diff --git a/external/llvm-project/.ci/compute_projects.py b/external/llvm-project/.ci/compute_projects.py
index 17a2136a270d..40dd0507a9ea 100644
--- a/external/llvm-project/.ci/compute_projects.py
+++ b/external/llvm-project/.ci/compute_projects.py
@@ -57,6 +57,13 @@
     ".ci": {"llvm", "clang", "lld", "lldb"},
 }
 
+# This mapping describes runtimes that should be enabled for a specific project,
+# but not necessarily run for testing. The only case of this currently is lldb
+# which needs some runtimes enabled for tests.
+DEPENDENT_RUNTIMES_TO_BUILD = {"lldb": {"libcxx", "libcxxabi", "libunwind"}}
+
+# This mapping describes runtimes that should be tested when the key project is
+# touched.
 DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
 
 EXCLUDE_LINUX = {
@@ -180,16 +187,20 @@ def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
 def _compute_runtimes_to_test(projects_to_test: Set[str]) -> Set[str]:
     runtimes_to_test = set()
     for project_to_test in projects_to_test:
-        if project_to_test not in DEPENDENT_RUNTIMES_TO_TEST:
-            continue
-        runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[project_to_test])
+        if project_to_test in DEPENDENT_RUNTIMES_TO_TEST:
+            runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[project_to_test])
+        if project_to_test in DEPENDENT_RUNTIMES_TO_BUILD:
+            runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_BUILD[project_to_test])
     return runtimes_to_test
 
 
-def _compute_runtime_check_targets(runtimes_to_test: Set[str]) -> Set[str]:
+def _compute_runtime_check_targets(projects_to_test: Set[str]) -> Set[str]:
     check_targets = set()
-    for runtime_to_test in runtimes_to_test:
-        check_targets.add(PROJECT_CHECK_TARGETS[runtime_to_test])
+    for project_to_test in projects_to_test:
+        if project_to_test not in DEPENDENT_RUNTIMES_TO_TEST:
+            continue
+        for runtime_to_test in DEPENDENT_RUNTIMES_TO_TEST[project_to_test]:
+            check_targets.add(PROJECT_CHECK_TARGETS[runtime_to_test])
     return check_targets
 
 
@@ -216,8 +227,8 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
     projects_to_test = _compute_projects_to_test(modified_projects, platform)
     projects_to_build = _compute_projects_to_build(projects_to_test)
     projects_check_targets = _compute_project_check_targets(projects_to_test)
-    runtimes_to_test = _compute_runtimes_to_test(projects_to_test)
-    runtimes_check_targets = _compute_runtime_check_targets(runtimes_to_test)
+    runtimes_to_build = _compute_runtimes_to_test(projects_to_test)
+    runtimes_check_targets = _compute_runtime_check_targets(projects_to_test)
     # We use a semicolon to separate the projects/runtimes as they get passed
     # to the CMake invocation and thus we need to use the CMake list separator
     # (;). We use spaces to separate the check targets as they end up getting
@@ -225,7 +236,7 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
     return {
         "projects_to_build": ";".join(sorted(projects_to_build)),
         "project_check_targets": " ".join(sorted(projects_check_targets)),
-        "runtimes_to_build": ";".join(sorted(runtimes_to_test)),
+        "runtimes_to_build": ";".join(sorted(runtimes_to_build)),
         "runtimes_check_targets": " ".join(sorted(runtimes_check_targets)),
     }
 
diff --git a/external/llvm-project/.ci/compute_projects_test.py b/external/llvm-project/.ci/compute_projects_test.py
index 885f1dd6af84..ae376ea6a43c 100644
--- a/external/llvm-project/.ci/compute_projects_test.py
+++ b/external/llvm-project/.ci/compute_projects_test.py
@@ -205,6 +205,17 @@ def test_ci(self):
             "check-cxx check-cxxabi check-unwind",
         )
 
+    def test_lldb(self):
+        env_variables = compute_projects.get_env_variables(
+            ["lldb/CMakeLists.txt"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "clang;lldb;llvm")
+        self.assertEqual(env_variables["project_check_targets"], "check-lldb")
+        self.assertEqual(
+            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
+        )
+        self.assertEqual(env_variables["runtimes_check_targets"], "")
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/external/llvm-project/.ci/metrics/metrics.py b/external/llvm-project/.ci/metrics/metrics.py
index 171bc20a9571..143e6ab4cf46 100644
--- a/external/llvm-project/.ci/metrics/metrics.py
+++ b/external/llvm-project/.ci/metrics/metrics.py
@@ -1,12 +1,9 @@
 import collections
 import datetime
-import dateutil
 import github
-import json
 import logging
 import os
 import requests
-import sys
 import time
 
 from dataclasses import dataclass
@@ -55,18 +52,6 @@
 # by trial and error).
 GRAFANA_METRIC_MAX_AGE_MN = 120
 
-# Lists the BuildKite jobs we want to track. Maps the BuildKite job name to
-# the metric name in Grafana. This is important not to lose metrics history
-# if the workflow name changes.
-BUILDKITE_WORKFLOW_TO_TRACK = {
-    ":linux: Linux x64": "buildkite_linux",
-    ":windows: Windows x64": "buildkite_windows",
-}
-
-# Number of builds to fetch per page. Since we scrape regularly, this can
-# remain small.
-BUILDKITE_GRAPHQL_BUILDS_PER_PAGE = 50
-
 
 @dataclass
 class JobMetrics:
@@ -86,181 +71,6 @@ class GaugeMetric:
     time_ns: int
 
 
-def buildkite_fetch_page_build_list(
-    buildkite_token: str, after_cursor: str = None
-) -> list[dict[str, str]]:
-    """Fetches a page of the build list using the GraphQL BuildKite API.
-
-    Returns the BUILDKITE_GRAPHQL_BUILDS_PER_PAGE last running/queued builds,
-    or the BUILDKITE_GRAPHQL_BUILDS_PER_PAGE running/queued builds
-    older than the one pointer by |after_cursor| if provided.
-    The |after_cursor| value is taken from the previous page returned by the
-    API.
-
-    Args:
-      buildkite_token: the secret token to authenticate GraphQL requests.
-      after_cursor: cursor after which to start the page fetch.
-
-    Returns:
-      The most recent builds after cursor (if set) with the following format:
-      [
-        {
-            "cursor": <value>,
-            "number": <build-number>,
-        }
-      ]
-    """
-
-    BUILDKITE_GRAPHQL_QUERY = """
-    query OrganizationShowQuery {{
-      organization(slug: "llvm-project") {{
-        pipelines(search: "Github pull requests", first: 1) {{
-          edges {{
-            node {{
-              builds (state: [CANCELING, CREATING, FAILING, RUNNING], first: {PAGE_SIZE}, after: {AFTER}) {{
-                edges {{
-                  cursor
-                  node {{
-                    number
-                  }}
-                }}
-              }}
-            }}
-          }}
-        }}
-      }}
-    }}
-    """
-    query = BUILDKITE_GRAPHQL_QUERY.format(
-        PAGE_SIZE=BUILDKITE_GRAPHQL_BUILDS_PER_PAGE,
-        AFTER="null" if after_cursor is None else '"{}"'.format(after_cursor),
-    )
-    query = json.dumps({"query": query})
-    url = "https://graphql.buildkite.com/v1"
-    headers = {
-        "Authorization": "Bearer " + buildkite_token,
-        "Content-Type": "application/json",
-    }
-    data = requests.post(url, data=query, headers=headers).json()
-    # De-nest the build list.
-    if "errors" in data:
-        logging.info("Failed to fetch BuildKite jobs: {}".format(data["errors"]))
-        return []
-    builds = data["data"]["organization"]["pipelines"]["edges"][0]["node"]["builds"][
-        "edges"
-    ]
-    # Fold cursor info into the node dictionnary.
-    return [{**x["node"], "cursor": x["cursor"]} for x in builds]
-
-
-def buildkite_get_build_info(build_number: str) -> dict:
-    """Returns all the info associated with the provided build number.
-
-    Note: for unknown reasons, graphql returns no jobs for a given build,
-    while this endpoint does, hence why this uses this API instead of graphql.
-
-    Args:
-      build_number: which build number to fetch info for.
-
-    Returns:
-      The info for the target build, a JSON dictionnary.
-    """
-
-    URL = "https://buildkite.com/llvm-project/github-pull-requests/builds/{}.json"
-    return requests.get(URL.format(build_number)).json()
-
-
-def buildkite_get_incomplete_tasks(buildkite_token: str) -> list:
-    """Returns all the running/pending BuildKite builds.
-
-    Args:
-     buildkite_token: the secret token to authenticate GraphQL requests.
-     last_cursor: the cursor to stop at if set. If None, a full page is fetched.
-    """
-    output = []
-    cursor = None
-    while True:
-        page = buildkite_fetch_page_build_list(buildkite_token, cursor)
-        if len(page) == 0:
-            break
-        cursor = page[-1]["cursor"]
-        output += page
-    return output
-
-
-def buildkite_get_metrics(
-    buildkite_token: str, previously_incomplete: set[int]
-) -> (list[JobMetrics], set[int]):
-    """Returns a tuple with:
-
-    - the metrics recorded for newly completed workflow jobs.
-    - the set of workflow still running now.
-
-    Args:
-      buildkite_token: the secret token to authenticate GraphQL requests.
-        previously_incomplete: the set of running workflows the last time this
-        function was called.
-    """
-
-    running_builds = buildkite_get_incomplete_tasks(buildkite_token)
-    incomplete_now = set([x["number"] for x in running_builds])
-    output = []
-
-    for build_id in previously_incomplete:
-        if build_id in incomplete_now:
-            continue
-
-        info = buildkite_get_build_info(build_id)
-        metric_timestamp = dateutil.parser.isoparse(info["finished_at"])
-        for job in info["jobs"]:
-            # This workflow is not interesting to us.
-            if job["name"] not in BUILDKITE_WORKFLOW_TO_TRACK:
-                continue
-
-            # Don't count canceled jobs.
-            if job["canceled_at"]:
-                continue
-
-            created_at = dateutil.parser.isoparse(job["created_at"])
-            scheduled_at = dateutil.parser.isoparse(job["scheduled_at"])
-            started_at = dateutil.parser.isoparse(job["started_at"])
-            finished_at = dateutil.parser.isoparse(job["finished_at"])
-
-            job_name = BUILDKITE_WORKFLOW_TO_TRACK[job["name"]]
-            queue_time = (started_at - scheduled_at).seconds
-            run_time = (finished_at - started_at).seconds
-            status = bool(job["passed"])
-
-            # Grafana will refuse to ingest metrics older than ~2 hours, so we
-            # should avoid sending historical data.
-            metric_age_mn = (
-                datetime.datetime.now(datetime.timezone.utc) - metric_timestamp
-            ).total_seconds() / 60
-            if metric_age_mn > GRAFANA_METRIC_MAX_AGE_MN:
-                logging.warning(
-                    f"Job {job['name']} from workflow {build_id} dropped due"
-                    + f" to staleness: {metric_age_mn}mn old."
-                )
-                continue
-
-            metric_timestamp_ns = int(metric_timestamp.timestamp()) * 10**9
-            workflow_id = build_id
-            workflow_name = "Github pull requests"
-            output.append(
-                JobMetrics(
-                    job_name,
-                    queue_time,
-                    run_time,
-                    status,
-                    metric_timestamp_ns,
-                    workflow_id,
-                    workflow_name,
-                )
-            )
-
-    return output, incomplete_now
-
-
 def github_get_metrics(
     github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
 ) -> tuple[list[JobMetrics], int]:
@@ -478,7 +288,6 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
 def main():
     # Authenticate with Github
     github_auth = Auth.Token(os.environ["GITHUB_TOKEN"])
-    buildkite_token = os.environ["BUILDKITE_TOKEN"]
     grafana_api_key = os.environ["GRAFANA_API_KEY"]
     grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
 
@@ -486,9 +295,6 @@ def main():
     # Because the Github queries are broken, we'll simply log a 'processed'
     # bit for the last COUNT_TO_PROCESS workflows.
     gh_last_workflows_seen_as_completed = set()
-    # Stores the list of pending/running builds in BuildKite we need to check
-    # at the next iteration.
-    bk_incomplete = set()
 
     # Enter the main loop. Every five minutes we wake up and dump metrics for
     # the relevant jobs.
@@ -500,13 +306,8 @@ def main():
             github_repo, gh_last_workflows_seen_as_completed
         )
 
-        bk_metrics, bk_incomplete = buildkite_get_metrics(
-            buildkite_token, bk_incomplete
-        )
-
-        metrics = gh_metrics + bk_metrics
-        upload_metrics(metrics, grafana_metrics_userid, grafana_api_key)
-        logging.info(f"Uploaded {len(metrics)} metrics")
+        upload_metrics(gh_metrics, grafana_metrics_userid, grafana_api_key)
+        logging.info(f"Uploaded {len(gh_metrics)} metrics")
 
         time.sleep(SCRAPE_INTERVAL_SECONDS)
 
diff --git a/external/llvm-project/.ci/monolithic-linux.sh b/external/llvm-project/.ci/monolithic-linux.sh
index a0ce8f3b80f1..7503ea4e6a99 100755
--- a/external/llvm-project/.ci/monolithic-linux.sh
+++ b/external/llvm-project/.ci/monolithic-linux.sh
@@ -47,7 +47,7 @@ function at-exit {
     python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":linux: Linux x64 Test Results" \
       "linux-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
   else
-    python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":linux: Linux x64 Test Results" \
+    python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":penguin: Linux x64 Test Results" \
       $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
   fi
 }
@@ -55,6 +55,8 @@ trap at-exit EXIT
 
 projects="${1}"
 targets="${2}"
+runtimes="${3}"
+runtime_targets="${4}"
 
 lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
@@ -70,7 +72,7 @@ export LLVM_SYMBOLIZER_PATH=`which llvm-symbolizer`
 # It will not be built unless it is used.
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
-      -D LLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
+      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
       -G Ninja \
       -D CMAKE_PREFIX_PATH="${HOME}/.local" \
       -D CMAKE_BUILD_TYPE=Release \
@@ -91,62 +93,28 @@ echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
 
-runtimes="${3}"
-runtime_targets="${4}"
-
 # Compiling runtimes with just-built Clang and running their tests
 # as an additional testing for Clang.
-if [[ "${runtimes}" != "" ]]; then
-  if [[ "${runtime_targets}" == "" ]]; then
-    echo "Runtimes to build are specified, but targets are not."
-    exit 1
-  fi
-
-  echo "--- ninja install-clang"
-
-  ninja -C ${BUILD_DIR} install-clang install-clang-resource-headers
-
-  RUNTIMES_BUILD_DIR="${MONOREPO_ROOT}/build-runtimes"
-  INSTALL_DIR="${BUILD_DIR}/install"
-  mkdir -p ${RUNTIMES_BUILD_DIR}
-
+if [[ "${runtimes_targets}" != "" ]]; then
   echo "--- cmake runtimes C++26"
 
-  rm -rf "${RUNTIMES_BUILD_DIR}"
-  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
-      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
-      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
-      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
-      -D LIBCXX_CXX_ABI=libcxxabi \
-      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-      -D LIBCXX_TEST_PARAMS="std=c++26" \
-      -D LIBCXXABI_TEST_PARAMS="std=c++26" \
-      -D LLVM_LIT_ARGS="${lit_args}"
+  cmake \
+    -D LIBCXX_TEST_PARAMS="std=c++26" \
+    -D LIBCXXABI_TEST_PARAMS="std=c++26" \
+    "${BUILD_DIR}"
 
   echo "--- ninja runtimes C++26"
 
-  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
+  ninja -C "${BUILD_DIR}" ${runtime_targets}
 
   echo "--- cmake runtimes clang modules"
 
-  # We don't need to do a clean build of runtimes, because LIBCXX_TEST_PARAMS
-  # and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
-  # propagates without a clean build. Other that those two variables, builds
-  # are supposed to be the same.
-
-  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
-      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
-      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
-      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
-      -D LIBCXX_CXX_ABI=libcxxabi \
-      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-      -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
-      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
-      -D LLVM_LIT_ARGS="${lit_args}"
+  cmake \
+    -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
+    -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
+    "${BUILD_DIR}"
 
   echo "--- ninja runtimes clang modules"
 
-  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
+  ninja -C "${BUILD_DIR}" ${runtime_targets}
 fi
diff --git a/external/llvm-project/.ci/monolithic-windows.sh b/external/llvm-project/.ci/monolithic-windows.sh
index 8dc49aa15819..a0997420b0d3 100755
--- a/external/llvm-project/.ci/monolithic-windows.sh
+++ b/external/llvm-project/.ci/monolithic-windows.sh
@@ -42,7 +42,7 @@ function at-exit {
     python "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":windows: Windows x64 Test Results" \
       "windows-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
   else
-    python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":windows: Windows x64 Test Results" \
+    python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":window: Windows x64 Test Results" \
       $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
   fi
 }
diff --git a/external/llvm-project/.git-blame-ignore-revs b/external/llvm-project/.git-blame-ignore-revs
index 30d9f6b883ce..01569c619f8f 100644
--- a/external/llvm-project/.git-blame-ignore-revs
+++ b/external/llvm-project/.git-blame-ignore-revs
@@ -100,3 +100,9 @@ d33bf2e9df578ff7e44fd22504d6ad5a122b7ee6
 
 # [lldb][NFC] clang-format MainLoopPosix.cpp
 66bdbfbaa08fa3d8e64a7fe136a8fb717f5cdbb7
+
+# [clang-tidy][NFC] Run clang-format on "clang-tools-extra/clang-tidy"
+65d66625b3e2b8322ed99d82edabecbafcd0885b
+ce46adb8b7ce645353eccaedf31ed9765dab77bb
+68070f908bb7ac5f0b5fa9722caa504ecf723f6b
+5213c57cb1f0d78aad9a253b7f6a2b62ff4c7859
diff --git a/external/llvm-project/.github/new-prs-labeler.yml b/external/llvm-project/.github/new-prs-labeler.yml
index f6a7a6072478..162161ff13fb 100644
--- a/external/llvm-project/.github/new-prs-labeler.yml
+++ b/external/llvm-project/.github/new-prs-labeler.yml
@@ -554,6 +554,12 @@ flang:fir-hlfir:
 flang:codegen:
   - flang/**/CodeGen/**
 
+llvm:codegen:
+  - llvm/lib/CodeGen/*
+  - llvm/lib/CodeGen/MIRParser/*
+  - llvm/lib/CodeGen/LiveDebugValues/*
+  - llvm/lib/CodeGen/AsmPrinter/*
+
 llvm:globalisel:
   - llvm/**/GlobalISel/**
   - llvm/utils/TableGen/GlobalISel*
diff --git a/external/llvm-project/.github/workflows/buildbot-psdb-trigger.yml b/external/llvm-project/.github/workflows/buildbot-psdb-trigger.yml
index 10ba788abf4d..471fd4001ae8 100644
--- a/external/llvm-project/.github/workflows/buildbot-psdb-trigger.yml
+++ b/external/llvm-project/.github/workflows/buildbot-psdb-trigger.yml
@@ -9,7 +9,8 @@ on:
 jobs:  
   trigger-build:
     if: github.event.pull_request.draft == false
-    runs-on: self-hosted
+    runs-on: 
+      group: compiler-generic-runners
     env:  
       PR_SHA: ${{ github.event.pull_request.head.sha != '' && github.event.pull_request.head.sha || github.sha }}
       PR_NUMBER: ${{ github.event.pull_request.number != '' && github.event.pull_request.number || 0 }}
diff --git a/external/llvm-project/.github/workflows/containers/github-action-ci-windows/Dockerfile b/external/llvm-project/.github/workflows/containers/github-action-ci-windows/Dockerfile
index 09cda4bb574b..eafe45fac8ea 100644
--- a/external/llvm-project/.github/workflows/containers/github-action-ci-windows/Dockerfile
+++ b/external/llvm-project/.github/workflows/containers/github-action-ci-windows/Dockerfile
@@ -85,7 +85,7 @@ RUN powershell -Command \
 RUN git config --system core.longpaths true & \
     git config --global core.autocrlf false
 
-ARG RUNNER_VERSION=2.324.0
+ARG RUNNER_VERSION=2.325.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \
diff --git a/external/llvm-project/.github/workflows/containers/github-action-ci/Dockerfile b/external/llvm-project/.github/workflows/containers/github-action-ci/Dockerfile
index 52c20e049ee7..f2716f0a9c6a 100644
--- a/external/llvm-project/.github/workflows/containers/github-action-ci/Dockerfile
+++ b/external/llvm-project/.github/workflows/containers/github-action-ci/Dockerfile
@@ -86,7 +86,7 @@ WORKDIR /home/gha
 
 FROM ci-container as ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.324.0
+ENV GITHUB_RUNNER_VERSION=2.325.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \
diff --git a/external/llvm-project/.github/workflows/docs.yml b/external/llvm-project/.github/workflows/docs.yml
index 0692633b71a5..3970271e4adb 100644
--- a/external/llvm-project/.github/workflows/docs.yml
+++ b/external/llvm-project/.github/workflows/docs.yml
@@ -205,7 +205,7 @@ jobs:
           steps.docs-changed-subprojects.outputs.workflow_any_changed == 'true'
         run: |
           cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON ./llvm
-          TZ=UTC ninja -C flang-build docs-flang-html
+          TZ=UTC ninja -C flang-build docs-flang-html docs-flang-man
           mkdir built-docs/flang
           cp -r flang-build/docs/* built-docs/flang/
       - name: Upload docs
diff --git a/external/llvm-project/.github/workflows/email-check.yaml b/external/llvm-project/.github/workflows/email-check.yaml
index f4481d5cf558..904ad718f97d 100644
--- a/external/llvm-project/.github/workflows/email-check.yaml
+++ b/external/llvm-project/.github/workflows/email-check.yaml
@@ -32,7 +32,8 @@ jobs:
           COMMENT: >-
             ⚠️ We detected that you are using a GitHub private e-mail address to contribute to the repo.<br/>
             Please turn off [Keep my email addresses private](https://github.com/settings/emails) setting in your account.<br/>
-            See [LLVM Discourse](https://discourse.llvm.org/t/hidden-emails-on-github-should-we-do-something-about-it) for more information.
+            See [LLVM Developer Policy](https://llvm.org/docs/DeveloperPolicy.html#email-addresses) and
+            [LLVM Discourse](https://discourse.llvm.org/t/hidden-emails-on-github-should-we-do-something-about-it) for more information.
         run: |
           cat << EOF > comments
           [{"body" : "$COMMENT"}]
diff --git a/external/llvm-project/.github/workflows/libcxx-build-and-test.yaml b/external/llvm-project/.github/workflows/libcxx-build-and-test.yaml
index 80f2432b78de..f0bdf6c0b589 100644
--- a/external/llvm-project/.github/workflows/libcxx-build-and-test.yaml
+++ b/external/llvm-project/.github/workflows/libcxx-build-and-test.yaml
@@ -52,8 +52,8 @@ jobs:
         cxx: [ 'clang++-21' ]
         include:
           - config: 'generic-gcc'
-            cc: 'gcc-14'
-            cxx: 'g++-14'
+            cc: 'gcc-15'
+            cxx: 'g++-15'
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: ${{ matrix.config }}.${{ matrix.cxx }}
@@ -92,8 +92,8 @@ jobs:
         cxx: [ 'clang++-21' ]
         include:
           - config: 'generic-gcc-cxx11'
-            cc: 'gcc-14'
-            cxx: 'g++-14'
+            cc: 'gcc-15'
+            cxx: 'g++-15'
           - config: 'generic-cxx26'
             cc: 'clang-20'
             cxx: 'clang++-20'
diff --git a/external/llvm-project/amd/comgr/CMakeLists.txt b/external/llvm-project/amd/comgr/CMakeLists.txt
index 922e040ea94d..c4fc39edba25 100644
--- a/external/llvm-project/amd/comgr/CMakeLists.txt
+++ b/external/llvm-project/amd/comgr/CMakeLists.txt
@@ -179,12 +179,24 @@ message("")
 option(COMGR_DISABLE_SPIRV "To disable SPIRV in Comgr" OFF)
 
 if (NOT COMGR_DISABLE_SPIRV)
-  # TODO: Explore switching this to CHECK_INCLUDE_FILE_CXX() macro
-  if (NOT EXISTS "${LLVM_INCLUDE_DIRS}/LLVMSPIRVLib/LLVMSPIRVLib.h")
+  # Candidate include paths for LLVMSPIRVLib.h:
+  # 1. ${LLVM_INCLUDE_DIRS}/LLVMSPIRVLib (standalone build)
+  # 2. ${LLVM_EXTERNAL_SPIRV_LLVM_TRANSLATOR_SOURCE_DIR}/include (external project)
+  # 3. ${CMAKE_SOURCE_DIR}/projects/SPIRV-LLVM-Translator/include (usual location)
+  find_path(
+    FOUND_SPIRV_INCLUDE_DIR
+    LLVMSPIRVLib.h
+    PATHS
+      "${LLVM_INCLUDE_DIRS}/LLVMSPIRVLib"
+      "${LLVM_EXTERNAL_SPIRV_LLVM_TRANSLATOR_SOURCE_DIR}/include"
+      "${CMAKE_SOURCE_DIR}/projects/SPIRV-LLVM-Translator/include"
+    NO_DEFAULT_PATH
+  )
+  if (NOT EXISTS "${FOUND_SPIRV_INCLUDE_DIR}/LLVMSPIRVLib.h")
     message("-- LLVMSPIRVLib/LLVMSPIRVLib.h not found")
     set(COMGR_DISABLE_SPIRV ON)
   else()
-    message("-- LLVMSPIRVLib/LLVMSPIRVLib.h found")
+    message("-- LLVMSPIRVLib/LLVMSPIRVLib.h found at ${FOUND_SPIRV_INCLUDE_DIR}")
   endif()
 endif()
 
@@ -195,7 +207,7 @@ else()
   message("-- Comgr SPIRV Enabled")
   target_include_directories(amd_comgr
       PRIVATE
-        "${LLVM_INCLUDE_DIRS}/LLVMSPIRVLib")
+        "${FOUND_SPIRV_INCLUDE_DIR}")
 endif()
 
 
diff --git a/external/llvm-project/amd/comgr/README.md b/external/llvm-project/amd/comgr/README.md
index 21710c8e5514..1e3056378698 100644
--- a/external/llvm-project/amd/comgr/README.md
+++ b/external/llvm-project/amd/comgr/README.md
@@ -198,13 +198,11 @@ improving performance by reducing on-disk file I/O. Currently, VFS is only suppo
 for the device library link step, but we aim to progressively add support for
 more actions.
 
-By default, VFS is turned off. Set the environment variable `AMD_COMGR_USE_VFS=1` to 
-enable it.
+By default, VFS is turned on.
 
-* `AMD_COMGR_USE_VFS`: When unset or "0", VFS support is turned off.
-* Users may use the API `amd_comgr_action_info_set_vfs` to enable VFS for individual actions
-  without having to modify system-wide environment variables. Note that API is effective only 
-  if `AMD_COMGR_USE_VFS` is unset.
+* `AMD_COMGR_USE_VFS`: When set to "0", VFS support is turned off.
+* Users may use the API `amd_comgr_action_info_set_vfs` to disable VFS for individual actions
+  without having to modify system-wide environment variables.
 * If `AMD_COMGR_SAVE_TEMPS` is set and not "0", VFS support is turned off irrespective
   of `AMD_COMGR_USE_VFS` or the use of `amd_comgr_action_info_set_vfs`.
 
diff --git a/external/llvm-project/amd/comgr/cmake/DeviceLibs.cmake b/external/llvm-project/amd/comgr/cmake/DeviceLibs.cmake
index d8f5654dfee5..3c5654d93dae 100644
--- a/external/llvm-project/amd/comgr/cmake/DeviceLibs.cmake
+++ b/external/llvm-project/amd/comgr/cmake/DeviceLibs.cmake
@@ -59,7 +59,8 @@ foreach(AMDGCN_LIB_TARGET ${AMD_DEVICE_LIBS_TARGETS})
   add_dependencies(amd_comgr ${AMDGCN_LIB_TARGET}_header)
 
   list(APPEND TARGETS_INCLUDES "#include \"${header}\"")
-  list(APPEND TARGETS_HEADERS "${INC_DIR}/${header}")
+  list(APPEND TARGETS_HEADERS_FILENAME "${header}")
+  list(APPEND TARGETS_HEADERS_REALPATH "${INC_DIR}/${header}")
 endforeach()
 
 list(JOIN TARGETS_INCLUDES "\n" TARGETS_INCLUDES)
@@ -113,8 +114,8 @@ find_package(Python3 REQUIRED Interpreter)
 set(DEVICE_LIBS_ID_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/device-libs-id.py")
 set(DEVICE_LIBS_ID_HEADER ${INC_DIR}/libraries_sha.inc)
 add_custom_command(OUTPUT ${DEVICE_LIBS_ID_HEADER}
-  COMMAND ${Python3_EXECUTABLE} ${DEVICE_LIBS_ID_SCRIPT} --varname DEVICE_LIBS_ID --output ${DEVICE_LIBS_ID_HEADER} ${TARGETS_HEADERS}
-  DEPENDS ${DEVICE_LIBS_ID_SCRIPT} ${TARGETS_HEADERS}
+    COMMAND ${Python3_EXECUTABLE} ${DEVICE_LIBS_ID_SCRIPT} --varname DEVICE_LIBS_ID --output ${DEVICE_LIBS_ID_HEADER} --parent-directory ${INC_DIR} ${TARGETS_HEADERS_FILENAME}
+    DEPENDS ${DEVICE_LIBS_ID_SCRIPT} ${TARGETS_HEADERS_REALPATH}
     COMMENT "Generating ${INC_DIR}/libraries_sha.inc"
 )
 set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${INC_DIR}/libraries_sha.inc)
diff --git a/external/llvm-project/amd/comgr/cmake/device-libs-id.py b/external/llvm-project/amd/comgr/cmake/device-libs-id.py
index 06ac7a4522f4..09362fb207ca 100644
--- a/external/llvm-project/amd/comgr/cmake/device-libs-id.py
+++ b/external/llvm-project/amd/comgr/cmake/device-libs-id.py
@@ -4,11 +4,14 @@
 
 from argparse import ArgumentParser
 from hashlib import sha256
-from functools import reduce
+from os.path import join as join_path
 
 if __name__ == "__main__":
     parser = ArgumentParser(description='Generate id by computing a hash of the generated headers')
     parser.add_argument("headers", nargs='+', help='List of headers to generate id from')
+    # On Windows, we cannot list the realpath for every individual header since we hit cmd.exe's
+    # maximum command line lenght. As a workaround, we pass the pwd and the headers separately.
+    parser.add_argument("--parent-directory", help='Parent directory for the headers', required=True)
     parser.add_argument("--varname", help='Name of the variable to generate', required=True)
     parser.add_argument("--output", help='Name of the header to generate', required=True)
 
@@ -16,8 +19,8 @@
     args.headers.sort()
     
     hash = sha256()
-    for x in args.headers:
-        hash.update(open(x, 'rb').read())
+    for header in args.headers:
+        hash.update(open(join_path(args.parent_directory, header), 'rb').read())
     digest_uchar = hash.digest()
     digest_elts = ", ".join(map(str, digest_uchar))
     print(f"static const unsigned char {args.varname}[] = {{{digest_elts}, 0}};", file=open(args.output, 'w'))
diff --git a/external/llvm-project/amd/comgr/docs/ReleaseNotes.md b/external/llvm-project/amd/comgr/docs/ReleaseNotes.md
index d3dcb3215340..abca63906622 100644
--- a/external/llvm-project/amd/comgr/docs/ReleaseNotes.md
+++ b/external/llvm-project/amd/comgr/docs/ReleaseNotes.md
@@ -38,8 +38,9 @@ Bug Fixes
 New APIs
 --------
 - amd\_comgr\_info\_set\_vfs\_() (v3.1)
-    - By setting this ActionInfo property, a user can explicitly dictate if
-    device libraries should be linked using a Virtual File System (VFS).
+    - By setting this ActionInfo property, users can explicitly dictate if
+    device libraries should be linked using the real file system or a
+    Virtual File System (VFS).
 
 Deprecated APIs
 ---------------
diff --git a/external/llvm-project/amd/comgr/src/comgr.h b/external/llvm-project/amd/comgr/src/comgr.h
index 1511e8632fe7..e419a46baf6d 100644
--- a/external/llvm-project/amd/comgr/src/comgr.h
+++ b/external/llvm-project/amd/comgr/src/comgr.h
@@ -204,7 +204,7 @@ struct DataAction {
   amd_comgr_language_t Language;
   bool Logging;
   bool ShouldLinkDeviceLibs = false;
-  bool ShouldUseVFS = false;
+  bool ShouldUseVFS = true;
 
   std::vector<std::string> BundleEntryIDs;
 
diff --git a/external/llvm-project/amd/comgr/test-lit/spirv-to-reloc-debuginfo.hip b/external/llvm-project/amd/comgr/test-lit/spirv-to-reloc-debuginfo.hip
index 7f5db667adcd..4fa7cc4cbe0c 100644
--- a/external/llvm-project/amd/comgr/test-lit/spirv-to-reloc-debuginfo.hip
+++ b/external/llvm-project/amd/comgr/test-lit/spirv-to-reloc-debuginfo.hip
@@ -1,4 +1,4 @@
-// REQUIRES: comgr-has-spirv,fix-rhel-issue
+// REQUIRES: comgr-has-spirv
 
 // COM: Generate a debuginfo SPIR-V file from a HIP kernel
 // RUN: clang -x hip --offload-arch=amdgcnspirv -nogpulib -nogpuinc \
diff --git a/external/llvm-project/amd/comgr/test-lit/vfs-tests/vfs-tests.cl b/external/llvm-project/amd/comgr/test-lit/vfs-tests/vfs-tests.cl
index 6c6932ab2a0f..bc7c66af8521 100644
--- a/external/llvm-project/amd/comgr/test-lit/vfs-tests/vfs-tests.cl
+++ b/external/llvm-project/amd/comgr/test-lit/vfs-tests/vfs-tests.cl
@@ -23,7 +23,7 @@
 // COM: Verify success of compilation for all scenarios
 // STATUS: ReturnStatus: AMD_COMGR_STATUS_SUCCESS
 
-// OUT-NA-NA-NA: File System: Real
+// OUT-NA-NA-NA: File System: VFS
 // OUT-NA-VFS-NOVFS: File System: VFS
 // OUT-NA-VFS-NA: File System: VFS
 // OUT-NA-NOVFS-VFS: File System: Real
diff --git a/external/llvm-project/amd/comgr/test/source/multiple-note-records-one-kernel.s b/external/llvm-project/amd/comgr/test/source/multiple-note-records-one-kernel.s
index 026cd939cc9a..48ba0e5a1683 100644
--- a/external/llvm-project/amd/comgr/test/source/multiple-note-records-one-kernel.s
+++ b/external/llvm-project/amd/comgr/test/source/multiple-note-records-one-kernel.s
@@ -120,7 +120,7 @@ _ZN17__HIP_CoordinatesI14__HIP_BlockDimE1xE:
 	.zero	1
 	.size	_ZN17__HIP_CoordinatesI14__HIP_BlockDimE1xE, 1
 
-	.ident	"AMD clang version 17.0.0 (https://github.com/RadeonOpenCompute/llvm-project roc-6.0.0 23483 7208e8d15fbf218deb74483ea8c549c67ca4985e)"
+	.ident	"AMD clang version 17.0.0 (https://github.com/ROCm/llvm-project roc-6.0.0 23483 7208e8d15fbf218deb74483ea8c549c67ca4985e)"
 	.section	".note.GNU-stack"
 	.addrsig
 	.amdgpu_metadata
diff --git a/external/llvm-project/amd/comgr/test/source/multiple-note-records.s b/external/llvm-project/amd/comgr/test/source/multiple-note-records.s
index 553deff220ab..ee268bd9b39f 100644
--- a/external/llvm-project/amd/comgr/test/source/multiple-note-records.s
+++ b/external/llvm-project/amd/comgr/test/source/multiple-note-records.s
@@ -216,7 +216,7 @@ _ZN17__HIP_CoordinatesI14__HIP_BlockDimE1xE:
 	.zero	1
 	.size	_ZN17__HIP_CoordinatesI14__HIP_BlockDimE1xE, 1
 
-	.ident	"AMD clang version 17.0.0 (https://github.com/RadeonOpenCompute/llvm-project roc-6.0.0 23483 7208e8d15fbf218deb74483ea8c549c67ca4985e)"
+	.ident	"AMD clang version 17.0.0 (https://github.com/ROCm/llvm-project roc-6.0.0 23483 7208e8d15fbf218deb74483ea8c549c67ca4985e)"
 	.section	".note.GNU-stack"
 	.addrsig
 	.amdgpu_metadata
diff --git a/external/llvm-project/amd/device-libs/CMakeLists.txt b/external/llvm-project/amd/device-libs/CMakeLists.txt
index b6cb5f28c425..a0d82249bfee 100644
--- a/external/llvm-project/amd/device-libs/CMakeLists.txt
+++ b/external/llvm-project/amd/device-libs/CMakeLists.txt
@@ -106,7 +106,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "Radeon Open Compute - device libraries" )
   set ( CPACK_PACKAGE_DESCRIPTION "This package includes LLVM bitcode libraries." )
   set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.TXT" )
-  set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCm-Device-Libs" )
+  set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/ROCm/llvm-project/tree/amd-staging/amd/device-libs" )
 
   # Install License file
   install ( FILES "${CPACK_RESOURCE_FILE_LICENSE}"
diff --git a/external/llvm-project/amd/device-libs/opencl/src/devenq/schedule_rocm.cl b/external/llvm-project/amd/device-libs/opencl/src/devenq/schedule_rocm.cl
index 731da8853e72..209eebeebc02 100644
--- a/external/llvm-project/amd/device-libs/opencl/src/devenq/schedule_rocm.cl
+++ b/external/llvm-project/amd/device-libs/opencl/src/devenq/schedule_rocm.cl
@@ -61,16 +61,23 @@ min_command(uint slot_num, __global AmdAqlWrap* wraps)
     return minCommand;
 }
 
+static inline bool
+check_pcie_support(__global SchedulerParam* param) {
+  #define kInvalidWriteIndex (ulong)(-1)
+  return (param->write_index == kInvalidWriteIndex) ? true : false;
+}
+
 static inline void
 EnqueueDispatch(__global hsa_kernel_dispatch_packet_t* aqlPkt, __global SchedulerParam* param)
 {
     __global hsa_queue_t* child_queue = param->child_queue;
 
-
-    // ulong index = __ockl_hsa_queue_add_write_index(child_queue, 1, __ockl_memory_order_relaxed);
-    // The original code seen above relies on PCIe 3 atomics, which might not be supported on some systems, so use a device side global
-    // for workaround.
-    ulong index = atomic_fetch_add_explicit((__global atomic_ulong*)&param->write_index, (ulong)1, memory_order_relaxed, memory_scope_device);
+    ulong index;
+    if (check_pcie_support(param)) {
+      index = __ockl_hsa_queue_add_write_index(child_queue, 1, __ockl_memory_order_relaxed);
+    } else {
+      index = atomic_fetch_add_explicit((__global atomic_ulong*)&param->write_index, (ulong)1, memory_order_relaxed, memory_scope_device);
+    }
 
     const ulong queueMask = child_queue->size - 1;
     __global hsa_kernel_dispatch_packet_t* dispatch_packet = &(((__global hsa_kernel_dispatch_packet_t*)(child_queue->base_address))[index & queueMask]);
@@ -82,17 +89,20 @@ EnqueueScheduler(__global SchedulerParam* param)
 {
     __global hsa_queue_t* child_queue = param->child_queue;
 
-    // ulong index = __ockl_hsa_queue_add_write_index(child_queue, 1, __ockl_memory_order_relaxed);
-    // The original code seen above relies on PCIe 3 atomics, which might not be supported on some systems, so use a device side global
-    // for workaround.
-    ulong index = atomic_fetch_add_explicit((__global atomic_ulong*)&param->write_index, (ulong)1, memory_order_relaxed, memory_scope_device);
+    ulong index;
+    if (check_pcie_support(param)) {
+      index = __ockl_hsa_queue_add_write_index(child_queue, 1, __ockl_memory_order_relaxed);
+    } else {
+      index = atomic_fetch_add_explicit((__global atomic_ulong*)&param->write_index, (ulong)1, memory_order_relaxed, memory_scope_device);
+    }
 
     const ulong queueMask = child_queue->size - 1;
     __global hsa_kernel_dispatch_packet_t* dispatch_packet = &(((__global hsa_kernel_dispatch_packet_t*)(child_queue->base_address))[index & queueMask]);
     *dispatch_packet = param->scheduler_aql;
 
-     // This is part of the PCIe 3 atomics workaround, to write the final write_index value back to the child_queue
-    __ockl_hsa_queue_store_write_index(child_queue, index + 1, __ockl_memory_order_relaxed);
+    if (!check_pcie_support(param)) {
+      __ockl_hsa_queue_store_write_index(child_queue, index + 1, __ockl_memory_order_relaxed);
+    }
 
     __ockl_hsa_signal_store(child_queue->doorbell_signal, index, __ockl_memory_order_release);
 }
diff --git a/external/llvm-project/amd/hipcc/CMakeLists.txt b/external/llvm-project/amd/hipcc/CMakeLists.txt
index 53f472c66e45..7512aa93baaa 100755
--- a/external/llvm-project/amd/hipcc/CMakeLists.txt
+++ b/external/llvm-project/amd/hipcc/CMakeLists.txt
@@ -70,7 +70,7 @@ set(CPACK_DEB_COMPONENT_INSTALL ON)
 set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT")
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core")
 set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "perl (>= 5.0)")
-set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/ROCm-Developer-Tools/HIPCC")
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/ROCm/llvm-project/tree/amd-staging/amd/hipcc")
 if(DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE})
   set(CPACK_DEBIAN_PACKAGE_RELEASE $ENV{CPACK_DEBIAN_PACKAGE_RELEASE})
 else()
diff --git a/external/llvm-project/amd/utils/translate-legacy-branch-to-omnibus-monorepo.sh b/external/llvm-project/amd/utils/translate-legacy-branch-to-omnibus-monorepo.sh
index d7c6ef65de24..49551cd7089c 100755
--- a/external/llvm-project/amd/utils/translate-legacy-branch-to-omnibus-monorepo.sh
+++ b/external/llvm-project/amd/utils/translate-legacy-branch-to-omnibus-monorepo.sh
@@ -13,15 +13,15 @@
 #|
 #|   Value        Repository
 #|   -----------  ---------------------------------------------------------
-#|   comgr        https://github.com/RadeonOpenCompute/ROCm-CompilerSupport
-#|   device-libs  https://github.com/RadeonOpenCompute/ROCm-Device-Libs
-#|   hipcc        https://github.com/ROCm-Developer-Tools/HIPCC
+#|   comgr        https://github.com/ROCm/llvm-project/tree/amd-staging/amd/comgr
+#|   device-libs  https://github.com/ROCm/llvm-project/tree/amd-staging/amd/device-libs
+#|   hipcc        https://github.com/ROCm/llvm-project/tree/amd-staging/amd/hipcc
 #|
 #| BASE-BRANCH identifies the branch upon which the to-be-translated branch is
 #| based (e.g. amd-stg-open).
 #|
 #| The TARGET-REPO-DIR must locate a fresh clone of the ROCm llvm-project
-#| repository (https://github.com/RadeonOpenCompute/llvm-project). If the
+#| repository (https://github.com/ROCm/llvm-project). If the
 #| script is to be invoked repeatedly the same clone may be reused. It is
 #| NOT recommended to use any clone which contains local refs that have not
 #| been saved elsewhere, as they may be clobbered!
diff --git a/external/llvm-project/bolt/include/bolt/Core/BinaryBasicBlock.h b/external/llvm-project/bolt/include/bolt/Core/BinaryBasicBlock.h
index 25cccc4edecf..629f0ce8314d 100644
--- a/external/llvm-project/bolt/include/bolt/Core/BinaryBasicBlock.h
+++ b/external/llvm-project/bolt/include/bolt/Core/BinaryBasicBlock.h
@@ -52,9 +52,8 @@ class BinaryBasicBlock {
     uint64_t MispredictedCount; /// number of branches mispredicted
 
     bool operator<(const BinaryBranchInfo &Other) const {
-      return (Count < Other.Count) ||
-             (Count == Other.Count &&
-              MispredictedCount < Other.MispredictedCount);
+      return std::tie(Count, MispredictedCount) <
+             std::tie(Other.Count, Other.MispredictedCount);
     }
   };
 
diff --git a/external/llvm-project/bolt/include/bolt/Core/BinaryFunction.h b/external/llvm-project/bolt/include/bolt/Core/BinaryFunction.h
index 6b22ff62c555..ca8b786f4ab6 100644
--- a/external/llvm-project/bolt/include/bolt/Core/BinaryFunction.h
+++ b/external/llvm-project/bolt/include/bolt/Core/BinaryFunction.h
@@ -388,6 +388,10 @@ class BinaryFunction {
   /// The profile data for the number of times the function was executed.
   uint64_t ExecutionCount{COUNT_NO_PROFILE};
 
+  /// Profile data for the number of times this function was entered from
+  /// external code (DSO, JIT, etc).
+  uint64_t ExternEntryCount{0};
+
   /// Profile match ratio.
   float ProfileMatchRatio{0.0f};
 
@@ -1877,6 +1881,10 @@ class BinaryFunction {
     return *this;
   }
 
+  /// Set the profile data for the number of times the function was entered from
+  /// external code (DSO/JIT).
+  void setExternEntryCount(uint64_t Count) { ExternEntryCount = Count; }
+
   /// Adjust execution count for the function by a given \p Count. The value
   /// \p Count will be subtracted from the current function count.
   ///
@@ -1904,6 +1912,10 @@ class BinaryFunction {
   /// Return COUNT_NO_PROFILE if there's no profile info.
   uint64_t getExecutionCount() const { return ExecutionCount; }
 
+  /// Return the profile information about the number of times the function was
+  /// entered from external code (DSO/JIT).
+  uint64_t getExternEntryCount() const { return ExternEntryCount; }
+
   /// Return the raw profile information about the number of branch
   /// executions corresponding to this function.
   uint64_t getRawSampleCount() const { return RawSampleCount; }
@@ -2348,6 +2360,7 @@ class BinaryFunction {
       releaseCFG();
       CurrentState = State::Emitted;
     }
+    clearList(Relocations);
   }
 
   /// Process LSDA information for the function.
diff --git a/external/llvm-project/bolt/include/bolt/Core/MCPlusBuilder.h b/external/llvm-project/bolt/include/bolt/Core/MCPlusBuilder.h
index b23345298550..804100db8079 100644
--- a/external/llvm-project/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/external/llvm-project/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -405,7 +405,7 @@ class MCPlusBuilder {
 
   bool equals(const MCExpr &A, const MCExpr &B, CompFuncTy Comp) const;
 
-  virtual bool equals(const MCTargetExpr &A, const MCTargetExpr &B,
+  virtual bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
                       CompFuncTy Comp) const;
 
   virtual bool isBranch(const MCInst &Inst) const {
diff --git a/external/llvm-project/bolt/include/bolt/Passes/PAuthGadgetScanner.h b/external/llvm-project/bolt/include/bolt/Passes/PAuthGadgetScanner.h
index 98a49df862eb..c6b9cc2eb4b9 100644
--- a/external/llvm-project/bolt/include/bolt/Passes/PAuthGadgetScanner.h
+++ b/external/llvm-project/bolt/include/bolt/Passes/PAuthGadgetScanner.h
@@ -77,9 +77,7 @@ struct MCInstInBFReference {
     return BF == RHS.BF && Offset == RHS.Offset;
   }
   bool operator<(const MCInstInBFReference &RHS) const {
-    if (BF != RHS.BF)
-      return BF < RHS.BF;
-    return Offset < RHS.Offset;
+    return std::tie(BF, Offset) < std::tie(RHS.BF, RHS.Offset);
   }
   operator MCInst &() const {
     assert(BF != nullptr);
diff --git a/external/llvm-project/bolt/include/bolt/Profile/DataAggregator.h b/external/llvm-project/bolt/include/bolt/Profile/DataAggregator.h
index 6d918134137d..3f07a6dc03a4 100644
--- a/external/llvm-project/bolt/include/bolt/Profile/DataAggregator.h
+++ b/external/llvm-project/bolt/include/bolt/Profile/DataAggregator.h
@@ -78,6 +78,13 @@ class DataAggregator : public DataReader {
   static bool checkPerfDataMagic(StringRef FileName);
 
 private:
+  struct LBREntry {
+    uint64_t From;
+    uint64_t To;
+    bool Mispred;
+  };
+  friend raw_ostream &operator<<(raw_ostream &OS, const LBREntry &);
+
   struct PerfBranchSample {
     SmallVector<LBREntry, 32> LBR;
   };
@@ -370,33 +377,46 @@ class DataAggregator : public DataReader {
   /// memory.
   ///
   /// File format syntax:
-  /// {B|F|f|T} [<start_id>:]<start_offset> [<end_id>:]<end_offset> [<ft_end>]
-  ///       <count> [<mispred_count>]
+  /// E <event>
+  /// S <start> <count>
+  /// T <start> <end> <ft_end> <count>
+  /// B <start> <end> <count> <mispred_count>
+  /// [Ff] <start> <end> <count>
+  ///
+  /// where <start>, <end>, <ft_end> have the format [<id>:]<offset>
   ///
-  /// B - indicates an aggregated branch
-  /// F - an aggregated fall-through
+  /// E - name of the sampling event used for subsequent entries
+  /// S - indicates an aggregated basic sample at <start>
+  /// B - indicates an aggregated branch from <start> to <end>
+  /// F - an aggregated fall-through from <start> to <end>
   /// f - an aggregated fall-through with external origin - used to disambiguate
   ///       between a return hitting a basic block head and a regular internal
   ///       jump to the block
-  /// T - an aggregated trace: branch with a fall-through (from, to, ft_end)
+  /// T - an aggregated trace: branch from <start> to <end> with a fall-through
+  ///       to <ft_end>
   ///
-  /// <start_id> - build id of the object containing the start address. We can
-  /// skip it for the main binary and use "X" for an unknown object. This will
-  /// save some space and facilitate human parsing.
+  /// <id> - build id of the object containing the address. We can skip it for
+  /// the main binary and use "X" for an unknown object. This will save some
+  /// space and facilitate human parsing.
   ///
-  /// <start_offset> - hex offset from the object base load address (0 for the
-  /// main executable unless it's PIE) to the start address.
+  /// <offset> - hex offset from the object base load address (0 for the
+  /// main executable unless it's PIE) to the address.
   ///
-  /// <end_id>, <end_offset> - same for the end address.
-  ///
-  /// <ft_end> - same for the fallthrough_end address.
-  ///
-  /// <count> - total aggregated count of the branch or a fall-through.
+  /// <count> - total aggregated count.
   ///
   /// <mispred_count> - the number of times the branch was mispredicted.
-  /// Omitted for fall-throughs.
   ///
   /// Example:
+  /// Basic samples profile:
+  /// E cycles
+  /// S 41be50 3
+  /// E br_inst_retired.near_taken
+  /// S 41be60 6
+  ///
+  /// Trace profile combining branches and fall-throughs:
+  /// T 4b196f 4b19e0 4b19ef 2
+  ///
+  /// Legacy branch profile with separate branches and fall-throughs:
   /// F 41be50 41be50 3
   /// F 41be90 41be90 4
   /// B 4b1942 39b57f0 3 0
@@ -463,7 +483,6 @@ class DataAggregator : public DataReader {
 
   /// Debugging dump methods
   void dump() const;
-  void dump(const LBREntry &LBR) const;
   void dump(const PerfBranchSample &Sample) const;
   void dump(const PerfMemSample &Sample) const;
 
@@ -491,6 +510,12 @@ class DataAggregator : public DataReader {
 
   friend class YAMLProfileWriter;
 };
+
+inline raw_ostream &operator<<(raw_ostream &OS,
+                               const DataAggregator::LBREntry &L) {
+  OS << formatv("{0:x} -> {1:x}/{2}", L.From, L.To, L.Mispred ? 'M' : 'P');
+  return OS;
+}
 } // namespace bolt
 } // namespace llvm
 
diff --git a/external/llvm-project/bolt/include/bolt/Profile/DataReader.h b/external/llvm-project/bolt/include/bolt/Profile/DataReader.h
index 5df1b5a8f4a0..6f527ba3931d 100644
--- a/external/llvm-project/bolt/include/bolt/Profile/DataReader.h
+++ b/external/llvm-project/bolt/include/bolt/Profile/DataReader.h
@@ -32,18 +32,6 @@ namespace bolt {
 
 class BinaryFunction;
 
-struct LBREntry {
-  uint64_t From;
-  uint64_t To;
-  bool Mispred;
-};
-
-inline raw_ostream &operator<<(raw_ostream &OS, const LBREntry &LBR) {
-  OS << "0x" << Twine::utohexstr(LBR.From) << " -> 0x"
-     << Twine::utohexstr(LBR.To);
-  return OS;
-}
-
 struct Location {
   bool IsSymbol;
   StringRef Name;
@@ -109,6 +97,9 @@ struct FuncBranchData {
   /// Total execution count for the function.
   int64_t ExecutionCount{0};
 
+  /// Total entry count from external code for the function.
+  uint64_t ExternEntryCount{0};
+
   /// Indicate if the data was used.
   bool Used{false};
 
diff --git a/external/llvm-project/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/external/llvm-project/bolt/include/bolt/Profile/ProfileYAMLMapping.h
index a8d9a15311d9..41e2bd1651ef 100644
--- a/external/llvm-project/bolt/include/bolt/Profile/ProfileYAMLMapping.h
+++ b/external/llvm-project/bolt/include/bolt/Profile/ProfileYAMLMapping.h
@@ -206,6 +206,7 @@ struct BinaryFunctionProfile {
   uint32_t Id{0};
   llvm::yaml::Hex64 Hash{0};
   uint64_t ExecCount{0};
+  uint64_t ExternEntryCount{0};
   std::vector<BinaryBasicBlockProfile> Blocks;
   std::vector<InlineTreeNode> InlineTree;
   bool Used{false};
@@ -218,6 +219,7 @@ template <> struct MappingTraits<bolt::BinaryFunctionProfile> {
     YamlIO.mapRequired("fid", BFP.Id);
     YamlIO.mapRequired("hash", BFP.Hash);
     YamlIO.mapRequired("exec", BFP.ExecCount);
+    YamlIO.mapOptional("extern", BFP.ExternEntryCount, 0);
     YamlIO.mapRequired("nblocks", BFP.NumBasicBlocks);
     YamlIO.mapOptional("blocks", BFP.Blocks,
                        std::vector<bolt::BinaryBasicBlockProfile>());
diff --git a/external/llvm-project/bolt/lib/Core/BinaryContext.cpp b/external/llvm-project/bolt/lib/Core/BinaryContext.cpp
index 36b789d42936..84f185346970 100644
--- a/external/llvm-project/bolt/lib/Core/BinaryContext.cpp
+++ b/external/llvm-project/bolt/lib/Core/BinaryContext.cpp
@@ -1032,10 +1032,8 @@ void BinaryContext::adjustCodePadding() {
 
     if (!hasValidCodePadding(BF)) {
       if (HasRelocations) {
-        if (opts::Verbosity >= 1) {
-          this->outs() << "BOLT-INFO: function " << BF
-                       << " has invalid padding. Ignoring the function.\n";
-        }
+        this->errs() << "BOLT-WARNING: function " << BF
+                     << " has invalid padding. Ignoring the function\n";
         BF.setIgnored();
       } else {
         BF.setMaxSize(BF.getSize());
diff --git a/external/llvm-project/bolt/lib/Core/BinaryFunction.cpp b/external/llvm-project/bolt/lib/Core/BinaryFunction.cpp
index ceb9390b546e..b998d7160aae 100644
--- a/external/llvm-project/bolt/lib/Core/BinaryFunction.cpp
+++ b/external/llvm-project/bolt/lib/Core/BinaryFunction.cpp
@@ -471,6 +471,8 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation) {
     OS << "\n  Sample Count: " << RawSampleCount;
     OS << "\n  Profile Acc : " << format("%.1f%%", ProfileMatchRatio * 100.0f);
   }
+  if (ExternEntryCount)
+    OS << "\n  Extern Entry Count: " << ExternEntryCount;
 
   if (opts::PrintDynoStats && !getLayout().block_empty()) {
     OS << '\n';
@@ -1526,8 +1528,6 @@ Error BinaryFunction::disassemble() {
   if (uint64_t Offset = getFirstInstructionOffset())
     Labels[Offset] = BC.Ctx->createNamedTempSymbol();
 
-  clearList(Relocations);
-
   if (!IsSimple) {
     clearList(Instructions);
     return createNonFatalBOLTError("");
@@ -3244,16 +3244,26 @@ void BinaryFunction::setTrapOnEntry() {
 }
 
 void BinaryFunction::setIgnored() {
+  IsIgnored = true;
+
   if (opts::processAllFunctions()) {
     // We can accept ignored functions before they've been disassembled.
-    // In that case, they would still get disassembled and emited, but not
+    // In that case, they would still get disassembled and emitted, but not
     // optimized.
-    assert(CurrentState == State::Empty &&
-           "cannot ignore non-empty functions in current mode");
-    IsIgnored = true;
+    if (CurrentState != State::Empty) {
+      BC.errs() << "BOLT-ERROR: cannot ignore non-empty function " << *this
+                << " in current mode\n";
+      exit(1);
+    }
     return;
   }
 
+  IsSimple = false;
+  LLVM_DEBUG(dbgs() << "Ignoring " << getPrintName() << '\n');
+
+  if (CurrentState == State::Empty)
+    return;
+
   clearDisasmState();
 
   // Clear CFG state too.
@@ -3273,9 +3283,11 @@ void BinaryFunction::setIgnored() {
 
   CurrentState = State::Empty;
 
-  IsIgnored = true;
-  IsSimple = false;
-  LLVM_DEBUG(dbgs() << "Ignoring " << getPrintName() << '\n');
+  // Fix external references in the original function body.
+  if (BC.HasRelocations) {
+    LLVM_DEBUG(dbgs() << "Scanning refs in " << *this << '\n');
+    scanExternalRefs();
+  }
 }
 
 void BinaryFunction::duplicateConstantIslands() {
@@ -3764,7 +3776,6 @@ void BinaryFunction::postProcessBranches() {
 
 MCSymbol *BinaryFunction::addEntryPointAtOffset(uint64_t Offset) {
   assert(Offset && "cannot add primary entry point");
-  assert(CurrentState == State::Empty || CurrentState == State::Disassembled);
 
   const uint64_t EntryPointAddress = getAddress() + Offset;
   MCSymbol *LocalSymbol = getOrCreateLocalLabel(EntryPointAddress);
@@ -3773,6 +3784,8 @@ MCSymbol *BinaryFunction::addEntryPointAtOffset(uint64_t Offset) {
   if (EntrySymbol)
     return EntrySymbol;
 
+  assert(CurrentState == State::Empty || CurrentState == State::Disassembled);
+
   if (BinaryData *EntryBD = BC.getBinaryDataAtAddress(EntryPointAddress)) {
     EntrySymbol = EntryBD->getSymbol();
   } else {
diff --git a/external/llvm-project/bolt/lib/Core/HashUtilities.cpp b/external/llvm-project/bolt/lib/Core/HashUtilities.cpp
index 6bdf30460f34..e01768387d6c 100644
--- a/external/llvm-project/bolt/lib/Core/HashUtilities.cpp
+++ b/external/llvm-project/bolt/lib/Core/HashUtilities.cpp
@@ -67,6 +67,7 @@ std::string hashExpr(BinaryContext &BC, const MCExpr &Expr) {
         .append(hashInteger(BinaryExpr.getOpcode()))
         .append(hashExpr(BC, *BinaryExpr.getRHS()));
   }
+  case MCExpr::Specifier:
   case MCExpr::Target:
     return std::string();
   }
diff --git a/external/llvm-project/bolt/lib/Core/MCPlusBuilder.cpp b/external/llvm-project/bolt/lib/Core/MCPlusBuilder.cpp
index 7752079b6153..fa8f4d1df308 100644
--- a/external/llvm-project/bolt/lib/Core/MCPlusBuilder.cpp
+++ b/external/llvm-project/bolt/lib/Core/MCPlusBuilder.cpp
@@ -114,17 +114,19 @@ bool MCPlusBuilder::equals(const MCExpr &A, const MCExpr &B,
            equals(*BinaryA.getRHS(), *BinaryB.getRHS(), Comp);
   }
 
-  case MCExpr::Target: {
-    const auto &TargetExprA = cast<MCTargetExpr>(A);
-    const auto &TargetExprB = cast<MCTargetExpr>(B);
+  case MCExpr::Specifier: {
+    const auto &TargetExprA = cast<MCSpecifierExpr>(A);
+    const auto &TargetExprB = cast<MCSpecifierExpr>(B);
     return equals(TargetExprA, TargetExprB, Comp);
   }
+  case MCExpr::Target:
+    llvm_unreachable("Not implemented");
   }
 
   llvm_unreachable("Invalid expression kind!");
 }
 
-bool MCPlusBuilder::equals(const MCTargetExpr &A, const MCTargetExpr &B,
+bool MCPlusBuilder::equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
                            CompFuncTy Comp) const {
   llvm_unreachable("target-specific expressions are unsupported");
 }
diff --git a/external/llvm-project/bolt/lib/Passes/ADRRelaxationPass.cpp b/external/llvm-project/bolt/lib/Passes/ADRRelaxationPass.cpp
index 4b37a061ac12..c3954c94a7f9 100644
--- a/external/llvm-project/bolt/lib/Passes/ADRRelaxationPass.cpp
+++ b/external/llvm-project/bolt/lib/Passes/ADRRelaxationPass.cpp
@@ -81,17 +81,15 @@ void ADRRelaxationPass::runOnFunction(BinaryFunction &BF) {
         It = BB.eraseInstruction(std::prev(It));
       } else if (std::next(It) != BB.end() && BC.MIB->isNoop(*std::next(It))) {
         BB.eraseInstruction(std::next(It));
-      } else if (!opts::StrictMode && !BF.isSimple()) {
+      } else if (!BF.isSimple()) {
         // If the function is not simple, it may contain a jump table undetected
         // by us. This jump table may use an offset from the branch instruction
         // to land in the desired place. If we add new instructions, we
         // invalidate this offset, so we have to rely on linker-inserted NOP to
         // replace it with ADRP, and abort if it is not present.
         auto L = BC.scopeLock();
-        BC.errs() << formatv(
-            "BOLT-ERROR: Cannot relax adr in non-simple function "
-            "{0}. Use --strict option to override\n",
-            BF.getOneName());
+        BC.errs() << "BOLT-ERROR: cannot relax ADR in non-simple function "
+                  << BF << '\n';
         PassFailed = true;
         return;
       }
diff --git a/external/llvm-project/bolt/lib/Passes/ProfileQualityStats.cpp b/external/llvm-project/bolt/lib/Passes/ProfileQualityStats.cpp
index dfd74d3dd571..64cc662c3ab2 100644
--- a/external/llvm-project/bolt/lib/Passes/ProfileQualityStats.cpp
+++ b/external/llvm-project/bolt/lib/Passes/ProfileQualityStats.cpp
@@ -532,6 +532,9 @@ void computeFlowMappings(const BinaryContext &BC, FlowInfo &TotalFlowMap) {
     std::vector<uint64_t> &MaxCountMap = TotalMaxCountMaps[FunctionNum];
     std::vector<uint64_t> &MinCountMap = TotalMinCountMaps[FunctionNum];
 
+    // Record external entry count into CallGraphIncomingFlows
+    CallGraphIncomingFlows[FunctionNum] += Function->getExternEntryCount();
+
     // Update MaxCountMap, MinCountMap, and CallGraphIncomingFlows
     auto recordCall = [&](const BinaryBasicBlock *SourceBB,
                           const MCSymbol *DestSymbol, uint64_t Count,
diff --git a/external/llvm-project/bolt/lib/Passes/RetpolineInsertion.cpp b/external/llvm-project/bolt/lib/Passes/RetpolineInsertion.cpp
index 98e5a8fba645..bda26206e16c 100644
--- a/external/llvm-project/bolt/lib/Passes/RetpolineInsertion.cpp
+++ b/external/llvm-project/bolt/lib/Passes/RetpolineInsertion.cpp
@@ -195,7 +195,7 @@ std::string createRetpolineFunctionTag(BinaryContext &BC,
 
   TagOS << "+";
   if (MemRef.DispExpr)
-    MemRef.DispExpr->print(TagOS, BC.AsmInfo.get());
+    BC.AsmInfo->printExpr(TagOS, *MemRef.DispExpr);
   else
     TagOS << MemRef.DispImm;
 
diff --git a/external/llvm-project/bolt/lib/Profile/DataAggregator.cpp b/external/llvm-project/bolt/lib/Profile/DataAggregator.cpp
index d0620b64cad2..ade8478f556e 100644
--- a/external/llvm-project/bolt/lib/Profile/DataAggregator.cpp
+++ b/external/llvm-project/bolt/lib/Profile/DataAggregator.cpp
@@ -61,6 +61,12 @@ FilterMemProfile("filter-mem-profile",
   cl::init(true),
   cl::cat(AggregatorCategory));
 
+static cl::opt<bool> ParseMemProfile(
+    "parse-mem-profile",
+    cl::desc("enable memory profile parsing if it's present in the input data, "
+             "on by default unless `--itrace` is set."),
+    cl::init(true), cl::cat(AggregatorCategory));
+
 static cl::opt<unsigned long long>
 FilterPID("pid",
   cl::desc("only use samples from process with specified PID"),
@@ -181,6 +187,10 @@ void DataAggregator::start() {
                       "script -F pid,event,ip",
                       /*Wait = */false);
   } else if (!opts::ITraceAggregation.empty()) {
+    // Disable parsing memory profile from trace data, unless requested by user.
+    if (!opts::ParseMemProfile.getNumOccurrences())
+      opts::ParseMemProfile = false;
+
     std::string ItracePerfScriptArgs = llvm::formatv(
         "script -F pid,brstack --itrace={0}", opts::ITraceAggregation);
     launchPerfProcess("branch events with itrace", MainEventsPPI,
@@ -191,12 +201,9 @@ void DataAggregator::start() {
                       /*Wait = */ false);
   }
 
-  // Note: we launch script for mem events regardless of the option, as the
-  //       command fails fairly fast if mem events were not collected.
-  launchPerfProcess("mem events",
-                    MemEventsPPI,
-                    "script -F pid,event,addr,ip",
-                    /*Wait = */false);
+  if (opts::ParseMemProfile)
+    launchPerfProcess("mem events", MemEventsPPI, "script -F pid,event,addr,ip",
+                      /*Wait = */ false);
 
   launchPerfProcess("process events", MMapEventsPPI,
                     "script --show-mmap-events --no-itrace",
@@ -217,7 +224,8 @@ void DataAggregator::abort() {
   sys::Wait(TaskEventsPPI.PI, 1, &Error);
   sys::Wait(MMapEventsPPI.PI, 1, &Error);
   sys::Wait(MainEventsPPI.PI, 1, &Error);
-  sys::Wait(MemEventsPPI.PI, 1, &Error);
+  if (opts::ParseMemProfile)
+    sys::Wait(MemEventsPPI.PI, 1, &Error);
 
   deleteTempFiles();
 
@@ -506,7 +514,8 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
     errs() << "PERF2BOLT: failed to parse samples\n";
 
   // Special handling for memory events
-  if (!prepareToParse("mem events", MemEventsPPI, MemEventsErrorCallback))
+  if (opts::ParseMemProfile &&
+      !prepareToParse("mem events", MemEventsPPI, MemEventsErrorCallback))
     if (const std::error_code EC = parseMemEvents())
       errs() << "PERF2BOLT: failed to parse memory events: " << EC.message()
              << '\n';
@@ -580,8 +589,10 @@ void DataAggregator::processProfile(BinaryContext &BC) {
     }
   }
 
-  for (auto &FuncBranches : NamesToBranches)
+  for (auto &FuncBranches : NamesToBranches) {
     llvm::stable_sort(FuncBranches.second.Data);
+    llvm::stable_sort(FuncBranches.second.EntryData);
+  }
 
   for (auto &MemEvents : NamesToMemEvents)
     llvm::stable_sort(MemEvents.second.Data);
@@ -733,8 +744,10 @@ bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
   // corresponds to a return (if \p IsFrom) or a call continuation (otherwise).
   auto handleAddress = [&](uint64_t &Addr, bool IsFrom) {
     BinaryFunction *Func = getBinaryFunctionContainingAddress(Addr);
-    if (!Func)
+    if (!Func) {
+      Addr = 0;
       return std::pair{Func, false};
+    }
 
     Addr -= Func->getAddress();
 
@@ -972,7 +985,7 @@ bool DataAggregator::recordExit(BinaryFunction &BF, uint64_t From, bool Mispred,
   return true;
 }
 
-ErrorOr<LBREntry> DataAggregator::parseLBREntry() {
+ErrorOr<DataAggregator::LBREntry> DataAggregator::parseLBREntry() {
   LBREntry Res;
   ErrorOr<StringRef> FromStrRes = parseString('/');
   if (std::error_code EC = FromStrRes.getError())
@@ -1204,60 +1217,74 @@ ErrorOr<Location> DataAggregator::parseLocationOrOffset() {
 }
 
 std::error_code DataAggregator::parseAggregatedLBREntry() {
-  while (checkAndConsumeFS()) {
-  }
+  enum AggregatedLBREntry : char {
+    INVALID = 0,
+    EVENT_NAME,        // E
+    TRACE,             // T
+    SAMPLE,            // S
+    BRANCH,            // B
+    FT,                // F
+    FT_EXTERNAL_ORIGIN // f
+  } Type = INVALID;
+
+  // The number of fields to parse, set based on Type.
+  int AddrNum = 0;
+  int CounterNum = 0;
+  // Storage for parsed fields.
+  StringRef EventName;
+  std::optional<Location> Addr[3];
+  int64_t Counters[2] = {0};
+
+  while (Type == INVALID || Type == EVENT_NAME) {
+    while (checkAndConsumeFS()) {
+    }
+    ErrorOr<StringRef> StrOrErr =
+        parseString(FieldSeparator, Type == EVENT_NAME);
+    if (std::error_code EC = StrOrErr.getError())
+      return EC;
+    StringRef Str = StrOrErr.get();
 
-  ErrorOr<StringRef> TypeOrErr = parseString(FieldSeparator);
-  if (std::error_code EC = TypeOrErr.getError())
-    return EC;
-  enum AggregatedLBREntry { TRACE, BRANCH, FT, FT_EXTERNAL_ORIGIN, INVALID };
-  auto Type = StringSwitch<AggregatedLBREntry>(TypeOrErr.get())
-                  .Case("T", TRACE)
-                  .Case("B", BRANCH)
-                  .Case("F", FT)
-                  .Case("f", FT_EXTERNAL_ORIGIN)
-                  .Default(INVALID);
-  if (Type == INVALID) {
-    reportError("expected T, B, F or f");
-    return make_error_code(llvm::errc::io_error);
-  }
+    if (Type == EVENT_NAME) {
+      EventName = Str;
+      break;
+    }
 
-  while (checkAndConsumeFS()) {
-  }
-  ErrorOr<Location> From = parseLocationOrOffset();
-  if (std::error_code EC = From.getError())
-    return EC;
+    Type = StringSwitch<AggregatedLBREntry>(Str)
+               .Case("T", TRACE)
+               .Case("S", SAMPLE)
+               .Case("E", EVENT_NAME)
+               .Case("B", BRANCH)
+               .Case("F", FT)
+               .Case("f", FT_EXTERNAL_ORIGIN)
+               .Default(INVALID);
+
+    if (Type == INVALID) {
+      reportError("expected T, S, E, B, F or f");
+      return make_error_code(llvm::errc::io_error);
+    }
 
-  while (checkAndConsumeFS()) {
+    using SSI = StringSwitch<int>;
+    AddrNum = SSI(Str).Case("T", 3).Case("S", 1).Case("E", 0).Default(2);
+    CounterNum = SSI(Str).Case("B", 2).Case("E", 0).Default(1);
   }
-  ErrorOr<Location> To = parseLocationOrOffset();
-  if (std::error_code EC = To.getError())
-    return EC;
 
-  ErrorOr<Location> TraceFtEnd = std::error_code();
-  if (Type == AggregatedLBREntry::TRACE) {
+  for (int I = 0; I < AddrNum; ++I) {
     while (checkAndConsumeFS()) {
     }
-    TraceFtEnd = parseLocationOrOffset();
-    if (std::error_code EC = TraceFtEnd.getError())
+    ErrorOr<Location> AddrOrErr = parseLocationOrOffset();
+    if (std::error_code EC = AddrOrErr.getError())
       return EC;
+    Addr[I] = AddrOrErr.get();
   }
 
-  while (checkAndConsumeFS()) {
-  }
-  ErrorOr<int64_t> Frequency =
-      parseNumberField(FieldSeparator, Type != AggregatedLBREntry::BRANCH);
-  if (std::error_code EC = Frequency.getError())
-    return EC;
-
-  uint64_t Mispreds = 0;
-  if (Type == AggregatedLBREntry::BRANCH) {
+  for (int I = 0; I < CounterNum; ++I) {
     while (checkAndConsumeFS()) {
     }
-    ErrorOr<int64_t> MispredsOrErr = parseNumberField(FieldSeparator, true);
-    if (std::error_code EC = MispredsOrErr.getError())
+    ErrorOr<int64_t> CountOrErr =
+        parseNumberField(FieldSeparator, I + 1 == CounterNum);
+    if (std::error_code EC = CountOrErr.getError())
       return EC;
-    Mispreds = static_cast<uint64_t>(MispredsOrErr.get());
+    Counters[I] = CountOrErr.get();
   }
 
   if (!checkAndConsumeNewLine()) {
@@ -1265,16 +1292,31 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
     return make_error_code(llvm::errc::io_error);
   }
 
-  BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(From->Offset);
-  BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(To->Offset);
+  if (Type == EVENT_NAME) {
+    EventNames.insert(EventName);
+    return std::error_code();
+  }
 
-  for (BinaryFunction *BF : {FromFunc, ToFunc})
-    if (BF)
-      BF->setHasProfileAvailable();
+  const uint64_t FromOffset = Addr[0]->Offset;
+  BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(FromOffset);
+  if (FromFunc)
+    FromFunc->setHasProfileAvailable();
+
+  int64_t Count = Counters[0];
+  int64_t Mispreds = Counters[1];
+
+  if (Type == SAMPLE) {
+    BasicSamples[FromOffset] += Count;
+    NumTotalSamples += Count;
+    return std::error_code();
+  }
 
-  uint64_t Count = static_cast<uint64_t>(Frequency.get());
+  const uint64_t ToOffset = Addr[1]->Offset;
+  BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(ToOffset);
+  if (ToFunc)
+    ToFunc->setHasProfileAvailable();
 
-  Trace Trace(From->Offset, To->Offset);
+  Trace Trace(FromOffset, ToOffset);
   // Taken trace
   if (Type == TRACE || Type == BRANCH) {
     TakenBranchInfo &Info = BranchLBRs[Trace];
@@ -1285,8 +1327,9 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
   }
   // Construct fallthrough part of the trace
   if (Type == TRACE) {
-    Trace.From = To->Offset;
-    Trace.To = TraceFtEnd->Offset;
+    const uint64_t TraceFtEndOffset = Addr[2]->Offset;
+    Trace.From = ToOffset;
+    Trace.To = TraceFtEndOffset;
     Type = FromFunc == ToFunc ? FT : FT_EXTERNAL_ORIGIN;
   }
   // Add fallthrough trace
@@ -1400,54 +1443,16 @@ void DataAggregator::parseLBRSample(const PerfBranchSample &Sample,
       const uint64_t TraceTo = NextLBR->From;
       const BinaryFunction *TraceBF =
           getBinaryFunctionContainingAddress(TraceFrom);
-      if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive) {
-        FTInfo &Info = FallthroughLBRs[Trace(TraceFrom, TraceTo)];
+      FTInfo &Info = FallthroughLBRs[Trace(TraceFrom, TraceTo)];
+      if (TraceBF && TraceBF->containsAddress(LBR.From))
         ++Info.InternCount;
-      } else if (TraceBF && TraceBF->containsAddress(TraceTo)) {
-        FTInfo &Info = FallthroughLBRs[Trace(TraceFrom, TraceTo)];
-        if (TraceBF->containsAddress(LBR.From))
-          ++Info.InternCount;
-        else
-          ++Info.ExternCount;
-      } else {
-        const BinaryFunction *ToFunc =
-            getBinaryFunctionContainingAddress(TraceTo);
-        if (TraceBF && ToFunc) {
-          LLVM_DEBUG({
-            dbgs() << "Invalid trace starting in " << TraceBF->getPrintName()
-                   << formatv(" @ {0:x}", TraceFrom - TraceBF->getAddress())
-                   << formatv(" and ending @ {0:x}\n", TraceTo);
-          });
-          ++NumInvalidTraces;
-        } else {
-          LLVM_DEBUG({
-            dbgs() << "Out of range trace starting in "
-                   << (TraceBF ? TraceBF->getPrintName() : "None")
-                   << formatv(" @ {0:x}",
-                              TraceFrom - (TraceBF ? TraceBF->getAddress() : 0))
-                   << " and ending in "
-                   << (ToFunc ? ToFunc->getPrintName() : "None")
-                   << formatv(" @ {0:x}\n",
-                              TraceTo - (ToFunc ? ToFunc->getAddress() : 0));
-          });
-          ++NumLongRangeTraces;
-        }
-      }
+      else
+        ++Info.ExternCount;
       ++NumTraces;
     }
     NextLBR = &LBR;
 
-    // Record branches outside binary functions for heatmap.
-    if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive) {
-      TakenBranchInfo &Info = BranchLBRs[Trace(LBR.From, LBR.To)];
-      ++Info.TakenCount;
-      continue;
-    }
-    uint64_t From = getBinaryFunctionContainingAddress(LBR.From) ? LBR.From : 0;
-    uint64_t To = getBinaryFunctionContainingAddress(LBR.To) ? LBR.To : 0;
-    if (!From && !To)
-      continue;
-    TakenBranchInfo &Info = BranchLBRs[Trace(From, To)];
+    TakenBranchInfo &Info = BranchLBRs[Trace(LBR.From, LBR.To)];
     ++Info.TakenCount;
     Info.MispredCount += LBR.Mispred;
   }
@@ -2259,6 +2264,7 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
       YamlBF.Id = BF->getFunctionNumber();
       YamlBF.Hash = BAT->getBFHash(FuncAddress);
       YamlBF.ExecCount = BF->getKnownExecutionCount();
+      YamlBF.ExternEntryCount = BF->getExternEntryCount();
       YamlBF.NumBasicBlocks = BAT->getNumBasicBlocks(FuncAddress);
       const BoltAddressTranslation::BBHashMapTy &BlockMap =
           BAT->getBBHashMap(FuncAddress);
@@ -2368,16 +2374,10 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
 
 void DataAggregator::dump() const { DataReader::dump(); }
 
-void DataAggregator::dump(const LBREntry &LBR) const {
-  Diag << "From: " << Twine::utohexstr(LBR.From)
-       << " To: " << Twine::utohexstr(LBR.To) << " Mispred? " << LBR.Mispred
-       << "\n";
-}
-
 void DataAggregator::dump(const PerfBranchSample &Sample) const {
   Diag << "Sample LBR entries: " << Sample.LBR.size() << "\n";
   for (const LBREntry &LBR : Sample.LBR)
-    dump(LBR);
+    Diag << LBR << '\n';
 }
 
 void DataAggregator::dump(const PerfMemSample &Sample) const {
diff --git a/external/llvm-project/bolt/lib/Profile/DataReader.cpp b/external/llvm-project/bolt/lib/Profile/DataReader.cpp
index c512394f26a3..afe24216d7f5 100644
--- a/external/llvm-project/bolt/lib/Profile/DataReader.cpp
+++ b/external/llvm-project/bolt/lib/Profile/DataReader.cpp
@@ -85,6 +85,7 @@ void FuncBranchData::appendFrom(const FuncBranchData &FBD, uint64_t Offset) {
   }
   llvm::stable_sort(Data);
   ExecutionCount += FBD.ExecutionCount;
+  ExternEntryCount += FBD.ExternEntryCount;
   for (auto I = FBD.EntryData.begin(), E = FBD.EntryData.end(); I != E; ++I) {
     assert(I->To.Name == FBD.Name);
     auto NewElmt = EntryData.insert(EntryData.end(), *I);
@@ -269,6 +270,7 @@ Error DataReader::preprocessProfile(BinaryContext &BC) {
     if (FuncBranchData *FuncData = getBranchDataForNames(Function.getNames())) {
       setBranchData(Function, FuncData);
       Function.ExecutionCount = FuncData->ExecutionCount;
+      Function.ExternEntryCount = FuncData->ExternEntryCount;
       FuncData->Used = true;
     }
   }
@@ -419,6 +421,7 @@ void DataReader::matchProfileData(BinaryFunction &BF) {
       if (fetchProfileForOtherEntryPoints(BF)) {
         BF.ProfileMatchRatio = evaluateProfileData(BF, *FBD);
         BF.ExecutionCount = FBD->ExecutionCount;
+        BF.ExternEntryCount = FBD->ExternEntryCount;
         BF.RawSampleCount = FBD->getNumExecutedBranches();
       }
       return;
@@ -449,6 +452,7 @@ void DataReader::matchProfileData(BinaryFunction &BF) {
     setBranchData(BF, NewBranchData);
     NewBranchData->Used = true;
     BF.ExecutionCount = NewBranchData->ExecutionCount;
+    BF.ExternEntryCount = NewBranchData->ExternEntryCount;
     BF.ProfileMatchRatio = 1.0f;
     break;
   }
@@ -1190,6 +1194,8 @@ std::error_code DataReader::parse() {
     if (BI.To.IsSymbol && BI.To.Offset == 0) {
       I = GetOrCreateFuncEntry(BI.To.Name);
       I->second.ExecutionCount += BI.Branches;
+      if (!BI.From.IsSymbol)
+        I->second.ExternEntryCount += BI.Branches;
     }
   }
 
diff --git a/external/llvm-project/bolt/lib/Profile/YAMLProfileReader.cpp b/external/llvm-project/bolt/lib/Profile/YAMLProfileReader.cpp
index 33ce40ac2eee..086e47b661e1 100644
--- a/external/llvm-project/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/external/llvm-project/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -176,6 +176,7 @@ bool YAMLProfileReader::parseFunctionProfile(
   uint64_t FunctionExecutionCount = 0;
 
   BF.setExecutionCount(YamlBF.ExecCount);
+  BF.setExternEntryCount(YamlBF.ExternEntryCount);
 
   uint64_t FuncRawBranchCount = 0;
   for (const yaml::bolt::BinaryBasicBlockProfile &YamlBB : YamlBF.Blocks)
diff --git a/external/llvm-project/bolt/lib/Profile/YAMLProfileWriter.cpp b/external/llvm-project/bolt/lib/Profile/YAMLProfileWriter.cpp
index f1fe45f21a0f..1632aa1c6bfe 100644
--- a/external/llvm-project/bolt/lib/Profile/YAMLProfileWriter.cpp
+++ b/external/llvm-project/bolt/lib/Profile/YAMLProfileWriter.cpp
@@ -226,6 +226,7 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
   YamlBF.Hash = BF.getHash();
   YamlBF.NumBasicBlocks = BF.size();
   YamlBF.ExecCount = BF.getKnownExecutionCount();
+  YamlBF.ExternEntryCount = BF.getExternEntryCount();
   DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t> InlineTreeNodeId;
   if (PseudoProbeDecoder && BF.getGUID()) {
     std::tie(YamlBF.InlineTree, InlineTreeNodeId) =
@@ -303,9 +304,8 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
       }
       // Sort targets in a similar way to getBranchData, see Location::operator<
       llvm::sort(CSTargets, [](const auto &RHS, const auto &LHS) {
-        if (RHS.first != LHS.first)
-          return RHS.first < LHS.first;
-        return RHS.second.Offset < LHS.second.Offset;
+        return std::tie(RHS.first, RHS.second.Offset) <
+               std::tie(LHS.first, LHS.second.Offset);
       });
       for (auto &KV : CSTargets)
         YamlBB.CallSites.push_back(KV.second);
diff --git a/external/llvm-project/bolt/lib/Rewrite/RewriteInstance.cpp b/external/llvm-project/bolt/lib/Rewrite/RewriteInstance.cpp
index 69c39cf5e5ba..e1aa00a3d749 100644
--- a/external/llvm-project/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/external/llvm-project/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -3446,6 +3446,7 @@ void RewriteInstance::disassembleFunctions() {
         BC->outs() << "BOLT-INFO: could not disassemble function " << Function
                    << ". Will ignore.\n";
       // Forcefully ignore the function.
+      Function.scanExternalRefs();
       Function.setIgnored();
     });
 
diff --git a/external/llvm-project/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/external/llvm-project/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 9d5a578cfbdf..eb1d9d8a1951 100644
--- a/external/llvm-project/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/external/llvm-project/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -177,7 +177,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     return true;
   }
 
-  bool equals(const MCTargetExpr &A, const MCTargetExpr &B,
+  bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
               CompFuncTy Comp) const override {
     const auto &AArch64ExprA = cast<AArch64MCExpr>(A);
     const auto &AArch64ExprB = cast<AArch64MCExpr>(B);
diff --git a/external/llvm-project/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/external/llvm-project/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 581f22f2a2b7..c7d664ab09d4 100644
--- a/external/llvm-project/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/external/llvm-project/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/RISCVMCExpr.h"
+#include "MCTargetDesc/RISCVMCAsmInfo.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "bolt/Core/MCPlusBuilder.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -31,10 +31,10 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
 public:
   using MCPlusBuilder::MCPlusBuilder;
 
-  bool equals(const MCTargetExpr &A, const MCTargetExpr &B,
+  bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
               CompFuncTy Comp) const override {
-    const auto &RISCVExprA = cast<RISCVMCExpr>(A);
-    const auto &RISCVExprB = cast<RISCVMCExpr>(B);
+    const auto &RISCVExprA = cast<MCSpecifierExpr>(A);
+    const auto &RISCVExprB = cast<MCSpecifierExpr>(B);
     if (RISCVExprA.getSpecifier() != RISCVExprB.getSpecifier())
       return false;
 
@@ -245,7 +245,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
                   MCContext *Ctx) {
     Inst.setOpcode(Opcode);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(RISCVMCExpr::create(
+    Inst.addOperand(MCOperand::createExpr(MCSpecifierExpr::create(
         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
         ELF::R_RISCV_CALL_PLT, *Ctx)));
   }
@@ -342,7 +342,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
   }
 
   const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
-    auto *RISCVExpr = dyn_cast<RISCVMCExpr>(Expr);
+    auto *RISCVExpr = dyn_cast<MCSpecifierExpr>(Expr);
     if (RISCVExpr && RISCVExpr->getSubExpr())
       return getTargetSymbol(RISCVExpr->getSubExpr());
 
@@ -435,19 +435,19 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     case ELF::R_RISCV_TLS_GD_HI20:
       // The GOT is reused so no need to create GOT relocations
     case ELF::R_RISCV_PCREL_HI20:
-      return RISCVMCExpr::create(Expr, ELF::R_RISCV_PCREL_HI20, Ctx);
+      return MCSpecifierExpr::create(Expr, ELF::R_RISCV_PCREL_HI20, Ctx);
     case ELF::R_RISCV_PCREL_LO12_I:
     case ELF::R_RISCV_PCREL_LO12_S:
-      return RISCVMCExpr::create(Expr, RISCVMCExpr::VK_PCREL_LO, Ctx);
+      return MCSpecifierExpr::create(Expr, RISCV::S_PCREL_LO, Ctx);
     case ELF::R_RISCV_HI20:
-      return RISCVMCExpr::create(Expr, ELF::R_RISCV_HI20, Ctx);
+      return MCSpecifierExpr::create(Expr, ELF::R_RISCV_HI20, Ctx);
     case ELF::R_RISCV_LO12_I:
     case ELF::R_RISCV_LO12_S:
-      return RISCVMCExpr::create(Expr, RISCVMCExpr::VK_LO, Ctx);
+      return MCSpecifierExpr::create(Expr, RISCV::S_LO, Ctx);
     case ELF::R_RISCV_CALL:
-      return RISCVMCExpr::create(Expr, ELF::R_RISCV_CALL_PLT, Ctx);
+      return MCSpecifierExpr::create(Expr, ELF::R_RISCV_CALL_PLT, Ctx);
     case ELF::R_RISCV_CALL_PLT:
-      return RISCVMCExpr::create(Expr, ELF::R_RISCV_CALL_PLT, Ctx);
+      return MCSpecifierExpr::create(Expr, ELF::R_RISCV_CALL_PLT, Ctx);
     }
   }
 
@@ -466,10 +466,10 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
       return false;
 
     const auto *ImmExpr = ImmOp.getExpr();
-    if (!isa<RISCVMCExpr>(ImmExpr))
+    if (!isa<MCSpecifierExpr>(ImmExpr))
       return false;
 
-    switch (cast<RISCVMCExpr>(ImmExpr)->getSpecifier()) {
+    switch (cast<MCSpecifierExpr>(ImmExpr)->getSpecifier()) {
     default:
       return false;
     case ELF::R_RISCV_CALL_PLT:
diff --git a/external/llvm-project/bolt/lib/Utils/CMakeLists.txt b/external/llvm-project/bolt/lib/Utils/CMakeLists.txt
index efba6d54449d..94933644ef5e 100644
--- a/external/llvm-project/bolt/lib/Utils/CMakeLists.txt
+++ b/external/llvm-project/bolt/lib/Utils/CMakeLists.txt
@@ -6,12 +6,25 @@ set(version_inc "${CMAKE_CURRENT_BINARY_DIR}/VCSVersion.inc")
 
 set(generate_vcs_version_script "${LLVM_CMAKE_DIR}/GenerateVersionFromVCS.cmake")
 
+if(llvm_vc AND LLVM_APPEND_VC_REV)
+  set(llvm_source_dir ${LLVM_MAIN_SRC_DIR})
+endif()
+if(LLVM_VC_REPOSITORY AND LLVM_VC_REVISION)
+  set(llvm_source_dir ${LLVM_SOURCE_DIR})
+  set(llvm_vc_repository ${LLVM_VC_REPOSITORY})
+  set(llvm_vc_revision ${LLVM_VC_REVISION})
+endif()
+if(bolt_vc AND LLVM_APPEND_VC_REV)
+  set(bolt_source_dir ${BOLT_SOURCE_DIR})
+endif()
+
 # Create custom target to generate the VC revision include.
 add_custom_command(OUTPUT "${version_inc}"
   DEPENDS "${llvm_vc}" "${bolt_vc}" "${generate_vcs_version_script}"
   COMMAND ${CMAKE_COMMAND} "-DNAMES=BOLT"
+                           "-DLLVM_SOURCE_DIR=${llvm_source_dir}"
+                           "-DBOLT_SOURCE_DIR=${bolt_source_dir}"
                            "-DHEADER_FILE=${version_inc}"
-                           "-DBOLT_SOURCE_DIR=${BOLT_SOURCE_DIR}"
                            "-DLLVM_VC_REPOSITORY=${llvm_vc_repository}"
                            "-DLLVM_VC_REVISION=${llvm_vc_revision}"
                            "-DLLVM_FORCE_VC_REVISION=${LLVM_FORCE_VC_REVISION}"
diff --git a/external/llvm-project/bolt/test/AArch64/adr-relaxation-fail.s b/external/llvm-project/bolt/test/AArch64/adr-relaxation-fail.s
new file mode 100644
index 000000000000..2b04bf8db697
--- /dev/null
+++ b/external/llvm-project/bolt/test/AArch64/adr-relaxation-fail.s
@@ -0,0 +1,33 @@
+## Check that llvm-bolt generates a proper error message when ADR instruction
+## cannot be relaxed.
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -static
+# RUN: not llvm-bolt %t.exe -o %t.bolt 2>&1 | FileCheck %s
+# RUN: not llvm-bolt %t.exe -o %t.bolt --strict 2>&1 | FileCheck %s
+
+# CHECK: BOLT-ERROR: cannot relax ADR in non-simple function _start
+
+## The function contains unknown control flow and llvm-bolt fails to recover
+## CFG. As BOLT has to preserve the function layout, the ADR instruction cannot
+## be relaxed into ADRP+ADD.
+  .text
+  .globl _start
+  .type _start, %function
+_start:
+  .cfi_startproc
+  adr x1, foo
+  adr x2, .L1
+.L1:
+  br x0
+  ret  x0
+  .cfi_endproc
+  .size _start, .-_start
+
+  .globl foo
+  .type foo, %function
+foo:
+  .cfi_startproc
+  ret  x0
+  .cfi_endproc
+  .size foo, .-foo
diff --git a/external/llvm-project/bolt/test/AArch64/adr-relaxation.s b/external/llvm-project/bolt/test/AArch64/adr-relaxation.s
index 0aa3c71f29aa..864650c3287d 100644
--- a/external/llvm-project/bolt/test/AArch64/adr-relaxation.s
+++ b/external/llvm-project/bolt/test/AArch64/adr-relaxation.s
@@ -3,7 +3,7 @@
 # RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
 # RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -static
 # RUN: llvm-bolt %t.exe -o %t.bolt --split-functions --split-strategy=random2
-# RUN: llvm-objdump -d --disassemble-symbols=_start %t.bolt | FileCheck %s
+# RUN: llvm-objdump -d -j .text %t.bolt | FileCheck %s
 # RUN: llvm-objdump -d --disassemble-symbols=foo.cold.0 %t.bolt \
 # RUN:   | FileCheck --check-prefix=CHECK-FOO %s
 
@@ -13,11 +13,10 @@
   .globl _start
   .type _start, %function
 _start:
-# CHECK: <_start>:
   .cfi_startproc
+# CHECK: <_start>:
+# CHECK-NEXT: adr
   adr x1, _start
-# CHECK-NOT: adrp
-# CHECK: adr
   cmp  x1, x11
   b.hi  .L1
   mov  x0, #0x0
@@ -35,15 +34,29 @@ foo:
   .cfi_startproc
   cmp  x1, x11
   b.hi  .L2
-  mov  x0, #0x0
 .L2:
 # CHECK-FOO: <foo.cold.0>:
-  adr x1, foo
-# CHECK-FOO: adrp
+# CHECK-FOO-NEXT: adrp
 # CHECK-FOO-NEXT: add
+  adr x1, foo
   ret  x30
   .cfi_endproc
 .size foo, .-foo
 
-## Force relocation mode.
-  .reloc 0, R_AARCH64_NONE
+## bar is a non-simple function. We can still relax ADR, because it has a
+## preceding NOP.
+  .globl bar
+  .type bar, %function
+bar:
+  .cfi_startproc
+# CHECK-LABEL: <bar>:
+# CHECK-NEXT: adrp
+# CHECK-NEXT: add
+  nop
+  adr x0, foo
+  adr x1, .L3
+  br x1
+.L3:
+  ret  x0
+  .cfi_endproc
+  .size bar, .-bar
diff --git a/external/llvm-project/bolt/test/AArch64/patch-ignored.s b/external/llvm-project/bolt/test/AArch64/patch-ignored.s
new file mode 100644
index 000000000000..377d4dacea60
--- /dev/null
+++ b/external/llvm-project/bolt/test/AArch64/patch-ignored.s
@@ -0,0 +1,33 @@
+## Check that llvm-bolt patches functions that are getting ignored after their
+## CFG was constructed.
+
+# RUN: %clang %cflags %s -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --force-patch 2>&1 | FileCheck %s
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OBJDUMP
+
+  .text
+
+## The function is too small to be patched and BOLT is forced to ignore it under
+## --force-patch. Check that the reference to _start is updated.
+# CHECK: BOLT-WARNING: failed to patch entries in unpatchable
+	.globl unpatchable
+  .type unpatchable, %function
+unpatchable:
+  .cfi_startproc
+# CHECK-OBJDUMP:      <unpatchable>:
+# CHECK-OBJDUMP-NEXT: bl {{.*}} <_start>
+  bl _start
+  ret
+  .cfi_endproc
+  .size unpatchable, .-unpatchable
+
+  .globl _start
+  .type _start, %function
+_start:
+  .cfi_startproc
+  cmp  x0, 1
+  b.eq  .L0
+.L0:
+  ret  x30
+  .cfi_endproc
+  .size _start, .-_start
diff --git a/external/llvm-project/bolt/test/AArch64/r_aarch64_prelxx.s b/external/llvm-project/bolt/test/AArch64/r_aarch64_prelxx.s
index 73bf8387d363..5cbe2c50b294 100644
--- a/external/llvm-project/bolt/test/AArch64/r_aarch64_prelxx.s
+++ b/external/llvm-project/bolt/test/AArch64/r_aarch64_prelxx.s
@@ -12,16 +12,14 @@
 // CHECKPREL-NEXT:  R_AARCH64_PREL32      {{.*}} _start + 4
 // CHECKPREL-NEXT:  R_AARCH64_PREL64      {{.*}} _start + 8
 
-// RUN: llvm-bolt %t.exe -o %t.bolt --strict
+// RUN: llvm-bolt %t.exe -o %t.bolt
 // RUN: llvm-objdump -D %t.bolt | FileCheck %s --check-prefix=CHECKPREL32
 
 // CHECKPREL32: [[#%x,DATATABLEADDR:]] <datatable>:
 // CHECKPREL32-NEXT: 00:
 // CHECKPREL32-NEXT: 04: {{.*}} .word 0x[[#%x,VALUE:]]
 
-// 4 is offset in datatable
-// 8 is addend
-// CHECKPREL32: [[#DATATABLEADDR + 4 - 8 + VALUE]] <_start>:
+// CHECKPREL32: [[#DATATABLEADDR + VALUE]] <_start>:
 
 // RUN: llvm-objdump -D %t.bolt | FileCheck %s --check-prefix=CHECKPREL64
 // CHECKPREL64: [[#%x,DATATABLEADDR:]] <datatable>:
@@ -30,16 +28,15 @@
 // CHECKPREL64-NEXT: 08: {{.*}} .word 0x[[#%x,VALUE:]]
 // CHECKPREL64-NEXT: 0c: {{.*}} .word 0x00000000
 
-// 8 is offset in datatable
-// 12 is addend
-// CHECKPREL64: [[#DATATABLEADDR + 8 - 12 + VALUE]] <_start>:
+// CHECKPREL64: [[#DATATABLEADDR + VALUE]] <_start>:
 
   .section .text
   .align 4
   .globl _start
   .type _start, %function
 _start:
-  adr x0, datatable
+  adrp x0, datatable
+  add x0, x0, :lo12:datable
   mov x0, #0
   ret 
 
diff --git a/external/llvm-project/bolt/test/AArch64/test-indirect-branch.s b/external/llvm-project/bolt/test/AArch64/test-indirect-branch.s
index b99737ee97ac..a9fa7737d3d2 100644
--- a/external/llvm-project/bolt/test/AArch64/test-indirect-branch.s
+++ b/external/llvm-project/bolt/test/AArch64/test-indirect-branch.s
@@ -7,8 +7,8 @@
 
 // RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown -mattr=+pauth %s -o %t.o
 // RUN: %clang %cflags --target=aarch64-unknown-linux %t.o -o %t.exe -Wl,-q
-// RUN: llvm-bolt %t.exe -o %t.bolt --print-cfg --strict --debug-only=mcplus \
-// RUN:  -v=1 2>&1 | FileCheck %s
+// RUN: llvm-bolt %t.exe -o %t.bolt --print-cfg --debug-only=mcplus -v=1 2>&1 \
+// RUN:   | FileCheck %s
 
 // Pattern 1: there is no shift amount after the 'add' instruction.
 //
@@ -45,6 +45,7 @@ _start:
   .type  test1, %function
 test1:
   mov     x1, #0
+  nop
   adr     x3, datatable
   add     x3, x3, x1, lsl #2
   ldr     w2, [x3]
@@ -56,6 +57,11 @@ test1_1:
    ret
 test1_2:
    ret
+   .size test1, .-test1
+
+// Temporary workaround for PC-relative relocations from datatable leaking into
+// test2 function and creating phantom entry points.
+.skip 0x100
 
 // Pattern 2
 // CHECK: BOLT-DEBUG: failed to match indirect branch: nop/adr instead of adrp/add
@@ -65,6 +71,7 @@ test2:
   nop
   adr     x3, jump_table
   ldrh    w3, [x3, x1, lsl #1]
+  nop
   adr     x1, test2_0
   add     x3, x1, w3, sxth #2
   br      x3
diff --git a/external/llvm-project/bolt/test/X86/Inputs/pre-aggregated-basic.txt b/external/llvm-project/bolt/test/X86/Inputs/pre-aggregated-basic.txt
new file mode 100644
index 000000000000..dcb85a1d4e44
--- /dev/null
+++ b/external/llvm-project/bolt/test/X86/Inputs/pre-aggregated-basic.txt
@@ -0,0 +1,19 @@
+E cycles
+S 4005f0 1
+S 4005f0 1
+S 400610 1
+S 400ad1 2
+S 400b10 1
+S 400bb7 1
+S 400bbc 2
+S 400d90 1
+S 400dae 1
+S 400e00 2
+S 401170 22
+S 401180 58
+S 4011a0 33
+S 4011a9 33
+S 4011ad 58
+S 4011b2 22
+S X:7f36d18d60c0 2
+S X:7f36d18f2ce0 1
diff --git a/external/llvm-project/bolt/test/X86/entry-point-fallthru.s b/external/llvm-project/bolt/test/X86/entry-point-fallthru.s
index edf14247b0c4..850247b97d27 100644
--- a/external/llvm-project/bolt/test/X86/entry-point-fallthru.s
+++ b/external/llvm-project/bolt/test/X86/entry-point-fallthru.s
@@ -6,6 +6,10 @@
 # RUN: link_fdata %s %t %t.preagg PREAGG
 # RUN: perf2bolt %t -p %t.preagg --pa -o %t.fdata | FileCheck %s
 # CHECK: traces mismatching disassembled function contents: 0
+# RUN: FileCheck %s --check-prefix=CHECK-FDATA --input-file %t.fdata
+# CHECK-FDATA:      1 main 0 1 main 6 0 1
+# CHECK-FDATA-NEXT: 1 main e 1 main 11 0 1
+# CHECK-FDATA-NEXT: 1 main 11 1 main 0 0 1
 
 	.globl main
 main:
diff --git a/external/llvm-project/bolt/test/X86/patch-ignored.s b/external/llvm-project/bolt/test/X86/patch-ignored.s
new file mode 100644
index 000000000000..4f98b51ca1ca
--- /dev/null
+++ b/external/llvm-project/bolt/test/X86/patch-ignored.s
@@ -0,0 +1,44 @@
+## Check that llvm-bolt patches functions that are getting ignored after their
+## CFG was constructed.
+
+# RUN: %clang %cflags %s -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt 2>&1 | FileCheck %s
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OBJDUMP
+
+  .text
+
+## After the CFG is built, the following function will be marked as ignored
+## due to the presence of the internal call.
+# CHECK:      BOLT-WARNING: will skip the following function
+# CHECK-NEXT: internal_call
+	.globl internal_call
+  .type internal_call, %function
+internal_call:
+  .cfi_startproc
+# CHECK-OBJDUMP:      <internal_call>:
+  call .L1
+  jmp .L2
+# CHECK-OBJDUMP:      jmp
+.L1:
+  jmp _start
+# CHECK-OBJDUMP:      jmp
+# CHECK-OBJDUMP-SAME: <_start>
+  ret
+.L2:
+  jmp _start
+# CHECK-OBJDUMP:      jmp
+# CHECK-OBJDUMP-SAME: <_start>
+  .cfi_endproc
+  .size internal_call, .-internal_call
+
+  .globl _start
+  .type _start, %function
+_start:
+  .cfi_startproc
+  cmpq  %rdi, 1
+  jne  .L0
+  movq %rdi, %rax
+.L0:
+  ret
+  .cfi_endproc
+  .size _start, .-_start
diff --git a/external/llvm-project/bolt/test/X86/pre-aggregated-perf.test b/external/llvm-project/bolt/test/X86/pre-aggregated-perf.test
index c05a06bf7494..c4f5b8f2ea44 100644
--- a/external/llvm-project/bolt/test/X86/pre-aggregated-perf.test
+++ b/external/llvm-project/bolt/test/X86/pre-aggregated-perf.test
@@ -36,37 +36,48 @@ RUN: llvm-bolt %t.exe -p %p/Inputs/pre-aggregated.txt --pa -o %t.null | FileChec
 
 CHECK: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty execution profile
 
-RUN: cat %t | sort | FileCheck %s -check-prefix=PERF2BOLT
-RUN: cat %t.new | FileCheck %s -check-prefix=NEWFORMAT
+RUN: FileCheck %s -check-prefix=PERF2BOLT --input-file %t
+RUN: FileCheck %s -check-prefix=NEWFORMAT --input-file %t.new
 
 ## Test --profile-format option with perf2bolt
 RUN: perf2bolt %t.exe -o %t.fdata --pa -p %p/Inputs/pre-aggregated.txt \
 RUN:   --profile-format=fdata
-RUN: cat %t.fdata | sort | FileCheck %s -check-prefix=PERF2BOLT
+RUN: FileCheck %s -check-prefix=PERF2BOLT --input-file %t.fdata
 
 RUN: perf2bolt %t.exe -o %t.yaml --pa -p %p/Inputs/pre-aggregated.txt \
 RUN:   --profile-format=yaml --profile-use-dfs
-RUN: cat %t.yaml | FileCheck %s -check-prefix=NEWFORMAT
+RUN: FileCheck %s -check-prefix=NEWFORMAT --input-file %t.yaml
 
 ## Test --profile-format option with llvm-bolt --aggregate-only
 RUN: llvm-bolt %t.exe -o %t.bolt.fdata --pa -p %p/Inputs/pre-aggregated.txt \
 RUN:   --aggregate-only --profile-format=fdata
-RUN: cat %t.bolt.fdata | sort | FileCheck %s -check-prefix=PERF2BOLT
+RUN: FileCheck %s -check-prefix=PERF2BOLT --input-file %t.bolt.fdata
 
 RUN: llvm-bolt %t.exe -o %t.bolt.yaml --pa -p %p/Inputs/pre-aggregated.txt \
 RUN:   --aggregate-only --profile-format=yaml --profile-use-dfs
-RUN: cat %t.bolt.yaml | FileCheck %s -check-prefix=NEWFORMAT
-
-PERF2BOLT: 0 [unknown] 7f36d18d60c0 1 main 53c 0 2
-PERF2BOLT: 1 main 451 1 SolveCubic 0 0 2
-PERF2BOLT: 1 main 490 0 [unknown] 4005f0 0 1
-PERF2BOLT: 1 main 537 0 [unknown] 400610 0 1
-PERF2BOLT: 1 usqrt 30 1 usqrt 32 0 22
-PERF2BOLT: 1 usqrt 30 1 usqrt 39 4 33
-PERF2BOLT: 1 usqrt 35 1 usqrt 39 0 22
-PERF2BOLT: 1 usqrt 3d 1 usqrt 10 0 58
-PERF2BOLT: 1 usqrt 3d 1 usqrt 3f 0 22
-PERF2BOLT: 1 usqrt a 1 usqrt 10 0 22
+RUN: FileCheck %s -check-prefix=NEWFORMAT --input-file %t.bolt.yaml
+
+## Test pre-aggregated basic profile
+RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated-basic.txt -o %t.ba \
+RUN:   2>&1 | FileCheck %s --check-prefix=BASIC-ERROR
+RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated-basic.txt -o %t.ba.nl \
+RUN:   -nl 2>&1 | FileCheck %s --check-prefix=BASIC-SUCCESS
+RUN: FileCheck %s --input-file %t.ba.nl --check-prefix CHECK-BASIC-NL
+BASIC-ERROR: BOLT-INFO: 0 out of 7 functions in the binary (0.0%) have non-empty execution profile
+BASIC-SUCCESS: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty execution profile
+CHECK-BASIC-NL: no_lbr cycles
+
+PERF2BOLT: 1 frame_dummy/1 1e 1 frame_dummy/1 0 0 1
+PERF2BOLT-NEXT: 1 main 451 1 SolveCubic 0 0 2
+PERF2BOLT-NEXT: 1 main 490 0 [unknown] 0 0 1
+PERF2BOLT-NEXT: 1 main 537 0 [unknown] 0 0 1
+PERF2BOLT-NEXT: 0 [unknown] 0 1 main 53c 0 2
+PERF2BOLT-NEXT: 1 usqrt a 1 usqrt 10 0 22
+PERF2BOLT-NEXT: 1 usqrt 30 1 usqrt 32 0 22
+PERF2BOLT-NEXT: 1 usqrt 30 1 usqrt 39 4 33
+PERF2BOLT-NEXT: 1 usqrt 35 1 usqrt 39 0 22
+PERF2BOLT-NEXT: 1 usqrt 3d 1 usqrt 10 0 58
+PERF2BOLT-NEXT: 1 usqrt 3d 1 usqrt 3f 0 22
 
 NEWFORMAT:  - name:            'frame_dummy/1'
 NEWFORMAT:    fid:             3
diff --git a/external/llvm-project/bolt/test/X86/shrinkwrapping.test b/external/llvm-project/bolt/test/X86/shrinkwrapping.test
index 8581d7e0c0f7..521b4561b3ba 100644
--- a/external/llvm-project/bolt/test/X86/shrinkwrapping.test
+++ b/external/llvm-project/bolt/test/X86/shrinkwrapping.test
@@ -8,6 +8,7 @@ REQUIRES: shell
 
 RUN: %clangxx %cxxflags -no-pie %S/Inputs/exc4sw.S -o %t.exe -Wl,-q
 RUN: llvm-bolt %t.exe -o %t --relocs --frame-opt=all \
+RUN:   --print-only=main --print-cfg \
 RUN:   --data=%p/Inputs/exc4sw.fdata --reorder-blocks=cache 2>&1 | \
 RUN:   FileCheck %s --check-prefix=CHECK-BOLT
 
@@ -19,6 +20,7 @@ RUN: llvm-objdump --dwarf=frames %t | grep -A20 -e \
 RUN:   `llvm-nm --numeric-sort %t | grep main | tail -n 1 | cut -f1 -d' ' | \
 RUN:    tail -c9` 2>&1 | FileCheck %s --check-prefix=CHECK-OUTPUT
 
+CHECK-BOLT: Extern Entry Count: 100
 CHECK-BOLT: Shrink wrapping moved 2 spills inserting load/stores and 0 spills inserting push/pops
 
 CHECK-INPUT:  DW_CFA_advance_loc: 2
diff --git a/external/llvm-project/bolt/test/link_fdata.py b/external/llvm-project/bolt/test/link_fdata.py
index bcf9a777922d..5a9752068bb9 100755
--- a/external/llvm-project/bolt/test/link_fdata.py
+++ b/external/llvm-project/bolt/test/link_fdata.py
@@ -33,12 +33,12 @@
 # <is symbol?> <closest elf symbol or DSO name> <relative FROM address>
 # <is symbol?> <closest elf symbol or DSO name> <relative TO address>
 # <number of mispredictions> <number of branches>
-fdata_pat = re.compile(r"([01].*) (?P<exec>\d+) (?P<mispred>\d+)")
+fdata_pat = re.compile(r"([01].*) (?P<mispred>\d+) (?P<exec>\d+)")
 
 # Pre-aggregated profile:
-# {T|B|F|f} [<start_id>:]<start_offset> [<end_id>:]<end_offset> [<ft_end>]
-# <count> [<mispred_count>]
-preagg_pat = re.compile(r"(?P<type>[TBFf]) (?P<offsets_count>.*)")
+# {T|S|E|B|F|f} <start> [<end>] [<ft_end>] <count> [<mispred_count>]
+# <loc>: [<id>:]<offset>
+preagg_pat = re.compile(r"(?P<type>[TSBFf]) (?P<offsets_count>.*)")
 
 # No-LBR profile:
 # <is symbol?> <closest elf symbol or DSO name> <relative address> <count>
@@ -61,7 +61,7 @@
         preagg_match = preagg_pat.match(profile_line)
         nolbr_match = nolbr_pat.match(profile_line)
         if fdata_match:
-            src_dst, execnt, mispred = fdata_match.groups()
+            src_dst, mispred, execnt = fdata_match.groups()
             # Split by whitespaces not preceded by a backslash (negative lookbehind)
             chunks = re.split(r"(?<!\\) +", src_dst)
             # Check if the number of records separated by non-escaped whitespace
@@ -69,7 +69,7 @@
             assert (
                 len(chunks) == 6
             ), f"ERROR: wrong format/whitespaces must be escaped:\n{line}"
-            exprs.append(("FDATA", (*chunks, execnt, mispred)))
+            exprs.append(("FDATA", (*chunks, mispred, execnt)))
         elif nolbr_match:
             loc, count = nolbr_match.groups()
             # Split by whitespaces not preceded by a backslash (negative lookbehind)
diff --git a/external/llvm-project/bolt/test/runtime/AArch64/adrrelaxationpass.s b/external/llvm-project/bolt/test/runtime/AArch64/adrrelaxationpass.s
deleted file mode 100644
index fa9fb63c613d..000000000000
--- a/external/llvm-project/bolt/test/runtime/AArch64/adrrelaxationpass.s
+++ /dev/null
@@ -1,62 +0,0 @@
-# The second and third ADR instructions are non-local to functions
-# and must be replaced with ADRP + ADD by BOLT
-# Also since main and test are non-simple, we can't change it's length so we
-# have to replace NOP with adrp, and if there is no nop before adr in non-simple
-# function, we can't guarantee we didn't break possible jump tables, so we
-# fail in non-strict mode
-
-# REQUIRES: system-linux
-
-# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \
-# RUN:   %s -o %t.o
-# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q
-# RUN: llvm-bolt %t.exe -o %t.bolt --adr-relaxation=true --strict
-# RUN: llvm-objdump --no-print-imm-hex -d --disassemble-symbols=main %t.bolt | FileCheck %s
-# RUN: %t.bolt
-# RUN: not llvm-bolt %t.exe -o %t.bolt --adr-relaxation=true \
-# RUN: 2>&1 | FileCheck %s --check-prefix CHECK-ERROR
-
-  .text
-  .align 4
-  .global test
-  .type test, %function
-test:
-  adr x2, Gvar
-  mov x0, xzr
-  ret
-  .size test, .-test
-
-  .align 4
-  .global main
-  .type main, %function
-main:
-  adr x0, .CI
-  nop
-  adr x1, test
-  adr x2, Gvar2
-  adr x3, br
-br:
-  br x1
-  .size main, .-main
-.CI:
-  .word 0xff
-
-  .data
-  .align 8
-  .global Gvar
-Gvar: .xword 0x0
-  .global Gvar2
-Gvar2: .xword 0x42
-  .balign 4
-jmptable:
-  .word 0
-  .word test - jmptable
-
-# CHECK: <main>:
-# CHECK-NEXT: adr x0, 0x{{[1-8a-f][0-9a-f]*}}
-# CHECK-NEXT: adrp x1, 0x{{[1-8a-f][0-9a-f]*}}
-# CHECK-NEXT: add x1, x1, #{{[1-8a-f][0-9a-f]*}}
-# CHECK-NEXT: adrp x2, 0x{{[1-8a-f][0-9a-f]*}}
-# CHECK-NEXT: add x2, x2, #{{[1-8a-f][0-9a-f]*}}
-# CHECK-NEXT: adr x3, 0x{{[1-8a-f][0-9a-f]*}}
-# CHECK-ERROR: BOLT-ERROR: Cannot relax adr in non-simple function
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/BitcodeReader.cpp b/external/llvm-project/clang-tools-extra/clang-doc/BitcodeReader.cpp
index f8e338eb7c6e..35058abab066 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -9,6 +9,7 @@
 #include "BitcodeReader.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -53,10 +54,8 @@ static llvm::Error decodeRecord(const Record &R, AccessSpecifier &Field,
   case AS_none:
     Field = (AccessSpecifier)R[0];
     return llvm::Error::success();
-  default:
-    return llvm::createStringError(llvm::inconvertibleErrorCode(),
-                                   "invalid value for AccessSpecifier");
   }
+  llvm_unreachable("invalid value for AccessSpecifier");
 }
 
 static llvm::Error decodeRecord(const Record &R, TagTypeKind &Field,
@@ -314,9 +313,13 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
 
 static llvm::Error parseRecord(const Record &R, unsigned ID,
                                llvm::StringRef Blob, CommentInfo *I) {
+  llvm::SmallString<16> KindStr;
   switch (ID) {
   case COMMENT_KIND:
-    return decodeRecord(R, I->Kind, Blob);
+    if (llvm::Error Err = decodeRecord(R, KindStr, Blob))
+      return Err;
+    I->Kind = stringToCommentKind(KindStr);
+    return llvm::Error::success();
   case COMMENT_TEXT:
     return decodeRecord(R, I->Text, Blob);
   case COMMENT_NAME:
@@ -672,6 +675,7 @@ llvm::Error ClangDocBitcodeReader::readRecord(unsigned ID, T I) {
 
 template <>
 llvm::Error ClangDocBitcodeReader::readRecord(unsigned ID, Reference *I) {
+  llvm::TimeTraceScope("Reducing infos", "readRecord");
   Record R;
   llvm::StringRef Blob;
   llvm::Expected<unsigned> MaybeRecID = Stream.readRecord(ID, R, &Blob);
@@ -683,6 +687,7 @@ llvm::Error ClangDocBitcodeReader::readRecord(unsigned ID, Reference *I) {
 // Read a block of records into a single info.
 template <typename T>
 llvm::Error ClangDocBitcodeReader::readBlock(unsigned ID, T I) {
+  llvm::TimeTraceScope("Reducing infos", "readBlock");
   if (llvm::Error Err = Stream.EnterSubBlock(ID))
     return Err;
 
@@ -713,6 +718,7 @@ llvm::Error ClangDocBitcodeReader::readBlock(unsigned ID, T I) {
 
 template <typename T>
 llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
+  llvm::TimeTraceScope("Reducing infos", "readSubBlock");
   switch (ID) {
   // Blocks can only have certain types of sub blocks.
   case BI_COMMENT_BLOCK_ID: {
@@ -819,6 +825,7 @@ llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
 
 ClangDocBitcodeReader::Cursor
 ClangDocBitcodeReader::skipUntilRecordOrBlock(unsigned &BlockOrRecordID) {
+  llvm::TimeTraceScope("Reducing infos", "skipUntilRecordOrBlock");
   BlockOrRecordID = 0;
 
   while (!Stream.AtEndOfStream()) {
@@ -880,6 +887,7 @@ llvm::Error ClangDocBitcodeReader::validateStream() {
 }
 
 llvm::Error ClangDocBitcodeReader::readBlockInfoBlock() {
+  llvm::TimeTraceScope("Reducing infos", "readBlockInfoBlock");
   Expected<std::optional<llvm::BitstreamBlockInfo>> MaybeBlockInfo =
       Stream.ReadBlockInfoBlock();
   if (!MaybeBlockInfo)
@@ -895,6 +903,7 @@ llvm::Error ClangDocBitcodeReader::readBlockInfoBlock() {
 template <typename T>
 llvm::Expected<std::unique_ptr<Info>>
 ClangDocBitcodeReader::createInfo(unsigned ID) {
+  llvm::TimeTraceScope("Reducing infos", "createInfo");
   std::unique_ptr<Info> I = std::make_unique<T>();
   if (auto Err = readBlock(ID, static_cast<T *>(I.get())))
     return std::move(Err);
@@ -903,6 +912,7 @@ ClangDocBitcodeReader::createInfo(unsigned ID) {
 
 llvm::Expected<std::unique_ptr<Info>>
 ClangDocBitcodeReader::readBlockToInfo(unsigned ID) {
+  llvm::TimeTraceScope("Reducing infos", "readBlockToInfo");
   switch (ID) {
   case BI_NAMESPACE_BLOCK_ID:
     return createInfo<NamespaceInfo>(ID);
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/BitcodeWriter.cpp b/external/llvm-project/clang-tools-extra/clang-doc/BitcodeWriter.cpp
index f0a445e606bf..f8a6859169b0 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/BitcodeWriter.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/BitcodeWriter.cpp
@@ -484,8 +484,9 @@ void ClangDocBitcodeWriter::emitBlock(const MemberTypeInfo &T) {
 
 void ClangDocBitcodeWriter::emitBlock(const CommentInfo &I) {
   StreamSubBlockGuard Block(Stream, BI_COMMENT_BLOCK_ID);
+  // Handle Kind (enum) separately, since it is not a string.
+  emitRecord(commentKindToString(I.Kind), COMMENT_KIND);
   for (const auto &L : std::vector<std::pair<llvm::StringRef, RecordId>>{
-           {I.Kind, COMMENT_KIND},
            {I.Text, COMMENT_TEXT},
            {I.Name, COMMENT_NAME},
            {I.Direction, COMMENT_DIRECTION},
@@ -663,7 +664,7 @@ bool ClangDocBitcodeWriter::dispatchInfoForWrite(Info *I) {
   case InfoType::IT_typedef:
     emitBlock(*static_cast<clang::doc::TypedefInfo *>(I));
     break;
-  default:
+  case InfoType::IT_default:
     llvm::errs() << "Unexpected info, unable to write.\n";
     return true;
   }
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/CMakeLists.txt b/external/llvm-project/clang-tools-extra/clang-doc/CMakeLists.txt
index 79563c41435e..5989e5fe60cf 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/CMakeLists.txt
+++ b/external/llvm-project/clang-tools-extra/clang-doc/CMakeLists.txt
@@ -17,6 +17,7 @@ add_clang_library(clangDoc STATIC
   Serialize.cpp
   YAMLGenerator.cpp
   HTMLMustacheGenerator.cpp
+  JSONGenerator.cpp
 
   DEPENDS
   omp_gen
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/Generators.cpp b/external/llvm-project/clang-tools-extra/clang-doc/Generators.cpp
index a3c2773412cf..3fb5b63c403a 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/Generators.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/Generators.cpp
@@ -105,5 +105,7 @@ static int LLVM_ATTRIBUTE_UNUSED HTMLGeneratorAnchorDest =
     HTMLGeneratorAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED MHTMLGeneratorAnchorDest =
     MHTMLGeneratorAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED JSONGeneratorAnchorDest =
+    JSONGeneratorAnchorSource;
 } // namespace doc
 } // namespace clang
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/Generators.h b/external/llvm-project/clang-tools-extra/clang-doc/Generators.h
index aee04b9d58d9..92d3006e6002 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/Generators.h
+++ b/external/llvm-project/clang-tools-extra/clang-doc/Generators.h
@@ -58,6 +58,7 @@ extern volatile int YAMLGeneratorAnchorSource;
 extern volatile int MDGeneratorAnchorSource;
 extern volatile int HTMLGeneratorAnchorSource;
 extern volatile int MHTMLGeneratorAnchorSource;
+extern volatile int JSONGeneratorAnchorSource;
 
 } // namespace doc
 } // namespace clang
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/external/llvm-project/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index 93b9279462a8..7293a129177c 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -635,7 +635,8 @@ genHTML(const Index &Index, StringRef InfoPath, bool IsOutermostList) {
 }
 
 static std::unique_ptr<HTMLNode> genHTML(const CommentInfo &I) {
-  if (I.Kind == "FullComment") {
+  switch (I.Kind) {
+  case CommentKind::CK_FullComment: {
     auto FullComment = std::make_unique<TagNode>(HTMLTag::TAG_DIV);
     for (const auto &Child : I.Children) {
       std::unique_ptr<HTMLNode> Node = genHTML(*Child);
@@ -645,7 +646,7 @@ static std::unique_ptr<HTMLNode> genHTML(const CommentInfo &I) {
     return std::move(FullComment);
   }
 
-  if (I.Kind == "ParagraphComment") {
+  case CommentKind::CK_ParagraphComment: {
     auto ParagraphComment = std::make_unique<TagNode>(HTMLTag::TAG_P);
     for (const auto &Child : I.Children) {
       std::unique_ptr<HTMLNode> Node = genHTML(*Child);
@@ -657,7 +658,7 @@ static std::unique_ptr<HTMLNode> genHTML(const CommentInfo &I) {
     return std::move(ParagraphComment);
   }
 
-  if (I.Kind == "BlockCommandComment") {
+  case CommentKind::CK_BlockCommandComment: {
     auto BlockComment = std::make_unique<TagNode>(HTMLTag::TAG_DIV);
     BlockComment->Children.emplace_back(
         std::make_unique<TagNode>(HTMLTag::TAG_DIV, I.Name));
@@ -670,12 +671,26 @@ static std::unique_ptr<HTMLNode> genHTML(const CommentInfo &I) {
       return nullptr;
     return std::move(BlockComment);
   }
-  if (I.Kind == "TextComment") {
-    if (I.Text == "")
+
+  case CommentKind::CK_TextComment: {
+    if (I.Text.empty())
       return nullptr;
     return std::make_unique<TextNode>(I.Text);
   }
-  return nullptr;
+
+  // For now, return nullptr for unsupported comment kinds
+  case CommentKind::CK_InlineCommandComment:
+  case CommentKind::CK_HTMLStartTagComment:
+  case CommentKind::CK_HTMLEndTagComment:
+  case CommentKind::CK_ParamCommandComment:
+  case CommentKind::CK_TParamCommandComment:
+  case CommentKind::CK_VerbatimBlockComment:
+  case CommentKind::CK_VerbatimBlockLineComment:
+  case CommentKind::CK_VerbatimLineComment:
+  case CommentKind::CK_Unknown:
+    return nullptr;
+  }
+  llvm_unreachable("Unhandled CommentKind");
 }
 
 static std::unique_ptr<TagNode> genHTML(const std::vector<CommentInfo> &C) {
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp b/external/llvm-project/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
index 65dc2e93582e..69c670b20844 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Mustache.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TimeProfiler.h"
 
 using namespace llvm;
 using namespace llvm::json;
@@ -125,13 +126,18 @@ static Error setupTemplateFiles(const clang::doc::ClangDocContext &CDCtx) {
 Error MustacheHTMLGenerator::generateDocs(
     StringRef RootDir, StringMap<std::unique_ptr<doc::Info>> Infos,
     const clang::doc::ClangDocContext &CDCtx) {
-  if (auto Err = setupTemplateFiles(CDCtx))
-    return Err;
+  {
+    llvm::TimeTraceScope TS("Setup Templates");
+    if (auto Err = setupTemplateFiles(CDCtx))
+      return Err;
+  }
+
   // Track which directories we already tried to create.
   StringSet<> CreatedDirs;
   // Collect all output by file name and create the necessary directories.
   StringMap<std::vector<doc::Info *>> FileToInfos;
   for (const auto &Group : Infos) {
+    llvm::TimeTraceScope TS("setup directories");
     doc::Info *Info = Group.getValue().get();
 
     SmallString<128> Path;
@@ -148,15 +154,19 @@ Error MustacheHTMLGenerator::generateDocs(
     FileToInfos[Path].push_back(Info);
   }
 
-  for (const auto &Group : FileToInfos) {
-    std::error_code FileErr;
-    raw_fd_ostream InfoOS(Group.getKey(), FileErr, sys::fs::OF_None);
-    if (FileErr)
-      return createFileOpenError(Group.getKey(), FileErr);
-
-    for (const auto &Info : Group.getValue())
-      if (Error Err = generateDocForInfo(Info, InfoOS, CDCtx))
-        return Err;
+  {
+    llvm::TimeTraceScope TS("Generate Docs");
+    for (const auto &Group : FileToInfos) {
+      llvm::TimeTraceScope TS("Info to Doc");
+      std::error_code FileErr;
+      raw_fd_ostream InfoOS(Group.getKey(), FileErr, sys::fs::OF_None);
+      if (FileErr)
+        return createFileOpenError(Group.getKey(), FileErr);
+
+      for (const auto &Info : Group.getValue())
+        if (Error Err = generateDocForInfo(Info, InfoOS, CDCtx))
+          return Err;
+    }
   }
   return Error::success();
 }
@@ -198,37 +208,110 @@ static json::Value extractValue(const TypedefInfo &I) {
 }
 
 static json::Value extractValue(const CommentInfo &I) {
-  assert((I.Kind == "BlockCommandComment" || I.Kind == "FullComment" ||
-          I.Kind == "ParagraphComment" || I.Kind == "TextComment") &&
-         "Unknown Comment type in CommentInfo.");
-
   Object Obj = Object();
-  json::Value Child = Object();
 
-  // TextComment has no children, so return it.
-  if (I.Kind == "TextComment") {
-    Obj.insert({"TextComment", I.Text});
-    return Obj;
-  }
+  json::Value ChildVal = Object();
+  Object &Child = *ChildVal.getAsObject();
 
-  // BlockCommandComment needs to generate a Command key.
-  if (I.Kind == "BlockCommandComment")
-    Child.getAsObject()->insert({"Command", I.Name});
-
-  // Use the same handling for everything else.
-  // Only valid for:
-  //  - BlockCommandComment
-  //  - FullComment
-  //  - ParagraphComment
   json::Value ChildArr = Array();
   auto &CARef = *ChildArr.getAsArray();
   CARef.reserve(I.Children.size());
   for (const auto &C : I.Children)
     CARef.emplace_back(extractValue(*C));
-  Child.getAsObject()->insert({"Children", ChildArr});
-  Obj.insert({I.Kind, Child});
 
-  return Obj;
+  switch (I.Kind) {
+  case CommentKind::CK_TextComment: {
+    Obj.insert({commentKindToString(I.Kind), I.Text});
+    return Obj;
+  }
+
+  case CommentKind::CK_BlockCommandComment: {
+    Child.insert({"Command", I.Name});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_InlineCommandComment: {
+    json::Value ArgsArr = Array();
+    auto &ARef = *ArgsArr.getAsArray();
+    ARef.reserve(I.Args.size());
+    for (const auto &Arg : I.Args)
+      ARef.emplace_back(Arg);
+    Child.insert({"Command", I.Name});
+    Child.insert({"Args", ArgsArr});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_ParamCommandComment:
+  case CommentKind::CK_TParamCommandComment: {
+    Child.insert({"ParamName", I.ParamName});
+    Child.insert({"Direction", I.Direction});
+    Child.insert({"Explicit", I.Explicit});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_VerbatimBlockComment: {
+    Child.insert({"Text", I.Text});
+    if (!I.CloseName.empty())
+      Child.insert({"CloseName", I.CloseName});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_VerbatimBlockLineComment:
+  case CommentKind::CK_VerbatimLineComment: {
+    Child.insert({"Text", I.Text});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_HTMLStartTagComment: {
+    json::Value AttrKeysArray = json::Array();
+    json::Value AttrValuesArray = json::Array();
+    auto &KeyArr = *AttrKeysArray.getAsArray();
+    auto &ValArr = *AttrValuesArray.getAsArray();
+    KeyArr.reserve(I.AttrKeys.size());
+    ValArr.reserve(I.AttrValues.size());
+    for (const auto &K : I.AttrKeys)
+      KeyArr.emplace_back(K);
+    for (const auto &V : I.AttrValues)
+      ValArr.emplace_back(V);
+    Child.insert({"Name", I.Name});
+    Child.insert({"SelfClosing", I.SelfClosing});
+    Child.insert({"AttrKeys", AttrKeysArray});
+    Child.insert({"AttrValues", AttrValuesArray});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_HTMLEndTagComment: {
+    Child.insert({"Name", I.Name});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_FullComment:
+  case CommentKind::CK_ParagraphComment: {
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_Unknown: {
+    Obj.insert({commentKindToString(I.Kind), I.Text});
+    return Obj;
+  }
+  }
+  llvm_unreachable("Unknown comment kind encountered.");
 }
 
 static void maybeInsertLocation(std::optional<Location> Loc,
@@ -245,6 +328,7 @@ static void extractDescriptionFromInfo(ArrayRef<CommentInfo> Descriptions,
     return;
   json::Value DescArr = Array();
   json::Array &DescARef = *DescArr.getAsArray();
+  DescARef.reserve(Descriptions.size());
   for (const CommentInfo &Child : Descriptions)
     DescARef.emplace_back(extractValue(Child));
   EnumValObj.insert({"EnumValueComments", DescArr});
@@ -260,6 +344,7 @@ static json::Value extractValue(const FunctionInfo &I, StringRef ParentInfoDir,
 
   json::Value ParamArr = Array();
   json::Array &ParamARef = *ParamArr.getAsArray();
+  ParamARef.reserve(I.Params.size());
   for (const auto Val : enumerate(I.Params)) {
     json::Value V = Object();
     auto &VRef = *V.getAsObject();
@@ -279,14 +364,14 @@ static json::Value extractValue(const EnumInfo &I,
   Object Obj = Object();
   std::string EnumType = I.Scoped ? "enum class " : "enum ";
   EnumType += I.Name;
-  bool HasComment = std::any_of(
-      I.Members.begin(), I.Members.end(),
-      [](const EnumValueInfo &M) { return !M.Description.empty(); });
+  bool HasComment = llvm::any_of(
+      I.Members, [](const EnumValueInfo &M) { return !M.Description.empty(); });
   Obj.insert({"EnumName", EnumType});
   Obj.insert({"HasComment", HasComment});
   Obj.insert({"ID", toHex(toStringRef(I.USR))});
   json::Value EnumArr = Array();
   json::Array &EnumARef = *EnumArr.getAsArray();
+  EnumARef.reserve(I.Members.size());
   for (const EnumValueInfo &M : I.Members) {
     json::Value EnumValue = Object();
     auto &EnumValObj = *EnumValue.getAsObject();
@@ -312,6 +397,7 @@ static void extractScopeChildren(const ScopeChildren &S, Object &Obj,
                                  const ClangDocContext &CDCtx) {
   json::Value NamespaceArr = Array();
   json::Array &NamespaceARef = *NamespaceArr.getAsArray();
+  NamespaceARef.reserve(S.Namespaces.size());
   for (const Reference &Child : S.Namespaces)
     NamespaceARef.emplace_back(extractValue(Child, ParentInfoDir));
 
@@ -320,6 +406,7 @@ static void extractScopeChildren(const ScopeChildren &S, Object &Obj,
 
   json::Value RecordArr = Array();
   json::Array &RecordARef = *RecordArr.getAsArray();
+  RecordARef.reserve(S.Records.size());
   for (const Reference &Child : S.Records)
     RecordARef.emplace_back(extractValue(Child, ParentInfoDir));
 
@@ -328,12 +415,15 @@ static void extractScopeChildren(const ScopeChildren &S, Object &Obj,
 
   json::Value FunctionArr = Array();
   json::Array &FunctionARef = *FunctionArr.getAsArray();
+  FunctionARef.reserve(S.Functions.size());
 
   json::Value PublicFunctionArr = Array();
   json::Array &PublicFunctionARef = *PublicFunctionArr.getAsArray();
+  PublicFunctionARef.reserve(S.Functions.size());
 
   json::Value ProtectedFunctionArr = Array();
   json::Array &ProtectedFunctionARef = *ProtectedFunctionArr.getAsArray();
+  ProtectedFunctionARef.reserve(S.Functions.size());
 
   for (const FunctionInfo &Child : S.Functions) {
     json::Value F = extractValue(Child, ParentInfoDir, CDCtx);
@@ -357,6 +447,7 @@ static void extractScopeChildren(const ScopeChildren &S, Object &Obj,
 
   json::Value EnumArr = Array();
   auto &EnumARef = *EnumArr.getAsArray();
+  EnumARef.reserve(S.Enums.size());
   for (const EnumInfo &Child : S.Enums)
     EnumARef.emplace_back(extractValue(Child, CDCtx));
 
@@ -365,6 +456,7 @@ static void extractScopeChildren(const ScopeChildren &S, Object &Obj,
 
   json::Value TypedefArr = Array();
   auto &TypedefARef = *TypedefArr.getAsArray();
+  TypedefARef.reserve(S.Typedefs.size());
   for (const TypedefInfo &Child : S.Typedefs)
     TypedefARef.emplace_back(extractValue(Child));
 
@@ -401,10 +493,13 @@ static json::Value extractValue(const RecordInfo &I,
   extractScopeChildren(I.Children, RecordValue, BasePath, CDCtx);
   json::Value PublicMembers = Array();
   json::Array &PubMemberRef = *PublicMembers.getAsArray();
+  PubMemberRef.reserve(I.Members.size());
   json::Value ProtectedMembers = Array();
   json::Array &ProtMemberRef = *ProtectedMembers.getAsArray();
+  ProtMemberRef.reserve(I.Members.size());
   json::Value PrivateMembers = Array();
   json::Array &PrivMemberRef = *PrivateMembers.getAsArray();
+  PrivMemberRef.reserve(I.Members.size());
   for (const MemberTypeInfo &Member : I.Members) {
     json::Value MemberValue = Object();
     auto &MVRef = *MemberValue.getAsObject();
@@ -436,20 +531,25 @@ static Error setupTemplateValue(const ClangDocContext &CDCtx, json::Value &V,
   auto InfoPath = I->getRelativeFilePath("");
   SmallString<128> RelativePath = computeRelativePath("", InfoPath);
   sys::path::native(RelativePath, sys::path::Style::posix);
+
+  auto *SSA = StylesheetArr.getAsArray();
+  SSA->reserve(CDCtx.UserStylesheets.size());
   for (const auto &FilePath : CDCtx.UserStylesheets) {
     SmallString<128> StylesheetPath = RelativePath;
     sys::path::append(StylesheetPath, sys::path::Style::posix,
                       sys::path::filename(FilePath));
-    StylesheetArr.getAsArray()->emplace_back(StylesheetPath);
+    SSA->emplace_back(StylesheetPath);
   }
   V.getAsObject()->insert({"Stylesheets", StylesheetArr});
 
   json::Value ScriptArr = Array();
+  auto *SCA = ScriptArr.getAsArray();
+  SCA->reserve(CDCtx.JsScripts.size());
   for (auto Script : CDCtx.JsScripts) {
     SmallString<128> JsPath = RelativePath;
     sys::path::append(JsPath, sys::path::Style::posix,
                       sys::path::filename(Script));
-    ScriptArr.getAsArray()->emplace_back(JsPath);
+    SCA->emplace_back(JsPath);
   }
   V.getAsObject()->insert({"Scripts", ScriptArr});
   return Error::success();
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/JSONGenerator.cpp b/external/llvm-project/clang-tools-extra/clang-doc/JSONGenerator.cpp
new file mode 100644
index 000000000000..0f7cbafcf513
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/clang-doc/JSONGenerator.cpp
@@ -0,0 +1,542 @@
+#include "Generators.h"
+#include "clang/Basic/Specifiers.h"
+#include "llvm/Support/JSON.h"
+
+using namespace llvm;
+using namespace llvm::json;
+
+namespace clang {
+namespace doc {
+
+class JSONGenerator : public Generator {
+public:
+  static const char *Format;
+
+  Error generateDocs(StringRef RootDir,
+                     llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+                     const ClangDocContext &CDCtx) override;
+  Error createResources(ClangDocContext &CDCtx) override;
+  Error generateDocForInfo(Info *I, llvm::raw_ostream &OS,
+                           const ClangDocContext &CDCtx) override;
+};
+
+const char *JSONGenerator::Format = "json";
+
+static void serializeInfo(const TypedefInfo &I, json::Object &Obj,
+                          std::optional<StringRef> RepositoryUrl);
+static void serializeInfo(const EnumInfo &I, json::Object &Obj,
+                          std::optional<StringRef> RepositoryUrl);
+
+static json::Object serializeLocation(const Location &Loc,
+                                      std::optional<StringRef> RepositoryUrl) {
+  Object LocationObj = Object();
+  LocationObj["LineNumber"] = Loc.StartLineNumber;
+  LocationObj["Filename"] = Loc.Filename;
+
+  if (!Loc.IsFileInRootDir || !RepositoryUrl)
+    return LocationObj;
+  SmallString<128> FileURL(*RepositoryUrl);
+  sys::path::append(FileURL, sys::path::Style::posix, Loc.Filename);
+  FileURL += "#" + std::to_string(Loc.StartLineNumber);
+  LocationObj["FileURL"] = FileURL;
+  return LocationObj;
+}
+
+static json::Value serializeComment(const CommentInfo &I) {
+  // taken from PR #142273
+  Object Obj = Object();
+
+  json::Value ChildVal = Object();
+  Object &Child = *ChildVal.getAsObject();
+
+  json::Value ChildArr = Array();
+  auto &CARef = *ChildArr.getAsArray();
+  CARef.reserve(I.Children.size());
+  for (const auto &C : I.Children)
+    CARef.emplace_back(serializeComment(*C));
+
+  switch (I.Kind) {
+  case CommentKind::CK_TextComment: {
+    Obj.insert({commentKindToString(I.Kind), I.Text});
+    return Obj;
+  }
+
+  case CommentKind::CK_BlockCommandComment: {
+    Child.insert({"Command", I.Name});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_InlineCommandComment: {
+    json::Value ArgsArr = Array();
+    auto &ARef = *ArgsArr.getAsArray();
+    ARef.reserve(I.Args.size());
+    for (const auto &Arg : I.Args)
+      ARef.emplace_back(Arg);
+    Child.insert({"Command", I.Name});
+    Child.insert({"Args", ArgsArr});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_ParamCommandComment:
+  case CommentKind::CK_TParamCommandComment: {
+    Child.insert({"ParamName", I.ParamName});
+    Child.insert({"Direction", I.Direction});
+    Child.insert({"Explicit", I.Explicit});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_VerbatimBlockComment: {
+    Child.insert({"Text", I.Text});
+    if (!I.CloseName.empty())
+      Child.insert({"CloseName", I.CloseName});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_VerbatimBlockLineComment:
+  case CommentKind::CK_VerbatimLineComment: {
+    Child.insert({"Text", I.Text});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_HTMLStartTagComment: {
+    json::Value AttrKeysArray = json::Array();
+    json::Value AttrValuesArray = json::Array();
+    auto &KeyArr = *AttrKeysArray.getAsArray();
+    auto &ValArr = *AttrValuesArray.getAsArray();
+    KeyArr.reserve(I.AttrKeys.size());
+    ValArr.reserve(I.AttrValues.size());
+    for (const auto &K : I.AttrKeys)
+      KeyArr.emplace_back(K);
+    for (const auto &V : I.AttrValues)
+      ValArr.emplace_back(V);
+    Child.insert({"Name", I.Name});
+    Child.insert({"SelfClosing", I.SelfClosing});
+    Child.insert({"AttrKeys", AttrKeysArray});
+    Child.insert({"AttrValues", AttrValuesArray});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_HTMLEndTagComment: {
+    Child.insert({"Name", I.Name});
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_FullComment:
+  case CommentKind::CK_ParagraphComment: {
+    Child.insert({"Children", ChildArr});
+    Obj.insert({commentKindToString(I.Kind), ChildVal});
+    return Obj;
+  }
+
+  case CommentKind::CK_Unknown: {
+    Obj.insert({commentKindToString(I.Kind), I.Text});
+    return Obj;
+  }
+  }
+  llvm_unreachable("Unknown comment kind encountered.");
+}
+
+static void serializeCommonAttributes(const Info &I, json::Object &Obj,
+                                      std::optional<StringRef> RepositoryUrl) {
+  Obj["Name"] = I.Name;
+  Obj["USR"] = toHex(toStringRef(I.USR));
+
+  if (!I.Path.empty())
+    Obj["Path"] = I.Path;
+
+  if (!I.Namespace.empty()) {
+    Obj["Namespace"] = json::Array();
+    for (const auto &NS : I.Namespace)
+      Obj["Namespace"].getAsArray()->push_back(NS.Name);
+  }
+
+  if (!I.Description.empty()) {
+    json::Value DescArray = json::Array();
+    auto &DescArrayRef = *DescArray.getAsArray();
+    DescArrayRef.reserve(I.Description.size());
+    for (const auto &Comment : I.Description)
+      DescArrayRef.push_back(serializeComment(Comment));
+    Obj["Description"] = DescArray;
+  }
+
+  // Namespaces aren't SymbolInfos, so they dont have a DefLoc
+  if (I.IT != InfoType::IT_namespace) {
+    const auto *Symbol = static_cast<const SymbolInfo *>(&I);
+    if (Symbol->DefLoc)
+      Obj["Location"] =
+          serializeLocation(Symbol->DefLoc.value(), RepositoryUrl);
+  }
+}
+
+static void serializeReference(const Reference &Ref, Object &ReferenceObj) {
+  ReferenceObj["Path"] = Ref.Path;
+  ReferenceObj["Name"] = Ref.Name;
+  ReferenceObj["QualName"] = Ref.QualName;
+  ReferenceObj["USR"] = toHex(toStringRef(Ref.USR));
+}
+
+static void serializeReference(const SmallVector<Reference, 4> &References,
+                               Object &Obj, std::string Key) {
+  json::Value ReferencesArray = Array();
+  json::Array &ReferencesArrayRef = *ReferencesArray.getAsArray();
+  ReferencesArrayRef.reserve(References.size());
+  for (const auto &Reference : References) {
+    json::Value ReferenceVal = Object();
+    auto &ReferenceObj = *ReferenceVal.getAsObject();
+    serializeReference(Reference, ReferenceObj);
+    ReferencesArrayRef.push_back(ReferenceVal);
+  }
+  Obj[Key] = ReferencesArray;
+}
+
+// Although namespaces and records both have ScopeChildren, they serialize them
+// differently. Only enums, records, and typedefs are handled here.
+static void serializeCommonChildren(const ScopeChildren &Children,
+                                    json::Object &Obj,
+                                    std::optional<StringRef> RepositoryUrl) {
+  if (!Children.Enums.empty()) {
+    json::Value EnumsArray = Array();
+    auto &EnumsArrayRef = *EnumsArray.getAsArray();
+    EnumsArrayRef.reserve(Children.Enums.size());
+    for (const auto &Enum : Children.Enums) {
+      json::Value EnumVal = Object();
+      auto &EnumObj = *EnumVal.getAsObject();
+      serializeInfo(Enum, EnumObj, RepositoryUrl);
+      EnumsArrayRef.push_back(EnumVal);
+    }
+    Obj["Enums"] = EnumsArray;
+  }
+
+  if (!Children.Typedefs.empty()) {
+    json::Value TypedefsArray = Array();
+    auto &TypedefsArrayRef = *TypedefsArray.getAsArray();
+    TypedefsArrayRef.reserve(Children.Typedefs.size());
+    for (const auto &Typedef : Children.Typedefs) {
+      json::Value TypedefVal = Object();
+      auto &TypedefObj = *TypedefVal.getAsObject();
+      serializeInfo(Typedef, TypedefObj, RepositoryUrl);
+      TypedefsArrayRef.push_back(TypedefVal);
+    }
+    Obj["Typedefs"] = TypedefsArray;
+  }
+
+  if (!Children.Records.empty()) {
+    json::Value RecordsArray = Array();
+    auto &RecordsArrayRef = *RecordsArray.getAsArray();
+    RecordsArrayRef.reserve(Children.Records.size());
+    for (const auto &Record : Children.Records) {
+      json::Value RecordVal = Object();
+      auto &RecordObj = *RecordVal.getAsObject();
+      serializeReference(Record, RecordObj);
+      RecordsArrayRef.push_back(RecordVal);
+    }
+    Obj["Records"] = RecordsArray;
+  }
+}
+
+static void serializeInfo(const TemplateInfo &Template, Object &Obj) {
+  json::Value TemplateVal = Object();
+  auto &TemplateObj = *TemplateVal.getAsObject();
+
+  if (Template.Specialization) {
+    json::Value TemplateSpecializationVal = Object();
+    auto &TemplateSpecializationObj = *TemplateSpecializationVal.getAsObject();
+    TemplateSpecializationObj["SpecializationOf"] =
+        toHex(toStringRef(Template.Specialization->SpecializationOf));
+    if (!Template.Specialization->Params.empty()) {
+      json::Value ParamsArray = Array();
+      auto &ParamsArrayRef = *ParamsArray.getAsArray();
+      ParamsArrayRef.reserve(Template.Specialization->Params.size());
+      for (const auto &Param : Template.Specialization->Params)
+        ParamsArrayRef.push_back(Param.Contents);
+      TemplateSpecializationObj["Parameters"] = ParamsArray;
+    }
+    TemplateObj["Specialization"] = TemplateSpecializationVal;
+  }
+
+  if (!Template.Params.empty()) {
+    json::Value ParamsArray = Array();
+    auto &ParamsArrayRef = *ParamsArray.getAsArray();
+    ParamsArrayRef.reserve(Template.Params.size());
+    for (const auto &Param : Template.Params)
+      ParamsArrayRef.push_back(Param.Contents);
+    TemplateObj["Parameters"] = ParamsArray;
+  }
+
+  Obj["Template"] = TemplateVal;
+}
+
+static void serializeInfo(const TypeInfo &I, Object &Obj) {
+  Obj["Name"] = I.Type.Name;
+  Obj["QualName"] = I.Type.QualName;
+  Obj["USR"] = toHex(toStringRef(I.Type.USR));
+  Obj["IsTemplate"] = I.IsTemplate;
+  Obj["IsBuiltIn"] = I.IsBuiltIn;
+}
+
+static void serializeInfo(const FunctionInfo &F, json::Object &Obj,
+                          std::optional<StringRef> RepositoryURL) {
+  serializeCommonAttributes(F, Obj, RepositoryURL);
+  Obj["IsStatic"] = F.IsStatic;
+
+  auto ReturnTypeObj = Object();
+  serializeInfo(F.ReturnType, ReturnTypeObj);
+  Obj["ReturnType"] = std::move(ReturnTypeObj);
+
+  if (!F.Params.empty()) {
+    json::Value ParamsArray = json::Array();
+    auto &ParamsArrayRef = *ParamsArray.getAsArray();
+    ParamsArrayRef.reserve(F.Params.size());
+    for (const auto &Param : F.Params) {
+      json::Value ParamVal = Object();
+      auto &ParamObj = *ParamVal.getAsObject();
+      ParamObj["Name"] = Param.Name;
+      ParamObj["Type"] = Param.Type.Name;
+      ParamsArrayRef.push_back(ParamVal);
+    }
+    Obj["Params"] = ParamsArray;
+  }
+
+  if (F.Template)
+    serializeInfo(F.Template.value(), Obj);
+}
+
+static void serializeInfo(const EnumInfo &I, json::Object &Obj,
+                          std::optional<StringRef> RepositoryUrl) {
+  serializeCommonAttributes(I, Obj, RepositoryUrl);
+  Obj["Scoped"] = I.Scoped;
+
+  if (I.BaseType) {
+    json::Value BaseTypeVal = Object();
+    auto &BaseTypeObj = *BaseTypeVal.getAsObject();
+    BaseTypeObj["Name"] = I.BaseType->Type.Name;
+    BaseTypeObj["QualName"] = I.BaseType->Type.QualName;
+    BaseTypeObj["USR"] = toHex(toStringRef(I.BaseType->Type.USR));
+    Obj["BaseType"] = BaseTypeVal;
+  }
+
+  if (!I.Members.empty()) {
+    json::Value MembersArray = Array();
+    auto &MembersArrayRef = *MembersArray.getAsArray();
+    MembersArrayRef.reserve(I.Members.size());
+    for (const auto &Member : I.Members) {
+      json::Value MemberVal = Object();
+      auto &MemberObj = *MemberVal.getAsObject();
+      MemberObj["Name"] = Member.Name;
+      if (!Member.ValueExpr.empty())
+        MemberObj["ValueExpr"] = Member.ValueExpr;
+      else
+        MemberObj["Value"] = Member.Value;
+      MembersArrayRef.push_back(MemberVal);
+    }
+    Obj["Members"] = MembersArray;
+  }
+}
+
+static void serializeInfo(const TypedefInfo &I, json::Object &Obj,
+                          std::optional<StringRef> RepositoryUrl) {
+  serializeCommonAttributes(I, Obj, RepositoryUrl);
+  Obj["TypeDeclaration"] = I.TypeDeclaration;
+  Obj["IsUsing"] = I.IsUsing;
+  json::Value TypeVal = Object();
+  auto &TypeObj = *TypeVal.getAsObject();
+  serializeInfo(I.Underlying, TypeObj);
+  Obj["Underlying"] = TypeVal;
+}
+
+static void serializeInfo(const RecordInfo &I, json::Object &Obj,
+                          std::optional<StringRef> RepositoryUrl) {
+  serializeCommonAttributes(I, Obj, RepositoryUrl);
+  Obj["FullName"] = I.FullName;
+  Obj["TagType"] = getTagType(I.TagType);
+  Obj["IsTypedef"] = I.IsTypeDef;
+
+  if (!I.Children.Functions.empty()) {
+    json::Value PubFunctionsArray = Array();
+    json::Array &PubFunctionsArrayRef = *PubFunctionsArray.getAsArray();
+    json::Value ProtFunctionsArray = Array();
+    json::Array &ProtFunctionsArrayRef = *ProtFunctionsArray.getAsArray();
+
+    for (const auto &Function : I.Children.Functions) {
+      json::Value FunctionVal = Object();
+      auto &FunctionObj = *FunctionVal.getAsObject();
+      serializeInfo(Function, FunctionObj, RepositoryUrl);
+      AccessSpecifier Access = Function.Access;
+      if (Access == AccessSpecifier::AS_public)
+        PubFunctionsArrayRef.push_back(FunctionVal);
+      else if (Access == AccessSpecifier::AS_protected)
+        ProtFunctionsArrayRef.push_back(FunctionVal);
+    }
+
+    if (!PubFunctionsArrayRef.empty())
+      Obj["PublicFunctions"] = PubFunctionsArray;
+    if (!ProtFunctionsArrayRef.empty())
+      Obj["ProtectedFunctions"] = ProtFunctionsArray;
+  }
+
+  if (!I.Members.empty()) {
+    json::Value PublicMembersArray = Array();
+    json::Array &PubMembersArrayRef = *PublicMembersArray.getAsArray();
+    json::Value ProtectedMembersArray = Array();
+    json::Array &ProtMembersArrayRef = *ProtectedMembersArray.getAsArray();
+
+    for (const MemberTypeInfo &Member : I.Members) {
+      json::Value MemberVal = Object();
+      auto &MemberObj = *MemberVal.getAsObject();
+      MemberObj["Name"] = Member.Name;
+      MemberObj["Type"] = Member.Type.Name;
+
+      if (Member.Access == AccessSpecifier::AS_public)
+        PubMembersArrayRef.push_back(MemberVal);
+      else if (Member.Access == AccessSpecifier::AS_protected)
+        ProtMembersArrayRef.push_back(MemberVal);
+    }
+
+    if (!PubMembersArrayRef.empty())
+      Obj["PublicMembers"] = PublicMembersArray;
+    if (!ProtMembersArrayRef.empty())
+      Obj["ProtectedMembers"] = ProtectedMembersArray;
+  }
+
+  if (!I.Bases.empty()) {
+    json::Value BasesArray = Array();
+    json::Array &BasesArrayRef = *BasesArray.getAsArray();
+    BasesArrayRef.reserve(I.Bases.size());
+    for (const auto &BaseInfo : I.Bases) {
+      json::Value BaseInfoVal = Object();
+      auto &BaseInfoObj = *BaseInfoVal.getAsObject();
+      serializeInfo(BaseInfo, BaseInfoObj, RepositoryUrl);
+      BaseInfoObj["IsVirtual"] = BaseInfo.IsVirtual;
+      BaseInfoObj["Access"] = getAccessSpelling(BaseInfo.Access);
+      BaseInfoObj["IsParent"] = BaseInfo.IsParent;
+      BasesArrayRef.push_back(BaseInfoVal);
+    }
+    Obj["Bases"] = BasesArray;
+  }
+
+  if (!I.Parents.empty())
+    serializeReference(I.Parents, Obj, "Parents");
+
+  if (!I.VirtualParents.empty())
+    serializeReference(I.VirtualParents, Obj, "VirtualParents");
+
+  if (I.Template)
+    serializeInfo(I.Template.value(), Obj);
+
+  serializeCommonChildren(I.Children, Obj, RepositoryUrl);
+}
+
+static void serializeInfo(const NamespaceInfo &I, json::Object &Obj,
+                          std::optional<StringRef> RepositoryUrl) {
+  serializeCommonAttributes(I, Obj, RepositoryUrl);
+
+  if (!I.Children.Namespaces.empty()) {
+    json::Value NamespacesArray = Array();
+    auto &NamespacesArrayRef = *NamespacesArray.getAsArray();
+    NamespacesArrayRef.reserve(I.Children.Namespaces.size());
+    for (auto &Namespace : I.Children.Namespaces) {
+      json::Value NamespaceVal = Object();
+      auto &NamespaceObj = *NamespaceVal.getAsObject();
+      serializeReference(Namespace, NamespaceObj);
+      NamespacesArrayRef.push_back(NamespaceVal);
+    }
+    Obj["Namespaces"] = NamespacesArray;
+  }
+
+  if (!I.Children.Functions.empty()) {
+    json::Value FunctionsArray = Array();
+    auto &FunctionsArrayRef = *FunctionsArray.getAsArray();
+    FunctionsArrayRef.reserve(I.Children.Functions.size());
+    for (const auto &Function : I.Children.Functions) {
+      json::Value FunctionVal = Object();
+      auto &FunctionObj = *FunctionVal.getAsObject();
+      serializeInfo(Function, FunctionObj, RepositoryUrl);
+      FunctionsArrayRef.push_back(FunctionVal);
+    }
+    Obj["Functions"] = FunctionsArray;
+  }
+
+  serializeCommonChildren(I.Children, Obj, RepositoryUrl);
+}
+
+Error JSONGenerator::generateDocs(
+    StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+    const ClangDocContext &CDCtx) {
+  StringSet<> CreatedDirs;
+  StringMap<std::vector<doc::Info *>> FileToInfos;
+  for (const auto &Group : Infos) {
+    Info *Info = Group.getValue().get();
+
+    SmallString<128> Path;
+    sys::path::native(RootDir, Path);
+    sys::path::append(Path, Info->getRelativeFilePath(""));
+    if (!CreatedDirs.contains(Path)) {
+      if (std::error_code Err = sys::fs::create_directories(Path);
+          Err != std::error_code())
+        return createFileError(Twine(Path), Err);
+      CreatedDirs.insert(Path);
+    }
+
+    sys::path::append(Path, Info->getFileBaseName() + ".json");
+    FileToInfos[Path].push_back(Info);
+  }
+
+  for (const auto &Group : FileToInfos) {
+    std::error_code FileErr;
+    raw_fd_ostream InfoOS(Group.getKey(), FileErr, sys::fs::OF_Text);
+    if (FileErr)
+      return createFileError("cannot open file " + Group.getKey(), FileErr);
+
+    for (const auto &Info : Group.getValue())
+      if (Error Err = generateDocForInfo(Info, InfoOS, CDCtx))
+        return Err;
+  }
+
+  return Error::success();
+}
+
+Error JSONGenerator::generateDocForInfo(Info *I, raw_ostream &OS,
+                                        const ClangDocContext &CDCtx) {
+  json::Object Obj = Object();
+
+  switch (I->IT) {
+  case InfoType::IT_namespace:
+    serializeInfo(*static_cast<NamespaceInfo *>(I), Obj, CDCtx.RepositoryUrl);
+    break;
+  case InfoType::IT_record:
+    serializeInfo(*static_cast<RecordInfo *>(I), Obj, CDCtx.RepositoryUrl);
+    break;
+  case InfoType::IT_enum:
+  case InfoType::IT_function:
+  case InfoType::IT_typedef:
+    break;
+  case InfoType::IT_default:
+    return createStringError(inconvertibleErrorCode(), "unexpected info type");
+  }
+  OS << llvm::formatv("{0:2}", llvm::json::Value(std::move(Obj)));
+  return Error::success();
+}
+
+Error JSONGenerator::createResources(ClangDocContext &CDCtx) {
+  return Error::success();
+}
+
+static GeneratorRegistry::Add<JSONGenerator> JSON(JSONGenerator::Format,
+                                                  "Generator for JSON output.");
+volatile int JSONGeneratorAnchorSource = 0;
+} // namespace doc
+} // namespace clang
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/MDGenerator.cpp b/external/llvm-project/clang-tools-extra/clang-doc/MDGenerator.cpp
index ccd6175c96cb..2becccf8b07d 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/MDGenerator.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/MDGenerator.cpp
@@ -75,39 +75,49 @@ static void maybeWriteSourceFileRef(llvm::raw_ostream &OS,
 }
 
 static void writeDescription(const CommentInfo &I, raw_ostream &OS) {
-  if (I.Kind == "FullComment") {
+  switch (I.Kind) {
+  case CommentKind::CK_FullComment:
     for (const auto &Child : I.Children)
       writeDescription(*Child, OS);
-  } else if (I.Kind == "ParagraphComment") {
+    break;
+
+  case CommentKind::CK_ParagraphComment:
     for (const auto &Child : I.Children)
       writeDescription(*Child, OS);
     writeNewLine(OS);
-  } else if (I.Kind == "BlockCommandComment") {
+    break;
+
+  case CommentKind::CK_BlockCommandComment:
     OS << genEmphasis(I.Name);
     for (const auto &Child : I.Children)
       writeDescription(*Child, OS);
-  } else if (I.Kind == "InlineCommandComment") {
+    break;
+
+  case CommentKind::CK_InlineCommandComment:
     OS << genEmphasis(I.Name) << " " << I.Text;
-  } else if (I.Kind == "ParamCommandComment") {
-    std::string Direction = I.Explicit ? (" " + I.Direction).str() : "";
-    OS << genEmphasis(I.ParamName) << I.Text << Direction;
-    for (const auto &Child : I.Children)
-      writeDescription(*Child, OS);
-  } else if (I.Kind == "TParamCommandComment") {
+    break;
+
+  case CommentKind::CK_ParamCommandComment:
+  case CommentKind::CK_TParamCommandComment: {
     std::string Direction = I.Explicit ? (" " + I.Direction).str() : "";
     OS << genEmphasis(I.ParamName) << I.Text << Direction;
     for (const auto &Child : I.Children)
       writeDescription(*Child, OS);
-  } else if (I.Kind == "VerbatimBlockComment") {
+    break;
+  }
+
+  case CommentKind::CK_VerbatimBlockComment:
     for (const auto &Child : I.Children)
       writeDescription(*Child, OS);
-  } else if (I.Kind == "VerbatimBlockLineComment") {
-    OS << I.Text;
-    writeNewLine(OS);
-  } else if (I.Kind == "VerbatimLineComment") {
+    break;
+
+  case CommentKind::CK_VerbatimBlockLineComment:
+  case CommentKind::CK_VerbatimLineComment:
     OS << I.Text;
     writeNewLine(OS);
-  } else if (I.Kind == "HTMLStartTagComment") {
+    break;
+
+  case CommentKind::CK_HTMLStartTagComment: {
     if (I.AttrKeys.size() != I.AttrValues.size())
       return;
     std::string Buffer;
@@ -117,12 +127,20 @@ static void writeDescription(const CommentInfo &I, raw_ostream &OS) {
 
     std::string CloseTag = I.SelfClosing ? "/>" : ">";
     writeLine("<" + I.Name + Attrs.str() + CloseTag, OS);
-  } else if (I.Kind == "HTMLEndTagComment") {
+    break;
+  }
+
+  case CommentKind::CK_HTMLEndTagComment:
     writeLine("</" + I.Name + ">", OS);
-  } else if (I.Kind == "TextComment") {
+    break;
+
+  case CommentKind::CK_TextComment:
     OS << I.Text;
-  } else {
-    OS << "Unknown comment kind: " << I.Kind << ".\n\n";
+    break;
+
+  case CommentKind::CK_Unknown:
+    OS << "Unknown comment kind: " << static_cast<int>(I.Kind) << ".\n\n";
+    break;
   }
 }
 
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/Mapper.cpp b/external/llvm-project/clang-tools-extra/clang-doc/Mapper.cpp
index a17645c3f3a3..9f640b5325da 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/Mapper.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/Mapper.cpp
@@ -7,13 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "Mapper.h"
-#include "BitcodeWriter.h"
 #include "Serialize.h"
 #include "clang/AST/Comment.h"
 #include "clang/Index/USRGeneration.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Mutex.h"
+#include "llvm/Support/TimeProfiler.h"
 
 namespace clang {
 namespace doc {
@@ -40,48 +40,66 @@ Location MapASTVisitor::getDeclLocation(const NamedDecl *D) const {
 }
 
 void MapASTVisitor::HandleTranslationUnit(ASTContext &Context) {
+  if (CDCtx.FTimeTrace)
+    llvm::timeTraceProfilerInitialize(200, "clang-doc");
   TraverseDecl(Context.getTranslationUnitDecl());
+  if (CDCtx.FTimeTrace)
+    llvm::timeTraceProfilerFinishThread();
 }
 
 template <typename T>
 bool MapASTVisitor::mapDecl(const T *D, bool IsDefinition) {
-  // If we're looking a decl not in user files, skip this decl.
-  if (D->getASTContext().getSourceManager().isInSystemHeader(D->getLocation()))
-    return true;
+  llvm::TimeTraceScope TS("Mapping declaration");
+  {
+    llvm::TimeTraceScope TS("Preamble");
+    // If we're looking a decl not in user files, skip this decl.
+    if (D->getASTContext().getSourceManager().isInSystemHeader(
+            D->getLocation()))
+      return true;
 
-  // Skip function-internal decls.
-  if (D->getParentFunctionOrMethod())
-    return true;
+    // Skip function-internal decls.
+    if (D->getParentFunctionOrMethod())
+      return true;
+  }
+
+  std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>> CP;
 
-  llvm::SmallString<128> USR;
-  // If there is an error generating a USR for the decl, skip this decl.
-  if (index::generateUSRForDecl(D, USR))
-    return true;
-  // Prevent Visiting USR twice
   {
-    llvm::sys::SmartScopedLock<true> Guard(USRVisitedGuard);
-    StringRef Visited = USR.str();
-    if (USRVisited.count(Visited) && !isTypedefAnonRecord<T>(D))
+    llvm::TimeTraceScope TS("emit info from astnode");
+    llvm::SmallString<128> USR;
+    // If there is an error generating a USR for the decl, skip this decl.
+    if (index::generateUSRForDecl(D, USR))
       return true;
-    // We considered a USR to be visited only when its defined
-    if (IsDefinition)
-      USRVisited.insert(Visited);
+    // Prevent Visiting USR twice
+    {
+      llvm::sys::SmartScopedLock<true> Guard(USRVisitedGuard);
+      StringRef Visited = USR.str();
+      if (USRVisited.count(Visited) && !isTypedefAnonRecord<T>(D))
+        return true;
+      // We considered a USR to be visited only when its defined
+      if (IsDefinition)
+        USRVisited.insert(Visited);
+    }
+    bool IsFileInRootDir;
+    llvm::SmallString<128> File =
+        getFile(D, D->getASTContext(), CDCtx.SourceRoot, IsFileInRootDir);
+    CP = serialize::emitInfo(D, getComment(D, D->getASTContext()),
+                             getDeclLocation(D), CDCtx.PublicOnly);
+  }
+
+  auto &[Child, Parent] = CP;
+
+  {
+    llvm::TimeTraceScope TS("serialized info into bitcode");
+    // A null in place of a valid Info indicates that the serializer is skipping
+    // this decl for some reason (e.g. we're only reporting public decls).
+    if (Child)
+      CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Child->USR)),
+                               serialize::serialize(Child));
+    if (Parent)
+      CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Parent->USR)),
+                               serialize::serialize(Parent));
   }
-  bool IsFileInRootDir;
-  llvm::SmallString<128> File =
-      getFile(D, D->getASTContext(), CDCtx.SourceRoot, IsFileInRootDir);
-  auto [Child, Parent] =
-      serialize::emitInfo(D, getComment(D, D->getASTContext()),
-                          getDeclLocation(D), CDCtx.PublicOnly);
-
-  // A null in place of a valid Info indicates that the serializer is skipping
-  // this decl for some reason (e.g. we're only reporting public decls).
-  if (Child)
-    CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Child->USR)),
-                             serialize::serialize(Child));
-  if (Parent)
-    CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Parent->USR)),
-                             serialize::serialize(Parent));
   return true;
 }
 
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/Representation.cpp b/external/llvm-project/clang-tools-extra/clang-doc/Representation.cpp
index 9ab2f342d969..71a926f1c73e 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/Representation.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/Representation.cpp
@@ -20,12 +20,68 @@
 //
 //===----------------------------------------------------------------------===//
 #include "Representation.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
 
 namespace clang {
 namespace doc {
 
+CommentKind stringToCommentKind(llvm::StringRef KindStr) {
+  static const llvm::StringMap<CommentKind> KindMap = {
+      {"FullComment", CommentKind::CK_FullComment},
+      {"ParagraphComment", CommentKind::CK_ParagraphComment},
+      {"TextComment", CommentKind::CK_TextComment},
+      {"InlineCommandComment", CommentKind::CK_InlineCommandComment},
+      {"HTMLStartTagComment", CommentKind::CK_HTMLStartTagComment},
+      {"HTMLEndTagComment", CommentKind::CK_HTMLEndTagComment},
+      {"BlockCommandComment", CommentKind::CK_BlockCommandComment},
+      {"ParamCommandComment", CommentKind::CK_ParamCommandComment},
+      {"TParamCommandComment", CommentKind::CK_TParamCommandComment},
+      {"VerbatimBlockComment", CommentKind::CK_VerbatimBlockComment},
+      {"VerbatimBlockLineComment", CommentKind::CK_VerbatimBlockLineComment},
+      {"VerbatimLineComment", CommentKind::CK_VerbatimLineComment},
+  };
+
+  auto It = KindMap.find(KindStr);
+  if (It != KindMap.end()) {
+    return It->second;
+  }
+  return CommentKind::CK_Unknown;
+}
+
+llvm::StringRef commentKindToString(CommentKind Kind) {
+  switch (Kind) {
+  case CommentKind::CK_FullComment:
+    return "FullComment";
+  case CommentKind::CK_ParagraphComment:
+    return "ParagraphComment";
+  case CommentKind::CK_TextComment:
+    return "TextComment";
+  case CommentKind::CK_InlineCommandComment:
+    return "InlineCommandComment";
+  case CommentKind::CK_HTMLStartTagComment:
+    return "HTMLStartTagComment";
+  case CommentKind::CK_HTMLEndTagComment:
+    return "HTMLEndTagComment";
+  case CommentKind::CK_BlockCommandComment:
+    return "BlockCommandComment";
+  case CommentKind::CK_ParamCommandComment:
+    return "ParamCommandComment";
+  case CommentKind::CK_TParamCommandComment:
+    return "TParamCommandComment";
+  case CommentKind::CK_VerbatimBlockComment:
+    return "VerbatimBlockComment";
+  case CommentKind::CK_VerbatimBlockLineComment:
+    return "VerbatimBlockLineComment";
+  case CommentKind::CK_VerbatimLineComment:
+    return "VerbatimLineComment";
+  case CommentKind::CK_Unknown:
+    return "Unknown";
+  }
+  llvm_unreachable("Unhandled CommentKind");
+}
+
 namespace {
 
 const SymbolID EmptySID = SymbolID();
@@ -87,10 +143,11 @@ mergeInfos(std::vector<std::unique_ptr<Info>> &Values) {
     return reduce<FunctionInfo>(Values);
   case InfoType::IT_typedef:
     return reduce<TypedefInfo>(Values);
-  default:
+  case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "unexpected info type");
   }
+  llvm_unreachable("unhandled enumerator");
 }
 
 bool CommentInfo::operator==(const CommentInfo &Other) const {
@@ -369,9 +426,11 @@ ClangDocContext::ClangDocContext(tooling::ExecutionContext *ECtx,
                                  StringRef OutDirectory, StringRef SourceRoot,
                                  StringRef RepositoryUrl,
                                  StringRef RepositoryLinePrefix, StringRef Base,
-                                 std::vector<std::string> UserStylesheets)
+                                 std::vector<std::string> UserStylesheets,
+                                 bool FTimeTrace)
     : ECtx(ECtx), ProjectName(ProjectName), PublicOnly(PublicOnly),
-      OutDirectory(OutDirectory), UserStylesheets(UserStylesheets), Base(Base) {
+      FTimeTrace(FTimeTrace), OutDirectory(OutDirectory),
+      UserStylesheets(UserStylesheets), Base(Base) {
   llvm::SmallString<128> SourceRootDir(SourceRoot);
   if (SourceRoot.empty())
     // If no SourceRoot was provided the current path is used as the default
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/Representation.h b/external/llvm-project/clang-tools-extra/clang-doc/Representation.h
index a3a6217f76bb..75da50064581 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/Representation.h
+++ b/external/llvm-project/clang-tools-extra/clang-doc/Representation.h
@@ -45,6 +45,25 @@ enum class InfoType {
   IT_typedef
 };
 
+enum class CommentKind {
+  CK_FullComment,
+  CK_ParagraphComment,
+  CK_TextComment,
+  CK_InlineCommandComment,
+  CK_HTMLStartTagComment,
+  CK_HTMLEndTagComment,
+  CK_BlockCommandComment,
+  CK_ParamCommandComment,
+  CK_TParamCommandComment,
+  CK_VerbatimBlockComment,
+  CK_VerbatimBlockLineComment,
+  CK_VerbatimLineComment,
+  CK_Unknown
+};
+
+CommentKind stringToCommentKind(llvm::StringRef KindStr);
+llvm::StringRef commentKindToString(CommentKind Kind);
+
 // A representation of a parsed comment.
 struct CommentInfo {
   CommentInfo() = default;
@@ -60,13 +79,13 @@ struct CommentInfo {
   // the vector.
   bool operator<(const CommentInfo &Other) const;
 
-  // TODO: The Kind field should be an enum, so we can switch on it easily.
-  SmallString<16>
-      Kind; // Kind of comment (FullComment, ParagraphComment, TextComment,
-            // InlineCommandComment, HTMLStartTagComment, HTMLEndTagComment,
-            // BlockCommandComment, ParamCommandComment,
-            // TParamCommandComment, VerbatimBlockComment,
-            // VerbatimBlockLineComment, VerbatimLineComment).
+  CommentKind Kind = CommentKind::
+      CK_Unknown; // Kind of comment (FullComment, ParagraphComment,
+                  // TextComment, InlineCommandComment, HTMLStartTagComment,
+                  // HTMLEndTagComment, BlockCommandComment,
+                  // ParamCommandComment, TParamCommandComment,
+                  // VerbatimBlockComment, VerbatimBlockLineComment,
+                  // VerbatimLineComment).
   SmallString<64> Text;      // Text of the comment.
   SmallString<16> Name;      // Name of the comment (for Verbatim and HTML).
   SmallString<8> Direction;  // Parameter direction (for (T)ParamCommand).
@@ -524,10 +543,13 @@ struct ClangDocContext {
   ClangDocContext(tooling::ExecutionContext *ECtx, StringRef ProjectName,
                   bool PublicOnly, StringRef OutDirectory, StringRef SourceRoot,
                   StringRef RepositoryUrl, StringRef RepositoryCodeLinePrefix,
-                  StringRef Base, std::vector<std::string> UserStylesheets);
+                  StringRef Base, std::vector<std::string> UserStylesheets,
+                  bool FTimeTrace = false);
   tooling::ExecutionContext *ECtx;
   std::string ProjectName; // Name of project clang-doc is documenting.
   bool PublicOnly; // Indicates if only public declarations are documented.
+  bool FTimeTrace; // Indicates if ftime trace is turned on
+  int Granularity; // Granularity of ftime trace
   std::string OutDirectory; // Directory for outputting generated files.
   std::string SourceRoot;   // Directory where processed files are stored. Links
                             // to definition locations will only be generated if
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/Serialize.cpp b/external/llvm-project/clang-tools-extra/clang-doc/Serialize.cpp
index 3932a939de97..820e8bfd8e64 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/Serialize.cpp
@@ -270,7 +270,7 @@ class ClangDocCommentVisitor
 };
 
 void ClangDocCommentVisitor::parseComment(const comments::Comment *C) {
-  CurrentCI.Kind = C->getCommentKindName();
+  CurrentCI.Kind = stringToCommentKind(C->getCommentKindName());
   ConstCommentVisitor<ClangDocCommentVisitor>::visit(C);
   for (comments::Comment *Child :
        llvm::make_range(C->child_begin(), C->child_end())) {
@@ -388,9 +388,11 @@ std::string serialize(std::unique_ptr<Info> &I) {
     return serialize(*static_cast<EnumInfo *>(I.get()));
   case InfoType::IT_function:
     return serialize(*static_cast<FunctionInfo *>(I.get()));
-  default:
+  case InfoType::IT_typedef:
+  case InfoType::IT_default:
     return "";
   }
+  llvm_unreachable("unhandled enumerator");
 }
 
 static void parseFullComment(const FullComment *C, CommentInfo &CI) {
@@ -525,9 +527,13 @@ static std::unique_ptr<Info> makeAndInsertIntoParent(ChildType Child) {
     InsertChild(ParentRec->Children, std::forward<ChildType>(Child));
     return ParentRec;
   }
-  default:
-    llvm_unreachable("Invalid reference type for parent namespace");
+  case InfoType::IT_default:
+  case InfoType::IT_enum:
+  case InfoType::IT_function:
+  case InfoType::IT_typedef:
+    break;
   }
+  llvm_unreachable("Invalid reference type for parent namespace");
 }
 
 // There are two uses for this function.
@@ -742,6 +748,7 @@ static void populateFunctionInfo(FunctionInfo &I, const FunctionDecl *D,
   I.ReturnType = getTypeInfoForType(D->getReturnType(), LO);
   I.Prototype = getFunctionPrototype(D);
   parseParameters(I, D);
+  I.IsStatic = D->isStatic();
 
   populateTemplateParameters(I.Template, D);
 
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/YAMLGenerator.cpp b/external/llvm-project/clang-tools-extra/clang-doc/YAMLGenerator.cpp
index 8c110b34e8e2..897b5d5ae4c9 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/YAMLGenerator.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/YAMLGenerator.cpp
@@ -65,6 +65,34 @@ template <> struct ScalarEnumerationTraits<InfoType> {
   }
 };
 
+template <> struct ScalarEnumerationTraits<clang::doc::CommentKind> {
+  static void enumeration(IO &IO, clang::doc::CommentKind &Value) {
+    IO.enumCase(Value, "FullComment", clang::doc::CommentKind::CK_FullComment);
+    IO.enumCase(Value, "ParagraphComment",
+                clang::doc::CommentKind::CK_ParagraphComment);
+    IO.enumCase(Value, "TextComment", clang::doc::CommentKind::CK_TextComment);
+    IO.enumCase(Value, "InlineCommandComment",
+                clang::doc::CommentKind::CK_InlineCommandComment);
+    IO.enumCase(Value, "HTMLStartTagComment",
+                clang::doc::CommentKind::CK_HTMLStartTagComment);
+    IO.enumCase(Value, "HTMLEndTagComment",
+                clang::doc::CommentKind::CK_HTMLEndTagComment);
+    IO.enumCase(Value, "BlockCommandComment",
+                clang::doc::CommentKind::CK_BlockCommandComment);
+    IO.enumCase(Value, "ParamCommandComment",
+                clang::doc::CommentKind::CK_ParamCommandComment);
+    IO.enumCase(Value, "TParamCommandComment",
+                clang::doc::CommentKind::CK_TParamCommandComment);
+    IO.enumCase(Value, "VerbatimBlockComment",
+                clang::doc::CommentKind::CK_VerbatimBlockComment);
+    IO.enumCase(Value, "VerbatimBlockLineComment",
+                clang::doc::CommentKind::CK_VerbatimBlockLineComment);
+    IO.enumCase(Value, "VerbatimLineComment",
+                clang::doc::CommentKind::CK_VerbatimLineComment);
+    IO.enumCase(Value, "Unknown", clang::doc::CommentKind::CK_Unknown);
+  }
+};
+
 // Scalars to YAML output.
 template <unsigned U> struct ScalarTraits<SmallString<U>> {
 
@@ -149,7 +177,7 @@ static void recordInfoMapping(IO &IO, RecordInfo &I) {
 }
 
 static void commentInfoMapping(IO &IO, CommentInfo &I) {
-  IO.mapOptional("Kind", I.Kind, SmallString<16>());
+  IO.mapOptional("Kind", I.Kind, CommentKind::CK_Unknown);
   IO.mapOptional("Text", I.Text, SmallString<64>());
   IO.mapOptional("Name", I.Name, SmallString<16>());
   IO.mapOptional("Direction", I.Direction, SmallString<8>());
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/assets/namespace-template.mustache b/external/llvm-project/clang-tools-extra/clang-doc/assets/namespace-template.mustache
index 12dc93069d1c..1a44ed3c3ccc 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/assets/namespace-template.mustache
+++ b/external/llvm-project/clang-tools-extra/clang-doc/assets/namespace-template.mustache
@@ -23,23 +23,79 @@
     </head>
     <body>
         <nav class="navbar">
-            Navbar
+            <div class="navbar__container">
+                {{#ProjectName}}
+                <div class="navbar__logo">
+                    {{ProjectName}}
+                </div>
+                {{/ProjectName}}
+                <div class="navbar__menu">
+                    <ul class="navbar__links">
+                        <li class="navbar__item">
+                            <a href="/" class="navbar__link">Namespace</a>
+                        </li>
+                        <li class="navbar__item">
+                            <a href="/" class="navbar__link">Class</a>
+                        </li>
+                    </ul>
+                </div>
+            </div>
         </nav>
         <main>
             <div class="container">
                 <div class="sidebar">
-                        Lorem ipsum dolor sit amet, consectetur adipiscing elit, 
-                        sed do eiusmod tempor incididunt ut labore et dolore magna 
-                        aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco 
-                        laboris nisi ut aliquip ex ea commodo consequat. 
-                        Duis aute irure dolor in reprehenderit in voluptate velit esse 
-                        cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat 
-                        cupidatat non proident, sunt in culpa qui officia deserunt mollit 
-                        anim id est laborum
+                    <h2>{{RecordType}} {{Name}}</h2>
+                    <ul>
+                        {{#Enums}}
+                        <li class="sidebar-section">
+                            <a class="sidebar-item" href="#Enums">Enums</a>
+                        </li>
+                        <ul>
+                            {{#Obj}}
+                            <li class="sidebar-item-container">
+                                <a class="sidebar-item" href="#{{ID}}">{{EnumName}}</a>
+                            </li>
+                            {{/Obj}}
+                        </ul>
+                        {{/Enums}}
+                        {{#Record}}
+                        <li class="sidebar-section">
+                            <a class="sidebar-item" href="#Classes">Inner Classes</a>
+                        </li>
+                        <ul>
+                            {{#Links}}
+                            <li class="sidebar-item-container">
+                                <a class="sidebar-item" href="#{{ID}}">{{Name}}</a>
+                            </li>
+                            {{/Links}}
+                        </ul>
+                        {{/Record}}
+                    </ul>
                 </div>
                 <div class="resizer" id="resizer"></div>
                 <div class="content">
-                    Content
+                    {{#Enums}}
+                    <section id="Enums" class="section-container">
+                        <h2>Enumerations</h2>
+                        <div>
+                            {{#Obj}}
+                            {{>EnumPartial}}
+                            {{/Obj}}
+                        </div>
+                    </section>
+                    {{/Enums}}
+                    {{#Record}}
+                    <section id="Classes" class="section-container">
+                        <h2>Inner Classes</h2>
+                        <ul class="class-container">
+                        {{#Links}}
+                            <li id="{{ID}}" style="max-height: 40px;">
+                                <a href="{{Link}}"><pre><code class="language-cpp code-clang-doc" >class {{Name}}</code></pre></a>
+                            </li>
+                        {{/Links}}
+                        </ul>
+                    </section>
+                    {{/Record}}
                 </div>
             </div>
         </main>
diff --git a/external/llvm-project/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp b/external/llvm-project/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
index 8253ef298db4..3bb67baf6573 100644
--- a/external/llvm-project/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -35,6 +35,7 @@
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <atomic>
 #include <mutex>
@@ -104,7 +105,12 @@ static llvm::cl::opt<std::string> RepositoryCodeLinePrefix(
     llvm::cl::desc("Prefix of line code for repository."),
     llvm::cl::cat(ClangDocCategory));
 
-enum OutputFormatTy { md, yaml, html, mustache };
+static llvm::cl::opt<bool> FTimeTrace("ftime-trace", llvm::cl::desc(R"(
+Turn on time profiler. Generates clang-doc-tracing.json)"),
+                                      llvm::cl::init(false),
+                                      llvm::cl::cat(ClangDocCategory));
+
+enum OutputFormatTy { md, yaml, html, mustache, json };
 
 static llvm::cl::opt<OutputFormatTy> FormatEnum(
     "format", llvm::cl::desc("Format for outputted docs."),
@@ -115,7 +121,9 @@ static llvm::cl::opt<OutputFormatTy> FormatEnum(
                      clEnumValN(OutputFormatTy::html, "html",
                                 "Documentation in HTML format."),
                      clEnumValN(OutputFormatTy::mustache, "mustache",
-                                "Documentation in mustache HTML format")),
+                                "Documentation in mustache HTML format"),
+                     clEnumValN(OutputFormatTy::json, "json",
+                                "Documentation in JSON format")),
     llvm::cl::init(OutputFormatTy::yaml), llvm::cl::cat(ClangDocCategory));
 
 static llvm::ExitOnError ExitOnErr;
@@ -130,6 +138,8 @@ static std::string getFormatString() {
     return "html";
   case OutputFormatTy::mustache:
     return "mustache";
+  case OutputFormatTy::json:
+    return "json";
   }
   llvm_unreachable("Unknown OutputFormatTy");
 }
@@ -261,7 +271,8 @@ static llvm::Error handleMappingFailures(llvm::Error Err) {
 
 static llvm::Error createDirectories(llvm::StringRef OutDirectory) {
   if (std::error_code Err = llvm::sys::fs::create_directories(OutDirectory))
-    return llvm::createFileError(OutDirectory, Err);
+    return llvm::createFileError(OutDirectory, Err,
+                                 "failed to create directory.");
   return llvm::Error::success();
 }
 
@@ -286,115 +297,162 @@ Example usage for a project using a compile commands database:
   auto Executor = ExitOnErr(clang::tooling::createExecutorFromCommandLineArgs(
       argc, argv, ClangDocCategory, Overview));
 
-  // Fail early if an invalid format was provided.
-  std::string Format = getFormatString();
-  llvm::outs() << "Emiting docs in " << Format << " format.\n";
-  auto G = ExitOnErr(doc::findGeneratorByName(Format));
-
-  ArgumentsAdjuster ArgAdjuster;
-  if (!DoxygenOnly)
-    ArgAdjuster = combineAdjusters(
-        getInsertArgumentAdjuster("-fparse-all-comments",
-                                  tooling::ArgumentInsertPosition::END),
-        ArgAdjuster);
-
-  clang::doc::ClangDocContext CDCtx = {
-      Executor->getExecutionContext(),
-      ProjectName,
-      PublicOnly,
-      OutDirectory,
-      SourceRoot,
-      RepositoryUrl,
-      RepositoryCodeLinePrefix,
-      BaseDirectory,
-      {UserStylesheets.begin(), UserStylesheets.end()}};
-
-  if (Format == "html") {
-    ExitOnErr(getHtmlAssetFiles(argv[0], CDCtx));
-  }
-
-  if (Format == "mustache") {
-    ExitOnErr(getMustacheHtmlFiles(argv[0], CDCtx));
-  }
-
-  // Mapping phase
-  llvm::outs() << "Mapping decls...\n";
-  ExitOnErr(handleMappingFailures(
-      Executor->execute(doc::newMapperActionFactory(CDCtx), ArgAdjuster)));
-
-  // Collect values into output by key.
-  // In ToolResults, the Key is the hashed USR and the value is the
-  // bitcode-encoded representation of the Info object.
-  llvm::outs() << "Collecting infos...\n";
-  llvm::StringMap<std::vector<StringRef>> USRToBitcode;
-  Executor->getToolResults()->forEachResult(
-      [&](StringRef Key, StringRef Value) {
-        USRToBitcode[Key].emplace_back(Value);
-      });
-
-  // Collects all Infos according to their unique USR value. This map is added
-  // to from the thread pool below and is protected by the USRToInfoMutex.
-  llvm::sys::Mutex USRToInfoMutex;
-  llvm::StringMap<std::unique_ptr<doc::Info>> USRToInfo;
-
-  // First reducing phase (reduce all decls into one info per decl).
-  llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
-  std::atomic<bool> Error;
-  Error = false;
-  llvm::sys::Mutex IndexMutex;
-  // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
-  llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ExecutorConcurrency));
-  for (auto &Group : USRToBitcode) {
-    Pool.async([&]() {
-      std::vector<std::unique_ptr<doc::Info>> Infos;
-      for (auto &Bitcode : Group.getValue()) {
-        llvm::BitstreamCursor Stream(Bitcode);
-        doc::ClangDocBitcodeReader Reader(Stream);
-        auto ReadInfos = Reader.readBitcode();
-        if (!ReadInfos) {
-          llvm::errs() << toString(ReadInfos.takeError()) << "\n";
-          Error = true;
-          return;
-        }
-        std::move(ReadInfos->begin(), ReadInfos->end(),
-                  std::back_inserter(Infos));
-      }
-
-      auto Reduced = doc::mergeInfos(Infos);
-      if (!Reduced) {
-        llvm::errs() << llvm::toString(Reduced.takeError());
-        return;
-      }
-
-      // Add a reference to this Info in the Index
-      {
-        std::lock_guard<llvm::sys::Mutex> Guard(IndexMutex);
-        clang::doc::Generator::addInfoToIndex(CDCtx.Idx, Reduced.get().get());
-      }
+  // turns on ftime trace profiling
+  if (FTimeTrace)
+    llvm::timeTraceProfilerInitialize(200, "clang-doc");
+  {
+    llvm::TimeTraceScope("main");
+
+    // Fail early if an invalid format was provided.
+    std::string Format = getFormatString();
+    llvm::outs() << "Emiting docs in " << Format << " format.\n";
+    auto G = ExitOnErr(doc::findGeneratorByName(Format));
+
+    ArgumentsAdjuster ArgAdjuster;
+    if (!DoxygenOnly)
+      ArgAdjuster = combineAdjusters(
+          getInsertArgumentAdjuster("-fparse-all-comments",
+                                    tooling::ArgumentInsertPosition::END),
+          ArgAdjuster);
+
+    clang::doc::ClangDocContext CDCtx = {
+        Executor->getExecutionContext(),
+        ProjectName,
+        PublicOnly,
+        OutDirectory,
+        SourceRoot,
+        RepositoryUrl,
+        RepositoryCodeLinePrefix,
+        BaseDirectory,
+        {UserStylesheets.begin(), UserStylesheets.end()},
+        FTimeTrace};
+
+    if (Format == "html") {
+      ExitOnErr(getHtmlAssetFiles(argv[0], CDCtx));
+    } else if (Format == "mustache") {
+      ExitOnErr(getMustacheHtmlFiles(argv[0], CDCtx));
+    }
 
-      // Save in the result map (needs a lock due to threaded access).
-      {
-        std::lock_guard<llvm::sys::Mutex> Guard(USRToInfoMutex);
-        USRToInfo[Group.getKey()] = std::move(Reduced.get());
+    llvm::timeTraceProfilerBegin("Executor Launch", "total runtime");
+    // Mapping phase
+    llvm::outs() << "Mapping decls...\n";
+    ExitOnErr(handleMappingFailures(
+        Executor->execute(doc::newMapperActionFactory(CDCtx), ArgAdjuster)));
+    llvm::timeTraceProfilerEnd();
+
+    // Collect values into output by key.
+    // In ToolResults, the Key is the hashed USR and the value is the
+    // bitcode-encoded representation of the Info object.
+    llvm::timeTraceProfilerBegin("Collect Info", "total runtime");
+    llvm::outs() << "Collecting infos...\n";
+    llvm::StringMap<std::vector<StringRef>> USRToBitcode;
+    Executor->getToolResults()->forEachResult(
+        [&](StringRef Key, StringRef Value) {
+          USRToBitcode[Key].emplace_back(Value);
+        });
+    llvm::timeTraceProfilerEnd();
+
+    // Collects all Infos according to their unique USR value. This map is added
+    // to from the thread pool below and is protected by the USRToInfoMutex.
+    llvm::sys::Mutex USRToInfoMutex;
+    llvm::StringMap<std::unique_ptr<doc::Info>> USRToInfo;
+
+    // First reducing phase (reduce all decls into one info per decl).
+    llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+    std::atomic<bool> Error;
+    Error = false;
+    llvm::sys::Mutex IndexMutex;
+    // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
+    llvm::DefaultThreadPool Pool(
+        llvm::hardware_concurrency(ExecutorConcurrency));
+    {
+      llvm::TimeTraceScope TS("Reduce");
+      for (auto &Group : USRToBitcode) {
+        Pool.async([&]() { // time trace decoding bitcode
+          if (FTimeTrace)
+            llvm::timeTraceProfilerInitialize(200, "clang-doc");
+
+          std::vector<std::unique_ptr<doc::Info>> Infos;
+          {
+            llvm::TimeTraceScope Red("decoding bitcode");
+            for (auto &Bitcode : Group.getValue()) {
+              llvm::BitstreamCursor Stream(Bitcode);
+              doc::ClangDocBitcodeReader Reader(Stream);
+              auto ReadInfos = Reader.readBitcode();
+              if (!ReadInfos) {
+                llvm::errs() << toString(ReadInfos.takeError()) << "\n";
+                Error = true;
+                return;
+              }
+              std::move(ReadInfos->begin(), ReadInfos->end(),
+                        std::back_inserter(Infos));
+            }
+          } // time trace decoding bitcode
+
+          std::unique_ptr<doc::Info> Reduced;
+
+          {
+            llvm::TimeTraceScope Merge("merging bitcode");
+            auto ExpReduced = doc::mergeInfos(Infos);
+
+            if (!ExpReduced) {
+              llvm::errs() << llvm::toString(ExpReduced.takeError());
+              return;
+            }
+            Reduced = std::move(*ExpReduced);
+          } // time trace merging bitcode
+
+          // Add a reference to this Info in the Index
+          {
+            llvm::TimeTraceScope Merge("addInfoToIndex");
+            std::lock_guard<llvm::sys::Mutex> Guard(IndexMutex);
+            clang::doc::Generator::addInfoToIndex(CDCtx.Idx, Reduced.get());
+          }
+          // Save in the result map (needs a lock due to threaded access).
+          {
+            llvm::TimeTraceScope Merge("USRToInfo");
+            std::lock_guard<llvm::sys::Mutex> Guard(USRToInfoMutex);
+            USRToInfo[Group.getKey()] = std::move(Reduced);
+          }
+
+          if (CDCtx.FTimeTrace)
+            llvm::timeTraceProfilerFinishThread();
+        });
       }
-    });
-  }
 
-  Pool.wait();
+      Pool.wait();
+    } // time trace reduce
 
-  if (Error)
-    return 1;
+    if (Error)
+      return 1;
 
-  sortUsrToInfo(USRToInfo);
-
-  // Ensure the root output directory exists.
-  ExitOnErr(createDirectories(OutDirectory));
-
-  // Run the generator.
-  llvm::outs() << "Generating docs...\n";
-  ExitOnErr(G->generateDocs(OutDirectory, std::move(USRToInfo), CDCtx));
-  llvm::outs() << "Generating assets for docs...\n";
-  ExitOnErr(G->createResources(CDCtx));
+    {
+      llvm::TimeTraceScope Sort("Sort USRToInfo");
+      sortUsrToInfo(USRToInfo);
+    }
 
+    llvm::timeTraceProfilerBegin("Writing output", "total runtime");
+    // Ensure the root output directory exists.
+    ExitOnErr(createDirectories(OutDirectory));
+
+    // Run the generator.
+    llvm::outs() << "Generating docs...\n";
+
+    ExitOnErr(G->generateDocs(OutDirectory, std::move(USRToInfo), CDCtx));
+    llvm::outs() << "Generating assets for docs...\n";
+    ExitOnErr(G->createResources(CDCtx));
+    llvm::timeTraceProfilerEnd();
+  } // time trace main
+
+  if (FTimeTrace) {
+    std::error_code EC;
+    llvm::raw_fd_ostream OS("clang-doc-tracing.json", EC,
+                            llvm::sys::fs::OF_Text);
+    if (!EC) {
+      llvm::timeTraceProfilerWrite(OS);
+      llvm::timeTraceProfilerCleanup();
+    } else
+      return 1;
+  }
   return 0;
 }
diff --git a/external/llvm-project/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp b/external/llvm-project/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp
index 746ba7bcea01..9f73f4799805 100644
--- a/external/llvm-project/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-include-fixer/tool/ClangIncludeFixer.cpp
@@ -167,13 +167,11 @@ createSymbolIndexManager(StringRef FilePath) {
     // Parse input and fill the database with it.
     // <symbol>=<header><, header...>
     // Multiple symbols can be given, separated by semicolons.
-    std::map<std::string, std::vector<std::string>> SymbolsMap;
     SmallVector<StringRef, 4> SemicolonSplits;
     StringRef(Input).split(SemicolonSplits, ";");
     std::vector<find_all_symbols::SymbolAndSignals> Symbols;
     for (StringRef Pair : SemicolonSplits) {
       auto Split = Pair.split('=');
-      std::vector<std::string> Headers;
       SmallVector<StringRef, 4> CommaSplits;
       Split.second.split(CommaSplits, ",");
       for (size_t I = 0, E = CommaSplits.size(); I != E; ++I)
diff --git a/external/llvm-project/clang-tools-extra/clang-move/Move.h b/external/llvm-project/clang-tools-extra/clang-move/Move.h
index ea241bbbc4f8..ba613abc03e1 100644
--- a/external/llvm-project/clang-tools-extra/clang-move/Move.h
+++ b/external/llvm-project/clang-tools-extra/clang-move/Move.h
@@ -49,9 +49,7 @@ class DeclarationReporter {
     bool Templated = false;    // Whether the declaration is templated.
   };
 
-  const std::vector<Declaration> getDeclarationList() const {
-    return DeclarationList;
-  }
+  ArrayRef<Declaration> getDeclarationList() const { return DeclarationList; }
 
 private:
   std::vector<Declaration> DeclarationList;
diff --git a/external/llvm-project/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp b/external/llvm-project/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp
index ea0207619fb2..3b1cd18d8034 100644
--- a/external/llvm-project/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-reorder-fields/ReorderFieldsAction.cpp
@@ -86,6 +86,10 @@ getNewFieldsOrder(const RecordDecl *Definition,
 static void
 addReplacement(SourceRange Old, SourceRange New, const ASTContext &Context,
                std::map<std::string, tooling::Replacements> &Replacements) {
+  if (Old.getBegin().isMacroID())
+    Old = Context.getSourceManager().getExpansionRange(Old).getAsRange();
+  if (New.getBegin().isMacroID())
+    New = Context.getSourceManager().getExpansionRange(New).getAsRange();
   StringRef NewText =
       Lexer::getSourceText(CharSourceRange::getTokenRange(New),
                            Context.getSourceManager(), Context.getLangOpts());
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidy.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 38d58bc13774..f4ab93b51f4a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -419,8 +419,8 @@ ClangTidyASTConsumerFactory::createASTConsumer(
 
   std::unique_ptr<ClangTidyProfiling> Profiling;
   if (Context.getEnableProfiling()) {
-    Profiling = std::make_unique<ClangTidyProfiling>(
-        Context.getProfileStorageParams());
+    Profiling =
+        std::make_unique<ClangTidyProfiling>(Context.getProfileStorageParams());
     FinderOptions.CheckProfiling.emplace(Profiling->Records);
   }
 
@@ -432,8 +432,8 @@ ClangTidyASTConsumerFactory::createASTConsumer(
 
   if (Context.canEnableModuleHeadersParsing() &&
       Context.getLangOpts().Modules && OverlayFS != nullptr) {
-    auto ModuleExpander = std::make_unique<ExpandModularHeadersPPCallbacks>(
-        &Compiler, OverlayFS);
+    auto ModuleExpander =
+        std::make_unique<ExpandModularHeadersPPCallbacks>(&Compiler, OverlayFS);
     ModuleExpanderPP = ModuleExpander->getPreprocessor();
     PP->addPPCallbacks(std::move(ModuleExpander));
   }
@@ -497,7 +497,7 @@ getCheckNames(const ClangTidyOptions &Options,
               bool AllowEnablingAnalyzerAlphaCheckers) {
   clang::tidy::ClangTidyContext Context(
       std::make_unique<DefaultOptionsProvider>(ClangTidyGlobalOptions(),
-                                                Options),
+                                               Options),
       AllowEnablingAnalyzerAlphaCheckers);
   ClangTidyASTConsumerFactory Factory(Context);
   return Factory.getCheckNames();
@@ -508,7 +508,7 @@ getCheckOptions(const ClangTidyOptions &Options,
                 bool AllowEnablingAnalyzerAlphaCheckers) {
   clang::tidy::ClangTidyContext Context(
       std::make_unique<DefaultOptionsProvider>(ClangTidyGlobalOptions(),
-                                                Options),
+                                               Options),
       AllowEnablingAnalyzerAlphaCheckers);
   ClangTidyDiagnosticConsumer DiagConsumer(Context);
   auto DiagOpts = std::make_unique<DiagnosticOptions>();
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyCheck.h
index 037526a0bd9a..399d45911549 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyCheck.h
@@ -530,7 +530,6 @@ void ClangTidyCheck::OptionsView::store<bool>(
     ClangTidyOptions::OptionMap &Options, StringRef LocalName,
     bool Value) const;
 
-
 } // namespace tidy
 } // namespace clang
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index bd7a1bf2c11c..a8851e794f24 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -217,11 +217,10 @@ class ClangTidyContext {
   using DiagLevelAndFormatString = std::pair<DiagnosticIDs::Level, std::string>;
   DiagLevelAndFormatString getDiagLevelAndFormatString(unsigned DiagnosticID,
                                                        SourceLocation Loc) {
-    return {
-        static_cast<DiagnosticIDs::Level>(
-            DiagEngine->getDiagnosticLevel(DiagnosticID, Loc)),
-        std::string(
-            DiagEngine->getDiagnosticIDs()->getDescription(DiagnosticID))};
+    return {static_cast<DiagnosticIDs::Level>(
+                DiagEngine->getDiagnosticLevel(DiagnosticID, Loc)),
+            std::string(
+                DiagEngine->getDiagnosticIDs()->getDescription(DiagnosticID))};
   }
 
   void setOptionsCollector(llvm::StringSet<> *Collector) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
index 1c480d107ae9..e59f157b468b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
@@ -70,7 +70,8 @@ struct NOptionMap {
   NOptionMap(IO &, const ClangTidyOptions::OptionMap &OptionMap) {
     Options.reserve(OptionMap.size());
     for (const auto &KeyValue : OptionMap)
-      Options.emplace_back(std::string(KeyValue.getKey()), KeyValue.getValue().Value);
+      Options.emplace_back(std::string(KeyValue.getKey()),
+                           KeyValue.getValue().Value);
   }
   ClangTidyOptions::OptionMap denormalize(IO &) {
     ClangTidyOptions::OptionMap Map;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.h b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.h
index dd78c570d25d..6ddc5f9b9cf9 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/ClangTidyOptions.h
@@ -204,7 +204,9 @@ class FileOptionsBaseProvider : public DefaultOptionsProvider {
 protected:
   // A pair of configuration file base name and a function parsing
   // configuration from text in the corresponding format.
-  using ConfigFileHandler = std::pair<std::string, std::function<llvm::ErrorOr<ClangTidyOptions> (llvm::MemoryBufferRef)>>;
+  using ConfigFileHandler =
+      std::pair<std::string, std::function<llvm::ErrorOr<ClangTidyOptions>(
+                                 llvm::MemoryBufferRef)>>;
 
   /// Configuration file handlers listed in the order of priority.
   ///
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp
index 6a84704434c3..2c17cd3b6e97 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp
@@ -49,8 +49,8 @@ class ExpandModularHeadersPPCallbacks::FileRecorder {
     FilesToRecord.erase(File);
   }
 
-  /// Makes sure we have contents for all the files we were interested in. Ideally
-  /// `FilesToRecord` should be empty.
+  /// Makes sure we have contents for all the files we were interested in.
+  /// Ideally `FilesToRecord` should be empty.
   void checkAllFilesRecorded() {
     LLVM_DEBUG({
       for (auto FileEntry : FilesToRecord)
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h b/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h
index c3478917ef49..e599bda92c25 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h
@@ -35,10 +35,10 @@ namespace tooling {
 /// including the contents of the modular headers and all their transitive
 /// includes.
 ///
-/// This allows existing tools based on PPCallbacks to retain their functionality
-/// when running with C++ modules enabled. This only works in the backwards
-/// compatible modules mode, i.e. when code can still be parsed in non-modular
-/// way.
+/// This allows existing tools based on PPCallbacks to retain their
+/// functionality when running with C++ modules enabled. This only works in the
+/// backwards compatible modules mode, i.e. when code can still be parsed in
+/// non-modular way.
 class ExpandModularHeadersPPCallbacks : public PPCallbacks {
 public:
   ExpandModularHeadersPPCallbacks(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/AbseilTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/AbseilTidyModule.cpp
index 09a09e6c599b..78605d59b442 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/AbseilTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/AbseilTidyModule.cpp
@@ -59,14 +59,12 @@ class AbseilModule : public ClangTidyModule {
     CheckFactories.registerCheck<NoNamespaceCheck>("abseil-no-namespace");
     CheckFactories.registerCheck<RedundantStrcatCallsCheck>(
         "abseil-redundant-strcat-calls");
-    CheckFactories.registerCheck<StrCatAppendCheck>(
-        "abseil-str-cat-append");
+    CheckFactories.registerCheck<StrCatAppendCheck>("abseil-str-cat-append");
     CheckFactories.registerCheck<StringFindStartswithCheck>(
         "abseil-string-find-startswith");
     CheckFactories.registerCheck<StringFindStrContainsCheck>(
         "abseil-string-find-str-contains");
-    CheckFactories.registerCheck<TimeComparisonCheck>(
-        "abseil-time-comparison");
+    CheckFactories.registerCheck<TimeComparisonCheck>("abseil-time-comparison");
     CheckFactories.registerCheck<TimeSubtractionCheck>(
         "abseil-time-subtraction");
     CheckFactories.registerCheck<UpgradeDurationConversionsCheck>(
@@ -82,6 +80,6 @@ static ClangTidyModuleRegistry::Add<AbseilModule> X("abseil-module",
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the AbseilModule.
-volatile int AbseilModuleAnchorSource = 0;
+volatile int AbseilModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp
index 0819642be055..8063fc540cce 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/CleanupCtadCheck.cpp
@@ -18,7 +18,7 @@ using namespace ::clang::transformer;
 
 namespace clang::tidy::abseil {
 
-RewriteRuleWith<std::string> cleanupCtadCheckImpl() {
+static RewriteRuleWith<std::string> cleanupCtadCheckImpl() {
   auto WarningMessage = cat("prefer absl::Cleanup's class template argument "
                             "deduction pattern in C++17 and higher");
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h
index 6b194e3d2664..c8f259521b64 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.h
@@ -15,7 +15,7 @@ namespace clang::tidy::abseil {
 
 // Find potential incorrect uses of integer division of absl::Duration objects.
 //
-// For the user-facing documentation see: 
+// For the user-facing documentation see:
 // http://clang.llvm.org/extra/clang-tidy/checks/abseil/duration-division.html
 
 class DurationDivisionCheck : public ClangTidyCheck {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.cpp
index edaa584a4d8e..19409d0616f0 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.cpp
@@ -27,7 +27,8 @@ void NoInternalDependenciesCheck::registerMatchers(MatchFinder *Finder) {
       this);
 }
 
-void NoInternalDependenciesCheck::check(const MatchFinder::MatchResult &Result) {
+void NoInternalDependenciesCheck::check(
+    const MatchFinder::MatchResult &Result) {
   const auto *InternalDependency =
       Result.Nodes.getNodeAs<NestedNameSpecifierLoc>("InternalDep");
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h
index dfaa2cbf8f90..9c16524ff6b5 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoInternalDependenciesCheck.h
@@ -1,4 +1,4 @@
-//===--- NoInternalDependenciesCheck.h - clang-tidy----------------------*- C++ -*-===//
+//===--- NoInternalDependenciesCheck.h - clang-tidy--------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.cpp
index b45a9c99d780..6c2baa4c4141 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/NoNamespaceCheck.cpp
@@ -15,10 +15,9 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::abseil {
 
 void NoNamespaceCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(
-      namespaceDecl(hasName("::absl"), unless(isInAbseilFile()))
-          .bind("abslNamespace"),
-      this);
+  Finder->addMatcher(namespaceDecl(hasName("::absl"), unless(isInAbseilFile()))
+                         .bind("abslNamespace"),
+                     this);
 }
 
 void NoNamespaceCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp
index 04ca00614350..ef26a8a76cb3 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp
@@ -23,7 +23,7 @@ namespace clang::tidy::abseil {
 //  - Make it work in macros if the outer and inner StrCats are both in the
 //    argument.
 
-void RedundantStrcatCallsCheck::registerMatchers(MatchFinder* Finder) {
+void RedundantStrcatCallsCheck::registerMatchers(MatchFinder *Finder) {
   const auto CallToStrcat =
       callExpr(callee(functionDecl(hasName("::absl::StrCat"))));
   const auto CallToStrappend =
@@ -62,7 +62,7 @@ const clang::CallExpr *processArgument(const Expr *Arg,
                                        const MatchFinder::MatchResult &Result,
                                        StrCatCheckResult *CheckResult) {
   const auto IsAlphanum = hasDeclaration(cxxMethodDecl(hasName("AlphaNum")));
-  static const auto* const Strcat = new auto(hasName("::absl::StrCat"));
+  static const auto *const Strcat = new auto(hasName("::absl::StrCat"));
   const auto IsStrcat = cxxBindTemporaryExpr(
       has(callExpr(callee(functionDecl(*Strcat))).bind("StrCat")));
   if (const auto *SubStrcatCall = selectFirst<const CallExpr>(
@@ -81,18 +81,18 @@ const clang::CallExpr *processArgument(const Expr *Arg,
 StrCatCheckResult processCall(const CallExpr *RootCall, bool IsAppend,
                               const MatchFinder::MatchResult &Result) {
   StrCatCheckResult CheckResult;
-  std::deque<const CallExpr*> CallsToProcess = {RootCall};
+  std::deque<const CallExpr *> CallsToProcess = {RootCall};
 
   while (!CallsToProcess.empty()) {
     ++CheckResult.NumCalls;
 
-    const CallExpr* CallExpr = CallsToProcess.front();
+    const CallExpr *CallExpr = CallsToProcess.front();
     CallsToProcess.pop_front();
 
     int StartArg = CallExpr == RootCall && IsAppend;
     for (const auto *Arg : CallExpr->arguments()) {
-      if (StartArg-- > 0) 
-      	continue;
+      if (StartArg-- > 0)
+        continue;
       if (const clang::CallExpr *Sub =
               processArgument(Arg, Result, &CheckResult)) {
         CallsToProcess.push_back(Sub);
@@ -101,18 +101,18 @@ StrCatCheckResult processCall(const CallExpr *RootCall, bool IsAppend,
   }
   return CheckResult;
 }
-}  // namespace
+} // namespace
 
-void RedundantStrcatCallsCheck::check(const MatchFinder::MatchResult& Result) {
+void RedundantStrcatCallsCheck::check(const MatchFinder::MatchResult &Result) {
   bool IsAppend = false;
 
   const CallExpr *RootCall = nullptr;
-  if ((RootCall = Result.Nodes.getNodeAs<CallExpr>("StrCat"))) 
-  	IsAppend = false;
-  else if ((RootCall = Result.Nodes.getNodeAs<CallExpr>("StrAppend"))) 
-  	IsAppend = true;
-  else 
-  	return;
+  if ((RootCall = Result.Nodes.getNodeAs<CallExpr>("StrCat")))
+    IsAppend = false;
+  else if ((RootCall = Result.Nodes.getNodeAs<CallExpr>("StrAppend")))
+    IsAppend = true;
+  else
+    return;
 
   if (RootCall->getBeginLoc().isMacroID()) {
     // Ignore calls within macros.
@@ -128,8 +128,8 @@ void RedundantStrcatCallsCheck::check(const MatchFinder::MatchResult& Result) {
     return;
   }
 
-  diag(RootCall->getBeginLoc(), 
-  	   "multiple calls to 'absl::StrCat' can be flattened into a single call")
+  diag(RootCall->getBeginLoc(),
+       "multiple calls to 'absl::StrCat' can be flattened into a single call")
       << CheckResult.Hints;
 }
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h
index 04c7c9cb4b78..a27899d3e821 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.h
@@ -13,8 +13,8 @@
 
 namespace clang::tidy::abseil {
 
-/// Flags redundant calls to absl::StrCat when the result is being passed to 
-///	another call of absl::StrCat/absl::StrAppend. Also suggests a fix to 
+/// Flags redundant calls to absl::StrCat when the result is being passed to
+///	another call of absl::StrCat/absl::StrAppend. Also suggests a fix to
 ///	collapse the calls.
 /// Example:
 ///   StrCat(1, StrCat(2, 3))  ==>  StrCat(1, 2, 3)
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp
index ab6ed701e59f..ced92590be02 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp
@@ -17,7 +17,7 @@ namespace clang::tidy::abseil {
 namespace {
 // Skips any combination of temporary materialization, temporary binding and
 // implicit casting.
-AST_MATCHER_P(Stmt, IgnoringTemporaries, ast_matchers::internal::Matcher<Stmt>,
+AST_MATCHER_P(Stmt, ignoringTemporaries, ast_matchers::internal::Matcher<Stmt>,
               InnerMatcher) {
   const Stmt *E = &Node;
   while (true) {
@@ -34,16 +34,16 @@ AST_MATCHER_P(Stmt, IgnoringTemporaries, ast_matchers::internal::Matcher<Stmt>,
   return InnerMatcher.matches(*E, Finder, Builder);
 }
 
-}  // namespace
+} // namespace
 
 // TODO: str += StrCat(...)
 //       str.append(StrCat(...))
 
 void StrCatAppendCheck::registerMatchers(MatchFinder *Finder) {
   const auto StrCat = functionDecl(hasName("::absl::StrCat"));
-  // The arguments of absl::StrCat are implicitly converted to AlphaNum. This 
-  // matches to the arguments because of that behavior. 
-  const auto AlphaNum = IgnoringTemporaries(cxxConstructExpr(
+  // The arguments of absl::StrCat are implicitly converted to AlphaNum. This
+  // matches to the arguments because of that behavior.
+  const auto AlphaNum = ignoringTemporaries(cxxConstructExpr(
       argumentCountIs(1), hasType(cxxRecordDecl(hasName("::absl::AlphaNum"))),
       hasArgument(0, ignoringImpCasts(declRefExpr(to(equalsBoundNode("LHS")),
                                                   expr().bind("Arg0"))))));
@@ -62,7 +62,7 @@ void StrCatAppendCheck::registerMatchers(MatchFinder *Finder) {
                    hasOverloadedOperatorName("="),
                    hasArgument(0, declRefExpr(to(decl().bind("LHS")))),
                    hasArgument(
-                       1, IgnoringTemporaries(
+                       1, ignoringTemporaries(
                               callExpr(callee(StrCat), hasArgument(0, AlphaNum),
                                        unless(HasAnotherReferenceToLhs))
                                   .bind("Call"))))
@@ -73,7 +73,7 @@ void StrCatAppendCheck::registerMatchers(MatchFinder *Finder) {
 void StrCatAppendCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *Op = Result.Nodes.getNodeAs<CXXOperatorCallExpr>("Op");
   const auto *Call = Result.Nodes.getNodeAs<CallExpr>("Call");
-  assert(Op != nullptr && Call != nullptr && "Matcher does not work as expected");
+  assert(Op && Call && "Matcher does not work as expected");
 
   // Handles the case 'x = absl::StrCat(x)', which has no effect.
   if (Call->getNumArgs() == 1) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h
index 22b55abfd83b..fcd9d4b6e188 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.h
@@ -14,7 +14,7 @@
 namespace clang::tidy::abseil {
 
 /// Flags uses of absl::StrCat to append to a string. Suggests absl::StrAppend
-/// should be used instead. 
+/// should be used instead.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/abseil/str-cat-append.html
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
index 6e89783bb51e..0c2fe285ce06 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
@@ -29,7 +29,9 @@ using ::clang::transformer::makeRule;
 using ::clang::transformer::node;
 using ::clang::transformer::RewriteRuleWith;
 
+namespace {
 AST_MATCHER(Type, isCharType) { return Node.isCharType(); }
+} // namespace
 
 static const char DefaultStringLikeClasses[] = "::std::basic_string;"
                                                "::std::basic_string_view;"
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
index 7e7dc7351181..c947f6bca7f3 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
@@ -26,7 +26,7 @@ class TimeSubtractionCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
 private:
-  void emitDiagnostic(const Expr* Node, llvm::StringRef Replacement);
+  void emitDiagnostic(const Expr *Node, llvm::StringRef Replacement);
 };
 
 } // namespace clang::tidy::abseil
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
index beb5479aa6aa..8e7d9829533d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h
@@ -16,7 +16,8 @@
 
 namespace clang::tidy::abseil {
 
-/// Finds deprecated uses of `absl::Duration` arithmetic operators and factories.
+/// Finds deprecated uses of `absl::Duration` arithmetic operators and
+/// factories.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/abseil/upgrade-duration-conversions.html
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp
index 21610c7d1ece..02a43ba86d7b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp
@@ -43,6 +43,6 @@ static ClangTidyModuleRegistry::Add<altera::AlteraModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the AlteraModule.
-volatile int AlteraModuleAnchorSource = 0;
+volatile int AlteraModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/altera/SingleWorkItemBarrierCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/altera/SingleWorkItemBarrierCheck.cpp
index c5da66a1f28b..c21b7cab1b8d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/altera/SingleWorkItemBarrierCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/altera/SingleWorkItemBarrierCheck.cpp
@@ -16,14 +16,14 @@ namespace clang::tidy::altera {
 
 void SingleWorkItemBarrierCheck::registerMatchers(MatchFinder *Finder) {
   // Find any function that calls barrier but does not call an ID function.
-  // hasAttr(attr::Kind::OpenCLKernel) restricts it to only kernel functions.
+  // hasAttr(attr::Kind::DeviceKernel) restricts it to only kernel functions.
   // FIXME: Have it accept all functions but check for a parameter that gets an
   // ID from one of the four ID functions.
   Finder->addMatcher(
       // Find function declarations...
       functionDecl(
-          // That are OpenCL kernels...
-          hasAttr(attr::Kind::OpenCLKernel),
+          // That are device kernels...
+          hasAttr(attr::Kind::DeviceKernel),
           // And call a barrier function (either 1.x or 2.x version)...
           forEachDescendant(callExpr(callee(functionDecl(hasAnyName(
                                          "barrier", "work_group_barrier"))))
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.cpp
index cdaedd440959..789327b196ab 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.cpp
@@ -47,7 +47,7 @@ void StructPackAlignCheck::check(const MatchFinder::MatchResult &Result) {
   // Do not trigger on templated struct declarations because the packing and
   // alignment requirements are unknown.
   if (Struct->isTemplated())
-     return;
+    return;
 
   // Packing and alignment requirements for invalid decls are meaningless.
   if (Struct->isInvalidDecl())
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.h
index 5edfd40ef77b..f360bcef4f14 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/altera/StructPackAlignCheck.h
@@ -22,7 +22,7 @@ class StructPackAlignCheck : public ClangTidyCheck {
 public:
   StructPackAlignCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context),
-    MaxConfiguredAlignment(Options.get("MaxConfiguredAlignment", 128)) {}
+        MaxConfiguredAlignment(Options.get("MaxConfiguredAlignment", 128)) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp
index 532a43bad741..0bb9d6e4a7ce 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp
@@ -247,8 +247,8 @@ bool UnrollLoopsCheck::extractValue(int &Value, const BinaryOperator *Op,
   return true;
 }
 
-bool UnrollLoopsCheck::exprHasLargeNumIterations(const Expr *Expression,
-                                                 const ASTContext *Context) const {
+bool UnrollLoopsCheck::exprHasLargeNumIterations(
+    const Expr *Expression, const ASTContext *Context) const {
   Expr::EvalResult Result;
   if (Expression->EvaluateAsRValue(Result, *Context)) {
     if (!Result.Val.isInt())
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/AndroidTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/AndroidTidyModule.cpp
index 4541e926e886..17efa10909d0 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/AndroidTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/AndroidTidyModule.cpp
@@ -34,7 +34,8 @@ namespace android {
 class AndroidModule : public ClangTidyModule {
 public:
   void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
-    CheckFactories.registerCheck<CloexecAccept4Check>("android-cloexec-accept4");
+    CheckFactories.registerCheck<CloexecAccept4Check>(
+        "android-cloexec-accept4");
     CheckFactories.registerCheck<CloexecAcceptCheck>("android-cloexec-accept");
     CheckFactories.registerCheck<CloexecCreatCheck>("android-cloexec-creat");
     CheckFactories.registerCheck<CloexecDupCheck>("android-cloexec-dup");
@@ -66,6 +67,6 @@ static ClangTidyModuleRegistry::Add<AndroidModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the AndroidModule.
-volatile int AndroidModuleAnchorSource = 0;
+volatile int AndroidModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecAcceptCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecAcceptCheck.cpp
index 8c34ecd6945c..c90fc7ba1bb0 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecAcceptCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecAcceptCheck.cpp
@@ -26,11 +26,10 @@ void CloexecAcceptCheck::registerMatchers(MatchFinder *Finder) {
 }
 
 void CloexecAcceptCheck::check(const MatchFinder::MatchResult &Result) {
-  std::string ReplacementText =
-      (Twine("accept4(") + getSpellingArg(Result, 0) + ", " +
-       getSpellingArg(Result, 1) + ", " + getSpellingArg(Result, 2) +
-       ", SOCK_CLOEXEC)")
-          .str();
+  std::string ReplacementText = (Twine("accept4(") + getSpellingArg(Result, 0) +
+                                 ", " + getSpellingArg(Result, 1) + ", " +
+                                 getSpellingArg(Result, 2) + ", SOCK_CLOEXEC)")
+                                    .str();
 
   replaceFunc(
       Result,
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCheck.cpp
index 39a0d74f3836..f4d657a7f4e9 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCheck.cpp
@@ -36,7 +36,7 @@ std::string buildFixMsgForStringFlag(const Expr *Arg, const SourceManager &SM,
 
 const char *CloexecCheck::FuncDeclBindingStr = "funcDecl";
 
-const char *CloexecCheck::FuncBindingStr ="func";
+const char *CloexecCheck::FuncBindingStr = "func";
 
 void CloexecCheck::registerMatchersImpl(
     MatchFinder *Finder, internal::Matcher<FunctionDecl> Function) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCreatCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCreatCheck.cpp
index ee270209e4fa..8d9f45c3567f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCreatCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecCreatCheck.cpp
@@ -16,11 +16,10 @@ namespace clang::tidy::android {
 void CloexecCreatCheck::registerMatchers(MatchFinder *Finder) {
   auto CharPointerType = hasType(pointerType(pointee(isAnyCharacter())));
   auto MODETType = hasType(namedDecl(hasName("mode_t")));
-  registerMatchersImpl(Finder,
-                       functionDecl(isExternC(), returns(isInteger()),
-                                    hasName("creat"),
-                                    hasParameter(0, CharPointerType),
-                                    hasParameter(1, MODETType)));
+  registerMatchersImpl(Finder, functionDecl(isExternC(), returns(isInteger()),
+                                            hasName("creat"),
+                                            hasParameter(0, CharPointerType),
+                                            hasParameter(1, MODETType)));
 }
 
 void CloexecCreatCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecFopenCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecFopenCheck.cpp
index 153e7af82d15..bb9d0d2cb3da 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecFopenCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecFopenCheck.cpp
@@ -15,11 +15,10 @@ namespace clang::tidy::android {
 
 void CloexecFopenCheck::registerMatchers(MatchFinder *Finder) {
   auto CharPointerType = hasType(pointerType(pointee(isAnyCharacter())));
-  registerMatchersImpl(Finder,
-                       functionDecl(isExternC(), returns(asString("FILE *")),
-                                    hasName("fopen"),
-                                    hasParameter(0, CharPointerType),
-                                    hasParameter(1, CharPointerType)));
+  registerMatchersImpl(
+      Finder, functionDecl(isExternC(), returns(asString("FILE *")),
+                           hasName("fopen"), hasParameter(0, CharPointerType),
+                           hasParameter(1, CharPointerType)));
 }
 
 void CloexecFopenCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecOpenCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecOpenCheck.cpp
index 11807f95def4..623b6ab02e7b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecOpenCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecOpenCheck.cpp
@@ -20,12 +20,11 @@ void CloexecOpenCheck::registerMatchers(MatchFinder *Finder) {
                                     hasAnyName("open", "open64"),
                                     hasParameter(0, CharPointerType),
                                     hasParameter(1, hasType(isInteger()))));
-  registerMatchersImpl(Finder,
-                       functionDecl(isExternC(), returns(isInteger()),
-                                    hasName("openat"),
-                                    hasParameter(0, hasType(isInteger())),
-                                    hasParameter(1, CharPointerType),
-                                    hasParameter(2, hasType(isInteger()))));
+  registerMatchersImpl(
+      Finder, functionDecl(isExternC(), returns(isInteger()), hasName("openat"),
+                           hasParameter(0, hasType(isInteger())),
+                           hasParameter(1, CharPointerType),
+                           hasParameter(2, hasType(isInteger()))));
 }
 
 void CloexecOpenCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipe2Check.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipe2Check.cpp
index 2798c80146e9..e32332bdfc95 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipe2Check.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipe2Check.cpp
@@ -14,10 +14,10 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::android {
 
 void CloexecPipe2Check::registerMatchers(MatchFinder *Finder) {
-  registerMatchersImpl(Finder,
-                       functionDecl(returns(isInteger()), hasName("pipe2"),
-                                    hasParameter(0, hasType(pointsTo(isInteger()))),
-                                    hasParameter(1, hasType(isInteger()))));
+  registerMatchersImpl(
+      Finder, functionDecl(returns(isInteger()), hasName("pipe2"),
+                           hasParameter(0, hasType(pointsTo(isInteger()))),
+                           hasParameter(1, hasType(isInteger()))));
 }
 
 void CloexecPipe2Check::check(const MatchFinder::MatchResult &Result) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipeCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipeCheck.cpp
index 9085be67b755..c59b127dc87a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipeCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecPipeCheck.cpp
@@ -14,19 +14,19 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::android {
 
 void CloexecPipeCheck::registerMatchers(MatchFinder *Finder) {
-  registerMatchersImpl(Finder,
-                       functionDecl(returns(isInteger()), hasName("pipe"),
-                                    hasParameter(0, hasType(pointsTo(isInteger())))));
+  registerMatchersImpl(
+      Finder, functionDecl(returns(isInteger()), hasName("pipe"),
+                           hasParameter(0, hasType(pointsTo(isInteger())))));
 }
 
 void CloexecPipeCheck::check(const MatchFinder::MatchResult &Result) {
   std::string ReplacementText =
       (Twine("pipe2(") + getSpellingArg(Result, 0) + ", O_CLOEXEC)").str();
 
-  replaceFunc(
-      Result,
-      "prefer pipe2() with O_CLOEXEC to avoid leaking file descriptors to child processes",
-      ReplacementText);
+  replaceFunc(Result,
+              "prefer pipe2() with O_CLOEXEC to avoid leaking file descriptors "
+              "to child processes",
+              ReplacementText);
 }
 
 } // namespace clang::tidy::android
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecSocketCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecSocketCheck.cpp
index 65c6a82036cc..12b31a050c2c 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecSocketCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/android/CloexecSocketCheck.cpp
@@ -14,12 +14,11 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::android {
 
 void CloexecSocketCheck::registerMatchers(MatchFinder *Finder) {
-  registerMatchersImpl(Finder,
-                       functionDecl(isExternC(), returns(isInteger()),
-                                    hasName("socket"),
-                                    hasParameter(0, hasType(isInteger())),
-                                    hasParameter(1, hasType(isInteger())),
-                                    hasParameter(2, hasType(isInteger()))));
+  registerMatchersImpl(
+      Finder, functionDecl(isExternC(), returns(isInteger()), hasName("socket"),
+                           hasParameter(0, hasType(isInteger())),
+                           hasParameter(1, hasType(isInteger())),
+                           hasParameter(2, hasType(isInteger()))));
 }
 
 void CloexecSocketCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/boost/BoostTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/boost/BoostTidyModule.cpp
index 79d0e380e402..f414fe750d02 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/boost/BoostTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/boost/BoostTidyModule.cpp
@@ -32,6 +32,6 @@ static ClangTidyModuleRegistry::Add<BoostModule> X("boost-module",
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the BoostModule.
-volatile int BoostModuleAnchorSource = 0;
+volatile int BoostModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
index 0b76cfb2ad8d..a6cd68edda55 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BranchCloneCheck.cpp
@@ -62,6 +62,7 @@ static bool isFallthroughSwitchBranch(const SwitchBranch &Branch) {
       return true; // Ignore sub-switches
     }
 
+    // NOLINTNEXTLINE(readability-identifier-naming) - FIXME
     bool TraverseSwitchCase(SwitchCase *, DataRecursionQueue * = nullptr) {
       return true; // Ignore cases
     }
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
index 64f4a524daf0..ed1fd138d8f1 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
@@ -291,6 +291,7 @@ static ClangTidyModuleRegistry::Add<bugprone::BugproneModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the BugproneModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int BugproneModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.cpp
index c6bb5bdf5ce9..7378f1a24ffd 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.cpp
@@ -34,8 +34,7 @@ AST_MATCHER(BinaryOperator,
 
 AST_MATCHER(CXXOperatorCallExpr,
             hasCppOperatorAChildComparisonOperatorWithoutParen) {
-  return std::any_of(Node.arg_begin(), Node.arg_end(),
-                     isExprAComparisonOperator);
+  return llvm::any_of(Node.arguments(), isExprAComparisonOperator);
 }
 
 struct ChainedComparisonData {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp
index 93f35cb2c1a3..3fe028b94771 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp
@@ -15,6 +15,8 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::bugprone {
 
+namespace {
+
 AST_MATCHER(clang::VarDecl, hasConstantDeclaration) {
   const Expr *Init = Node.getInit();
   if (Init && !Init->isValueDependent()) {
@@ -25,6 +27,8 @@ AST_MATCHER(clang::VarDecl, hasConstantDeclaration) {
   return false;
 }
 
+} // namespace
+
 DynamicStaticInitializersCheck::DynamicStaticInitializersCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
@@ -36,16 +40,18 @@ void DynamicStaticInitializersCheck::registerMatchers(MatchFinder *Finder) {
       this);
 }
 
-void DynamicStaticInitializersCheck::check(const MatchFinder::MatchResult &Result) {
+void DynamicStaticInitializersCheck::check(
+    const MatchFinder::MatchResult &Result) {
   const auto *Var = Result.Nodes.getNodeAs<VarDecl>("var");
   SourceLocation Loc = Var->getLocation();
-  if (!Loc.isValid() || !utils::isPresumedLocInHeaderFile(Loc, *Result.SourceManager,
-                                                          HeaderFileExtensions))
+  if (!Loc.isValid() || !utils::isPresumedLocInHeaderFile(
+                            Loc, *Result.SourceManager, HeaderFileExtensions))
     return;
   // If the initializer is a constant expression, then the compiler
   // doesn't have to dynamically initialize it.
-  diag(Loc, "static variable %0 may be dynamically initialized in this header file")
-    << Var;
+  diag(Loc,
+       "static variable %0 may be dynamically initialized in this header file")
+      << Var;
 }
 
 } // namespace clang::tidy::bugprone
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
index 10868129e76d..a179d4bf66b4 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
@@ -1497,11 +1497,13 @@ static MixableParameterRange modelMixingRange(
 
 } // namespace model
 
+namespace {
 /// Matches DeclRefExprs and their ignorable wrappers to ParmVarDecls.
 AST_MATCHER_FUNCTION(ast_matchers::internal::Matcher<Stmt>, paramRefExpr) {
   return expr(ignoringParenImpCasts(ignoringElidableConstructorCall(
       declRefExpr(to(parmVarDecl().bind("param"))))));
 }
+} // namespace
 
 namespace filter {
 
@@ -1574,8 +1576,8 @@ using ParamToSmallSetMap =
 /// Returns whether the sets mapped to the two elements in the map have at
 /// least one element in common.
 template <typename MapTy, typename ElemTy>
-bool lazyMapOfSetsIntersectionExists(const MapTy &Map, const ElemTy &E1,
-                                     const ElemTy &E2) {
+static bool lazyMapOfSetsIntersectionExists(const MapTy &Map, const ElemTy &E1,
+                                            const ElemTy &E2) {
   auto E1Iterator = Map.find(E1);
   auto E2Iterator = Map.find(E2);
   if (E1Iterator == Map.end() || E2Iterator == Map.end())
@@ -1882,6 +1884,8 @@ static bool prefixSuffixCoverUnderThreshold(std::size_t Threshold,
 
 } // namespace filter
 
+namespace {
+
 /// Matches functions that have at least the specified amount of parameters.
 AST_MATCHER_P(FunctionDecl, parameterCountGE, unsigned, N) {
   return Node.getNumParams() >= N;
@@ -1904,6 +1908,8 @@ AST_MATCHER(FunctionDecl, isOverloadedUnaryOrBinaryOperator) {
   }
 }
 
+} // namespace
+
 /// Returns the DefaultMinimumLength if the Value of requested minimum length
 /// is less than 2. Minimum lengths of 0 or 1 are not accepted.
 static inline unsigned clampMinimumLength(const unsigned Value) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp
index 7626bf89e694..92425890a0ea 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp
@@ -18,9 +18,9 @@ void InaccurateEraseCheck::registerMatchers(MatchFinder *Finder) {
   const auto EndCall =
       callExpr(
           callee(functionDecl(hasAnyName("remove", "remove_if", "unique"))),
-          hasArgument(
-              1, optionally(cxxMemberCallExpr(callee(cxxMethodDecl(hasName("end"))))
-                           .bind("end"))))
+          hasArgument(1, optionally(cxxMemberCallExpr(
+                                        callee(cxxMethodDecl(hasName("end"))))
+                                        .bind("end"))))
           .bind("alg");
 
   const auto DeclInStd = type(hasUnqualifiedDesugaredType(
@@ -35,10 +35,8 @@ void InaccurateEraseCheck::registerMatchers(MatchFinder *Finder) {
 }
 
 void InaccurateEraseCheck::check(const MatchFinder::MatchResult &Result) {
-  const auto *MemberCall =
-      Result.Nodes.getNodeAs<CXXMemberCallExpr>("erase");
-  const auto *EndExpr =
-      Result.Nodes.getNodeAs<CXXMemberCallExpr>("end");
+  const auto *MemberCall = Result.Nodes.getNodeAs<CXXMemberCallExpr>("erase");
+  const auto *EndExpr = Result.Nodes.getNodeAs<CXXMemberCallExpr>("end");
   const SourceLocation Loc = MemberCall->getBeginLoc();
 
   FixItHint Hint;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp
index 9b3b01eb0268..73bffe93146e 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp
@@ -15,6 +15,8 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::bugprone {
 
+namespace {
+
 AST_MATCHER(BinaryOperator, isLogicalOperator) { return Node.isLogicalOp(); }
 
 AST_MATCHER(UnaryOperator, isUnaryPrePostOperator) {
@@ -26,6 +28,8 @@ AST_MATCHER(CXXOperatorCallExpr, isPrePostOperator) {
          Node.getOperator() == OO_MinusMinus;
 }
 
+} // namespace
+
 void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) {
   auto OperatorMatcher = expr(
       anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp
index 2a8d36215cc7..c2b0732a3e7b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp
@@ -1,4 +1,4 @@
-//===--- IncorrectRoundingsCheck.cpp - clang-tidy ------------------------------===//
+//===--- IncorrectRoundingsCheck.cpp - clang-tidy--------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
index b7f0c08b2a7d..07116a7ff15f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
@@ -83,8 +83,8 @@ static bool isVarThatIsPossiblyChanged(const Decl *Func, const Stmt *LoopStmt,
              isChanged(LoopStmt, Var, Context);
       // FIXME: Track references.
     }
-  } else if (isa<MemberExpr, CallExpr,
-                 ObjCIvarRefExpr, ObjCPropertyRefExpr, ObjCMessageExpr>(Cond)) {
+  } else if (isa<MemberExpr, CallExpr, ObjCIvarRefExpr, ObjCPropertyRefExpr,
+                 ObjCMessageExpr>(Cond)) {
     // FIXME: Handle MemberExpr.
     return true;
   } else if (const auto *CE = dyn_cast<CastExpr>(Cond)) {
@@ -274,8 +274,7 @@ static bool hasRecursionOverStaticLoopCondVariables(const Expr *Cond,
 
 void InfiniteLoopCheck::registerMatchers(MatchFinder *Finder) {
   const auto LoopCondition = allOf(
-      hasCondition(
-          expr(forCallable(decl().bind("func"))).bind("condition")),
+      hasCondition(expr(forCallable(decl().bind("func"))).bind("condition")),
       unless(hasBody(hasDescendant(
           loopEndingStmt(forCallable(equalsBoundNode("func")))))));
 
@@ -324,7 +323,7 @@ void InfiniteLoopCheck::check(const MatchFinder::MatchResult &Result) {
     diag(LoopStmt->getBeginLoc(),
          "this loop is infinite; none of its condition variables (%0)"
          " are updated in the loop body")
-      << CondVarNames;
+        << CondVarNames;
   }
 }
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.cpp
index 32f5edddfe80..6d5c470616d1 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.cpp
@@ -34,12 +34,11 @@ class MacroExpansionsWithFileAndLine : public PPCallbacks {
       LambdaFunctionNameCheck::SourceRangeSet *SME)
       : SuppressMacroExpansions(SME) {}
 
-  void MacroExpands(const Token &MacroNameTok,
-                    const MacroDefinition &MD, SourceRange Range,
-                    const MacroArgs *Args) override {
+  void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
+                    SourceRange Range, const MacroArgs *Args) override {
     bool HasFile = false;
     bool HasLine = false;
-    for (const auto& T : MD.getMacroInfo()->tokens()) {
+    for (const Token &T : MD.getMacroInfo()->tokens()) {
       if (T.is(tok::identifier)) {
         StringRef IdentName = T.getIdentifierInfo()->getName();
         if (IdentName == "__FILE__") {
@@ -55,7 +54,7 @@ class MacroExpansionsWithFileAndLine : public PPCallbacks {
   }
 
 private:
-  LambdaFunctionNameCheck::SourceRangeSet* SuppressMacroExpansions;
+  LambdaFunctionNameCheck::SourceRangeSet *SuppressMacroExpansions;
 };
 
 AST_MATCHER(CXXMethodDecl, isInLambda) { return Node.getParent()->isLambda(); }
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
index 40e4ab6c8b12..5b166b4b3a9b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
@@ -74,7 +74,7 @@ void MisplacedOperatorInStrlenInAllocCheck::check(
   if (!Alloc)
     Alloc = Result.Nodes.getNodeAs<CXXNewExpr>("Alloc");
   assert(Alloc && "Matched node bound by 'Alloc' should be either 'CallExpr'"
-         " or 'CXXNewExpr'");
+                  " or 'CXXNewExpr'");
 
   const auto *StrLen = Result.Nodes.getNodeAs<CallExpr>("StrLen");
   const auto *BinOp = Result.Nodes.getNodeAs<BinaryOperator>("BinOp");
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.h
index 3a09ddb4a6c0..93cf50d0b1c6 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.h
@@ -21,7 +21,8 @@ namespace clang::tidy::bugprone {
 /// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/misplaced-operator-in-strlen-in-alloc.html
 class MisplacedOperatorInStrlenInAllocCheck : public ClangTidyCheck {
 public:
-  MisplacedOperatorInStrlenInAllocCheck(StringRef Name, ClangTidyContext *Context)
+  MisplacedOperatorInStrlenInAllocCheck(StringRef Name,
+                                        ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp
index b9129fa0811c..86785d36696d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MisplacedPointerArithmeticInAllocCheck.cpp
@@ -76,8 +76,8 @@ void MisplacedPointerArithmeticInAllocCheck::check(
     } else {
       const auto *CtrE = New->getConstructExpr();
       if (!CtrE || !CtrE->getArg(CtrE->getNumArgs() - 1)
-                                     ->getType()
-                                     ->isIntegralOrEnumerationType())
+                        ->getType()
+                        ->isIntegralOrEnumerationType())
         return;
       CallName = "operator new";
     }
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp
index 7a989b07119a..1a23473fdd22 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.cpp
@@ -57,6 +57,17 @@ AST_MATCHER(QualType, isPointerType) {
 
 } // namespace
 
+MultiLevelImplicitPointerConversionCheck::
+    MultiLevelImplicitPointerConversionCheck(StringRef Name,
+                                             ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context), EnableInC(Options.get("EnableInC", true)) {
+}
+
+void MultiLevelImplicitPointerConversionCheck::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "EnableInC", EnableInC);
+}
+
 void MultiLevelImplicitPointerConversionCheck::registerMatchers(
     MatchFinder *Finder) {
   Finder->addMatcher(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.h
index 13228145ff35..ef5f9f103c89 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultiLevelImplicitPointerConversionCheck.h
@@ -21,11 +21,17 @@ namespace clang::tidy::bugprone {
 class MultiLevelImplicitPointerConversionCheck : public ClangTidyCheck {
 public:
   MultiLevelImplicitPointerConversionCheck(StringRef Name,
-                                           ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
+                                           ClangTidyContext *Context);
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   std::optional<TraversalKind> getCheckTraversalKind() const override;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return EnableInC ? true : LangOpts.CPlusPlus;
+  }
+
+private:
+  bool const EnableInC;
 };
 
 } // namespace clang::tidy::bugprone
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
index 41191a3cfed2..b68888cb5b92 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
@@ -49,8 +49,6 @@ bool isExprValueStored(const Expr *E, ASTContext &C) {
   return isa<CallExpr, CXXConstructExpr>(ParentE);
 }
 
-} // namespace
-
 AST_MATCHER_P(CXXTryStmt, hasHandlerFor,
               ast_matchers::internal::Matcher<QualType>, InnerMatcher) {
   for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) {
@@ -74,6 +72,8 @@ AST_MATCHER(CXXNewExpr, mayThrow) {
   return !OperatorNew->getType()->castAs<FunctionProtoType>()->isNothrow();
 }
 
+} // namespace
+
 void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) {
   auto BadAllocType =
       recordType(hasDeclaration(cxxRecordDecl(hasName("::std::bad_alloc"))));
@@ -95,16 +95,14 @@ void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) {
 
   Finder->addMatcher(
       callExpr(
-          hasAnyArgument(
-              expr(HasNewExpr1).bind("arg1")),
+          hasAnyArgument(expr(HasNewExpr1).bind("arg1")),
           hasAnyArgument(
               expr(HasNewExpr2, unless(equalsBoundNode("arg1"))).bind("arg2")),
           hasAncestor(BadAllocCatchingTryBlock)),
       this);
   Finder->addMatcher(
       cxxConstructExpr(
-          hasAnyArgument(
-              expr(HasNewExpr1).bind("arg1")),
+          hasAnyArgument(expr(HasNewExpr1).bind("arg1")),
           hasAnyArgument(
               expr(HasNewExpr2, unless(equalsBoundNode("arg1"))).bind("arg2")),
           unless(isListInitialization()),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
index 977241e91b9a..203170d55f69 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
@@ -677,7 +677,7 @@ void NotNullTerminatedResultCheck::registerMatchers(MatchFinder *Finder) {
                 std::optional<unsigned> SourcePos, unsigned LengthPos,
                 bool WithIncrease)
         : Name(Name), DestinationPos(DestinationPos), SourcePos(SourcePos),
-          LengthPos(LengthPos), WithIncrease(WithIncrease){};
+          LengthPos(LengthPos), WithIncrease(WithIncrease) {};
 
     StringRef Name;
     std::optional<unsigned> DestinationPos;
@@ -702,17 +702,16 @@ void NotNullTerminatedResultCheck::registerMatchers(MatchFinder *Finder) {
     return hasArgument(
         CC.LengthPos,
         allOf(
-            anyOf(
-                ignoringImpCasts(integerLiteral().bind(WrongLengthExprName)),
-                allOf(unless(hasDefinition(SizeOfCharExpr)),
-                      allOf(CC.WithIncrease
-                                ? ignoringImpCasts(hasDefinition(HasIncOp))
-                                : ignoringImpCasts(allOf(
-                                      unless(hasDefinition(HasIncOp)),
-                                      anyOf(hasDefinition(binaryOperator().bind(
-                                                UnknownLengthName)),
-                                            hasDefinition(anything())))),
-                            AnyOfWrongLengthInit))),
+            anyOf(ignoringImpCasts(integerLiteral().bind(WrongLengthExprName)),
+                  allOf(unless(hasDefinition(SizeOfCharExpr)),
+                        allOf(CC.WithIncrease
+                                  ? ignoringImpCasts(hasDefinition(HasIncOp))
+                                  : ignoringImpCasts(
+                                        allOf(unless(hasDefinition(HasIncOp)),
+                                              hasDefinition(optionally(
+                                                  binaryOperator().bind(
+                                                      UnknownLengthName))))),
+                              AnyOfWrongLengthInit))),
             expr().bind(LengthExprName)));
   };
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h
index 254ae6c7f1ac..e9d2263b06bc 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h
@@ -13,7 +13,7 @@
 
 namespace clang::tidy::bugprone {
 
-class PosixReturnCheck: public ClangTidyCheck {
+class PosixReturnCheck : public ClangTidyCheck {
 public:
   PosixReturnCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp
index 95fbca2ad008..e717564847e4 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp
@@ -1,4 +1,4 @@
-//===--- RedundantBranchConditionCheck.cpp - clang-tidy -------------------------===//
+//===--- RedundantBranchConditionCheck.cpp - clang-tidy--------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -70,7 +70,8 @@ void RedundantBranchConditionCheck::registerMatchers(MatchFinder *Finder) {
   // FIXME: Handle longer conjunctive and disjunctive clauses.
 }
 
-void RedundantBranchConditionCheck::check(const MatchFinder::MatchResult &Result) {
+void RedundantBranchConditionCheck::check(
+    const MatchFinder::MatchResult &Result) {
   const auto *OuterIf = Result.Nodes.getNodeAs<IfStmt>(OuterIfStr);
   const auto *InnerIf = Result.Nodes.getNodeAs<IfStmt>(InnerIfStr);
   const auto *CondVar = Result.Nodes.getNodeAs<VarDecl>(CondVarStr);
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
index c066b3e7b19a..d5cbb847eea9 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
@@ -22,7 +22,9 @@ constexpr llvm::StringLiteral MinimalConformingFunctions[] = {
 // mentioned POSIX specification was not updated after 'quick_exit' appeared
 // in the C11 standard.
 // Also, we want to keep the "minimal set" a subset of the "POSIX set".
-// The list is repeated in bugprone-signal-handler.rst and should be kept up to date.
+// The list is repeated in bugprone-signal-handler.rst and should be kept up to
+// date.
+// clang-format off
 constexpr llvm::StringLiteral POSIXConformingFunctions[] = {
     "_Exit",
     "_exit",
@@ -215,7 +217,9 @@ constexpr llvm::StringLiteral POSIXConformingFunctions[] = {
     "wmemcpy",
     "wmemmove",
     "wmemset",
-    "write"};
+    "write"
+};
+// clang-format on
 
 using namespace clang::ast_matchers;
 
@@ -322,12 +326,12 @@ SourceRange getSourceRangeOfStmt(const Stmt *S, ASTContext &Ctx) {
   return P.getSourceRange();
 }
 
-} // namespace
-
 AST_MATCHER(FunctionDecl, isStandardFunction) {
   return isStandardFunction(&Node);
 }
 
+} // namespace
+
 SignalHandlerCheck::SignalHandlerCheck(StringRef Name,
                                        ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
index f3d4c2255d86..9eeba867f521 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
@@ -170,8 +170,6 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
 
     const auto PointerToStructType =
         hasUnqualifiedDesugaredType(pointerType(pointee(recordType())));
-    const auto PointerToStructTypeWithBinding =
-        type(PointerToStructType).bind("struct-type");
     const auto PointerToStructExpr =
         expr(hasType(hasCanonicalType(PointerToStructType)));
 
@@ -188,12 +186,10 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
         ignoringParenImpCasts(unaryOperator(hasOperatorName("*")));
 
     Finder->addMatcher(
-        expr(sizeOfExpr(anyOf(has(ignoringParenImpCasts(
-                                  expr(PointerToDetectedExpr, unless(DerefExpr),
-                                       unless(SubscriptExprWithZeroIndex),
-                                       unless(VarWithConstStrLiteralDecl),
-                                       unless(cxxThisExpr())))),
-                              has(PointerToStructTypeWithBinding))))
+        expr(sizeOfExpr(has(ignoringParenImpCasts(expr(
+                 PointerToDetectedExpr, unless(DerefExpr),
+                 unless(SubscriptExprWithZeroIndex),
+                 unless(VarWithConstStrLiteralDecl), unless(cxxThisExpr()))))))
             .bind("sizeof-pointer"),
         this);
   }
@@ -354,16 +350,9 @@ void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
          "suspicious usage of 'sizeof(char*)'; do you mean 'strlen'?")
         << E->getSourceRange();
   } else if (const auto *E = Result.Nodes.getNodeAs<Expr>("sizeof-pointer")) {
-    if (Result.Nodes.getNodeAs<Type>("struct-type")) {
-      diag(E->getBeginLoc(),
-           "suspicious usage of 'sizeof(A*)' on pointer-to-aggregate type; did "
-           "you mean 'sizeof(A)'?")
-          << E->getSourceRange();
-    } else {
-      diag(E->getBeginLoc(), "suspicious usage of 'sizeof()' on an expression "
-                             "of pointer type")
-          << E->getSourceRange();
-    }
+    diag(E->getBeginLoc(), "suspicious usage of 'sizeof()' on an expression "
+                           "of pointer type")
+        << E->getSourceRange();
   } else if (const auto *E = Result.Nodes.getNodeAs<BinaryOperator>(
                  "sizeof-compare-constant")) {
     diag(E->getOperatorLoc(),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.h
index 5e143411f9b6..098299aea7de 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SpuriouslyWakeUpFunctionsCheck.h
@@ -13,9 +13,9 @@
 
 namespace clang::tidy::bugprone {
 
-/// Finds ``cnd_wait``, ``cnd_timedwait``, ``wait``, ``wait_for``, or 
-/// ``wait_until`` function calls when the function is not invoked from a loop 
-/// that checks whether a condition predicate holds or the function has a 
+/// Finds ``cnd_wait``, ``cnd_timedwait``, ``wait``, ``wait_for``, or
+/// ``wait_until`` function calls when the function is not invoked from a loop
+/// that checks whether a condition predicate holds or the function has a
 /// condition parameter.
 ///
 /// For the user-facing documentation see:
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp
index 682478ecead0..5d9e91e0b82c 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StandaloneEmptyCheck.cpp
@@ -46,7 +46,8 @@ using ast_matchers::stmtExpr;
 using ast_matchers::unless;
 using ast_matchers::voidType;
 
-const Expr *getCondition(const BoundNodes &Nodes, const StringRef NodeId) {
+static const Expr *getCondition(const BoundNodes &Nodes,
+                                const StringRef NodeId) {
   const auto *If = Nodes.getNodeAs<IfStmt>(NodeId);
   if (If != nullptr)
     return If->getCond();
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringLiteralWithEmbeddedNulCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringLiteralWithEmbeddedNulCheck.cpp
index 72e680d25cb8..444f3081b704 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringLiteralWithEmbeddedNulCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringLiteralWithEmbeddedNulCheck.cpp
@@ -47,8 +47,9 @@ void StringLiteralWithEmbeddedNulCheck::registerMatchers(MatchFinder *Finder) {
 
   // Detect passing a suspicious string literal to a string constructor.
   // example: std::string str = "abc\0def";
-  Finder->addMatcher(traverse(TK_AsIs,
-      cxxConstructExpr(StringConstructorExpr, hasArgument(0, StrLitWithNul))),
+  Finder->addMatcher(
+      traverse(TK_AsIs, cxxConstructExpr(StringConstructorExpr,
+                                         hasArgument(0, StrLitWithNul))),
       this);
 
   // Detect passing a suspicious string literal through an overloaded operator.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp
index f944ae6c5b9e..20789b3123e2 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/StringviewNullptrCheck.cpp
@@ -33,7 +33,7 @@ AST_MATCHER(clang::VarDecl, isDirectInitialization) {
 
 } // namespace
 
-RewriteRuleWith<std::string> stringviewNullptrCheckImpl() {
+static RewriteRuleWith<std::string> stringviewNullptrCheckImpl() {
   auto ConstructionWarning =
       cat("constructing basic_string_view from null is undefined; replace with "
           "the default constructor");
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.h
index 63c4bea6e565..c8a70c5f0704 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.h
@@ -1,4 +1,4 @@
-//===--- SuspiciousEnumUsageCheck.h - clang-tidy--------------------------*- C++ -*-===//
+//===--- SuspiciousEnumUsageCheck.h - clang-tidy------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -28,7 +28,7 @@ class SuspiciousEnumUsageCheck : public ClangTidyCheck {
   }
 
 private:
-  void checkSuspiciousBitmaskUsage(const Expr*, const EnumDecl*);
+  void checkSuspiciousBitmaskUsage(const Expr *, const EnumDecl *);
   const bool StrictMode;
 };
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp
index a488d35ca744..cc1bd622039b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousMemsetUsageCheck.cpp
@@ -21,8 +21,7 @@ void SuspiciousMemsetUsageCheck::registerMatchers(MatchFinder *Finder) {
   // Match the standard memset:
   // void *memset(void *buffer, int fill_char, size_t byte_count);
   auto MemsetDecl =
-      functionDecl(hasName("::memset"),
-                   parameterCountIs(3),
+      functionDecl(hasName("::memset"), parameterCountIs(3),
                    hasParameter(0, hasType(pointerType(pointee(voidType())))),
                    hasParameter(1, hasType(isInteger())),
                    hasParameter(2, hasType(isInteger())));
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousSemicolonCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousSemicolonCheck.cpp
index 92b772dd26d4..e93ba760f447 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousSemicolonCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/SuspiciousSemicolonCheck.cpp
@@ -18,8 +18,7 @@ namespace clang::tidy::bugprone {
 void SuspiciousSemicolonCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       stmt(anyOf(ifStmt(hasThen(nullStmt().bind("semi")),
-                        unless(hasElse(stmt())),
-                        unless(isConstexpr())),
+                        unless(hasElse(stmt())), unless(isConstexpr())),
                  forStmt(hasBody(nullStmt().bind("semi"))),
                  cxxForRangeStmt(hasBody(nullStmt().bind("semi"))),
                  whileStmt(hasBody(nullStmt().bind("semi")))))
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp
index 9db14ab40ce5..a4c1fd53dfbe 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp
@@ -1,4 +1,4 @@
-//===--- UndelegatedConstructorCheck.cpp - clang-tidy --------------------------===//
+//===--- UndelegatedConstructorCheck.cpp - clang-tidy----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp
index ca4562aa9a44..5e220017c97f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnhandledExceptionAtNewCheck.cpp
@@ -12,6 +12,7 @@
 using namespace clang::ast_matchers;
 
 namespace clang::tidy::bugprone {
+namespace {
 
 AST_MATCHER_P(CXXTryStmt, hasHandlerFor,
               ast_matchers::internal::Matcher<QualType>, InnerMatcher) {
@@ -36,6 +37,8 @@ AST_MATCHER(CXXNewExpr, mayThrow) {
   return !OperatorNew->getType()->castAs<FunctionProtoType>()->isNothrow();
 }
 
+} // namespace
+
 UnhandledExceptionAtNewCheck::UnhandledExceptionAtNewCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context) {}
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
index d96e7524172b..b17d3868dd76 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
@@ -37,8 +37,8 @@ void UnusedRaiiCheck::registerMatchers(MatchFinder *Finder) {
 }
 
 template <typename T>
-void reportDiagnostic(DiagnosticBuilder D, const T *Node, SourceRange SR,
-                      bool DefaultConstruction) {
+static void reportDiagnostic(DiagnosticBuilder D, const T *Node, SourceRange SR,
+                             bool DefaultConstruction) {
   const char *Replacement = " give_me_a_name";
 
   // If this is a default ctor we have to remove the parens or we'll introduce a
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp
index 960133159dbb..1bcacf96a412 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp
@@ -242,7 +242,7 @@ void UseAfterMoveFinder::getUsesAndReinits(
   });
 }
 
-bool isStandardSmartPointer(const ValueDecl *VD) {
+static bool isStandardSmartPointer(const ValueDecl *VD) {
   const Type *TheType = VD->getType().getNonReferenceType().getTypePtrOrNull();
   if (!TheType)
     return false;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
index cc092a9627c5..66fedabaf3ca 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
@@ -250,8 +250,7 @@ class CERTModule : public ClangTidyModule {
         "cert-dcl51-cpp");
     CheckFactories.registerCheck<misc::NewDeleteOverloadsCheck>(
         "cert-dcl54-cpp");
-    CheckFactories.registerCheck<DontModifyStdNamespaceCheck>(
-        "cert-dcl58-cpp");
+    CheckFactories.registerCheck<DontModifyStdNamespaceCheck>("cert-dcl58-cpp");
     CheckFactories.registerCheck<google::build::UnnamedNamespaceInHeaderCheck>(
         "cert-dcl59-cpp");
     // ERR
@@ -278,8 +277,7 @@ class CERTModule : public ClangTidyModule {
         "cert-oop54-cpp");
     CheckFactories.registerCheck<NonTrivialTypesLibcMemoryCallsCheck>(
         "cert-oop57-cpp");
-    CheckFactories.registerCheck<MutatingCopyCheck>(
-        "cert-oop58-cpp");
+    CheckFactories.registerCheck<MutatingCopyCheck>("cert-oop58-cpp");
 
     // C checkers
     // ARR
@@ -360,6 +358,6 @@ static ClangTidyModuleRegistry::Add<cert::CERTModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the CERTModule.
-volatile int CERTModuleAnchorSource = 0;
+volatile int CERTModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/concurrency/ConcurrencyTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/concurrency/ConcurrencyTidyModule.cpp
index a88ee46fc4ab..6c58c506dc90 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/concurrency/ConcurrencyTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/concurrency/ConcurrencyTidyModule.cpp
@@ -27,12 +27,14 @@ class ConcurrencyModule : public ClangTidyModule {
 
 } // namespace concurrency
 
-// Register the ConcurrencyTidyModule using this statically initialized variable.
+// Register the ConcurrencyTidyModule using this statically initialized
+// variable.
 static ClangTidyModuleRegistry::Add<concurrency::ConcurrencyModule>
     X("concurrency-module", "Adds concurrency checks.");
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the ConcurrencyModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int ConcurrencyModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
index 6adef0426434..4dd9b0904f07 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
@@ -156,6 +156,7 @@ static ClangTidyModuleRegistry::Add<CppCoreGuidelinesModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the CppCoreGuidelinesModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int CppCoreGuidelinesModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
index d665c47d12bb..aaaaf6b66b51 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
@@ -433,21 +433,22 @@ void ProTypeMemberInitCheck::checkMissingMemberInitializer(
   // Gather all fields (direct and indirect) that need to be initialized.
   SmallPtrSet<const FieldDecl *, 16> FieldsToInit;
   bool AnyMemberHasInitPerUnion = false;
-  forEachFieldWithFilter(ClassDecl, ClassDecl.fields(),
-                         AnyMemberHasInitPerUnion, [&](const FieldDecl *F) {
-    if (IgnoreArrays && F->getType()->isArrayType())
-      return;
-    if (F->hasInClassInitializer() && F->getParent()->isUnion()) {
-      AnyMemberHasInitPerUnion = true;
-      removeFieldInitialized(F, FieldsToInit);
-    }
-    if (!F->hasInClassInitializer() &&
-        utils::type_traits::isTriviallyDefaultConstructible(F->getType(),
-                                                            Context) &&
-        !isEmpty(Context, F->getType()) && !F->isUnnamedBitField() &&
-        !AnyMemberHasInitPerUnion)
-      FieldsToInit.insert(F);
-  });
+  forEachFieldWithFilter(
+      ClassDecl, ClassDecl.fields(), AnyMemberHasInitPerUnion,
+      [&](const FieldDecl *F) {
+        if (IgnoreArrays && F->getType()->isArrayType())
+          return;
+        if (F->hasInClassInitializer() && F->getParent()->isUnion()) {
+          AnyMemberHasInitPerUnion = true;
+          removeFieldInitialized(F, FieldsToInit);
+        }
+        if (!F->hasInClassInitializer() &&
+            utils::type_traits::isTriviallyDefaultConstructible(F->getType(),
+                                                                Context) &&
+            !isEmpty(Context, F->getType()) && !F->isUnnamedBitField() &&
+            !AnyMemberHasInitPerUnion)
+          FieldsToInit.insert(F);
+      });
   if (FieldsToInit.empty())
     return;
 
@@ -500,17 +501,18 @@ void ProTypeMemberInitCheck::checkMissingMemberInitializer(
   AnyMemberHasInitPerUnion = false;
   forEachFieldWithFilter(ClassDecl, ClassDecl.fields(),
                          AnyMemberHasInitPerUnion, [&](const FieldDecl *F) {
-    if (!FieldsToInit.count(F))
-      return;
-    // Don't suggest fixes for enums because we don't know a good default.
-    // Don't suggest fixes for bitfields because in-class initialization is not
-    // possible until C++20.
-    if (F->getType()->isEnumeralType() ||
-        (!getLangOpts().CPlusPlus20 && F->isBitField()))
-      return;
-    FieldsToFix.insert(F);
-    AnyMemberHasInitPerUnion = true;
-  });
+                           if (!FieldsToInit.count(F))
+                             return;
+                           // Don't suggest fixes for enums because we don't
+                           // know a good default. Don't suggest fixes for
+                           // bitfields because in-class initialization is not
+                           // possible until C++20.
+                           if (F->getType()->isEnumeralType() ||
+                               (!getLangOpts().CPlusPlus20 && F->isBitField()))
+                             return;
+                           FieldsToFix.insert(F);
+                           AnyMemberHasInitPerUnion = true;
+                         });
   if (FieldsToFix.empty())
     return;
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
index 00f6e93fec20..02bfaf1205f4 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
@@ -14,11 +14,7 @@
 namespace clang::tidy::cppcoreguidelines {
 
 /// Flags slicing (incomplete copying of an object's state) of member variables
-/// or vtable. See:
-///   - https://github.com/isocpp/CppCoreGuidelines/blob/master/CppCoreGuidelines.md#es63-dont-slice
-///     for the former, and
-///   - https://github.com/isocpp/CppCoreGuidelines/blob/master/CppCoreGuidelines.md#c145-access-polymorphic-objects-through-pointers-and-references
-///     for the latter
+/// or vtable.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines/slicing.html
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp
index b1da3b9861c6..0de143dbb1b8 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp
@@ -116,7 +116,8 @@ void SpecialMemberFunctionsCheck::check(
   if (!MatchedDecl)
     return;
 
-  ClassDefId ID(MatchedDecl->getLocation(), std::string(MatchedDecl->getName()));
+  ClassDefId ID(MatchedDecl->getLocation(),
+                std::string(MatchedDecl->getName()));
 
   auto StoreMember = [this, &ID](SpecialMemberFunctionData Data) {
     llvm::SmallVectorImpl<SpecialMemberFunctionData> &Members =
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
index 9ebc03ed2fa1..dee01cb5a9fd 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
@@ -84,13 +84,12 @@ struct DenseMapInfo<
       clang::tidy::cppcoreguidelines::SpecialMemberFunctionsCheck::ClassDefId;
 
   static inline ClassDefId getEmptyKey() {
-    return {DenseMapInfo<clang::SourceLocation>::getEmptyKey(),
-                      "EMPTY"};
+    return {DenseMapInfo<clang::SourceLocation>::getEmptyKey(), "EMPTY"};
   }
 
   static inline ClassDefId getTombstoneKey() {
     return {DenseMapInfo<clang::SourceLocation>::getTombstoneKey(),
-                      "TOMBSTONE"};
+            "TOMBSTONE"};
   }
 
   static unsigned getHashValue(ClassDefId Val) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp
index aa70b3896f16..e31d04656567 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp
@@ -18,6 +18,8 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::cppcoreguidelines {
 
+namespace {
+
 AST_MATCHER(CXXRecordDecl, hasPublicVirtualOrProtectedNonVirtualDestructor) {
   // We need to call Node.getDestructor() instead of matching a
   // CXXDestructorDecl. Otherwise, tests will fail for class templates, since
@@ -33,6 +35,8 @@ AST_MATCHER(CXXRecordDecl, hasPublicVirtualOrProtectedNonVirtualDestructor) {
            !Destructor->isVirtual()));
 }
 
+} // namespace
+
 void VirtualClassDestructorCheck::registerMatchers(MatchFinder *Finder) {
   ast_matchers::internal::Matcher<CXXRecordDecl> InheritsVirtualMethod =
       hasAnyBase(hasType(cxxRecordDecl(has(cxxMethodDecl(isVirtual())))));
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.h
index 727878b258b2..5b5285710c3b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.h
@@ -19,7 +19,7 @@ namespace clang::tidy::darwin {
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/darwin/avoid-spinlock.html
 class AvoidSpinlockCheck : public ClangTidyCheck {
- public:
+public:
   AvoidSpinlockCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
@@ -28,4 +28,4 @@ class AvoidSpinlockCheck : public ClangTidyCheck {
 
 } // namespace clang::tidy::darwin
 
-#endif  // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_AVOIDSPINLOCKCHECK_H
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_AVOIDSPINLOCKCHECK_H
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/darwin/DarwinTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/darwin/DarwinTidyModule.cpp
index 3475deada499..bc8c91a9ed41 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/darwin/DarwinTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/darwin/DarwinTidyModule.cpp
@@ -18,8 +18,7 @@ namespace darwin {
 class DarwinModule : public ClangTidyModule {
 public:
   void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
-    CheckFactories.registerCheck<AvoidSpinlockCheck>(
-        "darwin-avoid-spinlock");
+    CheckFactories.registerCheck<AvoidSpinlockCheck>("darwin-avoid-spinlock");
     CheckFactories.registerCheck<DispatchOnceNonstaticCheck>(
         "darwin-dispatch-once-nonstatic");
   }
@@ -33,6 +32,6 @@ static ClangTidyModuleRegistry::Add<darwin::DarwinModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the DarwinModule.
-volatile int DarwinModuleAnchorSource = 0;
+volatile int DarwinModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/FuchsiaTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/FuchsiaTidyModule.cpp
index 45a79b75e093..d7a70b39bdc5 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/FuchsiaTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/FuchsiaTidyModule.cpp
@@ -52,6 +52,6 @@ static ClangTidyModuleRegistry::Add<FuchsiaModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the FuchsiaModule.
-volatile int FuchsiaModuleAnchorSource = 0;
+volatile int FuchsiaModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp
index b5ce23ae8fed..80ff97a76213 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp
@@ -49,7 +49,8 @@ bool MultipleInheritanceCheck::getInterfaceStatus(const CXXRecordDecl *Node,
 bool MultipleInheritanceCheck::isCurrentClassInterface(
     const CXXRecordDecl *Node) const {
   // Interfaces should have no fields.
-  if (!Node->field_empty()) return false;
+  if (!Node->field_empty())
+    return false;
 
   // Interfaces should have exclusively pure methods.
   return llvm::none_of(Node->methods(), [](const CXXMethodDecl *M) {
@@ -68,11 +69,14 @@ bool MultipleInheritanceCheck::isInterface(const CXXRecordDecl *Node) {
 
   // To be an interface, all base classes must be interfaces as well.
   for (const auto &I : Node->bases()) {
-    if (I.isVirtual()) continue;
+    if (I.isVirtual())
+      continue;
     const auto *Ty = I.getType()->getAs<RecordType>();
-    if (!Ty) continue;
+    if (!Ty)
+      continue;
     const RecordDecl *D = Ty->getDecl()->getDefinition();
-    if (!D) continue;
+    if (!D)
+      continue;
     const auto *Base = cast<CXXRecordDecl>(D);
     if (!isInterface(Base)) {
       addNodeToInterfaceMap(Node, false);
@@ -97,20 +101,25 @@ void MultipleInheritanceCheck::check(const MatchFinder::MatchResult &Result) {
     // concrete classes
     unsigned NumConcrete = 0;
     for (const auto &I : D->bases()) {
-      if (I.isVirtual()) continue;
+      if (I.isVirtual())
+        continue;
       const auto *Ty = I.getType()->getAs<RecordType>();
-      if (!Ty) continue;
+      if (!Ty)
+        continue;
       const auto *Base = cast<CXXRecordDecl>(Ty->getDecl()->getDefinition());
-      if (!isInterface(Base)) NumConcrete++;
+      if (!isInterface(Base))
+        NumConcrete++;
     }
 
     // Check virtual bases to see if there is more than one concrete
     // non-virtual base.
     for (const auto &V : D->vbases()) {
       const auto *Ty = V.getType()->getAs<RecordType>();
-      if (!Ty) continue;
+      if (!Ty)
+        continue;
       const auto *Base = cast<CXXRecordDecl>(Ty->getDecl()->getDefinition());
-      if (!isInterface(Base)) NumConcrete++;
+      if (!isInterface(Base))
+        NumConcrete++;
     }
 
     if (NumConcrete > 1) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.h
index fc18273ad73b..6c65c8cfeb22 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.h
@@ -14,7 +14,7 @@
 namespace clang::tidy::fuchsia {
 
 /// Constructing global, non-trivial objects with static storage is
-/// disallowed, unless the object is statically initialized with a constexpr 
+/// disallowed, unless the object is statically initialized with a constexpr
 /// constructor or has no explicit constructor.
 ///
 /// For the user-facing documentation see:
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/TrailingReturnCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/TrailingReturnCheck.h
index 2ba8e27d00bf..70551844898f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/TrailingReturnCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/TrailingReturnCheck.h
@@ -13,8 +13,8 @@
 
 namespace clang::tidy::fuchsia {
 
-/// Functions that have trailing returns are disallowed, except for those 
-/// using decltype specifiers and lambda with otherwise unutterable 
+/// Functions that have trailing returns are disallowed, except for those
+/// using decltype specifiers and lambda with otherwise unutterable
 /// return types.
 ///
 /// For the user-facing documentation see:
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.cpp
index 9c367423fdb5..20bd036fb265 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.cpp
@@ -16,10 +16,13 @@ namespace clang::tidy::fuchsia {
 
 namespace {
 AST_MATCHER(CXXRecordDecl, hasDirectVirtualBaseClass) {
-  if (!Node.hasDefinition()) return false;
-  if (!Node.getNumVBases()) return false;
+  if (!Node.hasDefinition())
+    return false;
+  if (!Node.getNumVBases())
+    return false;
   for (const CXXBaseSpecifier &Base : Node.bases())
-    if (Base.isVirtual()) return true;
+    if (Base.isVirtual())
+      return true;
   return false;
 }
 } // namespace
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h
index d109bd52a1a6..426d89d046a6 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h
@@ -18,7 +18,7 @@ namespace clang::tidy::fuchsia {
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/fuchsia/virtual-inheritance.html
 class VirtualInheritanceCheck : public ClangTidyCheck {
- public:
+public:
   VirtualInheritanceCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
@@ -27,4 +27,4 @@ class VirtualInheritanceCheck : public ClangTidyCheck {
 
 } // namespace clang::tidy::fuchsia
 
-#endif  // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_FUCHSIA_VIRTUAL_INHERITANCE_H
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_FUCHSIA_VIRTUAL_INHERITANCE_H
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
index 3109bbb3724c..e076b39b5d97 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
@@ -120,8 +120,8 @@ void AvoidCStyleCastsCheck::check(const MatchFinder::MatchResult &Result) {
       IsFunction(SourceTypeAsWritten) && IsFunction(DestTypeAsWritten);
 
   const bool ConstructorCast = !CastExpr->getTypeAsWritten().hasQualifiers() &&
-      DestTypeAsWritten->isRecordType() &&
-      !DestTypeAsWritten->isElaboratedTypeSpecifier();
+                               DestTypeAsWritten->isRecordType() &&
+                               !DestTypeAsWritten->isElaboratedTypeSpecifier();
 
   if (CastExpr->getCastKind() == CK_NoOp && !FnToFnCast) {
     // Function pointer/reference casts may be needed to resolve ambiguities in
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidThrowingObjCExceptionCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidThrowingObjCExceptionCheck.h
index f8b191a37620..58b46e0a075a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidThrowingObjCExceptionCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/AvoidThrowingObjCExceptionCheck.h
@@ -20,7 +20,7 @@ namespace clang::tidy::google::objc {
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/google/objc-avoid-throwing-exception.html
 class AvoidThrowingObjCExceptionCheck : public ClangTidyCheck {
- public:
+public:
   AvoidThrowingObjCExceptionCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
@@ -32,4 +32,4 @@ class AvoidThrowingObjCExceptionCheck : public ClangTidyCheck {
 
 } // namespace clang::tidy::google::objc
 
-#endif  // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_GOOGLE_OBJC_AVOID_THROWING_EXCEPTION_H
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_GOOGLE_OBJC_AVOID_THROWING_EXCEPTION_H
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp
index 6f26de988135..3deea0620514 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp
@@ -85,7 +85,7 @@ void ExplicitConstructorCheck::check(const MatchFinder::MatchResult &Result) {
       "%0 explicit expression evaluates to 'false'";
 
   if (const auto *Conversion =
-      Result.Nodes.getNodeAs<CXXConversionDecl>("conversion")) {
+          Result.Nodes.getNodeAs<CXXConversionDecl>("conversion")) {
     if (Conversion->isOutOfLine())
       return;
     SourceLocation Loc = Conversion->getLocation();
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp
index cba8d71ab6a3..459dee124752 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp
@@ -1,4 +1,4 @@
-//===--- GlobalNamesInHeadersCheck.cpp - clang-tidy -----------------*- C++ -*-===//
+//===--- GlobalNamesInHeadersCheck.cpp - clang-tidy --------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.cpp
index 018d49fa5a22..9082c9368d87 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.cpp
@@ -52,7 +52,7 @@ FixItHint generateFixItHint(const VarDecl *Decl, bool IsConst) {
       CharSourceRange::getTokenRange(SourceRange(Decl->getLocation())),
       llvm::StringRef(NewName));
 }
-}  // namespace
+} // namespace
 
 void GlobalVariableDeclarationCheck::registerMatchers(MatchFinder *Finder) {
   // need to add two matchers since we need to bind different ids to distinguish
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.h
index 550224b26d14..19e6c5dbc8e2 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/GlobalVariableDeclarationCheck.h
@@ -20,7 +20,7 @@ namespace clang::tidy::google::objc {
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/google/objc-global-variable-declaration.html
 class GlobalVariableDeclarationCheck : public ClangTidyCheck {
- public:
+public:
   GlobalVariableDeclarationCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
@@ -32,4 +32,4 @@ class GlobalVariableDeclarationCheck : public ClangTidyCheck {
 
 } // namespace clang::tidy::google::objc
 
-#endif  // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_GOOGLE_OBJC_GLOBAL_VARIABLE_DECLARATION_H
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_GOOGLE_OBJC_GLOBAL_VARIABLE_DECLARATION_H
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/GoogleTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/GoogleTidyModule.cpp
index 830a37af1acc..5343e2b3a597 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/GoogleTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/GoogleTidyModule.cpp
@@ -35,7 +35,7 @@ namespace clang::tidy {
 namespace google {
 
 class GoogleModule : public ClangTidyModule {
- public:
+public:
   void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
     CheckFactories.registerCheck<build::ExplicitMakePairCheck>(
         "google-build-explicit-make-pair");
@@ -96,10 +96,10 @@ class GoogleModule : public ClangTidyModule {
 static ClangTidyModuleRegistry::Add<GoogleModule> X("google-module",
                                                     "Adds Google lint checks.");
 
-}  // namespace google
+} // namespace google
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the GoogleModule.
-volatile int GoogleModuleAnchorSource = 0;
+volatile int GoogleModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp
index 359d8efd100b..711444fa9bcd 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp
@@ -172,8 +172,8 @@ void IntegerTypesCheck::check(const MatchFinder::MatchResult &Result) {
   // We don't add a fix-it as changing the type can easily break code,
   // e.g. when a function requires a 'long' argument on all platforms.
   // QualTypes are printed with implicit quotes.
-  diag(Loc, "consider replacing %0 with '%1'") << BuiltinLoc.getType()
-                                               << Replacement;
+  diag(Loc, "consider replacing %0 with '%1'")
+      << BuiltinLoc.getType() << Replacement;
 }
 
 } // namespace tidy::google::runtime
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp
index 3db27868fcca..c1e421308d77 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp
@@ -21,8 +21,8 @@ UnnamedNamespaceInHeaderCheck::UnnamedNamespaceInHeaderCheck(
 
 void UnnamedNamespaceInHeaderCheck::registerMatchers(
     ast_matchers::MatchFinder *Finder) {
-    Finder->addMatcher(namespaceDecl(isAnonymous()).bind("anonymousNamespace"),
-                       this);
+  Finder->addMatcher(namespaceDecl(isAnonymous()).bind("anonymousNamespace"),
+                     this);
 }
 
 void UnnamedNamespaceInHeaderCheck::check(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp
index e9734e54a13e..26d27c19f489 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp
@@ -16,7 +16,7 @@ namespace clang::tidy::google::build {
 
 void UsingNamespaceDirectiveCheck::registerMatchers(
     ast_matchers::MatchFinder *Finder) {
-    Finder->addMatcher(usingDirectiveDecl().bind("usingNamespace"), this);
+  Finder->addMatcher(usingDirectiveDecl().bind("usingNamespace"), this);
 }
 
 void UsingNamespaceDirectiveCheck::check(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.cpp
index 2b2acfdf5b08..ed39568ea554 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.cpp
@@ -24,14 +24,12 @@ void ExceptionBaseclassCheck::registerMatchers(MatchFinder *Finder) {
                   isSameOrDerivedFrom(hasName("::std::exception")))))))))),
           // This condition is always true, but will bind to the
           // template value if the thrown type is templated.
-          anyOf(has(expr(
-                    hasType(substTemplateTypeParmType().bind("templ_type")))),
-                anything()),
+          optionally(has(
+              expr(hasType(substTemplateTypeParmType().bind("templ_type"))))),
           // Bind to the declaration of the type of the value that
-          // is thrown. 'anything()' is necessary to always succeed
-          // in the 'eachOf' because builtin types are not
-          // 'namedDecl'.
-          eachOf(has(expr(hasType(namedDecl().bind("decl")))), anything()))
+          // is thrown. 'optionally' is necessary because builtin types
+          // are not 'namedDecl'.
+          optionally(has(expr(hasType(namedDecl().bind("decl"))))))
           .bind("bad_throw"),
       this);
 }
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h
index 036db35a2290..79d8cf925d1b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/ExceptionBaseclassCheck.h
@@ -13,7 +13,8 @@
 
 namespace clang::tidy::hicpp {
 
-/// Check for thrown exceptions and enforce they are all derived from std::exception.
+/// Check for thrown exceptions and enforce they are all derived from
+/// std::exception.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/hicpp/exception-baseclass.html
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp
index 38f3ab5ae85d..65a56be3e5a0 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp
@@ -119,6 +119,6 @@ static ClangTidyModuleRegistry::Add<HICPPModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the HICPPModule.
-volatile int HICPPModuleAnchorSource = 0;
+volatile int HICPPModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.cpp
index 7028c3958f10..3f5cd4b47390 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/hicpp/MultiwayPathsCoveredCheck.cpp
@@ -113,7 +113,7 @@ void MultiwayPathsCoveredCheck::check(const MatchFinder::MatchResult &Result) {
   }
   // Warns for degenerated 'switch' statements that neither define a case nor
   // a default label.
-  // FIXME: Evaluate, if emitting a fix-it to simplify that statement is 
+  // FIXME: Evaluate, if emitting a fix-it to simplify that statement is
   // reasonable.
   if (!SwitchHasDefault && SwitchCaseCount == 0) {
     diag(Switch->getBeginLoc(),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/LinuxKernelTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/LinuxKernelTidyModule.cpp
index e6d64ff61d26..b8b75b7ccaef 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/LinuxKernelTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/LinuxKernelTidyModule.cpp
@@ -30,6 +30,7 @@ static ClangTidyModuleRegistry::Add<LinuxKernelModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the LinuxKernelModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int LinuxKernelModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.cpp
index 149d0a94eaa5..ce501ac2acca 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.cpp
@@ -19,8 +19,7 @@ void MustCheckErrsCheck::registerMatchers(MatchFinder *Finder) {
                               "ERR_CAST", "PTR_ERR_OR_ZERO"));
   auto NonCheckingStmts = stmt(anyOf(compoundStmt(), labelStmt()));
   Finder->addMatcher(
-      callExpr(callee(ErrFn), hasParent(NonCheckingStmts)).bind("call"),
-      this);
+      callExpr(callee(ErrFn), hasParent(NonCheckingStmts)).bind("call"), this);
 
   auto ReturnToCheck = returnStmt(hasReturnValue(callExpr(callee(ErrFn))));
   auto ReturnsErrFn = functionDecl(hasDescendant(ReturnToCheck));
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp
index e749163699b3..ceebde1595e7 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/llvm/LLVMTidyModule.cpp
@@ -57,6 +57,6 @@ static ClangTidyModuleRegistry::Add<LLVMModule> X("llvm-module",
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the LLVMModule.
-volatile int LLVMModuleAnchorSource = 0;
+volatile int LLVMModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
index 933333777770..4c138bcc564d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
@@ -14,12 +14,11 @@
 
 using namespace clang::ast_matchers;
 
-namespace clang {
-namespace ast_matchers {
-AST_MATCHER(Expr, isMacroID) { return Node.getExprLoc().isMacroID(); }
-} // namespace ast_matchers
+namespace clang::tidy::llvm_check {
 
-namespace tidy::llvm_check {
+namespace {
+AST_MATCHER(Expr, isMacroID) { return Node.getExprLoc().isMacroID(); }
+} // namespace
 
 void PreferIsaOrDynCastInConditionalsCheck::registerMatchers(
     MatchFinder *Finder) {
@@ -125,5 +124,4 @@ void PreferIsaOrDynCastInConditionalsCheck::check(
   }
 }
 
-} // namespace tidy::llvm_check
-} // namespace clang
+} // namespace clang::tidy::llvm_check
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.h
index dd9c5fec6af5..5b611096c25f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.h
@@ -15,8 +15,9 @@ namespace clang::tidy::llvm_check {
 
 /// Looks at conditionals and finds and replaces cases of ``cast<>``, which will
 /// assert rather than return a null pointer, and ``dyn_cast<>`` where
-/// the return value is not captured.  Additionally, finds and replaces cases that match the
-/// pattern ``var && isa<X>(var)``, where ``var`` is evaluated twice.
+/// the return value is not captured.  Additionally, finds and replaces cases
+/// that match the pattern ``var && isa<X>(var)``, where ``var`` is evaluated
+/// twice.
 ///
 /// Finds cases like these:
 /// \code
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/CalleeNamespaceCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/CalleeNamespaceCheck.cpp
index d14cd2721d9a..4bc4d5a4691f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/CalleeNamespaceCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/CalleeNamespaceCheck.cpp
@@ -19,7 +19,7 @@ namespace clang::tidy::llvm_libc {
 
 // Gets the outermost namespace of a DeclContext, right under the Translation
 // Unit.
-const DeclContext *getOutermostNamespace(const DeclContext *Decl) {
+static const DeclContext *getOutermostNamespace(const DeclContext *Decl) {
   const DeclContext *Parent = Decl->getParent();
   if (Parent->isTranslationUnit())
     return Decl;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/LLVMLibcTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/LLVMLibcTidyModule.cpp
index 7f26840be737..562d71a0891c 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/LLVMLibcTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/llvmlibc/LLVMLibcTidyModule.cpp
@@ -39,6 +39,7 @@ static ClangTidyModuleRegistry::Add<LLVMLibcModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the LLVMLibcModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int LLVMLibcModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
index 54bcebca7e18..6ddebcbc0e15 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
@@ -92,6 +92,6 @@ static ClangTidyModuleRegistry::Add<misc::MiscModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the MiscModule.
-volatile int MiscModuleAnchorSource = 0;
+volatile int MiscModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp
index faff1c17fc61..37fbd8c0d725 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp
@@ -38,8 +38,7 @@ void StaticAssertCheck::registerMatchers(MatchFinder *Finder) {
       binaryOperator(
           hasAnyOperatorName("&&", "=="),
           hasEitherOperand(ignoringImpCasts(stringLiteral().bind("assertMSG"))),
-          anyOf(binaryOperator(hasEitherOperand(IsAlwaysFalseWithCast)),
-                anything()))
+          optionally(binaryOperator(hasEitherOperand(IsAlwaysFalseWithCast))))
           .bind("assertExprRoot"),
       IsAlwaysFalse);
   auto NonConstexprFunctionCall =
@@ -52,12 +51,10 @@ void StaticAssertCheck::registerMatchers(MatchFinder *Finder) {
   auto NonConstexprCode =
       expr(anyOf(NonConstexprFunctionCall, NonConstexprVariableReference));
   auto AssertCondition =
-      expr(
-          anyOf(expr(ignoringParenCasts(anyOf(
-                    AssertExprRoot, unaryOperator(hasUnaryOperand(
-                                        ignoringParenCasts(AssertExprRoot)))))),
-                anything()),
-          unless(NonConstexprCode), unless(hasDescendant(NonConstexprCode)))
+      expr(optionally(expr(ignoringParenCasts(anyOf(
+               AssertExprRoot, unaryOperator(hasUnaryOperand(
+                                   ignoringParenCasts(AssertExprRoot))))))),
+           unless(NonConstexprCode), unless(hasDescendant(NonConstexprCode)))
           .bind("condition");
   auto Condition =
       anyOf(ignoringParenImpCasts(callExpr(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/misc/ThrowByValueCatchByReferenceCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/misc/ThrowByValueCatchByReferenceCheck.h
index 0d4df97c7bdf..e3cc4c5e6cd4 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/misc/ThrowByValueCatchByReferenceCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/misc/ThrowByValueCatchByReferenceCheck.h
@@ -13,7 +13,7 @@
 
 namespace clang::tidy::misc {
 
-///checks for locations that do not throw by value
+/// Checks for locations that do not throw by value
 // or catch by reference.
 // The check is C++ only. It checks that all throw locations
 // throw by value and not by pointer. Additionally it
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
index a41524af5680..c9477327742d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
@@ -452,8 +452,8 @@ static bool isFixitSupported(const CallableInfo &Callee,
   return true;
 }
 
-const FunctionDecl *getCallOperator(const CXXRecordDecl *Callable,
-                                    size_t NumArgs) {
+static const FunctionDecl *getCallOperator(const CXXRecordDecl *Callable,
+                                           size_t NumArgs) {
   std::vector<const FunctionDecl *> Candidates =
       findCandidateCallOperators(Callable, NumArgs);
   if (Candidates.size() != 1)
@@ -462,7 +462,7 @@ const FunctionDecl *getCallOperator(const CXXRecordDecl *Callable,
   return Candidates.front();
 }
 
-const FunctionDecl *
+static const FunctionDecl *
 getCallMethodDecl(const MatchFinder::MatchResult &Result, CallableType Type,
                   CallableMaterializationKind Materialization) {
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.h
index eb791ec12814..09cfebef48d4 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.h
@@ -13,8 +13,8 @@
 
 namespace clang::tidy::modernize {
 
-/// This check warns the uses of the deprecated member types of ``std::ios_base``
-/// and replaces those that have a non-deprecated equivalent.
+/// This check warns the uses of the deprecated member types of
+/// ``std::ios_base`` and replaces those that have a non-deprecated equivalent.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/modernize/deprecated-ios-base-aliases.html
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
index eb11ae616202..6c6c626ec4fe 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
@@ -114,7 +114,7 @@ arrayConditionMatcher(internal::Matcher<Expr> LimitExpr) {
 /// Client code will need to make sure that:
 ///   - The index variable is only used as an array index.
 ///   - All arrays indexed by the loop are the same.
-StatementMatcher makeArrayLoopMatcher() {
+static StatementMatcher makeArrayLoopMatcher() {
   StatementMatcher ArrayBoundMatcher =
       expr(hasType(isInteger())).bind(ConditionBoundName);
 
@@ -155,7 +155,7 @@ StatementMatcher makeArrayLoopMatcher() {
 ///
 /// Client code will need to make sure that:
 ///   - The two containers on which 'begin' and 'end' are called are the same.
-StatementMatcher makeIteratorLoopMatcher(bool IsReverse) {
+static StatementMatcher makeIteratorLoopMatcher(bool IsReverse) {
 
   auto BeginNameMatcher = IsReverse ? hasAnyName("rbegin", "crbegin")
                                     : hasAnyName("begin", "cbegin");
@@ -267,7 +267,7 @@ StatementMatcher makeIteratorLoopMatcher(bool IsReverse) {
 ///   - The index variable is only used in overloaded operator[] or
 ///     container.at().
 ///   - The container's iterators would not be invalidated during the loop.
-StatementMatcher makePseudoArrayLoopMatcher() {
+static StatementMatcher makePseudoArrayLoopMatcher() {
   // Test that the incoming type has a record declaration that has methods
   // called 'begin' and 'end'. If the incoming type is const, then make sure
   // these methods are also marked const.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
index 93c231b0bdf0..3d0a1f01725f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
@@ -688,9 +688,8 @@ bool ForLoopIndexUseVisitor::TraverseArraySubscriptExpr(ArraySubscriptExpr *E) {
   if (!isIndexInSubscriptExpr(E->getIdx(), IndexVar))
     return VisitorBase::TraverseArraySubscriptExpr(E);
 
-  if ((ContainerExpr &&
-       !areSameExpr(Context, Arr->IgnoreParenImpCasts(),
-                    ContainerExpr->IgnoreParenImpCasts())) ||
+  if ((ContainerExpr && !areSameExpr(Context, Arr->IgnoreParenImpCasts(),
+                                     ContainerExpr->IgnoreParenImpCasts())) ||
       !arrayMatchesBoundExpr(Context, Arr->IgnoreImpCasts()->getType(),
                              ArrayBoundExpr)) {
     // If we have already discovered the array being indexed and this isn't it
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
index ae88ec2abaf7..c2db858f72e3 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
@@ -161,7 +161,7 @@ class MacroToEnumCallbacks : public PPCallbacks {
     checkName(MacroNameTok);
   }
   void Elifdef(SourceLocation Loc, SourceRange ConditionRange,
-      SourceLocation IfLoc) override {
+               SourceLocation IfLoc) override {
     PPCallbacks::Elifdef(Loc, ConditionRange, IfLoc);
   }
   void Elifndef(SourceLocation Loc, const Token &MacroNameTok,
@@ -169,7 +169,7 @@ class MacroToEnumCallbacks : public PPCallbacks {
     checkName(MacroNameTok);
   }
   void Elifndef(SourceLocation Loc, SourceRange ConditionRange,
-      SourceLocation IfLoc) override {
+                SourceLocation IfLoc) override {
     PPCallbacks::Elifndef(Loc, ConditionRange, IfLoc);
   }
   void Endif(SourceLocation Loc, SourceLocation IfLoc) override;
@@ -316,8 +316,7 @@ void MacroToEnumCallbacks::FileChanged(SourceLocation Loc,
   CurrentFile = &Files.back();
 }
 
-bool MacroToEnumCallbacks::isInitializer(ArrayRef<Token> MacroTokens)
-{
+bool MacroToEnumCallbacks::isInitializer(ArrayRef<Token> MacroTokens) {
   IntegralLiteralExpressionMatcher Matcher(MacroTokens, LangOpts.C99 == 0);
   bool Matched = Matcher.match();
   bool IsC = !LangOpts.CPlusPlus;
@@ -328,7 +327,6 @@ bool MacroToEnumCallbacks::isInitializer(ArrayRef<Token> MacroTokens)
   return Matched;
 }
 
-
 // Any defined but rejected macro is scanned for identifiers that
 // are to be excluded as enums.
 void MacroToEnumCallbacks::MacroDefined(const Token &MacroNameTok,
@@ -444,8 +442,8 @@ void MacroToEnumCallbacks::invalidateExpressionNames() {
 }
 
 void MacroToEnumCallbacks::EndOfMainFile() {
-    invalidateExpressionNames();
-    issueDiagnostics();
+  invalidateExpressionNames();
+  issueDiagnostics();
 }
 
 void MacroToEnumCallbacks::invalidateRange(SourceRange Range) {
@@ -517,7 +515,8 @@ void MacroToEnumCallbacks::fixEnumMacro(const MacroList &MacroList) const {
 void MacroToEnumCheck::registerPPCallbacks(const SourceManager &SM,
                                            Preprocessor *PP,
                                            Preprocessor *ModuleExpanderPP) {
-  auto Callback = std::make_unique<MacroToEnumCallbacks>(this, getLangOpts(), SM);
+  auto Callback =
+      std::make_unique<MacroToEnumCallbacks>(this, getLangOpts(), SM);
   PPCallback = Callback.get();
   PP->addPPCallbacks(std::move(Callback));
 }
@@ -540,7 +539,7 @@ void MacroToEnumCheck::check(
     const ast_matchers::MatchFinder::MatchResult &Result) {
   auto *TLDecl = Result.Nodes.getNodeAs<Decl>("top");
   if (TLDecl == nullptr)
-      return;
+    return;
 
   SourceRange Range = TLDecl->getSourceRange();
   if (auto *TemplateFn = Result.Nodes.getNodeAs<FunctionTemplateDecl>("top")) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
index d1d7e9dcfa9c..deef3586628c 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
@@ -361,8 +361,7 @@ bool MakeSmartPtrCheck::replaceNew(DiagnosticBuilder &Diag,
       Diag << FixItHint::CreateRemoval(
           SourceRange(NewStart, InitRange.getBegin()));
       Diag << FixItHint::CreateRemoval(SourceRange(InitRange.getEnd(), NewEnd));
-    }
-    else {
+    } else {
       // New array expression with default/value initialization:
       //   smart_ptr<Foo[]>(new int[5]());
       //   smart_ptr<Foo[]>(new Foo[5]());
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
index fc46c72982fd..0cf59b6e0216 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
@@ -110,11 +110,11 @@ class ModernizeModule : public ClangTidyModule {
     CheckFactories.registerCheck<UseDefaultMemberInitCheck>(
         "modernize-use-default-member-init");
     CheckFactories.registerCheck<UseEmplaceCheck>("modernize-use-emplace");
-    CheckFactories.registerCheck<UseEqualsDefaultCheck>("modernize-use-equals-default");
+    CheckFactories.registerCheck<UseEqualsDefaultCheck>(
+        "modernize-use-equals-default");
     CheckFactories.registerCheck<UseEqualsDeleteCheck>(
         "modernize-use-equals-delete");
-    CheckFactories.registerCheck<UseNodiscardCheck>(
-        "modernize-use-nodiscard");
+    CheckFactories.registerCheck<UseNodiscardCheck>("modernize-use-nodiscard");
     CheckFactories.registerCheck<UseNoexceptCheck>("modernize-use-noexcept");
     CheckFactories.registerCheck<UseNullptrCheck>("modernize-use-nullptr");
     CheckFactories.registerCheck<UseOverrideCheck>("modernize-use-override");
@@ -136,6 +136,7 @@ static ClangTidyModuleRegistry::Add<ModernizeModule> X("modernize-module",
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the ModernizeModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int ModernizeModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/RedundantVoidArgCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/RedundantVoidArgCheck.cpp
index 53447c2b960f..5eebccc366fd 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/RedundantVoidArgCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/RedundantVoidArgCheck.cpp
@@ -112,7 +112,8 @@ void RedundantVoidArgCheck::processFunctionDecl(
                              "function declaration");
 }
 
-bool isMacroIdentifier(const IdentifierTable &Idents, const Token &ProtoToken) {
+static bool isMacroIdentifier(const IdentifierTable &Idents,
+                              const Token &ProtoToken) {
   if (!ProtoToken.is(tok::TokenKind::raw_identifier))
     return false;
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
index 4587b0867078..1ad31d315dc2 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
@@ -141,8 +141,7 @@ void ReplaceAutoPtrCheck::check(const MatchFinder::MatchResult &Result) {
       "auto_ptr")
     return;
 
-  SourceLocation EndLoc =
-      AutoPtrLoc.getLocWithOffset(strlen("auto_ptr") - 1);
+  SourceLocation EndLoc = AutoPtrLoc.getLocWithOffset(strlen("auto_ptr") - 1);
   diag(AutoPtrLoc, "auto_ptr is deprecated, use unique_ptr instead")
       << FixItHint::CreateReplacement(SourceRange(AutoPtrLoc, EndLoc),
                                       "unique_ptr");
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
index 7a2d804e173c..f4b63087b723 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
@@ -38,12 +38,11 @@ size_t getTypeNameLength(bool RemoveStars, StringRef Text) {
     else if (C == '>')
       --TemplateTypenameCntr;
     const CharType NextChar =
-        isAlphanumeric(C)
-            ? Alpha
-            : (isWhitespace(C) ||
-               (!RemoveStars && TemplateTypenameCntr == 0 && C == '*'))
-                  ? Space
-                  : Punctuation;
+        isAlphanumeric(C) ? Alpha
+        : (isWhitespace(C) ||
+           (!RemoveStars && TemplateTypenameCntr == 0 && C == '*'))
+            ? Space
+            : Punctuation;
     if (NextChar != Space) {
       ++NumChars; // Count the non-space character.
       if (LastChar == Space && NextChar == Alpha && BeforeSpace == Alpha)
@@ -444,10 +443,10 @@ void UseAutoCheck::check(const MatchFinder::MatchResult &Result) {
     replaceIterators(Decl, Result.Context);
   } else if (const auto *Decl =
                  Result.Nodes.getNodeAs<DeclStmt>(DeclWithNewId)) {
-    replaceExpr(Decl, Result.Context,
-                [](const Expr *Expr) { return Expr->getType(); },
-                "use auto when initializing with new to avoid "
-                "duplicating the type name");
+    replaceExpr(
+        Decl, Result.Context, [](const Expr *Expr) { return Expr->getType(); },
+        "use auto when initializing with new to avoid "
+        "duplicating the type name");
   } else if (const auto *Decl =
                  Result.Nodes.getNodeAs<DeclStmt>(DeclWithCastId)) {
     replaceExpr(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp
index c8e6bf47bb82..339462093a6d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp
@@ -26,13 +26,12 @@ void UseBoolLiteralsCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 
 void UseBoolLiteralsCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
-      traverse(
-          TK_AsIs,
-          implicitCastExpr(
-              has(ignoringParenImpCasts(integerLiteral().bind("literal"))),
-              hasImplicitDestinationType(qualType(booleanType())),
-              unless(isInTemplateInstantiation()),
-              anyOf(hasParent(explicitCastExpr().bind("cast")), anything()))),
+      traverse(TK_AsIs,
+               implicitCastExpr(
+                   has(ignoringParenImpCasts(integerLiteral().bind("literal"))),
+                   hasImplicitDestinationType(qualType(booleanType())),
+                   unless(isInTemplateInstantiation()),
+                   optionally(hasParent(explicitCastExpr().bind("cast"))))),
       this);
 
   Finder->addMatcher(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
index 6040cddf0e52..9e4d184c4b6e 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
@@ -279,7 +279,7 @@ findInsertionForConstraint(const FunctionDecl *Function, ASTContext &Context) {
   return Body->getBeginLoc();
 }
 
-bool isPrimaryExpression(const Expr *Expression) {
+static bool isPrimaryExpression(const Expr *Expression) {
   // This function is an incomplete approximation of checking whether
   // an Expr is a primary expression. In particular, if this function
   // returns true, the expression is a primary expression. The converse
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
index 430455a38f39..aaf24eaa33c1 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
@@ -98,8 +98,8 @@ auto hasWantedType(llvm::ArrayRef<StringRef> TypeNames) {
 
 // Matches member call expressions of the named method on the listed container
 // types.
-auto cxxMemberCallExprOnContainer(
-    StringRef MethodName, llvm::ArrayRef<StringRef> ContainerNames) {
+auto cxxMemberCallExprOnContainer(StringRef MethodName,
+                                  llvm::ArrayRef<StringRef> ContainerNames) {
   return cxxMemberCallExpr(
       hasDeclaration(functionDecl(hasName(MethodName))),
       on(hasTypeOrPointeeType(hasWantedType(ContainerNames))));
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp
index 93151024064b..b361ae445653 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp
@@ -48,8 +48,8 @@ static std::set<const Type *> getAllDirectBases(const CXXRecordDecl *Record) {
 /// Returns a matcher that matches member expressions where the base is
 /// the variable declared as \p Var and the accessed member is the one declared
 /// as \p Field.
-internal::Matcher<Expr> accessToFieldInVar(const FieldDecl *Field,
-                                           const ValueDecl *Var) {
+static internal::Matcher<Expr> accessToFieldInVar(const FieldDecl *Field,
+                                                  const ValueDecl *Var) {
   return ignoringImpCasts(
       memberExpr(hasObjectExpression(declRefExpr(to(varDecl(equalsNode(Var))))),
                  member(fieldDecl(equalsNode(Field)))));
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.h
index 44f26c2bcfc5..04c2177704fb 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.h
@@ -1,4 +1,4 @@
-//===--- UseEqualsDefaultCheck.h - clang-tidy--------------------------*- C++ -*-===//
+//===--- UseEqualsDefaultCheck.h - clang-tidy---------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.h
index 8545aa2a6396..64f60351c065 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.h
@@ -1,4 +1,4 @@
-//===--- UseEqualsDeleteCheck.h - clang-tidy---------------------------*- C++ -*-===//
+//===--- UseEqualsDeleteCheck.h - clang-tidy----------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp
index a6b00be75abf..c38fb3a01d28 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp
@@ -229,7 +229,7 @@ class CastSequenceVisitor : public RecursiveASTVisitor<CastSequenceVisitor> {
       return true;
     }
 
-    auto* CastSubExpr = C->getSubExpr()->IgnoreParens();
+    auto *CastSubExpr = C->getSubExpr()->IgnoreParens();
     // Ignore cast expressions which cast nullptr literal.
     if (isa<CXXNullPtrLiteralExpr>(CastSubExpr)) {
       return true;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
index 2e059f24d47b..2af67f7ccb4c 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
@@ -33,6 +33,8 @@ static bool isNegativeComparison(const Expr *ComparisonExpr) {
   return false;
 }
 
+namespace {
+
 struct NotLengthExprForStringNode {
   NotLengthExprForStringNode(std::string ID, DynTypedNode Node,
                              ASTContext *Context)
@@ -91,6 +93,8 @@ AST_MATCHER_P(Expr, lengthExprForStringNode, std::string, ID) {
       ID, DynTypedNode::create(Node), &(Finder->getASTContext())));
 }
 
+} // namespace
+
 UseStartsEndsWithCheck::UseStartsEndsWithCheck(StringRef Name,
                                                ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context) {}
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
index 4e5c6a798be9..ced4825f79a9 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
@@ -17,6 +17,30 @@
 #include <cctype>
 #include <optional>
 
+namespace clang::tidy {
+
+template <>
+struct OptionEnumMapping<
+    modernize::UseTrailingReturnTypeCheck::TransformLambda> {
+  static llvm::ArrayRef<std::pair<
+      modernize::UseTrailingReturnTypeCheck::TransformLambda, StringRef>>
+  getEnumMapping() {
+    static constexpr std::pair<
+        modernize::UseTrailingReturnTypeCheck::TransformLambda, StringRef>
+        Mapping[] = {
+            {modernize::UseTrailingReturnTypeCheck::TransformLambda::All,
+             "all"},
+            {modernize::UseTrailingReturnTypeCheck::TransformLambda::
+                 AllExceptAuto,
+             "all_except_auto"},
+            {modernize::UseTrailingReturnTypeCheck::TransformLambda::None,
+             "none"}};
+    return Mapping;
+  }
+};
+
+} // namespace clang::tidy
+
 using namespace clang::ast_matchers;
 
 namespace clang::tidy::modernize {
@@ -111,10 +135,17 @@ struct UnqualNameVisitor : public RecursiveASTVisitor<UnqualNameVisitor> {
 private:
   const FunctionDecl &F;
 };
+
+AST_MATCHER(LambdaExpr, hasExplicitResultType) {
+  return Node.hasExplicitResultType();
+}
+
 } // namespace
 
 constexpr llvm::StringLiteral ErrorMessageOnFunction =
     "use a trailing return type for this function";
+constexpr llvm::StringLiteral ErrorMessageOnLambda =
+    "use a trailing return type for this lambda";
 
 static SourceLocation expandIfMacroId(SourceLocation Loc,
                                       const SourceManager &SM) {
@@ -285,7 +316,6 @@ findReturnTypeAndCVSourceRange(const FunctionDecl &F, const TypeLoc &ReturnLoc,
     return {};
   }
 
-
   // If the return type has no local qualifiers, it's source range is accurate.
   if (!hasAnyNestedLocalQualifiers(F.getReturnType()))
     return ReturnTypeRange;
@@ -329,6 +359,48 @@ findReturnTypeAndCVSourceRange(const FunctionDecl &F, const TypeLoc &ReturnLoc,
   return ReturnTypeRange;
 }
 
+static SourceLocation findLambdaTrailingReturnInsertLoc(
+    const CXXMethodDecl *Method, const SourceManager &SM,
+    const LangOptions &LangOpts, const ASTContext &Ctx) {
+  // 'requires' keyword is present in lambda declaration
+  if (Method->getTrailingRequiresClause()) {
+    SourceLocation ParamEndLoc;
+    if (Method->param_empty())
+      ParamEndLoc = Method->getBeginLoc();
+    else
+      ParamEndLoc = Method->getParametersSourceRange().getEnd();
+
+    std::pair<FileID, unsigned> ParamEndLocInfo =
+        SM.getDecomposedLoc(ParamEndLoc);
+    StringRef Buffer = SM.getBufferData(ParamEndLocInfo.first);
+
+    Lexer Lexer(SM.getLocForStartOfFile(ParamEndLocInfo.first), LangOpts,
+                Buffer.begin(), Buffer.data() + ParamEndLocInfo.second,
+                Buffer.end());
+
+    Token Token;
+    while (!Lexer.LexFromRawLexer(Token)) {
+      if (Token.is(tok::raw_identifier)) {
+        IdentifierInfo &Info = Ctx.Idents.get(StringRef(
+            SM.getCharacterData(Token.getLocation()), Token.getLength()));
+        Token.setIdentifierInfo(&Info);
+        Token.setKind(Info.getTokenID());
+      }
+
+      if (Token.is(tok::kw_requires))
+        return Token.getLocation().getLocWithOffset(-1);
+    }
+
+    return {};
+  }
+
+  // If no requires clause, insert before the body
+  if (const Stmt *Body = Method->getBody())
+    return Body->getBeginLoc().getLocWithOffset(-1);
+
+  return {};
+}
+
 static void keepSpecifiers(std::string &ReturnType, std::string &Auto,
                            SourceRange ReturnTypeCVRange, const FunctionDecl &F,
                            const FriendDecl *Fr, const ASTContext &Ctx,
@@ -382,14 +454,43 @@ static void keepSpecifiers(std::string &ReturnType, std::string &Auto,
   }
 }
 
+UseTrailingReturnTypeCheck::UseTrailingReturnTypeCheck(
+    StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      TransformFunctions(Options.get("TransformFunctions", true)),
+      TransformLambdas(Options.get("TransformLambdas", TransformLambda::All)) {
+
+  if (TransformFunctions == false && TransformLambdas == TransformLambda::None)
+    this->configurationDiag(
+        "The check 'modernize-use-trailing-return-type' will not perform any "
+        "analysis because 'TransformFunctions' and 'TransformLambdas' are "
+        "disabled.");
+}
+
+void UseTrailingReturnTypeCheck::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "TransformFunctions", TransformFunctions);
+  Options.store(Opts, "TransformLambdas", TransformLambdas);
+}
+
 void UseTrailingReturnTypeCheck::registerMatchers(MatchFinder *Finder) {
-  auto F = functionDecl(
-               unless(anyOf(hasTrailingReturn(), returns(voidType()),
-                            cxxConversionDecl(), cxxMethodDecl(isImplicit()))))
-               .bind("Func");
+  auto F =
+      functionDecl(
+          unless(anyOf(
+              hasTrailingReturn(), returns(voidType()), cxxConversionDecl(),
+              cxxMethodDecl(
+                  anyOf(isImplicit(),
+                        hasParent(cxxRecordDecl(hasParent(lambdaExpr()))))))))
+          .bind("Func");
+
+  if (TransformFunctions) {
+    Finder->addMatcher(F, this);
+    Finder->addMatcher(friendDecl(hasDescendant(F)).bind("Friend"), this);
+  }
 
-  Finder->addMatcher(F, this);
-  Finder->addMatcher(friendDecl(hasDescendant(F)).bind("Friend"), this);
+  if (TransformLambdas != TransformLambda::None)
+    Finder->addMatcher(
+        lambdaExpr(unless(hasExplicitResultType())).bind("Lambda"), this);
 }
 
 void UseTrailingReturnTypeCheck::registerPPCallbacks(
@@ -401,8 +502,13 @@ void UseTrailingReturnTypeCheck::check(const MatchFinder::MatchResult &Result) {
   assert(PP && "Expected registerPPCallbacks() to have been called before so "
                "preprocessor is available");
 
-  const auto *F = Result.Nodes.getNodeAs<FunctionDecl>("Func");
+  if (const auto *Lambda = Result.Nodes.getNodeAs<LambdaExpr>("Lambda")) {
+    diagOnLambda(Lambda, Result);
+    return;
+  }
+
   const auto *Fr = Result.Nodes.getNodeAs<FriendDecl>("Friend");
+  const auto *F = Result.Nodes.getNodeAs<FunctionDecl>("Func");
   assert(F && "Matcher is expected to find only FunctionDecls");
 
   // Three-way comparison operator<=> is syntactic sugar and generates implicit
@@ -495,4 +601,41 @@ void UseTrailingReturnTypeCheck::check(const MatchFinder::MatchResult &Result) {
       << FixItHint::CreateInsertion(InsertionLoc, " -> " + ReturnType);
 }
 
+void UseTrailingReturnTypeCheck::diagOnLambda(
+    const LambdaExpr *Lambda,
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+
+  const CXXMethodDecl *Method = Lambda->getCallOperator();
+  if (!Method || Lambda->hasExplicitResultType())
+    return;
+
+  const ASTContext *Ctx = Result.Context;
+  const QualType ReturnType = Method->getReturnType();
+
+  // We can't write 'auto' in C++11 mode, try to write generic msg and bail out.
+  if (ReturnType->isDependentType() &&
+      Ctx->getLangOpts().LangStd == LangStandard::lang_cxx11) {
+    if (TransformLambdas == TransformLambda::All)
+      diag(Lambda->getBeginLoc(), ErrorMessageOnLambda);
+    return;
+  }
+
+  if (ReturnType->isUndeducedAutoType() &&
+      TransformLambdas == TransformLambda::AllExceptAuto)
+    return;
+
+  const SourceLocation TrailingReturnInsertLoc =
+      findLambdaTrailingReturnInsertLoc(Method, *Result.SourceManager,
+                                        getLangOpts(), *Result.Context);
+
+  if (TrailingReturnInsertLoc.isValid())
+    diag(Lambda->getBeginLoc(), "use a trailing return type for this lambda")
+        << FixItHint::CreateInsertion(
+               TrailingReturnInsertLoc,
+               " -> " +
+                   ReturnType.getAsString(Result.Context->getPrintingPolicy()));
+  else
+    diag(Lambda->getBeginLoc(), ErrorMessageOnLambda);
+}
+
 } // namespace clang::tidy::modernize
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h
index 7f20674d15a5..91369919c5d3 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.h
@@ -26,18 +26,25 @@ struct ClassifiedToken {
 /// http://clang.llvm.org/extra/clang-tidy/checks/modernize/use-trailing-return-type.html
 class UseTrailingReturnTypeCheck : public ClangTidyCheck {
 public:
-  UseTrailingReturnTypeCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
+  UseTrailingReturnTypeCheck(StringRef Name, ClangTidyContext *Context);
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
     return LangOpts.CPlusPlus11;
   }
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP,
                            Preprocessor *ModuleExpanderPP) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
+  enum TransformLambda { All, AllExceptAuto, None };
+
 private:
   Preprocessor *PP = nullptr;
+  const bool TransformFunctions;
+  const TransformLambda TransformLambdas;
+
+  void diagOnLambda(const LambdaExpr *Lambda,
+                    const ast_matchers::MatchFinder::MatchResult &Result);
 };
 
 } // namespace clang::tidy::modernize
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.h
index 6f02becc9f9d..80f022159c67 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.h
@@ -26,6 +26,7 @@ class UseTransparentFunctorsCheck : public ClangTidyCheck {
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+
 private:
   const bool SafeMode;
 };
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp
index 109d701395cc..1e0a0a551339 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.cpp
@@ -39,10 +39,10 @@ void UseUncaughtExceptionsCheck::registerMatchers(MatchFinder *Finder) {
                      this);
   // CallExpr in initialisation list: warning, fix-it with avoiding narrowing
   // conversions.
-  Finder->addMatcher(callExpr(DirectCallToUncaughtException,
-                              hasAncestor(initListExpr()))
-                         .bind("init_call_expr"),
-                     this);
+  Finder->addMatcher(
+      callExpr(DirectCallToUncaughtException, hasAncestor(initListExpr()))
+          .bind("init_call_expr"),
+      this);
 }
 
 void UseUncaughtExceptionsCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.h
index 79b475c4f850..48677521181f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/modernize/UseUncaughtExceptionsCheck.h
@@ -13,9 +13,10 @@
 
 namespace clang::tidy::modernize {
 
-/// This check will warn on calls to std::uncaught_exception and replace them with calls to
-/// std::uncaught_exceptions, since std::uncaught_exception was deprecated in C++17. In case of
-/// macro ID there will be only a warning without fixits.
+/// This check will warn on calls to std::uncaught_exception and replace them
+/// with calls to std::uncaught_exceptions, since std::uncaught_exception was
+/// deprecated in C++17. In case of macro ID there will be only a warning
+/// without fixits.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/modernize/use-uncaught-exceptions.html
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/mpi/MPITidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/mpi/MPITidyModule.cpp
index 166d71e130db..67ae101c18cb 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/mpi/MPITidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/mpi/MPITidyModule.cpp
@@ -31,6 +31,6 @@ static ClangTidyModuleRegistry::Add<mpi::MPIModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the MPIModule.
-volatile int MPIModuleAnchorSource = 0;
+volatile int MPIModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/objc/AvoidNSErrorInitCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/objc/AvoidNSErrorInitCheck.h
index 72a6c6512f62..b343cb0c4fa8 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/objc/AvoidNSErrorInitCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/objc/AvoidNSErrorInitCheck.h
@@ -19,7 +19,7 @@ namespace clang::tidy::objc {
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/objc/avoid-nserror-init.html
 class AvoidNSErrorInitCheck : public ClangTidyCheck {
- public:
+public:
   AvoidNSErrorInitCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
@@ -31,4 +31,4 @@ class AvoidNSErrorInitCheck : public ClangTidyCheck {
 
 } // namespace clang::tidy::objc
 
-#endif  // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_AVOIDNSERRORINITCHECK_H
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_AVOIDNSERRORINITCHECK_H
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/objc/ForbiddenSubclassingCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/objc/ForbiddenSubclassingCheck.cpp
index 579aa2965071..089538d4c65a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/objc/ForbiddenSubclassingCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/objc/ForbiddenSubclassingCheck.cpp
@@ -35,14 +35,11 @@ constexpr char DefaultForbiddenSuperClassNames[] =
 
 } // namespace
 
-ForbiddenSubclassingCheck::ForbiddenSubclassingCheck(
-    StringRef Name,
-    ClangTidyContext *Context)
+ForbiddenSubclassingCheck::ForbiddenSubclassingCheck(StringRef Name,
+                                                     ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      ForbiddenSuperClassNames(
-          utils::options::parseStringList(
-              Options.get("ClassNames", DefaultForbiddenSuperClassNames))) {
-}
+      ForbiddenSuperClassNames(utils::options::parseStringList(
+          Options.get("ClassNames", DefaultForbiddenSuperClassNames))) {}
 
 void ForbiddenSubclassingCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
@@ -53,27 +50,22 @@ void ForbiddenSubclassingCheck::registerMatchers(MatchFinder *Finder) {
       this);
 }
 
-void ForbiddenSubclassingCheck::check(
-    const MatchFinder::MatchResult &Result) {
-  const auto *SubClass = Result.Nodes.getNodeAs<ObjCInterfaceDecl>(
-      "subclass");
+void ForbiddenSubclassingCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *SubClass = Result.Nodes.getNodeAs<ObjCInterfaceDecl>("subclass");
   assert(SubClass != nullptr);
-  const auto *SuperClass = Result.Nodes.getNodeAs<ObjCInterfaceDecl>(
-      "superclass");
+  const auto *SuperClass =
+      Result.Nodes.getNodeAs<ObjCInterfaceDecl>("superclass");
   assert(SuperClass != nullptr);
   diag(SubClass->getLocation(),
        "Objective-C interface %0 subclasses %1, which is not "
        "intended to be subclassed")
-      << SubClass
-      << SuperClass;
+      << SubClass << SuperClass;
 }
 
 void ForbiddenSubclassingCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(
-      Opts,
-      "ForbiddenSuperClassNames",
-      utils::options::serializeStringList(ForbiddenSuperClassNames));
+  Options.store(Opts, "ForbiddenSuperClassNames",
+                utils::options::serializeStringList(ForbiddenSuperClassNames));
 }
 
 } // namespace clang::tidy::objc
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/objc/NSDateFormatterCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/objc/NSDateFormatterCheck.cpp
index 5438c9c892e2..79e9d97d9594 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/objc/NSDateFormatterCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/objc/NSDateFormatterCheck.cpp
@@ -34,7 +34,7 @@ static char ValidDatePatternChars[] = {
 // A string pattern is valid if all the letters(a-z, A-Z) in it belong to the
 // set of reserved characters. See:
 // https://www.unicode.org/reports/tr35/tr35.html#Invalid_Patterns
-bool isValidDatePattern(StringRef Pattern) {
+static bool isValidDatePattern(StringRef Pattern) {
   return llvm::all_of(Pattern, [](const auto &PatternChar) {
     return !isalpha(PatternChar) ||
            llvm::is_contained(ValidDatePatternChars, PatternChar);
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp
index b6f8888ed389..56ccf33a6362 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp
@@ -35,27 +35,24 @@ class ObjCModule : public ClangTidyModule {
         "objc-dealloc-in-category");
     CheckFactories.registerCheck<ForbiddenSubclassingCheck>(
         "objc-forbidden-subclassing");
-    CheckFactories.registerCheck<MissingHashCheck>(
-        "objc-missing-hash");
+    CheckFactories.registerCheck<MissingHashCheck>("objc-missing-hash");
     CheckFactories.registerCheck<NSDateFormatterCheck>("objc-nsdate-formatter");
     CheckFactories.registerCheck<NSInvocationArgumentLifetimeCheck>(
         "objc-nsinvocation-argument-lifetime");
     CheckFactories.registerCheck<PropertyDeclarationCheck>(
         "objc-property-declaration");
-    CheckFactories.registerCheck<SuperSelfCheck>(
-        "objc-super-self");
+    CheckFactories.registerCheck<SuperSelfCheck>("objc-super-self");
   }
 };
 
 // Register the ObjCTidyModule using this statically initialized variable.
-static ClangTidyModuleRegistry::Add<ObjCModule> X(
-    "objc-module",
-    "Adds Objective-C lint checks.");
+static ClangTidyModuleRegistry::Add<ObjCModule>
+    X("objc-module", "Adds Objective-C lint checks.");
 
 } // namespace objc
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the ObjCModule.
-volatile int ObjCModuleAnchorSource = 0;
+volatile int ObjCModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp
index cda24de49186..3b847f51d217 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp
@@ -88,7 +88,7 @@ bool prefixedPropertyNameValid(llvm::StringRef PropertyName) {
   auto RegexExp = llvm::Regex(llvm::StringRef(validPropertyNameRegex(false)));
   return RegexExp.match(PropertyName.substr(Start + 1));
 }
-}  // namespace
+} // namespace
 
 void PropertyDeclarationCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(objcPropertyDecl(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.cpp
index 603d0954e1b1..42fb95bf1052 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/openmp/ExceptionEscapeCheck.cpp
@@ -20,8 +20,7 @@ ExceptionEscapeCheck::ExceptionEscapeCheck(StringRef Name,
                                            ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       RawIgnoredExceptions(Options.get("IgnoredExceptions", "")) {
-  llvm::SmallVector<StringRef, 8> FunctionsThatShouldNotThrowVec,
-      IgnoredExceptionsVec;
+  llvm::SmallVector<StringRef, 8> IgnoredExceptionsVec;
 
   llvm::StringSet<> IgnoredExceptions;
   StringRef(RawIgnoredExceptions).split(IgnoredExceptionsVec, ",", -1, false);
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/openmp/OpenMPTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/openmp/OpenMPTidyModule.cpp
index 37092f49b25d..d9c9d9067340 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/openmp/OpenMPTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/openmp/OpenMPTidyModule.cpp
@@ -34,6 +34,6 @@ static ClangTidyModuleRegistry::Add<OpenMPModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the OpenMPModule.
-volatile int OpenMPModuleAnchorSource = 0;
+volatile int OpenMPModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h
index 93ed32e5b878..83af95cd6954 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/FasterStringFindCheck.h
@@ -25,7 +25,7 @@ namespace clang::tidy::performance {
 class FasterStringFindCheck : public ClangTidyCheck {
 public:
   FasterStringFindCheck(StringRef Name, ClangTidyContext *Context);
-  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
     return LangOpts.CPlusPlus;
   }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h
index ab9b89d31f93..8fabbfa2ae7b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.h
@@ -20,7 +20,7 @@ namespace clang::tidy::performance {
 class ForRangeCopyCheck : public ClangTidyCheck {
 public:
   ForRangeCopyCheck(StringRef Name, ClangTidyContext *Context);
-  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
     return LangOpts.CPlusPlus11;
   }
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp
index 86fca0722dcd..1ecf1e14957a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp
@@ -65,8 +65,7 @@ void ImplicitConversionInLoopCheck::check(
     const MatchFinder::MatchResult &Result) {
   const auto *VD = Result.Nodes.getNodeAs<VarDecl>("faulty-var");
   const auto *Init = Result.Nodes.getNodeAs<Expr>("init");
-  const auto *OperatorCall =
-      Result.Nodes.getNodeAs<Expr>("operator-call");
+  const auto *OperatorCall = Result.Nodes.getNodeAs<Expr>("operator-call");
 
   if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(Init))
     Init = Cleanup->getSubExpr();
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h
index 2f3c9ae8f2db..d1764070bd4d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.h
@@ -20,9 +20,9 @@ class ImplicitConversionInLoopCheck : public ClangTidyCheck {
 public:
   ImplicitConversionInLoopCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
-      bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
-        return LangOpts.CPlusPlus11;
-      }
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus11;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
index 94cb7ec38087..9692fcb0c1de 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
@@ -126,15 +126,14 @@ void InefficientVectorOperationCheck::addMatcher(
   //
   // FIXME: Support more types of counter-based loops like decrement loops.
   Finder->addMatcher(
-      forStmt(
-          hasLoopInit(LoopVarInit),
-          hasCondition(binaryOperator(
-              hasOperatorName("<"), hasLHS(RefersToLoopVar),
-              hasRHS(expr(unless(hasDescendant(expr(RefersToLoopVar))))
-                         .bind(LoopEndExprName)))),
-          hasIncrement(unaryOperator(hasOperatorName("++"),
-                                     hasUnaryOperand(RefersToLoopVar))),
-          HasInterestingLoopBody, InInterestingCompoundStmt)
+      forStmt(hasLoopInit(LoopVarInit),
+              hasCondition(binaryOperator(
+                  hasOperatorName("<"), hasLHS(RefersToLoopVar),
+                  hasRHS(expr(unless(hasDescendant(expr(RefersToLoopVar))))
+                             .bind(LoopEndExprName)))),
+              hasIncrement(unaryOperator(hasOperatorName("++"),
+                                         hasUnaryOperand(RefersToLoopVar))),
+              HasInterestingLoopBody, InInterestingCompoundStmt)
           .bind(LoopCounterName),
       this);
 
@@ -179,7 +178,7 @@ void InefficientVectorOperationCheck::registerMatchers(MatchFinder *Finder) {
 
 void InefficientVectorOperationCheck::check(
     const MatchFinder::MatchResult &Result) {
-  auto* Context = Result.Context;
+  auto *Context = Result.Context;
   if (Context->getDiagnostics().hasUncompilableErrorOccurred())
     return;
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
index 7a2745f7ae5f..4f45ff490633 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
@@ -24,7 +24,7 @@ namespace clang::tidy::performance {
 class InefficientVectorOperationCheck : public ClangTidyCheck {
 public:
   InefficientVectorOperationCheck(StringRef Name, ClangTidyContext *Context);
-  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
     return LangOpts.CPlusPlus;
   }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
index 75ef7a697031..f458e26d964b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
@@ -78,9 +78,9 @@ void MoveConstArgCheck::registerMatchers(MatchFinder *Finder) {
       this);
 }
 
-bool isRValueReferenceParam(const Expr *Invocation,
-                            const QualType *InvocationParmType,
-                            const Expr *Arg) {
+static bool isRValueReferenceParam(const Expr *Invocation,
+                                   const QualType *InvocationParmType,
+                                   const Expr *Arg) {
   if (Invocation && (*InvocationParmType)->isRValueReferenceType() &&
       Arg->isLValue()) {
     if (!Invocation->getType()->isRecordType())
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp
index 9e0fa6f88b36..10ad9ec6fef4 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp
@@ -81,6 +81,7 @@ static ClangTidyModuleRegistry::Add<PerformanceModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the PerformanceModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int PerformanceModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
index ce360dded863..b6f19811f5e5 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
@@ -111,10 +111,11 @@ AST_MATCHER_FUNCTION(StatementMatcher, isConstRefReturningFunctionCall) {
   // an alias to one of its arguments and the arguments need to be checked
   // for const use as well.
   return callExpr(argumentCountIs(0),
-                  callee(functionDecl(returns(hasCanonicalType(matchers::isReferenceToConst())),
+                  callee(functionDecl(returns(hasCanonicalType(
+                                          matchers::isReferenceToConst())),
                                       unless(cxxMethodDecl(unless(isStatic()))))
-                         .bind(FunctionDeclId)))
-         .bind(InitFunctionCallId);
+                             .bind(FunctionDeclId)))
+      .bind(InitFunctionCallId);
 }
 
 AST_MATCHER_FUNCTION_P(StatementMatcher, initializerReturnsReferenceToConst,
@@ -234,12 +235,14 @@ UnnecessaryCopyInitialization::UnnecessaryCopyInitialization(
           Options.get("ExcludedContainerTypes", ""))) {}
 
 void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) {
-  auto LocalVarCopiedFrom = [this](const ast_matchers::internal::Matcher<Expr> &CopyCtorArg) {
-    return compoundStmt(
-               forEachDescendant(
-                   declStmt(
-                       unless(has(decompositionDecl())),
-                       has(varDecl(hasLocalStorage(),
+  auto LocalVarCopiedFrom =
+      [this](const ast_matchers::internal::Matcher<Expr> &CopyCtorArg) {
+        return compoundStmt(
+                   forEachDescendant(
+                       declStmt(
+                           unless(has(decompositionDecl())),
+                           has(varDecl(
+                                   hasLocalStorage(),
                                    hasType(qualType(
                                        hasCanonicalType(allOf(
                                            matchers::isExpensiveToCopy(),
@@ -256,10 +259,10 @@ void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) {
                                                isCopyConstructor())),
                                            hasArgument(0, CopyCtorArg))
                                            .bind("ctorCall"))))
-                               .bind("newVarDecl")))
-                       .bind("declStmt")))
-        .bind("blockStmt");
-  };
+                                   .bind("newVarDecl")))
+                           .bind("declStmt")))
+            .bind("blockStmt");
+      };
 
   Finder->addMatcher(
       LocalVarCopiedFrom(anyOf(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h b/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h
index ab0f1ecf6106..38f756f9b452 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h
@@ -25,7 +25,7 @@ namespace clang::tidy::performance {
 class UnnecessaryCopyInitialization : public ClangTidyCheck {
 public:
   UnnecessaryCopyInitialization(StringRef Name, ClangTidyContext *Context);
-  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override{
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
     return LangOpts.CPlusPlus;
   }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp
index 3caa222b9fa4..651a63b3aa97 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp
@@ -78,6 +78,7 @@ class ClangTidyPluginAction : public PluginASTAction {
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the clang-tidy plugin.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int ClangTidyPluginAnchorSource = 0;
 
 static clang::FrontendPluginRegistry::Add<clang::tidy::ClangTidyPluginAction>
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/portability/PortabilityTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/portability/PortabilityTidyModule.cpp
index a15cb36dfdaf..98853556588b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/portability/PortabilityTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/portability/PortabilityTidyModule.cpp
@@ -42,6 +42,7 @@ static ClangTidyModuleRegistry::Add<PortabilityModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the PortabilityModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int PortabilityModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.h
index c254a374ee9d..92fc0af98a25 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.h
@@ -30,7 +30,7 @@ class SIMDIntrinsicsCheck : public ClangTidyCheck {
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
- private:
+private:
   llvm::SmallString<32> Std;
   const bool Suggest;
 };
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConstReturnTypeCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConstReturnTypeCheck.h
index c3d779eafecb..e329b3d8496a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConstReturnTypeCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConstReturnTypeCheck.h
@@ -19,16 +19,16 @@ namespace clang::tidy::readability {
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/readability/const-return-type.html
 class ConstReturnTypeCheck : public ClangTidyCheck {
- public:
-   ConstReturnTypeCheck(StringRef Name, ClangTidyContext *Context)
-       : ClangTidyCheck(Name, Context),
-         IgnoreMacros(Options.getLocalOrGlobal("IgnoreMacros", true)) {}
-   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
-   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
-   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+public:
+  ConstReturnTypeCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context),
+        IgnoreMacros(Options.getLocalOrGlobal("IgnoreMacros", true)) {}
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
- private:
-   const bool IgnoreMacros;
+private:
+  const bool IgnoreMacros;
 };
 
 } // namespace clang::tidy::readability
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
index bf7a847dff10..ce736a8d1602 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
@@ -16,29 +16,31 @@
 
 using namespace clang::ast_matchers;
 
-namespace clang {
-namespace ast_matchers {
+namespace clang::tidy::readability {
+
+namespace {
 
 AST_POLYMORPHIC_MATCHER_P2(hasAnyArgumentWithParam,
                            AST_POLYMORPHIC_SUPPORTED_TYPES(CallExpr,
                                                            CXXConstructExpr),
-                           internal::Matcher<Expr>, ArgMatcher,
-                           internal::Matcher<ParmVarDecl>, ParamMatcher) {
-  BoundNodesTreeBuilder Result;
+                           ast_matchers::internal::Matcher<Expr>, ArgMatcher,
+                           ast_matchers::internal::Matcher<ParmVarDecl>,
+                           ParamMatcher) {
+  ast_matchers::internal::BoundNodesTreeBuilder Result;
   // The first argument of an overloaded member operator is the implicit object
   // argument of the method which should not be matched against a parameter, so
   // we skip over it here.
-  BoundNodesTreeBuilder Matches;
+  ast_matchers::internal::BoundNodesTreeBuilder Matches;
   unsigned ArgIndex = cxxOperatorCallExpr(callee(cxxMethodDecl()))
                               .matches(Node, Finder, &Matches)
                           ? 1
                           : 0;
   int ParamIndex = 0;
   for (; ArgIndex < Node.getNumArgs(); ++ArgIndex) {
-    BoundNodesTreeBuilder ArgMatches(*Builder);
+    ast_matchers::internal::BoundNodesTreeBuilder ArgMatches(*Builder);
     if (ArgMatcher.matches(*(Node.getArg(ArgIndex)->IgnoreParenCasts()), Finder,
                            &ArgMatches)) {
-      BoundNodesTreeBuilder ParamMatches(ArgMatches);
+      ast_matchers::internal::BoundNodesTreeBuilder ParamMatches(ArgMatches);
       if (expr(anyOf(cxxConstructExpr(hasDeclaration(cxxConstructorDecl(
                          hasParameter(ParamIndex, ParamMatcher)))),
                      callExpr(callee(functionDecl(
@@ -80,9 +82,10 @@ AST_MATCHER(Expr, usedInBooleanContext) {
                      binaryOperator(hasAnyOperatorName("&&", "||")),
                      unaryOperator(hasOperatorName("!")).bind("NegOnSize"))))))
           .matches(Node, Finder, Builder);
-  Builder->removeBindings([ExprName](const BoundNodesMap &Nodes) {
-    return Nodes.getNode(ExprName).getNodeKind().isNone();
-  });
+  Builder->removeBindings(
+      [ExprName](const ast_matchers::internal::BoundNodesMap &Nodes) {
+        return Nodes.getNode(ExprName).getNodeKind().isNone();
+      });
   return Result;
 }
 
@@ -107,8 +110,7 @@ AST_MATCHER_P(UserDefinedLiteral, hasLiteral,
   return false;
 }
 
-} // namespace ast_matchers
-namespace tidy::readability {
+} // namespace
 
 using utils::isBinaryOrTernary;
 
@@ -407,5 +409,4 @@ void ContainerSizeEmptyCheck::check(const MatchFinder::MatchResult &Result) {
       << Container;
 }
 
-} // namespace tidy::readability
-} // namespace clang
+} // namespace clang::tidy::readability
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp
index 1284df6bd99c..30df40bda57d 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.cpp
@@ -18,6 +18,8 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::readability {
 
+namespace {
+
 AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); }
 
 AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); }
@@ -74,6 +76,8 @@ AST_MATCHER(CXXMethodDecl, usesThis) {
   return UsageOfThis.Used;
 }
 
+} // namespace
+
 void ConvertMemberFunctionsToStatic::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       cxxMethodDecl(
@@ -81,7 +85,8 @@ void ConvertMemberFunctionsToStatic::registerMatchers(MatchFinder *Finder) {
           unless(anyOf(
               isExpansionInSystemHeader(), isVirtual(), isStatic(),
               hasTrivialBody(), isOverloadedOperator(), cxxConstructorDecl(),
-              cxxDestructorDecl(), cxxConversionDecl(), isTemplate(),
+              cxxDestructorDecl(), cxxConversionDecl(),
+              isExplicitObjectMemberFunction(), isTemplate(),
               isDependentContext(),
               ofClass(anyOf(
                   isLambda(),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
index f9fd1d903e23..20c73299915a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
@@ -348,8 +348,8 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) {
               implicitCastExpr().bind("implicitCastFromBool"),
               unless(hasParent(BitfieldConstruct)),
               // Check also for nested casts, for example: bool -> int -> float.
-              anyOf(hasParent(implicitCastExpr().bind("furtherImplicitCast")),
-                    anything()),
+              optionally(
+                  hasParent(implicitCastExpr().bind("furtherImplicitCast"))),
               unless(isInTemplateInstantiation()),
               unless(IsInCompilerGeneratedFunction))),
       this);
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp
index 5cc22a22b1f5..10aa779117bb 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp
@@ -131,8 +131,8 @@ findDifferingParamsInDeclaration(const FunctionDecl *ParameterSourceDeclaration,
 
 InconsistentDeclarationsContainer
 findInconsistentDeclarations(const FunctionDecl *OriginalDeclaration,
-                            const FunctionDecl *ParameterSourceDeclaration,
-                            SourceManager &SM, bool Strict) {
+                             const FunctionDecl *ParameterSourceDeclaration,
+                             SourceManager &SM, bool Strict) {
   InconsistentDeclarationsContainer InconsistentDeclarations;
   SourceLocation ParameterSourceLocation =
       ParameterSourceDeclaration->getLocation();
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h
index 79787845de61..70a17889d244 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.h
@@ -43,11 +43,12 @@ class MagicNumbersCheck : public ClangTidyCheck {
 
   bool isBitFieldWidth(const clang::ast_matchers::MatchFinder::MatchResult &,
                        const FloatingLiteral &) const {
-     return false;
+    return false;
   }
 
-  bool isBitFieldWidth(const clang::ast_matchers::MatchFinder::MatchResult &Result,
-                       const IntegerLiteral &Literal) const;
+  bool
+  isBitFieldWidth(const clang::ast_matchers::MatchFinder::MatchResult &Result,
+                  const IntegerLiteral &Literal) const;
 
   bool isUserDefinedLiteral(
       const clang::ast_matchers::MatchFinder::MatchResult &Result,
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
index d42fcba70e81..85852c2c829a 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
@@ -17,6 +17,8 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::readability {
 
+namespace {
+
 AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); }
 
 AST_MATCHER(CXXMethodDecl, hasTrivialBody) { return Node.hasTrivialBody(); }
@@ -214,6 +216,8 @@ AST_MATCHER(CXXMethodDecl, usesThisAsConst) {
   return UsageOfThis.Usage == Const;
 }
 
+} // namespace
+
 void MakeMemberFunctionConstCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp
index 64ce94e3fc1d..1d5c6cca5a82 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/MathMissingParenthesesCheck.cpp
@@ -16,13 +16,15 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::readability {
 
 void MathMissingParenthesesCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(binaryOperator(unless(hasParent(binaryOperator())),
-                                    unless(isAssignmentOperator()),
-                                    unless(isComparisonOperator()),
-                                    unless(hasAnyOperatorName("&&", "||")),
-                                    hasDescendant(binaryOperator()))
-                         .bind("binOp"),
-                     this);
+  Finder->addMatcher(
+      binaryOperator(
+          unless(hasParent(binaryOperator(unless(isAssignmentOperator()),
+                                          unless(isComparisonOperator())))),
+          unless(isAssignmentOperator()), unless(isComparisonOperator()),
+          unless(hasAnyOperatorName("&&", "||")),
+          hasDescendant(binaryOperator()))
+          .bind("binOp"),
+      this);
 }
 
 static int getPrecedence(const BinaryOperator *BinOp) {
@@ -88,10 +90,6 @@ static void addParantheses(const BinaryOperator *BinOp,
 void MathMissingParenthesesCheck::check(
     const MatchFinder::MatchResult &Result) {
   const auto *BinOp = Result.Nodes.getNodeAs<BinaryOperator>("binOp");
-  std::vector<
-      std::pair<clang::SourceRange, std::pair<const clang::BinaryOperator *,
-                                              const clang::BinaryOperator *>>>
-      Insertions;
   const SourceManager &SM = *Result.SourceManager;
   const clang::LangOptions &LO = Result.Context->getLangOpts();
   addParantheses(BinOp, nullptr, this, SM, LO);
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp
index 64dc941569a9..c04bf361c40c 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp
@@ -27,11 +27,15 @@ NamespaceCommentCheck::NamespaceCommentCheck(StringRef Name,
           "namespace( +(((inline )|([a-zA-Z0-9_:]))+))?\\.? *(\\*/)?$",
           llvm::Regex::IgnoreCase),
       ShortNamespaceLines(Options.get("ShortNamespaceLines", 1U)),
-      SpacesBeforeComments(Options.get("SpacesBeforeComments", 1U)) {}
+      SpacesBeforeComments(Options.get("SpacesBeforeComments", 1U)),
+      AllowOmittingNamespaceComments(
+          Options.get("AllowOmittingNamespaceComments", false)) {}
 
 void NamespaceCommentCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "ShortNamespaceLines", ShortNamespaceLines);
   Options.store(Opts, "SpacesBeforeComments", SpacesBeforeComments);
+  Options.store(Opts, "AllowOmittingNamespaceComments",
+                AllowOmittingNamespaceComments);
 }
 
 void NamespaceCommentCheck::registerMatchers(MatchFinder *Finder) {
@@ -106,7 +110,7 @@ void NamespaceCommentCheck::check(const MatchFinder::MatchResult &Result) {
   // Currently for nested namespace (n1::n2::...) the AST matcher will match foo
   // then bar instead of a single match. So if we got a nested namespace we have
   // to skip the next ones.
-  for (const auto &EndOfNameLocation : Ends) {
+  for (const SourceLocation &EndOfNameLocation : Ends) {
     if (Sources.isBeforeInTranslationUnit(ND->getLocation(), EndOfNameLocation))
       return;
   }
@@ -140,6 +144,7 @@ void NamespaceCommentCheck::check(const MatchFinder::MatchResult &Result) {
 
   SourceRange OldCommentRange(AfterRBrace, AfterRBrace);
   std::string Message = "%0 not terminated with a closing comment";
+  bool HasComment = false;
 
   // Try to find existing namespace closing comment on the same line.
   if (Tok.is(tok::comment) && NextTokenIsOnSameLine) {
@@ -158,6 +163,8 @@ void NamespaceCommentCheck::check(const MatchFinder::MatchResult &Result) {
         return;
       }
 
+      HasComment = true;
+
       // Otherwise we need to fix the comment.
       NeedLineBreak = Comment.starts_with("/*");
       OldCommentRange =
@@ -180,8 +187,13 @@ void NamespaceCommentCheck::check(const MatchFinder::MatchResult &Result) {
   }
 
   std::string NamespaceNameForDiag =
-      ND->isAnonymousNamespace() ? "anonymous namespace"
-                                 : ("namespace '" + *NamespaceNameAsWritten + "'");
+      ND->isAnonymousNamespace()
+          ? "anonymous namespace"
+          : ("namespace '" + *NamespaceNameAsWritten + "'");
+
+  // If no namespace comment is allowed
+  if (!HasComment && AllowOmittingNamespaceComments)
+    return;
 
   std::string Fix(SpacesBeforeComments, ' ');
   Fix.append("// namespace");
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.h
index 7607d37b1b2f..8edd77213f77 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.h
@@ -34,6 +34,7 @@ class NamespaceCommentCheck : public ClangTidyCheck {
   llvm::Regex NamespaceCommentPattern;
   const unsigned ShortNamespaceLines;
   const unsigned SpacesBeforeComments;
+  const bool AllowOmittingNamespaceComments;
   llvm::SmallVector<SourceLocation, 4> Ends;
 };
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp
index 00999ee8310c..91a08b9d8de6 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp
@@ -50,10 +50,9 @@ std::optional<Token> findQualToken(const VarDecl *Decl, Qualifier Qual,
   if (FileRange.isInvalid())
     return std::nullopt;
 
-  tok::TokenKind Tok =
-      Qual == Qualifier::Const
-          ? tok::kw_const
-          : Qual == Qualifier::Volatile ? tok::kw_volatile : tok::kw_restrict;
+  tok::TokenKind Tok = Qual == Qualifier::Const      ? tok::kw_const
+                       : Qual == Qualifier::Volatile ? tok::kw_volatile
+                                                     : tok::kw_restrict;
 
   return utils::lexer::getQualifyingToken(Tok, FileRange, *Result.Context,
                                           *Result.SourceManager);
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
index 4c0812f0e679..d59b0312673b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
@@ -186,6 +186,7 @@ static ClangTidyModuleRegistry::Add<ReadabilityModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the ReadabilityModule.
+// NOLINTNEXTLINE(misc-use-internal-linkage)
 volatile int ReadabilityModuleAnchorSource = 0;
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp
index 7850a6f29995..053892dffc7b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantDeclarationCheck.cpp
@@ -15,10 +15,14 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::readability {
 
+namespace {
+
 AST_MATCHER(FunctionDecl, doesDeclarationForceExternallyVisibleDefinition) {
   return Node.doesDeclarationForceExternallyVisibleDefinition();
 }
 
+} // namespace
+
 RedundantDeclarationCheck::RedundantDeclarationCheck(StringRef Name,
                                                      ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp
index 44053ebd21dc..a70719fd8a04 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp
@@ -13,7 +13,8 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::readability {
 
-void RedundantFunctionPtrDereferenceCheck::registerMatchers(MatchFinder *Finder) {
+void RedundantFunctionPtrDereferenceCheck::registerMatchers(
+    MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(TK_AsIs, unaryOperator(hasOperatorName("*"),
                                       has(implicitCastExpr(hasCastKind(
@@ -22,7 +23,8 @@ void RedundantFunctionPtrDereferenceCheck::registerMatchers(MatchFinder *Finder)
       this);
 }
 
-void RedundantFunctionPtrDereferenceCheck::check(const MatchFinder::MatchResult &Result) {
+void RedundantFunctionPtrDereferenceCheck::check(
+    const MatchFinder::MatchResult &Result) {
   const auto *Operator = Result.Nodes.getNodeAs<UnaryOperator>("op");
   diag(Operator->getOperatorLoc(),
        "redundant repeated dereference of function pointer")
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.h
index fdcf821bcf60..a04e9c165bc0 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.h
@@ -19,7 +19,8 @@ namespace clang::tidy::readability {
 /// http://clang.llvm.org/extra/clang-tidy/checks/readability/redundant-function-ptr-dereference.html
 class RedundantFunctionPtrDereferenceCheck : public ClangTidyCheck {
 public:
-  RedundantFunctionPtrDereferenceCheck(StringRef Name, ClangTidyContext *Context)
+  RedundantFunctionPtrDereferenceCheck(StringRef Name,
+                                       ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
index 1693e5c5e9cd..969bf2d1add6 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
@@ -38,8 +38,8 @@ AST_POLYMORPHIC_MATCHER(isInlineSpecified,
 AST_POLYMORPHIC_MATCHER_P(isInternalLinkage,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
                                                           VarDecl),
-                          bool, strictMode) {
-  if (!strictMode)
+                          bool, StrictMode) {
+  if (!StrictMode)
     return false;
   if (const auto *FD = dyn_cast<FunctionDecl>(&Node))
     return FD->getStorageClass() == SC_Static || FD->isInAnonymousNamespace();
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
index 57f13db07802..c90d1521e6b8 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
@@ -52,12 +52,11 @@ void RedundantStringCStrCheck::registerMatchers(
   const auto StringConstructorExpr = expr(anyOf(
       cxxConstructExpr(argumentCountIs(1),
                        hasDeclaration(cxxMethodDecl(hasName("basic_string")))),
-      cxxConstructExpr(
-          argumentCountIs(2),
-          hasDeclaration(cxxMethodDecl(hasName("basic_string"))),
-          // If present, the second argument is the alloc object which must not
-          // be present explicitly.
-          hasArgument(1, cxxDefaultArgExpr()))));
+      cxxConstructExpr(argumentCountIs(2),
+                       hasDeclaration(cxxMethodDecl(hasName("basic_string"))),
+                       // If present, the second argument is the alloc object
+                       // which must not be present explicitly.
+                       hasArgument(1, cxxDefaultArgExpr()))));
 
   // Match string constructor.
   const auto StringViewConstructorExpr = cxxConstructExpr(
@@ -105,8 +104,9 @@ void RedundantStringCStrCheck::registerMatchers(
 
   // Detect: 'dst.append(str.c_str())'  ->  'dst.append(str)'
   Finder->addMatcher(
-      cxxMemberCallExpr(on(StringExpr), callee(decl(cxxMethodDecl(hasAnyName(
-                                            "append", "assign", "compare")))),
+      cxxMemberCallExpr(on(StringExpr),
+                        callee(decl(cxxMethodDecl(
+                            hasAnyName("append", "assign", "compare")))),
                         argumentCountIs(1), hasArgument(0, StringCStrCallExpr)),
       this);
 
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h b/external/llvm-project/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h
index 83bab16b2f7a..deffb09f5db2 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h
@@ -25,7 +25,7 @@ class SimplifySubscriptExprCheck : public ClangTidyCheck {
   }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
-  void storeOptions(ClangTidyOptions::OptionMap& Opts) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   std::optional<TraversalKind> getCheckTraversalKind() const override {
     return TK_IgnoreUnlessSpelledInSource;
   }
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
index 7c3aee29c146..4336c723bd7c 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
@@ -337,8 +337,7 @@ Allow empty enabled checks. This suppresses
 the "no checks enabled" error when disabling
 all of the checks.
 )"),
-                                         cl::init(false),
-                                         cl::cat(ClangTidyCategory));
+                                   cl::init(false), cl::cat(ClangTidyCategory));
 
 namespace clang::tidy {
 
@@ -370,8 +369,8 @@ static void printStats(const ClangTidyStats &Stats) {
   }
 }
 
-static std::unique_ptr<ClangTidyOptionsProvider> createOptionsProvider(
-   llvm::IntrusiveRefCntPtr<vfs::FileSystem> FS) {
+static std::unique_ptr<ClangTidyOptionsProvider>
+createOptionsProvider(llvm::IntrusiveRefCntPtr<vfs::FileSystem> FS) {
   ClangTidyGlobalOptions GlobalOptions;
   if (std::error_code Err = parseLineFilter(LineFilter, GlobalOptions)) {
     llvm::errs() << "Invalid LineFilter: " << Err.message() << "\n\nUsage:\n";
@@ -448,7 +447,7 @@ static std::unique_ptr<ClangTidyOptionsProvider> createOptionsProvider(
       std::move(OverrideOptions), std::move(FS));
 }
 
-llvm::IntrusiveRefCntPtr<vfs::FileSystem>
+static llvm::IntrusiveRefCntPtr<vfs::FileSystem>
 getVfsFromFile(const std::string &OverlayFile,
                llvm::IntrusiveRefCntPtr<vfs::FileSystem> BaseFS) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
index 83248e131948..db1ea1bb514f 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
@@ -48,8 +48,7 @@ StringRef makeCanonicalName(StringRef Str, IncludeSorter::IncludeStyle Style) {
     if (StartIndex == StringRef::npos) {
       StartIndex = 0;
     }
-    return Canonical.substr(
-        0, Canonical.find_first_of('+', StartIndex));
+    return Canonical.substr(0, Canonical.find_first_of('+', StartIndex));
   }
   return removeFirstSuffix(
       removeFirstSuffix(Str, {".cc", ".cpp", ".c", ".h", ".hpp"}),
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp
index 4703ce126981..22cf23fb2446 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/utils/NamespaceAliaser.cpp
@@ -16,15 +16,16 @@
 namespace clang::tidy::utils {
 
 using namespace ast_matchers;
+namespace {
+AST_MATCHER_P(NamespaceAliasDecl, hasTargetNamespace,
+              ast_matchers::internal::Matcher<NamespaceDecl>, InnerMatcher) {
+  return InnerMatcher.matches(*Node.getNamespace(), Finder, Builder);
+}
+} // namespace
 
 NamespaceAliaser::NamespaceAliaser(const SourceManager &SourceMgr)
     : SourceMgr(SourceMgr) {}
 
-AST_MATCHER_P(NamespaceAliasDecl, hasTargetNamespace,
-              ast_matchers::internal::Matcher<NamespaceDecl>, innerMatcher) {
-  return innerMatcher.matches(*Node.getNamespace(), Finder, Builder);
-}
-
 std::optional<FixItHint>
 NamespaceAliaser::createAlias(ASTContext &Context, const Stmt &Statement,
                               StringRef Namespace,
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
index a40433e38a04..7d84a4a9331b 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
@@ -30,7 +30,7 @@ static void verifyRule(const RewriteRuleWith<std::string> &Rule) {
 // If a string unintentionally containing '%' is passed as a diagnostic, Clang
 // will claim the string is ill-formed and assert-fail. This function escapes
 // such strings so they can be safely used in diagnostics.
-std::string escapeForDiagnostic(std::string ToEscape) {
+static std::string escapeForDiagnostic(std::string ToEscape) {
   // Optimize for the common case that the string does not contain `%` at the
   // cost of an extra scan over the string in the slow case.
   auto Pos = ToEscape.find('%');
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
index 68574d22d15f..44db0c2aed60 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
@@ -41,8 +41,7 @@ std::optional<bool> isExpensiveToCopy(QualType Type,
     return std::nullopt;
   return !Type.isTriviallyCopyableType(Context) &&
          !classHasTrivialCopyAndDestroy(Type) &&
-         !hasDeletedCopyConstructor(Type) &&
-         !Type->isObjCLifetimeType();
+         !hasDeletedCopyConstructor(Type) && !Type->isObjCLifetimeType();
 }
 
 bool recordIsTriviallyDefaultConstructible(const RecordDecl &RecordDecl,
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp
index bb2976081a69..bb2c71913193 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp
@@ -17,11 +17,15 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::zircon {
 
+namespace {
+
 AST_MATCHER_P(CXXRecordDecl, matchesAnyName, ArrayRef<StringRef>, Names) {
   std::string QualifiedName = Node.getQualifiedNameAsString();
   return llvm::is_contained(Names, QualifiedName);
 }
 
+} // namespace
+
 void TemporaryObjectsCheck::registerMatchers(MatchFinder *Finder) {
   // Matcher for default constructors.
   Finder->addMatcher(
diff --git a/external/llvm-project/clang-tools-extra/clang-tidy/zircon/ZirconTidyModule.cpp b/external/llvm-project/clang-tools-extra/clang-tidy/zircon/ZirconTidyModule.cpp
index 000680455d61..0eb5683a94e4 100644
--- a/external/llvm-project/clang-tools-extra/clang-tidy/zircon/ZirconTidyModule.cpp
+++ b/external/llvm-project/clang-tools-extra/clang-tidy/zircon/ZirconTidyModule.cpp
@@ -32,6 +32,6 @@ static ClangTidyModuleRegistry::Add<ZirconModule>
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the ZirconModule.
-volatile int ZirconModuleAnchorSource = 0;
+volatile int ZirconModuleAnchorSource = 0; // NOLINT(misc-use-internal-linkage)
 
 } // namespace clang::tidy
diff --git a/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.cpp b/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.cpp
index dc4c8fc498b1..382ea3ffe342 100644
--- a/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.cpp
@@ -117,7 +117,9 @@ bool mayConsiderUnused(const Inclusion &Inc, ParsedAST &AST,
 
 std::vector<Diag> generateMissingIncludeDiagnostics(
     ParsedAST &AST, llvm::ArrayRef<MissingIncludeDiagInfo> MissingIncludes,
-    llvm::StringRef Code, HeaderFilter IgnoreHeaders, const ThreadsafeFS &TFS) {
+    llvm::StringRef Code, HeaderFilter IgnoreHeaders,
+    HeaderFilter AngledHeaders, HeaderFilter QuotedHeaders,
+    const ThreadsafeFS &TFS) {
   std::vector<Diag> Result;
   const SourceManager &SM = AST.getSourceManager();
   const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID());
@@ -141,7 +143,18 @@ std::vector<Diag> generateMissingIncludeDiagnostics(
          AST.getPreprocessor().getHeaderSearchInfo(), MainFile});
 
     llvm::StringRef HeaderRef{Spelling};
+
     bool Angled = HeaderRef.starts_with("<");
+    if (SymbolWithMissingInclude.Providers.front().kind() ==
+        include_cleaner::Header::Kind::Physical) {
+      for (auto &Filter : Angled ? QuotedHeaders : AngledHeaders) {
+        if (Filter(ResolvedPath)) {
+          Angled = !Angled;
+          break;
+        }
+      }
+    }
+
     // We might suggest insertion of an existing include in edge cases, e.g.,
     // include is present in a PP-disabled region, or spelling of the header
     // turns out to be the same as one of the unresolved includes in the
@@ -151,6 +164,11 @@ std::vector<Diag> generateMissingIncludeDiagnostics(
     if (!Replacement.has_value())
       continue;
 
+    if (Angled != (Spelling.front() == '<')) {
+      Spelling.front() = Angled ? '<' : '"';
+      Spelling.back() = Angled ? '>' : '"';
+    }
+
     Diag &D = Result.emplace_back();
     D.Message =
         llvm::formatv("No header providing \"{0}\" is directly included",
@@ -481,18 +499,19 @@ bool isPreferredProvider(const Inclusion &Inc,
   return false; // no header provides the symbol
 }
 
-std::vector<Diag>
-issueIncludeCleanerDiagnostics(ParsedAST &AST, llvm::StringRef Code,
-                               const IncludeCleanerFindings &Findings,
-                               const ThreadsafeFS &TFS,
-                               HeaderFilter IgnoreHeaders) {
+std::vector<Diag> issueIncludeCleanerDiagnostics(
+    ParsedAST &AST, llvm::StringRef Code,
+    const IncludeCleanerFindings &Findings, const ThreadsafeFS &TFS,
+    HeaderFilter IgnoreHeaders, HeaderFilter AngledHeaders,
+    HeaderFilter QuotedHeaders) {
   trace::Span Tracer("IncludeCleaner::issueIncludeCleanerDiagnostics");
   std::vector<Diag> UnusedIncludes = generateUnusedIncludeDiagnostics(
       AST.tuPath(), Findings.UnusedIncludes, Code, IgnoreHeaders);
   std::optional<Fix> RemoveAllUnused = removeAllUnusedIncludes(UnusedIncludes);
 
   std::vector<Diag> MissingIncludeDiags = generateMissingIncludeDiagnostics(
-      AST, Findings.MissingIncludes, Code, IgnoreHeaders, TFS);
+      AST, Findings.MissingIncludes, Code, IgnoreHeaders, AngledHeaders,
+      QuotedHeaders, TFS);
   std::optional<Fix> AddAllMissing = addAllMissingIncludes(MissingIncludeDiags);
 
   std::optional<Fix> FixAll;
diff --git a/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.h b/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.h
index 3f6e3b2fd45b..9439eeed6a3e 100644
--- a/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.h
+++ b/external/llvm-project/clang-tools-extra/clangd/IncludeCleaner.h
@@ -57,11 +57,11 @@ IncludeCleanerFindings
 computeIncludeCleanerFindings(ParsedAST &AST,
                               bool AnalyzeAngledIncludes = false);
 
-std::vector<Diag>
-issueIncludeCleanerDiagnostics(ParsedAST &AST, llvm::StringRef Code,
-                               const IncludeCleanerFindings &Findings,
-                               const ThreadsafeFS &TFS,
-                               HeaderFilter IgnoreHeader = {});
+std::vector<Diag> issueIncludeCleanerDiagnostics(
+    ParsedAST &AST, llvm::StringRef Code,
+    const IncludeCleanerFindings &Findings, const ThreadsafeFS &TFS,
+    HeaderFilter IgnoreHeader = {}, HeaderFilter AngledHeaders = {},
+    HeaderFilter QuotedHeaders = {});
 
 /// Converts the clangd include representation to include-cleaner
 /// include representation.
diff --git a/external/llvm-project/clang-tools-extra/clangd/InlayHints.cpp b/external/llvm-project/clang-tools-extra/clangd/InlayHints.cpp
index bdab2b8a9f37..20a238612a7e 100644
--- a/external/llvm-project/clang-tools-extra/clangd/InlayHints.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/InlayHints.cpp
@@ -33,6 +33,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/ADT/identity.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -375,7 +376,11 @@ static FunctionProtoTypeLoc getPrototypeLoc(Expr *Fn) {
   }
 
   if (auto F = Target.getAs<FunctionProtoTypeLoc>()) {
-    return F;
+    // In some edge cases the AST can contain a "trivial" FunctionProtoTypeLoc
+    // which has null parameters. Avoid these as they don't contain useful
+    // information.
+    if (llvm::all_of(F.getParams(), llvm::identity<ParmVarDecl *>()))
+      return F;
   }
 
   return {};
diff --git a/external/llvm-project/clang-tools-extra/clangd/ModulesBuilder.cpp b/external/llvm-project/clang-tools-extra/clangd/ModulesBuilder.cpp
index bf77f43bd28b..d88aa01aad05 100644
--- a/external/llvm-project/clang-tools-extra/clangd/ModulesBuilder.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/ModulesBuilder.cpp
@@ -430,10 +430,10 @@ class CachingProjectModules : public ProjectModules {
 /// Collect the directly and indirectly required module names for \param
 /// ModuleName in topological order. The \param ModuleName is guaranteed to
 /// be the last element in \param ModuleNames.
-llvm::SmallVector<StringRef> getAllRequiredModules(PathRef RequiredSource,
-                                                   CachingProjectModules &MDB,
-                                                   StringRef ModuleName) {
-  llvm::SmallVector<llvm::StringRef> ModuleNames;
+llvm::SmallVector<std::string> getAllRequiredModules(PathRef RequiredSource,
+                                                     CachingProjectModules &MDB,
+                                                     StringRef ModuleName) {
+  llvm::SmallVector<std::string> ModuleNames;
   llvm::StringSet<> ModuleNamesSet;
 
   auto VisitDeps = [&](StringRef ModuleName, auto Visitor) -> void {
@@ -444,7 +444,7 @@ llvm::SmallVector<StringRef> getAllRequiredModules(PathRef RequiredSource,
       if (ModuleNamesSet.insert(RequiredModuleName).second)
         Visitor(RequiredModuleName, Visitor);
 
-    ModuleNames.push_back(ModuleName);
+    ModuleNames.push_back(ModuleName.str());
   };
   VisitDeps(ModuleName, VisitDeps);
 
@@ -494,13 +494,13 @@ llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile(
   // Get Required modules in topological order.
   auto ReqModuleNames = getAllRequiredModules(RequiredSource, MDB, ModuleName);
   for (llvm::StringRef ReqModuleName : ReqModuleNames) {
-    if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
+    if (BuiltModuleFiles.isModuleUnitBuilt(ReqModuleName))
       continue;
 
     if (auto Cached = Cache.getModule(ReqModuleName)) {
       if (IsModuleFileUpToDate(Cached->getModuleFilePath(), BuiltModuleFiles,
                                TFS.view(std::nullopt))) {
-        log("Reusing module {0} from {1}", ModuleName,
+        log("Reusing module {0} from {1}", ReqModuleName,
             Cached->getModuleFilePath());
         BuiltModuleFiles.addModuleFile(std::move(Cached));
         continue;
@@ -508,14 +508,16 @@ llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile(
       Cache.remove(ReqModuleName);
     }
 
+    std::string ReqFileName =
+        MDB.getSourceForModuleName(ReqModuleName, RequiredSource);
     llvm::Expected<ModuleFile> MF = buildModuleFile(
-        ModuleName, ModuleUnitFileName, getCDB(), TFS, BuiltModuleFiles);
+        ReqModuleName, ReqFileName, getCDB(), TFS, BuiltModuleFiles);
     if (llvm::Error Err = MF.takeError())
       return Err;
 
-    log("Built module {0} to {1}", ModuleName, MF->getModuleFilePath());
+    log("Built module {0} to {1}", ReqModuleName, MF->getModuleFilePath());
     auto BuiltModuleFile = std::make_shared<const ModuleFile>(std::move(*MF));
-    Cache.add(ModuleName, BuiltModuleFile);
+    Cache.add(ReqModuleName, BuiltModuleFile);
     BuiltModuleFiles.addModuleFile(std::move(BuiltModuleFile));
   }
 
diff --git a/external/llvm-project/clang-tools-extra/clangd/ParsedAST.cpp b/external/llvm-project/clang-tools-extra/clangd/ParsedAST.cpp
index 9e1f6bb97722..48896e5f4ff8 100644
--- a/external/llvm-project/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/ParsedAST.cpp
@@ -381,8 +381,9 @@ std::vector<Diag> getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code,
     Findings.MissingIncludes.clear();
   if (SuppressUnused)
     Findings.UnusedIncludes.clear();
-  return issueIncludeCleanerDiagnostics(AST, Code, Findings, TFS,
-                                        Cfg.Diagnostics.Includes.IgnoreHeader);
+  return issueIncludeCleanerDiagnostics(
+      AST, Code, Findings, TFS, Cfg.Diagnostics.Includes.IgnoreHeader,
+      Cfg.Style.AngledHeaders, Cfg.Style.QuotedHeaders);
 }
 
 tidy::ClangTidyCheckFactories
diff --git a/external/llvm-project/clang-tools-extra/clangd/refactor/Rename.cpp b/external/llvm-project/clang-tools-extra/clangd/refactor/Rename.cpp
index d9b73b83e902..c56375b1a98d 100644
--- a/external/llvm-project/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -1308,7 +1308,7 @@ getMappedRanges(ArrayRef<Range> Indexed, ArrayRef<SymbolRange> Lexed) {
     return std::nullopt;
   }
   // Fast check for the special subset case.
-  if (std::includes(Indexed.begin(), Indexed.end(), Lexed.begin(), Lexed.end()))
+  if (llvm::includes(Indexed, Lexed))
     return Lexed.vec();
 
   std::vector<size_t> Best;
diff --git a/external/llvm-project/clang-tools-extra/clangd/test/module_dependencies.test b/external/llvm-project/clang-tools-extra/clangd/test/module_dependencies.test
new file mode 100644
index 000000000000..79306a73da43
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/clangd/test/module_dependencies.test
@@ -0,0 +1,95 @@
+# A smoke test to check that a simple dependency chain for modules can work.
+#
+# FIXME: The test fails on Windows; see comments on https://github.com/llvm/llvm-project/pull/142828
+# UNSUPPORTED: system-windows
+#
+# RUN: rm -fr %t
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+#
+# RUN: sed -e "s|DIR|%/t|g" %t/compile_commands.json.tmpl > %t/compile_commands.json.tmp
+# RUN: sed -e "s|CLANG_CC|%clang|g" %t/compile_commands.json.tmp > %t/compile_commands.json
+# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc.tmp
+#
+# On Windows, we need the URI in didOpen to look like "uri":"file:///C:/..."
+# (with the extra slash in the front), so we add it here.
+# RUN: sed -E -e 's|"file://([A-Z]):/|"file:///\1:/|g' %/t/definition.jsonrpc.tmp > %/t/definition.jsonrpc
+#
+# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc \
+# RUN:      | FileCheck -strict-whitespace %t/definition.jsonrpc
+
+#--- A-frag.cppm
+export module A:frag;
+export void printA() {}
+
+#--- A.cppm
+export module A;
+export import :frag;
+
+#--- Use.cpp
+import A;
+void foo() {
+    print
+}
+
+#--- compile_commands.json.tmpl
+[
+    {
+      "directory": "DIR",
+      "command": "CLANG_CC -fprebuilt-module-path=DIR -std=c++20 -o DIR/main.cpp.o -c DIR/Use.cpp",
+      "file": "DIR/Use.cpp"
+    },
+    {
+      "directory": "DIR",
+      "command": "CLANG_CC -std=c++20 DIR/A.cppm --precompile -o DIR/A.pcm",
+      "file": "DIR/A.cppm"
+    },
+    {
+      "directory": "DIR",
+      "command": "CLANG_CC -std=c++20 DIR/A-frag.cppm --precompile -o DIR/A-frag.pcm",
+      "file": "DIR/A-frag.cppm"
+    }
+]
+
+#--- definition.jsonrpc.tmpl
+{
+  "jsonrpc": "2.0",
+  "id": 0,
+  "method": "initialize",
+  "params": {
+    "processId": 123,
+    "rootPath": "clangd",
+    "capabilities": {
+      "textDocument": {
+        "completion": {
+          "completionItem": {
+            "snippetSupport": true
+          }
+        }
+      }
+    },
+    "trace": "off"
+  }
+}
+---
+{
+  "jsonrpc": "2.0",
+  "method": "textDocument/didOpen",
+  "params": {
+    "textDocument": {
+      "uri": "file://DIR/Use.cpp",
+      "languageId": "cpp",
+      "version": 1,
+      "text": "import A;\nvoid foo() {\n    print\n}\n"
+    }
+  }
+}
+
+# CHECK: "message"{{.*}}printA{{.*}}(fix available)
+
+---
+{"jsonrpc":"2.0","id":1,"method":"textDocument/completion","params":{"textDocument":{"uri":"file://DIR/Use.cpp"},"context":{"triggerKind":1},"position":{"line":2,"character":6}}}
+---
+{"jsonrpc":"2.0","id":2,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/external/llvm-project/clang-tools-extra/clangd/tool/ClangdMain.cpp b/external/llvm-project/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 4bd256d6be22..b11d194da04d 100644
--- a/external/llvm-project/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -913,7 +913,6 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
   if (!ResourceDir.empty())
     Opts.ResourceDir = ResourceDir;
   Opts.BuildDynamicSymbolIndex = true;
-  std::vector<std::unique_ptr<SymbolIndex>> IdxStack;
 #if CLANGD_ENABLE_REMOTE
   if (RemoteIndexAddress.empty() != ProjectRoot.empty()) {
     llvm::errs() << "remote-index-address and project-path have to be "
diff --git a/external/llvm-project/clang-tools-extra/clangd/unittests/HoverTests.cpp b/external/llvm-project/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 69f6df46c87c..775278ccf694 100644
--- a/external/llvm-project/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -974,7 +974,7 @@ class Foo final {})cpp";
          HI.Name = "abc";
          HI.Kind = index::SymbolKind::Variable;
          HI.NamespaceScope = "";
-         HI.Definition = "int abc = <recovery - expr>()";
+         HI.Definition = "int abc";
          HI.Type = "int";
          HI.AccessSpecifier = "public";
        }},
diff --git a/external/llvm-project/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp b/external/llvm-project/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
index 0ee748c1ed2d..ec733cbe9c42 100644
--- a/external/llvm-project/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
@@ -220,13 +220,16 @@ TEST(IncludeCleaner, ComputeMissingHeaders) {
 TEST(IncludeCleaner, GenerateMissingHeaderDiags) {
   Annotations MainFile(R"cpp(
 #include "a.h"
+#include "angled_wrapper.h"
 #include "all.h"
 $insert_b[[]]#include "baz.h"
 #include "dir/c.h"
 $insert_d[[]]$insert_foo[[]]#include "fuzz.h"
 #include "header.h"
-$insert_foobar[[]]#include <e.h>
-$insert_f[[]]$insert_vector[[]]
+$insert_foobar[[]]$insert_quoted[[]]$insert_quoted2[[]]#include "quoted_wrapper.h"
+$insert_angled[[]]#include <e.h>
+$insert_f[[]]#include <quoted2_wrapper.h>
+$insert_vector[[]]
 
 #define DEF(X) const Foo *X;
 #define BAZ(X) const X x
@@ -237,6 +240,9 @@ TEST(IncludeCleaner, GenerateMissingHeaderDiags) {
 
   void foo() {
     $b[[b]]();
+    $angled[[angled]]();
+    $quoted[[quoted]]();
+    $quoted2[[quoted2]]();
 
     ns::$bar[[Bar]] bar;
     bar.d();
@@ -263,12 +269,22 @@ TEST(IncludeCleaner, GenerateMissingHeaderDiags) {
   TU.AdditionalFiles["a.h"] = guard("#include \"b.h\"");
   TU.AdditionalFiles["b.h"] = guard("void b();");
 
+  TU.AdditionalFiles["angled_wrapper.h"] = guard("#include <angled.h>");
+  TU.AdditionalFiles["angled.h"] = guard("void angled();");
+  TU.ExtraArgs.push_back("-I" + testPath("."));
+
+  TU.AdditionalFiles["quoted_wrapper.h"] = guard("#include \"quoted.h\"");
+  TU.AdditionalFiles["quoted.h"] = guard("void quoted();");
+
   TU.AdditionalFiles["dir/c.h"] = guard("#include \"d.h\"");
   TU.AdditionalFiles["dir/d.h"] =
       guard("namespace ns { struct Bar { void d(); }; }");
 
   TU.AdditionalFiles["system/e.h"] = guard("#include <f.h>");
   TU.AdditionalFiles["system/f.h"] = guard("void f();");
+  TU.AdditionalFiles["system/quoted2_wrapper.h"] =
+      guard("#include <system/quoted2.h>");
+  TU.AdditionalFiles["system/quoted2.h"] = guard("void quoted2();");
   TU.ExtraArgs.push_back("-isystem" + testPath("system"));
 
   TU.AdditionalFiles["fuzz.h"] = guard("#include \"buzz.h\"");
@@ -297,7 +313,15 @@ TEST(IncludeCleaner, GenerateMissingHeaderDiags) {
   Findings.UnusedIncludes.clear();
   std::vector<clangd::Diag> Diags = issueIncludeCleanerDiagnostics(
       AST, TU.Code, Findings, MockFS(),
-      {[](llvm::StringRef Header) { return Header.ends_with("buzz.h"); }});
+      /*IgnoreHeaders=*/{[](llvm::StringRef Header) {
+        return Header.ends_with("buzz.h");
+      }},
+      /*AngledHeaders=*/{[](llvm::StringRef Header) {
+        return Header.contains("angled.h");
+      }},
+      /*QuotedHeaders=*/{[](llvm::StringRef Header) {
+        return Header.contains("quoted.h") || Header.contains("quoted2.h");
+      }});
   EXPECT_THAT(
       Diags,
       UnorderedElementsAre(
@@ -306,6 +330,23 @@ TEST(IncludeCleaner, GenerateMissingHeaderDiags) {
                 withFix({Fix(MainFile.range("insert_b"), "#include \"b.h\"\n",
                              "#include \"b.h\""),
                          FixMessage("add all missing includes")})),
+          AllOf(Diag(MainFile.range("angled"),
+                     "No header providing \"angled\" is directly included"),
+                withFix({Fix(MainFile.range("insert_angled"),
+                             "#include <angled.h>\n", "#include <angled.h>"),
+                         FixMessage("add all missing includes")})),
+          AllOf(
+              Diag(MainFile.range("quoted"),
+                   "No header providing \"quoted\" is directly included"),
+              withFix({Fix(MainFile.range("insert_quoted"),
+                           "#include \"quoted.h\"\n", "#include \"quoted.h\""),
+                       FixMessage("add all missing includes")})),
+          AllOf(Diag(MainFile.range("quoted2"),
+                     "No header providing \"quoted2\" is directly included"),
+                withFix(
+                    {Fix(MainFile.range("insert_quoted2"),
+                         "#include \"quoted2.h\"\n", "#include \"quoted2.h\""),
+                     FixMessage("add all missing includes")})),
           AllOf(Diag(MainFile.range("bar"),
                      "No header providing \"ns::Bar\" is directly included"),
                 withFix({Fix(MainFile.range("insert_d"),
diff --git a/external/llvm-project/clang-tools-extra/clangd/unittests/InlayHintTests.cpp b/external/llvm-project/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
index c3331d20730f..e0cd955bb1c9 100644
--- a/external/llvm-project/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
@@ -1011,11 +1011,16 @@ TEST(ParameterHints, FunctionPointer) {
     f3_t f3;
     using f4_t = void(__stdcall *)(int param);
     f4_t f4;
+    __attribute__((noreturn)) f4_t f5;
     void bar() {
       f1($f1[[42]]);
       f2($f2[[42]]);
       f3($f3[[42]]);
       f4($f4[[42]]);
+      // This one runs into an edge case in clang's type model
+      // and we can't extract the parameter name. But at least
+      // we shouldn't crash.
+      f5(42);
     }
   )cpp",
       ExpectedHint{"param: ", "f1"}, ExpectedHint{"param: ", "f2"},
diff --git a/external/llvm-project/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp b/external/llvm-project/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp
index 03e65768c26a..ae2d98ce90d8 100644
--- a/external/llvm-project/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp
@@ -247,13 +247,11 @@ import Dep;
       ProjectModules->getRequiredModules(getFullPath("M.cppm")).empty());
 
   // Set the mangler to filter out the invalid flag
-  ProjectModules->setCommandMangler(
-      [](tooling::CompileCommand &Command, PathRef) {
-        auto const It =
-            std::find(Command.CommandLine.begin(), Command.CommandLine.end(),
-                      "-invalid-unknown-flag");
-        Command.CommandLine.erase(It);
-      });
+  ProjectModules->setCommandMangler([](tooling::CompileCommand &Command,
+                                       PathRef) {
+    auto const It = llvm::find(Command.CommandLine, "-invalid-unknown-flag");
+    Command.CommandLine.erase(It);
+  });
 
   // And now it returns a non-empty list of required modules since the
   // compilation succeeded
diff --git a/external/llvm-project/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/external/llvm-project/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index 1892f87c8e82..b04d6431f89f 100644
--- a/external/llvm-project/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/external/llvm-project/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -2311,6 +2311,14 @@ TEST(FindReferences, WithinAST) {
             $(S::deleteObject)[[de^lete]] S;
           }
         };
+      )cpp",
+      // Array designators
+      R"cpp(
+        const int $def[[F^oo]] = 0;
+        int Bar[] = {
+          [$(Bar)[[F^oo]]...$(Bar)[[Fo^o]] + 1] = 0,
+          [$(Bar)[[^Foo]] + 2] = 1
+        };
       )cpp"};
   for (const char *Test : Tests)
     checkFindRefs(Test);
diff --git a/external/llvm-project/clang-tools-extra/docs/ReleaseNotes.rst b/external/llvm-project/clang-tools-extra/docs/ReleaseNotes.rst
index e0f81a032c38..19ccd1790e75 100644
--- a/external/llvm-project/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/external/llvm-project/clang-tools-extra/docs/ReleaseNotes.rst
@@ -159,6 +159,10 @@ Changes in existing checks
   false positives on deleted constructors that cannot be used to construct
   objects, even if they have public or protected access.
 
+- Added an option to :doc:`bugprone-multi-level-implicit-pointer-conversion
+  <clang-tidy/checks/bugprone/multi-level-implicit-pointer-conversion>` to
+  choose whether to enable the check in C code or not.
+
 - Improved :doc:`bugprone-optional-value-conversion
   <clang-tidy/checks/bugprone/optional-value-conversion>` check to detect
   conversion in argument of ``std::make_optional``.
@@ -187,6 +191,16 @@ Changes in existing checks
   <clang-tidy/checks/concurrency/mt-unsafe>` check by fixing a false positive
   where ``strerror`` was flagged as MT-unsafe.
 
+- Improved :doc:`google-readability-namespace-comments
+  <clang-tidy/checks/google/readability-namespace-comments>` check by adding
+  the option `AllowOmittingNamespaceComments` to accept if a namespace comment
+  is omitted entirely.
+
+- Improved :doc:`llvm-namespace-comment
+  <clang-tidy/checks/llvm/namespace-comment>` check by adding the option
+  `AllowOmittingNamespaceComments` to accept if a namespace comment is omitted
+  entirely.
+
 - Improved :doc:`misc-const-correctness
   <clang-tidy/checks/misc/const-correctness>` check by adding the option
   `AllowedTypes`, that excludes specified types from const-correctness
@@ -236,6 +250,13 @@ Changes in existing checks
   <clang-tidy/checks/modernize/use-std-numbers>` check to support math
   functions of different precisions.
 
+- Improved :doc:`modernize-use-trailing-return-type
+  <clang-tidy/checks/modernize/use-trailing-return-type>` check by adding
+  support to modernize lambda signatures to use trailing return type and adding
+  two new options: `TransformFunctions` and `TransformLambdas` to control
+  whether function declarations and lambdas should be transformed by the check.
+  Fixed false positives when lambda was matched as a function in C++11 mode.
+
 - Improved :doc:`performance-move-const-arg
   <clang-tidy/checks/performance/move-const-arg>` check by fixing false
   negatives on ternary operators calling ``std::move``.
@@ -245,6 +266,15 @@ Changes in existing checks
   tolerating fix-it breaking compilation when functions is used as pointers
   to avoid matching usage of functions within the current compilation unit.
 
+- Improved :doc:`readability-convert-member-functions-to-static
+  <clang-tidy/checks/readability/convert-member-functions-to-static>` check by
+  fixing false positives on member functions with an explicit object parameter.
+
+- Improved :doc:`readability-math-missing-parentheses
+  <clang-tidy/checks/readability/math-missing-parentheses>` check by fixing
+  false negatives where math expressions are the operand of assignment operators
+  or comparison operators.
+
 - Improved :doc:`readability-qualified-auto
   <clang-tidy/checks/readability/qualified-auto>` check by adding the option
   `AllowedTypes`, that excludes specified types from adding qualifiers.
diff --git a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/multi-level-implicit-pointer-conversion.rst b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/multi-level-implicit-pointer-conversion.rst
index e6dc5c13aa02..14df6bd9ccea 100644
--- a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/multi-level-implicit-pointer-conversion.rst
+++ b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/multi-level-implicit-pointer-conversion.rst
@@ -42,3 +42,11 @@ Additionally, it is recommended that developers thoroughly check and verify the
 safety of the conversion before using an explicit cast. This extra level of
 caution can help catch potential issues early on in the development process,
 improving the overall reliability and maintainability of the code.
+
+Options
+-------
+
+.. option:: EnableInC
+
+   If `true`, enables the check in C code (it is always enabled in C++ code).
+   Default is `true`.
diff --git a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst
index ba47399914de..325a0a2aa9cc 100644
--- a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst
+++ b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/bugprone/return-const-ref-from-parameter.rst
@@ -42,7 +42,7 @@ on the control flow of the function, an overload where all problematic
 
 This issue can also be resolved by adding ``[[clang::lifetimebound]]``. Clang
 enable ``-Wdangling`` warning by default which can detect mis-uses of the
-annotated function. See `lifetimebound attribute <https://clang.llvm.org/docs/AttributeReference.html#id11>`_
+annotated function. See `lifetimebound attribute <https://clang.llvm.org/docs/AttributeReference.html#lifetimebound>`_
 for details.
 
 .. code-block:: c++
diff --git a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/llvm/namespace-comment.rst b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/llvm/namespace-comment.rst
index be90260be73a..23673c9b4cfd 100644
--- a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/llvm/namespace-comment.rst
+++ b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/llvm/namespace-comment.rst
@@ -39,3 +39,9 @@ Options
 
    An unsigned integer specifying the number of spaces before the comment
    closing a namespace definition. Default is `1U`.
+
+.. option:: AllowOmittingNamespaceComments
+
+   When `true`, the check will accept if no namespace comment is present.
+   The check will only fail if the specified namespace comment is different
+   than expected. Default is `false`.
diff --git a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/deprecated-headers.rst b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/deprecated-headers.rst
index 298243fc3ced..6c35cd4e53d8 100644
--- a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/deprecated-headers.rst
+++ b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/deprecated-headers.rst
@@ -79,5 +79,5 @@ Options
    analyzed C++ source file is not included by any other C source files.
    Hence, to omit false-positives and wrong fixit-hints, we ignore emitting
    reports into header files. One can set this option to `true` if they know
-   that the header files in the project are only used by C++ source file.
+   that the header files in the project are only used by C++ source files.
    Default is `false`.
diff --git a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/use-trailing-return-type.rst b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/use-trailing-return-type.rst
index 0593a35326aa..2dac7cc76f41 100644
--- a/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/use-trailing-return-type.rst
+++ b/external/llvm-project/clang-tools-extra/docs/clang-tidy/checks/modernize/use-trailing-return-type.rst
@@ -3,7 +3,7 @@
 modernize-use-trailing-return-type
 ==================================
 
-Rewrites function signatures to use a trailing return type
+Rewrites function and lambda signatures to use a trailing return type
 (introduced in C++11). This transformation is purely stylistic.
 The return type before the function name is replaced by ``auto``
 and inserted after the function parameter list (and qualifiers).
@@ -16,6 +16,7 @@ Example
   int f1();
   inline int f2(int arg) noexcept;
   virtual float f3() const && = delete;
+  auto lambda = []() {};
 
 transforms to:
 
@@ -24,6 +25,7 @@ transforms to:
   auto f1() -> int;
   inline auto f2(int arg) -> int noexcept;
   virtual auto f3() const && -> float = delete;
+  auto lambda = []() -> void {};
 
 Known Limitations
 -----------------
@@ -66,3 +68,25 @@ a careless rewrite would produce the following output:
 This code fails to compile because the S in the context of f refers to the equally named function parameter.
 Similarly, the S in the context of m refers to the equally named class member.
 The check can currently only detect and avoid a clash with a function parameter name.
+
+Options
+-------
+
+.. option:: TransformFunctions
+
+  When set to `true`, function declarations will be transformed to use trailing
+  return. Default is `true`.
+
+.. option:: TransformLambdas
+
+  Controls how lambda expressions are transformed to use trailing
+  return type. Possible values are:
+
+  * `all` - Transform all lambda expressions without an explicit return type
+    to use trailing return type. If type can not be deduced, ``auto`` will be
+    used since C++14 and generic message will be emitted otherwise.
+  * `all_except_auto` - Transform all lambda expressions except those whose return
+    type can not be deduced.
+  * `none` - Do not transform any lambda expressions.
+
+  Default is `all`.
diff --git a/external/llvm-project/clang-tools-extra/modularize/CoverageChecker.cpp b/external/llvm-project/clang-tools-extra/modularize/CoverageChecker.cpp
index fe6711398ab7..1345a6ef8f48 100644
--- a/external/llvm-project/clang-tools-extra/modularize/CoverageChecker.cpp
+++ b/external/llvm-project/clang-tools-extra/modularize/CoverageChecker.cpp
@@ -329,10 +329,8 @@ bool CoverageChecker::collectFileSystemHeaders() {
   else {
     // Otherwise we only look at the sub-trees specified by the
     // include paths.
-    for (std::vector<std::string>::const_iterator I = IncludePaths.begin(),
-      E = IncludePaths.end();
-      I != E; ++I) {
-      if (!collectFileSystemHeaders(*I))
+    for (const std::string &IncludePath : IncludePaths) {
+      if (!collectFileSystemHeaders(IncludePath))
         return false;
     }
   }
diff --git a/external/llvm-project/clang-tools-extra/modularize/Modularize.cpp b/external/llvm-project/clang-tools-extra/modularize/Modularize.cpp
index 7f8a19280b11..2a90c5e3f678 100644
--- a/external/llvm-project/clang-tools-extra/modularize/Modularize.cpp
+++ b/external/llvm-project/clang-tools-extra/modularize/Modularize.cpp
@@ -339,8 +339,8 @@ static std::string findInputFile(const CommandLineArguments &CLArgs) {
   llvm::opt::Visibility VisibilityMask(options::CC1Option);
   unsigned MissingArgIndex, MissingArgCount;
   SmallVector<const char *, 256> Argv;
-  for (auto I = CLArgs.begin(), E = CLArgs.end(); I != E; ++I)
-    Argv.push_back(I->c_str());
+  for (const std::string &CLArg : CLArgs)
+    Argv.push_back(CLArg.c_str());
   InputArgList Args = getDriverOptTable().ParseArgs(
       Argv, MissingArgIndex, MissingArgCount, VisibilityMask);
   std::vector<std::string> Inputs = Args.getAllArgValues(OPT_INPUT);
diff --git a/external/llvm-project/clang-tools-extra/modularize/ModularizeUtilities.cpp b/external/llvm-project/clang-tools-extra/modularize/ModularizeUtilities.cpp
index 9ad1731915a8..8a24f21d658d 100644
--- a/external/llvm-project/clang-tools-extra/modularize/ModularizeUtilities.cpp
+++ b/external/llvm-project/clang-tools-extra/modularize/ModularizeUtilities.cpp
@@ -69,8 +69,7 @@ ModularizeUtilities *ModularizeUtilities::createModularizeUtilities(
 // Load all header lists and dependencies.
 std::error_code ModularizeUtilities::loadAllHeaderListsAndDependencies() {
   // For each input file.
-  for (auto I = InputFilePaths.begin(), E = InputFilePaths.end(); I != E; ++I) {
-    llvm::StringRef InputPath = *I;
+  for (llvm::StringRef InputPath : InputFilePaths) {
     // If it's a module map.
     if (InputPath.ends_with(".modulemap")) {
       // Load the module map.
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/DR-141990.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/DR-141990.cpp
new file mode 100644
index 000000000000..019afb0654c2
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/DR-141990.cpp
@@ -0,0 +1,18 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc -output=%t %s 2>&1 | FileCheck %s --implicit-check-not="{{warning|error}}"
+
+// COM: This case triggered an assertion before #141990:
+// COM: clang-doc: llvm-project/clang/lib/AST/Decl.cpp:2985:
+// COM:   Expr *clang::ParmVarDecl::getDefaultArg(): Assertion `!hasUninstantiatedDefaultArg()
+// COM:   && "Default argument is not yet instantiated!"' failed.
+
+template <class = int>
+class c;
+int e;
+
+template <class>
+class c {
+public:
+  void f(int n = e);
+};
+class B : c<> {};
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/invalid-options.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/invalid-options.cpp
index c8a4cef8364e..15e098ad4c78 100644
--- a/external/llvm-project/clang-tools-extra/test/clang-doc/invalid-options.cpp
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/invalid-options.cpp
@@ -2,7 +2,7 @@
 // RUN: rm -rf %t && touch %t
 // RUN: not clang-doc %s -output=%t/subdir 2>&1 | FileCheck %s --check-prefix=OUTPUT-FAIL
 // OUTPUT-FAIL: clang-doc error:
-// OUTPUT-FAIL: {{(Not a directory|no such file or directory)}}
+// OUTPUT-FAIL-SAME: failed to create directory.
 
 /// Invalid format option.
 // RUN: rm -rf %t && mkdir %t && touch %t/file
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/json/class-template.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/json/class-template.cpp
new file mode 100644
index 000000000000..e3ca086d1d9a
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/json/class-template.cpp
@@ -0,0 +1,29 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/MyClass.json
+
+template<typename T> struct MyClass {
+  T MemberTemplate;
+  T method(T Param); 
+};
+
+// CHECK:         "Name": "MyClass",
+// CHECK:         "Name": "method",
+// CHECK:         "Params": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "Name": "Param",
+// CHECK-NEXT:        "Type": "T"
+// CHECK-NEXT:      } 
+// CHECK-NEXT:    ], 
+// CHECK-NEXT:    "ReturnType": {
+// CHECK-NEXT:      "IsBuiltIn": false,
+// CHECK-NEXT:      "IsTemplate": false,
+// CHECK-NEXT:      "Name": "T",
+// CHECK-NEXT:      "QualName": "T"
+// CHECK-NEXT:      "USR": "0000000000000000000000000000000000000000"
+// CHECK:           "Name": "MemberTemplate",
+// CHECK:           "Type": "T"
+// CHECK:         "Template": {
+// CHECK-NEXT:      "Parameters": [
+// CHECK-NEXT:        "typename T"
+// CHECK-NEXT:      ] 
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/json/class.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/json/class.cpp
new file mode 100644
index 000000000000..9ae1f14e2d2a
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/json/class.cpp
@@ -0,0 +1,193 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/MyClass.json
+
+struct Foo;
+
+// This is a nice class.
+// It has some nice methods and fields.
+// @brief This is a brief description.
+struct MyClass {
+  int PublicField;
+
+  int myMethod(int MyParam);
+  static void staticMethod();
+  const int& getConst();
+  
+  enum Color {
+    RED,
+    GREEN,
+    BLUE = 5
+  };
+  
+  typedef int MyTypedef;
+  
+  class NestedClass;
+protected:
+  int protectedMethod();
+
+  int ProtectedField;
+};
+
+// CHECK:       {
+// CHECK-NEXT:    "Description": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:       "FullComment": {
+// CHECK-NEXT:         "Children": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:             "ParagraphComment": {
+// CHECK-NEXT:               "Children": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                   "TextComment": " This is a nice class."
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                   "TextComment": " It has some nice methods and fields."
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                   "TextComment": ""
+// CHECK-NEXT:                 }
+// CHECK-NEXT:               ]
+// CHECK:               {
+// CHECK-NEXT:             "BlockCommandComment": {
+// CHECK-NEXT:               "Children": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                   "ParagraphComment": {
+// CHECK-NEXT:                     "Children": [
+// CHECK-NEXT:                       { 
+// CHECK-NEXT:                         "TextComment": " This is a brief description." 
+// CHECK-NEXT:                       }
+// CHECK:                   "Command": "brief"
+// CHECK:         "Enums": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "Location": {
+// CHECK-NEXT:          "Filename": "{{.*}}class.cpp",
+// CHECK-NEXT:          "LineNumber": 17
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "Members": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "RED",
+// CHECK-NEXT:            "Value": "0"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "GREEN",
+// CHECK-NEXT:            "Value": "1"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "BLUE",
+// CHECK-NEXT:            "ValueExpr": "5"
+// CHECK-NEXT:          }
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "Name": "Color",
+// CHECK-NEXT:        "Namespace": [
+// CHECK-NEXT:          "MyClass",
+// CHECK-NEXT:          "GlobalNamespace"
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "Scoped": false,
+// CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
+// COM:           FIXME: FullName is not emitted correctly.
+// CHECK-NEXT:    "FullName": "",
+// CHECK-NEXT:    "IsTypedef": false,
+// CHECK-NEXT:    "Location": {
+// CHECK-NEXT:      "Filename": "{{.*}}class.cpp",
+// CHECK-NEXT:      "LineNumber": 10
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "Name": "MyClass",
+// CHECK-NEXT:    "Namespace": [
+// CHECK-NEXT:      "GlobalNamespace"
+// CHECK-NEXT:    ],
+// CHECK-NEXT:   "Path": "GlobalNamespace",
+// CHECK-NEXT:   "ProtectedFunctions": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "IsStatic": false,
+// CHECK-NEXT:       "Name": "protectedMethod",
+// CHECK-NEXT:       "Namespace": [
+// CHECK-NEXT:         "MyClass",
+// CHECK-NEXT:         "GlobalNamespace"
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "ReturnType": {
+// CHECK-NEXT:         "IsBuiltIn": false,
+// CHECK-NEXT:         "IsTemplate": false,
+// CHECK-NEXT:         "Name": "int",
+// CHECK-NEXT:         "QualName": "int",
+// CHECK-NEXT:         "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
+// CHECK-NEXT:    "ProtectedMembers": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "Name": "ProtectedField",
+// CHECK-NEXT:        "Type": "int"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
+// CHECK-NEXT:    "PublicFunctions": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "IsStatic": false,
+// CHECK-NEXT:        "Name": "myMethod",
+// CHECK-NEXT:        "Namespace": [
+// CHECK-NEXT:          "MyClass",
+// CHECK-NEXT:          "GlobalNamespace"
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "Params": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "MyParam",
+// CHECK-NEXT:            "Type": "int"
+// CHECK-NEXT:          }
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "ReturnType": {
+// CHECK-NEXT:          "IsBuiltIn": false,
+// CHECK-NEXT:          "IsTemplate": false,
+// CHECK-NEXT:          "Name": "int",
+// CHECK-NEXT:          "QualName": "int",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:      },
+// CHECK:             "IsStatic": true,
+// CHECK:             "Name": "getConst",
+// CHECK:             "ReturnType": {
+// CHECK-NEXT:          "IsBuiltIn": false,
+// CHECK-NEXT:          "IsTemplate": false,
+// CHECK-NEXT:          "Name": "const int &",
+// CHECK-NEXT:          "QualName": "const int &",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK:         "PublicMembers": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "Name": "PublicField",
+// CHECK-NEXT:        "Type": "int"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
+// CHECK-NEXT:    "Records": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "Name": "NestedClass",
+// CHECK-NEXT:        "Path": "GlobalNamespace{{[\/]+}}MyClass",
+// CHECK-NEXT:        "QualName": "NestedClass",
+// CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:    "TagType": "struct",
+// CHECK-NEXT:    "Typedefs": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "IsUsing": false,
+// CHECK-NEXT:        "Location": {
+// CHECK-NEXT:          "Filename": "{{.*}}class.cpp",
+// CHECK-NEXT:          "LineNumber": 23
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "Name": "MyTypedef",
+// CHECK-NEXT:        "Namespace": [
+// CHECK-NEXT:          "MyClass",
+// CHECK-NEXT:          "GlobalNamespace"
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "TypeDeclaration": "",
+// CHECK-NEXT:        "USR": "{{[0-9A-F]*}}",
+// CHECK-NEXT:        "Underlying": {
+// CHECK-NEXT:          "IsBuiltIn": false,
+// CHECK-NEXT:          "IsTemplate": false,
+// CHECK-NEXT:          "Name": "int",
+// CHECK-NEXT:          "QualName": "int",
+// CHECK-NEXT:          "USR": "0000000000000000000000000000000000000000"
+// CHECK:         "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:  }
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/json/function-specifiers.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/json/function-specifiers.cpp
new file mode 100644
index 000000000000..7005fb7b3e66
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/json/function-specifiers.cpp
@@ -0,0 +1,25 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/index.json
+
+static void myFunction() {}
+
+void noExceptFunction() noexcept {}
+
+inline void inlineFunction() {}
+
+extern void externFunction() {}
+
+constexpr void constexprFunction() {}
+
+// CHECK:          "Functions": [
+// CHECK-NEXT:       {
+// CHECK:              "IsStatic": true,
+// COM:                FIXME: Emit ExceptionSpecificationType
+// CHECK-NOT:          "ExceptionSpecifcation" : "noexcept",
+// COM:                FIXME: Emit inline
+// CHECK-NOT:          "IsInline": true,
+// COM:                FIXME: Emit extern
+// CHECK-NOT:          "IsExtern": true,
+// COM:                FIXME: Emit constexpr
+// CHECK-NOT:          "IsConstexpr": true,
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/json/method-template.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/json/method-template.cpp
new file mode 100644
index 000000000000..c51a2706d1c2
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/json/method-template.cpp
@@ -0,0 +1,40 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/MyClass.json
+
+struct MyClass {
+  template<class T> T methodTemplate(T param) {
+  }
+};
+
+// CHECK:           "PublicFunctions": [
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "IsStatic": false,
+// CHECK-NEXT:          "Location": {
+// CHECK-NEXT:            "Filename": "{{.*}}method-template.cpp",
+// CHECK-NEXT:            "LineNumber": 6
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "Name": "methodTemplate",
+// CHECK-NEXT:          "Namespace": [
+// CHECK-NEXT:            "MyClass",
+// CHECK-NEXT:            "GlobalNamespace"
+// CHECK-NEXT:          ],
+// CHECK-NEXT:          "Params": [
+// CHECK-NEXT:            {
+// CHECK-NEXT:              "Name": "param",
+// CHECK-NEXT:              "Type": "T"
+// CHECK-NEXT:            }
+// CHECK-NEXT:          ],
+// CHECK-NEXT:          "ReturnType": {
+// CHECK-NEXT:            "IsBuiltIn": false,
+// CHECK-NEXT:            "IsTemplate": false,
+// CHECK-NEXT:            "Name": "T",
+// CHECK-NEXT:            "QualName": "T",
+// CHECK-NEXT:            "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "Template": {
+// CHECK-NEXT:            "Parameters": [
+// CHECK-NEXT:              "class T"
+// CHECK-NEXT:            ]
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/json/namespace.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/json/namespace.cpp
new file mode 100644
index 000000000000..928864be1feb
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/json/namespace.cpp
@@ -0,0 +1,107 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/index.json
+
+class MyClass {};
+
+void myFunction(int Param);
+
+namespace NestedNamespace {
+} // namespace NestedNamespace
+
+// FIXME: Global variables are not mapped or serialized.
+static int Global;
+
+enum Color {
+  RED,
+  GREEN,
+  BLUE = 5
+};
+
+typedef int MyTypedef;
+
+// CHECK:       { 
+// CHECK-NEXT:    "Enums": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "Location": {
+// CHECK-NEXT:          "Filename": "{{.*}}namespace.cpp",
+// CHECK-NEXT:          "LineNumber": 15
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "Members": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "RED",
+// CHECK-NEXT:            "Value": "0"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "GREEN",
+// CHECK-NEXT:            "Value": "1"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "BLUE",
+// CHECK-NEXT:            "ValueExpr": "5"
+// CHECK-NEXT:          }
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "Name": "Color",
+// CHECK-NEXT:        "Scoped": false,
+// CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
+// CHECK-NEXT:   "Functions": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "IsStatic": false,
+// CHECK-NEXT:       "Name": "myFunction",
+// CHECK-NEXT:       "Params": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "Name": "Param",
+// CHECK-NEXT:           "Type": "int"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "ReturnType": {
+// CHECK-NEXT:         "IsBuiltIn": false,
+// CHECK-NEXT:         "IsTemplate": false,
+// CHECK-NEXT:         "Name": "void",
+// CHECK-NEXT:         "QualName": "void",
+// CHECK-NEXT:         "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ],
+// CHECK-NEXT:   "Name": "",
+// CHECK-NEXT:   "Namespaces": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "Name": "NestedNamespace",
+// CHECK-NEXT:       "Path": "",
+// CHECK-NEXT:       "QualName": "NestedNamespace",
+// CHECK-NEXT:       "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ],
+// CHECK-NEXT:   "Records": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "Name": "MyClass",
+// CHECK-NEXT:       "Path": "GlobalNamespace",
+// CHECK-NEXT:       "QualName": "MyClass",
+// CHECK-NEXT:       "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ],
+// CHECK-NEXT:   "Typedefs": [
+// CHECK-NEXT:    {
+// CHECK-NEXT:      "IsUsing": false,
+// CHECK-NEXT:      "Location": {
+// CHECK-NEXT:        "Filename": "{{.*}}namespace.cpp",
+// CHECK-NEXT:        "LineNumber": 21
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "Name": "MyTypedef",
+// CHECK-NEXT:      "TypeDeclaration": "",
+// CHECK-NEXT:      "USR": "{{[0-9A-F]*}}",
+// CHECK-NEXT:      "Underlying": {
+// CHECK-NEXT:        "IsBuiltIn": false,
+// CHECK-NEXT:        "IsTemplate": false,
+// CHECK-NEXT:        "Name": "int",
+// CHECK-NEXT:        "QualName": "int",
+// CHECK-NEXT:        "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
+// CHECK-NEXT:    "USR": "0000000000000000000000000000000000000000"
+// CHECK-NOT:     "Variables": [
+// CHECK-NEXT:  }
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/mustache-index.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/mustache-index.cpp
new file mode 100644
index 000000000000..cad4cc8b6931
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/mustache-index.cpp
@@ -0,0 +1,75 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --format=mustache --output=%t --executor=standalone %s 
+// RUN: FileCheck %s < %t/GlobalNamespace/index.html
+
+enum Color {
+  RED,
+  BLUE,
+  GREEN
+};
+
+class Foo;
+
+// CHECK:       <li class="sidebar-section">
+// CHECK-NEXT:      <a class="sidebar-item" href="#Enums">Enums</a>
+// CHECK-NEXT:  </li>
+// CHECK-NEXT:  <ul>
+// CHECK-NEXT:      <li class="sidebar-item-container">
+// CHECK-NEXT:          <a class="sidebar-item" href="#{{[0-9A-F]*}}">enum Color</a>
+// CHECK-NEXT:      </li>
+// CHECK-NEXT:  </ul>
+// CHECK:           <li class="sidebar-section">
+// CHECK-NEXT:          <a class="sidebar-item" href="#Classes">Inner Classes</a>
+// CHECK-NEXT:      </li>
+// CHECK-NEXT:  <ul>
+// CHECK-NEXT:      <li class="sidebar-item-container">
+// CHECK-NEXT:          <a class="sidebar-item" href="#{{[0-9A-F]*}}">Foo</a>
+// CHECK-NEXT:      </li>
+// CHECK-NEXT:  </ul>
+
+// CHECK:       <section id="Enums" class="section-container">
+// CHECK-NEXT:      <h2>Enumerations</h2>
+// CHECK-NEXT:      <div>
+// CHECK-NEXT:          <div id="{{[0-9A-F]*}}" class="delimiter-container">
+// CHECK-NEXT:              <div>
+// CHECK-NEXT:                  <pre>
+// CHECK-NEXT:                      <code class="language-cpp code-clang-doc">
+// CHECK-NEXT:                          enum Color
+// CHECK-NEXT:                      </code>
+// CHECK-NEXT:                  </pre>
+// CHECK-NEXT:              </div>
+// CHECK-NEXT:              <table class="table-wrapper">
+// CHECK-NEXT:                  <tbody>
+// CHECK-NEXT:                  <tr>
+// CHECK-NEXT:                      <th>Name</th>
+// CHECK-NEXT:                      <th>Value</th>
+// CHECK:                       </tr>
+// CHECK-NEXT:                      <tr>
+// CHECK-NEXT:                          <td>RED</td>
+// CHECK-NEXT:                          <td>0</td>
+// CHECK:                           </tr>
+// CHECK-NEXT:                      <tr>
+// CHECK-NEXT:                          <td>BLUE</td>
+// CHECK-NEXT:                          <td>1</td>
+// CHECK:                           </tr>
+// CHECK-NEXT:                      <tr>
+// CHECK-NEXT:                          <td>GREEN</td>
+// CHECK-NEXT:                          <td>2</td>
+// CHECK:                           </tr>
+// CHECK-NEXT:                  </tbody>
+// CHECK-NEXT:              </table>
+// CHECK-NEXT:              <div>
+// CHECK-NEXT:                  Defined at line 5 of file {{.*}}mustache-index.cpp
+// CHECK-NEXT:              </div>
+// CHECK-NEXT:          </div>
+// CHECK-NEXT:      </div>
+// CHECK-NEXT:  </section>
+
+// CHECK:       <section id="Classes" class="section-container">
+// CHECK-NEXT:      <h2>Inner Classes</h2>
+// CHECK-NEXT:      <ul class="class-container">
+// CHECK-NEXT:          <li id="{{[0-9A-F]*}}" style="max-height: 40px;">
+// CHECK-NEXT:              <a href="Foo.html"><pre><code class="language-cpp code-clang-doc" >class Foo</code></pre></a>
+// CHECK-NEXT:          </li>
+// CHECK-NEXT:      </ul>
+// CHECK-NEXT:  </section>
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/mustache-separate-namespace.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/mustache-separate-namespace.cpp
new file mode 100644
index 000000000000..ec29b2449169
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/mustache-separate-namespace.cpp
@@ -0,0 +1,13 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --format=mustache --output=%t --executor=standalone %s 
+// RUN: FileCheck %s < %t/MyNamespace/index.html
+
+namespace MyNamespace {
+  class Foo;
+}
+
+// CHECK:       <ul class="class-container">
+// CHECK-NEXT:    <li id="{{[0-9A-F]*}}" style="max-height: 40px;">
+// CHECK-NEXT:        <a href="Foo.html"><pre><code class="language-cpp code-clang-doc" >class Foo</code></pre></a>
+// CHECK-NEXT:    </li>
+// CHECK-NEXT: </ul>
diff --git a/external/llvm-project/clang-tools-extra/test/clang-doc/templates.cpp b/external/llvm-project/clang-tools-extra/test/clang-doc/templates.cpp
index 426a0b16befd..abe03a7d2d0e 100644
--- a/external/llvm-project/clang-tools-extra/test/clang-doc/templates.cpp
+++ b/external/llvm-project/clang-tools-extra/test/clang-doc/templates.cpp
@@ -112,22 +112,22 @@ tuple<int, int, bool> func_with_tuple_param(tuple<int, int, bool> t) { return t;
 // YAML-NEXT:   - USR:             '{{([0-9A-F]{40})}}'
 // YAML-NEXT:    Name:            'func_with_tuple_param'
 // YAML-NEXT:    Description:
-// YAML-NEXT:      - Kind:            'FullComment'
+// YAML-NEXT:      - Kind:            FullComment
 // YAML-NEXT:        Children:
-// YAML-NEXT:          - Kind:            'ParagraphComment'
+// YAML-NEXT:          - Kind:            ParagraphComment
 // YAML-NEXT:            Children:
-// YAML-NEXT:              - Kind:            'TextComment'
+// YAML-NEXT:              - Kind:            TextComment
 // YAML-NEXT:                Text:            ' A function with a tuple parameter'
-// YAML-NEXT:          - Kind:            'ParagraphComment'
+// YAML-NEXT:          - Kind:            ParagraphComment
 // YAML-NEXT:            Children:
-// YAML-NEXT:              - Kind:            'TextComment'
-// YAML-NEXT:          - Kind:            'ParamCommandComment'
+// YAML-NEXT:              - Kind:            TextComment
+// YAML-NEXT:          - Kind:            ParamCommandComment
 // YAML-NEXT:            Direction:       '[in]'
 // YAML-NEXT:            ParamName:       't'
 // YAML-NEXT:            Children:
-// YAML-NEXT:              - Kind:            'ParagraphComment'
+// YAML-NEXT:              - Kind:            ParagraphComment
 // YAML-NEXT:                Children:
-// YAML-NEXT:                  - Kind:            'TextComment'
+// YAML-NEXT:                  - Kind:            TextComment
 // YAML-NEXT:                    Text:            ' The input to func_with_tuple_param'
 // YAML-NEXT:    DefLocation:
 // YAML-NEXT:      LineNumber:      [[# @LINE - 23]]
diff --git a/external/llvm-project/clang-tools-extra/test/clang-reorder-fields/MacroExpansionField.cpp b/external/llvm-project/clang-tools-extra/test/clang-reorder-fields/MacroExpansionField.cpp
new file mode 100644
index 000000000000..a4c3cbc1e12f
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-reorder-fields/MacroExpansionField.cpp
@@ -0,0 +1,24 @@
+// RUN: clang-reorder-fields -record-name ::bar::Foo -fields-order z,y,x %s -- | FileCheck %s
+
+namespace bar {
+
+#define INT_DECL(NAME) int NAME // CHECK:      {{^#define INT_DECL\(NAME\) int NAME}}
+#define MACRO_DECL int x;       // CHECK-NEXT: {{^#define MACRO_DECL int x;}}
+
+struct Foo {
+  MACRO_DECL   // CHECK:      {{^ INT_DECL\(z\);}}
+  int y;       // CHECK-NEXT: {{^ int y;}}
+  INT_DECL(z); // CHECK-NEXT: {{^ MACRO_DECL}}
+};
+
+#define FOO 0 // CHECK:      {{^#define FOO 0}}
+#define BAR 1 // CHECK-NEXT: {{^#define BAR 1}}
+#define BAZ 2 // CHECK-NEXT: {{^#define BAZ 2}}
+
+struct Foo foo = {
+  FOO, // CHECK:      {{^ BAZ,}}
+  BAR, // CHECK-NEXT: {{^ BAR,}}
+  BAZ, // CHECK-NEXT: {{^ FOO,}}
+};
+
+} // end namespace bar
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/bugprone/multi-level-implicit-pointer-conversion.c b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/bugprone/multi-level-implicit-pointer-conversion.c
new file mode 100644
index 000000000000..9dc5dae5c80f
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/bugprone/multi-level-implicit-pointer-conversion.c
@@ -0,0 +1,11 @@
+// RUN: %check_clang_tidy -check-suffixes=ENABLE-IN-C %s bugprone-multi-level-implicit-pointer-conversion %t -- -config="{CheckOptions: {bugprone-multi-level-implicit-pointer-conversion.EnableInC: true}}"
+// RUN: %check_clang_tidy -check-suffixes=DISABLE-IN-C %s bugprone-multi-level-implicit-pointer-conversion %t -- -config="{CheckOptions: {bugprone-multi-level-implicit-pointer-conversion.EnableInC: false}}"
+
+void free(void*);
+
+void test() {
+  char **p;
+  free(p);
+  // CHECK-MESSAGES-ENABLE-IN-C: :[[@LINE-1]]:8: warning: multilevel pointer conversion from 'char **' to 'void *', please use explicit cast [bugprone-multi-level-implicit-pointer-conversion]
+  free((void *)p);
+}
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/google/readability-namespace-comments-missing-nested-namespaces.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/google/readability-namespace-comments-missing-nested-namespaces.cpp
new file mode 100644
index 000000000000..de42df30cb79
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/google/readability-namespace-comments-missing-nested-namespaces.cpp
@@ -0,0 +1,32 @@
+// RUN: %check_clang_tidy %s google-readability-namespace-comments %t -std=c++20 \
+// RUN:   '-config={CheckOptions: { \
+// RUN:     google-readability-namespace-comments.AllowOmittingNamespaceComments: true, \
+// RUN:     google-readability-namespace-comments.ShortNamespaceLines: 0, \
+// RUN:   }}'
+
+// accept if namespace comments are fully omitted
+namespace n1::n2 {
+namespace /*comment1*/n3/*comment2*/::/*comment3*/inline/*comment4*/n4/*comment5*/ {
+void f();
+}}
+
+namespace n5::inline n6 {
+void f();
+}
+
+namespace n7::inline n8 {
+void f();
+}
+
+// accept if namespace comments are partly omitted (e.g. only for nested namespace)
+namespace n1::n2 {
+namespace n3::n4 {
+void f();
+}
+} // namespace n1::n2
+
+// fail if namespace comment is different than expected
+namespace n9::inline n10 {
+void f();
+} // namespace n9::n10
+// CHECK-MESSAGES: :[[@LINE-1]]:2: warning: namespace 'n9::inline n10' ends with a comment that refers to a wrong namespace 'n9::n10' [google-readability-namespace-comments]
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/google/readability-namespace-comments-missing.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/google/readability-namespace-comments-missing.cpp
new file mode 100644
index 000000000000..019add1ed06d
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/google/readability-namespace-comments-missing.cpp
@@ -0,0 +1,40 @@
+// RUN: %check_clang_tidy %s google-readability-namespace-comments %t \
+// RUN:   -config='{CheckOptions: { \
+// RUN:     google-readability-namespace-comments.AllowOmittingNamespaceComments: true, \
+// RUN:     google-readability-namespace-comments.ShortNamespaceLines: 0, \
+// RUN:   }}'
+
+// accept if namespace comments are fully omitted
+namespace n1 {
+namespace /* a comment */ n2 /* another comment */ {
+void f();
+}}
+
+#define MACRO macro_expansion
+namespace MACRO {
+void f();
+}
+
+namespace [[deprecated("foo")]] namespace_with_attr {
+inline namespace inline_namespace {
+void f();
+}
+}
+
+namespace [[]] {
+void f();
+}
+
+// accept if namespace comments are partly omitted (e.g. only for nested namespace)
+namespace n3 {
+namespace n4 {
+void f();
+} // n4
+}
+
+// fail if namespace comment is different than expected
+namespace n1 {
+void f();
+} // namespace n2
+// CHECK-MESSAGES: :[[@LINE-1]]:2: warning: namespace 'n1' ends with a comment that refers to a wrong namespace 'n2' [google-readability-namespace-comments]
+
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas-cxx14.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas-cxx14.cpp
new file mode 100644
index 000000000000..33051c63e4f1
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas-cxx14.cpp
@@ -0,0 +1,24 @@
+// RUN: %check_clang_tidy -std=c++14-or-later %s modernize-use-trailing-return-type %t -- -- -fno-delayed-template-parsing
+
+namespace std {
+    template <typename T>
+    class vector {};
+
+    class string {};
+} // namespace std
+
+void test_lambda_positive() {
+  auto l1 = [](auto x) { return x; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l1 = [](auto x) -> auto { return x; };{{$}}
+}
+
+template <template <typename> class C>
+void test_lambda_positive_template() {
+  auto l1 = []() { return C<int>{}; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l1 = []() -> auto { return C<int>{}; };{{$}}
+  auto l2 = []() { return 0; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l2 = []() -> auto { return 0; };{{$}}
+}
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas-cxx20.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas-cxx20.cpp
new file mode 100644
index 000000000000..521b5c6d3618
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas-cxx20.cpp
@@ -0,0 +1,35 @@
+// RUN: %check_clang_tidy -std=c++20-or-later %s modernize-use-trailing-return-type %t -- -- -fno-delayed-template-parsing
+
+namespace std {
+template <typename T, typename U>
+struct is_same { static constexpr auto value = false; };
+
+template <typename T>
+struct is_same<T, T> { static constexpr auto value = true; };
+
+template <typename T>
+concept floating_point = std::is_same<T, float>::value || std::is_same<T, double>::value || std::is_same<T, long double>::value;
+}
+
+void test_lambda_positive() {
+  auto l1 = []<typename T, typename U>(T x, U y) { return x + y; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l1 = []<typename T, typename U>(T x, U y) -> auto { return x + y; };{{$}}
+  auto l2 = [](auto x) requires requires { x + x; } { return x; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l2 = [](auto x) -> auto requires requires { x + x; } { return x; };{{$}}
+  auto l3 = [](auto x) requires std::floating_point<decltype(x)> { return x; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l3 = [](auto x) -> auto requires std::floating_point<decltype(x)> { return x; };{{$}}
+  auto l4 = [](int x) consteval { return x; };
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l4 = [](int x) consteval -> int { return x; };{{$}}
+  // Complete complex example
+  auto l5 = []<typename T, typename U>(T x, U y) constexpr noexcept
+    requires std::floating_point<T> && std::floating_point<U>
+  { return x * y; };
+  // CHECK-MESSAGES: :[[@LINE-3]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES: {{^}}  auto l5 = []<typename T, typename U>(T x, U y) constexpr noexcept{{$}}
+  // CHECK-FIXES: {{^}}    -> auto requires std::floating_point<T> && std::floating_point<U>{{$}}
+  // CHECK-FIXES: {{^}}  { return x * y; };{{$}}
+}
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas.cpp
new file mode 100644
index 000000000000..0b7e7a15b2f1
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-transform-lambdas.cpp
@@ -0,0 +1,89 @@
+// RUN: %check_clang_tidy -check-suffix=ALL -std=c++11-or-later %s modernize-use-trailing-return-type %t --\
+// RUN:   -config="{CheckOptions: {modernize-use-trailing-return-type.TransformLambdas: all, \
+// RUN:                            modernize-use-trailing-return-type.TransformFunctions: false}}" \
+// RUN:   -- -fno-delayed-template-parsing
+// RUN: %check_clang_tidy -check-suffix=NOAUTO -std=c++11-or-later %s modernize-use-trailing-return-type %t --\
+// RUN:   -config="{CheckOptions: {modernize-use-trailing-return-type.TransformLambdas: all_except_auto, \
+// RUN:                            modernize-use-trailing-return-type.TransformFunctions: false}}" \
+// RUN:   -- -fno-delayed-template-parsing
+// RUN: %check_clang_tidy -check-suffix=NONE -std=c++11-or-later %s modernize-use-trailing-return-type %t --\
+// RUN:   -config="{CheckOptions: {modernize-use-trailing-return-type.TransformLambdas: none, \
+// RUN:                            modernize-use-trailing-return-type.TransformFunctions: true}}" \
+// RUN:   -- -fno-delayed-template-parsing
+
+namespace std {
+    template <typename T>
+    class vector {};
+
+    class string {};
+} // namespace std
+
+void test_lambda_positive() {
+  auto l01 = [] {};
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:14: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:14: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l01 = [] -> void {};{{$}}
+  auto l1 = []() {};
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l1 = []() -> void {};{{$}}
+  auto l2 = []() { return 42; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l2 = []() -> int { return 42; };{{$}}
+  auto l3 = [](int x, double y) { return x * y; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l3 = [](int x, double y) -> double { return x * y; };{{$}}
+
+  int capture_int = 10;
+  double capture_double = 3.14;
+  int* capture_ptr = nullptr;
+  
+  auto l4 = [capture_int]() { return capture_int; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l4 = [capture_int]() -> int { return capture_int; };{{$}}
+  auto l5 = [capture_int, &capture_double](char c) { return capture_int + capture_double + c; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l5 = [capture_int, &capture_double](char c) -> double { return capture_int + capture_double + c; };{{$}}
+  auto l6 = [capture_int]() constexpr mutable noexcept { return ++capture_int; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l6 = [capture_int]() constexpr mutable noexcept -> int { return ++capture_int; };{{$}}
+  auto l7 = [&capture_ptr]() { return capture_ptr; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l7 = [&capture_ptr]() -> int * { return capture_ptr; };{{$}}
+  auto l8 = [&capture_int]() { return capture_int; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l8 = [&capture_int]() -> int { return capture_int; };{{$}}
+  auto l9 = [] { return std::vector<std::vector<int>>{}; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l9 = [] -> std::vector<std::vector<int>> { return std::vector<std::vector<int>>{}; };{{$}}
+  auto l10 = [] { const char* const * const * const ptr = nullptr; return ptr; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:14: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-MESSAGES-NOAUTO: :[[@LINE-2]]:14: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  // CHECK-FIXES-ALL: {{^}}  auto l10 = [] -> const char *const *const * { const char* const * const * const ptr = nullptr; return ptr; };{{$}}
+}
+
+// In c++11 mode we can not write 'auto' type, see *-cxx14.cpp for fixes.
+template <template <typename> class C>
+void test_lambda_positive_template() {
+  auto l1 = []() { return C<int>{}; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+  auto l2 = []() { return 0; };
+  // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: use a trailing return type for this lambda [modernize-use-trailing-return-type]
+}
+
+void test_lambda_negative() {
+  auto l1_good = [](int arg) -> int { return 0; };  
+}
+
+// this function is solely used to not to get "wrong config error" from the check.
+int f();
+// CHECK-MESSAGES-NONE: :[[@LINE-1]]:5: warning: use a trailing return type for this function [modernize-use-trailing-return-type]
+// CHECK-FIXES-NONE: {{^}}auto f() -> int;{{$}}
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-wrong-config.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-wrong-config.cpp
new file mode 100644
index 000000000000..167e355f9fea
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type-wrong-config.cpp
@@ -0,0 +1,5 @@
+// RUN: %check_clang_tidy -std=c++11-or-later %s modernize-use-trailing-return-type %t \
+// RUN:   -config="{CheckOptions: {modernize-use-trailing-return-type.TransformLambdas: none, \
+// RUN:                            modernize-use-trailing-return-type.TransformFunctions: false}}"
+
+// CHECK-MESSAGES: warning: The check 'modernize-use-trailing-return-type' will not perform any analysis because 'TransformFunctions' and 'TransformLambdas' are disabled. [clang-tidy-config]
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type.cpp
index e1f36c52a7c0..c16c033f0b01 100644
--- a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type.cpp
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/use-trailing-return-type.cpp
@@ -594,5 +594,3 @@ struct D2 : B {
     operator double();
 };
 
-auto l1 = [](int arg) {};
-auto l2 = [](int arg) -> double { return 0; };
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/readability/convert-member-functions-to-static-deducing-this.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/readability/convert-member-functions-to-static-deducing-this.cpp
new file mode 100644
index 000000000000..7974301aecce
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/readability/convert-member-functions-to-static-deducing-this.cpp
@@ -0,0 +1,22 @@
+// RUN: %check_clang_tidy -std=c++23-or-later %s readability-convert-member-functions-to-static %t
+
+namespace std{
+  class string {};
+  void println(const char *format, const std::string &str) {}
+}
+
+struct Hello {
+  std::string str_;
+
+  void ByValueSelf(this Hello self) { std::println("Hello, {0}!", self.str_); }
+
+  void ByLRefSelf(this Hello &self) { std::println("Hello, {0}!", self.str_); }
+
+  void ByRRefSelf(this Hello&& self) {}
+
+  template<typename Self> void ByForwardRefSelf(this Self&& self) {}
+
+  void MultiParam(this Hello &self, int a, double b) {}
+
+  void UnnamedExplicitObjectParam(this Hello &) {}
+};
diff --git a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp
index 80d2bc304bb5..947eda07a767 100644
--- a/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp
+++ b/external/llvm-project/clang-tools-extra/test/clang-tidy/checkers/readability/math-missing-parentheses.cpp
@@ -157,3 +157,19 @@ namespace PR92516 {
     for (j = i + 1, 2; j < 1; ++j) {}
   }
 }
+
+namespace PR141249 {
+  void AssignAsParentBinOp(int* netChange, int* nums, int k, int i) {
+    //CHECK-MESSAGES: :[[@LINE+2]]:30: warning: '-' has higher precedence than '^'; add parentheses to explicitly specify the order of operations [readability-math-missing-parentheses]
+    //CHECK-FIXES: netChange[i] = nums[i] ^ (k - nums[i]);
+    netChange[i] = nums[i] ^ k - nums[i];
+  }
+}
+
+void CompareAsParentBinOp(int b) {
+  //CHECK-MESSAGES: :[[@LINE+2]]:12: warning: '*' has higher precedence than '-'; add parentheses to explicitly specify the order of operations [readability-math-missing-parentheses]
+  //CHECK-FIXES: if (b == (1 * 2) - 3)   {
+  if (b == 1 * 2 - 3)   {
+
+  }
+}
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp b/external/llvm-project/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
index 4f2466af9a6b..659870d2a5c0 100644
--- a/external/llvm-project/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
@@ -37,7 +37,7 @@ static std::string writeInfo(Info *I) {
     return writeInfo(*static_cast<FunctionInfo *>(I));
   case InfoType::IT_typedef:
     return writeInfo(*static_cast<TypedefInfo *>(I));
-  default:
+  case InfoType::IT_default:
     return "";
   }
 }
@@ -93,12 +93,12 @@ TEST(BitcodeTest, emitRecordInfoBitcode) {
 
   // Documentation for the data member.
   CommentInfo TopComment;
-  TopComment.Kind = "FullComment";
+  TopComment.Kind = CommentKind::CK_FullComment;
   TopComment.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Brief = TopComment.Children.back().get();
-  Brief->Kind = "ParagraphComment";
+  Brief->Kind = CommentKind::CK_ParagraphComment;
   Brief->Children.emplace_back(std::make_unique<CommentInfo>());
-  Brief->Children.back()->Kind = "TextComment";
+  Brief->Children.back()->Kind = CommentKind::CK_TextComment;
   Brief->Children.back()->Name = "ParagraphComment";
   Brief->Children.back()->Text = "Value of the thing.";
   I.Bases.back().Members.back().Description.emplace_back(std::move(TopComment));
@@ -184,13 +184,13 @@ TEST(BitcodeTest, emitTypedefInfoBitcode) {
   I.IsUsing = true;
 
   CommentInfo Top;
-  Top.Kind = "FullComment";
+  Top.Kind = CommentKind::CK_FullComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *BlankLine = Top.Children.back().get();
-  BlankLine->Kind = "ParagraphComment";
+  BlankLine->Kind = CommentKind::CK_ParagraphComment;
   BlankLine->Children.emplace_back(std::make_unique<CommentInfo>());
-  BlankLine->Children.back()->Kind = "TextComment";
+  BlankLine->Children.back()->Kind = CommentKind::CK_TextComment;
 
   I.Description.emplace_back(std::move(Top));
 
@@ -220,103 +220,105 @@ TEST(SerializeTest, emitInfoWithCommentBitcode) {
   F.Params.emplace_back(TypeInfo("int"), "I");
 
   CommentInfo Top;
-  Top.Kind = "FullComment";
+  Top.Kind = CommentKind::CK_FullComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *BlankLine = Top.Children.back().get();
-  BlankLine->Kind = "ParagraphComment";
+  BlankLine->Kind = CommentKind::CK_ParagraphComment;
   BlankLine->Children.emplace_back(std::make_unique<CommentInfo>());
-  BlankLine->Children.back()->Kind = "TextComment";
+  BlankLine->Children.back()->Kind = CommentKind::CK_TextComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Brief = Top.Children.back().get();
-  Brief->Kind = "ParagraphComment";
+  Brief->Kind = CommentKind::CK_ParagraphComment;
   Brief->Children.emplace_back(std::make_unique<CommentInfo>());
-  Brief->Children.back()->Kind = "TextComment";
+  Brief->Children.back()->Kind = CommentKind::CK_TextComment;
   Brief->Children.back()->Name = "ParagraphComment";
   Brief->Children.back()->Text = " Brief description.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Extended = Top.Children.back().get();
-  Extended->Kind = "ParagraphComment";
+  Extended->Kind = CommentKind::CK_ParagraphComment;
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " Extended description that";
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " continues onto the next line.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *HTML = Top.Children.back().get();
-  HTML->Kind = "ParagraphComment";
+  HTML->Kind = CommentKind::CK_ParagraphComment;
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "TextComment";
+  HTML->Children.back()->Kind = CommentKind::CK_TextComment;
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLStartTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLStartTagComment;
   HTML->Children.back()->Name = "ul";
   HTML->Children.back()->AttrKeys.emplace_back("class");
   HTML->Children.back()->AttrValues.emplace_back("test");
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLStartTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLStartTagComment;
   HTML->Children.back()->Name = "li";
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "TextComment";
+  HTML->Children.back()->Kind = CommentKind::CK_TextComment;
   HTML->Children.back()->Text = " Testing.";
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLEndTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLEndTagComment;
   HTML->Children.back()->Name = "ul";
   HTML->Children.back()->SelfClosing = true;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Verbatim = Top.Children.back().get();
-  Verbatim->Kind = "VerbatimBlockComment";
+  Verbatim->Kind = CommentKind::CK_VerbatimBlockComment;
   Verbatim->Name = "verbatim";
   Verbatim->CloseName = "endverbatim";
   Verbatim->Children.emplace_back(std::make_unique<CommentInfo>());
-  Verbatim->Children.back()->Kind = "VerbatimBlockLineComment";
+  Verbatim->Children.back()->Kind = CommentKind::CK_VerbatimBlockLineComment;
   Verbatim->Children.back()->Text = " The description continues.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *ParamOut = Top.Children.back().get();
-  ParamOut->Kind = "ParamCommandComment";
+  ParamOut->Kind = CommentKind::CK_ParamCommandComment;
   ParamOut->Direction = "[out]";
   ParamOut->ParamName = "I";
   ParamOut->Explicit = true;
   ParamOut->Children.emplace_back(std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Kind = "ParagraphComment";
+  ParamOut->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   ParamOut->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Children.back()->Kind = "TextComment";
+  ParamOut->Children.back()->Children.back()->Kind =
+      CommentKind::CK_TextComment;
   ParamOut->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Children.back()->Kind = "TextComment";
+  ParamOut->Children.back()->Children.back()->Kind =
+      CommentKind::CK_TextComment;
   ParamOut->Children.back()->Children.back()->Text = " is a parameter.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *ParamIn = Top.Children.back().get();
-  ParamIn->Kind = "ParamCommandComment";
+  ParamIn->Kind = CommentKind::CK_ParamCommandComment;
   ParamIn->Direction = "[in]";
   ParamIn->ParamName = "J";
   ParamIn->Children.emplace_back(std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Kind = "ParagraphComment";
+  ParamIn->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   ParamIn->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Children.back()->Kind = "TextComment";
+  ParamIn->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
   ParamIn->Children.back()->Children.back()->Text = " is a parameter.";
   ParamIn->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Children.back()->Kind = "TextComment";
+  ParamIn->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Return = Top.Children.back().get();
-  Return->Kind = "BlockCommandComment";
+  Return->Kind = CommentKind::CK_BlockCommandComment;
   Return->Name = "return";
   Return->Explicit = true;
   Return->Children.emplace_back(std::make_unique<CommentInfo>());
-  Return->Children.back()->Kind = "ParagraphComment";
+  Return->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   Return->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  Return->Children.back()->Children.back()->Kind = "TextComment";
+  Return->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
   Return->Children.back()->Children.back()->Text = "void";
 
   F.Description.emplace_back(std::move(Top));
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/CMakeLists.txt b/external/llvm-project/clang-tools-extra/unittests/clang-doc/CMakeLists.txt
index 59a856ed987d..18166acf9bbc 100644
--- a/external/llvm-project/clang-tools-extra/unittests/clang-doc/CMakeLists.txt
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/CMakeLists.txt
@@ -31,6 +31,7 @@ add_extra_unittest(ClangDocTests
   MergeTest.cpp
   SerializeTest.cpp
   YAMLGeneratorTest.cpp
+  JSONGeneratorTest.cpp
   )
 
 clang_target_link_libraries(ClangDocTests
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp b/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
index edf502475b51..143414354d36 100644
--- a/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
@@ -407,37 +407,37 @@ TEST(HTMLGeneratorTest, emitCommentHTML) {
   I.Access = AccessSpecifier::AS_none;
 
   CommentInfo Top;
-  Top.Kind = "FullComment";
+  Top.Kind = CommentKind::CK_FullComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *BlankLine = Top.Children.back().get();
-  BlankLine->Kind = "ParagraphComment";
+  BlankLine->Kind = CommentKind::CK_ParagraphComment;
   BlankLine->Children.emplace_back(std::make_unique<CommentInfo>());
-  BlankLine->Children.back()->Kind = "TextComment";
+  BlankLine->Children.back()->Kind = CommentKind::CK_TextComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Brief = Top.Children.back().get();
-  Brief->Kind = "ParagraphComment";
+  Brief->Kind = CommentKind::CK_ParagraphComment;
   Brief->Children.emplace_back(std::make_unique<CommentInfo>());
-  Brief->Children.back()->Kind = "TextComment";
+  Brief->Children.back()->Kind = CommentKind::CK_TextComment;
   Brief->Children.back()->Name = "ParagraphComment";
   Brief->Children.back()->Text = " Brief description.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Extended = Top.Children.back().get();
-  Extended->Kind = "ParagraphComment";
+  Extended->Kind = CommentKind::CK_ParagraphComment;
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " Extended description that";
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " continues onto the next line.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Entities = Top.Children.back().get();
-  Entities->Kind = "ParagraphComment";
+  Entities->Kind = CommentKind::CK_ParagraphComment;
   Entities->Children.emplace_back(std::make_unique<CommentInfo>());
-  Entities->Children.back()->Kind = "TextComment";
+  Entities->Children.back()->Kind = CommentKind::CK_TextComment;
   Entities->Children.back()->Name = "ParagraphComment";
   Entities->Children.back()->Text =
       " Comment with html entities: &, <, >, \", \'.";
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLMustacheGeneratorTest.cpp b/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLMustacheGeneratorTest.cpp
index 95acd363a958..32b0846a02db 100644
--- a/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLMustacheGeneratorTest.cpp
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/HTMLMustacheGeneratorTest.cpp
@@ -87,29 +87,6 @@ TEST(HTMLMustacheGeneratorTest, createResources) {
   }
 }
 
-TEST(HTMLMustacheGeneratorTest, generateDocs) {
-  auto G = getHTMLMustacheGenerator();
-  assert(G && "Could not find HTMLMustacheGenerator");
-  ClangDocContext CDCtx = getClangDocContext();
-
-  unittest::TempDir RootTestDirectory("generateDocsTest", /*Unique=*/true);
-  CDCtx.OutDirectory = RootTestDirectory.path();
-
-#if ENABLE_LOCAL_TEST
-  // FIXME: We can't read files during unit tests. Migrate to lit once
-  // tool support lands.
-  getMustacheHtmlFiles(CLANG_DOC_TEST_ASSET_DIR, CDCtx);
-
-  EXPECT_THAT_ERROR(G->generateDocs(RootTestDirectory.path(), {}, CDCtx),
-                    Succeeded())
-      << "Failed to generate docs.";
-#else
-  EXPECT_THAT_ERROR(G->generateDocs(RootTestDirectory.path(), {}, CDCtx),
-                    Failed())
-      << "Failed to generate docs.";
-#endif
-}
-
 TEST(HTMLGeneratorTest, emitFunctionHTML) {
 #if ENABLE_LOCAL_TEST
   auto G = getHTMLMustacheGenerator();
@@ -160,50 +137,6 @@ TEST(HTMLGeneratorTest, emitFunctionHTML) {
 #endif
 }
 
-TEST(HTMLMustacheGeneratorTest, emitEnumHTML) {
-#if ENABLE_LOCAL_TEST
-  auto G = getHTMLMustacheGenerator();
-  assert(G && "Could not find HTMLMustacheGenerator");
-  ClangDocContext CDCtx = getClangDocContext();
-  std::string Buffer;
-  llvm::raw_string_ostream Actual(Buffer);
-
-  unittest::TempDir RootTestDirectory("emitEnumHTML",
-                                      /*Unique=*/true);
-  CDCtx.OutDirectory = RootTestDirectory.path();
-
-  getMustacheHtmlFiles(CLANG_DOC_TEST_ASSET_DIR, CDCtx);
-
-  // FIXME: This is a terrible hack, since we can't initialize the templates
-  // directly. We'll need to update the interfaces so that we can call
-  // SetupTemplateFiles() from outsize of HTMLMustacheGenerator.cpp
-  EXPECT_THAT_ERROR(G->generateDocs(RootTestDirectory.path(), {}, CDCtx),
-                    Succeeded())
-      << "Failed to generate docs.";
-
-  CDCtx.RepositoryUrl = "http://www.repository.com";
-
-  EnumInfo I;
-  I.Name = "e";
-  I.Namespace.emplace_back(EmptySID, "A", InfoType::IT_namespace);
-
-  I.DefLoc = Location(10, 10, "test.cpp", true);
-  I.Loc.emplace_back(12, 12, "test.cpp");
-
-  I.Members.emplace_back("X");
-  I.Scoped = true;
-
-  auto Err = G->generateDocForInfo(&I, Actual, CDCtx);
-  assert(!Err);
-
-  std::string Expected = R"raw(IT_enum
-)raw";
-
-  // FIXME: Enums are not handled yet.
-  EXPECT_EQ(Expected, Actual.str());
-#endif
-}
-
 TEST(HTMLMustacheGeneratorTest, emitCommentHTML) {
 #if ENABLE_LOCAL_TEST
   auto G = getHTMLMustacheGenerator();
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/JSONGeneratorTest.cpp b/external/llvm-project/clang-tools-extra/unittests/clang-doc/JSONGeneratorTest.cpp
new file mode 100644
index 000000000000..09e522133d83
--- /dev/null
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/JSONGeneratorTest.cpp
@@ -0,0 +1,247 @@
+#include "ClangDocTest.h"
+#include "Generators.h"
+#include "Representation.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace doc {
+
+static std::unique_ptr<Generator> getJSONGenerator() {
+  auto G = doc::findGeneratorByName("json");
+  if (!G)
+    return nullptr;
+  return std::move(G.get());
+}
+
+TEST(JSONGeneratorTest, emitRecordJSON) {
+  RecordInfo I;
+  I.Name = "Foo";
+  // FIXME: FullName is not emitted correctly.
+  I.FullName = "";
+  I.IsTypeDef = false;
+  I.Namespace.emplace_back(EmptySID, "GlobalNamespace", InfoType::IT_namespace);
+  I.Path = "GlobalNamespace";
+  I.DefLoc = Location(1, 1, "main.cpp");
+  I.TagType = TagTypeKind::Class;
+
+  I.Template = TemplateInfo();
+  I.Template->Params.emplace_back("class T");
+
+  I.Children.Enums.emplace_back();
+  I.Children.Enums.back().Name = "Color";
+  I.Children.Enums.back().Scoped = false;
+  I.Children.Enums.back().Members.emplace_back();
+  I.Children.Enums.back().Members.back().Name = "RED";
+  I.Children.Enums.back().Members.back().Value = "0";
+
+  I.Members.emplace_back(TypeInfo("int"), "X", AccessSpecifier::AS_protected);
+
+  I.Bases.emplace_back(EmptySID, "F", "path/to/F", true,
+                       AccessSpecifier::AS_public, true);
+  I.Bases.back().Children.Functions.emplace_back();
+  I.Bases.back().Children.Functions.back().Name = "InheritedFunctionOne";
+  I.Bases.back().Members.emplace_back(TypeInfo("int"), "N",
+                                      AccessSpecifier::AS_public);
+
+  // F is in the global namespace
+  I.Parents.emplace_back(EmptySID, "F", InfoType::IT_record, "");
+  I.VirtualParents.emplace_back(EmptySID, "G", InfoType::IT_record,
+                                "path::to::G::G", "path/to/G");
+
+  I.Children.Records.emplace_back(EmptySID, "ChildStruct", InfoType::IT_record,
+                                  "path::to::A::r::ChildStruct", "path/to/A/r");
+  I.Children.Functions.emplace_back();
+  I.Children.Functions.back().Name = "OneFunction";
+
+  auto G = getJSONGenerator();
+  assert(G);
+  std::string Buffer;
+  llvm::raw_string_ostream Actual(Buffer);
+  auto Err = G->generateDocForInfo(&I, Actual, ClangDocContext());
+  assert(!Err);
+  std::string Expected = R"raw({
+  "Bases": [
+    {
+      "Access": "public",
+      "FullName": "",
+      "IsParent": true,
+      "IsTypedef": false,
+      "IsVirtual": true,
+      "Name": "F",
+      "Path": "path/to/F",
+      "PublicFunctions": [
+        {
+          "IsStatic": false,
+          "Name": "InheritedFunctionOne",
+          "ReturnType": {
+            "IsBuiltIn": false,
+            "IsTemplate": false,
+            "Name": "",
+            "QualName": "",
+            "USR": "0000000000000000000000000000000000000000"
+          },
+          "USR": "0000000000000000000000000000000000000000"
+        }
+      ],
+      "PublicMembers": [
+        {
+          "Name": "N",
+          "Type": "int"
+        }
+      ],
+      "TagType": "struct",
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "Enums": [
+    {
+      "Members": [
+        {
+          "Name": "RED",
+          "Value": "0"
+        }
+      ],
+      "Name": "Color",
+      "Scoped": false,
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "FullName": "",
+  "IsTypedef": false,
+  "Location": {
+    "Filename": "main.cpp",
+    "LineNumber": 1
+  },
+  "Name": "Foo",
+  "Namespace": [
+    "GlobalNamespace"
+  ],
+  "Parents": [
+    {
+      "Name": "F",
+      "Path": "",
+      "QualName": "",
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "Path": "GlobalNamespace",
+  "ProtectedMembers": [
+    {
+      "Name": "X",
+      "Type": "int"
+    }
+  ],
+  "PublicFunctions": [
+    {
+      "IsStatic": false,
+      "Name": "OneFunction",
+      "ReturnType": {
+        "IsBuiltIn": false,
+        "IsTemplate": false,
+        "Name": "",
+        "QualName": "",
+        "USR": "0000000000000000000000000000000000000000"
+      },
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "Records": [
+    {
+      "Name": "ChildStruct",
+      "Path": "path/to/A/r",
+      "QualName": "path::to::A::r::ChildStruct",
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "TagType": "class",
+  "Template": {
+    "Parameters": [
+      "class T"
+    ]
+  },
+  "USR": "0000000000000000000000000000000000000000",
+  "VirtualParents": [
+    {
+      "Name": "G",
+      "Path": "path/to/G",
+      "QualName": "path::to::G::G",
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ]
+})raw";
+  EXPECT_EQ(Expected, Actual.str());
+}
+
+TEST(JSONGeneratorTest, emitNamespaceJSON) {
+  NamespaceInfo I;
+  I.Name = "Namespace";
+  I.Path = "path/to/A";
+  I.Namespace.emplace_back(EmptySID, "A", InfoType::IT_namespace);
+
+  I.Children.Namespaces.emplace_back(
+      EmptySID, "ChildNamespace", InfoType::IT_namespace,
+      "path::to::A::Namespace::ChildNamespace", "path/to/A/Namespace");
+  I.Children.Records.emplace_back(EmptySID, "ChildStruct", InfoType::IT_record,
+                                  "path::to::A::Namespace::ChildStruct",
+                                  "path/to/A/Namespace");
+  I.Children.Functions.emplace_back();
+  I.Children.Functions.back().Name = "OneFunction";
+  I.Children.Functions.back().Access = AccessSpecifier::AS_none;
+  I.Children.Enums.emplace_back();
+  I.Children.Enums.back().Name = "OneEnum";
+
+  auto G = getJSONGenerator();
+  assert(G);
+  std::string Buffer;
+  llvm::raw_string_ostream Actual(Buffer);
+  auto Err = G->generateDocForInfo(&I, Actual, ClangDocContext());
+  assert(!Err);
+  std::string Expected = R"raw({
+  "Enums": [
+    {
+      "Name": "OneEnum",
+      "Scoped": false,
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "Functions": [
+    {
+      "IsStatic": false,
+      "Name": "OneFunction",
+      "ReturnType": {
+        "IsBuiltIn": false,
+        "IsTemplate": false,
+        "Name": "",
+        "QualName": "",
+        "USR": "0000000000000000000000000000000000000000"
+      },
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "Name": "Namespace",
+  "Namespace": [
+    "A"
+  ],
+  "Namespaces": [
+    {
+      "Name": "ChildNamespace",
+      "Path": "path/to/A/Namespace",
+      "QualName": "path::to::A::Namespace::ChildNamespace",
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "Path": "path/to/A",
+  "Records": [
+    {
+      "Name": "ChildStruct",
+      "Path": "path/to/A/Namespace",
+      "QualName": "path::to::A::Namespace::ChildStruct",
+      "USR": "0000000000000000000000000000000000000000"
+    }
+  ],
+  "USR": "0000000000000000000000000000000000000000"
+})raw";
+  EXPECT_EQ(Expected, Actual.str());
+}
+} // namespace doc
+} // namespace clang
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/MDGeneratorTest.cpp b/external/llvm-project/clang-tools-extra/unittests/clang-doc/MDGeneratorTest.cpp
index 1795ef5a46c3..f4c69cf868a1 100644
--- a/external/llvm-project/clang-tools-extra/unittests/clang-doc/MDGeneratorTest.cpp
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/MDGeneratorTest.cpp
@@ -218,103 +218,105 @@ TEST(MDGeneratorTest, emitCommentMD) {
   I.Access = AccessSpecifier::AS_none;
 
   CommentInfo Top;
-  Top.Kind = "FullComment";
+  Top.Kind = CommentKind::CK_FullComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *BlankLine = Top.Children.back().get();
-  BlankLine->Kind = "ParagraphComment";
+  BlankLine->Kind = CommentKind::CK_ParagraphComment;
   BlankLine->Children.emplace_back(std::make_unique<CommentInfo>());
-  BlankLine->Children.back()->Kind = "TextComment";
+  BlankLine->Children.back()->Kind = CommentKind::CK_TextComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Brief = Top.Children.back().get();
-  Brief->Kind = "ParagraphComment";
+  Brief->Kind = CommentKind::CK_ParagraphComment;
   Brief->Children.emplace_back(std::make_unique<CommentInfo>());
-  Brief->Children.back()->Kind = "TextComment";
+  Brief->Children.back()->Kind = CommentKind::CK_TextComment;
   Brief->Children.back()->Name = "ParagraphComment";
   Brief->Children.back()->Text = " Brief description.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Extended = Top.Children.back().get();
-  Extended->Kind = "ParagraphComment";
+  Extended->Kind = CommentKind::CK_ParagraphComment;
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " Extended description that";
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " continues onto the next line.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *HTML = Top.Children.back().get();
-  HTML->Kind = "ParagraphComment";
+  HTML->Kind = CommentKind::CK_ParagraphComment;
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "TextComment";
+  HTML->Children.back()->Kind = CommentKind::CK_TextComment;
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLStartTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLStartTagComment;
   HTML->Children.back()->Name = "ul";
   HTML->Children.back()->AttrKeys.emplace_back("class");
   HTML->Children.back()->AttrValues.emplace_back("test");
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLStartTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLStartTagComment;
   HTML->Children.back()->Name = "li";
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "TextComment";
+  HTML->Children.back()->Kind = CommentKind::CK_TextComment;
   HTML->Children.back()->Text = " Testing.";
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLEndTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLEndTagComment;
   HTML->Children.back()->Name = "ul";
   HTML->Children.back()->SelfClosing = true;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Verbatim = Top.Children.back().get();
-  Verbatim->Kind = "VerbatimBlockComment";
+  Verbatim->Kind = CommentKind::CK_VerbatimBlockComment;
   Verbatim->Name = "verbatim";
   Verbatim->CloseName = "endverbatim";
   Verbatim->Children.emplace_back(std::make_unique<CommentInfo>());
-  Verbatim->Children.back()->Kind = "VerbatimBlockLineComment";
+  Verbatim->Children.back()->Kind = CommentKind::CK_VerbatimBlockLineComment;
   Verbatim->Children.back()->Text = " The description continues.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *ParamOut = Top.Children.back().get();
-  ParamOut->Kind = "ParamCommandComment";
+  ParamOut->Kind = CommentKind::CK_ParamCommandComment;
   ParamOut->Direction = "[out]";
   ParamOut->ParamName = "I";
   ParamOut->Explicit = true;
   ParamOut->Children.emplace_back(std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Kind = "ParagraphComment";
+  ParamOut->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   ParamOut->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Children.back()->Kind = "TextComment";
+  ParamOut->Children.back()->Children.back()->Kind =
+      CommentKind::CK_TextComment;
   ParamOut->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Children.back()->Kind = "TextComment";
+  ParamOut->Children.back()->Children.back()->Kind =
+      CommentKind::CK_TextComment;
   ParamOut->Children.back()->Children.back()->Text = " is a parameter.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *ParamIn = Top.Children.back().get();
-  ParamIn->Kind = "ParamCommandComment";
+  ParamIn->Kind = CommentKind::CK_ParamCommandComment;
   ParamIn->Direction = "[in]";
   ParamIn->ParamName = "J";
   ParamIn->Children.emplace_back(std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Kind = "ParagraphComment";
+  ParamIn->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   ParamIn->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Children.back()->Kind = "TextComment";
+  ParamIn->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
   ParamIn->Children.back()->Children.back()->Text = " is a parameter.";
   ParamIn->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Children.back()->Kind = "TextComment";
+  ParamIn->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Return = Top.Children.back().get();
-  Return->Kind = "BlockCommandComment";
+  Return->Kind = CommentKind::CK_BlockCommandComment;
   Return->Name = "return";
   Return->Explicit = true;
   Return->Children.emplace_back(std::make_unique<CommentInfo>());
-  Return->Children.back()->Kind = "ParagraphComment";
+  Return->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   Return->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  Return->Children.back()->Children.back()->Kind = "TextComment";
+  Return->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
   Return->Children.back()->Children.back()->Text = "void";
 
   I.Description.emplace_back(std::move(Top));
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/MergeTest.cpp b/external/llvm-project/clang-tools-extra/unittests/clang-doc/MergeTest.cpp
index f99748fdf347..ade744e58ab0 100644
--- a/external/llvm-project/clang-tools-extra/unittests/clang-doc/MergeTest.cpp
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/MergeTest.cpp
@@ -166,11 +166,11 @@ TEST(MergeTest, mergeFunctionInfos) {
 
   One.Description.emplace_back();
   auto *OneFullComment = &One.Description.back();
-  OneFullComment->Kind = "FullComment";
+  OneFullComment->Kind = CommentKind::CK_FullComment;
   auto OneParagraphComment = std::make_unique<CommentInfo>();
-  OneParagraphComment->Kind = "ParagraphComment";
+  OneParagraphComment->Kind = CommentKind::CK_ParagraphComment;
   auto OneTextComment = std::make_unique<CommentInfo>();
-  OneTextComment->Kind = "TextComment";
+  OneTextComment->Kind = CommentKind::CK_TextComment;
   OneTextComment->Text = "This is a text comment.";
   OneParagraphComment->Children.push_back(std::move(OneTextComment));
   OneFullComment->Children.push_back(std::move(OneParagraphComment));
@@ -186,11 +186,11 @@ TEST(MergeTest, mergeFunctionInfos) {
 
   Two.Description.emplace_back();
   auto *TwoFullComment = &Two.Description.back();
-  TwoFullComment->Kind = "FullComment";
+  TwoFullComment->Kind = CommentKind::CK_FullComment;
   auto TwoParagraphComment = std::make_unique<CommentInfo>();
-  TwoParagraphComment->Kind = "ParagraphComment";
+  TwoParagraphComment->Kind = CommentKind::CK_ParagraphComment;
   auto TwoTextComment = std::make_unique<CommentInfo>();
-  TwoTextComment->Kind = "TextComment";
+  TwoTextComment->Kind = CommentKind::CK_TextComment;
   TwoTextComment->Text = "This is a text comment.";
   TwoParagraphComment->Children.push_back(std::move(TwoTextComment));
   TwoFullComment->Children.push_back(std::move(TwoParagraphComment));
@@ -213,11 +213,11 @@ TEST(MergeTest, mergeFunctionInfos) {
 
   Expected->Description.emplace_back();
   auto *ExpectedFullComment = &Expected->Description.back();
-  ExpectedFullComment->Kind = "FullComment";
+  ExpectedFullComment->Kind = CommentKind::CK_FullComment;
   auto ExpectedParagraphComment = std::make_unique<CommentInfo>();
-  ExpectedParagraphComment->Kind = "ParagraphComment";
+  ExpectedParagraphComment->Kind = CommentKind::CK_ParagraphComment;
   auto ExpectedTextComment = std::make_unique<CommentInfo>();
-  ExpectedTextComment->Kind = "TextComment";
+  ExpectedTextComment->Kind = CommentKind::CK_TextComment;
   ExpectedTextComment->Text = "This is a text comment.";
   ExpectedParagraphComment->Children.push_back(std::move(ExpectedTextComment));
   ExpectedFullComment->Children.push_back(std::move(ExpectedParagraphComment));
diff --git a/external/llvm-project/clang-tools-extra/unittests/clang-doc/YAMLGeneratorTest.cpp b/external/llvm-project/clang-tools-extra/unittests/clang-doc/YAMLGeneratorTest.cpp
index 407a503e2d43..a9e7c6413cf1 100644
--- a/external/llvm-project/clang-tools-extra/unittests/clang-doc/YAMLGeneratorTest.cpp
+++ b/external/llvm-project/clang-tools-extra/unittests/clang-doc/YAMLGeneratorTest.cpp
@@ -91,12 +91,12 @@ TEST(YAMLGeneratorTest, emitRecordYAML) {
 
   // Member documentation.
   CommentInfo TopComment;
-  TopComment.Kind = "FullComment";
+  TopComment.Kind = CommentKind::CK_FullComment;
   TopComment.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Brief = TopComment.Children.back().get();
-  Brief->Kind = "ParagraphComment";
+  Brief->Kind = CommentKind::CK_ParagraphComment;
   Brief->Children.emplace_back(std::make_unique<CommentInfo>());
-  Brief->Children.back()->Kind = "TextComment";
+  Brief->Children.back()->Kind = CommentKind::CK_TextComment;
   Brief->Children.back()->Name = "ParagraphComment";
   Brief->Children.back()->Text = "Value of the thing.";
   I.Members.back().Description.push_back(std::move(TopComment));
@@ -150,11 +150,11 @@ IsTypeDef:       true
     Name:            'X'
     Access:          Private
     Description:
-      - Kind:            'FullComment'
+      - Kind:            FullComment
         Children:
-          - Kind:            'ParagraphComment'
+          - Kind:            ParagraphComment
             Children:
-              - Kind:            'TextComment'
+              - Kind:            TextComment
                 Text:            'Value of the thing.'
                 Name:            'ParagraphComment'
 Bases:
@@ -375,103 +375,105 @@ TEST(YAMLGeneratorTest, emitCommentYAML) {
   I.Access = AccessSpecifier::AS_none;
 
   CommentInfo Top;
-  Top.Kind = "FullComment";
+  Top.Kind = CommentKind::CK_FullComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *BlankLine = Top.Children.back().get();
-  BlankLine->Kind = "ParagraphComment";
+  BlankLine->Kind = CommentKind::CK_ParagraphComment;
   BlankLine->Children.emplace_back(std::make_unique<CommentInfo>());
-  BlankLine->Children.back()->Kind = "TextComment";
+  BlankLine->Children.back()->Kind = CommentKind::CK_TextComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Brief = Top.Children.back().get();
-  Brief->Kind = "ParagraphComment";
+  Brief->Kind = CommentKind::CK_ParagraphComment;
   Brief->Children.emplace_back(std::make_unique<CommentInfo>());
-  Brief->Children.back()->Kind = "TextComment";
+  Brief->Children.back()->Kind = CommentKind::CK_TextComment;
   Brief->Children.back()->Name = "ParagraphComment";
   Brief->Children.back()->Text = " Brief description.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Extended = Top.Children.back().get();
-  Extended->Kind = "ParagraphComment";
+  Extended->Kind = CommentKind::CK_ParagraphComment;
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " Extended description that";
   Extended->Children.emplace_back(std::make_unique<CommentInfo>());
-  Extended->Children.back()->Kind = "TextComment";
+  Extended->Children.back()->Kind = CommentKind::CK_TextComment;
   Extended->Children.back()->Text = " continues onto the next line.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *HTML = Top.Children.back().get();
-  HTML->Kind = "ParagraphComment";
+  HTML->Kind = CommentKind::CK_ParagraphComment;
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "TextComment";
+  HTML->Children.back()->Kind = CommentKind::CK_TextComment;
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLStartTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLStartTagComment;
   HTML->Children.back()->Name = "ul";
   HTML->Children.back()->AttrKeys.emplace_back("class");
   HTML->Children.back()->AttrValues.emplace_back("test");
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLStartTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLStartTagComment;
   HTML->Children.back()->Name = "li";
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "TextComment";
+  HTML->Children.back()->Kind = CommentKind::CK_TextComment;
   HTML->Children.back()->Text = " Testing.";
   HTML->Children.emplace_back(std::make_unique<CommentInfo>());
-  HTML->Children.back()->Kind = "HTMLEndTagComment";
+  HTML->Children.back()->Kind = CommentKind::CK_HTMLEndTagComment;
   HTML->Children.back()->Name = "ul";
   HTML->Children.back()->SelfClosing = true;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Verbatim = Top.Children.back().get();
-  Verbatim->Kind = "VerbatimBlockComment";
+  Verbatim->Kind = CommentKind::CK_VerbatimBlockComment;
   Verbatim->Name = "verbatim";
   Verbatim->CloseName = "endverbatim";
   Verbatim->Children.emplace_back(std::make_unique<CommentInfo>());
-  Verbatim->Children.back()->Kind = "VerbatimBlockLineComment";
+  Verbatim->Children.back()->Kind = CommentKind::CK_VerbatimBlockLineComment;
   Verbatim->Children.back()->Text = " The description continues.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *ParamOut = Top.Children.back().get();
-  ParamOut->Kind = "ParamCommandComment";
+  ParamOut->Kind = CommentKind::CK_ParamCommandComment;
   ParamOut->Direction = "[out]";
   ParamOut->ParamName = "I";
   ParamOut->Explicit = true;
   ParamOut->Children.emplace_back(std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Kind = "ParagraphComment";
+  ParamOut->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   ParamOut->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Children.back()->Kind = "TextComment";
+  ParamOut->Children.back()->Children.back()->Kind =
+      CommentKind::CK_TextComment;
   ParamOut->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamOut->Children.back()->Children.back()->Kind = "TextComment";
+  ParamOut->Children.back()->Children.back()->Kind =
+      CommentKind::CK_TextComment;
   ParamOut->Children.back()->Children.back()->Text = " is a parameter.";
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *ParamIn = Top.Children.back().get();
-  ParamIn->Kind = "ParamCommandComment";
+  ParamIn->Kind = CommentKind::CK_ParamCommandComment;
   ParamIn->Direction = "[in]";
   ParamIn->ParamName = "J";
   ParamIn->Children.emplace_back(std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Kind = "ParagraphComment";
+  ParamIn->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   ParamIn->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Children.back()->Kind = "TextComment";
+  ParamIn->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
   ParamIn->Children.back()->Children.back()->Text = " is a parameter.";
   ParamIn->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  ParamIn->Children.back()->Children.back()->Kind = "TextComment";
+  ParamIn->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
 
   Top.Children.emplace_back(std::make_unique<CommentInfo>());
   CommentInfo *Return = Top.Children.back().get();
-  Return->Kind = "BlockCommandComment";
+  Return->Kind = CommentKind::CK_BlockCommandComment;
   Return->Name = "return";
   Return->Explicit = true;
   Return->Children.emplace_back(std::make_unique<CommentInfo>());
-  Return->Children.back()->Kind = "ParagraphComment";
+  Return->Children.back()->Kind = CommentKind::CK_ParagraphComment;
   Return->Children.back()->Children.emplace_back(
       std::make_unique<CommentInfo>());
-  Return->Children.back()->Children.back()->Kind = "TextComment";
+  Return->Children.back()->Children.back()->Kind = CommentKind::CK_TextComment;
   Return->Children.back()->Children.back()->Text = "void";
 
   I.Description.emplace_back(std::move(Top));
@@ -487,70 +489,70 @@ TEST(YAMLGeneratorTest, emitCommentYAML) {
 USR:             '0000000000000000000000000000000000000000'
 Name:            'f'
 Description:
-  - Kind:            'FullComment'
+  - Kind:            FullComment
     Children:
-      - Kind:            'ParagraphComment'
+      - Kind:            ParagraphComment
         Children:
-          - Kind:            'TextComment'
-      - Kind:            'ParagraphComment'
+          - Kind:            TextComment
+      - Kind:            ParagraphComment
         Children:
-          - Kind:            'TextComment'
+          - Kind:            TextComment
             Text:            ' Brief description.'
             Name:            'ParagraphComment'
-      - Kind:            'ParagraphComment'
+      - Kind:            ParagraphComment
         Children:
-          - Kind:            'TextComment'
+          - Kind:            TextComment
             Text:            ' Extended description that'
-          - Kind:            'TextComment'
+          - Kind:            TextComment
             Text:            ' continues onto the next line.'
-      - Kind:            'ParagraphComment'
+      - Kind:            ParagraphComment
         Children:
-          - Kind:            'TextComment'
-          - Kind:            'HTMLStartTagComment'
+          - Kind:            TextComment
+          - Kind:            HTMLStartTagComment
             Name:            'ul'
             AttrKeys:
               - 'class'
             AttrValues:
               - 'test'
-          - Kind:            'HTMLStartTagComment'
+          - Kind:            HTMLStartTagComment
             Name:            'li'
-          - Kind:            'TextComment'
+          - Kind:            TextComment
             Text:            ' Testing.'
-          - Kind:            'HTMLEndTagComment'
+          - Kind:            HTMLEndTagComment
             Name:            'ul'
             SelfClosing:     true
-      - Kind:            'VerbatimBlockComment'
+      - Kind:            VerbatimBlockComment
         Name:            'verbatim'
         CloseName:       'endverbatim'
         Children:
-          - Kind:            'VerbatimBlockLineComment'
+          - Kind:            VerbatimBlockLineComment
             Text:            ' The description continues.'
-      - Kind:            'ParamCommandComment'
+      - Kind:            ParamCommandComment
         Direction:       '[out]'
         ParamName:       'I'
         Explicit:        true
         Children:
-          - Kind:            'ParagraphComment'
+          - Kind:            ParagraphComment
             Children:
-              - Kind:            'TextComment'
-              - Kind:            'TextComment'
+              - Kind:            TextComment
+              - Kind:            TextComment
                 Text:            ' is a parameter.'
-      - Kind:            'ParamCommandComment'
+      - Kind:            ParamCommandComment
         Direction:       '[in]'
         ParamName:       'J'
         Children:
-          - Kind:            'ParagraphComment'
+          - Kind:            ParagraphComment
             Children:
-              - Kind:            'TextComment'
+              - Kind:            TextComment
                 Text:            ' is a parameter.'
-              - Kind:            'TextComment'
-      - Kind:            'BlockCommandComment'
+              - Kind:            TextComment
+      - Kind:            BlockCommandComment
         Name:            'return'
         Explicit:        true
         Children:
-          - Kind:            'ParagraphComment'
+          - Kind:            ParagraphComment
             Children:
-              - Kind:            'TextComment'
+              - Kind:            TextComment
                 Text:            'void'
 DefLocation:
   LineNumber:      10
diff --git a/external/llvm-project/clang/CMakeLists.txt b/external/llvm-project/clang/CMakeLists.txt
index a6122f4081ba..e0da146b6c5a 100644
--- a/external/llvm-project/clang/CMakeLists.txt
+++ b/external/llvm-project/clang/CMakeLists.txt
@@ -370,9 +370,6 @@ configure_file(
 # Add appropriate flags for GCC
 if (LLVM_COMPILER_IS_GCC_COMPATIBLE)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-common -Woverloaded-virtual")
-  if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
-  endif ()
 
   # Enable -pedantic for Clang even if it's not enabled for LLVM.
   if (NOT LLVM_ENABLE_PEDANTIC)
diff --git a/external/llvm-project/clang/bindings/python/clang/cindex.py b/external/llvm-project/clang/bindings/python/clang/cindex.py
index 6f7243cdf80a..824674309d26 100644
--- a/external/llvm-project/clang/bindings/python/clang/cindex.py
+++ b/external/llvm-project/clang/bindings/python/clang/cindex.py
@@ -335,9 +335,9 @@ def is_in_system_header(self):
         return conf.lib.clang_Location_isInSystemHeader(self)  # type: ignore [no-any-return]
 
     def __eq__(self, other):
-        if not isinstance(other, SourceLocation):
-            return False
-        return conf.lib.clang_equalLocations(self, other)  # type: ignore [no-any-return]
+        return isinstance(other, SourceLocation) and conf.lib.clang_equalLocations(
+            self, other
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -395,9 +395,9 @@ def end(self):
         return conf.lib.clang_getRangeEnd(self)  # type: ignore [no-any-return]
 
     def __eq__(self, other):
-        if not isinstance(other, SourceRange):
-            return False
-        return conf.lib.clang_equalRanges(self, other)  # type: ignore [no-any-return]
+        return isinstance(other, SourceRange) and conf.lib.clang_equalRanges(
+            self, other
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -1599,9 +1599,7 @@ def from_location(tu: TranslationUnit, location: SourceLocation) -> Cursor | Non
 
     # This function is not null-guarded because it is used in cursor_null_guard itself
     def __eq__(self, other: object) -> bool:
-        if not isinstance(other, Cursor):
-            return False
-        return conf.lib.clang_equalCursors(self, other)  # type: ignore [no-any-return]
+        return isinstance(other, Cursor) and conf.lib.clang_equalCursors(self, other)
 
     # Not null-guarded for consistency with __eq__
     def __ne__(self, other: object) -> bool:
@@ -2483,6 +2481,7 @@ def spelling(self):
     OBJCSEL = 29
     FLOAT128 = 30
     HALF = 31
+    FLOAT16 = 32
     IBM128 = 40
     COMPLEX = 100
     POINTER = 101
diff --git a/external/llvm-project/clang/bindings/python/tests/cindex/test_cdb.py b/external/llvm-project/clang/bindings/python/tests/cindex/test_cdb.py
index 757a14fbaef2..5abe56f0d65f 100644
--- a/external/llvm-project/clang/bindings/python/tests/cindex/test_cdb.py
+++ b/external/llvm-project/clang/bindings/python/tests/cindex/test_cdb.py
@@ -21,13 +21,16 @@ def test_create_fail(self):
 
         # clang_CompilationDatabase_fromDirectory calls fprintf(stderr, ...)
         # Suppress its output.
-        stderr = os.dup(2)
-        with open(os.devnull, "wb") as null:
-            os.dup2(null.fileno(), 2)
-        with self.assertRaises(CompilationDatabaseError) as cm:
-            CompilationDatabase.fromDirectory(path)
-        os.dup2(stderr, 2)
-        os.close(stderr)
+        try:
+            stderr = os.dup(2)
+            with open(os.devnull, "wb") as null:
+                os.dup2(null.fileno(), 2)
+            with self.assertRaises(CompilationDatabaseError) as cm:
+                CompilationDatabase.fromDirectory(path)
+        # Ensures that stderr is reset even if the above code crashes
+        finally:
+            os.dup2(stderr, 2)
+            os.close(stderr)
 
         e = cm.exception
         self.assertEqual(e.cdb_error, CompilationDatabaseError.ERROR_CANNOTLOADDATABASE)
diff --git a/external/llvm-project/clang/docs/HIPSupport.rst b/external/llvm-project/clang/docs/HIPSupport.rst
index 051a25396994..406e1c8e5a2f 100644
--- a/external/llvm-project/clang/docs/HIPSupport.rst
+++ b/external/llvm-project/clang/docs/HIPSupport.rst
@@ -17,7 +17,7 @@
 HIP Support
 =============
 
-HIP (Heterogeneous-Compute Interface for Portability) `<https://github.com/ROCm-Developer-Tools/HIP>`_ is
+HIP (Heterogeneous-Compute Interface for Portability) `<https://github.com/ROCm/HIP>`_ is
 a C++ Runtime API and Kernel Language. It enables developers to create portable applications for
 offloading computation to different hardware platforms from a single source code.
 
@@ -41,9 +41,9 @@ backend or the out-of-tree LLVM-SPIRV translator. The SPIR-V is then bundled and
 .. note::
    While Clang does not directly provide HIP support for NVIDIA GPUs and CPUs, these platforms are supported via other means:
 
-   - NVIDIA GPUs: HIP support is offered through the HIP project `<https://github.com/ROCm-Developer-Tools/HIP>`_, which provides a header-only library for translating HIP runtime APIs into CUDA runtime APIs. The code is subsequently compiled using NVIDIA's `nvcc`.
+   - NVIDIA GPUs: HIP support is offered through the HIP project `<https://github.com/ROCm/HIP>`_, which provides a header-only library for translating HIP runtime APIs into CUDA runtime APIs. The code is subsequently compiled using NVIDIA's `nvcc`.
 
-   - CPUs: HIP support is available through the HIP-CPU runtime library `<https://github.com/ROCm-Developer-Tools/HIP-CPU>`_. This header-only library enables CPUs to execute unmodified HIP code.
+   - CPUs: HIP support is available through the HIP-CPU runtime library `<https://github.com/ROCm/HIP-CPU>`_. This header-only library enables CPUs to execute unmodified HIP code.
 
 
 Example Usage
@@ -328,7 +328,7 @@ The `parallel_unsequenced_policy <https://en.cppreference.com/w/cpp/algorithm/ex
 maps relatively well to the execution model of AMD GPUs. This, coupled with the
 the availability and maturity of GPU accelerated algorithm libraries that
 implement most / all corresponding algorithms in the standard library
-(e.g. `rocThrust <https://github.com/ROCmSoftwarePlatform/rocThrust>`__), makes
+(e.g. `rocThrust <https://github.com/ROCm/rocm-libraries/tree/develop/projects/rocthrust>`__), makes
 it feasible to provide seamless accelerator offload for supported algorithms,
 when an accelerated version exists. Thus, it becomes possible to easily access
 the computational resources of an AMD accelerator, via a well specified,
@@ -483,7 +483,7 @@ such as GPUs, work.
      allocation / deallocation functions with accelerator-aware equivalents,
      based on a pre-established table; the list of functions that can be
      interposed is available
-     `here <https://github.com/ROCmSoftwarePlatform/roc-stdpar#allocation--deallocation-interposition-status>`__;
+     `here <https://github.com/ROCm/roc-stdpar#allocation--deallocation-interposition-status>`__;
    - This is only run when compiling for the host.
 
 The second pass is optional.
@@ -627,7 +627,7 @@ Linux operating system. Support is synthesised in the following table:
 The minimum Linux kernel version for running in HMM mode is 6.4.
 
 The forwarding header can be obtained from
-`its GitHub repository <https://github.com/ROCmSoftwarePlatform/roc-stdpar>`_.
+`its GitHub repository <https://github.com/ROCm/roc-stdpar>`_.
 It will be packaged with a future `ROCm <https://rocm.docs.amd.com/en/latest/>`_
 release. Because accelerated algorithms are provided via
 `rocThrust <https://rocm.docs.amd.com/projects/rocThrust/en/latest/>`_, a
@@ -636,7 +636,7 @@ transitive dependency on
 can be obtained either by installing their associated components of the
 `ROCm <https://rocm.docs.amd.com/en/latest/>`_ stack, or from their respective
 repositories. The list algorithms that can be offloaded is available
-`here <https://github.com/ROCmSoftwarePlatform/roc-stdpar#algorithm-support-status>`_.
+`here <https://github.com/ROCm/roc-stdpar#algorithm-support-status>`_.
 
 HIP Specific Elements
 ---------------------
diff --git a/external/llvm-project/clang/docs/LanguageExtensions.rst b/external/llvm-project/clang/docs/LanguageExtensions.rst
index 06fec7c6642d..cb188bc02b35 100644
--- a/external/llvm-project/clang/docs/LanguageExtensions.rst
+++ b/external/llvm-project/clang/docs/LanguageExtensions.rst
@@ -1001,6 +1001,7 @@ to ``float``; see below for more information on this emulation.
   * X86 (if SSE2 is available; natively if AVX512-FP16 is also available)
   * RISC-V (natively if Zfh or Zhinx is available)
   * SystemZ (emulated)
+  * LoongArch (emulated)
 
 * ``__bf16`` is supported on the following targets (currently never natively):
 
@@ -1008,6 +1009,7 @@ to ``float``; see below for more information on this emulation.
   * 64-bit ARM (AArch64)
   * RISC-V
   * X86 (when SSE2 is available)
+  * LoongArch
 
 (For X86, SSE2 is available on 64-bit and all recent 32-bit processors.)
 
@@ -2042,6 +2044,17 @@ references can be used instead of numeric references.
       return -1;
   }
 
+ASM Goto versus Branch Target Enforcement
+=========================================
+
+Some target architectures implement branch target enforcement, by requiring
+indirect (register-controlled) branch instructions to jump only to locations
+marked by a special instruction (such as AArch64 ``bti``).
+
+The assembler code inside an ``asm goto`` statement is expected not to use a
+branch instruction of that kind to transfer control to any of its destination
+labels. Therefore, using a label in an ``asm goto`` statement does not cause
+clang to put a ``bti`` or equivalent instruction at the label.
 
 Constexpr strings in GNU ASM statements
 =======================================
@@ -3062,6 +3075,41 @@ following way:
 
 Query for this feature with ``__has_builtin(__builtin_offsetof)``.
 
+``__builtin_get_vtable_pointer``
+--------------------------------
+
+``__builtin_get_vtable_pointer`` loads and authenticates the primary vtable
+pointer from an instance of a polymorphic C++ class. This builtin is needed
+for directly loading the vtable pointer when on platforms using
+:doc:`PointerAuthentication`.
+
+**Syntax**:
+
+.. code-block:: c++
+
+  __builtin_get_vtable_pointer(PolymorphicClass*)
+
+**Example of Use**:
+
+.. code-block:: c++
+
+  struct PolymorphicClass {
+    virtual ~PolymorphicClass();
+  };
+
+  PolymorphicClass anInstance;
+  const void* vtablePointer = __builtin_get_vtable_pointer(&anInstance);
+
+**Description**:
+
+The ``__builtin_get_vtable_pointer`` builtin loads the primary vtable
+pointer from a polymorphic C++ type. If the target platform authenticates
+vtable pointers, this builtin will perform the authentication and produce
+the underlying raw pointer. The object being queried must be polymorphic,
+and so must also be a complete type.
+
+Query for this feature with ``__has_builtin(__builtin_get_vtable_pointer)``.
+
 ``__builtin_call_with_static_chain``
 ------------------------------------
 
@@ -6568,3 +6616,77 @@ qualifications.
 Note, Clang does not allow an ``_Atomic`` function type because
 of explicit constraints against atomically qualified (arrays and) function
 types.
+
+
+Underspecified Object Declarations in C
+=======================================
+
+C23 introduced the notion of `underspecified object declarations <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3006.htm>`_
+(note, the final standards text is different from WG14 N3006 due to changes
+during national body comment review). When an object is declared with the
+``constexpr`` storage class specifier or has a deduced type (with the ``auto``
+specifier), it is said to be "underspecified". Underspecified declarations have
+different requirements than non-underspecified declarations. In particular, the
+identifier being declared cannot be used in its initialization. e.g.,
+
+.. code-block:: c
+
+  auto x = x; // Invalid
+  constexpr int y = y; // Invalid
+
+The standard leaves it implementation-defined whether an underspecified
+declaration may introduce additional identifiers as part of the declaration.
+
+Clang allows additional identifiers to be declared in the following cases:
+
+* A compound literal may introduce a new type. e.g.,
+
+.. code-block:: c
+
+  auto x = (struct S { int x, y; }){ 1, 2 };      // Accepted by Clang
+  constexpr int i = (struct T { int x; }){ 1 }.x; // Accepted by Clang
+
+* The type specifier for a ``constexpr`` declaration may define a new type.
+  e.g.,
+
+.. code-block:: c
+
+  constexpr struct S { int x; } s = { 1 }; // Accepted by Clang
+
+* A function declarator may be declared with parameters, including parameters
+  which introduce a new type. e.g.,
+
+.. code-block:: c
+
+  constexpr int (*fp)(int x) = nullptr;              // Accepted by Clang
+  auto f = (void (*)(struct S { int x; } s))nullptr; // Accepted by Clang
+
+* The initializer may contain a GNU statement expression which defines new
+  types or objects. e.g.,
+
+.. code-block:: c
+
+  constexpr int i = ({                              // Accepted by Clang
+    constexpr int x = 12;
+    constexpr struct S { int x; } s = { x };
+    s.x;
+  });
+  auto x = ({ struct S { int x; } s = { 0 }; s; }); // Accepted by Clang
+
+Clang intentionally does not implement the changed scoping rules from C23
+for underspecified declarations. Doing so would significantly complicate the
+implementation in order to get reasonable diagnostic behavior and also means
+Clang fails to reject some code that should be rejected. e.g.,
+
+.. code-block:: c
+
+  // This should be rejected because 'x' is not in scope within the initializer
+  // of an underspecified declaration. Clang accepts because it treats the scope
+  // of the identifier as beginning immediately after the declarator, same as with
+  // a non-underspecified declaration.
+  constexpr int x = sizeof(x);
+
+  // Clang rejects this code with a diagnostic about using the variable within its
+  // own initializer rather than rejecting the code with an undeclared identifier
+  // diagnostic.
+  auto x = x;
diff --git a/external/llvm-project/clang/docs/LibClang.rst b/external/llvm-project/clang/docs/LibClang.rst
index 08cf452a0df6..6c2b11ac7fc2 100644
--- a/external/llvm-project/clang/docs/LibClang.rst
+++ b/external/llvm-project/clang/docs/LibClang.rst
@@ -4,7 +4,7 @@
 Libclang tutorial
 =================
 The C Interface to Clang provides a relatively small API that exposes facilities for parsing source code into an abstract syntax tree (AST), loading already-parsed ASTs, traversing the AST, associating physical source locations with elements within the AST, and other facilities that support Clang-based development tools.
-This C interface to Clang will never provide all of the information representation stored in Clang's C++ AST, nor should it: the intent is to maintain an API that is relatively stable from one release to the next, providing only the basic functionality needed to support development tools.
+This C interface to Clang will never provide all of the information representation stored in Clang's C++ AST, nor should it: the intent is to maintain an API that is :ref:`relatively stable <Stability>` from one release to the next, providing only the basic functionality needed to support development tools.
 The entire C interface of libclang is available in the file `Index.h`_
 
 Essential types overview
@@ -358,3 +358,49 @@ Complete example code
 
 
 .. _Index.h: https://github.com/llvm/llvm-project/blob/main/clang/include/clang-c/Index.h
+
+.. _Stability:
+
+ABI and API Stability
+---------------------
+
+The C interfaces in libclang are intended to be relatively stable. This allows
+a programmer to use libclang without having to worry as much about Clang
+upgrades breaking existing code. However, the library is not unchanging. For
+example, the library will gain new interfaces over time as needs arise,
+existing APIs may be deprecated for eventual removal, etc. Also, the underlying
+implementation of the facilities by Clang may change behavior as bugs are
+fixed, features get implemented, etc.
+
+The library should be ABI and API stable over time, but ABI- and API-breaking
+changes can happen in the following (non-exhaustive) situations:
+
+* Adding new enumerator to an enumeration (can be ABI-breaking in C++).
+* Removing an explicitly deprecated API after a suitably long deprecation
+  period.
+* Using implementation details, such as names or comments that say something
+  is "private", "reserved", "internal", etc.
+* Bug fixes and changes to Clang's internal implementation happen routinely and
+  will change the behavior of callers.
+* Rarely, bug fixes to libclang itself.
+
+The library has version macros (``CINDEX_VERSION_MAJOR``,
+``CINDEX_VERSION_MINOR``, and ``CINDEX_VERSION``) which can be used to test for
+specific library versions at compile time. The ``CINDEX_VERSION_MAJOR`` macro
+is only incremented if there are major source- or ABI-breaking changes. Except
+for removing an explicitly deprecated API, the changes listed above are not
+considered major source- or ABI-breaking changes. Historically, the value this
+macro expands to has not changed, but may be incremented in the future should
+the need arise. The ``CINDEX_VERSION_MINOR`` macro is incremented as new APIs
+are added. The ``CINDEX_VERSION`` macro expands to a value based on the major
+and minor version macros.
+
+In an effort to allow the library to be modified as new needs arise, the
+following situations are explicitly unsupported:
+
+* Loading different library versions into the same executable and passing
+  objects between the libraries; despite general ABI stability, different
+  versions of the library may use different implementation details that are not
+  compatible across library versions.
+* For the same reason as above, serializing objects from one version of the
+  library and deserializing with a different version is also not supported.
diff --git a/external/llvm-project/clang/docs/OpenMPSupport.rst b/external/llvm-project/clang/docs/OpenMPSupport.rst
index d6507071d469..986aaabe1eed 100644
--- a/external/llvm-project/clang/docs/OpenMPSupport.rst
+++ b/external/llvm-project/clang/docs/OpenMPSupport.rst
@@ -406,7 +406,8 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Extensions to atomic construct                              | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+ 
-| Private reductions                                          | :part:`partial`           | :none:`unclaimed`         | Parse/Sema:https://github.com/llvm/llvm-project/pull/129938              |
+| Private reductions                                          | :good:`mostly`            | :none:`unclaimed`         | Parse/Sema:https://github.com/llvm/llvm-project/pull/129938              |
+|                                                             |                           |                           | Codegen: https://github.com/llvm/llvm-project/pull/134709                |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Self maps                                                   | :part:`partial`           | :none:`unclaimed`         | parsing/sema done: https://github.com/llvm/llvm-project/pull/129888      |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/external/llvm-project/clang/docs/ReleaseNotes.rst b/external/llvm-project/clang/docs/ReleaseNotes.rst
index 9f840dd351b0..12c68b96f253 100644
--- a/external/llvm-project/clang/docs/ReleaseNotes.rst
+++ b/external/llvm-project/clang/docs/ReleaseNotes.rst
@@ -41,6 +41,11 @@ Potentially Breaking Changes
 - For ARM targets when compiling assembly files, the features included in the selected CPU
   or Architecture's FPU are included. If you wish not to use a specific feature,
   the relevant ``+no`` option will need to be amended to the command line option.
+- When compiling with branch target enforcement, ``asm goto``
+  statements will no longer guarantee to place a ``bti`` or
+  ``endbr64`` instruction at the labels named as possible branch
+  destinations, so it is not safe to use a register-controlled branch
+  instruction to branch to one. (In line with gcc.)
 
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
@@ -148,8 +153,12 @@ Resolutions to C++ Defect Reports
 - Bumped the ``__cpp_constexpr`` feature-test macro to ``202002L`` in C++20 mode as indicated in
   `P2493R0 <https://wg21.link/P2493R0>`_.
 
+- Implemented `CWG2517 Useless restriction on use of parameter in `
+  `constraint-expression <https://cplusplus.github.io/CWG/issues/2517.html>`_.
 - Implemented `CWG3005 Function parameters should never be name-independent <https://wg21.link/CWG3005>`_.
 
+- Implemented `CWG2496 ref-qualifiers and virtual overriding <https://wg21.link/CWG2496>`_.
+
 C Language Changes
 ------------------
 
@@ -282,6 +291,8 @@ C23 Feature Support
   directive. Fixes #GH126940.
 - Fixed a crash when a declaration of a ``constexpr`` variable with an invalid
   type. Fixes #GH140887
+- Documented `WG14 N3006 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3006.htm>`_
+  which clarified how Clang is handling underspecified object declarations.
 
 C11 Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -310,6 +321,10 @@ Non-comprehensive list of changes in this release
   different than before.
 - Fixed a crash when a VLA with an invalid size expression was used within a
   ``sizeof`` or ``typeof`` expression. (#GH138444)
+- Deprecation warning is emitted for the deprecated ``__reference_binds_to_temporary`` intrinsic.
+  ``__reference_constructs_from_temporary`` should be used instead. (#GH44056)
+- Added `__builtin_get_vtable_pointer` to directly load the primary vtable pointer from a
+  polymorphic object.
 
 New Compiler Flags
 ------------------
@@ -617,6 +632,20 @@ Improvements to Clang's diagnostics
   trigger a ``'Blue' is deprecated`` warning, which can be turned off with
   ``-Wno-deprecated-declarations-switch-case``.
 
+- Split diagnosis of implicit integer comparison on negation to a new
+  diagnostic group ``-Wimplicit-int-comparison-on-negation``, grouped under
+  ``-Wimplicit-int-conversion``, so user can turn it off independently.
+
+- Improved the FixIts for unused lambda captures.
+
+- Delayed typo correction was removed from the compiler; immediate typo
+  correction behavior remains the same. Delayed typo correction facilities were
+  fragile and unmaintained, and the removal closed the following issues:
+  #GH142457, #GH139913, #GH138850, #GH137867, #GH137860, #GH107840, #GH93308,
+  #GH69470, #GH59391, #GH58172, #GH46215, #GH45915, #GH45891, #GH44490,
+  #GH36703, #GH32903, #GH23312, #GH69874.
+
+
 Improvements to Clang's time-trace
 ----------------------------------
 
@@ -686,6 +715,12 @@ Bug Fixes in This Version
 - Fixed an assertion failure in serialization of constexpr structs containing unions. (#GH140130)
 - Fixed duplicate entries in TableGen that caused the wrong attribute to be selected. (GH#140701)
 - Fixed type mismatch error when 'builtin-elementwise-math' arguments have different qualifiers, this should be well-formed. (#GH141397)
+- Constant evaluation now correctly runs the destructor of a variable declared in
+  the second clause of a C-style ``for`` loop. (#GH139818)
+- Fixed a bug with constexpr evaluation for structs containing unions in case of C++ modules. (#GH143168)
+- Fixed incorrect token location when emitting diagnostics for tokens expanded from macros. (#GH143216)
+- Fixed an infinite recursion when checking constexpr destructors. (#GH141789)
+- Fixed a crash when a malformed using declaration appears in a ``constexpr`` function. (#GH144264)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -708,6 +743,15 @@ Bug Fixes to Compiler Builtins
   are ``__builtin_is_cpp_trivially_relocatable``. It is recommended to use
   ``__builtin_trivially_relocate`` instead.
 
+- ``__reference_binds_to_temporary``, ``__reference_constructs_from_temporary``
+  and ``__reference_converts_from_temporary`` intrinsics no longer consider
+  function references can bind to temporary objects. (#GH114344)
+
+- ``__reference_constructs_from_temporary`` and
+  ``__reference_converts_from_temporary`` intrinsics detect reference binding
+  to prvalue instead of xvalue now if the second operand is an object type, per
+  `LWG3819 <https://cplusplus.github.io/LWG/issue3819>`_.
+
 Bug Fixes to Attribute Support
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  - Fixed crash when a parameter to the ``clang::annotate`` attribute evaluates to ``void``. See #GH119125
@@ -822,12 +866,17 @@ Bug Fixes to C++ Support
 - Clang modules now allow a module and its user to differ on TrivialAutoVarInit*
 - Fixed an access checking bug when initializing non-aggregates in default arguments (#GH62444), (#GH83608)
 - Fixed a pack substitution bug in deducing class template partial specializations. (#GH53609)
+- Fixed a crash when constant evaluating some explicit object member assignment operators. (#GH142835)
+- Fixed an access checking bug when substituting into concepts (#GH115838)
+- Fix a bug where private access specifier of overloaded function not respected. (#GH107629)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 - Fixed type checking when a statement expression ends in an l-value of atomic type. (#GH106576)
 - Fixed uninitialized use check in a lambda within CXXOperatorCallExpr. (#GH129198)
 - Fixed a malformed printout of ``CXXParenListInitExpr`` in certain contexts.
+- Fixed a malformed printout of certain calling convention function attributes. (#GH143160)
+- Fixed dependency calculation for TypedefTypes (#GH89774)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -838,7 +887,7 @@ Miscellaneous Clang Crashes Fixed
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Fixed crash when ``-print-stats`` is enabled in compiling IR files. (#GH131608)
-- Fix code completion crash involving PCH serialzied templates. (#GH139019)
+- Fix code completion crash involving PCH serialized templates. (#GH139019)
 
 OpenACC Specific Changes
 ------------------------
@@ -1045,7 +1094,9 @@ Sanitizers
 ----------
 
 - ``-fsanitize=vptr`` is no longer a part of ``-fsanitize=undefined``.
-- Sanitizer ignorelists now support the syntax ``src:*=sanitize``.
+- Sanitizer ignorelists now support the syntax ``src:*=sanitize``,
+  ``type:*=sanitize``, ``fun:*=sanitize``, ``global:*=sanitize``,
+  and ``mainfile:*=sanitize``.
 
 Python Binding Changes
 ----------------------
@@ -1081,6 +1132,10 @@ OpenMP Support
   open parenthesis. (#GH139665)
 - An error is now emitted when OpenMP ``collapse`` and ``ordered`` clauses have
   an argument larger than what can fit within a 64-bit integer.
+- Added support for private variable reduction.
+- Fixed mapping of arrays of structs containing nested structs with user defined
+  mappers, by using compiler-generated default mappers for the outer structs for
+  such maps.
 
 Improvements
 ^^^^^^^^^^^^
diff --git a/external/llvm-project/clang/docs/SanitizerSpecialCaseList.rst b/external/llvm-project/clang/docs/SanitizerSpecialCaseList.rst
index b82db78a9203..2c50778d0f49 100644
--- a/external/llvm-project/clang/docs/SanitizerSpecialCaseList.rst
+++ b/external/llvm-project/clang/docs/SanitizerSpecialCaseList.rst
@@ -79,7 +79,8 @@ instrumentation for arithmetic operations containing values of type ``int``.
 
 The ``=sanitize`` category is also supported. Any ``=sanitize`` category
 entries enable sanitizer instrumentation, even if it was ignored by entries
-before.
+before. Entries can be ``src``, ``type``, ``global``, ``fun``, and
+``mainfile``.
 
 With this, one may disable instrumentation for some or all types and
 specifically allow instrumentation for one or many types -- including types
@@ -102,23 +103,36 @@ supported sanitizers.
     char c = toobig; // also not instrumented
   }
 
-If multiple entries match the source, than the latest entry takes the
-precedence.
+If multiple entries match the source, then the latest entry takes the
+precedence. Here are a few examples.
 
 .. code-block:: bash
 
   $ cat ignorelist1.txt
-  # test.cc will be instrumented.
+  # test.cc will not be instrumented.
   src:*
   src:*/mylib/*=sanitize
   src:*/mylib/test.cc
 
   $ cat ignorelist2.txt
-  # test.cc will not be instrumented.
+  # test.cc will be instrumented.
   src:*
   src:*/mylib/test.cc
   src:*/mylib/*=sanitize
 
+  $ cat ignorelist3.txt
+  # Type T will not be instrumented.
+  type:*
+  type:T=sanitize
+  type:T
+
+  $ cat ignorelist4.txt
+  # Function `bad_bar`` will be instrumented.
+  # Function `good_bar` will not be instrumented.
+  fun:*
+  fun:*bar
+  fun:bad_bar=sanitize
+
 Format
 ======
 
diff --git a/external/llvm-project/clang/docs/UsersManual.rst b/external/llvm-project/clang/docs/UsersManual.rst
index eb9a812f0c1c..62844f7e6a2f 100644
--- a/external/llvm-project/clang/docs/UsersManual.rst
+++ b/external/llvm-project/clang/docs/UsersManual.rst
@@ -2300,12 +2300,14 @@ are listed below.
 .. option:: -f[no-]unique-source-file-names
 
    When enabled, allows the compiler to assume that each object file
-   passed to the linker has been compiled using a unique source file
-   path. This is useful for reducing link times when doing ThinLTO
-   in combination with whole-program devirtualization or CFI.
+   passed to the linker has a unique identifier. The identifier for
+   an object file is either the source file path or the value of the
+   argument `-funique-source-file-identifier` if specified. This is
+   useful for reducing link times when doing ThinLTO in combination with
+   whole-program devirtualization or CFI.
 
-   The full source path passed to the compiler must be unique. This
-   means that, for example, the following is a usage error:
+   The full source path or identifier passed to the compiler must be
+   unique. This means that, for example, the following is a usage error:
 
    .. code-block:: console
 
@@ -2327,6 +2329,11 @@ are listed below.
    A misuse of this flag may result in a duplicate symbol error at
    link time.
 
+.. option:: -funique-source-file-identifier=IDENTIFIER
+
+   Used with `-funique-source-file-names` to specify a source file
+   identifier.
+
 .. option:: -fforce-emit-vtables
 
    In order to improve devirtualization, forces emitting of vtables even in
@@ -3743,6 +3750,35 @@ Doxygen-style comments and ignores ordinary comments starting with ``//`` and
   ``-fcomment-block-commands=foo -fcomment-block-commands=bar`` does the same
   as above.
 
+.. _ccc-override-options:
+
+CCC_OVERRIDE_OPTIONS
+--------------------
+The environment variable ``CCC_OVERRIDE_OPTIONS`` can be used to edit clang's
+command line arguments. The value of this variable is a space-separated list of
+edits to perform. The edits are applied in the order in which they appear in
+``CCC_OVERRIDE_OPTIONS``. Each edit should be one of the following forms:
+
+- ``#``: Silence information about the changes to the command line arguments.
+
+- ``^FOO``: Add ``FOO`` as a new argument at the beginning of the command line
+  right after the name of the compiler executable.
+
+- ``+FOO``: Add ``FOO`` as a new argument at the end of the command line.
+
+- ``s/XXX/YYY/``: Substitute the regular expression ``XXX`` with ``YYY`` in the
+  command line.
+
+- ``xOPTION``: Removes all instances of the literal argument ``OPTION``.
+
+- ``XOPTION``: Removes all instances of the literal argument ``OPTION``, and the
+  following argument.
+
+- ``Ox``: Removes all flags matching ``O`` or ``O[sz0-9]`` and adds ``Ox`` at
+  the end of the command line.
+
+This environment variable does not affect the options added by the config files.
+
 .. _c:
 
 C Language Features
diff --git a/external/llvm-project/clang/include/clang/AST/ASTContext.h b/external/llvm-project/clang/include/clang/AST/ASTContext.h
index 8d24d393eab0..e01361e2466b 100644
--- a/external/llvm-project/clang/include/clang/AST/ASTContext.h
+++ b/external/llvm-project/clang/include/clang/AST/ASTContext.h
@@ -488,8 +488,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// if possible.
   ///
   /// Not serialized intentionally.
-  llvm::StringMap<const Module *> PrimaryModuleNameMap;
-  llvm::DenseMap<const Module *, const Module *> SameModuleLookupSet;
+  mutable llvm::StringMap<const Module *> PrimaryModuleNameMap;
+  mutable llvm::DenseMap<const Module *, const Module *> SameModuleLookupSet;
 
   static constexpr unsigned ConstantArrayTypesLog2InitSize = 8;
   static constexpr unsigned GeneralTypesLog2InitSize = 9;
@@ -629,10 +629,48 @@ class ASTContext : public RefCountedBase<ASTContext> {
   void setRelocationInfoForCXXRecord(const CXXRecordDecl *,
                                      CXXRecordDeclRelocationInfo);
 
+  /// Examines a given type, and returns whether the type itself
+  /// is address discriminated, or any transitively embedded types
+  /// contain data that is address discriminated. This includes
+  /// implicitly authenticated values like vtable pointers, as well as
+  /// explicitly qualified fields.
+  bool containsAddressDiscriminatedPointerAuth(QualType T) {
+    if (!isPointerAuthenticationAvailable())
+      return false;
+    return findPointerAuthContent(T) != PointerAuthContent::None;
+  }
+
+  /// Examines a given type, and returns whether the type itself
+  /// or any data it transitively contains has a pointer authentication
+  /// schema that is not safely relocatable. e.g. any data or fields
+  /// with address discrimination other than any otherwise similar
+  /// vtable pointers.
+  bool containsNonRelocatablePointerAuth(QualType T) {
+    if (!isPointerAuthenticationAvailable())
+      return false;
+    return findPointerAuthContent(T) ==
+           PointerAuthContent::AddressDiscriminatedData;
+  }
+
 private:
   llvm::DenseMap<const CXXRecordDecl *, CXXRecordDeclRelocationInfo>
       RelocatableClasses;
 
+  // FIXME: store in RecordDeclBitfields in future?
+  enum class PointerAuthContent : uint8_t {
+    None,
+    AddressDiscriminatedVTable,
+    AddressDiscriminatedData
+  };
+
+  // A simple helper function to short circuit pointer auth checks.
+  bool isPointerAuthenticationAvailable() const {
+    return LangOpts.PointerAuthCalls || LangOpts.PointerAuthIntrinsics;
+  }
+  PointerAuthContent findPointerAuthContent(QualType T);
+  llvm::DenseMap<const RecordDecl *, PointerAuthContent>
+      RecordContainsAddressDiscriminatedPointerAuth;
+
   ImportDecl *FirstLocalImport = nullptr;
   ImportDecl *LastLocalImport = nullptr;
 
@@ -1151,7 +1189,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   ///
   /// FIXME: The signature may be confusing since `clang::Module` means to
   /// a module fragment or a module unit but not a C++20 module.
-  bool isInSameModule(const Module *M1, const Module *M2);
+  bool isInSameModule(const Module *M1, const Module *M2) const;
 
   TranslationUnitDecl *getTranslationUnitDecl() const {
     return TUDecl->getMostRecentDecl();
@@ -3668,6 +3706,7 @@ OPT_LIST(V)
   /// authentication policy for the specified record.
   const CXXRecordDecl *
   baseForVTableAuthentication(const CXXRecordDecl *ThisClass);
+
   bool useAbbreviatedThunkName(GlobalDecl VirtualMethodDecl,
                                StringRef MangledName);
 
diff --git a/external/llvm-project/clang/include/clang/AST/ASTNodeTraverser.h b/external/llvm-project/clang/include/clang/AST/ASTNodeTraverser.h
index 01bc12ce33ef..8d02a50e2e8a 100644
--- a/external/llvm-project/clang/include/clang/AST/ASTNodeTraverser.h
+++ b/external/llvm-project/clang/include/clang/AST/ASTNodeTraverser.h
@@ -605,7 +605,7 @@ class ASTNodeTraverser
   }
 
   void VisitFileScopeAsmDecl(const FileScopeAsmDecl *D) {
-    Visit(D->getAsmString());
+    Visit(D->getAsmStringExpr());
   }
 
   void VisitTopLevelStmtDecl(const TopLevelStmtDecl *D) { Visit(D->getStmt()); }
diff --git a/external/llvm-project/clang/include/clang/AST/Decl.h b/external/llvm-project/clang/include/clang/AST/Decl.h
index 69ad18f4c058..05aac15b30cd 100644
--- a/external/llvm-project/clang/include/clang/AST/Decl.h
+++ b/external/llvm-project/clang/include/clang/AST/Decl.h
@@ -4490,18 +4490,18 @@ class RecordDecl : public TagDecl {
 };
 
 class FileScopeAsmDecl : public Decl {
-  StringLiteral *AsmString;
+  Expr *AsmString;
   SourceLocation RParenLoc;
 
-  FileScopeAsmDecl(DeclContext *DC, StringLiteral *asmstring,
-                   SourceLocation StartL, SourceLocation EndL)
-    : Decl(FileScopeAsm, DC, StartL), AsmString(asmstring), RParenLoc(EndL) {}
+  FileScopeAsmDecl(DeclContext *DC, Expr *asmstring, SourceLocation StartL,
+                   SourceLocation EndL)
+      : Decl(FileScopeAsm, DC, StartL), AsmString(asmstring), RParenLoc(EndL) {}
 
   virtual void anchor();
 
 public:
-  static FileScopeAsmDecl *Create(ASTContext &C, DeclContext *DC,
-                                  StringLiteral *Str, SourceLocation AsmLoc,
+  static FileScopeAsmDecl *Create(ASTContext &C, DeclContext *DC, Expr *Str,
+                                  SourceLocation AsmLoc,
                                   SourceLocation RParenLoc);
 
   static FileScopeAsmDecl *CreateDeserialized(ASTContext &C, GlobalDeclID ID);
@@ -4513,9 +4513,11 @@ class FileScopeAsmDecl : public Decl {
     return SourceRange(getAsmLoc(), getRParenLoc());
   }
 
-  const StringLiteral *getAsmString() const { return AsmString; }
-  StringLiteral *getAsmString() { return AsmString; }
-  void setAsmString(StringLiteral *Asm) { AsmString = Asm; }
+  const Expr *getAsmStringExpr() const { return AsmString; }
+  Expr *getAsmStringExpr() { return AsmString; }
+  void setAsmString(Expr *Asm) { AsmString = Asm; }
+
+  std::string getAsmString() const;
 
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == FileScopeAsm; }
diff --git a/external/llvm-project/clang/include/clang/AST/DeclBase.h b/external/llvm-project/clang/include/clang/AST/DeclBase.h
index 375e9e259250..dd67ebc9873f 100644
--- a/external/llvm-project/clang/include/clang/AST/DeclBase.h
+++ b/external/llvm-project/clang/include/clang/AST/DeclBase.h
@@ -646,6 +646,10 @@ class alignas(8) Decl {
     return getModuleOwnershipKind() == ModuleOwnershipKind::ModulePrivate;
   }
 
+  /// Whether this declaration was a local declaration to a C++20
+  /// named module.
+  bool isModuleLocal() const;
+
   /// Whether this declaration was exported in a lexical context.
   /// e.g.:
   ///
diff --git a/external/llvm-project/clang/include/clang/AST/DeclContextInternals.h b/external/llvm-project/clang/include/clang/AST/DeclContextInternals.h
index b17b7627ac90..a0f37886cc01 100644
--- a/external/llvm-project/clang/include/clang/AST/DeclContextInternals.h
+++ b/external/llvm-project/clang/include/clang/AST/DeclContextInternals.h
@@ -177,13 +177,12 @@ class StoredDeclsList {
       if (ND->isFromASTFile())
         return true;
       // FIXME: Can we get rid of this loop completely?
-      for (NamedDecl *D : Decls)
+      return llvm::any_of(Decls, [ND](NamedDecl *D) {
         // Only replace the local declaration if the external declaration has
-        // higher visibilities.
-        if (D->getModuleOwnershipKind() <= ND->getModuleOwnershipKind() &&
-            D->declarationReplaces(ND, /*IsKnownNewer=*/false))
-          return true;
-      return false;
+        // higher visiblities.
+        return D->getModuleOwnershipKind() <= ND->getModuleOwnershipKind() &&
+               D->declarationReplaces(ND, /*IsKnownNewer=*/false);
+      });
     });
 
     // Don't have any pending external decls any more.
diff --git a/external/llvm-project/clang/include/clang/AST/Expr.h b/external/llvm-project/clang/include/clang/AST/Expr.h
index 87bb9df9cef8..9fc23d30b733 100644
--- a/external/llvm-project/clang/include/clang/AST/Expr.h
+++ b/external/llvm-project/clang/include/clang/AST/Expr.h
@@ -240,8 +240,7 @@ class Expr : public ValueStmt {
     return static_cast<bool>(getDependence() & ExprDependence::UnexpandedPack);
   }
 
-  /// Whether this expression contains subexpressions which had errors, e.g. a
-  /// TypoExpr.
+  /// Whether this expression contains subexpressions which had errors.
   bool containsErrors() const {
     return static_cast<bool>(getDependence() & ExprDependence::Error);
   }
@@ -4579,7 +4578,6 @@ class ShuffleVectorExpr : public Expr {
   // indices.  The number of values in this list is always
   // 2+the number of indices in the vector type.
   Stmt **SubExprs;
-  unsigned NumExprs;
 
 public:
   ShuffleVectorExpr(const ASTContext &C, ArrayRef<Expr*> args, QualType Type,
@@ -4605,25 +4603,28 @@ class ShuffleVectorExpr : public Expr {
   /// getNumSubExprs - Return the size of the SubExprs array.  This includes the
   /// constant expression, the actual arguments passed in, and the function
   /// pointers.
-  unsigned getNumSubExprs() const { return NumExprs; }
+  unsigned getNumSubExprs() const { return ShuffleVectorExprBits.NumExprs; }
 
   /// Retrieve the array of expressions.
   Expr **getSubExprs() { return reinterpret_cast<Expr **>(SubExprs); }
 
   /// getExpr - Return the Expr at the specified index.
   Expr *getExpr(unsigned Index) {
-    assert((Index < NumExprs) && "Arg access out of range!");
+    assert((Index < ShuffleVectorExprBits.NumExprs) &&
+           "Arg access out of range!");
     return cast<Expr>(SubExprs[Index]);
   }
   const Expr *getExpr(unsigned Index) const {
-    assert((Index < NumExprs) && "Arg access out of range!");
+    assert((Index < ShuffleVectorExprBits.NumExprs) &&
+           "Arg access out of range!");
     return cast<Expr>(SubExprs[Index]);
   }
 
   void setExprs(const ASTContext &C, ArrayRef<Expr *> Exprs);
 
   llvm::APSInt getShuffleMaskIdx(unsigned N) const {
-    assert((N < NumExprs - 2) && "Shuffle idx out of range!");
+    assert((N < ShuffleVectorExprBits.NumExprs - 2) &&
+           "Shuffle idx out of range!");
     assert(isa<ConstantExpr>(getExpr(N + 2)) &&
            "Index expression must be a ConstantExpr");
     return cast<ConstantExpr>(getExpr(N + 2))->getAPValueResult().getInt();
@@ -4631,10 +4632,12 @@ class ShuffleVectorExpr : public Expr {
 
   // Iterators
   child_range children() {
-    return child_range(&SubExprs[0], &SubExprs[0]+NumExprs);
+    return child_range(&SubExprs[0],
+                       &SubExprs[0] + ShuffleVectorExprBits.NumExprs);
   }
   const_child_range children() const {
-    return const_child_range(&SubExprs[0], &SubExprs[0] + NumExprs);
+    return const_child_range(&SubExprs[0],
+                             &SubExprs[0] + ShuffleVectorExprBits.NumExprs);
   }
 };
 
@@ -4776,13 +4779,13 @@ class ChooseExpr : public Expr {
   enum { COND, LHS, RHS, END_EXPR };
   Stmt* SubExprs[END_EXPR]; // Left/Middle/Right hand sides.
   SourceLocation BuiltinLoc, RParenLoc;
-  bool CondIsTrue;
+
 public:
   ChooseExpr(SourceLocation BLoc, Expr *cond, Expr *lhs, Expr *rhs, QualType t,
              ExprValueKind VK, ExprObjectKind OK, SourceLocation RP,
              bool condIsTrue)
-      : Expr(ChooseExprClass, t, VK, OK), BuiltinLoc(BLoc), RParenLoc(RP),
-        CondIsTrue(condIsTrue) {
+      : Expr(ChooseExprClass, t, VK, OK), BuiltinLoc(BLoc), RParenLoc(RP) {
+    ChooseExprBits.CondIsTrue = condIsTrue;
     SubExprs[COND] = cond;
     SubExprs[LHS] = lhs;
     SubExprs[RHS] = rhs;
@@ -4798,9 +4801,9 @@ class ChooseExpr : public Expr {
   bool isConditionTrue() const {
     assert(!isConditionDependent() &&
            "Dependent condition isn't true or false");
-    return CondIsTrue;
+    return ChooseExprBits.CondIsTrue;
   }
-  void setIsConditionTrue(bool isTrue) { CondIsTrue = isTrue; }
+  void setIsConditionTrue(bool isTrue) { ChooseExprBits.CondIsTrue = isTrue; }
 
   bool isConditionDependent() const {
     return getCond()->isTypeDependent() || getCond()->isValueDependent();
@@ -6961,36 +6964,6 @@ class AtomicExpr : public Expr {
   }
 };
 
-/// TypoExpr - Internal placeholder for expressions where typo correction
-/// still needs to be performed and/or an error diagnostic emitted.
-class TypoExpr : public Expr {
-  // The location for the typo name.
-  SourceLocation TypoLoc;
-
-public:
-  TypoExpr(QualType T, SourceLocation TypoLoc)
-      : Expr(TypoExprClass, T, VK_LValue, OK_Ordinary), TypoLoc(TypoLoc) {
-    assert(T->isDependentType() && "TypoExpr given a non-dependent type");
-    setDependence(ExprDependence::TypeValueInstantiation |
-                  ExprDependence::Error);
-  }
-
-  child_range children() {
-    return child_range(child_iterator(), child_iterator());
-  }
-  const_child_range children() const {
-    return const_child_range(const_child_iterator(), const_child_iterator());
-  }
-
-  SourceLocation getBeginLoc() const LLVM_READONLY { return TypoLoc; }
-  SourceLocation getEndLoc() const LLVM_READONLY { return TypoLoc; }
-
-  static bool classof(const Stmt *T) {
-    return T->getStmtClass() == TypoExprClass;
-  }
-
-};
-
 /// This class represents BOTH the OpenMP Array Section and OpenACC 'subarray',
 /// with a boolean differentiator.
 /// OpenMP 5.0 [2.1.5, Array Sections].
diff --git a/external/llvm-project/clang/include/clang/AST/ExprCXX.h b/external/llvm-project/clang/include/clang/AST/ExprCXX.h
index 6ed049c91548..477373f07f25 100644
--- a/external/llvm-project/clang/include/clang/AST/ExprCXX.h
+++ b/external/llvm-project/clang/include/clang/AST/ExprCXX.h
@@ -1290,7 +1290,7 @@ class CXXDefaultArgExpr final
     CXXDefaultArgExprBits.Loc = Loc;
     CXXDefaultArgExprBits.HasRewrittenInit = RewrittenExpr != nullptr;
     if (RewrittenExpr)
-      *getTrailingObjects<Expr *>() = RewrittenExpr;
+      *getTrailingObjects() = RewrittenExpr;
     setDependence(computeDependence(this));
   }
 
@@ -1323,7 +1323,7 @@ class CXXDefaultArgExpr final
   }
 
   Expr *getRewrittenExpr() {
-    return hasRewrittenInit() ? *getTrailingObjects<Expr *>() : nullptr;
+    return hasRewrittenInit() ? *getTrailingObjects() : nullptr;
   }
 
   const Expr *getRewrittenExpr() const {
@@ -1421,14 +1421,14 @@ class CXXDefaultInitExpr final
   /// any.
   const Expr *getRewrittenExpr() const {
     assert(hasRewrittenInit() && "expected a rewritten init expression");
-    return *getTrailingObjects<Expr *>();
+    return *getTrailingObjects();
   }
 
   /// Retrieve the initializing expression with evaluated immediate calls, if
   /// any.
   Expr *getRewrittenExpr() {
     assert(hasRewrittenInit() && "expected a rewritten init expression");
-    return *getTrailingObjects<Expr *>();
+    return *getTrailingObjects();
   }
 
   const DeclContext *getUsedContext() const { return UsedContext; }
@@ -1982,8 +1982,8 @@ class LambdaExpr final : public Expr,
   /// Construct an empty lambda expression.
   LambdaExpr(EmptyShell Empty, unsigned NumCaptures);
 
-  Stmt **getStoredStmts() { return getTrailingObjects<Stmt *>(); }
-  Stmt *const *getStoredStmts() const { return getTrailingObjects<Stmt *>(); }
+  Stmt **getStoredStmts() { return getTrailingObjects(); }
+  Stmt *const *getStoredStmts() const { return getTrailingObjects(); }
 
   void initBodyIfNeeded() const;
 
@@ -2974,10 +2974,6 @@ class TypeTraitExpr final
 ///   __array_extent(int[10][20], 1) == 20
 /// \endcode
 class ArrayTypeTraitExpr : public Expr {
-  /// The trait. An ArrayTypeTrait enum in MSVC compat unsigned.
-  LLVM_PREFERRED_TYPE(ArrayTypeTrait)
-  unsigned ATT : 2;
-
   /// The value of the type trait. Unspecified if dependent.
   uint64_t Value = 0;
 
@@ -2999,21 +2995,27 @@ class ArrayTypeTraitExpr : public Expr {
   ArrayTypeTraitExpr(SourceLocation loc, ArrayTypeTrait att,
                      TypeSourceInfo *queried, uint64_t value, Expr *dimension,
                      SourceLocation rparen, QualType ty)
-      : Expr(ArrayTypeTraitExprClass, ty, VK_PRValue, OK_Ordinary), ATT(att),
+      : Expr(ArrayTypeTraitExprClass, ty, VK_PRValue, OK_Ordinary),
         Value(value), Dimension(dimension), Loc(loc), RParen(rparen),
         QueriedType(queried) {
     assert(att <= ATT_Last && "invalid enum value!");
-    assert(static_cast<unsigned>(att) == ATT && "ATT overflow!");
+    ArrayTypeTraitExprBits.ATT = att;
+    assert(static_cast<unsigned>(att) == ArrayTypeTraitExprBits.ATT &&
+           "ATT overflow!");
     setDependence(computeDependence(this));
   }
 
   explicit ArrayTypeTraitExpr(EmptyShell Empty)
-      : Expr(ArrayTypeTraitExprClass, Empty), ATT(0) {}
+      : Expr(ArrayTypeTraitExprClass, Empty) {
+    ArrayTypeTraitExprBits.ATT = 0;
+  }
 
   SourceLocation getBeginLoc() const LLVM_READONLY { return Loc; }
   SourceLocation getEndLoc() const LLVM_READONLY { return RParen; }
 
-  ArrayTypeTrait getTrait() const { return static_cast<ArrayTypeTrait>(ATT); }
+  ArrayTypeTrait getTrait() const {
+    return static_cast<ArrayTypeTrait>(ArrayTypeTraitExprBits.ATT);
+  }
 
   QualType getQueriedType() const { return QueriedType->getType(); }
 
@@ -3045,14 +3047,6 @@ class ArrayTypeTraitExpr : public Expr {
 ///   __is_lvalue_expr(1) == false
 /// \endcode
 class ExpressionTraitExpr : public Expr {
-  /// The trait. A ExpressionTrait enum in MSVC compatible unsigned.
-  LLVM_PREFERRED_TYPE(ExpressionTrait)
-  unsigned ET : 31;
-
-  /// The value of the type trait. Unspecified if dependent.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned Value : 1;
-
   /// The location of the type trait keyword.
   SourceLocation Loc;
 
@@ -3068,24 +3062,32 @@ class ExpressionTraitExpr : public Expr {
   ExpressionTraitExpr(SourceLocation loc, ExpressionTrait et, Expr *queried,
                       bool value, SourceLocation rparen, QualType resultType)
       : Expr(ExpressionTraitExprClass, resultType, VK_PRValue, OK_Ordinary),
-        ET(et), Value(value), Loc(loc), RParen(rparen),
-        QueriedExpression(queried) {
+        Loc(loc), RParen(rparen), QueriedExpression(queried) {
+    ExpressionTraitExprBits.ET = et;
+    ExpressionTraitExprBits.Value = value;
+
     assert(et <= ET_Last && "invalid enum value!");
-    assert(static_cast<unsigned>(et) == ET && "ET overflow!");
+    assert(static_cast<unsigned>(et) == ExpressionTraitExprBits.ET &&
+           "ET overflow!");
     setDependence(computeDependence(this));
   }
 
   explicit ExpressionTraitExpr(EmptyShell Empty)
-      : Expr(ExpressionTraitExprClass, Empty), ET(0), Value(false) {}
+      : Expr(ExpressionTraitExprClass, Empty) {
+    ExpressionTraitExprBits.ET = 0;
+    ExpressionTraitExprBits.Value = false;
+  }
 
   SourceLocation getBeginLoc() const LLVM_READONLY { return Loc; }
   SourceLocation getEndLoc() const LLVM_READONLY { return RParen; }
 
-  ExpressionTrait getTrait() const { return static_cast<ExpressionTrait>(ET); }
+  ExpressionTrait getTrait() const {
+    return static_cast<ExpressionTrait>(ExpressionTraitExprBits.ET);
+  }
 
   Expr *getQueriedExpression() const { return QueriedExpression; }
 
-  bool getValue() const { return Value; }
+  bool getValue() const { return ExpressionTraitExprBits.Value; }
 
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == ExpressionTraitExprClass;
@@ -3619,7 +3621,7 @@ class ExprWithCleanups final
                                   ArrayRef<CleanupObject> objects);
 
   ArrayRef<CleanupObject> getObjects() const {
-    return getTrailingObjects<CleanupObject>(getNumObjects());
+    return getTrailingObjects(getNumObjects());
   }
 
   unsigned getNumObjects() const { return ExprWithCleanupsBits.NumObjects; }
@@ -3740,14 +3742,14 @@ class CXXUnresolvedConstructExpr final
   using arg_iterator = Expr **;
   using arg_range = llvm::iterator_range<arg_iterator>;
 
-  arg_iterator arg_begin() { return getTrailingObjects<Expr *>(); }
+  arg_iterator arg_begin() { return getTrailingObjects(); }
   arg_iterator arg_end() { return arg_begin() + getNumArgs(); }
   arg_range arguments() { return arg_range(arg_begin(), arg_end()); }
 
   using const_arg_iterator = const Expr* const *;
   using const_arg_range = llvm::iterator_range<const_arg_iterator>;
 
-  const_arg_iterator arg_begin() const { return getTrailingObjects<Expr *>(); }
+  const_arg_iterator arg_begin() const { return getTrailingObjects(); }
   const_arg_iterator arg_end() const { return arg_begin() + getNumArgs(); }
   const_arg_range arguments() const {
     return const_arg_range(arg_begin(), arg_end());
@@ -3858,10 +3860,6 @@ class CXXDependentScopeMemberExpr final
     return getNumTemplateArgs();
   }
 
-  unsigned numTrailingObjects(OverloadToken<NamedDecl *>) const {
-    return hasFirstQualifierFoundInScope();
-  }
-
   CXXDependentScopeMemberExpr(const ASTContext &Ctx, Expr *Base,
                               QualType BaseType, bool IsArrow,
                               SourceLocation OperatorLoc,
@@ -4417,7 +4415,7 @@ class SizeOfPackExpr final
         Length(Length ? *Length : PartialArgs.size()), Pack(Pack) {
     assert((!Length || PartialArgs.empty()) &&
            "have partial args for non-dependent sizeof... expression");
-    auto *Args = getTrailingObjects<TemplateArgument>();
+    auto *Args = getTrailingObjects();
     llvm::uninitialized_copy(PartialArgs, Args);
     setDependence(Length ? ExprDependence::None
                          : ExprDependence::ValueInstantiation);
@@ -4470,8 +4468,7 @@ class SizeOfPackExpr final
   /// Get
   ArrayRef<TemplateArgument> getPartialArguments() const {
     assert(isPartiallySubstituted());
-    const auto *Args = getTrailingObjects<TemplateArgument>();
-    return llvm::ArrayRef(Args, Args + Length);
+    return getTrailingObjects(Length);
   }
 
   SourceLocation getBeginLoc() const LLVM_READONLY { return OperatorLoc; }
@@ -4506,24 +4503,16 @@ class PackIndexingExpr final
   // The pack being indexed, followed by the index
   Stmt *SubExprs[2];
 
-  // The size of the trailing expressions.
-  unsigned TransformedExpressions : 31;
-
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned FullySubstituted : 1;
-
   PackIndexingExpr(QualType Type, SourceLocation EllipsisLoc,
                    SourceLocation RSquareLoc, Expr *PackIdExpr, Expr *IndexExpr,
                    ArrayRef<Expr *> SubstitutedExprs = {},
                    bool FullySubstituted = false)
       : Expr(PackIndexingExprClass, Type, VK_LValue, OK_Ordinary),
         EllipsisLoc(EllipsisLoc), RSquareLoc(RSquareLoc),
-        SubExprs{PackIdExpr, IndexExpr},
-        TransformedExpressions(SubstitutedExprs.size()),
-        FullySubstituted(FullySubstituted) {
-
-    auto *Exprs = getTrailingObjects<Expr *>();
-    llvm::uninitialized_copy(SubstitutedExprs, Exprs);
+        SubExprs{PackIdExpr, IndexExpr} {
+    PackIndexingExprBits.TransformedExpressions = SubstitutedExprs.size();
+    PackIndexingExprBits.FullySubstituted = FullySubstituted;
+    llvm::uninitialized_copy(SubstitutedExprs, getTrailingObjects());
 
     setDependence(computeDependence(this));
     if (!isInstantiationDependent())
@@ -4534,7 +4523,7 @@ class PackIndexingExpr final
   PackIndexingExpr(EmptyShell Empty) : Expr(PackIndexingExprClass, Empty) {}
 
   unsigned numTrailingObjects(OverloadToken<Expr *>) const {
-    return TransformedExpressions;
+    return PackIndexingExprBits.TransformedExpressions;
   }
 
 public:
@@ -4548,11 +4537,14 @@ class PackIndexingExpr final
                                               unsigned NumTransformedExprs);
 
   // The index expression and all elements of the pack have been substituted.
-  bool isFullySubstituted() const { return FullySubstituted; }
+  bool isFullySubstituted() const {
+    return PackIndexingExprBits.FullySubstituted;
+  }
 
   /// Determine if the expression was expanded to empty.
   bool expandsToEmptyPack() const {
-    return isFullySubstituted() && TransformedExpressions == 0;
+    return isFullySubstituted() &&
+           PackIndexingExprBits.TransformedExpressions == 0;
   }
 
   /// Determine the location of the 'sizeof' keyword.
@@ -4585,12 +4577,12 @@ class PackIndexingExpr final
   Expr *getSelectedExpr() const {
     UnsignedOrNone Index = getSelectedIndex();
     assert(Index && "extracting the indexed expression of a dependant pack");
-    return getTrailingObjects<Expr *>()[*Index];
+    return getTrailingObjects()[*Index];
   }
 
   /// Return the trailing expressions, regardless of the expansion.
   ArrayRef<Expr *> getExpressions() const {
-    return {getTrailingObjects<Expr *>(), TransformedExpressions};
+    return getTrailingObjects(PackIndexingExprBits.TransformedExpressions);
   }
 
   static bool classof(const Stmt *T) {
@@ -4818,7 +4810,7 @@ class FunctionParmPackExpr final
   /// Iterators over the parameters which the parameter pack expanded
   /// into.
   using iterator = ValueDecl *const *;
-  iterator begin() const { return getTrailingObjects<ValueDecl *>(); }
+  iterator begin() const { return getTrailingObjects(); }
   iterator end() const { return begin() + NumParameters; }
 
   /// Get the number of parameters in this parameter pack.
@@ -4988,7 +4980,6 @@ class CXXFoldExpr : public Expr {
   // than the number of expansions.
   UnsignedOrNone NumExpansions = std::nullopt;
   Stmt *SubExprs[SubExpr::Count];
-  BinaryOperatorKind Opcode;
 
 public:
   CXXFoldExpr(QualType T, UnresolvedLookupExpr *Callee,
@@ -5021,7 +5012,7 @@ class CXXFoldExpr : public Expr {
   SourceLocation getLParenLoc() const { return LParenLoc; }
   SourceLocation getRParenLoc() const { return RParenLoc; }
   SourceLocation getEllipsisLoc() const { return EllipsisLoc; }
-  BinaryOperatorKind getOperator() const { return Opcode; }
+  BinaryOperatorKind getOperator() const { return CXXFoldExprBits.Opcode; }
 
   UnsignedOrNone getNumExpansions() const { return NumExpansions; }
 
@@ -5101,7 +5092,7 @@ class CXXParenListInitExpr final
       : Expr(CXXParenListInitExprClass, T, getValueKindForType(T), OK_Ordinary),
         NumExprs(Args.size()), NumUserSpecifiedExprs(NumUserSpecifiedExprs),
         InitLoc(InitLoc), LParenLoc(LParenLoc), RParenLoc(RParenLoc) {
-    std::copy(Args.begin(), Args.end(), getTrailingObjects<Expr *>());
+    llvm::copy(Args, getTrailingObjects());
     assert(NumExprs >= NumUserSpecifiedExprs &&
            "number of user specified inits is greater than the number of "
            "passed inits");
@@ -5126,19 +5117,17 @@ class CXXParenListInitExpr final
   void updateDependence() { setDependence(computeDependence(this)); }
 
   MutableArrayRef<Expr *> getInitExprs() {
-    return getTrailingObjects<Expr *>(NumExprs);
+    return getTrailingObjects(NumExprs);
   }
 
-  ArrayRef<Expr *> getInitExprs() const {
-    return getTrailingObjects<Expr *>(NumExprs);
-  }
+  ArrayRef<Expr *> getInitExprs() const { return getTrailingObjects(NumExprs); }
 
   ArrayRef<Expr *> getUserSpecifiedInitExprs() {
-    return getTrailingObjects<Expr *>(NumUserSpecifiedExprs);
+    return getTrailingObjects(NumUserSpecifiedExprs);
   }
 
   ArrayRef<Expr *> getUserSpecifiedInitExprs() const {
-    return getTrailingObjects<Expr *>(NumUserSpecifiedExprs);
+    return getTrailingObjects(NumUserSpecifiedExprs);
   }
 
   SourceLocation getBeginLoc() const LLVM_READONLY { return LParenLoc; }
@@ -5174,13 +5163,12 @@ class CXXParenListInitExpr final
   }
 
   child_range children() {
-    Stmt **Begin = reinterpret_cast<Stmt **>(getTrailingObjects<Expr *>());
+    Stmt **Begin = reinterpret_cast<Stmt **>(getTrailingObjects());
     return child_range(Begin, Begin + NumExprs);
   }
 
   const_child_range children() const {
-    Stmt *const *Begin =
-        reinterpret_cast<Stmt *const *>(getTrailingObjects<Expr *>());
+    Stmt *const *Begin = reinterpret_cast<Stmt *const *>(getTrailingObjects());
     return const_child_range(Begin, Begin + NumExprs);
   }
 
diff --git a/external/llvm-project/clang/include/clang/AST/ExprConcepts.h b/external/llvm-project/clang/include/clang/AST/ExprConcepts.h
index f988d40cf73c..8df5cdcaa9d7 100644
--- a/external/llvm-project/clang/include/clang/AST/ExprConcepts.h
+++ b/external/llvm-project/clang/include/clang/AST/ExprConcepts.h
@@ -514,10 +514,6 @@ class RequiresExpr final : public Expr,
     return NumLocalParameters;
   }
 
-  unsigned numTrailingObjects(OverloadToken<concepts::Requirement *>) const {
-    return NumRequirements;
-  }
-
   RequiresExpr(ASTContext &C, SourceLocation RequiresKWLoc,
                RequiresExprBodyDecl *Body, SourceLocation LParenLoc,
                ArrayRef<ParmVarDecl *> LocalParameters,
@@ -540,13 +536,13 @@ class RequiresExpr final : public Expr,
          unsigned NumRequirements);
 
   ArrayRef<ParmVarDecl *> getLocalParameters() const {
-    return {getTrailingObjects<ParmVarDecl *>(), NumLocalParameters};
+    return getTrailingObjects<ParmVarDecl *>(NumLocalParameters);
   }
 
   RequiresExprBodyDecl *getBody() const { return Body; }
 
   ArrayRef<concepts::Requirement *> getRequirements() const {
-    return {getTrailingObjects<concepts::Requirement *>(), NumRequirements};
+    return getTrailingObjects<concepts::Requirement *>(NumRequirements);
   }
 
   /// \brief Whether or not the requires clause is satisfied.
diff --git a/external/llvm-project/clang/include/clang/AST/ExprObjC.h b/external/llvm-project/clang/include/clang/AST/ExprObjC.h
index f87fa85569c4..3d9b3c4585e8 100644
--- a/external/llvm-project/clang/include/clang/AST/ExprObjC.h
+++ b/external/llvm-project/clang/include/clang/AST/ExprObjC.h
@@ -217,12 +217,10 @@ class ObjCArrayLiteral final
   SourceRange getSourceRange() const LLVM_READONLY { return Range; }
 
   /// Retrieve elements of array of literals.
-  Expr **getElements() { return getTrailingObjects<Expr *>(); }
+  Expr **getElements() { return getTrailingObjects(); }
 
   /// Retrieve elements of array of literals.
-  const Expr * const *getElements() const {
-    return getTrailingObjects<Expr *>();
-  }
+  const Expr *const *getElements() const { return getTrailingObjects(); }
 
   /// getNumElements - Return number of elements of objective-c array literal.
   unsigned getNumElements() const { return NumElements; }
diff --git a/external/llvm-project/clang/include/clang/AST/GlobalDecl.h b/external/llvm-project/clang/include/clang/AST/GlobalDecl.h
index baf5371d2682..97caff0198cb 100644
--- a/external/llvm-project/clang/include/clang/AST/GlobalDecl.h
+++ b/external/llvm-project/clang/include/clang/AST/GlobalDecl.h
@@ -164,7 +164,7 @@ class GlobalDecl {
   }
 
   static KernelReferenceKind getDefaultKernelReference(const FunctionDecl *D) {
-    return (D->hasAttr<OpenCLKernelAttr>() || D->getLangOpts().CUDAIsDevice)
+    return (D->hasAttr<DeviceKernelAttr>() || D->getLangOpts().CUDAIsDevice)
                ? KernelReferenceKind::Kernel
                : KernelReferenceKind::Stub;
   }
diff --git a/external/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h b/external/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h
index a11157c006f9..5cb2f57edffe 100644
--- a/external/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/external/llvm-project/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -1617,7 +1617,7 @@ DEF_TRAVERSE_DECL(LifetimeExtendedTemporaryDecl, {
 })
 
 DEF_TRAVERSE_DECL(FileScopeAsmDecl,
-                  { TRY_TO(TraverseStmt(D->getAsmString())); })
+                  { TRY_TO(TraverseStmt(D->getAsmStringExpr())); })
 
 DEF_TRAVERSE_DECL(TopLevelStmtDecl, { TRY_TO(TraverseStmt(D->getStmt())); })
 
@@ -2956,7 +2956,6 @@ DEF_TRAVERSE_STMT(CXXRewrittenBinaryOperator, {
   }
 })
 DEF_TRAVERSE_STMT(OpaqueValueExpr, {})
-DEF_TRAVERSE_STMT(TypoExpr, {})
 DEF_TRAVERSE_STMT(RecoveryExpr, {})
 DEF_TRAVERSE_STMT(CUDAKernelCallExpr, {})
 
diff --git a/external/llvm-project/clang/include/clang/AST/Stmt.h b/external/llvm-project/clang/include/clang/AST/Stmt.h
index 3be25c735772..6c4bd6f6946b 100644
--- a/external/llvm-project/clang/include/clang/AST/Stmt.h
+++ b/external/llvm-project/clang/include/clang/AST/Stmt.h
@@ -19,6 +19,7 @@
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/StmtIterator.h"
 #include "clang/Basic/CapturedStmt.h"
+#include "clang/Basic/ExpressionTraits.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Lambda.h"
@@ -736,6 +737,15 @@ class alignas(void *) Stmt {
     unsigned ProducedByFoldExpansion : 1;
   };
 
+  class ShuffleVectorExprBitfields {
+    friend class ShuffleVectorExpr;
+
+    LLVM_PREFERRED_TYPE(ExprBitfields)
+    unsigned : NumExprBits;
+
+    unsigned NumExprs;
+  };
+
   class StmtExprBitfields {
     friend class ASTStmtReader;
     friend class StmtExpr;
@@ -749,6 +759,17 @@ class alignas(void *) Stmt {
     unsigned TemplateDepth;
   };
 
+  class ChooseExprBitfields {
+    friend class ASTStmtReader;
+    friend class ChooseExpr;
+
+    LLVM_PREFERRED_TYPE(ExprBitfields)
+    unsigned : NumExprBits;
+
+    LLVM_PREFERRED_TYPE(bool)
+    bool CondIsTrue : 1;
+  };
+
   //===--- C++ Expression bitfields classes ---===//
 
   class CXXOperatorCallExprBitfields {
@@ -1184,6 +1205,57 @@ class alignas(void *) Stmt {
     SourceLocation RequiresKWLoc;
   };
 
+  class ArrayTypeTraitExprBitfields {
+    friend class ArrayTypeTraitExpr;
+    friend class ASTStmtReader;
+    LLVM_PREFERRED_TYPE(ExprBitfields)
+    unsigned : NumExprBits;
+
+    /// The trait. An ArrayTypeTrait enum in MSVC compat unsigned.
+    LLVM_PREFERRED_TYPE(ArrayTypeTrait)
+    unsigned ATT : 2;
+  };
+
+  class ExpressionTraitExprBitfields {
+    friend class ExpressionTraitExpr;
+    friend class ASTStmtReader;
+    LLVM_PREFERRED_TYPE(ExprBitfields)
+    unsigned : NumExprBits;
+
+    /// The trait. A ExpressionTrait enum in MSVC compatible unsigned.
+    LLVM_PREFERRED_TYPE(ExpressionTrait)
+    unsigned ET : 31;
+
+    /// The value of the type trait. Unspecified if dependent.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned Value : 1;
+  };
+
+  class CXXFoldExprBitfields {
+    friend class CXXFoldExpr;
+    friend class ASTStmtReader;
+    friend class ASTStmtWriter;
+
+    LLVM_PREFERRED_TYPE(ExprBitfields)
+    unsigned : NumExprBits;
+
+    BinaryOperatorKind Opcode;
+  };
+
+  class PackIndexingExprBitfields {
+    friend class PackIndexingExpr;
+    friend class ASTStmtWriter;
+    friend class ASTStmtReader;
+
+    LLVM_PREFERRED_TYPE(ExprBitfields)
+    unsigned : NumExprBits;
+    // The size of the trailing expressions.
+    unsigned TransformedExpressions : 31;
+
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned FullySubstituted : 1;
+  };
+
   //===--- C++ Coroutines bitfields classes ---===//
 
   class CoawaitExprBitfields {
@@ -1279,9 +1351,11 @@ class alignas(void *) Stmt {
     PseudoObjectExprBitfields PseudoObjectExprBits;
     SourceLocExprBitfields SourceLocExprBits;
     ParenExprBitfields ParenExprBits;
+    ShuffleVectorExprBitfields ShuffleVectorExprBits;
 
     // GNU Extensions.
     StmtExprBitfields StmtExprBits;
+    ChooseExprBitfields ChooseExprBits;
 
     // C++ Expressions
     CXXOperatorCallExprBitfields CXXOperatorCallExprBits;
@@ -1308,6 +1382,10 @@ class alignas(void *) Stmt {
     SubstNonTypeTemplateParmExprBitfields SubstNonTypeTemplateParmExprBits;
     LambdaExprBitfields LambdaExprBits;
     RequiresExprBitfields RequiresExprBits;
+    ArrayTypeTraitExprBitfields ArrayTypeTraitExprBits;
+    ExpressionTraitExprBitfields ExpressionTraitExprBits;
+    CXXFoldExprBitfields CXXFoldExprBits;
+    PackIndexingExprBitfields PackIndexingExprBits;
 
     // C++ Coroutines expressions
     CoawaitExprBitfields CoawaitBits;
@@ -1875,10 +1953,6 @@ class CaseStmt final
     return NumMandatoryStmtPtr + caseStmtIsGNURange();
   }
 
-  unsigned numTrailingObjects(OverloadToken<SourceLocation>) const {
-    return caseStmtIsGNURange();
-  }
-
   unsigned lhsOffset() const { return LhsOffset; }
   unsigned rhsOffset() const { return LhsOffset + caseStmtIsGNURange(); }
   unsigned subStmtOffset() const { return rhsOffset() + SubStmtOffsetFromRhs; }
@@ -2150,10 +2224,8 @@ class AttributedStmt final
     std::fill_n(getAttrArrayPtr(), NumAttrs, nullptr);
   }
 
-  const Attr *const *getAttrArrayPtr() const {
-    return getTrailingObjects<const Attr *>();
-  }
-  const Attr **getAttrArrayPtr() { return getTrailingObjects<const Attr *>(); }
+  const Attr *const *getAttrArrayPtr() const { return getTrailingObjects(); }
+  const Attr **getAttrArrayPtr() { return getTrailingObjects(); }
 
 public:
   static AttributedStmt *Create(const ASTContext &C, SourceLocation Loc,
@@ -2465,7 +2537,7 @@ class SwitchStmt final : public Stmt,
   SourceLocation LParenLoc;
   SourceLocation RParenLoc;
 
-  unsigned numTrailingObjects(OverloadToken<Stmt *>) const {
+  unsigned numTrailingStatements() const {
     return NumMandatoryStmtPtr + hasInitStorage() + hasVarStorage();
   }
 
@@ -2501,40 +2573,34 @@ class SwitchStmt final : public Stmt,
   bool hasVarStorage() const { return SwitchStmtBits.HasVar; }
 
   Expr *getCond() {
-    return reinterpret_cast<Expr *>(getTrailingObjects<Stmt *>()[condOffset()]);
+    return reinterpret_cast<Expr *>(getTrailingObjects()[condOffset()]);
   }
 
   const Expr *getCond() const {
-    return reinterpret_cast<Expr *>(getTrailingObjects<Stmt *>()[condOffset()]);
+    return reinterpret_cast<Expr *>(getTrailingObjects()[condOffset()]);
   }
 
   void setCond(Expr *Cond) {
-    getTrailingObjects<Stmt *>()[condOffset()] = reinterpret_cast<Stmt *>(Cond);
+    getTrailingObjects()[condOffset()] = reinterpret_cast<Stmt *>(Cond);
   }
 
-  Stmt *getBody() { return getTrailingObjects<Stmt *>()[bodyOffset()]; }
-  const Stmt *getBody() const {
-    return getTrailingObjects<Stmt *>()[bodyOffset()];
-  }
+  Stmt *getBody() { return getTrailingObjects()[bodyOffset()]; }
+  const Stmt *getBody() const { return getTrailingObjects()[bodyOffset()]; }
 
-  void setBody(Stmt *Body) {
-    getTrailingObjects<Stmt *>()[bodyOffset()] = Body;
-  }
+  void setBody(Stmt *Body) { getTrailingObjects()[bodyOffset()] = Body; }
 
   Stmt *getInit() {
-    return hasInitStorage() ? getTrailingObjects<Stmt *>()[initOffset()]
-                            : nullptr;
+    return hasInitStorage() ? getTrailingObjects()[initOffset()] : nullptr;
   }
 
   const Stmt *getInit() const {
-    return hasInitStorage() ? getTrailingObjects<Stmt *>()[initOffset()]
-                            : nullptr;
+    return hasInitStorage() ? getTrailingObjects()[initOffset()] : nullptr;
   }
 
   void setInit(Stmt *Init) {
     assert(hasInitStorage() &&
            "This switch statement has no storage for an init statement!");
-    getTrailingObjects<Stmt *>()[initOffset()] = Init;
+    getTrailingObjects()[initOffset()] = Init;
   }
 
   /// Retrieve the variable declared in this "switch" statement, if any.
@@ -2558,20 +2624,20 @@ class SwitchStmt final : public Stmt,
   /// If this SwitchStmt has a condition variable, return the faux DeclStmt
   /// associated with the creation of that condition variable.
   DeclStmt *getConditionVariableDeclStmt() {
-    return hasVarStorage() ? static_cast<DeclStmt *>(
-                                 getTrailingObjects<Stmt *>()[varOffset()])
-                           : nullptr;
+    return hasVarStorage()
+               ? static_cast<DeclStmt *>(getTrailingObjects()[varOffset()])
+               : nullptr;
   }
 
   const DeclStmt *getConditionVariableDeclStmt() const {
-    return hasVarStorage() ? static_cast<DeclStmt *>(
-                                 getTrailingObjects<Stmt *>()[varOffset()])
-                           : nullptr;
+    return hasVarStorage()
+               ? static_cast<DeclStmt *>(getTrailingObjects()[varOffset()])
+               : nullptr;
   }
 
   void setConditionVariableDeclStmt(DeclStmt *CondVar) {
     assert(hasVarStorage());
-    getTrailingObjects<Stmt *>()[varOffset()] = CondVar;
+    getTrailingObjects()[varOffset()] = CondVar;
   }
 
   SwitchCase *getSwitchCaseList() { return FirstCase; }
@@ -2615,15 +2681,13 @@ class SwitchStmt final : public Stmt,
 
   // Iterators
   child_range children() {
-    return child_range(getTrailingObjects<Stmt *>(),
-                       getTrailingObjects<Stmt *>() +
-                           numTrailingObjects(OverloadToken<Stmt *>()));
+    return child_range(getTrailingObjects(),
+                       getTrailingObjects() + numTrailingStatements());
   }
 
   const_child_range children() const {
-    return const_child_range(getTrailingObjects<Stmt *>(),
-                             getTrailingObjects<Stmt *>() +
-                                 numTrailingObjects(OverloadToken<Stmt *>()));
+    return const_child_range(getTrailingObjects(),
+                             getTrailingObjects() + numTrailingStatements());
   }
 
   static bool classof(const Stmt *T) {
@@ -2660,7 +2724,7 @@ class WhileStmt final : public Stmt,
   unsigned condOffset() const { return VarOffset + hasVarStorage(); }
   unsigned bodyOffset() const { return condOffset() + BodyOffsetFromCond; }
 
-  unsigned numTrailingObjects(OverloadToken<Stmt *>) const {
+  unsigned numTrailingStatements() const {
     return NumMandatoryStmtPtr + hasVarStorage();
   }
 
@@ -2686,25 +2750,21 @@ class WhileStmt final : public Stmt,
   bool hasVarStorage() const { return WhileStmtBits.HasVar; }
 
   Expr *getCond() {
-    return reinterpret_cast<Expr *>(getTrailingObjects<Stmt *>()[condOffset()]);
+    return reinterpret_cast<Expr *>(getTrailingObjects()[condOffset()]);
   }
 
   const Expr *getCond() const {
-    return reinterpret_cast<Expr *>(getTrailingObjects<Stmt *>()[condOffset()]);
+    return reinterpret_cast<Expr *>(getTrailingObjects()[condOffset()]);
   }
 
   void setCond(Expr *Cond) {
-    getTrailingObjects<Stmt *>()[condOffset()] = reinterpret_cast<Stmt *>(Cond);
+    getTrailingObjects()[condOffset()] = reinterpret_cast<Stmt *>(Cond);
   }
 
-  Stmt *getBody() { return getTrailingObjects<Stmt *>()[bodyOffset()]; }
-  const Stmt *getBody() const {
-    return getTrailingObjects<Stmt *>()[bodyOffset()];
-  }
+  Stmt *getBody() { return getTrailingObjects()[bodyOffset()]; }
+  const Stmt *getBody() const { return getTrailingObjects()[bodyOffset()]; }
 
-  void setBody(Stmt *Body) {
-    getTrailingObjects<Stmt *>()[bodyOffset()] = Body;
-  }
+  void setBody(Stmt *Body) { getTrailingObjects()[bodyOffset()] = Body; }
 
   /// Retrieve the variable declared in this "while" statement, if any.
   ///
@@ -2726,20 +2786,20 @@ class WhileStmt final : public Stmt,
   /// If this WhileStmt has a condition variable, return the faux DeclStmt
   /// associated with the creation of that condition variable.
   DeclStmt *getConditionVariableDeclStmt() {
-    return hasVarStorage() ? static_cast<DeclStmt *>(
-                                 getTrailingObjects<Stmt *>()[varOffset()])
-                           : nullptr;
+    return hasVarStorage()
+               ? static_cast<DeclStmt *>(getTrailingObjects()[varOffset()])
+               : nullptr;
   }
 
   const DeclStmt *getConditionVariableDeclStmt() const {
-    return hasVarStorage() ? static_cast<DeclStmt *>(
-                                 getTrailingObjects<Stmt *>()[varOffset()])
-                           : nullptr;
+    return hasVarStorage()
+               ? static_cast<DeclStmt *>(getTrailingObjects()[varOffset()])
+               : nullptr;
   }
 
   void setConditionVariableDeclStmt(DeclStmt *CondVar) {
     assert(hasVarStorage());
-    getTrailingObjects<Stmt *>()[varOffset()] = CondVar;
+    getTrailingObjects()[varOffset()] = CondVar;
   }
 
   SourceLocation getWhileLoc() const { return WhileStmtBits.WhileLoc; }
@@ -2761,15 +2821,13 @@ class WhileStmt final : public Stmt,
 
   // Iterators
   child_range children() {
-    return child_range(getTrailingObjects<Stmt *>(),
-                       getTrailingObjects<Stmt *>() +
-                           numTrailingObjects(OverloadToken<Stmt *>()));
+    return child_range(getTrailingObjects(),
+                       getTrailingObjects() + numTrailingStatements());
   }
 
   const_child_range children() const {
-    return const_child_range(getTrailingObjects<Stmt *>(),
-                             getTrailingObjects<Stmt *>() +
-                                 numTrailingObjects(OverloadToken<Stmt *>()));
+    return const_child_range(getTrailingObjects(),
+                             getTrailingObjects() + numTrailingStatements());
   }
 };
 
@@ -3080,10 +3138,6 @@ class ReturnStmt final
   /// True if this ReturnStmt has storage for an NRVO candidate.
   bool hasNRVOCandidate() const { return ReturnStmtBits.HasNRVOCandidate; }
 
-  unsigned numTrailingObjects(OverloadToken<const VarDecl *>) const {
-    return hasNRVOCandidate();
-  }
-
   /// Build a return statement.
   ReturnStmt(SourceLocation RL, Expr *E, const VarDecl *NRVOCandidate);
 
@@ -3109,8 +3163,7 @@ class ReturnStmt final
   /// The optimization itself can only be performed if the variable is
   /// also marked as an NRVO object.
   const VarDecl *getNRVOCandidate() const {
-    return hasNRVOCandidate() ? *getTrailingObjects<const VarDecl *>()
-                              : nullptr;
+    return hasNRVOCandidate() ? *getTrailingObjects() : nullptr;
   }
 
   /// Set the variable that might be used for the named return value
@@ -3119,7 +3172,7 @@ class ReturnStmt final
   void setNRVOCandidate(const VarDecl *Var) {
     assert(hasNRVOCandidate() &&
            "This return statement has no storage for an NRVO candidate!");
-    *getTrailingObjects<const VarDecl *>() = Var;
+    *getTrailingObjects() = Var;
   }
 
   SourceLocation getReturnLoc() const { return ReturnStmtBits.RetLoc; }
diff --git a/external/llvm-project/clang/include/clang/AST/StmtCXX.h b/external/llvm-project/clang/include/clang/AST/StmtCXX.h
index 8b4ef24ed376..5d68d3ef64a2 100644
--- a/external/llvm-project/clang/include/clang/AST/StmtCXX.h
+++ b/external/llvm-project/clang/include/clang/AST/StmtCXX.h
@@ -80,8 +80,8 @@ class CXXTryStmt final : public Stmt,
   CXXTryStmt(EmptyShell Empty, unsigned numHandlers)
     : Stmt(CXXTryStmtClass), NumHandlers(numHandlers) { }
 
-  Stmt *const *getStmts() const { return getTrailingObjects<Stmt *>(); }
-  Stmt **getStmts() { return getTrailingObjects<Stmt *>(); }
+  Stmt *const *getStmts() const { return getTrailingObjects(); }
+  Stmt **getStmts() { return getTrailingObjects(); }
 
 public:
   static CXXTryStmt *Create(const ASTContext &C, SourceLocation tryLoc,
@@ -133,11 +133,11 @@ class CXXTryStmt final : public Stmt,
 /// analysis of the constituent components. The original syntactic components
 /// can be extracted using getLoopVariable and getRangeInit.
 class CXXForRangeStmt : public Stmt {
-  SourceLocation ForLoc;
   enum { INIT, RANGE, BEGINSTMT, ENDSTMT, COND, INC, LOOPVAR, BODY, END };
   // SubExprs[RANGE] is an expression or declstmt.
   // SubExprs[COND] and SubExprs[INC] are expressions.
   Stmt *SubExprs[END];
+  SourceLocation ForLoc;
   SourceLocation CoawaitLoc;
   SourceLocation ColonLoc;
   SourceLocation RParenLoc;
@@ -339,9 +339,9 @@ class CoroutineBodyStmt final
   friend class ASTReader;
   friend TrailingObjects;
 
-  Stmt **getStoredStmts() { return getTrailingObjects<Stmt *>(); }
+  Stmt **getStoredStmts() { return getTrailingObjects(); }
 
-  Stmt *const *getStoredStmts() const { return getTrailingObjects<Stmt *>(); }
+  Stmt *const *getStoredStmts() const { return getTrailingObjects(); }
 
 public:
 
diff --git a/external/llvm-project/clang/include/clang/AST/StmtObjC.h b/external/llvm-project/clang/include/clang/AST/StmtObjC.h
index 03bc61f54cdf..af95d65fa2a3 100644
--- a/external/llvm-project/clang/include/clang/AST/StmtObjC.h
+++ b/external/llvm-project/clang/include/clang/AST/StmtObjC.h
@@ -166,7 +166,8 @@ class ObjCAtTryStmt final
     : public Stmt,
       private llvm::TrailingObjects<ObjCAtTryStmt, Stmt *> {
   friend TrailingObjects;
-  size_t numTrailingObjects(OverloadToken<Stmt *>) const {
+
+  size_t numTrailingStatements() const {
     return 1 + NumCatchStmts + HasFinally;
   }
 
@@ -185,8 +186,8 @@ class ObjCAtTryStmt final
   /// The order of the statements in memory follows the order in the source,
   /// with the \@try body first, followed by the \@catch statements (if any)
   /// and, finally, the \@finally (if it exists).
-  Stmt **getStmts() { return getTrailingObjects<Stmt *>(); }
-  Stmt *const *getStmts() const { return getTrailingObjects<Stmt *>(); }
+  Stmt **getStmts() { return getTrailingObjects(); }
+  Stmt *const *getStmts() const { return getTrailingObjects(); }
 
   ObjCAtTryStmt(SourceLocation atTryLoc, Stmt *atTryStmt,
                 Stmt **CatchStmts, unsigned NumCatchStmts,
@@ -262,8 +263,7 @@ class ObjCAtTryStmt final
   }
 
   child_range children() {
-    return child_range(
-        getStmts(), getStmts() + numTrailingObjects(OverloadToken<Stmt *>()));
+    return child_range(getStmts(), getStmts() + numTrailingStatements());
   }
 
   const_child_range children() const {
diff --git a/external/llvm-project/clang/include/clang/AST/TemplateBase.h b/external/llvm-project/clang/include/clang/AST/TemplateBase.h
index 279feb858e66..84a342ec053a 100644
--- a/external/llvm-project/clang/include/clang/AST/TemplateBase.h
+++ b/external/llvm-project/clang/include/clang/AST/TemplateBase.h
@@ -704,7 +704,7 @@ struct ASTTemplateArgumentListInfo final
 
   /// Retrieve the template arguments
   const TemplateArgumentLoc *getTemplateArgs() const {
-    return getTrailingObjects<TemplateArgumentLoc>();
+    return getTrailingObjects();
   }
   unsigned getNumTemplateArgs() const { return NumTemplateArgs; }
 
diff --git a/external/llvm-project/clang/include/clang/AST/Type.h b/external/llvm-project/clang/include/clang/AST/Type.h
index 48f6486e4266..3896cd914bf0 100644
--- a/external/llvm-project/clang/include/clang/AST/Type.h
+++ b/external/llvm-project/clang/include/clang/AST/Type.h
@@ -1985,6 +1985,10 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
     LLVM_PREFERRED_TYPE(bool)
     unsigned HasTrailingReturn : 1;
 
+    /// Whether this function has is a cfi unchecked callee.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned CFIUncheckedCallee : 1;
+
     /// Extra information which affects how the function is called, like
     /// regparm and the calling convention.
     LLVM_PREFERRED_TYPE(CallingConv)
@@ -2566,6 +2570,8 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
   bool isSignableIntegerType(const ASTContext &Ctx) const;
   bool isAnyPointerType() const;   // Any C pointer or ObjC object pointer
   bool isCountAttributedType() const;
+  bool isCFIUncheckedCalleeFunctionType() const;
+  bool hasPointeeToToCFIUncheckedCalleeFunctionType() const;
   bool isBlockPointerType() const;
   bool isVoidPointerType() const;
   bool isReferenceType() const;
@@ -4713,6 +4719,10 @@ class FunctionType : public Type {
   /// type.
   bool getNoReturnAttr() const { return getExtInfo().getNoReturn(); }
 
+  /// Determine whether this is a function prototype that includes the
+  /// cfi_unchecked_callee attribute.
+  bool getCFIUncheckedCalleeAttr() const;
+
   bool getCmseNSCallAttr() const { return getExtInfo().getCmseNSCall(); }
   CallingConv getCallConv() const { return getExtInfo().getCC(); }
   ExtInfo getExtInfo() const { return ExtInfo(FunctionTypeBits.ExtInfo); }
@@ -5249,8 +5259,12 @@ class FunctionProtoType final
   /// the various bits of extra information about a function prototype.
   struct ExtProtoInfo {
     FunctionType::ExtInfo ExtInfo;
+    LLVM_PREFERRED_TYPE(bool)
     unsigned Variadic : 1;
+    LLVM_PREFERRED_TYPE(bool)
     unsigned HasTrailingReturn : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned CFIUncheckedCallee : 1;
     unsigned AArch64SMEAttributes : 9;
     Qualifiers TypeQuals;
     RefQualifierKind RefQualifier = RQ_None;
@@ -5260,12 +5274,12 @@ class FunctionProtoType final
     FunctionEffectsRef FunctionEffects;
 
     ExtProtoInfo()
-        : Variadic(false), HasTrailingReturn(false),
+        : Variadic(false), HasTrailingReturn(false), CFIUncheckedCallee(false),
           AArch64SMEAttributes(SME_NormalFunction) {}
 
     ExtProtoInfo(CallingConv CC)
         : ExtInfo(CC), Variadic(false), HasTrailingReturn(false),
-          AArch64SMEAttributes(SME_NormalFunction) {}
+          CFIUncheckedCallee(false), AArch64SMEAttributes(SME_NormalFunction) {}
 
     ExtProtoInfo withExceptionSpec(const ExceptionSpecInfo &ESI) {
       ExtProtoInfo Result(*this);
@@ -5273,6 +5287,12 @@ class FunctionProtoType final
       return Result;
     }
 
+    ExtProtoInfo withCFIUncheckedCallee(bool CFIUncheckedCallee) {
+      ExtProtoInfo Result(*this);
+      Result.CFIUncheckedCallee = CFIUncheckedCallee;
+      return Result;
+    }
+
     bool requiresFunctionProtoTypeExtraBitfields() const {
       return ExceptionSpec.Type == EST_Dynamic ||
              requiresFunctionProtoTypeArmAttributes() ||
@@ -5332,10 +5352,6 @@ class FunctionProtoType final
     return getNumFunctionEffects();
   }
 
-  unsigned numTrailingObjects(OverloadToken<EffectConditionExpr>) const {
-    return getNumFunctionEffectConditions();
-  }
-
   /// Determine whether there are any argument types that
   /// contain an unexpanded parameter pack.
   static bool containsAnyUnexpandedParameterPack(const QualType *ArgArray,
@@ -5432,6 +5448,7 @@ class FunctionProtoType final
     EPI.Variadic = isVariadic();
     EPI.EllipsisLoc = getEllipsisLoc();
     EPI.HasTrailingReturn = hasTrailingReturn();
+    EPI.CFIUncheckedCallee = hasCFIUncheckedCallee();
     EPI.ExceptionSpec = getExceptionSpecInfo();
     EPI.TypeQuals = getMethodQuals();
     EPI.RefQualifier = getRefQualifier();
@@ -5557,6 +5574,10 @@ class FunctionProtoType final
   /// Whether this function prototype has a trailing return type.
   bool hasTrailingReturn() const { return FunctionTypeBits.HasTrailingReturn; }
 
+  bool hasCFIUncheckedCallee() const {
+    return FunctionTypeBits.CFIUncheckedCallee;
+  }
+
   Qualifiers getMethodQuals() const {
     if (hasExtQualifiers())
       return *getTrailingObjects<Qualifiers>();
@@ -5661,8 +5682,8 @@ class FunctionProtoType final
     if (hasExtraBitfields()) {
       const auto *Bitfields = getTrailingObjects<FunctionTypeExtraBitfields>();
       if (Bitfields->NumFunctionEffects > 0)
-        return {getTrailingObjects<FunctionEffect>(),
-                Bitfields->NumFunctionEffects};
+        return getTrailingObjects<FunctionEffect>(
+            Bitfields->NumFunctionEffects);
     }
     return {};
   }
@@ -5681,8 +5702,8 @@ class FunctionProtoType final
     if (hasExtraBitfields()) {
       const auto *Bitfields = getTrailingObjects<FunctionTypeExtraBitfields>();
       if (Bitfields->EffectsHaveConditions)
-        return {getTrailingObjects<EffectConditionExpr>(),
-                Bitfields->NumFunctionEffects};
+        return getTrailingObjects<EffectConditionExpr>(
+            Bitfields->NumFunctionEffects);
     }
     return {};
   }
@@ -5696,8 +5717,7 @@ class FunctionProtoType final
                                     ? Bitfields->NumFunctionEffects
                                     : 0;
         return FunctionEffectsRef(
-            {getTrailingObjects<FunctionEffect>(),
-             Bitfields->NumFunctionEffects},
+            getTrailingObjects<FunctionEffect>(Bitfields->NumFunctionEffects),
             {NumConds ? getTrailingObjects<EffectConditionExpr>() : nullptr,
              NumConds});
       }
@@ -5796,8 +5816,8 @@ class TypedefType final : public Type,
   friend class ASTContext; // ASTContext creates these.
   friend TrailingObjects;
 
-  TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType underlying,
-              QualType can);
+  TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType UnderlyingType,
+              bool HasTypeDifferentFromDecl);
 
 public:
   TypedefNameDecl *getDecl() const { return Decl; }
@@ -6038,8 +6058,6 @@ class PackIndexingType final
 
   static TypeDependence computeDependence(QualType Pattern, Expr *IndexExpr,
                                           ArrayRef<QualType> Expansions = {});
-
-  unsigned numTrailingObjects(OverloadToken<QualType>) const { return Size; }
 };
 
 /// A unary type transform, which is a type constructed from another.
@@ -6466,8 +6484,7 @@ class HLSLInlineSpirvType final
     for (size_t I = 0; I < NumOperands; I++) {
       // Since Operands are stored as a trailing object, they have not been
       // initialized yet. Call the constructor manually.
-      auto *Operand =
-          new (&getTrailingObjects<SpirvOperand>()[I]) SpirvOperand();
+      auto *Operand = new (&getTrailingObjects()[I]) SpirvOperand();
       *Operand = Operands[I];
     }
   }
@@ -6477,7 +6494,7 @@ class HLSLInlineSpirvType final
   uint32_t getSize() const { return Size; }
   uint32_t getAlignment() const { return Alignment; }
   ArrayRef<SpirvOperand> getOperands() const {
-    return {getTrailingObjects<SpirvOperand>(), NumOperands};
+    return getTrailingObjects<SpirvOperand>(NumOperands);
   }
 
   bool isSugared() const { return false; }
@@ -6577,7 +6594,7 @@ class SubstTemplateTypeParmType final
   /// parameter.
   QualType getReplacementType() const {
     return SubstTemplateTypeParmTypeBits.HasNonCanonicalUnderlyingType
-               ? *getTrailingObjects<QualType>()
+               ? *getTrailingObjects()
                : getCanonicalTypeInternal();
   }
 
@@ -7139,7 +7156,7 @@ class ElaboratedType final
     ElaboratedTypeBits.HasOwnedTagDecl = false;
     if (OwnedTagDecl) {
       ElaboratedTypeBits.HasOwnedTagDecl = true;
-      *getTrailingObjects<TagDecl *>() = OwnedTagDecl;
+      *getTrailingObjects() = OwnedTagDecl;
     }
   }
 
@@ -7159,8 +7176,7 @@ class ElaboratedType final
   /// Return the (re)declaration of this type owned by this occurrence of this
   /// type, or nullptr if there is none.
   TagDecl *getOwnedTagDecl() const {
-    return ElaboratedTypeBits.HasOwnedTagDecl ? *getTrailingObjects<TagDecl *>()
-                                              : nullptr;
+    return ElaboratedTypeBits.HasOwnedTagDecl ? *getTrailingObjects() : nullptr;
   }
 
   void Profile(llvm::FoldingSetNodeID &ID) {
@@ -8391,6 +8407,27 @@ inline bool Type::isObjectPointerType() const {
     return false;
 }
 
+inline bool Type::isCFIUncheckedCalleeFunctionType() const {
+  if (const auto *Fn = getAs<FunctionProtoType>())
+    return Fn->hasCFIUncheckedCallee();
+  return false;
+}
+
+inline bool Type::hasPointeeToToCFIUncheckedCalleeFunctionType() const {
+  QualType Pointee;
+  if (const auto *PT = getAs<PointerType>())
+    Pointee = PT->getPointeeType();
+  else if (const auto *RT = getAs<ReferenceType>())
+    Pointee = RT->getPointeeType();
+  else if (const auto *MPT = getAs<MemberPointerType>())
+    Pointee = MPT->getPointeeType();
+  else if (const auto *DT = getAs<DecayedType>())
+    Pointee = DT->getPointeeType();
+  else
+    return false;
+  return Pointee->isCFIUncheckedCalleeFunctionType();
+}
+
 inline bool Type::isFunctionPointerType() const {
   if (const auto *T = getAs<PointerType>())
     return T->getPointeeType()->isFunctionType();
diff --git a/external/llvm-project/clang/include/clang/AST/TypeProperties.td b/external/llvm-project/clang/include/clang/AST/TypeProperties.td
index f43d3ebd229d..6e44bce893e7 100644
--- a/external/llvm-project/clang/include/clang/AST/TypeProperties.td
+++ b/external/llvm-project/clang/include/clang/AST/TypeProperties.td
@@ -317,6 +317,9 @@ let Class = FunctionProtoType in {
   def : Property<"trailingReturn", Bool> {
     let Read = [{ node->hasTrailingReturn() }];
   }
+  def : Property<"cfiUncheckedCallee", Bool> {
+    let Read = [{ node->hasCFIUncheckedCallee() }];
+  }
   def : Property<"methodQualifiers", Qualifiers> {
     let Read = [{ node->getMethodQuals() }];
   }
@@ -353,6 +356,7 @@ let Class = FunctionProtoType in {
     epi.ExtInfo = extInfo;
     epi.Variadic = variadic;
     epi.HasTrailingReturn = trailingReturn;
+    epi.CFIUncheckedCallee = cfiUncheckedCallee;
     epi.TypeQuals = methodQualifiers;
     epi.RefQualifier = refQualifier;
     epi.ExceptionSpec = exceptionSpecifier;
diff --git a/external/llvm-project/clang/include/clang/Basic/AArch64ACLETypes.def b/external/llvm-project/clang/include/clang/Basic/AArch64ACLETypes.def
index 89e8e31d9ce3..bbe0c85f9ffb 100644
--- a/external/llvm-project/clang/include/clang/Basic/AArch64ACLETypes.def
+++ b/external/llvm-project/clang/include/clang/Basic/AArch64ACLETypes.def
@@ -6,7 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines various SVE builtin types.  The macros are:
+//  This file defines various Neon and SVE builtin types.  The macros are:
+//
+//    NEON_VECTOR_TYPE:
+//    - (Name, BaseType, ElBits, NumEls, VectorKind)
+//    Unlike the SVE types, the Neon vector types are not builtin types and
+//    mapped to the equivalent __attribute__(neon_vector_type(...)) vector type.
+//    They are not builtin types.
 //
 //    SVE_TYPE:
 //    - (Name, MangledName, Id, SingletonId)
@@ -57,6 +63,10 @@
 //  - IsBF true for vector of brain float elements.
 //===----------------------------------------------------------------------===//
 
+#ifndef NEON_VECTOR_TYPE
+#define NEON_VECTOR_TYPE(Name, BaseType, ElBits, NumEls, VectorKind)
+#endif
+
 #ifndef SVE_TYPE
 #define SVE_TYPE(Name, Id, SingletonId)
 #endif
@@ -111,7 +121,38 @@
   SVE_TYPE(Name, Id, SingletonId)
 #endif
 
-//===- Vector point types -----------------------------------------------===//
+//===- Neon Vector point types --------------------------------------------===//
+
+NEON_VECTOR_TYPE(__Int8x8_t, SignedCharTy, 8, 8, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Int16x4_t, ShortTy, 16, 4, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Int32x2_t, IntTy, 32, 2, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Uint8x8_t, UnsignedCharTy, 8, 8, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Uint16x4_t, UnsignedShortTy, 16, 4, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Uint32x2_t, UnsignedIntTy, 32, 2, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Float16x4_t, HalfTy, 16, 4, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Float32x2_t, FloatTy, 32, 2, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Poly8x8_t, UnsignedCharTy, 8, 8, VectorKind::NeonPoly)
+NEON_VECTOR_TYPE(__Poly16x4_t, UnsignedShortTy, 16, 4, VectorKind::NeonPoly)
+NEON_VECTOR_TYPE(__Bfloat16x4_t, BFloat16Ty, 16, 4, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Int8x16_t, SignedCharTy, 8, 16, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Int16x8_t, ShortTy, 16, 8, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Int32x4_t, IntTy, 32, 4, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Int64x2_t, LongTy, 64, 2, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Uint8x16_t, UnsignedCharTy, 8, 16, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Uint16x8_t, UnsignedShortTy, 16, 8, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Uint32x4_t, UnsignedIntTy, 32, 4, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Uint64x2_t, UnsignedLongTy, 64, 2, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Float16x8_t, HalfTy, 16, 8, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Float32x4_t, FloatTy, 32, 4, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Float64x2_t, DoubleTy, 64, 2, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Poly8x16_t, UnsignedCharTy, 8, 16, VectorKind::NeonPoly)
+NEON_VECTOR_TYPE(__Poly16x8_t, UnsignedShortTy, 16, 8, VectorKind::NeonPoly)
+NEON_VECTOR_TYPE(__Poly64x2_t, UnsignedLongTy, 64, 2, VectorKind::NeonPoly)
+NEON_VECTOR_TYPE(__Bfloat16x8_t, BFloat16Ty, 16, 8, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Mfloat8x8_t, MFloat8Ty, 8, 8, VectorKind::Neon)
+NEON_VECTOR_TYPE(__Mfloat8x16_t, MFloat8Ty, 8, 16, VectorKind::Neon)
+
+//===- SVE Vector point types ---------------------------------------------===//
 
 SVE_VECTOR_TYPE_INT(__SVInt8_t,  __SVInt8_t,  SveInt8,  SveInt8Ty, 16,  8, 1, true)
 SVE_VECTOR_TYPE_INT(__SVInt16_t, __SVInt16_t, SveInt16, SveInt16Ty, 8, 16, 1, true)
@@ -205,6 +246,7 @@ SVE_OPAQUE_TYPE(__SVCount_t, __SVCount_t, SveCount, SveCountTy)
 
 SVE_SCALAR_TYPE(__mfp8, __mfp8, MFloat8, MFloat8Ty, 8)
 
+#undef NEON_VECTOR_TYPE
 #undef SVE_VECTOR_TYPE
 #undef SVE_VECTOR_TYPE_MFLOAT
 #undef SVE_VECTOR_TYPE_BFLOAT
diff --git a/external/llvm-project/clang/include/clang/Basic/AddressSpaces.h b/external/llvm-project/clang/include/clang/Basic/AddressSpaces.h
index 519d959bb636..48e4a1c61fe0 100644
--- a/external/llvm-project/clang/include/clang/Basic/AddressSpaces.h
+++ b/external/llvm-project/clang/include/clang/Basic/AddressSpaces.h
@@ -61,6 +61,7 @@ enum class LangAS : unsigned {
   hlsl_constant,
   hlsl_private,
   hlsl_device,
+  hlsl_input,
 
   // Wasm specific address spaces.
   wasm_funcref,
diff --git a/external/llvm-project/clang/include/clang/Basic/Attr.td b/external/llvm-project/clang/include/clang/Basic/Attr.td
index 216084344c00..f113cd2ba2fb 100644
--- a/external/llvm-project/clang/include/clang/Basic/Attr.td
+++ b/external/llvm-project/clang/include/clang/Basic/Attr.td
@@ -140,6 +140,12 @@ def SharedVar : SubsetSubject<Var,
                               [{S->hasGlobalStorage() && !S->getTLSKind()}],
                               "global variables">;
 
+def HLSLInputBuiltin
+    : SubsetSubject<Var, [{S->hasGlobalStorage() &&
+                           S->getStorageClass() == StorageClass::SC_Static &&
+                           S->getType().isConstQualified()}],
+                    "static const globals">;
+
 def GlobalVar : SubsetSubject<Var,
                              [{S->hasGlobalStorage()}], "global variables">;
 
@@ -190,8 +196,10 @@ def FunctionPointer : SubsetSubject<DeclBase,
                                     "functions pointers">;
 
 def OpenCLKernelFunction
-    : SubsetSubject<Function, [{S->hasAttr<OpenCLKernelAttr>()}],
-                    "kernel functions">;
+    : SubsetSubject<Function, [{S->getASTContext().getLangOpts().OpenCL &&
+                               DeviceKernelAttr::isOpenCLSpelling(
+			                 S->getAttr<DeviceKernelAttr>()}],
+                               "kernel functions">;
 
 // HasFunctionProto is a more strict version of FunctionLike, so it should
 // never be specified in a Subjects list along with FunctionLike (due to the
@@ -1103,6 +1111,8 @@ static llvm::StringRef canonicalizePlatformName(llvm::StringRef Platform) {
     return llvm::StringSwitch<llvm::StringRef>(Platform)
              .Case("iOS", "ios")
              .Case("macOS", "macos")
+             .Case("macOSX", "macos")
+             .Case("macosx", "macos")
              .Case("tvOS", "tvos")
              .Case("watchOS", "watchos")
              .Case("iOSApplicationExtension", "ios_app_extension")
@@ -1167,6 +1177,26 @@ static llvm::Triple::EnvironmentType getEnvironmentType(llvm::StringRef Environm
              .Case("library", llvm::Triple::Library)
              .Default(llvm::Triple::UnknownEnvironment);
 }
+
+static llvm::Triple::OSType getOSType(llvm::StringRef Platform) {
+    using OSType = llvm::Triple::OSType;
+    return llvm::StringSwitch<OSType>(Platform)
+             .Case("ios", OSType::IOS)
+             .Case("macos", OSType::MacOSX)
+             .Case("maccatalyst", OSType::IOS)
+             .Case("tvos", OSType::TvOS)
+             .Case("watchos", OSType::WatchOS)
+             .Case("bridgeos", OSType::BridgeOS)
+             .Case("ios_app_extension", OSType::IOS)
+             .Case("maccatalyst_app_extension", OSType::IOS)
+             .Case("macos_app_extension", OSType::MacOSX)
+             .Case("tvos_app_extension", OSType::TvOS)
+             .Case("watchos_app_extension", OSType::WatchOS)
+             .Case("xros", OSType::XROS)
+             .Case("xros_app_extension", OSType::XROS)
+             .Default(OSType::UnknownOS);
+}
+
 }];
   let HasCustomParsing = 1;
   let InheritEvenIfAlreadyPresent = 1;
@@ -1509,12 +1539,6 @@ def CUDAGridConstant : InheritableAttr {
   let Documentation = [CUDAGridConstantAttrDocs];
 }
 
-def NVPTXKernel : InheritableAttr, TargetSpecificAttr<TargetNVPTX> {
-  let Spellings = [Clang<"nvptx_kernel">];
-  let Subjects = SubjectList<[Function]>;
-  let Documentation = [Undocumented];
-}
-
 def HIPManaged : InheritableAttr {
   let Spellings = [GNU<"managed">, Declspec<"__managed__">];
   let Subjects = SubjectList<[Var]>;
@@ -1549,11 +1573,52 @@ def CUDAShared : InheritableAttr {
 }
 def : MutualExclusions<[CUDAConstant, CUDAShared, HIPManaged]>;
 
-def SYCLKernel : InheritableAttr {
-  let Spellings = [Clang<"sycl_kernel">];
-  let Subjects = SubjectList<[FunctionTmpl]>;
-  let LangOpts = [SYCLDevice];
-  let Documentation = [SYCLKernelDocs];
+def DeviceKernel : DeclOrTypeAttr {
+  let Spellings = [Clang<"device_kernel">, Clang<"sycl_kernel">,
+                   Clang<"nvptx_kernel">, Clang<"amdgpu_kernel">,
+                   CustomKeyword<"__kernel">, CustomKeyword<"kernel">];
+  let Documentation = [DeviceKernelDocs];
+  let AdditionalMembers =
+  [{
+    static inline bool isAMDGPUSpelling(const AttributeCommonInfo& A) {
+      return A.getAttributeSpellingListIndex() == GNU_amdgpu_kernel ||
+             A.getAttributeSpellingListIndex() == CXX11_clang_amdgpu_kernel ||
+             A.getAttributeSpellingListIndex() == C23_clang_amdgpu_kernel;
+    }
+    static inline bool isAMDGPUSpelling(const AttributeCommonInfo* A) {
+      if(!A) return false;
+      return isAMDGPUSpelling(*A);
+    }
+    static inline bool isNVPTXSpelling(const AttributeCommonInfo& A) {
+      return A.getAttributeSpellingListIndex() == GNU_nvptx_kernel ||
+             A.getAttributeSpellingListIndex() == CXX11_clang_nvptx_kernel ||
+             A.getAttributeSpellingListIndex() == C23_clang_nvptx_kernel;
+    }
+    static inline bool isNVPTXSpelling(const AttributeCommonInfo* A) {
+      if(!A) return false;
+      return isNVPTXSpelling(*A);
+    }
+    static inline bool isSYCLSpelling(const AttributeCommonInfo& A) {
+      return A.getAttributeSpellingListIndex() == GNU_sycl_kernel ||
+             A.getAttributeSpellingListIndex() == CXX11_clang_sycl_kernel ||
+             A.getAttributeSpellingListIndex() == C23_clang_sycl_kernel;
+    }
+    static inline bool isSYCLSpelling(const AttributeCommonInfo* A) {
+      if(!A) return false;
+      return isSYCLSpelling(*A);
+    }
+    static inline bool isOpenCLSpelling(const AttributeCommonInfo& A) {
+    // Tablegen trips underscores from spellings to build the spelling
+    // list, but here we have the same spelling with unscores and without,
+    // so handle that case manually.
+      return A.getAttributeSpellingListIndex() == Keyword_kernel ||
+      (A.getAttrName() && A.getAttrName()->getName() == "kernel");
+    }
+    static inline bool isOpenCLSpelling(const AttributeCommonInfo* A) {
+      if (!A) return false;
+      return isOpenCLSpelling(*A);
+    }
+}];
 }
 
 def SYCLKernelEntryPoint : InheritableAttr {
@@ -1619,15 +1684,6 @@ def Allocating : TypeAttr {
   let Documentation = [AllocatingDocs];
 }
 
-// Similar to CUDA, OpenCL attributes do not receive a [[]] spelling because
-// the specification does not expose them with one currently.
-def OpenCLKernel : InheritableAttr {
-  let Spellings = [CustomKeyword<"__kernel">, CustomKeyword<"kernel">];
-  let Subjects = SubjectList<[Function], ErrorDiag>;
-  let Documentation = [Undocumented];
-  let SimpleHandler = 1;
-}
-
 def OpenCLUnrollHint : StmtAttr {
   let Spellings = [GNU<"opencl_unroll_hint">];
   let Subjects = SubjectList<[ForStmt, CXXForRangeStmt, WhileStmt, DoStmt],
@@ -2364,11 +2420,6 @@ def AMDGPUMaxNumWorkGroups : InheritableAttr {
   let Subjects = SubjectList<[Function], ErrorDiag, "kernel functions">;
 }
 
-def AMDGPUKernelCall : DeclOrTypeAttr {
-  let Spellings = [Clang<"amdgpu_kernel">];
-  let Documentation = [Undocumented];
-}
-
 def BPFPreserveAccessIndex : InheritableAttr,
                              TargetSpecificAttr<TargetBPF>  {
   let Spellings = [Clang<"preserve_access_index">];
@@ -3111,6 +3162,11 @@ def NoDeref : TypeAttr {
   let Documentation = [NoDerefDocs];
 }
 
+def CFIUncheckedCallee : TypeAttr {
+  let Spellings = [Clang<"cfi_unchecked_callee">];
+  let Documentation = [CFIUncheckedCalleeDocs];
+}
+
 def ReqdWorkGroupSize : InheritableAttr {
   // Does not have a [[]] spelling because it is an OpenCL-related attribute.
   let Spellings = [GNU<"reqd_work_group_size">];
@@ -4574,6 +4630,7 @@ def OMPDeclareVariant : InheritableAttr {
     OMPTraitInfoArgument<"TraitInfos">,
     VariadicExprArgument<"AdjustArgsNothing">,
     VariadicExprArgument<"AdjustArgsNeedDevicePtr">,
+    VariadicExprArgument<"AdjustArgsNeedDeviceAddr">,
     VariadicOMPInteropInfoArgument<"AppendArgs">,
   ];
   let AdditionalMembers = [{
@@ -4845,6 +4902,13 @@ def HLSLResourceBinding: InheritableAttr {
   }];
 }
 
+def HLSLSV_Position : HLSLAnnotationAttr {
+  let Spellings = [HLSLAnnotation<"sv_position">];
+  let Subjects = SubjectList<[ParmVar, Field]>;
+  let LangOpts = [HLSL];
+  let Documentation = [HLSLSV_PositionDocs];
+}
+
 def HLSLPackOffset: HLSLAnnotationAttr {
   let Spellings = [HLSLAnnotation<"packoffset">];
   let LangOpts = [HLSL];
@@ -4951,6 +5015,14 @@ def HLSLWaveSize: InheritableAttr {
   let Documentation = [WaveSizeDocs];
 }
 
+def HLSLVkExtBuiltinInput : InheritableAttr {
+  let Spellings = [CXX11<"vk", "ext_builtin_input">];
+  let Args = [UnsignedArgument<"BuiltIn">];
+  let Subjects = SubjectList<[HLSLInputBuiltin], ErrorDiag>;
+  let LangOpts = [HLSL];
+  let Documentation = [HLSLVkExtBuiltinInputDocs];
+}
+
 def RandomizeLayout : InheritableAttr {
   let Spellings = [GCC<"randomize_layout">];
   let Subjects = SubjectList<[Record]>;
diff --git a/external/llvm-project/clang/include/clang/Basic/AttrDocs.td b/external/llvm-project/clang/include/clang/Basic/AttrDocs.td
index 65d66dd398ad..047f51ffa59e 100644
--- a/external/llvm-project/clang/include/clang/Basic/AttrDocs.td
+++ b/external/llvm-project/clang/include/clang/Basic/AttrDocs.td
@@ -396,9 +396,13 @@ any option of a multiversioned function is undefined.
   }];
 }
 
-def SYCLKernelDocs : Documentation {
+def DeviceKernelDocs : Documentation {
   let Category = DocCatFunction;
+  let Heading = "device_kernel, sycl_kernel, nvptx_kernel, amdgpu_kernel, "
+                "kernel, __kernel";
   let Content = [{
+These attributes specify that the function represents a kernel for device offloading.
+The specific semantics depend on the offloading language, target, and attribute spelling.
 The ``sycl_kernel`` attribute specifies that a function template will be used
 to outline device code and to generate an OpenCL kernel.
 Here is a code example of the SYCL program, which demonstrates the compiler's
@@ -4392,8 +4396,6 @@ annotated parameter.
       addToSet(str, s); // Not detected.
     }
   }
-
-.. _`lifetimebound`: https://clang.llvm.org/docs/AttributeReference.html#lifetimebound
   }];
 }
 
@@ -4431,7 +4433,14 @@ destroy the object before returning. The lifetime of the copy of the parameter
 in the caller ends without a destructor call when the call begins.
 
 If a type is trivial for the purpose of calls, it is assumed to be trivially
-relocatable for the purpose of ``__is_trivially_relocatable``.
+relocatable for the purpose of ``__is_trivially_relocatable`` and
+``__builtin_is_cpp_trivially_relocatable``.
+When a type marked with ``[[trivial_abi]]`` is used as a function argument,
+the compiler may omit the call to the copy constructor.
+Thus, side effects of the copy constructor are potentially not performed.
+For example, objects that contain pointers to themselves or otherwise depend
+on their address (or the address or their subobjects) should not be declared
+``[[trivial_abi]]``.
 
 Attribute ``trivial_abi`` has no effect in the following cases:
 
@@ -6869,6 +6878,54 @@ for references or Objective-C object pointers.
 }];
 }
 
+def CFIUncheckedCalleeDocs : Documentation {
+  let Category = DocCatType;
+  let Content = [{
+``cfi_unchecked_callee`` is a function type attribute which prevents the compiler from instrumenting
+`Control Flow Integrity <https://clang.llvm.org/docs/ControlFlowIntegrity.html>`_ checks on indirect
+function calls. Specifically, the attribute has the following semantics:
+
+1. Indirect calls to a function type with this attribute will not be instrumented with CFI. That is,
+   the indirect call will not be checked. Note that this only changes the behavior for indirect calls
+   on pointers to function types having this attribute. It does not prevent all indirect function calls
+   for a given type from being checked.
+2. All direct references to a function whose type has this attribute will always reference the
+   function definition rather than an entry in the CFI jump table.
+3. When a pointer to a function with this attribute is implicitly cast to a pointer to a function
+   without this attribute, the compiler will give a warning saying this attribute is discarded. This
+   warning can be silenced with an explicit cast. Note an explicit cast just disables the warning, so
+   direct references to a function with a ``cfi_unchecked_callee`` attribute will still reference the
+   function definition rather than the CFI jump table.
+
+.. code-block:: c
+
+  #define CFI_UNCHECKED_CALLEE __attribute__((cfi_unchecked_callee))
+
+  void no_cfi() CFI_UNCHECKED_CALLEE {}
+
+  void (*with_cfi)() = no_cfi;  // warning: implicit conversion discards `cfi_unchecked_callee` attribute.
+                                // `with_cfi` also points to the actual definition of `no_cfi` rather than
+                                // its jump table entry.
+
+  void invoke(void (CFI_UNCHECKED_CALLEE *func)()) {
+    func();  // CFI will not instrument this indirect call.
+
+    void (*func2)() = func;  // warning: implicit conversion discards `cfi_unchecked_callee` attribute.
+
+    func2();  // CFI will instrument this indirect call. Users should be careful however because if this
+              // references a function with type `cfi_unchecked_callee`, then the CFI check may incorrectly
+              // fail because the reference will be to the function definition rather than the CFI jump
+              // table entry.
+  }
+
+This attribute can only be applied on functions or member functions. This attribute can be a good
+alternative to ``no_sanitize("cfi")`` if you only want to disable innstrumentation for specific indirect
+calls rather than applying ``no_sanitize("cfi")`` on the whole function containing indirect call. Note
+that ``cfi_unchecked_attribute`` is a type attribute doesn't disable CFI instrumentation on a function
+body.
+}];
+}
+
 def ReinitializesDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
@@ -8472,6 +8529,20 @@ The full documentation is available here: https://docs.microsoft.com/en-us/windo
   }];
 }
 
+def HLSLSV_PositionDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The ``SV_Position`` semantic, when applied to an input parameter in a pixel
+shader, contains the location of the pixel center (x, y) in screen space.
+This semantic can be applied to the parameter, or a field in a struct used
+as an input parameter.
+This attribute is supported as an input in pixel, hull, domain and mesh shaders.
+This attribute is supported as an output in vertex, geometry and domain shaders.
+
+The full documentation is available here: https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-semantics
+  }];
+}
+
 def HLSLGroupSharedAddressSpaceDocs : Documentation {
   let Category = DocCatVariable;
   let Content = [{
@@ -8508,6 +8579,29 @@ and copied back to the argument after the callee returns.
   }];
 }
 
+def HLSLVkExtBuiltinInputDocs : Documentation {
+  let Category = DocCatVariable;
+  let Content = [{
+Vulkan shaders have `Input` builtins. Those variables are externally
+initialized by the driver/pipeline, but each copy is private to the current
+lane.
+
+Those builtins can be declared using the `[[vk::ext_builtin_input]]` attribute
+like follows:
+
+.. code-block:: c++
+
+  [[vk::ext_builtin_input(/* WorkgroupId */ 26)]]
+  static const uint3 groupid;
+
+This variable will be lowered into a module-level variable, with the `Input`
+storage class, and the `BuiltIn 26` decoration.
+
+The full documentation for this inline SPIR-V attribute can be found here:
+https://github.com/microsoft/hlsl-specs/blob/main/proposals/0011-inline-spirv.md
+  }];
+}
+
 def AnnotateTypeDocs : Documentation {
   let Category = DocCatType;
   let Heading = "annotate_type";
@@ -8967,14 +9061,14 @@ def CoroLifetimeBoundDoc : Documentation {
   let Category = DocCatDecl;
   let Content = [{
 The ``[[clang::coro_lifetimebound]]`` is a class attribute which can be applied
-to a coroutine return type (`CRT`_) (i.e.
+to a coroutine return type (`coro_return_type, coro_wrapper`_) (i.e.
 it should also be annotated with ``[[clang::coro_return_type]]``).
 
 All parameters of a function are considered to be lifetime bound if the function returns a
 coroutine return type (CRT) annotated with ``[[clang::coro_lifetimebound]]``.
 This lifetime bound analysis can be disabled for a coroutine wrapper or a coroutine by annotating the function
 with ``[[clang::coro_disable_lifetimebound]]`` function attribute .
-See `documentation`_ of ``[[clang::lifetimebound]]`` for details about lifetime bound analysis.
+See documentation of `lifetimebound`_ for details about lifetime bound analysis.
 
 
 Reference parameters of a coroutine are susceptible to capturing references to temporaries or local variables.
@@ -9034,9 +9128,6 @@ but do not pass them to the underlying coroutine or pass them by value.
   void use() {
     auto task = coro_wrapper(1); // use of temporary is fine as the argument is not lifetime bound.
   }
-
-.. _`documentation`: https://clang.llvm.org/docs/AttributeReference.html#lifetimebound
-.. _`CRT`: https://clang.llvm.org/docs/AttributeReference.html#coro-return-type
 }];
 }
 
diff --git a/external/llvm-project/clang/include/clang/Basic/AttributeCommonInfo.h b/external/llvm-project/clang/include/clang/Basic/AttributeCommonInfo.h
index b4b8345b4ed4..21a7a88a3fb9 100644
--- a/external/llvm-project/clang/include/clang/Basic/AttributeCommonInfo.h
+++ b/external/llvm-project/clang/include/clang/Basic/AttributeCommonInfo.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_BASIC_ATTRIBUTECOMMONINFO_H
 #define LLVM_CLANG_BASIC_ATTRIBUTECOMMONINFO_H
 
+#include "clang/Basic/AttributeScopeInfo.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TokenKinds.h"
 
@@ -61,6 +62,7 @@ class AttributeCommonInfo {
     /// implicitly.
     AS_Implicit
   };
+
   enum Kind {
 #define PARSED_ATTR(NAME) AT_##NAME,
 #include "clang/Basic/AttrParsedAttrList.inc"
@@ -69,7 +71,7 @@ class AttributeCommonInfo {
     IgnoredAttribute,
     UnknownAttribute,
   };
-  enum class Scope { NONE, CLANG, GNU, MSVC, OMP, HLSL, GSL, RISCV };
+  enum class Scope { NONE, CLANG, GNU, MSVC, OMP, HLSL, VK, GSL, RISCV };
   enum class AttrArgsInfo {
     None,
     Optional,
@@ -78,9 +80,9 @@ class AttributeCommonInfo {
 
 private:
   const IdentifierInfo *AttrName = nullptr;
-  const IdentifierInfo *ScopeName = nullptr;
+  AttributeScopeInfo AttrScope;
   SourceRange AttrRange;
-  const SourceLocation ScopeLoc;
+
   // Corresponds to the Kind enum.
   LLVM_PREFERRED_TYPE(Kind)
   unsigned AttrKind : 16;
@@ -146,11 +148,10 @@ class AttributeCommonInfo {
   };
 
   AttributeCommonInfo(const IdentifierInfo *AttrName,
-                      const IdentifierInfo *ScopeName, SourceRange AttrRange,
-                      SourceLocation ScopeLoc, Kind AttrKind, Form FormUsed)
-      : AttrName(AttrName), ScopeName(ScopeName), AttrRange(AttrRange),
-        ScopeLoc(ScopeLoc), AttrKind(AttrKind),
-        SyntaxUsed(FormUsed.getSyntax()),
+                      AttributeScopeInfo AttrScope, SourceRange AttrRange,
+                      Kind AttrKind, Form FormUsed)
+      : AttrName(AttrName), AttrScope(AttrScope), AttrRange(AttrRange),
+        AttrKind(AttrKind), SyntaxUsed(FormUsed.getSyntax()),
         SpellingIndex(FormUsed.getSpellingIndex()),
         IsAlignas(FormUsed.isAlignas()),
         IsRegularKeywordAttribute(FormUsed.isRegularKeywordAttribute()) {
@@ -158,21 +159,20 @@ class AttributeCommonInfo {
            "Invalid syntax!");
   }
 
-  AttributeCommonInfo(const IdentifierInfo *AttrName,
-                      const IdentifierInfo *ScopeName, SourceRange AttrRange,
-                      SourceLocation ScopeLoc, Form FormUsed)
+  AttributeCommonInfo(const IdentifierInfo *AttrName, AttributeScopeInfo Scope,
+                      SourceRange AttrRange, Form FormUsed)
       : AttributeCommonInfo(
-            AttrName, ScopeName, AttrRange, ScopeLoc,
-            getParsedKind(AttrName, ScopeName, FormUsed.getSyntax()),
+            AttrName, Scope, AttrRange,
+            getParsedKind(AttrName, Scope.getName(), FormUsed.getSyntax()),
             FormUsed) {}
 
   AttributeCommonInfo(const IdentifierInfo *AttrName, SourceRange AttrRange,
                       Form FormUsed)
-      : AttributeCommonInfo(AttrName, nullptr, AttrRange, SourceLocation(),
+      : AttributeCommonInfo(AttrName, AttributeScopeInfo(), AttrRange,
                             FormUsed) {}
 
   AttributeCommonInfo(SourceRange AttrRange, Kind K, Form FormUsed)
-      : AttributeCommonInfo(nullptr, nullptr, AttrRange, SourceLocation(), K,
+      : AttributeCommonInfo(nullptr, AttributeScopeInfo(), AttrRange, K,
                             FormUsed) {}
 
   AttributeCommonInfo(AttributeCommonInfo &&) = default;
@@ -190,17 +190,27 @@ class AttributeCommonInfo {
   SourceRange getRange() const { return AttrRange; }
   void setRange(SourceRange R) { AttrRange = R; }
 
-  bool hasScope() const { return ScopeName; }
-  const IdentifierInfo *getScopeName() const { return ScopeName; }
-  SourceLocation getScopeLoc() const { return ScopeLoc; }
+  bool hasScope() const { return AttrScope.isValid(); }
+  bool isExplicitScope() const { return AttrScope.isExplicit(); }
+
+  const IdentifierInfo *getScopeName() const { return AttrScope.getName(); }
+  SourceLocation getScopeLoc() const { return AttrScope.getNameLoc(); }
 
   /// Gets the normalized full name, which consists of both scope and name and
   /// with surrounding underscores removed as appropriate (e.g.
   /// __gnu__::__attr__ will be normalized to gnu::attr).
   std::string getNormalizedFullName() const;
-  std::optional<std::string>
-  getCorrectedFullName(const TargetInfo &Target,
-                       const LangOptions &LangOpts) const;
+  std::string getNormalizedFullName(StringRef ScopeName,
+                                    StringRef AttrName) const;
+  StringRef getNormalizedScopeName() const;
+  StringRef getNormalizedAttrName(StringRef ScopeName) const;
+
+  std::optional<StringRef> tryGetCorrectedScopeName(StringRef ScopeName) const;
+  std::optional<StringRef>
+  tryGetCorrectedAttrName(StringRef ScopeName, StringRef AttrName,
+                          const TargetInfo &Target,
+                          const LangOptions &LangOpts) const;
+
   SourceRange getNormalizedRange() const;
 
   bool isDeclspecAttribute() const { return SyntaxUsed == AS_Declspec; }
diff --git a/external/llvm-project/clang/include/clang/Basic/AttributeScopeInfo.h b/external/llvm-project/clang/include/clang/Basic/AttributeScopeInfo.h
new file mode 100644
index 000000000000..cca4df7c11b0
--- /dev/null
+++ b/external/llvm-project/clang/include/clang/Basic/AttributeScopeInfo.h
@@ -0,0 +1,48 @@
+//==- AttributeScopeInfo.h - Base info about an Attribute Scope --*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the AttributeScopeInfo type, which represents information
+// about the scope of an attribute.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_BASIC_ATTRIBUTESCOPEINFO_H
+#define LLVM_CLANG_BASIC_ATTRIBUTESCOPEINFO_H
+
+#include "clang/Basic/SourceLocation.h"
+
+namespace clang {
+
+class IdentifierInfo;
+
+class AttributeScopeInfo {
+public:
+  AttributeScopeInfo() = default;
+
+  AttributeScopeInfo(const IdentifierInfo *Name, SourceLocation NameLoc)
+      : Name(Name), NameLoc(NameLoc) {}
+
+  AttributeScopeInfo(const IdentifierInfo *Name, SourceLocation NameLoc,
+                     SourceLocation CommonScopeLoc)
+      : Name(Name), NameLoc(NameLoc), CommonScopeLoc(CommonScopeLoc) {}
+
+  const IdentifierInfo *getName() const { return Name; }
+  SourceLocation getNameLoc() const { return NameLoc; }
+
+  bool isValid() const { return Name != nullptr; }
+  bool isExplicit() const { return CommonScopeLoc.isInvalid(); }
+
+private:
+  const IdentifierInfo *Name = nullptr;
+  SourceLocation NameLoc;
+  SourceLocation CommonScopeLoc;
+};
+
+} // namespace clang
+
+#endif // LLVM_CLANG_BASIC_ATTRIBUTESCOPEINFO_H
diff --git a/external/llvm-project/clang/include/clang/Basic/BuiltinHeaders.def b/external/llvm-project/clang/include/clang/Basic/BuiltinHeaders.def
index 8e4a2f9bee9a..22668ec7a339 100644
--- a/external/llvm-project/clang/include/clang/Basic/BuiltinHeaders.def
+++ b/external/llvm-project/clang/include/clang/Basic/BuiltinHeaders.def
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 HEADER(NO_HEADER, nullptr)
+HEADER(ARMACLE_H, "arm_acle.h")
 HEADER(BLOCKS_H, "Blocks.h")
 HEADER(COMPLEX_H, "complex.h")
 HEADER(CTYPE_H, "ctype.h")
diff --git a/external/llvm-project/clang/include/clang/Basic/Builtins.td b/external/llvm-project/clang/include/clang/Basic/Builtins.td
index e43b87fb3c13..68cd3d790e78 100644
--- a/external/llvm-project/clang/include/clang/Basic/Builtins.td
+++ b/external/llvm-project/clang/include/clang/Basic/Builtins.td
@@ -970,6 +970,12 @@ def IsWithinLifetime : LangBuiltin<"CXX_LANG"> {
   let Prototype = "bool(void*)";
 }
 
+def GetVtablePointer : LangBuiltin<"CXX_LANG"> {
+  let Spellings = ["__builtin_get_vtable_pointer"];
+  let Attributes = [CustomTypeChecking, NoThrow, Const];
+  let Prototype = "void*(void*)";
+}
+
 // GCC exception builtins
 def EHReturn : Builtin {
   let Spellings = ["__builtin_eh_return"];
@@ -4903,6 +4909,12 @@ def HLSLWaveReadLaneAt : LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void(...)";
 }
 
+def HLSLWaveGetLaneCount : LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_wave_get_lane_count"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "unsigned int()";
+}
+
 def HLSLClamp : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_elementwise_clamp"];
   let Attributes = [NoThrow, Const, CustomTypeChecking];
diff --git a/external/llvm-project/clang/include/clang/Basic/BuiltinsAArch64.def b/external/llvm-project/clang/include/clang/Basic/BuiltinsAArch64.def
index 0b2dba86d993..8867a9fe09fb 100644
--- a/external/llvm-project/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/external/llvm-project/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -50,6 +50,11 @@ BUILTIN(__builtin_arm_wfi, "v", "")
 BUILTIN(__builtin_arm_sev, "v", "")
 BUILTIN(__builtin_arm_sevl, "v", "")
 BUILTIN(__builtin_arm_chkfeat, "WUiWUi", "")
+TARGET_HEADER_BUILTIN(__yield, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfe,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfi,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sev,   "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sevl,  "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Like __builtin_trap but provide an 16-bit immediate reason code (which goes into `brk #N`).
 BUILTIN(__builtin_arm_trap, "vUIs", "nr")
@@ -82,6 +87,9 @@ TARGET_BUILTIN(__builtin_arm_mops_memset_tag, "v*v*iz", "", "mte,mops")
 BUILTIN(__builtin_arm_dmb, "vUi", "nc")
 BUILTIN(__builtin_arm_dsb, "vUi", "nc")
 BUILTIN(__builtin_arm_isb, "vUi", "nc")
+TARGET_HEADER_BUILTIN(__dmb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__dsb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__isb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
 
 TARGET_BUILTIN(__builtin_arm_jcvt, "Zid", "nc", "v8.3a")
 
@@ -98,16 +106,6 @@ BUILTIN(__builtin_arm_wsr64, "vcC*WUi", "nc")
 TARGET_BUILTIN(__builtin_arm_wsr128, "vcC*LLLUi", "nc", "d128")
 BUILTIN(__builtin_arm_wsrp, "vcC*vC*", "nc")
 
-// MSVC
-LANGBUILTIN(__dmb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__dsb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__isb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__yield, "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfe,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfi,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__sev,   "v", "",   ALL_MS_LANGUAGES)
-LANGBUILTIN(__sevl,  "v", "",   ALL_MS_LANGUAGES)
-
 // Misc
 BUILTIN(__builtin_sponentry, "v*", "c")
 
diff --git a/external/llvm-project/clang/include/clang/Basic/BuiltinsAMDGPU.def b/external/llvm-project/clang/include/clang/Basic/BuiltinsAMDGPU.def
index dee74bf68002..264cdb52671b 100644
--- a/external/llvm-project/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/external/llvm-project/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -257,7 +257,7 @@ TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_v2bf16, "V2sV2s*0V2s", "t", "at
 TARGET_BUILTIN(__builtin_amdgcn_global_atomic_fadd_v2bf16, "V2sV2s*1V2s", "t", "atomic-global-pk-add-bf16-inst")
 TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_v2bf16, "V2sV2s*3V2s", "t", "atomic-ds-pk-add-16-insts")
 TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_v2f16, "V2hV2h*3V2h", "t", "atomic-ds-pk-add-16-insts")
-TARGET_BUILTIN(__builtin_amdgcn_load_to_lds, "vv*v*3IUiIiIUi", "t", "vmem-to-lds-load-insts")
+TARGET_BUILTIN(__builtin_amdgcn_load_to_lds, "vv*v*3IUiIiIUi", "", "vmem-to-lds-load-insts")
 TARGET_BUILTIN(__builtin_amdgcn_global_load_lds, "vv*1v*3IUiIiIUi", "t", "vmem-to-lds-load-insts")
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/include/clang/Basic/BuiltinsARM.def b/external/llvm-project/clang/include/clang/Basic/BuiltinsARM.def
index cbab87cecbc7..2592e25e95c3 100644
--- a/external/llvm-project/clang/include/clang/Basic/BuiltinsARM.def
+++ b/external/llvm-project/clang/include/clang/Basic/BuiltinsARM.def
@@ -186,11 +186,19 @@ BUILTIN(__builtin_arm_wfi, "v", "")
 BUILTIN(__builtin_arm_sev, "v", "")
 BUILTIN(__builtin_arm_sevl, "v", "")
 BUILTIN(__builtin_arm_dbg, "vUi", "")
+TARGET_HEADER_BUILTIN(__yield, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfe, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__wfi, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sev, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sevl, "v", "h", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Data barrier
 BUILTIN(__builtin_arm_dmb, "vUi", "nc")
 BUILTIN(__builtin_arm_dsb, "vUi", "nc")
 BUILTIN(__builtin_arm_isb, "vUi", "nc")
+TARGET_HEADER_BUILTIN(__dmb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__dsb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__isb, "vUi", "nch", ARMACLE_H, ALL_LANGUAGES, "")
 
 // Prefetch
 BUILTIN(__builtin_arm_prefetch, "vvC*UiUi", "nc")
@@ -209,15 +217,6 @@ BUILTIN(__builtin_sponentry, "v*", "c")
 // MSVC
 LANGBUILTIN(__emit, "vIUiC", "", ALL_MS_LANGUAGES)
 
-LANGBUILTIN(__yield, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfe, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__wfi, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__sev, "v", "", ALL_MS_LANGUAGES)
-LANGBUILTIN(__sevl, "v", "", ALL_MS_LANGUAGES)
-
-LANGBUILTIN(__dmb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__dsb, "vUi", "nc", ALL_MS_LANGUAGES)
-LANGBUILTIN(__isb, "vUi", "nc", ALL_MS_LANGUAGES)
 LANGBUILTIN(__ldrexd, "WiWiCD*", "", ALL_MS_LANGUAGES)
 LANGBUILTIN(_MoveFromCoprocessor, "UiIUiIUiIUiIUiIUi", "", ALL_MS_LANGUAGES)
 LANGBUILTIN(_MoveFromCoprocessor2, "UiIUiIUiIUiIUiIUi", "", ALL_MS_LANGUAGES)
diff --git a/external/llvm-project/clang/include/clang/Basic/BuiltinsBase.td b/external/llvm-project/clang/include/clang/Basic/BuiltinsBase.td
index cf15a31235e7..09bc9f89059f 100644
--- a/external/llvm-project/clang/include/clang/Basic/BuiltinsBase.td
+++ b/external/llvm-project/clang/include/clang/Basic/BuiltinsBase.td
@@ -1,4 +1,4 @@
-//===--- BuiltinsBase.td - common structured used by builtins ---*- C++ -*-===//
+//===--- BuiltinsBase.td - common structured used by builtins -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/external/llvm-project/clang/include/clang/Basic/BuiltinsNVPTX.td b/external/llvm-project/clang/include/clang/Basic/BuiltinsNVPTX.td
index 2cea44e22467..6e531eff6dd1 100644
--- a/external/llvm-project/clang/include/clang/Basic/BuiltinsNVPTX.td
+++ b/external/llvm-project/clang/include/clang/Basic/BuiltinsNVPTX.td
@@ -177,6 +177,7 @@ let Attributes = [NoReturn] in {
 }
 let Attributes = [NoThrow] in {
   def __nvvm_nanosleep : NVPTXBuiltinSMAndPTX<"void(unsigned int)", SM_70, PTX63>;
+  def __nvvm_pm_event_mask : NVPTXBuiltin<"void(_Constant unsigned short)">;
 }
 
 // Min Max
diff --git a/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.def b/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.def
index aad4e107cbeb..2a30ff11464d 100644
--- a/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.def
+++ b/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.def
@@ -223,9 +223,11 @@ AFFECTING_VALUE_CODEGENOPT(OptimizeSize, 2, 0) ///< If -Os (==1) or -Oz (==2) is
 CODEGENOPT(AtomicProfileUpdate , 1, 0) ///< Set -fprofile-update=atomic
 CODEGENOPT(ContinuousProfileSync, 1, 0) ///< Enable continuous instrumentation profiling
 /// Choose profile instrumenation kind or no instrumentation.
-ENUM_CODEGENOPT(ProfileInstr, ProfileInstrKind, 4, ProfileNone)
+
+ENUM_CODEGENOPT(ProfileInstr, llvm::driver::ProfileInstrKind, 4, llvm::driver::ProfileInstrKind::ProfileNone)
+
 /// Choose profile kind for PGO use compilation.
-ENUM_CODEGENOPT(ProfileUse, ProfileInstrKind, 2, ProfileNone)
+ENUM_CODEGENOPT(ProfileUse, llvm::driver::ProfileInstrKind, 2, llvm::driver::ProfileInstrKind::ProfileNone)
 /// Partition functions into N groups and select only functions in group i to be
 /// instrumented. Selected group numbers can be 0 to N-1 inclusive.
 VALUE_CODEGENOPT(ProfileTotalFunctionGroups, 32, 1)
@@ -278,8 +280,6 @@ CODEGENOPT(SanitizeCfiICallNormalizeIntegers, 1, 0) ///< Normalize integer types
                                                     ///< CFI icall function signatures
 CODEGENOPT(SanitizeCfiCanonicalJumpTables, 1, 0) ///< Make jump table symbols canonical
                                                  ///< instead of creating a local jump table.
-CODEGENOPT(UniqueSourceFileNames, 1, 0) ///< Allow the compiler to assume that TUs
-                                        ///< have unique source file names at link time
 CODEGENOPT(SanitizeKcfiArity, 1, 0) ///< Embed arity in KCFI patchable function prefix
 CODEGENOPT(SanitizeCoverageType, 2, 0) ///< Type of sanitizer coverage
                                        ///< instrumentation.
diff --git a/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.h b/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.h
index e10eadfdcb29..70d928b4ec51 100644
--- a/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.h
+++ b/external/llvm-project/clang/include/clang/Basic/CodeGenOptions.h
@@ -80,16 +80,6 @@ class CodeGenOptions : public CodeGenOptionsBase {
     SRCK_InRegs    // Small structs in registers (-freg-struct-return).
   };
 
-  enum ProfileInstrKind {
-    ProfileNone,       // Profile instrumentation is turned off.
-    ProfileClangInstr, // Clang instrumentation to generate execution counts
-                       // to use with PGO.
-    ProfileIRInstr,    // IR level PGO instrumentation in LLVM.
-    ProfileCSIRInstr, // IR level PGO context sensitive instrumentation in LLVM.
-    ProfileIRSampleColdCov, // IR level sample pgo based cold function coverage
-                            // instrumentation in LLVM.
-  };
-
   enum EmbedBitcodeKind {
     Embed_Off,      // No embedded bitcode.
     Embed_All,      // Embed both bitcode and commandline in the output.
@@ -346,6 +336,10 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// -fsymbol-partition (see https://lld.llvm.org/Partitions.html).
   std::string SymbolPartition;
 
+  /// If non-empty, allow the compiler to assume that the given source file
+  /// identifier is unique at link time.
+  std::string UniqueSourceFileIdentifier;
+
   enum RemarkKind {
     RK_Missing,            // Remark argument not present on the command line.
     RK_Enabled,            // Remark enabled via '-Rgroup'.
@@ -526,35 +520,41 @@ class CodeGenOptions : public CodeGenOptionsBase {
 
   /// Check if Clang profile instrumenation is on.
   bool hasProfileClangInstr() const {
-    return getProfileInstr() == ProfileClangInstr;
+    return getProfileInstr() ==
+           llvm::driver::ProfileInstrKind::ProfileClangInstr;
   }
 
   /// Check if IR level profile instrumentation is on.
   bool hasProfileIRInstr() const {
-    return getProfileInstr() == ProfileIRInstr;
+    return getProfileInstr() == llvm::driver::ProfileInstrKind::ProfileIRInstr;
   }
 
   /// Check if CS IR level profile instrumentation is on.
   bool hasProfileCSIRInstr() const {
-    return getProfileInstr() == ProfileCSIRInstr;
+    return getProfileInstr() ==
+           llvm::driver::ProfileInstrKind::ProfileCSIRInstr;
   }
 
   /// Check if any form of instrumentation is on.
-  bool hasProfileInstr() const { return getProfileInstr() != ProfileNone; }
+  bool hasProfileInstr() const {
+    return getProfileInstr() != llvm::driver::ProfileInstrKind::ProfileNone;
+  }
 
   /// Check if Clang profile use is on.
   bool hasProfileClangUse() const {
-    return getProfileUse() == ProfileClangInstr;
+    return getProfileUse() == llvm::driver::ProfileInstrKind::ProfileClangInstr;
   }
 
   /// Check if IR level profile use is on.
   bool hasProfileIRUse() const {
-    return getProfileUse() == ProfileIRInstr ||
-           getProfileUse() == ProfileCSIRInstr;
+    return getProfileUse() == llvm::driver::ProfileInstrKind::ProfileIRInstr ||
+           getProfileUse() == llvm::driver::ProfileInstrKind::ProfileCSIRInstr;
   }
 
   /// Check if CSIR profile use is on.
-  bool hasProfileCSIRUse() const { return getProfileUse() == ProfileCSIRInstr; }
+  bool hasProfileCSIRUse() const {
+    return getProfileUse() == llvm::driver::ProfileInstrKind::ProfileCSIRInstr;
+  }
 
   /// Check if type and variable info should be emitted.
   bool hasReducedDebugInfo() const {
diff --git a/external/llvm-project/clang/include/clang/Basic/Diagnostic.h b/external/llvm-project/clang/include/clang/Basic/Diagnostic.h
index e9c54c3c487c..7ae4ef7df138 100644
--- a/external/llvm-project/clang/include/clang/Basic/Diagnostic.h
+++ b/external/llvm-project/clang/include/clang/Basic/Diagnostic.h
@@ -18,6 +18,7 @@
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
+#include "clang/Basic/UnsignedOrNone.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FunctionExtras.h"
@@ -49,6 +50,7 @@ class FileSystem;
 namespace clang {
 
 class DeclContext;
+class Diagnostic;
 class DiagnosticBuilder;
 class DiagnosticConsumer;
 class IdentifierInfo;
@@ -228,6 +230,8 @@ class DiagStorageAllocator {
 class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
 public:
   /// The level of the diagnostic, after it has been through mapping.
+  // FIXME: Make this an alias for DiagnosticIDs::Level as soon as
+  // we can use 'using enum'.
   enum Level {
     Ignored = DiagnosticIDs::Ignored,
     Note = DiagnosticIDs::Note,
@@ -420,10 +424,13 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
     bool empty() const { return Files.empty(); }
 
     /// Clear out this map.
-    void clear() {
+    void clear(bool Soft) {
+      // Just clear the cache when in soft mode.
       Files.clear();
-      FirstDiagState = CurDiagState = nullptr;
-      CurDiagStateLoc = SourceLocation();
+      if (!Soft) {
+        FirstDiagState = CurDiagState = nullptr;
+        CurDiagStateLoc = SourceLocation();
+      }
     }
 
     /// Produce a debugging dump of the diagnostic state.
@@ -532,7 +539,7 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   ///
   /// This is used to emit continuation diagnostics with the same level as the
   /// diagnostic that they follow.
-  DiagnosticIDs::Level LastDiagLevel;
+  Level LastDiagLevel;
 
   /// Number of warnings reported
   unsigned NumWarnings;
@@ -777,18 +784,16 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   /// the middle of another diagnostic.
   ///
   /// This can be used by clients who suppress diagnostics themselves.
-  void setLastDiagnosticIgnored(bool Ignored) {
-    if (LastDiagLevel == DiagnosticIDs::Fatal)
+  void setLastDiagnosticIgnored(bool IsIgnored) {
+    if (LastDiagLevel == Fatal)
       FatalErrorOccurred = true;
-    LastDiagLevel = Ignored ? DiagnosticIDs::Ignored : DiagnosticIDs::Warning;
+    LastDiagLevel = IsIgnored ? Ignored : Warning;
   }
 
   /// Determine whether the previous diagnostic was ignored. This can
   /// be used by clients that want to determine whether notes attached to a
   /// diagnostic will be suppressed.
-  bool isLastDiagnosticIgnored() const {
-    return LastDiagLevel == DiagnosticIDs::Ignored;
-  }
+  bool isLastDiagnosticIgnored() const { return LastDiagLevel == Ignored; }
 
   /// Controls whether otherwise-unmapped extension diagnostics are
   /// mapped onto ignore/warning/error.
@@ -918,6 +923,10 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   /// Reset the state of the diagnostic object to its initial configuration.
   /// \param[in] soft - if true, doesn't reset the diagnostic mappings and state
   void Reset(bool soft = false);
+  /// We keep a cache of FileIDs for diagnostics mapped by pragmas. These might
+  /// get invalidated when diagnostics engine is shared across different
+  /// compilations. Provide users with a way to reset that.
+  void ResetPragmas();
 
   //===--------------------------------------------------------------------===//
   // DiagnosticsEngine classification and reporting interfaces.
@@ -1024,9 +1033,10 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   /// Used to report a diagnostic that is finally fully formed.
   ///
   /// \returns true if the diagnostic was emitted, false if it was suppressed.
-  bool ProcessDiag(const DiagnosticBuilder &DiagBuilder) {
-    return Diags->ProcessDiag(*this, DiagBuilder);
-  }
+  bool ProcessDiag(const DiagnosticBuilder &DiagBuilder);
+
+  /// Forward a diagnostic to the DiagnosticConsumer.
+  void Report(Level DiagLevel, const Diagnostic &Info);
 
   /// @name Diagnostic Emission
   /// @{
diff --git a/external/llvm-project/clang/include/clang/Basic/DiagnosticDriverKinds.td b/external/llvm-project/clang/include/clang/Basic/DiagnosticDriverKinds.td
index aaa594471e76..fb947965c9c5 100644
--- a/external/llvm-project/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/external/llvm-project/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -219,6 +219,9 @@ def err_drv_cannot_open_randomize_layout_seed_file : Error<
   "cannot read randomize layout seed file '%0'">;
 def err_drv_invalid_version_number : Error<
   "invalid version number in '%0'">;
+def err_drv_invalid_version_number_inferred
+    : Error<"invalid version number '%0' inferred from '%1'">;
+def err_drv_missing_version_number : Error<"missing version number in '%0'">;
 def err_drv_kcfi_arity_unsupported_target : Error<
   "target '%0' is unsupported by -fsanitize-kcfi-arity">;
 def err_drv_no_linker_llvm_support : Error<
@@ -491,6 +494,9 @@ def warn_ignoring_ftabstop_value : Warning<
 def warn_drv_overriding_option : Warning<
   "overriding '%0' option with '%1'">,
   InGroup<DiagGroup<"overriding-option">>;
+def warn_drv_overriding_deployment_version
+    : Warning<"overriding deployment version from '%0' to '%1'">,
+      InGroup<DiagGroup<"overriding-deployment-version">>;
 def warn_drv_treating_input_as_cxx : Warning<
   "treating '%0' input as '%1' when in C++ mode, this behavior is deprecated">,
   InGroup<Deprecated>;
diff --git a/external/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td b/external/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td
index 277b461dd8b3..dafae9983d6c 100644
--- a/external/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/external/llvm-project/clang/include/clang/Basic/DiagnosticGroups.td
@@ -116,9 +116,11 @@ def DeprecatedOFast : DiagGroup<"deprecated-ofast">;
 def ObjCSignedCharBoolImplicitIntConversion :
   DiagGroup<"objc-signed-char-bool-implicit-int-conversion">;
 def Shorten64To32 : DiagGroup<"shorten-64-to-32">;
+def ImplicitIntConversionOnNegation : DiagGroup<"implicit-int-conversion-on-negation">;
 def ImplicitIntConversion : DiagGroup<"implicit-int-conversion",
                                      [Shorten64To32,
-                                      ObjCSignedCharBoolImplicitIntConversion]>;
+                                      ObjCSignedCharBoolImplicitIntConversion,
+                                      ImplicitIntConversionOnNegation]>;
 def ImplicitConstIntFloatConversion : DiagGroup<"implicit-const-int-float-conversion">;
 def ImplicitIntFloatConversion : DiagGroup<"implicit-int-float-conversion",
  [ImplicitConstIntFloatConversion]>;
@@ -804,32 +806,39 @@ def UniqueObjectDuplication : DiagGroup<"unique-object-duplication"> {
 Warns when objects which are supposed to be globally unique might get duplicated
 when built into a shared library.
 
-If an object with hidden visibility is built into a shared library, each instance
+This can occur to objects which are hidden from the dynamic linker, due to
+having hidden visibility (on posix) or lacking a dllimport/dllexport attribute
+(on windows). If such an object is built into a shared library, each instance
 of the library will get its own copy. This can cause very subtle bugs if there was
 only supposed to be one copy of the object in question: singletons aren't single,
 changes to one object won't affect the others, the object's initializer will run
 once per copy, etc.
 
 Specifically, this warning fires when it detects an object which:
-  1. Is defined as ``inline`` in a header file (so it might get compiled into multiple libaries), and
-  2. Has external linkage (otherwise it's supposed to be duplicated), and
-  3. Has hidden visibility.
+
+#. Is defined as ``inline`` in a header file (so it might get compiled into multiple libaries), and
+#. Has external linkage (otherwise it's supposed to be duplicated), and
+#. Has hidden visibility (posix) or lacks a dllimport/dllexport attribute (windows).
 
 As well as one of the following:
-  1. The object is mutable, or
-  2. The object's initializer definitely has side effects.
+
+#. The object is mutable, or
+#. The object's initializer definitely has side effects.
 
 The warning can be resolved by removing one of the conditions above. In rough
 order of preference, this may be done by:
-  1. Marking the object ``const`` (if possible)
-  2. Moving the object's definition to a source file
-  3. Giving the object non-hidden visibility, e.g. using ``__attribute((visibility("default")))``.
+
+#. Marking the object ``const`` (if possible)
+#. Moving the object's definition to a source file
+#. Making the object visible using ``__attribute((visibility("default")))``,
+   ``__declspec(dllimport)``, or ``__declspec(dllexport)``.
+
+When annotating an object with ``__declspec(dllimport)`` or ``__declspec(dllexport)``,
+take care to ensure that the object is only exported from one dll, and is imported
+everywhere else.
 
 Note that for (2), all levels of a pointer variable must be constant;
 ``const int*`` will trigger the warning because the pointer itself is mutable.
-
-This warning is not yet implemented for Windows, since Windows uses
-import/export rules instead of visibility.
 }];
 }
 
diff --git a/external/llvm-project/clang/include/clang/Basic/DiagnosticIDs.h b/external/llvm-project/clang/include/clang/Basic/DiagnosticIDs.h
index 80d52a0d0111..2b095f0fd674 100644
--- a/external/llvm-project/clang/include/clang/Basic/DiagnosticIDs.h
+++ b/external/llvm-project/clang/include/clang/Basic/DiagnosticIDs.h
@@ -483,18 +483,6 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
 
   Class getDiagClass(unsigned DiagID) const;
 
-  /// Used to report a diagnostic that is finally fully formed.
-  ///
-  /// \returns \c true if the diagnostic was emitted, \c false if it was
-  /// suppressed.
-  bool ProcessDiag(DiagnosticsEngine &Diag,
-                   const DiagnosticBuilder &DiagBuilder) const;
-
-  /// Used to emit a diagnostic that is finally fully formed,
-  /// ignoring suppression.
-  void EmitDiag(DiagnosticsEngine &Diag, const DiagnosticBuilder &DiagBuilder,
-                Level DiagLevel) const;
-
   /// Whether the diagnostic may leave the AST in a state where some
   /// invariants can break.
   bool isUnrecoverable(unsigned DiagID) const;
diff --git a/external/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td b/external/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td
index 3aa36ad59d0b..6c30da376daf 100644
--- a/external/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/external/llvm-project/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1581,8 +1581,10 @@ def err_omp_unexpected_append_op : Error<
   "unexpected operation specified in 'append_args' clause, expected 'interop'">;
 def err_omp_unexpected_execution_modifier : Error<
   "unexpected 'execution' modifier in non-executable context">;
-def err_omp_unknown_adjust_args_op : Error<
-  "incorrect adjust_args type, expected 'need_device_ptr' or 'nothing'">;
+def err_omp_unknown_adjust_args_op
+    : Error<
+          "incorrect 'adjust_args' type, expected 'need_device_ptr'%select{|, "
+          "'need_device_addr',}0 or 'nothing'">;
 def err_omp_declare_variant_wrong_clause : Error<
   "expected %select{'match'|'match', 'adjust_args', or 'append_args'}0 clause "
   "on 'omp declare variant' directive">;
diff --git a/external/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td b/external/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td
index e4c624c02e96..e5681a22c1e9 100644
--- a/external/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/external/llvm-project/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1763,24 +1763,39 @@ def err_user_defined_msg_constexpr : Error<
   "constant expression">;
 
 // Type traits explanations
-def note_unsatisfied_trait : Note<"%0 is not %enum_select<TraitName>{"
-                                  "%TriviallyRelocatable{trivially relocatable}"
-                                  "}1">;
+def note_unsatisfied_trait
+    : Note<"%0 is not %enum_select<TraitName>{"
+           "%TriviallyRelocatable{trivially relocatable}|"
+           "%Replaceable{replaceable}|"
+           "%TriviallyCopyable{trivially copyable}"
+           "}1">;
 
 def note_unsatisfied_trait_reason
     : Note<"because it "
            "%enum_select<TraitNotSatisfiedReason>{"
            "%Ref{is a reference type}|"
+           "%Const{is const}|"
+           "%Volatile{is volatile}|"
            "%HasArcLifetime{has an ARC lifetime qualifier}|"
            "%VLA{is a variably-modified type}|"
            "%VBase{has a virtual base %1}|"
-           "%NRBase{has a non-trivially-relocatable base %1}|"
-           "%NRField{has a non-trivially-relocatable member %1 of type %2}|"
+           "%NotScalarOrClass{is not %select{a|an array of objects of}1 scalar or "
+           "class type}|"
+           "%NTRBase{has a non-trivially-relocatable base %1}|"
+           "%NTRField{has a non-trivially-relocatable member %1 of type %2}|"
+           "%NonReplaceableBase{has a non-replaceable base %1}|"
+           "%NonReplaceableField{has a non-replaceable member %1 of type %2}|"
+           "%NTCBase{has a non-trivially-copyable base %1}|"
+           "%NTCField{has a non-trivially-copyable member %1 of type %2}|"
            "%DeletedDtr{has a %select{deleted|user-provided}1 destructor}|"
            "%UserProvidedCtr{has a user provided %select{copy|move}1 "
            "constructor}|"
+           "%DeletedCtr{has a deleted %select{copy|move}1 "
+           "constructor}|"
            "%UserProvidedAssign{has a user provided %select{copy|move}1 "
            "assignment operator}|"
+           "%DeletedAssign{has a deleted %select{copy|move}1 "
+           "assignment operator}|"
            "%UnionWithUserDeclaredSMF{is a union with a user-declared "
            "%sub{select_special_member_kind}1}"
            "}0">;
@@ -3105,8 +3120,6 @@ def err_reference_to_function_with_unsatisfied_constraints : Error<
   "invalid reference to function %0: constraints not satisfied">;
 def err_requires_expr_local_parameter_default_argument : Error<
   "default arguments not allowed for parameters of a requires expression">;
-def err_requires_expr_parameter_referenced_in_evaluated_context : Error<
-  "constraint variable %0 cannot be used in an evaluated context">;
 def note_expr_requirement_expr_substitution_error : Note<
   "%select{and|because}0 '%1' would be invalid: %2">;
 def note_expr_requirement_expr_unknown_substitution_error : Note<
@@ -4050,6 +4063,9 @@ def warn_at_available_unchecked_use : Warning<
   "%select{@available|__builtin_available}0 does not guard availability here; "
   "use if (%select{@available|__builtin_available}0) instead">,
   InGroup<DiagGroup<"unsupported-availability-guard">>;
+def warn_availability_invalid_os_version
+    : Warning<"invalid %1 version '%0' in availability attribute">, InGroup<DiagGroup<"invalid-version-availability">>;
+def note_availability_invalid_os_version_adjusted: Note<"implicitly treating version as '%0'">;
 
 def warn_missing_sdksettings_for_availability_checking : Warning<
   "%0 availability is ignored without a valid 'SDKSettings.json' in the SDK">,
@@ -4288,6 +4304,9 @@ def warn_impcast_integer_sign_conditional : Warning<
 def warn_impcast_integer_precision : Warning<
   "implicit conversion loses integer precision: %0 to %1">,
   InGroup<ImplicitIntConversion>, DefaultIgnore;
+def warn_impcast_integer_precision_on_negation : Warning<
+  "implicit conversion loses integer precision: %0 to %1 on negation">,
+  InGroup<ImplicitIntConversionOnNegation>, DefaultIgnore;
 def warn_impcast_high_order_zero_bits : Warning<
   "higher order bits are zeroes after implicit conversion">,
   InGroup<ImplicitIntConversion>, DefaultIgnore;
@@ -6244,14 +6263,19 @@ def warn_static_local_in_extern_inline : Warning<
 def note_convert_inline_to_static : Note<
   "use 'static' to give inline function %0 internal linkage">;
 
-def warn_possible_object_duplication_mutable : Warning<
-  "%0 may be duplicated when built into a shared library: "
-  "it is mutable, has hidden visibility, and external linkage">,
-  InGroup<UniqueObjectDuplication>, DefaultIgnore;
-def warn_possible_object_duplication_init : Warning<
-  "initialization of %0 may run twice when built into a shared library: "
-  "it has hidden visibility and external linkage">,
-  InGroup<UniqueObjectDuplication>, DefaultIgnore;
+def warn_possible_object_duplication_mutable
+    : Warning<"%0 may be duplicated when built into a shared library: "
+              "it is mutable, with external linkage and "
+              "%select{hidden visibility|no import/export annotation}1">,
+      InGroup<UniqueObjectDuplication>,
+      DefaultIgnore;
+def warn_possible_object_duplication_init
+    : Warning<"initialization of %0 may run twice when built into a shared "
+              "library: "
+              "it has external linkage and "
+              "%select{hidden visibility|no import/export annotation}1">,
+      InGroup<UniqueObjectDuplication>,
+      DefaultIgnore;
 
 def ext_redefinition_of_typedef : ExtWarn<
   "redefinition of typedef %0 is a C11 feature">,
@@ -9416,6 +9440,8 @@ def err_cuda_host_shared : Error<
     "%select{__device__|__global__|__host__|__host__ __device__}0 functions">;
 def err_cuda_nonstatic_constdev: Error<"__constant__, __device__, and "
     "__managed__ are not allowed on non-static local variables">;
+def err_cuda_address_space_gpuvar: Error<"__constant__, __device__, and "
+    "__shared__ variables must use default address space">;
 def err_cuda_grid_constant_not_allowed : Error<
   "__grid_constant__ is only allowed on const-qualified kernel parameters">;
 def err_cuda_ovl_target : Error<
@@ -12712,6 +12738,11 @@ def warn_noderef_on_non_pointer_or_array : Warning<
 def warn_noderef_to_dereferenceable_pointer : Warning<
   "casting to dereferenceable pointer removes 'noderef' attribute">, InGroup<NoDeref>;
 
+def warn_cast_discards_cfi_unchecked_callee
+    : Warning<"implicit conversion from %0 to %1 discards "
+              "'cfi_unchecked_callee' attribute">,
+      InGroup<DiagGroup<"cfi-unchecked-callee">>;
+
 def err_builtin_launder_invalid_arg : Error<
   "%select{non-pointer|function pointer|void pointer}0 argument to "
   "'__builtin_launder' is not allowed">;
@@ -12804,6 +12835,14 @@ def err_bit_cast_non_trivially_copyable : Error<
 def err_bit_cast_type_size_mismatch : Error<
   "size of '__builtin_bit_cast' source type %0 does not match destination type %1 (%2 vs %3 bytes)">;
 
+def err_get_vtable_pointer_incorrect_type
+    : Error<"__builtin_get_vtable_pointer requires an argument of%select{| "
+            "polymorphic}0 class pointer type"
+            ", but %1 %select{was provided|has no virtual methods}0">;
+def err_get_vtable_pointer_requires_complete_type
+    : Error<"__builtin_get_vtable_pointer requires an argument with a complete "
+            "type, but %0 is incomplete">;
+
 // SYCL-specific diagnostics
 def warn_sycl_kernel_num_of_template_params : Warning<
   "'sycl_kernel' attribute only applies to a function template with at least"
diff --git a/external/llvm-project/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/external/llvm-project/clang/include/clang/Basic/DiagnosticSerializationKinds.td
index 7965da593f21..584c8d62280b 100644
--- a/external/llvm-project/clang/include/clang/Basic/DiagnosticSerializationKinds.td
+++ b/external/llvm-project/clang/include/clang/Basic/DiagnosticSerializationKinds.td
@@ -13,36 +13,37 @@ def err_fe_unable_to_read_pch_file : Error<
     "unable to read PCH file %0: '%1'">;
 def err_fe_not_a_pch_file : Error<
     "input is not a PCH file: '%0'">;
-def err_fe_pch_malformed : Error<
-    "malformed or corrupted AST file: '%0'">, DefaultFatal;
+def err_fe_ast_file_malformed : Error<
+    "malformed or corrupted precompiled file: '%0'">, DefaultFatal;
 def err_fe_pch_malformed_block : Error<
     "malformed block record in PCH file: '%0'">, DefaultFatal;
 def err_fe_ast_file_modified : Error<
     "file '%0' has been modified since the "
-    "%select{precompiled header|module file|AST file}1 '%2' was built"
+    "%select{precompiled header|module file|precompiled file}1 '%2' was built"
     ": %select{size|mtime|content}3 changed%select{| (was %5, now %6)}4">,
     DefaultFatal;
 def err_fe_pch_file_overridden : Error<
     "file '%0' from the precompiled header has been overridden">;
-def note_pch_required_by : Note<"'%0' required by '%1'">;
-def note_pch_rebuild_required : Note<"please rebuild precompiled header '%0'">;
+def note_ast_file_required_by : Note<"'%0' required by '%1'">;
+def note_ast_file_rebuild_required
+    : Note<"please rebuild precompiled file '%0'">;
 def note_module_cache_path : Note<
     "after modifying system headers, please delete the module cache at '%0'">;
 
 def err_ast_file_targetopt_mismatch : Error<
-    "AST file '%0' was compiled for the %1 '%2' but the current translation "
+    "precompiled file '%0' was compiled for the %1 '%2' but the current translation "
     "unit is being compiled for target '%3'">;
 def err_ast_file_targetopt_feature_mismatch : Error<
-    "%select{AST file '%1' was|current translation unit is}0 compiled with the target "
-    "feature '%2' but the %select{current translation unit is|AST file '%1' was}0 "
+    "%select{precompiled file '%1' was|current translation unit is}0 compiled with the target "
+    "feature '%2' but the %select{current translation unit is|precompiled file '%1' was}0 "
     "not">;
 def err_ast_file_langopt_mismatch : Error<"%0 was %select{disabled|enabled}1 in "
-    "AST file '%3' but is currently %select{disabled|enabled}2">;
+    "precompiled file '%3' but is currently %select{disabled|enabled}2">;
 def err_ast_file_langopt_value_mismatch : Error<
-  "%0 differs in AST file '%1' vs. current file">;
+  "%0 differs in precompiled file '%1' vs. current file">;
 def err_ast_file_diagopt_mismatch : Error<"%0 is currently enabled, but was not in "
-  "the AST file '%1'">;
-def err_ast_file_modulecache_mismatch : Error<"AST file '%2' was compiled with module cache "
+  "the precompiled file '%1'">;
+def err_ast_file_modulecache_mismatch : Error<"precompiled file '%2' was compiled with module cache "
   "path '%0', but the path is currently '%1'">;
 def warn_pch_vfsoverlay_mismatch : Warning<
   "PCH was compiled with different VFS overlay files than are currently in use">,
@@ -51,27 +52,27 @@ def note_pch_vfsoverlay_files : Note<"%select{PCH|current translation unit}0 has
 def note_pch_vfsoverlay_empty : Note<"%select{PCH|current translation unit}0 has no VFS overlays">;
 
 def err_ast_file_version_too_old : Error<
-    "%select{PCH|module|AST}0 file '%1' uses an older format that is no longer supported">;
+    "%select{PCH|module|precompiled}0 file '%1' uses an older format that is no longer supported">;
 def err_ast_file_version_too_new : Error<
-    "%select{PCH|module|AST}0 file '%1' uses a newer format that cannot be read">;
+    "%select{PCH|module|precompiled }0 file '%1' uses a newer format that cannot be read">;
 def err_ast_file_different_branch : Error<
-    "%select{PCH|module|AST}0 file '%1' built from a different branch (%2) than the compiler (%3)">;
+    "%select{PCH|module|precompiled}0 file '%1' built from a different branch (%2) than the compiler (%3)">;
 def err_ast_file_with_compiler_errors : Error<
-    "%select{PCH|module|AST}0 file '%1' contains compiler errors">;
+    "%select{PCH|module|precompiled}0 file '%1' contains compiler errors">;
 
 def err_module_file_conflict : Error<
   "module '%0' is defined in both '%1' and '%2'">, DefaultFatal;
 def err_ast_file_not_found : Error<
-  "%select{PCH|module|AST}0 file '%1' not found%select{|: %3}2">, DefaultFatal;
+  "%select{PCH|module|precompiled}0 file '%1' not found%select{|: %3}2">, DefaultFatal;
 def err_ast_file_out_of_date : Error<
-  "%select{PCH|module|AST}0 file '%1' is out of date and "
+  "%select{PCH|module|precompiled}0 file '%1' is out of date and "
   "needs to be rebuilt%select{|: %3}2">, DefaultFatal;
 def err_ast_file_invalid : Error<
-  "file '%1' is not a valid precompiled %select{PCH|module|AST}0 file: %2">, DefaultFatal;
+  "file '%1' is not a valid %select{PCH|module|precompiled}0 file: %2">, DefaultFatal;
 def note_module_file_imported_by : Note<
   "imported by %select{|module '%2' in }1'%0'">;
 def err_module_file_not_module : Error<
-  "AST file '%0' was not built as a module">, DefaultFatal;
+  "precompiled file '%0' was not built as a module">, DefaultFatal;
 def err_module_file_missing_top_level_submodule : Error<
   "module file '%0' is missing its top-level submodule">, DefaultFatal;
 def note_module_file_conflict : Note<
@@ -83,14 +84,14 @@ def remark_module_import : Remark<
   InGroup<ModuleImport>;
 
 def err_imported_module_not_found : Error<
-    "module '%0' in AST file '%1' %select{(imported by AST file '%2') |}4"
+    "module '%0' in precompiled file '%1' %select{(imported by precompiled file '%2') |}4"
     "is not defined in any loaded module map file; "
     "maybe you need to load '%3'?">, DefaultFatal;
 def note_imported_by_pch_module_not_found : Note<
     "consider adding '%0' to the header search path">;
 def err_imported_module_modmap_changed : Error<
-    "module '%0' %select{in|imported by}4 AST file '%1' found in a different module map file"
-    " (%2) than when the importing AST file was built (%3)">, DefaultFatal;
+    "module '%0' %select{in|imported by}4 precompiled file '%1' found in a different module map file"
+    " (%2) than when the importing precompiled file was built (%3)">, DefaultFatal;
 def err_imported_module_relocated : Error<
     "module '%0' was built in directory '%1' but now resides in "
     "directory '%2'">, DefaultFatal;
@@ -99,18 +100,18 @@ def err_module_different_modmap : Error<
     "%select{| not}1 used when the module was built">;
 
 def err_ast_file_macro_def_undef : Error<
-    "macro '%0' was %select{defined|undef'd}1 in the AST file '%2' but "
+    "macro '%0' was %select{defined|undef'd}1 in the precompiled file '%2' but "
     "%select{undef'd|defined}1 on the command line">;
 def err_ast_file_macro_def_conflict : Error<
-    "definition of macro '%0' differs between the AST file '%3' ('%1') "
+    "definition of macro '%0' differs between the precompiled file '%3' ('%1') "
     "and the command line ('%2')">;
 def err_ast_file_undef : Error<
-    "%select{command line contains|AST file '%1' was built with}0 "
-    "'-undef' but %select{AST file '%1' was not built with it|"
+    "%select{command line contains|precompiled file '%1' was built with}0 "
+    "'-undef' but %select{precompiled file '%1' was not built with it|"
     "it is not present on the command line}0">;
 def err_ast_file_pp_detailed_record : Error<
-    "%select{command line contains|AST file '%1' was built with}0 "
-    "'-detailed-preprocessing-record' but %select{AST file '%1' was not "
+    "%select{command line contains|precompiled file '%1' was built with}0 "
+    "'-detailed-preprocessing-record' but %select{precompiled file '%1' was not "
     "built with it|it is not present on the command line}0">;
 
 def err_module_odr_violation_missing_decl : Error<
diff --git a/external/llvm-project/clang/include/clang/Basic/IdentifierTable.h b/external/llvm-project/clang/include/clang/Basic/IdentifierTable.h
index 54540193cfcc..e4044bcdfcc6 100644
--- a/external/llvm-project/clang/include/clang/Basic/IdentifierTable.h
+++ b/external/llvm-project/clang/include/clang/Basic/IdentifierTable.h
@@ -731,7 +731,7 @@ class IdentifierTable {
   /// introduce or modify an identifier. If they called get(), they would
   /// likely end up in a recursion.
   IdentifierInfo &getOwn(StringRef Name) {
-    auto &Entry = *HashTable.insert(std::make_pair(Name, nullptr)).first;
+    auto &Entry = *HashTable.try_emplace(Name).first;
 
     IdentifierInfo *&II = Entry.second;
     if (II)
diff --git a/external/llvm-project/clang/include/clang/Basic/OpenMPKinds.def b/external/llvm-project/clang/include/clang/Basic/OpenMPKinds.def
index b0de65df7e39..2b1dc1e0121b 100644
--- a/external/llvm-project/clang/include/clang/Basic/OpenMPKinds.def
+++ b/external/llvm-project/clang/include/clang/Basic/OpenMPKinds.def
@@ -214,6 +214,7 @@ OPENMP_ORIGINAL_SHARING_MODIFIER(default)
 // Adjust-op kinds for the 'adjust_args' clause.
 OPENMP_ADJUST_ARGS_KIND(nothing)
 OPENMP_ADJUST_ARGS_KIND(need_device_ptr)
+OPENMP_ADJUST_ARGS_KIND(need_device_addr)
 
 // Binding kinds for the 'bind' clause.
 OPENMP_BIND_KIND(teams)
diff --git a/external/llvm-project/clang/include/clang/Basic/ProfileList.h b/external/llvm-project/clang/include/clang/Basic/ProfileList.h
index b4217e49c18a..5338ef3992ad 100644
--- a/external/llvm-project/clang/include/clang/Basic/ProfileList.h
+++ b/external/llvm-project/clang/include/clang/Basic/ProfileList.h
@@ -49,17 +49,16 @@ class ProfileList {
   ~ProfileList();
 
   bool isEmpty() const { return Empty; }
-  ExclusionType getDefault(CodeGenOptions::ProfileInstrKind Kind) const;
+  ExclusionType getDefault(llvm::driver::ProfileInstrKind Kind) const;
 
   std::optional<ExclusionType>
   isFunctionExcluded(StringRef FunctionName,
-                     CodeGenOptions::ProfileInstrKind Kind) const;
+                     llvm::driver::ProfileInstrKind Kind) const;
   std::optional<ExclusionType>
   isLocationExcluded(SourceLocation Loc,
-                     CodeGenOptions::ProfileInstrKind Kind) const;
+                     llvm::driver::ProfileInstrKind Kind) const;
   std::optional<ExclusionType>
-  isFileExcluded(StringRef FileName,
-                 CodeGenOptions::ProfileInstrKind Kind) const;
+  isFileExcluded(StringRef FileName, llvm::driver::ProfileInstrKind Kind) const;
 };
 
 } // namespace clang
diff --git a/external/llvm-project/clang/include/clang/Basic/Specifiers.h b/external/llvm-project/clang/include/clang/Basic/Specifiers.h
index 491badcc804e..698fd9da5ced 100644
--- a/external/llvm-project/clang/include/clang/Basic/Specifiers.h
+++ b/external/llvm-project/clang/include/clang/Basic/Specifiers.h
@@ -289,14 +289,13 @@ namespace clang {
     CC_AAPCS_VFP,          // __attribute__((pcs("aapcs-vfp")))
     CC_IntelOclBicc,       // __attribute__((intel_ocl_bicc))
     CC_SpirFunction,       // default for OpenCL functions on SPIR target
-    CC_OpenCLKernel,       // inferred for OpenCL kernels
+    CC_DeviceKernel,       // __attribute__((device_kernel))
     CC_Swift,              // __attribute__((swiftcall))
     CC_SwiftAsync,         // __attribute__((swiftasynccall))
     CC_PreserveMost,       // __attribute__((preserve_most))
     CC_PreserveAll,        // __attribute__((preserve_all))
     CC_AArch64VectorCall,  // __attribute__((aarch64_vector_pcs))
     CC_AArch64SVEPCS,      // __attribute__((aarch64_sve_pcs))
-    CC_AMDGPUKernelCall,   // __attribute__((amdgpu_kernel))
     CC_M68kRTD,            // __attribute__((m68k_rtd))
     CC_PreserveNone,       // __attribute__((preserve_none))
     CC_RISCVVectorCall,    // __attribute__((riscv_vector_cc))
@@ -326,7 +325,7 @@ namespace clang {
     case CC_X86Pascal:
     case CC_X86VectorCall:
     case CC_SpirFunction:
-    case CC_OpenCLKernel:
+    case CC_DeviceKernel:
     case CC_Swift:
     case CC_SwiftAsync:
     case CC_M68kRTD:
diff --git a/external/llvm-project/clang/include/clang/Basic/StmtNodes.td b/external/llvm-project/clang/include/clang/Basic/StmtNodes.td
index 9526fa5808aa..c9c173f5c746 100644
--- a/external/llvm-project/clang/include/clang/Basic/StmtNodes.td
+++ b/external/llvm-project/clang/include/clang/Basic/StmtNodes.td
@@ -202,7 +202,6 @@ def ShuffleVectorExpr : StmtNode<Expr>;
 def ConvertVectorExpr : StmtNode<Expr>;
 def BlockExpr : StmtNode<Expr>;
 def OpaqueValueExpr : StmtNode<Expr>;
-def TypoExpr : StmtNode<Expr>;
 def RecoveryExpr : StmtNode<Expr>;
 def BuiltinBitCastExpr : StmtNode<ExplicitCastExpr>;
 def EmbedExpr : StmtNode<Expr>;
diff --git a/external/llvm-project/clang/include/clang/Basic/TargetInfo.h b/external/llvm-project/clang/include/clang/Basic/TargetInfo.h
index 652dc064a7b1..c3bce6e807f3 100644
--- a/external/llvm-project/clang/include/clang/Basic/TargetInfo.h
+++ b/external/llvm-project/clang/include/clang/Basic/TargetInfo.h
@@ -270,7 +270,7 @@ class TargetInfo : public TransferrableTargetInfo,
   unsigned HasBuiltinMSVaList : 1;
 
   LLVM_PREFERRED_TYPE(bool)
-  unsigned HasAArch64SVETypes : 1;
+  unsigned HasAArch64ACLETypes : 1;
 
   LLVM_PREFERRED_TYPE(bool)
   unsigned HasRISCVVTypes : 1;
@@ -1055,9 +1055,9 @@ class TargetInfo : public TransferrableTargetInfo,
   /// available on this target.
   bool hasBuiltinMSVaList() const { return HasBuiltinMSVaList; }
 
-  /// Returns whether or not the AArch64 SVE built-in types are
+  /// Returns whether or not the AArch64 ACLE built-in types are
   /// available on this target.
-  bool hasAArch64SVETypes() const { return HasAArch64SVETypes; }
+  bool hasAArch64ACLETypes() const { return HasAArch64ACLETypes; }
 
   /// Returns whether or not the RISC-V V built-in types are
   /// available on this target.
diff --git a/external/llvm-project/clang/include/clang/Basic/arm_mve.td b/external/llvm-project/clang/include/clang/Basic/arm_mve.td
index e33c065059c4..412ef9abac1b 100644
--- a/external/llvm-project/clang/include/clang/Basic/arm_mve.td
+++ b/external/llvm-project/clang/include/clang/Basic/arm_mve.td
@@ -710,7 +710,7 @@ defm vrndmq: vrnd<IRIntBase<"floor", [Vector]>, "m">;
 defm vrndpq: vrnd<IRIntBase<"ceil",  [Vector]>, "p">;
 defm vrndaq: vrnd<IRIntBase<"round", [Vector]>, "a">;
 defm vrndxq: vrnd<IRIntBase<"rint",  [Vector]>, "x">;
-defm vrndnq: vrnd<IRInt<"vrintn",    [Vector]>, "n">;
+defm vrndnq: vrnd<IRIntBase<"roundeven", [Vector]>, "n">;
 
 multiclass compare_with_pred<string condname, dag arguments,
                              dag cmp, string suffix> {
diff --git a/external/llvm-project/clang/include/clang/Basic/riscv_andes_vector.td b/external/llvm-project/clang/include/clang/Basic/riscv_andes_vector.td
index 1498ce2dcdf9..01019cf86d6e 100644
--- a/external/llvm-project/clang/include/clang/Basic/riscv_andes_vector.td
+++ b/external/llvm-project/clang/include/clang/Basic/riscv_andes_vector.td
@@ -30,7 +30,7 @@ multiclass RVVFPMAD {
   }
 }
 
-let RequiredFeatures = ["Xandesvpackfph"],
+let RequiredFeatures = ["xandesvpackfph"],
     UnMaskedPolicyScheme = HasPassthruOperand in {
 let ManualCodegen = [{
   {
@@ -86,7 +86,7 @@ let ManualCodegen = [{
 
 multiclass RVVD4DOT<list<list<string>> i_suffixes_prototypes,
                     list<list<string>> l_suffixes_prototypes> {
-  let RequiredFeatures = ["Xandesvdot"],
+  let RequiredFeatures = ["xandesvdot"],
       UnMaskedPolicyScheme = HasPolicyOperand,
       HasMaskedOffOperand = false,
       Log2LMUL = [-1, 0, 1, 2, 3],
diff --git a/external/llvm-project/clang/include/clang/Basic/riscv_sifive_vector.td b/external/llvm-project/clang/include/clang/Basic/riscv_sifive_vector.td
index f7996f362378..772fd3ef4201 100644
--- a/external/llvm-project/clang/include/clang/Basic/riscv_sifive_vector.td
+++ b/external/llvm-project/clang/include/clang/Basic/riscv_sifive_vector.td
@@ -50,7 +50,7 @@ multiclass RVVVCIXBuiltinSet<list<string> range, string prototype,
                              string suffix = "Uv"> {
   foreach r = range in
     let RequiredFeatures = !if(!and(UseGPR, !eq(r, "l")),
-                               ["Xsfvcp", "RV64"], ["Xsfvcp"]) in
+                               ["xsfvcp", "64bit"], ["xsfvcp"]) in
       defm : VCIXBuiltinSet<NAME, NAME, suffix, prototype, r, intrinsic_types>;
 }
 
@@ -126,7 +126,7 @@ multiclass RVVVFNRCLIPBuiltinSet<string suffix, string prototype, string type_ra
 }
 
 let UnMaskedPolicyScheme = HasPolicyOperand in
-  let RequiredFeatures = ["Xsfvqmaccdod"] in {
+  let RequiredFeatures = ["xsfvqmaccdod"] in {
     defm sf_vqmaccu_2x8x2 : RVVVQMACCDODBuiltinSet<[["", "v", "vv(FixedSEW:8)SUv(FixedSEW:8)Uv"]]>;
     defm sf_vqmacc_2x8x2 : RVVVQMACCDODBuiltinSet<[["", "v", "vv(FixedSEW:8)Sv(FixedSEW:8)v"]]>;
     defm sf_vqmaccus_2x8x2 : RVVVQMACCDODBuiltinSet<[["", "v", "vv(FixedSEW:8)SUv(FixedSEW:8)v"]]>;
@@ -134,7 +134,7 @@ let UnMaskedPolicyScheme = HasPolicyOperand in
   }
 
 let UnMaskedPolicyScheme = HasPolicyOperand in
-  let RequiredFeatures = ["Xsfvqmaccqoq"] in {
+  let RequiredFeatures = ["xsfvqmaccqoq"] in {
     defm sf_vqmaccu_4x8x4 : RVVVQMACCQOQBuiltinSet<[["", "w", "ww(FixedSEW:8)SUv(FixedSEW:8)Uv"]]>;
     defm sf_vqmacc_4x8x4 : RVVVQMACCQOQBuiltinSet<[["", "w", "ww(FixedSEW:8)Sv(FixedSEW:8)v"]]>;
     defm sf_vqmaccus_4x8x4 : RVVVQMACCQOQBuiltinSet<[["", "w", "ww(FixedSEW:8)SUv(FixedSEW:8)v"]]>;
@@ -142,10 +142,10 @@ let UnMaskedPolicyScheme = HasPolicyOperand in
   }
 
 let UnMaskedPolicyScheme = HasPolicyOperand in
-  let RequiredFeatures = ["Xsfvfwmaccqqq"] in
+  let RequiredFeatures = ["xsfvfwmaccqqq"] in
     defm sf_vfwmacc_4x4x4 : RVVVFWMACCBuiltinSet<[["", "Fw", "FwFwSvv"]]>;
 
-let UnMaskedPolicyScheme = HasPassthruOperand, RequiredFeatures = ["Xsfvfnrclipxfqf"] in {
+let UnMaskedPolicyScheme = HasPassthruOperand, RequiredFeatures = ["xsfvfnrclipxfqf"] in {
 let ManualCodegen = [{
   {
     // LLVM intrinsic
diff --git a/external/llvm-project/clang/include/clang/Basic/riscv_vector.td b/external/llvm-project/clang/include/clang/Basic/riscv_vector.td
index bff8699463c4..c6fd8a1a45fd 100644
--- a/external/llvm-project/clang/include/clang/Basic/riscv_vector.td
+++ b/external/llvm-project/clang/include/clang/Basic/riscv_vector.td
@@ -117,8 +117,8 @@ multiclass RVVIndexedLoad<string op> {
         defvar eew = eew_list[0];
         defvar eew_type = eew_list[1];
         let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask",
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin"],
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                   !if(!eq(type, "y"), ["zvfbfmin"],
                                                    []<string>)) in {
           def: RVVOutOp0Op1Builtin<"v", "vPCe" # eew_type # "Uv", type>;
             if !not(IsFloat<type>.val) then {
@@ -129,9 +129,9 @@ multiclass RVVIndexedLoad<string op> {
       defvar eew64 = "64";
       defvar eew64_type = "(Log2EEW:6)";
       let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask",
-          RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"],
-                                 !if(!eq(type, "y"), ["Zvfbfmin", "RV64"],
-                                                 ["RV64"])) in {
+          RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin", "64bit"],
+                                 !if(!eq(type, "y"), ["zvfbfmin", "64bit"],
+                                                 ["64bit"])) in {
           def: RVVOutOp0Op1Builtin<"v", "vPCe" # eew64_type # "Uv", type>;
             if !not(IsFloat<type>.val) then {
               def: RVVOutOp0Op1Builtin<"Uv", "UvPCUe" # eew64_type # "Uv", type>;
@@ -224,8 +224,8 @@ multiclass RVVIndexedStore<string op> {
           defvar eew = eew_list[0];
           defvar eew_type = eew_list[1];
           let Name = op # eew  # "_v", IRName = op, MaskedIRName = op # "_mask",
-          RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                 !if(!eq(type, "y"), ["Zvfbfmin"],
+          RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                 !if(!eq(type, "y"), ["zvfbfmin"],
                                                  []<string>)) in  {
             def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>;
             if !not(IsFloat<type>.val) then {
@@ -236,9 +236,9 @@ multiclass RVVIndexedStore<string op> {
         defvar eew64 = "64";
         defvar eew64_type = "(Log2EEW:6)";
         let Name = op # eew64  # "_v", IRName = op, MaskedIRName = op # "_mask",
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin", "RV64"],
-                                                   ["RV64"])) in  {
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin", "64bit"],
+                                   !if(!eq(type, "y"), ["zvfbfmin", "64bit"],
+                                                   ["64bit"])) in  {
           def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>;
           if !not(IsFloat<type>.val) then {
             def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>;
@@ -362,11 +362,11 @@ multiclass RVVNonTupleVCreateBuiltin<int dst_lmul, list<int> src_lmul_list> {
     def vcreate # src_v # dst_v : RVVBuiltin<src_v # dst_v,
                                              dst_v # src_s,
                                              "csilfd">;
-    let RequiredFeatures = ["Zvfhmin"] in
+    let RequiredFeatures = ["zvfhmin"] in
       def vcreate_h # src_v # dst_v : RVVBuiltin<src_v # dst_v,
                                                  dst_v # src_s,
                                                  "x", dst_v>;
-    let RequiredFeatures = ["Zvfbfmin"] in
+    let RequiredFeatures = ["zvfbfmin"] in
       def vcreate_bf16 # src_v # dst_v : RVVBuiltin<src_v # dst_v,
                                                     dst_v # src_s,
                                                     "y", dst_v>;
@@ -689,9 +689,9 @@ let HasBuiltinAlias = false,
 def vlm: RVVVLEMaskBuiltin;
 defm vle8: RVVVLEBuiltin<["c"]>;
 defm vle16: RVVVLEBuiltin<["s"]>;
-let Name = "vle16_v", RequiredFeatures = ["Zvfhmin"] in
+let Name = "vle16_v", RequiredFeatures = ["zvfhmin"] in
   defm vle16_h: RVVVLEBuiltin<["x"]>;
-let Name = "vle16_v", RequiredFeatures = ["Zvfbfmin"] in
+let Name = "vle16_v", RequiredFeatures = ["zvfbfmin"] in
   defm vle16_bf16 : RVVVLEBuiltin<["y"]>;
 defm vle32: RVVVLEBuiltin<["i","f"]>;
 defm vle64: RVVVLEBuiltin<["l","d"]>;
@@ -699,9 +699,9 @@ defm vle64: RVVVLEBuiltin<["l","d"]>;
 def vsm : RVVVSEMaskBuiltin;
 defm vse8 : RVVVSEBuiltin<["c"]>;
 defm vse16: RVVVSEBuiltin<["s"]>;
-let Name = "vse16_v", RequiredFeatures = ["Zvfhmin"] in
+let Name = "vse16_v", RequiredFeatures = ["zvfhmin"] in
   defm vse16_h: RVVVSEBuiltin<["x"]>;
-let Name = "vse16_v", RequiredFeatures = ["Zvfbfmin"] in
+let Name = "vse16_v", RequiredFeatures = ["zvfbfmin"] in
   defm vse16_bf16: RVVVSEBuiltin<["y"]>;
 defm vse32: RVVVSEBuiltin<["i","f"]>;
 defm vse64: RVVVSEBuiltin<["l","d"]>;
@@ -709,18 +709,18 @@ defm vse64: RVVVSEBuiltin<["l","d"]>;
 // 7.5. Vector Strided Instructions
 defm vlse8: RVVVLSEBuiltin<["c"]>;
 defm vlse16: RVVVLSEBuiltin<["s"]>;
-let Name = "vlse16_v", RequiredFeatures = ["Zvfhmin"] in
+let Name = "vlse16_v", RequiredFeatures = ["zvfhmin"] in
   defm vlse16_h: RVVVLSEBuiltin<["x"]>;
-let Name = "vlse16_v", RequiredFeatures = ["Zvfbfmin"] in
+let Name = "vlse16_v", RequiredFeatures = ["zvfbfmin"] in
   defm vlse16_bf16: RVVVLSEBuiltin<["y"]>;
 defm vlse32: RVVVLSEBuiltin<["i","f"]>;
 defm vlse64: RVVVLSEBuiltin<["l","d"]>;
 
 defm vsse8 : RVVVSSEBuiltin<["c"]>;
 defm vsse16: RVVVSSEBuiltin<["s"]>;
-let Name = "vsse16_v", RequiredFeatures = ["Zvfhmin"] in
+let Name = "vsse16_v", RequiredFeatures = ["zvfhmin"] in
   defm vsse16_h: RVVVSSEBuiltin<["x"]>;
-let Name = "vsse16_v", RequiredFeatures = ["Zvfbfmin"] in
+let Name = "vsse16_v", RequiredFeatures = ["zvfbfmin"] in
   defm vsse16_bf: RVVVSSEBuiltin<["y"]>;
 defm vsse32: RVVVSSEBuiltin<["i","f"]>;
 defm vsse64: RVVVSSEBuiltin<["l","d"]>;
@@ -735,9 +735,9 @@ defm : RVVIndexedStore<"vsoxei">;
 // 7.7. Unit-stride Fault-Only-First Loads
 defm vle8ff: RVVVLEFFBuiltin<["c"]>;
 defm vle16ff: RVVVLEFFBuiltin<["s"]>;
-let Name = "vle16ff_v", RequiredFeatures = ["Zvfhmin"] in
+let Name = "vle16ff_v", RequiredFeatures = ["zvfhmin"] in
   defm vle16ff: RVVVLEFFBuiltin<["x"]>;
-let Name = "vle16ff_v", RequiredFeatures = ["Zvfbfmin"] in
+let Name = "vle16ff_v", RequiredFeatures = ["zvfbfmin"] in
   defm vle16ff: RVVVLEFFBuiltin<["y"]>;
 defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>;
 defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>;
@@ -757,8 +757,8 @@ multiclass RVVUnitStridedSegLoadTuple<string op> {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin"],
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                   !if(!eq(type, "y"), ["zvfbfmin"],
                                                    []<string>)),
             ManualCodegen = [{
     {
@@ -824,8 +824,8 @@ multiclass RVVUnitStridedSegStoreTuple<string op> {
           MaskedIRName = op # nf # "_mask",
           NF = nf,
           HasMaskedOffOperand = false,
-          RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                 !if(!eq(type, "y"), ["Zvfbfmin"],
+          RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                 !if(!eq(type, "y"), ["zvfbfmin"],
                                                  []<string>)),
           ManualCodegen = [{
     {
@@ -878,8 +878,8 @@ multiclass RVVUnitStridedSegLoadFFTuple<string op> {
             IRName = op # nf # "ff",
             MaskedIRName = op # nf # "ff_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin"],
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                   !if(!eq(type, "y"), ["zvfbfmin"],
                                                    []<string>)),
             ManualCodegen = [{
     {
@@ -954,8 +954,8 @@ multiclass RVVStridedSegLoadTuple<string op> {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin"],
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                   !if(!eq(type, "y"), ["zvfbfmin"],
                                                    []<string>)),
             ManualCodegen = [{
     {
@@ -1023,8 +1023,8 @@ multiclass RVVStridedSegStoreTuple<string op> {
             NF = nf,
             HasMaskedOffOperand = false,
             MaskedPolicyScheme = NonePolicy,
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin"],
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                   !if(!eq(type, "y"), ["zvfbfmin"],
                                                    []<string>)),
             ManualCodegen = [{
     {
@@ -1073,8 +1073,8 @@ multiclass RVVIndexedSegLoadTuple<string op> {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin"],
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                   !if(!eq(type, "y"), ["zvfbfmin"],
                                                    []<string>)),
             ManualCodegen = [{
     {
@@ -1142,8 +1142,8 @@ multiclass RVVIndexedSegStoreTuple<string op> {
             NF = nf,
             HasMaskedOffOperand = false,
             MaskedPolicyScheme = NonePolicy,
-            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
-                                   !if(!eq(type, "y"), ["Zvfbfmin"],
+            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
+                                   !if(!eq(type, "y"), ["zvfbfmin"],
                                                    []<string>)),
             ManualCodegen = [{
     {
@@ -1387,10 +1387,10 @@ let HasMasked = false,
                                    [["v", "Uv", "UvUv"]]>;
     defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilfd",
                                    [["v", "v", "vv"]]>;
-    let RequiredFeatures = ["Zvfhmin"] in
+    let RequiredFeatures = ["zvfhmin"] in
       defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "x",
                                     [["v", "v", "vv"]]>;
-    let RequiredFeatures = ["Zvfbfmin"] in
+    let RequiredFeatures = ["zvfbfmin"] in
       defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "y",
                                     [["v", "v", "vv"]]>;
   let SupportOverloading = false in
@@ -1655,7 +1655,7 @@ let ManualCodegen = [{
       defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "f",
                                            [["vv", "w", "wvvu"],
                                             ["vf", "w", "wveu"]]>;
-      let RequiredFeatures = ["Zvfh"] in
+      let RequiredFeatures = ["zvfh"] in
         defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "x",
                                              [["vv", "w", "wvvu"],
                                               ["vf", "w", "wveu"]]>;
@@ -1671,7 +1671,7 @@ let ManualCodegen = [{
     defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "f",
                                          [["vv", "w", "wvv"],
                                           ["vf", "w", "wve"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "x",
                                            [["vv", "w", "wvv"],
                                             ["vf", "w", "wve"]]>;
@@ -1786,7 +1786,7 @@ let ManualCodegen = [{
 
     // Vector BF16 widening multiply-accumulate
     let Log2LMUL = [-2, -1, 0, 1, 2],
-        RequiredFeatures = ["Zvfbfwma"],
+        RequiredFeatures = ["zvfbfwma"],
         HasMaskedOffOperand = false in
     defm vfwmaccbf16 : RVVOutOp1Op2BuiltinSet<"vfwmaccbf16", "y",
                                               [["vv", "Fw", "FwFwvvu"],
@@ -1800,7 +1800,7 @@ let ManualCodegen = [{
 
   // Vector BF16 widening multiply-accumulate
   let Log2LMUL = [-2, -1, 0, 1, 2],
-      RequiredFeatures = ["Zvfbfwma"],
+      RequiredFeatures = ["zvfbfwma"],
       HasMaskedOffOperand = false in
   defm vfwmaccbf16 : RVVOutOp1Op2BuiltinSet<"vfwmaccbf16", "y",
                                             [["vv", "Fw", "FwFwvv"],
@@ -1856,28 +1856,28 @@ let ManualCodegen = [{
   let HasFRMRoundModeOp = 1 in {
     // 13.8. Vector Floating-Point Square-Root Instruction
     defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "fd", [["v", "v", "vvu"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "x", [["v", "v", "vvu"]]>;
 
     // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
     defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "fd", [["v", "v", "vvu"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "x", [["v", "v", "vvu"]]>;
   }
   // 13.8. Vector Floating-Point Square-Root Instruction
   defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "fd", [["v", "v", "vv"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "x", [["v", "v", "vv"]]>;
 
   // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
   defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "fd", [["v", "v", "vv"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "x", [["v", "v", "vv"]]>;
 }
 
 // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
 defm vfrsqrt7 : RVVOutBuiltinSet<"vfrsqrt7", "fd", [["v", "v", "vv"]]>;
-let RequiredFeatures = ["Zvfh"] in
+let RequiredFeatures = ["zvfh"] in
   defm vfrsqrt7 : RVVOutBuiltinSet<"vfrsqrt7", "x", [["v", "v", "vv"]]>;
 
 // 13.11. Vector Floating-Point MIN/MAX Instructions
@@ -1890,10 +1890,10 @@ defm vfsgnjn : RVVFloatingBinBuiltinSet;
 defm vfsgnjx : RVVFloatingBinBuiltinSet;
 }
 defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "fd">;
-let RequiredFeatures = ["Zvfh"] in
+let RequiredFeatures = ["zvfh"] in
   defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "x">;
 defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "fd">;
-let RequiredFeatures = ["Zvfh"] in
+let RequiredFeatures = ["zvfh"] in
   defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "x">;
 
 // 13.13. Vector Floating-Point Compare Instructions
@@ -1910,7 +1910,7 @@ defm vmfge : RVVFloatingMaskOutBuiltinSet;
 // 13.14. Vector Floating-Point Classify Instruction
 let UnMaskedPolicyScheme = HasPassthruOperand in {
 defm vfclass : RVVOp0BuiltinSet<"vfclass", "fd", [["v", "Uv", "Uvv"]]>;
-let RequiredFeatures = ["Zvfh"] in
+let RequiredFeatures = ["zvfh"] in
   defm vfclass : RVVOp0BuiltinSet<"vfclass", "x", [["v", "Uv", "Uvv"]]>;
 }
 
@@ -1927,15 +1927,15 @@ let HasMasked = false,
     }] in {
   defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "fd",
                                     [["vvm", "v", "vvvm"]]>;
-  let RequiredFeatures = ["Zvfhmin"] in
+  let RequiredFeatures = ["zvfhmin"] in
     defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "x",
                                       [["vvm", "v", "vvvm"]]>;
-  let RequiredFeatures = ["Zvfbfmin"] in
+  let RequiredFeatures = ["zvfbfmin"] in
     defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "y",
                                       [["vvm", "v", "vvvm"]]>;
   defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "fd",
                                      [["vfm", "v", "vvem"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "x",
                                        [["vfm", "v", "vvem"]]>;
 }
@@ -1948,7 +1948,7 @@ let HasMasked = false,
     OverloadedName = "vfmv_v" in {
   defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "fd",
                                  [["f", "v", "ve"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "x",
                                    [["f", "v", "ve"]]>;
 }
@@ -1957,12 +1957,12 @@ let HasMasked = false,
 let UnMaskedPolicyScheme = HasPassthruOperand in {
 let OverloadedName = "vfcvt_rtz_xu" in {
   defm : RVVConvBuiltinSet<"vfcvt_rtz_xu_f_v", "fd", [["Uv", "Uvv"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm : RVVConvBuiltinSet<"vfcvt_rtz_xu_f_v", "x", [["Uv", "Uvv"]]>;
 }
 let OverloadedName = "vfcvt_rtz_x" in {
   defm : RVVConvBuiltinSet<"vfcvt_rtz_x_f_v", "fd", [["Iv", "Ivv"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm : RVVConvBuiltinSet<"vfcvt_rtz_x_f_v", "x", [["Iv", "Ivv"]]>;
 }
 
@@ -1970,25 +1970,25 @@ let OverloadedName = "vfcvt_rtz_x" in {
 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
   let OverloadedName = "vfwcvt_rtz_xu" in {
     defm : RVVConvBuiltinSet<"vfwcvt_rtz_xu_f_v", "f", [["Uw", "Uwv"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm : RVVConvBuiltinSet<"vfwcvt_rtz_xu_f_v", "x", [["Uw", "Uwv"]]>;
   }
   let OverloadedName = "vfwcvt_rtz_x" in {
     defm : RVVConvBuiltinSet<"vfwcvt_rtz_x_f_v", "f", [["Iw", "Iwv"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm : RVVConvBuiltinSet<"vfwcvt_rtz_x_f_v", "x", [["Iw", "Iwv"]]>;
   }
   let OverloadedName = "vfwcvt_f" in {
     defm : RVVConvBuiltinSet<"vfwcvt_f_xu_v", "si", [["Fw", "FwUv"]]>;
     defm : RVVConvBuiltinSet<"vfwcvt_f_x_v", "si", [["Fw", "Fwv"]]>;
-    let RequiredFeatures = ["Zvfh"] in {
+    let RequiredFeatures = ["zvfh"] in {
       defm : RVVConvBuiltinSet<"vfwcvt_f_xu_v", "c", [["Fw", "FwUv"]]>;
       defm : RVVConvBuiltinSet<"vfwcvt_f_x_v", "c", [["Fw", "Fwv"]]>;
     }
   }
   let OverloadedName = "vfwcvt_f" in {
     defm : RVVConvBuiltinSet<"vfwcvt_f_f_v", "f", [["w", "wv"]]>;
-    let RequiredFeatures = ["Zvfhmin"] in
+    let RequiredFeatures = ["zvfhmin"] in
       defm : RVVConvBuiltinSet<"vfwcvt_f_f_v", "x", [["w", "wv"]]>;
   }
 }
@@ -1997,17 +1997,17 @@ let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
 let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
   let OverloadedName = "vfncvt_rtz_xu" in {
     defm : RVVConvBuiltinSet<"vfncvt_rtz_xu_f_w", "si", [["Uv", "UvFw"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm : RVVConvBuiltinSet<"vfncvt_rtz_xu_f_w", "c", [["Uv", "UvFw"]]>;
   }
   let OverloadedName = "vfncvt_rtz_x" in {
     defm : RVVConvBuiltinSet<"vfncvt_rtz_x_f_w", "si", [["Iv", "IvFw"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm : RVVConvBuiltinSet<"vfncvt_rtz_x_f_w", "c", [["Iv", "IvFw"]]>;
   }
   let OverloadedName = "vfncvt_rod_f" in {
     defm : RVVConvBuiltinSet<"vfncvt_rod_f_f_w", "f", [["v", "vw"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm : RVVConvBuiltinSet<"vfncvt_rod_f_f_w", "x", [["v", "vw"]]>;
   }
 }
@@ -2063,18 +2063,18 @@ let ManualCodegen = [{
     // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions
     let OverloadedName = "vfcvt_x" in {
       defm : RVVConvBuiltinSet<"vfcvt_x_f_v", "fd", [["Iv", "Ivvu"]]>;
-      let RequiredFeatures = ["Zvfh"] in
+      let RequiredFeatures = ["zvfh"] in
         defm : RVVConvBuiltinSet<"vfcvt_x_f_v", "x", [["Iv", "Ivvu"]]>;
     }
     let OverloadedName = "vfcvt_xu" in {
       defm : RVVConvBuiltinSet<"vfcvt_xu_f_v", "fd", [["Uv", "Uvvu"]]>;
-      let RequiredFeatures = ["Zvfh"] in
+      let RequiredFeatures = ["zvfh"] in
         defm : RVVConvBuiltinSet<"vfcvt_xu_f_v", "x", [["Uv", "Uvvu"]]>;
     }
     let OverloadedName = "vfcvt_f" in {
       defm : RVVConvBuiltinSet<"vfcvt_f_x_v", "fd", [["v", "vIvu"]]>;
       defm : RVVConvBuiltinSet<"vfcvt_f_xu_v", "fd", [["v", "vUvu"]]>;
-      let RequiredFeatures = ["Zvfh"] in {
+      let RequiredFeatures = ["zvfh"] in {
         defm : RVVConvBuiltinSet<"vfcvt_f_x_v", "x", [["v", "vIvu"]]>;
         defm : RVVConvBuiltinSet<"vfcvt_f_xu_v", "x", [["v", "vUvu"]]>;
       }
@@ -2084,12 +2084,12 @@ let ManualCodegen = [{
     let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
       let OverloadedName = "vfwcvt_x" in {
         defm : RVVConvBuiltinSet<"vfwcvt_x_f_v", "f", [["Iw", "Iwvu"]]>;
-        let RequiredFeatures = ["Zvfh"] in
+        let RequiredFeatures = ["zvfh"] in
           defm : RVVConvBuiltinSet<"vfwcvt_x_f_v", "x", [["Iw", "Iwvu"]]>;
       }
       let OverloadedName = "vfwcvt_xu" in {
         defm : RVVConvBuiltinSet<"vfwcvt_xu_f_v", "f", [["Uw", "Uwvu"]]>;
-        let RequiredFeatures = ["Zvfh"] in
+        let RequiredFeatures = ["zvfh"] in
           defm : RVVConvBuiltinSet<"vfwcvt_xu_f_v", "x", [["Uw", "Uwvu"]]>;
       }
     }
@@ -2097,25 +2097,25 @@ let ManualCodegen = [{
     let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
       let OverloadedName = "vfncvt_x" in {
         defm : RVVConvBuiltinSet<"vfncvt_x_f_w", "si", [["Iv", "IvFwu"]]>;
-        let RequiredFeatures = ["Zvfh"] in
+        let RequiredFeatures = ["zvfh"] in
           defm : RVVConvBuiltinSet<"vfncvt_x_f_w", "c", [["Iv", "IvFwu"]]>;
       }
       let OverloadedName = "vfncvt_xu" in {
         defm : RVVConvBuiltinSet<"vfncvt_xu_f_w", "si", [["Uv", "UvFwu"]]>;
-        let RequiredFeatures = ["Zvfh"] in
+        let RequiredFeatures = ["zvfh"] in
           defm : RVVConvBuiltinSet<"vfncvt_xu_f_w", "c", [["Uv", "UvFwu"]]>;
       }
       let OverloadedName = "vfncvt_f" in {
         defm : RVVConvBuiltinSet<"vfncvt_f_x_w", "f", [["v", "vIwu"]]>;
         defm : RVVConvBuiltinSet<"vfncvt_f_xu_w", "f", [["v", "vUwu"]]>;
-        let RequiredFeatures = ["Zvfh"] in {
+        let RequiredFeatures = ["zvfh"] in {
           defm : RVVConvBuiltinSet<"vfncvt_f_x_w", "x", [["v", "vIwu"]]>;
           defm : RVVConvBuiltinSet<"vfncvt_f_xu_w", "x", [["v", "vUwu"]]>;
         }
       }
       let OverloadedName = "vfncvt_f" in {
         defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vwu"]]>;
-        let RequiredFeatures = ["Zvfhmin"] in
+        let RequiredFeatures = ["zvfhmin"] in
           defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vwu"]]>;
       }
     }
@@ -2129,18 +2129,18 @@ let ManualCodegen = [{
   // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions
   let OverloadedName = "vfcvt_x" in {
     defm : RVVConvBuiltinSet<"vfcvt_x_f_v", "fd", [["Iv", "Ivv"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm : RVVConvBuiltinSet<"vfcvt_x_f_v", "x", [["Iv", "Ivv"]]>;
   }
   let OverloadedName = "vfcvt_xu" in {
     defm : RVVConvBuiltinSet<"vfcvt_xu_f_v", "fd", [["Uv", "Uvv"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm : RVVConvBuiltinSet<"vfcvt_xu_f_v", "x", [["Uv", "Uvv"]]>;
   }
   let OverloadedName = "vfcvt_f" in {
     defm : RVVConvBuiltinSet<"vfcvt_f_x_v", "fd", [["v", "vIv"]]>;
     defm : RVVConvBuiltinSet<"vfcvt_f_xu_v", "fd", [["v", "vUv"]]>;
-    let RequiredFeatures = ["Zvfh"] in {
+    let RequiredFeatures = ["zvfh"] in {
       defm : RVVConvBuiltinSet<"vfcvt_f_x_v", "x", [["v", "vIv"]]>;
       defm : RVVConvBuiltinSet<"vfcvt_f_xu_v", "x", [["v", "vUv"]]>;
     }
@@ -2150,12 +2150,12 @@ let ManualCodegen = [{
   let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
     let OverloadedName = "vfwcvt_x" in {
       defm : RVVConvBuiltinSet<"vfwcvt_x_f_v", "f", [["Iw", "Iwv"]]>;
-      let RequiredFeatures = ["Zvfh"] in
+      let RequiredFeatures = ["zvfh"] in
         defm : RVVConvBuiltinSet<"vfwcvt_x_f_v", "x", [["Iw", "Iwv"]]>;
     }
     let OverloadedName = "vfwcvt_xu" in {
       defm : RVVConvBuiltinSet<"vfwcvt_xu_f_v", "f", [["Uw", "Uwv"]]>;
-      let RequiredFeatures = ["Zvfh"] in
+      let RequiredFeatures = ["zvfh"] in
         defm : RVVConvBuiltinSet<"vfwcvt_xu_f_v", "x", [["Uw", "Uwv"]]>;
     }
   }
@@ -2163,25 +2163,25 @@ let ManualCodegen = [{
   let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
     let OverloadedName = "vfncvt_x" in {
       defm : RVVConvBuiltinSet<"vfncvt_x_f_w", "si", [["Iv", "IvFw"]]>;
-      let RequiredFeatures = ["Zvfh"] in
+      let RequiredFeatures = ["zvfh"] in
         defm : RVVConvBuiltinSet<"vfncvt_x_f_w", "c", [["Iv", "IvFw"]]>;
     }
     let OverloadedName = "vfncvt_xu" in {
       defm : RVVConvBuiltinSet<"vfncvt_xu_f_w", "si", [["Uv", "UvFw"]]>;
-      let RequiredFeatures = ["Zvfh"] in
+      let RequiredFeatures = ["zvfh"] in
         defm : RVVConvBuiltinSet<"vfncvt_xu_f_w", "c", [["Uv", "UvFw"]]>;
     }
     let OverloadedName = "vfncvt_f" in {
       defm : RVVConvBuiltinSet<"vfncvt_f_x_w", "f", [["v", "vIw"]]>;
       defm : RVVConvBuiltinSet<"vfncvt_f_xu_w", "f", [["v", "vUw"]]>;
-      let RequiredFeatures = ["Zvfh"] in {
+      let RequiredFeatures = ["zvfh"] in {
         defm : RVVConvBuiltinSet<"vfncvt_f_x_w", "x", [["v", "vIw"]]>;
         defm : RVVConvBuiltinSet<"vfncvt_f_xu_w", "x", [["v", "vUw"]]>;
       }
     }
     let OverloadedName = "vfncvt_f" in {
       defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vw"]]>;
-      let RequiredFeatures = ["Zvfhmin"] in
+      let RequiredFeatures = ["zvfhmin"] in
         defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vw"]]>;
     }
   }
@@ -2345,7 +2345,7 @@ let HasMasked = false, MaskedPolicyScheme = NonePolicy in {
   let HasVL = false, OverloadedName = "vfmv_f" in {
     defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "fd",
                                    [["s", "ve", "ev"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "x",
                                      [["s", "ve", "ev"]]>;
   }
@@ -2355,7 +2355,7 @@ let HasMasked = false, MaskedPolicyScheme = NonePolicy in {
     defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "fd",
                                    [["f", "v", "ve"],
                                     ["x", "Uv", "UvUe"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "x",
                                      [["f", "v", "ve"],
                                       ["x", "Uv", "UvUe"]]>;
@@ -2383,13 +2383,13 @@ defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilfd",
                                  [["vv", "v", "vvUv"]]>;
 defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilfd",
                                  [["vx", "v", "vvz"]]>;
-let RequiredFeatures = ["Zvfhmin"] in {
+let RequiredFeatures = ["zvfhmin"] in {
   defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "x",
                                    [["vv", "v", "vvUv"]]>;
   defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "x",
                                    [["vx", "v", "vvz"]]>;
 }
-let RequiredFeatures = ["Zvfbfmin"] in {
+let RequiredFeatures = ["zvfbfmin"] in {
   defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "y",
                                    [["vv", "v", "vvUv"]]>;
   defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "y",
@@ -2397,7 +2397,7 @@ let RequiredFeatures = ["Zvfbfmin"] in {
 }
 defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilfd",
                                      [["vv", "v", "vv(Log2EEW:4)Uv"]]>;
-let RequiredFeatures = ["Zvfh"] in
+let RequiredFeatures = ["zvfhmin"] in
 defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "x",
                                      [["vv", "v", "vv(Log2EEW:4)Uv"]]>;
 // unsigned type
@@ -2422,10 +2422,10 @@ let HasMasked = false,
   // signed and floating type
   defm vcompress : RVVOutBuiltinSet<"vcompress", "csilfd",
                                     [["vm", "v", "vvm"]]>;
-  let RequiredFeatures = ["Zvfhmin"] in
+  let RequiredFeatures = ["zvfhmin"] in
     defm vcompress : RVVOutBuiltinSet<"vcompress", "x",
                                       [["vm", "v", "vvm"]]>;
-  let RequiredFeatures = ["Zvfbfmin"] in
+  let RequiredFeatures = ["zvfbfmin"] in
     defm vcompress : RVVOutBuiltinSet<"vcompress", "y",
                                       [["vm", "v", "vvm"]]>;
   // unsigned type
@@ -2476,13 +2476,13 @@ let HasMasked = false, HasVL = false, IRName = "" in {
     def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "il", "Uv">;
     def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "il", "Fv">;
     def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "il", "Fv">;
-    let RequiredFeatures = ["Zvfhmin"] in {
+    let RequiredFeatures = ["zvfhmin"] in {
       def vreinterpret_i_h : RVVBuiltin<"Fvv", "vFv", "s", "v">;
       def vreinterpret_u_h : RVVBuiltin<"FvUv", "UvFv", "s", "Uv">;
       def vreinterpret_h_i : RVVBuiltin<"vFv", "Fvv", "s", "Fv">;
       def vreinterpret_h_u : RVVBuiltin<"UvFv", "FvUv", "s", "Fv">;
     }
-    let RequiredFeatures = ["Zvfbfmin"] in {
+    let RequiredFeatures = ["zvfbfmin"] in {
       def vreinterpret_i_bf16 : RVVBuiltin<"vIv", "Ivv", "y", "Iv">;
       def vreinterpret_u_bf16 : RVVBuiltin<"vUv", "Uvv", "y", "Uv">;
       def vreinterpret_bf16_i : RVVBuiltin<"Ivv", "vIv", "y", "v">;
@@ -2552,9 +2552,9 @@ let HasMasked = false, HasVL = false, IRName = "" in {
         return llvm::PoisonValue::get(ResultType);
       }] in {
     def vundefined : RVVBuiltin<"v", "v", "csilfd">;
-    let RequiredFeatures = ["Zvfhmin"] in
+    let RequiredFeatures = ["zvfhmin"] in
       def vundefined_h : RVVBuiltin<"v", "v", "x">;
-    let RequiredFeatures = ["Zvfbfmin"] in
+    let RequiredFeatures = ["zvfbfmin"] in
       def vundefined_bf16 : RVVBuiltin<"v", "v", "y">;
     def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">;
 
@@ -2562,9 +2562,9 @@ let HasMasked = false, HasVL = false, IRName = "" in {
       let NF = nf in {
         defvar T = "(Tuple:" # nf # ")";
         def : RVVBuiltin<T # "v", T # "v", "csilfd">;
-        let RequiredFeatures = ["Zvfhmin"] in
+        let RequiredFeatures = ["zvfhmin"] in
           def : RVVBuiltin<T # "v", T # "v", "x">;
-        let RequiredFeatures = ["Zvfbfmin"] in
+        let RequiredFeatures = ["zvfbfmin"] in
           def : RVVBuiltin<T # "v", T # "v", "y">;
         def : RVVBuiltin<T # "Uv", T # "Uv", "csil">;
       }
@@ -2584,10 +2584,10 @@ let HasMasked = false, HasVL = false, IRName = "" in {
                         "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in {
       def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
                                               dst_lmul # "vv", "csilfd", dst_lmul # "v">;
-      let RequiredFeatures = ["Zvfhmin"] in
+      let RequiredFeatures = ["zvfhmin"] in
         def vlmul_trunc_h # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
                                                   dst_lmul # "vv", "x", dst_lmul # "v">;
-      let RequiredFeatures = ["Zvfbfmin"] in
+      let RequiredFeatures = ["zvfbfmin"] in
         def vlmul_trunc_bf16 # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
                                                      dst_lmul # "vv", "y", dst_lmul # "v">;
       def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv",
@@ -2608,10 +2608,10 @@ let HasMasked = false, HasVL = false, IRName = "" in {
                         "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in {
       def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
                                             dst_lmul # "vv", "csilfd", dst_lmul # "v">;
-      let RequiredFeatures = ["Zvfhmin"] in
+      let RequiredFeatures = ["zvfhmin"] in
         def vlmul_ext_h # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
                                                 dst_lmul # "vv", "x", dst_lmul # "v">;
-      let RequiredFeatures = ["Zvfbfmin"] in
+      let RequiredFeatures = ["zvfbfmin"] in
         def vlmul_ext_bf16 # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
                                                    dst_lmul # "vv", "y", dst_lmul # "v">;
       def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv",
@@ -2643,18 +2643,18 @@ let HasMasked = false, HasVL = false, IRName = "" in {
       }] in {
     foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in {
       def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilfd", dst_lmul # "v">;
-      let RequiredFeatures = ["Zvfhmin"] in
+      let RequiredFeatures = ["zvfhmin"] in
         def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "x", dst_lmul # "v">;
-      let RequiredFeatures = ["Zvfbfmin"] in
+      let RequiredFeatures = ["zvfbfmin"] in
         def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "y", dst_lmul # "v">;
       def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">;
     }
     foreach nf = NFList in {
       defvar T = "(Tuple:" # nf # ")";
       def : RVVBuiltin<T # "vv", "v" # T # "vKz", "csilfd", "v">;
-      let RequiredFeatures = ["Zvfhmin"] in
+      let RequiredFeatures = ["zvfhmin"] in
         def : RVVBuiltin<T # "vv", "v" # T # "vKz", "x", "v">;
-      let RequiredFeatures = ["Zvfbfmin"] in
+      let RequiredFeatures = ["zvfbfmin"] in
         def : RVVBuiltin<T # "vv", "v" # T # "vKz", "y", "v">;
       def : RVVBuiltin<T # "UvUv", "Uv" # T # "UvKz", "csil", "Uv">;
     }
@@ -2684,18 +2684,18 @@ let HasMasked = false, HasVL = false, IRName = "" in {
       }] in {
     foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in {
       def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilfd">;
-      let RequiredFeatures = ["Zvfhmin"] in
+      let RequiredFeatures = ["zvfhmin"] in
         def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "x">;
-      let RequiredFeatures = ["Zvfbfmin"] in
+      let RequiredFeatures = ["zvfbfmin"] in
         def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "y">;
       def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">;
     }
     foreach nf = NFList in {
       defvar T = "(Tuple:" # nf # ")";
       def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "csilfd">;
-      let RequiredFeatures = ["Zvfhmin"] in
+      let RequiredFeatures = ["zvfhmin"] in
         def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "x">;
-      let RequiredFeatures = ["Zvfbfmin"] in
+      let RequiredFeatures = ["zvfbfmin"] in
         def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "y">;
       def : RVVBuiltin<"Uv" # T # "Uv", T # "Uv" # T # "UvKzUv", "csil">;
     }
@@ -2742,9 +2742,9 @@ let HasMasked = false, HasVL = false, IRName = "" in {
         defvar V = VString<nf, /*signed=*/true>.S;
         defvar UV = VString<nf, /*signed=*/false>.S;
         def : RVVBuiltin<T # "v", T # "v" # V, "csilfd">;
-        let RequiredFeatures = ["Zvfhmin"] in
+        let RequiredFeatures = ["zvfhmin"] in
           def : RVVBuiltin<T # "v", T # "v" # V, "x">;
-        let RequiredFeatures = ["Zvfbfmin"] in
+        let RequiredFeatures = ["zvfbfmin"] in
           def : RVVBuiltin<T # "v", T # "v" # V, "y">;
         def : RVVBuiltin<T # "Uv", T # "Uv" # UV, "csil">;
       }
@@ -2794,7 +2794,7 @@ multiclass RVVSignedWidenBinBuiltinSetVwsll
 
 let UnMaskedPolicyScheme = HasPassthruOperand in {
   // zvkb
-  let RequiredFeatures = ["Zvkb"] in {
+  let RequiredFeatures = ["zvkb"] in {
     defm vandn   : RVVUnsignedBinBuiltinSet;
     defm vbrev8  : RVVOutBuiltinSetZvbb;
     defm vrev8   : RVVOutBuiltinSetZvbb;
@@ -2803,7 +2803,7 @@ let UnMaskedPolicyScheme = HasPassthruOperand in {
   }
 
   // zvbb
-  let RequiredFeatures = ["Zvbb"] in {
+  let RequiredFeatures = ["zvbb"] in {
     defm vbrev   : RVVOutBuiltinSetZvbb;
     defm vclz    : RVVOutBuiltinSetZvbb;
     defm vctz    : RVVOutBuiltinSetZvbb;
@@ -2814,7 +2814,7 @@ let UnMaskedPolicyScheme = HasPassthruOperand in {
   }
 
   // zvbc
-  let RequiredFeatures = ["Zvbc"] in {
+  let RequiredFeatures = ["zvbc"] in {
     defm vclmul  : RVVInt64BinBuiltinSet;
     defm vclmulh : RVVInt64BinBuiltinSet;
   }
@@ -2822,13 +2822,13 @@ let UnMaskedPolicyScheme = HasPassthruOperand in {
 
 let UnMaskedPolicyScheme = HasPolicyOperand, HasMasked = false in {
   // zvkg
-  let RequiredFeatures = ["Zvkg"] in {
+  let RequiredFeatures = ["zvkg"] in {
     defm vghsh   : RVVOutOp2BuiltinSetVVZvk;
     defm vgmul   : RVVOutBuiltinSetZvk<HasVV=1, HasVS=0>;
   }
 
   // zvkned
-  let RequiredFeatures = ["Zvkned"] in {
+  let RequiredFeatures = ["zvkned"] in {
     defm vaesdf  : RVVOutBuiltinSetZvk;
     defm vaesdm  : RVVOutBuiltinSetZvk;
     defm vaesef  : RVVOutBuiltinSetZvk;
@@ -2839,29 +2839,21 @@ let UnMaskedPolicyScheme = HasPolicyOperand, HasMasked = false in {
     defm vaesz   : RVVOutBuiltinSetZvk<HasVV=0>;
   }
 
-  // zvknha
-  let RequiredFeatures = ["Zvknha"] in {
-    defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"i">;
-    defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"i">;
-    defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"i">;
-  }
-
-  // zvknhb
-  let RequiredFeatures = ["Zvknhb"] in {
-    defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"il">;
-    defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"il">;
-    defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"il">;
-  }
+  // zvknha and zvknhb has duplicated intrinsic but they don't imply each other,
+  // so we need to handle it manually in SemaRISCV.cpp.
+  defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"il">;
+  defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"il">;
+  defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"il">;
 
   // zvksed
-  let RequiredFeatures = ["Zvksed"] in {
+  let RequiredFeatures = ["zvksed"] in {
     let UnMaskedPolicyScheme = HasPassthruOperand in
     defm vsm4k   : RVVOutOp1BuiltinSet<"vsm4k", "i", [["vi", "Uv", "UvUvKz"]]>;
     defm vsm4r   : RVVOutBuiltinSetZvk;
   }
 
   // zvksh
-  let RequiredFeatures = ["Zvksh"] in {
+  let RequiredFeatures = ["zvksh"] in {
     defm vsm3c   : RVVOutOp2BuiltinSetVIZvk;
     let UnMaskedPolicyScheme = HasPassthruOperand in
     defm vsm3me  : RVVOutOp1BuiltinSet<"vsm3me", "i", [["vv", "Uv", "UvUvUv"]]>;
diff --git a/external/llvm-project/clang/include/clang/Basic/riscv_vector_common.td b/external/llvm-project/clang/include/clang/Basic/riscv_vector_common.td
index 5a81376208f7..e3d589699645 100644
--- a/external/llvm-project/clang/include/clang/Basic/riscv_vector_common.td
+++ b/external/llvm-project/clang/include/clang/Basic/riscv_vector_common.td
@@ -466,7 +466,7 @@ let HasMaskedOffOperand = false in {
     defm "" : RVVOutOp1BuiltinSet<NAME, "fd",
                                   [["vv", "v", "vvvv"],
                                    ["vf", "v", "vvev"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm "" : RVVOutOp1BuiltinSet<NAME, "x",
                                     [["vv", "v", "vvvv"],
                                      ["vf", "v", "vvev"]]>;
@@ -475,7 +475,7 @@ let HasMaskedOffOperand = false in {
     defm "" : RVVOutOp1BuiltinSet<NAME, "fd",
                                   [["vv", "v", "vvvvu"],
                                    ["vf", "v", "vvevu"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm "" : RVVOutOp1BuiltinSet<NAME, "x",
                                     [["vv", "v", "vvvvu"],
                                      ["vf", "v", "vvevu"]]>;
@@ -487,7 +487,7 @@ let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in {
     defm ""  : RVVOutOp1Op2BuiltinSet<NAME, "f",
                                       [["vv", "w", "wwvv"],
                                        ["vf", "w", "wwev"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm ""  : RVVOutOp1Op2BuiltinSet<NAME, "x",
                                         [["vv", "w", "wwvv"],
                                          ["vf", "w", "wwev"]]>;
@@ -496,7 +496,7 @@ let HasMaskedOffOperand = false, Log2LMUL = [-2, -1, 0, 1, 2] in {
     defm ""  : RVVOutOp1Op2BuiltinSet<NAME, "f",
                                       [["vv", "w", "wwvvu"],
                                        ["vf", "w", "wwevu"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm ""  : RVVOutOp1Op2BuiltinSet<NAME, "x",
                                         [["vv", "w", "wwvvu"],
                                          ["vf", "w", "wwevu"]]>;
@@ -507,7 +507,7 @@ multiclass RVVFloatingBinBuiltinSet {
   defm "" : RVVOutOp1BuiltinSet<NAME, "fd",
                                 [["vv", "v", "vvv"],
                                  ["vf", "v", "vve"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVOutOp1BuiltinSet<NAME, "x",
                                   [["vv", "v", "vvv"],
                                    ["vf", "v", "vve"]]>;
@@ -517,7 +517,7 @@ multiclass RVVFloatingBinBuiltinSetRoundingMode {
   defm "" : RVVOutOp1BuiltinSet<NAME, "fd",
                                 [["vv", "v", "vvvu"],
                                  ["vf", "v", "vveu"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVOutOp1BuiltinSet<NAME, "x",
                                   [["vv", "v", "vvvu"],
                                    ["vf", "v", "vveu"]]>;
@@ -526,7 +526,7 @@ multiclass RVVFloatingBinBuiltinSetRoundingMode {
 multiclass RVVFloatingBinVFBuiltinSet {
   defm "" : RVVOutOp1BuiltinSet<NAME, "fd",
                                 [["vf", "v", "vve"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVOutOp1BuiltinSet<NAME, "x",
                                   [["vf", "v", "vve"]]>;
 }
@@ -534,7 +534,7 @@ multiclass RVVFloatingBinVFBuiltinSet {
 multiclass RVVFloatingBinVFBuiltinSetRoundingMode {
   defm "" : RVVOutOp1BuiltinSet<NAME, "fd",
                                 [["vf", "v", "vveu"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVOutOp1BuiltinSet<NAME, "x",
                                   [["vf", "v", "vveu"]]>;
 }
@@ -543,7 +543,7 @@ multiclass RVVFloatingMaskOutBuiltinSet {
   defm "" : RVVOp0Op1BuiltinSet<NAME, "fd",
                                 [["vv", "vm", "mvv"],
                                  ["vf", "vm", "mve"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVOp0Op1BuiltinSet<NAME, "x",
                                   [["vv", "vm", "mvv"],
                                    ["vf", "vm", "mve"]]>;
@@ -593,7 +593,7 @@ let UnMaskedPolicyScheme = HasPolicyOperand,
   multiclass RVVSlideUpBuiltinSet {
     defm "" : RVVOutBuiltinSet<NAME, "csilfd",
                                [["vx","v", "vvvz"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfhmin"] in
       defm "" : RVVOutBuiltinSet<NAME, "x",
                                  [["vx","v", "vvvz"]]>;
     defm "" : RVVOutBuiltinSet<NAME, "csil",
@@ -618,7 +618,7 @@ let UnMaskedPolicyScheme = HasPassthruOperand,
   multiclass RVVSlideDownBuiltinSet {
     defm "" : RVVOutBuiltinSet<NAME, "csilfd",
                                [["vx","v", "vvz"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfhmin"] in
       defm "" : RVVOutBuiltinSet<NAME, "x",
                                  [["vx","v", "vvz"]]>;
     defm "" : RVVOutBuiltinSet<NAME, "csil",
@@ -663,28 +663,28 @@ let HasMaskedOffOperand = true in {
   multiclass RVVFloatingReductionBuiltin {
     defm "" : RVVOutOp0BuiltinSet<NAME, "fd",
                                   [["vs", "vSv", "SvvSv"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm "" : RVVOutOp0BuiltinSet<NAME, "x",
                                     [["vs", "vSv", "SvvSv"]]>;
   }
   multiclass RVVFloatingReductionBuiltinRoundingMode {
     defm "" : RVVOutOp0BuiltinSet<NAME, "fd",
                                   [["vs", "vSv", "SvvSvu"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm "" : RVVOutOp0BuiltinSet<NAME, "x",
                                     [["vs", "vSv", "SvvSvu"]]>;
   }
   multiclass RVVFloatingWidenReductionBuiltin {
     defm "" : RVVOutOp0BuiltinSet<NAME, "f",
                                   [["vs", "vSw", "SwvSw"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm "" : RVVOutOp0BuiltinSet<NAME, "x",
                                     [["vs", "vSw", "SwvSw"]]>;
   }
   multiclass RVVFloatingWidenReductionBuiltinRoundingMode {
     defm "" : RVVOutOp0BuiltinSet<NAME, "f",
                                   [["vs", "vSw", "SwvSwu"]]>;
-    let RequiredFeatures = ["Zvfh"] in
+    let RequiredFeatures = ["zvfh"] in
       defm "" : RVVOutOp0BuiltinSet<NAME, "x",
                                     [["vs", "vSw", "SwvSwu"]]>;
   }
@@ -750,7 +750,7 @@ multiclass RVVFloatingWidenBinBuiltinSet {
   defm "" : RVVWidenBuiltinSet<NAME, "f",
                                [["vv", "w", "wvv"],
                                 ["vf", "w", "wve"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVWidenBuiltinSet<NAME, "x",
                                  [["vv", "w", "wvv"],
                                   ["vf", "w", "wve"]]>;
@@ -760,7 +760,7 @@ multiclass RVVFloatingWidenBinBuiltinSetRoundingMode {
   defm "" : RVVWidenBuiltinSet<NAME, "f",
                                [["vv", "w", "wvvu"],
                                 ["vf", "w", "wveu"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVWidenBuiltinSet<NAME, "x",
                                  [["vv", "w", "wvvu"],
                                   ["vf", "w", "wveu"]]>;
@@ -770,7 +770,7 @@ multiclass RVVFloatingWidenOp0BinBuiltinSet {
   defm "" : RVVWidenWOp0BuiltinSet<NAME # "_w", "f",
                                    [["wv", "w", "wwv"],
                                     ["wf", "w", "wwe"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVWidenWOp0BuiltinSet<NAME # "_w", "x",
                                      [["wv", "w", "wwv"],
                                       ["wf", "w", "wwe"]]>;
@@ -780,7 +780,7 @@ multiclass RVVFloatingWidenOp0BinBuiltinSetRoundingMode {
   defm "" : RVVWidenWOp0BuiltinSet<NAME # "_w", "f",
                                    [["wv", "w", "wwvu"],
                                     ["wf", "w", "wweu"]]>;
-  let RequiredFeatures = ["Zvfh"] in
+  let RequiredFeatures = ["zvfh"] in
     defm "" : RVVWidenWOp0BuiltinSet<NAME # "_w", "x",
                                      [["wv", "w", "wwvu"],
                                       ["wf", "w", "wweu"]]>;
diff --git a/external/llvm-project/clang/include/clang/CIR/CIRGenerator.h b/external/llvm-project/clang/include/clang/CIR/CIRGenerator.h
index bb20fdf72693..dd48eec238fc 100644
--- a/external/llvm-project/clang/include/clang/CIR/CIRGenerator.h
+++ b/external/llvm-project/clang/include/clang/CIR/CIRGenerator.h
@@ -43,10 +43,32 @@ class CIRGenerator : public clang::ASTConsumer {
 
   const clang::CodeGenOptions &codeGenOpts;
 
+  unsigned handlingTopLevelDecls;
+
+  /// Use this when emitting decls to block re-entrant decl emission. It will
+  /// emit all deferred decls on scope exit. Set EmitDeferred to false if decl
+  /// emission must be deferred longer, like at the end of a tag definition.
+  struct HandlingTopLevelDeclRAII {
+    CIRGenerator &self;
+    bool emitDeferred;
+    HandlingTopLevelDeclRAII(CIRGenerator &self, bool emitDeferred = true)
+        : self{self}, emitDeferred{emitDeferred} {
+      ++self.handlingTopLevelDecls;
+    }
+    ~HandlingTopLevelDeclRAII() {
+      unsigned Level = --self.handlingTopLevelDecls;
+      if (Level == 0 && emitDeferred)
+        self.emitDeferredDecls();
+    }
+  };
+
 protected:
   std::unique_ptr<mlir::MLIRContext> mlirContext;
   std::unique_ptr<clang::CIRGen::CIRGenModule> cgm;
 
+private:
+  llvm::SmallVector<clang::FunctionDecl *, 8> deferredInlineMemberFuncDefs;
+
 public:
   CIRGenerator(clang::DiagnosticsEngine &diags,
                llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
@@ -54,6 +76,9 @@ class CIRGenerator : public clang::ASTConsumer {
   ~CIRGenerator() override;
   void Initialize(clang::ASTContext &astContext) override;
   bool HandleTopLevelDecl(clang::DeclGroupRef group) override;
+  void HandleTranslationUnit(clang::ASTContext &astContext) override;
+  void HandleInlineFunctionDefinition(clang::FunctionDecl *d) override;
+  void HandleTagDeclDefinition(clang::TagDecl *d) override;
   void CompleteTentativeDefinition(clang::VarDecl *d) override;
 
   mlir::ModuleOp getModule() const;
@@ -61,6 +86,8 @@ class CIRGenerator : public clang::ASTConsumer {
   const mlir::MLIRContext &getMLIRContext() const { return *mlirContext; };
 
   bool verifyModule() const;
+
+  void emitDeferredDecls();
 };
 
 } // namespace cir
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/external/llvm-project/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index e9d8a2baedf2..502d58d7db8b 100644
--- a/external/llvm-project/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -90,6 +90,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
       return cir::IntAttr::get(ty, 0);
     if (cir::isAnyFloatingPointType(ty))
       return cir::FPAttr::getZero(ty);
+    if (auto complexType = mlir::dyn_cast<cir::ComplexType>(ty))
+      return cir::ZeroAttr::get(complexType);
     if (auto arrTy = mlir::dyn_cast<cir::ArrayType>(ty))
       return cir::ZeroAttr::get(arrTy);
     if (auto vecTy = mlir::dyn_cast<cir::VectorType>(ty))
@@ -183,11 +185,25 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
                                     global.getSymName());
   }
 
+  mlir::Value createGetGlobal(cir::GlobalOp global) {
+    return createGetGlobal(global.getLoc(), global);
+  }
+
   cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
                            mlir::IntegerAttr align = {}) {
     return create<cir::StoreOp>(loc, val, dst, align);
   }
 
+  [[nodiscard]] cir::GlobalOp createGlobal(mlir::ModuleOp mlirModule,
+                                           mlir::Location loc,
+                                           mlir::StringRef name,
+                                           mlir::Type type,
+                                           cir::GlobalLinkageKind linkage) {
+    mlir::OpBuilder::InsertionGuard guard(*this);
+    setInsertionPointToStart(mlirModule.getBody());
+    return create<cir::GlobalOp>(loc, name, type, linkage);
+  }
+
   cir::GetMemberOp createGetMember(mlir::Location loc, mlir::Type resultTy,
                                    mlir::Value base, llvm::StringRef name,
                                    unsigned index) {
@@ -276,6 +292,11 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return createCast(loc, cir::CastKind::bitcast, src, newTy);
   }
 
+  mlir::Value createPtrBitcast(mlir::Value src, mlir::Type newPointeeTy) {
+    assert(mlir::isa<cir::PointerType>(src.getType()) && "expected ptr src");
+    return createBitcast(src, getPointerTo(newPointeeTy));
+  }
+
   //===--------------------------------------------------------------------===//
   // Binary Operators
   //===--------------------------------------------------------------------===//
@@ -300,6 +321,24 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return createBinop(loc, lhs, cir::BinOpKind::Or, rhs);
   }
 
+  mlir::Value createSelect(mlir::Location loc, mlir::Value condition,
+                           mlir::Value trueValue, mlir::Value falseValue) {
+    assert(trueValue.getType() == falseValue.getType() &&
+           "trueValue and falseValue should have the same type");
+    return create<cir::SelectOp>(loc, trueValue.getType(), condition, trueValue,
+                                 falseValue);
+  }
+
+  mlir::Value createLogicalAnd(mlir::Location loc, mlir::Value lhs,
+                               mlir::Value rhs) {
+    return createSelect(loc, lhs, rhs, getBool(false, loc));
+  }
+
+  mlir::Value createLogicalOr(mlir::Location loc, mlir::Value lhs,
+                              mlir::Value rhs) {
+    return createSelect(loc, lhs, getBool(true, loc), rhs);
+  }
+
   mlir::Value createMul(mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
                         OverflowBehavior ob = OverflowBehavior::None) {
     auto op =
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRAttrConstraints.td b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRAttrConstraints.td
new file mode 100644
index 000000000000..a8c81dbd71a0
--- /dev/null
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRAttrConstraints.td
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the CIR dialect attributes constraints.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CIR_DIALECT_IR_CIRATTRCONSTRAINTS_TD
+#define CLANG_CIR_DIALECT_IR_CIRATTRCONSTRAINTS_TD
+
+include "mlir/IR/CommonAttrConstraints.td"
+
+class CIR_IsAttrPred<code attr> : CPred<"::mlir::isa<" # attr # ">($_self)">;
+
+class CIR_AttrConstraint<code attr, string summary = "">
+    : Attr<CIR_IsAttrPred<attr>, summary>;
+
+//===----------------------------------------------------------------------===//
+// IntAttr constraints
+//===----------------------------------------------------------------------===//
+
+def CIR_AnyIntAttr : CIR_AttrConstraint<"::cir::IntAttr", "integer attribute">;
+
+//===----------------------------------------------------------------------===//
+// FPAttr constraints
+//===----------------------------------------------------------------------===//
+
+def CIR_AnyFPAttr : CIR_AttrConstraint<"::cir::FPAttr",
+  "floating-point attribute">;
+
+def CIR_AnyIntOrFloatAttr : AnyAttrOf<[CIR_AnyIntAttr, CIR_AnyFPAttr],
+    "integer or floating point type"> {
+  string cppType = "::mlir::TypedAttr";
+}
+
+//===----------------------------------------------------------------------===//
+// ArrayAttr constraints
+//===----------------------------------------------------------------------===//
+
+def CIR_IntArrayAttr : TypedArrayAttrBase<CIR_AnyIntAttr,
+   "integer array attribute">;
+
+#endif // CLANG_CIR_DIALECT_IR_CIRATTRCONSTRAINTS_TD
\ No newline at end of file
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index ecbced05a5fd..b48f4ed461cc 100644
--- a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -17,6 +17,7 @@ include "mlir/IR/BuiltinAttributeInterfaces.td"
 include "mlir/IR/EnumAttr.td"
 
 include "clang/CIR/Dialect/IR/CIRDialect.td"
+include "clang/CIR/Dialect/IR/CIRAttrConstraints.td"
 
 //===----------------------------------------------------------------------===//
 // CIR Attrs
@@ -276,6 +277,50 @@ def ConstPtrAttr : CIR_Attr<"ConstPtr", "ptr", [TypedAttrInterface]> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// ConstComplexAttr
+//===----------------------------------------------------------------------===//
+
+def ConstComplexAttr : CIR_Attr<"ConstComplex", "const_complex",
+                                [TypedAttrInterface]> {
+  let summary = "An attribute that contains a constant complex value";
+  let description = [{
+    The `#cir.const_complex` attribute contains a constant value of complex
+    number type. The `real` parameter gives the real part of the complex number
+    and the `imag` parameter gives the imaginary part of the complex number.
+
+    The `real` and `imag` parameters must both reference the same type and must
+    be either IntAttr or FPAttr.
+
+    ```mlir
+    %ci = #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i>
+        : !cir.complex<!s32i>
+    %cf = #cir.const_complex<#cir.fp<1.000000e+00> : !cir.float,
+        #cir.fp<2.000000e+00> : !cir.float> : !cir.complex<!cir.float>
+    ```
+  }];
+
+  let parameters = (ins
+    AttributeSelfTypeParameter<"", "cir::ComplexType">:$type,
+    CIR_AnyIntOrFloatAttr:$real,
+    CIR_AnyIntOrFloatAttr:$imag
+  );
+
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "mlir::TypedAttr":$real,
+                                        "mlir::TypedAttr":$imag), [{
+      auto type = cir::ComplexType::get(real.getType());
+      return $_get(type.getContext(), type, real, imag);
+    }]>,
+  ];
+
+  let genVerifyDecl = 1;
+
+  let assemblyFormat = [{
+    `<` qualified($real) `,` qualified($imag) `>`
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // VisibilityAttr
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
index 62fc53f2362f..8ef565d6afd3 100644
--- a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
@@ -12,6 +12,7 @@
 #ifndef CLANG_CIR_DIALECT_IR_CIRDATALAYOUT_H
 #define CLANG_CIR_DIALECT_IR_CIRDATALAYOUT_H
 
+#include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/IR/BuiltinOps.h"
 
 namespace cir {
@@ -21,11 +22,18 @@ namespace cir {
 class CIRDataLayout {
   // This is starting with the minimum functionality needed for code that is
   // being upstreamed. Additional methods and members will be added as needed.
+  bool bigEndian = false;
+
 public:
   mlir::DataLayout layout;
 
   /// Constructs a DataLayout the module's data layout attribute.
-  CIRDataLayout(mlir::ModuleOp modOp) : layout{modOp} {}
+  CIRDataLayout(mlir::ModuleOp modOp);
+
+  /// Parse a data layout string (with fallback to default values).
+  void reset(mlir::DataLayoutSpecInterface spec);
+
+  bool isBigEndian() const { return bigEndian; }
 };
 
 } // namespace cir
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIROps.td b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 07851610a2ab..bd36d228578b 100644
--- a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -17,6 +17,7 @@
 include "clang/CIR/Dialect/IR/CIRDialect.td"
 include "clang/CIR/Dialect/IR/CIRTypes.td"
 include "clang/CIR/Dialect/IR/CIRAttrs.td"
+include "clang/CIR/Dialect/IR/CIRAttrConstraints.td"
 
 include "clang/CIR/Interfaces/CIROpInterfaces.td"
 include "clang/CIR/Interfaces/CIRLoopOpInterface.td"
@@ -1464,9 +1465,13 @@ def ShiftOp : CIR_Op<"shift", [Pure]> {
     ```
   }];
 
-  let results = (outs CIR_AnyIntOrVecOfInt:$result);
-  let arguments = (ins CIR_AnyIntOrVecOfInt:$value, CIR_AnyIntOrVecOfInt:$amount,
-                       UnitAttr:$isShiftleft);
+  let arguments = (ins
+    CIR_AnyIntOrVecOfIntType:$value,
+    CIR_AnyIntOrVecOfIntType:$amount,
+    UnitAttr:$isShiftleft
+  );
+
+  let results = (outs CIR_AnyIntOrVecOfIntType:$result);
 
   let assemblyFormat = [{
     `(`
@@ -1774,11 +1779,6 @@ def GetMemberOp : CIR_Op<"get_member"> {
 
     /// Return the record type pointed by the base pointer.
     cir::PointerType getAddrTy() { return getAddr().getType(); }
-
-    /// Return the result type.
-    cir::PointerType getResultTy() {
-      return getResult().getType();
-    }
   }];
 
   let hasVerifier = 1;
@@ -2050,7 +2050,7 @@ def VecCreateOp : CIR_Op<"vec.create", [Pure]> {
     in the vector type.
   }];
 
-  let arguments = (ins Variadic<CIR_AnyType>:$elements);
+  let arguments = (ins Variadic<CIR_VectorElementType>:$elements);
   let results = (outs CIR_VectorType:$result);
 
   let assemblyFormat = [{
@@ -2059,6 +2059,7 @@ def VecCreateOp : CIR_Op<"vec.create", [Pure]> {
   }];
 
   let hasVerifier = 1;
+  let hasFolder = 1;
 }
 
 //===----------------------------------------------------------------------===//
@@ -2085,7 +2086,7 @@ def VecInsertOp : CIR_Op<"vec.insert", [Pure,
 
   let arguments = (ins
     CIR_VectorType:$vec,
-    AnyType:$value,
+    CIR_VectorElementType:$value,
     CIR_AnyFundamentalIntType:$index
   );
 
@@ -2118,7 +2119,7 @@ def VecExtractOp : CIR_Op<"vec.extract", [Pure,
   }];
 
   let arguments = (ins CIR_VectorType:$vec, CIR_AnyFundamentalIntType:$index);
-  let results = (outs CIR_AnyType:$result);
+  let results = (outs CIR_VectorElementType:$result);
 
   let assemblyFormat = [{
     $vec `[` $index `:` type($index) `]` attr-dict `:` qualified(type($vec))
@@ -2156,6 +2157,54 @@ def VecCmpOp : CIR_Op<"vec.cmp", [Pure, SameTypeOperands]> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// VecShuffleOp
+//===----------------------------------------------------------------------===//
+
+// TODO: Create an interface that both VecShuffleOp and VecShuffleDynamicOp
+// implement.  This could be useful for passes that don't care how the vector
+// shuffle was specified.
+
+def VecShuffleOp : CIR_Op<"vec.shuffle",
+                   [Pure, AllTypesMatch<["vec1", "vec2"]>]> {
+  let summary = "Combine two vectors using indices passed as constant integers";
+  let description = [{
+    The `cir.vec.shuffle` operation implements the documented form of Clang's
+    `__builtin_shufflevector`, where the indices of the shuffled result are
+    integer constants.
+
+    The two input vectors, which must have the same type, are concatenated.
+    Each of the integer constant arguments is interpreted as an index into that
+    concatenated vector, with a value of -1 meaning that the result value
+    doesn't matter. The result vector, which must have the same element type as
+    the input vectors and the same number of elements as the list of integer
+    constant indices, is constructed by taking the elements at the given
+    indices from the concatenated vector. The size of the result vector does
+    not have to match the size of the individual input vectors or of the
+    concatenated vector.
+
+    ```mlir
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<2 x !s32i>)
+        [#cir.int<3> : !s64i, #cir.int<1> : !s64i] : !cir.vector<2 x !s32i>
+    ```
+  }];
+
+  let arguments = (ins
+    CIR_VectorType:$vec1,
+    CIR_VectorType:$vec2,
+    CIR_IntArrayAttr:$indices
+  );
+
+  let results = (outs CIR_VectorType:$result);
+  let assemblyFormat = [{
+    `(` $vec1 `,` $vec2 `:` qualified(type($vec1)) `)` $indices `:`
+     qualified(type($result)) attr-dict
+  }];
+
+  let hasVerifier = 1;
+  let hasFolder = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // VecShuffleDynamicOp
 //===----------------------------------------------------------------------===//
@@ -2180,7 +2229,7 @@ def VecShuffleDynamicOp : CIR_Op<"vec.shuffle.dynamic",
     ```
   }];
 
-  let arguments = (ins CIR_VectorType:$vec, IntegerVector:$indices);
+  let arguments = (ins CIR_VectorType:$vec, CIR_VectorOfIntType:$indices);
   let results = (outs CIR_VectorType:$result);
   let assemblyFormat = [{
     $vec `:` qualified(type($vec)) `,` $indices `:` qualified(type($indices))
@@ -2188,6 +2237,152 @@ def VecShuffleDynamicOp : CIR_Op<"vec.shuffle.dynamic",
   }];
 
   let hasVerifier = 1;
+  let hasFolder = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// VecTernaryOp
+//===----------------------------------------------------------------------===//
+
+def VecTernaryOp : CIR_Op<"vec.ternary",
+                   [Pure, AllTypesMatch<["result", "lhs", "rhs"]>]> {
+  let summary = "The `cond ? a : b` ternary operator for vector types";
+  let description = [{
+    The `cir.vec.ternary` operation represents the C/C++ ternary operator,
+    `?:`, for vector types, which does a `select` on individual elements of the
+    vectors. Unlike a regular `?:` operator, there is no short circuiting. All
+    three arguments are always evaluated.  Because there is no short
+    circuiting, there are no regions in this operation, unlike cir.ternary.
+
+    The first argument is a vector of integral type. The second and third
+    arguments are vectors of the same type and have the same number of elements
+    as the first argument.
+
+    The result is a vector of the same type as the second and third arguments.
+    Each element of the result is `(bool)a[n] ? b[n] : c[n]`.
+  }];
+
+  let arguments = (ins
+    CIR_VectorOfIntType:$cond,
+    CIR_VectorType:$lhs,
+    CIR_VectorType:$rhs
+  );
+
+  let results = (outs CIR_VectorType:$result);
+  let assemblyFormat = [{
+    `(` $cond `,` $lhs`,` $rhs `)` `:` qualified(type($cond)) `,`
+    qualified(type($lhs)) attr-dict
+  }];
+
+  let hasVerifier = 1;
+  let hasFolder = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// VecSplatOp
+//===----------------------------------------------------------------------===//
+
+def VecSplatOp : CIR_Op<"vec.splat", [Pure,
+  TypesMatchWith<"type of 'value' matches element type of 'result'", "result",
+                 "value", "cast<VectorType>($_self).getElementType()">]> {
+
+  let summary = "Convert a scalar into a vector";
+  let description = [{
+    The `cir.vec.splat` operation creates a vector value from a scalar value.
+    All elements of the vector have the same value, that of the given scalar.
+
+    It's a separate operation from `cir.vec.create` because more
+    efficient LLVM IR can be generated for it, and because some optimization and
+    analysis passes can benefit from knowing that all elements of the vector
+    have the same value.
+
+    ```mlir
+    %value = cir.const #cir.int<3> : !s32i
+    %value_vec = cir.vec.splat %value : !s32i, !cir.vector<4 x !s32i>
+    ```
+  }];
+
+  let arguments = (ins CIR_VectorElementType:$value);
+  let results = (outs CIR_VectorType:$result);
+
+  let assemblyFormat = [{
+    $value `:` type($value) `,` qualified(type($result)) attr-dict
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// BaseClassAddrOp
+//===----------------------------------------------------------------------===//
+
+def BaseClassAddrOp : CIR_Op<"base_class_addr"> {
+  let summary = "Get the base class address for a class/struct";
+  let description = [{
+    The `cir.base_class_addr` operaration gets the address of a particular
+    non-virtual base class given a derived class pointer. The offset in bytes
+    of the base class must be passed in, since it is easier for the front end
+    to calculate that than the MLIR passes. The operation contains a flag for
+    whether or not the operand may be nullptr. That depends on the context and
+    cannot be known by the operation, and that information affects how the
+    operation is lowered.
+
+    Example:
+    ```c++
+    struct Base { };
+    struct Derived : Base { };
+    Derived d;
+    Base& b = d;
+    ```
+    will generate
+    ```mlir
+    %3 = cir.base_class_addr %1 : !cir.ptr<!rec_Derived> nonnull [0] -> !cir.ptr<!rec_Base>
+    ```
+  }];
+
+  // The validity of the relationship of derived and base cannot yet be
+  // verified, currently not worth adding a verifier.
+  let arguments = (ins
+    Arg<CIR_PointerType, "derived class pointer", [MemRead]>:$derived_addr,
+    IndexAttr:$offset, UnitAttr:$assume_not_null);
+
+  let results = (outs Res<CIR_PointerType, "">:$base_addr);
+
+  let assemblyFormat = [{
+      $derived_addr `:` qualified(type($derived_addr))
+      (`nonnull` $assume_not_null^)?
+      ` ` `[` $offset `]` `->` qualified(type($base_addr)) attr-dict
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// ComplexCreateOp
+//===----------------------------------------------------------------------===//
+
+def ComplexCreateOp : CIR_Op<"complex.create", [Pure, SameTypeOperands]> {
+  let summary = "Create a complex value from its real and imaginary parts";
+  let description = [{
+    The `cir.complex.create` operation takes two operands that represent the
+    real and imaginary part of a complex number, and yields the complex number.
+
+    ```mlir
+    %0 = cir.const #cir.fp<1.000000e+00> : !cir.double
+    %1 = cir.const #cir.fp<2.000000e+00> : !cir.double
+    %2 = cir.complex.create %0, %1 : !cir.double -> !cir.complex<!cir.double>
+    ```
+  }];
+
+  let results = (outs CIR_ComplexType:$result);
+  let arguments = (ins
+    CIR_AnyIntOrFloatType:$real,
+    CIR_AnyIntOrFloatType:$imag
+  );
+
+  let assemblyFormat = [{
+    $real `,` $imag
+    `:` qualified(type($real)) `->` qualified(type($result)) attr-dict
+  }];
+
+  let hasVerifier = 1;
+  let hasFolder = 1;
 }
 
 #endif // CLANG_CIR_DIALECT_IR_CIROPS_TD
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td
index ec461cab961c..bcd516e27cc7 100644
--- a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td
@@ -31,6 +31,18 @@ class CIR_ConfinedType<Type type, list<Pred> preds, string summary = "">
     : Type<And<[type.predicate, CIR_CastedSelfsToType<type.cppType, preds>]>,
          summary, type.cppType>;
 
+// Generates a type summary.
+// - For a single type: returns its summary.
+// - For multiple types: returns `any of <comma-separated summaries>`.
+class CIR_TypeSummaries<list<Type> types> {
+    assert !not(!empty(types)), "expects non-empty list of types";
+
+    list<string> summaries = !foreach(type, types, type.summary);
+    string joined = !interleave(summaries, ", ");
+
+    string value = !if(!eq(!size(types), 1), joined, "any of " # joined);
+}
+
 //===----------------------------------------------------------------------===//
 // Bool Type predicates
 //===----------------------------------------------------------------------===//
@@ -184,6 +196,24 @@ def CIR_PtrToVoidPtrType
 // Vector Type predicates
 //===----------------------------------------------------------------------===//
 
+def CIR_AnyVectorType : CIR_TypeBase<"::cir::VectorType", "vector type">;
+
+def CIR_VectorElementType : AnyTypeOf<[CIR_AnyIntOrFloatType, CIR_AnyPtrType],
+    "any cir integer, floating point or pointer type"
+> {
+    let cppFunctionName = "isValidVectorTypeElementType";
+}
+
+class CIR_ElementTypePred<Pred pred> : SubstLeaves<"$_self",
+    "::mlir::cast<::cir::VectorType>($_self).getElementType()", pred>;
+
+class CIR_VectorTypeOf<list<Type> types, string summary = "">
+    : CIR_ConfinedType<CIR_AnyVectorType,
+        [Or<!foreach(type, types, CIR_ElementTypePred<type.predicate>)>],
+        !if(!empty(summary),
+            "vector of " # CIR_TypeSummaries<types>.value,
+            summary)>;
+
 // Vector of integral type
 def IntegerVector : Type<
     And<[
@@ -196,8 +226,36 @@ def IntegerVector : Type<
     ]>, "!cir.vector of !cir.int"> {
 }
 
-// Any Integer or Vector of Integer Constraints
-def CIR_AnyIntOrVecOfInt: AnyTypeOf<[CIR_AnyIntType, IntegerVector]>;
+// Vector of type constraints
+def CIR_VectorOfIntType : CIR_VectorTypeOf<[CIR_AnyIntType]>;
+def CIR_VectorOfUIntType : CIR_VectorTypeOf<[CIR_AnyUIntType]>;
+def CIR_VectorOfSIntType : CIR_VectorTypeOf<[CIR_AnySIntType]>;
+def CIR_VectorOfFloatType : CIR_VectorTypeOf<[CIR_AnyFloatType]>;
+
+// Vector or Scalar type constraints
+def CIR_AnyIntOrVecOfIntType
+    : AnyTypeOf<[CIR_AnyIntType, CIR_VectorOfIntType],
+        "integer or vector of integer type"> {
+    let cppFunctionName = "isIntOrVectorOfIntType";
+}
+
+def CIR_AnySIntOrVecOfSIntType
+    : AnyTypeOf<[CIR_AnySIntType, CIR_VectorOfSIntType],
+        "signed integer or vector of signed integer type"> {
+    let cppFunctionName = "isSIntOrVectorOfSIntType";
+}
+
+def CIR_AnyUIntOrVecOfUIntType
+    : AnyTypeOf<[CIR_AnyUIntType, CIR_VectorOfUIntType],
+        "unsigned integer or vector of unsigned integer type"> {
+    let cppFunctionName = "isUIntOrVectorOfUIntType";
+}
+
+def CIR_AnyFloatOrVecOfFloatType
+    : AnyTypeOf<[CIR_AnyFloatType, CIR_VectorOfFloatType],
+        "floating point or vector of floating point type"> {
+    let cppFunctionName = "isFPOrVectorOfFPType";
+}
 
 //===----------------------------------------------------------------------===//
 // Scalar Type predicates
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.h b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
index 3845fd2a4b67..49933be724a0 100644
--- a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
@@ -26,8 +26,6 @@ struct RecordTypeStorage;
 
 bool isValidFundamentalIntWidth(unsigned width);
 
-bool isFPOrFPVectorTy(mlir::Type);
-
 } // namespace cir
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
index 26f1122a4b26..41d7d725a09e 100644
--- a/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
+++ b/external/llvm-project/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -161,6 +161,56 @@ def CIR_LongDouble : CIR_FloatType<"LongDouble", "long_double"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// ComplexType
+//===----------------------------------------------------------------------===//
+
+def CIR_ComplexType : CIR_Type<"Complex", "complex",
+    [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
+
+  let summary = "CIR complex type";
+  let description = [{
+    CIR type that represents a C complex number. `cir.complex` models the C type
+    `T _Complex`.
+
+    `cir.complex` type is not directly mapped to `std::complex`.
+
+    The type models complex values, per C99 6.2.5p11. It supports the C99
+    complex float types as well as the GCC integer complex extensions.
+
+    The parameter `elementType` gives the type of the real and imaginary part of
+    the complex number. `elementType` must be either a CIR integer type or a CIR
+    floating-point type.
+
+    ```mlir
+    !cir.complex<!s32i>
+    !cir.complex<!cir.float>
+    ```
+  }];
+
+  let parameters = (ins CIR_AnyIntOrFloatType:$elementType);
+
+  let builders = [
+    TypeBuilderWithInferredContext<(ins "mlir::Type":$elementType), [{
+      return $_get(elementType.getContext(), elementType);
+    }]>,
+  ];
+
+  let assemblyFormat = [{
+    `<` $elementType `>`
+  }];
+
+  let extraClassDeclaration = [{
+    bool isFloatingPointComplex() const {
+      return isAnyFloatingPointType(getElementType());
+    }
+
+    bool isIntegerComplex() const {
+      return mlir::isa<cir::IntType>(getElementType());
+    }
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // PointerType
 //===----------------------------------------------------------------------===//
@@ -275,18 +325,31 @@ def CIR_VectorType : CIR_Type<"Vector", "vector",
 
   let summary = "CIR vector type";
   let description = [{
-   `!cir.vector' represents fixed-size vector types, parameterized
-    by the element type and the number of elements.
+    The `!cir.vector` type represents a fixed-size, one-dimensional vector.
+    It takes two parameters: the element type and the number of elements.
 
-    Example:
+    Syntax:
+
+    ```mlir
+    vector-type ::= !cir.vector<size x element-type>
+    element-type ::= float-type | integer-type | pointer-type
+    ```
+
+    The `element-type` must be a scalar CIR type. Zero-sized vectors are not
+    allowed. The `size` must be a positive integer.
+
+    Examples:
 
     ```mlir
-    !cir.vector<!u64i x 2>
-    !cir.vector<!cir.float x 4>
+    !cir.vector<4 x !cir.int<u, 8>>
+    !cir.vector<2 x !cir.float>
     ```
   }];
 
-  let parameters = (ins "mlir::Type":$elementType, "uint64_t":$size);
+  let parameters = (ins
+    CIR_VectorElementType:$elementType,
+    "uint64_t":$size
+  );
 
   let assemblyFormat = [{
     `<` $size `x` $elementType `>`
@@ -418,6 +481,11 @@ def CIR_RecordType : CIR_Type<"Record", "record",
     will be the same type. Attempting to build a record with an existing name,
     but a different body will result in an error.
 
+    Each record type will have a `RecordKind` that is either `Class`, `Struct`,
+    or `Union`, depending on the C/C++ type that it is representing. Note that
+    `Class` and `Struct` are semantically identical, but the kind preserves the
+    keyword that was used to declare the type in the original source code.
+
     A few examples:
 
     ```mlir
@@ -456,6 +524,18 @@ def CIR_RecordType : CIR_Type<"Record", "record",
   let genVerifyDecl = 1;
 
   let builders = [
+    // Create an identified and complete record type.
+    TypeBuilder<(ins
+      "llvm::ArrayRef<mlir::Type>":$members,
+      "mlir::StringAttr":$name,
+      "bool":$packed,
+      "bool":$padded,
+      "RecordKind":$kind
+    ), [{
+      return $_get($_ctxt, members, name, /*complete=*/true, packed, padded,
+                   kind);
+    }]>,
+
     // Create an identified and incomplete record type.
     TypeBuilder<(ins
       "mlir::StringAttr":$name,
@@ -469,8 +549,9 @@ def CIR_RecordType : CIR_Type<"Record", "record",
   let extraClassDeclaration = [{
     using Base::verifyInvariants;
 
-    enum RecordKind : uint32_t { Struct, Union };
+    enum RecordKind : uint32_t { Class, Struct, Union };
 
+    bool isClass() const { return getKind() == RecordKind::Class; };
     bool isStruct() const { return getKind() == RecordKind::Struct; };
     bool isUnion() const { return getKind() == RecordKind::Union; };
     bool isComplete() const { return !isIncomplete(); };
@@ -480,6 +561,8 @@ def CIR_RecordType : CIR_Type<"Record", "record",
     size_t getNumElements() const { return getMembers().size(); };
     std::string getKindAsStr() {
       switch (getKind()) {
+      case RecordKind::Class:
+        return "class";
       case RecordKind::Union:
         return "union";
       case RecordKind::Struct:
@@ -517,7 +600,8 @@ def CIRRecordType : Type<
 
 def CIR_AnyType : AnyTypeOf<[
   CIR_VoidType, CIR_BoolType, CIR_ArrayType, CIR_VectorType, CIR_IntType,
-  CIR_AnyFloatType, CIR_PointerType, CIR_FuncType, CIR_RecordType
+  CIR_AnyFloatType, CIR_PointerType, CIR_FuncType, CIR_RecordType,
+  CIR_ComplexType
 ]>;
 
 #endif // MLIR_CIR_DIALECT_CIR_TYPES
diff --git a/external/llvm-project/clang/include/clang/CIR/MissingFeatures.h b/external/llvm-project/clang/include/clang/CIR/MissingFeatures.h
index b8a40666a98c..13ddc77835fb 100644
--- a/external/llvm-project/clang/include/clang/CIR/MissingFeatures.h
+++ b/external/llvm-project/clang/include/clang/CIR/MissingFeatures.h
@@ -34,17 +34,18 @@ struct MissingFeatures {
   static bool opGlobalThreadLocal() { return false; }
   static bool opGlobalConstant() { return false; }
   static bool opGlobalWeakRef() { return false; }
-  static bool opGlobalLinkage() { return false; }
   static bool opGlobalUnnamedAddr() { return false; }
   static bool opGlobalSection() { return false; }
   static bool opGlobalVisibility() { return false; }
   static bool opGlobalDLLImportExport() { return false; }
   static bool opGlobalPartition() { return false; }
+  static bool opGlobalUsedOrCompilerUsed() { return false; }
 
   static bool supportIFuncAttr() { return false; }
   static bool supportVisibility() { return false; }
   static bool hiddenVisibility() { return false; }
   static bool protectedVisibility() { return false; }
+  static bool defaultVisibility() { return false; }
 
   // Load/store attributes
   static bool opLoadStoreThreadLocal() { return false; }
@@ -77,9 +78,12 @@ struct MissingFeatures {
   static bool opFuncLinkage() { return false; }
   static bool opFuncVisibility() { return false; }
   static bool opFuncNoProto() { return false; }
+  static bool opFuncCPUAndFeaturesAttributes() { return false; }
+  static bool opFuncSection() { return false; }
+  static bool opFuncSetComdat() { return false; }
+  static bool opFuncAttributesForDefinition() { return false; }
 
   // CallOp handling
-  static bool opCallBuiltinFunc() { return false; }
   static bool opCallPseudoDtor() { return false; }
   static bool opCallAggregateArgs() { return false; }
   static bool opCallPaddingArgs() { return false; }
@@ -132,11 +136,28 @@ struct MissingFeatures {
   static bool cxxSupport() { return false; }
   static bool recordZeroInit() { return false; }
   static bool zeroSizeRecordMembers() { return false; }
+  static bool recordLayoutVirtualBases() { return false; }
 
-  // Misc
+  // Various handling of deferred processing in CIRGenModule.
+  static bool cgmRelease() { return false; }
+  static bool deferredVtables() { return false; }
+  static bool deferredFuncDecls() { return false; }
+
+  // CXXABI
   static bool cxxABI() { return false; }
+  static bool cxxabiThisAlignment() { return false; }
+  static bool cxxabiUseARMMethodPtrABI() { return false; }
+  static bool cxxabiUseARMGuardVarABI() { return false; }
+  static bool cxxabiAppleARM64CXXABI() { return false; }
+  static bool cxxabiStructorImplicitParam() { return false; }
+
+  // Address class
+  static bool addressOffset() { return false; }
+  static bool addressIsKnownNonNull() { return false; }
+  static bool addressPointerAuthInfo() { return false; }
+
+  // Misc
   static bool cirgenABIInfo() { return false; }
-  static bool cirgenTargetInfo() { return false; }
   static bool abiArgInfo() { return false; }
   static bool tryEmitAsConstant() { return false; }
   static bool constructABIArgDirectExtend() { return false; }
@@ -151,6 +172,11 @@ struct MissingFeatures {
   static bool astVarDeclInterface() { return false; }
   static bool stackSaveOp() { return false; }
   static bool aggValueSlot() { return false; }
+  static bool aggValueSlotMayOverlap() { return false; }
+  static bool aggValueSlotVolatile() { return false; }
+  static bool aggValueSlotDestructedFlag() { return false; }
+  static bool aggValueSlotAlias() { return false; }
+  static bool aggValueSlotGC() { return false; }
   static bool generateDebugInfo() { return false; }
   static bool pointerOverflowSanitizer() { return false; }
   static bool fpConstraints() { return false; }
@@ -183,8 +209,6 @@ struct MissingFeatures {
   static bool msabi() { return false; }
   static bool typeChecks() { return false; }
   static bool lambdaFieldToName() { return false; }
-  static bool updateCompletedType() { return false; }
-  static bool targetSpecificCXXABI() { return false; }
   static bool moduleNameHash() { return false; }
   static bool constantFoldSwitchStatement() { return false; }
   static bool cudaSupport() { return false; }
@@ -193,12 +217,25 @@ struct MissingFeatures {
   static bool constEmitterVectorILE() { return false; }
   static bool needsGlobalCtorDtor() { return false; }
   static bool emitTypeCheck() { return false; }
-  static bool cxxabiThisDecl() { return false; }
-  static bool cxxabiThisAlignment() { return false; }
   static bool writebacks() { return false; }
   static bool cleanupsToDeactivate() { return false; }
   static bool stackBase() { return false; }
-  static bool deferredDecls() { return false; }
+  static bool deferredCXXGlobalInit() { return false; }
+  static bool setTargetAttributes() { return false; }
+  static bool coverageMapping() { return false; }
+  static bool peepholeProtection() { return false; }
+  static bool instrumentation() { return false; }
+  static bool cleanupAfterErrorDiags() { return false; }
+  static bool cxxRecordStaticMembers() { return false; }
+  static bool isMemcpyEquivalentSpecialMember() { return false; }
+  static bool isTrivialCtorOrDtor() { return false; }
+  static bool implicitConstructorArgs() { return false; }
+  static bool intrinsics() { return false; }
+  static bool attributeNoBuiltin() { return false; }
+  static bool thunks() { return false; }
+  static bool runCleanupsScope() { return false; }
+  static bool lowerAggregateLoadStore() { return false; }
+  static bool dataLayoutTypeAllocSize() { return false; }
 
   // Missing types
   static bool dataMemberType() { return false; }
@@ -214,7 +251,6 @@ struct MissingFeatures {
   // Future CIR operations
   static bool awaitOp() { return false; }
   static bool callOp() { return false; }
-  static bool complexCreateOp() { return false; }
   static bool complexImagOp() { return false; }
   static bool complexRealOp() { return false; }
   static bool ifOp() { return false; }
@@ -223,8 +259,9 @@ struct MissingFeatures {
   static bool ptrDiffOp() { return false; }
   static bool ptrStrideOp() { return false; }
   static bool switchOp() { return false; }
-  static bool ternaryOp() { return false; }
+  static bool throwOp() { return false; }
   static bool tryOp() { return false; }
+  static bool vecTernaryOp() { return false; }
   static bool zextOp() { return false; }
 
   // Future CIR attributes
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.h b/external/llvm-project/clang/include/clang/Driver/CommonArgs.h
similarity index 96%
rename from external/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.h
rename to external/llvm-project/clang/include/clang/Driver/CommonArgs.h
index 90bb6d1532bd..bda73ea61f3b 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/external/llvm-project/clang/include/clang/Driver/CommonArgs.h
@@ -291,6 +291,17 @@ void handleVectorizeLoopsArgs(const llvm::opt::ArgList &Args,
 /// Enable -fslp-vectorize based on the optimization level selected.
 void handleVectorizeSLPArgs(const llvm::opt::ArgList &Args,
                             llvm::opt::ArgStringList &CmdArgs);
+
+// Parse -mprefer-vector-width=. Return the Value string if well-formed.
+// Otherwise, return an empty string and issue a diagnosic message if needed.
+StringRef parseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
+                                        const llvm::opt::ArgList &Args);
+
+// Parse -mrecip. Return the Value string if well-formed.
+// Otherwise, return an empty string and issue a diagnosic message if needed.
+StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
+                            const llvm::opt::ArgList &Args);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/external/llvm-project/clang/include/clang/Driver/Driver.h b/external/llvm-project/clang/include/clang/Driver/Driver.h
index 64e0cc6a6d4b..6593c862518c 100644
--- a/external/llvm-project/clang/include/clang/Driver/Driver.h
+++ b/external/llvm-project/clang/include/clang/Driver/Driver.h
@@ -895,7 +895,7 @@ llvm::Error expandResponseFiles(SmallVectorImpl<const char *> &Args,
 /// See applyOneOverrideOption.
 void applyOverrideOptions(SmallVectorImpl<const char *> &Args,
                           const char *OverrideOpts,
-                          llvm::StringSet<> &SavedStrings,
+                          llvm::StringSet<> &SavedStrings, StringRef EnvVar,
                           raw_ostream *OS = nullptr);
 
 } // end namespace driver
diff --git a/external/llvm-project/clang/include/clang/Driver/Options.td b/external/llvm-project/clang/include/clang/Driver/Options.td
index acce69dfcc7d..f10ac909d9da 100644
--- a/external/llvm-project/clang/include/clang/Driver/Options.td
+++ b/external/llvm-project/clang/include/clang/Driver/Options.td
@@ -1781,7 +1781,7 @@ def fmcdc_max_test_vectors_EQ : Joined<["-"], "fmcdc-max-test-vectors=">,
   HelpText<"Maximum number of test vectors in MC/DC coverage">,
   MarshallingInfoInt<CodeGenOpts<"MCDCMaxTVs">, "0x7FFFFFFE">;
 def fprofile_generate : Flag<["-"], "fprofile-generate">,
-    Group<f_Group>, Visibility<[ClangOption, CLOption]>,
+    Group<f_Group>, Visibility<[ClangOption, CLOption, FlangOption, FC1Option]>,
     HelpText<"Generate instrumented code to collect execution counts into default.profraw (overridden by LLVM_PROFILE_FILE env var)">;
 def fprofile_generate_EQ : Joined<["-"], "fprofile-generate=">,
     Group<f_Group>, Visibility<[ClangOption, CLOption]>,
@@ -1798,7 +1798,7 @@ def fprofile_use : Flag<["-"], "fprofile-use">, Group<f_Group>,
     Visibility<[ClangOption, CLOption]>, Alias<fprofile_instr_use>;
 def fprofile_use_EQ : Joined<["-"], "fprofile-use=">,
     Group<f_Group>,
-    Visibility<[ClangOption, CLOption]>,
+    Visibility<[ClangOption, CLOption, FlangOption, FC1Option]>,
     MetaVarName<"<pathname>">,
     HelpText<"Use instrumentation data for profile-guided optimization. If pathname is a directory, it reads from <pathname>/default.profdata. Otherwise, it reads from file <pathname>.">;
 def fno_profile_instr_generate : Flag<["-"], "fno-profile-instr-generate">,
@@ -4323,13 +4323,15 @@ def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>,
 def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>,
   HelpText<"Do not process trigraph sequences">,
   Visibility<[ClangOption, CC1Option]>;
-defm unique_source_file_names: BoolOption<"f", "unique-source-file-names",
-  CodeGenOpts<"UniqueSourceFileNames">, DefaultFalse,
-  PosFlag<SetTrue, [], [CC1Option], "Allow">,
-  NegFlag<SetFalse, [], [], "Do not allow">,
-  BothFlags<[], [ClangOption], " the compiler to assume that each translation unit has a unique "
-                               "source file name at link time">>,
-  Group<f_clang_Group>;
+def funique_source_file_names: Flag<["-"], "funique-source-file-names">, Group<f_Group>,
+  HelpText<"Allow the compiler to assume that each translation unit has a unique "                       
+           "source file identifier (see -funique-source-file-identifier) at link time">;
+def fno_unique_source_file_names: Flag<["-"], "fno-unique-source-file-names">;
+def unique_source_file_identifier_EQ: Joined<["-"], "funique-source-file-identifier=">, Group<f_Group>,
+  Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Specify the source file identifier for -funique-source-file-names; "
+           "uses the source file path if not specified">,
+  MarshallingInfoString<CodeGenOpts<"UniqueSourceFileIdentifier">>;
 def funsigned_bitfields : Flag<["-"], "funsigned-bitfields">, Group<f_Group>;
 def funsigned_char : Flag<["-"], "funsigned-char">, Group<f_Group>;
 def fno_unsigned_char : Flag<["-"], "fno-unsigned-char">;
@@ -5063,8 +5065,10 @@ def ffuchsia_api_level_EQ : Joined<["-"], "ffuchsia-api-level=">,
   HelpText<"Set Fuchsia API level">,
   MarshallingInfoInt<LangOpts<"FuchsiaAPILevel">>;
 def mmacos_version_min_EQ : Joined<["-"], "mmacos-version-min=">,
+  Visibility<[ClangOption, CC1Option, FlangOption]>,
   Group<m_Group>, HelpText<"Set macOS deployment target">;
 def : Joined<["-"], "mmacosx-version-min=">,
+  Visibility<[ClangOption, CC1Option, FC1Option, FlangOption]>,
   Group<m_Group>, Alias<mmacos_version_min_EQ>;
 def mms_bitfields : Flag<["-"], "mms-bitfields">, Group<m_Group>,
   Visibility<[ClangOption, CC1Option]>,
@@ -5190,9 +5194,10 @@ def mrvv_vector_bits_EQ : Joined<["-"], "mrvv-vector-bits=">, Group<m_Group>,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify the size in bits of an RVV vector register">,
   DocBrief<!strconcat(
-    "Defaults to the vector length agnostic value of \"scalable\". "
-    "Accepts power of 2 values between 64 and 65536. Also accepts "
-    "\"zvl\" to use the value implied by -march/-mcpu.",
+    "Specify the size in bits of an RVV vector register. Defaults to the "
+    "vector length agnostic value of \"scalable\". Accepts power of 2 values "
+    "between 64 and 65536. Also accepts \"zvl\" to use the value implied by "
+    "-march/-mcpu.",
     !cond(
       // Flang does not set the preprocessor define.
       !eq(GlobalDocumentation.Program, "Flang") : "",
@@ -5617,9 +5622,10 @@ def mno_implicit_float : Flag<["-"], "mno-implicit-float">, Group<m_Group>,
   HelpText<"Don't generate implicit floating point or vector instructions">;
 def mimplicit_float : Flag<["-"], "mimplicit-float">, Group<m_Group>;
 def mrecip : Flag<["-"], "mrecip">, Group<m_Group>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Equivalent to '-mrecip=all'">;
 def mrecip_EQ : CommaJoined<["-"], "mrecip=">, Group<m_Group>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Control use of approximate reciprocal and reciprocal square root instructions followed by <n> iterations of "
            "Newton-Raphson refinement. "
            "<value> = ( ['!'] ['vec-'] ('rcp'|'sqrt') [('h'|'s'|'d')] [':'<n>] ) | 'all' | 'default' | 'none'">,
@@ -6042,7 +6048,7 @@ def resource_dir : Separate<["-"], "resource-dir">,
   HelpText<"The directory which holds the compiler resource files">,
   MarshallingInfoString<HeaderSearchOpts<"ResourceDir">>;
 def resource_dir_EQ : Joined<["-"], "resource-dir=">, Flags<[NoXarchOption]>,
-  Visibility<[ClangOption, CC1Option, CLOption, DXCOption, FlangOption]>,
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
   Alias<resource_dir>;
 def rpath : Separate<["-"], "rpath">, Flags<[LinkerInput]>, Group<Link_Group>,
   Visibility<[ClangOption, FlangOption]>;
@@ -7305,6 +7311,15 @@ def fhermetic_module_files : Flag<["-"], "fhermetic-module-files">, Group<f_Grou
 def fdo_concurrent_to_openmp_EQ : Joined<["-"], "fdo-concurrent-to-openmp=">,
   HelpText<"Try to map `do concurrent` loops to OpenMP [none|host|device]">,
       Values<"none, host, device">;
+
+def fdefer_desc_map : Flag<["-"], "fdefer-desc-map">, Group<f_Group>,
+  HelpText<"Enable deferred descriptor mapping, which puts off top-level descriptor "
+    "mapping until target regions, this is the default behaviour">;
+
+def fno_defer_desc_map : Flag<["-"], "fno-defer-desc-map">, Group<f_Group>,
+  HelpText<"Disable deferred deferred descriptor mapping, which puts off top-level "
+          " descriptor mapping till target regions">;
+
 } // let Visibility = [FC1Option, FlangOption]
 
 def J : JoinedOrSeparate<["-"], "J">,
@@ -7925,7 +7940,7 @@ def fpatchable_function_entry_section_EQ
       MarshallingInfoString<CodeGenOpts<"PatchableFunctionEntrySection">>;
 def fprofile_instrument_EQ : Joined<["-"], "fprofile-instrument=">,
     HelpText<"Enable PGO instrumentation">, Values<"none,clang,llvm,csllvm,sample-coldcov">,
-    NormalizedValuesScope<"CodeGenOptions">,
+    NormalizedValuesScope<"llvm::driver::ProfileInstrKind">,
     NormalizedValues<["ProfileNone", "ProfileClangInstr", "ProfileIRInstr", "ProfileCSIRInstr", "ProfileIRSampleColdCov"]>,
     MarshallingInfoEnum<CodeGenOpts<"ProfileInstr">, "ProfileNone">;
 def fprofile_instrument_path_EQ : Joined<["-"], "fprofile-instrument-path=">,
diff --git a/external/llvm-project/clang/include/clang/ExtractAPI/API.h b/external/llvm-project/clang/include/clang/ExtractAPI/API.h
index c30e6fac66d6..1ace535686a3 100644
--- a/external/llvm-project/clang/include/clang/ExtractAPI/API.h
+++ b/external/llvm-project/clang/include/clang/ExtractAPI/API.h
@@ -1499,7 +1499,7 @@ APISet::createRecord(StringRef USR, StringRef Name,
                      CtorArgsContTy &&...CtorArgs) {
   // Ensure USR refers to a String stored in the allocator.
   auto USRString = copyString(USR);
-  auto Result = USRBasedLookupTable.insert({USRString, nullptr});
+  auto Result = USRBasedLookupTable.try_emplace(USRString);
   RecordTy *Record;
 
   // Create the record if it does not already exist
diff --git a/external/llvm-project/clang/include/clang/Frontend/ASTConsumers.h b/external/llvm-project/clang/include/clang/Frontend/ASTConsumers.h
index 0e068bf5cccb..890701b6ff18 100644
--- a/external/llvm-project/clang/include/clang/Frontend/ASTConsumers.h
+++ b/external/llvm-project/clang/include/clang/Frontend/ASTConsumers.h
@@ -35,6 +35,11 @@ CreateASTDumper(std::unique_ptr<raw_ostream> OS, StringRef FilterString,
                 bool DumpDecls, bool Deserialize, bool DumpLookups,
                 bool DumpDeclTypes, ASTDumpOutputFormat Format);
 
+std::unique_ptr<ASTConsumer>
+CreateASTDumper(raw_ostream &OS, StringRef FilterString, bool DumpDecls,
+                bool Deserialize, bool DumpLookups, bool DumpDeclTypes,
+                ASTDumpOutputFormat Format);
+
 // AST Decl node lister: prints qualified names of all filterable AST Decl
 // nodes.
 std::unique_ptr<ASTConsumer> CreateASTDeclNodeLister();
diff --git a/external/llvm-project/clang/include/clang/Lex/HLSLRootSignatureTokenKinds.def b/external/llvm-project/clang/include/clang/Lex/HLSLRootSignatureTokenKinds.def
index 1814ac4aeae5..a5cfeb34b2b5 100644
--- a/external/llvm-project/clang/include/clang/Lex/HLSLRootSignatureTokenKinds.def
+++ b/external/llvm-project/clang/include/clang/Lex/HLSLRootSignatureTokenKinds.def
@@ -53,6 +53,9 @@
 #ifndef SHADER_VISIBILITY_ENUM
 #define SHADER_VISIBILITY_ENUM(NAME, LIT) ENUM(NAME, LIT)
 #endif
+#ifndef FILTER_ENUM
+#define FILTER_ENUM(NAME, LIT) ENUM(NAME, LIT)
+#endif
 #ifndef TEXTURE_ADDRESS_MODE_ENUM
 #define TEXTURE_ADDRESS_MODE_ENUM(NAME, LIT) ENUM(NAME, LIT)
 #endif
@@ -110,6 +113,7 @@ KEYWORD(numDescriptors)
 KEYWORD(offset)
 
 // StaticSampler Keywords:
+KEYWORD(filter)
 KEYWORD(mipLODBias)
 KEYWORD(addressU)
 KEYWORD(addressV)
@@ -162,6 +166,44 @@ SHADER_VISIBILITY_ENUM(Pixel, "SHADER_VISIBILITY_PIXEL")
 SHADER_VISIBILITY_ENUM(Amplification, "SHADER_VISIBILITY_AMPLIFICATION")
 SHADER_VISIBILITY_ENUM(Mesh, "SHADER_VISIBILITY_MESH")
 
+// Filter Enums:
+FILTER_ENUM(MinMagMipPoint, "FILTER_MIN_MAG_MIP_POINT")
+FILTER_ENUM(MinMagPointMipLinear, "FILTER_MIN_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(MinPointMagLinearMipPoint, "FILTER_MIN_POINT_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(MinPointMagMipLinear, "FILTER_MIN_POINT_MAG_MIP_LINEAR")
+FILTER_ENUM(MinLinearMagMipPoint, "FILTER_MIN_LINEAR_MAG_MIP_POINT")
+FILTER_ENUM(MinLinearMagPointMipLinear, "FILTER_MIN_LINEAR_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(MinMagLinearMipPoint, "FILTER_MIN_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(MinMagMipLinear, "FILTER_MIN_MAG_MIP_LINEAR")
+FILTER_ENUM(Anisotropic, "FILTER_ANISOTROPIC")
+FILTER_ENUM(ComparisonMinMagMipPoint, "FILTER_COMPARISON_MIN_MAG_MIP_POINT")
+FILTER_ENUM(ComparisonMinMagPointMipLinear, "FILTER_COMPARISON_MIN_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(ComparisonMinPointMagLinearMipPoint, "FILTER_COMPARISON_MIN_POINT_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(ComparisonMinPointMagMipLinear, "FILTER_COMPARISON_MIN_POINT_MAG_MIP_LINEAR")
+FILTER_ENUM(ComparisonMinLinearMagMipPoint, "FILTER_COMPARISON_MIN_LINEAR_MAG_MIP_POINT")
+FILTER_ENUM(ComparisonMinLinearMagPointMipLinear, "FILTER_COMPARISON_MIN_LINEAR_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(ComparisonMinMagLinearMipPoint, "FILTER_COMPARISON_MIN_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(ComparisonMinMagMipLinear, "FILTER_COMPARISON_MIN_MAG_MIP_LINEAR")
+FILTER_ENUM(ComparisonAnisotropic, "FILTER_COMPARISON_ANISOTROPIC")
+FILTER_ENUM(MinimumMinMagMipPoint, "FILTER_MINIMUM_MIN_MAG_MIP_POINT")
+FILTER_ENUM(MinimumMinMagPointMipLinear, "FILTER_MINIMUM_MIN_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(MinimumMinPointMagLinearMipPoint, "FILTER_MINIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(MinimumMinPointMagMipLinear, "FILTER_MINIMUM_MIN_POINT_MAG_MIP_LINEAR")
+FILTER_ENUM(MinimumMinLinearMagMipPoint, "FILTER_MINIMUM_MIN_LINEAR_MAG_MIP_POINT")
+FILTER_ENUM(MinimumMinLinearMagPointMipLinear, "FILTER_MINIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(MinimumMinMagLinearMipPoint, "FILTER_MINIMUM_MIN_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(MinimumMinMagMipLinear, "FILTER_MINIMUM_MIN_MAG_MIP_LINEAR")
+FILTER_ENUM(MinimumAnisotropic, "FILTER_MINIMUM_ANISOTROPIC")
+FILTER_ENUM(MaximumMinMagMipPoint, "FILTER_MAXIMUM_MIN_MAG_MIP_POINT")
+FILTER_ENUM(MaximumMinMagPointMipLinear, "FILTER_MAXIMUM_MIN_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(MaximumMinPointMagLinearMipPoint, "FILTER_MAXIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(MaximumMinPointMagMipLinear, "FILTER_MAXIMUM_MIN_POINT_MAG_MIP_LINEAR")
+FILTER_ENUM(MaximumMinLinearMagMipPoint, "FILTER_MAXIMUM_MIN_LINEAR_MAG_MIP_POINT")
+FILTER_ENUM(MaximumMinLinearMagPointMipLinear, "FILTER_MAXIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR")
+FILTER_ENUM(MaximumMinMagLinearMipPoint, "FILTER_MAXIMUM_MIN_MAG_LINEAR_MIP_POINT")
+FILTER_ENUM(MaximumMinMagMipLinear, "FILTER_MAXIMUM_MIN_MAG_MIP_LINEAR")
+FILTER_ENUM(MaximumAnisotropic, "FILTER_MAXIMUM_ANISOTROPIC")
+
 // Texture Address Mode Enums:
 TEXTURE_ADDRESS_MODE_ENUM(Wrap, "TEXTURE_ADDRESS_WRAP")
 TEXTURE_ADDRESS_MODE_ENUM(Mirror, "TEXTURE_ADDRESS_MIRROR")
@@ -189,6 +231,7 @@ STATIC_BORDER_COLOR_ENUM(OpaqueWhiteUint, "STATIC_BORDER_COLOR_OPAQUE_WHITE_UINT
 #undef STATIC_BORDER_COLOR_ENUM
 #undef COMPARISON_FUNC_ENUM
 #undef TEXTURE_ADDRESS_MODE_ENUM
+#undef FILTER_ENUM
 #undef SHADER_VISIBILITY_ENUM
 #undef DESCRIPTOR_RANGE_FLAG_ENUM
 #undef DESCRIPTOR_RANGE_FLAG_ENUM_OFF
diff --git a/external/llvm-project/clang/include/clang/Lex/Preprocessor.h b/external/llvm-project/clang/include/clang/Lex/Preprocessor.h
index cab398ad4a86..78be2bd64d61 100644
--- a/external/llvm-project/clang/include/clang/Lex/Preprocessor.h
+++ b/external/llvm-project/clang/include/clang/Lex/Preprocessor.h
@@ -132,7 +132,7 @@ enum class EmbedResult {
 struct CXXStandardLibraryVersionInfo {
   enum Library { Unknown, LibStdCXX };
   Library Lib;
-  unsigned Version;
+  std::uint64_t Version;
 };
 
 /// Engages in a tight little dance with the lexer to efficiently
@@ -2717,8 +2717,8 @@ class Preprocessor {
   std::optional<CXXStandardLibraryVersionInfo> CXXStandardLibraryVersion;
 
 public:
-  std::optional<unsigned> getStdLibCxxVersion();
-  bool NeedsStdLibCxxWorkaroundBefore(unsigned FixedVersion);
+  std::optional<std::uint64_t> getStdLibCxxVersion();
+  bool NeedsStdLibCxxWorkaroundBefore(std::uint64_t FixedVersion);
 
 private:
   //===--------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/include/clang/Parse/ParseHLSLRootSignature.h b/external/llvm-project/clang/include/clang/Parse/ParseHLSLRootSignature.h
index 66a887bc12e6..afa2c4d8cfe5 100644
--- a/external/llvm-project/clang/include/clang/Parse/ParseHLSLRootSignature.h
+++ b/external/llvm-project/clang/include/clang/Parse/ParseHLSLRootSignature.h
@@ -111,10 +111,11 @@ class RootSignatureParser {
 
   struct ParsedStaticSamplerParams {
     std::optional<llvm::hlsl::rootsig::Register> Reg;
-    std::optional<float> MipLODBias;
+    std::optional<llvm::hlsl::rootsig::SamplerFilter> Filter;
     std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressU;
     std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressV;
     std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressW;
+    std::optional<float> MipLODBias;
     std::optional<uint32_t> MaxAnisotropy;
     std::optional<llvm::hlsl::rootsig::ComparisonFunc> CompFunc;
     std::optional<llvm::hlsl::rootsig::StaticBorderColor> BorderColor;
@@ -132,6 +133,7 @@ class RootSignatureParser {
 
   /// Parsing methods of various enums
   std::optional<llvm::hlsl::rootsig::ShaderVisibility> parseShaderVisibility();
+  std::optional<llvm::hlsl::rootsig::SamplerFilter> parseSamplerFilter();
   std::optional<llvm::hlsl::rootsig::TextureAddressMode>
   parseTextureAddressMode();
   std::optional<llvm::hlsl::rootsig::ComparisonFunc> parseComparisonFunc();
diff --git a/external/llvm-project/clang/include/clang/Parse/Parser.h b/external/llvm-project/clang/include/clang/Parse/Parser.h
index c4bef4729fd3..3243b94c5e5e 100644
--- a/external/llvm-project/clang/include/clang/Parse/Parser.h
+++ b/external/llvm-project/clang/include/clang/Parse/Parser.h
@@ -290,9 +290,7 @@ class Parser : public CodeCompletionHandler {
     return ConsumeToken();
   }
 
-  SourceLocation getEndOfPreviousToken() {
-    return PP.getLocForEndOfToken(PrevTokLocation);
-  }
+  SourceLocation getEndOfPreviousToken() const;
 
   /// GetLookAheadToken - This peeks ahead N tokens and returns that token
   /// without consuming any tokens.  LookAhead(0) returns 'Tok', LookAhead(1)
@@ -2598,8 +2596,7 @@ class Parser : public CodeCompletionHandler {
   void ParseTypeQualifierListOpt(
       DeclSpec &DS, unsigned AttrReqs = AR_AllAttributesParsed,
       bool AtomicOrPtrauthAllowed = true, bool IdentifierRequired = false,
-      std::optional<llvm::function_ref<void()>> CodeCompletionHandler =
-          std::nullopt);
+      llvm::function_ref<void()> CodeCompletionHandler = {});
 
   /// ParseDirectDeclarator
   /// \verbatim
@@ -4172,8 +4169,7 @@ class Parser : public CodeCompletionHandler {
   bool ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
                            llvm::function_ref<void()> ExpressionStarts =
                                llvm::function_ref<void()>(),
-                           bool FailImmediatelyOnInvalidExpr = false,
-                           bool EarlyTypoCorrection = false);
+                           bool FailImmediatelyOnInvalidExpr = false);
 
   /// ParseSimpleExpressionList - A simple comma-separated list of expressions,
   /// used for misc language extensions.
@@ -7074,6 +7070,10 @@ class Parser : public CodeCompletionHandler {
   bool HandlePragmaMSOptimize(StringRef PragmaName,
                               SourceLocation PragmaLocation);
 
+  // #pragma intrinsic("foo")
+  bool HandlePragmaMSIntrinsic(StringRef PragmaName,
+                               SourceLocation PragmaLocation);
+
   /// Handle the annotation token produced for
   /// #pragma align...
   void HandlePragmaAlign();
diff --git a/external/llvm-project/clang/include/clang/Sema/ParsedAttr.h b/external/llvm-project/clang/include/clang/Sema/ParsedAttr.h
index 428d3111de80..6b3c5a173417 100644
--- a/external/llvm-project/clang/include/clang/Sema/ParsedAttr.h
+++ b/external/llvm-project/clang/include/clang/Sema/ParsedAttr.h
@@ -130,9 +130,6 @@ class ParsedAttr final
   size_t numTrailingObjects(OverloadToken<ParsedType>) const {
     return HasParsedType;
   }
-  size_t numTrailingObjects(OverloadToken<detail::PropertyData>) const {
-    return IsProperty;
-  }
 
 private:
   IdentifierInfo *MacroII = nullptr;
@@ -207,10 +204,9 @@ class ParsedAttr final
 
   /// Constructor for attributes with expression arguments.
   ParsedAttr(IdentifierInfo *attrName, SourceRange attrRange,
-             IdentifierInfo *scopeName, SourceLocation scopeLoc,
-             ArgsUnion *args, unsigned numArgs, Form formUsed,
-             SourceLocation ellipsisLoc)
-      : AttributeCommonInfo(attrName, scopeName, attrRange, scopeLoc, formUsed),
+             AttributeScopeInfo scope, ArgsUnion *args, unsigned numArgs,
+             Form formUsed, SourceLocation ellipsisLoc)
+      : AttributeCommonInfo(attrName, scope, attrRange, formUsed),
         EllipsisLoc(ellipsisLoc), NumArgs(numArgs), Invalid(false),
         UsedAsTypeAttr(false), IsAvailability(false),
         IsTypeTagForDatatype(false), IsProperty(false), HasParsedType(false),
@@ -222,14 +218,14 @@ class ParsedAttr final
 
   /// Constructor for availability attributes.
   ParsedAttr(IdentifierInfo *attrName, SourceRange attrRange,
-             IdentifierInfo *scopeName, SourceLocation scopeLoc,
-             IdentifierLoc *Parm, const AvailabilityChange &introduced,
+             AttributeScopeInfo scope, IdentifierLoc *Parm,
+             const AvailabilityChange &introduced,
              const AvailabilityChange &deprecated,
              const AvailabilityChange &obsoleted, SourceLocation unavailable,
              const Expr *messageExpr, Form formUsed, SourceLocation strict,
              const Expr *replacementExpr, const IdentifierLoc *environmentLoc)
-      : AttributeCommonInfo(attrName, scopeName, attrRange, scopeLoc, formUsed),
-        NumArgs(1), Invalid(false), UsedAsTypeAttr(false), IsAvailability(true),
+      : AttributeCommonInfo(attrName, scope, attrRange, formUsed), NumArgs(1),
+        Invalid(false), UsedAsTypeAttr(false), IsAvailability(true),
         IsTypeTagForDatatype(false), IsProperty(false), HasParsedType(false),
         HasProcessingCache(false), IsPragmaClangAttribute(false),
         UnavailableLoc(unavailable), MessageExpr(messageExpr),
@@ -243,14 +239,13 @@ class ParsedAttr final
 
   /// Constructor for objc_bridge_related attributes.
   ParsedAttr(IdentifierInfo *attrName, SourceRange attrRange,
-             IdentifierInfo *scopeName, SourceLocation scopeLoc,
-             IdentifierLoc *Parm1, IdentifierLoc *Parm2, IdentifierLoc *Parm3,
-             Form formUsed)
-      : AttributeCommonInfo(attrName, scopeName, attrRange, scopeLoc, formUsed),
-        NumArgs(3), Invalid(false), UsedAsTypeAttr(false),
-        IsAvailability(false), IsTypeTagForDatatype(false), IsProperty(false),
-        HasParsedType(false), HasProcessingCache(false),
-        IsPragmaClangAttribute(false), Info(ParsedAttrInfo::get(*this)) {
+             AttributeScopeInfo scope, IdentifierLoc *Parm1,
+             IdentifierLoc *Parm2, IdentifierLoc *Parm3, Form formUsed)
+      : AttributeCommonInfo(attrName, scope, attrRange, formUsed), NumArgs(3),
+        Invalid(false), UsedAsTypeAttr(false), IsAvailability(false),
+        IsTypeTagForDatatype(false), IsProperty(false), HasParsedType(false),
+        HasProcessingCache(false), IsPragmaClangAttribute(false),
+        Info(ParsedAttrInfo::get(*this)) {
     ArgsUnion *Args = getArgsBuffer();
     Args[0] = Parm1;
     Args[1] = Parm2;
@@ -259,14 +254,14 @@ class ParsedAttr final
 
   /// Constructor for type_tag_for_datatype attribute.
   ParsedAttr(IdentifierInfo *attrName, SourceRange attrRange,
-             IdentifierInfo *scopeName, SourceLocation scopeLoc,
-             IdentifierLoc *ArgKind, ParsedType matchingCType,
-             bool layoutCompatible, bool mustBeNull, Form formUsed)
-      : AttributeCommonInfo(attrName, scopeName, attrRange, scopeLoc, formUsed),
-        NumArgs(1), Invalid(false), UsedAsTypeAttr(false),
-        IsAvailability(false), IsTypeTagForDatatype(true), IsProperty(false),
-        HasParsedType(false), HasProcessingCache(false),
-        IsPragmaClangAttribute(false), Info(ParsedAttrInfo::get(*this)) {
+             AttributeScopeInfo scope, IdentifierLoc *ArgKind,
+             ParsedType matchingCType, bool layoutCompatible, bool mustBeNull,
+             Form formUsed)
+      : AttributeCommonInfo(attrName, scope, attrRange, formUsed), NumArgs(1),
+        Invalid(false), UsedAsTypeAttr(false), IsAvailability(false),
+        IsTypeTagForDatatype(true), IsProperty(false), HasParsedType(false),
+        HasProcessingCache(false), IsPragmaClangAttribute(false),
+        Info(ParsedAttrInfo::get(*this)) {
     ArgsUnion PVal(ArgKind);
     memcpy(getArgsBuffer(), &PVal, sizeof(ArgsUnion));
     detail::TypeTagForDatatypeData &ExtraData = getTypeTagForDatatypeDataSlot();
@@ -277,9 +272,9 @@ class ParsedAttr final
 
   /// Constructor for attributes with a single type argument.
   ParsedAttr(IdentifierInfo *attrName, SourceRange attrRange,
-             IdentifierInfo *scopeName, SourceLocation scopeLoc,
-             ParsedType typeArg, Form formUsed, SourceLocation ellipsisLoc)
-      : AttributeCommonInfo(attrName, scopeName, attrRange, scopeLoc, formUsed),
+             AttributeScopeInfo scope, ParsedType typeArg, Form formUsed,
+             SourceLocation ellipsisLoc)
+      : AttributeCommonInfo(attrName, scope, attrRange, formUsed),
         EllipsisLoc(ellipsisLoc), NumArgs(0), Invalid(false),
         UsedAsTypeAttr(false), IsAvailability(false),
         IsTypeTagForDatatype(false), IsProperty(false), HasParsedType(true),
@@ -290,13 +285,13 @@ class ParsedAttr final
 
   /// Constructor for microsoft __declspec(property) attribute.
   ParsedAttr(IdentifierInfo *attrName, SourceRange attrRange,
-             IdentifierInfo *scopeName, SourceLocation scopeLoc,
-             IdentifierInfo *getterId, IdentifierInfo *setterId, Form formUsed)
-      : AttributeCommonInfo(attrName, scopeName, attrRange, scopeLoc, formUsed),
-        NumArgs(0), Invalid(false), UsedAsTypeAttr(false),
-        IsAvailability(false), IsTypeTagForDatatype(false), IsProperty(true),
-        HasParsedType(false), HasProcessingCache(false),
-        IsPragmaClangAttribute(false), Info(ParsedAttrInfo::get(*this)) {
+             AttributeScopeInfo scope, IdentifierInfo *getterId,
+             IdentifierInfo *setterId, Form formUsed)
+      : AttributeCommonInfo(attrName, scope, attrRange, formUsed), NumArgs(0),
+        Invalid(false), UsedAsTypeAttr(false), IsAvailability(false),
+        IsTypeTagForDatatype(false), IsProperty(true), HasParsedType(false),
+        HasProcessingCache(false), IsPragmaClangAttribute(false),
+        Info(ParsedAttrInfo::get(*this)) {
     new (&getPropertyDataBuffer()) detail::PropertyData(getterId, setterId);
   }
 
@@ -738,21 +733,21 @@ class AttributePool {
   void takeFrom(ParsedAttributesView &List, AttributePool &Pool);
 
   ParsedAttr *create(IdentifierInfo *attrName, SourceRange attrRange,
-                     IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                     ArgsUnion *args, unsigned numArgs, ParsedAttr::Form form,
+                     AttributeScopeInfo scope, ArgsUnion *args,
+                     unsigned numArgs, ParsedAttr::Form form,
                      SourceLocation ellipsisLoc = SourceLocation()) {
     void *memory = allocate(
         ParsedAttr::totalSizeToAlloc<ArgsUnion, detail::AvailabilityData,
                                      detail::TypeTagForDatatypeData, ParsedType,
                                      detail::PropertyData>(numArgs, 0, 0, 0,
                                                            0));
-    return add(new (memory) ParsedAttr(attrName, attrRange, scopeName, scopeLoc,
-                                       args, numArgs, form, ellipsisLoc));
+    return add(new (memory) ParsedAttr(attrName, attrRange, scope, args,
+                                       numArgs, form, ellipsisLoc));
   }
 
   ParsedAttr *create(IdentifierInfo *attrName, SourceRange attrRange,
-                     IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                     IdentifierLoc *Param, const AvailabilityChange &introduced,
+                     AttributeScopeInfo scope, IdentifierLoc *Param,
+                     const AvailabilityChange &introduced,
                      const AvailabilityChange &deprecated,
                      const AvailabilityChange &obsoleted,
                      SourceLocation unavailable, const Expr *MessageExpr,
@@ -760,58 +755,54 @@ class AttributePool {
                      const Expr *ReplacementExpr,
                      IdentifierLoc *EnvironmentLoc) {
     void *memory = allocate(AttributeFactory::AvailabilityAllocSize);
-    return add(new (memory) ParsedAttr(attrName, attrRange, scopeName, scopeLoc,
-                                       Param, introduced, deprecated, obsoleted,
-                                       unavailable, MessageExpr, form, strict,
-                                       ReplacementExpr, EnvironmentLoc));
+    return add(new (memory)
+                   ParsedAttr(attrName, attrRange, scope, Param, introduced,
+                              deprecated, obsoleted, unavailable, MessageExpr,
+                              form, strict, ReplacementExpr, EnvironmentLoc));
   }
 
   ParsedAttr *create(IdentifierInfo *attrName, SourceRange attrRange,
-                     IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                     IdentifierLoc *Param1, IdentifierLoc *Param2,
-                     IdentifierLoc *Param3, ParsedAttr::Form form) {
+                     AttributeScopeInfo scope, IdentifierLoc *Param1,
+                     IdentifierLoc *Param2, IdentifierLoc *Param3,
+                     ParsedAttr::Form form) {
     void *memory = allocate(
         ParsedAttr::totalSizeToAlloc<ArgsUnion, detail::AvailabilityData,
                                      detail::TypeTagForDatatypeData, ParsedType,
                                      detail::PropertyData>(3, 0, 0, 0, 0));
-    return add(new (memory) ParsedAttr(attrName, attrRange, scopeName, scopeLoc,
-                                       Param1, Param2, Param3, form));
+    return add(new (memory) ParsedAttr(attrName, attrRange, scope, Param1,
+                                       Param2, Param3, form));
   }
 
-  ParsedAttr *
-  createTypeTagForDatatype(IdentifierInfo *attrName, SourceRange attrRange,
-                           IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                           IdentifierLoc *argumentKind,
-                           ParsedType matchingCType, bool layoutCompatible,
-                           bool mustBeNull, ParsedAttr::Form form) {
+  ParsedAttr *createTypeTagForDatatype(
+      IdentifierInfo *attrName, SourceRange attrRange, AttributeScopeInfo scope,
+      IdentifierLoc *argumentKind, ParsedType matchingCType,
+      bool layoutCompatible, bool mustBeNull, ParsedAttr::Form form) {
     void *memory = allocate(AttributeFactory::TypeTagForDatatypeAllocSize);
-    return add(new (memory) ParsedAttr(attrName, attrRange, scopeName, scopeLoc,
-                                       argumentKind, matchingCType,
-                                       layoutCompatible, mustBeNull, form));
+    return add(new (memory) ParsedAttr(attrName, attrRange, scope, argumentKind,
+                                       matchingCType, layoutCompatible,
+                                       mustBeNull, form));
   }
 
   ParsedAttr *createTypeAttribute(IdentifierInfo *attrName,
                                   SourceRange attrRange,
-                                  IdentifierInfo *scopeName,
-                                  SourceLocation scopeLoc, ParsedType typeArg,
+                                  AttributeScopeInfo scope, ParsedType typeArg,
                                   ParsedAttr::Form formUsed,
                                   SourceLocation ellipsisLoc) {
     void *memory = allocate(
         ParsedAttr::totalSizeToAlloc<ArgsUnion, detail::AvailabilityData,
                                      detail::TypeTagForDatatypeData, ParsedType,
                                      detail::PropertyData>(0, 0, 0, 1, 0));
-    return add(new (memory) ParsedAttr(attrName, attrRange, scopeName, scopeLoc,
-                                       typeArg, formUsed, ellipsisLoc));
+    return add(new (memory) ParsedAttr(attrName, attrRange, scope, typeArg,
+                                       formUsed, ellipsisLoc));
   }
 
   ParsedAttr *
   createPropertyAttribute(IdentifierInfo *attrName, SourceRange attrRange,
-                          IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                          IdentifierInfo *getterId, IdentifierInfo *setterId,
-                          ParsedAttr::Form formUsed) {
+                          AttributeScopeInfo scope, IdentifierInfo *getterId,
+                          IdentifierInfo *setterId, ParsedAttr::Form formUsed) {
     void *memory = allocate(AttributeFactory::PropertyAllocSize);
-    return add(new (memory) ParsedAttr(attrName, attrRange, scopeName, scopeLoc,
-                                       getterId, setterId, formUsed));
+    return add(new (memory) ParsedAttr(attrName, attrRange, scope, getterId,
+                                       setterId, formUsed));
   }
 };
 
@@ -985,19 +976,19 @@ class ParsedAttributes : public ParsedAttributesView {
 
   /// Add attribute with expression arguments.
   ParsedAttr *addNew(IdentifierInfo *attrName, SourceRange attrRange,
-                     IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                     ArgsUnion *args, unsigned numArgs, ParsedAttr::Form form,
+                     AttributeScopeInfo scope, ArgsUnion *args,
+                     unsigned numArgs, ParsedAttr::Form form,
                      SourceLocation ellipsisLoc = SourceLocation()) {
-    ParsedAttr *attr = pool.create(attrName, attrRange, scopeName, scopeLoc,
-                                   args, numArgs, form, ellipsisLoc);
+    ParsedAttr *attr = pool.create(attrName, attrRange, scope, args, numArgs,
+                                   form, ellipsisLoc);
     addAtEnd(attr);
     return attr;
   }
 
   /// Add availability attribute.
   ParsedAttr *addNew(IdentifierInfo *attrName, SourceRange attrRange,
-                     IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                     IdentifierLoc *Param, const AvailabilityChange &introduced,
+                     AttributeScopeInfo scope, IdentifierLoc *Param,
+                     const AvailabilityChange &introduced,
                      const AvailabilityChange &deprecated,
                      const AvailabilityChange &obsoleted,
                      SourceLocation unavailable, const Expr *MessageExpr,
@@ -1005,33 +996,31 @@ class ParsedAttributes : public ParsedAttributesView {
                      const Expr *ReplacementExpr,
                      IdentifierLoc *EnvironmentLoc) {
     ParsedAttr *attr =
-        pool.create(attrName, attrRange, scopeName, scopeLoc, Param, introduced,
-                    deprecated, obsoleted, unavailable, MessageExpr, form,
-                    strict, ReplacementExpr, EnvironmentLoc);
+        pool.create(attrName, attrRange, scope, Param, introduced, deprecated,
+                    obsoleted, unavailable, MessageExpr, form, strict,
+                    ReplacementExpr, EnvironmentLoc);
     addAtEnd(attr);
     return attr;
   }
 
   /// Add objc_bridge_related attribute.
   ParsedAttr *addNew(IdentifierInfo *attrName, SourceRange attrRange,
-                     IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                     IdentifierLoc *Param1, IdentifierLoc *Param2,
-                     IdentifierLoc *Param3, ParsedAttr::Form form) {
-    ParsedAttr *attr = pool.create(attrName, attrRange, scopeName, scopeLoc,
-                                   Param1, Param2, Param3, form);
+                     AttributeScopeInfo scope, IdentifierLoc *Param1,
+                     IdentifierLoc *Param2, IdentifierLoc *Param3,
+                     ParsedAttr::Form form) {
+    ParsedAttr *attr =
+        pool.create(attrName, attrRange, scope, Param1, Param2, Param3, form);
     addAtEnd(attr);
     return attr;
   }
 
   /// Add type_tag_for_datatype attribute.
-  ParsedAttr *
-  addNewTypeTagForDatatype(IdentifierInfo *attrName, SourceRange attrRange,
-                           IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                           IdentifierLoc *argumentKind,
-                           ParsedType matchingCType, bool layoutCompatible,
-                           bool mustBeNull, ParsedAttr::Form form) {
+  ParsedAttr *addNewTypeTagForDatatype(
+      IdentifierInfo *attrName, SourceRange attrRange, AttributeScopeInfo scope,
+      IdentifierLoc *argumentKind, ParsedType matchingCType,
+      bool layoutCompatible, bool mustBeNull, ParsedAttr::Form form) {
     ParsedAttr *attr = pool.createTypeTagForDatatype(
-        attrName, attrRange, scopeName, scopeLoc, argumentKind, matchingCType,
+        attrName, attrRange, scope, argumentKind, matchingCType,
         layoutCompatible, mustBeNull, form);
     addAtEnd(attr);
     return attr;
@@ -1039,12 +1028,11 @@ class ParsedAttributes : public ParsedAttributesView {
 
   /// Add an attribute with a single type argument.
   ParsedAttr *addNewTypeAttr(IdentifierInfo *attrName, SourceRange attrRange,
-                             IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                             ParsedType typeArg, ParsedAttr::Form formUsed,
+                             AttributeScopeInfo scope, ParsedType typeArg,
+                             ParsedAttr::Form formUsed,
                              SourceLocation ellipsisLoc = SourceLocation()) {
-    ParsedAttr *attr =
-        pool.createTypeAttribute(attrName, attrRange, scopeName, scopeLoc,
-                                 typeArg, formUsed, ellipsisLoc);
+    ParsedAttr *attr = pool.createTypeAttribute(attrName, attrRange, scope,
+                                                typeArg, formUsed, ellipsisLoc);
     addAtEnd(attr);
     return attr;
   }
@@ -1052,11 +1040,10 @@ class ParsedAttributes : public ParsedAttributesView {
   /// Add microsoft __delspec(property) attribute.
   ParsedAttr *
   addNewPropertyAttr(IdentifierInfo *attrName, SourceRange attrRange,
-                     IdentifierInfo *scopeName, SourceLocation scopeLoc,
-                     IdentifierInfo *getterId, IdentifierInfo *setterId,
-                     ParsedAttr::Form formUsed) {
+                     AttributeScopeInfo scope, IdentifierInfo *getterId,
+                     IdentifierInfo *setterId, ParsedAttr::Form formUsed) {
     ParsedAttr *attr = pool.createPropertyAttribute(
-        attrName, attrRange, scopeName, scopeLoc, getterId, setterId, formUsed);
+        attrName, attrRange, scope, getterId, setterId, formUsed);
     addAtEnd(attr);
     return attr;
   }
diff --git a/external/llvm-project/clang/include/clang/Sema/ParsedTemplate.h b/external/llvm-project/clang/include/clang/Sema/ParsedTemplate.h
index cff7f0466635..3a8050f9a0a3 100644
--- a/external/llvm-project/clang/include/clang/Sema/ParsedTemplate.h
+++ b/external/llvm-project/clang/include/clang/Sema/ParsedTemplate.h
@@ -190,9 +190,7 @@ namespace clang {
     bool ArgsInvalid;
 
     /// Retrieves a pointer to the template arguments
-    ParsedTemplateArgument *getTemplateArgs() {
-      return getTrailingObjects<ParsedTemplateArgument>();
-    }
+    ParsedTemplateArgument *getTemplateArgs() { return getTrailingObjects(); }
 
     /// Creates a new TemplateIdAnnotation with NumArgs arguments and
     /// appends it to List.
diff --git a/external/llvm-project/clang/include/clang/Sema/Redeclaration.h b/external/llvm-project/clang/include/clang/Sema/Redeclaration.h
index ae18b922f5cd..638fa6fd6424 100644
--- a/external/llvm-project/clang/include/clang/Sema/Redeclaration.h
+++ b/external/llvm-project/clang/include/clang/Sema/Redeclaration.h
@@ -28,4 +28,4 @@ enum class RedeclarationKind {
   ForExternalRedeclaration
 };
 
-#endif // LLVM_CLANG_SEMA_REDECLARATION_H
\ No newline at end of file
+#endif // LLVM_CLANG_SEMA_REDECLARATION_H
diff --git a/external/llvm-project/clang/include/clang/Sema/Sema.h b/external/llvm-project/clang/include/clang/Sema/Sema.h
index 6ce9ae588b63..29452bb37260 100644
--- a/external/llvm-project/clang/include/clang/Sema/Sema.h
+++ b/external/llvm-project/clang/include/clang/Sema/Sema.h
@@ -972,6 +972,13 @@ class Sema final : public SemaBase {
   /// Calls \c Lexer::getLocForEndOfToken()
   SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0);
 
+  /// Calls \c Lexer::findNextToken() to find the next token, and if the
+  /// locations of both ends of the token can be resolved it return that
+  /// range; Otherwise it returns an invalid SourceRange.
+  SourceRange getRangeForNextToken(
+      SourceLocation Loc, bool IncludeMacros, bool IncludeComments,
+      std::optional<tok::TokenKind> ExpectedToken = std::nullopt);
+
   /// Retrieve the module loader associated with the preprocessor.
   ModuleLoader &getModuleLoader() const;
 
@@ -2645,6 +2652,16 @@ class Sema final : public SemaBase {
   /// void*).
   void DiscardMisalignedMemberAddress(const Type *T, Expr *E);
 
+  /// Returns true if `From` is a function or pointer to a function with the
+  /// `cfi_unchecked_callee` attribute but `To` is a function or pointer to
+  /// function without this attribute.
+  bool DiscardingCFIUncheckedCallee(QualType From, QualType To) const;
+
+  /// Returns true if `From` is a function or pointer to a function without the
+  /// `cfi_unchecked_callee` attribute but `To` is a function or pointer to
+  /// function with this attribute.
+  bool AddingCFIUncheckedCallee(QualType From, QualType To) const;
+
   /// This function calls Action when it determines that E designates a
   /// misaligned member due to the packed attribute. This is used to emit
   /// local diagnostics like in reference binding.
@@ -6696,10 +6713,6 @@ class Sema final : public SemaBase {
     /// this expression evaluation context.
     unsigned NumCleanupObjects;
 
-    /// The number of typos encountered during this expression evaluation
-    /// context (i.e. the number of TypoExprs created).
-    unsigned NumTypos;
-
     MaybeODRUseExprSet SavedMaybeODRUseExprs;
 
     /// The lambdas that are present within this context, if it
@@ -6796,7 +6809,7 @@ class Sema final : public SemaBase {
                                       Decl *ManglingContextDecl,
                                       ExpressionKind ExprContext)
         : Context(Context), ParentCleanup(ParentCleanup),
-          NumCleanupObjects(NumCleanupObjects), NumTypos(0),
+          NumCleanupObjects(NumCleanupObjects),
           ManglingContextDecl(ManglingContextDecl), ExprContext(ExprContext),
           InDiscardedStatement(false), InImmediateFunctionContext(false),
           InImmediateEscalatingFunctionContext(false) {}
@@ -7129,8 +7142,7 @@ class Sema final : public SemaBase {
                       CorrectionCandidateCallback &CCC,
                       TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr,
                       ArrayRef<Expr *> Args = {},
-                      DeclContext *LookupCtx = nullptr,
-                      TypoExpr **Out = nullptr);
+                      DeclContext *LookupCtx = nullptr);
 
   /// If \p D cannot be odr-used in the current expression evaluation context,
   /// return a reason explaining why. Otherwise, return NOUR_None.
@@ -8663,6 +8675,7 @@ class Sema final : public SemaBase {
   // FIXME: This is in Sema because it requires
   // overload resolution, can we move to ASTContext?
   bool IsCXXTriviallyRelocatableType(QualType T);
+  bool IsCXXTriviallyRelocatableType(const CXXRecordDecl &RD);
 
   //// Determines if a type is replaceable
   /// according to the C++26 rules.
@@ -8730,40 +8743,6 @@ class Sema final : public SemaBase {
 
   ExprResult CheckUnevaluatedOperand(Expr *E);
 
-  /// Process any TypoExprs in the given Expr and its children,
-  /// generating diagnostics as appropriate and returning a new Expr if there
-  /// were typos that were all successfully corrected and ExprError if one or
-  /// more typos could not be corrected.
-  ///
-  /// \param E The Expr to check for TypoExprs.
-  ///
-  /// \param InitDecl A VarDecl to avoid because the Expr being corrected is its
-  /// initializer.
-  ///
-  /// \param RecoverUncorrectedTypos If true, when typo correction fails, it
-  /// will rebuild the given Expr with all TypoExprs degraded to RecoveryExprs.
-  ///
-  /// \param Filter A function applied to a newly rebuilt Expr to determine if
-  /// it is an acceptable/usable result from a single combination of typo
-  /// corrections. As long as the filter returns ExprError, different
-  /// combinations of corrections will be tried until all are exhausted.
-  ExprResult CorrectDelayedTyposInExpr(
-      Expr *E, VarDecl *InitDecl = nullptr,
-      bool RecoverUncorrectedTypos = false,
-      llvm::function_ref<ExprResult(Expr *)> Filter =
-          [](Expr *E) -> ExprResult { return E; });
-
-  ExprResult CorrectDelayedTyposInExpr(
-      ExprResult ER, VarDecl *InitDecl = nullptr,
-      bool RecoverUncorrectedTypos = false,
-      llvm::function_ref<ExprResult(Expr *)> Filter =
-          [](Expr *E) -> ExprResult { return E; }) {
-    return ER.isInvalid()
-               ? ER
-               : CorrectDelayedTyposInExpr(ER.get(), InitDecl,
-                                           RecoverUncorrectedTypos, Filter);
-  }
-
   IfExistsResult
   CheckMicrosoftIfExistsSymbol(Scope *S, CXXScopeSpec &SS,
                                const DeclarationNameInfo &TargetNameInfo);
@@ -9134,6 +9113,7 @@ class Sema final : public SemaBase {
   /// Diagnose if an explicit lambda capture is unused. Returns true if a
   /// diagnostic is emitted.
   bool DiagnoseUnusedLambdaCapture(SourceRange CaptureRange,
+                                   SourceRange FixItRange,
                                    const sema::Capture &From);
 
   /// Build a FieldDecl suitable to hold the given capture.
@@ -9264,12 +9244,6 @@ class Sema final : public SemaBase {
   /// for C++ records.
   llvm::FoldingSet<SpecialMemberOverloadResultEntry> SpecialMemberCache;
 
-  /// Holds TypoExprs that are created from `createDelayedTypo`. This is used by
-  /// `TransformTypos` in order to keep track of any TypoExprs that are created
-  /// recursively during typo correction and wipe them away if the correction
-  /// fails.
-  llvm::SmallVector<TypoExpr *, 2> TypoExprs;
-
   enum class AcceptableKind { Visible, Reachable };
 
   // Members have to be NamespaceDecl* or TranslationUnitDecl*.
@@ -9357,10 +9331,6 @@ class Sema final : public SemaBase {
                       bool VolatileArg, bool RValueThis, bool ConstThis,
                       bool VolatileThis);
 
-  typedef std::function<void(const TypoCorrection &)> TypoDiagnosticGenerator;
-  typedef std::function<ExprResult(Sema &, TypoExpr *, TypoCorrection)>
-      TypoRecoveryCallback;
-
   RedeclarationKind forRedeclarationInCurContext() const;
 
   /// Look up a name, looking for a single declaration.  Return
@@ -9714,51 +9684,6 @@ class Sema final : public SemaBase {
                              const ObjCObjectPointerType *OPT = nullptr,
                              bool RecordFailure = true);
 
-  /// Try to "correct" a typo in the source code by finding
-  /// visible declarations whose names are similar to the name that was
-  /// present in the source code.
-  ///
-  /// \param TypoName the \c DeclarationNameInfo structure that contains
-  /// the name that was present in the source code along with its location.
-  ///
-  /// \param LookupKind the name-lookup criteria used to search for the name.
-  ///
-  /// \param S the scope in which name lookup occurs.
-  ///
-  /// \param SS the nested-name-specifier that precedes the name we're
-  /// looking for, if present.
-  ///
-  /// \param CCC A CorrectionCandidateCallback object that provides further
-  /// validation of typo correction candidates. It also provides flags for
-  /// determining the set of keywords permitted.
-  ///
-  /// \param TDG A TypoDiagnosticGenerator functor that will be used to print
-  /// diagnostics when the actual typo correction is attempted.
-  ///
-  /// \param TRC A TypoRecoveryCallback functor that will be used to build an
-  /// Expr from a typo correction candidate.
-  ///
-  /// \param MemberContext if non-NULL, the context in which to look for
-  /// a member access expression.
-  ///
-  /// \param EnteringContext whether we're entering the context described by
-  /// the nested-name-specifier SS.
-  ///
-  /// \param OPT when non-NULL, the search for visible declarations will
-  /// also walk the protocols in the qualified interfaces of \p OPT.
-  ///
-  /// \returns a new \c TypoExpr that will later be replaced in the AST with an
-  /// Expr representing the result of performing typo correction, or nullptr if
-  /// typo correction is not possible. If nullptr is returned, no diagnostics
-  /// will be emitted and it is the responsibility of the caller to emit any
-  /// that are needed.
-  TypoExpr *CorrectTypoDelayed(
-      const DeclarationNameInfo &Typo, Sema::LookupNameKind LookupKind,
-      Scope *S, CXXScopeSpec *SS, CorrectionCandidateCallback &CCC,
-      TypoDiagnosticGenerator TDG, TypoRecoveryCallback TRC,
-      CorrectTypoKind Mode, DeclContext *MemberContext = nullptr,
-      bool EnteringContext = false, const ObjCObjectPointerType *OPT = nullptr);
-
   /// Kinds of missing import. Note, the values of these enumerators correspond
   /// to %select values in diagnostics.
   enum class MissingImportKind {
@@ -9777,20 +9702,6 @@ class Sema final : public SemaBase {
                              SourceLocation DeclLoc, ArrayRef<Module *> Modules,
                              MissingImportKind MIK, bool Recover);
 
-  struct TypoExprState {
-    std::unique_ptr<TypoCorrectionConsumer> Consumer;
-    TypoDiagnosticGenerator DiagHandler;
-    TypoRecoveryCallback RecoveryHandler;
-    TypoExprState();
-    TypoExprState(TypoExprState &&other) noexcept;
-    TypoExprState &operator=(TypoExprState &&other) noexcept;
-  };
-
-  const TypoExprState &getTypoExprState(TypoExpr *TE) const;
-
-  /// Clears the state of the given TypoExpr.
-  void clearDelayedTypo(TypoExpr *TE);
-
   /// Called on #pragma clang __debug dump II
   void ActOnPragmaDump(Scope *S, SourceLocation Loc, IdentifierInfo *II);
 
@@ -9813,23 +9724,15 @@ class Sema final : public SemaBase {
   /// Determine if we could use all the declarations in the module.
   bool isUsableModule(const Module *M);
 
-  /// Helper for CorrectTypo and CorrectTypoDelayed used to create and
-  /// populate a new TypoCorrectionConsumer. Returns nullptr if typo correction
-  /// should be skipped entirely.
+  /// Helper for CorrectTypo used to create and populate a new
+  /// TypoCorrectionConsumer. Returns nullptr if typo correction should be
+  /// skipped entirely.
   std::unique_ptr<TypoCorrectionConsumer> makeTypoCorrectionConsumer(
       const DeclarationNameInfo &Typo, Sema::LookupNameKind LookupKind,
       Scope *S, CXXScopeSpec *SS, CorrectionCandidateCallback &CCC,
       DeclContext *MemberContext, bool EnteringContext,
       const ObjCObjectPointerType *OPT, bool ErrorRecovery);
 
-  /// The set of unhandled TypoExprs and their associated state.
-  llvm::MapVector<TypoExpr *, TypoExprState> DelayedTypos;
-
-  /// Creates a new TypoExpr AST node.
-  TypoExpr *createDelayedTypo(std::unique_ptr<TypoCorrectionConsumer> TCC,
-                              TypoDiagnosticGenerator TDG,
-                              TypoRecoveryCallback TRC, SourceLocation TypoLoc);
-
   /// Cache for module units which is usable for current module.
   llvm::DenseSet<const Module *> UsableModuleUnitsCache;
 
@@ -10254,9 +10157,15 @@ class Sema final : public SemaBase {
                                  bool CStyle, bool &ObjCLifetimeConversion);
 
   /// Determine whether the conversion from FromType to ToType is a valid
-  /// conversion that strips "noexcept" or "noreturn" off the nested function
-  /// type.
-  bool IsFunctionConversion(QualType FromType, QualType ToType) const;
+  /// conversion that strips "noexcept" or "noreturn" or "cfi_unchecked_callee"
+  /// off the nested function type. This also checks if "cfi_unchecked_callee"
+  /// was added to the function type. If "cfi_unchecked_callee" is added and
+  /// `AddingCFIUncheckedCallee` is provided, it will be set to true. The same
+  /// thing applies for `DiscardingCFIUncheckedCallee` if the attribute is
+  /// discarded.
+  bool IsFunctionConversion(QualType FromType, QualType ToType,
+                            bool *DiscardingCFIUncheckedCallee = nullptr,
+                            bool *AddingCFIUncheckedCallee = nullptr) const;
 
   /// Same as `IsFunctionConversion`, but if this would return true, it sets
   /// `ResultTy` to `ToType`.
@@ -10505,13 +10414,34 @@ class Sema final : public SemaBase {
       OverloadCandidateParamOrder PO = {},
       bool AggregateCandidateDeduction = false);
 
+  struct CheckNonDependentConversionsFlag {
+    /// Do not consider any user-defined conversions when constructing the
+    /// initializing sequence.
+    bool SuppressUserConversions;
+
+    /// Before constructing the initializing sequence, we check whether the
+    /// parameter type and argument type contain any user defined conversions.
+    /// If so, do not initialize them. This effectively bypasses some undesired
+    /// instantiation before checking constaints, which might otherwise result
+    /// in non-SFINAE errors e.g. recursive constraints.
+    bool OnlyInitializeNonUserDefinedConversions;
+
+    CheckNonDependentConversionsFlag(
+        bool SuppressUserConversions,
+        bool OnlyInitializeNonUserDefinedConversions)
+        : SuppressUserConversions(SuppressUserConversions),
+          OnlyInitializeNonUserDefinedConversions(
+              OnlyInitializeNonUserDefinedConversions) {}
+  };
+
   /// Check that implicit conversion sequences can be formed for each argument
   /// whose corresponding parameter has a non-dependent type, per DR1391's
   /// [temp.deduct.call]p10.
   bool CheckNonDependentConversions(
       FunctionTemplateDecl *FunctionTemplate, ArrayRef<QualType> ParamTypes,
       ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
-      ConversionSequenceList &Conversions, bool SuppressUserConversions,
+      ConversionSequenceList &Conversions,
+      CheckNonDependentConversionsFlag UserConversionFlag,
       CXXRecordDecl *ActingContext = nullptr, QualType ObjectType = QualType(),
       Expr::Classification ObjectClassification = {},
       OverloadCandidateParamOrder PO = {});
@@ -12555,6 +12485,11 @@ class Sema final : public SemaBase {
   ///
   /// \param OriginalCallArgs If non-NULL, the original call arguments against
   /// which the deduced argument types should be compared.
+  /// \param CheckNonDependent Callback before substituting into the declaration
+  /// with the deduced template arguments.
+  /// \param OnlyInitializeNonUserDefinedConversions is used as a workaround for
+  /// some breakages introduced by CWG2369, where non-user-defined conversions
+  /// are checked first before the constraints.
   TemplateDeductionResult FinishTemplateArgumentDeduction(
       FunctionTemplateDecl *FunctionTemplate,
       SmallVectorImpl<DeducedTemplateArgument> &Deduced,
@@ -12562,7 +12497,10 @@ class Sema final : public SemaBase {
       sema::TemplateDeductionInfo &Info,
       SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs,
       bool PartialOverloading, bool PartialOrdering,
-      llvm::function_ref<bool()> CheckNonDependent = [] { return false; });
+      llvm::function_ref<bool(bool)> CheckNonDependent =
+          [](bool /*OnlyInitializeNonUserDefinedConversions*/) {
+            return false;
+          });
 
   /// Perform template argument deduction from a function call
   /// (C++ [temp.deduct.call]).
@@ -12598,7 +12536,7 @@ class Sema final : public SemaBase {
       bool PartialOrdering, QualType ObjectType,
       Expr::Classification ObjectClassification,
       bool ForOverloadSetAddressResolution,
-      llvm::function_ref<bool(ArrayRef<QualType>)> CheckNonDependent);
+      llvm::function_ref<bool(ArrayRef<QualType>, bool)> CheckNonDependent);
 
   /// Deduce template arguments when taking the address of a function
   /// template (C++ [temp.deduct.funcaddr]) or matching a specialization to
@@ -13074,6 +13012,9 @@ class Sema final : public SemaBase {
     /// Was the enclosing context a non-instantiation SFINAE context?
     bool SavedInNonInstantiationSFINAEContext;
 
+    /// Whether we're substituting into constraints.
+    bool InConstraintSubstitution;
+
     /// The point of instantiation or synthesis within the source code.
     SourceLocation PointOfInstantiation;
 
@@ -13123,9 +13064,9 @@ class Sema final : public SemaBase {
 
     CodeSynthesisContext()
         : Kind(TemplateInstantiation),
-          SavedInNonInstantiationSFINAEContext(false), Entity(nullptr),
-          Template(nullptr), TemplateArgs(nullptr), NumTemplateArgs(0),
-          DeductionInfo(nullptr) {}
+          SavedInNonInstantiationSFINAEContext(false),
+          InConstraintSubstitution(false), Entity(nullptr), Template(nullptr),
+          TemplateArgs(nullptr), NumTemplateArgs(0), DeductionInfo(nullptr) {}
 
     /// Determines whether this template is an actual instantiation
     /// that should be counted toward the maximum instantiation depth.
@@ -13369,9 +13310,22 @@ class Sema final : public SemaBase {
   ///
   /// \param SkipForSpecialization when specified, any template specializations
   /// in a traversal would be ignored.
+  ///
   /// \param ForDefaultArgumentSubstitution indicates we should continue looking
   /// when encountering a specialized member function template, rather than
   /// returning immediately.
+  void getTemplateInstantiationArgs(
+      MultiLevelTemplateArgumentList &Result, const NamedDecl *D,
+      const DeclContext *DC = nullptr, bool Final = false,
+      std::optional<ArrayRef<TemplateArgument>> Innermost = std::nullopt,
+      bool RelativeToPrimary = false, const FunctionDecl *Pattern = nullptr,
+      bool ForConstraintInstantiation = false,
+      bool SkipForSpecialization = false,
+      bool ForDefaultArgumentSubstitution = false);
+
+  /// This creates a new \p MultiLevelTemplateArgumentList and invokes the other
+  /// overload with it as the first parameter. Prefer this overload in most
+  /// situations.
   MultiLevelTemplateArgumentList getTemplateInstantiationArgs(
       const NamedDecl *D, const DeclContext *DC = nullptr, bool Final = false,
       std::optional<ArrayRef<TemplateArgument>> Innermost = std::nullopt,
@@ -13644,7 +13598,7 @@ class Sema final : public SemaBase {
   ExprResult
   SubstConstraintExpr(Expr *E,
                       const MultiLevelTemplateArgumentList &TemplateArgs);
-  // Unlike the above, this does not evaluates constraints.
+  // Unlike the above, this does not evaluate constraints.
   ExprResult SubstConstraintExprWithoutSatisfaction(
       Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs);
 
@@ -13794,6 +13748,12 @@ class Sema final : public SemaBase {
     return CodeSynthesisContexts.size() > NonInstantiationEntries;
   }
 
+  /// Determine whether we are currently performing constraint substitution.
+  bool inConstraintSubstitution() const {
+    return !CodeSynthesisContexts.empty() &&
+           CodeSynthesisContexts.back().InConstraintSubstitution;
+  }
+
   using EntityPrinter = llvm::function_ref<void(llvm::raw_ostream &)>;
 
   /// \brief create a Requirement::SubstitutionDiagnostic with only a
@@ -13956,7 +13916,7 @@ class Sema final : public SemaBase {
                               Decl *D)
         : TmplAttr(A), Scope(S), NewDecl(D) {}
   };
-  typedef SmallVector<LateInstantiatedAttribute, 16> LateInstantiatedAttrVec;
+  typedef SmallVector<LateInstantiatedAttribute, 1> LateInstantiatedAttrVec;
 
   void InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
                         const Decl *Pattern, Decl *Inst,
@@ -14786,10 +14746,10 @@ class Sema final : public SemaBase {
       const MultiLevelTemplateArgumentList &TemplateArgs,
       SourceRange TemplateIDRange);
 
-  bool CheckInstantiatedFunctionTemplateConstraints(
-      SourceLocation PointOfInstantiation, FunctionDecl *Decl,
-      ArrayRef<TemplateArgument> TemplateArgs,
-      ConstraintSatisfaction &Satisfaction);
+  bool CheckFunctionTemplateConstraints(SourceLocation PointOfInstantiation,
+                                        FunctionDecl *Decl,
+                                        ArrayRef<TemplateArgument> TemplateArgs,
+                                        ConstraintSatisfaction &Satisfaction);
 
   /// \brief Emit diagnostics explaining why a constraint expression was deemed
   /// unsatisfied.
diff --git a/external/llvm-project/clang/include/clang/Sema/SemaHLSL.h b/external/llvm-project/clang/include/clang/Sema/SemaHLSL.h
index 15182bb27bbd..ba5f06f93dc3 100644
--- a/external/llvm-project/clang/include/clang/Sema/SemaHLSL.h
+++ b/external/llvm-project/clang/include/clang/Sema/SemaHLSL.h
@@ -125,12 +125,15 @@ class SemaHLSL : public SemaBase {
   void handleSV_DispatchThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupIDAttr(Decl *D, const ParsedAttr &AL);
+  void handleSV_PositionAttr(Decl *D, const ParsedAttr &AL);
   void handlePackOffsetAttr(Decl *D, const ParsedAttr &AL);
   void handleShaderAttr(Decl *D, const ParsedAttr &AL);
   void handleResourceBindingAttr(Decl *D, const ParsedAttr &AL);
   void handleParamModifierAttr(Decl *D, const ParsedAttr &AL);
   bool handleResourceTypeAttr(QualType T, const ParsedAttr &AL);
 
+  void handleVkExtBuiltinInputAttr(Decl *D, const ParsedAttr &AL);
+
   bool CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
   QualType ProcessResourceTypeAttributes(QualType Wrapped);
   HLSLAttributedResourceLocInfo
@@ -144,6 +147,7 @@ class SemaHLSL : public SemaBase {
 
   // Diagnose whether the input ID is uint/unit2/uint3 type.
   bool diagnoseInputIDType(QualType T, const ParsedAttr &AL);
+  bool diagnosePositionType(QualType T, const ParsedAttr &AL);
 
   bool CanPerformScalarCast(QualType SrcTy, QualType DestTy);
   bool ContainsBitField(QualType BaseTy);
diff --git a/external/llvm-project/clang/include/clang/Sema/SemaInternal.h b/external/llvm-project/clang/include/clang/Sema/SemaInternal.h
index 95874077050a..4d0da1102bb5 100644
--- a/external/llvm-project/clang/include/clang/Sema/SemaInternal.h
+++ b/external/llvm-project/clang/include/clang/Sema/SemaInternal.h
@@ -314,20 +314,6 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
   bool SearchNamespaces;
 };
 
-inline Sema::TypoExprState::TypoExprState() {}
-
-inline Sema::TypoExprState::TypoExprState(TypoExprState &&other) noexcept {
-  *this = std::move(other);
-}
-
-inline Sema::TypoExprState &Sema::TypoExprState::
-operator=(Sema::TypoExprState &&other) noexcept {
-  Consumer = std::move(other.Consumer);
-  DiagHandler = std::move(other.DiagHandler);
-  RecoveryHandler = std::move(other.RecoveryHandler);
-  return *this;
-}
-
 } // end namespace clang
 
 #endif
diff --git a/external/llvm-project/clang/include/clang/Sema/SemaOpenMP.h b/external/llvm-project/clang/include/clang/Sema/SemaOpenMP.h
index 6498390fe96f..7b169f56b680 100644
--- a/external/llvm-project/clang/include/clang/Sema/SemaOpenMP.h
+++ b/external/llvm-project/clang/include/clang/Sema/SemaOpenMP.h
@@ -283,7 +283,7 @@ class SemaOpenMP : public SemaBase {
   /// mapper' construct.
   QualType ActOnOpenMPDeclareMapperType(SourceLocation TyLoc,
                                         TypeResult ParsedType);
-  /// Called on start of '#pragma omp declare mapper'.
+  /// Called for '#pragma omp declare mapper'.
   DeclGroupPtrTy ActOnOpenMPDeclareMapperDirective(
       Scope *S, DeclContext *DC, DeclarationName Name, QualType MapperType,
       SourceLocation StartLoc, DeclarationName VN, AccessSpecifier AS,
@@ -849,6 +849,7 @@ class SemaOpenMP : public SemaBase {
       FunctionDecl *FD, Expr *VariantRef, OMPTraitInfo &TI,
       ArrayRef<Expr *> AdjustArgsNothing,
       ArrayRef<Expr *> AdjustArgsNeedDevicePtr,
+      ArrayRef<Expr *> AdjustArgsNeedDeviceAddr,
       ArrayRef<OMPInteropInfo> AppendArgs, SourceLocation AdjustArgsLoc,
       SourceLocation AppendArgsLoc, SourceRange SR);
 
diff --git a/external/llvm-project/clang/include/clang/Sema/SemaPseudoObject.h b/external/llvm-project/clang/include/clang/Sema/SemaPseudoObject.h
index 4753987070b7..24f45b75abf2 100644
--- a/external/llvm-project/clang/include/clang/Sema/SemaPseudoObject.h
+++ b/external/llvm-project/clang/include/clang/Sema/SemaPseudoObject.h
@@ -37,4 +37,4 @@ class SemaPseudoObject : public SemaBase {
 
 } // namespace clang
 
-#endif // LLVM_CLANG_SEMA_SEMAPSEUDOOBJECT_H
\ No newline at end of file
+#endif // LLVM_CLANG_SEMA_SEMAPSEUDOOBJECT_H
diff --git a/external/llvm-project/clang/include/clang/Sema/Template.h b/external/llvm-project/clang/include/clang/Sema/Template.h
index f9a10cfafb1f..fe907078af27 100644
--- a/external/llvm-project/clang/include/clang/Sema/Template.h
+++ b/external/llvm-project/clang/include/clang/Sema/Template.h
@@ -522,6 +522,12 @@ enum class TemplateSubstitutionKind : char {
     llvm::PointerUnion<Decl *, DeclArgumentPack *> *
     findInstantiationOf(const Decl *D);
 
+    /// Similar to \p findInstantiationOf(), but it wouldn't assert if the
+    /// instantiation was not found within the current instantiation scope. This
+    /// is helpful for on-demand declaration instantiation.
+    llvm::PointerUnion<Decl *, DeclArgumentPack *> *
+    getInstantiationOfIfExists(const Decl *D);
+
     void InstantiatedLocal(const Decl *D, Decl *Inst);
     void InstantiatedLocalPackArg(const Decl *D, VarDecl *Inst);
     void MakeInstantiatedLocalArgPack(const Decl *D);
@@ -581,16 +587,17 @@ enum class TemplateSubstitutionKind : char {
     /// specializations that will need to be instantiated after the
     /// enclosing class's instantiation is complete.
     SmallVector<std::pair<ClassTemplateDecl *,
-                                ClassTemplatePartialSpecializationDecl *>, 4>
-      OutOfLinePartialSpecs;
+                          ClassTemplatePartialSpecializationDecl *>,
+                1>
+        OutOfLinePartialSpecs;
 
     /// A list of out-of-line variable template partial
     /// specializations that will need to be instantiated after the
     /// enclosing variable's instantiation is complete.
     /// FIXME: Verify that this is needed.
     SmallVector<
-        std::pair<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>, 4>
-    OutOfLineVarPartialSpecs;
+        std::pair<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>, 1>
+        OutOfLineVarPartialSpecs;
 
   public:
     TemplateDeclInstantiator(Sema &SemaRef, DeclContext *Owner,
diff --git a/external/llvm-project/clang/include/clang/Serialization/ASTReader.h b/external/llvm-project/clang/include/clang/Serialization/ASTReader.h
index 6963611c6a81..ba676fd8698e 100644
--- a/external/llvm-project/clang/include/clang/Serialization/ASTReader.h
+++ b/external/llvm-project/clang/include/clang/Serialization/ASTReader.h
@@ -565,7 +565,7 @@ class ASTReader
   /// modules. It is used for the following cases:
   /// - Lambda inside a template function definition: The main declaration is
   ///   the enclosing function, and the related declarations are the lambda
-  ///   declarations.
+  ///   call operators.
   /// - Friend function defined inside a template CXXRecord declaration: The
   ///   main declaration is the enclosing record, and the related declarations
   ///   are the friend functions.
diff --git a/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/Checker.h b/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/Checker.h
index c6866cb56155..31cc095c29bf 100644
--- a/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/Checker.h
+++ b/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/Checker.h
@@ -608,20 +608,6 @@ class EventDispatcher {
   }
 };
 
-/// Tag that can use a checker name as a message provider
-/// (see SimpleProgramPointTag).
-/// FIXME: This is a cargo cult class which is copied into several checkers but
-/// does not provide anything useful.
-/// The only added functionality provided by this class (compared to
-/// SimpleProgramPointTag) is that it composes the tag description string from
-/// two arguments -- but tag descriptions only appear in debug output so there
-/// is no reason to bother with this.
-class CheckerProgramPointTag : public SimpleProgramPointTag {
-public:
-  CheckerProgramPointTag(StringRef CheckerName, StringRef Msg);
-  CheckerProgramPointTag(const CheckerBase *Checker, StringRef Msg);
-};
-
 /// We dereferenced a location that may be null.
 struct ImplicitNullDerefEvent {
   SVal Location;
diff --git a/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h b/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
index 63ca3efc6d22..f20b0031c152 100644
--- a/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
+++ b/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
@@ -51,6 +51,8 @@ class CheckerContext {
       wasInlined(wasInlined) {
     assert(Pred->getState() &&
            "We should not call the checkers on an empty state.");
+    assert(loc.getTag() && "The ProgramPoint associated with CheckerContext "
+                           "must be tagged with the active checker.");
   }
 
   AnalysisManager &getAnalysisManager() {
@@ -169,6 +171,9 @@ class CheckerContext {
   ///        tag is specified, a default tag, unique to the given checker,
   ///        will be used. Tags are used to prevent states generated at
   ///        different sites from caching out.
+  /// NOTE: If the State is unchanged and the Tag is nullptr, this may return a
+  /// node which is not tagged (instead of using the default tag corresponding
+  /// to the active checker). This is arguably a bug and should be fixed.
   ExplodedNode *addTransition(ProgramStateRef State = nullptr,
                               const ProgramPointTag *Tag = nullptr) {
     return addTransitionImpl(State ? State : getState(), false, nullptr, Tag);
@@ -181,6 +186,9 @@ class CheckerContext {
   /// @param Pred The transition will be generated from the specified Pred node
   ///             to the newly generated node.
   /// @param Tag The tag to uniquely identify the creation site.
+  /// NOTE: If the State is unchanged and the Tag is nullptr, this may return a
+  /// node which is not tagged (instead of using the default tag corresponding
+  /// to the active checker). This is arguably a bug and should be fixed.
   ExplodedNode *addTransition(ProgramStateRef State, ExplodedNode *Pred,
                               const ProgramPointTag *Tag = nullptr) {
     return addTransitionImpl(State, false, Pred, Tag);
diff --git a/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h b/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
index 580b49a38dc7..70a7953918ac 100644
--- a/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
+++ b/external/llvm-project/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
@@ -18,6 +18,8 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/Support/SMTAPI.h"
 
+#include <algorithm>
+
 namespace clang {
 namespace ento {
 
@@ -570,23 +572,35 @@ class SMTConv {
   // TODO: Refactor to put elsewhere
   static inline QualType getAPSIntType(ASTContext &Ctx,
                                        const llvm::APSInt &Int) {
-    return Ctx.getIntTypeForBitwidth(Int.getBitWidth(), Int.isSigned());
+    const QualType Ty =
+        Ctx.getIntTypeForBitwidth(Int.getBitWidth(), Int.isSigned());
+    if (!Ty.isNull())
+      return Ty;
+    // If Ty is Null, could be because the original type was a _BitInt.
+    // Get the size of the _BitInt type (expressed in bits) and round it up to
+    // the next power of 2 that is at least the bit size of 'char' (usually 8).
+    unsigned CharTypeSize = Ctx.getTypeSize(Ctx.CharTy);
+    unsigned Pow2DestWidth =
+        std::max(llvm::bit_ceil(Int.getBitWidth()), CharTypeSize);
+    return Ctx.getIntTypeForBitwidth(Pow2DestWidth, Int.isSigned());
   }
 
   // Get the QualTy for the input APSInt, and fix it if it has a bitwidth of 1.
   static inline std::pair<llvm::APSInt, QualType>
   fixAPSInt(ASTContext &Ctx, const llvm::APSInt &Int) {
     llvm::APSInt NewInt;
+    unsigned APSIntBitwidth = Int.getBitWidth();
+    QualType Ty = getAPSIntType(Ctx, Int);
 
     // FIXME: This should be a cast from a 1-bit integer type to a boolean type,
     // but the former is not available in Clang. Instead, extend the APSInt
     // directly.
-    if (Int.getBitWidth() == 1 && getAPSIntType(Ctx, Int).isNull()) {
-      NewInt = Int.extend(Ctx.getTypeSize(Ctx.BoolTy));
-    } else
-      NewInt = Int;
-
-    return std::make_pair(NewInt, getAPSIntType(Ctx, NewInt));
+    if (APSIntBitwidth == 1 && Ty.isNull())
+      return {Int.extend(Ctx.getTypeSize(Ctx.BoolTy)),
+              getAPSIntType(Ctx, NewInt)};
+    if (llvm::isPowerOf2_32(APSIntBitwidth) || Ty.isNull())
+      return {Int, Ty};
+    return {Int.extend(Ctx.getTypeSize(Ty)), Ty};
   }
 
   // Perform implicit type conversion on binary symbolic expressions.
diff --git a/external/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h b/external/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h
index ddb527597c71..29a07f198572 100644
--- a/external/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h
+++ b/external/llvm-project/clang/include/clang/Support/RISCVVIntrinsicUtils.h
@@ -11,7 +11,6 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitmaskEnum.h"
-#include "llvm/ADT/Bitset.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include <cstdint>
@@ -484,51 +483,6 @@ class RVVIntrinsic {
                                    Policy &PolicyAttrs, bool HasFRMRoundModeOp);
 };
 
-// RVVRequire should be sync'ed with target features, but only
-// required features used in riscv_vector.td.
-enum RVVRequire {
-  RVV_REQ_RV64,
-  RVV_REQ_Zvfhmin,
-  RVV_REQ_Xandesvdot,
-  RVV_REQ_Xandesvpackfph,
-  RVV_REQ_Xsfvcp,
-  RVV_REQ_Xsfvfnrclipxfqf,
-  RVV_REQ_Xsfvfwmaccqqq,
-  RVV_REQ_Xsfvqmaccdod,
-  RVV_REQ_Xsfvqmaccqoq,
-  RVV_REQ_Zvbb,
-  RVV_REQ_Zvbc,
-  RVV_REQ_Zvkb,
-  RVV_REQ_Zvkg,
-  RVV_REQ_Zvkned,
-  RVV_REQ_Zvknha,
-  RVV_REQ_Zvknhb,
-  RVV_REQ_Zvksed,
-  RVV_REQ_Zvksh,
-  RVV_REQ_Zvfbfwma,
-  RVV_REQ_Zvfbfmin,
-  RVV_REQ_Zvfh,
-  RVV_REQ_Experimental,
-  RVV_REQ_NUM,
-};
-
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum RVVRequire Require);
-
-struct RequiredExtensionBits {
-  llvm::Bitset<RVV_REQ_NUM> Bits;
-  RequiredExtensionBits() {}
-  RequiredExtensionBits(std::initializer_list<RVVRequire> Init) {
-    for (auto I : Init)
-      Bits.set(I);
-  }
-
-  void set(unsigned I) { Bits.set(I); }
-  bool operator[](unsigned I) const { return Bits[I]; }
-};
-
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
-                              const RequiredExtensionBits &Exts);
-
 // Raw RVV intrinsic info, used to expand later.
 // This struct is highly compact for minimized code size.
 struct RVVIntrinsicRecord {
@@ -540,7 +494,7 @@ struct RVVIntrinsicRecord {
   const char *OverloadedName;
 
   // Required target features for this intrinsic.
-  RequiredExtensionBits RequiredExtensions;
+  const char *RequiredExtensions;
 
   // Prototype for this intrinsic, index of RVVSignatureTable.
   uint16_t PrototypeIndex;
diff --git a/external/llvm-project/clang/lib/APINotes/APINotesManager.cpp b/external/llvm-project/clang/lib/APINotes/APINotesManager.cpp
index 7f8a126ffaa0..4dc6ffd66bd5 100644
--- a/external/llvm-project/clang/lib/APINotes/APINotesManager.cpp
+++ b/external/llvm-project/clang/lib/APINotes/APINotesManager.cpp
@@ -15,9 +15,7 @@
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/SourceMgrAdapter.h"
-#include "clang/Basic/Version.h"
 #include "llvm/ADT/APInt.h"
-#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/external/llvm-project/clang/lib/APINotes/APINotesReader.cpp b/external/llvm-project/clang/lib/APINotes/APINotesReader.cpp
index b55872446ac4..7cc4df2a9936 100644
--- a/external/llvm-project/clang/lib/APINotes/APINotesReader.cpp
+++ b/external/llvm-project/clang/lib/APINotes/APINotesReader.cpp
@@ -16,10 +16,8 @@
 #include "APINotesFormat.h"
 #include "clang/APINotes/Types.h"
 #include "llvm/ADT/Hashing.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/Support/DJB.h"
-#include "llvm/Support/EndianStream.h"
 #include "llvm/Support/OnDiskHashTable.h"
 
 namespace clang {
diff --git a/external/llvm-project/clang/lib/AST/ASTContext.cpp b/external/llvm-project/clang/lib/AST/ASTContext.cpp
index 5044d7c33ec3..189e67e4eed0 100644
--- a/external/llvm-project/clang/lib/AST/ASTContext.cpp
+++ b/external/llvm-project/clang/lib/AST/ASTContext.cpp
@@ -1175,7 +1175,7 @@ void ASTContext::setCurrentNamedModule(Module *M) {
   CurrentCXXNamedModule = M;
 }
 
-bool ASTContext::isInSameModule(const Module *M1, const Module *M2) {
+bool ASTContext::isInSameModule(const Module *M1, const Module *M2) const {
   if (!M1 != !M2)
     return false;
 
@@ -1447,10 +1447,10 @@ void ASTContext::InitBuiltinTypes(const TargetInfo &Target,
 #include "clang/Basic/HLSLIntangibleTypes.def"
   }
 
-  if (Target.hasAArch64SVETypes() ||
-      (AuxTarget && AuxTarget->hasAArch64SVETypes())) {
-#define SVE_TYPE(Name, Id, SingletonId) \
-    InitBuiltinType(SingletonId, BuiltinType::Id);
+  if (Target.hasAArch64ACLETypes() ||
+      (AuxTarget && AuxTarget->hasAArch64ACLETypes())) {
+#define SVE_TYPE(Name, Id, SingletonId)                                        \
+  InitBuiltinType(SingletonId, BuiltinType::Id);
 #include "clang/Basic/AArch64ACLETypes.def"
   }
 
@@ -1705,6 +1705,73 @@ void ASTContext::setRelocationInfoForCXXRecord(
   RelocatableClasses.insert({D, Info});
 }
 
+static bool primaryBaseHaseAddressDiscriminatedVTableAuthentication(
+    ASTContext &Context, const CXXRecordDecl *Class) {
+  if (!Class->isPolymorphic())
+    return false;
+  const CXXRecordDecl *BaseType = Context.baseForVTableAuthentication(Class);
+  using AuthAttr = VTablePointerAuthenticationAttr;
+  const AuthAttr *ExplicitAuth = BaseType->getAttr<AuthAttr>();
+  if (!ExplicitAuth)
+    return Context.getLangOpts().PointerAuthVTPtrAddressDiscrimination;
+  AuthAttr::AddressDiscriminationMode AddressDiscrimination =
+      ExplicitAuth->getAddressDiscrimination();
+  if (AddressDiscrimination == AuthAttr::DefaultAddressDiscrimination)
+    return Context.getLangOpts().PointerAuthVTPtrAddressDiscrimination;
+  return AddressDiscrimination == AuthAttr::AddressDiscrimination;
+}
+
+ASTContext::PointerAuthContent ASTContext::findPointerAuthContent(QualType T) {
+  assert(isPointerAuthenticationAvailable());
+
+  T = T.getCanonicalType();
+  if (T.hasAddressDiscriminatedPointerAuth())
+    return PointerAuthContent::AddressDiscriminatedData;
+  const RecordDecl *RD = T->getAsRecordDecl();
+  if (!RD)
+    return PointerAuthContent::None;
+
+  if (auto Existing = RecordContainsAddressDiscriminatedPointerAuth.find(RD);
+      Existing != RecordContainsAddressDiscriminatedPointerAuth.end())
+    return Existing->second;
+
+  PointerAuthContent Result = PointerAuthContent::None;
+
+  auto SaveResultAndReturn = [&]() -> PointerAuthContent {
+    auto [ResultIter, DidAdd] =
+        RecordContainsAddressDiscriminatedPointerAuth.try_emplace(RD, Result);
+    (void)ResultIter;
+    (void)DidAdd;
+    assert(DidAdd);
+    return Result;
+  };
+  auto ShouldContinueAfterUpdate = [&](PointerAuthContent NewResult) {
+    static_assert(PointerAuthContent::None <
+                  PointerAuthContent::AddressDiscriminatedVTable);
+    static_assert(PointerAuthContent::AddressDiscriminatedVTable <
+                  PointerAuthContent::AddressDiscriminatedData);
+    if (NewResult > Result)
+      Result = NewResult;
+    return Result != PointerAuthContent::AddressDiscriminatedData;
+  };
+  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+    if (primaryBaseHaseAddressDiscriminatedVTableAuthentication(*this, CXXRD) &&
+        !ShouldContinueAfterUpdate(
+            PointerAuthContent::AddressDiscriminatedVTable))
+      return SaveResultAndReturn();
+    for (auto Base : CXXRD->bases()) {
+      if (!ShouldContinueAfterUpdate(findPointerAuthContent(Base.getType())))
+        return SaveResultAndReturn();
+    }
+  }
+  for (auto *FieldDecl : RD->fields()) {
+    if (!ShouldContinueAfterUpdate(
+            findPointerAuthContent(FieldDecl->getType())))
+      return SaveResultAndReturn();
+  }
+  return SaveResultAndReturn();
+}
+
 void ASTContext::addedLocalImportDecl(ImportDecl *Import) {
   assert(!Import->getNextLocalImport() &&
          "Import declaration already in the chain");
@@ -4529,7 +4596,7 @@ QualType ASTContext::getWebAssemblyExternrefType() const {
 /// type.
 QualType ASTContext::getScalableVectorType(QualType EltTy, unsigned NumElts,
                                            unsigned NumFields) const {
-  if (Target->hasAArch64SVETypes()) {
+  if (Target->hasAArch64ACLETypes()) {
     uint64_t EltTySize = getTypeSize(EltTy);
 
 #define SVE_VECTOR_TYPE_INT(Name, MangledName, Id, SingletonId, NumEls,        \
@@ -5216,7 +5283,7 @@ QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
     if (Underlying.isNull())
       Underlying = Decl->getUnderlyingType();
     auto *NewType = new (*this, alignof(TypedefType)) TypedefType(
-        Type::Typedef, Decl, QualType(), getCanonicalType(Underlying));
+        Type::Typedef, Decl, Underlying, /*HasTypeDifferentFromDecl=*/false);
     Decl->TypeForDecl = NewType;
     Types.push_back(NewType);
     return QualType(NewType, 0);
@@ -5238,7 +5305,7 @@ QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
   void *Mem = Allocate(TypedefType::totalSizeToAlloc<QualType>(true),
                        alignof(TypedefType));
   auto *NewType = new (Mem) TypedefType(Type::Typedef, Decl, Underlying,
-                                        getCanonicalType(Underlying));
+                                        /*HasTypeDifferentFromDecl=*/true);
   TypedefTypes.InsertNode(NewType, InsertPos);
   Types.push_back(NewType);
   return QualType(NewType, 0);
@@ -7429,6 +7496,12 @@ bool ASTContext::isSameEntity(const NamedDecl *X, const NamedDecl *Y) const {
                           cast<Decl>(Y->getDeclContext()->getRedeclContext())))
     return false;
 
+  // If either X or Y are local to the owning module, they are only possible to
+  // be the same entity if they are in the same module.
+  if (X->isModuleLocal() || Y->isModuleLocal())
+    if (!isInSameModule(X->getOwningModule(), Y->getOwningModule()))
+      return false;
+
   // Two typedefs refer to the same entity if they have the same underlying
   // type.
   if (const auto *TypedefX = dyn_cast<TypedefNameDecl>(X))
diff --git a/external/llvm-project/clang/lib/AST/ASTImporter.cpp b/external/llvm-project/clang/lib/AST/ASTImporter.cpp
index 003bad225e30..5c44353d8b98 100644
--- a/external/llvm-project/clang/lib/AST/ASTImporter.cpp
+++ b/external/llvm-project/clang/lib/AST/ASTImporter.cpp
@@ -9333,8 +9333,9 @@ class AttrImporter {
     if (Err)
       return;
 
-    AttributeCommonInfo ToI(ToAttrName, ToScopeName, ToAttrRange, ToScopeLoc,
-                            FromAttr->getParsedKind(), FromAttr->getForm());
+    AttributeCommonInfo ToI(
+        ToAttrName, AttributeScopeInfo(ToScopeName, ToScopeLoc), ToAttrRange,
+        FromAttr->getParsedKind(), FromAttr->getForm());
     // The "SemanticSpelling" is not needed to be passed to the constructor.
     // That value is recalculated from the SpellingListIndex if needed.
     ToAttr = T::Create(Importer.getToContext(),
diff --git a/external/llvm-project/clang/lib/AST/AttrImpl.cpp b/external/llvm-project/clang/lib/AST/AttrImpl.cpp
index fefb8f55a9ee..5875a925d3fb 100644
--- a/external/llvm-project/clang/lib/AST/AttrImpl.cpp
+++ b/external/llvm-project/clang/lib/AST/AttrImpl.cpp
@@ -224,6 +224,12 @@ void OMPDeclareVariantAttr::printPrettyPragma(
     PrintExprs(adjustArgsNeedDevicePtr_begin(), adjustArgsNeedDevicePtr_end());
     OS << ")";
   }
+  if (adjustArgsNeedDeviceAddr_size()) {
+    OS << " adjust_args(need_device_addr:";
+    PrintExprs(adjustArgsNeedDeviceAddr_begin(),
+               adjustArgsNeedDeviceAddr_end());
+    OS << ")";
+  }
 
   auto PrintInteropInfo = [&OS](OMPInteropInfo *Begin, OMPInteropInfo *End) {
     for (OMPInteropInfo *I = Begin; I != End; ++I) {
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Boolean.h b/external/llvm-project/clang/lib/AST/ByteCode/Boolean.h
index 8380e85865ac..fd8d54665688 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Boolean.h
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Boolean.h
@@ -30,16 +30,10 @@ class Boolean final {
 public:
   /// Zero-initializes a boolean.
   Boolean() : V(false) {}
-  Boolean(const llvm::APSInt &I) : V(!I.isZero()) {}
   explicit Boolean(bool V) : V(V) {}
 
   bool operator<(Boolean RHS) const { return V < RHS.V; }
   bool operator>(Boolean RHS) const { return V > RHS.V; }
-  bool operator<=(Boolean RHS) const { return V <= RHS.V; }
-  bool operator>=(Boolean RHS) const { return V >= RHS.V; }
-  bool operator==(Boolean RHS) const { return V == RHS.V; }
-  bool operator!=(Boolean RHS) const { return V != RHS.V; }
-
   bool operator>(unsigned RHS) const { return static_cast<unsigned>(V) > RHS; }
 
   Boolean operator-() const { return Boolean(V); }
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Compiler.cpp b/external/llvm-project/clang/lib/AST/ByteCode/Compiler.cpp
index bf38b2e5d537..9fe4803ce98e 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Compiler.cpp
@@ -6591,10 +6591,6 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
     return T->isReferenceType();
   };
 
-  // DecompositionDecls are just proxies for us.
-  if (isa<DecompositionDecl>(VD))
-    return revisit(VD);
-
   if ((VD->hasGlobalStorage() || VD->isStaticDataMember()) &&
       typeShouldBeVisited(VD->getType())) {
     if (const Expr *Init = VD->getAnyInitializer();
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Context.cpp b/external/llvm-project/clang/lib/AST/ByteCode/Context.cpp
index 6b8f4e31e4ff..971eb7fd5887 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Context.cpp
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Context.cpp
@@ -22,8 +22,10 @@ using namespace clang;
 using namespace clang::interp;
 
 Context::Context(ASTContext &Ctx) : Ctx(Ctx), P(new Program(*this)) {
+  this->ShortWidth = Ctx.getTargetInfo().getShortWidth();
   this->IntWidth = Ctx.getTargetInfo().getIntWidth();
   this->LongWidth = Ctx.getTargetInfo().getLongWidth();
+  this->LongLongWidth = Ctx.getTargetInfo().getLongLongWidth();
   assert(Ctx.getTargetInfo().getCharWidth() == 8 &&
          "We're assuming 8 bit chars");
 }
@@ -47,7 +49,7 @@ bool Context::isPotentialConstantExpr(State &Parent, const FunctionDecl *FD) {
   if (!Run(Parent, Func))
     return false;
 
-  return Func->isConstexpr();
+  return Func->isValid();
 }
 
 bool Context::evaluateAsRValue(State &Parent, const Expr *E, APValue &Result) {
@@ -265,6 +267,11 @@ std::optional<PrimType> Context::classify(QualType T) const {
       return PT_MemberPtr;
 
     // Just trying to avoid the ASTContext::getIntWidth call below.
+    if (Kind == BuiltinType::Short)
+      return integralTypeToPrimTypeS(this->ShortWidth);
+    if (Kind == BuiltinType::UShort)
+      return integralTypeToPrimTypeU(this->ShortWidth);
+
     if (Kind == BuiltinType::Int)
       return integralTypeToPrimTypeS(this->IntWidth);
     if (Kind == BuiltinType::UInt)
@@ -273,6 +280,11 @@ std::optional<PrimType> Context::classify(QualType T) const {
       return integralTypeToPrimTypeS(this->LongWidth);
     if (Kind == BuiltinType::ULong)
       return integralTypeToPrimTypeU(this->LongWidth);
+    if (Kind == BuiltinType::LongLong)
+      return integralTypeToPrimTypeS(this->LongLongWidth);
+    if (Kind == BuiltinType::ULongLong)
+      return integralTypeToPrimTypeU(this->LongLongWidth);
+
     if (Kind == BuiltinType::SChar || Kind == BuiltinType::Char_S)
       return integralTypeToPrimTypeS(8);
     if (Kind == BuiltinType::UChar || Kind == BuiltinType::Char_U ||
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Context.h b/external/llvm-project/clang/lib/AST/ByteCode/Context.h
index 9a604ce8ebbe..5898ab5e5459 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Context.h
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Context.h
@@ -138,8 +138,10 @@ class Context final {
   /// ID identifying an evaluation.
   unsigned EvalID = 0;
   /// Cached widths (in bits) of common types, for a faster classify().
+  unsigned ShortWidth;
   unsigned IntWidth;
   unsigned LongWidth;
+  unsigned LongLongWidth;
 };
 
 } // namespace interp
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Floating.h b/external/llvm-project/clang/lib/AST/ByteCode/Floating.h
index 3a874fc6f0b4..3750568fc23c 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Floating.h
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Floating.h
@@ -87,7 +87,6 @@ class Floating final {
 
   bool isSigned() const { return true; }
   bool isNegative() const { return F.isNegative(); }
-  bool isPositive() const { return !F.isNegative(); }
   bool isZero() const { return F.isZero(); }
   bool isNonZero() const { return F.isNonZero(); }
   bool isMin() const { return F.isSmallest(); }
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Function.cpp b/external/llvm-project/clang/lib/AST/ByteCode/Function.cpp
index e421dad06281..0e639df3cafb 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Function.cpp
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Function.cpp
@@ -23,10 +23,11 @@ Function::Function(Program &P, FunctionDeclTy Source, unsigned ArgSize,
       ParamTypes(std::move(ParamTypes)), Params(std::move(Params)),
       ParamOffsets(std::move(ParamOffsets)), IsValid(false),
       IsFullyCompiled(false), HasThisPointer(HasThisPointer), HasRVO(HasRVO),
-      Defined(false) {
+      HasBody(false), Defined(false) {
   if (const auto *F = dyn_cast<const FunctionDecl *>(Source)) {
     Variadic = F->isVariadic();
     Immediate = F->isImmediateFunction();
+    Constexpr = F->isConstexpr() || F->hasAttr<MSConstexprAttr>();
     if (const auto *CD = dyn_cast<CXXConstructorDecl>(F)) {
       Virtual = CD->isVirtual();
       Kind = FunctionKind::Ctor;
@@ -48,6 +49,7 @@ Function::Function(Program &P, FunctionDeclTy Source, unsigned ArgSize,
     Variadic = false;
     Virtual = false;
     Immediate = false;
+    Constexpr = false;
   }
 }
 
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Function.h b/external/llvm-project/clang/lib/AST/ByteCode/Function.h
index 8b577858474a..de88f3ded34d 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Function.h
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Function.h
@@ -150,12 +150,13 @@ class Function final {
   /// Returns the source information at a given PC.
   SourceInfo getSource(CodePtr PC) const;
 
-  /// Checks if the function is valid to call in constexpr.
-  bool isConstexpr() const { return IsValid || isLambdaStaticInvoker(); }
+  /// Checks if the function is valid to call.
+  bool isValid() const { return IsValid || isLambdaStaticInvoker(); }
 
   /// Checks if the function is virtual.
   bool isVirtual() const { return Virtual; };
   bool isImmediate() const { return Immediate; }
+  bool isConstexpr() const { return Constexpr; }
 
   /// Checks if the function is a constructor.
   bool isConstructor() const { return Kind == FunctionKind::Ctor; }
@@ -303,6 +304,8 @@ class Function final {
   unsigned Virtual : 1;
   LLVM_PREFERRED_TYPE(bool)
   unsigned Immediate : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned Constexpr : 1;
 
 public:
   /// Dumps the disassembled bytecode to \c llvm::errs().
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Interp.cpp b/external/llvm-project/clang/lib/AST/ByteCode/Interp.cpp
index 4d1e5c6b98a8..5c8abffb3a99 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Interp.cpp
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Interp.cpp
@@ -857,23 +857,22 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
     return false;
   }
 
-  // Bail out if the function declaration itself is invalid.  We will
-  // have produced a relevant diagnostic while parsing it, so just
-  // note the problematic sub-expression.
-  if (F->getDecl()->isInvalidDecl())
-    return Invalid(S, OpPC);
-
   if (S.checkingPotentialConstantExpression() && S.Current->getDepth() != 0)
     return false;
 
-  if (F->isConstexpr() && F->hasBody() &&
-      (F->getDecl()->isConstexpr() || F->getDecl()->hasAttr<MSConstexprAttr>()))
+  if (F->isValid() && F->hasBody() && F->isConstexpr())
     return true;
 
   // Implicitly constexpr.
   if (F->isLambdaStaticInvoker())
     return true;
 
+  // Bail out if the function declaration itself is invalid.  We will
+  // have produced a relevant diagnostic while parsing it, so just
+  // note the problematic sub-expression.
+  if (F->getDecl()->isInvalidDecl())
+    return Invalid(S, OpPC);
+
   // Diagnose failed assertions specially.
   if (S.Current->getLocation(OpPC).isMacroID() &&
       F->getDecl()->getIdentifier()) {
@@ -923,7 +922,8 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
       // for a constant expression. It might be defined at the point we're
       // actually calling it.
       bool IsExtern = DiagDecl->getStorageClass() == SC_Extern;
-      if (!DiagDecl->isDefined() && !IsExtern && DiagDecl->isConstexpr() &&
+      bool IsDefined = F->isDefined();
+      if (!IsDefined && !IsExtern && DiagDecl->isConstexpr() &&
           S.checkingPotentialConstantExpression())
         return false;
 
@@ -2008,6 +2008,51 @@ bool DiagTypeid(InterpState &S, CodePtr OpPC) {
   return false;
 }
 
+bool arePotentiallyOverlappingStringLiterals(const Pointer &LHS,
+                                             const Pointer &RHS) {
+  unsigned LHSOffset = LHS.getIndex();
+  unsigned RHSOffset = RHS.getIndex();
+  unsigned LHSLength = (LHS.getNumElems() - 1) * LHS.elemSize();
+  unsigned RHSLength = (RHS.getNumElems() - 1) * RHS.elemSize();
+
+  StringRef LHSStr((const char *)LHS.atIndex(0).getRawAddress(), LHSLength);
+  StringRef RHSStr((const char *)RHS.atIndex(0).getRawAddress(), RHSLength);
+  int32_t IndexDiff = RHSOffset - LHSOffset;
+  if (IndexDiff < 0) {
+    if (static_cast<int32_t>(LHSLength) < -IndexDiff)
+      return false;
+    LHSStr = LHSStr.drop_front(-IndexDiff);
+  } else {
+    if (static_cast<int32_t>(RHSLength) < IndexDiff)
+      return false;
+    RHSStr = RHSStr.drop_front(IndexDiff);
+  }
+
+  unsigned ShorterCharWidth;
+  StringRef Shorter;
+  StringRef Longer;
+  if (LHSLength < RHSLength) {
+    ShorterCharWidth = LHS.elemSize();
+    Shorter = LHSStr;
+    Longer = RHSStr;
+  } else {
+    ShorterCharWidth = RHS.elemSize();
+    Shorter = RHSStr;
+    Longer = LHSStr;
+  }
+
+  // The null terminator isn't included in the string data, so check for it
+  // manually. If the longer string doesn't have a null terminator where the
+  // shorter string ends, they aren't potentially overlapping.
+  for (unsigned NullByte : llvm::seq(ShorterCharWidth)) {
+    if (Shorter.size() + NullByte >= Longer.size())
+      break;
+    if (Longer[Shorter.size() + NullByte])
+      return false;
+  }
+  return Shorter == Longer.take_front(Shorter.size());
+}
+
 // https://github.com/llvm/llvm-project/issues/102513
 #if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
 #pragma optimize("", off)
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Interp.h b/external/llvm-project/clang/lib/AST/ByteCode/Interp.h
index 33de09a263b3..ae3d4a441a79 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Interp.h
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Interp.h
@@ -1035,6 +1035,9 @@ static inline bool IsOpaqueConstantCall(const CallExpr *E) {
           Builtin == Builtin::BI__builtin_function_start);
 }
 
+bool arePotentiallyOverlappingStringLiterals(const Pointer &LHS,
+                                             const Pointer &RHS);
+
 template <>
 inline bool CmpHelperEQ<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   using BoolT = PrimConv<PT_Bool>::T;
@@ -1069,6 +1072,18 @@ inline bool CmpHelperEQ<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
     return true;
   }
 
+  // FIXME: The source check here isn't entirely correct.
+  if (LHS.pointsToStringLiteral() && RHS.pointsToStringLiteral() &&
+      LHS.getFieldDesc()->asExpr() != RHS.getFieldDesc()->asExpr()) {
+    if (arePotentiallyOverlappingStringLiterals(LHS, RHS)) {
+      const SourceInfo &Loc = S.Current->getSource(OpPC);
+      S.FFDiag(Loc, diag::note_constexpr_literal_comparison)
+          << LHS.toDiagnosticString(S.getASTContext())
+          << RHS.toDiagnosticString(S.getASTContext());
+      return false;
+    }
+  }
+
   if (Pointer::hasSameBase(LHS, RHS)) {
     if (LHS.inUnion() && RHS.inUnion()) {
       // If the pointers point into a union, things are a little more
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/external/llvm-project/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 89c626dd3cee..d01e3d042a8b 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/external/llvm-project/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -23,7 +23,7 @@
 namespace clang {
 namespace interp {
 
-static bool isNoopBuiltin(unsigned ID) {
+LLVM_ATTRIBUTE_UNUSED static bool isNoopBuiltin(unsigned ID) {
   switch (ID) {
   case Builtin::BIas_const:
   case Builtin::BIforward:
@@ -41,25 +41,12 @@ static bool isNoopBuiltin(unsigned ID) {
   return false;
 }
 
-static unsigned callArgSize(const InterpState &S, const CallExpr *C) {
-  unsigned O = 0;
-
-  for (const Expr *E : C->arguments()) {
-    O += align(primSize(*S.getContext().classify(E)));
-  }
-
-  return O;
+static void discard(InterpStack &Stk, PrimType T) {
+  TYPE_SWITCH(T, { Stk.discard<T>(); });
 }
 
-/// Peek an integer value from the stack into an APSInt.
-static APSInt peekToAPSInt(InterpStack &Stk, PrimType T, size_t Offset = 0) {
-  if (Offset == 0)
-    Offset = align(primSize(T));
-
-  APSInt R;
-  INT_TYPE_SWITCH(T, R = Stk.peek<T>(Offset).toAPSInt());
-
-  return R;
+static APSInt popToAPSInt(InterpStack &Stk, PrimType T) {
+  INT_TYPE_SWITCH(T, return Stk.pop<T>().toAPSInt());
 }
 
 /// Pushes \p Val on the stack as the type given by \p QT.
@@ -94,68 +81,12 @@ static void pushInteger(InterpState &S, T Val, QualType QT) {
                 QT);
 }
 
-static void assignInteger(Pointer &Dest, PrimType ValueT, const APSInt &Value) {
+static void assignInteger(const Pointer &Dest, PrimType ValueT,
+                          const APSInt &Value) {
   INT_TYPE_SWITCH_NO_BOOL(
       ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
 }
 
-template <PrimType Name, class V = typename PrimConv<Name>::T>
-static bool retBI(InterpState &S, const CallExpr *Call, unsigned BuiltinID) {
-  // The return value of the function is already on the stack.
-  // Remove it, get rid of all the arguments and add it back.
-  const V &Val = S.Stk.pop<V>();
-  if (!Context::isUnevaluatedBuiltin(BuiltinID)) {
-    for (int32_t I = Call->getNumArgs() - 1; I >= 0; --I) {
-      const Expr *A = Call->getArg(I);
-      PrimType Ty = S.getContext().classify(A).value_or(PT_Ptr);
-      TYPE_SWITCH(Ty, S.Stk.discard<T>());
-    }
-  }
-  S.Stk.push<V>(Val);
-  return true;
-}
-
-static bool retPrimValue(InterpState &S, CodePtr OpPC,
-                         std::optional<PrimType> &T, const CallExpr *Call,
-                         unsigned BuiltinID) {
-  if (isNoopBuiltin(BuiltinID))
-    return true;
-
-  if (!T) {
-    if (!Context::isUnevaluatedBuiltin(BuiltinID)) {
-      for (int32_t I = Call->getNumArgs() - 1; I >= 0; --I) {
-        const Expr *A = Call->getArg(I);
-        PrimType Ty = S.getContext().classify(A).value_or(PT_Ptr);
-        TYPE_SWITCH(Ty, S.Stk.discard<T>());
-      }
-    }
-
-    return true;
-  }
-
-#define RET_CASE(X)                                                            \
-  case X:                                                                      \
-    return retBI<X>(S, Call, BuiltinID);
-  switch (*T) {
-    RET_CASE(PT_Ptr);
-    RET_CASE(PT_Float);
-    RET_CASE(PT_Bool);
-    RET_CASE(PT_Sint8);
-    RET_CASE(PT_Uint8);
-    RET_CASE(PT_Sint16);
-    RET_CASE(PT_Uint16);
-    RET_CASE(PT_Sint32);
-    RET_CASE(PT_Uint32);
-    RET_CASE(PT_Sint64);
-    RET_CASE(PT_Uint64);
-    RET_CASE(PT_IntAP);
-    RET_CASE(PT_IntAPS);
-  default:
-    llvm_unreachable("Unsupported return type for builtin function");
-  }
-#undef RET_CASE
-}
-
 static QualType getElemType(const Pointer &P) {
   const Descriptor *Desc = P.getFieldDesc();
   QualType T = Desc->getType();
@@ -214,27 +145,30 @@ static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+// __builtin_assume(int)
+static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
+                                   const InterpFrame *Frame,
+                                   const CallExpr *Call) {
+  assert(Call->getNumArgs() == 1);
+  discard(S.Stk, *S.getContext().classify(Call->getArg(0)));
+  return true;
+}
+
 static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call, unsigned ID) {
-  unsigned LimitSize =
-      Call->getNumArgs() == 2
-          ? 0
-          : align(primSize(*S.getContext().classify(Call->getArg(2))));
-  const Pointer &A =
-      S.Stk.peek<Pointer>(align(primSize(PT_Ptr)) * 2 + LimitSize);
-  const Pointer &B = S.Stk.peek<Pointer>(align(primSize(PT_Ptr)) + LimitSize);
-
-  if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
-      ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
-    diagnoseNonConstexprBuiltin(S, OpPC, ID);
-
   uint64_t Limit = ~static_cast<uint64_t>(0);
   if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
       ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp)
-    Limit = peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)))
+    Limit = popToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)))
                 .getZExtValue();
 
+  const Pointer &B = S.Stk.pop<Pointer>();
+  const Pointer &A = S.Stk.pop<Pointer>();
+  if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
+      ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
+    diagnoseNonConstexprBuiltin(S, OpPC, ID);
+
   if (Limit == 0) {
     pushInteger(S, 0, Call->getType());
     return true;
@@ -305,7 +239,7 @@ static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call, unsigned ID) {
-  const Pointer &StrPtr = S.Stk.peek<Pointer>();
+  const Pointer &StrPtr = S.Stk.pop<Pointer>();
 
   if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
     diagnoseNonConstexprBuiltin(S, OpPC, ID);
@@ -360,7 +294,7 @@ static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame, const CallExpr *Call,
                                 bool Signaling) {
-  const Pointer &Arg = S.Stk.peek<Pointer>();
+  const Pointer &Arg = S.Stk.pop<Pointer>();
 
   if (!CheckLoad(S, OpPC, Arg))
     return false;
@@ -432,8 +366,8 @@ static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
 
 static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame) {
-  const Floating &Arg1 = S.Stk.peek<Floating>(align(primSize(PT_Float)) * 2);
-  const Floating &Arg2 = S.Stk.peek<Floating>();
+  const Floating &Arg2 = S.Stk.pop<Floating>();
+  const Floating &Arg1 = S.Stk.pop<Floating>();
 
   APFloat Copy = Arg1.getAPFloat();
   Copy.copySign(Arg2.getAPFloat());
@@ -444,8 +378,8 @@ static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
 
 static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
-  const Floating &LHS = S.Stk.peek<Floating>(align(primSize(PT_Float)) * 2);
-  const Floating &RHS = S.Stk.peek<Floating>();
+  const Floating &RHS = S.Stk.pop<Floating>();
+  const Floating &LHS = S.Stk.pop<Floating>();
 
   if (IsNumBuiltin)
     S.Stk.push<Floating>(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
@@ -456,8 +390,8 @@ static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
 
 static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
-  const Floating &LHS = S.Stk.peek<Floating>(align(primSize(PT_Float)) * 2);
-  const Floating &RHS = S.Stk.peek<Floating>();
+  const Floating &RHS = S.Stk.pop<Floating>();
+  const Floating &LHS = S.Stk.pop<Floating>();
 
   if (IsNumBuiltin)
     S.Stk.push<Floating>(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
@@ -472,7 +406,7 @@ static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
 
   pushInteger(S, Arg.isNan(), Call->getType());
   return true;
@@ -481,7 +415,7 @@ static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
                                         const InterpFrame *Frame,
                                         const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
 
   pushInteger(S, Arg.isSignaling(), Call->getType());
   return true;
@@ -490,7 +424,7 @@ static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame, bool CheckSign,
                                   const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
   bool IsInf = Arg.isInf();
 
   if (CheckSign)
@@ -503,7 +437,7 @@ static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame,
                                      const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
 
   pushInteger(S, Arg.isFinite(), Call->getType());
   return true;
@@ -512,7 +446,7 @@ static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame,
                                      const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
 
   pushInteger(S, Arg.isNormal(), Call->getType());
   return true;
@@ -521,7 +455,7 @@ static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
                                         const InterpFrame *Frame,
                                         const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
 
   pushInteger(S, Arg.isDenormal(), Call->getType());
   return true;
@@ -530,7 +464,7 @@ static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
 
   pushInteger(S, Arg.isZero(), Call->getType());
   return true;
@@ -539,7 +473,7 @@ static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
                                     const InterpFrame *Frame,
                                     const CallExpr *Call) {
-  const Floating &Arg = S.Stk.peek<Floating>();
+  const Floating &Arg = S.Stk.pop<Floating>();
 
   pushInteger(S, Arg.isNegative(), Call->getType());
   return true;
@@ -547,8 +481,8 @@ static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
 
 static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
                                        const CallExpr *Call, unsigned ID) {
-  const Floating &RHS = S.Stk.peek<Floating>();
-  const Floating &LHS = S.Stk.peek<Floating>(align(2u * primSize(PT_Float)));
+  const Floating &RHS = S.Stk.pop<Floating>();
+  const Floating &LHS = S.Stk.pop<Floating>();
 
   pushInteger(
       S,
@@ -584,9 +518,8 @@ static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
                                       const InterpFrame *Frame,
                                       const CallExpr *Call) {
   PrimType FPClassArgT = *S.getContext().classify(Call->getArg(1)->getType());
-  APSInt FPClassArg = peekToAPSInt(S.Stk, FPClassArgT);
-  const Floating &F =
-      S.Stk.peek<Floating>(align(primSize(FPClassArgT) + primSize(PT_Float)));
+  APSInt FPClassArg = popToAPSInt(S.Stk, FPClassArgT);
+  const Floating &F = S.Stk.pop<Floating>();
 
   int32_t Result =
       static_cast<int32_t>((F.classify() & FPClassArg).getZExtValue());
@@ -596,10 +529,16 @@ static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
 }
 
 /// Five int values followed by one floating value.
+/// __builtin_fpclassify(int, int, int, int, int, float)
 static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const CallExpr *Call) {
-  const Floating &Val = S.Stk.peek<Floating>();
+  const Floating &Val = S.Stk.pop<Floating>();
+
+  PrimType IntT = *S.getContext().classify(Call->getArg(0));
+  APSInt Values[5];
+  for (unsigned I = 0; I != 5; ++I)
+    Values[4 - I] = popToAPSInt(S.Stk, IntT);
 
   unsigned Index;
   switch (Val.getCategory()) {
@@ -619,13 +558,8 @@ static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
 
   // The last argument is first on the stack.
   assert(Index <= 4);
-  PrimType IntT = *S.getContext().classify(Call->getArg(0));
-  unsigned IntSize = primSize(IntT);
-  unsigned Offset =
-      align(primSize(PT_Float)) + ((1 + (4 - Index)) * align(IntSize));
 
-  APSInt I = peekToAPSInt(S.Stk, IntT, Offset);
-  pushInteger(S, I, Call->getType());
+  pushInteger(S, Values[Index], Call->getType());
   return true;
 }
 
@@ -636,7 +570,7 @@ static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
 // Reference, WG14 N2478 F.10.4.3
 static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame) {
-  const Floating &Val = S.Stk.peek<Floating>();
+  const Floating &Val = S.Stk.pop<Floating>();
 
   S.Stk.push<Floating>(Floating::abs(Val));
   return true;
@@ -646,7 +580,7 @@ static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame,
                                 const CallExpr *Call) {
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt Val = peekToAPSInt(S.Stk, ArgT);
+  APSInt Val = popToAPSInt(S.Stk, ArgT);
   if (Val ==
       APSInt(APInt::getSignedMinValue(Val.getBitWidth()), /*IsUnsigned=*/false))
     return false;
@@ -660,7 +594,7 @@ static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame,
                                      const CallExpr *Call) {
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt Val = peekToAPSInt(S.Stk, ArgT);
+  APSInt Val = popToAPSInt(S.Stk, ArgT);
   pushInteger(S, Val.popcount(), Call->getType());
   return true;
 }
@@ -669,7 +603,7 @@ static bool interp__builtin_parity(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call) {
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt Val = peekToAPSInt(S.Stk, ArgT);
+  APSInt Val = popToAPSInt(S.Stk, ArgT);
   pushInteger(S, Val.popcount() % 2, Call->getType());
   return true;
 }
@@ -678,7 +612,7 @@ static bool interp__builtin_clrsb(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call) {
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt Val = peekToAPSInt(S.Stk, ArgT);
+  APSInt Val = popToAPSInt(S.Stk, ArgT);
   pushInteger(S, Val.getBitWidth() - Val.getSignificantBits(), Call->getType());
   return true;
 }
@@ -687,7 +621,7 @@ static bool interp__builtin_bitreverse(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const CallExpr *Call) {
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt Val = peekToAPSInt(S.Stk, ArgT);
+  APSInt Val = popToAPSInt(S.Stk, ArgT);
   pushInteger(S, Val.reverseBits(), Call->getType());
   return true;
 }
@@ -717,11 +651,11 @@ static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
   assert(NumArgs == 2 || NumArgs == 3);
 
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  unsigned Offset = align(ArgT) * 2;
   if (NumArgs == 3)
-    Offset += align(primSize(PT_Float));
+    S.Stk.discard<Floating>();
+  discard(S.Stk, ArgT);
 
-  APSInt Val = peekToAPSInt(S.Stk, ArgT, Offset);
+  APSInt Val = popToAPSInt(S.Stk, ArgT);
   pushInteger(S, Val, Call->getType());
   return true;
 }
@@ -733,9 +667,8 @@ static bool interp__builtin_rotate(InterpState &S, CodePtr OpPC,
   PrimType AmountT = *S.getContext().classify(Call->getArg(1)->getType());
   PrimType ValueT = *S.getContext().classify(Call->getArg(0)->getType());
 
-  APSInt Amount = peekToAPSInt(S.Stk, AmountT);
-  APSInt Value = peekToAPSInt(
-      S.Stk, ValueT, align(primSize(AmountT)) + align(primSize(ValueT)));
+  APSInt Amount = popToAPSInt(S.Stk, AmountT);
+  APSInt Value = popToAPSInt(S.Stk, ValueT);
 
   APSInt Result;
   if (Right)
@@ -753,7 +686,7 @@ static bool interp__builtin_ffs(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame,
                                 const CallExpr *Call) {
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt Value = peekToAPSInt(S.Stk, ArgT);
+  APSInt Value = popToAPSInt(S.Stk, ArgT);
 
   uint64_t N = Value.countr_zero();
   pushInteger(S, N == Value.getBitWidth() ? 0 : N + 1, Call->getType());
@@ -782,7 +715,7 @@ static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
                                                  const InterpFrame *Frame,
                                                  const CallExpr *Call) {
   PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt Arg = peekToAPSInt(S.Stk, ArgT);
+  APSInt Arg = popToAPSInt(S.Stk, ArgT);
 
   int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
       Arg.getZExtValue());
@@ -794,17 +727,14 @@ static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
                                        const CallExpr *Call,
                                        unsigned BuiltinOp) {
-  Pointer &ResultPtr = S.Stk.peek<Pointer>();
+  const Pointer &ResultPtr = S.Stk.pop<Pointer>();
   if (ResultPtr.isDummy())
     return false;
 
   PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
   PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
-  APSInt RHS = peekToAPSInt(S.Stk, RHST,
-                            align(primSize(PT_Ptr)) + align(primSize(RHST)));
-  APSInt LHS = peekToAPSInt(S.Stk, LHST,
-                            align(primSize(PT_Ptr)) + align(primSize(RHST)) +
-                                align(primSize(LHST)));
+  APSInt RHS = popToAPSInt(S.Stk, RHST);
+  APSInt LHS = popToAPSInt(S.Stk, LHST);
   QualType ResultType = Call->getArg(2)->getType()->getPointeeType();
   PrimType ResultT = *S.getContext().classify(ResultType);
   bool Overflow;
@@ -887,7 +817,7 @@ static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
     Result = Temp;
   }
 
-  // Write Result to ResultPtr and put Overflow on the stacl.
+  // Write Result to ResultPtr and put Overflow on the stack.
   assignInteger(ResultPtr, ResultT, Result);
   ResultPtr.initialize();
   assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
@@ -899,18 +829,13 @@ static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
                                     const InterpFrame *Frame,
                                     const CallExpr *Call, unsigned BuiltinOp) {
+  const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
   PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
   PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
-  PrimType CarryT = *S.getContext().classify(Call->getArg(2)->getType());
-  APSInt RHS = peekToAPSInt(S.Stk, RHST,
-                            align(primSize(PT_Ptr)) + align(primSize(CarryT)) +
-                                align(primSize(RHST)));
-  APSInt LHS =
-      peekToAPSInt(S.Stk, LHST,
-                   align(primSize(PT_Ptr)) + align(primSize(RHST)) +
-                       align(primSize(CarryT)) + align(primSize(LHST)));
-  APSInt CarryIn = peekToAPSInt(
-      S.Stk, LHST, align(primSize(PT_Ptr)) + align(primSize(CarryT)));
+  APSInt CarryIn = popToAPSInt(S.Stk, LHST);
+  APSInt RHS = popToAPSInt(S.Stk, RHST);
+  APSInt LHS = popToAPSInt(S.Stk, LHST);
+
   APSInt CarryOut;
 
   APSInt Result;
@@ -944,7 +869,6 @@ static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
   // this is consistent.
   CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
 
-  Pointer &CarryOutPtr = S.Stk.peek<Pointer>();
   QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
   PrimType CarryOutT = *S.getContext().classify(CarryOutType);
   assignInteger(CarryOutPtr, CarryOutT, CarryOut);
@@ -958,9 +882,14 @@ static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame, const CallExpr *Call,
                                 unsigned BuiltinOp) {
-  unsigned CallSize = callArgSize(S, Call);
+
+  std::optional<APSInt> Fallback;
+  if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
+    PrimType FallbackT = *S.getContext().classify(Call->getArg(1));
+    Fallback = popToAPSInt(S.Stk, FallbackT);
+  }
   PrimType ValT = *S.getContext().classify(Call->getArg(0));
-  const APSInt &Val = peekToAPSInt(S.Stk, ValT, CallSize);
+  const APSInt &Val = popToAPSInt(S.Stk, ValT);
 
   // When the argument is 0, the result of GCC builtins is undefined, whereas
   // for Microsoft intrinsics, the result is the bit-width of the argument.
@@ -969,11 +898,8 @@ static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
                          BuiltinOp != Builtin::BI__lzcnt64;
 
   if (Val == 0) {
-    if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
-      // We have a fallback parameter.
-      PrimType FallbackT = *S.getContext().classify(Call->getArg(1));
-      const APSInt &Fallback = peekToAPSInt(S.Stk, FallbackT);
-      pushInteger(S, Fallback, Call->getType());
+    if (Fallback) {
+      pushInteger(S, *Fallback, Call->getType());
       return true;
     }
 
@@ -988,16 +914,17 @@ static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame, const CallExpr *Call,
                                 unsigned BuiltinID) {
-  unsigned CallSize = callArgSize(S, Call);
+  std::optional<APSInt> Fallback;
+  if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
+    PrimType FallbackT = *S.getContext().classify(Call->getArg(1));
+    Fallback = popToAPSInt(S.Stk, FallbackT);
+  }
   PrimType ValT = *S.getContext().classify(Call->getArg(0));
-  const APSInt &Val = peekToAPSInt(S.Stk, ValT, CallSize);
+  const APSInt &Val = popToAPSInt(S.Stk, ValT);
 
   if (Val == 0) {
-    if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
-      // We have a fallback parameter.
-      PrimType FallbackT = *S.getContext().classify(Call->getArg(1));
-      const APSInt &Fallback = peekToAPSInt(S.Stk, FallbackT);
-      pushInteger(S, Fallback, Call->getType());
+    if (Fallback) {
+      pushInteger(S, *Fallback, Call->getType());
       return true;
     }
     return false;
@@ -1012,7 +939,7 @@ static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
                                   const CallExpr *Call) {
   PrimType ReturnT = *S.getContext().classify(Call->getType());
   PrimType ValT = *S.getContext().classify(Call->getArg(0));
-  const APSInt &Val = peekToAPSInt(S.Stk, ValT);
+  const APSInt &Val = popToAPSInt(S.Stk, ValT);
   assert(Val.getActiveBits() <= 64);
 
   INT_TYPE_SWITCH(ReturnT,
@@ -1022,22 +949,19 @@ static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
 
 /// bool __atomic_always_lock_free(size_t, void const volatile*)
 /// bool __atomic_is_lock_free(size_t, void const volatile*)
-/// bool __c11_atomic_is_lock_free(size_t)
 static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
                                              const InterpFrame *Frame,
                                              const CallExpr *Call,
                                              unsigned BuiltinOp) {
-  PrimType ValT = *S.getContext().classify(Call->getArg(0));
-  unsigned SizeValOffset = 0;
-  if (BuiltinOp != Builtin::BI__c11_atomic_is_lock_free)
-    SizeValOffset = align(primSize(ValT)) + align(primSize(PT_Ptr));
-  const APSInt &SizeVal = peekToAPSInt(S.Stk, ValT, SizeValOffset);
-
   auto returnBool = [&S](bool Value) -> bool {
     S.Stk.push<Boolean>(Value);
     return true;
   };
 
+  PrimType ValT = *S.getContext().classify(Call->getArg(0));
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
+  const APSInt &SizeVal = popToAPSInt(S.Stk, ValT);
+
   // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
   // of two less than or equal to the maximum inline atomic width, we know it
   // is lock-free.  If the size isn't a power of two, or greater than the
@@ -1057,13 +981,11 @@ static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
 
       // OK, we will inline appropriately-aligned operations of this size,
       // and _Atomic(T) is appropriately-aligned.
-      if (BuiltinOp == Builtin::BI__c11_atomic_is_lock_free ||
-          Size == CharUnits::One())
+      if (Size == CharUnits::One())
         return returnBool(true);
 
       // Same for null pointers.
       assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
-      const Pointer &Ptr = S.Stk.peek<Pointer>();
       if (Ptr.isZero())
         return returnBool(true);
 
@@ -1099,14 +1021,38 @@ static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
   return false;
 }
 
+/// bool __c11_atomic_is_lock_free(size_t)
+static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
+                                                    CodePtr OpPC,
+                                                    const InterpFrame *Frame,
+                                                    const CallExpr *Call) {
+  PrimType ValT = *S.getContext().classify(Call->getArg(0));
+  const APSInt &SizeVal = popToAPSInt(S.Stk, ValT);
+
+  auto returnBool = [&S](bool Value) -> bool {
+    S.Stk.push<Boolean>(Value);
+    return true;
+  };
+
+  CharUnits Size = CharUnits::fromQuantity(SizeVal.getZExtValue());
+  if (Size.isPowerOfTwo()) {
+    // Check against inlining width.
+    unsigned InlineWidthBits =
+        S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
+    if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits))
+      return returnBool(true);
+  }
+
+  return false; // returnBool(false);
+}
+
 /// __builtin_complex(Float A, float B);
 static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
                                     const InterpFrame *Frame,
                                     const CallExpr *Call) {
-  const Floating &Arg2 = S.Stk.peek<Floating>();
-  const Floating &Arg1 = S.Stk.peek<Floating>(align(primSize(PT_Float)) * 2);
-  Pointer &Result = S.Stk.peek<Pointer>(align(primSize(PT_Float)) * 2 +
-                                        align(primSize(PT_Ptr)));
+  const Floating &Arg2 = S.Stk.pop<Floating>();
+  const Floating &Arg1 = S.Stk.pop<Floating>();
+  Pointer &Result = S.Stk.peek<Pointer>();
 
   Result.atIndex(0).deref<Floating>() = Arg1;
   Result.atIndex(0).initialize();
@@ -1126,10 +1072,8 @@ static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
                                                const InterpFrame *Frame,
                                                const CallExpr *Call,
                                                unsigned BuiltinOp) {
-  unsigned CallSize = callArgSize(S, Call);
-
   PrimType AlignmentT = *S.Ctx.classify(Call->getArg(1));
-  const APSInt &Alignment = peekToAPSInt(S.Stk, AlignmentT);
+  const APSInt &Alignment = popToAPSInt(S.Stk, AlignmentT);
 
   if (Alignment < 0 || !Alignment.isPowerOf2()) {
     S.FFDiag(Call, diag::note_constexpr_invalid_alignment) << Alignment;
@@ -1148,7 +1092,7 @@ static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
   PrimType FirstArgT = *S.Ctx.classify(Call->getArg(0));
 
   if (isIntegralType(FirstArgT)) {
-    const APSInt &Src = peekToAPSInt(S.Stk, FirstArgT, CallSize);
+    const APSInt &Src = popToAPSInt(S.Stk, FirstArgT);
     APSInt Align = Alignment.extOrTrunc(Src.getBitWidth());
     if (BuiltinOp == Builtin::BI__builtin_align_up) {
       APSInt AlignedVal =
@@ -1165,7 +1109,7 @@ static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
   }
 
   assert(FirstArgT == PT_Ptr);
-  const Pointer &Ptr = S.Stk.peek<Pointer>(CallSize);
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
 
   unsigned PtrOffset = Ptr.getByteOffset();
   PtrOffset = Ptr.getIndex();
@@ -1233,25 +1177,12 @@ static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
                                            const CallExpr *Call) {
   assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
 
-  // Might be called with function pointers in C.
-  std::optional<PrimType> PtrT = S.Ctx.classify(Call->getArg(0));
-  if (PtrT != PT_Ptr)
-    return false;
-
-  unsigned ArgSize = callArgSize(S, Call);
-  const Pointer &Ptr = S.Stk.peek<Pointer>(ArgSize);
   std::optional<APSInt> ExtraOffset;
-  APSInt Alignment;
-  if (Call->getNumArgs() == 2) {
-    Alignment = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)));
-  } else {
-    PrimType AlignmentT = *S.Ctx.classify(Call->getArg(1));
-    PrimType ExtraOffsetT = *S.Ctx.classify(Call->getArg(2));
-    Alignment = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)),
-                             align(primSize(AlignmentT)) +
-                                 align(primSize(ExtraOffsetT)));
-    ExtraOffset = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(2)));
-  }
+  if (Call->getNumArgs() == 3)
+    ExtraOffset = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(2)));
+
+  APSInt Alignment = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)));
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
 
   CharUnits Align = CharUnits::fromQuantity(Alignment.getZExtValue());
 
@@ -1300,9 +1231,8 @@ static bool interp__builtin_ia32_bextr(InterpState &S, CodePtr OpPC,
 
   PrimType ValT = *S.Ctx.classify(Call->getArg(0));
   PrimType IndexT = *S.Ctx.classify(Call->getArg(1));
-  APSInt Val = peekToAPSInt(S.Stk, ValT,
-                            align(primSize(ValT)) + align(primSize(IndexT)));
-  APSInt Index = peekToAPSInt(S.Stk, IndexT);
+  APSInt Index = popToAPSInt(S.Stk, IndexT);
+  APSInt Val = popToAPSInt(S.Stk, ValT);
 
   unsigned BitWidth = Val.getBitWidth();
   uint64_t Shift = Index.extractBitsAsZExtValue(8, 0);
@@ -1333,9 +1263,8 @@ static bool interp__builtin_ia32_bzhi(InterpState &S, CodePtr OpPC,
   PrimType ValT = *S.Ctx.classify(Call->getArg(0));
   PrimType IndexT = *S.Ctx.classify(Call->getArg(1));
 
-  APSInt Val = peekToAPSInt(S.Stk, ValT,
-                            align(primSize(ValT)) + align(primSize(IndexT)));
-  APSInt Idx = peekToAPSInt(S.Stk, IndexT);
+  APSInt Idx = popToAPSInt(S.Stk, IndexT);
+  APSInt Val = popToAPSInt(S.Stk, ValT);
 
   unsigned BitWidth = Val.getBitWidth();
   uint64_t Index = Idx.extractBitsAsZExtValue(8, 0);
@@ -1355,7 +1284,7 @@ static bool interp__builtin_ia32_lzcnt(InterpState &S, CodePtr OpPC,
       !Call->getArg(0)->getType()->isIntegerType())
     return false;
 
-  APSInt Val = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(0)));
+  APSInt Val = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(0)));
   pushInteger(S, Val.countLeadingZeros(), CallType);
   return true;
 }
@@ -1368,7 +1297,7 @@ static bool interp__builtin_ia32_tzcnt(InterpState &S, CodePtr OpPC,
       !Call->getArg(0)->getType()->isIntegerType())
     return false;
 
-  APSInt Val = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(0)));
+  APSInt Val = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(0)));
   pushInteger(S, Val.countTrailingZeros(), CallType);
   return true;
 }
@@ -1383,9 +1312,8 @@ static bool interp__builtin_ia32_pdep(InterpState &S, CodePtr OpPC,
   PrimType ValT = *S.Ctx.classify(Call->getArg(0));
   PrimType MaskT = *S.Ctx.classify(Call->getArg(1));
 
-  APSInt Val =
-      peekToAPSInt(S.Stk, ValT, align(primSize(ValT)) + align(primSize(MaskT)));
-  APSInt Mask = peekToAPSInt(S.Stk, MaskT);
+  APSInt Mask = popToAPSInt(S.Stk, MaskT);
+  APSInt Val = popToAPSInt(S.Stk, ValT);
 
   unsigned BitWidth = Val.getBitWidth();
   APInt Result = APInt::getZero(BitWidth);
@@ -1393,7 +1321,7 @@ static bool interp__builtin_ia32_pdep(InterpState &S, CodePtr OpPC,
     if (Mask[I])
       Result.setBitVal(I, Val[P++]);
   }
-  pushInteger(S, Result, Call->getType());
+  pushInteger(S, std::move(Result), Call->getType());
   return true;
 }
 
@@ -1407,9 +1335,8 @@ static bool interp__builtin_ia32_pext(InterpState &S, CodePtr OpPC,
   PrimType ValT = *S.Ctx.classify(Call->getArg(0));
   PrimType MaskT = *S.Ctx.classify(Call->getArg(1));
 
-  APSInt Val =
-      peekToAPSInt(S.Stk, ValT, align(primSize(ValT)) + align(primSize(MaskT)));
-  APSInt Mask = peekToAPSInt(S.Stk, MaskT);
+  APSInt Mask = popToAPSInt(S.Stk, MaskT);
+  APSInt Val = popToAPSInt(S.Stk, ValT);
 
   unsigned BitWidth = Val.getBitWidth();
   APInt Result = APInt::getZero(BitWidth);
@@ -1417,7 +1344,7 @@ static bool interp__builtin_ia32_pext(InterpState &S, CodePtr OpPC,
     if (Mask[I])
       Result.setBitVal(P++, Val[I]);
   }
-  pushInteger(S, Result, Call->getType());
+  pushInteger(S, std::move(Result), Call->getType());
   return true;
 }
 
@@ -1432,17 +1359,14 @@ static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
       !Call->getArg(2)->getType()->isIntegerType())
     return false;
 
+  const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
+
   PrimType CarryInT = *S.getContext().classify(Call->getArg(0));
   PrimType LHST = *S.getContext().classify(Call->getArg(1));
   PrimType RHST = *S.getContext().classify(Call->getArg(2));
-  unsigned PtrSize = align(primSize(PT_Ptr));
-  APSInt CarryIn =
-      peekToAPSInt(S.Stk, CarryInT,
-                   PtrSize + align(primSize(RHST)) + align(primSize(LHST)) +
-                       align(primSize(CarryInT)));
-  APSInt LHS = peekToAPSInt(
-      S.Stk, LHST, PtrSize + align(primSize(RHST)) + align(primSize(LHST)));
-  APSInt RHS = peekToAPSInt(S.Stk, RHST, PtrSize + align(primSize(RHST)));
+  APSInt RHS = popToAPSInt(S.Stk, RHST);
+  APSInt LHS = popToAPSInt(S.Stk, LHST);
+  APSInt CarryIn = popToAPSInt(S.Stk, CarryInT);
 
   bool IsAdd = BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u32 ||
                BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u64;
@@ -1457,7 +1381,6 @@ static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
   APSInt CarryOut =
       APSInt(ExResult.extractBits(1, BitWidth), /*IsUnsigned=*/true);
 
-  Pointer &CarryOutPtr = S.Stk.peek<Pointer>();
   QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
   PrimType CarryOutT = *S.getContext().classify(CarryOutType);
   assignInteger(CarryOutPtr, CarryOutT, APSInt(Result, true));
@@ -1481,7 +1404,7 @@ static bool
 interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
                                              const InterpFrame *Frame,
                                              const CallExpr *Call) {
-  const auto &Ptr = S.Stk.peek<Pointer>();
+  const auto &Ptr = S.Stk.pop<Pointer>();
   assert(Ptr.getFieldDesc()->isPrimitiveArray());
 
   // This should be created for a StringLiteral, so should alway shold at least
@@ -1515,7 +1438,25 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
     return false;
   }
 
-  APSInt Bytes = peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(0)));
+  // We only care about the first parameter (the size), so discard all the
+  // others.
+  {
+    unsigned NumArgs = Call->getNumArgs();
+    assert(NumArgs >= 1);
+
+    // The std::nothrow_t arg never gets put on the stack.
+    if (Call->getArg(NumArgs - 1)->getType()->isNothrowT())
+      --NumArgs;
+    auto Args = llvm::ArrayRef(Call->getArgs(), Call->getNumArgs());
+    // First arg is needed.
+    Args = Args.drop_front();
+
+    // Discard the rest.
+    for (const Expr *Arg : Args)
+      discard(S.Stk, *S.getContext().classify(Arg));
+  }
+
+  APSInt Bytes = popToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(0)));
   CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(ElemType);
   assert(!ElemSize.isZero());
   // Divide the number of bytes by sizeof(ElemType), so we get the number of
@@ -1597,17 +1538,20 @@ static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
   const Expr *Source = nullptr;
   const Block *BlockToDelete = nullptr;
 
-  if (S.checkingPotentialConstantExpression())
+  if (S.checkingPotentialConstantExpression()) {
+    S.Stk.discard<Pointer>();
     return false;
+  }
 
   // This is permitted only within a call to std::allocator<T>::deallocate.
   if (!S.getStdAllocatorCaller("deallocate")) {
     S.FFDiag(Call);
+    S.Stk.discard<Pointer>();
     return true;
   }
 
   {
-    const Pointer &Ptr = S.Stk.peek<Pointer>();
+    const Pointer &Ptr = S.Stk.pop<Pointer>();
 
     if (Ptr.isZero()) {
       S.CCEDiag(Call, diag::note_constexpr_deallocate_null);
@@ -1646,14 +1590,14 @@ static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
                                              const InterpFrame *Frame,
                                              const CallExpr *Call) {
-  const Floating &Arg0 = S.Stk.peek<Floating>();
+  const Floating &Arg0 = S.Stk.pop<Floating>();
   S.Stk.push<Floating>(Arg0);
   return true;
 }
 
 static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
                                           const CallExpr *Call, unsigned ID) {
-  const Pointer &Arg = S.Stk.peek<Pointer>();
+  const Pointer &Arg = S.Stk.pop<Pointer>();
   assert(Arg.getFieldDesc()->isPrimitiveArray());
 
   QualType ElemType = Arg.getFieldDesc()->getElemQualType();
@@ -1708,15 +1652,15 @@ static bool interp__builtin_elementwise_popcount(InterpState &S, CodePtr OpPC,
   assert(Call->getNumArgs() == 1);
   if (Call->getArg(0)->getType()->isIntegerType()) {
     PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
-    APSInt Val = peekToAPSInt(S.Stk, ArgT);
+    APSInt Val = popToAPSInt(S.Stk, ArgT);
     pushInteger(S, Val.popcount(), Call->getType());
     return true;
   }
   // Otherwise, the argument must be a vector.
   assert(Call->getArg(0)->getType()->isVectorType());
-  const Pointer &Arg = S.Stk.peek<Pointer>();
+  const Pointer &Arg = S.Stk.pop<Pointer>();
   assert(Arg.getFieldDesc()->isPrimitiveArray());
-  const Pointer &Dst = S.Stk.peek<Pointer>(primSize(PT_Ptr) * 2);
+  const Pointer &Dst = S.Stk.peek<Pointer>();
   assert(Dst.getFieldDesc()->isPrimitiveArray());
   assert(Arg.getFieldDesc()->getNumElems() ==
          Dst.getFieldDesc()->getNumElems());
@@ -1743,11 +1687,9 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
   assert(Call->getNumArgs() == 3);
   const ASTContext &ASTCtx = S.getASTContext();
   PrimType SizeT = *S.getContext().classify(Call->getArg(2));
-  Pointer DestPtr = S.Stk.peek<Pointer>(align(primSize(SizeT)) +
-                                        (align(primSize(PT_Ptr)) * 2));
-  const Pointer SrcPtr = S.Stk.peek<Pointer>(align(primSize(SizeT)) +
-                                             (align(primSize(PT_Ptr)) * 1));
-  APSInt Size = peekToAPSInt(S.Stk, SizeT);
+  APSInt Size = popToAPSInt(S.Stk, SizeT);
+  const Pointer SrcPtr = S.Stk.pop<Pointer>();
+  const Pointer DestPtr = S.Stk.pop<Pointer>();
 
   assert(!Size.isSigned() && "memcpy and friends take an unsigned size");
 
@@ -1902,11 +1844,9 @@ static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
                                    const CallExpr *Call, unsigned ID) {
   assert(Call->getNumArgs() == 3);
   PrimType SizeT = *S.getContext().classify(Call->getArg(2));
-  const APSInt &Size = peekToAPSInt(S.Stk, SizeT);
-  const Pointer &PtrA = S.Stk.peek<Pointer>(align(primSize(SizeT)) +
-                                            (align(primSize(PT_Ptr)) * 2));
-  const Pointer &PtrB = S.Stk.peek<Pointer>(align(primSize(SizeT)) +
-                                            (align(primSize(PT_Ptr)) * 1));
+  const APSInt &Size = popToAPSInt(S.Stk, SizeT);
+  const Pointer &PtrB = S.Stk.pop<Pointer>();
+  const Pointer &PtrA = S.Stk.pop<Pointer>();
 
   if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
       ID == Builtin::BIwmemcmp)
@@ -2008,27 +1948,22 @@ static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
   return false;
 }
 
+// __builtin_memchr(ptr, int, int)
+// __builtin_strchr(ptr, int)
 static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
                                    const CallExpr *Call, unsigned ID) {
   if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
       ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
     diagnoseNonConstexprBuiltin(S, OpPC, ID);
 
-  unsigned ArgOffset = 0;
-  APSInt Desired;
   std::optional<APSInt> MaxLength;
   PrimType DesiredT = *S.getContext().classify(Call->getArg(1));
   if (Call->getNumArgs() == 3) {
     PrimType MaxT = *S.getContext().classify(Call->getArg(2));
-    MaxLength = peekToAPSInt(S.Stk, MaxT);
-
-    ArgOffset += align(primSize(MaxT)) + align(primSize(DesiredT));
-    Desired = peekToAPSInt(S.Stk, DesiredT, ArgOffset);
-  } else {
-    Desired = peekToAPSInt(S.Stk, DesiredT);
-    ArgOffset += align(primSize(DesiredT));
+    MaxLength = popToAPSInt(S.Stk, MaxT);
   }
-  const Pointer &Ptr = S.Stk.peek<Pointer>(ArgOffset + align(primSize(PT_Ptr)));
+  APSInt Desired = popToAPSInt(S.Stk, DesiredT);
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
 
   if (MaxLength && MaxLength->isZero()) {
     S.Stk.push<Pointer>();
@@ -2182,12 +2117,11 @@ static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
                                         const InterpFrame *Frame,
                                         const CallExpr *Call) {
   PrimType KindT = *S.getContext().classify(Call->getArg(1));
-  [[maybe_unused]] unsigned Kind = peekToAPSInt(S.Stk, KindT).getZExtValue();
+  [[maybe_unused]] unsigned Kind = popToAPSInt(S.Stk, KindT).getZExtValue();
 
   assert(Kind <= 3 && "unexpected kind");
 
-  const Pointer &Ptr =
-      S.Stk.peek<Pointer>(align(primSize(KindT)) + align(primSize(PT_Ptr)));
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
 
   if (Ptr.isZero())
     return false;
@@ -2212,7 +2146,7 @@ static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
   if (!S.inConstantContext())
     return false;
 
-  const Pointer &Ptr = S.Stk.peek<Pointer>();
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
 
   auto Error = [&](int Diag) {
     bool CalledFromStd = false;
@@ -2256,17 +2190,14 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
     return Invalid(S, OpPC);
 
   const InterpFrame *Frame = S.Current;
-
-  std::optional<PrimType> ReturnT = S.getContext().classify(Call);
-
   switch (BuiltinID) {
   case Builtin::BI__builtin_is_constant_evaluated:
-    if (!interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_assume:
   case Builtin::BI__assume:
-    break;
+    return interp__builtin_assume(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_strcmp:
   case Builtin::BIstrcmp:
   case Builtin::BI__builtin_strncmp:
@@ -2275,32 +2206,27 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BIwcsncmp:
   case Builtin::BI__builtin_wcscmp:
   case Builtin::BIwcscmp:
-    if (!interp__builtin_strcmp(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_strcmp(S, OpPC, Frame, Call, BuiltinID);
+
   case Builtin::BI__builtin_strlen:
   case Builtin::BIstrlen:
   case Builtin::BI__builtin_wcslen:
   case Builtin::BIwcslen:
-    if (!interp__builtin_strlen(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_strlen(S, OpPC, Frame, Call, BuiltinID);
+
   case Builtin::BI__builtin_nan:
   case Builtin::BI__builtin_nanf:
   case Builtin::BI__builtin_nanl:
   case Builtin::BI__builtin_nanf16:
   case Builtin::BI__builtin_nanf128:
-    if (!interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false))
-      return false;
-    break;
+    return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
+
   case Builtin::BI__builtin_nans:
   case Builtin::BI__builtin_nansf:
   case Builtin::BI__builtin_nansl:
   case Builtin::BI__builtin_nansf16:
   case Builtin::BI__builtin_nansf128:
-    if (!interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true))
-      return false;
-    break;
+    return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
 
   case Builtin::BI__builtin_huge_val:
   case Builtin::BI__builtin_huge_valf:
@@ -2312,126 +2238,95 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI__builtin_infl:
   case Builtin::BI__builtin_inff16:
   case Builtin::BI__builtin_inff128:
-    if (!interp__builtin_inf(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_inf(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_copysign:
   case Builtin::BI__builtin_copysignf:
   case Builtin::BI__builtin_copysignl:
   case Builtin::BI__builtin_copysignf128:
-    if (!interp__builtin_copysign(S, OpPC, Frame))
-      return false;
-    break;
+    return interp__builtin_copysign(S, OpPC, Frame);
 
   case Builtin::BI__builtin_fmin:
   case Builtin::BI__builtin_fminf:
   case Builtin::BI__builtin_fminl:
   case Builtin::BI__builtin_fminf16:
   case Builtin::BI__builtin_fminf128:
-    if (!interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false))
-      return false;
-    break;
+    return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
 
   case Builtin::BI__builtin_fminimum_num:
   case Builtin::BI__builtin_fminimum_numf:
   case Builtin::BI__builtin_fminimum_numl:
   case Builtin::BI__builtin_fminimum_numf16:
   case Builtin::BI__builtin_fminimum_numf128:
-    if (!interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true))
-      return false;
-    break;
+    return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
 
   case Builtin::BI__builtin_fmax:
   case Builtin::BI__builtin_fmaxf:
   case Builtin::BI__builtin_fmaxl:
   case Builtin::BI__builtin_fmaxf16:
   case Builtin::BI__builtin_fmaxf128:
-    if (!interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false))
-      return false;
-    break;
+    return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
 
   case Builtin::BI__builtin_fmaximum_num:
   case Builtin::BI__builtin_fmaximum_numf:
   case Builtin::BI__builtin_fmaximum_numl:
   case Builtin::BI__builtin_fmaximum_numf16:
   case Builtin::BI__builtin_fmaximum_numf128:
-    if (!interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true))
-      return false;
-    break;
+    return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
 
   case Builtin::BI__builtin_isnan:
-    if (!interp__builtin_isnan(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_isnan(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_issignaling:
-    if (!interp__builtin_issignaling(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_issignaling(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_isinf:
-    if (!interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/false, Call))
-      return false;
-    break;
+    return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/false, Call);
 
   case Builtin::BI__builtin_isinf_sign:
-    if (!interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/true, Call))
-      return false;
-    break;
+    return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/true, Call);
 
   case Builtin::BI__builtin_isfinite:
-    if (!interp__builtin_isfinite(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_isfinite(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_isnormal:
-    if (!interp__builtin_isnormal(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_isnormal(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_issubnormal:
-    if (!interp__builtin_issubnormal(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_issubnormal(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_iszero:
-    if (!interp__builtin_iszero(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_iszero(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_signbit:
   case Builtin::BI__builtin_signbitf:
   case Builtin::BI__builtin_signbitl:
-    if (!interp__builtin_signbit(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_signbit(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_isgreater:
   case Builtin::BI__builtin_isgreaterequal:
   case Builtin::BI__builtin_isless:
   case Builtin::BI__builtin_islessequal:
   case Builtin::BI__builtin_islessgreater:
   case Builtin::BI__builtin_isunordered:
-    if (!interp_floating_comparison(S, OpPC, Call, BuiltinID))
-      return false;
-    break;
+    return interp_floating_comparison(S, OpPC, Call, BuiltinID);
+
   case Builtin::BI__builtin_isfpclass:
-    if (!interp__builtin_isfpclass(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_isfpclass(S, OpPC, Frame, Call);
+
   case Builtin::BI__builtin_fpclassify:
-    if (!interp__builtin_fpclassify(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_fpclassify(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_fabs:
   case Builtin::BI__builtin_fabsf:
   case Builtin::BI__builtin_fabsl:
   case Builtin::BI__builtin_fabsf128:
-    if (!interp__builtin_fabs(S, OpPC, Frame))
-      return false;
-    break;
+    return interp__builtin_fabs(S, OpPC, Frame);
 
   case Builtin::BI__builtin_abs:
   case Builtin::BI__builtin_labs:
   case Builtin::BI__builtin_llabs:
-    if (!interp__builtin_abs(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_abs(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_popcount:
   case Builtin::BI__builtin_popcountl:
@@ -2440,42 +2335,30 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI__popcnt16: // Microsoft variants of popcount
   case Builtin::BI__popcnt:
   case Builtin::BI__popcnt64:
-    if (!interp__builtin_popcount(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_popcount(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_parity:
   case Builtin::BI__builtin_parityl:
   case Builtin::BI__builtin_parityll:
-    if (!interp__builtin_parity(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_parity(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_clrsb:
   case Builtin::BI__builtin_clrsbl:
   case Builtin::BI__builtin_clrsbll:
-    if (!interp__builtin_clrsb(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_clrsb(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_bitreverse8:
   case Builtin::BI__builtin_bitreverse16:
   case Builtin::BI__builtin_bitreverse32:
   case Builtin::BI__builtin_bitreverse64:
-    if (!interp__builtin_bitreverse(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_bitreverse(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_classify_type:
-    if (!interp__builtin_classify_type(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_classify_type(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_expect:
   case Builtin::BI__builtin_expect_with_probability:
-    if (!interp__builtin_expect(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_expect(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_rotateleft8:
   case Builtin::BI__builtin_rotateleft16:
@@ -2486,9 +2369,7 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI_rotl:
   case Builtin::BI_lrotl:
   case Builtin::BI_rotl64:
-    if (!interp__builtin_rotate(S, OpPC, Frame, Call, /*Right=*/false))
-      return false;
-    break;
+    return interp__builtin_rotate(S, OpPC, Frame, Call, /*Right=*/false);
 
   case Builtin::BI__builtin_rotateright8:
   case Builtin::BI__builtin_rotateright16:
@@ -2499,23 +2380,18 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI_rotr:
   case Builtin::BI_lrotr:
   case Builtin::BI_rotr64:
-    if (!interp__builtin_rotate(S, OpPC, Frame, Call, /*Right=*/true))
-      return false;
-    break;
+    return interp__builtin_rotate(S, OpPC, Frame, Call, /*Right=*/true);
 
   case Builtin::BI__builtin_ffs:
   case Builtin::BI__builtin_ffsl:
   case Builtin::BI__builtin_ffsll:
-    if (!interp__builtin_ffs(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ffs(S, OpPC, Frame, Call);
+
   case Builtin::BIaddressof:
   case Builtin::BI__addressof:
   case Builtin::BI__builtin_addressof:
     assert(isNoopBuiltin(BuiltinID));
-    if (!interp__builtin_addressof(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_addressof(S, OpPC, Frame, Call);
 
   case Builtin::BIas_const:
   case Builtin::BIforward:
@@ -2523,18 +2399,14 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BImove:
   case Builtin::BImove_if_noexcept:
     assert(isNoopBuiltin(BuiltinID));
-    if (!interp__builtin_move(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_move(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_eh_return_data_regno:
-    if (!interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_launder:
     assert(isNoopBuiltin(BuiltinID));
-    break;
+    return true;
 
   case Builtin::BI__builtin_add_overflow:
   case Builtin::BI__builtin_sub_overflow:
@@ -2557,9 +2429,7 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI__builtin_smul_overflow:
   case Builtin::BI__builtin_smull_overflow:
   case Builtin::BI__builtin_smulll_overflow:
-    if (!interp__builtin_overflowop(S, OpPC, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_overflowop(S, OpPC, Call, BuiltinID);
 
   case Builtin::BI__builtin_addcb:
   case Builtin::BI__builtin_addcs:
@@ -2571,9 +2441,7 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI__builtin_subc:
   case Builtin::BI__builtin_subcl:
   case Builtin::BI__builtin_subcll:
-    if (!interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinID);
 
   case Builtin::BI__builtin_clz:
   case Builtin::BI__builtin_clzl:
@@ -2583,141 +2451,101 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
   case Builtin::BI__lzcnt:
   case Builtin::BI__lzcnt64:
-    if (!interp__builtin_clz(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinID);
 
   case Builtin::BI__builtin_ctz:
   case Builtin::BI__builtin_ctzl:
   case Builtin::BI__builtin_ctzll:
   case Builtin::BI__builtin_ctzs:
   case Builtin::BI__builtin_ctzg:
-    if (!interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
 
   case Builtin::BI__builtin_bswap16:
   case Builtin::BI__builtin_bswap32:
   case Builtin::BI__builtin_bswap64:
-    if (!interp__builtin_bswap(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_bswap(S, OpPC, Frame, Call);
 
   case Builtin::BI__atomic_always_lock_free:
   case Builtin::BI__atomic_is_lock_free:
+    return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinID);
+
   case Builtin::BI__c11_atomic_is_lock_free:
-    if (!interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_complex:
-    if (!interp__builtin_complex(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_complex(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_is_aligned:
   case Builtin::BI__builtin_align_up:
   case Builtin::BI__builtin_align_down:
-    if (!interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinID);
 
   case Builtin::BI__builtin_assume_aligned:
-    if (!interp__builtin_assume_aligned(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
 
   case clang::X86::BI__builtin_ia32_bextr_u32:
   case clang::X86::BI__builtin_ia32_bextr_u64:
   case clang::X86::BI__builtin_ia32_bextri_u32:
   case clang::X86::BI__builtin_ia32_bextri_u64:
-    if (!interp__builtin_ia32_bextr(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ia32_bextr(S, OpPC, Frame, Call);
 
   case clang::X86::BI__builtin_ia32_bzhi_si:
   case clang::X86::BI__builtin_ia32_bzhi_di:
-    if (!interp__builtin_ia32_bzhi(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ia32_bzhi(S, OpPC, Frame, Call);
 
   case clang::X86::BI__builtin_ia32_lzcnt_u16:
   case clang::X86::BI__builtin_ia32_lzcnt_u32:
   case clang::X86::BI__builtin_ia32_lzcnt_u64:
-    if (!interp__builtin_ia32_lzcnt(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ia32_lzcnt(S, OpPC, Frame, Call);
 
   case clang::X86::BI__builtin_ia32_tzcnt_u16:
   case clang::X86::BI__builtin_ia32_tzcnt_u32:
   case clang::X86::BI__builtin_ia32_tzcnt_u64:
-    if (!interp__builtin_ia32_tzcnt(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ia32_tzcnt(S, OpPC, Frame, Call);
 
   case clang::X86::BI__builtin_ia32_pdep_si:
   case clang::X86::BI__builtin_ia32_pdep_di:
-    if (!interp__builtin_ia32_pdep(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ia32_pdep(S, OpPC, Frame, Call);
 
   case clang::X86::BI__builtin_ia32_pext_si:
   case clang::X86::BI__builtin_ia32_pext_di:
-    if (!interp__builtin_ia32_pext(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ia32_pext(S, OpPC, Frame, Call);
 
   case clang::X86::BI__builtin_ia32_addcarryx_u32:
   case clang::X86::BI__builtin_ia32_addcarryx_u64:
   case clang::X86::BI__builtin_ia32_subborrow_u32:
   case clang::X86::BI__builtin_ia32_subborrow_u64:
-    if (!interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
-                                                 BuiltinID))
-      return false;
-    break;
+    return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
+                                                   BuiltinID);
 
   case Builtin::BI__builtin_os_log_format_buffer_size:
-    if (!interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_ptrauth_string_discriminator:
-    if (!interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
 
   case Builtin::BI__noop:
     pushInteger(S, 0, Call->getType());
-    break;
+    return true;
 
   case Builtin::BI__builtin_operator_new:
-    if (!interp__builtin_operator_new(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_operator_new(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_operator_delete:
-    if (!interp__builtin_operator_delete(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_operator_delete(S, OpPC, Frame, Call);
 
   case Builtin::BI__arithmetic_fence:
-    if (!interp__builtin_arithmetic_fence(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_reduce_add:
   case Builtin::BI__builtin_reduce_mul:
   case Builtin::BI__builtin_reduce_and:
   case Builtin::BI__builtin_reduce_or:
   case Builtin::BI__builtin_reduce_xor:
-    if (!interp__builtin_vector_reduce(S, OpPC, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_vector_reduce(S, OpPC, Call, BuiltinID);
 
   case Builtin::BI__builtin_elementwise_popcount:
-    if (!interp__builtin_elementwise_popcount(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_elementwise_popcount(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_memcpy:
   case Builtin::BImemcpy:
@@ -2727,9 +2555,7 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BImemmove:
   case Builtin::BI__builtin_wmemmove:
   case Builtin::BIwmemmove:
-    if (!interp__builtin_memcpy(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_memcpy(S, OpPC, Frame, Call, BuiltinID);
 
   case Builtin::BI__builtin_memcmp:
   case Builtin::BImemcmp:
@@ -2737,9 +2563,7 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BIbcmp:
   case Builtin::BI__builtin_wmemcmp:
   case Builtin::BIwmemcmp:
-    if (!interp__builtin_memcmp(S, OpPC, Frame, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_memcmp(S, OpPC, Frame, Call, BuiltinID);
 
   case Builtin::BImemchr:
   case Builtin::BI__builtin_memchr:
@@ -2750,20 +2574,14 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BIwcschr:
   case Builtin::BI__builtin_wcschr:
   case Builtin::BI__builtin_char_memchr:
-    if (!interp__builtin_memchr(S, OpPC, Call, BuiltinID))
-      return false;
-    break;
+    return interp__builtin_memchr(S, OpPC, Call, BuiltinID);
 
   case Builtin::BI__builtin_object_size:
   case Builtin::BI__builtin_dynamic_object_size:
-    if (!interp__builtin_object_size(S, OpPC, Frame, Call))
-      return false;
-    break;
+    return interp__builtin_object_size(S, OpPC, Frame, Call);
 
   case Builtin::BI__builtin_is_within_lifetime:
-    if (!interp__builtin_is_within_lifetime(S, OpPC, Call))
-      return false;
-    break;
+    return interp__builtin_is_within_lifetime(S, OpPC, Call);
 
   default:
     S.FFDiag(S.Current->getLocation(OpPC),
@@ -2773,7 +2591,7 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
     return false;
   }
 
-  return retPrimValue(S, OpPC, ReturnT, Call, BuiltinID);
+  llvm_unreachable("Unhandled builtin ID");
 }
 
 bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Opcodes.td b/external/llvm-project/clang/lib/AST/ByteCode/Opcodes.td
index 7031d7026fac..c76ac5f8ae86 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Opcodes.td
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Opcodes.td
@@ -93,6 +93,10 @@ def FixedSizeIntegralTypeClass : TypeClass {
                Uint32, Sint64, Uint64, Bool];
 }
 
+def FixedSizeIntegralNoBoolTypeClass : TypeClass {
+  let Types = [Sint8, Uint8, Sint16, Uint16, Sint32, Uint32, Sint64, Uint64];
+}
+
 def NumberTypeClass : TypeClass {
   let Types = !listconcat(IntegerTypeClass.Types, [Float]);
 }
@@ -650,10 +654,6 @@ def CastFixedPoint : Opcode {
   let Args = [ArgUint32];
 }
 
-def FixedSizeIntegralTypes : TypeClass {
-  let Types = [Uint8, Sint8, Uint16, Sint16, Uint32, Sint32, Uint64, Sint64, Bool];
-}
-
 def CastAP : Opcode {
   let Types = [AluTypeClass];
   let Args = [ArgUint32];
@@ -675,7 +675,7 @@ def CastIntegralFloating : Opcode {
 
 // Cast a floating to an integer type
 def CastFloatingIntegral : Opcode {
-  let Types = [FixedSizeIntegralTypes];
+  let Types = [FixedSizeIntegralTypeClass];
   let Args = [ArgUint32];
   let HasGroup = 1;
 }
@@ -699,7 +699,7 @@ def CastPointerIntegralAPS : Opcode {
   let Args = [ArgUint32];
 }
 def CastIntegralFixedPoint : Opcode {
-  let Types = [FixedSizeIntegralTypes];
+  let Types = [FixedSizeIntegralTypeClass];
   let Args = [ArgUint32];
   let HasGroup = 1;
 }
@@ -710,7 +710,7 @@ def CastFixedPointFloating : Opcode {
   let Args = [ArgFltSemantics];
 }
 def CastFixedPointIntegral : Opcode {
-  let Types = [FixedSizeIntegralTypes];
+  let Types = [FixedSizeIntegralNoBoolTypeClass];
   let HasGroup = 1;
 }
 def ShiftFixedPoint : Opcode {
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Pointer.cpp b/external/llvm-project/clang/lib/AST/ByteCode/Pointer.cpp
index 6c2566ba20bd..50453c72c582 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Pointer.cpp
@@ -571,6 +571,17 @@ bool Pointer::pointsToLiteral() const {
   return E && !isa<MaterializeTemporaryExpr, StringLiteral>(E);
 }
 
+bool Pointer::pointsToStringLiteral() const {
+  if (isZero() || !isBlockPointer())
+    return false;
+
+  if (block()->isDynamic())
+    return false;
+
+  const Expr *E = block()->getDescriptor()->asExpr();
+  return E && isa<StringLiteral>(E);
+}
+
 std::optional<std::pair<Pointer, Pointer>>
 Pointer::computeSplitPoint(const Pointer &A, const Pointer &B) {
   if (!A.isBlockPointer() || !B.isBlockPointer())
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Pointer.h b/external/llvm-project/clang/lib/AST/ByteCode/Pointer.h
index 479da0900468..0234ab02ab8f 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Pointer.h
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Pointer.h
@@ -756,6 +756,7 @@ class Pointer {
   /// Whether this points to a block that's been created for a "literal lvalue",
   /// i.e. a non-MaterializeTemporaryExpr Expr.
   bool pointsToLiteral() const;
+  bool pointsToStringLiteral() const;
 
   /// Prints the pointer.
   void print(llvm::raw_ostream &OS) const;
diff --git a/external/llvm-project/clang/lib/AST/ByteCode/Source.h b/external/llvm-project/clang/lib/AST/ByteCode/Source.h
index c74cda919347..d5d294e31d90 100644
--- a/external/llvm-project/clang/lib/AST/ByteCode/Source.h
+++ b/external/llvm-project/clang/lib/AST/ByteCode/Source.h
@@ -45,10 +45,6 @@ class CodePtr final {
     assert(Ptr != nullptr && "Invalid code pointer");
     return CodePtr(Ptr - RHS);
   }
-  CodePtr operator+(ssize_t RHS) const {
-    assert(Ptr != nullptr && "Invalid code pointer");
-    return CodePtr(Ptr + RHS);
-  }
 
   bool operator!=(const CodePtr &RHS) const { return Ptr != RHS.Ptr; }
   const std::byte *operator*() const { return Ptr; }
diff --git a/external/llvm-project/clang/lib/AST/Decl.cpp b/external/llvm-project/clang/lib/AST/Decl.cpp
index 8425e40567b2..860968939b4a 100644
--- a/external/llvm-project/clang/lib/AST/Decl.cpp
+++ b/external/llvm-project/clang/lib/AST/Decl.cpp
@@ -3541,7 +3541,7 @@ bool FunctionDecl::isExternC() const {
 }
 
 bool FunctionDecl::isInExternCContext() const {
-  if (hasAttr<OpenCLKernelAttr>())
+  if (DeviceKernelAttr::isOpenCLSpelling(getAttr<DeviceKernelAttr>()))
     return true;
   return getLexicalDeclContext()->isExternCContext();
 }
@@ -5510,7 +5510,8 @@ FunctionDecl *FunctionDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
 }
 
 bool FunctionDecl::isReferenceableKernel() const {
-  return hasAttr<CUDAGlobalAttr>() || hasAttr<OpenCLKernelAttr>();
+  return hasAttr<CUDAGlobalAttr>() ||
+         DeviceKernelAttr::isOpenCLSpelling(getAttr<DeviceKernelAttr>());
 }
 
 BlockDecl *BlockDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L) {
@@ -5712,8 +5713,7 @@ SourceRange TypeAliasDecl::getSourceRange() const {
 void FileScopeAsmDecl::anchor() {}
 
 FileScopeAsmDecl *FileScopeAsmDecl::Create(ASTContext &C, DeclContext *DC,
-                                           StringLiteral *Str,
-                                           SourceLocation AsmLoc,
+                                           Expr *Str, SourceLocation AsmLoc,
                                            SourceLocation RParenLoc) {
   return new (C, DC) FileScopeAsmDecl(DC, Str, AsmLoc, RParenLoc);
 }
@@ -5724,6 +5724,10 @@ FileScopeAsmDecl *FileScopeAsmDecl::CreateDeserialized(ASTContext &C,
                                       SourceLocation());
 }
 
+std::string FileScopeAsmDecl::getAsmString() const {
+  return GCCAsmStmt::ExtractStringFromGCCAsmStmtComponent(getAsmStringExpr());
+}
+
 void TopLevelStmtDecl::anchor() {}
 
 TopLevelStmtDecl *TopLevelStmtDecl::Create(ASTContext &C, Stmt *Statement) {
diff --git a/external/llvm-project/clang/lib/AST/DeclBase.cpp b/external/llvm-project/clang/lib/AST/DeclBase.cpp
index a1bb62bcb68f..48c60aa4e449 100644
--- a/external/llvm-project/clang/lib/AST/DeclBase.cpp
+++ b/external/llvm-project/clang/lib/AST/DeclBase.cpp
@@ -1132,6 +1132,12 @@ bool Decl::isInExportDeclContext() const {
   return isa_and_nonnull<ExportDecl>(DC);
 }
 
+bool Decl::isModuleLocal() const {
+  auto *M = getOwningModule();
+  return M && M->isNamedModule() &&
+         getModuleOwnershipKind() == ModuleOwnershipKind::ReachableWhenImported;
+}
+
 bool Decl::isInAnotherModuleUnit() const {
   auto *M = getOwningModule();
 
diff --git a/external/llvm-project/clang/lib/AST/DeclPrinter.cpp b/external/llvm-project/clang/lib/AST/DeclPrinter.cpp
index 3616419dce4e..944385744334 100644
--- a/external/llvm-project/clang/lib/AST/DeclPrinter.cpp
+++ b/external/llvm-project/clang/lib/AST/DeclPrinter.cpp
@@ -1042,8 +1042,8 @@ void DeclPrinter::VisitParmVarDecl(ParmVarDecl *D) {
 
 void DeclPrinter::VisitFileScopeAsmDecl(FileScopeAsmDecl *D) {
   Out << "__asm (";
-  D->getAsmString()->printPretty(Out, nullptr, Policy, Indentation, "\n",
-                                 &Context);
+  D->getAsmStringExpr()->printPretty(Out, nullptr, Policy, Indentation, "\n",
+                                     &Context);
   Out << ")";
 }
 
diff --git a/external/llvm-project/clang/lib/AST/Expr.cpp b/external/llvm-project/clang/lib/AST/Expr.cpp
index e95eaeef5970..7a11b9e55d1a 100644
--- a/external/llvm-project/clang/lib/AST/Expr.cpp
+++ b/external/llvm-project/clang/lib/AST/Expr.cpp
@@ -3611,7 +3611,6 @@ bool Expr::HasSideEffects(const ASTContext &Ctx,
   case PackExpansionExprClass:
   case SubstNonTypeTemplateParmPackExprClass:
   case FunctionParmPackExprClass:
-  case TypoExprClass:
   case RecoveryExprClass:
   case CXXFoldExprClass:
     // Make a conservative assumption for dependent nodes.
@@ -4393,7 +4392,8 @@ ShuffleVectorExpr::ShuffleVectorExpr(const ASTContext &C, ArrayRef<Expr *> args,
                                      QualType Type, SourceLocation BLoc,
                                      SourceLocation RP)
     : Expr(ShuffleVectorExprClass, Type, VK_PRValue, OK_Ordinary),
-      BuiltinLoc(BLoc), RParenLoc(RP), NumExprs(args.size()) {
+      BuiltinLoc(BLoc), RParenLoc(RP) {
+  ShuffleVectorExprBits.NumExprs = args.size();
   SubExprs = new (C) Stmt*[args.size()];
   for (unsigned i = 0; i != args.size(); i++)
     SubExprs[i] = args[i];
@@ -4404,8 +4404,8 @@ ShuffleVectorExpr::ShuffleVectorExpr(const ASTContext &C, ArrayRef<Expr *> args,
 void ShuffleVectorExpr::setExprs(const ASTContext &C, ArrayRef<Expr *> Exprs) {
   if (SubExprs) C.Deallocate(SubExprs);
 
-  this->NumExprs = Exprs.size();
-  SubExprs = new (C) Stmt*[NumExprs];
+  this->ShuffleVectorExprBits.NumExprs = Exprs.size();
+  SubExprs = new (C) Stmt *[ShuffleVectorExprBits.NumExprs];
   memcpy(SubExprs, Exprs.data(), sizeof(Expr *) * Exprs.size());
 }
 
diff --git a/external/llvm-project/clang/lib/AST/ExprCXX.cpp b/external/llvm-project/clang/lib/AST/ExprCXX.cpp
index 5c712e146e5a..43b1c39d7379 100644
--- a/external/llvm-project/clang/lib/AST/ExprCXX.cpp
+++ b/external/llvm-project/clang/lib/AST/ExprCXX.cpp
@@ -1067,7 +1067,7 @@ CXXDefaultInitExpr::CXXDefaultInitExpr(const ASTContext &Ctx,
   CXXDefaultInitExprBits.HasRewrittenInit = RewrittenInitExpr != nullptr;
 
   if (CXXDefaultInitExprBits.HasRewrittenInit)
-    *getTrailingObjects<Expr *>() = RewrittenInitExpr;
+    *getTrailingObjects() = RewrittenInitExpr;
 
   assert(Field->hasInClassInitializer());
 
@@ -1437,8 +1437,7 @@ ExprWithCleanups::ExprWithCleanups(Expr *subexpr,
     : FullExpr(ExprWithCleanupsClass, subexpr) {
   ExprWithCleanupsBits.CleanupsHaveSideEffects = CleanupsHaveSideEffects;
   ExprWithCleanupsBits.NumObjects = objects.size();
-  for (unsigned i = 0, e = objects.size(); i != e; ++i)
-    getTrailingObjects<CleanupObject>()[i] = objects[i];
+  llvm::copy(objects, getTrailingObjects());
 }
 
 ExprWithCleanups *ExprWithCleanups::Create(const ASTContext &C, Expr *subexpr,
@@ -1474,7 +1473,7 @@ CXXUnresolvedConstructExpr::CXXUnresolvedConstructExpr(
       TypeAndInitForm(TSI, IsListInit), LParenLoc(LParenLoc),
       RParenLoc(RParenLoc) {
   CXXUnresolvedConstructExprBits.NumArgs = Args.size();
-  auto **StoredArgs = getTrailingObjects<Expr *>();
+  auto **StoredArgs = getTrailingObjects();
   for (unsigned I = 0; I != Args.size(); ++I)
     StoredArgs[I] = Args[I];
   setDependence(computeDependence(this));
@@ -1800,8 +1799,7 @@ FunctionParmPackExpr::FunctionParmPackExpr(QualType T, ValueDecl *ParamPack,
     : Expr(FunctionParmPackExprClass, T, VK_LValue, OK_Ordinary),
       ParamPack(ParamPack), NameLoc(NameLoc), NumParameters(NumParams) {
   if (Params)
-    std::uninitialized_copy(Params, Params + NumParams,
-                            getTrailingObjects<ValueDecl *>());
+    std::uninitialized_copy(Params, Params + NumParams, getTrailingObjects());
   setDependence(ExprDependence::TypeValueInstantiation |
                 ExprDependence::UnexpandedPack);
 }
@@ -2003,7 +2001,8 @@ CXXFoldExpr::CXXFoldExpr(QualType T, UnresolvedLookupExpr *Callee,
                          UnsignedOrNone NumExpansions)
     : Expr(CXXFoldExprClass, T, VK_PRValue, OK_Ordinary), LParenLoc(LParenLoc),
       EllipsisLoc(EllipsisLoc), RParenLoc(RParenLoc),
-      NumExpansions(NumExpansions), Opcode(Opcode) {
+      NumExpansions(NumExpansions) {
+  CXXFoldExprBits.Opcode = Opcode;
   // We rely on asserted invariant to distinguish left and right folds.
   assert(((LHS && LHS->containsUnexpandedParameterPack()) !=
           (RHS && RHS->containsUnexpandedParameterPack())) &&
diff --git a/external/llvm-project/clang/lib/AST/ExprClassification.cpp b/external/llvm-project/clang/lib/AST/ExprClassification.cpp
index 3f37d06cc8f3..ad66335138a4 100644
--- a/external/llvm-project/clang/lib/AST/ExprClassification.cpp
+++ b/external/llvm-project/clang/lib/AST/ExprClassification.cpp
@@ -129,7 +129,6 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
     // FIXME: Is this wise? Should they get their own kind?
   case Expr::UnresolvedLookupExprClass:
   case Expr::UnresolvedMemberExprClass:
-  case Expr::TypoExprClass:
   case Expr::DependentCoawaitExprClass:
   case Expr::CXXDependentScopeMemberExprClass:
   case Expr::DependentScopeDeclRefExprClass:
diff --git a/external/llvm-project/clang/lib/AST/ExprConstant.cpp b/external/llvm-project/clang/lib/AST/ExprConstant.cpp
index b20e2690d0ee..f1580255a462 100644
--- a/external/llvm-project/clang/lib/AST/ExprConstant.cpp
+++ b/external/llvm-project/clang/lib/AST/ExprConstant.cpp
@@ -3550,7 +3550,12 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
   // should begin within the evaluation of E
   // Used to be C++20 [expr.const]p5.12.2:
   // ... its lifetime began within the evaluation of E;
-  if (isa<ParmVarDecl>(VD) && !AllowConstexprUnknown) {
+  if (isa<ParmVarDecl>(VD)) {
+    if (AllowConstexprUnknown) {
+      Result = &Info.CurrentCall->createConstexprUnknownAPValues(VD, Base);
+      return true;
+    }
+
     // Assume parameters of a potential constant expression are usable in
     // constant expressions.
     if (!Info.checkingPotentialConstantExpression() ||
@@ -5756,8 +5761,12 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
       if (FS->getCond() && !EvaluateCond(Info, FS->getConditionVariable(),
                                          FS->getCond(), Continue))
         return ESR_Failed;
-      if (!Continue)
+
+      if (!Continue) {
+        if (!IterScope.destroy())
+          return ESR_Failed;
         break;
+      }
 
       EvalStmtResult ESR = EvaluateLoopBody(Result, Info, FS->getBody());
       if (ESR != ESR_Continue) {
@@ -6544,8 +6553,8 @@ static bool MaybeHandleUnionActiveMemberChange(EvalInfo &Info,
 }
 
 static bool EvaluateCallArg(const ParmVarDecl *PVD, const Expr *Arg,
-                            CallRef Call, EvalInfo &Info,
-                            bool NonNull = false) {
+                            CallRef Call, EvalInfo &Info, bool NonNull = false,
+                            APValue **EvaluatedArg = nullptr) {
   LValue LV;
   // Create the parameter slot and register its destruction. For a vararg
   // argument, create a temporary.
@@ -6565,13 +6574,17 @@ static bool EvaluateCallArg(const ParmVarDecl *PVD, const Expr *Arg,
     return false;
   }
 
+  if (EvaluatedArg)
+    *EvaluatedArg = &V;
+
   return true;
 }
 
 /// Evaluate the arguments to a function call.
 static bool EvaluateArgs(ArrayRef<const Expr *> Args, CallRef Call,
                          EvalInfo &Info, const FunctionDecl *Callee,
-                         bool RightToLeft = false) {
+                         bool RightToLeft = false,
+                         LValue *ObjectArg = nullptr) {
   bool Success = true;
   llvm::SmallBitVector ForbiddenNullArgs;
   if (Callee->hasAttr<NonNullAttr>()) {
@@ -6594,13 +6607,16 @@ static bool EvaluateArgs(ArrayRef<const Expr *> Args, CallRef Call,
     const ParmVarDecl *PVD =
         Idx < Callee->getNumParams() ? Callee->getParamDecl(Idx) : nullptr;
     bool NonNull = !ForbiddenNullArgs.empty() && ForbiddenNullArgs[Idx];
-    if (!EvaluateCallArg(PVD, Args[Idx], Call, Info, NonNull)) {
+    APValue *That = nullptr;
+    if (!EvaluateCallArg(PVD, Args[Idx], Call, Info, NonNull, &That)) {
       // If we're checking for a potential constant expression, evaluate all
       // initializers even if some of them fail.
       if (!Info.noteFailure())
         return false;
       Success = false;
     }
+    if (PVD && PVD->isExplicitObjectParameter() && That && That->isLValue())
+      ObjectArg->setFrom(Info.Ctx, *That);
   }
   return Success;
 }
@@ -6628,14 +6644,15 @@ static bool handleTrivialCopy(EvalInfo &Info, const ParmVarDecl *Param,
 
 /// Evaluate a function call.
 static bool HandleFunctionCall(SourceLocation CallLoc,
-                               const FunctionDecl *Callee, const LValue *This,
-                               const Expr *E, ArrayRef<const Expr *> Args,
-                               CallRef Call, const Stmt *Body, EvalInfo &Info,
+                               const FunctionDecl *Callee,
+                               const LValue *ObjectArg, const Expr *E,
+                               ArrayRef<const Expr *> Args, CallRef Call,
+                               const Stmt *Body, EvalInfo &Info,
                                APValue &Result, const LValue *ResultSlot) {
   if (!Info.CheckCallLimit(CallLoc))
     return false;
 
-  CallStackFrame Frame(Info, E->getSourceRange(), Callee, This, E, Call);
+  CallStackFrame Frame(Info, E->getSourceRange(), Callee, ObjectArg, E, Call);
 
   // For a trivial copy or move assignment, perform an APValue copy. This is
   // essential for unions, where the operations performed by the assignment
@@ -6648,16 +6665,20 @@ static bool HandleFunctionCall(SourceLocation CallLoc,
       (MD->getParent()->isUnion() ||
        (MD->isTrivial() &&
         isReadByLvalueToRvalueConversion(MD->getParent())))) {
-    assert(This &&
+    unsigned ExplicitOffset = MD->isExplicitObjectMemberFunction() ? 1 : 0;
+    assert(ObjectArg &&
            (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()));
     APValue RHSValue;
     if (!handleTrivialCopy(Info, MD->getParamDecl(0), Args[0], RHSValue,
                            MD->getParent()->isUnion()))
       return false;
-    if (!handleAssignment(Info, Args[0], *This, MD->getThisType(),
+
+    LValue Obj;
+    if (!handleAssignment(Info, Args[ExplicitOffset], *ObjectArg,
+                          MD->getFunctionObjectParameterReferenceType(),
                           RHSValue))
       return false;
-    This->moveInto(Result);
+    ObjectArg->moveInto(Result);
     return true;
   } else if (MD && isLambdaCallOperator(MD)) {
     // We're in a lambda; determine the lambda capture field maps unless we're
@@ -6823,7 +6844,8 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
         // FIXME: In this case, the values of the other subobjects are
         // specified, since zero-initialization sets all padding bits to zero.
         if (!Value->hasValue() ||
-            (Value->isUnion() && Value->getUnionField() != FD)) {
+            (Value->isUnion() &&
+             !declaresSameEntity(Value->getUnionField(), FD))) {
           if (CD->isUnion())
             *Value = APValue(FD);
           else
@@ -8284,7 +8306,7 @@ class ExprEvaluatorBase
     QualType CalleeType = Callee->getType();
 
     const FunctionDecl *FD = nullptr;
-    LValue *This = nullptr, ThisVal;
+    LValue *This = nullptr, ObjectArg;
     auto Args = llvm::ArrayRef(E->getArgs(), E->getNumArgs());
     bool HasQualifier = false;
 
@@ -8295,28 +8317,28 @@ class ExprEvaluatorBase
       const CXXMethodDecl *Member = nullptr;
       if (const MemberExpr *ME = dyn_cast<MemberExpr>(Callee)) {
         // Explicit bound member calls, such as x.f() or p->g();
-        if (!EvaluateObjectArgument(Info, ME->getBase(), ThisVal))
+        if (!EvaluateObjectArgument(Info, ME->getBase(), ObjectArg))
           return false;
         Member = dyn_cast<CXXMethodDecl>(ME->getMemberDecl());
         if (!Member)
           return Error(Callee);
-        This = &ThisVal;
+        This = &ObjectArg;
         HasQualifier = ME->hasQualifier();
       } else if (const BinaryOperator *BE = dyn_cast<BinaryOperator>(Callee)) {
         // Indirect bound member calls ('.*' or '->*').
         const ValueDecl *D =
-            HandleMemberPointerAccess(Info, BE, ThisVal, false);
+            HandleMemberPointerAccess(Info, BE, ObjectArg, false);
         if (!D)
           return false;
         Member = dyn_cast<CXXMethodDecl>(D);
         if (!Member)
           return Error(Callee);
-        This = &ThisVal;
+        This = &ObjectArg;
       } else if (const auto *PDE = dyn_cast<CXXPseudoDestructorExpr>(Callee)) {
         if (!Info.getLangOpts().CPlusPlus20)
           Info.CCEDiag(PDE, diag::note_constexpr_pseudo_destructor);
-        return EvaluateObjectArgument(Info, PDE->getBase(), ThisVal) &&
-               HandleDestruction(Info, PDE, ThisVal, PDE->getDestroyedType());
+        return EvaluateObjectArgument(Info, PDE->getBase(), ObjectArg) &&
+               HandleDestruction(Info, PDE, ObjectArg, PDE->getDestroyedType());
       } else
         return Error(Callee);
       FD = Member;
@@ -8353,7 +8375,7 @@ class ExprEvaluatorBase
         if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
           HasThis = MD->isImplicitObjectMemberFunction();
         if (!EvaluateArgs(HasThis ? Args.slice(1) : Args, Call, Info, FD,
-                          /*RightToLeft=*/true))
+                          /*RightToLeft=*/true, &ObjectArg))
           return false;
       }
 
@@ -8368,20 +8390,20 @@ class ExprEvaluatorBase
         if (Args.empty())
           return Error(E);
 
-        if (!EvaluateObjectArgument(Info, Args[0], ThisVal))
+        if (!EvaluateObjectArgument(Info, Args[0], ObjectArg))
           return false;
 
         // If we are calling a static operator, the 'this' argument needs to be
         // ignored after being evaluated.
         if (MD->isInstance())
-          This = &ThisVal;
+          This = &ObjectArg;
 
         // If this is syntactically a simple assignment using a trivial
         // assignment operator, start the lifetimes of union members as needed,
         // per C++20 [class.union]5.
         if (Info.getLangOpts().CPlusPlus20 && OCE &&
             OCE->getOperator() == OO_Equal && MD->isTrivial() &&
-            !MaybeHandleUnionActiveMemberChange(Info, Args[0], ThisVal))
+            !MaybeHandleUnionActiveMemberChange(Info, Args[0], ObjectArg))
           return false;
 
         Args = Args.slice(1);
@@ -8436,7 +8458,8 @@ class ExprEvaluatorBase
     // Evaluate the arguments now if we've not already done so.
     if (!Call) {
       Call = Info.CurrentCall->createCall(FD);
-      if (!EvaluateArgs(Args, Call, Info, FD))
+      if (!EvaluateArgs(Args, Call, Info, FD, /*RightToLeft*/ false,
+                        &ObjectArg))
         return false;
     }
 
@@ -8470,6 +8493,11 @@ class ExprEvaluatorBase
     Stmt *Body = FD->getBody(Definition);
     SourceLocation Loc = E->getExprLoc();
 
+    // Treat the object argument as `this` when evaluating defaulted
+    // special menmber functions
+    if (FD->hasCXXExplicitFunctionObjectParameter())
+      This = &ObjectArg;
+
     if (!CheckConstexprFunction(Info, Loc, FD, Definition, Body) ||
         !HandleFunctionCall(Loc, Definition, This, E, Args, Call, Body, Info,
                             Result, ResultSlot))
@@ -15218,21 +15246,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
       return Info.Ctx.getTypeSize(DestType) == Info.Ctx.getTypeSize(SrcType);
     }
 
-    if (Info.Ctx.getLangOpts().CPlusPlus && Info.InConstantContext &&
-        Info.EvalMode == EvalInfo::EM_ConstantExpression &&
-        DestType->isEnumeralType()) {
-
-      bool ConstexprVar = true;
-
-      // We know if we are here that we are in a context that we might require
-      // a constant expression or a context that requires a constant
-      // value. But if we are initializing a value we don't know if it is a
-      // constexpr variable or not. We can check the EvaluatingDecl to determine
-      // if it constexpr or not. If not then we don't want to emit a diagnostic.
-      if (const auto *VD = dyn_cast_or_null<VarDecl>(
-              Info.EvaluatingDecl.dyn_cast<const ValueDecl *>()))
-        ConstexprVar = VD->isConstexpr();
-
+    if (Info.Ctx.getLangOpts().CPlusPlus && DestType->isEnumeralType()) {
       const EnumType *ET = dyn_cast<EnumType>(DestType.getCanonicalType());
       const EnumDecl *ED = ET->getDecl();
       // Check that the value is within the range of the enumeration values.
@@ -15252,13 +15266,13 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
         ED->getValueRange(Max, Min);
         --Max;
 
-        if (ED->getNumNegativeBits() && ConstexprVar &&
+        if (ED->getNumNegativeBits() &&
             (Max.slt(Result.getInt().getSExtValue()) ||
              Min.sgt(Result.getInt().getSExtValue())))
           Info.CCEDiag(E, diag::note_constexpr_unscoped_enum_out_of_range)
               << llvm::toString(Result.getInt(), 10) << Min.getSExtValue()
               << Max.getSExtValue() << ED;
-        else if (!ED->getNumNegativeBits() && ConstexprVar &&
+        else if (!ED->getNumNegativeBits() &&
                  Max.ult(Result.getInt().getZExtValue()))
           Info.CCEDiag(E, diag::note_constexpr_unscoped_enum_out_of_range)
               << llvm::toString(Result.getInt(), 10) << Min.getZExtValue()
@@ -17313,7 +17327,6 @@ static ICEDiag CheckICE(const Expr* E, const ASTContext &Ctx) {
   case Expr::CXXDeleteExprClass:
   case Expr::CXXPseudoDestructorExprClass:
   case Expr::UnresolvedLookupExprClass:
-  case Expr::TypoExprClass:
   case Expr::RecoveryExprClass:
   case Expr::DependentScopeDeclRefExprClass:
   case Expr::CXXConstructExprClass:
diff --git a/external/llvm-project/clang/lib/AST/ItaniumMangle.cpp b/external/llvm-project/clang/lib/AST/ItaniumMangle.cpp
index f7c620dc09df..487933a748ab 100644
--- a/external/llvm-project/clang/lib/AST/ItaniumMangle.cpp
+++ b/external/llvm-project/clang/lib/AST/ItaniumMangle.cpp
@@ -1557,7 +1557,8 @@ void CXXNameMangler::mangleUnqualifiedName(
           FD && FD->hasAttr<CUDAGlobalAttr>() &&
           GD.getKernelReferenceKind() == KernelReferenceKind::Stub;
       bool IsOCLDeviceStub =
-          FD && FD->hasAttr<OpenCLKernelAttr>() &&
+          FD &&
+          DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()) &&
           GD.getKernelReferenceKind() == KernelReferenceKind::Stub;
       if (IsDeviceStub)
         mangleDeviceStubName(II);
@@ -3532,10 +3533,9 @@ StringRef CXXNameMangler::getCallingConvQualifierName(CallingConv CC) {
   case CC_AAPCS_VFP:
   case CC_AArch64VectorCall:
   case CC_AArch64SVEPCS:
-  case CC_AMDGPUKernelCall:
   case CC_IntelOclBicc:
   case CC_SpirFunction:
-  case CC_OpenCLKernel:
+  case CC_DeviceKernel:
   case CC_PreserveMost:
   case CC_PreserveAll:
   case CC_M68kRTD:
@@ -4994,7 +4994,6 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
   case Expr::ParenListExprClass:
   case Expr::MSPropertyRefExprClass:
   case Expr::MSPropertySubscriptExprClass:
-  case Expr::TypoExprClass: // This should no longer exist in the AST by now.
   case Expr::RecoveryExprClass:
   case Expr::ArraySectionExprClass:
   case Expr::OMPArrayShapingExprClass:
diff --git a/external/llvm-project/clang/lib/AST/MicrosoftMangle.cpp b/external/llvm-project/clang/lib/AST/MicrosoftMangle.cpp
index 290521a9bd53..bc47e0506add 100644
--- a/external/llvm-project/clang/lib/AST/MicrosoftMangle.cpp
+++ b/external/llvm-project/clang/lib/AST/MicrosoftMangle.cpp
@@ -1164,7 +1164,9 @@ void MicrosoftCXXNameMangler::mangleUnqualifiedName(GlobalDecl GD,
                   ->hasAttr<CUDAGlobalAttr>())) &&
             GD.getKernelReferenceKind() == KernelReferenceKind::Stub;
         bool IsOCLDeviceStub =
-            ND && isa<FunctionDecl>(ND) && ND->hasAttr<OpenCLKernelAttr>() &&
+            ND && isa<FunctionDecl>(ND) &&
+            DeviceKernelAttr::isOpenCLSpelling(
+                ND->getAttr<DeviceKernelAttr>()) &&
             GD.getKernelReferenceKind() == KernelReferenceKind::Stub;
         if (IsDeviceStub)
           mangleSourceName(
@@ -2828,6 +2830,13 @@ void MicrosoftCXXNameMangler::mangleType(const BuiltinType *T, Qualifiers,
     break;
 #include "clang/Basic/HLSLIntangibleTypes.def"
 
+#define SVE_TYPE(Name, Id, SingletonId)                                        \
+  case BuiltinType::Id:                                                        \
+    mangleArtificialTagType(TagTypeKind::Struct, #Name, {"__clang"});          \
+    break;
+#define SVE_SCALAR_TYPE(Name, MangledName, Id, SingletonId, Bits)
+#include "clang/Basic/AArch64ACLETypes.def"
+
     // Issue an error for any type not explicitly handled.
   default:
     Error(Range.getBegin(), "built-in type: ",
diff --git a/external/llvm-project/clang/lib/AST/Stmt.cpp b/external/llvm-project/clang/lib/AST/Stmt.cpp
index be4ba6878bf0..0b0289cd5ec0 100644
--- a/external/llvm-project/clang/lib/AST/Stmt.cpp
+++ b/external/llvm-project/clang/lib/AST/Stmt.cpp
@@ -1154,12 +1154,12 @@ void SwitchStmt::setConditionVariable(const ASTContext &Ctx, VarDecl *V) {
          "This switch statement has no storage for a condition variable!");
 
   if (!V) {
-    getTrailingObjects<Stmt *>()[varOffset()] = nullptr;
+    getTrailingObjects()[varOffset()] = nullptr;
     return;
   }
 
   SourceRange VarRange = V->getSourceRange();
-  getTrailingObjects<Stmt *>()[varOffset()] = new (Ctx)
+  getTrailingObjects()[varOffset()] = new (Ctx)
       DeclStmt(DeclGroupRef(V), VarRange.getBegin(), VarRange.getEnd());
 }
 
@@ -1215,12 +1215,12 @@ void WhileStmt::setConditionVariable(const ASTContext &Ctx, VarDecl *V) {
          "This while statement has no storage for a condition variable!");
 
   if (!V) {
-    getTrailingObjects<Stmt *>()[varOffset()] = nullptr;
+    getTrailingObjects()[varOffset()] = nullptr;
     return;
   }
 
   SourceRange VarRange = V->getSourceRange();
-  getTrailingObjects<Stmt *>()[varOffset()] = new (Ctx)
+  getTrailingObjects()[varOffset()] = new (Ctx)
       DeclStmt(DeclGroupRef(V), VarRange.getBegin(), VarRange.getEnd());
 }
 
diff --git a/external/llvm-project/clang/lib/AST/StmtPrinter.cpp b/external/llvm-project/clang/lib/AST/StmtPrinter.cpp
index 13c3bc038789..28317911d825 100644
--- a/external/llvm-project/clang/lib/AST/StmtPrinter.cpp
+++ b/external/llvm-project/clang/lib/AST/StmtPrinter.cpp
@@ -2914,11 +2914,6 @@ void StmtPrinter::VisitOpaqueValueExpr(OpaqueValueExpr *Node) {
   PrintExpr(Node->getSourceExpr());
 }
 
-void StmtPrinter::VisitTypoExpr(TypoExpr *Node) {
-  // TODO: Print something reasonable for a TypoExpr, if necessary.
-  llvm_unreachable("Cannot print TypoExpr nodes");
-}
-
 void StmtPrinter::VisitRecoveryExpr(RecoveryExpr *Node) {
   OS << "<recovery-expr>(";
   const char *Sep = "";
diff --git a/external/llvm-project/clang/lib/AST/StmtProfile.cpp b/external/llvm-project/clang/lib/AST/StmtProfile.cpp
index f7d1655f67ed..c666d966a6e5 100644
--- a/external/llvm-project/clang/lib/AST/StmtProfile.cpp
+++ b/external/llvm-project/clang/lib/AST/StmtProfile.cpp
@@ -2361,10 +2361,6 @@ void StmtProfiler::VisitOpaqueValueExpr(const OpaqueValueExpr *E) {
   VisitExpr(E);
 }
 
-void StmtProfiler::VisitTypoExpr(const TypoExpr *E) {
-  VisitExpr(E);
-}
-
 void StmtProfiler::VisitSourceLocExpr(const SourceLocExpr *E) {
   VisitExpr(E);
 }
diff --git a/external/llvm-project/clang/lib/AST/TemplateBase.cpp b/external/llvm-project/clang/lib/AST/TemplateBase.cpp
index 8ecd3c7aaca0..7c89dea4629c 100644
--- a/external/llvm-project/clang/lib/AST/TemplateBase.cpp
+++ b/external/llvm-project/clang/lib/AST/TemplateBase.cpp
@@ -726,7 +726,7 @@ ASTTemplateArgumentListInfo::ASTTemplateArgumentListInfo(
   RAngleLoc = Info.getRAngleLoc();
   NumTemplateArgs = Info.size();
 
-  TemplateArgumentLoc *ArgBuffer = getTrailingObjects<TemplateArgumentLoc>();
+  TemplateArgumentLoc *ArgBuffer = getTrailingObjects();
   for (unsigned i = 0; i != NumTemplateArgs; ++i)
     new (&ArgBuffer[i]) TemplateArgumentLoc(Info[i]);
 }
@@ -737,7 +737,7 @@ ASTTemplateArgumentListInfo::ASTTemplateArgumentListInfo(
   RAngleLoc = Info->getRAngleLoc();
   NumTemplateArgs = Info->getNumTemplateArgs();
 
-  TemplateArgumentLoc *ArgBuffer = getTrailingObjects<TemplateArgumentLoc>();
+  TemplateArgumentLoc *ArgBuffer = getTrailingObjects();
   for (unsigned i = 0; i != NumTemplateArgs; ++i)
     new (&ArgBuffer[i]) TemplateArgumentLoc((*Info)[i]);
 }
diff --git a/external/llvm-project/clang/lib/AST/TextNodeDumper.cpp b/external/llvm-project/clang/lib/AST/TextNodeDumper.cpp
index 112e902dfb37..1b84b8824047 100644
--- a/external/llvm-project/clang/lib/AST/TextNodeDumper.cpp
+++ b/external/llvm-project/clang/lib/AST/TextNodeDumper.cpp
@@ -24,7 +24,7 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TypeTraits.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
 
 #include <algorithm>
 #include <utility>
diff --git a/external/llvm-project/clang/lib/AST/Type.cpp b/external/llvm-project/clang/lib/AST/Type.cpp
index 1ab18a5c26c0..a461dbde4093 100644
--- a/external/llvm-project/clang/lib/AST/Type.cpp
+++ b/external/llvm-project/clang/lib/AST/Type.cpp
@@ -100,6 +100,7 @@ bool Qualifiers::isTargetAddressSpaceSupersetOf(LangAS A, LangAS B,
          // address spaces to default to work around this problem.
          (A == LangAS::Default && B == LangAS::hlsl_private) ||
          (A == LangAS::Default && B == LangAS::hlsl_device) ||
+         (A == LangAS::Default && B == LangAS::hlsl_input) ||
          // Conversions from target specific address spaces may be legal
          // depending on the target information.
          Ctx.getTargetInfo().isAddressSpaceSupersetOf(A, B);
@@ -3571,6 +3572,12 @@ QualType QualType::getNonLValueExprType(const ASTContext &Context) const {
   return *this;
 }
 
+bool FunctionType::getCFIUncheckedCalleeAttr() const {
+  if (const auto *FPT = getAs<FunctionProtoType>())
+    return FPT->hasCFIUncheckedCallee();
+  return false;
+}
+
 StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   switch (CC) {
   case CC_C:
@@ -3599,14 +3606,12 @@ StringRef FunctionType::getNameForCallConv(CallingConv CC) {
     return "aarch64_vector_pcs";
   case CC_AArch64SVEPCS:
     return "aarch64_sve_pcs";
-  case CC_AMDGPUKernelCall:
-    return "amdgpu_kernel";
   case CC_IntelOclBicc:
     return "intel_ocl_bicc";
   case CC_SpirFunction:
     return "spir_function";
-  case CC_OpenCLKernel:
-    return "opencl_kernel";
+  case CC_DeviceKernel:
+    return "device_kernel";
   case CC_Swift:
     return "swiftcall";
   case CC_SwiftAsync:
@@ -3662,6 +3667,7 @@ FunctionProtoType::FunctionProtoType(QualType result, ArrayRef<QualType> params,
   FunctionTypeBits.HasExtParameterInfos = !!epi.ExtParameterInfos;
   FunctionTypeBits.Variadic = epi.Variadic;
   FunctionTypeBits.HasTrailingReturn = epi.HasTrailingReturn;
+  FunctionTypeBits.CFIUncheckedCallee = epi.CFIUncheckedCallee;
 
   if (epi.requiresFunctionProtoTypeExtraBitfields()) {
     FunctionTypeBits.HasExtraBitfields = true;
@@ -3929,6 +3935,7 @@ void FunctionProtoType::Profile(llvm::FoldingSetNodeID &ID, QualType Result,
 
   ID.AddInteger((EffectCount << 3) | (HasConds << 2) |
                 (epi.AArch64SMEAttributes << 1) | epi.HasTrailingReturn);
+  ID.AddInteger(epi.CFIUncheckedCallee);
 
   for (unsigned Idx = 0; Idx != EffectCount; ++Idx) {
     ID.AddInteger(epi.FunctionEffects.Effects[Idx].toOpaqueInt32());
@@ -4006,18 +4013,17 @@ StringRef CountAttributedType::getAttributeName(bool WithMacroPrefix) const {
 }
 
 TypedefType::TypedefType(TypeClass tc, const TypedefNameDecl *D,
-                         QualType Underlying, QualType can)
-    : Type(tc, can, toSemanticDependence(can->getDependence())),
+                         QualType UnderlyingType, bool HasTypeDifferentFromDecl)
+    : Type(tc, UnderlyingType.getCanonicalType(),
+           toSemanticDependence(UnderlyingType->getDependence())),
       Decl(const_cast<TypedefNameDecl *>(D)) {
-  assert(!isa<TypedefType>(can) && "Invalid canonical type");
-  TypedefBits.hasTypeDifferentFromDecl = !Underlying.isNull();
+  TypedefBits.hasTypeDifferentFromDecl = HasTypeDifferentFromDecl;
   if (!typeMatchesDecl())
-    *getTrailingObjects<QualType>() = Underlying;
+    *getTrailingObjects() = UnderlyingType;
 }
 
 QualType TypedefType::desugar() const {
-  return typeMatchesDecl() ? Decl->getUnderlyingType()
-                           : *getTrailingObjects<QualType>();
+  return typeMatchesDecl() ? Decl->getUnderlyingType() : *getTrailingObjects();
 }
 
 UsingType::UsingType(const UsingShadowDecl *Found, QualType Underlying,
@@ -4026,14 +4032,14 @@ UsingType::UsingType(const UsingShadowDecl *Found, QualType Underlying,
       Found(const_cast<UsingShadowDecl *>(Found)) {
   UsingBits.hasTypeDifferentFromDecl = !Underlying.isNull();
   if (!typeMatchesDecl())
-    *getTrailingObjects<QualType>() = Underlying;
+    *getTrailingObjects() = Underlying;
 }
 
 QualType UsingType::getUnderlyingType() const {
   return typeMatchesDecl()
              ? QualType(
                    cast<TypeDecl>(Found->getTargetDecl())->getTypeForDecl(), 0)
-             : *getTrailingObjects<QualType>();
+             : *getTrailingObjects();
 }
 
 QualType MacroQualifiedType::desugar() const { return getUnderlyingType(); }
@@ -4139,7 +4145,7 @@ PackIndexingType::PackIndexingType(QualType Canonical, QualType Pattern,
       Pattern(Pattern), IndexExpr(IndexExpr), Size(Expansions.size()),
       FullySubstituted(FullySubstituted) {
 
-  llvm::uninitialized_copy(Expansions, getTrailingObjects<QualType>());
+  llvm::uninitialized_copy(Expansions, getTrailingObjects());
 }
 
 UnsignedOrNone PackIndexingType::getSelectedIndex() const {
@@ -4319,7 +4325,7 @@ bool AttributedType::isCallingConv() const {
   case attr::VectorCall:
   case attr::AArch64VectorPcs:
   case attr::AArch64SVEPcs:
-  case attr::AMDGPUKernelCall:
+  case attr::DeviceKernel:
   case attr::Pascal:
   case attr::MSABI:
   case attr::SysVABI:
@@ -4362,7 +4368,7 @@ SubstTemplateTypeParmType::SubstTemplateTypeParmType(QualType Replacement,
   SubstTemplateTypeParmTypeBits.HasNonCanonicalUnderlyingType =
       Replacement != getCanonicalTypeInternal();
   if (SubstTemplateTypeParmTypeBits.HasNonCanonicalUnderlyingType)
-    *getTrailingObjects<QualType>() = Replacement;
+    *getTrailingObjects() = Replacement;
 
   SubstTemplateTypeParmTypeBits.Index = Index;
   SubstTemplateTypeParmTypeBits.Final = Final;
diff --git a/external/llvm-project/clang/lib/AST/TypePrinter.cpp b/external/llvm-project/clang/lib/AST/TypePrinter.cpp
index 4793ef38c2c4..d18723d807c6 100644
--- a/external/llvm-project/clang/lib/AST/TypePrinter.cpp
+++ b/external/llvm-project/clang/lib/AST/TypePrinter.cpp
@@ -1050,6 +1050,9 @@ void TypePrinter::printFunctionProtoAfter(const FunctionProtoType *T,
     OS << "))";
   }
 
+  if (T->hasCFIUncheckedCallee())
+    OS << " __attribute__((cfi_unchecked_callee))";
+
   if (T->hasTrailingReturn()) {
     OS << " -> ";
     print(T->getReturnType(), OS, StringRef());
@@ -1092,13 +1095,13 @@ void TypePrinter::printFunctionAfter(const FunctionType::ExtInfo &Info,
       OS << " __attribute__((pcs(\"aapcs-vfp\")))";
       break;
     case CC_AArch64VectorCall:
-      OS << "__attribute__((aarch64_vector_pcs))";
+      OS << " __attribute__((aarch64_vector_pcs))";
       break;
     case CC_AArch64SVEPCS:
-      OS << "__attribute__((aarch64_sve_pcs))";
+      OS << " __attribute__((aarch64_sve_pcs))";
       break;
-    case CC_AMDGPUKernelCall:
-      OS << "__attribute__((amdgpu_kernel))";
+    case CC_DeviceKernel:
+      OS << " __attribute__((device_kernel))";
       break;
     case CC_IntelOclBicc:
       OS << " __attribute__((intel_ocl_bicc))";
@@ -1113,7 +1116,6 @@ void TypePrinter::printFunctionAfter(const FunctionType::ExtInfo &Info,
       OS << " __attribute__((regcall))";
       break;
     case CC_SpirFunction:
-    case CC_OpenCLKernel:
       // Do nothing. These CCs are not available as attributes.
       break;
     case CC_Swift:
@@ -2066,7 +2068,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   }
   case attr::AArch64VectorPcs: OS << "aarch64_vector_pcs"; break;
   case attr::AArch64SVEPcs: OS << "aarch64_sve_pcs"; break;
-  case attr::AMDGPUKernelCall: OS << "amdgpu_kernel"; break;
+  case attr::DeviceKernel:
+    OS << T->getAttr()->getSpelling();
+    break;
   case attr::IntelOclBicc: OS << "inteloclbicc"; break;
   case attr::PreserveMost:
     OS << "preserve_most";
@@ -2090,6 +2094,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::NoDeref:
     OS << "noderef";
     break;
+  case attr::CFIUncheckedCallee:
+    OS << "cfi_unchecked_callee";
+    break;
   case attr::AcquireHandle:
     OS << "acquire_handle";
     break;
@@ -2667,6 +2674,8 @@ std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
     return "hlsl_private";
   case LangAS::hlsl_device:
     return "hlsl_device";
+  case LangAS::hlsl_input:
+    return "hlsl_input";
   case LangAS::wasm_funcref:
     return "__funcref";
   default:
diff --git a/external/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/external/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 9c7943a98d65..9cc50a656d37 100644
--- a/external/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/external/llvm-project/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -26,13 +26,10 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
-#include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <optional>
diff --git a/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Parser.cpp b/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Parser.cpp
index 6a16c2184fcf..8a5ac4d0f9d0 100644
--- a/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Parser.cpp
+++ b/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Parser.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
-#include <algorithm>
 #include <cassert>
 #include <cerrno>
 #include <cstddef>
diff --git a/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 336d3a14f795..562df715e08a 100644
--- a/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/external/llvm-project/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -17,13 +17,11 @@
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/ASTMatchers/Dynamic/Diagnostics.h"
 #include "clang/ASTMatchers/Dynamic/VariantValue.h"
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
-#include <iterator>
 #include <memory>
 #include <optional>
 #include <set>
diff --git a/external/llvm-project/clang/lib/ASTMatchers/Dynamic/VariantValue.cpp b/external/llvm-project/clang/lib/ASTMatchers/Dynamic/VariantValue.cpp
index 4f6b021b26f0..2309c8a9b556 100644
--- a/external/llvm-project/clang/lib/ASTMatchers/Dynamic/VariantValue.cpp
+++ b/external/llvm-project/clang/lib/ASTMatchers/Dynamic/VariantValue.cpp
@@ -13,7 +13,6 @@
 
 #include "clang/ASTMatchers/Dynamic/VariantValue.h"
 #include "clang/Basic/LLVM.h"
-#include "llvm/ADT/STLExtras.h"
 #include <optional>
 
 namespace clang {
diff --git a/external/llvm-project/clang/lib/ASTMatchers/GtestMatchers.cpp b/external/llvm-project/clang/lib/ASTMatchers/GtestMatchers.cpp
index a556d8ef2da0..7c135bbfe7cb 100644
--- a/external/llvm-project/clang/lib/ASTMatchers/GtestMatchers.cpp
+++ b/external/llvm-project/clang/lib/ASTMatchers/GtestMatchers.cpp
@@ -16,11 +16,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/ASTMatchers/GtestMatchers.h"
-#include "clang/AST/ASTConsumer.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/RecursiveASTVisitor.h"
-#include "clang/ASTMatchers/ASTMatchFinder.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace clang {
diff --git a/external/llvm-project/clang/lib/Analysis/CFG.cpp b/external/llvm-project/clang/lib/Analysis/CFG.cpp
index 7f37a8b18d46..cf7595952be2 100644
--- a/external/llvm-project/clang/lib/Analysis/CFG.cpp
+++ b/external/llvm-project/clang/lib/Analysis/CFG.cpp
@@ -1261,6 +1261,28 @@ class CFGBuilder {
         L2Result.Val.getKind() == APValue::Float) {
       llvm::APFloat L1 = L1Result.Val.getFloat();
       llvm::APFloat L2 = L2Result.Val.getFloat();
+      // Note that L1 and L2 do not necessarily have the same type.  For example
+      // `x != 0 || x != 1.0`, if `x` is a float16, the two literals `0` and
+      // `1.0` are float16 and double respectively.  In this case, we should do
+      // a conversion before comparing L1 and L2.  Their types must be
+      // compatible since they are comparing with the same DRE.
+      int Order = Context->getFloatingTypeSemanticOrder(NumExpr1->getType(),
+                                                        NumExpr2->getType());
+      bool Ignored = false;
+
+      if (Order > 0) {
+        // type rank L1 > L2:
+        if (llvm::APFloat::opOK !=
+            L2.convert(L1.getSemantics(), llvm::APFloat::rmNearestTiesToEven,
+                       &Ignored))
+          return {};
+      } else if (Order < 0)
+        // type rank L1 < L2:
+        if (llvm::APFloat::opOK !=
+            L1.convert(L2.getSemantics(), llvm::APFloat::rmNearestTiesToEven,
+                       &Ignored))
+          return {};
+
       llvm::APFloat MidValue = L1;
       MidValue.add(L2, llvm::APFloat::rmNearestTiesToEven);
       MidValue.divide(llvm::APFloat(MidValue.getSemantics(), "2.0"),
diff --git a/external/llvm-project/clang/lib/Analysis/FlowSensitive/Formula.cpp b/external/llvm-project/clang/lib/Analysis/FlowSensitive/Formula.cpp
index 12f4e25097d7..7ca1141c39e7 100644
--- a/external/llvm-project/clang/lib/Analysis/FlowSensitive/Formula.cpp
+++ b/external/llvm-project/clang/lib/Analysis/FlowSensitive/Formula.cpp
@@ -90,4 +90,4 @@ void Formula::print(llvm::raw_ostream &OS, const AtomNames *Names) const {
   }
 }
 
-} // namespace clang::dataflow
\ No newline at end of file
+} // namespace clang::dataflow
diff --git a/external/llvm-project/clang/lib/Analysis/ThreadSafety.cpp b/external/llvm-project/clang/lib/Analysis/ThreadSafety.cpp
index d4d06b6dedca..80e7c8eff671 100644
--- a/external/llvm-project/clang/lib/Analysis/ThreadSafety.cpp
+++ b/external/llvm-project/clang/lib/Analysis/ThreadSafety.cpp
@@ -1066,9 +1066,9 @@ class ScopedLockableFactEntry : public FactEntry {
         return;
       }
 
-      FSet.removeLock(FactMan, Cp);
-      FSet.addLock(FactMan,
-                   std::make_unique<LockableFactEntry>(!Cp, LK_Exclusive, loc));
+      FSet.replaceLock(
+          FactMan, It,
+          std::make_unique<LockableFactEntry>(!Cp, LK_Exclusive, loc));
     } else if (Handler) {
       SourceLocation PrevLoc;
       if (const FactEntry *Neg = FSet.findLock(FactMan, !Cp))
diff --git a/external/llvm-project/clang/lib/Analysis/UnsafeBufferUsage.cpp b/external/llvm-project/clang/lib/Analysis/UnsafeBufferUsage.cpp
index ec648e1a17af..631a546b45ff 100644
--- a/external/llvm-project/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/external/llvm-project/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -30,14 +30,12 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Casting.h"
 #include <cstddef>
 #include <optional>
 #include <queue>
 #include <set>
 #include <sstream>
 
-using namespace llvm;
 using namespace clang;
 
 #ifndef NDEBUG
@@ -474,7 +472,7 @@ static bool isSafeSpanTwoParamConstruct(const CXXConstructExpr &Node,
   auto HaveEqualConstantValues = [&Ctx](const Expr *E0, const Expr *E1) {
     if (auto E0CV = E0->getIntegerConstantExpr(Ctx))
       if (auto E1CV = E1->getIntegerConstantExpr(Ctx)) {
-        return APSInt::compareValues(*E0CV, *E1CV) == 0;
+        return llvm::APSInt::compareValues(*E0CV, *E1CV) == 0;
       }
     return false;
   };
@@ -485,7 +483,7 @@ static bool isSafeSpanTwoParamConstruct(const CXXConstructExpr &Node,
       }
     return false;
   };
-  std::optional<APSInt> Arg1CV = Arg1->getIntegerConstantExpr(Ctx);
+  std::optional<llvm::APSInt> Arg1CV = Arg1->getIntegerConstantExpr(Ctx);
 
   if (Arg1CV && Arg1CV->isZero())
     // Check form 5:
@@ -528,10 +526,10 @@ static bool isSafeSpanTwoParamConstruct(const CXXConstructExpr &Node,
   QualType Arg0Ty = Arg0->IgnoreImplicit()->getType();
 
   if (auto *ConstArrTy = Ctx.getAsConstantArrayType(Arg0Ty)) {
-    const APSInt ConstArrSize = APSInt(ConstArrTy->getSize());
+    const llvm::APSInt ConstArrSize = llvm::APSInt(ConstArrTy->getSize());
 
     // Check form 4:
-    return Arg1CV && APSInt::compareValues(ConstArrSize, *Arg1CV) == 0;
+    return Arg1CV && llvm::APSInt::compareValues(ConstArrSize, *Arg1CV) == 0;
   }
   // Check form 6:
   if (auto CCast = dyn_cast<CStyleCastExpr>(Arg0)) {
@@ -1099,9 +1097,10 @@ static bool hasUnsafeSnprintfBuffer(const CallExpr &Node,
       // explicit cast will be needed, which will make this check unreachable.
       // Therefore, the array extent is same as its' bytewise size.
       if (Size->EvaluateAsInt(ER, Ctx)) {
-        APSInt EVal = ER.Val.getInt(); // Size must have integer type
+        llvm::APSInt EVal = ER.Val.getInt(); // Size must have integer type
 
-        return APSInt::compareValues(EVal, APSInt(CAT->getSize(), true)) != 0;
+        return llvm::APSInt::compareValues(
+                   EVal, llvm::APSInt(CAT->getSize(), true)) != 0;
       }
     }
   }
@@ -2148,8 +2147,8 @@ namespace {
 // declarations to its uses and make sure we've covered all uses with our
 // analysis before we try to fix the declaration.
 class DeclUseTracker {
-  using UseSetTy = SmallSet<const DeclRefExpr *, 16>;
-  using DefMapTy = DenseMap<const VarDecl *, const DeclStmt *>;
+  using UseSetTy = llvm::SmallSet<const DeclRefExpr *, 16>;
+  using DefMapTy = llvm::DenseMap<const VarDecl *, const DeclStmt *>;
 
   // Allocate on the heap for easier move.
   std::unique_ptr<UseSetTy> Uses{std::make_unique<UseSetTy>()};
@@ -3640,7 +3639,7 @@ static FixItList fixVarDeclWithArray(const VarDecl *D, const ASTContext &Ctx,
     }
 
     SmallString<32> Replacement;
-    raw_svector_ostream OS(Replacement);
+    llvm::raw_svector_ostream OS(Replacement);
     OS << "std::array<" << ElemTypeTxt << ", " << ArraySizeTxt << "> "
        << IdentText->str();
 
@@ -4064,7 +4063,8 @@ static void applyGadgets(const Decl *D, FixableGadgetList FixableGadgets,
 #endif
 
   // Fixpoint iteration for pointer assignments
-  using DepMapTy = DenseMap<const VarDecl *, llvm::SetVector<const VarDecl *>>;
+  using DepMapTy =
+      llvm::DenseMap<const VarDecl *, llvm::SetVector<const VarDecl *>>;
   DepMapTy DependenciesMap{};
   DepMapTy PtrAssignmentGraph{};
 
diff --git a/external/llvm-project/clang/lib/Basic/Attributes.cpp b/external/llvm-project/clang/lib/Basic/Attributes.cpp
index bcd2c9bd4504..004e5209a44a 100644
--- a/external/llvm-project/clang/lib/Basic/Attributes.cpp
+++ b/external/llvm-project/clang/lib/Basic/Attributes.cpp
@@ -119,7 +119,6 @@ normalizeAttrScopeName(const IdentifierInfo *ScopeName,
                        AttributeCommonInfo::Syntax SyntaxUsed) {
   if (ScopeName)
     return normalizeAttrScopeName(ScopeName->getName(), SyntaxUsed);
-
   return "";
 }
 
@@ -141,12 +140,23 @@ static StringRef normalizeAttrName(StringRef AttrName,
   return AttrName;
 }
 
+StringRef AttributeCommonInfo::getNormalizedScopeName() const {
+  return normalizeAttrScopeName(getScopeName(), getSyntax());
+}
+
+StringRef
+AttributeCommonInfo::getNormalizedAttrName(StringRef ScopeName) const {
+  return normalizeAttrName(getAttrName()->getName(), ScopeName, getSyntax());
+}
+
 bool AttributeCommonInfo::isGNUScope() const {
-  return ScopeName && (ScopeName->isStr("gnu") || ScopeName->isStr("__gnu__"));
+  return AttrScope.isValid() && (AttrScope.getName()->isStr("gnu") ||
+                                 AttrScope.getName()->isStr("__gnu__"));
 }
 
 bool AttributeCommonInfo::isClangScope() const {
-  return ScopeName && (ScopeName->isStr("clang") || ScopeName->isStr("_Clang"));
+  return AttrScope.isValid() && (AttrScope.getName()->isStr("clang") ||
+                                 AttrScope.getName()->isStr("_Clang"));
 }
 
 #include "clang/Sema/AttrParsedAttrKinds.inc"
@@ -198,8 +208,16 @@ std::string AttributeCommonInfo::getNormalizedFullName() const {
       normalizeName(getAttrName(), getScopeName(), getSyntax()));
 }
 
+std::string
+AttributeCommonInfo::getNormalizedFullName(StringRef ScopeName,
+                                           StringRef AttrName) const {
+  return static_cast<std::string>(
+      normalizeName(AttrName, ScopeName, getSyntax()));
+}
+
 SourceRange AttributeCommonInfo::getNormalizedRange() const {
-  return hasScope() ? SourceRange(ScopeLoc, AttrRange.getEnd()) : AttrRange;
+  return hasScope() ? SourceRange(AttrScope.getNameLoc(), AttrRange.getEnd())
+                    : AttrRange;
 }
 
 static AttributeCommonInfo::Scope
@@ -210,6 +228,7 @@ getScopeFromNormalizedScopeName(StringRef ScopeName) {
       .Case("gnu", AttributeCommonInfo::Scope::GNU)
       .Case("gsl", AttributeCommonInfo::Scope::GSL)
       .Case("hlsl", AttributeCommonInfo::Scope::HLSL)
+      .Case("vk", AttributeCommonInfo::Scope::VK)
       .Case("msvc", AttributeCommonInfo::Scope::MSVC)
       .Case("omp", AttributeCommonInfo::Scope::OMP)
       .Case("riscv", AttributeCommonInfo::Scope::RISCV);
@@ -238,10 +257,8 @@ static constexpr const char *AttrScopeSpellingList[] = {
 #include "clang/Basic/AttributeSpellingList.inc"
 };
 
-std::optional<std::string>
-AttributeCommonInfo::getCorrectedFullName(const TargetInfo &Target,
-                                          const LangOptions &LangOpts) const {
-  StringRef ScopeName = normalizeAttrScopeName(getScopeName(), getSyntax());
+std::optional<StringRef>
+AttributeCommonInfo::tryGetCorrectedScopeName(StringRef ScopeName) const {
   if (ScopeName.size() > 0 &&
       llvm::none_of(AttrScopeSpellingList,
                     [&](const char *S) { return S == ScopeName; })) {
@@ -250,25 +267,26 @@ AttributeCommonInfo::getCorrectedFullName(const TargetInfo &Target,
       STC.add(Scope);
 
     if (auto CorrectedScopeName = STC.getCorrection())
-      ScopeName = *CorrectedScopeName;
+      return CorrectedScopeName;
   }
+  return std::nullopt;
+}
 
-  StringRef AttrName =
-      normalizeAttrName(getAttrName()->getName(), ScopeName, getSyntax());
+std::optional<StringRef> AttributeCommonInfo::tryGetCorrectedAttrName(
+    StringRef ScopeName, StringRef AttrName, const TargetInfo &Target,
+    const LangOptions &LangOpts) const {
   if (llvm::none_of(AttrSpellingList,
                     [&](const char *A) { return A == AttrName; })) {
     SimpleTypoCorrection STC(AttrName);
     for (const auto &Attr : AttrSpellingList)
       STC.add(Attr);
 
-    if (auto CorrectedAttrName = STC.getCorrection())
-      AttrName = *CorrectedAttrName;
+    if (auto CorrectedAttrName = STC.getCorrection()) {
+      if (hasAttribute(getSyntax(), ScopeName, *CorrectedAttrName, Target,
+                       LangOpts,
+                       /*CheckPlugins=*/true))
+        return CorrectedAttrName;
+    }
   }
-
-  if (hasAttribute(getSyntax(), ScopeName, AttrName, Target, LangOpts,
-                   /*CheckPlugins=*/true))
-    return static_cast<std::string>(
-        normalizeName(AttrName, ScopeName, getSyntax()));
-
   return std::nullopt;
 }
diff --git a/external/llvm-project/clang/lib/Basic/Diagnostic.cpp b/external/llvm-project/clang/lib/Basic/Diagnostic.cpp
index 694224071347..a30bfa28eca7 100644
--- a/external/llvm-project/clang/lib/Basic/Diagnostic.cpp
+++ b/external/llvm-project/clang/lib/Basic/Diagnostic.cpp
@@ -119,6 +119,8 @@ bool DiagnosticsEngine::popMappings(SourceLocation Loc) {
   return true;
 }
 
+void DiagnosticsEngine::ResetPragmas() { DiagStatesByLoc.clear(/*Soft=*/true); }
+
 void DiagnosticsEngine::Reset(bool soft /*=false*/) {
   ErrorOccurred = false;
   UncompilableErrorOccurred = false;
@@ -130,12 +132,12 @@ void DiagnosticsEngine::Reset(bool soft /*=false*/) {
   TrapNumErrorsOccurred = 0;
   TrapNumUnrecoverableErrorsOccurred = 0;
 
-  LastDiagLevel = DiagnosticIDs::Ignored;
+  LastDiagLevel = Ignored;
 
   if (!soft) {
     // Clear state related to #pragma diagnostic.
     DiagStates.clear();
-    DiagStatesByLoc.clear();
+    DiagStatesByLoc.clear(false);
     DiagStateOnPushStack.clear();
 
     // Create a DiagState and DiagStatePoint representing diagnostic changes
@@ -658,13 +660,95 @@ void DiagnosticsEngine::Report(const StoredDiagnostic &storedDiag) {
   Level DiagLevel = storedDiag.getLevel();
   Diagnostic Info(this, storedDiag.getLocation(), storedDiag.getID(),
                   DiagStorage, storedDiag.getMessage());
+  Report(DiagLevel, Info);
+}
+
+void DiagnosticsEngine::Report(Level DiagLevel, const Diagnostic &Info) {
+  assert(DiagLevel != Ignored && "Cannot emit ignored diagnostics!");
   Client->HandleDiagnostic(DiagLevel, Info);
   if (Client->IncludeInDiagnosticCounts()) {
-    if (DiagLevel == DiagnosticsEngine::Warning)
+    if (DiagLevel == Warning)
       ++NumWarnings;
   }
 }
 
+/// ProcessDiag - This is the method used to report a diagnostic that is
+/// finally fully formed.
+bool DiagnosticsEngine::ProcessDiag(const DiagnosticBuilder &DiagBuilder) {
+  Diagnostic Info(this, DiagBuilder);
+
+  assert(getClient() && "DiagnosticClient not set!");
+
+  // Figure out the diagnostic level of this message.
+  unsigned DiagID = Info.getID();
+  Level DiagLevel = getDiagnosticLevel(DiagID, Info.getLocation());
+
+  // Update counts for DiagnosticErrorTrap even if a fatal error occurred
+  // or diagnostics are suppressed.
+  if (DiagLevel >= Error) {
+    ++TrapNumErrorsOccurred;
+    if (Diags->isUnrecoverable(DiagID))
+      ++TrapNumUnrecoverableErrorsOccurred;
+  }
+
+  if (SuppressAllDiagnostics)
+    return false;
+
+  if (DiagLevel != Note) {
+    // Record that a fatal error occurred only when we see a second
+    // non-note diagnostic. This allows notes to be attached to the
+    // fatal error, but suppresses any diagnostics that follow those
+    // notes.
+    if (LastDiagLevel == Fatal)
+      FatalErrorOccurred = true;
+
+    LastDiagLevel = DiagLevel;
+  }
+
+  // If a fatal error has already been emitted, silence all subsequent
+  // diagnostics.
+  if (FatalErrorOccurred) {
+    if (DiagLevel >= Error && Client->IncludeInDiagnosticCounts())
+      ++NumErrors;
+
+    return false;
+  }
+
+  // If the client doesn't care about this message, don't issue it.  If this is
+  // a note and the last real diagnostic was ignored, ignore it too.
+  if (DiagLevel == Ignored || (DiagLevel == Note && LastDiagLevel == Ignored))
+    return false;
+
+  if (DiagLevel >= Error) {
+    if (Diags->isUnrecoverable(DiagID))
+      UnrecoverableErrorOccurred = true;
+
+    // Warnings which have been upgraded to errors do not prevent compilation.
+    if (Diags->isDefaultMappingAsError(DiagID))
+      UncompilableErrorOccurred = true;
+
+    ErrorOccurred = true;
+    if (Client->IncludeInDiagnosticCounts())
+      ++NumErrors;
+
+    // If we've emitted a lot of errors, emit a fatal error instead of it to
+    // stop a flood of bogus errors.
+    if (ErrorLimit && NumErrors > ErrorLimit && DiagLevel == Error) {
+      Report(diag::fatal_too_many_errors);
+      return false;
+    }
+  }
+
+  // Make sure we set FatalErrorOccurred to ensure that the notes from the
+  // diagnostic that caused `fatal_too_many_errors` won't be emitted.
+  if (Info.getID() == diag::fatal_too_many_errors)
+    FatalErrorOccurred = true;
+
+  // Finally, report it.
+  Report(DiagLevel, Info);
+  return true;
+}
+
 bool DiagnosticsEngine::EmitDiagnostic(const DiagnosticBuilder &DB,
                                        bool Force) {
   assert(getClient() && "DiagnosticClient not set!");
@@ -674,14 +758,12 @@ bool DiagnosticsEngine::EmitDiagnostic(const DiagnosticBuilder &DB,
     Diagnostic Info(this, DB);
 
     // Figure out the diagnostic level of this message.
-    DiagnosticIDs::Level DiagLevel =
-        Diags->getDiagnosticLevel(Info.getID(), Info.getLocation(), *this);
+    Level DiagLevel = getDiagnosticLevel(Info.getID(), Info.getLocation());
 
-    Emitted = (DiagLevel != DiagnosticIDs::Ignored);
-    if (Emitted) {
-      // Emit the diagnostic regardless of suppression level.
-      Diags->EmitDiag(*this, DB, DiagLevel);
-    }
+    // Emit the diagnostic regardless of suppression level.
+    Emitted = DiagLevel != Ignored;
+    if (Emitted)
+      Report(DiagLevel, Info);
   } else {
     // Process the diagnostic, sending the accumulated information to the
     // DiagnosticConsumer.
diff --git a/external/llvm-project/clang/lib/Basic/DiagnosticIDs.cpp b/external/llvm-project/clang/lib/Basic/DiagnosticIDs.cpp
index 6fa4066d95f8..dcf0c6cb5428 100644
--- a/external/llvm-project/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/external/llvm-project/clang/lib/Basic/DiagnosticIDs.cpp
@@ -542,26 +542,32 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
     return Result;
 
   const auto &SM = Diag.getSourceManager();
-
-  bool ShowInSystemHeader =
-      IsCustomDiag
-          ? CustomDiagInfo->getDescription(DiagID).ShouldShowInSystemHeader()
-          : !GetDiagInfo(DiagID) || GetDiagInfo(DiagID)->WarnShowInSystemHeader;
-
   // If we are in a system header, we ignore it. We look at the diagnostic class
   // because we also want to ignore extensions and warnings in -Werror and
   // -pedantic-errors modes, which *map* warnings/extensions to errors.
-  if (State->SuppressSystemWarnings && !ShowInSystemHeader && Loc.isValid() &&
-      SM.isInSystemHeader(SM.getExpansionLoc(Loc)))
-    return diag::Severity::Ignored;
-
+  if (State->SuppressSystemWarnings && Loc.isValid() &&
+      SM.isInSystemHeader(SM.getExpansionLoc(Loc))) {
+    bool ShowInSystemHeader = true;
+    if (IsCustomDiag)
+      ShowInSystemHeader =
+          CustomDiagInfo->getDescription(DiagID).ShouldShowInSystemHeader();
+    else if (const StaticDiagInfoRec *Rec = GetDiagInfo(DiagID))
+      ShowInSystemHeader = Rec->WarnShowInSystemHeader;
+
+    if (!ShowInSystemHeader)
+      return diag::Severity::Ignored;
+  }
   // We also ignore warnings due to system macros
-  bool ShowInSystemMacro =
-      !GetDiagInfo(DiagID) || GetDiagInfo(DiagID)->WarnShowInSystemMacro;
-  if (State->SuppressSystemWarnings && !ShowInSystemMacro && Loc.isValid() &&
-      SM.isInSystemMacro(Loc))
-    return diag::Severity::Ignored;
+  if (State->SuppressSystemWarnings && Loc.isValid() &&
+      SM.isInSystemMacro(Loc)) {
+
+    bool ShowInSystemMacro = true;
+    if (const StaticDiagInfoRec *Rec = GetDiagInfo(DiagID))
+      ShowInSystemMacro = Rec->WarnShowInSystemMacro;
 
+    if (!ShowInSystemMacro)
+      return diag::Severity::Ignored;
+  }
   // Clang-diagnostics pragmas always take precedence over suppression mapping.
   if (!Mapping.isPragma() && Diag.isSuppressedViaMapping(DiagID, Loc))
     return diag::Severity::Ignored;
@@ -817,103 +823,6 @@ unsigned DiagnosticIDs::getCXXCompatDiagId(const LangOptions &LangOpts,
   return StdVer >= D.StdVer ? D.DiagId : D.PreDiagId;
 }
 
-/// ProcessDiag - This is the method used to report a diagnostic that is
-/// finally fully formed.
-bool DiagnosticIDs::ProcessDiag(DiagnosticsEngine &Diag,
-                                const DiagnosticBuilder &DiagBuilder) const {
-  Diagnostic Info(&Diag, DiagBuilder);
-
-  assert(Diag.getClient() && "DiagnosticClient not set!");
-
-  // Figure out the diagnostic level of this message.
-  unsigned DiagID = Info.getID();
-  DiagnosticIDs::Level DiagLevel
-    = getDiagnosticLevel(DiagID, Info.getLocation(), Diag);
-
-  // Update counts for DiagnosticErrorTrap even if a fatal error occurred
-  // or diagnostics are suppressed.
-  if (DiagLevel >= DiagnosticIDs::Error) {
-    ++Diag.TrapNumErrorsOccurred;
-    if (isUnrecoverable(DiagID))
-      ++Diag.TrapNumUnrecoverableErrorsOccurred;
-  }
-
-  if (Diag.SuppressAllDiagnostics)
-    return false;
-
-  if (DiagLevel != DiagnosticIDs::Note) {
-    // Record that a fatal error occurred only when we see a second
-    // non-note diagnostic. This allows notes to be attached to the
-    // fatal error, but suppresses any diagnostics that follow those
-    // notes.
-    if (Diag.LastDiagLevel == DiagnosticIDs::Fatal)
-      Diag.FatalErrorOccurred = true;
-
-    Diag.LastDiagLevel = DiagLevel;
-  }
-
-  // If a fatal error has already been emitted, silence all subsequent
-  // diagnostics.
-  if (Diag.FatalErrorOccurred) {
-    if (DiagLevel >= DiagnosticIDs::Error &&
-        Diag.Client->IncludeInDiagnosticCounts()) {
-      ++Diag.NumErrors;
-    }
-
-    return false;
-  }
-
-  // If the client doesn't care about this message, don't issue it.  If this is
-  // a note and the last real diagnostic was ignored, ignore it too.
-  if (DiagLevel == DiagnosticIDs::Ignored ||
-      (DiagLevel == DiagnosticIDs::Note &&
-       Diag.LastDiagLevel == DiagnosticIDs::Ignored))
-    return false;
-
-  if (DiagLevel >= DiagnosticIDs::Error) {
-    if (isUnrecoverable(DiagID))
-      Diag.UnrecoverableErrorOccurred = true;
-
-    // Warnings which have been upgraded to errors do not prevent compilation.
-    if (isDefaultMappingAsError(DiagID))
-      Diag.UncompilableErrorOccurred = true;
-
-    Diag.ErrorOccurred = true;
-    if (Diag.Client->IncludeInDiagnosticCounts()) {
-      ++Diag.NumErrors;
-    }
-
-    // If we've emitted a lot of errors, emit a fatal error instead of it to
-    // stop a flood of bogus errors.
-    if (Diag.ErrorLimit && Diag.NumErrors > Diag.ErrorLimit &&
-        DiagLevel == DiagnosticIDs::Error) {
-      Diag.Report(diag::fatal_too_many_errors);
-      return false;
-    }
-  }
-
-  // Make sure we set FatalErrorOccurred to ensure that the notes from the
-  // diagnostic that caused `fatal_too_many_errors` won't be emitted.
-  if (Info.getID() == diag::fatal_too_many_errors)
-    Diag.FatalErrorOccurred = true;
-  // Finally, report it.
-  EmitDiag(Diag, DiagBuilder, DiagLevel);
-  return true;
-}
-
-void DiagnosticIDs::EmitDiag(DiagnosticsEngine &Diag,
-                             const DiagnosticBuilder &DiagBuilder,
-                             Level DiagLevel) const {
-  Diagnostic Info(&Diag, DiagBuilder);
-  assert(DiagLevel != DiagnosticIDs::Ignored && "Cannot emit ignored diagnostics!");
-
-  Diag.Client->HandleDiagnostic((DiagnosticsEngine::Level)DiagLevel, Info);
-  if (Diag.Client->IncludeInDiagnosticCounts()) {
-    if (DiagLevel == DiagnosticIDs::Warning)
-      ++Diag.NumWarnings;
-  }
-}
-
 bool DiagnosticIDs::isUnrecoverable(unsigned DiagID) const {
   // Only errors may be unrecoverable.
   if (getDiagClass(DiagID) < CLASS_ERROR)
diff --git a/external/llvm-project/clang/lib/Basic/LangOptions.cpp b/external/llvm-project/clang/lib/Basic/LangOptions.cpp
index 7e696620993f..912b890569cf 100644
--- a/external/llvm-project/clang/lib/Basic/LangOptions.cpp
+++ b/external/llvm-project/clang/lib/Basic/LangOptions.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/LangOptions.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/Path.h"
 
 using namespace clang;
diff --git a/external/llvm-project/clang/lib/Basic/NoSanitizeList.cpp b/external/llvm-project/clang/lib/Basic/NoSanitizeList.cpp
index a1ca31ea0e97..96f79fb2a2a2 100644
--- a/external/llvm-project/clang/lib/Basic/NoSanitizeList.cpp
+++ b/external/llvm-project/clang/lib/Basic/NoSanitizeList.cpp
@@ -44,7 +44,7 @@ bool NoSanitizeList::containsPrefix(SanitizerMask Mask, StringRef Prefix,
 
 bool NoSanitizeList::containsGlobal(SanitizerMask Mask, StringRef GlobalName,
                                     StringRef Category) const {
-  return SSCL->inSection(Mask, "global", GlobalName, Category);
+  return containsPrefix(Mask, "global", GlobalName, Category);
 }
 
 bool NoSanitizeList::containsType(SanitizerMask Mask, StringRef MangledTypeName,
@@ -54,7 +54,7 @@ bool NoSanitizeList::containsType(SanitizerMask Mask, StringRef MangledTypeName,
 
 bool NoSanitizeList::containsFunction(SanitizerMask Mask,
                                       StringRef FunctionName) const {
-  return SSCL->inSection(Mask, "fun", FunctionName);
+  return containsPrefix(Mask, "fun", FunctionName, {});
 }
 
 bool NoSanitizeList::containsFile(SanitizerMask Mask, StringRef FileName,
@@ -64,7 +64,7 @@ bool NoSanitizeList::containsFile(SanitizerMask Mask, StringRef FileName,
 
 bool NoSanitizeList::containsMainFile(SanitizerMask Mask, StringRef FileName,
                                       StringRef Category) const {
-  return SSCL->inSection(Mask, "mainfile", FileName, Category);
+  return containsPrefix(Mask, "mainfile", FileName, Category);
 }
 
 bool NoSanitizeList::containsLocation(SanitizerMask Mask, SourceLocation Loc,
diff --git a/external/llvm-project/clang/lib/Basic/ProfileList.cpp b/external/llvm-project/clang/lib/Basic/ProfileList.cpp
index aaea5a00ab6a..8481deffe2a7 100644
--- a/external/llvm-project/clang/lib/Basic/ProfileList.cpp
+++ b/external/llvm-project/clang/lib/Basic/ProfileList.cpp
@@ -69,24 +69,24 @@ ProfileList::ProfileList(ArrayRef<std::string> Paths, SourceManager &SM)
 
 ProfileList::~ProfileList() = default;
 
-static StringRef getSectionName(CodeGenOptions::ProfileInstrKind Kind) {
+static StringRef getSectionName(llvm::driver::ProfileInstrKind Kind) {
   switch (Kind) {
-  case CodeGenOptions::ProfileNone:
+  case llvm::driver::ProfileInstrKind::ProfileNone:
     return "";
-  case CodeGenOptions::ProfileClangInstr:
+  case llvm::driver::ProfileInstrKind::ProfileClangInstr:
     return "clang";
-  case CodeGenOptions::ProfileIRInstr:
+  case llvm::driver::ProfileInstrKind::ProfileIRInstr:
     return "llvm";
-  case CodeGenOptions::ProfileCSIRInstr:
+  case llvm::driver::ProfileInstrKind::ProfileCSIRInstr:
     return "csllvm";
-  case CodeGenOptions::ProfileIRSampleColdCov:
+  case llvm::driver::ProfileInstrKind::ProfileIRSampleColdCov:
     return "sample-coldcov";
   }
-  llvm_unreachable("Unhandled CodeGenOptions::ProfileInstrKind enum");
+  llvm_unreachable("Unhandled llvm::driver::ProfileInstrKind enum");
 }
 
 ProfileList::ExclusionType
-ProfileList::getDefault(CodeGenOptions::ProfileInstrKind Kind) const {
+ProfileList::getDefault(llvm::driver::ProfileInstrKind Kind) const {
   StringRef Section = getSectionName(Kind);
   // Check for "default:<type>"
   if (SCL->inSection(Section, "default", "allow"))
@@ -117,7 +117,7 @@ ProfileList::inSection(StringRef Section, StringRef Prefix,
 
 std::optional<ProfileList::ExclusionType>
 ProfileList::isFunctionExcluded(StringRef FunctionName,
-                                CodeGenOptions::ProfileInstrKind Kind) const {
+                                llvm::driver::ProfileInstrKind Kind) const {
   StringRef Section = getSectionName(Kind);
   // Check for "function:<regex>=<case>"
   if (auto V = inSection(Section, "function", FunctionName))
@@ -131,13 +131,13 @@ ProfileList::isFunctionExcluded(StringRef FunctionName,
 
 std::optional<ProfileList::ExclusionType>
 ProfileList::isLocationExcluded(SourceLocation Loc,
-                                CodeGenOptions::ProfileInstrKind Kind) const {
+                                llvm::driver::ProfileInstrKind Kind) const {
   return isFileExcluded(SM.getFilename(SM.getFileLoc(Loc)), Kind);
 }
 
 std::optional<ProfileList::ExclusionType>
 ProfileList::isFileExcluded(StringRef FileName,
-                            CodeGenOptions::ProfileInstrKind Kind) const {
+                            llvm::driver::ProfileInstrKind Kind) const {
   StringRef Section = getSectionName(Kind);
   // Check for "source:<regex>=<case>"
   if (auto V = inSection(Section, "source", FileName))
diff --git a/external/llvm-project/clang/lib/Basic/SourceManager.cpp b/external/llvm-project/clang/lib/Basic/SourceManager.cpp
index 0c7f237ad9f0..053e82683a4a 100644
--- a/external/llvm-project/clang/lib/Basic/SourceManager.cpp
+++ b/external/llvm-project/clang/lib/Basic/SourceManager.cpp
@@ -24,6 +24,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/AutoConvert.h"
 #include "llvm/Support/Capacity.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
@@ -343,6 +344,9 @@ void SourceManager::clearIDTables() {
   NextLocalOffset = 0;
   CurrentLoadedOffset = MaxLoadedOffset;
   createExpansionLoc(SourceLocation(), SourceLocation(), SourceLocation(), 1);
+  // Diagnostics engine keeps some references to fileids, mostly for dealing
+  // with diagnostic pragmas, make sure they're reset as well.
+  Diag.ResetPragmas();
 }
 
 bool SourceManager::isMainFile(const FileEntry &SourceFile) {
diff --git a/external/llvm-project/clang/lib/Basic/TargetInfo.cpp b/external/llvm-project/clang/lib/Basic/TargetInfo.cpp
index ab13c32f6943..9429a316a919 100644
--- a/external/llvm-project/clang/lib/Basic/TargetInfo.cpp
+++ b/external/llvm-project/clang/lib/Basic/TargetInfo.cpp
@@ -49,6 +49,8 @@ static const LangASMap FakeAddrSpaceMap = {
     13, // hlsl_groupshared
     14, // hlsl_constant
     15, // hlsl_private
+    16, // hlsl_device
+    17, // hlsl_input
     20, // wasm_funcref
 };
 
@@ -157,7 +159,7 @@ TargetInfo::TargetInfo(const llvm::Triple &T) : Triple(T) {
   SSERegParmMax = 0;
   HasAlignMac68kSupport = false;
   HasBuiltinMSVaList = false;
-  HasAArch64SVETypes = false;
+  HasAArch64ACLETypes = false;
   HasRISCVVTypes = false;
   AllowAMDGPUUnsafeFPAtomics = false;
   HasUnalignedAccess = false;
@@ -553,8 +555,7 @@ void TargetInfo::adjust(DiagnosticsEngine &Diags, LangOptions &Opts) {
 bool TargetInfo::initFeatureMap(
     llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef CPU,
     const std::vector<std::string> &FeatureVec) const {
-  for (const auto &F : FeatureVec) {
-    StringRef Name = F;
+  for (StringRef Name : FeatureVec) {
     if (Name.empty())
       continue;
     // Apply the feature via the target.
diff --git a/external/llvm-project/clang/lib/Basic/Targets/AArch64.cpp b/external/llvm-project/clang/lib/Basic/Targets/AArch64.cpp
index e14eff6c4183..124b340b62d9 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/AArch64.cpp
+++ b/external/llvm-project/clang/lib/Basic/Targets/AArch64.cpp
@@ -239,15 +239,15 @@ AArch64TargetInfo::AArch64TargetInfo(const llvm::Triple &Triple,
   // Make __builtin_ms_va_list available.
   HasBuiltinMSVaList = true;
 
-  // Make the SVE types available.  Note that this deliberately doesn't
-  // depend on SveMode, since in principle it should be possible to turn
+  // Make the Neon ACLE and SVE types available.  Note that this deliberately
+  // doesn't depend on SveMode, since in principle it should be possible to turn
   // SVE on and off within a translation unit.  It should also be possible
   // to compile the global declaration:
   //
   // __SVInt8_t *ptr;
   //
   // even without SVE.
-  HasAArch64SVETypes = true;
+  HasAArch64ACLETypes = true;
 
   // {} in inline assembly are neon specifiers, not assembly variant
   // specifiers.
@@ -625,6 +625,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasCRC)
     Builder.defineMacro("__ARM_FEATURE_CRC32", "1");
 
+  if (HasCSSC)
+    Builder.defineMacro("__ARM_FEATURE_CSSC", "1");
+
   if (HasRCPC3)
     Builder.defineMacro("__ARM_FEATURE_RCPC", "3");
   else if (HasRCPC)
@@ -874,6 +877,7 @@ bool AArch64TargetInfo::hasFeature(StringRef Feature) const {
       .Case("rdm", HasRDM)
       .Case("lse", HasLSE)
       .Case("crc", HasCRC)
+      .Case("cssc", HasCSSC)
       .Case("sha2", HasSHA2)
       .Case("sha3", HasSHA3)
       .Cases("aes", "pmull", HasAES)
@@ -1249,6 +1253,8 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasPAuthLR = true;
       HasPAuth = true;
     }
+    if (Feature == "+cssc")
+      HasCSSC = true;
   }
 
   // Check features that are manually disabled by command line options.
@@ -1400,7 +1406,7 @@ AArch64TargetInfo::checkCallingConvention(CallingConv CC) const {
   case CC_PreserveMost:
   case CC_PreserveAll:
   case CC_PreserveNone:
-  case CC_OpenCLKernel:
+  case CC_DeviceKernel:
   case CC_AArch64VectorCall:
   case CC_AArch64SVEPCS:
   case CC_Win64:
@@ -1758,7 +1764,7 @@ WindowsARM64TargetInfo::checkCallingConvention(CallingConv CC) const {
   case CC_X86FastCall:
     return CCCR_Ignore;
   case CC_C:
-  case CC_OpenCLKernel:
+  case CC_DeviceKernel:
   case CC_PreserveMost:
   case CC_PreserveAll:
   case CC_PreserveNone:
diff --git a/external/llvm-project/clang/lib/Basic/Targets/AArch64.h b/external/llvm-project/clang/lib/Basic/Targets/AArch64.h
index 7230f22d5bb8..1951e0679d2e 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/AArch64.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/AArch64.h
@@ -47,6 +47,7 @@ static const unsigned ARM64AddrSpaceMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -65,6 +66,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
 
   unsigned FPU = FPUMode;
   bool HasCRC = false;
+  bool HasCSSC = false;
   bool HasAES = false;
   bool HasSHA2 = false;
   bool HasSHA3 = false;
diff --git a/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.cpp b/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.cpp
index 9a595c5e6c5c..8d5489becc02 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -62,6 +62,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
     // will break loudly.
     llvm::AMDGPUAS::PRIVATE_ADDRESS, // hlsl_private
     llvm::AMDGPUAS::GLOBAL_ADDRESS,  // hlsl_device
+    llvm::AMDGPUAS::PRIVATE_ADDRESS, // hlsl_input
 };
 
 const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
@@ -89,6 +90,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
     llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
     llvm::AMDGPUAS::PRIVATE_ADDRESS,  // hlsl_private
     llvm::AMDGPUAS::GLOBAL_ADDRESS,   // hlsl_device
+    llvm::AMDGPUAS::PRIVATE_ADDRESS,  // hlsl_input
 };
 } // namespace targets
 } // namespace clang
diff --git a/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.h b/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.h
index 8ea544ba28b1..509128f3cf07 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/AMDGPU.h
@@ -415,8 +415,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
     default:
       return CCCR_Warning;
     case CC_C:
-    case CC_OpenCLKernel:
-    case CC_AMDGPUKernelCall:
+    case CC_DeviceKernel:
       return CCCR_OK;
     }
   }
diff --git a/external/llvm-project/clang/lib/Basic/Targets/ARM.cpp b/external/llvm-project/clang/lib/Basic/Targets/ARM.cpp
index bd12350367ce..65d4ed1e9654 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/ARM.cpp
+++ b/external/llvm-project/clang/lib/Basic/Targets/ARM.cpp
@@ -1404,7 +1404,7 @@ ARMTargetInfo::checkCallingConvention(CallingConv CC) const {
   case CC_AAPCS_VFP:
   case CC_Swift:
   case CC_SwiftAsync:
-  case CC_OpenCLKernel:
+  case CC_DeviceKernel:
     return CCCR_OK;
   default:
     return CCCR_Warning;
@@ -1479,7 +1479,7 @@ WindowsARMTargetInfo::checkCallingConvention(CallingConv CC) const {
   case CC_X86VectorCall:
     return CCCR_Ignore;
   case CC_C:
-  case CC_OpenCLKernel:
+  case CC_DeviceKernel:
   case CC_PreserveMost:
   case CC_PreserveAll:
   case CC_Swift:
diff --git a/external/llvm-project/clang/lib/Basic/Targets/AVR.cpp b/external/llvm-project/clang/lib/Basic/Targets/AVR.cpp
index 85ca4bc30c46..bbe7b01ca036 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/AVR.cpp
+++ b/external/llvm-project/clang/lib/Basic/Targets/AVR.cpp
@@ -336,6 +336,9 @@ static MCUInfo AVRMcus[] = {
     {"attiny1624", "__AVR_ATtiny1624__", "103", 1},
     {"attiny1626", "__AVR_ATtiny1626__", "103", 1},
     {"attiny1627", "__AVR_ATtiny1627__", "103", 1},
+    {"attiny3224", "__AVR_ATtiny3224__", "103", 1},
+    {"attiny3226", "__AVR_ATtiny3226__", "103", 1},
+    {"attiny3227", "__AVR_ATtiny3227__", "103", 1},
     {"atmega808", "__AVR_ATmega808__", "103", 1},
     {"atmega809", "__AVR_ATmega809__", "103", 1},
     {"atmega1608", "__AVR_ATmega1608__", "103", 1},
@@ -344,6 +347,72 @@ static MCUInfo AVRMcus[] = {
     {"atmega3209", "__AVR_ATmega3209__", "103", 1},
     {"atmega4808", "__AVR_ATmega4808__", "103", 1},
     {"atmega4809", "__AVR_ATmega4809__", "103", 1},
+
+    // gcc 14 additions:
+
+    {"avr64da28", "__AVR_AVR64DA28__", "102", 1},
+    {"avr64da32", "__AVR_AVR64DA32__", "102", 1},
+    {"avr64da48", "__AVR_AVR64DA48__", "102", 1},
+    {"avr64da64", "__AVR_AVR64DA64__", "102", 1},
+    {"avr64db28", "__AVR_AVR64DB28__", "102", 1},
+    {"avr64db32", "__AVR_AVR64DB32__", "102", 1},
+    {"avr64db48", "__AVR_AVR64DB48__", "102", 1},
+    {"avr64db64", "__AVR_AVR64DB64__", "102", 1},
+    {"avr64dd14", "__AVR_AVR64DD14__", "102", 1},
+    {"avr64dd20", "__AVR_AVR64DD20__", "102", 1},
+    {"avr64dd28", "__AVR_AVR64DD28__", "102", 1},
+    {"avr64dd32", "__AVR_AVR64DD32__", "102", 1},
+    {"avr64du28", "__AVR_AVR64DU28__", "102", 1},
+    {"avr64du32", "__AVR_AVR64DU32__", "102", 1},
+    {"avr64ea28", "__AVR_AVR64EA28__", "102", 1},
+    {"avr64ea32", "__AVR_AVR64EA32__", "102", 1},
+    {"avr64ea48", "__AVR_AVR64EA48__", "102", 1},
+    {"avr64sd28", "__AVR_AVR64SD28__", "102", 1},
+    {"avr64sd32", "__AVR_AVR64SD32__", "102", 1},
+    {"avr64sd48", "__AVR_AVR64SD48__", "102", 1},
+
+    {"avr16dd20", "__AVR_AVR16DD20__", "103", 1},
+    {"avr16dd28", "__AVR_AVR16DD28__", "103", 1},
+    {"avr16dd32", "__AVR_AVR16DD32__", "103", 1},
+    {"avr16du14", "__AVR_AVR16DU14__", "103", 1},
+    {"avr16du20", "__AVR_AVR16DU20__", "103", 1},
+    {"avr16du28", "__AVR_AVR16DU28__", "103", 1},
+    {"avr16du32", "__AVR_AVR16DU32__", "103", 1},
+    {"avr32da28", "__AVR_AVR32DA28__", "103", 1},
+    {"avr32da32", "__AVR_AVR32DA32__", "103", 1},
+    {"avr32da48", "__AVR_AVR32DA48__", "103", 1},
+    {"avr32db28", "__AVR_AVR32DB28__", "103", 1},
+    {"avr32db32", "__AVR_AVR32DB32__", "103", 1},
+    {"avr32db48", "__AVR_AVR32DB48__", "103", 1},
+    {"avr32dd14", "__AVR_AVR32DD14__", "103", 1},
+    {"avr32dd20", "__AVR_AVR32DD20__", "103", 1},
+    {"avr32dd28", "__AVR_AVR32DD28__", "103", 1},
+    {"avr32dd32", "__AVR_AVR32DD32__", "103", 1},
+    {"avr32du14", "__AVR_AVR32DU14__", "103", 1},
+    {"avr32du20", "__AVR_AVR32DU20__", "103", 1},
+    {"avr32du28", "__AVR_AVR32DU28__", "103", 1},
+    {"avr32du32", "__AVR_AVR32DU32__", "103", 1},
+    {"avr16eb14", "__AVR_AVR16EB14__", "103", 1},
+    {"avr16eb20", "__AVR_AVR16EB20__", "103", 1},
+    {"avr16eb28", "__AVR_AVR16EB28__", "103", 1},
+    {"avr16eb32", "__AVR_AVR16EB32__", "103", 1},
+    {"avr16ea28", "__AVR_AVR16EA28__", "103", 1},
+    {"avr16ea32", "__AVR_AVR16EA32__", "103", 1},
+    {"avr16ea48", "__AVR_AVR16EA48__", "103", 1},
+    {"avr32ea28", "__AVR_AVR32EA28__", "103", 1},
+    {"avr32ea32", "__AVR_AVR32EA32__", "103", 1},
+    {"avr32ea48", "__AVR_AVR32EA48__", "103", 1},
+    {"avr32sd20", "__AVR_AVR32SD20__", "103", 1},
+    {"avr32sd28", "__AVR_AVR32SD28__", "103", 1},
+    {"avr32sd32", "__AVR_AVR32SD32__", "103", 1},
+    {"avr128da28", "__AVR_AVR128DA28__", "104", 2},
+    {"avr128da32", "__AVR_AVR128DA32__", "104", 2},
+    {"avr128da48", "__AVR_AVR128DA48__", "104", 2},
+    {"avr128da64", "__AVR_AVR128DA64__", "104", 2},
+    {"avr128db28", "__AVR_AVR128DB28__", "104", 2},
+    {"avr128db32", "__AVR_AVR128DB32__", "104", 2},
+    {"avr128db48", "__AVR_AVR128DB48__", "104", 2},
+    {"avr128db64", "__AVR_AVR128DB64__", "104", 2},
 };
 
 } // namespace targets
diff --git a/external/llvm-project/clang/lib/Basic/Targets/BPF.h b/external/llvm-project/clang/lib/Basic/Targets/BPF.h
index d1f68b842348..d9e5cf4d8a92 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/BPF.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/BPF.h
@@ -94,7 +94,7 @@ class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo {
     default:
       return CCCR_Warning;
     case CC_C:
-    case CC_OpenCLKernel:
+    case CC_DeviceKernel:
       return CCCR_OK;
     }
   }
diff --git a/external/llvm-project/clang/lib/Basic/Targets/DirectX.h b/external/llvm-project/clang/lib/Basic/Targets/DirectX.h
index 566acd6bb3cf..1729a014c2de 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/DirectX.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/DirectX.h
@@ -45,6 +45,7 @@ static const unsigned DirectXAddrSpaceMap[] = {
     2, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/external/llvm-project/clang/lib/Basic/Targets/LoongArch.cpp b/external/llvm-project/clang/lib/Basic/Targets/LoongArch.cpp
index 67476ee651c2..5312767498d9 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/LoongArch.cpp
+++ b/external/llvm-project/clang/lib/Basic/Targets/LoongArch.cpp
@@ -392,6 +392,73 @@ bool LoongArchTargetInfo::handleTargetFeatures(
   return true;
 }
 
+enum class AttrFeatureKind { Arch, Tune, NoFeature, Feature };
+
+static std::pair<AttrFeatureKind, llvm::StringRef>
+getAttrFeatureTypeAndValue(llvm::StringRef AttrFeature) {
+  if (auto Split = AttrFeature.split("="); !Split.second.empty()) {
+    if (Split.first.trim() == "arch")
+      return {AttrFeatureKind::Arch, Split.second.trim()};
+    if (Split.first.trim() == "tune")
+      return {AttrFeatureKind::Tune, Split.second.trim()};
+  }
+  if (AttrFeature.starts_with("no-"))
+    return {AttrFeatureKind::NoFeature, AttrFeature.drop_front(3)};
+  return {AttrFeatureKind::Feature, AttrFeature};
+}
+
+ParsedTargetAttr
+LoongArchTargetInfo::parseTargetAttr(StringRef Features) const {
+  ParsedTargetAttr Ret;
+  if (Features == "default")
+    return Ret;
+  SmallVector<StringRef, 1> AttrFeatures;
+  Features.split(AttrFeatures, ",");
+
+  for (auto &Feature : AttrFeatures) {
+    auto [Kind, Value] = getAttrFeatureTypeAndValue(Feature.trim());
+
+    switch (Kind) {
+    case AttrFeatureKind::Arch: {
+      if (llvm::LoongArch::isValidArchName(Value) || Value == "la64v1.0" ||
+          Value == "la64v1.1") {
+        std::vector<llvm::StringRef> ArchFeatures;
+        if (llvm::LoongArch::getArchFeatures(Value, ArchFeatures)) {
+          Ret.Features.insert(Ret.Features.end(), ArchFeatures.begin(),
+                              ArchFeatures.end());
+        }
+
+        if (!Ret.CPU.empty())
+          Ret.Duplicate = "arch=";
+        else if (Value == "la64v1.0" || Value == "la64v1.1")
+          Ret.CPU = "loongarch64";
+        else
+          Ret.CPU = Value;
+      } else {
+        Ret.Features.push_back("!arch=" + Value.str());
+      }
+      break;
+    }
+
+    case AttrFeatureKind::Tune:
+      if (!Ret.Tune.empty())
+        Ret.Duplicate = "tune=";
+      else
+        Ret.Tune = Value;
+      break;
+
+    case AttrFeatureKind::NoFeature:
+      Ret.Features.push_back("-" + Value.str());
+      break;
+
+    case AttrFeatureKind::Feature:
+      Ret.Features.push_back("+" + Value.str());
+      break;
+    }
+  }
+  return Ret;
+}
+
 bool LoongArchTargetInfo::isValidCPUName(StringRef Name) const {
   return llvm::LoongArch::isValidCPUName(Name);
 }
@@ -400,3 +467,7 @@ void LoongArchTargetInfo::fillValidCPUList(
     SmallVectorImpl<StringRef> &Values) const {
   llvm::LoongArch::fillValidCPUList(Values);
 }
+
+bool LoongArchTargetInfo::isValidFeatureName(StringRef Name) const {
+  return llvm::LoongArch::isValidFeatureName(Name);
+}
diff --git a/external/llvm-project/clang/lib/Basic/Targets/LoongArch.h b/external/llvm-project/clang/lib/Basic/Targets/LoongArch.h
index 4c7b53abfef9..3ad5abca927b 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/LoongArch.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/LoongArch.h
@@ -49,10 +49,14 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
     HasFeatureLD_SEQ_SA = false;
     HasFeatureDiv32 = false;
     HasFeatureSCQ = false;
+    BFloat16Width = 16;
+    BFloat16Align = 16;
+    BFloat16Format = &llvm::APFloat::BFloat();
     LongDoubleWidth = 128;
     LongDoubleAlign = 128;
     LongDoubleFormat = &llvm::APFloat::IEEEquad();
     MCountName = "_mcount";
+    HasFloat16 = true;
     SuitableAlign = 128;
     WCharType = SignedInt;
     WIntType = UnsignedInt;
@@ -98,9 +102,16 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
 
   bool hasBitIntType() const override { return true; }
 
+  bool hasBFloat16Type() const override { return true; }
+
+  bool useFP16ConversionIntrinsics() const override { return false; }
+
   bool handleTargetFeatures(std::vector<std::string> &Features,
                             DiagnosticsEngine &Diags) override;
 
+  ParsedTargetAttr parseTargetAttr(StringRef Str) const override;
+  bool supportsTargetAttributeTune() const override { return true; }
+
   bool
   initFeatureMap(llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags,
                  StringRef CPU,
@@ -110,6 +121,7 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
 
   bool isValidCPUName(StringRef Name) const override;
   void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override;
+  bool isValidFeatureName(StringRef Name) const override;
 };
 
 class LLVM_LIBRARY_VISIBILITY LoongArch32TargetInfo
diff --git a/external/llvm-project/clang/lib/Basic/Targets/Mips.cpp b/external/llvm-project/clang/lib/Basic/Targets/Mips.cpp
index d693b19a2902..34837cc363a3 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/Mips.cpp
+++ b/external/llvm-project/clang/lib/Basic/Targets/Mips.cpp
@@ -336,7 +336,7 @@ WindowsMipsTargetInfo::checkCallingConvention(CallingConv CC) const {
   case CC_X86VectorCall:
     return CCCR_Ignore;
   case CC_C:
-  case CC_OpenCLKernel:
+  case CC_DeviceKernel:
   case CC_PreserveMost:
   case CC_PreserveAll:
   case CC_Swift:
diff --git a/external/llvm-project/clang/lib/Basic/Targets/NVPTX.h b/external/llvm-project/clang/lib/Basic/Targets/NVPTX.h
index fbb46001b0f9..33c29586359b 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/NVPTX.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/NVPTX.h
@@ -49,6 +49,7 @@ static const unsigned NVPTXAddrSpaceMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/external/llvm-project/clang/lib/Basic/Targets/SPIR.cpp b/external/llvm-project/clang/lib/Basic/Targets/SPIR.cpp
index d9632cf92174..610f4a3eecc6 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/SPIR.cpp
+++ b/external/llvm-project/clang/lib/Basic/Targets/SPIR.cpp
@@ -87,7 +87,8 @@ void SPIR64TargetInfo::getTargetDefines(const LangOptions &Opts,
 void BaseSPIRVTargetInfo::getTargetDefines(const LangOptions &Opts,
                                            MacroBuilder &Builder) const {
   DefineStd(Builder, "SPIRV", Opts);
-  DefineStd(Builder, "spirv", Opts);
+  if (Opts.HLSL)
+    DefineStd(Builder, "spirv", Opts);
 }
 
 void SPIRVTargetInfo::getTargetDefines(const LangOptions &Opts,
diff --git a/external/llvm-project/clang/lib/Basic/Targets/SPIR.h b/external/llvm-project/clang/lib/Basic/Targets/SPIR.h
index ce01f2c399a3..10b3108b5fe8 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/SPIR.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/SPIR.h
@@ -46,10 +46,11 @@ static const unsigned SPIRDefIsPrivMap[] = {
     0,  // ptr32_sptr
     0,  // ptr32_uptr
     0,  // ptr64
-    0,  // hlsl_groupshared
+    3,  // hlsl_groupshared
     12, // hlsl_constant
     10, // hlsl_private
     11, // hlsl_device
+    7,  // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -81,10 +82,11 @@ static const unsigned SPIRDefIsGenMap[] = {
     0,  // ptr32_sptr
     0,  // ptr32_uptr
     0,  // ptr64
-    0,  // hlsl_groupshared
+    3,  // hlsl_groupshared
     0,  // hlsl_constant
     10, // hlsl_private
     11, // hlsl_device
+    7,  // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -191,7 +193,7 @@ class LLVM_LIBRARY_VISIBILITY BaseSPIRTargetInfo : public TargetInfo {
   }
 
   CallingConvCheckResult checkCallingConvention(CallingConv CC) const override {
-    return (CC == CC_SpirFunction || CC == CC_OpenCLKernel) ? CCCR_OK
+    return (CC == CC_SpirFunction || CC == CC_DeviceKernel) ? CCCR_OK
                                                             : CCCR_Warning;
   }
 
diff --git a/external/llvm-project/clang/lib/Basic/Targets/SystemZ.h b/external/llvm-project/clang/lib/Basic/Targets/SystemZ.h
index 1f69530c4757..1af6122c7048 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/SystemZ.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/SystemZ.h
@@ -45,6 +45,7 @@ static const unsigned ZOSAddressMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     0  // wasm_funcref
 };
 
@@ -244,7 +245,7 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
     switch (CC) {
     case CC_C:
     case CC_Swift:
-    case CC_OpenCLKernel:
+    case CC_DeviceKernel:
       return CCCR_OK;
     case CC_SwiftAsync:
       return CCCR_Error;
diff --git a/external/llvm-project/clang/lib/Basic/Targets/TCE.h b/external/llvm-project/clang/lib/Basic/Targets/TCE.h
index f6e582120447..005cab981947 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/TCE.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/TCE.h
@@ -54,6 +54,7 @@ static const unsigned TCEOpenCLAddrSpaceMap[] = {
     0, // hlsl_constant
     0, // hlsl_private
     0, // hlsl_device
+    0, // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/external/llvm-project/clang/lib/Basic/Targets/WebAssembly.h b/external/llvm-project/clang/lib/Basic/Targets/WebAssembly.h
index 04f0cb5df460..d5aee5c0bd0e 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/WebAssembly.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/WebAssembly.h
@@ -45,6 +45,7 @@ static const unsigned WebAssemblyAddrSpaceMap[] = {
     0,  // hlsl_constant
     0,  // hlsl_private
     0,  // hlsl_device
+    0,  // hlsl_input
     20, // wasm_funcref
 };
 
diff --git a/external/llvm-project/clang/lib/Basic/Targets/X86.h b/external/llvm-project/clang/lib/Basic/Targets/X86.h
index 780385f9c9bc..ecb31ffa4750 100644
--- a/external/llvm-project/clang/lib/Basic/Targets/X86.h
+++ b/external/llvm-project/clang/lib/Basic/Targets/X86.h
@@ -49,6 +49,7 @@ static const unsigned X86AddrSpaceMap[] = {
     0,   // hlsl_constant
     0,   // hlsl_private
     0,   // hlsl_device
+    0,   // hlsl_input
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -408,10 +409,11 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
     case CC_Swift:
     case CC_X86Pascal:
     case CC_IntelOclBicc:
-    case CC_OpenCLKernel:
       return CCCR_OK;
     case CC_SwiftAsync:
       return CCCR_Error;
+    case CC_DeviceKernel:
+      return IsOpenCL ? CCCR_OK : CCCR_Warning;
     default:
       return CCCR_Warning;
     }
@@ -439,7 +441,13 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   uint64_t getPointerAlignV(LangAS AddrSpace) const override {
     return getPointerWidthV(AddrSpace);
   }
+  void adjust(DiagnosticsEngine &Diags, LangOptions &Opts) override {
+    TargetInfo::adjust(Diags, Opts);
+    IsOpenCL = Opts.OpenCL;
+  }
 
+private:
+  bool IsOpenCL = false;
 };
 
 // X86-32 generic target
@@ -639,6 +647,7 @@ class LLVM_LIBRARY_VISIBILITY CygwinX86_32TargetInfo : public X86_32TargetInfo {
   CygwinX86_32TargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts)
       : X86_32TargetInfo(Triple, Opts) {
     this->WCharType = TargetInfo::UnsignedShort;
+    this->WIntType = TargetInfo::UnsignedInt;
     DoubleAlign = LongLongAlign = 64;
     resetDataLayout("e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-"
                     "i128:128-f80:32-n8:16:32-a:0:32-S32",
@@ -785,8 +794,9 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo {
     case CC_PreserveAll:
     case CC_PreserveNone:
     case CC_X86RegCall:
-    case CC_OpenCLKernel:
       return CCCR_OK;
+    case CC_DeviceKernel:
+      return IsOpenCL ? CCCR_OK : CCCR_Warning;
     default:
       return CCCR_Warning;
     }
@@ -817,7 +827,6 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo {
     return X86TargetInfo::validateGlobalRegisterVariable(RegName, RegSize,
                                                          HasSizeMismatch);
   }
-
   void setMaxAtomicWidth() override {
     if (hasFeature("cx16"))
       MaxAtomicInlineWidth = 128;
@@ -829,6 +838,14 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo {
   size_t getMaxBitIntWidth() const override {
     return llvm::IntegerType::MAX_INT_BITS;
   }
+
+  void adjust(DiagnosticsEngine &Diags, LangOptions &Opts) override {
+    TargetInfo::adjust(Diags, Opts);
+    IsOpenCL = Opts.OpenCL;
+  }
+
+private:
+  bool IsOpenCL = false;
 };
 
 // x86-64 UEFI target
@@ -914,7 +931,7 @@ class LLVM_LIBRARY_VISIBILITY WindowsX86_64TargetInfo
     case CC_Swift:
     case CC_SwiftAsync:
     case CC_X86RegCall:
-    case CC_OpenCLKernel:
+    case CC_DeviceKernel:
       return CCCR_OK;
     default:
       return CCCR_Warning;
@@ -966,6 +983,7 @@ class LLVM_LIBRARY_VISIBILITY CygwinX86_64TargetInfo : public X86_64TargetInfo {
   CygwinX86_64TargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts)
       : X86_64TargetInfo(Triple, Opts) {
     this->WCharType = TargetInfo::UnsignedShort;
+    this->WIntType = TargetInfo::UnsignedInt;
   }
 
   void getTargetDefines(const LangOptions &Opts,
@@ -979,6 +997,10 @@ class LLVM_LIBRARY_VISIBILITY CygwinX86_64TargetInfo : public X86_64TargetInfo {
     if (Opts.CPlusPlus)
       Builder.defineMacro("_GNU_SOURCE");
   }
+
+  BuiltinVaListKind getBuiltinVaListKind() const override {
+    return TargetInfo::CharPtrBuiltinVaList;
+  }
 };
 
 class LLVM_LIBRARY_VISIBILITY DarwinX86_64TargetInfo
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/Address.h b/external/llvm-project/clang/lib/CIR/CodeGen/Address.h
index 2cc8ada78319..6f76c3ebb1c5 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/Address.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/Address.h
@@ -21,6 +21,9 @@
 
 namespace clang::CIRGen {
 
+// Forward declaration to avoid a circular dependency
+class CIRGenBuilderTy;
+
 class Address {
 
   // The boolean flag indicates whether the pointer is known to be non-null.
@@ -65,11 +68,22 @@ class Address {
     return pointerAndKnownNonNull.getPointer() != nullptr;
   }
 
+  /// Return address with different element type, a bitcast pointer, and
+  /// the same alignment.
+  Address withElementType(CIRGenBuilderTy &builder, mlir::Type ElemTy) const;
+
   mlir::Value getPointer() const {
     assert(isValid());
     return pointerAndKnownNonNull.getPointer();
   }
 
+  mlir::Value getBasePointer() const {
+    // TODO(cir): Remove the version above when we catchup with OG codegen on
+    // ptr auth.
+    assert(isValid() && "pointer isn't valid");
+    return getPointer();
+  }
+
   mlir::Type getType() const {
     assert(mlir::cast<cir::PointerType>(
                pointerAndKnownNonNull.getPointer().getType())
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
index 5620821a5375..9cec17bcb2fd 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
@@ -38,3 +38,43 @@ mlir::Value CIRGenBuilderTy::getArrayElement(mlir::Location arrayLocBegin,
   const mlir::Type flatPtrTy = basePtr.getType();
   return create<cir::PtrStrideOp>(arrayLocEnd, flatPtrTy, basePtr, idx);
 }
+
+cir::ConstantOp CIRGenBuilderTy::getConstInt(mlir::Location loc,
+                                             llvm::APSInt intVal) {
+  bool isSigned = intVal.isSigned();
+  unsigned width = intVal.getBitWidth();
+  cir::IntType t = isSigned ? getSIntNTy(width) : getUIntNTy(width);
+  return getConstInt(loc, t,
+                     isSigned ? intVal.getSExtValue() : intVal.getZExtValue());
+}
+
+cir::ConstantOp CIRGenBuilderTy::getConstInt(mlir::Location loc,
+                                             llvm::APInt intVal) {
+  return getConstInt(loc, llvm::APSInt(intVal));
+}
+
+cir::ConstantOp CIRGenBuilderTy::getConstInt(mlir::Location loc, mlir::Type t,
+                                             uint64_t c) {
+  assert(mlir::isa<cir::IntType>(t) && "expected cir::IntType");
+  return create<cir::ConstantOp>(loc, cir::IntAttr::get(t, c));
+}
+
+cir::ConstantOp
+clang::CIRGen::CIRGenBuilderTy::getConstFP(mlir::Location loc, mlir::Type t,
+                                           llvm::APFloat fpVal) {
+  assert(mlir::isa<cir::CIRFPTypeInterface>(t) &&
+         "expected floating point type");
+  return create<cir::ConstantOp>(loc, getAttr<cir::FPAttr>(t, fpVal));
+}
+
+// This can't be defined in Address.h because that file is included by
+// CIRGenBuilder.h
+Address Address::withElementType(CIRGenBuilderTy &builder,
+                                 mlir::Type elemTy) const {
+  assert(!cir::MissingFeatures::addressOffset());
+  assert(!cir::MissingFeatures::addressIsKnownNonNull());
+  assert(!cir::MissingFeatures::addressPointerAuthInfo());
+
+  return Address(builder.createPtrBitcast(getBasePointer(), elemTy), elemTy,
+                 getAlignment());
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.h
index b8548487d5b3..e38faba83b80 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -11,10 +11,12 @@
 
 #include "Address.h"
 #include "CIRGenTypeCache.h"
+#include "clang/CIR/Interfaces/CIRFPTypeInterface.h"
 #include "clang/CIR/MissingFeatures.h"
 
 #include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
 #include "clang/CIR/MissingFeatures.h"
+#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
 
 namespace clang::CIRGen {
@@ -22,6 +24,7 @@ namespace clang::CIRGen {
 class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   const CIRGenTypeCache &typeCache;
   llvm::StringMap<unsigned> recordNames;
+  llvm::StringMap<unsigned> globalsVersioning;
 
 public:
   CIRGenBuilderTy(mlir::MLIRContext &mlirContext, const CIRGenTypeCache &tc)
@@ -83,6 +86,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   cir::RecordType::RecordKind getRecordKind(const clang::TagTypeKind kind) {
     switch (kind) {
     case clang::TagTypeKind::Class:
+      return cir::RecordType::Class;
     case clang::TagTypeKind::Struct:
       return cir::RecordType::Struct;
     case clang::TagTypeKind::Union:
@@ -95,6 +99,34 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     llvm_unreachable("Unsupported record kind");
   }
 
+  /// Get a CIR named record type.
+  ///
+  /// If a record already exists and is complete, but the client tries to fetch
+  /// it with a different set of attributes, this method will crash.
+  cir::RecordType getCompleteRecordTy(llvm::ArrayRef<mlir::Type> members,
+                                      llvm::StringRef name, bool packed,
+                                      bool padded) {
+    const auto nameAttr = getStringAttr(name);
+    auto kind = cir::RecordType::RecordKind::Struct;
+    assert(!cir::MissingFeatures::astRecordDeclAttr());
+
+    // Create or get the record.
+    auto type =
+        getType<cir::RecordType>(members, nameAttr, packed, padded, kind);
+
+    // If we found an existing type, verify that either it is incomplete or
+    // it matches the requested attributes.
+    assert(!type.isIncomplete() ||
+           (type.getMembers() == members && type.getPacked() == packed &&
+            type.getPadded() == padded));
+
+    // Complete an incomplete record or ensure the existing complete record
+    // matches the requested attributes.
+    type.complete(members, packed, padded);
+
+    return type;
+  }
+
   /// Get an incomplete CIR struct type. If we have a complete record
   /// declaration, we may create an incomplete type and then add the
   /// members, so \p rd here may be complete.
@@ -108,8 +140,9 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   }
 
   bool isSized(mlir::Type ty) {
-    if (mlir::isa<cir::PointerType, cir::ArrayType, cir::BoolType,
-                  cir::IntType>(ty))
+    if (mlir::isa<cir::PointerType, cir::ArrayType, cir::BoolType, cir::IntType,
+                  cir::CIRFPTypeInterface, cir::ComplexType, cir::RecordType>(
+            ty))
       return true;
 
     if (const auto vt = mlir::dyn_cast<cir::VectorType>(ty))
@@ -200,6 +233,15 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   cir::IntType getUInt32Ty() { return typeCache.UInt32Ty; }
   cir::IntType getUInt64Ty() { return typeCache.UInt64Ty; }
 
+  cir::ConstantOp getConstInt(mlir::Location loc, llvm::APSInt intVal);
+
+  cir::ConstantOp getConstInt(mlir::Location loc, llvm::APInt intVal);
+
+  cir::ConstantOp getConstInt(mlir::Location loc, mlir::Type t, uint64_t c);
+
+  cir::ConstantOp getConstFP(mlir::Location loc, mlir::Type t,
+                             llvm::APFloat fpVal);
+
   bool isInt8Ty(mlir::Type i) {
     return i == typeCache.UInt8Ty || i == typeCache.SInt8Ty;
   }
@@ -280,6 +322,30 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     return create<cir::BinOp>(loc, cir::BinOpKind::Div, lhs, rhs);
   }
 
+  Address createBaseClassAddr(mlir::Location loc, Address addr,
+                              mlir::Type destType, unsigned offset,
+                              bool assumeNotNull) {
+    if (destType == addr.getElementType())
+      return addr;
+
+    auto ptrTy = getPointerTo(destType);
+    auto baseAddr = create<cir::BaseClassAddrOp>(
+        loc, ptrTy, addr.getPointer(), mlir::APInt(64, offset), assumeNotNull);
+    return Address(baseAddr, destType, addr.getAlignment());
+  }
+
+  /// Cast the element type of the given address to a different type,
+  /// preserving information like the alignment.
+  Address createElementBitCast(mlir::Location loc, Address addr,
+                               mlir::Type destType) {
+    if (destType == addr.getElementType())
+      return addr;
+
+    auto ptrTy = getPointerTo(destType);
+    return Address(createBitcast(loc, addr.getPointer(), ptrTy), destType,
+                   addr.getAlignment());
+  }
+
   cir::LoadOp createLoad(mlir::Location loc, Address addr,
                          bool isVolatile = false) {
     mlir::IntegerAttr align = getAlignmentAttr(addr.getAlignment());
@@ -294,6 +360,12 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), align);
   }
 
+  mlir::Value createComplexCreate(mlir::Location loc, mlir::Value real,
+                                  mlir::Value imag) {
+    auto resultComplexTy = cir::ComplexType::get(real.getType());
+    return create<cir::ComplexCreateOp>(loc, resultComplexTy, real, imag);
+  }
+
   /// Create a cir.ptr_stride operation to get access to an array element.
   /// \p idx is the index of the element to access, \p shouldDecay is true if
   /// the result should decay to a pointer to the element type.
@@ -306,6 +378,23 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   /// pointed to by \p arrayPtr.
   mlir::Value maybeBuildArrayDecay(mlir::Location loc, mlir::Value arrayPtr,
                                    mlir::Type eltTy);
+
+  /// Creates a versioned global variable. If the symbol is already taken, an ID
+  /// will be appended to the symbol. The returned global must always be queried
+  /// for its name so it can be referenced correctly.
+  [[nodiscard]] cir::GlobalOp
+  createVersionedGlobal(mlir::ModuleOp module, mlir::Location loc,
+                        mlir::StringRef name, mlir::Type type,
+                        cir::GlobalLinkageKind linkage) {
+    // Create a unique name if the given name is already taken.
+    std::string uniqueName;
+    if (unsigned version = globalsVersioning[name.str()]++)
+      uniqueName = name.str() + "." + std::to_string(version);
+    else
+      uniqueName = name.str();
+
+    return createGlobal(module, loc, uniqueName, type, linkage);
+  }
 };
 
 } // namespace clang::CIRGen
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
new file mode 100644
index 000000000000..c59ac78210f8
--- /dev/null
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code to emit Builtin calls as CIR or a function call to be
+// later resolved.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenCall.h"
+#include "CIRGenFunction.h"
+#include "CIRGenModule.h"
+#include "CIRGenValue.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Value.h"
+#include "mlir/Support/LLVM.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/GlobalDecl.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
+                                       const CallExpr *e,
+                                       ReturnValueSlot returnValue) {
+  // See if we can constant fold this builtin.  If so, don't emit it at all.
+  // TODO: Extend this handling to all builtin calls that we can constant-fold.
+  Expr::EvalResult result;
+  if (e->isPRValue() && e->EvaluateAsRValue(result, cgm.getASTContext()) &&
+      !result.hasSideEffects()) {
+    if (result.Val.isInt()) {
+      return RValue::get(builder.getConstInt(getLoc(e->getSourceRange()),
+                                             result.Val.getInt()));
+    }
+    if (result.Val.isFloat()) {
+      // Note: we are using result type of CallExpr to determine the type of
+      // the constant. Classic codegen uses the result value to determine the
+      // type. We feel it should be Ok to use expression type because it is
+      // hard to imagine a builtin function evaluates to a value that
+      // over/underflows its own defined type.
+      mlir::Type type = convertType(e->getType());
+      return RValue::get(builder.getConstFP(getLoc(e->getExprLoc()), type,
+                                            result.Val.getFloat()));
+    }
+  }
+
+  mlir::Location loc = getLoc(e->getExprLoc());
+  cgm.errorNYI(loc, "non constant foldable builtin calls");
+  return getUndefRValue(e->getType());
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXX.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXX.cpp
new file mode 100644
index 000000000000..51751483d34e
--- /dev/null
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXX.cpp
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with C++ code generation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenFunction.h"
+#include "CIRGenModule.h"
+
+#include "clang/AST/GlobalDecl.h"
+#include "clang/CIR/MissingFeatures.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+cir::FuncOp CIRGenModule::codegenCXXStructor(GlobalDecl gd) {
+  const CIRGenFunctionInfo &fnInfo =
+      getTypes().arrangeCXXStructorDeclaration(gd);
+  cir::FuncType funcType = getTypes().getFunctionType(fnInfo);
+  cir::FuncOp fn = getAddrOfCXXStructor(gd, &fnInfo, /*FnType=*/nullptr,
+                                        /*DontDefer=*/true, ForDefinition);
+  assert(!cir::MissingFeatures::opFuncLinkage());
+  CIRGenFunction cgf{*this, builder};
+  curCGF = &cgf;
+  {
+    mlir::OpBuilder::InsertionGuard guard(builder);
+    cgf.generateCode(gd, fn, funcType);
+  }
+  curCGF = nullptr;
+
+  setNonAliasAttributes(gd, fn);
+  assert(!cir::MissingFeatures::opFuncAttributesForDefinition());
+  return fn;
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp
index e368ccd006bb..6cf4e5c658fb 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.cpp
@@ -33,13 +33,24 @@ void CIRGenCXXABI::buildThisParam(CIRGenFunction &cgf,
                                 &cgm.getASTContext().Idents.get("this"),
                                 md->getThisType(), ImplicitParamKind::CXXThis);
   params.push_back(thisDecl);
-
-  // Classic codegen save thisDecl in CodeGenFunction::CXXABIThisDecl, but it
-  // doesn't seem to be needed in CIRGen.
-  assert(!cir::MissingFeatures::cxxabiThisDecl());
+  cgf.cxxabiThisDecl = thisDecl;
 
   // Classic codegen computes the alignment of thisDecl and saves it in
-  // CodeGenFunction::CXXABIThisAlignment, but it doesn't seem to be needed in
-  // CIRGen.
+  // CodeGenFunction::CXXABIThisAlignment, but it is only used in emitTypeCheck
+  // in CodeGenFunction::StartFunction().
   assert(!cir::MissingFeatures::cxxabiThisAlignment());
 }
+
+mlir::Value CIRGenCXXABI::loadIncomingCXXThis(CIRGenFunction &cgf) {
+  ImplicitParamDecl *vd = getThisDecl(cgf);
+  Address addr = cgf.getAddrOfLocalVar(vd);
+  return cgf.getBuilder().create<cir::LoadOp>(
+      cgf.getLoc(vd->getLocation()), addr.getElementType(), addr.getPointer());
+}
+
+void CIRGenCXXABI::setCXXABIThisValue(CIRGenFunction &cgf,
+                                      mlir::Value thisPtr) {
+  /// Initialize the 'this' slot.
+  assert(getThisDecl(cgf) && "no 'this' variable for function");
+  cgf.cxxabiThisValue = thisPtr;
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.h
index b5c33a29442c..2d967fd307e0 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCXXABI.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_LIB_CIR_CIRGENCXXABI_H
 
 #include "CIRGenCall.h"
+#include "CIRGenFunction.h"
 #include "CIRGenModule.h"
 
 #include "clang/AST/Mangle.h"
@@ -34,7 +35,21 @@ class CIRGenCXXABI {
       : cgm(cgm), mangleContext(cgm.getASTContext().createMangleContext()) {}
   virtual ~CIRGenCXXABI();
 
+  void setCXXABIThisValue(CIRGenFunction &cgf, mlir::Value thisPtr);
+
+  /// Emit a single constructor/destructor with the gen type from a C++
+  /// constructor/destructor Decl.
+  virtual void emitCXXStructor(clang::GlobalDecl gd) = 0;
+
 public:
+  clang::ImplicitParamDecl *getThisDecl(CIRGenFunction &cgf) {
+    return cgf.cxxabiThisDecl;
+  }
+
+  /// Emit the ABI-specific prolog for the function
+  virtual void emitInstanceFunctionProlog(SourceLocation Loc,
+                                          CIRGenFunction &cgf) = 0;
+
   /// Get the type of the implicit "this" parameter used by a method. May return
   /// zero if no specific type is applicable, e.g. if the ABI expects the "this"
   /// parameter to point to some artificial offset in a complete object due to
@@ -44,9 +59,19 @@ class CIRGenCXXABI {
     return md->getParent();
   }
 
+  /// Return whether the given global decl needs a VTT (virtual table table)
+  /// parameter.
+  virtual bool needsVTTParameter(clang::GlobalDecl gd) { return false; }
+
   /// Build a parameter variable suitable for 'this'.
   void buildThisParam(CIRGenFunction &cgf, FunctionArgList &params);
 
+  /// Loads the incoming C++ this pointer as it was passed by the caller.
+  mlir::Value loadIncomingCXXThis(CIRGenFunction &cgf);
+
+  /// Emit constructor variants required by this ABI.
+  virtual void emitCXXConstructors(const clang::CXXConstructorDecl *d) = 0;
+
   /// Returns true if the given constructor or destructor is one of the kinds
   /// that the ABI says returns 'this' (only applies when called non-virtually
   /// for destructors).
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.cpp
index b194a8670bfb..0d9064425fa9 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.cpp
@@ -60,6 +60,30 @@ CIRGenCallee CIRGenCallee::prepareConcreteCallee(CIRGenFunction &cgf) const {
   return *this;
 }
 
+void CIRGenFunction::emitAggregateStore(mlir::Value value, Address dest) {
+  // In classic codegen:
+  // Function to store a first-class aggregate into memory. We prefer to
+  // store the elements rather than the aggregate to be more friendly to
+  // fast-isel.
+  // In CIR codegen:
+  // Emit the most simple cir.store possible (e.g. a store for a whole
+  // record), which can later be broken down in other CIR levels (or prior
+  // to dialect codegen).
+
+  // Stored result for the callers of this function expected to be in the same
+  // scope as the value, don't make assumptions about current insertion point.
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointAfter(value.getDefiningOp());
+  builder.createStore(*currSrcLoc, value, dest);
+}
+
+/// Returns the canonical formal type of the given C++ method.
+static CanQual<FunctionProtoType> getFormalType(const CXXMethodDecl *md) {
+  return md->getType()
+      ->getCanonicalTypeUnqualified()
+      .getAs<FunctionProtoType>();
+}
+
 /// Adds the formal parameters in FPT to the given prefix. If any parameter in
 /// FPT has pass_object_size_attrs, then we'll add parameters for those, too.
 /// TODO(cir): this should be shared with LLVM codegen
@@ -76,6 +100,48 @@ static void appendParameterTypes(const CIRGenTypes &cgt,
   cgt.getCGModule().errorNYI("appendParameterTypes: hasExtParameterInfos");
 }
 
+const CIRGenFunctionInfo &
+CIRGenTypes::arrangeCXXStructorDeclaration(GlobalDecl gd) {
+  auto *md = cast<CXXMethodDecl>(gd.getDecl());
+
+  llvm::SmallVector<CanQualType, 16> argTypes;
+  argTypes.push_back(deriveThisType(md->getParent(), md));
+
+  bool passParams = true;
+
+  if (auto *cd = dyn_cast<CXXConstructorDecl>(md)) {
+    // A base class inheriting constructor doesn't get forwarded arguments
+    // needed to construct a virtual base (or base class thereof)
+    if (cd->getInheritedConstructor())
+      cgm.errorNYI(cd->getSourceRange(),
+                   "arrangeCXXStructorDeclaration: inheriting constructor");
+  }
+
+  CanQual<FunctionProtoType> fpt = getFormalType(md);
+
+  if (passParams)
+    appendParameterTypes(*this, argTypes, fpt);
+
+  assert(!cir::MissingFeatures::implicitConstructorArgs());
+
+  RequiredArgs required =
+      (passParams && md->isVariadic() ? RequiredArgs(argTypes.size())
+                                      : RequiredArgs::All);
+
+  CanQualType resultType = theCXXABI.hasThisReturn(gd) ? argTypes.front()
+                           : theCXXABI.hasMostDerivedReturn(gd)
+                               ? astContext.VoidPtrTy
+                               : astContext.VoidTy;
+
+  assert(!theCXXABI.hasThisReturn(gd) &&
+         "Please send PR with a test and remove this");
+
+  assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
+  assert(!cir::MissingFeatures::opCallFnInfoOpts());
+
+  return arrangeCIRFunctionInfo(resultType, argTypes, required);
+}
+
 /// Derives the 'this' type for CIRGen purposes, i.e. ignoring method CVR
 /// qualification. Either or both of `rd` and `md` may be null. A null `rd`
 /// indicates that there is no meaningful 'this' type, and a null `md` can occur
@@ -103,16 +169,56 @@ CanQualType CIRGenTypes::deriveThisType(const CXXRecordDecl *rd,
 /// top of any implicit parameters already stored.
 static const CIRGenFunctionInfo &
 arrangeCIRFunctionInfo(CIRGenTypes &cgt, SmallVectorImpl<CanQualType> &prefix,
-                       CanQual<FunctionProtoType> ftp) {
+                       CanQual<FunctionProtoType> fpt) {
   assert(!cir::MissingFeatures::opCallFnInfoOpts());
   RequiredArgs required =
-      RequiredArgs::getFromProtoWithExtraSlots(ftp, prefix.size());
+      RequiredArgs::getFromProtoWithExtraSlots(fpt, prefix.size());
   assert(!cir::MissingFeatures::opCallExtParameterInfo());
-  appendParameterTypes(cgt, prefix, ftp);
-  CanQualType resultType = ftp->getReturnType().getUnqualifiedType();
+  appendParameterTypes(cgt, prefix, fpt);
+  CanQualType resultType = fpt->getReturnType().getUnqualifiedType();
   return cgt.arrangeCIRFunctionInfo(resultType, prefix, required);
 }
 
+void CIRGenFunction::emitDelegateCallArg(CallArgList &args,
+                                         const VarDecl *param,
+                                         SourceLocation loc) {
+  // StartFunction converted the ABI-lowered parameter(s) into a local alloca.
+  // We need to turn that into an r-value suitable for emitCall
+  Address local = getAddrOfLocalVar(param);
+
+  QualType type = param->getType();
+
+  if (type->getAsCXXRecordDecl()) {
+    cgm.errorNYI(param->getSourceRange(),
+                 "emitDelegateCallArg: record argument");
+    return;
+  }
+
+  // GetAddrOfLocalVar returns a pointer-to-pointer for references, but the
+  // argument needs to be the original pointer.
+  if (type->isReferenceType()) {
+    args.add(
+        RValue::get(builder.createLoad(getLoc(param->getSourceRange()), local)),
+        type);
+  } else if (getLangOpts().ObjCAutoRefCount) {
+    cgm.errorNYI(param->getSourceRange(),
+                 "emitDelegateCallArg: ObjCAutoRefCount");
+    // For the most part, we just need to load the alloca, except that aggregate
+    // r-values are actually pointers to temporaries.
+  } else {
+    args.add(convertTempToRValue(local, type, loc), type);
+  }
+
+  // Deactivate the cleanup for the callee-destructed param that was pushed.
+  assert(!cir::MissingFeatures::thunks());
+  if (type->isRecordType() &&
+      type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
+      param->needsDestruction(getContext())) {
+    cgm.errorNYI(param->getSourceRange(),
+                 "emitDelegateCallArg: callee-destructed param");
+  }
+}
+
 static const CIRGenFunctionInfo &
 arrangeFreeFunctionLikeCall(CIRGenTypes &cgt, CIRGenModule &cgm,
                             const CallArgList &args,
@@ -141,6 +247,44 @@ arrangeFreeFunctionLikeCall(CIRGenTypes &cgt, CIRGenModule &cgm,
   return cgt.arrangeCIRFunctionInfo(retType, argTypes, required);
 }
 
+/// Arrange a call to a C++ method, passing the given arguments.
+///
+/// passProtoArgs indicates whether `args` has args for the parameters in the
+/// given CXXConstructorDecl.
+const CIRGenFunctionInfo &CIRGenTypes::arrangeCXXConstructorCall(
+    const CallArgList &args, const CXXConstructorDecl *d, CXXCtorType ctorKind,
+    bool passProtoArgs) {
+
+  // FIXME: Kill copy.
+  llvm::SmallVector<CanQualType, 16> argTypes;
+  for (const auto &arg : args)
+    argTypes.push_back(astContext.getCanonicalParamType(arg.ty));
+
+  assert(!cir::MissingFeatures::implicitConstructorArgs());
+  // +1 for implicit this, which should always be args[0]
+  unsigned totalPrefixArgs = 1;
+
+  CanQual<FunctionProtoType> fpt = getFormalType(d);
+  RequiredArgs required =
+      passProtoArgs
+          ? RequiredArgs::getFromProtoWithExtraSlots(fpt, totalPrefixArgs)
+          : RequiredArgs::All;
+
+  GlobalDecl gd(d, ctorKind);
+  if (theCXXABI.hasThisReturn(gd))
+    cgm.errorNYI(d->getSourceRange(),
+                 "arrangeCXXConstructorCall: hasThisReturn");
+  if (theCXXABI.hasMostDerivedReturn(gd))
+    cgm.errorNYI(d->getSourceRange(),
+                 "arrangeCXXConstructorCall: hasMostDerivedReturn");
+  CanQualType resultType = astContext.VoidTy;
+
+  assert(!cir::MissingFeatures::opCallFnInfoOpts());
+  assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
+
+  return arrangeCIRFunctionInfo(resultType, argTypes, required);
+}
+
 /// Arrange a call to a C++ method, passing the given arguments.
 ///
 /// numPrefixArgs is the number of the ABI-specific prefix arguments we have. It
@@ -198,7 +342,7 @@ CIRGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *md) {
 /// constructor or destructor.
 const CIRGenFunctionInfo &
 CIRGenTypes::arrangeCXXMethodType(const CXXRecordDecl *rd,
-                                  const FunctionProtoType *ftp,
+                                  const FunctionProtoType *fpt,
                                   const CXXMethodDecl *md) {
   llvm::SmallVector<CanQualType, 16> argTypes;
 
@@ -208,7 +352,7 @@ CIRGenTypes::arrangeCXXMethodType(const CXXRecordDecl *rd,
   assert(!cir::MissingFeatures::opCallFnInfoOpts());
   return ::arrangeCIRFunctionInfo(
       *this, argTypes,
-      ftp->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
+      fpt->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
 }
 
 /// Arrange the argument and result information for the declaration or
@@ -312,8 +456,49 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
       assert(!cir::MissingFeatures::opCallBitcastArg());
       cirCallArgs[argNo] = v;
     } else {
-      assert(!cir::MissingFeatures::opCallAggregateArgs());
-      cgm.errorNYI("emitCall: aggregate function call argument");
+      Address src = Address::invalid();
+      if (!arg.isAggregate())
+        cgm.errorNYI(loc, "emitCall: non-aggregate call argument");
+      else
+        src = arg.hasLValue() ? arg.getKnownLValue().getAddress()
+                              : arg.getKnownRValue().getAggregateAddress();
+
+      // Fast-isel and the optimizer generally like scalar values better than
+      // FCAs, so we flatten them if this is safe to do for this argument.
+      auto argRecordTy = cast<cir::RecordType>(argType);
+      mlir::Type srcTy = src.getElementType();
+      // FIXME(cir): get proper location for each argument.
+      mlir::Location argLoc = loc;
+
+      // If the source type is smaller than the destination type of the
+      // coerce-to logic, copy the source value into a temp alloca the size
+      // of the destination type to allow loading all of it. The bits past
+      // the source value are left undef.
+      // FIXME(cir): add data layout info and compare sizes instead of
+      // matching the types.
+      //
+      // uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
+      // uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
+      // if (SrcSize < DstSize) {
+      assert(!cir::MissingFeatures::dataLayoutTypeAllocSize());
+      if (srcTy != argRecordTy) {
+        cgm.errorNYI(loc, "emitCall: source type does not match argument type");
+      } else {
+        // FIXME(cir): this currently only runs when the types are exactly the
+        // same, but should be when alloc sizes are the same, fix this as soon
+        // as datalayout gets introduced.
+        assert(!cir::MissingFeatures::dataLayoutTypeAllocSize());
+      }
+
+      // assert(NumCIRArgs == STy.getMembers().size());
+      // In LLVMGen: Still only pass the struct without any gaps but mark it
+      // as such somehow.
+      //
+      // In CIRGen: Emit a load from the "whole" struct,
+      // which shall be broken later by some lowering step into multiple
+      // loads.
+      assert(!cir::MissingFeatures::lowerAggregateLoadStore());
+      cirCallArgs[argNo] = builder.createLoad(argLoc, src);
     }
   }
 
@@ -352,6 +537,7 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
 
   assert(!cir::MissingFeatures::opCallAttrs());
 
+  mlir::Location callLoc = loc;
   cir::CIRCallOpInterface theCall = emitCallLikeOp(
       *this, loc, indirectFuncTy, indirectFuncVal, directFuncOp, cirCallArgs);
 
@@ -365,6 +551,19 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
   if (isa<cir::VoidType>(retCIRTy))
     return getUndefRValue(retTy);
   switch (getEvaluationKind(retTy)) {
+  case cir::TEK_Aggregate: {
+    Address destPtr = returnValue.getValue();
+
+    if (!destPtr.isValid())
+      destPtr = createMemTemp(retTy, callLoc, getCounterAggTmpAsString());
+
+    mlir::ResultRange results = theCall->getOpResults();
+    assert(results.size() <= 1 && "multiple returns from a call");
+
+    SourceLocRAIIObject loc{*this, callLoc};
+    emitAggregateStore(results[0], destPtr);
+    return RValue::getAggregate(destPtr);
+  }
   case cir::TEK_Scalar: {
     mlir::ResultRange results = theCall->getOpResults();
     assert(results.size() == 1 && "unexpected number of returns");
@@ -381,7 +580,6 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
     return RValue::get(results[0]);
   }
   case cir::TEK_Complex:
-  case cir::TEK_Aggregate:
     cgm.errorNYI(loc, "unsupported evaluation kind of function call result");
     return getUndefRValue(retTy);
   }
@@ -400,10 +598,21 @@ void CIRGenFunction::emitCallArg(CallArgList &args, const clang::Expr *e,
 
   bool hasAggregateEvalKind = hasAggregateEvaluationKind(argType);
 
-  if (hasAggregateEvalKind) {
-    assert(!cir::MissingFeatures::opCallAggregateArgs());
-    cgm.errorNYI(e->getSourceRange(),
-                 "emitCallArg: aggregate function call argument");
+  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
+  // However, we still have to push an EH-only cleanup in case we unwind before
+  // we make it to the call.
+  if (argType->isRecordType() &&
+      argType->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
+    assert(!cir::MissingFeatures::msabi());
+    cgm.errorNYI(e->getSourceRange(), "emitCallArg: msabi is NYI");
+  }
+
+  if (hasAggregateEvalKind && isa<ImplicitCastExpr>(e) &&
+      cast<CastExpr>(e)->getCastKind() == CK_LValueToRValue) {
+    LValue lv = emitLValue(cast<CastExpr>(e)->getSubExpr());
+    assert(lv.isSimple());
+    args.addUncopiedAggregate(lv, argType);
+    return;
   }
 
   args.add(emitAnyExprToTemp(e), argType);
@@ -424,12 +633,13 @@ QualType CIRGenFunction::getVarArgType(const Expr *arg) {
 /// Similar to emitAnyExpr(), however, the result will always be accessible
 /// even if no aggregate location is provided.
 RValue CIRGenFunction::emitAnyExprToTemp(const Expr *e) {
-  assert(!cir::MissingFeatures::opCallAggregateArgs());
+  AggValueSlot aggSlot = AggValueSlot::ignored();
 
   if (hasAggregateEvaluationKind(e->getType()))
-    cgm.errorNYI(e->getSourceRange(), "emit aggregate value to temp");
+    aggSlot = createAggTemp(e->getType(), getLoc(e->getSourceRange()),
+                            getCounterAggTmpAsString());
 
-  return emitAnyExpr(e);
+  return emitAnyExpr(e, aggSlot);
 }
 
 void CIRGenFunction::emitCallArgs(
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.h
index 605625705a75..0353848f3ec0 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenCall.h
@@ -44,16 +44,25 @@ class CIRGenCalleeInfo {
 class CIRGenCallee {
   enum class SpecialKind : uintptr_t {
     Invalid,
+    Builtin,
 
-    Last = Invalid,
+    Last = Builtin,
+  };
+
+  struct BuiltinInfoStorage {
+    const clang::FunctionDecl *decl;
+    unsigned id;
   };
 
   SpecialKind kindOrFunctionPtr;
 
   union {
     CIRGenCalleeInfo abstractInfo;
+    BuiltinInfoStorage builtinInfo;
   };
 
+  explicit CIRGenCallee(SpecialKind kind) : kindOrFunctionPtr(kind) {}
+
 public:
   CIRGenCallee() : kindOrFunctionPtr(SpecialKind::Invalid) {}
 
@@ -69,6 +78,25 @@ class CIRGenCallee {
     return CIRGenCallee(abstractInfo, funcPtr);
   }
 
+  bool isBuiltin() const { return kindOrFunctionPtr == SpecialKind::Builtin; }
+
+  const clang::FunctionDecl *getBuiltinDecl() const {
+    assert(isBuiltin());
+    return builtinInfo.decl;
+  }
+  unsigned getBuiltinID() const {
+    assert(isBuiltin());
+    return builtinInfo.id;
+  }
+
+  static CIRGenCallee forBuiltin(unsigned builtinID,
+                                 const clang::FunctionDecl *builtinDecl) {
+    CIRGenCallee result(SpecialKind::Builtin);
+    result.builtinInfo.decl = builtinDecl;
+    result.builtinInfo.id = builtinID;
+    return result;
+  }
+
   bool isOrdinary() const {
     return uintptr_t(kindOrFunctionPtr) > uintptr_t(SpecialKind::Last);
   }
@@ -105,8 +133,16 @@ struct CallArg {
   CallArg(RValue rv, clang::QualType ty)
       : rv(rv), hasLV(false), isUsed(false), ty(ty) {}
 
+  CallArg(LValue lv, clang::QualType ty)
+      : lv(lv), hasLV(true), isUsed(false), ty(ty) {}
+
   bool hasLValue() const { return hasLV; }
 
+  LValue getKnownLValue() const {
+    assert(hasLV && !isUsed);
+    return lv;
+  }
+
   RValue getKnownRValue() const {
     assert(!hasLV && !isUsed);
     return rv;
@@ -119,6 +155,10 @@ class CallArgList : public llvm::SmallVector<CallArg, 8> {
 public:
   void add(RValue rvalue, clang::QualType type) { emplace_back(rvalue, type); }
 
+  void addUncopiedAggregate(LValue lvalue, clang::QualType type) {
+    emplace_back(lvalue, type);
+  }
+
   /// Add all the arguments from another CallArgList to this one. After doing
   /// this, the old CallArgList retains its list of arguments, but must not
   /// be used to emit a call.
@@ -134,7 +174,15 @@ class CallArgList : public llvm::SmallVector<CallArg, 8> {
 
 /// Contains the address where the return value of a function can be stored, and
 /// whether the address is volatile or not.
-class ReturnValueSlot {};
+class ReturnValueSlot {
+  Address addr = Address::invalid();
+
+public:
+  ReturnValueSlot() = default;
+  ReturnValueSlot(Address addr) : addr(addr) {}
+
+  Address getValue() const { return addr; }
+};
 
 } // namespace clang::CIRGen
 
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenClass.cpp
new file mode 100644
index 000000000000..e59a1fdb837c
--- /dev/null
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenClass.cpp
@@ -0,0 +1,258 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with C++ code generation of classes
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenCXXABI.h"
+#include "CIRGenFunction.h"
+
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/AST/Type.h"
+#include "clang/CIR/MissingFeatures.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+/// Checks whether the given constructor is a valid subject for the
+/// complete-to-base constructor delegation optimization, i.e. emitting the
+/// complete constructor as a simple call to the base constructor.
+bool CIRGenFunction::isConstructorDelegationValid(
+    const CXXConstructorDecl *ctor) {
+  // Currently we disable the optimization for classes with virtual bases
+  // because (1) the address of parameter variables need to be consistent across
+  // all initializers but (2) the delegate function call necessarily creates a
+  // second copy of the parameter variable.
+  //
+  // The limiting example (purely theoretical AFAIK):
+  //   struct A { A(int &c) { c++; } };
+  //   struct A : virtual A {
+  //     B(int count) : A(count) { printf("%d\n", count); }
+  //   };
+  // ...although even this example could in principle be emitted as a delegation
+  // since the address of the parameter doesn't escape.
+  if (ctor->getParent()->getNumVBases())
+    return false;
+
+  // We also disable the optimization for variadic functions because it's
+  // impossible to "re-pass" varargs.
+  if (ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
+    return false;
+
+  // FIXME: Decide if we can do a delegation of a delegating constructor.
+  if (ctor->isDelegatingConstructor())
+    return false;
+
+  return true;
+}
+
+/// This routine generates necessary code to initialize base classes and
+/// non-static data members belonging to this constructor.
+void CIRGenFunction::emitCtorPrologue(const CXXConstructorDecl *cd,
+                                      CXXCtorType ctorType,
+                                      FunctionArgList &args) {
+  if (cd->isDelegatingConstructor())
+    return emitDelegatingCXXConstructorCall(cd, args);
+
+  if (cd->getNumCtorInitializers() != 0) {
+    // There's much more to do here.
+    cgm.errorNYI(cd->getSourceRange(), "emitCtorPrologue: any initializer");
+    return;
+  }
+}
+
+Address CIRGenFunction::loadCXXThisAddress() {
+  assert(curFuncDecl && "loading 'this' without a func declaration?");
+  assert(isa<CXXMethodDecl>(curFuncDecl));
+
+  // Lazily compute CXXThisAlignment.
+  if (cxxThisAlignment.isZero()) {
+    // Just use the best known alignment for the parent.
+    // TODO: if we're currently emitting a complete-object ctor/dtor, we can
+    // always use the complete-object alignment.
+    auto rd = cast<CXXMethodDecl>(curFuncDecl)->getParent();
+    cxxThisAlignment = cgm.getClassPointerAlignment(rd);
+  }
+
+  return Address(loadCXXThis(), cxxThisAlignment);
+}
+
+void CIRGenFunction::emitDelegateCXXConstructorCall(
+    const CXXConstructorDecl *ctor, CXXCtorType ctorType,
+    const FunctionArgList &args, SourceLocation loc) {
+  CallArgList delegateArgs;
+
+  FunctionArgList::const_iterator i = args.begin(), e = args.end();
+  assert(i != e && "no parameters to constructor");
+
+  // this
+  Address thisAddr = loadCXXThisAddress();
+  delegateArgs.add(RValue::get(thisAddr.getPointer()), (*i)->getType());
+  ++i;
+
+  // FIXME: The location of the VTT parameter in the parameter list is specific
+  // to the Itanium ABI and shouldn't be hardcoded here.
+  if (cgm.getCXXABI().needsVTTParameter(curGD)) {
+    cgm.errorNYI(loc, "emitDelegateCXXConstructorCall: VTT parameter");
+    return;
+  }
+
+  // Explicit arguments.
+  for (; i != e; ++i) {
+    const VarDecl *param = *i;
+    // FIXME: per-argument source location
+    emitDelegateCallArg(delegateArgs, param, loc);
+  }
+
+  assert(!cir::MissingFeatures::sanitizers());
+
+  emitCXXConstructorCall(ctor, ctorType, /*ForVirtualBase=*/false,
+                         /*Delegating=*/true, thisAddr, delegateArgs, loc);
+}
+
+void CIRGenFunction::emitDelegatingCXXConstructorCall(
+    const CXXConstructorDecl *ctor, const FunctionArgList &args) {
+  assert(ctor->isDelegatingConstructor());
+
+  Address thisPtr = loadCXXThisAddress();
+
+  assert(!cir::MissingFeatures::objCGC());
+  assert(!cir::MissingFeatures::sanitizers());
+  AggValueSlot aggSlot = AggValueSlot::forAddr(
+      thisPtr, Qualifiers(), AggValueSlot::IsDestructed,
+      AggValueSlot::IsNotAliased, AggValueSlot::MayOverlap,
+      AggValueSlot::IsNotZeroed);
+
+  emitAggExpr(ctor->init_begin()[0]->getInit(), aggSlot);
+
+  const CXXRecordDecl *classDecl = ctor->getParent();
+  if (cgm.getLangOpts().Exceptions && !classDecl->hasTrivialDestructor()) {
+    cgm.errorNYI(ctor->getSourceRange(),
+                 "emitDelegatingCXXConstructorCall: exception");
+    return;
+  }
+}
+
+Address CIRGenFunction::getAddressOfBaseClass(
+    Address value, const CXXRecordDecl *derived,
+    llvm::iterator_range<CastExpr::path_const_iterator> path,
+    bool nullCheckValue, SourceLocation loc) {
+  assert(!path.empty() && "Base path should not be empty!");
+
+  if ((*path.begin())->isVirtual()) {
+    // The implementation here is actually complete, but let's flag this
+    // as an error until the rest of the virtual base class support is in place.
+    cgm.errorNYI(loc, "getAddrOfBaseClass: virtual base");
+    return Address::invalid();
+  }
+
+  // Compute the static offset of the ultimate destination within its
+  // allocating subobject (the virtual base, if there is one, or else
+  // the "complete" object that we see).
+  CharUnits nonVirtualOffset =
+      cgm.computeNonVirtualBaseClassOffset(derived, path);
+
+  // Get the base pointer type.
+  mlir::Type baseValueTy = convertType((path.end()[-1])->getType());
+  assert(!cir::MissingFeatures::addressSpace());
+
+  // The if statement here is redundant now, but it will be needed when we add
+  // support for virtual base classes.
+  // If there is no virtual base, use cir.base_class_addr.  It takes care of
+  // the adjustment and the null pointer check.
+  if (nonVirtualOffset.isZero()) {
+    assert(!cir::MissingFeatures::sanitizers());
+    return builder.createBaseClassAddr(getLoc(loc), value, baseValueTy, 0,
+                                       /*assumeNotNull=*/true);
+  }
+
+  assert(!cir::MissingFeatures::sanitizers());
+
+  // Apply the offset
+  value = builder.createBaseClassAddr(getLoc(loc), value, baseValueTy,
+                                      nonVirtualOffset.getQuantity(),
+                                      /*assumeNotNull=*/true);
+
+  // Cast to the destination type.
+  value = value.withElementType(builder, baseValueTy);
+
+  return value;
+}
+
+void CIRGenFunction::emitCXXConstructorCall(const clang::CXXConstructorDecl *d,
+                                            clang::CXXCtorType type,
+                                            bool forVirtualBase,
+                                            bool delegating,
+                                            AggValueSlot thisAVS,
+                                            const clang::CXXConstructExpr *e) {
+  CallArgList args;
+  Address thisAddr = thisAVS.getAddress();
+  QualType thisType = d->getThisType();
+  mlir::Value thisPtr = thisAddr.getPointer();
+
+  assert(!cir::MissingFeatures::addressSpace());
+
+  args.add(RValue::get(thisPtr), thisType);
+
+  // In LLVM Codegen: If this is a trivial constructor, just emit what's needed.
+  // If this is a union copy constructor, we must emit a memcpy, because the AST
+  // does not model that copy.
+  assert(!cir::MissingFeatures::isMemcpyEquivalentSpecialMember());
+
+  const FunctionProtoType *fpt = d->getType()->castAs<FunctionProtoType>();
+
+  assert(!cir::MissingFeatures::opCallArgEvaluationOrder());
+
+  emitCallArgs(args, fpt, e->arguments(), e->getConstructor(),
+               /*ParamsToSkip=*/0);
+
+  assert(!cir::MissingFeatures::sanitizers());
+  emitCXXConstructorCall(d, type, forVirtualBase, delegating, thisAddr, args,
+                         e->getExprLoc());
+}
+
+void CIRGenFunction::emitCXXConstructorCall(
+    const CXXConstructorDecl *d, CXXCtorType type, bool forVirtualBase,
+    bool delegating, Address thisAddr, CallArgList &args, SourceLocation loc) {
+
+  const CXXRecordDecl *crd = d->getParent();
+
+  // If this is a call to a trivial default constructor:
+  // In LLVM: do nothing.
+  // In CIR: emit as a regular call, other later passes should lower the
+  // ctor call into trivial initialization.
+  assert(!cir::MissingFeatures::isTrivialCtorOrDtor());
+
+  assert(!cir::MissingFeatures::isMemcpyEquivalentSpecialMember());
+
+  bool passPrototypeArgs = true;
+
+  // Check whether we can actually emit the constructor before trying to do so.
+  if (d->getInheritedConstructor()) {
+    cgm.errorNYI(d->getSourceRange(),
+                 "emitCXXConstructorCall: inherited constructor");
+    return;
+  }
+
+  // Insert any ABI-specific implicit constructor arguments.
+  assert(!cir::MissingFeatures::implicitConstructorArgs());
+
+  // Emit the call.
+  auto calleePtr = cgm.getAddrOfCXXStructor(GlobalDecl(d, type));
+  const CIRGenFunctionInfo &info = cgm.getTypes().arrangeCXXConstructorCall(
+      args, d, type, passPrototypeArgs);
+  CIRGenCallee callee = CIRGenCallee::forDirect(calleePtr, GlobalDecl(d, type));
+  cir::CIRCallOpInterface c;
+  emitCall(info, callee, ReturnValueSlot(), args, &c, getLoc(loc));
+
+  if (cgm.getCodeGenOpts().OptimizationLevel != 0 && !crd->isDynamicClass() &&
+      type != Ctor_Base && cgm.getCodeGenOpts().StrictVTablePointers)
+    cgm.errorNYI(d->getSourceRange(), "vtable assumption loads");
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenDecl.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
index 61af33053dc0..afbe92aded80 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
@@ -50,8 +50,7 @@ CIRGenFunction::emitAutoVarAlloca(const VarDecl &d) {
   // A normal fixed sized variable becomes an alloca in the entry block,
   mlir::Type allocaTy = convertTypeForMem(ty);
   // Create the temp alloca and declare variable using it.
-  address = createTempAlloca(allocaTy, alignment, loc, d.getName(),
-                             /*insertIntoFnEntryBlock=*/false);
+  address = createTempAlloca(allocaTy, alignment, loc, d.getName());
   declare(address.getPointer(), &d, ty, getLoc(d.getSourceRange()), alignment);
 
   emission.Addr = address;
@@ -209,8 +208,25 @@ void CIRGenFunction::emitVarDecl(const VarDecl &d) {
   if (d.hasExternalStorage())
     return;
 
-  if (d.getStorageDuration() != SD_Automatic)
-    cgm.errorNYI(d.getSourceRange(), "emitVarDecl automatic storage duration");
+  if (d.getStorageDuration() != SD_Automatic) {
+    // Static sampler variables translated to function calls.
+    if (d.getType()->isSamplerT()) {
+      // Nothing needs to be done here, but let's flag it as an error until we
+      // have a test. It requires OpenCL support.
+      cgm.errorNYI(d.getSourceRange(), "emitVarDecl static sampler type");
+      return;
+    }
+
+    cir::GlobalLinkageKind linkage =
+        cgm.getCIRLinkageVarDefinition(&d, /*IsConstant=*/false);
+
+    // FIXME: We need to force the emission/use of a guard variable for
+    // some variables even if we can constant-evaluate them because
+    // we can't guarantee every translation unit will constant-evaluate them.
+
+    return emitStaticVarDecl(d, linkage);
+  }
+
   if (d.getType().getAddressSpace() == LangAS::opencl_local)
     cgm.errorNYI(d.getSourceRange(), "emitVarDecl openCL address space");
 
@@ -220,6 +236,233 @@ void CIRGenFunction::emitVarDecl(const VarDecl &d) {
   return emitAutoVarDecl(d);
 }
 
+static std::string getStaticDeclName(CIRGenModule &cgm, const VarDecl &d) {
+  if (cgm.getLangOpts().CPlusPlus)
+    return cgm.getMangledName(&d).str();
+
+  // If this isn't C++, we don't need a mangled name, just a pretty one.
+  assert(!d.isExternallyVisible() && "name shouldn't matter");
+  std::string contextName;
+  const DeclContext *dc = d.getDeclContext();
+  if (auto *cd = dyn_cast<CapturedDecl>(dc))
+    dc = cast<DeclContext>(cd->getNonClosureContext());
+  if (const auto *fd = dyn_cast<FunctionDecl>(dc))
+    contextName = std::string(cgm.getMangledName(fd));
+  else if (isa<BlockDecl>(dc))
+    cgm.errorNYI(d.getSourceRange(), "block decl context for static var");
+  else if (isa<ObjCMethodDecl>(dc))
+    cgm.errorNYI(d.getSourceRange(), "ObjC decl context for static var");
+  else
+    cgm.errorNYI(d.getSourceRange(), "Unknown context for static var decl");
+
+  contextName += "." + d.getNameAsString();
+  return contextName;
+}
+
+// TODO(cir): LLVM uses a Constant base class. Maybe CIR could leverage an
+// interface for all constants?
+cir::GlobalOp
+CIRGenModule::getOrCreateStaticVarDecl(const VarDecl &d,
+                                       cir::GlobalLinkageKind linkage) {
+  // In general, we don't always emit static var decls once before we reference
+  // them. It is possible to reference them before emitting the function that
+  // contains them, and it is possible to emit the containing function multiple
+  // times.
+  if (cir::GlobalOp existingGV = getStaticLocalDeclAddress(&d))
+    return existingGV;
+
+  QualType ty = d.getType();
+  assert(ty->isConstantSizeType() && "VLAs can't be static");
+
+  // Use the label if the variable is renamed with the asm-label extension.
+  if (d.hasAttr<AsmLabelAttr>())
+    errorNYI(d.getSourceRange(), "getOrCreateStaticVarDecl: asm label");
+
+  std::string name = getStaticDeclName(*this, d);
+
+  mlir::Type lty = getTypes().convertTypeForMem(ty);
+  assert(!cir::MissingFeatures::addressSpace());
+
+  if (d.hasAttr<LoaderUninitializedAttr>() || d.hasAttr<CUDASharedAttr>())
+    errorNYI(d.getSourceRange(),
+             "getOrCreateStaticVarDecl: LoaderUninitializedAttr");
+  assert(!cir::MissingFeatures::addressSpace());
+
+  mlir::Attribute init = builder.getZeroInitAttr(convertType(ty));
+
+  cir::GlobalOp gv = builder.createVersionedGlobal(
+      getModule(), getLoc(d.getLocation()), name, lty, linkage);
+  // TODO(cir): infer visibility from linkage in global op builder.
+  gv.setVisibility(getMLIRVisibilityFromCIRLinkage(linkage));
+  gv.setInitialValueAttr(init);
+  gv.setAlignment(getASTContext().getDeclAlign(&d).getAsAlign().value());
+
+  if (supportsCOMDAT() && gv.isWeakForLinker())
+    gv.setComdat(true);
+
+  assert(!cir::MissingFeatures::opGlobalThreadLocal());
+
+  setGVProperties(gv, &d);
+
+  // OG checks if the expected address space, denoted by the type, is the
+  // same as the actual address space indicated by attributes. If they aren't
+  // the same, an addrspacecast is emitted when this variable is accessed.
+  // In CIR however, cir.get_global already carries that information in
+  // !cir.ptr type - if this global is in OpenCL local address space, then its
+  // type would be !cir.ptr<..., addrspace(offload_local)>. Therefore we don't
+  // need an explicit address space cast in CIR: they will get emitted when
+  // lowering to LLVM IR.
+
+  // Ensure that the static local gets initialized by making sure the parent
+  // function gets emitted eventually.
+  const Decl *dc = cast<Decl>(d.getDeclContext());
+
+  // We can't name blocks or captured statements directly, so try to emit their
+  // parents.
+  if (isa<BlockDecl>(dc) || isa<CapturedDecl>(dc)) {
+    dc = dc->getNonClosureContext();
+    // FIXME: Ensure that global blocks get emitted.
+    if (!dc)
+      errorNYI(d.getSourceRange(), "non-closure context");
+  }
+
+  GlobalDecl gd;
+  if (isa<CXXConstructorDecl>(dc))
+    errorNYI(d.getSourceRange(), "C++ constructors static var context");
+  else if (isa<CXXDestructorDecl>(dc))
+    errorNYI(d.getSourceRange(), "C++ destructors static var context");
+  else if (const auto *fd = dyn_cast<FunctionDecl>(dc))
+    gd = GlobalDecl(fd);
+  else {
+    // Don't do anything for Obj-C method decls or global closures. We should
+    // never defer them.
+    assert(isa<ObjCMethodDecl>(dc) && "unexpected parent code decl");
+  }
+  if (gd.getDecl() && cir::MissingFeatures::openMP()) {
+    // Disable emission of the parent function for the OpenMP device codegen.
+    errorNYI(d.getSourceRange(), "OpenMP");
+  }
+
+  return gv;
+}
+
+/// Add the initializer for 'd' to the global variable that has already been
+/// created for it. If the initializer has a different type than gv does, this
+/// may free gv and return a different one. Otherwise it just returns gv.
+cir::GlobalOp CIRGenFunction::addInitializerToStaticVarDecl(
+    const VarDecl &d, cir::GlobalOp gv, cir::GetGlobalOp gvAddr) {
+  ConstantEmitter emitter(*this);
+  mlir::TypedAttr init =
+      mlir::cast<mlir::TypedAttr>(emitter.tryEmitForInitializer(d));
+
+  // If constant emission failed, then this should be a C++ static
+  // initializer.
+  if (!init) {
+    cgm.errorNYI(d.getSourceRange(), "static var without initializer");
+    return gv;
+  }
+
+  // TODO(cir): There should be debug code here to assert that the decl size
+  // matches the CIR data layout type alloc size, but the code for calculating
+  // the type alloc size is not implemented yet.
+  assert(!cir::MissingFeatures::dataLayoutTypeAllocSize());
+
+  // The initializer may differ in type from the global. Rewrite
+  // the global to match the initializer.  (We have to do this
+  // because some types, like unions, can't be completely represented
+  // in the LLVM type system.)
+  if (gv.getSymType() != init.getType()) {
+    gv.setSymType(init.getType());
+
+    // Normally this should be done with a call to cgm.replaceGlobal(oldGV, gv),
+    // but since at this point the current block hasn't been really attached,
+    // there's no visibility into the GetGlobalOp corresponding to this Global.
+    // Given those constraints, thread in the GetGlobalOp and update it
+    // directly.
+    assert(!cir::MissingFeatures::addressSpace());
+    gvAddr.getAddr().setType(builder.getPointerTo(init.getType()));
+  }
+
+  bool needsDtor =
+      d.needsDestruction(getContext()) == QualType::DK_cxx_destructor;
+
+  assert(!cir::MissingFeatures::opGlobalConstant());
+  gv.setInitialValueAttr(init);
+
+  emitter.finalize(gv);
+
+  if (needsDtor) {
+    // We have a constant initializer, but a nontrivial destructor. We still
+    // need to perform a guarded "initialization" in order to register the
+    // destructor.
+    cgm.errorNYI(d.getSourceRange(), "C++ guarded init");
+  }
+
+  return gv;
+}
+
+void CIRGenFunction::emitStaticVarDecl(const VarDecl &d,
+                                       cir::GlobalLinkageKind linkage) {
+  // Check to see if we already have a global variable for this
+  // declaration.  This can happen when double-emitting function
+  // bodies, e.g. with complete and base constructors.
+  cir::GlobalOp globalOp = cgm.getOrCreateStaticVarDecl(d, linkage);
+  // TODO(cir): we should have a way to represent global ops as values without
+  // having to emit a get global op. Sometimes these emissions are not used.
+  mlir::Value addr = builder.createGetGlobal(globalOp);
+  auto getAddrOp = mlir::cast<cir::GetGlobalOp>(addr.getDefiningOp());
+
+  CharUnits alignment = getContext().getDeclAlign(&d);
+
+  // Store into LocalDeclMap before generating initializer to handle
+  // circular references.
+  mlir::Type elemTy = convertTypeForMem(d.getType());
+  setAddrOfLocalVar(&d, Address(addr, elemTy, alignment));
+
+  // We can't have a VLA here, but we can have a pointer to a VLA,
+  // even though that doesn't really make any sense.
+  // Make sure to evaluate VLA bounds now so that we have them for later.
+  if (d.getType()->isVariablyModifiedType()) {
+    cgm.errorNYI(d.getSourceRange(),
+                 "emitStaticVarDecl: variably modified type");
+  }
+
+  // Save the type in case adding the initializer forces a type change.
+  mlir::Type expectedType = addr.getType();
+
+  cir::GlobalOp var = globalOp;
+
+  assert(!cir::MissingFeatures::cudaSupport());
+
+  // If this value has an initializer, emit it.
+  if (d.getInit())
+    var = addInitializerToStaticVarDecl(d, var, getAddrOp);
+
+  var.setAlignment(alignment.getAsAlign().value());
+
+  // There are a lot of attributes that need to be handled here. Until
+  // we start to support them, we just report an error if there are any.
+  if (d.hasAttrs())
+    cgm.errorNYI(d.getSourceRange(), "static var with attrs");
+
+  if (cgm.getCodeGenOpts().KeepPersistentStorageVariables)
+    cgm.errorNYI(d.getSourceRange(), "static var keep persistent storage");
+
+  // From traditional codegen:
+  // We may have to cast the constant because of the initializer
+  // mismatch above.
+  //
+  // FIXME: It is really dangerous to store this in the map; if anyone
+  // RAUW's the GV uses of this constant will be invalid.
+  mlir::Value castedAddr =
+      builder.createBitcast(getAddrOp.getAddr(), expectedType);
+  localDeclMap.find(&d)->second = Address(castedAddr, elemTy, alignment);
+  cgm.setStaticLocalDeclAddress(&d, var);
+
+  assert(!cir::MissingFeatures::sanitizers());
+  assert(!cir::MissingFeatures::generateDebugInfo());
+}
+
 void CIRGenFunction::emitScalarInit(const Expr *init, mlir::Location loc,
                                     LValue lvalue, bool capturedByInit) {
   assert(!cir::MissingFeatures::objCLifetime());
@@ -256,11 +499,22 @@ void CIRGenFunction::emitExprAsInit(const Expr *init, const ValueDecl *d,
     emitScalarInit(init, getLoc(d->getSourceRange()), lvalue);
     return;
   case cir::TEK_Complex: {
-    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: complex type");
+    mlir::Value complex = emitComplexExpr(init);
+    if (capturedByInit)
+      cgm.errorNYI(init->getSourceRange(),
+                   "emitExprAsInit: complex type captured by init");
+    mlir::Location loc = getLoc(init->getExprLoc());
+    emitStoreOfComplex(loc, complex, lvalue,
+                       /*isInit*/ true);
     return;
   }
   case cir::TEK_Aggregate:
-    emitAggExpr(init, AggValueSlot::forLValue(lvalue));
+    // The overlap flag here should be calculated.
+    assert(!cir::MissingFeatures::aggValueSlotMayOverlap());
+    emitAggExpr(init,
+                AggValueSlot::forLValue(lvalue, AggValueSlot::IsDestructed,
+                                        AggValueSlot::IsNotAliased,
+                                        AggValueSlot::MayOverlap));
     return;
   }
   llvm_unreachable("bad evaluation kind");
@@ -345,8 +599,8 @@ void CIRGenFunction::emitDecl(const Decl &d) {
     // None of these decls require codegen support.
     return;
 
-  case Decl::Enum:   // enum X;
-  case Decl::Record: // struct/union/class X;
+  case Decl::Enum:      // enum X;
+  case Decl::Record:    // struct/union/class X;
   case Decl::CXXRecord: // struct/union/class X; [C++]
   case Decl::NamespaceAlias:
   case Decl::Using:          // using X; [C++]
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index 5424c6a8d6f3..2e43f10be132 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -11,10 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "Address.h"
+#include "CIRGenConstantEmitter.h"
 #include "CIRGenFunction.h"
 #include "CIRGenModule.h"
 #include "CIRGenValue.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Value.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Decl.h"
@@ -22,6 +24,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 #include "clang/CIR/MissingFeatures.h"
+#include <optional>
 
 using namespace clang;
 using namespace clang::CIRGen;
@@ -95,9 +98,14 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
 
     case CK_UncheckedDerivedToBase:
     case CK_DerivedToBase: {
-      cgm.errorNYI(expr->getSourceRange(),
-                   "emitPointerWithAlignment: derived-to-base cast");
-      return Address::invalid();
+      assert(!cir::MissingFeatures::opTBAA());
+      assert(!cir::MissingFeatures::addressIsKnownNonNull());
+      Address addr = emitPointerWithAlignment(ce->getSubExpr(), baseInfo);
+      const CXXRecordDecl *derived =
+          ce->getSubExpr()->getType()->getPointeeCXXRecordDecl();
+      return getAddressOfBaseClass(addr, derived, ce->path(),
+                                   shouldNullCheckClassCastValue(ce),
+                                   ce->getExprLoc());
     }
 
     case CK_AnyPointerToBlockPointerCast:
@@ -229,7 +237,7 @@ void CIRGenFunction::emitStoreThroughLValue(RValue src, LValue dst,
 
 static LValue emitGlobalVarDeclLValue(CIRGenFunction &cgf, const Expr *e,
                                       const VarDecl *vd) {
-  QualType T = e->getType();
+  QualType t = e->getType();
 
   // If it's thread_local, emit a call to its wrapper function instead.
   assert(!cir::MissingFeatures::opGlobalThreadLocal());
@@ -259,7 +267,7 @@ static LValue emitGlobalVarDeclLValue(CIRGenFunction &cgf, const Expr *e,
     cgf.cgm.errorNYI(e->getSourceRange(),
                      "emitGlobalVarDeclLValue: reference type");
   else
-    lv = cgf.makeAddrLValue(addr, T, AlignmentSource::Decl);
+    lv = cgf.makeAddrLValue(addr, t, AlignmentSource::Decl);
   assert(!cir::MissingFeatures::setObjCGCLValueClass());
   return lv;
 }
@@ -821,8 +829,6 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
   case CK_NonAtomicToAtomic:
   case CK_AtomicToNonAtomic:
   case CK_Dynamic:
-  case CK_UncheckedDerivedToBase:
-  case CK_DerivedToBase:
   case CK_ToUnion:
   case CK_BaseToDerived:
   case CK_LValueBitCast:
@@ -861,6 +867,27 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
     return lv;
   }
 
+  case CK_UncheckedDerivedToBase:
+  case CK_DerivedToBase: {
+    const auto *derivedClassTy =
+        e->getSubExpr()->getType()->castAs<clang::RecordType>();
+    auto *derivedClassDecl = cast<CXXRecordDecl>(derivedClassTy->getDecl());
+
+    LValue lv = emitLValue(e->getSubExpr());
+    Address thisAddr = lv.getAddress();
+
+    // Perform the derived-to-base conversion
+    Address baseAddr =
+        getAddressOfBaseClass(thisAddr, derivedClassDecl, e->path(),
+                              /*NullCheckValue=*/false, e->getExprLoc());
+
+    // TODO: Support accesses to members of base classes in TBAA. For now, we
+    // conservatively pretend that the complete object is of the base class
+    // type.
+    assert(!cir::MissingFeatures::opTBAA());
+    return makeAddrLValue(baseAddr, e->getType(), lv.getBaseInfo());
+  }
+
   case CK_ZeroToOCLOpaqueType:
     llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
   }
@@ -983,16 +1010,20 @@ LValue CIRGenFunction::emitBinaryOperatorLValue(const BinaryOperator *e) {
 
 /// Emit code to compute the specified expression which
 /// can have any type.  The result is returned as an RValue struct.
-RValue CIRGenFunction::emitAnyExpr(const Expr *e) {
+RValue CIRGenFunction::emitAnyExpr(const Expr *e, AggValueSlot aggSlot) {
   switch (CIRGenFunction::getEvaluationKind(e->getType())) {
   case cir::TEK_Scalar:
     return RValue::get(emitScalarExpr(e));
   case cir::TEK_Complex:
     cgm.errorNYI(e->getSourceRange(), "emitAnyExpr: complex type");
     return RValue::get(nullptr);
-  case cir::TEK_Aggregate:
-    cgm.errorNYI(e->getSourceRange(), "emitAnyExpr: aggregate type");
-    return RValue::get(nullptr);
+  case cir::TEK_Aggregate: {
+    if (aggSlot.isIgnored())
+      aggSlot = createAggTemp(e->getType(), getLoc(e->getSourceRange()),
+                              getCounterAggTmpAsString());
+    emitAggExpr(e, aggSlot);
+    return aggSlot.asRValue();
+  }
   }
   llvm_unreachable("bad evaluation kind");
 }
@@ -1002,8 +1033,49 @@ static cir::FuncOp emitFunctionDeclPointer(CIRGenModule &cgm, GlobalDecl gd) {
   return cgm.getAddrOfFunction(gd);
 }
 
-static CIRGenCallee emitDirectCallee(CIRGenModule &cgm, GlobalDecl gd) {
-  assert(!cir::MissingFeatures::opCallBuiltinFunc());
+// Detect the unusual situation where an inline version is shadowed by a
+// non-inline version. In that case we should pick the external one
+// everywhere. That's GCC behavior too.
+static bool onlyHasInlineBuiltinDeclaration(const FunctionDecl *fd) {
+  for (const FunctionDecl *pd = fd; pd; pd = pd->getPreviousDecl())
+    if (!pd->isInlineBuiltinDeclaration())
+      return false;
+  return true;
+}
+
+CIRGenCallee CIRGenFunction::emitDirectCallee(const GlobalDecl &gd) {
+  const auto *fd = cast<FunctionDecl>(gd.getDecl());
+
+  if (unsigned builtinID = fd->getBuiltinID()) {
+    if (fd->getAttr<AsmLabelAttr>()) {
+      cgm.errorNYI("AsmLabelAttr");
+    }
+
+    StringRef ident = fd->getName();
+    std::string fdInlineName = (ident + ".inline").str();
+
+    bool isPredefinedLibFunction =
+        cgm.getASTContext().BuiltinInfo.isPredefinedLibFunction(builtinID);
+    // Assume nobuiltins everywhere until we actually read the attributes.
+    bool hasAttributeNoBuiltin = true;
+    assert(!cir::MissingFeatures::attributeNoBuiltin());
+
+    // When directing calling an inline builtin, call it through it's mangled
+    // name to make it clear it's not the actual builtin.
+    auto fn = cast<cir::FuncOp>(curFn);
+    if (fn.getName() != fdInlineName && onlyHasInlineBuiltinDeclaration(fd)) {
+      cgm.errorNYI("Inline only builtin function calls");
+    }
+
+    // Replaceable builtins provide their own implementation of a builtin. If we
+    // are in an inline builtin implementation, avoid trivial infinite
+    // recursion. Honor __attribute__((no_builtin("foo"))) or
+    // __attribute__((no_builtin)) on the current function unless foo is
+    // not a predefined library function which means we must generate the
+    // builtin no matter what.
+    else if (!isPredefinedLibFunction || !hasAttributeNoBuiltin)
+      return CIRGenCallee::forBuiltin(builtinID, fd);
+  }
 
   cir::FuncOp callee = emitFunctionDeclPointer(cgm, gd);
 
@@ -1079,7 +1151,7 @@ CIRGenCallee CIRGenFunction::emitCallee(const clang::Expr *e) {
   } else if (const auto *declRef = dyn_cast<DeclRefExpr>(e)) {
     // Resolve direct calls.
     const auto *funcDecl = cast<FunctionDecl>(declRef->getDecl());
-    return emitDirectCallee(cgm, funcDecl);
+    return emitDirectCallee(funcDecl);
   } else if (isa<MemberExpr>(e)) {
     cgm.errorNYI(e->getSourceRange(),
                  "emitCallee: call to member function is NYI");
@@ -1135,10 +1207,9 @@ RValue CIRGenFunction::emitCallExpr(const clang::CallExpr *e,
 
   CIRGenCallee callee = emitCallee(e->getCallee());
 
-  if (e->getBuiltinCallee()) {
-    cgm.errorNYI(e->getSourceRange(), "call to builtin functions");
-  }
-  assert(!cir::MissingFeatures::opCallBuiltinFunc());
+  if (callee.isBuiltin())
+    return emitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(), e,
+                           returnValue);
 
   if (isa<CXXPseudoDestructorExpr>(e->getCallee())) {
     cgm.errorNYI(e->getSourceRange(), "call to pseudo destructor");
@@ -1195,6 +1266,23 @@ Address CIRGenFunction::emitArrayToPointerDecay(const Expr *e) {
   return Address(ptr, addr.getAlignment());
 }
 
+/// Given the address of a temporary variable, produce an r-value of its type.
+RValue CIRGenFunction::convertTempToRValue(Address addr, clang::QualType type,
+                                           clang::SourceLocation loc) {
+  LValue lvalue = makeAddrLValue(addr, type, AlignmentSource::Decl);
+  switch (getEvaluationKind(type)) {
+  case cir::TEK_Complex:
+    cgm.errorNYI(loc, "convertTempToRValue: complex type");
+    return RValue::get(nullptr);
+  case cir::TEK_Aggregate:
+    cgm.errorNYI(loc, "convertTempToRValue: aggregate type");
+    return RValue::get(nullptr);
+  case cir::TEK_Scalar:
+    return RValue::get(emitLoadOfScalar(lvalue, loc));
+  }
+  llvm_unreachable("bad evaluation kind");
+}
+
 /// Emit an `if` on a boolean condition, filling `then` and `else` into
 /// appropriated regions.
 mlir::LogicalResult CIRGenFunction::emitIfOnBoolExpr(const Expr *cond,
@@ -1259,10 +1347,28 @@ mlir::Value CIRGenFunction::emitOpOnBoolExpr(mlir::Location loc,
   //  cir.ternary(!x, t, f) -> cir.ternary(x, f, t)
   assert(!cir::MissingFeatures::shouldReverseUnaryCondOnBoolExpr());
 
-  if (isa<ConditionalOperator>(cond)) {
-    cgm.errorNYI(cond->getExprLoc(), "Ternary NYI");
-    assert(!cir::MissingFeatures::ternaryOp());
-    return createDummyValue(loc, cond->getType());
+  if (const ConditionalOperator *condOp = dyn_cast<ConditionalOperator>(cond)) {
+    Expr *trueExpr = condOp->getTrueExpr();
+    Expr *falseExpr = condOp->getFalseExpr();
+    mlir::Value condV = emitOpOnBoolExpr(loc, condOp->getCond());
+
+    mlir::Value ternaryOpRes =
+        builder
+            .create<cir::TernaryOp>(
+                loc, condV, /*thenBuilder=*/
+                [this, trueExpr](mlir::OpBuilder &b, mlir::Location loc) {
+                  mlir::Value lhs = emitScalarExpr(trueExpr);
+                  b.create<cir::YieldOp>(loc, lhs);
+                },
+                /*elseBuilder=*/
+                [this, falseExpr](mlir::OpBuilder &b, mlir::Location loc) {
+                  mlir::Value rhs = emitScalarExpr(falseExpr);
+                  b.create<cir::YieldOp>(loc, rhs);
+                })
+            .getResult();
+
+    return emitScalarConversion(ternaryOpRes, condOp->getType(),
+                                getContext().BoolTy, condOp->getExprLoc());
   }
 
   if (isa<CXXThrowExpr>(cond)) {
@@ -1348,6 +1454,61 @@ RValue CIRGenFunction::emitCXXMemberCallExpr(const CXXMemberCallExpr *ce,
       ce, md, returnValue, hasQualifier, qualifier, isArrow, base);
 }
 
+void CIRGenFunction::emitCXXConstructExpr(const CXXConstructExpr *e,
+                                          AggValueSlot dest) {
+  assert(!dest.isIgnored() && "Must have a destination!");
+  const CXXConstructorDecl *cd = e->getConstructor();
+
+  // If we require zero initialization before (or instead of) calling the
+  // constructor, as can be the case with a non-user-provided default
+  // constructor, emit the zero initialization now, unless destination is
+  // already zeroed.
+  if (e->requiresZeroInitialization() && !dest.isZeroed()) {
+    cgm.errorNYI(e->getSourceRange(),
+                 "emitCXXConstructExpr: requires initialization");
+    return;
+  }
+
+  // If this is a call to a trivial default constructor:
+  // In LLVM: do nothing.
+  // In CIR: emit as a regular call, other later passes should lower the
+  // ctor call into trivial initialization.
+
+  // Elide the constructor if we're constructing from a temporary
+  if (getLangOpts().ElideConstructors && e->isElidable()) {
+    cgm.errorNYI(e->getSourceRange(),
+                 "emitCXXConstructExpr: elidable constructor");
+    return;
+  }
+
+  if (getContext().getAsArrayType(e->getType())) {
+    cgm.errorNYI(e->getSourceRange(), "emitCXXConstructExpr: array type");
+    return;
+  }
+
+  clang::CXXCtorType type = Ctor_Complete;
+  bool forVirtualBase = false;
+  bool delegating = false;
+
+  switch (e->getConstructionKind()) {
+  case CXXConstructionKind::Complete:
+    type = Ctor_Complete;
+    break;
+  case CXXConstructionKind::Delegating:
+    // We should be emitting a constructor; GlobalDecl will assert this
+    type = curGD.getCtorType();
+    delegating = true;
+    break;
+  case CXXConstructionKind::VirtualBase:
+  case CXXConstructionKind::NonVirtualBase:
+    cgm.errorNYI(e->getSourceRange(),
+                 "emitCXXConstructExpr: other construction kind");
+    return;
+  }
+
+  emitCXXConstructorCall(cd, type, forVirtualBase, delegating, dest, e);
+}
+
 RValue CIRGenFunction::emitReferenceBindingToExpr(const Expr *e) {
   // Emit the expression as an lvalue.
   LValue lv = emitLValue(e);
@@ -1394,13 +1555,149 @@ mlir::Value CIRGenFunction::createDummyValue(mlir::Location loc,
   return builder.createDummyValue(loc, t, alignment);
 }
 
-/// This creates an alloca and inserts it into the entry block if
-/// \p insertIntoFnEntryBlock is true, otherwise it inserts it at the current
-/// insertion point of the builder.
+//===----------------------------------------------------------------------===//
+// CIR builder helpers
+//===----------------------------------------------------------------------===//
+
+Address CIRGenFunction::createMemTemp(QualType ty, mlir::Location loc,
+                                      const Twine &name, Address *alloca,
+                                      mlir::OpBuilder::InsertPoint ip) {
+  // FIXME: Should we prefer the preferred type alignment here?
+  return createMemTemp(ty, getContext().getTypeAlignInChars(ty), loc, name,
+                       alloca, ip);
+}
+
+Address CIRGenFunction::createMemTemp(QualType ty, CharUnits align,
+                                      mlir::Location loc, const Twine &name,
+                                      Address *alloca,
+                                      mlir::OpBuilder::InsertPoint ip) {
+  Address result = createTempAlloca(convertTypeForMem(ty), align, loc, name,
+                                    /*ArraySize=*/nullptr, alloca, ip);
+  if (ty->isConstantMatrixType()) {
+    assert(!cir::MissingFeatures::matrixType());
+    cgm.errorNYI(loc, "temporary matrix value");
+  }
+  return result;
+}
+
+/// This creates a alloca and inserts it into the entry block of the
+/// current region.
+Address CIRGenFunction::createTempAllocaWithoutCast(
+    mlir::Type ty, CharUnits align, mlir::Location loc, const Twine &name,
+    mlir::Value arraySize, mlir::OpBuilder::InsertPoint ip) {
+  cir::AllocaOp alloca = ip.isSet()
+                             ? createTempAlloca(ty, loc, name, ip, arraySize)
+                             : createTempAlloca(ty, loc, name, arraySize);
+  alloca.setAlignmentAttr(cgm.getSize(align));
+  return Address(alloca, ty, align);
+}
+
+/// This creates a alloca and inserts it into the entry block. The alloca is
+/// casted to default address space if necessary.
 Address CIRGenFunction::createTempAlloca(mlir::Type ty, CharUnits align,
                                          mlir::Location loc, const Twine &name,
-                                         bool insertIntoFnEntryBlock) {
-  mlir::Value alloca =
-      emitAlloca(name.str(), ty, loc, align, insertIntoFnEntryBlock);
-  return Address(alloca, ty, align);
+                                         mlir::Value arraySize,
+                                         Address *allocaAddr,
+                                         mlir::OpBuilder::InsertPoint ip) {
+  Address alloca =
+      createTempAllocaWithoutCast(ty, align, loc, name, arraySize, ip);
+  if (allocaAddr)
+    *allocaAddr = alloca;
+  mlir::Value v = alloca.getPointer();
+  // Alloca always returns a pointer in alloca address space, which may
+  // be different from the type defined by the language. For example,
+  // in C++ the auto variables are in the default address space. Therefore
+  // cast alloca to the default address space when necessary.
+  assert(!cir::MissingFeatures::addressSpace());
+  return Address(v, ty, align);
+}
+
+/// This creates an alloca and inserts it into the entry block if \p ArraySize
+/// is nullptr, otherwise inserts it at the current insertion point of the
+/// builder.
+cir::AllocaOp CIRGenFunction::createTempAlloca(mlir::Type ty,
+                                               mlir::Location loc,
+                                               const Twine &name,
+                                               mlir::Value arraySize,
+                                               bool insertIntoFnEntryBlock) {
+  return cast<cir::AllocaOp>(emitAlloca(name.str(), ty, loc, CharUnits(),
+                                        insertIntoFnEntryBlock, arraySize)
+                                 .getDefiningOp());
+}
+
+/// This creates an alloca and inserts it into the provided insertion point
+cir::AllocaOp CIRGenFunction::createTempAlloca(mlir::Type ty,
+                                               mlir::Location loc,
+                                               const Twine &name,
+                                               mlir::OpBuilder::InsertPoint ip,
+                                               mlir::Value arraySize) {
+  assert(ip.isSet() && "Insertion point is not set");
+  return cast<cir::AllocaOp>(
+      emitAlloca(name.str(), ty, loc, CharUnits(), ip, arraySize)
+          .getDefiningOp());
+}
+
+/// Try to emit a reference to the given value without producing it as
+/// an l-value.  For many cases, this is just an optimization, but it avoids
+/// us needing to emit global copies of variables if they're named without
+/// triggering a formal use in a context where we can't emit a direct
+/// reference to them, for instance if a block or lambda or a member of a
+/// local class uses a const int variable or constexpr variable from an
+/// enclosing function.
+///
+/// For named members of enums, this is the only way they are emitted.
+CIRGenFunction::ConstantEmission
+CIRGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
+  ValueDecl *value = refExpr->getDecl();
+
+  // There is a lot more to do here, but for now only EnumConstantDecl is
+  // supported.
+  assert(!cir::MissingFeatures::tryEmitAsConstant());
+
+  // The value needs to be an enum constant or a constant variable.
+  if (!isa<EnumConstantDecl>(value))
+    return ConstantEmission();
+
+  Expr::EvalResult result;
+  if (!refExpr->EvaluateAsRValue(result, getContext()))
+    return ConstantEmission();
+
+  QualType resultType = refExpr->getType();
+
+  // As long as we're only handling EnumConstantDecl, there should be no
+  // side-effects.
+  assert(!result.HasSideEffects);
+
+  // Emit as a constant.
+  // FIXME(cir): have emitAbstract build a TypedAttr instead (this requires
+  // somewhat heavy refactoring...)
+  mlir::Attribute c = ConstantEmitter(*this).emitAbstract(
+      refExpr->getLocation(), result.Val, resultType);
+  mlir::TypedAttr cstToEmit = mlir::dyn_cast_if_present<mlir::TypedAttr>(c);
+  assert(cstToEmit && "expected a typed attribute");
+
+  assert(!cir::MissingFeatures::generateDebugInfo());
+
+  return ConstantEmission::forValue(cstToEmit);
+}
+
+mlir::Value CIRGenFunction::emitScalarConstant(
+    const CIRGenFunction::ConstantEmission &constant, Expr *e) {
+  assert(constant && "not a constant");
+  if (constant.isReference()) {
+    cgm.errorNYI(e->getSourceRange(), "emitScalarConstant: reference");
+    return {};
+  }
+  return builder.getConstant(getLoc(e->getSourceRange()), constant.getValue());
+}
+
+/// An LValue is a candidate for having its loads and stores be made atomic if
+/// we are operating under /volatile:ms *and* the LValue itself is volatile and
+/// performing such an operation can be performed without a libcall.
+bool CIRGenFunction::isLValueSuitableForInlineAtomic(LValue lv) {
+  if (!cgm.getLangOpts().MSVolatile)
+    return false;
+
+  cgm.errorNYI("LValueSuitableForInlineAtomic LangOpts MSVolatile");
+  return false;
 }
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
index b375bcf2f483..ffe1b701b244 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
@@ -28,6 +28,15 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   CIRGenFunction &cgf;
   AggValueSlot dest;
 
+  // Calls `fn` with a valid return value slot, potentially creating a temporary
+  // to do so. If a temporary is created, an appropriate copy into `Dest` will
+  // be emitted, as will lifetime markers.
+  //
+  // The given function should take a ReturnValueSlot, and return an RValue that
+  // points to said slot.
+  void withReturnValueSlot(const Expr *e,
+                           llvm::function_ref<RValue(ReturnValueSlot)> fn);
+
   AggValueSlot ensureSlot(mlir::Location loc, QualType t) {
     if (!dest.isIgnored())
       return dest;
@@ -40,17 +49,30 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   AggExprEmitter(CIRGenFunction &cgf, AggValueSlot dest)
       : cgf(cgf), dest(dest) {}
 
+  /// Given an expression with aggregate type that represents a value lvalue,
+  /// this method emits the address of the lvalue, then loads the result into
+  /// DestPtr.
+  void emitAggLoadOfLValue(const Expr *e);
+
   void emitArrayInit(Address destPtr, cir::ArrayType arrayTy, QualType arrayQTy,
                      Expr *exprToVisit, ArrayRef<Expr *> args,
                      Expr *arrayFiller);
 
+  /// Perform the final copy to DestPtr, if desired.
+  void emitFinalDestCopy(QualType type, const LValue &src);
+
   void emitInitializationToLValue(Expr *e, LValue lv);
 
   void emitNullInitializationToLValue(mlir::Location loc, LValue lv);
 
   void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(e); }
 
+  void VisitCallExpr(const CallExpr *e);
+
+  void VisitDeclRefExpr(DeclRefExpr *e) { emitAggLoadOfLValue(e); }
+
   void VisitInitListExpr(InitListExpr *e);
+  void VisitCXXConstructExpr(const CXXConstructExpr *e);
 
   void visitCXXParenListOrInitListExpr(Expr *e, ArrayRef<Expr *> args,
                                        FieldDecl *initializedFieldInUnion,
@@ -79,6 +101,17 @@ static bool isTrivialFiller(Expr *e) {
   return false;
 }
 
+/// Given an expression with aggregate type that represents a value lvalue, this
+/// method emits the address of the lvalue, then loads the result into DestPtr.
+void AggExprEmitter::emitAggLoadOfLValue(const Expr *e) {
+  LValue lv = cgf.emitLValue(e);
+
+  // If the type of the l-value is atomic, then do an atomic load.
+  assert(!cir::MissingFeatures::opLoadStoreAtomic());
+
+  emitFinalDestCopy(e->getType(), lv);
+}
+
 void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
                                    QualType arrayQTy, Expr *e,
                                    ArrayRef<Expr *> args, Expr *arrayFiller) {
@@ -155,8 +188,7 @@ void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
     // Allocate the temporary variable
     // to store the pointer to first unitialized element
     const Address tmpAddr = cgf.createTempAlloca(
-        cirElementPtrType, cgf.getPointerAlign(), loc, "arrayinit.temp",
-        /*insertIntoFnEntryBlock=*/false);
+        cirElementPtrType, cgf.getPointerAlign(), loc, "arrayinit.temp");
     LValue tmpLV = cgf.makeAddrLValue(tmpAddr, elementPtrType);
     cgf.emitStoreThroughLValue(RValue::get(element), tmpLV);
 
@@ -182,6 +214,18 @@ void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
   }
 }
 
+/// Perform the final copy to destPtr, if desired.
+void AggExprEmitter::emitFinalDestCopy(QualType type, const LValue &src) {
+  // If dest is ignored, then we're evaluating an aggregate expression
+  // in a context that doesn't care about the result.  Note that loads
+  // from volatile l-values force the existence of a non-ignored
+  // destination.
+  if (dest.isIgnored())
+    return;
+
+  cgf.cgm.errorNYI("emitFinalDestCopy: non-ignored dest is NYI");
+}
+
 void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
   const QualType type = lv.getType();
 
@@ -203,7 +247,11 @@ void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
     cgf.cgm.errorNYI("emitInitializationToLValue TEK_Complex");
     break;
   case cir::TEK_Aggregate:
-    cgf.emitAggExpr(e, AggValueSlot::forLValue(lv));
+    cgf.emitAggExpr(e, AggValueSlot::forLValue(lv, AggValueSlot::IsDestructed,
+                                               AggValueSlot::IsNotAliased,
+                                               AggValueSlot::MayOverlap,
+                                               dest.isZeroed()));
+
     return;
   case cir::TEK_Scalar:
     if (lv.isSimple())
@@ -214,6 +262,11 @@ void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
   }
 }
 
+void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *e) {
+  AggValueSlot slot = ensureSlot(cgf.getLoc(e->getSourceRange()), e->getType());
+  cgf.emitCXXConstructExpr(e, slot);
+}
+
 void AggExprEmitter::emitNullInitializationToLValue(mlir::Location loc,
                                                     LValue lv) {
   const QualType type = lv.getType();
@@ -241,6 +294,44 @@ void AggExprEmitter::emitNullInitializationToLValue(mlir::Location loc,
   cgf.emitNullInitialization(loc, lv.getAddress(), lv.getType());
 }
 
+void AggExprEmitter::VisitCallExpr(const CallExpr *e) {
+  if (e->getCallReturnType(cgf.getContext())->isReferenceType()) {
+    cgf.cgm.errorNYI(e->getSourceRange(), "reference return type");
+    return;
+  }
+
+  withReturnValueSlot(
+      e, [&](ReturnValueSlot slot) { return cgf.emitCallExpr(e, slot); });
+}
+
+void AggExprEmitter::withReturnValueSlot(
+    const Expr *e, llvm::function_ref<RValue(ReturnValueSlot)> fn) {
+  QualType retTy = e->getType();
+
+  assert(!cir::MissingFeatures::aggValueSlotDestructedFlag());
+  bool requiresDestruction =
+      retTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
+  if (requiresDestruction)
+    cgf.cgm.errorNYI(
+        e->getSourceRange(),
+        "withReturnValueSlot: return value requiring destruction is NYI");
+
+  // If it makes no observable difference, save a memcpy + temporary.
+  //
+  // We need to always provide our own temporary if destruction is required.
+  // Otherwise, fn will emit its own, notice that it's "unused", and end its
+  // lifetime before we have the chance to emit a proper destructor call.
+  assert(!cir::MissingFeatures::aggValueSlotAlias());
+  assert(!cir::MissingFeatures::aggValueSlotGC());
+
+  Address retAddr = dest.getAddress();
+  assert(!cir::MissingFeatures::emitLifetimeMarkers());
+
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+  assert(!cir::MissingFeatures::aggValueSlotDestructedFlag());
+  fn(ReturnValueSlot(retAddr));
+}
+
 void AggExprEmitter::VisitInitListExpr(InitListExpr *e) {
   if (e->hadArrayRangeDesignator())
     llvm_unreachable("GNU array range designator extension");
@@ -274,3 +365,13 @@ void AggExprEmitter::visitCXXParenListOrInitListExpr(
 void CIRGenFunction::emitAggExpr(const Expr *e, AggValueSlot slot) {
   AggExprEmitter(*this, slot).Visit(const_cast<Expr *>(e));
 }
+
+LValue CIRGenFunction::emitAggExprToLValue(const Expr *e) {
+  assert(hasAggregateEvaluationKind(e->getType()) && "Invalid argument!");
+  Address temp = createMemTemp(e->getType(), getLoc(e->getSourceRange()));
+  LValue lv = makeAddrLValue(temp, e->getType());
+  emitAggExpr(e, AggValueSlot::forLValue(lv, AggValueSlot::IsNotDestructed,
+                                         AggValueSlot::IsNotAliased,
+                                         AggValueSlot::DoesNotOverlap));
+  return lv;
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
new file mode 100644
index 000000000000..2ffe75a388e9
--- /dev/null
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -0,0 +1,79 @@
+#include "CIRGenBuilder.h"
+#include "CIRGenFunction.h"
+
+#include "clang/AST/StmtVisitor.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+namespace {
+class ComplexExprEmitter : public StmtVisitor<ComplexExprEmitter, mlir::Value> {
+  CIRGenFunction &cgf;
+  CIRGenBuilderTy &builder;
+
+public:
+  explicit ComplexExprEmitter(CIRGenFunction &cgf)
+      : cgf(cgf), builder(cgf.getBuilder()) {}
+
+  /// Store the specified real/imag parts into the
+  /// specified value pointer.
+  void emitStoreOfComplex(mlir::Location loc, mlir::Value val, LValue lv,
+                          bool isInit);
+
+  mlir::Value VisitInitListExpr(InitListExpr *e);
+};
+
+} // namespace
+
+static const ComplexType *getComplexType(QualType type) {
+  type = type.getCanonicalType();
+  if (const ComplexType *comp = dyn_cast<ComplexType>(type))
+    return comp;
+  return cast<ComplexType>(cast<AtomicType>(type)->getValueType());
+}
+
+void ComplexExprEmitter::emitStoreOfComplex(mlir::Location loc, mlir::Value val,
+                                            LValue lv, bool isInit) {
+  if (lv.getType()->isAtomicType() ||
+      (!isInit && cgf.isLValueSuitableForInlineAtomic(lv))) {
+    cgf.cgm.errorNYI("StoreOfComplex with Atomic LV");
+    return;
+  }
+
+  const Address destAddr = lv.getAddress();
+  builder.createStore(loc, val, destAddr);
+}
+
+mlir::Value ComplexExprEmitter::VisitInitListExpr(InitListExpr *e) {
+  mlir::Location loc = cgf.getLoc(e->getExprLoc());
+  if (e->getNumInits() == 2) {
+    mlir::Value real = cgf.emitScalarExpr(e->getInit(0));
+    mlir::Value imag = cgf.emitScalarExpr(e->getInit(1));
+    return builder.createComplexCreate(loc, real, imag);
+  }
+
+  if (e->getNumInits() == 1) {
+    cgf.cgm.errorNYI("Create Complex with InitList with size 1");
+    return {};
+  }
+
+  assert(e->getNumInits() == 0 && "Unexpected number of inits");
+  QualType complexElemTy =
+      e->getType()->castAs<clang::ComplexType>()->getElementType();
+  mlir::Type complexElemLLVMTy = cgf.convertType(complexElemTy);
+  mlir::TypedAttr defaultValue = builder.getZeroInitAttr(complexElemLLVMTy);
+  auto complexAttr = cir::ConstComplexAttr::get(defaultValue, defaultValue);
+  return builder.create<cir::ConstantOp>(loc, complexAttr);
+}
+
+mlir::Value CIRGenFunction::emitComplexExpr(const Expr *e) {
+  assert(e && getComplexType(e->getType()) &&
+         "Invalid complex expression to emit");
+
+  return ComplexExprEmitter(*this).Visit(const_cast<Expr *>(e));
+}
+
+void CIRGenFunction::emitStoreOfComplex(mlir::Location loc, mlir::Value v,
+                                        LValue dest, bool isInit) {
+  ComplexExprEmitter(*this).emitStoreOfComplex(loc, v, dest, isInit);
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index 9085ee2dfe50..c41ab54be09c 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -577,12 +577,33 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
   case APValue::Union:
     cgm.errorNYI("ConstExprEmitter::tryEmitPrivate struct or union");
     return {};
-  case APValue::FixedPoint:
   case APValue::ComplexInt:
-  case APValue::ComplexFloat:
+  case APValue::ComplexFloat: {
+    mlir::Type desiredType = cgm.convertType(destType);
+    cir::ComplexType complexType =
+        mlir::dyn_cast<cir::ComplexType>(desiredType);
+
+    mlir::Type complexElemTy = complexType.getElementType();
+    if (isa<cir::IntType>(complexElemTy)) {
+      llvm::APSInt real = value.getComplexIntReal();
+      llvm::APSInt imag = value.getComplexIntImag();
+      return builder.getAttr<cir::ConstComplexAttr>(
+          complexType, builder.getAttr<cir::IntAttr>(complexElemTy, real),
+          builder.getAttr<cir::IntAttr>(complexElemTy, imag));
+    }
+
+    assert(isa<cir::CIRFPTypeInterface>(complexElemTy) &&
+           "expected floating-point type");
+    llvm::APFloat real = value.getComplexFloatReal();
+    llvm::APFloat imag = value.getComplexFloatImag();
+    return builder.getAttr<cir::ConstComplexAttr>(
+        complexType, builder.getAttr<cir::FPAttr>(complexElemTy, real),
+        builder.getAttr<cir::FPAttr>(complexElemTy, imag));
+  }
+  case APValue::FixedPoint:
   case APValue::AddrLabelDiff:
-    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate fixed point, complex int, "
-                 "complex float, addr label diff");
+    cgm.errorNYI(
+        "ConstExprEmitter::tryEmitPrivate fixed point, addr label diff");
     return {};
   }
   llvm_unreachable("Unknown APValue kind");
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 8448c164a5e5..75b4d2a637e6 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -98,6 +98,15 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
 
   mlir::Value emitPromoted(const Expr *e, QualType promotionType);
 
+  mlir::Value maybePromoteBoolResult(mlir::Value value,
+                                     mlir::Type dstTy) const {
+    if (mlir::isa<cir::IntType>(dstTy))
+      return builder.createBoolToInt(value, dstTy);
+    if (mlir::isa<cir::BoolType>(dstTy))
+      return value;
+    llvm_unreachable("Can only promote integer or boolean types");
+  }
+
   //===--------------------------------------------------------------------===//
   //                            Visitor Methods
   //===--------------------------------------------------------------------===//
@@ -131,7 +140,9 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
 
   // l-values
   mlir::Value VisitDeclRefExpr(DeclRefExpr *e) {
-    assert(!cir::MissingFeatures::tryEmitAsConstant());
+    if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(e))
+      return cgf.emitScalarConstant(constant, e);
+
     return emitLoadOfLValue(e);
   }
 
@@ -151,6 +162,12 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
         builder.getAttr<cir::FPAttr>(type, e->getValue()));
   }
 
+  mlir::Value VisitCharacterLiteral(const CharacterLiteral *e) {
+    mlir::Type ty = cgf.convertType(e->getType());
+    auto init = cir::IntAttr::get(ty, e->getValue());
+    return builder.create<cir::ConstantOp>(cgf.getLoc(e->getExprLoc()), init);
+  }
+
   mlir::Value VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *e) {
     return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
   }
@@ -180,9 +197,24 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
           cgf.getLoc(e->getSourceRange()), inputVec, indexVec);
     }
 
-    cgf.getCIRGenModule().errorNYI(e->getSourceRange(),
-                                   "ShuffleVectorExpr with indices");
-    return {};
+    mlir::Value vec1 = Visit(e->getExpr(0));
+    mlir::Value vec2 = Visit(e->getExpr(1));
+
+    // The documented form of __builtin_shufflevector, where the indices are
+    // a variable number of integer constants. The constants will be stored
+    // in an ArrayAttr.
+    SmallVector<mlir::Attribute, 8> indices;
+    for (unsigned i = 2; i < e->getNumSubExprs(); ++i) {
+      indices.push_back(
+          cir::IntAttr::get(cgf.builder.getSInt64Ty(),
+                            e->getExpr(i)
+                                ->EvaluateKnownConstInt(cgf.getContext())
+                                .getSExtValue()));
+    }
+
+    return cgf.builder.create<cir::VecShuffleOp>(
+        cgf.getLoc(e->getSourceRange()), cgf.convertType(e->getType()), vec1,
+        vec2, cgf.builder.getArrayAttr(indices));
   }
 
   mlir::Value VisitConvertVectorExpr(ConvertVectorExpr *e) {
@@ -320,8 +352,8 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
         assert(!cir::MissingFeatures::fpConstraints());
         castKind = cir::CastKind::float_to_int;
       } else if (mlir::isa<cir::CIRFPTypeInterface>(dstTy)) {
-        cgf.getCIRGenModule().errorNYI("floating point casts");
-        return cgf.createDummyValue(src.getLoc(), dstType);
+        // TODO: split this to createFPExt/createFPTrunc
+        return builder.createFloatingCast(src, fullDstTy);
       } else {
         llvm_unreachable("Internal error: Cast to unexpected type");
       }
@@ -334,6 +366,8 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
   }
 
   mlir::Value VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *e);
+  mlir::Value
+  VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
 
   // Unary Operators.
   mlir::Value VisitUnaryPostDec(const UnaryOperator *e) {
@@ -569,6 +603,8 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
 
   mlir::Value VisitUnaryLNot(const UnaryOperator *e);
 
+  mlir::Value VisitCXXThisExpr(CXXThisExpr *te) { return cgf.loadCXXThis(); }
+
   /// Emit a conversion from the specified type to the specified destination
   /// type, both of which are CIR scalar types.
   /// TODO: do we need ScalarConversionOpts here? Should be done in another
@@ -932,6 +968,71 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     // NOTE: We don't need to EnsureInsertPoint() like LLVM codegen.
     return Visit(e->getRHS());
   }
+
+  mlir::Value VisitBinLAnd(const clang::BinaryOperator *e) {
+    if (e->getType()->isVectorType()) {
+      assert(!cir::MissingFeatures::vectorType());
+      return {};
+    }
+
+    assert(!cir::MissingFeatures::instrumentation());
+    mlir::Type resTy = cgf.convertType(e->getType());
+    mlir::Location loc = cgf.getLoc(e->getExprLoc());
+
+    CIRGenFunction::ConditionalEvaluation eval(cgf);
+
+    mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS());
+    auto resOp = builder.create<cir::TernaryOp>(
+        loc, lhsCondV, /*trueBuilder=*/
+        [&](mlir::OpBuilder &b, mlir::Location loc) {
+          CIRGenFunction::LexicalScope lexScope{cgf, loc,
+                                                b.getInsertionBlock()};
+          cgf.curLexScope->setAsTernary();
+          b.create<cir::YieldOp>(loc, cgf.evaluateExprAsBool(e->getRHS()));
+        },
+        /*falseBuilder*/
+        [&](mlir::OpBuilder &b, mlir::Location loc) {
+          CIRGenFunction::LexicalScope lexScope{cgf, loc,
+                                                b.getInsertionBlock()};
+          cgf.curLexScope->setAsTernary();
+          auto res = b.create<cir::ConstantOp>(loc, builder.getFalseAttr());
+          b.create<cir::YieldOp>(loc, res.getRes());
+        });
+    return maybePromoteBoolResult(resOp.getResult(), resTy);
+  }
+
+  mlir::Value VisitBinLOr(const clang::BinaryOperator *e) {
+    if (e->getType()->isVectorType()) {
+      assert(!cir::MissingFeatures::vectorType());
+      return {};
+    }
+
+    assert(!cir::MissingFeatures::instrumentation());
+    mlir::Type resTy = cgf.convertType(e->getType());
+    mlir::Location loc = cgf.getLoc(e->getExprLoc());
+
+    CIRGenFunction::ConditionalEvaluation eval(cgf);
+
+    mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS());
+    auto resOp = builder.create<cir::TernaryOp>(
+        loc, lhsCondV, /*trueBuilder=*/
+        [&](mlir::OpBuilder &b, mlir::Location loc) {
+          CIRGenFunction::LexicalScope lexScope{cgf, loc,
+                                                b.getInsertionBlock()};
+          cgf.curLexScope->setAsTernary();
+          auto res = b.create<cir::ConstantOp>(loc, builder.getTrueAttr());
+          b.create<cir::YieldOp>(loc, res.getRes());
+        },
+        /*falseBuilder*/
+        [&](mlir::OpBuilder &b, mlir::Location loc) {
+          CIRGenFunction::LexicalScope lexScope{cgf, loc,
+                                                b.getInsertionBlock()};
+          cgf.curLexScope->setAsTernary();
+          b.create<cir::YieldOp>(loc, cgf.evaluateExprAsBool(e->getRHS()));
+        });
+
+    return maybePromoteBoolResult(resOp.getResult(), resTy);
+  }
 };
 
 LValue ScalarExprEmitter::emitCompoundAssignLValue(
@@ -1311,7 +1412,7 @@ mlir::Value ScalarExprEmitter::emitMul(const BinOpInfo &ops) {
       !canElideOverflowCheck(cgf.getContext(), ops))
     cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
-  if (cir::isFPOrFPVectorTy(ops.lhs.getType())) {
+  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
     assert(!cir::MissingFeatures::cgFPOptionsRAII());
     return builder.createFMul(loc, ops.lhs, ops.rhs);
   }
@@ -1370,7 +1471,7 @@ mlir::Value ScalarExprEmitter::emitAdd(const BinOpInfo &ops) {
       !canElideOverflowCheck(cgf.getContext(), ops))
     cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
-  if (cir::isFPOrFPVectorTy(ops.lhs.getType())) {
+  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
     assert(!cir::MissingFeatures::cgFPOptionsRAII());
     return builder.createFAdd(loc, ops.lhs, ops.rhs);
   }
@@ -1418,7 +1519,7 @@ mlir::Value ScalarExprEmitter::emitSub(const BinOpInfo &ops) {
         !canElideOverflowCheck(cgf.getContext(), ops))
       cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
-    if (cir::isFPOrFPVectorTy(ops.lhs.getType())) {
+    if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
       assert(!cir::MissingFeatures::cgFPOptionsRAII());
       return builder.createFSub(loc, ops.lhs, ops.rhs);
     }
@@ -1685,6 +1786,14 @@ mlir::Value ScalarExprEmitter::VisitCastExpr(CastExpr *ce) {
                               cgf.convertType(destTy));
   }
 
+  case CK_VectorSplat: {
+    // Create a vector object and fill all elements with the same scalar value.
+    assert(destTy->isVectorType() && "CK_VectorSplat to non-vector type");
+    return builder.create<cir::VecSplatOp>(
+        cgf.getLoc(subExpr->getSourceRange()), cgf.convertType(destTy),
+        Visit(subExpr));
+  }
+
   default:
     cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                    "CastExpr: ", ce->getCastKindName());
@@ -1779,14 +1888,7 @@ mlir::Value ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *e) {
   boolVal = builder.createNot(boolVal);
 
   // ZExt result to the expr type.
-  mlir::Type dstTy = cgf.convertType(e->getType());
-  if (mlir::isa<cir::IntType>(dstTy))
-    return builder.createBoolToInt(boolVal, dstTy);
-  if (mlir::isa<cir::BoolType>(dstTy))
-    return boolVal;
-
-  cgf.cgm.errorNYI("destination type for logical-not unary operator is NYI");
-  return {};
+  return maybePromoteBoolResult(boolVal, cgf.convertType(e->getType()));
 }
 
 /// Return the size or alignment of the type of argument of the sizeof
@@ -1826,6 +1928,171 @@ mlir::Value ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
                cgf.cgm.UInt64Ty, e->EvaluateKnownConstInt(cgf.getContext())));
 }
 
+/// Return true if the specified expression is cheap enough and side-effect-free
+/// enough to evaluate unconditionally instead of conditionally.  This is used
+/// to convert control flow into selects in some cases.
+/// TODO(cir): can be shared with LLVM codegen.
+static bool isCheapEnoughToEvaluateUnconditionally(const Expr *e,
+                                                   CIRGenFunction &cgf) {
+  // Anything that is an integer or floating point constant is fine.
+  return e->IgnoreParens()->isEvaluatable(cgf.getContext());
+
+  // Even non-volatile automatic variables can't be evaluated unconditionally.
+  // Referencing a thread_local may cause non-trivial initialization work to
+  // occur. If we're inside a lambda and one of the variables is from the scope
+  // outside the lambda, that function may have returned already. Reading its
+  // locals is a bad idea. Also, these reads may introduce races there didn't
+  // exist in the source-level program.
+}
+
+mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator(
+    const AbstractConditionalOperator *e) {
+  CIRGenBuilderTy &builder = cgf.getBuilder();
+  mlir::Location loc = cgf.getLoc(e->getSourceRange());
+  ignoreResultAssign = false;
+
+  // Bind the common expression if necessary.
+  CIRGenFunction::OpaqueValueMapping binding(cgf, e);
+
+  Expr *condExpr = e->getCond();
+  Expr *lhsExpr = e->getTrueExpr();
+  Expr *rhsExpr = e->getFalseExpr();
+
+  // If the condition constant folds and can be elided, try to avoid emitting
+  // the condition and the dead arm.
+  bool condExprBool;
+  if (cgf.constantFoldsToBool(condExpr, condExprBool)) {
+    Expr *live = lhsExpr, *dead = rhsExpr;
+    if (!condExprBool)
+      std::swap(live, dead);
+
+    // If the dead side doesn't have labels we need, just emit the Live part.
+    if (!cgf.containsLabel(dead)) {
+      if (condExprBool)
+        assert(!cir::MissingFeatures::incrementProfileCounter());
+      mlir::Value result = Visit(live);
+
+      // If the live part is a throw expression, it acts like it has a void
+      // type, so evaluating it returns a null Value.  However, a conditional
+      // with non-void type must return a non-null Value.
+      if (!result && !e->getType()->isVoidType()) {
+        cgf.cgm.errorNYI(e->getSourceRange(),
+                         "throw expression in conditional operator");
+        result = {};
+      }
+
+      return result;
+    }
+  }
+
+  QualType condType = condExpr->getType();
+
+  // OpenCL: If the condition is a vector, we can treat this condition like
+  // the select function.
+  if ((cgf.getLangOpts().OpenCL && condType->isVectorType()) ||
+      condType->isExtVectorType()) {
+    assert(!cir::MissingFeatures::vectorType());
+    cgf.cgm.errorNYI(e->getSourceRange(), "vector ternary op");
+  }
+
+  if (condType->isVectorType() || condType->isSveVLSBuiltinType()) {
+    if (!condType->isVectorType()) {
+      assert(!cir::MissingFeatures::vecTernaryOp());
+      cgf.cgm.errorNYI(loc, "TernaryOp for SVE vector");
+      return {};
+    }
+
+    mlir::Value condValue = Visit(condExpr);
+    mlir::Value lhsValue = Visit(lhsExpr);
+    mlir::Value rhsValue = Visit(rhsExpr);
+    return builder.create<cir::VecTernaryOp>(loc, condValue, lhsValue,
+                                             rhsValue);
+  }
+
+  // If this is a really simple expression (like x ? 4 : 5), emit this as a
+  // select instead of as control flow.  We can only do this if it is cheap
+  // and safe to evaluate the LHS and RHS unconditionally.
+  if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, cgf) &&
+      isCheapEnoughToEvaluateUnconditionally(rhsExpr, cgf)) {
+    bool lhsIsVoid = false;
+    mlir::Value condV = cgf.evaluateExprAsBool(condExpr);
+    assert(!cir::MissingFeatures::incrementProfileCounter());
+
+    mlir::Value lhs = Visit(lhsExpr);
+    if (!lhs) {
+      lhs = builder.getNullValue(cgf.VoidTy, loc);
+      lhsIsVoid = true;
+    }
+
+    mlir::Value rhs = Visit(rhsExpr);
+    if (lhsIsVoid) {
+      assert(!rhs && "lhs and rhs types must match");
+      rhs = builder.getNullValue(cgf.VoidTy, loc);
+    }
+
+    return builder.createSelect(loc, condV, lhs, rhs);
+  }
+
+  mlir::Value condV = cgf.emitOpOnBoolExpr(loc, condExpr);
+  CIRGenFunction::ConditionalEvaluation eval(cgf);
+  SmallVector<mlir::OpBuilder::InsertPoint, 2> insertPoints{};
+  mlir::Type yieldTy{};
+
+  auto emitBranch = [&](mlir::OpBuilder &b, mlir::Location loc, Expr *expr) {
+    CIRGenFunction::LexicalScope lexScope{cgf, loc, b.getInsertionBlock()};
+    cgf.curLexScope->setAsTernary();
+
+    assert(!cir::MissingFeatures::incrementProfileCounter());
+    eval.beginEvaluation();
+    mlir::Value branch = Visit(expr);
+    eval.endEvaluation();
+
+    if (branch) {
+      yieldTy = branch.getType();
+      b.create<cir::YieldOp>(loc, branch);
+    } else {
+      // If LHS or RHS is a throw or void expression we need to patch
+      // arms as to properly match yield types.
+      insertPoints.push_back(b.saveInsertionPoint());
+    }
+  };
+
+  mlir::Value result = builder
+                           .create<cir::TernaryOp>(
+                               loc, condV,
+                               /*trueBuilder=*/
+                               [&](mlir::OpBuilder &b, mlir::Location loc) {
+                                 emitBranch(b, loc, lhsExpr);
+                               },
+                               /*falseBuilder=*/
+                               [&](mlir::OpBuilder &b, mlir::Location loc) {
+                                 emitBranch(b, loc, rhsExpr);
+                               })
+                           .getResult();
+
+  if (!insertPoints.empty()) {
+    // If both arms are void, so be it.
+    if (!yieldTy)
+      yieldTy = cgf.VoidTy;
+
+    // Insert required yields.
+    for (mlir::OpBuilder::InsertPoint &toInsert : insertPoints) {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      builder.restoreInsertionPoint(toInsert);
+
+      // Block does not return: build empty yield.
+      if (mlir::isa<cir::VoidType>(yieldTy)) {
+        builder.create<cir::YieldOp>(loc);
+      } else { // Block returns: set null yield value.
+        mlir::Value op0 = builder.getNullValue(yieldTy, loc);
+        builder.create<cir::YieldOp>(loc, op0);
+      }
+    }
+  }
+
+  return result;
+}
+
 mlir::Value CIRGenFunction::emitScalarPrePostIncDec(const UnaryOperator *e,
                                                     LValue lv, bool isInc,
                                                     bool isPre) {
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index bd67a3407d74..fd413fe86383 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -16,6 +16,7 @@
 #include "CIRGenCall.h"
 #include "CIRGenValue.h"
 #include "mlir/IR/Location.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/GlobalDecl.h"
 #include "clang/CIR/MissingFeatures.h"
 
@@ -343,7 +344,9 @@ void CIRGenFunction::startFunction(GlobalDecl gd, QualType returnType,
 
   curFn = fn;
 
-  const auto *fd = dyn_cast_or_null<FunctionDecl>(gd.getDecl());
+  const Decl *d = gd.getDecl();
+  const auto *fd = dyn_cast_or_null<FunctionDecl>(d);
+  curFuncDecl = d->getNonClosureContext();
 
   mlir::Block *entryBB = &fn.getBlocks().front();
   builder.setInsertionPointToStart(entryBB);
@@ -385,6 +388,24 @@ void CIRGenFunction::startFunction(GlobalDecl gd, QualType returnType,
   if (!returnType->isVoidType())
     emitAndUpdateRetAlloca(returnType, getLoc(fd->getBody()->getEndLoc()),
                            getContext().getTypeAlignInChars(returnType));
+
+  if (isa_and_nonnull<CXXMethodDecl>(d) &&
+      cast<CXXMethodDecl>(d)->isInstance()) {
+    cgm.getCXXABI().emitInstanceFunctionProlog(loc, *this);
+
+    const auto *md = cast<CXXMethodDecl>(d);
+    if (md->getParent()->isLambda() && md->getOverloadedOperator() == OO_Call) {
+      cgm.errorNYI(loc, "lambda call operator");
+    } else {
+      // Not in a lambda; just use 'this' from the method.
+      // FIXME: Should we generate a new load for each use of 'this'? The fast
+      // register allocator would be happier...
+      cxxThisValue = cxxabiThisValue;
+    }
+
+    assert(!cir::MissingFeatures::sanitizers());
+    assert(!cir::MissingFeatures::emitTypeCheck());
+  }
 }
 
 void CIRGenFunction::finishFunction(SourceLocation endLoc) {}
@@ -444,7 +465,7 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
     if (isa<CXXDestructorDecl>(funcDecl))
       getCIRGenModule().errorNYI(bodyRange, "C++ destructor definition");
     else if (isa<CXXConstructorDecl>(funcDecl))
-      getCIRGenModule().errorNYI(bodyRange, "C++ constructor definition");
+      emitConstructorBody(args);
     else if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice &&
              funcDecl->hasAttr<CUDAGlobalAttr>())
       getCIRGenModule().errorNYI(bodyRange, "CUDA kernel");
@@ -475,6 +496,48 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
   return fn;
 }
 
+void CIRGenFunction::emitConstructorBody(FunctionArgList &args) {
+  assert(!cir::MissingFeatures::sanitizers());
+  const auto *ctor = cast<CXXConstructorDecl>(curGD.getDecl());
+  CXXCtorType ctorType = curGD.getCtorType();
+
+  assert((cgm.getTarget().getCXXABI().hasConstructorVariants() ||
+          ctorType == Ctor_Complete) &&
+         "can only generate complete ctor for this ABI");
+
+  if (ctorType == Ctor_Complete && isConstructorDelegationValid(ctor) &&
+      cgm.getTarget().getCXXABI().hasConstructorVariants()) {
+    emitDelegateCXXConstructorCall(ctor, Ctor_Base, args, ctor->getEndLoc());
+    return;
+  }
+
+  const FunctionDecl *definition = nullptr;
+  Stmt *body = ctor->getBody(definition);
+  assert(definition == ctor && "emitting wrong constructor body");
+
+  if (isa_and_nonnull<CXXTryStmt>(body)) {
+    cgm.errorNYI(ctor->getSourceRange(), "emitConstructorBody: try body");
+    return;
+  }
+
+  assert(!cir::MissingFeatures::incrementProfileCounter());
+  assert(!cir::MissingFeatures::runCleanupsScope());
+
+  // TODO: in restricted cases, we can emit the vbase initializers of a
+  // complete ctor and then delegate to the base ctor.
+
+  // Emit the constructor prologue, i.e. the base and member initializers.
+  emitCtorPrologue(ctor, ctorType, args);
+
+  // TODO(cir): propagate this result via mlir::logical result. Just unreachable
+  // now just to have it handled.
+  if (mlir::failed(emitStmt(body, true))) {
+    cgm.errorNYI(ctor->getSourceRange(),
+                 "emitConstructorBody: emit body statement failed.");
+    return;
+  }
+}
+
 /// Given a value of type T* that may not be to a complete object, construct
 /// an l-vlaue withi the natural pointee alignment of T.
 LValue CIRGenFunction::makeNaturalAlignPointeeAddrLValue(mlir::Value val,
@@ -501,16 +564,16 @@ clang::QualType CIRGenFunction::buildFunctionArgList(clang::GlobalDecl gd,
     cgm.getCXXABI().buildThisParam(*this, args);
   }
 
-  if (isa<CXXConstructorDecl>(fd))
-    cgm.errorNYI(fd->getSourceRange(),
-                 "buildFunctionArgList: CXXConstructorDecl");
+  if (const auto *cd = dyn_cast<CXXConstructorDecl>(fd))
+    if (cd->getInheritedConstructor())
+      cgm.errorNYI(fd->getSourceRange(),
+                   "buildFunctionArgList: inherited constructor");
 
   for (auto *param : fd->parameters())
     args.push_back(param);
 
   if (md && (isa<CXXConstructorDecl>(md) || isa<CXXDestructorDecl>(md)))
-    cgm.errorNYI(fd->getSourceRange(),
-                 "buildFunctionArgList: implicit structor params");
+    assert(!cir::MissingFeatures::cxxabiStructorImplicitParam());
 
   return retTy;
 }
@@ -566,6 +629,17 @@ LValue CIRGenFunction::emitLValue(const Expr *e) {
   }
 }
 
+static std::string getVersionedTmpName(llvm::StringRef name, unsigned cnt) {
+  SmallString<256> buffer;
+  llvm::raw_svector_ostream out(buffer);
+  out << name << cnt;
+  return std::string(out.str());
+}
+
+std::string CIRGenFunction::getCounterAggTmpAsString() {
+  return getVersionedTmpName("agg.tmp", counterAggTmp++);
+}
+
 void CIRGenFunction::emitNullInitialization(mlir::Location loc, Address destPtr,
                                             QualType ty) {
   // Ignore empty classes in C++.
@@ -609,4 +683,25 @@ void CIRGenFunction::emitNullInitialization(mlir::Location loc, Address destPtr,
   builder.createStore(loc, zeroValue, destPtr);
 }
 
+// TODO(cir): should be shared with LLVM codegen.
+bool CIRGenFunction::shouldNullCheckClassCastValue(const CastExpr *ce) {
+  const Expr *e = ce->getSubExpr();
+
+  if (ce->getCastKind() == CK_UncheckedDerivedToBase)
+    return false;
+
+  if (isa<CXXThisExpr>(e->IgnoreParens())) {
+    // We always assume that 'this' is never null.
+    return false;
+  }
+
+  if (const ImplicitCastExpr *ice = dyn_cast<ImplicitCastExpr>(ce)) {
+    // And that glvalue casts are never null.
+    if (ice->isGLValue())
+      return false;
+  }
+
+  return true;
+}
+
 } // namespace clang::CIRGen
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 0badde024b16..de6ef2a69faf 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -34,6 +34,12 @@ namespace {
 class ScalarExprEmitter;
 } // namespace
 
+namespace mlir {
+namespace acc {
+class LoopOp;
+} // namespace acc
+} // namespace mlir
+
 namespace clang::CIRGen {
 
 class CIRGenFunction : public CIRGenTypeCache {
@@ -55,6 +61,16 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// The compiler-generated variable that holds the return value.
   std::optional<mlir::Value> fnRetAlloca;
 
+  /// CXXThisDecl - When generating code for a C++ member function,
+  /// this will hold the implicit 'this' declaration.
+  ImplicitParamDecl *cxxabiThisDecl = nullptr;
+  mlir::Value cxxabiThisValue = nullptr;
+  mlir::Value cxxThisValue = nullptr;
+  clang::CharUnits cxxThisAlignment;
+
+  // Holds the Decl for the current outermost non-closure context
+  const clang::Decl *curFuncDecl = nullptr;
+
   /// The function for which code is currently being generated.
   cir::FuncOp curFn;
 
@@ -109,6 +125,144 @@ class CIRGenFunction : public CIRGenTypeCache {
   const TargetInfo &getTarget() const { return cgm.getTarget(); }
   mlir::MLIRContext &getMLIRContext() { return cgm.getMLIRContext(); }
 
+  // ---------------------
+  // Opaque value handling
+  // ---------------------
+
+  /// Keeps track of the current set of opaque value expressions.
+  llvm::DenseMap<const OpaqueValueExpr *, LValue> opaqueLValues;
+  llvm::DenseMap<const OpaqueValueExpr *, RValue> opaqueRValues;
+
+public:
+  /// A non-RAII class containing all the information about a bound
+  /// opaque value.  OpaqueValueMapping, below, is a RAII wrapper for
+  /// this which makes individual mappings very simple; using this
+  /// class directly is useful when you have a variable number of
+  /// opaque values or don't want the RAII functionality for some
+  /// reason.
+  class OpaqueValueMappingData {
+    const OpaqueValueExpr *opaqueValue;
+    bool boundLValue;
+
+    OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
+        : opaqueValue(ov), boundLValue(boundLValue) {}
+
+  public:
+    OpaqueValueMappingData() : opaqueValue(nullptr) {}
+
+    static bool shouldBindAsLValue(const Expr *expr) {
+      // gl-values should be bound as l-values for obvious reasons.
+      // Records should be bound as l-values because IR generation
+      // always keeps them in memory.  Expressions of function type
+      // act exactly like l-values but are formally required to be
+      // r-values in C.
+      return expr->isGLValue() || expr->getType()->isFunctionType() ||
+             hasAggregateEvaluationKind(expr->getType());
+    }
+
+    static OpaqueValueMappingData
+    bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const Expr *e) {
+      if (shouldBindAsLValue(ov))
+        return bind(cgf, ov, cgf.emitLValue(e));
+      return bind(cgf, ov, cgf.emitAnyExpr(e));
+    }
+
+    static OpaqueValueMappingData
+    bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const LValue &lv) {
+      assert(shouldBindAsLValue(ov));
+      cgf.opaqueLValues.insert(std::make_pair(ov, lv));
+      return OpaqueValueMappingData(ov, true);
+    }
+
+    static OpaqueValueMappingData
+    bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const RValue &rv) {
+      assert(!shouldBindAsLValue(ov));
+      cgf.opaqueRValues.insert(std::make_pair(ov, rv));
+
+      OpaqueValueMappingData data(ov, false);
+
+      // Work around an extremely aggressive peephole optimization in
+      // EmitScalarConversion which assumes that all other uses of a
+      // value are extant.
+      assert(!cir::MissingFeatures::peepholeProtection() && "NYI");
+      return data;
+    }
+
+    bool isValid() const { return opaqueValue != nullptr; }
+    void clear() { opaqueValue = nullptr; }
+
+    void unbind(CIRGenFunction &cgf) {
+      assert(opaqueValue && "no data to unbind!");
+
+      if (boundLValue) {
+        cgf.opaqueLValues.erase(opaqueValue);
+      } else {
+        cgf.opaqueRValues.erase(opaqueValue);
+        assert(!cir::MissingFeatures::peepholeProtection() && "NYI");
+      }
+    }
+  };
+
+  /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
+  class OpaqueValueMapping {
+    CIRGenFunction &cgf;
+    OpaqueValueMappingData data;
+
+  public:
+    static bool shouldBindAsLValue(const Expr *expr) {
+      return OpaqueValueMappingData::shouldBindAsLValue(expr);
+    }
+
+    /// Build the opaque value mapping for the given conditional
+    /// operator if it's the GNU ?: extension.  This is a common
+    /// enough pattern that the convenience operator is really
+    /// helpful.
+    ///
+    OpaqueValueMapping(CIRGenFunction &cgf,
+                       const AbstractConditionalOperator *op)
+        : cgf(cgf) {
+      if (mlir::isa<ConditionalOperator>(op))
+        // Leave Data empty.
+        return;
+
+      const BinaryConditionalOperator *e =
+          mlir::cast<BinaryConditionalOperator>(op);
+      data = OpaqueValueMappingData::bind(cgf, e->getOpaqueValue(),
+                                          e->getCommon());
+    }
+
+    /// Build the opaque value mapping for an OpaqueValueExpr whose source
+    /// expression is set to the expression the OVE represents.
+    OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *ov)
+        : cgf(cgf) {
+      if (ov) {
+        assert(ov->getSourceExpr() && "wrong form of OpaqueValueMapping used "
+                                      "for OVE with no source expression");
+        data = OpaqueValueMappingData::bind(cgf, ov, ov->getSourceExpr());
+      }
+    }
+
+    OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *opaqueValue,
+                       LValue lvalue)
+        : cgf(cgf),
+          data(OpaqueValueMappingData::bind(cgf, opaqueValue, lvalue)) {}
+
+    OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *opaqueValue,
+                       RValue rvalue)
+        : cgf(cgf),
+          data(OpaqueValueMappingData::bind(cgf, opaqueValue, rvalue)) {}
+
+    void pop() {
+      data.unbind(cgf);
+      data.clear();
+    }
+
+    ~OpaqueValueMapping() {
+      if (data.isValid())
+        data.unbind(cgf);
+    }
+  };
+
 private:
   /// Declare a variable in the current scope, return success if the variable
   /// wasn't declared yet.
@@ -162,6 +316,10 @@ class CIRGenFunction : public CIRGenTypeCache {
     ~SourceLocRAIIObject() { restore(); }
   };
 
+  /// Hold counters for incrementally naming temporaries
+  unsigned counterAggTmp = 0;
+  std::string getCounterAggTmpAsString();
+
   /// Helpers to convert Clang's SourceLocation to a MLIR Location.
   mlir::Location getLoc(clang::SourceLocation srcLoc);
   mlir::Location getLoc(clang::SourceRange srcLoc);
@@ -180,6 +338,8 @@ class CIRGenFunction : public CIRGenTypeCache {
     PrototypeWrapper(const clang::ObjCMethodDecl *md) : p(md) {}
   };
 
+  bool isLValueSuitableForInlineAtomic(LValue lv);
+
   /// An abstract representation of regular/ObjC call/message targets.
   class AbstractCallee {
     /// The function declaration of the callee.
@@ -227,6 +387,41 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// that we can just remove the code.
   bool containsLabel(const clang::Stmt *s, bool ignoreCaseStmts = false);
 
+  class ConstantEmission {
+    // Cannot use mlir::TypedAttr directly here because of bit availability.
+    llvm::PointerIntPair<mlir::Attribute, 1, bool> valueAndIsReference;
+    ConstantEmission(mlir::TypedAttr c, bool isReference)
+        : valueAndIsReference(c, isReference) {}
+
+  public:
+    ConstantEmission() {}
+    static ConstantEmission forReference(mlir::TypedAttr c) {
+      return ConstantEmission(c, true);
+    }
+    static ConstantEmission forValue(mlir::TypedAttr c) {
+      return ConstantEmission(c, false);
+    }
+
+    explicit operator bool() const {
+      return valueAndIsReference.getOpaqueValue() != nullptr;
+    }
+
+    bool isReference() const { return valueAndIsReference.getInt(); }
+    LValue getReferenceLValue(CIRGenFunction &cgf, Expr *refExpr) const {
+      assert(isReference());
+      cgf.cgm.errorNYI(refExpr->getSourceRange(),
+                       "ConstantEmission::getReferenceLValue");
+      return {};
+    }
+
+    mlir::TypedAttr getValue() const {
+      assert(!isReference());
+      return mlir::cast<mlir::TypedAttr>(valueAndIsReference.getPointer());
+    }
+  };
+
+  ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr);
+
   struct AutoVarEmission {
     const clang::VarDecl *Variable;
     /// The address of the alloca for languages with explicit address space
@@ -263,7 +458,7 @@ class CIRGenFunction : public CIRGenTypeCache {
     /// Returns the address of the object within this declaration.
     /// Note that this does not chase the forwarding pointer for
     /// __block decls.
-    Address getObjectAddress(CIRGenFunction &CGF) const {
+    Address getObjectAddress(CIRGenFunction &cgf) const {
       if (!IsEscapingByRef)
         return Addr;
 
@@ -276,6 +471,10 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// compare the result against zero, returning an Int1Ty value.
   mlir::Value evaluateExprAsBool(const clang::Expr *e);
 
+  cir::GlobalOp addInitializerToStaticVarDecl(const VarDecl &d,
+                                              cir::GlobalOp gv,
+                                              cir::GetGlobalOp gvAddr);
+
   /// Set the address of a local variable.
   void setAddrOfLocalVar(const clang::VarDecl *vd, Address addr) {
     assert(!localDeclMap.count(vd) && "Decl already exists in LocalDeclMap!");
@@ -283,6 +482,14 @@ class CIRGenFunction : public CIRGenTypeCache {
     // TODO: Add symbol table support
   }
 
+  bool shouldNullCheckClassCastValue(const CastExpr *ce);
+
+  RValue convertTempToRValue(Address addr, clang::QualType type,
+                             clang::SourceLocation loc);
+
+  static bool
+  isConstructorDelegationValid(const clang::CXXConstructorDecl *ctor);
+
   LValue makeNaturalAlignPointeeAddrLValue(mlir::Value v, clang::QualType t);
 
   /// Construct an address with the natural alignment of T. If a pointer to T
@@ -298,6 +505,11 @@ class CIRGenFunction : public CIRGenTypeCache {
     return Address(ptr, convertTypeForMem(t), alignment);
   }
 
+  Address getAddressOfBaseClass(
+      Address value, const CXXRecordDecl *derived,
+      llvm::iterator_range<CastExpr::path_const_iterator> path,
+      bool nullCheckValue, SourceLocation loc);
+
   LValue makeAddrLValue(Address addr, QualType ty,
                         AlignmentSource source = AlignmentSource::Type) {
     return makeAddrLValue(addr, ty, LValueBaseInfo(source));
@@ -307,6 +519,23 @@ class CIRGenFunction : public CIRGenTypeCache {
     return LValue::makeAddr(addr, ty, baseInfo);
   }
 
+  /// Return the address of a local variable.
+  Address getAddrOfLocalVar(const clang::VarDecl *vd) {
+    auto it = localDeclMap.find(vd);
+    assert(it != localDeclMap.end() &&
+           "Invalid argument to getAddrOfLocalVar(), no decl!");
+    return it->second;
+  }
+
+  /// Load the value for 'this'. This function is only valid while generating
+  /// code for an C++ member function.
+  /// FIXME(cir): this should return a mlir::Value!
+  mlir::Value loadCXXThis() {
+    assert(cxxThisValue && "no 'this' value for this function");
+    return cxxThisValue;
+  }
+  Address loadCXXThisAddress();
+
   /// Get an appropriate 'undef' rvalue for the given type.
   /// TODO: What's the equivalent for MLIR? Currently we're only using this for
   /// void types so it just returns RValue::get(nullptr) but it'll need
@@ -460,6 +689,8 @@ class CIRGenFunction : public CIRGenTypeCache {
   void emitAndUpdateRetAlloca(clang::QualType type, mlir::Location loc,
                               clang::CharUnits alignment);
 
+  CIRGenCallee emitDirectCallee(const GlobalDecl &gd);
+
 public:
   Address emitAddrOfFieldStorage(Address base, const FieldDecl *field,
                                  llvm::StringRef fieldName,
@@ -474,13 +705,18 @@ class CIRGenFunction : public CIRGenTypeCache {
                          mlir::OpBuilder::InsertPoint ip,
                          mlir::Value arraySize = nullptr);
 
+  void emitAggregateStore(mlir::Value value, Address dest);
+
   void emitAggExpr(const clang::Expr *e, AggValueSlot slot);
 
+  LValue emitAggExprToLValue(const Expr *e);
+
   /// Emit code to compute the specified expression which can have any type. The
   /// result is returned as an RValue struct. If this is an aggregate
   /// expression, the aggloc/agglocvolatile arguments indicate where the result
   /// should be returned.
-  RValue emitAnyExpr(const clang::Expr *e);
+  RValue emitAnyExpr(const clang::Expr *e,
+                     AggValueSlot aggSlot = AggValueSlot::ignored());
 
   /// Similarly to emitAnyExpr(), however, the result will always be accessible
   /// even if no aggregate location is provided.
@@ -504,6 +740,9 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitBreakStmt(const clang::BreakStmt &s);
 
+  RValue emitBuiltinExpr(const clang::GlobalDecl &gd, unsigned builtinID,
+                         const clang::CallExpr *e, ReturnValueSlot returnValue);
+
   RValue emitCall(const CIRGenFunctionInfo &funcInfo,
                   const CIRGenCallee &callee, ReturnValueSlot returnValue,
                   const CallArgList &args, cir::CIRCallOpInterface *callOp,
@@ -535,8 +774,23 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   LValue emitCompoundAssignmentLValue(const clang::CompoundAssignOperator *e);
 
+  void emitConstructorBody(FunctionArgList &args);
+
   mlir::LogicalResult emitContinueStmt(const clang::ContinueStmt &s);
 
+  void emitCXXConstructExpr(const clang::CXXConstructExpr *e,
+                            AggValueSlot dest);
+
+  void emitCXXConstructorCall(const clang::CXXConstructorDecl *d,
+                              clang::CXXCtorType type, bool forVirtualBase,
+                              bool delegating, AggValueSlot thisAVS,
+                              const clang::CXXConstructExpr *e);
+
+  void emitCXXConstructorCall(const clang::CXXConstructorDecl *d,
+                              clang::CXXCtorType type, bool forVirtualBase,
+                              bool delegating, Address thisAddr,
+                              CallArgList &args, clang::SourceLocation loc);
+
   mlir::LogicalResult emitCXXForRangeStmt(const CXXForRangeStmt &s,
                                           llvm::ArrayRef<const Attr *> attrs);
 
@@ -559,6 +813,16 @@ class CIRGenFunction : public CIRGenTypeCache {
                                        const CXXMethodDecl *md,
                                        ReturnValueSlot returnValue);
 
+  void emitCtorPrologue(const clang::CXXConstructorDecl *ctor,
+                        clang::CXXCtorType ctorType, FunctionArgList &args);
+
+  // It's important not to confuse this and emitDelegateCXXConstructorCall.
+  // Delegating constructors are the C++11 feature. The constructor delegate
+  // optimization is used to reduce duplication in the base and complete
+  // constructors where they are substantially the same.
+  void emitDelegatingCXXConstructorCall(const CXXConstructorDecl *ctor,
+                                        const FunctionArgList &args);
+
   mlir::LogicalResult emitDoStmt(const clang::DoStmt &s);
 
   /// Emit an expression as an initializer for an object (variable, field, etc.)
@@ -598,6 +862,10 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitForStmt(const clang::ForStmt &s);
 
+  /// Emit the computation of the specified expression of complex type,
+  /// returning the result.
+  mlir::Value emitComplexExpr(const Expr *e);
+
   void emitCompoundStmt(const clang::CompoundStmt &s);
 
   void emitCompoundStmtWithoutScope(const clang::CompoundStmt &s);
@@ -610,6 +878,17 @@ class CIRGenFunction : public CIRGenTypeCache {
                                       mlir::Type condType,
                                       bool buildingTopLevelCase);
 
+  void emitDelegateCXXConstructorCall(const clang::CXXConstructorDecl *ctor,
+                                      clang::CXXCtorType ctorType,
+                                      const FunctionArgList &args,
+                                      clang::SourceLocation loc);
+
+  /// We are performing a delegate call; that is, the current function is
+  /// delegating to another one. Produce a r-value suitable for passing the
+  /// given parameter.
+  void emitDelegateCallArg(CallArgList &args, const clang::VarDecl *param,
+                           clang::SourceLocation loc);
+
   /// Emit an `if` on a boolean condition to the specified blocks.
   /// FIXME: Based on the condition, this might try to simplify the codegen of
   /// the conditional based on the branch.
@@ -675,6 +954,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitReturnStmt(const clang::ReturnStmt &s);
 
+  mlir::Value emitScalarConstant(const ConstantEmission &constant, Expr *e);
+
   /// Emit a conversion from the specified type to the specified destination
   /// type, both of which are CIR scalar types.
   mlir::Value emitScalarConversion(mlir::Value src, clang::QualType srcType,
@@ -684,6 +965,11 @@ class CIRGenFunction : public CIRGenTypeCache {
   void emitScalarInit(const clang::Expr *init, mlir::Location loc,
                       LValue lvalue, bool capturedByInit = false);
 
+  void emitStaticVarDecl(const VarDecl &d, cir::GlobalLinkageKind linkage);
+
+  void emitStoreOfComplex(mlir::Location loc, mlir::Value v, LValue dest,
+                          bool isInit);
+
   void emitStoreOfScalar(mlir::Value value, Address addr, bool isVolatile,
                          clang::QualType ty, bool isInit = false,
                          bool isNontemporal = false);
@@ -722,12 +1008,101 @@ class CIRGenFunction : public CIRGenTypeCache {
   void emitNullabilityCheck(LValue lhs, mlir::Value rhs,
                             clang::SourceLocation loc);
 
+  /// An object to manage conditionally-evaluated expressions.
+  class ConditionalEvaluation {
+    CIRGenFunction &cgf;
+    mlir::OpBuilder::InsertPoint insertPt;
+
+  public:
+    ConditionalEvaluation(CIRGenFunction &cgf)
+        : cgf(cgf), insertPt(cgf.builder.saveInsertionPoint()) {}
+    ConditionalEvaluation(CIRGenFunction &cgf, mlir::OpBuilder::InsertPoint ip)
+        : cgf(cgf), insertPt(ip) {}
+
+    void beginEvaluation() {
+      assert(cgf.outermostConditional != this);
+      if (!cgf.outermostConditional)
+        cgf.outermostConditional = this;
+    }
+
+    void endEvaluation() {
+      assert(cgf.outermostConditional != nullptr);
+      if (cgf.outermostConditional == this)
+        cgf.outermostConditional = nullptr;
+    }
+
+    /// Returns the insertion point which will be executed prior to each
+    /// evaluation of the conditional code. In LLVM OG, this method
+    /// is called getStartingBlock.
+    mlir::OpBuilder::InsertPoint getInsertPoint() const { return insertPt; }
+  };
+
+  struct ConditionalInfo {
+    std::optional<LValue> lhs{}, rhs{};
+    mlir::Value result{};
+  };
+
+  // Return true if we're currently emitting one branch or the other of a
+  // conditional expression.
+  bool isInConditionalBranch() const { return outermostConditional != nullptr; }
+
+  void setBeforeOutermostConditional(mlir::Value value, Address addr) {
+    assert(isInConditionalBranch());
+    {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      builder.restoreInsertionPoint(outermostConditional->getInsertPoint());
+      builder.createStore(
+          value.getLoc(), value, addr,
+          mlir::IntegerAttr::get(
+              mlir::IntegerType::get(value.getContext(), 64),
+              (uint64_t)addr.getAlignment().getAsAlign().value()));
+    }
+  }
+
+  // Points to the outermost active conditional control. This is used so that
+  // we know if a temporary should be destroyed conditionally.
+  ConditionalEvaluation *outermostConditional = nullptr;
+
+  template <typename FuncTy>
+  ConditionalInfo emitConditionalBlocks(const AbstractConditionalOperator *e,
+                                        const FuncTy &branchGenFunc);
+
+  mlir::Value emitTernaryOnBoolExpr(const clang::Expr *cond, mlir::Location loc,
+                                    const clang::Stmt *thenS,
+                                    const clang::Stmt *elseS);
+
   /// ----------------------
   /// CIR build helpers
   /// -----------------
 public:
+  cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc,
+                                 const Twine &name = "tmp",
+                                 mlir::Value arraySize = nullptr,
+                                 bool insertIntoFnEntryBlock = false);
+  cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc,
+                                 const Twine &name = "tmp",
+                                 mlir::OpBuilder::InsertPoint ip = {},
+                                 mlir::Value arraySize = nullptr);
   Address createTempAlloca(mlir::Type ty, CharUnits align, mlir::Location loc,
-                           const Twine &name, bool insertIntoFnEntryBlock);
+                           const Twine &name = "tmp",
+                           mlir::Value arraySize = nullptr,
+                           Address *alloca = nullptr,
+                           mlir::OpBuilder::InsertPoint ip = {});
+  Address createTempAllocaWithoutCast(mlir::Type ty, CharUnits align,
+                                      mlir::Location loc,
+                                      const Twine &name = "tmp",
+                                      mlir::Value arraySize = nullptr,
+                                      mlir::OpBuilder::InsertPoint ip = {});
+
+  /// Create a temporary memory object of the given type, with
+  /// appropriate alignmen and cast it to the default address space. Returns
+  /// the original alloca instruction by \p Alloca if it is not nullptr.
+  Address createMemTemp(QualType t, mlir::Location loc,
+                        const Twine &name = "tmp", Address *alloca = nullptr,
+                        mlir::OpBuilder::InsertPoint ip = {});
+  Address createMemTemp(QualType t, CharUnits align, mlir::Location loc,
+                        const Twine &name = "tmp", Address *alloca = nullptr,
+                        mlir::OpBuilder::InsertPoint ip = {});
 
   //===--------------------------------------------------------------------===//
   //                         OpenACC Emission
@@ -766,6 +1141,12 @@ class CIRGenFunction : public CIRGenTypeCache {
                           OpenACCDirectiveKind dirKind, SourceLocation dirLoc,
                           ArrayRef<const OpenACCClause *> clauses);
 
+  // The OpenACC LoopOp requires that we have auto, seq, or independent on all
+  // LoopOp operations for the 'none' device type case. This function checks if
+  // the LoopOp has one, else it updates it to have one.
+  void updateLoopOpParallelism(mlir::acc::LoopOp &op, bool isOrphan,
+                               OpenACCDirectiveKind dk);
+
 public:
   mlir::LogicalResult
   emitOpenACCComputeConstruct(const OpenACCComputeConstruct &s);
@@ -793,6 +1174,17 @@ class CIRGenFunction : public CIRGenTypeCache {
   void emitOpenACCDeclare(const OpenACCDeclareDecl &d);
   void emitOpenACCRoutine(const OpenACCRoutineDecl &d);
 
+  /// Create a temporary memory object for the given aggregate type.
+  AggValueSlot createAggTemp(QualType ty, mlir::Location loc,
+                             const Twine &name = "tmp",
+                             Address *alloca = nullptr) {
+    assert(!cir::MissingFeatures::aggValueSlot());
+    return AggValueSlot::forAddr(
+        createMemTemp(ty, loc, name, alloca), ty.getQualifiers(),
+        AggValueSlot::IsNotDestructed, AggValueSlot::IsNotAliased,
+        AggValueSlot::DoesNotOverlap);
+  }
+
 private:
   QualType getVarArgType(const Expr *arg);
 };
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h
index 9886574fd463..4f5754cb4398 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenFunctionInfo.h
@@ -76,10 +76,8 @@ class CIRGenFunctionInfo final
 
   unsigned numArgs;
 
-  CanQualType *getArgTypes() { return getTrailingObjects<CanQualType>(); }
-  const CanQualType *getArgTypes() const {
-    return getTrailingObjects<CanQualType>();
-  }
+  CanQualType *getArgTypes() { return getTrailingObjects(); }
+  const CanQualType *getArgTypes() const { return getTrailingObjects(); }
 
   CIRGenFunctionInfo() : required(RequiredArgs::All) {}
 
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
new file mode 100644
index 000000000000..cd9096a0188a
--- /dev/null
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -0,0 +1,151 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides C++ code generation targeting the Itanium C++ ABI.  The class
+// in this file generates structures that follow the Itanium C++ ABI, which is
+// documented at:
+//  https://itanium-cxx-abi.github.io/cxx-abi/abi.html
+//  https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
+//
+// It also supports the closely-related ARM ABI, documented at:
+// https://developer.arm.com/documentation/ihi0041/g/
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenCXXABI.h"
+#include "CIRGenFunction.h"
+
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/GlobalDecl.h"
+#include "clang/CIR/MissingFeatures.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+namespace {
+
+class CIRGenItaniumCXXABI : public CIRGenCXXABI {
+public:
+  CIRGenItaniumCXXABI(CIRGenModule &cgm) : CIRGenCXXABI(cgm) {
+    assert(!cir::MissingFeatures::cxxabiUseARMMethodPtrABI());
+    assert(!cir::MissingFeatures::cxxabiUseARMGuardVarABI());
+  }
+
+  bool needsVTTParameter(clang::GlobalDecl gd) override;
+
+  void emitInstanceFunctionProlog(SourceLocation loc,
+                                  CIRGenFunction &cgf) override;
+
+  void emitCXXConstructors(const clang::CXXConstructorDecl *d) override;
+  void emitCXXStructor(clang::GlobalDecl gd) override;
+};
+
+} // namespace
+
+void CIRGenItaniumCXXABI::emitInstanceFunctionProlog(SourceLocation loc,
+                                                     CIRGenFunction &cgf) {
+  // Naked functions have no prolog.
+  if (cgf.curFuncDecl && cgf.curFuncDecl->hasAttr<NakedAttr>()) {
+    cgf.cgm.errorNYI(cgf.curFuncDecl->getLocation(),
+                     "emitInstanceFunctionProlog: Naked");
+  }
+
+  /// Initialize the 'this' slot. In the Itanium C++ ABI, no prologue
+  /// adjustments are required, because they are all handled by thunks.
+  setCXXABIThisValue(cgf, loadIncomingCXXThis(cgf));
+
+  /// Classic codegen has code here to initialize the 'vtt' slot if
+  // getStructorImplicitParamDecl(cgf) returns a non-null value, but in the
+  // current implementation (of classic codegen) it never does.
+  assert(!cir::MissingFeatures::cxxabiStructorImplicitParam());
+
+  /// If this is a function that the ABI specifies returns 'this', initialize
+  /// the return slot to this' at the start of the function.
+  ///
+  /// Unlike the setting of return types, this is done within the ABI
+  /// implementation instead of by clients of CIRGenCXXBI because:
+  /// 1) getThisValue is currently protected
+  /// 2) in theory, an ABI could implement 'this' returns some other way;
+  ///    HasThisReturn only specifies a contract, not the implementation
+  if (hasThisReturn(cgf.curGD)) {
+    cgf.cgm.errorNYI(cgf.curFuncDecl->getLocation(),
+                     "emitInstanceFunctionProlog: hasThisReturn");
+  }
+}
+
+void CIRGenItaniumCXXABI::emitCXXStructor(GlobalDecl gd) {
+  auto *md = cast<CXXMethodDecl>(gd.getDecl());
+  auto *cd = dyn_cast<CXXConstructorDecl>(md);
+
+  if (!cd) {
+    cgm.errorNYI(md->getSourceRange(), "CXCABI emit destructor");
+    return;
+  }
+
+  if (cgm.getCodeGenOpts().CXXCtorDtorAliases)
+    cgm.errorNYI(md->getSourceRange(), "Ctor/Dtor aliases");
+
+  auto fn = cgm.codegenCXXStructor(gd);
+
+  cgm.maybeSetTrivialComdat(*md, fn);
+}
+
+void CIRGenItaniumCXXABI::emitCXXConstructors(const CXXConstructorDecl *d) {
+  // Just make sure we're in sync with TargetCXXABI.
+  assert(cgm.getTarget().getCXXABI().hasConstructorVariants());
+
+  // The constructor used for constructing this as a base class;
+  // ignores virtual bases.
+  cgm.emitGlobal(GlobalDecl(d, Ctor_Base));
+
+  // The constructor used for constructing this as a complete class;
+  // constructs the virtual bases, then calls the base constructor.
+  if (!d->getParent()->isAbstract()) {
+    // We don't need to emit the complete ctro if the class is abstract.
+    cgm.emitGlobal(GlobalDecl(d, Ctor_Complete));
+  }
+}
+
+/// Return whether the given global decl needs a VTT (virtual table table)
+/// parameter, which it does if it's a base constructor or destructor with
+/// virtual bases.
+bool CIRGenItaniumCXXABI::needsVTTParameter(GlobalDecl gd) {
+  auto *md = cast<CXXMethodDecl>(gd.getDecl());
+
+  // We don't have any virtual bases, just return early.
+  if (!md->getParent()->getNumVBases())
+    return false;
+
+  // Check if we have a base constructor.
+  if (isa<CXXConstructorDecl>(md) && gd.getCtorType() == Ctor_Base)
+    return true;
+
+  // Check if we have a base destructor.
+  if (isa<CXXDestructorDecl>(md) && gd.getDtorType() == Dtor_Base)
+    return true;
+
+  return false;
+}
+
+CIRGenCXXABI *clang::CIRGen::CreateCIRGenItaniumCXXABI(CIRGenModule &cgm) {
+  switch (cgm.getASTContext().getCXXABIKind()) {
+  case TargetCXXABI::GenericItanium:
+  case TargetCXXABI::GenericAArch64:
+    return new CIRGenItaniumCXXABI(cgm);
+
+  case TargetCXXABI::AppleARM64:
+    // The general Itanium ABI will do until we implement something that
+    // requires special handling.
+    assert(!cir::MissingFeatures::cxxabiAppleARM64CXXABI());
+    return new CIRGenItaniumCXXABI(cgm);
+
+  default:
+    llvm_unreachable("bad or NYI ABI kind");
+  }
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 11c3bd4d7be8..434dd376208e 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclOpenACC.h"
 #include "clang/AST/GlobalDecl.h"
+#include "clang/AST/RecordLayout.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 #include "clang/CIR/Interfaces/CIROpInterfaces.h"
@@ -55,14 +56,6 @@ static CIRGenCXXABI *createCXXABI(CIRGenModule &cgm) {
   llvm_unreachable("invalid C++ ABI kind");
 }
 
-namespace clang::CIRGen {
-// TODO(cir): Implement target-specific CIRGenCXXABIs
-CIRGenCXXABI *CreateCIRGenItaniumCXXABI(CIRGenModule &cgm) {
-  assert(!cir::MissingFeatures::targetSpecificCXXABI());
-  return new CIRGenCXXABI(cgm);
-}
-} // namespace clang::CIRGen
-
 CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
                            clang::ASTContext &astContext,
                            const clang::CodeGenOptions &cgo,
@@ -110,6 +103,25 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
 
 CIRGenModule::~CIRGenModule() = default;
 
+/// FIXME: this could likely be a common helper and not necessarily related
+/// with codegen.
+/// Return the best known alignment for an unknown pointer to a
+/// particular class.
+CharUnits CIRGenModule::getClassPointerAlignment(const CXXRecordDecl *rd) {
+  if (!rd->hasDefinition())
+    return CharUnits::One(); // Hopefully won't be used anywhere.
+
+  auto &layout = astContext.getASTRecordLayout(rd);
+
+  // If the class is final, then we know that the pointer points to an
+  // object of that type and can use the full alignment.
+  if (rd->isEffectivelyFinal())
+    return layout.getAlignment();
+
+  // Otherwise, we have to assume it could be a subclass.
+  return layout.getNonVirtualAlignment();
+}
+
 CharUnits CIRGenModule::getNaturalTypeAlignment(QualType t,
                                                 LValueBaseInfo *baseInfo) {
   assert(!cir::MissingFeatures::opTBAA());
@@ -210,6 +222,102 @@ mlir::Location CIRGenModule::getLoc(SourceRange cRange) {
   return mlir::FusedLoc::get({begin, end}, metadata, builder.getContext());
 }
 
+mlir::Operation *
+CIRGenModule::getAddrOfGlobal(GlobalDecl gd, ForDefinition_t isForDefinition) {
+  const Decl *d = gd.getDecl();
+
+  if (isa<CXXConstructorDecl>(d) || isa<CXXDestructorDecl>(d))
+    return getAddrOfCXXStructor(gd, /*FnInfo=*/nullptr, /*FnType=*/nullptr,
+                                /*DontDefer=*/false, isForDefinition);
+
+  if (isa<CXXMethodDecl>(d)) {
+    const CIRGenFunctionInfo &fi =
+        getTypes().arrangeCXXMethodDeclaration(cast<CXXMethodDecl>(d));
+    cir::FuncType ty = getTypes().getFunctionType(fi);
+    return getAddrOfFunction(gd, ty, /*ForVTable=*/false, /*DontDefer=*/false,
+                             isForDefinition);
+  }
+
+  if (isa<FunctionDecl>(d)) {
+    const CIRGenFunctionInfo &fi = getTypes().arrangeGlobalDeclaration(gd);
+    cir::FuncType ty = getTypes().getFunctionType(fi);
+    return getAddrOfFunction(gd, ty, /*ForVTable=*/false, /*DontDefer=*/false,
+                             isForDefinition);
+  }
+
+  return getAddrOfGlobalVar(cast<VarDecl>(d), /*ty=*/nullptr, isForDefinition)
+      .getDefiningOp();
+}
+
+void CIRGenModule::emitGlobalDecl(const clang::GlobalDecl &d) {
+  // We call getAddrOfGlobal with isForDefinition set to ForDefinition in
+  // order to get a Value with exactly the type we need, not something that
+  // might have been created for another decl with the same mangled name but
+  // different type.
+  mlir::Operation *op = getAddrOfGlobal(d, ForDefinition);
+
+  // In case of different address spaces, we may still get a cast, even with
+  // IsForDefinition equal to ForDefinition. Query mangled names table to get
+  // GlobalValue.
+  if (!op)
+    op = getGlobalValue(getMangledName(d));
+
+  assert(op && "expected a valid global op");
+
+  // Check to see if we've already emitted this. This is necessary for a
+  // couple of reasons: first, decls can end up in deferred-decls queue
+  // multiple times, and second, decls can end up with definitions in unusual
+  // ways (e.g. by an extern inline function acquiring a strong function
+  // redefinition). Just ignore those cases.
+  // TODO: Not sure what to map this to for MLIR
+  mlir::Operation *globalValueOp = op;
+  if (auto gv = dyn_cast<cir::GetGlobalOp>(op))
+    globalValueOp =
+        mlir::SymbolTable::lookupSymbolIn(getModule(), gv.getNameAttr());
+
+  if (auto cirGlobalValue =
+          dyn_cast<cir::CIRGlobalValueInterface>(globalValueOp))
+    if (!cirGlobalValue.isDeclaration())
+      return;
+
+  // If this is OpenMP, check if it is legal to emit this global normally.
+  assert(!cir::MissingFeatures::openMP());
+
+  // Otherwise, emit the definition and move on to the next one.
+  emitGlobalDefinition(d, op);
+}
+
+void CIRGenModule::emitDeferred() {
+  // Emit code for any potentially referenced deferred decls. Since a previously
+  // unused static decl may become used during the generation of code for a
+  // static function, iterate until no changes are made.
+
+  assert(!cir::MissingFeatures::openMP());
+  assert(!cir::MissingFeatures::deferredVtables());
+  assert(!cir::MissingFeatures::cudaSupport());
+
+  // Stop if we're out of both deferred vtables and deferred declarations.
+  if (deferredDeclsToEmit.empty())
+    return;
+
+  // Grab the list of decls to emit. If emitGlobalDefinition schedules more
+  // work, it will not interfere with this.
+  std::vector<GlobalDecl> curDeclsToEmit;
+  curDeclsToEmit.swap(deferredDeclsToEmit);
+
+  for (const GlobalDecl &d : curDeclsToEmit) {
+    emitGlobalDecl(d);
+
+    // If we found out that we need to emit more decls, do that recursively.
+    // This has the advantage that the decls are emitted in a DFS and related
+    // ones are close together, which is convenient for testing.
+    if (!deferredDeclsToEmit.empty()) {
+      emitDeferred();
+      assert(deferredDeclsToEmit.empty());
+    }
+  }
+}
+
 void CIRGenModule::emitGlobal(clang::GlobalDecl gd) {
   if (const auto *cd = dyn_cast<clang::OpenACCConstructDecl>(gd.getDecl())) {
     emitGlobalOpenACCDecl(cd);
@@ -248,8 +356,33 @@ void CIRGenModule::emitGlobal(clang::GlobalDecl gd) {
     }
   }
 
-  // TODO(CIR): Defer emitting some global definitions until later
-  emitGlobalDefinition(gd);
+  // Defer code generation to first use when possible, e.g. if this is an inline
+  // function. If the global must always be emitted, do it eagerly if possible
+  // to benefit from cache locality. Deferring code generation is necessary to
+  // avoid adding initializers to external declarations.
+  if (mustBeEmitted(global) && mayBeEmittedEagerly(global)) {
+    // Emit the definition if it can't be deferred.
+    emitGlobalDefinition(gd);
+    return;
+  }
+
+  // If we're deferring emission of a C++ variable with an initializer, remember
+  // the order in which it appeared on the file.
+  assert(!cir::MissingFeatures::deferredCXXGlobalInit());
+
+  llvm::StringRef mangledName = getMangledName(gd);
+  if (getGlobalValue(mangledName) != nullptr) {
+    // The value has already been used and should therefore be emitted.
+    addDeferredDeclToEmit(gd);
+  } else if (mustBeEmitted(global)) {
+    // The value must be emitted, but cannot be emitted eagerly.
+    assert(!mayBeEmittedEagerly(global));
+    addDeferredDeclToEmit(gd);
+  } else {
+    // Otherwise, remember that we saw a deferred decl with this name. The first
+    // use of the mangled name will cause it to move into deferredDeclsToEmit.
+    deferredDecls[mangledName] = gd;
+  }
 }
 
 void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
@@ -276,6 +409,7 @@ void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
     cgf.generateCode(gd, funcOp, funcType);
   }
   curCGF = nullptr;
+  assert(!cir::MissingFeatures::opFuncAttributesForDefinition());
 }
 
 mlir::Operation *CIRGenModule::getGlobalValue(StringRef name) {
@@ -317,6 +451,36 @@ cir::GlobalOp CIRGenModule::createGlobalOp(CIRGenModule &cgm,
   return g;
 }
 
+void CIRGenModule::setCommonAttributes(GlobalDecl gd, mlir::Operation *gv) {
+  const Decl *d = gd.getDecl();
+  if (isa_and_nonnull<NamedDecl>(d))
+    setGVProperties(gv, dyn_cast<NamedDecl>(d));
+  assert(!cir::MissingFeatures::defaultVisibility());
+  assert(!cir::MissingFeatures::opGlobalUsedOrCompilerUsed());
+}
+
+void CIRGenModule::setNonAliasAttributes(GlobalDecl gd, mlir::Operation *op) {
+  setCommonAttributes(gd, op);
+
+  assert(!cir::MissingFeatures::opGlobalUsedOrCompilerUsed());
+  assert(!cir::MissingFeatures::opGlobalSection());
+  assert(!cir::MissingFeatures::opFuncCPUAndFeaturesAttributes());
+  assert(!cir::MissingFeatures::opFuncSection());
+
+  assert(!cir::MissingFeatures::setTargetAttributes());
+}
+
+static void setLinkageForGV(cir::GlobalOp &gv, const NamedDecl *nd) {
+  // Set linkage and visibility in case we never see a definition.
+  LinkageInfo lv = nd->getLinkageAndVisibility();
+  // Don't set internal linkage on declarations.
+  // "extern_weak" is overloaded in LLVM; we probably should have
+  // separate linkage types for this.
+  if (isExternallyVisible(lv.getLinkage()) &&
+      (nd->hasAttr<WeakAttr>() || nd->isWeakImported()))
+    gv.setLinkage(cir::GlobalLinkageKind::ExternalWeakLinkage);
+}
+
 /// If the specified mangled name is not in the module,
 /// create and return an mlir GlobalOp with the specified type (TODO(cir):
 /// address space).
@@ -380,6 +544,17 @@ CIRGenModule::getOrCreateCIRGlobal(StringRef mangledName, mlir::Type ty,
       CIRGenModule::createGlobalOp(*this, loc, mangledName, ty,
                                    /*insertPoint=*/entry.getOperation());
 
+  // This is the first use or definition of a mangled name.  If there is a
+  // deferred decl with this name, remember that we need to emit it at the end
+  // of the file.
+  auto ddi = deferredDecls.find(mangledName);
+  if (ddi != deferredDecls.end()) {
+    // Move the potentially referenced deferred decl to the DeferredDeclsToEmit
+    // list, and remove it from DeferredDecls (since we don't need it anymore).
+    addDeferredDeclToEmit(ddi->second);
+    deferredDecls.erase(ddi);
+  }
+
   // Handle things which are present even on external declarations.
   if (d) {
     if (langOpts.OpenMP && !langOpts.OpenMPSimd)
@@ -387,7 +562,8 @@ CIRGenModule::getOrCreateCIRGlobal(StringRef mangledName, mlir::Type ty,
 
     gv.setAlignmentAttr(getSize(astContext.getDeclAlign(d)));
     assert(!cir::MissingFeatures::opGlobalConstant());
-    assert(!cir::MissingFeatures::opGlobalLinkage());
+
+    setLinkageForGV(gv, d);
 
     if (d->getTLSKind())
       errorNYI(d->getSourceRange(), "thread local global variable");
@@ -555,8 +731,6 @@ void CIRGenModule::emitGlobalVarDefinition(const clang::VarDecl *vd,
     errorNYI(vd->getSourceRange(), "annotate global variable");
   }
 
-  assert(!cir::MissingFeatures::opGlobalLinkage());
-
   if (langOpts.CUDA) {
     errorNYI(vd->getSourceRange(), "CUDA global variable");
   }
@@ -577,6 +751,12 @@ void CIRGenModule::emitGlobalVarDefinition(const clang::VarDecl *vd,
   assert(!cir::MissingFeatures::opGlobalDLLImportExport());
   if (linkage == cir::GlobalLinkageKind::CommonLinkage)
     errorNYI(initExpr->getSourceRange(), "common linkage");
+
+  setNonAliasAttributes(vd, gv);
+
+  assert(!cir::MissingFeatures::opGlobalThreadLocal());
+
+  maybeSetTrivialComdat(*vd, gv);
 }
 
 void CIRGenModule::emitGlobalDefinition(clang::GlobalDecl gd,
@@ -590,7 +770,7 @@ void CIRGenModule::emitGlobalDefinition(clang::GlobalDecl gd,
       // Make sure to emit the definition(s) before we emit the thunks. This is
       // necessary for the generation of certain thunks.
       if (isa<CXXConstructorDecl>(method) || isa<CXXDestructorDecl>(method))
-        errorNYI(method->getSourceRange(), "C++ ctor/dtor");
+        abi->emitCXXStructor(gd);
       else if (fd->isMultiVersion())
         errorNYI(method->getSourceRange(), "multiversion functions");
       else
@@ -668,6 +848,20 @@ static bool shouldBeInCOMDAT(CIRGenModule &cgm, const Decl &d) {
   llvm_unreachable("No such linkage");
 }
 
+void CIRGenModule::maybeSetTrivialComdat(const Decl &d, mlir::Operation *op) {
+  if (!shouldBeInCOMDAT(*this, d))
+    return;
+  if (auto globalOp = dyn_cast_or_null<cir::GlobalOp>(op))
+    globalOp.setComdat(true);
+
+  assert(!cir::MissingFeatures::opFuncSetComdat());
+}
+
+void CIRGenModule::updateCompletedType(const TagDecl *td) {
+  // Make sure that this type is translated.
+  genTypes.updateCompletedType(td);
+}
+
 // TODO(CIR): this could be a common method between LLVM codegen.
 static bool isVarDeclStrongDefinition(const ASTContext &astContext,
                                       CIRGenModule &cgm, const VarDecl *vd,
@@ -830,10 +1024,10 @@ CIRGenModule::getCIRLinkageVarDefinition(const VarDecl *vd, bool isConstant) {
   return getCIRLinkageForDeclarator(vd, linkage, isConstant);
 }
 
-static cir::GlobalOp generateStringLiteral(mlir::Location loc,
-                                           mlir::TypedAttr c, CIRGenModule &cgm,
-                                           StringRef globalName,
-                                           CharUnits alignment) {
+static cir::GlobalOp
+generateStringLiteral(mlir::Location loc, mlir::TypedAttr c,
+                      cir::GlobalLinkageKind lt, CIRGenModule &cgm,
+                      StringRef globalName, CharUnits alignment) {
   assert(!cir::MissingFeatures::addressSpace());
 
   // Create a global variable for this string
@@ -843,7 +1037,8 @@ static cir::GlobalOp generateStringLiteral(mlir::Location loc,
 
   // Set up extra information and add to the module
   gv.setAlignmentAttr(cgm.getSize(alignment));
-  assert(!cir::MissingFeatures::opGlobalLinkage());
+  gv.setLinkageAttr(
+      cir::GlobalLinkageKindAttr::get(cgm.getBuilder().getContext(), lt));
   assert(!cir::MissingFeatures::opGlobalThreadLocal());
   assert(!cir::MissingFeatures::opGlobalUnnamedAddr());
   CIRGenModule::setInitializer(gv, c);
@@ -907,7 +1102,8 @@ cir::GlobalOp CIRGenModule::getGlobalForStringLiteral(const StringLiteral *s,
   mlir::Location loc = getLoc(s->getSourceRange());
   auto typedC = llvm::cast<mlir::TypedAttr>(c);
   cir::GlobalOp gv =
-      generateStringLiteral(loc, typedC, *this, uniqueName, alignment);
+      generateStringLiteral(loc, typedC, cir::GlobalLinkageKind::PrivateLinkage,
+                            *this, uniqueName, alignment);
   setDSOLocal(static_cast<mlir::Operation *>(gv));
 
   assert(!cir::MissingFeatures::sanitizers());
@@ -963,19 +1159,34 @@ void CIRGenModule::emitTopLevelDecl(Decl *decl) {
     emitGlobalOpenACCDecl(cast<OpenACCDeclareDecl>(decl));
     break;
   case Decl::Enum:
+  case Decl::Using:          // using X; [C++]
   case Decl::UsingDirective: // using namespace X; [C++]
   case Decl::Typedef:
   case Decl::TypeAlias: // using foo = bar; [C++11]
   case Decl::Record:
-  case Decl::CXXRecord:
     assert(!cir::MissingFeatures::generateDebugInfo());
     break;
 
+  // No code generation needed.
+  case Decl::UsingShadow:
+  case Decl::Empty:
+    break;
+
+  case Decl::CXXConstructor:
+    getCXXABI().emitCXXConstructors(cast<CXXConstructorDecl>(decl));
+    break;
+
   // C++ Decls
   case Decl::LinkageSpec:
   case Decl::Namespace:
     emitDeclContext(Decl::castToDeclContext(decl));
     break;
+
+  case Decl::ClassTemplateSpecialization:
+  case Decl::CXXRecord:
+    assert(!cir::MissingFeatures::generateDebugInfo());
+    assert(!cir::MissingFeatures::cxxRecordStaticMembers());
+    break;
   }
 }
 
@@ -985,6 +1196,34 @@ void CIRGenModule::setInitializer(cir::GlobalOp &op, mlir::Attribute value) {
   assert(!cir::MissingFeatures::opGlobalVisibility());
 }
 
+std::pair<cir::FuncType, cir::FuncOp> CIRGenModule::getAddrAndTypeOfCXXStructor(
+    GlobalDecl gd, const CIRGenFunctionInfo *fnInfo, cir::FuncType fnType,
+    bool dontDefer, ForDefinition_t isForDefinition) {
+  auto *md = cast<CXXMethodDecl>(gd.getDecl());
+
+  if (isa<CXXDestructorDecl>(md)) {
+    // Always alias equivalent complete destructors to base destructors in the
+    // MS ABI.
+    if (getTarget().getCXXABI().isMicrosoft() &&
+        gd.getDtorType() == Dtor_Complete &&
+        md->getParent()->getNumVBases() == 0)
+      errorNYI(md->getSourceRange(),
+               "getAddrAndTypeOfCXXStructor: MS ABI complete destructor");
+  }
+
+  if (!fnType) {
+    if (!fnInfo)
+      fnInfo = &getTypes().arrangeCXXStructorDeclaration(gd);
+    fnType = getTypes().getFunctionType(*fnInfo);
+  }
+
+  auto fn = getOrCreateCIRFunction(getMangledName(gd), fnType, gd,
+                                   /*ForVtable=*/false, dontDefer,
+                                   /*IsThunk=*/false, isForDefinition);
+
+  return {fnType, fn};
+}
+
 cir::FuncOp CIRGenModule::getAddrOfFunction(clang::GlobalDecl gd,
                                             mlir::Type funcType, bool forVTable,
                                             bool dontDefer,
@@ -1059,8 +1298,11 @@ StringRef CIRGenModule::getMangledName(GlobalDecl gd) {
   // Some ABIs don't have constructor variants. Make sure that base and complete
   // constructors get mangled the same.
   if (const auto *cd = dyn_cast<CXXConstructorDecl>(canonicalGd.getDecl())) {
-    errorNYI(cd->getSourceRange(), "getMangledName: C++ constructor");
-    return cast<NamedDecl>(gd.getDecl())->getIdentifier()->getName();
+    if (!getTarget().getCXXABI().hasConstructorVariants()) {
+      errorNYI(cd->getSourceRange(),
+               "getMangledName: C++ constructor without variants");
+      return cast<NamedDecl>(gd.getDecl())->getIdentifier()->getName();
+    }
   }
 
   // Keep the first result in the case of a mangling collision.
@@ -1083,12 +1325,83 @@ void CIRGenModule::emitTentativeDefinition(const VarDecl *d) {
   if (gv && !mlir::cast<cir::GlobalOp>(gv).isDeclaration())
     return;
 
-  assert(!cir::MissingFeatures::deferredDecls());
+  // If we have not seen a reference to this variable yet, place it into the
+  // deferred declarations table to be emitted if needed later.
+  if (!mustBeEmitted(d) && !gv) {
+    deferredDecls[mangledName] = d;
+    return;
+  }
 
   // The tentative definition is the only definition.
   emitGlobalVarDefinition(d);
 }
 
+bool CIRGenModule::mustBeEmitted(const ValueDecl *global) {
+  // Never defer when EmitAllDecls is specified.
+  if (langOpts.EmitAllDecls)
+    return true;
+
+  const auto *vd = dyn_cast<VarDecl>(global);
+  if (vd &&
+      ((codeGenOpts.KeepPersistentStorageVariables &&
+        (vd->getStorageDuration() == SD_Static ||
+         vd->getStorageDuration() == SD_Thread)) ||
+       (codeGenOpts.KeepStaticConsts && vd->getStorageDuration() == SD_Static &&
+        vd->getType().isConstQualified())))
+    return true;
+
+  return getASTContext().DeclMustBeEmitted(global);
+}
+
+bool CIRGenModule::mayBeEmittedEagerly(const ValueDecl *global) {
+  // In OpenMP 5.0 variables and function may be marked as
+  // device_type(host/nohost) and we should not emit them eagerly unless we sure
+  // that they must be emitted on the host/device. To be sure we need to have
+  // seen a declare target with an explicit mentioning of the function, we know
+  // we have if the level of the declare target attribute is -1. Note that we
+  // check somewhere else if we should emit this at all.
+  if (langOpts.OpenMP >= 50 && !langOpts.OpenMPSimd) {
+    std::optional<OMPDeclareTargetDeclAttr *> activeAttr =
+        OMPDeclareTargetDeclAttr::getActiveAttr(global);
+    if (!activeAttr || (*activeAttr)->getLevel() != (unsigned)-1)
+      return false;
+  }
+
+  const auto *fd = dyn_cast<FunctionDecl>(global);
+  if (fd) {
+    // Implicit template instantiations may change linkage if they are later
+    // explicitly instantiated, so they should not be emitted eagerly.
+    if (fd->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
+      return false;
+    // Defer until all versions have been semantically checked.
+    if (fd->hasAttr<TargetVersionAttr>() && !fd->isMultiVersion())
+      return false;
+    if (langOpts.SYCLIsDevice) {
+      errorNYI(fd->getSourceRange(), "mayBeEmittedEagerly: SYCL");
+      return false;
+    }
+  }
+  const auto *vd = dyn_cast<VarDecl>(global);
+  if (vd)
+    if (astContext.getInlineVariableDefinitionKind(vd) ==
+        ASTContext::InlineVariableDefinitionKind::WeakUnknown)
+      // A definition of an inline constexpr static data member may change
+      // linkage later if it's redeclared outside the class.
+      return false;
+
+  // If OpenMP is enabled and threadprivates must be generated like TLS, delay
+  // codegen for global variables, because they may be marked as threadprivate.
+  if (langOpts.OpenMP && langOpts.OpenMPUseTLS &&
+      astContext.getTargetInfo().isTLSSupported() && isa<VarDecl>(global) &&
+      !global->getType().isConstantStorage(astContext, false, false) &&
+      !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(global))
+    return false;
+
+  assert((fd || vd) &&
+         "Only FunctionDecl and VarDecl should hit this path so far.");
+  return true;
+}
+
 static bool shouldAssumeDSOLocal(const CIRGenModule &cgm,
                                  cir::CIRGlobalValueInterface gv) {
   if (gv.hasLocalLinkage())
@@ -1280,6 +1593,56 @@ cir::FuncOp CIRGenModule::getOrCreateCIRFunction(
   cir::FuncOp funcOp = createCIRFunction(
       invalidLoc ? theModule->getLoc() : getLoc(funcDecl->getSourceRange()),
       mangledName, mlir::cast<cir::FuncType>(funcType), funcDecl);
+
+  // 'dontDefer' actually means don't move this to the deferredDeclsToEmit list.
+  if (dontDefer) {
+    // TODO(cir): This assertion will need an additional condition when we
+    // support incomplete functions.
+    assert(funcOp.getFunctionType() == funcType);
+    return funcOp;
+  }
+
+  // All MSVC dtors other than the base dtor are linkonce_odr and delegate to
+  // each other bottoming out wiht the base dtor. Therefore we emit non-base
+  // dtors on usage, even if there is no dtor definition in the TU.
+  if (isa_and_nonnull<CXXDestructorDecl>(d))
+    errorNYI(d->getSourceRange(), "getOrCreateCIRFunction: dtor");
+
+  // This is the first use or definition of a mangled name. If there is a
+  // deferred decl with this name, remember that we need to emit it at the end
+  // of the file.
+  auto ddi = deferredDecls.find(mangledName);
+  if (ddi != deferredDecls.end()) {
+    // Move the potentially referenced deferred decl to the
+    // DeferredDeclsToEmit list, and remove it from DeferredDecls (since we
+    // don't need it anymore).
+    addDeferredDeclToEmit(ddi->second);
+    deferredDecls.erase(ddi);
+
+    // Otherwise, there are cases we have to worry about where we're using a
+    // declaration for which we must emit a definition but where we might not
+    // find a top-level definition.
+    //   - member functions defined inline in their classes
+    //   - friend functions defined inline in some class
+    //   - special member functions with implicit definitions
+    // If we ever change our AST traversal to walk into class methods, this
+    // will be unnecessary.
+    //
+    // We also don't emit a definition for a function if it's going to be an
+    // entry in a vtable, unless it's already marked as used.
+  } else if (getLangOpts().CPlusPlus && d) {
+    // Look for a declaration that's lexically in a record.
+    for (const auto *fd = cast<FunctionDecl>(d)->getMostRecentDecl(); fd;
+         fd = fd->getPreviousDecl()) {
+      if (isa<CXXRecordDecl>(fd->getLexicalDeclContext())) {
+        if (fd->doesThisDeclarationHaveABody()) {
+          addDeferredDeclToEmit(gd.getWithDecl(fd));
+          break;
+        }
+      }
+    }
+  }
+
   return funcOp;
 }
 
@@ -1356,6 +1719,13 @@ CIRGenModule::getGlobalVisibilityAttrFromDecl(const Decl *decl) {
   return cirVisibility;
 }
 
+void CIRGenModule::release() {
+  emitDeferred();
+
+  // There's a lot of code that is not implemented yet.
+  assert(!cir::MissingFeatures::cgmRelease());
+}
+
 mlir::Type CIRGenModule::convertType(QualType type) {
   return genTypes.convertType(type);
 }
@@ -1367,6 +1737,33 @@ bool CIRGenModule::verifyModule() const {
   return mlir::verify(theModule).succeeded();
 }
 
+// TODO(cir): this can be shared with LLVM codegen.
+CharUnits CIRGenModule::computeNonVirtualBaseClassOffset(
+    const CXXRecordDecl *derivedClass,
+    llvm::iterator_range<CastExpr::path_const_iterator> path) {
+  CharUnits offset = CharUnits::Zero();
+
+  const ASTContext &astContext = getASTContext();
+  const CXXRecordDecl *rd = derivedClass;
+
+  for (const CXXBaseSpecifier *base : path) {
+    assert(!base->isVirtual() && "Should not see virtual bases here!");
+
+    // Get the layout.
+    const ASTRecordLayout &layout = astContext.getASTRecordLayout(rd);
+
+    const auto *baseDecl = cast<CXXRecordDecl>(
+        base->getType()->castAs<clang::RecordType>()->getDecl());
+
+    // Add the offset.
+    offset += layout.getBaseClassOffset(baseDecl);
+
+    rd = baseDecl;
+  }
+
+  return offset;
+}
+
 DiagnosticBuilder CIRGenModule::errorNYI(SourceLocation loc,
                                          llvm::StringRef feature) {
   unsigned diagID = diags.getCustomDiagID(
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.h
index 697c2450ec38..03606dba200f 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -113,8 +113,21 @@ class CIRGenModule : public CIRGenTypeCache {
 
   mlir::Operation *lastGlobalOp = nullptr;
 
+  llvm::DenseMap<const Decl *, cir::GlobalOp> staticLocalDeclMap;
+
   mlir::Operation *getGlobalValue(llvm::StringRef ref);
 
+  cir::GlobalOp getStaticLocalDeclAddress(const VarDecl *d) {
+    return staticLocalDeclMap[d];
+  }
+
+  void setStaticLocalDeclAddress(const VarDecl *d, cir::GlobalOp c) {
+    staticLocalDeclMap[d] = c;
+  }
+
+  cir::GlobalOp getOrCreateStaticVarDecl(const VarDecl &d,
+                                         cir::GlobalLinkageKind linkage);
+
   /// If the specified mangled name is not in the module, create and return an
   /// mlir::GlobalOp value
   cir::GlobalOp getOrCreateCIRGlobal(llvm::StringRef mangledName, mlir::Type ty,
@@ -141,6 +154,10 @@ class CIRGenModule : public CIRGenTypeCache {
   getAddrOfGlobalVar(const VarDecl *d, mlir::Type ty = {},
                      ForDefinition_t isForDefinition = NotForDefinition);
 
+  CharUnits computeNonVirtualBaseClassOffset(
+      const CXXRecordDecl *derivedClass,
+      llvm::iterator_range<CastExpr::path_const_iterator> path);
+
   /// Return a constant array for the given string.
   mlir::Attribute getConstantArrayFromStringLiteral(const StringLiteral *e);
 
@@ -149,6 +166,12 @@ class CIRGenModule : public CIRGenTypeCache {
   cir::GlobalOp getGlobalForStringLiteral(const StringLiteral *s,
                                           llvm::StringRef name = ".str");
 
+  /// Set attributes which are common to any form of a global definition (alias,
+  /// Objective-C method, function, global variable).
+  ///
+  /// NOTE: This should only be called for definitions.
+  void setCommonAttributes(GlobalDecl gd, mlir::Operation *op);
+
   const TargetCIRGenInfo &getTargetCIRGenInfo();
 
   /// Helpers to convert the presumed location of Clang's SourceLocation to an
@@ -156,13 +179,55 @@ class CIRGenModule : public CIRGenTypeCache {
   mlir::Location getLoc(clang::SourceLocation cLoc);
   mlir::Location getLoc(clang::SourceRange cRange);
 
+  /// Return the best known alignment for an unknown pointer to a
+  /// particular class.
+  clang::CharUnits getClassPointerAlignment(const clang::CXXRecordDecl *rd);
+
   /// FIXME: this could likely be a common helper and not necessarily related
   /// with codegen.
   clang::CharUnits getNaturalTypeAlignment(clang::QualType t,
                                            LValueBaseInfo *baseInfo);
 
+  cir::FuncOp
+  getAddrOfCXXStructor(clang::GlobalDecl gd,
+                       const CIRGenFunctionInfo *fnInfo = nullptr,
+                       cir::FuncType fnType = nullptr, bool dontDefer = false,
+                       ForDefinition_t isForDefinition = NotForDefinition) {
+    return getAddrAndTypeOfCXXStructor(gd, fnInfo, fnType, dontDefer,
+                                       isForDefinition)
+        .second;
+  }
+
+  std::pair<cir::FuncType, cir::FuncOp> getAddrAndTypeOfCXXStructor(
+      clang::GlobalDecl gd, const CIRGenFunctionInfo *fnInfo = nullptr,
+      cir::FuncType fnType = nullptr, bool dontDefer = false,
+      ForDefinition_t isForDefinition = NotForDefinition);
+
+  /// This contains all the decls which have definitions but which are deferred
+  /// for emission and therefore should only be output if they are actually
+  /// used. If a decl is in this, then it is known to have not been referenced
+  /// yet.
+  std::map<llvm::StringRef, clang::GlobalDecl> deferredDecls;
+
+  // This is a list of deferred decls which we have seen that *are* actually
+  // referenced. These get code generated when the module is done.
+  std::vector<clang::GlobalDecl> deferredDeclsToEmit;
+  void addDeferredDeclToEmit(clang::GlobalDecl GD) {
+    deferredDeclsToEmit.emplace_back(GD);
+  }
+
   void emitTopLevelDecl(clang::Decl *decl);
 
+  /// Determine whether the definition must be emitted; if this returns \c
+  /// false, the definition can be emitted lazily if it's used.
+  bool mustBeEmitted(const clang::ValueDecl *d);
+
+  /// Determine whether the definition can be emitted eagerly, or should be
+  /// delayed until the end of the translation unit. This is relevant for
+  /// definitions whose linkage can change, e.g. implicit function
+  /// instantiations which may later be explicitly instantiated.
+  bool mayBeEmittedEagerly(const clang::ValueDecl *d);
+
   bool verifyModule() const;
 
   /// Return the address of the given function. If funcType is non-null, then
@@ -173,6 +238,10 @@ class CIRGenModule : public CIRGenTypeCache {
                     bool forVTable = false, bool dontDefer = false,
                     ForDefinition_t isForDefinition = NotForDefinition);
 
+  mlir::Operation *
+  getAddrOfGlobal(clang::GlobalDecl gd,
+                  ForDefinition_t isForDefinition = NotForDefinition);
+
   /// Emit code for a single global function or variable declaration. Forward
   /// declarations are emitted lazily.
   void emitGlobal(clang::GlobalDecl gd);
@@ -208,7 +277,16 @@ class CIRGenModule : public CIRGenTypeCache {
 
   void emitTentativeDefinition(const VarDecl *d);
 
+  // Make sure that this type is translated.
+  void updateCompletedType(const clang::TagDecl *td);
+
+  // Produce code for this constructor/destructor. This method doesn't try to
+  // apply any ABI rules about which other constructors/destructors are needed
+  // or if they are alias to each other.
+  cir::FuncOp codegenCXXStructor(clang::GlobalDecl gd);
+
   bool supportsCOMDAT() const;
+  void maybeSetTrivialComdat(const clang::Decl &d, mlir::Operation *op);
 
   static void setInitializer(cir::GlobalOp &op, mlir::Attribute value);
 
@@ -227,8 +305,17 @@ class CIRGenModule : public CIRGenTypeCache {
     return builder.getSizeFromCharUnits(size);
   }
 
+  /// Emit any needed decls for which code generation was deferred.
+  void emitDeferred();
+
+  /// Helper for `emitDeferred` to apply actual codegen.
+  void emitGlobalDecl(const clang::GlobalDecl &d);
+
   const llvm::Triple &getTriple() const { return target.getTriple(); }
 
+  // Finalize CIR code generation.
+  void release();
+
   /// -------
   /// Visibility and Linkage
   /// -------
@@ -285,6 +372,8 @@ class CIRGenModule : public CIRGenTypeCache {
   // An ordered map of canonical GlobalDecls to their mangled names.
   llvm::MapVector<clang::GlobalDecl, llvm::StringRef> mangledDeclNames;
   llvm::StringMap<clang::GlobalDecl, llvm::BumpPtrAllocator> manglings;
+
+  void setNonAliasAttributes(GlobalDecl gd, mlir::Operation *op);
 };
 } // namespace CIRGen
 
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
index 8b61d3fae3ad..899e91574e91 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
@@ -14,6 +14,8 @@
 
 #include "CIRGenFunction.h"
 
+#include "clang/AST/ExprCXX.h"
+
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -188,7 +190,7 @@ class OpenACCClauseCIREmitter final
   struct DataOperandInfo {
     mlir::Location beginLoc;
     mlir::Value varValue;
-    llvm::StringRef name;
+    std::string name;
     llvm::SmallVector<mlir::Value> bounds;
   };
 
@@ -226,6 +228,10 @@ class OpenACCClauseCIREmitter final
     mlir::Location exprLoc = cgf.cgm.getLoc(curVarExpr->getBeginLoc());
     llvm::SmallVector<mlir::Value> bounds;
 
+    std::string exprString;
+    llvm::raw_string_ostream os(exprString);
+    e->printPretty(os, nullptr, cgf.getContext().getPrintingPolicy());
+
     // Assemble the list of bounds.
     while (isa<ArraySectionExpr, ArraySubscriptExpr>(curVarExpr)) {
       mlir::Location boundLoc = cgf.cgm.getLoc(curVarExpr->getBeginLoc());
@@ -267,20 +273,16 @@ class OpenACCClauseCIREmitter final
       bounds.push_back(createBound(boundLoc, lowerBound, upperBound, extent));
     }
 
-    // TODO: OpenACC: if this is a member expr, emit the VarPtrPtr correctly.
-    if (isa<MemberExpr>(curVarExpr)) {
-      cgf.cgm.errorNYI(curVarExpr->getSourceRange(),
-                       "OpenACC Data clause member expr");
-      return {exprLoc, {}, {}, std::move(bounds)};
-    }
+    if (const auto *memExpr = dyn_cast<MemberExpr>(curVarExpr))
+      return {exprLoc, cgf.emitMemberExpr(memExpr).getPointer(), exprString,
+              std::move(bounds)};
 
     // Sema has made sure that only 4 types of things can get here, array
     // subscript, array section, member expr, or DRE to a var decl (or the
     // former 3 wrapping a var-decl), so we should be able to assume this is
     // right.
     const auto *dre = cast<DeclRefExpr>(curVarExpr);
-    const auto *vd = cast<VarDecl>(dre->getFoundDecl()->getCanonicalDecl());
-    return {exprLoc, cgf.emitDeclRefLValue(dre).getPointer(), vd->getName(),
+    return {exprLoc, cgf.emitDeclRefLValue(dre).getPointer(), exprString,
             std::move(bounds)};
   }
 
@@ -303,9 +305,19 @@ class OpenACCClauseCIREmitter final
     {
       mlir::OpBuilder::InsertionGuard guardCase(builder);
       builder.setInsertionPointAfter(operation);
-      afterOp = builder.create<AfterOpTy>(opInfo.beginLoc, beforeOp.getResult(),
-                                          opInfo.varValue, structured, implicit,
-                                          opInfo.name, opInfo.bounds);
+
+      if constexpr (std::is_same_v<AfterOpTy, mlir::acc::DeleteOp> ||
+                    std::is_same_v<AfterOpTy, mlir::acc::DetachOp>) {
+        // Detach/Delete ops don't have the variable reference here, so they
+        // take 1 fewer argument to their build function.
+        afterOp = builder.create<AfterOpTy>(
+            opInfo.beginLoc, beforeOp.getResult(), structured, implicit,
+            opInfo.name, opInfo.bounds);
+      } else {
+        afterOp = builder.create<AfterOpTy>(
+            opInfo.beginLoc, beforeOp.getResult(), opInfo.varValue, structured,
+            implicit, opInfo.name, opInfo.bounds);
+      }
     }
 
     // Set the 'rest' of the info for both operations.
@@ -317,6 +329,21 @@ class OpenACCClauseCIREmitter final
     dataOperands.push_back(afterOp.getOperation());
   }
 
+  template <typename BeforeOpTy>
+  void addDataOperand(const Expr *varOperand, mlir::acc::DataClause dataClause,
+                      bool structured, bool implicit) {
+    DataOperandInfo opInfo = getDataOperandInfo(dirKind, varOperand);
+    auto beforeOp =
+        builder.create<BeforeOpTy>(opInfo.beginLoc, opInfo.varValue, structured,
+                                   implicit, opInfo.name, opInfo.bounds);
+    operation.getDataClauseOperandsMutable().append(beforeOp.getResult());
+
+    // Set the 'rest' of the info for the operation.
+    beforeOp.setDataClause(dataClause);
+    // Make sure we record these, so 'async' values can be updated later.
+    dataOperands.push_back(beforeOp.getOperation());
+  }
+
   // Helper function that covers for the fact that we don't have this function
   // on all operation types.
   mlir::ArrayAttr getAsyncOnlyAttr() {
@@ -548,7 +575,8 @@ class OpenACCClauseCIREmitter final
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
                                mlir::acc::KernelsOp, mlir::acc::InitOp,
                                mlir::acc::ShutdownOp, mlir::acc::SetOp,
-                               mlir::acc::DataOp, mlir::acc::WaitOp>) {
+                               mlir::acc::DataOp, mlir::acc::WaitOp,
+                               mlir::acc::HostDataOp>) {
       operation.getIfCondMutable().append(
           createCondition(clause.getConditionExpr()));
     } else if constexpr (isCombinedType<OpTy>) {
@@ -564,6 +592,17 @@ class OpenACCClauseCIREmitter final
     }
   }
 
+  void VisitIfPresentClause(const OpenACCIfPresentClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp>) {
+      operation.setIfPresent(true);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
+      // Last unimplemented one here, so just put it in this way instead.
+      return clauseNotImplemented(clause);
+    } else {
+      llvm_unreachable("unknown construct kind in VisitIfPresentClause");
+    }
+  }
+
   void VisitDeviceNumClause(const OpenACCDeviceNumClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::InitOp, mlir::acc::ShutdownOp,
                                mlir::acc::SetOp>) {
@@ -789,6 +828,81 @@ class OpenACCClauseCIREmitter final
       return clauseNotImplemented(clause);
     }
   }
+
+  void VisitUseDeviceClause(const OpenACCUseDeviceClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::UseDeviceOp>(
+            var, mlir::acc::DataClause::acc_use_device,
+            /*structured=*/true, /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitUseDeviceClause");
+    }
+  }
+
+  void VisitDevicePtrClause(const OpenACCDevicePtrClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::DevicePtrOp>(
+            var, mlir::acc::DataClause::acc_deviceptr, /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. data, declare remain.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitNoCreateClause(const OpenACCNoCreateClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::NoCreateOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_no_create, /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. data remains.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitPresentClause(const OpenACCPresentClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::PresentOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_present, /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. data & declare remain.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitAttachClause(const OpenACCAttachClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::AttachOp, mlir::acc::DetachOp>(
+            var, mlir::acc::DataClause::acc_attach, /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. data, enter data remain.
+      return clauseNotImplemented(clause);
+    }
+  }
 };
 
 template <typename OpTy>
@@ -824,6 +938,7 @@ EXPL_SPEC(mlir::acc::InitOp)
 EXPL_SPEC(mlir::acc::ShutdownOp)
 EXPL_SPEC(mlir::acc::SetOp)
 EXPL_SPEC(mlir::acc::WaitOp)
+EXPL_SPEC(mlir::acc::HostDataOp)
 #undef EXPL_SPEC
 
 template <typename ComputeOp, typename LoopOp>
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
index 2ece85b8aa0a..ac8832b8c9b2 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
@@ -29,10 +29,18 @@ class CIRGenRecordLayout {
   /// as a complete object.
   cir::RecordType completeObjectType;
 
+  /// The CIR type for the non-virtual part of this record layout; used when
+  /// laying it out as a base subobject.
+  cir::RecordType baseSubobjectType;
+
   /// Map from (non-bit-field) record field to the corresponding cir record type
   /// field no. This info is populated by the record builder.
   llvm::DenseMap<const clang::FieldDecl *, unsigned> fieldIdxMap;
 
+  // FIXME: Maybe we could use CXXBaseSpecifier as the key and use a single map
+  // for both virtual and non-virtual bases.
+  llvm::DenseMap<const clang::CXXRecordDecl *, unsigned> nonVirtualBases;
+
   /// False if any direct or indirect subobject of this class, when considered
   /// as a complete object, requires a non-zero bitpattern when
   /// zero-initialized.
@@ -45,9 +53,11 @@ class CIRGenRecordLayout {
   unsigned zeroInitializableAsBase : 1;
 
 public:
-  CIRGenRecordLayout(cir::RecordType completeObjectType, bool zeroInitializable,
+  CIRGenRecordLayout(cir::RecordType completeObjectType,
+                     cir::RecordType baseSubobjectType, bool zeroInitializable,
                      bool zeroInitializableAsBase)
       : completeObjectType(completeObjectType),
+        baseSubobjectType(baseSubobjectType),
         zeroInitializable(zeroInitializable),
         zeroInitializableAsBase(zeroInitializableAsBase) {}
 
@@ -55,6 +65,10 @@ class CIRGenRecordLayout {
   /// this record.
   cir::RecordType getCIRType() const { return completeObjectType; }
 
+  /// Return the "base subobject" LLVM type associated with
+  /// this record.
+  cir::RecordType getBaseSubobjectCIRType() const { return baseSubobjectType; }
+
   /// Return cir::RecordType element number that corresponds to the field FD.
   unsigned getCIRFieldNo(const clang::FieldDecl *fd) const {
     fd = fd->getCanonicalDecl();
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
index 53aa0aee36fc..0aeef7fd89ae 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
@@ -40,15 +40,18 @@ struct CIRRecordLowering final {
   // member type that ensures correct rounding.
   struct MemberInfo final {
     CharUnits offset;
-    enum class InfoKind { Field } kind;
+    enum class InfoKind { Field, Base } kind;
     mlir::Type data;
     union {
       const FieldDecl *fieldDecl;
-      // CXXRecordDecl will be used here when base types are supported.
+      const CXXRecordDecl *cxxRecordDecl;
     };
     MemberInfo(CharUnits offset, InfoKind kind, mlir::Type data,
                const FieldDecl *fieldDecl = nullptr)
-        : offset(offset), kind(kind), data(data), fieldDecl(fieldDecl) {};
+        : offset{offset}, kind{kind}, data{data}, fieldDecl{fieldDecl} {}
+    MemberInfo(CharUnits offset, InfoKind kind, mlir::Type data,
+               const CXXRecordDecl *rd)
+        : offset{offset}, kind{kind}, data{data}, cxxRecordDecl{rd} {}
     // MemberInfos are sorted so we define a < operator.
     bool operator<(const MemberInfo &other) const {
       return offset < other.offset;
@@ -71,6 +74,8 @@ struct CIRRecordLowering final {
   /// Inserts padding everywhere it's needed.
   void insertPadding();
 
+  void accumulateBases(const CXXRecordDecl *cxxRecordDecl);
+  void accumulateVPtrs();
   void accumulateFields();
 
   CharUnits bitsToCharUnits(uint64_t bitOffset) {
@@ -89,6 +94,9 @@ struct CIRRecordLowering final {
   bool isZeroInitializable(const FieldDecl *fd) {
     return cirGenTypes.isZeroInitializable(fd->getType());
   }
+  bool isZeroInitializable(const RecordDecl *rd) {
+    return cirGenTypes.isZeroInitializable(rd);
+  }
 
   /// Wraps cir::IntType with some implicit arguments.
   mlir::Type getUIntNType(uint64_t numBits) {
@@ -112,6 +120,11 @@ struct CIRRecordLowering final {
                : cir::ArrayType::get(type, numberOfChars.getQuantity());
   }
 
+  // Gets the CIR BaseSubobject type from a CXXRecordDecl.
+  mlir::Type getStorageType(const CXXRecordDecl *RD) {
+    return cirGenTypes.getCIRGenRecordLayout(RD).getBaseSubobjectCIRType();
+  }
+
   mlir::Type getStorageType(const FieldDecl *fieldDecl) {
     mlir::Type type = cirGenTypes.convertTypeForMem(fieldDecl->getType());
     if (fieldDecl->isBitField()) {
@@ -145,6 +158,7 @@ struct CIRRecordLowering final {
   // Output fields, consumed by CIRGenTypes::computeRecordLayout
   llvm::SmallVector<mlir::Type, 16> fieldTypes;
   llvm::DenseMap<const FieldDecl *, unsigned> fieldIdxMap;
+  llvm::DenseMap<const CXXRecordDecl *, unsigned> nonVirtualBases;
   cir::CIRDataLayout dataLayout;
 
   LLVM_PREFERRED_TYPE(bool)
@@ -179,24 +193,20 @@ void CIRRecordLowering::lower() {
     return;
   }
 
-  assert(!cir::MissingFeatures::cxxSupport());
-
+  assert(!cir::MissingFeatures::recordLayoutVirtualBases());
   CharUnits size = astRecordLayout.getSize();
 
   accumulateFields();
 
   if (const auto *cxxRecordDecl = dyn_cast<CXXRecordDecl>(recordDecl)) {
-    if (cxxRecordDecl->getNumBases() > 0) {
-      CIRGenModule &cgm = cirGenTypes.getCGModule();
-      cgm.errorNYI(recordDecl->getSourceRange(),
-                   "CIRRecordLowering::lower: derived CXXRecordDecl");
-      return;
-    }
+    accumulateVPtrs();
+    accumulateBases(cxxRecordDecl);
     if (members.empty()) {
       appendPaddingBytes(size);
       assert(!cir::MissingFeatures::bitfields());
       return;
     }
+    assert(!cir::MissingFeatures::recordLayoutVirtualBases());
   }
 
   llvm::stable_sort(members);
@@ -223,8 +233,10 @@ void CIRRecordLowering::fillOutputFields() {
             fieldTypes.size() - 1;
       // A field without storage must be a bitfield.
       assert(!cir::MissingFeatures::bitfields());
+    } else if (member.kind == MemberInfo::InfoKind::Base) {
+      nonVirtualBases[member.cxxRecordDecl] = fieldTypes.size() - 1;
     }
-    assert(!cir::MissingFeatures::cxxSupport());
+    assert(!cir::MissingFeatures::recordLayoutVirtualBases());
   }
 }
 
@@ -254,9 +266,14 @@ void CIRRecordLowering::calculateZeroInit() {
         continue;
       zeroInitializable = zeroInitializableAsBase = false;
       return;
+    } else if (member.kind == MemberInfo::InfoKind::Base) {
+      if (isZeroInitializable(member.cxxRecordDecl))
+        continue;
+      zeroInitializable = false;
+      if (member.kind == MemberInfo::InfoKind::Base)
+        zeroInitializableAsBase = false;
     }
-    // TODO(cir): handle base types
-    assert(!cir::MissingFeatures::cxxSupport());
+    assert(!cir::MissingFeatures::recordLayoutVirtualBases());
   }
 }
 
@@ -317,6 +334,27 @@ CIRGenTypes::computeRecordLayout(const RecordDecl *rd, cir::RecordType *ty) {
   lowering.lower();
 
   // If we're in C++, compute the base subobject type.
+  cir::RecordType baseTy;
+  if (llvm::isa<CXXRecordDecl>(rd) && !rd->isUnion() &&
+      !rd->hasAttr<FinalAttr>()) {
+    baseTy = *ty;
+    if (lowering.astRecordLayout.getNonVirtualSize() !=
+        lowering.astRecordLayout.getSize()) {
+      CIRRecordLowering baseLowering(*this, rd, /*Packed=*/lowering.packed);
+      baseLowering.lower();
+      std::string baseIdentifier = getRecordTypeName(rd, ".base");
+      baseTy =
+          builder.getCompleteRecordTy(baseLowering.fieldTypes, baseIdentifier,
+                                      baseLowering.packed, baseLowering.padded);
+      // TODO(cir): add something like addRecordTypeName
+
+      // BaseTy and Ty must agree on their packedness for getCIRFieldNo to work
+      // on both of them with the same index.
+      assert(lowering.packed == baseLowering.packed &&
+             "Non-virtual and complete types must agree on packedness");
+    }
+  }
+
   if (llvm::isa<CXXRecordDecl>(rd) && !rd->isUnion() &&
       !rd->hasAttr<FinalAttr>()) {
     if (lowering.astRecordLayout.getNonVirtualSize() !=
@@ -332,10 +370,13 @@ CIRGenTypes::computeRecordLayout(const RecordDecl *rd, cir::RecordType *ty) {
   ty->complete(lowering.fieldTypes, lowering.packed, lowering.padded);
 
   auto rl = std::make_unique<CIRGenRecordLayout>(
-      ty ? *ty : cir::RecordType(), (bool)lowering.zeroInitializable,
-      (bool)lowering.zeroInitializableAsBase);
+      ty ? *ty : cir::RecordType{}, baseTy ? baseTy : cir::RecordType{},
+      (bool)lowering.zeroInitializable, (bool)lowering.zeroInitializableAsBase);
 
   assert(!cir::MissingFeatures::recordZeroInit());
+
+  rl->nonVirtualBases.swap(lowering.nonVirtualBases);
+
   assert(!cir::MissingFeatures::cxxSupport());
   assert(!cir::MissingFeatures::bitfields());
 
@@ -415,3 +456,38 @@ void CIRRecordLowering::lowerUnion() {
   if (layoutSize % getAlignment(storageType))
     packed = true;
 }
+
+void CIRRecordLowering::accumulateBases(const CXXRecordDecl *cxxRecordDecl) {
+  // If we've got a primary virtual base, we need to add it with the bases.
+  if (astRecordLayout.isPrimaryBaseVirtual()) {
+    cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                       "accumulateBases: primary virtual base");
+  }
+
+  // Accumulate the non-virtual bases.
+  for ([[maybe_unused]] const auto &base : cxxRecordDecl->bases()) {
+    if (base.isVirtual()) {
+      cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                         "accumulateBases: virtual base");
+      continue;
+    }
+    // Bases can be zero-sized even if not technically empty if they
+    // contain only a trailing array member.
+    const CXXRecordDecl *baseDecl = base.getType()->getAsCXXRecordDecl();
+    if (!baseDecl->isEmpty() &&
+        !astContext.getASTRecordLayout(baseDecl).getNonVirtualSize().isZero()) {
+      members.push_back(MemberInfo(astRecordLayout.getBaseClassOffset(baseDecl),
+                                   MemberInfo::InfoKind::Base,
+                                   getStorageType(baseDecl), baseDecl));
+    }
+  }
+}
+
+void CIRRecordLowering::accumulateVPtrs() {
+  if (astRecordLayout.hasOwnVFPtr())
+    cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                       "accumulateVPtrs: hasOwnVFPtr");
+  if (astRecordLayout.hasOwnVBPtr())
+    cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
+                                       "accumulateVPtrs: hasOwnVBPtr");
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index d922ca0c74d5..1feefa55eb27 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -102,6 +102,8 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct(
 
     emitOpenACCClauses(computeOp, loopOp, dirKind, dirLoc, clauses);
 
+    updateLoopOpParallelism(loopOp, /*isOrphan=*/false, dirKind);
+
     builder.create<TermOp>(end);
   }
 
@@ -235,6 +237,17 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCCombinedConstruct(
     llvm_unreachable("invalid compute construct kind");
   }
 }
+
+mlir::LogicalResult CIRGenFunction::emitOpenACCHostDataConstruct(
+    const OpenACCHostDataConstruct &s) {
+  mlir::Location start = getLoc(s.getSourceRange().getBegin());
+  mlir::Location end = getLoc(s.getSourceRange().getEnd());
+
+  return emitOpenACCOpAssociatedStmt<HostDataOp, mlir::acc::TerminatorOp>(
+      start, end, s.getDirectiveKind(), s.getDirectiveLoc(), s.clauses(),
+      s.getStructuredBlock());
+}
+
 mlir::LogicalResult CIRGenFunction::emitOpenACCEnterDataConstruct(
     const OpenACCEnterDataConstruct &s) {
   cgm.errorNYI(s.getSourceRange(), "OpenACC EnterData Construct");
@@ -245,11 +258,6 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCExitDataConstruct(
   cgm.errorNYI(s.getSourceRange(), "OpenACC ExitData Construct");
   return mlir::failure();
 }
-mlir::LogicalResult CIRGenFunction::emitOpenACCHostDataConstruct(
-    const OpenACCHostDataConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC HostData Construct");
-  return mlir::failure();
-}
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s) {
   cgm.errorNYI(s.getSourceRange(), "OpenACC Update Construct");
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
index 24cd1d399de6..71f3ccb8e040 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
@@ -22,6 +22,36 @@ using namespace clang::CIRGen;
 using namespace cir;
 using namespace mlir::acc;
 
+void CIRGenFunction::updateLoopOpParallelism(mlir::acc::LoopOp &op,
+                                             bool isOrphan,
+                                             OpenACCDirectiveKind dk) {
+  // Check that at least one of auto, independent, or seq is present
+  // for the device-independent default clauses.
+  if (op.hasParallelismFlag(mlir::acc::DeviceType::None))
+    return;
+
+  switch (dk) {
+  default:
+    llvm_unreachable("Invalid parent directive kind");
+  case OpenACCDirectiveKind::Invalid:
+  case OpenACCDirectiveKind::Parallel:
+  case OpenACCDirectiveKind::ParallelLoop:
+    op.addIndependent(builder.getContext(), {});
+    return;
+  case OpenACCDirectiveKind::Kernels:
+  case OpenACCDirectiveKind::KernelsLoop:
+    op.addAuto(builder.getContext(), {});
+    return;
+  case OpenACCDirectiveKind::Serial:
+  case OpenACCDirectiveKind::SerialLoop:
+    if (op.hasDefaultGangWorkerVector())
+      op.addAuto(builder.getContext(), {});
+    else
+      op.addSeq(builder.getContext(), {});
+    return;
+  };
+}
+
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s) {
   mlir::Location start = getLoc(s.getSourceRange().getBegin());
@@ -90,6 +120,9 @@ CIRGenFunction::emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s) {
   emitOpenACCClauses(op, s.getDirectiveKind(), s.getDirectiveLoc(),
                      s.clauses());
 
+  updateLoopOpParallelism(op, s.isOrphanedLoopConstruct(),
+                          s.getParentComputeConstructKind());
+
   mlir::LogicalResult stmtRes = mlir::success();
   // Emit body.
   {
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index d962372a4bcc..bab47924dd71 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -239,9 +239,10 @@ mlir::Type CIRGenTypes::convertRecordDeclType(const clang::RecordDecl *rd) {
 
   // Force conversion of non-virtual base classes recursively.
   if (const auto *cxxRecordDecl = dyn_cast<CXXRecordDecl>(rd)) {
-    if (cxxRecordDecl->getNumBases() > 0) {
-      cgm.errorNYI(rd->getSourceRange(),
-                   "convertRecordDeclType: derived CXXRecordDecl");
+    for (const auto &base : cxxRecordDecl->bases()) {
+      if (base.isVirtual())
+        continue;
+      convertRecordDeclType(base.getType()->castAs<RecordType>()->getDecl());
     }
   }
 
@@ -387,6 +388,13 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
     break;
   }
 
+  case Type::Complex: {
+    const auto *ct = cast<clang::ComplexType>(ty);
+    mlir::Type elementTy = convertType(ct->getElementType());
+    resultType = cir::ComplexType::get(elementTy);
+    break;
+  }
+
   case Type::LValueReference:
   case Type::RValueReference: {
     const ReferenceType *refTy = cast<ReferenceType>(ty);
@@ -411,6 +419,15 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
   case Type::ConstantArray: {
     const ConstantArrayType *arrTy = cast<ConstantArrayType>(ty);
     mlir::Type elemTy = convertTypeForMem(arrTy->getElementType());
+
+    // TODO(CIR): In LLVM, "lower arrays of undefined struct type to arrays of
+    // i8 just to have a concrete type"
+    if (!builder.isSized(elemTy)) {
+      cgm.errorNYI(SourceLocation(), "arrays of undefined struct type", type);
+      resultType = cgm.UInt32Ty;
+      break;
+    }
+
     resultType = cir::ArrayType::get(elemTy, arrTy->getSize().getZExtValue());
     break;
   }
@@ -424,10 +441,8 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
   }
 
   case Type::Enum: {
-    // TODO(cir): Implement updateCompletedType for enums.
-    assert(!cir::MissingFeatures::updateCompletedType());
-    const EnumDecl *ED = cast<EnumType>(ty)->getDecl();
-    if (auto integerType = ED->getIntegerType(); !integerType.isNull())
+    const EnumDecl *ed = cast<EnumType>(ty)->getDecl();
+    if (auto integerType = ed->getIntegerType(); !integerType.isNull())
       return convertType(integerType);
     // Return a placeholder 'i32' type.  This can be changed later when the
     // type is defined (see UpdateCompletedType), but is likely to be the
@@ -578,3 +593,40 @@ const CIRGenFunctionInfo &CIRGenTypes::arrangeGlobalDeclaration(GlobalDecl gd) {
 
   return arrangeFunctionDeclaration(fd);
 }
+
+// When we find the full definition for a TagDecl, replace the 'opaque' type we
+// previously made for it if applicable.
+void CIRGenTypes::updateCompletedType(const TagDecl *td) {
+  // If this is an enum being completed, then we flush all non-struct types
+  // from the cache. This allows function types and other things that may be
+  // derived from the enum to be recomputed.
+  if (const auto *ed = dyn_cast<EnumDecl>(td)) {
+    // Classic codegen clears the type cache if it contains an entry for this
+    // enum type that doesn't use i32 as the underlying type, but I can't find
+    // a test case that meets that condition. C++ doesn't allow forward
+    // declaration of enums, and C doesn't allow an incomplete forward
+    // declaration with a non-default type.
+    assert(
+        !typeCache.count(ed->getTypeForDecl()) ||
+        (convertType(ed->getIntegerType()) == typeCache[ed->getTypeForDecl()]));
+    // If necessary, provide the full definition of a type only used with a
+    // declaration so far.
+    assert(!cir::MissingFeatures::generateDebugInfo());
+    return;
+  }
+
+  // If we completed a RecordDecl that we previously used and converted to an
+  // anonymous type, then go ahead and complete it now.
+  const auto *rd = cast<RecordDecl>(td);
+  if (rd->isDependentType())
+    return;
+
+  // Only complete if we converted it already. If we haven't converted it yet,
+  // we'll just do it lazily.
+  if (recordDeclTypes.count(astContext.getTagDeclType(rd).getTypePtr()))
+    convertRecordDeclType(rd);
+
+  // If necessary, provide the full definition of a type only used with a
+  // declaration so far.
+  assert(!cir::MissingFeatures::generateDebugInfo());
+}
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.h
index 7a4301ed38d0..c2813d79bf63 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -19,6 +19,7 @@
 
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Type.h"
+#include "clang/Basic/ABI.h"
 #include "clang/CIR/Dialect/IR/CIRTypes.h"
 
 #include "llvm/ADT/SmallPtrSet.h"
@@ -151,6 +152,10 @@ class CIRGenTypes {
 
   const CIRGenFunctionInfo &arrangeGlobalDeclaration(GlobalDecl gd);
 
+  /// UpdateCompletedType - when we find the full definition for a TagDecl,
+  /// replace the 'opaque' type we previously made for it if applicable.
+  void updateCompletedType(const clang::TagDecl *td);
+
   /// Free functions are functions that are compatible with an ordinary C
   /// function pointer type.
   const CIRGenFunctionInfo &
@@ -161,6 +166,10 @@ class CIRGenTypes {
   bool isZeroInitializable(clang::QualType ty);
   bool isZeroInitializable(const RecordDecl *rd);
 
+  const CIRGenFunctionInfo &arrangeCXXConstructorCall(
+      const CallArgList &args, const clang::CXXConstructorDecl *d,
+      clang::CXXCtorType ctorKind, bool passProtoArgs = true);
+
   const CIRGenFunctionInfo &
   arrangeCXXMethodCall(const CallArgList &args,
                        const clang::FunctionProtoType *type,
@@ -169,6 +178,7 @@ class CIRGenTypes {
   /// C++ methods have some special rules and also have implicit parameters.
   const CIRGenFunctionInfo &
   arrangeCXXMethodDeclaration(const clang::CXXMethodDecl *md);
+  const CIRGenFunctionInfo &arrangeCXXStructorDeclaration(clang::GlobalDecl gd);
 
   const CIRGenFunctionInfo &
   arrangeCXXMethodType(const clang::CXXRecordDecl *rd,
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenValue.h b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenValue.h
index 3feadfaf5635..c1e08ba1e9b6 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenValue.h
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenValue.h
@@ -19,8 +19,6 @@
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Type.h"
 
-#include "llvm/ADT/PointerIntPair.h"
-
 #include "mlir/IR/Value.h"
 
 #include "clang/CIR/MissingFeatures.h"
@@ -34,28 +32,85 @@ namespace clang::CIRGen {
 class RValue {
   enum Flavor { Scalar, Complex, Aggregate };
 
-  // Stores first value and flavor.
-  llvm::PointerIntPair<mlir::Value, 2, Flavor> v1;
-  // Stores second value and volatility.
-  llvm::PointerIntPair<llvm::PointerUnion<mlir::Value, int *>, 1, bool> v2;
-  // Stores element type for aggregate values.
-  mlir::Type elementType;
+  union {
+    // Stores first and second value.
+    struct {
+      mlir::Value first;
+      mlir::Value second;
+    } vals;
+
+    // Stores aggregate address.
+    Address aggregateAddr;
+  };
+
+  unsigned isVolatile : 1;
+  unsigned flavor : 2;
 
 public:
-  bool isScalar() const { return v1.getInt() == Scalar; }
-  bool isAggregate() const { return v1.getInt() == Aggregate; }
+  RValue() : vals{nullptr, nullptr}, flavor(Scalar) {}
 
-  /// Return the mlir::Value of this scalar value.
+  bool isScalar() const { return flavor == Scalar; }
+  bool isComplex() const { return flavor == Complex; }
+  bool isAggregate() const { return flavor == Aggregate; }
+
+  bool isVolatileQualified() const { return isVolatile; }
+
+  /// Return the value of this scalar value.
   mlir::Value getScalarVal() const {
     assert(isScalar() && "Not a scalar!");
-    return v1.getPointer();
+    return vals.first;
+  }
+
+  /// Return the real/imag components of this complex value.
+  std::pair<mlir::Value, mlir::Value> getComplexVal() const {
+    return std::make_pair(vals.first, vals.second);
+  }
+
+  /// Return the value of the address of the aggregate.
+  Address getAggregateAddress() const {
+    assert(isAggregate() && "Not an aggregate!");
+    return aggregateAddr;
+  }
+
+  mlir::Value getAggregatePointer(QualType pointeeType) const {
+    return getAggregateAddress().getPointer();
+  }
+
+  static RValue getIgnored() {
+    // FIXME: should we make this a more explicit state?
+    return get(nullptr);
   }
 
   static RValue get(mlir::Value v) {
     RValue er;
-    er.v1.setPointer(v);
-    er.v1.setInt(Scalar);
-    er.v2.setInt(false);
+    er.vals.first = v;
+    er.flavor = Scalar;
+    er.isVolatile = false;
+    return er;
+  }
+
+  static RValue getComplex(mlir::Value v1, mlir::Value v2) {
+    RValue er;
+    er.vals = {v1, v2};
+    er.flavor = Complex;
+    er.isVolatile = false;
+    return er;
+  }
+  static RValue getComplex(const std::pair<mlir::Value, mlir::Value> &c) {
+    return getComplex(c.first, c.second);
+  }
+  // FIXME: Aggregate rvalues need to retain information about whether they are
+  // volatile or not.  Remove default to find all places that probably get this
+  // wrong.
+
+  /// Convert an Address to an RValue. If the Address is not
+  /// signed, create an RValue using the unsigned address. Otherwise, resign the
+  /// address using the provided type.
+  static RValue getAggregate(Address addr, bool isVolatile = false) {
+    RValue er;
+    er.aggregateAddr = addr;
+    er.flavor = Aggregate;
+    er.isVolatile = isVolatile;
     return er;
   }
 };
@@ -164,6 +219,7 @@ class LValue {
   clang::Qualifiers &getQuals() { return quals; }
 
   LValueBaseInfo getBaseInfo() const { return baseInfo; }
+  void setBaseInfo(LValueBaseInfo info) { baseInfo = info; }
 
   static LValue makeAddr(Address address, clang::QualType t,
                          LValueBaseInfo baseInfo) {
@@ -211,23 +267,71 @@ class AggValueSlot {
   Address addr;
   clang::Qualifiers quals;
 
+  /// This is set to true if some external code is responsible for setting up a
+  /// destructor for the slot.  Otherwise the code which constructs it should
+  /// push the appropriate cleanup.
+  LLVM_PREFERRED_TYPE(bool)
+  LLVM_ATTRIBUTE_UNUSED unsigned destructedFlag : 1;
+
   /// This is set to true if the memory in the slot is known to be zero before
   /// the assignment into it.  This means that zero fields don't need to be set.
-  bool zeroedFlag : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned zeroedFlag : 1;
+
+  /// This is set to true if the slot might be aliased and it's not undefined
+  /// behavior to access it through such an alias.  Note that it's always
+  /// undefined behavior to access a C++ object that's under construction
+  /// through an alias derived from outside the construction process.
+  ///
+  /// This flag controls whether calls that produce the aggregate
+  /// value may be evaluated directly into the slot, or whether they
+  /// must be evaluated into an unaliased temporary and then memcpy'ed
+  /// over.  Since it's invalid in general to memcpy a non-POD C++
+  /// object, it's important that this flag never be set when
+  /// evaluating an expression which constructs such an object.
+  LLVM_PREFERRED_TYPE(bool)
+  LLVM_ATTRIBUTE_UNUSED unsigned aliasedFlag : 1;
+
+  /// This is set to true if the tail padding of this slot might overlap
+  /// another object that may have already been initialized (and whose
+  /// value must be preserved by this initialization). If so, we may only
+  /// store up to the dsize of the type. Otherwise we can widen stores to
+  /// the size of the type.
+  LLVM_PREFERRED_TYPE(bool)
+  LLVM_ATTRIBUTE_UNUSED unsigned overlapFlag : 1;
 
 public:
+  enum IsDestructed_t { IsNotDestructed, IsDestructed };
   enum IsZeroed_t { IsNotZeroed, IsZeroed };
+  enum IsAliased_t { IsNotAliased, IsAliased };
+  enum Overlap_t { MayOverlap, DoesNotOverlap };
+
+  /// Returns an aggregate value slot indicating that the aggregate
+  /// value is being ignored.
+  static AggValueSlot ignored() {
+    return forAddr(Address::invalid(), clang::Qualifiers(), IsNotDestructed,
+                   IsNotAliased, DoesNotOverlap);
+  }
 
-  AggValueSlot(Address addr, clang::Qualifiers quals, bool zeroedFlag)
-      : addr(addr), quals(quals), zeroedFlag(zeroedFlag) {}
+  AggValueSlot(Address addr, clang::Qualifiers quals, bool destructedFlag,
+               bool zeroedFlag, bool aliasedFlag, bool overlapFlag)
+      : addr(addr), quals(quals), destructedFlag(destructedFlag),
+        zeroedFlag(zeroedFlag), aliasedFlag(aliasedFlag),
+        overlapFlag(overlapFlag) {}
 
   static AggValueSlot forAddr(Address addr, clang::Qualifiers quals,
+                              IsDestructed_t isDestructed,
+                              IsAliased_t isAliased, Overlap_t mayOverlap,
                               IsZeroed_t isZeroed = IsNotZeroed) {
-    return AggValueSlot(addr, quals, isZeroed);
+    return AggValueSlot(addr, quals, isDestructed, isZeroed, isAliased,
+                        mayOverlap);
   }
 
-  static AggValueSlot forLValue(const LValue &lv) {
-    return forAddr(lv.getAddress(), lv.getQuals());
+  static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed,
+                                IsAliased_t isAliased, Overlap_t mayOverlap,
+                                IsZeroed_t isZeroed = IsNotZeroed) {
+    return forAddr(LV.getAddress(), LV.getQuals(), isDestructed, isAliased,
+                   mayOverlap, isZeroed);
   }
 
   clang::Qualifiers getQualifiers() const { return quals; }
@@ -236,7 +340,16 @@ class AggValueSlot {
 
   bool isIgnored() const { return !addr.isValid(); }
 
+  mlir::Value getPointer() const { return addr.getPointer(); }
+
   IsZeroed_t isZeroed() const { return IsZeroed_t(zeroedFlag); }
+
+  RValue asRValue() const {
+    if (isIgnored())
+      return RValue::getIgnored();
+    assert(!cir::MissingFeatures::aggValueSlot());
+    return RValue::getAggregate(getAddress());
+  }
 };
 
 } // namespace clang::CIRGen
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenerator.cpp b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenerator.cpp
index 726da5b01326..99d652841be2 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenerator.cpp
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CIRGenerator.cpp
@@ -14,11 +14,13 @@
 
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/IR/MLIRContext.h"
+#include "mlir/Target/LLVMIR/Import.h"
 
 #include "clang/AST/DeclGroup.h"
 #include "clang/CIR/CIRGenerator.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 #include "clang/CIR/Dialect/OpenACC/RegisterOpenACCExtensions.h"
+#include "llvm/IR/DataLayout.h"
 
 using namespace cir;
 using namespace clang;
@@ -28,8 +30,19 @@ void CIRGenerator::anchor() {}
 CIRGenerator::CIRGenerator(clang::DiagnosticsEngine &diags,
                            llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs,
                            const CodeGenOptions &cgo)
-    : diags(diags), fs(std::move(vfs)), codeGenOpts{cgo} {}
-CIRGenerator::~CIRGenerator() = default;
+    : diags(diags), fs(std::move(vfs)), codeGenOpts{cgo},
+      handlingTopLevelDecls{0} {}
+CIRGenerator::~CIRGenerator() {
+  // There should normally not be any leftover inline method definitions.
+  assert(deferredInlineMemberFuncDefs.empty() || diags.hasErrorOccurred());
+}
+
+static void setMLIRDataLayout(mlir::ModuleOp &mod, const llvm::DataLayout &dl) {
+  mlir::MLIRContext *mlirContext = mod.getContext();
+  mlir::DataLayoutSpecInterface dlSpec =
+      mlir::translateDataLayout(dl, mlirContext);
+  mod->setAttr(mlir::DLTIDialect::kDataLayoutAttrName, dlSpec);
+}
 
 void CIRGenerator::Initialize(ASTContext &astContext) {
   using namespace llvm;
@@ -37,6 +50,7 @@ void CIRGenerator::Initialize(ASTContext &astContext) {
   this->astContext = &astContext;
 
   mlirContext = std::make_unique<mlir::MLIRContext>();
+  mlirContext->loadDialect<mlir::DLTIDialect>();
   mlirContext->loadDialect<cir::CIRDialect>();
   mlirContext->getOrLoadDialect<mlir::acc::OpenACCDialect>();
 
@@ -47,6 +61,10 @@ void CIRGenerator::Initialize(ASTContext &astContext) {
 
   cgm = std::make_unique<clang::CIRGen::CIRGenModule>(
       *mlirContext.get(), astContext, codeGenOpts, diags);
+  mlir::ModuleOp mod = cgm->getModule();
+  llvm::DataLayout layout =
+      llvm::DataLayout(astContext.getTargetInfo().getDataLayoutString());
+  setMLIRDataLayout(mod, layout);
 }
 
 bool CIRGenerator::verifyModule() const { return cgm->verifyModule(); }
@@ -57,12 +75,83 @@ bool CIRGenerator::HandleTopLevelDecl(DeclGroupRef group) {
   if (diags.hasUnrecoverableErrorOccurred())
     return true;
 
+  HandlingTopLevelDeclRAII handlingDecl(*this);
+
   for (Decl *decl : group)
     cgm->emitTopLevelDecl(decl);
 
   return true;
 }
 
+void CIRGenerator::HandleTranslationUnit(ASTContext &astContext) {
+  // Release the Builder when there is no error.
+  if (!diags.hasErrorOccurred() && cgm)
+    cgm->release();
+
+  // If there are errors before or when releasing the cgm, reset the module to
+  // stop here before invoking the backend.
+  assert(!cir::MissingFeatures::cleanupAfterErrorDiags());
+}
+
+void CIRGenerator::HandleInlineFunctionDefinition(FunctionDecl *d) {
+  if (diags.hasErrorOccurred())
+    return;
+
+  assert(d->doesThisDeclarationHaveABody());
+
+  // We may want to emit this definition. However, that decision might be
+  // based on computing the linkage, and we have to defer that in case we are
+  // inside of something that will chagne the method's final linkage, e.g.
+  //   typedef struct {
+  //     void bar();
+  //     void foo() { bar(); }
+  //   } A;
+  deferredInlineMemberFuncDefs.push_back(d);
+
+  // Provide some coverage mapping even for methods that aren't emitted.
+  // Don't do this for templated classes though, as they may not be
+  // instantiable.
+  assert(!cir::MissingFeatures::coverageMapping());
+}
+
+void CIRGenerator::emitDeferredDecls() {
+  if (deferredInlineMemberFuncDefs.empty())
+    return;
+
+  // Emit any deferred inline method definitions. Note that more deferred
+  // methods may be added during this loop, since ASTConsumer callbacks can be
+  // invoked if AST inspection results in declarations being added. Therefore,
+  // we use an index to loop over the deferredInlineMemberFuncDefs rather than
+  // a range.
+  HandlingTopLevelDeclRAII handlingDecls(*this);
+  for (unsigned i = 0; i != deferredInlineMemberFuncDefs.size(); ++i)
+    cgm->emitTopLevelDecl(deferredInlineMemberFuncDefs[i]);
+  deferredInlineMemberFuncDefs.clear();
+}
+
+/// HandleTagDeclDefinition - This callback is invoked each time a TagDecl to
+/// (e.g. struct, union, enum, class) is completed. This allows the client to
+/// hack on the type, which can occur at any point in the file (because these
+/// can be defined in declspecs).
+void CIRGenerator::HandleTagDeclDefinition(TagDecl *d) {
+  if (diags.hasErrorOccurred())
+    return;
+
+  // Don't allow re-entrant calls to CIRGen triggered by PCH deserialization to
+  // emit deferred decls.
+  HandlingTopLevelDeclRAII handlingDecl(*this, /*EmitDeferred=*/false);
+
+  cgm->updateCompletedType(d);
+
+  // For MSVC compatibility, treat declarations of static data members with
+  // inline initializers as definitions.
+  if (astContext->getTargetInfo().getCXXABI().isMicrosoft())
+    cgm->errorNYI(d->getSourceRange(), "HandleTagDeclDefinition: MSABI");
+  // For OpenMP emit declare reduction functions, if required.
+  if (astContext->getLangOpts().OpenMP)
+    cgm->errorNYI(d->getSourceRange(), "HandleTagDeclDefinition: OpenMP");
+}
+
 void CIRGenerator::CompleteTentativeDefinition(VarDecl *d) {
   if (diags.hasErrorOccurred())
     return;
diff --git a/external/llvm-project/clang/lib/CIR/CodeGen/CMakeLists.txt b/external/llvm-project/clang/lib/CIR/CodeGen/CMakeLists.txt
index 4c0aa9a3d0a6..385bea066c61 100644
--- a/external/llvm-project/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/external/llvm-project/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -10,15 +10,20 @@ add_clang_library(clangCIR
   CIRGenerator.cpp
   CIRGenBuilder.cpp
   CIRGenCall.cpp
+  CIRGenClass.cpp
+  CIRGenCXX.cpp
   CIRGenCXXABI.cpp
   CIRGenCXXExpr.cpp
+  CIRGenBuiltin.cpp
   CIRGenDecl.cpp
   CIRGenDeclOpenACC.cpp
   CIRGenExpr.cpp
   CIRGenExprAggregate.cpp
+  CIRGenExprComplex.cpp
   CIRGenExprConstant.cpp
   CIRGenExprScalar.cpp
   CIRGenFunction.cpp
+  CIRGenItaniumCXXABI.cpp
   CIRGenModule.cpp
   CIRGenOpenACCClause.cpp
   CIRGenRecordLayoutBuilder.cpp
@@ -41,4 +46,5 @@ add_clang_library(clangCIR
   CIROpenACCSupport
   MLIRCIR
   MLIRCIRInterfaces
+  MLIRTargetLLVMIRImport
 )
diff --git a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
index 52f0b33afaba..b52e9c3e4919 100644
--- a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
+++ b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
@@ -183,6 +183,26 @@ LogicalResult FPAttr::verify(function_ref<InFlightDiagnostic()> emitError,
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// ConstComplexAttr definitions
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+ConstComplexAttr::verify(function_ref<InFlightDiagnostic()> emitError,
+                         cir::ComplexType type, mlir::TypedAttr real,
+                         mlir::TypedAttr imag) {
+  mlir::Type elemType = type.getElementType();
+  if (real.getType() != elemType)
+    return emitError()
+           << "type of the real part does not match the complex type";
+
+  if (imag.getType() != elemType)
+    return emitError()
+           << "type of the imaginary part does not match the complex type";
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // CIR ConstArrayAttr
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRDataLayout.cpp b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRDataLayout.cpp
new file mode 100644
index 000000000000..d835c4076224
--- /dev/null
+++ b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRDataLayout.cpp
@@ -0,0 +1,22 @@
+#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
+
+using namespace cir;
+
+//===----------------------------------------------------------------------===//
+//                       DataLayout Class Implementation
+//===----------------------------------------------------------------------===//
+
+CIRDataLayout::CIRDataLayout(mlir::ModuleOp modOp) : layout(modOp) {
+  reset(modOp.getDataLayoutSpec());
+}
+
+void CIRDataLayout::reset(mlir::DataLayoutSpecInterface spec) {
+  bigEndian = false;
+  if (spec) {
+    mlir::StringAttr key = mlir::StringAttr::get(
+        spec.getContext(), mlir::DLTIDialect::kDataLayoutEndiannessKey);
+    if (mlir::DataLayoutEntryInterface entry = spec.getSpecForIdentifier(key))
+      if (auto str = llvm::dyn_cast<mlir::StringAttr>(entry.getValue()))
+        bigEndian = str == mlir::DLTIDialect::kDataLayoutEndiannessBig;
+  }
+}
diff --git a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 36f050de9f8b..5578d4f5825a 100644
--- a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -21,6 +21,7 @@
 #include "clang/CIR/Dialect/IR/CIROpsDialect.cpp.inc"
 #include "clang/CIR/Dialect/IR/CIROpsEnums.cpp.inc"
 #include "clang/CIR/MissingFeatures.h"
+
 #include <numeric>
 
 using namespace mlir;
@@ -230,9 +231,11 @@ static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType,
   }
 
   if (isa<cir::ZeroAttr>(attrType)) {
-    if (isa<cir::RecordType, cir::ArrayType, cir::VectorType>(opType))
+    if (isa<cir::RecordType, cir::ArrayType, cir::VectorType, cir::ComplexType>(
+            opType))
       return success();
-    return op->emitOpError("zero expects struct or array type");
+    return op->emitOpError(
+        "zero expects struct, array, vector, or complex type");
   }
 
   if (mlir::isa<cir::BoolAttr>(attrType)) {
@@ -252,7 +255,8 @@ static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType,
     return success();
   }
 
-  if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr>(attrType))
+  if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
+                cir::ConstComplexAttr>(attrType))
     return success();
 
   assert(isa<TypedAttr>(attrType) && "What else could we be looking at here?");
@@ -286,7 +290,7 @@ LogicalResult cir::ContinueOp::verify() {
 //===----------------------------------------------------------------------===//
 
 LogicalResult cir::CastOp::verify() {
-  mlir::Type resType = getResult().getType();
+  mlir::Type resType = getType();
   mlir::Type srcType = getSrc().getType();
 
   if (mlir::isa<cir::VectorType>(srcType) &&
@@ -444,7 +448,7 @@ static Value tryFoldCastChain(cir::CastOp op) {
 }
 
 OpFoldResult cir::CastOp::fold(FoldAdaptor adaptor) {
-  if (getSrc().getType() == getResult().getType()) {
+  if (getSrc().getType() == getType()) {
     switch (getKind()) {
     case cir::CastKind::integral: {
       // TODO: for sign differences, it's possible in certain conditions to
@@ -1446,7 +1450,7 @@ LogicalResult cir::ShiftOp::verify() {
     if (op0VecTy.getSize() != op1VecTy.getSize())
       return emitOpError() << "input vector types must have the same size";
 
-    auto opResultTy = mlir::dyn_cast<cir::VectorType>(getResult().getType());
+    auto opResultTy = mlir::dyn_cast<cir::VectorType>(getType());
     if (!opResultTy)
       return emitOpError() << "the type of the result must be a vector "
                            << "if it is vector shift";
@@ -1519,7 +1523,7 @@ LogicalResult cir::GetMemberOp::verify() {
   if (recordTy.getMembers().size() <= getIndex())
     return emitError() << "member index out of bounds";
 
-  if (recordTy.getMembers()[getIndex()] != getResultTy().getPointee())
+  if (recordTy.getMembers()[getIndex()] != getType().getPointee())
     return emitError() << "member type mismatch";
 
   return mlir::success();
@@ -1529,11 +1533,21 @@ LogicalResult cir::GetMemberOp::verify() {
 // VecCreateOp
 //===----------------------------------------------------------------------===//
 
+OpFoldResult cir::VecCreateOp::fold(FoldAdaptor adaptor) {
+  if (llvm::any_of(getElements(), [](mlir::Value value) {
+        return !mlir::isa<cir::ConstantOp>(value.getDefiningOp());
+      }))
+    return {};
+
+  return cir::ConstVectorAttr::get(
+      getType(), mlir::ArrayAttr::get(getContext(), adaptor.getElements()));
+}
+
 LogicalResult cir::VecCreateOp::verify() {
   // Verify that the number of arguments matches the number of elements in the
   // vector, and that the type of all the arguments matches the type of the
   // elements in the vector.
-  const VectorType vecTy = getResult().getType();
+  const cir::VectorType vecTy = getType();
   if (getElements().size() != vecTy.getSize()) {
     return emitOpError() << "operand count of " << getElements().size()
                          << " doesn't match vector type " << vecTy
@@ -1575,10 +1589,106 @@ OpFoldResult cir::VecExtractOp::fold(FoldAdaptor adaptor) {
   return elements[index];
 }
 
+//===----------------------------------------------------------------------===//
+// VecShuffleOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult cir::VecShuffleOp::fold(FoldAdaptor adaptor) {
+  auto vec1Attr =
+      mlir::dyn_cast_if_present<cir::ConstVectorAttr>(adaptor.getVec1());
+  auto vec2Attr =
+      mlir::dyn_cast_if_present<cir::ConstVectorAttr>(adaptor.getVec2());
+  if (!vec1Attr || !vec2Attr)
+    return {};
+
+  mlir::Type vec1ElemTy =
+      mlir::cast<cir::VectorType>(vec1Attr.getType()).getElementType();
+
+  mlir::ArrayAttr vec1Elts = vec1Attr.getElts();
+  mlir::ArrayAttr vec2Elts = vec2Attr.getElts();
+  mlir::ArrayAttr indicesElts = adaptor.getIndices();
+
+  SmallVector<mlir::Attribute, 16> elements;
+  elements.reserve(indicesElts.size());
+
+  uint64_t vec1Size = vec1Elts.size();
+  for (const auto &idxAttr : indicesElts.getAsRange<cir::IntAttr>()) {
+    if (idxAttr.getSInt() == -1) {
+      elements.push_back(cir::UndefAttr::get(vec1ElemTy));
+      continue;
+    }
+
+    uint64_t idxValue = idxAttr.getUInt();
+    elements.push_back(idxValue < vec1Size ? vec1Elts[idxValue]
+                                           : vec2Elts[idxValue - vec1Size]);
+  }
+
+  return cir::ConstVectorAttr::get(
+      getType(), mlir::ArrayAttr::get(getContext(), elements));
+}
+
+LogicalResult cir::VecShuffleOp::verify() {
+  // The number of elements in the indices array must match the number of
+  // elements in the result type.
+  if (getIndices().size() != getResult().getType().getSize()) {
+    return emitOpError() << ": the number of elements in " << getIndices()
+                         << " and " << getResult().getType() << " don't match";
+  }
+
+  // The element types of the two input vectors and of the result type must
+  // match.
+  if (getVec1().getType().getElementType() !=
+      getResult().getType().getElementType()) {
+    return emitOpError() << ": element types of " << getVec1().getType()
+                         << " and " << getResult().getType() << " don't match";
+  }
+
+  const uint64_t maxValidIndex =
+      getVec1().getType().getSize() + getVec2().getType().getSize() - 1;
+  if (llvm::any_of(
+          getIndices().getAsRange<cir::IntAttr>(), [&](cir::IntAttr idxAttr) {
+            return idxAttr.getSInt() != -1 && idxAttr.getUInt() > maxValidIndex;
+          })) {
+    return emitOpError() << ": index for __builtin_shufflevector must be "
+                            "less than the total number of vector elements";
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // VecShuffleDynamicOp
 //===----------------------------------------------------------------------===//
 
+OpFoldResult cir::VecShuffleDynamicOp::fold(FoldAdaptor adaptor) {
+  mlir::Attribute vec = adaptor.getVec();
+  mlir::Attribute indices = adaptor.getIndices();
+  if (mlir::isa_and_nonnull<cir::ConstVectorAttr>(vec) &&
+      mlir::isa_and_nonnull<cir::ConstVectorAttr>(indices)) {
+    auto vecAttr = mlir::cast<cir::ConstVectorAttr>(vec);
+    auto indicesAttr = mlir::cast<cir::ConstVectorAttr>(indices);
+
+    mlir::ArrayAttr vecElts = vecAttr.getElts();
+    mlir::ArrayAttr indicesElts = indicesAttr.getElts();
+
+    const uint64_t numElements = vecElts.size();
+
+    SmallVector<mlir::Attribute, 16> elements;
+    elements.reserve(numElements);
+
+    const uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1;
+    for (const auto &idxAttr : indicesElts.getAsRange<cir::IntAttr>()) {
+      uint64_t idxValue = idxAttr.getUInt();
+      uint64_t newIdx = idxValue & maskBits;
+      elements.push_back(vecElts[newIdx]);
+    }
+
+    return cir::ConstVectorAttr::get(
+        getType(), mlir::ArrayAttr::get(getContext(), elements));
+  }
+
+  return {};
+}
+
 LogicalResult cir::VecShuffleDynamicOp::verify() {
   // The number of elements in the two input vectors must match.
   if (getVec().getType().getSize() !=
@@ -1589,6 +1699,82 @@ LogicalResult cir::VecShuffleDynamicOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// VecTernaryOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::VecTernaryOp::verify() {
+  // Verify that the condition operand has the same number of elements as the
+  // other operands.  (The automatic verification already checked that all
+  // operands are vector types and that the second and third operands are the
+  // same type.)
+  if (getCond().getType().getSize() != getLhs().getType().getSize()) {
+    return emitOpError() << ": the number of elements in "
+                         << getCond().getType() << " and " << getLhs().getType()
+                         << " don't match";
+  }
+  return success();
+}
+
+OpFoldResult cir::VecTernaryOp::fold(FoldAdaptor adaptor) {
+  mlir::Attribute cond = adaptor.getCond();
+  mlir::Attribute lhs = adaptor.getLhs();
+  mlir::Attribute rhs = adaptor.getRhs();
+
+  if (!mlir::isa_and_nonnull<cir::ConstVectorAttr>(cond) ||
+      !mlir::isa_and_nonnull<cir::ConstVectorAttr>(lhs) ||
+      !mlir::isa_and_nonnull<cir::ConstVectorAttr>(rhs))
+    return {};
+  auto condVec = mlir::cast<cir::ConstVectorAttr>(cond);
+  auto lhsVec = mlir::cast<cir::ConstVectorAttr>(lhs);
+  auto rhsVec = mlir::cast<cir::ConstVectorAttr>(rhs);
+
+  mlir::ArrayAttr condElts = condVec.getElts();
+
+  SmallVector<mlir::Attribute, 16> elements;
+  elements.reserve(condElts.size());
+
+  for (const auto &[idx, condAttr] :
+       llvm::enumerate(condElts.getAsRange<cir::IntAttr>())) {
+    if (condAttr.getSInt()) {
+      elements.push_back(lhsVec.getElts()[idx]);
+    } else {
+      elements.push_back(rhsVec.getElts()[idx]);
+    }
+  }
+
+  cir::VectorType vecTy = getLhs().getType();
+  return cir::ConstVectorAttr::get(
+      vecTy, mlir::ArrayAttr::get(getContext(), elements));
+}
+
+//===----------------------------------------------------------------------===//
+// ComplexCreateOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::ComplexCreateOp::verify() {
+  if (getType().getElementType() != getReal().getType()) {
+    emitOpError()
+        << "operand type of cir.complex.create does not match its result type";
+    return failure();
+  }
+
+  return success();
+}
+
+OpFoldResult cir::ComplexCreateOp::fold(FoldAdaptor adaptor) {
+  mlir::Attribute real = adaptor.getReal();
+  mlir::Attribute imag = adaptor.getImag();
+  if (!real || !imag)
+    return {};
+
+  // When both of real and imag are constants, we can fold the operation into an
+  // `#cir.const_complex` operation.
+  auto realAttr = mlir::cast<mlir::TypedAttr>(real);
+  auto imagAttr = mlir::cast<mlir::TypedAttr>(imag);
+  return cir::ConstComplexAttr::get(realAttr, imagAttr);
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
index 20b086ffdd85..2550c369a927 100644
--- a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
+++ b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp
@@ -74,7 +74,7 @@ bool cir::LoadOp::canUsesBeRemoved(
     return false;
   Value blockingUse = (*blockingUses.begin())->get();
   return blockingUse == slot.ptr && getAddr() == slot.ptr &&
-         getResult().getType() == slot.elemType;
+         getType() == slot.elemType;
 }
 
 DeletionKind cir::LoadOp::removeBlockingUses(
diff --git a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRTypes.cpp b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
index b402177a5ec1..09926a93eb6f 100644
--- a/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
+++ b/external/llvm-project/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
@@ -98,6 +98,8 @@ Type RecordType::parse(mlir::AsmParser &parser) {
     kind = RecordKind::Struct;
   else if (parser.parseOptionalKeyword("union").succeeded())
     kind = RecordKind::Union;
+  else if (parser.parseOptionalKeyword("class").succeeded())
+    kind = RecordKind::Class;
   else {
     parser.emitError(loc, "unknown record type");
     return {};
@@ -168,6 +170,9 @@ void RecordType::print(mlir::AsmPrinter &printer) const {
   case RecordKind::Union:
     printer << "union ";
     break;
+  case RecordKind::Class:
+    printer << "class ";
+    break;
   }
 
   if (getName())
@@ -553,17 +558,30 @@ LongDoubleType::getABIAlignment(const mlir::DataLayout &dataLayout,
 }
 
 //===----------------------------------------------------------------------===//
-// Floating-point and Float-point Vector type helpers
+// ComplexType Definitions
 //===----------------------------------------------------------------------===//
 
-bool cir::isFPOrFPVectorTy(mlir::Type t) {
-  assert(!cir::MissingFeatures::vectorType());
-  return isAnyFloatingPointType(t);
+llvm::TypeSize
+cir::ComplexType::getTypeSizeInBits(const mlir::DataLayout &dataLayout,
+                                    mlir::DataLayoutEntryListRef params) const {
+  // C17 6.2.5p13:
+  //   Each complex type has the same representation and alignment requirements
+  //   as an array type containing exactly two elements of the corresponding
+  //   real type.
+
+  return dataLayout.getTypeSizeInBits(getElementType()) * 2;
 }
 
-//===----------------------------------------------------------------------===//
-// FuncType Definitions
-//===----------------------------------------------------------------------===//
+uint64_t
+cir::ComplexType::getABIAlignment(const mlir::DataLayout &dataLayout,
+                                  mlir::DataLayoutEntryListRef params) const {
+  // C17 6.2.5p13:
+  //   Each complex type has the same representation and alignment requirements
+  //   as an array type containing exactly two elements of the corresponding
+  //   real type.
+
+  return dataLayout.getTypeABIAlignment(getElementType());
+}
 
 FuncType FuncType::clone(TypeRange inputs, TypeRange results) const {
   assert(results.size() == 1 && "expected exactly one result type");
@@ -693,17 +711,7 @@ mlir::LogicalResult cir::VectorType::verify(
     mlir::Type elementType, uint64_t size) {
   if (size == 0)
     return emitError() << "the number of vector elements must be non-zero";
-
-  // Check if it a valid FixedVectorType
-  if (mlir::isa<cir::PointerType, cir::FP128Type>(elementType))
-    return success();
-
-  // Check if it a valid VectorType
-  if (mlir::isa<cir::IntType>(elementType) ||
-      isAnyFloatingPointType(elementType))
-    return success();
-
-  return emitError() << "unsupported element type for CIR vector";
+  return success();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/lib/CIR/Dialect/IR/CMakeLists.txt b/external/llvm-project/clang/lib/CIR/Dialect/IR/CMakeLists.txt
index e5256bf961f4..98575941035f 100644
--- a/external/llvm-project/clang/lib/CIR/Dialect/IR/CMakeLists.txt
+++ b/external/llvm-project/clang/lib/CIR/Dialect/IR/CMakeLists.txt
@@ -3,6 +3,7 @@ add_clang_library(MLIRCIR
   CIRDialect.cpp
   CIRMemorySlot.cpp
   CIRTypes.cpp
+  CIRDataLayout.cpp
 
   DEPENDS
   MLIRCIROpsIncGen
diff --git a/external/llvm-project/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp b/external/llvm-project/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
index fb000adee04c..20c634d6c66f 100644
--- a/external/llvm-project/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
+++ b/external/llvm-project/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
@@ -134,14 +134,15 @@ void CIRCanonicalizePass::runOnOperation() {
   getOperation()->walk([&](Operation *op) {
     assert(!cir::MissingFeatures::switchOp());
     assert(!cir::MissingFeatures::tryOp());
-    assert(!cir::MissingFeatures::complexCreateOp());
     assert(!cir::MissingFeatures::complexRealOp());
     assert(!cir::MissingFeatures::complexImagOp());
     assert(!cir::MissingFeatures::callOp());
-    // CastOp, UnaryOp and VecExtractOp are here to perform a manual `fold` in
+
+    // Many operations are here to perform a manual `fold` in
     // applyOpPatternsGreedily.
     if (isa<BrOp, BrCondOp, CastOp, ScopeOp, SwitchOp, SelectOp, UnaryOp,
-            VecExtractOp>(op))
+            ComplexCreateOp, VecCreateOp, VecExtractOp, VecShuffleOp,
+            VecShuffleDynamicOp, VecTernaryOp>(op))
       ops.push_back(op);
   });
 
diff --git a/external/llvm-project/clang/lib/CIR/FrontendAction/CIRGenAction.cpp b/external/llvm-project/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
index 48ae2a05df70..9264aa6b18b5 100644
--- a/external/llvm-project/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
+++ b/external/llvm-project/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
@@ -84,6 +84,10 @@ class CIRGenConsumer : public clang::ASTConsumer {
     return true;
   }
 
+  void HandleInlineFunctionDefinition(FunctionDecl *D) override {
+    Gen->HandleInlineFunctionDefinition(D);
+  }
+
   void HandleTranslationUnit(ASTContext &C) override {
     Gen->HandleTranslationUnit(C);
 
@@ -136,6 +140,13 @@ class CIRGenConsumer : public clang::ASTConsumer {
     }
   }
 
+  void HandleTagDeclDefinition(TagDecl *D) override {
+    PrettyStackTraceDecl CrashInfo(D, SourceLocation(),
+                                   Context->getSourceManager(),
+                                   "CIR generation of declaration");
+    Gen->HandleTagDeclDefinition(D);
+  }
+
   void CompleteTentativeDefinition(VarDecl *D) override {
     Gen->CompleteTentativeDefinition(D);
   }
diff --git a/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index b07e61638c3b..6a4e4e4a7df3 100644
--- a/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -105,7 +105,7 @@ static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter,
                                   cir::LoadOp op, mlir::Value value) {
 
   // TODO(cir): Handle other types similarly to clang's codegen EmitFromMemory
-  if (auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getResult().getType())) {
+  if (auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getType())) {
     // Create a cast value from specified size in datalayout to i1
     assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy)));
     return createIntCast(rewriter, value, rewriter.getI1Type());
@@ -188,14 +188,15 @@ class CIRAttrToValue {
 
   mlir::Value visit(mlir::Attribute attr) {
     return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)
-        .Case<cir::IntAttr, cir::FPAttr, cir::ConstArrayAttr,
-              cir::ConstVectorAttr, cir::ConstPtrAttr, cir::ZeroAttr>(
-            [&](auto attrT) { return visitCirAttr(attrT); })
+        .Case<cir::IntAttr, cir::FPAttr, cir::ConstComplexAttr,
+              cir::ConstArrayAttr, cir::ConstVectorAttr, cir::ConstPtrAttr,
+              cir::ZeroAttr>([&](auto attrT) { return visitCirAttr(attrT); })
         .Default([&](auto attrT) { return mlir::Value(); });
   }
 
   mlir::Value visitCirAttr(cir::IntAttr intAttr);
   mlir::Value visitCirAttr(cir::FPAttr fltAttr);
+  mlir::Value visitCirAttr(cir::ConstComplexAttr complexAttr);
   mlir::Value visitCirAttr(cir::ConstPtrAttr ptrAttr);
   mlir::Value visitCirAttr(cir::ConstArrayAttr attr);
   mlir::Value visitCirAttr(cir::ConstVectorAttr attr);
@@ -226,6 +227,42 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
       loc, converter->convertType(intAttr.getType()), intAttr.getValue());
 }
 
+/// FPAttr visitor.
+mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) {
+  mlir::Location loc = parentOp->getLoc();
+  return rewriter.create<mlir::LLVM::ConstantOp>(
+      loc, converter->convertType(fltAttr.getType()), fltAttr.getValue());
+}
+
+/// ConstComplexAttr visitor.
+mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) {
+  auto complexType = mlir::cast<cir::ComplexType>(complexAttr.getType());
+  mlir::Type complexElemTy = complexType.getElementType();
+  mlir::Type complexElemLLVMTy = converter->convertType(complexElemTy);
+
+  mlir::Attribute components[2];
+  if (const auto intType = mlir::dyn_cast<cir::IntType>(complexElemTy)) {
+    components[0] = rewriter.getIntegerAttr(
+        complexElemLLVMTy,
+        mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
+    components[1] = rewriter.getIntegerAttr(
+        complexElemLLVMTy,
+        mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
+  } else {
+    components[0] = rewriter.getFloatAttr(
+        complexElemLLVMTy,
+        mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
+    components[1] = rewriter.getFloatAttr(
+        complexElemLLVMTy,
+        mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
+  }
+
+  mlir::Location loc = parentOp->getLoc();
+  return rewriter.create<mlir::LLVM::ConstantOp>(
+      loc, converter->convertType(complexAttr.getType()),
+      rewriter.getArrayAttr(components));
+}
+
 /// ConstPtrAttr visitor.
 mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) {
   mlir::Location loc = parentOp->getLoc();
@@ -241,13 +278,6 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) {
       loc, converter->convertType(ptrAttr.getType()), ptrVal);
 }
 
-/// FPAttr visitor.
-mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) {
-  mlir::Location loc = parentOp->getLoc();
-  return rewriter.create<mlir::LLVM::ConstantOp>(
-      loc, converter->convertType(fltAttr.getType()), fltAttr.getValue());
-}
-
 // ConstArrayAttr visitor
 mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) {
   mlir::Type llvmTy = converter->convertType(attr.getType());
@@ -341,9 +371,11 @@ class GlobalInitAttrRewriter {
   mlir::Attribute visitCirAttr(cir::IntAttr attr) {
     return rewriter.getIntegerAttr(llvmType, attr.getValue());
   }
+
   mlir::Attribute visitCirAttr(cir::FPAttr attr) {
     return rewriter.getFloatAttr(llvmType, attr.getValue());
   }
+
   mlir::Attribute visitCirAttr(cir::BoolAttr attr) {
     return rewriter.getBoolAttr(attr.getValue());
   }
@@ -405,7 +437,7 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   switch (castOp.getKind()) {
   case cir::CastKind::array_to_ptrdecay: {
     const auto ptrTy = mlir::cast<cir::PointerType>(castOp.getType());
-    mlir::Value sourceValue = adaptor.getOperands().front();
+    mlir::Value sourceValue = adaptor.getSrc();
     mlir::Type targetType = convertTy(ptrTy);
     mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout,
                                                 ptrTy.getPointee());
@@ -415,7 +447,7 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
     break;
   }
   case cir::CastKind::int_to_bool: {
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Value zeroInt = rewriter.create<mlir::LLVM::ConstantOp>(
         castOp.getLoc(), llvmSrcVal.getType(), 0);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
@@ -424,8 +456,8 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::integral: {
     mlir::Type srcType = castOp.getSrc().getType();
-    mlir::Type dstType = castOp.getResult().getType();
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Type dstType = castOp.getType();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Type llvmDstType = getTypeConverter()->convertType(dstType);
     cir::IntType srcIntType =
         mlir::cast<cir::IntType>(elementTypeIfVector(srcType));
@@ -438,12 +470,11 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
     break;
   }
   case cir::CastKind::floating: {
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
-    mlir::Type llvmDstTy =
-        getTypeConverter()->convertType(castOp.getResult().getType());
+    mlir::Value llvmSrcVal = adaptor.getSrc();
+    mlir::Type llvmDstTy = getTypeConverter()->convertType(castOp.getType());
 
     mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType());
-    mlir::Type dstTy = elementTypeIfVector(castOp.getResult().getType());
+    mlir::Type dstTy = elementTypeIfVector(castOp.getType());
 
     if (!mlir::isa<cir::CIRFPTypeInterface>(dstTy) ||
         !mlir::isa<cir::CIRFPTypeInterface>(srcTy))
@@ -463,7 +494,7 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::int_to_ptr: {
     auto dstTy = mlir::cast<cir::PointerType>(castOp.getType());
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
     rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(castOp, llvmDstTy,
                                                         llvmSrcVal);
@@ -471,14 +502,14 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::ptr_to_int: {
     auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
     rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(castOp, llvmDstTy,
                                                         llvmSrcVal);
     return mlir::success();
   }
   case cir::CastKind::float_to_bool: {
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     auto kind = mlir::LLVM::FCmpPredicate::une;
 
     // Check if float is not equal to zero.
@@ -494,7 +525,7 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::bool_to_int: {
     auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     auto llvmSrcTy = mlir::cast<mlir::IntegerType>(llvmSrcVal.getType());
     auto llvmDstTy =
         mlir::cast<mlir::IntegerType>(getTypeConverter()->convertType(dstTy));
@@ -508,7 +539,7 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::bool_to_float: {
     mlir::Type dstTy = castOp.getType();
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
     rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
                                                       llvmSrcVal);
@@ -516,7 +547,7 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::int_to_float: {
     mlir::Type dstTy = castOp.getType();
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
     if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getSrc().getType()))
             .isSigned())
@@ -529,10 +560,9 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::float_to_int: {
     mlir::Type dstTy = castOp.getType();
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
-    if (mlir::cast<cir::IntType>(
-            elementTypeIfVector(castOp.getResult().getType()))
+    if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getType()))
             .isSigned())
       rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(castOp, llvmDstTy,
                                                         llvmSrcVal);
@@ -548,13 +578,13 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
     assert(!MissingFeatures::cxxABI());
     assert(!MissingFeatures::dataMemberType());
 
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
                                                        llvmSrcVal);
     return mlir::success();
   }
   case cir::CastKind::ptr_to_bool: {
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Value zeroPtr = rewriter.create<mlir::LLVM::ZeroOp>(
         castOp.getLoc(), llvmSrcVal.getType());
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
@@ -563,7 +593,7 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   }
   case cir::CastKind::address_space: {
     mlir::Type dstTy = castOp.getType();
-    mlir::Value llvmSrcVal = adaptor.getOperands().front();
+    mlir::Value llvmSrcVal = adaptor.getSrc();
     mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
     rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(castOp, llvmDstTy,
                                                              llvmSrcVal);
@@ -641,6 +671,38 @@ mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite(
+    cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  const mlir::Type resultType =
+      getTypeConverter()->convertType(baseClassOp.getType());
+  mlir::Value derivedAddr = adaptor.getDerivedAddr();
+  llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {
+      adaptor.getOffset().getZExtValue()};
+  mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,
+                                               mlir::IntegerType::Signless);
+  if (adaptor.getOffset().getZExtValue() == 0) {
+    rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(
+        baseClassOp, resultType, adaptor.getDerivedAddr());
+    return mlir::success();
+  }
+
+  if (baseClassOp.getAssumeNotNull()) {
+    rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
+        baseClassOp, resultType, byteType, derivedAddr, offset);
+  } else {
+    auto loc = baseClassOp.getLoc();
+    mlir::Value isNull = rewriter.create<mlir::LLVM::ICmpOp>(
+        loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr,
+        rewriter.create<mlir::LLVM::ZeroOp>(loc, derivedAddr.getType()));
+    mlir::Value adjusted = rewriter.create<mlir::LLVM::GEPOp>(
+        loc, resultType, byteType, derivedAddr, offset);
+    rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull,
+                                                      derivedAddr, adjusted);
+  }
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(
     cir::AllocaOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -649,8 +711,8 @@ mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(
       op.getLoc(), typeConverter->convertType(rewriter.getIndexType()), 1);
   mlir::Type elementTy =
       convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType());
-  mlir::Type resultTy = convertTypeForMemory(*getTypeConverter(), dataLayout,
-                                             op.getResult().getType());
+  mlir::Type resultTy =
+      convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
 
   assert(!cir::MissingFeatures::addressSpace());
   assert(!cir::MissingFeatures::opAllocaAnnotations());
@@ -722,8 +784,8 @@ mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite(
 mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
     cir::LoadOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
-  const mlir::Type llvmTy = convertTypeForMemory(
-      *getTypeConverter(), dataLayout, op.getResult().getType());
+  const mlir::Type llvmTy =
+      convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
   assert(!cir::MissingFeatures::opLoadStoreMemOrder());
   std::optional<size_t> opAlign = op.getAlignment();
   unsigned alignment =
@@ -839,7 +901,36 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
       rewriter.eraseOp(op);
       return mlir::success();
     }
-  } else {
+  } else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) {
+    rewriter.replaceOp(op, lowerCirAttrAsValue(op, op.getValue(), rewriter,
+                                               getTypeConverter()));
+    return mlir::success();
+  } else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) {
+    auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());
+    mlir::Type complexElemTy = complexTy.getElementType();
+    mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy);
+
+    mlir::Attribute components[2];
+    if (mlir::isa<cir::IntType>(complexElemTy)) {
+      components[0] = rewriter.getIntegerAttr(
+          complexElemLLVMTy,
+          mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
+      components[1] = rewriter.getIntegerAttr(
+          complexElemLLVMTy,
+          mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
+    } else {
+      components[0] = rewriter.getFloatAttr(
+          complexElemLLVMTy,
+          mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
+      components[1] = rewriter.getFloatAttr(
+          complexElemLLVMTy,
+          mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
+    }
+
+    attr = rewriter.getArrayAttr(components);
+  }
+
+  else {
     return op.emitError() << "unsupported constant type " << op.getType();
   }
 
@@ -969,13 +1060,13 @@ void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
   const uint64_t alignment = op.getAlignment().value_or(0);
   const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
   const StringRef symbol = op.getSymName();
+  mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
 
   SmallVector<mlir::NamedAttribute> attributes;
   mlir::LLVM::GlobalOp newGlobalOp =
       rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
           op, llvmType, isConst, linkage, symbol, nullptr, alignment, addrSpace,
-          isDsoLocal, isThreadLocal,
-          /*comdat=*/mlir::SymbolRefAttr(), attributes);
+          isDsoLocal, isThreadLocal, comdatAttr, attributes);
   newGlobalOp.getRegion().emplaceBlock();
   rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
 }
@@ -986,7 +1077,7 @@ CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
     mlir::ConversionPatternRewriter &rewriter) const {
   // TODO: Generalize this handling when more types are needed here.
   assert((isa<cir::ConstArrayAttr, cir::ConstVectorAttr, cir::ConstPtrAttr,
-              cir::ZeroAttr>(init)));
+              cir::ConstComplexAttr, cir::ZeroAttr>(init)));
 
   // TODO(cir): once LLVM's dialect has proper equivalent attributes this
   // should be updated. For now, we use a custom op to initialize globals
@@ -1024,6 +1115,7 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
   const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
   const StringRef symbol = op.getSymName();
   SmallVector<mlir::NamedAttribute> attributes;
+  mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
 
   if (init.has_value()) {
     if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(init.value())) {
@@ -1038,7 +1130,8 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
         return mlir::failure();
       }
     } else if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
-                         cir::ConstPtrAttr, cir::ZeroAttr>(init.value())) {
+                         cir::ConstPtrAttr, cir::ConstComplexAttr,
+                         cir::ZeroAttr>(init.value())) {
       // TODO(cir): once LLVM's dialect has proper equivalent attributes this
       // should be updated. For now, we use a custom op to initialize globals
       // to the appropriate value.
@@ -1054,12 +1147,33 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
   // Rewrite op.
   rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
       op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()),
-      alignment, addrSpace, isDsoLocal, isThreadLocal,
-      /*comdat=*/mlir::SymbolRefAttr(), attributes);
-
+      alignment, addrSpace, isDsoLocal, isThreadLocal, comdatAttr, attributes);
   return mlir::success();
 }
 
+mlir::SymbolRefAttr
+CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op,
+                                         mlir::OpBuilder &builder) const {
+  if (!op.getComdat())
+    return mlir::SymbolRefAttr{};
+
+  mlir::ModuleOp module = op->getParentOfType<mlir::ModuleOp>();
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  StringRef comdatName("__llvm_comdat_globals");
+  if (!comdatOp) {
+    builder.setInsertionPointToStart(module.getBody());
+    comdatOp =
+        builder.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
+  }
+
+  builder.setInsertionPointToStart(&comdatOp.getBody().back());
+  auto selectorOp = builder.create<mlir::LLVM::ComdatSelectorOp>(
+      comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
+  return mlir::SymbolRefAttr::get(
+      builder.getContext(), comdatName,
+      mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr()));
+}
+
 mlir::LogicalResult CIRToLLVMSwitchFlatOpLowering::matchAndRewrite(
     cir::SwitchFlatOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -1549,6 +1663,14 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
   converter.addConversion([&](cir::BF16Type type) -> mlir::Type {
     return mlir::BFloat16Type::get(type.getContext());
   });
+  converter.addConversion([&](cir::ComplexType type) -> mlir::Type {
+    // A complex type is lowered to an LLVM struct that contains the real and
+    // imaginary part as data fields.
+    mlir::Type elementTy = converter.convertType(type.getElementType());
+    mlir::Type structFields[2] = {elementTy, elementTy};
+    return mlir::LLVM::LLVMStructType::getLiteral(type.getContext(),
+                                                  structFields);
+  });
   converter.addConversion([&](cir::FuncType type) -> std::optional<mlir::Type> {
     auto result = converter.convertType(type.getReturnType());
     llvm::SmallVector<mlir::Type> arguments;
@@ -1562,6 +1684,7 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
     // Convert struct members.
     llvm::SmallVector<mlir::Type> llvmMembers;
     switch (type.getKind()) {
+    case cir::RecordType::Class:
     case cir::RecordType::Struct:
       for (mlir::Type ty : type.getMembers())
         llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, ty));
@@ -1688,6 +1811,7 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                                              dl);
   patterns.add<
       // clang-format off
+               CIRToLLVMBaseClassAddrOpLowering,
                CIRToLLVMBinOpLowering,
                CIRToLLVMBrCondOpLowering,
                CIRToLLVMBrOpLowering,
@@ -1708,7 +1832,11 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMVecExtractOpLowering,
                CIRToLLVMVecInsertOpLowering,
                CIRToLLVMVecCmpOpLowering,
-               CIRToLLVMVecShuffleDynamicOpLowering
+               CIRToLLVMVecSplatOpLowering,
+               CIRToLLVMVecShuffleOpLowering,
+               CIRToLLVMVecShuffleDynamicOpLowering,
+               CIRToLLVMVecTernaryOpLowering,
+               CIRToLLVMComplexCreateOpLowering
       // clang-format on
       >(converter, patterns.getContext());
 
@@ -1745,6 +1873,7 @@ mlir::LogicalResult CIRToLLVMGetMemberOpLowering::matchAndRewrite(
   assert(recordTy && "expected record type");
 
   switch (recordTy.getKind()) {
+  case cir::RecordType::Class:
   case cir::RecordType::Struct: {
     // Since the base address is a pointer to an aggregate, the first offset
     // is always zero. The second offset tell us which member it will access.
@@ -1836,10 +1965,6 @@ mlir::LogicalResult CIRToLLVMVecInsertOpLowering::matchAndRewrite(
 mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
     cir::VecCmpOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
-  assert(mlir::isa<cir::VectorType>(op.getType()) &&
-         mlir::isa<cir::VectorType>(op.getLhs().getType()) &&
-         mlir::isa<cir::VectorType>(op.getRhs().getType()) &&
-         "Vector compare with non-vector type");
   mlir::Type elementType = elementTypeIfVector(op.getLhs().getType());
   mlir::Value bitResult;
   if (auto intType = mlir::dyn_cast<cir::IntType>(elementType)) {
@@ -1862,6 +1987,73 @@ mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite(
+    cir::VecSplatOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  // Vector splat can be implemented with an `insertelement` and a
+  // `shufflevector`, which is better than an `insertelement` for each
+  // element in the vector. Start with an undef vector. Insert the value into
+  // the first element. Then use a `shufflevector` with a mask of all 0 to
+  // fill out the entire vector with that value.
+  cir::VectorType vecTy = op.getType();
+  mlir::Type llvmTy = typeConverter->convertType(vecTy);
+  mlir::Location loc = op.getLoc();
+  mlir::Value poison = rewriter.create<mlir::LLVM::PoisonOp>(loc, llvmTy);
+
+  mlir::Value elementValue = adaptor.getValue();
+  if (mlir::isa<mlir::LLVM::PoisonOp>(elementValue.getDefiningOp())) {
+    // If the splat value is poison, then we can just use poison value
+    // for the entire vector.
+    rewriter.replaceOp(op, poison);
+    return mlir::success();
+  }
+
+  if (auto constValue =
+          dyn_cast<mlir::LLVM::ConstantOp>(elementValue.getDefiningOp())) {
+    if (auto intAttr = dyn_cast<mlir::IntegerAttr>(constValue.getValue())) {
+      mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get(
+          mlir::cast<mlir::ShapedType>(llvmTy), intAttr.getValue());
+      rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
+          op, denseVec.getType(), denseVec);
+      return mlir::success();
+    }
+
+    if (auto fpAttr = dyn_cast<mlir::FloatAttr>(constValue.getValue())) {
+      mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get(
+          mlir::cast<mlir::ShapedType>(llvmTy), fpAttr.getValue());
+      rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
+          op, denseVec.getType(), denseVec);
+      return mlir::success();
+    }
+  }
+
+  mlir::Value indexValue =
+      rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), 0);
+  mlir::Value oneElement = rewriter.create<mlir::LLVM::InsertElementOp>(
+      loc, poison, elementValue, indexValue);
+  SmallVector<int32_t> zeroValues(vecTy.getSize(), 0);
+  rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement,
+                                                           poison, zeroValues);
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMVecShuffleOpLowering::matchAndRewrite(
+    cir::VecShuffleOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  // LLVM::ShuffleVectorOp takes an ArrayRef of int for the list of indices.
+  // Convert the ClangIR ArrayAttr of IntAttr constants into a
+  // SmallVector<int>.
+  SmallVector<int, 8> indices;
+  std::transform(
+      op.getIndices().begin(), op.getIndices().end(),
+      std::back_inserter(indices), [](mlir::Attribute intAttr) {
+        return mlir::cast<cir::IntAttr>(intAttr).getValue().getSExtValue();
+      });
+  rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(
+      op, adaptor.getVec1(), adaptor.getVec2(), indices);
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite(
     cir::VecShuffleDynamicOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -1916,6 +2108,38 @@ mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite(
+    cir::VecTernaryOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  // Convert `cond` into a vector of i1, then use that in a `select` op.
+  mlir::Value bitVec = rewriter.create<mlir::LLVM::ICmpOp>(
+      op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(),
+      rewriter.create<mlir::LLVM::ZeroOp>(
+          op.getCond().getLoc(),
+          typeConverter->convertType(op.getCond().getType())));
+  rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
+      op, bitVec, adaptor.getLhs(), adaptor.getRhs());
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite(
+    cir::ComplexCreateOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Type complexLLVMTy =
+      getTypeConverter()->convertType(op.getResult().getType());
+  auto initialComplex =
+      rewriter.create<mlir::LLVM::UndefOp>(op->getLoc(), complexLLVMTy);
+
+  auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>(
+      op->getLoc(), initialComplex, adaptor.getReal(), 0);
+
+  auto complex = rewriter.create<mlir::LLVM::InsertValueOp>(
+      op->getLoc(), realComplex, adaptor.getImag(), 1);
+
+  rewriter.replaceOp(op, complex);
+  return mlir::success();
+}
+
 std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
   return std::make_unique<ConvertCIRToLLVMPass>();
 }
diff --git a/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index 6b8862db2c8b..a80981806354 100644
--- a/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/external/llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -13,6 +13,7 @@
 #define CLANG_CIR_LOWERTOLLVM_H
 
 #include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 
@@ -192,6 +193,10 @@ class CIRToLLVMGlobalOpLowering
 
   void setupRegionInitializedLLVMGlobalOp(
       cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const;
+
+  mutable mlir::LLVM::ComdatOp comdatOp = nullptr;
+  mlir::SymbolRefAttr getComdatAttr(cir::GlobalOp &op,
+                                    mlir::OpBuilder &builder) const;
 };
 
 class CIRToLLVMUnaryOpLowering
@@ -292,6 +297,16 @@ class CIRToLLVMPtrStrideOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMBaseClassAddrOpLowering
+    : public mlir::OpConversionPattern<cir::BaseClassAddrOp> {
+public:
+  using mlir::OpConversionPattern<cir::BaseClassAddrOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BaseClassAddrOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMStackSaveOpLowering
     : public mlir::OpConversionPattern<cir::StackSaveOp> {
 public:
@@ -352,6 +367,26 @@ class CIRToLLVMVecCmpOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMVecSplatOpLowering
+    : public mlir::OpConversionPattern<cir::VecSplatOp> {
+public:
+  using mlir::OpConversionPattern<cir::VecSplatOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VecSplatOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMVecShuffleOpLowering
+    : public mlir::OpConversionPattern<cir::VecShuffleOp> {
+public:
+  using mlir::OpConversionPattern<cir::VecShuffleOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VecShuffleOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMVecShuffleDynamicOpLowering
     : public mlir::OpConversionPattern<cir::VecShuffleDynamicOp> {
 public:
@@ -363,6 +398,26 @@ class CIRToLLVMVecShuffleDynamicOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMVecTernaryOpLowering
+    : public mlir::OpConversionPattern<cir::VecTernaryOp> {
+public:
+  using mlir::OpConversionPattern<cir::VecTernaryOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VecTernaryOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMComplexCreateOpLowering
+    : public mlir::OpConversionPattern<cir::ComplexCreateOp> {
+public:
+  using mlir::OpConversionPattern<cir::ComplexCreateOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::ComplexCreateOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 } // namespace direct
 } // namespace cir
 
diff --git a/external/llvm-project/clang/lib/CodeGen/BackendUtil.cpp b/external/llvm-project/clang/lib/CodeGen/BackendUtil.cpp
index cd5fc48c4a22..7e0a3cf5591c 100644
--- a/external/llvm-project/clang/lib/CodeGen/BackendUtil.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/BackendUtil.cpp
@@ -69,7 +69,8 @@
 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
 #include "llvm/Transforms/Instrumentation/KCFI.h"
 #include "llvm/Transforms/Instrumentation/LowerAllowCheckPass.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 #include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
@@ -123,17 +124,10 @@ namespace clang {
 extern llvm::cl::opt<bool> ClSanitizeGuardChecks;
 }
 
-// Default filename used for profile generation.
-static std::string getDefaultProfileGenName() {
-  return DebugInfoCorrelate || ProfileCorrelate != InstrProfCorrelator::NONE
-             ? "default_%m.proflite"
-             : "default_%m.profraw";
-}
-
 // Path and name of file used for profile generation
 static std::string getProfileGenName(const CodeGenOptions &CodeGenOpts) {
   std::string FileName = CodeGenOpts.InstrProfileOutput.empty()
-                             ? getDefaultProfileGenName()
+                             ? llvm::driver::getDefaultProfileGenName()
                              : CodeGenOpts.InstrProfileOutput;
   if (CodeGenOpts.ContinuousProfileSync)
     FileName = "%c" + FileName;
@@ -1351,7 +1345,7 @@ runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex,
   // Context sensitive profile.
   if (CGOpts.hasProfileCSIRInstr()) {
     Conf.RunCSIRInstr = true;
-    Conf.CSIRProfile = std::move(CGOpts.InstrProfileOutput);
+    Conf.CSIRProfile = getProfileGenName(CGOpts);
   } else if (CGOpts.hasProfileCSIRUse()) {
     Conf.RunCSIRInstr = false;
     Conf.CSIRProfile = std::move(CGOpts.ProfileInstrumentUsePath);
diff --git a/external/llvm-project/clang/lib/CodeGen/CGBlocks.cpp b/external/llvm-project/clang/lib/CodeGen/CGBlocks.cpp
index adb353d568cd..729758ddce56 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGBlocks.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGBlocks.cpp
@@ -17,6 +17,7 @@
 #include "CGOpenCLRuntime.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenPGO.h"
 #include "ConstantEmitter.h"
 #include "TargetInfo.h"
 #include "clang/AST/Attr.h"
@@ -1522,7 +1523,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction(
   if (IsLambdaConversionToBlock)
     EmitLambdaBlockInvokeBody();
   else {
-    PGO.assignRegionCounters(GlobalDecl(blockDecl), fn);
+    PGO->assignRegionCounters(GlobalDecl(blockDecl), fn);
     incrementProfileCounter(blockDecl->getBody());
     EmitStmt(blockDecl->getBody());
   }
diff --git a/external/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp b/external/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp
index a11fb8d6093c..803fafcf8bc9 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2008,11 +2008,12 @@ Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
   if (!SanOpts.has(SanitizerKind::Builtin))
     return ArgValue;
 
-  SanitizerScope SanScope(this);
+  auto CheckOrdinal = SanitizerKind::SO_Builtin;
+  auto CheckHandler = SanitizerHandler::InvalidBuiltin;
+  SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
   Value *Cond = Builder.CreateICmpNE(
       ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
-  EmitCheck(std::make_pair(Cond, SanitizerKind::SO_Builtin),
-            SanitizerHandler::InvalidBuiltin,
+  EmitCheck(std::make_pair(Cond, CheckOrdinal), CheckHandler,
             {EmitCheckSourceLocation(E->getExprLoc()),
              llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
             {});
@@ -2024,10 +2025,11 @@ Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
   if (!SanOpts.has(SanitizerKind::Builtin))
     return ArgValue;
 
-  SanitizerScope SanScope(this);
+  auto CheckOrdinal = SanitizerKind::SO_Builtin;
+  auto CheckHandler = SanitizerHandler::InvalidBuiltin;
+  SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
   EmitCheck(
-      std::make_pair(ArgValue, SanitizerKind::SO_Builtin),
-      SanitizerHandler::InvalidBuiltin,
+      std::make_pair(ArgValue, CheckOrdinal), CheckHandler,
       {EmitCheckSourceLocation(E->getExprLoc()),
        llvm::ConstantInt::get(Builder.getInt8Ty(), BCK_AssumePassedFalse)},
       std::nullopt);
@@ -2050,7 +2052,15 @@ static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
       return EmitAbs(CGF, ArgValue, true);
   }
 
-  CodeGenFunction::SanitizerScope SanScope(&CGF);
+  SmallVector<SanitizerKind::SanitizerOrdinal, 1> Ordinals;
+  SanitizerHandler CheckHandler;
+  if (SanitizeOverflow) {
+    Ordinals.push_back(SanitizerKind::SO_SignedIntegerOverflow);
+    CheckHandler = SanitizerHandler::NegateOverflow;
+  } else
+    CheckHandler = SanitizerHandler::SubOverflow;
+
+  SanitizerDebugLocation SanScope(&CGF, Ordinals, CheckHandler);
 
   Constant *Zero = Constant::getNullValue(ArgValue->getType());
   Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
@@ -2062,12 +2072,12 @@ static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
   // TODO: support -ftrapv-handler.
   if (SanitizeOverflow) {
     CGF.EmitCheck({{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
-                  SanitizerHandler::NegateOverflow,
+                  CheckHandler,
                   {CGF.EmitCheckSourceLocation(E->getArg(0)->getExprLoc()),
                    CGF.EmitCheckTypeDescriptor(E->getType())},
                   {ArgValue});
   } else
-    CGF.EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
+    CGF.EmitTrapCheck(NotOverflow, CheckHandler);
 
   Value *CmpResult = CGF.Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
   return CGF.Builder.CreateSelect(CmpResult, Result, ArgValue, "abs");
@@ -5621,6 +5631,18 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(Result);
   }
 
+  case Builtin::BI__builtin_get_vtable_pointer: {
+    const Expr *Target = E->getArg(0);
+    QualType TargetType = Target->getType();
+    const CXXRecordDecl *Decl = TargetType->getPointeeCXXRecordDecl();
+    assert(Decl);
+    auto ThisAddress = EmitPointerWithAlignment(Target);
+    assert(ThisAddress.isValid());
+    llvm::Value *VTablePointer =
+        GetVTablePtr(ThisAddress, Int8PtrTy, Decl, VTableAuthMode::MustTrap);
+    return RValue::get(VTablePointer);
+  }
+
   case Builtin::BI__exception_code:
   case Builtin::BI_exception_code:
     return RValue::get(EmitSEHExceptionCode());
diff --git a/external/llvm-project/clang/lib/CodeGen/CGCall.cpp b/external/llvm-project/clang/lib/CodeGen/CGCall.cpp
index 3c0bf64947c3..4fa3ce84e96b 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGCall.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGCall.cpp
@@ -21,6 +21,7 @@
 #include "CGRecordLayout.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenPGO.h"
 #include "TargetInfo.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
@@ -80,12 +81,19 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
     return llvm::CallingConv::AArch64_VectorCall;
   case CC_AArch64SVEPCS:
     return llvm::CallingConv::AArch64_SVE_VectorCall;
-  case CC_AMDGPUKernelCall:
-    return llvm::CallingConv::AMDGPU_KERNEL;
   case CC_SpirFunction:
     return llvm::CallingConv::SPIR_FUNC;
-  case CC_OpenCLKernel:
-    return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
+  case CC_DeviceKernel: {
+    if (CGM.getLangOpts().OpenCL)
+      return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
+    if (CGM.getTriple().isSPIROrSPIRV())
+      return llvm::CallingConv::SPIR_KERNEL;
+    if (CGM.getTriple().isAMDGPU())
+      return llvm::CallingConv::AMDGPU_KERNEL;
+    if (CGM.getTriple().isNVPTX())
+      return llvm::CallingConv::PTX_Kernel;
+    llvm_unreachable("Unknown kernel calling convention");
+  }
   case CC_PreserveMost:
     return llvm::CallingConv::PreserveMost;
   case CC_PreserveAll:
@@ -283,8 +291,8 @@ static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
   if (D->hasAttr<AArch64SVEPcsAttr>())
     return CC_AArch64SVEPCS;
 
-  if (D->hasAttr<AMDGPUKernelCallAttr>())
-    return CC_AMDGPUKernelCall;
+  if (D->hasAttr<DeviceKernelAttr>())
+    return CC_DeviceKernel;
 
   if (D->hasAttr<IntelOclBiccAttr>())
     return CC_IntelOclBicc;
@@ -532,7 +540,7 @@ CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
   assert(isa<FunctionType>(FTy));
   setCUDAKernelCallingConvention(FTy, CGM, FD);
 
-  if (FD->hasAttr<OpenCLKernelAttr>() &&
+  if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()) &&
       GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
     const FunctionType *FT = FTy->getAs<FunctionType>();
     CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
@@ -760,7 +768,7 @@ CodeGenTypes::arrangeSYCLKernelCallerDeclaration(QualType resultType,
 
   return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,
                                  argTypes,
-                                 FunctionType::ExtInfo(CC_OpenCLKernel),
+                                 FunctionType::ExtInfo(CC_DeviceKernel),
                                  /*paramInfos=*/{}, RequiredArgs::All);
 }
 
@@ -1425,7 +1433,8 @@ void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst,
         SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) {
       // If the value is supposed to be a pointer, convert it before storing it.
       Src = CoerceIntOrPtrToIntOrPtr(Src, Dst.getElementType(), *this);
-      Builder.CreateStore(Src, Dst, DstIsVolatile);
+      auto *I = Builder.CreateStore(Src, Dst, DstIsVolatile);
+      addInstToCurrentSourceAtom(I, Src);
     } else if (llvm::StructType *STy =
                    dyn_cast<llvm::StructType>(Src->getType())) {
       // Prefer scalar stores to first-class aggregate stores.
@@ -1433,16 +1442,21 @@ void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst,
       for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
         Address EltPtr = Builder.CreateStructGEP(Dst, i);
         llvm::Value *Elt = Builder.CreateExtractValue(Src, i);
-        Builder.CreateStore(Elt, EltPtr, DstIsVolatile);
+        auto *I = Builder.CreateStore(Elt, EltPtr, DstIsVolatile);
+        addInstToCurrentSourceAtom(I, Elt);
       }
     } else {
-      Builder.CreateStore(Src, Dst.withElementType(SrcTy), DstIsVolatile);
+      auto *I =
+          Builder.CreateStore(Src, Dst.withElementType(SrcTy), DstIsVolatile);
+      addInstToCurrentSourceAtom(I, Src);
     }
   } else if (SrcTy->isIntegerTy()) {
     // If the source is a simple integer, coerce it directly.
     llvm::Type *DstIntTy = Builder.getIntNTy(DstSize.getFixedValue() * 8);
     Src = CoerceIntOrPtrToIntOrPtr(Src, DstIntTy, *this);
-    Builder.CreateStore(Src, Dst.withElementType(DstIntTy), DstIsVolatile);
+    auto *I =
+        Builder.CreateStore(Src, Dst.withElementType(DstIntTy), DstIsVolatile);
+    addInstToCurrentSourceAtom(I, Src);
   } else {
     // Otherwise do coercion through memory. This is stupid, but
     // simple.
@@ -1456,10 +1470,11 @@ void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst,
     RawAddress Tmp =
         CreateTempAllocaForCoercion(*this, SrcTy, Dst.getAlignment());
     Builder.CreateStore(Src, Tmp);
-    Builder.CreateMemCpy(Dst.emitRawPointer(*this),
-                         Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
-                         Tmp.getAlignment().getAsAlign(),
-                         Builder.CreateTypeSize(IntPtrTy, DstSize));
+    auto *I = Builder.CreateMemCpy(
+        Dst.emitRawPointer(*this), Dst.getAlignment().getAsAlign(),
+        Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
+        Builder.CreateTypeSize(IntPtrTy, DstSize));
+    addInstToCurrentSourceAtom(I, Src);
   }
 }
 
@@ -2357,9 +2372,8 @@ static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
 
   if (llvm::StructType *ST = dyn_cast<llvm::StructType>(IRTy)) {
     return !IsReturn && AI.getCanBeFlattened() &&
-           llvm::all_of(ST->elements(), [](llvm::Type *Ty) {
-             return llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty);
-           });
+           llvm::all_of(ST->elements(),
+                        llvm::AttributeFuncs::isNoFPClassCompatibleType);
   }
 
   return false;
@@ -2529,7 +2543,8 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
                                  NumElemsParam);
     }
 
-    if (TargetDecl->hasAttr<OpenCLKernelAttr>() &&
+    if (DeviceKernelAttr::isOpenCLSpelling(
+            TargetDecl->getAttr<DeviceKernelAttr>()) &&
         CallingConv != CallingConv::CC_C &&
         CallingConv != CallingConv::CC_SpirFunction) {
       // Check CallingConv to avoid adding uniform-work-group-size attribute to
@@ -2912,7 +2927,9 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
     // > For arguments to a __kernel function declared to be a pointer to a
     // > data type, the OpenCL compiler can assume that the pointee is always
     // > appropriately aligned as required by the data type.
-    if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>() &&
+    if (TargetDecl &&
+        DeviceKernelAttr::isOpenCLSpelling(
+            TargetDecl->getAttr<DeviceKernelAttr>()) &&
         ParamType->isPointerType()) {
       QualType PTy = ParamType->getPointeeType();
       if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
@@ -3933,9 +3950,9 @@ llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
   return R;
 }
 
-void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
-                                         bool EmitRetDbgLoc,
-                                         SourceLocation EndLoc) {
+void CodeGenFunction::EmitFunctionEpilog(
+    const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
+    uint64_t RetKeyInstructionsSourceAtom) {
   if (FI.isNoReturn()) {
     // Noreturn functions don't return.
     EmitUnreachable(EndLoc);
@@ -3950,7 +3967,11 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
 
   // Functions with no result always return void.
   if (!ReturnValue.isValid()) {
-    Builder.CreateRetVoid();
+    auto *I = Builder.CreateRetVoid();
+    if (RetKeyInstructionsSourceAtom)
+      addInstToSpecificSourceAtom(I, nullptr, RetKeyInstructionsSourceAtom);
+    else
+      addInstToNewSourceAtom(I, nullptr);
     return;
   }
 
@@ -4130,6 +4151,12 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
 
   if (RetDbgLoc)
     Ret->setDebugLoc(std::move(RetDbgLoc));
+
+  llvm::Value *Backup = RV ? Ret->getOperand(0) : nullptr;
+  if (RetKeyInstructionsSourceAtom)
+    addInstToSpecificSourceAtom(Ret, Backup, RetKeyInstructionsSourceAtom);
+  else
+    addInstToNewSourceAtom(Ret, Backup);
 }
 
 void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
@@ -4168,7 +4195,7 @@ void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
     Handler = SanitizerHandler::NullabilityReturn;
   }
 
-  SanitizerScope SanScope(this);
+  SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
 
   // Make sure the "return" source location is valid. If we're checking a
   // nullability annotation, make sure the preconditions for the check are met.
@@ -4553,7 +4580,7 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
     Handler = SanitizerHandler::NullabilityArg;
   }
 
-  SanitizerScope SanScope(this);
+  SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
   llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
   llvm::Constant *StaticData[] = {
       EmitCheckSourceLocation(ArgLoc),
@@ -5912,7 +5939,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // For more details, see the comment before the definition of
   // IPVK_IndirectCallTarget in InstrProfData.inc.
   if (!CI->getCalledFunction())
-    PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget, CI, CalleePtr);
+    PGO->valueProfile(Builder, llvm::IPVK_IndirectCallTarget, CI, CalleePtr);
 
   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
   // optimizer it can aggressively ignore unwind edges.
diff --git a/external/llvm-project/clang/lib/CodeGen/CGClass.cpp b/external/llvm-project/clang/lib/CodeGen/CGClass.cpp
index 0a3447817652..8d1e57ab2aaf 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGClass.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGClass.cpp
@@ -2820,7 +2820,8 @@ void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
     RD = LeastDerivedClassWithSameLayout(RD);
 
   auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
-  ApplyDebugLocation ApplyTrapDI(*this, SanitizerAnnotateDebugInfo(Ordinal));
+  SanitizerDebugLocation SanScope(this, {Ordinal},
+                                  SanitizerHandler::CFICheckFail);
 
   EmitVTablePtrCheck(RD, VTable, TCK, Loc);
 }
@@ -2845,7 +2846,8 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
     ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
 
   auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
-  ApplyDebugLocation ApplyTrapDI(*this, SanitizerAnnotateDebugInfo(Ordinal));
+  SanitizerDebugLocation SanScope(this, {Ordinal},
+                                  SanitizerHandler::CFICheckFail);
 
   llvm::BasicBlock *ContBlock = nullptr;
 
@@ -2877,6 +2879,8 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
                                          llvm::Value *VTable,
                                          CFITypeCheckKind TCK,
                                          SourceLocation Loc) {
+  assert(IsSanitizerScope);
+
   if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
       !CGM.HasHiddenLTOVisibility(RD))
     return;
@@ -2888,7 +2892,6 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
           SanitizerMask::bitPosToMask(M), TypeName))
     return;
 
-  SanitizerScope SanScope(this);
   EmitSanitizerStatReport(SSK);
 
   llvm::Metadata *MD =
@@ -2945,11 +2948,11 @@ bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
 llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
     const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
     uint64_t VTableByteOffset) {
-  SanitizerScope SanScope(this);
+  auto CheckOrdinal = SanitizerKind::SO_CFIVCall;
+  auto CheckHandler = SanitizerHandler::CFICheckFail;
+  SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
 
   EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
-  ApplyDebugLocation ApplyTrapDI(
-      *this, SanitizerAnnotateDebugInfo(SanitizerKind::SO_CFIVCall));
 
   llvm::Metadata *MD =
       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
@@ -2968,8 +2971,7 @@ llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
   if (SanOpts.has(SanitizerKind::CFIVCall) &&
       !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
                                                      TypeName)) {
-    EmitCheck(std::make_pair(CheckResult, SanitizerKind::SO_CFIVCall),
-              SanitizerHandler::CFICheckFail, {}, {});
+    EmitCheck(std::make_pair(CheckResult, CheckOrdinal), CheckHandler, {}, {});
   }
 
   return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0),
diff --git a/external/llvm-project/clang/lib/CodeGen/CGCleanup.cpp b/external/llvm-project/clang/lib/CodeGen/CGCleanup.cpp
index 7e1c5b7da955..4ed2c5183c47 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGCleanup.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGCleanup.cpp
@@ -1118,6 +1118,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
 
   // Create the branch.
   llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
+  addInstToCurrentSourceAtom(BI, nullptr);
 
   // Calculate the innermost active normal cleanup.
   EHScopeStack::stable_iterator
diff --git a/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp b/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp
index 4ed0305ce1d5..b446114ce545 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -58,6 +58,13 @@
 using namespace clang;
 using namespace clang::CodeGen;
 
+// TODO: consider deprecating ClArrayBoundsPseudoFn; functionality is subsumed
+//       by -fsanitize-annotate-debug-info
+static llvm::cl::opt<bool> ClArrayBoundsPseudoFn(
+    "array-bounds-pseudofn", llvm::cl::Hidden, llvm::cl::Optional,
+    llvm::cl::desc("Emit debug info that places array-bounds instrumentation "
+                   "in an inline function called __ubsan_check_array_bounds."));
+
 static uint32_t getTypeAlignIfRequired(const Type *Ty, const ASTContext &Ctx) {
   auto TI = Ctx.getTypeInfo(Ty);
   if (TI.isAlignRequired())
@@ -1608,28 +1615,7 @@ llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty,
 
   SourceLocation Loc = AliasDecl->getLocation();
 
-  if (CGM.getCodeGenOpts().DebugTemplateAlias &&
-      // FIXME: This is a workaround for the issue
-      //        https://github.com/llvm/llvm-project/issues/89774
-      // The TemplateSpecializationType doesn't contain any instantiation
-      // information; dependent template arguments can't be resolved. For now,
-      // fall back to DW_TAG_typedefs for template aliases that are
-      // instantiation dependent, e.g.:
-      // ```
-      // template <int>
-      // using A = int;
-      //
-      // template<int I>
-      // struct S {
-      //   using AA = A<I>; // Instantiation dependent.
-      //   AA aa;
-      // };
-      //
-      // S<0> s;
-      // ```
-      // S::AA's underlying type A<I> is dependent on I so will be emitted as a
-      // DW_TAG_typedef.
-      !Ty->isInstantiationDependentType()) {
+  if (CGM.getCodeGenOpts().DebugTemplateAlias) {
     auto ArgVector = ::GetTemplateArgs(TD, Ty);
     TemplateArgs Args = {TD->getTemplateParameters(), ArgVector};
 
@@ -1743,9 +1729,8 @@ static unsigned getDwarfCC(CallingConv CC) {
     return llvm::dwarf::DW_CC_LLVM_IntelOclBicc;
   case CC_SpirFunction:
     return llvm::dwarf::DW_CC_LLVM_SpirFunction;
-  case CC_OpenCLKernel:
-  case CC_AMDGPUKernelCall:
-    return llvm::dwarf::DW_CC_LLVM_OpenCLKernel;
+  case CC_DeviceKernel:
+    return llvm::dwarf::DW_CC_LLVM_DeviceKernel;
   case CC_Swift:
     return llvm::dwarf::DW_CC_LLVM_Swift;
   case CC_SwiftAsync:
@@ -4821,7 +4806,7 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc,
 
   // Preserve btf_decl_tag attributes for parameters of extern functions
   // for BPF target. The parameters created in this loop are attached as
-  // DISubprogram's retainedNodes in the subsequent finalizeSubprogram call.
+  // DISubprogram's retainedNodes in the DIBuilder::finalize() call.
   if (IsDeclForCallSite && CGM.getTarget().getTriple().isBPF()) {
     if (auto *FD = dyn_cast<FunctionDecl>(D)) {
       llvm::DITypeRefArray ParamTypes = STy->getTypeArray();
@@ -4839,8 +4824,6 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc,
 
   if (IsDeclForCallSite)
     Fn->setSubprogram(SP);
-
-  DBuilder.finalizeSubprogram(SP);
 }
 
 void CGDebugInfo::EmitFuncDeclForCallSite(llvm::CallBase *CallOrInvoke,
@@ -6983,3 +6966,78 @@ CodeGenFunction::LexicalScope::~LexicalScope() {
     ForceCleanup();
   }
 }
+
+static std::string SanitizerHandlerToCheckLabel(SanitizerHandler Handler) {
+  std::string Label;
+  switch (Handler) {
+#define SANITIZER_CHECK(Enum, Name, Version)                                   \
+  case Enum:                                                                   \
+    Label = "__ubsan_check_" #Name;                                            \
+    break;
+
+    LIST_SANITIZER_CHECKS
+#undef SANITIZER_CHECK
+  };
+
+  // Label doesn't require sanitization
+  return Label;
+}
+
+static std::string
+SanitizerOrdinalToCheckLabel(SanitizerKind::SanitizerOrdinal Ordinal) {
+  std::string Label;
+  switch (Ordinal) {
+#define SANITIZER(NAME, ID)                                                    \
+  case SanitizerKind::SO_##ID:                                                 \
+    Label = "__ubsan_check_" NAME;                                             \
+    break;
+#include "clang/Basic/Sanitizers.def"
+  default:
+    llvm_unreachable("unexpected sanitizer kind");
+  }
+
+  // Sanitize label (convert hyphens to underscores; also futureproof against
+  // non-alpha)
+  for (unsigned int i = 0; i < Label.length(); i++)
+    if (!std::isalpha(Label[i]))
+      Label[i] = '_';
+
+  return Label;
+}
+
+llvm::DILocation *CodeGenFunction::SanitizerAnnotateDebugInfo(
+    ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
+    SanitizerHandler Handler) {
+  std::string Label;
+  if (Ordinals.size() == 1)
+    Label = SanitizerOrdinalToCheckLabel(Ordinals[0]);
+  else
+    Label = SanitizerHandlerToCheckLabel(Handler);
+
+  llvm::DILocation *CheckDI = Builder.getCurrentDebugLocation();
+
+  for (auto Ord : Ordinals) {
+    // TODO: deprecate ClArrayBoundsPseudoFn
+    if (((ClArrayBoundsPseudoFn && Ord == SanitizerKind::SO_ArrayBounds) ||
+         CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo.has(Ord)) &&
+        CheckDI) {
+      return getDebugInfo()->CreateSyntheticInlineAt(CheckDI, Label);
+    }
+  }
+
+  return CheckDI;
+}
+
+SanitizerDebugLocation::SanitizerDebugLocation(
+    CodeGenFunction *CGF, ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
+    SanitizerHandler Handler)
+    : CGF(CGF),
+      Apply(*CGF, CGF->SanitizerAnnotateDebugInfo(Ordinals, Handler)) {
+  assert(!CGF->IsSanitizerScope);
+  CGF->IsSanitizerScope = true;
+}
+
+SanitizerDebugLocation::~SanitizerDebugLocation() {
+  assert(CGF->IsSanitizerScope);
+  CGF->IsSanitizerScope = false;
+}
diff --git a/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.h b/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.h
index 9d61bc9503de..6a1b0f6f361c 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.h
+++ b/external/llvm-project/clang/lib/CodeGen/CGDebugInfo.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_LIB_CODEGEN_CGDEBUGINFO_H
 
 #include "CGBuilder.h"
+#include "SanitizerHandler.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExternalASTSource.h"
@@ -1010,6 +1011,17 @@ class ApplyInlineDebugLocation {
   ~ApplyInlineDebugLocation();
 };
 
+class SanitizerDebugLocation {
+  CodeGenFunction *CGF;
+  ApplyDebugLocation Apply;
+
+public:
+  SanitizerDebugLocation(CodeGenFunction *CGF,
+                         ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
+                         SanitizerHandler Handler);
+  ~SanitizerDebugLocation();
+};
+
 } // namespace CodeGen
 } // namespace clang
 
diff --git a/external/llvm-project/clang/lib/CodeGen/CGDecl.cpp b/external/llvm-project/clang/lib/CodeGen/CGDecl.cpp
index 2fb1db9df080..0b3dd5028aa6 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGDecl.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGDecl.cpp
@@ -18,6 +18,7 @@
 #include "CGOpenMPRuntime.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenPGO.h"
 #include "ConstantEmitter.h"
 #include "EHScopeStack.h"
 #include "PatternInit.h"
@@ -368,7 +369,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
     return GV;
   }
 
-  PGO.markStmtMaybeUsed(D.getInit()); // FIXME: Too lazy
+  PGO->markStmtMaybeUsed(D.getInit()); // FIXME: Too lazy
 
 #ifndef NDEBUG
   CharUnits VarSize = CGM.getContext().getTypeSizeInChars(D.getType()) +
@@ -771,14 +772,15 @@ void CodeGenFunction::EmitNullabilityCheck(LValue LHS, llvm::Value *RHS,
 
   // Check if the right hand side of the assignment is nonnull, if the left
   // hand side must be nonnull.
-  SanitizerScope SanScope(this);
+  auto CheckOrdinal = SanitizerKind::SO_NullabilityAssign;
+  auto CheckHandler = SanitizerHandler::TypeMismatch;
+  SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
   llvm::Value *IsNotNull = Builder.CreateIsNotNull(RHS);
   llvm::Constant *StaticData[] = {
       EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(LHS.getType()),
       llvm::ConstantInt::get(Int8Ty, 0), // The LogAlignment info is unused.
       llvm::ConstantInt::get(Int8Ty, TCK_NonnullAssign)};
-  EmitCheck({{IsNotNull, SanitizerKind::SO_NullabilityAssign}},
-            SanitizerHandler::TypeMismatch, StaticData, RHS);
+  EmitCheck({{IsNotNull, CheckOrdinal}}, CheckHandler, StaticData, RHS);
 }
 
 void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D,
@@ -1963,7 +1965,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
   // unless it contains a label.
   if (!HaveInsertPoint()) {
     if (!Init || !ContainsLabel(Init)) {
-      PGO.markStmtMaybeUsed(Init);
+      PGO->markStmtMaybeUsed(Init);
       return;
     }
     EnsureInsertPoint();
@@ -2076,7 +2078,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
     return EmitExprAsInit(Init, &D, lv, capturedByInit);
   }
 
-  PGO.markStmtMaybeUsed(Init);
+  PGO->markStmtMaybeUsed(Init);
 
   if (!emission.IsConstantAggregate) {
     // For simple scalar/complex initialization, store the value directly.
diff --git a/external/llvm-project/clang/lib/CodeGen/CGExpr.cpp b/external/llvm-project/clang/lib/CodeGen/CGExpr.cpp
index e8dad5dbeb96..5793c9fbdd93 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGExpr.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGExpr.cpp
@@ -21,6 +21,7 @@
 #include "CGRecordLayout.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenPGO.h"
 #include "ConstantEmitter.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
@@ -64,13 +65,6 @@ llvm::cl::opt<bool> ClSanitizeGuardChecks(
 
 } // namespace clang
 
-// TODO: consider deprecating ClArrayBoundsPseudoFn; functionality is subsumed
-//       by -fsanitize-annotate-debug-info
-static llvm::cl::opt<bool> ClArrayBoundsPseudoFn(
-    "array-bounds-pseudofn", llvm::cl::Hidden, llvm::cl::Optional,
-    llvm::cl::desc("Emit debug info that places array-bounds instrumentation "
-                   "in an inline function called __ubsan_check_array_bounds."));
-
 //===--------------------------------------------------------------------===//
 //                        Defines for metadata
 //===--------------------------------------------------------------------===//
@@ -207,7 +201,7 @@ RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
 /// expression and compare the result against zero, returning an Int1Ty value.
 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
-  PGO.setCurrentStmt(E);
+  PGO->setCurrentStmt(E);
   if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
     llvm::Value *MemPtr = EmitScalarExpr(E);
     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
@@ -750,104 +744,116 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
   if (Ty.isVolatileQualified())
     return;
 
-  SanitizerScope SanScope(this);
-
-  SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>
-      Checks;
-  llvm::BasicBlock *Done = nullptr;
-
   // Quickly determine whether we have a pointer to an alloca. It's possible
   // to skip null checks, and some alignment checks, for these pointers. This
   // can reduce compile-time significantly.
   auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
 
-  llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
   llvm::Value *IsNonNull = nullptr;
   bool IsGuaranteedNonNull =
       SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;
-  bool AllowNullPointers = isNullPointerAllowed(TCK);
-  if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
-      !IsGuaranteedNonNull) {
-    // The glvalue must not be an empty glvalue.
-    IsNonNull = Builder.CreateIsNotNull(Ptr);
-
-    // The IR builder can constant-fold the null check if the pointer points to
-    // a constant.
-    IsGuaranteedNonNull = IsNonNull == True;
 
-    // Skip the null check if the pointer is known to be non-null.
-    if (!IsGuaranteedNonNull) {
-      if (AllowNullPointers) {
-        // When performing pointer casts, it's OK if the value is null.
-        // Skip the remaining checks in that case.
-        Done = createBasicBlock("null");
-        llvm::BasicBlock *Rest = createBasicBlock("not.null");
-        Builder.CreateCondBr(IsNonNull, Rest, Done);
-        EmitBlock(Rest);
-      } else {
-        Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::SO_Null));
+  llvm::BasicBlock *Done = nullptr;
+  bool DoneViaNullSanitize = false;
+
+  {
+    auto CheckHandler = SanitizerHandler::TypeMismatch;
+    SanitizerDebugLocation SanScope(this,
+                                    {SanitizerKind::SO_Null,
+                                     SanitizerKind::SO_ObjectSize,
+                                     SanitizerKind::SO_Alignment},
+                                    CheckHandler);
+
+    SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>
+        Checks;
+
+    llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
+    bool AllowNullPointers = isNullPointerAllowed(TCK);
+    if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
+        !IsGuaranteedNonNull) {
+      // The glvalue must not be an empty glvalue.
+      IsNonNull = Builder.CreateIsNotNull(Ptr);
+
+      // The IR builder can constant-fold the null check if the pointer points
+      // to a constant.
+      IsGuaranteedNonNull = IsNonNull == True;
+
+      // Skip the null check if the pointer is known to be non-null.
+      if (!IsGuaranteedNonNull) {
+        if (AllowNullPointers) {
+          // When performing pointer casts, it's OK if the value is null.
+          // Skip the remaining checks in that case.
+          Done = createBasicBlock("null");
+          DoneViaNullSanitize = true;
+          llvm::BasicBlock *Rest = createBasicBlock("not.null");
+          Builder.CreateCondBr(IsNonNull, Rest, Done);
+          EmitBlock(Rest);
+        } else {
+          Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::SO_Null));
+        }
       }
     }
-  }
 
-  if (SanOpts.has(SanitizerKind::ObjectSize) &&
-      !SkippedChecks.has(SanitizerKind::ObjectSize) &&
-      !Ty->isIncompleteType()) {
-    uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
-    llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
-    if (ArraySize)
-      Size = Builder.CreateMul(Size, ArraySize);
-
-    // Degenerate case: new X[0] does not need an objectsize check.
-    llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
-    if (!ConstantSize || !ConstantSize->isNullValue()) {
-      // The glvalue must refer to a large enough storage region.
-      // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
-      //        to check this.
-      // FIXME: Get object address space
-      llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
-      llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
-      llvm::Value *Min = Builder.getFalse();
-      llvm::Value *NullIsUnknown = Builder.getFalse();
-      llvm::Value *Dynamic = Builder.getFalse();
-      llvm::Value *LargeEnough = Builder.CreateICmpUGE(
-          Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size);
-      Checks.push_back(
-          std::make_pair(LargeEnough, SanitizerKind::SO_ObjectSize));
+    if (SanOpts.has(SanitizerKind::ObjectSize) &&
+        !SkippedChecks.has(SanitizerKind::ObjectSize) &&
+        !Ty->isIncompleteType()) {
+      uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
+      llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
+      if (ArraySize)
+        Size = Builder.CreateMul(Size, ArraySize);
+
+      // Degenerate case: new X[0] does not need an objectsize check.
+      llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
+      if (!ConstantSize || !ConstantSize->isNullValue()) {
+        // The glvalue must refer to a large enough storage region.
+        // FIXME: If Address Sanitizer is enabled, insert dynamic
+        // instrumentation
+        //        to check this.
+        // FIXME: Get object address space
+        llvm::Type *Tys[2] = {IntPtrTy, Int8PtrTy};
+        llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
+        llvm::Value *Min = Builder.getFalse();
+        llvm::Value *NullIsUnknown = Builder.getFalse();
+        llvm::Value *Dynamic = Builder.getFalse();
+        llvm::Value *LargeEnough = Builder.CreateICmpUGE(
+            Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size);
+        Checks.push_back(
+            std::make_pair(LargeEnough, SanitizerKind::SO_ObjectSize));
+      }
     }
-  }
 
-  llvm::MaybeAlign AlignVal;
-  llvm::Value *PtrAsInt = nullptr;
-
-  if (SanOpts.has(SanitizerKind::Alignment) &&
-      !SkippedChecks.has(SanitizerKind::Alignment)) {
-    AlignVal = Alignment.getAsMaybeAlign();
-    if (!Ty->isIncompleteType() && !AlignVal)
-      AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
-                                             /*ForPointeeType=*/true)
-                     .getAsMaybeAlign();
-
-    // The glvalue must be suitably aligned.
-    if (AlignVal && *AlignVal > llvm::Align(1) &&
-        (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
-      PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
-      llvm::Value *Align = Builder.CreateAnd(
-          PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
-      llvm::Value *Aligned =
-          Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
-      if (Aligned != True)
-        Checks.push_back(std::make_pair(Aligned, SanitizerKind::SO_Alignment));
+    llvm::MaybeAlign AlignVal;
+    llvm::Value *PtrAsInt = nullptr;
+
+    if (SanOpts.has(SanitizerKind::Alignment) &&
+        !SkippedChecks.has(SanitizerKind::Alignment)) {
+      AlignVal = Alignment.getAsMaybeAlign();
+      if (!Ty->isIncompleteType() && !AlignVal)
+        AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
+                                               /*ForPointeeType=*/true)
+                       .getAsMaybeAlign();
+
+      // The glvalue must be suitably aligned.
+      if (AlignVal && *AlignVal > llvm::Align(1) &&
+          (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
+        PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
+        llvm::Value *Align = Builder.CreateAnd(
+            PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
+        llvm::Value *Aligned =
+            Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
+        if (Aligned != True)
+          Checks.push_back(
+              std::make_pair(Aligned, SanitizerKind::SO_Alignment));
+      }
     }
-  }
 
-  if (Checks.size() > 0) {
-    llvm::Constant *StaticData[] = {
-        EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),
-        llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
-        llvm::ConstantInt::get(Int8Ty, TCK)};
-    EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
-              PtrAsInt ? PtrAsInt : Ptr);
+    if (Checks.size() > 0) {
+      llvm::Constant *StaticData[] = {
+          EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),
+          llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
+          llvm::ConstantInt::get(Int8Ty, TCK)};
+      EmitCheck(Checks, CheckHandler, StaticData, PtrAsInt ? PtrAsInt : Ptr);
+    }
   }
 
   // If possible, check that the vptr indicates that there is a subobject of
@@ -860,6 +866,9 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
   //       or call a non-static member function
   if (SanOpts.has(SanitizerKind::Vptr) &&
       !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
+    SanitizerDebugLocation SanScope(this, {SanitizerKind::SO_Vptr},
+                                    SanitizerHandler::DynamicTypeCacheMiss);
+
     // Ensure that the pointer is non-null before loading it. If there is no
     // compile-time guarantee, reuse the run-time null check or emit a new one.
     if (!IsGuaranteedNonNull) {
@@ -928,6 +937,11 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
   }
 
   if (Done) {
+    SanitizerDebugLocation SanScope(
+        this,
+        {DoneViaNullSanitize ? SanitizerKind::SO_Null : SanitizerKind::SO_Vptr},
+        DoneViaNullSanitize ? SanitizerHandler::TypeMismatch
+                            : SanitizerHandler::DynamicTypeCacheMiss);
     Builder.CreateBr(Done);
     EmitBlock(Done);
   }
@@ -1226,10 +1240,9 @@ void CodeGenFunction::EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
   if (!Bound)
     return;
 
-  SanitizerScope SanScope(this);
-
   auto CheckKind = SanitizerKind::SO_ArrayBounds;
-  ApplyDebugLocation ApplyTrapDI(*this, SanitizerAnnotateDebugInfo(CheckKind));
+  auto CheckHandler = SanitizerHandler::OutOfBounds;
+  SanitizerDebugLocation SanScope(this, {CheckKind}, CheckHandler);
 
   bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
   llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
@@ -1242,37 +1255,7 @@ void CodeGenFunction::EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
   };
   llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)
                                 : Builder.CreateICmpULE(IndexVal, BoundVal);
-  EmitCheck(std::make_pair(Check, CheckKind), SanitizerHandler::OutOfBounds,
-            StaticData, Index);
-}
-
-llvm::DILocation *CodeGenFunction::SanitizerAnnotateDebugInfo(
-    SanitizerKind::SanitizerOrdinal CheckKindOrdinal) {
-  std::string Label;
-  switch (CheckKindOrdinal) {
-#define SANITIZER(NAME, ID)                                                    \
-  case SanitizerKind::SO_##ID:                                                 \
-    Label = "__ubsan_check_" NAME;                                             \
-    break;
-#include "clang/Basic/Sanitizers.def"
-  default:
-    llvm_unreachable("unexpected sanitizer kind");
-  }
-
-  // Sanitize label
-  for (unsigned int i = 0; i < Label.length(); i++)
-    if (!std::isalpha(Label[i]))
-      Label[i] = '_';
-
-  llvm::DILocation *CheckDI = Builder.getCurrentDebugLocation();
-  // TODO: deprecate ClArrayBoundsPseudoFn
-  if (((ClArrayBoundsPseudoFn &&
-        CheckKindOrdinal == SanitizerKind::SO_ArrayBounds) ||
-       CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo.has(CheckKindOrdinal)) &&
-      CheckDI)
-    CheckDI = getDebugInfo()->CreateSyntheticInlineAt(CheckDI, Label);
-
-  return CheckDI;
+  EmitCheck(std::make_pair(Check, CheckKind), CheckHandler, StaticData, Index);
 }
 
 CodeGenFunction::ComplexPairTy CodeGenFunction::
@@ -1996,8 +1979,12 @@ bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
   if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
     return true;
 
+  SanitizerKind::SanitizerOrdinal Kind =
+      NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;
+
   auto &Ctx = getLLVMContext();
-  SanitizerScope SanScope(this);
+  auto CheckHandler = SanitizerHandler::LoadInvalidValue;
+  SanitizerDebugLocation SanScope(this, {Kind}, CheckHandler);
   llvm::Value *Check;
   --End;
   if (!Min) {
@@ -2011,10 +1998,7 @@ bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
   }
   llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
                                   EmitCheckTypeDescriptor(Ty)};
-  SanitizerKind::SanitizerOrdinal Kind =
-      NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;
-  EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
-            StaticArgs, Value);
+  EmitCheck(std::make_pair(Check, Kind), CheckHandler, StaticArgs, Value);
   return true;
 }
 
@@ -3052,9 +3036,13 @@ static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
                                      GlobalDecl GD) {
   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
   llvm::Constant *V = CGF.CGM.getFunctionPointer(GD);
+  QualType ETy = E->getType();
+  if (ETy->isCFIUncheckedCalleeFunctionType()) {
+    if (auto *GV = dyn_cast<llvm::GlobalValue>(V))
+      V = llvm::NoCFIValue::get(GV);
+  }
   CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
-  return CGF.MakeAddrLValue(V, E->getType(), Alignment,
-                            AlignmentSource::Decl);
+  return CGF.MakeAddrLValue(V, ETy, Alignment, AlignmentSource::Decl);
 }
 
 static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
@@ -3933,7 +3921,17 @@ void CodeGenFunction::EmitCfiCheckStub() {
 // can be nullptr if the calling module has -fsanitize-trap behavior for this
 // check kind; in this case __cfi_check_fail traps as well.
 void CodeGenFunction::EmitCfiCheckFail() {
-  SanitizerScope SanScope(this);
+  auto CheckHandler = SanitizerHandler::CFICheckFail;
+  // TODO: the SanitizerKind is not yet determined for this check (and might
+  // not even be available, if Data == nullptr). However, we still want to
+  // annotate the instrumentation. We approximate this by using all the CFI
+  // kinds.
+  SanitizerDebugLocation SanScope(
+      this,
+      {SanitizerKind::SO_CFIVCall, SanitizerKind::SO_CFINVCall,
+       SanitizerKind::SO_CFIDerivedCast, SanitizerKind::SO_CFIUnrelatedCast,
+       SanitizerKind::SO_CFIICall},
+      CheckHandler);
   FunctionArgList Args;
   ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,
                             ImplicitParamKind::Other);
@@ -3996,8 +3994,6 @@ void CodeGenFunction::EmitCfiCheckFail() {
                          {Addr, AllVtables}),
       IntPtrTy);
 
-  // TODO: the instructions above are not annotated with debug info. It is
-  // inconvenient to do so because we have not determined SanitizerKind yet.
   const std::pair<int, SanitizerKind::SanitizerOrdinal> CheckKinds[] = {
       {CFITCK_VCall, SanitizerKind::SO_CFIVCall},
       {CFITCK_NVCall, SanitizerKind::SO_CFINVCall},
@@ -4009,7 +4005,8 @@ void CodeGenFunction::EmitCfiCheckFail() {
     int Kind = CheckKindOrdinalPair.first;
     SanitizerKind::SanitizerOrdinal Ordinal = CheckKindOrdinalPair.second;
 
-    ApplyDebugLocation ApplyTrapDI(*this, SanitizerAnnotateDebugInfo(Ordinal));
+    // TODO: we could apply SanitizerAnnotateDebugInfo(Ordinal) instead of
+    //       relying on the SanitizerScope with all CFI ordinals
 
     llvm::Value *Cond =
         Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));
@@ -4021,7 +4018,7 @@ void CodeGenFunction::EmitCfiCheckFail() {
       // Although the compiler allows SanitizeMergeHandlers to be set
       // independently of CGM.getLangOpts().Sanitize, Driver/SanitizerArgs.cpp
       // requires that SanitizeMergeHandlers is a subset of Sanitize.
-      EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail, /*NoMerge=*/false);
+      EmitTrapCheck(Cond, CheckHandler, /*NoMerge=*/false);
   }
 
   FinishFunction();
@@ -4032,11 +4029,12 @@ void CodeGenFunction::EmitCfiCheckFail() {
 
 void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
   if (SanOpts.has(SanitizerKind::Unreachable)) {
-    SanitizerScope SanScope(this);
+    auto CheckOrdinal = SanitizerKind::SO_Unreachable;
+    auto CheckHandler = SanitizerHandler::BuiltinUnreachable;
+    SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
     EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
-                             SanitizerKind::SO_Unreachable),
-              SanitizerHandler::BuiltinUnreachable,
-              EmitCheckSourceLocation(Loc), {});
+                             CheckOrdinal),
+              CheckHandler, EmitCheckSourceLocation(Loc), {});
   }
   Builder.CreateUnreachable();
 }
@@ -4590,7 +4588,32 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
         E->getType(), !getLangOpts().PointerOverflowDefined, SignedIndices,
         E->getExprLoc(), &arrayType, E->getBase());
     EltBaseInfo = ArrayLV.getBaseInfo();
-    EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
+    if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
+      // Since CodeGenTBAA::getTypeInfoHelper only handles array types for
+      // new struct path TBAA, we must a use a plain access.
+      EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
+    } else if (ArrayLV.getTBAAInfo().isMayAlias()) {
+      EltTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
+    } else if (ArrayLV.getTBAAInfo().isIncomplete()) {
+      // The array element is complete, even if the array is not.
+      EltTBAAInfo = CGM.getTBAAAccessInfo(E->getType());
+    } else {
+      // The TBAA access info from the array (base) lvalue is ordinary. We will
+      // adapt it to create access info for the element.
+      EltTBAAInfo = ArrayLV.getTBAAInfo();
+
+      // We retain the TBAA struct path (BaseType and Offset members) from the
+      // array. In the TBAA representation, we map any array access to the
+      // element at index 0, as the index is generally a runtime value. This
+      // element has the same offset in the base type as the array itself.
+      // If the array lvalue had no base type, there is no point trying to
+      // generate one, since an array itself is not a valid base type.
+
+      // We also retain the access type from the base lvalue, but the access
+      // size must be updated to the size of an individual element.
+      EltTBAAInfo.Size =
+          getContext().getTypeSizeInChars(E->getType()).getQuantity();
+    }
   } else {
     // The base must be a pointer; emit it with an estimate of its alignment.
     Address BaseAddr =
@@ -5914,7 +5937,7 @@ static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
 }
 
 static GlobalDecl getGlobalDeclForDirectCall(const FunctionDecl *FD) {
-  if (FD->hasAttr<OpenCLKernelAttr>())
+  if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()))
     return GlobalDecl(FD, KernelReferenceKind::Stub);
   return GlobalDecl(FD);
 }
@@ -6273,7 +6296,9 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType,
       !isa<FunctionNoProtoType>(PointeeType)) {
     if (llvm::Constant *PrefixSig =
             CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
-      SanitizerScope SanScope(this);
+      auto CheckOrdinal = SanitizerKind::SO_Function;
+      auto CheckHandler = SanitizerHandler::FunctionTypeMismatch;
+      SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
       auto *TypeHash = getUBSanFunctionTypeHash(PointeeType);
 
       llvm::Type *PrefixSigType = PrefixSig->getType();
@@ -6333,9 +6358,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType,
           Builder.CreateICmpEQ(CalleeTypeHash, TypeHash);
       llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()),
                                       EmitCheckTypeDescriptor(CalleeType)};
-      EmitCheck(std::make_pair(CalleeTypeHashMatch, SanitizerKind::SO_Function),
-                SanitizerHandler::FunctionTypeMismatch, StaticData,
-                {CalleePtr});
+      EmitCheck(std::make_pair(CalleeTypeHashMatch, CheckOrdinal), CheckHandler,
+                StaticData, {CalleePtr});
 
       Builder.CreateBr(Cont);
       EmitBlock(Cont);
@@ -6345,17 +6369,20 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType,
   const auto *FnType = cast<FunctionType>(PointeeType);
 
   if (const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);
-      FD && FD->hasAttr<OpenCLKernelAttr>())
+      FD && DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()))
     CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FnType);
 
+  bool CFIUnchecked =
+      CalleeType->hasPointeeToToCFIUncheckedCalleeFunctionType();
+
   // If we are checking indirect calls and this call is indirect, check that the
   // function pointer is a member of the bit set for the function type.
   if (SanOpts.has(SanitizerKind::CFIICall) &&
-      (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
-    SanitizerScope SanScope(this);
+      (!TargetDecl || !isa<FunctionDecl>(TargetDecl)) && !CFIUnchecked) {
+    auto CheckOrdinal = SanitizerKind::SO_CFIICall;
+    auto CheckHandler = SanitizerHandler::CFICheckFail;
+    SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
     EmitSanitizerStatReport(llvm::SanStat_CFI_ICall);
-    ApplyDebugLocation ApplyTrapDI(
-        *this, SanitizerAnnotateDebugInfo(SanitizerKind::SO_CFIICall));
 
     llvm::Metadata *MD;
     if (CGM.getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
@@ -6376,12 +6403,11 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType,
         EmitCheckTypeDescriptor(QualType(FnType, 0)),
     };
     if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
-      EmitCfiSlowPathCheck(SanitizerKind::SO_CFIICall, TypeTest, CrossDsoTypeId,
-                           CalleePtr, StaticData);
+      EmitCfiSlowPathCheck(CheckOrdinal, TypeTest, CrossDsoTypeId, CalleePtr,
+                           StaticData);
     } else {
-      EmitCheck(std::make_pair(TypeTest, SanitizerKind::SO_CFIICall),
-                SanitizerHandler::CFICheckFail, StaticData,
-                {CalleePtr, llvm::UndefValue::get(IntPtrTy)});
+      EmitCheck(std::make_pair(TypeTest, CheckOrdinal), CheckHandler,
+                StaticData, {CalleePtr, llvm::UndefValue::get(IntPtrTy)});
     }
   }
 
diff --git a/external/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp b/external/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp
index 024254b0affe..359e30cb8f5c 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1446,8 +1446,6 @@ namespace {
     unsigned NumPlacementArgs : 30;
     LLVM_PREFERRED_TYPE(AlignedAllocationMode)
     unsigned PassAlignmentToPlacementDelete : 1;
-    LLVM_PREFERRED_TYPE(TypeAwareAllocationMode)
-    unsigned PassTypeToPlacementDelete : 1;
     const FunctionDecl *OperatorDelete;
     RValueTy TypeIdentity;
     ValueTy Ptr;
diff --git a/external/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp b/external/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp
index a4bfac878a5b..0ca5a415eb73 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp
@@ -2234,8 +2234,12 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
       return ConstantLValue(C);
     };
 
-    if (const auto *FD = dyn_cast<FunctionDecl>(D))
-      return PtrAuthSign(CGM.getRawFunctionPointer(FD));
+    if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
+      llvm::Constant *C = CGM.getRawFunctionPointer(FD);
+      if (FD->getType()->isCFIUncheckedCalleeFunctionType())
+        C = llvm::NoCFIValue::get(cast<llvm::GlobalValue>(C));
+      return PtrAuthSign(C);
+    }
 
     if (const auto *VD = dyn_cast<VarDecl>(D)) {
       // We can never refer to a variable with local storage.
diff --git a/external/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp b/external/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp
index 111b5805c6a9..193710bef2d1 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGExprScalar.cpp
@@ -999,7 +999,9 @@ void ScalarExprEmitter::EmitFloatConversionCheck(
   if (!isa<llvm::IntegerType>(DstTy))
     return;
 
-  CodeGenFunction::SanitizerScope SanScope(&CGF);
+  auto CheckOrdinal = SanitizerKind::SO_FloatCastOverflow;
+  auto CheckHandler = SanitizerHandler::FloatCastOverflow;
+  SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
   using llvm::APFloat;
   using llvm::APSInt;
 
@@ -1056,8 +1058,8 @@ void ScalarExprEmitter::EmitFloatConversionCheck(
   llvm::Constant *StaticArgs[] = {CGF.EmitCheckSourceLocation(Loc),
                                   CGF.EmitCheckTypeDescriptor(OrigSrcType),
                                   CGF.EmitCheckTypeDescriptor(DstType)};
-  CGF.EmitCheck(std::make_pair(Check, SanitizerKind::SO_FloatCastOverflow),
-                SanitizerHandler::FloatCastOverflow, StaticArgs, OrigSrc);
+  CGF.EmitCheck(std::make_pair(Check, CheckOrdinal), CheckHandler, StaticArgs,
+                OrigSrc);
 }
 
 // Should be called within CodeGenFunction::SanitizerScope RAII scope.
@@ -1134,18 +1136,31 @@ void ScalarExprEmitter::EmitIntegerTruncationCheck(Value *Src, QualType SrcType,
       (!SrcSigned && DstSigned))
     return;
 
-  CodeGenFunction::SanitizerScope SanScope(&CGF);
-
   std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
             std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
-      Check =
-          EmitIntegerTruncationCheckHelper(Src, SrcType, Dst, DstType, Builder);
-  // If the comparison result is 'i1 false', then the truncation was lossy.
+      Check;
+
+  auto CheckHandler = SanitizerHandler::ImplicitConversion;
+  {
+    // We don't know the check kind until we call
+    // EmitIntegerTruncationCheckHelper, but we want to annotate
+    // EmitIntegerTruncationCheckHelper's instructions too.
+    SanitizerDebugLocation SanScope(
+        &CGF,
+        {SanitizerKind::SO_ImplicitUnsignedIntegerTruncation,
+         SanitizerKind::SO_ImplicitSignedIntegerTruncation},
+        CheckHandler);
+    Check =
+        EmitIntegerTruncationCheckHelper(Src, SrcType, Dst, DstType, Builder);
+    // If the comparison result is 'i1 false', then the truncation was lossy.
+  }
 
   // Do we care about this type of truncation?
   if (!CGF.SanOpts.has(Check.second.second))
     return;
 
+  SanitizerDebugLocation SanScope(&CGF, {Check.second.second}, CheckHandler);
+
   // Does some SSCL ignore this type?
   if (CGF.getContext().isTypeIgnoredBySanitizer(
           SanitizerMask::bitPosToMask(Check.second.second), DstType))
@@ -1157,8 +1172,7 @@ void ScalarExprEmitter::EmitIntegerTruncationCheck(Value *Src, QualType SrcType,
       llvm::ConstantInt::get(Builder.getInt8Ty(), Check.first),
       llvm::ConstantInt::get(Builder.getInt32Ty(), 0)};
 
-  CGF.EmitCheck(Check.second, SanitizerHandler::ImplicitConversion, StaticArgs,
-                {Src, Dst});
+  CGF.EmitCheck(Check.second, CheckHandler, StaticArgs, {Src, Dst});
 }
 
 static llvm::Value *EmitIsNegativeTestHelper(Value *V, QualType VType,
@@ -1272,7 +1286,13 @@ void ScalarExprEmitter::EmitIntegerSignChangeCheck(Value *Src, QualType SrcType,
     return;
   // That's it. We can't rule out any more cases with the data we have.
 
-  CodeGenFunction::SanitizerScope SanScope(&CGF);
+  auto CheckHandler = SanitizerHandler::ImplicitConversion;
+  SanitizerDebugLocation SanScope(
+      &CGF,
+      {SanitizerKind::SO_ImplicitIntegerSignChange,
+       SanitizerKind::SO_ImplicitUnsignedIntegerTruncation,
+       SanitizerKind::SO_ImplicitSignedIntegerTruncation},
+      CheckHandler);
 
   std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
             std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
@@ -1308,8 +1328,7 @@ void ScalarExprEmitter::EmitIntegerSignChangeCheck(Value *Src, QualType SrcType,
       llvm::ConstantInt::get(Builder.getInt8Ty(), CheckKind),
       llvm::ConstantInt::get(Builder.getInt32Ty(), 0)};
   // EmitCheck() will 'and' all the checks together.
-  CGF.EmitCheck(Checks, SanitizerHandler::ImplicitConversion, StaticArgs,
-                {Src, Dst});
+  CGF.EmitCheck(Checks, CheckHandler, StaticArgs, {Src, Dst});
 }
 
 // Should be called within CodeGenFunction::SanitizerScope RAII scope.
@@ -1393,7 +1412,9 @@ void CodeGenFunction::EmitBitfieldConversionCheck(Value *Src, QualType SrcType,
   bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType();
   bool DstSigned = DstType->isSignedIntegerOrEnumerationType();
 
-  CodeGenFunction::SanitizerScope SanScope(this);
+  auto CheckHandler = SanitizerHandler::ImplicitConversion;
+  SanitizerDebugLocation SanScope(
+      this, {SanitizerKind::SO_ImplicitBitfieldConversion}, CheckHandler);
 
   std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
             std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
@@ -1439,8 +1460,7 @@ void CodeGenFunction::EmitBitfieldConversionCheck(Value *Src, QualType SrcType,
       llvm::ConstantInt::get(Builder.getInt8Ty(), CheckKind),
       llvm::ConstantInt::get(Builder.getInt32Ty(), Info.Size)};
 
-  EmitCheck(Check.second, SanitizerHandler::ImplicitConversion, StaticArgs,
-            {Src, Dst});
+  EmitCheck(Check.second, CheckHandler, StaticArgs, {Src, Dst});
 }
 
 Value *ScalarExprEmitter::EmitScalarCast(Value *Src, QualType SrcType,
@@ -3975,7 +3995,11 @@ void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck(
 
 Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
   {
-    CodeGenFunction::SanitizerScope SanScope(&CGF);
+    SanitizerDebugLocation SanScope(&CGF,
+                                    {SanitizerKind::SO_IntegerDivideByZero,
+                                     SanitizerKind::SO_SignedIntegerOverflow,
+                                     SanitizerKind::SO_FloatDivideByZero},
+                                    SanitizerHandler::DivremOverflow);
     if ((CGF.SanOpts.has(SanitizerKind::IntegerDivideByZero) ||
          CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) &&
         Ops.Ty->isIntegerType() &&
@@ -4029,7 +4053,10 @@ Value *ScalarExprEmitter::EmitRem(const BinOpInfo &Ops) {
        CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) &&
       Ops.Ty->isIntegerType() &&
       (Ops.mayHaveIntegerDivisionByZero() || Ops.mayHaveIntegerOverflow())) {
-    CodeGenFunction::SanitizerScope SanScope(&CGF);
+    SanitizerDebugLocation SanScope(&CGF,
+                                    {SanitizerKind::SO_IntegerDivideByZero,
+                                     SanitizerKind::SO_SignedIntegerOverflow},
+                                    SanitizerHandler::DivremOverflow);
     llvm::Value *Zero = llvm::Constant::getNullValue(ConvertType(Ops.Ty));
     EmitUndefinedBehaviorIntegerDivAndRemCheck(Ops, Zero, false);
   }
@@ -4078,7 +4105,10 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
   if (isSigned)
     OpID |= 1;
 
-  CodeGenFunction::SanitizerScope SanScope(&CGF);
+  SanitizerDebugLocation SanScope(&CGF,
+                                  {SanitizerKind::SO_SignedIntegerOverflow,
+                                   SanitizerKind::SO_UnsignedIntegerOverflow},
+                                  OverflowKind);
   llvm::Type *opTy = CGF.CGM.getTypes().ConvertType(Ops.Ty);
 
   llvm::Function *intrinsic = CGF.CGM.getIntrinsic(IID, opTy);
@@ -4204,7 +4234,9 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
                              PtrTy->getPointerAddressSpace()))
       return Ptr;
     // The inbounds GEP of null is valid iff the index is zero.
-    CodeGenFunction::SanitizerScope SanScope(&CGF);
+    auto CheckOrdinal = SanitizerKind::SO_PointerOverflow;
+    auto CheckHandler = SanitizerHandler::PointerOverflow;
+    SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
     Value *IsZeroIndex = CGF.Builder.CreateIsNull(index);
     llvm::Constant *StaticArgs[] = {
         CGF.EmitCheckSourceLocation(op.E->getExprLoc())};
@@ -4212,8 +4244,8 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
     Value *IntPtr = llvm::Constant::getNullValue(IntPtrTy);
     Value *ComputedGEP = CGF.Builder.CreateZExtOrTrunc(index, IntPtrTy);
     Value *DynamicArgs[] = {IntPtr, ComputedGEP};
-    CGF.EmitCheck({{IsZeroIndex, SanitizerKind::SO_PointerOverflow}},
-                  SanitizerHandler::PointerOverflow, StaticArgs, DynamicArgs);
+    CGF.EmitCheck({{IsZeroIndex, CheckOrdinal}}, CheckHandler, StaticArgs,
+                  DynamicArgs);
     return Ptr;
   }
 
@@ -4734,7 +4766,16 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) {
     RHS = ConstrainShiftValue(Ops.LHS, RHS, "shl.mask");
   else if ((SanitizeBase || SanitizeExponent) &&
            isa<llvm::IntegerType>(Ops.LHS->getType())) {
-    CodeGenFunction::SanitizerScope SanScope(&CGF);
+    SmallVector<SanitizerKind::SanitizerOrdinal, 3> Ordinals;
+    if (SanitizeSignedBase)
+      Ordinals.push_back(SanitizerKind::SO_ShiftBase);
+    if (SanitizeUnsignedBase)
+      Ordinals.push_back(SanitizerKind::SO_UnsignedShiftBase);
+    if (SanitizeExponent)
+      Ordinals.push_back(SanitizerKind::SO_ShiftExponent);
+
+    SanitizerDebugLocation SanScope(&CGF, Ordinals,
+                                    SanitizerHandler::ShiftOutOfBounds);
     SmallVector<std::pair<Value *, SanitizerKind::SanitizerOrdinal>, 2> Checks;
     bool RHSIsSigned = Ops.rhsHasSignedIntegerRepresentation();
     llvm::Value *WidthMinusOne =
@@ -4805,7 +4846,8 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) {
     RHS = ConstrainShiftValue(Ops.LHS, RHS, "shr.mask");
   else if (CGF.SanOpts.has(SanitizerKind::ShiftExponent) &&
            isa<llvm::IntegerType>(Ops.LHS->getType())) {
-    CodeGenFunction::SanitizerScope SanScope(&CGF);
+    SanitizerDebugLocation SanScope(&CGF, {SanitizerKind::SO_ShiftExponent},
+                                    SanitizerHandler::ShiftOutOfBounds);
     bool RHSIsSigned = Ops.rhsHasSignedIntegerRepresentation();
     llvm::Value *Valid = Builder.CreateICmpULE(
         Ops.RHS, GetMaximumShiftAmount(Ops.LHS, Ops.RHS, RHSIsSigned));
@@ -6037,7 +6079,9 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
 
   const auto &DL = CGM.getDataLayout();
 
-  SanitizerScope SanScope(this);
+  auto CheckOrdinal = SanitizerKind::SO_PointerOverflow;
+  auto CheckHandler = SanitizerHandler::PointerOverflow;
+  SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
   llvm::Type *IntPtrTy = DL.getIntPtrType(PtrTy);
 
   GEPOffsetAndOverflow EvaluatedGEP =
@@ -6074,7 +6118,7 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
     auto *BaseIsNotNullptr = Builder.CreateIsNotNull(Ptr);
     auto *ResultIsNotNullptr = Builder.CreateIsNotNull(ComputedGEP);
     auto *Valid = Builder.CreateICmpEQ(BaseIsNotNullptr, ResultIsNotNullptr);
-    Checks.emplace_back(Valid, SanitizerKind::SO_PointerOverflow);
+    Checks.emplace_back(Valid, CheckOrdinal);
   }
 
   if (PerformOverflowCheck) {
@@ -6110,7 +6154,7 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
       ValidGEP = Builder.CreateICmpULE(ComputedGEP, IntPtr);
     }
     ValidGEP = Builder.CreateAnd(ValidGEP, NoOffsetOverflow);
-    Checks.emplace_back(ValidGEP, SanitizerKind::SO_PointerOverflow);
+    Checks.emplace_back(ValidGEP, CheckOrdinal);
   }
 
   assert(!Checks.empty() && "Should have produced some checks.");
@@ -6118,7 +6162,7 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
   llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc)};
   // Pass the computed GEP to the runtime to avoid emitting poisoned arguments.
   llvm::Value *DynamicArgs[] = {IntPtr, ComputedGEP};
-  EmitCheck(Checks, SanitizerHandler::PointerOverflow, StaticArgs, DynamicArgs);
+  EmitCheck(Checks, CheckHandler, StaticArgs, DynamicArgs);
 
   return GEPVal;
 }
diff --git a/external/llvm-project/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/external/llvm-project/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index 10dd9fd04eb9..ccf45c0c6ff1 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -295,17 +295,16 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     Value *SpaceOp = EmitScalarExpr(E->getArg(2));
     Value *RangeOp = EmitScalarExpr(E->getArg(3));
     Value *IndexOp = EmitScalarExpr(E->getArg(4));
+    Value *Name = EmitScalarExpr(E->getArg(5));
     // FIXME: NonUniformResourceIndex bit is not yet implemented
     // (llvm/llvm-project#135452)
     Value *NonUniform =
         llvm::ConstantInt::get(llvm::Type::getInt1Ty(getLLVMContext()), false);
 
-    auto [IntrinsicID, HasNameArg] =
+    llvm::Intrinsic::ID IntrinsicID =
         CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
-    SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp,
-                              NonUniform};
-    if (HasNameArg)
-      Args.push_back(EmitScalarExpr(E->getArg(5)));
+    SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp,
+                              IndexOp, NonUniform, Name};
     return Builder.CreateIntrinsic(HandleTy, IntrinsicID, Args);
   }
   case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
@@ -314,16 +313,16 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     Value *RangeOp = EmitScalarExpr(E->getArg(2));
     Value *IndexOp = EmitScalarExpr(E->getArg(3));
     Value *OrderID = EmitScalarExpr(E->getArg(4));
+    Value *Name = EmitScalarExpr(E->getArg(5));
     // FIXME: NonUniformResourceIndex bit is not yet implemented
     // (llvm/llvm-project#135452)
     Value *NonUniform =
         llvm::ConstantInt::get(llvm::Type::getInt1Ty(getLLVMContext()), false);
 
-    auto [IntrinsicID, HasNameArg] =
+    llvm::Intrinsic::ID IntrinsicID =
         CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
-    SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, NonUniform};
-    if (HasNameArg)
-      Args.push_back(EmitScalarExpr(E->getArg(5)));
+    SmallVector<Value *> Args{OrderID, SpaceOp,    RangeOp,
+                              IndexOp, NonUniform, Name};
     return Builder.CreateIntrinsic(HandleTy, IntrinsicID, Args);
   }
   case Builtin::BI__builtin_hlsl_all: {
@@ -691,6 +690,11 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     return EmitRuntimeCall(
         Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
   }
+  case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
+    Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
+    return EmitRuntimeCall(
+        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
+  }
   case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
     // Due to the use of variadic arguments we must explicitly retreive them and
     // create our function type.
diff --git a/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp b/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp
index cfe9dc1192d9..571ff53b7d64 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -23,6 +23,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/TargetOptions.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -34,7 +35,6 @@
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
-#include <utility>
 
 using namespace clang;
 using namespace CodeGen;
@@ -236,35 +236,6 @@ static void fillPackoffsetLayout(const HLSLBufferDecl *BufDecl,
   }
 }
 
-std::pair<llvm::Intrinsic::ID, bool>
-CGHLSLRuntime::getCreateHandleFromBindingIntrinsic() {
-  switch (getArch()) {
-  case llvm::Triple::dxil:
-    return std::pair(llvm::Intrinsic::dx_resource_handlefrombinding, true);
-  case llvm::Triple::spirv:
-    return std::pair(llvm::Intrinsic::spv_resource_handlefrombinding, false);
-  default:
-    llvm_unreachable("Intrinsic resource_handlefrombinding not supported by "
-                     "target architecture");
-  }
-}
-
-std::pair<llvm::Intrinsic::ID, bool>
-CGHLSLRuntime::getCreateHandleFromImplicitBindingIntrinsic() {
-  switch (getArch()) {
-  case llvm::Triple::dxil:
-    return std::pair(llvm::Intrinsic::dx_resource_handlefromimplicitbinding,
-                     true);
-  case llvm::Triple::spirv:
-    return std::pair(llvm::Intrinsic::spv_resource_handlefromimplicitbinding,
-                     false);
-  default:
-    llvm_unreachable(
-        "Intrinsic resource_handlefromimplicitbinding not supported by "
-        "target architecture");
-  }
-}
-
 // Codegen for HLSLBufferDecl
 void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
 
@@ -383,6 +354,30 @@ static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
   return B.CreateCall(F, {B.getInt32(0)});
 }
 
+static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
+                                      unsigned BuiltIn) {
+  LLVMContext &Ctx = GV->getContext();
+  IRBuilder<> B(GV->getContext());
+  MDNode *Operands = MDNode::get(
+      Ctx,
+      {ConstantAsMetadata::get(B.getInt32(/* Spirv::Decoration::BuiltIn */ 11)),
+       ConstantAsMetadata::get(B.getInt32(BuiltIn))});
+  MDNode *Decoration = MDNode::get(Ctx, {Operands});
+  GV->addMetadata("spirv.Decorations", *Decoration);
+}
+
+static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
+                                           llvm::Type *Ty, const Twine &Name,
+                                           unsigned BuiltInID) {
+  auto *GV = new llvm::GlobalVariable(
+      M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
+      /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
+      llvm::GlobalVariable::GeneralDynamicTLSModel,
+      /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
+  addSPIRVBuiltinDecoration(GV, BuiltInID);
+  return B.CreateLoad(Ty, GV);
+}
+
 llvm::Value *CGHLSLRuntime::emitInputSemantic(IRBuilder<> &B,
                                               const ParmVarDecl &D,
                                               llvm::Type *Ty) {
@@ -406,6 +401,12 @@ llvm::Value *CGHLSLRuntime::emitInputSemantic(IRBuilder<> &B,
     llvm::Function *GroupIDIntrinsic = CGM.getIntrinsic(getGroupIdIntrinsic());
     return buildVectorInput(B, GroupIDIntrinsic, Ty);
   }
+  if (D.hasAttr<HLSLSV_PositionAttr>()) {
+    if (getArch() == llvm::Triple::spirv)
+      return createSPIRVBuiltinLoad(B, CGM.getModule(), Ty, "sv_position",
+                                    /* BuiltIn::Position */ 0);
+    llvm_unreachable("SV_Position semantic not implemented for this target.");
+  }
   assert(false && "Unhandled parameter attribute");
   return nullptr;
 }
@@ -594,35 +595,37 @@ void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
       llvm::ConstantInt::get(CGM.IntTy, RBA ? RBA->getSpaceNumber() : 0);
   Value *Name = nullptr;
 
-  auto [IntrinsicID, HasNameArg] =
+  llvm::Intrinsic::ID IntrinsicID =
       RBA->hasRegisterSlot()
           ? CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic()
           : CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
 
-  if (HasNameArg) {
-    std::string Str(BufDecl->getName());
-    std::string GlobalName(Str + ".str");
-    Name = CGM.GetAddrOfConstantCString(Str, GlobalName.c_str()).getPointer();
-  }
+  std::string Str(BufDecl->getName());
+  std::string GlobalName(Str + ".str");
+  Name = CGM.GetAddrOfConstantCString(Str, GlobalName.c_str()).getPointer();
 
   // buffer with explicit binding
   if (RBA->hasRegisterSlot()) {
     auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, RBA->getSlotNumber());
-    SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, NonUniform};
-    if (Name)
-      Args.push_back(Name);
+    SmallVector<Value *> Args{Space, RegSlot,    RangeSize,
+                              Index, NonUniform, Name};
     initializeBuffer(CGM, GV, IntrinsicID, Args);
   } else {
     // buffer with implicit binding
     auto *OrderID =
         llvm::ConstantInt::get(CGM.IntTy, RBA->getImplicitBindingOrderID());
-    SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, NonUniform};
-    if (Name)
-      Args.push_back(Name);
+    SmallVector<Value *> Args{OrderID, Space,      RangeSize,
+                              Index,   NonUniform, Name};
     initializeBuffer(CGM, GV, IntrinsicID, Args);
   }
 }
 
+void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
+                                              llvm::GlobalVariable *GV) {
+  if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
+    addSPIRVBuiltinDecoration(GV, Attr->getBuiltIn());
+}
+
 llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
   if (!CGM.shouldEmitConvergenceTokens())
     return nullptr;
diff --git a/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h b/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h
index 34c6b8d0a336..89d2aff85d91 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/external/llvm-project/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -107,6 +107,7 @@ class CGHLSLRuntime {
   GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
   GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
   GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
+  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveGetLaneCount, wave_get_lane_count)
   GENERATE_HLSL_INTRINSIC_FUNCTION(WaveReadLaneAt, wave_readlane)
   GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitUHigh, firstbituhigh)
   GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitSHigh, firstbitshigh)
@@ -117,6 +118,10 @@ class CGHLSLRuntime {
 
   GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetPointer,
                                    resource_getpointer)
+  GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromBinding,
+                                   resource_handlefrombinding)
+  GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromImplicitBinding,
+                                   resource_handlefromimplicitbinding)
   GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, resource_updatecounter)
   GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync,
                                    group_memory_barrier_with_group_sync)
@@ -125,15 +130,6 @@ class CGHLSLRuntime {
   // End of reserved area for HLSL intrinsic getters.
   //===----------------------------------------------------------------------===//
 
-  // Returns ID of the intrinsic that initializes resource handle from binding
-  // and a bool value indicating whether the last argument of the intrinsic is
-  // the resource name (not all targets need that).
-  std::pair<llvm::Intrinsic::ID, bool> getCreateHandleFromBindingIntrinsic();
-
-  // Same as above but for implicit binding.
-  std::pair<llvm::Intrinsic::ID, bool>
-  getCreateHandleFromImplicitBindingIntrinsic();
-
 protected:
   CodeGenModule &CGM;
 
@@ -157,6 +153,7 @@ class CGHLSLRuntime {
 
   void emitEntryFunction(const FunctionDecl *FD, llvm::Function *Fn);
   void setHLSLFunctionAttributes(const FunctionDecl *FD, llvm::Function *Fn);
+  void handleGlobalVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Var);
 
   llvm::Instruction *getConvergenceToken(llvm::BasicBlock &BB);
 
diff --git a/external/llvm-project/clang/lib/CodeGen/CGObjC.cpp b/external/llvm-project/clang/lib/CodeGen/CGObjC.cpp
index 73071bc60901..8a1b44a0c157 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGObjC.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGObjC.cpp
@@ -14,6 +14,7 @@
 #include "CGObjCRuntime.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenPGO.h"
 #include "ConstantEmitter.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
@@ -805,7 +806,7 @@ static llvm::Value *emitARCRetainLoadOfScalar(CodeGenFunction &CGF,
 /// its pointer, name, and types registered in the class structure.
 void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) {
   StartObjCMethod(OMD, OMD->getClassInterface());
-  PGO.assignRegionCounters(GlobalDecl(OMD), CurFn);
+  PGO->assignRegionCounters(GlobalDecl(OMD), CurFn);
   assert(isa<CompoundStmt>(OMD->getBody()));
   incrementProfileCounter(OMD->getBody());
   EmitCompoundStmtWithoutScope(*cast<CompoundStmt>(OMD->getBody()));
@@ -1967,7 +1968,9 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
     const ObjCInterfaceType *InterfaceTy =
         ObjPtrTy ? ObjPtrTy->getInterfaceType() : nullptr;
     if (InterfaceTy) {
-      SanitizerScope SanScope(this);
+      auto CheckOrdinal = SanitizerKind::SO_ObjCCast;
+      auto CheckHandler = SanitizerHandler::InvalidObjCCast;
+      SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
       auto &C = CGM.getContext();
       assert(InterfaceTy->getDecl() && "No decl for ObjC interface type");
       Selector IsKindOfClassSel = GetUnarySelector("isKindOfClass", C);
@@ -1984,8 +1987,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
       llvm::Constant *StaticData[] = {
           EmitCheckSourceLocation(S.getBeginLoc()),
           EmitCheckTypeDescriptor(QualType(InterfaceTy, 0))};
-      EmitCheck({{IsClass, SanitizerKind::SO_ObjCCast}},
-                SanitizerHandler::InvalidObjCCast,
+      EmitCheck({{IsClass, CheckOrdinal}}, CheckHandler,
                 ArrayRef<llvm::Constant *>(StaticData), CurrentItem);
     }
   }
diff --git a/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 8c44ed058dec..75c4442d3fe9 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1375,7 +1375,12 @@ static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
   llvm::raw_svector_ostream OS(Buffer);
   // Build debug location
   PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
-  OS << ";" << PLoc.getFilename() << ";";
+  OS << ";";
+  if (auto *DbgInfo = CGF.getDebugInfo())
+    OS << DbgInfo->remapDIPath(PLoc.getFilename());
+  else
+    OS << PLoc.getFilename();
+  OS << ";";
   if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
     OS << FD->getQualifiedNameAsString();
   OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
@@ -1393,10 +1398,14 @@ llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
     SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
   } else {
     std::string FunctionName;
+    std::string FileName;
     if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
       FunctionName = FD->getQualifiedNameAsString();
     PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
-    const char *FileName = PLoc.getFilename();
+    if (auto *DbgInfo = CGF.getDebugInfo())
+      FileName = DbgInfo->remapDIPath(PLoc.getFilename());
+    else
+      FileName = PLoc.getFilename();
     unsigned Line = PLoc.getLine();
     unsigned Column = PLoc.getColumn();
     SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
@@ -4944,11 +4953,255 @@ void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
   }
 }
 
+static std::string generateUniqueName(CodeGenModule &CGM,
+                                      llvm::StringRef Prefix, const Expr *Ref);
+
+void CGOpenMPRuntime::emitPrivateReduction(
+    CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,
+    const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {
+
+  //  Create a shared global variable (__shared_reduction_var) to accumulate the
+  //  final result.
+  //
+  //  Call __kmpc_barrier to synchronize threads before initialization.
+  //
+  //  The master thread (thread_id == 0) initializes __shared_reduction_var
+  //    with the identity value or initializer.
+  //
+  //  Call __kmpc_barrier to synchronize before combining.
+  //  For each i:
+  //    - Thread enters critical section.
+  //    - Reads its private value from LHSExprs[i].
+  //    - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
+  //    Privates[i]).
+  //    - Exits critical section.
+  //
+  //  Call __kmpc_barrier after combining.
+  //
+  //  Each thread copies __shared_reduction_var[i] back to RHSExprs[i].
+  //
+  //  Final __kmpc_barrier to synchronize after broadcasting
+  QualType PrivateType = Privates->getType();
+  llvm::Type *LLVMType = CGF.ConvertTypeForMem(PrivateType);
+
+  const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOps);
+  std::string ReductionVarNameStr;
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates->IgnoreParenCasts()))
+    ReductionVarNameStr =
+        generateUniqueName(CGM, DRE->getDecl()->getNameAsString(), Privates);
+  else
+    ReductionVarNameStr = "unnamed_priv_var";
+
+  // Create an internal shared variable
+  std::string SharedName =
+      CGM.getOpenMPRuntime().getName({"internal_pivate_", ReductionVarNameStr});
+  llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(
+      LLVMType, ".omp.reduction." + SharedName);
+
+  SharedVar->setAlignment(
+      llvm::MaybeAlign(CGF.getContext().getTypeAlign(PrivateType) / 8));
+
+  Address SharedResult =
+      CGF.MakeNaturalAlignRawAddrLValue(SharedVar, PrivateType).getAddress();
+
+  llvm::Value *ThreadId = getThreadID(CGF, Loc);
+  llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
+  llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
+
+  llvm::BasicBlock *InitBB = CGF.createBasicBlock("init");
+  llvm::BasicBlock *InitEndBB = CGF.createBasicBlock("init.end");
+
+  llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(
+      ThreadId, llvm::ConstantInt::get(ThreadId->getType(), 0));
+  CGF.Builder.CreateCondBr(IsWorker, InitBB, InitEndBB);
+
+  CGF.EmitBlock(InitBB);
+
+  auto EmitSharedInit = [&]() {
+    if (UDR) { // Check if it's a User-Defined Reduction
+      if (const Expr *UDRInitExpr = UDR->getInitializer()) {
+        std::pair<llvm::Function *, llvm::Function *> FnPair =
+            getUserDefinedReduction(UDR);
+        llvm::Function *InitializerFn = FnPair.second;
+        if (InitializerFn) {
+          if (const auto *CE =
+                  dyn_cast<CallExpr>(UDRInitExpr->IgnoreParenImpCasts())) {
+            const auto *OutDRE = cast<DeclRefExpr>(
+                cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts())
+                    ->getSubExpr());
+            const VarDecl *OutVD = cast<VarDecl>(OutDRE->getDecl());
+
+            CodeGenFunction::OMPPrivateScope LocalScope(CGF);
+            LocalScope.addPrivate(OutVD, SharedResult);
+
+            (void)LocalScope.Privatize();
+            if (const auto *OVE = dyn_cast<OpaqueValueExpr>(
+                    CE->getCallee()->IgnoreParenImpCasts())) {
+              CodeGenFunction::OpaqueValueMapping OpaqueMap(
+                  CGF, OVE, RValue::get(InitializerFn));
+              CGF.EmitIgnoredExpr(CE);
+            } else {
+              CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,
+                                   PrivateType.getQualifiers(),
+                                   /*IsInitializer=*/true);
+            }
+          } else {
+            CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,
+                                 PrivateType.getQualifiers(),
+                                 /*IsInitializer=*/true);
+          }
+        } else {
+          CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,
+                               PrivateType.getQualifiers(),
+                               /*IsInitializer=*/true);
+        }
+      } else {
+        // EmitNullInitialization handles default construction for C++ classes
+        // and zeroing for scalars, which is a reasonable default.
+        CGF.EmitNullInitialization(SharedResult, PrivateType);
+      }
+      return; // UDR initialization handled
+    }
+    if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates)) {
+      if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
+        if (const Expr *InitExpr = VD->getInit()) {
+          CGF.EmitAnyExprToMem(InitExpr, SharedResult,
+                               PrivateType.getQualifiers(), true);
+          return;
+        }
+      }
+    }
+    CGF.EmitNullInitialization(SharedResult, PrivateType);
+  };
+  EmitSharedInit();
+  CGF.Builder.CreateBr(InitEndBB);
+  CGF.EmitBlock(InitEndBB);
+
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                          CGM.getModule(), OMPRTL___kmpc_barrier),
+                      BarrierArgs);
+
+  const Expr *ReductionOp = ReductionOps;
+  const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
+  LValue SharedLV = CGF.MakeAddrLValue(SharedResult, PrivateType);
+  LValue LHSLV = CGF.EmitLValue(Privates);
+
+  auto EmitCriticalReduction = [&](auto ReductionGen) {
+    std::string CriticalName = getName({"reduction_critical"});
+    emitCriticalRegion(CGF, CriticalName, ReductionGen, Loc);
+  };
+
+  if (CurrentUDR) {
+    // Handle user-defined reduction.
+    auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
+      Action.Enter(CGF);
+      std::pair<llvm::Function *, llvm::Function *> FnPair =
+          getUserDefinedReduction(CurrentUDR);
+      if (FnPair.first) {
+        if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) {
+          const auto *OutDRE = cast<DeclRefExpr>(
+              cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts())
+                  ->getSubExpr());
+          const auto *InDRE = cast<DeclRefExpr>(
+              cast<UnaryOperator>(CE->getArg(1)->IgnoreParenImpCasts())
+                  ->getSubExpr());
+          CodeGenFunction::OMPPrivateScope LocalScope(CGF);
+          LocalScope.addPrivate(cast<VarDecl>(OutDRE->getDecl()),
+                                SharedLV.getAddress());
+          LocalScope.addPrivate(cast<VarDecl>(InDRE->getDecl()),
+                                LHSLV.getAddress());
+          (void)LocalScope.Privatize();
+          emitReductionCombiner(CGF, ReductionOp);
+        }
+      }
+    };
+    EmitCriticalReduction(ReductionGen);
+  } else {
+    // Handle built-in reduction operations.
+#ifndef NDEBUG
+    const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();
+    if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
+      ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
+
+    const Expr *AssignRHS = nullptr;
+    if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
+      if (BinOp->getOpcode() == BO_Assign)
+        AssignRHS = BinOp->getRHS();
+    } else if (const auto *OpCall =
+                   dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
+      if (OpCall->getOperator() == OO_Equal)
+        AssignRHS = OpCall->getArg(1);
+    }
+
+    assert(AssignRHS &&
+           "Private Variable Reduction : Invalid ReductionOp expression");
+#endif
+
+    auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
+      Action.Enter(CGF);
+      const auto *OmpOutDRE =
+          dyn_cast<DeclRefExpr>(LHSExprs->IgnoreParenImpCasts());
+      const auto *OmpInDRE =
+          dyn_cast<DeclRefExpr>(RHSExprs->IgnoreParenImpCasts());
+      assert(
+          OmpOutDRE && OmpInDRE &&
+          "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
+      const VarDecl *OmpOutVD = cast<VarDecl>(OmpOutDRE->getDecl());
+      const VarDecl *OmpInVD = cast<VarDecl>(OmpInDRE->getDecl());
+      CodeGenFunction::OMPPrivateScope LocalScope(CGF);
+      LocalScope.addPrivate(OmpOutVD, SharedLV.getAddress());
+      LocalScope.addPrivate(OmpInVD, LHSLV.getAddress());
+      (void)LocalScope.Privatize();
+      // Emit the actual reduction operation
+      CGF.EmitIgnoredExpr(ReductionOp);
+    };
+    EmitCriticalReduction(ReductionGen);
+  }
+
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                          CGM.getModule(), OMPRTL___kmpc_barrier),
+                      BarrierArgs);
+
+  // Broadcast final result
+  bool IsAggregate = PrivateType->isAggregateType();
+  LValue SharedLV1 = CGF.MakeAddrLValue(SharedResult, PrivateType);
+  llvm::Value *FinalResultVal = nullptr;
+  Address FinalResultAddr = Address::invalid();
+
+  if (IsAggregate)
+    FinalResultAddr = SharedResult;
+  else
+    FinalResultVal = CGF.EmitLoadOfScalar(SharedLV1, Loc);
+
+  LValue TargetLHSLV = CGF.EmitLValue(RHSExprs);
+  if (IsAggregate) {
+    CGF.EmitAggregateCopy(TargetLHSLV,
+                          CGF.MakeAddrLValue(FinalResultAddr, PrivateType),
+                          PrivateType, AggValueSlot::DoesNotOverlap, false);
+  } else {
+    CGF.EmitStoreOfScalar(FinalResultVal, TargetLHSLV);
+  }
+  // Final synchronization barrier
+  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                          CGM.getModule(), OMPRTL___kmpc_barrier),
+                      BarrierArgs);
+
+  // Combiner with original list item
+  auto OriginalListCombiner = [&](CodeGenFunction &CGF,
+                                  PrePostActionTy &Action) {
+    Action.Enter(CGF);
+    emitSingleReductionCombiner(CGF, ReductionOps, Privates,
+                                cast<DeclRefExpr>(LHSExprs),
+                                cast<DeclRefExpr>(RHSExprs));
+  };
+  EmitCriticalReduction(OriginalListCombiner);
+}
+
 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
-                                    ArrayRef<const Expr *> Privates,
-                                    ArrayRef<const Expr *> LHSExprs,
-                                    ArrayRef<const Expr *> RHSExprs,
-                                    ArrayRef<const Expr *> ReductionOps,
+                                    ArrayRef<const Expr *> OrgPrivates,
+                                    ArrayRef<const Expr *> OrgLHSExprs,
+                                    ArrayRef<const Expr *> OrgRHSExprs,
+                                    ArrayRef<const Expr *> OrgReductionOps,
                                     ReductionOptionsTy Options) {
   if (!CGF.HaveInsertPoint())
     return;
@@ -4995,10 +5248,10 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
 
   if (SimpleReduction) {
     CodeGenFunction::RunCleanupsScope Scope(CGF);
-    const auto *IPriv = Privates.begin();
-    const auto *ILHS = LHSExprs.begin();
-    const auto *IRHS = RHSExprs.begin();
-    for (const Expr *E : ReductionOps) {
+    const auto *IPriv = OrgPrivates.begin();
+    const auto *ILHS = OrgLHSExprs.begin();
+    const auto *IRHS = OrgRHSExprs.begin();
+    for (const Expr *E : OrgReductionOps) {
       emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
                                   cast<DeclRefExpr>(*IRHS));
       ++IPriv;
@@ -5008,6 +5261,26 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
     return;
   }
 
+  // Filter out shared  reduction variables based on IsPrivateVarReduction flag.
+  // Only keep entries where the corresponding variable is not private.
+  SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
+      FilteredRHSExprs, FilteredReductionOps;
+  for (unsigned I : llvm::seq<unsigned>(
+           std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
+    if (!Options.IsPrivateVarReduction[I]) {
+      FilteredPrivates.emplace_back(OrgPrivates[I]);
+      FilteredLHSExprs.emplace_back(OrgLHSExprs[I]);
+      FilteredRHSExprs.emplace_back(OrgRHSExprs[I]);
+      FilteredReductionOps.emplace_back(OrgReductionOps[I]);
+    }
+  }
+  // Wrap filtered vectors in ArrayRef for downstream shared reduction
+  // processing.
+  ArrayRef<const Expr *> Privates = FilteredPrivates;
+  ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
+  ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
+  ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
+
   // 1. Build a list of reduction variables.
   // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
   auto Size = RHSExprs.size();
@@ -5199,7 +5472,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
       } else {
         // Emit as a critical region.
         auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
-                                           const Expr *, const Expr *) {
+                                     const Expr *, const Expr *) {
           CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
           std::string Name = RT.getName({"atomic_reduction"});
           RT.emitCriticalRegion(
@@ -5246,6 +5519,16 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
 
   CGF.EmitBranch(DefaultBB);
   CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
+  assert(OrgLHSExprs.size() == OrgPrivates.size() &&
+         "PrivateVarReduction: Privates size mismatch");
+  assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
+         "PrivateVarReduction: ReductionOps size mismatch");
+  for (unsigned I : llvm::seq<unsigned>(
+           std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
+    if (Options.IsPrivateVarReduction[I])
+      emitPrivateReduction(CGF, Loc, OrgPrivates[I], OrgLHSExprs[I],
+                           OrgRHSExprs[I], OrgReductionOps[I]);
+  }
 }
 
 /// Generates unique name for artificial threadprivate variables.
@@ -8882,10 +9165,14 @@ emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
     ExprName = MapExprs.getMapDecl()->getNameAsString();
   }
 
+  std::string FileName;
   PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
-  return OMPBuilder.getOrCreateSrcLocStr(PLoc.getFilename(), ExprName,
-                                         PLoc.getLine(), PLoc.getColumn(),
-                                         SrcLocStrSize);
+  if (auto *DbgInfo = CGF.getDebugInfo())
+    FileName = DbgInfo->remapDIPath(PLoc.getFilename());
+  else
+    FileName = PLoc.getFilename();
+  return OMPBuilder.getOrCreateSrcLocStr(FileName, ExprName, PLoc.getLine(),
+                                         PLoc.getColumn(), SrcLocStrSize);
 }
 /// Emit the arrays used to pass the captures and map information to the
 /// offloading runtime library. If there is no map or capture information,
@@ -9707,6 +9994,17 @@ static void emitTargetCallKernelLaunch(
         ++RedVarCount;
       }
     }
+    // Process debug info.
+    if (CGF.CGM.getCodeGenOpts().getDebugInfo() !=
+        llvm::codegenoptions::NoDebugInfo) {
+      auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
+        return emitMappingInformation(CGF, OMPBuilder, MapExpr);
+      };
+
+      CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
+      llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
+                      FillInfoMap);
+    }
   }
 
   CGOpenMPRuntime::TargetDataInfo Info;
diff --git a/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h b/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h
index 83af17c215c4..8db83354b441 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -1214,8 +1214,20 @@ class CGOpenMPRuntime {
   struct ReductionOptionsTy {
     bool WithNowait;
     bool SimpleReduction;
+    llvm::SmallVector<bool, 8> IsPrivateVarReduction;
     OpenMPDirectiveKind ReductionKind;
   };
+
+  /// Emits code for private variable reduction
+  /// \param Privates List of private copies for original reduction arguments.
+  /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
+  /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
+  /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
+  /// or 'operator binop(LHS, RHS)'.
+  void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc,
+                            const Expr *Privates, const Expr *LHSExprs,
+                            const Expr *RHSExprs, const Expr *ReductionOps);
+
   /// Emit a code for reduction clause. Next code should be emitted for
   /// reduction:
   /// \code
diff --git a/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 8647a9f4642b..2dc5689f5284 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/Cuda.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
diff --git a/external/llvm-project/clang/lib/CodeGen/CGPointerAuth.cpp b/external/llvm-project/clang/lib/CodeGen/CGPointerAuth.cpp
index bf22912988d3..dcef01a5eb6d 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGPointerAuth.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGPointerAuth.cpp
@@ -520,6 +520,11 @@ llvm::Constant *CodeGenModule::getMemberFunctionPointer(llvm::Constant *Pointer,
         Pointer, PointerAuth.getKey(), nullptr,
         cast_or_null<llvm::ConstantInt>(PointerAuth.getDiscriminator()));
 
+  if (const auto *MFT = dyn_cast<MemberPointerType>(FT.getTypePtr())) {
+    if (MFT->hasPointeeToToCFIUncheckedCalleeFunctionType())
+      Pointer = llvm::NoCFIValue::get(cast<llvm::GlobalValue>(Pointer));
+  }
+
   return Pointer;
 }
 
diff --git a/external/llvm-project/clang/lib/CodeGen/CGStmt.cpp b/external/llvm-project/clang/lib/CodeGen/CGStmt.cpp
index 98838df1b652..f92d052c253f 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGStmt.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGStmt.cpp
@@ -15,6 +15,7 @@
 #include "CGOpenMPRuntimeGPU.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenPGO.h"
 #include "TargetInfo.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Expr.h"
@@ -601,7 +602,7 @@ void CodeGenFunction::EmitBigJumpLoopCode(const OMPExecutableDirective &D,
   // Add pre-processing code from start of EmitStmt function so that the
   // code path is identical.
   assert(CapturedForStmt && "Null statement?");
-  PGO.setCurrentStmt(CapturedForStmt);
+  PGO->setCurrentStmt(CapturedForStmt);
 
   // These statements have their own debug info handling.
   if (EmitSimpleStmt(CapturedForStmt, nullptr))
@@ -1108,9 +1109,9 @@ CodeGenFunction::ExtractXteamRedRhsExpr(const CallExpr *Call,
   // then extracting the other argument.
   CodeGenModule::XteamRedOpKind Opcode;
   std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
-  if (CGM.isOptKernelAMDGCNMax(CallName))
+  if (CGM.isOptKernelAMDGCNMax(Call))
     Opcode = CodeGenModule::XR_OP_max;
-  else if (CGM.isOptKernelAMDGCNMin(CallName))
+  else if (CGM.isOptKernelAMDGCNMin(Call))
     Opcode = CodeGenModule::XR_OP_min;
   else
     llvm_unreachable("Epecting either min or max");
@@ -1135,7 +1136,7 @@ void CodeGenFunction::EmitXteamRedStmtForBuiltinCall(
 
 void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   assert(S && "Null statement?");
-  PGO.setCurrentStmt(S);
+  PGO->setCurrentStmt(S);
 
   // These statements have their own debug info handling.
   if (EmitSimpleStmt(S, Attrs))
@@ -1152,7 +1153,7 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
       // Verify that any decl statements were handled as simple, they may be in
       // scope of subsequent reachable statements.
       assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
-      PGO.markStmtMaybeUsed(S);
+      PGO->markStmtMaybeUsed(S);
       return;
     }
 
@@ -1994,7 +1995,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
         RunCleanupsScope ExecutedScope(*this);
         EmitStmt(Executed);
       }
-      PGO.markStmtMaybeUsed(Skipped);
+      PGO->markStmtMaybeUsed(Skipped);
       return;
     }
   }
@@ -2693,6 +2694,7 @@ void CodeGenFunction::EmitForStmtWithArgs(const ForStmt &S,
 
   if (CGM.shouldEmitConvergenceTokens())
     ConvergenceTokenStack.pop_back();
+
 }
 
 void CodeGenFunction::EmitForStmt(const ForStmt &S,
@@ -2745,7 +2747,14 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
   if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
     BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
         BoolCondVal, Stmt::getLikelihood(S.getBody()));
-  Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
+  auto *I = Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
+  // Key Instructions: Emit the condition and branch as separate atoms to
+  // match existing loop stepping behaviour. FIXME: We could have the branch as
+  // the backup location for the condition, which would probably be a better
+  // experience.
+  if (auto *CondI = dyn_cast<llvm::Instruction>(BoolCondVal))
+    addInstToNewSourceAtom(CondI, nullptr);
+  addInstToNewSourceAtom(I, nullptr);
 
   if (ExitBlock != LoopExit.getBlock()) {
     EmitBlock(ExitBlock);
@@ -2770,6 +2779,9 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
     EmitStmt(S.getLoopVarStmt());
     EmitStmt(S.getBody());
   }
+  // The last block in the loop's body (which unconditionally branches to the
+  // `inc` block if there is one).
+  auto *FinalBodyBB = Builder.GetInsertBlock();
 
   EmitStopPoint(&S);
   // If there is an increment, emit it next.
@@ -2794,6 +2806,12 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
 
   if (CGM.shouldEmitConvergenceTokens())
     ConvergenceTokenStack.pop_back();
+
+  if (FinalBodyBB) {
+    // We want the for closing brace to be step-able on to match existing
+    // behaviour.
+    addInstToNewSourceAtom(FinalBodyBB->getTerminator(), nullptr);
+  }
 }
 
 void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
@@ -2851,6 +2869,7 @@ static bool isSwiftAsyncCallee(const CallExpr *CE) {
 /// if the function returns void, or may be missing one if the function returns
 /// non-void.  Fun stuff :).
 void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
+  ApplyAtomGroup Grp(getDebugInfo());
   if (requiresReturnValueCheck()) {
     llvm::Constant *SLoc = EmitCheckSourceLocation(S.getBeginLoc());
     auto *SLocPtr =
@@ -2926,16 +2945,19 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
     // If this function returns a reference, take the address of the expression
     // rather than the value.
     RValue Result = EmitReferenceBindingToExpr(RV);
-    Builder.CreateStore(Result.getScalarVal(), ReturnValue);
+    auto *I = Builder.CreateStore(Result.getScalarVal(), ReturnValue);
+    addInstToCurrentSourceAtom(I, I->getValueOperand());
   } else {
     switch (getEvaluationKind(RV->getType())) {
     case TEK_Scalar: {
       llvm::Value *Ret = EmitScalarExpr(RV);
-      if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect)
+      if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
         EmitStoreOfScalar(Ret, MakeAddrLValue(ReturnValue, RV->getType()),
                           /*isInit*/ true);
-      else
-        Builder.CreateStore(Ret, ReturnValue);
+      } else {
+        auto *I = Builder.CreateStore(Ret, ReturnValue);
+        addInstToCurrentSourceAtom(I, I->getValueOperand());
+      }
       break;
     }
     case TEK_Complex:
@@ -2980,6 +3002,7 @@ void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
   if (HaveInsertPoint())
     EmitStopPoint(&S);
 
+  ApplyAtomGroup Grp(getDebugInfo());
   EmitBranchThroughCleanup(BreakContinueStack.back().BreakBlock);
 }
 
@@ -2992,6 +3015,7 @@ void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
   if (HaveInsertPoint())
     EmitStopPoint(&S);
 
+  ApplyAtomGroup Grp(getDebugInfo());
   EmitBranchThroughCleanup(BreakContinueStack.back().ContinueBlock);
 }
 
@@ -3545,7 +3569,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
       for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i)
         EmitStmt(CaseStmts[i]);
       incrementProfileCounter(&S);
-      PGO.markStmtMaybeUsed(S.getBody());
+      PGO->markStmtMaybeUsed(S.getBody());
 
       // Now we want to restore the saved switch instance so that nested
       // switches continue to function properly
@@ -3588,7 +3612,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
                             llvm::MDNode::get(CGM.getLLVMContext(), Vals));
   }
 
-  if (PGO.haveRegionCounts()) {
+  if (PGO->haveRegionCounts()) {
     // Walk the SwitchCase list to find how many there are.
     uint64_t DefaultCount = 0;
     unsigned NumCases = 0;
@@ -4609,7 +4633,7 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
     CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();
   }
 
-  PGO.assignRegionCounters(GlobalDecl(CD), F);
+  PGO->assignRegionCounters(GlobalDecl(CD), F);
   CapturedStmtInfo->EmitBody(*this, CD->getBody());
   FinishFunction(CD->getBodyRBrace());
 
diff --git a/external/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp b/external/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 4d9b5191c46a..d0ec866e448d 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -15,6 +15,7 @@
 #include "CGOpenMPRuntime.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenPGO.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
@@ -1028,7 +1029,7 @@ llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunction(
   (void)LocalScope.Privatize();
   for (const auto &VLASizePair : WrapperVLASizes)
     VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
-  PGO.assignRegionCounters(GlobalDecl(CD), F);
+  PGO->assignRegionCounters(GlobalDecl(CD), F);
 
   // Generate specialized kernels for device only
   if (CGM.getLangOpts().OpenMPIsTargetDevice && D.hasAssociatedStmt() &&
@@ -1822,6 +1823,7 @@ void CodeGenFunction::EmitOMPReductionClauseFinal(
   llvm::SmallVector<const Expr *, 8> LHSExprs;
   llvm::SmallVector<const Expr *, 8> RHSExprs;
   llvm::SmallVector<const Expr *, 8> ReductionOps;
+  llvm::SmallVector<bool, 8> IsPrivateVarReduction;
   bool HasAtLeastOneReduction = false;
   bool IsReductionWithTaskMod = false;
   for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
@@ -1832,6 +1834,8 @@ void CodeGenFunction::EmitOMPReductionClauseFinal(
     Privates.append(C->privates().begin(), C->privates().end());
     LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
     RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
+    IsPrivateVarReduction.append(C->private_var_reduction_flags().begin(),
+                                 C->private_var_reduction_flags().end());
     ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
     IsReductionWithTaskMod =
         IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
@@ -1853,7 +1857,7 @@ void CodeGenFunction::EmitOMPReductionClauseFinal(
     // parallel directive (it always has implicit barrier).
     CGM.getOpenMPRuntime().emitReduction(
         *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
-        {WithNowait, SimpleReduction, ReductionKind});
+        {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});
   }
 }
 
@@ -4449,7 +4453,8 @@ static void emitScanBasedDirective(
       PrivScope.Privatize();
       CGF.CGM.getOpenMPRuntime().emitReduction(
           CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
-          {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_unknown});
+          {/*WithNowait=*/true, /*SimpleReduction=*/true,
+           /*IsPrivateVarReduction*/ {}, OMPD_unknown});
     }
     llvm::Value *NextIVal =
         CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
@@ -6267,7 +6272,8 @@ void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) {
       }
       CGM.getOpenMPRuntime().emitReduction(
           *this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
-          {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_simd});
+          {/*WithNowait=*/true, /*SimpleReduction=*/true,
+           /*IsPrivateVarReduction*/ {}, OMPD_simd});
       for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
         const Expr *PrivateExpr = Privates[I];
         LValue DestLVal;
diff --git a/external/llvm-project/clang/lib/CodeGen/CGVTables.cpp b/external/llvm-project/clang/lib/CodeGen/CGVTables.cpp
index c7447273a42f..2897ccdf8866 100644
--- a/external/llvm-project/clang/lib/CodeGen/CGVTables.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CGVTables.cpp
@@ -124,7 +124,7 @@ static void resolveTopLevelMetadata(llvm::Function *Fn,
   auto *DIS = Fn->getSubprogram();
   if (!DIS)
     return;
-  auto *NewDIS = DIS->replaceWithDistinct(DIS->clone());
+  auto *NewDIS = llvm::MDNode::replaceWithDistinct(DIS->clone());
   VMap.MD()[DIS].reset(NewDIS);
 
   // Find all llvm.dbg.declare intrinsics and resolve the DILocalVariable nodes
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp b/external/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp
index 25659f9cb80b..2d6afb4530ac 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp
@@ -274,8 +274,8 @@ void BackendConsumer::HandleTranslationUnit(ASTContext &C) {
   std::unique_ptr<llvm::ToolOutputFile> OptRecordFile =
     std::move(*OptRecordFileOrErr);
 
-  if (OptRecordFile &&
-      CodeGenOpts.getProfileUse() != CodeGenOptions::ProfileNone)
+  if (OptRecordFile && CodeGenOpts.getProfileUse() !=
+                           llvm::driver::ProfileInstrKind::ProfileNone)
     Ctx.setDiagnosticsHotnessRequested(true);
 
   if (CodeGenOpts.MisExpect) {
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp b/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp
index 4193f0a1b278..13d0633e9b1c 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -80,7 +80,8 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
       Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
               CGBuilderInserterTy(this)),
       SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
-      DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
+      DebugInfo(CGM.getModuleDebugInfo()),
+      PGO(std::make_unique<CodeGenPGO>(cgm)),
       ShouldEmitLifetimeMarkers(
           shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
   if (!suppressNewContext)
@@ -443,8 +444,10 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
 
   // Reset the debug location to that of the simple 'return' expression, if any
   // rather than that of the end of the function's scope '}'.
+  uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;
   ApplyDebugLocation AL(*this, Loc);
-  EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
+  EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc,
+                     RetKeyInstructionsAtomGroup);
   EmitEndEHSpec(CurCodeDecl);
 
   assert(EHStack.empty() &&
@@ -623,7 +626,7 @@ CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
 
 void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
                                          llvm::Function *Fn) {
-  if (!FD->hasAttr<OpenCLKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
+  if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
     return;
 
   llvm::LLVMContext &Context = getLLVMContext();
@@ -940,7 +943,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
     }
   }
 
-  if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone) {
+  if (CGM.getCodeGenOpts().getProfileInstr() !=
+      llvm::driver::ProfileInstrKind::ProfileNone) {
     switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
     case ProfileList::Skip:
       Fn->addFnAttr(llvm::Attribute::SkipProfile);
@@ -1564,7 +1568,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
     CurFn->addFnAttr(llvm::Attribute::MustProgress);
 
   // Generate the body of the function.
-  PGO.assignRegionCounters(GD, CurFn);
+  PGO->assignRegionCounters(GD, CurFn);
   if (isa<CXXDestructorDecl>(FD))
     EmitDestructorBody(Args);
   else if (isa<CXXConstructorDecl>(FD))
@@ -1595,7 +1599,8 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
     // Implicit copy-assignment gets the same special treatment as implicit
     // copy-constructors.
     emitImplicitAssignmentOperatorBody(Args);
-  } else if (FD->hasAttr<OpenCLKernelAttr>() &&
+  } else if (DeviceKernelAttr::isOpenCLSpelling(
+                 FD->getAttr<DeviceKernelAttr>()) &&
              GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {
     CallArgList CallArgs;
     for (unsigned i = 0; i < Args.size(); ++i) {
@@ -1632,10 +1637,11 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
         CGM.getCodeGenOpts().StrictReturn ||
         !CGM.MayDropFunctionReturn(FD->getASTContext(), FD->getReturnType());
     if (SanOpts.has(SanitizerKind::Return)) {
-      SanitizerScope SanScope(this);
+      auto CheckOrdinal = SanitizerKind::SO_Return;
+      auto CheckHandler = SanitizerHandler::MissingReturn;
+      SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
       llvm::Value *IsFalse = Builder.getFalse();
-      EmitCheck(std::make_pair(IsFalse, SanitizerKind::SO_Return),
-                SanitizerHandler::MissingReturn,
+      EmitCheck(std::make_pair(IsFalse, CheckOrdinal), CheckHandler,
                 EmitCheckSourceLocation(FD->getLocation()), {});
     } else if (ShouldEmitUnreachable) {
       if (CGM.getCodeGenOpts().OptimizationLevel == 0)
@@ -1650,7 +1656,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
   // Emit the standard function epilogue.
   FinishFunction(BodyRange.getEnd());
 
-  PGO.verifyCounterMap();
+  PGO->verifyCounterMap();
 
   // If we haven't marked the function nothrow through other means, do
   // a quick pass now to see if we can.
@@ -1774,7 +1780,7 @@ bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
   if (!AllowLabels && CodeGenFunction::ContainsLabel(Cond))
     return false;  // Contains a label.
 
-  PGO.markStmtMaybeUsed(Cond);
+  PGO->markStmtMaybeUsed(Cond);
   ResultInt = Int;
   return true;
 }
@@ -2537,7 +2543,9 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
           //   expression [...] each time it is evaluated it shall have a value
           //   greater than zero.
           if (SanOpts.has(SanitizerKind::VLABound)) {
-            SanitizerScope SanScope(this);
+            auto CheckOrdinal = SanitizerKind::SO_VLABound;
+            auto CheckHandler = SanitizerHandler::VLABoundNotPositive;
+            SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
             llvm::Value *Zero = llvm::Constant::getNullValue(size->getType());
             clang::QualType SEType = sizeExpr->getType();
             llvm::Value *CheckCondition =
@@ -2547,9 +2555,8 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
             llvm::Constant *StaticArgs[] = {
                 EmitCheckSourceLocation(sizeExpr->getBeginLoc()),
                 EmitCheckTypeDescriptor(SEType)};
-            EmitCheck(
-                std::make_pair(CheckCondition, SanitizerKind::SO_VLABound),
-                SanitizerHandler::VLABoundNotPositive, StaticArgs, size);
+            EmitCheck(std::make_pair(CheckCondition, CheckOrdinal),
+                      CheckHandler, StaticArgs, size);
           }
 
           // Always zexting here would be wrong if it weren't
@@ -3192,7 +3199,9 @@ void CodeGenFunction::emitAlignmentAssumptionCheck(
   Assumption->removeFromParent();
 
   {
-    SanitizerScope SanScope(this);
+    auto CheckOrdinal = SanitizerKind::SO_Alignment;
+    auto CheckHandler = SanitizerHandler::AlignmentAssumption;
+    SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
 
     if (!OffsetValue)
       OffsetValue = Builder.getInt1(false); // no offset.
@@ -3201,8 +3210,8 @@ void CodeGenFunction::emitAlignmentAssumptionCheck(
                                     EmitCheckSourceLocation(SecondaryLoc),
                                     EmitCheckTypeDescriptor(Ty)};
     llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};
-    EmitCheck({std::make_pair(TheCheck, SanitizerKind::SO_Alignment)},
-              SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
+    EmitCheck({std::make_pair(TheCheck, CheckOrdinal)}, CheckHandler,
+              StaticData, DynamicData);
   }
 
   // We are now in the (new, empty) "cont" basic block.
@@ -3355,4 +3364,4 @@ void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
     ApplyAtomGroup Grp(getDebugInfo());
     DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
   }
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.h b/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.h
index 3b1e060cc6f6..5842ebaaf9b3 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.h
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenFunction.h
@@ -17,8 +17,8 @@
 #include "CGLoopInfo.h"
 #include "CGValue.h"
 #include "CodeGenModule.h"
-#include "CodeGenPGO.h"
 #include "EHScopeStack.h"
+#include "SanitizerHandler.h"
 #include "VarBypassDetector.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/CurrentSourceLocExprScope.h"
@@ -92,6 +92,7 @@ class OSLogBufferLayout;
 
 namespace CodeGen {
 class CodeGenTypes;
+class CodeGenPGO;
 class CGCallee;
 class CGFunctionInfo;
 class CGBlockInfo;
@@ -117,40 +118,6 @@ enum TypeEvaluationKind {
 };
 // clang-format on
 
-#define LIST_SANITIZER_CHECKS                                                  \
-  SANITIZER_CHECK(AddOverflow, add_overflow, 0)                                \
-  SANITIZER_CHECK(BuiltinUnreachable, builtin_unreachable, 0)                  \
-  SANITIZER_CHECK(CFICheckFail, cfi_check_fail, 0)                             \
-  SANITIZER_CHECK(DivremOverflow, divrem_overflow, 0)                          \
-  SANITIZER_CHECK(DynamicTypeCacheMiss, dynamic_type_cache_miss, 0)            \
-  SANITIZER_CHECK(FloatCastOverflow, float_cast_overflow, 0)                   \
-  SANITIZER_CHECK(FunctionTypeMismatch, function_type_mismatch, 0)             \
-  SANITIZER_CHECK(ImplicitConversion, implicit_conversion, 0)                  \
-  SANITIZER_CHECK(InvalidBuiltin, invalid_builtin, 0)                          \
-  SANITIZER_CHECK(InvalidObjCCast, invalid_objc_cast, 0)                       \
-  SANITIZER_CHECK(LoadInvalidValue, load_invalid_value, 0)                     \
-  SANITIZER_CHECK(MissingReturn, missing_return, 0)                            \
-  SANITIZER_CHECK(MulOverflow, mul_overflow, 0)                                \
-  SANITIZER_CHECK(NegateOverflow, negate_overflow, 0)                          \
-  SANITIZER_CHECK(NullabilityArg, nullability_arg, 0)                          \
-  SANITIZER_CHECK(NullabilityReturn, nullability_return, 1)                    \
-  SANITIZER_CHECK(NonnullArg, nonnull_arg, 0)                                  \
-  SANITIZER_CHECK(NonnullReturn, nonnull_return, 1)                            \
-  SANITIZER_CHECK(OutOfBounds, out_of_bounds, 0)                               \
-  SANITIZER_CHECK(PointerOverflow, pointer_overflow, 0)                        \
-  SANITIZER_CHECK(ShiftOutOfBounds, shift_out_of_bounds, 0)                    \
-  SANITIZER_CHECK(SubOverflow, sub_overflow, 0)                                \
-  SANITIZER_CHECK(TypeMismatch, type_mismatch, 1)                              \
-  SANITIZER_CHECK(AlignmentAssumption, alignment_assumption, 0)                \
-  SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0)              \
-  SANITIZER_CHECK(BoundsSafety, bounds_safety, 0)
-
-enum SanitizerHandler {
-#define SANITIZER_CHECK(Enum, Name, Version) Enum,
-  LIST_SANITIZER_CHECKS
-#undef SANITIZER_CHECK
-};
-
 /// Helper class with most of the code for saving a value for a
 /// conditional expression cleanup.
 struct DominatingLLVMValue {
@@ -1672,7 +1639,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
                                                     Stmt::Likelihood LH);
 
-  CodeGenPGO PGO;
+  std::unique_ptr<CodeGenPGO> PGO;
 
   /// Bitmap used by MC/DC to track condition outcomes of a boolean expression.
   Address MCDCCondBitmapAddr = Address::invalid();
@@ -1685,12 +1652,9 @@ class CodeGenFunction : public CodeGenTypeCache {
                                             uint64_t LoopCount) const;
 
 public:
-  auto getIsCounterPair(const Stmt *S) const { return PGO.getIsCounterPair(S); }
-
-  void markStmtAsUsed(bool Skipped, const Stmt *S) {
-    PGO.markStmtAsUsed(Skipped, S);
-  }
-  void markStmtMaybeUsed(const Stmt *S) { PGO.markStmtMaybeUsed(S); }
+  std::pair<bool, bool> getIsCounterPair(const Stmt *S) const;
+  void markStmtAsUsed(bool Skipped, const Stmt *S);
+  void markStmtMaybeUsed(const Stmt *S);
 
   /// Increment the profiler's counter for the given statement by \p StepV.
   /// If \p StepV is null, the default increment is 1.
@@ -1704,13 +1668,7 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   /// Allocate a temp value on the stack that MCDC can use to track condition
   /// results.
-  void maybeCreateMCDCCondBitmap() {
-    if (isMCDCCoverageEnabled()) {
-      PGO.emitMCDCParameters(Builder);
-      MCDCCondBitmapAddr =
-          CreateIRTemp(getContext().UnsignedIntTy, "mcdc.addr");
-    }
-  }
+  void maybeCreateMCDCCondBitmap();
 
   bool isBinaryLogicalOp(const Expr *E) const {
     const BinaryOperator *BOp = dyn_cast<BinaryOperator>(E->IgnoreParens());
@@ -1718,43 +1676,24 @@ class CodeGenFunction : public CodeGenTypeCache {
   }
 
   /// Zero-init the MCDC temp value.
-  void maybeResetMCDCCondBitmap(const Expr *E) {
-    if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
-      PGO.emitMCDCCondBitmapReset(Builder, E, MCDCCondBitmapAddr);
-      PGO.setCurrentStmt(E);
-    }
-  }
+  void maybeResetMCDCCondBitmap(const Expr *E);
 
   /// Increment the profiler's counter for the given expression by \p StepV.
   /// If \p StepV is null, the default increment is 1.
-  void maybeUpdateMCDCTestVectorBitmap(const Expr *E) {
-    if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
-      PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this);
-      PGO.setCurrentStmt(E);
-    }
-  }
+  void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
 
   /// Update the MCDC temp value with the condition's evaluated result.
-  void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) {
-    if (isMCDCCoverageEnabled()) {
-      PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this);
-      PGO.setCurrentStmt(E);
-    }
-  }
+  void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
 
   /// Get the profiler's count for the given statement.
-  uint64_t getProfileCount(const Stmt *S) {
-    return PGO.getStmtCount(S).value_or(0);
-  }
+  uint64_t getProfileCount(const Stmt *S);
 
   /// Set the profiler's current count.
-  void setCurrentProfileCount(uint64_t Count) {
-    PGO.setCurrentRegionCount(Count);
-  }
+  void setCurrentProfileCount(uint64_t Count);
 
   /// Get the profiler's current count. This is generally the count for the most
   /// recently incremented counter.
-  uint64_t getCurrentProfileCount() { return PGO.getCurrentRegionCount(); }
+  uint64_t getCurrentProfileCount();
 
   /// See CGDebugInfo::addInstToCurrentSourceAtom.
   void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
@@ -2614,9 +2553,12 @@ class CodeGenFunction : public CodeGenTypeCache {
                           const FunctionArgList &Args);
 
   /// EmitFunctionEpilog - Emit the target specific LLVM code to return the
-  /// given temporary.
+  /// given temporary. Specify the source location atom group (Key Instructions
+  /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
+  /// If it's 0, the `ret` will get added to a new source atom group.
   void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
-                          SourceLocation EndLoc);
+                          SourceLocation EndLoc,
+                          uint64_t RetKeyInstructionsSourceAtom);
 
   /// Emit a test that checks if the return value \p RV is nonnull.
   void EmitReturnValueCheck(llvm::Value *RV);
@@ -3396,10 +3338,12 @@ class CodeGenFunction : public CodeGenTypeCache {
                            llvm::Value *Index, QualType IndexType,
                            QualType IndexedType, bool Accessed);
 
-  /// Returns debug info, with additional annotation if enabled by
-  /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[CheckKindOrdinal].
+  /// Returns debug info, with additional annotation if
+  /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
+  /// any of the ordinals.
   llvm::DILocation *
-  SanitizerAnnotateDebugInfo(SanitizerKind::SanitizerOrdinal CheckKindOrdinal);
+  SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
+                             SanitizerHandler Handler);
 
   llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
                                         const FieldDecl *CountDecl);
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp b/external/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp
index 61372b5284d7..a4eda9984217 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp
@@ -418,6 +418,11 @@ CodeGenModule::CodeGenModule(ASTContext &C,
       CodeGenOpts.CoverageNotesFile.size() ||
       CodeGenOpts.CoverageDataFile.size())
     DebugInfo.reset(new CGDebugInfo(*this));
+  else if (getTriple().isOSWindows())
+    // On Windows targets, we want to emit compiler info even if debug info is
+    // otherwise disabled. Use a temporary CGDebugInfo instance to emit only
+    // basic compiler metadata.
+    CGDebugInfo(*this);
 
   Block.GlobalUniqueCount = 0;
 
@@ -1055,7 +1060,7 @@ void CodeGenModule::Release() {
                               "StrictVTablePointersRequirement",
                               llvm::MDNode::get(VMContext, Ops));
   }
-  if (getModuleDebugInfo())
+  if (getModuleDebugInfo() || getTriple().isOSWindows())
     // We support a single version in the linked module. The LLVM
     // parser will drop debug info with a different version number
     // (and warn about it, too).
@@ -1150,8 +1155,13 @@ void CodeGenModule::Release() {
                               1);
   }
 
-  if (CodeGenOpts.UniqueSourceFileNames) {
-    getModule().addModuleFlag(llvm::Module::Max, "Unique Source File Names", 1);
+  if (!CodeGenOpts.UniqueSourceFileIdentifier.empty()) {
+    getModule().addModuleFlag(
+        llvm::Module::Append, "Unique Source File Identifier",
+        llvm::MDTuple::get(
+            TheModule.getContext(),
+            llvm::MDString::get(TheModule.getContext(),
+                                CodeGenOpts.UniqueSourceFileIdentifier)));
   }
 
   if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) {
@@ -1917,7 +1927,9 @@ static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD,
     } else if (FD && FD->hasAttr<CUDAGlobalAttr>() &&
                GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
       Out << "__device_stub__" << II->getName();
-    } else if (FD && FD->hasAttr<OpenCLKernelAttr>() &&
+    } else if (FD &&
+               DeviceKernelAttr::isOpenCLSpelling(
+                   FD->getAttr<DeviceKernelAttr>()) &&
                GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
       Out << "__clang_ocl_kern_imp_" << II->getName();
     } else {
@@ -3608,7 +3620,7 @@ CodeGenModule::isFunctionBlockedByProfileList(llvm::Function *Fn,
   // If the profile list is empty, then instrument everything.
   if (ProfileList.isEmpty())
     return ProfileList::Allow;
-  CodeGenOptions::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr();
+  llvm::driver::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr();
   // First, check the function name.
   if (auto V = ProfileList.isFunctionExcluded(Fn->getName(), Kind))
     return *V;
@@ -3937,7 +3949,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
 
   // Ignore declarations, they will be emitted on their first use.
   if (const auto *FD = dyn_cast<FunctionDecl>(Global)) {
-    if (FD->hasAttr<OpenCLKernelAttr>() && FD->doesThisDeclarationHaveABody())
+    if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()) &&
+        FD->doesThisDeclarationHaveABody())
       addDeferredDeclToEmit(GlobalDecl(FD, KernelReferenceKind::Stub));
 
     // Update deferred annotations with the latest declaration if the function
@@ -4244,19 +4257,19 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
       EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
   } else if (auto *TC = FD->getAttr<TargetClonesAttr>()) {
     for (unsigned I = 0; I < TC->featuresStrs_size(); ++I)
-      // AArch64 favors the default target version over the clone if any.
-      if ((!TC->isDefaultVersion(I) || !getTarget().getTriple().isAArch64()) &&
-          TC->isFirstOfVersion(I))
+      if (TC->isFirstOfVersion(I))
         EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
-    // Ensure that the resolver function is also emitted.
-    GetOrCreateMultiVersionResolver(GD);
   } else
     EmitGlobalFunctionDefinition(GD, GV);
 
-  // Defer the resolver emission until we can reason whether the TU
-  // contains a default target version implementation.
-  if (FD->isTargetVersionMultiVersion())
-    AddDeferredMultiVersionResolverToEmit(GD);
+  // Ensure that the resolver function is also emitted.
+  if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) {
+    // On AArch64 defer the resolver emission until the entire TU is processed.
+    if (getTarget().getTriple().isAArch64())
+      AddDeferredMultiVersionResolverToEmit(GD);
+    else
+      GetOrCreateMultiVersionResolver(GD);
+  }
 }
 
 void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
@@ -4358,7 +4371,7 @@ void CodeGenModule::emitMultiVersionFunctions() {
     };
 
     // For AArch64, a resolver is only emitted if a function marked with
-    // target_version("default")) or target_clones() is present and defined
+    // target_version("default")) or target_clones("default") is defined
     // in this TU. For other architectures it is always emitted.
     bool ShouldEmitResolver = !getTarget().getTriple().isAArch64();
     SmallVector<CodeGenFunction::FMVResolverOption, 10> Options;
@@ -4381,12 +4394,11 @@ void CodeGenModule::emitMultiVersionFunctions() {
             TVA->getFeatures(Feats, Delim);
             Options.emplace_back(Func, Feats);
           } else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
-            if (IsDefined)
-              ShouldEmitResolver = true;
             for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
               if (!TC->isFirstOfVersion(I))
                 continue;
-
+              if (TC->isDefaultVersion(I) && IsDefined)
+                ShouldEmitResolver = true;
               llvm::Function *Func = createFunction(CurFD, I);
               Feats.clear();
               if (getTarget().getTriple().isX86()) {
@@ -4903,7 +4915,7 @@ CodeGenModule::GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty, bool ForVTable,
   if (!Ty) {
     const auto *FD = cast<FunctionDecl>(GD.getDecl());
     Ty = getTypes().ConvertType(FD->getType());
-    if (FD->hasAttr<OpenCLKernelAttr>() &&
+    if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()) &&
         GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
       const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
       Ty = getTypes().GetFunctionType(FI);
@@ -5771,7 +5783,17 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
     getCUDARuntime().handleVarRegistration(D, *GV);
   }
 
-  GV->setInitializer(Init);
+  if (LangOpts.HLSL && GetGlobalVarAddressSpace(D) == LangAS::hlsl_input) {
+    // HLSL Input variables are considered to be set by the driver/pipeline, but
+    // only visible to a single thread/wave.
+    GV->setExternallyInitialized(true);
+  } else {
+    GV->setInitializer(Init);
+  }
+
+  if (LangOpts.HLSL)
+    getHLSLRuntime().handleGlobalVarDefinition(D, GV);
+
   if (emitter)
     emitter->finalize(GV);
 
@@ -5814,6 +5836,12 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
       !D->hasAttr<ConstInitAttr>())
     Linkage = llvm::GlobalValue::InternalLinkage;
 
+  // HLSL variables in the input address space maps like memory-mapped
+  // variables. Even if they are 'static', they are externally initialized and
+  // read/write by the hardware/driver/pipeline.
+  if (LangOpts.HLSL && GetGlobalVarAddressSpace(D) == LangAS::hlsl_input)
+    Linkage = llvm::GlobalValue::ExternalLinkage;
+
   GV->setLinkage(Linkage);
   if (D->hasAttr<DLLImportAttr>())
     GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
@@ -6187,7 +6215,7 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
                           (CodeGenOpts.OptimizationLevel == 0) &&
                           !D->hasAttr<MinSizeAttr>();
 
-  if (D->hasAttr<OpenCLKernelAttr>()) {
+  if (DeviceKernelAttr::isOpenCLSpelling(D->getAttr<DeviceKernelAttr>())) {
     if (GD.getKernelReferenceKind() == KernelReferenceKind::Stub &&
         !D->hasAttr<NoInlineAttr>() &&
         !Fn->hasFnAttribute(llvm::Attribute::NoInline) &&
@@ -7255,7 +7283,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
     if (LangOpts.SYCLIsDevice)
       break;
     auto *AD = cast<FileScopeAsmDecl>(D);
-    getModule().appendModuleInlineAsm(AD->getAsmString()->getString());
+    getModule().appendModuleInlineAsm(AD->getAsmString());
     break;
   }
 
@@ -9668,9 +9696,34 @@ void CodeGenModule::resetOptKernelMetadata(const Stmt *DirectiveStmt) {
     eraseOptKernelNestElem(getOptKernelKey(*Dir));
 }
 
+bool CodeGenModule::isStdNameSpace(const CallExpr *Call) const {
+  // Examine the first child, the call itself.
+  const Stmt *CE = nullptr;
+  for (const Stmt *Child : Call->children())
+    if (Child) {
+      CE = Child;
+      break;
+    }
+  if (CE) {
+    while (isa<ImplicitCastExpr>(CE))
+      CE = cast<ImplicitCastExpr>(CE)->getSubExpr();
+    if (isa<DeclRefExpr>(CE)) {
+      const DeclRefExpr *DRE = cast<DeclRefExpr>(CE);
+      if (DRE->hasQualifier()) {
+        const NestedNameSpecifier *NS = DRE->getQualifier();
+        if (NS->getKind() == NestedNameSpecifier::Namespace &&
+            !NS->getAsNamespace()->getNameAsString().compare("std"))
+          return true;
+      }
+    }
+  }
+  return false;
+}
+
 CodeGenModule::NoLoopXteamErr
-CodeGenModule::getStatusOptKernelHostBuiltin(std::string CallName) const {
-  if (isOptKernelHostMin(CallName) || isOptKernelHostMax(CallName))
+CodeGenModule::getStatusOptKernelHostBuiltin(const CallExpr *Call) const {
+  std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
+  if (isOptKernelHostMin(Call) || isOptKernelHostMax(Call))
     return NxSuccess;
   auto emitDebugMsg = [](std::string Msg) {
     Msg += ": Not recognized as builtin in host compile";
@@ -9681,8 +9734,9 @@ CodeGenModule::getStatusOptKernelHostBuiltin(std::string CallName) const {
 }
 
 CodeGenModule::NoLoopXteamErr
-CodeGenModule::getStatusOptKernelAMDGCNBuiltin(std::string CallName) const {
-  if (isOptKernelAMDGCNMin(CallName) || isOptKernelAMDGCNMax(CallName))
+CodeGenModule::getStatusOptKernelAMDGCNBuiltin(const CallExpr *Call) const {
+  std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
+  if (isOptKernelAMDGCNMin(Call) || isOptKernelAMDGCNMax(Call))
     return NxSuccess;
   auto emitDebugMsg = [](std::string Msg) {
     Msg += ": Not recognized as builtin in device compile";
@@ -9694,12 +9748,11 @@ CodeGenModule::getStatusOptKernelAMDGCNBuiltin(std::string CallName) const {
 
 CodeGenModule::NoLoopXteamErr
 CodeGenModule::getStatusOptKernelBuiltin(const CallExpr *Call) {
-  std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
   if (getLangOpts().OpenMPIsTargetDevice) {
-    if (auto NxStatus = getStatusOptKernelAMDGCNBuiltin(CallName))
+    if (auto NxStatus = getStatusOptKernelAMDGCNBuiltin(Call))
       return NxStatus;
   } else {
-    if (auto NxStatus = getStatusOptKernelHostBuiltin(CallName))
+    if (auto NxStatus = getStatusOptKernelHostBuiltin(Call))
       return NxStatus;
   }
   return NxSuccess;
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenModule.h b/external/llvm-project/clang/lib/CodeGen/CodeGenModule.h
index a357238646d8..38659b949fcf 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenModule.h
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenModule.h
@@ -2132,10 +2132,9 @@ class CodeGenModule : public CodeGenTypeCache {
   void updateXteamRedVarOpcode(const CallExpr *Call, const VarDecl *VD,
                                XteamRedVarMap *RedMap) {
     XteamRedOpKind Opcode;
-    std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
-    if (isOptKernelAMDGCNMax(CallName))
+    if (isOptKernelAMDGCNMax(Call))
       Opcode = XR_OP_max;
-    else if (isOptKernelAMDGCNMin(CallName))
+    else if (isOptKernelAMDGCNMin(Call))
       Opcode = XR_OP_min;
     else
       llvm_unreachable("Expected either min or max");
@@ -2200,10 +2199,16 @@ class CodeGenModule : public CodeGenTypeCache {
   /// Return status indicating whether the call is an Xteam-supported host
   /// builtin.
   CodeGenModule::NoLoopXteamErr
-  getStatusOptKernelHostBuiltin(std::string CallName) const;
+  getStatusOptKernelHostBuiltin(const CallExpr *C) const;
+
+  /// Is the callee in std namespace?
+  bool isStdNameSpace(const CallExpr *Call) const;
 
   /// Is the function name recognized as a min builtin by the host compile?
-  bool isOptKernelHostMin(std::string CallName) const {
+  bool isOptKernelHostMin(const CallExpr *Call) const {
+    std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
+    if (isStdNameSpace(Call) && !CallName.compare("min"))
+      return true;
     return (!CallName.compare("fmin") || !CallName.compare("fminf") ||
             !CallName.compare("fminl") || !CallName.compare("__builtin_fmin") ||
             !CallName.compare("__builtin_fminf") ||
@@ -2211,7 +2216,10 @@ class CodeGenModule : public CodeGenTypeCache {
   }
 
   /// Is the function name recognized as a max builtin by the host compile?
-  bool isOptKernelHostMax(std::string CallName) const {
+  bool isOptKernelHostMax(const CallExpr *Call) const {
+    std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
+    if (isStdNameSpace(Call) && !CallName.compare("max"))
+      return true;
     return (!CallName.compare("fmax") || !CallName.compare("fmaxf") ||
             !CallName.compare("fmaxl") || !CallName.compare("__builtin_fmax") ||
             !CallName.compare("__builtin_fmaxf") ||
@@ -2221,10 +2229,13 @@ class CodeGenModule : public CodeGenTypeCache {
   /// Return status indicating whether the amdgcn device function is supported
   /// by Xteam.
   CodeGenModule::NoLoopXteamErr
-  getStatusOptKernelAMDGCNBuiltin(std::string CallName) const;
+  getStatusOptKernelAMDGCNBuiltin(const CallExpr *C) const;
 
   /// Is the function name recognized as a min builtin by the device compile?
-  bool isOptKernelAMDGCNMin(std::string CallName) const {
+  bool isOptKernelAMDGCNMin(const CallExpr *Call) const {
+    std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
+    if (isStdNameSpace(Call) && !CallName.compare("min"))
+      return true;
     return (!CallName.compare("fmin[device={arch(amdgcn)}]") ||
             !CallName.compare("fminf[device={arch(amdgcn)}]") ||
             !CallName.compare("fminl[device={arch(amdgcn)}]") ||
@@ -2235,7 +2246,10 @@ class CodeGenModule : public CodeGenTypeCache {
   }
 
   // Is the function name recognized as a max builtin by the device compile?
-  bool isOptKernelAMDGCNMax(std::string CallName) const {
+  bool isOptKernelAMDGCNMax(const CallExpr *Call) const {
+    std::string CallName = Call->getDirectCallee()->getNameInfo().getAsString();
+    if (isStdNameSpace(Call) && !CallName.compare("max"))
+      return true;
     return (!CallName.compare("fmax[device={arch(amdgcn)}]") ||
             !CallName.compare("fmaxf[device={arch(amdgcn)}]") ||
             !CallName.compare("fmaxl[device={arch(amdgcn)}]") ||
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp b/external/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp
index 8197c5f9d37b..a80bebbb4cf4 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -1505,9 +1505,9 @@ CodeGenFunction::createProfileWeights(ArrayRef<uint64_t> Weights) const {
 llvm::MDNode *
 CodeGenFunction::createProfileWeightsForLoop(const Stmt *Cond,
                                              uint64_t LoopCount) const {
-  if (!PGO.haveRegionCounts())
+  if (!PGO->haveRegionCounts())
     return nullptr;
-  std::optional<uint64_t> CondCount = PGO.getStmtCount(Cond);
+  std::optional<uint64_t> CondCount = PGO->getStmtCount(Cond);
   if (!CondCount || *CondCount == 0)
     return nullptr;
   return createProfileWeights(LoopCount,
@@ -1520,7 +1520,59 @@ void CodeGenFunction::incrementProfileCounter(const Stmt *S,
       !CurFn->hasFnAttribute(llvm::Attribute::NoProfile) &&
       !CurFn->hasFnAttribute(llvm::Attribute::SkipProfile)) {
     auto AL = ApplyDebugLocation::CreateArtificial(*this);
-    PGO.emitCounterSetOrIncrement(Builder, S, StepV);
+    PGO->emitCounterSetOrIncrement(Builder, S, StepV);
   }
-  PGO.setCurrentStmt(S);
+  PGO->setCurrentStmt(S);
+}
+
+std::pair<bool, bool> CodeGenFunction::getIsCounterPair(const Stmt *S) const {
+  return PGO->getIsCounterPair(S);
+}
+void CodeGenFunction::markStmtAsUsed(bool Skipped, const Stmt *S) {
+  PGO->markStmtAsUsed(Skipped, S);
+}
+void CodeGenFunction::markStmtMaybeUsed(const Stmt *S) {
+  PGO->markStmtMaybeUsed(S);
+}
+
+void CodeGenFunction::maybeCreateMCDCCondBitmap() {
+  if (isMCDCCoverageEnabled()) {
+    PGO->emitMCDCParameters(Builder);
+    MCDCCondBitmapAddr = CreateIRTemp(getContext().UnsignedIntTy, "mcdc.addr");
+  }
+}
+void CodeGenFunction::maybeResetMCDCCondBitmap(const Expr *E) {
+  if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
+    PGO->emitMCDCCondBitmapReset(Builder, E, MCDCCondBitmapAddr);
+    PGO->setCurrentStmt(E);
+  }
+}
+void CodeGenFunction::maybeUpdateMCDCTestVectorBitmap(const Expr *E) {
+  if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
+    PGO->emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this);
+    PGO->setCurrentStmt(E);
+  }
+}
+
+void CodeGenFunction::maybeUpdateMCDCCondBitmap(const Expr *E,
+                                                llvm::Value *Val) {
+  if (isMCDCCoverageEnabled()) {
+    PGO->emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this);
+    PGO->setCurrentStmt(E);
+  }
+}
+
+uint64_t CodeGenFunction::getProfileCount(const Stmt *S) {
+  return PGO->getStmtCount(S).value_or(0);
+}
+
+/// Set the profiler's current count.
+void CodeGenFunction::setCurrentProfileCount(uint64_t Count) {
+  PGO->setCurrentRegionCount(Count);
+}
+
+/// Get the profiler's current count. This is generally the count for the most
+/// recently incremented counter.
+uint64_t CodeGenFunction::getCurrentProfileCount() {
+  return PGO->getCurrentRegionCount();
 }
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp b/external/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp
index 15fb2400e3ff..bb278f308f0c 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -129,6 +129,13 @@ static bool TypeHasMayAlias(QualType QTy) {
       return true;
     QTy = TT->desugar();
   }
+
+  // Also consider an array type as may_alias when its element type (at
+  // any level) is marked as such.
+  if (auto *ArrayTy = QTy->getAsArrayTypeUnsafe())
+    if (TypeHasMayAlias(ArrayTy->getElementType()))
+      return true;
+
   return false;
 }
 
diff --git a/external/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp b/external/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp
index 09f6b20f53ec..c98503e4bbd2 100644
--- a/external/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -726,7 +726,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
     auto *MPTy = cast<MemberPointerType>(Ty);
     if (!getCXXABI().isMemberPointerConvertible(MPTy)) {
       auto *C = MPTy->getMostRecentCXXRecordDecl()->getTypeForDecl();
-      auto Insertion = RecordsWithOpaqueMemberPointers.insert({C, nullptr});
+      auto Insertion = RecordsWithOpaqueMemberPointers.try_emplace(C);
       if (Insertion.second)
         Insertion.first->second = llvm::StructType::create(getLLVMContext());
       ResultType = Insertion.first->second;
diff --git a/external/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp b/external/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp
index 6170dc7f5910..4aafac349e3e 100644
--- a/external/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -12,6 +12,7 @@
 
 #include "CoverageMappingGen.h"
 #include "CodeGenFunction.h"
+#include "CodeGenPGO.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Lex/Lexer.h"
@@ -2621,8 +2622,9 @@ void CoverageMappingModuleGen::emit() {
   CGM.addUsedGlobal(CovData);
   // Create the deferred function records array
   if (!FunctionNames.empty()) {
-    auto NamesArrTy = llvm::ArrayType::get(llvm::PointerType::getUnqual(Ctx),
-                                           FunctionNames.size());
+    auto AddrSpace = FunctionNames.front()->getType()->getPointerAddressSpace();
+    auto NamesArrTy = llvm::ArrayType::get(
+        llvm::PointerType::get(Ctx, AddrSpace), FunctionNames.size());
     auto NamesArrVal = llvm::ConstantArray::get(NamesArrTy, FunctionNames);
     // This variable will *NOT* be emitted to the object file. It is used
     // to pass the list of names referenced to codegen.
diff --git a/external/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp b/external/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp
index a2984941947c..f4a99467010a 100644
--- a/external/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -693,6 +693,17 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
   llvm::Constant *CheckTypeDesc;
   bool ShouldEmitCFICheck = CGF.SanOpts.has(SanitizerKind::CFIMFCall) &&
                             CGM.HasHiddenLTOVisibility(RD);
+
+  if (ShouldEmitCFICheck) {
+    if (const auto *BinOp = dyn_cast<BinaryOperator>(E)) {
+      if (BinOp->isPtrMemOp() &&
+          BinOp->getRHS()
+              ->getType()
+              ->hasPointeeToToCFIUncheckedCalleeFunctionType())
+        ShouldEmitCFICheck = false;
+    }
+  }
+
   bool ShouldEmitVFEInfo = CGM.getCodeGenOpts().VirtualFunctionElimination &&
                            CGM.HasHiddenLTOVisibility(RD);
   bool ShouldEmitWPDInfo =
@@ -702,9 +713,9 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
   llvm::Value *VirtualFn = nullptr;
 
   {
-    CodeGenFunction::SanitizerScope SanScope(&CGF);
-    ApplyDebugLocation ApplyTrapDI(
-        CGF, CGF.SanitizerAnnotateDebugInfo(SanitizerKind::SO_CFIMFCall));
+    auto CheckOrdinal = SanitizerKind::SO_CFIMFCall;
+    auto CheckHandler = SanitizerHandler::CFICheckFail;
+    SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
 
     llvm::Value *TypeId = nullptr;
     llvm::Value *CheckResult = nullptr;
@@ -774,16 +785,15 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
       };
 
       if (CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIMFCall)) {
-        CGF.EmitTrapCheck(CheckResult, SanitizerHandler::CFICheckFail);
+        CGF.EmitTrapCheck(CheckResult, CheckHandler);
       } else {
         llvm::Value *AllVtables = llvm::MetadataAsValue::get(
             CGM.getLLVMContext(),
             llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
         llvm::Value *ValidVtable = Builder.CreateCall(
             CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
-        CGF.EmitCheck(std::make_pair(CheckResult, SanitizerKind::SO_CFIMFCall),
-                      SanitizerHandler::CFICheckFail, StaticData,
-                      {VTable, ValidVtable});
+        CGF.EmitCheck(std::make_pair(CheckResult, CheckOrdinal), CheckHandler,
+                      StaticData, {VTable, ValidVtable});
       }
 
       FnVirtual = Builder.GetInsertBlock();
@@ -802,9 +812,9 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
   if (ShouldEmitCFICheck) {
     CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
     if (RD->hasDefinition()) {
-      CodeGenFunction::SanitizerScope SanScope(&CGF);
-      ApplyDebugLocation ApplyTrapDI(
-          CGF, CGF.SanitizerAnnotateDebugInfo(SanitizerKind::SO_CFIMFCall));
+      auto CheckOrdinal = SanitizerKind::SO_CFIMFCall;
+      auto CheckHandler = SanitizerHandler::CFICheckFail;
+      SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
 
       llvm::Constant *StaticData[] = {
           llvm::ConstantInt::get(CGF.Int8Ty, CodeGenFunction::CFITCK_NVMFCall),
@@ -827,8 +837,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
         Bit = Builder.CreateOr(Bit, TypeTest);
       }
 
-      CGF.EmitCheck(std::make_pair(Bit, SanitizerKind::SO_CFIMFCall),
-                    SanitizerHandler::CFICheckFail, StaticData,
+      CGF.EmitCheck(std::make_pair(Bit, CheckOrdinal), CheckHandler, StaticData,
                     {NonVirtualFn, llvm::UndefValue::get(CGF.IntPtrTy)});
 
       FnNonVirtual = Builder.GetInsertBlock();
diff --git a/external/llvm-project/clang/lib/CodeGen/SanitizerHandler.h b/external/llvm-project/clang/lib/CodeGen/SanitizerHandler.h
new file mode 100644
index 000000000000..bb42e3947cf1
--- /dev/null
+++ b/external/llvm-project/clang/lib/CodeGen/SanitizerHandler.h
@@ -0,0 +1,50 @@
+//===-- SanitizerHandler.h - Definition of sanitizer handlers ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the internal per-function state used for llvm translation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CODEGEN_SANITIZER_HANDLER_H
+#define LLVM_CLANG_LIB_CODEGEN_SANITIZER_HANDLER_H
+
+#define LIST_SANITIZER_CHECKS                                                  \
+  SANITIZER_CHECK(AddOverflow, add_overflow, 0)                                \
+  SANITIZER_CHECK(BuiltinUnreachable, builtin_unreachable, 0)                  \
+  SANITIZER_CHECK(CFICheckFail, cfi_check_fail, 0)                             \
+  SANITIZER_CHECK(DivremOverflow, divrem_overflow, 0)                          \
+  SANITIZER_CHECK(DynamicTypeCacheMiss, dynamic_type_cache_miss, 0)            \
+  SANITIZER_CHECK(FloatCastOverflow, float_cast_overflow, 0)                   \
+  SANITIZER_CHECK(FunctionTypeMismatch, function_type_mismatch, 0)             \
+  SANITIZER_CHECK(ImplicitConversion, implicit_conversion, 0)                  \
+  SANITIZER_CHECK(InvalidBuiltin, invalid_builtin, 0)                          \
+  SANITIZER_CHECK(InvalidObjCCast, invalid_objc_cast, 0)                       \
+  SANITIZER_CHECK(LoadInvalidValue, load_invalid_value, 0)                     \
+  SANITIZER_CHECK(MissingReturn, missing_return, 0)                            \
+  SANITIZER_CHECK(MulOverflow, mul_overflow, 0)                                \
+  SANITIZER_CHECK(NegateOverflow, negate_overflow, 0)                          \
+  SANITIZER_CHECK(NullabilityArg, nullability_arg, 0)                          \
+  SANITIZER_CHECK(NullabilityReturn, nullability_return, 1)                    \
+  SANITIZER_CHECK(NonnullArg, nonnull_arg, 0)                                  \
+  SANITIZER_CHECK(NonnullReturn, nonnull_return, 1)                            \
+  SANITIZER_CHECK(OutOfBounds, out_of_bounds, 0)                               \
+  SANITIZER_CHECK(PointerOverflow, pointer_overflow, 0)                        \
+  SANITIZER_CHECK(ShiftOutOfBounds, shift_out_of_bounds, 0)                    \
+  SANITIZER_CHECK(SubOverflow, sub_overflow, 0)                                \
+  SANITIZER_CHECK(TypeMismatch, type_mismatch, 1)                              \
+  SANITIZER_CHECK(AlignmentAssumption, alignment_assumption, 0)                \
+  SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0)              \
+  SANITIZER_CHECK(BoundsSafety, bounds_safety, 0)
+
+enum SanitizerHandler {
+#define SANITIZER_CHECK(Enum, Name, Version) Enum,
+  LIST_SANITIZER_CHECKS
+#undef SANITIZER_CHECK
+};
+
+#endif // LLVM_CLANG_LIB_CODEGEN_SANITIZER_HANDLER_H
diff --git a/external/llvm-project/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/external/llvm-project/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 1cf8f6819b75..dab311903f6d 100644
--- a/external/llvm-project/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -4792,12 +4792,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   case SVE::BI__builtin_sve_svlen_u32:
   case SVE::BI__builtin_sve_svlen_u64: {
     SVETypeFlags TF(Builtin->TypeModifier);
-    auto VTy = cast<llvm::VectorType>(getSVEType(TF));
-    auto *NumEls =
-        llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
-
-    Function *F = CGM.getIntrinsic(Intrinsic::vscale, Ty);
-    return Builder.CreateMul(NumEls, Builder.CreateCall(F));
+    return Builder.CreateElementCount(Ty, getSVEType(TF)->getElementCount());
   }
 
   case SVE::BI__builtin_sve_svtbl2_u8:
@@ -4813,8 +4808,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   case SVE::BI__builtin_sve_svtbl2_f32:
   case SVE::BI__builtin_sve_svtbl2_f64: {
     SVETypeFlags TF(Builtin->TypeModifier);
-    auto VTy = cast<llvm::ScalableVectorType>(getSVEType(TF));
-    Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, VTy);
+    Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, getSVEType(TF));
     return Builder.CreateCall(F, Ops);
   }
 
diff --git a/external/llvm-project/clang/lib/CodeGen/TargetInfo.cpp b/external/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
index 7d176e421ac4..f3df92c44bb6 100644
--- a/external/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
@@ -191,7 +191,7 @@ llvm::Value *TargetCodeGenInfo::createEnqueuedBlockKernel(
   auto *F = llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage, Name,
                                    &CGF.CGM.getModule());
   llvm::CallingConv::ID KernelCC =
-      CGF.getTypes().ClangCallConvToLLVMCallConv(CallingConv::CC_OpenCLKernel);
+      CGF.getTypes().ClangCallConvToLLVMCallConv(CallingConv::CC_DeviceKernel);
   F->setCallingConv(KernelCC);
 
   llvm::AttrBuilder KernelAttrs(C);
diff --git a/external/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp b/external/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp
index a318b619d76a..b82c46966cf0 100644
--- a/external/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -487,9 +487,38 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
     }
     Size = llvm::alignTo(Size, Alignment);
 
+    // If the Aggregate is made up of pointers, use an array of pointers for the
+    // coerced type. This prevents having to convert ptr2int->int2ptr through
+    // the call, allowing alias analysis to produce better code.
+    auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {
+      if (isEmptyRecord(getContext(), Ty, true))
+        return false;
+      const RecordType *RT = Ty->getAs<RecordType>();
+      if (!RT)
+        return false;
+      const RecordDecl *RD = RT->getDecl();
+      if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+        for (const auto &I : CXXRD->bases())
+          if (!Self(Self, I.getType()))
+            return false;
+      }
+      return all_of(RD->fields(), [&](FieldDecl *FD) {
+        QualType FDTy = FD->getType();
+        if (FDTy->isArrayType())
+          FDTy = getContext().getBaseElementType(FDTy);
+        return (FDTy->isPointerOrReferenceType() &&
+                getContext().getTypeSize(FDTy) == 64 &&
+                !FDTy->getPointeeType().hasAddressSpace()) ||
+               Self(Self, FDTy);
+      });
+    };
+
     // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
     // For aggregates with 16-byte alignment, we use i128.
     llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment);
+    if ((Size == 64 || Size == 128) && Alignment == 64 &&
+        ContainsOnlyPointers(ContainsOnlyPointers, Ty))
+      BaseTy = llvm::PointerType::getUnqual(getVMContext());
     return ABIArgInfo::getDirect(
         Size == Alignment ? BaseTy
                           : llvm::ArrayType::get(BaseTy, Size / Alignment));
diff --git a/external/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp b/external/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp
index 02314d27abe6..8660373c3927 100644
--- a/external/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp
@@ -8,6 +8,7 @@
 
 #include "ABIInfoImpl.h"
 #include "TargetInfo.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 
 using namespace clang;
@@ -336,7 +337,7 @@ static bool requiresAMDGPUProtectedVisibility(const Decl *D,
     return false;
 
   return !D->hasAttr<OMPDeclareTargetDeclAttr>() &&
-         (D->hasAttr<OpenCLKernelAttr>() ||
+         (D->hasAttr<DeviceKernelAttr>() ||
           (isa<FunctionDecl>(D) && D->hasAttr<CUDAGlobalAttr>()) ||
           (isa<VarDecl>(D) &&
            (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
@@ -349,7 +350,7 @@ void AMDGPUTargetCodeGenInfo::setFunctionDeclAttributes(
   const auto *ReqdWGS =
       M.getLangOpts().OpenCL ? FD->getAttr<ReqdWorkGroupSizeAttr>() : nullptr;
   const bool IsOpenCLKernel =
-      M.getLangOpts().OpenCL && FD->hasAttr<OpenCLKernelAttr>();
+      M.getLangOpts().OpenCL && FD->hasAttr<DeviceKernelAttr>();
   const bool IsHIPKernel = M.getLangOpts().HIP && FD->hasAttr<CUDAGlobalAttr>();
 
   const auto *FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>();
@@ -571,7 +572,7 @@ bool AMDGPUTargetCodeGenInfo::shouldEmitDWARFBitFieldSeparators() const {
 void AMDGPUTargetCodeGenInfo::setCUDAKernelCallingConvention(
     const FunctionType *&FT) const {
   FT = getABIInfo().getContext().adjustFunctionType(
-      FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
+      FT, FT->getExtInfo().withCallingConv(CC_DeviceKernel));
 }
 
 /// Return IR struct type for rtinfo struct in rocm-device-libs used for device
diff --git a/external/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp b/external/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp
index 0f689371a60d..7640f3779816 100644
--- a/external/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/Targets/LoongArch.cpp
@@ -110,10 +110,9 @@ bool LoongArchABIInfo::detectFARsEligibleStructHelper(
     uint64_t Size = getContext().getTypeSize(Ty);
     if (IsInt && Size > GRLen)
       return false;
-    // Can't be eligible if larger than the FP registers. Half precision isn't
-    // currently supported on LoongArch and the ABI hasn't been confirmed, so
-    // default to the integer ABI in that case.
-    if (IsFloat && (Size > FRLen || Size < 32))
+    // Can't be eligible if larger than the FP registers. Handling of half
+    // precision values has been specified in the ABI, so don't block those.
+    if (IsFloat && Size > FRLen)
       return false;
     // Can't be eligible if an integer type was already found (int+int pairs
     // are not eligible).
diff --git a/external/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp b/external/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp
index 8ce70e2111cc..523165c11750 100644
--- a/external/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/Targets/MSP430.cpp
@@ -8,6 +8,7 @@
 
 #include "ABIInfoImpl.h"
 #include "TargetInfo.h"
+#include "llvm/ADT/StringExtras.h"
 
 using namespace clang;
 using namespace clang::CodeGen;
diff --git a/external/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp b/external/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp
index 25ab28c54b65..ad802c9131de 100644
--- a/external/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/Targets/NVPTX.cpp
@@ -9,6 +9,7 @@
 #include "ABIInfoImpl.h"
 #include "TargetInfo.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
 
@@ -259,40 +260,31 @@ void NVPTXTargetCodeGenInfo::setTargetAttributes(
 
   llvm::Function *F = cast<llvm::Function>(GV);
 
-  // Perform special handling in OpenCL mode
-  if (M.getLangOpts().OpenCL) {
-    // Use OpenCL function attributes to check for kernel functions
+  // Perform special handling in OpenCL/CUDA mode
+  if (M.getLangOpts().OpenCL || M.getLangOpts().CUDA) {
+    // Use function attributes to check for kernel functions
     // By default, all functions are device functions
-    if (FD->hasAttr<OpenCLKernelAttr>()) {
-      // OpenCL __kernel functions get kernel metadata
+    if (FD->hasAttr<DeviceKernelAttr>() || FD->hasAttr<CUDAGlobalAttr>()) {
+      // OpenCL/CUDA kernel functions get kernel metadata
       // Create !{<func-ref>, metadata !"kernel", i32 1} node
-      F->setCallingConv(llvm::CallingConv::PTX_Kernel);
       // And kernel functions are not subject to inlining
       F->addFnAttr(llvm::Attribute::NoInline);
+      if (FD->hasAttr<CUDAGlobalAttr>()) {
+        SmallVector<int, 10> GCI;
+        for (auto IV : llvm::enumerate(FD->parameters()))
+          if (IV.value()->hasAttr<CUDAGridConstantAttr>())
+            // For some reason arg indices are 1-based in NVVM
+            GCI.push_back(IV.index() + 1);
+        // Create !{<func-ref>, metadata !"kernel", i32 1} node
+        F->setCallingConv(llvm::CallingConv::PTX_Kernel);
+        addGridConstantNVVMMetadata(F, GCI);
+      }
+      if (CUDALaunchBoundsAttr *Attr = FD->getAttr<CUDALaunchBoundsAttr>())
+        M.handleCUDALaunchBoundsAttr(F, Attr);
     }
   }
-
-  // Perform special handling in CUDA mode.
-  if (M.getLangOpts().CUDA) {
-    // CUDA __global__ functions get a kernel metadata entry.  Since
-    // __global__ functions cannot be called from the device, we do not
-    // need to set the noinline attribute.
-    if (FD->hasAttr<CUDAGlobalAttr>()) {
-      SmallVector<int, 10> GCI;
-      for (auto IV : llvm::enumerate(FD->parameters()))
-        if (IV.value()->hasAttr<CUDAGridConstantAttr>())
-          // For some reason arg indices are 1-based in NVVM
-          GCI.push_back(IV.index() + 1);
-      // Create !{<func-ref>, metadata !"kernel", i32 1} node
-      F->setCallingConv(llvm::CallingConv::PTX_Kernel);
-      addGridConstantNVVMMetadata(F, GCI);
-    }
-    if (CUDALaunchBoundsAttr *Attr = FD->getAttr<CUDALaunchBoundsAttr>())
-      M.handleCUDALaunchBoundsAttr(F, Attr);
-  }
-
   // Attach kernel metadata directly if compiling for NVPTX.
-  if (FD->hasAttr<NVPTXKernelAttr>()) {
+  if (FD->hasAttr<DeviceKernelAttr>()) {
     F->setCallingConv(llvm::CallingConv::PTX_Kernel);
   }
 }
diff --git a/external/llvm-project/clang/lib/CodeGen/Targets/SPIR.cpp b/external/llvm-project/clang/lib/CodeGen/Targets/SPIR.cpp
index 92ae46234e6b..2f1e43cdc8cc 100644
--- a/external/llvm-project/clang/lib/CodeGen/Targets/SPIR.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/Targets/SPIR.cpp
@@ -228,7 +228,7 @@ void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention(
   // Convert HIP kernels to SPIR-V kernels.
   if (getABIInfo().getContext().getLangOpts().HIP) {
     FT = getABIInfo().getContext().adjustFunctionType(
-        FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
+        FT, FT->getExtInfo().withCallingConv(CC_DeviceKernel));
     return;
   }
 }
diff --git a/external/llvm-project/clang/lib/CodeGen/Targets/TCE.cpp b/external/llvm-project/clang/lib/CodeGen/Targets/TCE.cpp
index f3685ccd9825..df49aea49a1e 100644
--- a/external/llvm-project/clang/lib/CodeGen/Targets/TCE.cpp
+++ b/external/llvm-project/clang/lib/CodeGen/Targets/TCE.cpp
@@ -39,7 +39,7 @@ void TCETargetCodeGenInfo::setTargetAttributes(
   llvm::Function *F = cast<llvm::Function>(GV);
 
   if (M.getLangOpts().OpenCL) {
-    if (FD->hasAttr<OpenCLKernelAttr>()) {
+    if (FD->hasAttr<DeviceKernelAttr>()) {
       // OpenCL C Kernel functions are not subject to inlining
       F->addFnAttr(llvm::Attribute::NoInline);
       const ReqdWorkGroupSizeAttr *Attr = FD->getAttr<ReqdWorkGroupSizeAttr>();
diff --git a/external/llvm-project/clang/lib/Driver/Driver.cpp b/external/llvm-project/clang/lib/Driver/Driver.cpp
index 038afb972d2a..879eb803f3d7 100644
--- a/external/llvm-project/clang/lib/Driver/Driver.cpp
+++ b/external/llvm-project/clang/lib/Driver/Driver.cpp
@@ -149,6 +149,7 @@ getNVIDIAOffloadTargetTriple(const Driver &D, const ArgList &Args,
   D.Diag(diag::err_drv_invalid_or_unsupported_offload_target) << TT->str();
   return std::nullopt;
 }
+
 static std::optional<llvm::Triple>
 getHIPOffloadTargetTriple(const Driver &D, const ArgList &Args) {
   if (!Args.hasArg(options::OPT_offload_EQ)) {
@@ -170,6 +171,17 @@ getHIPOffloadTargetTriple(const Driver &D, const ArgList &Args) {
   return std::nullopt;
 }
 
+template <typename F> static bool usesInput(const ArgList &Args, F &&Fn) {
+  return llvm::any_of(Args, [&](Arg *A) {
+    return (A->getOption().matches(options::OPT_x) &&
+            Fn(types::lookupTypeForTypeSpecifier(A->getValue()))) ||
+           (A->getOption().getKind() == Option::InputClass &&
+            StringRef(A->getValue()).rfind('.') != StringRef::npos &&
+            Fn(types::lookupTypeForExtension(
+                &A->getValue()[StringRef(A->getValue()).rfind('.') + 1])));
+  });
+}
+
 // static
 std::string Driver::GetResourcesPath(StringRef BinaryPath) {
   // Since the resource directory is embedded in the module hash, it's important
@@ -1586,28 +1598,27 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
       A->claim();
 
       if (Args.hasArg(options::OPT_spirv)) {
+        const llvm::StringMap<llvm::Triple::SubArchType> ValidTargets = {
+            {"vulkan1.2", llvm::Triple::SPIRVSubArch_v15},
+            {"vulkan1.3", llvm::Triple::SPIRVSubArch_v16}};
         llvm::Triple T(TargetTriple);
-        T.setArch(llvm::Triple::spirv);
-        T.setOS(llvm::Triple::Vulkan);
 
-        // Set specific Vulkan version if applicable.
+        // Set specific Vulkan version. Default to vulkan1.3.
+        auto TargetInfo = ValidTargets.find("vulkan1.3");
+        assert(TargetInfo != ValidTargets.end());
         if (const Arg *A = Args.getLastArg(options::OPT_fspv_target_env_EQ)) {
-          const llvm::StringMap<llvm::Triple::SubArchType> ValidTargets = {
-              {"vulkan1.2", llvm::Triple::SPIRVSubArch_v15},
-              {"vulkan1.3", llvm::Triple::SPIRVSubArch_v16}};
-
-          auto TargetInfo = ValidTargets.find(A->getValue());
-          if (TargetInfo != ValidTargets.end()) {
-            T.setOSName(TargetInfo->getKey());
-            T.setArch(llvm::Triple::spirv, TargetInfo->getValue());
-          } else {
+          TargetInfo = ValidTargets.find(A->getValue());
+          if (TargetInfo == ValidTargets.end()) {
             Diag(diag::err_drv_invalid_value)
                 << A->getAsString(Args) << A->getValue();
           }
           A->claim();
         }
-
-        TargetTriple = T.str();
+        if (TargetInfo != ValidTargets.end()) {
+          T.setOSName(TargetInfo->getKey());
+          T.setArch(llvm::Triple::spirv, TargetInfo->getValue());
+          TargetTriple = T.str();
+        }
       }
     } else {
       Diag(diag::err_drv_dxc_missing_target_profile);
@@ -4236,6 +4247,39 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
                              const InputList &Inputs,
                              ActionList &Actions) const {
 
+  // Diagnose misuse of /Fo.
+  if (Arg *A = Args.getLastArg(options::OPT__SLASH_Fo)) {
+    StringRef V = A->getValue();
+    if (Inputs.size() > 1 && !V.empty() &&
+        !llvm::sys::path::is_separator(V.back())) {
+      // Check whether /Fo tries to name an output file for multiple inputs.
+      Diag(clang::diag::err_drv_out_file_argument_with_multiple_sources)
+          << A->getSpelling() << V;
+      Args.eraseArg(options::OPT__SLASH_Fo);
+    }
+  }
+
+  // Diagnose misuse of /Fa.
+  if (Arg *A = Args.getLastArg(options::OPT__SLASH_Fa)) {
+    StringRef V = A->getValue();
+    if (Inputs.size() > 1 && !V.empty() &&
+        !llvm::sys::path::is_separator(V.back())) {
+      // Check whether /Fa tries to name an asm file for multiple inputs.
+      Diag(clang::diag::err_drv_out_file_argument_with_multiple_sources)
+          << A->getSpelling() << V;
+      Args.eraseArg(options::OPT__SLASH_Fa);
+    }
+  }
+
+  // Diagnose misuse of /o.
+  if (Arg *A = Args.getLastArg(options::OPT__SLASH_o)) {
+    if (A->getValue()[0] == '\0') {
+      // It has to have a value.
+      Diag(clang::diag::err_drv_missing_argument) << A->getSpelling() << 1;
+      Args.eraseArg(options::OPT__SLASH_o);
+    }
+  }
+
   // Ignore /Yc/Yu if both /Yc and /Yu passed but with different filenames.
   Arg *YcArg = Args.getLastArg(options::OPT__SLASH_Yc);
   Arg *YuArg = Args.getLastArg(options::OPT__SLASH_Yu);
@@ -4381,39 +4425,6 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
   Arg *FinalPhaseArg;
   phases::ID FinalPhase = getFinalPhase(Args, &FinalPhaseArg);
 
-  // Diagnose misuse of /Fo.
-  if (Arg *A = Args.getLastArg(options::OPT__SLASH_Fo)) {
-    StringRef V = A->getValue();
-    if (Inputs.size() > 1 && !V.empty() &&
-        !llvm::sys::path::is_separator(V.back())) {
-      // Check whether /Fo tries to name an output file for multiple inputs.
-      Diag(clang::diag::err_drv_out_file_argument_with_multiple_sources)
-          << A->getSpelling() << V;
-      Args.eraseArg(options::OPT__SLASH_Fo);
-    }
-  }
-
-  // Diagnose misuse of /Fa.
-  if (Arg *A = Args.getLastArg(options::OPT__SLASH_Fa)) {
-    StringRef V = A->getValue();
-    if (Inputs.size() > 1 && !V.empty() &&
-        !llvm::sys::path::is_separator(V.back())) {
-      // Check whether /Fa tries to name an asm file for multiple inputs.
-      Diag(clang::diag::err_drv_out_file_argument_with_multiple_sources)
-          << A->getSpelling() << V;
-      Args.eraseArg(options::OPT__SLASH_Fa);
-    }
-  }
-
-  // Diagnose misuse of /o.
-  if (Arg *A = Args.getLastArg(options::OPT__SLASH_o)) {
-    if (A->getValue()[0] == '\0') {
-      // It has to have a value.
-      Diag(clang::diag::err_drv_missing_argument) << A->getSpelling() << 1;
-      Args.eraseArg(options::OPT__SLASH_o);
-    }
-  }
-
   handleArguments(C, Args, Inputs, Actions);
 
   bool UseNewOffloadingDriver =
@@ -4903,7 +4914,10 @@ Action *Driver::BuildOffloadingActions(Compilation &C,
   // For HIP non-rdc non-device-only compilation, create a linker wrapper
   // action for each host object to link, bundle and wrap device files in
   // it.
-  if (isa<AssembleJobAction>(HostAction) && HIPNoRDC && !offloadDeviceOnly()) {
+  if ((isa<AssembleJobAction>(HostAction) ||
+       (isa<BackendJobAction>(HostAction) &&
+        HostAction->getType() == types::TY_LTO_BC)) &&
+      HIPNoRDC && !offloadDeviceOnly()) {
     ActionList AL{HostAction};
     HostAction = C.MakeAction<LinkerWrapperJobAction>(AL, types::TY_Object);
     HostAction->propagateHostOffloadInfo(C.getActiveOffloadKinds(),
@@ -6895,9 +6909,14 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
     case llvm::Triple::CUDA:
       TC = std::make_unique<toolchains::NVPTXToolChain>(*this, Target, Args);
       break;
-    case llvm::Triple::AMDHSA:
-      TC = std::make_unique<toolchains::ROCMToolChain>(*this, Target, Args);
+    case llvm::Triple::AMDHSA: {
+      bool DL =
+          usesInput(Args, types::isOpenCL) || usesInput(Args, types::isLLVMIR);
+      TC = DL ? std::make_unique<toolchains::ROCMToolChain>(*this, Target, Args)
+              : std::make_unique<toolchains::AMDGPUToolChain>(*this, Target,
+                                                              Args);
       break;
+    }
     case llvm::Triple::AMDPAL:
     case llvm::Triple::Mesa3D:
       TC = std::make_unique<toolchains::AMDGPUToolChain>(*this, Target, Args);
@@ -7265,9 +7284,10 @@ static const char *GetStableCStr(llvm::StringSet<> &SavedStrings, StringRef S) {
 ///
 ///  '#': Silence information about the changes to the command line arguments.
 ///
-///  '^': Add FOO as a new argument at the beginning of the command line.
+///  '^FOO': Add FOO as a new argument at the beginning of the command line
+///  right after the name of the compiler executable.
 ///
-///  '+': Add FOO as a new argument at the end of the command line.
+///  '+FOO': Add FOO as a new argument at the end of the command line.
 ///
 ///  's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
 ///  line.
@@ -7355,7 +7375,7 @@ static void applyOneOverrideOption(raw_ostream &OS,
 void driver::applyOverrideOptions(SmallVectorImpl<const char *> &Args,
                                   const char *OverrideStr,
                                   llvm::StringSet<> &SavedStrings,
-                                  raw_ostream *OS) {
+                                  StringRef EnvVar, raw_ostream *OS) {
   if (!OS)
     OS = &llvm::nulls();
 
@@ -7364,7 +7384,7 @@ void driver::applyOverrideOptions(SmallVectorImpl<const char *> &Args,
     OS = &llvm::nulls();
   }
 
-  *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
+  *OS << "### " << EnvVar << ": " << OverrideStr << "\n";
 
   // This does not need to be efficient.
 
diff --git a/external/llvm-project/clang/lib/Driver/MultilibBuilder.cpp b/external/llvm-project/clang/lib/Driver/MultilibBuilder.cpp
index 8440a82f78be..9d44d8681cf8 100644
--- a/external/llvm-project/clang/lib/Driver/MultilibBuilder.cpp
+++ b/external/llvm-project/clang/lib/Driver/MultilibBuilder.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/MultilibBuilder.h"
-#include "ToolChains/CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChain.cpp b/external/llvm-project/clang/lib/Driver/ToolChain.cpp
index 5847b80d13a4..5352c4a90c5c 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChain.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChain.cpp
@@ -11,13 +11,13 @@
 #include "ToolChains/Arch/ARM.h"
 #include "ToolChains/Arch/RISCV.h"
 #include "ToolChains/Clang.h"
-#include "ToolChains/CommonArgs.h"
 #include "ToolChains/Flang.h"
 #include "ToolChains/InterfaceStubs.h"
 #include "clang/Basic/ObjCRuntime.h"
 #include "clang/Basic/Sanitizers.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/Action.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Job.h"
@@ -932,11 +932,16 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   if (auto Path = getPathForTriple(T))
     return *Path;
 
-  if (T.isOSAIX() && T.getEnvironment() == Triple::UnknownEnvironment) {
-    // Strip unknown environment from the triple.
-    const llvm::Triple AIXTriple(
-        llvm::Triple(T.getArchName(), T.getVendorName(),
-                     llvm::Triple::getOSTypeName(T.getOS())));
+  if (T.isOSAIX()) {
+    llvm::Triple AIXTriple;
+    if (T.getEnvironment() == Triple::UnknownEnvironment) {
+      // Strip unknown environment and the OS version from the triple.
+      AIXTriple = llvm::Triple(T.getArchName(), T.getVendorName(),
+                               llvm::Triple::getOSTypeName(T.getOS()));
+    } else {
+      // Strip the OS version from the triple.
+      AIXTriple = getTripleWithoutOSVersion();
+    }
     if (auto Path = getPathForTriple(AIXTriple))
       return *Path;
   }
@@ -986,14 +991,6 @@ std::optional<std::string> ToolChain::getRuntimePath() const {
   if (Triple.isOSDarwin())
     return {};
 
-  // For AIX, get the triple without the OS version.
-  if (Triple.isOSAIX()) {
-    const llvm::Triple &TripleWithoutVersion = getTripleWithoutOSVersion();
-    llvm::sys::path::append(P, TripleWithoutVersion.str());
-    if (getVFS().exists(P))
-      return std::string(P);
-    return {};
-  }
   llvm::sys::path::append(P, Triple.str());
   return std::string(P);
 }
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp
index 4885b885ccd6..ffd7b6920544 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/AIX.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AIX.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/SanitizerArgs.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp
index bda4f310f03f..b53f75fc3ff5 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
-#include "CommonArgs.h"
 #include "clang/Basic/TargetID.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
index b248a45efed7..9c9942955358 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
@@ -8,7 +8,7 @@
 
 #include "AMDGPUOpenMP.h"
 #include "AMDGPU.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/AVR.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/AVR.cpp
index f1529f857eef..731076d9754a 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/AVR.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/AVR.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AVR.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
@@ -326,8 +326,78 @@ constexpr struct {
     {"attiny1624", "avrxmega3", "avrxmega3", 0x803800},
     {"attiny1626", "avrxmega3", "avrxmega3", 0x803800},
     {"attiny1627", "avrxmega3", "avrxmega3", 0x803800},
+    {"attiny3224", "avrxmega3", "avrxmega3", 0x803400},
+    {"attiny3226", "avrxmega3", "avrxmega3", 0x803400},
+    {"attiny3227", "avrxmega3", "avrxmega3", 0x803400},
     {"attiny3216", "avrxmega3", "avrxmega3", 0x803800},
     {"attiny3217", "avrxmega3", "avrxmega3", 0x803800},
+
+    // gcc 14 additions:
+
+    {"avr64da28", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64da32", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64da48", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64da64", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64db28", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64db32", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64db48", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64db64", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64dd14", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64dd20", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64dd28", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64dd32", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64du28", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64du32", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64ea28", "avrxmega2", "avrxmega2", 0x806800},
+    {"avr64ea32", "avrxmega2", "avrxmega2", 0x806800},
+    {"avr64ea48", "avrxmega2", "avrxmega2", 0x806800},
+    {"avr64sd28", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64sd32", "avrxmega2", "avrxmega2", 0x806000},
+    {"avr64sd48", "avrxmega2", "avrxmega2", 0x806000},
+
+    {"avr16dd20", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16dd28", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16dd32", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16du14", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16du20", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16du28", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16du32", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr32da28", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32da32", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32da48", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32db28", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32db32", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32db48", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32dd14", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32dd20", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32dd28", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32dd32", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32du14", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32du20", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32du28", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32du32", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr16eb14", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16eb20", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16eb28", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16eb32", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16ea28", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16ea32", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr16ea48", "avrxmega3", "avrxmega3", 0x807800},
+    {"avr32ea28", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32ea32", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32ea48", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32sd20", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32sd28", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr32sd32", "avrxmega3", "avrxmega3", 0x807000},
+    {"avr128da28", "avrxmega4", "avrxmega4", 0x804000},
+    {"avr128da32", "avrxmega4", "avrxmega4", 0x804000},
+    {"avr128da48", "avrxmega4", "avrxmega4", 0x804000},
+    {"avr128da64", "avrxmega4", "avrxmega4", 0x804000},
+    {"avr128db28", "avrxmega4", "avrxmega4", 0x804000},
+    {"avr128db32", "avrxmega4", "avrxmega4", 0x804000},
+    {"avr128db48", "avrxmega4", "avrxmega4", 0x804000},
+    {"avr128db64", "avrxmega4", "avrxmega4", 0x804000},
+
 };
 
 std::string GetMCUSubPath(StringRef MCUName) {
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index eaae9f876e3a..ea906d2e2acf 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64.h"
-#include "../CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Option/ArgList.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
index a35f8276737f..33a655870b01 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
@@ -8,8 +8,8 @@
 
 #include "LoongArch.h"
 #include "../Clang.h"
-#include "ToolChains/CommonArgs.h"
 #include "clang/Basic/DiagnosticDriver.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "llvm/TargetParser/Host.h"
@@ -252,6 +252,7 @@ void loongarch::getLoongArchTargetFeatures(const Driver &D,
         Features.push_back("+lsx");
     } else /*-mno-lsx*/ {
       Features.push_back("-lsx");
+      Features.push_back("-lasx");
     }
   }
 
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/Mips.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/Mips.cpp
index 0d847587e866..8787c8276721 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/Mips.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/Mips.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Mips.h"
-#include "ToolChains/CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/PPC.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/PPC.cpp
index afa86230085e..361a68a892a8 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/PPC.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/PPC.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "PPC.h"
-#include "ToolChains/CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index d0b620980bce..baa2c8c0bcfb 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -8,7 +8,7 @@
 
 #include "RISCV.h"
 #include "../Clang.h"
-#include "ToolChains/CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Option/ArgList.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/BareMetal.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/BareMetal.cpp
index 6bf2302d3bd9..d8168ed15feb 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -8,8 +8,8 @@
 
 #include "BareMetal.h"
 
-#include "CommonArgs.h"
 #include "Gnu.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/InputInfo.h"
 
 #include "Arch/ARM.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/CSKYToolChain.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
index e944cba0eb23..e4db3307ee3a 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYToolChain.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp
index 5694c7fea142..92acddf6e8e2 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Clang.cpp
@@ -14,7 +14,6 @@
 #include "Arch/RISCV.h"
 #include "Arch/Sparc.h"
 #include "Arch/SystemZ.h"
-#include "CommonArgs.h"
 #include "Hexagon.h"
 #include "PS4CPU.h"
 #include "clang/Basic/CLWarnings.h"
@@ -27,6 +26,7 @@
 #include "clang/Basic/Version.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/Action.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Distro.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
@@ -126,166 +126,6 @@ forAllAssociatedToolChains(Compilation &C, const JobAction &JA,
   //
 }
 
-/// This is a helper function for validating the optional refinement step
-/// parameter in reciprocal argument strings. Return false if there is an error
-/// parsing the refinement step. Otherwise, return true and set the Position
-/// of the refinement step in the input string.
-static bool getRefinementStep(StringRef In, const Driver &D,
-                              const Arg &A, size_t &Position) {
-  const char RefinementStepToken = ':';
-  Position = In.find(RefinementStepToken);
-  if (Position != StringRef::npos) {
-    StringRef Option = A.getOption().getName();
-    StringRef RefStep = In.substr(Position + 1);
-    // Allow exactly one numeric character for the additional refinement
-    // step parameter. This is reasonable for all currently-supported
-    // operations and architectures because we would expect that a larger value
-    // of refinement steps would cause the estimate "optimization" to
-    // under-perform the native operation. Also, if the estimate does not
-    // converge quickly, it probably will not ever converge, so further
-    // refinement steps will not produce a better answer.
-    if (RefStep.size() != 1) {
-      D.Diag(diag::err_drv_invalid_value) << Option << RefStep;
-      return false;
-    }
-    char RefStepChar = RefStep[0];
-    if (RefStepChar < '0' || RefStepChar > '9') {
-      D.Diag(diag::err_drv_invalid_value) << Option << RefStep;
-      return false;
-    }
-  }
-  return true;
-}
-
-/// The -mrecip flag requires processing of many optional parameters.
-static void ParseMRecip(const Driver &D, const ArgList &Args,
-                        ArgStringList &OutStrings) {
-  StringRef DisabledPrefixIn = "!";
-  StringRef DisabledPrefixOut = "!";
-  StringRef EnabledPrefixOut = "";
-  StringRef Out = "-mrecip=";
-
-  Arg *A = Args.getLastArg(options::OPT_mrecip, options::OPT_mrecip_EQ);
-  if (!A)
-    return;
-
-  unsigned NumOptions = A->getNumValues();
-  if (NumOptions == 0) {
-    // No option is the same as "all".
-    OutStrings.push_back(Args.MakeArgString(Out + "all"));
-    return;
-  }
-
-  // Pass through "all", "none", or "default" with an optional refinement step.
-  if (NumOptions == 1) {
-    StringRef Val = A->getValue(0);
-    size_t RefStepLoc;
-    if (!getRefinementStep(Val, D, *A, RefStepLoc))
-      return;
-    StringRef ValBase = Val.slice(0, RefStepLoc);
-    if (ValBase == "all" || ValBase == "none" || ValBase == "default") {
-      OutStrings.push_back(Args.MakeArgString(Out + Val));
-      return;
-    }
-  }
-
-  // Each reciprocal type may be enabled or disabled individually.
-  // Check each input value for validity, concatenate them all back together,
-  // and pass through.
-
-  llvm::StringMap<bool> OptionStrings;
-  OptionStrings.insert(std::make_pair("divd", false));
-  OptionStrings.insert(std::make_pair("divf", false));
-  OptionStrings.insert(std::make_pair("divh", false));
-  OptionStrings.insert(std::make_pair("vec-divd", false));
-  OptionStrings.insert(std::make_pair("vec-divf", false));
-  OptionStrings.insert(std::make_pair("vec-divh", false));
-  OptionStrings.insert(std::make_pair("sqrtd", false));
-  OptionStrings.insert(std::make_pair("sqrtf", false));
-  OptionStrings.insert(std::make_pair("sqrth", false));
-  OptionStrings.insert(std::make_pair("vec-sqrtd", false));
-  OptionStrings.insert(std::make_pair("vec-sqrtf", false));
-  OptionStrings.insert(std::make_pair("vec-sqrth", false));
-
-  for (unsigned i = 0; i != NumOptions; ++i) {
-    StringRef Val = A->getValue(i);
-
-    bool IsDisabled = Val.starts_with(DisabledPrefixIn);
-    // Ignore the disablement token for string matching.
-    if (IsDisabled)
-      Val = Val.substr(1);
-
-    size_t RefStep;
-    if (!getRefinementStep(Val, D, *A, RefStep))
-      return;
-
-    StringRef ValBase = Val.slice(0, RefStep);
-    llvm::StringMap<bool>::iterator OptionIter = OptionStrings.find(ValBase);
-    if (OptionIter == OptionStrings.end()) {
-      // Try again specifying float suffix.
-      OptionIter = OptionStrings.find(ValBase.str() + 'f');
-      if (OptionIter == OptionStrings.end()) {
-        // The input name did not match any known option string.
-        D.Diag(diag::err_drv_unknown_argument) << Val;
-        return;
-      }
-      // The option was specified without a half or float or double suffix.
-      // Make sure that the double or half entry was not already specified.
-      // The float entry will be checked below.
-      if (OptionStrings[ValBase.str() + 'd'] ||
-          OptionStrings[ValBase.str() + 'h']) {
-        D.Diag(diag::err_drv_invalid_value) << A->getOption().getName() << Val;
-        return;
-      }
-    }
-
-    if (OptionIter->second == true) {
-      // Duplicate option specified.
-      D.Diag(diag::err_drv_invalid_value) << A->getOption().getName() << Val;
-      return;
-    }
-
-    // Mark the matched option as found. Do not allow duplicate specifiers.
-    OptionIter->second = true;
-
-    // If the precision was not specified, also mark the double and half entry
-    // as found.
-    if (ValBase.back() != 'f' && ValBase.back() != 'd' && ValBase.back() != 'h') {
-      OptionStrings[ValBase.str() + 'd'] = true;
-      OptionStrings[ValBase.str() + 'h'] = true;
-    }
-
-    // Build the output string.
-    StringRef Prefix = IsDisabled ? DisabledPrefixOut : EnabledPrefixOut;
-    Out = Args.MakeArgString(Out + Prefix + Val);
-    if (i != NumOptions - 1)
-      Out = Args.MakeArgString(Out + ",");
-  }
-
-  OutStrings.push_back(Args.MakeArgString(Out));
-}
-
-/// The -mprefer-vector-width option accepts either a positive integer
-/// or the string "none".
-static void ParseMPreferVectorWidth(const Driver &D, const ArgList &Args,
-                                    ArgStringList &CmdArgs) {
-  Arg *A = Args.getLastArg(options::OPT_mprefer_vector_width_EQ);
-  if (!A)
-    return;
-
-  StringRef Value = A->getValue();
-  if (Value == "none") {
-    CmdArgs.push_back("-mprefer-vector-width=none");
-  } else {
-    unsigned Width;
-    if (Value.getAsInteger(10, Width)) {
-      D.Diag(diag::err_drv_invalid_value) << A->getOption().getName() << Value;
-      return;
-    }
-    CmdArgs.push_back(Args.MakeArgString("-mprefer-vector-width=" + Value));
-  }
-}
-
 static bool
 shouldUseExceptionTablesForObjCExceptions(const ObjCRuntime &runtime,
                                           const llvm::Triple &Triple) {
@@ -3030,8 +2870,9 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
   StringRef Float16ExcessPrecision = "";
   StringRef BFloat16ExcessPrecision = "";
   LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_None;
-  std::string ComplexRangeStr = "";
-  std::string GccRangeComplexOption = "";
+  std::string ComplexRangeStr;
+  std::string GccRangeComplexOption;
+  std::string LastComplexRangeOption;
 
   auto setComplexRange = [&](LangOptions::ComplexRangeKind NewRange) {
     // Warn if user expects to perform full implementation of complex
@@ -3115,6 +2956,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
           EmitComplexRangeDiag(D, GccRangeComplexOption, "-fcx-limited-range");
       }
       GccRangeComplexOption = "-fcx-limited-range";
+      LastComplexRangeOption = A->getSpelling();
       Range = LangOptions::ComplexRangeKind::CX_Basic;
       break;
     case options::OPT_fno_cx_limited_range:
@@ -3128,6 +2970,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                                "-fno-cx-limited-range");
       }
       GccRangeComplexOption = "-fno-cx-limited-range";
+      LastComplexRangeOption = A->getSpelling();
       Range = LangOptions::ComplexRangeKind::CX_Full;
       break;
     case options::OPT_fcx_fortran_rules:
@@ -3137,6 +2980,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       else
         EmitComplexRangeDiag(D, GccRangeComplexOption, "-fcx-fortran-rules");
       GccRangeComplexOption = "-fcx-fortran-rules";
+      LastComplexRangeOption = A->getSpelling();
       Range = LangOptions::ComplexRangeKind::CX_Improved;
       break;
     case options::OPT_fno_cx_fortran_rules:
@@ -3149,6 +2993,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                                "-fno-cx-fortran-rules");
       }
       GccRangeComplexOption = "-fno-cx-fortran-rules";
+      LastComplexRangeOption = A->getSpelling();
       Range = LangOptions::ComplexRangeKind::CX_Full;
       break;
     case options::OPT_fcomplex_arithmetic_EQ: {
@@ -3183,6 +3028,8 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                                  ComplexArithmeticStr(RangeVal));
         }
       }
+      LastComplexRangeOption =
+          Args.MakeArgString(A->getSpelling() + A->getValue());
       Range = RangeVal;
       break;
     }
@@ -3236,6 +3083,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       } else
         D.Diag(diag::err_drv_unsupported_option_argument)
             << A->getSpelling() << Val;
+      LastComplexRangeOption = A->getSpelling();
       break;
     }
 
@@ -3426,6 +3274,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       [[fallthrough]];
     case options::OPT_ffast_math:
       applyFastMath(true);
+      LastComplexRangeOption = A->getSpelling();
       if (A->getOption().getID() == options::OPT_Ofast)
         LastFpContractOverrideOption = "-Ofast";
       else
@@ -3443,6 +3292,15 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       ApproxFunc = false;
       SignedZeros = true;
       restoreFPContractState();
+      // If the last specified option related to complex range is not
+      // -ffast-math or -ffp-model=, emit warning.
+      if (LastComplexRangeOption != "-ffast-math" &&
+          LastComplexRangeOption != "-ffp-model=" &&
+          Range != LangOptions::ComplexRangeKind::CX_Full)
+        EmitComplexRangeDiag(D, LastComplexRangeOption, "-fno-fast-math");
+      Range = LangOptions::ComplexRangeKind::CX_None;
+      LastComplexRangeOption = "";
+      GccRangeComplexOption = "";
       LastFpContractOverrideOption = "";
       break;
     } // End switch (A->getOption().getID())
@@ -3555,7 +3413,9 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     CmdArgs.push_back(Args.MakeArgString("-fbfloat16-excess-precision=" +
                                          BFloat16ExcessPrecision));
 
-  ParseMRecip(D, Args, CmdArgs);
+  StringRef Recip = parseMRecipOption(D.getDiags(), Args);
+  if (!Recip.empty())
+    CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
 
   // -ffast-math enables the __FAST_MATH__ preprocessor macro, but check for the
   // individual features enabled by -ffast-math instead of the option itself as
@@ -7218,9 +7078,17 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.addOptInFlag(CmdArgs, options::OPT_mstackrealign,
                     options::OPT_mno_stackrealign);
 
-  if (Args.hasArg(options::OPT_mstack_alignment)) {
-    StringRef alignment = Args.getLastArgValue(options::OPT_mstack_alignment);
-    CmdArgs.push_back(Args.MakeArgString("-mstack-alignment=" + alignment));
+  if (const Arg *A = Args.getLastArg(options::OPT_mstack_alignment)) {
+    StringRef Value = A->getValue();
+    int64_t Alignment = 0;
+    if (Value.getAsInteger(10, Alignment) || Alignment < 0)
+      D.Diag(diag::err_drv_invalid_argument_to_option)
+          << Value << A->getOption().getName();
+    else if (Alignment & (Alignment - 1))
+      D.Diag(diag::err_drv_alignment_not_power_of_two)
+          << A->getAsString(Args) << Value;
+    else
+      CmdArgs.push_back(Args.MakeArgString("-mstack-alignment=" + Value));
   }
 
   if (Args.hasArg(options::OPT_mstack_probe_size)) {
@@ -7779,7 +7647,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   handleVectorizeLoopsArgs(Args, CmdArgs);
   handleVectorizeSLPArgs(Args, CmdArgs);
-  ParseMPreferVectorWidth(D, Args, CmdArgs);
+
+  StringRef VecWidth = parseMPreferVectorWidthOption(D.getDiags(), Args);
+  if (!VecWidth.empty())
+    CmdArgs.push_back(Args.MakeArgString("-mprefer-vector-width=" + VecWidth));
 
   Args.AddLastArg(CmdArgs, options::OPT_fshow_overloads_EQ);
   Args.AddLastArg(CmdArgs,
@@ -7912,8 +7783,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.addOptInFlag(CmdArgs, options::OPT_fexperimental_late_parse_attributes,
                     options::OPT_fno_experimental_late_parse_attributes);
 
-  Args.addOptInFlag(CmdArgs, options::OPT_funique_source_file_names,
-                    options::OPT_fno_unique_source_file_names);
+  if (Args.hasFlag(options::OPT_funique_source_file_names,
+                   options::OPT_fno_unique_source_file_names, false)) {
+    if (Arg *A = Args.getLastArg(options::OPT_unique_source_file_identifier_EQ))
+      A->render(Args, CmdArgs);
+    else
+      CmdArgs.push_back(Args.MakeArgString(
+          Twine("-funique-source-file-identifier=") + Input.getBaseInput()));
+  }
 
   // Setup statistics file output.
   SmallString<128> StatsFile = getStatsFileName(Args, Output, Input, D);
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp
index a479fc191987..a73392da1f77 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "Arch/AArch64.h"
 #include "Arch/ARM.h"
 #include "Arch/CSKY.h"
@@ -3369,3 +3369,166 @@ void tools::handleInterchangeLoopsArgs(const ArgList &Args,
                    options::OPT_fno_loop_interchange, EnableInterchange))
     CmdArgs.push_back("-floop-interchange");
 }
+
+// Parse -mprefer-vector-width=. Return the Value string if well-formed.
+// Otherwise, return an empty string and issue a diagnosic message if needed.
+StringRef tools::parseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
+                                               const llvm::opt::ArgList &Args) {
+  Arg *A = Args.getLastArg(clang::driver::options::OPT_mprefer_vector_width_EQ);
+  if (!A)
+    return "";
+
+  StringRef Value = A->getValue();
+  unsigned Width LLVM_ATTRIBUTE_UNINITIALIZED;
+
+  // Only "none" and Integer values are accepted by
+  // -mprefer-vector-width=<value>.
+  if (Value != "none" && Value.getAsInteger(10, Width)) {
+    Diags.Report(clang::diag::err_drv_invalid_value)
+        << A->getOption().getName() << Value;
+    return "";
+  }
+
+  return Value;
+}
+
+// This is a helper function for validating the optional refinement step
+// parameter in reciprocal argument strings. Return false if there is an error
+// parsing the refinement step. Otherwise, return true and set the Position
+// of the refinement step in the input string.
+static bool getRefinementStep(StringRef In, clang::DiagnosticsEngine &Diags,
+                              const Arg &A, size_t &Position) {
+  const char RefinementStepToken = ':';
+  Position = In.find(RefinementStepToken);
+  if (Position != StringRef::npos) {
+    StringRef Option = A.getOption().getName();
+    StringRef RefStep = In.substr(Position + 1);
+    // Allow exactly one numeric character for the additional refinement
+    // step parameter. This is reasonable for all currently-supported
+    // operations and architectures because we would expect that a larger value
+    // of refinement steps would cause the estimate "optimization" to
+    // under-perform the native operation. Also, if the estimate does not
+    // converge quickly, it probably will not ever converge, so further
+    // refinement steps will not produce a better answer.
+    if (RefStep.size() != 1) {
+      Diags.Report(diag::err_drv_invalid_value) << Option << RefStep;
+      return false;
+    }
+    char RefStepChar = RefStep[0];
+    if (RefStepChar < '0' || RefStepChar > '9') {
+      Diags.Report(diag::err_drv_invalid_value) << Option << RefStep;
+      return false;
+    }
+  }
+  return true;
+}
+
+// Parse -mrecip. Return the Value string if well-formed.
+// Otherwise, return an empty string and issue a diagnosic message if needed.
+StringRef tools::parseMRecipOption(clang::DiagnosticsEngine &Diags,
+                                   const ArgList &Args) {
+  StringRef DisabledPrefixIn = "!";
+  StringRef DisabledPrefixOut = "!";
+  StringRef EnabledPrefixOut = "";
+  StringRef Out = "";
+
+  Arg *A = Args.getLastArg(options::OPT_mrecip, options::OPT_mrecip_EQ);
+  if (!A)
+    return "";
+
+  unsigned NumOptions = A->getNumValues();
+  if (NumOptions == 0) {
+    // No option is the same as "all".
+    return "all";
+  }
+
+  // Pass through "all", "none", or "default" with an optional refinement step.
+  if (NumOptions == 1) {
+    StringRef Val = A->getValue(0);
+    size_t RefStepLoc;
+    if (!getRefinementStep(Val, Diags, *A, RefStepLoc))
+      return "";
+    StringRef ValBase = Val.slice(0, RefStepLoc);
+    if (ValBase == "all" || ValBase == "none" || ValBase == "default") {
+      return Val;
+    }
+  }
+
+  // Each reciprocal type may be enabled or disabled individually.
+  // Check each input value for validity, concatenate them all back together,
+  // and pass through.
+
+  llvm::StringMap<bool> OptionStrings;
+  OptionStrings.insert(std::make_pair("divd", false));
+  OptionStrings.insert(std::make_pair("divf", false));
+  OptionStrings.insert(std::make_pair("divh", false));
+  OptionStrings.insert(std::make_pair("vec-divd", false));
+  OptionStrings.insert(std::make_pair("vec-divf", false));
+  OptionStrings.insert(std::make_pair("vec-divh", false));
+  OptionStrings.insert(std::make_pair("sqrtd", false));
+  OptionStrings.insert(std::make_pair("sqrtf", false));
+  OptionStrings.insert(std::make_pair("sqrth", false));
+  OptionStrings.insert(std::make_pair("vec-sqrtd", false));
+  OptionStrings.insert(std::make_pair("vec-sqrtf", false));
+  OptionStrings.insert(std::make_pair("vec-sqrth", false));
+
+  for (unsigned i = 0; i != NumOptions; ++i) {
+    StringRef Val = A->getValue(i);
+
+    bool IsDisabled = Val.starts_with(DisabledPrefixIn);
+    // Ignore the disablement token for string matching.
+    if (IsDisabled)
+      Val = Val.substr(1);
+
+    size_t RefStep;
+    if (!getRefinementStep(Val, Diags, *A, RefStep))
+      return "";
+
+    StringRef ValBase = Val.slice(0, RefStep);
+    llvm::StringMap<bool>::iterator OptionIter = OptionStrings.find(ValBase);
+    if (OptionIter == OptionStrings.end()) {
+      // Try again specifying float suffix.
+      OptionIter = OptionStrings.find(ValBase.str() + 'f');
+      if (OptionIter == OptionStrings.end()) {
+        // The input name did not match any known option string.
+        Diags.Report(diag::err_drv_unknown_argument) << Val;
+        return "";
+      }
+      // The option was specified without a half or float or double suffix.
+      // Make sure that the double or half entry was not already specified.
+      // The float entry will be checked below.
+      if (OptionStrings[ValBase.str() + 'd'] ||
+          OptionStrings[ValBase.str() + 'h']) {
+        Diags.Report(diag::err_drv_invalid_value)
+            << A->getOption().getName() << Val;
+        return "";
+      }
+    }
+
+    if (OptionIter->second == true) {
+      // Duplicate option specified.
+      Diags.Report(diag::err_drv_invalid_value)
+          << A->getOption().getName() << Val;
+      return "";
+    }
+
+    // Mark the matched option as found. Do not allow duplicate specifiers.
+    OptionIter->second = true;
+
+    // If the precision was not specified, also mark the double and half entry
+    // as found.
+    if (ValBase.back() != 'f' && ValBase.back() != 'd' &&
+        ValBase.back() != 'h') {
+      OptionStrings[ValBase.str() + 'd'] = true;
+      OptionStrings[ValBase.str() + 'h'] = true;
+    }
+
+    // Build the output string.
+    StringRef Prefix = IsDisabled ? DisabledPrefixOut : EnabledPrefixOut;
+    Out = Args.MakeArgString(Out + Prefix + Val);
+    if (i != NumOptions - 1)
+      Out = Args.MakeArgString(Out + ",");
+  }
+
+  return Out;
+}
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/CrossWindows.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/CrossWindows.cpp
index 3c5dfba329cf..51c892fc9171 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/CrossWindows.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/CrossWindows.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CrossWindows.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Cuda.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Cuda.cpp
index 54c0b1484283..e2194655dfab 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "Cuda.h"
-#include "CommonArgs.h"
 #include "clang/Basic/Cuda.h"
 #include "clang/Basic/TargetID.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Distro.h"
 #include "clang/Driver/Driver.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Cygwin.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Cygwin.cpp
index f0e90deee98b..d9c16347daa3 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Cygwin.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Cygwin.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "Cygwin.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Support/Path.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Darwin.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Darwin.cpp
index 0be718009835..e5075cbcaf66 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -8,10 +8,10 @@
 
 #include "Darwin.h"
 #include "Arch/ARM.h"
-#include "CommonArgs.h"
 #include "clang/Basic/AlignedAllocation.h"
 #include "clang/Basic/ObjCRuntime.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
@@ -1642,14 +1642,16 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
     CmdArgs.push_back("-lSystem");
 
   // Select the dynamic runtime library and the target specific static library.
-  if (isTargetIOSBased()) {
-    // If we are compiling as iOS / simulator, don't attempt to link libgcc_s.1,
-    // it never went into the SDK.
-    // Linking against libgcc_s.1 isn't needed for iOS 5.0+
-    if (isIPhoneOSVersionLT(5, 0) && !isTargetIOSSimulator() &&
-        getTriple().getArch() != llvm::Triple::aarch64)
-      CmdArgs.push_back("-lgcc_s.1");
-  }
+  // Some old Darwin versions put builtins, libunwind, and some other stuff in
+  // libgcc_s.1.dylib. MacOS X 10.6 and iOS 5 moved those functions to
+  // libSystem, and made libgcc_s.1.dylib a stub. We never link libgcc_s when
+  // building for aarch64 or iOS simulator, since libgcc_s was made obsolete
+  // before either existed.
+  if (getTriple().getArch() != llvm::Triple::aarch64 &&
+      ((isTargetIOSBased() && isIPhoneOSVersionLT(5, 0) &&
+        !isTargetIOSSimulator()) ||
+       (isTargetMacOSBased() && isMacosxVersionLT(10, 6))))
+    CmdArgs.push_back("-lgcc_s.1");
   AddLinkRuntimeLib(Args, CmdArgs, "builtins");
 }
 
@@ -1678,7 +1680,8 @@ static std::string getSystemOrSDKMacOSVersion(StringRef MacOSSDKVersion) {
 
 namespace {
 
-/// The Darwin OS that was selected or inferred from arguments / environment.
+/// The Darwin OS and version that was selected or inferred from arguments or
+/// environment.
 struct DarwinPlatform {
   enum SourceKind {
     /// The OS was specified using the -target argument.
@@ -1707,23 +1710,37 @@ struct DarwinPlatform {
     InferSimulatorFromArch = false;
   }
 
-  StringRef getOSVersion() const {
-    if (Kind == OSVersionArg)
-      return Argument->getValue();
-    return OSVersion;
+  const VersionTuple getOSVersion() const {
+    return UnderlyingOSVersion.value_or(VersionTuple());
   }
 
-  void setOSVersion(StringRef S) {
-    assert(Kind == TargetArg && "Unexpected kind!");
-    OSVersion = std::string(S);
+  VersionTuple takeOSVersion() {
+    assert(UnderlyingOSVersion.has_value() &&
+           "attempting to get an unset OS version");
+    VersionTuple Result = *UnderlyingOSVersion;
+    UnderlyingOSVersion.reset();
+    return Result;
+  }
+  bool isValidOSVersion() const {
+    return llvm::Triple::isValidVersionForOS(getOSFromPlatform(Platform),
+                                             getOSVersion());
   }
 
-  bool hasOSVersion() const { return HasOSVersion; }
+  VersionTuple getCanonicalOSVersion() const {
+    return llvm::Triple::getCanonicalVersionForOS(
+        getOSFromPlatform(Platform), getOSVersion(), /*IsInValidRange=*/true);
+  }
 
-  VersionTuple getNativeTargetVersion() const {
+  void setOSVersion(const VersionTuple &Version) {
+    UnderlyingOSVersion = Version;
+  }
+
+  bool hasOSVersion() const { return UnderlyingOSVersion.has_value(); }
+
+  VersionTuple getZipperedOSVersion() const {
     assert(Environment == DarwinEnvironmentKind::MacCatalyst &&
-           "native target version is specified only for Mac Catalyst");
-    return NativeTargetVersion;
+           "zippered target version is specified only for Mac Catalyst");
+    return ZipperedOSVersion;
   }
 
   /// Returns true if the target OS was explicitly specified.
@@ -1738,7 +1755,8 @@ struct DarwinPlatform {
 
   /// Adds the -m<os>-version-min argument to the compiler invocation.
   void addOSVersionMinArgument(DerivedArgList &Args, const OptTable &Opts) {
-    if (Argument)
+    auto &[Arg, OSVersionStr] = Arguments;
+    if (Arg)
       return;
     assert(Kind != TargetArg && Kind != MTargetOSArg && Kind != OSVersionArg &&
            "Invalid kind");
@@ -1763,27 +1781,35 @@ struct DarwinPlatform {
       // DriverKit always explicitly provides a version in the triple.
       return;
     }
-    Argument = Args.MakeJoinedArg(nullptr, Opts.getOption(Opt), OSVersion);
-    Args.append(Argument);
+    Arg = Args.MakeJoinedArg(nullptr, Opts.getOption(Opt), OSVersionStr);
+    Args.append(Arg);
   }
 
   /// Returns the OS version with the argument / environment variable that
   /// specified it.
   std::string getAsString(DerivedArgList &Args, const OptTable &Opts) {
+    auto &[Arg, OSVersionStr] = Arguments;
     switch (Kind) {
     case TargetArg:
     case MTargetOSArg:
     case OSVersionArg:
+      assert(Arg && "OS version argument not yet inferred");
+      return Arg->getAsString(Args);
+    case DeploymentTargetEnv:
+      return (llvm::Twine(EnvVarName) + "=" + OSVersionStr).str();
     case InferredFromSDK:
     case InferredFromArch:
-      assert(Argument && "OS version argument not yet inferred");
-      return Argument->getAsString(Args);
-    case DeploymentTargetEnv:
-      return (llvm::Twine(EnvVarName) + "=" + OSVersion).str();
+      llvm_unreachable("Cannot print arguments for inferred OS version");
     }
     llvm_unreachable("Unsupported Darwin Source Kind");
   }
 
+  // Returns the inferred source of how the OS version was resolved.
+  std::string getInferredSource() {
+    assert(!isExplicitlySpecified() && "OS version was not inferred");
+    return InferredSource.str();
+  }
+
   void setEnvironment(llvm::Triple::EnvironmentType EnvType,
                       const VersionTuple &OSVersion,
                       const std::optional<DarwinSDKInfo> &SDKInfo) {
@@ -1794,13 +1820,13 @@ struct DarwinPlatform {
     case llvm::Triple::MacABI: {
       Environment = DarwinEnvironmentKind::MacCatalyst;
       // The minimum native macOS target for MacCatalyst is macOS 10.15.
-      NativeTargetVersion = VersionTuple(10, 15);
-      if (HasOSVersion && SDKInfo) {
+      ZipperedOSVersion = VersionTuple(10, 15);
+      if (hasOSVersion() && SDKInfo) {
         if (const auto *MacCatalystToMacOSMapping = SDKInfo->getVersionMapping(
                 DarwinSDKInfo::OSEnvPair::macCatalystToMacOSPair())) {
           if (auto MacOSVersion = MacCatalystToMacOSMapping->map(
-                  OSVersion, NativeTargetVersion, std::nullopt)) {
-            NativeTargetVersion = *MacOSVersion;
+                  OSVersion, ZipperedOSVersion, std::nullopt)) {
+            ZipperedOSVersion = *MacOSVersion;
           }
         }
       }
@@ -1810,8 +1836,8 @@ struct DarwinPlatform {
       if (TargetVariantTriple) {
         auto TargetVariantVersion = TargetVariantTriple->getOSVersion();
         if (TargetVariantVersion.getMajor()) {
-          if (TargetVariantVersion < NativeTargetVersion)
-            NativeTargetVersion = TargetVariantVersion;
+          if (TargetVariantVersion < ZipperedOSVersion)
+            ZipperedOSVersion = TargetVariantVersion;
         }
       }
       break;
@@ -1822,14 +1848,12 @@ struct DarwinPlatform {
   }
 
   static DarwinPlatform
-  createFromTarget(const llvm::Triple &TT, StringRef OSVersion, Arg *A,
+  createFromTarget(const llvm::Triple &TT, Arg *A,
                    std::optional<llvm::Triple> TargetVariantTriple,
                    const std::optional<DarwinSDKInfo> &SDKInfo) {
-    DarwinPlatform Result(TargetArg, getPlatformFromOS(TT.getOS()), OSVersion,
-                          A);
+    DarwinPlatform Result(TargetArg, getPlatformFromOS(TT.getOS()),
+                          TT.getOSVersion(), A);
     VersionTuple OsVersion = TT.getOSVersion();
-    if (OsVersion.getMajor() == 0)
-      Result.HasOSVersion = false;
     Result.TargetVariantTriple = TargetVariantTriple;
     Result.setEnvironment(TT.getEnvironment(), OsVersion, SDKInfo);
     return Result;
@@ -1838,38 +1862,45 @@ struct DarwinPlatform {
   createFromMTargetOS(llvm::Triple::OSType OS, VersionTuple OSVersion,
                       llvm::Triple::EnvironmentType Environment, Arg *A,
                       const std::optional<DarwinSDKInfo> &SDKInfo) {
-    DarwinPlatform Result(MTargetOSArg, getPlatformFromOS(OS),
-                          OSVersion.getAsString(), A);
+    DarwinPlatform Result(MTargetOSArg, getPlatformFromOS(OS), OSVersion, A);
     Result.InferSimulatorFromArch = false;
     Result.setEnvironment(Environment, OSVersion, SDKInfo);
     return Result;
   }
   static DarwinPlatform createOSVersionArg(DarwinPlatformKind Platform, Arg *A,
                                            bool IsSimulator) {
-    DarwinPlatform Result{OSVersionArg, Platform, A};
+    DarwinPlatform Result{OSVersionArg, Platform,
+                          getVersionFromString(A->getValue()), A};
     if (IsSimulator)
       Result.Environment = DarwinEnvironmentKind::Simulator;
     return Result;
   }
   static DarwinPlatform createDeploymentTargetEnv(DarwinPlatformKind Platform,
                                                   StringRef EnvVarName,
-                                                  StringRef Value) {
-    DarwinPlatform Result(DeploymentTargetEnv, Platform, Value);
+                                                  StringRef OSVersion) {
+    DarwinPlatform Result(DeploymentTargetEnv, Platform,
+                          getVersionFromString(OSVersion));
     Result.EnvVarName = EnvVarName;
     return Result;
   }
-  static DarwinPlatform createFromSDK(DarwinPlatformKind Platform,
+  static DarwinPlatform createFromSDK(StringRef SDKRoot,
+                                      DarwinPlatformKind Platform,
                                       StringRef Value,
                                       bool IsSimulator = false) {
-    DarwinPlatform Result(InferredFromSDK, Platform, Value);
+    DarwinPlatform Result(InferredFromSDK, Platform,
+                          getVersionFromString(Value));
     if (IsSimulator)
       Result.Environment = DarwinEnvironmentKind::Simulator;
     Result.InferSimulatorFromArch = false;
+    Result.InferredSource = SDKRoot;
     return Result;
   }
-  static DarwinPlatform createFromArch(llvm::Triple::OSType OS,
-                                       StringRef Value) {
-    return DarwinPlatform(InferredFromArch, getPlatformFromOS(OS), Value);
+  static DarwinPlatform createFromArch(StringRef Arch, llvm::Triple::OSType OS,
+                                       VersionTuple Version) {
+    auto Result =
+        DarwinPlatform(InferredFromArch, getPlatformFromOS(OS), Version);
+    Result.InferredSource = Arch;
+    return Result;
   }
 
   /// Constructs an inferred SDKInfo value based on the version inferred from
@@ -1877,22 +1908,31 @@ struct DarwinPlatform {
   /// the platform from the SDKPath.
   DarwinSDKInfo inferSDKInfo() {
     assert(Kind == InferredFromSDK && "can infer SDK info only");
-    llvm::VersionTuple Version;
-    bool IsValid = !Version.tryParse(OSVersion);
-    (void)IsValid;
-    assert(IsValid && "invalid SDK version");
-    return DarwinSDKInfo(
-        Version,
-        /*MaximumDeploymentTarget=*/VersionTuple(Version.getMajor(), 0, 99),
-        getOSFromPlatform(Platform));
+    return DarwinSDKInfo(getOSVersion(),
+                         /*MaximumDeploymentTarget=*/
+                         VersionTuple(getOSVersion().getMajor(), 0, 99),
+                         getOSFromPlatform(Platform));
   }
 
 private:
   DarwinPlatform(SourceKind Kind, DarwinPlatformKind Platform, Arg *Argument)
-      : Kind(Kind), Platform(Platform), Argument(Argument) {}
-  DarwinPlatform(SourceKind Kind, DarwinPlatformKind Platform, StringRef Value,
-                 Arg *Argument = nullptr)
-      : Kind(Kind), Platform(Platform), OSVersion(Value), Argument(Argument) {}
+      : Kind(Kind), Platform(Platform),
+        Arguments({Argument, VersionTuple().getAsString()}) {}
+  DarwinPlatform(SourceKind Kind, DarwinPlatformKind Platform,
+                 VersionTuple Value, Arg *Argument = nullptr)
+      : Kind(Kind), Platform(Platform),
+        Arguments({Argument, Value.getAsString()}) {
+    if (!Value.empty())
+      UnderlyingOSVersion = Value;
+  }
+
+  static VersionTuple getVersionFromString(const StringRef Input) {
+    llvm::VersionTuple Version;
+    bool IsValid = !Version.tryParse(Input);
+    assert(IsValid && "unable to convert input version to version tuple");
+    (void)IsValid;
+    return Version;
+  }
 
   static DarwinPlatformKind getPlatformFromOS(llvm::Triple::OSType OS) {
     switch (OS) {
@@ -1935,11 +1975,23 @@ struct DarwinPlatform {
   SourceKind Kind;
   DarwinPlatformKind Platform;
   DarwinEnvironmentKind Environment = DarwinEnvironmentKind::NativeEnvironment;
-  VersionTuple NativeTargetVersion;
-  std::string OSVersion;
-  bool HasOSVersion = true, InferSimulatorFromArch = true;
-  Arg *Argument;
+  // When compiling for a zippered target, this means both target &
+  // target variant is set on the command line, ZipperedOSVersion holds the
+  // OSVersion tied to the main target value.
+  VersionTuple ZipperedOSVersion;
+  // We allow multiple ways to set or default the OS
+  // version used for compilation. When set, UnderlyingOSVersion represents
+  // the intended version to match the platform information computed from
+  // arguments.
+  std::optional<VersionTuple> UnderlyingOSVersion;
+  bool InferSimulatorFromArch = true;
+  std::pair<Arg *, std::string> Arguments;
   StringRef EnvVarName;
+  // If the DarwinPlatform information is derived from an inferred source, this
+  // captures what that source input was for error reporting.
+  StringRef InferredSource;
+  // When compiling for a zippered target, this value represents the target
+  // triple encoded in the target variant.
   std::optional<llvm::Triple> TargetVariantTriple;
 };
 
@@ -1957,6 +2009,19 @@ getDeploymentTargetFromOSVersionArg(DerivedArgList &Args,
   Arg *WatchOSVersion =
       Args.getLastArg(options::OPT_mwatchos_version_min_EQ,
                       options::OPT_mwatchos_simulator_version_min_EQ);
+
+  auto GetDarwinPlatform =
+      [&](DarwinPlatform::DarwinPlatformKind Platform, Arg *VersionArg,
+          bool IsSimulator) -> std::optional<DarwinPlatform> {
+    if (StringRef(VersionArg->getValue()).empty()) {
+      TheDriver.Diag(diag::err_drv_missing_version_number)
+          << VersionArg->getAsString(Args);
+      return std::nullopt;
+    }
+    return DarwinPlatform::createOSVersionArg(Platform, VersionArg,
+                                              /*IsSimulator=*/IsSimulator);
+  };
+
   if (macOSVersion) {
     if (iOSVersion || TvOSVersion || WatchOSVersion) {
       TheDriver.Diag(diag::err_drv_argument_not_allowed_with)
@@ -1965,30 +2030,29 @@ getDeploymentTargetFromOSVersionArg(DerivedArgList &Args,
                          : TvOSVersion ? TvOSVersion : WatchOSVersion)
                  ->getAsString(Args);
     }
-    return DarwinPlatform::createOSVersionArg(Darwin::MacOS, macOSVersion,
-                                              /*IsSimulator=*/false);
+    return GetDarwinPlatform(Darwin::MacOS, macOSVersion,
+                             /*IsSimulator=*/false);
+
   } else if (iOSVersion) {
     if (TvOSVersion || WatchOSVersion) {
       TheDriver.Diag(diag::err_drv_argument_not_allowed_with)
           << iOSVersion->getAsString(Args)
           << (TvOSVersion ? TvOSVersion : WatchOSVersion)->getAsString(Args);
     }
-    return DarwinPlatform::createOSVersionArg(
-        Darwin::IPhoneOS, iOSVersion,
-        iOSVersion->getOption().getID() ==
-            options::OPT_mios_simulator_version_min_EQ);
+    return GetDarwinPlatform(Darwin::IPhoneOS, iOSVersion,
+                             iOSVersion->getOption().getID() ==
+                                 options::OPT_mios_simulator_version_min_EQ);
   } else if (TvOSVersion) {
     if (WatchOSVersion) {
       TheDriver.Diag(diag::err_drv_argument_not_allowed_with)
           << TvOSVersion->getAsString(Args)
           << WatchOSVersion->getAsString(Args);
     }
-    return DarwinPlatform::createOSVersionArg(
-        Darwin::TvOS, TvOSVersion,
-        TvOSVersion->getOption().getID() ==
-            options::OPT_mtvos_simulator_version_min_EQ);
+    return GetDarwinPlatform(Darwin::TvOS, TvOSVersion,
+                             TvOSVersion->getOption().getID() ==
+                                 options::OPT_mtvos_simulator_version_min_EQ);
   } else if (WatchOSVersion)
-    return DarwinPlatform::createOSVersionArg(
+    return GetDarwinPlatform(
         Darwin::WatchOS, WatchOSVersion,
         WatchOSVersion->getOption().getID() ==
             options::OPT_mwatchos_simulator_version_min_EQ);
@@ -2094,26 +2158,27 @@ inferDeploymentTargetFromSDK(DerivedArgList &Args,
       [&](StringRef SDK) -> std::optional<DarwinPlatform> {
     if (SDK.starts_with("iPhoneOS") || SDK.starts_with("iPhoneSimulator"))
       return DarwinPlatform::createFromSDK(
-          Darwin::IPhoneOS, Version,
+          isysroot, Darwin::IPhoneOS, Version,
           /*IsSimulator=*/SDK.starts_with("iPhoneSimulator"));
     else if (SDK.starts_with("MacOSX"))
-      return DarwinPlatform::createFromSDK(Darwin::MacOS,
+      return DarwinPlatform::createFromSDK(isysroot, Darwin::MacOS,
                                            getSystemOrSDKMacOSVersion(Version));
     else if (SDK.starts_with("WatchOS") || SDK.starts_with("WatchSimulator"))
       return DarwinPlatform::createFromSDK(
-          Darwin::WatchOS, Version,
+          isysroot, Darwin::WatchOS, Version,
           /*IsSimulator=*/SDK.starts_with("WatchSimulator"));
     else if (SDK.starts_with("AppleTVOS") ||
              SDK.starts_with("AppleTVSimulator"))
       return DarwinPlatform::createFromSDK(
-          Darwin::TvOS, Version,
+          isysroot, Darwin::TvOS, Version,
           /*IsSimulator=*/SDK.starts_with("AppleTVSimulator"));
     else if (SDK.starts_with("XR"))
       return DarwinPlatform::createFromSDK(
-          Darwin::XROS, Version,
+          isysroot, Darwin::XROS, Version,
           /*IsSimulator=*/SDK.contains("Simulator"));
     else if (SDK.starts_with("DriverKit"))
-      return DarwinPlatform::createFromSDK(Darwin::DriverKit, Version);
+      return DarwinPlatform::createFromSDK(isysroot, Darwin::DriverKit,
+                                           Version);
     return std::nullopt;
   };
   if (auto Result = CreatePlatformFromSDKName(SDK))
@@ -2121,9 +2186,10 @@ inferDeploymentTargetFromSDK(DerivedArgList &Args,
   // The SDK can be an SDK variant with a name like `<prefix>.<platform>`.
   return CreatePlatformFromSDKName(dropSDKNamePrefix(SDK));
 }
-
-std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple,
-                         const Driver &TheDriver) {
+// Compute & get the OS Version when the target triple omitted one.
+VersionTuple getInferredOSVersion(llvm::Triple::OSType OS,
+                                  const llvm::Triple &Triple,
+                                  const Driver &TheDriver) {
   VersionTuple OsVersion;
   llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
   switch (OS) {
@@ -2162,12 +2228,7 @@ std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple,
     llvm_unreachable("Unexpected OS type");
     break;
   }
-
-  std::string OSVersion;
-  llvm::raw_string_ostream(OSVersion)
-      << OsVersion.getMajor() << '.' << OsVersion.getMinor().value_or(0) << '.'
-      << OsVersion.getSubminor().value_or(0);
-  return OSVersion;
+  return OsVersion;
 }
 
 /// Tries to infer the target OS from the -arch.
@@ -2190,8 +2251,8 @@ inferDeploymentTargetFromArch(DerivedArgList &Args, const Darwin &Toolchain,
     OSTy = llvm::Triple::MacOSX;
   if (OSTy == llvm::Triple::UnknownOS)
     return std::nullopt;
-  return DarwinPlatform::createFromArch(OSTy,
-                                        getOSVersion(OSTy, Triple, TheDriver));
+  return DarwinPlatform::createFromArch(
+      MachOArchName, OSTy, getInferredOSVersion(OSTy, Triple, TheDriver));
 }
 
 /// Returns the deployment target that's specified using the -target option.
@@ -2203,7 +2264,6 @@ std::optional<DarwinPlatform> getDeploymentTargetFromTargetArg(
   if (Triple.getOS() == llvm::Triple::Darwin ||
       Triple.getOS() == llvm::Triple::UnknownOS)
     return std::nullopt;
-  std::string OSVersion = getOSVersion(Triple.getOS(), Triple, TheDriver);
   std::optional<llvm::Triple> TargetVariantTriple;
   for (const Arg *A : Args.filtered(options::OPT_darwin_target_variant)) {
     llvm::Triple TVT(A->getValue());
@@ -2229,9 +2289,11 @@ std::optional<DarwinPlatform> getDeploymentTargetFromTargetArg(
           << A->getSpelling() << A->getValue();
     }
   }
-  return DarwinPlatform::createFromTarget(Triple, OSVersion,
-                                          Args.getLastArg(options::OPT_target),
-                                          TargetVariantTriple, SDKInfo);
+  DarwinPlatform PlatformAndVersion = DarwinPlatform::createFromTarget(
+      Triple, Args.getLastArg(options::OPT_target), TargetVariantTriple,
+      SDKInfo);
+
+  return PlatformAndVersion;
 }
 
 /// Returns the deployment target that's specified using the -mtargetos option.
@@ -2310,119 +2372,153 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   SDKInfo = parseSDKSettings(getVFS(), Args, getDriver());
 
   // The OS and the version can be specified using the -target argument.
-  std::optional<DarwinPlatform> OSTarget =
+  std::optional<DarwinPlatform> PlatformAndVersion =
       getDeploymentTargetFromTargetArg(Args, getTriple(), getDriver(), SDKInfo);
-  if (OSTarget) {
+  if (PlatformAndVersion) {
     // Disallow mixing -target and -mtargetos=.
     if (const auto *MTargetOSArg = Args.getLastArg(options::OPT_mtargetos_EQ)) {
-      std::string TargetArgStr = OSTarget->getAsString(Args, Opts);
+      std::string TargetArgStr = PlatformAndVersion->getAsString(Args, Opts);
       std::string MTargetOSArgStr = MTargetOSArg->getAsString(Args);
       getDriver().Diag(diag::err_drv_cannot_mix_options)
           << TargetArgStr << MTargetOSArgStr;
     }
-    std::optional<DarwinPlatform> OSVersionArgTarget =
+    // Implicitly allow resolving the OS version when it wasn't explicitly set.
+    bool TripleProvidedOSVersion = PlatformAndVersion->hasOSVersion();
+    if (!TripleProvidedOSVersion)
+      PlatformAndVersion->setOSVersion(
+          getInferredOSVersion(getTriple().getOS(), getTriple(), getDriver()));
+
+    std::optional<DarwinPlatform> PlatformAndVersionFromOSVersionArg =
         getDeploymentTargetFromOSVersionArg(Args, getDriver());
-    if (OSVersionArgTarget) {
+    if (PlatformAndVersionFromOSVersionArg) {
       unsigned TargetMajor, TargetMinor, TargetMicro;
       bool TargetExtra;
       unsigned ArgMajor, ArgMinor, ArgMicro;
       bool ArgExtra;
-      if (OSTarget->getPlatform() != OSVersionArgTarget->getPlatform() ||
-          (Driver::GetReleaseVersion(OSTarget->getOSVersion(), TargetMajor,
-                                     TargetMinor, TargetMicro, TargetExtra) &&
-           Driver::GetReleaseVersion(OSVersionArgTarget->getOSVersion(),
-                                     ArgMajor, ArgMinor, ArgMicro, ArgExtra) &&
+      if (PlatformAndVersion->getPlatform() !=
+              PlatformAndVersionFromOSVersionArg->getPlatform() ||
+          (Driver::GetReleaseVersion(
+               PlatformAndVersion->getOSVersion().getAsString(), TargetMajor,
+               TargetMinor, TargetMicro, TargetExtra) &&
+           Driver::GetReleaseVersion(
+               PlatformAndVersionFromOSVersionArg->getOSVersion().getAsString(),
+               ArgMajor, ArgMinor, ArgMicro, ArgExtra) &&
            (VersionTuple(TargetMajor, TargetMinor, TargetMicro) !=
                 VersionTuple(ArgMajor, ArgMinor, ArgMicro) ||
             TargetExtra != ArgExtra))) {
         // Select the OS version from the -m<os>-version-min argument when
         // the -target does not include an OS version.
-        if (OSTarget->getPlatform() == OSVersionArgTarget->getPlatform() &&
-            !OSTarget->hasOSVersion()) {
-          OSTarget->setOSVersion(OSVersionArgTarget->getOSVersion());
+        if (PlatformAndVersion->getPlatform() ==
+                PlatformAndVersionFromOSVersionArg->getPlatform() &&
+            !TripleProvidedOSVersion) {
+          PlatformAndVersion->setOSVersion(
+              PlatformAndVersionFromOSVersionArg->getOSVersion());
         } else {
           // Warn about -m<os>-version-min that doesn't match the OS version
           // that's specified in the target.
           std::string OSVersionArg =
-              OSVersionArgTarget->getAsString(Args, Opts);
-          std::string TargetArg = OSTarget->getAsString(Args, Opts);
+              PlatformAndVersionFromOSVersionArg->getAsString(Args, Opts);
+          std::string TargetArg = PlatformAndVersion->getAsString(Args, Opts);
           getDriver().Diag(clang::diag::warn_drv_overriding_option)
               << OSVersionArg << TargetArg;
         }
       }
     }
-  } else if ((OSTarget = getDeploymentTargetFromMTargetOSArg(Args, getDriver(),
-                                                             SDKInfo))) {
+  } else if ((PlatformAndVersion = getDeploymentTargetFromMTargetOSArg(
+                  Args, getDriver(), SDKInfo))) {
     // The OS target can be specified using the -mtargetos= argument.
     // Disallow mixing -mtargetos= and -m<os>version-min=.
-    std::optional<DarwinPlatform> OSVersionArgTarget =
+    std::optional<DarwinPlatform> PlatformAndVersionFromOSVersionArg =
         getDeploymentTargetFromOSVersionArg(Args, getDriver());
-    if (OSVersionArgTarget) {
-      std::string MTargetOSArgStr = OSTarget->getAsString(Args, Opts);
-      std::string OSVersionArgStr = OSVersionArgTarget->getAsString(Args, Opts);
+    if (PlatformAndVersionFromOSVersionArg) {
+      std::string MTargetOSArgStr = PlatformAndVersion->getAsString(Args, Opts);
+      std::string OSVersionArgStr =
+          PlatformAndVersionFromOSVersionArg->getAsString(Args, Opts);
       getDriver().Diag(diag::err_drv_cannot_mix_options)
           << MTargetOSArgStr << OSVersionArgStr;
     }
   } else {
     // The OS target can be specified using the -m<os>version-min argument.
-    OSTarget = getDeploymentTargetFromOSVersionArg(Args, getDriver());
+    PlatformAndVersion = getDeploymentTargetFromOSVersionArg(Args, getDriver());
     // If no deployment target was specified on the command line, check for
     // environment defines.
-    if (!OSTarget) {
-      OSTarget =
+    if (!PlatformAndVersion) {
+      PlatformAndVersion =
           getDeploymentTargetFromEnvironmentVariables(getDriver(), getTriple());
-      if (OSTarget) {
+      if (PlatformAndVersion) {
         // Don't infer simulator from the arch when the SDK is also specified.
         std::optional<DarwinPlatform> SDKTarget =
             inferDeploymentTargetFromSDK(Args, SDKInfo);
         if (SDKTarget)
-          OSTarget->setEnvironment(SDKTarget->getEnvironment());
+          PlatformAndVersion->setEnvironment(SDKTarget->getEnvironment());
       }
     }
     // If there is no command-line argument to specify the Target version and
     // no environment variable defined, see if we can set the default based
     // on -isysroot using SDKSettings.json if it exists.
-    if (!OSTarget) {
-      OSTarget = inferDeploymentTargetFromSDK(Args, SDKInfo);
+    if (!PlatformAndVersion) {
+      PlatformAndVersion = inferDeploymentTargetFromSDK(Args, SDKInfo);
       /// If the target was successfully constructed from the SDK path, try to
       /// infer the SDK info if the SDK doesn't have it.
-      if (OSTarget && !SDKInfo)
-        SDKInfo = OSTarget->inferSDKInfo();
+      if (PlatformAndVersion && !SDKInfo)
+        SDKInfo = PlatformAndVersion->inferSDKInfo();
     }
     // If no OS targets have been specified, try to guess platform from -target
     // or arch name and compute the version from the triple.
-    if (!OSTarget)
-      OSTarget =
+    if (!PlatformAndVersion)
+      PlatformAndVersion =
           inferDeploymentTargetFromArch(Args, *this, getTriple(), getDriver());
   }
 
-  assert(OSTarget && "Unable to infer Darwin variant");
-  OSTarget->addOSVersionMinArgument(Args, Opts);
-  DarwinPlatformKind Platform = OSTarget->getPlatform();
+  assert(PlatformAndVersion && "Unable to infer Darwin variant");
+  if (!PlatformAndVersion->isValidOSVersion()) {
+    if (PlatformAndVersion->isExplicitlySpecified())
+      getDriver().Diag(diag::err_drv_invalid_version_number)
+          << PlatformAndVersion->getAsString(Args, Opts);
+    else
+      getDriver().Diag(diag::err_drv_invalid_version_number_inferred)
+          << PlatformAndVersion->getOSVersion().getAsString()
+          << PlatformAndVersion->getInferredSource();
+  }
+  // After the deployment OS version has been resolved, set it to the canonical
+  // version before further error detection and converting to a proper target
+  // triple.
+  VersionTuple CanonicalVersion = PlatformAndVersion->getCanonicalOSVersion();
+  if (CanonicalVersion != PlatformAndVersion->getOSVersion()) {
+    getDriver().Diag(diag::warn_drv_overriding_deployment_version)
+        << PlatformAndVersion->getOSVersion().getAsString()
+        << CanonicalVersion.getAsString();
+    PlatformAndVersion->setOSVersion(CanonicalVersion);
+  }
+
+  PlatformAndVersion->addOSVersionMinArgument(Args, Opts);
+  DarwinPlatformKind Platform = PlatformAndVersion->getPlatform();
 
   unsigned Major, Minor, Micro;
   bool HadExtra;
   // The major version should not be over this number.
   const unsigned MajorVersionLimit = 1000;
+  const VersionTuple OSVersion = PlatformAndVersion->takeOSVersion();
+  const std::string OSVersionStr = OSVersion.getAsString();
   // Set the tool chain target information.
   if (Platform == MacOS) {
-    if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
-                                   Micro, HadExtra) ||
+    if (!Driver::GetReleaseVersion(OSVersionStr, Major, Minor, Micro,
+                                   HadExtra) ||
         HadExtra || Major < 10 || Major >= MajorVersionLimit || Minor >= 100 ||
         Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
-          << OSTarget->getAsString(Args, Opts);
+          << PlatformAndVersion->getAsString(Args, Opts);
   } else if (Platform == IPhoneOS) {
-    if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
-                                   Micro, HadExtra) ||
+    if (!Driver::GetReleaseVersion(OSVersionStr, Major, Minor, Micro,
+                                   HadExtra) ||
         HadExtra || Major >= MajorVersionLimit || Minor >= 100 || Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
-          << OSTarget->getAsString(Args, Opts);
+          << PlatformAndVersion->getAsString(Args, Opts);
     ;
-    if (OSTarget->getEnvironment() == MacCatalyst &&
+    if (PlatformAndVersion->getEnvironment() == MacCatalyst &&
         (Major < 13 || (Major == 13 && Minor < 1))) {
       getDriver().Diag(diag::err_drv_invalid_version_number)
-          << OSTarget->getAsString(Args, Opts);
+          << PlatformAndVersion->getAsString(Args, Opts);
       Major = 13;
       Minor = 1;
       Micro = 0;
@@ -2431,12 +2527,12 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
     // iOS 11.
     if (getTriple().isArch32Bit() && Major >= 11) {
       // If the deployment target is explicitly specified, print a diagnostic.
-      if (OSTarget->isExplicitlySpecified()) {
-        if (OSTarget->getEnvironment() == MacCatalyst)
+      if (PlatformAndVersion->isExplicitlySpecified()) {
+        if (PlatformAndVersion->getEnvironment() == MacCatalyst)
           getDriver().Diag(diag::err_invalid_macos_32bit_deployment_target);
         else
           getDriver().Diag(diag::warn_invalid_ios_deployment_target)
-              << OSTarget->getAsString(Args, Opts);
+              << PlatformAndVersion->getAsString(Args, Opts);
         // Otherwise, set it to 10.99.99.
       } else {
         Major = 10;
@@ -2445,46 +2541,52 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
       }
     }
   } else if (Platform == TvOS) {
-    if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
-                                   Micro, HadExtra) ||
+    if (!Driver::GetReleaseVersion(OSVersionStr, Major, Minor, Micro,
+                                   HadExtra) ||
         HadExtra || Major >= MajorVersionLimit || Minor >= 100 || Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
-          << OSTarget->getAsString(Args, Opts);
+          << PlatformAndVersion->getAsString(Args, Opts);
   } else if (Platform == WatchOS) {
-    if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
-                                   Micro, HadExtra) ||
+    if (!Driver::GetReleaseVersion(OSVersionStr, Major, Minor, Micro,
+                                   HadExtra) ||
         HadExtra || Major >= MajorVersionLimit || Minor >= 100 || Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
-          << OSTarget->getAsString(Args, Opts);
+          << PlatformAndVersion->getAsString(Args, Opts);
   } else if (Platform == DriverKit) {
-    if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
-                                   Micro, HadExtra) ||
+    if (!Driver::GetReleaseVersion(OSVersionStr, Major, Minor, Micro,
+                                   HadExtra) ||
         HadExtra || Major < 19 || Major >= MajorVersionLimit || Minor >= 100 ||
         Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
-          << OSTarget->getAsString(Args, Opts);
+          << PlatformAndVersion->getAsString(Args, Opts);
   } else if (Platform == XROS) {
-    if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
-                                   Micro, HadExtra) ||
+    if (!Driver::GetReleaseVersion(OSVersionStr, Major, Minor, Micro,
+                                   HadExtra) ||
         HadExtra || Major < 1 || Major >= MajorVersionLimit || Minor >= 100 ||
         Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
-          << OSTarget->getAsString(Args, Opts);
+          << PlatformAndVersion->getAsString(Args, Opts);
   } else
     llvm_unreachable("unknown kind of Darwin platform");
 
-  DarwinEnvironmentKind Environment = OSTarget->getEnvironment();
+  DarwinEnvironmentKind Environment = PlatformAndVersion->getEnvironment();
   // Recognize iOS targets with an x86 architecture as the iOS simulator.
   if (Environment == NativeEnvironment && Platform != MacOS &&
-      Platform != DriverKit && OSTarget->canInferSimulatorFromArch() &&
-      getTriple().isX86())
+      Platform != DriverKit &&
+      PlatformAndVersion->canInferSimulatorFromArch() && getTriple().isX86())
     Environment = Simulator;
 
-  VersionTuple NativeTargetVersion;
+  VersionTuple ZipperedOSVersion;
   if (Environment == MacCatalyst)
-    NativeTargetVersion = OSTarget->getNativeTargetVersion();
-  setTarget(Platform, Environment, Major, Minor, Micro, NativeTargetVersion);
-  TargetVariantTriple = OSTarget->getTargetVariantTriple();
+    ZipperedOSVersion = PlatformAndVersion->getZipperedOSVersion();
+  setTarget(Platform, Environment, Major, Minor, Micro, ZipperedOSVersion);
+  TargetVariantTriple = PlatformAndVersion->getTargetVariantTriple();
+  if (TargetVariantTriple &&
+      !llvm::Triple::isValidVersionForOS(TargetVariantTriple->getOS(),
+                                         TargetVariantTriple->getOSVersion())) {
+    getDriver().Diag(diag::err_drv_invalid_version_number)
+        << TargetVariantTriple->str();
+  }
 
   if (const Arg *A = Args.getLastArg(options::OPT_isysroot)) {
     StringRef SDK = getSDKName(A->getValue());
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/DragonFly.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/DragonFly.cpp
index 38a29ae49e8d..524f5f2ff391 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/DragonFly.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/DragonFly.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DragonFly.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Flang.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Flang.cpp
index 1ed3ae71ad4b..b4b337b2476e 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Flang.cpp
@@ -8,9 +8,9 @@
 
 #include "Flang.h"
 #include "Arch/RISCV.h"
-#include "CommonArgs.h"
 
 #include "clang/Basic/CodeGenOptions.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Frontend/Debug/Options.h"
 #include "llvm/Support/Path.h"
@@ -184,7 +184,8 @@ void Flang::addCodegenOptions(const ArgList &Args,
        options::OPT_frepack_arrays_contiguity_EQ,
        options::OPT_fstack_repack_arrays, options::OPT_fno_stack_repack_arrays,
        options::OPT_ftime_report, options::OPT_ftime_report_EQ,
-       options::OPT_funroll_loops, options::OPT_fno_unroll_loops});
+       options::OPT_funroll_loops, options::OPT_fno_unroll_loops,
+       options::OPT_fdefer_desc_map, options::OPT_fno_defer_desc_map});
 }
 
 void Flang::addPicOptions(const ArgList &Args, ArgStringList &CmdArgs) const {
@@ -530,15 +531,6 @@ void Flang::addTargetOptions(const ArgList &Args,
       CmdArgs.push_back(A->getValue());
   }
 
-  // FIXME: Remove it when local alias tags are generated by default
-  //  See: https://github.com/llvm/llvm-project/pull/139682
-  //  We need this downstream hack due to ARM failure for test case:
-  //  https://github.com/fujitsu/compiler-test-suite/blob/main/Fortran/0614/0614_0005.f
-  if (!Triple.isARM()) {
-    CmdArgs.push_back("-mmlir");
-    CmdArgs.push_back("-tbaa-local-alloc");
-  }
-
   Args.addAllArgs(CmdArgs,
                   {options::OPT_fverbose_asm, options::OPT_fno_verbose_asm});
 }
@@ -711,6 +703,10 @@ static void addFloatingPointOptions(const Driver &D, const ArgList &Args,
     A->claim();
   }
 
+  StringRef Recip = parseMRecipOption(D.getDiags(), Args);
+  if (!Recip.empty())
+    CmdArgs.push_back(Args.MakeArgString("-mrecip=" + Recip));
+
   if (!HonorINFs && !HonorNaNs && AssociativeMath && ReciprocalMath &&
       ApproxFunc && !SignedZeros &&
       (FPContract == "fast" || FPContract.empty())) {
@@ -905,6 +901,10 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   // TODO: Handle interactions between -w, -pedantic, -Wall, -WOption
   Args.AddLastArg(CmdArgs, options::OPT_w);
 
+  // recognise options: fprofile-generate -fprofile-use=
+  Args.addAllArgs(
+      CmdArgs, {options::OPT_fprofile_generate, options::OPT_fprofile_use_EQ});
+
   // Forward flags for OpenMP. We don't do this if the current action is an
   // device offloading action other than OpenMP.
   if (Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp
index ccf6987a2ea6..b17b76233ad3 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -10,8 +10,8 @@
 #include "Arch/ARM.h"
 #include "Arch/Mips.h"
 #include "Arch/Sparc.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/SanitizerArgs.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Fuchsia.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Fuchsia.cpp
index 910db1dde0d1..1c165bbfe84f 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "Fuchsia.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/MultilibBuilder.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp
index abaf57f99b2c..ffd04fad0aa6 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -15,8 +15,8 @@
 #include "Arch/RISCV.h"
 #include "Arch/Sparc.h"
 #include "Arch/SystemZ.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h" // for GCC_INSTALL_PREFIX
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/MultilibBuilder.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/HIPAMD.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/HIPAMD.cpp
index daa56135b47a..3b0a7b613146 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -8,10 +8,10 @@
 
 #include "HIPAMD.h"
 #include "AMDGPU.h"
-#include "CommonArgs.h"
 #include "HIPUtility.h"
 #include "SPIRV.h"
 #include "clang/Basic/Cuda.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/HIPSPV.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/HIPSPV.cpp
index 02ec36df273a..ec29c62976e1 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/HIPSPV.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/HIPSPV.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "HIPSPV.h"
-#include "CommonArgs.h"
 #include "HIPUtility.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/HIPUtility.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/HIPUtility.cpp
index 8a3dd494f3b4..cb061ffede23 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/HIPUtility.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/HIPUtility.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "HIPUtility.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Options.h"
 #include "llvm/ADT/StringExtras.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/HLSL.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/HLSL.cpp
index eb8f6881bffc..dcc51e182924 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "HLSL.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Job.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Haiku.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Haiku.cpp
index 5d0d8b7f5024..dd6ada8f5697 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Haiku.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Haiku.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "Haiku.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/SanitizerArgs.h"
 #include "llvm/Support/Path.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Hexagon.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Hexagon.cpp
index bed8de94130c..9f8b676fc7dc 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Hexagon.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Hurd.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Hurd.cpp
index 0bc114b90ffc..a22a8face179 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Hurd.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Hurd.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "Hurd.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Support/Path.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/InterfaceStubs.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/InterfaceStubs.cpp
index 05a13db8d0cf..59a6742c406c 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/InterfaceStubs.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/InterfaceStubs.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "InterfaceStubs.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "llvm/Support/Path.h"
 
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp
index 92e8a1baddac..082da1feca75 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp
@@ -12,8 +12,8 @@
 #include "Arch/Mips.h"
 #include "Arch/PPC.h"
 #include "Arch/RISCV.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Distro.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/MSP430.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/MSP430.cpp
index be0f9f214538..9eca1ad5f286 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/MSP430.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/MSP430.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "MSP430.h"
-#include "CommonArgs.h"
 #include "Gnu.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Multilib.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp
index f41d05248427..e2c88e22952a 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "MSVC.h"
-#include "CommonArgs.h"
 #include "Darwin.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/MinGW.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/MinGW.cpp
index 664576b4abfc..3b387f633273 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "MinGW.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/NaCl.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/NaCl.cpp
index ab619615d67e..f7acaa1f3a78 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/NaCl.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/NaCl.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "NaCl.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/NetBSD.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/NetBSD.cpp
index ae164be1b4e8..8db00deeb80d 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/NetBSD.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/NetBSD.cpp
@@ -10,8 +10,8 @@
 #include "Arch/ARM.h"
 #include "Arch/Mips.h"
 #include "Arch/Sparc.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/OHOS.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/OHOS.cpp
index 72f36d08f4da..00991504e97a 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/OHOS.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/OHOS.cpp
@@ -8,8 +8,8 @@
 
 #include "OHOS.h"
 #include "Arch/ARM.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/OpaqueOffloadLinker.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/OpaqueOffloadLinker.cpp
index 1d358fc44ac7..4ac297635a74 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/OpaqueOffloadLinker.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/OpaqueOffloadLinker.cpp
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 #include "Clang.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
+#include "llvm/ADT/StringMap.h"    
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Path.h"
 
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp
index 668626259809..79b1b6960da1 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/OpenBSD.cpp
@@ -10,8 +10,8 @@
 #include "Arch/ARM.h"
 #include "Arch/Mips.h"
 #include "Arch/Sparc.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/SanitizerArgs.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/PS4CPU.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/PS4CPU.cpp
index cb56a7ebeba2..e965786d269f 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "PS4CPU.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/RISCVToolchain.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
index d88ddc264d72..42bbc130637b 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "RISCVToolchain.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/SPIRV.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/SPIRV.cpp
index 8a2d1fae93c2..afb9e63b4b34 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/SPIRV.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/SPIRV.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "SPIRV.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/SPIRVOpenMP.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/SPIRVOpenMP.cpp
index a688fd8d9fb5..4e09d5eac26f 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/SPIRVOpenMP.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/SPIRVOpenMP.cpp
@@ -6,7 +6,7 @@
 //
 //==------------------------------------------------------------------------==//
 #include "SPIRVOpenMP.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 
 using namespace clang::driver;
 using namespace clang::driver::toolchains;
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/SYCL.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/SYCL.cpp
index 6611c142a5ef..0232b047a6c4 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/SYCL.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/SYCL.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "SYCL.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 
 using namespace clang::driver;
 using namespace clang::driver::toolchains;
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp
index 48d100238176..a3574e1b701e 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "Solaris.h"
-#include "CommonArgs.h"
 #include "Gnu.h"
 #include "clang/Basic/LangStandard.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/UEFI.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/UEFI.cpp
index 2ef719bb43aa..ac6668e6bdd5 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/UEFI.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/UEFI.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "UEFI.h"
-#include "CommonArgs.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/VEToolchain.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/VEToolchain.cpp
index cf85a691dbdb..ad9129046c3e 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/VEToolchain.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/VEToolchain.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "VEToolchain.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/WebAssembly.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/WebAssembly.cpp
index b3ed49662999..5054868b5ff4 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssembly.h"
-#include "CommonArgs.h"
 #include "Gnu.h"
 #include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/XCore.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/XCore.cpp
index c95ebabdd30c..6a2a75cb9973 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/XCore.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/XCore.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "XCore.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
diff --git a/external/llvm-project/clang/lib/Driver/ToolChains/ZOS.cpp b/external/llvm-project/clang/lib/Driver/ToolChains/ZOS.cpp
index d05202972825..57bcb3c306ce 100644
--- a/external/llvm-project/clang/lib/Driver/ToolChains/ZOS.cpp
+++ b/external/llvm-project/clang/lib/Driver/ToolChains/ZOS.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ZOS.h"
-#include "CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Option/ArgList.h"
diff --git a/external/llvm-project/clang/lib/Driver/XRayArgs.cpp b/external/llvm-project/clang/lib/Driver/XRayArgs.cpp
index 701dd2906dcc..ceed7cb6acbb 100644
--- a/external/llvm-project/clang/lib/Driver/XRayArgs.cpp
+++ b/external/llvm-project/clang/lib/Driver/XRayArgs.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "clang/Driver/XRayArgs.h"
-#include "ToolChains/CommonArgs.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/ToolChain.h"
diff --git a/external/llvm-project/clang/lib/Edit/EditedSource.cpp b/external/llvm-project/clang/lib/Edit/EditedSource.cpp
index a3386b2489b0..398cce71d5e2 100644
--- a/external/llvm-project/clang/lib/Edit/EditedSource.cpp
+++ b/external/llvm-project/clang/lib/Edit/EditedSource.cpp
@@ -16,10 +16,8 @@
 #include "clang/Edit/FileOffset.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include <algorithm>
 #include <cassert>
 #include <tuple>
 #include <utility>
diff --git a/external/llvm-project/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp b/external/llvm-project/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
index 764c345a9db9..1087eb300185 100644
--- a/external/llvm-project/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
+++ b/external/llvm-project/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
@@ -43,7 +43,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
diff --git a/external/llvm-project/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/external/llvm-project/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index e881d56258e5..139023f32e8d 100644
--- a/external/llvm-project/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/external/llvm-project/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -19,14 +19,12 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/raw_ostream.h"
 #include <iterator>
 #include <optional>
-#include <type_traits>
 
 using namespace clang;
 using namespace clang::extractapi;
diff --git a/external/llvm-project/clang/lib/Format/ContinuationIndenter.cpp b/external/llvm-project/clang/lib/Format/ContinuationIndenter.cpp
index 4e4e48f90a89..b4745477b96e 100644
--- a/external/llvm-project/clang/lib/Format/ContinuationIndenter.cpp
+++ b/external/llvm-project/clang/lib/Format/ContinuationIndenter.cpp
@@ -329,9 +329,9 @@ bool ContinuationIndenter::canBreak(const LineState &State) {
   // statement and we are aligning lambda blocks to their signatures.
   if (Previous.is(tok::l_brace) && State.Stack.size() > 1 &&
       State.Stack[State.Stack.size() - 2].NestedBlockInlined &&
-      State.Stack[State.Stack.size() - 2].HasMultipleNestedBlocks &&
-      Style.LambdaBodyIndentation == FormatStyle::LBI_Signature) {
-    return false;
+      State.Stack[State.Stack.size() - 2].HasMultipleNestedBlocks) {
+    return Style.isCpp() &&
+           Style.LambdaBodyIndentation == FormatStyle::LBI_OuterScope;
   }
 
   // Don't break after very short return types (e.g. "void") as that is often
@@ -706,42 +706,48 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
   const FormatToken &Previous = *State.NextToken->Previous;
   auto &CurrentState = State.Stack.back();
 
-  bool DisallowLineBreaksOnThisLine =
-      Style.LambdaBodyIndentation == FormatStyle::LBI_Signature &&
-      // Deal with lambda arguments in C++. The aim here is to ensure that we
-      // don't over-indent lambda function bodies when lambdas are passed as
-      // arguments to function calls. We do this by ensuring that either all
-      // arguments (including any lambdas) go on the same line as the function
-      // call, or we break before the first argument.
-      Style.isCpp() && [&] {
-        // For example, `/*Newline=*/false`.
-        if (Previous.is(TT_BlockComment) && Current.SpacesRequiredBefore == 0)
-          return false;
-        const auto *PrevNonComment = Current.getPreviousNonComment();
-        if (!PrevNonComment || PrevNonComment->isNot(tok::l_paren))
-          return false;
-        if (Current.isOneOf(tok::comment, tok::l_paren, TT_LambdaLSquare))
-          return false;
-        auto BlockParameterCount = PrevNonComment->BlockParameterCount;
-        if (BlockParameterCount == 0)
-          return false;
+  // Deal with lambda arguments in C++. The aim here is to ensure that we don't
+  // over-indent lambda function bodies when lambdas are passed as arguments to
+  // function calls. We do this by ensuring that either all arguments (including
+  // any lambdas) go on the same line as the function call, or we break before
+  // the first argument.
+  auto DisallowLineBreaks = [&] {
+    if (!Style.isCpp() ||
+        Style.LambdaBodyIndentation == FormatStyle::LBI_OuterScope) {
+      return false;
+    }
 
-        // Multiple lambdas in the same function call.
-        if (BlockParameterCount > 1)
-          return true;
+    // For example, `/*Newline=*/false`.
+    if (Previous.is(TT_BlockComment) && Current.SpacesRequiredBefore == 0)
+      return false;
 
-        // A lambda followed by another arg.
-        if (!PrevNonComment->Role)
-          return false;
-        auto Comma = PrevNonComment->Role->lastComma();
-        if (!Comma)
-          return false;
-        auto Next = Comma->getNextNonComment();
-        return Next &&
-               !Next->isOneOf(TT_LambdaLSquare, tok::l_brace, tok::caret);
-      }();
+    if (Current.isOneOf(tok::comment, tok::l_paren, TT_LambdaLSquare))
+      return false;
+
+    const auto *Prev = Current.getPreviousNonComment();
+    if (!Prev || Prev->isNot(tok::l_paren))
+      return false;
+
+    if (Prev->BlockParameterCount == 0)
+      return false;
+
+    // Multiple lambdas in the same function call.
+    if (Prev->BlockParameterCount > 1)
+      return true;
+
+    // A lambda followed by another arg.
+    if (!Prev->Role)
+      return false;
+
+    const auto *Comma = Prev->Role->lastComma();
+    if (!Comma)
+      return false;
+
+    const auto *Next = Comma->getNextNonComment();
+    return Next && !Next->isOneOf(TT_LambdaLSquare, tok::l_brace, tok::caret);
+  };
 
-  if (DisallowLineBreaksOnThisLine)
+  if (DisallowLineBreaks())
     State.NoLineBreak = true;
 
   if (Current.is(tok::equal) &&
@@ -2248,7 +2254,6 @@ unsigned ContinuationIndenter::reformatRawStringLiteral(
       /*Status=*/nullptr);
 
   auto NewCode = applyAllReplacements(RawText, Fixes.first);
-  tooling::Replacements NoFixes;
   if (!NewCode)
     return addMultilineToken(Current, State);
   if (!DryRun) {
diff --git a/external/llvm-project/clang/lib/Format/MacroCallReconstructor.cpp b/external/llvm-project/clang/lib/Format/MacroCallReconstructor.cpp
index 116bbad320e1..895d9f93dfce 100644
--- a/external/llvm-project/clang/lib/Format/MacroCallReconstructor.cpp
+++ b/external/llvm-project/clang/lib/Format/MacroCallReconstructor.cpp
@@ -528,10 +528,10 @@ MacroCallReconstructor::createUnwrappedLine(const ReconstructedLine &Line,
       // 1. One level below the current line's level.
       // 2. At the correct level relative to each other.
       unsigned MinChildLevel =
-          std::min_element(N->Children.begin(), N->Children.end(),
-                           [](const auto &E1, const auto &E2) {
-                             return E1->Level < E2->Level;
-                           })
+          llvm::min_element(N->Children,
+                            [](const auto &E1, const auto &E2) {
+                              return E1->Level < E2->Level;
+                            })
               ->get()
               ->Level;
       for (const auto &Child : N->Children) {
diff --git a/external/llvm-project/clang/lib/Format/TokenAnnotator.cpp b/external/llvm-project/clang/lib/Format/TokenAnnotator.cpp
index 52100d195fe7..d2f8b2703a9a 100644
--- a/external/llvm-project/clang/lib/Format/TokenAnnotator.cpp
+++ b/external/llvm-project/clang/lib/Format/TokenAnnotator.cpp
@@ -1306,6 +1306,7 @@ class AnnotatingParser {
     if (Tok->is(TT_TableGenMultiLineString))
       return true;
     auto *Prev = Tok->getPreviousNonComment();
+    auto *Next = Tok->getNextNonComment();
     switch (bool IsIf = false; Tok->Tok.getKind()) {
     case tok::plus:
     case tok::minus:
@@ -1435,10 +1436,10 @@ class AnnotatingParser {
           if (Prev->isAccessSpecifierKeyword())
             Line.Type = LT_AccessModifier;
         }
-      } else if (canBeObjCSelectorComponent(*Prev) && Tok->Next &&
-                 (Tok->Next->isOneOf(tok::r_paren, tok::comma) ||
-                  (canBeObjCSelectorComponent(*Tok->Next) && Tok->Next->Next &&
-                   Tok->Next->Next->is(tok::colon)))) {
+      } else if (canBeObjCSelectorComponent(*Prev) && Next &&
+                 (Next->isOneOf(tok::r_paren, tok::comma) ||
+                  (canBeObjCSelectorComponent(*Next) && Next->Next &&
+                   Next->Next->is(tok::colon)))) {
         // This handles a special macro in ObjC code where selectors including
         // the colon are passed as macro arguments.
         Tok->setType(TT_ObjCMethodExpr);
@@ -1476,10 +1477,8 @@ class AnnotatingParser {
     case tok::kw_for:
       if (Style.isJavaScript()) {
         // x.for and {for: ...}
-        if ((Prev && Prev->is(tok::period)) ||
-            (Tok->Next && Tok->Next->is(tok::colon))) {
+        if ((Prev && Prev->is(tok::period)) || (Next && Next->is(tok::colon)))
           break;
-        }
         // JS' for await ( ...
         if (CurrentToken && CurrentToken->is(Keywords.kw_await))
           next();
@@ -1690,9 +1689,9 @@ class AnnotatingParser {
         CurrentToken->Previous->setType(TT_OverloadedOperator);
       break;
     case tok::question:
-      if (Style.isJavaScript() && Tok->Next &&
-          Tok->Next->isOneOf(tok::semi, tok::comma, tok::colon, tok::r_paren,
-                             tok::r_brace, tok::r_square)) {
+      if (Style.isJavaScript() && Next &&
+          Next->isOneOf(tok::semi, tok::comma, tok::colon, tok::r_paren,
+                        tok::r_brace, tok::r_square)) {
         // Question marks before semicolons, colons, etc. indicate optional
         // types (fields, parameters), e.g.
         //   function(x?: string, y?) {...}
@@ -1709,8 +1708,7 @@ class AnnotatingParser {
       if (Style.isCSharp()) {
         // `Type?)`, `Type?>`, `Type? name;`, and `Type? name =` can only be
         // nullable types.
-        if (const auto *Next = Tok->getNextNonComment();
-            Next && (Next->isOneOf(tok::r_paren, tok::greater) ||
+        if (Next && (Next->isOneOf(tok::r_paren, tok::greater) ||
                      Next->startsSequence(tok::identifier, tok::semi) ||
                      Next->startsSequence(tok::identifier, tok::equal))) {
           Tok->setType(TT_CSharpNullable);
@@ -1723,10 +1721,8 @@ class AnnotatingParser {
         // cond ? id : "B";
         // cond ? cond2 ? "A" : "B" : "C";
         if (!Contexts.back().IsExpression && Line.MustBeDeclaration &&
-            (!Tok->Next ||
-             !Tok->Next->isOneOf(tok::identifier, tok::string_literal) ||
-             !Tok->Next->Next ||
-             !Tok->Next->Next->isOneOf(tok::colon, tok::question))) {
+            (!Next || !Next->isOneOf(tok::identifier, tok::string_literal) ||
+             !Next->Next || !Next->Next->isOneOf(tok::colon, tok::question))) {
           Tok->setType(TT_CSharpNullable);
           break;
         }
@@ -1773,20 +1769,27 @@ class AnnotatingParser {
                        Keywords.kw___has_include_next)) {
         parseHasInclude();
       }
-      if (Style.isCSharp() && Tok->is(Keywords.kw_where) && Tok->Next &&
-          Tok->Next->isNot(tok::l_paren)) {
-        Tok->setType(TT_CSharpGenericTypeConstraint);
-        parseCSharpGenericTypeConstraint();
-        if (!Prev)
-          Line.IsContinuation = true;
-      }
-      if (Style.isTableGen()) {
+      if (IsCpp) {
+        if (Next && Next->is(tok::l_paren) && Prev &&
+            Prev->isOneOf(tok::kw___cdecl, tok::kw___stdcall,
+                          tok::kw___fastcall, tok::kw___thiscall,
+                          tok::kw___regcall, tok::kw___vectorcall)) {
+          Tok->setFinalizedType(TT_FunctionDeclarationName);
+          Next->setFinalizedType(TT_FunctionDeclarationLParen);
+        }
+      } else if (Style.isCSharp()) {
+        if (Tok->is(Keywords.kw_where) && Next && Next->isNot(tok::l_paren)) {
+          Tok->setType(TT_CSharpGenericTypeConstraint);
+          parseCSharpGenericTypeConstraint();
+          if (!Prev)
+            Line.IsContinuation = true;
+        }
+      } else if (Style.isTableGen()) {
         if (Tok->is(Keywords.kw_assert)) {
           if (!parseTableGenValue())
             return false;
         } else if (Tok->isOneOf(Keywords.kw_def, Keywords.kw_defm) &&
-                   (!Tok->Next ||
-                    !Tok->Next->isOneOf(tok::colon, tok::l_brace))) {
+                   (!Next || !Next->isOneOf(tok::colon, tok::l_brace))) {
           // The case NameValue appears.
           if (!parseTableGenValue(true))
             return false;
@@ -3848,6 +3851,8 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts,
   } else {
     if (Current.isNot(TT_StartOfName) || Current.NestingLevel != 0)
       return false;
+    while (Next && Next->startsSequence(tok::hashhash, tok::identifier))
+      Next = Next->Next->Next;
     for (; Next; Next = Next->Next) {
       if (Next->is(TT_TemplateOpener) && Next->MatchingParen) {
         Next = Next->MatchingParen;
@@ -3973,16 +3978,11 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const {
   for (auto *Tok = FirstNonComment && FirstNonComment->isNot(tok::kw_using)
                        ? FirstNonComment->Next
                        : nullptr;
-       Tok; Tok = Tok->Next) {
+       Tok && Tok->isNot(BK_BracedInit); Tok = Tok->Next) {
     if (Tok->is(TT_StartOfName))
       SeenName = true;
-    const auto *Previous = Tok->Previous;
-    if (Previous->EndsCppAttributeGroup) {
+    if (Tok->Previous->EndsCppAttributeGroup)
       AfterLastAttribute = Tok;
-    } else if (Line.InMacroBody &&
-               Previous->endsSequence(tok::hashhash, TT_StartOfName)) {
-      Tok->setType(TT_StartOfName);
-    }
     if (const bool IsCtorOrDtor = Tok->is(TT_CtorDtorDeclName);
         IsCtorOrDtor ||
         isFunctionDeclarationName(LangOpts, *Tok, Line, ClosingParen)) {
diff --git a/external/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp b/external/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp
index 6c78f0b7dbd9..61b84126fe1b 100644
--- a/external/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/external/llvm-project/clang/lib/Format/UnwrappedLineParser.cpp
@@ -269,7 +269,7 @@ void UnwrappedLineParser::parseFile() {
   bool MustBeDeclaration = !Line->InPPDirective && !Style.isJavaScript();
   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
                                           MustBeDeclaration);
-  if (Style.isTextProto())
+  if (Style.isTextProto() || (Style.isJson() && FormatTok->IsFirst))
     parseBracedList();
   else
     parseLevel();
@@ -3485,6 +3485,7 @@ bool UnwrappedLineParser::parseRequires(bool SeenEqual) {
   case tok::r_paren:
   case tok::kw_noexcept:
   case tok::kw_const:
+  case tok::star:
   case tok::amp:
     // This is a requires clause.
     parseRequiresClause(RequiresToken);
diff --git a/external/llvm-project/clang/lib/Frontend/ASTConsumers.cpp b/external/llvm-project/clang/lib/Frontend/ASTConsumers.cpp
index a5ec6961fbf0..ab8a35a18925 100644
--- a/external/llvm-project/clang/lib/Frontend/ASTConsumers.cpp
+++ b/external/llvm-project/clang/lib/Frontend/ASTConsumers.cpp
@@ -38,6 +38,13 @@ namespace {
           OutputKind(K), OutputFormat(Format), FilterString(FilterString),
           DumpLookups(DumpLookups), DumpDeclTypes(DumpDeclTypes) {}
 
+    ASTPrinter(raw_ostream &Out, Kind K, ASTDumpOutputFormat Format,
+               StringRef FilterString, bool DumpLookups = false,
+               bool DumpDeclTypes = false)
+        : Out(Out), OwnedOut(nullptr), OutputKind(K), OutputFormat(Format),
+          FilterString(FilterString), DumpLookups(DumpLookups),
+          DumpDeclTypes(DumpDeclTypes) {}
+
     void HandleTranslationUnit(ASTContext &Context) override {
       TranslationUnitDecl *D = Context.getTranslationUnitDecl();
 
@@ -173,6 +180,19 @@ clang::CreateASTDumper(std::unique_ptr<raw_ostream> Out, StringRef FilterString,
       Format, FilterString, DumpLookups, DumpDeclTypes);
 }
 
+std::unique_ptr<ASTConsumer>
+clang::CreateASTDumper(raw_ostream &Out, StringRef FilterString, bool DumpDecls,
+                       bool Deserialize, bool DumpLookups, bool DumpDeclTypes,
+                       ASTDumpOutputFormat Format) {
+  assert((DumpDecls || Deserialize || DumpLookups) && "nothing to dump");
+  return std::make_unique<ASTPrinter>(Out,
+                                      Deserialize ? ASTPrinter::DumpFull
+                                      : DumpDecls ? ASTPrinter::Dump
+                                                  : ASTPrinter::None,
+                                      Format, FilterString, DumpLookups,
+                                      DumpDeclTypes);
+}
+
 std::unique_ptr<ASTConsumer> clang::CreateASTDeclNodeLister() {
   return std::make_unique<ASTDeclNodeLister>(nullptr);
 }
diff --git a/external/llvm-project/clang/lib/Frontend/CompilerInstance.cpp b/external/llvm-project/clang/lib/Frontend/CompilerInstance.cpp
index 587b0d1af9c8..09a66b652518 100644
--- a/external/llvm-project/clang/lib/Frontend/CompilerInstance.cpp
+++ b/external/llvm-project/clang/lib/Frontend/CompilerInstance.cpp
@@ -1191,7 +1191,7 @@ std::unique_ptr<CompilerInstance> CompilerInstance::cloneForModuleCompileImpl(
   FrontendOpts.OriginalModuleMap = std::string(OriginalModuleMapFile);
   // Force implicitly-built modules to hash the content of the module file.
   HSOpts.ModulesHashContent = true;
-  FrontendOpts.Inputs = {Input};
+  FrontendOpts.Inputs = {std::move(Input)};
 
   // Don't free the remapped file buffers; they are owned by our caller.
   PPOpts.RetainRemappedFileBuffers = true;
diff --git a/external/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp b/external/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp
index 8e136f3a3da6..f26aa5630e8f 100644
--- a/external/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/external/llvm-project/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1500,11 +1500,11 @@ static void setPGOUseInstrumentor(CodeGenOptions &Opts,
   // which is available (might be one or both).
   if (PGOReader->isIRLevelProfile() || PGOReader->hasMemoryProfile()) {
     if (PGOReader->hasCSIRLevelProfile())
-      Opts.setProfileUse(CodeGenOptions::ProfileCSIRInstr);
+      Opts.setProfileUse(llvm::driver::ProfileInstrKind::ProfileCSIRInstr);
     else
-      Opts.setProfileUse(CodeGenOptions::ProfileIRInstr);
+      Opts.setProfileUse(llvm::driver::ProfileInstrKind::ProfileIRInstr);
   } else
-    Opts.setProfileUse(CodeGenOptions::ProfileClangInstr);
+    Opts.setProfileUse(llvm::driver::ProfileInstrKind::ProfileClangInstr);
 }
 
 void CompilerInvocation::setDefaultPointerAuthOptions(
@@ -4603,6 +4603,8 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
         Opts.setClangABICompat(LangOptions::ClangABI::Ver18);
       else if (Major <= 19)
         Opts.setClangABICompat(LangOptions::ClangABI::Ver19);
+      else if (Major <= 20)
+        Opts.setClangABICompat(LangOptions::ClangABI::Ver20);
     } else if (Ver != "latest") {
       Diags.Report(diag::err_drv_invalid_value)
           << A->getAsString(Args) << A->getValue();
diff --git a/external/llvm-project/clang/lib/Frontend/FrontendAction.cpp b/external/llvm-project/clang/lib/Frontend/FrontendAction.cpp
index af9d18ab6610..ef7ae27a2694 100644
--- a/external/llvm-project/clang/lib/Frontend/FrontendAction.cpp
+++ b/external/llvm-project/clang/lib/Frontend/FrontendAction.cpp
@@ -181,7 +181,7 @@ class DeserializedDeclsSourceRangePrinter : public ASTConsumer,
         if (MergedRanges.back().second < It->second)
           MergedRanges.back().second = It->second;
       }
-      Result.push_back({Data.Ref->getName(), MergedRanges});
+      Result.push_back({Data.Ref->getName(), std::move(MergedRanges)});
     }
     printJson(Result);
   }
diff --git a/external/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp b/external/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp
index 69a91eef6aed..f64613fb4a6c 100644
--- a/external/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/external/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Basic/DiagnosticLex.h"
 #include "clang/Basic/HLSLRuntime.h"
 #include "clang/Basic/MacroBuilder.h"
 #include "clang/Basic/SourceManager.h"
@@ -1644,4 +1645,11 @@ void clang::InitializePreprocessor(Preprocessor &PP,
 
   // Copy PredefinedBuffer into the Preprocessor.
   PP.setPredefines(std::move(PredefineBuffer));
+
+  // Match gcc behavior regarding gnu-line-directive diagnostics, assuming that
+  // '-x <*>-cpp-output' is analogous to '-fpreprocessed'.
+  if (FEOpts.DashX.isPreprocessed()) {
+    PP.getDiagnostics().setSeverity(diag::ext_pp_gnu_line_directive,
+                                    diag::Severity::Ignored, SourceLocation());
+  }
 }
diff --git a/external/llvm-project/clang/lib/Headers/amxmovrstransposeintrin.h b/external/llvm-project/clang/lib/Headers/amxmovrstransposeintrin.h
index 17a9f7506a04..5f48cba949f3 100644
--- a/external/llvm-project/clang/lib/Headers/amxmovrstransposeintrin.h
+++ b/external/llvm-project/clang/lib/Headers/amxmovrstransposeintrin.h
@@ -197,4 +197,4 @@ static void __tile_2rpntlvwz1rst1(__tile1024i *dst0, __tile1024i *dst1,
 
 #undef __DEFAULT_FN_ATTRS
 #endif /* __x86_64__ */
-#endif /* __AMX_MOVRS_TRANSPOSEINTRIN_H */
\ No newline at end of file
+#endif /* __AMX_MOVRS_TRANSPOSEINTRIN_H */
diff --git a/external/llvm-project/clang/lib/Headers/arm_acle.h b/external/llvm-project/clang/lib/Headers/arm_acle.h
index a7a62b69e046..5cfa3d023a7d 100644
--- a/external/llvm-project/clang/lib/Headers/arm_acle.h
+++ b/external/llvm-project/clang/lib/Headers/arm_acle.h
@@ -29,52 +29,16 @@ extern "C" {
 
 /* 7 SYNCHRONIZATION, BARRIER AND HINT INTRINSICS */
 /* 7.3 Memory barriers */
-#if !__has_builtin(__dmb)
-#define __dmb(i) __builtin_arm_dmb(i)
-#endif
-#if !__has_builtin(__dsb)
-#define __dsb(i) __builtin_arm_dsb(i)
-#endif
-#if !__has_builtin(__isb)
-#define __isb(i) __builtin_arm_isb(i)
-#endif
+void __dmb(unsigned int);
+void __dsb(unsigned int);
+void __isb(unsigned int);
 
 /* 7.4 Hints */
-
-#if !__has_builtin(__wfi)
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
-__wfi(void) {
-  __builtin_arm_wfi();
-}
-#endif
-
-#if !__has_builtin(__wfe)
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
-__wfe(void) {
-  __builtin_arm_wfe();
-}
-#endif
-
-#if !__has_builtin(__sev)
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
-__sev(void) {
-  __builtin_arm_sev();
-}
-#endif
-
-#if !__has_builtin(__sevl)
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
-__sevl(void) {
-  __builtin_arm_sevl();
-}
-#endif
-
-#if !__has_builtin(__yield)
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
-__yield(void) {
-  __builtin_arm_yield();
-}
-#endif
+void __wfi(void);
+void __wfe(void);
+void __sev(void);
+void __sevl(void);
+void __yield(void);
 
 #if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
 #define __dbg(t) __builtin_arm_dbg(t)
diff --git a/external/llvm-project/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h b/external/llvm-project/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
index 3835eb78cec5..cbc518d05058 100644
--- a/external/llvm-project/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
+++ b/external/llvm-project/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
@@ -2350,6 +2350,10 @@ _HLSL_AVAILABILITY(shadermodel, 6.0)
 _HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_is_first_lane)
 __attribute__((convergent)) bool WaveIsFirstLane();
 
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_get_lane_count)
+__attribute__((convergent)) uint WaveGetLaneCount();
+
 //===----------------------------------------------------------------------===//
 // WaveReadLaneAt builtins
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/lib/Headers/hlsl/hlsl_spirv.h b/external/llvm-project/clang/lib/Headers/hlsl/hlsl_spirv.h
index 711da2fea46a..3e272c7144a3 100644
--- a/external/llvm-project/clang/lib/Headers/hlsl/hlsl_spirv.h
+++ b/external/llvm-project/clang/lib/Headers/hlsl/hlsl_spirv.h
@@ -25,4 +25,4 @@ using SpirvOpaqueType = __hlsl_spirv_type<Opcode, 0, 0, Operands...>;
 } // namespace vk
 } // namespace hlsl
 
-#endif // _HLSL_HLSL_SPIRV_H_
\ No newline at end of file
+#endif // _HLSL_HLSL_SPIRV_H_
diff --git a/external/llvm-project/clang/lib/Headers/module.modulemap b/external/llvm-project/clang/lib/Headers/module.modulemap
index a805282903ee..3b5468c9532a 100644
--- a/external/llvm-project/clang/lib/Headers/module.modulemap
+++ b/external/llvm-project/clang/lib/Headers/module.modulemap
@@ -35,6 +35,14 @@ module _Builtin_intrinsics [system] [extern_c] {
     }
   }
 
+  explicit module arm64 {
+    requires arm64
+    requires windows
+
+    header "arm64intr.h"
+    export *
+  }
+
   explicit module intel {
     requires x86
     export *
diff --git a/external/llvm-project/clang/lib/Headers/movrsintrin.h b/external/llvm-project/clang/lib/Headers/movrsintrin.h
index 250f4004cd2b..94510485b290 100644
--- a/external/llvm-project/clang/lib/Headers/movrsintrin.h
+++ b/external/llvm-project/clang/lib/Headers/movrsintrin.h
@@ -56,4 +56,4 @@ _m_prefetchrs(volatile const void *__P) {
 }
 
 #undef __DEFAULT_FN_ATTRS
-#endif // __MOVRSINTRIN_H
\ No newline at end of file
+#endif // __MOVRSINTRIN_H
diff --git a/external/llvm-project/clang/lib/Index/CommentToXML.cpp b/external/llvm-project/clang/lib/Index/CommentToXML.cpp
index cd7226e71171..672386171ecf 100644
--- a/external/llvm-project/clang/lib/Index/CommentToXML.cpp
+++ b/external/llvm-project/clang/lib/Index/CommentToXML.cpp
@@ -11,7 +11,6 @@
 #include "clang/AST/Attr.h"
 #include "clang/AST/Comment.h"
 #include "clang/AST/CommentVisitor.h"
-#include "clang/Basic/FileManager.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Format/Format.h"
diff --git a/external/llvm-project/clang/lib/Index/FileIndexRecord.cpp b/external/llvm-project/clang/lib/Index/FileIndexRecord.cpp
index cf40a596f509..d6b0ee5c4ff1 100644
--- a/external/llvm-project/clang/lib/Index/FileIndexRecord.cpp
+++ b/external/llvm-project/clang/lib/Index/FileIndexRecord.cpp
@@ -7,10 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "FileIndexRecord.h"
-#include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/Basic/SourceManager.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/Path.h"
 
 using namespace clang;
diff --git a/external/llvm-project/clang/lib/Index/IndexBody.cpp b/external/llvm-project/clang/lib/Index/IndexBody.cpp
index 2ed20df22bda..98ce6f73ec84 100644
--- a/external/llvm-project/clang/lib/Index/IndexBody.cpp
+++ b/external/llvm-project/clang/lib/Index/IndexBody.cpp
@@ -435,6 +435,13 @@ class BodyIndexer : public RecursiveASTVisitor<BodyIndexer> {
                                             ParentDC, SymbolRoleSet(),
                                             /*Relations=*/{}, E);
           }
+        } else {
+          if (D.isArrayDesignator())
+            TraverseStmt(E->getArrayIndex(D));
+          else if (D.isArrayRangeDesignator()) {
+            TraverseStmt(E->getArrayRangeStart(D));
+            TraverseStmt(E->getArrayRangeEnd(D));
+          }
         }
       }
       return true;
diff --git a/external/llvm-project/clang/lib/Index/IndexingAction.cpp b/external/llvm-project/clang/lib/Index/IndexingAction.cpp
index 81c46a0d08de..73a6a8c62af2 100644
--- a/external/llvm-project/clang/lib/Index/IndexingAction.cpp
+++ b/external/llvm-project/clang/lib/Index/IndexingAction.cpp
@@ -10,12 +10,10 @@
 #include "IndexingContext.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendAction.h"
-#include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Index/IndexDataConsumer.h"
 #include "clang/Lex/PPCallbacks.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Serialization/ASTReader.h"
-#include "llvm/ADT/STLExtras.h"
 #include <memory>
 
 using namespace clang;
diff --git a/external/llvm-project/clang/lib/Index/USRGeneration.cpp b/external/llvm-project/clang/lib/Index/USRGeneration.cpp
index a23e3578d3ab..1d5dcacdaa41 100644
--- a/external/llvm-project/clang/lib/Index/USRGeneration.cpp
+++ b/external/llvm-project/clang/lib/Index/USRGeneration.cpp
@@ -13,7 +13,6 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/ODRHash.h"
-#include "clang/Basic/FileManager.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/external/llvm-project/clang/lib/InstallAPI/DiagnosticBuilderWrappers.cpp b/external/llvm-project/clang/lib/InstallAPI/DiagnosticBuilderWrappers.cpp
index fd9db8113a41..37b428216c91 100644
--- a/external/llvm-project/clang/lib/InstallAPI/DiagnosticBuilderWrappers.cpp
+++ b/external/llvm-project/clang/lib/InstallAPI/DiagnosticBuilderWrappers.cpp
@@ -8,7 +8,6 @@
 
 #include "DiagnosticBuilderWrappers.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TextAPI/Platform.h"
 
diff --git a/external/llvm-project/clang/lib/InstallAPI/DirectoryScanner.cpp b/external/llvm-project/clang/lib/InstallAPI/DirectoryScanner.cpp
index be43a96f3d97..f8f708fda4ca 100644
--- a/external/llvm-project/clang/lib/InstallAPI/DirectoryScanner.cpp
+++ b/external/llvm-project/clang/lib/InstallAPI/DirectoryScanner.cpp
@@ -9,7 +9,6 @@
 #include "clang/InstallAPI/DirectoryScanner.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
-#include "llvm/TextAPI/DylibReader.h"
 
 using namespace llvm;
 using namespace llvm::MachO;
diff --git a/external/llvm-project/clang/lib/InstallAPI/FileList.cpp b/external/llvm-project/clang/lib/InstallAPI/FileList.cpp
index 65610903840a..8f8ed6e8a5db 100644
--- a/external/llvm-project/clang/lib/InstallAPI/FileList.cpp
+++ b/external/llvm-project/clang/lib/InstallAPI/FileList.cpp
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/InstallAPI/FileList.h"
-#include "clang/Basic/DiagnosticFrontend.h"
 #include "clang/InstallAPI/FileList.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Error.h"
diff --git a/external/llvm-project/clang/lib/InstallAPI/Frontend.cpp b/external/llvm-project/clang/lib/InstallAPI/Frontend.cpp
index 9e8c60fbda3d..cce0b19b5061 100644
--- a/external/llvm-project/clang/lib/InstallAPI/Frontend.cpp
+++ b/external/llvm-project/clang/lib/InstallAPI/Frontend.cpp
@@ -9,7 +9,6 @@
 #include "clang/InstallAPI/Frontend.h"
 #include "clang/AST/Availability.h"
 #include "clang/InstallAPI/FrontendRecords.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 
 using namespace llvm;
diff --git a/external/llvm-project/clang/lib/InstallAPI/Visitor.cpp b/external/llvm-project/clang/lib/InstallAPI/Visitor.cpp
index a73ea0b0d124..487be2c30088 100644
--- a/external/llvm-project/clang/lib/InstallAPI/Visitor.cpp
+++ b/external/llvm-project/clang/lib/InstallAPI/Visitor.cpp
@@ -13,7 +13,6 @@
 #include "clang/Basic/Linkage.h"
 #include "clang/InstallAPI/DylibVerifier.h"
 #include "clang/InstallAPI/FrontendRecords.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Mangler.h"
diff --git a/external/llvm-project/clang/lib/Interpreter/DeviceOffload.cpp b/external/llvm-project/clang/lib/Interpreter/DeviceOffload.cpp
index 05625ddedb72..9a25a264b2d5 100644
--- a/external/llvm-project/clang/lib/Interpreter/DeviceOffload.cpp
+++ b/external/llvm-project/clang/lib/Interpreter/DeviceOffload.cpp
@@ -151,7 +151,7 @@ llvm::Error IncrementalCUDADeviceParser::GenerateFatbinary() {
                    llvm::StringRef(FatbinContent.data(), FatbinContent.size()),
                    "", false));
 
-  CodeGenOpts.CudaGpuBinaryFileName = FatbinFileName;
+  CodeGenOpts.CudaGpuBinaryFileName = std::move(FatbinFileName);
 
   FatbinContent.clear();
 
diff --git a/external/llvm-project/clang/lib/Interpreter/IncrementalParser.cpp b/external/llvm-project/clang/lib/Interpreter/IncrementalParser.cpp
index 41d6304bd5f6..6343f17ed822 100644
--- a/external/llvm-project/clang/lib/Interpreter/IncrementalParser.cpp
+++ b/external/llvm-project/clang/lib/Interpreter/IncrementalParser.cpp
@@ -176,7 +176,7 @@ void IncrementalParser::CleanUpPTU(TranslationUnitDecl *MostRecentTU) {
   // FIXME: We should de-allocate MostRecentTU
   for (Decl *D : MostRecentTU->decls()) {
     auto *ND = dyn_cast<NamedDecl>(D);
-    if (!ND)
+    if (!ND || ND->getDeclName().isEmpty())
       continue;
     // Check if we need to clean up the IdResolver chain.
     if (ND->getDeclName().getFETokenInfo() && !D->getLangOpts().ObjC &&
diff --git a/external/llvm-project/clang/lib/Interpreter/InterpreterValuePrinter.cpp b/external/llvm-project/clang/lib/Interpreter/InterpreterValuePrinter.cpp
index 3e3fbfd172ca..3e7e32b2e855 100644
--- a/external/llvm-project/clang/lib/Interpreter/InterpreterValuePrinter.cpp
+++ b/external/llvm-project/clang/lib/Interpreter/InterpreterValuePrinter.cpp
@@ -18,7 +18,6 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Interpreter/Interpreter.h"
 #include "clang/Interpreter/Value.h"
-#include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Sema.h"
 
@@ -26,7 +25,6 @@
 #include "llvm/Support/raw_ostream.h"
 
 #include <cassert>
-#include <string>
 
 #include <cstdarg>
 
diff --git a/external/llvm-project/clang/lib/Interpreter/Value.cpp b/external/llvm-project/clang/lib/Interpreter/Value.cpp
index eb2ce9c9fd33..afdf406b3725 100644
--- a/external/llvm-project/clang/lib/Interpreter/Value.cpp
+++ b/external/llvm-project/clang/lib/Interpreter/Value.cpp
@@ -16,10 +16,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Interpreter/Interpreter.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_os_ostream.h"
 #include <cassert>
-#include <cstdint>
 #include <utility>
 
 namespace {
diff --git a/external/llvm-project/clang/lib/Lex/DependencyDirectivesScanner.cpp b/external/llvm-project/clang/lib/Lex/DependencyDirectivesScanner.cpp
index 088d1cc96e3a..1b6b16c56114 100644
--- a/external/llvm-project/clang/lib/Lex/DependencyDirectivesScanner.cpp
+++ b/external/llvm-project/clang/lib/Lex/DependencyDirectivesScanner.cpp
@@ -323,10 +323,6 @@ static unsigned skipNewline(const char *&First, const char *End) {
   return Len;
 }
 
-static bool wasLineContinuation(const char *First, unsigned EOLLen) {
-  return *(First - (int)EOLLen - 1) == '\\';
-}
-
 static void skipToNewlineRaw(const char *&First, const char *const End) {
   for (;;) {
     if (First == End)
@@ -336,13 +332,16 @@ static void skipToNewlineRaw(const char *&First, const char *const End) {
     if (Len)
       return;
 
+    char LastNonWhitespace = ' ';
     do {
+      if (!isHorizontalWhitespace(*First))
+        LastNonWhitespace = *First;
       if (++First == End)
         return;
       Len = isEOL(First, End);
     } while (!Len);
 
-    if (First[-1] != '\\')
+    if (LastNonWhitespace != '\\')
       return;
 
     First += Len;
@@ -394,6 +393,7 @@ static bool isQuoteCppDigitSeparator(const char *const Start,
 }
 
 void Scanner::skipLine(const char *&First, const char *const End) {
+  char LastNonWhitespace = ' ';
   for (;;) {
     assert(First <= End);
     if (First == End)
@@ -419,6 +419,8 @@ void Scanner::skipLine(const char *&First, const char *const End) {
       // Iterate over comments correctly.
       if (*First != '/' || End - First < 2) {
         LastTokenPtr = First;
+        if (!isWhitespace(*First))
+          LastNonWhitespace = *First;
         ++First;
         continue;
       }
@@ -431,6 +433,8 @@ void Scanner::skipLine(const char *&First, const char *const End) {
 
       if (First[1] != '*') {
         LastTokenPtr = First;
+        if (!isWhitespace(*First))
+          LastNonWhitespace = *First;
         ++First;
         continue;
       }
@@ -442,8 +446,9 @@ void Scanner::skipLine(const char *&First, const char *const End) {
       return;
 
     // Skip over the newline.
-    unsigned Len = skipNewline(First, End);
-    if (!wasLineContinuation(First, Len)) // Continue past line-continuations.
+    skipNewline(First, End);
+
+    if (LastNonWhitespace != '\\')
       break;
   }
 }
@@ -468,9 +473,16 @@ static void skipWhitespace(const char *&First, const char *const End) {
     if (End - First < 2)
       return;
 
-    if (First[0] == '\\' && isVerticalWhitespace(First[1])) {
-      skipNewline(++First, End);
-      continue;
+    if (*First == '\\') {
+      const char *Ptr = First + 1;
+      while (Ptr < End && isHorizontalWhitespace(*Ptr))
+        ++Ptr;
+      if (Ptr != End && isVerticalWhitespace(*Ptr)) {
+        skipNewline(Ptr, End);
+        First = Ptr;
+        continue;
+      }
+      return;
     }
 
     // Check for a non-comment character.
@@ -496,7 +508,15 @@ bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
                                      const char *const End) {
   const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
   for (;;) {
+    // Keep a copy of the First char incase it needs to be reset.
+    const char *Previous = First;
     const dependency_directives_scan::Token &Tok = lexToken(First, End);
+    if ((Tok.is(tok::hash) || Tok.is(tok::at)) &&
+        (Tok.Flags & clang::Token::StartOfLine)) {
+      CurDirToks.pop_back();
+      First = Previous;
+      return false;
+    }
     if (Tok.is(tok::eof))
       return reportError(
           DirectiveLoc,
@@ -846,6 +866,7 @@ bool Scanner::lexPPLine(const char *&First, const char *const End) {
   if (*First == '@')
     return lexAt(First, End);
 
+  // Handle module directives for C++20 modules.
   if (*First == 'i' || *First == 'e' || *First == 'm')
     return lexModule(First, End);
 
diff --git a/external/llvm-project/clang/lib/Lex/HeaderMap.cpp b/external/llvm-project/clang/lib/Lex/HeaderMap.cpp
index 588b32ee9ca8..a7b670f00ac6 100644
--- a/external/llvm-project/clang/lib/Lex/HeaderMap.cpp
+++ b/external/llvm-project/clang/lib/Lex/HeaderMap.cpp
@@ -18,7 +18,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SystemZ/zOSSupport.h"
 #include <cstring>
 #include <memory>
 #include <optional>
diff --git a/external/llvm-project/clang/lib/Lex/LiteralSupport.cpp b/external/llvm-project/clang/lib/Lex/LiteralSupport.cpp
index 75ad977d64b2..a62508e3e27b 100644
--- a/external/llvm-project/clang/lib/Lex/LiteralSupport.cpp
+++ b/external/llvm-project/clang/lib/Lex/LiteralSupport.cpp
@@ -1420,7 +1420,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
   }
 
   // Parse a potential octal literal prefix.
-  bool SawOctalPrefix = false, IsSingleZero = false;
+  bool IsSingleZero = false;
   if ((c1 == 'O' || c1 == 'o') && (s[1] >= '0' && s[1] <= '7')) {
     unsigned DiagId;
     if (LangOpts.C2y)
@@ -1432,14 +1432,26 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
     Diags.Report(TokLoc, DiagId);
     ++s;
     DigitsBegin = s;
-    SawOctalPrefix = true;
+    radix = 8;
+    s = SkipOctalDigits(s);
+    if (s == ThisTokEnd) {
+      // Done
+    } else if ((isHexDigit(*s) && *s != 'e' && *s != 'E' && *s != '.') &&
+               !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) {
+      auto InvalidDigitLoc = Lexer::AdvanceToTokenCharacter(
+          TokLoc, s - ThisTokBegin, SM, LangOpts);
+      Diags.Report(InvalidDigitLoc, diag::err_invalid_digit)
+          << StringRef(s, 1) << 1;
+      hadError = true;
+    }
+    // Other suffixes will be diagnosed by the caller.
+    return;
   }
 
   auto _ = llvm::make_scope_exit([&] {
     // If we still have an octal value but we did not see an octal prefix,
     // diagnose as being an obsolescent feature starting in C2y.
-    if (radix == 8 && LangOpts.C2y && !SawOctalPrefix && !hadError &&
-        !IsSingleZero)
+    if (radix == 8 && LangOpts.C2y && !hadError && !IsSingleZero)
       Diags.Report(TokLoc, diag::warn_unprefixed_octal_deprecated);
   });
 
diff --git a/external/llvm-project/clang/lib/Lex/PPExpressions.cpp b/external/llvm-project/clang/lib/Lex/PPExpressions.cpp
index 118d4ae6791d..d18e56e7d478 100644
--- a/external/llvm-project/clang/lib/Lex/PPExpressions.cpp
+++ b/external/llvm-project/clang/lib/Lex/PPExpressions.cpp
@@ -984,7 +984,6 @@ static std::optional<CXXStandardLibraryVersionInfo>
 getCXXStandardLibraryVersion(Preprocessor &PP, StringRef MacroName,
                              CXXStandardLibraryVersionInfo::Library Lib) {
   MacroInfo *Macro = PP.getMacroInfo(PP.getIdentifierInfo(MacroName));
-
   if (!Macro || Macro->getNumTokens() != 1 || !Macro->isObjectLike())
     return std::nullopt;
 
@@ -995,7 +994,7 @@ getCXXStandardLibraryVersion(Preprocessor &PP, StringRef MacroName,
   llvm::StringRef RevisionDate =
       PP.getSpelling(RevisionDateTok, Buffer, &Invalid);
   if (!Invalid) {
-    unsigned Value;
+    std::uint64_t Value;
     // We don't use NumericParser to avoid diagnostics
     if (!RevisionDate.consumeInteger(10, Value))
       return CXXStandardLibraryVersionInfo{Lib, Value};
@@ -1004,7 +1003,7 @@ getCXXStandardLibraryVersion(Preprocessor &PP, StringRef MacroName,
                                        0};
 }
 
-std::optional<unsigned> Preprocessor::getStdLibCxxVersion() {
+std::optional<uint64_t> Preprocessor::getStdLibCxxVersion() {
   if (!CXXStandardLibraryVersion)
     CXXStandardLibraryVersion = getCXXStandardLibraryVersion(
         *this, "__GLIBCXX__", CXXStandardLibraryVersionInfo::LibStdCXX);
@@ -1017,10 +1016,10 @@ std::optional<unsigned> Preprocessor::getStdLibCxxVersion() {
   return std::nullopt;
 }
 
-bool Preprocessor::NeedsStdLibCxxWorkaroundBefore(unsigned FixedVersion) {
+bool Preprocessor::NeedsStdLibCxxWorkaroundBefore(uint64_t FixedVersion) {
   assert(FixedVersion >= 2000'00'00 && FixedVersion <= 2100'00'00 &&
          "invalid value for __GLIBCXX__");
-  std::optional<unsigned> Ver = getStdLibCxxVersion();
+  std::optional<std::uint64_t> Ver = getStdLibCxxVersion();
   if (!Ver)
     return false;
   return *Ver < FixedVersion;
diff --git a/external/llvm-project/clang/lib/Parse/ParseCXXInlineMethods.cpp b/external/llvm-project/clang/lib/Parse/ParseCXXInlineMethods.cpp
index e215c64cccd1..9a010fb5f342 100644
--- a/external/llvm-project/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -422,7 +422,6 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
         DefArgResult = ParseBraceInitializer();
       } else
         DefArgResult = ParseAssignmentExpression();
-      DefArgResult = Actions.CorrectDelayedTyposInExpr(DefArgResult, Param);
       if (DefArgResult.isInvalid()) {
         Actions.ActOnParamDefaultArgumentError(Param, EqualLoc,
                                                /*DefaultArg=*/nullptr);
diff --git a/external/llvm-project/clang/lib/Parse/ParseDecl.cpp b/external/llvm-project/clang/lib/Parse/ParseDecl.cpp
index d6c36616bab4..02f33511dbd6 100644
--- a/external/llvm-project/clang/lib/Parse/ParseDecl.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseDecl.cpp
@@ -151,7 +151,7 @@ bool Parser::ParseSingleGNUAttribute(ParsedAttributes &Attrs,
   SourceLocation AttrNameLoc = ConsumeToken();
 
   if (Tok.isNot(tok::l_paren)) {
-    Attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+    Attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                  ParsedAttr::Form::GNU());
     return false;
   }
@@ -396,12 +396,12 @@ void Parser::ParseAttributeWithTypeArg(IdentifierInfo &AttrName,
     return;
 
   if (T.isUsable())
-    Attrs.addNewTypeAttr(&AttrName,
-                         SourceRange(AttrNameLoc, Parens.getCloseLocation()),
-                         ScopeName, ScopeLoc, T.get(), Form);
+    Attrs.addNewTypeAttr(
+        &AttrName, SourceRange(AttrNameLoc, Parens.getCloseLocation()),
+        AttributeScopeInfo(ScopeName, ScopeLoc), T.get(), Form);
   else
     Attrs.addNew(&AttrName, SourceRange(AttrNameLoc, Parens.getCloseLocation()),
-                 ScopeName, ScopeLoc, nullptr, 0, Form);
+                 AttributeScopeInfo(ScopeName, ScopeLoc), nullptr, 0, Form);
 }
 
 ExprResult
@@ -436,7 +436,6 @@ bool Parser::ParseAttributeArgumentList(
     } else {
       Expr = ParseAssignmentExpression();
     }
-    Expr = Actions.CorrectDelayedTyposInExpr(Expr);
 
     if (Tok.is(tok::ellipsis))
       Expr = Actions.ActOnPackExpansion(Expr.get(), ConsumeToken());
@@ -472,15 +471,6 @@ bool Parser::ParseAttributeArgumentList(
     Arg++;
   }
 
-  if (SawError) {
-    // Ensure typos get diagnosed when errors were encountered while parsing the
-    // expression list.
-    for (auto &E : Exprs) {
-      ExprResult Expr = Actions.CorrectDelayedTyposInExpr(E);
-      if (Expr.isUsable())
-        E = Expr.get();
-    }
-  }
   return SawError;
 }
 
@@ -565,9 +555,7 @@ unsigned Parser::ParseAttributeArgsCommon(
               nullptr,
               Sema::ExpressionEvaluationContextRecord::EK_AttrArgument);
 
-          ExprResult ArgExpr(
-              Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression()));
-
+          ExprResult ArgExpr = ParseAssignmentExpression();
           if (ArgExpr.isInvalid()) {
             SkipUntil(tok::r_paren, StopAtSemi);
             return 0;
@@ -621,10 +609,12 @@ unsigned Parser::ParseAttributeArgsCommon(
 
     if (AttributeIsTypeArgAttr && !TheParsedType.get().isNull()) {
       Attrs.addNewTypeAttr(AttrName, SourceRange(AttrNameLoc, RParen),
-                           ScopeName, ScopeLoc, TheParsedType, Form);
+                           AttributeScopeInfo(ScopeName, ScopeLoc),
+                           TheParsedType, Form);
     } else {
-      Attrs.addNew(AttrName, SourceRange(AttrLoc, RParen), ScopeName, ScopeLoc,
-                   ArgExprs.data(), ArgExprs.size(), Form);
+      Attrs.addNew(AttrName, SourceRange(AttrLoc, RParen),
+                   AttributeScopeInfo(ScopeName, ScopeLoc), ArgExprs.data(),
+                   ArgExprs.size(), Form);
     }
   }
 
@@ -866,7 +856,7 @@ bool Parser::ParseMicrosoftDeclSpecArgs(IdentifierInfo *AttrName,
 
     // Only add the property attribute if it was well-formed.
     if (!HasInvalidAccessor)
-      Attrs.addNewPropertyAttr(AttrName, AttrNameLoc, nullptr, SourceLocation(),
+      Attrs.addNewPropertyAttr(AttrName, AttrNameLoc, AttributeScopeInfo(),
                                AccessorNames[AK_Get], AccessorNames[AK_Put],
                                ParsedAttr::Form::Declspec());
     T.skipToEnd();
@@ -952,7 +942,7 @@ void Parser::ParseMicrosoftDeclSpecs(ParsedAttributes &Attrs) {
             << AttrName->getName();
 
       if (!AttrHandled)
-        Attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+        Attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                      ParsedAttr::Form::Declspec());
     }
     T.consumeClose();
@@ -980,7 +970,7 @@ void Parser::ParseMicrosoftTypeAttributes(ParsedAttributes &attrs) {
     case tok::kw___uptr: {
       IdentifierInfo *AttrName = Tok.getIdentifierInfo();
       SourceLocation AttrNameLoc = ConsumeToken();
-      attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+      attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                    Kind);
       break;
     }
@@ -1001,9 +991,8 @@ void Parser::ParseWebAssemblyFuncrefTypeAttribute(ParsedAttributes &attrs) {
 
   IdentifierInfo *AttrName = Tok.getIdentifierInfo();
   SourceLocation AttrNameLoc = ConsumeToken();
-  attrs.addNew(AttrName, AttrNameLoc, /*ScopeName=*/nullptr,
-               /*ScopeLoc=*/SourceLocation{}, /*Args=*/nullptr, /*numArgs=*/0,
-               tok::kw___funcref);
+  attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), /*Args=*/nullptr,
+               /*numArgs=*/0, tok::kw___funcref);
 }
 
 void Parser::DiagnoseAndSkipExtendedMicrosoftTypeAttributes() {
@@ -1047,7 +1036,7 @@ void Parser::ParseBorlandTypeAttributes(ParsedAttributes &attrs) {
   while (Tok.is(tok::kw___pascal)) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
     SourceLocation AttrNameLoc = ConsumeToken();
-    attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+    attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                  tok::kw___pascal);
   }
 }
@@ -1057,7 +1046,7 @@ void Parser::ParseOpenCLKernelAttributes(ParsedAttributes &attrs) {
   while (Tok.is(tok::kw___kernel)) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
     SourceLocation AttrNameLoc = ConsumeToken();
-    attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+    attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                  tok::kw___kernel);
   }
 }
@@ -1066,7 +1055,7 @@ void Parser::ParseCUDAFunctionAttributes(ParsedAttributes &attrs) {
   while (Tok.is(tok::kw___noinline__)) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
     SourceLocation AttrNameLoc = ConsumeToken();
-    attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+    attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                  tok::kw___noinline__);
   }
 }
@@ -1074,7 +1063,7 @@ void Parser::ParseCUDAFunctionAttributes(ParsedAttributes &attrs) {
 void Parser::ParseOpenCLQualifiers(ParsedAttributes &Attrs) {
   IdentifierInfo *AttrName = Tok.getIdentifierInfo();
   SourceLocation AttrNameLoc = Tok.getLocation();
-  Attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+  Attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                Tok.getKind());
 }
 
@@ -1086,7 +1075,7 @@ void Parser::ParseHLSLQualifiers(ParsedAttributes &Attrs) {
   IdentifierInfo *AttrName = Tok.getIdentifierInfo();
   auto Kind = Tok.getKind();
   SourceLocation AttrNameLoc = ConsumeToken();
-  Attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0, Kind);
+  Attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0, Kind);
 }
 
 void Parser::ParseNullabilityTypeSpecifiers(ParsedAttributes &attrs) {
@@ -1103,7 +1092,7 @@ void Parser::ParseNullabilityTypeSpecifiers(ParsedAttributes &attrs) {
       if (!getLangOpts().ObjC)
         Diag(AttrNameLoc, diag::ext_nullability)
           << AttrName;
-      attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+      attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                    Kind);
       break;
     }
@@ -1447,10 +1436,11 @@ void Parser::ParseAvailabilityAttribute(
 
   // Record this attribute
   attrs.addNew(&Availability,
-               SourceRange(AvailabilityLoc, T.getCloseLocation()), ScopeName,
-               ScopeLoc, Platform, Changes[Introduced], Changes[Deprecated],
-               Changes[Obsoleted], UnavailableLoc, MessageExpr.get(), Form,
-               StrictLoc, ReplacementExpr.get(), EnvironmentLoc);
+               SourceRange(AvailabilityLoc, T.getCloseLocation()),
+               AttributeScopeInfo(ScopeName, ScopeLoc), Platform,
+               Changes[Introduced], Changes[Deprecated], Changes[Obsoleted],
+               UnavailableLoc, MessageExpr.get(), Form, StrictLoc,
+               ReplacementExpr.get(), EnvironmentLoc);
 }
 
 void Parser::ParseExternalSourceSymbolAttribute(
@@ -1568,7 +1558,8 @@ void Parser::ParseExternalSourceSymbolAttribute(
   ArgsUnion Args[] = {Language.get(), DefinedInExpr.get(), GeneratedDeclaration,
                       USR.get()};
   Attrs.addNew(&ExternalSourceSymbol, SourceRange(Loc, T.getCloseLocation()),
-               ScopeName, ScopeLoc, Args, std::size(Args), Form);
+               AttributeScopeInfo(ScopeName, ScopeLoc), Args, std::size(Args),
+               Form);
 }
 
 void Parser::ParseObjCBridgeRelatedAttribute(
@@ -1636,8 +1627,8 @@ void Parser::ParseObjCBridgeRelatedAttribute(
   // Record this attribute
   Attrs.addNew(&ObjCBridgeRelated,
                SourceRange(ObjCBridgeRelatedLoc, T.getCloseLocation()),
-               ScopeName, ScopeLoc, RelatedClass, ClassMethod, InstanceMethod,
-               Form);
+               AttributeScopeInfo(ScopeName, ScopeLoc), RelatedClass,
+               ClassMethod, InstanceMethod, Form);
 }
 
 void Parser::ParseSwiftNewTypeAttribute(
@@ -1678,7 +1669,8 @@ void Parser::ParseSwiftNewTypeAttribute(
 
   ArgsUnion Args[] = {SwiftType};
   Attrs.addNew(&AttrName, SourceRange(AttrNameLoc, T.getCloseLocation()),
-               ScopeName, ScopeLoc, Args, std::size(Args), Form);
+               AttributeScopeInfo(ScopeName, ScopeLoc), Args, std::size(Args),
+               Form);
 }
 
 void Parser::ParseTypeTagForDatatypeAttribute(
@@ -1731,9 +1723,9 @@ void Parser::ParseTypeTagForDatatypeAttribute(
   }
 
   if (!T.consumeClose()) {
-    Attrs.addNewTypeTagForDatatype(&AttrName, AttrNameLoc, ScopeName, ScopeLoc,
-                                   ArgumentKind, MatchingCType.get(),
-                                   LayoutCompatible, MustBeNull, Form);
+    Attrs.addNewTypeTagForDatatype(
+        &AttrName, AttrNameLoc, AttributeScopeInfo(ScopeName, ScopeLoc),
+        ArgumentKind, MatchingCType.get(), LayoutCompatible, MustBeNull, Form);
   }
 
   if (EndLoc)
@@ -1840,9 +1832,10 @@ void Parser::ProhibitCXX11Attributes(ParsedAttributes &Attrs,
     if (!AL.isStandardAttributeSyntax())
       continue;
     if (AL.getKind() == ParsedAttr::UnknownAttribute) {
-      if (WarnOnUnknownAttrs)
-        Diag(AL.getLoc(), diag::warn_unknown_attribute_ignored)
-            << AL << AL.getRange();
+      if (WarnOnUnknownAttrs) {
+        Actions.DiagnoseUnknownAttribute(AL);
+        AL.setInvalid();
+      }
     } else {
       Diag(AL.getLoc(), AttrDiagID) << AL;
       AL.setInvalid();
@@ -3129,12 +3122,12 @@ void Parser::ParseAlignmentSpecifier(ParsedAttributes &Attrs,
     *EndLoc = T.getCloseLocation();
 
   if (IsType) {
-    Attrs.addNewTypeAttr(KWName, KWLoc, nullptr, KWLoc, TypeResult, Kind,
+    Attrs.addNewTypeAttr(KWName, KWLoc, AttributeScopeInfo(), TypeResult, Kind,
                          EllipsisLoc);
   } else {
     ArgsVector ArgExprs;
     ArgExprs.push_back(ArgExpr.get());
-    Attrs.addNew(KWName, KWLoc, nullptr, KWLoc, ArgExprs.data(), 1, Kind,
+    Attrs.addNew(KWName, KWLoc, AttributeScopeInfo(), ArgExprs.data(), 1, Kind,
                  EllipsisLoc);
   }
 }
@@ -3180,9 +3173,8 @@ void Parser::ParsePtrauthQualifier(ParsedAttributes &Attrs) {
     return;
   }
 
-  Attrs.addNew(KwName, SourceRange(KwLoc, EndLoc),
-               /*scope*/ nullptr, SourceLocation(), ArgExprs.data(),
-               ArgExprs.size(),
+  Attrs.addNew(KwName, SourceRange(KwLoc, EndLoc), AttributeScopeInfo(),
+               ArgExprs.data(), ArgExprs.size(),
                ParsedAttr::Form::Keyword(/*IsAlignAs=*/false,
                                          /*IsRegularKeywordAttribute=*/false));
 }
@@ -3212,9 +3204,7 @@ void Parser::ParseBoundsAttribute(IdentifierInfo &AttrName,
       Actions, Sema::ExpressionEvaluationContext::PotentiallyEvaluated, nullptr,
       ExpressionKind::EK_AttrArgument);
 
-  ExprResult ArgExpr(
-      Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression()));
-
+  ExprResult ArgExpr = ParseAssignmentExpression();
   if (ArgExpr.isInvalid()) {
     Parens.skipToEnd();
     return;
@@ -3230,7 +3220,7 @@ void Parser::ParseBoundsAttribute(IdentifierInfo &AttrName,
       Ctx.getSizeType(), SourceLocation()));
 
   Attrs.addNew(&AttrName, SourceRange(AttrNameLoc, Parens.getCloseLocation()),
-               ScopeName, ScopeLoc, ArgExprs.data(), ArgExprs.size(), Form);
+               AttributeScopeInfo(), ArgExprs.data(), ArgExprs.size(), Form);
 }
 
 ExprResult Parser::ParseExtIntegerArgument() {
@@ -4009,7 +3999,7 @@ void Parser::ParseDeclarationSpecifiers(
       isInvalid = DS.setFunctionSpecForceInline(Loc, PrevSpec, DiagID);
       IdentifierInfo *AttrName = Tok.getIdentifierInfo();
       SourceLocation AttrNameLoc = Tok.getLocation();
-      DS.getAttributes().addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc,
+      DS.getAttributes().addNew(AttrName, AttrNameLoc, AttributeScopeInfo(),
                                 nullptr, 0, tok::kw___forceinline);
       break;
     }
@@ -4067,8 +4057,9 @@ void Parser::ParseDeclarationSpecifiers(
 
     // Objective-C 'kindof' types.
     case tok::kw___kindof:
-      DS.getAttributes().addNew(Tok.getIdentifierInfo(), Loc, nullptr, Loc,
-                                nullptr, 0, tok::kw___kindof);
+      DS.getAttributes().addNew(Tok.getIdentifierInfo(), Loc,
+                                AttributeScopeInfo(), nullptr, 0,
+                                tok::kw___kindof);
       (void)ConsumeToken();
       continue;
 
@@ -6124,8 +6115,7 @@ bool Parser::isConstructorDeclarator(bool IsUnqualified, bool DeductionGuide,
 
 void Parser::ParseTypeQualifierListOpt(
     DeclSpec &DS, unsigned AttrReqs, bool AtomicOrPtrauthAllowed,
-    bool IdentifierRequired,
-    std::optional<llvm::function_ref<void()>> CodeCompletionHandler) {
+    bool IdentifierRequired, llvm::function_ref<void()> CodeCompletionHandler) {
   if ((AttrReqs & AR_CXX11AttributesParsed) &&
       isAllowedCXX11AttributeSpecifier()) {
     ParsedAttributes Attrs(AttrFactory);
@@ -6145,7 +6135,7 @@ void Parser::ParseTypeQualifierListOpt(
     case tok::code_completion:
       cutOffParsing();
       if (CodeCompletionHandler)
-        (*CodeCompletionHandler)();
+        CodeCompletionHandler();
       else
         Actions.CodeCompletion().CodeCompleteTypeQualifiers(DS);
       return;
@@ -6253,8 +6243,9 @@ void Parser::ParseTypeQualifierListOpt(
 
     // Objective-C 'kindof' types.
     case tok::kw___kindof:
-      DS.getAttributes().addNew(Tok.getIdentifierInfo(), Loc, nullptr, Loc,
-                                nullptr, 0, tok::kw___kindof);
+      DS.getAttributes().addNew(Tok.getIdentifierInfo(), Loc,
+                                AttributeScopeInfo(), nullptr, 0,
+                                tok::kw___kindof);
       (void)ConsumeToken();
       continue;
 
@@ -6891,8 +6882,8 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
       //   void (f()) requires true;
       Diag(Tok, diag::err_requires_clause_inside_parens);
       ConsumeToken();
-      ExprResult TrailingRequiresClause = Actions.CorrectDelayedTyposInExpr(
-         ParseConstraintLogicalOrExpression(/*IsTrailingRequiresClause=*/true));
+      ExprResult TrailingRequiresClause =
+          ParseConstraintLogicalOrExpression(/*IsTrailingRequiresClause=*/true);
       if (TrailingRequiresClause.isUsable() && D.isFunctionDeclarator() &&
           !D.hasTrailingRequiresClause())
         // We're already ill-formed if we got here but we'll accept it anyway.
@@ -7236,9 +7227,9 @@ void Parser::ParseFunctionDeclarator(Declarator &D,
       ParseTypeQualifierListOpt(
           DS, AR_NoAttributesParsed,
           /*AtomicOrPtrauthAllowed=*/false,
-          /*IdentifierRequired=*/false, llvm::function_ref<void()>([&]() {
+          /*IdentifierRequired=*/false, [&]() {
             Actions.CodeCompletion().CodeCompleteFunctionQualifiers(DS, D);
-          }));
+          });
       if (!DS.getSourceRange().getEnd().isInvalid()) {
         EndLoc = DS.getSourceRange().getEnd();
       }
@@ -7539,8 +7530,7 @@ void Parser::ParseParameterDeclarationClause(
       Diag(Tok,
            diag::err_requires_clause_on_declarator_not_declaring_a_function);
       ConsumeToken();
-      Actions.CorrectDelayedTyposInExpr(
-         ParseConstraintLogicalOrExpression(/*IsTrailingRequiresClause=*/true));
+      ParseConstraintLogicalOrExpression(/*IsTrailingRequiresClause=*/true);
     }
 
     // Remember this parsed parameter in ParamInfo.
@@ -7654,7 +7644,6 @@ void Parser::ParseParameterDeclarationClause(
             }
             DefArgResult = ParseAssignmentExpression();
           }
-          DefArgResult = Actions.CorrectDelayedTyposInExpr(DefArgResult);
           if (DefArgResult.isInvalid()) {
             Actions.ActOnParamDefaultArgumentError(Param, EqualLoc,
                                                    /*DefaultArg=*/nullptr);
@@ -7800,8 +7789,7 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
     } else {
       EnterExpressionEvaluationContext Unevaluated(
           Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
-      NumElements =
-          Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+      NumElements = ParseAssignmentExpression();
     }
   } else {
     if (StaticLoc.isValid()) {
@@ -7938,8 +7926,8 @@ void Parser::ParseTypeofSpecifier(DeclSpec &DS) {
   bool isCastExpr;
   ParsedType CastTy;
   SourceRange CastRange;
-  ExprResult Operand = Actions.CorrectDelayedTyposInExpr(
-      ParseExprAfterUnaryExprOrTypeTrait(OpTok, isCastExpr, CastTy, CastRange));
+  ExprResult Operand =
+      ParseExprAfterUnaryExprOrTypeTrait(OpTok, isCastExpr, CastTy, CastRange);
   if (HasParens)
     DS.setTypeArgumentRange(CastRange);
 
diff --git a/external/llvm-project/clang/lib/Parse/ParseDeclCXX.cpp b/external/llvm-project/clang/lib/Parse/ParseDeclCXX.cpp
index 2cf33a856c4f..a5c76501c7c1 100644
--- a/external/llvm-project/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseDeclCXX.cpp
@@ -760,6 +760,10 @@ Parser::DeclGroupPtrTy Parser::ParseUsingDeclaration(
 
     Decl *AD = ParseAliasDeclarationAfterDeclarator(
         TemplateInfo, UsingLoc, D, DeclEnd, AS, Attrs, &DeclFromDeclSpec);
+
+    if (!AD)
+      return nullptr;
+
     return Actions.ConvertDeclToDeclGroup(AD, DeclFromDeclSpec);
   }
 
@@ -1071,10 +1075,7 @@ SourceLocation Parser::ParseDecltypeSpecifier(DeclSpec &DS) {
       EnterExpressionEvaluationContext Unevaluated(
           Actions, Sema::ExpressionEvaluationContext::Unevaluated, nullptr,
           Sema::ExpressionEvaluationContextRecord::EK_Decltype);
-      Result = Actions.CorrectDelayedTyposInExpr(
-          ParseExpression(), /*InitDecl=*/nullptr,
-          /*RecoverUncorrectedTypos=*/false,
-          [](Expr *E) { return E->hasPlaceholderType() ? ExprError() : E; });
+      Result = ParseExpression();
       if (Result.isInvalid()) {
         DS.SetTypeSpecError();
         if (SkipUntil(tok::r_paren, StopAtSemi | StopBeforeMatch)) {
@@ -1433,7 +1434,7 @@ void Parser::ParseMicrosoftInheritanceClassAttributes(ParsedAttributes &attrs) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
     auto Kind = Tok.getKind();
     SourceLocation AttrNameLoc = ConsumeToken();
-    attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0, Kind);
+    attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0, Kind);
   }
 }
 
@@ -1442,7 +1443,7 @@ void Parser::ParseNullabilityClassAttributes(ParsedAttributes &attrs) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
     auto Kind = Tok.getKind();
     SourceLocation AttrNameLoc = ConsumeToken();
-    attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0, Kind);
+    attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0, Kind);
   }
 }
 
@@ -2649,9 +2650,9 @@ void Parser::MaybeParseAndDiagnoseDeclSpecAfterCXX11VirtSpecifierSeq(
   // handled by the caller.  Diagnose everything else.
   ParseTypeQualifierListOpt(
       DS, AR_NoAttributesParsed, /*AtomicOrPtrauthAllowed=*/false,
-      /*IdentifierRequired=*/false, llvm::function_ref<void()>([&]() {
+      /*IdentifierRequired=*/false, [&]() {
         Actions.CodeCompletion().CodeCompleteFunctionQualifiers(DS, D, &VS);
-      }));
+      });
   D.ExtendWithDeclSpec(DS);
 
   if (D.isFunctionDeclarator()) {
@@ -4465,8 +4466,7 @@ bool Parser::ParseCXXAssumeAttributeArg(
       Actions, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
 
   TentativeParsingAction TPA(*this);
-  ExprResult Res(
-      Actions.CorrectDelayedTyposInExpr(ParseConditionalExpression()));
+  ExprResult Res = ParseConditionalExpression();
   if (Res.isInvalid()) {
     TPA.Commit();
     SkipUntil(tok::r_paren, tok::r_square, StopAtSemi | StopBeforeMatch);
@@ -4497,8 +4497,8 @@ bool Parser::ParseCXXAssumeAttributeArg(
   ArgsUnion Assumption = Res.get();
   auto RParen = Tok.getLocation();
   T.consumeClose();
-  Attrs.addNew(AttrName, SourceRange(AttrNameLoc, RParen), ScopeName, ScopeLoc,
-               &Assumption, 1, Form);
+  Attrs.addNew(AttrName, SourceRange(AttrNameLoc, RParen),
+               AttributeScopeInfo(ScopeName, ScopeLoc), &Assumption, 1, Form);
 
   if (EndLoc)
     *EndLoc = RParen;
@@ -4578,7 +4578,7 @@ bool Parser::ParseCXX11AttributeArgs(
 
     // Ignore attributes that don't exist for the target.
     if (!Attr.existsInTarget(getTargetInfo())) {
-      Diag(LParenLoc, diag::warn_unknown_attribute_ignored) << AttrName;
+      Actions.DiagnoseUnknownAttribute(Attr);
       Attr.setInvalid(true);
       return true;
     }
@@ -4633,7 +4633,7 @@ void Parser::ParseCXX11AttributeSpecifierInternal(ParsedAttributes &Attrs,
                                  /*ScopeName*/ nullptr,
                                  /*ScopeLoc*/ Loc, Form);
     } else
-      Attrs.addNew(AttrName, Loc, nullptr, Loc, nullptr, 0, Form);
+      Attrs.addNew(AttrName, Loc, AttributeScopeInfo(), nullptr, 0, Form);
     return;
   }
 
@@ -4728,12 +4728,15 @@ void Parser::ParseCXX11AttributeSpecifierInternal(ParsedAttributes &Attrs,
                                            ScopeName, ScopeLoc, OpenMPTokens);
 
     if (!AttrParsed) {
-      Attrs.addNew(
-          AttrName,
-          SourceRange(ScopeLoc.isValid() ? ScopeLoc : AttrLoc, AttrLoc),
-          ScopeName, ScopeLoc, nullptr, 0,
-          getLangOpts().CPlusPlus ? ParsedAttr::Form::CXX11()
-                                  : ParsedAttr::Form::C23());
+      Attrs.addNew(AttrName,
+                   SourceRange(ScopeLoc.isValid() && CommonScopeLoc.isInvalid()
+                                   ? ScopeLoc
+                                   : AttrLoc,
+                               AttrLoc),
+                   AttributeScopeInfo(ScopeName, ScopeLoc, CommonScopeLoc),
+                   nullptr, 0,
+                   getLangOpts().CPlusPlus ? ParsedAttr::Form::CXX11()
+                                           : ParsedAttr::Form::C23());
       AttrParsed = true;
     }
 
@@ -4894,8 +4897,8 @@ void Parser::ParseMicrosoftUuidAttributeArgs(ParsedAttributes &Attrs) {
   }
 
   if (!T.consumeClose()) {
-    Attrs.addNew(UuidIdent, SourceRange(UuidLoc, T.getCloseLocation()), nullptr,
-                 SourceLocation(), ArgExprs.data(), ArgExprs.size(),
+    Attrs.addNew(UuidIdent, SourceRange(UuidLoc, T.getCloseLocation()),
+                 AttributeScopeInfo(), ArgExprs.data(), ArgExprs.size(),
                  ParsedAttr::Form::Microsoft());
   }
 }
@@ -4979,8 +4982,8 @@ void Parser::ParseMicrosoftRootSignatureAttributeArgs(ParsedAttributes &Attrs) {
 
   if (!T.consumeClose())
     Attrs.addNew(RootSignatureIdent,
-                 SourceRange(RootSignatureLoc, T.getCloseLocation()), nullptr,
-                 SourceLocation(), Args.data(), Args.size(),
+                 SourceRange(RootSignatureLoc, T.getCloseLocation()),
+                 AttributeScopeInfo(), Args.data(), Args.size(),
                  ParsedAttr::Form::Microsoft());
 }
 
@@ -5030,7 +5033,7 @@ void Parser::ParseMicrosoftAttributes(ParsedAttributes &Attrs) {
             ReplayOpenMPAttributeTokens(OpenMPTokens);
           }
           if (!AttrParsed) {
-            Attrs.addNew(II, NameLoc, nullptr, SourceLocation(), nullptr, 0,
+            Attrs.addNew(II, NameLoc, AttributeScopeInfo(), nullptr, 0,
                          ParsedAttr::Form::Microsoft());
           }
         }
diff --git a/external/llvm-project/clang/lib/Parse/ParseExpr.cpp b/external/llvm-project/clang/lib/Parse/ParseExpr.cpp
index 951a157305dd..3cf3d4ea7d70 100644
--- a/external/llvm-project/clang/lib/Parse/ParseExpr.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseExpr.cpp
@@ -183,7 +183,6 @@ ExprResult Parser::ParseConstraintExpression() {
   ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr));
   ExprResult Res(ParseRHSOfBinaryExpression(LHS, prec::LogicalOr));
   if (Res.isUsable() && !Actions.CheckConstraintExpression(Res.get())) {
-    Actions.CorrectDelayedTyposInExpr(Res);
     return ExprError();
   }
   return Res;
@@ -244,7 +243,6 @@ Parser::ParseConstraintLogicalAndExpression(bool IsTrailingRequiresClause) {
       // the rest of the addition expression). Try to parse the rest of it here.
       if (PossibleNonPrimary)
         E = RecoverFromNonPrimary(E, /*Note=*/!IsConstraintExpr);
-      Actions.CorrectDelayedTyposInExpr(E);
       return ExprError();
     }
     return E;
@@ -256,14 +254,11 @@ Parser::ParseConstraintLogicalAndExpression(bool IsTrailingRequiresClause) {
     SourceLocation LogicalAndLoc = ConsumeToken();
     ExprResult RHS = ParsePrimary();
     if (RHS.isInvalid()) {
-      Actions.CorrectDelayedTyposInExpr(LHS);
       return ExprError();
     }
     ExprResult Op = Actions.ActOnBinOp(getCurScope(), LogicalAndLoc,
                                        tok::ampamp, LHS.get(), RHS.get());
     if (!Op.isUsable()) {
-      Actions.CorrectDelayedTyposInExpr(RHS);
-      Actions.CorrectDelayedTyposInExpr(LHS);
       return ExprError();
     }
     LHS = Op;
@@ -281,14 +276,11 @@ Parser::ParseConstraintLogicalOrExpression(bool IsTrailingRequiresClause) {
     ExprResult RHS =
         ParseConstraintLogicalAndExpression(IsTrailingRequiresClause);
     if (!RHS.isUsable()) {
-      Actions.CorrectDelayedTyposInExpr(LHS);
       return ExprError();
     }
     ExprResult Op = Actions.ActOnBinOp(getCurScope(), LogicalOrLoc,
                                        tok::pipepipe, LHS.get(), RHS.get());
     if (!Op.isUsable()) {
-      Actions.CorrectDelayedTyposInExpr(RHS);
-      Actions.CorrectDelayedTyposInExpr(LHS);
       return ExprError();
     }
     LHS = Op;
@@ -408,7 +400,6 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
       }
 
       if (TernaryMiddle.isInvalid()) {
-        Actions.CorrectDelayedTyposInExpr(LHS);
         LHS = ExprError();
         TernaryMiddle = nullptr;
       }
@@ -466,11 +457,6 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
       RHS = ParseCastExpression(CastParseKind::AnyCastExpr);
 
     if (RHS.isInvalid()) {
-      // FIXME: Errors generated by the delayed typo correction should be
-      // printed before errors from parsing the RHS, not after.
-      Actions.CorrectDelayedTyposInExpr(LHS);
-      if (TernaryMiddle.isUsable())
-        TernaryMiddle = Actions.CorrectDelayedTyposInExpr(TernaryMiddle);
       LHS = ExprError();
     }
 
@@ -503,11 +489,6 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
       RHSIsInitList = false;
 
       if (RHS.isInvalid()) {
-        // FIXME: Errors generated by the delayed typo correction should be
-        // printed before errors from ParseRHSOfBinaryExpression, not after.
-        Actions.CorrectDelayedTyposInExpr(LHS);
-        if (TernaryMiddle.isUsable())
-          TernaryMiddle = Actions.CorrectDelayedTyposInExpr(TernaryMiddle);
         LHS = ExprError();
       }
 
@@ -532,7 +513,6 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
       }
     }
 
-    ExprResult OrigLHS = LHS;
     if (!LHS.isInvalid()) {
       // Combine the LHS and RHS into the LHS (e.g. build AST).
       if (TernaryMiddle.isInvalid()) {
@@ -571,17 +551,6 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
 
         LHS = CondOp;
       }
-      // In this case, ActOnBinOp or ActOnConditionalOp performed the
-      // CorrectDelayedTyposInExpr check.
-      if (!getLangOpts().CPlusPlus)
-        continue;
-    }
-
-    // Ensure potential typos aren't left undiagnosed.
-    if (LHS.isInvalid()) {
-      Actions.CorrectDelayedTyposInExpr(OrigLHS);
-      Actions.CorrectDelayedTyposInExpr(TernaryMiddle);
-      Actions.CorrectDelayedTyposInExpr(RHS);
     }
   }
 }
@@ -1711,7 +1680,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       // Reject array indices starting with a lambda-expression. '[[' is
       // reserved for attributes.
       if (CheckProhibitedCXX11Attribute()) {
-        (void)Actions.CorrectDelayedTyposInExpr(LHS);
         return ExprError();
       }
       BalancedDelimiterTracker T(*this, tok::l_square);
@@ -1737,8 +1705,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
           } else {
             Idx = ParseExpression(); // May be a comma expression
           }
-          LHS = Actions.CorrectDelayedTyposInExpr(LHS);
-          Idx = Actions.CorrectDelayedTyposInExpr(Idx);
           if (Idx.isInvalid()) {
             HasError = true;
           } else {
@@ -1746,7 +1712,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
           }
         } else if (Tok.isNot(tok::r_square)) {
           if (ParseExpressionList(ArgExprs)) {
-            LHS = Actions.CorrectDelayedTyposInExpr(LHS);
             HasError = true;
           }
         }
@@ -1762,7 +1727,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
           // Consume ':'
           ColonLocFirst = ConsumeToken();
           if (Tok.isNot(tok::r_square))
-            Length = Actions.CorrectDelayedTyposInExpr(ParseExpression());
+            Length = ParseExpression();
         }
       } else if (ArgExprs.size() <= 1 && getLangOpts().OpenMP) {
         ColonProtectionRAIIObject RAII(*this);
@@ -1773,7 +1738,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
               (getLangOpts().OpenMP < 50 ||
                ((Tok.isNot(tok::colon) && getLangOpts().OpenMP >= 50)))) {
             Length = ParseExpression();
-            Length = Actions.CorrectDelayedTyposInExpr(Length);
           }
         }
         if (getLangOpts().OpenMP >= 50 &&
@@ -1789,8 +1753,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       }
 
       SourceLocation RLoc = Tok.getLocation();
-      LHS = Actions.CorrectDelayedTyposInExpr(LHS);
-
       if (!LHS.isInvalid() && !HasError && !Length.isInvalid() &&
           !Stride.isInvalid() && Tok.is(tok::r_square)) {
         if (ColonLocFirst.isValid() || ColonLocSecond.isValid()) {
@@ -1838,7 +1800,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         SourceLocation OpenLoc = ConsumeToken();
 
         if (ParseSimpleExpressionList(ExecConfigExprs)) {
-          (void)Actions.CorrectDelayedTyposInExpr(LHS);
           LHS = ExprError();
         }
 
@@ -1889,16 +1850,12 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
                  PreferredType.enterFunctionArgument(Tok.getLocation(),
                                                      RunSignatureHelp);
                }))) {
-            (void)Actions.CorrectDelayedTyposInExpr(LHS);
             // If we got an error when parsing expression list, we don't call
             // the CodeCompleteCall handler inside the parser. So call it here
             // to make sure we get overload suggestions even when we are in the
             // middle of a parameter.
             if (PP.isCodeCompletionReached() && !CalledSignatureHelp)
               RunSignatureHelp();
-          } else if (LHS.isInvalid()) {
-            for (auto &E : ArgExprs)
-              Actions.CorrectDelayedTyposInExpr(E);
           }
         }
       }
@@ -1913,16 +1870,16 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
                                          ArgExprs);
         SkipUntil(tok::r_paren, StopAtSemi);
       } else if (Tok.isNot(tok::r_paren)) {
-        bool HadDelayedTypo = false;
-        if (Actions.CorrectDelayedTyposInExpr(LHS).get() != LHS.get())
-          HadDelayedTypo = true;
+        bool HadErrors = false;
+        if (LHS.get()->containsErrors())
+          HadErrors = true;
         for (auto &E : ArgExprs)
-          if (Actions.CorrectDelayedTyposInExpr(E).get() != E)
-            HadDelayedTypo = true;
-        // If there were delayed typos in the LHS or ArgExprs, call SkipUntil
-        // instead of PT.consumeClose() to avoid emitting extra diagnostics for
-        // the unmatched l_paren.
-        if (HadDelayedTypo)
+          if (E->containsErrors())
+            HadErrors = true;
+        // If there were errors in the LHS or ArgExprs, call SkipUntil instead
+        // of PT.consumeClose() to avoid emitting extra diagnostics for the
+        // unmatched l_paren.
+        if (HadErrors)
           SkipUntil(tok::r_paren, StopAtSemi);
         else
           PT.consumeClose();
@@ -2050,7 +2007,6 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
                      /*AllowConstructorName=*/
                      getLangOpts().MicrosoftExt && SS.isNotEmpty(),
                      /*AllowDeductionGuide=*/false, &TemplateKWLoc, Name)) {
-        (void)Actions.CorrectDelayedTyposInExpr(LHS);
         LHS = ExprError();
       }
 
@@ -2921,8 +2877,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     do {
       BalancedDelimiterTracker TS(*this, tok::l_square);
       TS.consumeOpen();
-      ExprResult NumElements =
-          Actions.CorrectDelayedTyposInExpr(ParseExpression());
+      ExprResult NumElements = ParseExpression();
       if (!NumElements.isUsable()) {
         ErrorFound = true;
         while (!SkipUntil(tok::r_square, tok::r_paren,
@@ -2936,7 +2891,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     // Match the ')'.
     T.consumeClose();
     RParenLoc = T.getCloseLocation();
-    Result = Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+    Result = ParseAssignmentExpression();
     if (ErrorFound) {
       Result = ExprError();
     } else if (!Result.isInvalid()) {
@@ -2948,12 +2903,6 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     InMessageExpressionRAIIObject InMessage(*this, false);
 
     Result = ParseExpression(TypeCastState::MaybeTypeCast);
-    if (!getLangOpts().CPlusPlus && Result.isUsable()) {
-      // Correct typos in non-C++ code earlier so that implicit-cast-like
-      // expressions are parsed correctly.
-      Result = Actions.CorrectDelayedTyposInExpr(Result);
-    }
-
     if (ExprType >= ParenParseOption::FoldExpr &&
         isFoldOperator(Tok.getKind()) && NextToken().is(tok::ellipsis)) {
       ExprType = ParenParseOption::FoldExpr;
@@ -3057,8 +3006,7 @@ ExprResult Parser::ParseGenericSelectionExpression() {
     // not evaluated."
     EnterExpressionEvaluationContext Unevaluated(
         Actions, Sema::ExpressionEvaluationContext::Unevaluated);
-    ControllingExpr =
-        Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+    ControllingExpr = ParseAssignmentExpression();
     if (ControllingExpr.isInvalid()) {
       SkipUntil(tok::r_paren, StopAtSemi);
       return ExprError();
@@ -3104,8 +3052,7 @@ ExprResult Parser::ParseGenericSelectionExpression() {
 
     // FIXME: These expressions should be parsed in a potentially potentially
     // evaluated context.
-    ExprResult ER(
-        Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression()));
+    ExprResult ER = ParseAssignmentExpression();
     if (ER.isInvalid()) {
       SkipUntil(tok::r_paren, StopAtSemi);
       return ExprError();
@@ -3199,8 +3146,7 @@ void Parser::injectEmbedTokens() {
 
 bool Parser::ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
                                  llvm::function_ref<void()> ExpressionStarts,
-                                 bool FailImmediatelyOnInvalidExpr,
-                                 bool EarlyTypoCorrection) {
+                                 bool FailImmediatelyOnInvalidExpr) {
   bool SawError = false;
   while (true) {
     if (ExpressionStarts)
@@ -3213,9 +3159,6 @@ bool Parser::ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
     } else
       Expr = ParseAssignmentExpression();
 
-    if (EarlyTypoCorrection)
-      Expr = Actions.CorrectDelayedTyposInExpr(Expr);
-
     if (Tok.is(tok::ellipsis))
       Expr = Actions.ActOnPackExpansion(Expr.get(), ConsumeToken());
     else if (Tok.is(tok::code_completion)) {
@@ -3244,14 +3187,6 @@ bool Parser::ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
     ConsumeToken();
     checkPotentialAngleBracketDelimiter(Comma);
   }
-  if (SawError) {
-    // Ensure typos get diagnosed when errors were encountered while parsing the
-    // expression list.
-    for (auto &E : Exprs) {
-      ExprResult Expr = Actions.CorrectDelayedTyposInExpr(E);
-      if (Expr.isUsable()) E = Expr.get();
-    }
-  }
   return SawError;
 }
 
diff --git a/external/llvm-project/clang/lib/Parse/ParseExprCXX.cpp b/external/llvm-project/clang/lib/Parse/ParseExprCXX.cpp
index d95260829e4a..1ea0cf52933f 100644
--- a/external/llvm-project/clang/lib/Parse/ParseExprCXX.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseExprCXX.cpp
@@ -421,8 +421,8 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
       // like we never saw it.
       Token Identifier = Tok; // Stash away the identifier.
       ConsumeToken();         // Eat the identifier, current token is now '::'.
-      Diag(PP.getLocForEndOfToken(ConsumeToken()), diag::err_expected)
-          << tok::identifier;
+      ConsumeToken();
+      Diag(getEndOfPreviousToken(), diag::err_expected) << tok::identifier;
       UnconsumeToken(Identifier); // Stick the identifier back.
       Next = NextToken();         // Point Next at the '{' token.
     }
@@ -972,8 +972,6 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
           SourceLocation StartLoc = Tok.getLocation();
           InMessageExpressionRAIIObject MaybeInMessageExpression(*this, true);
           Init = ParseInitializer();
-          if (!Init.isInvalid())
-            Init = Actions.CorrectDelayedTyposInExpr(Init.get());
 
           if (Tok.getLocation() != StartLoc) {
             // Back out the lexing of the token after the initializer.
@@ -1065,8 +1063,6 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
     // enclosing the lambda-expression, rather than in the context of the
     // lambda-expression itself.
     ParsedType InitCaptureType;
-    if (Init.isUsable())
-      Init = Actions.CorrectDelayedTyposInExpr(Init.get());
     if (Init.isUsable()) {
       NonTentativeAction([&] {
         // Get the pointer and store it in an lvalue, so we can use it as an
@@ -1238,8 +1234,8 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
       if (Tok.is(tok::kw___noinline__)) {
         IdentifierInfo *AttrName = Tok.getIdentifierInfo();
         SourceLocation AttrNameLoc = ConsumeToken();
-        Attributes.addNew(AttrName, AttrNameLoc, /*ScopeName=*/nullptr,
-                          AttrNameLoc, /*ArgsUnion=*/nullptr,
+        Attributes.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(),
+                          /*ArgsUnion=*/nullptr,
                           /*numArgs=*/0, tok::kw___noinline__);
       } else if (Tok.is(tok::kw___attribute))
         ParseGNUAttributes(Attributes, /*LatePArsedAttrList=*/nullptr, &D);
@@ -3202,8 +3198,7 @@ ExprResult Parser::ParseRequiresExpression() {
         //             cv-qualifier-seq[opt] abstract-declarator[opt]
         BalancedDelimiterTracker ExprBraces(*this, tok::l_brace);
         ExprBraces.consumeOpen();
-        ExprResult Expression =
-            Actions.CorrectDelayedTyposInExpr(ParseExpression());
+        ExprResult Expression = ParseExpression();
         if (!Expression.isUsable()) {
           ExprBraces.skipToEnd();
           SkipUntil(tok::semi, tok::r_brace, SkipUntilFlags::StopBeforeMatch);
@@ -3306,8 +3301,7 @@ ExprResult Parser::ParseRequiresExpression() {
             // C++ [expr.prim.req.nested]
             //     nested-requirement:
             //         'requires' constraint-expression ';'
-            ExprResult ConstraintExpr =
-                Actions.CorrectDelayedTyposInExpr(ParseConstraintExpression());
+            ExprResult ConstraintExpr = ParseConstraintExpression();
             if (ConstraintExpr.isInvalid() || !ConstraintExpr.isUsable()) {
               SkipUntil(tok::semi, tok::r_brace,
                         SkipUntilFlags::StopBeforeMatch);
@@ -3373,8 +3367,7 @@ ExprResult Parser::ParseRequiresExpression() {
         //     simple-requirement:
         //         expression ';'
         SourceLocation StartLoc = Tok.getLocation();
-        ExprResult Expression =
-            Actions.CorrectDelayedTyposInExpr(ParseExpression());
+        ExprResult Expression = ParseExpression();
         if (!Expression.isUsable()) {
           SkipUntil(tok::semi, tok::r_brace, SkipUntilFlags::StopBeforeMatch);
           break;
diff --git a/external/llvm-project/clang/lib/Parse/ParseHLSL.cpp b/external/llvm-project/clang/lib/Parse/ParseHLSL.cpp
index 5569605c287b..e6caa81b309c 100644
--- a/external/llvm-project/clang/lib/Parse/ParseHLSL.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseHLSL.cpp
@@ -289,12 +289,13 @@ void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
   case ParsedAttr::AT_HLSLSV_GroupID:
   case ParsedAttr::AT_HLSLSV_GroupIndex:
   case ParsedAttr::AT_HLSLSV_DispatchThreadID:
+  case ParsedAttr::AT_HLSLSV_Position:
     break;
   default:
     llvm_unreachable("invalid HLSL Annotation");
     break;
   }
 
-  Attrs.addNew(II, Loc, nullptr, SourceLocation(), ArgExprs.data(),
-               ArgExprs.size(), ParsedAttr::Form::HLSLAnnotation());
+  Attrs.addNew(II, Loc, AttributeScopeInfo(), ArgExprs.data(), ArgExprs.size(),
+               ParsedAttr::Form::HLSLAnnotation());
 }
diff --git a/external/llvm-project/clang/lib/Parse/ParseHLSLRootSignature.cpp b/external/llvm-project/clang/lib/Parse/ParseHLSLRootSignature.cpp
index 39a28edee8e2..e510302c3aae 100644
--- a/external/llvm-project/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -377,6 +377,9 @@ std::optional<StaticSampler> RootSignatureParser::parseStaticSampler() {
   Sampler.Reg = Params->Reg.value();
 
   // Fill in optional values
+  if (Params->Filter.has_value())
+    Sampler.Filter = Params->Filter.value();
+
   if (Params->AddressU.has_value())
     Sampler.AddressU = Params->AddressU.value();
 
@@ -696,6 +699,23 @@ RootSignatureParser::parseStaticSamplerParams() {
       Params.Reg = Reg;
     }
 
+    // `filter` `=` FILTER
+    if (tryConsumeExpectedToken(TokenKind::kw_filter)) {
+      if (Params.Filter.has_value()) {
+        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
+            << CurToken.TokKind;
+        return std::nullopt;
+      }
+
+      if (consumeExpectedToken(TokenKind::pu_equal))
+        return std::nullopt;
+
+      auto Filter = parseSamplerFilter();
+      if (!Filter.has_value())
+        return std::nullopt;
+      Params.Filter = Filter;
+    }
+
     // `addressU` `=` TEXTURE_ADDRESS
     if (tryConsumeExpectedToken(TokenKind::kw_addressU)) {
       if (Params.AddressU.has_value()) {
@@ -989,6 +1009,32 @@ RootSignatureParser::parseShaderVisibility() {
   return std::nullopt;
 }
 
+std::optional<llvm::hlsl::rootsig::SamplerFilter>
+RootSignatureParser::parseSamplerFilter() {
+  assert(CurToken.TokKind == TokenKind::pu_equal &&
+         "Expects to only be invoked starting at given keyword");
+
+  TokenKind Expected[] = {
+#define FILTER_ENUM(NAME, LIT) TokenKind::en_##NAME,
+#include "clang/Lex/HLSLRootSignatureTokenKinds.def"
+  };
+
+  if (!tryConsumeExpectedToken(Expected))
+    return std::nullopt;
+
+  switch (CurToken.TokKind) {
+#define FILTER_ENUM(NAME, LIT)                                                 \
+  case TokenKind::en_##NAME:                                                   \
+    return SamplerFilter::NAME;                                                \
+    break;
+#include "clang/Lex/HLSLRootSignatureTokenKinds.def"
+  default:
+    llvm_unreachable("Switch for consumed enum token was not provided");
+  }
+
+  return std::nullopt;
+}
+
 std::optional<llvm::hlsl::rootsig::TextureAddressMode>
 RootSignatureParser::parseTextureAddressMode() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
diff --git a/external/llvm-project/clang/lib/Parse/ParseInit.cpp b/external/llvm-project/clang/lib/Parse/ParseInit.cpp
index df8372b995e5..a3be3744a932 100644
--- a/external/llvm-project/clang/lib/Parse/ParseInit.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseInit.cpp
@@ -477,8 +477,6 @@ ExprResult Parser::ParseBraceInitializer() {
     if (Tok.is(tok::ellipsis))
       SubElt = Actions.ActOnPackExpansion(SubElt.get(), ConsumeToken());
 
-    SubElt = Actions.CorrectDelayedTyposInExpr(SubElt.get());
-
     // If we couldn't parse the subelement, bail out.
     if (SubElt.isUsable()) {
       InitExprs.push_back(SubElt.get());
diff --git a/external/llvm-project/clang/lib/Parse/ParseObjc.cpp b/external/llvm-project/clang/lib/Parse/ParseObjc.cpp
index 6afb7809d3cd..291c70e7bad4 100644
--- a/external/llvm-project/clang/lib/Parse/ParseObjc.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseObjc.cpp
@@ -370,7 +370,7 @@ static void addContextSensitiveTypeNullability(Parser &P,
   // Create the attribute.
   auto getNullabilityAttr = [&](AttributePool &Pool) -> ParsedAttr * {
     return Pool.create(P.getNullabilityKeyword(nullability),
-                       SourceRange(nullabilityLoc), nullptr, SourceLocation(),
+                       SourceRange(nullabilityLoc), AttributeScopeInfo(),
                        nullptr, 0, ParsedAttr::Form::ContextSensitiveKeyword());
   };
 
@@ -2629,10 +2629,7 @@ bool Parser::ParseObjCXXMessageReceiver(bool &IsExpr, void *&TypeOrExpr) {
   if (!Tok.isSimpleTypeSpecifier(getLangOpts())) {
     //   objc-receiver:
     //     expression
-    // Make sure any typos in the receiver are corrected or diagnosed, so that
-    // proper recovery can happen. FIXME: Perhaps filter the corrected expr to
-    // only the things that are valid ObjC receivers?
-    ExprResult Receiver = Actions.CorrectDelayedTyposInExpr(ParseExpression());
+    ExprResult Receiver = ParseExpression();
     if (Receiver.isInvalid())
       return true;
 
@@ -2809,7 +2806,7 @@ ExprResult Parser::ParseObjCMessageExpression() {
   }
 
   // Otherwise, an arbitrary expression can be the receiver of a send.
-  ExprResult Res = Actions.CorrectDelayedTyposInExpr(ParseExpression());
+  ExprResult Res = ParseExpression();
   if (Res.isInvalid()) {
     SkipUntil(tok::r_square, StopAtSemi);
     return Res;
@@ -2930,8 +2927,6 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
       SourceLocation commaLoc = ConsumeToken(); // Eat the ','.
       ///  Parse the expression after ','
       ExprResult Res(ParseAssignmentExpression());
-      if (Tok.is(tok::colon))
-        Res = Actions.CorrectDelayedTyposInExpr(Res);
       if (Res.isInvalid()) {
         if (Tok.is(tok::colon)) {
           Diag(commaLoc, diag::note_extra_comma_message_arg) <<
@@ -3078,10 +3073,6 @@ ExprResult Parser::ParseObjCArrayLiteral(SourceLocation AtLoc) {
       return Res;
     }
 
-    Res = Actions.CorrectDelayedTyposInExpr(Res.get());
-    if (Res.isInvalid())
-      HasInvalidEltExpr = true;
-
     // Parse the ellipsis that indicates a pack expansion.
     if (Tok.is(tok::ellipsis))
       Res = Actions.ActOnPackExpansion(Res.get(), ConsumeToken());
@@ -3108,7 +3099,6 @@ ExprResult Parser::ParseObjCArrayLiteral(SourceLocation AtLoc) {
 ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
   SmallVector<ObjCDictionaryElement, 4> Elements; // dictionary elements.
   ConsumeBrace(); // consume the l_square.
-  bool HasInvalidEltExpr = false;
   while (Tok.isNot(tok::r_brace)) {
     // Parse the comma separated key : value expressions.
     ExprResult KeyExpr;
@@ -3138,12 +3128,6 @@ ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
       return ValueExpr;
     }
 
-    // Check the key and value for possible typos
-    KeyExpr = Actions.CorrectDelayedTyposInExpr(KeyExpr.get());
-    ValueExpr = Actions.CorrectDelayedTyposInExpr(ValueExpr.get());
-    if (KeyExpr.isInvalid() || ValueExpr.isInvalid())
-      HasInvalidEltExpr = true;
-
     // Parse the ellipsis that designates this as a pack expansion. Do not
     // ActOnPackExpansion here, leave it to template instantiation time where
     // we can get better diagnostics.
@@ -3163,9 +3147,6 @@ ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
   }
   SourceLocation EndLoc = ConsumeBrace();
 
-  if (HasInvalidEltExpr)
-    return ExprError();
-
   // Create the ObjCDictionaryLiteral.
   return Actions.ObjC().BuildObjCDictionaryLiteral(SourceRange(AtLoc, EndLoc),
                                                    Elements);
diff --git a/external/llvm-project/clang/lib/Parse/ParseOpenACC.cpp b/external/llvm-project/clang/lib/Parse/ParseOpenACC.cpp
index ca4f878464c4..f2849c4eac7c 100644
--- a/external/llvm-project/clang/lib/Parse/ParseOpenACC.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseOpenACC.cpp
@@ -653,7 +653,7 @@ ExprResult Parser::ParseOpenACCConditionExpr() {
   // it does in an if/while/etc (See ParseCXXCondition), however as it was
   // written with Fortran/C in mind, we're going to assume it just means an
   // 'expression evaluating to boolean'.
-  ExprResult ER = getActions().CorrectDelayedTyposInExpr(ParseExpression());
+  ExprResult ER = ParseExpression();
 
   if (!ER.isUsable())
     return ER;
@@ -761,12 +761,6 @@ Parser::ParseOpenACCIntExpr(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
   if (!ER.isUsable())
     return {ER, OpenACCParseCanContinue::Cannot};
 
-  // Parsing can continue after the initial assignment expression parsing, so
-  // even if there was a typo, we can continue.
-  ER = getActions().CorrectDelayedTyposInExpr(ER);
-  if (!ER.isUsable())
-    return {ER, OpenACCParseCanContinue::Can};
-
   return {getActions().OpenACC().ActOnIntExpr(DK, CK, Loc, ER.get()),
           OpenACCParseCanContinue::Can};
 }
@@ -836,8 +830,7 @@ ExprResult Parser::ParseOpenACCSizeExpr(OpenACCClauseKind CK) {
     return getActions().OpenACC().ActOnOpenACCAsteriskSizeExpr(AsteriskLoc);
   }
 
-  ExprResult SizeExpr =
-      getActions().CorrectDelayedTyposInExpr(ParseConstantExpression());
+  ExprResult SizeExpr = ParseConstantExpression();
 
   if (!SizeExpr.isUsable())
     return SizeExpr;
@@ -891,8 +884,7 @@ Parser::OpenACCGangArgRes Parser::ParseOpenACCGangArg(SourceLocation GangLoc) {
     ConsumeToken();
     // Parse this as a const-expression, and we'll check its integer-ness/value
     // in CheckGangExpr.
-    ExprResult Res =
-        getActions().CorrectDelayedTyposInExpr(ParseConstantExpression());
+    ExprResult Res = ParseConstantExpression();
     return {OpenACCGangKind::Dim, Res};
   }
 
@@ -1089,8 +1081,7 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
     case OpenACCClauseKind::Collapse: {
       bool HasForce = tryParseAndConsumeSpecialTokenKind(
           *this, OpenACCSpecialTokenKind::Force, ClauseKind);
-      ExprResult LoopCount =
-          getActions().CorrectDelayedTyposInExpr(ParseConstantExpression());
+      ExprResult LoopCount = ParseConstantExpression();
       if (LoopCount.isInvalid()) {
         Parens.skipToEnd();
         return OpenACCCanContinue();
@@ -1387,7 +1378,7 @@ ExprResult Parser::ParseOpenACCIDExpression() {
                                     /*isAddressOfOperand=*/false);
   }
 
-  return getActions().CorrectDelayedTyposInExpr(Res);
+  return Res;
 }
 
 std::variant<std::monostate, clang::StringLiteral *, IdentifierInfo *>
@@ -1414,9 +1405,8 @@ Parser::ParseOpenACCBindClauseArgument() {
     return std::monostate{};
   }
 
-  ExprResult Res =
-      getActions().CorrectDelayedTyposInExpr(ParseStringLiteralExpression(
-          /*AllowUserDefinedLiteral=*/false, /*Unevaluated=*/true));
+  ExprResult Res = ParseStringLiteralExpression(
+      /*AllowUserDefinedLiteral=*/false, /*Unevaluated=*/true);
   if (!Res.isUsable())
     return std::monostate{};
   return cast<StringLiteral>(Res.get());
@@ -1430,10 +1420,6 @@ Parser::OpenACCVarParseResult Parser::ParseOpenACCVar(OpenACCDirectiveKind DK,
   if (!Res.isUsable())
     return {Res, OpenACCParseCanContinue::Cannot};
 
-  Res = getActions().CorrectDelayedTyposInExpr(Res.get());
-  if (!Res.isUsable())
-    return {Res, OpenACCParseCanContinue::Can};
-
   Res = getActions().OpenACC().ActOnVar(DK, CK, Res.get());
 
   return {Res, OpenACCParseCanContinue::Can};
diff --git a/external/llvm-project/clang/lib/Parse/ParseOpenMP.cpp b/external/llvm-project/clang/lib/Parse/ParseOpenMP.cpp
index e41e5ba8596b..78d3503d8eb6 100644
--- a/external/llvm-project/clang/lib/Parse/ParseOpenMP.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseOpenMP.cpp
@@ -576,6 +576,7 @@ Parser::ParseOpenMPDeclareMapperDirective(AccessSpecifier AS) {
     return DeclGroupPtrTy();
   }
 
+  Scope *OuterScope = getCurScope();
   // Enter scope.
   DeclarationNameInfo DirName;
   SourceLocation Loc = Tok.getLocation();
@@ -614,12 +615,17 @@ Parser::ParseOpenMPDeclareMapperDirective(AccessSpecifier AS) {
     IsCorrect = false;
   }
 
+  // This needs to be called within the scope because
+  // processImplicitMapsWithDefaultMappers may add clauses when analyzing nested
+  // types. The scope used for calling ActOnOpenMPDeclareMapperDirective,
+  // however, needs to be the outer one, otherwise declared mappers don't become
+  // visible.
+  DeclGroupPtrTy DG = Actions.OpenMP().ActOnOpenMPDeclareMapperDirective(
+      OuterScope, Actions.getCurLexicalContext(), MapperId, MapperType,
+      Range.getBegin(), VName, AS, MapperVarRef.get(), Clauses);
   // Exit scope.
   Actions.OpenMP().EndOpenMPDSABlock(nullptr);
   OMPDirectiveScope.Exit();
-  DeclGroupPtrTy DG = Actions.OpenMP().ActOnOpenMPDeclareMapperDirective(
-      getCurScope(), Actions.getCurLexicalContext(), MapperId, MapperType,
-      Range.getBegin(), VName, AS, MapperVarRef.get(), Clauses);
   if (!IsCorrect)
     return DeclGroupPtrTy();
 
@@ -1483,6 +1489,7 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
   OMPTraitInfo &TI = ASTCtx.getNewOMPTraitInfo();
   SmallVector<Expr *, 6> AdjustNothing;
   SmallVector<Expr *, 6> AdjustNeedDevicePtr;
+  SmallVector<Expr *, 6> AdjustNeedDeviceAddr;
   SmallVector<OMPInteropInfo, 3> AppendArgs;
   SourceLocation AdjustArgsLoc, AppendArgsLoc;
 
@@ -1515,11 +1522,21 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
         SmallVector<Expr *> Vars;
         IsError = ParseOpenMPVarList(OMPD_declare_variant, OMPC_adjust_args,
                                      Vars, Data);
-        if (!IsError)
-          llvm::append_range(Data.ExtraModifier == OMPC_ADJUST_ARGS_nothing
-                                 ? AdjustNothing
-                                 : AdjustNeedDevicePtr,
-                             Vars);
+        if (!IsError) {
+          switch (Data.ExtraModifier) {
+          case OMPC_ADJUST_ARGS_nothing:
+            llvm::append_range(AdjustNothing, Vars);
+            break;
+          case OMPC_ADJUST_ARGS_need_device_ptr:
+            llvm::append_range(AdjustNeedDevicePtr, Vars);
+            break;
+          case OMPC_ADJUST_ARGS_need_device_addr:
+            llvm::append_range(AdjustNeedDeviceAddr, Vars);
+            break;
+          default:
+            llvm_unreachable("Unexpected 'adjust_args' clause modifier.");
+          }
+        }
         break;
       }
       case OMPC_append_args:
@@ -1559,8 +1576,8 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
   if (DeclVarData && !TI.Sets.empty())
     Actions.OpenMP().ActOnOpenMPDeclareVariantDirective(
         DeclVarData->first, DeclVarData->second, TI, AdjustNothing,
-        AdjustNeedDevicePtr, AppendArgs, AdjustArgsLoc, AppendArgsLoc,
-        SourceRange(Loc, Tok.getLocation()));
+        AdjustNeedDevicePtr, AdjustNeedDeviceAddr, AppendArgs, AdjustArgsLoc,
+        AppendArgsLoc, SourceRange(Loc, Tok.getLocation()));
 
   // Skip the last annot_pragma_openmp_end.
   (void)ConsumeAnnotationToken();
@@ -3589,8 +3606,7 @@ bool Parser::ParseOMPInteropInfo(OMPInteropInfo &InteropInfo,
       while (Tok.isNot(tok::r_paren)) {
         SourceLocation Loc = Tok.getLocation();
         ExprResult LHS = ParseCastExpression(CastParseKind::AnyCastExpr);
-        ExprResult PTExpr = Actions.CorrectDelayedTyposInExpr(
-            ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+        ExprResult PTExpr = ParseRHSOfBinaryExpression(LHS, prec::Conditional);
         PTExpr = Actions.ActOnFinishFullExpr(PTExpr.get(), Loc,
                                              /*DiscardedValue=*/false);
         if (PTExpr.isUsable()) {
@@ -3651,8 +3667,7 @@ OMPClause *Parser::ParseOpenMPInteropClause(OpenMPClauseKind Kind,
 
   // Parse the variable.
   SourceLocation VarLoc = Tok.getLocation();
-  ExprResult InteropVarExpr =
-      Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+  ExprResult InteropVarExpr = ParseAssignmentExpression();
   if (!InteropVarExpr.isUsable()) {
     SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openmp_end,
               StopBeforeMatch);
@@ -4277,8 +4292,7 @@ ExprResult Parser::ParseOpenMPIteratorsExpr() {
     // Parse <begin>
     SourceLocation Loc = Tok.getLocation();
     ExprResult LHS = ParseCastExpression(CastParseKind::AnyCastExpr);
-    ExprResult Begin = Actions.CorrectDelayedTyposInExpr(
-        ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+    ExprResult Begin = ParseRHSOfBinaryExpression(LHS, prec::Conditional);
     Begin = Actions.ActOnFinishFullExpr(Begin.get(), Loc,
                                         /*DiscardedValue=*/false);
     // Parse ':'.
@@ -4289,8 +4303,7 @@ ExprResult Parser::ParseOpenMPIteratorsExpr() {
     // Parse <end>
     Loc = Tok.getLocation();
     LHS = ParseCastExpression(CastParseKind::AnyCastExpr);
-    ExprResult End = Actions.CorrectDelayedTyposInExpr(
-        ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+    ExprResult End = ParseRHSOfBinaryExpression(LHS, prec::Conditional);
     End = Actions.ActOnFinishFullExpr(End.get(), Loc,
                                       /*DiscardedValue=*/false);
 
@@ -4303,8 +4316,7 @@ ExprResult Parser::ParseOpenMPIteratorsExpr() {
       // Parse <step>
       Loc = Tok.getLocation();
       LHS = ParseCastExpression(CastParseKind::AnyCastExpr);
-      Step = Actions.CorrectDelayedTyposInExpr(
-          ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+      Step = ParseRHSOfBinaryExpression(LHS, prec::Conditional);
       Step = Actions.ActOnFinishFullExpr(Step.get(), Loc,
                                          /*DiscardedValue=*/false);
     }
@@ -4786,7 +4798,6 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
       EnterScope(Scope::OpenMPDirectiveScope | Scope::DeclScope);
       Tail = ParseOpenMPIteratorsExpr();
     }
-    Tail = Actions.CorrectDelayedTyposInExpr(Tail);
     Tail = Actions.ActOnFinishFullExpr(Tail.get(), T.getOpenLocation(),
                                        /*DiscardedValue=*/false);
     if (Tail.isUsable() || Data.AllocateAlignment) {
@@ -4818,7 +4829,8 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
         getLangOpts());
     Data.ExtraModifierLoc = Tok.getLocation();
     if (Data.ExtraModifier == OMPC_ADJUST_ARGS_unknown) {
-      Diag(Tok, diag::err_omp_unknown_adjust_args_op);
+      Diag(Tok, diag::err_omp_unknown_adjust_args_op)
+          << (getLangOpts().OpenMP >= 60 ? 1 : 0);
       SkipUntil(tok::r_paren, tok::annot_pragma_openmp_end, StopBeforeMatch);
     } else {
       ConsumeToken();
@@ -4846,8 +4858,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     ColonProtectionRAIIObject ColonRAII(*this, MayHaveTail);
     if (!ParseOpenMPReservedLocator(Kind, Data, getLangOpts())) {
       // Parse variable
-      ExprResult VarExpr =
-          Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+      ExprResult VarExpr = ParseAssignmentExpression();
       if (VarExpr.isUsable()) {
         Vars.push_back(VarExpr.get());
       } else {
@@ -4884,6 +4895,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     SourceLocation ELoc = ConsumeToken();
 
     if (getLangOpts().OpenMP >= 52 && Kind == OMPC_linear) {
+      bool Malformed = false;
       while (Tok.isNot(tok::r_paren)) {
         if (Tok.is(tok::identifier)) {
           // identifier could be a linear kind (val, uval, ref) or step
@@ -4920,6 +4932,11 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
             ModifierFound = true;
           } else {
             StepFound = parseStepSize(*this, Data, Kind, Tok.getLocation());
+            if (!StepFound) {
+              Malformed = true;
+              SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openmp_end,
+                        StopBeforeMatch);
+            }
           }
         } else {
           // parse an integer expression as step size
@@ -4931,7 +4948,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
         if (Tok.is(tok::r_paren) || Tok.is(tok::annot_pragma_openmp_end))
           break;
       }
-      if (!StepFound && !ModifierFound)
+      if (!Malformed && !StepFound && !ModifierFound)
         Diag(ELoc, diag::err_expected_expression);
     } else {
       // for OMPC_aligned and OMPC_linear (with OpenMP <= 5.1)
diff --git a/external/llvm-project/clang/lib/Parse/ParsePragma.cpp b/external/llvm-project/clang/lib/Parse/ParsePragma.cpp
index 77b61af76899..98933811265e 100644
--- a/external/llvm-project/clang/lib/Parse/ParsePragma.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParsePragma.cpp
@@ -300,12 +300,6 @@ struct PragmaMSRuntimeChecksHandler : public EmptyPragmaHandler {
   PragmaMSRuntimeChecksHandler() : EmptyPragmaHandler("runtime_checks") {}
 };
 
-struct PragmaMSIntrinsicHandler : public PragmaHandler {
-  PragmaMSIntrinsicHandler() : PragmaHandler("intrinsic") {}
-  void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
-                    Token &FirstToken) override;
-};
-
 // "\#pragma fenv_access (on)".
 struct PragmaMSFenvAccessHandler : public PragmaHandler {
   PragmaMSFenvAccessHandler() : PragmaHandler("fenv_access") {}
@@ -517,7 +511,7 @@ void Parser::initializePragmaHandlers() {
     PP.AddPragmaHandler(MSOptimize.get());
     MSRuntimeChecks = std::make_unique<PragmaMSRuntimeChecksHandler>();
     PP.AddPragmaHandler(MSRuntimeChecks.get());
-    MSIntrinsic = std::make_unique<PragmaMSIntrinsicHandler>();
+    MSIntrinsic = std::make_unique<PragmaMSPragma>("intrinsic");
     PP.AddPragmaHandler(MSIntrinsic.get());
     MSFenvAccess = std::make_unique<PragmaMSFenvAccessHandler>();
     PP.AddPragmaHandler(MSFenvAccess.get());
@@ -1046,7 +1040,8 @@ void Parser::HandlePragmaMSPragma() {
           .Case("strict_gs_check", &Parser::HandlePragmaMSStrictGuardStackCheck)
           .Case("function", &Parser::HandlePragmaMSFunction)
           .Case("alloc_text", &Parser::HandlePragmaMSAllocText)
-          .Case("optimize", &Parser::HandlePragmaMSOptimize);
+          .Case("optimize", &Parser::HandlePragmaMSOptimize)
+          .Case("intrinsic", &Parser::HandlePragmaMSIntrinsic);
 
   if (!(this->*Handler)(PragmaName, PragmaLocation)) {
     // Pragma handling failed, and has been diagnosed.  Slurp up the tokens
@@ -1931,7 +1926,7 @@ void Parser::HandlePragmaAttribute() {
       SourceLocation AttrNameLoc = ConsumeToken();
 
       if (Tok.isNot(tok::l_paren))
-        Attrs.addNew(AttrName, AttrNameLoc, nullptr, AttrNameLoc, nullptr, 0,
+        Attrs.addNew(AttrName, AttrNameLoc, AttributeScopeInfo(), nullptr, 0,
                      ParsedAttr::Form::GNU());
       else
         ParseGNUAttributeArgs(AttrName, AttrNameLoc, Attrs, /*EndLoc=*/nullptr,
@@ -3762,56 +3757,6 @@ void PragmaUnrollHintHandler::HandlePragma(Preprocessor &PP,
                       /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
 }
 
-/// Handle the Microsoft \#pragma intrinsic extension.
-///
-/// The syntax is:
-/// \code
-///  #pragma intrinsic(memset)
-///  #pragma intrinsic(strlen, memcpy)
-/// \endcode
-///
-/// Pragma intrisic tells the compiler to use a builtin version of the
-/// function. Clang does it anyway, so the pragma doesn't really do anything.
-/// Anyway, we emit a warning if the function specified in \#pragma intrinsic
-/// isn't an intrinsic in clang and suggest to include intrin.h.
-void PragmaMSIntrinsicHandler::HandlePragma(Preprocessor &PP,
-                                            PragmaIntroducer Introducer,
-                                            Token &Tok) {
-  PP.Lex(Tok);
-
-  if (Tok.isNot(tok::l_paren)) {
-    PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen)
-        << "intrinsic";
-    return;
-  }
-  PP.Lex(Tok);
-
-  bool SuggestIntrinH = !PP.isMacroDefined("__INTRIN_H");
-
-  while (Tok.is(tok::identifier)) {
-    IdentifierInfo *II = Tok.getIdentifierInfo();
-    if (!II->getBuiltinID())
-      PP.Diag(Tok.getLocation(), diag::warn_pragma_intrinsic_builtin)
-          << II << SuggestIntrinH;
-
-    PP.Lex(Tok);
-    if (Tok.isNot(tok::comma))
-      break;
-    PP.Lex(Tok);
-  }
-
-  if (Tok.isNot(tok::r_paren)) {
-    PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen)
-        << "intrinsic";
-    return;
-  }
-  PP.Lex(Tok);
-
-  if (Tok.isNot(tok::eod))
-    PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
-        << "intrinsic";
-}
-
 bool Parser::HandlePragmaMSFunction(StringRef PragmaName,
                                     SourceLocation PragmaLocation) {
   Token FirstTok = Tok;
@@ -3907,6 +3852,56 @@ bool Parser::HandlePragmaMSOptimize(StringRef PragmaName,
   return true;
 }
 
+/// Handle the Microsoft \#pragma intrinsic extension.
+///
+/// The syntax is:
+/// \code
+///  #pragma intrinsic(memset)
+///  #pragma intrinsic(strlen, memcpy)
+/// \endcode
+///
+/// Pragma intrisic tells the compiler to use a builtin version of the
+/// function. Clang does it anyway, so the pragma doesn't really do anything.
+/// Anyway, we emit a warning if the function specified in \#pragma intrinsic
+/// isn't an intrinsic in clang and suggest to include intrin.h, as well as
+/// declare the builtin if it has not been declared.
+bool Parser::HandlePragmaMSIntrinsic(StringRef PragmaName,
+                                     SourceLocation PragmaLocation) {
+  if (ExpectAndConsume(tok::l_paren, diag::warn_pragma_expected_lparen,
+                       PragmaName))
+    return false;
+
+  bool SuggestIntrinH = !PP.isMacroDefined("__INTRIN_H");
+
+  while (Tok.is(tok::identifier)) {
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+    if (!II->getBuiltinID())
+      PP.Diag(Tok.getLocation(), diag::warn_pragma_intrinsic_builtin)
+          << II << SuggestIntrinH;
+    // If the builtin hasn't already been declared, declare it now.
+    DeclarationNameInfo NameInfo(II, Tok.getLocation());
+    LookupResult Previous(Actions, NameInfo, Sema::LookupOrdinaryName,
+                          RedeclarationKind::NotForRedeclaration);
+    Actions.LookupName(Previous, Actions.getCurScope(),
+                       /*CreateBuiltins*/ false);
+    if (Previous.empty())
+      Actions.LazilyCreateBuiltin(II, II->getBuiltinID(), Actions.getCurScope(),
+                                  /*ForRedeclaration*/ true, Tok.getLocation());
+    PP.Lex(Tok);
+    if (Tok.isNot(tok::comma))
+      break;
+    PP.Lex(Tok);
+  }
+  if (ExpectAndConsume(tok::r_paren, diag::warn_pragma_expected_rparen,
+                       PragmaName))
+    return false;
+
+  if (ExpectAndConsume(tok::eof, diag::warn_pragma_extra_tokens_at_eol,
+                       PragmaName))
+    return false;
+  return true;
+}
+
 void PragmaForceCUDAHostDeviceHandler::HandlePragma(
     Preprocessor &PP, PragmaIntroducer Introducer, Token &Tok) {
   Token FirstTok = Tok;
diff --git a/external/llvm-project/clang/lib/Parse/ParseStmt.cpp b/external/llvm-project/clang/lib/Parse/ParseStmt.cpp
index c788723023c8..bc40b726bf41 100644
--- a/external/llvm-project/clang/lib/Parse/ParseStmt.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseStmt.cpp
@@ -602,7 +602,7 @@ StmtResult Parser::ParseSEHExceptBlock(SourceLocation ExceptLoc) {
   {
     ParseScopeFlags FilterScope(this, getCurScope()->getFlags() |
                                           Scope::SEHFilterScope);
-    FilterExpr = Actions.CorrectDelayedTyposInExpr(ParseExpression());
+    FilterExpr = ParseExpression();
   }
 
   if (getLangOpts().Borland) {
@@ -832,10 +832,12 @@ StmtResult Parser::ParseCaseStatement(ParsedStmtContext StmtCtx,
           << "'case'" << tok::colon
           << FixItHint::CreateReplacement(ColonLoc, ":");
     } else {
-      SourceLocation ExpectedLoc = PP.getLocForEndOfToken(PrevTokLocation);
+      SourceLocation ExpectedLoc = getEndOfPreviousToken();
+
       Diag(ExpectedLoc, diag::err_expected_after)
           << "'case'" << tok::colon
           << FixItHint::CreateInsertion(ExpectedLoc, ":");
+
       ColonLoc = ExpectedLoc;
     }
 
@@ -1829,11 +1831,7 @@ StmtResult Parser::ParseDoStatement() {
 
   SourceLocation Start = Tok.getLocation();
   ExprResult Cond = ParseExpression();
-  // Correct the typos in condition before closing the scope.
-  if (Cond.isUsable())
-    Cond = Actions.CorrectDelayedTyposInExpr(Cond, /*InitDecl=*/nullptr,
-                                             /*RecoverUncorrectedTypos=*/true);
-  else {
+  if (!Cond.isUsable()) {
     if (!Tok.isOneOf(tok::r_paren, tok::r_square, tok::r_brace))
       SkipUntil(tok::semi);
     Cond = Actions.CreateRecoveryExpr(
@@ -2015,7 +2013,7 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
     }
   } else {
     ProhibitAttributes(attrs);
-    Value = Actions.CorrectDelayedTyposInExpr(ParseExpression());
+    Value = ParseExpression();
 
     ForEach = isTokIdentifier_in();
 
@@ -2174,12 +2172,10 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
   StmtResult ForEachStmt;
 
   if (ForRangeInfo.ParsedForRangeDecl()) {
-    ExprResult CorrectedRange =
-        Actions.CorrectDelayedTyposInExpr(ForRangeInfo.RangeExpr.get());
     ForRangeStmt = Actions.ActOnCXXForRangeStmt(
         getCurScope(), ForLoc, CoawaitLoc, FirstPart.get(),
-        ForRangeInfo.LoopVar.get(), ForRangeInfo.ColonLoc, CorrectedRange.get(),
-        T.getCloseLocation(), Sema::BFRK_Build,
+        ForRangeInfo.LoopVar.get(), ForRangeInfo.ColonLoc,
+        ForRangeInfo.RangeExpr.get(), T.getCloseLocation(), Sema::BFRK_Build,
         ForRangeInfo.LifetimeExtendTemps);
   } else if (ForEach) {
     // Similarly, we need to do the semantic analysis for a for-range
@@ -2349,8 +2345,8 @@ StmtResult Parser::ParsePragmaLoopHint(StmtVector &Stmts,
     ArgsUnion ArgHints[] = {Hint.PragmaNameLoc, Hint.OptionLoc, Hint.StateLoc,
                             ArgsUnion(Hint.ValueExpr)};
     TempAttrs.addNew(Hint.PragmaNameLoc->getIdentifierInfo(), Hint.Range,
-                     /*scopeName=*/nullptr, Hint.PragmaNameLoc->getLoc(),
-                     ArgHints, /*numArgs=*/4, ParsedAttr::Form::Pragma());
+                     AttributeScopeInfo(), ArgHints, /*numArgs=*/4,
+                     ParsedAttr::Form::Pragma());
   }
 
   // Get the next statement.
diff --git a/external/llvm-project/clang/lib/Parse/ParseStmtAsm.cpp b/external/llvm-project/clang/lib/Parse/ParseStmtAsm.cpp
index f2417479a0e7..182907df5607 100644
--- a/external/llvm-project/clang/lib/Parse/ParseStmtAsm.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseStmtAsm.cpp
@@ -864,7 +864,7 @@ bool Parser::ParseAsmOperandsOpt(SmallVectorImpl<IdentifierInfo *> &Names,
     // Read the parenthesized expression.
     BalancedDelimiterTracker T(*this, tok::l_paren);
     T.consumeOpen();
-    ExprResult Res = Actions.CorrectDelayedTyposInExpr(ParseExpression());
+    ExprResult Res = ParseExpression();
     T.consumeClose();
     if (Res.isInvalid()) {
       SkipUntil(tok::r_paren, StopAtSemi);
diff --git a/external/llvm-project/clang/lib/Parse/ParseTemplate.cpp b/external/llvm-project/clang/lib/Parse/ParseTemplate.cpp
index d3c9ca029c9a..a16dbe95b788 100644
--- a/external/llvm-project/clang/lib/Parse/ParseTemplate.cpp
+++ b/external/llvm-project/clang/lib/Parse/ParseTemplate.cpp
@@ -296,8 +296,7 @@ Parser::ParseConceptDefinition(const ParsedTemplateInfo &TemplateInfo,
     return nullptr;
   }
 
-  ExprResult ConstraintExprResult =
-      Actions.CorrectDelayedTyposInExpr(ParseConstraintExpression());
+  ExprResult ConstraintExprResult = ParseConstraintExpression();
   if (ConstraintExprResult.isInvalid()) {
     SkipUntil(tok::semi);
     if (D)
diff --git a/external/llvm-project/clang/lib/Parse/Parser.cpp b/external/llvm-project/clang/lib/Parse/Parser.cpp
index db65c05cc114..788ed79e0c1f 100644
--- a/external/llvm-project/clang/lib/Parse/Parser.cpp
+++ b/external/llvm-project/clang/lib/Parse/Parser.cpp
@@ -1873,6 +1873,11 @@ Parser::TryAnnotateName(CorrectionCandidateCallback *CCC,
   return AnnotatedNameKind::Unresolved;
 }
 
+SourceLocation Parser::getEndOfPreviousToken() const {
+  SourceLocation TokenEndLoc = PP.getLocForEndOfToken(PrevTokLocation);
+  return TokenEndLoc.isValid() ? TokenEndLoc : Tok.getLocation();
+}
+
 bool Parser::TryKeywordIdentFallback(bool DisableKeyword) {
   assert(Tok.isNot(tok::identifier));
   Diag(Tok, diag::ext_keyword_as_ident)
diff --git a/external/llvm-project/clang/lib/Rewrite/HTMLRewrite.cpp b/external/llvm-project/clang/lib/Rewrite/HTMLRewrite.cpp
index 1829a4ff3504..109cdf990543 100644
--- a/external/llvm-project/clang/lib/Rewrite/HTMLRewrite.cpp
+++ b/external/llvm-project/clang/lib/Rewrite/HTMLRewrite.cpp
@@ -17,9 +17,7 @@
 #include "clang/Lex/TokenConcatenation.h"
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "llvm/ADT/RewriteBuffer.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
 
diff --git a/external/llvm-project/clang/lib/Sema/ParsedAttr.cpp b/external/llvm-project/clang/lib/Sema/ParsedAttr.cpp
index 4e4859d538cd..294f88eae931 100644
--- a/external/llvm-project/clang/lib/Sema/ParsedAttr.cpp
+++ b/external/llvm-project/clang/lib/Sema/ParsedAttr.cpp
@@ -91,7 +91,8 @@ void AttributePool::takePool(AttributePool &pool) {
 
 void AttributePool::takeFrom(ParsedAttributesView &List, AttributePool &Pool) {
   assert(&Pool != this && "AttributePool can't take attributes from itself");
-  llvm::for_each(List.AttrList, [&Pool](ParsedAttr *A) { Pool.remove(A); });
+  for (ParsedAttr *A : List.AttrList)
+    Pool.remove(A);
   llvm::append_range(Attrs, List.AttrList);
 }
 
diff --git a/external/llvm-project/clang/lib/Sema/Sema.cpp b/external/llvm-project/clang/lib/Sema/Sema.cpp
index 6a2cca6c14ac..42ebf2a508a2 100644
--- a/external/llvm-project/clang/lib/Sema/Sema.cpp
+++ b/external/llvm-project/clang/lib/Sema/Sema.cpp
@@ -84,6 +84,28 @@ SourceLocation Sema::getLocForEndOfToken(SourceLocation Loc, unsigned Offset) {
   return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
 }
 
+SourceRange
+Sema::getRangeForNextToken(SourceLocation Loc, bool IncludeMacros,
+                           bool IncludeComments,
+                           std::optional<tok::TokenKind> ExpectedToken) {
+  if (!Loc.isValid())
+    return SourceRange();
+  std::optional<Token> NextToken =
+      Lexer::findNextToken(Loc, SourceMgr, LangOpts, IncludeComments);
+  if (!NextToken)
+    return SourceRange();
+  if (ExpectedToken && NextToken->getKind() != *ExpectedToken)
+    return SourceRange();
+  SourceLocation TokenStart = NextToken->getLocation();
+  SourceLocation TokenEnd = NextToken->getLastLoc();
+  if (!TokenStart.isValid() || !TokenEnd.isValid())
+    return SourceRange();
+  if (!IncludeMacros && (TokenStart.isMacroID() || TokenEnd.isMacroID()))
+    return SourceRange();
+
+  return SourceRange(TokenStart, TokenEnd);
+}
+
 ModuleLoader &Sema::getModuleLoader() const { return PP.getModuleLoader(); }
 
 DarwinSDKInfo *
@@ -515,11 +537,14 @@ void Sema::Initialize() {
 #include "clang/Basic/OpenCLExtensionTypes.def"
   }
 
-  if (Context.getTargetInfo().hasAArch64SVETypes() ||
+  if (Context.getTargetInfo().hasAArch64ACLETypes() ||
       (Context.getAuxTargetInfo() &&
-       Context.getAuxTargetInfo()->hasAArch64SVETypes())) {
+       Context.getAuxTargetInfo()->hasAArch64ACLETypes())) {
 #define SVE_TYPE(Name, Id, SingletonId)                                        \
   addImplicitTypedef(#Name, Context.SingletonId);
+#define NEON_VECTOR_TYPE(Name, BaseType, ElBits, NumEls, VectorKind)           \
+  addImplicitTypedef(                                                          \
+      #Name, Context.getVectorType(Context.BaseType, NumEls, VectorKind));
 #include "clang/Basic/AArch64ACLETypes.def"
   }
 
@@ -1202,15 +1227,6 @@ void Sema::ActOnEndOfTranslationUnitFragment(TUFragmentKind Kind) {
   assert(LateParsedInstantiations.empty() &&
          "end of TU template instantiation should not create more "
          "late-parsed templates");
-
-  // Report diagnostics for uncorrected delayed typos. Ideally all of them
-  // should have been corrected by that time, but it is very hard to cover all
-  // cases in practice.
-  for (const auto &Typo : DelayedTypos) {
-    // We pass an empty TypoCorrection to indicate no correction was performed.
-    Typo.second.DiagHandler(TypoCorrection());
-  }
-  DelayedTypos.clear();
 }
 
 void Sema::ActOnEndOfTranslationUnit() {
@@ -1350,8 +1366,7 @@ void Sema::ActOnEndOfTranslationUnit() {
 
     CurrentModule->NamedModuleHasInit =
         DoesModNeedInit(CurrentModule) ||
-        llvm::any_of(CurrentModule->submodules(),
-                     [&](auto *SubM) { return DoesModNeedInit(SubM); });
+        llvm::any_of(CurrentModule->submodules(), DoesModNeedInit);
   }
 
   if (TUKind == TU_ClangModule) {
diff --git a/external/llvm-project/clang/lib/Sema/SemaAPINotes.cpp b/external/llvm-project/clang/lib/Sema/SemaAPINotes.cpp
index def909fc2478..f21cbbbdb44e 100644
--- a/external/llvm-project/clang/lib/Sema/SemaAPINotes.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaAPINotes.cpp
@@ -303,10 +303,9 @@ static void ProcessAPINotes(Sema &S, Decl *D,
           AttributeFactory AF{};
           AttributePool AP{AF};
           auto &C = S.getASTContext();
-          ParsedAttr *SNA =
-              AP.create(&C.Idents.get("swift_name"), SourceRange(), nullptr,
-                        SourceLocation(), nullptr, nullptr, nullptr,
-                        ParsedAttr::Form::GNU());
+          ParsedAttr *SNA = AP.create(
+              &C.Idents.get("swift_name"), SourceRange(), AttributeScopeInfo(),
+              nullptr, nullptr, nullptr, ParsedAttr::Form::GNU());
 
           if (!S.Swift().DiagnoseName(D, Info.SwiftName, D->getLocation(), *SNA,
                                       /*IsAsync=*/false))
diff --git a/external/llvm-project/clang/lib/Sema/SemaARM.cpp b/external/llvm-project/clang/lib/Sema/SemaARM.cpp
index 146cd7819566..e992a1012fde 100644
--- a/external/llvm-project/clang/lib/Sema/SemaARM.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaARM.cpp
@@ -1010,8 +1010,11 @@ bool SemaARM::CheckARMBuiltinFunctionCall(const TargetInfo &TI,
   case ARM::BI__builtin_arm_vcvtr_d:
     return SemaRef.BuiltinConstantArgRange(TheCall, 1, 0, 1);
   case ARM::BI__builtin_arm_dmb:
+  case ARM::BI__dmb:
   case ARM::BI__builtin_arm_dsb:
+  case ARM::BI__dsb:
   case ARM::BI__builtin_arm_isb:
+  case ARM::BI__isb:
   case ARM::BI__builtin_arm_dbg:
     return SemaRef.BuiltinConstantArgRange(TheCall, 0, 0, 15);
   case ARM::BI__builtin_arm_cdp:
@@ -1108,8 +1111,11 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
   switch (BuiltinID) {
   default: return false;
   case AArch64::BI__builtin_arm_dmb:
+  case AArch64::BI__dmb:
   case AArch64::BI__builtin_arm_dsb:
+  case AArch64::BI__dsb:
   case AArch64::BI__builtin_arm_isb:
+  case AArch64::BI__isb:
     l = 0;
     u = 15;
     break;
diff --git a/external/llvm-project/clang/lib/Sema/SemaCUDA.cpp b/external/llvm-project/clang/lib/Sema/SemaCUDA.cpp
index 5ca47adbed81..24cb8c369790 100644
--- a/external/llvm-project/clang/lib/Sema/SemaCUDA.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaCUDA.cpp
@@ -321,7 +321,7 @@ void SemaCUDA::EraseUnwantedMatches(
   if (Matches.size() <= 1)
     return;
 
-  using Pair = std::pair<DeclAccessPair, FunctionDecl*>;
+  using Pair = std::pair<DeclAccessPair, FunctionDecl *>;
 
   // Gets the CUDA function preference for a call from Caller to Match.
   auto GetCFP = [&](const Pair &Match) {
@@ -504,7 +504,6 @@ bool SemaCUDA::inferTargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
     }
   }
 
-
   // If no target was inferred, mark this member as __host__ __device__;
   // it's the least restrictive option that can be invoked from any target.
   bool NeedsH = true, NeedsD = true;
@@ -679,16 +678,22 @@ void SemaCUDA::checkAllowedInitializer(VarDecl *VD) {
       FD && FD->isDependentContext())
     return;
 
+  bool IsSharedVar = VD->hasAttr<CUDASharedAttr>();
+  bool IsDeviceOrConstantVar =
+      !IsSharedVar &&
+      (VD->hasAttr<CUDADeviceAttr>() || VD->hasAttr<CUDAConstantAttr>());
+  if ((IsSharedVar || IsDeviceOrConstantVar) &&
+      VD->getType().getQualifiers().getAddressSpace() != LangAS::Default) {
+    Diag(VD->getLocation(), diag::err_cuda_address_space_gpuvar);
+    VD->setInvalidDecl();
+    return;
+  }
   // Do not check dependent variables since the ctor/dtor/initializer are not
   // determined. Do it after instantiation.
   if (VD->isInvalidDecl() || !VD->hasInit() || !VD->hasGlobalStorage() ||
       IsDependentVar(VD))
     return;
   const Expr *Init = VD->getInit();
-  bool IsSharedVar = VD->hasAttr<CUDASharedAttr>();
-  bool IsDeviceOrConstantVar =
-      !IsSharedVar &&
-      (VD->hasAttr<CUDADeviceAttr>() || VD->hasAttr<CUDAConstantAttr>());
   if (IsDeviceOrConstantVar || IsSharedVar) {
     if (HasAllowedCUDADeviceStaticInitializer(
             *this, VD, IsSharedVar ? CICK_Shared : CICK_DeviceOrConstant))
diff --git a/external/llvm-project/clang/lib/Sema/SemaChecking.cpp b/external/llvm-project/clang/lib/Sema/SemaChecking.cpp
index d61b71e01452..10a425eec026 100644
--- a/external/llvm-project/clang/lib/Sema/SemaChecking.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaChecking.cpp
@@ -1257,6 +1257,8 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
   switch (BuiltinID) {
   default:
     return;
+  case Builtin::BI__builtin_stpcpy:
+  case Builtin::BIstpcpy:
   case Builtin::BI__builtin_strcpy:
   case Builtin::BIstrcpy: {
     DiagID = diag::warn_fortify_strlen_overflow;
@@ -1265,6 +1267,7 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
     break;
   }
 
+  case Builtin::BI__builtin___stpcpy_chk:
   case Builtin::BI__builtin___strcpy_chk: {
     DiagID = diag::warn_fortify_strlen_overflow;
     SourceSize = ComputeStrLenArgument(1);
@@ -1826,6 +1829,40 @@ static ExprResult PointerAuthStringDiscriminator(Sema &S, CallExpr *Call) {
   return Call;
 }
 
+static ExprResult GetVTablePointer(Sema &S, CallExpr *Call) {
+  if (S.checkArgCount(Call, 1))
+    return ExprError();
+  Expr *FirstArg = Call->getArg(0);
+  ExprResult FirstValue = S.DefaultFunctionArrayLvalueConversion(FirstArg);
+  if (FirstValue.isInvalid())
+    return ExprError();
+  Call->setArg(0, FirstValue.get());
+  QualType FirstArgType = FirstArg->getType();
+  if (FirstArgType->canDecayToPointerType() && FirstArgType->isArrayType())
+    FirstArgType = S.Context.getDecayedType(FirstArgType);
+
+  const CXXRecordDecl *FirstArgRecord = FirstArgType->getPointeeCXXRecordDecl();
+  if (!FirstArgRecord) {
+    S.Diag(FirstArg->getBeginLoc(), diag::err_get_vtable_pointer_incorrect_type)
+        << /*isPolymorphic=*/0 << FirstArgType;
+    return ExprError();
+  }
+  if (S.RequireCompleteType(
+          FirstArg->getBeginLoc(), FirstArgType->getPointeeType(),
+          diag::err_get_vtable_pointer_requires_complete_type)) {
+    return ExprError();
+  }
+
+  if (!FirstArgRecord->isPolymorphic()) {
+    S.Diag(FirstArg->getBeginLoc(), diag::err_get_vtable_pointer_incorrect_type)
+        << /*isPolymorphic=*/1 << FirstArgRecord;
+    return ExprError();
+  }
+  QualType ReturnType = S.Context.getPointerType(S.Context.VoidTy.withConst());
+  Call->setType(ReturnType);
+  return Call;
+}
+
 static ExprResult BuiltinLaunder(Sema &S, CallExpr *TheCall) {
   if (S.checkArgCount(TheCall, 1))
     return ExprError();
@@ -2611,8 +2648,6 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     bool IsDelete = BuiltinID == Builtin::BI__builtin_operator_delete;
     ExprResult Res =
         BuiltinOperatorNewDeleteOverloaded(TheCallResult, IsDelete);
-    if (Res.isInvalid())
-      CorrectDelayedTyposInExpr(TheCallResult.get());
     return Res;
   }
   case Builtin::BI__builtin_dump_struct:
@@ -2724,6 +2759,10 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     return PointerAuthAuthAndResign(*this, TheCall);
   case Builtin::BI__builtin_ptrauth_string_discriminator:
     return PointerAuthStringDiscriminator(*this, TheCall);
+
+  case Builtin::BI__builtin_get_vtable_pointer:
+    return GetVTablePointer(*this, TheCall);
+
   // OpenCL v2.0, s6.13.16 - Pipe functions
   case Builtin::BIread_pipe:
   case Builtin::BIwrite_pipe:
@@ -11911,6 +11950,46 @@ static void DiagnoseMixedUnicodeImplicitConversion(Sema &S, const Type *Source,
   }
 }
 
+enum CFIUncheckedCalleeChange {
+  None,
+  Adding,
+  Discarding,
+};
+
+static CFIUncheckedCalleeChange AdjustingCFIUncheckedCallee(QualType From,
+                                                            QualType To) {
+  QualType MaybePointee = From->getPointeeType();
+  if (!MaybePointee.isNull() && MaybePointee->getAs<FunctionType>())
+    From = MaybePointee;
+  MaybePointee = To->getPointeeType();
+  if (!MaybePointee.isNull() && MaybePointee->getAs<FunctionType>())
+    To = MaybePointee;
+
+  if (const auto *FromFn = From->getAs<FunctionType>()) {
+    if (const auto *ToFn = To->getAs<FunctionType>()) {
+      if (FromFn->getCFIUncheckedCalleeAttr() &&
+          !ToFn->getCFIUncheckedCalleeAttr())
+        return Discarding;
+      if (!FromFn->getCFIUncheckedCalleeAttr() &&
+          ToFn->getCFIUncheckedCalleeAttr())
+        return Adding;
+    }
+  }
+  return None;
+}
+
+bool Sema::DiscardingCFIUncheckedCallee(QualType From, QualType To) const {
+  From = Context.getCanonicalType(From);
+  To = Context.getCanonicalType(To);
+  return ::AdjustingCFIUncheckedCallee(From, To) == Discarding;
+}
+
+bool Sema::AddingCFIUncheckedCallee(QualType From, QualType To) const {
+  From = Context.getCanonicalType(From);
+  To = Context.getCanonicalType(To);
+  return ::AdjustingCFIUncheckedCallee(From, To) == Adding;
+}
+
 void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
                                    bool *ICContext, bool IsListInit) {
   if (E->isTypeDependent() || E->isValueDependent()) return;
@@ -12256,6 +12335,11 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
   if (Target->isBooleanType())
     DiagnoseIntInBoolContext(*this, E);
 
+  if (DiscardingCFIUncheckedCallee(QualType(Source, 0), QualType(Target, 0))) {
+    Diag(CC, diag::warn_cast_discards_cfi_unchecked_callee)
+        << QualType(Source, 0) << QualType(Target, 0);
+  }
+
   if (!Source->isIntegerType() || !Target->isIntegerType())
     return;
 
@@ -12306,6 +12390,12 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
     if (SourceMgr.isInSystemMacro(CC))
       return;
 
+    if (const auto *UO = dyn_cast<UnaryOperator>(E)) {
+      if (UO->getOpcode() == UO_Minus)
+        return DiagnoseImpCast(
+            *this, E, T, CC, diag::warn_impcast_integer_precision_on_negation);
+    }
+
     if (TargetRange.Width == 32 && Context.getIntWidth(E->getType()) == 64)
       return DiagnoseImpCast(*this, E, T, CC, diag::warn_impcast_integer_64_32,
                              /* pruneControlFlow */ true);
diff --git a/external/llvm-project/clang/lib/Sema/SemaConcept.cpp b/external/llvm-project/clang/lib/Sema/SemaConcept.cpp
index c6a54dc141de..a4f660c39b4b 100644
--- a/external/llvm-project/clang/lib/Sema/SemaConcept.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaConcept.cpp
@@ -792,7 +792,7 @@ bool Sema::CheckFunctionConstraints(const FunctionDecl *FD,
                                     bool ForOverloadResolution) {
   // Don't check constraints if the function is dependent. Also don't check if
   // this is a function template specialization, as the call to
-  // CheckinstantiatedFunctionTemplateConstraints after this will check it
+  // CheckFunctionTemplateConstraints after this will check it
   // better.
   if (FD->isDependentContext() ||
       FD->getTemplatedKind() ==
@@ -1060,12 +1060,55 @@ bool Sema::EnsureTemplateArgumentListConstraints(
   return false;
 }
 
-bool Sema::CheckInstantiatedFunctionTemplateConstraints(
+static bool CheckFunctionConstraintsWithoutInstantiation(
+    Sema &SemaRef, SourceLocation PointOfInstantiation,
+    FunctionTemplateDecl *Template, ArrayRef<TemplateArgument> TemplateArgs,
+    ConstraintSatisfaction &Satisfaction) {
+  SmallVector<AssociatedConstraint, 3> TemplateAC;
+  Template->getAssociatedConstraints(TemplateAC);
+  if (TemplateAC.empty()) {
+    Satisfaction.IsSatisfied = true;
+    return false;
+  }
+
+  LocalInstantiationScope Scope(SemaRef);
+
+  FunctionDecl *FD = Template->getTemplatedDecl();
+  // Collect the list of template arguments relative to the 'primary'
+  // template. We need the entire list, since the constraint is completely
+  // uninstantiated at this point.
+
+  // FIXME: Add TemplateArgs through the 'Innermost' parameter once
+  // the refactoring of getTemplateInstantiationArgs() relands.
+  MultiLevelTemplateArgumentList MLTAL;
+  MLTAL.addOuterTemplateArguments(Template, std::nullopt, /*Final=*/false);
+  SemaRef.getTemplateInstantiationArgs(
+      MLTAL, /*D=*/FD, FD,
+      /*Final=*/false, /*Innermost=*/std::nullopt, /*RelativeToPrimary=*/true,
+      /*Pattern=*/nullptr, /*ForConstraintInstantiation=*/true);
+  MLTAL.replaceInnermostTemplateArguments(Template, TemplateArgs);
+
+  Sema::ContextRAII SavedContext(SemaRef, FD);
+  std::optional<Sema::CXXThisScopeRAII> ThisScope;
+  if (auto *Method = dyn_cast<CXXMethodDecl>(FD))
+    ThisScope.emplace(SemaRef, /*Record=*/Method->getParent(),
+                      /*ThisQuals=*/Method->getMethodQualifiers());
+  return SemaRef.CheckConstraintSatisfaction(
+      Template, TemplateAC, MLTAL, PointOfInstantiation, Satisfaction);
+}
+
+bool Sema::CheckFunctionTemplateConstraints(
     SourceLocation PointOfInstantiation, FunctionDecl *Decl,
     ArrayRef<TemplateArgument> TemplateArgs,
     ConstraintSatisfaction &Satisfaction) {
   // In most cases we're not going to have constraints, so check for that first.
   FunctionTemplateDecl *Template = Decl->getPrimaryTemplate();
+
+  if (!Template)
+    return ::CheckFunctionConstraintsWithoutInstantiation(
+        *this, PointOfInstantiation, Decl->getDescribedFunctionTemplate(),
+        TemplateArgs, Satisfaction);
+
   // Note - code synthesis context for the constraints check is created
   // inside CheckConstraintsSatisfaction.
   SmallVector<AssociatedConstraint, 3> TemplateAC;
diff --git a/external/llvm-project/clang/lib/Sema/SemaCoroutine.cpp b/external/llvm-project/clang/lib/Sema/SemaCoroutine.cpp
index 425b32e53a7b..a1389c6c034b 100644
--- a/external/llvm-project/clang/lib/Sema/SemaCoroutine.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaCoroutine.cpp
@@ -309,15 +309,6 @@ static ExprResult buildMemberCall(Sema &S, Expr *Base, SourceLocation Loc,
   if (Result.isInvalid())
     return ExprError();
 
-  // We meant exactly what we asked for. No need for typo correction.
-  if (auto *TE = dyn_cast<TypoExpr>(Result.get())) {
-    S.clearDelayedTypo(TE);
-    S.Diag(Loc, diag::err_no_member)
-        << NameInfo.getName() << Base->getType()->getAsCXXRecordDecl()
-        << Base->getSourceRange();
-    return ExprError();
-  }
-
   auto EndLoc = Args.empty() ? Loc : Args.back()->getEndLoc();
   return S.BuildCallExpr(nullptr, Result.get(), Loc, Args, EndLoc, nullptr);
 }
@@ -811,7 +802,6 @@ ExprResult Sema::ActOnCoawaitExpr(Scope *S, SourceLocation Loc, Expr *E) {
     return ExprError();
 
   if (!ActOnCoroutineBodyStart(S, Loc, "co_await")) {
-    CorrectDelayedTyposInExpr(E);
     return ExprError();
   }
 
@@ -970,7 +960,6 @@ ExprResult Sema::ActOnCoyieldExpr(Scope *S, SourceLocation Loc, Expr *E) {
     return ExprError();
 
   if (!ActOnCoroutineBodyStart(S, Loc, "co_yield")) {
-    CorrectDelayedTyposInExpr(E);
     return ExprError();
   }
 
@@ -1025,7 +1014,6 @@ ExprResult Sema::BuildCoyieldExpr(SourceLocation Loc, Expr *E) {
 
 StmtResult Sema::ActOnCoreturnStmt(Scope *S, SourceLocation Loc, Expr *E) {
   if (!ActOnCoroutineBodyStart(S, Loc, "co_return")) {
-    CorrectDelayedTyposInExpr(E);
     return StmtError();
   }
   return BuildCoreturnStmt(Loc, E);
diff --git a/external/llvm-project/clang/lib/Sema/SemaDecl.cpp b/external/llvm-project/clang/lib/Sema/SemaDecl.cpp
index 86b871396ec9..5cffd82e3372 100644
--- a/external/llvm-project/clang/lib/Sema/SemaDecl.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaDecl.cpp
@@ -8789,7 +8789,7 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) {
         FunctionDecl *FD = getCurFunctionDecl();
         // OpenCL v1.1 s6.5.2 and s6.5.3: no local or constant variables
         // in functions.
-        if (FD && !FD->hasAttr<OpenCLKernelAttr>()) {
+        if (FD && !FD->hasAttr<DeviceKernelAttr>()) {
           if (T.getAddressSpace() == LangAS::opencl_constant)
             Diag(NewVD->getLocation(), diag::err_opencl_function_variable)
                 << 0 /*non-kernel only*/ << "constant";
@@ -8801,7 +8801,7 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) {
         }
         // OpenCL v2.0 s6.5.2 and s6.5.3: local and constant variables must be
         // in the outermost scope of a kernel function.
-        if (FD && FD->hasAttr<OpenCLKernelAttr>()) {
+        if (FD && FD->hasAttr<DeviceKernelAttr>()) {
           if (!getCurScope()->isFunctionScope()) {
             if (T.getAddressSpace() == LangAS::opencl_constant)
               Diag(NewVD->getLocation(), diag::err_opencl_addrspace_scope)
@@ -10930,9 +10930,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 
   MarkUnusedFileScopedDecl(NewFD);
 
-
-
-  if (getLangOpts().OpenCL && NewFD->hasAttr<OpenCLKernelAttr>()) {
+  if (getLangOpts().OpenCL && NewFD->hasAttr<DeviceKernelAttr>()) {
     // OpenCL v1.2 s6.8 static is invalid for kernel functions.
     if (SC == SC_Static) {
       Diag(D.getIdentifierLoc(), diag::err_static_kernel);
@@ -12437,7 +12435,7 @@ void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
 
   if (getLangOpts().OpenCL) {
     Diag(FD->getLocation(), diag::err_opencl_no_main)
-        << FD->hasAttr<OpenCLKernelAttr>();
+        << FD->hasAttr<DeviceKernelAttr>();
     FD->setInvalidDecl();
     return;
   }
@@ -13520,8 +13518,28 @@ bool Sema::GloballyUniqueObjectMightBeAccidentallyDuplicated(
 
   // If the object isn't hidden, the dynamic linker will prevent duplication.
   clang::LinkageInfo Lnk = Target->getLinkageAndVisibility();
-  if (Lnk.getVisibility() != HiddenVisibility)
+
+  // The target is "hidden" (from the dynamic linker) if:
+  // 1. On posix, it has hidden visibility, or
+  // 2. On windows, it has no import/export annotation
+  if (Context.getTargetInfo().shouldDLLImportComdatSymbols()) {
+    if (Target->hasAttr<DLLExportAttr>() || Target->hasAttr<DLLImportAttr>())
+      return false;
+
+    // If the variable isn't directly annotated, check to see if it's a member
+    // of an annotated class.
+    const VarDecl *VD = dyn_cast<VarDecl>(Target);
+
+    if (VD && VD->isStaticDataMember()) {
+      const CXXRecordDecl *Ctx = dyn_cast<CXXRecordDecl>(VD->getDeclContext());
+      if (Ctx &&
+          (Ctx->hasAttr<DLLExportAttr>() || Ctx->hasAttr<DLLImportAttr>()))
+        return false;
+    }
+  } else if (Lnk.getVisibility() != HiddenVisibility) {
+    // Posix case
     return false;
+  }
 
   // If the obj doesn't have external linkage, it's supposed to be duplicated.
   if (!isExternalFormalLinkage(Lnk.getLinkage()))
@@ -13552,19 +13570,16 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
   // duplicated when built into a shared library, which causes problems if it's
   // mutable (since the copies won't be in sync) or its initialization has side
   // effects (since it will run once per copy instead of once globally).
-  // FIXME: Windows uses dllexport/dllimport instead of visibility, and we don't
-  // handle that yet. Disable the warning on Windows for now.
 
   // Don't diagnose if we're inside a template, because it's not practical to
   // fix the warning in most cases.
-  if (!Context.getTargetInfo().shouldDLLImportComdatSymbols() &&
-      !VD->isTemplated() &&
+  if (!VD->isTemplated() &&
       GloballyUniqueObjectMightBeAccidentallyDuplicated(VD)) {
 
     QualType Type = VD->getType();
     if (looksMutable(Type, VD->getASTContext())) {
       Diag(VD->getLocation(), diag::warn_possible_object_duplication_mutable)
-          << VD;
+          << VD << Context.getTargetInfo().shouldDLLImportComdatSymbols();
     }
 
     // To keep false positives low, only warn if we're certain that the
@@ -13577,7 +13592,7 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
                              /*IncludePossibleEffects=*/false) &&
         !isa<CXXNewExpr>(Init->IgnoreParenImpCasts())) {
       Diag(Init->getExprLoc(), diag::warn_possible_object_duplication_init)
-          << VD;
+          << VD << Context.getTargetInfo().shouldDLLImportComdatSymbols();
     }
   }
 }
@@ -13586,7 +13601,6 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
   // If there is no declaration, there was an error parsing it.  Just ignore
   // the initializer.
   if (!RealDecl) {
-    CorrectDelayedTyposInExpr(Init, dyn_cast_or_null<VarDecl>(RealDecl));
     return;
   }
 
@@ -13609,12 +13623,8 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
   }
 
   if (VDecl->isInvalidDecl()) {
-    ExprResult Res = CorrectDelayedTyposInExpr(Init, VDecl);
-    SmallVector<Expr *> SubExprs;
-    if (Res.isUsable())
-      SubExprs.push_back(Res.get());
     ExprResult Recovery =
-        CreateRecoveryExpr(Init->getBeginLoc(), Init->getEndLoc(), SubExprs);
+        CreateRecoveryExpr(Init->getBeginLoc(), Init->getEndLoc(), {Init});
     if (Expr *E = Recovery.get())
       VDecl->setInit(E);
     return;
@@ -13629,23 +13639,12 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
 
   // C++11 [decl.spec.auto]p6. Deduce the type which 'auto' stands in for.
   if (VDecl->getType()->isUndeducedType()) {
-    // Attempt typo correction early so that the type of the init expression can
-    // be deduced based on the chosen correction if the original init contains a
-    // TypoExpr.
-    ExprResult Res = CorrectDelayedTyposInExpr(Init, VDecl);
-    if (!Res.isUsable()) {
-      // There are unresolved typos in Init, just drop them.
-      // FIXME: improve the recovery strategy to preserve the Init.
-      RealDecl->setInvalidDecl();
-      return;
-    }
-    if (Res.get()->containsErrors()) {
+    if (Init->containsErrors()) {
       // Invalidate the decl as we don't know the type for recovery-expr yet.
       RealDecl->setInvalidDecl();
-      VDecl->setInit(Res.get());
+      VDecl->setInit(Init);
       return;
     }
-    Init = Res.get();
 
     if (DeduceVariableDeclarationType(VDecl, DirectInit, Init))
       return;
@@ -13791,23 +13790,6 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
       InitializedFromParenListExpr = true;
     }
 
-    // Try to correct any TypoExprs in the initialization arguments.
-    for (size_t Idx = 0; Idx < Args.size(); ++Idx) {
-      ExprResult Res = CorrectDelayedTyposInExpr(
-          Args[Idx], VDecl, /*RecoverUncorrectedTypos=*/true,
-          [this, Entity, Kind](Expr *E) {
-            InitializationSequence Init(*this, Entity, Kind, MultiExprArg(E));
-            return Init.Failed() ? ExprError() : E;
-          });
-      if (!Res.isUsable()) {
-        VDecl->setInvalidDecl();
-      } else if (Res.get() != Args[Idx]) {
-        Args[Idx] = Res.get();
-      }
-    }
-    if (VDecl->isInvalidDecl())
-      return;
-
     InitializationSequence InitSeq(*this, Entity, Kind, Args,
                                    /*TopLevelOfInitList=*/false,
                                    /*TreatUnavailableAsInvalid=*/false);
@@ -14419,6 +14401,12 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
     if (getLangOpts().HLSL && HLSL().ActOnUninitializedVarDecl(Var))
       return;
 
+    // HLSL input variables are expected to be externally initialized, even
+    // when marked `static`.
+    if (getLangOpts().HLSL &&
+        Var->getType().getAddressSpace() == LangAS::hlsl_input)
+      return;
+
     // C++03 [dcl.init]p9:
     //   If no initializer is specified for an object, and the
     //   object is of (possibly cv-qualified) non-POD class type (or
@@ -15707,7 +15695,7 @@ ShouldWarnAboutMissingPrototype(const FunctionDecl *FD,
     return false;
 
   // Don't warn for OpenCL kernels.
-  if (FD->hasAttr<OpenCLKernelAttr>())
+  if (FD->hasAttr<DeviceKernelAttr>())
     return false;
 
   // Don't warn on explicitly deleted functions.
@@ -20510,14 +20498,11 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceRange BraceRange,
     CheckAlignasUnderalignment(Enum);
 }
 
-Decl *Sema::ActOnFileScopeAsmDecl(Expr *expr,
-                                  SourceLocation StartLoc,
+Decl *Sema::ActOnFileScopeAsmDecl(Expr *expr, SourceLocation StartLoc,
                                   SourceLocation EndLoc) {
-  StringLiteral *AsmString = cast<StringLiteral>(expr);
 
-  FileScopeAsmDecl *New = FileScopeAsmDecl::Create(Context, CurContext,
-                                                   AsmString, StartLoc,
-                                                   EndLoc);
+  FileScopeAsmDecl *New =
+      FileScopeAsmDecl::Create(Context, CurContext, expr, StartLoc, EndLoc);
   CurContext->addDecl(New);
   return New;
 }
@@ -20601,7 +20586,7 @@ Sema::FunctionEmissionStatus Sema::getEmissionStatus(const FunctionDecl *FD,
 
   // SYCL functions can be template, so we check if they have appropriate
   // attribute prior to checking if it is a template.
-  if (LangOpts.SYCLIsDevice && FD->hasAttr<SYCLKernelAttr>())
+  if (LangOpts.SYCLIsDevice && FD->hasAttr<DeviceKernelAttr>())
     return FunctionEmissionStatus::Emitted;
 
   // Templates are emitted when they're instantiated.
diff --git a/external/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp b/external/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp
index f34b70ea61a5..1c2fa80e782d 100644
--- a/external/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaDeclAttr.cpp
@@ -53,7 +53,6 @@
 #include "clang/Sema/SemaOpenCL.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaRISCV.h"
-#include "clang/Sema/SemaSPIRV.h"
 #include "clang/Sema/SemaSYCL.h"
 #include "clang/Sema/SemaSwift.h"
 #include "clang/Sema/SemaWasm.h"
@@ -872,6 +871,21 @@ static void handleDiagnoseIfAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
       cast<NamedDecl>(D)));
 }
 
+static void handleCFIUncheckedCalleeAttr(Sema &S, Decl *D,
+                                         const ParsedAttr &Attrs) {
+  if (hasDeclarator(D))
+    return;
+
+  if (!isa<ObjCMethodDecl>(D)) {
+    S.Diag(Attrs.getLoc(), diag::warn_attribute_wrong_decl_type)
+        << Attrs << Attrs.isRegularKeywordAttribute()
+        << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  D->addAttr(::new (S.Context) CFIUncheckedCalleeAttr(S.Context, Attrs));
+}
+
 static void handleNoBuiltinAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   static constexpr const StringRef kWildcard = "*";
 
@@ -1971,14 +1985,13 @@ bool Sema::CheckAttrNoArgs(const ParsedAttr &Attrs) {
 bool Sema::CheckAttrTarget(const ParsedAttr &AL) {
   // Check whether the attribute is valid on the current target.
   if (!AL.existsInTarget(Context.getTargetInfo())) {
-    Diag(AL.getLoc(), AL.isRegularKeywordAttribute()
-                          ? diag::err_keyword_not_supported_on_target
-                          : diag::warn_unknown_attribute_ignored)
-        << AL << AL.getRange();
+    if (AL.isRegularKeywordAttribute())
+      Diag(AL.getLoc(), diag::err_keyword_not_supported_on_target);
+    else
+      DiagnoseUnknownAttribute(AL);
     AL.setInvalid();
     return true;
   }
-
   return false;
 }
 
@@ -2359,7 +2372,8 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   IdentifierLoc *Platform = AL.getArgAsIdent(0);
 
   IdentifierInfo *II = Platform->getIdentifierInfo();
-  if (AvailabilityAttr::getPrettyPlatformName(II->getName()).empty())
+  StringRef PrettyName = AvailabilityAttr::getPrettyPlatformName(II->getName());
+  if (PrettyName.empty())
     S.Diag(Platform->getLoc(), diag::warn_availability_unknown_platform)
         << Platform->getIdentifierInfo();
 
@@ -2370,6 +2384,32 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   AvailabilityChange Introduced = AL.getAvailabilityIntroduced();
   AvailabilityChange Deprecated = AL.getAvailabilityDeprecated();
   AvailabilityChange Obsoleted = AL.getAvailabilityObsoleted();
+
+  const llvm::Triple::OSType PlatformOS = AvailabilityAttr::getOSType(
+      AvailabilityAttr::canonicalizePlatformName(II->getName()));
+
+  auto reportAndUpdateIfInvalidOS = [&](auto &InputVersion) -> void {
+    const bool IsInValidRange =
+        llvm::Triple::isValidVersionForOS(PlatformOS, InputVersion);
+    // Canonicalize availability versions.
+    auto CanonicalVersion = llvm::Triple::getCanonicalVersionForOS(
+        PlatformOS, InputVersion, IsInValidRange);
+    if (!IsInValidRange) {
+      S.Diag(Platform->getLoc(), diag::warn_availability_invalid_os_version)
+          << InputVersion.getAsString() << PrettyName;
+      S.Diag(Platform->getLoc(),
+             diag::note_availability_invalid_os_version_adjusted)
+          << CanonicalVersion.getAsString();
+    }
+    InputVersion = CanonicalVersion;
+  };
+
+  if (PlatformOS != llvm::Triple::OSType::UnknownOS) {
+    reportAndUpdateIfInvalidOS(Introduced.Version);
+    reportAndUpdateIfInvalidOS(Deprecated.Version);
+    reportAndUpdateIfInvalidOS(Obsoleted.Version);
+  }
+
   bool IsUnavailable = AL.getUnavailableLoc().isValid();
   bool IsStrict = AL.getStrictLoc().isValid();
   StringRef Str;
@@ -2461,7 +2501,11 @@ static void handleAvailabilityAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
         }
 
         auto Major = Version.getMajor();
-        auto NewMajor = Major >= 9 ? Major - 7 : 0;
+        auto NewMajor = Major;
+        if (Major < 9)
+          NewMajor = 0;
+        else if (Major < 12)
+          NewMajor = Major - 7;
         if (NewMajor >= 2) {
           if (Version.getMinor()) {
             if (Version.getSubminor())
@@ -3187,14 +3231,23 @@ bool Sema::checkTargetAttr(SourceLocation LiteralLoc, StringRef AttrStr) {
     if (ParsedAttrs.Duplicate != "")
       return Diag(LiteralLoc, diag::err_duplicate_target_attribute)
              << Duplicate << None << ParsedAttrs.Duplicate << Target;
-    for (const auto &Feature : ParsedAttrs.Features) {
-      StringRef CurFeature = Feature;
+    for (StringRef CurFeature : ParsedAttrs.Features) {
       if (!CurFeature.starts_with('+') && !CurFeature.starts_with('-'))
         return Diag(LiteralLoc, diag::warn_unsupported_target_attribute)
                << Unsupported << None << AttrStr << Target;
     }
   }
 
+  if (Context.getTargetInfo().getTriple().isLoongArch()) {
+    for (StringRef CurFeature : ParsedAttrs.Features) {
+      if (CurFeature.starts_with("!arch=")) {
+        StringRef ArchValue = CurFeature.split("=").second.trim();
+        return Diag(LiteralLoc, diag::err_attribute_unsupported)
+               << "target(arch=..)" << ArchValue;
+      }
+    }
+  }
+
   if (ParsedAttrs.Duplicate != "")
     return Diag(LiteralLoc, diag::warn_unsupported_target_attribute)
            << Duplicate << None << ParsedAttrs.Duplicate << Target;
@@ -3495,13 +3548,7 @@ static void handleTargetClonesAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (HasCommas && AL.getNumArgs() > 1)
     S.Diag(AL.getLoc(), diag::warn_target_clone_mixed_values);
 
-  if (S.Context.getTargetInfo().getTriple().isAArch64() && !HasDefault) {
-    // Add default attribute if there is no one
-    HasDefault = true;
-    Strings.push_back("default");
-  }
-
-  if (!HasDefault) {
+  if (!HasDefault && !S.Context.getTargetInfo().getTriple().isAArch64()) {
     S.Diag(AL.getLoc(), diag::err_target_clone_must_have_default);
     return;
   }
@@ -5088,8 +5135,8 @@ static void handleGlobalAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (FD->isInlineSpecified() && !S.getLangOpts().CUDAIsDevice)
     S.Diag(FD->getBeginLoc(), diag::warn_kern_is_inline) << FD;
 
-  if (AL.getKind() == ParsedAttr::AT_NVPTXKernel)
-    D->addAttr(::new (S.Context) NVPTXKernelAttr(S.Context, AL));
+  if (AL.getKind() == ParsedAttr::AT_DeviceKernel)
+    D->addAttr(::new (S.Context) DeviceKernelAttr(S.Context, AL));
   else
     D->addAttr(::new (S.Context) CUDAGlobalAttr(S.Context, AL));
   // In host compilation the kernel is emitted as a stub function, which is
@@ -5224,9 +5271,11 @@ static void handleCallConvAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   case ParsedAttr::AT_AArch64SVEPcs:
     D->addAttr(::new (S.Context) AArch64SVEPcsAttr(S.Context, AL));
     return;
-  case ParsedAttr::AT_AMDGPUKernelCall:
-    D->addAttr(::new (S.Context) AMDGPUKernelCallAttr(S.Context, AL));
+  case ParsedAttr::AT_DeviceKernel: {
+    // The attribute should already be applied.
+    assert(D->hasAttr<DeviceKernelAttr>() && "Expected attribute");
     return;
+  }
   case ParsedAttr::AT_IntelOclBicc:
     D->addAttr(::new (S.Context) IntelOclBiccAttr(S.Context, AL));
     return;
@@ -5269,6 +5318,33 @@ static void handleCallConvAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 }
 
+static void handleDeviceKernelAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  const auto *FD = dyn_cast_or_null<FunctionDecl>(D);
+  bool IsFunctionTemplate = FD && FD->getDescribedFunctionTemplate();
+  if (S.getLangOpts().SYCLIsDevice) {
+    if (!IsFunctionTemplate) {
+      S.Diag(AL.getLoc(), diag::warn_attribute_wrong_decl_type_str)
+          << AL << AL.isRegularKeywordAttribute() << "function templates";
+    } else {
+      S.SYCL().handleKernelAttr(D, AL);
+    }
+  } else if (DeviceKernelAttr::isSYCLSpelling(AL)) {
+    S.Diag(AL.getLoc(), diag::warn_attribute_ignored) << AL;
+  } else if (S.getASTContext().getTargetInfo().getTriple().isNVPTX()) {
+    handleGlobalAttr(S, D, AL);
+  } else {
+    // OpenCL C++ will throw a more specific error.
+    if (!S.getLangOpts().OpenCLCPlusPlus && (!FD || IsFunctionTemplate)) {
+      S.Diag(AL.getLoc(), diag::err_attribute_wrong_decl_type_str)
+          << AL << AL.isRegularKeywordAttribute() << "functions";
+    }
+    handleSimpleAttribute<DeviceKernelAttr>(S, D, AL);
+  }
+  // Make sure we validate the CC with the target
+  // and warn/error if necessary.
+  handleCallConvAttr(S, D, AL);
+}
+
 static void handleSuppressAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (AL.getAttributeSpellingListIndex() == SuppressAttr::CXX11_gsl_suppress) {
     // Suppression attribute with GSL spelling requires at least 1 argument.
@@ -5433,9 +5509,6 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
   case ParsedAttr::AT_AArch64SVEPcs:
     CC = CC_AArch64SVEPCS;
     break;
-  case ParsedAttr::AT_AMDGPUKernelCall:
-    CC = CC_AMDGPUKernelCall;
-    break;
   case ParsedAttr::AT_RegCall:
     CC = CC_X86RegCall;
     break;
@@ -5505,6 +5578,11 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
                                   llvm::Log2_64(ABIVLen) - 5);
     break;
   }
+  case ParsedAttr::AT_DeviceKernel: {
+    // Validation was handled in handleDeviceKernelAttr.
+    CC = CC_DeviceKernel;
+    break;
+  }
   default: llvm_unreachable("unexpected attribute kind");
   }
 
@@ -7110,6 +7188,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_NoBuiltin:
     handleNoBuiltinAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_CFIUncheckedCallee:
+    handleCFIUncheckedCalleeAttr(S, D, AL);
+    break;
   case ParsedAttr::AT_ExtVectorType:
     handleExtVectorTypeAttr(S, D, AL);
     break;
@@ -7125,9 +7206,6 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_EnumExtensibility:
     handleEnumExtensibilityAttr(S, D, AL);
     break;
-  case ParsedAttr::AT_SYCLKernel:
-    S.SYCL().handleKernelAttr(D, AL);
-    break;
   case ParsedAttr::AT_SYCLKernelEntryPoint:
     S.SYCL().handleKernelEntryPointAttr(D, AL);
     break;
@@ -7152,7 +7230,6 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_CalledOnce:
     handleCalledOnceAttr(S, D, AL);
     break;
-  case ParsedAttr::AT_NVPTXKernel:
   case ParsedAttr::AT_CUDAGlobal:
     handleGlobalAttr(S, D, AL);
     break;
@@ -7416,13 +7493,15 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_PreserveAll:
   case ParsedAttr::AT_AArch64VectorPcs:
   case ParsedAttr::AT_AArch64SVEPcs:
-  case ParsedAttr::AT_AMDGPUKernelCall:
   case ParsedAttr::AT_M68kRTD:
   case ParsedAttr::AT_PreserveNone:
   case ParsedAttr::AT_RISCVVectorCC:
   case ParsedAttr::AT_RISCVVLSCC:
     handleCallConvAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_DeviceKernel:
+    handleDeviceKernelAttr(S, D, AL);
+    break;
   case ParsedAttr::AT_Suppress:
     handleSuppressAttr(S, D, AL);
     break;
@@ -7505,6 +7584,12 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLWaveSize:
     S.HLSL().handleWaveSizeAttr(D, AL);
     break;
+  case ParsedAttr::AT_HLSLSV_Position:
+    S.HLSL().handleSV_PositionAttr(D, AL);
+    break;
+  case ParsedAttr::AT_HLSLVkExtBuiltinInput:
+    S.HLSL().handleVkExtBuiltinInputAttr(D, AL);
+    break;
   case ParsedAttr::AT_HLSLSV_GroupThreadID:
     S.HLSL().handleSV_GroupThreadIDAttr(D, AL);
     break;
@@ -7738,9 +7823,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
 
 static bool isKernelDecl(Decl *D) {
   const FunctionType *FnTy = D->getFunctionType();
-  return D->hasAttr<OpenCLKernelAttr>() ||
-         (FnTy && FnTy->getCallConv() == CallingConv::CC_AMDGPUKernelCall) ||
-         D->hasAttr<CUDAGlobalAttr>() || D->getAttr<NVPTXKernelAttr>();
+  return D->hasAttr<DeviceKernelAttr>() ||
+         (FnTy && FnTy->getCallConv() == CallingConv::CC_DeviceKernel) ||
+         D->hasAttr<CUDAGlobalAttr>();
 }
 
 void Sema::ProcessDeclAttributeList(
@@ -7767,7 +7852,7 @@ void Sema::ProcessDeclAttributeList(
   // good to have a way to specify "these attributes must appear as a group",
   // for these. Additionally, it would be good to have a way to specify "these
   // attribute must never appear as a group" for attributes like cold and hot.
-  if (!(D->hasAttr<OpenCLKernelAttr>() ||
+  if (!(D->hasAttr<DeviceKernelAttr>() ||
         (D->hasAttr<CUDAGlobalAttr>() &&
          Context.getTargetInfo().getTriple().isSPIRV()))) {
     // These attributes cannot be applied to a non-kernel function.
@@ -7867,8 +7952,7 @@ static void checkUnusedDeclAttributes(Sema &S, const ParsedAttributesView &A) {
       continue;
 
     if (AL.getKind() == ParsedAttr::UnknownAttribute) {
-      S.Diag(AL.getLoc(), diag::warn_unknown_attribute_ignored)
-          << AL << AL.getRange();
+      S.DiagnoseUnknownAttribute(AL);
     } else {
       S.Diag(AL.getLoc(), diag::warn_attribute_not_on_decl) << AL
                                                             << AL.getRange();
@@ -7886,15 +7970,45 @@ void Sema::checkUnusedDeclAttributes(Declarator &D) {
 
 void Sema::DiagnoseUnknownAttribute(const ParsedAttr &AL) {
   std::string NormalizedFullName = '\'' + AL.getNormalizedFullName() + '\'';
-  if (auto CorrectedFullName =
-          AL.getCorrectedFullName(Context.getTargetInfo(), getLangOpts())) {
-    Diag(AL.getNormalizedRange().getBegin(),
-         diag::warn_unknown_attribute_ignored_suggestion)
-        << NormalizedFullName << *CorrectedFullName << AL.getNormalizedRange();
+  SourceRange NR = AL.getNormalizedRange();
+
+  StringRef ScopeName = AL.getNormalizedScopeName();
+  std::optional<StringRef> CorrectedScopeName =
+      AL.tryGetCorrectedScopeName(ScopeName);
+  if (CorrectedScopeName) {
+    ScopeName = *CorrectedScopeName;
+  }
+
+  StringRef AttrName = AL.getNormalizedAttrName(ScopeName);
+  std::optional<StringRef> CorrectedAttrName = AL.tryGetCorrectedAttrName(
+      ScopeName, AttrName, Context.getTargetInfo(), getLangOpts());
+  if (CorrectedAttrName) {
+    AttrName = *CorrectedAttrName;
+  }
+
+  if (CorrectedScopeName || CorrectedAttrName) {
+    std::string CorrectedFullName =
+        AL.getNormalizedFullName(ScopeName, AttrName);
+    SemaDiagnosticBuilder D =
+        Diag(CorrectedScopeName ? NR.getBegin() : AL.getRange().getBegin(),
+             diag::warn_unknown_attribute_ignored_suggestion);
+
+    D << NormalizedFullName << CorrectedFullName;
+
+    if (AL.isExplicitScope()) {
+      D << FixItHint::CreateReplacement(NR, CorrectedFullName) << NR;
+    } else {
+      if (CorrectedScopeName) {
+        D << FixItHint::CreateReplacement(SourceRange(AL.getScopeLoc()),
+                                          ScopeName);
+      }
+      if (CorrectedAttrName) {
+        D << FixItHint::CreateReplacement(AL.getRange(), AttrName);
+      }
+    }
   } else {
-    Diag(AL.getNormalizedRange().getBegin(),
-         diag::warn_unknown_attribute_ignored)
-        << NormalizedFullName << AL.getNormalizedRange();
+    Diag(NR.getBegin(), diag::warn_unknown_attribute_ignored)
+        << NormalizedFullName << NR;
   }
 }
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaDeclCXX.cpp b/external/llvm-project/clang/lib/Sema/SemaDeclCXX.cpp
index 55e078f3180a..16645ecf411e 100644
--- a/external/llvm-project/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaDeclCXX.cpp
@@ -2752,8 +2752,7 @@ CXXBaseSpecifier *Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
       std::string Quals =
           BaseType.getQualifiers().getAsString(Context.getPrintingPolicy());
       Diag(BaseLoc, diag::warn_qual_base_type)
-          << Quals << std::count(Quals.begin(), Quals.end(), ' ') + 1
-          << BaseType;
+          << Quals << llvm::count(Quals, ' ') + 1 << BaseType;
       Diag(BaseLoc, diag::note_base_class_specified_here) << BaseType;
     }
 
@@ -2866,8 +2865,7 @@ BaseResult Sema::ActOnBaseSpecifier(Decl *classdecl, SourceRange SpecifierRange,
     if (AL.isInvalid() || AL.getKind() == ParsedAttr::IgnoredAttribute)
       continue;
     if (AL.getKind() == ParsedAttr::UnknownAttribute)
-      Diag(AL.getLoc(), diag::warn_unknown_attribute_ignored)
-          << AL << AL.getRange();
+      DiagnoseUnknownAttribute(AL);
     else
       Diag(AL.getLoc(), diag::err_base_specifier_attribute)
           << AL << AL.isRegularKeywordAttribute() << AL.getRange();
@@ -4155,10 +4153,6 @@ ExprResult Sema::ActOnRequiresClause(ExprResult ConstraintExpr) {
   if (ConstraintExpr.isInvalid())
     return ExprError();
 
-  ConstraintExpr = CorrectDelayedTyposInExpr(ConstraintExpr);
-  if (ConstraintExpr.isInvalid())
-    return ExprError();
-
   if (DiagnoseUnexpandedParameterPack(ConstraintExpr.get(),
                                       UPPC_RequiresClause))
     return ExprError();
@@ -4208,23 +4202,20 @@ void Sema::ActOnFinishCXXInClassMemberInitializer(Decl *D,
     return;
   }
 
-  ExprResult Init = CorrectDelayedTyposInExpr(InitExpr, /*InitDecl=*/nullptr,
-                                              /*RecoverUncorrectedTypos=*/true);
-  assert(Init.isUsable() && "Init should at least have a RecoveryExpr");
-  if (!FD->getType()->isDependentType() && !Init.get()->isTypeDependent()) {
-    Init = ConvertMemberDefaultInitExpression(FD, Init.get(), InitLoc);
+  if (!FD->getType()->isDependentType() && !InitExpr.get()->isTypeDependent()) {
+    InitExpr = ConvertMemberDefaultInitExpression(FD, InitExpr.get(), InitLoc);
     // C++11 [class.base.init]p7:
     //   The initialization of each base and member constitutes a
     //   full-expression.
-    if (!Init.isInvalid())
-      Init = ActOnFinishFullExpr(Init.get(), /*DiscarededValue=*/false);
-    if (Init.isInvalid()) {
+    if (!InitExpr.isInvalid())
+      InitExpr = ActOnFinishFullExpr(InitExpr.get(), /*DiscarededValue=*/false);
+    if (InitExpr.isInvalid()) {
       FD->setInvalidDecl();
       return;
     }
   }
 
-  FD->setInClassInitializer(Init.get());
+  FD->setInClassInitializer(InitExpr.get());
 }
 
 /// Find the direct and/or virtual base specifiers that
@@ -4394,13 +4385,7 @@ Sema::BuildMemInitializer(Decl *ConstructorD,
                           SourceLocation IdLoc,
                           Expr *Init,
                           SourceLocation EllipsisLoc) {
-  ExprResult Res = CorrectDelayedTyposInExpr(Init, /*InitDecl=*/nullptr,
-                                             /*RecoverUncorrectedTypos=*/true);
-  if (!Res.isUsable())
-    return true;
-  Init = Res.get();
-
-  if (!ConstructorD)
+  if (!ConstructorD || !Init)
     return true;
 
   AdjustDeclIfTemplate(ConstructorD);
@@ -7173,7 +7158,10 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
     // "effectively constexpr" for better compatibility.
     // See https://github.com/llvm/llvm-project/issues/102293 for more info.
     if (isa<CXXDestructorDecl>(M)) {
-      auto Check = [](QualType T, auto &&Check) -> bool {
+      llvm::SmallDenseSet<QualType> Visited;
+      auto Check = [&Visited](QualType T, auto &&Check) -> bool {
+        if (!Visited.insert(T->getCanonicalTypeUnqualified()).second)
+          return false;
         const CXXRecordDecl *RD =
             T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
         if (!RD || !RD->isCompleteDefinition())
@@ -7182,16 +7170,11 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
         if (!RD->hasConstexprDestructor())
           return false;
 
-        QualType CanUnqualT = T.getCanonicalType().getUnqualifiedType();
         for (const CXXBaseSpecifier &B : RD->bases())
-          if (B.getType().getCanonicalType().getUnqualifiedType() !=
-                  CanUnqualT &&
-              !Check(B.getType(), Check))
+          if (!Check(B.getType(), Check))
             return false;
         for (const FieldDecl *FD : RD->fields())
-          if (FD->getType().getCanonicalType().getUnqualifiedType() !=
-                  CanUnqualT &&
-              !Check(FD->getType(), Check))
+          if (!Check(FD->getType(), Check))
             return false;
         return true;
       };
@@ -10572,29 +10555,6 @@ void Sema::checkIllFormedTrivialABIStruct(CXXRecordDecl &RD) {
     RD.dropAttr<TrivialABIAttr>();
   };
 
-  // Ill-formed if the copy and move constructors are deleted.
-  auto HasNonDeletedCopyOrMoveConstructor = [&]() {
-    // If the type is dependent, then assume it might have
-    // implicit copy or move ctor because we won't know yet at this point.
-    if (RD.isDependentType())
-      return true;
-    if (RD.needsImplicitCopyConstructor() &&
-        !RD.defaultedCopyConstructorIsDeleted())
-      return true;
-    if (RD.needsImplicitMoveConstructor() &&
-        !RD.defaultedMoveConstructorIsDeleted())
-      return true;
-    for (const CXXConstructorDecl *CD : RD.ctors())
-      if (CD->isCopyOrMoveConstructor() && !CD->isDeleted())
-        return true;
-    return false;
-  };
-
-  if (!HasNonDeletedCopyOrMoveConstructor()) {
-    PrintDiagAndRemoveAttr(0);
-    return;
-  }
-
   // Ill-formed if the struct has virtual functions.
   if (RD.isPolymorphic()) {
     PrintDiagAndRemoveAttr(1);
@@ -10638,6 +10598,32 @@ void Sema::checkIllFormedTrivialABIStruct(CXXRecordDecl &RD) {
         return;
       }
   }
+
+  if (IsCXXTriviallyRelocatableType(RD))
+    return;
+
+  // Ill-formed if the copy and move constructors are deleted.
+  auto HasNonDeletedCopyOrMoveConstructor = [&]() {
+    // If the type is dependent, then assume it might have
+    // implicit copy or move ctor because we won't know yet at this point.
+    if (RD.isDependentType())
+      return true;
+    if (RD.needsImplicitCopyConstructor() &&
+        !RD.defaultedCopyConstructorIsDeleted())
+      return true;
+    if (RD.needsImplicitMoveConstructor() &&
+        !RD.defaultedMoveConstructorIsDeleted())
+      return true;
+    for (const CXXConstructorDecl *CD : RD.ctors())
+      if (CD->isCopyOrMoveConstructor() && !CD->isDeleted())
+        return true;
+    return false;
+  };
+
+  if (!HasNonDeletedCopyOrMoveConstructor()) {
+    PrintDiagAndRemoveAttr(0);
+    return;
+  }
 }
 
 void Sema::checkIncorrectVTablePointerAuthenticationAttribute(
diff --git a/external/llvm-project/clang/lib/Sema/SemaExceptionSpec.cpp b/external/llvm-project/clang/lib/Sema/SemaExceptionSpec.cpp
index c692f824da42..0a6cea8869c1 100644
--- a/external/llvm-project/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1368,7 +1368,6 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Expr::UnaryExprOrTypeTraitExprClass:
   case Expr::UnresolvedLookupExprClass:
   case Expr::UnresolvedMemberExprClass:
-  case Expr::TypoExprClass:
     // FIXME: Many of the above can throw.
     return CT_Cannot;
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaExpr.cpp b/external/llvm-project/clang/lib/Sema/SemaExpr.cpp
index 3aa710bb8e3b..4ef3de1e270c 100644
--- a/external/llvm-project/clang/lib/Sema/SemaExpr.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaExpr.cpp
@@ -397,17 +397,6 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, ArrayRef<SourceLocation> Locs,
           targetDiag(*Locs.begin(), diag::err_thread_unsupported);
   }
 
-  if (isa<ParmVarDecl>(D) && isa<RequiresExprBodyDecl>(D->getDeclContext()) &&
-      !isUnevaluatedContext()) {
-    // C++ [expr.prim.req.nested] p3
-    //   A local parameter shall only appear as an unevaluated operand
-    //   (Clause 8) within the constraint-expression.
-    Diag(Loc, diag::err_requires_expr_parameter_referenced_in_evaluated_context)
-        << D;
-    Diag(D->getLocation(), diag::note_entity_declared_at) << D;
-    return true;
-  }
-
   return false;
 }
 
@@ -2457,42 +2446,6 @@ Sema::DecomposeUnqualifiedId(const UnqualifiedId &Id,
   }
 }
 
-static void emitEmptyLookupTypoDiagnostic(const TypoCorrection &TC,
-                                          Sema &SemaRef, const CXXScopeSpec &SS,
-                                          DeclarationName Typo,
-                                          SourceRange TypoRange,
-                                          unsigned DiagnosticID,
-                                          unsigned DiagnosticSuggestID) {
-  DeclContext *Ctx =
-      SS.isEmpty() ? nullptr : SemaRef.computeDeclContext(SS, false);
-  if (!TC) {
-    // Emit a special diagnostic for failed member lookups.
-    // FIXME: computing the declaration context might fail here (?)
-    if (Ctx)
-      SemaRef.Diag(TypoRange.getBegin(), diag::err_no_member)
-          << Typo << Ctx << TypoRange;
-    else
-      SemaRef.Diag(TypoRange.getBegin(), DiagnosticID) << Typo << TypoRange;
-    return;
-  }
-
-  std::string CorrectedStr = TC.getAsString(SemaRef.getLangOpts());
-  bool DroppedSpecifier =
-      TC.WillReplaceSpecifier() && Typo.getAsString() == CorrectedStr;
-  unsigned NoteID = TC.getCorrectionDeclAs<ImplicitParamDecl>()
-                        ? diag::note_implicit_param_decl
-                        : diag::note_previous_decl;
-  if (!Ctx)
-    SemaRef.diagnoseTypo(
-        TC, SemaRef.PDiag(DiagnosticSuggestID) << Typo << TypoRange,
-        SemaRef.PDiag(NoteID));
-  else
-    SemaRef.diagnoseTypo(TC,
-                         SemaRef.PDiag(diag::err_no_member_suggest)
-                             << Typo << Ctx << DroppedSpecifier << TypoRange,
-                         SemaRef.PDiag(NoteID));
-}
-
 bool Sema::DiagnoseDependentMemberLookup(const LookupResult &R) {
   // During a default argument instantiation the CurContext points
   // to a CXXMethodDecl; but we can't apply a this-> fixit inside a
@@ -2555,8 +2508,7 @@ bool Sema::DiagnoseDependentMemberLookup(const LookupResult &R) {
 bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                                CorrectionCandidateCallback &CCC,
                                TemplateArgumentListInfo *ExplicitTemplateArgs,
-                               ArrayRef<Expr *> Args, DeclContext *LookupCtx,
-                               TypoExpr **Out) {
+                               ArrayRef<Expr *> Args, DeclContext *LookupCtx) {
   DeclarationName Name = R.getLookupName();
   SourceRange NameRange = R.getLookupNameInfo().getSourceRange();
 
@@ -2615,21 +2567,9 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
 
   // We didn't find anything, so try to correct for a typo.
   TypoCorrection Corrected;
-  if (S && Out) {
-    assert(!ExplicitTemplateArgs &&
-           "Diagnosing an empty lookup with explicit template args!");
-    *Out = CorrectTypoDelayed(
-        R.getLookupNameInfo(), R.getLookupKind(), S, &SS, CCC,
-        [=](const TypoCorrection &TC) {
-          emitEmptyLookupTypoDiagnostic(TC, *this, SS, Name, NameRange,
-                                        diagnostic, diagnostic_suggest);
-        },
-        nullptr, CorrectTypoKind::ErrorRecovery, LookupCtx);
-    if (*Out)
-      return true;
-  } else if (S && (Corrected = CorrectTypo(
-                       R.getLookupNameInfo(), R.getLookupKind(), S, &SS, CCC,
-                       CorrectTypoKind::ErrorRecovery, LookupCtx))) {
+  if (S && (Corrected =
+                CorrectTypo(R.getLookupNameInfo(), R.getLookupKind(), S, &SS,
+                            CCC, CorrectTypoKind::ErrorRecovery, LookupCtx))) {
     std::string CorrectedStr(Corrected.getAsString(getLangOpts()));
     bool DroppedSpecifier =
         Corrected.WillReplaceSpecifier() && Name.getAsString() == CorrectedStr;
@@ -2891,7 +2831,6 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
 
     // If this name wasn't predeclared and if this is not a function
     // call, diagnose the problem.
-    TypoExpr *TE = nullptr;
     DefaultFilterCCC DefaultValidator(II, SS.isValid() ? SS.getScopeRep()
                                                        : nullptr);
     DefaultValidator.IsAddressOfOperand = IsAddressOfOperand;
@@ -2907,29 +2846,8 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
     // a template name, but we happen to have always already looked up the name
     // before we get here if it must be a template name.
     if (DiagnoseEmptyLookup(S, SS, R, CCC ? *CCC : DefaultValidator, nullptr,
-                            {}, nullptr, &TE)) {
-      if (TE && KeywordReplacement) {
-        auto &State = getTypoExprState(TE);
-        auto BestTC = State.Consumer->getNextCorrection();
-        if (BestTC.isKeyword()) {
-          auto *II = BestTC.getCorrectionAsIdentifierInfo();
-          if (State.DiagHandler)
-            State.DiagHandler(BestTC);
-          KeywordReplacement->startToken();
-          KeywordReplacement->setKind(II->getTokenID());
-          KeywordReplacement->setIdentifierInfo(II);
-          KeywordReplacement->setLocation(BestTC.getCorrectionRange().getBegin());
-          // Clean up the state associated with the TypoExpr, since it has
-          // now been diagnosed (without a call to CorrectDelayedTyposInExpr).
-          clearDelayedTypo(TE);
-          // Signal that a correction to a keyword was performed by returning a
-          // valid-but-null ExprResult.
-          return (Expr*)nullptr;
-        }
-        State.Consumer->resetCorrectionStream();
-      }
-      return TE ? TE : ExprError();
-    }
+                            {}, nullptr))
+      return ExprError();
 
     assert(!R.empty() &&
            "DiagnoseEmptyLookup returned false but added no results");
@@ -6395,7 +6313,8 @@ static FunctionDecl *rewriteBuiltinFunctionDecl(Sema *Sema, ASTContext &Context,
       return nullptr;
     Expr *Arg = ArgRes.get();
     QualType ArgType = Arg->getType();
-    if (!ParamType->isPointerType() || ParamType.hasAddressSpace() ||
+    if (!ParamType->isPointerType() ||
+        ParamType->getPointeeType().hasAddressSpace() ||
         !ArgType->isPointerType() ||
         !ArgType->getPointeeType().hasAddressSpace() ||
         isPtrSizeAddressSpace(ArgType->getPointeeType().getAddressSpace())) {
@@ -6404,9 +6323,6 @@ static FunctionDecl *rewriteBuiltinFunctionDecl(Sema *Sema, ASTContext &Context,
     }
 
     QualType PointeeType = ParamType->getPointeeType();
-    if (PointeeType.hasAddressSpace())
-      continue;
-
     NeedsNewDecl = true;
     LangAS AS = ArgType->getPointeeType().getAddressSpace();
 
@@ -7038,40 +6954,6 @@ ExprResult Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
                          CurFPFeatureOverrides(), NumParams, UsesADL);
   }
 
-  if (!Context.isDependenceAllowed()) {
-    // Forget about the nulled arguments since typo correction
-    // do not handle them well.
-    TheCall->shrinkNumArgs(Args.size());
-    // C cannot always handle TypoExpr nodes in builtin calls and direct
-    // function calls as their argument checking don't necessarily handle
-    // dependent types properly, so make sure any TypoExprs have been
-    // dealt with.
-    ExprResult Result = CorrectDelayedTyposInExpr(TheCall);
-    if (!Result.isUsable()) return ExprError();
-    CallExpr *TheOldCall = TheCall;
-    TheCall = dyn_cast<CallExpr>(Result.get());
-    bool CorrectedTypos = TheCall != TheOldCall;
-    if (!TheCall) return Result;
-    Args = llvm::ArrayRef(TheCall->getArgs(), TheCall->getNumArgs());
-
-    // A new call expression node was created if some typos were corrected.
-    // However it may not have been constructed with enough storage. In this
-    // case, rebuild the node with enough storage. The waste of space is
-    // immaterial since this only happens when some typos were corrected.
-    if (CorrectedTypos && Args.size() < NumParams) {
-      if (Config)
-        TheCall = CUDAKernelCallExpr::Create(
-            Context, Fn, cast<CallExpr>(Config), Args, ResultTy, VK_PRValue,
-            RParenLoc, CurFPFeatureOverrides(), NumParams);
-      else
-        TheCall =
-            CallExpr::Create(Context, Fn, Args, ResultTy, VK_PRValue, RParenLoc,
-                             CurFPFeatureOverrides(), NumParams, UsesADL);
-    }
-    // We can now handle the nulled arguments for the default arguments.
-    TheCall->setNumArgsUnsafe(std::max<unsigned>(Args.size(), NumParams));
-  }
-
   // Bail out early if calling a builtin with custom type checking.
   if (BuiltinID && Context.BuiltinInfo.hasCustomTypechecking(BuiltinID)) {
     ExprResult E = CheckBuiltinFunctionCall(FDecl, BuiltinID, TheCall);
@@ -7962,12 +7844,6 @@ Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
   if (getLangOpts().CPlusPlus) {
     // Check that there are no default arguments (C++ only).
     CheckExtraCXXDefaultArguments(D);
-  } else {
-    // Make sure any TypoExprs have been dealt with.
-    ExprResult Res = CorrectDelayedTyposInExpr(CastExpr);
-    if (!Res.isUsable())
-      return ExprError();
-    CastExpr = Res.get();
   }
 
   checkUnusedDeclAttributes(D);
@@ -9013,30 +8889,6 @@ ExprResult Sema::ActOnConditionalOp(SourceLocation QuestionLoc,
                                     SourceLocation ColonLoc,
                                     Expr *CondExpr, Expr *LHSExpr,
                                     Expr *RHSExpr) {
-  if (!Context.isDependenceAllowed()) {
-    // C cannot handle TypoExpr nodes in the condition because it
-    // doesn't handle dependent types properly, so make sure any TypoExprs have
-    // been dealt with before checking the operands.
-    ExprResult CondResult = CorrectDelayedTyposInExpr(CondExpr);
-    ExprResult LHSResult = CorrectDelayedTyposInExpr(LHSExpr);
-    ExprResult RHSResult = CorrectDelayedTyposInExpr(RHSExpr);
-
-    if (!CondResult.isUsable())
-      return ExprError();
-
-    if (LHSExpr) {
-      if (!LHSResult.isUsable())
-        return ExprError();
-    }
-
-    if (!RHSResult.isUsable())
-      return ExprError();
-
-    CondExpr = CondResult.get();
-    LHSExpr = LHSResult.get();
-    RHSExpr = RHSResult.get();
-  }
-
   // If this is the gnu "x ?: y" extension, analyze the types as though the LHS
   // was the condition.
   OpaqueValueExpr *opaqueValue = nullptr;
@@ -9299,8 +9151,14 @@ static AssignConvertType checkPointerTypesForAssignment(Sema &S,
       return AssignConvertType::IncompatibleFunctionPointer;
     return AssignConvertType::IncompatiblePointer;
   }
-  if (!S.getLangOpts().CPlusPlus && S.IsFunctionConversion(ltrans, rtrans))
-    return AssignConvertType::IncompatibleFunctionPointer;
+  bool DiscardingCFIUncheckedCallee, AddingCFIUncheckedCallee;
+  if (!S.getLangOpts().CPlusPlus &&
+      S.IsFunctionConversion(ltrans, rtrans, &DiscardingCFIUncheckedCallee,
+                             &AddingCFIUncheckedCallee)) {
+    // Allow conversions between CFIUncheckedCallee-ness.
+    if (!DiscardingCFIUncheckedCallee && !AddingCFIUncheckedCallee)
+      return AssignConvertType::IncompatibleFunctionPointer;
+  }
   if (IsInvalidCmseNSCallConversion(S, ltrans, rtrans))
     return AssignConvertType::IncompatibleFunctionPointer;
   if (S.IsInvalidSMECallConversion(rtrans, ltrans))
@@ -12756,9 +12614,17 @@ QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS,
       LangAS AddrSpaceR = RCanPointeeTy.getAddressSpace();
       CastKind Kind = AddrSpaceL != AddrSpaceR ? CK_AddressSpaceConversion
                                                : CK_BitCast;
+
+      const FunctionType *LFn = LCanPointeeTy->getAs<FunctionType>();
+      const FunctionType *RFn = RCanPointeeTy->getAs<FunctionType>();
+      bool LHSHasCFIUncheckedCallee = LFn && LFn->getCFIUncheckedCalleeAttr();
+      bool RHSHasCFIUncheckedCallee = RFn && RFn->getCFIUncheckedCalleeAttr();
+      bool ChangingCFIUncheckedCallee =
+          LHSHasCFIUncheckedCallee != RHSHasCFIUncheckedCallee;
+
       if (LHSIsNull && !RHSIsNull)
         LHS = ImpCastExprToType(LHS.get(), RHSType, Kind);
-      else
+      else if (!ChangingCFIUncheckedCallee)
         RHS = ImpCastExprToType(RHS.get(), LHSType, Kind);
     }
     return computeResultTy();
@@ -15053,18 +14919,6 @@ static void checkObjCPointerIntrospection(Sema &S, ExprResult &L, ExprResult &R,
   }
 }
 
-static NamedDecl *getDeclFromExpr(Expr *E) {
-  if (!E)
-    return nullptr;
-  if (auto *DRE = dyn_cast<DeclRefExpr>(E))
-    return DRE->getDecl();
-  if (auto *ME = dyn_cast<MemberExpr>(E))
-    return ME->getMemberDecl();
-  if (auto *IRE = dyn_cast<ObjCIvarRefExpr>(E))
-    return IRE->getDecl();
-  return nullptr;
-}
-
 // This helper function promotes a binary operator's operands (which are of a
 // half vector type) to a vector of floats and then truncates the result to
 // a vector of either half or short.
@@ -15100,28 +14954,6 @@ static ExprResult convertHalfVecBinOp(Sema &S, ExprResult LHS, ExprResult RHS,
   return convertVector(BO, ResultTy->castAs<VectorType>()->getElementType(), S);
 }
 
-static std::pair<ExprResult, ExprResult>
-CorrectDelayedTyposInBinOp(Sema &S, BinaryOperatorKind Opc, Expr *LHSExpr,
-                           Expr *RHSExpr) {
-  ExprResult LHS = LHSExpr, RHS = RHSExpr;
-  if (!S.Context.isDependenceAllowed()) {
-    // C cannot handle TypoExpr nodes on either side of a binop because it
-    // doesn't handle dependent types properly, so make sure any TypoExprs have
-    // been dealt with before checking the operands.
-    LHS = S.CorrectDelayedTyposInExpr(LHS);
-    RHS = S.CorrectDelayedTyposInExpr(
-        RHS, /*InitDecl=*/nullptr, /*RecoverUncorrectedTypos=*/false,
-        [Opc, LHS](Expr *E) {
-          if (Opc != BO_Assign)
-            return ExprResult(E);
-          // Avoid correcting the RHS to the same Expr as the LHS.
-          Decl *D = getDeclFromExpr(E);
-          return (D && D == getDeclFromExpr(LHS.get())) ? ExprError() : E;
-        });
-  }
-  return std::make_pair(LHS, RHS);
-}
-
 /// Returns true if conversion between vectors of halfs and vectors of floats
 /// is needed.
 static bool needsConversionOfHalfVec(bool OpRequiresConversion, ASTContext &Ctx,
@@ -15178,7 +15010,6 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   ExprObjectKind OK = OK_Ordinary;
   bool ConvertHalfVec = false;
 
-  std::tie(LHS, RHS) = CorrectDelayedTyposInBinOp(*this, Opc, LHSExpr, RHSExpr);
   if (!LHS.isUsable() || !RHS.isUsable())
     return ExprError();
 
@@ -15694,12 +15525,8 @@ static ExprResult BuildOverloadedBinOp(Sema &S, Scope *Sc, SourceLocation OpLoc,
 ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
                             BinaryOperatorKind Opc, Expr *LHSExpr,
                             Expr *RHSExpr, bool ForFoldExpression) {
-  ExprResult LHS, RHS;
-  std::tie(LHS, RHS) = CorrectDelayedTyposInBinOp(*this, Opc, LHSExpr, RHSExpr);
-  if (!LHS.isUsable() || !RHS.isUsable())
+  if (!LHSExpr || !RHSExpr)
     return ExprError();
-  LHSExpr = LHS.get();
-  RHSExpr = RHS.get();
 
   // We want to end up calling one of SemaPseudoObject::checkAssignment
   // (if the LHS is a pseudo-object), BuildOverloadedBinOp (if
@@ -18260,8 +18087,6 @@ HandleImmediateInvocations(Sema &SemaRef,
 
 void Sema::PopExpressionEvaluationContext() {
   ExpressionEvaluationContextRecord& Rec = ExprEvalContexts.back();
-  unsigned NumTypos = Rec.NumTypos;
-
   if (!Rec.Lambdas.empty()) {
     using ExpressionKind = ExpressionEvaluationContextRecord::ExpressionKind;
     if (!getLangOpts().CPlusPlus20 &&
@@ -18329,9 +18154,6 @@ void Sema::PopExpressionEvaluationContext() {
 
   // Pop the current expression evaluation context off the stack.
   ExprEvalContexts.pop_back();
-
-  // The global expression evaluation context record is never popped.
-  ExprEvalContexts.back().NumTypos += NumTypos;
 }
 
 void Sema::DiscardCleanupsInEvaluationContext() {
@@ -20089,8 +19911,6 @@ ExprResult Sema::CheckLValueToRValueConversionOperand(Expr *E) {
 }
 
 ExprResult Sema::ActOnConstantExpression(ExprResult Res) {
-  Res = CorrectDelayedTyposInExpr(Res);
-
   if (!Res.isUsable())
     return Res;
 
@@ -21506,15 +21326,6 @@ static ExprResult diagnoseUnknownAnyExpr(Sema &S, Expr *E) {
 }
 
 ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
-  if (!Context.isDependenceAllowed()) {
-    // C cannot handle TypoExpr nodes on either side of a binop because it
-    // doesn't handle dependent types properly, so make sure any TypoExprs have
-    // been dealt with before checking the operands.
-    ExprResult Result = CorrectDelayedTyposInExpr(E);
-    if (!Result.isUsable()) return ExprError();
-    E = Result.get();
-  }
-
   const BuiltinType *placeholderType = E->getType()->getAsPlaceholderType();
   if (!placeholderType) return E;
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaExprCXX.cpp b/external/llvm-project/clang/lib/Sema/SemaExprCXX.cpp
index 7b08fb869ccc..4a86cbd0633b 100644
--- a/external/llvm-project/clang/lib/Sema/SemaExprCXX.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaExprCXX.cpp
@@ -40,7 +40,6 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaHLSL.h"
-#include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaLambda.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaPPC.h"
@@ -1500,13 +1499,7 @@ Sema::ActOnCXXTypeConstructExpr(ParsedType TypeRep,
 
   auto Result = BuildCXXTypeConstructExpr(TInfo, LParenOrBraceLoc, exprs,
                                           RParenOrBraceLoc, ListInitialization);
-  // Avoid creating a non-type-dependent expression that contains typos.
-  // Non-type-dependent expressions are liable to be discarded without
-  // checking for embedded typos.
-  if (!Result.isInvalid() && Result.get()->isInstantiationDependent() &&
-      !Result.get()->isTypeDependent())
-    Result = CorrectDelayedTyposInExpr(Result.get());
-  else if (Result.isInvalid())
+  if (Result.isInvalid())
     Result = CreateRecoveryExpr(TInfo->getTypeLoc().getBeginLoc(),
                                 RParenOrBraceLoc, exprs, Ty);
   return Result;
@@ -2159,6 +2152,10 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
     assert(InitStyle == CXXNewInitializationStyle::Parens &&
            "paren init for non-call init");
     Exprs = MultiExprArg(List->getExprs(), List->getNumExprs());
+  } else if (auto *List = dyn_cast_or_null<CXXParenListInitExpr>(Initializer)) {
+    assert(InitStyle == CXXNewInitializationStyle::Parens &&
+           "paren init for non-call init");
+    Exprs = List->getInitExprs();
   }
 
   // C++11 [expr.new]p15:
@@ -2884,7 +2881,7 @@ static bool resolveAllocationOverload(
     // type-identity-less argument list.
     IAP.PassTypeIdentity = TypeAwareAllocationMode::No;
     IAP.PassAlignment = InitialAlignmentMode;
-    Args = UntypedParameters;
+    Args = std::move(UntypedParameters);
   }
   assert(!S.isStdTypeIdentity(Args[0]->getType(), nullptr));
   return resolveAllocationOverloadInterior(
@@ -6407,6 +6404,11 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
         EPI1.ExtInfo = EPI1.ExtInfo.withNoReturn(Noreturn);
         EPI2.ExtInfo = EPI2.ExtInfo.withNoReturn(Noreturn);
 
+        bool CFIUncheckedCallee =
+            EPI1.CFIUncheckedCallee || EPI2.CFIUncheckedCallee;
+        EPI1.CFIUncheckedCallee = CFIUncheckedCallee;
+        EPI2.CFIUncheckedCallee = CFIUncheckedCallee;
+
         // The result is nothrow if both operands are.
         SmallVector<QualType, 8> ExceptionTypeStorage;
         EPI1.ExceptionSpec = EPI2.ExceptionSpec = Context.mergeExceptionSpecs(
@@ -7644,417 +7646,6 @@ static void CheckIfAnyEnclosingLambdasMustCaptureAnyPotentialCaptures(
   CurrentLSI->clearPotentialCaptures();
 }
 
-static ExprResult attemptRecovery(Sema &SemaRef,
-                                  const TypoCorrectionConsumer &Consumer,
-                                  const TypoCorrection &TC) {
-  LookupResult R(SemaRef, Consumer.getLookupResult().getLookupNameInfo(),
-                 Consumer.getLookupResult().getLookupKind());
-  const CXXScopeSpec *SS = Consumer.getSS();
-  CXXScopeSpec NewSS;
-
-  // Use an approprate CXXScopeSpec for building the expr.
-  if (auto *NNS = TC.getCorrectionSpecifier())
-    NewSS.MakeTrivial(SemaRef.Context, NNS, TC.getCorrectionRange());
-  else if (SS && !TC.WillReplaceSpecifier())
-    NewSS = *SS;
-
-  if (auto *ND = TC.getFoundDecl()) {
-    R.setLookupName(ND->getDeclName());
-    R.addDecl(ND);
-    if (ND->isCXXClassMember()) {
-      // Figure out the correct naming class to add to the LookupResult.
-      CXXRecordDecl *Record = nullptr;
-      if (auto *NNS = TC.getCorrectionSpecifier())
-        Record = NNS->getAsType()->getAsCXXRecordDecl();
-      if (!Record)
-        Record =
-            dyn_cast<CXXRecordDecl>(ND->getDeclContext()->getRedeclContext());
-      if (Record)
-        R.setNamingClass(Record);
-
-      // Detect and handle the case where the decl might be an implicit
-      // member.
-      if (SemaRef.isPotentialImplicitMemberAccess(
-              NewSS, R, Consumer.isAddressOfOperand()))
-        return SemaRef.BuildPossibleImplicitMemberExpr(
-            NewSS, /*TemplateKWLoc*/ SourceLocation(), R,
-            /*TemplateArgs*/ nullptr, /*S*/ nullptr);
-    } else if (auto *Ivar = dyn_cast<ObjCIvarDecl>(ND)) {
-      return SemaRef.ObjC().LookupInObjCMethod(R, Consumer.getScope(),
-                                               Ivar->getIdentifier());
-    }
-  }
-
-  return SemaRef.BuildDeclarationNameExpr(NewSS, R, /*NeedsADL*/ false,
-                                          /*AcceptInvalidDecl*/ true);
-}
-
-namespace {
-class FindTypoExprs : public DynamicRecursiveASTVisitor {
-  llvm::SmallSetVector<TypoExpr *, 2> &TypoExprs;
-
-public:
-  explicit FindTypoExprs(llvm::SmallSetVector<TypoExpr *, 2> &TypoExprs)
-      : TypoExprs(TypoExprs) {}
-  bool VisitTypoExpr(TypoExpr *TE) override {
-    TypoExprs.insert(TE);
-    return true;
-  }
-};
-
-class TransformTypos : public TreeTransform<TransformTypos> {
-  typedef TreeTransform<TransformTypos> BaseTransform;
-
-  VarDecl *InitDecl; // A decl to avoid as a correction because it is in the
-                     // process of being initialized.
-  llvm::function_ref<ExprResult(Expr *)> ExprFilter;
-  llvm::SmallSetVector<TypoExpr *, 2> TypoExprs, AmbiguousTypoExprs;
-  llvm::SmallDenseMap<TypoExpr *, ExprResult, 2> TransformCache;
-  llvm::SmallDenseMap<OverloadExpr *, Expr *, 4> OverloadResolution;
-
-  /// Emit diagnostics for all of the TypoExprs encountered.
-  ///
-  /// If the TypoExprs were successfully corrected, then the diagnostics should
-  /// suggest the corrections. Otherwise the diagnostics will not suggest
-  /// anything (having been passed an empty TypoCorrection).
-  ///
-  /// If we've failed to correct due to ambiguous corrections, we need to
-  /// be sure to pass empty corrections and replacements. Otherwise it's
-  /// possible that the Consumer has a TypoCorrection that failed to ambiguity
-  /// and we don't want to report those diagnostics.
-  void EmitAllDiagnostics(bool IsAmbiguous) {
-    for (TypoExpr *TE : TypoExprs) {
-      auto &State = SemaRef.getTypoExprState(TE);
-      if (State.DiagHandler) {
-        TypoCorrection TC = IsAmbiguous
-            ? TypoCorrection() : State.Consumer->getCurrentCorrection();
-        ExprResult Replacement = IsAmbiguous ? ExprError() : TransformCache[TE];
-
-        // Extract the NamedDecl from the transformed TypoExpr and add it to the
-        // TypoCorrection, replacing the existing decls. This ensures the right
-        // NamedDecl is used in diagnostics e.g. in the case where overload
-        // resolution was used to select one from several possible decls that
-        // had been stored in the TypoCorrection.
-        if (auto *ND = getDeclFromExpr(
-                Replacement.isInvalid() ? nullptr : Replacement.get()))
-          TC.setCorrectionDecl(ND);
-
-        State.DiagHandler(TC);
-      }
-      SemaRef.clearDelayedTypo(TE);
-    }
-  }
-
-  /// Try to advance the typo correction state of the first unfinished TypoExpr.
-  /// We allow advancement of the correction stream by removing it from the
-  /// TransformCache which allows `TransformTypoExpr` to advance during the
-  /// next transformation attempt.
-  ///
-  /// Any substitution attempts for the previous TypoExprs (which must have been
-  /// finished) will need to be retried since it's possible that they will now
-  /// be invalid given the latest advancement.
-  ///
-  /// We need to be sure that we're making progress - it's possible that the
-  /// tree is so malformed that the transform never makes it to the
-  /// `TransformTypoExpr`.
-  ///
-  /// Returns true if there are any untried correction combinations.
-  bool CheckAndAdvanceTypoExprCorrectionStreams() {
-    for (auto *TE : TypoExprs) {
-      auto &State = SemaRef.getTypoExprState(TE);
-      TransformCache.erase(TE);
-      if (!State.Consumer->hasMadeAnyCorrectionProgress())
-        return false;
-      if (!State.Consumer->finished())
-        return true;
-      State.Consumer->resetCorrectionStream();
-    }
-    return false;
-  }
-
-  NamedDecl *getDeclFromExpr(Expr *E) {
-    if (auto *OE = dyn_cast_or_null<OverloadExpr>(E))
-      E = OverloadResolution[OE];
-
-    if (!E)
-      return nullptr;
-    if (auto *DRE = dyn_cast<DeclRefExpr>(E))
-      return DRE->getFoundDecl();
-    if (auto *ME = dyn_cast<MemberExpr>(E))
-      return ME->getFoundDecl();
-    // FIXME: Add any other expr types that could be seen by the delayed typo
-    // correction TreeTransform for which the corresponding TypoCorrection could
-    // contain multiple decls.
-    return nullptr;
-  }
-
-  ExprResult TryTransform(Expr *E) {
-    Sema::SFINAETrap Trap(SemaRef);
-    ExprResult Res = TransformExpr(E);
-    if (Trap.hasErrorOccurred() || Res.isInvalid())
-      return ExprError();
-
-    return ExprFilter(Res.get());
-  }
-
-  // Since correcting typos may intoduce new TypoExprs, this function
-  // checks for new TypoExprs and recurses if it finds any. Note that it will
-  // only succeed if it is able to correct all typos in the given expression.
-  ExprResult CheckForRecursiveTypos(ExprResult Res, bool &IsAmbiguous) {
-    if (Res.isInvalid()) {
-      return Res;
-    }
-    // Check to see if any new TypoExprs were created. If so, we need to recurse
-    // to check their validity.
-    Expr *FixedExpr = Res.get();
-
-    auto SavedTypoExprs = std::move(TypoExprs);
-    auto SavedAmbiguousTypoExprs = std::move(AmbiguousTypoExprs);
-    TypoExprs.clear();
-    AmbiguousTypoExprs.clear();
-
-    FindTypoExprs(TypoExprs).TraverseStmt(FixedExpr);
-    if (!TypoExprs.empty()) {
-      // Recurse to handle newly created TypoExprs. If we're not able to
-      // handle them, discard these TypoExprs.
-      ExprResult RecurResult =
-          RecursiveTransformLoop(FixedExpr, IsAmbiguous);
-      if (RecurResult.isInvalid()) {
-        Res = ExprError();
-        // Recursive corrections didn't work, wipe them away and don't add
-        // them to the TypoExprs set. Remove them from Sema's TypoExpr list
-        // since we don't want to clear them twice. Note: it's possible the
-        // TypoExprs were created recursively and thus won't be in our
-        // Sema's TypoExprs - they were created in our `RecursiveTransformLoop`.
-        auto &SemaTypoExprs = SemaRef.TypoExprs;
-        for (auto *TE : TypoExprs) {
-          TransformCache.erase(TE);
-          SemaRef.clearDelayedTypo(TE);
-
-          auto SI = find(SemaTypoExprs, TE);
-          if (SI != SemaTypoExprs.end()) {
-            SemaTypoExprs.erase(SI);
-          }
-        }
-      } else {
-        // TypoExpr is valid: add newly created TypoExprs since we were
-        // able to correct them.
-        Res = RecurResult;
-        SavedTypoExprs.set_union(TypoExprs);
-      }
-    }
-
-    TypoExprs = std::move(SavedTypoExprs);
-    AmbiguousTypoExprs = std::move(SavedAmbiguousTypoExprs);
-
-    return Res;
-  }
-
-  // Try to transform the given expression, looping through the correction
-  // candidates with `CheckAndAdvanceTypoExprCorrectionStreams`.
-  //
-  // If valid ambiguous typo corrections are seen, `IsAmbiguous` is set to
-  // true and this method immediately will return an `ExprError`.
-  ExprResult RecursiveTransformLoop(Expr *E, bool &IsAmbiguous) {
-    ExprResult Res;
-    auto SavedTypoExprs = std::move(SemaRef.TypoExprs);
-    SemaRef.TypoExprs.clear();
-
-    while (true) {
-      Res = CheckForRecursiveTypos(TryTransform(E), IsAmbiguous);
-
-      // Recursion encountered an ambiguous correction. This means that our
-      // correction itself is ambiguous, so stop now.
-      if (IsAmbiguous)
-        break;
-
-      // If the transform is still valid after checking for any new typos,
-      // it's good to go.
-      if (!Res.isInvalid())
-        break;
-
-      // The transform was invalid, see if we have any TypoExprs with untried
-      // correction candidates.
-      if (!CheckAndAdvanceTypoExprCorrectionStreams())
-        break;
-    }
-
-    // If we found a valid result, double check to make sure it's not ambiguous.
-    if (!IsAmbiguous && !Res.isInvalid() && !AmbiguousTypoExprs.empty()) {
-      auto SavedTransformCache =
-          llvm::SmallDenseMap<TypoExpr *, ExprResult, 2>(TransformCache);
-
-      // Ensure none of the TypoExprs have multiple typo correction candidates
-      // with the same edit length that pass all the checks and filters.
-      while (!AmbiguousTypoExprs.empty()) {
-        auto TE  = AmbiguousTypoExprs.back();
-
-        // TryTransform itself can create new Typos, adding them to the TypoExpr map
-        // and invalidating our TypoExprState, so always fetch it instead of storing.
-        SemaRef.getTypoExprState(TE).Consumer->saveCurrentPosition();
-
-        TypoCorrection TC = SemaRef.getTypoExprState(TE).Consumer->peekNextCorrection();
-        TypoCorrection Next;
-        do {
-          // Fetch the next correction by erasing the typo from the cache and calling
-          // `TryTransform` which will iterate through corrections in
-          // `TransformTypoExpr`.
-          TransformCache.erase(TE);
-          ExprResult AmbigRes = CheckForRecursiveTypos(TryTransform(E), IsAmbiguous);
-
-          if (!AmbigRes.isInvalid() || IsAmbiguous) {
-            SemaRef.getTypoExprState(TE).Consumer->resetCorrectionStream();
-            SavedTransformCache.erase(TE);
-            Res = ExprError();
-            IsAmbiguous = true;
-            break;
-          }
-        } while ((Next = SemaRef.getTypoExprState(TE).Consumer->peekNextCorrection()) &&
-                 Next.getEditDistance(false) == TC.getEditDistance(false));
-
-        if (IsAmbiguous)
-          break;
-
-        AmbiguousTypoExprs.remove(TE);
-        SemaRef.getTypoExprState(TE).Consumer->restoreSavedPosition();
-        TransformCache[TE] = SavedTransformCache[TE];
-      }
-      TransformCache = std::move(SavedTransformCache);
-    }
-
-    // Wipe away any newly created TypoExprs that we don't know about. Since we
-    // clear any invalid TypoExprs in `CheckForRecursiveTypos`, this is only
-    // possible if a `TypoExpr` is created during a transformation but then
-    // fails before we can discover it.
-    auto &SemaTypoExprs = SemaRef.TypoExprs;
-    for (auto Iterator = SemaTypoExprs.begin(); Iterator != SemaTypoExprs.end();) {
-      auto TE = *Iterator;
-      auto FI = find(TypoExprs, TE);
-      if (FI != TypoExprs.end()) {
-        Iterator++;
-        continue;
-      }
-      SemaRef.clearDelayedTypo(TE);
-      Iterator = SemaTypoExprs.erase(Iterator);
-    }
-    SemaRef.TypoExprs = std::move(SavedTypoExprs);
-
-    return Res;
-  }
-
-public:
-  TransformTypos(Sema &SemaRef, VarDecl *InitDecl, llvm::function_ref<ExprResult(Expr *)> Filter)
-      : BaseTransform(SemaRef), InitDecl(InitDecl), ExprFilter(Filter) {}
-
-  ExprResult RebuildCallExpr(Expr *Callee, SourceLocation LParenLoc,
-                                   MultiExprArg Args,
-                                   SourceLocation RParenLoc,
-                                   Expr *ExecConfig = nullptr) {
-    auto Result = BaseTransform::RebuildCallExpr(Callee, LParenLoc, Args,
-                                                 RParenLoc, ExecConfig);
-    if (auto *OE = dyn_cast<OverloadExpr>(Callee)) {
-      if (Result.isUsable()) {
-        Expr *ResultCall = Result.get();
-        if (auto *BE = dyn_cast<CXXBindTemporaryExpr>(ResultCall))
-          ResultCall = BE->getSubExpr();
-        if (auto *CE = dyn_cast<CallExpr>(ResultCall))
-          OverloadResolution[OE] = CE->getCallee();
-      }
-    }
-    return Result;
-  }
-
-  ExprResult TransformLambdaExpr(LambdaExpr *E) { return Owned(E); }
-
-  ExprResult TransformBlockExpr(BlockExpr *E) { return Owned(E); }
-
-  ExprResult Transform(Expr *E) {
-    bool IsAmbiguous = false;
-    ExprResult Res = RecursiveTransformLoop(E, IsAmbiguous);
-
-    if (!Res.isUsable())
-      FindTypoExprs(TypoExprs).TraverseStmt(E);
-
-    EmitAllDiagnostics(IsAmbiguous);
-
-    return Res;
-  }
-
-  ExprResult TransformTypoExpr(TypoExpr *E) {
-    // If the TypoExpr hasn't been seen before, record it. Otherwise, return the
-    // cached transformation result if there is one and the TypoExpr isn't the
-    // first one that was encountered.
-    auto &CacheEntry = TransformCache[E];
-    if (!TypoExprs.insert(E) && !CacheEntry.isUnset()) {
-      return CacheEntry;
-    }
-
-    auto &State = SemaRef.getTypoExprState(E);
-    assert(State.Consumer && "Cannot transform a cleared TypoExpr");
-
-    // For the first TypoExpr and an uncached TypoExpr, find the next likely
-    // typo correction and return it.
-    while (TypoCorrection TC = State.Consumer->getNextCorrection()) {
-      if (InitDecl && TC.getFoundDecl() == InitDecl)
-        continue;
-      // FIXME: If we would typo-correct to an invalid declaration, it's
-      // probably best to just suppress all errors from this typo correction.
-      ExprResult NE = State.RecoveryHandler ?
-          State.RecoveryHandler(SemaRef, E, TC) :
-          attemptRecovery(SemaRef, *State.Consumer, TC);
-      if (!NE.isInvalid()) {
-        // Check whether there may be a second viable correction with the same
-        // edit distance; if so, remember this TypoExpr may have an ambiguous
-        // correction so it can be more thoroughly vetted later.
-        TypoCorrection Next;
-        if ((Next = State.Consumer->peekNextCorrection()) &&
-            Next.getEditDistance(false) == TC.getEditDistance(false)) {
-          AmbiguousTypoExprs.insert(E);
-        } else {
-          AmbiguousTypoExprs.remove(E);
-        }
-        assert(!NE.isUnset() &&
-               "Typo was transformed into a valid-but-null ExprResult");
-        return CacheEntry = NE;
-      }
-    }
-    return CacheEntry = ExprError();
-  }
-};
-}
-
-ExprResult
-Sema::CorrectDelayedTyposInExpr(Expr *E, VarDecl *InitDecl,
-                                bool RecoverUncorrectedTypos,
-                                llvm::function_ref<ExprResult(Expr *)> Filter) {
-  // If the current evaluation context indicates there are uncorrected typos
-  // and the current expression isn't guaranteed to not have typos, try to
-  // resolve any TypoExpr nodes that might be in the expression.
-  if (E && !ExprEvalContexts.empty() && ExprEvalContexts.back().NumTypos &&
-      (E->isTypeDependent() || E->isValueDependent() ||
-       E->isInstantiationDependent())) {
-    auto TyposResolved = DelayedTypos.size();
-    auto Result = TransformTypos(*this, InitDecl, Filter).Transform(E);
-    TyposResolved -= DelayedTypos.size();
-    if (Result.isInvalid() || Result.get() != E) {
-      ExprEvalContexts.back().NumTypos -= TyposResolved;
-      if (Result.isInvalid() && RecoverUncorrectedTypos) {
-        struct TyposReplace : TreeTransform<TyposReplace> {
-          TyposReplace(Sema &SemaRef) : TreeTransform(SemaRef) {}
-          ExprResult TransformTypoExpr(clang::TypoExpr *E) {
-            return this->SemaRef.CreateRecoveryExpr(E->getBeginLoc(),
-                                                    E->getEndLoc(), {});
-          }
-        } TT(*this);
-        return TT.TransformExpr(E);
-      }
-      return Result;
-    }
-    assert(TyposResolved == 0 && "Corrected typo but got same Expr back?");
-  }
-  return E;
-}
-
 ExprResult Sema::ActOnFinishFullExpr(Expr *FE, SourceLocation CC,
                                      bool DiscardedValue, bool IsConstexpr,
                                      bool IsTemplateArgument) {
@@ -8086,8 +7677,6 @@ ExprResult Sema::ActOnFinishFullExpr(Expr *FE, SourceLocation CC,
     DiagnoseUnusedExprResult(FullExpr.get(), diag::warn_unused_expr);
   }
 
-  FullExpr = CorrectDelayedTyposInExpr(FullExpr.get(), /*InitDecl=*/nullptr,
-                                       /*RecoverUncorrectedTypos=*/true);
   if (FullExpr.isInvalid())
     return ExprError();
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaExprMember.cpp b/external/llvm-project/clang/lib/Sema/SemaExprMember.cpp
index 39c162c3b835..5dca509d46fd 100644
--- a/external/llvm-project/clang/lib/Sema/SemaExprMember.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaExprMember.cpp
@@ -650,64 +650,11 @@ bool Sema::CheckQualifiedMemberReference(Expr *BaseExpr,
   return true;
 }
 
-namespace {
-
-// Callback to only accept typo corrections that are either a ValueDecl or a
-// FunctionTemplateDecl and are declared in the current record or, for a C++
-// classes, one of its base classes.
-class RecordMemberExprValidatorCCC final : public CorrectionCandidateCallback {
-public:
-  explicit RecordMemberExprValidatorCCC(QualType RTy)
-      : Record(RTy->getAsRecordDecl()) {
-    // Don't add bare keywords to the consumer since they will always fail
-    // validation by virtue of not being associated with any decls.
-    WantTypeSpecifiers = false;
-    WantExpressionKeywords = false;
-    WantCXXNamedCasts = false;
-    WantFunctionLikeCasts = false;
-    WantRemainingKeywords = false;
-  }
-
-  bool ValidateCandidate(const TypoCorrection &candidate) override {
-    NamedDecl *ND = candidate.getCorrectionDecl();
-    // Don't accept candidates that cannot be member functions, constants,
-    // variables, or templates.
-    if (!ND || !(isa<ValueDecl>(ND) || isa<FunctionTemplateDecl>(ND)))
-      return false;
-
-    // Accept candidates that occur in the current record.
-    if (Record->containsDecl(ND))
-      return true;
-
-    if (const auto *RD = dyn_cast<CXXRecordDecl>(Record)) {
-      // Accept candidates that occur in any of the current class' base classes.
-      for (const auto &BS : RD->bases()) {
-        if (const auto *BSTy = BS.getType()->getAs<RecordType>()) {
-          if (BSTy->getDecl()->containsDecl(ND))
-            return true;
-        }
-      }
-    }
-
-    return false;
-  }
-
-  std::unique_ptr<CorrectionCandidateCallback> clone() override {
-    return std::make_unique<RecordMemberExprValidatorCCC>(*this);
-  }
-
-private:
-  const RecordDecl *const Record;
-};
-
-}
-
 static bool LookupMemberExprInRecord(Sema &SemaRef, LookupResult &R,
                                      Expr *BaseExpr, QualType RTy,
                                      SourceLocation OpLoc, bool IsArrow,
                                      CXXScopeSpec &SS, bool HasTemplateArgs,
-                                     SourceLocation TemplateKWLoc,
-                                     TypoExpr *&TE) {
+                                     SourceLocation TemplateKWLoc) {
   SourceRange BaseRange = BaseExpr ? BaseExpr->getSourceRange() : SourceRange();
   if (!RTy->isDependentType() &&
       !SemaRef.isThisOutsideMemberFunctionBody(RTy) &&
@@ -724,56 +671,6 @@ static bool LookupMemberExprInRecord(Sema &SemaRef, LookupResult &R,
                                       /*EnteringContext=*/false, TemplateKWLoc);
 
   SemaRef.LookupParsedName(R, /*S=*/nullptr, &SS, ObjectType);
-
-  if (!R.empty() || R.wasNotFoundInCurrentInstantiation())
-    return false;
-
-  DeclarationName Typo = R.getLookupName();
-  SourceLocation TypoLoc = R.getNameLoc();
-  // Recompute the lookup context.
-  DeclContext *DC = SS.isSet() ? SemaRef.computeDeclContext(SS)
-                               : SemaRef.computeDeclContext(RTy);
-
-  struct QueryState {
-    Sema &SemaRef;
-    DeclarationNameInfo NameInfo;
-    Sema::LookupNameKind LookupKind;
-    RedeclarationKind Redecl;
-  };
-  QueryState Q = {R.getSema(), R.getLookupNameInfo(), R.getLookupKind(),
-                  R.redeclarationKind()};
-  RecordMemberExprValidatorCCC CCC(RTy);
-  TE = SemaRef.CorrectTypoDelayed(
-      R.getLookupNameInfo(), R.getLookupKind(), nullptr, &SS, CCC,
-      [=, &SemaRef](const TypoCorrection &TC) {
-        if (TC) {
-          assert(!TC.isKeyword() &&
-                 "Got a keyword as a correction for a member!");
-          bool DroppedSpecifier =
-              TC.WillReplaceSpecifier() &&
-              Typo.getAsString() == TC.getAsString(SemaRef.getLangOpts());
-          SemaRef.diagnoseTypo(TC, SemaRef.PDiag(diag::err_no_member_suggest)
-                                       << Typo << DC << DroppedSpecifier
-                                       << SS.getRange());
-        } else {
-          SemaRef.Diag(TypoLoc, diag::err_no_member)
-              << Typo << DC << (SS.isSet() ? SS.getRange() : BaseRange);
-        }
-      },
-      [=](Sema &SemaRef, TypoExpr *TE, TypoCorrection TC) mutable {
-        LookupResult R(Q.SemaRef, Q.NameInfo, Q.LookupKind, Q.Redecl);
-        R.clear(); // Ensure there's no decls lingering in the shared state.
-        R.suppressDiagnostics();
-        R.setLookupName(TC.getCorrection());
-        for (NamedDecl *ND : TC)
-          R.addDecl(ND);
-        R.resolveKind();
-        return SemaRef.BuildMemberReferenceExpr(
-            BaseExpr, BaseExpr->getType(), OpLoc, IsArrow, SS, SourceLocation(),
-            nullptr, R, nullptr, nullptr);
-      },
-      CorrectTypoKind::ErrorRecovery, DC);
-
   return false;
 }
 
@@ -793,15 +690,11 @@ ExprResult Sema::BuildMemberReferenceExpr(
 
   // Implicit member accesses.
   if (!Base) {
-    TypoExpr *TE = nullptr;
     QualType RecordTy = BaseType;
     if (IsArrow) RecordTy = RecordTy->castAs<PointerType>()->getPointeeType();
     if (LookupMemberExprInRecord(*this, R, nullptr, RecordTy, OpLoc, IsArrow,
-                                 SS, TemplateArgs != nullptr, TemplateKWLoc,
-                                 TE))
+                                 SS, TemplateArgs != nullptr, TemplateKWLoc))
       return ExprError();
-    if (TE)
-      return TE;
 
   // Explicit member accesses.
   } else {
@@ -1396,16 +1289,15 @@ static ExprResult LookupMemberExpr(Sema &S, LookupResult &R,
 
   // Handle field access to simple records.
   if (BaseType->getAsRecordDecl()) {
-    TypoExpr *TE = nullptr;
     if (LookupMemberExprInRecord(S, R, BaseExpr.get(), BaseType, OpLoc, IsArrow,
-                                 SS, HasTemplateArgs, TemplateKWLoc, TE))
+                                 SS, HasTemplateArgs, TemplateKWLoc))
       return ExprError();
 
     // Returning valid-but-null is how we indicate to the caller that
     // the lookup result was filled in. If typo correction was attempted and
     // failed, the lookup result will have been cleared--that combined with the
     // valid-but-null ExprResult will trigger the appropriate diagnostics.
-    return ExprResult(TE);
+    return ExprResult{};
   } else if (BaseType->isDependentType()) {
     R.setNotFoundInCurrentInstantiation();
     return ExprEmpty();
diff --git a/external/llvm-project/clang/lib/Sema/SemaExprObjC.cpp b/external/llvm-project/clang/lib/Sema/SemaExprObjC.cpp
index b248ea380a52..3505d9f38d23 100644
--- a/external/llvm-project/clang/lib/Sema/SemaExprObjC.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaExprObjC.cpp
@@ -5151,7 +5151,8 @@ ExprResult SemaObjC::ActOnObjCAvailabilityCheckExpr(
     SourceLocation RParen) {
   ASTContext &Context = getASTContext();
   auto FindSpecVersion =
-      [&](StringRef Platform) -> std::optional<VersionTuple> {
+      [&](StringRef Platform,
+          const llvm::Triple::OSType &OS) -> std::optional<VersionTuple> {
     auto Spec = llvm::find_if(AvailSpecs, [&](const AvailabilitySpec &Spec) {
       return Spec.getPlatform() == Platform;
     });
@@ -5164,12 +5165,16 @@ ExprResult SemaObjC::ActOnObjCAvailabilityCheckExpr(
     }
     if (Spec == AvailSpecs.end())
       return std::nullopt;
-    return Spec->getVersion();
+
+    return llvm::Triple::getCanonicalVersionForOS(
+        OS, Spec->getVersion(),
+        llvm::Triple::isValidVersionForOS(OS, Spec->getVersion()));
   };
 
   VersionTuple Version;
   if (auto MaybeVersion =
-          FindSpecVersion(Context.getTargetInfo().getPlatformName()))
+          FindSpecVersion(Context.getTargetInfo().getPlatformName(),
+                          Context.getTargetInfo().getTriple().getOS()))
     Version = *MaybeVersion;
 
   // The use of `@available` in the enclosing context should be analyzed to
diff --git a/external/llvm-project/clang/lib/Sema/SemaHLSL.cpp b/external/llvm-project/clang/lib/Sema/SemaHLSL.cpp
index 43db85594de3..ba491b613429 100644
--- a/external/llvm-project/clang/lib/Sema/SemaHLSL.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaHLSL.cpp
@@ -764,6 +764,13 @@ void SemaHLSL::CheckSemanticAnnotation(
       return;
     DiagnoseAttrStageMismatch(AnnotationAttr, ST, {llvm::Triple::Compute});
     break;
+  case attr::HLSLSV_Position:
+    // TODO(#143523): allow use on other shader types & output once the overall
+    // semantic logic is implemented.
+    if (ST == llvm::Triple::Pixel)
+      return;
+    DiagnoseAttrStageMismatch(AnnotationAttr, ST, {llvm::Triple::Pixel});
+    break;
   default:
     llvm_unreachable("Unknown HLSLAnnotationAttr");
   }
@@ -1117,6 +1124,14 @@ void SemaHLSL::handleWaveSizeAttr(Decl *D, const ParsedAttr &AL) {
     D->addAttr(NewAttr);
 }
 
+void SemaHLSL::handleVkExtBuiltinInputAttr(Decl *D, const ParsedAttr &AL) {
+  uint32_t ID;
+  if (!SemaRef.checkUInt32Argument(AL, AL.getArgAsExpr(0), ID))
+    return;
+  D->addAttr(::new (getASTContext())
+                 HLSLVkExtBuiltinInputAttr(getASTContext(), AL, ID));
+}
+
 bool SemaHLSL::diagnoseInputIDType(QualType T, const ParsedAttr &AL) {
   const auto *VT = T->getAs<VectorType>();
 
@@ -1139,6 +1154,26 @@ void SemaHLSL::handleSV_DispatchThreadIDAttr(Decl *D, const ParsedAttr &AL) {
                  HLSLSV_DispatchThreadIDAttr(getASTContext(), AL));
 }
 
+bool SemaHLSL::diagnosePositionType(QualType T, const ParsedAttr &AL) {
+  const auto *VT = T->getAs<VectorType>();
+
+  if (!T->hasFloatingRepresentation() || (VT && VT->getNumElements() > 4)) {
+    Diag(AL.getLoc(), diag::err_hlsl_attr_invalid_type)
+        << AL << "float/float1/float2/float3/float4";
+    return false;
+  }
+
+  return true;
+}
+
+void SemaHLSL::handleSV_PositionAttr(Decl *D, const ParsedAttr &AL) {
+  auto *VD = cast<ValueDecl>(D);
+  if (!diagnosePositionType(VD->getType(), AL))
+    return;
+
+  D->addAttr(::new (getASTContext()) HLSLSV_PositionAttr(getASTContext(), AL));
+}
+
 void SemaHLSL::handleSV_GroupThreadIDAttr(Decl *D, const ParsedAttr &AL) {
   auto *VD = cast<ValueDecl>(D);
   if (!diagnoseInputIDType(VD->getType(), AL))
@@ -3158,6 +3193,14 @@ void SemaHLSL::deduceAddressSpace(VarDecl *Decl) {
     return;
 
   QualType Type = Decl->getType();
+
+  if (Decl->hasAttr<HLSLVkExtBuiltinInputAttr>()) {
+    LangAS ImplAS = LangAS::hlsl_input;
+    Type = SemaRef.getASTContext().getAddrSpaceQualType(Type, ImplAS);
+    Decl->setType(Type);
+    return;
+  }
+
   if (Type->isSamplerT() || Type->isVoidType())
     return;
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaInit.cpp b/external/llvm-project/clang/lib/Sema/SemaInit.cpp
index 5d870f07093e..da56225b2f92 100644
--- a/external/llvm-project/clang/lib/Sema/SemaInit.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaInit.cpp
@@ -6620,7 +6620,7 @@ void InitializationSequence::InitializeFrom(Sema &S,
       // initializer present. However, we only do this for structure types, not
       // union types, because an unitialized field in a union is generally
       // reasonable, especially in C where unions can be used for type punning.
-      if (!Initializer && !Rec->isUnion() && !Rec->isInvalidDecl()) {
+      if (Var && !Initializer && !Rec->isUnion() && !Rec->isInvalidDecl()) {
         if (const FieldDecl *FD = getConstField(Rec)) {
           unsigned DiagID = diag::warn_default_init_const_field_unsafe;
           if (Var->getStorageDuration() == SD_Static ||
diff --git a/external/llvm-project/clang/lib/Sema/SemaLambda.cpp b/external/llvm-project/clang/lib/Sema/SemaLambda.cpp
index d80a5f6ae856..db103819f789 100644
--- a/external/llvm-project/clang/lib/Sema/SemaLambda.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaLambda.cpp
@@ -2021,6 +2021,7 @@ bool Sema::CaptureHasSideEffects(const Capture &From) {
 }
 
 bool Sema::DiagnoseUnusedLambdaCapture(SourceRange CaptureRange,
+                                       SourceRange FixItRange,
                                        const Capture &From) {
   if (CaptureHasSideEffects(From))
     return false;
@@ -2040,7 +2041,12 @@ bool Sema::DiagnoseUnusedLambdaCapture(SourceRange CaptureRange,
   else
     diag << From.getVariable();
   diag << From.isNonODRUsed();
-  diag << FixItHint::CreateRemoval(CaptureRange);
+  // If we were able to resolve the fixit range we'll create a fixit,
+  // otherwise we just use the raw capture range for the diagnostic.
+  if (FixItRange.isValid())
+    diag << FixItHint::CreateRemoval(FixItRange);
+  else
+    diag << CaptureRange;
   return true;
 }
 
@@ -2094,6 +2100,39 @@ FieldDecl *Sema::BuildCaptureField(RecordDecl *RD,
   return Field;
 }
 
+static SourceRange
+ConstructFixItRangeForUnusedCapture(Sema &S, SourceRange CaptureRange,
+                                    SourceLocation PrevCaptureLoc,
+                                    bool CurHasPreviousCapture, bool IsLast) {
+  if (!CaptureRange.isValid())
+    return SourceRange();
+
+  auto GetTrailingEndLocation = [&](SourceLocation StartPoint) {
+    SourceRange NextToken = S.getRangeForNextToken(
+        StartPoint, /*IncludeMacros=*/false, /*IncludeComments=*/true);
+    if (!NextToken.isValid())
+      return SourceLocation();
+    // Return the last location preceding the next token
+    return NextToken.getBegin().getLocWithOffset(-1);
+  };
+
+  if (!CurHasPreviousCapture && !IsLast) {
+    // If there are no captures preceding this capture, remove the
+    // trailing comma and anything up to the next token
+    SourceRange CommaRange =
+        S.getRangeForNextToken(CaptureRange.getEnd(), /*IncludeMacros=*/false,
+                               /*IncludeComments=*/false, tok::comma);
+    SourceLocation FixItEnd = GetTrailingEndLocation(CommaRange.getBegin());
+    return SourceRange(CaptureRange.getBegin(), FixItEnd);
+  }
+
+  // Otherwise, remove the comma since the last used capture, and
+  // anything up to the next token
+  SourceLocation FixItStart = S.getLocForEndOfToken(PrevCaptureLoc);
+  SourceLocation FixItEnd = GetTrailingEndLocation(CaptureRange.getEnd());
+  return SourceRange(FixItStart, FixItEnd);
+}
+
 ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
                                  LambdaScopeInfo *LSI) {
   // Collect information from the lambda scope.
@@ -2161,21 +2200,11 @@ ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
             IsGenericLambda && From.isNonODRUsed() && From.isInitCapture();
         if (!NonODRUsedInitCapture) {
           bool IsLast = (I + 1) == LSI->NumExplicitCaptures;
-          SourceRange FixItRange;
-          if (CaptureRange.isValid()) {
-            if (!CurHasPreviousCapture && !IsLast) {
-              // If there are no captures preceding this capture, remove the
-              // following comma.
-              FixItRange = SourceRange(CaptureRange.getBegin(),
-                                       getLocForEndOfToken(CaptureRange.getEnd()));
-            } else {
-              // Otherwise, remove the comma since the last used capture.
-              FixItRange = SourceRange(getLocForEndOfToken(PrevCaptureLoc),
-                                       CaptureRange.getEnd());
-            }
-          }
-
-          IsCaptureUsed = !DiagnoseUnusedLambdaCapture(FixItRange, From);
+          SourceRange FixItRange = ConstructFixItRangeForUnusedCapture(
+              *this, CaptureRange, PrevCaptureLoc, CurHasPreviousCapture,
+              IsLast);
+          IsCaptureUsed =
+              !DiagnoseUnusedLambdaCapture(CaptureRange, FixItRange, From);
         }
       }
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaLookup.cpp b/external/llvm-project/clang/lib/Sema/SemaLookup.cpp
index eef134b15843..5ad9dd8ed0d3 100644
--- a/external/llvm-project/clang/lib/Sema/SemaLookup.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaLookup.cpp
@@ -1978,6 +1978,8 @@ bool LookupResult::isReachableSlow(Sema &SemaRef, NamedDecl *D) {
   if (D->isModulePrivate())
     return false;
 
+  Module *DeclTopModule = DeclModule->getTopLevelModule();
+
   // [module.reach]/p1
   //   A translation unit U is necessarily reachable from a point P if U is a
   //   module interface unit on which the translation unit containing P has an
@@ -1996,17 +1998,28 @@ bool LookupResult::isReachableSlow(Sema &SemaRef, NamedDecl *D) {
   //
   // Here we only check for the first condition. Since we couldn't see
   // DeclModule if it isn't (transitively) imported.
-  if (DeclModule->getTopLevelModule()->isModuleInterfaceUnit())
+  if (DeclTopModule->isModuleInterfaceUnit())
     return true;
 
-  // [module.reach]/p2
+  // [module.reach]/p1,2
+  //   A translation unit U is necessarily reachable from a point P if U is a
+  //   module interface unit on which the translation unit containing P has an
+  //   interface dependency, or the translation unit containing P imports U, in
+  //   either case prior to P
+  //
   //   Additional translation units on
   //   which the point within the program has an interface dependency may be
   //   considered reachable, but it is unspecified which are and under what
   //   circumstances.
-  //
-  // The decision here is to treat all additional tranditional units as
-  // unreachable.
+  Module *CurrentM = SemaRef.getCurrentModule();
+
+  // Directly imported module are necessarily reachable.
+  // Since we can't export import a module implementation partition unit, we
+  // don't need to count for Exports here.
+  if (CurrentM && CurrentM->getTopLevelModule()->Imports.count(DeclTopModule))
+    return true;
+
+  // Then we treat all module implementation partition unit as unreachable.
   return false;
 }
 
@@ -5431,40 +5444,6 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
   return FailedCorrection(Typo, TypoName.getLoc(), RecordFailure && !SecondBestTC);
 }
 
-TypoExpr *Sema::CorrectTypoDelayed(
-    const DeclarationNameInfo &TypoName, Sema::LookupNameKind LookupKind,
-    Scope *S, CXXScopeSpec *SS, CorrectionCandidateCallback &CCC,
-    TypoDiagnosticGenerator TDG, TypoRecoveryCallback TRC, CorrectTypoKind Mode,
-    DeclContext *MemberContext, bool EnteringContext,
-    const ObjCObjectPointerType *OPT) {
-  auto Consumer = makeTypoCorrectionConsumer(
-      TypoName, LookupKind, S, SS, CCC, MemberContext, EnteringContext, OPT,
-      Mode == CorrectTypoKind::ErrorRecovery);
-
-  // Give the external sema source a chance to correct the typo.
-  TypoCorrection ExternalTypo;
-  if (ExternalSource && Consumer) {
-    ExternalTypo = ExternalSource->CorrectTypo(
-        TypoName, LookupKind, S, SS, *Consumer->getCorrectionValidator(),
-        MemberContext, EnteringContext, OPT);
-    if (ExternalTypo)
-      Consumer->addCorrection(ExternalTypo);
-  }
-
-  if (!Consumer || Consumer->empty())
-    return nullptr;
-
-  // Make sure the best edit distance (prior to adding any namespace qualifiers)
-  // is not more that about a third of the length of the typo's identifier.
-  unsigned ED = Consumer->getBestEditDistance(true);
-  IdentifierInfo *Typo = TypoName.getName().getAsIdentifierInfo();
-  if (!ExternalTypo && ED > 0 && Typo->getName().size() / ED < 3)
-    return nullptr;
-  ExprEvalContexts.back().NumTypos++;
-  return createDelayedTypo(std::move(Consumer), std::move(TDG), std::move(TRC),
-                           TypoName.getLoc());
-}
-
 void TypoCorrection::addCorrectionDecl(NamedDecl *CDecl) {
   if (!CDecl) return;
 
@@ -5789,32 +5768,6 @@ void Sema::diagnoseTypo(const TypoCorrection &Correction,
     Diag(Correction.getCorrectionRange().getBegin(), PD);
 }
 
-TypoExpr *Sema::createDelayedTypo(std::unique_ptr<TypoCorrectionConsumer> TCC,
-                                  TypoDiagnosticGenerator TDG,
-                                  TypoRecoveryCallback TRC,
-                                  SourceLocation TypoLoc) {
-  assert(TCC && "createDelayedTypo requires a valid TypoCorrectionConsumer");
-  auto TE = new (Context) TypoExpr(Context.DependentTy, TypoLoc);
-  auto &State = DelayedTypos[TE];
-  State.Consumer = std::move(TCC);
-  State.DiagHandler = std::move(TDG);
-  State.RecoveryHandler = std::move(TRC);
-  if (TE)
-    TypoExprs.push_back(TE);
-  return TE;
-}
-
-const Sema::TypoExprState &Sema::getTypoExprState(TypoExpr *TE) const {
-  auto Entry = DelayedTypos.find(TE);
-  assert(Entry != DelayedTypos.end() &&
-         "Failed to get the state for a TypoExpr!");
-  return Entry->second;
-}
-
-void Sema::clearDelayedTypo(TypoExpr *TE) {
-  DelayedTypos.erase(TE);
-}
-
 void Sema::ActOnPragmaDump(Scope *S, SourceLocation IILoc, IdentifierInfo *II) {
   DeclarationNameInfo Name(II, IILoc);
   LookupResult R(*this, Name, LookupAnyName,
diff --git a/external/llvm-project/clang/lib/Sema/SemaModule.cpp b/external/llvm-project/clang/lib/Sema/SemaModule.cpp
index 6c4df0aa35af..9fcaad48d305 100644
--- a/external/llvm-project/clang/lib/Sema/SemaModule.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaModule.cpp
@@ -712,7 +712,13 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
       Mod->Kind == Module::ModuleKind::ModulePartitionImplementation) {
     Diag(ExportLoc, diag::err_export_partition_impl)
         << SourceRange(ExportLoc, Path.back().getLoc());
-  } else if (!ModuleScopes.empty() && !currentModuleIsImplementation()) {
+  } else if (ExportLoc.isValid() &&
+             (ModuleScopes.empty() || currentModuleIsImplementation())) {
+    // [module.interface]p1:
+    // An export-declaration shall inhabit a namespace scope and appear in the
+    // purview of a module interface unit.
+    Diag(ExportLoc, diag::err_export_not_in_module_interface);
+  } else if (!ModuleScopes.empty()) {
     // Re-export the module if the imported module is exported.
     // Note that we don't need to add re-exported module to Imports field
     // since `Exports` implies the module is imported already.
@@ -720,11 +726,6 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
       getCurrentModule()->Exports.emplace_back(Mod, false);
     else
       getCurrentModule()->Imports.insert(Mod);
-  } else if (ExportLoc.isValid()) {
-    // [module.interface]p1:
-    // An export-declaration shall inhabit a namespace scope and appear in the
-    // purview of a module interface unit.
-    Diag(ExportLoc, diag::err_export_not_in_module_interface);
   }
 
   return Import;
diff --git a/external/llvm-project/clang/lib/Sema/SemaObjC.cpp b/external/llvm-project/clang/lib/Sema/SemaObjC.cpp
index 56815cd2731a..0f39a9817ce7 100644
--- a/external/llvm-project/clang/lib/Sema/SemaObjC.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaObjC.cpp
@@ -124,17 +124,12 @@ ExprResult SemaObjC::CheckObjCForCollectionOperand(SourceLocation forLoc,
   if (!collection)
     return ExprError();
 
-  ExprResult result = SemaRef.CorrectDelayedTyposInExpr(collection);
-  if (!result.isUsable())
-    return ExprError();
-  collection = result.get();
-
   // Bail out early if we've got a type-dependent expression.
   if (collection->isTypeDependent())
     return collection;
 
   // Perform normal l-value conversion.
-  result = SemaRef.DefaultFunctionArrayLvalueConversion(collection);
+  ExprResult result = SemaRef.DefaultFunctionArrayLvalueConversion(collection);
   if (result.isInvalid())
     return ExprError();
   collection = result.get();
diff --git a/external/llvm-project/clang/lib/Sema/SemaOpenMP.cpp b/external/llvm-project/clang/lib/Sema/SemaOpenMP.cpp
index a3b198d897e4..00fb04786410 100644
--- a/external/llvm-project/clang/lib/Sema/SemaOpenMP.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaOpenMP.cpp
@@ -496,7 +496,7 @@ class DSAStackTy {
   void addCriticalWithHint(const OMPCriticalDirective *D, llvm::APSInt Hint) {
     Criticals.try_emplace(D->getDirectiveName().getAsString(), D, Hint);
   }
-  const std::pair<const OMPCriticalDirective *, llvm::APSInt>
+  std::pair<const OMPCriticalDirective *, llvm::APSInt>
   getCriticalWithHint(const DeclarationNameInfo &Name) const {
     auto I = Criticals.find(Name.getAsString());
     if (I != Criticals.end())
@@ -7565,6 +7565,7 @@ void SemaOpenMP::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
       getASTContext(), VariantFuncRef, DVScope.TI,
       /*NothingArgs=*/nullptr, /*NothingArgsSize=*/0,
       /*NeedDevicePtrArgs=*/nullptr, /*NeedDevicePtrArgsSize=*/0,
+      /*NeedDeviceAddrArgs=*/nullptr, /*NeedDeviceAddrArgsSize=*/0,
       /*AppendArgs=*/nullptr, /*AppendArgsSize=*/0);
   for (FunctionDecl *BaseFD : Bases)
     BaseFD->addAttr(OMPDeclareVariantA);
@@ -7996,6 +7997,7 @@ void SemaOpenMP::ActOnOpenMPDeclareVariantDirective(
     FunctionDecl *FD, Expr *VariantRef, OMPTraitInfo &TI,
     ArrayRef<Expr *> AdjustArgsNothing,
     ArrayRef<Expr *> AdjustArgsNeedDevicePtr,
+    ArrayRef<Expr *> AdjustArgsNeedDeviceAddr,
     ArrayRef<OMPInteropInfo> AppendArgs, SourceLocation AdjustArgsLoc,
     SourceLocation AppendArgsLoc, SourceRange SR) {
 
@@ -8007,6 +8009,7 @@ void SemaOpenMP::ActOnOpenMPDeclareVariantDirective(
   SmallVector<Expr *, 8> AllAdjustArgs;
   llvm::append_range(AllAdjustArgs, AdjustArgsNothing);
   llvm::append_range(AllAdjustArgs, AdjustArgsNeedDevicePtr);
+  llvm::append_range(AllAdjustArgs, AdjustArgsNeedDeviceAddr);
 
   if (!AllAdjustArgs.empty() || !AppendArgs.empty()) {
     VariantMatchInfo VMI;
@@ -8057,6 +8060,8 @@ void SemaOpenMP::ActOnOpenMPDeclareVariantDirective(
       const_cast<Expr **>(AdjustArgsNothing.data()), AdjustArgsNothing.size(),
       const_cast<Expr **>(AdjustArgsNeedDevicePtr.data()),
       AdjustArgsNeedDevicePtr.size(),
+      const_cast<Expr **>(AdjustArgsNeedDeviceAddr.data()),
+      AdjustArgsNeedDeviceAddr.size(),
       const_cast<OMPInteropInfo *>(AppendArgs.data()), AppendArgs.size(), SR);
   FD->addAttr(NewAttr);
 }
@@ -19490,34 +19495,14 @@ static bool actOnOMPReductionKindClause(
         reportOriginalDsa(S, Stack, D, DVar);
         continue;
       }
-      // OpenMP 6.0 [ 7.6.10 ]
-      // Support Reduction over private variables with reduction clause.
-      // A list item in a reduction clause can now be private in the enclosing
-      // context. For orphaned constructs it is assumed to be shared unless the
-      // original(private) modifier appears in the clause.
-      DVar = Stack->getImplicitDSA(D, true);
-      bool IsOrphaned = false;
-      OpenMPDirectiveKind CurrDir = Stack->getCurrentDirective();
-      OpenMPDirectiveKind ParentDir = Stack->getParentDirective();
-      // Check if the construct is orphaned (has no enclosing OpenMP context)
-      IsOrphaned = ParentDir == OMPD_unknown;
-      // OpenMP 6.0: Private DSA check
-      IsPrivate =
-          (S.getLangOpts().OpenMP > 52) &&
-          ((isOpenMPPrivate(DVar.CKind) && DVar.CKind != OMPC_reduction &&
-            isOpenMPWorksharingDirective(CurrDir) &&
-            !isOpenMPParallelDirective(CurrDir) &&
-            !isOpenMPTeamsDirective(CurrDir) &&
-            !isOpenMPSimdDirective(ParentDir)) ||
-           (IsOrphaned && DVar.CKind == OMPC_unknown) ||
-           RD.OrigSharingModifier != OMPC_ORIGINAL_SHARING_shared);
 
       // OpenMP [2.14.3.6, Restrictions, p.1]
       //  A list item that appears in a reduction clause of a worksharing
       //  construct must be shared in the parallel regions to which any of the
       //  worksharing regions arising from the worksharing construct bind.
 
-      if (!IsPrivate && isOpenMPWorksharingDirective(CurrDir) &&
+      if (S.getLangOpts().OpenMP <= 52 &&
+          isOpenMPWorksharingDirective(CurrDir) &&
           !isOpenMPParallelDirective(CurrDir) &&
           !isOpenMPTeamsDirective(CurrDir)) {
         DVar = Stack->getImplicitDSA(D, true);
@@ -19528,6 +19513,23 @@ static bool actOnOMPReductionKindClause(
           reportOriginalDsa(S, Stack, D, DVar);
           continue;
         }
+      } else if (isOpenMPWorksharingDirective(CurrDir) &&
+                 !isOpenMPParallelDirective(CurrDir) &&
+                 !isOpenMPTeamsDirective(CurrDir)) {
+        // OpenMP 6.0 [ 7.6.10 ]
+        // Support Reduction over private variables with reduction clause.
+        // A list item in a reduction clause can now be private in the enclosing
+        // context. For orphaned constructs it is assumed to be shared unless
+        // the original(private) modifier appears in the clause.
+        DVar = Stack->getImplicitDSA(D, true);
+        // Determine if the variable should be considered private
+        IsPrivate = DVar.CKind != OMPC_shared;
+        bool IsOrphaned = false;
+        OpenMPDirectiveKind ParentDir = Stack->getParentDirective();
+        IsOrphaned = ParentDir == OMPD_unknown;
+        if ((IsOrphaned &&
+             RD.OrigSharingModifier == OMPC_ORIGINAL_SHARING_private))
+          IsPrivate = true;
       }
     } else {
       // Threadprivates cannot be shared between threads, so dignose if the base
@@ -22505,20 +22507,34 @@ static void checkMappableExpressionList(
         Type.getCanonicalType(), UnresolvedMapper);
     if (ER.isInvalid())
       continue;
-    if (!ER.get() && isa<ArraySectionExpr>(VE)) {
-      // Create implicit mapper as needed.
-      QualType BaseType = VE->getType().getCanonicalType();
-      if (BaseType->isSpecificBuiltinType(BuiltinType::ArraySection)) {
-        const auto *OASE = cast<ArraySectionExpr>(VE->IgnoreParenImpCasts());
-        QualType BType = ArraySectionExpr::getBaseOriginalType(OASE->getBase());
-        QualType ElemType;
-        if (const auto *ATy = BType->getAsArrayTypeUnsafe())
-          ElemType = ATy->getElementType();
-        else
-          ElemType = BType->getPointeeType();
+
+    // If no user-defined mapper is found, we need to create an implicit one for
+    // arrays/array-sections on structs that have members that have
+    // user-defined mappers. This is needed to ensure that the mapper for the
+    // member is invoked when mapping each element of the array/array-section.
+    if (!ER.get()) {
+      QualType BaseType;
+
+      if (isa<ArraySectionExpr>(VE)) {
+        BaseType = VE->getType().getCanonicalType();
+        if (BaseType->isSpecificBuiltinType(BuiltinType::ArraySection)) {
+          const auto *OASE = cast<ArraySectionExpr>(VE->IgnoreParenImpCasts());
+          QualType BType =
+              ArraySectionExpr::getBaseOriginalType(OASE->getBase());
+          QualType ElemType;
+          if (const auto *ATy = BType->getAsArrayTypeUnsafe())
+            ElemType = ATy->getElementType();
+          else
+            ElemType = BType->getPointeeType();
+          BaseType = ElemType.getCanonicalType();
+        }
+      } else if (VE->getType()->isArrayType()) {
+        const ArrayType *AT = VE->getType()->getAsArrayTypeUnsafe();
+        const QualType ElemType = AT->getElementType();
         BaseType = ElemType.getCanonicalType();
       }
-      if (BaseType->getAsRecordDecl() &&
+
+      if (!BaseType.isNull() && BaseType->getAsRecordDecl() &&
           isImplicitMapperNeeded(SemaRef, DSAS, BaseType, VE)) {
         ER = buildImplicitMapper(SemaRef, BaseType, DSAS);
       }
diff --git a/external/llvm-project/clang/lib/Sema/SemaOverload.cpp b/external/llvm-project/clang/lib/Sema/SemaOverload.cpp
index 69768dd3a721..8c5f81f126c7 100644
--- a/external/llvm-project/clang/lib/Sema/SemaOverload.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaOverload.cpp
@@ -1490,7 +1490,7 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
   // If the function is a class member, its signature includes the
   // cv-qualifiers (if any) and ref-qualifier (if any) on the function itself.
   auto DiagnoseInconsistentRefQualifiers = [&]() {
-    if (SemaRef.LangOpts.CPlusPlus23)
+    if (SemaRef.LangOpts.CPlusPlus23 && !UseOverrideRules)
       return false;
     if (OldMethod->getRefQualifier() == NewMethod->getRefQualifier())
       return false;
@@ -1889,7 +1889,14 @@ bool Sema::TryFunctionConversion(QualType FromType, QualType ToType,
   return Changed;
 }
 
-bool Sema::IsFunctionConversion(QualType FromType, QualType ToType) const {
+bool Sema::IsFunctionConversion(QualType FromType, QualType ToType,
+                                bool *DiscardingCFIUncheckedCallee,
+                                bool *AddingCFIUncheckedCallee) const {
+  if (DiscardingCFIUncheckedCallee)
+    *DiscardingCFIUncheckedCallee = false;
+  if (AddingCFIUncheckedCallee)
+    *AddingCFIUncheckedCallee = false;
+
   if (Context.hasSameUnqualifiedType(FromType, ToType))
     return false;
 
@@ -1944,9 +1951,34 @@ bool Sema::IsFunctionConversion(QualType FromType, QualType ToType) const {
     Changed = true;
   }
 
+  const auto *FromFPT = dyn_cast<FunctionProtoType>(FromFn);
+  const auto *ToFPT = dyn_cast<FunctionProtoType>(ToFn);
+
+  if (FromFPT && ToFPT) {
+    if (FromFPT->hasCFIUncheckedCallee() && !ToFPT->hasCFIUncheckedCallee()) {
+      QualType NewTy = Context.getFunctionType(
+          FromFPT->getReturnType(), FromFPT->getParamTypes(),
+          FromFPT->getExtProtoInfo().withCFIUncheckedCallee(false));
+      FromFPT = cast<FunctionProtoType>(NewTy.getTypePtr());
+      FromFn = FromFPT;
+      Changed = true;
+      if (DiscardingCFIUncheckedCallee)
+        *DiscardingCFIUncheckedCallee = true;
+    } else if (!FromFPT->hasCFIUncheckedCallee() &&
+               ToFPT->hasCFIUncheckedCallee()) {
+      QualType NewTy = Context.getFunctionType(
+          FromFPT->getReturnType(), FromFPT->getParamTypes(),
+          FromFPT->getExtProtoInfo().withCFIUncheckedCallee(true));
+      FromFPT = cast<FunctionProtoType>(NewTy.getTypePtr());
+      FromFn = FromFPT;
+      Changed = true;
+      if (AddingCFIUncheckedCallee)
+        *AddingCFIUncheckedCallee = true;
+    }
+  }
+
   // Drop 'noexcept' if not present in target type.
-  if (const auto *FromFPT = dyn_cast<FunctionProtoType>(FromFn)) {
-    const auto *ToFPT = cast<FunctionProtoType>(ToFn);
+  if (FromFPT) {
     if (FromFPT->isNothrow() && !ToFPT->isNothrow()) {
       FromFn = cast<FunctionType>(
           Context.getFunctionTypeWithExceptionSpec(QualType(FromFPT, 0),
@@ -2510,12 +2542,15 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
 
   SCS.setToType(2, FromType);
 
-  if (CanonFrom == CanonTo)
-    return true;
-
   // If we have not converted the argument type to the parameter type,
   // this is a bad conversion sequence, unless we're resolving an overload in C.
-  if (S.getLangOpts().CPlusPlus || !InOverloadResolution)
+  //
+  // Permit conversions from a function without `cfi_unchecked_callee` to a
+  // function with `cfi_unchecked_callee`.
+  if (CanonFrom == CanonTo || S.AddingCFIUncheckedCallee(CanonFrom, CanonTo))
+    return true;
+
+  if ((S.getLangOpts().CPlusPlus || !InOverloadResolution))
     return false;
 
   ExprResult ER = ExprResult{From};
@@ -7848,11 +7883,14 @@ static void AddMethodTemplateCandidateImmediately(
           /*PartialOrdering=*/false, ObjectType, ObjectClassification,
           CandidateSet.getKind() ==
               clang::OverloadCandidateSet::CSK_AddressOfOverloadSet,
-          [&](ArrayRef<QualType> ParamTypes) {
+          [&](ArrayRef<QualType> ParamTypes,
+              bool OnlyInitializeNonUserDefinedConversions) {
             return S.CheckNonDependentConversions(
                 MethodTmpl, ParamTypes, Args, CandidateSet, Conversions,
-                SuppressUserConversions, ActingContext, ObjectType,
-                ObjectClassification, PO);
+                Sema::CheckNonDependentConversionsFlag(
+                    SuppressUserConversions,
+                    OnlyInitializeNonUserDefinedConversions),
+                ActingContext, ObjectType, ObjectClassification, PO);
           });
       Result != TemplateDeductionResult::Success) {
     OverloadCandidate &Candidate =
@@ -7964,10 +8002,14 @@ static void AddTemplateOverloadCandidateImmediately(
           /*ObjectClassification=*/Expr::Classification(),
           CandidateSet.getKind() ==
               OverloadCandidateSet::CSK_AddressOfOverloadSet,
-          [&](ArrayRef<QualType> ParamTypes) {
+          [&](ArrayRef<QualType> ParamTypes,
+              bool OnlyInitializeNonUserDefinedConversions) {
             return S.CheckNonDependentConversions(
                 FunctionTemplate, ParamTypes, Args, CandidateSet, Conversions,
-                SuppressUserConversions, nullptr, QualType(), {}, PO);
+                Sema::CheckNonDependentConversionsFlag(
+                    SuppressUserConversions,
+                    OnlyInitializeNonUserDefinedConversions),
+                nullptr, QualType(), {}, PO);
           });
       Result != TemplateDeductionResult::Success) {
     OverloadCandidate &Candidate =
@@ -8041,7 +8083,8 @@ void Sema::AddTemplateOverloadCandidate(
 bool Sema::CheckNonDependentConversions(
     FunctionTemplateDecl *FunctionTemplate, ArrayRef<QualType> ParamTypes,
     ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet,
-    ConversionSequenceList &Conversions, bool SuppressUserConversions,
+    ConversionSequenceList &Conversions,
+    CheckNonDependentConversionsFlag UserConversionFlag,
     CXXRecordDecl *ActingContext, QualType ObjectType,
     Expr::Classification ObjectClassification, OverloadCandidateParamOrder PO) {
   // FIXME: The cases in which we allow explicit conversions for constructor
@@ -8054,8 +8097,9 @@ bool Sema::CheckNonDependentConversions(
   bool HasThisConversion = Method && !isa<CXXConstructorDecl>(Method);
   unsigned ThisConversions = HasThisConversion ? 1 : 0;
 
-  Conversions =
-      CandidateSet.allocateConversionSequences(ThisConversions + Args.size());
+  if (Conversions.empty())
+    Conversions =
+        CandidateSet.allocateConversionSequences(ThisConversions + Args.size());
 
   // Overload resolution is always an unevaluated context.
   EnterExpressionEvaluationContext Unevaluated(
@@ -8079,6 +8123,46 @@ bool Sema::CheckNonDependentConversions(
     }
   }
 
+  // A speculative workaround for self-dependent constraint bugs that manifest
+  // after CWG2369.
+  // FIXME: Add references to the standard once P3606 is adopted.
+  auto MaybeInvolveUserDefinedConversion = [&](QualType ParamType,
+                                               QualType ArgType) {
+    ParamType = ParamType.getNonReferenceType();
+    ArgType = ArgType.getNonReferenceType();
+    bool PointerConv = ParamType->isPointerType() && ArgType->isPointerType();
+    if (PointerConv) {
+      ParamType = ParamType->getPointeeType();
+      ArgType = ArgType->getPointeeType();
+    }
+
+    if (auto *RT = ParamType->getAs<RecordType>())
+      if (auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
+          RD && RD->hasDefinition()) {
+        if (llvm::any_of(LookupConstructors(RD), [](NamedDecl *ND) {
+              auto Info = getConstructorInfo(ND);
+              if (!Info)
+                return false;
+              CXXConstructorDecl *Ctor = Info.Constructor;
+              /// isConvertingConstructor takes copy/move constructors into
+              /// account!
+              return !Ctor->isCopyOrMoveConstructor() &&
+                     Ctor->isConvertingConstructor(
+                         /*AllowExplicit=*/true);
+            }))
+          return true;
+      }
+
+    if (auto *RT = ArgType->getAs<RecordType>())
+      if (auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
+          RD && RD->hasDefinition() &&
+          !RD->getVisibleConversionFunctions().empty()) {
+        return true;
+      }
+
+    return false;
+  };
+
   unsigned Offset =
       Method && Method->hasCXXExplicitFunctionObjectParameter() ? 1 : 0;
 
@@ -8099,13 +8183,16 @@ bool Sema::CheckNonDependentConversions(
         // For members, 'this' got ConvIdx = 0 previously.
         ConvIdx = ThisConversions + I;
       }
-      Conversions[ConvIdx]
-        = TryCopyInitialization(*this, Args[I], ParamType,
-                                SuppressUserConversions,
-                                /*InOverloadResolution=*/true,
-                                /*AllowObjCWritebackConversion=*/
-                                  getLangOpts().ObjCAutoRefCount,
-                                AllowExplicit);
+      if (Conversions[ConvIdx].isInitialized())
+        continue;
+      if (UserConversionFlag.OnlyInitializeNonUserDefinedConversions &&
+          MaybeInvolveUserDefinedConversion(ParamType, Args[I]->getType()))
+        continue;
+      Conversions[ConvIdx] = TryCopyInitialization(
+          *this, Args[I], ParamType, UserConversionFlag.SuppressUserConversions,
+          /*InOverloadResolution=*/true,
+          /*AllowObjCWritebackConversion=*/
+          getLangOpts().ObjCAutoRefCount, AllowExplicit);
       if (Conversions[ConvIdx].isBad())
         return true;
     }
@@ -9185,11 +9272,10 @@ class BuiltinOperatorOverloadBuilder {
     /// the candidates into a unique set, then move from that set into the list
     /// of arithmetic types.
     llvm::SmallSetVector<CanQualType, 2> BitIntCandidates;
-    llvm::for_each(CandidateTypes, [&BitIntCandidates](
-                                       BuiltinCandidateTypeSet &Candidate) {
+    for (BuiltinCandidateTypeSet &Candidate : CandidateTypes) {
       for (QualType BitTy : Candidate.bitint_types())
         BitIntCandidates.insert(CanQualType::CreateUnsafe(BitTy));
-    });
+    }
     llvm::move(BitIntCandidates, std::back_inserter(ArithmeticTypes));
     LastPromotedIntegralType = ArithmeticTypes.size();
     LastPromotedArithmeticType = ArithmeticTypes.size();
@@ -11235,9 +11321,9 @@ OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
                                                            SourceLocation Loc,
                                                            iterator &Best) {
 
-  assert(shouldDeferTemplateArgumentDeduction(S.getLangOpts()) ||
-         DeferredCandidatesCount == 0 &&
-             "Unexpected deferred template candidates");
+  assert((shouldDeferTemplateArgumentDeduction(S.getLangOpts()) ||
+          DeferredCandidatesCount == 0) &&
+         "Unexpected deferred template candidates");
 
   bool TwoPhaseResolution =
       DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
@@ -13968,8 +14054,10 @@ FunctionDecl *Sema::ResolveSingleFunctionTemplateSpecialization(
     //   specified and it, along with any default template arguments,
     //   identifies a single function template specialization, then the
     //   template-id is an lvalue for the function template specialization.
-    FunctionTemplateDecl *FunctionTemplate
-      = cast<FunctionTemplateDecl>((*I)->getUnderlyingDecl());
+    FunctionTemplateDecl *FunctionTemplate =
+        dyn_cast<FunctionTemplateDecl>((*I)->getUnderlyingDecl());
+    if (!FunctionTemplate)
+      continue;
 
     // C++ [over.over]p2:
     //   If the name is a function template, template argument deduction is
@@ -16894,6 +16982,9 @@ ExprResult Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
   }
 
   if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(E)) {
+    if (Found.getAccess() == AS_none) {
+      CheckUnresolvedLookupAccess(ULE, Found);
+    }
     // FIXME: avoid copy.
     TemplateArgumentListInfo TemplateArgsBuffer, *TemplateArgs = nullptr;
     if (ULE->hasExplicitTemplateArgs()) {
diff --git a/external/llvm-project/clang/lib/Sema/SemaRISCV.cpp b/external/llvm-project/clang/lib/Sema/SemaRISCV.cpp
index ac88f5e059b7..9eab0c2a0df6 100644
--- a/external/llvm-project/clang/lib/Sema/SemaRISCV.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaRISCV.cpp
@@ -46,6 +46,9 @@ struct RVVIntrinsicDef {
   /// Mapping to which clang built-in function, e.g. __builtin_rvv_vadd.
   std::string BuiltinName;
 
+  /// Mapping to RequiredFeatures in riscv_vector.td
+  StringRef RequiredExtensions;
+
   /// Function signature, first element is return type.
   RVVTypes Signature;
 };
@@ -177,7 +180,6 @@ namespace {
 class RISCVIntrinsicManagerImpl : public sema::RISCVIntrinsicManager {
 private:
   Sema &S;
-  ASTContext &Context;
   RVVTypeCache TypeCache;
   bool ConstructedRISCVVBuiltins;
   bool ConstructedRISCVSiFiveVectorBuiltins;
@@ -204,7 +206,7 @@ class RISCVIntrinsicManagerImpl : public sema::RISCVIntrinsicManager {
                               IntrinsicKind K);
 
 public:
-  RISCVIntrinsicManagerImpl(clang::Sema &S) : S(S), Context(S.Context) {
+  RISCVIntrinsicManagerImpl(clang::Sema &S) : S(S) {
     ConstructedRISCVVBuiltins = false;
     ConstructedRISCVSiFiveVectorBuiltins = false;
     ConstructedRISCVAndesVectorBuiltins = false;
@@ -222,40 +224,9 @@ class RISCVIntrinsicManagerImpl : public sema::RISCVIntrinsicManager {
 
 void RISCVIntrinsicManagerImpl::ConstructRVVIntrinsics(
     ArrayRef<RVVIntrinsicRecord> Recs, IntrinsicKind K) {
-  const TargetInfo &TI = Context.getTargetInfo();
-  static const std::pair<const char *, unsigned> FeatureCheckList[] = {
-      {"64bit", RVV_REQ_RV64},
-      {"xandesvdot", RVV_REQ_Xandesvdot},
-      {"xandesvpackfph", RVV_REQ_Xandesvpackfph},
-      {"xsfvcp", RVV_REQ_Xsfvcp},
-      {"xsfvfnrclipxfqf", RVV_REQ_Xsfvfnrclipxfqf},
-      {"xsfvfwmaccqqq", RVV_REQ_Xsfvfwmaccqqq},
-      {"xsfvqmaccdod", RVV_REQ_Xsfvqmaccdod},
-      {"xsfvqmaccqoq", RVV_REQ_Xsfvqmaccqoq},
-      {"zvbb", RVV_REQ_Zvbb},
-      {"zvbc", RVV_REQ_Zvbc},
-      {"zvkb", RVV_REQ_Zvkb},
-      {"zvkg", RVV_REQ_Zvkg},
-      {"zvkned", RVV_REQ_Zvkned},
-      {"zvknha", RVV_REQ_Zvknha},
-      {"zvknhb", RVV_REQ_Zvknhb},
-      {"zvksed", RVV_REQ_Zvksed},
-      {"zvksh", RVV_REQ_Zvksh},
-      {"zvfbfwma", RVV_REQ_Zvfbfwma},
-      {"zvfbfmin", RVV_REQ_Zvfbfmin},
-      {"zvfh", RVV_REQ_Zvfh},
-      {"experimental", RVV_REQ_Experimental}};
-
   // Construction of RVVIntrinsicRecords need to sync with createRVVIntrinsics
   // in RISCVVEmitter.cpp.
   for (auto &Record : Recs) {
-    // Check requirements.
-    if (llvm::any_of(FeatureCheckList, [&](const auto &Item) {
-          return Record.RequiredExtensions[Item.second] &&
-                 !TI.hasFeature(Item.first);
-        }))
-      continue;
-
     // Create Intrinsics for each type and LMUL.
     BasicType BaseType = BasicType::Unknown;
     ArrayRef<PrototypeDescriptor> BasicProtoSeq =
@@ -414,7 +385,7 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic(
   uint32_t Index = IntrinsicList.size();
   assert(IntrinsicList.size() == (size_t)Index &&
          "Intrinsics indices overflow.");
-  IntrinsicList.push_back({BuiltinName, Signature});
+  IntrinsicList.push_back({BuiltinName, Record.RequiredExtensions, Signature});
 
   // Creating mapping to Intrinsics.
   Intrinsics.insert({Name, Index});
@@ -477,6 +448,9 @@ void RISCVIntrinsicManagerImpl::CreateRVVIntrinsicDecl(LookupResult &LR,
   if (IsOverload)
     RVVIntrinsicDecl->addAttr(OverloadableAttr::CreateImplicit(Context));
 
+  if (IDef.RequiredExtensions != "")
+    RVVIntrinsicDecl->addAttr(
+        TargetAttr::CreateImplicit(Context, IDef.RequiredExtensions));
   // Setup alias to __builtin_rvv_*
   IdentifierInfo &IntrinsicII =
       PP.getIdentifierTable().get("__builtin_rvv_" + IDef.BuiltinName);
@@ -585,6 +559,17 @@ bool SemaRISCV::CheckBuiltinFunctionCall(const TargetInfo &TI,
   llvm::StringMap<bool> FunctionFeatureMap;
   Context.getFunctionFeatureMap(FunctionFeatureMap, FD);
 
+  if (const auto *A = TheCall->getCalleeDecl()->getAttr<TargetAttr>()) {
+    StringRef FeaturesStr = A->getFeaturesStr();
+    llvm::SmallVector<StringRef> RequiredFeatures;
+    FeaturesStr.split(RequiredFeatures, ',');
+    for (auto RF : RequiredFeatures)
+      if (!TI.hasFeature(RF) && !FunctionFeatureMap.lookup(RF))
+        return Diag(TheCall->getBeginLoc(),
+                    diag::err_riscv_builtin_requires_extension)
+               << /* IsExtension */ true << TheCall->getSourceRange() << RF;
+  }
+
   // vmulh.vv, vmulh.vx, vmulhu.vv, vmulhu.vx, vmulhsu.vv, vmulhsu.vx,
   // vsmul.vv, vsmul.vx are not included for EEW=64 in Zve64*.
   switch (BuiltinID) {
@@ -782,6 +767,13 @@ bool SemaRISCV::CheckBuiltinFunctionCall(const TargetInfo &TI,
       return Diag(TheCall->getBeginLoc(),
                   diag::err_riscv_builtin_requires_extension)
              << /* IsExtension */ true << TheCall->getSourceRange() << "zvknhb";
+    // If ElemSize is 32, check at least zvknha or zvknhb is enabled.
+    if (!TI.hasFeature("zvknha") && !FunctionFeatureMap.lookup("zvknha") &&
+        !TI.hasFeature("zvknhb") && !FunctionFeatureMap.lookup("zvknhb"))
+      return Diag(TheCall->getBeginLoc(),
+                  diag::err_riscv_builtin_requires_extension)
+             << /* IsExtension */ true << TheCall->getSourceRange()
+             << "zvknha or zvknhb";
 
     return CheckInvalidVLENandLMUL(TI, FunctionFeatureMap, TheCall, SemaRef,
                                    Arg0Type, ElemSize * 4) ||
diff --git a/external/llvm-project/clang/lib/Sema/SemaSYCL.cpp b/external/llvm-project/clang/lib/Sema/SemaSYCL.cpp
index 1969d7b0ba83..3e03cb4bd5f9 100644
--- a/external/llvm-project/clang/lib/Sema/SemaSYCL.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaSYCL.cpp
@@ -199,7 +199,7 @@ void SemaSYCL::handleKernelAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  handleSimpleAttribute<SYCLKernelAttr>(*this, D, AL);
+  handleSimpleAttribute<DeviceKernelAttr>(*this, D, AL);
 }
 
 void SemaSYCL::handleKernelEntryPointAttr(Decl *D, const ParsedAttr &AL) {
diff --git a/external/llvm-project/clang/lib/Sema/SemaStmt.cpp b/external/llvm-project/clang/lib/Sema/SemaStmt.cpp
index 50f5757dff5b..923a9e81fbd6 100644
--- a/external/llvm-project/clang/lib/Sema/SemaStmt.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaStmt.cpp
@@ -535,12 +535,7 @@ Sema::ActOnCaseExpr(SourceLocation CaseLoc, ExprResult Val) {
     return ER;
   };
 
-  ExprResult Converted = CorrectDelayedTyposInExpr(
-      Val, /*InitDecl=*/nullptr, /*RecoverUncorrectedTypos=*/false,
-      CheckAndFinish);
-  if (Converted.get() == Val.get())
-    Converted = CheckAndFinish(Val.get());
-  return Converted;
+  return CheckAndFinish(Val.get());
 }
 
 StmtResult
@@ -2344,7 +2339,7 @@ StmtResult Sema::ActOnForEachLValueExpr(Expr *E) {
 static bool FinishForRangeVarDecl(Sema &SemaRef, VarDecl *Decl, Expr *Init,
                                   SourceLocation Loc, int DiagID) {
   if (Decl->getType()->isUndeducedType()) {
-    ExprResult Res = SemaRef.CorrectDelayedTyposInExpr(Init);
+    ExprResult Res = Init;
     if (!Res.isUsable()) {
       Decl->setInvalidDecl();
       return true;
@@ -3845,10 +3840,7 @@ bool Sema::DeduceFunctionTypeFromReturnExpr(FunctionDecl *FD,
 StmtResult
 Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp,
                       Scope *CurScope) {
-  // Correct typos, in case the containing function returns 'auto' and
-  // RetValExp should determine the deduced type.
-  ExprResult RetVal = CorrectDelayedTyposInExpr(
-      RetValExp, nullptr, /*RecoverUncorrectedTypos=*/true);
+  ExprResult RetVal = RetValExp;
   if (RetVal.isInvalid())
     return StmtError();
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaStmtAttr.cpp b/external/llvm-project/clang/lib/Sema/SemaStmtAttr.cpp
index 17da5fd8325b..857d46af9ada 100644
--- a/external/llvm-project/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaStmtAttr.cpp
@@ -672,12 +672,14 @@ static Attr *ProcessStmtAttribute(Sema &S, Stmt *St, const ParsedAttr &A,
       !(A.existsInTarget(S.Context.getTargetInfo()) ||
         (S.Context.getLangOpts().SYCLIsDevice && Aux &&
          A.existsInTarget(*Aux)))) {
-    S.Diag(A.getLoc(), A.isRegularKeywordAttribute()
-                           ? (unsigned)diag::err_keyword_not_supported_on_target
-                       : A.isDeclspecAttribute()
-                           ? (unsigned)diag::warn_unhandled_ms_attribute_ignored
-                           : (unsigned)diag::warn_unknown_attribute_ignored)
-        << A << A.getRange();
+    if (A.isRegularKeywordAttribute() || A.isDeclspecAttribute()) {
+      S.Diag(A.getLoc(), A.isRegularKeywordAttribute()
+                             ? diag::err_keyword_not_supported_on_target
+                             : diag::warn_unhandled_ms_attribute_ignored)
+          << A << A.getRange();
+    } else {
+      S.DiagnoseUnknownAttribute(A);
+    }
     return nullptr;
   }
 
@@ -782,11 +784,10 @@ ExprResult Sema::ActOnCXXAssumeAttr(Stmt *St, const ParsedAttr &A,
 ExprResult Sema::BuildCXXAssumeExpr(Expr *Assumption,
                                     const IdentifierInfo *AttrName,
                                     SourceRange Range) {
-  ExprResult Res = CorrectDelayedTyposInExpr(Assumption);
-  if (Res.isInvalid())
+  if (!Assumption)
     return ExprError();
 
-  Res = CheckPlaceholderExpr(Res.get());
+  ExprResult Res = CheckPlaceholderExpr(Assumption);
   if (Res.isInvalid())
     return ExprError();
 
diff --git a/external/llvm-project/clang/lib/Sema/SemaTemplate.cpp b/external/llvm-project/clang/lib/Sema/SemaTemplate.cpp
index 1d8fea1d7200..ccc532e1fd3d 100644
--- a/external/llvm-project/clang/lib/Sema/SemaTemplate.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaTemplate.cpp
@@ -4749,6 +4749,8 @@ Sema::CheckConceptTemplateId(const CXXScopeSpec &SS,
   EnterExpressionEvaluationContext EECtx{
       *this, ExpressionEvaluationContext::Unevaluated, CSD};
 
+  ContextRAII CurContext(*this, CSD->getDeclContext(),
+                         /*NewThisContext=*/false);
   if (!AreArgsDependent &&
       CheckConstraintSatisfaction(
           NamedConcept, AssociatedConstraint(NamedConcept->getConstraintExpr()),
diff --git a/external/llvm-project/clang/lib/Sema/SemaTemplateDeduction.cpp b/external/llvm-project/clang/lib/Sema/SemaTemplateDeduction.cpp
index 5566057cccef..94edd553aacd 100644
--- a/external/llvm-project/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -3885,7 +3885,7 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
     TemplateDeductionInfo &Info,
     SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs,
     bool PartialOverloading, bool PartialOrdering,
-    llvm::function_ref<bool()> CheckNonDependent) {
+    llvm::function_ref<bool(bool)> CheckNonDependent) {
   // Unevaluated SFINAE context.
   EnterExpressionEvaluationContext Unevaluated(
       *this, Sema::ExpressionEvaluationContext::Unevaluated);
@@ -3914,18 +3914,6 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
       Result != TemplateDeductionResult::Success)
     return Result;
 
-  // C++ [temp.deduct.call]p10: [DR1391]
-  //   If deduction succeeds for all parameters that contain
-  //   template-parameters that participate in template argument deduction,
-  //   and all template arguments are explicitly specified, deduced, or
-  //   obtained from default template arguments, remaining parameters are then
-  //   compared with the corresponding arguments. For each remaining parameter
-  //   P with a type that was non-dependent before substitution of any
-  //   explicitly-specified template arguments, if the corresponding argument
-  //   A cannot be implicitly converted to P, deduction fails.
-  if (CheckNonDependent())
-    return TemplateDeductionResult::NonDependentConversionFailure;
-
   // Form the template argument list from the deduced template arguments.
   TemplateArgumentList *SugaredDeducedArgumentList =
       TemplateArgumentList::CreateCopy(Context, CTAI.SugaredConverted);
@@ -3940,6 +3928,42 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
     Owner = FunctionTemplate->getLexicalDeclContext();
   FunctionDecl *FD = FunctionTemplate->getTemplatedDecl();
 
+  if (CheckNonDependent(/*OnlyInitializeNonUserDefinedConversions=*/true))
+    return TemplateDeductionResult::NonDependentConversionFailure;
+
+  // C++20 [temp.deduct.general]p5: [CWG2369]
+  //   If the function template has associated constraints, those constraints
+  //   are checked for satisfaction. If the constraints are not satisfied, type
+  //   deduction fails.
+  //
+  // FIXME: We haven't implemented CWG2369 for lambdas yet, because we need
+  // to figure out how to instantiate lambda captures to the scope without
+  // first instantiating the lambda.
+  bool IsLambda = isLambdaCallOperator(FD) || isLambdaConversionOperator(FD);
+  if (!IsLambda && !IsIncomplete) {
+    if (CheckFunctionTemplateConstraints(
+            Info.getLocation(),
+            FunctionTemplate->getCanonicalDecl()->getTemplatedDecl(),
+            CTAI.CanonicalConverted, Info.AssociatedConstraintsSatisfaction))
+      return TemplateDeductionResult::MiscellaneousDeductionFailure;
+    if (!Info.AssociatedConstraintsSatisfaction.IsSatisfied) {
+      Info.reset(Info.takeSugared(), TemplateArgumentList::CreateCopy(
+                                         Context, CTAI.CanonicalConverted));
+      return TemplateDeductionResult::ConstraintsNotSatisfied;
+    }
+  }
+  // C++ [temp.deduct.call]p10: [CWG1391]
+  //   If deduction succeeds for all parameters that contain
+  //   template-parameters that participate in template argument deduction,
+  //   and all template arguments are explicitly specified, deduced, or
+  //   obtained from default template arguments, remaining parameters are then
+  //   compared with the corresponding arguments. For each remaining parameter
+  //   P with a type that was non-dependent before substitution of any
+  //   explicitly-specified template arguments, if the corresponding argument
+  //   A cannot be implicitly converted to P, deduction fails.
+  if (CheckNonDependent(/*OnlyInitializeNonUserDefinedConversions=*/false))
+    return TemplateDeductionResult::NonDependentConversionFailure;
+
   MultiLevelTemplateArgumentList SubstArgs(
       FunctionTemplate, CanonicalDeducedArgumentList->asArray(),
       /*Final=*/false);
@@ -3974,8 +3998,8 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
   //   ([temp.constr.decl]), those constraints are checked for satisfaction
   //   ([temp.constr.constr]). If the constraints are not satisfied, type
   //   deduction fails.
-  if (!IsIncomplete) {
-    if (CheckInstantiatedFunctionTemplateConstraints(
+  if (IsLambda && !IsIncomplete) {
+    if (CheckFunctionTemplateConstraints(
             Info.getLocation(), Specialization, CTAI.CanonicalConverted,
             Info.AssociatedConstraintsSatisfaction))
       return TemplateDeductionResult::MiscellaneousDeductionFailure;
@@ -4435,7 +4459,7 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
     bool PartialOrdering, QualType ObjectType,
     Expr::Classification ObjectClassification,
     bool ForOverloadSetAddressResolution,
-    llvm::function_ref<bool(ArrayRef<QualType>)> CheckNonDependent) {
+    llvm::function_ref<bool(ArrayRef<QualType>, bool)> CheckNonDependent) {
   if (FunctionTemplate->isInvalidDecl())
     return TemplateDeductionResult::Invalid;
 
@@ -4657,9 +4681,10 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
     Result = FinishTemplateArgumentDeduction(
         FunctionTemplate, Deduced, NumExplicitlySpecified, Specialization, Info,
         &OriginalCallArgs, PartialOverloading, PartialOrdering,
-        [&, CallingCtx]() {
+        [&, CallingCtx](bool OnlyInitializeNonUserDefinedConversions) {
           ContextRAII SavedContext(*this, CallingCtx);
-          return CheckNonDependent(ParamTypesForArgChecking);
+          return CheckNonDependent(ParamTypesForArgChecking,
+                                   OnlyInitializeNonUserDefinedConversions);
         });
   });
   return Result;
@@ -4740,8 +4765,8 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
                                           /*AdjustExceptionSpec*/false);
 
   // Unevaluated SFINAE context.
-  EnterExpressionEvaluationContext Unevaluated(
-      *this, Sema::ExpressionEvaluationContext::Unevaluated);
+  std::optional<EnterExpressionEvaluationContext> Unevaluated(
+      std::in_place, *this, Sema::ExpressionEvaluationContext::Unevaluated);
   SFINAETrap Trap(*this);
 
   Deduced.resize(TemplateParams->size());
@@ -4784,13 +4809,14 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
       DeduceReturnType(Specialization, Info.getLocation(), false))
     return TemplateDeductionResult::MiscellaneousDeductionFailure;
 
+  Unevaluated = std::nullopt;
   // [C++26][expr.const]/p17
   // An expression or conversion is immediate-escalating if it is not initially
   // in an immediate function context and it is [...]
   // a potentially-evaluated id-expression that denotes an immediate function.
   if (IsAddressOfFunction && getLangOpts().CPlusPlus20 &&
       Specialization->isImmediateEscalating() &&
-      parentEvaluationContext().isPotentiallyEvaluated() &&
+      currentEvaluationContext().isPotentiallyEvaluated() &&
       CheckIfFunctionSpecializationIsImmediate(Specialization,
                                                Info.getLocation()))
     return TemplateDeductionResult::MiscellaneousDeductionFailure;
diff --git a/external/llvm-project/clang/lib/Sema/SemaTemplateDeductionGuide.cpp b/external/llvm-project/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
index c75013b05d99..bdc46a0115a4 100644
--- a/external/llvm-project/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
@@ -955,10 +955,12 @@ Expr *buildIsDeducibleConstraint(Sema &SemaRef,
       Context.getTrivialTypeSourceInfo(
           Context.getDeducedTemplateSpecializationType(
               TemplateName(AliasTemplate), /*DeducedType=*/QualType(),
-              /*IsDependent=*/true)), // template specialization type whose
-                                      // arguments will be deduced.
+              /*IsDependent=*/true),
+          AliasTemplate->getLocation()), // template specialization type whose
+                                         // arguments will be deduced.
       Context.getTrivialTypeSourceInfo(
-          ReturnType), // type from which template arguments are deduced.
+          ReturnType, AliasTemplate->getLocation()), // type from which template
+                                                     // arguments are deduced.
   };
   return TypeTraitExpr::Create(
       Context, Context.getLogicalOperationType(), AliasTemplate->getLocation(),
diff --git a/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiate.cpp b/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 3a04a24d3aa7..a1c7143e3b87 100644
--- a/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -474,6 +474,21 @@ MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
   assert((ND || DC) && "Can't find arguments for a decl if one isn't provided");
   // Accumulate the set of template argument lists in this structure.
   MultiLevelTemplateArgumentList Result;
+  getTemplateInstantiationArgs(
+      Result, ND, DC, Final, Innermost, RelativeToPrimary, Pattern,
+      ForConstraintInstantiation, SkipForSpecialization,
+      ForDefaultArgumentSubstitution);
+  return Result;
+}
+
+void Sema::getTemplateInstantiationArgs(
+    MultiLevelTemplateArgumentList &Result, const NamedDecl *ND,
+    const DeclContext *DC, bool Final,
+    std::optional<ArrayRef<TemplateArgument>> Innermost, bool RelativeToPrimary,
+    const FunctionDecl *Pattern, bool ForConstraintInstantiation,
+    bool SkipForSpecialization, bool ForDefaultArgumentSubstitution) {
+  assert((ND || DC) && "Can't find arguments for a decl if one isn't provided");
+  // Accumulate the set of template argument lists in this structure.
 
   using namespace TemplateInstArgsHelpers;
   const Decl *CurDecl = ND;
@@ -533,14 +548,12 @@ MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
     }
 
     if (R.IsDone)
-      return Result;
+      return;
     if (R.ClearRelativeToPrimary)
       RelativeToPrimary = false;
     assert(R.NextDecl);
     CurDecl = R.NextDecl;
   }
-
-  return Result;
 }
 
 bool Sema::CodeSynthesisContext::isInstantiationRecord() const {
@@ -609,6 +622,12 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
     Inst.NumTemplateArgs = TemplateArgs.size();
     Inst.DeductionInfo = DeductionInfo;
     Inst.InstantiationRange = InstantiationRange;
+    Inst.InConstraintSubstitution =
+        Inst.Kind == CodeSynthesisContext::ConstraintSubstitution;
+    if (!SemaRef.CodeSynthesisContexts.empty())
+      Inst.InConstraintSubstitution |=
+          SemaRef.CodeSynthesisContexts.back().InConstraintSubstitution;
+
     SemaRef.pushCodeSynthesisContext(Inst);
 
     AlreadyInstantiating = !Inst.Entity ? false :
@@ -1375,6 +1394,12 @@ namespace {
     // Whether an incomplete substituion should be treated as an error.
     bool BailOutOnIncomplete;
 
+  private:
+    // CWG2770: Function parameters should be instantiated when they are
+    // needed by a satisfaction check of an atomic constraint or
+    // (recursively) by another function parameter.
+    bool maybeInstantiateFunctionParameterToScope(ParmVarDecl *OldParm);
+
   public:
     typedef TreeTransform<TemplateInstantiator> inherited;
 
@@ -1431,12 +1456,19 @@ namespace {
                                  ArrayRef<UnexpandedParameterPack> Unexpanded,
                                  bool &ShouldExpand, bool &RetainExpansion,
                                  UnsignedOrNone &NumExpansions) {
-      return getSema().CheckParameterPacksForExpansion(EllipsisLoc,
-                                                       PatternRange, Unexpanded,
-                                                       TemplateArgs,
-                                                       ShouldExpand,
-                                                       RetainExpansion,
-                                                       NumExpansions);
+      if (SemaRef.CurrentInstantiationScope &&
+          SemaRef.inConstraintSubstitution()) {
+        for (UnexpandedParameterPack ParmPack : Unexpanded) {
+          NamedDecl *VD = ParmPack.first.dyn_cast<NamedDecl *>();
+          if (auto *PVD = dyn_cast_if_present<ParmVarDecl>(VD);
+              PVD && maybeInstantiateFunctionParameterToScope(PVD))
+            return true;
+        }
+      }
+
+      return getSema().CheckParameterPacksForExpansion(
+          EllipsisLoc, PatternRange, Unexpanded, TemplateArgs, ShouldExpand,
+          RetainExpansion, NumExpansions);
     }
 
     void ExpandingFunctionParameterPack(ParmVarDecl *Pack) {
@@ -1920,9 +1952,57 @@ Decl *TemplateInstantiator::TransformDecl(SourceLocation Loc, Decl *D) {
     // template parameter.
   }
 
+  if (ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(D);
+      PVD && SemaRef.CurrentInstantiationScope &&
+      SemaRef.inConstraintSubstitution() &&
+      maybeInstantiateFunctionParameterToScope(PVD))
+    return nullptr;
+
   return SemaRef.FindInstantiatedDecl(Loc, cast<NamedDecl>(D), TemplateArgs);
 }
 
+bool TemplateInstantiator::maybeInstantiateFunctionParameterToScope(
+    ParmVarDecl *OldParm) {
+  if (SemaRef.CurrentInstantiationScope->getInstantiationOfIfExists(OldParm))
+    return false;
+
+  if (!OldParm->isParameterPack())
+    return !TransformFunctionTypeParam(OldParm, /*indexAdjustment=*/0,
+                                       /*NumExpansions=*/std::nullopt,
+                                       /*ExpectParameterPack=*/false);
+
+  SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+
+  // Find the parameter packs that could be expanded.
+  TypeLoc TL = OldParm->getTypeSourceInfo()->getTypeLoc();
+  PackExpansionTypeLoc ExpansionTL = TL.castAs<PackExpansionTypeLoc>();
+  TypeLoc Pattern = ExpansionTL.getPatternLoc();
+  SemaRef.collectUnexpandedParameterPacks(Pattern, Unexpanded);
+  assert(!Unexpanded.empty() && "Pack expansion without parameter packs?");
+
+  bool ShouldExpand = false;
+  bool RetainExpansion = false;
+  UnsignedOrNone OrigNumExpansions =
+      ExpansionTL.getTypePtr()->getNumExpansions();
+  UnsignedOrNone NumExpansions = OrigNumExpansions;
+  if (TryExpandParameterPacks(ExpansionTL.getEllipsisLoc(),
+                              Pattern.getSourceRange(), Unexpanded,
+                              ShouldExpand, RetainExpansion, NumExpansions))
+    return true;
+
+  assert(ShouldExpand && !RetainExpansion &&
+         "Shouldn't preserve pack expansion when evaluating constraints");
+  ExpandingFunctionParameterPack(OldParm);
+  for (unsigned I = 0; I != *NumExpansions; ++I) {
+    Sema::ArgPackSubstIndexRAII SubstIndex(getSema(), I);
+    if (!TransformFunctionTypeParam(OldParm, /*indexAdjustment=*/0,
+                                    /*NumExpansions=*/OrigNumExpansions,
+                                    /*ExpectParameterPack=*/false))
+      return true;
+  }
+  return false;
+}
+
 Decl *TemplateInstantiator::TransformDefinition(SourceLocation Loc, Decl *D) {
   Decl *Inst = getSema().SubstDecl(D, getSema().CurContext, TemplateArgs);
   if (!Inst)
@@ -4502,9 +4582,8 @@ static const Decl *getCanonicalParmVarDecl(const Decl *D) {
   return D;
 }
 
-
 llvm::PointerUnion<Decl *, LocalInstantiationScope::DeclArgumentPack *> *
-LocalInstantiationScope::findInstantiationOf(const Decl *D) {
+LocalInstantiationScope::getInstantiationOfIfExists(const Decl *D) {
   D = getCanonicalParmVarDecl(D);
   for (LocalInstantiationScope *Current = this; Current;
        Current = Current->Outer) {
@@ -4529,6 +4608,14 @@ LocalInstantiationScope::findInstantiationOf(const Decl *D) {
       break;
   }
 
+  return nullptr;
+}
+
+llvm::PointerUnion<Decl *, LocalInstantiationScope::DeclArgumentPack *> *
+LocalInstantiationScope::findInstantiationOf(const Decl *D) {
+  auto *Result = getInstantiationOfIfExists(D);
+  if (Result)
+    return Result;
   // If we're performing a partial substitution during template argument
   // deduction, we may not have values for template parameters yet.
   if (isa<NonTypeTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
diff --git a/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 4ef2b653ee87..a25bfd1c48de 100644
--- a/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -12,7 +12,6 @@
 #include "TreeTransform.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
-#include "clang/AST/ASTLambda.h"
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DependentDiagnostic.h"
@@ -528,6 +527,7 @@ static void instantiateOMPDeclareVariantAttr(
 
   SmallVector<Expr *, 8> NothingExprs;
   SmallVector<Expr *, 8> NeedDevicePtrExprs;
+  SmallVector<Expr *, 8> NeedDeviceAddrExprs;
   SmallVector<OMPInteropInfo, 4> AppendArgs;
 
   for (Expr *E : Attr.adjustArgsNothing()) {
@@ -542,14 +542,20 @@ static void instantiateOMPDeclareVariantAttr(
       continue;
     NeedDevicePtrExprs.push_back(ER.get());
   }
+  for (Expr *E : Attr.adjustArgsNeedDeviceAddr()) {
+    ExprResult ER = Subst(E);
+    if (ER.isInvalid())
+      continue;
+    NeedDeviceAddrExprs.push_back(ER.get());
+  }
   for (OMPInteropInfo &II : Attr.appendArgs()) {
     // When prefer_type is implemented for append_args handle them here too.
     AppendArgs.emplace_back(II.IsTarget, II.IsTargetSync);
   }
 
   S.OpenMP().ActOnOpenMPDeclareVariantDirective(
-      FD, E, TI, NothingExprs, NeedDevicePtrExprs, AppendArgs, SourceLocation(),
-      SourceLocation(), Attr.getRange());
+      FD, E, TI, NothingExprs, NeedDevicePtrExprs, NeedDeviceAddrExprs,
+      AppendArgs, SourceLocation(), SourceLocation(), Attr.getRange());
 }
 
 static void instantiateDependentAMDGPUFlatWorkGroupSizeAttr(
@@ -677,9 +683,9 @@ static void instantiateDependentAMDGPUMaxNumWorkGroupsAttr(
 // This doesn't take any template parameters, but we have a custom action that
 // needs to happen when the kernel itself is instantiated. We need to run the
 // ItaniumMangler to mark the names required to name this kernel.
-static void instantiateDependentSYCLKernelAttr(
+static void instantiateDependentDeviceKernelAttr(
     Sema &S, const MultiLevelTemplateArgumentList &TemplateArgs,
-    const SYCLKernelAttr &Attr, Decl *New) {
+    const DeviceKernelAttr &Attr, Decl *New) {
   New->addAttr(Attr.clone(S.getASTContext()));
 }
 
@@ -921,8 +927,8 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
       continue;
     }
 
-    if (auto *A = dyn_cast<SYCLKernelAttr>(TmplAttr)) {
-      instantiateDependentSYCLKernelAttr(*this, TemplateArgs, *A, New);
+    if (auto *A = dyn_cast<DeviceKernelAttr>(TmplAttr)) {
+      instantiateDependentDeviceKernelAttr(*this, TemplateArgs, *A, New);
       continue;
     }
 
@@ -2395,7 +2401,7 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateDecl(VarTemplateDecl *D) {
     // Queue up any out-of-line partial specializations of this member
     // variable template; the client will force their instantiation once
     // the enclosing class has been instantiated.
-    SmallVector<VarTemplatePartialSpecializationDecl *, 4> PartialSpecs;
+    SmallVector<VarTemplatePartialSpecializationDecl *, 1> PartialSpecs;
     D->getPartialSpecializations(PartialSpecs);
     for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I)
       if (PartialSpecs[I]->getFirstDecl()->isOutOfLine())
@@ -5846,8 +5852,24 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
     savedContext.pop();
   }
 
-  DeclGroupRef DG(Function);
-  Consumer.HandleTopLevelDecl(DG);
+  // We never need to emit the code for a lambda in unevaluated context.
+  // We also can't mangle a lambda in the require clause of a function template
+  // during constraint checking as the MSI ABI would need to mangle the (not yet
+  // specialized) enclosing declaration
+  // FIXME: Should we try to skip this for non-lambda functions too?
+  bool ShouldSkipCG = [&] {
+    auto *RD = dyn_cast<CXXRecordDecl>(Function->getParent());
+    if (!RD || !RD->isLambda())
+      return false;
+
+    return llvm::any_of(ExprEvalContexts, [](auto &Context) {
+      return Context.isUnevaluated() || Context.isImmediateFunctionContext();
+    });
+  }();
+  if (!ShouldSkipCG) {
+    DeclGroupRef DG(Function);
+    Consumer.HandleTopLevelDecl(DG);
+  }
 
   // This class may have local implicit instantiations that need to be
   // instantiation within this scope.
diff --git a/external/llvm-project/clang/lib/Sema/SemaTemplateVariadic.cpp b/external/llvm-project/clang/lib/Sema/SemaTemplateVariadic.cpp
index 5f0e968ff18c..572dbf2e7393 100644
--- a/external/llvm-project/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -741,7 +741,6 @@ ExprResult Sema::CheckPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc,
   if (!Pattern->containsUnexpandedParameterPack()) {
     Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
     << Pattern->getSourceRange();
-    CorrectDelayedTyposInExpr(Pattern);
     return ExprError();
   }
 
@@ -1201,11 +1200,9 @@ ExprResult Sema::ActOnPackIndexingExpr(Scope *S, Expr *PackExpression,
                                        SourceLocation RSquareLoc) {
   bool isParameterPack = ::isParameterPack(PackExpression);
   if (!isParameterPack) {
-    if (!PackExpression->containsErrors()) {
-      CorrectDelayedTyposInExpr(IndexExpr);
+    if (!PackExpression->containsErrors())
       Diag(PackExpression->getBeginLoc(), diag::err_expected_name_of_pack)
           << PackExpression;
-    }
     return ExprError();
   }
   ExprResult Res =
@@ -1403,11 +1400,6 @@ ExprResult Sema::ActOnCXXFoldExpr(Scope *S, SourceLocation LParenLoc, Expr *LHS,
   CheckFoldOperand(*this, LHS);
   CheckFoldOperand(*this, RHS);
 
-  auto DiscardOperands = [&] {
-    CorrectDelayedTyposInExpr(LHS);
-    CorrectDelayedTyposInExpr(RHS);
-  };
-
   // [expr.prim.fold]p3:
   //   In a binary fold, op1 and op2 shall be the same fold-operator, and
   //   either e1 shall contain an unexpanded parameter pack or e2 shall contain
@@ -1415,7 +1407,6 @@ ExprResult Sema::ActOnCXXFoldExpr(Scope *S, SourceLocation LParenLoc, Expr *LHS,
   if (LHS && RHS &&
       LHS->containsUnexpandedParameterPack() ==
           RHS->containsUnexpandedParameterPack()) {
-    DiscardOperands();
     return Diag(EllipsisLoc,
                 LHS->containsUnexpandedParameterPack()
                     ? diag::err_fold_expression_packs_both_sides
@@ -1430,7 +1421,6 @@ ExprResult Sema::ActOnCXXFoldExpr(Scope *S, SourceLocation LParenLoc, Expr *LHS,
     Expr *Pack = LHS ? LHS : RHS;
     assert(Pack && "fold expression with neither LHS nor RHS");
     if (!Pack->containsUnexpandedParameterPack()) {
-      DiscardOperands();
       return Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
              << Pack->getSourceRange();
     }
diff --git a/external/llvm-project/clang/lib/Sema/SemaType.cpp b/external/llvm-project/clang/lib/Sema/SemaType.cpp
index 338b81fe8974..785d7b89e778 100644
--- a/external/llvm-project/clang/lib/Sema/SemaType.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaType.cpp
@@ -134,7 +134,7 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   case ParsedAttr::AT_VectorCall:                                              \
   case ParsedAttr::AT_AArch64VectorPcs:                                        \
   case ParsedAttr::AT_AArch64SVEPcs:                                           \
-  case ParsedAttr::AT_AMDGPUKernelCall:                                        \
+  case ParsedAttr::AT_DeviceKernel:                                            \
   case ParsedAttr::AT_MSABI:                                                   \
   case ParsedAttr::AT_SysVABI:                                                 \
   case ParsedAttr::AT_Pcs:                                                     \
@@ -155,6 +155,7 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   case ParsedAttr::AT_Blocking:                                                \
   case ParsedAttr::AT_Allocating:                                              \
   case ParsedAttr::AT_Regparm:                                                 \
+  case ParsedAttr::AT_CFIUncheckedCallee:                                      \
   case ParsedAttr::AT_CmseNSCall:                                              \
   case ParsedAttr::AT_ArmStreaming:                                            \
   case ParsedAttr::AT_ArmStreamingCompatible:                                  \
@@ -3754,18 +3755,7 @@ static CallingConv getCCForDeclaratorChunk(
   CallingConv CC = S.Context.getDefaultCallingConvention(FTI.isVariadic,
                                                          IsCXXInstanceMethod);
 
-  // Attribute AT_OpenCLKernel affects the calling convention for SPIR
-  // and AMDGPU targets, hence it cannot be treated as a calling
-  // convention attribute. This is the simplest place to infer
-  // calling convention for OpenCL kernels.
-  if (S.getLangOpts().OpenCL) {
-    for (const ParsedAttr &AL : D.getDeclSpec().getAttributes()) {
-      if (AL.getKind() == ParsedAttr::AT_OpenCLKernel) {
-        CC = CC_OpenCLKernel;
-        break;
-      }
-    }
-  } else if (S.getLangOpts().CUDA) {
+  if (S.getLangOpts().CUDA) {
     // If we're compiling CUDA/HIP code and targeting HIPSPV we need to make
     // sure the kernels will be marked with the right calling convention so that
     // they will be visible by the APIs that ingest SPIR-V. We do not do this
@@ -3774,13 +3764,20 @@ static CallingConv getCCForDeclaratorChunk(
     if (Triple.isSPIRV() && Triple.getVendor() != llvm::Triple::AMD) {
       for (const ParsedAttr &AL : D.getDeclSpec().getAttributes()) {
         if (AL.getKind() == ParsedAttr::AT_CUDAGlobal) {
-          CC = CC_OpenCLKernel;
+          CC = CC_DeviceKernel;
           break;
         }
       }
     }
   }
-
+  if (!S.getLangOpts().isSYCL()) {
+    for (const ParsedAttr &AL : D.getDeclSpec().getAttributes()) {
+      if (AL.getKind() == ParsedAttr::AT_DeviceKernel) {
+        CC = CC_DeviceKernel;
+        break;
+      }
+    }
+  }
   return CC;
 }
 
@@ -4555,7 +4552,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
                                           false /*IsRegularKeywordAttribute*/);
       ParsedAttr *nullabilityAttr = Pool.create(
           S.getNullabilityKeyword(*inferNullability), SourceRange(pointerLoc),
-          nullptr, SourceLocation(), nullptr, 0, form);
+          AttributeScopeInfo(), nullptr, 0, form);
 
       attrs.addAtEnd(nullabilityAttr);
 
@@ -5738,10 +5735,10 @@ static void transferARCOwnershipToDeclaratorChunk(TypeProcessingState &state,
 
   // If there wasn't one, add one (with an invalid source location
   // so that we don't make an AttributedType for it).
-  ParsedAttr *attr = D.getAttributePool().create(
-      &S.Context.Idents.get("objc_ownership"), SourceLocation(),
-      /*scope*/ nullptr, SourceLocation(),
-      /*args*/ &Args, 1, ParsedAttr::Form::GNU());
+  ParsedAttr *attr =
+      D.getAttributePool().create(&S.Context.Idents.get("objc_ownership"),
+                                  SourceLocation(), AttributeScopeInfo(),
+                                  /*args*/ &Args, 1, ParsedAttr::Form::GNU());
   chunk.getAttrs().addAtEnd(attr);
   // TODO: mark whether we did this inference?
 }
@@ -7531,8 +7528,8 @@ static Attr *getCCTypeAttr(ASTContext &Ctx, ParsedAttr &Attr) {
     return createSimpleAttr<AArch64SVEPcsAttr>(Ctx, Attr);
   case ParsedAttr::AT_ArmStreaming:
     return createSimpleAttr<ArmStreamingAttr>(Ctx, Attr);
-  case ParsedAttr::AT_AMDGPUKernelCall:
-    return createSimpleAttr<AMDGPUKernelCallAttr>(Ctx, Attr);
+  case ParsedAttr::AT_DeviceKernel:
+    return createSimpleAttr<DeviceKernelAttr>(Ctx, Attr);
   case ParsedAttr::AT_Pcs: {
     // The attribute may have had a fixit applied where we treated an
     // identifier as a string literal.  The contents of the string are valid,
@@ -7814,6 +7811,27 @@ static bool handleFunctionTypeAttr(TypeProcessingState &state, ParsedAttr &attr,
     return true;
   }
 
+  if (attr.getKind() == ParsedAttr::AT_CFIUncheckedCallee) {
+    // Delay if this is not a prototyped function type.
+    if (!unwrapped.isFunctionType())
+      return false;
+
+    if (!unwrapped.get()->isFunctionProtoType()) {
+      S.Diag(attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+          << attr << attr.isRegularKeywordAttribute()
+          << ExpectedFunctionWithProtoType;
+      attr.setInvalid();
+      return true;
+    }
+
+    const auto *FPT = unwrapped.get()->getAs<FunctionProtoType>();
+    type = S.Context.getFunctionType(
+        FPT->getReturnType(), FPT->getParamTypes(),
+        FPT->getExtProtoInfo().withCFIUncheckedCallee(true));
+    type = unwrapped.wrap(S, cast<FunctionType>(type.getTypePtr()));
+    return true;
+  }
+
   if (attr.getKind() == ParsedAttr::AT_CmseNSCall) {
     // Delay if this is not a function type.
     if (!unwrapped.isFunctionType())
@@ -8720,6 +8738,16 @@ static void HandleHLSLParamModifierAttr(TypeProcessingState &State,
   }
 }
 
+static bool isMultiSubjectAttrAllowedOnType(const ParsedAttr &Attr) {
+  // The DeviceKernel attribute is shared for many targets, and
+  // it is only allowed to be a type attribute with the AMDGPU
+  // spelling, so skip processing the attr as a type attr
+  // unless it has that spelling.
+  if (Attr.getKind() != ParsedAttr::AT_DeviceKernel)
+    return true;
+  return DeviceKernelAttr::isAMDGPUSpelling(Attr);
+}
+
 static void processTypeAttrs(TypeProcessingState &state, QualType &type,
                              TypeAttrLocation TAL,
                              const ParsedAttributesView &attrs,
@@ -8973,6 +9001,9 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
         break;
       [[fallthrough]];
     FUNCTION_TYPE_ATTRS_CASELIST:
+      if (!isMultiSubjectAttrAllowedOnType(attr))
+        break;
+
       attr.setUsedAsTypeAttr();
 
       // Attributes with standard syntax have strict rules for what they
diff --git a/external/llvm-project/clang/lib/Sema/SemaTypeTraits.cpp b/external/llvm-project/clang/lib/Sema/SemaTypeTraits.cpp
index 7bf3c8eaabf4..4dbb2450857e 100644
--- a/external/llvm-project/clang/lib/Sema/SemaTypeTraits.cpp
+++ b/external/llvm-project/clang/lib/Sema/SemaTypeTraits.cpp
@@ -11,8 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/Type.h"
 #include "clang/Basic/DiagnosticParse.h"
 #include "clang/Basic/DiagnosticSema.h"
+#include "clang/Basic/TypeTraits.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
@@ -102,7 +104,8 @@ static CXXMethodDecl *LookupSpecialMemberFromXValue(Sema &SemaRef,
   OverloadCandidateSet::iterator Best;
   switch (OCS.BestViableFunction(SemaRef, LookupLoc, Best)) {
   case OR_Success:
-    return cast<CXXMethodDecl>(Best->Function);
+  case OR_Deleted:
+    return cast<CXXMethodDecl>(Best->Function)->getCanonicalDecl();
   default:
     return nullptr;
   }
@@ -118,7 +121,8 @@ static bool hasSuitableConstructorForRelocation(Sema &SemaRef,
 
   CXXMethodDecl *Decl =
       LookupSpecialMemberFromXValue(SemaRef, D, /*Assign=*/false);
-  return Decl && Decl->isUserProvided() == AllowUserDefined;
+  return Decl && Decl->isUserProvided() == AllowUserDefined &&
+         !Decl->isDeleted();
 }
 
 static bool hasSuitableMoveAssignmentOperatorForRelocation(
@@ -133,7 +137,8 @@ static bool hasSuitableMoveAssignmentOperatorForRelocation(
   if (!Decl)
     return false;
 
-  return Decl && Decl->isUserProvided() == AllowUserDefined;
+  return Decl && Decl->isUserProvided() == AllowUserDefined &&
+         !Decl->isDeleted();
 }
 
 // [C++26][class.prop]
@@ -159,6 +164,8 @@ static bool IsDefaultMovable(Sema &SemaRef, const CXXRecordDecl *D) {
   if (!Dtr)
     return true;
 
+  Dtr = Dtr->getCanonicalDecl();
+
   if (Dtr->isUserProvided() && (!Dtr->isDefaulted() || Dtr->isDeleted()))
     return false;
 
@@ -181,6 +188,7 @@ static bool IsEligibleForTrivialRelocation(Sema &SemaRef,
       return false;
   }
 
+  bool IsUnion = D->isUnion();
   for (const FieldDecl *Field : D->fields()) {
     if (Field->getType()->isDependentType())
       continue;
@@ -190,6 +198,12 @@ static bool IsEligibleForTrivialRelocation(Sema &SemaRef,
     // of a trivially relocatable type
     if (!SemaRef.IsCXXTriviallyRelocatableType(Field->getType()))
       return false;
+
+    // A union contains values with address discriminated pointer auth
+    // cannot be relocated.
+    if (IsUnion && SemaRef.Context.containsAddressDiscriminatedPointerAuth(
+                       Field->getType()))
+      return false;
   }
   return !D->hasDeletedDestructor();
 }
@@ -295,18 +309,17 @@ Sema::CheckCXX2CRelocatableAndReplaceable(const CXXRecordDecl *D) {
   return Info;
 }
 
-static bool IsCXXTriviallyRelocatableType(Sema &S, const CXXRecordDecl *RD) {
+bool Sema::IsCXXTriviallyRelocatableType(const CXXRecordDecl &RD) {
   if (std::optional<ASTContext::CXXRecordDeclRelocationInfo> Info =
-          S.getASTContext().getRelocationInfoForCXXRecord(RD))
+          getASTContext().getRelocationInfoForCXXRecord(&RD))
     return Info->IsRelocatable;
   ASTContext::CXXRecordDeclRelocationInfo Info =
-      S.CheckCXX2CRelocatableAndReplaceable(RD);
-  S.getASTContext().setRelocationInfoForCXXRecord(RD, Info);
+      CheckCXX2CRelocatableAndReplaceable(&RD);
+  getASTContext().setRelocationInfoForCXXRecord(&RD, Info);
   return Info.IsRelocatable;
 }
 
 bool Sema::IsCXXTriviallyRelocatableType(QualType Type) {
-
   QualType BaseElementType = getASTContext().getBaseElementType(Type);
 
   if (Type->isVariableArrayType())
@@ -315,17 +328,17 @@ bool Sema::IsCXXTriviallyRelocatableType(QualType Type) {
   if (BaseElementType.hasNonTrivialObjCLifetime())
     return false;
 
-  if (BaseElementType.hasAddressDiscriminatedPointerAuth())
+  if (BaseElementType->isIncompleteType())
     return false;
 
-  if (BaseElementType->isIncompleteType())
+  if (Context.containsNonRelocatablePointerAuth(Type))
     return false;
 
   if (BaseElementType->isScalarType() || BaseElementType->isVectorType())
     return true;
 
   if (const auto *RD = BaseElementType->getAsCXXRecordDecl())
-    return ::IsCXXTriviallyRelocatableType(*this, RD);
+    return IsCXXTriviallyRelocatableType(*RD);
 
   return false;
 }
@@ -663,11 +676,14 @@ static bool IsTriviallyRelocatableType(Sema &SemaRef, QualType T) {
   if (!BaseElementType->isObjectType())
     return false;
 
-  if (T.hasAddressDiscriminatedPointerAuth())
+  // The deprecated __builtin_is_trivially_relocatable does not have
+  // an equivalent to __builtin_trivially_relocate, so there is no
+  // safe way to use it if there are any address discriminated values.
+  if (SemaRef.getASTContext().containsAddressDiscriminatedPointerAuth(T))
     return false;
 
   if (const auto *RD = BaseElementType->getAsCXXRecordDecl();
-      RD && !RD->isPolymorphic() && IsCXXTriviallyRelocatableType(SemaRef, RD))
+      RD && !RD->isPolymorphic() && SemaRef.IsCXXTriviallyRelocatableType(*RD))
     return true;
 
   if (const auto *RD = BaseElementType->getAsRecordDecl())
@@ -1343,12 +1359,21 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
     if (RD && RD->isAbstract())
       return false;
 
+    // LWG3819: For reference_meows_from_temporary traits, && is not added to
+    // the source object type.
+    // Otherwise, compute the result of add_rvalue_reference_t.
+    bool UseRawObjectType =
+        Kind == clang::BTT_ReferenceBindsToTemporary ||
+        Kind == clang::BTT_ReferenceConstructsFromTemporary ||
+        Kind == clang::BTT_ReferenceConvertsFromTemporary;
+
     llvm::BumpPtrAllocator OpaqueExprAllocator;
     SmallVector<Expr *, 2> ArgExprs;
     ArgExprs.reserve(Args.size() - 1);
     for (unsigned I = 1, N = Args.size(); I != N; ++I) {
       QualType ArgTy = Args[I]->getType();
-      if (ArgTy->isObjectType() || ArgTy->isFunctionType())
+      if ((ArgTy->isObjectType() && !UseRawObjectType) ||
+          ArgTy->isFunctionType())
         ArgTy = S.Context.getRValueReferenceType(ArgTy);
       ArgExprs.push_back(
           new (OpaqueExprAllocator.Allocate<OpaqueValueExpr>())
@@ -1386,6 +1411,10 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
       if (!T->isReferenceType())
         return false;
 
+      // A function reference never binds to a temporary object.
+      if (T.getNonReferenceType()->isFunctionType())
+        return false;
+
       if (!Init.isDirectReferenceBinding())
         return true;
 
@@ -1458,6 +1487,9 @@ void DiagnoseBuiltinDeprecation(Sema &S, TypeTrait Kind, SourceLocation KWLoc) {
   case UTT_IsTriviallyRelocatable:
     Replacement = clang::UTT_IsCppTriviallyRelocatable;
     break;
+  case BTT_ReferenceBindsToTemporary:
+    Replacement = clang::BTT_ReferenceConstructsFromTemporary;
+    break;
   default:
     return;
   }
@@ -1922,6 +1954,8 @@ static std::optional<TypeTrait> StdNameToTypeTrait(StringRef Name) {
   return llvm::StringSwitch<std::optional<TypeTrait>>(Name)
       .Case("is_trivially_relocatable",
             TypeTrait::UTT_IsCppTriviallyRelocatable)
+      .Case("is_replaceable", TypeTrait::UTT_IsReplaceable)
+      .Case("is_trivially_copyable", TypeTrait::UTT_IsTriviallyCopyable)
       .Default(std::nullopt);
 }
 
@@ -1986,6 +2020,50 @@ static ExtractedTypeTraitInfo ExtractTypeTraitFromExpression(const Expr *E) {
   return std::nullopt;
 }
 
+static void DiagnoseNonDefaultMovable(Sema &SemaRef, SourceLocation Loc,
+                                      const CXXRecordDecl *D) {
+  if (D->isUnion()) {
+    auto DiagSPM = [&](CXXSpecialMemberKind K, bool Has) {
+      if (Has)
+        SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+            << diag::TraitNotSatisfiedReason::UnionWithUserDeclaredSMF << K;
+    };
+    DiagSPM(CXXSpecialMemberKind::CopyConstructor,
+            D->hasUserDeclaredCopyConstructor());
+    DiagSPM(CXXSpecialMemberKind::CopyAssignment,
+            D->hasUserDeclaredCopyAssignment());
+    DiagSPM(CXXSpecialMemberKind::MoveConstructor,
+            D->hasUserDeclaredMoveConstructor());
+    DiagSPM(CXXSpecialMemberKind::MoveAssignment,
+            D->hasUserDeclaredMoveAssignment());
+    return;
+  }
+
+  if (!D->hasSimpleMoveConstructor() && !D->hasSimpleCopyConstructor()) {
+    const auto *Decl = cast_or_null<CXXConstructorDecl>(
+        LookupSpecialMemberFromXValue(SemaRef, D, /*Assign=*/false));
+    if (Decl && Decl->isUserProvided())
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::UserProvidedCtr
+          << Decl->isMoveConstructor() << Decl->getSourceRange();
+  }
+  if (!D->hasSimpleMoveAssignment() && !D->hasSimpleCopyAssignment()) {
+    CXXMethodDecl *Decl =
+        LookupSpecialMemberFromXValue(SemaRef, D, /*Assign=*/true);
+    if (Decl && Decl->isUserProvided())
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::UserProvidedAssign
+          << Decl->isMoveAssignmentOperator() << Decl->getSourceRange();
+  }
+  if (CXXDestructorDecl *Dtr = D->getDestructor()) {
+    Dtr = Dtr->getCanonicalDecl();
+    if (Dtr->isUserProvided() && !Dtr->isDefaulted())
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::DeletedDtr << /*User Provided*/ 1
+          << Dtr->getSourceRange();
+  }
+}
+
 static void DiagnoseNonTriviallyRelocatableReason(Sema &SemaRef,
                                                   SourceLocation Loc,
                                                   const CXXRecordDecl *D) {
@@ -1997,15 +2075,15 @@ static void DiagnoseNonTriviallyRelocatableReason(Sema &SemaRef,
           << B.getSourceRange();
     if (!SemaRef.IsCXXTriviallyRelocatableType(B.getType()))
       SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
-          << diag::TraitNotSatisfiedReason::NRBase << B.getType()
+          << diag::TraitNotSatisfiedReason::NTRBase << B.getType()
           << B.getSourceRange();
   }
   for (const FieldDecl *Field : D->fields()) {
     if (!Field->getType()->isReferenceType() &&
         !SemaRef.IsCXXTriviallyRelocatableType(Field->getType()))
       SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
-          << diag::TraitNotSatisfiedReason::NRField << Field << Field->getType()
-          << Field->getSourceRange();
+          << diag::TraitNotSatisfiedReason::NTRField << Field
+          << Field->getType() << Field->getSourceRange();
   }
   if (D->hasDeletedDestructor())
     SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
@@ -2014,52 +2092,84 @@ static void DiagnoseNonTriviallyRelocatableReason(Sema &SemaRef,
 
   if (D->hasAttr<TriviallyRelocatableAttr>())
     return;
+  DiagnoseNonDefaultMovable(SemaRef, Loc, D);
+}
 
-  if (D->isUnion()) {
-    auto DiagSPM = [&](CXXSpecialMemberKind K, bool Has) {
-      if (Has)
-        SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
-            << diag::TraitNotSatisfiedReason::UnionWithUserDeclaredSMF << K;
-    };
-    DiagSPM(CXXSpecialMemberKind::CopyConstructor,
-            D->hasUserDeclaredCopyConstructor());
-    DiagSPM(CXXSpecialMemberKind::CopyAssignment,
-            D->hasUserDeclaredCopyAssignment());
-    DiagSPM(CXXSpecialMemberKind::MoveConstructor,
-            D->hasUserDeclaredMoveConstructor());
-    DiagSPM(CXXSpecialMemberKind::MoveAssignment,
-            D->hasUserDeclaredMoveAssignment());
+static void DiagnoseNonTriviallyRelocatableReason(Sema &SemaRef,
+                                                  SourceLocation Loc,
+                                                  QualType T) {
+  SemaRef.Diag(Loc, diag::note_unsatisfied_trait)
+      << T << diag::TraitName::TriviallyRelocatable;
+  if (T->isVariablyModifiedType())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::VLA;
+
+  if (T->isReferenceType())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::Ref;
+  T = T.getNonReferenceType();
+
+  if (T.hasNonTrivialObjCLifetime())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::HasArcLifetime;
+
+  const CXXRecordDecl *D = T->getAsCXXRecordDecl();
+  if (!D || D->isInvalidDecl())
     return;
+
+  if (D->hasDefinition())
+    DiagnoseNonTriviallyRelocatableReason(SemaRef, Loc, D);
+
+  SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
+}
+
+static void DiagnoseNonReplaceableReason(Sema &SemaRef, SourceLocation Loc,
+                                         const CXXRecordDecl *D) {
+  for (const CXXBaseSpecifier &B : D->bases()) {
+    assert(B.getType()->getAsCXXRecordDecl() && "invalid base?");
+    if (!SemaRef.IsCXXReplaceableType(B.getType()))
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NonReplaceableBase << B.getType()
+          << B.getSourceRange();
+  }
+  for (const FieldDecl *Field : D->fields()) {
+    if (!SemaRef.IsCXXReplaceableType(Field->getType()))
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NonReplaceableField << Field
+          << Field->getType() << Field->getSourceRange();
   }
+  if (D->hasDeletedDestructor())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::DeletedDtr << /*Deleted*/ 0
+        << D->getDestructor()->getSourceRange();
 
   if (!D->hasSimpleMoveConstructor() && !D->hasSimpleCopyConstructor()) {
     const auto *Decl = cast<CXXConstructorDecl>(
         LookupSpecialMemberFromXValue(SemaRef, D, /*Assign=*/false));
-    if (Decl && Decl->isUserProvided())
+    if (Decl && Decl->isDeleted())
       SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
-          << diag::TraitNotSatisfiedReason::UserProvidedCtr
+          << diag::TraitNotSatisfiedReason::DeletedCtr
           << Decl->isMoveConstructor() << Decl->getSourceRange();
   }
   if (!D->hasSimpleMoveAssignment() && !D->hasSimpleCopyAssignment()) {
     CXXMethodDecl *Decl =
         LookupSpecialMemberFromXValue(SemaRef, D, /*Assign=*/true);
-    if (Decl && Decl->isUserProvided())
+    if (Decl && Decl->isDeleted())
       SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
-          << diag::TraitNotSatisfiedReason::UserProvidedAssign
+          << diag::TraitNotSatisfiedReason::DeletedAssign
           << Decl->isMoveAssignmentOperator() << Decl->getSourceRange();
   }
-  CXXDestructorDecl *Dtr = D->getDestructor();
-  if (Dtr && Dtr->isUserProvided() && !Dtr->isDefaulted())
-    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
-        << diag::TraitNotSatisfiedReason::DeletedDtr << /*User Provided*/ 1
-        << Dtr->getSourceRange();
+
+  if (D->hasAttr<ReplaceableAttr>())
+    return;
+  DiagnoseNonDefaultMovable(SemaRef, Loc, D);
 }
 
-static void DiagnoseNonTriviallyRelocatableReason(Sema &SemaRef,
-                                                  SourceLocation Loc,
-                                                  QualType T) {
+static void DiagnoseNonReplaceableReason(Sema &SemaRef, SourceLocation Loc,
+                                         QualType T) {
   SemaRef.Diag(Loc, diag::note_unsatisfied_trait)
-      << T << diag::TraitName::TriviallyRelocatable;
+      << T << diag::TraitName::Replaceable;
+
   if (T->isVariablyModifiedType())
     SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
         << diag::TraitNotSatisfiedReason::VLA;
@@ -2069,16 +2179,108 @@ static void DiagnoseNonTriviallyRelocatableReason(Sema &SemaRef,
         << diag::TraitNotSatisfiedReason::Ref;
   T = T.getNonReferenceType();
 
-  if (T.hasNonTrivialObjCLifetime())
+  if (T.isConstQualified())
     SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
-        << diag::TraitNotSatisfiedReason::HasArcLifetime;
+        << diag::TraitNotSatisfiedReason::Const;
+
+  if (T.isVolatileQualified())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::Volatile;
+
+  bool IsArray = T->isArrayType();
+  T = SemaRef.getASTContext().getBaseElementType(T.getUnqualifiedType());
+
+  if (T->isScalarType())
+    return;
+
+  const CXXRecordDecl *D = T->getAsCXXRecordDecl();
+  if (!D) {
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::NotScalarOrClass << IsArray;
+    return;
+  }
+
+  if (D->isInvalidDecl())
+    return;
+
+  if (D->hasDefinition())
+    DiagnoseNonReplaceableReason(SemaRef, Loc, D);
+
+  SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
+}
+
+static void DiagnoseNonTriviallyCopyableReason(Sema &SemaRef,
+                                               SourceLocation Loc,
+                                               const CXXRecordDecl *D) {
+  for (const CXXBaseSpecifier &B : D->bases()) {
+    assert(B.getType()->getAsCXXRecordDecl() && "invalid base?");
+    if (B.isVirtual())
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::VBase << B.getType()
+          << B.getSourceRange();
+    if (!B.getType().isTriviallyCopyableType(D->getASTContext())) {
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NTCBase << B.getType()
+          << B.getSourceRange();
+    }
+  }
+  for (const FieldDecl *Field : D->fields()) {
+    if (!Field->getType().isTriviallyCopyableType(Field->getASTContext()))
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NTCField << Field
+          << Field->getType() << Field->getSourceRange();
+  }
+  CXXDestructorDecl *Dtr = D->getDestructor();
+  if (D->hasDeletedDestructor() || (Dtr && !Dtr->isTrivial()))
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::DeletedDtr
+        << !D->hasDeletedDestructor() << D->getDestructor()->getSourceRange();
+
+  for (const CXXMethodDecl *Method : D->methods()) {
+    if (Method->isTrivial() || !Method->isUserProvided()) {
+      continue;
+    }
+    auto SpecialMemberKind =
+        SemaRef.getDefaultedFunctionKind(Method).asSpecialMember();
+    switch (SpecialMemberKind) {
+    case CXXSpecialMemberKind::CopyConstructor:
+    case CXXSpecialMemberKind::MoveConstructor:
+    case CXXSpecialMemberKind::CopyAssignment:
+    case CXXSpecialMemberKind::MoveAssignment: {
+      bool IsAssignment =
+          SpecialMemberKind == CXXSpecialMemberKind::CopyAssignment ||
+          SpecialMemberKind == CXXSpecialMemberKind::MoveAssignment;
+      bool IsMove =
+          SpecialMemberKind == CXXSpecialMemberKind::MoveConstructor ||
+          SpecialMemberKind == CXXSpecialMemberKind::MoveAssignment;
+
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << (IsAssignment ? diag::TraitNotSatisfiedReason::UserProvidedAssign
+                           : diag::TraitNotSatisfiedReason::UserProvidedCtr)
+          << IsMove << Method->getSourceRange();
+      break;
+    }
+    default:
+      break;
+    }
+  }
+}
+
+static void DiagnoseNonTriviallyCopyableReason(Sema &SemaRef,
+                                               SourceLocation Loc, QualType T) {
+  SemaRef.Diag(Loc, diag::note_unsatisfied_trait)
+      << T << diag::TraitName::TriviallyCopyable;
+
+  if (T->isReferenceType())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::Ref;
 
   const CXXRecordDecl *D = T->getAsCXXRecordDecl();
   if (!D || D->isInvalidDecl())
     return;
 
   if (D->hasDefinition())
-    DiagnoseNonTriviallyRelocatableReason(SemaRef, Loc, D);
+    DiagnoseNonTriviallyCopyableReason(SemaRef, Loc, D);
 
   SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
 }
@@ -2097,6 +2299,12 @@ void Sema::DiagnoseTypeTraitDetails(const Expr *E) {
   case UTT_IsCppTriviallyRelocatable:
     DiagnoseNonTriviallyRelocatableReason(*this, E->getBeginLoc(), Args[0]);
     break;
+  case UTT_IsReplaceable:
+    DiagnoseNonReplaceableReason(*this, E->getBeginLoc(), Args[0]);
+    break;
+  case UTT_IsTriviallyCopyable:
+    DiagnoseNonTriviallyCopyableReason(*this, E->getBeginLoc(), Args[0]);
+    break;
   default:
     break;
   }
diff --git a/external/llvm-project/clang/lib/Sema/TreeTransform.h b/external/llvm-project/clang/lib/Sema/TreeTransform.h
index bd839087c66e..3e33fb73e01b 100644
--- a/external/llvm-project/clang/lib/Sema/TreeTransform.h
+++ b/external/llvm-project/clang/lib/Sema/TreeTransform.h
@@ -715,7 +715,7 @@ class TreeTransform {
   /// variables vector are acceptable.
   ///
   /// LastParamTransformed, if non-null, will be set to the index of the last
-  /// parameter on which transfromation was started. In the event of an error,
+  /// parameter on which transformation was started. In the event of an error,
   /// this will contain the parameter which failed to instantiate.
   ///
   /// Return true on error.
@@ -13121,12 +13121,6 @@ TreeTransform<Derived>::TransformOpaqueValueExpr(OpaqueValueExpr *E) {
   return E;
 }
 
-template<typename Derived>
-ExprResult
-TreeTransform<Derived>::TransformTypoExpr(TypoExpr *E) {
-  return E;
-}
-
 template <typename Derived>
 ExprResult TreeTransform<Derived>::TransformRecoveryExpr(RecoveryExpr *E) {
   llvm::SmallVector<Expr *, 8> Children;
diff --git a/external/llvm-project/clang/lib/Serialization/ASTReader.cpp b/external/llvm-project/clang/lib/Serialization/ASTReader.cpp
index 352ccc07fb46..acda5a7c879d 100644
--- a/external/llvm-project/clang/lib/Serialization/ASTReader.cpp
+++ b/external/llvm-project/clang/lib/Serialization/ASTReader.cpp
@@ -1532,7 +1532,7 @@ bool ASTReader::ReadSpecializations(ModuleFile &M, BitstreamCursor &Cursor,
 }
 
 void ASTReader::Error(StringRef Msg) const {
-  Error(diag::err_fe_pch_malformed, Msg);
+  Error(diag::err_fe_ast_file_malformed, Msg);
   if (PP.getLangOpts().Modules &&
       !PP.getHeaderSearchInfo().getModuleCachePath().empty()) {
     Diag(diag::note_module_cache_path)
@@ -2854,25 +2854,25 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
       while (!ImportStack.back()->ImportedBy.empty())
         ImportStack.push_back(ImportStack.back()->ImportedBy[0]);
 
-      // The top-level PCH is stale.
-      StringRef TopLevelPCHName(ImportStack.back()->FileName);
+      // The top-level AST file is stale.
+      StringRef TopLevelASTFileName(ImportStack.back()->FileName);
       Diag(diag::err_fe_ast_file_modified)
           << *Filename << moduleKindForDiagnostic(ImportStack.back()->Kind)
-          << TopLevelPCHName << FileChange.Kind
+          << TopLevelASTFileName << FileChange.Kind
           << (FileChange.Old && FileChange.New)
           << llvm::itostr(FileChange.Old.value_or(0))
           << llvm::itostr(FileChange.New.value_or(0));
 
       // Print the import stack.
       if (ImportStack.size() > 1) {
-        Diag(diag::note_pch_required_by)
+        Diag(diag::note_ast_file_required_by)
             << *Filename << ImportStack[0]->FileName;
         for (unsigned I = 1; I < ImportStack.size(); ++I)
-          Diag(diag::note_pch_required_by)
-            << ImportStack[I-1]->FileName << ImportStack[I]->FileName;
+          Diag(diag::note_ast_file_required_by)
+              << ImportStack[I - 1]->FileName << ImportStack[I]->FileName;
       }
 
-      Diag(diag::note_pch_rebuild_required) << TopLevelPCHName;
+      Diag(diag::note_ast_file_rebuild_required) << TopLevelASTFileName;
     }
 
     IsOutOfDate = true;
@@ -4953,20 +4953,21 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName, ModuleKind Type,
 
 static ASTFileSignature readASTFileSignature(StringRef PCH);
 
-/// Whether \p Stream doesn't start with the AST/PCH file magic number 'CPCH'.
+/// Whether \p Stream doesn't start with the AST file magic number 'CPCH'.
 static llvm::Error doesntStartWithASTFileMagic(BitstreamCursor &Stream) {
   // FIXME checking magic headers is done in other places such as
   // SerializedDiagnosticReader and GlobalModuleIndex, but error handling isn't
   // always done the same. Unify it all with a helper.
   if (!Stream.canSkipToPos(4))
-    return llvm::createStringError(std::errc::illegal_byte_sequence,
-                                   "file too small to contain AST file magic");
+    return llvm::createStringError(
+        std::errc::illegal_byte_sequence,
+        "file too small to contain precompiled file magic");
   for (unsigned C : {'C', 'P', 'C', 'H'})
     if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Stream.Read(8)) {
       if (Res.get() != C)
         return llvm::createStringError(
             std::errc::illegal_byte_sequence,
-            "file doesn't start with AST file magic");
+            "file doesn't start with precompiled file magic");
     } else
       return Res.takeError();
   return llvm::Error::success();
diff --git a/external/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp b/external/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp
index fb31613ded65..a1368a48351c 100644
--- a/external/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/external/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -3198,8 +3198,8 @@ Attr *ASTRecordReader::readAttr() {
                     SpellingIndex == AlignedAttr::Keyword_alignas);
   bool IsRegularKeywordAttribute = Record.readBool();
 
-  AttributeCommonInfo Info(AttrName, ScopeName, AttrRange, ScopeLoc,
-                           AttributeCommonInfo::Kind(ParsedKind),
+  AttributeCommonInfo Info(AttrName, AttributeScopeInfo(ScopeName, ScopeLoc),
+                           AttrRange, AttributeCommonInfo::Kind(ParsedKind),
                            {AttributeCommonInfo::Syntax(Syntax), SpellingIndex,
                             IsAlignas, IsRegularKeywordAttribute});
 
@@ -4631,11 +4631,10 @@ namespace {
 
       // Perform a binary search to find the local redeclarations for this
       // declaration (if any).
-      const ObjCCategoriesInfo Compare = { LocalID, 0 };
-      const ObjCCategoriesInfo *Result
-        = std::lower_bound(M.ObjCCategoriesMap,
-                           M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap,
-                           Compare);
+      const ObjCCategoriesInfo Compare = {LocalID, 0};
+      const ObjCCategoriesInfo *Result = std::lower_bound(
+          M.ObjCCategoriesMap,
+          M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap, Compare);
       if (Result == M.ObjCCategoriesMap + M.LocalNumObjCCategoriesInMap ||
           LocalID != Result->getDefinitionID()) {
         // We didn't find anything. If the class definition is in this module
diff --git a/external/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp b/external/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp
index f1bb33a19107..65102b64030c 100644
--- a/external/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/external/llvm-project/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2156,7 +2156,7 @@ void ASTStmtReader::VisitTypeTraitExpr(TypeTraitExpr *E) {
 
 void ASTStmtReader::VisitArrayTypeTraitExpr(ArrayTypeTraitExpr *E) {
   VisitExpr(E);
-  E->ATT = (ArrayTypeTrait)Record.readInt();
+  E->ArrayTypeTraitExprBits.ATT = (ArrayTypeTrait)Record.readInt();
   E->Value = (unsigned int)Record.readInt();
   SourceRange Range = readSourceRange();
   E->Loc = Range.getBegin();
@@ -2167,8 +2167,8 @@ void ASTStmtReader::VisitArrayTypeTraitExpr(ArrayTypeTraitExpr *E) {
 
 void ASTStmtReader::VisitExpressionTraitExpr(ExpressionTraitExpr *E) {
   VisitExpr(E);
-  E->ET = (ExpressionTrait)Record.readInt();
-  E->Value = (bool)Record.readInt();
+  E->ExpressionTraitExprBits.ET = (ExpressionTrait)Record.readInt();
+  E->ExpressionTraitExprBits.Value = (bool)Record.readInt();
   SourceRange Range = readSourceRange();
   E->QueriedExpression = Record.readSubExpr();
   E->Loc = Range.getBegin();
@@ -2209,14 +2209,14 @@ void ASTStmtReader::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
 
 void ASTStmtReader::VisitPackIndexingExpr(PackIndexingExpr *E) {
   VisitExpr(E);
-  E->TransformedExpressions = Record.readInt();
-  E->FullySubstituted = Record.readInt();
+  E->PackIndexingExprBits.TransformedExpressions = Record.readInt();
+  E->PackIndexingExprBits.FullySubstituted = Record.readInt();
   E->EllipsisLoc = readSourceLocation();
   E->RSquareLoc = readSourceLocation();
   E->SubExprs[0] = Record.readStmt();
   E->SubExprs[1] = Record.readStmt();
   auto **Exprs = E->getTrailingObjects<Expr *>();
-  for (unsigned I = 0; I < E->TransformedExpressions; ++I)
+  for (unsigned I = 0; I < E->PackIndexingExprBits.TransformedExpressions; ++I)
     Exprs[I] = Record.readExpr();
 }
 
@@ -2275,7 +2275,7 @@ void ASTStmtReader::VisitCXXFoldExpr(CXXFoldExpr *E) {
   E->SubExprs[0] = Record.readSubExpr();
   E->SubExprs[1] = Record.readSubExpr();
   E->SubExprs[2] = Record.readSubExpr();
-  E->Opcode = (BinaryOperatorKind)Record.readInt();
+  E->CXXFoldExprBits.Opcode = (BinaryOperatorKind)Record.readInt();
 }
 
 void ASTStmtReader::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
@@ -2310,10 +2310,6 @@ void ASTStmtReader::VisitOpaqueValueExpr(OpaqueValueExpr *E) {
   E->setIsUnique(Record.readInt());
 }
 
-void ASTStmtReader::VisitTypoExpr(TypoExpr *E) {
-  llvm_unreachable("Cannot read TypoExpr nodes");
-}
-
 void ASTStmtReader::VisitRecoveryExpr(RecoveryExpr *E) {
   VisitExpr(E);
   unsigned NumArgs = Record.readInt();
diff --git a/external/llvm-project/clang/lib/Serialization/ASTWriter.cpp b/external/llvm-project/clang/lib/Serialization/ASTWriter.cpp
index 46bfdb2a92d1..ab1b5b333e06 100644
--- a/external/llvm-project/clang/lib/Serialization/ASTWriter.cpp
+++ b/external/llvm-project/clang/lib/Serialization/ASTWriter.cpp
@@ -4972,6 +4972,9 @@ void ASTWriter::WriteCUDAPragmas(Sema &SemaRef) {
 }
 
 void ASTWriter::WriteObjCCategories() {
+  if (ObjCClassesWithCategories.empty())
+    return;
+
   SmallVector<ObjCCategoriesInfo, 2> CategoriesMap;
   RecordData Categories;
 
diff --git a/external/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp b/external/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp
index f3d260e8671b..052cb5a253bf 100644
--- a/external/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/external/llvm-project/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -221,6 +221,48 @@ namespace clang {
         Record.AddDeclRef(F.second);
     }
 
+    template <typename T> bool shouldSkipWritingSpecializations(T *Spec) {
+      // Now we will only avoid writing specializations if we're generating
+      // reduced BMI.
+      if (!GeneratingReducedBMI)
+        return false;
+
+      assert((isa<FunctionDecl, ClassTemplateSpecializationDecl,
+                  VarTemplateSpecializationDecl>(Spec)));
+
+      ArrayRef<TemplateArgument> Args;
+      if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(Spec))
+        Args = CTSD->getTemplateArgs().asArray();
+      else if (auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(Spec))
+        Args = VTSD->getTemplateArgs().asArray();
+      else
+        Args = cast<FunctionDecl>(Spec)
+                   ->getTemplateSpecializationArgs()
+                   ->asArray();
+
+      // If there is any template argument is TULocal, we can avoid writing the
+      // specialization since the consumers of reduced BMI won't get the
+      // specialization anyway.
+      for (const TemplateArgument &TA : Args) {
+        switch (TA.getKind()) {
+        case TemplateArgument::Type: {
+          Linkage L = TA.getAsType()->getLinkage();
+          if (!isExternallyVisible(L))
+            return true;
+          break;
+        }
+        case TemplateArgument::Declaration:
+          if (!TA.getAsDecl()->isExternallyVisible())
+            return true;
+          break;
+        default:
+          break;
+        }
+      }
+
+      return false;
+    }
+
     /// Add to the record the first template specialization from each module
     /// file that provides a declaration of D. We store the DeclId and an
     /// ODRHash of the template arguments of D which should provide enough
@@ -235,6 +277,9 @@ namespace clang {
       CollectFirstDeclFromEachModule(D, /*IncludeLocal*/ true, Firsts);
 
       for (const auto &F : Firsts) {
+        if (shouldSkipWritingSpecializations(F.second))
+          continue;
+
         if (isa<ClassTemplatePartialSpecializationDecl,
                 VarTemplatePartialSpecializationDecl>(F.second))
           PartialSpecsInMap.push_back(F.second);
@@ -1389,7 +1434,7 @@ void ASTDeclWriter::VisitBindingDecl(BindingDecl *D) {
 
 void ASTDeclWriter::VisitFileScopeAsmDecl(FileScopeAsmDecl *D) {
   VisitDecl(D);
-  Record.AddStmt(D->getAsmString());
+  Record.AddStmt(D->getAsmStringExpr());
   Record.AddSourceLocation(D->getRParenLoc());
   Code = serialization::DECL_FILE_SCOPE_ASM;
 }
@@ -1655,7 +1700,7 @@ void ASTDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
     if (auto *FD = llvm::dyn_cast_or_null<FunctionDecl>(D->getDeclContext());
         FD && isDefinitionInDependentContext(FD)) {
       Writer.RelatedDeclsMap[Writer.GetDeclRef(FD)].push_back(
-          Writer.GetDeclRef(D));
+          Writer.GetDeclRef(D->getLambdaCallOperator()));
     }
   } else {
     Record.push_back(CXXRecNotTemplate);
diff --git a/external/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp b/external/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp
index fc24b76b79de..a6e320c7f3eb 100644
--- a/external/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/external/llvm-project/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2212,8 +2212,8 @@ void ASTStmtWriter::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
 
 void ASTStmtWriter::VisitPackIndexingExpr(PackIndexingExpr *E) {
   VisitExpr(E);
-  Record.push_back(E->TransformedExpressions);
-  Record.push_back(E->FullySubstituted);
+  Record.push_back(E->PackIndexingExprBits.TransformedExpressions);
+  Record.push_back(E->PackIndexingExprBits.FullySubstituted);
   Record.AddSourceLocation(E->getEllipsisLoc());
   Record.AddSourceLocation(E->getRSquareLoc());
   Record.AddStmt(E->getPackIdExpression());
@@ -2278,7 +2278,7 @@ void ASTStmtWriter::VisitCXXFoldExpr(CXXFoldExpr *E) {
   Record.AddStmt(E->SubExprs[0]);
   Record.AddStmt(E->SubExprs[1]);
   Record.AddStmt(E->SubExprs[2]);
-  Record.push_back(E->Opcode);
+  Record.push_back(E->CXXFoldExprBits.Opcode);
   Code = serialization::EXPR_CXX_FOLD;
 }
 
@@ -2314,12 +2314,6 @@ void ASTStmtWriter::VisitOpaqueValueExpr(OpaqueValueExpr *E) {
   Code = serialization::EXPR_OPAQUE_VALUE;
 }
 
-void ASTStmtWriter::VisitTypoExpr(TypoExpr *E) {
-  VisitExpr(E);
-  // TODO: Figure out sane writer behavior for a TypoExpr, if necessary
-  llvm_unreachable("Cannot write TypoExpr nodes");
-}
-
 //===----------------------------------------------------------------------===//
 // CUDA Expressions and Statements.
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 410260478175..86476b32309c 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -671,7 +671,6 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
                                               ProgramStateRef state,
                                               const ObjCMethodCall &Msg) const {
   ASTContext &Ctx = C.getASTContext();
-  static CheckerProgramPointTag Tag(this, "NilReceiver");
 
   // Check the return type of the message expression.  A message to nil will
   // return different values depending on the return type and the architecture.
@@ -682,7 +681,7 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
   if (CanRetTy->isStructureOrClassType()) {
     // Structure returns are safe since the compiler zeroes them out.
     SVal V = C.getSValBuilder().makeZeroVal(RetTy);
-    C.addTransition(state->BindExpr(Msg.getOriginExpr(), LCtx, V), &Tag);
+    C.addTransition(state->BindExpr(Msg.getOriginExpr(), LCtx, V));
     return;
   }
 
@@ -701,7 +700,7 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
             Ctx.LongDoubleTy == CanRetTy ||
             Ctx.LongLongTy == CanRetTy ||
             Ctx.UnsignedLongLongTy == CanRetTy)))) {
-      if (ExplodedNode *N = C.generateErrorNode(state, &Tag))
+      if (ExplodedNode *N = C.generateErrorNode(state))
         emitNilReceiverBug(C, Msg, N);
       return;
     }
@@ -720,7 +719,7 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
     // of this case unless we have *a lot* more knowledge.
     //
     SVal V = C.getSValBuilder().makeZeroVal(RetTy);
-    C.addTransition(state->BindExpr(Msg.getOriginExpr(), LCtx, V), &Tag);
+    C.addTransition(state->BindExpr(Msg.getOriginExpr(), LCtx, V));
     return;
   }
 
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
index e58329817d7c..344be0b176c5 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
@@ -674,9 +674,16 @@ void DynamicTypePropagation::checkPostStmt(const CastExpr *CE,
   if (TrackedType &&
       !ASTCtxt.canAssignObjCInterfaces(DestObjectPtrType, *TrackedType) &&
       !ASTCtxt.canAssignObjCInterfaces(*TrackedType, DestObjectPtrType)) {
-    static CheckerProgramPointTag IllegalConv(this, "IllegalConversion");
-    ExplodedNode *N = C.addTransition(State, AfterTypeProp, &IllegalConv);
-    reportGenericsBug(*TrackedType, DestObjectPtrType, N, Sym, C);
+    // This distinct program point tag is needed because `State` can be
+    // identical to the state of the node `AfterTypeProp`, and in that case
+    // `generateNonFatalErrorNode` would "cache out" and return nullptr
+    // (instead of re-creating an already existing node).
+    static SimpleProgramPointTag IllegalConv("DynamicTypePropagation",
+                                             "IllegalConversion");
+    ExplodedNode *N =
+        C.generateNonFatalErrorNode(State, AfterTypeProp, &IllegalConv);
+    if (N)
+      reportGenericsBug(*TrackedType, DestObjectPtrType, N, Sym, C);
     return;
   }
 
@@ -885,8 +892,7 @@ void DynamicTypePropagation::checkPreObjCMessage(const ObjCMethodCall &M,
     // Warn when argument is incompatible with the parameter.
     if (!ASTCtxt.canAssignObjCInterfaces(ParamObjectPtrType,
                                          ArgObjectPtrType)) {
-      static CheckerProgramPointTag Tag(this, "ArgTypeMismatch");
-      ExplodedNode *N = C.addTransition(State, &Tag);
+      ExplodedNode *N = C.generateNonFatalErrorNode(State);
       reportGenericsBug(ArgObjectPtrType, ParamObjectPtrType, N, Sym, C, Arg);
       return;
     }
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
index 6b536822024b..bea8f3f13ba2 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
@@ -1043,8 +1043,7 @@ bool GenericTaintChecker::generateReportIfTainted(const Expr *E, StringRef Msg,
 
   // Generate diagnostic.
   assert(BT);
-  static CheckerProgramPointTag Tag(BT->getCheckerName(), Msg);
-  if (ExplodedNode *N = C.generateNonFatalErrorNode(C.getState(), &Tag)) {
+  if (ExplodedNode *N = C.generateNonFatalErrorNode(C.getState())) {
     auto report = std::make_unique<PathSensitiveBugReport>(*BT, Msg, N);
     report->addRange(E->getSourceRange());
     for (auto TaintedSym : getTaintedSymbols(C.getState(), *TaintedSVal)) {
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
index e9abc9ab2a1f..2539d3b9b71e 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
@@ -747,9 +747,7 @@ void NonLocalizedStringChecker::reportLocalizationError(
   if (isDebuggingContext(C))
     return;
 
-  static CheckerProgramPointTag Tag("NonLocalizedStringChecker",
-                                    "UnlocalizedString");
-  ExplodedNode *ErrNode = C.addTransition(C.getState(), C.getPredecessor(), &Tag);
+  ExplodedNode *ErrNode = C.generateNonFatalErrorNode();
 
   if (!ErrNode)
     return;
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp
index 4c0a8ba2c7c0..2932cc133657 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp
@@ -73,7 +73,6 @@ void MPIChecker::checkUnmatchedWaits(const CallEvent &PreCallEvent,
     return;
 
   ProgramStateRef State = Ctx.getState();
-  static CheckerProgramPointTag Tag("MPI-Checker", "UnmatchedWait");
   ExplodedNode *ErrorNode{nullptr};
 
   // Check all request regions used by the wait function.
@@ -82,7 +81,7 @@ void MPIChecker::checkUnmatchedWaits(const CallEvent &PreCallEvent,
     State = State->set<RequestMap>(ReqRegion, Request::State::Wait);
     if (!Req) {
       if (!ErrorNode) {
-        ErrorNode = Ctx.generateNonFatalErrorNode(State, &Tag);
+        ErrorNode = Ctx.generateNonFatalErrorNode(State);
         State = ErrorNode->getState();
       }
       // A wait has no matching nonblocking call.
@@ -105,7 +104,6 @@ void MPIChecker::checkMissingWaits(SymbolReaper &SymReaper,
   if (Requests.isEmpty())
     return;
 
-  static CheckerProgramPointTag Tag("MPI-Checker", "MissingWait");
   ExplodedNode *ErrorNode{nullptr};
 
   auto ReqMap = State->get<RequestMap>();
@@ -114,7 +112,7 @@ void MPIChecker::checkMissingWaits(SymbolReaper &SymReaper,
       if (Req.second.CurrentState == Request::State::Nonblocking) {
 
         if (!ErrorNode) {
-          ErrorNode = Ctx.generateNonFatalErrorNode(State, &Tag);
+          ErrorNode = Ctx.generateNonFatalErrorNode(State);
           State = ErrorNode->getState();
         }
         BReporter.reportMissingWait(Req.second, Req.first, ErrorNode,
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
index e65ab78657b9..4f829a1dda09 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
@@ -556,8 +556,7 @@ void MacOSKeychainAPIChecker::checkDeadSymbols(SymbolReaper &SR,
     return;
   }
 
-  static CheckerProgramPointTag Tag(this, "DeadSymbolsLeak");
-  ExplodedNode *N = C.generateNonFatalErrorNode(C.getState(), &Tag);
+  ExplodedNode *N = C.generateNonFatalErrorNode(C.getState());
   if (!N)
     return;
 
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 74ad4b3eb609..fef33509c0b6 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -3098,8 +3098,7 @@ void MallocChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   // Generate leak node.
   ExplodedNode *N = C.getPredecessor();
   if (!Errors.empty()) {
-    static CheckerProgramPointTag Tag("MallocChecker", "DeadSymbolsLeak");
-    N = C.generateNonFatalErrorNode(C.getState(), &Tag);
+    N = C.generateNonFatalErrorNode(C.getState());
     if (N) {
       for (SymbolRef Sym : Errors) {
         HandleLeak(Sym, N, C);
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
index 04472bb3895a..461d01b452fd 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
@@ -69,6 +69,7 @@ const char *getNullabilityString(Nullability Nullab) {
 }
 
 // These enums are used as an index to ErrorMessages array.
+// FIXME: ErrorMessages no longer exists, perhaps remove this as well?
 enum class ErrorKind : int {
   NilAssignedToNonnull,
   NilPassedToNonnull,
@@ -714,8 +715,7 @@ void NullabilityChecker::checkPreStmt(const ReturnStmt *S,
   if (ChecksEnabled[CK_NullReturnedFromNonnull] && NullReturnedFromNonNull &&
       RetExprTypeLevelNullability != Nullability::Nonnull &&
       !InSuppressedMethodFamily) {
-    static CheckerProgramPointTag Tag(this, "NullReturnedFromNonnull");
-    ExplodedNode *N = C.generateErrorNode(State, &Tag);
+    ExplodedNode *N = C.generateErrorNode(State);
     if (!N)
       return;
 
@@ -750,8 +750,7 @@ void NullabilityChecker::checkPreStmt(const ReturnStmt *S,
         Nullness != NullConstraint::IsNotNull &&
         TrackedNullabValue == Nullability::Nullable &&
         RequiredNullability == Nullability::Nonnull) {
-      static CheckerProgramPointTag Tag(this, "NullableReturnedFromNonnull");
-      ExplodedNode *N = C.addTransition(State, C.getPredecessor(), &Tag);
+      ExplodedNode *N = C.addTransition(State, C.getPredecessor());
 
       SmallString<256> SBuf;
       llvm::raw_svector_ostream OS(SBuf);
@@ -1299,8 +1298,7 @@ void NullabilityChecker::checkBind(SVal L, SVal V, const Stmt *S,
       ValNullability != Nullability::Nonnull &&
       ValueExprTypeLevelNullability != Nullability::Nonnull &&
       !isARCNilInitializedLocal(C, S)) {
-    static CheckerProgramPointTag Tag(this, "NullPassedToNonnull");
-    ExplodedNode *N = C.generateErrorNode(State, &Tag);
+    ExplodedNode *N = C.generateErrorNode(State);
     if (!N)
       return;
 
@@ -1342,8 +1340,7 @@ void NullabilityChecker::checkBind(SVal L, SVal V, const Stmt *S,
       return;
     if (ChecksEnabled[CK_NullablePassedToNonnull] &&
         LocNullability == Nullability::Nonnull) {
-      static CheckerProgramPointTag Tag(this, "NullablePassedToNonnull");
-      ExplodedNode *N = C.addTransition(State, C.getPredecessor(), &Tag);
+      ExplodedNode *N = C.addTransition(State, C.getPredecessor());
       reportBugIfInvariantHolds("Nullable pointer is assigned to a pointer "
                                 "which is expected to have non-null value",
                                 ErrorKind::NullableAssignedToNonnull,
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
index 85e22daaedee..62bc3218d9ce 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "RetainCountChecker.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include <optional>
 
@@ -1031,8 +1030,7 @@ ExplodedNode * RetainCountChecker::processReturn(const ReturnStmt *S,
     return nullptr;
 
   // Update the autorelease counts.
-  static CheckerProgramPointTag AutoreleaseTag(this, "Autorelease");
-  state = handleAutoreleaseCounts(state, Pred, &AutoreleaseTag, C, Sym, X, S);
+  state = handleAutoreleaseCounts(state, Pred, C, Sym, X, S);
 
   // Have we generated a sink node?
   if (!state)
@@ -1089,8 +1087,7 @@ ExplodedNode * RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
         // Generate an error node.
         state = setRefBinding(state, Sym, X);
 
-        static CheckerProgramPointTag ReturnOwnLeakTag(this, "ReturnsOwnLeak");
-        ExplodedNode *N = C.addTransition(state, Pred, &ReturnOwnLeakTag);
+        ExplodedNode *N = C.addTransition(state, Pred);
         if (N) {
           const LangOptions &LOpts = C.getASTContext().getLangOpts();
           auto R =
@@ -1113,10 +1110,7 @@ ExplodedNode * RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
         // owned object.
         state = setRefBinding(state, Sym, X ^ RefVal::ErrorReturnedNotOwned);
 
-        static CheckerProgramPointTag
-            ReturnNotOwnedTag(this, "ReturnNotOwnedForOwned");
-
-        ExplodedNode *N = C.addTransition(state, Pred, &ReturnNotOwnedTag);
+        ExplodedNode *N = C.addTransition(state, Pred);
         if (N) {
           auto R = std::make_unique<RefCountReport>(
               *ReturnNotOwnedForOwned, C.getASTContext().getLangOpts(), N, Sym);
@@ -1202,14 +1196,9 @@ ProgramStateRef RetainCountChecker::checkRegionChanges(
   return state;
 }
 
-ProgramStateRef
-RetainCountChecker::handleAutoreleaseCounts(ProgramStateRef state,
-                                            ExplodedNode *Pred,
-                                            const ProgramPointTag *Tag,
-                                            CheckerContext &Ctx,
-                                            SymbolRef Sym,
-                                            RefVal V,
-                                            const ReturnStmt *S) const {
+ProgramStateRef RetainCountChecker::handleAutoreleaseCounts(
+    ProgramStateRef state, ExplodedNode *Pred, CheckerContext &Ctx,
+    SymbolRef Sym, RefVal V, const ReturnStmt *S) const {
   unsigned ACnt = V.getAutoreleaseCount();
 
   // No autorelease counts?  Nothing to be done.
@@ -1260,7 +1249,7 @@ RetainCountChecker::handleAutoreleaseCounts(ProgramStateRef state,
   V = V ^ RefVal::ErrorOverAutorelease;
   state = setRefBinding(state, Sym, V);
 
-  ExplodedNode *N = Ctx.generateSink(state, Pred, Tag);
+  ExplodedNode *N = Ctx.generateSink(state, Pred);
   if (N) {
     SmallString<128> sbuf;
     llvm::raw_svector_ostream os(sbuf);
@@ -1383,8 +1372,7 @@ void RetainCountChecker::checkEndFunction(const ReturnStmt *RS,
   }
 
   for (auto &I : B) {
-    state = handleAutoreleaseCounts(state, Pred, /*Tag=*/nullptr, Ctx,
-                                    I.first, I.second);
+    state = handleAutoreleaseCounts(state, Pred, Ctx, I.first, I.second);
     if (!state)
       return;
   }
@@ -1416,9 +1404,8 @@ void RetainCountChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   for (const auto &I: state->get<RefBindings>()) {
     SymbolRef Sym = I.first;
     if (SymReaper.isDead(Sym)) {
-      static CheckerProgramPointTag Tag(this, "DeadSymbolAutorelease");
       const RefVal &V = I.second;
-      state = handleAutoreleaseCounts(state, Pred, &Tag, C, Sym, V);
+      state = handleAutoreleaseCounts(state, Pred, C, Sym, V);
       if (!state)
         return;
 
@@ -1472,15 +1459,15 @@ void RetainCountChecker::printState(raw_ostream &Out, ProgramStateRef State,
 // Checker registration.
 //===----------------------------------------------------------------------===//
 
-std::unique_ptr<CheckerProgramPointTag> RetainCountChecker::DeallocSentTag;
-std::unique_ptr<CheckerProgramPointTag> RetainCountChecker::CastFailTag;
+std::unique_ptr<SimpleProgramPointTag> RetainCountChecker::DeallocSentTag;
+std::unique_ptr<SimpleProgramPointTag> RetainCountChecker::CastFailTag;
 
 void ento::registerRetainCountBase(CheckerManager &Mgr) {
   auto *Chk = Mgr.registerChecker<RetainCountChecker>();
-  Chk->DeallocSentTag =
-      std::make_unique<CheckerProgramPointTag>(Chk, "DeallocSent");
-  Chk->CastFailTag =
-      std::make_unique<CheckerProgramPointTag>(Chk, "DynamicCastFail");
+  Chk->DeallocSentTag = std::make_unique<SimpleProgramPointTag>(
+      "RetainCountChecker", "DeallocSent");
+  Chk->CastFailTag = std::make_unique<SimpleProgramPointTag>(
+      "RetainCountChecker", "DynamicCastFail");
 }
 
 bool ento::shouldRegisterRetainCountBase(const CheckerManager &mgr) {
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h
index d4d7c4c74c56..0e811436605f 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h
@@ -262,8 +262,8 @@ class RetainCountChecker
 
   mutable std::unique_ptr<RetainSummaryManager> Summaries;
 
-  static std::unique_ptr<CheckerProgramPointTag> DeallocSentTag;
-  static std::unique_ptr<CheckerProgramPointTag> CastFailTag;
+  static std::unique_ptr<SimpleProgramPointTag> DeallocSentTag;
+  static std::unique_ptr<SimpleProgramPointTag> CastFailTag;
 
   /// Track Objective-C and CoreFoundation objects.
   bool TrackObjCAndCFObjects = false;
@@ -347,23 +347,22 @@ class RetainCountChecker
                                     SymbolRef sid, RefVal V,
                                     SmallVectorImpl<SymbolRef> &Leaked) const;
 
-  ProgramStateRef
-  handleAutoreleaseCounts(ProgramStateRef state, ExplodedNode *Pred,
-                          const ProgramPointTag *Tag, CheckerContext &Ctx,
-                          SymbolRef Sym,
-                          RefVal V,
-                          const ReturnStmt *S=nullptr) const;
+  ProgramStateRef handleAutoreleaseCounts(ProgramStateRef state,
+                                          ExplodedNode *Pred,
+                                          CheckerContext &Ctx, SymbolRef Sym,
+                                          RefVal V,
+                                          const ReturnStmt *S = nullptr) const;
 
   ExplodedNode *processLeaks(ProgramStateRef state,
                              SmallVectorImpl<SymbolRef> &Leaked,
                              CheckerContext &Ctx,
                              ExplodedNode *Pred = nullptr) const;
 
-  static const CheckerProgramPointTag &getDeallocSentTag() {
+  static const SimpleProgramPointTag &getDeallocSentTag() {
     return *DeallocSentTag;
   }
 
-  static const CheckerProgramPointTag &getCastFailTag() { return *CastFailTag; }
+  static const SimpleProgramPointTag &getCastFailTag() { return *CastFailTag; }
 
 private:
   /// Perform the necessary checks and state adjustments at the end of the
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
index 7ce9a5b5bb6d..ff7615212d34 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
@@ -256,4 +256,4 @@ void ento::registerUnreachableCodeChecker(CheckerManager &mgr) {
 
 bool ento::shouldRegisterUnreachableCodeChecker(const CheckerManager &mgr) {
   return true;
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
index f087fc8fa19f..cfcc47c5e71c 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
@@ -119,6 +119,11 @@ bool tryToFindPtrOrigin(
         }
       }
 
+      if (call->isCallToStdMove() && call->getNumArgs() == 1) {
+        E = call->getArg(0)->IgnoreParenCasts();
+        continue;
+      }
+
       if (auto *callee = call->getDirectCallee()) {
         if (isCtorOfSafePtr(callee)) {
           if (StopAtFirstRefCountedObj)
@@ -231,6 +236,26 @@ bool isConstOwnerPtrMemberExpr(const clang::Expr *E) {
   return isOwnerPtrType(T) && T.isConstQualified();
 }
 
+bool isExprToGetCheckedPtrCapableMember(const clang::Expr *E) {
+  auto *ME = dyn_cast<MemberExpr>(E);
+  if (!ME)
+    return false;
+  auto *Base = ME->getBase();
+  if (!Base)
+    return false;
+  if (!isa<CXXThisExpr>(Base->IgnoreParenCasts()))
+    return false;
+  auto *D = ME->getMemberDecl();
+  if (!D)
+    return false;
+  auto T = D->getType();
+  auto *CXXRD = T->getAsCXXRecordDecl();
+  if (!CXXRD)
+    return false;
+  auto result = isCheckedPtrCapable(CXXRD);
+  return result && *result;
+}
+
 class EnsureFunctionVisitor
     : public ConstStmtVisitor<EnsureFunctionVisitor, bool> {
 public:
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h
index e2cc7b976adf..8302bbe3084c 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h
@@ -69,6 +69,10 @@ bool isASafeCallArg(const clang::Expr *E);
 /// \returns true if E is a MemberExpr accessing a const smart pointer type.
 bool isConstOwnerPtrMemberExpr(const clang::Expr *E);
 
+/// \returns true if E is a MemberExpr accessing a member variable which
+/// supports CheckedPtr.
+bool isExprToGetCheckedPtrCapableMember(const clang::Expr *E);
+
 /// \returns true if E is a CXXMemberCallExpr which returns a const smart
 /// pointer type.
 class EnsureFunctionAnalysis {
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 4ddd11495f53..72199af2f80a 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -236,6 +236,7 @@ std::optional<bool> isUnchecked(const QualType T) {
 void RetainTypeChecker::visitTranslationUnitDecl(
     const TranslationUnitDecl *TUD) {
   IsARCEnabled = TUD->getLangOpts().ObjCAutoRefCount;
+  DefaultSynthProperties = TUD->getLangOpts().ObjCDefaultSynthProperties;
 }
 
 void RetainTypeChecker::visitTypedef(const TypedefDecl *TD) {
@@ -468,6 +469,18 @@ bool isPtrConversion(const FunctionDecl *F) {
       FunctionName == "checked_objc_cast")
     return true;
 
+  auto ReturnType = F->getReturnType();
+  if (auto *Type = ReturnType.getTypePtrOrNull()) {
+    if (auto *AttrType = dyn_cast<AttributedType>(Type)) {
+      if (auto *Attr = AttrType->getAttr()) {
+        if (auto *AnnotateType = dyn_cast<AnnotateTypeAttr>(Attr)) {
+          if (AnnotateType->getAnnotation() == "webkit.pointerconversion")
+            return true;
+        }
+      }
+    }
+  }
+
   return false;
 }
 
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
index f9fcfe9878d5..3c9560cb8059 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
@@ -76,12 +76,14 @@ class RetainTypeChecker {
   llvm::DenseSet<const RecordType *> CFPointees;
   llvm::DenseSet<const Type *> RecordlessTypes;
   bool IsARCEnabled{false};
+  bool DefaultSynthProperties{true};
 
 public:
   void visitTranslationUnitDecl(const TranslationUnitDecl *);
   void visitTypedef(const TypedefDecl *);
   bool isUnretained(const QualType, bool ignoreARC = false);
   bool isARCEnabled() const { return IsARCEnabled; }
+  bool defaultSynthProperties() const { return DefaultSynthProperties; }
 };
 
 /// \returns true if \p Class is NS or CF objects AND not retained, false if
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
index 78675e55cf0b..6bc39ab56504 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
@@ -443,6 +443,10 @@ class UncheckedCallArgsChecker final : public RawPtrRefCallArgsChecker {
     return isRefOrCheckedPtrType(type);
   }
 
+  bool isSafeExpr(const Expr *E) const final {
+    return isExprToGetCheckedPtrCapableMember(E);
+  }
+
   const char *ptrKind() const final { return "unchecked"; }
 };
 
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp
index 4fb47703e398..7cd86a682c0a 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLocalVarsChecker.cpp
@@ -417,6 +417,9 @@ class UncheckedLocalVarsChecker final : public RawPtrRefLocalVarsChecker {
   bool isSafePtrType(const QualType type) const final {
     return isRefOrCheckedPtrType(type);
   }
+  bool isSafeExpr(const Expr *E) const final {
+    return isExprToGetCheckedPtrCapableMember(E);
+  }
   const char *ptrKind() const final { return "unchecked"; }
 };
 
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
index b1350b909302..8faf6a219450 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
@@ -28,6 +28,7 @@ class RawPtrRefMemberChecker
 private:
   BugType Bug;
   mutable BugReporter *BR;
+  mutable llvm::DenseSet<const ObjCIvarDecl *> IvarDeclsToIgnore;
 
 protected:
   mutable std::optional<RetainTypeChecker> RTC;
@@ -36,7 +37,8 @@ class RawPtrRefMemberChecker
   RawPtrRefMemberChecker(const char *description)
       : Bug(this, description, "WebKit coding guidelines") {}
 
-  virtual std::optional<bool> isUnsafePtr(QualType) const = 0;
+  virtual std::optional<bool> isUnsafePtr(QualType,
+                                          bool ignoreARC = false) const = 0;
   virtual const char *typeName() const = 0;
   virtual const char *invariant() const = 0;
 
@@ -138,6 +140,8 @@ class RawPtrRefMemberChecker
       return;
     }
     if (auto *ID = dyn_cast<ObjCImplementationDecl>(CD)) {
+      for (auto *PropImpl : ID->property_impls())
+        visitPropImpl(CD, PropImpl);
       for (auto *Ivar : ID->ivars())
         visitIvarDecl(CD, Ivar);
       return;
@@ -148,6 +152,10 @@ class RawPtrRefMemberChecker
                      const ObjCIvarDecl *Ivar) const {
     if (BR->getSourceManager().isInSystemHeader(Ivar->getLocation()))
       return;
+
+    if (IvarDeclsToIgnore.contains(Ivar))
+      return;
+
     auto QT = Ivar->getType();
     const Type *IvarType = QT.getTypePtrOrNull();
     if (!IvarType)
@@ -157,6 +165,8 @@ class RawPtrRefMemberChecker
     if (!IsUnsafePtr || !*IsUnsafePtr)
       return;
 
+    IvarDeclsToIgnore.insert(Ivar);
+
     if (auto *MemberCXXRD = IvarType->getPointeeCXXRecordDecl())
       reportBug(Ivar, IvarType, MemberCXXRD, CD);
     else if (auto *ObjCDecl = getObjCDecl(IvarType))
@@ -167,13 +177,15 @@ class RawPtrRefMemberChecker
                              const ObjCPropertyDecl *PD) const {
     if (BR->getSourceManager().isInSystemHeader(PD->getLocation()))
       return;
-    auto QT = PD->getType();
-    const Type *PropType = QT.getTypePtrOrNull();
-    if (!PropType)
-      return;
 
-    auto IsUnsafePtr = isUnsafePtr(QT);
-    if (!IsUnsafePtr || !*IsUnsafePtr)
+    if (const ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(CD)) {
+      if (!RTC || !RTC->defaultSynthProperties() ||
+          ID->isObjCRequiresPropertyDefs())
+        return;
+    }
+
+    auto [IsUnsafe, PropType] = isPropImplUnsafePtr(PD);
+    if (!IsUnsafe)
       return;
 
     if (auto *MemberCXXRD = PropType->getPointeeCXXRecordDecl())
@@ -182,6 +194,47 @@ class RawPtrRefMemberChecker
       reportBug(PD, PropType, ObjCDecl, CD);
   }
 
+  void visitPropImpl(const ObjCContainerDecl *CD,
+                     const ObjCPropertyImplDecl *PID) const {
+    if (BR->getSourceManager().isInSystemHeader(PID->getLocation()))
+      return;
+
+    if (PID->getPropertyImplementation() != ObjCPropertyImplDecl::Synthesize)
+      return;
+
+    auto *PropDecl = PID->getPropertyDecl();
+    if (auto *IvarDecl = PID->getPropertyIvarDecl()) {
+      if (IvarDeclsToIgnore.contains(IvarDecl))
+        return;
+      IvarDeclsToIgnore.insert(IvarDecl);
+    }
+    auto [IsUnsafe, PropType] = isPropImplUnsafePtr(PropDecl);
+    if (!IsUnsafe)
+      return;
+
+    if (auto *MemberCXXRD = PropType->getPointeeCXXRecordDecl())
+      reportBug(PropDecl, PropType, MemberCXXRD, CD);
+    else if (auto *ObjCDecl = getObjCDecl(PropType))
+      reportBug(PropDecl, PropType, ObjCDecl, CD);
+  }
+
+  std::pair<bool, const Type *>
+  isPropImplUnsafePtr(const ObjCPropertyDecl *PD) const {
+    if (!PD)
+      return {false, nullptr};
+
+    auto QT = PD->getType();
+    const Type *PropType = QT.getTypePtrOrNull();
+    if (!PropType)
+      return {false, nullptr};
+
+    // "assign" property doesn't retain even under ARC so treat it as unsafe.
+    bool ignoreARC =
+        !PD->isReadOnly() && PD->getSetterKind() == ObjCPropertyDecl::Assign;
+    auto IsUnsafePtr = isUnsafePtr(QT, ignoreARC);
+    return {IsUnsafePtr && *IsUnsafePtr, PropType};
+  }
+
   bool shouldSkipDecl(const RecordDecl *RD) const {
     if (!RD->isThisDeclarationADefinition())
       return true;
@@ -272,7 +325,7 @@ class NoUncountedMemberChecker final : public RawPtrRefMemberChecker {
       : RawPtrRefMemberChecker("Member variable is a raw-pointer/reference to "
                                "reference-countable type") {}
 
-  std::optional<bool> isUnsafePtr(QualType QT) const final {
+  std::optional<bool> isUnsafePtr(QualType QT, bool) const final {
     return isUncountedPtr(QT.getCanonicalType());
   }
 
@@ -289,7 +342,7 @@ class NoUncheckedPtrMemberChecker final : public RawPtrRefMemberChecker {
       : RawPtrRefMemberChecker("Member variable is a raw-pointer/reference to "
                                "checked-pointer capable type") {}
 
-  std::optional<bool> isUnsafePtr(QualType QT) const final {
+  std::optional<bool> isUnsafePtr(QualType QT, bool) const final {
     return isUncheckedPtr(QT.getCanonicalType());
   }
 
@@ -309,8 +362,8 @@ class NoUnretainedMemberChecker final : public RawPtrRefMemberChecker {
     RTC = RetainTypeChecker();
   }
 
-  std::optional<bool> isUnsafePtr(QualType QT) const final {
-    return RTC->isUnretained(QT);
+  std::optional<bool> isUnsafePtr(QualType QT, bool ignoreARC) const final {
+    return RTC->isUnretained(QT, ignoreARC);
   }
 
   const char *typeName() const final { return "retainable type"; }
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 49cf4174415b..5fe64dc5e927 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -2164,6 +2164,7 @@ PathSensitiveBugReport::PathSensitiveBugReport(
     : BugReport(Kind::PathSensitive, bt, shortDesc, desc), ErrorNode(errorNode),
       ErrorNodeRange(getStmt() ? getStmt()->getSourceRange() : SourceRange()),
       UniqueingLocation(LocationToUnique), UniqueingDecl(DeclToUnique) {
+  assert(ErrorNode && "The error node must be non-null!");
   assert(!isDependency(ErrorNode->getState()
                            ->getAnalysisManager()
                            .getCheckerManager()
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Core/Checker.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Core/Checker.cpp
index af34ad3b5766..f06ae7d093b6 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Core/Checker.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Core/Checker.cpp
@@ -21,11 +21,3 @@ StringRef CheckerBase::getDebugTag() const { return getName(); }
 
 void CheckerBackend::printState(raw_ostream &Out, ProgramStateRef State,
                                 const char *NL, const char *Sep) const {}
-
-CheckerProgramPointTag::CheckerProgramPointTag(StringRef CheckerName,
-                                               StringRef Msg)
-  : SimpleProgramPointTag(CheckerName, Msg) {}
-
-CheckerProgramPointTag::CheckerProgramPointTag(const CheckerBase *Checker,
-                                               StringRef Msg)
-    : SimpleProgramPointTag(Checker->getDebugTag(), Msg) {}
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
index 4ed4113919c1..8b404377186e 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
@@ -129,11 +129,19 @@ std::optional<int> tryExpandAsInteger(StringRef Macro, const Preprocessor &PP) {
 
   // Parse an integer at the end of the macro definition.
   const Token &T = FilteredTokens.back();
-  // FIXME: EOF macro token coming from a PCH file on macOS while marked as
-  //        literal, doesn't contain any literal data
-  if (!T.isLiteral() || !T.getLiteralData())
+
+  if (!T.isLiteral())
     return std::nullopt;
-  StringRef ValueStr = StringRef(T.getLiteralData(), T.getLength());
+
+  bool InvalidSpelling = false;
+  SmallVector<char> Buffer(T.getLength());
+  // `Preprocessor::getSpelling` can get the spelling of the token regardless of
+  // whether the macro is defined in a PCH or not:
+  StringRef ValueStr = PP.getSpelling(T, Buffer, &InvalidSpelling);
+
+  if (InvalidSpelling)
+    return std::nullopt;
+
   llvm::APInt IntValue;
   constexpr unsigned AutoSenseRadix = 0;
   if (ValueStr.getAsInteger(AutoSenseRadix, IntValue))
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
index d2b7b2bfbb01..0fe677e4ee43 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -585,8 +585,8 @@ namespace {
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
       llvm::TimeTraceScope TimeScope(
           checkerScopeName("Allocator", checkFn.Checker));
-      ProgramPoint L =
-          PostAllocatorCall(Call.getOriginExpr(), Pred->getLocationContext());
+      ProgramPoint L = PostAllocatorCall(
+          Call.getOriginExpr(), Pred->getLocationContext(), checkFn.Checker);
       CheckerContext C(Bldr, Eng, Pred, L, WasInlined);
       checkFn(cast<CXXAllocatorCall>(*Call.cloneWithState(Pred->getState())),
               C);
diff --git a/external/llvm-project/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/external/llvm-project/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index b28deee41d1c..c77ef26da568 100644
--- a/external/llvm-project/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/external/llvm-project/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1732,7 +1732,6 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::ExpressionTraitExprClass:
     case Stmt::UnresolvedLookupExprClass:
     case Stmt::UnresolvedMemberExprClass:
-    case Stmt::TypoExprClass:
     case Stmt::RecoveryExprClass:
     case Stmt::CXXNoexceptExprClass:
     case Stmt::PackExpansionExprClass:
diff --git a/external/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp b/external/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp
index 37f95411af19..5a4e805d4a9d 100644
--- a/external/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp
+++ b/external/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp
@@ -8,13 +8,10 @@
 
 #include "clang/Support/RISCVVIntrinsicUtils.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include <numeric>
 #include <optional>
 
 using namespace llvm;
@@ -1210,50 +1207,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum PolicyScheme PS) {
   return OS;
 }
 
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum RVVRequire Require) {
-  switch (Require) {
-    STRINGIFY(RVV_REQ_RV64)
-    STRINGIFY(RVV_REQ_Zvfhmin)
-    STRINGIFY(RVV_REQ_Xandesvdot)
-    STRINGIFY(RVV_REQ_Xandesvpackfph)
-    STRINGIFY(RVV_REQ_Xsfvcp)
-    STRINGIFY(RVV_REQ_Xsfvfnrclipxfqf)
-    STRINGIFY(RVV_REQ_Xsfvfwmaccqqq)
-    STRINGIFY(RVV_REQ_Xsfvqmaccdod)
-    STRINGIFY(RVV_REQ_Xsfvqmaccqoq)
-    STRINGIFY(RVV_REQ_Zvbb)
-    STRINGIFY(RVV_REQ_Zvbc)
-    STRINGIFY(RVV_REQ_Zvkb)
-    STRINGIFY(RVV_REQ_Zvkg)
-    STRINGIFY(RVV_REQ_Zvkned)
-    STRINGIFY(RVV_REQ_Zvknha)
-    STRINGIFY(RVV_REQ_Zvknhb)
-    STRINGIFY(RVV_REQ_Zvksed)
-    STRINGIFY(RVV_REQ_Zvksh)
-    STRINGIFY(RVV_REQ_Zvfbfwma)
-    STRINGIFY(RVV_REQ_Zvfbfmin)
-    STRINGIFY(RVV_REQ_Zvfh)
-    STRINGIFY(RVV_REQ_Experimental)
-  default:
-    llvm_unreachable("Unsupported RVVRequire!");
-    break;
-  }
-  return OS;
-}
-
 #undef STRINGIFY
 
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
-                              const RequiredExtensionBits &Exts) {
-  OS << "{";
-  ListSeparator LS;
-  for (unsigned I = 0; I < RVV_REQ_NUM; I++)
-    if (Exts[I])
-      OS << LS << static_cast<RVVRequire>(I);
-  OS << "}";
-  return OS;
-}
-
 raw_ostream &operator<<(raw_ostream &OS, const RVVIntrinsicRecord &Record) {
   OS << "{";
   OS << "/*Name=*/\"" << Record.Name << "\", ";
@@ -1262,7 +1217,7 @@ raw_ostream &operator<<(raw_ostream &OS, const RVVIntrinsicRecord &Record) {
     OS << "/*OverloadedName=*/nullptr, ";
   else
     OS << "/*OverloadedName=*/\"" << Record.OverloadedName << "\", ";
-  OS << "/*RequiredExtensions=*/" << Record.RequiredExtensions << ", ";
+  OS << "/*RequiredExtensions=*/\"" << Record.RequiredExtensions << "\", ";
   OS << "/*PrototypeIndex=*/" << Record.PrototypeIndex << ", ";
   OS << "/*SuffixIndex=*/" << Record.SuffixIndex << ", ";
   OS << "/*OverloadedSuffixIndex=*/" << Record.OverloadedSuffixIndex << ", ";
diff --git a/external/llvm-project/clang/lib/Tooling/ArgumentsAdjusters.cpp b/external/llvm-project/clang/lib/Tooling/ArgumentsAdjusters.cpp
index d01c57ee69c0..999fa790124c 100644
--- a/external/llvm-project/clang/lib/Tooling/ArgumentsAdjusters.cpp
+++ b/external/llvm-project/clang/lib/Tooling/ArgumentsAdjusters.cpp
@@ -22,8 +22,7 @@ namespace clang {
 namespace tooling {
 
 static StringRef getDriverMode(const CommandLineArguments &Args) {
-  for (const auto &Arg : Args) {
-    StringRef ArgRef = Arg;
+  for (StringRef ArgRef : Args) {
     if (ArgRef.consume_front("--driver-mode=")) {
       return ArgRef;
     }
diff --git a/external/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/external/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 4316b584a2f3..140833050f4e 100644
--- a/external/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/external/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -173,7 +173,7 @@ DependencyScanningFilesystemSharedCache::CacheShard::getOrEmplaceEntryForUID(
     llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
     std::unique_ptr<llvm::MemoryBuffer> Contents) {
   std::lock_guard<std::mutex> LockGuard(CacheLock);
-  auto [It, Inserted] = EntriesByUID.insert({UID, nullptr});
+  auto [It, Inserted] = EntriesByUID.try_emplace(UID);
   auto &CachedEntry = It->getSecond();
   if (Inserted) {
     CachedFileContents *StoredContents = nullptr;
diff --git a/external/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/external/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 44a270d5f7b3..b1495163ccc2 100644
--- a/external/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/external/llvm-project/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -657,7 +657,7 @@ void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
     P1689ModuleInfo RequiredModule;
     RequiredModule.ModuleName = Path[0].getIdentifierInfo()->getName().str();
     RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
-    MDC.RequiredStdCXXModules.push_back(RequiredModule);
+    MDC.RequiredStdCXXModules.push_back(std::move(RequiredModule));
     return;
   }
 
@@ -920,7 +920,7 @@ void ModuleDepCollectorPP::addAllSubmoduleDeps(
 
 void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID,
                                            ModuleDeps &MD) {
-  MD.ClangModuleDeps.push_back(ID);
+  MD.ClangModuleDeps.push_back(std::move(ID));
   if (MD.IsInStableDirectories)
     MD.IsInStableDirectories = MDC.ModularDeps[M]->IsInStableDirectories;
 }
diff --git a/external/llvm-project/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc b/external/llvm-project/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc
index 44ad4a0d9684..a2c12007134d 100644
--- a/external/llvm-project/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc
+++ b/external/llvm-project/clang/lib/Tooling/Inclusions/Stdlib/StdSpecialSymbolMap.inc
@@ -452,6 +452,8 @@ SYMBOL(equal_range, std::ranges::, <algorithm>)
 SYMBOL(lower_bound, std::ranges::, <algorithm>)
 SYMBOL(upper_bound, std::ranges::, <algorithm>)
 
+SYMBOL(unwrap_reference_t, std::, <type_traits>)
+
 // These are C symbols that are not under std namespace.
 SYMBOL(localtime_r, None, <ctime>)
 SYMBOL(localtime_r, None, <time.h>)
diff --git a/external/llvm-project/clang/test/AST/ByteCode/cxx11.cpp b/external/llvm-project/clang/test/AST/ByteCode/cxx11.cpp
index 44725f13e6a5..b34e7823220e 100644
--- a/external/llvm-project/clang/test/AST/ByteCode/cxx11.cpp
+++ b/external/llvm-project/clang/test/AST/ByteCode/cxx11.cpp
@@ -260,3 +260,31 @@ namespace ZeroSizeCmp {
   static_assert(&start != &end, ""); // both-error {{constant expression}} \
                                      // both-note {{comparison of pointers '&start' and '&end' to unrelated zero-sized objects}}
 }
+
+namespace OverlappingStrings {
+  static_assert(+"foo" != +"bar", "");
+  static_assert(&"xfoo"[1] != &"yfoo"[1], "");
+  static_assert(+"foot" != +"foo", "");
+  static_assert(+"foo\0bar" != +"foo\0baz", "");
+
+
+#define fold(x) (__builtin_constant_p(x) ? (x) : (x))
+  static_assert(fold((const char*)u"A" != (const char*)"\0A\0x"), "");
+  static_assert(fold((const char*)u"A" != (const char*)"A\0\0x"), "");
+  static_assert(fold((const char*)u"AAA" != (const char*)"AAA\0\0x"), "");
+
+  constexpr const char *string = "hello";
+  constexpr const char *also_string = string;
+  static_assert(string == string, "");
+  static_assert(string == also_string, "");
+
+
+  // These strings may overlap, and so the result of the comparison is unknown.
+  constexpr bool may_overlap_1 = +"foo" == +"foo"; // both-error {{}} both-note {{addresses of potentially overlapping literals}}
+  constexpr bool may_overlap_2 = +"foo" == +"foo\0bar"; // both-error {{}} both-note {{addresses of potentially overlapping literals}}
+  constexpr bool may_overlap_3 = +"foo" == &"bar\0foo"[4]; // both-error {{}} both-note {{addresses of potentially overlapping literals}}
+  constexpr bool may_overlap_4 = &"xfoo"[1] == &"xfoo"[1]; // both-error {{}} both-note {{addresses of potentially overlapping literals}}
+
+
+
+}
diff --git a/external/llvm-project/clang/test/AST/ByteCode/cxx17.cpp b/external/llvm-project/clang/test/AST/ByteCode/cxx17.cpp
index 08a40e0a9286..0cf3a4f666d6 100644
--- a/external/llvm-project/clang/test/AST/ByteCode/cxx17.cpp
+++ b/external/llvm-project/clang/test/AST/ByteCode/cxx17.cpp
@@ -141,3 +141,11 @@ template <int x> constexpr auto c() {
 }
 
 auto y = c<1>(); // both-note {{in instantiation of function template specialization 'c<1>' requested here}}
+
+namespace NonConstexprStructuredBinding {
+  void f1() {
+    int arr[2] = {};
+    auto [a, b] = arr;
+    static_assert(&a != &b);
+  }
+}
diff --git a/external/llvm-project/clang/test/AST/ByteCode/cxx20.cpp b/external/llvm-project/clang/test/AST/ByteCode/cxx20.cpp
index 0b2234ef8329..e0fb38e10610 100644
--- a/external/llvm-project/clang/test/AST/ByteCode/cxx20.cpp
+++ b/external/llvm-project/clang/test/AST/ByteCode/cxx20.cpp
@@ -122,8 +122,8 @@ static_assert(!b4);
 constexpr auto bar(const char *p) { return p + __builtin_strlen(p); }
 constexpr auto b5 = bar(p1) == p1;
 static_assert(!b5);
-constexpr auto b6 = bar(p1) == ""; // ref-error {{must be initialized by a constant expression}} \
-                                   // ref-note {{comparison of addresses of potentially overlapping literals}}
+constexpr auto b6 = bar(p1) == ""; // both-error {{must be initialized by a constant expression}} \
+                                   // both-note {{comparison of addresses of potentially overlapping literals}}
 constexpr auto b7 = bar(p1) + 1 == ""; // both-error {{must be initialized by a constant expression}} \
                                        // both-note {{comparison against pointer '&"test1"[6]' that points past the end of a complete object has unspecified value}}
 
diff --git a/external/llvm-project/clang/test/AST/ByteCode/fixed-point.cpp b/external/llvm-project/clang/test/AST/ByteCode/fixed-point.cpp
index 4bf80ba7c58f..5237d758fa25 100644
--- a/external/llvm-project/clang/test/AST/ByteCode/fixed-point.cpp
+++ b/external/llvm-project/clang/test/AST/ByteCode/fixed-point.cpp
@@ -17,6 +17,14 @@ constexpr _Accum acc = (0.5r, 6.9k);
 constexpr _Accum A{};
 static_assert(A == 0.0k);
 static_assert(A == 0);
+static_assert(!A);
+
+constexpr bool toBool() {
+  if (A)
+    return true;
+  return false;
+}
+static_assert(!toBool());
 
 namespace IntToFixedPointCast {
   constexpr _Accum B = 13;
diff --git a/external/llvm-project/clang/test/AST/ByteCode/literals.cpp b/external/llvm-project/clang/test/AST/ByteCode/literals.cpp
index 2fa7b69b9347..699746c0b2c4 100644
--- a/external/llvm-project/clang/test/AST/ByteCode/literals.cpp
+++ b/external/llvm-project/clang/test/AST/ByteCode/literals.cpp
@@ -910,7 +910,8 @@ namespace CompoundLiterals {
   constexpr int f2(int *x =(int[]){1,2,3}) {
     return x[0];
   }
-  constexpr int g = f2(); // Should evaluate to 1?
+  // Should evaluate to 1?
+  constexpr int g = f2(); // #g_decl
   static_assert(g == 1, "");
 
   // This example should be rejected because the lifetime of the compound
@@ -1347,7 +1348,10 @@ namespace NTTP {
 namespace UnaryOpError {
   constexpr int foo() {
     int f = 0;
-    ++g; // both-error {{use of undeclared identifier 'g'}}
+    ++g; // both-error {{use of undeclared identifier 'g'}} \
+            both-error {{cannot assign to variable 'g' with const-qualified type 'const int'}} \
+            both-note@#g_decl {{'CompoundLiterals::g' declared here}} \
+            both-note@#g_decl {{variable 'g' declared const here}}
     return f;
   }
 }
diff --git a/external/llvm-project/clang/test/AST/ByteCode/new-delete.cpp b/external/llvm-project/clang/test/AST/ByteCode/new-delete.cpp
index 1ee41a98e13b..9c293e5d15fc 100644
--- a/external/llvm-project/clang/test/AST/ByteCode/new-delete.cpp
+++ b/external/llvm-project/clang/test/AST/ByteCode/new-delete.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -std=c++20 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -triple=i686-linux-gnu -std=c++20 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -verify=ref,both %s
-// RUN: %clang_cc1 -std=c++20 -verify=ref,both %s
-// RUN: %clang_cc1 -triple=i686-linux-gnu -std=c++20 -verify=ref,both %s
+// RUN: %clang_cc1            -verify=expected,both                        -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -std=c++20 -verify=expected,both                        -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -std=c++20 -verify=expected,both -triple=i686-linux-gnu -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1            -verify=ref,both                                                                     %s
+// RUN: %clang_cc1 -std=c++20 -verify=ref,both                                                                     %s
+// RUN: %clang_cc1 -std=c++20 -verify=ref,both      -triple=i686-linux-gnu                                         %s
 
 #if __cplusplus >= 202002L
 
@@ -1012,6 +1012,16 @@ constexpr int no_deallocate_nonalloc = (std::allocator<int>().deallocate((int*)&
                                                                                                              // both-note {{in call}} \
                                                                                                              // both-note {{declared here}}
 
+namespace OpNewNothrow {
+  constexpr int f() {
+      int *v = (int*)operator new(sizeof(int), std::align_val_t(2), std::nothrow); // both-note {{cannot allocate untyped memory in a constant expression; use 'std::allocator<T>::allocate' to allocate memory of type 'T'}}
+      operator delete(v, std::align_val_t(2), std::nothrow);
+      return 1;
+  }
+  static_assert(f()); // both-error {{not an integral constant expression}} \
+                      // both-note {{in call to}}
+}
+
 #else
 /// Make sure we reject this prior to C++20
 constexpr int a() { // both-error {{never produces a constant expression}}
diff --git a/external/llvm-project/clang/test/AST/ast-dump-aarch64-neon-types.c b/external/llvm-project/clang/test/AST/ast-dump-aarch64-neon-types.c
new file mode 100644
index 000000000000..f509bd880c14
--- /dev/null
+++ b/external/llvm-project/clang/test/AST/ast-dump-aarch64-neon-types.c
@@ -0,0 +1,125 @@
+// Test that NEON types are defined, even when arm_neon.h is not included.
+// as required by AAPCS64 "Support for Advanced SIMD Extensions".
+
+// RUN: %clang_cc1 -ast-dump -triple aarch64-linux-gnu %s -x c | FileCheck %s
+// RUN: %clang_cc1 -ast-dump -triple aarch64-linux-gnu %s -x c++ | FileCheck %s
+// RUN: %clang_cc1 -verify -verify-ignore-unexpected=note -triple x86_64 %s -x c
+// RUN: %clang_cc1 -verify -verify-ignore-unexpected=note -triple x86_64 %s -x c++
+// RUN: %clang_cc1 -verify -verify-ignore-unexpected=note -triple arm-linux-gnu %s -x c
+// RUN: %clang_cc1 -verify -verify-ignore-unexpected=note -triple arm-linux-gnu %s -x c++
+
+__Int8x8_t Int8x8;
+// CHECK: Int8x8 '__Int8x8_t':'__attribute__((neon_vector_type(8))) signed char'
+// expected-error@-2{{unknown type name '__Int8x8_t'}}
+
+__Int16x4_t Int16x4;
+// CHECK: Int16x4 '__Int16x4_t':'__attribute__((neon_vector_type(4))) short'
+// expected-error@-2{{unknown type name '__Int16x4_t'}}
+
+__Int32x2_t Int32x2;
+// CHECK: Int32x2 '__Int32x2_t':'__attribute__((neon_vector_type(2))) int'
+// expected-error@-2{{unknown type name '__Int32x2_t'}}
+
+__Uint8x8_t Uint8x8;
+// CHECK: Uint8x8 '__Uint8x8_t':'__attribute__((neon_vector_type(8))) unsigned char'
+// expected-error@-2{{unknown type name '__Uint8x8_t'}}
+
+__Uint16x4_t Uint16x4;
+// CHECK: Uint16x4 '__Uint16x4_t':'__attribute__((neon_vector_type(4))) unsigned short'
+// expected-error@-2{{unknown type name '__Uint16x4_t'}}
+
+__Uint32x2_t Uint32x2;
+// CHECK: Uint32x2 '__Uint32x2_t':'__attribute__((neon_vector_type(2))) unsigned int'
+// expected-error@-2{{unknown type name '__Uint32x2_t'}}
+
+__Float16x4_t Float16x4;
+// CHECK: Float16x4 '__Float16x4_t':'__attribute__((neon_vector_type(4))) __fp16'
+// expected-error@-2{{unknown type name '__Float16x4_t'}}
+
+__Float32x2_t Float32x2;
+// CHECK: Float32x2 '__Float32x2_t':'__attribute__((neon_vector_type(2))) float'
+// expected-error@-2{{unknown type name '__Float32x2_t'}}
+
+__Poly8x8_t Poly8x8;
+// CHECK: Poly8x8 '__Poly8x8_t':'__attribute__((neon_polyvector_type(8))) unsigned char'
+// expected-error@-2{{unknown type name '__Poly8x8_t'}}
+
+__Poly16x4_t Poly16x4;
+// CHECK: Poly16x4 '__Poly16x4_t':'__attribute__((neon_polyvector_type(4))) unsigned short'
+// expected-error@-2{{unknown type name '__Poly16x4_t'}}
+
+__Bfloat16x4_t Bfloat16x4;
+// CHECK: Bfloat16x4 '__Bfloat16x4_t':'__attribute__((neon_vector_type(4))) __bf16'
+// expected-error@-2{{unknown type name '__Bfloat16x4_t'}}
+
+__Int8x16_t Int8x16;
+// CHECK: Int8x16 '__Int8x16_t':'__attribute__((neon_vector_type(16))) signed char'
+// expected-error@-2{{unknown type name '__Int8x16_t'}}
+
+__Int16x8_t Int16x8;
+// CHECK: Int16x8 '__Int16x8_t':'__attribute__((neon_vector_type(8))) short'
+// expected-error@-2{{unknown type name '__Int16x8_t'}}
+
+__Int32x4_t Int32x4;
+// CHECK: Int32x4 '__Int32x4_t':'__attribute__((neon_vector_type(4))) int'
+// expected-error@-2{{unknown type name '__Int32x4_t'}}
+
+__Int64x2_t Int64x2;
+// CHECK: Int64x2 '__Int64x2_t':'__attribute__((neon_vector_type(2))) long'
+// expected-error@-2{{unknown type name '__Int64x2_t'}}
+
+__Uint8x16_t Uint8x16;
+// CHECK: Uint8x16 '__Uint8x16_t':'__attribute__((neon_vector_type(16))) unsigned char'
+// expected-error@-2{{unknown type name '__Uint8x16_t'}}
+
+__Uint16x8_t Uint16x8;
+// CHECK: Uint16x8 '__Uint16x8_t':'__attribute__((neon_vector_type(8))) unsigned short'
+// expected-error@-2{{unknown type name '__Uint16x8_t'}}
+
+__Uint32x4_t Uint32x4;
+// CHECK: Uint32x4 '__Uint32x4_t':'__attribute__((neon_vector_type(4))) unsigned int'
+// expected-error@-2{{unknown type name '__Uint32x4_t'}}
+
+__Uint64x2_t Uint64x2;
+// CHECK: Uint64x2 '__Uint64x2_t':'__attribute__((neon_vector_type(2))) unsigned long'
+// expected-error@-2{{unknown type name '__Uint64x2_t'}}
+
+__Float16x8_t Float16x8;
+// CHECK: Float16x8 '__Float16x8_t':'__attribute__((neon_vector_type(8))) __fp16'
+// expected-error@-2{{unknown type name '__Float16x8_t'}}
+
+__Float32x4_t Float32x4;
+// CHECK: Float32x4 '__Float32x4_t':'__attribute__((neon_vector_type(4))) float'
+// expected-error@-2{{unknown type name '__Float32x4_t'}}
+
+__Float64x2_t Float64x2;
+// CHECK: Float64x2 '__Float64x2_t':'__attribute__((neon_vector_type(2))) double'
+// expected-error@-2{{unknown type name '__Float64x2_t'}}
+
+__Poly8x16_t Poly8x16;
+// CHECK: Poly8x16 '__Poly8x16_t':'__attribute__((neon_polyvector_type(16))) unsigned char'
+// expected-error@-2{{unknown type name '__Poly8x16_t'}}
+
+__Poly16x8_t Poly16x8;
+// CHECK: Poly16x8 '__Poly16x8_t':'__attribute__((neon_polyvector_type(8))) unsigned short'
+// expected-error@-2{{unknown type name '__Poly16x8_t'}}
+
+__Poly64x2_t Poly64x2;
+// CHECK: Poly64x2 '__Poly64x2_t':'__attribute__((neon_polyvector_type(2))) unsigned long'
+// expected-error@-2{{unknown type name '__Poly64x2_t'}}
+
+__Bfloat16x8_t Bfloat16x8;
+// CHECK: Bfloat16x8 '__Bfloat16x8_t':'__attribute__((neon_vector_type(8))) __bf16'
+// expected-error@-2{{unknown type name '__Bfloat16x8_t'}}
+
+__mfp8 mfp8;
+// CHECK: mfp8 '__mfp8'
+// expected-error@-2{{unknown type name '__mfp8'}}
+
+__Mfloat8x8_t Mfloat8x8;
+// CHECK: Mfloat8x8 '__Mfloat8x8_t':'__attribute__((neon_vector_type(8))) __mfp8'
+// expected-error@-2{{unknown type name '__Mfloat8x8_t'}}
+
+__Mfloat8x16_t Mfloat8x16;
+// CHECK: Mfloat8x16 '__Mfloat8x16_t':'__attribute__((neon_vector_type(16))) __mfp8'
+// expected-error@-2{{unknown type name '__Mfloat8x16_t'}}
diff --git a/external/llvm-project/clang/test/AST/ast-dump-recovery.c b/external/llvm-project/clang/test/AST/ast-dump-recovery.c
index 68d3f182dd9f..09a03fb9d6fd 100644
--- a/external/llvm-project/clang/test/AST/ast-dump-recovery.c
+++ b/external/llvm-project/clang/test/AST/ast-dump-recovery.c
@@ -23,13 +23,6 @@ int postfix_inc = a++;
 // CHECK-NEXT:      `-IntegerLiteral {{.*}} 'int'
 int unary_address = &(a + 1);
 
-// CHECK:       VarDecl {{.*}} ternary 'int' cinit
-// CHECK-NEXT:  `-ConditionalOperator {{.*}}
-// CHECK-NEXT:    |-DeclRefExpr {{.*}} 'a'
-// CHECK-NEXT:    |-RecoveryExpr {{.*}}
-// CHECK-NEXT:    `-DeclRefExpr {{.*}} 'a'
-int ternary = a ? undef : a;
-
 void test1() {
   // CHECK:     `-RecoveryExpr {{.*}} contains-errors
   // CHECK-NEXT:  `-DeclRefExpr {{.*}} 'a' 'const int'
@@ -91,12 +84,6 @@ void test3() {
   // CHECK-NEXT: |   `-DeclRefExpr {{.*}} '__builtin_classify_type'
   // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 1
   (*__builtin_classify_type)(1);
-
-  extern void ext();
-  // CHECK:     CallExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT: |-DeclRefExpr {{.*}} 'ext'
-  // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>'
-  ext(undef_var);
 }
 
 // Verify no crash.
@@ -110,23 +97,6 @@ void test4() {
   };
 }
 
-// Verify no crash
-void test5_GH62711() {
-  // CHECK:      VAArgExpr {{.*}} 'int' contains-errors
-  // CHECK-NEXT: | `-ImplicitCastExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT: |   `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  if (__builtin_va_arg(undef, int) << 1);
-}
-
-void test6_GH50244() {
-  double array[16];
-  // CHECK:      UnaryExprOrTypeTraitExpr {{.*}} 'unsigned long' contains-errors sizeof
-  // CHECK-NEXT: `-CallExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   |-DeclRefExpr {{.*}} 'int ()'
-  // CHECK-NEXT:   `-RecoveryExpr {{.*}} '<dependent type>'
-  sizeof array / sizeof foo(undef);
-}
-
 // No crash on DeclRefExpr that refers to ValueDecl with invalid initializers.
 void test7() {
   int b[] = {""()};
diff --git a/external/llvm-project/clang/test/AST/ast-dump-recovery.cpp b/external/llvm-project/clang/test/AST/ast-dump-recovery.cpp
index b8195950f2fa..a8e30f1759e9 100644
--- a/external/llvm-project/clang/test/AST/ast-dump-recovery.cpp
+++ b/external/llvm-project/clang/test/AST/ast-dump-recovery.cpp
@@ -9,28 +9,6 @@ int some_func(int *);
 // CHECK-NEXT:    `-IntegerLiteral {{.*}} 123
 // DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
 int invalid_call = some_func(123);
-void test_invalid_call_1(int s) {
-  // CHECK:      CallExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT: |-UnresolvedLookupExpr {{.*}} 'some_func'
-  // CHECK-NEXT: |-RecoveryExpr {{.*}} <col:13>
-  // CHECK-NEXT: `-BinaryOperator {{.*}}
-  // CHECK-NEXT:   |-RecoveryExpr {{.*}}
-  // CHECK-NEXT:   `-IntegerLiteral {{.*}} <col:28> 'int' 1
-  some_func(undef1, undef2+1);
-
-  // CHECK:      BinaryOperator {{.*}} '<dependent type>' contains-errors '='
-  // CHECK-NEXT: |-DeclRefExpr {{.*}} 's'
-  // CHECK-NEXT: `-CallExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   |-UnresolvedLookupExpr {{.*}} 'some_func'
-  // CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
-  s = some_func(undef1);
-
-  // CHECK:     VarDecl {{.*}} var 'int'
-  // CHECK-NEXT: `-CallExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   |-UnresolvedLookupExpr {{.*}} 'some_func'
-  // CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
-  int var = some_func(undef1);
-}
 
 int some_func2(int a, int b);
 void test_invalid_call_2() {
@@ -63,22 +41,6 @@ int ambig_func(float);
 // DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
 int ambig_call = ambig_func(123);
 
-// CHECK:     VarDecl {{.*}} unresolved_call1
-// CHECK-NEXT:`-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-// CHECK-NEXT:  `-UnresolvedLookupExpr {{.*}} 'bar'
-// DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
-int unresolved_call1 = bar();
-
-// CHECK:     VarDecl {{.*}} unresolved_call2
-// CHECK-NEXT:`-CallExpr {{.*}} contains-errors
-// CHECK-NEXT:  |-UnresolvedLookupExpr {{.*}} 'bar'
-// CHECK-NEXT:  |-RecoveryExpr {{.*}} contains-errors
-// CHECK-NEXT:  | `-UnresolvedLookupExpr {{.*}} 'baz'
-// CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
-// CHECK-NEXT:     `-UnresolvedLookupExpr {{.*}} 'qux'
-// DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
-int unresolved_call2 = bar(baz(), qux());
-
 constexpr int a = 10;
 
 // CHECK:     VarDecl {{.*}} postfix_inc
@@ -177,11 +139,6 @@ void test2(Foo2 f) {
   f.overload(1);
 }
 
-// CHECK:     |-AlignedAttr {{.*}} alignas
-// CHECK-NEXT:| `-RecoveryExpr {{.*}} contains-errors
-// CHECK-NEXT:|   `-UnresolvedLookupExpr {{.*}} 'invalid'
-struct alignas(invalid()) Aligned {};
-
 auto f();
 int f(double);
 // CHECK:      VarDecl {{.*}} unknown_type_call 'int'
@@ -203,16 +160,6 @@ void InvalidInitalizer(int x) {
   // CHECK-NEXT:  `-InitListExpr
   // CHECK-NEDT:   `-DeclRefExpr {{.*}} 'x'
   Bar a3{x};
-  // CHECK:     `-VarDecl {{.*}} a4 'Bar'
-  // CHECK-NEXT: `-ParenListExpr {{.*}} 'NULL TYPE' contains-errors
-  // CHECK-NEXT:  `-RecoveryExpr {{.*}} contains-errors
-  // CHECK-NEXT:   `-UnresolvedLookupExpr {{.*}} 'invalid'
-  Bar a4(invalid());
-  // CHECK:     `-VarDecl {{.*}} a5 'Bar'
-  // CHECK-NEXT: `-InitListExpr {{.*}} contains-errors
-  // CHECK-NEXT:  `-RecoveryExpr {{.*}} contains-errors
-  // CHECK-NEXT:   `-UnresolvedLookupExpr {{.*}} 'invalid'
-  Bar a5{invalid()};
 
   // CHECK:     `-VarDecl {{.*}} b1 'Bar'
   // CHECK-NEXT: `-RecoveryExpr {{.*}} contains-errors
@@ -231,52 +178,12 @@ void InvalidInitalizer(int x) {
   // CHECK-NEXT:    `-InitListExpr {{.*}} 'void'
   // CHECK-NEXT:      `-DeclRefExpr {{.*}} 'x' 'int'
   Bar b4 = Bar{x};
-  // CHECK:     `-VarDecl {{.*}} b5 'Bar'
-  // CHECK-NEXT: `-CXXUnresolvedConstructExpr {{.*}} 'Bar' contains-errors 'Bar'
-  // CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
-  // CHECK-NEXT:     `-UnresolvedLookupExpr {{.*}} 'invalid'
-  Bar b5 = Bar(invalid());
-  // CHECK:     `-VarDecl {{.*}} b6 'Bar'
-  // CHECK-NEXT: `-CXXUnresolvedConstructExpr {{.*}} 'Bar' contains-errors 'Bar'
-  // CHECK-NEXT:  `-InitListExpr {{.*}} contains-errors
-  // CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
-  // CHECK-NEXT:     `-UnresolvedLookupExpr {{.*}} 'invalid'
-  Bar b6 = Bar{invalid()};
 
   // CHECK:     RecoveryExpr {{.*}} 'Bar' contains-errors
   // CHECK-NEXT:  `-IntegerLiteral {{.*}} 'int' 1
   Bar(1);
-
-  // CHECK:     `-VarDecl {{.*}} var1
-  // CHECK-NEXT: `-BinaryOperator {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   `-IntegerLiteral {{.*}} 'int' 1
-  int var1 = undef + 1;
-}
-void InitializerForAuto() {
-  // CHECK:     `-VarDecl {{.*}} invalid a 'auto'
-  // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   `-UnresolvedLookupExpr {{.*}} 'invalid'
-  auto a = invalid();
-
-  // CHECK:     `-VarDecl {{.*}} invalid b 'auto'
-  // CHECK-NEXT: `-CallExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   |-UnresolvedLookupExpr {{.*}} 'some_func'
-  // CHECK-NEXT:   `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:     `-UnresolvedLookupExpr {{.*}} 'invalid'
-  auto b = some_func(invalid());
-
-  decltype(ned);
-  // very bad initailizer: there is an unresolved typo expr internally, we just
-  // drop it.
-  // CHECK: `-VarDecl {{.*}} invalid unresolved_typo 'auto'
-  auto unresolved_typo = gned.*[] {};
 }
 
-// Verified that the generated call operator is invalid.
-// CHECK: |-CXXMethodDecl {{.*}} invalid operator() 'auto () const -> auto'
-using Escape = decltype([] { return undef(); }());
-
 // CHECK:      VarDecl {{.*}} NoCrashOnInvalidInitList
 // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>' contains-errors lvalue
 // CHECK-NEXT:   `-InitListExpr
@@ -301,56 +208,8 @@ void ValueCategory() {
   xvalue(); // call to a function (rvalue reference return type) yields an xvalue.
 }
 
-void InvalidCondition() {
-  // CHECK:      IfStmt {{.*}}
-  // CHECK-NEXT: |-RecoveryExpr {{.*}} <col:7, col:15> '<dependent type>' contains-errors
-  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}} <col:7>
-  if (invalid()) {}
-
-  // CHECK:      WhileStmt {{.*}}
-  // CHECK-NEXT: |-RecoveryExpr {{.*}} <col:10, col:18> '<dependent type>' contains-errors
-  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}} <col:10>
-  while (invalid()) {}
-
-  // CHECK:      SwitchStmt {{.*}}
-  // CHECK-NEXT: |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}} <col:10>
-  switch(invalid()) {
-    case 1:
-      break;
-  }
-  // FIXME: figure out why the type of ConditionalOperator is not int.
-  // CHECK:      ConditionalOperator {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT: |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}}
-  // CHECK-NEXT: |-IntegerLiteral {{.*}} 'int' 1
-  // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 2
-  invalid() ? 1 : 2;
-}
-
 void CtorInitializer() {
   struct S{int m};
-  class MemberInit {
-    int x, y, z;
-    S s;
-    MemberInit() : x(invalid), y(invalid, invalid), z(invalid()), s(1,2) {}
-    // CHECK:      CXXConstructorDecl {{.*}} MemberInit 'void ()'
-    // CHECK-NEXT: |-CXXCtorInitializer Field {{.*}} 'x' 'int'
-    // CHECK-NEXT: | `-ParenListExpr
-    // CHECK-NEXT: |   `-RecoveryExpr {{.*}} '<dependent type>'
-    // CHECK-NEXT: |-CXXCtorInitializer Field {{.*}} 'y' 'int'
-    // CHECK-NEXT: | `-ParenListExpr
-    // CHECK-NEXT: |   |-RecoveryExpr {{.*}} '<dependent type>'
-    // CHECK-NEXT: |   `-RecoveryExpr {{.*}} '<dependent type>'
-    // CHECK-NEXT: |-CXXCtorInitializer Field {{.*}} 'z' 'int'
-    // CHECK-NEXT: | `-ParenListExpr
-    // CHECK-NEXT: |   `-RecoveryExpr {{.*}} '<dependent type>'
-    // CHECK-NEXT: |     `-UnresolvedLookupExpr {{.*}} '<overloaded function type>'
-    // CHECK-NEXT: |-CXXCtorInitializer Field {{.*}} 's' 'S'
-    // CHECK-NEXT: | `-RecoveryExpr {{.*}} 'S' contains-errors
-    // CHECK-NEXT: |   |-IntegerLiteral {{.*}} 1
-    // CHECK-NEXT: |   `-IntegerLiteral {{.*}} 2
-  };
   class BaseInit : S {
     BaseInit(float) : S("no match") {}
     // CHECK:      CXXConstructorDecl {{.*}} BaseInit 'void (float)'
@@ -358,13 +217,6 @@ void CtorInitializer() {
     // CHECK-NEXT: |-CXXCtorInitializer 'S'
     // CHECK-NEXT: | `-RecoveryExpr {{.*}} 'S'
     // CHECK-NEXT: |   `-StringLiteral
-
-    BaseInit(double) : S(invalid) {}
-    // CHECK:      CXXConstructorDecl {{.*}} BaseInit 'void (double)'
-    // CHECK-NEXT: |-ParmVarDecl
-    // CHECK-NEXT: |-CXXCtorInitializer 'S'
-    // CHECK-NEXT: | `-ParenListExpr
-    // CHECK-NEXT: |   `-RecoveryExpr {{.*}} '<dependent type>'
   };
   class DelegatingInit {
     DelegatingInit(float) : DelegatingInit("no match") {}
@@ -373,13 +225,6 @@ void CtorInitializer() {
     // CHECK-NEXT: |-CXXCtorInitializer 'DelegatingInit'
     // CHECK-NEXT: | `-RecoveryExpr {{.*}} 'DelegatingInit'
     // CHECK-NEXT: |   `-StringLiteral
-
-    DelegatingInit(double) : DelegatingInit(invalid) {}
-    // CHECK:      CXXConstructorDecl {{.*}} DelegatingInit 'void (double)'
-    // CHECK-NEXT: |-ParmVarDecl
-    // CHECK-NEXT: |-CXXCtorInitializer 'DelegatingInit'
-    // CHECK-NEXT: | `-ParenListExpr
-    // CHECK-NEXT: |   `-RecoveryExpr {{.*}} '<dependent type>'
   };
 }
 
@@ -423,65 +268,6 @@ void returnInitListFromVoid() {
   // CHECK-NEXT:   `-IntegerLiteral {{.*}} 'int' 8
 }
 
-void RecoveryExprForInvalidDecls(Unknown InvalidDecl) {
-  InvalidDecl + 1;
-  // CHECK:      BinaryOperator {{.*}}
-  // CHECK-NEXT: |-RecoveryExpr {{.*}} '<dependent type>'
-  // CHECK-NEXT: | | `-DeclRefExpr {{.*}} 'InvalidDecl' 'int'
-  // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 1
-  InvalidDecl();
-  // CHECK:      CallExpr {{.*}}
-  // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>'
-}
-
-void InitializerOfInvalidDecl() {
-  int ValidDecl;
-  Unkown InvalidDecl = ValidDecl;
-  // CHECK:      VarDecl {{.*}} invalid InvalidDecl
-  // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   `-DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'ValidDecl'
-
-  Unknown InvalidDeclWithInvalidInit = Invalid;
-  // CHECK:      VarDecl {{.*}} invalid InvalidDeclWithInvalidInit
-  // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NOT:    `-TypoExpr
-}
-
-void RecoverToAnInvalidDecl() {
-  Unknown* foo; // invalid decl
-  goo; // the typo was correct to the invalid foo.
-  // Verify that RecoveryExpr has an inner DeclRefExpr.
-  // CHECK:      RecoveryExpr {{.*}} '<dependent type>' contains-errors lvalue
-  // CHECK-NEXT: `-DeclRefExpr {{.*}} 'foo' 'int *'
-}
-
-void RecoveryToDoWhileStmtCond() {
-  // CHECK:       FunctionDecl {{.*}} RecoveryToDoWhileStmtCond
-  // CHECK:       `-DoStmt {{.*}}
-  // CHECK-NEXT:    |-CompoundStmt {{.*}}
-  // CHECK-NEXT:    `-BinaryOperator {{.*}} '<dependent type>' contains-errors '<'
-  // CHECK-NEXT:      |-BinaryOperator {{.*}} '<dependent type>' contains-errors '+'
-  // CHECK-NEXT:      | |-RecoveryExpr {{.*}} '<dependent type>' contains-errors lvalue
-  // CHECK-NEXT:      | `-IntegerLiteral {{.*}} 'int' 1
-  // CHECK-NEXT:      `-IntegerLiteral {{.*}} 'int' 10
-  do {} while (some_invalid_val + 1 < 10);
-}
-
-void RecoveryForStmtCond() {
-  // CHECK:FunctionDecl {{.*}} RecoveryForStmtCond
-  // CHECK-NEXT:`-CompoundStmt {{.*}}
-  // CHECK-NEXT:  `-ForStmt {{.*}}
-  // CHECK-NEXT:    |-DeclStmt {{.*}}
-  // CHECK-NEXT:    | `-VarDecl {{.*}}
-  // CHECK-NEXT:    |   `-IntegerLiteral {{.*}} <col:16> 'int' 0
-  // CHECK-NEXT:    |-<<<NULL>>>
-  // CHECK-NEXT:    |-RecoveryExpr {{.*}} 'bool' contains-errors
-  // CHECK-NEXT:    |-UnaryOperator {{.*}} 'int' lvalue prefix '++'
-  // CHECK-NEXT:    | `-DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'i' 'int'
-  // CHECK-NEXT:    `-CompoundStmt {{.*}}
-  for (int i = 0; i < invalid; ++i) {}
-}
-
 // Fix crash issue https://github.com/llvm/llvm-project/issues/112560.
 // Make sure clang compiles the following code without crashing:
 
diff --git a/external/llvm-project/clang/test/AST/ast-dump-recovery.m b/external/llvm-project/clang/test/AST/ast-dump-recovery.m
deleted file mode 100644
index 37fa8045c0b9..000000000000
--- a/external/llvm-project/clang/test/AST/ast-dump-recovery.m
+++ /dev/null
@@ -1,32 +0,0 @@
-// RUN: not %clang_cc1 -triple x86_64-unknown-unknown -frecovery-ast -frecovery-ast-type -fblocks -ast-dump %s | FileCheck -strict-whitespace %s
-
-@interface Foo
-- (void)method:(int)n;
-@end
-
-void k(Foo *foo) {
-  // CHECK:       ObjCMessageExpr {{.*}} 'void' contains-errors
-  // CHECK-CHECK:  |-ImplicitCastExpr {{.*}} 'Foo *' <LValueToRValue>
-  // CHECK-CHECK:  | `-DeclRefExpr {{.*}} 'foo'
-  // CHECK-CHECK:  `-RecoveryExpr {{.*}}
-  [foo method:undef];
-
-  // CHECK:      ImplicitCastExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
-  // CHECK-NEXT:   `-DeclRefExpr {{.*}} 'foo'
-  foo.undef;
-}
-
-// CHECK:      |-VarDecl {{.*}} 'int (^)()' cinit
-// CHECK-NEXT: | `-RecoveryExpr {{.*}} '<dependent type> (^)(void)' contains-errors lvalue
-// CHECK-NEXT: |   `-BlockExpr {{.*}} '<dependent type> (^)(void)'
-// CHECK-NEXT: |     `-BlockDecl {{.*}} invalid
-int (^gh63863)() = ^() {
-  return undef;
-};
-
-// CHECK:      `-BlockExpr {{.*}} 'int (^)(int, int)'
-// CHECK-NEXT:   `-BlockDecl {{.*}} invalid
-int (^gh64005)(int, int) = ^(int, undefined b) {
-   return 1;
-};
diff --git a/external/llvm-project/clang/test/AST/ast-dump-type-callingconv.cpp b/external/llvm-project/clang/test/AST/ast-dump-type-callingconv.cpp
new file mode 100644
index 000000000000..1611223acbc8
--- /dev/null
+++ b/external/llvm-project/clang/test/AST/ast-dump-type-callingconv.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -triple aarch64 -ast-dump -ast-dump-filter foo %s \
+// RUN: | FileCheck --strict-whitespace %s
+
+// CHECK: foo1 'void () __attribute__((device_kernel))'{{$}}
+void foo1() __attribute__((device_kernel));
+
+// CHECK: foo2 'void () __attribute__((aarch64_vector_pcs))'{{$}}
+void foo2()  __attribute__((aarch64_vector_pcs));
+
+// CHECK: foo3 'void () __attribute__((aarch64_sve_pcs))'{{$}}
+void foo3()  __attribute__((aarch64_sve_pcs));
diff --git a/external/llvm-project/clang/test/AST/attr-target-version.c b/external/llvm-project/clang/test/AST/attr-target-version.c
index 52ac0e61b5a5..b537f5e685a3 100644
--- a/external/llvm-project/clang/test/AST/attr-target-version.c
+++ b/external/llvm-project/clang/test/AST/attr-target-version.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -ast-dump %s | FileCheck %s
 
 int __attribute__((target_version("sve2-bitperm + sha2"))) foov(void) { return 1; }
-int __attribute__((target_clones(" lse + fp + sha3 "))) fooc(void) { return 2; }
+int __attribute__((target_clones(" lse + fp + sha3 ", "default"))) fooc(void) { return 2; }
 // CHECK: TargetVersionAttr
 // CHECK: sve2-bitperm + sha2
 // CHECK: TargetClonesAttr
diff --git a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp
index e24b04dcd3cf..b2fb042e7def 100644
--- a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp
+++ b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-checked.cpp
@@ -2,6 +2,20 @@
 
 #include "mock-types.h"
 
+namespace std {
+
+template <typename T> struct remove_reference {
+  typedef T type;
+};
+
+template <typename T> struct remove_reference<T&> {
+  typedef T type;
+};
+
+template<typename T> typename remove_reference<T>::type&& move(T&& t);
+
+} // namespace std
+
 RefCountableAndCheckable* makeObj();
 CheckedRef<RefCountableAndCheckable> makeObjChecked();
 void someFunction(RefCountableAndCheckable*);
@@ -32,6 +46,26 @@ static void baz() {
 
 } // namespace call_args_checked
 
+namespace call_args_member {
+
+void consume(CheckedObj&);
+
+struct WrapperObj {
+  CheckedObj checked;
+  CheckedObj& checkedRef;
+  void foo() {
+    consume(checked);
+    consume(checkedRef);
+    // expected-warning@-1{{Call argument is unchecked and unsafe [alpha.webkit.UncheckedCallArgsChecker]}}
+  }
+  void bar(WrapperObj& other) {
+    consume(other.checked);
+    // expected-warning@-1{{Call argument is unchecked and unsafe [alpha.webkit.UncheckedCallArgsChecker]}}
+  }
+};
+
+} // namespace call_args_checked
+
 namespace call_args_default {
 
 void someFunction(RefCountableAndCheckable* = makeObj());
@@ -54,3 +88,12 @@ void foo() {
 }
 
 }
+
+namespace call_with_std_move {
+
+void consume(CheckedObj&&);
+void foo(CheckedObj&& obj) {
+  consume(std::move(obj));
+}
+
+}
diff --git a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.cpp b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.cpp
index a87446564870..5c540a58deba 100644
--- a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.cpp
+++ b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.cpp
@@ -1,10 +1,12 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
-// expected-no-diagnostics
+
+#include "mock-types.h"
 
 class Base {
 public:
-    inline void ref();
-    inline void deref();
+    void ref();
+    void deref();
+    void doWork();
 };
 
 class Derived : public Base {
@@ -21,6 +23,7 @@ class SubDerived final : public Derived {
 class OtherObject {
 public:
     Derived* obj();
+    Base* base();
 };
 
 class String {
@@ -44,6 +47,12 @@ inline Target* uncheckedDowncast(Source* source)
     return static_cast<Target*>(source);
 }
 
+template<typename Target, typename Source>
+Target* [[clang::annotate_type("webkit.pointerconversion")]] newCastFunction(Source*);
+
+template<typename Target, typename Source>
+Target* [[clang::annotate_type("unrelated-annotation")]] badCastFunction(Source*);
+
 template<typename... Types>
 String toString(const Types&... values);
 
@@ -52,5 +61,17 @@ void foo(OtherObject* other)
     dynamicDowncast<SubDerived>(other->obj());
     checkedDowncast<SubDerived>(other->obj());
     uncheckedDowncast<SubDerived>(other->obj());
+    newCastFunction<SubDerived>(other->obj());
+    badCastFunction<SubDerived>(other->obj());
+    // expected-warning@-1{{Call argument is uncounted and unsafe}}
     toString(other->obj());
 }
+
+struct SomeStruct {
+  Derived* [[clang::annotate_type("webkit.pointerconversion")]] ptrConversion(Base*);
+
+  void foo(OtherObject& otherObj) {
+    RefPtr ptr = otherObj.base();
+    ptrConversion(ptr.get())->doWork();
+  }
+};
diff --git a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
index 93e7dfd77b9e..9e4356a71f1b 100644
--- a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
+++ b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
@@ -22,6 +22,7 @@ typedef struct CF_BRIDGED_TYPE(id) CGImage *CGImageRef;
 #define NS_RETURNS_RETAINED __attribute__((ns_returns_retained))
 #define CF_CONSUMED __attribute__((cf_consumed))
 #define CF_RETURNS_RETAINED __attribute__((cf_returns_retained))
+#define NS_REQUIRES_PROPERTY_DEFINITIONS __attribute__((objc_requires_property_definitions))
 
 extern const CFAllocatorRef kCFAllocatorDefault;
 typedef struct _NSZone NSZone;
diff --git a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unchecked-local-vars.cpp b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unchecked-local-vars.cpp
index 3bc75230fcf8..2984f8ba4eef 100644
--- a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unchecked-local-vars.cpp
+++ b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unchecked-local-vars.cpp
@@ -290,6 +290,28 @@ void foo() {
 
 } // namespace local_assignment_to_global
 
+namespace member_var {
+
+  struct WrapperObj {
+    CheckedObj checked;
+    CheckedObj& checkedRef;
+    void foo() {
+      auto *a = &checked;
+      a->method();
+      auto *b = &checkedRef;
+      // expected-warning@-1{{Local variable 'b' is unchecked and unsafe [alpha.webkit.UncheckedLocalVarsChecker]}}
+      b->method();
+    }
+
+    void bar(WrapperObj& wrapper) {
+      CheckedObj* ptr = &wrapper.checked;
+      // expected-warning@-1{{Local variable 'ptr' is unchecked and unsafe [alpha.webkit.UncheckedLocalVarsChecker]}}
+      ptr->method();
+    }
+  };
+
+}
+
 namespace local_refcountable_checkable_object {
 
 RefCountableAndCheckable* provide_obj();
diff --git a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members-arc.mm b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members-arc.mm
index 3491bc93ed98..00e6e6ec1dcf 100644
--- a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members-arc.mm
+++ b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members-arc.mm
@@ -64,3 +64,39 @@ void forceTmplToInstantiate(FooTmpl<SomeObj, CFMutableArrayRef>) {}
   };
 
 } // namespace ptr_to_ptr_to_retained
+
+@interface AnotherObject : NSObject {
+  NSString *ns_string;
+  CFStringRef cf_string;
+  // expected-warning@-1{{Instance variable 'cf_string' in 'AnotherObject' is a retainable type 'CFStringRef'; member variables must be a RetainPtr}}
+}
+@property(nonatomic, strong) NSString *prop_string1;
+@property(nonatomic, assign) NSString *prop_string2;
+// expected-warning@-1{{Property 'prop_string2' in 'AnotherObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
+@property(nonatomic, unsafe_unretained) NSString *prop_string3;
+// expected-warning@-1{{Property 'prop_string3' in 'AnotherObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
+@property(nonatomic, readonly) NSString *prop_string4;
+@end
+
+NS_REQUIRES_PROPERTY_DEFINITIONS
+@interface NoSynthObject : NSObject {
+  NSString *ns_string;
+  CFStringRef cf_string;
+  // expected-warning@-1{{Instance variable 'cf_string' in 'NoSynthObject' is a retainable type 'CFStringRef'; member variables must be a RetainPtr}}
+}
+@property(nonatomic, readonly, strong) NSString *prop_string1;
+@property(nonatomic, readonly, strong) NSString *prop_string2;
+@property(nonatomic, assign) NSString *prop_string3;
+// expected-warning@-1{{Property 'prop_string3' in 'NoSynthObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
+@property(nonatomic, unsafe_unretained) NSString *prop_string4;
+// expected-warning@-1{{Property 'prop_string4' in 'NoSynthObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
+@end
+
+@implementation NoSynthObject
+- (NSString *)prop_string1 {
+  return nil;
+}
+@synthesize prop_string2;
+@synthesize prop_string3;
+@synthesize prop_string4;
+@end
diff --git a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members.mm b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members.mm
index 0cb4c4ac0f6a..46f65dfa603a 100644
--- a/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members.mm
+++ b/external/llvm-project/clang/test/Analysis/Checkers/WebKit/unretained-members.mm
@@ -99,3 +99,28 @@ @interface AnotherObject : NSObject {
 @property(nonatomic, strong) NSString *prop_string;
 // expected-warning@-1{{Property 'prop_string' in 'AnotherObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
 @end
+
+NS_REQUIRES_PROPERTY_DEFINITIONS
+@interface NoSynthObject : NSObject {
+  NSString *ns_string;
+  // expected-warning@-1{{Instance variable 'ns_string' in 'NoSynthObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
+  CFStringRef cf_string;
+  // expected-warning@-1{{Instance variable 'cf_string' in 'NoSynthObject' is a retainable type 'CFStringRef'; member variables must be a RetainPtr}}
+}
+@property(nonatomic, readonly, strong) NSString *prop_string1;
+@property(nonatomic, readonly, strong) NSString *prop_string2;
+// expected-warning@-1{{Property 'prop_string2' in 'NoSynthObject' is a raw pointer to retainable type 'NSString'}}
+@property(nonatomic, assign) NSString *prop_string3;
+// expected-warning@-1{{Property 'prop_string3' in 'NoSynthObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
+@property(nonatomic, unsafe_unretained) NSString *prop_string4;
+// expected-warning@-1{{Property 'prop_string4' in 'NoSynthObject' is a raw pointer to retainable type 'NSString'; member variables must be a RetainPtr}}
+@end
+
+@implementation NoSynthObject
+- (NSString *)prop_string1 {
+  return nil;
+}
+@synthesize prop_string2;
+@synthesize prop_string3;
+@synthesize prop_string4;
+@end
diff --git a/external/llvm-project/clang/test/Analysis/bitint-z3.c b/external/llvm-project/clang/test/Analysis/bitint-z3.c
new file mode 100644
index 000000000000..4cb97f9de829
--- /dev/null
+++ b/external/llvm-project/clang/test/Analysis/bitint-z3.c
@@ -0,0 +1,22 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -w \
+// RUN:   -analyzer-config crosscheck-with-z3=true -verify %s
+// REQUIRES: z3
+
+// Previously these tests were crashing because the SMTConv layer did not
+// comprehend the _BitInt types.
+
+void clang_analyzer_warnIfReached();
+
+void c(int b, _BitInt(35) a) {
+  int d = 0;
+  if (a)
+    b = d;
+  clang_analyzer_warnIfReached(); // expected-warning{{REACHABLE}}
+}
+
+void f(int *d, _BitInt(3) e) {
+  int g;
+  d = &g;
+  e ?: 0;
+  clang_analyzer_warnIfReached(); // expected-warning{{REACHABLE}}
+}
diff --git a/external/llvm-project/clang/test/Analysis/pch_crash.cpp b/external/llvm-project/clang/test/Analysis/pch_crash.cpp
deleted file mode 100644
index 7ad2cb2d2ab5..000000000000
--- a/external/llvm-project/clang/test/Analysis/pch_crash.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.15.0 -emit-pch -o %t %s
-// RUN: %clang_analyze_cc1 -triple x86_64-apple-macosx10.15.0 -include-pch %t \
-// RUN:   -analyzer-checker=core,apiModeling -verify %s
-//
-// RUN: %clang_cc1 -emit-pch -o %t %s
-// RUN: %clang_analyze_cc1 -include-pch %t \
-// RUN:   -analyzer-checker=core,apiModeling -verify %s
-
-// expected-no-diagnostics
-
-#ifndef HEADER
-#define HEADER
-// Pre-compiled header
-
-int foo();
-
-// Literal data for this macro value will be null
-#define EOF -1
-
-#else
-// Source file
-
-int test() {
-  // we need a function call here to initiate erroneous routine
-  return foo(); // no-crash
-}
-
-#endif
diff --git a/external/llvm-project/clang/test/Analysis/pch_macro.cpp b/external/llvm-project/clang/test/Analysis/pch_macro.cpp
new file mode 100644
index 000000000000..afc371d50460
--- /dev/null
+++ b/external/llvm-project/clang/test/Analysis/pch_macro.cpp
@@ -0,0 +1,50 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// DEFINE: %{emit-pch-action} = \
+// DEFINE:   %clang_cc1 -x c++ -emit-pch -o %t/header.pch %t/header.h
+
+// DEFINE: %{analyze-action} = \
+// DEFINE:   %clang_analyze_cc1 -include-pch %t/header.pch \
+// DEFINE:   -analyzer-checker=core,apiModeling,unix.StdCLibraryFunctions \
+// DEFINE:   -verify %t/main.cpp
+
+// RUN: %{emit-pch-action} -triple x86_64-apple-macosx10.15.0
+// RUN: %{analyze-action}  -triple x86_64-apple-macosx10.15.0
+// RUN: %{emit-pch-action}
+// RUN: %{analyze-action}
+
+
+//--- header.h
+
+
+// Pre-compiled header
+
+int foo();
+
+// Literal data for macro values will be null as they are defined in a PCH
+#define EOF -1
+#define AT_FDCWD -2
+
+
+//--- main.cpp
+
+
+// Source file
+// expected-no-diagnostics
+int test() {
+  // we need a function call here to initiate erroneous routine
+  return foo(); // no-crash
+}
+
+// Test that StdLibraryFunctionsChecker can obtain the definition of
+// AT_FDCWD even if it is from a PCH:
+int faccessat(int, const char *, int, int);
+
+void test_faccessat() {
+  char fileSystemPath[10] = { 0 };
+
+  if (0 != faccessat(AT_FDCWD, fileSystemPath, 2, 0x0030)) {}
+}
+
diff --git a/external/llvm-project/clang/test/C/C23/n3006.c b/external/llvm-project/clang/test/C/C23/n3006.c
new file mode 100644
index 000000000000..e0713fa9969c
--- /dev/null
+++ b/external/llvm-project/clang/test/C/C23/n3006.c
@@ -0,0 +1,113 @@
+// RUN: %clang_cc1 -std=c23 -verify %s
+
+/* WG14 N3006: Yes
+ * Underspecified object declarations
+ */
+
+void struct_test(void) {
+  struct S1 { int x, y; };                                                  // expected-note {{field 'x' has type 'int' here}}
+
+  auto normal_struct = (struct S1){ 1, 2 };
+  auto normal_struct2 = (struct S1) { .x = 1, .y = 2 };
+  auto underspecified_struct = (struct S2 { int x, y; }){ 1, 2 };
+  auto underspecified_struct_redef = (struct S1 { char x, y; }){ 'A', 'B'}; // expected-error {{type 'struct S1' has incompatible definitions}} \
+                                                                               expected-error {{cannot use 'auto' with array in C}} \
+                                                                               expected-note {{field 'x' has type 'char' here}}
+  auto underspecified_empty_struct = (struct S3 { }){ };
+  auto zero_init_struct = (struct S4 { int x; }){ 0 };
+  int field_struct = (struct S5 { int y; }){ 0 }.y;
+}
+
+void union_test(void) {
+  union U1 { int a; double b; };                                                  // expected-note {{field 'a' has type 'int' here}}
+
+  auto normal_union_int = (union U1){ .a = 12 };
+  auto normal_union_double = (union U1){ .b = 2.4 };
+  auto underspecified_union = (union U2 { int a; double b; }){ .a = 34 };
+  auto underspecified_union_redef = (union U1 { char a; double b; }){ .a = 'A' }; // expected-error {{type 'union U1' has incompatible definitions}} \
+                                                                                     expected-error {{cannot use 'auto' with array in C}} \
+                                                                                     expected-note {{field 'a' has type 'char' here}}
+  auto underspecified_empty_union = (union U3 {  }){  };
+}
+
+void enum_test(void) {
+  enum E1 { FOO, BAR };                                           // expected-note {{enumerator 'BAR' with value 1 here}}
+
+  auto normal_enum_foo = (enum E1){ FOO };
+  auto normal_enum_bar = (enum E1){ BAR };
+  auto underspecified_enum = (enum E2 { BAZ, QUX }){ BAZ };
+  auto underspecified_enum_redef = (enum E1 { ONE, TWO }){ ONE }; // expected-error {{type 'enum E1' has incompatible definitions}} \
+                                                                     expected-error {{cannot use 'auto' with array in C}} \
+                                                                     expected-note {{enumerator 'ONE' with value 0 here}}
+  auto underspecified_empty_enum = (enum E3 {  }){ };             // expected-error {{use of empty enum}}
+  auto underspecified_undeclared_enum = (enum E4){ FOO };         // expected-error {{variable has incomplete type 'enum E4'}} \
+                                                                     expected-note {{forward declaration of 'enum E4'}}
+}
+
+void constexpr_test(void) {
+  constexpr auto ce_struct = (struct S1){ 1, 2 };                   // expected-error {{variable has incomplete type 'struct S1'}} \
+                                                                       expected-note {{forward declaration of 'struct S1'}}
+  constexpr auto ce_struct_zero_init = (struct S2 { int x; }){ 0 };
+  constexpr int ce_struct_field = (struct S3 { int y; }){ 0 }.y;
+  constexpr auto ce_union = (union U1){ .a = 12 };                  // expected-error {{variable has incomplete type 'union U1'}} \
+                                                                       expected-note {{forward declaration of 'union U1'}}
+
+  constexpr auto ce_enum = (enum E1 { BAZ, QUX }){ BAZ };
+  constexpr auto ce_empty_enum = (enum E2){ FOO };                  // expected-error {{use of undeclared identifier 'FOO'}}
+}
+
+void self_reference_test(void) {
+  constexpr int i = i;  // expected-error {{constexpr variable 'i' must be initialized by a constant expression}} \
+                           expected-note {{read of object outside its lifetime is not allowed in a constant expression}}
+  auto j = j;           // expected-error {{variable 'j' declared with deduced type 'auto' cannot appear in its own initializer}}
+}
+
+void redefinition_test(void) {
+  const struct S { int x; } s;  // expected-warning {{default initialization of an object of type 'const struct S' leaves the object uninitialized}} \
+                                   expected-note {{previous definition is here}}
+  constexpr struct S s = {0};   // expected-error {{redefinition of 's'}}
+}
+
+void declaring_an_underspecified_defied_object_test(void) {
+  struct S { int x, y; };
+  constexpr int i = (struct T { int a, b; }){0, 1}.a;
+
+  struct T t = { 1, 2 };
+}
+
+void constexpr_complience_test(void) {
+  int x = (struct Foo { int x; }){ 0 }.x;
+  constexpr int y = (struct Bar { int x; }){ 0 }.x;
+}
+
+void builtin_functions_test(void) {
+  constexpr typeof(struct s *) x = 0;
+  auto so = sizeof(struct S {});
+  auto to = (typeof(struct S {})){};
+}
+
+void misc_test(void) {
+  constexpr struct S { int a, b; } y = { 0 };
+  constexpr int a = 0, b = 0;
+  auto c = (struct T { int x, y; }){0, 0};
+  auto s2 = ({struct T { int x; } s = {}; s.x; });
+  auto s3 = ((struct {}){},0); // expected-warning {{left operand of comma operator has no effect}}
+  constexpr int (*fp)(struct X { int x; } val) = 0;
+  auto v = (void (*)(int y))0;
+}
+
+void misc_struct_test(void) {
+  constexpr struct {
+      int a;
+  } a = {};
+
+  constexpr struct {
+      int b;
+  } b = (struct S { int x; }){ 0 };  // expected-error-re {{initializing 'const struct (unnamed struct at {{.*}}n3006.c:104:13)' with an expression of incompatible type 'struct S'}}
+
+  auto z = ({
+      int a = 12;
+      struct {} s;
+      a;
+  });
+}
diff --git a/external/llvm-project/clang/test/C/C2y/n3353.c b/external/llvm-project/clang/test/C/C2y/n3353.c
index a616228f1bad..cd61cbf03906 100644
--- a/external/llvm-project/clang/test/C/C2y/n3353.c
+++ b/external/llvm-project/clang/test/C/C2y/n3353.c
@@ -92,6 +92,37 @@ int o2 = 0xG; /* expected-error {{invalid suffix 'xG' on integer constant}}
                  c2y-warning {{octal literals without a '0o' prefix are deprecated}}
                */
 
+// Show that floating-point suffixes on octal literals are rejected.
+auto f1 = 0o0.;  /* expected-error {{invalid suffix '.' on integer constant}}
+                    compat-warning {{octal integer literals are incompatible with standards before C2y}}
+                    ext-warning {{octal integer literals are a C2y extension}}
+                    cpp-warning {{octal integer literals are a Clang extension}}
+                */
+auto f2 = 0o0.1; /* expected-error {{invalid suffix '.1' on integer constant}}
+                    compat-warning {{octal integer literals are incompatible with standards before C2y}}
+                    ext-warning {{octal integer literals are a C2y extension}}
+                    cpp-warning {{octal integer literals are a Clang extension}}
+                */
+auto f3 = 0o0e1; /* expected-error {{invalid suffix 'e1' on integer constant}}
+                    compat-warning {{octal integer literals are incompatible with standards before C2y}}
+                    ext-warning {{octal integer literals are a C2y extension}}
+                    cpp-warning {{octal integer literals are a Clang extension}}
+                 */
+auto f4 = 0o0E1; /* expected-error {{invalid suffix 'E1' on integer constant}}
+                    compat-warning {{octal integer literals are incompatible with standards before C2y}}
+                    ext-warning {{octal integer literals are a C2y extension}}
+                    cpp-warning {{octal integer literals are a Clang extension}}
+                 */
+
+// Show that valid floating-point literals with a leading 0 do not produce octal-related warnings.
+auto f5 = 0.;
+auto f7 = 00.;
+auto f8 = 01.;
+auto f9 = 0e1;
+auto f10 = 0E1;
+auto f11 = 00e1;
+auto f12 = 00E1;
+
 // Ensure digit separators work as expected.
 constexpr int p = 0o0'1'2'3'4'5'6'7; /* compat-warning {{octal integer literals are incompatible with standards before C2y}}
                                         ext-warning {{octal integer literals are a C2y extension}}
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/array.cpp b/external/llvm-project/clang/test/CIR/CodeGen/array.cpp
index 7b90c1682ec4..26e172a00645 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/array.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/array.cpp
@@ -473,3 +473,26 @@ void func10(int *a) {
 // OGCG:  %[[ELE:.*]] = getelementptr inbounds i32, ptr %[[TMP_1]], i64 5
 // OGCG:  %[[TMP_2:.*]] = load i32, ptr %[[ELE]], align 4
 // OGCG:  store i32 %[[TMP_2]], ptr %[[INIT]], align 4
+
+void func11() { int _Complex a[4]; }
+
+// CIR: %[[ARR:.*]] = cir.alloca !cir.array<!cir.complex<!s32i> x 4>, !cir.ptr<!cir.array<!cir.complex<!s32i> x 4>>, ["a"]
+
+// LLVM: %[[ARR:.*]] = alloca [4 x { i32, i32 }], i64 1, align 16
+
+// OGCG: %[[ARR:.*]] = alloca [4 x { i32, i32 }], align 16
+
+void func12() {
+  struct Point {
+    int x;
+    int y;
+  };
+
+  Point a[4];
+}
+
+// CIR: %[[ARR:.*]] = cir.alloca !cir.array<!rec_Point x 4>, !cir.ptr<!cir.array<!rec_Point x 4>>, ["a"]
+
+// LLVM: %[[ARR:.*]] = alloca [4 x %struct.Point], i64 1, align 16
+
+// OGCG: %[[ARR:.*]] = alloca [4 x %struct.Point], align 16
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/basic.c b/external/llvm-project/clang/test/CIR/CodeGen/basic.c
index abc1a45fd433..7ff73ee95f79 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/basic.c
+++ b/external/llvm-project/clang/test/CIR/CodeGen/basic.c
@@ -309,3 +309,17 @@ size_type max_size(void) {
 // CHECK:   %6 = cir.load{{.*}} %0 : !cir.ptr<!u64i>, !u64i
 // CHECK:   cir.return %6 : !u64i
 // CHECK:   }
+
+void test_char_literal() {
+  char c;
+  c = 'X';
+}
+
+// CIR: cir.func @test_char_literal
+// CIR:   cir.const #cir.int<88>
+
+// LLVM: define void @test_char_literal()
+// LLVM:   store i8 88, ptr %{{.*}}, align 1
+
+// OGCG: define{{.*}} void @test_char_literal()
+// OGCG:   store i8 88, ptr %{{.*}}, align 1
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/binop.c b/external/llvm-project/clang/test/CIR/CodeGen/binop.c
new file mode 100644
index 000000000000..280fd29b067f
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/binop.c
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+
+void conditionalResultIimplicitCast(int a, int b, float f) {
+  // Should implicit cast back to int.
+  int x = a && b;
+  // CHECK: %[[#INT:]] = cir.ternary
+  // CHECK: %{{.+}} = cir.cast(bool_to_int, %[[#INT]] : !cir.bool), !s32i
+  float y = f && f;
+  // CHECK: %[[#BOOL:]] = cir.ternary
+  // CHECK: %[[#INT:]] = cir.cast(bool_to_int, %[[#BOOL]] : !cir.bool), !s32i
+  // CHECK: %{{.+}} = cir.cast(int_to_float, %[[#INT]] : !s32i), !cir.float
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/binop.cpp b/external/llvm-project/clang/test/CIR/CodeGen/binop.cpp
index dbd17fb7ba83..c728f0d0c1bc 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/binop.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/binop.cpp
@@ -540,3 +540,235 @@ void long_shift_example(long long a, short b) {
 // OGCG:         store i64 %[[SHL]], ptr %[[X]]
 
 // OGCG:         ret void
+
+void b1(bool a, bool b) {
+  bool x = a && b;
+  x = x || b;
+}
+
+// CIR-LABEL: cir.func @_Z2b1bb(
+// CIR-SAME: %[[ARG0:.*]]: !cir.bool {{.*}}, %[[ARG1:.*]]: !cir.bool {{.*}})
+// CIR: [[A:%[0-9]+]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["a", init]
+// CIR: [[B:%[0-9]+]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b", init]
+// CIR: [[X:%[0-9]+]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init]
+// CIR: cir.store %[[ARG0]], [[A]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: cir.store %[[ARG1]], [[B]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: [[AVAL:%[0-9]+]] = cir.load align(1) [[A]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR: [[RES1:%[0-9]+]] = cir.ternary([[AVAL]], true {
+// CIR: [[BVAL:%[0-9]+]] = cir.load align(1) [[B]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR: cir.yield [[BVAL]] : !cir.bool
+// CIR: }, false {
+// CIR: [[FALSE:%[0-9]+]] = cir.const #false
+// CIR: cir.yield [[FALSE]] : !cir.bool
+// CIR: }) : (!cir.bool) -> !cir.bool
+// CIR: cir.store align(1) [[RES1]], [[X]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: [[XVAL:%[0-9]+]] = cir.load align(1) [[X]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR: [[RES2:%[0-9]+]] = cir.ternary([[XVAL]], true {
+// CIR: [[TRUE:%[0-9]+]] = cir.const #true
+// CIR: cir.yield [[TRUE]] : !cir.bool
+// CIR: }, false {
+// CIR: [[BVAL2:%[0-9]+]] = cir.load align(1) [[B]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR: cir.yield [[BVAL2]] : !cir.bool
+// CIR: }) : (!cir.bool) -> !cir.bool
+// CIR: cir.store align(1) [[RES2]], [[X]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: cir.return
+
+
+// LLVM-LABEL: define void @_Z2b1bb(
+// LLVM-SAME: i1 %[[ARG0:.+]], i1 %[[ARG1:.+]])
+// LLVM: %[[A_ADDR:.*]] = alloca i8
+// LLVM: %[[B_ADDR:.*]] = alloca i8
+// LLVM: %[[X:.*]] = alloca i8
+// LLVM: %[[ZEXT0:.*]] = zext i1 %[[ARG0]] to i8
+// LLVM: store i8 %[[ZEXT0]], ptr %[[A_ADDR]]
+// LLVM: %[[ZEXT1:.*]] = zext i1 %[[ARG1]] to i8
+// LLVM: store i8 %[[ZEXT1]], ptr %[[B_ADDR]]
+// LLVM: %[[A_VAL:.*]] = load i8, ptr %[[A_ADDR]]
+// LLVM: %[[A_BOOL:.*]] = trunc i8 %[[A_VAL]] to i1
+// LLVM: br i1 %[[A_BOOL]], label %[[AND_TRUE:.+]], label %[[AND_FALSE:.+]]
+// LLVM: [[AND_TRUE]]:
+// LLVM: %[[B_VAL:.*]] = load i8, ptr %[[B_ADDR]]
+// LLVM: %[[B_BOOL:.*]] = trunc i8 %[[B_VAL]] to i1
+// LLVM: br label %[[AND_MERGE:.+]]
+// LLVM: [[AND_FALSE]]:
+// LLVM: br label %[[AND_MERGE]]
+// LLVM: [[AND_MERGE]]:
+// LLVM: %[[AND_PHI:.*]] = phi i1 [ false, %[[AND_FALSE]] ], [ %[[B_BOOL]], %[[AND_TRUE]] ]
+// LLVM: %[[ZEXT_AND:.*]] = zext i1 %[[AND_PHI]] to i8
+// LLVM: store i8 %[[ZEXT_AND]], ptr %[[X]]
+// LLVM: %[[X_VAL:.*]] = load i8, ptr %[[X]]
+// LLVM: %[[X_BOOL:.*]] = trunc i8 %[[X_VAL]] to i1
+// LLVM: br i1 %[[X_BOOL]], label %[[OR_TRUE:.+]], label %[[OR_FALSE:.+]]
+// LLVM: [[OR_TRUE]]:
+// LLVM: br label %[[OR_MERGE:.+]]
+// LLVM: [[OR_FALSE]]:
+// LLVM: %[[B_VAL2:.*]] = load i8, ptr %[[B_ADDR]]
+// LLVM: %[[B_BOOL2:.*]] = trunc i8 %[[B_VAL2]] to i1
+// LLVM: br label %[[OR_MERGE]]
+// LLVM: [[OR_MERGE]]:
+// LLVM: %[[OR_PHI:.*]] = phi i1 [ %[[B_BOOL2]], %[[OR_FALSE]] ], [ true, %[[OR_TRUE]] ]
+// LLVM: %[[ZEXT_OR:.*]] = zext i1 %[[OR_PHI]] to i8
+// LLVM: store i8 %[[ZEXT_OR]], ptr %[[X]]
+// LLVM: ret void
+
+// OGCG-LABEL: define dso_local void @_Z2b1bb
+// OGCG-SAME: (i1 {{.*}} %[[ARG0:.+]], i1 {{.*}} %[[ARG1:.+]])
+// OGCG: [[ENTRY:.*]]:
+// OGCG: %[[A_ADDR:.*]] = alloca i8
+// OGCG: %[[B_ADDR:.*]] = alloca i8
+// OGCG: %[[X:.*]] = alloca i8
+// OGCG: %[[ZEXT0:.*]] = zext i1 %[[ARG0]] to i8
+// OGCG: store i8 %[[ZEXT0]], ptr %[[A_ADDR]]
+// OGCG: %[[ZEXT1:.*]] = zext i1 %[[ARG1]] to i8
+// OGCG: store i8 %[[ZEXT1]], ptr %[[B_ADDR]]
+// OGCG: %[[A_VAL:.*]] = load i8, ptr %[[A_ADDR]]
+// OGCG: %[[A_BOOL:.*]] = trunc i8 %[[A_VAL]] to i1
+// OGCG: br i1 %[[A_BOOL]], label %[[AND_TRUE:.+]], label %[[AND_MERGE:.+]]
+// OGCG: [[AND_TRUE]]:
+// OGCG: %[[B_VAL:.*]] = load i8, ptr %[[B_ADDR]]
+// OGCG: %[[B_BOOL:.*]] = trunc i8 %[[B_VAL]] to i1
+// OGCG: br label %[[AND_MERGE:.+]]
+// OGCG: [[AND_MERGE]]:
+// OGCG: %[[AND_PHI:.*]] = phi i1 [ false, %[[ENTRY]] ], [ %[[B_BOOL]], %[[AND_TRUE]] ]
+// OGCG: %[[ZEXT_AND:.*]] = zext i1 %[[AND_PHI]] to i8
+// OGCG: store i8 %[[ZEXT_AND]], ptr %[[X]]
+// OGCG: %[[X_VAL:.*]] = load i8, ptr %[[X]]
+// OGCG: %[[X_BOOL:.*]] = trunc i8 %[[X_VAL]] to i1
+// OGCG: br i1 %[[X_BOOL]], label %[[OR_MERGE:.+]], label %[[OR_FALSE:.+]]
+// OGCG: [[OR_FALSE]]:
+// OGCG: %[[B_VAL2:.*]] = load i8, ptr %[[B_ADDR]]
+// OGCG: %[[B_BOOL2:.*]] = trunc i8 %[[B_VAL2]] to i1
+// OGCG: br label %[[OR_MERGE]]
+// OGCG: [[OR_MERGE]]:
+// OGCG: %[[OR_PHI:.*]] = phi i1 [ true, %[[AND_MERGE]] ], [ %[[B_BOOL2]], %[[OR_FALSE]] ]
+// OGCG: %[[ZEXT_OR:.*]] = zext i1 %[[OR_PHI]] to i8
+// OGCG: store i8 %[[ZEXT_OR]], ptr %[[X]]
+// OGCG: ret void
+
+void b3(int a, int b, int c, int d) {
+  bool x = (a == b) && (c == d);
+  x = (a == b) || (c == d);
+}
+
+// CIR-LABEL: cir.func @_Z2b3iiii(
+// CIR-SAME: %[[ARG0:.*]]: !s32i {{.*}}, %[[ARG1:.*]]: !s32i {{.*}}, %[[ARG2:.*]]: !s32i {{.*}}, %[[ARG3:.*]]: !s32i {{.*}})
+// CIR: [[A:%[0-9]+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
+// CIR: [[B:%[0-9]+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
+// CIR: [[C:%[0-9]+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["c", init]
+// CIR: [[D:%[0-9]+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["d", init]
+// CIR: [[X:%[0-9]+]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init]
+// CIR: cir.store %[[ARG0]], [[A]] : !s32i, !cir.ptr<!s32i>
+// CIR: cir.store %[[ARG1]], [[B]] : !s32i, !cir.ptr<!s32i>
+// CIR: cir.store %[[ARG2]], [[C]] : !s32i, !cir.ptr<!s32i>
+// CIR: cir.store %[[ARG3]], [[D]] : !s32i, !cir.ptr<!s32i>
+// CIR: [[AVAL1:%[0-9]+]] = cir.load align(4) [[A]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[BVAL1:%[0-9]+]] = cir.load align(4) [[B]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[CMP1:%[0-9]+]] = cir.cmp(eq, [[AVAL1]], [[BVAL1]]) : !s32i, !cir.bool
+// CIR: [[AND_RESULT:%[0-9]+]] = cir.ternary([[CMP1]], true {
+// CIR: [[CVAL1:%[0-9]+]] = cir.load align(4) [[C]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[DVAL1:%[0-9]+]] = cir.load align(4) [[D]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[CMP2:%[0-9]+]] = cir.cmp(eq, [[CVAL1]], [[DVAL1]]) : !s32i, !cir.bool
+// CIR: cir.yield [[CMP2]] : !cir.bool
+// CIR: }, false {
+// CIR: [[FALSE:%[0-9]+]] = cir.const #false
+// CIR: cir.yield [[FALSE]] : !cir.bool
+// CIR: }) : (!cir.bool) -> !cir.bool
+// CIR: cir.store align(1) [[AND_RESULT]], [[X]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: [[AVAL2:%[0-9]+]] = cir.load align(4) [[A]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[BVAL2:%[0-9]+]] = cir.load align(4) [[B]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[CMP3:%[0-9]+]] = cir.cmp(eq, [[AVAL2]], [[BVAL2]]) : !s32i, !cir.bool
+// CIR: [[OR_RESULT:%[0-9]+]] = cir.ternary([[CMP3]], true {
+// CIR: [[TRUE:%[0-9]+]] = cir.const #true
+// CIR: cir.yield [[TRUE]] : !cir.bool
+// CIR: }, false {
+// CIR: [[CVAL2:%[0-9]+]] = cir.load align(4) [[C]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[DVAL2:%[0-9]+]] = cir.load align(4) [[D]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[CMP4:%[0-9]+]] = cir.cmp(eq, [[CVAL2]], [[DVAL2]]) : !s32i, !cir.bool
+// CIR: cir.yield [[CMP4]] : !cir.bool
+// CIR: }) : (!cir.bool) -> !cir.bool
+// CIR: cir.store align(1) [[OR_RESULT]], [[X]] : !cir.bool, !cir.ptr<!cir.bool>
+// CIR: cir.return
+
+
+// LLVM-LABEL: define void @_Z2b3iiii(
+// LLVM-SAME: i32 %[[ARG0:.+]], i32 %[[ARG1:.+]], i32 %[[ARG2:.+]], i32 %[[ARG3:.+]])
+// LLVM: %[[A_ADDR:.*]] = alloca i32, i64 1
+// LLVM: %[[B_ADDR:.*]] = alloca i32, i64 1
+// LLVM: %[[C_ADDR:.*]] = alloca i32, i64 1
+// LLVM: %[[D_ADDR:.*]] = alloca i32, i64 1
+// LLVM: %[[X:.*]] = alloca i8, i64 1
+// LLVM: store i32 %[[ARG0]], ptr %[[A_ADDR]]
+// LLVM: store i32 %[[ARG1]], ptr %[[B_ADDR]]
+// LLVM: store i32 %[[ARG2]], ptr %[[C_ADDR]]
+// LLVM: store i32 %[[ARG3]], ptr %[[D_ADDR]]
+// LLVM: %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]]
+// LLVM: %[[B_VAL:.*]] = load i32, ptr %[[B_ADDR]]
+// LLVM: %[[CMP1:.*]] = icmp eq i32 %[[A_VAL]], %[[B_VAL]]
+// LLVM: br i1 %[[CMP1]], label %[[AND_TRUE:.+]], label %[[AND_FALSE:.+]]
+// LLVM: [[AND_TRUE]]:
+// LLVM: %[[C_VAL:.*]] = load i32, ptr %[[C_ADDR]]
+// LLVM: %[[D_VAL:.*]] = load i32, ptr %[[D_ADDR]]
+// LLVM: %[[CMP2:.*]] = icmp eq i32 %[[C_VAL]], %[[D_VAL]]
+// LLVM: br label %[[AND_MERGE:.+]]
+// LLVM: [[AND_FALSE]]:
+// LLVM: br label %[[AND_MERGE]]
+// LLVM: [[AND_MERGE]]:
+// LLVM: %[[AND_PHI:.*]] = phi i1 [ false, %[[AND_FALSE]] ], [ %[[CMP2]], %[[AND_TRUE]] ]
+// LLVM: %[[ZEXT_AND:.*]] = zext i1 %[[AND_PHI]] to i8
+// LLVM: store i8 %[[ZEXT_AND]], ptr %[[X]]
+// LLVM: %[[A_VAL2:.*]] = load i32, ptr %[[A_ADDR]]
+// LLVM: %[[B_VAL2:.*]] = load i32, ptr %[[B_ADDR]]
+// LLVM: %[[CMP3:.*]] = icmp eq i32 %[[A_VAL2]], %[[B_VAL2]]
+// LLVM: br i1 %[[CMP3]], label %[[OR_TRUE:.+]], label %[[OR_FALSE:.+]]
+// LLVM: [[OR_TRUE]]:
+// LLVM: br label %[[OR_MERGE:.+]]
+// LLVM: [[OR_FALSE]]:
+// LLVM: %[[C_VAL2:.*]] = load i32, ptr %[[C_ADDR]]
+// LLVM: %[[D_VAL2:.*]] = load i32, ptr %[[D_ADDR]]
+// LLVM: %[[CMP4:.*]] = icmp eq i32 %[[C_VAL2]], %[[D_VAL2]]
+// LLVM: br label %[[OR_MERGE]]
+// LLVM: [[OR_MERGE]]:
+// LLVM: %[[OR_PHI:.*]] = phi i1 [ %[[CMP4]], %[[OR_FALSE]] ], [ true, %[[OR_TRUE]] ]
+// LLVM: %[[ZEXT_OR:.*]] = zext i1 %[[OR_PHI]] to i8
+// LLVM: store i8 %[[ZEXT_OR]], ptr %[[X]]
+// LLVM: ret void
+
+// OGCG-LABEL: define dso_local void @_Z2b3iiii(
+// OGCG-SAME: i32 {{.*}} %[[ARG0:.+]], i32 {{.*}} %[[ARG1:.+]], i32 {{.*}} %[[ARG2:.+]], i32 {{.*}} %[[ARG3:.+]])
+// OGCG: [[ENTRY:.*]]:
+// OGCG: %[[A_ADDR:.*]] = alloca i32
+// OGCG: %[[B_ADDR:.*]] = alloca i32
+// OGCG: %[[C_ADDR:.*]] = alloca i32
+// OGCG: %[[D_ADDR:.*]] = alloca i32
+// OGCG: %[[X:.*]] = alloca i8
+// OGCG: store i32 %[[ARG0]], ptr %[[A_ADDR]]
+// OGCG: store i32 %[[ARG1]], ptr %[[B_ADDR]]
+// OGCG: store i32 %[[ARG2]], ptr %[[C_ADDR]]
+// OGCG: store i32 %[[ARG3]], ptr %[[D_ADDR]]
+// OGCG: %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]]
+// OGCG: %[[B_VAL:.*]] = load i32, ptr %[[B_ADDR]]
+// OGCG: %[[CMP1:.*]] = icmp eq i32 %[[A_VAL]], %[[B_VAL]]
+// OGCG: br i1 %[[CMP1]], label %[[AND_TRUE:.+]], label %[[AND_MERGE:.+]]
+// OGCG: [[AND_TRUE]]:
+// OGCG: %[[C_VAL:.*]] = load i32, ptr %[[C_ADDR]]
+// OGCG: %[[D_VAL:.*]] = load i32, ptr %[[D_ADDR]]
+// OGCG: %[[CMP2:.*]] = icmp eq i32 %[[C_VAL]], %[[D_VAL]]
+// OGCG: br label %[[AND_MERGE:.+]]
+// OGCG: [[AND_MERGE]]:
+// OGCG: %[[AND_PHI:.*]] = phi i1 [ false, %[[ENTRY]] ], [ %[[CMP2]], %[[AND_TRUE]] ]
+// OGCG: %[[ZEXT_AND:.*]] = zext i1 %[[AND_PHI]] to i8
+// OGCG: store i8 %[[ZEXT_AND]], ptr %[[X]]
+// OGCG: %[[A_VAL2:.*]] = load i32, ptr %[[A_ADDR]]
+// OGCG: %[[B_VAL2:.*]] = load i32, ptr %[[B_ADDR]]
+// OGCG: %[[CMP3:.*]] = icmp eq i32 %[[A_VAL2]], %[[B_VAL2]]
+// OGCG: br i1 %[[CMP3]], label %[[OR_MERGE:.+]], label %[[OR_FALSE:.+]]
+// OGCG: [[OR_FALSE]]:
+// OGCG: %[[C_VAL2:.*]] = load i32, ptr %[[C_ADDR]]
+// OGCG: %[[D_VAL2:.*]] = load i32, ptr %[[D_ADDR]]
+// OGCG: %[[CMP4:.*]] = icmp eq i32 %[[C_VAL2]], %[[D_VAL2]]
+// OGCG: br label %[[OR_MERGE]]
+// OGCG: [[OR_MERGE]]:
+// OGCG: %[[OR_PHI:.*]] = phi i1 [ true, %[[AND_MERGE]] ], [ %[[CMP4]], %[[OR_FALSE]] ]
+// OGCG: %[[ZEXT_OR:.*]] = zext i1 %[[OR_PHI]] to i8
+// OGCG: store i8 %[[ZEXT_OR]], ptr %[[X]]
+// OGCG: ret void
\ No newline at end of file
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/builtin_call.cpp b/external/llvm-project/clang/test/CIR/CodeGen/builtin_call.cpp
new file mode 100644
index 000000000000..2706ea7f8f85
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/builtin_call.cpp
@@ -0,0 +1,78 @@
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+constexpr extern int cx_var = __builtin_is_constant_evaluated();
+
+// CIR: cir.global {{.*}} @cx_var = #cir.int<1> : !s32i
+// LLVM: @cx_var = {{.*}} i32 1
+// OGCG: @cx_var = {{.*}} i32 1
+
+constexpr extern float cx_var_single = __builtin_huge_valf();
+
+// CIR: cir.global {{.*}} @cx_var_single = #cir.fp<0x7F800000> : !cir.float
+// LLVM: @cx_var_single = {{.*}} float 0x7FF0000000000000
+// OGCG: @cx_var_single = {{.*}} float 0x7FF0000000000000
+
+constexpr extern long double cx_var_ld = __builtin_huge_vall();
+
+// CIR: cir.global {{.*}} @cx_var_ld = #cir.fp<0x7FFF8000000000000000> : !cir.long_double<!cir.f80>
+// LLVM: @cx_var_ld = {{.*}} x86_fp80 0xK7FFF8000000000000000
+// OGCG: @cx_var_ld = {{.*}} x86_fp80 0xK7FFF8000000000000000
+
+int is_constant_evaluated() {
+  return __builtin_is_constant_evaluated();
+}
+
+// CIR: cir.func @_Z21is_constant_evaluatedv() -> !s32i
+// CIR: %[[ZERO:.+]] = cir.const #cir.int<0>
+
+// LLVM: define {{.*}}i32 @_Z21is_constant_evaluatedv()
+// LLVM: %[[MEM:.+]] = alloca i32
+// LLVM: store i32 0, ptr %[[MEM]]
+// LLVM: %[[RETVAL:.+]] = load i32, ptr %[[MEM]]
+// LLVM: ret i32 %[[RETVAL]]
+// LLVM: }
+
+// OGCG: define {{.*}}i32 @_Z21is_constant_evaluatedv()
+// OGCG: ret i32 0
+// OGCG: }
+
+long double constant_fp_builtin_ld() {
+  return __builtin_fabsl(-0.1L);
+}
+
+// CIR: cir.func @_Z22constant_fp_builtin_ldv() -> !cir.long_double<!cir.f80>
+// CIR: %[[PONE:.+]] = cir.const #cir.fp<1.000000e-01> : !cir.long_double<!cir.f80>
+
+// LLVM: define {{.*}}x86_fp80 @_Z22constant_fp_builtin_ldv()
+// LLVM: %[[MEM:.+]] = alloca x86_fp80
+// LLVM: store x86_fp80 0xK3FFBCCCCCCCCCCCCCCCD, ptr %[[MEM]]
+// LLVM: %[[RETVAL:.+]] = load x86_fp80, ptr %[[MEM]]
+// LLVM: ret x86_fp80 %[[RETVAL]]
+// LLVM: }
+
+// OGCG: define {{.*}}x86_fp80 @_Z22constant_fp_builtin_ldv()
+// OGCG: ret x86_fp80 0xK3FFBCCCCCCCCCCCCCCCD
+// OGCG: }
+
+float constant_fp_builtin_single() {
+  return __builtin_fabsf(-0.1f);
+}
+
+// CIR: cir.func @_Z26constant_fp_builtin_singlev() -> !cir.float
+// CIR: %[[PONE:.+]] = cir.const #cir.fp<1.000000e-01> : !cir.float
+
+// LLVM: define {{.*}}float @_Z26constant_fp_builtin_singlev()
+// LLVM: %[[MEM:.+]] = alloca float
+// LLVM: store float 0x3FB99999A0000000, ptr %[[MEM]]
+// LLVM: %[[RETVAL:.+]] = load float, ptr %[[MEM]]
+// LLVM: ret float %[[RETVAL]]
+// LLVM: }
+
+// OGCG: define {{.*}}float @_Z26constant_fp_builtin_singlev()
+// OGCG: ret float 0x3FB99999A0000000
+// OGCG: }
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/call.c b/external/llvm-project/clang/test/CIR/CodeGen/call.c
new file mode 100644
index 000000000000..13f3c5a21ceb
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/call.c
@@ -0,0 +1,111 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+struct S {
+  int x;
+  int y;
+};
+
+void f1(struct S);
+void f2() {
+  struct S s;
+  f1(s);
+}
+
+// CIR-LABEL: cir.func @f2()
+// CIR:         %[[S:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!rec_S>, !rec_S
+// CIR-NEXT:    cir.call @f1(%[[S]]) : (!rec_S) -> ()
+
+// LLVM-LABEL: define void @f2()
+// LLVM:         %[[S:.+]] = load %struct.S, ptr %{{.+}}, align 4
+// LLVM-NEXT:    call void @f1(%struct.S %[[S]])
+
+// OGCG-LABEL: define dso_local void @f2()
+// OGCG:         %[[S:.+]] = load i64, ptr %{{.+}}, align 4
+// OGCG-NEXT:    call void @f1(i64 %[[S]])
+
+struct S f3();
+void f4() {
+  struct S s = f3();
+}
+
+// CIR-LABEL: cir.func @f4() {
+// CIR:         %[[S:.+]] = cir.call @f3() : () -> !rec_S
+// CIR-NEXT:    cir.store align(4) %[[S]], %{{.+}} : !rec_S, !cir.ptr<!rec_S>
+
+// LLVM-LABEL: define void @f4() {
+// LLVM:         %[[S:.+]] = call %struct.S (...) @f3()
+// LLVM-NEXT:    store %struct.S %[[S]], ptr %{{.+}}, align 4
+
+// OGCG-LABEL: define dso_local void @f4() #0 {
+// OGCG:         %[[S:.+]] = call i64 (...) @f3()
+// OGCG-NEXT:    store i64 %[[S]], ptr %{{.+}}, align 4
+
+struct Big {
+  int data[10];
+};
+
+void f5(struct Big);
+struct Big f6();
+
+void f7() {
+  struct Big b;
+  f5(b);
+}
+
+// CIR-LABEL: cir.func @f7()
+// CIR:         %[[B:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!rec_Big>, !rec_Big
+// CIR-NEXT:    cir.call @f5(%[[B]]) : (!rec_Big) -> ()
+
+// LLVM-LABEL: define void @f7() {
+// LLVM:         %[[B:.+]] = load %struct.Big, ptr %{{.+}}, align 4
+// LLVM-NEXT:    call void @f5(%struct.Big %[[B]])
+
+// OGCG-LABEL: define dso_local void @f7() #0 {
+// OGCG:         %[[B:.+]] = alloca %struct.Big, align 8
+// OGCG-NEXT:    call void @f5(ptr noundef byval(%struct.Big) align 8 %[[B]])
+
+void f8() {
+  struct Big b = f6();
+}
+
+// CIR-LABEL: cir.func @f8()
+// CIR:         %[[B:.+]] = cir.call @f6() : () -> !rec_Big
+// CIR:         cir.store align(4) %[[B]], %{{.+}} : !rec_Big, !cir.ptr<!rec_Big>
+
+// LLVM-LABEL: define void @f8() {
+// LLVM:        %[[B:.+]] = call %struct.Big (...) @f6()
+// LLVM-NEXT:   store %struct.Big %[[B]], ptr %{{.+}}, align 4
+
+// OGCG-LABEL: define dso_local void @f8() #0 {
+// OGCG:         %[[B:.+]] = alloca %struct.Big, align 4
+// OGCG-NEXT:    call void (ptr, ...) @f6(ptr dead_on_unwind writable sret(%struct.Big) align 4 %[[B]])
+
+void f9() {
+  f1(f3());
+}
+
+// CIR-LABEL: cir.func @f9()
+// CIR:         %[[SLOT:.+]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["agg.tmp0"] {alignment = 4 : i64}
+// CIR-NEXT:    %[[RET:.+]] = cir.call @f3() : () -> !rec_S
+// CIR-NEXT:    cir.store align(4) %[[RET]], %[[SLOT]] : !rec_S, !cir.ptr<!rec_S>
+// CIR-NEXT:    %[[ARG:.+]] = cir.load align(4) %[[SLOT]] : !cir.ptr<!rec_S>, !rec_S
+// CIR-NEXT:    cir.call @f1(%[[ARG]]) : (!rec_S) -> ()
+
+// LLVM-LABEL: define void @f9() {
+// LLVM:         %[[SLOT:.+]] = alloca %struct.S, i64 1, align 4
+// LLVM-NEXT:    %[[RET:.+]] = call %struct.S (...) @f3()
+// LLVM-NEXT:    store %struct.S %[[RET]], ptr %[[SLOT]], align 4
+// LLVM-NEXT:    %[[ARG:.+]] = load %struct.S, ptr %[[SLOT]], align 4
+// LLVM-NEXT:    call void @f1(%struct.S %[[ARG]])
+
+// OGCG-LABEL: define dso_local void @f9() #0 {
+// OGCG:         %[[SLOT:.+]] = alloca %struct.S, align 4
+// OGCG-NEXT:    %[[RET:.+]] = call i64 (...) @f3()
+// OGCG-NEXT:    store i64 %[[RET]], ptr %[[SLOT]], align 4
+// OGCG-NEXT:    %[[ARG:.+]] = load i64, ptr %[[SLOT]], align 4
+// OGCG-NEXT:    call void @f1(i64 %[[ARG]])
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/call.cpp b/external/llvm-project/clang/test/CIR/CodeGen/call.cpp
index 741cadeb5c76..cc25afce1e5a 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/call.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/call.cpp
@@ -70,3 +70,35 @@ void f9() {
 // LLVM-LABEL: define void @_Z2f9v()
 // LLVM:         call void (i32, ...) @_Z2f8iz(i32 1)
 // LLVM:         call void (i32, ...) @_Z2f8iz(i32 1, i32 2, i32 3, i32 4)
+
+struct S {
+  int x;
+  int y;
+};
+
+S f10();
+void f11() {
+  S s = f10();
+}
+
+// CIR-LABEL: cir.func @_Z3f11v()
+// CIR:         %[[#s:]] = cir.call @_Z3f10v() : () -> !rec_S
+// CIR-NEXT:    cir.store align(4) %[[#s]], %{{.+}} : !rec_S, !cir.ptr<!rec_S>
+
+// LLVM-LABEL: define void @_Z3f11v()
+// LLVM:         %[[#s:]] = call %struct.S @_Z3f10v()
+// LLVM-NEXT:    store %struct.S %[[#s]], ptr %{{.+}}, align 4
+
+void f12() {
+  f10();
+}
+
+// CIR-LABEL: cir.func @_Z3f12v()
+// CIR:         %[[#slot:]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["agg.tmp0"]
+// CIR-NEXT:    %[[#ret:]] = cir.call @_Z3f10v() : () -> !rec_S
+// CIR-NEXT:    cir.store align(4) %[[#ret]], %[[#slot]] : !rec_S, !cir.ptr<!rec_S>
+
+// LLVM-LABEL: define void @_Z3f12v() {
+// LLVM:         %[[#slot:]] = alloca %struct.S, i64 1, align 4
+// LLVM-NEXT:    %[[#ret:]] = call %struct.S @_Z3f10v()
+// LLVM-NEXT:    store %struct.S %[[#ret]], ptr %[[#slot]], align 4
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/cast.cpp b/external/llvm-project/clang/test/CIR/CodeGen/cast.cpp
index a7c11b1939ba..84f55242a611 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/cast.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/cast.cpp
@@ -73,6 +73,14 @@ int cStyleCasts_0(unsigned x1, int x2, float x3, short x4, double x5) {
   // LLVM: %{{[0-9]+}} = fcmp une float %{{[0-9]+}}, 0.000000e+00
   // LLVM: %{{[0-9]+}} = zext i1 %{{[0-9]+}} to i8
 
+  double d2 = f; // float to double
+  // CIR: %{{[0-9]+}} = cir.cast(floating, %{{[0-9]+}} : !cir.float), !cir.double
+  // LLVM: %{{[0-9]+}} = fpext float %{{[0-9]+}} to double
+
+  f = d2; // double to float
+  // CIR: %{{[0-9]+}} = cir.cast(floating, %{{[0-9]+}} : !cir.double), !cir.float
+  // LLVM: %{{[0-9]+}} = fptrunc double %{{[0-9]+}} to float
+
   return 0;
 }
 
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/class.cpp b/external/llvm-project/clang/test/CIR/CodeGen/class.cpp
new file mode 100644
index 000000000000..d7f3772c9582
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/class.cpp
@@ -0,0 +1,102 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+// CIR: !rec_IncompleteC = !cir.record<class "IncompleteC" incomplete>
+// CIR: !rec_Base = !cir.record<class "Base" {!s32i}>
+// CIR: !rec_CompleteC = !cir.record<class "CompleteC" {!s32i, !s8i}>
+// CIR: !rec_Derived = !cir.record<class "Derived" {!rec_Base, !s32i}>
+
+// Note: LLVM and OGCG do not emit the type for incomplete classes.
+
+// LLVM: %class.CompleteC = type { i32, i8 }
+// LLVM: %class.Derived = type { %class.Base, i32 }
+// LLVM: %class.Base = type { i32 }
+
+// OGCG: %class.CompleteC = type { i32, i8 }
+// OGCG: %class.Derived = type { %class.Base, i32 }
+// OGCG: %class.Base = type { i32 }
+
+class IncompleteC;
+IncompleteC *p;
+
+// CIR: cir.global external @p = #cir.ptr<null> : !cir.ptr<!rec_IncompleteC>
+// LLVM: @p = global ptr null
+// OGCG: @p = global ptr null, align 8
+
+class CompleteC {
+public:    
+  int a;
+  char b;
+};
+
+CompleteC cc;
+
+// CIR:       cir.global external @cc = #cir.zero : !rec_CompleteC
+// LLVM:  @cc = global %class.CompleteC zeroinitializer
+// OGCG:  @cc = global %class.CompleteC zeroinitializer
+
+class Base {
+public:
+  int a;
+};
+
+class Derived : public Base {
+public:
+  int b;
+};
+
+int use(Derived *d) { return d->b; }
+
+// CIR: cir.func @_Z3useP7Derived(%[[ARG0:.*]]: !cir.ptr<!rec_Derived>
+// CIR:  %[[D_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Derived>, !cir.ptr<!cir.ptr<!rec_Derived>>, ["d", init]
+// CIR:  cir.store %[[ARG0]], %[[D_ADDR]]
+// CIR:  %[[D_PTR:.*]] = cir.load align(8) %0
+// CIR:  %[[D_B_ADDR:.*]] = cir.get_member %[[D_PTR]][1] {name = "b"}
+// CIR:  %[[D_B:.*]] = cir.load align(4) %[[D_B_ADDR]]
+
+// LLVM: define{{.*}} i32 @_Z3useP7Derived
+// LLVM:   getelementptr %class.Derived, ptr %{{.*}}, i32 0, i32 1
+
+// OGCG: define{{.*}} i32 @_Z3useP7Derived
+// OGCG:   getelementptr inbounds nuw %class.Derived, ptr %{{.*}}, i32 0, i32 1
+
+int use_base() {
+  Derived d;
+  return d.a;
+}
+
+// CIR: cir.func @_Z8use_basev
+// CIR:   %[[D_ADDR:.*]] = cir.alloca !rec_Derived, !cir.ptr<!rec_Derived>, ["d"]
+// CIR:   %[[BASE_ADDR:.*]] cir.base_class_addr %[[D_ADDR]] : !cir.ptr<!rec_Derived> nonnull [0] -> !cir.ptr<!rec_Base>
+// CIR:   %[[D_A_ADDR:.*]] = cir.get_member %2[0] {name = "a"} : !cir.ptr<!rec_Base> -> !cir.ptr<!s32i>
+// CIR:   %[[D_A:.*]] = cir.load align(4) %3 : !cir.ptr<!s32i>, !s32i
+
+// LLVM: define{{.*}} i32 @_Z8use_basev
+// LLVM:   %[[D:.*]] = alloca %class.Derived
+// LLVM:   %[[D_A_ADDR:.*]] = getelementptr %class.Base, ptr %[[D]], i32 0, i32 0
+
+// OGCG: define{{.*}} i32 @_Z8use_basev
+// OGCG:   %[[D:.*]] = alloca %class.Derived
+// OGCG:   %[[D_A_ADDR:.*]] = getelementptr inbounds nuw %class.Base, ptr %[[D]], i32 0, i32 0
+
+int use_base_via_pointer(Derived *d) {
+  return d->a;
+}
+
+// CIR: cir.func @_Z20use_base_via_pointerP7Derived(%[[ARG0:.*]]: !cir.ptr<!rec_Derived>
+// CIR:   %[[D_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Derived>, !cir.ptr<!cir.ptr<!rec_Derived>>, ["d", init]
+// CIR:   cir.store %[[ARG0]], %[[D_ADDR]]
+// CIR:   %[[D:.*]] = cir.load align(8) %[[D_ADDR]]
+// CIR:   %[[BASE_ADDR:.*]] = cir.base_class_addr %[[D]] : !cir.ptr<!rec_Derived> nonnull [0] -> !cir.ptr<!rec_Base>
+// CIR:   %[[D_A_ADDR:.*]] = cir.get_member %[[BASE_ADDR]][0] {name = "a"}
+// CIR:   %[[D_A:.*]] = cir.load align(4) %[[D_A_ADDR]]
+
+// LLVM: define{{.*}} i32 @_Z20use_base_via_pointerP7Derived
+// LLVM:   %[[D_A_ADDR:.*]] = getelementptr %class.Base, ptr %{{.*}}, i32 0, i32 0
+
+// OGCG: define{{.*}} i32 @_Z20use_base_via_pointerP7Derived
+// OGCG:   %[[D_A_ADDR:.*]] = getelementptr inbounds nuw %class.Base, ptr %{{.*}}, i32 0, i32 0
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/complex.cpp b/external/llvm-project/clang/test/CIR/CodeGen/complex.cpp
new file mode 100644
index 000000000000..d193b9f32efb
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/complex.cpp
@@ -0,0 +1,178 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+int _Complex ci;
+
+float _Complex cf;
+
+int _Complex ci2 = { 1, 2 };
+
+float _Complex cf2 = { 1.0f, 2.0f };
+
+// CIR: cir.global external @ci = #cir.zero : !cir.complex<!s32i>
+// CIR: cir.global external @cf = #cir.zero : !cir.complex<!cir.float>
+// CIR: cir.global external @ci2 = #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+// CIR: cir.global external @cf2 = #cir.const_complex<#cir.fp<1.000000e+00> : !cir.float, #cir.fp<2.000000e+00> : !cir.float> : !cir.complex<!cir.float>
+
+// LLVM: {{.*}} = global { i32, i32 } zeroinitializer, align 4
+// LLVM: {{.*}} = global { float, float } zeroinitializer, align 4
+// LLVM: {{.*}} = global { i32, i32 } { i32 1, i32 2 }, align 4
+// LLVM: {{.*}} = global { float, float } { float 1.000000e+00, float 2.000000e+00 }, align 4
+
+// OGCG: {{.*}} = global { i32, i32 } zeroinitializer, align 4
+// OGCG: {{.*}} = global { float, float } zeroinitializer, align 4
+// OGCG: {{.*}} = global { i32, i32 } { i32 1, i32 2 }, align 4
+// OGCG: {{.*}} = global { float, float } { float 1.000000e+00, float 2.000000e+00 }, align 4
+
+void foo() { int _Complex c = {}; }
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.int<0> : !s32i, #cir.int<0> : !s32i> : !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[INIT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: store { i32, i32 } zeroinitializer, ptr %[[INIT]], align 4
+
+// OGCG: %[[COMPLEX:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store i32 0, ptr %[[C_REAL_PTR]], align 4
+// OGCG: store i32 0, ptr %[[C_IMAG_PTR]], align 4
+
+void foo2() { int _Complex c = {1, 2}; }
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[INIT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: store { i32, i32 } { i32 1, i32 2 }, ptr %[[INIT]], align 4
+
+// OGCG: %[[COMPLEX:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store i32 1, ptr %[[C_REAL_PTR]], align 4
+// OGCG: store i32 2, ptr %[[C_IMAG_PTR]], align 4
+
+void foo3() {
+  int a;
+  int b;
+  int _Complex c = {a, b};
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!s32i>
+// CIR: %[[COMPLEX:.*]] = cir.complex.create %[[TMP_A]], %[[TMP_B]] : !s32i -> !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[INIT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load i32, ptr {{.*}}, align 4
+// LLVM: %[[TMP_B:.*]] = load i32, ptr {{.*}}, align 4
+// LLVM: %[[TMP:.*]] = insertvalue { i32, i32 } undef, i32 %[[TMP_A]], 0
+// LLVM: %[[TMP_2:.*]] = insertvalue { i32, i32 } %[[TMP]], i32 %[[TMP_B]], 1
+// LLVM: store { i32, i32 } %[[TMP_2]], ptr %[[INIT]], align 4
+
+// OGCG: %[[COMPLEX:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[REAL_VAL:.*]] = load i32, ptr {{.*}}, align 4
+// OGCG: %[[IMAG_VAL:.*]] = load i32, ptr {{.*}}, align 4
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store i32 %[[REAL_VAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG: store i32 %[[IMAG_VAL]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo4() {
+  int a;
+  int _Complex c = {1, a};
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!s32i>, !s32i
+// CIR: %[[COMPLEX:.*]] = cir.complex.create %[[CONST_1]], %[[TMP_A]] : !s32i -> !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[INIT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load i32, ptr {{.*}}, align 4
+// LLVM: %[[COMPLEX:.*]] = insertvalue { i32, i32 } { i32 1, i32 undef }, i32 %[[TMP_A]], 1
+// LLVM: store { i32, i32 } %[[COMPLEX]], ptr %[[INIT]], align 4
+
+// OGCG: %[[COMPLEX:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[TMP_A:.*]] = load i32, ptr {{.*}}, align 4
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store i32 1, ptr %[[C_REAL_PTR]], align 4
+// OGCG: store i32 %[[TMP_A]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo5() {
+  float _Complex c = {1.0f, 2.0f};
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.fp<1.000000e+00> : !cir.float, #cir.fp<2.000000e+00> : !cir.float> : !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[INIT:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: store { float, float } { float 1.000000e+00, float 2.000000e+00 }, ptr %[[INIT]], align 4
+
+// OGCG: %[[COMPLEX]] = alloca { float, float }, align 4
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store float 1.000000e+00, ptr %[[C_REAL_PTR]], align 4
+// OGCG: store float 2.000000e+00, ptr %[[C_IMAG_PTR]], align 4
+
+void foo6() {
+  float a;
+  float b;
+  float _Complex c = {a, b};
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.float>, !cir.float
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.float>, !cir.float
+// CIR: %[[COMPLEX:.*]] = cir.complex.create %[[TMP_A]], %[[TMP_B]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[COMPLEX:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load float, ptr {{.*}}, align 4
+// LLVM: %[[TMP_B:.*]] = load float, ptr {{.*}}, align 4
+// LLVM: %[[TMP:.*]] = insertvalue { float, float } undef, float %[[TMP_A]], 0
+// LLVM: %[[TMP_2:.*]] = insertvalue { float, float } %[[TMP]], float %[[TMP_B]], 1
+// LLVM: store { float, float } %[[TMP_2]], ptr %[[COMPLEX]], align 4
+
+// OGCG: %[[COMPLEX]] = alloca { float, float }, align 4
+// OGCG: %[[TMP_A:.*]] = load float, ptr {{.*}}, align 4
+// OGCG: %[[TMP_B:.*]] = load float, ptr {{.*}}, align 4
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store float %[[TMP_A]], ptr %[[C_REAL_PTR]], align 4
+// OGCG: store float %[[TMP_B]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo7() {
+  float a;
+  float _Complex c = {a, 2.0f};
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.float>, !cir.float
+// CIR: %[[CONST_2F:.*]] = cir.const #cir.fp<2.000000e+00> : !cir.float
+// CIR: %[[COMPLEX:.*]] = cir.complex.create %[[TMP_A]], %[[CONST_2F]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[COMPLEX:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load float, ptr {{.*}}, align 4
+// LLVM: %[[TMP:.*]] = insertvalue { float, float } undef, float %[[TMP_A]], 0
+// LLVM: %[[TMP_2:.*]] = insertvalue { float, float } %[[TMP]], float 2.000000e+00, 1
+// LLVM: store { float, float } %[[TMP_2]], ptr %[[COMPLEX]], align 4
+
+// OGCG: %[[COMPLEX:.*]] = alloca { float, float }, align 4
+// OGCG: %[[TMP_A:.*]] = load float, ptr {{.*}}, align 4
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store float %[[TMP_A]], ptr %[[C_REAL_PTR]], align 4
+// OGCG: store float 2.000000e+00, ptr %[[C_IMAG_PTR]], align 4
+
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/ctor.cpp b/external/llvm-project/clang/test/CIR/CodeGen/ctor.cpp
new file mode 100644
index 000000000000..1a36eb0d9d3a
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/ctor.cpp
@@ -0,0 +1,115 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+
+struct Struk {
+  int a;
+  Struk() {}
+};
+
+void baz() {
+  Struk s;
+}
+
+// CHECK: !rec_Struk = !cir.record<struct "Struk" {!s32i}>
+
+// Note: In the absence of the '-mconstructor-aliases' option, we emit two
+//       constructors here. The handling of constructor aliases is currently
+//       NYI, but when it is added this test should be updated to add a RUN
+//       line that passes '-mconstructor-aliases' to clang_cc1.
+// CHECK:   cir.func @_ZN5StrukC2Ev(%arg0: !cir.ptr<!rec_Struk>
+// CHECK-NEXT:     %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>, ["this", init] {alignment = 8 : i64}
+// CHECK-NEXT:     cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>
+// CHECK-NEXT:     %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_Struk>>, !cir.ptr<!rec_Struk>
+// CHECK-NEXT:     cir.return
+
+// CHECK:   cir.func @_ZN5StrukC1Ev(%arg0: !cir.ptr<!rec_Struk>
+// CHECK-NEXT:     %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>, ["this", init] {alignment = 8 : i64}
+// CHECK-NEXT:     cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>
+// CHECK-NEXT:     %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_Struk>>, !cir.ptr<!rec_Struk>
+// CHECK-NEXT:     cir.call @_ZN5StrukC2Ev(%[[THIS]]) : (!cir.ptr<!rec_Struk>) -> ()
+// CHECK-NEXT:     cir.return
+
+// CHECK:   cir.func @_Z3bazv()
+// CHECK-NEXT:     %[[S_ADDR:.*]] = cir.alloca !rec_Struk, !cir.ptr<!rec_Struk>, ["s", init] {alignment = 4 : i64}
+// CHECK-NEXT:     cir.call @_ZN5StrukC1Ev(%[[S_ADDR]]) : (!cir.ptr<!rec_Struk>) -> ()
+// CHECK-NEXT:     cir.return
+
+struct VariadicStruk {
+  int a;
+  VariadicStruk(int n, ...) { a = n;}
+};
+
+void bar() {
+  VariadicStruk s(1, 2, 3);
+}
+
+// When a variadic constructor is present, we call the C2 constructor directly.
+
+// CHECK-NOT: cir.func @_ZN13VariadicStrukC2Eiz
+
+// CHECK:      cir.func @_ZN13VariadicStrukC1Eiz(%arg0: !cir.ptr<!rec_VariadicStruk>
+// CHECK-SAME:                                   %arg1: !s32i
+// CHECK-SAME:                                   ...) {
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   %[[N_ADDR:.*]] = cir.alloca {{.*}} ["n", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   cir.store %arg1, %[[N_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load{{.*}} %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[N:.*]] = cir.load{{.*}} %[[N_ADDR]]
+// CHECK-NEXT:   %[[A_ADDR:.*]] = cir.get_member %[[THIS]][0] {name = "a"}
+// CHECK-NEXT:   cir.store{{.*}} %[[N]], %[[A_ADDR]]
+// CHECK-NEXT:   cir.return
+
+// CHECK:  cir.func @_Z3barv
+// CHECK-NEXT:    %[[S_ADDR:.*]] = cir.alloca !rec_VariadicStruk, !cir.ptr<!rec_VariadicStruk>, ["s", init]
+// CHECK-NEXT:    %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CHECK-NEXT:    %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+// CHECK-NEXT:    %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+// CHECK-NEXT:    cir.call @_ZN13VariadicStrukC1Eiz(%[[S_ADDR]], %[[ONE]], %[[TWO]], %[[THREE]])
+// CHECK-NEXT:    cir.return
+
+struct DelegatingStruk {
+  int a;
+  DelegatingStruk(int n) { a = n; }
+  DelegatingStruk() : DelegatingStruk(0) {}
+};
+
+void bam() {
+  DelegatingStruk s;
+}
+
+// CHECK:       cir.func @_ZN15DelegatingStrukC2Ei(%arg0: !cir.ptr<!rec_DelegatingStruk>
+// CHECK-SAME:                                     %arg1: !s32i
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   %[[N_ADDR:.*]] = cir.alloca {{.*}} ["n", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   cir.store %arg1, %[[N_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load{{.*}} %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[N:.*]] = cir.load{{.*}} %[[N_ADDR]]
+// CHECK-NEXT:   %[[A_ADDR:.*]] = cir.get_member %[[THIS]][0] {name = "a"}
+// CHECK-NEXT:   cir.store{{.*}} %[[N]], %[[A_ADDR]]
+// CHECK-NEXT:   cir.return
+
+// CHECK:       cir.func @_ZN15DelegatingStrukC1Ei(%arg0: !cir.ptr<!rec_DelegatingStruk>
+// CHECK-SAME:                                     %arg1: !s32i
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   %[[N_ADDR:.*]] = cir.alloca {{.*}} ["n", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   cir.store %arg1, %[[N_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load{{.*}} %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[N:.*]] = cir.load{{.*}} %[[N_ADDR]]
+// CHECK-NEXT:   cir.call @_ZN15DelegatingStrukC2Ei(%[[THIS]], %[[N]])
+// CHECK-NEXT:   cir.return
+
+// CHECK: cir.func @_ZN15DelegatingStrukC1Ev(%arg0: !cir.ptr<!rec_DelegatingStruk>
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load{{.*}} %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CHECK-NEXT:   cir.call @_ZN15DelegatingStrukC1Ei(%[[THIS]], %[[ZERO]])
+// CHECK-NEXT:   cir.return
+
+// CHECK: cir.func @_Z3bamv
+// CHECK-NEXT:    %[[S_ADDR:.*]] = cir.alloca {{.*}} ["s", init]
+// CHECK-NEXT:    cir.call @_ZN15DelegatingStrukC1Ev(%[[S_ADDR]])
+// CHECK-NEXT:    cir.return
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/deferred-defs.cpp b/external/llvm-project/clang/test/CIR/CodeGen/deferred-defs.cpp
new file mode 100644
index 000000000000..4230a4ed3e24
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/deferred-defs.cpp
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR --implicit-check-not=unusedExternal \
+// RUN:   --implicit-check-not=unusedInternal
+
+extern int usedExternal;
+extern int unusedExternal;
+
+int locallyDefined;
+
+namespace {
+  int unsedInternal;
+  int usedInternal;
+}
+
+void use() {
+  usedInternal;
+  usedExternal;
+}
+
+// CIR: cir.global external @locallyDefined = #cir.int<0> : !s32i
+// CIR: cir.global "private" internal dsolocal @_ZN12_GLOBAL__N_112usedInternalE = #cir.int<0> : !s32i
+// CIR: cir.global "private" external @usedExternal : !s32i
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/deferred-fn-defs.cpp b/external/llvm-project/clang/test/CIR/CodeGen/deferred-fn-defs.cpp
new file mode 100644
index 000000000000..e7088ba35eca
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/deferred-fn-defs.cpp
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR --implicit-check-not=externNotCalled \
+// RUN:   --implicit-check-not=internalNotCalled --implicit-check-not=inlineNotCalled
+
+extern int externCalled();
+extern int externNotCalled();
+
+namespace {
+  int internalCalled() { return 1; }
+  int internalNotCalled() { return 2; }
+}
+
+struct S {
+  int inlineCalled() { return 3; }
+  int inlineNotCalled() { return 4; }
+};
+
+void use() {
+  S s;
+  externCalled();
+  internalCalled();
+  s.inlineCalled();
+}
+
+// CIR: cir.func{{.*}} @_Z12externCalledv
+// This shouldn't have a body.
+// CIR-NOT: cir.return
+
+// CIR: cir.func{{.*}} @_ZN12_GLOBAL__N_114internalCalledEv
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1>
+// CIR:   cir.store %[[ONE]], %[[RET_ADDR:.*]]
+
+// CIR: cir.func{{.*}} @_ZN1S12inlineCalledEv
+// CIR:   %[[THIS:.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["this", init]
+// CIR:   %[[THREE:.*]] = cir.const #cir.int<3>
+// CIR:   cir.store %[[THREE]], %[[RET_ADDR:.*]]
+
+// CIR: cir.func{{.*}} @_Z3usev()
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/dlti.c b/external/llvm-project/clang/test/CIR/CodeGen/dlti.c
new file mode 100644
index 000000000000..3fa15d2ab584
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/dlti.c
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=LITTLE
+
+void foo() {}
+
+//  LITTLE-DAG: dlti.dl_spec =
+//  LITTLE-DAG:   #dlti.dl_spec<
+//  LITTLE-DAG:     i16 = dense<16> : vector<2xi64>,
+//  LITTLE-DAG:     i32 = dense<32> : vector<2xi64>,
+//  LITTLE-DAG:     i8 = dense<8> : vector<2xi64>,
+//  LITTLE-DAG:     i1 = dense<8> : vector<2xi64>,
+//  LITTLE-DAG:     !llvm.ptr = dense<64> : vector<4xi64>,
+//  LITTLE-DAG:     f80 = dense<128> : vector<2xi64>,
+//  LITTLE-DAG:     i128 = dense<128> : vector<2xi64>,
+//  LITTLE-DAG:     !llvm.ptr<272> = dense<64> : vector<4xi64>,
+//  LITTLE-DAG:     i64 = dense<64> : vector<2xi64>,
+//  LITTLE-DAG:     !llvm.ptr<270> = dense<32> : vector<4xi64>,
+//  LITTLE-DAG:     !llvm.ptr<271> = dense<32> : vector<4xi64>,
+//  LITTLE-DAG:     f128 = dense<128> : vector<2xi64>,
+//  LITTLE-DAG:     f16 = dense<16> : vector<2xi64>,
+//  LITTLE-DAG:     f64 = dense<64> : vector<2xi64>,
+//  LITTLE-DAG:     "dlti.stack_alignment" = 128 : i64
+//  LITTLE-DAG:     "dlti.endianness" = "little"
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/dlti_be.c b/external/llvm-project/clang/test/CIR/CodeGen/dlti_be.c
new file mode 100644
index 000000000000..918124ac0a23
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/dlti_be.c
@@ -0,0 +1,23 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64_be-linux-gnu -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=BIG
+
+void foo() {}
+
+// BIG-DAG: dlti.dl_spec =
+// BIG-DAG:   #dlti.dl_spec<
+// BIG-DAG:     i16 = dense<[16, 32]> : vector<2xi64>,
+// BIG-DAG:     i32 = dense<32> : vector<2xi64>,
+// BIG-DAG:     i8 = dense<[8, 32]> : vector<2xi64>,
+// BIG-DAG:     i1 = dense<8> : vector<2xi64>,
+// BIG-DAG:     !llvm.ptr = dense<64> : vector<4xi64>,
+// BIG-DAG:     i128 = dense<128> : vector<2xi64>,
+// BIG-DAG:     !llvm.ptr<272> = dense<64> : vector<4xi64>,
+// BIG-DAG:     i64 = dense<64> : vector<2xi64>,
+// BIG-DAG:     !llvm.ptr<270> = dense<32> : vector<4xi64>,
+// BIG-DAG:     !llvm.ptr<271> = dense<32> : vector<4xi64>,
+// BIG-DAG:     f128 = dense<128> : vector<2xi64>,
+// BIG-DAG:     f16 = dense<16> : vector<2xi64>,
+// BIG-DAG:     f64 = dense<64> : vector<2xi64>,
+// BIG-DAG:     "dlti.stack_alignment" = 128 : i64
+// BIG-DAG:     "dlti.endianness" = "big"
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/dso-local.c b/external/llvm-project/clang/test/CIR/CodeGen/dso-local.c
new file mode 100644
index 000000000000..07c833d2fbc9
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/dso-local.c
@@ -0,0 +1,76 @@
+// This test is copied from clang/test/CodeGen/dso-local-executable.c with
+// unsupported targets, thread_local variables, and function checks removed.
+
+// These are here so we find this test when grepping for missing features.
+// cir::MissingFeatures::opGlobalThreadLocal()
+// cir::MissingFeatures::opFuncDsoLocal()
+
+/// Static relocation model defaults to -fdirect-access-external-data and sets
+/// dso_local on most global objects.
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -mrelocation-model static %s -o - | FileCheck --check-prefix=STATIC %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -mrelocation-model static -fdirect-access-external-data %s -o - | FileCheck --check-prefix=STATIC %s
+// STATIC:      @baz = dso_local global i32 42
+// STATIC-NEXT: @import_var = external dso_local global i32
+// STATIC-NEXT: @weak_bar = extern_weak dso_local global i32
+// STATIC-NEXT: @bar = external dso_local global i32
+
+/// If -fno-direct-access-external-data is set, drop dso_local from global variable
+/// declarations.
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -mrelocation-model static -fno-direct-access-external-data  %s -o - | FileCheck --check-prefix=STATIC-INDIRECT %s
+// STATIC-INDIRECT:      @baz = dso_local global i32 42
+// STATIC-INDIRECT-NEXT: @import_var = external global i32
+// STATIC-INDIRECT-NEXT: @weak_bar = extern_weak global i32
+// STATIC-INDIRECT-NEXT: @bar = external global i32
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 1 -pic-is-pie %s -o - | FileCheck --check-prefix=PIE %s
+// PIE:      @baz = dso_local global i32 42
+// PIE-NEXT: @import_var = external global i32
+// PIE-NEXT: @weak_bar = extern_weak global i32
+// PIE-NEXT: @bar = external global i32
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 1 -pic-is-pie -fdirect-access-external-data %s -o - | FileCheck --check-prefix=PIE-DIRECT %s
+// PIE-DIRECT:      @baz = dso_local global i32 42
+// PIE-DIRECT-NEXT: @import_var = external dso_local global i32
+// PIE-DIRECT-NEXT: @weak_bar = extern_weak global i32
+// PIE-DIRECT-NEXT: @bar = external dso_local global i32
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -mrelocation-model static -fno-plt %s -o - | FileCheck --check-prefix=NOPLT %s
+// NOPLT:      @baz = dso_local global i32 42
+// NOPLT-NEXT: @import_var = external dso_local global i32
+// NOPLT-NEXT: @weak_bar = extern_weak dso_local global i32
+// NOPLT-NEXT: @bar = external dso_local global i32
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -fno-plt -pic-level 1 -pic-is-pie -fdirect-access-external-data %s -o - | FileCheck --check-prefix=PIE-DIRECT-NOPLT %s
+// PIE-DIRECT-NOPLT:      @baz = dso_local global i32 42
+// PIE-DIRECT-NOPLT-NEXT: @import_var = external dso_local global i32
+// PIE-DIRECT-NOPLT-NEXT: @weak_bar = extern_weak global i32
+// PIE-DIRECT-NOPLT-NEXT: @bar = external dso_local global i32
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 1 -pic-is-pie -fno-plt %s -o - | FileCheck --check-prefix=PIE-NO-PLT %s
+// RUN: %clang_cc1 -triple powerpc64le -fclangir -emit-llvm -mrelocation-model static %s -o - | FileCheck --check-prefix=PIE-NO-PLT %s
+// PIE-NO-PLT:      @baz = dso_local global i32 42
+// PIE-NO-PLT-NEXT: @import_var = external global i32
+// PIE-NO-PLT-NEXT: @weak_bar = extern_weak global i32
+// PIE-NO-PLT-NEXT: @bar = external global i32
+
+/// -fdirect-access-external-data is currently ignored for -fPIC.
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 2 %s -o - | FileCheck --check-prefix=SHARED %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 2 -fdirect-access-external-data %s -o - | FileCheck --check-prefix=SHARED %s
+// SHARED-DAG: @bar = external global i32
+// SHARED-DAG: @weak_bar = extern_weak global i32
+// SHARED-DAG: @baz ={{.*}} global i32 42
+
+int baz = 42;
+__attribute__((dllimport)) extern int import_var;
+__attribute__((weak)) extern int weak_bar;
+extern int bar;
+
+int *use_import(void) {
+  return &import_var;
+}
+
+int *zed(void) {
+  if (baz)
+    return &weak_bar;
+  return &bar;
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/enum.cpp b/external/llvm-project/clang/test/CIR/CodeGen/enum.cpp
new file mode 100644
index 000000000000..5d9b1057aaa1
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/enum.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck %s --input-file=%t.cir
+
+enum Numbers {
+  Zero,
+  One,
+  Two,
+  Three
+};
+
+int f() {
+  return Numbers::One;
+}
+
+// CHECK: cir.func{{.*}} @_Z1fv
+// CHECK:    cir.const #cir.int<1> : !u32i
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/forrange.cpp b/external/llvm-project/clang/test/CIR/CodeGen/forrange.cpp
index 6b6ccc79e59d..45e146e9091d 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/forrange.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/forrange.cpp
@@ -115,8 +115,8 @@ void for_range3() {
 // CIR:    %[[C_ADDR:.*]] = cir.alloca !rec_C3{{.*}} ["c"]
 // CIR:    cir.scope {
 // CIR:      %[[RANGE_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C3>{{.*}} ["__range1", init, const]
-// CIR:      %[[BEGIN_ADDR:.*]] = cir.alloca !rec_Iterator, !cir.ptr<!rec_Iterator>{{.*}} ["__begin1"]
-// CIR:      %[[END_ADDR:.*]] = cir.alloca !rec_Iterator, !cir.ptr<!rec_Iterator>{{.*}} ["__end1"]
+// CIR:      %[[BEGIN_ADDR:.*]] = cir.alloca !rec_Iterator, !cir.ptr<!rec_Iterator>{{.*}} ["__begin1", init]
+// CIR:      %[[END_ADDR:.*]] = cir.alloca !rec_Iterator, !cir.ptr<!rec_Iterator>{{.*}} ["__end1", init]
 // CIR:      %[[E_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Element>{{.*}} ["e", init, const]
 // CIR:      cir.store{{.*}} %[[C_ADDR]], %[[RANGE_ADDR]]
 // CIR:      cir.for : cond {
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/forward-decls.cpp b/external/llvm-project/clang/test/CIR/CodeGen/forward-decls.cpp
new file mode 100644
index 000000000000..5a7e6b0afbed
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/forward-decls.cpp
@@ -0,0 +1,124 @@
+// RUN: split-file %s %t
+
+
+//--- incomplete_struct
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %t/incomplete_struct -o %t/incomplete_struct.cir
+// RUN: FileCheck %s --input-file=%t/incomplete_struct.cir --check-prefix=CHECK1
+
+// Forward declaration of the record is never defined, so it is created as
+// an incomplete struct in CIR and will remain as such.
+
+// CHECK1: ![[INC_STRUCT:.+]] = !cir.record<struct "IncompleteStruct" incomplete>
+struct IncompleteStruct;
+// CHECK1: testIncompleteStruct(%arg0: !cir.ptr<![[INC_STRUCT]]>
+void testIncompleteStruct(struct IncompleteStruct *s) {};
+
+
+
+//--- mutated_struct
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %t/mutated_struct -o %t/mutated_struct.cir
+// RUN: FileCheck %s --input-file=%t/mutated_struct.cir --check-prefix=CHECK2
+
+// Foward declaration of the struct is followed by usage, then definition.
+// This means it will initially be created as incomplete, then completed.
+
+// CHECK2: ![[COMPLETE:.+]] = !cir.record<struct "ForwardDeclaredStruct" {!s32i}>
+// CHECK2: testForwardDeclaredStruct(%arg0: !cir.ptr<![[COMPLETE]]>
+struct ForwardDeclaredStruct;
+void testForwardDeclaredStruct(struct ForwardDeclaredStruct *fds) {};
+struct ForwardDeclaredStruct {
+  int testVal;
+};
+
+
+
+//--- recursive_struct
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %t/recursive_struct -o %t/recursive_struct.cir
+// RUN: FileCheck --check-prefix=CHECK3 --input-file=%t/recursive_struct.cir %s
+
+// Struct is initially forward declared since the self-reference is generated
+// first. Then, once the type is fully generated, it is completed.
+
+// CHECK3: ![[STRUCT:.+]] = !cir.record<struct "RecursiveStruct" {!s32i, !cir.ptr<!cir.record<struct "RecursiveStruct">>}>
+struct RecursiveStruct {
+  int value;
+  struct RecursiveStruct *next;
+};
+// CHECK3: testRecursiveStruct(%arg0: !cir.ptr<![[STRUCT]]>
+void testRecursiveStruct(struct RecursiveStruct *arg) {
+  // CHECK3: %[[#NEXT:]] = cir.get_member %{{.+}}[1] {name = "next"} : !cir.ptr<![[STRUCT]]> -> !cir.ptr<!cir.ptr<![[STRUCT]]>>
+  // CHECK3: %[[#DEREF:]] = cir.load{{.*}} %[[#NEXT]] : !cir.ptr<!cir.ptr<![[STRUCT]]>>, !cir.ptr<![[STRUCT]]>
+  // CHECK3: cir.get_member %[[#DEREF]][0] {name = "value"} : !cir.ptr<![[STRUCT]]> -> !cir.ptr<!s32i>
+  arg->next->value;
+}
+
+
+
+//--- indirect_recursive_struct
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %t/indirect_recursive_struct -o %t/indirect_recursive_struct.cir
+// RUN: FileCheck --check-prefix=CHECK4 --input-file=%t/indirect_recursive_struct.cir %s
+
+// Node B refers to A, and vice-versa, so a forward declaration is used to
+// ensure the classes can be defined. Since types alias are not yet supported
+// in recursive type, each struct is expanded until there are no more recursive
+// types, or all the recursive types are self references.
+
+// CHECK4: ![[B:.+]] = !cir.record<struct "StructNodeB" {!s32i, !cir.ptr<!cir.record<struct "StructNodeA" {!s32i, !cir.ptr<!cir.record<struct "StructNodeB">>}
+// CHECK4: ![[A:.+]] = !cir.record<struct "StructNodeA" {!s32i, !cir.ptr<![[B]]>}>
+struct StructNodeB;
+struct StructNodeA {
+  int value;
+  struct StructNodeB *next;
+};
+struct StructNodeB {
+  int value;
+  struct StructNodeA *next;
+};
+
+void testIndirectSelfReference(struct StructNodeA arg) {
+  // CHECK4: %[[#V1:]] = cir.get_member %{{.+}}[1] {name = "next"} : !cir.ptr<![[A]]> -> !cir.ptr<!cir.ptr<![[B]]>>
+  // CHECK4: %[[#V2:]] = cir.load{{.*}} %[[#V1]] : !cir.ptr<!cir.ptr<![[B]]>>, !cir.ptr<![[B]]>
+  // CHECK4: %[[#V3:]] = cir.get_member %[[#V2]][1] {name = "next"} : !cir.ptr<![[B]]> -> !cir.ptr<!cir.ptr<![[A]]>>
+  // CHECK4: %[[#V4:]] = cir.load{{.*}} %[[#V3]] : !cir.ptr<!cir.ptr<![[A]]>>, !cir.ptr<![[A]]>
+  // CHECK4: cir.get_member %[[#V4]][0] {name = "value"} : !cir.ptr<![[A]]> -> !cir.ptr<!s32i>
+  arg.next->next->value;
+}
+
+
+
+//--- complex_struct
+
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %t/complex_struct -o %t/complex_struct.cir
+// RUN: FileCheck --check-prefix=CHECK5 --input-file=%t/complex_struct.cir %s
+
+// A sizeable complex struct just to double check that stuff is working.
+// CHECK5: !cir.record<struct "anon.0" {!cir.ptr<!cir.record<struct "A" {!cir.record<struct "anon.0">, !cir.record<struct "B" {!cir.ptr<!cir.record<struct "B">>, !cir.record<struct "C" {!cir.ptr<!cir.record<struct "A">>, !cir.ptr<!cir.record<struct "B">>, !cir.ptr<!cir.record<struct "C">>}>, !cir.record<union "anon.1" {!cir.ptr<!cir.record<struct "A">>, !cir.record<struct "anon.2" {!cir.ptr<!cir.record<struct "B">>}>}>}>}>>}>
+// CHECK5: !cir.record<struct "C" {!cir.ptr<!cir.record<struct "A" {!rec_anon2E0, !cir.record<struct "B" {!cir.ptr<!cir.record<struct "B">>, !cir.record<struct "C">, !cir.record<union "anon.1" {!cir.ptr<!cir.record<struct "A">>, !cir.record<struct "anon.2" {!cir.ptr<!cir.record<struct "B">>}>}>}>}>>, !cir.ptr<!cir.record<struct "B" {!cir.ptr<!cir.record<struct "B">>, !cir.record<struct "C">, !cir.record<union "anon.1" {!cir.ptr<!cir.record<struct "A" {!rec_anon2E0, !cir.record<struct "B">}>>, !cir.record<struct "anon.2" {!cir.ptr<!cir.record<struct "B">>}>}>}>>, !cir.ptr<!cir.record<struct "C">>}>
+// CHECK5: !cir.record<struct "anon.2" {!cir.ptr<!cir.record<struct "B" {!cir.ptr<!cir.record<struct "B">>, !rec_C, !cir.record<union "anon.1" {!cir.ptr<!cir.record<struct "A" {!rec_anon2E0, !cir.record<struct "B">}>>, !cir.record<struct "anon.2">}>}>>}>
+// CHECK5: !cir.record<union "anon.1" {!cir.ptr<!cir.record<struct "A" {!rec_anon2E0, !cir.record<struct "B" {!cir.ptr<!cir.record<struct "B">>, !rec_C, !cir.record<union "anon.1">}>}>>, !rec_anon2E2}>
+// CHECK5: !cir.record<struct "B" {!cir.ptr<!cir.record<struct "B">>, !rec_C, !rec_anon2E1}>
+// CHECK5: !cir.record<struct "A" {!rec_anon2E0, !rec_B}>
+struct A {
+  struct {
+    struct A *a1;
+  };
+  struct B {
+    struct B *b1;
+    struct C {
+      struct A *a2;
+      struct B *b2;
+      struct C *c1;
+    } c;
+    union {
+      struct A *a2;
+      struct {
+        struct B *b3;
+      };
+    } u;
+  } b;
+};
+void test(struct A *a){};
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/forward-enum.c b/external/llvm-project/clang/test/CIR/CodeGen/forward-enum.c
new file mode 100644
index 000000000000..f5479262908a
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/forward-enum.c
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck %s --input-file=%t.cir
+
+extern enum X x;
+void f(void) {
+  x;
+}
+
+enum X {
+  One,
+  Two
+};
+
+// CHECK: cir.global "private" external @x : !u32i
+// CHECK: cir.func{{.*}} @f
+// CHECK:   cir.get_global @x : !cir.ptr<!u32i>
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/inline-cxx-func.cpp b/external/llvm-project/clang/test/CIR/CodeGen/inline-cxx-func.cpp
new file mode 100644
index 000000000000..31d0255f18df
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/inline-cxx-func.cpp
@@ -0,0 +1,70 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+struct S {
+  int Member;
+
+  int InlineFunc() {
+    return Member;
+  }
+};
+
+// CIR: !rec_S = !cir.record<struct "S" {!s32i}>
+// LLVM: %struct.S = type { i32 }
+// OGCG: %struct.S = type { i32 }
+
+// CIR: cir.func @_ZN1S10InlineFuncEv(%arg0: !cir.ptr<!rec_S> {{.*}}) -> !s32i
+// CIR:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["this", init]
+// CIR:   %[[RET_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>
+// CIR:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+// CIR:   %[[MEMBER_ADDR:.*]] = cir.get_member %[[THIS]][0] {name = "Member"} : !cir.ptr<!rec_S> -> !cir.ptr<!s32i>
+// CIR:   %[[MEMBER:.*]] = cir.load{{.*}} %[[MEMBER_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.store %[[MEMBER]], %[[RET_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[RET_VAL:.*]] = cir.load %[[RET_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[RET_VAL]] : !s32i
+
+// LLVM: define{{.*}} i32 @_ZN1S10InlineFuncEv(ptr %[[ARG0:.*]])
+// LLVM:   %[[THIS_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[RET_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM:   store ptr %[[ARG0]], ptr %[[THIS_ADDR]], align 8
+// LLVM:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]], align 8
+// LLVM:   %[[MEMBER_ADDR:.*]] = getelementptr %struct.S, ptr %[[THIS]], i32 0, i32 0
+// LLVM:   %[[MEMBER:.*]] = load i32, ptr %[[MEMBER_ADDR]], align 4
+// LLVM:   store i32 %[[MEMBER]], ptr %[[RET_ADDR]], align 4
+// LLVM:   %[[RET_VAL:.*]] = load i32, ptr %[[RET_ADDR]], align 4
+// LLVM:   ret i32 %[[RET_VAL]]
+
+// The inlined function is defined after use() in OGCG
+
+void use() {
+  S s;
+  s.InlineFunc();
+}
+
+// CIR: cir.func @_Z3usev()
+// CIR:   %[[S_ADDR:.*]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["s"]
+// CIR:   %[[RET_VAL:.*]] = cir.call @_ZN1S10InlineFuncEv(%[[S_ADDR]]) : (!cir.ptr<!rec_S>) -> !s32i
+// CIR:   cir.return
+
+// LLVM: define{{.*}} void @_Z3usev()
+// LLVM:   %[[S_ADDR:.*]] = alloca %struct.S
+// LLVM:   %[[RET_VAL:.*]] = call i32 @_ZN1S10InlineFuncEv(ptr %[[S_ADDR]])
+// LLVM:   ret void
+
+// OGCG: define{{.*}} void @_Z3usev()
+// OGCG:   %[[S_ADDR:.*]] = alloca %struct.S
+// OGCG:   %[[RET_VAL:.*]] = call{{.*}} i32 @_ZN1S10InlineFuncEv(ptr{{.*}} %[[S_ADDR]])
+// OGCG:   ret void
+
+// OGCG: define{{.*}} i32 @_ZN1S10InlineFuncEv(ptr{{.*}} %[[ARG0:.*]])
+// OGCG:   %[[THIS_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   store ptr %[[ARG0]], ptr %[[THIS_ADDR]], align 8
+// OGCG:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]], align 8
+// OGCG:   %[[MEMBER_ADDR:.*]] = getelementptr inbounds nuw %struct.S, ptr %[[THIS]], i32 0, i32 0
+// OGCG:   %[[MEMBER:.*]] = load i32, ptr %[[MEMBER_ADDR]], align 4
+// OGCG:   ret i32 %[[MEMBER]]
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/libc.c b/external/llvm-project/clang/test/CIR/CodeGen/libc.c
new file mode 100644
index 000000000000..f65fe92cd36a
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/libc.c
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+
+// Note: In the final implementation, we will want these to generate
+//       CIR-specific libc operations. This test is just a placeholder
+//       to make sure we can compile these to normal function calls
+//       until the special handling is implemented.
+
+void *memcpy(void *, const void *, unsigned long);
+void testMemcpy(void *dst, const void *src, unsigned long size) {
+  memcpy(dst, src, size);
+  // CHECK: cir.call @memcpy
+}
+
+void *memmove(void *, const void *, unsigned long);
+void testMemmove(void *src, const void *dst, unsigned long size) {
+  memmove(dst, src, size);
+  // CHECK: cir.call @memmove
+}
+
+void *memset(void *, int, unsigned long);
+void testMemset(void *dst, int val, unsigned long size) {
+  memset(dst, val, size);
+  // CHECK: cir.call @memset
+}
+
+double fabs(double);
+double testFabs(double x) {
+  return fabs(x);
+  // CHECK: cir.call @fabs
+}
+
+float fabsf(float);
+float testFabsf(float x) {
+  return fabsf(x);
+  // CHECK: cir.call @fabsf
+}
+
+int abs(int);
+int testAbs(int x) {
+  return abs(x);
+  // CHECK: cir.call @abs
+}
+
+long labs(long);
+long testLabs(long x) {
+  return labs(x);
+  // CHECK: cir.call @labs
+}
+
+long long llabs(long long);
+long long testLlabs(long long x) {
+  return llabs(x);
+  // CHECK: cir.call @llabs
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/member-functions.cpp b/external/llvm-project/clang/test/CIR/CodeGen/member-functions.cpp
index 3d7dc8c4184a..c1d49ac4d8f3 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/member-functions.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/member-functions.cpp
@@ -13,6 +13,7 @@ void C::f() {}
 // CIR: cir.func @_ZN1C1fEv(%[[THIS_ARG:.*]]: !cir.ptr<!rec_C>
 // CIR:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["this", init]
 // CIR:   cir.store %[[THIS_ARG]], %[[THIS_ADDR]] : !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>
+// CIR:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_C>>, !cir.ptr<!rec_C>
 // CIR:   cir.return
 // CIR: }
 
@@ -23,8 +24,9 @@ void C::f2(int a, int b) {}
 // CIR-NEXT:   %[[A_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR-NEXT:   %[[B_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
 // CIR-NEXT:   cir.store %[[THIS_ARG]], %[[THIS_ADDR]] : !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>
-// CIR-NEXT:   cir.store %[[A_ARG]], %[[A_ADDR]] : !s32i, !cir.ptr<!s32i> loc(#loc12)
-// CIR-NEXT:   cir.store %[[B_ARG]], %[[B_ADDR]] : !s32i, !cir.ptr<!s32i> loc(#loc12)
+// CIR-NEXT:   cir.store %[[A_ARG]], %[[A_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR-NEXT:   cir.store %[[B_ARG]], %[[B_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR-NEXT:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_C>>, !cir.ptr<!rec_C>
 // CIR-NEXT:   cir.return
 // CIR-NEXT: }
 
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/namespace.cpp b/external/llvm-project/clang/test/CIR/CodeGen/namespace.cpp
index e0cf6333c119..14490d505140 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/namespace.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/namespace.cpp
@@ -20,7 +20,7 @@ namespace test {
   }
 }
 
-// CHECK-DAG: cir.global "private" internal @_ZN12_GLOBAL__N_12g1E = #cir.int<1> : !s32i
+// CHECK-DAG: cir.global "private" internal dsolocal @_ZN12_GLOBAL__N_12g1E = #cir.int<1> : !s32i
 // CHECK-DAG: cir.global external @_ZN4test2g2E = #cir.int<2> : !s32i
 // CHECK-DAG: cir.global external @_ZN4test5test22g3E = #cir.int<3> : !s32i
 // CHECK-DAG: cir.func @_ZN12_GLOBAL__N_12f1Ev()
@@ -50,3 +50,46 @@ int f4(void) {
 // CHECK:   %[[G3_ADDR:.*]] = cir.get_global @_ZN4test5test22g3E : !cir.ptr<!s32i>
 // CHECK:   %[[G3_VAL:.*]] = cir.load{{.*}} %[[G3_ADDR]] : !cir.ptr<!s32i>, !s32i
 // CHECK:   %[[SUM2:.*]] = cir.binop(add, %[[SUM]], %[[G3_VAL]]) nsw : !s32i
+
+using test2::f3;
+using test2::g3;
+
+int f5() {
+  f3();
+  return g3;
+}
+
+// CHECK: cir.func @_Z2f5v()
+// CHECK:   cir.call @_ZN4test5test22f3Ev()
+// CHECK:   %[[G3_ADDR:.*]] = cir.get_global @_ZN4test5test22g3E : !cir.ptr<!s32i>
+// CHECK:   %[[G3_VAL:.*]] = cir.load{{.*}} %[[G3_ADDR]] : !cir.ptr<!s32i>, !s32i
+
+namespace test3 {
+  struct S {
+    int a;
+  } s;
+}
+
+using test3::s;
+
+int f6() {
+  return s.a;
+}
+
+// CHECK: cir.func @_Z2f6v()
+// CHECK:   cir.get_global @_ZN5test31sE : !cir.ptr<!rec_test33A3AS>
+// CHECK:   cir.get_member %{{.*}}[0] {name = "a"}
+
+int shadowedFunc() {
+  return 3;
+}
+
+namespace shadow {
+  using ::shadowedFunc;
+}
+
+void f7() {
+  shadow::shadowedFunc();
+}
+
+// CHECK: cir.func @_Z2f7v()
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/static-vars.c b/external/llvm-project/clang/test/CIR/CodeGen/static-vars.c
new file mode 100644
index 000000000000..f45a41d9a00f
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/static-vars.c
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+
+void func1(void) {
+  // Should lower default-initialized static vars.
+  static int i;
+  // CHECK-DAG: cir.global "private" internal dsolocal @func1.i = #cir.int<0> : !s32i
+
+  // Should lower constant-initialized static vars.
+  static int j = 1;
+  // CHECK-DAG: cir.global "private" internal dsolocal @func1.j = #cir.int<1> : !s32i
+
+  // Should properly shadow static vars in nested scopes.
+  {
+    static int j = 2;
+    // CHECK-DAG: cir.global "private" internal dsolocal @func1.j.1 = #cir.int<2> : !s32i
+  }
+  {
+    static int j = 3;
+    // CHECK-DAG: cir.global "private" internal dsolocal @func1.j.2 = #cir.int<3> : !s32i
+  }
+
+  // Should lower basic static vars arithmetics.
+  j++;
+  // CHECK-DAG: %[[#V2:]] = cir.get_global @func1.j : !cir.ptr<!s32i>
+  // CHECK-DAG: %[[#V3:]] = cir.load{{.*}} %[[#V2]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-DAG: %[[#V4:]] = cir.unary(inc, %[[#V3]]) nsw : !s32i, !s32i
+  // CHECK-DAG: cir.store{{.*}} %[[#V4]], %[[#V2]] : !s32i, !cir.ptr<!s32i>
+}
+
+// Should shadow static vars on different functions.
+void func2(void) {
+  static char i;
+  // CHECK-DAG: cir.global "private" internal dsolocal @func2.i = #cir.int<0> : !s8i
+  static float j;
+  // CHECK-DAG: cir.global "private" internal dsolocal @func2.j = #cir.fp<0.000000e+00> : !cir.float
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/static-vars.cpp b/external/llvm-project/clang/test/CIR/CodeGen/static-vars.cpp
new file mode 100644
index 000000000000..9b892c69a6fe
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/static-vars.cpp
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t1.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t1.ll %s
+
+void func1(void) {
+  // Should lower default-initialized static vars.
+  static int i;
+  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1i = #cir.int<0> : !s32i
+
+  // Should lower constant-initialized static vars.
+  static int j = 1;
+  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1j = #cir.int<1> : !s32i
+
+  // Should properly shadow static vars in nested scopes.
+  {
+    static int j = 2;
+    // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1j_0 = #cir.int<2> : !s32i
+  }
+  {
+    static int j = 3;
+    // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1j_1 = #cir.int<3> : !s32i
+  }
+
+  // Should lower basic static vars arithmetics.
+  j++;
+  // CHECK-DAG: %[[#V2:]] = cir.get_global @_ZZ5func1vE1j : !cir.ptr<!s32i>
+  // CHECK-DAG: %[[#V3:]] = cir.load{{.*}} %[[#V2]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-DAG: %[[#V4:]] = cir.unary(inc, %[[#V3]]) nsw : !s32i, !s32i
+  // CHECK-DAG: cir.store{{.*}} %[[#V4]], %[[#V2]] : !s32i, !cir.ptr<!s32i>
+}
+
+// Should shadow static vars on different functions.
+void func2(void) {
+  static char i;
+  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func2vE1i = #cir.int<0> : !s8i
+  static float j;
+  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func2vE1j = #cir.fp<0.000000e+00> : !cir.float
+}
+
+// CHECK-DAG: cir.global linkonce_odr comdat @_ZZ4testvE1c = #cir.int<0> : !s32i
+
+// LLVM-DAG: $_ZZ4testvE1c = comdat any
+// LLVM-DAG: @_ZZ4testvE1c = linkonce_odr global i32 0, comdat, align 4
+
+inline void test() { static int c; }
+// CHECK-LABEL: @_Z4testv
+// CHECK: {{%.*}} = cir.get_global @_ZZ4testvE1c : !cir.ptr<!s32i>
+void foo() { test(); }
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/string-literals.c b/external/llvm-project/clang/test/CIR/CodeGen/string-literals.c
index 549fdcb74f48..00f59b09400c 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/string-literals.c
+++ b/external/llvm-project/clang/test/CIR/CodeGen/string-literals.c
@@ -5,13 +5,13 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-android21 -emit-llvm %s -o %t.ll
 // RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
 
-// CIR: cir.global "private" external @[[STR1_GLOBAL:.*]] = #cir.const_array<"1\00" : !cir.array<!s8i x 2>> : !cir.array<!s8i x 2>
-// CIR: cir.global "private" external @[[STR2_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 1>
-// CIR: cir.global "private" external @[[STR3_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 2>
+// CIR: cir.global "private" cir_private dsolocal @[[STR1_GLOBAL:.*]] = #cir.const_array<"1\00" : !cir.array<!s8i x 2>> : !cir.array<!s8i x 2>
+// CIR: cir.global "private" cir_private dsolocal @[[STR2_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 1>
+// CIR: cir.global "private" cir_private dsolocal @[[STR3_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 2>
 
-// LLVM: @[[STR1_GLOBAL:.*]] = global [2 x i8] c"1\00"
-// LLVM: @[[STR2_GLOBAL:.*]] = global [1 x i8] zeroinitializer
-// LLVM: @[[STR3_GLOBAL:.*]] = global [2 x i8] zeroinitializer
+// LLVM: @[[STR1_GLOBAL:.*]] = private global [2 x i8] c"1\00"
+// LLVM: @[[STR2_GLOBAL:.*]] = private global [1 x i8] zeroinitializer
+// LLVM: @[[STR3_GLOBAL:.*]] = private global [2 x i8] zeroinitializer
 
 // OGCG: @[[STR1_GLOBAL:.*]] = private unnamed_addr constant [2 x i8] c"1\00"
 // OGCG: @[[STR2_GLOBAL:.*]] = private unnamed_addr constant [1 x i8] zeroinitializer
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/template-specialization.cpp b/external/llvm-project/clang/test/CIR/CodeGen/template-specialization.cpp
new file mode 100644
index 000000000000..5151f8ce1542
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/template-specialization.cpp
@@ -0,0 +1,103 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+template<typename T, typename U>
+class Templ {};
+
+template<typename T>
+class Templ<T, int>{};
+
+Templ<int, int> t;
+
+// CIR: !rec_Templ3Cint2C_int3E = !cir.record<class "Templ<int, int>" padded {!u8i}>
+// CIR: cir.global external @t = #cir.zero : !rec_Templ3Cint2C_int3E
+
+// LLVM: %"class.Templ<int, int>" = type { i8 }
+// LLVM: @t = global %"class.Templ<int, int>" zeroinitializer
+
+// OGCG: %class.Templ = type { i8 }
+// OGCG: @t = global %class.Templ zeroinitializer
+
+template<class T>
+class X {
+public:
+  int f() { return 0; }
+};
+
+template<> class X<int> {
+public:
+  int f() { return 1; }
+};
+
+void test_double() {
+  X<double> d;
+  d.f();
+}
+
+// CIR: cir.func{{.*}} @_ZN1XIdE1fEv
+// CIR:   cir.const #cir.int<0>
+//
+// CIR: cir.func{{.*}} @_Z11test_doublev()
+// CIR:   cir.call @_ZN1XIdE1fEv
+
+// LLVM: define{{.*}} i32 @_ZN1XIdE1fEv
+// LLVM:   store i32 0
+//
+// LLVM: define{{.*}} void @_Z11test_doublev()
+// LLVM:   call i32 @_ZN1XIdE1fEv
+
+// OGCG: define{{.*}} void @_Z11test_doublev()
+// OGCG:   call{{.*}} i32 @_ZN1XIdE1fEv
+//
+// OGCG: define{{.*}} i32 @_ZN1XIdE1fEv
+// OGCG:   ret i32 0
+
+void test_int() {
+  X<int> n;
+  n.f();
+}
+
+// CIR: cir.func{{.*}} @_ZN1XIiE1fEv
+// CIR:   cir.const #cir.int<1>
+//
+// CIR: cir.func{{.*}} @_Z8test_intv()
+// CIR:   cir.call @_ZN1XIiE1fEv
+
+// LLVM: define{{.*}} i32 @_ZN1XIiE1fEv
+// LLVM:   store i32 1
+//
+// LLVM: define{{.*}} void @_Z8test_intv()
+// LLVM:   call i32 @_ZN1XIiE1fEv
+
+// OGCG: define{{.*}} void @_Z8test_intv()
+// OGCG:   call{{.*}} i32 @_ZN1XIiE1fEv
+//
+// OGCG: define{{.*}} i32 @_ZN1XIiE1fEv
+// OGCG:   ret i32 1
+
+void test_short() {
+  X<short> s;
+  s.f();
+}
+
+// CIR: cir.func{{.*}} @_ZN1XIsE1fEv
+// CIR:   cir.const #cir.int<0>
+//
+// CIR: cir.func{{.*}} @_Z10test_shortv()
+// CIR:   cir.call @_ZN1XIsE1fEv
+
+// LLVM: define{{.*}} i32 @_ZN1XIsE1fEv
+// LLVM: store i32 0
+//
+// LLVM: define{{.*}} void @_Z10test_shortv()
+// LLVM:   call i32 @_ZN1XIsE1fEv
+
+// OGCG: define{{.*}} void @_Z10test_shortv()
+// OGCG:   call{{.*}} i32 @_ZN1XIsE1fEv
+//
+// OGCG: define{{.*}} i32 @_ZN1XIsE1fEv
+// OGCG:   ret i32 0
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/ternary.cpp b/external/llvm-project/clang/test/CIR/CodeGen/ternary.cpp
new file mode 100644
index 000000000000..3b66f7ccdf54
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGen/ternary.cpp
@@ -0,0 +1,147 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+int x(int y) {
+  return y > 0 ? 3 : 5;
+}
+
+// CIR-LABEL: cir.func @_Z1xi(
+// CIR-SAME: %[[ARG0:.*]]: !s32i {{.*}}) -> !s32i {
+// CIR: [[Y:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["y", init] {alignment = 4 : i64}
+// CIR: [[RETVAL:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
+// CIR: cir.store %[[ARG0]], [[Y]] : !s32i, !cir.ptr<!s32i>
+// CIR: [[YVAL:%.+]] = cir.load align(4) [[Y]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[ZERO:%.+]] = cir.const #cir.int<0> : !s32i
+// CIR: [[CMP:%.+]] = cir.cmp(gt, [[YVAL]], [[ZERO]]) : !s32i, !cir.bool
+// CIR: [[THREE:%.+]] = cir.const #cir.int<3> : !s32i
+// CIR: [[FIVE:%.+]] = cir.const #cir.int<5> : !s32i
+// CIR: [[SELECT_RES:%.+]] = cir.select if [[CMP]] then [[THREE]] else [[FIVE]] : (!cir.bool, !s32i, !s32i) -> !s32i
+// CIR: cir.store [[SELECT_RES]], [[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: [[RETVAL_VAL:%.+]] = cir.load [[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return [[RETVAL_VAL]] : !s32i
+
+// LLVM-LABEL: define i32 @_Z1xi(
+// LLVM-SAME: i32 %[[ARG0:.+]])
+// LLVM: %[[Y:.*]] = alloca i32
+// LLVM: %[[RETVAL:.*]] = alloca i32
+// LLVM: store i32 %[[ARG0]], ptr %[[Y]]
+// LLVM: %[[YVAL:.*]] = load i32, ptr %[[Y]]
+// LLVM: %[[CMP:.*]] = icmp sgt i32 %[[YVAL]], 0
+// LLVM: %[[SELECT:.*]] = select i1 %[[CMP]], i32 3, i32 5
+// LLVM: store i32 %[[SELECT]], ptr %[[RETVAL]]
+// LLVM: %[[RESULT:.*]] = load i32, ptr %[[RETVAL]]
+// LLVM: ret i32 %[[RESULT]]
+
+// OGCG-LABEL: define dso_local noundef i32 @_Z1xi(
+// OGCG-SAME: i32 {{.*}} %[[ARG0:.+]])
+// OGCG: %[[Y:.*]] = alloca i32
+// OGCG: store i32 %[[ARG0]], ptr %[[Y]]
+// OGCG: %[[YVAL:.*]] = load i32, ptr %[[Y]]
+// OGCG: %[[CMP:.*]] = icmp sgt i32 %[[YVAL]], 0
+// OGCG: %[[SELECT:.*]] = select i1 %[[CMP]], i32 3, i32 5
+// OGCG: ret i32 %[[SELECT]]
+
+int foo(int a, int b) {
+  if (a < b ? 0 : a)
+    return -1;
+  return 0;
+}
+
+// CIR-LABEL: cir.func @_Z3fooii(
+// CIR-SAME: %[[ARG0:.*]]: !s32i {{.*}}, %[[ARG1:.*]]: !s32i {{.*}}) -> !s32i {
+// CIR: [[A:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init] {alignment = 4 : i64}
+// CIR: [[B:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init] {alignment = 4 : i64}
+// CIR: [[RETVAL:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
+// CIR: cir.store %[[ARG0]], [[A]] : !s32i, !cir.ptr<!s32i>
+// CIR: cir.store %[[ARG1]], [[B]] : !s32i, !cir.ptr<!s32i>
+// CIR: cir.scope {
+// CIR: [[ALOAD:%.+]] = cir.load align(4) [[A]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[BLOAD:%.+]] = cir.load align(4) [[B]] : !cir.ptr<!s32i>, !s32i
+// CIR: [[CMP:%.+]] = cir.cmp(lt, [[ALOAD]], [[BLOAD]]) : !s32i, !cir.bool
+// CIR: [[TERNARY_RES:%.+]] = cir.ternary([[CMP]], true {
+// CIR: [[ZERO:%.+]] = cir.const #cir.int<0> : !s32i
+// CIR: cir.yield [[ZERO]] : !s32i
+// CIR: }, false {
+// CIR: [[ALOAD2:%.+]] = cir.load align(4) [[A]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.yield [[ALOAD2]] : !s32i
+// CIR: }) : (!cir.bool) -> !s32i
+// CIR: [[CAST:%.+]] = cir.cast(int_to_bool, [[TERNARY_RES]] : !s32i), !cir.bool
+// CIR: cir.if [[CAST]] {
+// CIR: [[ONE:%.+]] = cir.const #cir.int<1> : !s32i
+// CIR: [[MINUS_ONE:%.+]] = cir.unary(minus, [[ONE]]) nsw : !s32i, !s32i
+// CIR: cir.store [[MINUS_ONE]], [[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: [[RETVAL_VAL:%.+]] = cir.load [[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return [[RETVAL_VAL]] : !s32i
+// CIR: }
+// CIR: }
+// CIR: [[ZERO2:%.+]] = cir.const #cir.int<0> : !s32i
+// CIR: cir.store [[ZERO2]], [[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: [[RETVAL_VAL2:%.+]] = cir.load [[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return [[RETVAL_VAL2]] : !s32i
+
+// LLVM-LABEL: define i32 @_Z3fooii(
+// LLVM-SAME: i32 %[[ARG0:.*]], i32 %[[ARG1:.*]])
+// LLVM: %[[A:.*]] = alloca i32
+// LLVM: %[[B:.*]] = alloca i32
+// LLVM: %[[RETVAL:.*]] = alloca i32
+// LLVM: store i32 %[[ARG0]], ptr %[[A]]
+// LLVM: store i32 %[[ARG1]], ptr %[[B]]
+// LLVM: br label %[[ENTRY_BB:.*]]
+// LLVM: [[ENTRY_BB]]:
+// LLVM: %[[AVAL:.*]] = load i32, ptr %[[A]]
+// LLVM: %[[BVAL:.*]] = load i32, ptr %[[B]]
+// LLVM: %[[CMP:.*]] = icmp slt i32 %[[AVAL]], %[[BVAL]]
+// LLVM: br i1 %[[CMP]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVM: [[TRUE_BB]]:
+// LLVM: br label %[[MERGE_BB:.*]]
+// LLVM: [[FALSE_BB]]:
+// LLVM: %[[AVAL2:.*]] = load i32, ptr %[[A]]
+// LLVM: br label %[[MERGE_BB]]
+// LLVM: [[MERGE_BB]]:
+// LLVM: %[[PHI:.*]] = phi i32 [ %[[AVAL2]], %[[FALSE_BB]] ], [ 0, %[[TRUE_BB]] ]
+// LLVM: %[[COND:.*]] = icmp ne i32 %[[PHI]], 0
+// LLVM: br i1 %[[COND]], label %[[RETURN_MINUS_ONE:.*]], label %[[CONT_BB:.*]]
+// LLVM: [[RETURN_MINUS_ONE]]:
+// LLVM: store i32 -1, ptr %[[RETVAL]]
+// LLVM: %[[RET1:.*]] = load i32, ptr %[[RETVAL]]
+// LLVM: ret i32 %[[RET1]]
+// LLVM: [[CONT_BB]]:
+// LLVM: br label %[[RETURN_ZERO:.*]]
+// LLVM: [[RETURN_ZERO]]:
+// LLVM: store i32 0, ptr %[[RETVAL]]
+// LLVM: %[[RET2:.*]] = load i32, ptr %[[RETVAL]]
+// LLVM: ret i32 %[[RET2]]
+
+// OGCG-LABEL: define dso_local noundef i32 @_Z3fooii(
+// OGCG-SAME: i32 {{.*}} %[[ARG0:.*]], i32 {{.*}} %[[ARG1:.*]])
+// OGCG: %[[RETVAL:.*]] = alloca i32
+// OGCG: %[[A:.*]] = alloca i32
+// OGCG: %[[B:.*]] = alloca i32
+// OGCG: store i32 %[[ARG0]], ptr %[[A]]
+// OGCG: store i32 %[[ARG1]], ptr %[[B]]
+// OGCG: %[[AVAL:.*]] = load i32, ptr %[[A]]
+// OGCG: %[[BVAL:.*]] = load i32, ptr %[[B]]
+// OGCG: %[[CMP:.*]] = icmp slt i32 %[[AVAL]], %[[BVAL]]
+// OGCG: br i1 %[[CMP]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// OGCG: [[TRUE_BB]]:
+// OGCG: br label %[[MERGE_BB:.*]]
+// OGCG: [[FALSE_BB]]:
+// OGCG: %[[AVAL2:.*]] = load i32, ptr %[[A]]
+// OGCG: br label %[[MERGE_BB]]
+// OGCG: [[MERGE_BB]]:
+// OGCG: %[[PHI:.*]] = phi i32 [ 0, %[[TRUE_BB]] ], [ %[[AVAL2]], %[[FALSE_BB]] ]
+// OGCG: %[[COND:.*]] = icmp ne i32 %[[PHI]], 0
+// OGCG: br i1 %[[COND]], label %[[RETURN_MINUS_ONE:.*]], label %[[RETURN_ZERO:.*]]
+// OGCG: [[RETURN_MINUS_ONE]]:
+// OGCG: store i32 -1, ptr %[[RETVAL]]
+// OGCG: br label %[[RETURN:.+]]
+// OGCG: [[RETURN_ZERO]]:
+// OGCG: store i32 0, ptr %[[RETVAL]]
+// OGCG: br label %[[RETURN]]
+// OGCG: [[RETURN]]:
+// OGCG: %[[RET2:.*]] = load i32, ptr %[[RETVAL]]
+// OGCG: ret i32 %[[RET2]]
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/vector-ext.cpp b/external/llvm-project/clang/test/CIR/CodeGen/vector-ext.cpp
index 8a0479fc1d08..fe4919ec0478 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/vector-ext.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/vector-ext.cpp
@@ -77,12 +77,8 @@ void foo() {
 // CIR: %[[VEC_F:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["f", init]
 // CIR: %[[VEC_G:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["g", init]
 // CIR: %[[VEC_H:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["h", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_E_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_E_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_E_VAL]], %[[VEC_E]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[GLOBAL_X:.*]] = cir.get_global @x : !cir.ptr<!s32i>
 // CIR: %[[X_VAL:.*]] = cir.load{{.*}} %[[GLOBAL_X]] : !cir.ptr<!s32i>, !s32i
@@ -95,13 +91,11 @@ void foo() {
 // CIR: %[[VEC_F_VAL:.*]] = cir.vec.create(%[[X_VAL]], %[[CONST_5]], %[[CONST_6]], %[[X_PLUS_1]] :
 // CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_F_VAL]], %[[VEC_F]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[CONST_5:.*]] = cir.const #cir.int<5> : !s32i
-// CIR: %[[CONST_0:.*]] = cir.const #cir.int<0> : !s32i
-// CIR: %[[VEC_G_VAL:.*]] = cir.vec.create(%[[CONST_5]], %[[CONST_0]], %[[CONST_0]], %[[CONST_0]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_G_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<0> : !s32i,
+// CIR-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_G_VAL]], %[[VEC_G]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
-// CIR: %[[VEC_H_VAL:.*]] = cir.vec.create(%[[ZERO]], %[[ZERO]], %[[ZERO]], %[[ZERO]] : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_H_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
+// CIR-SAME; #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_H_VAL]], %[[VEC_H]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 
 // LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
@@ -148,12 +142,8 @@ void foo3() {
 
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[IDX:.*]] = cir.const #cir.int<1> : !s32i
@@ -184,12 +174,8 @@ void foo4() {
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[IDX:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["idx", init]
 // CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
 // CIR: cir.store{{.*}} %[[CONST_IDX]], %[[IDX]] : !s32i, !cir.ptr<!s32i>
@@ -225,12 +211,8 @@ void foo5() {
 }
 
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_VAL:.*]] = cir.const #cir.int<5> : !s32i
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
@@ -260,12 +242,8 @@ void foo6() {
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[IDX:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["idx", init]
 // CIR: %[[VAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["value", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
 // CIR: cir.store{{.*}} %[[CONST_IDX]], %[[IDX]] : !s32i, !cir.ptr<!s32i>
@@ -307,12 +285,8 @@ void foo7() {
 }
 
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_VAL:.*]] = cir.const #cir.int<5> : !s32i
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
@@ -353,12 +327,8 @@ void foo8() {
 // CIR: %[[PLUS_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["plus_res", init]
 // CIR: %[[MINUS_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["minus_res", init]
 // CIR: %[[NOT_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["not_res", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP1:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[PLUS:.*]] = cir.unary(plus, %[[TMP1]]) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
@@ -410,19 +380,11 @@ void foo9() {
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b", init]
 // CIR: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
 // CIR: %[[SHR_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shr", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[CONST_5:.*]] = cir.const #cir.int<5> : !s32i
-// CIR: %[[CONST_6:.*]] = cir.const #cir.int<6> : !s32i
-// CIR: %[[CONST_7:.*]] = cir.const #cir.int<7> : !s32i
-// CIR: %[[CONST_8:.*]] = cir.const #cir.int<8> : !s32i
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create(%[[CONST_5]], %[[CONST_6]], %[[CONST_7]], %[[CONST_8]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<6> : !s32i,
+// CIR-SAME: #cir.int<7> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
@@ -475,9 +437,11 @@ void foo10() {
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["b", init]
 // CIR: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
 // CIR: %[[SHR_RES:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["shr", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} :  !u32i, !u32i, !u32i, !u32i) : !cir.vector<4 x !u32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !u32i, #cir.int<6> : !u32i,
+// CIR-SAME: #cir.int<7> : !u32i, #cir.int<8> : !u32i]> : !cir.vector<4 x !u32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
@@ -534,11 +498,11 @@ void foo11() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<6> : !s32i,
+// CIR-SAME: #cir.int<7> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
@@ -661,11 +625,11 @@ void foo12() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<6> : !s32i,
+// CIR-SAME: #cir.int<7> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
@@ -776,11 +740,11 @@ void foo13() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !u32i, !u32i, !u32i, !u32i) :
-// CIR-SAME: !cir.vector<4 x !u32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !u32i, #cir.int<2> : !u32i,
+// CIR-SAME: #cir.int<3> : !u32i, #cir.int<4> : !u32i]> : !cir.vector<4 x !u32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !u32i, !u32i, !u32i, !u32i) :
-// CIR-SAME: !cir.vector<4 x !u32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !u32i, #cir.int<6> : !u32i,
+// CIR-SAME: #cir.int<7> : !u32i, #cir.int<8> : !u32i]> : !cir.vector<4 x !u32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
@@ -891,11 +855,11 @@ void foo14() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !cir.float, !cir.float, !cir.float, !cir.float) :
-// CIR-SAME: !cir.vector<4 x !cir.float>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.fp<1.000000e+00> : !cir.float, #cir.fp<2.000000e+00> : !cir.float,
+// CIR-SAME: #cir.fp<3.000000e+00> : !cir.float, #cir.fp<4.000000e+00> : !cir.float]> : !cir.vector<4 x !cir.float>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !cir.float, !cir.float, !cir.float, !cir.float) :
-// CIR-SAME: !cir.vector<4 x !cir.float>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.fp<5.000000e+00> : !cir.float, #cir.fp<6.000000e+00> : !cir.float,
+// CIR-SAME: #cir.fp<7.000000e+00> : !cir.float, #cir.fp<8.000000e+00> : !cir.float]> : !cir.vector<4 x !cir.float>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !cir.float>>, !cir.vector<4 x !cir.float>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !cir.float>>, !cir.vector<4 x !cir.float>
@@ -990,6 +954,7 @@ void foo14() {
 // OGCG: %[[TMP_B:.*]] = load <4 x float>, ptr %[[VEC_B]], align 16
 // OGCG: %[[GE:.*]] = fcmp oge <4 x float> %[[TMP_A]], %[[TMP_B]]
 // OGCG: %[[RES:.*]] = sext <4 x i1> %[[GE]] to <4 x i32>
+// OGCG: store <4 x i32> %[[RES]], ptr {{.*}}, align 16
 
 void foo15() {
   vi4 a;
@@ -1091,3 +1056,108 @@ void foo17() {
 // OGCG: %[[VEC_A:.*]] = alloca <2 x double>, align 16
 // OGCG: %[[TMP:.*]] = load <2 x double>, ptr %[[VEC_A]], align 16
 // OGCG: %[[RES:.*]]= fptoui <2 x double> %[[TMP]] to <2 x i16>
+
+void foo18() {
+  vi4 a = {1, 2, 3, 4};
+  vi4 shl = a << 3;
+
+  uvi4 b = {1u, 2u, 3u, 4u};
+  uvi4 shr = b >> 3u;
+}
+
+// CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
+// CIR: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
+// CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["b", init]
+// CIR: %[[SHR_RES:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["shr", init]
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
+// CIR-SAME: #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+// CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[SH_AMOUNT:.*]] = cir.const #cir.int<3> : !s32i
+// CIR: %[[SPLAT_VEC:.*]] = cir.vec.splat %[[SH_AMOUNT]] : !s32i, !cir.vector<4 x !s32i>
+// CIR: %[[SHL:.*]] = cir.shift(left, %[[TMP_A]] : !cir.vector<4 x !s32i>, %[[SPLAT_VEC]] : !cir.vector<4 x !s32i>) -> !cir.vector<4 x !s32i>
+// CIR: cir.store{{.*}} %[[SHL]], %[[SHL_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !u32i, #cir.int<2> : !u32i,
+// CIR-SAME: #cir.int<3> : !u32i, #cir.int<4> : !u32i]> : !cir.vector<4 x !u32i>
+// CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
+// CIR: %[[SH_AMOUNT:.*]] = cir.const #cir.int<3> : !u32i
+// CIR: %[[SPLAT_VEC:.*]] = cir.vec.splat %[[SH_AMOUNT]] : !u32i, !cir.vector<4 x !u32i>
+// CIR: %[[SHR:.*]] = cir.shift(right, %[[TMP_B]] : !cir.vector<4 x !u32i>, %[[SPLAT_VEC]] : !cir.vector<4 x !u32i>) -> !cir.vector<4 x !u32i>
+// CIR: cir.store{{.*}} %[[SHR]], %[[SHR_RES]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
+
+// LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[SHL_RES:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[VEC_B:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[SHR_RES:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_A]], align 16
+// LLVM: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// LLVM: %[[SHL:.*]] = shl <4 x i32> %[[TMP_A]], splat (i32 3)
+// LLVM: store <4 x i32> %[[SHL]], ptr %[[SHL_RES]], align 16
+// LLVM: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_B]], align 16
+// LLVM: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// LLVM: %[[SHR:.*]] = lshr <4 x i32> %[[TMP_B]], splat (i32 3)
+// LLVM: store <4 x i32> %[[SHR]], ptr %[[SHR_RES]], align 16
+
+// OGCG: %[[VEC_A:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[SHL_RES:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[VEC_B:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[SHR_RES:.*]] = alloca <4 x i32>, align 16
+// OGCG: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_A]], align 16
+// OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// OGCG: %[[SHL:.*]] = shl <4 x i32> %[[TMP_A]], splat (i32 3)
+// OGCG: store <4 x i32> %[[SHL]], ptr %[[SHL_RES]], align 16
+// OGCG: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_B]], align 16
+// OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// OGCG: %[[SHR:.*]] = lshr <4 x i32> %[[TMP_B]], splat (i32 3)
+// OGCG: store <4 x i32> %[[SHR]], ptr %[[SHR_RES]], align 16
+
+void foo19() {
+  vi4 a;
+  vi4 b;
+  vi4 u = __builtin_shufflevector(a, b, 7, 5, 3, 1);
+}
+
+// CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a"]
+// CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[SHUF:.*]] = cir.vec.shuffle(%[[TMP_A]], %[[TMP_B]] : !cir.vector<4 x !s32i>) [#cir.int<7> :
+// CIR-SAME: !s64i, #cir.int<5> : !s64i, #cir.int<3> : !s64i, #cir.int<1> : !s64i] : !cir.vector<4 x !s32i>
+
+// LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[VEC_B:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// LLVM: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// LLVM: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 7, i32 5, i32 3, i32 1>
+
+// OGCG: %[[VEC_A:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[VEC_B:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// OGCG: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 7, i32 5, i32 3, i32 1>
+
+void foo20() {
+  vi4 a;
+  vi4 b;
+  vi4 u = __builtin_shufflevector(a, b, -1, 1, -1, 1);
+}
+
+// CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a"]
+// CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[SHUF:.*]] = cir.vec.shuffle(%[[TMP_A]], %[[TMP_B]] : !cir.vector<4 x !s32i>) [#cir.int<-1> :
+// CIR-SAME: !s64i, #cir.int<1> : !s64i, #cir.int<-1> : !s64i, #cir.int<1> : !s64i] : !cir.vector<4 x !s32i>
+
+// LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[VEC_B:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// LLVM: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// LLVM: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
+
+// OGCG: %[[VEC_A:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[VEC_B:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// OGCG: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
diff --git a/external/llvm-project/clang/test/CIR/CodeGen/vector.cpp b/external/llvm-project/clang/test/CIR/CodeGen/vector.cpp
index 4c50f68a5616..d0c5b83cd5b0 100644
--- a/external/llvm-project/clang/test/CIR/CodeGen/vector.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGen/vector.cpp
@@ -66,12 +66,8 @@ void foo() {
 // CIR: %[[VEC_E:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["e", init]
 // CIR: %[[VEC_F:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["f", init]
 // CIR: %[[VEC_G:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["g", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_D_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_D_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_D_VAL]], %[[VEC_D]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[GLOBAL_X:.*]] = cir.get_global @x : !cir.ptr<!s32i>
 // CIR: %[[X_VAL:.*]] = cir.load{{.*}} %[[GLOBAL_X]] : !cir.ptr<!s32i>, !s32i
@@ -84,14 +80,11 @@ void foo() {
 // CIR: %[[VEC_E_VAL:.*]] = cir.vec.create(%[[X_VAL]], %[[CONST_5]], %[[CONST_6]], %[[X_PLUS_1]] :
 // CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_E_VAL]], %[[VEC_E]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[CONST_5:.*]] = cir.const #cir.int<5> : !s32i
-// CIR: %[[CONST_0:.*]] = cir.const #cir.int<0> : !s32i
-// CIR: %[[VEC_F_VAL:.*]] = cir.vec.create(%[[CONST_5]], %[[CONST_0]], %[[CONST_0]], %[[CONST_0]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_F_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<0> : !s32i,
+// CIR-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_F_VAL]], %[[VEC_F]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[CONST_0:.*]] = cir.const #cir.int<0> : !s32i
-// CIR: %[[VEC_G_VAL:.*]] = cir.vec.create(%[[CONST_0]], %[[CONST_0]], %[[CONST_0]], %[[CONST_0]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_G_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
+// CIR-SAME; #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_G_VAL]], %[[VEC_G]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 
 // LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
@@ -136,12 +129,8 @@ void foo3() {
 
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[IDX:.*]] = cir.const #cir.int<1> : !s32i
@@ -172,12 +161,8 @@ void foo4() {
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[IDX:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["idx", init]
 // CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
 // CIR: cir.store{{.*}} %[[CONST_IDX]], %[[IDX]] : !s32i, !cir.ptr<!s32i>
@@ -213,12 +198,8 @@ void foo5() {
 }
 
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_VAL:.*]] = cir.const #cir.int<5> : !s32i
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
@@ -248,12 +229,8 @@ void foo6() {
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[IDX:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["idx", init]
 // CIR: %[[VAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["value", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
 // CIR: cir.store{{.*}} %[[CONST_IDX]], %[[IDX]] : !s32i, !cir.ptr<!s32i>
@@ -295,12 +272,8 @@ void foo7() {
 }
 
 // CIR: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[CONST_VAL:.*]] = cir.const #cir.int<5> : !s32i
 // CIR: %[[CONST_IDX:.*]] = cir.const #cir.int<2> : !s32i
@@ -341,12 +314,8 @@ void foo8() {
 // CIR: %[[PLUS_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["plus_res", init]
 // CIR: %[[MINUS_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["minus_res", init]
 // CIR: %[[NOT_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["not_res", init]
-// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
-// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
-// CIR: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
-// CIR: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
-// CIR: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] :
-// CIR-SAME: !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP1:.*]] = cir.load{{.*}} %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[PLUS:.*]] = cir.unary(plus, %[[TMP1]]) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
@@ -398,9 +367,11 @@ void foo9() {
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b", init]
 // CIR: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
 // CIR: %[[SHR_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shr", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} :  !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<6> : !s32i,
+// CIR-SAME: #cir.int<7> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
@@ -453,9 +424,11 @@ void foo10() {
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["b", init]
 // CIR: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
 // CIR: %[[SHR_RES:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["shr", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} :  !u32i, !u32i, !u32i, !u32i) : !cir.vector<4 x !u32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !u32i, #cir.int<6> : !u32i,
+// CIR-SAME: #cir.int<7> : !u32i, #cir.int<8> : !u32i]> : !cir.vector<4 x !u32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
@@ -512,11 +485,11 @@ void foo11() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<6> : !s32i,
+// CIR-SAME: #cir.int<7> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
@@ -639,11 +612,11 @@ void foo12() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !s32i, !s32i, !s32i, !s32i) :
-// CIR-SAME: !cir.vector<4 x !s32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<6> : !s32i,
+// CIR-SAME: #cir.int<7> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
@@ -754,11 +727,11 @@ void foo13() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !u32i, !u32i, !u32i, !u32i) :
-// CIR-SAME: !cir.vector<4 x !u32i>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !u32i, #cir.int<2> : !u32i,
+// CIR-SAME: #cir.int<3> : !u32i, #cir.int<4> : !u32i]> : !cir.vector<4 x !u32i>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !u32i, !u32i, !u32i, !u32i) :
-// CIR-SAME: !cir.vector<4 x !u32i>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<5> : !u32i, #cir.int<6> : !u32i,
+// CIR-SAME: #cir.int<7> : !u32i, #cir.int<8> : !u32i]> : !cir.vector<4 x !u32i>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
@@ -869,11 +842,11 @@ void foo14() {
 
 // CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>, ["a", init]
 // CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>, ["b", init]
-// CIR: %[[VEC_A_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !cir.float, !cir.float, !cir.float, !cir.float) :
-// CIR-SAME: !cir.vector<4 x !cir.float>
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.fp<1.000000e+00> : !cir.float, #cir.fp<2.000000e+00> : !cir.float,
+// CIR-SAME: #cir.fp<3.000000e+00> : !cir.float, #cir.fp<4.000000e+00> : !cir.float]> : !cir.vector<4 x !cir.float>
 // CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>
-// CIR: %[[VEC_B_VAL:.*]] = cir.vec.create({{.*}}, {{.*}}, {{.*}}, {{.*}} : !cir.float, !cir.float, !cir.float, !cir.float) :
-// CIR-SAME: !cir.vector<4 x !cir.float>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.fp<5.000000e+00> : !cir.float, #cir.fp<6.000000e+00> : !cir.float,
+// CIR-SAME: #cir.fp<7.000000e+00> : !cir.float, #cir.fp<8.000000e+00> : !cir.float]> : !cir.vector<4 x !cir.float>
 // CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !cir.float>, !cir.ptr<!cir.vector<4 x !cir.float>>
 // CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !cir.float>>, !cir.vector<4 x !cir.float>
 // CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !cir.float>>, !cir.vector<4 x !cir.float>
@@ -1069,4 +1042,164 @@ void foo17() {
 
 // OGCG: %[[VEC_A:.*]] = alloca <2 x double>, align 16
 // OGCG: %[[TMP:.*]] = load <2 x double>, ptr %[[VEC_A]], align 16
-// OGCG: %[[RES:.*]]= fptoui <2 x double> %[[TMP]] to <2 x i16>
\ No newline at end of file
+// OGCG: %[[RES:.*]]= fptoui <2 x double> %[[TMP]] to <2 x i16>
+
+void foo18() {
+  vi4 a = {1, 2, 3, 4};
+  vi4 shl = a << 3;
+
+  uvi4 b = {1u, 2u, 3u, 4u};
+  uvi4 shr = b >> 3u;
+}
+
+// CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
+// CIR: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
+// CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["b", init]
+// CIR: %[[SHR_RES:.*]] = cir.alloca !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>, ["shr", init]
+// CIR: %[[VEC_A_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
+// CIR-SAME: #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+// CIR: cir.store{{.*}} %[[VEC_A_VAL]], %[[VEC_A]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[SH_AMOUNT:.*]] = cir.const #cir.int<3> : !s32i
+// CIR: %[[SPLAT_VEC:.*]] = cir.vec.splat %[[SH_AMOUNT]] : !s32i, !cir.vector<4 x !s32i>
+// CIR: %[[SHL:.*]] = cir.shift(left, %[[TMP_A]] : !cir.vector<4 x !s32i>, %[[SPLAT_VEC]] : !cir.vector<4 x !s32i>) -> !cir.vector<4 x !s32i>
+// CIR: cir.store{{.*}} %[[SHL]], %[[SHL_RES]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CIR: %[[VEC_B_VAL:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !u32i, #cir.int<2> : !u32i,
+// CIR-SAME: #cir.int<3> : !u32i, #cir.int<4> : !u32i]> : !cir.vector<4 x !u32i>
+// CIR: cir.store{{.*}} %[[VEC_B_VAL]], %[[VEC_B]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !u32i>>, !cir.vector<4 x !u32i>
+// CIR: %[[SH_AMOUNT:.*]] = cir.const #cir.int<3> : !u32i
+// CIR: %[[SPLAT_VEC:.*]] = cir.vec.splat %[[SH_AMOUNT]] : !u32i, !cir.vector<4 x !u32i>
+// CIR: %[[SHR:.*]] = cir.shift(right, %[[TMP_B]] : !cir.vector<4 x !u32i>, %[[SPLAT_VEC]] : !cir.vector<4 x !u32i>) -> !cir.vector<4 x !u32i>
+// CIR: cir.store{{.*}} %[[SHR]], %[[SHR_RES]] : !cir.vector<4 x !u32i>, !cir.ptr<!cir.vector<4 x !u32i>>
+
+// LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[SHL_RES:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[VEC_B:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[SHR_RES:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_A]], align 16
+// LLVM: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// LLVM: %[[SHL:.*]] = shl <4 x i32> %[[TMP_A]], splat (i32 3)
+// LLVM: store <4 x i32> %[[SHL]], ptr %[[SHL_RES]], align 16
+// LLVM: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_B]], align 16
+// LLVM: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// LLVM: %[[SHR:.*]] = lshr <4 x i32> %[[TMP_B]], splat (i32 3)
+// LLVM: store <4 x i32> %[[SHR]], ptr %[[SHR_RES]], align 16
+
+// OGCG: %[[VEC_A:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[SHL_RES:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[VEC_B:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[SHR_RES:.*]] = alloca <4 x i32>, align 16
+// OGCG: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_A]], align 16
+// OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// OGCG: %[[SHL:.*]] = shl <4 x i32> %[[TMP_A]], splat (i32 3)
+// OGCG: store <4 x i32> %[[SHL]], ptr %[[SHL_RES]], align 16
+// OGCG: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr %[[VEC_B]], align 16
+// OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// OGCG: %[[SHR:.*]] = lshr <4 x i32> %[[TMP_B]], splat (i32 3)
+// OGCG: store <4 x i32> %[[SHR]], ptr %[[SHR_RES]], align 16
+
+void foo19() {
+  vi4 a;
+  vi4 b;
+  vi4 u = __builtin_shufflevector(a, b, 7, 5, 3, 1);
+}
+
+// CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a"]
+// CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[SHUF:.*]] = cir.vec.shuffle(%[[TMP_A]], %[[TMP_B]] : !cir.vector<4 x !s32i>) [#cir.int<7> :
+// CIR-SAME: !s64i, #cir.int<5> : !s64i, #cir.int<3> : !s64i, #cir.int<1> : !s64i] : !cir.vector<4 x !s32i>
+
+// LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[VEC_B:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// LLVM: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// LLVM: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 7, i32 5, i32 3, i32 1>
+
+// OGCG: %[[VEC_A:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[VEC_B:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// OGCG: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 7, i32 5, i32 3, i32 1>
+
+void foo20() {
+  vi4 a;
+  vi4 b;
+  vi4 c;
+  vi4 r = c ? a : b;
+}
+
+// CIR: %[[RES:.*]] = cir.vec.ternary({{.*}}, {{.*}}, {{.*}}) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
+
+// LLVM: %[[VEC_COND:.*]] = icmp ne <4 x i32> {{.*}}, zeroinitializer
+// LLVM: %[[RES:.*]] = select <4 x i1> %[[VEC_COND]], <4 x i32> {{.*}}, <4 x i32> {{.*}}
+
+// OGCG: %[[VEC_COND:.*]] = icmp ne <4 x i32> {{.*}}, zeroinitializer
+// OGCG: %[[RES:.*]] = select <4 x i1> %[[VEC_COND]], <4 x i32> {{.*}}, <4 x i32> {{.*}}
+
+void foo21() {
+  vi4 a;
+  vi4 b;
+  vi4 r = (a > b) ? (a - b) : (b - a);
+}
+
+// CIR: %[[VEC_COND:.*]] = cir.vec.cmp(gt, {{.*}}, {{.*}}) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
+// CIR: %[[LHS:.*]] = cir.binop(sub, {{.*}}, {{.*}}) : !cir.vector<4 x !s32i>
+// CIR: %[[RHS:.*]] = cir.binop(sub, {{.*}}, {{.*}}) : !cir.vector<4 x !s32i>
+// CIR: %[[RES:.*]] = cir.vec.ternary(%[[VEC_COND]], %[[LHS]], %[[RHS]]) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
+
+// LLVM: %[[CMP:.*]] = icmp sgt <4 x i32> {{.*}}, {{.*}}
+// LLVM: %[[SEXT:.*]] = sext <4 x i1> %[[CMP]] to <4 x i32>
+// LLVM: %[[LHS:.*]] = sub <4 x i32> {{.*}}, {{.*}}
+// LLVM: %[[RHS:.*]] = sub <4 x i32> {{.*}}, {{.*}}
+// LLVM: %[[VEC_COND:.*]] = icmp ne <4 x i32> %[[SEXT]], zeroinitializer
+// LLVM: %[[RES:.*]] = select <4 x i1> %[[VEC_COND]], <4 x i32> %[[LHS]], <4 x i32> %[[RHS]]
+
+// OGCG: %[[CMP:.*]] = icmp sgt <4 x i32> {{.*}}, {{.*}}
+// OGCG: %[[SEXT:.*]] = sext <4 x i1> %[[CMP]] to <4 x i32>
+// OGCG: %[[LHS:.*]] = sub <4 x i32> {{.*}}, {{.*}}
+// OGCG: %[[RHS:.*]] = sub <4 x i32> {{.*}}, {{.*}}
+// OGCG: %[[VEC_COND:.*]] = icmp ne <4 x i32> %[[SEXT]], zeroinitializer
+// OGCG: %[[RES:.*]] = select <4 x i1> %[[VEC_COND]], <4 x i32> %[[LHS]], <4 x i32> %[[RHS]]
+
+void foo22() {
+  vf4 a;
+  vf4 b;
+  vi4 c;
+  vf4 r = c ? a : b;
+}
+
+// CIR: %[[RES:.*]] = cir.vec.ternary({{.*}}, {{.*}}, {{.*}}) : !cir.vector<4 x !s32i>, !cir.vector<4 x !cir.float>
+
+// LLVM: %[[VEC_COND:.*]] = icmp ne <4 x i32> {{.*}}, zeroinitializer
+// LLVM: %[[RES:.*]] = select <4 x i1> %[[VEC_COND]], <4 x float> {{.*}}, <4 x float> {{.*}}
+
+// OGCG: %[[VEC_COND:.*]] = icmp ne <4 x i32> {{.*}}, zeroinitializer
+// OGCG: %[[RES:.*]] = select <4 x i1> %[[VEC_COND]], <4 x float> {{.*}}, <4 x float> {{.*}}
+
+void foo23() {
+  vi4 a;
+  vi4 b;
+  vi4 u = __builtin_shufflevector(a, b, -1, 1, -1, 1);
+}
+
+// CIR: %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a"]
+// CIR: %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[VEC_A]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[VEC_B]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CIR: %[[SHUF:.*]] = cir.vec.shuffle(%[[TMP_A]], %[[TMP_B]] : !cir.vector<4 x !s32i>) [#cir.int<-1> :
+// CIR-SAME: !s64i, #cir.int<1> : !s64i, #cir.int<-1> : !s64i, #cir.int<1> : !s64i] : !cir.vector<4 x !s32i>
+
+// LLVM: %[[VEC_A:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[VEC_B:.*]] = alloca <4 x i32>, i64 1, align 16
+// LLVM: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// LLVM: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// LLVM: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
+
+// OGCG: %[[VEC_A:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[VEC_B:.*]] = alloca <4 x i32>, align 16
+// OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
+// OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
+// OGCG: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined-copy.c b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined-copy.c
index 50c0519f0f29..72471d4ec787 100644
--- a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined-copy.c
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined-copy.c
@@ -272,14 +272,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[3]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[3]"} loc
   
 #pragma acc serial loop copy(localArray[1:3])
   for(int i = 0; i < 5; ++i);
@@ -290,14 +290,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:3]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:3]"} loc
 
 #pragma acc kernels loop copy(localArray[:3])
   for(int i = 0; i < 5; ++i);
@@ -307,14 +307,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[:3]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[:3]"} loc
 
 #pragma acc parallel loop copy(localArray[1:])
   for(int i = 0; i < 5; ++i);
@@ -324,14 +324,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:]"} loc
 
 #pragma acc serial loop copy(localArray[localVar1:localVar2])
   for(int i = 0; i < 5; ++i);
@@ -342,14 +342,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:localVar2]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:localVar2]"} loc
 
 #pragma acc kernels loop copy(localArray[:localVar2])
   for(int i = 0; i < 5; ++i);
@@ -359,14 +359,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[:localVar2]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[:localVar2]"} loc
 
 #pragma acc parallel loop copy(localArray[localVar1:])
   for(int i = 0; i < 5; ++i);
@@ -376,14 +376,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:]"} loc
 
 #pragma acc serial loop copy(localPointer[3])
   for(int i = 0; i < 5; ++i);
@@ -393,14 +393,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[3]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[3]"} loc
 
 #pragma acc kernels loop copy(localPointer[1:3])
   for(int i = 0; i < 5; ++i);
@@ -411,14 +411,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[1:3]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[1:3]"} loc
 
 #pragma acc parallel loop copy(localPointer[:3])
   for(int i = 0; i < 5; ++i);
@@ -428,14 +428,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:3]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:3]"} loc
 
 #pragma acc serial loop copy(localPointer[localVar1:localVar2])
   for(int i = 0; i < 5; ++i);
@@ -446,14 +446,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[localVar1:localVar2]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[localVar1:localVar2]"} loc
 
 #pragma acc kernels loop copy(localPointer[:localVar2])
   for(int i = 0; i < 5; ++i);
@@ -463,14 +463,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:localVar2]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:localVar2]"} loc
 
   float *localArrayOfPtrs[5];
 #pragma acc parallel loop copy(localArrayOfPtrs[3])
@@ -481,14 +481,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3]"} loc
 
 #pragma acc serial loop copy(localArrayOfPtrs[3][2])
   for(int i = 0; i < 5; ++i);
@@ -504,14 +504,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3][2]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3][2]"} loc
 
 #pragma acc kernels loop copy(localArrayOfPtrs[localVar1:localVar2])
   for(int i = 0; i < 5; ++i);
@@ -522,14 +522,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2]"} loc
 
 #pragma acc parallel loop copy(localArrayOfPtrs[localVar1:])
   for(int i = 0; i < 5; ++i);
@@ -539,14 +539,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:]"} loc
 
 #pragma acc serial loop copy(localArrayOfPtrs[:localVar2])
   for(int i = 0; i < 5; ++i);
@@ -556,14 +556,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[:localVar2]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[:localVar2]"} loc
 
 #pragma acc kernels loop copy(localArrayOfPtrs[localVar1])
   for(int i = 0; i < 5; ++i);
@@ -573,14 +573,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1]"} loc
 
 #pragma acc parallel loop copy(localArrayOfPtrs[localVar1][localVar2])
   for(int i = 0; i < 5; ++i);
@@ -596,14 +596,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2]"} loc
   
 #pragma acc serial loop copy(localArrayOfPtrs[localVar1][localVar2:parmVar])
   for(int i = 0; i < 5; ++i);
@@ -620,14 +620,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2:parmVar]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2:parmVar]"} loc
  
 #pragma acc kernels loop copy(localArrayOfPtrs[localVar1][:parmVar])
   for(int i = 0; i < 5; ++i);
@@ -643,14 +643,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][:parmVar]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][:parmVar]"} loc
 
 #pragma acc parallel loop copy(localArrayOfPtrs[localVar1:localVar2][:1])
   for(int i = 0; i < 5; ++i);
@@ -667,14 +667,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][:1]"} loc
   // CHECK-NEXT: acc.parallel combined(loop)  dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][:1]"} loc
 
 #pragma acc serial loop copy(localArrayOfPtrs[localVar1:localVar2][1:1])
   for(int i = 0; i < 5; ++i);
@@ -692,14 +692,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][1:1]"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][1:1]"} loc
 
   double threeDArray[5][6][7];
 #pragma acc kernels loop copy(threeDArray[1][2][3])
@@ -722,14 +722,14 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS3:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1][2][3]"} loc
   // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {
   // CHECK-NEXT: acc.loop combined(kernels) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1][2][3]"} loc
 
 #pragma acc parallel loop copy(threeDArray[1:1][2:1][3:1])
   for(int i = 0; i < 5; ++i);
@@ -754,12 +754,351 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS3:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1:1][2:1][3:1]"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1:1][2:1][3:1]"} loc
+}
+
+typedef struct StructTy {
+  int scalarMember;
+  int arrayMember[5];
+  short twoDArrayMember[5][3];
+  float *ptrArrayMember[5];
+  double **ptrPtrMember;
+} Struct ;
+
+void acc_compute_members() {
+  // CHECK: cir.func @acc_compute_members()
+  Struct localStruct;
+  // CHECK-NEXT: %[[LOCALSTRUCT:.*]] = cir.alloca !rec_StructTy, !cir.ptr<!rec_StructTy>, ["localStruct"]
+
+#pragma acc parallel loop copy(localStruct)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALSTRUCT]] : !cir.ptr<!rec_StructTy>) -> !cir.ptr<!rec_StructTy> {dataClause = #acc<data_clause acc_copy>, name = "localStruct"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_StructTy>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_StructTy>) to varPtr(%[[LOCALSTRUCT]] : !cir.ptr<!rec_StructTy>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct"}
+
+#pragma acc serial loop copy(localStruct.scalarMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMBER]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.scalarMember"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETMEMBER]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.scalarMember"}
+
+#pragma acc kernels loop copy(localStruct.arrayMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember"} loc
+
+#pragma acc parallel loop copy(localStruct.arrayMember[2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) 
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[2]"} 
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[2]"} loc
+
+#pragma acc serial loop copy(localStruct.arrayMember[1:2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[TWO_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:2]"} 
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:2]"} loc
+
+#pragma acc kernels loop copy(localStruct.arrayMember[1:])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FOUR_CONST:.*]] = arith.constant 4 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[ONE_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:]"} 
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:]"} loc
+
+#pragma acc parallel loop copy(localStruct.arrayMember[:2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[TWO_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[:2]"} 
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[:2]"} loc
+
+#pragma acc serial loop copy(localStruct.twoDArrayMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GET2DARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember"} 
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) to varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember"} 
+
+#pragma acc kernels loop copy(localStruct.twoDArrayMember[3][2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[3][2]"} 
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[3][2]"} 
+
+#pragma acc parallel loop copy(localStruct.twoDArrayMember[1:3][1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[1:3][1:1]"} 
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[1:3][1:1]"} 
+
+#pragma acc serial loop copy(localStruct.ptrArrayMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETPTRARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][3] {name = "ptrArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) to varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember"}
+
+#pragma acc kernels loop copy(localStruct.ptrArrayMember[3][2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][3] {name = "ptrArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[3][2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT:  acc.terminator
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[3][2]"}
+
+#pragma acc parallel loop copy(localStruct.ptrArrayMember[1:3][1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][3] {name = "ptrArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[1:3][1:1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[1:3][1:1]"}
+
+#pragma acc serial loop copy(localStruct.ptrPtrMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETPTRPTRMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][4] {name = "ptrPtrMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>
+  // CHECK-NEXT:  %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember"}
+  // CHECK-NEXT:  acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) to varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember"}
+
+#pragma acc kernels loop copy(localStruct.ptrPtrMember[3][2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRPTRMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][4] {name = "ptrPtrMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>
+  // CHECK-NEXT:  %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[3][2]"}
+  // CHECK-NEXT:  acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT:  acc.terminator
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[3][2]"}
+
+#pragma acc parallel loop copy(localStruct.ptrPtrMember[1:3][1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRPTRMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][4] {name = "ptrPtrMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>
+  // CHECK-NEXT:  %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[1:3][1:1]"}
+  // CHECK-NEXT:  acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[1:3][1:1]"}
+}
+
+typedef struct InnerTy {
+  int a;
+  int b;
+} Inner;
+
+typedef struct OuterTy {
+  Inner inner[4];
+} Outer;
+
+void copy_member_of_array_element_member() {
+  // CHECK: cir.func @copy_member_of_array_element_member() {
+  Outer outer;
+  // CHECK-NEXT: %[[OUTER:.*]] = cir.alloca !rec_OuterTy, !cir.ptr<!rec_OuterTy>, ["outer"]
+
+  #pragma acc parallel loop copy(outer.inner[2].b)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[GETINNER:.*]] = cir.get_member %[[OUTER]][0] {name = "inner"} : !cir.ptr<!rec_OuterTy> -> !cir.ptr<!cir.array<!rec_InnerTy x 4>>
+  // CHECK-NEXT: %[[INNERDECAY:.*]] = cir.cast(array_to_ptrdecay, %[[GETINNER]] : !cir.ptr<!cir.array<!rec_InnerTy x 4>>), !cir.ptr<!rec_InnerTy>
+  // CHECK-NEXT: %[[STRIDE:.*]] = cir.ptr_stride(%[[INNERDECAY]] : !cir.ptr<!rec_InnerTy>, %[[TWO]] : !s32i), !cir.ptr<!rec_InnerTy>
+  // CHECK-NEXT: %[[GETB:.*]] = cir.get_member %[[STRIDE]][1] {name = "b"} : !cir.ptr<!rec_InnerTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT:  %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETB]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "outer.inner[2].b"}
+  // CHECK-NEXT:  acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETB]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "outer.inner[2].b"}
 }
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined-copy.cpp b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined-copy.cpp
new file mode 100644
index 000000000000..525cf3de187e
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined-copy.cpp
@@ -0,0 +1,418 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+struct InnerStructTy {
+  int Member[5];
+};
+struct StructTy {
+  int scalarMember;
+  int arrayMember[5];
+  short twoDArrayMember[5][3];
+  InnerStructTy iSTy;
+
+void InlineFunc() {
+  // CHECK: cir.func {{.*}}InlineFunc{{.*}}
+  // CHECK-NEXT: %[[THIS:.*]] = cir.alloca !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>, ["this", init]
+  // CHECK-NEXT: cir.store %[[THIS_ARG:.*]], %[[THIS]] : !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>
+  // CHECK-NEXT: %[[THIS_LOAD:.*]] = cir.load %[[THIS]] : !cir.ptr<!cir.ptr<!rec_StructTy>>, !cir.ptr<!rec_StructTy>
+
+#pragma acc parallel loop copy(scalarMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+
+#pragma acc kernels loop copy(arrayMember[2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+
+#pragma acc kernels loop copy(twoDArrayMember[1][2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+
+#pragma acc kernels loop copy(iSTy)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel loop copy(iSTy.Member)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial loop copy(iSTy.Member[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+
+#pragma acc parallel loop copy(this->scalarMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+
+#pragma acc kernels loop copy(this->arrayMember[2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+#pragma acc kernels loop copy(this->twoDArrayMember[1][2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+ 
+#pragma acc kernels loop copy(this->iSTy)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel loop copy(this->iSTy.Member)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial loop copy(this->iSTy.Member[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+}
+
+void OutlineFunc();
+};
+
+void InlineUse() {
+  StructTy s;
+  s.InlineFunc();
+}
+
+void StructTy::OutlineFunc() {
+  // CHECK: cir.func {{.*}}OutlineFunc{{.*}}
+  // CHECK-NEXT: %[[THIS:.*]] = cir.alloca !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>, ["this", init]
+  // CHECK-NEXT: cir.store %[[THIS_ARG:.*]], %[[THIS]] : !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>
+  // CHECK-NEXT: %[[THIS_LOAD:.*]] = cir.load %[[THIS]] : !cir.ptr<!cir.ptr<!rec_StructTy>>, !cir.ptr<!rec_StructTy>
+#pragma acc parallel loop copy(scalarMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+#pragma acc kernels loop copy(arrayMember[2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+#pragma acc kernels loop copy(twoDArrayMember[1][2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+#pragma acc kernels loop copy(iSTy)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel loop copy(iSTy.Member)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial loop copy(iSTy.Member[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+
+#pragma acc parallel loop copy(this->scalarMember)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+#pragma acc kernels loop copy(this->arrayMember[2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+
+#pragma acc kernels loop copy(this->twoDArrayMember[1][2])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+
+#pragma acc kernels loop copy(this->iSTy)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel loop copy(this->iSTy.Member)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial loop copy(this->iSTy.Member[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined.cpp b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined.cpp
index 6124f3896728..5b83a9cb9189 100644
--- a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/combined.cpp
@@ -74,7 +74,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK: acc.serial combined(loop) {
   // CHECK: acc.loop combined(serial) {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {seq = [#acc.device_type<nvidia>, #acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {seq = [#acc.device_type<nvidia>, #acc.device_type<radeon>, #acc.device_type<none>]} loc
   // CHECK: acc.yield
   // CHECK-NEXT: } loc
 #pragma acc kernels loop seq device_type(nvidia, radeon)
@@ -99,7 +99,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK: acc.serial combined(loop) {
   // CHECK: acc.loop combined(serial) {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<nvidia>, #acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<nvidia>, #acc.device_type<radeon>], seq = [#acc.device_type<none>]} loc
   // CHECK: acc.yield
   // CHECK-NEXT: } loc
 #pragma acc kernels loop auto device_type(nvidia, radeon)
@@ -124,7 +124,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK: acc.serial combined(loop) {
   // CHECK: acc.loop combined(serial) {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {independent = [#acc.device_type<nvidia>, #acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<nvidia>, #acc.device_type<radeon>], seq = [#acc.device_type<none>]} loc
   // CHECK: acc.yield
   // CHECK-NEXT: } loc
 #pragma acc kernels loop independent device_type(nvidia, radeon)
@@ -143,7 +143,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK: acc.parallel combined(loop) {
   // CHECK: acc.loop combined(parallel) {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1], collapseDeviceType = [#acc.device_type<none>]}
+  // CHECK-NEXT: } attributes {collapse = [1], collapseDeviceType = [#acc.device_type<none>], independent = [#acc.device_type<none>]}
   // CHECK: acc.yield
   // CHECK-NEXT: } loc
 
@@ -154,7 +154,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK: acc.serial combined(loop) {
   // CHECK: acc.loop combined(serial) {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>]}
+  // CHECK-NEXT: } attributes {collapse = [1, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>], seq = [#acc.device_type<none>]}
   // CHECK: acc.yield
   // CHECK-NEXT: } loc
 
@@ -165,7 +165,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK: acc.kernels combined(loop) {
   // CHECK: acc.loop combined(kernels) {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1, 2, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>]}
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>], collapse = [1, 2, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>]}
   // CHECK: acc.terminator
   // CHECK-NEXT: } loc
   #pragma acc parallel loop collapse(1) device_type(radeon, nvidia) collapse(2) device_type(host) collapse(3)
@@ -175,7 +175,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK: acc.parallel combined(loop) {
   // CHECK: acc.loop combined(parallel) {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1, 2, 2, 3], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>, #acc.device_type<host>]}
+  // CHECK-NEXT: } attributes {collapse = [1, 2, 2, 3], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>, #acc.device_type<host>], independent = [#acc.device_type<none>]}
   // CHECK: acc.yield
   // CHECK-NEXT: } loc
 
@@ -1011,3 +1011,232 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
 }
+extern "C" void acc_combined_data_clauses(int *arg1, int *arg2) {
+  // CHECK: cir.func @acc_combined_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
+  // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
+  // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+  // CHECK-NEXT: cir.store %[[ARG2_PARAM]], %[[ARG2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc parallel loop deviceptr(arg1)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial loop deviceptr(arg2)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels loop deviceptr(arg1, arg2)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel loop deviceptr(arg1) async
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) async {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial loop deviceptr(arg2) async device_type(nvidia)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>) async {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels loop deviceptr(arg1, arg2) device_type(nvidia) async
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel loop no_create(arg1)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+
+#pragma acc serial loop no_create(arg2)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[NOCREATE2:.*]] = acc.nocreate varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_no_create>, name = "arg2"}
+
+#pragma acc kernels loop no_create(arg1, arg2) device_type(host) async
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[NOCREATE2:.*]] = acc.nocreate varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[NOCREATE1]], %[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+
+#pragma acc parallel loop present(arg1)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+
+#pragma acc serial loop present(arg2)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[PRESENT2:.*]] = acc.present varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_present>, name = "arg2"}
+
+#pragma acc kernels loop present(arg1, arg2) device_type(host) async
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[PRESENT2:.*]] = acc.present varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[PRESENT1]], %[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {dataClause = #acc<data_clause acc_present>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+
+#pragma acc parallel loop attach(arg1)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+
+#pragma acc serial loop attach(arg2)
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[ATTACH2:.*]] = acc.attach varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_attach>, name = "arg2"}
+
+#pragma acc kernels loop attach(arg1, arg2) device_type(host) async
+  for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[ATTACH2:.*]] = acc.attach varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[ATTACH1]], %[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {dataClause = #acc<data_clause acc_attach>, name = "arg2"}
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<host>]) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+
+  // Checking the automatic-addition of parallelism clauses.
+#pragma acc parallel loop
+    for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: acc.parallel combined(loop) {
+  // CHECK-NEXT:  acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]} loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels loop
+    for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: acc.kernels combined(loop) {
+  // CHECK-NEXT:  acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc serial loop
+    for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: acc.serial combined(loop) {
+  // CHECK-NEXT:  acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {seq = [#acc.device_type<none>]} loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial loop worker
+    for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: acc.serial combined(loop) {
+  // CHECK-NEXT:  acc.loop combined(serial) worker {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial loop vector
+    for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: acc.serial combined(loop) {
+  // CHECK-NEXT:  acc.loop combined(serial) vector {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial loop gang
+    for(unsigned I = 0; I < 5; ++I);
+  // CHECK-NEXT: acc.serial combined(loop) {
+  // CHECK-NEXT:  acc.loop combined(serial) gang {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/compute-copy.c b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/compute-copy.c
index 549af7802c54..888bad29caa7 100644
--- a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/compute-copy.c
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/compute-copy.c
@@ -222,11 +222,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[3]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[3]"} loc
 
 #pragma acc serial copy(localArray[1:3])
   ;
@@ -237,11 +237,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:3]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:3]"} loc
 
 #pragma acc kernels copy(localArray[:3])
   ;
@@ -251,11 +251,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[:3]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[:3]"} loc
 
 #pragma acc parallel copy(localArray[1:])
   ;
@@ -265,11 +265,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[1:]"} loc
 #pragma acc serial copy(localArray[localVar1:localVar2])
   ;
   // CHECK-NEXT: %[[LV1:.*]] = cir.load{{.*}} %[[LOCAL1]] : !cir.ptr<!s32i>, !s32i 
@@ -279,11 +279,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:localVar2]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:localVar2]"} loc
 
 #pragma acc kernels copy(localArray[:localVar2])
   ;
@@ -293,11 +293,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[:localVar2]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[:localVar2]"} loc
 
 #pragma acc parallel copy(localArray[localVar1:])
   ;
@@ -307,11 +307,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAY]] : !cir.ptr<!cir.array<!cir.float x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArray[localVar1:]"} loc
 
 #pragma acc serial copy(localPointer[3])
   ;
@@ -321,11 +321,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[3]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[3]"} loc
 
 #pragma acc kernels copy(localPointer[1:3])
   ;
@@ -336,11 +336,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[1:3]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[1:3]"} loc
 
 #pragma acc parallel copy(localPointer[:3])
   ;
@@ -350,11 +350,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:3]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:3]"} loc
 
 #pragma acc serial copy(localPointer[localVar1:localVar2])
   ;
@@ -365,11 +365,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[localVar1:localVar2]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[localVar1:localVar2]"} loc
 
 #pragma acc kernels copy(localPointer[:localVar2])
   ;
@@ -379,11 +379,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.ptr<!s16i>> {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:localVar2]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!s16i>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALPTR]] : !cir.ptr<!cir.ptr<!s16i>>) {dataClause = #acc<data_clause acc_copy>, name = "localPointer[:localVar2]"} loc
 
   float *localArrayOfPtrs[5];
 #pragma acc parallel copy(localArrayOfPtrs[3])
@@ -394,11 +394,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3]"} loc
 #pragma acc serial copy(localArrayOfPtrs[3][2])
   ;
   // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
@@ -413,11 +413,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3][2]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[3][2]"} loc
 
 #pragma acc kernels copy(localArrayOfPtrs[localVar1:localVar2])
   ;
@@ -428,11 +428,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2]"} loc
 
 #pragma acc parallel copy(localArrayOfPtrs[localVar1:])
   ;
@@ -442,11 +442,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:]"} loc
 
 #pragma acc serial copy(localArrayOfPtrs[:localVar2])
   ;
@@ -456,11 +456,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[:localVar2]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[:localVar2]"} loc
 
 #pragma acc kernels copy(localArrayOfPtrs[localVar1])
   ;
@@ -470,11 +470,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1]"} loc
 
 #pragma acc parallel copy(localArrayOfPtrs[localVar1][localVar2])
   ;
@@ -490,11 +490,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2]"} loc
 
 #pragma acc serial copy(localArrayOfPtrs[localVar1][localVar2:parmVar])
   ;
@@ -511,11 +511,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2:parmVar]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][localVar2:parmVar]"} loc
 
 #pragma acc kernels copy(localArrayOfPtrs[localVar1][:parmVar])
   ;
@@ -531,11 +531,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][:parmVar]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1][:parmVar]"} loc
 
 #pragma acc parallel copy(localArrayOfPtrs[localVar1:localVar2][:1])
   ;
@@ -552,11 +552,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][:1]"} loc
   // CHECK-NEXT: acc.parallel  dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][:1]"} loc
 
 #pragma acc serial copy(localArrayOfPtrs[localVar1:localVar2][1:1])
   ;
@@ -574,11 +574,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[LV1_CAST]] : si32) extent(%[[LV2_CAST]] : si16) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][1:1]"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]]) to varPtr(%[[LOCALARRAYOFPTRS]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localArrayOfPtrs[localVar1:localVar2][1:1]"} loc
 
   double threeDArray[5][6][7];
 #pragma acc kernels copy(threeDArray[1][2][3])
@@ -601,11 +601,11 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS3:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1][2][3]"} loc
   // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1][2][3]"} loc
 
 #pragma acc parallel copy(threeDArray[1:1][2:1][3:1])
   ;
@@ -630,9 +630,270 @@ void acc_compute(int parmVar) {
   // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
   // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
   // CHECK-NEXT: %[[BOUNDS3:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) loc
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) -> !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1:1][2:1][3:1]"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) bounds(%[[BOUNDS]], %[[BOUNDS2]], %[[BOUNDS3]]) to varPtr(%[[THREEDARRAY]] : !cir.ptr<!cir.array<!cir.array<!cir.array<!cir.double x 7> x 6> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "threeDArray[1:1][2:1][3:1]"} loc
+}
+
+typedef struct StructTy {
+  int scalarMember;
+  int arrayMember[5];
+  short twoDArrayMember[5][3];
+  float *ptrArrayMember[5];
+  double **ptrPtrMember;
+} Struct ;
+
+void acc_compute_members() {
+  // CHECK: cir.func @acc_compute_members()
+  Struct localStruct;
+  // CHECK-NEXT: %[[LOCALSTRUCT:.*]] = cir.alloca !rec_StructTy, !cir.ptr<!rec_StructTy>, ["localStruct"]
+
+#pragma acc parallel copy(localStruct)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALSTRUCT]] : !cir.ptr<!rec_StructTy>) -> !cir.ptr<!rec_StructTy> {dataClause = #acc<data_clause acc_copy>, name = "localStruct"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_StructTy>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_StructTy>) to varPtr(%[[LOCALSTRUCT]] : !cir.ptr<!rec_StructTy>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct"}
+
+#pragma acc serial copy(localStruct.scalarMember)
+  ;
+  // CHECK-NEXT: %[[GETMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMBER]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.scalarMember"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETMEMBER]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.scalarMember"}
+
+#pragma acc kernels copy(localStruct.arrayMember)
+  ;
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember"} loc
+
+#pragma acc parallel copy(localStruct.arrayMember[2])
+  ;
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64) 
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[2]"} 
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[2]"} loc
+
+#pragma acc serial copy(localStruct.arrayMember[1:2])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[TWO_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:2]"} 
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:2]"} loc
+
+#pragma acc kernels copy(localStruct.arrayMember[1:])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FOUR_CONST:.*]] = arith.constant 4 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[ONE_CAST]] : si32) upperbound(%[[FOUR_CONST]] : i64) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:]"} 
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[1:]"} loc
+
+#pragma acc parallel copy(localStruct.arrayMember[:2])
+  ;
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST2:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]]  = acc.bounds lowerbound(%[[ZERO_CONST]] : i64) extent(%[[TWO_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST2]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[:2]"} 
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) to varPtr(%[[GETARRAYMEMBER]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.arrayMember[:2]"} loc
+
+#pragma acc serial copy(localStruct.twoDArrayMember)
+  ;
+  // CHECK-NEXT: %[[GET2DARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember"} 
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.yield 
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) to varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember"} 
+
+#pragma acc kernels copy(localStruct.twoDArrayMember[3][2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[3][2]"} 
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[3][2]"} 
+
+#pragma acc parallel copy(localStruct.twoDArrayMember[1:3][1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[1:3][1:1]"} 
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.twoDArrayMember[1:3][1:1]"} 
+
+#pragma acc serial copy(localStruct.ptrArrayMember)
+  ;
+  // CHECK-NEXT: %[[GETPTRARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][3] {name = "ptrArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) to varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember"}
+
+#pragma acc kernels copy(localStruct.ptrArrayMember[3][2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][3] {name = "ptrArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[3][2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
+  // CHECK-NEXT:  acc.terminator
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[3][2]"}
+
+#pragma acc parallel copy(localStruct.ptrArrayMember[1:3][1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRARRAYMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][3] {name = "ptrArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[1:3][1:1]"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRARRAYMEMBER]] : !cir.ptr<!cir.array<!cir.ptr<!cir.float> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrArrayMember[1:3][1:1]"}
+
+#pragma acc serial copy(localStruct.ptrPtrMember)
+  ;
+  // CHECK-NEXT: %[[GETPTRPTRMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][4] {name = "ptrPtrMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>
+  // CHECK-NEXT:  %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember"}
+  // CHECK-NEXT:  acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) to varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember"}
+
+#pragma acc kernels copy(localStruct.ptrPtrMember[3][2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[THREE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRPTRMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][4] {name = "ptrPtrMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>
+  // CHECK-NEXT:  %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[3][2]"}
+  // CHECK-NEXT:  acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {
+  // CHECK-NEXT:  acc.terminator
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[3][2]"}
+
+#pragma acc parallel copy(localStruct.ptrPtrMember[1:3][1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+  // CHECK-NEXT: %[[THREE_CAST:.*]] = builtin.unrealized_conversion_cast %[[THREE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[THREE_CAST]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETPTRPTRMEMBER:.*]] = cir.get_member %[[LOCALSTRUCT]][4] {name = "ptrPtrMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>
+  // CHECK-NEXT:  %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>> {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[1:3][1:1]"}
+  // CHECK-NEXT:  acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {
+  // CHECK-NEXT:  acc.yield
+  // CHECK-NEXT:  } loc
+  // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[1:3][1:1]"}
+
 }
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/compute-copy.cpp b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/compute-copy.cpp
new file mode 100644
index 000000000000..e1692f5a5f3c
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/compute-copy.cpp
@@ -0,0 +1,346 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+struct InnerStructTy {
+  int Member[5];
+};
+struct StructTy {
+  int scalarMember;
+  int arrayMember[5];
+  short twoDArrayMember[5][3];
+  InnerStructTy iSTy;
+
+void InlineFunc() {
+  // CHECK: cir.func {{.*}}InlineFunc{{.*}}
+  // CHECK-NEXT: %[[THIS:.*]] = cir.alloca !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>, ["this", init]
+  // CHECK-NEXT: cir.store %[[THIS_ARG:.*]], %[[THIS]] : !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>
+  // CHECK-NEXT: %[[THIS_LOAD:.*]] = cir.load %[[THIS]] : !cir.ptr<!cir.ptr<!rec_StructTy>>, !cir.ptr<!rec_StructTy>
+
+#pragma acc parallel copy(scalarMember)
+  ;
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+
+#pragma acc kernels copy(arrayMember[2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+
+#pragma acc kernels copy(twoDArrayMember[1][2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+
+#pragma acc kernels copy(iSTy)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel copy(iSTy.Member)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial copy(iSTy.Member[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+
+#pragma acc parallel copy(this->scalarMember)
+  ;
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+
+#pragma acc kernels copy(this->arrayMember[2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+#pragma acc kernels copy(this->twoDArrayMember[1][2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+ 
+#pragma acc kernels copy(this->iSTy)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel copy(this->iSTy.Member)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial copy(this->iSTy.Member[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+}
+
+void OutlineFunc();
+};
+
+void InlineUse() {
+  StructTy s;
+  s.InlineFunc();
+}
+
+void StructTy::OutlineFunc() {
+  // CHECK: cir.func {{.*}}OutlineFunc{{.*}}
+  // CHECK-NEXT: %[[THIS:.*]] = cir.alloca !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>, ["this", init]
+  // CHECK-NEXT: cir.store %[[THIS_ARG:.*]], %[[THIS]] : !cir.ptr<!rec_StructTy>, !cir.ptr<!cir.ptr<!rec_StructTy>>
+  // CHECK-NEXT: %[[THIS_LOAD:.*]] = cir.load %[[THIS]] : !cir.ptr<!cir.ptr<!rec_StructTy>>, !cir.ptr<!rec_StructTy>
+#pragma acc parallel copy(scalarMember)
+  ;
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+#pragma acc kernels copy(arrayMember[2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+#pragma acc kernels copy(twoDArrayMember[1][2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+#pragma acc kernels copy(iSTy)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel copy(iSTy.Member)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial copy(iSTy.Member[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+
+#pragma acc parallel copy(this->scalarMember)
+  ;
+  // CHECK-NEXT: %[[GETSCALARMEM:.*]] = cir.get_member %[[THIS_LOAD]][0] {name = "scalarMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETSCALARMEM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "this->scalarMember"}
+#pragma acc kernels copy(this->arrayMember[2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][1] {name = "arrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETARRAYMEM]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->arrayMember[2]"}
+
+#pragma acc kernels copy(this->twoDArrayMember[1][2])
+  ;
+  // CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[TWO_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS2:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GET2DARRAYMEM:.*]] = cir.get_member %[[THIS_LOAD]][2] {name = "twoDArrayMember"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) -> !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GET2DARRAYMEM]] : !cir.ptr<!cir.array<!cir.array<!s16i x 3> x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->twoDArrayMember[1][2]"}
+
+#pragma acc kernels copy(this->iSTy)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) -> !cir.ptr<!rec_InnerStructTy> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!rec_InnerStructTy>) to varPtr(%[[GETSTRUCTMEM]] : !cir.ptr<!rec_InnerStructTy>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy"}
+
+#pragma acc parallel copy(this->iSTy.Member)
+  ;
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member"}
+
+#pragma acc serial copy(this->iSTy.Member[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS1:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[GETSTRUCTMEM:.*]] = cir.get_member %[[THIS_LOAD]][3] {name = "iSTy"} : !cir.ptr<!rec_StructTy> -> !cir.ptr<!rec_InnerStructTy>
+  // CHECK-NEXT: %[[GETMEMOFSTRUCT:.*]] = cir.get_member %[[GETSTRUCTMEM]][0] {name = "Member"} : !cir.ptr<!rec_InnerStructTy> -> !cir.ptr<!cir.array<!s32i x 5>>
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) -> !cir.ptr<!cir.array<!s32i x 5>> {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS1]]) to varPtr(%[[GETMEMOFSTRUCT]] : !cir.ptr<!cir.array<!s32i x 5>>) {dataClause = #acc<data_clause acc_copy>, name = "this->iSTy.Member[1]"}
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/host_data.c b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/host_data.c
new file mode 100644
index 000000000000..4c3f7dd092a2
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/host_data.c
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_host_data(int cond, int var1, int var2) {
+  // CHECK: cir.func @acc_host_data(%[[ARG_COND:.*]]: !s32i {{.*}}, %[[ARG_V1:.*]]: !s32i {{.*}}, %[[ARG_V2:.*]]: !s32i {{.*}}) {
+  // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
+  // CHECK-NEXT: %[[V1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["var1", init]
+  // CHECK-NEXT: %[[V2:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["var2", init]
+  // CHECK-NEXT: cir.store %[[ARG_COND]], %[[COND]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[ARG_V1]], %[[V1]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[ARG_V2]], %[[V2]] : !s32i, !cir.ptr<!s32i>
+
+#pragma acc host_data use_device(var1)
+  {}
+  // CHECK-NEXT: %[[USE_DEV1:.*]] = acc.use_device varPtr(%[[V1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var1"}
+  // CHECK-NEXT: acc.host_data dataOperands(%[[USE_DEV1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+#pragma acc host_data use_device(var1, var2)
+  {}
+  // CHECK-NEXT: %[[USE_DEV1:.*]] = acc.use_device varPtr(%[[V1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var1"}
+  // CHECK-NEXT: %[[USE_DEV2:.*]] = acc.use_device varPtr(%[[V2]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var2"}
+  // CHECK-NEXT: acc.host_data dataOperands(%[[USE_DEV1]], %[[USE_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc host_data use_device(var1, var2) if_present
+  {}
+  // CHECK-NEXT: %[[USE_DEV1:.*]] = acc.use_device varPtr(%[[V1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var1"}
+  // CHECK-NEXT: %[[USE_DEV2:.*]] = acc.use_device varPtr(%[[V2]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var2"}
+  // CHECK-NEXT: acc.host_data dataOperands(%[[USE_DEV1]], %[[USE_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } attributes {ifPresent}
+
+#pragma acc host_data use_device(var1, var2) if(cond)
+  {}
+  // CHECK-NEXT: %[[USE_DEV1:.*]] = acc.use_device varPtr(%[[V1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var1"}
+  // CHECK-NEXT: %[[USE_DEV2:.*]] = acc.use_device varPtr(%[[V2]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var2"}
+  // CHECK-NEXT: %[[LOAD_COND:.*]] = cir.load{{.*}} %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[COND_BOOL:.*]] = cir.cast(int_to_bool, %[[LOAD_COND]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[COND_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_BOOL]] : !cir.bool to i1
+  // CHECK-NEXT: acc.host_data if(%[[COND_CAST]]) dataOperands(%[[USE_DEV1]], %[[USE_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc host_data use_device(var1, var2) if(cond) if_present
+  {}
+  // CHECK-NEXT: %[[USE_DEV1:.*]] = acc.use_device varPtr(%[[V1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var1"}
+  // CHECK-NEXT: %[[USE_DEV2:.*]] = acc.use_device varPtr(%[[V2]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "var2"}
+  // CHECK-NEXT: %[[LOAD_COND:.*]] = cir.load{{.*}} %[[COND]] : !cir.ptr<!s32i>, !s32i
+  // CHECK-NEXT: %[[COND_BOOL:.*]] = cir.cast(int_to_bool, %[[LOAD_COND]] : !s32i), !cir.bool
+  // CHECK-NEXT: %[[COND_CAST:.*]] = builtin.unrealized_conversion_cast %[[COND_BOOL]] : !cir.bool to i1
+  // CHECK-NEXT: acc.host_data if(%[[COND_CAST]]) dataOperands(%[[USE_DEV1]], %[[USE_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } attributes {ifPresent}
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/kernels.c b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/kernels.c
index 333669c8de7d..d0c6f1134c8d 100644
--- a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/kernels.c
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/kernels.c
@@ -416,3 +416,93 @@ void acc_kernels(int cond) {
 
   // CHECK-NEXT: cir.return
 }
+
+void acc_kernels_data_clauses(int *arg1, int *arg2) {
+  // CHECK: cir.func @acc_kernels_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
+  // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
+  // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+  // CHECK-NEXT: cir.store %[[ARG2_PARAM]], %[[ARG2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc kernels deviceptr(arg1)
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+#pragma acc kernels deviceptr(arg1, arg2)
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels deviceptr(arg1) async
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) async {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+#pragma acc kernels deviceptr(arg1, arg2) device_type(radeon) async
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels no_create(arg1)
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+
+#pragma acc kernels no_create(arg1, arg2) device_type(nvidia) async
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[NOCREATE2:.*]] = acc.nocreate varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[NOCREATE1]], %[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+
+#pragma acc kernels present(arg1)
+  ;
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+
+#pragma acc kernels present(arg1, arg2) device_type(nvidia) async
+  ;
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[PRESENT2:.*]] = acc.present varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[PRESENT1]], %[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_present>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+
+#pragma acc kernels attach(arg1)
+  ;
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+
+#pragma acc kernels attach(arg1, arg2) device_type(nvidia) async
+  ;
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[ATTACH2:.*]] = acc.attach varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[ATTACH1]], %[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_attach>, name = "arg2"}
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+}
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/loop.cpp b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/loop.cpp
index db94e2819b30..c0bf11e35395 100644
--- a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/loop.cpp
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/loop.cpp
@@ -41,12 +41,12 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) {
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {seq = [#acc.device_type<nvidia>, #acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>], seq = [#acc.device_type<nvidia>, #acc.device_type<radeon>]} loc
 #pragma acc loop device_type(radeon) seq
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {seq = [#acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>], seq = [#acc.device_type<radeon>]} loc
 #pragma acc loop seq device_type(nvidia, radeon)
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
@@ -67,12 +67,12 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) {
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {independent = [#acc.device_type<nvidia>, #acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<nvidia>, #acc.device_type<radeon>, #acc.device_type<none>]} loc
 #pragma acc loop device_type(radeon) independent
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {independent = [#acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<radeon>, #acc.device_type<none>]} loc
 #pragma acc loop independent device_type(nvidia, radeon)
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
@@ -93,12 +93,12 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) {
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<nvidia>, #acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<nvidia>, #acc.device_type<radeon>], independent = [#acc.device_type<none>]} loc
 #pragma acc loop device_type(radeon) auto
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<radeon>]} loc
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<radeon>], independent = [#acc.device_type<none>]} loc
 #pragma acc loop auto device_type(nvidia, radeon)
   for(unsigned I = 0; I < N; ++I);
   // CHECK: acc.loop {
@@ -116,7 +116,7 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) {
       for(unsigned K = 0; K < N; ++K);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1], collapseDeviceType = [#acc.device_type<none>]}
+  // CHECK-NEXT: } attributes {collapse = [1], collapseDeviceType = [#acc.device_type<none>], independent = [#acc.device_type<none>]}
 
   #pragma acc loop collapse(1) device_type(radeon) collapse (2)
   for(unsigned I = 0; I < N; ++I)
@@ -124,7 +124,7 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) {
       for(unsigned K = 0; K < N; ++K);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>]}
+  // CHECK-NEXT: } attributes {collapse = [1, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>], independent = [#acc.device_type<none>]}
 
   #pragma acc loop collapse(1) device_type(radeon, nvidia) collapse (2)
   for(unsigned I = 0; I < N; ++I)
@@ -132,14 +132,14 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) {
       for(unsigned K = 0; K < N; ++K);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1, 2, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>]}
+  // CHECK-NEXT: } attributes {collapse = [1, 2, 2], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>], independent = [#acc.device_type<none>]}
   #pragma acc loop collapse(1) device_type(radeon, nvidia) collapse(2) device_type(host) collapse(3)
   for(unsigned I = 0; I < N; ++I)
     for(unsigned J = 0; J < N; ++J)
       for(unsigned K = 0; K < N; ++K);
   // CHECK: acc.loop {
   // CHECK: acc.yield
-  // CHECK-NEXT: } attributes {collapse = [1, 2, 2, 3], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>, #acc.device_type<host>]}
+  // CHECK-NEXT: } attributes {collapse = [1, 2, 2, 3], collapseDeviceType = [#acc.device_type<none>, #acc.device_type<radeon>, #acc.device_type<nvidia>, #acc.device_type<host>], independent = [#acc.device_type<none>]}
 
   #pragma acc loop tile(1, 2, 3)
   for(unsigned I = 0; I < N; ++I)
@@ -392,4 +392,85 @@ extern "C" void acc_loop(int *A, int *B, int *C, int N) {
   // CHECK: acc.yield
   // CHECK-NEXT: } loc
   }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+  // Checking the automatic-addition of parallelism clauses.
+#pragma acc loop
+  for(unsigned I = 0; I < N; ++I);
+  // CHECK-NEXT:  acc.loop {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]} loc
+
+#pragma acc parallel
+  {
+    // CHECK-NEXT: acc.parallel {
+#pragma acc loop
+    for(unsigned I = 0; I < N; ++I);
+  // CHECK-NEXT:  acc.loop {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]} loc
+  }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc kernels
+  {
+    // CHECK-NEXT: acc.kernels {
+#pragma acc loop
+    for(unsigned I = 0; I < N; ++I);
+  // CHECK-NEXT:  acc.loop {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  }
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc serial
+  {
+    // CHECK-NEXT: acc.serial {
+#pragma acc loop
+    for(unsigned I = 0; I < N; ++I);
+  // CHECK-NEXT:  acc.loop {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {seq = [#acc.device_type<none>]} loc
+  }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial
+  {
+    // CHECK-NEXT: acc.serial {
+#pragma acc loop worker
+    for(unsigned I = 0; I < N; ++I);
+  // CHECK-NEXT:  acc.loop worker {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial
+  {
+    // CHECK-NEXT: acc.serial {
+#pragma acc loop vector
+    for(unsigned I = 0; I < N; ++I);
+  // CHECK-NEXT:  acc.loop vector {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial
+  {
+    // CHECK-NEXT: acc.serial {
+#pragma acc loop gang
+    for(unsigned I = 0; I < N; ++I);
+  // CHECK-NEXT:  acc.loop gang {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {auto_ = [#acc.device_type<none>]} loc
+  }
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
 }
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/parallel.c b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/parallel.c
index a6d60ee3ad27..0127613233ec 100644
--- a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/parallel.c
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/parallel.c
@@ -443,3 +443,94 @@ void acc_parallel(int cond) {
 
   // CHECK-NEXT: cir.return
 }
+
+void acc_parallel_data_clauses(int *arg1, int *arg2) {
+  // CHECK: cir.func @acc_parallel_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
+  // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
+  // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+  // CHECK-NEXT: cir.store %[[ARG2_PARAM]], %[[ARG2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc parallel deviceptr(arg1)
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel deviceptr(arg1, arg2)
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel deviceptr(arg1) async
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) async {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel deviceptr(arg1, arg2) device_type(radeon) async
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel no_create(arg1)
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+
+#pragma acc parallel no_create(arg1, arg2) device_type(radeon) async
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[NOCREATE2:.*]] = acc.nocreate varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[NOCREATE1]], %[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+
+#pragma acc parallel present(arg1)
+  ;
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+
+#pragma acc parallel present(arg1, arg2) device_type(radeon) async
+  ;
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[PRESENT2:.*]] = acc.present varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[PRESENT1]], %[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_present>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+
+#pragma acc parallel attach(arg1)
+  ;
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+
+#pragma acc parallel attach(arg1, arg2) device_type(radeon) async
+  ;
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[ATTACH2:.*]] = acc.attach varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[ATTACH1]], %[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_attach>, name = "arg2"}
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+}
+
diff --git a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/serial.c b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/serial.c
index cf077ca2834e..1c9695b34833 100644
--- a/external/llvm-project/clang/test/CIR/CodeGenOpenACC/serial.c
+++ b/external/llvm-project/clang/test/CIR/CodeGenOpenACC/serial.c
@@ -266,3 +266,91 @@ void acc_serial(int cond) {
 
   // CHECK-NEXT: cir.return
 }
+
+void acc_serial_data_clauses(int *arg1, int *arg2) {
+  // CHECK: cir.func @acc_serial_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
+  // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
+  // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+  // CHECK-NEXT: cir.store %[[ARG2_PARAM]], %[[ARG2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc serial deviceptr(arg1)
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial deviceptr(arg1, arg2)
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial deviceptr(arg1) async
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[DEVPTR1]] : !cir.ptr<!cir.ptr<!s32i>>) async {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial deviceptr(arg1, arg2) device_type(radeon) async
+  ;
+  // CHECK-NEXT: %[[DEVPTR1:.*]] = acc.deviceptr varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[DEVPTR2:.*]] = acc.deviceptr varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[DEVPTR1]], %[[DEVPTR2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial no_create(arg1)
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+#pragma acc serial no_create(arg1, arg2) device_type(nvidia) async
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[NOCREATE2:.*]] = acc.nocreate varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[NOCREATE1]], %[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_no_create>, name = "arg1"}
+
+#pragma acc serial present(arg1)
+  ;
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+#pragma acc serial present(arg1, arg2) device_type(nvidia) async
+  ;
+  // CHECK-NEXT: %[[PRESENT1:.*]] = acc.present varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[PRESENT2:.*]] = acc.present varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[PRESENT1]], %[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_present>, name = "arg2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_present>, name = "arg1"}
+
+#pragma acc serial attach(arg1)
+  ;
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+#pragma acc serial attach(arg1, arg2) device_type(nvidia) async
+  ;
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[ARG1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg1"}
+  // CHECK-NEXT: %[[ATTACH2:.*]] = acc.attach varPtr(%[[ARG2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "arg2"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[ATTACH1]], %[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH2]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_attach>, name = "arg2"}
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<nvidia>]) {dataClause = #acc<data_clause acc_attach>, name = "arg1"}
+}
diff --git a/external/llvm-project/clang/test/CIR/IR/complex.cir b/external/llvm-project/clang/test/CIR/IR/complex.cir
new file mode 100644
index 000000000000..a73a8654ca27
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/IR/complex.cir
@@ -0,0 +1,16 @@
+// RUN: cir-opt %s | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module  {
+
+cir.global external @ci = #cir.zero : !cir.complex<!s32i>
+// CHECK: cir.global external {{.*}} = #cir.zero : !cir.complex<!s32i>
+
+cir.global external @cf = #cir.zero : !cir.complex<!cir.float>
+// CHECK: cir.global external {{.*}} = #cir.zero : !cir.complex<!cir.float>
+
+cir.global external @ci2 = #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+// CHECK: cir.global external {{.*}} = #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+
+}
diff --git a/external/llvm-project/clang/test/CIR/IR/invalid-complex.cir b/external/llvm-project/clang/test/CIR/IR/invalid-complex.cir
new file mode 100644
index 000000000000..8c6d89057932
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/IR/invalid-complex.cir
@@ -0,0 +1,23 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module {
+
+// expected-error @below {{type of the real part does not match the complex type}}
+cir.global external @ci2 = #cir.const_complex<#cir.int<1> : !s64i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module {
+
+// expected-error @below {{type of the imaginary part does not match the complex type}}
+cir.global external @ci2 = #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s64i> : !cir.complex<!s32i>
+
+}
diff --git a/external/llvm-project/clang/test/CIR/IR/invalid-vector-shuffle-wrong-index.cir b/external/llvm-project/clang/test/CIR/IR/invalid-vector-shuffle-wrong-index.cir
new file mode 100644
index 000000000000..375b2d3dc563
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/IR/invalid-vector-shuffle-wrong-index.cir
@@ -0,0 +1,16 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+    %vec_1 = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<3> : !s32i, #cir.int<5> : !s32i, #cir.int<7> : !s32i]> : !cir.vector<4 x !s32i>
+    %vec_2 = cir.const #cir.const_vector<[#cir.int<2> : !s32i, #cir.int<4> : !s32i, #cir.int<6> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
+
+    // expected-error @below {{index for __builtin_shufflevector must be less than the total number of vector elements}}
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<9> : !s64i, #cir.int<4> : !s64i,
+      #cir.int<1> : !s64i, #cir.int<5> : !s64i] : !cir.vector<4 x !s32i>
+    cir.return %new_vec : !cir.vector<4 x !s32i>
+  }
+}
diff --git a/external/llvm-project/clang/test/CIR/IR/invalid-vector.cir b/external/llvm-project/clang/test/CIR/IR/invalid-vector.cir
index d94eacedec1f..98e1109df84f 100644
--- a/external/llvm-project/clang/test/CIR/IR/invalid-vector.cir
+++ b/external/llvm-project/clang/test/CIR/IR/invalid-vector.cir
@@ -4,7 +4,45 @@
 
 module  {
 
-// expected-error @below {{unsupported element type for CIR vector}}
+// expected-error @below {{failed to verify 'elementType'}}
 cir.global external @vec_b = #cir.zero : !cir.vector<4 x !cir.array<!s32i x 10>>
 
 }
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @invalid_vector_shuffle() {
+    %1 = cir.const #cir.int<1> : !s32i
+    %2 = cir.const #cir.int<2> : !s32i
+    %3 = cir.const #cir.int<3> : !s32i
+    %4 = cir.const #cir.int<4> : !s32i
+    %vec_1 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    %vec_2 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    // expected-error @below {{element types of '!cir.vector<4 x !cir.int<s, 32>>' and '!cir.vector<4 x !cir.int<s, 64>>' don't match}}
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<7> : !s64i, #cir.int<5> : !s64i, #cir.int<3> : !s64i, #cir.int<1> : !s64i] : !cir.vector<4 x !s64i>
+    cir.return
+  }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @invalid_vector_shuffle() {
+    %1 = cir.const #cir.int<1> : !s32i
+    %2 = cir.const #cir.int<2> : !s32i
+    %3 = cir.const #cir.int<3> : !s32i
+    %4 = cir.const #cir.int<4> : !s32i
+    %vec_1 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    %vec_2 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    // expected-error @below {{the number of elements in [#cir.int<7> : !cir.int<s, 64>, #cir.int<5> : !cir.int<s, 64>, #cir.int<3> : !cir.int<s, 64>] and '!cir.vector<4 x !cir.int<s, 64>>' don't match}}
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<7> : !s64i, #cir.int<5> : !s64i, #cir.int<3> : !s64i] : !cir.vector<4 x !s64i>
+    cir.return
+  }
+}
diff --git a/external/llvm-project/clang/test/CIR/IR/struct.cir b/external/llvm-project/clang/test/CIR/IR/struct.cir
index 7f0ce0763118..5ff5ff7cd1f6 100644
--- a/external/llvm-project/clang/test/CIR/IR/struct.cir
+++ b/external/llvm-project/clang/test/CIR/IR/struct.cir
@@ -1,15 +1,19 @@
 // RUN: cir-opt %s | FileCheck %s
 
+!rec_C = !cir.record<class "C" incomplete>
 !rec_S = !cir.record<struct "S" incomplete>
 !rec_U = !cir.record<union "U" incomplete>
 
+// CHECK: !rec_C = !cir.record<class "C" incomplete>
 // CHECK: !rec_S = !cir.record<struct "S" incomplete>
 // CHECK: !rec_U = !cir.record<union "U" incomplete>
 
 module  {
     cir.global external @p1 = #cir.ptr<null> : !cir.ptr<!rec_S>
     cir.global external @p2 = #cir.ptr<null> : !cir.ptr<!rec_U>
+    cir.global external @p3 = #cir.ptr<null> : !cir.ptr<!rec_C>
 }
 
 // CHECK: cir.global external @p1 = #cir.ptr<null> : !cir.ptr<!rec_S>
 // CHECK: cir.global external @p2 = #cir.ptr<null> : !cir.ptr<!rec_U>
+// CHECK: cir.global external @p3 = #cir.ptr<null> : !cir.ptr<!rec_C>
diff --git a/external/llvm-project/clang/test/CIR/IR/vector.cir b/external/llvm-project/clang/test/CIR/IR/vector.cir
index a455acf92ab6..f23f5de9692d 100644
--- a/external/llvm-project/clang/test/CIR/IR/vector.cir
+++ b/external/llvm-project/clang/test/CIR/IR/vector.cir
@@ -187,4 +187,37 @@ cir.func @vector_shuffle_dynamic_test() {
 // CHECK:    cir.return
 // CHECK: }
 
+cir.func @vector_splat_test() {
+    %0 = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
+    %1 = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
+    %2 = cir.const #cir.int<1> : !s32i
+    %3 = cir.const #cir.int<2> : !s32i
+    %4 = cir.const #cir.int<3> : !s32i
+    %5 = cir.const #cir.int<4> : !s32i
+    %6 = cir.vec.create(%2, %3, %4, %5 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    cir.store %6, %0 : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+    %7 = cir.load %0 : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+    %8 = cir.const #cir.int<3> : !s32i
+    %9 = cir.vec.splat %8 : !s32i, !cir.vector<4 x !s32i>
+    %10 = cir.shift(left, %7 : !cir.vector<4 x !s32i>, %9 : !cir.vector<4 x !s32i>) -> !cir.vector<4 x !s32i>
+    cir.store %10, %1 : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+    cir.return
+}
+
+// CHECK: cir.func @vector_splat_test() {
+// CHECK-NEXT: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
+// CHECK-NEXT: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
+// CHECK-NEXT: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
+// CHECK-NEXT: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s32i
+// CHECK-NEXT: %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
+// CHECK-NEXT: %[[CONST_4:.*]] = cir.const #cir.int<4> : !s32i
+// CHECK-NEXT: %[[VEC_VAL:.*]] = cir.vec.create(%[[CONST_1]], %[[CONST_2]], %[[CONST_3]], %[[CONST_4]] : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+// CHECK-NEXT: cir.store %[[VEC_VAL]], %[[VEC]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CHECK-NEXT: %[[TMP:.*]] = cir.load %[[VEC]] : !cir.ptr<!cir.vector<4 x !s32i>>, !cir.vector<4 x !s32i>
+// CHECK-NEXT: %[[SPLAT_VAL:.*]] = cir.const #cir.int<3> : !s32i
+// CHECK-NEXT: %[[SPLAT_VEC:.*]] = cir.vec.splat %[[SPLAT_VAL]] : !s32i, !cir.vector<4 x !s32i>
+// CHECK-NEXT: %[[SHL:.*]] = cir.shift(left, %[[TMP]] : !cir.vector<4 x !s32i>, %[[SPLAT_VEC]] : !cir.vector<4 x !s32i>) -> !cir.vector<4 x !s32i>
+// CHECK-NEXT: cir.store %[[SHL]], %[[SHL_RES:.*]] : !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>
+// CHECK-NEXT: cir.return
+
 }
diff --git a/external/llvm-project/clang/test/CIR/Lowering/func-simple.cpp b/external/llvm-project/clang/test/CIR/Lowering/func-simple.cpp
index 3d6a837daf6e..96306babc867 100644
--- a/external/llvm-project/clang/test/CIR/Lowering/func-simple.cpp
+++ b/external/llvm-project/clang/test/CIR/Lowering/func-simple.cpp
@@ -41,9 +41,9 @@ int scopes() {
 
 long longfunc() { return 42l; }
 // CHECK: define{{.*}} i64 @_Z8longfuncv() {
-// CHECK:   %[[RV]] = alloca i64, i64 1, align 8
-// CHECK:   store i64 42, ptr %[[RV]], align 4
-// CHECK:   %[[R:.*]] = load i64, ptr %[[RV]], align 4
+// CHECK:   %[[RV:.*]] = alloca i64, i64 1, align 8
+// CHECK:   store i64 42, ptr %[[RV]], align 8
+// CHECK:   %[[R:.*]] = load i64, ptr %[[RV]], align 8
 // CHECK:   ret i64 %[[R]]
 // CHECK: }
 
@@ -58,8 +58,8 @@ unsigned unsignedfunc() { return 42u; }
 unsigned long long ullfunc() { return 42ull; }
 // CHECK: define{{.*}} i64 @_Z7ullfuncv() {
 // CHECK:   %[[RV:.*]] = alloca i64, i64 1, align 8
-// CHECK:   store i64 42, ptr %[[RV]], align 4
-// CHECK:   %[[R:.*]] = load i64, ptr %[[RV]], align 4
+// CHECK:   store i64 42, ptr %[[RV]], align 8
+// CHECK:   %[[R:.*]] = load i64, ptr %[[RV]], align 8
 // CHECK:   ret i64 %[[R]]
 // CHECK: }
 
diff --git a/external/llvm-project/clang/test/CIR/Transforms/complex-create-fold.cir b/external/llvm-project/clang/test/CIR/Transforms/complex-create-fold.cir
new file mode 100644
index 000000000000..5d9d22112c8b
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/Transforms/complex-create-fold.cir
@@ -0,0 +1,30 @@
+// RUN: cir-opt %s -cir-canonicalize -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module {
+  cir.func @fold_complex_create_test() -> !cir.complex<!s32i>  {
+    %0 = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["__retval"]
+    %1 = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+    %2 = cir.const #cir.int<1> : !s32i
+    %3 = cir.const #cir.int<2> : !s32i
+    %4 = cir.complex.create %2, %3 : !s32i -> !cir.complex<!s32i>
+    cir.store align(4) %4, %1 : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+    %5 = cir.load align(4) %1 : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+    cir.store align(4) %5, %0 : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+    %6 = cir.load %0 : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+    cir.return %6 : !cir.complex<!s32i>
+  }
+
+// CHECK: cir.func @fold_complex_create_test() -> !cir.complex<!s32i> {
+// CHECK:   %[[RET:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["__retval"]
+// CHECK:   %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CHECK:   %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+// CHECK:   cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+// CHECK:   %[[TMP:.*]] = cir.load{{.*}} %[[INIT]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CHECK:   cir.store{{.*}} %[[TMP:.*]], %[[RET]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+// CHECK:   %[[TMP_2:.*]] = cir.load %[[RET]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CHECK:   cir.return %[[TMP_2]] : !cir.complex<!s32i>
+// CHECK: }
+
+}
diff --git a/external/llvm-project/clang/test/CIR/Transforms/vector-create-fold.cir b/external/llvm-project/clang/test/CIR/Transforms/vector-create-fold.cir
new file mode 100644
index 000000000000..fb8f66dc4deb
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/Transforms/vector-create-fold.cir
@@ -0,0 +1,19 @@
+// RUN: cir-opt %s -cir-canonicalize -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module  {
+  cir.func @fold_create_vector_op_test() -> !cir.vector<4 x !s32i> {
+    %2 = cir.const #cir.int<1> : !s32i
+    %3 = cir.const #cir.int<2> : !s32i
+    %4 = cir.const #cir.int<3> : !s32i
+    %5 = cir.const #cir.int<4> : !s32i
+    %vec = cir.vec.create(%2, %3, %4, %5 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    cir.return %vec : !cir.vector<4 x !s32i>
+  }
+
+  // CHECK:  cir.func @fold_create_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK-NEXT: %[[VEC:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+  // CHECK-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+  // CHECK-NEXT: cir.return %[[VEC]] : !cir.vector<4 x !s32i>
+}
diff --git a/external/llvm-project/clang/test/CIR/Transforms/vector-shuffle-dynamic-fold.cir b/external/llvm-project/clang/test/CIR/Transforms/vector-shuffle-dynamic-fold.cir
new file mode 100644
index 000000000000..46ab04502afe
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/Transforms/vector-shuffle-dynamic-fold.cir
@@ -0,0 +1,31 @@
+// RUN: cir-opt %s -cir-canonicalize -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module {
+  cir.func @fold_shuffle_dynamic_vector_op_test() -> !cir.vector<4 x !s32i> {
+    %vec = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+    %indices = cir.const #cir.const_vector<[#cir.int<8> : !s32i, #cir.int<7> : !s32i, #cir.int<6> : !s32i, #cir.int<5> : !s32i]> : !cir.vector<4 x !s32i>
+    %new_vec = cir.vec.shuffle.dynamic %vec : !cir.vector<4 x !s32i>, %indices : !cir.vector<4 x !s32i>
+    cir.return %new_vec : !cir.vector<4 x !s32i>
+  }
+
+  // Masking indices [8, 7, 6, 5] AND 3 = [0, 3, 2, 1]
+  // CHECK: cir.func @fold_shuffle_dynamic_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK-NEXT: %[[NEW_VEC:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<4> : !s32i, #cir.int<3> : !s32i, #cir.int<2> : !s32i]> : !cir.vector<4 x !s32i>
+  // CHECK-NEXT: cir.return %[[NEW_VEC:.*]] : !cir.vector<4 x !s32i>
+
+  cir.func @fold_shuffle_dynamic_vector_op_test_2() -> !cir.vector<4 x !s32i> {
+    %vec = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+    %indices = cir.const #cir.const_vector<[#cir.int<3> : !s32i, #cir.int<2> : !s32i, #cir.int<1> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
+    %new_vec = cir.vec.shuffle.dynamic %vec : !cir.vector<4 x !s32i>, %indices : !cir.vector<4 x !s32i>
+    cir.return %new_vec : !cir.vector<4 x !s32i>
+  }
+
+  // Masking indices [3, 2, 1, 0] AND 3 = [3, 2, 1, 0]
+  // CHECK: cir.func @fold_shuffle_dynamic_vector_op_test_2() -> !cir.vector<4 x !s32i> {
+  // CHECK-NEXT: %[[NEW_VEC:.*]] = cir.const #cir.const_vector<[#cir.int<4> : !s32i, #cir.int<3> : !s32i, #cir.int<2> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
+  // CHECK-NEXT: cir.return %[[NEW_VEC:.*]] : !cir.vector<4 x !s32i>
+}
+
+
diff --git a/external/llvm-project/clang/test/CIR/Transforms/vector-shuffle-fold.cir b/external/llvm-project/clang/test/CIR/Transforms/vector-shuffle-fold.cir
new file mode 100644
index 000000000000..87d409728989
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/Transforms/vector-shuffle-fold.cir
@@ -0,0 +1,59 @@
+// RUN: cir-opt %s -cir-canonicalize -o - -split-input-file | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+    %vec_1 = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<3> : !s32i, #cir.int<5> : !s32i, #cir.int<7> : !s32i]> : !cir.vector<4 x !s32i>
+    %vec_2 = cir.const #cir.const_vector<[#cir.int<2> : !s32i, #cir.int<4> : !s32i, #cir.int<6> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<0> : !s64i, #cir.int<4> : !s64i,
+      #cir.int<1> : !s64i, #cir.int<5> : !s64i] : !cir.vector<4 x !s32i>
+    cir.return %new_vec : !cir.vector<4 x !s32i>
+  }
+
+  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
+  // CHECK-SAME: #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+  // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @fold_shuffle_vector_op_test() -> !cir.vector<6 x !s32i> {
+    %vec_1 = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<3> : !s32i, #cir.int<5> : !s32i, #cir.int<7> : !s32i]> : !cir.vector<4 x !s32i>
+    %vec_2 = cir.const #cir.const_vector<[#cir.int<2> : !s32i, #cir.int<4> : !s32i, #cir.int<6> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<0> : !s64i, #cir.int<4> : !s64i,
+      #cir.int<1> : !s64i, #cir.int<5> : !s64i, #cir.int<2> : !s64i, #cir.int<6> : !s64i] : !cir.vector<6 x !s32i>
+    cir.return %new_vec : !cir.vector<6 x !s32i>
+  }
+
+  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<6 x !s32i> {
+  // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
+  // CHECK-SAME: #cir.int<4> : !s32i, #cir.int<5> : !s32i, #cir.int<6> : !s32i]> : !cir.vector<6 x !s32i>
+  // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<6 x !s32i>
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+    %vec_1 = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<3> : !s32i, #cir.int<5> : !s32i, #cir.int<7> : !s32i]> : !cir.vector<4 x !s32i>
+    %vec_2 = cir.const #cir.const_vector<[#cir.int<2> : !s32i, #cir.int<4> : !s32i, #cir.int<6> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<-1> : !s64i, #cir.int<4> : !s64i,
+      #cir.int<1> : !s64i, #cir.int<5> : !s64i] : !cir.vector<4 x !s32i>
+    cir.return %new_vec : !cir.vector<4 x !s32i>
+  }
+
+  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK: cir.const #cir.const_vector<[#cir.undef : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
+  // CHECK-SAME: #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+  // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
+}
diff --git a/external/llvm-project/clang/test/CIR/Transforms/vector-ternary-fold.cir b/external/llvm-project/clang/test/CIR/Transforms/vector-ternary-fold.cir
new file mode 100644
index 000000000000..f2e18576da74
--- /dev/null
+++ b/external/llvm-project/clang/test/CIR/Transforms/vector-ternary-fold.cir
@@ -0,0 +1,20 @@
+// RUN: cir-opt %s -cir-canonicalize -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module {
+  cir.func @vector_ternary_fold_test() -> !cir.vector<4 x !s32i> {
+    %cond = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
+    %lhs = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
+    %rhs = cir.const #cir.const_vector<[#cir.int<5> : !s32i, #cir.int<6> : !s32i, #cir.int<7> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
+    %res = cir.vec.ternary(%cond, %lhs, %rhs) : !cir.vector<4 x !s32i>, !cir.vector<4 x !s32i>
+    cir.return %res : !cir.vector<4 x !s32i>
+  }
+
+  // [1, 0, 1, 0] ? [1, 2, 3, 4] : [5, 6, 7, 8] Will be fold to [1, 6, 3, 8]
+  // CHECK: cir.func @vector_ternary_fold_test() -> !cir.vector<4 x !s32i> {
+  // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<6> : !s32i, #cir.int<3> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
+  // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
+}
+
+
diff --git a/external/llvm-project/clang/test/CIR/global-var-linkage.cpp b/external/llvm-project/clang/test/CIR/global-var-linkage.cpp
index 1c0c1b121427..3a288b8990ad 100644
--- a/external/llvm-project/clang/test/CIR/global-var-linkage.cpp
+++ b/external/llvm-project/clang/test/CIR/global-var-linkage.cpp
@@ -12,22 +12,19 @@ int aaaa;
 // OGCG: @aaaa = global i32 0
 
 [[gnu::selectany]] int dddd;
-// CIR: cir.global weak_odr @dddd
-// LLVM: @dddd = weak_odr global i32 0
-// This actually should be here, but we aren't emitting it yet so I want the
-// test to reflect that.
-// LLVM-NOT: comdat
+// CIR: cir.global weak_odr comdat @dddd
+// LLVM: @dddd = weak_odr global i32 0, comdat
 // OGCG: @dddd = weak_odr global i32 0, comdat
 
 static int bbbb;
-// CIR: cir.global "private" internal @_ZL4bbbb
+// CIR: cir.global "private" internal dsolocal @_ZL4bbbb
 // LLVM: @_ZL4bbbb = internal global i32 0
 // OGCG: @_ZL4bbbb = internal global i32 0
 
 inline int cccc;
-// CIR: cir.global linkonce_odr @cccc
-// LLVM: @cccc = linkonce_odr global i32 0
-// OGCG: @cccc = linkonce_odr global i32 0
+// CIR: cir.global linkonce_odr comdat @cccc
+// LLVM: @cccc = linkonce_odr global i32 0, comdat
+// OGCG: @cccc = linkonce_odr global i32 0, comdat
 
 // Force the global variables to be emitted
 void reference_vars() {
diff --git a/external/llvm-project/clang/test/CXX/class.access/p4.cpp b/external/llvm-project/clang/test/CXX/class.access/p4.cpp
index ca98c9f90bd8..6d4c8c004911 100644
--- a/external/llvm-project/clang/test/CXX/class.access/p4.cpp
+++ b/external/llvm-project/clang/test/CXX/class.access/p4.cpp
@@ -21,11 +21,13 @@ namespace test0 {
   public:
     void foo(Public&);
   protected:
-    void foo(Protected&); // expected-note 2 {{declared protected here}}
+    void foo(Protected&); // expected-note 4 {{declared protected here}}
   private:
-    void foo(Private&); // expected-note 2 {{declared private here}}
+    void foo(Private&); // expected-note 4 {{declared private here}}
   };
 
+  class B : public A {};
+
   void test(A *op) {
     op->foo(PublicInst);
     op->foo(ProtectedInst); // expected-error {{'foo' is a protected member}}
@@ -35,6 +37,16 @@ namespace test0 {
     void (A::*b)(Protected&) = &A::foo; // expected-error {{'foo' is a protected member}}
     void (A::*c)(Private&) = &A::foo; // expected-error {{'foo' is a private member}}
   }
+
+  void test(B *op) {
+    op->foo(PublicInst);
+    op->foo(ProtectedInst); // expected-error {{'foo' is a protected member}}
+    op->foo(PrivateInst); // expected-error {{'foo' is a private member}}
+
+    void (B::*a)(Public&) = &B::foo;
+    void (B::*b)(Protected&) = &B::foo; // expected-error {{'foo' is a protected member}}
+    void (B::*c)(Private&) = &B::foo; // expected-error {{'foo' is a private member}}
+  }
 }
 
 // Member operators.
diff --git a/external/llvm-project/clang/test/CXX/drs/cwg1xx.cpp b/external/llvm-project/clang/test/CXX/drs/cwg1xx.cpp
index 8b84de0ab5a9..c9dce77b772d 100644
--- a/external/llvm-project/clang/test/CXX/drs/cwg1xx.cpp
+++ b/external/llvm-project/clang/test/CXX/drs/cwg1xx.cpp
@@ -702,8 +702,7 @@ namespace cwg141 { // cwg141: 3.1
     // cxx98-error@#cwg141-a {{lookup of 'S' in member access expression is ambiguous; using member of 'struct A'}}
     //   cxx98-note@#cwg141-A-S {{lookup in the object type 'struct A' refers here}}
     //   cxx98-note@#cwg141-S {{lookup from the current scope refers here}}
-    // expected-error@#cwg141-a {{no member named 'n' in 'cwg141::A::S<int>'; did you mean '::cwg141::S<int>::n'?}}
-    //   expected-note@#cwg141-S {{'::cwg141::S<int>::n' declared here}}
+    // expected-error@#cwg141-a {{no member named 'n' in 'cwg141::A::S<int>'}}
     // FIXME: we issue a useful diagnostic first, then some bogus ones.
     b.f<int>();
     // expected-error@-1 {{no member named 'f' in 'cwg141::B'}}
diff --git a/external/llvm-project/clang/test/CXX/drs/cwg23xx.cpp b/external/llvm-project/clang/test/CXX/drs/cwg23xx.cpp
index 74e72f2371e2..128566537b57 100644
--- a/external/llvm-project/clang/test/CXX/drs/cwg23xx.cpp
+++ b/external/llvm-project/clang/test/CXX/drs/cwg23xx.cpp
@@ -365,6 +365,35 @@ struct A {
 #endif
 } // namespace cwg2363
 
+namespace cwg2369 { // cwg2369: partial
+#if __cplusplus >= 202002L
+template <class T> struct Z {
+  typedef typename T::x xx;
+};
+
+template <class T>
+concept C = requires { typename T::A; };
+template <C T> typename Z<T>::xx f(void *, T); // #1
+template <class T> void f(int, T);             // #2
+
+struct A {
+} a;
+
+struct ZZ {
+  template <class T, class = typename Z<T>::xx> operator T *();
+  operator int();
+};
+
+void foo() {
+  ZZ zz;
+  f(1, a); // OK, deduction fails for #1 because there is no conversion from int
+           // to void*
+  f(zz, 42); // OK, deduction fails for #1 because C<int> is not satisfied
+}
+
+#endif
+} // namespace cwg2369
+
 namespace cwg2370 { // cwg2370: no
 namespace N {
 typedef int type;
diff --git a/external/llvm-project/clang/test/CXX/drs/cwg24xx.cpp b/external/llvm-project/clang/test/CXX/drs/cwg24xx.cpp
index 9c9a3f14b9e8..c499a2dfcccb 100644
--- a/external/llvm-project/clang/test/CXX/drs/cwg24xx.cpp
+++ b/external/llvm-project/clang/test/CXX/drs/cwg24xx.cpp
@@ -215,3 +215,34 @@ void (*q)() throw() = S();
 // since-cxx17-error@-1 {{no viable conversion from 'S' to 'void (*)() throw()'}}
 //   since-cxx17-note@#cwg2486-conv {{candidate function}}
 } // namespace cwg2486
+
+
+namespace cwg2496 { // cwg2496: 21
+#if __cplusplus >= 201102L
+struct S {
+    virtual void f(); // #cwg2496-f
+    virtual void g() &; // #cwg2496-g
+    virtual void h(); // #cwg2496-h
+    virtual void i();
+    virtual void j() &;
+    virtual void k() &&;
+    virtual void l() &;
+};
+
+struct T : S {
+    virtual void f() &;
+    // expected-error@-1 {{cannot overload a member function with ref-qualifier '&' with a member function without a ref-qualifier}}
+    // expected-note@#cwg2496-f {{previous declaration is here}}
+    virtual void g();
+    // expected-error@-1 {{cannot overload a member function without a ref-qualifier with a member function with ref-qualifier '&'}}
+    // expected-note@#cwg2496-g {{previous declaration is here}}
+    virtual void h() &&;
+    // expected-error@-1 {{cannot overload a member function with ref-qualifier '&&' with a member function without a ref-qualifier}}
+    // expected-note@#cwg2496-h {{previous declaration is here}}
+    virtual void i();
+    virtual void j() &;
+    virtual void k() &;
+    virtual void l() &&;
+};
+#endif
+}
diff --git a/external/llvm-project/clang/test/CXX/drs/cwg25xx.cpp b/external/llvm-project/clang/test/CXX/drs/cwg25xx.cpp
index d9a7d2bbb267..5c2948f67d0e 100644
--- a/external/llvm-project/clang/test/CXX/drs/cwg25xx.cpp
+++ b/external/llvm-project/clang/test/CXX/drs/cwg25xx.cpp
@@ -32,6 +32,26 @@ enum E2 : S<E2>::I { e };
 #endif
 } // namespace cwg2516
 
+namespace cwg2517 { // cwg2517: 21
+#if __cplusplus >= 202002L
+template<typename ArrayType>
+concept LargeArray = requires (ArrayType my_array) {
+  requires my_array.size() > 5;
+};
+
+struct Big {
+  constexpr int size() const { return 100; }
+};
+
+struct Small {
+  constexpr int size() const { return 3; }
+};
+
+static_assert(LargeArray<Big>);
+static_assert(!LargeArray<Small>);
+#endif
+} // namespace cwg2517
+
 namespace cwg2518 { // cwg2518: 17
 
 #if __cplusplus >= 201103L
diff --git a/external/llvm-project/clang/test/CXX/drs/cwg26xx.cpp b/external/llvm-project/clang/test/CXX/drs/cwg26xx.cpp
index 3eb70583b602..60d896443ecd 100644
--- a/external/llvm-project/clang/test/CXX/drs/cwg26xx.cpp
+++ b/external/llvm-project/clang/test/CXX/drs/cwg26xx.cpp
@@ -220,7 +220,6 @@ int x = cwg2640_a\N{abc});
 int y = cwg2640_a\N{LOTUS});
 // expected-error@-1 {{character <U+1FAB7> not allowed in an identifier}}
 // expected-error@-2 {{use of undeclared identifier 'cwg2640_a🪷'}}
-// expected-error@-3 {{extraneous ')' before ';'}}
 } // namespace cwg2640
 
 // cwg2642: na
@@ -310,7 +309,7 @@ void f(T) requires requires { []() { T::invalid; } (); };
 //   since-cxx20-note@-3 {{in instantiation of requirement here}}
 //   since-cxx20-note@-4 {{while substituting template arguments into constraint expression here}}
 //   since-cxx20-note@#cwg2672-f-0 {{while checking constraint satisfaction for template 'f<int>' required here}}
-//   since-cxx20-note@#cwg2672-f-0 {{in instantiation of function template specialization 'cwg2672::f<int>' requested here}}
+//   since-cxx20-note@#cwg2672-f-0 {{while substituting deduced template arguments into function template 'f' [with T = int]}}
 void f(...);
 
 template <class T>
diff --git a/external/llvm-project/clang/test/CXX/drs/cwg27xx.cpp b/external/llvm-project/clang/test/CXX/drs/cwg27xx.cpp
index a87d26dfc9ac..7caf36a9f23b 100644
--- a/external/llvm-project/clang/test/CXX/drs/cwg27xx.cpp
+++ b/external/llvm-project/clang/test/CXX/drs/cwg27xx.cpp
@@ -174,6 +174,26 @@ static_assert(!__is_layout_compatible(StructWithAnonUnion, StructWithAnonUnion3)
 #endif
 } // namespace cwg2759
 
+namespace cwg2770 { // cwg2770: 20 open 2023-07-14
+#if __cplusplus >= 202002L
+template<typename T>
+struct B {
+  static_assert(sizeof(T) == 1);
+  using type = int;
+};
+
+template<typename T>
+int f(T t, typename B<T>::type u) requires (sizeof(t) == 1);
+
+template<typename T>
+int f(T t, long);
+
+int i = f(1, 2);
+int j = f('a', 2);
+
+#endif
+} // namespace cwg2770
+
 namespace cwg2789 { // cwg2789: 18
 #if __cplusplus >= 202302L
 template <typename T = int>
diff --git a/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/nested-requirement.cpp b/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/nested-requirement.cpp
index 651cca927d51..033ae349a02e 100644
--- a/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/nested-requirement.cpp
+++ b/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/nested-requirement.cpp
@@ -41,9 +41,17 @@ namespace std_example {
   template<typename T>
   concept C2 = requires (T a) {
       requires sizeof(a) == 4; // OK
-      requires a == 0; // expected-note{{because 'a == 0' would be invalid: constraint variable 'a' cannot be used in an evaluated context}}
+      requires a == 0; // expected-error{{substitution into constraint expression resulted in a non-constant expression}}
+      // expected-note@-1{{while checking the satisfaction of nested requirement requested here}}
+      // expected-note@-2{{in instantiation of requirement here}}
+      // expected-note@-3{{while checking the satisfaction of nested requirement requested here}}
+      // expected-note@-6{{while substituting template arguments into constraint expression here}}
+      // expected-note@-5{{function parameter 'a' with unknown value cannot be used in a constant expression}}
+      // expected-note@-8{{declared here}}
     };
-  static_assert(C2<int>); // expected-note{{because 'int' does not satisfy 'C2'}} expected-error{{static assertion failed}}
+    static_assert(C2<int>); // expected-error{{static assertion failed}}
+    // expected-note@-1{{while checking the satisfaction of concept 'C2<int>' requested here}}
+    // expected-note@-2{{because 'int' does not satisfy 'C2'}}
 }
 
 template<typename T>
@@ -154,7 +162,7 @@ void func() {
 
   bar<int>();
   // expected-note@-1 {{while checking constraint satisfaction for template 'bar<int>' required here}} \
-  // expected-note@-1 {{in instantiation of function template specialization}}
+  // expected-note@-1 {{while substituting deduced template arguments into function template 'bar' [with T = int]}}
   // expected-note@#bar {{in instantiation of static data member}}
   // expected-note@#bar {{in instantiation of requirement here}}
   // expected-note@#bar {{while checking the satisfaction of nested requirement requested here}}
diff --git a/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/simple-requirement.cpp b/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/simple-requirement.cpp
index 7515f5c62d5e..5199708cd816 100644
--- a/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/simple-requirement.cpp
+++ b/external/llvm-project/clang/test/CXX/expr/expr.prim/expr.prim.req/simple-requirement.cpp
@@ -102,8 +102,10 @@ namespace std_example {
 // of a polymorphic type.
 class X { virtual ~X(); };
 constexpr bool b = requires (X &x) { static_cast<int(*)[(typeid(x), 0)]>(nullptr); };
-// expected-error@-1{{constraint variable 'x' cannot be used in an evaluated context}}
-// expected-note@-2{{'x' declared here}}
+// expected-warning@-1 {{left operand of comma operator has no effect}}
+// expected-warning@-2 {{variable length arrays in C++ are a Clang extension}}
+// expected-note@-3{{function parameter 'x' with unknown value cannot be used in a constant expression}}
+// expected-note@-4{{declared here}}
 
 namespace access_checks {
 namespace in_requires_expression {
diff --git a/external/llvm-project/clang/test/CXX/module/basic/basic.link/p2.cppm b/external/llvm-project/clang/test/CXX/module/basic/basic.link/p2.cppm
index d7d2b5992a23..6a2c67526c9a 100644
--- a/external/llvm-project/clang/test/CXX/module/basic/basic.link/p2.cppm
+++ b/external/llvm-project/clang/test/CXX/module/basic/basic.link/p2.cppm
@@ -51,7 +51,7 @@ void use_from_module_impl() {
   (void)external_linkage_var;
   (void)module_linkage_var;
 
-  (void)internal_linkage_class{}; // expected-error {{use of undeclared identifier 'internal_linkage_class'}} //expected-error{{}}
+  (void)internal_linkage_class{}; // expected-error {{use of undeclared identifier 'internal_linkage_class'}} // expected-note@* {{}}
   (void)internal_linkage_var; // expected-error {{use of undeclared identifier 'internal_linkage_var'}}
 }
 
@@ -64,7 +64,7 @@ void use_from_module_impl() {
   internal_linkage_fn(); // expected-error {{use of undeclared identifier 'internal_linkage_fn'}}
   (void)external_linkage_class{};
   (void)module_linkage_class{}; // expected-error {{undeclared identifier}} expected-error 0+{{}} // expected-note@* {{}}
-  (void)internal_linkage_class{}; // expected-error {{undeclared identifier}} expected-error 0+{{}}
+  (void)internal_linkage_class{}; // expected-error {{undeclared identifier}} expected-error 0+{{}} // expected-note@* {{}}
   (void)external_linkage_var;
   (void)module_linkage_var; // expected-error {{undeclared identifier}}
   (void)internal_linkage_var; // expected-error {{undeclared identifier}}
diff --git a/external/llvm-project/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm b/external/llvm-project/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
index 873e4c0edeac..3670f9430ed4 100644
--- a/external/llvm-project/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
+++ b/external/llvm-project/clang/test/CXX/module/dcl.dcl/dcl.module/dcl.module.import/p1.cppm
@@ -45,7 +45,7 @@ import x;
 import x [[]];
 import x [[foo]]; // expected-warning {{unknown attribute 'foo' ignored}}
 import x [[noreturn]]; // expected-error {{'noreturn' attribute cannot be applied to a module import}}
-import x [[blarg::noreturn]]; // expected-warning {{unknown attribute 'noreturn' ignored}}
+import x [[blarg::noreturn]]; // expected-warning {{unknown attribute 'blarg::noreturn' ignored}}
 
 import x.y;
 import x.; // expected-error {{expected a module name after 'import'}}
diff --git a/external/llvm-project/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp b/external/llvm-project/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
index 083e74381812..59e6a48e4887 100644
--- a/external/llvm-project/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
+++ b/external/llvm-project/clang/test/CXX/temp/temp.constr/temp.constr.atomic/constrant-satisfaction-conversions.cpp
@@ -11,7 +11,7 @@ template<typename T> struct S {
 
 // expected-error@+3{{atomic constraint must be of type 'bool' (found 'S<int>')}}
 // expected-note@#FINST{{while checking constraint satisfaction}}
-// expected-note@#FINST{{in instantiation of function template specialization}}
+// expected-note@#FINST{{while substituting deduced template arguments into function template 'f' [with T = int]}}
 template<typename T> requires (S<T>{})
 void f(T);
 void f(long);
@@ -19,7 +19,7 @@ void f(long);
 // Ensure this applies to operator && as well.
 // expected-error@+3{{atomic constraint must be of type 'bool' (found 'S<int>')}}
 // expected-note@#F2INST{{while checking constraint satisfaction}}
-// expected-note@#F2INST{{in instantiation of function template specialization}}
+// expected-note@#F2INST{{while substituting deduced template arguments into function template 'f2' [with T = int]}}
 template<typename T> requires (S<T>{} && true)
 void f2(T);
 void f2(long);
@@ -32,7 +32,7 @@ template<typename T> requires requires {
   // expected-note@-4{{while checking the satisfaction}}
   // expected-note@-6{{while substituting template arguments}}
   // expected-note@#F3INST{{while checking constraint satisfaction}}
-  // expected-note@#F3INST{{in instantiation of function template specialization}}
+  // expected-note@#F3INST{{while substituting deduced template arguments into function template 'f3' [with T = int]}}
   //
 }
 void f3(T);
diff --git a/external/llvm-project/clang/test/CodeGen/AArch64/fmv-detection.c b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-detection.c
index 44702a04e532..e585140a1eb0 100644
--- a/external/llvm-project/clang/test/CodeGen/AArch64/fmv-detection.c
+++ b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-detection.c
@@ -97,7 +97,7 @@ __attribute__((target_version("wfxt"))) int fmv(void) { return 0; }
 
 __attribute__((target_version("cssc+fp"))) int fmv(void);
 
-__attribute__((target_version("default"))) int fmv(void);
+__attribute__((target_version("default"))) int fmv(void) { return 0; }
 
 int caller() {
   return fmv();
@@ -121,380 +121,6 @@ int caller() {
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@fmv.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 2304
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2304
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @fmv._McsscMfp
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 2048
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2048
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @fmv._Mcssc
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 576460752303423488
-// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 576460752303423488
-// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
-// CHECK:       resolver_return3:
-// CHECK-NEXT:    ret ptr @fmv._Mmops
-// CHECK:       resolver_else4:
-// CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 144119586256651008
-// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 144119586256651008
-// CHECK-NEXT:    [[TMP15:%.*]] = and i1 true, [[TMP14]]
-// CHECK-NEXT:    br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
-// CHECK:       resolver_return5:
-// CHECK-NEXT:    ret ptr @fmv._Msme2
-// CHECK:       resolver_else6:
-// CHECK-NEXT:    [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72061992218723072
-// CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 72061992218723072
-// CHECK-NEXT:    [[TMP19:%.*]] = and i1 true, [[TMP18]]
-// CHECK-NEXT:    br i1 [[TMP19]], label [[RESOLVER_RETURN7:%.*]], label [[RESOLVER_ELSE8:%.*]]
-// CHECK:       resolver_return7:
-// CHECK-NEXT:    ret ptr @fmv._Msme-i16i64
-// CHECK:       resolver_else8:
-// CHECK-NEXT:    [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 36033195199759104
-// CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 36033195199759104
-// CHECK-NEXT:    [[TMP23:%.*]] = and i1 true, [[TMP22]]
-// CHECK-NEXT:    br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]]
-// CHECK:       resolver_return9:
-// CHECK-NEXT:    ret ptr @fmv._Msme-f64f64
-// CHECK:       resolver_else10:
-// CHECK-NEXT:    [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 18014398509481984
-// CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 18014398509481984
-// CHECK-NEXT:    [[TMP27:%.*]] = and i1 true, [[TMP26]]
-// CHECK-NEXT:    br i1 [[TMP27]], label [[RESOLVER_RETURN11:%.*]], label [[RESOLVER_ELSE12:%.*]]
-// CHECK:       resolver_return11:
-// CHECK-NEXT:    ret ptr @fmv._Mwfxt
-// CHECK:       resolver_else12:
-// CHECK-NEXT:    [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 1125899906842624
-// CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 1125899906842624
-// CHECK-NEXT:    [[TMP31:%.*]] = and i1 true, [[TMP30]]
-// CHECK-NEXT:    br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]]
-// CHECK:       resolver_return13:
-// CHECK-NEXT:    ret ptr @fmv._Mbti
-// CHECK:       resolver_else14:
-// CHECK-NEXT:    [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP33:%.*]] = and i64 [[TMP32]], 562949953421312
-// CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 562949953421312
-// CHECK-NEXT:    [[TMP35:%.*]] = and i1 true, [[TMP34]]
-// CHECK-NEXT:    br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]]
-// CHECK:       resolver_return15:
-// CHECK-NEXT:    ret ptr @fmv._Mssbs
-// CHECK:       resolver_else16:
-// CHECK-NEXT:    [[TMP36:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 70368744177664
-// CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 70368744177664
-// CHECK-NEXT:    [[TMP39:%.*]] = and i1 true, [[TMP38]]
-// CHECK-NEXT:    br i1 [[TMP39]], label [[RESOLVER_RETURN17:%.*]], label [[RESOLVER_ELSE18:%.*]]
-// CHECK:       resolver_return17:
-// CHECK-NEXT:    ret ptr @fmv._Msb
-// CHECK:       resolver_else18:
-// CHECK-NEXT:    [[TMP40:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP41:%.*]] = and i64 [[TMP40]], 17592186044416
-// CHECK-NEXT:    [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 17592186044416
-// CHECK-NEXT:    [[TMP43:%.*]] = and i1 true, [[TMP42]]
-// CHECK-NEXT:    br i1 [[TMP43]], label [[RESOLVER_RETURN19:%.*]], label [[RESOLVER_ELSE20:%.*]]
-// CHECK:       resolver_return19:
-// CHECK-NEXT:    ret ptr @fmv._Mmemtag
-// CHECK:       resolver_else20:
-// CHECK-NEXT:    [[TMP44:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP45:%.*]] = and i64 [[TMP44]], 4398180795136
-// CHECK-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 4398180795136
-// CHECK-NEXT:    [[TMP47:%.*]] = and i1 true, [[TMP46]]
-// CHECK-NEXT:    br i1 [[TMP47]], label [[RESOLVER_RETURN21:%.*]], label [[RESOLVER_ELSE22:%.*]]
-// CHECK:       resolver_return21:
-// CHECK-NEXT:    ret ptr @fmv._Msme
-// CHECK:       resolver_else22:
-// CHECK-NEXT:    [[TMP48:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP49:%.*]] = and i64 [[TMP48]], 2268816540448
-// CHECK-NEXT:    [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 2268816540448
-// CHECK-NEXT:    [[TMP51:%.*]] = and i1 true, [[TMP50]]
-// CHECK-NEXT:    br i1 [[TMP51]], label [[RESOLVER_RETURN23:%.*]], label [[RESOLVER_ELSE24:%.*]]
-// CHECK:       resolver_return23:
-// CHECK-NEXT:    ret ptr @fmv._Msve2-sm4
-// CHECK:       resolver_else24:
-// CHECK-NEXT:    [[TMP52:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP53:%.*]] = and i64 [[TMP52]], 1169304924928
-// CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 1169304924928
-// CHECK-NEXT:    [[TMP55:%.*]] = and i1 true, [[TMP54]]
-// CHECK-NEXT:    br i1 [[TMP55]], label [[RESOLVER_RETURN25:%.*]], label [[RESOLVER_ELSE26:%.*]]
-// CHECK:       resolver_return25:
-// CHECK-NEXT:    ret ptr @fmv._Msve2-sha3
-// CHECK:       resolver_else26:
-// CHECK-NEXT:    [[TMP56:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP57:%.*]] = and i64 [[TMP56]], 619549098240
-// CHECK-NEXT:    [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 619549098240
-// CHECK-NEXT:    [[TMP59:%.*]] = and i1 true, [[TMP58]]
-// CHECK-NEXT:    br i1 [[TMP59]], label [[RESOLVER_RETURN27:%.*]], label [[RESOLVER_ELSE28:%.*]]
-// CHECK:       resolver_return27:
-// CHECK-NEXT:    ret ptr @fmv._Msve2-bitperm
-// CHECK:       resolver_else28:
-// CHECK-NEXT:    [[TMP60:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP61:%.*]] = and i64 [[TMP60]], 344671224576
-// CHECK-NEXT:    [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 344671224576
-// CHECK-NEXT:    [[TMP63:%.*]] = and i1 true, [[TMP62]]
-// CHECK-NEXT:    br i1 [[TMP63]], label [[RESOLVER_RETURN29:%.*]], label [[RESOLVER_ELSE30:%.*]]
-// CHECK:       resolver_return29:
-// CHECK-NEXT:    ret ptr @fmv._Msve2-aes
-// CHECK:       resolver_else30:
-// CHECK-NEXT:    [[TMP64:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP65:%.*]] = and i64 [[TMP64]], 69793284352
-// CHECK-NEXT:    [[TMP66:%.*]] = icmp eq i64 [[TMP65]], 69793284352
-// CHECK-NEXT:    [[TMP67:%.*]] = and i1 true, [[TMP66]]
-// CHECK-NEXT:    br i1 [[TMP67]], label [[RESOLVER_RETURN31:%.*]], label [[RESOLVER_ELSE32:%.*]]
-// CHECK:       resolver_return31:
-// CHECK-NEXT:    ret ptr @fmv._Msve2
-// CHECK:       resolver_else32:
-// CHECK-NEXT:    [[TMP68:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP69:%.*]] = and i64 [[TMP68]], 35433545984
-// CHECK-NEXT:    [[TMP70:%.*]] = icmp eq i64 [[TMP69]], 35433545984
-// CHECK-NEXT:    [[TMP71:%.*]] = and i1 true, [[TMP70]]
-// CHECK-NEXT:    br i1 [[TMP71]], label [[RESOLVER_RETURN33:%.*]], label [[RESOLVER_ELSE34:%.*]]
-// CHECK:       resolver_return33:
-// CHECK-NEXT:    ret ptr @fmv._Mf64mm
-// CHECK:       resolver_else34:
-// CHECK-NEXT:    [[TMP72:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP73:%.*]] = and i64 [[TMP72]], 18253676800
-// CHECK-NEXT:    [[TMP74:%.*]] = icmp eq i64 [[TMP73]], 18253676800
-// CHECK-NEXT:    [[TMP75:%.*]] = and i1 true, [[TMP74]]
-// CHECK-NEXT:    br i1 [[TMP75]], label [[RESOLVER_RETURN35:%.*]], label [[RESOLVER_ELSE36:%.*]]
-// CHECK:       resolver_return35:
-// CHECK-NEXT:    ret ptr @fmv._Mf32mm
-// CHECK:       resolver_else36:
-// CHECK-NEXT:    [[TMP76:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP77:%.*]] = and i64 [[TMP76]], 1073807616
-// CHECK-NEXT:    [[TMP78:%.*]] = icmp eq i64 [[TMP77]], 1073807616
-// CHECK-NEXT:    [[TMP79:%.*]] = and i1 true, [[TMP78]]
-// CHECK-NEXT:    br i1 [[TMP79]], label [[RESOLVER_RETURN37:%.*]], label [[RESOLVER_ELSE38:%.*]]
-// CHECK:       resolver_return37:
-// CHECK-NEXT:    ret ptr @fmv._Msve
-// CHECK:       resolver_else38:
-// CHECK-NEXT:    [[TMP80:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP81:%.*]] = and i64 [[TMP80]], 134218496
-// CHECK-NEXT:    [[TMP82:%.*]] = icmp eq i64 [[TMP81]], 134218496
-// CHECK-NEXT:    [[TMP83:%.*]] = and i1 true, [[TMP82]]
-// CHECK-NEXT:    br i1 [[TMP83]], label [[RESOLVER_RETURN39:%.*]], label [[RESOLVER_ELSE40:%.*]]
-// CHECK:       resolver_return39:
-// CHECK-NEXT:    ret ptr @fmv._Mbf16
-// CHECK:       resolver_else40:
-// CHECK-NEXT:    [[TMP84:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP85:%.*]] = and i64 [[TMP84]], 67109632
-// CHECK-NEXT:    [[TMP86:%.*]] = icmp eq i64 [[TMP85]], 67109632
-// CHECK-NEXT:    [[TMP87:%.*]] = and i1 true, [[TMP86]]
-// CHECK-NEXT:    br i1 [[TMP87]], label [[RESOLVER_RETURN41:%.*]], label [[RESOLVER_ELSE42:%.*]]
-// CHECK:       resolver_return41:
-// CHECK-NEXT:    ret ptr @fmv._Mi8mm
-// CHECK:       resolver_else42:
-// CHECK-NEXT:    [[TMP88:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP89:%.*]] = and i64 [[TMP88]], 16777472
-// CHECK-NEXT:    [[TMP90:%.*]] = icmp eq i64 [[TMP89]], 16777472
-// CHECK-NEXT:    [[TMP91:%.*]] = and i1 true, [[TMP90]]
-// CHECK-NEXT:    br i1 [[TMP91]], label [[RESOLVER_RETURN43:%.*]], label [[RESOLVER_ELSE44:%.*]]
-// CHECK:       resolver_return43:
-// CHECK-NEXT:    ret ptr @fmv._Mfrintts
-// CHECK:       resolver_else44:
-// CHECK-NEXT:    [[TMP92:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP93:%.*]] = and i64 [[TMP92]], 288230376164294656
-// CHECK-NEXT:    [[TMP94:%.*]] = icmp eq i64 [[TMP93]], 288230376164294656
-// CHECK-NEXT:    [[TMP95:%.*]] = and i1 true, [[TMP94]]
-// CHECK-NEXT:    br i1 [[TMP95]], label [[RESOLVER_RETURN45:%.*]], label [[RESOLVER_ELSE46:%.*]]
-// CHECK:       resolver_return45:
-// CHECK-NEXT:    ret ptr @fmv._Mrcpc3
-// CHECK:       resolver_else46:
-// CHECK-NEXT:    [[TMP96:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP97:%.*]] = and i64 [[TMP96]], 12582912
-// CHECK-NEXT:    [[TMP98:%.*]] = icmp eq i64 [[TMP97]], 12582912
-// CHECK-NEXT:    [[TMP99:%.*]] = and i1 true, [[TMP98]]
-// CHECK-NEXT:    br i1 [[TMP99]], label [[RESOLVER_RETURN47:%.*]], label [[RESOLVER_ELSE48:%.*]]
-// CHECK:       resolver_return47:
-// CHECK-NEXT:    ret ptr @fmv._Mrcpc2
-// CHECK:       resolver_else48:
-// CHECK-NEXT:    [[TMP100:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP101:%.*]] = and i64 [[TMP100]], 4194304
-// CHECK-NEXT:    [[TMP102:%.*]] = icmp eq i64 [[TMP101]], 4194304
-// CHECK-NEXT:    [[TMP103:%.*]] = and i1 true, [[TMP102]]
-// CHECK-NEXT:    br i1 [[TMP103]], label [[RESOLVER_RETURN49:%.*]], label [[RESOLVER_ELSE50:%.*]]
-// CHECK:       resolver_return49:
-// CHECK-NEXT:    ret ptr @fmv._Mrcpc
-// CHECK:       resolver_else50:
-// CHECK-NEXT:    [[TMP104:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP105:%.*]] = and i64 [[TMP104]], 2097920
-// CHECK-NEXT:    [[TMP106:%.*]] = icmp eq i64 [[TMP105]], 2097920
-// CHECK-NEXT:    [[TMP107:%.*]] = and i1 true, [[TMP106]]
-// CHECK-NEXT:    br i1 [[TMP107]], label [[RESOLVER_RETURN51:%.*]], label [[RESOLVER_ELSE52:%.*]]
-// CHECK:       resolver_return51:
-// CHECK-NEXT:    ret ptr @fmv._Mfcma
-// CHECK:       resolver_else52:
-// CHECK-NEXT:    [[TMP108:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP109:%.*]] = and i64 [[TMP108]], 1048832
-// CHECK-NEXT:    [[TMP110:%.*]] = icmp eq i64 [[TMP109]], 1048832
-// CHECK-NEXT:    [[TMP111:%.*]] = and i1 true, [[TMP110]]
-// CHECK-NEXT:    br i1 [[TMP111]], label [[RESOLVER_RETURN53:%.*]], label [[RESOLVER_ELSE54:%.*]]
-// CHECK:       resolver_return53:
-// CHECK-NEXT:    ret ptr @fmv._Mjscvt
-// CHECK:       resolver_else54:
-// CHECK-NEXT:    [[TMP112:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP113:%.*]] = and i64 [[TMP112]], 786432
-// CHECK-NEXT:    [[TMP114:%.*]] = icmp eq i64 [[TMP113]], 786432
-// CHECK-NEXT:    [[TMP115:%.*]] = and i1 true, [[TMP114]]
-// CHECK-NEXT:    br i1 [[TMP115]], label [[RESOLVER_RETURN55:%.*]], label [[RESOLVER_ELSE56:%.*]]
-// CHECK:       resolver_return55:
-// CHECK-NEXT:    ret ptr @fmv._Mdpb2
-// CHECK:       resolver_else56:
-// CHECK-NEXT:    [[TMP116:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP117:%.*]] = and i64 [[TMP116]], 262144
-// CHECK-NEXT:    [[TMP118:%.*]] = icmp eq i64 [[TMP117]], 262144
-// CHECK-NEXT:    [[TMP119:%.*]] = and i1 true, [[TMP118]]
-// CHECK-NEXT:    br i1 [[TMP119]], label [[RESOLVER_RETURN57:%.*]], label [[RESOLVER_ELSE58:%.*]]
-// CHECK:       resolver_return57:
-// CHECK-NEXT:    ret ptr @fmv._Mdpb
-// CHECK:       resolver_else58:
-// CHECK-NEXT:    [[TMP120:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP121:%.*]] = and i64 [[TMP120]], 131072
-// CHECK-NEXT:    [[TMP122:%.*]] = icmp eq i64 [[TMP121]], 131072
-// CHECK-NEXT:    [[TMP123:%.*]] = and i1 true, [[TMP122]]
-// CHECK-NEXT:    br i1 [[TMP123]], label [[RESOLVER_RETURN59:%.*]], label [[RESOLVER_ELSE60:%.*]]
-// CHECK:       resolver_return59:
-// CHECK-NEXT:    ret ptr @fmv._Mdit
-// CHECK:       resolver_else60:
-// CHECK-NEXT:    [[TMP124:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP125:%.*]] = and i64 [[TMP124]], 66312
-// CHECK-NEXT:    [[TMP126:%.*]] = icmp eq i64 [[TMP125]], 66312
-// CHECK-NEXT:    [[TMP127:%.*]] = and i1 true, [[TMP126]]
-// CHECK-NEXT:    br i1 [[TMP127]], label [[RESOLVER_RETURN61:%.*]], label [[RESOLVER_ELSE62:%.*]]
-// CHECK:       resolver_return61:
-// CHECK-NEXT:    ret ptr @fmv._Mfp16fml
-// CHECK:       resolver_else62:
-// CHECK-NEXT:    [[TMP128:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP129:%.*]] = and i64 [[TMP128]], 65792
-// CHECK-NEXT:    [[TMP130:%.*]] = icmp eq i64 [[TMP129]], 65792
-// CHECK-NEXT:    [[TMP131:%.*]] = and i1 true, [[TMP130]]
-// CHECK-NEXT:    br i1 [[TMP131]], label [[RESOLVER_RETURN63:%.*]], label [[RESOLVER_ELSE64:%.*]]
-// CHECK:       resolver_return63:
-// CHECK-NEXT:    ret ptr @fmv._Mfp16
-// CHECK:       resolver_else64:
-// CHECK-NEXT:    [[TMP132:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP133:%.*]] = and i64 [[TMP132]], 33536
-// CHECK-NEXT:    [[TMP134:%.*]] = icmp eq i64 [[TMP133]], 33536
-// CHECK-NEXT:    [[TMP135:%.*]] = and i1 true, [[TMP134]]
-// CHECK-NEXT:    br i1 [[TMP135]], label [[RESOLVER_RETURN65:%.*]], label [[RESOLVER_ELSE66:%.*]]
-// CHECK:       resolver_return65:
-// CHECK-NEXT:    ret ptr @fmv._Maes
-// CHECK:       resolver_else66:
-// CHECK-NEXT:    [[TMP136:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP137:%.*]] = and i64 [[TMP136]], 13056
-// CHECK-NEXT:    [[TMP138:%.*]] = icmp eq i64 [[TMP137]], 13056
-// CHECK-NEXT:    [[TMP139:%.*]] = and i1 true, [[TMP138]]
-// CHECK-NEXT:    br i1 [[TMP139]], label [[RESOLVER_RETURN67:%.*]], label [[RESOLVER_ELSE68:%.*]]
-// CHECK:       resolver_return67:
-// CHECK-NEXT:    ret ptr @fmv._Msha3
-// CHECK:       resolver_else68:
-// CHECK-NEXT:    [[TMP140:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP141:%.*]] = and i64 [[TMP140]], 4864
-// CHECK-NEXT:    [[TMP142:%.*]] = icmp eq i64 [[TMP141]], 4864
-// CHECK-NEXT:    [[TMP143:%.*]] = and i1 true, [[TMP142]]
-// CHECK-NEXT:    br i1 [[TMP143]], label [[RESOLVER_RETURN69:%.*]], label [[RESOLVER_ELSE70:%.*]]
-// CHECK:       resolver_return69:
-// CHECK-NEXT:    ret ptr @fmv._Msha2
-// CHECK:       resolver_else70:
-// CHECK-NEXT:    [[TMP144:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP145:%.*]] = and i64 [[TMP144]], 1024
-// CHECK-NEXT:    [[TMP146:%.*]] = icmp eq i64 [[TMP145]], 1024
-// CHECK-NEXT:    [[TMP147:%.*]] = and i1 true, [[TMP146]]
-// CHECK-NEXT:    br i1 [[TMP147]], label [[RESOLVER_RETURN71:%.*]], label [[RESOLVER_ELSE72:%.*]]
-// CHECK:       resolver_return71:
-// CHECK-NEXT:    ret ptr @fmv._Mcrc
-// CHECK:       resolver_else72:
-// CHECK-NEXT:    [[TMP148:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP149:%.*]] = and i64 [[TMP148]], 832
-// CHECK-NEXT:    [[TMP150:%.*]] = icmp eq i64 [[TMP149]], 832
-// CHECK-NEXT:    [[TMP151:%.*]] = and i1 true, [[TMP150]]
-// CHECK-NEXT:    br i1 [[TMP151]], label [[RESOLVER_RETURN73:%.*]], label [[RESOLVER_ELSE74:%.*]]
-// CHECK:       resolver_return73:
-// CHECK-NEXT:    ret ptr @fmv._Mrdm
-// CHECK:       resolver_else74:
-// CHECK-NEXT:    [[TMP152:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP153:%.*]] = and i64 [[TMP152]], 800
-// CHECK-NEXT:    [[TMP154:%.*]] = icmp eq i64 [[TMP153]], 800
-// CHECK-NEXT:    [[TMP155:%.*]] = and i1 true, [[TMP154]]
-// CHECK-NEXT:    br i1 [[TMP155]], label [[RESOLVER_RETURN75:%.*]], label [[RESOLVER_ELSE76:%.*]]
-// CHECK:       resolver_return75:
-// CHECK-NEXT:    ret ptr @fmv._Msm4
-// CHECK:       resolver_else76:
-// CHECK-NEXT:    [[TMP156:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP157:%.*]] = and i64 [[TMP156]], 784
-// CHECK-NEXT:    [[TMP158:%.*]] = icmp eq i64 [[TMP157]], 784
-// CHECK-NEXT:    [[TMP159:%.*]] = and i1 true, [[TMP158]]
-// CHECK-NEXT:    br i1 [[TMP159]], label [[RESOLVER_RETURN77:%.*]], label [[RESOLVER_ELSE78:%.*]]
-// CHECK:       resolver_return77:
-// CHECK-NEXT:    ret ptr @fmv._Mdotprod
-// CHECK:       resolver_else78:
-// CHECK-NEXT:    [[TMP160:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP161:%.*]] = and i64 [[TMP160]], 768
-// CHECK-NEXT:    [[TMP162:%.*]] = icmp eq i64 [[TMP161]], 768
-// CHECK-NEXT:    [[TMP163:%.*]] = and i1 true, [[TMP162]]
-// CHECK-NEXT:    br i1 [[TMP163]], label [[RESOLVER_RETURN79:%.*]], label [[RESOLVER_ELSE80:%.*]]
-// CHECK:       resolver_return79:
-// CHECK-NEXT:    ret ptr @fmv._Msimd
-// CHECK:       resolver_else80:
-// CHECK-NEXT:    [[TMP164:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP165:%.*]] = and i64 [[TMP164]], 256
-// CHECK-NEXT:    [[TMP166:%.*]] = icmp eq i64 [[TMP165]], 256
-// CHECK-NEXT:    [[TMP167:%.*]] = and i1 true, [[TMP166]]
-// CHECK-NEXT:    br i1 [[TMP167]], label [[RESOLVER_RETURN81:%.*]], label [[RESOLVER_ELSE82:%.*]]
-// CHECK:       resolver_return81:
-// CHECK-NEXT:    ret ptr @fmv._Mfp
-// CHECK:       resolver_else82:
-// CHECK-NEXT:    [[TMP168:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP169:%.*]] = and i64 [[TMP168]], 128
-// CHECK-NEXT:    [[TMP170:%.*]] = icmp eq i64 [[TMP169]], 128
-// CHECK-NEXT:    [[TMP171:%.*]] = and i1 true, [[TMP170]]
-// CHECK-NEXT:    br i1 [[TMP171]], label [[RESOLVER_RETURN83:%.*]], label [[RESOLVER_ELSE84:%.*]]
-// CHECK:       resolver_return83:
-// CHECK-NEXT:    ret ptr @fmv._Mlse
-// CHECK:       resolver_else84:
-// CHECK-NEXT:    [[TMP172:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP173:%.*]] = and i64 [[TMP172]], 6
-// CHECK-NEXT:    [[TMP174:%.*]] = icmp eq i64 [[TMP173]], 6
-// CHECK-NEXT:    [[TMP175:%.*]] = and i1 true, [[TMP174]]
-// CHECK-NEXT:    br i1 [[TMP175]], label [[RESOLVER_RETURN85:%.*]], label [[RESOLVER_ELSE86:%.*]]
-// CHECK:       resolver_return85:
-// CHECK-NEXT:    ret ptr @fmv._Mflagm2
-// CHECK:       resolver_else86:
-// CHECK-NEXT:    [[TMP176:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP177:%.*]] = and i64 [[TMP176]], 2
-// CHECK-NEXT:    [[TMP178:%.*]] = icmp eq i64 [[TMP177]], 2
-// CHECK-NEXT:    [[TMP179:%.*]] = and i1 true, [[TMP178]]
-// CHECK-NEXT:    br i1 [[TMP179]], label [[RESOLVER_RETURN87:%.*]], label [[RESOLVER_ELSE88:%.*]]
-// CHECK:       resolver_return87:
-// CHECK-NEXT:    ret ptr @fmv._Mflagm
-// CHECK:       resolver_else88:
-// CHECK-NEXT:    [[TMP180:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP181:%.*]] = and i64 [[TMP180]], 1
-// CHECK-NEXT:    [[TMP182:%.*]] = icmp eq i64 [[TMP181]], 1
-// CHECK-NEXT:    [[TMP183:%.*]] = and i1 true, [[TMP182]]
-// CHECK-NEXT:    br i1 [[TMP183]], label [[RESOLVER_RETURN89:%.*]], label [[RESOLVER_ELSE90:%.*]]
-// CHECK:       resolver_return89:
-// CHECK-NEXT:    ret ptr @fmv._Mrng
-// CHECK:       resolver_else90:
-// CHECK-NEXT:    ret ptr @fmv.default
-//
-//
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._Mbti
 // CHECK-SAME: () #[[ATTR2:[0-9]+]] {
@@ -544,14 +170,14 @@ int caller() {
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Mf32mm
 // CHECK-SAME: () #[[ATTR9:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Mf64mm
 // CHECK-SAME: () #[[ATTR10:[0-9]+]] {
 // CHECK-NEXT:  entry:
@@ -747,42 +373,42 @@ int caller() {
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Msve
 // CHECK-SAME: () #[[ATTR38:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Msve2
 // CHECK-SAME: () #[[ATTR39:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Msve2-aes
 // CHECK-SAME: () #[[ATTR40:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Msve2-bitperm
 // CHECK-SAME: () #[[ATTR41:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Msve2-sha3
 // CHECK-SAME: () #[[ATTR42:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: noinline nounwind optnone vscale_range(1,16)
 // CHECK-LABEL: define {{[^@]+}}@fmv._Msve2-sm4
 // CHECK-SAME: () #[[ATTR43:[0-9]+]] {
 // CHECK-NEXT:  entry:
@@ -797,18 +423,392 @@ int caller() {
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@caller
+// CHECK-LABEL: define {{[^@]+}}@fmv.default
 // CHECK-SAME: () #[[ATTR45:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 @fmv()
-// CHECK-NEXT:    ret i32 [[CALL]]
+// CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv.default
+// CHECK-LABEL: define {{[^@]+}}@caller
 // CHECK-SAME: () #[[ATTR46:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @fmv()
+// CHECK-NEXT:    ret i32 [[CALL]]
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@fmv.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 2304
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2304
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @fmv._McsscMfp
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 2048
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2048
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @fmv._Mcssc
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 576460752303423488
+// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 576460752303423488
+// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
+// CHECK:       resolver_return3:
+// CHECK-NEXT:    ret ptr @fmv._Mmops
+// CHECK:       resolver_else4:
+// CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 144119586256651008
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 144119586256651008
+// CHECK-NEXT:    [[TMP15:%.*]] = and i1 true, [[TMP14]]
+// CHECK-NEXT:    br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
+// CHECK:       resolver_return5:
+// CHECK-NEXT:    ret ptr @fmv._Msme2
+// CHECK:       resolver_else6:
+// CHECK-NEXT:    [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72061992218723072
+// CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 72061992218723072
+// CHECK-NEXT:    [[TMP19:%.*]] = and i1 true, [[TMP18]]
+// CHECK-NEXT:    br i1 [[TMP19]], label [[RESOLVER_RETURN7:%.*]], label [[RESOLVER_ELSE8:%.*]]
+// CHECK:       resolver_return7:
+// CHECK-NEXT:    ret ptr @fmv._Msme-i16i64
+// CHECK:       resolver_else8:
+// CHECK-NEXT:    [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 36033195199759104
+// CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 36033195199759104
+// CHECK-NEXT:    [[TMP23:%.*]] = and i1 true, [[TMP22]]
+// CHECK-NEXT:    br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]]
+// CHECK:       resolver_return9:
+// CHECK-NEXT:    ret ptr @fmv._Msme-f64f64
+// CHECK:       resolver_else10:
+// CHECK-NEXT:    [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 18014398509481984
+// CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 18014398509481984
+// CHECK-NEXT:    [[TMP27:%.*]] = and i1 true, [[TMP26]]
+// CHECK-NEXT:    br i1 [[TMP27]], label [[RESOLVER_RETURN11:%.*]], label [[RESOLVER_ELSE12:%.*]]
+// CHECK:       resolver_return11:
+// CHECK-NEXT:    ret ptr @fmv._Mwfxt
+// CHECK:       resolver_else12:
+// CHECK-NEXT:    [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 1125899906842624
+// CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 1125899906842624
+// CHECK-NEXT:    [[TMP31:%.*]] = and i1 true, [[TMP30]]
+// CHECK-NEXT:    br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]]
+// CHECK:       resolver_return13:
+// CHECK-NEXT:    ret ptr @fmv._Mbti
+// CHECK:       resolver_else14:
+// CHECK-NEXT:    [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP33:%.*]] = and i64 [[TMP32]], 562949953421312
+// CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 562949953421312
+// CHECK-NEXT:    [[TMP35:%.*]] = and i1 true, [[TMP34]]
+// CHECK-NEXT:    br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]]
+// CHECK:       resolver_return15:
+// CHECK-NEXT:    ret ptr @fmv._Mssbs
+// CHECK:       resolver_else16:
+// CHECK-NEXT:    [[TMP36:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 70368744177664
+// CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 70368744177664
+// CHECK-NEXT:    [[TMP39:%.*]] = and i1 true, [[TMP38]]
+// CHECK-NEXT:    br i1 [[TMP39]], label [[RESOLVER_RETURN17:%.*]], label [[RESOLVER_ELSE18:%.*]]
+// CHECK:       resolver_return17:
+// CHECK-NEXT:    ret ptr @fmv._Msb
+// CHECK:       resolver_else18:
+// CHECK-NEXT:    [[TMP40:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP41:%.*]] = and i64 [[TMP40]], 17592186044416
+// CHECK-NEXT:    [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 17592186044416
+// CHECK-NEXT:    [[TMP43:%.*]] = and i1 true, [[TMP42]]
+// CHECK-NEXT:    br i1 [[TMP43]], label [[RESOLVER_RETURN19:%.*]], label [[RESOLVER_ELSE20:%.*]]
+// CHECK:       resolver_return19:
+// CHECK-NEXT:    ret ptr @fmv._Mmemtag
+// CHECK:       resolver_else20:
+// CHECK-NEXT:    [[TMP44:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP45:%.*]] = and i64 [[TMP44]], 4398180795136
+// CHECK-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 4398180795136
+// CHECK-NEXT:    [[TMP47:%.*]] = and i1 true, [[TMP46]]
+// CHECK-NEXT:    br i1 [[TMP47]], label [[RESOLVER_RETURN21:%.*]], label [[RESOLVER_ELSE22:%.*]]
+// CHECK:       resolver_return21:
+// CHECK-NEXT:    ret ptr @fmv._Msme
+// CHECK:       resolver_else22:
+// CHECK-NEXT:    [[TMP48:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP49:%.*]] = and i64 [[TMP48]], 2268816540448
+// CHECK-NEXT:    [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 2268816540448
+// CHECK-NEXT:    [[TMP51:%.*]] = and i1 true, [[TMP50]]
+// CHECK-NEXT:    br i1 [[TMP51]], label [[RESOLVER_RETURN23:%.*]], label [[RESOLVER_ELSE24:%.*]]
+// CHECK:       resolver_return23:
+// CHECK-NEXT:    ret ptr @fmv._Msve2-sm4
+// CHECK:       resolver_else24:
+// CHECK-NEXT:    [[TMP52:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP53:%.*]] = and i64 [[TMP52]], 1169304924928
+// CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 1169304924928
+// CHECK-NEXT:    [[TMP55:%.*]] = and i1 true, [[TMP54]]
+// CHECK-NEXT:    br i1 [[TMP55]], label [[RESOLVER_RETURN25:%.*]], label [[RESOLVER_ELSE26:%.*]]
+// CHECK:       resolver_return25:
+// CHECK-NEXT:    ret ptr @fmv._Msve2-sha3
+// CHECK:       resolver_else26:
+// CHECK-NEXT:    [[TMP56:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP57:%.*]] = and i64 [[TMP56]], 619549098240
+// CHECK-NEXT:    [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 619549098240
+// CHECK-NEXT:    [[TMP59:%.*]] = and i1 true, [[TMP58]]
+// CHECK-NEXT:    br i1 [[TMP59]], label [[RESOLVER_RETURN27:%.*]], label [[RESOLVER_ELSE28:%.*]]
+// CHECK:       resolver_return27:
+// CHECK-NEXT:    ret ptr @fmv._Msve2-bitperm
+// CHECK:       resolver_else28:
+// CHECK-NEXT:    [[TMP60:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP61:%.*]] = and i64 [[TMP60]], 344671224576
+// CHECK-NEXT:    [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 344671224576
+// CHECK-NEXT:    [[TMP63:%.*]] = and i1 true, [[TMP62]]
+// CHECK-NEXT:    br i1 [[TMP63]], label [[RESOLVER_RETURN29:%.*]], label [[RESOLVER_ELSE30:%.*]]
+// CHECK:       resolver_return29:
+// CHECK-NEXT:    ret ptr @fmv._Msve2-aes
+// CHECK:       resolver_else30:
+// CHECK-NEXT:    [[TMP64:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP65:%.*]] = and i64 [[TMP64]], 69793284352
+// CHECK-NEXT:    [[TMP66:%.*]] = icmp eq i64 [[TMP65]], 69793284352
+// CHECK-NEXT:    [[TMP67:%.*]] = and i1 true, [[TMP66]]
+// CHECK-NEXT:    br i1 [[TMP67]], label [[RESOLVER_RETURN31:%.*]], label [[RESOLVER_ELSE32:%.*]]
+// CHECK:       resolver_return31:
+// CHECK-NEXT:    ret ptr @fmv._Msve2
+// CHECK:       resolver_else32:
+// CHECK-NEXT:    [[TMP68:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP69:%.*]] = and i64 [[TMP68]], 35433545984
+// CHECK-NEXT:    [[TMP70:%.*]] = icmp eq i64 [[TMP69]], 35433545984
+// CHECK-NEXT:    [[TMP71:%.*]] = and i1 true, [[TMP70]]
+// CHECK-NEXT:    br i1 [[TMP71]], label [[RESOLVER_RETURN33:%.*]], label [[RESOLVER_ELSE34:%.*]]
+// CHECK:       resolver_return33:
+// CHECK-NEXT:    ret ptr @fmv._Mf64mm
+// CHECK:       resolver_else34:
+// CHECK-NEXT:    [[TMP72:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP73:%.*]] = and i64 [[TMP72]], 18253676800
+// CHECK-NEXT:    [[TMP74:%.*]] = icmp eq i64 [[TMP73]], 18253676800
+// CHECK-NEXT:    [[TMP75:%.*]] = and i1 true, [[TMP74]]
+// CHECK-NEXT:    br i1 [[TMP75]], label [[RESOLVER_RETURN35:%.*]], label [[RESOLVER_ELSE36:%.*]]
+// CHECK:       resolver_return35:
+// CHECK-NEXT:    ret ptr @fmv._Mf32mm
+// CHECK:       resolver_else36:
+// CHECK-NEXT:    [[TMP76:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP77:%.*]] = and i64 [[TMP76]], 1073807616
+// CHECK-NEXT:    [[TMP78:%.*]] = icmp eq i64 [[TMP77]], 1073807616
+// CHECK-NEXT:    [[TMP79:%.*]] = and i1 true, [[TMP78]]
+// CHECK-NEXT:    br i1 [[TMP79]], label [[RESOLVER_RETURN37:%.*]], label [[RESOLVER_ELSE38:%.*]]
+// CHECK:       resolver_return37:
+// CHECK-NEXT:    ret ptr @fmv._Msve
+// CHECK:       resolver_else38:
+// CHECK-NEXT:    [[TMP80:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP81:%.*]] = and i64 [[TMP80]], 134218496
+// CHECK-NEXT:    [[TMP82:%.*]] = icmp eq i64 [[TMP81]], 134218496
+// CHECK-NEXT:    [[TMP83:%.*]] = and i1 true, [[TMP82]]
+// CHECK-NEXT:    br i1 [[TMP83]], label [[RESOLVER_RETURN39:%.*]], label [[RESOLVER_ELSE40:%.*]]
+// CHECK:       resolver_return39:
+// CHECK-NEXT:    ret ptr @fmv._Mbf16
+// CHECK:       resolver_else40:
+// CHECK-NEXT:    [[TMP84:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP85:%.*]] = and i64 [[TMP84]], 67109632
+// CHECK-NEXT:    [[TMP86:%.*]] = icmp eq i64 [[TMP85]], 67109632
+// CHECK-NEXT:    [[TMP87:%.*]] = and i1 true, [[TMP86]]
+// CHECK-NEXT:    br i1 [[TMP87]], label [[RESOLVER_RETURN41:%.*]], label [[RESOLVER_ELSE42:%.*]]
+// CHECK:       resolver_return41:
+// CHECK-NEXT:    ret ptr @fmv._Mi8mm
+// CHECK:       resolver_else42:
+// CHECK-NEXT:    [[TMP88:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP89:%.*]] = and i64 [[TMP88]], 16777472
+// CHECK-NEXT:    [[TMP90:%.*]] = icmp eq i64 [[TMP89]], 16777472
+// CHECK-NEXT:    [[TMP91:%.*]] = and i1 true, [[TMP90]]
+// CHECK-NEXT:    br i1 [[TMP91]], label [[RESOLVER_RETURN43:%.*]], label [[RESOLVER_ELSE44:%.*]]
+// CHECK:       resolver_return43:
+// CHECK-NEXT:    ret ptr @fmv._Mfrintts
+// CHECK:       resolver_else44:
+// CHECK-NEXT:    [[TMP92:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP93:%.*]] = and i64 [[TMP92]], 288230376164294656
+// CHECK-NEXT:    [[TMP94:%.*]] = icmp eq i64 [[TMP93]], 288230376164294656
+// CHECK-NEXT:    [[TMP95:%.*]] = and i1 true, [[TMP94]]
+// CHECK-NEXT:    br i1 [[TMP95]], label [[RESOLVER_RETURN45:%.*]], label [[RESOLVER_ELSE46:%.*]]
+// CHECK:       resolver_return45:
+// CHECK-NEXT:    ret ptr @fmv._Mrcpc3
+// CHECK:       resolver_else46:
+// CHECK-NEXT:    [[TMP96:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP97:%.*]] = and i64 [[TMP96]], 12582912
+// CHECK-NEXT:    [[TMP98:%.*]] = icmp eq i64 [[TMP97]], 12582912
+// CHECK-NEXT:    [[TMP99:%.*]] = and i1 true, [[TMP98]]
+// CHECK-NEXT:    br i1 [[TMP99]], label [[RESOLVER_RETURN47:%.*]], label [[RESOLVER_ELSE48:%.*]]
+// CHECK:       resolver_return47:
+// CHECK-NEXT:    ret ptr @fmv._Mrcpc2
+// CHECK:       resolver_else48:
+// CHECK-NEXT:    [[TMP100:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP101:%.*]] = and i64 [[TMP100]], 4194304
+// CHECK-NEXT:    [[TMP102:%.*]] = icmp eq i64 [[TMP101]], 4194304
+// CHECK-NEXT:    [[TMP103:%.*]] = and i1 true, [[TMP102]]
+// CHECK-NEXT:    br i1 [[TMP103]], label [[RESOLVER_RETURN49:%.*]], label [[RESOLVER_ELSE50:%.*]]
+// CHECK:       resolver_return49:
+// CHECK-NEXT:    ret ptr @fmv._Mrcpc
+// CHECK:       resolver_else50:
+// CHECK-NEXT:    [[TMP104:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP105:%.*]] = and i64 [[TMP104]], 2097920
+// CHECK-NEXT:    [[TMP106:%.*]] = icmp eq i64 [[TMP105]], 2097920
+// CHECK-NEXT:    [[TMP107:%.*]] = and i1 true, [[TMP106]]
+// CHECK-NEXT:    br i1 [[TMP107]], label [[RESOLVER_RETURN51:%.*]], label [[RESOLVER_ELSE52:%.*]]
+// CHECK:       resolver_return51:
+// CHECK-NEXT:    ret ptr @fmv._Mfcma
+// CHECK:       resolver_else52:
+// CHECK-NEXT:    [[TMP108:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP109:%.*]] = and i64 [[TMP108]], 1048832
+// CHECK-NEXT:    [[TMP110:%.*]] = icmp eq i64 [[TMP109]], 1048832
+// CHECK-NEXT:    [[TMP111:%.*]] = and i1 true, [[TMP110]]
+// CHECK-NEXT:    br i1 [[TMP111]], label [[RESOLVER_RETURN53:%.*]], label [[RESOLVER_ELSE54:%.*]]
+// CHECK:       resolver_return53:
+// CHECK-NEXT:    ret ptr @fmv._Mjscvt
+// CHECK:       resolver_else54:
+// CHECK-NEXT:    [[TMP112:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP113:%.*]] = and i64 [[TMP112]], 786432
+// CHECK-NEXT:    [[TMP114:%.*]] = icmp eq i64 [[TMP113]], 786432
+// CHECK-NEXT:    [[TMP115:%.*]] = and i1 true, [[TMP114]]
+// CHECK-NEXT:    br i1 [[TMP115]], label [[RESOLVER_RETURN55:%.*]], label [[RESOLVER_ELSE56:%.*]]
+// CHECK:       resolver_return55:
+// CHECK-NEXT:    ret ptr @fmv._Mdpb2
+// CHECK:       resolver_else56:
+// CHECK-NEXT:    [[TMP116:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP117:%.*]] = and i64 [[TMP116]], 262144
+// CHECK-NEXT:    [[TMP118:%.*]] = icmp eq i64 [[TMP117]], 262144
+// CHECK-NEXT:    [[TMP119:%.*]] = and i1 true, [[TMP118]]
+// CHECK-NEXT:    br i1 [[TMP119]], label [[RESOLVER_RETURN57:%.*]], label [[RESOLVER_ELSE58:%.*]]
+// CHECK:       resolver_return57:
+// CHECK-NEXT:    ret ptr @fmv._Mdpb
+// CHECK:       resolver_else58:
+// CHECK-NEXT:    [[TMP120:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP121:%.*]] = and i64 [[TMP120]], 131072
+// CHECK-NEXT:    [[TMP122:%.*]] = icmp eq i64 [[TMP121]], 131072
+// CHECK-NEXT:    [[TMP123:%.*]] = and i1 true, [[TMP122]]
+// CHECK-NEXT:    br i1 [[TMP123]], label [[RESOLVER_RETURN59:%.*]], label [[RESOLVER_ELSE60:%.*]]
+// CHECK:       resolver_return59:
+// CHECK-NEXT:    ret ptr @fmv._Mdit
+// CHECK:       resolver_else60:
+// CHECK-NEXT:    [[TMP124:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP125:%.*]] = and i64 [[TMP124]], 66312
+// CHECK-NEXT:    [[TMP126:%.*]] = icmp eq i64 [[TMP125]], 66312
+// CHECK-NEXT:    [[TMP127:%.*]] = and i1 true, [[TMP126]]
+// CHECK-NEXT:    br i1 [[TMP127]], label [[RESOLVER_RETURN61:%.*]], label [[RESOLVER_ELSE62:%.*]]
+// CHECK:       resolver_return61:
+// CHECK-NEXT:    ret ptr @fmv._Mfp16fml
+// CHECK:       resolver_else62:
+// CHECK-NEXT:    [[TMP128:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP129:%.*]] = and i64 [[TMP128]], 65792
+// CHECK-NEXT:    [[TMP130:%.*]] = icmp eq i64 [[TMP129]], 65792
+// CHECK-NEXT:    [[TMP131:%.*]] = and i1 true, [[TMP130]]
+// CHECK-NEXT:    br i1 [[TMP131]], label [[RESOLVER_RETURN63:%.*]], label [[RESOLVER_ELSE64:%.*]]
+// CHECK:       resolver_return63:
+// CHECK-NEXT:    ret ptr @fmv._Mfp16
+// CHECK:       resolver_else64:
+// CHECK-NEXT:    [[TMP132:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP133:%.*]] = and i64 [[TMP132]], 33536
+// CHECK-NEXT:    [[TMP134:%.*]] = icmp eq i64 [[TMP133]], 33536
+// CHECK-NEXT:    [[TMP135:%.*]] = and i1 true, [[TMP134]]
+// CHECK-NEXT:    br i1 [[TMP135]], label [[RESOLVER_RETURN65:%.*]], label [[RESOLVER_ELSE66:%.*]]
+// CHECK:       resolver_return65:
+// CHECK-NEXT:    ret ptr @fmv._Maes
+// CHECK:       resolver_else66:
+// CHECK-NEXT:    [[TMP136:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP137:%.*]] = and i64 [[TMP136]], 13056
+// CHECK-NEXT:    [[TMP138:%.*]] = icmp eq i64 [[TMP137]], 13056
+// CHECK-NEXT:    [[TMP139:%.*]] = and i1 true, [[TMP138]]
+// CHECK-NEXT:    br i1 [[TMP139]], label [[RESOLVER_RETURN67:%.*]], label [[RESOLVER_ELSE68:%.*]]
+// CHECK:       resolver_return67:
+// CHECK-NEXT:    ret ptr @fmv._Msha3
+// CHECK:       resolver_else68:
+// CHECK-NEXT:    [[TMP140:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP141:%.*]] = and i64 [[TMP140]], 4864
+// CHECK-NEXT:    [[TMP142:%.*]] = icmp eq i64 [[TMP141]], 4864
+// CHECK-NEXT:    [[TMP143:%.*]] = and i1 true, [[TMP142]]
+// CHECK-NEXT:    br i1 [[TMP143]], label [[RESOLVER_RETURN69:%.*]], label [[RESOLVER_ELSE70:%.*]]
+// CHECK:       resolver_return69:
+// CHECK-NEXT:    ret ptr @fmv._Msha2
+// CHECK:       resolver_else70:
+// CHECK-NEXT:    [[TMP144:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP145:%.*]] = and i64 [[TMP144]], 1024
+// CHECK-NEXT:    [[TMP146:%.*]] = icmp eq i64 [[TMP145]], 1024
+// CHECK-NEXT:    [[TMP147:%.*]] = and i1 true, [[TMP146]]
+// CHECK-NEXT:    br i1 [[TMP147]], label [[RESOLVER_RETURN71:%.*]], label [[RESOLVER_ELSE72:%.*]]
+// CHECK:       resolver_return71:
+// CHECK-NEXT:    ret ptr @fmv._Mcrc
+// CHECK:       resolver_else72:
+// CHECK-NEXT:    [[TMP148:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP149:%.*]] = and i64 [[TMP148]], 832
+// CHECK-NEXT:    [[TMP150:%.*]] = icmp eq i64 [[TMP149]], 832
+// CHECK-NEXT:    [[TMP151:%.*]] = and i1 true, [[TMP150]]
+// CHECK-NEXT:    br i1 [[TMP151]], label [[RESOLVER_RETURN73:%.*]], label [[RESOLVER_ELSE74:%.*]]
+// CHECK:       resolver_return73:
+// CHECK-NEXT:    ret ptr @fmv._Mrdm
+// CHECK:       resolver_else74:
+// CHECK-NEXT:    [[TMP152:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP153:%.*]] = and i64 [[TMP152]], 800
+// CHECK-NEXT:    [[TMP154:%.*]] = icmp eq i64 [[TMP153]], 800
+// CHECK-NEXT:    [[TMP155:%.*]] = and i1 true, [[TMP154]]
+// CHECK-NEXT:    br i1 [[TMP155]], label [[RESOLVER_RETURN75:%.*]], label [[RESOLVER_ELSE76:%.*]]
+// CHECK:       resolver_return75:
+// CHECK-NEXT:    ret ptr @fmv._Msm4
+// CHECK:       resolver_else76:
+// CHECK-NEXT:    [[TMP156:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP157:%.*]] = and i64 [[TMP156]], 784
+// CHECK-NEXT:    [[TMP158:%.*]] = icmp eq i64 [[TMP157]], 784
+// CHECK-NEXT:    [[TMP159:%.*]] = and i1 true, [[TMP158]]
+// CHECK-NEXT:    br i1 [[TMP159]], label [[RESOLVER_RETURN77:%.*]], label [[RESOLVER_ELSE78:%.*]]
+// CHECK:       resolver_return77:
+// CHECK-NEXT:    ret ptr @fmv._Mdotprod
+// CHECK:       resolver_else78:
+// CHECK-NEXT:    [[TMP160:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP161:%.*]] = and i64 [[TMP160]], 768
+// CHECK-NEXT:    [[TMP162:%.*]] = icmp eq i64 [[TMP161]], 768
+// CHECK-NEXT:    [[TMP163:%.*]] = and i1 true, [[TMP162]]
+// CHECK-NEXT:    br i1 [[TMP163]], label [[RESOLVER_RETURN79:%.*]], label [[RESOLVER_ELSE80:%.*]]
+// CHECK:       resolver_return79:
+// CHECK-NEXT:    ret ptr @fmv._Msimd
+// CHECK:       resolver_else80:
+// CHECK-NEXT:    [[TMP164:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP165:%.*]] = and i64 [[TMP164]], 256
+// CHECK-NEXT:    [[TMP166:%.*]] = icmp eq i64 [[TMP165]], 256
+// CHECK-NEXT:    [[TMP167:%.*]] = and i1 true, [[TMP166]]
+// CHECK-NEXT:    br i1 [[TMP167]], label [[RESOLVER_RETURN81:%.*]], label [[RESOLVER_ELSE82:%.*]]
+// CHECK:       resolver_return81:
+// CHECK-NEXT:    ret ptr @fmv._Mfp
+// CHECK:       resolver_else82:
+// CHECK-NEXT:    [[TMP168:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP169:%.*]] = and i64 [[TMP168]], 128
+// CHECK-NEXT:    [[TMP170:%.*]] = icmp eq i64 [[TMP169]], 128
+// CHECK-NEXT:    [[TMP171:%.*]] = and i1 true, [[TMP170]]
+// CHECK-NEXT:    br i1 [[TMP171]], label [[RESOLVER_RETURN83:%.*]], label [[RESOLVER_ELSE84:%.*]]
+// CHECK:       resolver_return83:
+// CHECK-NEXT:    ret ptr @fmv._Mlse
+// CHECK:       resolver_else84:
+// CHECK-NEXT:    [[TMP172:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP173:%.*]] = and i64 [[TMP172]], 6
+// CHECK-NEXT:    [[TMP174:%.*]] = icmp eq i64 [[TMP173]], 6
+// CHECK-NEXT:    [[TMP175:%.*]] = and i1 true, [[TMP174]]
+// CHECK-NEXT:    br i1 [[TMP175]], label [[RESOLVER_RETURN85:%.*]], label [[RESOLVER_ELSE86:%.*]]
+// CHECK:       resolver_return85:
+// CHECK-NEXT:    ret ptr @fmv._Mflagm2
+// CHECK:       resolver_else86:
+// CHECK-NEXT:    [[TMP176:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP177:%.*]] = and i64 [[TMP176]], 2
+// CHECK-NEXT:    [[TMP178:%.*]] = icmp eq i64 [[TMP177]], 2
+// CHECK-NEXT:    [[TMP179:%.*]] = and i1 true, [[TMP178]]
+// CHECK-NEXT:    br i1 [[TMP179]], label [[RESOLVER_RETURN87:%.*]], label [[RESOLVER_ELSE88:%.*]]
+// CHECK:       resolver_return87:
+// CHECK-NEXT:    ret ptr @fmv._Mflagm
+// CHECK:       resolver_else88:
+// CHECK-NEXT:    [[TMP180:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP181:%.*]] = and i64 [[TMP180]], 1
+// CHECK-NEXT:    [[TMP182:%.*]] = icmp eq i64 [[TMP181]], 1
+// CHECK-NEXT:    [[TMP183:%.*]] = and i1 true, [[TMP182]]
+// CHECK-NEXT:    br i1 [[TMP183]], label [[RESOLVER_RETURN89:%.*]], label [[RESOLVER_ELSE90:%.*]]
+// CHECK:       resolver_return89:
+// CHECK-NEXT:    ret ptr @fmv._Mrng
+// CHECK:       resolver_else90:
+// CHECK-NEXT:    ret ptr @fmv.default
 //
 //.
 // CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
diff --git a/external/llvm-project/clang/test/CodeGen/AArch64/fmv-duplicate-mangled-name.c b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-duplicate-mangled-name.c
new file mode 100644
index 000000000000..e7e611e09542
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-duplicate-mangled-name.c
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -verify -emit-llvm-only %s -DCHECK_IMPLICIT_DEFAULT
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -verify -emit-llvm-only %s -DCHECK_EXPLICIT_DEFAULT
+
+#if defined(CHECK_IMPLICIT_DEFAULT)
+
+int implicit_default_ok(void) { return 0; }
+__attribute__((target_clones("aes", "lse"))) int implicit_default_ok(void) { return 1; }
+
+int implicit_default_bad(void) { return 0; }
+// expected-error@+2 {{definition with same mangled name 'implicit_default_bad.default' as another definition}}
+// expected-note@-2 {{previous definition is here}}
+__attribute__((target_clones("aes", "lse", "default"))) int implicit_default_bad(void) { return 1; }
+
+#elif defined(CHECK_EXPLICIT_DEFAULT)
+
+__attribute__((target_version("default"))) int explicit_default_ok(void) { return 0; }
+__attribute__((target_clones("aes", "lse"))) int explicit_default_ok(void) { return 1; }
+
+__attribute__((target_version("default"))) int explicit_default_bad(void) { return 0; }
+// expected-error@+2 {{definition with same mangled name 'explicit_default_bad.default' as another definition}}
+// expected-note@-2 {{previous definition is here}}
+__attribute__((target_clones("aes", "lse", "default"))) int explicit_default_bad(void) { return 1; }
+
+#endif
diff --git a/external/llvm-project/clang/test/CodeGen/AArch64/fmv-features.c b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-features.c
index fdc64e2cd395..f8b5a9b15456 100644
--- a/external/llvm-project/clang/test/CodeGen/AArch64/fmv-features.c
+++ b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-features.c
@@ -140,7 +140,7 @@ __attribute__((target_version("crc+bti+bti+bti+aes+aes+bf16"))) int fmv(void) {
 __attribute__((target_version("non_existent_extension"))) int fmv(void);
 
 // CHECK: define dso_local i32 @fmv.default() #[[default:[0-9]+]] {
-__attribute__((target_version("default"))) int fmv(void);
+__attribute__((target_version("default"))) int fmv(void) { return 0; }
 
 int caller() {
   return fmv();
diff --git a/external/llvm-project/clang/test/CodeGen/AArch64/fmv-resolver-emission.c b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-resolver-emission.c
index 84667910c6e5..591625d4d0da 100644
--- a/external/llvm-project/clang/test/CodeGen/AArch64/fmv-resolver-emission.c
+++ b/external/llvm-project/clang/test/CodeGen/AArch64/fmv-resolver-emission.c
@@ -68,6 +68,14 @@ inline __attribute__((target_version("default"))) void linkonce_func(void) {}
 void call_linkonce(void) { linkonce_func(); }
 
 
+// Test that an ifunc is generated when the clones attribute has a default version.
+__attribute__((target_clones("default", "aes"))) void clones_with_default(void) {}
+
+
+// Test that an ifunc is NOT generated when the clones attribute does not have a default version.
+__attribute__((target_clones("aes"))) void clones_without_default(void) {}
+
+
 //.
 // CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
 // CHECK: @used_before_default_def = weak_odr ifunc void (), ptr @used_before_default_def.resolver
@@ -76,6 +84,7 @@ void call_linkonce(void) { linkonce_func(); }
 // CHECK: @indirect_use = weak_odr ifunc void (), ptr @indirect_use.resolver
 // CHECK: @internal_func = internal ifunc void (), ptr @internal_func.resolver
 // CHECK: @linkonce_func = weak_odr ifunc void (), ptr @linkonce_func.resolver
+// CHECK: @clones_with_default = weak_odr ifunc void (), ptr @clones_with_default.resolver
 //.
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@used_before_default_def._Maes
@@ -228,6 +237,27 @@ void call_linkonce(void) { linkonce_func(); }
 // CHECK-NEXT:    ret void
 //
 //
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@clones_with_default.default
+// CHECK-SAME: () #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@clones_with_default._Maes
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@clones_without_default._Maes
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret void
+//
+//
 // CHECK-LABEL: define {{[^@]+}}@used_before_default_def.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -339,6 +369,20 @@ void call_linkonce(void) { linkonce_func(); }
 // CHECK:       resolver_else:
 // CHECK-NEXT:    ret ptr @linkonce_func.default
 //
+//
+// CHECK-LABEL: define {{[^@]+}}@clones_with_default.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 33536
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 33536
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @clones_with_default._Maes
+// CHECK:       resolver_else:
+// CHECK-NEXT:    ret ptr @clones_with_default.default
+//
 //.
 // CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 // CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
diff --git a/external/llvm-project/clang/test/CodeGen/AArch64/mixed-neon-types.c b/external/llvm-project/clang/test/CodeGen/AArch64/mixed-neon-types.c
new file mode 100644
index 000000000000..34fbe499f405
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/AArch64/mixed-neon-types.c
@@ -0,0 +1,547 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1  -triple aarch64-linux-gnu -target-feature +neon -x c %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-C
+// RUN: %clang_cc1  -triple aarch64-linux-gnu -target-feature +neon -x c++ %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-CPP
+// REQUIRES: aarch64-registered-target
+
+typedef __Uint8x16_t X;
+
+// CHECK-C-LABEL: define dso_local <16 x i8> @test(
+// CHECK-C-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <16 x i8> @_Z4test12__Uint8x16_t(
+// CHECK-CPP-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <16 x i8> [[TMP0]]
+//
+X test(X x) {
+  return x;
+}
+
+#include <arm_neon.h>
+
+// CHECK-C-LABEL: define dso_local <16 x i8> @testboth(
+// CHECK-C-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[__P0_ADDR_I:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    [[__P1_ADDR_I:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    [[__RET_I:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    store <16 x i8> [[TMP0]], ptr [[__P0_ADDR_I]], align 16
+// CHECK-C-NEXT:    store <16 x i8> [[TMP1]], ptr [[__P1_ADDR_I]], align 16
+// CHECK-C-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr [[__P0_ADDR_I]], align 16
+// CHECK-C-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr [[__P1_ADDR_I]], align 16
+// CHECK-C-NEXT:    [[ADD_I:%.*]] = add <16 x i8> [[TMP2]], [[TMP3]]
+// CHECK-C-NEXT:    store <16 x i8> [[ADD_I]], ptr [[__RET_I]], align 16
+// CHECK-C-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr [[__RET_I]], align 16
+// CHECK-C-NEXT:    ret <16 x i8> [[TMP4]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <16 x i8> @_Z8testboth12__Uint8x16_t(
+// CHECK-CPP-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[__P0_ADDR_I:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    [[__P1_ADDR_I:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    [[__RET_I:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[TMP0]], ptr [[__P0_ADDR_I]], align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[TMP1]], ptr [[__P1_ADDR_I]], align 16
+// CHECK-CPP-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr [[__P0_ADDR_I]], align 16
+// CHECK-CPP-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr [[__P1_ADDR_I]], align 16
+// CHECK-CPP-NEXT:    [[ADD_I:%.*]] = add <16 x i8> [[TMP2]], [[TMP3]]
+// CHECK-CPP-NEXT:    store <16 x i8> [[ADD_I]], ptr [[__RET_I]], align 16
+// CHECK-CPP-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr [[__RET_I]], align 16
+// CHECK-CPP-NEXT:    ret <16 x i8> [[TMP4]]
+//
+uint8x16_t testboth(X x) {
+   return vaddq_u8(x, x);
+}
+
+// CHECK-C-LABEL: define dso_local <8 x i8> @test__Int8x8_t(
+// CHECK-C-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-C-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x i8> @_Z14test__Int8x8_t10__Int8x8_t(
+// CHECK-CPP-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-CPP-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <8 x i8> [[TMP0]]
+//
+int8x8_t test__Int8x8_t(__Int8x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x i16> @test__Int16x4_t(
+// CHECK-C-SAME: <4 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i16>, align 8
+// CHECK-C-NEXT:    store <4 x i16> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <4 x i16> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x i16> @_Z15test__Int16x4_t11__Int16x4_t(
+// CHECK-CPP-SAME: <4 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i16>, align 8
+// CHECK-CPP-NEXT:    store <4 x i16> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <4 x i16> [[TMP0]]
+//
+int16x4_t test__Int16x4_t(__Int16x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <2 x i32> @test__Int32x2_t(
+// CHECK-C-SAME: <2 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i32>, align 8
+// CHECK-C-NEXT:    store <2 x i32> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <2 x i32> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <2 x i32> @_Z15test__Int32x2_t11__Int32x2_t(
+// CHECK-CPP-SAME: <2 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i32>, align 8
+// CHECK-CPP-NEXT:    store <2 x i32> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <2 x i32> [[TMP0]]
+//
+int32x2_t test__Int32x2_t(__Int32x2_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x i8> @test__Uint8x8_t(
+// CHECK-C-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-C-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x i8> @_Z15test__Uint8x8_t11__Uint8x8_t(
+// CHECK-CPP-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-CPP-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <8 x i8> [[TMP0]]
+//
+uint8x8_t test__Uint8x8_t(__Uint8x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x i16> @test__Uint16x4_t(
+// CHECK-C-SAME: <4 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i16>, align 8
+// CHECK-C-NEXT:    store <4 x i16> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <4 x i16> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x i16> @_Z16test__Uint16x4_t12__Uint16x4_t(
+// CHECK-CPP-SAME: <4 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i16>, align 8
+// CHECK-CPP-NEXT:    store <4 x i16> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <4 x i16> [[TMP0]]
+//
+uint16x4_t test__Uint16x4_t(__Uint16x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <2 x i32> @test__Uint32x2_t(
+// CHECK-C-SAME: <2 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i32>, align 8
+// CHECK-C-NEXT:    store <2 x i32> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <2 x i32> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <2 x i32> @_Z16test__Uint32x2_t12__Uint32x2_t(
+// CHECK-CPP-SAME: <2 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i32>, align 8
+// CHECK-CPP-NEXT:    store <2 x i32> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <2 x i32> [[TMP0]]
+//
+uint32x2_t test__Uint32x2_t(__Uint32x2_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x half> @test__Float16x4_t(
+// CHECK-C-SAME: <4 x half> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x half>, align 8
+// CHECK-C-NEXT:    store <4 x half> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x half>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <4 x half> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x half> @_Z17test__Float16x4_t13__Float16x4_t(
+// CHECK-CPP-SAME: <4 x half> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x half>, align 8
+// CHECK-CPP-NEXT:    store <4 x half> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x half>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <4 x half> [[TMP0]]
+//
+float16x4_t test__Float16x4_t(__Float16x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <2 x float> @test__Float32x2_t(
+// CHECK-C-SAME: <2 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <2 x float>, align 8
+// CHECK-C-NEXT:    store <2 x float> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <2 x float> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <2 x float> @_Z17test__Float32x2_t13__Float32x2_t(
+// CHECK-CPP-SAME: <2 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <2 x float>, align 8
+// CHECK-CPP-NEXT:    store <2 x float> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <2 x float> [[TMP0]]
+//
+float32x2_t test__Float32x2_t(__Float32x2_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x i8> @test__Poly8x8_t(
+// CHECK-C-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-C-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x i8> @_Z15test__Poly8x8_t11__Poly8x8_t(
+// CHECK-CPP-SAME: <8 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-CPP-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <8 x i8> [[TMP0]]
+//
+poly8x8_t test__Poly8x8_t(__Poly8x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x i16> @test__Poly16x4_t(
+// CHECK-C-SAME: <4 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i16>, align 8
+// CHECK-C-NEXT:    store <4 x i16> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <4 x i16> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x i16> @_Z16test__Poly16x4_t12__Poly16x4_t(
+// CHECK-CPP-SAME: <4 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i16>, align 8
+// CHECK-CPP-NEXT:    store <4 x i16> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <4 x i16> [[TMP0]]
+//
+poly16x4_t test__Poly16x4_t(__Poly16x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x bfloat> @test__Bfloat16x4_t(
+// CHECK-C-SAME: <4 x bfloat> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x bfloat>, align 8
+// CHECK-C-NEXT:    store <4 x bfloat> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x bfloat>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <4 x bfloat> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x bfloat> @_Z18test__Bfloat16x4_t14__Bfloat16x4_t(
+// CHECK-CPP-SAME: <4 x bfloat> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x bfloat>, align 8
+// CHECK-CPP-NEXT:    store <4 x bfloat> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x bfloat>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <4 x bfloat> [[TMP0]]
+//
+bfloat16x4_t test__Bfloat16x4_t(__Bfloat16x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <16 x i8> @test__Int8x16_t(
+// CHECK-C-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <16 x i8> @_Z15test__Int8x16_t11__Int8x16_t(
+// CHECK-CPP-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test__Int8x16_t(__Int8x16_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x i16> @test__Int16x8_t(
+// CHECK-C-SAME: <8 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i16>, align 16
+// CHECK-C-NEXT:    store <8 x i16> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <8 x i16> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x i16> @_Z15test__Int16x8_t11__Int16x8_t(
+// CHECK-CPP-SAME: <8 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i16>, align 16
+// CHECK-CPP-NEXT:    store <8 x i16> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <8 x i16> [[TMP0]]
+//
+int16x8_t test__Int16x8_t(__Int16x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x i32> @test__Int32x4_t(
+// CHECK-C-SAME: <4 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i32>, align 16
+// CHECK-C-NEXT:    store <4 x i32> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <4 x i32> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x i32> @_Z15test__Int32x4_t11__Int32x4_t(
+// CHECK-CPP-SAME: <4 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i32>, align 16
+// CHECK-CPP-NEXT:    store <4 x i32> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <4 x i32> [[TMP0]]
+//
+int32x4_t test__Int32x4_t(__Int32x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <2 x i64> @test__Int64x2_t(
+// CHECK-C-SAME: <2 x i64> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i64>, align 16
+// CHECK-C-NEXT:    store <2 x i64> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <2 x i64> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <2 x i64> @_Z15test__Int64x2_t11__Int64x2_t(
+// CHECK-CPP-SAME: <2 x i64> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i64>, align 16
+// CHECK-CPP-NEXT:    store <2 x i64> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <2 x i64> [[TMP0]]
+//
+int64x2_t test__Int64x2_t(__Int64x2_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <16 x i8> @test__Uint8x16_t(
+// CHECK-C-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <16 x i8> @_Z16test__Uint8x16_t12__Uint8x16_t(
+// CHECK-CPP-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test__Uint8x16_t(__Uint8x16_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x i16> @test__Uint16x8_t(
+// CHECK-C-SAME: <8 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i16>, align 16
+// CHECK-C-NEXT:    store <8 x i16> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <8 x i16> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x i16> @_Z16test__Uint16x8_t12__Uint16x8_t(
+// CHECK-CPP-SAME: <8 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i16>, align 16
+// CHECK-CPP-NEXT:    store <8 x i16> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <8 x i16> [[TMP0]]
+//
+uint16x8_t test__Uint16x8_t(__Uint16x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x i32> @test__Uint32x4_t(
+// CHECK-C-SAME: <4 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i32>, align 16
+// CHECK-C-NEXT:    store <4 x i32> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <4 x i32> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x i32> @_Z16test__Uint32x4_t12__Uint32x4_t(
+// CHECK-CPP-SAME: <4 x i32> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x i32>, align 16
+// CHECK-CPP-NEXT:    store <4 x i32> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <4 x i32> [[TMP0]]
+//
+uint32x4_t test__Uint32x4_t(__Uint32x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <2 x i64> @test__Uint64x2_t(
+// CHECK-C-SAME: <2 x i64> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i64>, align 16
+// CHECK-C-NEXT:    store <2 x i64> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <2 x i64> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <2 x i64> @_Z16test__Uint64x2_t12__Uint64x2_t(
+// CHECK-CPP-SAME: <2 x i64> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i64>, align 16
+// CHECK-CPP-NEXT:    store <2 x i64> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <2 x i64> [[TMP0]]
+//
+uint64x2_t test__Uint64x2_t(__Uint64x2_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x half> @test__Float16x8_t(
+// CHECK-C-SAME: <8 x half> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x half>, align 16
+// CHECK-C-NEXT:    store <8 x half> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x half>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <8 x half> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x half> @_Z17test__Float16x8_t13__Float16x8_t(
+// CHECK-CPP-SAME: <8 x half> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x half>, align 16
+// CHECK-CPP-NEXT:    store <8 x half> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x half>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <8 x half> [[TMP0]]
+//
+float16x8_t test__Float16x8_t(__Float16x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <4 x float> @test__Float32x4_t(
+// CHECK-C-SAME: <4 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <4 x float>, align 16
+// CHECK-C-NEXT:    store <4 x float> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <4 x float> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <4 x float> @_Z17test__Float32x4_t13__Float32x4_t(
+// CHECK-CPP-SAME: <4 x float> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <4 x float>, align 16
+// CHECK-CPP-NEXT:    store <4 x float> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <4 x float> [[TMP0]]
+//
+float32x4_t test__Float32x4_t(__Float32x4_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <2 x double> @test__Float64x2_t(
+// CHECK-C-SAME: <2 x double> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <2 x double>, align 16
+// CHECK-C-NEXT:    store <2 x double> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <2 x double> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <2 x double> @_Z17test__Float64x2_t13__Float64x2_t(
+// CHECK-CPP-SAME: <2 x double> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <2 x double>, align 16
+// CHECK-CPP-NEXT:    store <2 x double> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <2 x double> [[TMP0]]
+//
+float64x2_t test__Float64x2_t(__Float64x2_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <16 x i8> @test__Poly8x16_t(
+// CHECK-C-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <16 x i8> @_Z16test__Poly8x16_t12__Poly8x16_t(
+// CHECK-CPP-SAME: <16 x i8> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <16 x i8> [[TMP0]]
+//
+poly8x16_t test__Poly8x16_t(__Poly8x16_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x i16> @test__Poly16x8_t(
+// CHECK-C-SAME: <8 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i16>, align 16
+// CHECK-C-NEXT:    store <8 x i16> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <8 x i16> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x i16> @_Z16test__Poly16x8_t12__Poly16x8_t(
+// CHECK-CPP-SAME: <8 x i16> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i16>, align 16
+// CHECK-CPP-NEXT:    store <8 x i16> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <8 x i16> [[TMP0]]
+//
+poly16x8_t test__Poly16x8_t(__Poly16x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <2 x i64> @test__Poly64x2_t(
+// CHECK-C-SAME: <2 x i64> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i64>, align 16
+// CHECK-C-NEXT:    store <2 x i64> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <2 x i64> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <2 x i64> @_Z16test__Poly64x2_t12__Poly64x2_t(
+// CHECK-CPP-SAME: <2 x i64> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <2 x i64>, align 16
+// CHECK-CPP-NEXT:    store <2 x i64> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <2 x i64> [[TMP0]]
+//
+poly64x2_t test__Poly64x2_t(__Poly64x2_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x bfloat> @test__Bfloat16x8_t(
+// CHECK-C-SAME: <8 x bfloat> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x bfloat>, align 16
+// CHECK-C-NEXT:    store <8 x bfloat> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x bfloat>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <8 x bfloat> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local noundef <8 x bfloat> @_Z18test__Bfloat16x8_t14__Bfloat16x8_t(
+// CHECK-CPP-SAME: <8 x bfloat> noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x bfloat>, align 16
+// CHECK-CPP-NEXT:    store <8 x bfloat> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x bfloat>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <8 x bfloat> [[TMP0]]
+//
+bfloat16x8_t test__Bfloat16x8_t(__Bfloat16x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <8 x i8> @test__Mfloat8x8_t(
+// CHECK-C-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-C-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-C-NEXT:    ret <8 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local <8 x i8> @_Z17test__Mfloat8x8_t13__Mfloat8x8_t(
+// CHECK-CPP-SAME: <8 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <8 x i8>, align 8
+// CHECK-CPP-NEXT:    store <8 x i8> [[X]], ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X_ADDR]], align 8
+// CHECK-CPP-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test__Mfloat8x8_t(__Mfloat8x8_t x) { return x; }
+// CHECK-C-LABEL: define dso_local <16 x i8> @test__Mfloat8x16_t(
+// CHECK-C-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-C-NEXT:  [[ENTRY:.*:]]
+// CHECK-C-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-C-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-C-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CHECK-CPP-LABEL: define dso_local <16 x i8> @_Z18test__Mfloat8x16_t14__Mfloat8x16_t(
+// CHECK-CPP-SAME: <16 x i8> [[X:%.*]]) #[[ATTR0]] {
+// CHECK-CPP-NEXT:  [[ENTRY:.*:]]
+// CHECK-CPP-NEXT:    [[X_ADDR:%.*]] = alloca <16 x i8>, align 16
+// CHECK-CPP-NEXT:    store <16 x i8> [[X]], ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X_ADDR]], align 16
+// CHECK-CPP-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test__Mfloat8x16_t(__Mfloat8x16_t x) { return x; }
diff --git a/external/llvm-project/clang/test/CodeGen/AArch64/mixed-target-attributes.c b/external/llvm-project/clang/test/CodeGen/AArch64/mixed-target-attributes.c
index 1ccb0c6177c8..ef47c8a3bc73 100644
--- a/external/llvm-project/clang/test/CodeGen/AArch64/mixed-target-attributes.c
+++ b/external/llvm-project/clang/test/CodeGen/AArch64/mixed-target-attributes.c
@@ -4,19 +4,19 @@
 
 // The following is guarded because in NOFMV we get an error for redefining the default.
 #ifdef __HAVE_FUNCTION_MULTI_VERSIONING
-int explicit_default(void) { return 0; }
-__attribute__((target_version("jscvt"))) int explicit_default(void) { return 1; }
-__attribute__((target_clones("dotprod", "lse"))) int explicit_default(void) { return 2; }
-__attribute__((target_version("rdma"))) int explicit_default(void) { return 3; }
-
-int foo(void) { return explicit_default(); }
-#endif
-
+int implicit_default(void) { return 0; }
 __attribute__((target_version("jscvt"))) int implicit_default(void) { return 1; }
 __attribute__((target_clones("dotprod", "lse"))) int implicit_default(void) { return 2; }
 __attribute__((target_version("rdma"))) int implicit_default(void) { return 3; }
 
-int bar(void) { return implicit_default(); }
+int foo(void) { return implicit_default(); }
+#endif
+
+__attribute__((target_version("jscvt"))) int explicit_default(void) { return 1; }
+__attribute__((target_clones("dotprod", "lse", "default"))) int explicit_default(void) { return 2; }
+__attribute__((target_version("rdma"))) int explicit_default(void) { return 3; }
+
+int bar(void) { return explicit_default(); }
 
 // These shouldn't generate anything.
 int unused_version_declarations(void);
@@ -30,78 +30,40 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void
 
 //.
 // CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
-// CHECK: @explicit_default = weak_odr ifunc i32 (), ptr @explicit_default.resolver
 // CHECK: @implicit_default = weak_odr ifunc i32 (), ptr @implicit_default.resolver
+// CHECK: @explicit_default = weak_odr ifunc i32 (), ptr @explicit_default.resolver
 // CHECK: @default_def_with_version_decls = weak_odr ifunc i32 (), ptr @default_def_with_version_decls.resolver
 //.
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@explicit_default.default
+// CHECK-LABEL: define {{[^@]+}}@implicit_default.default
 // CHECK-SAME: () #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mjscvt
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mjscvt
 // CHECK-SAME: () #[[ATTR1:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mdotprod
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mdotprod
 // CHECK-SAME: () #[[ATTR2:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mlse
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mlse
 // CHECK-SAME: () #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@explicit_default.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048832
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048832
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @explicit_default._Mjscvt
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 832
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 832
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @explicit_default._Mrdm
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 784
-// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 784
-// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
-// CHECK:       resolver_return3:
-// CHECK-NEXT:    ret ptr @explicit_default._Mdotprod
-// CHECK:       resolver_else4:
-// CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 128
-// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 128
-// CHECK-NEXT:    [[TMP15:%.*]] = and i1 true, [[TMP14]]
-// CHECK-NEXT:    br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
-// CHECK:       resolver_return5:
-// CHECK-NEXT:    ret ptr @explicit_default._Mlse
-// CHECK:       resolver_else6:
-// CHECK-NEXT:    ret ptr @explicit_default.default
-//
-//
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mrdm
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mrdm
 // CHECK-SAME: () #[[ATTR4:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
@@ -111,31 +73,60 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void
 // CHECK-LABEL: define {{[^@]+}}@foo
 // CHECK-SAME: () #[[ATTR0]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 @explicit_default()
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @implicit_default()
 // CHECK-NEXT:    ret i32 [[CALL]]
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mjscvt
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mjscvt
 // CHECK-SAME: () #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mdotprod
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mdotprod
 // CHECK-SAME: () #[[ATTR2]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mlse
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mlse
 // CHECK-SAME: () #[[ATTR3]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@explicit_default.default
+// CHECK-SAME: () #[[ATTR5:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mrdm
+// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 3
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@bar
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @explicit_default()
+// CHECK-NEXT:    ret i32 [[CALL]]
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
 // CHECK-LABEL: define {{[^@]+}}@implicit_default.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -174,33 +165,42 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void
 // CHECK-NEXT:    ret ptr @implicit_default.default
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mrdm
-// CHECK-SAME: () #[[ATTR4]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 3
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@bar
-// CHECK-SAME: () #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CALL:%.*]] = call i32 @implicit_default()
-// CHECK-NEXT:    ret i32 [[CALL]]
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default
-// CHECK-SAME: () #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@implicit_default.default
-// CHECK-SAME: () #[[ATTR6:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
+// CHECK-LABEL: define {{[^@]+}}@explicit_default.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048832
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048832
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @explicit_default._Mjscvt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 832
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 832
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @explicit_default._Mrdm
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 784
+// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 784
+// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
+// CHECK:       resolver_return3:
+// CHECK-NEXT:    ret ptr @explicit_default._Mdotprod
+// CHECK:       resolver_else4:
+// CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 128
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 128
+// CHECK-NEXT:    [[TMP15:%.*]] = and i1 true, [[TMP14]]
+// CHECK-NEXT:    br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
+// CHECK:       resolver_return5:
+// CHECK-NEXT:    ret ptr @explicit_default._Mlse
+// CHECK:       resolver_else6:
+// CHECK-NEXT:    ret ptr @explicit_default.default
 //
 //
 // CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.resolver() comdat {
@@ -234,7 +234,7 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void
 //
 //
 // CHECK-NOFMV: Function Attrs: noinline nounwind optnone
-// CHECK-NOFMV-LABEL: define {{[^@]+}}@implicit_default
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@explicit_default
 // CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] {
 // CHECK-NOFMV-NEXT:  entry:
 // CHECK-NOFMV-NEXT:    ret i32 2
@@ -244,7 +244,7 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void
 // CHECK-NOFMV-LABEL: define {{[^@]+}}@bar
 // CHECK-NOFMV-SAME: () #[[ATTR0]] {
 // CHECK-NOFMV-NEXT:  entry:
-// CHECK-NOFMV-NEXT:    [[CALL:%.*]] = call i32 @implicit_default()
+// CHECK-NOFMV-NEXT:    [[CALL:%.*]] = call i32 @explicit_default()
 // CHECK-NOFMV-NEXT:    ret i32 [[CALL]]
 //
 //
diff --git a/external/llvm-project/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp b/external/llvm-project/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
index f7a44a599988..b1232921df36 100644
--- a/external/llvm-project/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
+++ b/external/llvm-project/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
@@ -29,12 +29,11 @@ struct Sp {
     int *x;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z2Tp2Sp(
-// CHECK-A64-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SP:%.*]], align 8
 // CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr
-// CHECK-A64-NEXT:    store ptr [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    store ptr [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SP]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
@@ -58,10 +57,10 @@ struct Spp {
     int *x, *y;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z3Tpp3Spp(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPP:%.*]], align 8
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[S]], align 8
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[S]], align 8
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SPP]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
@@ -135,12 +134,12 @@ struct Srp {
     int &x, *y;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z3Trp3Srp(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRP:%.*]], align 8
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[S]], align 8
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[S]], align 8
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRP]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
 // CHECK-A64-NEXT:    ret void
 //
@@ -150,7 +149,7 @@ struct Srp {
 // CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRP:%.*]], align 4
 // CHECK-A64_32-NEXT:    store i64 [[S_COERCE]], ptr [[S]], align 4
 // CHECK-A64_32-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRP]], ptr [[S]], i32 0, i32 0
-// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 4
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 4, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
 // CHECK-A64_32-NEXT:    store i32 1, ptr [[TMP0]], align 4
 // CHECK-A64_32-NEXT:    ret void
 //
@@ -160,10 +159,10 @@ struct __attribute__((__packed__)) Spp_packed {
     int *x, *y;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z10Tpp_packed10Spp_packed(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPP_PACKED:%.*]], align 1
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[S]], align 1
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[S]], align 1
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SPP_PACKED]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 1
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
@@ -185,11 +184,11 @@ struct __attribute__((__packed__)) Spp_superpacked {
     Spp_packed x;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z15Tpp_superpacked15Spp_superpacked(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPP_SUPERPACKED:%.*]], align 1
 // CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPP_SUPERPACKED]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[COERCE_DIVE]], align 1
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[COERCE_DIVE]], align 1
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SPP_SUPERPACKED]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[X1:%.*]] = getelementptr inbounds nuw [[STRUCT_SPP_PACKED:%.*]], ptr [[X]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X1]], align 1
@@ -215,12 +214,11 @@ union Upp {
     long long *y;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z11Tupp_packed3Upp(
-// CHECK-A64-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[UNION_UPP:%.*]], align 8
 // CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[UNION_UPP]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr
-// CHECK-A64-NEXT:    store ptr [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    store ptr [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S]], align 8
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
 // CHECK-A64-NEXT:    ret void
@@ -326,10 +324,10 @@ struct SSpSp {
     struct Sp a, b;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z5TSpSp5SSpSp(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SSPSP:%.*]], align 8
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[S]], align 8
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[S]], align 8
 // CHECK-A64-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_SSPSP]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SP:%.*]], ptr [[A]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8
@@ -353,11 +351,11 @@ struct SSpp {
     Spp a;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4TSpp4SSpp(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SSPP:%.*]], align 8
 // CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SSPP]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_SSPP]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SPP:%.*]], ptr [[A]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8
@@ -382,10 +380,10 @@ struct SSp : public Sp {
     int* b;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z3TSp3SSp(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SSP:%.*]], align 8
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[S]], align 8
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[S]], align 8
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SP:%.*]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
@@ -433,11 +431,11 @@ struct Spa {
     int* xs[1];
 };
 // CHECK-A64-LABEL: define dso_local void @_Z3Tpa3Spa(
-// CHECK-A64-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPA:%.*]], align 8
 // CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPA]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    store i64 [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    store ptr [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    [[XS:%.*]] = getelementptr inbounds nuw [[STRUCT_SPA]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1 x ptr], ptr [[XS]], i64 0, i64 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
@@ -463,11 +461,11 @@ struct Spa2 {
     int* xs[2];
 };
 // CHECK-A64-LABEL: define dso_local void @_Z4Tpa24Spa2(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPA2:%.*]], align 8
 // CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPA2]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    [[XS:%.*]] = getelementptr inbounds nuw [[STRUCT_SPA2]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x ptr], ptr [[XS]], i64 0, i64 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
@@ -523,10 +521,10 @@ struct __attribute__((aligned(16))) Spp_align16 {
     int *x, *y;
 };
 // CHECK-A64-LABEL: define dso_local void @_Z11Tpp_align1611Spp_align16(
-// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: [2 x ptr] [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPP_ALIGN16:%.*]], align 16
-// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[S]], align 16
+// CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[S]], align 16
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SPP_ALIGN16]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 16
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
@@ -620,3 +618,180 @@ struct SpSempty {
 // CHECK-A64_32-NEXT:    ret void
 //
 void TpSempty(SpSempty s) { *s.x = 1; }
+
+
+struct Spaddrspace {
+    __attribute__((address_space(100))) int *x;
+};
+// CHECK-A64-LABEL: define dso_local void @_Z11Tpaddrspace11Spaddrspace(
+// CHECK-A64-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPADDRSPACE:%.*]], align 8
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr addrspace(100)
+// CHECK-A64-NEXT:    store ptr addrspace(100) [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 8
+// CHECK-A64-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
+// CHECK-A64-NEXT:    ret void
+//
+// CHECK-A64_32-LABEL: define void @_Z11Tpaddrspace11Spaddrspace(
+// CHECK-A64_32-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPADDRSPACE:%.*]], align 4
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[COERCE_VAL_II:%.*]] = trunc i64 [[S_COERCE]] to i32
+// CHECK-A64_32-NEXT:    store i32 [[COERCE_VAL_II]], ptr [[COERCE_DIVE]], align 4
+// CHECK-A64_32-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 4
+// CHECK-A64_32-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
+// CHECK-A64_32-NEXT:    ret void
+//
+void Tpaddrspace(Spaddrspace s) { *s.x = 1; }
+// CHECK-A64-LABEL: define dso_local void @_Z11Cpaddrspacev(
+// CHECK-A64-SAME: ) #[[ATTR0]] {
+// CHECK-A64-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPADDRSPACE:%.*]], align 8
+// CHECK-A64-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SPADDRSPACE]], align 8
+// CHECK-A64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP]], ptr align 8 [[S]], i64 8, i1 false)
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i64
+// CHECK-A64-NEXT:    call void @_Z11Tpaddrspace11Spaddrspace(i64 [[COERCE_VAL_PI]])
+// CHECK-A64-NEXT:    ret void
+//
+// CHECK-A64_32-LABEL: define void @_Z11Cpaddrspacev(
+// CHECK-A64_32-SAME: ) #[[ATTR0]] {
+// CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPADDRSPACE:%.*]], align 4
+// CHECK-A64_32-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SPADDRSPACE]], align 4
+// CHECK-A64_32-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_TMP]], ptr align 4 [[S]], i32 4, i1 false)
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE]], align 4
+// CHECK-A64_32-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i32
+// CHECK-A64_32-NEXT:    [[COERCE_VAL_II:%.*]] = zext i32 [[COERCE_VAL_PI]] to i64
+// CHECK-A64_32-NEXT:    call void @_Z11Tpaddrspace11Spaddrspace(i64 [[COERCE_VAL_II]])
+// CHECK-A64_32-NEXT:    ret void
+//
+void Cpaddrspace() { Spaddrspace s; Tpaddrspace(s); }
+
+struct Sp2addrspace {
+    __attribute__((address_space(100))) int *x[2];
+};
+// CHECK-A64-LABEL: define dso_local void @_Z12Tp2addrspace12Sp2addrspace(
+// CHECK-A64-SAME: [2 x i64] [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SP2ADDRSPACE:%.*]], align 8
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64-NEXT:    store [2 x i64] [[S_COERCE]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x ptr addrspace(100)], ptr [[X]], i64 0, i64 0
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[ARRAYIDX]], align 8
+// CHECK-A64-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
+// CHECK-A64-NEXT:    ret void
+//
+// CHECK-A64_32-LABEL: define void @_Z12Tp2addrspace12Sp2addrspace(
+// CHECK-A64_32-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SP2ADDRSPACE:%.*]], align 4
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    store i64 [[S_COERCE]], ptr [[COERCE_DIVE]], align 4
+// CHECK-A64_32-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x ptr addrspace(100)], ptr [[X]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[ARRAYIDX]], align 4
+// CHECK-A64_32-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
+// CHECK-A64_32-NEXT:    ret void
+//
+void Tp2addrspace(Sp2addrspace s) { *s.x[0] = 1; }
+// CHECK-A64-LABEL: define dso_local void @_Z12Cp2addrspacev(
+// CHECK-A64-SAME: ) #[[ATTR0]] {
+// CHECK-A64-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SP2ADDRSPACE:%.*]], align 8
+// CHECK-A64-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SP2ADDRSPACE]], align 8
+// CHECK-A64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP]], ptr align 8 [[S]], i64 16, i1 false)
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load [2 x i64], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    call void @_Z12Tp2addrspace12Sp2addrspace([2 x i64] [[TMP0]])
+// CHECK-A64-NEXT:    ret void
+//
+// CHECK-A64_32-LABEL: define void @_Z12Cp2addrspacev(
+// CHECK-A64_32-SAME: ) #[[ATTR0]] {
+// CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SP2ADDRSPACE:%.*]], align 4
+// CHECK-A64_32-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SP2ADDRSPACE]], align 4
+// CHECK-A64_32-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_TMP]], ptr align 4 [[S]], i32 8, i1 false)
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load i64, ptr [[COERCE_DIVE]], align 4
+// CHECK-A64_32-NEXT:    call void @_Z12Tp2addrspace12Sp2addrspace(i64 [[TMP0]])
+// CHECK-A64_32-NEXT:    ret void
+//
+void Cp2addrspace() { Sp2addrspace s; Tp2addrspace(s); }
+
+struct Sraddrspace {
+    __attribute__((address_space(100))) int &x;
+};
+// CHECK-A64-LABEL: define dso_local void @_Z11Traddrspace11Sraddrspace(
+// CHECK-A64-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRADDRSPACE:%.*]], align 8
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr addrspace(100)
+// CHECK-A64-NEXT:    store ptr addrspace(100) [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 8, !align [[META3]]
+// CHECK-A64-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
+// CHECK-A64-NEXT:    ret void
+//
+// CHECK-A64_32-LABEL: define void @_Z11Traddrspace11Sraddrspace(
+// CHECK-A64_32-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRADDRSPACE:%.*]], align 4
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[COERCE_VAL_II:%.*]] = trunc i64 [[S_COERCE]] to i32
+// CHECK-A64_32-NEXT:    store i32 [[COERCE_VAL_II]], ptr [[COERCE_DIVE]], align 4
+// CHECK-A64_32-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 4, !align [[META3]]
+// CHECK-A64_32-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
+// CHECK-A64_32-NEXT:    ret void
+//
+void Traddrspace(Sraddrspace s) { s.x = 1; }
+// CHECK-A64-LABEL: define dso_local void @_Z11Craddrspace11Sraddrspace(
+// CHECK-A64-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRADDRSPACE:%.*]], align 8
+// CHECK-A64-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SRADDRSPACE]], align 8
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr addrspace(100)
+// CHECK-A64-NEXT:    store ptr addrspace(100) [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8
+// CHECK-A64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP]], ptr align 8 [[S]], i64 8, i1 false)
+// CHECK-A64-NEXT:    [[COERCE_DIVE1:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE1]], align 8
+// CHECK-A64-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i64
+// CHECK-A64-NEXT:    call void @_Z11Traddrspace11Sraddrspace(i64 [[COERCE_VAL_PI]])
+// CHECK-A64-NEXT:    ret void
+//
+// CHECK-A64_32-LABEL: define void @_Z11Craddrspace11Sraddrspace(
+// CHECK-A64_32-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
+// CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRADDRSPACE:%.*]], align 4
+// CHECK-A64_32-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SRADDRSPACE]], align 4
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[COERCE_VAL_II:%.*]] = trunc i64 [[S_COERCE]] to i32
+// CHECK-A64_32-NEXT:    store i32 [[COERCE_VAL_II]], ptr [[COERCE_DIVE]], align 4
+// CHECK-A64_32-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_TMP]], ptr align 4 [[S]], i32 4, i1 false)
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE1:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE1]], align 4
+// CHECK-A64_32-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i32
+// CHECK-A64_32-NEXT:    [[COERCE_VAL_II2:%.*]] = zext i32 [[COERCE_VAL_PI]] to i64
+// CHECK-A64_32-NEXT:    call void @_Z11Traddrspace11Sraddrspace(i64 [[COERCE_VAL_II2]])
+// CHECK-A64_32-NEXT:    ret void
+//
+void Craddrspace(Sraddrspace s) { Traddrspace(s); }
+
+//.
+// CHECK-A64: [[META2]] = !{}
+// CHECK-A64: [[META3]] = !{i64 4}
+//.
+// CHECK-A64_32: [[META2]] = !{}
+// CHECK-A64_32: [[META3]] = !{i64 4}
+//.
diff --git a/external/llvm-project/clang/test/CodeGen/LoongArch/__fp16-convert.c b/external/llvm-project/clang/test/CodeGen/LoongArch/__fp16-convert.c
new file mode 100644
index 000000000000..84ef5de960b4
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/LoongArch/__fp16-convert.c
@@ -0,0 +1,30 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple loongarch64 -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
+
+__fp16 y;
+short z;
+// CHECK-LABEL: define dso_local void @bar1(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load half, ptr @y, align 2
+// CHECK-NEXT:    [[CONV:%.*]] = fpext half [[TMP0]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptosi float [[CONV]] to i16
+// CHECK-NEXT:    store i16 [[CONV1]], ptr @z, align 2
+// CHECK-NEXT:    ret void
+//
+void bar1(){
+    z = y;
+}
+// CHECK-LABEL: define dso_local void @bar2(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @z, align 2
+// CHECK-NEXT:    [[CONV:%.*]] = sitofp i16 [[TMP0]] to float
+// CHECK-NEXT:    [[CONV1:%.*]] = fptrunc float [[CONV]] to half
+// CHECK-NEXT:    store half [[CONV1]], ptr @y, align 2
+// CHECK-NEXT:    ret void
+//
+void bar2(){
+    y = z;
+}
diff --git a/external/llvm-project/clang/test/CodeGen/LoongArch/abi-lp64d.c b/external/llvm-project/clang/test/CodeGen/LoongArch/abi-lp64d.c
index fc7f1eada586..9f64cfd662e5 100644
--- a/external/llvm-project/clang/test/CodeGen/LoongArch/abi-lp64d.c
+++ b/external/llvm-project/clang/test/CodeGen/LoongArch/abi-lp64d.c
@@ -48,6 +48,9 @@ unsigned long check_ulong() { return 0; }
 // CHECK-LABEL: define{{.*}} i64 @check_ulonglong()
 unsigned long long check_ulonglong() { return 0; }
 
+// CHECK-LABEL: define{{.*}}  half @check_float16()
+_Float16 check_float16() { return 0; }
+
 // CHECK-LABEL: define{{.*}} float @check_float()
 float check_float() { return 0; }
 
@@ -127,6 +130,14 @@ struct i16x4_s f_i16x4_s(struct i16x4_s x) {
 /// available, the value is passed in a GAR; if no GAR is available, the value
 /// is passed on the stack.
 
+struct f16x1_s {
+  __fp16 a;
+};
+
+struct float16x1_s {
+  _Float16 a;
+};
+
 struct f32x1_s {
   float a;
 };
@@ -135,6 +146,16 @@ struct f64x1_s {
   double a;
 };
 
+// CHECK-LABEL: define{{.*}} half @f_f16x1_s(half %0)
+struct f16x1_s f_f16x1_s(struct f16x1_s x) {
+  return x;
+}
+
+// CHECK-LABEL: define{{.*}} half @f_float16x1_s(half %0)
+struct float16x1_s f_float16x1_s(struct float16x1_s x) {
+  return x;
+}
+
 // CHECK-LABEL: define{{.*}} float @f_f32x1_s(float %0)
 struct f32x1_s f_f32x1_s(struct f32x1_s x) {
   return x;
@@ -151,10 +172,20 @@ struct f64x1_s f_f64x1_s(struct f64x1_s x) {
 /// number of available FAR is less than 2, it’s passed in a GAR, and passed on
 /// the stack if no GAR is available.
 
+struct f16x2_s {
+  __fp16 a;
+  _Float16 b;
+};
+
 struct f32x2_s {
   float a, b;
 };
 
+// CHECK-LABEL: define{{.*}} { half, half } @f_f16x2_s(half %0, half %1)
+struct f16x2_s f_f16x2_s(struct f16x2_s x) {
+  return x;
+}
+
 // CHECK-LABEL: define{{.*}} { float, float } @f_f32x2_s(float %0, float %1)
 struct f32x2_s f_f32x2_s(struct f32x2_s x) {
   return x;
@@ -165,11 +196,21 @@ struct f32x2_s f_f32x2_s(struct f32x2_s x) {
 /// i. Multiple fixed-point members. If there are available GAR, the structure
 /// is passed in a GAR, and passed on the stack if no GAR is available.
 
+struct f16x1_i16x2_s {
+  _Float16 a;
+  int16_t b, c;
+};
+
 struct f32x1_i16x2_s {
   float a;
   int16_t b, c;
 };
 
+// CHECK-LABEL: define{{.*}} i64 @f_f16x1_i16x2_s(i64 %x.coerce)
+struct f16x1_i16x2_s f_f16x1_i16x2_s(struct f16x1_i16x2_s x) {
+  return x;
+}
+
 // CHECK-LABEL: define{{.*}} i64 @f_f32x1_i16x2_s(i64 %x.coerce)
 struct f32x1_i16x2_s f_f32x1_i16x2_s(struct f32x1_i16x2_s x) {
   return x;
@@ -181,11 +222,21 @@ struct f32x1_i16x2_s f_f32x1_i16x2_s(struct f32x1_i16x2_s x) {
 /// but one GAR is available, it’s passed in GAR; If no GAR is available, it’s
 /// passed on the stack.
 
+struct f16x1_i32x1_s {
+  _Float16 a;
+  int32_t b;
+};
+
 struct f32x1_i32x1_s {
   float a;
   int32_t b;
 };
 
+// CHECK-LABEL: define{{.*}} { half, i32 } @f_f16x1_i32x1_s(half %0, i32 %1)
+struct f16x1_i32x1_s f_f16x1_i32x1_s(struct f16x1_i32x1_s x) {
+  return x;
+}
+
 // CHECK-LABEL: define{{.*}} { float, i32 } @f_f32x1_i32x1_s(float %0, i32 %1)
 struct f32x1_i32x1_s f_f32x1_i32x1_s(struct f32x1_i32x1_s x) {
   return x;
@@ -253,6 +304,16 @@ struct f32x4_s f_f32x4_s(struct f32x4_s x) {
   return x;
 }
 
+struct f16x5_s {
+  _Float16 a, b, c, d;
+  __fp16 e;
+};
+
+// CHECK-LABEL: define{{.*}} [2 x i64] @f_f16x5_s([2 x i64] %x.coerce)
+struct f16x5_s f_f16x5_s(struct f16x5_s x) {
+  return x;
+}
+
 /// ii. The structure with two double members is passed in a pair of available
 /// FARs. If no a pair of available FARs, it’s passed in GARs. A structure with
 /// one double member and one float member is same.
@@ -312,6 +373,16 @@ struct f32x2_i32x2_s f_f32x2_i32x2_s(struct f32x2_i32x2_s x) {
   return x;
 }
 
+struct f16x4_i32x2_s {
+  _Float16 a, b, c, d;
+  int32_t e, f;
+};
+
+// CHECK-LABEL: define{{.*}} [2 x i64] @f_f16x4_i32x2_s([2 x i64] %x.coerce)
+struct f16x4_i32x2_s f_f16x4_i32x2_s(struct f16x4_i32x2_s x) {
+  return x;
+}
+
 /// 3. WOA > 2 × GRLEN
 /// a. It’s passed by reference and are replaced in the argument list with the
 /// address. If there is an available GAR, the reference is passed in the GAR,
diff --git a/external/llvm-project/clang/test/CodeGen/LoongArch/bfloat-abi.c b/external/llvm-project/clang/test/CodeGen/LoongArch/bfloat-abi.c
new file mode 100644
index 000000000000..a8a252919ef3
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/LoongArch/bfloat-abi.c
@@ -0,0 +1,532 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// RUN: %clang_cc1 -triple loongarch64 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-LA64
+// RUN: %clang_cc1 -triple loongarch32 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-LA32
+
+struct bfloat1 {
+  __bf16 a;
+};
+
+// CHECK-LABEL: define dso_local bfloat @h1
+// CHECK-SAME: (bfloat noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT1:%.*]], align 2
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT1]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw { bfloat }, ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[TMP1]], align 2
+// CHECK-NEXT:    ret bfloat [[TMP2]]
+//
+struct bfloat1 h1(__bf16 a) {
+  struct bfloat1 x;
+  x.a = a;
+  return x;
+}
+
+struct bfloat2 {
+  __bf16 a;
+  __bf16 b;
+};
+
+// CHECK-LABEL: define dso_local { bfloat, bfloat } @h2
+// CHECK-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT2:%.*]], align 2
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw { bfloat, bfloat }, ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[TMP2]], align 2
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw { bfloat, bfloat }, ptr [[RETVAL]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP5:%.*]] = load bfloat, ptr [[TMP4]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { bfloat, bfloat } poison, bfloat [[TMP3]], 0
+// CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { bfloat, bfloat } [[TMP6]], bfloat [[TMP5]], 1
+// CHECK-NEXT:    ret { bfloat, bfloat } [[TMP7]]
+//
+struct bfloat2 h2(__bf16 a, __bf16 b) {
+  struct bfloat2 x;
+  x.a = a;
+  x.b = b;
+  return x;
+}
+
+struct bfloat3 {
+  __bf16 a;
+  __bf16 b;
+  __bf16 c;
+};
+
+// CHECK-LA64-LABEL: define dso_local i64 @h3
+// CHECK-LA64-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]]) #[[ATTR0]] {
+// CHECK-LA64-NEXT:  entry:
+// CHECK-LA64-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT3:%.*]], align 2
+// CHECK-LA64-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[RETVAL_COERCE:%.*]] = alloca i64, align 8
+// CHECK-LA64-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT3]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA64-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-LA64-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT3]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA64-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA64-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT3]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA64-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[RETVAL_COERCE]], ptr align 2 [[RETVAL]], i64 6, i1 false)
+// CHECK-LA64-NEXT:    [[TMP3:%.*]] = load i64, ptr [[RETVAL_COERCE]], align 8
+// CHECK-LA64-NEXT:    ret i64 [[TMP3]]
+//
+// CHECK-LA32-LABEL: define dso_local [2 x i32] @h3
+// CHECK-LA32-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]]) #[[ATTR0]] {
+// CHECK-LA32-NEXT:  entry:
+// CHECK-LA32-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT3:%.*]], align 2
+// CHECK-LA32-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[RETVAL_COERCE:%.*]] = alloca [2 x i32], align 4
+// CHECK-LA32-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT3]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA32-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-LA32-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT3]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA32-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA32-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT3]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA32-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA32-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[RETVAL_COERCE]], ptr align 2 [[RETVAL]], i32 6, i1 false)
+// CHECK-LA32-NEXT:    [[TMP3:%.*]] = load [2 x i32], ptr [[RETVAL_COERCE]], align 4
+// CHECK-LA32-NEXT:    ret [2 x i32] [[TMP3]]
+//
+struct bfloat3 h3(__bf16 a, __bf16 b, __bf16 c) {
+  struct bfloat3 x;
+  x.a = a;
+  x.b = b;
+  x.c = c;
+  return x;
+}
+
+struct bfloat4 {
+  __bf16 a;
+  __bf16 b;
+  __bf16 c;
+  __bf16 d;
+};
+
+// CHECK-LA64-LABEL: define dso_local i64 @h4
+// CHECK-LA64-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]], bfloat noundef [[D:%.*]]) #[[ATTR0]] {
+// CHECK-LA64-NEXT:  entry:
+// CHECK-LA64-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT4:%.*]], align 2
+// CHECK-LA64-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[D_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[D]], ptr [[D_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA64-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-LA64-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA64-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA64-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA64-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA64-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[D_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[D4:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 3
+// CHECK-LA64-NEXT:    store bfloat [[TMP3]], ptr [[D4]], align 2
+// CHECK-LA64-NEXT:    [[TMP4:%.*]] = load i64, ptr [[RETVAL]], align 2
+// CHECK-LA64-NEXT:    ret i64 [[TMP4]]
+//
+// CHECK-LA32-LABEL: define dso_local [2 x i32] @h4
+// CHECK-LA32-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]], bfloat noundef [[D:%.*]]) #[[ATTR0]] {
+// CHECK-LA32-NEXT:  entry:
+// CHECK-LA32-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT4:%.*]], align 2
+// CHECK-LA32-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[D_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[D]], ptr [[D_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA32-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-LA32-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA32-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA32-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA32-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA32-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[D_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[D4:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT4]], ptr [[RETVAL]], i32 0, i32 3
+// CHECK-LA32-NEXT:    store bfloat [[TMP3]], ptr [[D4]], align 2
+// CHECK-LA32-NEXT:    [[TMP4:%.*]] = load [2 x i32], ptr [[RETVAL]], align 2
+// CHECK-LA32-NEXT:    ret [2 x i32] [[TMP4]]
+//
+struct bfloat4 h4(__bf16 a, __bf16 b, __bf16 c, __bf16 d) {
+  struct bfloat4 x;
+  x.a = a;
+  x.b = b;
+  x.c = c;
+  x.d = d;
+  return x;
+}
+
+struct floatbfloat {
+  float a;
+  __bf16 b;
+};
+
+// CHECK-LABEL: define dso_local { float, bfloat } @fh
+// CHECK-SAME: (float noundef [[A:%.*]], bfloat noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_FLOATBFLOAT:%.*]], align 4
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    store float [[A]], ptr [[A_ADDR]], align 4
+// CHECK-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[A_ADDR]], align 4
+// CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    store float [[TMP0]], ptr [[A1]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw { float, bfloat }, ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw { float, bfloat }, ptr [[RETVAL]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP5:%.*]] = load bfloat, ptr [[TMP4]], align 4
+// CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { float, bfloat } poison, float [[TMP3]], 0
+// CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { float, bfloat } [[TMP6]], bfloat [[TMP5]], 1
+// CHECK-NEXT:    ret { float, bfloat } [[TMP7]]
+//
+struct floatbfloat fh(float a, __bf16 b) {
+  struct floatbfloat x;
+  x.a = a;
+  x.b = b;
+  return x;
+}
+
+struct floatbfloat2 {
+  float a;
+  __bf16 b;
+  __bf16 c;
+};
+
+// CHECK-LA64-LABEL: define dso_local i64 @fh2
+// CHECK-LA64-SAME: (float noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]]) #[[ATTR0]] {
+// CHECK-LA64-NEXT:  entry:
+// CHECK-LA64-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_FLOATBFLOAT2:%.*]], align 4
+// CHECK-LA64-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// CHECK-LA64-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    store float [[A]], ptr [[A_ADDR]], align 4
+// CHECK-LA64-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[TMP0:%.*]] = load float, ptr [[A_ADDR]], align 4
+// CHECK-LA64-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT2]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA64-NEXT:    store float [[TMP0]], ptr [[A1]], align 4
+// CHECK-LA64-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT2]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA64-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 4
+// CHECK-LA64-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT2]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA64-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA64-NEXT:    [[TMP3:%.*]] = load i64, ptr [[RETVAL]], align 4
+// CHECK-LA64-NEXT:    ret i64 [[TMP3]]
+//
+// CHECK-LA32-LABEL: define dso_local [2 x i32] @fh2
+// CHECK-LA32-SAME: (float noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]]) #[[ATTR0]] {
+// CHECK-LA32-NEXT:  entry:
+// CHECK-LA32-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_FLOATBFLOAT2:%.*]], align 4
+// CHECK-LA32-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// CHECK-LA32-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    store float [[A]], ptr [[A_ADDR]], align 4
+// CHECK-LA32-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[TMP0:%.*]] = load float, ptr [[A_ADDR]], align 4
+// CHECK-LA32-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT2]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA32-NEXT:    store float [[TMP0]], ptr [[A1]], align 4
+// CHECK-LA32-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT2]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA32-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 4
+// CHECK-LA32-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT2]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA32-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA32-NEXT:    [[TMP3:%.*]] = load [2 x i32], ptr [[RETVAL]], align 4
+// CHECK-LA32-NEXT:    ret [2 x i32] [[TMP3]]
+//
+struct floatbfloat2 fh2(float a, __bf16 b, __bf16 c) {
+  struct floatbfloat2 x;
+  x.a = a;
+  x.b = b;
+  x.c = c;
+  return x;
+}
+
+struct bfloatfloat {
+  __bf16 a;
+  float b;
+};
+
+// CHECK-LABEL: define dso_local { bfloat, float } @hf
+// CHECK-SAME: (bfloat noundef [[A:%.*]], float noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOATFLOAT:%.*]], align 4
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca float, align 4
+// CHECK-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    store float [[B]], ptr [[B_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOATFLOAT]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[B_ADDR]], align 4
+// CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOATFLOAT]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-NEXT:    store float [[TMP1]], ptr [[B2]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw { bfloat, float }, ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[TMP2]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw { bfloat, float }, ptr [[RETVAL]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[TMP4]], align 4
+// CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { bfloat, float } poison, bfloat [[TMP3]], 0
+// CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { bfloat, float } [[TMP6]], float [[TMP5]], 1
+// CHECK-NEXT:    ret { bfloat, float } [[TMP7]]
+//
+struct bfloatfloat hf(__bf16 a, float b) {
+  struct bfloatfloat x;
+  x.a = a;
+  x.b = b;
+  return x;
+}
+
+struct bfloat2float {
+  __bf16 a;
+  __bf16 b;
+  float c;
+};
+
+// CHECK-LA64-LABEL: define dso_local i64 @h2f
+// CHECK-LA64-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], float noundef [[C:%.*]]) #[[ATTR0]] {
+// CHECK-LA64-NEXT:  entry:
+// CHECK-LA64-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT2FLOAT:%.*]], align 4
+// CHECK-LA64-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[C_ADDR:%.*]] = alloca float, align 4
+// CHECK-LA64-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    store float [[C]], ptr [[C_ADDR]], align 4
+// CHECK-LA64-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2FLOAT]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA64-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 4
+// CHECK-LA64-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2FLOAT]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA64-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA64-NEXT:    [[TMP2:%.*]] = load float, ptr [[C_ADDR]], align 4
+// CHECK-LA64-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2FLOAT]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA64-NEXT:    store float [[TMP2]], ptr [[C3]], align 4
+// CHECK-LA64-NEXT:    [[TMP3:%.*]] = load i64, ptr [[RETVAL]], align 4
+// CHECK-LA64-NEXT:    ret i64 [[TMP3]]
+//
+// CHECK-LA32-LABEL: define dso_local [2 x i32] @h2f
+// CHECK-LA32-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], float noundef [[C:%.*]]) #[[ATTR0]] {
+// CHECK-LA32-NEXT:  entry:
+// CHECK-LA32-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT2FLOAT:%.*]], align 4
+// CHECK-LA32-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[C_ADDR:%.*]] = alloca float, align 4
+// CHECK-LA32-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    store float [[C]], ptr [[C_ADDR]], align 4
+// CHECK-LA32-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2FLOAT]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA32-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 4
+// CHECK-LA32-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2FLOAT]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA32-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA32-NEXT:    [[TMP2:%.*]] = load float, ptr [[C_ADDR]], align 4
+// CHECK-LA32-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT2FLOAT]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA32-NEXT:    store float [[TMP2]], ptr [[C3]], align 4
+// CHECK-LA32-NEXT:    [[TMP3:%.*]] = load [2 x i32], ptr [[RETVAL]], align 4
+// CHECK-LA32-NEXT:    ret [2 x i32] [[TMP3]]
+//
+struct bfloat2float h2f(__bf16 a, __bf16 b, float c) {
+  struct bfloat2float x;
+  x.a = a;
+  x.b = b;
+  x.c = c;
+  return x;
+}
+
+struct floatbfloat3 {
+  float a;
+  __bf16 b;
+  __bf16 c;
+  __bf16 d;
+};
+
+// CHECK-LA64-LABEL: define dso_local [2 x i64] @fh3
+// CHECK-LA64-SAME: (float noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]], bfloat noundef [[D:%.*]]) #[[ATTR0]] {
+// CHECK-LA64-NEXT:  entry:
+// CHECK-LA64-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_FLOATBFLOAT3:%.*]], align 4
+// CHECK-LA64-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// CHECK-LA64-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[D_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[RETVAL_COERCE:%.*]] = alloca [2 x i64], align 8
+// CHECK-LA64-NEXT:    store float [[A]], ptr [[A_ADDR]], align 4
+// CHECK-LA64-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[D]], ptr [[D_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[TMP0:%.*]] = load float, ptr [[A_ADDR]], align 4
+// CHECK-LA64-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA64-NEXT:    store float [[TMP0]], ptr [[A1]], align 4
+// CHECK-LA64-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA64-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 4
+// CHECK-LA64-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA64-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA64-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[D_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[D4:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[RETVAL]], i32 0, i32 3
+// CHECK-LA64-NEXT:    store bfloat [[TMP3]], ptr [[D4]], align 4
+// CHECK-LA64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[RETVAL_COERCE]], ptr align 4 [[RETVAL]], i64 12, i1 false)
+// CHECK-LA64-NEXT:    [[TMP4:%.*]] = load [2 x i64], ptr [[RETVAL_COERCE]], align 8
+// CHECK-LA64-NEXT:    ret [2 x i64] [[TMP4]]
+//
+// CHECK-LA32-LABEL: define dso_local void @fh3
+// CHECK-LA32-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_FLOATBFLOAT3:%.*]]) align 4 [[AGG_RESULT:%.*]], float noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]], bfloat noundef [[D:%.*]]) #[[ATTR0]] {
+// CHECK-LA32-NEXT:  entry:
+// CHECK-LA32-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 4
+// CHECK-LA32-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// CHECK-LA32-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[D_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 4
+// CHECK-LA32-NEXT:    store float [[A]], ptr [[A_ADDR]], align 4
+// CHECK-LA32-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[D]], ptr [[D_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[TMP0:%.*]] = load float, ptr [[A_ADDR]], align 4
+// CHECK-LA32-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[AGG_RESULT]], i32 0, i32 0
+// CHECK-LA32-NEXT:    store float [[TMP0]], ptr [[A1]], align 4
+// CHECK-LA32-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[AGG_RESULT]], i32 0, i32 1
+// CHECK-LA32-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 4
+// CHECK-LA32-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[AGG_RESULT]], i32 0, i32 2
+// CHECK-LA32-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA32-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[D_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[D4:%.*]] = getelementptr inbounds nuw [[STRUCT_FLOATBFLOAT3]], ptr [[AGG_RESULT]], i32 0, i32 3
+// CHECK-LA32-NEXT:    store bfloat [[TMP3]], ptr [[D4]], align 4
+// CHECK-LA32-NEXT:    ret void
+//
+struct floatbfloat3 fh3(float a, __bf16 b, __bf16 c, __bf16 d) {
+  struct floatbfloat3 x;
+  x.a = a;
+  x.b = b;
+  x.c = c;
+  x.d = d;
+  return x;
+}
+
+struct bfloat5 {
+  __bf16 a;
+  __bf16 b;
+  __bf16 c;
+  __bf16 d;
+  __bf16 e;
+};
+
+// CHECK-LA64-LABEL: define dso_local [2 x i64] @h5
+// CHECK-LA64-SAME: (bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]], bfloat noundef [[D:%.*]], bfloat noundef [[E:%.*]]) #[[ATTR0]] {
+// CHECK-LA64-NEXT:  entry:
+// CHECK-LA64-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_BFLOAT5:%.*]], align 2
+// CHECK-LA64-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[D_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[E_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA64-NEXT:    [[RETVAL_COERCE:%.*]] = alloca [2 x i64], align 8
+// CHECK-LA64-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[D]], ptr [[D_ADDR]], align 2
+// CHECK-LA64-NEXT:    store bfloat [[E]], ptr [[E_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-LA64-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-LA64-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[RETVAL]], i32 0, i32 1
+// CHECK-LA64-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA64-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[RETVAL]], i32 0, i32 2
+// CHECK-LA64-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA64-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[D_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[D4:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[RETVAL]], i32 0, i32 3
+// CHECK-LA64-NEXT:    store bfloat [[TMP3]], ptr [[D4]], align 2
+// CHECK-LA64-NEXT:    [[TMP4:%.*]] = load bfloat, ptr [[E_ADDR]], align 2
+// CHECK-LA64-NEXT:    [[E5:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[RETVAL]], i32 0, i32 4
+// CHECK-LA64-NEXT:    store bfloat [[TMP4]], ptr [[E5]], align 2
+// CHECK-LA64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[RETVAL_COERCE]], ptr align 2 [[RETVAL]], i64 10, i1 false)
+// CHECK-LA64-NEXT:    [[TMP5:%.*]] = load [2 x i64], ptr [[RETVAL_COERCE]], align 8
+// CHECK-LA64-NEXT:    ret [2 x i64] [[TMP5]]
+//
+// CHECK-LA32-LABEL: define dso_local void @h5
+// CHECK-LA32-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_BFLOAT5:%.*]]) align 2 [[AGG_RESULT:%.*]], bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]], bfloat noundef [[C:%.*]], bfloat noundef [[D:%.*]], bfloat noundef [[E:%.*]]) #[[ATTR0]] {
+// CHECK-LA32-NEXT:  entry:
+// CHECK-LA32-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 4
+// CHECK-LA32-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[C_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[D_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    [[E_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-LA32-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 4
+// CHECK-LA32-NEXT:    store bfloat [[A]], ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[C]], ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[D]], ptr [[D_ADDR]], align 2
+// CHECK-LA32-NEXT:    store bfloat [[E]], ptr [[E_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[AGG_RESULT]], i32 0, i32 0
+// CHECK-LA32-NEXT:    store bfloat [[TMP0]], ptr [[A1]], align 2
+// CHECK-LA32-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[AGG_RESULT]], i32 0, i32 1
+// CHECK-LA32-NEXT:    store bfloat [[TMP1]], ptr [[B2]], align 2
+// CHECK-LA32-NEXT:    [[TMP2:%.*]] = load bfloat, ptr [[C_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[C3:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[AGG_RESULT]], i32 0, i32 2
+// CHECK-LA32-NEXT:    store bfloat [[TMP2]], ptr [[C3]], align 2
+// CHECK-LA32-NEXT:    [[TMP3:%.*]] = load bfloat, ptr [[D_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[D4:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[AGG_RESULT]], i32 0, i32 3
+// CHECK-LA32-NEXT:    store bfloat [[TMP3]], ptr [[D4]], align 2
+// CHECK-LA32-NEXT:    [[TMP4:%.*]] = load bfloat, ptr [[E_ADDR]], align 2
+// CHECK-LA32-NEXT:    [[E5:%.*]] = getelementptr inbounds nuw [[STRUCT_BFLOAT5]], ptr [[AGG_RESULT]], i32 0, i32 4
+// CHECK-LA32-NEXT:    store bfloat [[TMP4]], ptr [[E5]], align 2
+// CHECK-LA32-NEXT:    ret void
+//
+struct bfloat5 h5(__bf16 a, __bf16 b, __bf16 c, __bf16 d, __bf16 e) {
+  struct bfloat5 x;
+  x.a = a;
+  x.b = b;
+  x.c = c;
+  x.d = d;
+  x.e = e;
+  return x;
+}
diff --git a/external/llvm-project/clang/test/CodeGen/LoongArch/bfloat-mangle.cpp b/external/llvm-project/clang/test/CodeGen/LoongArch/bfloat-mangle.cpp
new file mode 100644
index 000000000000..de4a10dbe44b
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/LoongArch/bfloat-mangle.cpp
@@ -0,0 +1,12 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// RUN: %clang_cc1 -triple loongarch64 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple loongarch32 -emit-llvm -o - %s | FileCheck %s
+
+// CHECK-LABEL: define dso_local void @_Z3fooDF16b
+// CHECK-SAME: (bfloat noundef [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2
+// CHECK-NEXT:    store bfloat [[B]], ptr [[B_ADDR]], align 2
+// CHECK-NEXT:    ret void
+//
+void foo(__bf16 b) {}
diff --git a/external/llvm-project/clang/test/CodeGen/LoongArch/targetattr.c b/external/llvm-project/clang/test/CodeGen/LoongArch/targetattr.c
new file mode 100644
index 000000000000..91156e6006e7
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/LoongArch/targetattr.c
@@ -0,0 +1,89 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --version 5
+// RUN: %clang_cc1 -triple loongarch64 -emit-llvm %s -o - | FileCheck %s
+
+__attribute__((target("div32")))
+// CHECK-LABEL: define dso_local void @testdiv32(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void testdiv32() {}
+
+__attribute__((target("arch=loongarch64")))
+// CHECK-LABEL: define dso_local void @testLoongarch64(
+// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void testLoongarch64() {}
+
+__attribute__((target("arch=la64v1.0")))
+// CHECK-LABEL: define dso_local void @testLa64v10(
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void testLa64v10() {}
+
+__attribute__((target("arch=la64v1.1")))
+// CHECK-LABEL: define dso_local void @testLa64v11(
+// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void testLa64v11() {}
+
+__attribute__((target("arch=la464")))
+// CHECK-LABEL: define dso_local void @testLa464(
+// CHECK-SAME: ) #[[ATTR4:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void testLa464() {}
+
+__attribute__((target("arch=la664")))
+// CHECK-LABEL: define dso_local void @testLa664(
+// CHECK-SAME: ) #[[ATTR5:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void testLa664() {}
+
+__attribute__((target("arch=la664, no-div32")))
+// CHECK-LABEL: define dso_local void @la664Nodiv32(
+// CHECK-SAME: ) #[[ATTR6:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void la664Nodiv32() {}
+
+__attribute__((target("tune=la464")))
+// CHECK-LABEL: define dso_local void @tuneLa464(
+// CHECK-SAME: ) #[[ATTR7:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void tuneLa464() {}
+
+__attribute__((target("arch=la464, tune=la664")))
+// CHECK-LABEL: define dso_local void @archLa464tuneLa664(
+// CHECK-SAME: ) #[[ATTR8:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void archLa464tuneLa664() {}
+
+//.
+// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+64bit,+div32" }
+// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="loongarch64" "target-features"="+64bit,+d,+f,+ual" }
+// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="loongarch64" "target-features"="+64bit,+d,+lsx,+ual" }
+// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="loongarch64" "target-features"="+64bit,+d,+div32,+frecipe,+lam-bh,+lamcas,+ld-seq-sa,+lsx,+scq,+ual" }
+// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="la464" "target-features"="+64bit,+d,+f,+lasx,+lsx,+ual" }
+// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="la664" "target-features"="+64bit,+d,+div32,+f,+frecipe,+lam-bh,+lamcas,+lasx,+ld-seq-sa,+lsx,+scq,+ual" }
+// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="la664" "target-features"="+64bit,+d,+f,+frecipe,+lam-bh,+lamcas,+lasx,+ld-seq-sa,+lsx,+scq,+ual,-div32" }
+// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+64bit" "tune-cpu"="la464" }
+// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="la464" "target-features"="+64bit,+d,+f,+lasx,+lsx,+ual" "tune-cpu"="la664" }
+//.
+// CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vrgatherei16.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vrgatherei16.c
index 32469731d114..41214f7cdce2 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vrgatherei16.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vrgatherei16.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslidedown.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslidedown.c
index c4e7d86e7d53..8b97ce8f760c 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslidedown.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslidedown.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslideup.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslideup.c
index 0ab387525f6a..c302b2940bc6 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslideup.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/vslideup.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vrgatherei16.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vrgatherei16.c
index f69613c4777f..a63f0a59a34e 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vrgatherei16.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vrgatherei16.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslidedown.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslidedown.c
index 14b3a99a6f0f..fb99a750a670 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslidedown.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslidedown.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslideup.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslideup.c
index 1b3c3f6c0f85..77e8122890ab 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslideup.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/vslideup.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vrgatherei16.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vrgatherei16.c
index e22da32dbfa8..cf98549c41af 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vrgatherei16.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vrgatherei16.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslidedown.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslidedown.c
index 205866db3566..4f1c00bef076 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslidedown.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslidedown.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslideup.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslideup.c
index b32264fd88e7..c9fa994e51b3 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslideup.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/vslideup.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vrgatherei16.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vrgatherei16.c
index 3d53e46b4885..c50f1f731ffb 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vrgatherei16.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vrgatherei16.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslidedown.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslidedown.c
index c275ee9bb2f6..476b9b59dc19 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslidedown.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslidedown.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslideup.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslideup.c
index 9bd602fa5d76..1e0228e17caf 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslideup.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/vslideup.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh \
-// RUN:   -target-feature +zvfh -disable-O0-optnone  \
+// RUN:   -target-feature +zvfhmin -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
 
diff --git a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/zvfhmin-error.c b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/zvfhmin-error.c
index d45ed77da633..7a07d2339178 100644
--- a/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/zvfhmin-error.c
+++ b/external/llvm-project/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/zvfhmin-error.c
@@ -17,7 +17,7 @@
 // CHECK-ZVF-NEXT:    ret <vscale x 4 x half> [[TMP0]]
 //
 
-// CHECK-ZVFHMIN-ERR: no matching function for call to '__riscv_vfadd'
+// CHECK-ZVFHMIN-ERR: builtin requires at least one of the following extensions: zvfh
 
 vfloat16m1_t test_vfadd_vv_f16m1(vfloat16m1_t op1, vfloat16m1_t op2, size_t vl) {
   return __riscv_vfadd(op1, op2, vl);
diff --git a/external/llvm-project/clang/test/CodeGen/X86/avx-cxx-record.cpp b/external/llvm-project/clang/test/CodeGen/X86/avx-cxx-record.cpp
index bcd9c361fda9..6ce6815a521a 100644
--- a/external/llvm-project/clang/test/CodeGen/X86/avx-cxx-record.cpp
+++ b/external/llvm-project/clang/test/CodeGen/X86/avx-cxx-record.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -O2 -target-cpu x86-64-v3 -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -O2 -target-cpu x86-64-v3 -fclang-abi-compat=20 -o - | FileCheck --check-prefix CLANG-20 %s
 
 using UInt64x2 = unsigned long long __attribute__((__vector_size__(16), may_alias));
 
@@ -11,6 +12,7 @@ struct XMM2 : XMM1<0>, XMM1<1> {
 };
 
 // CHECK: define{{.*}} @_Z3foov({{.*}} [[ARG:%.*]]){{.*}}
+// CLANG-20: define{{.*}} <4 x double> @_Z3foov()
 // CHECK: entry:
 // CHECK-NEXT: store {{.*}}, ptr [[ARG]]{{.*}}
 // CHECK-NEXT: [[TMP1:%.*]] = getelementptr {{.*}}, ptr [[ARG]]{{.*}}
diff --git a/external/llvm-project/clang/test/CodeGen/X86/cygwin-varargs.c b/external/llvm-project/clang/test/CodeGen/X86/cygwin-varargs.c
new file mode 100644
index 000000000000..4eea7d64bcb3
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/X86/cygwin-varargs.c
@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 -triple x86_64-windows-gnu -emit-llvm < %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-cygwin -emit-llvm < %s | FileCheck %s
+
+struct foo {
+  int x;
+  float y;
+  char z;
+};
+// CHECK: %[[STRUCT_FOO:.*]] = type { i32, float, i8 }
+
+void f(int a, ...) {
+  // CHECK-LABEL: define dso_local void @f
+  __builtin_va_list ap;
+  __builtin_va_start(ap, a);
+  // CHECK: %[[AP:.*]] = alloca ptr
+  // CHECK: call void @llvm.va_start
+  int b = __builtin_va_arg(ap, int);
+  // CHECK: %[[AP_CUR:.*]] = load ptr, ptr %[[AP]]
+  // CHECK-NEXT: %[[AP_NEXT:.*]] = getelementptr inbounds i8, ptr %[[AP_CUR]], i64 8
+  // CHECK-NEXT: store ptr %[[AP_NEXT]], ptr %[[AP]]
+  double _Complex c = __builtin_va_arg(ap, double _Complex);
+  // CHECK: %[[AP_CUR2:.*]] = load ptr, ptr %[[AP]]
+  // CHECK-NEXT: %[[AP_NEXT2:.*]] = getelementptr inbounds i8, ptr %[[AP_CUR2]], i64 8
+  // CHECK-NEXT: store ptr %[[AP_NEXT2]], ptr %[[AP]]
+  // CHECK-NEXT: load ptr, ptr %[[AP_CUR2]]
+  struct foo d = __builtin_va_arg(ap, struct foo);
+  // CHECK: %[[AP_CUR3:.*]] = load ptr, ptr %[[AP]]
+  // CHECK-NEXT: %[[AP_NEXT3:.*]] = getelementptr inbounds i8, ptr %[[AP_CUR3]], i64 8
+  // CHECK-NEXT: store ptr %[[AP_NEXT3]], ptr %[[AP]]
+  __builtin_va_list ap2;
+  __builtin_va_copy(ap2, ap);
+  // CHECK: call void @llvm.va_copy
+  __builtin_va_end(ap);
+  // CHECK: call void @llvm.va_end
+}
diff --git a/external/llvm-project/clang/test/CodeGen/alias.c b/external/llvm-project/clang/test/CodeGen/alias.c
index bc4167adf53f..9403c55beae0 100644
--- a/external/llvm-project/clang/test/CodeGen/alias.c
+++ b/external/llvm-project/clang/test/CodeGen/alias.c
@@ -29,20 +29,20 @@ const int wacom_usb_ids[] = {1, 1, 2, 3, 5, 8, 13, 0};
 extern const int __mod_usb_device_table __attribute__ ((alias("wacom_usb_ids")));
 // CHECKBASIC-DAG: @__mod_usb_device_table ={{.*}} alias i32, ptr @wacom_usb_ids
 // CHECKASM-DAG: .globl __mod_usb_device_table
-// CHECKASM-DAG: .set __mod_usb_device_table, wacom_usb_ids
+// CHECKASM-DAG: __mod_usb_device_table = wacom_usb_ids
 // CHECKASM-NOT: .size __mod_usb_device_table
 
 extern int g1;
 extern int g1 __attribute((alias("g0")));
 // CHECKBASIC-DAG: @g1 ={{.*}} alias i32, ptr @g0
 // CHECKASM-DAG: .globl g1
-// CHECKASM-DAG: .set g1, g0
+// CHECKASM-DAG: g1 = g0
 // CHECKASM-NOT: .size g1
 
 extern __thread int __libc_errno __attribute__ ((alias ("TL_WITH_ALIAS")));
 // CHECKBASIC-DAG: @__libc_errno ={{.*}} thread_local alias i32, ptr @TL_WITH_ALIAS
 // CHECKASM-DAG: .globl __libc_errno
-// CHECKASM-DAG: .set __libc_errno, TL_WITH_ALIAS
+// CHECKASM-DAG: __libc_errno = TL_WITH_ALIAS
 // CHECKASM-NOT: .size __libc_errno
 
 void f0(void) { }
diff --git a/external/llvm-project/clang/test/CodeGen/arm-former-microsoft-intrinsics-header-warning.c b/external/llvm-project/clang/test/CodeGen/arm-former-microsoft-intrinsics-header-warning.c
new file mode 100644
index 000000000000..8edcbbeb0375
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/arm-former-microsoft-intrinsics-header-warning.c
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple thumbv7-windows -fms-compatibility -Wno-everything -Wimplicit-function-declaration -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple armv7-eabi -Wno-everything -Wimplicit-function-declaration -fsyntax-only -verify %s
+
+void check__dmb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dmb(0);
+}
+
+void check__dsb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dsb(0);
+}
+
+void check__isb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __isb(0);
+}
+
+void check__yield(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __yield();
+}
+
+void check__wfe(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfe();
+}
+
+void check__wfi(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfi();
+}
+
+void check__sev(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sev();
+}
+
+void check__sevl(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sevl();
+}
diff --git a/external/llvm-project/clang/test/CodeGen/arm-former-microsoft-intrinsics.c b/external/llvm-project/clang/test/CodeGen/arm-former-microsoft-intrinsics.c
new file mode 100644
index 000000000000..8a02602dc987
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/arm-former-microsoft-intrinsics.c
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -triple thumbv7-windows -fms-compatibility -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+
+// RUN: %clang_cc1 -triple armv7-eabi -Werror -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+#include <arm_acle.h>
+void check__dmb(void) {
+  __dmb(0);
+}
+
+// CHECK: @llvm.arm.dmb(i32 0)
+
+void check__dsb(void) {
+  __dsb(0);
+}
+
+// CHECK: @llvm.arm.dsb(i32 0)
+
+void check__isb(void) {
+  __isb(0);
+}
+
+// CHECK: @llvm.arm.isb(i32 0)
+
+void check__yield(void) {
+  __yield();
+}
+
+// CHECK: @llvm.arm.hint(i32 1)
+
+void check__wfe(void) {
+  __wfe();
+}
+
+// CHECK: @llvm.arm.hint(i32 2)
+
+void check__wfi(void) {
+  __wfi();
+}
+
+// CHECK: @llvm.arm.hint(i32 3)
+
+void check__sev(void) {
+  __sev();
+}
+
+// CHECK: @llvm.arm.hint(i32 4)
+
+void check__sevl(void) {
+  __sevl();
+}
+
+// CHECK: @llvm.arm.hint(i32 5)
diff --git a/external/llvm-project/clang/test/CodeGen/arm-microsoft-intrinsics.c b/external/llvm-project/clang/test/CodeGen/arm-microsoft-intrinsics.c
index 6793b3879eed..a5f3e2c71922 100644
--- a/external/llvm-project/clang/test/CodeGen/arm-microsoft-intrinsics.c
+++ b/external/llvm-project/clang/test/CodeGen/arm-microsoft-intrinsics.c
@@ -4,27 +4,6 @@
 // RUN: not %clang_cc1 -triple armv7-eabi -Werror -S -o /dev/null %s 2>&1 \
 // RUN:    | FileCheck %s -check-prefix CHECK-EABI
 
-void check__dmb(void) {
-  __dmb(0);
-}
-
-// CHECK-MSVC: @llvm.arm.dmb(i32 0)
-// CHECK-EABI: error: call to undeclared function '__dmb'
-
-void check__dsb(void) {
-  __dsb(0);
-}
-
-// CHECK-MSVC: @llvm.arm.dsb(i32 0)
-// CHECK-EABI: error: call to undeclared function '__dsb'
-
-void check__isb(void) {
-  __isb(0);
-}
-
-// CHECK-MSVC: @llvm.arm.isb(i32 0)
-// CHECK-EABI: error: call to undeclared function '__isb'
-
 __INT64_TYPE__ check__ldrexd(void) {
   __INT64_TYPE__ i64;
   return __ldrexd(&i64);
diff --git a/external/llvm-project/clang/test/CodeGen/arm-mve-intrinsics/vrnd.c b/external/llvm-project/clang/test/CodeGen/arm-mve-intrinsics/vrnd.c
index 3e625c739bde..4888bc8c5e98 100644
--- a/external/llvm-project/clang/test/CodeGen/arm-mve-intrinsics/vrnd.c
+++ b/external/llvm-project/clang/test/CodeGen/arm-mve-intrinsics/vrnd.c
@@ -148,7 +148,7 @@ float32x4_t test_vrndxq_f32(float32x4_t a)
 
 // CHECK-LABEL: @test_vrndnq_f16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.arm.mve.vrintn.v8f16(<8 x half> [[A:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.roundeven.v8f16(<8 x half> [[A:%.*]])
 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
 //
 float16x8_t test_vrndnq_f16(float16x8_t a)
@@ -162,7 +162,7 @@ float16x8_t test_vrndnq_f16(float16x8_t a)
 
 // CHECK-LABEL: @test_vrndnq_f32(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.arm.mve.vrintn.v4f32(<4 x float> [[A:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.roundeven.v4f32(<4 x float> [[A:%.*]])
 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
 //
 float32x4_t test_vrndnq_f32(float32x4_t a)
diff --git a/external/llvm-project/clang/test/CodeGen/arm64-former-microsoft-intrinsics-header-warning.c b/external/llvm-project/clang/test/CodeGen/arm64-former-microsoft-intrinsics-header-warning.c
new file mode 100644
index 000000000000..52fed49db4dd
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/arm64-former-microsoft-intrinsics-header-warning.c
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -triple arm64-windows -Wno-everything -Wimplicit-function-declaration -fms-compatibility -fsyntax-only -verify %s
+
+// RUN: %clang_cc1 -triple arm64-linux -Wno-everything -Wimplicit-function-declaration -fsyntax-only -verify %s
+
+// RUN: %clang_cc1 -triple arm64-darwin -Wno-everything -Wimplicit-function-declaration -fms-compatibility -fsyntax-only -verify %s
+
+void check__dmb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dmb(0);
+}
+
+void check__dsb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __dsb(0);
+}
+
+void check__isb(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __isb(0);
+}
+
+void check__yield(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __yield();
+}
+
+void check__wfe(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfe();
+}
+
+void check__wfi(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __wfi();
+}
+
+void check__sev(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sev();
+}
+
+void check__sevl(void) {
+  // expected-warning@+2{{call to undeclared library function}}
+  // expected-note@+1{{include the header <arm_acle.h> or explicitly provide a declaration for}}
+  __sevl();
+}
diff --git a/external/llvm-project/clang/test/CodeGen/arm64-former-microsoft-intrinsics.c b/external/llvm-project/clang/test/CodeGen/arm64-former-microsoft-intrinsics.c
new file mode 100644
index 000000000000..f1d9af2c6d25
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/arm64-former-microsoft-intrinsics.c
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -triple arm64-windows -Wno-implicit-function-declaration -fms-compatibility -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+
+// RUN: %clang_cc1 -triple arm64-linux -Werror -emit-llvm -o - %s 2>&1 \
+// RUN:    | FileCheck %s
+
+// RUN: %clang_cc1 -triple arm64-darwin -Wno-implicit-function-declaration -fms-compatibility -emit-llvm -o - %s \
+// RUN:    | FileCheck %s
+
+#include <arm_acle.h>
+
+void check__dmb(void) {
+  __dmb(0);
+}
+
+// CHECK: @llvm.aarch64.dmb(i32 0)
+
+void check__dsb(void) {
+  __dsb(0);
+}
+
+// CHECK: @llvm.aarch64.dsb(i32 0)
+
+void check__isb(void) {
+  __isb(0);
+}
+
+// CHECK: @llvm.aarch64.isb(i32 0)
+
+void check__yield(void) {
+  __yield();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 1)
+
+void check__wfe(void) {
+  __wfe();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 2)
+
+void check__wfi(void) {
+  __wfi();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 3)
+
+void check__sev(void) {
+  __sev();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 4)
+
+void check__sevl(void) {
+  __sevl();
+}
+
+// CHECK: @llvm.aarch64.hint(i32 5)
+
diff --git a/external/llvm-project/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/external/llvm-project/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 74c6f2f79e63..e18977a4559b 100644
--- a/external/llvm-project/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/external/llvm-project/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -35,62 +35,6 @@ __int64 test_InterlockedAdd64_constant(__int64 volatile *Addend) {
 // CHECK-MSVC: ret i64 %[[NEWVAL:[0-9]+]]
 // CHECK-LINUX: error: call to undeclared function '_InterlockedAdd64'
 
-void check__dmb(void) {
-  __dmb(0);
-}
-
-// CHECK-MSVC: @llvm.aarch64.dmb(i32 0)
-// CHECK-LINUX: error: call to undeclared function '__dmb'
-
-void check__dsb(void) {
-  __dsb(0);
-}
-
-// CHECK-MSVC: @llvm.aarch64.dsb(i32 0)
-// CHECK-LINUX: error: call to undeclared function '__dsb'
-
-void check__isb(void) {
-  __isb(0);
-}
-
-// CHECK-MSVC: @llvm.aarch64.isb(i32 0)
-// CHECK-LINUX: error: call to undeclared function '__isb'
-
-void check__yield(void) {
-  __yield();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 1)
-// CHECK-LINUX: error: call to undeclared function '__yield'
-
-void check__wfe(void) {
-  __wfe();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 2)
-// CHECK-LINUX: error: call to undeclared function '__wfe'
-
-void check__wfi(void) {
-  __wfi();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 3)
-// CHECK-LINUX: error: call to undeclared function '__wfi'
-
-void check__sev(void) {
-  __sev();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 4)
-// CHECK-LINUX: error: call to undeclared function '__sev'
-
-void check__sevl(void) {
-  __sevl();
-}
-
-// CHECK-MSVC: @llvm.aarch64.hint(i32 5)
-// CHECK-LINUX: error: call to undeclared function '__sevl'
-
 void check_ReadWriteBarrier(void) {
   _ReadWriteBarrier();
 }
diff --git a/external/llvm-project/clang/test/CodeGen/asan-global-ignorelist.test b/external/llvm-project/clang/test/CodeGen/asan-global-ignorelist.test
new file mode 100644
index 000000000000..2bb37c100c06
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/asan-global-ignorelist.test
@@ -0,0 +1,37 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+// RUN: %clang -target x86_64-unknown-linux-gnu -S -emit-llvm -fsanitize=address %t/test.c -o - | FileCheck %s --check-prefixes=SANITIZE
+
+// RUN: %clang -target x86_64-unknown-linux-gnu -S -emit-llvm -fsanitize=address -fsanitize-ignorelist=%t/order-0.ignorelist %t/test.c -o - | FileCheck %s --check-prefixes=IGNORE
+// RUN: %clang -target x86_64-unknown-linux-gnu -S -emit-llvm -fsanitize=address -fsanitize-ignorelist=%t/order-1.ignorelist %t/test.c -o - | FileCheck %s --check-prefixes=SANITIZE
+// RUN: %clang -target x86_64-unknown-linux-gnu -S -emit-llvm -fsanitize=address -fsanitize-ignorelist=%t/order-2.ignorelist %t/test.c -o - | FileCheck %s --check-prefixes=IGNORE
+// RUN: %clang -target x86_64-unknown-linux-gnu -S -emit-llvm -fsanitize=address -fsanitize-ignorelist=%t/order-3.ignorelist %t/test.c -o - | FileCheck %s --check-prefixes=SANITIZE
+
+// The same type can appear multiple times within an ignorelist. Any ``=sanitize`` type
+// entries enable sanitizer instrumentation, even if it was ignored by entries before.
+// If multiple entries match the source, then the latest entry takes the
+// precedence.
+
+//--- order-0.ignorelist
+global:global_array
+
+//--- order-1.ignorelist
+global:global_array
+global:global_array=sanitize
+
+//--- order-2.ignorelist
+global:*
+global:global_array=sanitize
+global:global_array
+
+//--- order-3.ignorelist
+global:*
+global:global_array=sanitize
+global:global*
+global:*array=sanitize
+
+//--- test.c
+// SANITIZE: @global_array ={{.*}} global {{.*}}, comdat, align {{.*}} 
+// IGNORE: @global_array ={{.*}} global {{.*}}, no_sanitize_address, align {{.*}}
+unsigned global_array[100] = {-1};
diff --git a/external/llvm-project/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c b/external/llvm-project/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c
index e1e2220f94d6..fcd4314249ff 100644
--- a/external/llvm-project/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c
+++ b/external/llvm-project/clang/test/CodeGen/attr-arm-sve-vector-bits-cast.c
@@ -62,10 +62,7 @@ fixed_bool_t from_svbool_t(svbool_t type) {
 
 // CHECK-LABEL: @lax_cast(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[SAVED_VALUE:%.*]] = alloca <16 x i32>, align 64
-// CHECK-NEXT:    [[TYPE:%.*]] = tail call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[TYPE_COERCE:%.*]], i64 0)
-// CHECK-NEXT:    store <16 x i32> [[TYPE]], ptr [[SAVED_VALUE]], align 64, !tbaa [[TBAA6:![0-9]+]]
-// CHECK-NEXT:    [[TMP0:%.*]] = load <vscale x 2 x i64>, ptr [[SAVED_VALUE]], align 64, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <vscale x 4 x i32> [[TYPE_COERCE:%.*]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
 svint64_t lax_cast(fixed_int32_t type) {
@@ -74,9 +71,9 @@ svint64_t lax_cast(fixed_int32_t type) {
 
 // CHECK-LABEL: @to_svint32_t__from_gnu_int32_t(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TYPE:%.*]] = load <16 x i32>, ptr [[TMP0:%.*]], align 16, !tbaa [[TBAA6]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TYPE]], i64 0)
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
+// CHECK-NEXT:    [[TYPE:%.*]] = load <16 x i32>, ptr [[TMP0:%.*]], align 16, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 svint32_t to_svint32_t__from_gnu_int32_t(gnu_int32_t type) {
   return type;
@@ -84,8 +81,8 @@ svint32_t to_svint32_t__from_gnu_int32_t(gnu_int32_t type) {
 
 // CHECK-LABEL: @from_svint32_t__to_gnu_int32_t(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[TYPE:%.*]], i64 0)
-// CHECK-NEXT:    store <16 x i32> [[CASTFIXEDSVE]], ptr [[AGG_RESULT:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[CAST_FIXED:%.*]] = tail call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[TYPE:%.*]], i64 0)
+// CHECK-NEXT:    store <16 x i32> [[CAST_FIXED]], ptr [[AGG_RESULT:%.*]], align 16, !tbaa [[TBAA2]]
 // CHECK-NEXT:    ret void
 //
 gnu_int32_t from_svint32_t__to_gnu_int32_t(svint32_t type) {
@@ -94,9 +91,9 @@ gnu_int32_t from_svint32_t__to_gnu_int32_t(svint32_t type) {
 
 // CHECK-LABEL: @to_fixed_int32_t__from_gnu_int32_t(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TYPE:%.*]] = load <16 x i32>, ptr [[TMP0:%.*]], align 16, !tbaa [[TBAA6]]
-// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TYPE]], i64 0)
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
+// CHECK-NEXT:    [[TYPE:%.*]] = load <16 x i32>, ptr [[TMP0:%.*]], align 16, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST_SCALABLE]]
 //
 fixed_int32_t to_fixed_int32_t__from_gnu_int32_t(gnu_int32_t type) {
   return type;
@@ -105,7 +102,7 @@ fixed_int32_t to_fixed_int32_t__from_gnu_int32_t(gnu_int32_t type) {
 // CHECK-LABEL: @from_fixed_int32_t__to_gnu_int32_t(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TYPE:%.*]] = tail call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[TYPE_COERCE:%.*]], i64 0)
-// CHECK-NEXT:    store <16 x i32> [[TYPE]], ptr [[AGG_RESULT:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-NEXT:    store <16 x i32> [[TYPE]], ptr [[AGG_RESULT:%.*]], align 16, !tbaa [[TBAA2]]
 // CHECK-NEXT:    ret void
 //
 gnu_int32_t from_fixed_int32_t__to_gnu_int32_t(fixed_int32_t type) {
diff --git a/external/llvm-project/clang/test/CodeGen/attr-availability-aligned-versions.c b/external/llvm-project/clang/test/CodeGen/attr-availability-aligned-versions.c
new file mode 100644
index 000000000000..5555fa8e7f69
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/attr-availability-aligned-versions.c
@@ -0,0 +1,28 @@
+/// This test verifies IR generated for APIs protected with availability annotations with a common versions.
+// RUN: %clang_cc1 -fvisibility=hidden "-triple" "arm64-apple-ios26.0" -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -fvisibility=hidden "-triple" "arm64-apple-tvos26" -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -fvisibility=hidden "-triple" "arm64-apple-watchos26" -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -fvisibility=hidden "-triple" "arm64-apple-ios18" -emit-llvm -o - %s | FileCheck -check-prefix=OLD %s
+
+__attribute__((availability(ios,introduced=19)))
+void f0(void);
+
+__attribute__((availability(ios,introduced=26)))
+void f1(void);
+
+__attribute__((availability(ios,introduced=27)))
+void f2(void);
+
+// OLD: declare extern_weak void @f0
+// OLD: declare extern_weak void @f1
+// OLD: declare extern_weak void @f2
+
+// CHECK:  declare void @f0
+// CHECK:  declare void @f1
+// CHECK:  declare extern_weak void @f2
+
+void test() {
+  f0();
+  f1();
+  f2();
+}
diff --git a/external/llvm-project/clang/test/CodeGen/attr-target-clones-aarch64.c b/external/llvm-project/clang/test/CodeGen/attr-target-clones-aarch64.c
index ac926f2329cc..57add8b8c8ab 100644
--- a/external/llvm-project/clang/test/CodeGen/attr-target-clones-aarch64.c
+++ b/external/llvm-project/clang/test/CodeGen/attr-target-clones-aarch64.c
@@ -6,15 +6,15 @@
 int __attribute__((target_clones("lse+aes", "sve2"))) ftc(void) { return 0; }
 int __attribute__((target_clones("sha2", "sha2+memtag", " default "))) ftc_def(void) { return 1; }
 int __attribute__((target_clones("sha2", "default"))) ftc_dup1(void) { return 2; }
-int __attribute__((target_clones("fp", "crc+dotprod"))) ftc_dup2(void) { return 3; }
-int __attribute__((target_clones("memtag", "bti"))) ftc_dup3(void) { return 4; }
+int __attribute__((target_clones("fp", "crc+dotprod", "default"))) ftc_dup2(void) { return 3; }
+int __attribute__((target_clones("memtag", "bti", "default"))) ftc_dup3(void) { return 4; }
 int foo() {
   return ftc() + ftc_def() + ftc_dup1() + ftc_dup2() + ftc_dup3();
 }
 
 inline int __attribute__((target_clones("rng+simd", "rcpc", "sve2-aes+wfxt"))) ftc_inline1(void) { return 1; }
 inline int __attribute__((target_clones("fp16", "fcma+sve2-bitperm", "default"))) ftc_inline2(void);
-inline int __attribute__((target_clones("bti", "sve+sb"))) ftc_inline3(void) { return 3; }
+inline int __attribute__((target_clones("bti", "sve+sb", "default"))) ftc_inline3(void) { return 3; }
 
 int __attribute__((target_clones("default"))) ftc_direct(void) { return 4; }
 
@@ -27,23 +27,19 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 
 //.
 // CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
-// CHECK: @ftc = weak_odr ifunc i32 (), ptr @ftc.resolver
 // CHECK: @ftc_def = weak_odr ifunc i32 (), ptr @ftc_def.resolver
 // CHECK: @ftc_dup1 = weak_odr ifunc i32 (), ptr @ftc_dup1.resolver
 // CHECK: @ftc_dup2 = weak_odr ifunc i32 (), ptr @ftc_dup2.resolver
 // CHECK: @ftc_dup3 = weak_odr ifunc i32 (), ptr @ftc_dup3.resolver
 // CHECK: @ftc_inline2 = weak_odr ifunc i32 (), ptr @ftc_inline2.resolver
-// CHECK: @ftc_inline1 = weak_odr ifunc i32 (), ptr @ftc_inline1.resolver
 // CHECK: @ftc_inline3 = weak_odr ifunc i32 (), ptr @ftc_inline3.resolver
 //.
 // CHECK-MTE-BTI: @__aarch64_cpu_features = external dso_local global { i64 }
-// CHECK-MTE-BTI: @ftc = weak_odr ifunc i32 (), ptr @ftc.resolver
 // CHECK-MTE-BTI: @ftc_def = weak_odr ifunc i32 (), ptr @ftc_def.resolver
 // CHECK-MTE-BTI: @ftc_dup1 = weak_odr ifunc i32 (), ptr @ftc_dup1.resolver
 // CHECK-MTE-BTI: @ftc_dup2 = weak_odr ifunc i32 (), ptr @ftc_dup2.resolver
 // CHECK-MTE-BTI: @ftc_dup3 = weak_odr ifunc i32 (), ptr @ftc_dup3.resolver
 // CHECK-MTE-BTI: @ftc_inline2 = weak_odr ifunc i32 (), ptr @ftc_inline2.resolver
-// CHECK-MTE-BTI: @ftc_inline1 = weak_odr ifunc i32 (), ptr @ftc_inline1.resolver
 // CHECK-MTE-BTI: @ftc_inline3 = weak_odr ifunc i32 (), ptr @ftc_inline3.resolver
 //.
 // CHECK: Function Attrs: noinline nounwind optnone
@@ -60,28 +56,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@ftc.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 69793284352
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 69793284352
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc._Msve2
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 33664
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 33664
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc._MaesMlse
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @ftc.default
-//
-//
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_def._Msha2
 // CHECK-SAME: () #[[ATTR2:[0-9]+]] {
@@ -96,26 +70,11 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@ftc_def.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 17592186049280
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17592186049280
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_def._MmemtagMsha2
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 4864
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4864
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc_def._Msha2
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @ftc_def.default
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@ftc_def.default
+// CHECK-SAME: () #[[ATTR4:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
@@ -125,95 +84,58 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@ftc_dup1.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 4864
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4864
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_dup1._Msha2
-// CHECK:       resolver_else:
-// CHECK-NEXT:    ret ptr @ftc_dup1.default
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@ftc_dup1.default
+// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_dup2._Mfp
-// CHECK-SAME: () #[[ATTR4:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR5:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_dup2._McrcMdotprod
-// CHECK-SAME: () #[[ATTR5:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR6:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@ftc_dup2.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1808
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1808
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_dup2._McrcMdotprod
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 256
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 256
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc_dup2._Mfp
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @ftc_dup2.default
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@ftc_dup2.default
+// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_dup3._Mmemtag
-// CHECK-SAME: () #[[ATTR6:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR7:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_dup3._Mbti
-// CHECK-SAME: () #[[ATTR7:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR8:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
 //
 //
-// CHECK-LABEL: define {{[^@]+}}@ftc_dup3.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1125899906842624
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1125899906842624
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_dup3._Mbti
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 17592186044416
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17592186044416
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc_dup3._Mmemtag
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    ret ptr @ftc_dup3.default
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@ftc_dup3.default
+// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 4
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@foo
-// CHECK-SAME: () #[[ATTR8:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR9:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[CALL:%.*]] = call i32 @ftc()
 // CHECK-NEXT:    [[CALL1:%.*]] = call i32 @ftc_def()
@@ -229,14 +151,14 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_direct
-// CHECK-SAME: () #[[ATTR8]] {
+// CHECK-SAME: () #[[ATTR9]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@main
-// CHECK-SAME: () #[[ATTR8]] {
+// CHECK-SAME: () #[[ATTR9]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store i32 0, ptr [[RETVAL]], align 4
@@ -250,58 +172,103 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 [[ADD5]]
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@ftc.default
-// CHECK-SAME: () #[[ATTR9:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@ftc_def.default
-// CHECK-SAME: () #[[ATTR9]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
+// CHECK-LABEL: define {{[^@]+}}@ftc_def.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 17592186049280
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17592186049280
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @ftc_def._MmemtagMsha2
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 4864
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4864
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @ftc_def._Msha2
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @ftc_def.default
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@ftc_dup1.default
-// CHECK-SAME: () #[[ATTR9]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
+// CHECK-LABEL: define {{[^@]+}}@ftc_dup1.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 4864
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4864
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @ftc_dup1._Msha2
+// CHECK:       resolver_else:
+// CHECK-NEXT:    ret ptr @ftc_dup1.default
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@ftc_dup2.default
-// CHECK-SAME: () #[[ATTR9]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 3
+// CHECK-LABEL: define {{[^@]+}}@ftc_dup2.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1808
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1808
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @ftc_dup2._McrcMdotprod
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 256
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 256
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @ftc_dup2._Mfp
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @ftc_dup2.default
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@ftc_dup3.default
-// CHECK-SAME: () #[[ATTR9]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 4
+// CHECK-LABEL: define {{[^@]+}}@ftc_dup3.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1125899906842624
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1125899906842624
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @ftc_dup3._Mbti
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 17592186044416
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17592186044416
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @ftc_dup3._Mmemtag
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @ftc_dup3.default
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline2._Mfp16
-// CHECK-SAME: () #[[ATTR10:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR12:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline2._MfcmaMsve2-bitperm
-// CHECK-SAME: () #[[ATTR11:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR13:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline2.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
@@ -330,79 +297,42 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline1._MrngMsimd
-// CHECK-SAME: () #[[ATTR12:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR14:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline1._Mrcpc
-// CHECK-SAME: () #[[ATTR13:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR15:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline1._Msve2-aesMwfxt
-// CHECK-SAME: () #[[ATTR14:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR16:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@ftc_inline1.default
-// CHECK-SAME: () #[[ATTR9]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define {{[^@]+}}@ftc_inline1.resolver() comdat {
-// CHECK-NEXT:  resolver_entry:
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 18014743180706560
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014743180706560
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK:       resolver_return:
-// CHECK-NEXT:    ret ptr @ftc_inline1._Msve2-aesMwfxt
-// CHECK:       resolver_else:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 4194304
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4194304
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK:       resolver_return1:
-// CHECK-NEXT:    ret ptr @ftc_inline1._Mrcpc
-// CHECK:       resolver_else2:
-// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 769
-// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 769
-// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
-// CHECK:       resolver_return3:
-// CHECK-NEXT:    ret ptr @ftc_inline1._MrngMsimd
-// CHECK:       resolver_else4:
-// CHECK-NEXT:    ret ptr @ftc_inline1.default
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline3._Mbti
-// CHECK-SAME: () #[[ATTR7]] {
+// CHECK-SAME: () #[[ATTR8]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline3._MsbMsve
-// CHECK-SAME: () #[[ATTR15:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR17:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@ftc_inline3.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
@@ -517,28 +447,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI-NEXT:    ret i32 0
 //
 //
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc.resolver() comdat {
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 69793284352
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 69793284352
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc._Msve2
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 33664
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 33664
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc._MaesMlse
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc.default
-//
-//
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_def._Msha2
 // CHECK-MTE-BTI-SAME: () #[[ATTR2:[0-9]+]] {
@@ -553,26 +461,11 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI-NEXT:    ret i32 1
 //
 //
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_def.resolver() comdat {
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 17592186049280
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17592186049280
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_def._MmemtagMsha2
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 4864
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4864
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_def._Msha2
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_def.default
+// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_def.default
+// CHECK-MTE-BTI-SAME: () #[[ATTR4:[0-9]+]] {
+// CHECK-MTE-BTI-NEXT:  entry:
+// CHECK-MTE-BTI-NEXT:    ret i32 1
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
@@ -582,95 +475,58 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI-NEXT:    ret i32 2
 //
 //
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup1.resolver() comdat {
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 4864
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4864
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup1._Msha2
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup1.default
+// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup1.default
+// CHECK-MTE-BTI-SAME: () #[[ATTR4]] {
+// CHECK-MTE-BTI-NEXT:  entry:
+// CHECK-MTE-BTI-NEXT:    ret i32 2
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup2._Mfp
-// CHECK-MTE-BTI-SAME: () #[[ATTR4:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR5:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 3
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup2._McrcMdotprod
-// CHECK-MTE-BTI-SAME: () #[[ATTR5:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR6:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 3
 //
 //
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup2.resolver() comdat {
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1808
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1808
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2._McrcMdotprod
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 256
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 256
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2._Mfp
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2.default
+// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup2.default
+// CHECK-MTE-BTI-SAME: () #[[ATTR4]] {
+// CHECK-MTE-BTI-NEXT:  entry:
+// CHECK-MTE-BTI-NEXT:    ret i32 3
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup3._Mmemtag
-// CHECK-MTE-BTI-SAME: () #[[ATTR6:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR7:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 4
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup3._Mbti
-// CHECK-MTE-BTI-SAME: () #[[ATTR7:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR8:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 4
 //
 //
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup3.resolver() comdat {
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1125899906842624
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1125899906842624
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup3._Mbti
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 17592186044416
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17592186044416
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup3._Mmemtag
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup3.default
+// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup3.default
+// CHECK-MTE-BTI-SAME: () #[[ATTR4]] {
+// CHECK-MTE-BTI-NEXT:  entry:
+// CHECK-MTE-BTI-NEXT:    ret i32 4
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@foo
-// CHECK-MTE-BTI-SAME: () #[[ATTR8:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR9:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    [[CALL:%.*]] = call i32 @ftc()
 // CHECK-MTE-BTI-NEXT:    [[CALL1:%.*]] = call i32 @ftc_def()
@@ -686,14 +542,14 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_direct
-// CHECK-MTE-BTI-SAME: () #[[ATTR8]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 4
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@main
-// CHECK-MTE-BTI-SAME: () #[[ATTR8]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
 // CHECK-MTE-BTI-NEXT:    store i32 0, ptr [[RETVAL]], align 4
@@ -707,58 +563,103 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-MTE-BTI-NEXT:    ret i32 [[ADD5]]
 //
 //
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9:[0-9]+]] {
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 0
-//
-//
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_def.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 1
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_def.resolver() comdat {
+// CHECK-MTE-BTI-NEXT:  resolver_entry:
+// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 17592186049280
+// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17592186049280
+// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK-MTE-BTI:       resolver_return:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_def._MmemtagMsha2
+// CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 4864
+// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4864
+// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-MTE-BTI:       resolver_return1:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_def._Msha2
+// CHECK-MTE-BTI:       resolver_else2:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_def.default
 //
 //
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup1.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 2
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup1.resolver() comdat {
+// CHECK-MTE-BTI-NEXT:  resolver_entry:
+// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 4864
+// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4864
+// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK-MTE-BTI:       resolver_return:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup1._Msha2
+// CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup1.default
 //
 //
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup2.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 3
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup2.resolver() comdat {
+// CHECK-MTE-BTI-NEXT:  resolver_entry:
+// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1808
+// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1808
+// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK-MTE-BTI:       resolver_return:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2._McrcMdotprod
+// CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 256
+// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 256
+// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-MTE-BTI:       resolver_return1:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2._Mfp
+// CHECK-MTE-BTI:       resolver_else2:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup2.default
 //
 //
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup3.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 4
+// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_dup3.resolver() comdat {
+// CHECK-MTE-BTI-NEXT:  resolver_entry:
+// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1125899906842624
+// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1125899906842624
+// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK-MTE-BTI:       resolver_return:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup3._Mbti
+// CHECK-MTE-BTI:       resolver_else:
+// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 17592186044416
+// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17592186044416
+// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-MTE-BTI:       resolver_return1:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup3._Mmemtag
+// CHECK-MTE-BTI:       resolver_else2:
+// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_dup3.default
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline2._Mfp16
-// CHECK-MTE-BTI-SAME: () #[[ATTR10:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR12:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 2
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline2._MfcmaMsve2-bitperm
-// CHECK-MTE-BTI-SAME: () #[[ATTR11:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR13:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 2
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline2.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR4]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 2
 //
@@ -787,79 +688,42 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline1._MrngMsimd
-// CHECK-MTE-BTI-SAME: () #[[ATTR12:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR14:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 1
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline1._Mrcpc
-// CHECK-MTE-BTI-SAME: () #[[ATTR13:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR15:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 1
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline1._Msve2-aesMwfxt
-// CHECK-MTE-BTI-SAME: () #[[ATTR14:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR16:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 1
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline1.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
-// CHECK-MTE-BTI-NEXT:  entry:
-// CHECK-MTE-BTI-NEXT:    ret i32 1
-//
-//
-// CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline1.resolver() comdat {
-// CHECK-MTE-BTI-NEXT:  resolver_entry:
-// CHECK-MTE-BTI-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-MTE-BTI-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 18014743180706560
-// CHECK-MTE-BTI-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014743180706560
-// CHECK-MTE-BTI-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK-MTE-BTI:       resolver_return:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._Msve2-aesMwfxt
-// CHECK-MTE-BTI:       resolver_else:
-// CHECK-MTE-BTI-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 4194304
-// CHECK-MTE-BTI-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4194304
-// CHECK-MTE-BTI-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK-MTE-BTI:       resolver_return1:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._Mrcpc
-// CHECK-MTE-BTI:       resolver_else2:
-// CHECK-MTE-BTI-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-MTE-BTI-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 769
-// CHECK-MTE-BTI-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 769
-// CHECK-MTE-BTI-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-MTE-BTI-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
-// CHECK-MTE-BTI:       resolver_return3:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1._MrngMsimd
-// CHECK-MTE-BTI:       resolver_else4:
-// CHECK-MTE-BTI-NEXT:    ret ptr @ftc_inline1.default
-//
-//
-// CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline3._Mbti
-// CHECK-MTE-BTI-SAME: () #[[ATTR7]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR8]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 3
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline3._MsbMsve
-// CHECK-MTE-BTI-SAME: () #[[ATTR15:[0-9]+]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR17:[0-9]+]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 3
 //
 //
 // CHECK-MTE-BTI: Function Attrs: noinline nounwind optnone
 // CHECK-MTE-BTI-LABEL: define {{[^@]+}}@ftc_inline3.default
-// CHECK-MTE-BTI-SAME: () #[[ATTR9]] {
+// CHECK-MTE-BTI-SAME: () #[[ATTR4]] {
 // CHECK-MTE-BTI-NEXT:  entry:
 // CHECK-MTE-BTI-NEXT:    ret i32 3
 //
diff --git a/external/llvm-project/clang/test/CodeGen/attr-target-version-riscv.c b/external/llvm-project/clang/test/CodeGen/attr-target-version-riscv.c
index 3eff52a7c771..fbead04caf45 100644
--- a/external/llvm-project/clang/test/CodeGen/attr-target-version-riscv.c
+++ b/external/llvm-project/clang/test/CodeGen/attr-target-version-riscv.c
@@ -49,163 +49,6 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: define dso_local signext i32 @foo1.default(
-// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo2._zbb(
-// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo2._m(
-// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo2.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo3._c_zbb(
-// CHECK-SAME: ) #[[ATTR4:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo3._m(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo3.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo4._zba(
-// CHECK-SAME: ) #[[ATTR5:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo4._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo4._zba_zbb(
-// CHECK-SAME: ) #[[ATTR6:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo4.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo5._zba(
-// CHECK-SAME: ) #[[ATTR5]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo5._zba_zbb(
-// CHECK-SAME: ) #[[ATTR6]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo5._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo5.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo6._zba(
-// CHECK-SAME: ) #[[ATTR5]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo6._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo6._zba_zbb(
-// CHECK-SAME: ) #[[ATTR6]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo6.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo7._zba(
-// CHECK-SAME: ) #[[ATTR5]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo7._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo7._zba_zbb(
-// CHECK-SAME: ) #[[ATTR6]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @foo7.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local signext i32 @bar(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 @foo1()
-// CHECK-NEXT:    [[CALL1:%.*]] = call signext i32 @foo2()
-// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
-// CHECK-NEXT:    [[CALL2:%.*]] = call signext i32 @foo3()
-// CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[ADD]], [[CALL2]]
-// CHECK-NEXT:    [[CALL4:%.*]] = call signext i32 @foo4()
-// CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[ADD3]], [[CALL4]]
-// CHECK-NEXT:    [[CALL6:%.*]] = call signext i32 @foo5()
-// CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 [[ADD5]], [[CALL6]]
-// CHECK-NEXT:    [[CALL8:%.*]] = call signext i32 @foo6()
-// CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[ADD7]], [[CALL8]]
-// CHECK-NEXT:    [[CALL10:%.*]] = call signext i32 @foo7()
-// CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD9]], [[CALL10]]
-// CHECK-NEXT:    ret i32 [[ADD11]]
-//
-//
 // CHECK-LABEL: define weak_odr ptr @foo1.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -219,6 +62,18 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK-NEXT:    ret ptr @foo1.default
 //
 //
+// CHECK-LABEL: define dso_local signext i32 @foo1.default(
+// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo2._zbb(
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @foo2.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -239,6 +94,24 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK-NEXT:    ret ptr @foo2.default
 //
 //
+// CHECK-LABEL: define dso_local signext i32 @foo2._m(
+// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo2.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo3._c_zbb(
+// CHECK-SAME: ) #[[ATTR4:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @foo3.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -259,6 +132,24 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK-NEXT:    ret ptr @foo3.default
 //
 //
+// CHECK-LABEL: define dso_local signext i32 @foo3._m(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo3.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo4._zba(
+// CHECK-SAME: ) #[[ATTR5:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @foo4.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -286,6 +177,30 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK-NEXT:    ret ptr @foo4.default
 //
 //
+// CHECK-LABEL: define dso_local signext i32 @foo4._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo4._zba_zbb(
+// CHECK-SAME: ) #[[ATTR6:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo4.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo5._zba(
+// CHECK-SAME: ) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @foo5.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -313,6 +228,30 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK-NEXT:    ret ptr @foo5.default
 //
 //
+// CHECK-LABEL: define dso_local signext i32 @foo5._zba_zbb(
+// CHECK-SAME: ) #[[ATTR6]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo5._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo5.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo6._zba(
+// CHECK-SAME: ) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @foo6.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -340,6 +279,30 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK-NEXT:    ret ptr @foo6.default
 //
 //
+// CHECK-LABEL: define dso_local signext i32 @foo6._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo6._zba_zbb(
+// CHECK-SAME: ) #[[ATTR6]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo6.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo7._zba(
+// CHECK-SAME: ) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @foo7.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -366,6 +329,43 @@ int bar() { return foo1() + foo2() + foo3() + foo4() + foo5() + foo6() + foo7();
 // CHECK:       resolver_else4:
 // CHECK-NEXT:    ret ptr @foo7.default
 //
+//
+// CHECK-LABEL: define dso_local signext i32 @foo7._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo7._zba_zbb(
+// CHECK-SAME: ) #[[ATTR6]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @foo7.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local signext i32 @bar(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call signext i32 @foo1()
+// CHECK-NEXT:    [[CALL1:%.*]] = call signext i32 @foo2()
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
+// CHECK-NEXT:    [[CALL2:%.*]] = call signext i32 @foo3()
+// CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[ADD]], [[CALL2]]
+// CHECK-NEXT:    [[CALL4:%.*]] = call signext i32 @foo4()
+// CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[ADD3]], [[CALL4]]
+// CHECK-NEXT:    [[CALL6:%.*]] = call signext i32 @foo5()
+// CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 [[ADD5]], [[CALL6]]
+// CHECK-NEXT:    [[CALL8:%.*]] = call signext i32 @foo6()
+// CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[ADD7]], [[CALL8]]
+// CHECK-NEXT:    [[CALL10:%.*]] = call signext i32 @foo7()
+// CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD9]], [[CALL10]]
+// CHECK-NEXT:    ret i32 [[ADD11]]
+//
 //.
 // CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+64bit,+d,+f,+i,+v,+zicsr,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b" }
 // CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+64bit,+i" }
diff --git a/external/llvm-project/clang/test/CodeGen/builtins-arm-microsoft.c b/external/llvm-project/clang/test/CodeGen/builtins-arm-microsoft.c
index 841a8a500d53..b2681197e33e 100644
--- a/external/llvm-project/clang/test/CodeGen/builtins-arm-microsoft.c
+++ b/external/llvm-project/clang/test/CodeGen/builtins-arm-microsoft.c
@@ -4,38 +4,40 @@
 // RUN:     | FileCheck %s -check-prefix CHECK-EABI
 // REQUIRES: arm-registered-target
 
+#include <arm_acle.h>
+
 void test_yield_intrinsic() {
   __yield();
 }
 
 // CHECK-MSVC: call void @llvm.arm.hint(i32 1)
-// CHECK-EABI-NOT: call void @llvm.arm.hint(i32 1)
+// CHECK-EABI: call void @llvm.arm.hint(i32 1)
 
 void wfe() {
   __wfe();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 2)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 2)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 2)
 
 void wfi() {
   __wfi();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 3)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 3)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 3)
 
 void sev() {
   __sev();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 4)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 4)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 4)
 
 void sevl() {
   __sevl();
 }
 
 // CHECK-MSVC: call {{.*}} @llvm.arm.hint(i32 5)
-// CHECK-EABI-NOT: call {{.*}} @llvm.arm.hint(i32 5)
+// CHECK-EABI: call {{.*}} @llvm.arm.hint(i32 5)
 
diff --git a/external/llvm-project/clang/test/CodeGen/builtins-nvptx.c b/external/llvm-project/clang/test/CodeGen/builtins-nvptx.c
index cef529163bb3..f994adb14e45 100644
--- a/external/llvm-project/clang/test/CodeGen/builtins-nvptx.c
+++ b/external/llvm-project/clang/test/CodeGen/builtins-nvptx.c
@@ -883,6 +883,13 @@ __device__ void nvvm_vote(int pred) {
   // CHECK: ret void
 }
 
+// CHECK-LABEL: nvvm_pm_event_mask
+__device__ void nvvm_pm_event_mask() {
+  // CHECK: call void @llvm.nvvm.pm.event.mask(i16 255)
+  __nvvm_pm_event_mask(255);
+  // CHECK: ret void
+}
+
 // CHECK-LABEL: nvvm_nanosleep
 __device__ void nvvm_nanosleep(int d) {
 #if __CUDA_ARCH__ >= 700
diff --git a/external/llvm-project/clang/test/CodeGen/cfi-unchecked-callee-attribute-member-function.cpp b/external/llvm-project/clang/test/CodeGen/cfi-unchecked-callee-attribute-member-function.cpp
new file mode 100644
index 000000000000..637b5201ad6b
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/cfi-unchecked-callee-attribute-member-function.cpp
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=cfi-mfcall -o - %s -fvisibility=hidden | FileCheck %s
+
+#define CFI_UNCHECKED_CALLEE __attribute__((cfi_unchecked_callee))
+
+class A {};
+
+// CHECK-LABEL: _Z14MemberFuncCallP1AMS_FvvE
+void MemberFuncCall(A *s, void (A::*p)()) {
+  // CHECK:      memptr.virtual:
+  // CHECK-NEXT:   [[VTABLE:%.*]] = load ptr, ptr {{.*}}, align 8
+  // CHECK-NEXT:   [[OFFSET:%.*]] = sub i64 %memptr.ptr, 1
+  // CHECK-NEXT:   [[FUNC:%.*]] = getelementptr i8, ptr [[VTABLE]], i64 [[OFFSET]]
+  // CHECK-NEXT:   [[VALID:%.*]] = call i1 @llvm.type.test(ptr [[FUNC]], metadata !"_ZTSM1AFvvE.virtual")
+  // CHECK-NEXT:   [[FUNC:%.*]] = getelementptr i8, ptr [[VTABLE]], i64 [[OFFSET]]
+  // CHECK-NEXT:   %memptr.virtualfn = load ptr, ptr [[FUNC]], align 8
+  // CHECK-NEXT:   {{.*}}= call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"all-vtables")
+  // CHECK-NEXT:   br i1 [[VALID]], label %[[CONT:.*]], label %handler.cfi_check_fail{{.*}}
+
+  // CHECK:      [[CONT]]:
+  // CHECK-NEXT:   br label %memptr.end
+
+  // CHECK:      memptr.nonvirtual:
+  // CHECK-NEXT:   %memptr.nonvirtualfn = inttoptr i64 %memptr.ptr to ptr
+  // CHECK-NEXT:   [[VALID:%.*]] = call i1 @llvm.type.test(ptr %memptr.nonvirtualfn, metadata !"_ZTSM1AFvvE")
+  // CHECK-NEXT:   [[VALID2:%.*]] = or i1 false, [[VALID]]
+  // CHECK-NEXT:   br i1 [[VALID2]], label %[[CONT2:.*]], label %handler.cfi_check_fail{{.*}}
+
+  // CHECK:      [[CONT2]]:
+  // CHECK-NEXT:   br label %memptr.end
+
+  // CHECK:      memptr.end:
+  // CHECK-NEXT:   {{.*}} = phi ptr [ %memptr.virtualfn, %[[CONT]] ], [ %memptr.nonvirtualfn, %[[CONT2]] ]
+  (s->*p)();
+}
+
+// CHECK-LABEL: _Z19MemberFuncCallNoCFIP1AMS_FvvE
+// CHECK-NOT: llvm.type.test
+void MemberFuncCallNoCFI(A *s, void (CFI_UNCHECKED_CALLEE A::*p)()) {
+  // CHECK:      memptr.virtual:
+  // CHECK-NEXT:   [[VTABLE:%.*]] = load ptr, ptr {{.*}}, align 8
+  // CHECK-NEXT:   [[OFFSET:%.*]] = sub i64 %memptr.ptr, 1
+  // CHECK-NEXT:   [[FUNC:%.*]] = getelementptr i8, ptr [[VTABLE]], i64 [[OFFSET]]
+  // CHECK-NEXT:   %memptr.virtualfn = load ptr, ptr [[FUNC]], align 8
+  // CHECK-NEXT:   br label %memptr.end
+
+  // CHECK:      memptr.nonvirtual:
+  // CHECK-NEXT:   %memptr.nonvirtualfn = inttoptr i64 %memptr.ptr to ptr
+  // CHECK-NEXT:   br label %memptr.end
+
+  // CHECK:      memptr.end:
+  // CHECK-NEXT:   {{.*}} = phi ptr [ %memptr.virtualfn, %memptr.virtual ], [ %memptr.nonvirtualfn, %memptr.nonvirtual ]
+  (s->*p)();
+}
diff --git a/external/llvm-project/clang/test/CodeGen/cfi-unchecked-callee-attribute.cpp b/external/llvm-project/clang/test/CodeGen/cfi-unchecked-callee-attribute.cpp
new file mode 100644
index 000000000000..feb7c9c30a21
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/cfi-unchecked-callee-attribute.cpp
@@ -0,0 +1,77 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=cfi-icall -o - %s | FileCheck %s
+
+#define CFI_UNCHECKED_CALLEE __attribute__((cfi_unchecked_callee))
+
+void unchecked(void) CFI_UNCHECKED_CALLEE {}
+
+/// All references to unchecked function with `cfi_unchecked_callee` should have the `cfi_unchecked_callee` wrapper.
+// CHECK: @checked = global ptr no_cfi @_Z9uncheckedv
+void (*checked)(void) = unchecked;
+
+// CHECK: @unchecked2 = global ptr no_cfi @_Z9uncheckedv
+void (CFI_UNCHECKED_CALLEE *unchecked2)(void) = unchecked;
+
+// CHECK: @checked2 = global ptr no_cfi @_Z9uncheckedv
+constexpr void (CFI_UNCHECKED_CALLEE *unchecked_constexpr)(void) = unchecked;
+void (*checked2)(void) = unchecked_constexpr;
+
+/// Note we still reference the `no_cfi` function rather than the jump table entry.
+/// The explicit cast will only silence the warning.
+// CHECK: @checked_explicit_cast = global ptr no_cfi @_Z9uncheckedv
+void (*checked_explicit_cast)(void) = (void (*)(void))unchecked;
+
+// CHECK: @checked_array = global [3 x ptr] [ptr no_cfi @_Z9uncheckedv, ptr no_cfi @_Z9uncheckedv, ptr no_cfi @_Z9uncheckedv]
+void (*checked_array[])(void) = {
+  unchecked,
+  (void (*)(void))unchecked,
+  reinterpret_cast<void (*)(void)>(unchecked),
+};
+
+void func_accepting_checked(void (*p)(void)) {}
+
+// CHECK-LABEL: _Z9InvokeCFIv
+void InvokeCFI() {
+  // CHECK: %0 = load ptr, ptr @checked, align 8
+  // CHECK: %1 = call i1 @llvm.type.test(ptr %0, metadata !"_ZTSFvvE")
+  checked();
+}
+
+// CHECK-LABEL: _Z11InvokeNoCFIv
+void InvokeNoCFI() {
+  // CHECK:  %0 = load ptr, ptr @unchecked2, align 8
+  // CHECK:  call void %0()
+  unchecked2();
+}
+
+struct A {
+  void unchecked_method() CFI_UNCHECKED_CALLEE {}
+  virtual void unchecked_virtual_method() CFI_UNCHECKED_CALLEE {}
+  static void unchecked_static_method() CFI_UNCHECKED_CALLEE {}
+  int unchecked_const_method() const CFI_UNCHECKED_CALLEE { return 0; }
+  int unchecked_const_method_int_arg(int n) const CFI_UNCHECKED_CALLEE { return 0; }
+};
+
+void h(void) {
+  // CHECK: store ptr no_cfi @_Z9uncheckedv, ptr %unchecked_local
+  void (*unchecked_local)(void) = unchecked;
+
+  // CHECK: call void @_Z22func_accepting_checkedPFvvE(ptr noundef no_cfi @_Z9uncheckedv)
+  func_accepting_checked(unchecked);
+
+  // CHECK:      [[B:%.*]] = load ptr, ptr @checked
+  // CHECK-NEXT: call void @_Z22func_accepting_checkedPFvvE(ptr noundef [[B]]) 
+  func_accepting_checked(checked);
+
+  // CHECK: store { i64, i64 } { i64 ptrtoint (ptr no_cfi @_ZN1A16unchecked_methodEv to i64), i64 0 }, ptr %A1
+  auto A1 = &A::unchecked_method;
+  /// Storing unchecked virtual function pointer stores an offset instead. This is part of the
+  /// normal Itanium C++ ABI, but let's make sure we don't change anything.
+  // CHECK: store { i64, i64 } { i64 1, i64 0 }, ptr %A2
+  auto A2 = &A::unchecked_virtual_method;
+  // CHECK: store ptr no_cfi @_ZN1A23unchecked_static_methodEv, ptr %A3
+  auto A3 = &A::unchecked_static_method;
+  // CHECK: store { i64, i64 } { i64 ptrtoint (ptr no_cfi @_ZNK1A22unchecked_const_methodEv to i64), i64 0 }, ptr %A4
+  auto A4 = (int(CFI_UNCHECKED_CALLEE A::*)() const)(&A::unchecked_const_method);
+  // CHECK: store { i64, i64 } { i64 ptrtoint (ptr no_cfi @_ZNK1A30unchecked_const_method_int_argEi to i64), i64 0 }, ptr %A5
+  auto A5 = (int(CFI_UNCHECKED_CALLEE A::*)(int) const)(&A::unchecked_const_method_int_arg);
+}
diff --git a/external/llvm-project/clang/test/CodeGen/debug-info-version-coff.c b/external/llvm-project/clang/test/CodeGen/debug-info-version-coff.c
new file mode 100644
index 000000000000..6497a5829236
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/debug-info-version-coff.c
@@ -0,0 +1,8 @@
+// REQUIRES: x86-registered-target
+// RUN: %clang --target=x86_64-windows -g -S -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang --target=x86_64-windows -S -emit-llvm -o - %s | FileCheck %s
+int main (void) {
+  return 0;
+}
+
+// CHECK:  i32 2, !"Debug Info Version", i32 3}
diff --git a/external/llvm-project/clang/test/CodeGen/debug-info-version.c b/external/llvm-project/clang/test/CodeGen/debug-info-version.c
index fa7e20e7f527..c7c2bb95017a 100644
--- a/external/llvm-project/clang/test/CodeGen/debug-info-version.c
+++ b/external/llvm-project/clang/test/CodeGen/debug-info-version.c
@@ -1,3 +1,4 @@
+// REQUIRES: !system-windows
 // RUN: %clang -g -S -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang -S -emit-llvm -o - %s | FileCheck %s --check-prefix=NO_DEBUG
 int main (void) {
diff --git a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-backend-skip.ll b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/backend-skip.ll
similarity index 87%
rename from external/llvm-project/clang/test/CodeGen/thinlto-distributed-backend-skip.ll
rename to external/llvm-project/clang/test/CodeGen/distributed-thin-lto/backend-skip.ll
index d7b8225ee269..62aa8aa8e7dc 100644
--- a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-backend-skip.ll
+++ b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/backend-skip.ll
@@ -6,7 +6,7 @@
 ; RUN: opt -thinlto-bc -o %t.o %s
 
 ; RUN: %clang_cc1 -triple x86_64-grtev4-linux-gnu \
-; RUN:   -fthinlto-index=%S/Inputs/thinlto-distributed-backend-skip.bc \
+; RUN:   -fthinlto-index=%S/../Inputs/thinlto-distributed-backend-skip.bc \
 ; RUN:   -emit-llvm -o - -x ir %t.o | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/external/llvm-project/clang/test/CodeGen/thinlto-distributed.ll b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/basic.ll
similarity index 100%
rename from external/llvm-project/clang/test/CodeGen/thinlto-distributed.ll
rename to external/llvm-project/clang/test/CodeGen/distributed-thin-lto/basic.ll
diff --git a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-cfi-devirt.ll b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cfi-devirt.ll
similarity index 94%
rename from external/llvm-project/clang/test/CodeGen/thinlto-distributed-cfi-devirt.ll
rename to external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cfi-devirt.ll
index 64a1cfdc0f4c..a5792eee8abf 100644
--- a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-cfi-devirt.ll
+++ b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cfi-devirt.ll
@@ -30,10 +30,10 @@
 ; CHECK: blob data = '_ZTS1A_ZN1A1nEi'
 ; CHECK-LABEL: </STRTAB_BLOCK
 
-; RUN: llvm-dis  %t.o.thinlto.bc -o - | FileCheck %s --check-prefix=CHECK-DIS
+; RUN: llvm-dis %t.o.thinlto.bc -o - | FileCheck %s --check-prefix=CHECK-DIS
 ; Round trip it through llvm-as
-; RUN: llvm-dis  %t.o.thinlto.bc -o - | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-DIS
-; CHECK-DIS: ^0 = module: (path: "{{.*}}thinlto-distributed-cfi-devirt.ll.tmp.o", hash: ({{.*}}, {{.*}}, {{.*}}, {{.*}}, {{.*}}))
+; RUN: llvm-dis %t.o.thinlto.bc -o - | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-DIS
+; CHECK-DIS: ^0 = module: (path: "{{.*}}cfi-devirt.ll.tmp.o", hash: ({{.*}}, {{.*}}, {{.*}}, {{.*}}, {{.*}}))
 ; CHECK-DIS: ^1 = gv: (guid: 8346051122425466633, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0, importType: definition), insts: 18, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), typeIdInfo: (typeTests: (^2), typeCheckedLoadVCalls: (vFuncId: (^2, offset: 8), vFuncId: (^2, offset: 0))))))
 ; CHECK-DIS: ^2 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi"))))) ; guid = 7004155349499253778
 
diff --git a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-cfi.ll b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cfi.ll
similarity index 95%
rename from external/llvm-project/clang/test/CodeGen/thinlto-distributed-cfi.ll
rename to external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cfi.ll
index 6023ba8f32df..696f2dc4ea63 100644
--- a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-cfi.ll
+++ b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cfi.ll
@@ -23,7 +23,7 @@
 ; RUN: llvm-dis %t.o.thinlto.bc -o - | FileCheck %s --check-prefix=CHECK-DIS
 ; Round trip it through llvm-as
 ; RUN: llvm-dis %t.o.thinlto.bc -o - | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-DIS
-; CHECK-DIS: ^0 = module: (path: "{{.*}}thinlto-distributed-cfi.ll.tmp.o", hash: ({{.*}}, {{.*}}, {{.*}}, {{.*}}, {{.*}}))
+; CHECK-DIS: ^0 = module: (path: "{{.*}}cfi.ll.tmp.o", hash: ({{.*}}, {{.*}}, {{.*}}, {{.*}}, {{.*}}))
 ; CHECK-DIS: ^1 = gv: (guid: 8346051122425466633, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0, importType: definition), insts: 7, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), typeIdInfo: (typeTests: (^2)))))
 ; CHECK-DIS: ^2 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: single, sizeM1BitWidth: 0))) ; guid = 7004155349499253778
 
diff --git a/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cs-irpgo.c b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cs-irpgo.c
new file mode 100644
index 000000000000..d4c2e5d4fe6e
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/cs-irpgo.c
@@ -0,0 +1,8 @@
+// RUN: mkdir -p %t
+// RUN: %clang -target x86_64-apple-darwin -fcs-profile-generate -flto=thin -c -o %t/main.bc %s
+// RUN: llvm-lto2 run --thinlto-distributed-indexes %t/main.bc -o %t/index -r=%t/main.bc,_main,px
+// RUN: %clang -target x86_64-apple-darwin -fcs-profile-generate -c -o %t/main.o -fthinlto-index=%t/main.bc.thinlto.bc %t/main.bc
+
+int main() {
+    return 0;
+}
diff --git a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-newpm.ll b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/newpm.ll
similarity index 100%
rename from external/llvm-project/clang/test/CodeGen/thinlto-distributed-newpm.ll
rename to external/llvm-project/clang/test/CodeGen/distributed-thin-lto/newpm.ll
diff --git a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-objc-contract-pass.ll b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/objc-contract-pass.ll
similarity index 100%
rename from external/llvm-project/clang/test/CodeGen/thinlto-distributed-objc-contract-pass.ll
rename to external/llvm-project/clang/test/CodeGen/distributed-thin-lto/objc-contract-pass.ll
diff --git a/external/llvm-project/clang/test/CodeGen/thinlto-distributed-supports-hot-cold-new.ll b/external/llvm-project/clang/test/CodeGen/distributed-thin-lto/supports-hot-cold-new.ll
similarity index 100%
rename from external/llvm-project/clang/test/CodeGen/thinlto-distributed-supports-hot-cold-new.ll
rename to external/llvm-project/clang/test/CodeGen/distributed-thin-lto/supports-hot-cold-new.ll
diff --git a/external/llvm-project/clang/test/CodeGen/openmp-prefix-map.c b/external/llvm-project/clang/test/CodeGen/openmp-prefix-map.c
new file mode 100644
index 000000000000..be3429c26721
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/openmp-prefix-map.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple i386-unknown-unknown -debug-info-kind=standalone -fopenmp %s -emit-llvm -o - -disable-llvm-optzns -fdebug-prefix-map=%S=.| FileCheck -DPREFIX=%S %s
+
+// CHECK-NOT: @{{[0-9]+}} = private unnamed_addr constant [{{[0-9]+}} x i8] c";[[PREFIX]]{{.*}}.c;foo;{{[0-9]+}};{{[0-9]+}};;\00"
+
+void work1(int, int);
+void work2(int, int);
+void work12(int, int);
+
+void foo(int q) {
+  int p = 2;
+
+  #pragma omp parallel firstprivate(q, p)
+  work1(p, q);
+
+  #pragma omp parallel for firstprivate(p, q)
+  for (int i = 0; i < q; i++)
+    work2(i, p);
+
+  #pragma omp target teams firstprivate(p)
+  work12(p, p);
+}
diff --git a/external/llvm-project/clang/test/CodeGen/ptrauth-in-c-struct.c b/external/llvm-project/clang/test/CodeGen/ptrauth-in-c-struct.c
index d415c18d0da5..2aec31ec3baf 100644
--- a/external/llvm-project/clang/test/CodeGen/ptrauth-in-c-struct.c
+++ b/external/llvm-project/clang/test/CodeGen/ptrauth-in-c-struct.c
@@ -158,7 +158,7 @@ void test_copy_constructor_SI(SI *s) {
   SI t = *s;
 }
 
-// CHECK: define void @test_parameter_SI(i64 %{{.*}})
+// CHECK: define void @test_parameter_SI(ptr %{{.*}})
 // CHECK-NOT: call
 // CHECK: ret void
 
diff --git a/external/llvm-project/clang/test/CodeGen/sanitize-ignorelist-mainfile.c b/external/llvm-project/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
index 419e0417c1d8..599ff8e597c9 100644
--- a/external/llvm-project/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
+++ b/external/llvm-project/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
@@ -1,8 +1,10 @@
 /// Test mainfile in a sanitizer special case list.
 // RUN: rm -rf %t && split-file %s %t
-// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,DEFAULT
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,SANITIZE
 // RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/a.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
 // RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/b.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/c.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,SANITIZE
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/d.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
 
 //--- a.list
 mainfile:*a.c
@@ -14,6 +16,15 @@ mainfile:*a.c
 [alignment]
 mainfile:*.c
 
+//--- c.list
+mainfile:*a.c
+mainfile:*a.c=sanitize
+
+//--- d.list
+mainfile:*a.c
+mainfile:*a.c=sanitize
+mainfile:*a.c
+
 //--- a.h
 int global_h;
 
@@ -30,12 +41,12 @@ int foo(void *x) {
   return load(x);
 }
 
-// DEFAULT:     @___asan_gen_{{.*}} = {{.*}} c"global_h\00"
-// DEFAULT:     @___asan_gen_{{.*}} = {{.*}} c"global_c\00"
+// SANITIZE:     @___asan_gen_{{.*}} = {{.*}} c"global_h\00"
+// SANITIZE:     @___asan_gen_{{.*}} = {{.*}} c"global_c\00"
 // IGNORE-NOT:  @___asan_gen_
 
 // CHECK-LABEL: define {{.*}}@load(
-// DEFAULT:       call void @__ubsan_handle_type_mismatch_v1_abort(
-// DEFAULT:       call void @__asan_report_load4(
+// SANITIZE:       call void @__ubsan_handle_type_mismatch_v1_abort(
+// SANITIZE:       call void @__asan_report_load4(
 // IGNORE-NOT:    call void @__ubsan_handle_type_mismatch_v1_abort(
 // IGNORE-NOT:    call void @__asan_report_load4(
diff --git a/external/llvm-project/clang/test/CodeGen/tbaa-array.cpp b/external/llvm-project/clang/test/CodeGen/tbaa-array.cpp
index 4a6576e2eeb7..35e525cfa67c 100644
--- a/external/llvm-project/clang/test/CodeGen/tbaa-array.cpp
+++ b/external/llvm-project/clang/test/CodeGen/tbaa-array.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s \
+// RUN: %clang_cc1 -triple x86_64-linux -O1 %s \
 // RUN:     -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-linux -O1 -disable-llvm-passes %s \
+// RUN: %clang_cc1 -triple x86_64-linux -O1 %s \
 // RUN:     -new-struct-path-tbaa -emit-llvm -o - | \
 // RUN:     FileCheck -check-prefix=CHECK-NEW %s
 //
@@ -10,6 +10,12 @@
 struct A { int i; };
 struct B { A a[1]; };
 struct C { int i; int x[3]; };
+struct D { int n; int arr[]; }; // flexible array member
+extern int AA[];                // incomplete array type
+
+typedef int __attribute__((may_alias)) aliasing_int;
+typedef int __attribute__((may_alias)) aliasing_array[10];
+struct E { aliasing_int x[4]; aliasing_array y; };
 
 int foo(B *b) {
 // CHECK-LABEL: _Z3fooP1B
@@ -28,16 +34,42 @@ int bar(C *c) {
 
 int bar2(C *c) {
 // CHECK-NEW-LABEL: _Z4bar2P1C
-// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_int:!.*]]
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_C_x:!.*]]
   return c->x[2];
 }
 
 int bar3(C *c, int j) {
 // CHECK-NEW-LABEL: _Z4bar3P1Ci
-// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_int:!.*]]
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_C_x]]
   return c->x[j];
 }
 
+int bar4(D *d) {
+// CHECK-NEW-LABEL: _Z4bar4P1D
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_int:!.*]]
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_int:!.*]]
+    return d->arr[d->n];
+}
+
+int bar5(int j) {
+// CHECK-NEW-LABEL: _Z4bar5i
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_int:!.*]]
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_int:!.*]]
+    return AA[2] + AA[j];
+}
+
+int bar6(E *e, int j) {
+// CHECK-NEW-LABEL: _Z4bar6P1Ei
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_E_x:!.*]]
+    return e->x[j];
+}
+
+int bar7(E *e, int j) {
+// CHECK-NEW-LABEL: _Z4bar7P1Ei
+// CHECK-NEW: load i32, {{.*}}, !tbaa [[TAG_E_y:!.*]]
+    return e->y[j];
+}
+
 // CHECK-DAG: [[TAG_A_i]] = !{[[TYPE_A:!.*]], [[TYPE_int:!.*]], i64 0}
 // CHECK-DAG: [[TYPE_A]] = !{!"_ZTS1A", !{{.*}}, i64 0}
 // CHECK-DAG: [[TYPE_int]] = !{!"int", !{{.*}}, i64 0}
@@ -45,8 +77,11 @@ int bar3(C *c, int j) {
 // CHECK-NEW-DAG: [[TYPE_char:!.*]] = !{{{.*}}, i64 1, !"omnipotent char"}
 // CHECK-NEW-DAG: [[TYPE_int:!.*]] = !{[[TYPE_char]], i64 4, !"int"}
 // CHECK-NEW-DAG: [[TAG_int]] = !{[[TYPE_int]], [[TYPE_int]], i64 0, i64 4}
-// CHECK-NEW-DAG: [[TYPE_pointer:!.*]] = !{[[TYPE_char]], i64 8, !"any pointer"}
 // CHECK-NEW-DAG: [[TYPE_A:!.*]] = !{[[TYPE_char]], i64 4, !"_ZTS1A", [[TYPE_int]], i64 0, i64 4}
 // CHECK-NEW-DAG: [[TAG_A_i]] = !{[[TYPE_A]], [[TYPE_int]], i64 0, i64 4}
 // CHECK-NEW-DAG: [[TYPE_C:!.*]] = !{[[TYPE_char]], i64 16, !"_ZTS1C", [[TYPE_int]], i64 0, i64 4, [[TYPE_int]], i64 4, i64 12}
-// CHECK-NEW-DAG: [[TAG_C_i]] = !{[[TYPE_C:!.*]], [[TYPE_int:!.*]], i64 0, i64 4}
+// CHECK-NEW-DAG: [[TAG_C_i]] = !{[[TYPE_C]], [[TYPE_int]], i64 0, i64 4}
+// CHECK-NEW-DAG: [[TAG_C_x]] = !{[[TYPE_C]], [[TYPE_int]], i64 4, i64 4}
+// CHECK-NEW-DAG: [[TYPE_E:!.*]] = !{[[TYPE_char]], i64 56, !"_ZTS1E", [[TYPE_char]], i64 0, i64 16, [[TYPE_char]], i64 16, i64 40}
+// CHECK-NEW-DAG: [[TAG_E_x]] = !{[[TYPE_E]], [[TYPE_char]], i64 0, i64 4}
+// CHECK-NEW-DAG: [[TAG_E_y]] = !{[[TYPE_E]], [[TYPE_char]], i64 16, i64 4}
diff --git a/external/llvm-project/clang/test/CodeGen/ubsan-function-debuginfo.c b/external/llvm-project/clang/test/CodeGen/ubsan-function-debuginfo.c
index 21cc7e41e1a3..106aeceadf36 100644
--- a/external/llvm-project/clang/test/CodeGen/ubsan-function-debuginfo.c
+++ b/external/llvm-project/clang/test/CodeGen/ubsan-function-debuginfo.c
@@ -18,26 +18,26 @@ void call_no_prototype(void (*f)()) { f(); }
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:      #dbg_value(ptr [[F]], [[META25:![0-9]+]], !DIExpression(), [[META27:![0-9]+]])
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[F]], i64 -8, !dbg [[DBG28:![0-9]+]]
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !dbg [[DBG28]], !nosanitize [[META29:![0-9]+]]
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -1056584962, !dbg [[DBG28]], !nosanitize [[META29]]
-// CHECK-NEXT:    br i1 [[TMP2]], label %[[TYPECHECK:.*]], label %[[CONT1:.*]], !dbg [[DBG28]], !nosanitize [[META29]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !dbg [[DBG28]], !nosanitize [[META32:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -1056584962, !dbg [[DBG28]], !nosanitize [[META32]]
+// CHECK-NEXT:    br i1 [[TMP2]], label %[[TYPECHECK:.*]], label %[[CONT1:.*]], !dbg [[DBG28]], !nosanitize [[META32]]
 // CHECK:       [[TYPECHECK]]:
 // CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[F]], i64 -4, !dbg [[DBG28]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 8, !dbg [[DBG28]], !nosanitize [[META29]]
-// CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 905068220, !dbg [[DBG28]], !nosanitize [[META29]]
-// CHECK-NEXT:    br i1 [[TMP5]], label %[[CONT1]], label %[[HANDLER_FUNCTION_TYPE_MISMATCH:.*]], !dbg [[DBG28]], !prof [[PROF30:![0-9]+]], !nosanitize [[META29]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 8, !dbg [[DBG28]], !nosanitize [[META32]]
+// CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 905068220, !dbg [[DBG28]], !nosanitize [[META32]]
+// CHECK-NEXT:    br i1 [[TMP5]], label %[[CONT1]], label %[[HANDLER_FUNCTION_TYPE_MISMATCH:.*]], !dbg [[DBG28]], !prof [[PROF33:![0-9]+]], !nosanitize [[META32]]
 // CHECK:       [[HANDLER_FUNCTION_TYPE_MISMATCH]]:
-// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[F]] to i64, !dbg [[DBG28]], !nosanitize [[META29]]
-// CHECK-NEXT:    tail call void @__ubsan_handle_function_type_mismatch_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP6]]) #[[ATTR3:[0-9]+]], !dbg [[DBG28]], !nosanitize [[META29]]
-// CHECK-NEXT:    unreachable, !dbg [[DBG28]], !nosanitize [[META29]]
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[F]] to i64, !dbg [[DBG28]], !nosanitize [[META32]]
+// CHECK-NEXT:    tail call void @__ubsan_handle_function_type_mismatch_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP6]]) #[[ATTR3:[0-9]+]], !dbg [[DBG28]], !nosanitize [[META32]]
+// CHECK-NEXT:    unreachable, !dbg [[DBG28]], !nosanitize [[META32]]
 // CHECK:       [[CONT1]]:
-// CHECK-NEXT:    tail call void [[F]]() #[[ATTR2]], !dbg [[DBG28]]
-// CHECK-NEXT:    ret void, !dbg [[DBG31:![0-9]+]]
+// CHECK-NEXT:    tail call void [[F]]() #[[ATTR2]], !dbg [[DBG31:![0-9]+]]
+// CHECK-NEXT:    ret void, !dbg [[DBG34:![0-9]+]]
 //
 void call_prototype(void (*f)(void)) { f(); }
 //.
 // CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C11, file: [[META1:![0-9]+]], isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-// CHECK: [[META1]] = !DIFile(filename: "<stdin>", directory: {{.*}})
+// CHECK: [[META1]] = !DIFile(filename: "{{.*}}<stdin>", directory: {{.*}})
 // CHECK: [[DBG5]] = distinct !DISubprogram(name: "call_no_prototype", scope: [[META6:![0-9]+]], file: [[META6]], line: 14, type: [[META7:![0-9]+]], scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12:![0-9]+]])
 // CHECK: [[META6]] = !DIFile(filename: "{{.*}}ubsan-function-debuginfo.c", directory: {{.*}})
 // CHECK: [[META7]] = !DISubroutineType(types: [[META8:![0-9]+]])
@@ -61,8 +61,11 @@ void call_prototype(void (*f)(void)) { f(); }
 // CHECK: [[META25]] = !DILocalVariable(name: "f", arg: 1, scope: [[DBG18]], file: [[META6]], line: 37, type: [[META21]])
 // CHECK: [[META26]] = !{i32 -1056584962, i32 -747727454}
 // CHECK: [[META27]] = !DILocation(line: 0, scope: [[DBG18]])
-// CHECK: [[DBG28]] = !DILocation(line: 37, column: 40, scope: [[DBG18]])
-// CHECK: [[META29]] = !{}
-// CHECK: [[PROF30]] = !{!"branch_weights", i32 1048575, i32 1}
-// CHECK: [[DBG31]] = !DILocation(line: 37, column: 45, scope: [[DBG18]])
+// CHECK: [[DBG28]] = !DILocation(line: 0, scope: [[META29:![0-9]+]], inlinedAt: [[DBG31]])
+// CHECK: [[META29]] = distinct !DISubprogram(name: "__ubsan_check_function", scope: [[META6]], file: [[META6]], type: [[META30:![0-9]+]], flags: DIFlagArtificial, spFlags: DISPFlagDefinition, unit: [[META0]])
+// CHECK: [[META30]] = !DISubroutineType(types: null)
+// CHECK: [[DBG31]] = !DILocation(line: 37, column: 40, scope: [[DBG18]])
+// CHECK: [[META32]] = !{}
+// CHECK: [[PROF33]] = !{!"branch_weights", i32 1048575, i32 1}
+// CHECK: [[DBG34]] = !DILocation(line: 37, column: 45, scope: [[DBG18]])
 //.
diff --git a/external/llvm-project/clang/test/CodeGen/ubsan-function-ignorelist.test b/external/llvm-project/clang/test/CodeGen/ubsan-function-ignorelist.test
new file mode 100644
index 000000000000..311b2ffd7f73
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGen/ubsan-function-ignorelist.test
@@ -0,0 +1,83 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-0.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,SANITIZE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-1.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-2.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,SANITIZE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-3.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-4.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,SANITIZE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-5.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-6.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,SANITIZE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-7.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-ignorelist=%t/order-8.ignorelist -emit-llvm %t/test.c -o - | FileCheck %s --check-prefixes=CHECK,SANITIZE
+
+
+// The same type can appear multiple times within an ignorelist. Any ``=sanitize`` type
+// entries enable sanitizer instrumentation, even if it was ignored by entries before.
+// If multiple entries match the source, then the latest entry takes the
+// precedence.
+
+
+//--- order-0.ignorelist
+fun:add
+fun:add=sanitize
+
+//--- order-1.ignorelist
+fun:add=sanitize
+fun:add
+
+//--- order-2.ignorelist
+fun:ad*
+fun:add=sanitize
+
+//--- order-3.ignorelist
+fun:ad*=sanitize
+fun:add
+
+//--- order-4.ignorelist
+fun:add
+fun:ad*=sanitize
+
+//--- order-5.ignorelist
+fun:add=sanitize
+fun:ad*
+
+//--- order-6.ignorelist
+fun:add
+fun:add=sanitize
+fun:a*d
+fun:*dd=sanitize
+
+//--- order-7.ignorelist
+[{unsigned-integer-overflow,signed-integer-overflow}]
+fun:*
+fun:add=sanitize
+fun:a*d
+fun:*dd=sanitize
+[{unsigned-integer-overflow,signed-integer-overflow}]
+fun:*
+fun:add
+fun:a*d=sanitize
+fun:*d
+
+//--- order-8.ignorelist
+[{unsigned-integer-overflow,signed-integer-overflow}]
+fun:*
+fun:add
+fun:a*d=sanitize
+fun:*dd
+[{unsigned-integer-overflow,signed-integer-overflow}]
+fun:*
+fun:add=sanitize
+fun:a*d
+fun:*dd=sanitize
+
+
+//--- test.c
+// CHECK-LABEL: define dso_local void @add
+void add(int A) {
+// IGNORE: %inc = add nsw
+// SANITIZE: @llvm.sadd.with.overflow.i32
+  ++A;
+}
+
diff --git a/external/llvm-project/clang/test/CodeGen/unique-source-file-names.c b/external/llvm-project/clang/test/CodeGen/unique-source-file-names.c
index 1d5a4a5e8e4c..df8e3025870a 100644
--- a/external/llvm-project/clang/test/CodeGen/unique-source-file-names.c
+++ b/external/llvm-project/clang/test/CodeGen/unique-source-file-names.c
@@ -1,2 +1,3 @@
-// RUN: %clang_cc1 -funique-source-file-names -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck %s
-// CHECK:  !{i32 7, !"Unique Source File Names", i32 1}
+// RUN: %clang_cc1 -funique-source-file-identifier=foo -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// CHECK:  !{i32 5, !"Unique Source File Identifier", ![[MD:[0-9]*]]}
+// CHECK: ![[MD]] = !{!"foo"}
diff --git a/external/llvm-project/clang/test/CodeGen/unsigned-promotion-debuginfo.c b/external/llvm-project/clang/test/CodeGen/unsigned-promotion-debuginfo.c
index 163f306f60a1..88e691d65334 100644
--- a/external/llvm-project/clang/test/CodeGen/unsigned-promotion-debuginfo.c
+++ b/external/llvm-project/clang/test/CodeGen/unsigned-promotion-debuginfo.c
@@ -18,19 +18,19 @@ unsigned short si, sj, sk;
 // CHECKS-NEXT:    [[CONV:%.*]] = zext i16 [[TMP0]] to i32, !dbg [[DBG16]]
 // CHECKS-NEXT:    [[TMP1:%.*]] = load i16, ptr @sk, align 2, !dbg [[DBG21:![0-9]+]], !tbaa [[TBAA17]]
 // CHECKS-NEXT:    [[CONV1:%.*]] = zext i16 [[TMP1]] to i32, !dbg [[DBG21]]
-// CHECKS-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[CONV]], i32 [[CONV1]]), !dbg [[DBG22:![0-9]+]], !nosanitize [[META23:![0-9]+]]
-// CHECKS-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !dbg [[DBG22]], !nosanitize [[META23]]
-// CHECKS-NEXT:    br i1 [[TMP3]], label %[[HANDLER_MUL_OVERFLOW:.*]], label %[[CONT:.*]], !dbg [[DBG22]], !prof [[PROF24:![0-9]+]], !nosanitize [[META23]]
+// CHECKS-NEXT:    [[TMP2:%.*]] = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[CONV]], i32 [[CONV1]]), !dbg [[DBG22:![0-9]+]], !nosanitize [[META26:![0-9]+]]
+// CHECKS-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !dbg [[DBG22]], !nosanitize [[META26]]
+// CHECKS-NEXT:    br i1 [[TMP3]], label %[[HANDLER_MUL_OVERFLOW:.*]], label %[[CONT:.*]], !dbg [[DBG22]], !prof [[PROF27:![0-9]+]], !nosanitize [[META26]]
 // CHECKS:       [[HANDLER_MUL_OVERFLOW]]:
 // CHECKS-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP0]] to i64, !dbg [[DBG22]]
 // CHECKS-NEXT:    [[TMP5:%.*]] = zext i16 [[TMP1]] to i64, !dbg [[DBG22]]
-// CHECKS-NEXT:    tail call void @__ubsan_handle_mul_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP4]], i64 [[TMP5]]) #[[ATTR3:[0-9]+]], !dbg [[DBG22]], !nosanitize [[META23]]
-// CHECKS-NEXT:    unreachable, !dbg [[DBG22]], !nosanitize [[META23]]
+// CHECKS-NEXT:    tail call void @__ubsan_handle_mul_overflow_abort(ptr nonnull @[[GLOB1:[0-9]+]], i64 [[TMP4]], i64 [[TMP5]]) #[[ATTR3:[0-9]+]], !dbg [[DBG22]], !nosanitize [[META26]]
+// CHECKS-NEXT:    unreachable, !dbg [[DBG22]], !nosanitize [[META26]]
 // CHECKS:       [[CONT]]:
-// CHECKS-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !dbg [[DBG22]], !nosanitize [[META23]]
+// CHECKS-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0, !dbg [[DBG22]], !nosanitize [[META26]]
 // CHECKS-NEXT:    [[CONV2:%.*]] = trunc i32 [[TMP6]] to i16, !dbg [[DBG16]]
-// CHECKS-NEXT:    store i16 [[CONV2]], ptr @si, align 2, !dbg [[DBG25:![0-9]+]], !tbaa [[TBAA17]]
-// CHECKS-NEXT:    ret void, !dbg [[DBG26:![0-9]+]]
+// CHECKS-NEXT:    store i16 [[CONV2]], ptr @si, align 2, !dbg [[DBG28:![0-9]+]], !tbaa [[TBAA17]]
+// CHECKS-NEXT:    ret void, !dbg [[DBG29:![0-9]+]]
 //
 // CHECKU-LABEL: define dso_local void @testshortmul(
 // CHECKU-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] !dbg [[DBG13:![0-9]+]] {
@@ -67,11 +67,14 @@ void testshortmul(void) {
 // CHECKS: [[META19]] = !{!"omnipotent char", [[META20:![0-9]+]], i64 0}
 // CHECKS: [[META20]] = !{!"Simple C/C++ TBAA"}
 // CHECKS: [[DBG21]] = !DILocation(line: 47, column: 13, scope: [[DBG13]])
-// CHECKS: [[DBG22]] = !DILocation(line: 47, column: 11, scope: [[DBG13]])
-// CHECKS: [[META23]] = !{}
-// CHECKS: [[PROF24]] = !{!"branch_weights", i32 1, i32 1048575}
-// CHECKS: [[DBG25]] = !DILocation(line: 47, column: 6, scope: [[DBG13]])
-// CHECKS: [[DBG26]] = !DILocation(line: 48, column: 1, scope: [[DBG13]])
+// CHECKS: [[DBG22]] = !DILocation(line: 0, scope: [[META23:![0-9]+]], inlinedAt: [[META25:![0-9]+]])
+// CHECKS: [[META23]] = distinct !DISubprogram(name: "__ubsan_check_mul_overflow", scope: [[META7]], file: [[META7]], type: [[META24:![0-9]+]], flags: DIFlagArtificial, spFlags: DISPFlagDefinition, unit: [[META2]])
+// CHECKS: [[META24]] = !DISubroutineType(types: null)
+// CHECKS: [[META25]] = !DILocation(line: 47, column: 11, scope: [[DBG13]])
+// CHECKS: [[META26]] = !{}
+// CHECKS: [[PROF27]] = !{!"branch_weights", i32 1, i32 1048575}
+// CHECKS: [[DBG28]] = !DILocation(line: 47, column: 6, scope: [[DBG13]])
+// CHECKS: [[DBG29]] = !DILocation(line: 48, column: 1, scope: [[DBG13]])
 //.
 // CHECKU: [[META0:![0-9]+]] = !DIGlobalVariableExpression(var: [[META1:![0-9]+]], expr: !DIExpression())
 // CHECKU: [[META1]] = distinct !DIGlobalVariable(name: "sj", scope: [[META2:![0-9]+]], file: [[META7:![0-9]+]], line: 12, type: [[META8:![0-9]+]], isLocal: false, isDefinition: true)
diff --git a/external/llvm-project/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors-msvc.cpp b/external/llvm-project/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors-msvc.cpp
index 435feec774ec..3ed75b9494c2 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors-msvc.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors-msvc.cpp
@@ -1,7 +1,13 @@
-// RUN: not %clang_cc1 -triple aarch64-unknown-windows-msvc %s -emit-llvm \
-// RUN:   -o - 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-unknown-windows-msvc %s -emit-llvm \
+// RUN:   -o - | FileCheck %s
 
 template<typename T> struct S {};
 
-// CHECK: cannot mangle this built-in type: __SVInt8_t yet
+// CHECK: void @"?f1@@YAXU?$S@U__SVInt8_t@__clang@@@@@Z"
 void f1(S<__SVInt8_t>) {}
+// CHECK: void @"?f2@@YAXU?$S@U__SVInt32_t@__clang@@@@@Z"
+void f2(S<__SVInt32_t>) {}
+// CHECK: void @"?f3@@YAXU?$S@U__SVBool_t@__clang@@@@@Z"
+void f3(S<__SVBool_t>) {}
+// CHECK: void @"?f4@@YAXU?$S@U__clang_svfloat64x4_t@__clang@@@@@Z"
+void f4(S<__clang_svfloat64x4_t>) {}
diff --git a/external/llvm-project/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp b/external/llvm-project/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
index 4f553262c73b..a502d24f1788 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
@@ -9,7 +9,7 @@ int bar() {
 }
 
 template <typename T1, typename T2> struct MyClass {
-  int __attribute__((target_clones("frintts", "ssbs+sme-f64f64"))) foo_tml() { return 1; }
+  int __attribute__((target_clones("frintts", "ssbs+sme-f64f64", "default"))) foo_tml() { return 1; }
 };
 
 template <typename T> struct MyClass<int, T> {
@@ -41,9 +41,7 @@ void run_foo_tml() {
 //.
 // CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
 // CHECK: @_Z7foo_ovli = weak_odr ifunc i32 (i32), ptr @_Z7foo_ovli.resolver
-// CHECK: @_Z7foo_ovlv = weak_odr ifunc i32 (), ptr @_Z7foo_ovlv.resolver
 // CHECK: @_ZN7MyClassIssE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIssE7foo_tmlEv.resolver
-// CHECK: @_ZN7MyClassIisE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIisE7foo_tmlEv.resolver
 //.
 // CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovli._Mfp16(
 // CHECK-SAME: i32 noundef [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -53,18 +51,12 @@ void run_foo_tml() {
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: define weak_odr ptr @_Z7foo_ovli.resolver() comdat {
-// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 65792
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65792
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
-// CHECK:       [[RESOLVER_RETURN]]:
-// CHECK-NEXT:    ret ptr @_Z7foo_ovli._Mfp16
-// CHECK:       [[RESOLVER_ELSE]]:
-// CHECK-NEXT:    ret ptr @_Z7foo_ovli.default
+// CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovli.default(
+// CHECK-SAME: i32 noundef [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovlv._Mfp16(
@@ -73,22 +65,8 @@ void run_foo_tml() {
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: define weak_odr ptr @_Z7foo_ovlv.resolver() comdat {
-// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 65792
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65792
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
-// CHECK:       [[RESOLVER_RETURN]]:
-// CHECK-NEXT:    ret ptr @_Z7foo_ovlv._Mfp16
-// CHECK:       [[RESOLVER_ELSE]]:
-// CHECK-NEXT:    ret ptr @_Z7foo_ovlv.default
-//
-//
 // CHECK-LABEL: define dso_local noundef i32 @_Z3barv(
-// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[CALL:%.*]] = call noundef i32 @_Z7foo_ovli(i32 noundef 1)
 // CHECK-NEXT:    [[CALL1:%.*]] = call noundef i32 @_Z7foo_ovlv()
@@ -97,7 +75,7 @@ void run_foo_tml() {
 //
 //
 // CHECK-LABEL: define dso_local void @_Z11run_foo_tmlv(
-// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-SAME: ) #[[ATTR2]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MC1:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1
 // CHECK-NEXT:    [[MC2:%.*]] = alloca [[STRUCT_MYCLASS_0:%.*]], align 1
@@ -111,7 +89,7 @@ void run_foo_tml() {
 //
 //
 // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIfsE7foo_tmlEv(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR1]] comdat {
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
@@ -120,7 +98,7 @@ void run_foo_tml() {
 //
 //
 // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIdfE7foo_tmlEv(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR1]] comdat {
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
@@ -128,22 +106,22 @@ void run_foo_tml() {
 // CHECK-NEXT:    ret i32 4
 //
 //
-// CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovli.default(
-// CHECK-SAME: i32 noundef [[TMP0:%.*]]) #[[ATTR2:[0-9]+]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[DOTADDR]], align 4
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef i32 @_Z7foo_ovlv.default(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    ret i32 2
+// CHECK-LABEL: define weak_odr ptr @_Z7foo_ovli.resolver() comdat {
+// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 65792
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65792
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK:       [[RESOLVER_RETURN]]:
+// CHECK-NEXT:    ret ptr @_Z7foo_ovli._Mfp16
+// CHECK:       [[RESOLVER_ELSE]]:
+// CHECK-NEXT:    ret ptr @_Z7foo_ovli.default
 //
 //
 // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIssE7foo_tmlEv._Mfrintts(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR3:[0-9]+]] comdat {
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR5:[0-9]+]] comdat {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
@@ -152,7 +130,7 @@ void run_foo_tml() {
 //
 //
 // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIssE7foo_tmlEv._Msme-f64f64Mssbs(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR4:[0-9]+]] comdat {
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR6:[0-9]+]] comdat {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
@@ -161,7 +139,7 @@ void run_foo_tml() {
 //
 //
 // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIssE7foo_tmlEv.default(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR1]] comdat {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
@@ -192,7 +170,7 @@ void run_foo_tml() {
 //
 //
 // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIisE7foo_tmlEv._Mfrintts(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR3]] comdat {
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR5]] comdat {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
@@ -201,44 +179,13 @@ void run_foo_tml() {
 //
 //
 // CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR4]] comdat {
+// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR6]] comdat {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK-NEXT:    ret i32 2
 //
-//
-// CHECK-LABEL: define linkonce_odr noundef i32 @_ZN7MyClassIisE7foo_tmlEv.default(
-// CHECK-SAME: ptr noundef nonnull align 1 dereferenceable(1) [[THIS:%.*]]) #[[ATTR2]] comdat {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK-NEXT:    ret i32 2
-//
-//
-// CHECK-LABEL: define weak_odr ptr @_ZN7MyClassIisE7foo_tmlEv.resolver() comdat {
-// CHECK-NEXT:  [[RESOLVER_ENTRY:.*:]]
-// CHECK-NEXT:    call void @__init_cpu_features_resolver()
-// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 36596145153180416
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36596145153180416
-// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT:    br i1 [[TMP3]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
-// CHECK:       [[RESOLVER_RETURN]]:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs
-// CHECK:       [[RESOLVER_ELSE]]:
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16777472
-// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777472
-// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT:    br i1 [[TMP7]], label %[[RESOLVER_RETURN1:.*]], label %[[RESOLVER_ELSE2:.*]]
-// CHECK:       [[RESOLVER_RETURN1]]:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv._Mfrintts
-// CHECK:       [[RESOLVER_ELSE2]]:
-// CHECK-NEXT:    ret ptr @_ZN7MyClassIisE7foo_tmlEv.default
-//
 //.
 // CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 // CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
diff --git a/external/llvm-project/clang/test/CodeGenCXX/attr-target-version-riscv.cpp b/external/llvm-project/clang/test/CodeGenCXX/attr-target-version-riscv.cpp
index 827cd1ef9750..ffb4576b3cd3 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/attr-target-version-riscv.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/attr-target-version-riscv.cpp
@@ -49,155 +49,6 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo1v.default(
-// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo2v._zbb(
-// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo2v._m(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo2v.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo3v._c_zbb(
-// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo3v._m(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo3v.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v._zba(
-// CHECK-SAME: ) #[[ATTR4:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v._zba_zbb(
-// CHECK-SAME: ) #[[ATTR5:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v._zba(
-// CHECK-SAME: ) #[[ATTR4]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v._zba_zbb(
-// CHECK-SAME: ) #[[ATTR5]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v._zba(
-// CHECK-SAME: ) #[[ATTR4]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v._zba_zbb(
-// CHECK-SAME: ) #[[ATTR5]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v._zba(
-// CHECK-SAME: ) #[[ATTR4]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v._zbb(
-// CHECK-SAME: ) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v._zba_zbb(
-// CHECK-SAME: ) #[[ATTR5]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v.default(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: define dso_local noundef signext i32 @_Z3barv(
-// CHECK-SAME: ) #[[ATTR1]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[CALL:%.*]] = call noundef signext i32 @_Z4foo1v()
-// CHECK-NEXT:    [[CALL1:%.*]] = call noundef signext i32 @_Z4foo2v()
-// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
-// CHECK-NEXT:    [[CALL2:%.*]] = call noundef signext i32 @_Z4foo3v()
-// CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[ADD]], [[CALL2]]
-// CHECK-NEXT:    ret i32 [[ADD3]]
-//
-//
 // CHECK-LABEL: define weak_odr ptr @_Z4foo1v.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -211,6 +62,18 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK-NEXT:    ret ptr @_Z4foo1v.default
 //
 //
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo1v.default(
+// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo2v._zbb(
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @_Z4foo2v.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -231,6 +94,24 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK-NEXT:    ret ptr @_Z4foo2v.default
 //
 //
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo2v._m(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo2v.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo3v._c_zbb(
+// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @_Z4foo3v.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -251,6 +132,24 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK-NEXT:    ret ptr @_Z4foo3v.default
 //
 //
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo3v._m(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo3v.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v._zba(
+// CHECK-SAME: ) #[[ATTR4:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @_Z4foo4v.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -278,6 +177,30 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK-NEXT:    ret ptr @_Z4foo4v.default
 //
 //
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v._zba_zbb(
+// CHECK-SAME: ) #[[ATTR5:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo4v.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v._zba(
+// CHECK-SAME: ) #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @_Z4foo5v.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -305,6 +228,30 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK-NEXT:    ret ptr @_Z4foo5v.default
 //
 //
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v._zba_zbb(
+// CHECK-SAME: ) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo5v.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v._zba(
+// CHECK-SAME: ) #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @_Z4foo6v.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -332,6 +279,30 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK-NEXT:    ret ptr @_Z4foo6v.default
 //
 //
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v._zba_zbb(
+// CHECK-SAME: ) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo6v.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v._zba(
+// CHECK-SAME: ) #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: define weak_odr ptr @_Z4foo7v.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_riscv_feature_bits(ptr null)
@@ -358,6 +329,35 @@ int bar() { return foo1() + foo2() + foo3(); }
 // CHECK:       resolver_else4:
 // CHECK-NEXT:    ret ptr @_Z4foo7v.default
 //
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v._zbb(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v._zba_zbb(
+// CHECK-SAME: ) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z4foo7v.default(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: define dso_local noundef signext i32 @_Z3barv(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call noundef signext i32 @_Z4foo1v()
+// CHECK-NEXT:    [[CALL1:%.*]] = call noundef signext i32 @_Z4foo2v()
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
+// CHECK-NEXT:    [[CALL2:%.*]] = call noundef signext i32 @_Z4foo3v()
+// CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[ADD]], [[CALL2]]
+// CHECK-NEXT:    ret i32 [[ADD3]]
+//
 //.
 // CHECK: attributes #[[ATTR0]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+64bit,+d,+f,+i,+m,+v,+zicsr,+zmmul,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b" }
 // CHECK: attributes #[[ATTR1]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+64bit,+i,+m,+zmmul" }
diff --git a/external/llvm-project/clang/test/CodeGenCXX/builtin-get-vtable-pointer.cpp b/external/llvm-project/clang/test/CodeGenCXX/builtin-get-vtable-pointer.cpp
new file mode 100644
index 000000000000..604fb6c5585a
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGenCXX/builtin-get-vtable-pointer.cpp
@@ -0,0 +1,370 @@
+// RUN: %clang_cc1 %s -x c++ -std=c++23  -triple x86_64-apple-darwin10 -emit-llvm -O1 -disable-llvm-passes -no-enable-noundef-analysis  -o - | FileCheck --check-prefix=CHECK-NOAUTH %s
+// RUN: %clang_cc1 %s -x c++ -std=c++23  -triple arm64-apple-ios -fptrauth-calls -fptrauth-vtable-pointer-type-discrimination -emit-llvm -O1 -disable-llvm-passes -no-enable-noundef-analysis   -o - | FileCheck --check-prefix=CHECK-TYPEAUTH %s
+// RUN: %clang_cc1 %s -x c++ -std=c++23  -triple arm64-apple-ios -fptrauth-calls -fptrauth-vtable-pointer-address-discrimination -emit-llvm -O1 -disable-llvm-passes -no-enable-noundef-analysis  -o - | FileCheck --check-prefix=CHECK-ADDRESSAUTH %s
+// RUN: %clang_cc1 %s -x c++ -std=c++23  -triple arm64-apple-ios -fptrauth-calls -fptrauth-vtable-pointer-type-discrimination -fptrauth-vtable-pointer-address-discrimination -emit-llvm -O1 -disable-llvm-passes -no-enable-noundef-analysis  -o - | FileCheck --check-prefix=CHECK-BOTHAUTH %s
+// FIXME: Assume load should not require -fstrict-vtable-pointers
+
+namespace test1 {
+struct A {
+  A();
+  virtual void bar();
+};
+
+struct B : A {
+  B();
+  virtual void foo();
+};
+
+struct Z : A {};
+struct C : Z, B {
+  C();
+  virtual void wibble();
+};
+
+struct D : virtual A {
+};
+
+struct E : D, B {
+};
+
+template <class A, class B> struct same_type {
+  static const bool value = false;
+};
+
+template <class A> struct same_type<A, A> {
+  static const bool value = true;
+};
+
+const void *a(A *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test11aEPNS_1AE(ptr %o) #0 {
+  // CHECK-TYPEAUTH: define ptr @_ZN5test11aEPNS_1AE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer(o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %0 = load ptr, ptr %o.addr, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 %1)
+  // CHECK-ADDRESSAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-ADDRESSAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-BOTHAUTH: [[T1:%.*]] = ptrtoint ptr %0 to i64
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T1]], i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *b(B *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-TYPEAUTH: define ptr @_ZN5test11bEPNS_1BE(ptr %o) #0 {
+  // CHECK-NOAUTH: define ptr @_ZN5test11bEPNS_1BE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer(o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 %1)
+  // CHECK-ADDRESSAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-ADDRESSAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 %1, i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *b_as_A(B *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test16b_as_AEPNS_1BE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer((A *)o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 %1)
+  // CHECK-ADDRESSAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-ADDRESSAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 %1, i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *c(C *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test11cEPNS_1CE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer(o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 %1)
+  // CHECK-ADDRESSAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-ADDRESSAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 %1, i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *c_as_Z(C *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test16c_as_ZEPNS_1CE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer((Z *)o);
+  // CHECK-NOAUTH: %0 = load ptr, ptr %o.addr, align 8
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 %1)
+  // CHECK-ADDRESSAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-ADDRESSAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 %1, i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *c_as_B(C *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test16c_as_BEPNS_1CE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer((B *)o);
+  // CHECK-NOAUTH: %add.ptr = getelementptr inbounds i8, ptr %0, i64 8
+  // CHECK-NOAUTH: br label %cast.end
+  // CHECK-NOAUTH: %cast.result = phi ptr [ %add.ptr, %cast.notnull ], [ null, %entry ]
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %cast.result, align 8
+  // CHECK-TYPEAUTH: %cast.result = phi ptr [ %add.ptr, %cast.notnull ], [ null, %entry ]
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %cast.result, align 8
+  // CHECK-TYPEAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-TYPEAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-ADDRESSAUTH: %2 = ptrtoint ptr %cast.result to i64
+  // CHECK-ADDRESSAUTH: %3 = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: %4 = call i64 @llvm.ptrauth.auth(i64 %3, i32 2, i64 %2)
+  // CHECK-ADDRESSAUTH: %5 = inttoptr i64 %4 to ptr
+  // CHECK-ADDRESSAUTH: %6 = load volatile i8, ptr %5, align 8
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 %2, i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *d(D *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test11dEPNS_1DE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer(o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: %1 = ptrtoint ptr %0 to i64
+  // CHECK-ADDRESSAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 %1)
+  // CHECK-ADDRESSAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-ADDRESSAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-BOTHAUTH: [[T1:%.*]] = ptrtoint ptr %0 to i64
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T1]], i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *d_as_A(D *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test16d_as_AEPNS_1DE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer((A *)o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-NOAUTH: %vbase.offset.ptr = getelementptr i8, ptr %vtable, i64 -32
+  // CHECK-NOAUTH: %vbase.offset = load i64, ptr %vbase.offset.ptr, align 8
+  // CHECK-NOAUTH: %add.ptr = getelementptr inbounds i8, ptr %0, i64 %vbase.offset
+  // CHECK-NOAUTH: %cast.result = phi ptr [ %add.ptr, %cast.notnull ], [ null, %entry ]
+  // CHECK-NOAUTH: %vtable1 = load ptr, ptr %cast.result, align 8
+  // CHECK-TYPEAUTH: %vtable1 = load ptr, ptr %cast.result, align 8
+  // CHECK-TYPEAUTH: %5 = ptrtoint ptr %vtable1 to i64
+  // CHECK-TYPEAUTH: %6 = call i64 @llvm.ptrauth.auth(i64 %5, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %7 = inttoptr i64 %6 to ptr
+  // CHECK-TYPEAUTH: %8 = load volatile i8, ptr %7, align 8
+  // CHECK-ADDRESSAUTH: %6 = ptrtoint ptr %cast.result to i64
+  // CHECK-ADDRESSAUTH: %7 = ptrtoint ptr %vtable1 to i64
+  // CHECK-ADDRESSAUTH: %8 = call i64 @llvm.ptrauth.auth(i64 %7, i32 2, i64 %6)
+  // CHECK-ADDRESSAUTH: %9 = inttoptr i64 %8 to ptr
+  // CHECK-ADDRESSAUTH: %10 = load volatile i8, ptr %9, align 8
+  // CHECK-BOTHAUTH: [[T1:%.*]] = ptrtoint ptr %cast.result to i64
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T1]], i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable1 to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *e(E *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test11eEPNS_1EE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer(o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: [[T1:%.*]] = ptrtoint ptr %0 to i64
+  // CHECK-ADDRESSAUTH: [[T2:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: [[T3:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T2]], i32 2, i64 [[T1]])
+  // CHECK-ADDRESSAUTH: [[T4:%.*]] = inttoptr i64 [[T3]] to ptr
+  // CHECK-ADDRESSAUTH: [[T5:%.*]] = load volatile i8, ptr [[T4]], align 8
+  // CHECK-BOTHAUTH: [[T1:%.*]] = ptrtoint ptr %0 to i64
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T1]], i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *e_as_B(E *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test16e_as_BEPNS_1EE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer((B *)o);
+  // CHECK-NOAUTH: %add.ptr = getelementptr inbounds i8, ptr %0, i64 8
+  // CHECK-NOAUTH: %cast.result = phi ptr [ %add.ptr, %cast.notnull ], [ null, %entry ]
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %cast.result, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %cast.result, align 8
+  // CHECK-TYPEAUTH: %2 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %3 = call i64 @llvm.ptrauth.auth(i64 %2, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %4 = inttoptr i64 %3 to ptr
+  // CHECK-TYPEAUTH: %5 = load volatile i8, ptr %4, align 8
+  // CHECK-ADDRESSAUTH: [[T1:%.*]] = ptrtoint ptr %cast.result to i64
+  // CHECK-ADDRESSAUTH: [[T2:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: [[T3:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T2]], i32 2, i64 [[T1]])
+  // CHECK-ADDRESSAUTH: [[T4:%.*]] = inttoptr i64 [[T3]] to ptr
+  // CHECK-ADDRESSAUTH: [[T5:%.*]] = load volatile i8, ptr [[T4]], align 8
+  // CHECK-BOTHAUTH: [[T1:%.*]] = ptrtoint ptr %cast.result to i64
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T1]], i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+const void *e_as_D(E *o) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(o)), const void*>::value);
+  // CHECK-NOAUTH: define ptr @_ZN5test16e_as_DEPNS_1EE(ptr %o) #0 {
+  return __builtin_get_vtable_pointer((D *)o);
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %0, align 8
+  // CHECK-TYPEAUTH: %1 = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: %2 = call i64 @llvm.ptrauth.auth(i64 %1, i32 2, i64 48388)
+  // CHECK-TYPEAUTH: %3 = inttoptr i64 %2 to ptr
+  // CHECK-TYPEAUTH: %4 = load volatile i8, ptr %3, align 8
+  // CHECK-ADDRESSAUTH: [[T1:%.*]] = ptrtoint ptr %0 to i64
+  // CHECK-ADDRESSAUTH: [[T2:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: [[T3:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T2]], i32 2, i64 [[T1]])
+  // CHECK-ADDRESSAUTH: [[T4:%.*]] = inttoptr i64 [[T3]] to ptr
+  // CHECK-ADDRESSAUTH: [[T5:%.*]] = load volatile i8, ptr [[T4]], align 8
+  // CHECK-BOTHAUTH: [[T1:%.*]] = ptrtoint ptr %0 to i64
+  // CHECK-BOTHAUTH: [[T2:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[T1]], i64 48388)
+  // CHECK-BOTHAUTH: [[T3:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[T4:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[T3]], i32 2, i64 [[T2]])
+  // CHECK-BOTHAUTH: [[T5:%.*]] = inttoptr i64 [[T4]] to ptr
+  // CHECK-BOTHAUTH: [[T6:%.*]] = load volatile i8, ptr [[T5]], align 8
+}
+
+extern "C" const void *aArrayParameter(A aArray[]) {
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(aArray)), const void*>::value);
+  // CHECK-NOAUTH: [[THIS_OBJ:%.*]] = load ptr, ptr %aArray.addr
+  // CHECK-NOAUTH: %vtable = load ptr, ptr [[THIS_OBJ]]
+  // CHECK-TYPEAUTH: [[THIS_OBJ:%.*]] = load ptr, ptr %aArray.addr
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr [[THIS_OBJ]]
+  // CHECK-TYPEAUTH: [[VTABLEI:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: [[AUTHENTICATED:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[VTABLEI]], i32 2, i64 48388)
+  // CHECK-ADDRESSAUTH: [[VTABLE_ADDR:%.*]] = load ptr, ptr %aArray.addr, align 8, !tbaa !2
+  // CHECK-ADDRESSAUTH: %vtable = load ptr, ptr %0, align 8, !tbaa !7
+  // CHECK-ADDRESSAUTH: [[VTABLE_ADDRI:%.*]] = ptrtoint ptr [[VTABLE_ADDR]] to i64
+  // CHECK-ADDRESSAUTH: [[VTABLEI:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: [[AUTHENTICATED:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[VTABLEI]], i32 2, i64 [[VTABLE_ADDRI]])
+  // CHECK-BOTHAUTH: [[VTABLE_ADDR:%.*]] = load ptr, ptr %aArray.addr, align 8, !tbaa !2
+  // CHECK-BOTHAUTH: %vtable = load ptr, ptr [[VTABLE_ADDR]], align 8, !tbaa !7
+  // CHECK-BOTHAUTH: [[VTABLE_ADDRI:%.*]] = ptrtoint ptr [[VTABLE_ADDR]] to i64
+  // CHECK-BOTHAUTH: [[VTABLE_DISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[VTABLE_ADDRI]], i64 48388)
+  // CHECK-BOTHAUTH: [[VTABLE_PTR:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[AUTHENTICATED:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[VTABLE_PTR]], i32 2, i64 [[VTABLE_DISC]])
+  return __builtin_get_vtable_pointer(aArray);
+}
+
+extern "C" const void *aArrayLocal() {
+  A array[] = { A() };
+  static_assert(same_type<decltype(__builtin_get_vtable_pointer(array)), const void*>::value);
+  // CHECK-NOAUTH: [[THIS_OBJ:%.*]] = getelementptr inbounds [1 x %"struct.test1::A"], ptr %array
+  // CHECK-NOAUTH: %vtable = load ptr, ptr %arraydecay
+  // CHECK-TYPEAUTH: %arraydecay = getelementptr inbounds [1 x %"struct.test1::A"]
+  // CHECK-TYPEAUTH: %vtable = load ptr, ptr %arraydecay
+  // CHECK-TYPEAUTH: [[VTABLEI:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-TYPEAUTH: [[AUTHENTICATED:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[VTABLEI]], i32 2, i64 48388)
+  // CHECK-ADDRESSAUTH: %arraydecay = getelementptr inbounds [1 x %"struct.test1::A"], ptr %array, i64 0, i64 0
+  // CHECK-ADDRESSAUTH: %vtable = load ptr, ptr %arraydecay, align 8, !tbaa !7
+  // CHECK-ADDRESSAUTH: [[VTABLE_ADDRI:%.*]] = ptrtoint ptr %arraydecay to i64
+  // CHECK-ADDRESSAUTH: [[VTABLEI:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-ADDRESSAUTH: [[AUTHENTICATED:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[VTABLEI]], i32 2, i64 [[VTABLE_ADDRI]])
+  // CHECK-BOTHAUTH: %arraydecay = getelementptr inbounds [1 x %"struct.test1::A"], ptr %array, i64 0, i64 0
+  // CHECK-BOTHAUTH: %vtable = load ptr, ptr %arraydecay, align 8, !tbaa !7
+  // CHECK-BOTHAUTH: [[VTABLE_ADDRI:%.*]] = ptrtoint ptr %arraydecay to i64
+  // CHECK-BOTHAUTH: [[VTABLE_DISC:%.*]] = call i64 @llvm.ptrauth.blend(i64 %0, i64 48388)
+  // CHECK-BOTHAUTH: [[VTABLEI:%.*]] = ptrtoint ptr %vtable to i64
+  // CHECK-BOTHAUTH: [[AUTHENTICATED:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[VTABLEI]], i32 2, i64 [[VTABLE_DISC]])
+  return __builtin_get_vtable_pointer(array);
+}
+
+void test() {
+  A aInstance;
+  B bInstance;
+  C cInstance;
+  D dInstance;
+  E eInstance;
+  E eArray[] = { E() };
+  a(&aInstance);
+  a(&bInstance);
+  a((B *)&cInstance);
+  a(&dInstance);
+  a((D *)&eInstance);
+  a((B *)&eInstance);
+  b(&bInstance);
+  b(&cInstance);
+  b(&eInstance);
+  b_as_A(&bInstance);
+  c(&cInstance);
+  c_as_Z(&cInstance);
+  c_as_B(&cInstance);
+  d(&dInstance);
+  d(&eInstance);
+  d_as_A(&dInstance);
+  d_as_A(&eInstance);
+  e(&eInstance);
+  e_as_B(&eInstance);
+  e_as_D(&eInstance);
+  (void)__builtin_get_vtable_pointer(eArray);
+}
+} // namespace test1
diff --git a/external/llvm-project/clang/test/CodeGenCXX/debug-info-coff.cpp b/external/llvm-project/clang/test/CodeGenCXX/debug-info-coff.cpp
new file mode 100644
index 000000000000..2535c5cc7511
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGenCXX/debug-info-coff.cpp
@@ -0,0 +1,37 @@
+// REQUIRES: x86-registered-target
+
+// Check that CodeView compiler version is emitted even when debug info is otherwise disabled.
+
+// RUN: %clang --target=i686-pc-windows-msvc -S -emit-llvm %s -o - | FileCheck --check-prefix=IR %s
+// IR: !llvm.dbg.cu = !{!0}
+// IR: !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "{{.*}}", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, splitDebugInlining: false, nameTableKind: None)
+
+// RUN: %clang --target=i686-pc-windows-msvc -c %s -o %t.o
+// RUN: llvm-readobj --codeview %t.o | FileCheck %s
+// CHECK:      CodeViewDebugInfo [
+// CHECK-NEXT:   Section: .debug$S (4)
+// CHECK-NEXT:   Magic: 0x4
+// CHECK-NEXT:   Subsection [
+// CHECK-NEXT:     SubSectionType: Symbols (0xF1)
+// CHECK-NEXT:     SubSectionSize:
+// CHECK-NEXT:     ObjNameSym {
+// CHECK-NEXT:       Kind: S_OBJNAME (0x1101)
+// CHECK-NEXT:       Signature: 0x0
+// CHECK-NEXT:       ObjectName:
+// CHECK-NEXT:     }
+// CHECK-NEXT:     Compile3Sym {
+// CHECK-NEXT:       Kind: S_COMPILE3 (0x113C)
+// CHECK-NEXT:       Language: Cpp (0x1)
+// CHECK-NEXT:       Flags [ (0x0)
+// CHECK-NEXT:       ]
+// CHECK-NEXT:       Machine: Pentium3 (0x7)
+// CHECK-NEXT:       FrontendVersion:
+// CHECK-NEXT:       BackendVersion:
+// CHECK-NEXT:       VersionName: {{.*}}clang version
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ]
+// CHECK-NEXT: ]
+
+int main() {
+  return 0;
+}
diff --git a/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-aarch64.cpp b/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-aarch64.cpp
index 10fb1750f2c5..ff2dfc19961c 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-aarch64.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-aarch64.cpp
@@ -11,12 +11,9 @@
 // RUN: llvm-pdbutil dump -symbols %t.obj | FileCheck %s --check-prefix=HOTPATCH
 // HOTPATCH: S_COMPILE3 [size = [[#]]]
 // HOTPATCH: flags = hot patchable
-///
-/// Unfortunately we need /Z7, Clang does not systematically generate S_COMPILE3.
-///
+//
 // RUN: %clang_cl --target=aarch64-pc-windows-msvc /c -o %t.obj -- %s
-// RUN: llvm-pdbutil dump -symbols %t.obj | FileCheck %s --check-prefix=NO-HOTPATCH
-// NO-HOTPATCH-NOT: flags = hot patchable
+// RUN: llvm-pdbutil dump -symbols %t.obj | FileCheck %s --check-prefix=HOTPATCH
 
 int main() {
   return 0;
diff --git a/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-arm.cpp b/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-arm.cpp
index 48a61f7fb197..e31c762b0887 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-arm.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/debug-info-hotpatch-arm.cpp
@@ -11,12 +11,9 @@
 // RUN: llvm-pdbutil dump -symbols %t.obj | FileCheck %s --check-prefix=HOTPATCH
 // HOTPATCH: S_COMPILE3 [size = [[#]]]
 // HOTPATCH: flags = hot patchable
-///
-/// Unfortunately we need /Z7, Clang does not systematically generate S_COMPILE3.
-///
+//
 // RUN: %clang_cl --target=arm-pc-windows-msvc /c -o %t.obj -- %s
-// RUN: llvm-pdbutil dump -symbols %t.obj | FileCheck %s --check-prefix=NO-HOTPATCH
-// NO-HOTPATCH-NOT: flags = hot patchable
+// RUN: llvm-pdbutil dump -symbols %t.obj | FileCheck %s --check-prefix=HOTPATCH
 
 int main() {
   return 0;
diff --git a/external/llvm-project/clang/test/CodeGenCXX/dependent-template-alias.cpp b/external/llvm-project/clang/test/CodeGenCXX/dependent-template-alias.cpp
index deb243f9fc88..324b16f1919d 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/dependent-template-alias.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/dependent-template-alias.cpp
@@ -1,9 +1,6 @@
 // RUN: %clang_cc1 -triple x86_64-unk-unk -o - -emit-llvm -debug-info-kind=standalone -gtemplate-alias %s -gsimple-template-names=simple \
 // RUN: | FileCheck %s
 
-//// Check that -gtemplate-alias falls back to DW_TAG_typedef emission
-//// for instantiation dependent type aliases.
-
 template <int>
 using A = int;
 
@@ -16,6 +13,6 @@ struct S {
 S<0> s;
 
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "aa", scope: ![[#]], file: ![[#]], line: [[#]], baseType: ![[AA:[0-9]+]], size: 32)
-// CHECK: [[AA]] = !DIDerivedType(tag: DW_TAG_typedef, name: "AA", file: ![[#]], line: [[#]], baseType: ![[A:[0-9]+]])
-// CHECK: [[A]] = !DIDerivedType(tag: DW_TAG_typedef, name: "A<I>", file: ![[#]], line: [[#]], baseType: ![[int:[0-9]+]])
+// CHECK: [[AA]] = !DIDerivedType(tag: DW_TAG_typedef, name: "AA", scope: ![[#]], file: ![[#]], line: [[#]], baseType: ![[A:[0-9]+]])
+// CHECK: [[A]] = !DIDerivedType(tag: DW_TAG_template_alias, name: "A", file: ![[#]], line: [[#]], baseType: ![[int:[0-9]+]], extraData: ![[#]])
 // CHECK: [[int]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
diff --git a/external/llvm-project/clang/test/CodeGenCXX/gnu-asm-constexpr.cpp b/external/llvm-project/clang/test/CodeGenCXX/gnu-asm-constexpr.cpp
index d9c633ea6928..175a3b7bc588 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/gnu-asm-constexpr.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/gnu-asm-constexpr.cpp
@@ -16,6 +16,12 @@ struct string_view {
     }
 };
 
+namespace GH143242 {
+    constexpr string_view code2 = R"(nop; nop; nop; nop)";
+    asm((code2));
+    // CHECK: module asm "nop; nop; nop; nop"
+}
+
 int func() {return 0;};
 
 void f() {
@@ -49,4 +55,4 @@ void test_srcloc() {
         foobar)o")
 
     ) ::(string_view("r"))(func()));
-  }
+}
diff --git a/external/llvm-project/clang/test/CodeGenCXX/ms-mangle-requires.cpp b/external/llvm-project/clang/test/CodeGenCXX/ms-mangle-requires.cpp
new file mode 100644
index 000000000000..15318ecffaf0
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGenCXX/ms-mangle-requires.cpp
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -std=c++20 -triple=x86_64-windows-msvc -Wno-defaulted-function-deleted -fms-compatibility -fms-extensions -emit-llvm %s -o - | FileCheck %s
+
+namespace CWG2369 {
+
+namespace Regression1 {
+
+template <class, class Up>
+using compare_three_way_result_t = Up::type;
+
+struct sfinae_assign_base {};
+
+template <class Tp>
+concept is_derived_from_optional =
+    requires(Tp param) { []<class Up>(Up) {}(param); };
+
+template <class Tp, class Up>
+  requires(is_derived_from_optional<Up> && []<class W>(W) { return true; }(Up()))
+compare_three_way_result_t<Tp, Up> operator<=>(Tp, Up);
+
+struct RuntimeModeArgs {
+  auto operator<=>(const RuntimeModeArgs &) const = default;
+  sfinae_assign_base needs_admin;
+};
+
+RuntimeModeArgs foo() {
+  return {};
+}
+
+// CHECK: ?foo@Regression1@CWG2369@@YA?AURuntimeModeArgs@12@XZ
+
+} // namespace Regression1
+
+namespace Regression2 {
+
+template <class _Tp>
+constexpr _Tp * __to_address(_Tp *) {
+  return nullptr;
+}
+
+template <class _Ip>
+concept contiguous_iterator = requires(_Ip __i) { __to_address(__i); };
+
+struct basic_string_view {
+  template <contiguous_iterator _It>
+  basic_string_view(_It, _It);
+};
+
+const char *str;
+void sv() { basic_string_view(str, str); }
+
+void m_fn2() {
+  const char __trans_tmp_1 = *__to_address(&__trans_tmp_1);
+}
+
+// CHECK: define {{.*}} @"??$__to_address@$$CBD@Regression2@CWG2369@@YAPEBDPEBD@Z"
+
+} // namespace Regression2
+
+}
diff --git a/external/llvm-project/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp b/external/llvm-project/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
index 7d6de50d926b..daeea77774ec 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
@@ -99,7 +99,7 @@ void testMoveAssignment(SA a) {
   t = static_cast<SA &&>(a);
 }
 
-// CHECK: define {{.*}}void @_Z19testCopyConstructor2SI(i
+// CHECK: define {{.*}}void @_Z19testCopyConstructor2SI(
 // CHECK: call void @llvm.memcpy.p0.p0.i64(
 
 void testCopyConstructor(SI a) {
diff --git a/external/llvm-project/clang/test/CodeGenCXX/tmp-md-nodes2.cpp b/external/llvm-project/clang/test/CodeGenCXX/tmp-md-nodes2.cpp
index e5081884447b..8500cf3c4239 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/tmp-md-nodes2.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/tmp-md-nodes2.cpp
@@ -1,8 +1,6 @@
 // REQUIRES: asserts
 // RUN: %clang_cc1 -O0 -triple %itanium_abi_triple -debug-info-kind=limited -emit-llvm %s -o - | \
 // RUN: FileCheck %s
-// RUN: %clang_cc1 -O0 -triple %itanium_abi_triple -debug-info-kind=limited -emit-llvm -mllvm --experimental-debuginfo-iterators=true %s -o - | \
-// RUN: FileCheck %s
 
 // This test simply checks that the varargs thunk is created. The failing test
 // case asserts.
diff --git a/external/llvm-project/clang/test/CodeGenCXX/trivial_abi.cpp b/external/llvm-project/clang/test/CodeGenCXX/trivial_abi.cpp
index 90054dbf37ae..b8cc0d1cc652 100644
--- a/external/llvm-project/clang/test/CodeGenCXX/trivial_abi.cpp
+++ b/external/llvm-project/clang/test/CodeGenCXX/trivial_abi.cpp
@@ -68,11 +68,10 @@ struct D0 : B0, B1 {
 
 Small D0::m0() { return {}; }
 
-// CHECK: define{{.*}} void @_Z14testParamSmall5Small(i64 %[[A_COERCE:.*]])
+// CHECK: define{{.*}} void @_Z14testParamSmall5Small(ptr %[[A_COERCE:.*]])
 // CHECK: %[[A:.*]] = alloca %[[STRUCT_SMALL]], align 8
 // CHECK: %[[COERCE_DIVE:.*]] = getelementptr inbounds nuw %[[STRUCT_SMALL]], ptr %[[A]], i32 0, i32 0
-// CHECK: %[[COERCE_VAL_IP:.*]] = inttoptr i64 %[[A_COERCE]] to ptr
-// CHECK: store ptr %[[COERCE_VAL_IP]], ptr %[[COERCE_DIVE]], align 8
+// CHECK: store ptr %[[A_COERCE]], ptr %[[COERCE_DIVE]], align 8
 // CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5SmallD1Ev(ptr {{[^,]*}} %[[A]])
 // CHECK: ret void
 // CHECK: }
@@ -101,8 +100,7 @@ Small testReturnSmall() {
 // CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN5SmallC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(8) %[[T]])
 // CHECK: %[[COERCE_DIVE:.*]] = getelementptr inbounds nuw %[[STRUCT_SMALL]], ptr %[[AGG_TMP]], i32 0, i32 0
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[COERCE_DIVE]], align 8
-// CHECK: %[[COERCE_VAL_PI:.*]] = ptrtoint ptr %[[V0]] to i64
-// CHECK: call void @_Z14testParamSmall5Small(i64 %[[COERCE_VAL_PI]])
+// CHECK: call void @_Z14testParamSmall5Small(ptr %[[V0]])
 // CHECK: %[[CALL2:.*]] = call noundef ptr @_ZN5SmallD1Ev(ptr {{[^,]*}} %[[T]])
 // CHECK: ret void
 // CHECK: }
@@ -120,8 +118,7 @@ void testCallSmall0() {
 // CHECK: store ptr %[[COERCE_VAL_IP]], ptr %[[COERCE_DIVE]], align 8
 // CHECK: %[[COERCE_DIVE1:.*]] = getelementptr inbounds nuw %[[STRUCT_SMALL]], ptr %[[AGG_TMP]], i32 0, i32 0
 // CHECK: %[[V0:.*]] = load ptr, ptr %[[COERCE_DIVE1]], align 8
-// CHECK: %[[COERCE_VAL_PI:.*]] = ptrtoint ptr %[[V0]] to i64
-// CHECK: call void @_Z14testParamSmall5Small(i64 %[[COERCE_VAL_PI]])
+// CHECK: call void @_Z14testParamSmall5Small(ptr %[[V0]])
 // CHECK: ret void
 // CHECK: }
 
@@ -226,7 +223,7 @@ NonTrivial testReturnHasNonTrivial() {
 // CHECK: call noundef ptr @_ZN5SmallC1Ev(ptr {{[^,]*}} %[[AGG_TMP]])
 // CHECK: invoke noundef ptr @_ZN5SmallC1Ev(ptr {{[^,]*}} %[[AGG_TMP1]])
 
-// CHECK: call void @_Z20calleeExceptionSmall5SmallS_(i64 %{{.*}}, i64 %{{.*}})
+// CHECK: call void @_Z20calleeExceptionSmall5SmallS_(ptr %{{.*}}, ptr %{{.*}})
 // CHECK-NEXT: ret void
 
 // CHECK: landingpad { ptr, i32 }
diff --git a/external/llvm-project/clang/test/CodeGenHLSL/RootSignature.hlsl b/external/llvm-project/clang/test/CodeGenHLSL/RootSignature.hlsl
index 34caa3eb6b71..ca843ffbb1ce 100644
--- a/external/llvm-project/clang/test/CodeGenHLSL/RootSignature.hlsl
+++ b/external/llvm-project/clang/test/CodeGenHLSL/RootSignature.hlsl
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -o - %s | FileCheck %s
 
 // CHECK: !dx.rootsignatures = !{![[#EMPTY_ENTRY:]], ![[#DT_ENTRY:]],
-// CHECK-SAME: ![[#RF_ENTRY:]], ![[#RC_ENTRY:]], ![[#RD_ENTRY:]]}
+// CHECK-SAME: ![[#RF_ENTRY:]], ![[#RC_ENTRY:]], ![[#RD_ENTRY:]], ![[#SS_ENTRY:]]}
 
 // CHECK: ![[#EMPTY_ENTRY]] = !{ptr @EmptyEntry, ![[#EMPTY:]]}
 // CHECK: ![[#EMPTY]] = !{}
@@ -66,6 +66,40 @@ void RootConstantsEntry() {}
 [numthreads(1,1,1)]
 void RootDescriptorsEntry() {}
 
+// CHECK: ![[#SS_ENTRY]] = !{ptr @StaticSamplerEntry, ![[#SS_RS:]]}
+// CHECK: ![[#SS_RS]] = !{![[#STATIC_SAMPLER:]]}
+
+// checking filter = 0x4
+// CHECK: ![[#STATIC_SAMPLER]] = !{!"StaticSampler", i32 4,
+
+// checking texture address[U|V|W]
+// CHECK-SAME: i32 2, i32 3, i32 5,
+
+// checking mipLODBias, maxAnisotropy, comparisonFunc, borderColor
+// CHECK-SAME: float 0x40403999A0000000, i32 9, i32 3, i32 2,
+
+// checking minLOD, maxLOD
+// CHECK-SAME: float -1.280000e+02, float 1.280000e+02,
+
+// checking register, space and visibility
+// CHECK-SAME: i32 42, i32 0, i32 0}
+
+#define SampleStaticSampler \
+  "StaticSampler(s42, " \
+  " filter = FILTER_MIN_POINT_MAG_LINEAR_MIP_POINT, " \
+  " addressU = TEXTURE_ADDRESS_MIRROR, " \
+  " addressV = TEXTURE_ADDRESS_CLAMP, " \
+  " addressW = TEXTURE_ADDRESS_MIRRORONCE, " \
+  " mipLODBias = 32.45f, maxAnisotropy = 9, " \
+  " comparisonFunc = COMPARISON_EQUAL, " \
+  " borderColor = STATIC_BORDER_COLOR_OPAQUE_WHITE, " \
+  " minLOD = -128.f, maxLOD = 128.f, " \
+  " space = 0, visibility = SHADER_VISIBILITY_ALL, " \
+  ")"
+[shader("compute"), RootSignature(SampleStaticSampler)]
+[numthreads(1,1,1)]
+void StaticSamplerEntry() {}
+
 // Sanity test to ensure no root is added for this function as there is only
 // two entries in !dx.roosignatures
 [shader("compute")]
diff --git a/external/llvm-project/clang/test/CodeGenHLSL/builtins/wave_get_lane_count.hlsl b/external/llvm-project/clang/test/CodeGenHLSL/builtins/wave_get_lane_count.hlsl
new file mode 100644
index 000000000000..8072f6d4ea20
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGenHLSL/builtins/wave_get_lane_count.hlsl
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple   \
+// RUN:   spirv-pc-vulkan-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN:   FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple   \
+// RUN:   dxil-pc-shadermodel6.3-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN:   FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
+
+[numthreads(1, 1, 1)]
+void main() {
+  uint a, b;
+// CHECK-SPIRV: %[[#entry_tok:]] = call token @llvm.experimental.convergence.entry()
+
+// CHECK-SPIRV: %[[#loop_tok:]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %[[#entry_tok]]) ]
+  while (a) {
+
+// CHECK-DXIL:  %[[#]] = call i32 @llvm.dx.wave.get.lane.count()
+// CHECK-SPIRV: %[[#]] = call spir_func i32 @llvm.spv.wave.get.lane.count()
+// CHECK-SPIRV-SAME: [ "convergencectrl"(token %[[#loop_tok]]) ]
+    a = WaveGetLaneCount();
+  }
+
+// CHECK-DXIL:  %[[#]] = call i32 @llvm.dx.wave.get.lane.count()
+// CHECK-SPIRV: %[[#]] = call spir_func i32 @llvm.spv.wave.get.lane.count()
+// CHECK-SPIRV-SAME: [ "convergencectrl"(token %[[#entry_tok]]) ]
+  b = WaveGetLaneCount();
+}
+
+// CHECK-DXIL:  i32 @llvm.dx.wave.get.lane.count() #[[#attr:]]
+// CHECK-SPIRV: i32 @llvm.spv.wave.get.lane.count() #[[#attr:]]
+
+// CHECK: attributes #[[#attr]] = {{{.*}} convergent {{.*}}}
+
diff --git a/external/llvm-project/clang/test/CodeGenHLSL/group_shared.hlsl b/external/llvm-project/clang/test/CodeGenHLSL/group_shared.hlsl
index 4b2e2beba4f1..a562e75b3488 100644
--- a/external/llvm-project/clang/test/CodeGenHLSL/group_shared.hlsl
+++ b/external/llvm-project/clang/test/CodeGenHLSL/group_shared.hlsl
@@ -3,6 +3,10 @@
 // RUN:   dxil-pc-shadermodel6.3-library %s \
 // RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
 
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan1.3-compute %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
 // Make sure groupshared translated into address space 3.
 // CHECK:@a = addrspace(3) global [10 x float]
 
diff --git a/external/llvm-project/clang/test/CodeGenHLSL/semantics/SV_Position.ps.hlsl b/external/llvm-project/clang/test/CodeGenHLSL/semantics/SV_Position.ps.hlsl
new file mode 100644
index 000000000000..58b91fc9264d
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGenHLSL/semantics/SV_Position.ps.hlsl
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-pixel -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s | FileCheck %s
+
+// CHECK: @sv_position = external thread_local addrspace(7) externally_initialized constant <4 x float>, !spirv.Decorations !0
+
+// CHECK: define void @main() {{.*}} {
+float4 main(float4 p : SV_Position) {
+  // CHECK: %[[#P:]] = load <4 x float>, ptr addrspace(7) @sv_position, align 16
+  // CHECK: %[[#R:]] = call spir_func <4 x float> @_Z4mainDv4_f(<4 x float> %[[#P]])
+  return p;
+}
diff --git a/external/llvm-project/clang/test/CodeGenHLSL/vk-input-builtin.hlsl b/external/llvm-project/clang/test/CodeGenHLSL/vk-input-builtin.hlsl
new file mode 100644
index 000000000000..1cc7963c0e28
--- /dev/null
+++ b/external/llvm-project/clang/test/CodeGenHLSL/vk-input-builtin.hlsl
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan1.3-compute %s -emit-llvm -O3 -o - | FileCheck %s
+
+[[vk::ext_builtin_input(/* WorkgroupId */ 26)]]
+static const uint3 groupid;
+// CHECK: @_ZL7groupid = external local_unnamed_addr addrspace(7) externally_initialized constant <3 x i32>, align 16, !spirv.Decorations [[META0:![0-9]+]]
+
+RWStructuredBuffer<int> output : register(u1, space0);
+
+[numthreads(1, 1, 1)]
+void main() {
+  output[0] = groupid;
+}
+// CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+// CHECK: [[META1]] = !{i32 11, i32 26}
diff --git a/external/llvm-project/clang/test/CodeGenHipStdPar/select-accelerator-code-pass-ordering.cpp b/external/llvm-project/clang/test/CodeGenHipStdPar/select-accelerator-code-pass-ordering.cpp
index 44557284fc58..f03b783199a4 100644
--- a/external/llvm-project/clang/test/CodeGenHipStdPar/select-accelerator-code-pass-ordering.cpp
+++ b/external/llvm-project/clang/test/CodeGenHipStdPar/select-accelerator-code-pass-ordering.cpp
@@ -7,6 +7,7 @@
 // HIPSTDPAR-PRE: Running pass: EntryExitInstrumenterPass
 // HIPSTDPAR-PRE-NEXT: Running pass: EntryExitInstrumenterPass
 // HIPSTDPAR-PRE-NOT: Running pass: HipStdParAcceleratorCodeSelectionPass
+// HIPSTDPAR-PRE-NEXT: Running pass: AMDGPUExpandFeaturePredicatesPass
 // HIPSTDPAR-PRE-NEXT: Running pass: AlwaysInlinerPass
 
 // Ensure Pass HipStdParAcceleratorCodeSelectionPass is invoked in PostLink.
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/agg.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/agg.c
index 33b3a0141a0f..58d923d1d932 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/agg.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/agg.c
@@ -24,6 +24,8 @@ void fun(Struct a) {
 // CHECK: %matins = insertelement <25 x float> %3, float 0.000000e+00, i64 0, !dbg [[G4R2:!.*]]
 // CHECK: store <25 x float> %matins, ptr @m{{.*}}, !dbg [[G4R1:!.*]]
   m[0][0] = 0;
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
@@ -32,3 +34,4 @@ void fun(Struct a) {
 // CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
 // CHECK: [[G4R2]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 2)
 // CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/assign-scalar.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/assign-scalar.c
index afb0ed8ebdfd..0bba1830e42a 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/assign-scalar.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/assign-scalar.c
@@ -58,6 +58,7 @@ void fun() {
 // CHECK-NEXT: %inc4 = add i64 %7, 1, !dbg [[G11R2:!.*]]
 // CHECK-NEXT: store i64 %inc4, ptr @i{{.*}}, !dbg [[G11R1:!.*]]
     g = ++h, ++i;
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
@@ -83,3 +84,4 @@ void fun() {
 // CHECK: [[load_i_loc]] = !DILocation(line: [[#]], column: [[#]], scope: ![[#]])
 // CHECK: [[G11R2]] = !DILocation({{.*}}, atomGroup: 11, atomRank: 2)
 // CHECK: [[G11R1]] = !DILocation({{.*}}, atomGroup: 11, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 12, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/bitfield.cpp b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/bitfield.cpp
index a14b4fbaf585..e1d4c264a426 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/bitfield.cpp
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/bitfield.cpp
@@ -10,7 +10,10 @@ void foo(int x, S s) {
 // CHECK: %bf.set = or i8 %bf.clear, %bf.value, !dbg [[G1R2:!.*]]
 // CHECK: store i8 %bf.set, ptr %s, align 4, !dbg [[G1R1:!.*]]
   s.a = x;
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/builtin.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/builtin.c
index 359d52ca3862..ce8c6124fe92 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/builtin.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/builtin.c
@@ -64,6 +64,7 @@ void fun() {
 // CHECK-NEXT: %4 = trunc i32 %3 to i8, !dbg [[G15R2:!.*]]
 // CHECK-NEXT: call void @llvm.memset{{.*}}, !dbg [[G15R1:!.*]]
     __builtin_memset(f4, v, sizeof(float) * 4);
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
@@ -86,3 +87,4 @@ void fun() {
 // CHECK: [[G15R3]] = !DILocation({{.*}}, atomGroup: 15, atomRank: 3)
 // CHECK: [[G15R2]] = !DILocation({{.*}}, atomGroup: 15, atomRank: 2)
 // CHECK: [[G15R1]] = !DILocation({{.*}}, atomGroup: 15, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 16, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/cast.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/cast.c
new file mode 100644
index 000000000000..62f2c3d1d3e5
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/cast.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -gno-column-info -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o -  \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -gno-column-info -x c %s -debug-info-kind=line-tables-only -emit-llvm -o -  \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// Check that when a cast is a Key Instruction backup we add its operand
+// instruction, if there is one, to the source atom too.
+
+float g;
+void a() {
+// CHECK: %0 = load float, ptr @g{{.*}}, !dbg [[G1R3:!.*]]
+// CHECK: %conv = fptosi float %0 to i32{{.*}}, !dbg [[G1R2:!.*]]
+// CHECK: store i32 %conv, ptr %a{{.*}}, !dbg [[G1R1:!.*]]
+    int a = g;
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+// CHECK: [[G1R3]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 3)
+// CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-packed.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-packed.c
new file mode 100644
index 000000000000..1bd276014970
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-packed.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple arm64-apple-ios11 \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple arm64-apple-ios11 \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+typedef struct {
+  char a;
+  int x;
+} __attribute((packed)) S;
+
+S getS();
+void f() {
+// CHECK: [[call:%.*]] = call i40{{.*}}getS{{.*}}, !dbg [[G1R2:!.*]]
+// CHECK: store i40 [[call]], ptr %s, align 1, !dbg [[G1R1:!.*]]
+    S s = getS();
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+// CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-ptr.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-ptr.c
new file mode 100644
index 000000000000..1205b312e43c
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-ptr.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple x86_64-windows-msvc \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple x86_64-windows-msvc \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+typedef struct { int *p; } Ptr;
+Ptr getPtr();
+void f() {
+// CHECK: %call = call i64{{.*}}, !dbg [[G1R3:!.*]]
+// CHECK: [[gep:%.*]] = getelementptr inbounds nuw %struct.Ptr, ptr %p, i32 0, i32 0
+// CHECK: [[i2p:%.*]] = inttoptr i64 %call to ptr, !dbg [[G1R2:!.*]]
+// CHECK: store ptr [[i2p]], ptr [[gep]], align 8, !dbg [[G1R1:!.*]]
+    Ptr p = getPtr();
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+// CHECK: [[G1R3]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 3)
+// CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-through-memory.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-through-memory.c
new file mode 100644
index 000000000000..f8667b73b111
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced-through-memory.c
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple aarch64-windows-msvc \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple aarch64-windows-msvc \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+typedef struct {
+  short a;
+  int b;
+  short c;
+} S;
+
+S getS(void);
+
+void f() {
+// CHECK: %call = call [2 x i64] {{.*}}getS{{.*}}(), !dbg [[G1R2:!.*]]
+//// Note: The store to the tmp alloca isn't part of the atom.
+// CHECK: store [2 x i64] %call, ptr %tmp.coerce, align 8
+// CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %s, ptr align 8 %tmp.coerce, i64 12, i1 false), !dbg [[G1R1:!.*]]
+  S s = getS();
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+// CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced.c
new file mode 100644
index 000000000000..036f6d66fa67
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/coerced.c
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple x86_64-unknown-linux \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-CXX
+
+// RUN: %clang_cc1 -gkey-instructions -gno-column-info -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - -triple x86_64-unknown-linux \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-C
+
+typedef struct {
+  void* a;
+  void* b;
+} Struct;
+Struct get();
+
+void test() {
+// CHECK: %1 = extractvalue { ptr, ptr } %call, 0, !dbg [[G1R2:!.*]]
+// CHECK: store ptr %1, ptr {{.*}}, !dbg [[G1R1:!.*]]
+// CHECK: %3 = extractvalue { ptr, ptr } %call, 1, !dbg [[G1R2]]
+// CHECK: store ptr %3, ptr {{.*}}, !dbg [[G1R1:!.*]]
+  Struct s = get();
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+typedef struct { int i; } Int;
+Int getInt(void);
+
+// CHECK-C: @test2
+// CHECK-CXX: @_Z5test2v
+void test2() {
+// CHECK: %call = call i32 @{{(_Z6)?}}getInt{{v?}}(), !dbg [[T2_G1R2:!.*]]
+// CHECK: [[gep:%.*]] = getelementptr inbounds nuw %struct.Int, ptr %i, i32 0, i32 0
+// CHECK: store i32 %call, ptr [[gep]]{{.*}}, !dbg [[T2_G1R1:!.*]]
+// CHECK: ret{{.*}}, !dbg [[T2_RET:!.*]]
+  Int i = getInt();
+}
+
+// CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[T2_G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[T2_G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[T2_RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/complex.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/complex.c
index 9647d0e4009e..d5cb67fed99d 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/complex.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/complex.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - \
-// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-CXX
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - \
 // RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-C
@@ -71,6 +71,7 @@ void test() {
 // CHECK-C: store float %mul.rl, ptr @f, align 4, !dbg [[G11R1:!.*]]
   f *= ci;
 #endif
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
@@ -95,3 +96,5 @@ void test() {
 // CHECK-C: [[G10R1]] = !DILocation({{.*}}, atomGroup: 10, atomRank: 1)
 // CHECK-C: [[G11R2]] = !DILocation({{.*}}, atomGroup: 11, atomRank: 2)
 // CHECK-C: [[G11R1]] = !DILocation({{.*}}, atomGroup: 11, atomRank: 1)
+// CHECK-C: [[RET]] = !DILocation({{.*}}, atomGroup: 12, atomRank: 1)
+// CHECK-CXX: [[RET]] = !DILocation({{.*}}, atomGroup: 8, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/do.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/do.c
index 4f0d388f9404..cc3749e6e56c 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/do.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/do.c
@@ -25,9 +25,12 @@ void a(int A) {
 // CHECK: %tobool = icmp ne i32 %dec, 0, !dbg [[G2R1:!.*]]
 // CHECK: br i1 %tobool, label %do.body, label %do.end, !dbg [[G3R1:!.*]], !llvm.loop
     do { } while (--A);
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
 // CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
 // CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/for-range.cpp b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/for-range.cpp
new file mode 100644
index 000000000000..433f900219f1
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/for-range.cpp
@@ -0,0 +1,99 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu  -gkey-instructions %s -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s
+
+// Perennial question: should the inc be its own source atom or not
+// (currently it is).
+
+// FIXME: See do.c and while.c regarding cmp and cond br groups.
+
+// The stores in the setup (stores to __RANGE1, __BEGIN1, __END1) are all
+// marked as Key. Unclear whether that's desirable. Keep for now as that's
+// least risky (at worst it introduces an unnecessary step while debugging,
+// as opposed to potentially losing one we want).
+
+// Check the conditional branch (and the condition) in FOR_COND and
+// unconditional branch in FOR_BODY are Key Instructions.
+
+struct Range {
+    int *begin();
+    int *end();
+} r;
+
+// CHECK-LABEL: define dso_local void @_Z1av(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] !dbg [[DBG10:![0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[__RANGE1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[__BEGIN1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[__END1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store ptr @r, ptr [[__RANGE1]], align 8, !dbg [[DBG14:![0-9]+]]
+// CHECK-NEXT:    [[CALL:%.*]] = call noundef ptr @_ZN5Range5beginEv(ptr noundef nonnull align 1 dereferenceable(1) @r), !dbg [[DBG15:![0-9]+]]
+// CHECK-NEXT:    store ptr [[CALL]], ptr [[__BEGIN1]], align 8, !dbg [[DBG16:![0-9]+]]
+// CHECK-NEXT:    [[CALL1:%.*]] = call noundef ptr @_ZN5Range3endEv(ptr noundef nonnull align 1 dereferenceable(1) @r), !dbg [[DBG17:![0-9]+]]
+// CHECK-NEXT:    store ptr [[CALL1]], ptr [[__END1]], align 8, !dbg [[DBG18:![0-9]+]]
+// CHECK-NEXT:    br label %[[FOR_COND:.*]], !dbg [[DBG19:![0-9]+]]
+// CHECK:       [[FOR_COND]]:
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !dbg [[DBG19]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__END1]], align 8, !dbg [[DBG19]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[TMP0]], [[TMP1]], !dbg [[DBG20:![0-9]+]]
+// CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]], !dbg [[DBG21:![0-9]+]]
+// CHECK:       [[FOR_BODY]]:
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !dbg [[DBG19]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4, !dbg [[DBG22:![0-9]+]]
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[I]], align 4, !dbg [[DBG23:![0-9]+]]
+// CHECK-NEXT:    br label %[[FOR_INC:.*]], !dbg [[DBG24:![0-9]+]]
+// CHECK:       [[FOR_INC]]:
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !dbg [[DBG19]]
+// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP4]], i32 1, !dbg [[DBG25:![0-9]+]]
+// CHECK-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__BEGIN1]], align 8, !dbg [[DBG26:![0-9]+]]
+// CHECK-NEXT:    br label %[[FOR_COND]], !dbg [[DBG19]], !llvm.loop
+// CHECK:       [[FOR_END]]:
+// CHECK-NEXT:    ret void, !dbg [[DBG30:![0-9]+]]
+//
+void a() {
+  for (int i: r)
+    ;
+}
+
+// - Check the branch out of the body gets an atom group (and gets it correct
+// if there's ctrl-flow in the body).
+void b() {
+  for (int i: r) {
+    if (i)
+      ;
+// CHECK: entry:
+// CHECK: if.end:
+// CHECK-NEXT: br label %for.inc, !dbg [[b_br:!.*]]
+  }
+}
+
+// - Check an immediate loop exit (`break` in this case) gets an atom group and
+// - doesn't cause a crash.
+void c() {
+  for (int i: r)
+// CHECK: entry:
+// CHECK: for.body:
+// CHECK: br label %for.end, !dbg [[c_br:!.*]]
+    break;
+}
+
+//.
+// CHECK: [[DBG14]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[DBG15]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
+// CHECK: [[DBG16]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[DBG17]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 2)
+// CHECK: [[DBG18]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[DBG19]] = !DILocation({{.*}})
+// CHECK: [[DBG20]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK: [[DBG21]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
+// CHECK: [[DBG22]] = !DILocation({{.*}}, atomGroup: 6, atomRank: 2)
+// CHECK: [[DBG23]] = !DILocation({{.*}}, atomGroup: 6, atomRank: 1)
+// CHECK: [[DBG24]] = !DILocation({{.*}} atomGroup: 8, atomRank: 1)
+// CHECK: [[DBG25]] = !DILocation({{.*}}, atomGroup: 7, atomRank: 2)
+// CHECK: [[DBG26]] = !DILocation({{.*}}, atomGroup: 7, atomRank: 1)
+// CHECK: [[DBG30]] = !DILocation({{.*}})
+//
+// CHECK: [[b_br]] = !DILocation({{.*}}, atomGroup: [[#]], atomRank: [[#]])
+//
+// CHECK: [[c_br]] = !DILocation({{.*}}, atomGroup: [[#]], atomRank: [[#]])
+//.
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/for.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/for.c
new file mode 100644
index 000000000000..e7c1567c14d6
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/for.c
@@ -0,0 +1,183 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c %s -debug-info-kind=line-tables-only -emit-llvm -o -  \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// Perennial question: should the inc be its own source atom or not
+// (currently it is).
+
+// TODO: See do.c and while.c regarding cmp and cond br groups.
+
+void a(int A) {
+// CHECK: entry:
+// CHECK: store i32 0, ptr %i{{.*}}, !dbg [[G1R1:!.*]]
+// CHECK: for.cond:
+// CHECK: %cmp = icmp slt i32 %0, %1, !dbg [[G2R1:!.*]]
+// CHECK: br i1 %cmp, label %for.body, label %for.end, !dbg [[G3R1:!.*]]
+
+// TODO: The unconditional br is given an atom group here which is useful for
+// O0. Since we're no longer targeting O0 we should reevaluate whether this
+// adds any value.
+// CHECK: for.body:
+// CHECK-NEXT: br label %for.inc, !dbg [[G5R1:!.*]]
+
+// CHECK: for.inc:
+// CHECK: %inc = add{{.*}}, !dbg [[G4R2:!.*]]
+// CHECK: store i32 %inc, ptr %i{{.*}}, !dbg [[G4R1:!.*]]
+  for (int i = 0; i < A; ++i) {
+  }
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+void b(int A) {
+// CHECK: entry:
+// CHECK: store i32 0, ptr %i{{.*}}, !dbg [[bG1R1:!.*]]
+// CHECK: for.cond:
+// CHECK: %cmp = icmp slt i32 %0, %1, !dbg [[bG2R1:!.*]]
+// CHECK: br i1 %cmp, label %for.body, label %for.end, !dbg [[bG3R1:!.*]]
+
+// CHECK: for.body:
+// CHECK-NEXT: %2 = load i32, ptr %A.addr
+// - If stmt atom:
+// CHECK-NEXT: %cmp1 = icmp sgt i32 %2, 1, !dbg [[bG4R2:!.*]]
+// CHECK-NEXT: br i1 %cmp1, label %if.then, label %if.end, !dbg [[bG4R1:!.*]]
+// CHECK: if.then:
+// CHECK-NEXT: br label %if.end
+
+// - For closing brace.
+// CHECK: if.end:
+// CHECK-NEXT: br label %for.inc, !dbg [[bG6R1:!.*]]
+
+// CHECK: for.inc:
+// CHECK: %inc = add{{.*}}, !dbg [[bG5R2:!.*]]
+// CHECK: store i32 %inc, ptr %i{{.*}}, !dbg [[bG5R1:!.*]]
+  for (int i = 0; i < A; ++i) {
+    if (A > 1)
+      ;
+  }
+// CHECK: ret{{.*}}, !dbg [[bRET:!.*]]
+}
+
+void c(int A) {
+// CHECK: entry:
+// CHECK: for.cond:
+// CHECK-NEXT: %0 = load i32, ptr %A.addr
+// - If stmt atom:
+// CHECK-NEXT: %cmp = icmp sgt i32 %0, 1, !dbg [[cG1R2:!.*]]
+// CHECK-NEXT: br i1 %cmp, label %if.then, label %if.end, !dbg [[cG1R1:!.*]]
+// CHECK: if.then:
+// CHECK-NEXT: br label %if.end
+
+// - For closing brace.
+// CHECK: if.end:
+// CHECK-NEXT: br label %for.inc, !dbg [[cG3R1:!.*]]
+
+// CHECK: for.inc:
+// CHECK-NEXT: %1 = load i32, ptr %A.addr
+// CHECK-NEXT: %inc = add{{.*}}, !dbg [[cG2R2:!.*]]
+// CHECK-NEXT: store i32 %inc, ptr %A.addr{{.*}}, !dbg [[cG2R1:!.*]]
+  for (; /*no cond*/ ; ++A) {
+    if (A > 1)
+      ;
+  }
+}
+
+void d() {
+// - Check the `for` keyword gets an atom group.
+// CHECK: entry:
+// CHECK-NEXT: br label %for.cond
+
+// CHECK: for.cond:
+// CHECK: br label %for.cond, !dbg [[dG1R1:!.*]], !llvm.loop
+  for ( ; ; )
+  {
+  }
+}
+
+int x, i;
+void ee();
+void e() {
+// - Check we assign an atom group to `for.body`s `br`, even without braces.
+// - TODO: Investigate whether this is needed.
+// CHECK: entry:
+
+// CHECK: for.cond:
+// CHECK-NEXT: %0 = load i32, ptr @i
+// CHECK-NEXT: %cmp = icmp slt i32 %0, 3, !dbg [[eG1R1:!.*]]
+// CHECK-NEXT: br i1 %cmp, label %for.body, label %for.end, !dbg [[eG2R1:!.*]]
+
+// CHECK: for.body:
+// CHECK-NEXT: %1 = load i32, ptr @i{{.*}}, !dbg [[eG3R2:!.*]]
+// CHECK-NEXT: store i32 %1, ptr @x{{.*}}, !dbg [[eG3R1:!.*]]
+// CHECK-NEXT: br label %for.inc, !dbg [[eG4R1:!.*]]
+  for (; i < 3; ee())
+    x = i;
+// CHECK: ret{{.*}}, !dbg [[eRET:!.*]]
+}
+
+
+void f() {
+// - Check the `continue` keyword gets an atom group.
+// CHECK: entry:
+// CHECK-NEXT: br label %for.cond
+
+// CHECK: for.cond:
+// CHECK: br label %for.cond, !dbg [[fG1R1:!.*]], !llvm.loop
+  for ( ; ; )
+  {
+    continue;
+  }
+}
+
+void g() {
+// - Check the `break` keyword gets an atom group.
+// CHECK: entry:
+// CHECK-NEXT: br label %for.cond
+
+// CHECK: for.cond:
+// CHECK: br label %for.end, !dbg [[gG1R1:!.*]]
+  for ( ; ; )
+  {
+    break;
+  }
+// CHECK: ret{{.*}}, !dbg [[gRET:!.*]]
+}
+
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[G5R1]] = !DILocation(line: 29,{{.*}} atomGroup: 5, atomRank: 1)
+// CHECK: [[G4R2]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 2)
+// CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 6, atomRank: 1)
+
+// CHECK: [[bG1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[bG2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[bG3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[bG4R2]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 2)
+// CHECK: [[bG4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK: [[bG6R1]] = !DILocation(line: 58,{{.*}} atomGroup: 6, atomRank: 1)
+// CHECK: [[bG5R2]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 2)
+// CHECK: [[bG5R1]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
+// CHECK: [[bRET]] = !DILocation({{.*}}, atomGroup: 7, atomRank: 1)
+
+// CHECK: [[cG1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[cG1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[cG3R1]] = !DILocation(line: 83,{{.*}} atomGroup: 3, atomRank: 1)
+// CHECK: [[cG2R2]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
+// CHECK: [[cG2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+
+// CHECK: [[dG1R1]] = !DILocation(line: 93, column: 3, scope: ![[#]], atomGroup: 1, atomRank: 1)
+
+// CHECK: [[eG1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[eG2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[eG3R2]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 2)
+// CHECK: [[eG3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[eG4R1]] = !DILocation(line: 115, column: 5, scope: ![[#]], atomGroup: 4, atomRank: 1)
+// CHECK: [[eRET]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
+
+// CHECK: [[fG1R1]] = !DILocation(line: 129, column: 5, scope: ![[#]], atomGroup: 1, atomRank: 1)
+
+// CHECK: [[gG1R1]] = !DILocation(line: 142, column: 5, scope: ![[#]], atomGroup: 1, atomRank: 1)
+// CHECK: [[gRET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/if.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/if.c
index b16dec7b91c4..edadd04cf0e1 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/if.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/if.c
@@ -2,7 +2,7 @@
 // RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-CXX
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - \
-// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-C
 
 int g;
 void a(int A) {
@@ -32,6 +32,8 @@ void a(int A) {
     if (int B = A; B)
         ;
 #endif
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
@@ -44,3 +46,5 @@ void a(int A) {
 // CHECK-CXX: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
 // CHECK-CXX: [[G5R2]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 2)
 // CHECK-CXX: [[G5R1]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
+// CHECK-CXX: [[RET]] = !DILocation({{.*}}, atomGroup: 6, atomRank: 1)
+// CHECK-C: [[RET]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-agg.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-agg.c
index 1021490bd75b..711d8ad3de57 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-agg.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-agg.c
@@ -37,6 +37,8 @@ void a() {
 
 // CHECK: store i8 {{.*}}, ptr %uninit{{.*}}, !dbg [[G5R1:!.*]], !annotation
     char uninit; // -ftrivial-auto-var-init=pattern
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
@@ -47,3 +49,4 @@ void a() {
 // CHECK: [[big_LINE]] = !DILocation(line: 33, scope: ![[#]])
 // CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
 // CHECK: [[G5R1]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 6, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-member.cpp b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-member.cpp
index 4d8e1b9dace3..0c45c07886d4 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-member.cpp
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-member.cpp
@@ -17,6 +17,8 @@ void fun() {
 
 // CHECK: store i32 1, ptr %x{{.*}}, !dbg [[G1R1:!.*]]
 // CHECK: store float 5.000000e+00, ptr %y{{.*}}, !dbg [[G2R1:!.*]]
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
 // CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-scalar.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-scalar.c
index d4802a709b89..217a2496c675 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-scalar.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-scalar.c
@@ -10,8 +10,10 @@ void a() {
 // CHECK: %add = add {{.*}}, !dbg [[G2R2:!.*]]
 // CHECK: store i32 %add, ptr %B, align 4, !dbg [[G2R1:!.*]]
     int B = 2 * A + 1;
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
 // CHECK: [[G2R2]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
 // CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-static.cpp b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-static.cpp
index 2cac57e1d9ff..ee73275edb76 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-static.cpp
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/init-static.cpp
@@ -4,10 +4,12 @@
 // CHECK: [[b_addr:@.*]] = {{.*}}global ptr
 
 void g(int *a) {
-    // CHECK: [[v:%.*]] = load ptr, ptr %a.addr{{.*}}, !dbg [[G1R2:!.*]]
-    // CHECK: store ptr [[v]], ptr [[b_addr]]{{.*}}, !dbg [[G1R1:!.*]]
+// CHECK: [[v:%.*]] = load ptr, ptr %a.addr{{.*}}, !dbg [[G1R2:!.*]]
+// CHECK: store ptr [[v]], ptr [[b_addr]]{{.*}}, !dbg [[G1R1:!.*]]
     static int &b = *a;
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/multi-func.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/multi-func.c
index 6e225eed81de..ce311bad00d1 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/multi-func.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/multi-func.c
@@ -8,12 +8,17 @@
 
 int g;
 // CHECK: store{{.*}}, !dbg [[AG:!.*]]
+// CHECK: ret{{.*}}, !dbg [[ARET:!.*]]
 void a() { g = 0; }
 
 // CHECK: store{{.*}}, !dbg [[BG:!.*]]
+// CHECK: ret{{.*}}, !dbg [[BRET:!.*]]
 void b() { g = 0; }
 
 // CHECK: [[A:!.*]] = distinct !DISubprogram(name: "a",
-// CHECK: [[AG]] = !DILocation(line: 11, scope: [[A]], atomGroup: 1, atomRank: 1)
+// CHECK: [[AG]] = !DILocation(line: 12, scope: [[A]], atomGroup: 1, atomRank: 1)
+// CHECK: [[ARET]] = !DILocation(line: 12, scope: [[A]], atomGroup: 2, atomRank: 1)
+
 // CHECK: [[B:!.*]] = distinct !DISubprogram(name: "b",
-// CHECK: [[BG]] = !DILocation(line: 14, scope: [[B]], atomGroup: 1, atomRank: 1)
+// CHECK: [[BG]] = !DILocation(line: 16, scope: [[B]], atomGroup: 1, atomRank: 1)
+// CHECK: [[BRET]] = !DILocation(line: 16, scope: [[B]], atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/new.cpp b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/new.cpp
new file mode 100644
index 000000000000..f939e784a457
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/new.cpp
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions %s -debug-info-kind=line-tables-only -emit-llvm -o - -Wno-unused-variable \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// Check both the addr-store and value-store are part of the same atom.
+
+void f(int x) {
+// CHECK: %call = call noalias noundef nonnull ptr @_Znwm{{.*}}, !dbg [[G1R2_C12:!.*]]
+// CHECK: %0 = load i32, ptr %x.addr{{.*}}, !dbg [[G1R2_C20:!.*]]
+// CHECK: store i32 %0, ptr %call{{.*}}, !dbg [[G1R1_C12:!.*]]
+// CHECK: store ptr %call, ptr %{{.*}}, !dbg [[G1R1_C8:!.*]]
+  int *n = new int(x);
+// CHECK: %call1 = call noalias noundef nonnull ptr @_Znwm{{.*}}, !dbg [[G2R2_C7:!.*]]
+// CHECK: %1 = load i32, ptr %x.addr{{.*}}, !dbg [[G2R2_C15:!.*]]
+// CHECK: store i32 %1, ptr %call{{.*}}, !dbg [[G2R1_C7:!.*]]
+// CHECK: store ptr %call1, ptr %{{.*}}, !dbg [[G2R1_C5:!.*]]
+  n = new int(x);
+// CHECK: %2 = load i32, ptr %x.addr{{.*}}, !dbg [[G3R2:!.*]]
+// CHECK: %3 = load ptr, ptr %n
+// CHECK: store i32 %2, ptr %3{{.*}}, !dbg [[G3R1:!.*]]
+  *n = x;
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+// CHECK: [[G1R2_C12]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R2_C20]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R1_C12]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[G1R1_C8]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+
+// CHECK: [[G2R2_C7]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
+// CHECK: [[G2R2_C15]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
+// CHECK: [[G2R1_C7]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[G2R1_C5]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+
+// CHECK: [[G3R2]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 2)
+// CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/return-va-arg.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/return-va-arg.c
new file mode 100644
index 000000000000..0773bf535317
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/return-va-arg.c
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -gno-column-info -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -gno-column-info -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+typedef struct {
+  struct{} a;
+  double b;
+} s1;
+
+s1 f(int z, ...) {
+  __builtin_va_list list;
+  __builtin_va_start(list, z);
+// CHECK: vaarg.end:
+// CHECK-NEXT: %vaarg.addr = phi ptr
+// CHECK-NEXT: call void @llvm.memcpy{{.*}}, !dbg [[G1R1:!.*]]
+// CHECK-NEXT: {{.*}} = getelementptr{{.*}}
+// CHECK-NEXT: [[LOAD:%.*]] = load double{{.*}}, !dbg [[G1R2:!.*]]
+// CHECK-NEXT: ret double [[LOAD]], !dbg [[G1R1]]
+  return __builtin_va_arg(list, s1);
+}
+
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/return.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/return.c
new file mode 100644
index 000000000000..c79e9d930f31
--- /dev/null
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/return.c
@@ -0,0 +1,101 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -gno-column-info -x c++ %s -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-CXX
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -gno-column-info -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s
+
+// Check the stores to `retval` allocas and branches to `return` block are in
+// the same atom group. They are both rank 1, which could in theory introduce
+// an extra step in some optimized code. This low risk currently feels an
+// acceptable for keeping the code a bit simpler (as opposed to adding
+// scaffolding to make the store rank 2).
+
+// Also check that in the case of a single return (no control flow) the
+// return instruction inherits the atom group of the branch to the return
+// block when the blocks get folded togather.
+
+#ifdef __cplusplus
+#define nomangle extern "C"
+#else
+#define nomangle
+#endif
+
+int g;
+nomangle float a() {
+// CHECK: float @a()
+  if (g)
+// CHECK: if.then:
+// CHECK-NEXT: %1 = load i32, ptr @g{{.*}}, !dbg [[G2R3:!.*]]
+// CHECK-NEXT: %conv = sitofp i32 %1 to float{{.*}}, !dbg [[G2R2:!.*]]
+// CHECK-NEXT: store float %conv, ptr %retval{{.*}}, !dbg [[G2R1:!.*]]
+// CHECK-NEXT: br label %return{{.*}}, !dbg [[G2R1]]
+    return g;
+// CHECK: if.end:
+// CHECK-NEXT: store float 1.000000e+00, ptr %retval{{.*}}, !dbg [[G3R1:!.*]]
+// CHECK-NEXT: br label %return, !dbg [[G3R1]]
+
+// CHECK: return:
+// CHECK-NEXT:  %2 = load float, ptr %retval{{.*}}, !dbg [[G4R2:!.*]]
+// CHECK-NEXT:  ret float %2{{.*}}, !dbg [[G4R1:!.*]]
+  return 1;
+}
+
+// CHECK: void @b()
+// CHECK: ret void{{.*}}, !dbg [[B_G1R1:!.*]]
+nomangle void b() { return; }
+
+// CHECK: i32 @c()
+// CHECK: %add = add{{.*}}, !dbg [[C_G1R2:!.*]]
+// CHECK: ret i32 %add{{.*}}, !dbg [[C_G1R1:!.*]]
+nomangle int c() { return g + 1; }
+
+// NOTE: (return) (g = 1) are two separate atoms.
+// CHECK: i32 @d()
+// CHECK: store{{.*}}, !dbg [[D_G2R1:!.*]]
+// CHECK: ret i32 1{{.*}}, !dbg [[D_G1R1:!.*]]
+nomangle int d() { return g = 1; }
+
+// The implicit return here get the line number of the closing brace; make it
+// key to match existing behaviour.
+// CHECK: void @e()
+// CHECK: ret void, !dbg [[E_G1R1:!.*]]
+nomangle void e() {}
+
+#ifdef __cplusplus
+// CHECK-CXX: ptr @_Z1fRi
+int &f(int &r) {
+// Include ctrl-flow to stop ret value store being elided.
+    if (r)
+// CHECK-CXX: if.then:
+// CHECK-CXX-NEXT: %2 = load ptr, ptr %r.addr{{.*}}, !dbg [[F_G2R2:!.*]], !nonnull
+// CHECK-CXX-NEXT: store ptr %2, ptr %retval{{.*}}, !dbg [[F_G2R1:!.*]]
+// CHECK-CXX-NEXT: br label %return, !dbg [[F_G2R1:!.*]]
+    return r;
+
+// CHECK-CXX: if.end:
+// CHECK-CXX-NEXT: store ptr @g, ptr %retval{{.*}}, !dbg [[F_G3R1:!.*]]
+// CHECK-CXX-NEXT: br label %return, !dbg [[F_G3R1:!.*]]
+// CHECK-CXX: return:
+// CHECK-CXX-NEXT: %3 = load ptr, ptr %retval{{.*}}, !dbg [[F_G4R2:!.*]]
+// CHECK-CXX-NEXT: ret ptr %3, !dbg [[F_G4R1:!.*]]
+  return g;
+}
+#endif
+
+// CHECK: [[G2R3]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 3)
+// CHECK: [[G2R2]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
+// CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[G4R2]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 2)
+// CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK: [[B_G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[C_G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[C_G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[D_G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[D_G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[E_G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK-CXX: [[F_G2R2]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
+// CHECK-CXX: [[F_G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK-CXX: [[F_G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK-CXX: [[F_G4R2]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 2)
+// CHECK-CXX: [[F_G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/switch.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/switch.c
index 5142f204ff58..cd8fced67eed 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/switch.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/switch.c
@@ -2,7 +2,7 @@
 // RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-CXX
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - \
-// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank --check-prefixes=CHECK,CHECK-C
 
 int g;
 void a(int A, int B) {
@@ -16,18 +16,19 @@ void a(int A, int B) {
 // CHECK:   i32 1, label %sw.bb1
 // CHECK: ], !dbg [[G2R1:!.*]]
     switch ((g = A)) {
+// CHECK: br label %sw.epilog[[#]], !dbg [[G3R1:!.*]]
     case 0: break;
     case 1: {
 // CHECK: sw.bb1:
-// CHECK: %1 = load i32, ptr %B.addr{{.*}}, !dbg [[G3R2:!.*]]
+// CHECK: %1 = load i32, ptr %B.addr{{.*}}, !dbg [[G4R2:!.*]]
 // CHECK: switch i32 %1, label %{{.*}} [
 // CHECK:   i32 0, label %sw.bb2
-// CHECK: ], !dbg [[G3R1:!.*]]
+// CHECK: ], !dbg [[G4R1:!.*]]
     switch ((B)) {
         case 0: {
 // Test that assignments in constant-folded switches don't go missing.
 // CHECK-CXX: sw.bb2:
-// CHECK-CXX: store i32 1, ptr %C{{.*}}, !dbg [[G4R1:!.*]]
+// CHECK-CXX: store i32 1, ptr %C{{.*}}, !dbg [[G5R1:!.*]]
 #ifdef __cplusplus
             switch (const int C = 1; C) {
                 case 0: break;
@@ -35,17 +36,19 @@ void a(int A, int B) {
                 default: break;
             }
 #endif
-        } break;
-        default: break;
+        };
     }
-    } break;
-    default: break;
+    };
     }
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G2R2]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
 // CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
-// CHECK: [[G3R2]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 2)
 // CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
-// CHECK-CXX: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK: [[G4R2]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 2)
+// CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK-CXX: [[G5R1]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
+// CHECK-CXX: [[RET]] = !DILocation({{.*}}, atomGroup: 6, atomRank: 1)
+// CHECK-C: [[RET]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/try-catch.cpp b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/try-catch.cpp
index 918eb4c97db9..c1887bc52518 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/try-catch.cpp
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/try-catch.cpp
@@ -14,7 +14,10 @@ void attempt() {
 // CHECK: store i32 %5, ptr %e{{.*}}, !dbg [[G1R1:!.*]]
 // CHECK: call void @__cxa_end_catch()
   catch (int e) { }
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
diff --git a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/while.c b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/while.c
index 38cdb8433dd2..9c9eab399f34 100644
--- a/external/llvm-project/clang/test/DebugInfo/KeyInstructions/while.c
+++ b/external/llvm-project/clang/test/DebugInfo/KeyInstructions/while.c
@@ -26,9 +26,12 @@ void a(int A) {
 // CHECK: %tobool = icmp ne i32 %dec, 0, !dbg [[G2R1:!.*]]
 // CHECK: br i1 %tobool, label %while.body, label %while.end, !dbg [[G3R1:!.*]]
     while (--A) { };
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
 }
 
 // CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
 // CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
 // CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
 // CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
diff --git a/external/llvm-project/clang/test/Driver/aix-print-runtime-dir.c b/external/llvm-project/clang/test/Driver/aix-print-runtime-dir.c
index ef133728d731..474012944254 100644
--- a/external/llvm-project/clang/test/Driver/aix-print-runtime-dir.c
+++ b/external/llvm-project/clang/test/Driver/aix-print-runtime-dir.c
@@ -1,13 +1,5 @@
 // Test output of -print-runtime-dir on AIX
 
-// RUN: %clang -print-runtime-dir --target=powerpc-ibm-aix \
-// RUN:        -resource-dir=%S/Inputs/resource_dir \
-// RUN:      | FileCheck --check-prefix=PRINT-RUNTIME-DIR %s
-
-// RUN: %clang -print-runtime-dir --target=powerpc64-ibm-aix \
-// RUN:        -resource-dir=%S/Inputs/resource_dir \
-// RUN:      | FileCheck --check-prefix=PRINT-RUNTIME-DIR %s
-
 // RUN: %clang -print-runtime-dir --target=powerpc-ibm-aix \
 // RUN:        -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir\
 // RUN:      | FileCheck --check-prefix=PRINT-RUNTIME-DIR32-PER-TARGET %s
@@ -24,7 +16,6 @@
 // RUN:        -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
 // RUN:      | FileCheck --check-prefix=PRINT-RUNTIME-DIR64-UNKNOWN-ENV %s 
 
-// PRINT-RUNTIME-DIR: lib{{/|\\}}aix{{$}}
 // PRINT-RUNTIME-DIR32-PER-TARGET: lib{{/|\\}}powerpc-ibm-aix{{$}}
 // PRINT-RUNTIME-DIR64-PER-TARGET: lib{{/|\\}}powerpc64-ibm-aix{{$}}
 // PRINT-RUNTIME-DIR32-UNKNOWN-ENV: lib{{/|\\}}powerpc-ibm-aix
diff --git a/external/llvm-project/clang/test/Driver/amdgpu-toolchain-opencl.cl b/external/llvm-project/clang/test/Driver/amdgpu-toolchain-opencl.cl
index 755d5e0ad8c5..fb21301744b9 100644
--- a/external/llvm-project/clang/test/Driver/amdgpu-toolchain-opencl.cl
+++ b/external/llvm-project/clang/test/Driver/amdgpu-toolchain-opencl.cl
@@ -35,3 +35,8 @@
 
 // RUN: %clang -### --target=amdgcn-amd-amdhsa-opencl -x cl -c -emit-llvm -mcpu=fiji -nogpulib %s 2>&1 | FileCheck -check-prefix=CHECK-WARN-ATOMIC %s
 // CHECK-WARN-ATOMIC: "-cc1"{{.*}} "-Werror=atomic-alignment"
+
+// RUN: %clang -### --target=amdgcn-amd-amdhsa -x cl -c -emit-llvm -mcpu=fiji %s 2>&1 \
+// RUN:  -mcode-object-version=5 --rocm-device-lib-path=%S/Inputs/rocm/amdgcn/bitcode \
+// RUN: | FileCheck -check-prefix=DEVICE-LIBS %s
+// DEVICE-LIBS: "-mlink-builtin-bitcode" "[[ROCM_PATH:.+]]opencl.bc"
diff --git a/external/llvm-project/clang/test/Driver/amdgpu-toolchain.c b/external/llvm-project/clang/test/Driver/amdgpu-toolchain.c
index 2a35c13746a0..459c1bdac246 100644
--- a/external/llvm-project/clang/test/Driver/amdgpu-toolchain.c
+++ b/external/llvm-project/clang/test/Driver/amdgpu-toolchain.c
@@ -38,3 +38,11 @@
 // RUN: %clang -target amdgcn-amd-amdhsa -march=gfx90a -stdlib -startfiles \
 // RUN:   -nogpulib -nogpuinc -### %s 2>&1 | FileCheck -check-prefix=STARTUP %s
 // STARTUP: ld.lld{{.*}}"-lc" "-lm" "{{.*}}crt1.o"
+
+// RUN: %clang -### --target=amdgcn-amd-amdhsa -mcpu=gfx906 %s 2>&1 | FileCheck -check-prefix=ROCM %s
+// ROCM-NOT: -mlink-builtin-bitcode
+
+// RUN: %clang -### --target=amdgcn-amd-amdhsa -mcpu=fiji -x ir %s \
+// RUN:  --rocm-device-lib-path=%S/Inputs/rocm/amdgcn/bitcode 2>&1 \
+// RUN: | FileCheck -check-prefix=DEVICE-LIBS %s
+// DEVICE-LIBS: "-mlink-builtin-bitcode" "[[ROCM_PATH:.+]]ockl.bc"
diff --git a/external/llvm-project/clang/test/Driver/darwin-debug-flags.c b/external/llvm-project/clang/test/Driver/darwin-debug-flags.c
index 918ee345658a..90209bb179bf 100644
--- a/external/llvm-project/clang/test/Driver/darwin-debug-flags.c
+++ b/external/llvm-project/clang/test/Driver/darwin-debug-flags.c
@@ -7,7 +7,7 @@
 // CHECK-SAME:                flags:
 // CHECK-SAME:                -I path\\ with\\ \\\\spaces
 // CHECK-SAME:                -g -Os
-// CHECK-SAME:                -mmacos-version-min=10.7.0
+// CHECK-SAME:                -mmacos-version-min=10.7
 
 int x;
 
diff --git a/external/llvm-project/clang/test/Driver/darwin-infer-simulator-sdkroot.c b/external/llvm-project/clang/test/Driver/darwin-infer-simulator-sdkroot.c
index 519e6d27540e..a946015028aa 100644
--- a/external/llvm-project/clang/test/Driver/darwin-infer-simulator-sdkroot.c
+++ b/external/llvm-project/clang/test/Driver/darwin-infer-simulator-sdkroot.c
@@ -41,9 +41,9 @@
 //
 // RUN: rm -rf %t/SDKs/WatchOS3.0.sdk
 // RUN: mkdir -p %t/SDKs/WatchOS3.0.sdk
-// RUN: env SDKROOT=%t/SDKs/WatchOS3.0.sdk %clang %s -fuse-ld= -mlinker-version=400 -### 2>&1 \
+// RUN: env SDKROOT=%t/SDKs/WatchOS3.0.sdk %clang %s -fuse-ld= -arch arm64_32 -mlinker-version=400 -### 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-WATCH %s
-// RUN: env WATCHOS_DEPLOYMENT_TARGET=3.0 %clang %s -fuse-ld= -isysroot %t/SDKs/WatchOS3.0.sdk -mlinker-version=400 -### 2>&1 \
+// RUN: env WATCHOS_DEPLOYMENT_TARGET=3.0 %clang %s -fuse-ld= -arch arm64_32 -isysroot %t/SDKs/WatchOS3.0.sdk -mlinker-version=400 -### 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-WATCH %s
 //
 // CHECK-WATCH: clang
diff --git a/external/llvm-project/clang/test/Driver/darwin-invalid-os-versions.c b/external/llvm-project/clang/test/Driver/darwin-invalid-os-versions.c
new file mode 100644
index 000000000000..d9424c1e6728
--- /dev/null
+++ b/external/llvm-project/clang/test/Driver/darwin-invalid-os-versions.c
@@ -0,0 +1,22 @@
+/// Verify invalid OSVersions are diagnosed.
+
+// RUN: not %clang -target arm64-apple-ios20 -c %s 2>&1 | FileCheck %s --check-prefix=IOS
+// IOS: error: invalid version number in '-target arm64-apple-ios20'
+
+// RUN: not %clang -target arm64-apple-watchos20 -c %s 2>&1 | FileCheck %s --check-prefix=WATCHOS
+// WATCHOS: error: invalid version number in '-target arm64-apple-watchos20'
+
+// RUN: not %clang -target arm64-apple-macosx19 -c %s 2>&1 | FileCheck %s --check-prefix=MAC
+// MAC: error: invalid version number in '-target arm64-apple-macosx19'
+
+// RUN: not %clang -target arm64-apple-ios22-macabi -c %s 2>&1 | FileCheck %s --check-prefix=IOSMAC
+// IOSMAC: error: invalid version number in '-target arm64-apple-ios22-macabi'
+
+// RUN: not %clang -target arm64-apple-macosx16 -darwin-target-variant arm64-apple-ios22-macabi  -c %s 2>&1 | FileCheck %s --check-prefix=ZIPPERED
+// ZIPPERED: error: invalid version number in 'arm64-apple-ios22-macabi'
+
+// RUN: not %clang -target arm64-apple-visionos5 -c %s 2>&1 | FileCheck %s --check-prefix=VISION
+// VISION: error: invalid version number in '-target arm64-apple-visionos5'
+
+// RUN: not %clang -target arm64-apple-tvos21 -c %s 2>&1 | FileCheck %s --check-prefix=TV
+// TV: error: invalid version number in '-target arm64-apple-tvos21'
diff --git a/external/llvm-project/clang/test/Driver/darwin-invalid-version-range.c b/external/llvm-project/clang/test/Driver/darwin-invalid-version-range.c
new file mode 100644
index 000000000000..84603aec1d2f
--- /dev/null
+++ b/external/llvm-project/clang/test/Driver/darwin-invalid-version-range.c
@@ -0,0 +1,29 @@
+/// This test validates that the various ways to assign an invalid deployment version are captured and detected.
+// REQUIRES: system-darwin && native
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+// RUN: env SDKROOT=%t/iPhoneOS21.0.sdk not %clang -m64 -c -### %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=SDKROOT
+
+// RUN: not %clang -isysroot %t/iPhoneOS21.0.sdk -m64 -c -### %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=SYSROOT
+
+// RUN: not %clang -target arm64-apple-ios21 -c -### %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=TARGET
+
+// RUN: not %clang -mtargetos=ios21 -arch arm64 -c -### %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=MTARGET
+
+// RUN: env IPHONEOS_DEPLOYMENT_TARGET=21.0 not %clang -arch arm64 -c -### %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=DEPLOY_VAR
+
+// SDKROOT:    error: invalid version number '21.0' inferred from '{{.*}}.sdk'
+// SYSROOT:    error: invalid version number '21.0' inferred from '{{.*}}.sdk'
+// TARGET:     error: invalid version number in '-target arm64-apple-ios21'
+// MTARGET:    error: invalid version number in '-mtargetos=ios21'
+// DEPLOY_VAR: error: invalid version number in 'IPHONEOS_DEPLOYMENT_TARGET=21.0'
+
+//--- iPhoneOS21.0.sdk/SDKSettings.json
+{"Version":"21.0", "MaximumDeploymentTarget": "21.0.99"}
diff --git a/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-macos.c b/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-macos.c
index 355df8dfc1bc..b16ca8a85374 100644
--- a/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-macos.c
+++ b/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-macos.c
@@ -48,3 +48,8 @@
 // RUN:   -### %t.o 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOSDK %s
 // NOSDK: "-platform_version" "macos" "10.13.0" "10.13.0"
+
+// RUN: %clang -target arm64-apple-macos26 -mlinker-version=520 \
+// RUN:   -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=VERSION_BUMP %s
+// VERSION_BUMP: "-platform_version" "macos" "26.0.0" "26.0.0"
diff --git a/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-watchos.c b/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-watchos.c
index fc8e859f30ef..afbc3194963f 100644
--- a/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-watchos.c
+++ b/external/llvm-project/clang/test/Driver/darwin-ld-platform-version-watchos.c
@@ -17,6 +17,54 @@
 // RUN:   -### %t.o 2>&1 \
 // RUN:   | FileCheck --check-prefix=SIMUL %s
 
+// RUN: %clang -target arm64-apple-watchos6.3 -fuse-ld= \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=400 \
+// RUN:   -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-LINKER-OLD %s
+
+// RUN: %clang -target arm64e-apple-watchos6.3 -fuse-ld= \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=400 \
+// RUN:   -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-LINKER-OLD %s
+
+// RUN: %clang -target arm64-apple-watchos26.1 -fuse-ld= \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=400 \
+// RUN:   -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-LINKER-OLD-261 %s
+
+// RUN: %clang -target arm64-apple-watchos6.3 -fuse-ld=lld \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=0 \
+// RUN:   -### %t.o -B%S/Inputs/lld 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-LINKER-NEW %s
+
+// RUN: %clang -target arm64e-apple-watchos6.3 -fuse-ld=lld \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=0 \
+// RUN:   -### %t.o -B%S/Inputs/lld 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-LINKER-NEW %s
+
+// RUN: %clang -target arm64-apple-watchos6.3 -fuse-ld= \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=520 \
+// RUN:   -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-LINKER-NEW %s
+
+// RUN: %clang -target arm64-apple-watchos26.1 -fuse-ld= \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=520 \
+// RUN:   -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-LINKER-NEW-261 %s
+
+// RUN: %clang -target arm64-apple-watchos6-simulator -fuse-ld= \
+// RUN:   -isysroot %S/Inputs/WatchOS6.0.sdk -mlinker-version=520 \
+// RUN:   -### %t.o 2>&1 \
+// RUN:   | FileCheck --check-prefix=ARM64-SIMUL %s
+
 // LINKER-OLD: "-watchos_version_min" "5.2.0"
 // LINKER-NEW: "-platform_version" "watchos" "5.2.0" "6.0"
 // SIMUL: "-platform_version" "watchos-simulator" "6.0.0" "6.0"
+
+// ARM64-LINKER-OLD: "-watchos_version_min" "26.0.0"
+// ARM64-LINKER-OLD-261: "-watchos_version_min" "26.1.0"
+
+// ARM64-LINKER-NEW: "-platform_version" "watchos" "26.0.0" "6.0"
+// ARM64-LINKER-NEW-261: "-platform_version" "watchos" "26.1.0" "6.0"
+
+// ARM64-SIMUL: "-platform_version" "watchos-simulator" "7.0.0" "6.0"
diff --git a/external/llvm-project/clang/test/Driver/darwin-ld.c b/external/llvm-project/clang/test/Driver/darwin-ld.c
index f0ca411430cc..9a8d98cdb9c2 100644
--- a/external/llvm-project/clang/test/Driver/darwin-ld.c
+++ b/external/llvm-project/clang/test/Driver/darwin-ld.c
@@ -240,6 +240,15 @@
 // RUN: FileCheck -check-prefix=LINK_NO_IOS_ARM64_LIBGCC_S %s < %t.log
 // LINK_NO_IOS_ARM64_LIBGCC_S-NOT: lgcc_s.1
 
+// Check that clang links with libgcc_s.1 for Mac OS X 10.5 and earlier, but not arm64
+// RUN: %clang -target x86_64-apple-macosx10.5 -mmacosx-version-min=10.5 -### %t.o 2> %t.log
+// RUN: FileCheck -check-prefix=LINK_OSX_LIBGCC_S %s < %t.log
+// LINK_OSX_LIBGCC_S: lgcc_s.1
+
+// RUN: %clang -target arm64-apple-macosx10.5 -mmacosx-version-min=10.5 -### %t.o 2> %t.log
+// RUN: FileCheck -check-prefix=LINK_NO_OSX_ARM64_LIBGCC_S %s < %t.log
+// LINK_NO_OSX_ARM64_LIBGCC_S-NOT: lgcc_s.1
+
 // RUN: %clang -target x86_64-apple-darwin12 -rdynamic -### %t.o \
 // RUN:   -fuse-ld= -mlinker-version=100 2> %t.log
 // RUN: FileCheck -check-prefix=LINK_NO_EXPORT_DYNAMIC %s < %t.log
@@ -385,4 +394,4 @@
 // RUN:   %clang -target armv7em-apple-darwin -mno-outline -### %t.o 2> %t.log
 // RUN: FileCheck -check-prefix=ARMV7EM-MNO_OUTLINE %s < %t.log
 // ARMV7EM-MNO_OUTLINE: {{ld(.exe)?"}}
-// ARMV7EM-MNO_OUTLINE-SAME: "-mllvm" "-enable-machine-outliner=never" "-mllvm" "-enable-linkonceodr-outlining"
\ No newline at end of file
+// ARMV7EM-MNO_OUTLINE-SAME: "-mllvm" "-enable-machine-outliner=never" "-mllvm" "-enable-linkonceodr-outlining"
diff --git a/external/llvm-project/clang/test/Driver/darwin-version.c b/external/llvm-project/clang/test/Driver/darwin-version.c
index a00bd832dc80..9b9c6034bb75 100644
--- a/external/llvm-project/clang/test/Driver/darwin-version.c
+++ b/external/llvm-project/clang/test/Driver/darwin-version.c
@@ -86,7 +86,7 @@
 // RUN:   FileCheck --check-prefix=CHECK-VERSION-MISSING %s
 // RUN: not %clang -target x86_64-apple-darwin -mmacos-version-min= -c %s -### 2>&1 | \
 // RUN:   FileCheck --check-prefix=CHECK-VERSION-MISSING %s
-// CHECK-VERSION-MISSING: invalid version number
+// CHECK-VERSION-MISSING: missing version number
 // RUN: %clang -target armv7k-apple-darwin -mwatchos-version-min=2.0 -c %s -### 2>&1 | \
 // RUN:   FileCheck --check-prefix=CHECK-VERSION-WATCHOS20 %s
 // RUN: %clang -target armv7-apple-darwin -mtvos-version-min=8.3 -c %s -### 2>&1 | \
@@ -330,9 +330,13 @@
 // RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
 // RUN: %clang -target x86_64-apple-darwin -mmacos-version-min=11 -c %s -### 2>&1 | \
 // RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
-
 // CHECK-MACOS11: "x86_64-apple-macosx11.0.0"
 
+// RUN: %clang -target arm64-apple-macosx10.16 -c %s -### 2>&1 | \
+// RUN:   FileCheck --check-prefix=CHECK-IMPLICIT-MACOS11 %s
+// CHECK-IMPLICIT-MACOS11: warning: overriding deployment version
+// CHECK-IMPLICIT-MACOS11: "arm64-apple-macosx11.0.0"
+
 // RUN: %clang -target arm64-apple-macos999 -c %s -### 2>&1 | \
 // RUN:   FileCheck --check-prefix=CHECK-MACOS999 %s
 
diff --git a/external/llvm-project/clang/test/Driver/dxc_spirv.hlsl b/external/llvm-project/clang/test/Driver/dxc_spirv.hlsl
index e6624e5f1b3f..65c9018dc54c 100644
--- a/external/llvm-project/clang/test/Driver/dxc_spirv.hlsl
+++ b/external/llvm-project/clang/test/Driver/dxc_spirv.hlsl
@@ -3,7 +3,7 @@
 // RUN: %clang_dxc -T cs_6_0 -spirv -fspv-target-env=vulkan1.3 -### %s 2>&1 | FileCheck %s --check-prefix=CHECK-VULKAN13
 // RUN: not %clang_dxc -T cs_6_0 -spirv -fspv-target-env=vulkan1.0 -### %s 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-// CHECK: "-triple" "spirv-unknown-vulkan-compute"
+// CHECK: "-triple" "spirv1.6-unknown-vulkan1.3-compute"
 // CHECK-SAME: "-x" "hlsl"
 
 // CHECK-VULKAN12: "-triple" "spirv1.5-unknown-vulkan1.2-compute"
diff --git a/external/llvm-project/clang/test/Driver/hip-phases.hip b/external/llvm-project/clang/test/Driver/hip-phases.hip
index 66c0789fc284..c22def297bc6 100644
--- a/external/llvm-project/clang/test/Driver/hip-phases.hip
+++ b/external/llvm-project/clang/test/Driver/hip-phases.hip
@@ -12,6 +12,9 @@
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --offload-new-driver --cuda-gpu-arch=gfx803 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN,NEW,NEWN %s
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: --offload-new-driver --cuda-gpu-arch=gfx803 -flto -c %s 2>&1 \
+// RUN: | FileCheck -check-prefixes=BIN,NEW,NEWLTO %s
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 -fgpu-rdc %s 2>&1 \
@@ -45,10 +48,14 @@
 // OLDR-DAG: [[P11:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P10]]}, object
 // OLDN-DAG: [[P12:[0-9]+]]: backend, {[[P11]]}, assembler, (host-[[T]])
 // OLDN-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]])
-// NEW-DAG: [[P12:[0-9]+]]: backend, {[[P11]]}, assembler, (host-[[T]])
-// NEW-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]])
+// NEWN-DAG: [[P12:[0-9]+]]: backend, {[[P11]]}, assembler, (host-[[T]])
+// NEWN-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]])
+// NEWLTO-DAG: [[P13:[0-9]+]]: backend, {[[P11]]}, lto-bc, (host-hip)
+// NEWR-DAG: [[P12:[0-9]+]]: backend, {[[P11]]}, assembler, (host-[[T]])
+// NEWR-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]])
 // OLDN-DAG: [[P14:[0-9]+]]: linker, {[[P13]]}, image, (host-[[T]])
 // NEWN-DAG: [[P14:[0-9]+]]: clang-linker-wrapper, {[[P13]]}, object, (host-[[T]])
+// NEWLTO-DAG: [[P14:[0-9]+]]: clang-linker-wrapper, {[[P13]]}, object, (host-[[T]])
 // OLDR-DAG: [[P14:[0-9]+]]: linker, {[[P13]], [[P11]]}, image, (host-[[T]])
 // NEWR-DAG: [[P14:[0-9]+]]: clang-linker-wrapper, {[[P13]]}, image, (host-[[T]])
 // NEWN-DAG: [[P15:[0-9]+]]: linker, {[[P14]]}, image
diff --git a/external/llvm-project/clang/test/Driver/linker-wrapper.c b/external/llvm-project/clang/test/Driver/linker-wrapper.c
index a7e98e7351d9..80b1a5745a12 100644
--- a/external/llvm-project/clang/test/Driver/linker-wrapper.c
+++ b/external/llvm-project/clang/test/Driver/linker-wrapper.c
@@ -223,6 +223,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RELOCATABLE-LINK-HIP: clang-offload-bundler{{.*}} -type=o -bundle-align=4096 -targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx90a -input={{/dev/null|NUL}} -input={{.*}} -output={{.*}}
 // RELOCATABLE-LINK-HIP: /usr/bin/ld.lld{{.*}}-r
 // RELOCATABLE-LINK-HIP: llvm-objcopy{{.*}}a.out --remove-section .llvm.offloading
+// RELOCATABLE-LINK-HIP: --rename-section llvm_offload_entries
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=cuda,triple=nvptx64-nvidia-cuda,arch=sm_89 \
diff --git a/external/llvm-project/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c b/external/llvm-project/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
new file mode 100644
index 000000000000..3dc4d6a6b4fa
--- /dev/null
+++ b/external/llvm-project/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
@@ -0,0 +1,71 @@
+// RUN: %clang --target=riscv64 -mcpu=sifive-x390 -menable-experimental-extensions --print-enabled-extensions | FileCheck %s
+// REQUIRES: riscv-registered-target
+
+// CHECK:     Name                 Version   Description
+// CHECK-NEXT:     i                    2.1       'I' (Base Integer Instruction Set)
+// CHECK-NEXT:     m                    2.0       'M' (Integer Multiplication and Division)
+// CHECK-NEXT:     a                    2.1       'A' (Atomic Instructions)
+// CHECK-NEXT:     f                    2.2       'F' (Single-Precision Floating-Point)
+// CHECK-NEXT:     d                    2.2       'D' (Double-Precision Floating-Point)
+// CHECK-NEXT:     c                    2.0       'C' (Compressed Instructions)
+// CHECK-NEXT:     b                    1.0       'B' (the collection of the Zba, Zbb, Zbs extensions)
+// CHECK-NEXT:     v                    1.0       'V' (Vector Extension for Application Processors)
+// CHECK-NEXT:     zic64b               1.0       'Zic64b' (Cache Block Size Is 64 Bytes)
+// CHECK-NEXT:     zicbom               1.0       'Zicbom' (Cache-Block Management Instructions)
+// CHECK-NEXT:     zicbop               1.0       'Zicbop' (Cache-Block Prefetch Instructions)
+// CHECK-NEXT:     zicboz               1.0       'Zicboz' (Cache-Block Zero Instructions)
+// CHECK-NEXT:     ziccamoa             1.0       'Ziccamoa' (Main Memory Supports All Atomics in A)
+// CHECK-NEXT:     ziccif               1.0       'Ziccif' (Main Memory Supports Instruction Fetch with Atomicity Requirement)
+// CHECK-NEXT:     ziccrse              1.0       'Ziccrse' (Main Memory Supports Forward Progress on LR/SC Sequences)
+// CHECK-NEXT:     zicntr               2.0       'Zicntr' (Base Counters and Timers)
+// CHECK-NEXT:     zicond               1.0       'Zicond' (Integer Conditional Operations)
+// CHECK-NEXT:     zicsr                2.0       'Zicsr' (CSRs)
+// CHECK-NEXT:     zifencei             2.0       'Zifencei' (fence.i)
+// CHECK-NEXT:     zihintntl            1.0       'Zihintntl' (Non-Temporal Locality Hints)
+// CHECK-NEXT:     zihintpause          2.0       'Zihintpause' (Pause Hint)
+// CHECK-NEXT:     zihpm                2.0       'Zihpm' (Hardware Performance Counters)
+// CHECK-NEXT:     zimop                1.0       'Zimop' (May-Be-Operations)
+// CHECK-NEXT:     zmmul                1.0       'Zmmul' (Integer Multiplication)
+// CHECK-NEXT:     za64rs               1.0       'Za64rs' (Reservation Set Size of at Most 64 Bytes)
+// CHECK-NEXT:     zaamo                1.0       'Zaamo' (Atomic Memory Operations)
+// CHECK-NEXT:     zalrsc               1.0       'Zalrsc' (Load-Reserved/Store-Conditional)
+// CHECK-NEXT:     zawrs                1.0       'Zawrs' (Wait on Reservation Set)
+// CHECK-NEXT:     zfa                  1.0       'Zfa' (Additional Floating-Point)
+// CHECK-NEXT:     zfbfmin              1.0       'Zfbfmin' (Scalar BF16 Converts)
+// CHECK-NEXT:     zfh                  1.0       'Zfh' (Half-Precision Floating-Point)
+// CHECK-NEXT:     zfhmin               1.0       'Zfhmin' (Half-Precision Floating-Point Minimal)
+// CHECK-NEXT:     zca                  1.0       'Zca' (part of the C extension, excluding compressed floating point loads/stores)
+// CHECK-NEXT:     zcb                  1.0       'Zcb' (Compressed basic bit manipulation instructions)
+// CHECK-NEXT:     zcd                  1.0       'Zcd' (Compressed Double-Precision Floating-Point Instructions)
+// CHECK-NEXT:     zcmop                1.0       'Zcmop' (Compressed May-Be-Operations)
+// CHECK-NEXT:     zba                  1.0       'Zba' (Address Generation Instructions)
+// CHECK-NEXT:     zbb                  1.0       'Zbb' (Basic Bit-Manipulation)
+// CHECK-NEXT:     zbs                  1.0       'Zbs' (Single-Bit Instructions)
+// CHECK-NEXT:     zkr                  1.0       'Zkr' (Entropy Source Extension)
+// CHECK-NEXT:     zkt                  1.0       'Zkt' (Data Independent Execution Latency)
+// CHECK-NEXT:     zvbb                 1.0       'Zvbb' (Vector basic bit-manipulation instructions)
+// CHECK-NEXT:     zve32f               1.0       'Zve32f' (Vector Extensions for Embedded Processors with maximal 32 EEW and F extension)
+// CHECK-NEXT:     zve32x               1.0       'Zve32x' (Vector Extensions for Embedded Processors with maximal 32 EEW)
+// CHECK-NEXT:     zve64d               1.0       'Zve64d' (Vector Extensions for Embedded Processors with maximal 64 EEW, F and D extension)
+// CHECK-NEXT:     zve64f               1.0       'Zve64f' (Vector Extensions for Embedded Processors with maximal 64 EEW and F extension)
+// CHECK-NEXT:     zve64x               1.0       'Zve64x' (Vector Extensions for Embedded Processors with maximal 64 EEW)
+// CHECK-NEXT:     zvfbfmin             1.0       'Zvfbfmin' (Vector BF16 Converts)
+// CHECK-NEXT:     zvfbfwma             1.0       'Zvfbfwma' (Vector BF16 widening mul-add)
+// CHECK-NEXT:     zvfh                 1.0       'Zvfh' (Vector Half-Precision Floating-Point)
+// CHECK-NEXT:     zvfhmin              1.0       'Zvfhmin' (Vector Half-Precision Floating-Point Minimal)
+// CHECK-NEXT:     zvkb                 1.0       'Zvkb' (Vector Bit-manipulation used in Cryptography)
+// CHECK-NEXT:     zvkt                 1.0       'Zvkt' (Vector Data-Independent Execution Latency)
+// CHECK-NEXT:     zvl1024b             1.0       'Zvl1024b' (Minimum Vector Length 1024)
+// CHECK-NEXT:     zvl128b              1.0       'Zvl128b' (Minimum Vector Length 128)
+// CHECK-NEXT:     zvl256b              1.0       'Zvl256b' (Minimum Vector Length 256)
+// CHECK-NEXT:     zvl32b               1.0       'Zvl32b' (Minimum Vector Length 32)
+// CHECK-NEXT:     zvl512b              1.0       'Zvl512b' (Minimum Vector Length 512)
+// CHECK-NEXT:     zvl64b               1.0       'Zvl64b' (Minimum Vector Length 64)
+// CHECK-NEXT:     xsifivecdiscarddlone 1.0       'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction)
+// CHECK-NEXT:     xsifivecflushdlone   1.0       'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction)
+// CHECK-EMPTY:
+// CHECK-NEXT: Experimental extensions
+// CHECK-NEXT:     zicfilp              1.0       'Zicfilp' (Landing pad)
+// CHECK-NEXT:     zicfiss              1.0       'Zicfiss' (Shadow stack)
+// CHECK-EMPTY:
+// CHECK-NEXT: ISA String: rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_b1p0_v1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_ziccrse1p0_zicfilp1p0_zicfiss1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zaamo1p0_zalrsc1p0_zawrs1p0_zfa1p0_zfbfmin1p0_zfh1p0_zfhmin1p0_zca1p0_zcb1p0_zcd1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkr1p0_zkt1p0_zvbb1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfbfmin1p0_zvfbfwma1p0_zvfh1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl1024b1p0_zvl128b1p0_zvl256b1p0_zvl32b1p0_zvl512b1p0_zvl64b1p0_xsifivecdiscarddlone1p0_xsifivecflushdlone1p0
diff --git a/external/llvm-project/clang/test/Driver/range.c b/external/llvm-project/clang/test/Driver/range.c
index da5748d7c723..30140f3c208e 100644
--- a/external/llvm-project/clang/test/Driver/range.c
+++ b/external/llvm-project/clang/test/Driver/range.c
@@ -177,14 +177,83 @@
 // RUN: %clang -### -target x86_64 -ffast-math -fcomplex-arithmetic=basic -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=BASIC %s
 
-// BASIC: -complex-range=basic
-// FULL: -complex-range=full
-// PRMTD: -complex-range=promoted
-// BASIC-NOT: -complex-range=improved
-// CHECK-NOT: -complex-range=basic
-// IMPRVD: -complex-range=improved
-// IMPRVD-NOT: -complex-range=basic
-// CHECK-NOT: -complex-range=improved
+// RUN: %clang -### --target=x86_64 -fcx-limited-range -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE,WARN21 %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-cx-limited-range -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### --target=x86_64 -fcx-fortran-rules -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE,WARN22 %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-cx-fortran-rules -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### -Werror --target=x86_64 -ffast-math -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### --target=x86_64 -fcomplex-arithmetic=basic -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE,WARN23 %s
+
+// RUN: %clang -### --target=x86_64 -fcomplex-arithmetic=promoted -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE,WARN24 %s
+
+// RUN: %clang -### --target=x86_64 -fcomplex-arithmetic=improved -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE,WARN25 %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fcomplex-arithmetic=full -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### -Werror --target=x86_64 -ffp-model=aggressive -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### -Werror --target=x86_64 -ffp-model=fast -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### -Werror --target=x86_64 -ffp-model=precise -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### -Werror --target=x86_64 -ffp-model=strict -fno-fast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=RANGE %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fcx-limited-range \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=BASIC %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fno-cx-limited-range \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=FULL %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fcx-fortran-rules \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=IMPRVD %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fno-cx-fortran-rules \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=FULL %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -ffast-math \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=BASIC %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fcomplex-arithmetic=basic \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=BASIC %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fcomplex-arithmetic=promoted \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=PRMTD %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fcomplex-arithmetic=improved \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=IMPRVD %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -fcomplex-arithmetic=full \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=FULL %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -ffp-model=aggressive \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=BASIC %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -ffp-model=fast \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=PRMTD %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -ffp-model=precise \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=FULL %s
+
+// RUN: %clang -### -Werror --target=x86_64 -fno-fast-math -ffp-model=strict \
+// RUN:   -c %s 2>&1 | FileCheck --check-prefixes=FULL %s
 
 // WARN1: warning: overriding '-fcx-limited-range' option with '-fcx-fortran-rules' [-Woverriding-option]
 // WARN2: warning: overriding '-fno-cx-limited-range' option with '-fcx-fortran-rules' [-Woverriding-option]
@@ -196,5 +265,20 @@
 // WARN14: overriding '-complex-range=promoted' option with '-fcx-limited-range' [-Woverriding-option]
 // WARN17: warning: overriding '-fcomplex-arithmetic=full' option with '-fcomplex-arithmetic=basic' [-Woverriding-option]
 // WARN20: warning: overriding '-fcx-fortran-rules' option with '-fcx-limited-range' [-Woverriding-option]
+// WARN21: warning: overriding '-fcx-limited-range' option with '-fno-fast-math' [-Woverriding-option]
+// WARN22: warning: overriding '-fcx-fortran-rules' option with '-fno-fast-math' [-Woverriding-option]
+// WARN23: warning: overriding '-fcomplex-arithmetic=basic' option with '-fno-fast-math' [-Woverriding-option]
+// WARN24: warning: overriding '-fcomplex-arithmetic=promoted' option with '-fno-fast-math' [-Woverriding-option]
+// WARN25: warning: overriding '-fcomplex-arithmetic=improved' option with '-fno-fast-math' [-Woverriding-option]
+
+// BASIC: -complex-range=basic
+// FULL: -complex-range=full
+// PRMTD: -complex-range=promoted
+// BASIC-NOT: -complex-range=improved
+// CHECK-NOT: -complex-range=basic
+// IMPRVD: -complex-range=improved
+// IMPRVD-NOT: -complex-range=basic
+// CHECK-NOT: -complex-range=improved
+// RANGE-NOT: -complex-range=
 
 // ERR: error: unsupported argument 'foo' to option '-fcomplex-arithmetic='
diff --git a/external/llvm-project/clang/test/Driver/riscv-cpus.c b/external/llvm-project/clang/test/Driver/riscv-cpus.c
index 9b29ea2d76c7..7605a3056d43 100644
--- a/external/llvm-project/clang/test/Driver/riscv-cpus.c
+++ b/external/llvm-project/clang/test/Driver/riscv-cpus.c
@@ -429,6 +429,11 @@
 // MCPU-SIFIVE-X280-SAME: "-target-feature" "+zvl512b" "-target-feature" "+zvl64b"
 // MCPU-SIFIVE-X280-SAME: "-target-abi" "lp64d"
 
+// RUN: %clang -target riscv64 -### -c %s 2>&1 -menable-experimental-extensions -mcpu=sifive-x390 | FileCheck -check-prefix=MCPU-SIFIVE-X390 %s
+// MCPU-SIFIVE-X390: "-target-cpu" "sifive-x390"
+// COM: The list of extensions are tested in `test/Driver/print-enabled-extensions/riscv-sifive-x390.c`
+// MCPU-SIFIVE-X390-SAME: "-target-abi" "lp64d"
+
 // RUN: %clang -target riscv64 -### -c %s 2>&1 -mcpu=sifive-p450 | FileCheck -check-prefix=MCPU-SIFIVE-P450 %s
 // MCPU-SIFIVE-P450: "-nostdsysteminc" "-target-cpu" "sifive-p450"
 // MCPU-SIFIVE-P450-SAME: "-target-feature" "+m"
diff --git a/external/llvm-project/clang/test/Driver/stack-alignment.c b/external/llvm-project/clang/test/Driver/stack-alignment.c
new file mode 100644
index 000000000000..e1e62c05c32a
--- /dev/null
+++ b/external/llvm-project/clang/test/Driver/stack-alignment.c
@@ -0,0 +1,11 @@
+// RUN: not %clang -### -mstack-alignment=-1 %s 2>&1 | FileCheck %s --check-prefix=CHECK_NEG_1
+// RUN: %clang -### -mstack-alignment=0 %s 2>&1 | FileCheck %s --check-prefix=CHECK_0
+// RUN: %clang -### -mstack-alignment=1 %s 2>&1 | FileCheck %s --check-prefix=CHECK_1
+// RUN: %clang -### -mstack-alignment=4 %s 2>&1 | FileCheck %s --check-prefix=CHECK_4
+// RUN: not %clang -### -mstack-alignment=5 %s 2>&1 | FileCheck %s --check-prefix=CHECK_5
+
+// CHECK_NEG_1: error: invalid argument '-1' to -mstack-alignment=
+// CHECK_0: -mstack-alignment=0
+// CHECK_1: -mstack-alignment=1
+// CHECK_4: -mstack-alignment=4
+// CHECK_5: error: alignment is not a power of 2 in '-mstack-alignment=5'
diff --git a/external/llvm-project/clang/test/Driver/unique-source-file-names.c b/external/llvm-project/clang/test/Driver/unique-source-file-names.c
index 8322f0e37b0c..0dc71345d745 100644
--- a/external/llvm-project/clang/test/Driver/unique-source-file-names.c
+++ b/external/llvm-project/clang/test/Driver/unique-source-file-names.c
@@ -1,5 +1,11 @@
 // RUN: %clang -funique-source-file-names -### %s 2> %t
-// RUN: FileCheck < %t %s
+// RUN: FileCheck --check-prefix=SRC < %t %s
 
-// CHECK: "-cc1"
-// CHECK: "-funique-source-file-names"
+// SRC: "-cc1"
+// SRC: "-funique-source-file-identifier={{.*}}unique-source-file-names.c"
+
+// RUN: %clang -funique-source-file-names -funique-source-file-identifier=foo -### %s 2> %t
+// RUN: FileCheck --check-prefix=ID < %t %s
+
+// ID: "-cc1"
+// ID: "-funique-source-file-identifier=foo"
diff --git a/external/llvm-project/clang/test/ExtractAPI/availability.c b/external/llvm-project/clang/test/ExtractAPI/availability.c
index 237b2ffa55d7..4f7e21a5c376 100644
--- a/external/llvm-project/clang/test/ExtractAPI/availability.c
+++ b/external/llvm-project/clang/test/ExtractAPI/availability.c
@@ -17,7 +17,7 @@ void a(void) __attribute__((availability(macos, introduced=12.0)));
 // A-NEXT: ]
 
 // RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix B
-void b(void) __attribute__((availability(macos, introduced=11.0, deprecated=12.0, obsoleted=20.0)));
+void b(void) __attribute__((availability(macos, introduced=11.0, deprecated=12.0, obsoleted=30.0)));
 // B-LABEL: "!testLabel": "c:@F@b"
 // B:      "availability": [
 // B-NEXT:   {
@@ -33,7 +33,7 @@ void b(void) __attribute__((availability(macos, introduced=11.0, deprecated=12.0
 // B-NEXT:       "patch": 0
 // B-NEXT:     },
 // B-NEXT:     "obsoleted": {
-// B-NEXT:       "major": 20,
+// B-NEXT:       "major": 30,
 // B-NEXT:       "minor": 0,
 // B-NEXT:       "patch": 0
 // B-NEXT:     }
@@ -41,7 +41,7 @@ void b(void) __attribute__((availability(macos, introduced=11.0, deprecated=12.0
 // B-NEXT: ]
 
 // RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix E
-void c(void) __attribute__((availability(macos, introduced=11.0, deprecated=12.0, obsoleted=20.0))) __attribute__((availability(ios, introduced=13.0)));
+void c(void) __attribute__((availability(macos, introduced=11.0, deprecated=12.0, obsoleted=30.0))) __attribute__((availability(ios, introduced=13.0)));
 // C-LABEL: "!testLabel": "c:@F@c"
 // C:       "availability": [
 // C-NEXT:    {
@@ -57,7 +57,7 @@ void c(void) __attribute__((availability(macos, introduced=11.0, deprecated=12.0
 // C-NEXT:        "patch": 0
 // C-NEXT:      },
 // C-NEXT:      "obsoleted": {
-// C-NEXT:        "major": 20,
+// C-NEXT:        "major": 30,
 // C-NEXT:        "minor": 0,
 // C-NEXT:        "patch": 0
 // C-NEXT:      }
diff --git a/external/llvm-project/clang/test/ExtractAPI/inherited_availability.m b/external/llvm-project/clang/test/ExtractAPI/inherited_availability.m
index c24e7fa8e208..3d9f953bbda1 100644
--- a/external/llvm-project/clang/test/ExtractAPI/inherited_availability.m
+++ b/external/llvm-project/clang/test/ExtractAPI/inherited_availability.m
@@ -4,7 +4,7 @@
 
 
 // RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix A
-__attribute__((availability(macos, introduced=9.0, deprecated=12.0, obsoleted=20.0)))
+__attribute__((availability(macos, introduced=9.0, deprecated=12.0, obsoleted=30.0)))
 @interface A
 // A-LABEL: "!testLabel": "c:objc(cs)A"
 // A:      "availability": [
@@ -21,7 +21,7 @@ @interface A
 // A-NEXT:       "patch": 0
 // A-NEXT:     }
 // A-NEXT:     "obsoleted": {
-// A-NEXT:       "major": 20,
+// A-NEXT:       "major": 30,
 // A-NEXT:       "minor": 0,
 // A-NEXT:       "patch": 0
 // A-NEXT:     }
@@ -45,7 +45,7 @@ @interface A
 // CP-NEXT:       "patch": 0
 // CP-NEXT:     }
 // CP-NEXT:     "obsoleted": {
-// CP-NEXT:       "major": 20,
+// CP-NEXT:       "major": 30,
 // CP-NEXT:       "minor": 0,
 // CP-NEXT:       "patch": 0
 // CP-NEXT:     }
@@ -69,7 +69,7 @@ @interface A
 // IP-NEXT:       "patch": 0
 // IP-NEXT:     }
 // IP-NEXT:     "obsoleted": {
-// IP-NEXT:       "major": 20,
+// IP-NEXT:       "major": 30,
 // IP-NEXT:       "minor": 0,
 // IP-NEXT:       "patch": 0
 // IP-NEXT:     }
@@ -77,7 +77,7 @@ @interface A
 // IP-NEXT: ]
 
 // RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix MR
-@property int moreRestrictive __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=19.0)));
+@property int moreRestrictive __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=29.0)));
 // MR-LABEL: "!testLabel": "c:objc(cs)A(py)moreRestrictive"
 // MR:      "availability": [
 // MR-NEXT:   {
@@ -93,7 +93,7 @@ @interface A
 // MR-NEXT:       "patch": 0
 // MR-NEXT:     }
 // MR-NEXT:     "obsoleted": {
-// MR-NEXT:       "major": 19,
+// MR-NEXT:       "major": 29,
 // MR-NEXT:       "minor": 0,
 // MR-NEXT:       "patch": 0
 // MR-NEXT:     }
@@ -148,7 +148,7 @@ @interface C
 
 @interface D
 // RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix DIP
-@property int DIP __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=19.0)));
+@property int DIP __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=29.0)));
 // DIP-LABEL: "!testLabel": "c:objc(cs)D(py)DIP"
 // DIP:      "availability": [
 // DIP-NEXT:   {
@@ -164,7 +164,7 @@ @interface D
 // DIP-NEXT:       "patch": 0
 // DIP-NEXT:     }
 // DIP-NEXT:     "obsoleted": {
-// DIP-NEXT:       "major": 19,
+// DIP-NEXT:       "major": 29,
 // DIP-NEXT:       "minor": 0,
 // DIP-NEXT:       "patch": 0
 // DIP-NEXT:     }
diff --git a/external/llvm-project/clang/test/FixIt/fixit-punctuator-spelling.cpp b/external/llvm-project/clang/test/FixIt/fixit-punctuator-spelling.cpp
new file mode 100644
index 000000000000..3cba0e7b6459
--- /dev/null
+++ b/external/llvm-project/clang/test/FixIt/fixit-punctuator-spelling.cpp
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: not %clang_cc1 -fsyntax-only -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s
+
+void f(int x) {
+  switch (x) {
+    case 1 // expected-error {{expected ':' after 'case'}}
+      break;
+  }
+}
+// CHECK: fix-it:"{{.*}}":{6:11-6:11}:":"
diff --git a/external/llvm-project/clang/test/FixIt/fixit-unknown-attributes.cpp b/external/llvm-project/clang/test/FixIt/fixit-unknown-attributes.cpp
new file mode 100644
index 000000000000..7dff510f5ddf
--- /dev/null
+++ b/external/llvm-project/clang/test/FixIt/fixit-unknown-attributes.cpp
@@ -0,0 +1,74 @@
+// RUN: %clang_cc1 -Wunknown-attributes -fsyntax-only -verify %s
+// RUN: %clang_cc1 -Wunknown-attributes -fsyntax-only -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s
+
+[[gmu::deprected]] // expected-warning {{unknown attribute 'gmu::deprected' ignored; did you mean 'gnu::deprecated'?}}
+int f1(void) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:3-[[@LINE-4]]:17}:"gnu::deprecated"
+
+[[gmu::deprecated]] // expected-warning {{unknown attribute 'gmu::deprecated' ignored; did you mean 'gnu::deprecated'?}}
+int f2(void) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:3-[[@LINE-4]]:18}:"gnu::deprecated"
+
+[[gnu::deprected]] // expected-warning {{unknown attribute 'gnu::deprected' ignored; did you mean 'gnu::deprecated'?}}
+int f3(void) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:3-[[@LINE-4]]:17}:"gnu::deprecated"
+
+[[deprected]] // expected-warning {{unknown attribute 'deprected' ignored; did you mean 'deprecated'?}}
+int f4(void) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:3-[[@LINE-4]]:12}:"deprecated"
+
+[[using gmu : deprected]] // expected-warning {{unknown attribute 'gmu::deprected' ignored; did you mean 'gnu::deprecated'?}}
+int f5(void) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:9-[[@LINE-4]]:12}:"gnu"
+// CHECK: fix-it:"{{.*}}":{[[@LINE-5]]:15-[[@LINE-5]]:24}:"deprecated"
+
+[[using gmu : deprecated]] // expected-warning {{unknown attribute 'gmu::deprecated' ignored; did you mean 'gnu::deprecated'?}}
+int f6(void) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:9-[[@LINE-4]]:12}:"gnu"
+
+[[using gnu : deprected]] // expected-warning {{unknown attribute 'gnu::deprected' ignored; did you mean 'gnu::deprecated'?}}
+int f7(void) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:15-[[@LINE-4]]:24}:"deprecated"
+
+[[using gnu : deprecated, noretyrn]] // expected-warning {{unknown attribute 'gnu::noretyrn' ignored; did you mean 'gnu::noreturn'?}}
+void f8(void) {
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:27-[[@LINE-3]]:35}:"noreturn"
+
+[[using gmu : deprected, noretyrn]] // expected-warning {{unknown attribute 'gmu::deprected' ignored; did you mean 'gnu::deprecated'?}} \
+                                    // expected-warning {{unknown attribute 'gmu::noretyrn' ignored; did you mean 'gnu::noreturn'?}}
+void f9(void) {
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:9-[[@LINE-4]]:12}:"gnu"
+// CHECK: fix-it:"{{.*}}":{[[@LINE-5]]:15-[[@LINE-5]]:24}:"deprecated"
+
+// CHECK: fix-it:"{{.*}}":{[[@LINE-7]]:9-[[@LINE-7]]:12}:"gnu"
+// CHECK: fix-it:"{{.*}}":{[[@LINE-8]]:26-[[@LINE-8]]:34}:"noreturn"
+
+__attribute__((cont, deprected)) // expected-warning {{unknown attribute 'cont' ignored; did you mean 'const'?}} \
+                                 // expected-warning {{unknown attribute 'deprected' ignored; did you mean 'deprecated'?}}
+int f10(int) {
+  return 0;
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-5]]:16-[[@LINE-5]]:20}:"const"
+// CHECK: fix-it:"{{.*}}":{[[@LINE-6]]:22-[[@LINE-6]]:31}:"deprecated"
+
+[[using gnu: noretyrn, address_spaci(0)]] // expected-warning {{unknown attribute 'gnu::noretyrn' ignored; did you mean 'gnu::noreturn'?}} \
+                                          // expected-warning {{unknown attribute 'gnu::address_spaci' ignored}}
+void f11(void) {
+}
+// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:14-[[@LINE-4]]:22}:"noreturn"
diff --git a/external/llvm-project/clang/test/FixIt/fixit-unused-lambda-capture-trailing-tokens.cpp b/external/llvm-project/clang/test/FixIt/fixit-unused-lambda-capture-trailing-tokens.cpp
new file mode 100644
index 000000000000..80250e6d1851
--- /dev/null
+++ b/external/llvm-project/clang/test/FixIt/fixit-unused-lambda-capture-trailing-tokens.cpp
@@ -0,0 +1,491 @@
+// RUN: cp %s %t
+// RUN: %clang_cc1 -x c++ -Wunused-lambda-capture -Wno-unused-value -std=c++1z -fixit %t
+// RUN: grep -v CHECK %t | FileCheck %s
+// RUN: %clang_cc1 -x c++ -Wunused-lambda-capture -Wno-unused-value -std=c++1z -fsyntax-only %s 2>&1 | FileCheck --match-full-lines --strict-whitespace --check-prefix POINTER %s
+
+
+#define MACRO_CAPTURE(...) __VA_ARGS__
+#define M_A a
+#define M_B b
+
+int main() {
+    int a = 0, b = 0, c = 0, d = 0;
+    auto F0 = [a, &b]() mutable {
+    // CHECK: auto F0 = [a]()
+         // POINTER:{{^.*}}|     auto F0 = [a, &b]() mutable {
+    // POINTER-NEXT:{{^.*}}|                 ~~~^
+
+        a++;
+    };
+
+    auto F1 = [&a, &b]() {
+    // CHECK: auto F1 = []() {
+         // POINTER:{{^.*}}|     auto F1 = [&a, &b]() {
+    // POINTER-NEXT:{{^.*}}|                ~^~
+         // POINTER:{{^.*}}|     auto F1 = [&a, &b]() {
+    // POINTER-NEXT:{{^.*}}|                  ~~~^
+    };
+
+    auto F2 = [&a, b]() {
+    // CHECK: auto F2 = []() {
+         // POINTER:{{^.*}}|     auto F2 = [&a, b]() {
+    // POINTER-NEXT:{{^.*}}|                ~^~
+         // POINTER:{{^.*}}|     auto F2 = [&a, b]() {
+    // POINTER-NEXT:{{^.*}}|                  ~~^
+    };
+
+    auto F3 = [&a,
+         &b]() {
+    // CHECK: auto F3 = []() {
+         // POINTER:{{^.*}}|     auto F3 = [&a,
+    // POINTER-NEXT:{{^.*}}|                ~^~
+    // POINTER-NEXT:{{^.*}}|          &b]() {
+         // POINTER:{{^.*}}|     auto F3 = [&a,
+    // POINTER-NEXT:{{^.*}}|                  ~
+    // POINTER-NEXT:{{^.*}}|          &b]() {
+    // POINTER-NEXT:{{^.*}}|          ~^
+
+    };
+
+    auto F4 = [&a
+        , &b]() {
+    // CHECK: auto F4 = []() {
+         // POINTER:{{^.*}}|     auto F4 = [&a
+    // POINTER-NEXT:{{^.*}}|                ~^
+    // POINTER-NEXT:{{^.*}}|         , &b]() {
+    // POINTER-NEXT:{{^.*}}|         ~
+         // POINTER:{{^.*}}|     auto F4 = [&a
+    // POINTER-NEXT:{{^.*}}|                  {{$}}
+    // POINTER-NEXT:{{^.*}}|         , &b]() {
+    // POINTER-NEXT:{{^.*}}|         ~~~^
+    };
+    auto F5 = [&a ,&b]()  {
+    // CHECK: auto F5 = []() {
+         // POINTER:{{^.*}}|     auto F5 = [&a ,&b]()  {
+    // POINTER-NEXT:{{^.*}}|                ~^~~
+         // POINTER:{{^.*}}|     auto F5 = [&a ,&b]()  {
+    // POINTER-NEXT:{{^.*}}|                   ~~^
+    };
+
+    auto F0a = [a, &b]() mutable {
+    // CHECK: auto F0a = [a]() mutable {
+         // POINTER:{{^.*}}|     auto F0a = [a, &b]() mutable {
+    // POINTER-NEXT:{{^.*}}|                  ~~~^
+        a++;
+    };
+
+    auto F1a = [&a, &b]() {
+    // CHECK: auto F1a = [&a]() {
+         // POINTER:{{^.*}}|     auto F1a = [&a, &b]() {
+    // POINTER-NEXT:{{^.*}}|                   ~~~^
+        a++;
+    };
+
+    auto F2a = [&a, b]() {
+    // CHECK: auto F2a = [&a]() {
+         // POINTER:{{^.*}}|     auto F2a = [&a, b]() {
+    // POINTER-NEXT:{{^.*}}|                   ~~^
+        a++;
+    };
+
+    auto F3a = [&a,
+         &b]() {
+    // CHECK: auto F3a = [&a]() {
+         // POINTER:{{^.*}}|     auto F3a = [&a,
+    // POINTER-NEXT:{{^.*}}|                   ~
+    // POINTER-NEXT:{{^.*}}|          &b]() {
+    // POINTER-NEXT:{{^.*}}|          ~^
+        a++;
+    };
+
+    auto F4a = [&a
+        , &b]() {
+    // CHECK: auto F4a = [&a]() {
+         // POINTER:{{^.*}}|     auto F4a = [&a
+    // POINTER-NEXT:{{^.*}}|                   {{$}}
+    // POINTER-NEXT:{{^.*}}|         , &b]() {
+    // POINTER-NEXT:{{^.*}}|         ~~~^
+        a++;
+    };
+
+    auto F5a = [&a ,&b]() {
+    // CHECK: auto F5a = [&a]() {
+         // POINTER:{{^.*}}|     auto F5a = [&a ,&b]() {
+    // POINTER-NEXT:{{^.*}}|                    ~~^
+        a++;
+    };
+    auto F0b = [a, &b]() mutable {
+    // CHECK: auto F0b = [ &b]() mutable
+         // POINTER:{{^.*}}|     auto F0b = [a, &b]() mutable {
+    // POINTER-NEXT:{{^.*}}|                 ^~
+
+        b++;
+    };
+
+    auto F1b = [&a, &b]() {
+    // CHECK: auto F1b = [ &b]() {
+         // POINTER:{{^.*}}|     auto F1b = [&a, &b]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^~
+        b++;
+    };
+
+    auto F2b = [&a, b]() mutable {
+    // CHECK: auto F2b = [ b]() mutable {
+         // POINTER:{{^.*}}|     auto F2b = [&a, b]() mutable {
+    // POINTER-NEXT:{{^.*}}|                 ~^~
+        b++;
+    };
+
+    auto F3b = [&a,
+         &b]() {
+    // CHECK: auto F3b = [ &b]() {
+         // POINTER:{{^.*}}|     auto F3b = [&a,
+    // POINTER-NEXT:{{^.*}}|                 ~^~
+    // POINTER-NEXT:{{^.*}}|          &b]() {
+        b++;
+    };
+
+    auto F4b = [&a
+        , &b]() {
+    // CHECK: auto F4b = [ &b]() {
+         // POINTER:{{^.*}}|     auto F4b = [&a
+    // POINTER-NEXT:{{^.*}}|                 ~^
+    // POINTER-NEXT:{{^.*}}|         , &b]() {
+    // POINTER-NEXT:{{^.*}}|         ~
+
+        b++;
+    };
+    auto F5b = [&a ,&b]() {
+    // CHECK: auto F5b = [&b]() {
+         // POINTER:{{^.*}}|     auto F5b = [&a ,&b]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^~~
+        b++;
+    };
+
+    auto F6 = [&a, &b, &c]() {
+    // CHECK: auto F6 = [&a, &b]() {
+         // POINTER:{{^.*}}|     auto F6 = [&a, &b, &c]() {
+    // POINTER-NEXT:{{^.*}}|                      ~~~^
+        a++;
+        b++;
+    };
+    auto F7 = [&a, &b, &c]() {
+    // CHECK: auto F7 = [&a, &c]() {
+         // POINTER:{{^.*}}|     auto F7 = [&a, &b, &c]() {
+    // POINTER-NEXT:{{^.*}}|                  ~~~^
+        a++;
+        c++;
+    };
+    auto F8 = [&a, &b, &c]() {
+    // CHECK: auto F8 = [ &b, &c]() {
+         // POINTER:{{^.*}}|     auto F8 = [&a, &b, &c]() {
+    // POINTER-NEXT:{{^.*}}|                ~^~
+        b++;
+        c++;
+    };
+    auto F9 = [&a, &b    , &c]() {
+    // CHECK: auto F9 = [&a   , &c]() {
+         // POINTER:{{^.*}}|     auto F9 = [&a, &b    , &c]() {
+    // POINTER-NEXT:{{^.*}}|                  ~~~^
+        a++;
+        c++;
+    };
+    auto F10 = [&a,
+         &b, &c]() {
+    // CHECK: auto F10 = [&a, &c]() {
+         // POINTER:{{^.*}}|     auto F10 = [&a,
+    // POINTER-NEXT:{{^.*}}|                   ~
+    // POINTER-NEXT:{{^.*}}|          &b, &c]() {
+    // POINTER-NEXT:{{^.*}}|          ~^
+        a++;
+        c++;
+    };
+    auto F11 = [&a,  &b  ,
+         &c]() {
+    // CHECK: auto F11 = [&a ,
+    // CHECK-NEXT:      &c]() {
+         // POINTER:{{^.*}}|     auto F11 = [&a,  &b  ,
+    // POINTER-NEXT:{{^.*}}|                   ~~~~^
+        a++;
+        c++;
+    };
+    auto F12 = [a = 0,  b  ,
+         c]() mutable {
+    // CHECK: auto F12 = [ b  ,
+    // CHECK-NEXT:     c]() mutable {
+         // POINTER:{{^.*}}|     auto F12 = [a = 0,  b  ,
+    // POINTER-NEXT:{{^.*}}|                 ^~~~~~
+        b++;
+        c++;
+    };
+    auto F13 = [a,  b = 0 ,
+         c]() mutable {
+    // CHECK: auto F13 = [a ,
+    // CHECK-NEXT:     c]() mutable {
+         // POINTER:{{^.*}}|     auto F13 = [a,  b = 0 ,
+    // POINTER-NEXT:{{^.*}}|                  ~~~^~~~~
+        a++;
+        c++;
+    };
+    auto F14 = [a,  b ,
+         c
+        = 0]() mutable {
+    // CHECK: auto F14 = [a,  b]() mutable {
+         // POINTER:{{^.*}}|     auto F14 = [a,  b ,
+    // POINTER-NEXT:{{^.*}}|                       ~
+    // POINTER-NEXT:{{^.*}}|          c
+    // POINTER-NEXT:{{^.*}}|          ^
+    // POINTER-NEXT:{{^.*}}|         = 0]() mutable {
+    // POINTER-NEXT:{{^.*}}|         ~~~
+        a++;
+        b++;
+    };
+
+    // We want to remove everything including the comment
+    // as well as the comma following the capture of `a`
+    auto F15 = [&a /* comment */, &b]() {
+    // CHECK: auto F15 = [ &b]() {
+         // POINTER:{{^.*}}|     auto F15 = [&a /* comment */, &b]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^~~~~~~~~~~~~~~~
+        b++;
+    };
+
+    // The comment preceding the first capture remains. This is more
+    // by design of the fixit logic than anything else, but we should
+    // consider the preceding comment might actually be a comment for
+    // the entire capture set, so maybe we do want it to hang around
+    auto F16 = [/* comment */ &a , &b]() {
+    // CHECK: auto F16 = [/* comment */ &b]() {
+         // POINTER:{{^.*}}|     auto F16 = [/* comment */ &a , &b]() {
+    // POINTER-NEXT:{{^.*}}|                               ~^~~
+        b++;
+    };
+
+    auto F16b = [&a ,    /* comment */ &b]() {
+    // CHECK: auto F16b = [ /* comment */ &b]() {
+         // POINTER:{{^.*}}|     auto F16b = [&a ,    /* comment */ &b]() {
+    // POINTER-NEXT:{{^.*}}|                  ~^~~
+        b++;
+    };
+
+    auto F17 = [&a /* comment */, &b]() {
+    // CHECK: auto F17 = [&a]() {
+         // POINTER:{{^.*}}|     auto F17 = [&a /* comment */, &b]() {
+    // POINTER-NEXT:{{^.*}}|                    ~~~~~~~~~~~~~~~~^
+
+        a++;
+    };
+
+    auto F18 = [&a , /* comment */ &b]() {
+    // CHECK: auto F18 = [&a]() {
+         // POINTER:{{^.*}}|     auto F18 = [&a , /* comment */ &b]() {
+    // POINTER-NEXT:{{^.*}}|                    ~~~~~~~~~~~~~~~~~^
+
+        a++;
+    };
+
+    auto F19 = [&a /* comment */, &b /* comment */]() {
+    // CHECK: auto F19 = [&a /* comment */]() {
+         // POINTER:{{^.*}}|     auto F19 = [&a /* comment */, &b /* comment */]() {
+    // POINTER-NEXT:{{^.*}}|                    ~~~~~~~~~~~~~~~~^
+
+        a++;
+    };
+
+    auto F20 = [MACRO_CAPTURE(&a, &b)]() {
+    // CHECK: auto F20 = [MACRO_CAPTURE(&a, &b)]() {
+         // POINTER:{{^.*}}|     auto F20 = [MACRO_CAPTURE(&a, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                               ~^
+          // POINTER:{{^.*}}|     auto F20 = [MACRO_CAPTURE(&a, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                                   ~^
+    };
+
+    auto F21 = [MACRO_CAPTURE(&a), &b]() {
+    // CHECK: auto F21 = []() {
+         // POINTER:{{^.*}}|     auto F21 = [MACRO_CAPTURE(&a), &b]() {
+    // POINTER-NEXT:{{^.*}}|                 ~~~~~~~~~~~~~~~^~~
+         // POINTER:{{^.*}}|     auto F21 = [MACRO_CAPTURE(&a), &b]() {
+    // POINTER-NEXT:{{^.*}}|                                  ~~~^
+    };
+
+    auto F22 = [MACRO_CAPTURE(&a,) &b]() {
+    // CHECK: auto F22 = [MACRO_CAPTURE(&a,) &b]() {
+         // POINTER:{{^.*}}|     auto F22 = [MACRO_CAPTURE(&a,) &b]() {
+    // POINTER-NEXT:{{^.*}}|                               ~^
+         // POINTER:{{^.*}}|     auto F22 = [MACRO_CAPTURE(&a,) &b]() {
+    // POINTER-NEXT:{{^.*}}|                                    ~^
+    };
+    auto F23 = [&a MACRO_CAPTURE(, &b)]() {
+    // CHECK: auto F23 = [&a]() {
+         // POINTER:{{^.*}}|     auto F23 = [&a MACRO_CAPTURE(, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^
+         // POINTER:{{^.*}}|     auto F23 = [&a MACRO_CAPTURE(, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                    ~~~~~~~~~~~~~~~~~^~
+    };
+    auto F24 = [&a, MACRO_CAPTURE(&b)]() {
+    // CHECK: auto F24 = []() {
+         // POINTER:{{^.*}}|     auto F24 = [&a, MACRO_CAPTURE(&b)]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^~
+         // POINTER:{{^.*}}|     auto F24 = [&a, MACRO_CAPTURE(&b)]() {
+    // POINTER-NEXT:{{^.*}}|                   ~~~~~~~~~~~~~~~~~^~
+    };
+
+    auto F20a = [MACRO_CAPTURE(&a, &b)]() {
+    // CHECK: auto F20a = [MACRO_CAPTURE(&a, &b)]() {
+         // POINTER:{{^.*}}|     auto F20a = [MACRO_CAPTURE(&a, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                                    ~^
+      a++;
+    };
+
+    auto F21a = [MACRO_CAPTURE(&a), &b]() {
+    // CHECK: auto F21a = [MACRO_CAPTURE(&a)]() {
+         // POINTER:{{^.*}}|     auto F21a = [MACRO_CAPTURE(&a), &b]() {
+    // POINTER-NEXT:{{^.*}}|                                   ~~~^
+      a++;
+    };
+
+    auto F22a = [MACRO_CAPTURE(&a,) &b]() {
+    // CHECK: auto F22a = [MACRO_CAPTURE(&a,) &b]() {
+         // POINTER:{{^.*}}|     auto F22a = [MACRO_CAPTURE(&a,) &b]() {
+    // POINTER-NEXT:{{^.*}}|                                     ~^
+      a++;
+    };
+    auto F23a = [&a MACRO_CAPTURE(, &b)]() {
+    // CHECK: auto F23a = [&a]() {
+         // POINTER:{{^.*}}|     auto F23a = [&a MACRO_CAPTURE(, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                     ~~~~~~~~~~~~~~~~~^~
+      a++;
+    };
+    auto F24a = [&a, MACRO_CAPTURE(&b)]() {
+    // CHECK: auto F24a = [&a]() {
+         // POINTER:{{^.*}}|     auto F24a = [&a, MACRO_CAPTURE(&b)]() {
+    // POINTER-NEXT:{{^.*}}|                    ~~~~~~~~~~~~~~~~~^~
+      a++;
+    };
+    auto F20b = [MACRO_CAPTURE(&a, &b)]() {
+    // CHECK: auto F20b = [MACRO_CAPTURE(&a, &b)]() {
+         // POINTER:{{^.*}}|     auto F20b = [MACRO_CAPTURE(&a, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                                ~^
+      b++;
+    };
+
+    auto F21b = [MACRO_CAPTURE(&a), &b]() {
+    // CHECK: auto F21b = [ &b]() {
+         // POINTER:{{^.*}}|     auto F21b = [MACRO_CAPTURE(&a), &b]() {
+    // POINTER-NEXT:{{^.*}}|                  ~~~~~~~~~~~~~~~^~~
+      b++;
+    };
+
+    auto F22b = [MACRO_CAPTURE(&a,) &b]() {
+    // CHECK: auto F22b = [MACRO_CAPTURE(&a,) &b]() {
+         // POINTER:{{^.*}}|     auto F22b = [MACRO_CAPTURE(&a,) &b]() {
+    // POINTER-NEXT:{{^.*}}|                                ~^
+      b++;
+    };
+    auto F23b = [&a MACRO_CAPTURE(, &b)]() {
+    // CHECK: auto F23b = [&a MACRO_CAPTURE(, &b)]() {
+         // POINTER:{{^.*}}|     auto F23b = [&a MACRO_CAPTURE(, &b)]() {
+    // POINTER-NEXT:{{^.*}}|                  ~^
+      b++;
+    };
+    auto F24b = [&a, MACRO_CAPTURE(&b)]() {
+    // CHECK: auto F24b = [ MACRO_CAPTURE(&b)]() {
+         // POINTER:{{^.*}}|     auto F24b = [&a, MACRO_CAPTURE(&b)]() {
+    // POINTER-NEXT:{{^.*}}|                  ~^~
+      b++;
+    };
+
+    auto F25ma = [&M_A, &b]() {
+    // CHECK:     auto F25ma = []() {
+         // POINTER:{{^.*}}|     auto F25ma = [&M_A, &b]() {
+    // POINTER-NEXT:{{^.*}}|                   ~^~~~
+         // POINTER:{{^.*}}|     auto F25ma = [&M_A, &b]() {
+    // POINTER-NEXT:{{^.*}}|                       ~~~^
+    };
+    auto F25mb = [&a, &M_B]() {
+    // CHECK: auto F25mb = []() {
+         // POINTER:{{^.*}}|     auto F25mb = [&a, &M_B]() {
+    // POINTER-NEXT:{{^.*}}|                   ~^~
+         // POINTER:{{^.*}}|     auto F25mb = [&a, &M_B]() {
+    // POINTER-NEXT:{{^.*}}|                     ~~~^~~
+    };
+
+    auto F25mab = [&M_A, &b]() {
+    // CHECK: auto F25mab = [ &b]() {
+         // POINTER:{{^.*}}|     auto F25mab = [&M_A, &b]() {
+    // POINTER-NEXT:{{^.*}}|                    ~^~~~
+        b++;
+    };
+    auto F25amb = [&a, &M_B]() {
+    //CHECK: auto F25amb = []() {
+         // POINTER:{{^.*}}|     auto F25amb = [&a, &M_B]() {
+    // POINTER-NEXT:{{^.*}}|                    ~^~
+         // POINTER:{{^.*}}|     auto F25amb = [&a, &M_B]() {
+    // POINTER-NEXT:{{^.*}}|                      ~~~^~~
+    };
+
+    auto F26 = [&a, &b, &c, &d]() {
+    // CHECK: auto F26 = [&a, &b]() {
+         // POINTER:{{^.*}}|     auto F26 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                       ~~~^
+         // POINTER:{{^.*}}|     auto F26 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                           ~~~^
+
+        (void)a;
+        (void)b;
+    };
+
+    auto F27 = [&a, &b, &c, &d]() {
+    // CHECK: auto F27 = [&a, &c]() {
+         // POINTER:{{^.*}}|     auto F27 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                   ~~~^
+         // POINTER:{{^.*}}|     auto F27 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                           ~~~^
+
+        (void)a;
+        (void)c;
+    };
+
+    auto F28 = [&a, &b, &c, &d]() {
+        // CHECK: auto F28 = [&a, &d]() {
+         // POINTER:{{^.*}}|     auto F28 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                   ~~~^
+         // POINTER:{{^.*}}|     auto F28 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                       ~~~^
+
+        (void)a;
+        (void)d;
+    };
+
+    auto F29 = [&a, &b, &c, &d]() {
+    // CHECK: auto F29 = [ &b, &c]() {
+         // POINTER:{{^.*}}|     auto F29 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^~
+         // POINTER:{{^.*}}|     auto F29 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                           ~~~^
+
+        (void)b;
+        (void)c;
+    };
+
+    auto F30 = [&a, &b, &c, &d]() {
+    // CHECK: auto F30 = [ &b, &d]() {
+         // POINTER:{{^.*}}|     auto F30 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^~
+         // POINTER:{{^.*}}|     auto F30 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                       ~~~^
+
+        (void)b;
+        (void)d;
+    };
+
+    auto F31 = [&a, &b, &c, &d]() {
+    // CHECK: auto F31 = [ &c, &d]() {
+         // POINTER:{{^.*}}|     auto F31 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                 ~^~
+         // POINTER:{{^.*}}|     auto F31 = [&a, &b, &c, &d]() {
+    // POINTER-NEXT:{{^.*}}|                     ~^~
+        (void)c;
+        (void)d;
+    };
+}
diff --git a/external/llvm-project/clang/test/FixIt/typo.cpp b/external/llvm-project/clang/test/FixIt/typo.cpp
deleted file mode 100644
index e489fbbcaa1d..000000000000
--- a/external/llvm-project/clang/test/FixIt/typo.cpp
+++ /dev/null
@@ -1,137 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
-// RUN: cp %s %t
-// RUN: not %clang_cc1 -fixit -x c++ %t
-// RUN: %clang_cc1 -fsyntax-only -pedantic -Werror -x c++ %t
-// RUN: grep test_string %t
-
-namespace std {
-  template<typename T> class basic_string { // expected-note 3{{'basic_string' declared here}}
-  public:
-    int find(const char *substr); // expected-note{{'find' declared here}}
-    static const int npos = -1; // expected-note{{'npos' declared here}}
-  };
-
-  typedef basic_string<char> string; // expected-note 2{{'string' declared here}}
-}
-
-namespace otherstd { // expected-note 2{{'otherstd' declared here}} \
-                     // expected-note{{namespace 'otherstd' defined here}}
-  using namespace std;
-}
-
-using namespace std;
-
-other_std::strng str1; // expected-error{{use of undeclared identifier 'other_std'; did you mean 'otherstd'?}} \
-// expected-error{{no type named 'strng' in namespace 'otherstd'; did you mean 'string'?}}
-tring str2; // expected-error{{unknown type name 'tring'; did you mean 'string'?}}
-
-::other_std::string str3; // expected-error{{no member named 'other_std' in the global namespace; did you mean 'otherstd'?}}
-
-float area(float radius, // expected-note{{'radius' declared here}}
-           float pi) {
-  return radious * pi; // expected-error{{did you mean 'radius'?}}
-}
-
-using namespace othestd; // expected-error{{no namespace named 'othestd'; did you mean 'otherstd'?}}
-namespace blargh = otherstd; // expected-note 3{{namespace 'blargh' defined here}}
-using namespace ::blarg; // expected-error{{no namespace named 'blarg' in the global namespace; did you mean 'blargh'?}}
-
-namespace wibble = blarg; // expected-error{{no namespace named 'blarg'; did you mean 'blargh'?}}
-namespace wobble = ::blarg; // expected-error{{no namespace named 'blarg' in the global namespace; did you mean 'blargh'?}}
-
-bool test_string(std::string s) {
-  basc_string<char> b1; // expected-error{{no template named 'basc_string'; did you mean 'basic_string'?}}
-  std::basic_sting<char> b2; // expected-error{{no template named 'basic_sting' in namespace 'std'; did you mean 'basic_string'?}}
-  (void)b1;
-  (void)b2;
-  return s.fnd("hello") // expected-error{{no member named 'fnd' in 'std::basic_string<char>'; did you mean 'find'?}}
-    == std::string::pos; // expected-error{{no member named 'pos' in 'std::basic_string<char>'; did you mean 'npos'?}}
-}
-
-struct Base { };
-struct Derived : public Base { // expected-note{{base class 'Base' specified here}}
-  int member; // expected-note 3{{'member' declared here}}
-
-  Derived() : base(), // expected-error{{initializer 'base' does not name a non-static data member or base class; did you mean the base class 'Base'?}}
-              ember() { } // expected-error{{initializer 'ember' does not name a non-static data member or base class; did you mean the member 'member'?}}
-
-  int getMember() const {
-    return ember; // expected-error{{use of undeclared identifier 'ember'; did you mean 'member'?}}
-  }
-
-  int &getMember();
-};
-
-int &Derived::getMember() {
-  return ember; // expected-error{{use of undeclared identifier 'ember'; did you mean 'member'?}}
-}
-
-typedef int Integer; // expected-note{{'Integer' declared here}}
-int global_value; // expected-note{{'global_value' declared here}}
-
-int foo() {
-  integer * i = 0; // expected-error{{unknown type name 'integer'; did you mean 'Integer'?}}
-  unsinged *ptr = 0; // expected-error{{use of undeclared identifier 'unsinged'; did you mean 'unsigned'?}}
-  return *i + *ptr + global_val; // expected-error{{use of undeclared identifier 'global_val'; did you mean 'global_value'?}}
-}
-
-namespace nonstd {
-  typedef std::basic_string<char> yarn; // expected-note 2 {{'nonstd::yarn' declared here}}
-  int narf; // expected-note{{'nonstd::narf' declared here}}
-}
-
-yarn str4; // expected-error{{unknown type name 'yarn'; did you mean 'nonstd::yarn'?}}
-wibble::yarn str5; // expected-error{{no type named 'yarn' in namespace 'otherstd'; did you mean 'nonstd::yarn'?}}
-
-namespace another {
-  template<typename T> class wide_string {}; // expected-note {{'another::wide_string' declared here}}
-}
-int poit() {
-  nonstd::basic_string<char> str; // expected-error{{no template named 'basic_string' in namespace 'nonstd'; did you mean simply 'basic_string'?}}
-  nonstd::wide_string<char> str2; // expected-error{{no template named 'wide_string' in namespace 'nonstd'; did you mean 'another::wide_string'?}}
-  return wibble::narf; // expected-error{{no member named 'narf' in namespace 'otherstd'; did you mean 'nonstd::narf'?}}
-}
-
-namespace check_bool {
-  void f() {
-    Bool b; // expected-error{{use of undeclared identifier 'Bool'; did you mean 'bool'?}}
-  }
-}
-
-namespace outr {
-}
-namespace outer {
-  namespace inner { // expected-note{{'outer::inner' declared here}} \
-                    // expected-note{{namespace 'outer::inner' defined here}} \
-                    // expected-note{{'inner' declared here}}
-    int i;
-  }
-}
-
-using namespace outr::inner; // expected-error{{no namespace named 'inner' in namespace 'outr'; did you mean 'outer::inner'?}}
-
-void func() {
-  outr::inner::i = 3; // expected-error{{no member named 'inner' in namespace 'outr'; did you mean 'outer::inner'?}}
-  outer::innr::i = 4; // expected-error{{no member named 'innr' in namespace 'outer'; did you mean 'inner'?}}
-}
-
-struct base {
-};
-struct derived : base {
-  int i;
-};
-
-void func2() {
-  derived d;
-  // FIXME: we should offer a fix here. We do if the 'i' is misspelled, but we don't do name qualification changes
-  //        to replace base::i with derived::i as we would for other qualified name misspellings.
-  // d.base::i = 3;
-}
-
-class A {
-  void bar(int);
-};
-void bar(int, int);  // expected-note{{'::bar' declared here}}
-void A::bar(int x) {
-  bar(x, 5);  // expected-error{{too many arguments to function call, expected 1, have 2; did you mean '::bar'?}}
-}
diff --git a/external/llvm-project/clang/test/Format/multiple-inputs-error.cpp b/external/llvm-project/clang/test/Format/multiple-inputs-error.cpp
index 1aa9c9f3e2fa..7cb835d39f23 100644
--- a/external/llvm-project/clang/test/Format/multiple-inputs-error.cpp
+++ b/external/llvm-project/clang/test/Format/multiple-inputs-error.cpp
@@ -1,6 +1,6 @@
 // RUN: cp %s %t-1.cpp
 // RUN: cp %s %t-2.cpp
-// RUN: not clang-format 2>&1 >/dev/null -offset=1 -length=0 %t-1.cpp %t-2.cpp |FileCheck %s
+// RUN: not clang-format 2>&1 >/dev/null -offset=1 -length=1 %t-1.cpp %t-2.cpp |FileCheck %s
 // RUN: not clang-format 2>&1 >/dev/null -lines=1:1 %t-1.cpp %t-2.cpp |FileCheck %s -check-prefix=CHECK-LINE
 // CHECK: error: -offset, -length and -lines can only be used for single file.
 // CHECK-LINE: error: -offset, -length and -lines can only be used for single file.
diff --git a/external/llvm-project/clang/test/Format/ranges.cpp b/external/llvm-project/clang/test/Format/ranges.cpp
index 66b984e037b3..f42492e43f84 100644
--- a/external/llvm-project/clang/test/Format/ranges.cpp
+++ b/external/llvm-project/clang/test/Format/ranges.cpp
@@ -1,5 +1,5 @@
 // RUN: grep -Ev "// *[A-Z-]+:" %s \
-// RUN:   | clang-format -style=LLVM -offset=2 -length=0 -offset=28 -length=0 \
+// RUN:   | clang-format -style=LLVM -offset=2 -length=1 -offset=28 -length=1 -offset=35 -length=8 \
 // RUN:   | FileCheck -strict-whitespace %s
 // CHECK: {{^int\ \*i;$}}
 int*i;
@@ -9,3 +9,12 @@ int  *  i;
 
 // CHECK: {{^int\ \*i;$}}
 int   *   i;
+
+// CHECK: int I;
+// CHECK-NEXT: int J ;
+int I ;
+int J ;
+
+// RUN: not clang-format -length=0 %s 2>&1 \
+// RUN:   | FileCheck -strict-whitespace -check-prefix=CHECK0 %s
+// CHECK0: error: length should be at least 1
diff --git a/external/llvm-project/clang/test/Frontend/aarch64-print-enabled-extensions-cc1.c b/external/llvm-project/clang/test/Frontend/aarch64-print-enabled-extensions-cc1.c
new file mode 100644
index 000000000000..d761e12e8392
--- /dev/null
+++ b/external/llvm-project/clang/test/Frontend/aarch64-print-enabled-extensions-cc1.c
@@ -0,0 +1,139 @@
+// Test how -cc1 -target-feature interacts with -print-enabled-extensions.
+// The current behaviour does not look correct, since dependent features are
+// removed from the printed list when one of their dependencies are disabled,
+// but they are actually still enabled during compilation, and then actually
+// disabled for parsing assembly.
+
+// REQUIRES: aarch64-registered-target
+
+// Behaviour with two positive features.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +neon -target-feature +sve \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=POS_ONLY
+
+// Negative -target-feature disables the extension but keeps any dependencies of it (FEAT_FP16).
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +neon -target-feature +sve -target-feature -sve \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=POS_NEG
+
+// Disabling then re-enabling a feature is the same as never disabling it.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +neon -target-feature -sve -target-feature +sve \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=POS_ONLY
+
+// Disabling then re-enabling a feature is the same as never disabling it.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +neon -target-feature +sve -target-feature -sve -target-feature +sve \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=POS_ONLY
+
+// Only disabling it is the same as never having enabled it.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +neon \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=NEG_ONLY
+
+// Only disabling it is the same as never having enabled it.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +neon -target-feature -sve \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=NEG_ONLY
+
+// Disabling a dependency (after enabling the dependent) appears to disable the dependent feature.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +sve2 -target-feature -sve \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=DISABLE_DEP
+
+// Disabling a dependency before enabling the dependent appears to have no effect.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature -sve -target-feature +sve2 \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=DISABLE_DEP2
+
+// Disabling a dependency before enabling the dependent appears to have no effect.
+// RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -print-enabled-extensions \
+// RUN:     -target-feature +sve2 \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=DISABLE_DEP2
+
+// Driver --print-enabled-extensions indicates that negative -target-features disable dependent features.
+// RUN: %clang --target=aarch64 -march=armv8-a+sve2 --print-enabled-extensions \
+// RUN:     -Xclang -target-feature -Xclang -sve \
+// RUN:     | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s --check-prefix=DISABLE_VIA_XCLANG
+
+// However, sve2 is actually enabled in clang but disabled for MC.
+// RUN: %clang --target=aarch64 -march=armv8-a+sve2 -c %s -o %t \
+// RUN:     -Xclang -target-feature -Xclang -sve \
+// RUN:     -Xclang -verify -Xclang -verify-ignore-unexpected=note
+
+
+// POS_ONLY: Extensions enabled for the given AArch64 target
+// POS_ONLY-EMPTY:
+// POS_ONLY-NEXT:     Architecture Feature(s)                                Description
+// POS_ONLY-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// POS_ONLY-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// POS_ONLY-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// POS_ONLY-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// POS_ONLY-NEXT:     FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
+// POS_ONLY-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+
+// POS_NEG: Extensions enabled for the given AArch64 target
+// POS_NEG-EMPTY:
+// POS_NEG-NEXT:     Architecture Feature(s)                                Description
+// POS_NEG-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// POS_NEG-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// POS_NEG-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// POS_NEG-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// POS_NEG-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+
+// NEG_POS: Extensions enabled for the given AArch64 target
+// NEG_POS-EMPTY:
+// NEG_POS-NEXT:     Architecture Feature(s)                                Description
+// NEG_POS-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// NEG_POS-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// NEG_POS-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// NEG_POS-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// NEG_POS-NEXT:     FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
+// NEG_POS-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+
+// NEG_ONLY: Extensions enabled for the given AArch64 target
+// NEG_ONLY-EMPTY:
+// NEG_ONLY-NEXT:     Architecture Feature(s)                                Description
+// NEG_ONLY-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// NEG_ONLY-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// NEG_ONLY-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// NEG_ONLY-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+
+// DISABLE_DEP: Extensions enabled for the given AArch64 target
+// DISABLE_DEP-EMPTY: 
+// DISABLE_DEP-NEXT:     Architecture Feature(s)                                Description
+// DISABLE_DEP-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// DISABLE_DEP-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// DISABLE_DEP-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// DISABLE_DEP-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// DISABLE_DEP-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+
+// DISABLE_DEP2: Extensions enabled for the given AArch64 target
+// DISABLE_DEP2-EMPTY: 
+// DISABLE_DEP2-NEXT:     Architecture Feature(s)                                Description
+// DISABLE_DEP2-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// DISABLE_DEP2-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// DISABLE_DEP2-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// DISABLE_DEP2-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// DISABLE_DEP2-NEXT:     FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
+// DISABLE_DEP2-NEXT:     FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
+// DISABLE_DEP2-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+
+// DISABLE_VIA_XCLANG: Extensions enabled for the given AArch64 target
+// DISABLE_VIA_XCLANG-EMPTY: 
+// DISABLE_VIA_XCLANG-NEXT:     Architecture Feature(s)                                Description
+// DISABLE_VIA_XCLANG-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// DISABLE_VIA_XCLANG-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// DISABLE_VIA_XCLANG-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// DISABLE_VIA_XCLANG-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// DISABLE_VIA_XCLANG-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+
+#if __ARM_FEATURE_SVE2
+#warning "SVE2 is enabled"
+// expected-warning@-1 {{SVE2 is enabled}}
+#endif
+
+void fn_that_requires_sve2() {
+    __asm__("ldnt1sh z0.s, p0/z, [z1.s]");
+    // expected-error@-1 {{instruction requires: sve2}}
+}
diff --git a/external/llvm-project/clang/test/Frontend/ast-main.c b/external/llvm-project/clang/test/Frontend/ast-main.c
index cdc74219f73a..6a64497f4109 100644
--- a/external/llvm-project/clang/test/Frontend/ast-main.c
+++ b/external/llvm-project/clang/test/Frontend/ast-main.c
@@ -1,6 +1,6 @@
-// RUN: env SDKROOT="/" %clang -emit-llvm -S -o %t1.ll -x c - < %s
+// RUN: env SDKROOT="/" %clang -emit-llvm -S -o - -x c - < %s | grep -v DIFile > %t1.ll
 // RUN: env SDKROOT="/" %clang -emit-ast -o %t.ast %s
-// RUN: env SDKROOT="/" %clang -emit-llvm -S -o %t2.ll -x ast - < %t.ast
+// RUN: env SDKROOT="/" %clang -emit-llvm -S -o - -x ast - < %t.ast | grep -v DIFile > %t2.ll
 // RUN: diff %t1.ll %t2.ll
 
 int main(void) {
diff --git a/external/llvm-project/clang/test/Frontend/ast-main.cpp b/external/llvm-project/clang/test/Frontend/ast-main.cpp
index fe47ce435f06..fc09e6437f93 100644
--- a/external/llvm-project/clang/test/Frontend/ast-main.cpp
+++ b/external/llvm-project/clang/test/Frontend/ast-main.cpp
@@ -1,6 +1,6 @@
-// RUN: env SDKROOT="/" %clang -Wno-error=return-type -emit-llvm -S -o %t1.ll -x c++ - < %s
+// RUN: env SDKROOT="/" %clang -Wno-error=return-type -emit-llvm -S -o - -x c++ - < %s | grep -v DIFile > %t1.ll
 // RUN: env SDKROOT="/" %clang -Wno-error=return-type -fno-delayed-template-parsing -emit-ast -o %t.ast %s
-// RUN: env SDKROOT="/" %clang -Wno-error=return-type -emit-llvm -S -o %t2.ll -x ast - < %t.ast
+// RUN: env SDKROOT="/" %clang -Wno-error=return-type -emit-llvm -S -o - -x ast - < %t.ast | grep -v DIFile > %t2.ll
 // RUN: diff %t1.ll %t2.ll
 
 // http://llvm.org/bugs/show_bug.cgi?id=15377
diff --git a/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute-n3037.c b/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute-n3037.c
new file mode 100644
index 000000000000..ae5c14096f83
--- /dev/null
+++ b/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute-n3037.c
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -Wall -Wno-unused -Wno-uninitialized -verify -std=c17 %s
+// RUN: %clang_cc1 -Wall -Wno-unused -Wno-uninitialized -verify -std=c23 %s
+
+#define CFI_UNCHECKED_CALLEE __attribute__((cfi_unchecked_callee))
+
+#if __STDC_VERSION__ >= 202311L
+// expected-no-diagnostics
+#endif
+
+#if __STDC_VERSION__ < 202311L
+// expected-note@+2 2 {{previous definition is here}}
+#endif
+struct field_attr_test {
+  void (CFI_UNCHECKED_CALLEE *func)(void);
+};
+
+#if __STDC_VERSION__ < 202311L
+// expected-error@+2{{redefinition of 'field_attr_test'}}
+#endif
+struct field_attr_test {
+  void (CFI_UNCHECKED_CALLEE *func)(void);
+};
+
+typedef void (CFI_UNCHECKED_CALLEE func_t)(void);
+
+#if __STDC_VERSION__ < 202311L
+// expected-error@+2{{redefinition of 'field_attr_test'}}
+#endif
+struct field_attr_test {
+  func_t *func;
+};
diff --git a/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute.c b/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute.c
new file mode 100644
index 000000000000..889c106c3ccf
--- /dev/null
+++ b/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute.c
@@ -0,0 +1,71 @@
+// RUN: %clang_cc1 -Wall -Wno-unused -Wno-uninitialized -verify %s
+
+#define CFI_UNCHECKED_CALLEE __attribute__((cfi_unchecked_callee))
+
+void unchecked(void) CFI_UNCHECKED_CALLEE {}
+void checked(void) {}
+
+void (*checked_ptr)(void) = unchecked;  // expected-warning{{implicit conversion from 'void (void) __attribute__((cfi_unchecked_callee))' to 'void (*)(void)' discards 'cfi_unchecked_callee' attribute}}
+void (CFI_UNCHECKED_CALLEE *unchecked_ptr)(void) = unchecked;
+void (CFI_UNCHECKED_CALLEE *from_normal)(void) = checked;
+void (CFI_UNCHECKED_CALLEE *c_no_function_decay)(void) = &unchecked;
+
+typedef void (CFI_UNCHECKED_CALLEE unchecked_func_t)(void);
+typedef void (checked_func_t)(void);
+typedef void (CFI_UNCHECKED_CALLEE *cfi_unchecked_func_ptr_t)(void);
+typedef void (*checked_func_ptr_t)(void);
+checked_func_t *cfi_func = unchecked;  // expected-warning{{implicit conversion from 'void (void) __attribute__((cfi_unchecked_callee))' to 'void (*)(void)' discards 'cfi_unchecked_callee' attribute}}
+unchecked_func_t *unchecked_func = unchecked;
+
+void CFI_UNCHECKED_CALLEE after_ret_type(void);
+CFI_UNCHECKED_CALLEE void before_ret_type(void);
+void (* CFI_UNCHECKED_CALLEE after_name)(void);
+
+void UsageOnImproperTypes() {
+  int CFI_UNCHECKED_CALLEE i;  // expected-warning{{'cfi_unchecked_callee' only applies to function types; type here is 'int'}}
+
+  /// The attribute must be used only on functions with prototypes. The omission of `void` means it is not prototyped.
+  void (CFI_UNCHECKED_CALLEE *noproto)(void);  // expecteed-warning{{'cfi_unchecked_callee' attribute only applies to non-K&R-style functions}}
+}
+
+/// Explicit casts suppress the warning.
+void CheckCasts() {
+  void (*should_warn)(void) = unchecked;  // expected-warning{{implicit conversion from 'void (void) __attribute__((cfi_unchecked_callee))' to 'void (*)(void)' discards 'cfi_unchecked_callee' attribute}}
+
+  void (*no_warn_c_style_cast)(void) = (void (*)(void))unchecked;
+
+  struct B {} CFI_UNCHECKED_CALLEE b;  // expected-warning{{'cfi_unchecked_callee' attribute only applies to functions and methods}}
+  struct CFI_UNCHECKED_CALLEE C {} c;  // expected-warning{{'cfi_unchecked_callee' attribute only applies to functions and methods}}
+  CFI_UNCHECKED_CALLEE struct D {} d;  // expected-warning{{'cfi_unchecked_callee' only applies to function types; type here is 'struct D'}}
+
+  void *ptr2 = (void *)unchecked;
+}
+
+int checked_arg_func(checked_func_t *checked_func);
+
+void CheckDifferentConstructions() {
+  void (CFI_UNCHECKED_CALLEE *arr[10])(void);
+  void (*cfi_elem)(void) = arr[1];  // expected-warning{{implicit conversion from 'void (*)(void) __attribute__((cfi_unchecked_callee))' to 'void (*)(void)' discards 'cfi_unchecked_callee' attribute}}
+  void (CFI_UNCHECKED_CALLEE *cfi_unchecked_elem)(void) = arr[1];
+
+  int invoke = checked_arg_func(unchecked);  // expected-warning{{implicit conversion from 'void (void) __attribute__((cfi_unchecked_callee))' to 'void (*)(void)' discards 'cfi_unchecked_callee' attribute}}
+}
+
+checked_func_t *returning_checked_func() {
+  return unchecked;  // expected-warning{{implicit conversion from 'void (void) __attribute__((cfi_unchecked_callee))' to 'void (*)(void)' discards 'cfi_unchecked_callee' attribute}}
+}
+
+void no_args(void) __attribute__((cfi_unchecked_callee(10)));  // expected-error{{'cfi_unchecked_callee' attribute takes no arguments}}
+
+void Comparisons() {
+  /// Let's be able to compare checked and unchecked pointers without warnings.
+  unchecked == checked_ptr;
+  checked_ptr == unchecked;
+  unchecked == unchecked_ptr;
+  unchecked != checked_ptr;
+  checked_ptr != unchecked;
+  unchecked != unchecked_ptr;
+
+  (void (*)(void))unchecked == checked_ptr;
+  checked_ptr == (void (*)(void))unchecked;
+}
diff --git a/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute.cpp b/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute.cpp
new file mode 100644
index 000000000000..f2c4e9e2f889
--- /dev/null
+++ b/external/llvm-project/clang/test/Frontend/cfi-unchecked-callee-attribute.cpp
@@ -0,0 +1,235 @@
+// RUN: %clang_cc1 -Wall -Wno-unused -Wno-uninitialized -std=c++2b -verify %s
+
+#define CFI_UNCHECKED_CALLEE __attribute__((cfi_unchecked_callee))
+
+void unchecked(void) CFI_UNCHECKED_CALLEE {}
+void checked(void) {}
+
+void (*checked_ptr)(void) = unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+void (CFI_UNCHECKED_CALLEE *unchecked_ptr)(void) = unchecked;
+void (CFI_UNCHECKED_CALLEE *from_normal)(void) = checked;
+void (CFI_UNCHECKED_CALLEE *c_no_function_decay)(void) = &unchecked;
+void (CFI_UNCHECKED_CALLEE *arr[10])(void);
+void (*cfi_elem)(void) = arr[1];  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+void (CFI_UNCHECKED_CALLEE *cfi_unchecked_elem)(void) = arr[1];
+void (CFI_UNCHECKED_CALLEE &ref)(void) = unchecked;
+void (CFI_UNCHECKED_CALLEE &ref2)(void) = *unchecked;
+void (&ref_cfi_checked)(void) = unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void ()' discards 'cfi_unchecked_callee' attribute}}
+void (&ref_cfi_checked2)(void) = *unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void ()' discards 'cfi_unchecked_callee' attribute}}
+
+void (CFI_UNCHECKED_CALLEE *unchecked_from_deref)(void) = &*unchecked;
+void (*checked_from_deref)(void) = &*unchecked;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+
+typedef void (CFI_UNCHECKED_CALLEE unchecked_func_t)(void);
+typedef void (checked_func_t)(void);
+typedef void (CFI_UNCHECKED_CALLEE *unchecked_func_ptr_t)(void);
+typedef void (*checked_func_ptr_t)(void);
+checked_func_t *checked_func = unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+unchecked_func_t *unchecked_func = unchecked;
+
+void CFI_UNCHECKED_CALLEE before_func(void);
+CFI_UNCHECKED_CALLEE void before_return_type(void);
+void (* CFI_UNCHECKED_CALLEE after_name)(void);
+
+void UsageOnImproperTypes() {
+  int CFI_UNCHECKED_CALLEE i;  // expected-warning{{'cfi_unchecked_callee' only applies to function types; type here is 'int'}}
+}
+
+/// Explicit casts suppress the warning.
+void CheckCasts() {
+  void (*should_warn)(void) = unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+
+  void (*no_warn_c_style_cast)(void) = (void (*)(void))unchecked;
+  void (*no_warn_static_cast)(void) = static_cast<void (*)(void)>(unchecked);
+  void (*no_warn_reinterpret_cast)(void) = reinterpret_cast<void (*)(void)>(unchecked);
+  unsigned long long ull = (unsigned long long)unchecked;
+
+  struct A {};
+  void (CFI_UNCHECKED_CALLEE A::*cfi_unchecked_member_ptr)(void);
+  void (A::*member_ptr)(void) = cfi_unchecked_member_ptr;  // expected-warning{{implicit conversion from 'void (A::*)() __attribute__((cfi_unchecked_callee))' to 'void (A::*)()' discards 'cfi_unchecked_callee' attribute}}
+
+  struct B {} CFI_UNCHECKED_CALLEE b;  // expected-warning{{'cfi_unchecked_callee' attribute only applies to functions and methods}}
+  struct CFI_UNCHECKED_CALLEE C {} c;  // expected-warning{{'cfi_unchecked_callee' attribute only applies to functions and methods}}
+  CFI_UNCHECKED_CALLEE struct D {} d;  // expected-warning{{'cfi_unchecked_callee' only applies to function types; type here is 'struct D'}}
+
+  void *ptr2 = (void *)unchecked;
+}
+
+void CheckDifferentConstructions() {
+  checked_func_t *checked_func(unchecked_func);  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+  new (checked_func_t *)(unchecked_func);  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+  struct S {
+    checked_func_t *checked_func;
+
+    S(unchecked_func_t *unchecked_func) : checked_func(unchecked_func) {}  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+  };
+
+  checked_func_t *checked_func2{unchecked_func};  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+  checked_ptr = checked_func_ptr_t(unchecked);
+
+  auto checked_auto = checked;
+  auto unchecked_auto = unchecked;
+  unchecked_ptr = checked_auto;
+  unchecked_ptr = unchecked_auto;
+  checked_ptr = checked_auto;
+  checked_ptr = unchecked_auto;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+}
+
+checked_func_t *returning_checked_func() {
+  return unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+}
+
+int checked_arg_func(checked_func_t *checked_func);
+int invoke = checked_arg_func(unchecked);  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+
+template <typename T>
+struct S {
+  S(T *ptr) {}
+};
+S<unchecked_func_t> s(checked);
+S<unchecked_func_t> s2(unchecked);
+S<checked_func_t> s3(checked);
+S<checked_func_t> s4(unchecked);  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+S s5(checked);
+S s6(unchecked);
+
+template <typename T, typename U>
+struct is_same {
+  static constexpr bool value = false;
+};
+template <typename T>
+struct is_same<T,T> {
+  static constexpr bool value = true;
+};
+
+template <typename T>
+struct ExpectingCFIUncheckedCallee {
+  static_assert(is_same<T, unchecked_func_t>::value);
+  ExpectingCFIUncheckedCallee(T *) {}
+  ExpectingCFIUncheckedCallee() = default;
+};
+ExpectingCFIUncheckedCallee<unchecked_func_t> expecting;
+ExpectingCFIUncheckedCallee expecting2(unchecked);
+
+void no_args() __attribute__((cfi_unchecked_callee(10)));  // expected-error{{'cfi_unchecked_callee' attribute takes no arguments}}
+
+void bracket_cfi_unchecked(void) [[clang::cfi_unchecked_callee]] {}
+
+void BracketNotation() {
+  checked_ptr = bracket_cfi_unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+}
+
+void Comparisons() {
+  /// Let's be able to compare checked and unchecked pointers without warnings.
+  unchecked == checked_ptr;
+  checked_ptr == unchecked;
+  unchecked == unchecked_ptr;
+  unchecked != checked_ptr;
+  checked_ptr != unchecked;
+  unchecked != unchecked_ptr;
+
+  (void (*)(void))unchecked == checked_ptr;
+  checked_ptr == (void (*)(void))unchecked;
+
+  struct S {
+    typedef void CB() CFI_UNCHECKED_CALLEE;
+    constexpr bool operator==(const S &other) const {
+      return cb == other.cb;
+    }
+    CB *cb;
+  };
+}
+
+/// Type aliasing
+typedef void (BaseType)(void);
+using WithoutAttr = BaseType;
+using WithAttr = __attribute__((cfi_unchecked_callee)) BaseType;
+
+WithoutAttr *checked_func_alias = unchecked;  // expected-warning{{implicit conversion from 'void () __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+WithAttr *unchecked_func_allias = unchecked;
+WithoutAttr *checked_func_alias2 = checked;
+WithAttr *unchecked_func_alias2 = checked;
+
+using MyType = WithAttr;  // expected-note{{previous definition is here}}
+using MyType = WithoutAttr;  // expected-error{{type alias redefinition with different types ('WithoutAttr' (aka 'void ()') vs 'WithAttr' (aka 'void () __attribute__((cfi_unchecked_callee))'))}}
+
+void MemberFunctionPointer() {
+  struct A {
+    void unchecked() CFI_UNCHECKED_CALLEE {}
+    virtual void unchecked_virtual() CFI_UNCHECKED_CALLEE {}
+    static void unchecked_static() CFI_UNCHECKED_CALLEE {}
+    void unchecked_explicit_this(this A&) CFI_UNCHECKED_CALLEE {}
+    int operator+=(int i) CFI_UNCHECKED_CALLEE { return i; }
+
+    void checked() {}
+    virtual void checked_virtual() {}
+    static void checked_static() {}
+    void checked_explicit_this(this A&) {}
+    int operator-=(int i) { return i; }
+  };
+
+  void (CFI_UNCHECKED_CALLEE A::*unchecked)() = &A::unchecked;
+  unchecked = &A::unchecked_virtual;
+  void (CFI_UNCHECKED_CALLEE *unchecked_explicit_this)(A&) = &A::unchecked_explicit_this;
+  void (CFI_UNCHECKED_CALLEE *unchecked_static)() = &A::unchecked_static;
+  int (CFI_UNCHECKED_CALLEE A::*unchecked_overloaded)(int) = &A::operator+=;
+
+  void (A::*checked)() = &A::unchecked;  // expected-warning{{implicit conversion from 'void (A::*)() __attribute__((cfi_unchecked_callee))' to 'void (A::*)()' discards 'cfi_unchecked_callee' attribute}}
+  checked = &A::unchecked_virtual;  // expected-warning{{implicit conversion from 'void (A::*)() __attribute__((cfi_unchecked_callee))' to 'void (A::*)()' discards 'cfi_unchecked_callee' attribute}}
+  void (*checked_explicit_this)(A&) = &A::unchecked_explicit_this;  // expected-warning{{implicit conversion from 'void (*)(A &) __attribute__((cfi_unchecked_callee))' to 'void (*)(A &)' discards 'cfi_unchecked_callee' attribute}}
+  void (*checked_static)() = &A::unchecked_static;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+  int (A::*checked_overloaded)(int) = &A::operator+=;  // expected-warning{{implicit conversion from 'int (A::*)(int) __attribute__((cfi_unchecked_callee))' to 'int (A::*)(int)' discards 'cfi_unchecked_callee' attribute}}
+
+  unchecked = &A::checked;
+  unchecked = &A::checked_virtual;
+  unchecked_explicit_this = &A::checked_explicit_this;
+  unchecked_static = &A::checked_static;
+  unchecked_overloaded = &A::operator-=;
+
+  checked = &A::checked;
+  checked = &A::checked_virtual;
+  checked_explicit_this = &A::checked_explicit_this;
+  checked_static = &A::checked_static;
+  checked_overloaded = &A::operator-=;
+
+  typedef void (CFI_UNCHECKED_CALLEE A::*WithAttr)();
+  typedef void (CFI_UNCHECKED_CALLEE A::*WithoutAttr)();
+  using WithoutAttr = decltype(unchecked);
+}
+
+void lambdas() {
+  auto unchecked_lambda = [](void) CFI_UNCHECKED_CALLEE -> void {};
+  auto checked_lambda = [](void) -> void {};
+  void (CFI_UNCHECKED_CALLEE *unchecked_func)(void) = unchecked_lambda;
+  unchecked_func = checked_lambda;
+  void (*checked_func)(void) = unchecked_lambda;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+  checked_func = checked_lambda;
+
+  auto capture_by_value = [unchecked_lambda, checked_lambda]() {
+    void (CFI_UNCHECKED_CALLEE *unchecked_func)(void) = unchecked_lambda;
+    unchecked_func = checked_lambda;
+    void (*checked_func)(void) = unchecked_lambda;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+    checked_func = checked_lambda;
+  };
+
+  auto capture_by_ref = [&unchecked_lambda, &checked_lambda]() {
+    void (CFI_UNCHECKED_CALLEE *unchecked_func)(void) = unchecked_lambda;
+    unchecked_func = checked_lambda;
+    void (*checked_func)(void) = unchecked_lambda;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+    checked_func = checked_lambda;
+  };
+
+  auto capture_all_by_value = [=]() {
+    void (CFI_UNCHECKED_CALLEE *unchecked_func)(void) = unchecked_lambda;
+    unchecked_func = checked_lambda;
+    void (*checked_func)(void) = unchecked_lambda;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+    checked_func = checked_lambda;
+  };
+
+  auto capture_all_by_ref = [&]() {
+    void (CFI_UNCHECKED_CALLEE *unchecked_func)(void) = unchecked_lambda;
+    unchecked_func = checked_lambda;
+    void (*checked_func)(void) = unchecked_lambda;  // expected-warning{{implicit conversion from 'void (*)() __attribute__((cfi_unchecked_callee))' to 'void (*)()' discards 'cfi_unchecked_callee' attribute}}
+    checked_func = checked_lambda;
+  };
+}
diff --git a/external/llvm-project/clang/test/Headers/Inputs/include/stdint.h b/external/llvm-project/clang/test/Headers/Inputs/include/stdint.h
index 67b27b8dfc7b..eb09272b40c3 100644
--- a/external/llvm-project/clang/test/Headers/Inputs/include/stdint.h
+++ b/external/llvm-project/clang/test/Headers/Inputs/include/stdint.h
@@ -1,11 +1,23 @@
 #ifndef STDINT_H
 #define STDINT_H
 
+#ifdef __INT8_TYPE__
+typedef __INT8_TYPE__ int8_t;
+#endif
+typedef unsigned char uint8_t;
+
+#ifdef __INT16_TYPE__
+typedef __INT16_TYPE__ int16_t;
+typedef unsigned __INT16_TYPE__ uint16_t;
+#endif
+
 #ifdef __INT32_TYPE__
+typedef __INT32_TYPE__ int32_t;
 typedef unsigned __INT32_TYPE__ uint32_t;
 #endif
 
 #ifdef __INT64_TYPE__
+typedef __INT64_TYPE__ int64_t;
 typedef unsigned __INT64_TYPE__ uint64_t;
 #endif
 
diff --git a/external/llvm-project/clang/test/Headers/crash-instantiated-in-scope-cxx-modules6.cpp b/external/llvm-project/clang/test/Headers/crash-instantiated-in-scope-cxx-modules6.cpp
new file mode 100644
index 000000000000..077f69ea6598
--- /dev/null
+++ b/external/llvm-project/clang/test/Headers/crash-instantiated-in-scope-cxx-modules6.cpp
@@ -0,0 +1,149 @@
+// RUN: rm -fR %t
+// RUN: split-file %s %t
+// RUN: cd %t
+// RUN: %clang_cc1 -std=c++20 -xc++ -emit-module -fmodules foo.cppmap -fmodule-name=foo -fmodule-name=foo -o foo.pcm
+// RUN: %clang_cc1 -std=c++20 -xc++ -emit-module -fmodules bar.cppmap -fmodule-name=bar -o bar.pcm
+// RUN: %clang_cc1 -std=c++20 -xc++ -emit-module -fmodules experiment_context.cppmap -fmodule-name=experiment_context -fmodule-file=foo.pcm -fmodule-file=bar.pcm -o experiment_context.pcm
+// RUN: %clang_cc1 -verify -std=c++20 -xc++ -fmodule-file=experiment_context.pcm experiment_context_test.cc -o experiment_context_test.o
+
+// https://github.com/llvm/llvm-project/issues/141582
+//--- bar.cppmap
+module "bar" {
+  export *
+  header "co.h"
+}
+
+//--- foo.cppmap
+module "foo" {
+  export *
+  header "co.h"
+}
+
+//--- experiment_context.cppmap
+module "experiment_context" {
+  export *
+  header "lazy.h"
+
+  use "foo"
+  use "bar"
+}
+
+//--- experiment_context_test.cc
+// expected-no-diagnostics
+#include "lazy.h"
+
+namespace c9 {
+
+template <typename T>
+void WaitForCoroutine() {
+  MakeCo<T>([]() -> Co<void> {
+    co_return;
+  });
+}
+
+void test() {
+  c9::WaitForCoroutine<void>();
+}
+}
+
+//--- lazy.h
+#pragma once
+
+#include "co.h"
+
+namespace c9 {
+template <typename T, typename F>
+Co<T> MakeCo(F f)
+{
+  co_return co_await f();
+}
+}
+
+inline c9::Co<void> DoNothing() { co_return; }
+
+
+//--- co.h
+#pragma  once
+namespace std {
+
+template <class _Ret, class... _Args>
+struct coroutine_traits {};
+
+template <typename Ret, typename... Args>
+  requires requires { typename Ret::promise_type; }
+struct coroutine_traits<Ret, Args...> {
+  using promise_type = typename Ret::promise_type;
+};
+
+template <typename Promise = void>
+struct coroutine_handle;
+
+template <>
+struct coroutine_handle<void> {};
+
+template <typename Promise = void>
+struct coroutine_handle : coroutine_handle<> {
+  static coroutine_handle from_address(void *addr);
+};
+
+struct suspend_always {
+  bool await_ready() noexcept;
+  void await_suspend(coroutine_handle<>) noexcept;
+  void await_resume() noexcept;
+};
+
+}  // namespace std
+
+namespace c9 {
+
+template <typename T>
+class Co;
+
+namespace internal {
+
+template <typename T>
+class CoroutinePromise {
+ public:
+  template <typename... Args>
+  explicit CoroutinePromise(Args&&... args) {
+    // Ensure that the 'dummy_color' VarDecl referenced by the inner DeclRefExpr
+    // is the same declaration as the one outside the lambda.
+    // This is guaranteed because both CoroutinePromise and the lambda's call operator
+    // (CXXMethodDecl) are loaded from the same module.
+    const int dummy_color = 1;
+    [&]{ (void)dummy_color; }();
+  }
+
+  ~CoroutinePromise();
+  void return_void();
+  auto get_return_object() {
+    return Co<T>();
+  }
+  void unhandled_exception();
+  std::suspend_always  initial_suspend();
+
+  struct result_t {
+    ~result_t();
+    bool await_ready() const noexcept;
+    void await_suspend(std::coroutine_handle<void>) noexcept;
+    void await_resume() const noexcept;
+  };
+
+  template <typename U>
+  result_t await_transform(Co<U> co);
+
+  std::suspend_always final_suspend() noexcept;
+};
+}  // namespace internal
+
+template <typename T>
+class Co {
+ public:
+  using promise_type = internal::CoroutinePromise<void>;
+};
+
+class CoIncomingModuleBase {
+ public:
+    Co<void> CoAfterFinish() { co_return; }
+};
+}  // namespace c9
diff --git a/external/llvm-project/clang/test/Headers/ms-intrin.cpp b/external/llvm-project/clang/test/Headers/ms-intrin.cpp
index d630883e79d6..985f8577c46b 100644
--- a/external/llvm-project/clang/test/Headers/ms-intrin.cpp
+++ b/external/llvm-project/clang/test/Headers/ms-intrin.cpp
@@ -36,6 +36,10 @@ typedef __SIZE_TYPE__ size_t;
 
 #include <intrin.h>
 
+#ifdef __ARM_ACLE
+#include <arm_acle.h>
+#endif
+
 // Use some C++ to make sure we closed the extern "C" brackets.
 template <typename T>
 void foo(T V) {}
diff --git a/external/llvm-project/clang/test/Index/complete-switch.c b/external/llvm-project/clang/test/Index/complete-switch.c
deleted file mode 100644
index 4a7885459554..000000000000
--- a/external/llvm-project/clang/test/Index/complete-switch.c
+++ /dev/null
@@ -1,10 +0,0 @@
-void f() {
-  auto foo = bar;
-  switch(foo) {
-    case x:
-      break;
-  }
-}
-
-// RUN: not %clang_cc1 -fsyntax-only -fno-recovery-ast -code-completion-at=%s:4:10 %s | FileCheck %s -allow-empty
-// CHECK-NOT: COMPLETION: foo
diff --git a/external/llvm-project/clang/test/Index/fix-its.c b/external/llvm-project/clang/test/Index/fix-its.c
index 1e710c28afcc..8378fd9da9b4 100644
--- a/external/llvm-project/clang/test/Index/fix-its.c
+++ b/external/llvm-project/clang/test/Index/fix-its.c
@@ -1,27 +1,12 @@
-// RUN: c-index-test -test-load-source all -fspell-checking %s 2> %t  
+// RUN: c-index-test -test-load-source all -fspell-checking %s 2> %t
 // RUN: FileCheck %s < %t
-struct X {
-  int wibble;
-};
-
 #define MACRO(X) X
 
-void f(struct X *x) {
-  // CHECK: error: no member named 'wobble' in 'struct X'; did you mean 'wibble'?
-  // CHECK: FIX-IT: Replace [13:12 - 13:18] with "wibble"
-  // CHECK: note: 'wibble' declared here
-  MACRO(x->wobble = 17);
-  // CHECK: error: no member named 'wabble' in 'struct X'; did you mean 'wibble'?
-  // CHECK: FIX-IT: Replace [17:6 - 17:12] with "wibble"
-  // CHECK: note: 'wibble' declared here
-  x->wabble = 17;
-}
-
 int printf(const char *restrict, ...);
 
 void f2() {
   unsigned long index;
   // CHECK: warning: format specifies type 'int' but the argument has type 'unsigned long'
-  // CHECK: FIX-IT: Replace [26:17 - 26:19] with "%lu"
+  // CHECK: FIX-IT: Replace [11:17 - 11:19] with "%lu"
   MACRO(printf("%d", index));
 }
diff --git a/external/llvm-project/clang/test/Interpreter/lambda.cpp b/external/llvm-project/clang/test/Interpreter/lambda.cpp
index df75274a050b..db8c9db9b344 100644
--- a/external/llvm-project/clang/test/Interpreter/lambda.cpp
+++ b/external/llvm-project/clang/test/Interpreter/lambda.cpp
@@ -1,7 +1,10 @@
 // REQUIRES: host-supports-jit
 // UNSUPPORTED: system-aix
 // RUN: cat %s | clang-repl | FileCheck %s
-// RUN: cat %s | clang-repl -Xcc -O2 | FileCheck %s
+// At -O2, somehow "x = 42" appears first when piped into FileCheck,
+// see https://github.com/llvm/llvm-project/issues/143547.
+// RUN: %if !system-windows %{ cat %s | clang-repl -Xcc -Xclang -Xcc -verify -Xcc -O2 | FileCheck %s %}
+
 extern "C" int printf(const char *, ...);
 
 auto l1 = []() { printf("ONE\n"); return 42; };
@@ -14,4 +17,14 @@ auto r2 = l2();
 auto r3 = l2();
 // CHECK: TWO
 
-%quit
+// Verify non-local lambda capture error is correctly reported
+int x = 42;
+
+// expected-error {{non-local lambda expression cannot have a capture-default}}
+auto capture = [&]() { return x * 2; };
+
+// Ensure interpreter continues and x is still valid
+printf("x = %d\n", x);
+// CHECK: x = 42
+
+%quit
\ No newline at end of file
diff --git a/external/llvm-project/clang/test/Lexer/raw-string-ext.c b/external/llvm-project/clang/test/Lexer/raw-string-ext.c
index de318b616df7..8ed96e5c19f0 100644
--- a/external/llvm-project/clang/test/Lexer/raw-string-ext.c
+++ b/external/llvm-project/clang/test/Lexer/raw-string-ext.c
@@ -27,13 +27,13 @@
 // no-warning@* {{ignoring '-fno-raw-string-literals'}}
 
 void f() {
-  (void) R"foo()foo"; // unsupported-error {{use of undeclared identifier 'R'}} cxx-unsupported-error {{expected ';' after expression}}
-  (void) LR"foo()foo"; // unsupported-error {{use of undeclared identifier 'LR'}} cxx-unsupported-error {{expected ';' after expression}}
+  (void) R"foo()foo"; // unsupported-error {{use of undeclared identifier 'R'}}
+  (void) LR"foo()foo"; // unsupported-error {{use of undeclared identifier 'LR'}}
 
 #ifdef UNICODE
-  (void) uR"foo()foo"; // unsupported-error {{use of undeclared identifier 'uR'}} cxx-unsupported-error {{expected ';' after expression}}
-  (void) u8R"foo()foo"; // unsupported-error {{use of undeclared identifier 'u8R'}} cxx-unsupported-error {{expected ';' after expression}}
-  (void) UR"foo()foo"; // unsupported-error {{use of undeclared identifier 'UR'}} cxx-unsupported-error {{expected ';' after expression}}
+  (void) uR"foo()foo"; // unsupported-error {{use of undeclared identifier 'uR'}}
+  (void) u8R"foo()foo"; // unsupported-error {{use of undeclared identifier 'u8R'}}
+  (void) UR"foo()foo"; // unsupported-error {{use of undeclared identifier 'UR'}}
 #endif
 }
 
diff --git a/external/llvm-project/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/external/llvm-project/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index bf64c388b043..41d00dae3f69 100644
--- a/external/llvm-project/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/external/llvm-project/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -109,7 +109,6 @@
 // CHECK-NEXT: NSConsumed (SubjectMatchRule_variable_is_parameter)
 // CHECK-NEXT: NSConsumesSelf (SubjectMatchRule_objc_method)
 // CHECK-NEXT: NSErrorDomain (SubjectMatchRule_enum)
-// CHECK-NEXT: NVPTXKernel (SubjectMatchRule_function)
 // CHECK-NEXT: Naked (SubjectMatchRule_function)
 // CHECK-NEXT: NoBuiltin (SubjectMatchRule_function)
 // CHECK-NEXT: NoCommon (SubjectMatchRule_variable)
diff --git a/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/avr.c b/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/avr.c
index 86ffbb683858..49d68bcc2edf 100644
--- a/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/avr.c
+++ b/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/avr.c
@@ -311,6 +311,9 @@
 // CHECK-SAME: {{^}}, attiny1624
 // CHECK-SAME: {{^}}, attiny1626
 // CHECK-SAME: {{^}}, attiny1627
+// CHECK-SAME: {{^}}, attiny3224
+// CHECK-SAME: {{^}}, attiny3226
+// CHECK-SAME: {{^}}, attiny3227
 // CHECK-SAME: {{^}}, atmega808
 // CHECK-SAME: {{^}}, atmega809
 // CHECK-SAME: {{^}}, atmega1608
@@ -319,4 +322,66 @@
 // CHECK-SAME: {{^}}, atmega3209
 // CHECK-SAME: {{^}}, atmega4808
 // CHECK-SAME: {{^}}, atmega4809
+// CHECK-SAME: {{^}}, avr64da28
+// CHECK-SAME: {{^}}, avr64da32
+// CHECK-SAME: {{^}}, avr64da48
+// CHECK-SAME: {{^}}, avr64da64
+// CHECK-SAME: {{^}}, avr64db28
+// CHECK-SAME: {{^}}, avr64db32
+// CHECK-SAME: {{^}}, avr64db48
+// CHECK-SAME: {{^}}, avr64db64
+// CHECK-SAME: {{^}}, avr64dd14
+// CHECK-SAME: {{^}}, avr64dd20
+// CHECK-SAME: {{^}}, avr64dd28
+// CHECK-SAME: {{^}}, avr64dd32
+// CHECK-SAME: {{^}}, avr64du28
+// CHECK-SAME: {{^}}, avr64du32
+// CHECK-SAME: {{^}}, avr64ea28
+// CHECK-SAME: {{^}}, avr64ea32
+// CHECK-SAME: {{^}}, avr64ea48
+// CHECK-SAME: {{^}}, avr64sd28
+// CHECK-SAME: {{^}}, avr64sd32
+// CHECK-SAME: {{^}}, avr64sd48
+// CHECK-SAME: {{^}}, avr16dd20
+// CHECK-SAME: {{^}}, avr16dd28
+// CHECK-SAME: {{^}}, avr16dd32
+// CHECK-SAME: {{^}}, avr16du14
+// CHECK-SAME: {{^}}, avr16du20
+// CHECK-SAME: {{^}}, avr16du28
+// CHECK-SAME: {{^}}, avr16du32
+// CHECK-SAME: {{^}}, avr32da28
+// CHECK-SAME: {{^}}, avr32da32
+// CHECK-SAME: {{^}}, avr32da48
+// CHECK-SAME: {{^}}, avr32db28
+// CHECK-SAME: {{^}}, avr32db32
+// CHECK-SAME: {{^}}, avr32db48
+// CHECK-SAME: {{^}}, avr32dd14
+// CHECK-SAME: {{^}}, avr32dd20
+// CHECK-SAME: {{^}}, avr32dd28
+// CHECK-SAME: {{^}}, avr32dd32
+// CHECK-SAME: {{^}}, avr32du14
+// CHECK-SAME: {{^}}, avr32du20
+// CHECK-SAME: {{^}}, avr32du28
+// CHECK-SAME: {{^}}, avr32du32
+// CHECK-SAME: {{^}}, avr16eb14
+// CHECK-SAME: {{^}}, avr16eb20
+// CHECK-SAME: {{^}}, avr16eb28
+// CHECK-SAME: {{^}}, avr16eb32
+// CHECK-SAME: {{^}}, avr16ea28
+// CHECK-SAME: {{^}}, avr16ea32
+// CHECK-SAME: {{^}}, avr16ea48
+// CHECK-SAME: {{^}}, avr32ea28
+// CHECK-SAME: {{^}}, avr32ea32
+// CHECK-SAME: {{^}}, avr32ea48
+// CHECK-SAME: {{^}}, avr32sd20
+// CHECK-SAME: {{^}}, avr32sd28
+// CHECK-SAME: {{^}}, avr32sd32
+// CHECK-SAME: {{^}}, avr128da28
+// CHECK-SAME: {{^}}, avr128da32
+// CHECK-SAME: {{^}}, avr128da48
+// CHECK-SAME: {{^}}, avr128da64
+// CHECK-SAME: {{^}}, avr128db28
+// CHECK-SAME: {{^}}, avr128db32
+// CHECK-SAME: {{^}}, avr128db48
+// CHECK-SAME: {{^}}, avr128db64
 // CHECK-SAME: {{$}}
diff --git a/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/riscv.c b/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/riscv.c
index c57bde8aa531..d736695b4883 100644
--- a/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/riscv.c
+++ b/external/llvm-project/clang/test/Misc/target-invalid-cpu-note/riscv.c
@@ -45,6 +45,7 @@
 // RISCV64-SAME: {{^}}, sifive-u54
 // RISCV64-SAME: {{^}}, sifive-u74
 // RISCV64-SAME: {{^}}, sifive-x280
+// RISCV64-SAME: {{^}}, sifive-x390
 // RISCV64-SAME: {{^}}, spacemit-x60
 // RISCV64-SAME: {{^}}, syntacore-scr3-rv64
 // RISCV64-SAME: {{^}}, syntacore-scr4-rv64
@@ -76,6 +77,7 @@
 // TUNE-RISCV32-SAME: {{^}}, syntacore-scr3-rv32
 // TUNE-RISCV32-SAME: {{^}}, syntacore-scr4-rv32
 // TUNE-RISCV32-SAME: {{^}}, syntacore-scr5-rv32
+// TUNE-RISCV32-SAME: {{^}}, andes-45-series
 // TUNE-RISCV32-SAME: {{^}}, generic
 // TUNE-RISCV32-SAME: {{^}}, generic-ooo
 // TUNE-RISCV32-SAME: {{^}}, rocket
@@ -103,6 +105,7 @@
 // TUNE-RISCV64-SAME: {{^}}, sifive-u54
 // TUNE-RISCV64-SAME: {{^}}, sifive-u74
 // TUNE-RISCV64-SAME: {{^}}, sifive-x280
+// TUNE-RISCV64-SAME: {{^}}, sifive-x390
 // TUNE-RISCV64-SAME: {{^}}, spacemit-x60
 // TUNE-RISCV64-SAME: {{^}}, syntacore-scr3-rv64
 // TUNE-RISCV64-SAME: {{^}}, syntacore-scr4-rv64
@@ -112,6 +115,7 @@
 // TUNE-RISCV64-SAME: {{^}}, veyron-v1
 // TUNE-RISCV64-SAME: {{^}}, xiangshan-kunminghu
 // TUNE-RISCV64-SAME: {{^}}, xiangshan-nanhu
+// TUNE-RISCV64-SAME: {{^}}, andes-45-series
 // TUNE-RISCV64-SAME: {{^}}, generic
 // TUNE-RISCV64-SAME: {{^}}, generic-ooo
 // TUNE-RISCV64-SAME: {{^}}, rocket
diff --git a/external/llvm-project/clang/test/Modules/check-for-sanitizer-feature.cpp b/external/llvm-project/clang/test/Modules/check-for-sanitizer-feature.cpp
index 861b571f0efa..1814da2d7333 100644
--- a/external/llvm-project/clang/test/Modules/check-for-sanitizer-feature.cpp
+++ b/external/llvm-project/clang/test/Modules/check-for-sanitizer-feature.cpp
@@ -43,7 +43,7 @@
 //
 // Import the PCH without ASan enabled (we expect an error).
 // RUN: not %clang_cc1 -x c -include-pch %t.asan_pch %s -verify 2>&1 | FileCheck %s --check-prefix=PCH_MISMATCH
-// PCH_MISMATCH: AST file '{{.*}}.asan_pch' was compiled with the target feature '-fsanitize=address' but the current translation unit is not
+// PCH_MISMATCH: precompiled file '{{.*}}.asan_pch' was compiled with the target feature '-fsanitize=address' but the current translation unit is not
 //
 // Emit a PCH with UBSan enabled.
 // RUN: %clang_cc1 -x c -fsanitize=null %S/Inputs/check-for-sanitizer-feature/check.h -emit-pch -o %t.ubsan_pch
diff --git a/external/llvm-project/clang/test/Modules/constexpr-initialization-failure.cpp b/external/llvm-project/clang/test/Modules/constexpr-initialization-failure.cpp
new file mode 100644
index 000000000000..8ff20f2fc8ac
--- /dev/null
+++ b/external/llvm-project/clang/test/Modules/constexpr-initialization-failure.cpp
@@ -0,0 +1,44 @@
+// RUN: rm -fR %t
+// RUN: split-file %s %t
+// RUN: cd %t
+// RUN: %clang_cc1 -verify -w -std=c++20 -fmodule-name=h1.h -emit-header-unit -xc++-user-header h1.h -o h1.pcm
+// RUN: %clang_cc1 -verify -w -std=c++20 -fmodule-map-file=module.modulemap -fmodule-file=h1.h=h1.pcm main.cpp -o main.o
+
+//--- module.modulemap
+module "h1.h" {
+  header "h1.h"
+  export *
+}
+
+//--- h0.h
+// expected-no-diagnostics
+#pragma once
+
+template <typename T> struct A {
+  union {
+    struct {
+      T x, y, z;
+    };
+  };
+  constexpr A(T, T, T) : x(), y(), z() {}
+};
+typedef A<float> packed_vec3;
+
+//--- h1.h
+// expected-no-diagnostics
+#pragma once
+
+#include "h0.h"
+
+constexpr packed_vec3 kMessThingsUp = packed_vec3(5.0f, 5.0f, 5.0f);
+
+//--- main.cpp
+// expected-no-diagnostics
+#include "h0.h"
+
+static_assert(sizeof(packed_vec3) == sizeof(float) * 3);
+static_assert(alignof(packed_vec3) == sizeof(float));
+
+import "h1.h";
+
+constexpr packed_vec3 kDefaultHalfExtents = packed_vec3(5.0f, 5.0f, 5.0f);
diff --git a/external/llvm-project/clang/test/Modules/cxx20-force-check-input.cppm b/external/llvm-project/clang/test/Modules/cxx20-force-check-input.cppm
index 312f9b1990bc..a649fe8581f3 100644
--- a/external/llvm-project/clang/test/Modules/cxx20-force-check-input.cppm
+++ b/external/llvm-project/clang/test/Modules/cxx20-force-check-input.cppm
@@ -29,5 +29,5 @@ module;
 export module a;
 export using ::foo;
 
-// CHECK-HEADER-FAILURE: fatal error:{{.*}}foo.h' has been modified since the AST file {{.*}}was built
-// CHECK-FAILURE: fatal error:{{.*}}a.cppm' has been modified since the AST file {{.*}}was built
+// CHECK-HEADER-FAILURE: fatal error:{{.*}}foo.h' has been modified since the precompiled file {{.*}}was built
+// CHECK-FAILURE: fatal error:{{.*}}a.cppm' has been modified since the precompiled file {{.*}}was built
diff --git a/external/llvm-project/clang/test/Modules/diagnose-missing-import.m b/external/llvm-project/clang/test/Modules/diagnose-missing-import.m
index 8fb8e6b25f68..b34bc1a62b6b 100644
--- a/external/llvm-project/clang/test/Modules/diagnose-missing-import.m
+++ b/external/llvm-project/clang/test/Modules/diagnose-missing-import.m
@@ -7,11 +7,9 @@
 void foo(void) {
   XYZLogEvent(xyzRiskyCloseOpenParam, xyzRiskyCloseOpenParam); // expected-error {{call to undeclared function 'XYZLogEvent'; ISO C99 and later do not support implicit function declarations}} \
                                                                   expected-error {{declaration of 'XYZLogEvent' must be imported}} \
-                                                                  expected-error {{declaration of 'xyzRiskyCloseOpenParam' must be imported from module 'NCI.A'}} \
                                                                   expected-error {{declaration of 'xyzRiskyCloseOpenParam' must be imported from module 'NCI.A'}}
 }
 
-// expected-note@Inputs/diagnose-missing-import/a.h:5 {{declaration here is not visible}}
 // expected-note@Inputs/diagnose-missing-import/a.h:5 {{declaration here is not visible}}
 // expected-note@Inputs/diagnose-missing-import/a.h:6 {{declaration here is not visible}}
 
diff --git a/external/llvm-project/clang/test/Modules/explicit-build.cpp b/external/llvm-project/clang/test/Modules/explicit-build.cpp
index 50bba0d09966..c2fe2024a362 100644
--- a/external/llvm-project/clang/test/Modules/explicit-build.cpp
+++ b/external/llvm-project/clang/test/Modules/explicit-build.cpp
@@ -1,17 +1,20 @@
+// Requires a specific target as the "module file has a different size than expected" is not reliable on all architectures.
+// REQUIRES: x86-registered-target
+
 // RUN: rm -rf %t
 
 // -------------------------------
 // Build chained modules A, B, and C
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-name=a -emit-module %S/Inputs/explicit-build/module.modulemap -o %t/a.pcm \
 // RUN:            2>&1 | FileCheck --check-prefix=CHECK-NO-IMPLICIT-BUILD %s --allow-empty
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-name=b -emit-module %S/Inputs/explicit-build/module.modulemap -o %t/b.pcm \
 // RUN:            2>&1 | FileCheck --check-prefix=CHECK-NO-IMPLICIT-BUILD %s --allow-empty
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/b.pcm \
 // RUN:            -fmodule-name=c -emit-module %S/Inputs/explicit-build/module.modulemap -o %t/c.pcm \
 // RUN:            2>&1 | FileCheck --check-prefix=CHECK-NO-IMPLICIT-BUILD %s --allow-empty
@@ -20,7 +23,7 @@
 
 // -------------------------------
 // Build B with an implicit build of A
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-name=b -emit-module %S/Inputs/explicit-build/module.modulemap -o %t/b-not-a.pcm \
 // RUN:            2>&1 | FileCheck --check-prefix=CHECK-B-NO-A %s
 //
@@ -29,36 +32,36 @@
 
 // -------------------------------
 // Check that we can use the explicitly-built A, B, and C modules.
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -verify %s -DHAVE_A
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-map-file=%S/Inputs/explicit-build/module.modulemap \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -verify %s -DHAVE_A
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/b.pcm \
 // RUN:            -verify %s -DHAVE_A -DHAVE_B
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-file=%t/b.pcm \
 // RUN:            -verify %s -DHAVE_A -DHAVE_B
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-file=%t/b.pcm \
 // RUN:            -fmodule-file=%t/c.pcm \
 // RUN:            -verify %s -DHAVE_A -DHAVE_B -DHAVE_C
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-file=%t/c.pcm \
@@ -67,12 +70,12 @@
 // -------------------------------
 // Check that -fmodule-file= in a module build makes the file transitively
 // available even if it's not used.
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fno-implicit-modules -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fno-implicit-modules -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/b.pcm \
 // RUN:            -fmodule-name=d -emit-module %S/Inputs/explicit-build/module.modulemap -o %t/d.pcm \
 // RUN:            2>&1 | FileCheck --check-prefix=CHECK-NO-IMPLICIT-BUILD %s --allow-empty
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fno-implicit-modules -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fno-implicit-modules -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/d.pcm \
 // RUN:            -verify %s -DHAVE_A -DHAVE_B
@@ -104,35 +107,35 @@
 
 // -------------------------------
 // Check that we can use a mixture of implicit and explicit modules.
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/b-not-a.pcm \
 // RUN:            -verify %s -DHAVE_A -DHAVE_B
 
 // -------------------------------
 // Try to use two different flavors of the 'a' module.
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-file=%t/b-not-a.pcm \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-MULTIPLE-AS %s
 //
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-file=%t/b-not-a.pcm \
 // RUN:            -fmodule-map-file=%S/Inputs/explicit-build/module.modulemap \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-MULTIPLE-AS %s
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-name=a -emit-module %S/Inputs/explicit-build/module.modulemap -o %t/a-alt.pcm \
 // RUN:            2>&1 | FileCheck --check-prefix=CHECK-NO-IMPLICIT-BUILD %s --allow-empty
 //
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-file=%t/a-alt.pcm \
 // RUN:            -fmodule-map-file=%S/Inputs/explicit-build/module.modulemap \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-MULTIPLE-AS %s
 //
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/a-alt.pcm \
 // RUN:            -fmodule-file=%t/a.pcm \
 // RUN:            -fmodule-map-file=%S/Inputs/explicit-build/module.modulemap \
@@ -142,35 +145,35 @@
 
 // -------------------------------
 // Try to import a PCH with -fmodule-file=
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-name=a -emit-pch %S/Inputs/explicit-build/a.h -o %t/a.pch -DBUILDING_A_PCH \
 // RUN:            2>&1 | FileCheck --check-prefix=CHECK-NO-IMPLICIT-BUILD %s --allow-empty
 //
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/a.pch \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-A-AS-PCH %s
 //
-// CHECK-A-AS-PCH: fatal error: AST file '{{.*}}a.pch' was not built as a module
+// CHECK-A-AS-PCH: fatal error: precompiled file '{{.*}}a.pch' was not built as a module
 
 // -------------------------------
 // Try to import a non-AST file with -fmodule-file=
 //
 // RUN: touch %t/not.pcm
 //
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/not.pcm \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-BAD-FILE %s
 //
-// CHECK-BAD-FILE: fatal error: file '{{.*}}not.pcm' is not a valid precompiled module file: file too small to contain AST file magic
+// CHECK-BAD-FILE: fatal error: file '{{.*}}not.pcm' is not a valid module file: file too small to contain precompiled file magic
 
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/nonexistent.pcm \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-NO-FILE %s
 //
 // CHECK-NO-FILE: fatal error: module file '{{.*}}nonexistent.pcm' not found: module file not found
 
 // RUN: mv %t/a.pcm %t/a-tmp.pcm
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/c.pcm \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-NO-FILE-INDIRECT %s
@@ -185,7 +188,7 @@
 // Check that we don't get upset if B's timestamp is newer than C's.
 // RUN: touch %t/b.pcm
 //
-// RUN: %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/c.pcm \
 // RUN:            -verify %s -DHAVE_A -DHAVE_B -DHAVE_C
@@ -194,7 +197,7 @@
 //
 // RUN: cp %t/b-not-a.pcm %t/b.pcm
 //
-// RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
+// RUN: not %clang_cc1 -triple=x86_64-linux-gnu -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -I%S/Inputs/explicit-build \
 // RUN:            -fmodule-file=%t/c.pcm \
 // RUN:            %s -DHAVE_A -DHAVE_B -DHAVE_C 2>&1 | FileCheck --check-prefix=CHECK-MISMATCHED-B %s
diff --git a/external/llvm-project/clang/test/Modules/fatal-module-loader-error.m b/external/llvm-project/clang/test/Modules/fatal-module-loader-error.m
index 6e6131e00010..d5bf8fa79358 100644
--- a/external/llvm-project/clang/test/Modules/fatal-module-loader-error.m
+++ b/external/llvm-project/clang/test/Modules/fatal-module-loader-error.m
@@ -8,13 +8,13 @@
 
 #ifdef IMPLICIT
 
-// expected-error@+1{{Module.pcm' is not a valid precompiled module file}}
+// expected-error@+1{{Module.pcm' is not a valid module file}}
 #import <Module/Module.h>
 #pragma clang __debug crash;
 
 #else
 
-// expected-error@+1{{Module.pcm' is not a valid precompiled module file}}
+// expected-error@+1{{Module.pcm' is not a valid module file}}
 @import Module;
 #pragma clang __debug crash;
 
diff --git a/external/llvm-project/clang/test/Modules/ignored_macros.m b/external/llvm-project/clang/test/Modules/ignored_macros.m
index 33801dfa4f47..fe5a6f320629 100644
--- a/external/llvm-project/clang/test/Modules/ignored_macros.m
+++ b/external/llvm-project/clang/test/Modules/ignored_macros.m
@@ -10,7 +10,7 @@
 // RUN: %clang_cc1 -fmodules-cache-path=%t.modules -fmodules -fimplicit-module-maps -I %S/Inputs -emit-pch -o %t.pch -x objective-c-header %s -verify
 // RUN: not %clang_cc1 -fmodules-cache-path=%t.modules -DIGNORED=1 -fmodules -fimplicit-module-maps -I %S/Inputs -include-pch %t.pch %s > %t.err 2>&1
 // RUN: FileCheck -check-prefix=CHECK-CONFLICT %s < %t.err
-// CHECK-CONFLICT: AST file '{{.*}}' was compiled with module cache path
+// CHECK-CONFLICT: precompiled file '{{.*}}' was compiled with module cache path
 
 // Third trial: pass -DIGNORED=1 only to the second invocation, but
 // make it ignored. There should be no failure, IGNORED is defined in
diff --git a/external/llvm-project/clang/test/Modules/load_failure.c b/external/llvm-project/clang/test/Modules/load_failure.c
index 662b39b6f187..ca284c50cedf 100644
--- a/external/llvm-project/clang/test/Modules/load_failure.c
+++ b/external/llvm-project/clang/test/Modules/load_failure.c
@@ -15,7 +15,7 @@
 // RUN: FileCheck -check-prefix=CHECK-FAILURE %s < %t.out
 
 // FIXME: Clean up diagnostic text below and give it a location
-// CHECK-FAILURE: error: C99 was disabled in AST file '{{.*}}load_failure.pcm' but is currently enabled
+// CHECK-FAILURE: error: C99 was disabled in precompiled file '{{.*}}load_failure.pcm' but is currently enabled
 // FIXME: When we have a syntax for modules in C, use that.
 
 
diff --git a/external/llvm-project/clang/test/Modules/merge-target-features.cpp b/external/llvm-project/clang/test/Modules/merge-target-features.cpp
index cc2bbfa077e9..73474e1388fb 100644
--- a/external/llvm-project/clang/test/Modules/merge-target-features.cpp
+++ b/external/llvm-project/clang/test/Modules/merge-target-features.cpp
@@ -20,7 +20,7 @@
 // RUN:   -target-cpu i386 \
 // RUN:   -fsyntax-only merge-target-features.cpp 2>&1 \
 // RUN:   | FileCheck --check-prefix=SUBSET --implicit-check-not=error: %s
-// SUBSET: error: AST file '{{.*}}foo.pcm' was compiled with the target feature '+sse2' but the current translation unit is not
+// SUBSET: error: precompiled file '{{.*}}foo.pcm' was compiled with the target feature '+sse2' but the current translation unit is not
 // SUBSET: error: {{.*}} configuration mismatch
 //
 // RUN: %clang_cc1 -fmodules -x c++ -fmodules-cache-path=%t \
@@ -57,8 +57,8 @@
 // RUN:   -target-cpu i386 -target-feature +cx16 \
 // RUN:   -fsyntax-only merge-target-features.cpp 2>&1 \
 // RUN:   | FileCheck --check-prefix=MISMATCH --implicit-check-not=error: %s
-// MISMATCH: error: AST file '{{.*}}foo.pcm' was compiled with the target feature '+sse2' but the current translation unit is not
-// MISMATCH: error: current translation unit is compiled with the target feature '+cx16' but the AST file '{{.*}}foo.pcm' was not
+// MISMATCH: error: precompiled file '{{.*}}foo.pcm' was compiled with the target feature '+sse2' but the current translation unit is not
+// MISMATCH: error: current translation unit is compiled with the target feature '+cx16' but the precompiled file '{{.*}}foo.pcm' was not
 // MISMATCH: error: {{.*}} configuration mismatch
 
 #include "foo.h"
diff --git a/external/llvm-project/clang/test/Modules/mismatch-diagnostics.cpp b/external/llvm-project/clang/test/Modules/mismatch-diagnostics.cpp
index dffd4b46a678..be58f638cfa8 100644
--- a/external/llvm-project/clang/test/Modules/mismatch-diagnostics.cpp
+++ b/external/llvm-project/clang/test/Modules/mismatch-diagnostics.cpp
@@ -29,5 +29,5 @@ export module mismatching_module;
 
 //--- use.cpp
 import mismatching_module;
-// CHECK: error: POSIX thread support was enabled in AST file '{{.*[/|\\\\]}}mismatching_module.pcm' but is currently disabled
+// CHECK: error: POSIX thread support was enabled in precompiled file '{{.*[/|\\\\]}}mismatching_module.pcm' but is currently disabled
 // CHECK-NEXT: module file {{.*[/|\\\\]}}mismatching_module.pcm cannot be loaded due to a configuration mismatch with the current compilation
diff --git a/external/llvm-project/clang/test/Modules/module-file-modified.c b/external/llvm-project/clang/test/Modules/module-file-modified.c
index 85018b521e28..57160f34a46c 100644
--- a/external/llvm-project/clang/test/Modules/module-file-modified.c
+++ b/external/llvm-project/clang/test/Modules/module-file-modified.c
@@ -8,4 +8,5 @@
 #include "a.h"
 int foo = 0; // redefinition of 'foo'
 // CHECK: fatal error: file {{.*}} has been modified since the module file {{.*}} was built
+// CHECK: note: please rebuild precompiled file
 // REQUIRES: shell
diff --git a/external/llvm-project/clang/test/Modules/module-imported-by-pch-path.m b/external/llvm-project/clang/test/Modules/module-imported-by-pch-path.m
index 04c6caf24dec..1435d011ccbe 100644
--- a/external/llvm-project/clang/test/Modules/module-imported-by-pch-path.m
+++ b/external/llvm-project/clang/test/Modules/module-imported-by-pch-path.m
@@ -12,6 +12,6 @@ void test() {
   (void)MyModuleVersion; // should be found by implicit import
 }
 
-// CHECK: module 'MyModule' in AST file '{{.*MyModule.*pcm}}' (imported by AST file '[[PCH:.*header.pch]]') is not defined in any loaded module map file; maybe you need to load '[[PATH:.*folder-with-modulemap]]
+// CHECK: module 'MyModule' in precompiled file '{{.*MyModule.*pcm}}' (imported by precompiled file '[[PCH:.*header.pch]]') is not defined in any loaded module map file; maybe you need to load '[[PATH:.*folder-with-modulemap]]
 // CHECK: consider adding '[[PATH]]' to the header search path
 // CHECK: imported by '[[PCH]]'
diff --git a/external/llvm-project/clang/test/Modules/module-local-declarations-02.cppm b/external/llvm-project/clang/test/Modules/module-local-declarations-02.cppm
new file mode 100644
index 000000000000..0670c4295abc
--- /dev/null
+++ b/external/llvm-project/clang/test/Modules/module-local-declarations-02.cppm
@@ -0,0 +1,31 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -fprebuilt-module-path=%t -emit-llvm -o %t/B.ll
+
+//--- A.cppm
+export module A;
+
+export template<typename>
+struct holder {
+};
+
+struct foo {};
+
+export struct a {
+	holder<foo> m;
+};
+
+//--- B.cppm
+// expected-no-diagnostics
+export module B;
+
+import A;
+
+struct foo {};
+
+struct b {
+	holder<foo> m;
+};
\ No newline at end of file
diff --git a/external/llvm-project/clang/test/Modules/module-local-declarations.cppm b/external/llvm-project/clang/test/Modules/module-local-declarations.cppm
new file mode 100644
index 000000000000..4fbcf09e4d79
--- /dev/null
+++ b/external/llvm-project/clang/test/Modules/module-local-declarations.cppm
@@ -0,0 +1,30 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/Base.cppm -emit-module-interface -o %t/Base.pcm
+// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm -fprebuilt-module-path=%t
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -fsyntax-only -verify -fprebuilt-module-path=%t
+
+//--- Base.cppm
+export module Base;
+export template <class T>
+class Base {};
+
+//--- A.cppm
+export module A;
+import Base;
+struct S {};
+
+export Base<S> a;
+
+//--- B.cppm
+// expected-no-diagnostics
+export module B;
+
+import A;
+import Base;
+
+struct S {};
+
+export Base<S> b;
diff --git a/external/llvm-project/clang/test/Modules/module-pch-different-cache-path.c b/external/llvm-project/clang/test/Modules/module-pch-different-cache-path.c
index 8dd04a166eab..af8fcff12638 100644
--- a/external/llvm-project/clang/test/Modules/module-pch-different-cache-path.c
+++ b/external/llvm-project/clang/test/Modules/module-pch-different-cache-path.c
@@ -14,5 +14,5 @@
 
 pch_int x = 0;
 
-// CHECK-ERROR: AST file '{{.*}}' was compiled with module cache path '{{.*}}', but the path is currently '{{.*}}'
-// CHECK-SUCCESS-NOT: AST file '{{.*}}' was compiled with module cache path '{{.*}}', but the path is currently '{{.*}}'
+// CHECK-ERROR: precompiled file '{{.*}}' was compiled with module cache path '{{.*}}', but the path is currently '{{.*}}'
+// CHECK-SUCCESS-NOT: precompiled file '{{.*}}' was compiled with module cache path '{{.*}}', but the path is currently '{{.*}}'
diff --git a/external/llvm-project/clang/test/Modules/objc-categories.cpp b/external/llvm-project/clang/test/Modules/objc-categories.cpp
new file mode 100644
index 000000000000..2c004c4fd19c
--- /dev/null
+++ b/external/llvm-project/clang/test/Modules/objc-categories.cpp
@@ -0,0 +1,14 @@
+// RUN: rm -rf %t.pcm
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -x c++ -std=c++11  -emit-module -fmodules -fmodule-name=cxx_library  %S/Inputs/module.modulemap -o %t.pcm
+// RUN: llvm-bcanalyzer %t.pcm | FileCheck %s --check-prefix=CXX_LIBRARY
+// CXX_LIBRARY: AST_BLOCK
+// CXX_LIBRARY-NOT: OBJC_CATEGORIES
+// CXX_LIBRARY-NOT: OBJC_CATEGORIES_MAP
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -x objective-c -emit-module -fmodules -fmodule-name=category_top  %S/Inputs/module.modulemap -o %t.pcm
+// RUN: llvm-bcanalyzer %t.pcm | FileCheck %s --check-prefix=CATEGORY_TOP
+// CATEGORY_TOP: AST_BLOCK
+// CATEGORY_TOP: OBJC_CATEGORIES
+// CATEGORY_TOP: OBJC_CATEGORIES_MAP
+
diff --git a/external/llvm-project/clang/test/Modules/pr119947.cppm b/external/llvm-project/clang/test/Modules/pr119947.cppm
new file mode 100644
index 000000000000..40de2cad3c0d
--- /dev/null
+++ b/external/llvm-project/clang/test/Modules/pr119947.cppm
@@ -0,0 +1,54 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/use.cpp -fprebuilt-module-path=%t -emit-llvm -o -
+
+
+//--- a.cppm
+export module a;
+
+struct a_inner {
+	~a_inner() {
+	}
+	void f(auto) {
+	}
+};
+
+export template<typename T>
+struct a {
+	a() {
+		struct local {};
+		inner.f(local());
+	}
+private:
+	a_inner inner;
+};
+
+
+namespace {
+
+struct s {
+};
+
+} // namespace
+
+void f() {
+	a<s> x;
+}
+
+//--- use.cpp
+import a;
+
+namespace {
+
+struct s {
+};
+
+} // namespace
+
+void g() {
+	a<s> x;
+}
+
diff --git a/external/llvm-project/clang/test/Modules/pr143788.cppm b/external/llvm-project/clang/test/Modules/pr143788.cppm
new file mode 100644
index 000000000000..5ae36d8d0e85
--- /dev/null
+++ b/external/llvm-project/clang/test/Modules/pr143788.cppm
@@ -0,0 +1,28 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/M.cppm -emit-module-interface -o %t/M.pcm
+// RUN: %clang_cc1 -std=c++20 %t/P.cppm -emit-module-interface -o %t/P.pcm
+// RUN: %clang_cc1 -std=c++20 %t/I.cpp -fmodule-file=M:P=%t/P.pcm -fmodule-file=M=%t/M.pcm -fsyntax-only -verify
+
+//--- H.hpp
+struct S{};
+
+//--- M.cppm
+export module M;
+
+
+//--- P.cppm
+module;
+#include "H.hpp"
+module M:P;
+
+using T = S;
+
+//--- I.cpp
+// expected-no-diagnostics
+module M;
+import :P;
+
+T f() { return {}; }
diff --git a/external/llvm-project/clang/test/Modules/pr61360.cppm b/external/llvm-project/clang/test/Modules/pr61360.cppm
new file mode 100644
index 000000000000..a16f65d4be2f
--- /dev/null
+++ b/external/llvm-project/clang/test/Modules/pr61360.cppm
@@ -0,0 +1,25 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -fprebuilt-module-path=%t -emit-llvm -o %t/B.ll
+
+//--- A.cppm
+export module A;
+export template<typename>
+struct holder {
+};
+
+struct a {
+	holder<struct foo> m;
+};
+
+//--- B.cppm
+// expected-no-diagnostics
+export module B;
+import A;
+
+struct b {
+	holder<struct foo> m;
+};
diff --git a/external/llvm-project/clang/test/Modules/pr62359.cppm b/external/llvm-project/clang/test/Modules/pr62359.cppm
index 7d9d3eec26cc..fab0b7d03d81 100644
--- a/external/llvm-project/clang/test/Modules/pr62359.cppm
+++ b/external/llvm-project/clang/test/Modules/pr62359.cppm
@@ -43,7 +43,7 @@ int use() {
   return 0;
 }
 
-// CHECK: OpenMP{{.*}}differs in AST file '{{.*}}Hello.pcm' vs. current file
+// CHECK: OpenMP{{.*}}differs in precompiled file '{{.*}}Hello.pcm' vs. current file
 
 //--- use2.cpp
 // expected-no-diagnostics
@@ -55,5 +55,5 @@ int use2() {
   return 0;
 }
 
-// CHECK: OpenMP{{.*}}differs in AST file '{{.*}}Hello.pcm' vs. current file
+// CHECK: OpenMP{{.*}}differs in precompiled file '{{.*}}Hello.pcm' vs. current file
 // CHECK: use of undeclared identifier 'pragma'
diff --git a/external/llvm-project/clang/test/Modules/resolution-change.m b/external/llvm-project/clang/test/Modules/resolution-change.m
index d413dd3b042a..e2cc3a634a39 100644
--- a/external/llvm-project/clang/test/Modules/resolution-change.m
+++ b/external/llvm-project/clang/test/Modules/resolution-change.m
@@ -11,11 +11,11 @@
 
 // Use the PCH with no way to resolve DependsOnA
 // RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -include-pch %t-A.pch %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CHECK-NODOA %s
-// CHECK-NODOA: module 'DependsOnA' in AST file '{{.*DependsOnA.*pcm}}' (imported by AST file '{{.*A.pch}}') is not defined in any loaded module map
+// CHECK-NODOA: module 'DependsOnA' in precompiled file '{{.*DependsOnA.*pcm}}' (imported by precompiled file '{{.*A.pch}}') is not defined in any loaded module map
 
 // Use the PCH with no way to resolve A
 // RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I %S/Inputs/modules-with-same-name/DependsOnA -include-pch %t-A.pch %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CHECK-NOA %s
-// CHECK-NOA: module 'A' in AST file '{{.*A.*pcm}}' (imported by AST file '{{.*DependsOnA.*pcm}}') is not defined in any loaded module map
+// CHECK-NOA: module 'A' in precompiled file '{{.*A.*pcm}}' (imported by precompiled file '{{.*DependsOnA.*pcm}}') is not defined in any loaded module map
 
 // Use the PCH and have it resolve to the other A
 // RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I %S/Inputs/modules-with-same-name/DependsOnA -I %S/Inputs/modules-with-same-name/path2/A -include-pch %t-A.pch %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CHECK-WRONGA %s
diff --git a/external/llvm-project/clang/test/Modules/validate-file-content.m b/external/llvm-project/clang/test/Modules/validate-file-content.m
index 327a68a9b641..9977aa4665f0 100644
--- a/external/llvm-project/clang/test/Modules/validate-file-content.m
+++ b/external/llvm-project/clang/test/Modules/validate-file-content.m
@@ -29,5 +29,5 @@
 // CHECK: file '[[M_H:.*[/\\]m\.h]]' has been modified since the precompiled header '[[A_PCH:.*/a\.pch]]' was built: content changed
 // CHECK: '[[M_H]]' required by '[[M_PCM:.*[/\\]m.*\.pcm]]'
 // CHECK: '[[M_PCM]]' required by '[[A_PCH]]'
-// CHECK: please rebuild precompiled header '[[A_PCH]]'
+// CHECK: please rebuild precompiled file '[[A_PCH]]'
 // expected-no-diagnostics
diff --git a/external/llvm-project/clang/test/OpenMP/begin_declare_variant_messages.c b/external/llvm-project/clang/test/OpenMP/begin_declare_variant_messages.c
index 2c64ae74289b..ea68fb52d3a3 100644
--- a/external/llvm-project/clang/test/OpenMP/begin_declare_variant_messages.c
+++ b/external/llvm-project/clang/test/OpenMP/begin_declare_variant_messages.c
@@ -83,7 +83,7 @@ const int var;
 #pragma omp end declare variant
 #pragma omp begin declare variant match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}}
 #pragma omp end declare variant
-#pragma omp begin declare variant match(device = {kind(score(ibm) }) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<recovery-expr>()'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp begin declare variant match(device = {kind(score(ibm) }) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp end declare variant
 #pragma omp begin declare variant match(device={kind(score(2 gpu)}) // expected-error {{expected ')'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('2'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{to match this '('}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp end declare variant
diff --git a/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.c b/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.c
index 3c554a106fe4..bb83f23a0c18 100644
--- a/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.c
+++ b/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.c
@@ -49,6 +49,23 @@ struct dat {
 #pragma omp declare mapper(struct dat d) map(to: d.d)
 // CHECK: #pragma omp declare mapper (default : struct dat d) map(to: d.d){{$}}
 
+// Verify that nested default mappers do not lead to a crash during parsing / sema.
+// CHECK: struct inner {
+struct inner {
+  int size;
+  int *data;
+};
+#pragma omp declare mapper(struct inner i) map(i, i.data[0 : i.size])
+// CHECK: #pragma omp declare mapper (default : struct inner i) map(tofrom: default::i,i.data[0:i.size]){{$}}
+
+// CHECK: struct outer {
+struct outer {
+  int a;
+  struct inner i;
+};
+#pragma omp declare mapper(struct outer o) map(o)
+// CHECK: #pragma omp declare mapper (default : struct outer o) map(tofrom: default::o) map(tofrom: o.i){{$}}
+
 // CHECK: int main(void) {
 int main(void) {
 #pragma omp declare mapper(id: struct vec v) map(v.len)
@@ -77,6 +94,14 @@ int main(void) {
 #pragma omp declare mapper(id1: struct vec vvec) map(iterator(it=0:vvec.len:2), tofrom:vvec.data[it])
 // OMP52: #pragma omp declare mapper (id1 : struct vec vvec) map(iterator(int it = 0:vvec.len:2),tofrom: vvec.data[it]);
 #endif
+
+  {
+    struct outer outer;
+#pragma omp target map(outer)
+// CHECK: #pragma omp target map(tofrom: outer)
+    { }
+  }
+
   return 0;
 }
 // CHECK: }
diff --git a/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.cpp b/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.cpp
index 422fa9981672..9ca3412e3e3d 100644
--- a/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.cpp
+++ b/external/llvm-project/clang/test/OpenMP/declare_mapper_ast_print.cpp
@@ -34,6 +34,28 @@ class vecchild : public vec {
 // CHECK: }
 // CHECK: ;
 
+// Verify that nested default mappers do not lead to a crash during parsing / sema.
+// CHECK: namespace N2 {
+namespace N2
+{
+// CHECK: struct inner {
+struct inner {
+  int size;
+  int *data;
+};
+#pragma omp declare mapper(struct inner i) map(i, i.data[0 : i.size])
+// CHECK: #pragma omp declare mapper (default : struct inner i) map(tofrom: N2::default::i,i.data[0:i.size]){{$}}
+
+// CHECK: struct outer {
+struct outer {
+  int a;
+  struct inner i;
+};
+#pragma omp declare mapper(struct outer o) map(o)
+// CHECK: #pragma omp declare mapper (default : struct outer o) map(tofrom: N2::default::o) map(tofrom: o.i){{$}}
+} // namespace N2
+// CHECK: }
+
 template <class T>
 class dat {
 public:
@@ -122,6 +144,7 @@ T foo(T a) {
 int main() {
   N1::vec vv, vvv;
   N1::vecchild vc;
+  N2::outer outer;
   dat<double> dd;
 #pragma omp target map(mapper(N1::id) tofrom: vv) map(mapper(dat<double>::id) alloc: vvv)
 // CHECK: #pragma omp target map(mapper(N1::id),tofrom: vv) map(mapper(dat<double>::id),alloc: vvv)
@@ -132,6 +155,9 @@ int main() {
 #pragma omp target map(mapper(default) tofrom: dd)
 // CHECK: #pragma omp target map(mapper(default),tofrom: dd)
   { dd.d++; }
+#pragma omp target map(outer)
+// CHECK: #pragma omp target map(tofrom: outer)
+  { }
 
 #pragma omp target update to(mapper(N1::id) : vc)
 // CHECK: #pragma omp target update to(mapper(N1::id): vc)
diff --git a/external/llvm-project/clang/test/OpenMP/declare_reduction_messages.cpp b/external/llvm-project/clang/test/OpenMP/declare_reduction_messages.cpp
index 752cc4fb05a1..f91d952dfa14 100644
--- a/external/llvm-project/clang/test/OpenMP/declare_reduction_messages.cpp
+++ b/external/llvm-project/clang/test/OpenMP/declare_reduction_messages.cpp
@@ -69,7 +69,7 @@ class Class2 : public Class1<T> {
 #pragma omp declare reduction(fun77 : long : omp_out += omp_in) initializer(omp_priv Class2 < int > ()) // expected-error {{expected ')'}} expected-note {{to match this '('}}
 #pragma omp declare reduction(fun8 : long : omp_out += omp_in) initializer(omp_priv 23)                 // expected-error {{expected ')'}} expected-note {{to match this '('}}
 #pragma omp declare reduction(fun88 : long : omp_out += omp_in) initializer(omp_priv 23))               // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{extra tokens at the end of '#pragma omp declare reduction' are ignored}}
-#pragma omp declare reduction(fun9 : long : omp_out += omp_priv) initializer(omp_in = 23)               // expected-error {{use of undeclared identifier 'omp_priv'; did you mean 'omp_in'?}} expected-note {{'omp_in' declared here}}
+#pragma omp declare reduction(fun9 : long : omp_out += omp_priv) initializer(omp_in = 23)               // expected-error {{use of undeclared identifier 'omp_priv'}}
 #pragma omp declare reduction(fun10 : long : omp_out += omp_in) initializer(omp_priv = 23)
 
 template <typename T>
diff --git a/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_ast_print.cpp b/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_ast_print.cpp
index 172dd1670421..c14e19cc8b7e 100644
--- a/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_ast_print.cpp
+++ b/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_ast_print.cpp
@@ -54,9 +54,9 @@ void foo_v3(float *AAA, float *BBB, int *I) {return;}
 //DUMP: DeclRefExpr{{.*}}Function{{.*}}foo_v1
 //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'AAA'
 //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'BBB'
-//PRINT: #pragma omp declare variant(foo_v3) match(construct={dispatch}, device={arch(x86, x86_64)}) adjust_args(nothing:I) adjust_args(need_device_ptr:BBB)
+//PRINT: #pragma omp declare variant(foo_v3) match(construct={dispatch}, device={arch(x86, x86_64)}) adjust_args(nothing:I) adjust_args(need_device_ptr:BBB) adjust_args(need_device_addr:AAA)
 
-//PRINT: #pragma omp declare variant(foo_v2) match(construct={dispatch}, device={arch(ppc)}) adjust_args(need_device_ptr:AAA)
+//PRINT: #pragma omp declare variant(foo_v2) match(construct={dispatch}, device={arch(ppc)}) adjust_args(need_device_ptr:AAA) adjust_args(need_device_addr:BBB)
 
 //PRINT: omp declare variant(foo_v1) match(construct={dispatch}, device={arch(arm)}) adjust_args(need_device_ptr:AAA,BBB)
 
@@ -66,42 +66,48 @@ void foo_v3(float *AAA, float *BBB, int *I) {return;}
 
 #pragma omp declare variant(foo_v2)                        \
    match(construct={dispatch}, device={arch(ppc)}),        \
-   adjust_args(need_device_ptr:AAA)
+   adjust_args(need_device_ptr:AAA)                        \
+   adjust_args(need_device_addr:BBB)
 
 #pragma omp declare variant(foo_v3)                        \
    adjust_args(need_device_ptr:BBB) adjust_args(nothing:I) \
+   adjust_args(need_device_addr:AAA)                      \
    match(construct={dispatch}, device={arch(x86,x86_64)})
 
 void foo(float *AAA, float *BBB, int *I) {return;}
 
-void Foo_Var(float *AAA, float *BBB) {return;}
+void Foo_Var(float *AAA, float *BBB, float *CCC) {return;}
 
 #pragma omp declare variant(Foo_Var) \
    match(construct={dispatch}, device={arch(x86_64)}) \
-   adjust_args(need_device_ptr:AAA) adjust_args(nothing:BBB)
+   adjust_args(need_device_ptr:AAA) adjust_args(nothing:BBB) \
+   adjust_args(need_device_addr:CCC)
 template<typename T>
-void Foo(T *AAA, T *BBB) {return;}
+void Foo(T *AAA, T *BBB, T *CCC) {return;}
 
-//PRINT: #pragma omp declare variant(Foo_Var) match(construct={dispatch}, device={arch(x86_64)}) adjust_args(nothing:BBB) adjust_args(need_device_ptr:AAA)
-//DUMP: FunctionDecl{{.*}} Foo 'void (T *, T *)'
+//PRINT: #pragma omp declare variant(Foo_Var) match(construct={dispatch}, device={arch(x86_64)}) adjust_args(nothing:BBB) adjust_args(need_device_ptr:AAA) adjust_args(need_device_addr:CCC)
+//DUMP: FunctionDecl{{.*}} Foo 'void (T *, T *, T *)'
 //DUMP: OMPDeclareVariantAttr{{.*}}device={arch(x86_64)}
 //DUMP: DeclRefExpr{{.*}}Function{{.*}}Foo_Var
 //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'BBB'
 //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'AAA'
+//DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'CCC'
 //
-//DUMP: FunctionDecl{{.*}} Foo 'void (float *, float *)'
+//DUMP: FunctionDecl{{.*}} Foo 'void (float *, float *, float *)'
 //DUMP: OMPDeclareVariantAttr{{.*}}device={arch(x86_64)}
 //DUMP: DeclRefExpr{{.*}}Function{{.*}}Foo_Var
 //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'BBB'
 //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'AAA'
+//DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'CCC'
 
 void func()
 {
   float *A;
   float *B;
+  float *C;
 
   //#pragma omp dispatch
-  Foo(A, B);
+  Foo(A, B, C);
 }
 
 typedef void *omp_interop_t;
diff --git a/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_messages.cpp b/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_messages.cpp
index 284e49bbd21b..aadded7699ea 100644
--- a/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_messages.cpp
+++ b/external/llvm-project/clang/test/OpenMP/declare_variant_clauses_messages.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -std=c++11 -o - %s
-// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -std=c++11 \
+// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -fopenmp-version=60 -std=c++11 -o - %s
+// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -fopenmp-version=60 -std=c++11 \
 // RUN:  -DNO_INTEROP_T_DEF -o - %s
-// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -std=c++11 -o - %s
-// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -Wno-strict-prototypes -DC -x c -o - %s
+// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -fopenmp-version=60 -std=c++11 -o - %s
+// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux -fopenmp -fopenmp-version=60 -Wno-strict-prototypes -DC -x c -o - %s
 // RUN: %clang_cc1 -verify -triple x86_64-pc-windows-msvc -fms-compatibility \
-// RUN:  -fopenmp -Wno-strict-prototypes -DC -DWIN -x c -o - %s
+// RUN:  -fopenmp -fopenmp-version=60 -Wno-strict-prototypes -DC -DWIN -x c -o - %s
 
 #ifdef NO_INTEROP_T_DEF
 void foo_v1(float *, void *);
@@ -114,6 +114,16 @@ void vararg_bar2(const char *fmt) { return; }
    match(construct={dispatch}, device={arch(ppc)}),          \
    adjust_args(need_device_ptr:AAA) adjust_args(nothing:AAA)
 
+// expected-error@+3 {{'adjust_arg' argument 'AAA' used in multiple clauses}}
+#pragma omp declare variant(foo_v1)                          \
+   match(construct={dispatch}, device={arch(arm)})           \
+   adjust_args(need_device_ptr:AAA,BBB) adjust_args(need_device_addr:AAA)
+
+// expected-error@+3 {{'adjust_arg' argument 'AAA' used in multiple clauses}}
+#pragma omp declare variant(foo_v1)                          \
+   match(construct={dispatch}, device={arch(ppc)}),          \
+   adjust_args(need_device_addr:AAA) adjust_args(nothing:AAA)
+
 // expected-error@+2 {{use of undeclared identifier 'J'}}
 #pragma omp declare variant(foo_v1)                          \
    adjust_args(nothing:J)                                    \
@@ -186,12 +196,12 @@ void vararg_bar2(const char *fmt) { return; }
 // expected-error@+1 {{variant in '#pragma omp declare variant' with type 'void (float *, float *, int *, omp_interop_t)' (aka 'void (float *, float *, int *, void *)') is incompatible with type 'void (float *, float *, int *)'}}
 #pragma omp declare variant(foo_v4) match(construct={dispatch})
 
-// expected-error@+3 {{incorrect adjust_args type, expected 'need_device_ptr' or 'nothing'}}
+// expected-error@+3 {{incorrect 'adjust_args' type, expected 'need_device_ptr', 'need_device_addr', or 'nothing'}}
 #pragma omp declare variant(foo_v1)                        \
    match(construct={dispatch}, device={arch(arm)})         \
    adjust_args(badaaop:AAA,BBB)
 
-// expected-error@+3 {{incorrect adjust_args type, expected 'need_device_ptr' or 'nothing'}}
+// expected-error@+3 {{incorrect 'adjust_args' type, expected 'need_device_ptr', 'need_device_addr', or 'nothing'}}
 #pragma omp declare variant(foo_v1)                        \
    match(construct={dispatch}, device={arch(arm)})         \
    adjust_args(badaaop AAA,BBB)
diff --git a/external/llvm-project/clang/test/OpenMP/declare_variant_messages.c b/external/llvm-project/clang/test/OpenMP/declare_variant_messages.c
index 4184a0e4279c..3275c6c8f95c 100644
--- a/external/llvm-project/clang/test/OpenMP/declare_variant_messages.c
+++ b/external/llvm-project/clang/test/OpenMP/declare_variant_messages.c
@@ -11,7 +11,7 @@ int foo(void);
 #pragma omp declare variant // expected-error {{expected '(' after 'declare variant'}}
 #pragma omp declare variant( // expected-error {{expected expression}} expected-error {{expected ')'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo // expected-error {{expected ')'}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}} expected-note {{to match this '('}}
-#pragma omp declare variant(x) // expected-error {{use of undeclared identifier 'x'}} omp50-error {{expected 'match' clause on}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(x) // expected-error {{use of undeclared identifier 'x'}}
 #pragma omp declare variant(foo) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foo) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foo) xxx // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
@@ -42,7 +42,7 @@ int foo(void);
 #pragma omp declare variant(foo) match(device={kind(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo) match(device={kind()}) // expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}}
 #pragma omp declare variant(foo) match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}}
-#pragma omp declare variant(foo) match(device = {kind(score(ibm) }) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<recovery-expr>()'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match(device = {kind(score(ibm) }) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo) match(device={kind(score(2 gpu)}) // expected-error {{expected ')'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('2'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{to match this '('}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo) match(device={kind(score(foo()) ibm)}) // expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo()'); score ignored}} expected-warning {{'ibm' is not a valid context property for the context selector 'kind' and the context set 'device'; property ignored}} expected-note {{try 'match(implementation={vendor(ibm)})'}} expected-note {{the ignored property spans until here}}
 #pragma omp declare variant(foo) match(device={kind(score(5): host), kind(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'kind' was used already in the same 'omp declare variant' directive; selector ignored}} expected-note {{the previous context selector 'kind' used here}} expected-note {{the ignored selector spans until here}}
@@ -56,7 +56,7 @@ int foo(void);
 #pragma omp declare variant(foo) match(target_device={device_num}) // expected-warning {{the context selector 'device_num' in context set 'target_device' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foo) match(target_device={device_num()}) // expected-error {{expected expression}}
 #pragma omp declare variant(foo) match(target_device={device_num(-1)}) // expected-error {{argument to 'device_num' clause must be a non-negative integer value}}
-#pragma omp declare variant(foo) match(target_device={device_num(abc)}) // expected-error {{expected expression}} expected-error {{use of undeclared identifier 'abc'}}
+#pragma omp declare variant(foo) match(target_device={device_num(abc)}) // expected-error {{use of undeclared identifier 'abc'}}
 int bar(void);
 
 
diff --git a/external/llvm-project/clang/test/OpenMP/declare_variant_messages.cpp b/external/llvm-project/clang/test/OpenMP/declare_variant_messages.cpp
index 4e65a529fc41..b15940b9d8cd 100644
--- a/external/llvm-project/clang/test/OpenMP/declare_variant_messages.cpp
+++ b/external/llvm-project/clang/test/OpenMP/declare_variant_messages.cpp
@@ -16,7 +16,7 @@ T foofoo();
 #pragma omp declare variant // expected-error {{expected '(' after 'declare variant'}}
 #pragma omp declare variant( // expected-error {{expected expression}} expected-error {{expected ')'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo // expected-error {{expected ')'}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}} expected-note {{to match this '('}}
-#pragma omp declare variant(x) // expected-error {{use of undeclared identifier 'x'}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(x) // expected-error {{use of undeclared identifier 'x'}}
 #pragma omp declare variant(foo) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo <int>) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo <int>) xxx // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
@@ -57,7 +57,7 @@ int bar();
 #pragma omp declare variant // expected-error {{expected '(' after 'declare variant'}}
 #pragma omp declare variant( // expected-error {{expected expression}} expected-error {{expected ')'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foofoo <T> // expected-error {{expected ')'}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}} expected-note {{to match this '('}}
-#pragma omp declare variant(x) // expected-error {{use of undeclared identifier 'x'}} omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(x) // expected-error {{use of undeclared identifier 'x'}}
 #pragma omp declare variant(foo) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
 #pragma omp declare variant(foofoo <T>) // omp50-error {{expected 'match' clause on 'omp declare variant' directive}} omp51-error {{expected 'match', 'adjust_args', or 'append_args' clause on 'omp declare variant' directive}}
diff --git a/external/llvm-project/clang/test/OpenMP/distribute_simd_misc_messages.c b/external/llvm-project/clang/test/OpenMP/distribute_simd_misc_messages.c
index 8cbf96cd7a01..270e17dcb89b 100644
--- a/external/llvm-project/clang/test/OpenMP/distribute_simd_misc_messages.c
+++ b/external/llvm-project/clang/test/OpenMP/distribute_simd_misc_messages.c
@@ -508,6 +508,7 @@ void test_collapse(void) {
 #pragma omp distribute simd collapse(5 - 5)
   for (i = 0; i < 16; ++i)
     ;
+#if defined(_OPENMP) && (_OPENMP <= 202111)
 // expected-note@+3 2 {{defined as reduction}}
 #pragma omp target
 #pragma omp teams
@@ -520,7 +521,7 @@ void test_collapse(void) {
 #pragma omp for reduction(+ : i, j)
       for (int k = 0; k < 16; ++k)
         i += j;
-
+#endif
 #pragma omp target
 #pragma omp teams
   for (i = 0; i < 16; ++i)
diff --git a/external/llvm-project/clang/test/OpenMP/for_private_reduction_codegen.cpp b/external/llvm-project/clang/test/OpenMP/for_private_reduction_codegen.cpp
new file mode 100644
index 000000000000..c8a6863299fb
--- /dev/null
+++ b/external/llvm-project/clang/test/OpenMP/for_private_reduction_codegen.cpp
@@ -0,0 +1,710 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-globals --include-generated-funcs --replace-value-regex "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex ".omp.reduction..internal[a-zA-Z_0-9.]+"
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -fopenmp-version=60 -x c++ -std=c++17  -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+#define N 10
+class Sum {
+  int val;
+
+public:
+  Sum(int v = 0) : val(v) {}
+  Sum operator+(const Sum &rhs) const { return Sum(val + rhs.val); }
+  Sum &operator+=(const Sum &rhs) {
+    val += rhs.val;
+    return *this;
+  }
+};
+#pragma omp declare reduction(sum_reduction:Sum : omp_out += omp_in)           \
+    initializer(omp_priv = Sum(0))
+
+void func_red() {
+  Sum result(0);
+  Sum array[N];
+
+  for (int i = 0; i < N; i++) {
+    array[i] = Sum(i);
+  }
+
+#pragma omp parallel private(result) num_threads(4)
+  {
+#pragma omp for reduction(sum_reduction : result)
+    for (int i = 0; i < N; i++) {
+      result = result + array[i];
+    }
+  }
+}
+
+void do_red(int n, int *v, int &sum_v) {
+  sum_v = 0;
+#pragma omp for reduction(original(private), + : sum_v)
+  for (int i = 0; i < n; i++) {
+    sum_v += v[i];
+  }
+}
+void do_red_extended(int n, int *v, int &sum_v, int &prod_v) {
+  sum_v = 0;
+  prod_v = 1;
+
+#pragma omp for reduction(original(private), + : sum_v)                        \
+    reduction(original(private), * : prod_v)
+  for (int i = 0; i < n; i++) {
+    sum_v += v[i];
+    prod_v *= v[i];
+  }
+}
+int main(void) {
+  int v[N];
+  for (int i = 0; i < N; i++)
+    v[i] = i;
+#pragma omp parallel num_threads(4)
+  {
+    int s_v;
+    do_red(N, v, s_v);
+  }
+
+  int sum_v_ext = 0, prod_v_ext = 1;
+#pragma omp parallel num_threads(4)
+  {
+    do_red_extended(N, v, sum_v_ext, prod_v_ext);
+  }
+  return 0;
+}
+
+//.
+// CHECK: @.omp.reduction..internal_pivate_.result.result_996 = common global %class.Sum zeroinitializer, align 4
+// CHECK: @.omp.reduction..internal_pivate_.sum_v.sum_v_1188 = common global i32 0, align 4
+// CHECK: @.omp.reduction..internal_pivate_.sum_v.sum_v_1392 = common global i32 0, align 4
+// CHECK: @.omp.reduction..internal_pivate_.prod_v.prod_v_1461 = common global i32 0, align 4
+//.
+// CHECK-LABEL: define {{[^@]+}}@_Z8func_redv
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca [[CLASS_SUM:%.*]], align 4
+// CHECK-NEXT:    [[ARRAY:%.*]] = alloca [10 x %class.Sum], align 16
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[REF_TMP:%.*]] = alloca [[CLASS_SUM]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB3:[0-9]+]])
+// CHECK-NEXT:    call void @_ZN3SumC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[RESULT]], i32 noundef 0)
+// CHECK-NEXT:    [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [10 x %class.Sum], ptr [[ARRAY]], i32 0, i32 0
+// CHECK-NEXT:    [[ARRAYCTOR_END:%.*]] = getelementptr inbounds [[CLASS_SUM]], ptr [[ARRAY_BEGIN]], i64 10
+// CHECK-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
+// CHECK:       arrayctor.loop:
+// CHECK-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
+// CHECK-NEXT:    call void @_ZN3SumC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]], i32 noundef 0)
+// CHECK-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[CLASS_SUM]], ptr [[ARRAYCTOR_CUR]], i64 1
+// CHECK-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
+// CHECK-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
+// CHECK:       arrayctor.cont:
+// CHECK-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 10
+// CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    call void @_ZN3SumC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[REF_TMP]], i32 noundef [[TMP2]])
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP3]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x %class.Sum], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[ARRAYIDX]], ptr align 4 [[REF_TMP]], i64 4, i1 false)
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
+// CHECK-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    call void @__kmpc_push_num_threads(ptr @[[GLOB3]], i32 [[TMP0]], i32 4)
+// CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 1, ptr @_Z8func_redv.omp_outlined, ptr [[ARRAY]])
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_ZN3SumC1Ei
+// CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]], i32 noundef [[V:%.*]]) unnamed_addr #[[ATTR0]] comdat align 2 {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[V_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[V]], ptr [[V_ADDR]], align 4
+// CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[V_ADDR]], align 4
+// CHECK-NEXT:    call void @_ZN3SumC2Ei(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], i32 noundef [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z8func_redv.omp_outlined
+// CHECK-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef nonnull align 4 dereferenceable(40) [[ARRAY:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[ARRAY_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca [[CLASS_SUM:%.*]], align 4
+// CHECK-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[RESULT1:%.*]] = alloca [[CLASS_SUM]], align 4
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[REF_TMP:%.*]] = alloca [[CLASS_SUM]], align 4
+// CHECK-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [0 x ptr], align 8
+// CHECK-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
+// CHECK-NEXT:    store ptr [[ARRAY]], ptr [[ARRAY_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARRAY_ADDR]], align 8
+// CHECK-NEXT:    call void @_ZN3SumC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[RESULT]], i32 noundef 0)
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK-NEXT:    store i32 9, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK-NEXT:    call void @.omp_initializer.(ptr noundef [[RESULT1]], ptr noundef [[RESULT]])
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
+// CHECK-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP2]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP3]], 9
+// CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK:       cond.true:
+// CHECK-NEXT:    br label [[COND_END:%.*]]
+// CHECK:       cond.false:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    br label [[COND_END]]
+// CHECK:       cond.end:
+// CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 9, [[COND_TRUE]] ], [ [[TMP4]], [[COND_FALSE]] ]
+// CHECK-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK-NEXT:    store i32 [[TMP5]], ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK:       omp.inner.for.cond:
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK:       omp.inner.for.body:
+// CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 1
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP9]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x %class.Sum], ptr [[TMP0]], i64 0, i64 [[IDXPROM]]
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @_ZNK3SumplERKS_(ptr noundef nonnull align 4 dereferenceable(4) [[RESULT1]], ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYIDX]])
+// CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[CLASS_SUM]], ptr [[REF_TMP]], i32 0, i32 0
+// CHECK-NEXT:    store i32 [[CALL]], ptr [[COERCE_DIVE]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[RESULT1]], ptr align 4 [[REF_TMP]], i64 4, i1 false)
+// CHECK-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK:       omp.body.continue:
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK:       omp.inner.for.inc:
+// CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK-NEXT:    store i32 [[ADD3]], ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK:       omp.inner.for.end:
+// CHECK-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK:       omp.loop.exit:
+// CHECK-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP2]])
+// CHECK-NEXT:    [[TMP11:%.*]] = call i32 @__kmpc_reduce(ptr @[[GLOB2:[0-9]+]], i32 [[TMP2]], i32 0, i64 0, ptr [[DOTOMP_REDUCTION_RED_LIST]], ptr @_Z8func_redv.omp_outlined.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    switch i32 [[TMP11]], label [[DOTOMP_REDUCTION_DEFAULT:%.*]] [
+// CHECK-NEXT:      i32 1, label [[DOTOMP_REDUCTION_CASE1:%.*]]
+// CHECK-NEXT:      i32 2, label [[DOTOMP_REDUCTION_CASE2:%.*]]
+// CHECK-NEXT:    ]
+// CHECK:       .omp.reduction.case1:
+// CHECK-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP2]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
+// CHECK:       .omp.reduction.case2:
+// CHECK-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP2]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
+// CHECK:       .omp.reduction.default:
+// CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP2]], 0
+// CHECK-NEXT:    br i1 [[TMP12]], label [[INIT:%.*]], label [[INIT_END:%.*]]
+// CHECK:       init:
+// CHECK-NEXT:    call void @_ZN3SumC1Ei(ptr noundef nonnull align 4 dereferenceable(4) @.omp.reduction..internal_pivate_.result.result_996, i32 noundef 0)
+// CHECK-NEXT:    br label [[INIT_END]]
+// CHECK:       init.end:
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP2]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP2]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    call void @.omp_combiner.(ptr noundef @.omp.reduction..internal_pivate_.result.result_996, ptr noundef [[RESULT1]])
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP2]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP2]])
+// CHECK-NEXT:    [[TMP13:%.*]] = load [[CLASS_SUM]], ptr @.omp.reduction..internal_pivate_.result.result_996, align 4
+// CHECK-NEXT:    store [[CLASS_SUM]] [[TMP13]], ptr [[RESULT1]], align 4
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP2]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP2]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    call void @.omp_combiner.(ptr noundef [[RESULT]], ptr noundef [[RESULT1]])
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP2]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4:[0-9]+]], i32 [[TMP2]])
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@.omp_combiner.
+// CHECK-SAME: (ptr noalias noundef [[TMP0:%.*]], ptr noalias noundef [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTADDR1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    [[CALL:%.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @_ZN3SumpLERKS_(ptr noundef nonnull align 4 dereferenceable(4) [[TMP3]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]])
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_ZN3SumpLERKS_
+// CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[RHS:%.*]]) #[[ATTR0]] comdat align 2 {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[RHS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[RHS]], ptr [[RHS_ADDR]], align 8
+// CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[RHS_ADDR]], align 8
+// CHECK-NEXT:    [[VAL:%.*]] = getelementptr inbounds nuw [[CLASS_SUM:%.*]], ptr [[TMP0]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[VAL]], align 4
+// CHECK-NEXT:    [[VAL2:%.*]] = getelementptr inbounds nuw [[CLASS_SUM]], ptr [[THIS1]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VAL2]], align 4
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+// CHECK-NEXT:    store i32 [[ADD]], ptr [[VAL2]], align 4
+// CHECK-NEXT:    ret ptr [[THIS1]]
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@.omp_initializer.
+// CHECK-SAME: (ptr noalias noundef [[TMP0:%.*]], ptr noalias noundef [[TMP1:%.*]]) #[[ATTR3]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTADDR1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    call void @_ZN3SumC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[TMP3]], i32 noundef 0)
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_ZNK3SumplERKS_
+// CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[RHS:%.*]]) #[[ATTR0]] comdat align 2 {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[CLASS_SUM:%.*]], align 4
+// CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[RHS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[RHS]], ptr [[RHS_ADDR]], align 8
+// CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    [[VAL:%.*]] = getelementptr inbounds nuw [[CLASS_SUM]], ptr [[THIS1]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[VAL]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[RHS_ADDR]], align 8
+// CHECK-NEXT:    [[VAL2:%.*]] = getelementptr inbounds nuw [[CLASS_SUM]], ptr [[TMP1]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[VAL2]], align 4
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP2]]
+// CHECK-NEXT:    call void @_ZN3SumC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[RETVAL]], i32 noundef [[ADD]])
+// CHECK-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[CLASS_SUM]], ptr [[RETVAL]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[COERCE_DIVE]], align 4
+// CHECK-NEXT:    ret i32 [[TMP3]]
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z8func_redv.omp_outlined.omp.reduction.reduction_func
+// CHECK-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR5:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTADDR1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_ZN3SumC2Ei
+// CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]], i32 noundef [[V:%.*]]) unnamed_addr #[[ATTR0]] comdat align 2 {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[V_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[V]], ptr [[V_ADDR]], align 4
+// CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    [[VAL:%.*]] = getelementptr inbounds nuw [[CLASS_SUM:%.*]], ptr [[THIS1]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[V_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[VAL]], align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z6do_rediPiRi
+// CHECK-SAME: (i32 noundef [[N:%.*]], ptr noundef [[V:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[SUM_V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[V_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[SUM_V_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[SUM_V4:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[_TMP5:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[I6:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [0 x ptr], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB3]])
+// CHECK-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[V]], ptr [[V_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[SUM_V]], ptr [[SUM_V_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[SUM_V_ADDR]], align 8
+// CHECK-NEXT:    store i32 0, ptr [[TMP1]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[SUM_V_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP2]], ptr [[TMP]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP4]], 0
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK-NEXT:    [[SUB3:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK-NEXT:    store i32 [[SUB3]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 0, [[TMP5]]
+// CHECK-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK:       omp.precond.then:
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK-NEXT:    store i32 [[TMP6]], ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK-NEXT:    store i32 0, ptr [[SUM_V4]], align 4
+// CHECK-NEXT:    store ptr [[SUM_V4]], ptr [[_TMP5]], align 8
+// CHECK-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[TMP8]], [[TMP9]]
+// CHECK-NEXT:    br i1 [[CMP7]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK:       cond.true:
+// CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK-NEXT:    br label [[COND_END:%.*]]
+// CHECK:       cond.false:
+// CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    br label [[COND_END]]
+// CHECK:       cond.end:
+// CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP10]], [[COND_TRUE]] ], [ [[TMP11]], [[COND_FALSE]] ]
+// CHECK-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK-NEXT:    store i32 [[TMP12]], ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK:       omp.inner.for.cond:
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP13]], [[TMP14]]
+// CHECK-NEXT:    br i1 [[CMP8]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK:       omp.inner.for.body:
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP15]], 1
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK-NEXT:    store i32 [[ADD]], ptr [[I6]], align 4
+// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[V_ADDR]], align 8
+// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I6]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[_TMP5]], align 8
+// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4
+// CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP20]], [[TMP18]]
+// CHECK-NEXT:    store i32 [[ADD9]], ptr [[TMP19]], align 4
+// CHECK-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK:       omp.body.continue:
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK:       omp.inner.for.inc:
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK-NEXT:    store i32 [[ADD10]], ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK:       omp.inner.for.end:
+// CHECK-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK:       omp.loop.exit:
+// CHECK-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP22:%.*]] = call i32 @__kmpc_reduce(ptr @[[GLOB2]], i32 [[TMP0]], i32 0, i64 0, ptr [[DOTOMP_REDUCTION_RED_LIST]], ptr @_Z6do_rediPiRi.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    switch i32 [[TMP22]], label [[DOTOMP_REDUCTION_DEFAULT:%.*]] [
+// CHECK-NEXT:      i32 1, label [[DOTOMP_REDUCTION_CASE1:%.*]]
+// CHECK-NEXT:      i32 2, label [[DOTOMP_REDUCTION_CASE2:%.*]]
+// CHECK-NEXT:    ]
+// CHECK:       .omp.reduction.case1:
+// CHECK-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
+// CHECK:       .omp.reduction.case2:
+// CHECK-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
+// CHECK:       .omp.reduction.default:
+// CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[TMP0]], 0
+// CHECK-NEXT:    br i1 [[TMP23]], label [[INIT:%.*]], label [[INIT_END:%.*]]
+// CHECK:       init:
+// CHECK-NEXT:    store i32 0, ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1188, align 4
+// CHECK-NEXT:    br label [[INIT_END]]
+// CHECK:       init.end:
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1188, align 4
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[SUM_V4]], align 4
+// CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP24]], [[TMP25]]
+// CHECK-NEXT:    store i32 [[ADD11]], ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1188, align 4
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1188, align 4
+// CHECK-NEXT:    store i32 [[TMP26]], ptr [[SUM_V4]], align 4
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[TMP7]], align 4
+// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[SUM_V4]], align 4
+// CHECK-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP27]], [[TMP28]]
+// CHECK-NEXT:    store i32 [[ADD12]], ptr [[TMP7]], align 4
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    br label [[OMP_PRECOND_END]]
+// CHECK:       omp.precond.end:
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z6do_rediPiRi.omp.reduction.reduction_func
+// CHECK-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTADDR1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z15do_red_extendediPiRiS0_
+// CHECK-SAME: (i32 noundef [[N:%.*]], ptr noundef [[V:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[SUM_V:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[PROD_V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[V_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[SUM_V_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[PROD_V_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[_TMP1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[SUM_V5:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[_TMP6:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[PROD_V7:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[_TMP8:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[I9:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [0 x ptr], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB3]])
+// CHECK-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
+// CHECK-NEXT:    store ptr [[V]], ptr [[V_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[SUM_V]], ptr [[SUM_V_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[PROD_V]], ptr [[PROD_V_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[SUM_V_ADDR]], align 8
+// CHECK-NEXT:    store i32 0, ptr [[TMP1]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PROD_V_ADDR]], align 8
+// CHECK-NEXT:    store i32 1, ptr [[TMP2]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[SUM_V_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP3]], ptr [[TMP]], align 8
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[PROD_V_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP4]], ptr [[_TMP1]], align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP5]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP6]], 0
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[SUB]], 1
+// CHECK-NEXT:    [[SUB4:%.*]] = sub nsw i32 [[DIV]], 1
+// CHECK-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_3]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 0, [[TMP7]]
+// CHECK-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK:       omp.precond.then:
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
+// CHECK-NEXT:    store i32 [[TMP8]], ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK-NEXT:    store i32 0, ptr [[SUM_V5]], align 4
+// CHECK-NEXT:    store ptr [[SUM_V5]], ptr [[_TMP6]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK-NEXT:    store i32 1, ptr [[PROD_V7]], align 4
+// CHECK-NEXT:    store ptr [[PROD_V7]], ptr [[_TMP8]], align 8
+// CHECK-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
+// CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
+// CHECK-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK:       cond.true:
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
+// CHECK-NEXT:    br label [[COND_END:%.*]]
+// CHECK:       cond.false:
+// CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    br label [[COND_END]]
+// CHECK:       cond.end:
+// CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP13]], [[COND_TRUE]] ], [ [[TMP14]], [[COND_FALSE]] ]
+// CHECK-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK-NEXT:    store i32 [[TMP15]], ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+// CHECK:       omp.inner.for.cond:
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK-NEXT:    [[CMP11:%.*]] = icmp sle i32 [[TMP16]], [[TMP17]]
+// CHECK-NEXT:    br i1 [[CMP11]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK:       omp.inner.for.body:
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP18]], 1
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK-NEXT:    store i32 [[ADD]], ptr [[I9]], align 4
+// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[V_ADDR]], align 8
+// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I9]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP20]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[_TMP6]], align 8
+// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
+// CHECK-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP23]], [[TMP21]]
+// CHECK-NEXT:    store i32 [[ADD12]], ptr [[TMP22]], align 4
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[V_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I9]], align 4
+// CHECK-NEXT:    [[IDXPROM13:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, ptr [[TMP24]], i64 [[IDXPROM13]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[ARRAYIDX14]], align 4
+// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[TMP27]], align 4
+// CHECK-NEXT:    [[MUL15:%.*]] = mul nsw i32 [[TMP28]], [[TMP26]]
+// CHECK-NEXT:    store i32 [[MUL15]], ptr [[TMP27]], align 4
+// CHECK-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK:       omp.body.continue:
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
+// CHECK:       omp.inner.for.inc:
+// CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP29]], 1
+// CHECK-NEXT:    store i32 [[ADD16]], ptr [[DOTOMP_IV]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND]]
+// CHECK:       omp.inner.for.end:
+// CHECK-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+// CHECK:       omp.loop.exit:
+// CHECK-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP30:%.*]] = call i32 @__kmpc_reduce(ptr @[[GLOB2]], i32 [[TMP0]], i32 0, i64 0, ptr [[DOTOMP_REDUCTION_RED_LIST]], ptr @_Z15do_red_extendediPiRiS0_.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    switch i32 [[TMP30]], label [[DOTOMP_REDUCTION_DEFAULT:%.*]] [
+// CHECK-NEXT:      i32 1, label [[DOTOMP_REDUCTION_CASE1:%.*]]
+// CHECK-NEXT:      i32 2, label [[DOTOMP_REDUCTION_CASE2:%.*]]
+// CHECK-NEXT:    ]
+// CHECK:       .omp.reduction.case1:
+// CHECK-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
+// CHECK:       .omp.reduction.case2:
+// CHECK-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
+// CHECK:       .omp.reduction.default:
+// CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[TMP0]], 0
+// CHECK-NEXT:    br i1 [[TMP31]], label [[INIT:%.*]], label [[INIT_END:%.*]]
+// CHECK:       init:
+// CHECK-NEXT:    store i32 0, ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1392, align 4
+// CHECK-NEXT:    br label [[INIT_END]]
+// CHECK:       init.end:
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1392, align 4
+// CHECK-NEXT:    [[TMP33:%.*]] = load i32, ptr [[SUM_V5]], align 4
+// CHECK-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP32]], [[TMP33]]
+// CHECK-NEXT:    store i32 [[ADD17]], ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1392, align 4
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP34:%.*]] = load i32, ptr @.omp.reduction..internal_pivate_.sum_v.sum_v_1392, align 4
+// CHECK-NEXT:    store i32 [[TMP34]], ptr [[SUM_V5]], align 4
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    [[TMP35:%.*]] = load i32, ptr [[TMP9]], align 4
+// CHECK-NEXT:    [[TMP36:%.*]] = load i32, ptr [[SUM_V5]], align 4
+// CHECK-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP35]], [[TMP36]]
+// CHECK-NEXT:    store i32 [[ADD18]], ptr [[TMP9]], align 4
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    [[TMP37:%.*]] = icmp eq i32 [[TMP0]], 0
+// CHECK-NEXT:    br i1 [[TMP37]], label [[INIT19:%.*]], label [[INIT_END20:%.*]]
+// CHECK:       init19:
+// CHECK-NEXT:    store i32 1, ptr @.omp.reduction..internal_pivate_.prod_v.prod_v_1461, align 4
+// CHECK-NEXT:    br label [[INIT_END20]]
+// CHECK:       init.end20:
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    [[TMP38:%.*]] = load i32, ptr @.omp.reduction..internal_pivate_.prod_v.prod_v_1461, align 4
+// CHECK-NEXT:    [[TMP39:%.*]] = load i32, ptr [[PROD_V7]], align 4
+// CHECK-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP38]], [[TMP39]]
+// CHECK-NEXT:    store i32 [[MUL21]], ptr @.omp.reduction..internal_pivate_.prod_v.prod_v_1461, align 4
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP40:%.*]] = load i32, ptr @.omp.reduction..internal_pivate_.prod_v.prod_v_1461, align 4
+// CHECK-NEXT:    store i32 [[TMP40]], ptr [[PROD_V7]], align 4
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[TMP0]])
+// CHECK-NEXT:    call void @__kmpc_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    [[TMP41:%.*]] = load i32, ptr [[TMP10]], align 4
+// CHECK-NEXT:    [[TMP42:%.*]] = load i32, ptr [[PROD_V7]], align 4
+// CHECK-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP41]], [[TMP42]]
+// CHECK-NEXT:    store i32 [[MUL22]], ptr [[TMP10]], align 4
+// CHECK-NEXT:    call void @__kmpc_end_critical(ptr @[[GLOB3]], i32 [[TMP0]], ptr @.gomp_critical_user_.reduction_critical.var)
+// CHECK-NEXT:    br label [[OMP_PRECOND_END]]
+// CHECK:       omp.precond.end:
+// CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z15do_red_extendediPiRiS0_.omp.reduction.reduction_func
+// CHECK-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTADDR1:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[DOTADDR1]], align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@main
+// CHECK-SAME: () #[[ATTR7:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[V:%.*]] = alloca [10 x i32], align 16
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[SUM_V_EXT:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[PROD_V_EXT:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB3]])
+// CHECK-NEXT:    store i32 0, ptr [[RETVAL]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 10
+// CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP3]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x i32], ptr [[V]], i64 0, i64 [[IDXPROM]]
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
+// CHECK-NEXT:    store i32 [[INC]], ptr [[I]], align 4
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    call void @__kmpc_push_num_threads(ptr @[[GLOB3]], i32 [[TMP0]], i32 4)
+// CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 1, ptr @main.omp_outlined, ptr [[V]])
+// CHECK-NEXT:    store i32 0, ptr [[SUM_V_EXT]], align 4
+// CHECK-NEXT:    store i32 1, ptr [[PROD_V_EXT]], align 4
+// CHECK-NEXT:    call void @__kmpc_push_num_threads(ptr @[[GLOB3]], i32 [[TMP0]], i32 4)
+// CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 3, ptr @main.omp_outlined.1, ptr [[V]], ptr [[SUM_V_EXT]], ptr [[PROD_V_EXT]])
+// CHECK-NEXT:    ret i32 0
+
diff --git a/external/llvm-project/clang/test/OpenMP/for_reduction_messages.cpp b/external/llvm-project/clang/test/OpenMP/for_reduction_messages.cpp
index de28ba2c3be0..2fdac3048c9c 100644
--- a/external/llvm-project/clang/test/OpenMP/for_reduction_messages.cpp
+++ b/external/llvm-project/clang/test/OpenMP/for_reduction_messages.cpp
@@ -417,10 +417,12 @@ int main(int argc, char **argv) {
 #pragma omp for reduction(+ : qa[1], qa[0])
   for (int i = 0; i < 10; ++i)
     foo();
+#if defined(_OPENMP) && (_OPENMP <= 202111)
 #pragma omp parallel reduction(* : fl) // expected-note {{defined as reduction}}
 #pragma omp for reduction(+ : fl)      // expected-error {{reduction variable must be shared}}
   for (int i = 0; i < 10; ++i)
     foo();
+#endif
   static int m=0;
 #pragma omp for reduction(+:m)
   for (int i = 0; i < 10; ++i)
diff --git a/external/llvm-project/clang/test/OpenMP/for_simd_reduction_messages.cpp b/external/llvm-project/clang/test/OpenMP/for_simd_reduction_messages.cpp
index 96b3805b10a8..a9ef6c39cb5d 100644
--- a/external/llvm-project/clang/test/OpenMP/for_simd_reduction_messages.cpp
+++ b/external/llvm-project/clang/test/OpenMP/for_simd_reduction_messages.cpp
@@ -396,11 +396,11 @@ int main(int argc, char **argv) {
 #pragma omp for simd reduction(+ : fl) // expected-error {{reduction variable must be shared}}
   for (int i = 0; i < 10; ++i)
     foo();
-#endif
 #pragma omp parallel reduction(* : fl) // expected-note {{defined as reduction}}
 #pragma omp for simd reduction(+ : fl)      // expected-error {{reduction variable must be shared}}
   for (int i = 0; i < 10; ++i)
     foo();
+#endif
   static int m;
 #pragma omp for simd reduction(+ : m)
   for (int i = 0; i < 10; ++i)
diff --git a/external/llvm-project/clang/test/OpenMP/sections_reduction_messages.cpp b/external/llvm-project/clang/test/OpenMP/sections_reduction_messages.cpp
index 42ec3ed6d58e..8cde6489f325 100644
--- a/external/llvm-project/clang/test/OpenMP/sections_reduction_messages.cpp
+++ b/external/llvm-project/clang/test/OpenMP/sections_reduction_messages.cpp
@@ -461,12 +461,12 @@ int main(int argc, char **argv) {
   {
     foo();
   }
-#endif
 #pragma omp parallel reduction(* : fl) // expected-note {{defined as reduction}}
 #pragma omp sections reduction(+ : fl) // expected-error {{reduction variable must be shared}}
   {
     foo();
   }
+#endif
   static int m;
 #pragma omp sections reduction(+ : m) // OK
   {
diff --git a/external/llvm-project/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_ast_dump.cpp b/external/llvm-project/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_ast_dump.cpp
new file mode 100644
index 000000000000..a5847709d3e7
--- /dev/null
+++ b/external/llvm-project/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_ast_dump.cpp
@@ -0,0 +1,34 @@
+//RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -ast-dump  %s | FileCheck %s --check-prefix=DUM
+
+typedef struct {
+  int a;
+} C;
+#pragma omp declare mapper(C s) map(to : s.a)
+
+typedef struct {
+  int e;
+  C f;
+  int h;
+} D;
+
+void foo() {
+  D sa[10];
+  sa[1].e = 111;
+  sa[1].f.a = 222;
+
+#pragma omp target map(tofrom : sa)
+  {
+    sa[0].e = 333;
+    sa[1].f.a = 444;
+  }
+}
+
+// DUM: -OMPDeclareMapperDecl{{.*}}<<invalid sloc>> <invalid sloc>
+// DUM-NEXT:  |-OMPMapClause {{.*}}<<invalid sloc>> <implicit>
+// DUM-NEXT:  | |-MemberExpr {{.*}}<line:9:3> 'int' lvalue .e
+// DUM-NEXT:  | | `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
+// DUM-NEXT:  | |-MemberExpr {{.*}}<line:10:3> 'C' lvalue .f {{.*}}
+// DUM-NEXT:  | | `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
+// DUM-NEXT:  | `-MemberExpr {{.*}}<line:11:3> 'int' lvalue .h {{.*}}
+// DUM-NEXT:  |   `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
+// DUM-NEXT:  `-VarDecl {{.*}} <line:12:1> col:1 implicit used _s 'D'
diff --git a/external/llvm-project/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_codegen.cpp b/external/llvm-project/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_codegen.cpp
new file mode 100644
index 000000000000..5df1e958ad55
--- /dev/null
+++ b/external/llvm-project/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_codegen.cpp
@@ -0,0 +1,323 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-globals --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex "\.offload_.*"
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+typedef struct {
+  int a;
+} C;
+#pragma omp declare mapper(C s) map(to : s.a)
+
+typedef struct {
+  int e;
+  C f;
+  int h;
+} D;
+
+void foo() {
+  D sa[10];
+  sa[1].e = 111;
+  sa[1].f.a = 222;
+
+#pragma omp target map(tofrom : sa)
+  {
+    sa[1].e = 333;
+    sa[1].f.a = 444;
+  }
+}
+#endif
+//.
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 120]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 35]
+//.
+// CHECK-LABEL: define {{[^@]+}}@_Z3foov
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[SA:%.*]] = alloca [10 x %struct.D], align 4
+// CHECK-NEXT:    [[DOTOFFLOAD_BASEPTRS:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_PTRS:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_MAPPERS:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[KERNEL_ARGS:%.*]] = alloca [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x %struct.D], ptr [[SA]], i64 0, i64 1
+// CHECK-NEXT:    [[E:%.*]] = getelementptr inbounds nuw [[STRUCT_D:%.*]], ptr [[ARRAYIDX]], i32 0, i32 0
+// CHECK-NEXT:    store i32 111, ptr [[E]], align 4
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [10 x %struct.D], ptr [[SA]], i64 0, i64 1
+// CHECK-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_D]], ptr [[ARRAYIDX1]], i32 0, i32 1
+// CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_C:%.*]], ptr [[F]], i32 0, i32 0
+// CHECK-NEXT:    store i32 222, ptr [[A]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK-NEXT:    store ptr [[SA]], ptr [[TMP0]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK-NEXT:    store ptr [[SA]], ptr [[TMP1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 0
+// CHECK-NEXT:    store ptr @.omp_mapper._ZTS1D.default, ptr [[TMP2]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0
+// CHECK-NEXT:    store i32 3, ptr [[TMP5]], align 4
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 1
+// CHECK-NEXT:    store i32 1, ptr [[TMP6]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 2
+// CHECK-NEXT:    store ptr [[TMP3]], ptr [[TMP7]], align 8
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 3
+// CHECK-NEXT:    store ptr [[TMP4]], ptr [[TMP8]], align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 4
+// CHECK-NEXT:    store ptr @.offload_sizes, ptr [[TMP9]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 5
+// CHECK-NEXT:    store ptr @.offload_maptypes, ptr [[TMP10]], align 8
+// CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 6
+// CHECK-NEXT:    store ptr null, ptr [[TMP11]], align 8
+// CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 7
+// CHECK-NEXT:    store ptr [[DOTOFFLOAD_MAPPERS]], ptr [[TMP12]], align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 8
+// CHECK-NEXT:    store i64 0, ptr [[TMP13]], align 8
+// CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 9
+// CHECK-NEXT:    store i64 0, ptr [[TMP14]], align 8
+// CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 10
+// CHECK-NEXT:    store [3 x i32] [i32 -1, i32 0, i32 0], ptr [[TMP15]], align 4
+// CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 11
+// CHECK-NEXT:    store [3 x i32] zeroinitializer, ptr [[TMP16]], align 4
+// CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 12
+// CHECK-NEXT:    store i32 0, ptr [[TMP17]], align 4
+// CHECK-NEXT:    [[TMP18:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1:[0-9]+]], i64 -1, i32 -1, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l26.region_id, ptr [[KERNEL_ARGS]])
+// CHECK-NEXT:    [[TMP19:%.*]] = icmp ne i32 [[TMP18]], 0
+// CHECK-NEXT:    br i1 [[TMP19]], label [[OMP_OFFLOAD_FAILED:%.*]], label [[OMP_OFFLOAD_CONT:%.*]]
+// CHECK:       omp_offload.failed:
+// CHECK-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l26(ptr [[SA]]) #[[ATTR3:[0-9]+]]
+// CHECK-NEXT:    br label [[OMP_OFFLOAD_CONT]]
+// CHECK:       omp_offload.cont:
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l26
+// CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(120) [[SA:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[SA_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[SA]], ptr [[SA_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SA_ADDR]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x %struct.D], ptr [[TMP0]], i64 0, i64 1
+// CHECK-NEXT:    [[E:%.*]] = getelementptr inbounds nuw [[STRUCT_D:%.*]], ptr [[ARRAYIDX]], i32 0, i32 0
+// CHECK-NEXT:    store i32 333, ptr [[E]], align 4
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [10 x %struct.D], ptr [[TMP0]], i64 0, i64 1
+// CHECK-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_D]], ptr [[ARRAYIDX1]], i32 0, i32 1
+// CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_C:%.*]], ptr [[F]], i32 0, i32 0
+// CHECK-NEXT:    store i32 444, ptr [[A]], align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@.omp_mapper._ZTS1D.default
+// CHECK-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]], i64 noundef [[TMP3:%.*]], i64 noundef [[TMP4:%.*]], ptr noundef [[TMP5:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP6:%.*]] = udiv exact i64 [[TMP3]], 12
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr [[STRUCT_D:%.*]], ptr [[TMP2]], i64 [[TMP6]]
+// CHECK-NEXT:    [[OMP_ARRAYINIT_ISARRAY:%.*]] = icmp sgt i64 [[TMP6]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP4]], 8
+// CHECK-NEXT:    [[TMP9:%.*]] = icmp ne ptr [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP4]], 16
+// CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i64 [[TMP10]], 0
+// CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[TMP9]], [[TMP11]]
+// CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[OMP_ARRAYINIT_ISARRAY]], [[TMP12]]
+// CHECK-NEXT:    [[DOTOMP_ARRAY__INIT__DELETE:%.*]] = icmp eq i64 [[TMP8]], 0
+// CHECK-NEXT:    [[TMP14:%.*]] = and i1 [[TMP13]], [[DOTOMP_ARRAY__INIT__DELETE]]
+// CHECK-NEXT:    br i1 [[TMP14]], label [[DOTOMP_ARRAY__INIT:%.*]], label [[OMP_ARRAYMAP_HEAD:%.*]]
+// CHECK:       .omp.array..init:
+// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP6]], 12
+// CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP4]], -4
+// CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP16]], 512
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP15]], i64 [[TMP17]], ptr [[TMP5]])
+// CHECK-NEXT:    br label [[OMP_ARRAYMAP_HEAD]]
+// CHECK:       omp.arraymap.head:
+// CHECK-NEXT:    [[OMP_ARRAYMAP_ISEMPTY:%.*]] = icmp eq ptr [[TMP2]], [[TMP7]]
+// CHECK-NEXT:    br i1 [[OMP_ARRAYMAP_ISEMPTY]], label [[OMP_DONE:%.*]], label [[OMP_ARRAYMAP_BODY:%.*]]
+// CHECK:       omp.arraymap.body:
+// CHECK-NEXT:    [[OMP_ARRAYMAP_PTRCURRENT:%.*]] = phi ptr [ [[TMP2]], [[OMP_ARRAYMAP_HEAD]] ], [ [[OMP_ARRAYMAP_NEXT:%.*]], [[OMP_TYPE_END20:%.*]] ]
+// CHECK-NEXT:    [[E:%.*]] = getelementptr inbounds nuw [[STRUCT_D]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0
+// CHECK-NEXT:    [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_D]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 1
+// CHECK-NEXT:    [[H:%.*]] = getelementptr inbounds nuw [[STRUCT_D]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 2
+// CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i32, ptr [[H]], i32 1
+// CHECK-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[TMP18]] to i64
+// CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[E]] to i64
+// CHECK-NEXT:    [[TMP21:%.*]] = sub i64 [[TMP19]], [[TMP20]]
+// CHECK-NEXT:    [[TMP22:%.*]] = sdiv exact i64 [[TMP21]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK-NEXT:    [[TMP23:%.*]] = call i64 @__tgt_mapper_num_components(ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP24:%.*]] = shl i64 [[TMP23]], 48
+// CHECK-NEXT:    [[TMP25:%.*]] = add nuw i64 0, [[TMP24]]
+// CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP4]], 3
+// CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0
+// CHECK-NEXT:    br i1 [[TMP27]], label [[OMP_TYPE_ALLOC:%.*]], label [[OMP_TYPE_ALLOC_ELSE:%.*]]
+// CHECK:       omp.type.alloc:
+// CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP25]], -4
+// CHECK-NEXT:    br label [[OMP_TYPE_END:%.*]]
+// CHECK:       omp.type.alloc.else:
+// CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[TMP26]], 1
+// CHECK-NEXT:    br i1 [[TMP29]], label [[OMP_TYPE_TO:%.*]], label [[OMP_TYPE_TO_ELSE:%.*]]
+// CHECK:       omp.type.to:
+// CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP25]], -3
+// CHECK-NEXT:    br label [[OMP_TYPE_END]]
+// CHECK:       omp.type.to.else:
+// CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[TMP26]], 2
+// CHECK-NEXT:    br i1 [[TMP31]], label [[OMP_TYPE_FROM:%.*]], label [[OMP_TYPE_END]]
+// CHECK:       omp.type.from:
+// CHECK-NEXT:    [[TMP32:%.*]] = and i64 [[TMP25]], -2
+// CHECK-NEXT:    br label [[OMP_TYPE_END]]
+// CHECK:       omp.type.end:
+// CHECK-NEXT:    [[OMP_MAPTYPE:%.*]] = phi i64 [ [[TMP28]], [[OMP_TYPE_ALLOC]] ], [ [[TMP30]], [[OMP_TYPE_TO]] ], [ [[TMP32]], [[OMP_TYPE_FROM]] ], [ [[TMP25]], [[OMP_TYPE_TO_ELSE]] ]
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[E]], i64 [[TMP22]], i64 [[OMP_MAPTYPE]], ptr null)
+// CHECK-NEXT:    [[TMP33:%.*]] = add nuw i64 281474976711171, [[TMP24]]
+// CHECK-NEXT:    [[TMP34:%.*]] = and i64 [[TMP4]], 3
+// CHECK-NEXT:    [[TMP35:%.*]] = icmp eq i64 [[TMP34]], 0
+// CHECK-NEXT:    br i1 [[TMP35]], label [[OMP_TYPE_ALLOC1:%.*]], label [[OMP_TYPE_ALLOC_ELSE2:%.*]]
+// CHECK:       omp.type.alloc1:
+// CHECK-NEXT:    [[TMP36:%.*]] = and i64 [[TMP33]], -4
+// CHECK-NEXT:    br label [[OMP_TYPE_END6:%.*]]
+// CHECK:       omp.type.alloc.else2:
+// CHECK-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[TMP34]], 1
+// CHECK-NEXT:    br i1 [[TMP37]], label [[OMP_TYPE_TO3:%.*]], label [[OMP_TYPE_TO_ELSE4:%.*]]
+// CHECK:       omp.type.to3:
+// CHECK-NEXT:    [[TMP38:%.*]] = and i64 [[TMP33]], -3
+// CHECK-NEXT:    br label [[OMP_TYPE_END6]]
+// CHECK:       omp.type.to.else4:
+// CHECK-NEXT:    [[TMP39:%.*]] = icmp eq i64 [[TMP34]], 2
+// CHECK-NEXT:    br i1 [[TMP39]], label [[OMP_TYPE_FROM5:%.*]], label [[OMP_TYPE_END6]]
+// CHECK:       omp.type.from5:
+// CHECK-NEXT:    [[TMP40:%.*]] = and i64 [[TMP33]], -2
+// CHECK-NEXT:    br label [[OMP_TYPE_END6]]
+// CHECK:       omp.type.end6:
+// CHECK-NEXT:    [[OMP_MAPTYPE7:%.*]] = phi i64 [ [[TMP36]], [[OMP_TYPE_ALLOC1]] ], [ [[TMP38]], [[OMP_TYPE_TO3]] ], [ [[TMP40]], [[OMP_TYPE_FROM5]] ], [ [[TMP33]], [[OMP_TYPE_TO_ELSE4]] ]
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[E]], i64 4, i64 [[OMP_MAPTYPE7]], ptr null)
+// CHECK-NEXT:    [[TMP41:%.*]] = add nuw i64 281474976711171, [[TMP24]]
+// CHECK-NEXT:    [[TMP42:%.*]] = and i64 [[TMP4]], 3
+// CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i64 [[TMP42]], 0
+// CHECK-NEXT:    br i1 [[TMP43]], label [[OMP_TYPE_ALLOC8:%.*]], label [[OMP_TYPE_ALLOC_ELSE9:%.*]]
+// CHECK:       omp.type.alloc8:
+// CHECK-NEXT:    [[TMP44:%.*]] = and i64 [[TMP41]], -4
+// CHECK-NEXT:    br label [[OMP_TYPE_END13:%.*]]
+// CHECK:       omp.type.alloc.else9:
+// CHECK-NEXT:    [[TMP45:%.*]] = icmp eq i64 [[TMP42]], 1
+// CHECK-NEXT:    br i1 [[TMP45]], label [[OMP_TYPE_TO10:%.*]], label [[OMP_TYPE_TO_ELSE11:%.*]]
+// CHECK:       omp.type.to10:
+// CHECK-NEXT:    [[TMP46:%.*]] = and i64 [[TMP41]], -3
+// CHECK-NEXT:    br label [[OMP_TYPE_END13]]
+// CHECK:       omp.type.to.else11:
+// CHECK-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[TMP42]], 2
+// CHECK-NEXT:    br i1 [[TMP47]], label [[OMP_TYPE_FROM12:%.*]], label [[OMP_TYPE_END13]]
+// CHECK:       omp.type.from12:
+// CHECK-NEXT:    [[TMP48:%.*]] = and i64 [[TMP41]], -2
+// CHECK-NEXT:    br label [[OMP_TYPE_END13]]
+// CHECK:       omp.type.end13:
+// CHECK-NEXT:    [[OMP_MAPTYPE14:%.*]] = phi i64 [ [[TMP44]], [[OMP_TYPE_ALLOC8]] ], [ [[TMP46]], [[OMP_TYPE_TO10]] ], [ [[TMP48]], [[OMP_TYPE_FROM12]] ], [ [[TMP41]], [[OMP_TYPE_TO_ELSE11]] ]
+// CHECK-NEXT:    call void @.omp_mapper._ZTS1C.default(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[F]], i64 4, i64 [[OMP_MAPTYPE14]], ptr null) #[[ATTR3]]
+// CHECK-NEXT:    [[TMP49:%.*]] = add nuw i64 281474976711171, [[TMP24]]
+// CHECK-NEXT:    [[TMP50:%.*]] = and i64 [[TMP4]], 3
+// CHECK-NEXT:    [[TMP51:%.*]] = icmp eq i64 [[TMP50]], 0
+// CHECK-NEXT:    br i1 [[TMP51]], label [[OMP_TYPE_ALLOC15:%.*]], label [[OMP_TYPE_ALLOC_ELSE16:%.*]]
+// CHECK:       omp.type.alloc15:
+// CHECK-NEXT:    [[TMP52:%.*]] = and i64 [[TMP49]], -4
+// CHECK-NEXT:    br label [[OMP_TYPE_END20]]
+// CHECK:       omp.type.alloc.else16:
+// CHECK-NEXT:    [[TMP53:%.*]] = icmp eq i64 [[TMP50]], 1
+// CHECK-NEXT:    br i1 [[TMP53]], label [[OMP_TYPE_TO17:%.*]], label [[OMP_TYPE_TO_ELSE18:%.*]]
+// CHECK:       omp.type.to17:
+// CHECK-NEXT:    [[TMP54:%.*]] = and i64 [[TMP49]], -3
+// CHECK-NEXT:    br label [[OMP_TYPE_END20]]
+// CHECK:       omp.type.to.else18:
+// CHECK-NEXT:    [[TMP55:%.*]] = icmp eq i64 [[TMP50]], 2
+// CHECK-NEXT:    br i1 [[TMP55]], label [[OMP_TYPE_FROM19:%.*]], label [[OMP_TYPE_END20]]
+// CHECK:       omp.type.from19:
+// CHECK-NEXT:    [[TMP56:%.*]] = and i64 [[TMP49]], -2
+// CHECK-NEXT:    br label [[OMP_TYPE_END20]]
+// CHECK:       omp.type.end20:
+// CHECK-NEXT:    [[OMP_MAPTYPE21:%.*]] = phi i64 [ [[TMP52]], [[OMP_TYPE_ALLOC15]] ], [ [[TMP54]], [[OMP_TYPE_TO17]] ], [ [[TMP56]], [[OMP_TYPE_FROM19]] ], [ [[TMP49]], [[OMP_TYPE_TO_ELSE18]] ]
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[H]], i64 4, i64 [[OMP_MAPTYPE21]], ptr null)
+// CHECK-NEXT:    [[OMP_ARRAYMAP_NEXT]] = getelementptr [[STRUCT_D]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 1
+// CHECK-NEXT:    [[OMP_ARRAYMAP_ISDONE:%.*]] = icmp eq ptr [[OMP_ARRAYMAP_NEXT]], [[TMP7]]
+// CHECK-NEXT:    br i1 [[OMP_ARRAYMAP_ISDONE]], label [[OMP_ARRAYMAP_EXIT:%.*]], label [[OMP_ARRAYMAP_BODY]]
+// CHECK:       omp.arraymap.exit:
+// CHECK-NEXT:    [[OMP_ARRAYINIT_ISARRAY22:%.*]] = icmp sgt i64 [[TMP6]], 1
+// CHECK-NEXT:    [[TMP57:%.*]] = and i64 [[TMP4]], 8
+// CHECK-NEXT:    [[DOTOMP_ARRAY__DEL__DELETE:%.*]] = icmp ne i64 [[TMP57]], 0
+// CHECK-NEXT:    [[TMP58:%.*]] = and i1 [[OMP_ARRAYINIT_ISARRAY22]], [[DOTOMP_ARRAY__DEL__DELETE]]
+// CHECK-NEXT:    br i1 [[TMP58]], label [[DOTOMP_ARRAY__DEL:%.*]], label [[OMP_DONE]]
+// CHECK:       .omp.array..del:
+// CHECK-NEXT:    [[TMP59:%.*]] = mul nuw i64 [[TMP6]], 12
+// CHECK-NEXT:    [[TMP60:%.*]] = and i64 [[TMP4]], -4
+// CHECK-NEXT:    [[TMP61:%.*]] = or i64 [[TMP60]], 512
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP59]], i64 [[TMP61]], ptr [[TMP5]])
+// CHECK-NEXT:    br label [[OMP_DONE]]
+// CHECK:       omp.done:
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@.omp_mapper._ZTS1C.default
+// CHECK-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]], ptr noundef [[TMP2:%.*]], i64 noundef [[TMP3:%.*]], i64 noundef [[TMP4:%.*]], ptr noundef [[TMP5:%.*]]) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP6:%.*]] = udiv exact i64 [[TMP3]], 4
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr [[STRUCT_C:%.*]], ptr [[TMP2]], i64 [[TMP6]]
+// CHECK-NEXT:    [[OMP_ARRAYINIT_ISARRAY:%.*]] = icmp sgt i64 [[TMP6]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP4]], 8
+// CHECK-NEXT:    [[TMP9:%.*]] = icmp ne ptr [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP4]], 16
+// CHECK-NEXT:    [[TMP11:%.*]] = icmp ne i64 [[TMP10]], 0
+// CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[TMP9]], [[TMP11]]
+// CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[OMP_ARRAYINIT_ISARRAY]], [[TMP12]]
+// CHECK-NEXT:    [[DOTOMP_ARRAY__INIT__DELETE:%.*]] = icmp eq i64 [[TMP8]], 0
+// CHECK-NEXT:    [[TMP14:%.*]] = and i1 [[TMP13]], [[DOTOMP_ARRAY__INIT__DELETE]]
+// CHECK-NEXT:    br i1 [[TMP14]], label [[DOTOMP_ARRAY__INIT:%.*]], label [[OMP_ARRAYMAP_HEAD:%.*]]
+// CHECK:       .omp.array..init:
+// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP6]], 4
+// CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP4]], -4
+// CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP16]], 512
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP15]], i64 [[TMP17]], ptr [[TMP5]])
+// CHECK-NEXT:    br label [[OMP_ARRAYMAP_HEAD]]
+// CHECK:       omp.arraymap.head:
+// CHECK-NEXT:    [[OMP_ARRAYMAP_ISEMPTY:%.*]] = icmp eq ptr [[TMP2]], [[TMP7]]
+// CHECK-NEXT:    br i1 [[OMP_ARRAYMAP_ISEMPTY]], label [[OMP_DONE:%.*]], label [[OMP_ARRAYMAP_BODY:%.*]]
+// CHECK:       omp.arraymap.body:
+// CHECK-NEXT:    [[OMP_ARRAYMAP_PTRCURRENT:%.*]] = phi ptr [ [[TMP2]], [[OMP_ARRAYMAP_HEAD]] ], [ [[OMP_ARRAYMAP_NEXT:%.*]], [[OMP_TYPE_END:%.*]] ]
+// CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_C]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP18:%.*]] = call i64 @__tgt_mapper_num_components(ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP18]], 48
+// CHECK-NEXT:    [[TMP20:%.*]] = add nuw i64 1, [[TMP19]]
+// CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP4]], 3
+// CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 0
+// CHECK-NEXT:    br i1 [[TMP22]], label [[OMP_TYPE_ALLOC:%.*]], label [[OMP_TYPE_ALLOC_ELSE:%.*]]
+// CHECK:       omp.type.alloc:
+// CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP20]], -4
+// CHECK-NEXT:    br label [[OMP_TYPE_END]]
+// CHECK:       omp.type.alloc.else:
+// CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[TMP21]], 1
+// CHECK-NEXT:    br i1 [[TMP24]], label [[OMP_TYPE_TO:%.*]], label [[OMP_TYPE_TO_ELSE:%.*]]
+// CHECK:       omp.type.to:
+// CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP20]], -3
+// CHECK-NEXT:    br label [[OMP_TYPE_END]]
+// CHECK:       omp.type.to.else:
+// CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[TMP21]], 2
+// CHECK-NEXT:    br i1 [[TMP26]], label [[OMP_TYPE_FROM:%.*]], label [[OMP_TYPE_END]]
+// CHECK:       omp.type.from:
+// CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], -2
+// CHECK-NEXT:    br label [[OMP_TYPE_END]]
+// CHECK:       omp.type.end:
+// CHECK-NEXT:    [[OMP_MAPTYPE:%.*]] = phi i64 [ [[TMP23]], [[OMP_TYPE_ALLOC]] ], [ [[TMP25]], [[OMP_TYPE_TO]] ], [ [[TMP27]], [[OMP_TYPE_FROM]] ], [ [[TMP20]], [[OMP_TYPE_TO_ELSE]] ]
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[A]], i64 4, i64 [[OMP_MAPTYPE]], ptr null)
+// CHECK-NEXT:    [[OMP_ARRAYMAP_NEXT]] = getelementptr [[STRUCT_C]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 1
+// CHECK-NEXT:    [[OMP_ARRAYMAP_ISDONE:%.*]] = icmp eq ptr [[OMP_ARRAYMAP_NEXT]], [[TMP7]]
+// CHECK-NEXT:    br i1 [[OMP_ARRAYMAP_ISDONE]], label [[OMP_ARRAYMAP_EXIT:%.*]], label [[OMP_ARRAYMAP_BODY]]
+// CHECK:       omp.arraymap.exit:
+// CHECK-NEXT:    [[OMP_ARRAYINIT_ISARRAY1:%.*]] = icmp sgt i64 [[TMP6]], 1
+// CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP4]], 8
+// CHECK-NEXT:    [[DOTOMP_ARRAY__DEL__DELETE:%.*]] = icmp ne i64 [[TMP28]], 0
+// CHECK-NEXT:    [[TMP29:%.*]] = and i1 [[OMP_ARRAYINIT_ISARRAY1]], [[DOTOMP_ARRAY__DEL__DELETE]]
+// CHECK-NEXT:    br i1 [[TMP29]], label [[DOTOMP_ARRAY__DEL:%.*]], label [[OMP_DONE]]
+// CHECK:       .omp.array..del:
+// CHECK-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP6]], 4
+// CHECK-NEXT:    [[TMP31:%.*]] = and i64 [[TMP4]], -4
+// CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP31]], 512
+// CHECK-NEXT:    call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP30]], i64 [[TMP32]], ptr [[TMP5]])
+// CHECK-NEXT:    br label [[OMP_DONE]]
+// CHECK:       omp.done:
+// CHECK-NEXT:    ret void
+//
diff --git a/external/llvm-project/clang/test/OpenMP/target_map_nest_defalut_mapper_ast_dump.cpp b/external/llvm-project/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_ast_dump.cpp
similarity index 100%
rename from external/llvm-project/clang/test/OpenMP/target_map_nest_defalut_mapper_ast_dump.cpp
rename to external/llvm-project/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_ast_dump.cpp
diff --git a/external/llvm-project/clang/test/OpenMP/target_map_nest_defalut_mapper_codegen.cpp b/external/llvm-project/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_codegen.cpp
similarity index 100%
rename from external/llvm-project/clang/test/OpenMP/target_map_nest_defalut_mapper_codegen.cpp
rename to external/llvm-project/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_codegen.cpp
diff --git a/external/llvm-project/clang/test/OpenMP/target_update_messages.cpp b/external/llvm-project/clang/test/OpenMP/target_update_messages.cpp
index 83191059202c..000cc80e513e 100644
--- a/external/llvm-project/clang/test/OpenMP/target_update_messages.cpp
+++ b/external/llvm-project/clang/test/OpenMP/target_update_messages.cpp
@@ -113,9 +113,11 @@ int main(int argc, char **argv) {
   // Check parsing with two modifiers.
   // lt51-warning@+1 {{missing ':' after ) - ignoring}}
   #pragma omp target update to(mapper(id), present: s)
-  // lt51-error@+3 {{use of undeclared identifier 'present'}}
-  // lt51-error@+2 {{use of undeclared identifier 'id'}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  // lt51-error@+5 {{use of undeclared identifier 'present'}}
+  // lt51-error@+4 {{use of undeclared identifier 'id'}}
+  // lt51-error@+3 {{expected ',' or ')' in 'to' clause}}
+  // lt51-error@+2 {{expected ')'}}
+  // lt51-note@+1 {{to match this '('}}
   #pragma omp target update to(present, mapper(id): s)
   // lt51-warning@+1 {{missing ':' after ) - ignoring}}
   #pragma omp target update to(mapper(id) present: s)
@@ -141,10 +143,9 @@ int main(int argc, char **argv) {
   #pragma omp target update to(present,,: s)
   // lt51-warning@+1 {{missing ':' after ) - ignoring}}
   #pragma omp target update to(mapper(id), present,: s)
-  // lt51-error@+4 {{use of undeclared identifier 'present'}}
-  // lt51-error@+3 {{use of undeclared identifier 'id'}}
-  // lt51-error@+2 {{expected expression}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  // lt51-error@+3 {{use of undeclared identifier 'present'}}
+  // lt51-error@+2 {{use of undeclared identifier 'id'}}
+  // lt51-error@+1 {{expected expression}}
   #pragma omp target update to(present, mapper(id),: s)
 
   #pragma omp target update from(m) allocate(m) // expected-error {{unexpected OpenMP clause 'allocate' in directive '#pragma omp target update'}}
diff --git a/external/llvm-project/clang/test/OpenMP/xteam_red_debug_info.c b/external/llvm-project/clang/test/OpenMP/xteam_red_debug_info.c
new file mode 100644
index 000000000000..68223177d57c
--- /dev/null
+++ b/external/llvm-project/clang/test/OpenMP/xteam_red_debug_info.c
@@ -0,0 +1,16 @@
+// RUN: %clang -g %s -fopenmp --offload-arch=gfx90a -S --offload-host-only -emit-llvm -o - | FileCheck %s
+
+void test_xteam_red_debug_info() {
+  int N = 100000;
+  double c[N];
+  double sum = 0.0;
+  #pragma omp target teams distribute parallel for reduction(+: sum)
+  for (int i=0; i<N; i++){
+    sum += c[i];
+  }
+  sum = sum/(double)N;
+}
+
+// CHECK:       @.offload_sizes = private unnamed_addr constant [6 x i64]
+// CHECK-NEXT:  @.offload_maptypes = private unnamed_addr constant [6 x i64]
+// CHECK-NEXT:  @.offload_mapnames = private constant [6 x ptr]
\ No newline at end of file
diff --git a/external/llvm-project/clang/test/PCH/arc.m b/external/llvm-project/clang/test/PCH/arc.m
index e4ad71a469b9..e98dcedea85e 100644
--- a/external/llvm-project/clang/test/PCH/arc.m
+++ b/external/llvm-project/clang/test/PCH/arc.m
@@ -1,5 +1,6 @@
 // REQUIRES: x86-registered-target
 // Test this without pch.
+
 // RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin11 -fobjc-arc -include %S/Inputs/arc.h -emit-llvm-only %s
 
 // Test with pch.
@@ -14,5 +15,5 @@
 array0 a0;
 array1 a1;
 
-// CHECK-ERR1: Objective-C automated reference counting was enabled in AST file '{{.*}}' but is currently disabled
-// CHECK-ERR2: Objective-C automated reference counting was disabled in AST file '{{.*}}' but is currently enabled
+// CHECK-ERR1: Objective-C automated reference counting was enabled in precompiled file '{{.*}}' but is currently disabled
+// CHECK-ERR2: Objective-C automated reference counting was disabled in precompiled file '{{.*}}' but is currently enabled
diff --git a/external/llvm-project/clang/test/PCH/fuzzy-pch.c b/external/llvm-project/clang/test/PCH/fuzzy-pch.c
index 53985866dc08..fc90a398b958 100644
--- a/external/llvm-project/clang/test/PCH/fuzzy-pch.c
+++ b/external/llvm-project/clang/test/PCH/fuzzy-pch.c
@@ -24,8 +24,8 @@ BAR bar = 17;
 #  error BAR was not defined
 #endif
 
-// CHECK-FOO: definition of macro 'FOO' differs between the AST file '{{.*}}' ('1') and the command line ('blah')
-// CHECK-NOFOO: macro 'FOO' was defined in the AST file '{{.*}}' but undef'd on the command line
+// CHECK-FOO: definition of macro 'FOO' differs between the precompiled file '{{.*}}' ('1') and the command line ('blah')
+// CHECK-NOFOO: macro 'FOO' was defined in the precompiled file '{{.*}}' but undef'd on the command line
 
-// CHECK-UNDEF: command line contains '-undef' but AST file '{{.*}}' was not built with it
+// CHECK-UNDEF: command line contains '-undef' but precompiled file '{{.*}}' was not built with it
 
diff --git a/external/llvm-project/clang/test/PCH/modified-module-dependency.m b/external/llvm-project/clang/test/PCH/modified-module-dependency.m
index 44c14b3a0231..a4710dea5116 100644
--- a/external/llvm-project/clang/test/PCH/modified-module-dependency.m
+++ b/external/llvm-project/clang/test/PCH/modified-module-dependency.m
@@ -17,4 +17,4 @@
 // CHECK: file '[[TEST_H:.*[/\\]test\.h]]' has been modified since the precompiled header '[[PREFIX_PCH:.*/prefix\.pch]]' was built
 // CHECK: '[[TEST_H]]' required by '[[TEST_PCM:.*[/\\]test\.pcm]]'
 // CHECK: '[[TEST_PCM]]' required by '[[PREFIX_PCH]]'
-// CHECK: please rebuild precompiled header '[[PREFIX_PCH]]'
+// CHECK: please rebuild precompiled file '[[PREFIX_PCH]]'
diff --git a/external/llvm-project/clang/test/PCH/module-hash-difference.m b/external/llvm-project/clang/test/PCH/module-hash-difference.m
index 73cf536f88b4..a48a15398903 100644
--- a/external/llvm-project/clang/test/PCH/module-hash-difference.m
+++ b/external/llvm-project/clang/test/PCH/module-hash-difference.m
@@ -4,5 +4,5 @@
 // RUN: not %clang_cc1 -fsyntax-only -include-pch %t.pch %s -I %S/Inputs/modules -fmodules -fimplicit-module-maps -fmodules-cache-path=%t.mcp -fdisable-module-hash 2> %t.err
 // RUN: FileCheck -input-file=%t.err %s
 
-// CHECK: error: AST file '{{.*}}' was compiled with module cache path {{.*}}, but the path is currently {{.*}}
+// CHECK: error: precompiled file '{{.*}}' was compiled with module cache path {{.*}}, but the path is currently {{.*}}
 @import Foo;
diff --git a/external/llvm-project/clang/test/PCH/ms-pch-macro.c b/external/llvm-project/clang/test/PCH/ms-pch-macro.c
index 4d4900cc4f90..c34732bf5c2f 100644
--- a/external/llvm-project/clang/test/PCH/ms-pch-macro.c
+++ b/external/llvm-project/clang/test/PCH/ms-pch-macro.c
@@ -33,7 +33,7 @@ BAR bar = 17;
 #  error BAR was not defined
 #endif
 
-// CHECK-FOO: definition of macro 'FOO' differs between the AST file '{{.*}}1.pch' ('1') and the command line ('blah')
-// CHECK-NOFOO: macro 'FOO' was defined in the AST file '{{.*}}1.pch' but undef'd on the command line
+// CHECK-FOO: definition of macro 'FOO' differs between the precompiled file '{{.*}}1.pch' ('1') and the command line ('blah')
+// CHECK-NOFOO: macro 'FOO' was defined in the precompiled file '{{.*}}1.pch' but undef'd on the command line
 
 // expected-warning@2 {{definition of macro 'BAR' does not match definition in precompiled header}}
diff --git a/external/llvm-project/clang/test/PCH/no-validate-pch.cl b/external/llvm-project/clang/test/PCH/no-validate-pch.cl
index aa228ee20521..f6a7c65461fe 100644
--- a/external/llvm-project/clang/test/PCH/no-validate-pch.cl
+++ b/external/llvm-project/clang/test/PCH/no-validate-pch.cl
@@ -16,8 +16,8 @@
 // CHECK: note: previous definition is here
 // CHECK: #define X 4
 
-// CHECK-VAL: error: __OPTIMIZE__ predefined macro was enabled in AST file '{{.*}}' but is currently disabled
-// CHECK-VAL: error: definition of macro 'X' differs between the AST file '{{.*}}' ('4') and the command line ('5')
+// CHECK-VAL: error: __OPTIMIZE__ predefined macro was enabled in precompiled file '{{.*}}' but is currently disabled
+// CHECK-VAL: error: definition of macro 'X' differs between the precompiled file '{{.*}}' ('4') and the command line ('5')
 
 void test(void) {
   int a = ONE;
diff --git a/external/llvm-project/clang/test/PCH/validate-file-content.m b/external/llvm-project/clang/test/PCH/validate-file-content.m
index aba4944a28e9..b98979341b76 100644
--- a/external/llvm-project/clang/test/PCH/validate-file-content.m
+++ b/external/llvm-project/clang/test/PCH/validate-file-content.m
@@ -25,5 +25,5 @@
 // RUN: FileCheck %s < %t/stderr
 //
 // CHECK: file '[[M_H:.*[/\\]m\.h]]' has been modified since the precompiled header '[[A_PCH:.*/a\.pch]]' was built: content changed
-// CHECK: please rebuild precompiled header '[[A_PCH]]'
+// CHECK: please rebuild precompiled file '[[A_PCH]]'
 // expected-no-diagnostics
diff --git a/external/llvm-project/clang/test/Parser/cxx-invalid-using-decl-in-constexpr-crash.cpp b/external/llvm-project/clang/test/Parser/cxx-invalid-using-decl-in-constexpr-crash.cpp
new file mode 100644
index 000000000000..94fa8c8c820a
--- /dev/null
+++ b/external/llvm-project/clang/test/Parser/cxx-invalid-using-decl-in-constexpr-crash.cpp
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+// issue144264
+constexpr void test() 
+{ 
+    using TT = struct T[; 
+    // expected-error@-1 {{expected expression}}
+}
diff --git a/external/llvm-project/clang/test/Parser/cxx11-base-spec-attributes.cpp b/external/llvm-project/clang/test/Parser/cxx11-base-spec-attributes.cpp
index 7338c5116c16..6f2f54ead62b 100644
--- a/external/llvm-project/clang/test/Parser/cxx11-base-spec-attributes.cpp
+++ b/external/llvm-project/clang/test/Parser/cxx11-base-spec-attributes.cpp
@@ -7,4 +7,4 @@ struct D : [[]] public virtual A {};
 struct E : public [[]] virtual A {}; // expected-error {{an attribute list cannot appear here}}
 struct F : virtual [[]] public A {}; // expected-error {{an attribute list cannot appear here}}
 struct G : [[noreturn]] A {}; // expected-error {{'noreturn' attribute cannot be applied to a base specifier}}
-struct H : [[unknown::foobar]] A {}; // expected-warning {{unknown attribute 'foobar' ignored}}
+struct H : [[unknown::foobar]] A {}; // expected-warning {{unknown attribute 'unknown::foobar' ignored}}
diff --git a/external/llvm-project/clang/test/Parser/cxx1z-decomposition.cpp b/external/llvm-project/clang/test/Parser/cxx1z-decomposition.cpp
index 3e2526979be8..b7a8d30bd16c 100644
--- a/external/llvm-project/clang/test/Parser/cxx1z-decomposition.cpp
+++ b/external/llvm-project/clang/test/Parser/cxx1z-decomposition.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-unknown-linux-gnu -verify=expected,cxx2c,post2b -fcxx-exceptions
 // RUN: not %clang_cc1 -std=c++17 %s -triple x86_64-unknown-linux-gnu -emit-llvm-only -fcxx-exceptions
 
-struct S { int a, b, c; };
+struct S { int a, b, c; }; // expected-note 2 {{'S::a' declared here}}
 
 // A simple-declaration can be a decompsition declaration.
 namespace SimpleDecl {
@@ -32,7 +32,7 @@ namespace ForRangeDecl {
 namespace OtherDecl {
   // A parameter-declaration is not a simple-declaration.
   // This parses as an array declaration.
-  void f(auto [a, b, c]); // cxx17-error {{'auto' not allowed in function prototype}} expected-error {{'a'}}
+  void f(auto [a, b, c]); // cxx17-error {{'auto' not allowed in function prototype}} expected-error 1+{{'a'}}
 
   void g() {
     // A condition is allowed as a Clang extension.
@@ -46,7 +46,7 @@ namespace OtherDecl {
 
     // An exception-declaration is not a simple-declaration.
     try {}
-    catch (auto [a, b, c]) {} // expected-error {{'auto' not allowed in exception declaration}} expected-error {{'a'}}
+    catch (auto [a, b, c]) {} // expected-error {{'auto' not allowed in exception declaration}} expected-error 1+{{'a'}}
   }
 
   // A member-declaration is not a simple-declaration.
diff --git a/external/llvm-project/clang/test/Parser/cxx1z-fold-expressions.cpp b/external/llvm-project/clang/test/Parser/cxx1z-fold-expressions.cpp
index 4a329646b799..d798a9cbb99b 100644
--- a/external/llvm-project/clang/test/Parser/cxx1z-fold-expressions.cpp
+++ b/external/llvm-project/clang/test/Parser/cxx1z-fold-expressions.cpp
@@ -37,14 +37,14 @@ template<int ...N> int bad12() { return (... N); } // expected-error {{expected
 
 template<typename ...T> void as_operand_of_cast(int a, T ...t) {
   return
-    (int)(a + ... + undeclared_junk) + // expected-error {{undeclared}} expected-error {{does not contain any unexpanded}}
+    (int)(a + ... + undeclared_junk) + // expected-error {{undeclared}}
     (int)(t + ... + undeclared_junk) + // expected-error {{undeclared}}
-    (int)(... + undeclared_junk) + // expected-error {{undeclared}} expected-error {{does not contain any unexpanded}}
+    (int)(... + undeclared_junk) + // expected-error {{undeclared}}
     (int)(undeclared_junk + ...) + // expected-error {{undeclared}}
     (int)(a + ...) + // expected-error {{does not contain any unexpanded}}
     (int)(a, ...) + // expected-error {{does not contain any unexpanded}}
     (int)(..., a) + // expected-error {{does not contain any unexpanded}}
-    (int)(a, ..., undeclared_junk) + // expected-error {{undeclared}} expected-error {{does not contain any unexpanded}}
+    (int)(a, ..., undeclared_junk) + // expected-error {{undeclared}}
     (int)(t, ...) +
     (int)(..., t) +
     (int)(t, ..., a);
diff --git a/external/llvm-project/clang/test/Parser/cxx2c-pack-indexing.cpp b/external/llvm-project/clang/test/Parser/cxx2c-pack-indexing.cpp
index 72e286322fa9..79069a86ea70 100644
--- a/external/llvm-project/clang/test/Parser/cxx2c-pack-indexing.cpp
+++ b/external/llvm-project/clang/test/Parser/cxx2c-pack-indexing.cpp
@@ -69,7 +69,8 @@ template <typename... T>
 requires( ); // expected-error {{expected expression}}
 struct SS {
     void f( ) {
-        (*p).~T...[](); // expected-error {{use of undeclared identifier 'p'}}
+        (*p).~T...[](); // expected-error {{use of undeclared identifier 'p'}} \
+                           expected-error {{undeclared identifier 'T' in destructor name}}
     }
 };
 }
diff --git a/external/llvm-project/clang/test/Parser/macro-expansion-recovery.cpp b/external/llvm-project/clang/test/Parser/macro-expansion-recovery.cpp
new file mode 100644
index 000000000000..6826cc04e4df
--- /dev/null
+++ b/external/llvm-project/clang/test/Parser/macro-expansion-recovery.cpp
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+namespace GH143216 {
+#define A x y
+enum { A }; // expected-error {{missing ',' between enumerators}}
+
+#define B x y
+void f() {
+    int a[2];
+    auto [B] = a; // expected-error {{expected ','}}
+}
+
+#define C <int!
+template <class T> class D;
+D C; // expected-error {{expected unqualified-id}} \
+     // expected-error {{expected '>'}} \
+     // expected-note {{to match this '<'}}
+
+#define E F::{
+class F { E }}; // expected-error {{expected identifier}} \
+                // expected-error {{expected member name or ';' after declaration specifiers}}
+}
diff --git a/external/llvm-project/clang/test/Parser/objc-foreach-syntax.m b/external/llvm-project/clang/test/Parser/objc-foreach-syntax.m
index 2158d8062f6c..1ff84f393b9f 100644
--- a/external/llvm-project/clang/test/Parser/objc-foreach-syntax.m
+++ b/external/llvm-project/clang/test/Parser/objc-foreach-syntax.m
@@ -21,6 +21,5 @@ - (void)compilerTestAgainst {
 
 
 static int test7(id keys) {
-  for (id key; in keys) ;  // expected-error {{use of undeclared identifier 'in'}} \
-                           // expected-error {{expected ';' in 'for' statement specifier}}
+  for (id key; in keys) ;  // expected-error {{use of undeclared identifier 'in'}}
 }
diff --git a/external/llvm-project/clang/test/Parser/objcxx11-attributes.mm b/external/llvm-project/clang/test/Parser/objcxx11-attributes.mm
index d7ba609ebd74..88fa3103593e 100644
--- a/external/llvm-project/clang/test/Parser/objcxx11-attributes.mm
+++ b/external/llvm-project/clang/test/Parser/objcxx11-attributes.mm
@@ -57,7 +57,7 @@ void f(X *noreturn) {
 
 template<typename...Ts> void f(Ts ...x) {
   [[test::foo(bar, baz)...]]; // expected-error {{attribute 'foo' cannot be used as an attribute pack}} \
-  // expected-warning {{unknown attribute 'foo' ignored}}
+  // expected-warning {{unknown attribute 'test::foo' ignored}}
 
   [[used(x)...]]; // expected-error {{attribute 'used' cannot be used as an attribute pack}} \
   // expected-warning {{unknown attribute 'used' ignored}}
diff --git a/external/llvm-project/clang/test/Parser/opencl-atomics-cl20.cl b/external/llvm-project/clang/test/Parser/opencl-atomics-cl20.cl
index 2648142f28e7..2cd2c6ca133e 100644
--- a/external/llvm-project/clang/test/Parser/opencl-atomics-cl20.cl
+++ b/external/llvm-project/clang/test/Parser/opencl-atomics-cl20.cl
@@ -39,23 +39,17 @@ void atomic_types_test(void) {
 // expected-error@-11 {{use of undeclared identifier 'atomic_ulong'}}
 // expected-error@-11 {{use of undeclared identifier 'atomic_double'}}
 #if defined(LANG_VER_OK)
-// expected-error@-15 {{expected ';' after expression}}
-// expected-error@-16 {{use of undeclared identifier 'l'}}
-// expected-error@-16 {{expected ';' after expression}}
-// expected-error@-17 {{use of undeclared identifier 'ul'}}
 #endif
 #if !defined(LANG_VER_OK) || defined(__SPIR64__)
-// expected-error@-18 {{use of undeclared identifier 'atomic_size_t'}}
-// expected-error@-16 {{use of undeclared identifier 'atomic_ptrdiff_t'}}
+// expected-error@-14 {{use of undeclared identifier 'atomic_size_t'}}
+// expected-error@-12 {{use of undeclared identifier 'atomic_ptrdiff_t'}}
 #if !defined(LANG_VER_OK)
-// expected-error@-20 {{use of undeclared identifier 'atomic_intptr_t'}}
-// expected-error@-20 {{use of undeclared identifier 'atomic_uintptr_t'}}
+// expected-error@-16 {{use of undeclared identifier 'atomic_intptr_t'}}
+// expected-error@-16 {{use of undeclared identifier 'atomic_uintptr_t'}}
 #else
-// expected-error@-24 {{expected ';' after expression}}
-// expected-error@-25 {{use of undeclared identifier 's'}}
-// expected-error@-25 {{unknown type name 'atomic_intptr_t'; did you mean 'atomic_int'?}}
+// expected-error@-19 {{unknown type name 'atomic_intptr_t'; did you mean 'atomic_int'?}}
 // expected-note@* {{'atomic_int' declared here}}
-// expected-error@-26 {{unknown type name 'atomic_uintptr_t'; did you mean 'atomic_uint'?}}
+// expected-error@-20 {{unknown type name 'atomic_uintptr_t'; did you mean 'atomic_uint'?}}
 // expected-note@* {{'atomic_uint' declared here}}
 #endif
 #endif
diff --git a/external/llvm-project/clang/test/Parser/recovery.c b/external/llvm-project/clang/test/Parser/recovery.c
index 6fdbedffd236..0d86bd0608bf 100644
--- a/external/llvm-project/clang/test/Parser/recovery.c
+++ b/external/llvm-project/clang/test/Parser/recovery.c
@@ -11,7 +11,7 @@ float test2241[2] = {
 static void f (char * (*g) (char **, int), char **p, ...) {
   char *s;
   va_list v;                              // expected-error {{identifier}}
-  s = g (p, __builtin_va_arg(v, int));    // expected-error {{identifier}}
+  s = g (p, __builtin_va_arg(v, int));    // expected-error {{identifier}} expected-error {{extraneous ')' before ';'}}
 }
 
 
diff --git a/external/llvm-project/clang/test/Parser/switch-recovery.cpp b/external/llvm-project/clang/test/Parser/switch-recovery.cpp
index baf703cd03ae..40712799933c 100644
--- a/external/llvm-project/clang/test/Parser/switch-recovery.cpp
+++ b/external/llvm-project/clang/test/Parser/switch-recovery.cpp
@@ -104,7 +104,7 @@ void test9(int x) { // expected-note {{'x' declared here}}
               expected-error {{expected expression}}
     8:: x; // expected-error {{expected ';' after expression}} \
               expected-error {{no member named 'x' in the global namespace; did you mean simply 'x'?}} \
-              expected-warning {{expression result unused}}
+              expected-warning 2 {{expression result unused}}
     9:: :y; // expected-error {{expected ';' after expression}} \
                expected-error {{expected unqualified-id}} \
                expected-warning {{expression result unused}}
@@ -229,3 +229,16 @@ void fn1() {
     }
 } // expected-error{{expected statement}}
 }
+
+namespace GH143216 {
+#define FOO 1 case 3:
+
+int f(int x) {
+  switch (x) {
+  case FOO // expected-error {{expected ':' after 'case'}}
+    return 0;
+  default:
+    return 1;
+  }
+}
+}
diff --git a/external/llvm-project/clang/test/Parser/switch-typo-correction.cpp b/external/llvm-project/clang/test/Parser/switch-typo-correction.cpp
index ebf1c18f2b86..95d610b9cdd2 100644
--- a/external/llvm-project/clang/test/Parser/switch-typo-correction.cpp
+++ b/external/llvm-project/clang/test/Parser/switch-typo-correction.cpp
@@ -1,9 +1,9 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 
-namespace c { double xxx; } // expected-note{{'c::xxx' declared here}}
+namespace c { double xxx; }
 namespace d { float xxx; }
 namespace z { namespace xxx {} }
 
 void crash() {
-  switch (xxx) {} // expected-error{{use of undeclared identifier 'xxx'; did you mean }}
+  switch (xxx) {} // expected-error{{use of undeclared identifier 'xxx'}}
 }
diff --git a/external/llvm-project/clang/test/ParserOpenACC/parse-cache-construct.cpp b/external/llvm-project/clang/test/ParserOpenACC/parse-cache-construct.cpp
index a5a1e58028c3..948f2e30f149 100644
--- a/external/llvm-project/clang/test/ParserOpenACC/parse-cache-construct.cpp
+++ b/external/llvm-project/clang/test/ParserOpenACC/parse-cache-construct.cpp
@@ -1,8 +1,8 @@
 // RUN: %clang_cc1 %s -verify -fopenacc
 
 namespace NS {
-  static char* NSArray;// expected-note{{declared here}}
-  static int NSInt;// expected-note 2{{declared here}}
+  static char* NSArray; // expected-note {{'NS::NSArray' declared here}}
+  static int NSInt;     // expected-note 2 {{'NS::NSInt' declared here}}
 }
 char *getArrayPtr();
 template<typename T, int I>
@@ -21,17 +21,17 @@ void func() {
   }
 
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{use of undeclared identifier 'NSArray'; did you mean 'NS::NSArray'}}
+    // expected-error@+1{{use of undeclared identifier 'NSArray'}}
     #pragma acc cache(NSArray[NS::NSInt : NS::NSInt])
   }
 
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{use of undeclared identifier 'NSInt'; did you mean 'NS::NSInt'}}
+    // expected-error@+1{{use of undeclared identifier 'NSInt'}}
     #pragma acc cache(NS::NSArray[NSInt : NS::NSInt])
   }
 
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{use of undeclared identifier 'NSInt'; did you mean 'NS::NSInt'}}
+    // expected-error@+1{{use of undeclared identifier 'NSInt'}}
     #pragma acc cache(NS::NSArray[NS::NSInt : NSInt])
   }
 }
diff --git a/external/llvm-project/clang/test/ParserOpenACC/parse-clauses.c b/external/llvm-project/clang/test/ParserOpenACC/parse-clauses.c
index 6d771e858d24..a9ad7ab176cb 100644
--- a/external/llvm-project/clang/test/ParserOpenACC/parse-clauses.c
+++ b/external/llvm-project/clang/test/ParserOpenACC/parse-clauses.c
@@ -347,9 +347,7 @@ void SelfUpdate() {
 #pragma acc update host(s) self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+3{{use of undeclared identifier 'zero'}}
-  // expected-error@+2{{expected ','}}
-  // expected-error@+1{{expected expression}}
+  // expected-error@+1{{use of undeclared identifier 'zero'}}
 #pragma acc update self(zero : s.array[s.value : 5], s.value), if_present
   for(int i = 0; i < 5;++i) {}
 
@@ -453,8 +451,6 @@ void VarListClauses() {
 #pragma acc parallel copy(always, alwaysin, always: HasMem.MemArr[3:]) self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+3{{use of undeclared identifier 'always'}}
-  // expected-error@+2{{use of undeclared identifier 'alwaysin'}}
   // expected-error@+1{{use of undeclared identifier 'always'}}
 #pragma acc parallel copy(always, alwaysin, always, HasMem.MemArr[3:]) self
   for(int i = 0; i < 5;++i) {}
@@ -591,8 +587,7 @@ void VarListClauses() {
 #pragma acc serial copyout(zero : s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+2{{use of undeclared identifier 'zero'}}
-  // expected-error@+1{{expected ','}}
+  // expected-error@+1{{use of undeclared identifier 'zero'}}
 #pragma acc serial copyout(zero s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
@@ -608,8 +603,7 @@ void VarListClauses() {
 #pragma acc serial copyout(invalid:s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+2{{use of undeclared identifier 'invalid'}}
-  // expected-error@+1{{expected ','}}
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
 #pragma acc serial copyout(invalid s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
@@ -657,8 +651,7 @@ void VarListClauses() {
 #pragma acc serial create(zero : s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+2{{use of undeclared identifier 'zero'}}
-  // expected-error@+1{{expected ','}}
+  // expected-error@+1{{use of undeclared identifier 'zero'}}
 #pragma acc serial create(zero s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
@@ -674,8 +667,7 @@ void VarListClauses() {
 #pragma acc serial create(invalid:s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+2{{use of undeclared identifier 'invalid'}}
-  // expected-error@+1{{expected ','}}
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
 #pragma acc serial create(invalid s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
@@ -700,8 +692,7 @@ void VarListClauses() {
 #pragma acc serial copyin(readonly : s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+2{{use of undeclared identifier 'readonly'}}
-  // expected-error@+1{{expected ','}}
+  // expected-error@+1{{use of undeclared identifier 'readonly'}}
 #pragma acc serial copyin(readonly s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
@@ -717,8 +708,7 @@ void VarListClauses() {
 #pragma acc serial copyin(invalid:s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
-  // expected-error@+2{{use of undeclared identifier 'invalid'}}
-  // expected-error@+1{{expected ','}}
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
 #pragma acc serial copyin(invalid s.array[s.value : 5], s.value), self
   for(int i = 0; i < 5;++i) {}
 
diff --git a/external/llvm-project/clang/test/ParserOpenACC/parse-constructs.cpp b/external/llvm-project/clang/test/ParserOpenACC/parse-constructs.cpp
index 814f6a1fd09f..69b04bcbad9e 100644
--- a/external/llvm-project/clang/test/ParserOpenACC/parse-constructs.cpp
+++ b/external/llvm-project/clang/test/ParserOpenACC/parse-constructs.cpp
@@ -18,13 +18,13 @@ namespace NS {
 #pragma acc routine(NS::foo) seq
 
 // expected-error@+2{{use of undeclared identifier 'templ'; did you mean 'NS::templ'?}}
-// expected-error@+1{{OpenACC routine name 'NS::templ' names a set of overloads}}
+// expected-error@+1{{OpenACC routine name 'templ' names a set of overloads}}
 #pragma acc routine(templ) seq
 // expected-error@+1{{OpenACC routine name 'NS::templ' names a set of overloads}}
 #pragma acc routine(NS::templ) seq
 
 // expected-error@+2{{use of undeclared identifier 'templ'; did you mean 'NS::templ'?}}
-// expected-error@+1{{OpenACC routine name 'NS::templ' names a set of overloads}}
+// expected-error@+1{{OpenACC routine name 'templ<int>' names a set of overloads}}
 #pragma acc routine(templ<int>) seq
 // expected-error@+1{{OpenACC routine name 'NS::templ<int>' names a set of overloads}}
 #pragma acc routine(NS::templ<int>) seq
diff --git a/external/llvm-project/clang/test/ParserOpenACC/parse-wait-clause.c b/external/llvm-project/clang/test/ParserOpenACC/parse-wait-clause.c
index 16e31a67c094..5c006b4379a2 100644
--- a/external/llvm-project/clang/test/ParserOpenACC/parse-wait-clause.c
+++ b/external/llvm-project/clang/test/ParserOpenACC/parse-wait-clause.c
@@ -85,19 +85,16 @@ void func() {
   #pragma acc parallel wait (devnum: i + j:queues:) clause-list
     {}
 
-  // expected-error@+4{{use of undeclared identifier 'devnum'}}
-  // expected-error@+3{{expected ','}}
+  // expected-error@+3{{use of undeclared identifier 'devnum'}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait (queues:devnum: i + j
     {}
 
-  // expected-error@+2{{expected ','}}
   // expected-error@+1{{use of undeclared identifier 'devnum'}}
   #pragma acc parallel wait (queues:devnum: i + j)
     {}
 
-  // expected-error@+3{{expected ','}}
   // expected-error@+2{{use of undeclared identifier 'devnum'}}
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait (queues:devnum: i + j) clause-list
diff --git a/external/llvm-project/clang/test/ParserOpenACC/parse-wait-construct.c b/external/llvm-project/clang/test/ParserOpenACC/parse-wait-construct.c
index 491c3bee4ac5..27a3a02dc263 100644
--- a/external/llvm-project/clang/test/ParserOpenACC/parse-wait-construct.c
+++ b/external/llvm-project/clang/test/ParserOpenACC/parse-wait-construct.c
@@ -68,18 +68,15 @@ void func() {
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc wait (devnum: i + j:queues:) clause-list
 
-  // expected-error@+4{{use of undeclared identifier 'devnum'}}
-  // expected-error@+3{{expected ','}}
+  // expected-error@+3{{use of undeclared identifier 'devnum'}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc wait (queues:devnum: i + j
 
-  // expected-error@+2{{use of undeclared identifier 'devnum'}}
-  // expected-error@+1{{expected ','}}
+  // expected-error@+1{{use of undeclared identifier 'devnum'}}
   #pragma acc wait (queues:devnum: i + j)
 
-  // expected-error@+3{{use of undeclared identifier 'devnum'}}
-  // expected-error@+2{{expected ','}}
+  // expected-error@+2{{use of undeclared identifier 'devnum'}}
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc wait (queues:devnum: i + j) clause-list
 
diff --git a/external/llvm-project/clang/test/Preprocessor/aarch64-target-features.c b/external/llvm-project/clang/test/Preprocessor/aarch64-target-features.c
index 4cb9b6ce53b0..fd83e4b689a2 100644
--- a/external/llvm-project/clang/test/Preprocessor/aarch64-target-features.c
+++ b/external/llvm-project/clang/test/Preprocessor/aarch64-target-features.c
@@ -744,7 +744,10 @@
 // CHECK-SMEB16B16: __ARM_FEATURE_SME2 1
 // CHECK-SMEB16B16: __ARM_FEATURE_SME_B16B16 1
 // CHECK-SMEB16B16: __ARM_FEATURE_SVE_B16B16 1
-//
+
+// RUN: %clang --target=aarch64 -march=armv9-a+cssc -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-CSSC %s
+// CHECK-CSSC: __ARM_FEATURE_CSSC 1
+
 //  RUN: %clang --target=aarch64 -march=armv9-a+fp8 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-FP8 %s
 // CHECK-FP8: __ARM_FEATURE_FP8 1
 
diff --git a/external/llvm-project/clang/test/Preprocessor/init-loongarch.c b/external/llvm-project/clang/test/Preprocessor/init-loongarch.c
index ac461b371162..71a266b8a915 100644
--- a/external/llvm-project/clang/test/Preprocessor/init-loongarch.c
+++ b/external/llvm-project/clang/test/Preprocessor/init-loongarch.c
@@ -946,6 +946,10 @@
 // RUN:   | FileCheck --match-full-lines --check-prefix=MNO-LSX %s
 // RUN: %clang --target=loongarch64 -mno-lasx -mno-lsx -x c -E -dM %s -o - \
 // RUN:   | FileCheck --match-full-lines --check-prefix=MNO-LSX %s
+// RUN: %clang --target=loongarch64 -march=la464 -mno-lsx -x c -E -dM %s -o - \
+// RUN:   | FileCheck --match-full-lines --check-prefix=MNO-LSX %s
+// RUN: %clang --target=loongarch64 -mno-lsx -march=la464 -x c -E -dM %s -o - \
+// RUN:   | FileCheck --match-full-lines --check-prefix=MNO-LSX %s
 // MNO-LSX-NOT: #define __loongarch_asx
 // MNO-LSX-NOT: #define __loongarch_simd_width
 // MNO-LSX-NOT: #define __loongarch_sx
diff --git a/external/llvm-project/clang/test/Preprocessor/init-x86.c b/external/llvm-project/clang/test/Preprocessor/init-x86.c
index cb77b5583407..8ea4ce791510 100644
--- a/external/llvm-project/clang/test/Preprocessor/init-x86.c
+++ b/external/llvm-project/clang/test/Preprocessor/init-x86.c
@@ -1486,3 +1486,431 @@
 
 // RUN: %clang_cc1 -E -dM -triple=i386-unknown-openbsd -x c++ < /dev/null | FileCheck -match-full-lines -check-prefix I386-OPENBSD-CXX %s
 // I386-OPENBSD-CXX: #define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
+
+// RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=i386-pc-windows-cygnus -target-cpu pentium4 < /dev/null | FileCheck -match-full-lines -check-prefix I386-CYGWIN %s
+// I386-CYGWIN-NOT:#define _LP64
+// I386-CYGWIN:#define __BIGGEST_ALIGNMENT__ 16
+// I386-CYGWIN:#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+// I386-CYGWIN:#define __CHAR16_TYPE__ unsigned short
+// I386-CYGWIN:#define __CHAR32_TYPE__ unsigned int
+// I386-CYGWIN:#define __CHAR_BIT__ 8
+// I386-CYGWIN:#define __CYGWIN32__ 1
+// I386-CYGWIN:#define __CYGWIN__ 1
+// I386-CYGWIN:#define __DBL_DENORM_MIN__ 4.9406564584124654e-324
+// I386-CYGWIN:#define __DBL_DIG__ 15
+// I386-CYGWIN:#define __DBL_EPSILON__ 2.2204460492503131e-16
+// I386-CYGWIN:#define __DBL_HAS_DENORM__ 1
+// I386-CYGWIN:#define __DBL_HAS_INFINITY__ 1
+// I386-CYGWIN:#define __DBL_HAS_QUIET_NAN__ 1
+// I386-CYGWIN:#define __DBL_MANT_DIG__ 53
+// I386-CYGWIN:#define __DBL_MAX_10_EXP__ 308
+// I386-CYGWIN:#define __DBL_MAX_EXP__ 1024
+// I386-CYGWIN:#define __DBL_MAX__ 1.7976931348623157e+308
+// I386-CYGWIN:#define __DBL_MIN_10_EXP__ (-307)
+// I386-CYGWIN:#define __DBL_MIN_EXP__ (-1021)
+// I386-CYGWIN:#define __DBL_MIN__ 2.2250738585072014e-308
+// I386-CYGWIN:#define __DECIMAL_DIG__ __LDBL_DECIMAL_DIG__
+// I386-CYGWIN:#define __FLT_DENORM_MIN__ 1.40129846e-45F
+// I386-CYGWIN:#define __FLT_DIG__ 6
+// I386-CYGWIN:#define __FLT_EPSILON__ 1.19209290e-7F
+// I386-CYGWIN:#define __FLT_HAS_DENORM__ 1
+// I386-CYGWIN:#define __FLT_HAS_INFINITY__ 1
+// I386-CYGWIN:#define __FLT_HAS_QUIET_NAN__ 1
+// I386-CYGWIN:#define __FLT_MANT_DIG__ 24
+// I386-CYGWIN:#define __FLT_MAX_10_EXP__ 38
+// I386-CYGWIN:#define __FLT_MAX_EXP__ 128
+// I386-CYGWIN:#define __FLT_MAX__ 3.40282347e+38F
+// I386-CYGWIN:#define __FLT_MIN_10_EXP__ (-37)
+// I386-CYGWIN:#define __FLT_MIN_EXP__ (-125)
+// I386-CYGWIN:#define __FLT_MIN__ 1.17549435e-38F
+// I386-CYGWIN:#define __FLT_RADIX__ 2
+// I386-CYGWIN:#define __GCC_ATOMIC_BOOL_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_CHAR16_T_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_CHAR32_T_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_CHAR_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_INT_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_LLONG_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_LONG_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_POINTER_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_SHORT_LOCK_FREE 2
+// I386-CYGWIN:#define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
+// I386-CYGWIN:#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
+// I386-CYGWIN:#define __ILP32__ 1
+// I386-CYGWIN:#define __INT16_C(c) c
+// I386-CYGWIN:#define __INT16_C_SUFFIX__
+// I386-CYGWIN:#define __INT16_FMTd__ "hd"
+// I386-CYGWIN:#define __INT16_FMTi__ "hi"
+// I386-CYGWIN:#define __INT16_MAX__ 32767
+// I386-CYGWIN:#define __INT16_TYPE__ short
+// I386-CYGWIN:#define __INT32_C(c) c
+// I386-CYGWIN:#define __INT32_C_SUFFIX__
+// I386-CYGWIN:#define __INT32_FMTd__ "d"
+// I386-CYGWIN:#define __INT32_FMTi__ "i"
+// I386-CYGWIN:#define __INT32_MAX__ 2147483647
+// I386-CYGWIN:#define __INT32_TYPE__ int
+// I386-CYGWIN:#define __INT64_C(c) c##LL
+// I386-CYGWIN:#define __INT64_C_SUFFIX__ LL
+// I386-CYGWIN:#define __INT64_FMTd__ "lld"
+// I386-CYGWIN:#define __INT64_FMTi__ "lli"
+// I386-CYGWIN:#define __INT64_MAX__ 9223372036854775807LL
+// I386-CYGWIN:#define __INT64_TYPE__ long long int
+// I386-CYGWIN:#define __INT8_C(c) c
+// I386-CYGWIN:#define __INT8_C_SUFFIX__
+// I386-CYGWIN:#define __INT8_FMTd__ "hhd"
+// I386-CYGWIN:#define __INT8_FMTi__ "hhi"
+// I386-CYGWIN:#define __INT8_MAX__ 127
+// I386-CYGWIN:#define __INT8_TYPE__ signed char
+// I386-CYGWIN:#define __INTMAX_C(c) c##LL
+// I386-CYGWIN:#define __INTMAX_C_SUFFIX__ LL
+// I386-CYGWIN:#define __INTMAX_FMTd__ "lld"
+// I386-CYGWIN:#define __INTMAX_FMTi__ "lli"
+// I386-CYGWIN:#define __INTMAX_MAX__ 9223372036854775807LL
+// I386-CYGWIN:#define __INTMAX_TYPE__ long long int
+// I386-CYGWIN:#define __INTMAX_WIDTH__ 64
+// I386-CYGWIN:#define __INTPTR_FMTd__ "d"
+// I386-CYGWIN:#define __INTPTR_FMTi__ "i"
+// I386-CYGWIN:#define __INTPTR_MAX__ 2147483647
+// I386-CYGWIN:#define __INTPTR_TYPE__ int
+// I386-CYGWIN:#define __INTPTR_WIDTH__ 32
+// I386-CYGWIN:#define __INT_FAST16_FMTd__ "hd"
+// I386-CYGWIN:#define __INT_FAST16_FMTi__ "hi"
+// I386-CYGWIN:#define __INT_FAST16_MAX__ 32767
+// I386-CYGWIN:#define __INT_FAST16_TYPE__ short
+// I386-CYGWIN:#define __INT_FAST32_FMTd__ "d"
+// I386-CYGWIN:#define __INT_FAST32_FMTi__ "i"
+// I386-CYGWIN:#define __INT_FAST32_MAX__ 2147483647
+// I386-CYGWIN:#define __INT_FAST32_TYPE__ int
+// I386-CYGWIN:#define __INT_FAST64_FMTd__ "lld"
+// I386-CYGWIN:#define __INT_FAST64_FMTi__ "lli"
+// I386-CYGWIN:#define __INT_FAST64_MAX__ 9223372036854775807LL
+// I386-CYGWIN:#define __INT_FAST64_TYPE__ long long int
+// I386-CYGWIN:#define __INT_FAST8_FMTd__ "hhd"
+// I386-CYGWIN:#define __INT_FAST8_FMTi__ "hhi"
+// I386-CYGWIN:#define __INT_FAST8_MAX__ 127
+// I386-CYGWIN:#define __INT_FAST8_TYPE__ signed char
+// I386-CYGWIN:#define __INT_LEAST16_FMTd__ "hd"
+// I386-CYGWIN:#define __INT_LEAST16_FMTi__ "hi"
+// I386-CYGWIN:#define __INT_LEAST16_MAX__ 32767
+// I386-CYGWIN:#define __INT_LEAST16_TYPE__ short
+// I386-CYGWIN:#define __INT_LEAST32_FMTd__ "d"
+// I386-CYGWIN:#define __INT_LEAST32_FMTi__ "i"
+// I386-CYGWIN:#define __INT_LEAST32_MAX__ 2147483647
+// I386-CYGWIN:#define __INT_LEAST32_TYPE__ int
+// I386-CYGWIN:#define __INT_LEAST64_FMTd__ "lld"
+// I386-CYGWIN:#define __INT_LEAST64_FMTi__ "lli"
+// I386-CYGWIN:#define __INT_LEAST64_MAX__ 9223372036854775807LL
+// I386-CYGWIN:#define __INT_LEAST64_TYPE__ long long int
+// I386-CYGWIN:#define __INT_LEAST8_FMTd__ "hhd"
+// I386-CYGWIN:#define __INT_LEAST8_FMTi__ "hhi"
+// I386-CYGWIN:#define __INT_LEAST8_MAX__ 127
+// I386-CYGWIN:#define __INT_LEAST8_TYPE__ signed char
+// I386-CYGWIN:#define __INT_MAX__ 2147483647
+// I386-CYGWIN:#define __LDBL_DENORM_MIN__ 3.64519953188247460253e-4951L
+// I386-CYGWIN:#define __LDBL_DIG__ 18
+// I386-CYGWIN:#define __LDBL_EPSILON__ 1.08420217248550443401e-19L
+// I386-CYGWIN:#define __LDBL_HAS_DENORM__ 1
+// I386-CYGWIN:#define __LDBL_HAS_INFINITY__ 1
+// I386-CYGWIN:#define __LDBL_HAS_QUIET_NAN__ 1
+// I386-CYGWIN:#define __LDBL_MANT_DIG__ 64
+// I386-CYGWIN:#define __LDBL_MAX_10_EXP__ 4932
+// I386-CYGWIN:#define __LDBL_MAX_EXP__ 16384
+// I386-CYGWIN:#define __LDBL_MAX__ 1.18973149535723176502e+4932L
+// I386-CYGWIN:#define __LDBL_MIN_10_EXP__ (-4931)
+// I386-CYGWIN:#define __LDBL_MIN_EXP__ (-16381)
+// I386-CYGWIN:#define __LDBL_MIN__ 3.36210314311209350626e-4932L
+// I386-CYGWIN:#define __LITTLE_ENDIAN__ 1
+// I386-CYGWIN:#define __LONG_LONG_MAX__ 9223372036854775807LL
+// I386-CYGWIN:#define __LONG_MAX__ 2147483647L
+// I386-CYGWIN-NOT:#define __LP64__
+// I386-CYGWIN:#define __NO_MATH_INLINES 1
+// I386-CYGWIN:#define __POINTER_WIDTH__ 32
+// I386-CYGWIN:#define __PTRDIFF_TYPE__ int
+// I386-CYGWIN:#define __PTRDIFF_WIDTH__ 32
+// I386-CYGWIN:#define __REGISTER_PREFIX__
+// I386-CYGWIN:#define __SCHAR_MAX__ 127
+// I386-CYGWIN:#define __SHRT_MAX__ 32767
+// I386-CYGWIN:#define __SIG_ATOMIC_MAX__ 2147483647
+// I386-CYGWIN:#define __SIG_ATOMIC_WIDTH__ 32
+// I386-CYGWIN:#define __SIZEOF_DOUBLE__ 8
+// I386-CYGWIN:#define __SIZEOF_FLOAT__ 4
+// I386-CYGWIN:#define __SIZEOF_INT__ 4
+// I386-CYGWIN:#define __SIZEOF_LONG_DOUBLE__ 12
+// I386-CYGWIN:#define __SIZEOF_LONG_LONG__ 8
+// I386-CYGWIN:#define __SIZEOF_LONG__ 4
+// I386-CYGWIN:#define __SIZEOF_POINTER__ 4
+// I386-CYGWIN:#define __SIZEOF_PTRDIFF_T__ 4
+// I386-CYGWIN:#define __SIZEOF_SHORT__ 2
+// I386-CYGWIN:#define __SIZEOF_SIZE_T__ 4
+// I386-CYGWIN:#define __SIZEOF_WCHAR_T__ 2
+// I386-CYGWIN:#define __SIZEOF_WINT_T__ 4
+// I386-CYGWIN:#define __SIZE_MAX__ 4294967295U
+// I386-CYGWIN:#define __SIZE_TYPE__ unsigned int
+// I386-CYGWIN:#define __SIZE_WIDTH__ 32
+// I386-CYGWIN:#define __UINT16_C(c) c
+// I386-CYGWIN:#define __UINT16_C_SUFFIX__
+// I386-CYGWIN:#define __UINT16_MAX__ 65535
+// I386-CYGWIN:#define __UINT16_TYPE__ unsigned short
+// I386-CYGWIN:#define __UINT32_C(c) c##U
+// I386-CYGWIN:#define __UINT32_C_SUFFIX__ U
+// I386-CYGWIN:#define __UINT32_MAX__ 4294967295U
+// I386-CYGWIN:#define __UINT32_TYPE__ unsigned int
+// I386-CYGWIN:#define __UINT64_C(c) c##ULL
+// I386-CYGWIN:#define __UINT64_C_SUFFIX__ ULL
+// I386-CYGWIN:#define __UINT64_MAX__ 18446744073709551615ULL
+// I386-CYGWIN:#define __UINT64_TYPE__ long long unsigned int
+// I386-CYGWIN:#define __UINT8_C(c) c
+// I386-CYGWIN:#define __UINT8_C_SUFFIX__
+// I386-CYGWIN:#define __UINT8_MAX__ 255
+// I386-CYGWIN:#define __UINT8_TYPE__ unsigned char
+// I386-CYGWIN:#define __UINTMAX_C(c) c##ULL
+// I386-CYGWIN:#define __UINTMAX_C_SUFFIX__ ULL
+// I386-CYGWIN:#define __UINTMAX_MAX__ 18446744073709551615ULL
+// I386-CYGWIN:#define __UINTMAX_TYPE__ long long unsigned int
+// I386-CYGWIN:#define __UINTMAX_WIDTH__ 64
+// I386-CYGWIN:#define __UINTPTR_MAX__ 4294967295U
+// I386-CYGWIN:#define __UINTPTR_TYPE__ unsigned int
+// I386-CYGWIN:#define __UINTPTR_WIDTH__ 32
+// I386-CYGWIN:#define __UINT_FAST16_MAX__ 65535
+// I386-CYGWIN:#define __UINT_FAST16_TYPE__ unsigned short
+// I386-CYGWIN:#define __UINT_FAST32_MAX__ 4294967295U
+// I386-CYGWIN:#define __UINT_FAST32_TYPE__ unsigned int
+// I386-CYGWIN:#define __UINT_FAST64_MAX__ 18446744073709551615ULL
+// I386-CYGWIN:#define __UINT_FAST64_TYPE__ long long unsigned int
+// I386-CYGWIN:#define __UINT_FAST8_MAX__ 255
+// I386-CYGWIN:#define __UINT_FAST8_TYPE__ unsigned char
+// I386-CYGWIN:#define __UINT_LEAST16_MAX__ 65535
+// I386-CYGWIN:#define __UINT_LEAST16_TYPE__ unsigned short
+// I386-CYGWIN:#define __UINT_LEAST32_MAX__ 4294967295U
+// I386-CYGWIN:#define __UINT_LEAST32_TYPE__ unsigned int
+// I386-CYGWIN:#define __UINT_LEAST64_MAX__ 18446744073709551615ULL
+// I386-CYGWIN:#define __UINT_LEAST64_TYPE__ long long unsigned int
+// I386-CYGWIN:#define __UINT_LEAST8_MAX__ 255
+// I386-CYGWIN:#define __UINT_LEAST8_TYPE__ unsigned char
+// I386-CYGWIN:#define __USER_LABEL_PREFIX__ _
+// I386-CYGWIN:#define __WCHAR_MAX__ 65535
+// I386-CYGWIN:#define __WCHAR_TYPE__ unsigned short
+// I386-CYGWIN:#define __WCHAR_WIDTH__ 16
+// I386-CYGWIN:#define __WINT_TYPE__ unsigned int
+// I386-CYGWIN:#define __WINT_WIDTH__ 32
+// I386-CYGWIN:#define __i386 1
+// I386-CYGWIN:#define __i386__ 1
+// I386-CYGWIN:#define __unix 1
+// I386-CYGWIN:#define __unix__ 1
+// I386-CYGWIN:#define i386 1
+// I386-CYGWIN:#define unix 1
+
+// RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=x86_64-pc-windows-cygnus < /dev/null | FileCheck -match-full-lines -check-prefix X86_64-CYGWIN %s
+// X86_64-CYGWIN:#define _LP64 1
+// X86_64-CYGWIN:#define __BIGGEST_ALIGNMENT__ 16
+// X86_64-CYGWIN:#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+// X86_64-CYGWIN:#define __CHAR16_TYPE__ unsigned short
+// X86_64-CYGWIN:#define __CHAR32_TYPE__ unsigned int
+// X86_64-CYGWIN:#define __CHAR_BIT__ 8
+// X86_64-CYGWIN:#define __CYGWIN__ 1
+// X86_64-CYGWIN:#define __DBL_DENORM_MIN__ 4.9406564584124654e-324
+// X86_64-CYGWIN:#define __DBL_DIG__ 15
+// X86_64-CYGWIN:#define __DBL_EPSILON__ 2.2204460492503131e-16
+// X86_64-CYGWIN:#define __DBL_HAS_DENORM__ 1
+// X86_64-CYGWIN:#define __DBL_HAS_INFINITY__ 1
+// X86_64-CYGWIN:#define __DBL_HAS_QUIET_NAN__ 1
+// X86_64-CYGWIN:#define __DBL_MANT_DIG__ 53
+// X86_64-CYGWIN:#define __DBL_MAX_10_EXP__ 308
+// X86_64-CYGWIN:#define __DBL_MAX_EXP__ 1024
+// X86_64-CYGWIN:#define __DBL_MAX__ 1.7976931348623157e+308
+// X86_64-CYGWIN:#define __DBL_MIN_10_EXP__ (-307)
+// X86_64-CYGWIN:#define __DBL_MIN_EXP__ (-1021)
+// X86_64-CYGWIN:#define __DBL_MIN__ 2.2250738585072014e-308
+// X86_64-CYGWIN:#define __DECIMAL_DIG__ __LDBL_DECIMAL_DIG__
+// X86_64-CYGWIN:#define __FLT_DENORM_MIN__ 1.40129846e-45F
+// X86_64-CYGWIN:#define __FLT_DIG__ 6
+// X86_64-CYGWIN:#define __FLT_EPSILON__ 1.19209290e-7F
+// X86_64-CYGWIN:#define __FLT_HAS_DENORM__ 1
+// X86_64-CYGWIN:#define __FLT_HAS_INFINITY__ 1
+// X86_64-CYGWIN:#define __FLT_HAS_QUIET_NAN__ 1
+// X86_64-CYGWIN:#define __FLT_MANT_DIG__ 24
+// X86_64-CYGWIN:#define __FLT_MAX_10_EXP__ 38
+// X86_64-CYGWIN:#define __FLT_MAX_EXP__ 128
+// X86_64-CYGWIN:#define __FLT_MAX__ 3.40282347e+38F
+// X86_64-CYGWIN:#define __FLT_MIN_10_EXP__ (-37)
+// X86_64-CYGWIN:#define __FLT_MIN_EXP__ (-125)
+// X86_64-CYGWIN:#define __FLT_MIN__ 1.17549435e-38F
+// X86_64-CYGWIN:#define __FLT_RADIX__ 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_BOOL_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_CHAR16_T_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_CHAR32_T_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_CHAR_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_INT_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_LLONG_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_LONG_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_POINTER_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_SHORT_LOCK_FREE 2
+// X86_64-CYGWIN:#define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
+// X86_64-CYGWIN:#define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
+// X86_64-CYGWIN:#define __INT16_C(c) c
+// X86_64-CYGWIN:#define __INT16_C_SUFFIX__
+// X86_64-CYGWIN:#define __INT16_FMTd__ "hd"
+// X86_64-CYGWIN:#define __INT16_FMTi__ "hi"
+// X86_64-CYGWIN:#define __INT16_MAX__ 32767
+// X86_64-CYGWIN:#define __INT16_TYPE__ short
+// X86_64-CYGWIN:#define __INT32_C(c) c
+// X86_64-CYGWIN:#define __INT32_C_SUFFIX__
+// X86_64-CYGWIN:#define __INT32_FMTd__ "d"
+// X86_64-CYGWIN:#define __INT32_FMTi__ "i"
+// X86_64-CYGWIN:#define __INT32_MAX__ 2147483647
+// X86_64-CYGWIN:#define __INT32_TYPE__ int
+// X86_64-CYGWIN:#define __INT64_C(c) c##L
+// X86_64-CYGWIN:#define __INT64_C_SUFFIX__ L
+// X86_64-CYGWIN:#define __INT64_FMTd__ "ld"
+// X86_64-CYGWIN:#define __INT64_FMTi__ "li"
+// X86_64-CYGWIN:#define __INT64_MAX__ 9223372036854775807L
+// X86_64-CYGWIN:#define __INT64_TYPE__ long int
+// X86_64-CYGWIN:#define __INT8_C(c) c
+// X86_64-CYGWIN:#define __INT8_C_SUFFIX__
+// X86_64-CYGWIN:#define __INT8_FMTd__ "hhd"
+// X86_64-CYGWIN:#define __INT8_FMTi__ "hhi"
+// X86_64-CYGWIN:#define __INT8_MAX__ 127
+// X86_64-CYGWIN:#define __INT8_TYPE__ signed char
+// X86_64-CYGWIN:#define __INTMAX_C(c) c##L
+// X86_64-CYGWIN:#define __INTMAX_C_SUFFIX__ L
+// X86_64-CYGWIN:#define __INTMAX_FMTd__ "ld"
+// X86_64-CYGWIN:#define __INTMAX_FMTi__ "li"
+// X86_64-CYGWIN:#define __INTMAX_MAX__ 9223372036854775807L
+// X86_64-CYGWIN:#define __INTMAX_TYPE__ long int
+// X86_64-CYGWIN:#define __INTMAX_WIDTH__ 64
+// X86_64-CYGWIN:#define __INTPTR_FMTd__ "ld"
+// X86_64-CYGWIN:#define __INTPTR_FMTi__ "li"
+// X86_64-CYGWIN:#define __INTPTR_MAX__ 9223372036854775807L
+// X86_64-CYGWIN:#define __INTPTR_TYPE__ long int
+// X86_64-CYGWIN:#define __INTPTR_WIDTH__ 64
+// X86_64-CYGWIN:#define __INT_FAST16_FMTd__ "hd"
+// X86_64-CYGWIN:#define __INT_FAST16_FMTi__ "hi"
+// X86_64-CYGWIN:#define __INT_FAST16_MAX__ 32767
+// X86_64-CYGWIN:#define __INT_FAST16_TYPE__ short
+// X86_64-CYGWIN:#define __INT_FAST32_FMTd__ "d"
+// X86_64-CYGWIN:#define __INT_FAST32_FMTi__ "i"
+// X86_64-CYGWIN:#define __INT_FAST32_MAX__ 2147483647
+// X86_64-CYGWIN:#define __INT_FAST32_TYPE__ int
+// X86_64-CYGWIN:#define __INT_FAST64_FMTd__ "ld"
+// X86_64-CYGWIN:#define __INT_FAST64_FMTi__ "li"
+// X86_64-CYGWIN:#define __INT_FAST64_MAX__ 9223372036854775807L
+// X86_64-CYGWIN:#define __INT_FAST64_TYPE__ long int
+// X86_64-CYGWIN:#define __INT_FAST8_FMTd__ "hhd"
+// X86_64-CYGWIN:#define __INT_FAST8_FMTi__ "hhi"
+// X86_64-CYGWIN:#define __INT_FAST8_MAX__ 127
+// X86_64-CYGWIN:#define __INT_FAST8_TYPE__ signed char
+// X86_64-CYGWIN:#define __INT_LEAST16_FMTd__ "hd"
+// X86_64-CYGWIN:#define __INT_LEAST16_FMTi__ "hi"
+// X86_64-CYGWIN:#define __INT_LEAST16_MAX__ 32767
+// X86_64-CYGWIN:#define __INT_LEAST16_TYPE__ short
+// X86_64-CYGWIN:#define __INT_LEAST32_FMTd__ "d"
+// X86_64-CYGWIN:#define __INT_LEAST32_FMTi__ "i"
+// X86_64-CYGWIN:#define __INT_LEAST32_MAX__ 2147483647
+// X86_64-CYGWIN:#define __INT_LEAST32_TYPE__ int
+// X86_64-CYGWIN:#define __INT_LEAST64_FMTd__ "ld"
+// X86_64-CYGWIN:#define __INT_LEAST64_FMTi__ "li"
+// X86_64-CYGWIN:#define __INT_LEAST64_MAX__ 9223372036854775807L
+// X86_64-CYGWIN:#define __INT_LEAST64_TYPE__ long int
+// X86_64-CYGWIN:#define __INT_LEAST8_FMTd__ "hhd"
+// X86_64-CYGWIN:#define __INT_LEAST8_FMTi__ "hhi"
+// X86_64-CYGWIN:#define __INT_LEAST8_MAX__ 127
+// X86_64-CYGWIN:#define __INT_LEAST8_TYPE__ signed char
+// X86_64-CYGWIN:#define __INT_MAX__ 2147483647
+// X86_64-CYGWIN:#define __LDBL_DENORM_MIN__ 3.64519953188247460253e-4951L
+// X86_64-CYGWIN:#define __LDBL_DIG__ 18
+// X86_64-CYGWIN:#define __LDBL_EPSILON__ 1.08420217248550443401e-19L
+// X86_64-CYGWIN:#define __LDBL_HAS_DENORM__ 1
+// X86_64-CYGWIN:#define __LDBL_HAS_INFINITY__ 1
+// X86_64-CYGWIN:#define __LDBL_HAS_QUIET_NAN__ 1
+// X86_64-CYGWIN:#define __LDBL_MANT_DIG__ 64
+// X86_64-CYGWIN:#define __LDBL_MAX_10_EXP__ 4932
+// X86_64-CYGWIN:#define __LDBL_MAX_EXP__ 16384
+// X86_64-CYGWIN:#define __LDBL_MAX__ 1.18973149535723176502e+4932L
+// X86_64-CYGWIN:#define __LDBL_MIN_10_EXP__ (-4931)
+// X86_64-CYGWIN:#define __LDBL_MIN_EXP__ (-16381)
+// X86_64-CYGWIN:#define __LDBL_MIN__ 3.36210314311209350626e-4932L
+// X86_64-CYGWIN:#define __LITTLE_ENDIAN__ 1
+// X86_64-CYGWIN:#define __LONG_LONG_MAX__ 9223372036854775807LL
+// X86_64-CYGWIN:#define __LONG_MAX__ 9223372036854775807L
+// X86_64-CYGWIN:#define __LP64__ 1
+// X86_64-CYGWIN:#define __MMX__ 1
+// X86_64-CYGWIN:#define __NO_MATH_INLINES 1
+// X86_64-CYGWIN:#define __POINTER_WIDTH__ 64
+// X86_64-CYGWIN:#define __PTRDIFF_TYPE__ long int
+// X86_64-CYGWIN:#define __PTRDIFF_WIDTH__ 64
+// X86_64-CYGWIN:#define __REGISTER_PREFIX__
+// X86_64-CYGWIN:#define __SCHAR_MAX__ 127
+// X86_64-CYGWIN:#define __SHRT_MAX__ 32767
+// X86_64-CYGWIN:#define __SIG_ATOMIC_MAX__ 2147483647
+// X86_64-CYGWIN:#define __SIG_ATOMIC_WIDTH__ 32
+// X86_64-CYGWIN:#define __SIZEOF_DOUBLE__ 8
+// X86_64-CYGWIN:#define __SIZEOF_FLOAT__ 4
+// X86_64-CYGWIN:#define __SIZEOF_INT__ 4
+// X86_64-CYGWIN:#define __SIZEOF_LONG_DOUBLE__ 16
+// X86_64-CYGWIN:#define __SIZEOF_LONG_LONG__ 8
+// X86_64-CYGWIN:#define __SIZEOF_LONG__ 8
+// X86_64-CYGWIN:#define __SIZEOF_POINTER__ 8
+// X86_64-CYGWIN:#define __SIZEOF_PTRDIFF_T__ 8
+// X86_64-CYGWIN:#define __SIZEOF_SHORT__ 2
+// X86_64-CYGWIN:#define __SIZEOF_SIZE_T__ 8
+// X86_64-CYGWIN:#define __SIZEOF_WCHAR_T__ 2
+// X86_64-CYGWIN:#define __SIZEOF_WINT_T__ 4
+// X86_64-CYGWIN:#define __SIZE_MAX__ 18446744073709551615UL
+// X86_64-CYGWIN:#define __SIZE_TYPE__ long unsigned int
+// X86_64-CYGWIN:#define __SIZE_WIDTH__ 64
+// X86_64-CYGWIN:#define __SSE2_MATH__ 1
+// X86_64-CYGWIN:#define __SSE2__ 1
+// X86_64-CYGWIN:#define __SSE_MATH__ 1
+// X86_64-CYGWIN:#define __SSE__ 1
+// X86_64-CYGWIN:#define __UINT16_C(c) c
+// X86_64-CYGWIN:#define __UINT16_C_SUFFIX__
+// X86_64-CYGWIN:#define __UINT16_MAX__ 65535
+// X86_64-CYGWIN:#define __UINT16_TYPE__ unsigned short
+// X86_64-CYGWIN:#define __UINT32_C(c) c##U
+// X86_64-CYGWIN:#define __UINT32_C_SUFFIX__ U
+// X86_64-CYGWIN:#define __UINT32_MAX__ 4294967295U
+// X86_64-CYGWIN:#define __UINT32_TYPE__ unsigned int
+// X86_64-CYGWIN:#define __UINT64_C(c) c##UL
+// X86_64-CYGWIN:#define __UINT64_C_SUFFIX__ UL
+// X86_64-CYGWIN:#define __UINT64_MAX__ 18446744073709551615UL
+// X86_64-CYGWIN:#define __UINT64_TYPE__ long unsigned int
+// X86_64-CYGWIN:#define __UINT8_C(c) c
+// X86_64-CYGWIN:#define __UINT8_C_SUFFIX__
+// X86_64-CYGWIN:#define __UINT8_MAX__ 255
+// X86_64-CYGWIN:#define __UINT8_TYPE__ unsigned char
+// X86_64-CYGWIN:#define __UINTMAX_C(c) c##UL
+// X86_64-CYGWIN:#define __UINTMAX_C_SUFFIX__ UL
+// X86_64-CYGWIN:#define __UINTMAX_MAX__ 18446744073709551615UL
+// X86_64-CYGWIN:#define __UINTMAX_TYPE__ long unsigned int
+// X86_64-CYGWIN:#define __UINTMAX_WIDTH__ 64
+// X86_64-CYGWIN:#define __UINTPTR_MAX__ 18446744073709551615UL
+// X86_64-CYGWIN:#define __UINTPTR_TYPE__ long unsigned int
+// X86_64-CYGWIN:#define __UINTPTR_WIDTH__ 64
+// X86_64-CYGWIN:#define __UINT_FAST16_MAX__ 65535
+// X86_64-CYGWIN:#define __UINT_FAST16_TYPE__ unsigned short
+// X86_64-CYGWIN:#define __UINT_FAST32_MAX__ 4294967295U
+// X86_64-CYGWIN:#define __UINT_FAST32_TYPE__ unsigned int
+// X86_64-CYGWIN:#define __UINT_FAST64_MAX__ 18446744073709551615UL
+// X86_64-CYGWIN:#define __UINT_FAST64_TYPE__ long unsigned int
+// X86_64-CYGWIN:#define __UINT_FAST8_MAX__ 255
+// X86_64-CYGWIN:#define __UINT_FAST8_TYPE__ unsigned char
+// X86_64-CYGWIN:#define __UINT_LEAST16_MAX__ 65535
+// X86_64-CYGWIN:#define __UINT_LEAST16_TYPE__ unsigned short
+// X86_64-CYGWIN:#define __UINT_LEAST32_MAX__ 4294967295U
+// X86_64-CYGWIN:#define __UINT_LEAST32_TYPE__ unsigned int
+// X86_64-CYGWIN:#define __UINT_LEAST64_MAX__ 18446744073709551615UL
+// X86_64-CYGWIN:#define __UINT_LEAST64_TYPE__ long unsigned int
+// X86_64-CYGWIN:#define __UINT_LEAST8_MAX__ 255
+// X86_64-CYGWIN:#define __UINT_LEAST8_TYPE__ unsigned char
+// X86_64-CYGWIN:#define __USER_LABEL_PREFIX__
+// X86_64-CYGWIN:#define __WCHAR_MAX__ 65535
+// X86_64-CYGWIN:#define __WCHAR_TYPE__ unsigned short
+// X86_64-CYGWIN:#define __WCHAR_WIDTH__ 16
+// X86_64-CYGWIN:#define __WINT_TYPE__ unsigned int
+// X86_64-CYGWIN:#define __WINT_WIDTH__ 32
+// X86_64-CYGWIN:#define __amd64 1
+// X86_64-CYGWIN:#define __amd64__ 1
+// X86_64-CYGWIN:#define __unix 1
+// X86_64-CYGWIN:#define __unix__ 1
+// X86_64-CYGWIN:#define __x86_64 1
+// X86_64-CYGWIN:#define __x86_64__ 1
+// X86_64-CYGWIN:#define unix 1
diff --git a/external/llvm-project/clang/test/Preprocessor/init.c b/external/llvm-project/clang/test/Preprocessor/init.c
index 4c8d519c2b66..031a6c1a755b 100644
--- a/external/llvm-project/clang/test/Preprocessor/init.c
+++ b/external/llvm-project/clang/test/Preprocessor/init.c
@@ -2090,12 +2090,6 @@
 // WEBASSEMBLY-CXX-ATOMICS:#define _REENTRANT 1
 // WEBASSEMBLY-CXX-ATOMICS:#define __STDCPP_THREADS__ 1
 
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple i686-windows-cygnus < /dev/null | FileCheck -match-full-lines -check-prefix CYGWIN-X32 %s
-// CYGWIN-X32: #define __USER_LABEL_PREFIX__ _
-
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple x86_64-windows-cygnus < /dev/null | FileCheck -match-full-lines -check-prefix CYGWIN-X64 %s
-// CYGWIN-X64: #define __USER_LABEL_PREFIX__
-
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=avr \
 // RUN:   < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix=AVR %s
diff --git a/external/llvm-project/clang/test/Preprocessor/line-directive-suppressed.c b/external/llvm-project/clang/test/Preprocessor/line-directive-suppressed.c
new file mode 100644
index 000000000000..b03c49b8a9f9
--- /dev/null
+++ b/external/llvm-project/clang/test/Preprocessor/line-directive-suppressed.c
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -std=c99 -fsyntax-only -pedantic %s 2>&1 | FileCheck %s
+
+// RUN: cp %s %t.i
+// RUN: %clang_cc1 -std=c99 -fsyntax-only -pedantic %t.i 2>&1 | FileCheck %s --check-prefix=NO-WARNING --allow-empty
+// RUN: %clang_cc1 -std=c99 -fsyntax-only -pedantic -x cpp-output %s 2>&1 | FileCheck %s --check-prefix=NO-WARNING --allow-empty
+
+# 0 "zero"
+// CHECK: line-directive-suppressed.c:[[@LINE-1]]:5: warning: {{.*}} [-Wgnu-line-marker]
+
+# 1 "one" 1
+// CHECK: zero:2:5: warning: {{.*}} [-Wgnu-line-marker]
+
+# 2 "two" 1 3 4
+// CHECK: one:3:5: warning: {{.*}} [-Wgnu-line-marker]
+
+// NO-WARNING-NOT: warning:
diff --git a/external/llvm-project/clang/test/Preprocessor/predefined-macros.c b/external/llvm-project/clang/test/Preprocessor/predefined-macros.c
index b7765bfa2fb1..da25b1efa398 100644
--- a/external/llvm-project/clang/test/Preprocessor/predefined-macros.c
+++ b/external/llvm-project/clang/test/Preprocessor/predefined-macros.c
@@ -228,6 +228,7 @@
 // CHECK-SPIRV32-DAG: #define __SPIRV__ 1
 // CHECK-SPIRV32-DAG: #define __SPIRV32__ 1
 // CHECK-SPIRV32-NOT: #define __SPIRV64__ 1
+// CHECK-SPIRV32-NOT: #define __spirv__ 1
 
 // RUN: %clang_cc1 %s -E -dM -o - -x cl -triple spirv64-unknown-unknown \
 // RUN:   | FileCheck -match-full-lines %s --check-prefix=CHECK-SPIRV64
@@ -235,6 +236,7 @@
 // CHECK-SPIRV64-DAG: #define __SPIRV__ 1
 // CHECK-SPIRV64-DAG: #define __SPIRV64__ 1
 // CHECK-SPIRV64-NOT: #define __SPIRV32__ 1
+// CHECK-SPIRV64-NOT: #define __spirv__ 1
 
 // RUN: %clang_cc1 %s -E -dM -o - -x cl -triple spirv64-amd-amdhsa \
 // RUN:   | FileCheck -match-full-lines %s --check-prefix=CHECK-SPIRV64-AMDGCN
@@ -245,6 +247,7 @@
 // CHECK-SPIRV64-AMDGCN-DAG: #define __AMDGCN__ 1
 // CHECK-SPIRV64-AMDGCN-DAG: #define __AMDGPU__ 1
 // CHECK-SPIRV64-AMDGCN-NOT: #define __SPIRV32__ 1
+// CHECK-SPIRV64-AMDGCN-NOT: #define __spirv__ 1
 
 // RUN: %clang_cc1 %s -E -dM -o - -x hip -triple x86_64-unknown-linux-gnu \
 // RUN:   | FileCheck -match-full-lines %s --check-prefix=CHECK-HIP
diff --git a/external/llvm-project/clang/test/Preprocessor/riscv-target-features-andes.c b/external/llvm-project/clang/test/Preprocessor/riscv-target-features-andes.c
new file mode 100644
index 000000000000..3cd9b0435413
--- /dev/null
+++ b/external/llvm-project/clang/test/Preprocessor/riscv-target-features-andes.c
@@ -0,0 +1,32 @@
+// RUN: %clang --target=riscv32-unknown-linux-gnu -march=rv32i -E -dM %s \
+// RUN:   -o - | FileCheck %s
+// RUN: %clang --target=riscv64-unknown-linux-gnu -march=rv64i -E -dM %s \
+// RUN:   -o - | FileCheck %s
+
+// CHECK-NOT: __riscv_xandesperf {{.*$}}
+// CHECK-NOT: __riscv_xandesvpackfph {{.*$}}
+// CHECK-NOT: __riscv_xandesvdot {{.*$}}
+
+// RUN: %clang --target=riscv32 \
+// RUN:   -march=rv32i_xandesperf -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESPERF %s
+// RUN: %clang --target=riscv64 \
+// RUN:   -march=rv64i_xandesperf -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESPERF %s
+// CHECK-XANDESPERF: __riscv_xandesperf  5000000{{$}}
+
+// RUN: %clang --target=riscv32 \
+// RUN:   -march=rv32i_xandesvpackfph -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVPACKFPH %s
+// RUN: %clang --target=riscv64 \
+// RUN:   -march=rv64i_xandesvpackfph -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVPACKFPH %s
+// CHECK-XANDESVPACKFPH: __riscv_xandesvpackfph  5000000{{$}}
+
+// RUN: %clang --target=riscv32 \
+// RUN:   -march=rv32i_xandesvdot -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVDOT %s
+// RUN: %clang --target=riscv64 \
+// RUN:   -march=rv64i_xandesvdot -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVDOT %s
+// CHECK-XANDESVDOT: __riscv_xandesvdot  5000000{{$}}
diff --git a/external/llvm-project/clang/test/Sema/Inputs/XROS.sdk/SDKSettings.json b/external/llvm-project/clang/test/Sema/Inputs/XROS.sdk/SDKSettings.json
new file mode 100644
index 000000000000..a56a7d61431b
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/Inputs/XROS.sdk/SDKSettings.json
@@ -0,0 +1,9 @@
+{
+  "DefaultVariant": "xrOS", "DisplayName": "xrOS",
+  "Version": "26.0",
+  "CanonicalName": "xros26.0",
+  "MaximumDeploymentTarget": "26.0.99",
+  "VersionMap": {
+    "iOS_xrOS":{"15.0":"1.0", "16.0":"2.0", "19.0":"26.0", "26.0":"26.0"}
+  }
+}
diff --git a/external/llvm-project/clang/test/Sema/Inputs/builtin-system-header.h b/external/llvm-project/clang/test/Sema/Inputs/builtin-system-header.h
new file mode 100644
index 000000000000..7eeb8d811fcf
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/Inputs/builtin-system-header.h
@@ -0,0 +1,9 @@
+#ifdef USE_PRAGMA_BEFORE
+#pragma intrinsic(_InterlockedOr64)
+#endif
+
+#define MACRO(x,y) _InterlockedOr64(x,y);
+
+#ifdef USE_PRAGMA_AFTER
+#pragma intrinsic(_InterlockedOr64)
+#endif
diff --git a/external/llvm-project/clang/test/Sema/PR28181.c b/external/llvm-project/clang/test/Sema/PR28181.c
index 8d0a4ad33562..7e9d5cc91038 100644
--- a/external/llvm-project/clang/test/Sema/PR28181.c
+++ b/external/llvm-project/clang/test/Sema/PR28181.c
@@ -5,9 +5,9 @@ struct spinlock_t {
 } audit_skb_queue;
 
 void fn1(void) {
-  audit_skb_queue = (lock); // expected-error {{use of undeclared identifier 'lock'; did you mean 'long'?}}
-}                           // expected-error@-1 {{assigning to 'struct spinlock_t' from incompatible type '<overloaded function type>'}}
+  audit_skb_queue = (lock); // expected-error {{use of undeclared identifier 'lock'}}
+}
 
 void fn2(void) {
-  audit_skb_queue + (lock); // expected-error {{use of undeclared identifier 'lock'; did you mean 'long'?}}
-}                           // expected-error@-1 {{reference to overloaded function could not be resolved; did you mean to call it?}}
+  audit_skb_queue + (lock); // expected-error {{use of undeclared identifier 'lock'}}
+}
diff --git a/external/llvm-project/clang/test/Sema/attr-availability-invalid-os-versions.c b/external/llvm-project/clang/test/Sema/attr-availability-invalid-os-versions.c
new file mode 100644
index 000000000000..871263fc62cb
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/attr-availability-invalid-os-versions.c
@@ -0,0 +1,35 @@
+/// This test verifies diagnostic reporting for OS versions within an invalid range.
+// RUN: %clang_cc1 -triple x86_64-apple-darwin25 -fsyntax-only -verify %s
+
+ __attribute__((availability(macosx,introduced=16, deprecated=20))) 
+void invalid_dep(void); // expected-warning@-1 {{invalid macOS version '20' in availability attribute}}
+                        // expected-note@-2 {{implicitly treating version as '30'}}
+ 
+__attribute__((availability(macosx,introduced=15,  obsoleted=19)))
+void invalid_obsolete(void); // expected-warning@-1 {{invalid macOS version '19' in availability attribute}}
+                             // expected-note@-2 {{implicitly treating version as '29'}}
+
+__attribute__((availability(macosx,introduced=24)))
+void invalid_intro(void); // expected-warning@-1 {{invalid macOS version '24' in availability attribute}}
+                          // expected-note@-2 {{implicitly treating version as '34'}}
+
+__attribute__((availability(watchos,introduced=24)))
+void invalid_watch_intro(void); // expected-warning@-1 {{invalid watchOS version '24' in availability attribute}}
+                                // expected-note@-2 {{implicitly treating version as '38'}}
+
+__attribute__((availability(maccatalyst, introduced=18.1, deprecated=21.1)))
+void invalid_catalyst_intro(void); // expected-warning@-1 {{invalid macCatalyst version '21.1' in availability attribute}}
+                                   // expected-note@-2 {{implicitly treating version as '28.1'}}
+
+__attribute__((availability(macos, introduced=15, deprecated=26)))
+__attribute__((availability(watchos, introduced=15)))
+__attribute__((availability(ios, introduced=15, deprecated=18)))
+void invalid_intro_multiple(void); // expected-warning@-2 {{invalid watchOS version '15' in availability attribute}}
+                                   // expected-note@-3 {{implicitly treating version as '29'}}
+ 
+__attribute__((availability(macCatalyst, introduced=19.2, deprecated=22.2))) 
+void invalid_dep(void); // expected-warning@-1 {{invalid macCatalyst version '19.2' in availability attribute}}
+                        // expected-note@-2 {{implicitly treating version as '26.2'}}
+                        // expected-warning@-3 {{invalid macCatalyst version '22.2' in availability attribute}}
+                        // expected-note@-4 {{implicitly treating version as '29.2'}}
+
diff --git a/external/llvm-project/clang/test/Sema/attr-availability-ios-aligned-versions.c b/external/llvm-project/clang/test/Sema/attr-availability-ios-aligned-versions.c
new file mode 100644
index 000000000000..703ef42d881d
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/attr-availability-ios-aligned-versions.c
@@ -0,0 +1,116 @@
+/// This test verifies several different patterns of iOS, and app extension, availability declarations & usages.
+// RUN: %clang_cc1 "-triple" "arm64-apple-ios26" -DNEW -fsyntax-only -verify %s
+// RUN: %clang_cc1 "-triple" "arm64-apple-ios18" -fsyntax-only -verify -fapplication-extension -DAPP_EXT %s
+// RUN: %clang_cc1 "-triple" "arm64-apple-ios18" -fsyntax-only -verify %s
+
+__attribute__((availability(ios,strict,introduced=19)))
+void fNew1();
+#ifndef NEW
+// expected-note@-2 {{here}}
+#endif
+
+__attribute__((availability(ios,strict,introduced=19)))
+void fNew();
+
+__attribute__((availability(ios,strict,introduced=26)))
+void fNew() { }
+#ifndef NEW
+// expected-note@-2 {{here}}
+#endif
+
+__attribute__((availability(ios,strict,deprecated=19)))
+void fDep();
+
+__attribute__((availability(ios,strict,deprecated=26)))
+void fDep() { }
+#ifdef NEW
+// expected-note@-2 {{here}}
+#endif
+
+__attribute__((availability(ios,strict,obsoleted=19)))
+void fObs();
+
+__attribute__((availability(ios,strict,obsoleted=26)))
+void fObs() { }
+#ifdef NEW
+// expected-note@-2 {{here}}
+#endif
+
+__attribute__((availability(ios_app_extension,strict,introduced=19)))
+void fAppExt();
+
+__attribute__((availability(ios_app_extension,strict,introduced=26)))
+void fAppExt() { }
+#ifdef APP_EXT
+// expected-note@-2 {{here}}
+#endif
+
+void testVersionRemapping() {
+  fNew1();
+#ifndef NEW
+  // expected-error@-2 {{'fNew1' is unavailable: introduced in iOS 26.0}}
+#endif
+  fNew();
+#ifndef NEW
+  // expected-error@-2 {{'fNew' is unavailable: introduced in iOS 26}}
+#endif
+  fDep();
+#ifdef NEW
+  // expected-warning@-2 {{'fDep' is deprecated: first deprecated in iOS 26}}
+#endif
+  fObs();
+#ifdef NEW
+  // expected-error@-2 {{'fObs' is unavailable: obsoleted in iOS 26}}
+#endif
+
+  fAppExt();
+#ifdef APP_EXT
+  // expected-error@-2 {{'fAppExt' is unavailable: introduced in iOS (App Extension) 26}}
+#endif
+}
+
+__attribute__((availability(ios,strict,introduced=18.5))) // expected-note {{here}}
+void fMatchErr();
+
+__attribute__((availability(ios,strict,introduced=26))) // expected-warning {{availability does not match previous declaration}}
+void fMatchErr() { }
+
+__attribute__((availability(ios_app_extension,strict,introduced=19))) // expected-note {{here}}
+void fAppExtErr();
+
+__attribute__((availability(ios_app_extension,strict,introduced=26.1))) // expected-warning {{availability does not match previous declaration}}
+void fAppExtErr() { }
+
+__attribute__((availability(ios,introduced=26)))
+void fNew2();
+#ifndef NEW
+  // expected-note@-2 {{'fNew2' has been marked as being introduced in iOS 26 here, but the deployment target is iOS 18}}
+#endif
+__attribute__((availability(ios,introduced=19)))
+void fNew3();
+
+__attribute__((availability(ios,introduced=27)))
+void evenNewer();
+#ifdef NEW
+  // expected-note@-2 {{'evenNewer' has been marked as being introduced in iOS 27 here, but the deployment target is iOS 26}}
+#endif
+
+void testAvailabilityCheck() {
+  if (__builtin_available(iOS 19, *)) {
+    fNew2();
+    fNew3();
+  }
+  if (__builtin_available(iOS 26, *)) {
+    fNew2();
+    fNew3();
+  }
+  fNew2();
+#ifndef NEW
+  // expected-warning@-2 {{'fNew2' is only available on iOS 26 or newer}} expected-note@-2 {{enclose}}
+#endif
+#ifdef NEW
+  evenNewer(); // expected-warning {{'evenNewer' is only available on iOS 27 or newer}} expected-note {{enclose}}
+#endif
+}
+
+
diff --git a/external/llvm-project/clang/test/Sema/attr-availability-ios-fallback-aligned-versions.c b/external/llvm-project/clang/test/Sema/attr-availability-ios-fallback-aligned-versions.c
new file mode 100644
index 000000000000..8947bc7dda46
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/attr-availability-ios-fallback-aligned-versions.c
@@ -0,0 +1,32 @@
+// This validates that all expected OSVersions that allow fallbacks 
+// from iOS behave as expected against a common version bump.
+
+// RUN: %clang_cc1 "-triple" "arm64-apple-ios26" -fsyntax-only -verify %s
+// RUN: %clang_cc1 "-triple" "arm64-apple-watchos26" -fsyntax-only -verify %s
+// RUN: %clang_cc1 "-triple" "arm64-apple-tvos26" -fsyntax-only -verify %s
+
+// VisionOS requires SDKSettings support to enable remappings.
+// RUN: %clang_cc1 "-triple" "arm64-apple-visionos26" -isysroot %S/Inputs/XROS.sdk -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+__attribute__((availability(ios,strict,introduced=19)))
+int iOSExistingAPI(void);
+
+__attribute__((availability(ios,strict,introduced=26)))
+int iOSExistingAPI2(void);
+
+void testAvailabilityCheck(void) {
+  
+  if (__builtin_available(iOS 19, *)) {
+    iOSExistingAPI();
+    iOSExistingAPI2();
+  }
+  
+  if (__builtin_available(iOS 26, *)) {
+    iOSExistingAPI();
+    iOSExistingAPI2();
+  }
+
+  iOSExistingAPI2();
+}
diff --git a/external/llvm-project/clang/test/Sema/attr-availability-iosmac-infer-from-macos.c b/external/llvm-project/clang/test/Sema/attr-availability-iosmac-infer-from-macos.c
index 12c6d59b61ac..73ff8c33ebe5 100644
--- a/external/llvm-project/clang/test/Sema/attr-availability-iosmac-infer-from-macos.c
+++ b/external/llvm-project/clang/test/Sema/attr-availability-iosmac-infer-from-macos.c
@@ -24,7 +24,7 @@ void fObs(void) __attribute__((availability(macOS, introduced = 10.11, obsoleted
 
 void fAPItoDepr(void) __attribute__((availability(macOS, introduced = 10.11, deprecated = 100000)));
 
-void dontRemapFutureVers(void) __attribute__((availability(macOS, introduced = 20)));
+void dontRemapFutureVers(void) __attribute__((availability(macOS, introduced = 30)));
 
 void usage(void) {
   f0();
diff --git a/external/llvm-project/clang/test/Sema/attr-target-clones-aarch64.c b/external/llvm-project/clang/test/Sema/attr-target-clones-aarch64.c
index 784646ed15d2..93d87cef5456 100644
--- a/external/llvm-project/clang/test/Sema/attr-target-clones-aarch64.c
+++ b/external/llvm-project/clang/test/Sema/attr-target-clones-aarch64.c
@@ -13,6 +13,8 @@ int redecl(void);
 int __attribute__((target_clones("frintts", "simd+fp", "default"))) redecl(void) { return 1; }
 
 int __attribute__((target_clones("jscvt+fcma", "rcpc", "default"))) redecl2(void);
+// expected-error@+2 {{'target_clones' attribute does not match previous declaration}}
+// expected-note@-2 {{previous declaration is here}}
 int __attribute__((target_clones("jscvt+fcma", "rcpc"))) redecl2(void) { return 1; }
 
 int __attribute__((target_clones("sve+dotprod"))) redecl3(void);
diff --git a/external/llvm-project/clang/test/Sema/attr-target-loongarch.c b/external/llvm-project/clang/test/Sema/attr-target-loongarch.c
new file mode 100644
index 000000000000..10ac8334bdb0
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/attr-target-loongarch.c
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -triple loongarch64-linux-gnu  -fsyntax-only -verify %s
+
+// expected-error@+1 {{function multiversioning is not supported on the current target}}
+void __attribute__((target("default"))) bar(void) {}
+
+// expected-error@+1 {{target(arch=..) attribute is not supported on targets missing invalid; specify an appropriate -march= or -mcpu=}}
+void __attribute__((target("arch=invalid"))) foo(void) {}
+
+// expected-warning@+1 {{unsupported '+div32' in the 'target' attribute string; 'target' attribute ignored}}
+void __attribute__((target("+div32"))) plusfeature(void) {}
+
+// expected-warning@+1 {{unsupported '-div32' in the 'target' attribute string; 'target' attribute ignored}}
+void __attribute__((target("-div32"))) minusfeature(void) {}
+
+// expected-warning@+1 {{unsupported 'aaa' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("aaa"))) test_feature(void) { return 4; }
+
+// expected-warning@+1 {{unsupported 'aaa' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("no-aaa"))) test_nofeature(void) { return 4; }
+
+// expected-warning@+1 {{duplicate 'arch=' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("arch=la464,arch=la664"))) test_duplarch(void) { return 4; }
+
+// expected-warning@+1 {{unknown tune CPU 'la64v1.0' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("tune=la64v1.0"))) test_tune(void) { return 4; }
diff --git a/external/llvm-project/clang/test/Sema/builtin-pragma-intrinsic-namespace.cpp b/external/llvm-project/clang/test/Sema/builtin-pragma-intrinsic-namespace.cpp
new file mode 100644
index 000000000000..ffdc24b3d468
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/builtin-pragma-intrinsic-namespace.cpp
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fms-compatibility -fsyntax-only -verify -DOUTSIDE %s
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fms-compatibility -fsyntax-only -verify -DINSIDE %s
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fms-compatibility -fsyntax-only -verify -DNESTED %s
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fms-compatibility -fsyntax-only -verify -DOUTSIDE -DEXTERN %s
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fms-compatibility -fsyntax-only -verify -DINSIDE -DEXTERN %s
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fms-compatibility -fsyntax-only -verify -DNESTED -DEXTERN %s
+
+// expected-no-diagnostics
+#ifdef EXTERN
+extern "C"
+#endif
+unsigned __int64 _umul128(unsigned __int64, unsigned __int64,
+                          unsigned __int64 *);
+namespace {
+#ifdef INSIDE
+  #pragma intrinsic(_umul128)
+#endif
+#ifdef NESTED
+  namespace {
+#pragma intrinsic(_umul128)
+  }
+#endif
+}
+
+#ifdef OUTSIDE
+#pragma intrinsic(_umul128)
+#endif
+
+void foo() {
+  unsigned __int64 carry;
+  unsigned __int64 low = _umul128(0, 0, &carry);
+}
diff --git a/external/llvm-project/clang/test/Sema/builtin-pragma-intrinsic.c b/external/llvm-project/clang/test/Sema/builtin-pragma-intrinsic.c
new file mode 100644
index 000000000000..1e8507bfd37d
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/builtin-pragma-intrinsic.c
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -fms-extensions -fsyntax-only -verify -triple arm64-windows -isystem %S/Inputs %s -DUSE_PRAGMA_BEFORE
+// RUN: %clang_cc1 -fms-extensions -fsyntax-only -verify -triple arm64-windows -isystem %S/Inputs %s -DUSE_PRAGMA_AFTER
+// RUN: %clang_cc1 -fms-extensions -fsyntax-only -verify -triple arm64-windows -isystem %S/Inputs %s -DUSE_PRAGMA_AFTER_USE
+// RUN: %clang_cc1 -fms-extensions -fsyntax-only -verify -triple arm64-windows -isystem %S/Inputs %s -DUSE_PRAGMA_SAME_FILE
+// RUN: %clang_cc1 -fms-extensions -fsyntax-only -verify -triple arm64-windows -isystem %S/Inputs %s
+
+#if defined(USE_PRAGMA_BEFORE) || defined(USE_PRAGMA_AFTER) || defined(USE_PRAGMA_SAME_FILE)
+// expected-no-diagnostics
+#else
+// expected-error@+10 {{call to undeclared library function '_InterlockedOr64'}}
+// expected-note@+9 {{include the header <intrin.h> or explicitly provide a declaration for '_InterlockedOr64'}}
+#endif
+#include <builtin-system-header.h>
+
+#ifdef USE_PRAGMA_SAME_FILE
+#pragma intrinsic(_InterlockedOr64)
+#endif
+
+void foo() {
+  MACRO(0,0);
+}
+
+#ifdef USE_PRAGMA_AFTER_USE
+#pragma intrinsic(_InterlockedOr64)
+#endif
diff --git a/external/llvm-project/clang/test/Sema/builtin-unary-fp.c b/external/llvm-project/clang/test/Sema/builtin-unary-fp.c
index fb8e341156a5..9bfcb30b9eba 100644
--- a/external/llvm-project/clang/test/Sema/builtin-unary-fp.c
+++ b/external/llvm-project/clang/test/Sema/builtin-unary-fp.c
@@ -17,5 +17,4 @@ void a(void) {
 
   check(__builtin_fpclassify(0,0,0,0,0, (invalid))); // expected-error{{use of undeclared identifier 'invalid'}}
   check(__builtin_fpclassify(0,0,0,0,0, (inf))); // expected-error{{use of undeclared identifier 'inf'}}
-                                                // expected-error@-1{{reference to overloaded function could not be resolved}}
 }
diff --git a/external/llvm-project/clang/test/Sema/c23-delayed-typo-correction-crashes.c b/external/llvm-project/clang/test/Sema/c23-delayed-typo-correction-crashes.c
new file mode 100644
index 000000000000..6afd3fd32c36
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/c23-delayed-typo-correction-crashes.c
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 -std=c23 -fsyntax-only -verify %s
+
+void GH139913(...);
+void GH139913_test() {
+  GH139913(CONCAT(foo, )); // expected-error {{use of undeclared identifier 'CONCAT'}} \
+                              expected-error {{use of undeclared identifier 'foo'}} \
+                              expected-error {{expected expression}}
+}
+
+struct GH137867 {
+ char value;
+};
+void GH137867_test() {
+  _Atomic(struct GH137867) t;
+  while (!atomic_load(&t.value)->value) // expected-error {{use of undeclared identifier 'atomic_load'}} \
+                                           expected-error {{accessing a member of an atomic structure or union is undefined behavior}}
+    ;
+}
diff --git a/external/llvm-project/clang/test/Sema/delayed-typo-correction-crashes.c b/external/llvm-project/clang/test/Sema/delayed-typo-correction-crashes.c
new file mode 100644
index 000000000000..81c966789ccb
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/delayed-typo-correction-crashes.c
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 -fsyntax-only -fblocks -ffixed-point -verify %s
+
+void GH137860_test(void) {
+  struct S {
+    char h;
+  };
+  _Atomic struct S s = { .h = UINT8_MIN }; // expected-error {{use of undeclared identifier 'UINT8_MIN'}}
+  __c11_atomic_fetch_add(&s.h, UINT8_MIN); // expected-error {{use of undeclared identifier 'UINT8_MIN'}} \
+                                              expected-error {{accessing a member of an atomic structure or union is undefined behavior}}
+}
+
+int (^GH69470) (int i, int j) = ^(int i, int j)
+{ return i / j; }/ j; // expected-error {{use of undeclared identifier 'j'}}
+
+void GH69874(void) {
+  *a = (a_struct){0}; // expected-error {{use of undeclared identifier 'a'}} \
+                         expected-error {{use of undeclared identifier 'a_struct'}}
+}
diff --git a/external/llvm-project/clang/test/Sema/implicit-int-conversion-on-int.c b/external/llvm-project/clang/test/Sema/implicit-int-conversion-on-int.c
new file mode 100644
index 000000000000..f555893d9e17
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/implicit-int-conversion-on-int.c
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 %s -verify=expected -Wimplicit-int-conversion
+// RUN: %clang_cc1 %s -verify=none -Wimplicit-int-conversion -Wno-implicit-int-conversion-on-negation
+
+// none-no-diagnostics
+
+char test_char(char x) {
+  return -x; // expected-warning {{implicit conversion loses integer precision: 'int' to 'char' on negation}}
+}
+
+unsigned char test_unsigned_char(unsigned char x) {
+  return -x; // expected-warning {{implicit conversion loses integer precision: 'int' to 'unsigned char' on negation}}
+}
+
+short test_short(short x) {
+  return -x; // expected-warning {{implicit conversion loses integer precision: 'int' to 'short' on negation}}
+}
+
+unsigned short test_unsigned_short(unsigned short x) {
+  return -x; // expected-warning {{implicit conversion loses integer precision: 'int' to 'unsigned short' on negation}}
+}
+
+// --- int-width and wider (should NOT warn) ---
+
+int test_i(int x) {
+  return -x;
+}
+
+unsigned int test_ui(unsigned int x) {
+  return -x;
+}
+
+long test_l(long x) {
+  return -x;
+}
+
+unsigned long test_ul(unsigned long x) {
+  return -x;
+}
+
+long long test_ll(long long x) {
+  return -x;
+}
+
+unsigned long long test_ull(unsigned long long x) {
+  return -x;
+}
+
+unsigned _BitInt(16) test_unsigned_bit_int(unsigned _BitInt(16) x) {
+  return -x;
+}
diff --git a/external/llvm-project/clang/test/Sema/invalid-member.cpp b/external/llvm-project/clang/test/Sema/invalid-member.cpp
index 57ee187ccf4d..0e3fec1b18ee 100644
--- a/external/llvm-project/clang/test/Sema/invalid-member.cpp
+++ b/external/llvm-project/clang/test/Sema/invalid-member.cpp
@@ -20,10 +20,12 @@ class Z {
 // Should be able to evaluate sizeof without crashing.
 static_assert(sizeof(Z) == 1, "No valid members");
 
-constexpr int N = undef; // expected-error {{use of undeclared identifier}}
+constexpr int N = undef; // expected-error {{use of undeclared identifier}} \
+                            expected-note {{declared here}}
 template<int a>
 class ABC {};
 class T {
-  ABC<N> abc;
+  ABC<N> abc; // expected-error {{non-type template argument is not a constant expression}} \
+                 expected-note {{initializer of 'N' is unknown}}
 };
 static_assert(sizeof(T) == 1, "No valid members");
diff --git a/external/llvm-project/clang/test/Sema/rvv-required-features-invalid.c b/external/llvm-project/clang/test/Sema/rvv-required-features-invalid.c
index e73ffda95ab1..c96d0e158062 100644
--- a/external/llvm-project/clang/test/Sema/rvv-required-features-invalid.c
+++ b/external/llvm-project/clang/test/Sema/rvv-required-features-invalid.c
@@ -5,37 +5,46 @@
 #include <sifive_vector.h>
 
 vint8m1_t test_vloxei64_v_i8m1(const int8_t *base, vuint64m8_t bindex, size_t vl) {
-  return __riscv_vloxei64(base, bindex, vl); // expected-error {{call to undeclared function '__riscv_vloxei64'}} expected-error {{returning 'int' from a function with incompatible result type 'vint8m1_t'}}
+  return __riscv_vloxei64(base, bindex, vl); // expected-error {{builtin requires at least one of the following extensions: 64bit}}
 }
 
 void test_vsoxei64_v_i8m1(int8_t *base, vuint64m8_t bindex, vint8m1_t value, size_t vl) {
-  __riscv_vsoxei64(base, bindex, value, vl); // expected-error {{call to undeclared function '__riscv_vsoxei64'}}
+  __riscv_vsoxei64(base, bindex, value, vl); // expected-error {{builtin requires at least one of the following extensions: 64bit}}
 }
 
 void test_xsfvcp_sf_vc_x_se_u64m1(uint64_t rs1, size_t vl) {
   __riscv_sf_vc_x_se_u64m1(1, 1, 1, rs1, vl); // expected-error {{call to undeclared function '__riscv_sf_vc_x_se_u64m1'}}
 }
 
-void test_xsfvqmaccdod() {
-  __riscv_sf_vqmacc_2x8x2(); // expected-error {{call to undeclared function '__riscv_sf_vqmacc_2x8x2'}}
+void test_xsfvqmaccdod(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  __riscv_sf_vqmacc_2x8x2(vd, vs1, vs2, vl); // expected-error {{builtin requires at least one of the following extensions: xsfvqmaccdod}}
 }
 
-void test_xsfvqmaccqoq() {
-  __riscv_sf_vqmacc_4x8x4(); // expected-error {{call to undeclared function '__riscv_sf_vqmacc_4x8x4'}}
+void test_xsfvqmaccqoq(vint32m1_t vd, vint8m1_t vs1, vint8mf2_t vs2, size_t vl) {
+  __riscv_sf_vqmacc_4x8x4(vd, vs1, vs2, vl); // expected-error {{builtin requires at least one of the following extensions: xsfvqmaccqoq}}
 }
 
-void test_xsfvfwmaccqqq() {
-  __riscv_sf_vfwmacc_4x4x4(); // expected-error {{call to undeclared function '__riscv_sf_vfwmacc_4x4x4'}}
+void test_xsfvfwmaccqqq(vfloat32m4_t vd, vbfloat16m1_t vs1, vbfloat16m2_t vs2, size_t vl) {
+  // expected-error@-1 {{RISC-V type 'vbfloat16m1_t' (aka '__rvv_bfloat16m1_t') requires the 'zvfbfmin' extension}}
+  // expected-error@-2 {{RISC-V type 'vbfloat16m2_t' (aka '__rvv_bfloat16m2_t') requires the 'zvfbfmin' extension}}
+  __riscv_sf_vfwmacc_4x4x4(vd, vs1, vs2, vl); // expected-error {{RISC-V type 'vbfloat16m1_t' (aka '__rvv_bfloat16m1_t') requires the 'zvfbfmin' extension}}
+  // expected-error@-1 {{RISC-V type 'vbfloat16m2_t' (aka '__rvv_bfloat16m2_t') requires the 'zvfbfmin' extension}}
+  // expected-error@-2 {{RISC-V type '__rvv_bfloat16m1_t' requires the 'zvfbfmin' extension}}
+  // expected-error@-3 {{RISC-V type '__rvv_bfloat16m2_t' requires the 'zvfbfmin' extension}}
+  // expected-error@-4 {{builtin requires at least one of the following extensions: xsfvfwmaccqqq}}
 }
 
-void test_xsfvfnrclipxfqf() {
-  __riscv_sf_vfnrclip_x_f_qf(); // expected-error {{call to undeclared function '__riscv_sf_vfnrclip_x_f_qf'}}
+void test_xsfvfnrclipxfqf(vfloat32m1_t vs2, float rs1, size_t vl) {
+  __riscv_sf_vfnrclip_x_f_qf(vs2, rs1, vl); // expected-error {{builtin requires at least one of the following extensions: xsfvfnrclipxfqf}}
 }
 
-void test_xsfvfnrclipxufqf() {
-  __riscv_sf_vfnrclip_xu_f_qf(); // expected-error {{call to undeclared function '__riscv_sf_vfnrclip_xu_f_qf'}}
+void test_xsfvfnrclipxufqf(vfloat32mf2_t vs2, float rs1, size_t vl) {
+  __riscv_sf_vfnrclip_xu_f_qf(vs2, rs1, 2, vl); // expected-error {{builtin requires at least one of the following extensions: xsfvfnrclipxfqf}}
 }
 
-void test_zvfbfwma_vfwmaccbf16() {
-  __riscv_vfwmaccbf16(); // expected-error {{call to undeclared function '__riscv_vfwmaccbf16'}}
+void test_zvfbfwma_vfwmaccbf16(vfloat32m4_t vd, __bf16 vs1, vbfloat16m2_t vs2, size_t vl) {
+  // expected-error@-1 {{RISC-V type 'vbfloat16m2_t' (aka '__rvv_bfloat16m2_t') requires the 'zvfbfmin' extension}}
+  __riscv_vfwmaccbf16(vd, vs1, vs2, vl); // expected-error {{RISC-V type 'vbfloat16m2_t' (aka '__rvv_bfloat16m2_t') requires the 'zvfbfmin' extension}}
+  // expected-error@-1 {{RISC-V type '__rvv_bfloat16m2_t' requires the 'zvfbfmin' extension}}
+  // expected-error@-2 {{builtin requires at least one of the following extensions: zvfbfwma}}
 }
diff --git a/external/llvm-project/clang/test/Sema/typo-correction-ambiguity.cpp b/external/llvm-project/clang/test/Sema/typo-correction-ambiguity.cpp
index 9dcff3d68c82..b2dae1d7696c 100644
--- a/external/llvm-project/clang/test/Sema/typo-correction-ambiguity.cpp
+++ b/external/llvm-project/clang/test/Sema/typo-correction-ambiguity.cpp
@@ -18,12 +18,12 @@ void testAmbiguousNoSuggestions()
 
 namespace MultipleCorrectionsButNotAmbiguous
 {
-  int PrefixType_Name(int value);  // expected-note {{'PrefixType_Name' declared here}}
+  int PrefixType_Name(int value);
   int PrefixType_MIN();
   int PrefixType_MAX();
 };
 
 int testMultipleCorrectionsButNotAmbiguous() {
-  int val = MultipleCorrectionsButNotAmbiguous::PrefixType_Enum(0);  // expected-error {{no member named 'PrefixType_Enum' in namespace 'MultipleCorrectionsButNotAmbiguous'; did you mean 'PrefixType_Name'?}}
+  int val = MultipleCorrectionsButNotAmbiguous::PrefixType_Enum(0);  // expected-error {{no member named 'PrefixType_Enum' in namespace 'MultipleCorrectionsButNotAmbiguous'}}
   return val;
 }
diff --git a/external/llvm-project/clang/test/Sema/typo-correction-no-hang.c b/external/llvm-project/clang/test/Sema/typo-correction-no-hang.c
index e6041704ff32..da234a2c7373 100644
--- a/external/llvm-project/clang/test/Sema/typo-correction-no-hang.c
+++ b/external/llvm-project/clang/test/Sema/typo-correction-no-hang.c
@@ -2,16 +2,15 @@
 
 // PR50797
 struct a {
-  int xxx; // expected-note {{'xxx' declared here}}
+  int xxx;
 };
 
 int g_107;
 int g_108;
 int g_109;
 
-struct a g_999; // expected-note 4{{'g_999' declared here}}
+struct a g_999;
 
-void b(void) { (g_910.xxx = g_910.xxx); } //expected-error 2{{use of undeclared identifier 'g_910'; did you mean 'g_999'}}
+void b(void) { (g_910.xxx = g_910.xxx); } //expected-error 2{{use of undeclared identifier 'g_910'}}
 
-void c(void) { (g_910.xxx = g_910.xxx1); } //expected-error 2{{use of undeclared identifier 'g_910'; did you mean 'g_999'}} \
-                                             expected-error {{no member named 'xxx1' in 'struct a'; did you mean 'xxx'}}
+void c(void) { (g_910.xxx = g_910.xxx1); } //expected-error 2{{use of undeclared identifier 'g_910'}}
diff --git a/external/llvm-project/clang/test/Sema/typo-correction-no-hang.cpp b/external/llvm-project/clang/test/Sema/typo-correction-no-hang.cpp
index 3c591645be25..34b8486bed90 100644
--- a/external/llvm-project/clang/test/Sema/typo-correction-no-hang.cpp
+++ b/external/llvm-project/clang/test/Sema/typo-correction-no-hang.cpp
@@ -8,10 +8,12 @@ struct rdar38642201 {
 
 void rdar38642201_callee(int x, int y);
 void rdar38642201_caller() {
-  struct rdar38642201 structVar;
+  struct rdar38642201 structVar;      //expected-note 2{{'structVar' declared here}}
   rdar38642201_callee(
-      structVar1.fieldName1.member1,  //expected-error{{use of undeclared identifier 'structVar1'}}
-      structVar2.fieldName2.member2); //expected-error{{use of undeclared identifier 'structVar2'}}
+      structVar1.fieldName1.member1,  //expected-error{{use of undeclared identifier 'structVar1'}} \
+                                        expected-error{{no member named 'fieldName1' in 'rdar38642201'}}
+      structVar2.fieldName2.member2); //expected-error{{use of undeclared identifier 'structVar2'}} \
+                                        expected-error{{no member named 'fieldName2' in 'rdar38642201'}}
 }
 
 // Similar reproducer.
@@ -20,7 +22,7 @@ class A {
   int minut() const = delete;
   int hour() const = delete;
 
-  int longit() const; //expected-note{{'longit' declared here}}
+  int longit() const;
   int latit() const;
 };
 
@@ -35,6 +37,6 @@ int Foo(const B &b) {
 }
 
 int Bar(const B &b) {
-  return b.depar().longitude() + //expected-error{{no member named 'longitude' in 'A'; did you mean 'longit'?}}
+  return b.depar().longitude() + //expected-error{{no member named 'longitude' in 'A'}}
          b.depar().latitude();   //expected-error{{no member named 'latitude' in 'A'}}
 }
diff --git a/external/llvm-project/clang/test/Sema/typo-correction-recursive.cpp b/external/llvm-project/clang/test/Sema/typo-correction-recursive.cpp
index b39beb5493f6..a7d7127564b7 100644
--- a/external/llvm-project/clang/test/Sema/typo-correction-recursive.cpp
+++ b/external/llvm-project/clang/test/Sema/typo-correction-recursive.cpp
@@ -8,13 +8,13 @@
 class DeepClass
 {
 public:
-  void trigger() const;  // expected-note {{'trigger' declared here}}
+  void trigger() const;
 };
 
 class Y
 {
 public:
-  const DeepClass& getX() const { return m_deepInstance; }  // expected-note {{'getX' declared here}}
+  const DeepClass& getX() const { return m_deepInstance; }
 private:
   DeepClass m_deepInstance;
   int m_n;
@@ -23,7 +23,7 @@ class Y
 class Z
 {
 public:
-  const Y& getY0() const { return m_y0; }  // expected-note {{'getY0' declared here}}
+  const Y& getY0() const { return m_y0; }
   const Y& getActiveY() const { return m_y0; }
 
 private:
@@ -35,9 +35,9 @@ Z z_obj;
 
 void testMultipleCorrections()
 {
-  z_obj.getY2().  // expected-error {{no member named 'getY2' in 'Z'; did you mean 'getY0'}}
-      getM().     // expected-error {{no member named 'getM' in 'Y'; did you mean 'getX'}}
-      triggee();  // expected-error {{no member named 'triggee' in 'DeepClass'; did you mean 'trigger'}}
+  z_obj.getY2().  // expected-error {{no member named 'getY2' in 'Z'}}
+      getM().
+      triggee();
 }
 
 void testNoCorrections()
@@ -53,19 +53,19 @@ struct A {
   C get_me_a_C();
 };
 struct B {
-  D get_me_a_D();  // expected-note {{'get_me_a_D' declared here}}
+  D get_me_a_D();
 };
 class Scope {
 public:
   A make_an_A();
-  B make_a_B();  // expected-note {{'make_a_B' declared here}}
+  B make_a_B();
 };
 
 Scope scope_obj;
 
 int testDiscardedCorrections() {
-  return scope_obj.make_an_E().  // expected-error {{no member named 'make_an_E' in 'Scope'; did you mean 'make_a_B'}}
-      get_me_a_Z().value;        // expected-error {{no member named 'get_me_a_Z' in 'B'; did you mean 'get_me_a_D'}}
+  return scope_obj.make_an_E().  // expected-error {{no member named 'make_an_E' in 'Scope'}}
+      get_me_a_Z().value;
 }
 
 class AmbiguousHelper {
@@ -120,13 +120,13 @@ int testDeepAmbiguity() {
 }
 
 struct Dog {
-  int age;  //expected-note{{'age' declared here}}
-  int size; //expected-note{{'size' declared here}}
+  int age;
+  int size;
 };
 
 int from_dog_years(int DogYears, int DogSize);
 int get_dog_years() {
   struct Dog doggo;
-  return from_dog_years(doggo.agee,   //expected-error{{no member named 'agee' in 'Dog'; did you mean 'age'}}
-                        doggo.sizee); //expected-error{{no member named 'sizee' in 'Dog'; did you mean 'size'}}
+  return from_dog_years(doggo.agee,   //expected-error{{no member named 'agee' in 'Dog'}}
+                        doggo.sizee); //expected-error{{no member named 'sizee' in 'Dog'}}
 }
diff --git a/external/llvm-project/clang/test/Sema/typo-correction.c b/external/llvm-project/clang/test/Sema/typo-correction.c
index 4157207a9ac4..510a67e725f9 100644
--- a/external/llvm-project/clang/test/Sema/typo-correction.c
+++ b/external/llvm-project/clang/test/Sema/typo-correction.c
@@ -50,10 +50,12 @@ void fn1(void) {
   cabs(errij);  // expected-error {{use of undeclared identifier 'errij'}}
 }
 
-extern long afunction(int);
+extern long afunction(int); // expected-note {{'afunction' declared here}} \
+                               expected-note {{passing argument to parameter here}}
 void fn2(void) {
   f(THIS_IS_AN_ERROR,       // expected-error {{use of undeclared identifier 'THIS_IS_AN_ERROR'}}
-    afunction(afunction_)); // expected-error {{use of undeclared identifier 'afunction_'}}
+    afunction(afunction_)); // expected-error {{use of undeclared identifier 'afunction_'}} \
+                               expected-error {{incompatible pointer to integer conversion passing 'long (int)' to parameter of type 'int'}}
 }
 
 int d = X ? d : L; // expected-error 2 {{use of undeclared identifier}}
@@ -94,22 +96,24 @@ struct rdar38642201 {
 
 void rdar38642201_callee(int x, int y);
 void rdar38642201_caller(void) {
-  struct rdar38642201 structVar;
+  struct rdar38642201 structVar;     // expected-note 2{{'structVar' declared here}}
   rdar38642201_callee(
-      structVar1.fieldName1.member1, //expected-error{{use of undeclared identifier 'structVar1'}}
-      structVar2.fieldName2.member2); //expected-error{{use of undeclared identifier 'structVar2'}}
+      structVar1.fieldName1.member1, //expected-error{{use of undeclared identifier 'structVar1'}} \
+                                       expected-error{{no member named 'fieldName1' in 'struct rdar38642201'}}
+      structVar2.fieldName2.member2); //expected-error{{use of undeclared identifier 'structVar2'}} \
+                                        expected-error{{no member named 'fieldName2' in 'struct rdar38642201'}}
 }
 
 void PR40286_g(int x, int y);
 void PR40286_h(int x, int y, int z);
-void PR40286_1(int the_value) {
-  PR40286_g(the_walue); // expected-error {{use of undeclared identifier 'the_walue'}}
+void PR40286_1(int the_value) { // expected-note {{'the_value' declared here}}
+  PR40286_g(the_walue, 0); // expected-error {{use of undeclared identifier 'the_walue'}}
 }
-void PR40286_2(int the_value) {
-  PR40286_h(the_value, the_walue); // expected-error {{use of undeclared identifier 'the_walue'}}
+void PR40286_2(int the_value) { // expected-note {{'the_value' declared here}}
+  PR40286_h(the_value, the_walue, 0); // expected-error {{use of undeclared identifier 'the_walue'}}
 }
-void PR40286_3(int the_value) {
-  PR40286_h(the_walue); // expected-error {{use of undeclared identifier 'the_walue'}}
+void PR40286_3(int the_value) { // expected-note {{'the_value' declared here}}
+  PR40286_h(the_walue, 0, 0); // expected-error {{use of undeclared identifier 'the_walue'}}
 }
 void PR40286_4(int the_value) { // expected-note {{'the_value' declared here}}
   PR40286_h(the_value, the_value, the_walue); // expected-error {{use of undeclared identifier 'the_walue'; did you mean 'the_value'?}}
diff --git a/external/llvm-project/clang/test/Sema/unknown-attributes.c b/external/llvm-project/clang/test/Sema/unknown-attributes.c
index a701650c9e05..4711c9fa667b 100644
--- a/external/llvm-project/clang/test/Sema/unknown-attributes.c
+++ b/external/llvm-project/clang/test/Sema/unknown-attributes.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -Wunknown-attributes -fsyntax-only -verify %s
-// RUN: %clang_cc1 -x c++ -Wunknown-attributes -fsyntax-only -verify %s
+// RUN: %clang_cc1 -Wunknown-attributes -fsyntax-only -verify=expected,c %s
+// RUN: %clang_cc1 -x c++ -Wunknown-attributes -fsyntax-only -verify=expected,cxx %s
 
 [[gmu::deprected]] // expected-warning {{unknown attribute 'gmu::deprected' ignored; did you mean 'gnu::deprecated'?}}
 int f1(void) {
@@ -20,3 +20,10 @@ int f3(void) {
 int f4(void) {
   return 0;
 }
+
+[[using gnu : deprected]] // c-error {{expected ','}} \
+                          // c-warning {{unknown attribute 'using' ignored}} \
+                          // cxx-warning {{unknown attribute 'gnu::deprected' ignored; did you mean 'gnu::deprecated'?}}
+int f5(void) {
+  return 0;
+}
diff --git a/external/llvm-project/clang/test/Sema/warn-fortify-source.c b/external/llvm-project/clang/test/Sema/warn-fortify-source.c
index a12460b963cd..f48ea0907c65 100644
--- a/external/llvm-project/clang/test/Sema/warn-fortify-source.c
+++ b/external/llvm-project/clang/test/Sema/warn-fortify-source.c
@@ -71,6 +71,14 @@ void call_strcpy_nowarn(void) {
   __builtin_strcpy(dst, src);
 }
 
+void call_stpcpy(void) {
+  const char *const src = "abcd";
+  char dst1[5];
+  char dst2[4];
+  __builtin_stpcpy(dst1, src);
+  __builtin_stpcpy(dst2, src); // expected-warning {{'stpcpy' will always overflow; destination buffer has size 4, but the source string has length 5 (including NUL byte)}}
+}
+
 void call_memmove(void) {
   char s1[10], s2[20];
   __builtin_memmove(s2, s1, 20);
diff --git a/external/llvm-project/clang/test/Sema/warn-unreachable_crash.cpp b/external/llvm-project/clang/test/Sema/warn-unreachable_crash.cpp
new file mode 100644
index 000000000000..628abcc83f81
--- /dev/null
+++ b/external/llvm-project/clang/test/Sema/warn-unreachable_crash.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -verify -Wunreachable-code %s
+
+// Previously this test will crash
+static void test(__fp16& x) {
+  if (x != 0 || x != 1.0) { // expected-note{{}} no-crash
+      x = 0.9;
+    } else
+      x = 0.8; // expected-warning{{code will never be executed}}
+}
+
+static void test2(__fp16& x) {
+  if (x != 1 && x == 1.0) { // expected-note{{}} no-crash
+      x = 0.9; // expected-warning{{code will never be executed}}
+    } else
+      x = 0.8;
+}
diff --git a/external/llvm-project/clang/test/Sema/zvk-invalid-features.c b/external/llvm-project/clang/test/Sema/zvk-invalid-features.c
index 05dd6498ce01..4b666b4480f7 100644
--- a/external/llvm-project/clang/test/Sema/zvk-invalid-features.c
+++ b/external/llvm-project/clang/test/Sema/zvk-invalid-features.c
@@ -3,48 +3,48 @@
 
 #include <riscv_vector.h>
 
-void test_zvk_features() {
+void test_zvk_features(vuint32m4_t vd, vuint32m4_t vs2, vuint32m4_t vs1, vuint64m1_t vs2_64, vuint64m1_t vs1_64, size_t vl) {
   // zvbb
-  __riscv_vbrev(); // expected-error {{call to undeclared function '__riscv_vbrev'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vclz(); // expected-error {{call to undeclared function '__riscv_vclz'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vctz(); // expected-error {{call to undeclared function '__riscv_vctz'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vcpopv(); // expected-error {{call to undeclared function '__riscv_vcpopv'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vwsll(); // expected-error {{call to undeclared function '__riscv_vwsll'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vbrev(vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvbb}}
+  __riscv_vclz(vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvbb}}
+  __riscv_vctz(vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvbb}}
+  __riscv_vcpop(vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvbb}}
+  __riscv_vwsll(vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvbb}}
 
   // zvbc
-  __riscv_vclmul(); // expected-error {{call to undeclared function '__riscv_vclmul'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vclmulh(); // expected-error {{call to undeclared function '__riscv_vclmulh'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vclmul(vs2_64, vs1_64, vl); // expected-error {{builtin requires at least one of the following extensions: zvbc}}
+  __riscv_vclmulh(vs2_64, vs1_64, vl); // expected-error {{builtin requires at least one of the following extensions: zvbc}}
 
   // zvkb
-  __riscv_vandn(); // expected-error {{call to undeclared function '__riscv_vandn'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vbrev8(); // expected-error {{call to undeclared function '__riscv_vbrev8'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vrev8(); // expected-error {{call to undeclared function '__riscv_vrev8'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vrol(); // expected-error {{call to undeclared function '__riscv_vrol'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vror(); // expected-error {{call to undeclared function '__riscv_vror'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vandn(vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvkb}}
+  __riscv_vbrev8(vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvkb}}
+  __riscv_vrev8(vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvkb}}
+  __riscv_vrol(vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvkb}}
+  __riscv_vror(vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvkb}}
 
   // zvkg
-  __riscv_vghsh(); // expected-error {{call to undeclared function '__riscv_vghsh'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vgmul(); // expected-error {{call to undeclared function '__riscv_vgmul'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vghsh(vd, vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvkg}}
+  __riscv_vgmul(vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvkg}}
 
   // zvkned
-  __riscv_vaesdf(); // expected-error {{call to undeclared function '__riscv_vaesdf'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vaesdm(); // expected-error {{call to undeclared function '__riscv_vaesdm'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vaesef(); // expected-error {{call to undeclared function '__riscv_vaesef'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vaesem(); // expected-error {{call to undeclared function '__riscv_vaesem'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vaeskf1(); // expected-error {{call to undeclared function '__riscv_vaeskf1'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vaeskf2(); // expected-error {{call to undeclared function '__riscv_vaeskf2'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vaesz(); // expected-error {{call to undeclared function '__riscv_vaesz'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vaesdf_vv(vd, vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvkned}}
+  __riscv_vaesdm_vv(vd, vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvkned}}
+  __riscv_vaesef_vv(vd, vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvkned}}
+  __riscv_vaesem_vv(vd, vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvkned}}
+  __riscv_vaeskf1(vs2, 0, vl); // expected-error {{builtin requires at least one of the following extensions: zvkned}}
+  __riscv_vaeskf2(vd, vs2, 0, vl); // expected-error {{builtin requires at least one of the following extensions: zvkned}}
+  __riscv_vaesz(vd, vs2, vl); // expected-error {{builtin requires at least one of the following extensions: zvkned}}
 
   // zvknha or zvknhb
-  __riscv_vsha2ch(); // expected-error {{call to undeclared function '__riscv_vsha2ch'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vsha2cl(); // expected-error {{call to undeclared function '__riscv_vsha2cl'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vsha2ms(); // expected-error {{call to undeclared function '__riscv_vsha2ms'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vsha2ch(vd, vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvknha or zvknhb}}
+  __riscv_vsha2cl(vd, vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvknha or zvknhb}}
+  __riscv_vsha2ms(vd, vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvknha or zvknhb}}
 
   //zvksed
-  __riscv_vsm4k(); // expected-error {{call to undeclared function '__riscv_vsm4k'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vsm4r(); // expected-error {{call to undeclared function '__riscv_vsm4r'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vsm4k(vs2, 0, vl); // expected-error {{builtin requires at least one of the following extensions: zvksed}}
+  __riscv_vsm4r_vv(vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvksed}}
 
   // zvksh
-  __riscv_vsm3c(); // expected-error {{call to undeclared function '__riscv_vsm3c'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vsm3me(); // expected-error {{call to undeclared function '__riscv_vsm3me'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vsm3c(vd, vs2, 0, vl); // expected-error {{builtin requires at least one of the following extensions: zvksh}}
+  __riscv_vsm3me(vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvksh}}
 }
diff --git a/external/llvm-project/clang/test/Sema/zvk-invalid-zvknha.c b/external/llvm-project/clang/test/Sema/zvk-invalid-zvknha.c
index a0a4cf8fb86d..f7817aeee2fa 100644
--- a/external/llvm-project/clang/test/Sema/zvk-invalid-zvknha.c
+++ b/external/llvm-project/clang/test/Sema/zvk-invalid-zvknha.c
@@ -3,9 +3,9 @@
 
 #include <riscv_vector.h>
 
-void test_zvk_features() {
+void test_zvk_features(vuint64m1_t vd, vuint64m1_t vs2, vuint64m1_t vs1, size_t vl) {
   // zvknhb
-  __riscv_vsha2ch_vv_u64m1(); // expected-error {{call to undeclared function '__riscv_vsha2ch_vv_u64m1'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vsha2cl_vv_u64m1(); // expected-error {{call to undeclared function '__riscv_vsha2cl_vv_u64m1'; ISO C99 and later do not support implicit function declarations}}
-  __riscv_vsha2ms_vv_u64m1(); // expected-error {{call to undeclared function '__riscv_vsha2ms_vv_u64m1'; ISO C99 and later do not support implicit function declarations}}
+  __riscv_vsha2ch_vv_u64m1(vd, vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvknhb}}
+  __riscv_vsha2cl_vv_u64m1(vd, vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvknhb}}
+  __riscv_vsha2ms_vv_u64m1(vd, vs2, vs1, vl); // expected-error {{builtin requires at least one of the following extensions: zvknhb}}
 }
diff --git a/external/llvm-project/clang/test/Sema/zvk-target-attributes.c b/external/llvm-project/clang/test/Sema/zvk-target-attributes.c
index dad2e5b16ac8..ce3061201730 100644
--- a/external/llvm-project/clang/test/Sema/zvk-target-attributes.c
+++ b/external/llvm-project/clang/test/Sema/zvk-target-attributes.c
@@ -9,3 +9,27 @@ __attribute__((target("arch=+zvl128b")))
 void test_zvk_features(vuint32m1_t vd, vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
   __riscv_vsha2ch_vv_u32m1(vd, vs2, vs1, vl);
 }
+
+__attribute__((target("arch=+v,+zvkn")))
+vuint32m4_t testcase1(vuint32m4_t pt, vuint32m1_t rk, size_t vl)
+{
+  return __riscv_vaesz_vs_u32m1_u32m4(pt, rk, vl);
+}
+
+__attribute__((target("arch=+v,+zvknc")))
+vuint32m4_t testcase2(vuint32m4_t pt, vuint32m1_t rk, size_t vl)
+{
+  return __riscv_vaesz_vs_u32m1_u32m4(pt, rk, vl);
+}
+
+__attribute__((target("arch=+v,+zvkned")))
+vuint32m4_t testcase3(vuint32m4_t pt, vuint32m1_t rk, size_t vl)
+{
+  return __riscv_vaesz_vs_u32m1_u32m4(pt, rk, vl);
+}
+
+__attribute__((target("arch=+v,+zvkng")))
+vuint32m4_t testcase4(vuint32m4_t pt, vuint32m1_t rk, size_t vl)
+{
+  return __riscv_vaesz_vs_u32m1_u32m4(pt, rk, vl);
+}
diff --git a/external/llvm-project/clang/test/SemaCUDA/bad-attributes.cu b/external/llvm-project/clang/test/SemaCUDA/bad-attributes.cu
index 8ac6fa1ab5c2..acb2f38a7528 100644
--- a/external/llvm-project/clang/test/SemaCUDA/bad-attributes.cu
+++ b/external/llvm-project/clang/test/SemaCUDA/bad-attributes.cu
@@ -50,6 +50,14 @@ __global__ __device__ void z11();  // expected-error {{attributes are not compat
 __global__ __host__ void z12();  // expected-error {{attributes are not compatible}}
 // expected-note@-1 {{conflicting attribute is here}}
 
+// Make sure GPU-side variables do not allow __attribute((address_space(N)))
+// expected-error@+1 {{__constant__, __device__, and __shared__ variables must use default address space}}
+__shared__ __attribute__((address_space(999))) int as_s;
+// expected-error@+1 {{__constant__, __device__, and __shared__ variables must use default address space}}
+__device__ __attribute__((address_space(999))) int as_d;
+// expected-error@+1 {{__constant__, __device__, and __shared__ variables must use default address space}}
+__constant__ __attribute__((address_space(999))) int as_c;
+
 struct S {
   __global__ void foo() {};  // expected-error {{must be a free function or static member function}}
   __global__ static void bar(); // expected-warning {{kernel function 'bar' is a member function}}
diff --git a/external/llvm-project/clang/test/SemaCXX/arrow-operator.cpp b/external/llvm-project/clang/test/SemaCXX/arrow-operator.cpp
index 295dea3c1756..a789c4e36e4c 100644
--- a/external/llvm-project/clang/test/SemaCXX/arrow-operator.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/arrow-operator.cpp
@@ -47,23 +47,22 @@ class wrapped_ptr {
  public:
   wrapped_ptr(T* ptr) : ptr_(ptr) {}
   T* operator->() { return ptr_; }
-  void Check(); // expected-note {{'Check' declared here}}
+  void Check();
  private:
   T *ptr_;
 };
 
 class Worker {
  public:
-  void DoSomething(); // expected-note {{'DoSomething' declared here}}
+  void DoSomething();
   void Chuck();
 };
 
 void test() {
   wrapped_ptr<Worker> worker(new Worker);
   worker.DoSomething(); // expected-error {{no member named 'DoSomething' in 'arrow_suggest::wrapped_ptr<arrow_suggest::Worker>'; did you mean to use '->' instead of '.'?}}
-  worker.DoSamething(); // expected-error {{no member named 'DoSamething' in 'arrow_suggest::wrapped_ptr<arrow_suggest::Worker>'; did you mean to use '->' instead of '.'?}} \
-                        // expected-error {{no member named 'DoSamething' in 'arrow_suggest::Worker'; did you mean 'DoSomething'?}}
-  worker.Chuck(); // expected-error {{no member named 'Chuck' in 'arrow_suggest::wrapped_ptr<arrow_suggest::Worker>'; did you mean 'Check'?}}
+  worker.DoSamething(); // expected-error {{no member named 'DoSamething' in 'arrow_suggest::wrapped_ptr<arrow_suggest::Worker>'}}
+  worker.Chuck(); // expected-error {{no member named 'Chuck' in 'arrow_suggest::wrapped_ptr<arrow_suggest::Worker>'}}
 }
 
 } // namespace arrow_suggest
diff --git a/external/llvm-project/clang/test/SemaCXX/attr-non-x86-no_caller_saved_registers.cpp b/external/llvm-project/clang/test/SemaCXX/attr-non-x86-no_caller_saved_registers.cpp
index 00fa5bd7336b..acd9846bb20f 100644
--- a/external/llvm-project/clang/test/SemaCXX/attr-non-x86-no_caller_saved_registers.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/attr-non-x86-no_caller_saved_registers.cpp
@@ -11,7 +11,7 @@ __attribute__((no_caller_saved_registers(999))) void bar(int *) {} // expected-w
 
 __attribute__((no_caller_saved_registers)) void foo(int *){} // expected-warning {{unknown attribute 'no_caller_saved_registers' ignored}}
 
-[[gnu::no_caller_saved_registers]] void foo2(int *) {} // expected-warning {{unknown attribute 'no_caller_saved_registers' ignored}}
+[[gnu::no_caller_saved_registers]] void foo2(int *) {} // expected-warning {{unknown attribute 'gnu::no_caller_saved_registers' ignored}}
 
 typedef __attribute__((no_caller_saved_registers)) void (*foo3)(int *); // expected-warning {{unknown attribute 'no_caller_saved_registers' ignored}}
 
diff --git a/external/llvm-project/clang/test/SemaCXX/attr-trivial-abi.cpp b/external/llvm-project/clang/test/SemaCXX/attr-trivial-abi.cpp
index 0347dc37e839..1d0a9be4ca30 100644
--- a/external/llvm-project/clang/test/SemaCXX/attr-trivial-abi.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/attr-trivial-abi.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++11
-// RUN: %clang_cc1 -fsyntax-only -verify %s -triple x86_64-windows-msvc -std=c++11
-// RUN: %clang_cc1 -fsyntax-only -verify %s -triple x86_64-scei-ps4 -std=c++11
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,itanium %s -triple x86_64-unknown-linux -std=c++11
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,windows %s -triple x86_64-windows-msvc -std=c++11
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,scei %s -triple x86_64-scei-ps4 -std=c++11
 
 
 void __attribute__((trivial_abi)) foo(); // expected-warning {{'trivial_abi' attribute only applies to classes}}
@@ -165,7 +165,10 @@ static_assert(!__builtin_is_cpp_trivially_relocatable(CopyMoveDeleted), "");
 
 #endif
 
-struct __attribute__((trivial_abi)) S18 { // expected-warning {{'trivial_abi' cannot be applied to 'S18'}} expected-note {{copy constructors and move constructors are all deleted}}
+struct __attribute__((trivial_abi)) S18 { // expected-warning {{'trivial_abi' cannot be applied to 'S18'}} \
+                                          // itanium-note {{'trivial_abi' is disallowed on 'S18' because it has a field of a non-trivial class type}} \
+                                          // windows-note {{'trivial_abi' is disallowed on 'S18' because it has a field of a non-trivial class type}} \
+                                          // scei-note {{'trivial_abi' is disallowed on 'S18' because its copy constructors and move constructors are all deleted}}
   CopyMoveDeleted a;
 };
 #ifdef __ORBIS__
@@ -195,7 +198,10 @@ struct __attribute__((trivial_abi)) MoveDeleted {
 };
 static_assert(__is_trivially_relocatable(MoveDeleted), ""); // expected-warning{{deprecated}}
 static_assert(!__builtin_is_cpp_trivially_relocatable(MoveDeleted), "");
-struct __attribute__((trivial_abi)) S19 { // expected-warning {{'trivial_abi' cannot be applied to 'S19'}} expected-note {{copy constructors and move constructors are all deleted}}
+struct __attribute__((trivial_abi)) S19 { // expected-warning {{'trivial_abi' cannot be applied to 'S19'}} \
+                                          // itanium-note {{'trivial_abi' is disallowed on 'S19' because its copy constructors and move constructors are all deleted}} \
+                                          // windows-note {{'trivial_abi' is disallowed on 'S19' because it has a field of a non-trivial class type}} \
+                                          // scei-note {{'trivial_abi' is disallowed on 'S19' because its copy constructors and move constructors are all deleted}}
   CopyDeleted a;
   MoveDeleted b;
 };
diff --git a/external/llvm-project/clang/test/SemaCXX/builtin-get-vtable-pointer.cpp b/external/llvm-project/clang/test/SemaCXX/builtin-get-vtable-pointer.cpp
new file mode 100644
index 000000000000..273f9c3b4c66
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaCXX/builtin-get-vtable-pointer.cpp
@@ -0,0 +1,123 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2a %s
+
+namespace basic {
+struct ForwardDeclaration; // expected-note{{forward declaration of 'basic::ForwardDeclaration'}}
+                           // expected-note@-1{{forward declaration of 'basic::ForwardDeclaration'}}
+struct NonPolymorphic {};
+struct Polymorphic {
+  virtual ~Polymorphic();
+};
+
+template <typename T>
+struct Foo {
+  virtual ~Foo();
+};
+
+template <>
+struct Foo<int> {
+};
+
+template <typename T>
+struct Bar {
+  using SubType = typename T::SubType;
+  SubType *ty() const;
+};
+
+struct Thing1 {
+  using SubType = Thing1;
+};
+
+struct Thing2 {
+  using SubType = Thing2;
+  virtual ~Thing2();
+};
+
+struct Thing3 {
+  using SubType = int;
+};
+
+struct Thing4 {
+  using SubType = Polymorphic;
+};
+
+struct Thing5 {
+  using SubType = NonPolymorphic;
+};
+
+struct Thing6 {
+  using SubType = ForwardDeclaration;
+};
+
+template <typename T>
+const void *getThing(const Bar<T> *b = nullptr) {
+  return __builtin_get_vtable_pointer(b->ty()); // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'SubType *' (aka 'int *') was provided}}
+                                                // expected-error@-1{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'Thing1' has no virtual methods}}
+                                                // expected-error@-2{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'NonPolymorphic' has no virtual methods}}
+                                                // expected-error@-3{{__builtin_get_vtable_pointer requires an argument with a complete type, but 'SubType' (aka 'basic::ForwardDeclaration') is incomplete}}
+}
+template <typename>
+struct IncompleteTemplate; // expected-note{{template is declared here}}
+template <typename>
+struct MonomorphicTemplate {
+};
+template <typename>
+struct PolymorphicTemplate {
+  virtual ~PolymorphicTemplate();
+};
+
+void test_function(int);    // expected-note{{possible target for call}}
+                            // expected-note@-1{{possible target for call}}
+void test_function(double); // expected-note{{possible target for call}}
+                            // expected-note@-1{{possible target for call}}
+
+void getVTablePointer() {
+  ForwardDeclaration *fd = nullptr;
+  NonPolymorphic np;
+  Polymorphic p;
+  NonPolymorphic np_array[1];
+  Polymorphic p_array[1];
+  __builtin_get_vtable_pointer(0);             // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'int' was provided}}
+  __builtin_get_vtable_pointer(nullptr);       // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'std::nullptr_t' was provided}}
+  __builtin_get_vtable_pointer(0.5);           // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'double' was provided}}
+  __builtin_get_vtable_pointer(fd);            // expected-error{{__builtin_get_vtable_pointer requires an argument with a complete type, but 'ForwardDeclaration' is incomplete}}
+  __builtin_get_vtable_pointer(np);            // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'NonPolymorphic' was provided}}
+  __builtin_get_vtable_pointer(&np);           // expected-error{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'NonPolymorphic' has no virtual methods}}
+  __builtin_get_vtable_pointer(p);             // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'Polymorphic' was provided}}
+  __builtin_get_vtable_pointer(&p);            // expected-warning{{ignoring return value of function declared with const attribute}}
+  __builtin_get_vtable_pointer(p_array);       // expected-warning{{ignoring return value of function declared with const attribute}}
+  __builtin_get_vtable_pointer(&p_array);      // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'Polymorphic (*)[1]' was provided}}
+  __builtin_get_vtable_pointer(np_array);      // expected-error{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'NonPolymorphic' has no virtual methods}}
+  __builtin_get_vtable_pointer(&np_array);     // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'NonPolymorphic (*)[1]' was provided}}
+  __builtin_get_vtable_pointer(test_function); // expected-error{{reference to overloaded function could not be resolved; did you mean to call it?}}
+                                               // expected-error@-1{{reference to overloaded function could not be resolved; did you mean to call it?}}
+  Foo<double> Food;
+  Foo<int> Fooi;
+  __builtin_get_vtable_pointer(Food); // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'Foo<double>' was provided}}
+  (void)__builtin_get_vtable_pointer(&Food);
+  __builtin_get_vtable_pointer(Fooi);  // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'Foo<int>' was provided}}
+  __builtin_get_vtable_pointer(&Fooi); // expected-error{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'Foo<int>' has no virtual methods}}
+
+  IncompleteTemplate<bool> *incomplete = nullptr;
+  (void)__builtin_get_vtable_pointer(incomplete); // expected-error{{implicit instantiation of undefined template 'basic::IncompleteTemplate<bool>'}}
+  PolymorphicTemplate<bool> *ptb = nullptr;
+  MonomorphicTemplate<bool> *mtb = nullptr;
+  PolymorphicTemplate<int> pti;
+  MonomorphicTemplate<int> mti;
+  PolymorphicTemplate<float> ptf;
+  MonomorphicTemplate<float> mtf;
+  (void)__builtin_get_vtable_pointer(ptb);
+  __builtin_get_vtable_pointer(mtb); // expected-error{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'MonomorphicTemplate<bool>' has no virtual methods}}
+  __builtin_get_vtable_pointer(pti); // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'PolymorphicTemplate<int>' was provided}}
+  __builtin_get_vtable_pointer(mti); // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'MonomorphicTemplate<int>' was provided}}
+  (void)__builtin_get_vtable_pointer(&ptf);
+  __builtin_get_vtable_pointer(&mtf); // expected-error{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'MonomorphicTemplate<float>' has no virtual methods}}
+
+  getThing<Thing1>(); // expected-note{{in instantiation of function template specialization 'basic::getThing<basic::Thing1>' requested here}}
+  getThing<Thing2>();
+  getThing<Thing3>(); // expected-note{{in instantiation of function template specialization 'basic::getThing<basic::Thing3>' requested here}}
+  getThing<Thing4>();
+  getThing<Thing5>(); // expected-note{{in instantiation of function template specialization 'basic::getThing<basic::Thing5>' requested here}}
+  getThing<Thing6>(); // expected-note{{in instantiation of function template specialization 'basic::getThing<basic::Thing6>' requested here}}
+}
+
+} // namespace basic
diff --git a/external/llvm-project/clang/test/SemaCXX/builtin-is-constant-evaluated.cpp b/external/llvm-project/clang/test/SemaCXX/builtin-is-constant-evaluated.cpp
index a1c003c85f73..c775fe71069d 100644
--- a/external/llvm-project/clang/test/SemaCXX/builtin-is-constant-evaluated.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/builtin-is-constant-evaluated.cpp
@@ -143,3 +143,14 @@ namespace fold_initializer {
   const float A::f = __builtin_is_constant_evaluated();
   static_assert(fold(A::f == 1.0f));
 }
+
+namespace narrowing {
+  struct X { unsigned u; };
+  constexpr int f(X x) {return x.u;}
+  void g() {
+    static_assert(f({0xFFFFFFFFLL + __builtin_is_constant_evaluated()}) == 0);
+    f({0x100000000LL - __builtin_is_constant_evaluated()}); // expected-error {{constant expression evaluates to 4294967296}} \
+    // expected-warning {{implicit conversion}} \
+    // expected-note {{insert an explicit cast to silence this issue}}
+  }
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/concept-crash-on-diagnostic.cpp b/external/llvm-project/clang/test/SemaCXX/concept-crash-on-diagnostic.cpp
index 1efed72522fe..b02930e14570 100644
--- a/external/llvm-project/clang/test/SemaCXX/concept-crash-on-diagnostic.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/concept-crash-on-diagnostic.cpp
@@ -31,7 +31,7 @@ void function() {
 // expected-note@#3 {{checking the satisfaction of concept 'convertible_to<bool, bool>'}}
 // expected-note@#2 {{substituting template arguments into constraint expression here}}
 // expected-note@#5 {{checking constraint satisfaction for template 'compare<Object *, Object *>'}}
-// expected-note@#5 {{in instantiation of function template specialization 'compare<Object *, Object *>' requested here}}
+// expected-note@#5 {{while substituting deduced template arguments into function template 'compare' [with IteratorL = Object *, IteratorR = Object *]}}
 
 // expected-note@#4 {{candidate template ignored: constraints not satisfied [with IteratorL = Object *, IteratorR = Object *]}}
 // We don't know exactly the substituted type for `lhs == rhs`, thus a placeholder 'expr-type' is emitted.
diff --git a/external/llvm-project/clang/test/SemaCXX/constant-expression-cxx11.cpp b/external/llvm-project/clang/test/SemaCXX/constant-expression-cxx11.cpp
index 0a135654fab1..ab4e50072f65 100644
--- a/external/llvm-project/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1888,10 +1888,11 @@ namespace PR15884 {
 }
 
 namespace AfterError {
-  constexpr int error() {
+  constexpr int error() { // pre-cxx23-error {{no return statement in constexpr function}}
     return foobar; // expected-error {{undeclared identifier}}
-  }
-  constexpr int k = error(); // expected-error {{constexpr variable 'k' must be initialized by a constant expression}}
+  } // cxx23-note {{control reached end of constexpr function}}
+  constexpr int k = error(); // cxx23-error {{constexpr variable 'k' must be initialized by a constant expression}} \
+                                cxx23-note {{in call to 'error()'}}
 }
 
 namespace std {
@@ -2513,6 +2514,9 @@ void testValueInRangeOfEnumerationValues() {
   // expected-error@-1 {{constexpr variable 'x2' must be initialized by a constant expression}}
   // expected-note@-2 {{integer value 8 is outside the valid range of values [-8, 7] for the enumeration type 'E1'}}
   E1 x2b = static_cast<E1>(8); // ok, not a constant expression context
+  static_assert(static_cast<E1>(8), "");
+  // expected-error@-1 {{static assertion expression is not an integral constant expression}}
+  // expected-note@-2 {{integer value 8 is outside the valid range of values [-8, 7] for the enumeration type 'E1'}}
 
   constexpr E2 x3 = static_cast<E2>(-8);
   // expected-error@-1 {{constexpr variable 'x3' must be initialized by a constant expression}}
@@ -2551,6 +2555,10 @@ void testValueInRangeOfEnumerationValues() {
   // expected-note@-2 {{integer value 2147483648 is outside the valid range of values [-2147483648, 2147483647] for the enumeration type 'EMaxInt'}}
 
   const NumberType neg_one = (NumberType) ((NumberType) 0 - (NumberType) 1); // ok, not a constant expression context
+  constexpr NumberType neg_one_constexpr = neg_one;
+  // expected-error@-1 {{constexpr variable 'neg_one_constexpr' must be initialized by a constant expression}}
+  // expected-note@-2 {{initializer of 'neg_one' is not a constant expression}}
+  // expected-note@-4 {{declared here}}
 
   CONSTEXPR_CAST_TO_SYSTEM_ENUM_OUTSIDE_OF_RANGE;
   // expected-error@-1 {{constexpr variable 'system_enum' must be initialized by a constant expression}}
diff --git a/external/llvm-project/clang/test/SemaCXX/constant-expression-p2280r4.cpp b/external/llvm-project/clang/test/SemaCXX/constant-expression-p2280r4.cpp
index c14250a268f6..50637917ba21 100644
--- a/external/llvm-project/clang/test/SemaCXX/constant-expression-p2280r4.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/constant-expression-p2280r4.cpp
@@ -250,3 +250,15 @@ namespace uninit_reference_used {
   // expected-note {{in call to 'g5()'}}
 
 }
+
+namespace param_reference {
+  constexpr int arbitrary = -12345;
+  constexpr void f(const int &x = arbitrary) { // expected-note {{declared here}}
+    constexpr const int &v1 = x; // expected-error {{must be initialized by a constant expression}} \
+    // expected-note {{reference to 'x' is not a constant expression}}
+    constexpr const int &v2 = (x, arbitrary); // expected-warning {{left operand of comma operator has no effect}}
+    constexpr int v3 = x; // expected-error {{must be initialized by a constant expression}}
+    static_assert(x==arbitrary); // expected-error {{static assertion expression is not an integral constant expression}}
+    static_assert(&x - &x == 0);
+  }
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/conversion-function.cpp b/external/llvm-project/clang/test/SemaCXX/conversion-function.cpp
index b653a3bf1a1d..717c73c4786e 100644
--- a/external/llvm-project/clang/test/SemaCXX/conversion-function.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/conversion-function.cpp
@@ -458,7 +458,7 @@ namespace PR18234 {
 #endif
   } a;
   A::S s = a; // expected-error {{no viable conversion from 'struct A' to 'A::S'}}
-  A::E e = a;
+  A::E e = a; // expected-note {{'e' declared here}}
   bool k1 = e == A::e; // expected-error {{no member named 'e'}}
   bool k2 = e.n == 0;
 }
diff --git a/external/llvm-project/clang/test/SemaCXX/coroutines.cpp b/external/llvm-project/clang/test/SemaCXX/coroutines.cpp
index 068fdab4bfe3..c9cefeb30c15 100644
--- a/external/llvm-project/clang/test/SemaCXX/coroutines.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/coroutines.cpp
@@ -8,19 +8,16 @@
 // RUN: not %clang_cc1 -std=c++20 -fsyntax-only %s -fcxx-exceptions -fexceptions -Wunused-result 2>&1 | FileCheck %s
 
 void no_coroutine_traits_bad_arg_await() {
-  co_await a; // expected-error {{include <coroutine>}}
-  // expected-error@-1 {{use of undeclared identifier 'a'}}
+  co_await a; // expected-error {{use of undeclared identifier 'a'}}
 }
 
 void no_coroutine_traits_bad_arg_yield() {
-  co_yield a; // expected-error {{include <coroutine>}}
-  // expected-error@-1 {{use of undeclared identifier 'a'}}
+  co_yield a; // expected-error {{use of undeclared identifier 'a'}}
 }
 
 
 void no_coroutine_traits_bad_arg_return() {
-  co_return a; // expected-error {{include <coroutine>}}
-  // expected-error@-1 {{use of undeclared identifier 'a'}}
+  co_return a; // expected-error {{use of undeclared identifier 'a'}}
 }
 
 void no_coroutine_traits() {
@@ -208,8 +205,7 @@ void mixed_yield() {
 
 void mixed_yield_invalid() {
   co_yield blah; // expected-error {{use of undeclared identifier}}
-  // expected-note@-1 {{function is a coroutine due to use of 'co_yield'}}
-  return; // expected-error {{return statement not allowed in coroutine}}
+  return;
 }
 
 void mixed_yield_return_first(bool b) {
@@ -231,8 +227,7 @@ void mixed_return_for_range(bool b, T t) {
 template <class T>
 void mixed_yield_template(T) {
   co_yield blah; // expected-error {{use of undeclared identifier}}
-  // expected-note@-1 {{function is a coroutine due to use of 'co_yield'}}
-  return; // expected-error {{return statement not allowed in coroutine}}
+  return;
 }
 
 template <class T>
@@ -314,10 +309,9 @@ template void mixed_coreturn_template(void_tag, bool, int); // expected-note {{r
 template <class T>
 void mixed_coreturn_template2(bool b, T) {
   if (b)
-    co_return v; // expected-note {{use of 'co_return'}}
-    // expected-error@-1 {{use of undeclared identifier 'v'}}
+    co_return v; // expected-error {{use of undeclared identifier 'v'}}
   else
-    return; // expected-error {{not allowed in coroutine}}
+    return;
 }
 
 struct promise_handle;
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx-delayed-typo-correction-crashes.cpp b/external/llvm-project/clang/test/SemaCXX/cxx-delayed-typo-correction-crashes.cpp
new file mode 100644
index 000000000000..f3aa05153281
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaCXX/cxx-delayed-typo-correction-crashes.cpp
@@ -0,0 +1,67 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+namespace GH138850 {
+void test() {
+  int tmp = add(int, 0, 0);    // expected-error {{expected '(' for function-style cast or type construction}} \
+                                  expected-note {{previous definition is here}}
+  uint tmp = add(uint, 1, 1);  // expected-error {{use of undeclared identifier 'uint'; did you mean 'int'?}} \
+                                  expected-error {{redefinition of 'tmp'}} \
+                                  expected-error {{use of undeclared identifier 'uint'}}
+  call(void, f, (int)tmp);     // expected-error {{expected '(' for function-style cast or type construction}} \
+                                  expected-error {{use of undeclared identifier 'f'}}
+}
+}
+
+namespace GH107840 {
+struct tm {};          // expected-note {{'tm' declared here}}
+
+auto getCache = [&] {  // expected-error {{non-local lambda expression cannot have a capture-default}}
+  ::foo([=] {          // expected-error {{no member named 'foo' in the global namespace}}
+    tms time;          // expected-error {{unknown type name 'tms'; did you mean 'tm'?}}
+    (void)time;
+  });
+};
+}
+
+namespace GH59391 {
+template <typename b> class c {
+  c(b);
+  b e;
+  void f() {
+    for (auto core : a::c(cores)) { // expected-error {{use of undeclared identifier 'cores'}} \
+                                       expected-error {{use of undeclared identifier 'a'}}
+    }
+  }
+};
+}
+
+namespace GH45915 {
+short g_volatile_ushort;                   // expected-note {{'g_volatile_ushort' declared here}}
+namespace a {
+   int b = l_volatile_uwchar.a ::c ::~d<>; // expected-error {{use of undeclared identifier 'l_volatile_uwchar'}} \
+                                              expected-error {{no member named 'd' in namespace 'GH45915::a'}}
+}
+}
+
+namespace GH45891 {
+int a = b.c < enum , > :: template ~d < > [ e; // expected-error {{use of undeclared identifier 'b'}} \
+                                                  expected-error {{expected identifier or '{'}} \
+                                                  expected-error {{expected ';' after top level declarator}}
+}
+
+namespace GH32903 {
+void
+B(
+  char cat_dog_3, char cat_dog_2, char cat_dog_1, char cat_dog_0, char pigeon_dog_3, char pigeon_dog_2,
+  char pigeon_dog_1, char pigeon_dog_0, short &elefant15_lion, short &elefant14_lion, short &elefant13_lion,       // expected-note 3 {{declared here}}
+  short &elefant12_lion, short &elefant11_lion, short &elefant10_lion, short &elefant9_lion, short &elefant8_lion, // expected-note 5 {{declared here}}
+  short &elefant7_lion, short &elefant6_lion, short &elefant5_lion, short &elefant4_lion, short &elefant3_lion,    // expected-note 2 {{declared here}}
+  short &elefant2_lion, short &elefant1_lion, short &elefant0_lion, char& no_animal)
+{
+
+    A(  // FIXME: it's surprising that we don't issue a "use of undeclared identifier" diagnostic for the call itself.
+        elefant_15_lion, elefant_14_lion, elefant_13_lion, elefant_12_lion, elefant_11_lion, elefant_10_lion, elefant_9_lion, // expected-error 7 {{use of undeclared identifier}}
+        elefant_8_lion, elefant_7_lion, elefant_6_lion, elefant_5_lion, elefant_4_lion, elefant_3_lion, elefant_2_lion,       // expected-error 7 {{use of undeclared identifier}}
+        elefant_1_lion, elefant_0_lion, no_animal, other_mammal);                                                             // expected-error 3 {{use of undeclared identifier}}
+}
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx1z-decomposition.cpp b/external/llvm-project/clang/test/SemaCXX/cxx1z-decomposition.cpp
index 95c64bc3b8bf..6ee1249a66c3 100644
--- a/external/llvm-project/clang/test/SemaCXX/cxx1z-decomposition.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/cxx1z-decomposition.cpp
@@ -121,7 +121,8 @@ void for_range() {
 }
 
 int error_recovery() {
-  auto [foobar]; // expected-error {{requires an initializer}}
+  auto [foobar]; // expected-error {{requires an initializer}} \
+                    expected-note {{'foobar' declared here}}
   return foobar_; // expected-error {{undeclared identifier 'foobar_'}}
 }
 
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/external/llvm-project/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index c863cc841af4..aeb02c9e4898 100644
--- a/external/llvm-project/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -196,7 +196,7 @@ struct Foo {
 
 template <int K>
 using Bar = Foo<double, K>; // expected-note {{constraints not satisfied for class template 'Foo'}}
-// expected-note@-1 {{candidate template ignored: could not match}}
+// expected-note@-1 {{candidate template ignored: could not match}} expected-note@-1 {{candidate template ignored: constraints not satisfied}}
 // expected-note@-2 {{implicit deduction guide declared as 'template <int K> requires __is_deducible(test14::Bar, Foo<double, K>) Bar(Foo<double, K>) -> Foo<double, K>'}}
 // expected-note@-3 {{implicit deduction guide declared as 'template <int K> requires __is_deducible(test14::Bar, Foo<double, K>) Bar(const double (&)[K]) -> Foo<double, K>'}}
 double abc[3];
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx20-delayed-typo-correction-crashes.cpp b/external/llvm-project/clang/test/SemaCXX/cxx20-delayed-typo-correction-crashes.cpp
new file mode 100644
index 000000000000..a16a7f8255f7
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaCXX/cxx20-delayed-typo-correction-crashes.cpp
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
+
+#include "Inputs/std-coroutine.h"
+
+namespace GH58172 {
+template<typename Fn>
+int f2(int, Fn&&)
+{
+  return 0;
+}
+
+int f1()
+{
+  return f2(v1, []() -> task<int> {   // expected-error {{no template named 'task'}} \
+                                         expected-error {{use of undeclared identifier 'v1'}}
+    co_return v2;                     // expected-error {{use of undeclared identifier 'v2'}}
+  });
+}
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx23-assume.cpp b/external/llvm-project/clang/test/SemaCXX/cxx23-assume.cpp
index 7f80cdfe7d45..726cb3bff652 100644
--- a/external/llvm-project/clang/test/SemaCXX/cxx23-assume.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/cxx23-assume.cpp
@@ -129,12 +129,12 @@ constexpr int f5() requires (!C<T>) { return 2; } // expected-note 4 {{while che
 
 static_assert(f5<int>() == 1);
 static_assert(f5<D>() == 1); // expected-note 3 {{while checking constraint satisfaction}}
-                             // expected-note@-1 3 {{in instantiation of}}
+                             // expected-note@-1 3 {{while substituting deduced template arguments}}
                              // expected-error@-2 {{no matching function for call}}
 
 static_assert(f5<double>() == 2);
-static_assert(f5<E>() == 1); // expected-note {{while checking constraint satisfaction}} expected-note {{in instantiation of}}
-static_assert(f5<F>() == 2); // expected-note {{while checking constraint satisfaction}} expected-note {{in instantiation of}}
+static_assert(f5<E>() == 1); // expected-note {{while checking constraint satisfaction}} expected-note {{while substituting deduced template arguments}}
+static_assert(f5<F>() == 2); // expected-note {{while checking constraint satisfaction}} expected-note {{while substituting deduced template arguments}}
 
 // Do not validate assumptions whose evaluation would have side-effects.
 constexpr int foo() {
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp b/external/llvm-project/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp
index 5c0d89d9125f..1bc7f2cce3c9 100644
--- a/external/llvm-project/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp
@@ -61,7 +61,7 @@ struct A : X<int> { // expected-error {{no template named 'X'}}
 // Similarly for treating overload sets of functions as template names.
 struct g<int> {}; // expected-error {{'g' refers to a function template}}
 g<int>::Y xy; // expected-error {{no template named 'g'}} FIXME lies
-void xf(g<int> x); // expected-error {{variable has incomplete type 'void'}} expected-error 1+{{}} expected-note {{}}
+void xf(g<int> x); // expected-error {{variable has incomplete type 'void'}} expected-error 1+{{}}
 struct B : g<int> { // expected-error {{expected class name}}
   B() : g<int>() {} // expected-error {{expected class member or base class name}}
 };
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx2b-deducing-this-constexpr.cpp b/external/llvm-project/clang/test/SemaCXX/cxx2b-deducing-this-constexpr.cpp
index 191fb013e031..5b06c7d1827a 100644
--- a/external/llvm-project/clang/test/SemaCXX/cxx2b-deducing-this-constexpr.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/cxx2b-deducing-this-constexpr.cpp
@@ -73,3 +73,32 @@ int test() {
 }
 
 }
+
+namespace GH142835 {
+struct MoveMe {
+  MoveMe& operator=(this MoveMe&, const MoveMe&) = default;
+  constexpr MoveMe& operator=(this MoveMe& self, MoveMe&& other) {
+    self.value = other.value;
+    other.value = 0;
+    return self;
+  }
+  int value = 4242;
+};
+
+struct S {
+  constexpr S& operator=(this S&, const S&) = default;
+  S& operator=(this S&, S&&) = default;
+
+  MoveMe move_me;
+};
+
+constexpr bool f() {
+  S s1{};
+  S s2{};
+  s2 = s1;
+  return true;
+}
+
+static_assert(f());
+
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx2c-fold-exprs.cpp b/external/llvm-project/clang/test/SemaCXX/cxx2c-fold-exprs.cpp
index 48061439941f..4220486d3aed 100644
--- a/external/llvm-project/clang/test/SemaCXX/cxx2c-fold-exprs.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/cxx2c-fold-exprs.cpp
@@ -233,7 +233,7 @@ void g() {
     A<Thingy, Thingy> *ap;
     f(ap, ap); // expected-error{{no matching function for call to 'f'}} \
                // expected-note {{while checking constraint satisfaction}} \
-               // expected-note {{in instantiation of function template specialization}}
+               // expected-note {{while substituting deduced template arguments}}
 }
 
 }
diff --git a/external/llvm-project/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp b/external/llvm-project/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp
index 062becd4f577..7152a5937d9b 100644
--- a/external/llvm-project/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -std=c++2c -verify %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -fptrauth-intrinsics -fptrauth-calls -std=c++2c -verify %s
 
 class Trivial {};
 static_assert(__builtin_is_cpp_trivially_relocatable(Trivial));
@@ -333,6 +334,31 @@ struct CopyAssign1 {
    CopyAssign1 & operator=(CopyAssign1 const &) = default;
 };
 
+struct UserDeleted1 {
+    UserDeleted1(const UserDeleted1&) = delete;
+};
+static_assert(!__builtin_is_cpp_trivially_relocatable(UserDeleted1));
+static_assert(!__builtin_is_replaceable(UserDeleted1));
+
+struct UserDeleted2 {
+    UserDeleted2(UserDeleted2&&) = delete;
+};
+static_assert(!__builtin_is_cpp_trivially_relocatable(UserDeleted2));
+static_assert(!__builtin_is_replaceable(UserDeleted2));
+
+
+struct UserDeleted3 {
+    UserDeleted3 operator=(UserDeleted3);
+};
+static_assert(!__builtin_is_cpp_trivially_relocatable(UserDeleted3));
+static_assert(!__builtin_is_replaceable(UserDeleted3));
+
+struct UserDeleted4 {
+    UserDeleted4 operator=(UserDeleted4&&);
+};
+static_assert(!__builtin_is_cpp_trivially_relocatable(UserDeleted4));
+static_assert(!__builtin_is_replaceable(UserDeleted4));
+
 }
 
 
@@ -363,3 +389,24 @@ void do_test__builtin_trivially_relocate() {
     // expected-note@-1 {{'test__builtin_trivially_relocate<S *, S *, int>' requested here}}
     // expected-error@#reloc1 {{first argument to '__builtin_trivially_relocate' must be relocatable}}
 }
+
+
+namespace GH143599 {
+struct A { ~A (); };
+A::~A () = default;
+
+static_assert (!__builtin_is_cpp_trivially_relocatable(A));
+static_assert (!__builtin_is_replaceable(A));
+
+struct B { B(const B&); };
+B::B (const B&) = default;
+
+static_assert (!__builtin_is_cpp_trivially_relocatable(B));
+static_assert (!__builtin_is_replaceable(B));
+
+struct C { C& operator=(const C&); };
+C& C::operator=(const C&) = default;
+
+static_assert (!__builtin_is_cpp_trivially_relocatable(C));
+static_assert (!__builtin_is_replaceable(C));
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/deprecated-builtins.cpp b/external/llvm-project/clang/test/SemaCXX/deprecated-builtins.cpp
index fafc1da4da13..1234c8354fca 100644
--- a/external/llvm-project/clang/test/SemaCXX/deprecated-builtins.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/deprecated-builtins.cpp
@@ -15,6 +15,7 @@ void f() {
     a = __has_trivial_constructor(A);  // expected-warning-re {{__has_trivial_constructor {{.*}} use __is_trivially_constructible}}
     a = __has_trivial_move_constructor(A);  // expected-warning-re {{__has_trivial_move_constructor {{.*}} use __is_trivially_constructible}}
     a = __has_trivial_destructor(A);  // expected-warning-re {{__has_trivial_destructor {{.*}} use __is_trivially_destructible}}
+    a = __reference_binds_to_temporary(const A&, A); // expected-warning-re {{__reference_binds_to_temporary {{.*}} use __reference_constructs_from_temporary}}
 
 }
 
diff --git a/external/llvm-project/clang/test/SemaCXX/destructor.cpp b/external/llvm-project/clang/test/SemaCXX/destructor.cpp
index 589616ef8e43..b9e0b17d510a 100644
--- a/external/llvm-project/clang/test/SemaCXX/destructor.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/destructor.cpp
@@ -58,7 +58,7 @@ struct D {
 
 struct D2 {
   void ~D2() { } //                          \
-  // expected-error{{destructor cannot have a return type}}  
+  // expected-error{{destructor cannot have a return type}}
 };
 
 
@@ -86,7 +86,7 @@ struct G {
 G::~G() { }
 
 struct H {
-  ~H(void) { } 
+  ~H(void) { }
 };
 
 struct X {};
@@ -103,7 +103,7 @@ namespace PR6421 {
     template<typename U>
     void foo(T t) // expected-error{{variable has incomplete type}}
     { }
-    
+
     void disconnect()
     {
       T* t;
@@ -364,7 +364,7 @@ struct __is_destructor_wellformed {
                        decltype(_Tp1().~_Tp1())>::type);
   template <typename _Tp1>
   static __two __test (...);
-              
+
   static const bool value = sizeof(__test<_Tp>(12)) == sizeof(char);
 };
 
@@ -553,14 +553,11 @@ namespace crash_on_invalid_base_dtor {
 struct Test {
   virtual ~Test();
 };
-struct Baz : public Test { // expected-warning {{non-virtual destructor}}
+struct Baz : public Test {
   Baz() {}
-  ~Baz() = defaul; // expected-error {{undeclared identifier 'defaul'}} \
-                   // expected-error {{initializer on function}} \
-                   // expected-note {{overridden virtual function is here}}
+  ~Baz() = defaul; // expected-error {{undeclared identifier 'defaul'}}
 };
-struct Foo : public Baz { // expected-error {{cannot override a non-deleted function}} \
-                          // expected-note {{destructor of 'Foo' is implicitly deleted}}
+struct Foo : public Baz {
   Foo() {}
 };
 }
@@ -579,11 +576,9 @@ static_assert(!__is_trivially_constructible(Foo, Foo &&), "");
 
 namespace GH97230 {
 struct X {
-  ~X() = defaul; // expected-error {{initializer on function does not look like a pure-specifier}} \
-                 // expected-error {{use of undeclared identifier 'defaul'}}
+  ~X() = defaul; // expected-error {{use of undeclared identifier 'defaul'}}
 };
-struct Y : X {} y1{ }; // expected-error {{call to implicitly-deleted default constructor of 'struct Y'}} \
-                       // expected-note {{default constructor of 'Y' is implicitly deleted because base class 'X' has no destructor}}
+struct Y : X {} y1{ };
 }
 
 namespace GH121706 {
diff --git a/external/llvm-project/clang/test/SemaCXX/gh102293.cpp b/external/llvm-project/clang/test/SemaCXX/gh102293.cpp
index d4218cc13dce..fe417e697841 100644
--- a/external/llvm-project/clang/test/SemaCXX/gh102293.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/gh102293.cpp
@@ -45,3 +45,20 @@ class quux : quux { // expected-error {{base class has incomplete type}} \
   virtual int c();
 };
 }
+
+// Ensure we don't get infinite recursion from the check, however. See GH141789
+namespace GH141789 {
+template <typename Ty>
+struct S {
+  Ty t; // expected-error {{field has incomplete type 'GH141789::X'}}
+};
+
+struct T {
+  ~T();
+};
+
+struct X { // expected-note {{definition of 'GH141789::X' is not complete until the closing '}'}}
+  S<X> next; // expected-note {{in instantiation of template class 'GH141789::S<GH141789::X>' requested here}}
+  T m;
+};
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/gh139818.cpp b/external/llvm-project/clang/test/SemaCXX/gh139818.cpp
new file mode 100644
index 000000000000..71a1e6e3b454
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaCXX/gh139818.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only %s -verify -fcxx-exceptions
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only %s -verify -fexperimental-new-constant-interpreter -fcxx-exceptions
+
+namespace GH139818{
+    struct A {
+      constexpr ~A() { ref = false; }
+      constexpr operator bool() {
+        return b;
+      }
+      bool b;
+      bool& ref;
+    };
+
+    constexpr bool f1() {
+      bool ret = true;
+      for (bool b = false; A x{b, ret}; b = true) {}
+      return ret;
+    }
+
+    static_assert(!f1());
+
+    struct Y {
+      constexpr ~Y() noexcept(false) { throw "oops"; }  // expected-note {{subexpression not valid in a constant expression}}
+                                                        
+      constexpr operator bool() {
+        return b;
+      }
+      bool b;
+    };
+    constexpr bool f2() {
+      for (bool b = false; Y x = {b}; b = true) {} // expected-note {{in call to 'x.~Y()'}}
+      return true;
+    }
+    static_assert(f2()); // expected-error {{static assertion expression is not an integral constant expression}}
+                         // expected-note@-1 {{in call to 'f2()'}}
+};
diff --git a/external/llvm-project/clang/test/SemaCXX/invalid-if-constexpr.cpp b/external/llvm-project/clang/test/SemaCXX/invalid-if-constexpr.cpp
index 0007f2739cbb..9f2774187148 100644
--- a/external/llvm-project/clang/test/SemaCXX/invalid-if-constexpr.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/invalid-if-constexpr.cpp
@@ -2,12 +2,16 @@
 
 namespace GH61885 {
 void similar() { // expected-note {{'similar' declared here}}
-  if constexpr (similer<>) {} // expected-error {{use of undeclared identifier 'similer'; did you mean 'similar'?}}
+  if constexpr (similer<>) {} // expected-error {{use of undeclared identifier 'similer'; did you mean 'similar'?}} \
+                                 expected-warning {{address of function 'similar<>' will always evaluate to 'true'}} \
+                                 expected-note {{prefix with the address-of operator to silence this warning}}
 }
-void a() { if constexpr (__adl_swap<>) {}} // expected-error{{use of undeclared identifier '__adl_swap'; did you mean '__sync_swap'?}}
+void a() { if constexpr (__adl_swap<>) {}} // expected-error{{use of undeclared identifier '__adl_swap'}}
 
 int AA() { return true;} // expected-note {{'AA' declared here}}
 
-void b() { if constexpr (AAA<>) {}} // expected-error {{use of undeclared identifier 'AAA'; did you mean 'AA'?}}
+void b() { if constexpr (AAA<>) {}} // expected-error {{use of undeclared identifier 'AAA'; did you mean 'AA'?}} \
+                                       expected-warning {{address of function 'AA<>' will always evaluate to 'true'}} \
+                                       expected-note {{prefix with the address-of operator to silence this warning}}
 }
 
diff --git a/external/llvm-project/clang/test/SemaCXX/lambda-unevaluated.cpp b/external/llvm-project/clang/test/SemaCXX/lambda-unevaluated.cpp
index 40f8a729fd91..9289fc9ec124 100644
--- a/external/llvm-project/clang/test/SemaCXX/lambda-unevaluated.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/lambda-unevaluated.cpp
@@ -174,7 +174,7 @@ int* func(T) requires requires { []() { T::foo(); }; }; // expected-error{{type
 double* func(...);
 
 static_assert(__is_same(decltype(func(0)), double*)); // expected-note {{while checking constraint satisfaction for template 'func<int>' required here}}
-                                                      // expected-note@-1 {{in instantiation of function template specialization 'lambda_in_constraints::func<int>'}}
+                                                      // expected-note@-1 {{while substituting deduced template arguments into function template 'func' [with T = int]}}
 static_assert(__is_same(decltype(func(WithFoo())), int*));
 
 template <class T>
@@ -252,7 +252,7 @@ S s("a"); // #use
 // expected-note@#S-requires {{substituting template arguments into constraint expression here}}
 // expected-note@#S-requires {{in instantiation of requirement here}}
 // expected-note@#use {{checking constraint satisfaction for template 'S<const char *>' required here}}
-// expected-note@#use {{requested here}}
+// expected-note@#use {{while substituting deduced template arguments into function template 'S' [with value:auto = const char *]}}
 // expected-note-re@#S 2{{candidate constructor {{.*}} not viable}}
 // expected-note@#S-ctor {{constraints not satisfied}}
 // expected-note-re@#S-requires {{because {{.*}} would be invalid}}
diff --git a/external/llvm-project/clang/test/SemaCXX/member-expr.cpp b/external/llvm-project/clang/test/SemaCXX/member-expr.cpp
index 0596e40f6c2f..902b09097a12 100644
--- a/external/llvm-project/clang/test/SemaCXX/member-expr.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/member-expr.cpp
@@ -96,11 +96,11 @@ namespace test5 {
 namespace PR7508 {
   struct A {
     struct CleanupScope {};
-    void PopCleanupBlock(); // expected-note{{'PopCleanupBlock' declared here}}
+    void PopCleanupBlock();
   };
 
   void foo(A &a) {
-    a.PopCleanupScope(); // expected-error{{no member named 'PopCleanupScope' in 'PR7508::A'; did you mean 'PopCleanupBlock'?}}
+    a.PopCleanupScope(); // expected-error{{no member named 'PopCleanupScope' in 'PR7508::A'}}
   }
 }
 
@@ -189,7 +189,7 @@ namespace PR15045 {
   }
 
   struct bar {
-    void func();  // expected-note {{'func' declared here}}
+    void func();
   };
 
   struct foo {
@@ -207,7 +207,7 @@ namespace PR15045 {
 
     // Show that recovery has happened by also triggering typo correction
     e->Func();  // expected-error {{member reference type 'bar' is not a pointer; did you mean to use '.'?}} \
-                // expected-error {{no member named 'Func' in 'PR15045::bar'; did you mean 'func'?}}
+                // expected-error {{no member named 'Func' in 'PR15045::bar'}}
 
     // Make sure a fixit isn't given in the case that the '->' isn't actually
     // the problem (the problem is with the return value of an operator->).
diff --git a/external/llvm-project/clang/test/SemaCXX/nested-name-spec.cpp b/external/llvm-project/clang/test/SemaCXX/nested-name-spec.cpp
index 36398aed7ac5..abeaba9d8dde 100644
--- a/external/llvm-project/clang/test/SemaCXX/nested-name-spec.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/nested-name-spec.cpp
@@ -409,7 +409,8 @@ T1<C2::N1> var_1a;
 T1<C2:N1> var_1b;  // expected-error{{unexpected ':' in nested name specifier; did you mean '::'?}}
 template<int N> int F() {}
 int (*X1)() = (B1::B2 ? F<1> : F<2>);
-int (*X2)() = (B1:B2 ? F<1> : F<2>);  // expected-error{{unexpected ':' in nested name specifier; did you mean '::'?}}
+int (*X2)() = (B1:B2 ? F<1> : F<2>);  // expected-error{{unexpected ':' in nested name specifier; did you mean '::'?}} \
+                                         expected-note{{'PR18587::X2' declared here}}
 
 // Bit fields + templates
 struct S7a {
@@ -445,7 +446,8 @@ namespace PR16951 {
 
   int x4 = enumerator_2::ENUMERATOR_2; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
   int x5 = enumerator_2::X2; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
-                             // expected-error{{no member named 'X2' in 'PR16951::enumerator_2'}}
+                             // expected-error{{no member named 'X2' in 'PR16951::enumerator_2'}} \
+                             // expected-error{{cannot initialize a variable of type 'int' with an lvalue of type 'int (*)()'}}
 
 }
 
diff --git a/external/llvm-project/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/external/llvm-project/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
index 4c5ac5dcbbd0..7cb71e075d50 100644
--- a/external/llvm-project/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -187,7 +187,7 @@ void f(int);
 void g(int n) { f(n); } // OK
 void h(short n) { f(n); }
 // expected-error@#GH62096-err {{static assertion failed due to requirement 'sizeof(short) == 0'}} \
-// expected-note@-1{{in instantiation of function template specialization}} \
+// expected-note@-1{{while substituting deduced template arguments}} \
 // expected-note@-1{{while checking constraint satisfaction for template}}
 // expected-note@#GH62096-note1{{in instantiation}}
 // expected-note@#GH62096-note1{{while substituting template arguments into constraint expression here}}
diff --git a/external/llvm-project/clang/test/SemaCXX/paren-list-init-expr.cpp b/external/llvm-project/clang/test/SemaCXX/paren-list-init-expr.cpp
new file mode 100644
index 000000000000..c47239ed0f32
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaCXX/paren-list-init-expr.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -ast-dump %s | FileCheck %s
+struct Node {
+  long val;
+};
+template <bool>
+void CallNew() {
+    new Node(0);
+}
+// CHECK-LABEL: FunctionTemplateDecl {{.*}} CallNew
+// CHECK: |-FunctionDecl {{.*}} CallNew 'void ()'
+// CHECK:  `-CXXNewExpr {{.*}} 'operator new'
+// CHECK:  `-CXXParenListInitExpr {{.*}} 'Node'
+// CHECK:  `-ImplicitCastExpr {{.*}} 'long' <IntegralCast>
+// CHECK:  `-IntegerLiteral {{.*}} 'int' 0
+// CHECK: `-FunctionDecl {{.*}} used CallNew 'void ()' implicit_instantiation
+// CHECK:   |-TemplateArgument integral 'true'
+// CHECK:   `-CXXNewExpr {{.*}} 'operator new'
+// CHECK:   `-CXXParenListInitExpr {{.*}} 'Node'
+// CHECK:   `-ImplicitCastExpr {{.*}} 'long' <IntegralCast>
+// CHECK:   `-IntegerLiteral {{.*}} 'int' 0
+void f() {
+    (void)CallNew<true>; 
+}
diff --git a/external/llvm-project/clang/test/SemaCXX/pr13394-crash-on-invalid.cpp b/external/llvm-project/clang/test/SemaCXX/pr13394-crash-on-invalid.cpp
deleted file mode 100644
index 304ee92f6a8d..000000000000
--- a/external/llvm-project/clang/test/SemaCXX/pr13394-crash-on-invalid.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
-// Don't crash (PR13394).
-
-namespace stretch_v1 {
-  struct closure_t {
-    const stretch_v1::ops_t* d_methods; // expected-error {{no type named 'ops_t' in namespace 'stretch_v1'}}
-  };
-}
-namespace gatekeeper_v1 {
-  namespace gatekeeper_factory_v1 {
-    struct closure_t { // expected-note {{'closure_t' declared here}} expected-note {{'gatekeeper_factory_v1::closure_t' declared here}}
-      gatekeeper_v1::closure_t* create(); // expected-error {{no type named 'closure_t' in namespace 'gatekeeper_v1'; did you mean simply 'closure_t'?}}
-    };
-  }
-  // FIXME: Typo correction should remove the 'gatekeeper_v1::' name specifier
-  gatekeeper_v1::closure_t *x; // expected-error {{no type named 'closure_t' in namespace 'gatekeeper_v1'; did you mean 'gatekeeper_factory_v1::closure_t'}}
-}
-
-namespace Foo {
-struct Base {
-  void Bar() {} // expected-note{{'Bar' declared here}}
-};
-}
-
-struct Derived : public Foo::Base {
-  void test() {
-    Foo::Bar(); // expected-error{{no member named 'Bar' in namespace 'Foo'; did you mean simply 'Bar'?}}
-  }
-};
diff --git a/external/llvm-project/clang/test/SemaCXX/ptrauth-triviality.cpp b/external/llvm-project/clang/test/SemaCXX/ptrauth-triviality.cpp
index 60d1b57230f1..ba8a8273d5c0 100644
--- a/external/llvm-project/clang/test/SemaCXX/ptrauth-triviality.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/ptrauth-triviality.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple arm64-apple-ios -std=c++20 -fptrauth-calls -fptrauth-intrinsics -verify -fsyntax-only %s
-// RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c++20 -fptrauth-calls -fptrauth-intrinsics -verify -fsyntax-only %s
+// RUN: %clang_cc1 -triple arm64-apple-ios -std=c++26 -fptrauth-calls -fptrauth-intrinsics -verify -fsyntax-only %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c++26 -fptrauth-calls -fptrauth-intrinsics -verify -fsyntax-only %s
 
 #define AQ __ptrauth(1,1,50)
 #define IQ __ptrauth(1,0,50)
@@ -83,7 +83,7 @@ static_assert(!__is_trivially_constructible(Holder<S3>, const Holder<S3>&));
 static_assert(!__is_trivially_assignable(Holder<S3>, const Holder<S3>&));
 static_assert(__is_trivially_destructible(Holder<S3>));
 static_assert(!__is_trivially_copyable(Holder<S3>));
-static_assert(__is_trivially_relocatable(Holder<S3>)); // expected-warning{{deprecated}}
+static_assert(!__is_trivially_relocatable(Holder<S3>)); // expected-warning{{deprecated}}
 static_assert(__builtin_is_cpp_trivially_relocatable(Holder<S3>));
 static_assert(!__is_trivially_equality_comparable(Holder<S3>));
 
@@ -99,7 +99,6 @@ static_assert(!__is_trivially_assignable(S4, const S4&));
 static_assert(__is_trivially_destructible(S4));
 static_assert(!__is_trivially_copyable(S4));
 static_assert(!__is_trivially_relocatable(S4)); // expected-warning{{deprecated}}
-//FIXME
 static_assert(__builtin_is_cpp_trivially_relocatable(S4));
 static_assert(!__is_trivially_equality_comparable(S4));
 
@@ -124,7 +123,6 @@ static_assert(!__is_trivially_assignable(S5, const S5&));
 static_assert(__is_trivially_destructible(S5));
 static_assert(!__is_trivially_copyable(S5));
 static_assert(!__is_trivially_relocatable(S5)); // expected-warning{{deprecated}}
-//FIXME
 static_assert(__builtin_is_cpp_trivially_relocatable(S5));
 static_assert(!__is_trivially_equality_comparable(S5));
 
@@ -182,3 +180,39 @@ static_assert(__is_trivially_copyable(Holder<S7>));
 static_assert(__is_trivially_relocatable(Holder<S7>)); // expected-warning{{deprecated}}
 static_assert(__builtin_is_cpp_trivially_relocatable(Holder<S7>));
 static_assert(__is_trivially_equality_comparable(Holder<S7>));
+
+template <class... Bases> struct MultipleInheriter : Bases... {
+};
+
+template <class T> static const bool test_is_trivially_relocatable_v = __builtin_is_cpp_trivially_relocatable(T);
+template <class... Types> static const bool multiple_inheritance_is_relocatable = test_is_trivially_relocatable_v<MultipleInheriter<Types...>>;
+template <class... Types> static const bool inheritance_relocatability_matches_bases_v =
+  (test_is_trivially_relocatable_v<Types> && ...) == multiple_inheritance_is_relocatable<Types...>;
+
+static_assert(multiple_inheritance_is_relocatable<S4, S5> == multiple_inheritance_is_relocatable<S5, S4>);
+static_assert(inheritance_relocatability_matches_bases_v<S4, S5>);
+static_assert(inheritance_relocatability_matches_bases_v<S5, S4>);
+
+struct AA AddressDiscriminatedPolymorphicBase trivially_relocatable_if_eligible {
+  virtual void foo();
+};
+
+struct IA NoAddressDiscriminatedPolymorphicBase trivially_relocatable_if_eligible {
+  virtual void bar();
+};
+
+template <class T> struct UnionWrapper trivially_relocatable_if_eligible {
+  union U {
+    T field1;
+  } u;
+};
+
+static_assert(test_is_trivially_relocatable_v<AddressDiscriminatedPolymorphicBase>);
+static_assert(test_is_trivially_relocatable_v<NoAddressDiscriminatedPolymorphicBase>);
+static_assert(inheritance_relocatability_matches_bases_v<AddressDiscriminatedPolymorphicBase, NoAddressDiscriminatedPolymorphicBase>);
+static_assert(inheritance_relocatability_matches_bases_v<NoAddressDiscriminatedPolymorphicBase, AddressDiscriminatedPolymorphicBase>);
+
+static_assert(!test_is_trivially_relocatable_v<UnionWrapper<AddressDiscriminatedPolymorphicBase>>);
+static_assert(test_is_trivially_relocatable_v<UnionWrapper<NoAddressDiscriminatedPolymorphicBase>>);
+static_assert(!test_is_trivially_relocatable_v<UnionWrapper<MultipleInheriter<NoAddressDiscriminatedPolymorphicBase, AddressDiscriminatedPolymorphicBase>>>);
+static_assert(!test_is_trivially_relocatable_v<UnionWrapper<MultipleInheriter<AddressDiscriminatedPolymorphicBase, NoAddressDiscriminatedPolymorphicBase>>>);
diff --git a/external/llvm-project/clang/test/SemaCXX/return.cpp b/external/llvm-project/clang/test/SemaCXX/return.cpp
index 17d7892d8dbd..796c9ae91ded 100644
--- a/external/llvm-project/clang/test/SemaCXX/return.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/return.cpp
@@ -130,5 +130,5 @@ void cxx_unresolved_expr() {
   // CXXUnresolvedConstructExpr, and the missing ')' gives it an invalid source
   // location for its rparen.  Check that emitting a diag on the range of the
   // expr doesn't assert.
-  return int(undeclared, 4; // expected-error {{expected ')'}} expected-note{{to match this '('}} expected-error {{use of undeclared identifier 'undeclared'}}
+  return int(undeclared, 4; // expected-error {{use of undeclared identifier 'undeclared'}}
 }
diff --git a/external/llvm-project/clang/test/SemaCXX/trivially-relocatable-ptrauth.cpp b/external/llvm-project/clang/test/SemaCXX/trivially-relocatable-ptrauth.cpp
new file mode 100644
index 000000000000..b38499a634fc
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaCXX/trivially-relocatable-ptrauth.cpp
@@ -0,0 +1,109 @@
+// RUN: %clang_cc1 -triple arm64 -fptrauth-calls -fptrauth-intrinsics -std=c++26 -verify %s
+
+// This test intentionally does not enable the global address discrimination
+// of vtable pointers. This lets us configure them with different schemas
+// and verify that we're correctly tracking the existence of address discrimination
+
+// expected-no-diagnostics
+
+struct NonAddressDiscPtrauth {
+  void * __ptrauth(1, 0, 1234) p;
+};
+
+static_assert(__builtin_is_cpp_trivially_relocatable(NonAddressDiscPtrauth));
+
+struct AddressDiscPtrauth {
+  void * __ptrauth(1, 1, 1234) p;
+};
+
+static_assert(!__builtin_is_cpp_trivially_relocatable(AddressDiscPtrauth));
+
+struct MultipleBaseClasses : NonAddressDiscPtrauth, AddressDiscPtrauth {
+
+};
+
+static_assert(!__builtin_is_cpp_trivially_relocatable(MultipleBaseClasses));
+
+struct MultipleMembers1 {
+   NonAddressDiscPtrauth field0;
+   AddressDiscPtrauth field1;
+};
+
+static_assert(!__builtin_is_cpp_trivially_relocatable(MultipleMembers1));
+
+struct MultipleMembers2 {
+   NonAddressDiscPtrauth field0;
+   NonAddressDiscPtrauth field1;
+};
+
+static_assert(__builtin_is_cpp_trivially_relocatable(MultipleMembers2));
+
+struct UnionOfPtrauth {
+    union {
+        NonAddressDiscPtrauth field0;
+        AddressDiscPtrauth field1;
+    } u;
+};
+
+static_assert(!__builtin_is_cpp_trivially_relocatable(UnionOfPtrauth));
+
+struct [[clang::ptrauth_vtable_pointer(process_independent,address_discrimination,no_extra_discrimination)]] Polymorphic trivially_relocatable_if_eligible {
+  virtual ~Polymorphic();
+};
+
+struct Foo : Polymorphic {
+  Foo(const Foo&);
+  ~Foo();
+};
+
+
+static_assert(__builtin_is_cpp_trivially_relocatable(Polymorphic));
+
+struct [[clang::ptrauth_vtable_pointer(process_independent,no_address_discrimination,no_extra_discrimination)]] NonAddressDiscriminatedPolymorphic trivially_relocatable_if_eligible {
+  virtual ~NonAddressDiscriminatedPolymorphic();
+};
+
+static_assert(__builtin_is_cpp_trivially_relocatable(NonAddressDiscriminatedPolymorphic));
+
+
+struct PolymorphicMembers {
+    Polymorphic field;
+};
+
+static_assert(__builtin_is_cpp_trivially_relocatable(PolymorphicMembers));
+
+struct UnionOfPolymorphic {
+  union trivially_relocatable_if_eligible {
+    Polymorphic p;
+    int i;
+  } u;
+};
+
+static_assert(!__builtin_is_cpp_trivially_relocatable(UnionOfPolymorphic));
+
+
+struct UnionOfNonAddressDiscriminatedPolymorphic {
+  union trivially_relocatable_if_eligible {
+    NonAddressDiscriminatedPolymorphic p;
+    int i;
+  } u;
+};
+static_assert(!__builtin_is_cpp_trivially_relocatable(UnionOfNonAddressDiscriminatedPolymorphic));
+
+struct UnionOfNonAddressDiscriminatedPtrauth {
+  union {
+    NonAddressDiscPtrauth p;
+    int i;
+  } u;
+};
+
+static_assert(__builtin_is_cpp_trivially_relocatable(UnionOfNonAddressDiscriminatedPtrauth));
+
+struct UnionOfAddressDisriminatedPtrauth {
+  union {
+    AddressDiscPtrauth p;
+    int i;
+  } u;
+};
+
+static_assert(!__builtin_is_cpp_trivially_relocatable(UnionOfAddressDisriminatedPtrauth));
diff --git a/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp b/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
index 90cff1e66000..329b611110c1 100644
--- a/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
@@ -12,6 +12,14 @@ struct is_trivially_relocatable {
 
 template <typename T>
 constexpr bool is_trivially_relocatable_v = __builtin_is_cpp_trivially_relocatable(T);
+
+template <typename T>
+struct is_trivially_copyable {
+    static constexpr bool value = __is_trivially_copyable(T);
+};
+
+template <typename T>
+constexpr bool is_trivially_copyable_v = __is_trivially_copyable(T);
 #endif
 
 #ifdef STD2
@@ -25,6 +33,17 @@ using is_trivially_relocatable  = __details_is_trivially_relocatable<T>;
 
 template <typename T>
 constexpr bool is_trivially_relocatable_v = __builtin_is_cpp_trivially_relocatable(T);
+
+template <typename T>
+struct __details_is_trivially_copyable {
+    static constexpr bool value = __is_trivially_copyable(T);
+};
+
+template <typename T>
+using is_trivially_copyable  = __details_is_trivially_copyable<T>;
+
+template <typename T>
+constexpr bool is_trivially_copyable_v = __is_trivially_copyable(T);
 #endif
 
 
@@ -45,6 +64,15 @@ using is_trivially_relocatable  = __details_is_trivially_relocatable<T>;
 
 template <typename T>
 constexpr bool is_trivially_relocatable_v = is_trivially_relocatable<T>::value;
+
+template <typename T>
+struct __details_is_trivially_copyable : bool_constant<__is_trivially_copyable(T)> {};
+
+template <typename T>
+using is_trivially_copyable  = __details_is_trivially_copyable<T>;
+
+template <typename T>
+constexpr bool is_trivially_copyable_v = is_trivially_copyable<T>::value;
 #endif
 
 }
@@ -60,6 +88,18 @@ static_assert(std::is_trivially_relocatable_v<int&>);
 // expected-note@-1 {{'int &' is not trivially relocatable}} \
 // expected-note@-1 {{because it is a reference type}}
 
+static_assert(std::is_trivially_copyable<int>::value);
+
+static_assert(std::is_trivially_copyable<int&>::value);
+// expected-error-re@-1 {{static assertion failed due to requirement 'std::{{.*}}is_trivially_copyable<int &>::value'}} \
+// expected-note@-1 {{'int &' is not trivially copyable}} \
+// expected-note@-1 {{because it is a reference type}}
+static_assert(std::is_trivially_copyable_v<int&>);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_trivially_copyable_v<int &>'}} \
+// expected-note@-1 {{'int &' is not trivially copyable}} \
+// expected-note@-1 {{because it is a reference type}}
+
+
 namespace test_namespace {
     using namespace std;
     static_assert(is_trivially_relocatable<int&>::value);
@@ -70,6 +110,15 @@ namespace test_namespace {
     // expected-error@-1 {{static assertion failed due to requirement 'is_trivially_relocatable_v<int &>'}} \
     // expected-note@-1 {{'int &' is not trivially relocatable}} \
     // expected-note@-1 {{because it is a reference type}}
+
+    static_assert(is_trivially_copyable<int&>::value);
+    // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_trivially_copyable<int &>::value'}} \
+    // expected-note@-1 {{'int &' is not trivially copyable}} \
+    // expected-note@-1 {{because it is a reference type}}
+    static_assert(is_trivially_copyable_v<int&>);
+    // expected-error@-1 {{static assertion failed due to requirement 'is_trivially_copyable_v<int &>'}} \
+    // expected-note@-1 {{'int &' is not trivially copyable}} \
+    // expected-note@-1 {{because it is a reference type}}
 }
 
 
@@ -82,6 +131,14 @@ concept C = std::is_trivially_relocatable_v<T>; // #concept2
 
 template <C T> void g();  // #cand2
 
+template <typename T>
+requires std::is_trivially_copyable<T>::value void f2();  // #cand3
+
+template <typename T>
+concept C2 = std::is_trivially_copyable_v<T>; // #concept4
+
+template <C2 T> void g2();  // #cand4
+
 void test() {
     f<int&>();
     // expected-error@-1 {{no matching function for call to 'f'}} \
@@ -97,5 +154,41 @@ void test() {
     // expected-note@#concept2 {{because 'std::is_trivially_relocatable_v<int &>' evaluated to false}} \
     // expected-note@#concept2 {{'int &' is not trivially relocatable}} \
     // expected-note@#concept2 {{because it is a reference type}}
+
+    f2<int&>();
+    // expected-error@-1 {{no matching function for call to 'f2'}} \
+    // expected-note@#cand3 {{candidate template ignored: constraints not satisfied [with T = int &]}} \
+    // expected-note-re@#cand3 {{because '{{.*}}is_trivially_copyable<int &>::value' evaluated to false}} \
+    // expected-note@#cand3 {{'int &' is not trivially copyable}} \
+    // expected-note@#cand3 {{because it is a reference type}}
+
+    g2<int&>();
+    // expected-error@-1 {{no matching function for call to 'g2'}} \
+    // expected-note@#cand4 {{candidate template ignored: constraints not satisfied [with T = int &]}} \
+    // expected-note@#cand4 {{because 'int &' does not satisfy 'C2'}} \
+    // expected-note@#concept4 {{because 'std::is_trivially_copyable_v<int &>' evaluated to false}} \
+    // expected-note@#concept4 {{'int &' is not trivially copyable}} \
+    // expected-note@#concept4 {{because it is a reference type}}
+}
 }
+
+
+namespace std {
+template <typename T>
+struct is_replaceable {
+    static constexpr bool value = __builtin_is_replaceable(T);
+};
+
+template <typename T>
+constexpr bool is_replaceable_v = __builtin_is_replaceable(T);
+
 }
+
+static_assert(std::is_replaceable<int&>::value);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_replaceable<int &>::value'}} \
+// expected-note@-1 {{'int &' is not replaceable}} \
+// expected-note@-1 {{because it is a reference type}}
+static_assert(std::is_replaceable_v<int&>);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_replaceable_v<int &>'}} \
+// expected-note@-1 {{'int &' is not replaceable}} \
+// expected-note@-1 {{because it is a reference type}}
diff --git a/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp b/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
index d9cab20f4feb..5210354a66d4 100644
--- a/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
@@ -142,5 +142,349 @@ static_assert(__builtin_is_cpp_trivially_relocatable(U2));
 // expected-note@-1 {{because it has a deleted destructor}} \
 // expected-note@-1 {{because it has a non-trivially-relocatable member 'b' of type 'B'}} \
 // expected-note@#tr-U2 {{'U2' defined here}}
+}
+
+namespace replaceable {
+
+extern int vla_size;
+static_assert(__builtin_is_replaceable(int[vla_size]));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(int[vla_size])'}} \
+// expected-note@-1 {{'int[vla_size]' is not replaceable}} \
+// expected-note@-1 {{because it is a variably-modified type}}
+
+struct S; // expected-note {{forward declaration of 'replaceable::S'}}
+static_assert(__builtin_is_replaceable(S));
+// expected-error@-1 {{incomplete type 'S' used in type trait expression}}
+
+static_assert(__builtin_is_replaceable(const volatile int));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(const volatile int)}} \
+// expected-note@-1 {{'const volatile int' is not replaceable}} \
+// expected-note@-1 {{because it is const}} \
+// expected-note@-1 {{because it is volatile}}
+
+
+static_assert(__builtin_is_replaceable(void()));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(void ())}} \
+// expected-note@-1 {{'void ()' is not replaceable}} \
+// expected-note@-1 {{because it is not a scalar or class type}}
+
+struct B {
+ virtual ~B();
+};
+struct S : virtual B { // #replaceable-S
+    S();
+    int & a;
+    const int ci;
+    B & b;
+    B c;
+    ~S();
+};
+static_assert(__builtin_is_replaceable(S));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::S)'}} \
+// expected-note@-1 {{'S' is not replaceable}} \
+// expected-note@-1 {{because it has a non-replaceable base 'B'}} \
+// expected-note@-1 {{because it has a non-replaceable member 'a' of type 'int &'}} \
+// expected-note@-1 {{because it has a non-replaceable member 'ci' of type 'const int'}} \
+// expected-note@-1 {{because it has a non-replaceable member 'b' of type 'B &'}} \
+// expected-note@-1 {{because it has a non-replaceable member 'c' of type 'B'}} \
+// expected-note@-1 {{because it has a user-provided destructor}} \
+// expected-note@-1 {{because it has a deleted copy assignment operator}}
+// expected-note@#replaceable-S {{'S' defined here}}
+
+struct S2 { // #replaceable-S2
+    S2(S2&&);
+    S2& operator=(const S2&);
+};
+static_assert(__builtin_is_replaceable(S2));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::S2)'}} \
+// expected-note@-1 {{'S2' is not replaceable}} \
+// expected-note@-1 {{because it has a user provided move constructor}} \
+// expected-note@-1 {{because it has a user provided copy assignment operator}} \
+// expected-note@#replaceable-S2 {{'S2' defined here}}
+
+
+struct S3 { // #replaceable-S3
+    ~S3() = delete;
+};
+static_assert(__builtin_is_replaceable(S3));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::S3)'}} \
+// expected-note@-1 {{'S3' is not replaceable}} \
+// expected-note@-1 {{because it has a deleted destructor}} \
+// expected-note@#replaceable-S3 {{'S3' defined here}}
+
+
+union U { // #replaceable-U
+    U(const U&);
+    U(U&&);
+    U& operator=(const U&);
+    U& operator=(U&&);
+};
+static_assert(__builtin_is_replaceable(U));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::U)'}} \
+// expected-note@-1 {{'U' is not replaceable}} \
+// expected-note@-1 {{because it is a union with a user-declared copy constructor}} \
+// expected-note@-1 {{because it is a union with a user-declared copy assignment operator}} \
+// expected-note@-1 {{because it is a union with a user-declared move constructor}} \
+// expected-note@-1 {{because it is a union with a user-declared move assignment operator}}
+// expected-note@#replaceable-U {{'U' defined here}}
+struct S4 replaceable_if_eligible { // #replaceable-S4
+    ~S4();
+    B b;
+};
+static_assert(__builtin_is_replaceable(S4));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::S4)'}} \
+// expected-note@-1 {{'S4' is not replaceable}} \
+// expected-note@-1 {{because it has a non-replaceable member 'b' of type 'B'}} \
+// expected-note@#replaceable-S4 {{'S4' defined here}}
+
+union U2 replaceable_if_eligible { // #replaceable-U2
+    U2(const U2&);
+    U2(U2&&);
+    B b;
+};
+static_assert(__builtin_is_replaceable(U2));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::U2)'}} \
+// expected-note@-1 {{'U2' is not replaceable}} \
+// expected-note@-1 {{because it has a deleted destructor}} \
+// expected-note@-1 {{because it has a non-replaceable member 'b' of type 'B'}} \
+// expected-note@-1 {{because it has a deleted copy assignment operator}} \
+// expected-note@#replaceable-U2 {{'U2' defined here}}
+
+struct UD1 {  // #replaceable-UD1
+    UD1(const UD1&) = delete;
+    UD1 & operator=(const UD1&) = delete;
+
+};
+static_assert(__builtin_is_replaceable(UD1));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::UD1)'}} \
+// expected-note@-1 {{'UD1' is not replaceable}} \
+// expected-note@-1 {{because it has a deleted copy constructor}} \
+// expected-note@-1 {{because it has a deleted copy assignment operator}} \
+// expected-note@#replaceable-UD1 {{'UD1' defined here}}
+
+
+struct UD2 {  // #replaceable-UD2
+    UD2(UD2&&) = delete;
+    UD2 & operator=(UD2&&) = delete;
+};
+static_assert(__builtin_is_replaceable(UD2));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(replaceable::UD2)'}} \
+// expected-note@-1 {{'UD2' is not replaceable}} \
+// expected-note@-1 {{because it has a deleted move constructor}} \
+// expected-note@-1 {{because it has a deleted move assignment operator}} \
+// expected-note@#replaceable-UD2 {{'UD2' defined here}}
+
+}
+
+
+namespace GH143325 {
+struct Foo  { // expected-note {{previous definition is here}}
+  Foo(const Foo&);
+  ~Foo();
+};
+
+struct Foo { // expected-error {{redefinition of 'Foo'}}
+  Foo();
+  int;
+};
+struct Wrapper { // #GH143325-Wrapper
+  union {
+    Foo p;
+  } u;
+};
+
+static_assert(__builtin_is_cpp_trivially_relocatable(Wrapper));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_cpp_trivially_relocatable(GH143325::Wrapper)'}} \
+// expected-note@-1 {{'Wrapper' is not trivially relocatable}} \
+// expected-note@-1 {{because it has a non-trivially-relocatable member 'u' of type 'union}} \
+// expected-note@-1 {{because it has a deleted destructor}}
+// expected-note@#GH143325-Wrapper {{'Wrapper' defined here}}
+
+struct Polymorphic  {
+  virtual ~Polymorphic();
+};
+
+struct UnionOfPolymorphic { // #GH143325-UnionOfPolymorphic
+  union {
+    Polymorphic p;
+    int i;
+  } u;
+};
 
+static_assert(__builtin_is_cpp_trivially_relocatable(UnionOfPolymorphic));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_cpp_trivially_relocatable(GH143325::UnionOfPolymorphic)'}} \
+// expected-note@-1 {{'UnionOfPolymorphic' is not trivially relocatable}} \
+// expected-note@-1 {{because it has a non-trivially-relocatable member 'u' of type 'union}} \
+// expected-note@-1 {{because it has a deleted destructor}} \
+// expected-note@#GH143325-UnionOfPolymorphic {{'UnionOfPolymorphic' defined here}}
+
+}
+
+struct GH143599 {  // expected-note 2 {{'GH143599' defined here}}
+    ~GH143599 ();
+     GH143599(const GH143599&);
+     GH143599& operator=(const GH143599&);
+};
+GH143599::~GH143599 () = default;
+GH143599::GH143599 (const GH143599&) = default;
+GH143599& GH143599::operator=(const GH143599&) = default;
+
+static_assert (__builtin_is_cpp_trivially_relocatable(GH143599));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_cpp_trivially_relocatable(GH143599)'}} \
+// expected-note@-1 {{'GH143599' is not trivially relocatable}} \
+// expected-note@-1 {{because it has a user provided copy constructor}} \
+// expected-note@-1 {{because it has a user provided copy assignment operator}} \
+// expected-note@-1 {{because it has a user-provided destructor}}
+
+static_assert (__builtin_is_replaceable(GH143599));
+// expected-error@-1 {{static assertion failed due to requirement '__builtin_is_replaceable(GH143599)'}} \
+// expected-note@-1 {{'GH143599' is not replaceable}} \
+// expected-note@-1 {{because it has a user provided copy constructor}} \
+// expected-note@-1 {{because it has a user provided copy assignment operator}} \
+// expected-note@-1 {{because it has a user-provided destructor}}
+
+namespace trivially_copyable {
+struct B {
+ virtual ~B();
+};
+struct S : virtual B { // #tc-S
+    S();
+    int & a;
+    const int ci;
+    B & b;
+    B c;
+    ~S();
+};
+static_assert(__is_trivially_copyable(S));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S)'}} \
+// expected-note@-1 {{'S' is not trivially copyable}} \
+// expected-note@-1 {{because it has a virtual base 'B'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable base 'B'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable member 'c' of type 'B'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable member 'b' of type 'B &'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable member 'a' of type 'int &'}} \
+// expected-note@-1 {{because it has a user-provided destructor}}
+// expected-note@#tc-S {{'S' defined here}}
+
+struct S2 { // #tc-S2
+    S2(S2&&);
+    S2& operator=(const S2&);
+};
+static_assert(__is_trivially_copyable(S2));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S2)'}} \
+// expected-note@-1 {{'S2' is not trivially copyable}} \
+// expected-note@-1 {{because it has a user provided move constructor}} \
+// expected-note@-1 {{because it has a user provided copy assignment operator}} \
+// expected-note@#tc-S2 {{'S2' defined here}}
+
+struct S3 {
+    ~S3() = delete;
+};
+static_assert(__is_trivially_copyable(S3));
+
+struct S4 { // #tc-S4
+    ~S4();
+    B b;
+};
+static_assert(__is_trivially_copyable(S4));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S4)'}} \
+// expected-note@-1 {{'S4' is not trivially copyable}} \
+// expected-note@-1 {{because it has a non-trivially-copyable member 'b' of type 'B'}} \
+// expected-note@-1 {{because it has a user-provided destructor}} \
+// expected-note@#tc-S4 {{'S4' defined here}}
+
+struct B1 {
+    int & a;
+};
+
+struct B2 {
+    int & a;
+};
+
+struct S5 : virtual B1, virtual B2 { // #tc-S5
+};
+static_assert(__is_trivially_copyable(S5));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S5)'}} \
+// expected-note@-1 {{'S5' is not trivially copyable}} \
+// expected-note@-1 {{because it has a virtual base 'B1'}} \
+// expected-note@-1 {{because it has a virtual base 'B2'}} \
+// expected-note@#tc-S5 {{'S5' defined here}}
+
+struct B3 {
+    ~B3();
+};
+
+struct B4 {
+    ~B4();
+};
+
+struct S6 : B3, B4 { // #tc-S6
+};
+static_assert(__is_trivially_copyable(S6));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S6)'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable base 'B3'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable base 'B4'}} \
+// expected-note@-1 {{because it has a user-provided destructor}} \
+// expected-note@-1 {{'S6' is not trivially copyable}} \
+// expected-note@#tc-S6 {{'S6' defined here}}
+
+struct S7 { // #tc-S7
+    S7(const S7&);
+};
+static_assert(__is_trivially_copyable(S7));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S7)'}} \
+// expected-note@-1 {{because it has a user provided copy constructor}} \
+// expected-note@-1 {{'S7' is not trivially copyable}} \
+// expected-note@#tc-S7 {{'S7' defined here}}
+
+struct S8 { // #tc-S8
+    S8(S8&&);
+};
+static_assert(__is_trivially_copyable(S8));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S8)'}} \
+// expected-note@-1 {{because it has a user provided move constructor}} \
+// expected-note@-1 {{'S8' is not trivially copyable}} \
+// expected-note@#tc-S8 {{'S8' defined here}}
+
+struct S9 { // #tc-S9
+    S9& operator=(const S9&);
+};
+static_assert(__is_trivially_copyable(S9));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S9)'}} \
+// expected-note@-1 {{because it has a user provided copy assignment operator}} \
+// expected-note@-1 {{'S9' is not trivially copyable}} \
+// expected-note@#tc-S9 {{'S9' defined here}}
+
+struct S10 { // #tc-S10
+    S10& operator=(S10&&);
+};
+static_assert(__is_trivially_copyable(S10));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S10)'}} \
+// expected-note@-1 {{because it has a user provided move assignment operator}} \
+// expected-note@-1 {{'S10' is not trivially copyable}} \
+// expected-note@#tc-S10 {{'S10' defined here}}
+
+struct B5 : B4 {
+};
+
+struct B6 : B5  {
+};
+
+struct S11 : B6 { // #tc-S11
+};
+static_assert(__is_trivially_copyable(S11));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S11)'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable base 'B6'}} \
+// expected-note@-1 {{'S11' is not trivially copyable}} \
+// expected-note@#tc-S11 {{'S11' defined here}}
+
+struct S12 : B6 { // #tc-S12
+    ~S12() = delete;
+};
+static_assert(__is_trivially_copyable(S12));
+// expected-error@-1 {{static assertion failed due to requirement '__is_trivially_copyable(trivially_copyable::S12)'}} \
+// expected-note@-1 {{because it has a non-trivially-copyable base 'B6'}} \
+// expected-note@-1 {{because it has a deleted destructor}} \
+// expected-note@-1 {{'S12' is not trivially copyable}} \
+// expected-note@#tc-S12 {{'S12' defined here}}
 }
diff --git a/external/llvm-project/clang/test/SemaCXX/type-traits.cpp b/external/llvm-project/clang/test/SemaCXX/type-traits.cpp
index 56cca05402f4..3f0124755c67 100644
--- a/external/llvm-project/clang/test/SemaCXX/type-traits.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/type-traits.cpp
@@ -3143,6 +3143,10 @@ void reference_binds_to_temporary_checks() {
   static_assert(!(__reference_binds_to_temporary(int, long)));
 
   static_assert((__reference_binds_to_temporary(const int &, long)));
+
+  // Test that function references are never considered bound to temporaries.
+  static_assert(!__reference_binds_to_temporary(void(&)(), void()));
+  static_assert(!__reference_binds_to_temporary(void(&&)(), void()));
 }
 
 
@@ -3156,6 +3160,14 @@ struct ExplicitConversionRef {
     explicit operator int&();
 };
 
+struct NonMovable {
+  NonMovable(NonMovable&&) = delete;
+};
+
+struct ConvertsFromNonMovable {
+  ConvertsFromNonMovable(NonMovable);
+};
+
 void reference_constructs_from_temporary_checks() {
   static_assert(!__reference_constructs_from_temporary(int &, int &));
   static_assert(!__reference_constructs_from_temporary(int &, int &&));
@@ -3193,6 +3205,16 @@ void reference_constructs_from_temporary_checks() {
 
   static_assert(__reference_constructs_from_temporary(const int &, long));
 
+  // Test that function references are never considered bound to temporaries.
+  static_assert(!__reference_constructs_from_temporary(void(&&)(), void()));
+  static_assert(!__reference_constructs_from_temporary(void(&)(), void()));
+
+  // LWG3819: reference_meows_from_temporary should not use is_meowible
+  static_assert(__reference_constructs_from_temporary(ConvertsFromNonMovable&&, NonMovable) == __cplusplus >= 201703L);
+  // For scalar types, cv-qualifications are dropped first for prvalues.
+  static_assert(__reference_constructs_from_temporary(int&&, const int));
+  static_assert(__reference_constructs_from_temporary(int&&, volatile int));
+
   // Additional checks
   static_assert(__reference_constructs_from_temporary(POD const&, Derives));
   static_assert(__reference_constructs_from_temporary(int&&, int));
@@ -3250,6 +3272,16 @@ void reference_converts_from_temporary_checks() {
 
   static_assert(__reference_converts_from_temporary(const int &, long));
 
+  // Test that function references are never considered bound to temporaries.
+  static_assert(!__reference_converts_from_temporary(void(&)(), void()));
+  static_assert(!__reference_converts_from_temporary(void(&&)(), void()));
+
+  // LWG3819: reference_meows_from_temporary should not use is_meowible
+  static_assert(__reference_converts_from_temporary(ConvertsFromNonMovable&&, NonMovable) == __cplusplus >= 201703L);
+  // For scalar types, cv-qualifications are dropped first for prvalues.
+  static_assert(__reference_converts_from_temporary(int&&, const int));
+  static_assert(__reference_converts_from_temporary(int&&, volatile int));
+
   // Additional checks
   static_assert(__reference_converts_from_temporary(POD const&, Derives));
   static_assert(__reference_converts_from_temporary(int&&, int));
diff --git a/external/llvm-project/clang/test/SemaCXX/typo-correction-crash.cpp b/external/llvm-project/clang/test/SemaCXX/typo-correction-crash.cpp
index 2a77c9df505e..434b70e3c509 100644
--- a/external/llvm-project/clang/test/SemaCXX/typo-correction-crash.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/typo-correction-crash.cpp
@@ -4,10 +4,10 @@ auto check1() {
   return s; // expected-error {{use of undeclared identifier 's'}}
 }
 
-int test = 11; // expected-note 2 {{'test' declared here}}
+int test = 11; // expected-note 3 {{'test' declared here}}
 auto check2() {
   return "s";
-  return tes; // expected-error {{use of undeclared identifier 'tes'; did you mean 'test'?}}
+  return tes; // expected-error {{use of undeclared identifier 'tes'}}
               // expected-error@-1 {{deduced as 'int' here but deduced as 'const char *' in earlier}}
 }
 
@@ -16,9 +16,8 @@ template <class A> struct is_same<A,A> { static constexpr bool value = true; };
 
 auto L1 = [] { return s; }; // expected-error {{use of undeclared identifier 's'}}
 using T1 = decltype(L1());
-// FIXME: Suppress the 'undeclared identifier T1' diagnostic, the UsingDecl T1 is discarded because of an invalid L1().
-static_assert(is_same<T1, void>::value, "Return statement should be discarded"); // expected-error {{use of undeclared identifier 'T1'}}
-auto L2 = [] { return tes; }; // expected-error {{use of undeclared identifier 'tes'; did you mean 'test'?}}
+static_assert(is_same<T1, void>::value, "Return statement should be discarded");
+auto L2 = [] { return tes; }; // expected-error {{use of undeclared identifier 'tes'}}
 using T2 = decltype(L2());
 static_assert(is_same<T2, int>::value, "Return statement was corrected");
 
@@ -32,13 +31,13 @@ FooRecord::NestedNamespace::type x; // expected-error {{no member named 'NestedN
 
 void cast_expr(int g) { +int(n)(g); } // expected-error {{undeclared identifier 'n'}}
 
-void bind() { for (const auto& [test,_] : _test_) { }; } // expected-error {{undeclared identifier '_test_'}}
+void bind() { for (const auto& [test,_] : _test_) { }; } // expected-error {{undeclared identifier '_test_'}} \
+                                                            expected-error {{invalid range expression of type 'int'; no viable 'begin' function available}}
 
 namespace NoCrash {
 class S {
   void Function(int a) {
-    unknown1(unknown2, Function, unknown3); // expected-error 2{{use of undeclared identifier}} \
-                                               expected-error {{reference to non-static member function must be called}}
+    unknown1(unknown2, Function, unknown3); // expected-error 2{{use of undeclared identifier}}
   }
 };
 }
@@ -46,8 +45,6 @@ class S {
 namespace NoCrashOnCheckArgAlignment {
 template <typename a> void b(a &);
 void test() {
-  for (auto file_data :b(files_db_data)); // expected-error {{use of undeclared identifier 'files_db_data'; did you mean 'file_data'?}} \
-                                          // expected-note {{'file_data' declared here}} \
-                                          // expected-error {{cannot use type 'void' as a range}}
+  for (auto file_data :b(files_db_data)); // expected-error {{use of undeclared identifier 'files_db_data'}}
 }
 }
diff --git a/external/llvm-project/clang/test/SemaCXX/typo-correction-cxx11.cpp b/external/llvm-project/clang/test/SemaCXX/typo-correction-cxx11.cpp
index 8c588203cc12..9eb5f9c29962 100644
--- a/external/llvm-project/clang/test/SemaCXX/typo-correction-cxx11.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/typo-correction-cxx11.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s
 
 namespace PR23186 {
-decltype(ned);  // expected-error-re {{use of undeclared identifier 'ned'{{$}}}}
+decltype(ned);  // expected-error {{use of undeclared identifier 'ned'}}
 // The code below was triggering an UNREACHABLE in ASTContext::getTypeInfoImpl
 // once the above code failed to recover properly after making the bogus
 // correction of 'ned' to 'new'.
@@ -19,8 +19,9 @@ struct S {
 namespace PR23140 {
 auto lneed = gned.*[] {};  // expected-error-re {{use of undeclared identifier 'gned'{{$}}}}
 
-void test(int aaa, int bbb, int thisvar) {  // expected-note {{'thisvar' declared here}}
-  int thatval = aaa * (bbb + thatvar);  // expected-error {{use of undeclared identifier 'thatvar'; did you mean 'thisvar'?}}
+void test(int aaa, int bbb, int thisvar) {
+  int thatval = aaa * (bbb + thatvar);  // expected-error {{use of undeclared identifier 'thatvar'; did you mean 'thatval'}} \
+                                           expected-note {{'thatval' declared here}}
 }
 }
 
@@ -54,7 +55,7 @@ void run(A *annotations) {
 
   auto &annotation = *annotations;
   auto new_it = new_annotations.find(5);
-  auto &new_anotation = new_it.second;  // expected-note {{'new_anotation' declared here}}
-  new_annotation->Swap(&annotation);  // expected-error {{use of undeclared identifier 'new_annotation'; did you mean 'new_anotation'?}}
+  auto &new_anotation = new_it.second;
+  new_annotation->Swap(&annotation);  // expected-error {{use of undeclared identifier 'new_annotation'}}
 }
 }
diff --git a/external/llvm-project/clang/test/SemaCXX/typo-correction-delayed.cpp b/external/llvm-project/clang/test/SemaCXX/typo-correction-delayed.cpp
deleted file mode 100644
index fdb1f740fda6..000000000000
--- a/external/llvm-project/clang/test/SemaCXX/typo-correction-delayed.cpp
+++ /dev/null
@@ -1,216 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -Wno-c++11-extensions %s
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 -Wno-c++11-extensions %s
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
-
-struct A {};
-struct B {};
-struct D {
-  A fizbin;  // expected-note 2 {{declared here}}
-  A foobar;  // expected-note 2 {{declared here}}
-  B roxbin;  // expected-note 2 {{declared here}}
-  B toobad;  // expected-note 2 {{declared here}}
-  void BooHoo();
-  void FoxBox();
-};
-
-void something(A, B);
-void test() {
-  D obj;
-  something(obj.fixbin,   // expected-error {{did you mean 'fizbin'?}}
-            obj.toobat);  // expected-error {{did you mean 'toobad'?}}
-  something(obj.toobat,   // expected-error {{did you mean 'foobar'?}}
-            obj.fixbin);  // expected-error {{did you mean 'roxbin'?}}
-  something(obj.fixbin,   // expected-error {{did you mean 'fizbin'?}}
-            obj.fixbin);  // expected-error {{did you mean 'roxbin'?}}
-  something(obj.toobat,   // expected-error {{did you mean 'foobar'?}}
-            obj.toobat);  // expected-error {{did you mean 'toobad'?}}
-  // Both members could be corrected to methods, but that isn't valid.
-  something(obj.boohoo,   // expected-error-re {{no member named 'boohoo' in 'D'{{$}}}}
-            obj.foxbox);  // expected-error-re {{no member named 'foxbox' in 'D'{{$}}}}
-  // The first argument has a usable correction but the second doesn't.
-  something(obj.boobar,   // expected-error-re {{no member named 'boobar' in 'D'{{$}}}}
-            obj.foxbox);  // expected-error-re {{no member named 'foxbox' in 'D'{{$}}}}
-}
-
-// Ensure the delayed typo correction does the right thing when trying to
-// recover using a seemingly-valid correction for which a valid expression to
-// replace the TypoExpr cannot be created (but which does have a second
-// correction candidate that would be a valid and usable correction).
-class Foo {
-public:
-  template <> void testIt();  // expected-error {{no function template matches}}
-  void textIt();  // expected-note {{'textIt' declared here}}
-};
-void testMemberExpr(Foo *f) {
-  f->TestIt();  // expected-error {{no member named 'TestIt' in 'Foo'; did you mean 'textIt'?}}
-}
-
-void callee(double, double);
-void testNoCandidates() {
-  callee(xxxxxx,   // expected-error-re {{use of undeclared identifier 'xxxxxx'{{$}}}}
-         zzzzzz);  // expected-error-re {{use of undeclared identifier 'zzzzzz'{{$}}}}
-}
-
-class string {};
-struct Item {
-  void Nest();
-  string text();
-  Item* next();  // expected-note {{'next' declared here}}
-};
-void testExprFilter(Item *i) {
-  Item *j;
-  j = i->Next();  // expected-error {{no member named 'Next' in 'Item'; did you mean 'next'?}}
-}
-
-// Test that initializer expressions are handled correctly and that the type
-// being initialized is taken into account when choosing a correction.
-namespace initializerCorrections {
-struct Node {
-  string text() const;
-  // Node* Next() is not implemented yet
-};
-void f(Node *node) {
-  // text is only an edit distance of 1 from Next, but would trigger type
-  // conversion errors if used in this initialization expression.
-  Node *next = node->Next();  // expected-error-re {{no member named 'Next' in 'initializerCorrections::Node'{{$}}}}
-}
-
-struct LinkedNode {
-  LinkedNode* next();  // expected-note {{'next' declared here}}
-  string text() const;
-};
-void f(LinkedNode *node) {
-  // text and next are equidistant from Next, but only one results in a valid
-  // initialization expression.
-  LinkedNode *next = node->Next();  // expected-error {{no member named 'Next' in 'initializerCorrections::LinkedNode'; did you mean 'next'?}}
-}
-
-struct NestedNode {
-  NestedNode* Nest();
-  NestedNode* next();
-  string text() const;
-};
-void f(NestedNode *node) {
-  // There are two equidistant, usable corrections for Next: next and Nest
-  NestedNode *next = node->Next();  // expected-error-re {{no member named 'Next' in 'initializerCorrections::NestedNode'{{$}}}}
-}
-}
-
-namespace PR21669 {
-void f(int *i) {
-  // Check that arguments to a builtin with custom type checking are corrected
-  // properly, since calls to such builtins bypass much of the normal code path
-  // for building and checking the call.
-  __atomic_load(i, i, something_something);  // expected-error-re {{use of undeclared identifier 'something_something'{{$}}}}
-}
-}
-
-const int DefaultArg = 9;  // expected-note {{'DefaultArg' declared here}}
-template <int I = defaultArg> struct S {};  // expected-error {{use of undeclared identifier 'defaultArg'; did you mean 'DefaultArg'?}}
-S<1> s;
-
-namespace foo {}
-void test_paren_suffix() {
-  foo::bar({5, 6});  // expected-error-re {{no member named 'bar' in namespace 'foo'{{$}}}}
-#if __cplusplus <= 199711L
-  // expected-error@-2 {{expected expression}}
-#endif
-}
-
-const int kNum = 10;  // expected-note {{'kNum' declared here}}
-class SomeClass {
-  int Kind;
-public:
-  explicit SomeClass() : Kind(kSum) {}  // expected-error {{use of undeclared identifier 'kSum'; did you mean 'kNum'?}}
-};
-
-// There used to be an issue with typo resolution inside overloads.
-struct AssertionResult { ~AssertionResult(); };
-AssertionResult Overload(const char *a);
-AssertionResult Overload(int a);
-void UseOverload() {
-  // expected-note@+1 {{'result' declared here}}
-  const char *result;
-  // expected-error@+1 {{use of undeclared identifier 'resulta'; did you mean 'result'?}}
-  Overload(resulta);
-}
-
-namespace PR21925 {
-struct X {
-  int get() { return 7; }  // expected-note {{'get' declared here}}
-};
-void test() {
-  X variable;  // expected-note {{'variable' declared here}}
-
-  // expected-error@+2 {{use of undeclared identifier 'variableX'; did you mean 'variable'?}}
-  // expected-error@+1 {{no member named 'getX' in 'PR21925::X'; did you mean 'get'?}}
-  int x = variableX.getX();
-}
-}
-
-namespace PR21905 {
-int (*a)() = (void)Z; // expected-error-re {{use of undeclared identifier 'Z'{{$}}}} \
-                      // expected-error {{cannot initialize a variable of type 'int (*)()' with an rvalue of type 'void'}}
-}
-
-namespace PR21947 {
-int blue;  // expected-note {{'blue' declared here}}
-__typeof blur y;  // expected-error {{use of undeclared identifier 'blur'; did you mean 'blue'?}}
-}
-
-namespace PR22092 {
-a = b ? : 0;  // expected-error {{a type specifier is required for all declarations}} \
-              // expected-error-re {{use of undeclared identifier 'b'{{$}}}}
-}
-
-extern long clock (void);
-struct Pointer {
-  void set_xpos(int);
-  void set_ypos(int);
-};
-void MovePointer(Pointer &Click, int x, int y) {  // expected-note 2 {{'Click' declared here}}
-  click.set_xpos(x);  // expected-error {{use of undeclared identifier 'click'; did you mean 'Click'?}}
-  click.set_ypos(x);  // expected-error {{use of undeclared identifier 'click'; did you mean 'Click'?}}
-}
-
-namespace PR22250 {
-// expected-error@+4 {{use of undeclared identifier 'size_t'; did you mean 'sizeof'?}}
-// expected-error-re@+3 {{use of undeclared identifier 'y'{{$}}}}
-// expected-error-re@+2 {{use of undeclared identifier 'z'{{$}}}}
-// expected-error@+1 {{expected ';' after top level declarator}}
-int getenv_s(size_t *y, char(&z)) {}
-}
-
-namespace PR22291 {
-template <unsigned I> void f() {
-  unsigned *prio_bits_array;  // expected-note {{'prio_bits_array' declared here}}
-  // expected-error@+1 {{use of undeclared identifier 'prio_op_array'; did you mean 'prio_bits_array'?}}
-  __atomic_store_n(prio_op_array + I, false, __ATOMIC_RELAXED);
-}
-}
-
-namespace PR22297 {
-double pow(double x, double y);
-struct TimeTicks {
-  static void Now();  // expected-note {{'Now' declared here}}
-};
-void f() {
-  TimeTicks::now();  // expected-error {{no member named 'now' in 'PR22297::TimeTicks'; did you mean 'Now'?}}
-}
-}
-
-namespace PR23005 {
-void f() { int a = Unknown::b(c); }  // expected-error {{use of undeclared identifier 'Unknown'}}
-// expected-error@-1 {{use of undeclared identifier 'c'}}
-}
-
-namespace PR23350 {
-int z = 1 ? N : ;  // expected-error {{expected expression}}
-// expected-error-re@-1 {{use of undeclared identifier 'N'{{$}}}}
-}
-
-// PR 23285. This test must be at the end of the file to avoid additional,
-// unwanted diagnostics.
-// expected-error-re@+2 {{use of undeclared identifier 'uintmax_t'{{$}}}}
-// expected-error@+1 {{expected ';' after top level declarator}}
-unsigned int a = 0(uintmax_t
diff --git a/external/llvm-project/clang/test/SemaCXX/typo-correction.cpp b/external/llvm-project/clang/test/SemaCXX/typo-correction.cpp
index 45f42c426035..e4dadf83e0a0 100644
--- a/external/llvm-project/clang/test/SemaCXX/typo-correction.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/typo-correction.cpp
@@ -3,7 +3,6 @@
 
 namespace PR21817{
 int a(-rsing[2]); // expected-error {{undeclared identifier 'rsing'; did you mean 'using'?}}
-                  // expected-error@-1 {{expected expression}}
 }
 
 struct errc {
@@ -43,14 +42,14 @@ inline error_condition make_error_condition(errc _e) {
 // refer to a base class or non-static data member.
 struct BaseType { };
 struct Derived : public BaseType { // expected-note {{base class 'BaseType' specified here}}
-  static int base_type; // expected-note {{'base_type' declared here}}
+  static int base_type;
   Derived() : basetype() {} // expected-error{{initializer 'basetype' does not name a non-static data member or base class; did you mean the base class 'BaseType'?}}
 };
 
 // Test the improvement from passing a callback object to CorrectTypo in
 // the helper function LookupMemberExprInRecord.
 int get_type(struct Derived *st) {
-  return st->Base_Type; // expected-error{{no member named 'Base_Type' in 'Derived'; did you mean 'base_type'?}}
+  return st->Base_Type; // expected-error{{no member named 'Base_Type' in 'Derived'}}
 }
 
 // In this example, somename should not be corrected to the cached correction
@@ -212,12 +211,11 @@ namespace PR13051 {
   };
 
   void foo(); // expected-note{{'foo' declared here}}
-  void g(void(*)()); // expected-note{{candidate function not viable}}
-  void g(bool(S<int>::*)() const); // expected-note{{candidate function not viable}}
+  void g(void(*)());
+  void g(bool(S<int>::*)() const);
 
   void test() {
-    g(&S<int>::tempalte f<int>); // expected-error{{did you mean 'template'?}} \
-                                 // expected-error{{no matching function for call to 'g'}}
+    g(&S<int>::tempalte f<int>); // expected-error{{did you mean 'template'?}}
     g(&S<int>::opeartor bool); // expected-error{{did you mean 'operator'?}}
     g(&S<int>::foo); // expected-error{{no member named 'foo' in 'PR13051::S<int>'; did you mean simply 'foo'?}}
   }
@@ -251,13 +249,13 @@ namespace b6956809_test1 {
 
   struct S1 {
     void method(A*);  // no note here
-    void method(B*);  // expected-note{{'method' declared here}}
+    void method(B*);
   };
 
   void test1() {
     B b;
     S1 s;
-    s.methodd(&b);  // expected-error{{no member named 'methodd' in 'b6956809_test1::S1'; did you mean 'method'}}
+    s.methodd(&b);  // expected-error{{no member named 'methodd' in 'b6956809_test1::S1'}}
   }
 
   struct S2 {
@@ -275,15 +273,15 @@ namespace b6956809_test1 {
 }
 
 namespace b6956809_test2 {
-  template<typename T> struct Err { typename T::error n; };  // expected-error{{type 'void *' cannot be used prior to '::' because it has no members}}
+  template<typename T> struct Err { typename T::error n; };
   struct S {
-    template<typename T> typename Err<T>::type method(T);  // expected-note{{in instantiation of template class 'b6956809_test2::Err<void *>' requested here}}
-    template<typename T> int method(T *);  // expected-note{{'method' declared here}}
+    template<typename T> typename Err<T>::type method(T);
+    template<typename T> int method(T *);
   };
 
   void test() {
     S s;
-    int k = s.methodd((void*)0);  // expected-error{{no member named 'methodd' in 'b6956809_test2::S'; did you mean 'method'?}} expected-note{{while substituting deduced template arguments into function template 'method' [with T = void *]}}
+    int k = s.methodd((void*)0);  // expected-error{{no member named 'methodd' in 'b6956809_test2::S'}}
   }
 }
 
@@ -309,12 +307,12 @@ struct A {
   void CreateBar(float, float);
 };
 struct B : A {
-  using A::CreateFoo; // expected-note {{'CreateFoo' declared here}}
-  void CreateFoo(int, int);  // expected-note {{'CreateFoo' declared here}}
+  using A::CreateFoo;
+  void CreateFoo(int, int);
 };
 void f(B &x) {
-  x.Createfoo(0,0);  // expected-error {{no member named 'Createfoo' in 'PR13387::B'; did you mean 'CreateFoo'?}}
-  x.Createfoo(0.f,0.f);  // expected-error {{no member named 'Createfoo' in 'PR13387::B'; did you mean 'CreateFoo'?}}
+  x.Createfoo(0,0);  // expected-error {{no member named 'Createfoo' in 'PR13387::B'}}
+  x.Createfoo(0.f,0.f);  // expected-error {{no member named 'Createfoo' in 'PR13387::B'}}
 }
 }
 
@@ -649,12 +647,12 @@ class AddObservation { // expected-note {{declared here}}
 
 namespace testNonStaticMemberHandling {
 struct Foo {
-  bool usesMetadata;  // expected-note {{'usesMetadata' declared here}}
+  bool usesMetadata;
 };
 int test(Foo f) {
   if (UsesMetadata)  // expected-error-re {{use of undeclared identifier 'UsesMetadata'{{$}}}}
     return 5;
-  if (f.UsesMetadata)  // expected-error {{no member named 'UsesMetadata' in 'testNonStaticMemberHandling::Foo'; did you mean 'usesMetadata'?}}
+  if (f.UsesMetadata)  // expected-error {{no member named 'UsesMetadata' in 'testNonStaticMemberHandling::Foo'}}
     return 11;
   return 0;
 }
@@ -707,7 +705,7 @@ using C::D::Foofoo;  // expected-error {{no member named 'Foofoo' in namespace '
 int d = ? L : d; // expected-error {{expected expression}} expected-error {{undeclared identifier}}
 
 struct B0 {
-  int : 0 |         // expected-error {{invalid operands to binary expression}}
+  int : 0 |
       (struct B0)e; // expected-error {{use of undeclared identifier}}
 };
 
diff --git a/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.cpp b/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.cpp
index 4b41bfbfdc2f..ff3b85d19fa6 100644
--- a/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.cpp
@@ -1,7 +1,9 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=hidden -Wunique-object-duplication -fvisibility=hidden -Wno-unused-value %s
-// RUN: %clang_cc1 -fsyntax-only -verify -Wunique-object-duplication -Wno-unused-value %s
-// The check is currently disabled on windows in MSVC-like environments. The test should fail because we're not getting the expected warnings.
-// XFAIL: target={{.*}}-windows-msvc, {{.*}}-ps{{(4|5)(-.+)?}}
+// RUN: %clang_cc1 -fsyntax-only -Wunique-object-duplication -Wno-unused-value \
+// RUN:   -verify -triple=x86_64-pc-linux-gnu %s
+// RUN: %clang_cc1 -fsyntax-only -Wunique-object-duplication -Wno-unused-value \
+// RUN:   -verify=hidden -triple=x86_64-pc-linux-gnu -fvisibility=hidden  %s
+// RUN: %clang_cc1 -fsyntax-only -Wunique-object-duplication -Wno-unused-value \
+// RUN:   -verify=windows -triple=x86_64-windows-msvc -DWINDOWS_TEST -fdeclspec %s
 
 #include "unique_object_duplication.h"
 
diff --git a/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.h b/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.h
index 537429d9ebda..bd0ee6bd14d6 100644
--- a/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.h
+++ b/external/llvm-project/clang/test/SemaCXX/unique_object_duplication.h
@@ -3,8 +3,14 @@
  * See the warning's documentation for more information.
  */
 
+#ifdef WINDOWS_TEST
+#define HIDDEN
+// dllimport also suffices for visibility, but those can't have definitions
+#define VISIBLE __declspec(dllexport)
+#else
 #define HIDDEN __attribute__((visibility("hidden")))
-#define DEFAULT __attribute__((visibility("default")))
+#define VISIBLE __attribute__((visibility("default")))
+#endif
 
 // Helper functions
 constexpr int init_constexpr(int x) { return x; };
@@ -17,10 +23,11 @@ namespace StaticLocalTest {
 
 inline void has_static_locals_external() {
   // Mutable
-  static int disallowedStatic1 = 0; // hidden-warning {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  static int disallowedStatic1 = 0; // hidden-warning {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                    // windows-warning@-1 {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
   // Initialization might run more than once
-  static const double disallowedStatic2 = disallowedStatic1++; // hidden-warning {{initialization of 'disallowedStatic2' may run twice when built into a shared library: it has hidden visibility and external linkage}}
-  
+  static const double disallowedStatic2 = disallowedStatic1++; // hidden-warning {{initialization of 'disallowedStatic2' may run twice when built into a shared library: it has external linkage and hidden visibility}}
+                                                               // windows-warning@-1 {{initialization of 'disallowedStatic2' may run twice when built into a shared library: it has external linkage and no import/export annotation}}
   // OK, because immutable and compile-time-initialized
   static constexpr int allowedStatic1 = 0;
   static const float allowedStatic2 = 1;
@@ -53,29 +60,33 @@ void has_static_locals_anon() {
   static double allowedStatic2 = init_dynamic(2);
   static char allowedStatic3 = []() { return allowedStatic1++; }();
   static constexpr int allowedStatic4 = init_constexpr(3);
-} 
+}
 
 } // Anonymous namespace
 
 HIDDEN inline void static_local_always_hidden() {
-    static int disallowedStatic1 = 3; // hidden-warning {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
-                                      // expected-warning@-1 {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+    static int disallowedStatic1 = 3; // hidden-warning {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                      // expected-warning@-1 {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                      // windows-warning@-2 {{'disallowedStatic1' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
     {
-      static int disallowedStatic2 = 3; // hidden-warning {{'disallowedStatic2' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
-                                        // expected-warning@-1 {{'disallowedStatic2' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+      static int disallowedStatic2 = 3; // hidden-warning {{'disallowedStatic2' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                        // expected-warning@-1 {{'disallowedStatic2' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                        // windows-warning@-2 {{'disallowedStatic2' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
     }
 
     auto lmb = []() {
-      static int disallowedStatic3 = 3; // hidden-warning {{'disallowedStatic3' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
-                                        // expected-warning@-1 {{'disallowedStatic3' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+      static int disallowedStatic3 = 3; // hidden-warning {{'disallowedStatic3' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                        // expected-warning@-1 {{'disallowedStatic3' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                        // windows-warning@-2 {{'disallowedStatic3' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
     };
 }
 
-DEFAULT void static_local_never_hidden() {
-    static int allowedStatic1 = 3; 
+// Always visible
+VISIBLE void static_local_never_hidden() {
+    static int allowedStatic1 = 3;
 
     {
-      static int allowedStatic2 = 3; 
+      static int allowedStatic2 = 3;
     }
 
     auto lmb = []() {
@@ -96,7 +107,8 @@ inline void has_regular_local() {
 
 inline void has_thread_local() {
   // thread_local variables are static by default
-  thread_local int disallowedThreadLocal = 0; // hidden-warning {{'disallowedThreadLocal' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  thread_local int disallowedThreadLocal = 0; // hidden-warning {{'disallowedThreadLocal' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                              // windows-warning@-1 {{'disallowedThreadLocal' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
 }
 
 // Functions themselves are always immutable, so referencing them is okay
@@ -109,11 +121,13 @@ inline auto& allowedFunctionReference = has_static_locals_external;
  ******************************************************************************/
 namespace GlobalTest {
   // Mutable
-  inline float disallowedGlobal1 = 3.14; // hidden-warning {{'disallowedGlobal1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
-  
-  // Initialization might run more than once
-  inline const double disallowedGlobal5 = disallowedGlobal1++; // hidden-warning {{initialization of 'disallowedGlobal5' may run twice when built into a shared library: it has hidden visibility and external linkage}}
+  inline float disallowedGlobal1 = 3.14; // hidden-warning {{'disallowedGlobal1' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                         // windows-warning@-1 {{'disallowedGlobal1' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+
 
+  // Initialization might run more than once
+  inline const double disallowedGlobal5 = disallowedGlobal1++; // hidden-warning {{initialization of 'disallowedGlobal5' may run twice when built into a shared library: it has external linkage and hidden visibility}}
+                                                               // windows-warning@-1 {{initialization of 'disallowedGlobal5' may run twice when built into a shared library: it has external linkage and no import/export annotation}}
   // OK because internal linkage, so duplication is intended
   static float allowedGlobal1 = 3.14;
   const double allowedGlobal2 = init_dynamic(2);
@@ -129,34 +143,52 @@ namespace GlobalTest {
   // We don't warn on this because non-inline variables can't (legally) appear
   // in more than one TU.
   float allowedGlobal9 = 3.14;
-  
+
   // Pointers need to be double-const-qualified
-  inline float& nonConstReference = disallowedGlobal1; // hidden-warning {{'nonConstReference' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  inline float& nonConstReference = disallowedGlobal1; // hidden-warning {{'nonConstReference' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                                       // windows-warning@-1 {{'nonConstReference' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
   const inline int& constReference = allowedGlobal5;
 
-  inline int* nonConstPointerToNonConst = nullptr; // hidden-warning {{'nonConstPointerToNonConst' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
-  inline int const* nonConstPointerToConst = nullptr; // hidden-warning {{'nonConstPointerToConst' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
-  inline int* const constPointerToNonConst = nullptr; // hidden-warning {{'constPointerToNonConst' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  inline int* nonConstPointerToNonConst = nullptr; // hidden-warning {{'nonConstPointerToNonConst' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                                   // windows-warning@-1 {{'nonConstPointerToNonConst' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+  inline int const* nonConstPointerToConst = nullptr; // hidden-warning {{'nonConstPointerToConst' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                                      // windows-warning@-1 {{'nonConstPointerToConst' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+  inline int* const constPointerToNonConst = nullptr; // hidden-warning {{'constPointerToNonConst' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                                      // windows-warning@-1 {{'constPointerToNonConst' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
   inline int const* const constPointerToConst = nullptr;
   // Don't warn on new because it tends to generate false positives
   inline int const* const constPointerToConstNew = new int(7);
 
   inline int const * const * const * const nestedConstPointer = nullptr;
-  inline int const * const ** const * const nestedNonConstPointer = nullptr; // hidden-warning {{'nestedNonConstPointer' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  inline int const * const ** const * const nestedNonConstPointer = nullptr; // hidden-warning {{'nestedNonConstPointer' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                                                             // windows-warning@-1 {{'nestedNonConstPointer' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
 
   struct Test {
-    static inline float disallowedStaticMember1; // hidden-warning {{'disallowedStaticMember1' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}       
+    static inline float disallowedStaticMember1; // hidden-warning {{'disallowedStaticMember1' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                                 // windows-warning@-1 {{'disallowedStaticMember1' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
     // Defined below, in the header file
-    static float disallowedStaticMember2;                                       
+    static float disallowedStaticMember2;
     // Defined in the cpp file, so won't get duplicated
     static float allowedStaticMember1;
 
+    // Always visible
+    VISIBLE static inline float allowedStaticMember2 = 0.0;
+
     // Tests here are sparse because the AddrTest case below will define plenty
     // more, which aren't problematic to define (because they're immutable), but
     // may still cause problems if their address is taken.
   };
 
-  inline float Test::disallowedStaticMember2 = 2.3; // hidden-warning {{'disallowedStaticMember2' may be duplicated when built into a shared library: it is mutable, has hidden visibility, and external linkage}}
+  inline float Test::disallowedStaticMember2 = 2.3; // hidden-warning {{'disallowedStaticMember2' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
+                                                    // windows-warning@-1 {{'disallowedStaticMember2' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+
+  // This is always visible, so nothing inside it will get duplicated
+  struct VISIBLE NeverHidden {
+    static inline float allowedStaticMember3;
+    static float allowedStaticMember4;
+  };
+
+  inline float NeverHidden::allowedStaticMember4 = 3.4;
 } // namespace GlobalTest
 
 /******************************************************************************
@@ -165,7 +197,7 @@ namespace GlobalTest {
 
 namespace TemplateTest {
 
-// We never warn inside templates because it's frequently infeasible to actually
+// We never warn inside templates because it's usually infeasible to actually
 // fix the warning.
 
 template <typename T>
diff --git a/external/llvm-project/clang/test/SemaCXX/virtuals.cpp b/external/llvm-project/clang/test/SemaCXX/virtuals.cpp
index 2a22ab9fc2b0..f6f52d51f650 100644
--- a/external/llvm-project/clang/test/SemaCXX/virtuals.cpp
+++ b/external/llvm-project/clang/test/SemaCXX/virtuals.cpp
@@ -58,10 +58,8 @@ struct Base {
 };
 
 struct Derived final : Base {
-  virtual ~Derived() = defaul; // #default
+  virtual ~Derived() = defaul; // expected-error {{use of undeclared identifier 'defaul'}}
 } do_not_crash;
-// expected-error@#default {{initializer on function does not look like a pure-specifier}}
-// expected-error@#default {{use of undeclared identifier 'defaul'}}
 }
 
 namespace VirtualFriend {
diff --git a/external/llvm-project/clang/test/SemaHLSL/Semantics/position.ps.hlsl b/external/llvm-project/clang/test/SemaHLSL/Semantics/position.ps.hlsl
new file mode 100644
index 000000000000..32bc5f55b2ab
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaHLSL/Semantics/position.ps.hlsl
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-pixel -x hlsl -finclude-default-header -o - %s -ast-dump | FileCheck %s
+
+float4 main(float4 a : SV_Position) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:8 main 'float4 (float4)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:20 a 'float4':'vector<float, 4>'
+// CHECK-NEXT: HLSLSV_PositionAttr 0x{{[0-9a-fA-F]+}} <{{.*}}>
+}
diff --git a/external/llvm-project/clang/test/SemaHLSL/Semantics/position.ps.size.hlsl b/external/llvm-project/clang/test/SemaHLSL/Semantics/position.ps.size.hlsl
new file mode 100644
index 000000000000..124d401a9990
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaHLSL/Semantics/position.ps.size.hlsl
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -finclude-default-header -o - %s -verify -verify-ignore-unexpected
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-library  -x hlsl -finclude-default-header -o - %s -verify -verify-ignore-unexpected
+
+// expected-error@+1 {{attribute 'SV_Position' only applies to a field or parameter of type 'float/float1/float2/float3/float4'}}
+void main(vector<float, 5> a : SV_Position) {
+}
+
+// expected-error@+1 {{attribute 'SV_Position' only applies to a field or parameter of type 'float/float1/float2/float3/float4'}}
+void main(int2 a : SV_Position) {
+}
diff --git a/external/llvm-project/clang/test/SemaHLSL/Semantics/position.vs.hlsl b/external/llvm-project/clang/test/SemaHLSL/Semantics/position.vs.hlsl
new file mode 100644
index 000000000000..19f781fa3757
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaHLSL/Semantics/position.vs.hlsl
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-vertex -x hlsl -finclude-default-header -o - %s -verify
+
+// expected-error@+1 {{attribute 'SV_Position' is unsupported in 'vertex' shaders, requires pixel}}
+float4 main(float4 a : SV_Position) {
+  return a;
+}
diff --git a/external/llvm-project/clang/test/SemaHLSL/vk-ext-input-builtin.hlsl b/external/llvm-project/clang/test/SemaHLSL/vk-ext-input-builtin.hlsl
new file mode 100644
index 000000000000..e636e146cf68
--- /dev/null
+++ b/external/llvm-project/clang/test/SemaHLSL/vk-ext-input-builtin.hlsl
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple spirv-unkown-vulkan1.3-compute -x hlsl -hlsl-entry foo  -finclude-default-header -o - %s -verify
+
+// expected-error@+1 {{'ext_builtin_input' attribute only applies to static const globals}}
+[[vk::ext_builtin_input(/* WorkgroupId */ 26)]]
+const uint3 groupid1;
+
+// expected-error@+1 {{'ext_builtin_input' attribute only applies to static const globals}}
+[[vk::ext_builtin_input(/* WorkgroupId */ 26)]]
+static uint3 groupid2;
+
+// expected-error@+1 {{'ext_builtin_input' attribute takes one argument}}
+[[vk::ext_builtin_input()]]
+// expected-error@+1 {{default initialization of an object of const type 'const hlsl_private uint3' (aka 'const hlsl_private vector<uint, 3>')}}
+static const uint3 groupid3;
+
+// expected-error@+1 {{'ext_builtin_input' attribute requires an integer constant}}
+[[vk::ext_builtin_input(0.4f)]]
+// expected-error@+1 {{default initialization of an object of const type 'const hlsl_private uint3' (aka 'const hlsl_private vector<uint, 3>')}}
+static const uint3 groupid4;
+
+// expected-error@+1 {{'ext_builtin_input' attribute only applies to static const globals}}
+[[vk::ext_builtin_input(1)]]
+void some_function() {
+}
+
+[numthreads(1,1,1)]
+void foo() {
+}
+
diff --git a/external/llvm-project/clang/test/SemaObjC/call-super-2.m b/external/llvm-project/clang/test/SemaObjC/call-super-2.m
index 01acff70c230..885f392e353a 100644
--- a/external/llvm-project/clang/test/SemaObjC/call-super-2.m
+++ b/external/llvm-project/clang/test/SemaObjC/call-super-2.m
@@ -115,7 +115,7 @@ @interface B : A
 @end
 
 @implementation B
-- (instancetype)initWithCoder:(C *)coder {
+- (instancetype)initWithCoder:(C *)coder {     // expected-note {{'coder' declared here}}
   if (0 != (self = [super initWithCode:code])) // expected-error {{use of undeclared identifier 'code'}} expected-warning {{instance method '-initWithCode:' not found}}
     return (void *)0;
   return (void *)0;
diff --git a/external/llvm-project/clang/test/SemaObjC/typo-correction-subscript.m b/external/llvm-project/clang/test/SemaObjC/typo-correction-subscript.m
index 340f3cfe2743..6c09127dbb8d 100644
--- a/external/llvm-project/clang/test/SemaObjC/typo-correction-subscript.m
+++ b/external/llvm-project/clang/test/SemaObjC/typo-correction-subscript.m
@@ -7,8 +7,7 @@ @interface Test
 @implementation Test
 - (void)rdar47403222:(Dictionary *)opts {
   [self undeclaredMethod:undeclaredArg];
-  // expected-error@-1{{no visible @interface for 'Test' declares the selector 'undeclaredMethod:'}}
-  // expected-error@-2{{use of undeclared identifier 'undeclaredArg}}
+  // expected-error@-1{{use of undeclared identifier 'undeclaredArg}}
   opts[(__bridge id)undeclaredKey] = 0;
   // expected-error@-1{{use of undeclared identifier 'undeclaredKey'}}
 }
diff --git a/external/llvm-project/clang/test/SemaObjC/undef-arg-super-method-call.m b/external/llvm-project/clang/test/SemaObjC/undef-arg-super-method-call.m
index 11fd97f2c00d..b8cbe7f69f2f 100644
--- a/external/llvm-project/clang/test/SemaObjC/undef-arg-super-method-call.m
+++ b/external/llvm-project/clang/test/SemaObjC/undef-arg-super-method-call.m
@@ -11,12 +11,12 @@ @interface DBGViewDebuggerSupport_iOS : DBGViewDebuggerSupport
 @end
 
 @implementation DBGViewDebuggerSupport_iOS
-+ (void)addViewLayerInfo:(id)aView; // expected-note {{'aView' declared here}}
++ (void)addViewLayerInfo:(id)aView;
 {
-    [super addViewLayerInfo:view]; // expected-error {{use of undeclared identifier 'view'; did you mean 'aView'?}}
+    [super addViewLayerInfo:view]; // expected-error {{use of undeclared identifier 'view'}}
 }
-- (void)addInstViewLayerInfo:(id)aView; // expected-note {{'aView' declared here}}
+- (void)addInstViewLayerInfo:(id)aView;
 {
-    [super addInstViewLayerInfo:view]; // expected-error {{use of undeclared identifier 'view'; did you mean 'aView'?}}
+    [super addInstViewLayerInfo:view]; // expected-error {{use of undeclared identifier 'view'}}
 }
 @end
diff --git a/external/llvm-project/clang/test/SemaObjCXX/attr-trivial-abi.mm b/external/llvm-project/clang/test/SemaObjCXX/attr-trivial-abi.mm
index 87b79c14d07a..a87c4d3f990d 100644
--- a/external/llvm-project/clang/test/SemaObjCXX/attr-trivial-abi.mm
+++ b/external/llvm-project/clang/test/SemaObjCXX/attr-trivial-abi.mm
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++11 -fobjc-runtime-has-weak -fobjc-weak -fobjc-arc -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++11 -fobjc-runtime-has-weak -fobjc-weak -fobjc-arc -triple x86_64-apple-darwin -fsyntax-only -verify %s
 
 void __attribute__((trivial_abi)) foo(); // expected-warning {{'trivial_abi' attribute only applies to classes}}
 
@@ -108,7 +108,8 @@ struct __attribute__((trivial_abi)) CopyMoveDeleted { // expected-warning {{'tri
     CopyMoveDeleted(CopyMoveDeleted &&) = delete;
   };
 
-  struct __attribute__((trivial_abi)) S18 { // expected-warning {{'trivial_abi' cannot be applied to 'S18'}} expected-note {{copy constructors and move constructors are all deleted}}
+  struct __attribute__((trivial_abi)) S18 { // expected-warning {{'trivial_abi' cannot be applied to 'S18'}} \
+                                            // expected-note {{'trivial_abi' is disallowed on 'S18' because it has a field of a non-trivial class type}}
     CopyMoveDeleted a;
   };
 
diff --git a/external/llvm-project/clang/test/SemaObjCXX/block-for-lambda-conversion.mm b/external/llvm-project/clang/test/SemaObjCXX/block-for-lambda-conversion.mm
index 671e83dc2201..a3bcfab67719 100644
--- a/external/llvm-project/clang/test/SemaObjCXX/block-for-lambda-conversion.mm
+++ b/external/llvm-project/clang/test/SemaObjCXX/block-for-lambda-conversion.mm
@@ -8,19 +8,20 @@
   NSEventMaskLeftMouseDown = 1
 };
 
-static const NSEventType NSFlagsChanged = NSEventTypeFlagsChanged;
+static const NSEventType NSFlagsChanged = NSEventTypeFlagsChanged; // expected-note {{'NSFlagsChanged' declared here}}
 
 @interface NSObject
 @end
 @interface NSEvent : NSObject {
 }
 + (nullable id)
-addMonitor:(NSEventMask)mask handler:(NSEvent *_Nullable (^)(NSEvent *))block;
+addMonitor:(NSEventMask)mask handler:(NSEvent *_Nullable (^)(NSEvent *))block; // expected-note {{passing argument to parameter 'mask' here}}
 @end
 
 void test(id weakThis) {
   id m_flagsChangedEventMonitor = [NSEvent
-      addMonitor:NSFlagsChangedMask //expected-error {{use of undeclared identifier 'NSFlagsChangedMask'}}
+      addMonitor:NSFlagsChangedMask //expected-error {{use of undeclared identifier 'NSFlagsChangedMask'}} \
+                                      expected-error {{cannot initialize a parameter of type 'NSEventMask' with an lvalue of type 'const NSEventType'}}
          handler:[weakThis](NSEvent *flagsChangedEvent) {
              return flagsChangedEvent;
          }];
diff --git a/external/llvm-project/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.cpp b/external/llvm-project/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.cpp
index c6dbe4db2be6..0cf27666dd03 100644
--- a/external/llvm-project/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.cpp
+++ b/external/llvm-project/clang/test/SemaOpenACC/compute-construct-num_gangs-clause.cpp
@@ -119,8 +119,7 @@ struct HasInt {
 
 template <typename T>
 void TestInst() {
-  // expected-error@+2{{no member named 'Invalid' in 'HasInt'}}
-  // expected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'serial' directive}}
+  // expected-error@+1{{no member named 'Invalid' in 'HasInt'}}
 #pragma acc serial num_gangs(HasInt::Invalid)
   while(1);
 
@@ -137,8 +136,7 @@ void TestInst() {
 #pragma acc parallel num_gangs(T::Invalid, 1)
   while(1);
 
-  // expected-error@+2{{no member named 'Invalid' in 'HasInt'}}
-  // expected-error@+1{{OpenACC 'num_gangs' clause is not valid on 'serial' directive}}
+  // expected-error@+1{{no member named 'Invalid' in 'HasInt'}}
 #pragma acc serial num_gangs(1, HasInt::Invalid)
   while(1);
 
diff --git a/external/llvm-project/clang/test/SemaOpenCL/atomic-ops.cl b/external/llvm-project/clang/test/SemaOpenCL/atomic-ops.cl
index 7a273546db77..babebba31e82 100644
--- a/external/llvm-project/clang/test/SemaOpenCL/atomic-ops.cl
+++ b/external/llvm-project/clang/test/SemaOpenCL/atomic-ops.cl
@@ -167,7 +167,7 @@ void syncscope_checks(atomic_int *Ap, int scope) {
   (void)__opencl_atomic_load(Ap, memory_order_relaxed, memory_scope_all_devices);
 #if __OPENCL_C_VERSION__ < CL_VERSION_3_0
   // expected-error@-2{{use of undeclared identifier 'memory_scope_all_devices'}}
-  // expected-note@* {{'memory_scope_all_svm_devices' declared here}}
+  // expected-note@opencl-c-base.h:*{{'memory_scope_all_svm_devices' declared here}}
 #endif
   (void)__opencl_atomic_load(Ap, memory_order_relaxed, memory_scope_sub_group);
   (void)__opencl_atomic_load(Ap, memory_order_relaxed, scope);
diff --git a/external/llvm-project/clang/test/SemaOpenCL/clang-builtin-version.cl b/external/llvm-project/clang/test/SemaOpenCL/clang-builtin-version.cl
index ec6eecee3106..21cbf2d8f28d 100644
--- a/external/llvm-project/clang/test/SemaOpenCL/clang-builtin-version.cl
+++ b/external/llvm-project/clang/test/SemaOpenCL/clang-builtin-version.cl
@@ -17,12 +17,8 @@ kernel void dse_builtins(void) {
   });
 #if (__OPENCL_C_VERSION__ >= CL_VERSION_3_0) && !defined(__opencl_c_device_enqueue)
 // expected-error@-10{{support disabled - compile with -fblocks or for OpenCL C 2.0 or OpenCL C 3.0 with __opencl_c_device_enqueue feature}}
-// FIXME: the typo correction for the undeclared identifiers finds alternative
-// suggestions, but instantiating the typo correction causes us to
-// re-instantiate the argument to the call, which triggers the support
-// diagnostic a second time.
-// expected-error@-12 2{{support disabled - compile with -fblocks or for OpenCL C 2.0 or OpenCL C 3.0 with __opencl_c_device_enqueue feature}}
-// expected-error@-10 2{{support disabled - compile with -fblocks or for OpenCL C 2.0 or OpenCL C 3.0 with __opencl_c_device_enqueue feature}}
+// expected-error@-8 {{support disabled - compile with -fblocks or for OpenCL C 2.0 or OpenCL C 3.0 with __opencl_c_device_enqueue feature}}
+// expected-error@-6 {{support disabled - compile with -fblocks or for OpenCL C 2.0 or OpenCL C 3.0 with __opencl_c_device_enqueue feature}}
 #endif
 }
 
diff --git a/external/llvm-project/clang/test/SemaTemplate/address_space-dependent.cpp b/external/llvm-project/clang/test/SemaTemplate/address_space-dependent.cpp
index 4eabcfa0fcf2..e17bf60e6a20 100644
--- a/external/llvm-project/clang/test/SemaTemplate/address_space-dependent.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/address_space-dependent.cpp
@@ -43,7 +43,7 @@ void neg() {
 
 template <long int I>
 void tooBig() {
-  __attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388583)}}
+  __attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388582)}}
 }
 
 template <long int I>
@@ -101,7 +101,7 @@ int main() {
   car<1, 2, 3>(); // expected-note {{in instantiation of function template specialization 'car<1, 2, 3>' requested here}}
   HasASTemplateFields<1> HASTF;
   neg<-1>(); // expected-note {{in instantiation of function template specialization 'neg<-1>' requested here}}
-  correct<0x7FFFE7>();
+  correct<0x7FFFE6>();
   tooBig<8388650>(); // expected-note {{in instantiation of function template specialization 'tooBig<8388650L>' requested here}}
 
   __attribute__((address_space(1))) char *x;
diff --git a/external/llvm-project/clang/test/SemaTemplate/concepts-recovery-expr.cpp b/external/llvm-project/clang/test/SemaTemplate/concepts-recovery-expr.cpp
index b338f3bc271b..6bed1790051f 100644
--- a/external/llvm-project/clang/test/SemaTemplate/concepts-recovery-expr.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/concepts-recovery-expr.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -std=c++20 -verify %s
 
-// expected-error@+1{{use of undeclared identifier 'b'}}
-constexpr bool CausesRecoveryExpr = b;
+// expected-error@+1 {{invalid operands to binary expression ('const char[5]' and 'float')}}
+constexpr bool CausesRecoveryExpr = "test" + 1.0f;
 
 template<typename T>
 concept ReferencesCRE = CausesRecoveryExpr;
diff --git a/external/llvm-project/clang/test/SemaTemplate/concepts-recursive-inst.cpp b/external/llvm-project/clang/test/SemaTemplate/concepts-recursive-inst.cpp
index 9330df8cdd03..097cad1a6417 100644
--- a/external/llvm-project/clang/test/SemaTemplate/concepts-recursive-inst.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/concepts-recursive-inst.cpp
@@ -76,7 +76,7 @@ auto it = begin(rng); // #BEGIN_CALL
 // expected-note@#INF_BEGIN {{while checking the satisfaction of concept 'Inf<DirectRecursiveCheck::my_range>' requested here}}
 // expected-note@#INF_BEGIN {{while substituting template arguments into constraint expression here}}
 // expected-note@#BEGIN_CALL {{while checking constraint satisfaction for template 'begin<DirectRecursiveCheck::my_range>' required here}}
-// expected-note@#BEGIN_CALL {{in instantiation of function template specialization}}
+// expected-note@#BEGIN_CALL {{while substituting deduced template arguments into function template}}
 
 // Fallout of the failure is failed lookup, which is necessary to stop odd
 // cascading errors.
@@ -103,7 +103,7 @@ namespace GH50891 {
   // expected-note@#OP_TO {{while checking the satisfaction of concept 'Numeric<GH50891::Deferred>' requested here}}
   // expected-note@#OP_TO {{while substituting template arguments into constraint expression here}}
   // expected-note@#FOO_CALL {{while checking constraint satisfaction for template}}
-  // expected-note@#FOO_CALL {{in instantiation of function template specialization}}
+  // expected-note@#FOO_CALL {{while substituting deduced template arguments into function template}}
   // expected-note@#FOO_CALL {{in instantiation of requirement here}}
   // expected-note@#NUMERIC {{while substituting template arguments into constraint expression here}}
 
@@ -143,3 +143,222 @@ namespace GH60323 {
             Size().sizeparens(i);
   }
 }
+
+namespace CWG2369_Regressions {
+
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109397
+namespace GCC_103997 {
+
+template<typename _type, typename _stream>
+concept streamable = requires(_stream &s, _type &&v) {
+  s << static_cast<_type &&>(v);
+};
+
+struct type_a {
+  template<typename _arg>
+  type_a &operator<<(_arg &&) {
+    // std::clog << "type_a" << std::endl;
+    return *this;
+  }
+};
+
+struct type_b {
+  type_b &operator<<(type_a const &) {
+    // std::clog << "type_b" << std::endl;
+    return *this;
+  }
+};
+
+struct type_c {
+  type_b b;
+  template<typename _arg>
+  requires streamable<_arg, type_b>
+  friend type_c &operator<<(type_c &c, _arg &&a) {
+    // std::clog << "type_c" << std::endl;
+    c.b << static_cast<_arg &&>(a);
+    return c;
+  }
+};
+
+void foo() {
+  type_a a;
+  type_c c;
+  a << c; // "type_a\n" (gcc gives error here)
+  c << a; // "type_c\ntype_b\n"
+}
+
+}
+
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108393
+namespace GCC_108393 {
+
+template<class>
+struct iterator_traits
+{};
+
+template<class T>
+  requires requires(T __t, T __u) { __t == __u; }
+struct iterator_traits<T>
+{};
+
+template<class T>
+concept C = requires { typename iterator_traits<T>::A; };
+
+struct unreachable_sentinel_t
+{
+  template<C _Iter>
+  friend constexpr bool operator==(unreachable_sentinel_t, const _Iter&) noexcept;
+};
+
+template<class T>
+struct S
+{};
+
+static_assert(!C<S<unreachable_sentinel_t>>);
+
+}
+
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107429
+namespace GCC_107429 {
+
+struct tag_foo { } inline constexpr foo;
+struct tag_bar { } inline constexpr bar;
+
+template<typename... T>
+auto f(tag_foo, T... x)
+{
+  return (x + ...);
+}
+
+template<typename... T>
+concept fooable = requires (T... x) { f(foo, x...); };
+
+template<typename... T> requires (fooable<T...>)
+auto f(tag_bar, T... x)
+{
+  return f(foo, x...);
+}
+
+auto test()
+{
+  return f(bar, 1, 2, 3);
+}
+
+}
+
+namespace GCC_99599 {
+
+struct foo_tag {};
+struct bar_tag {};
+
+template <class T>
+concept fooable = requires(T it) {
+  invoke_tag(foo_tag{}, it); // <-- here
+};
+
+template <class T> auto invoke_tag(foo_tag, T in) { return in; }
+
+template <fooable T> auto invoke_tag(bar_tag, T it) { return it; }
+
+int main() {
+  // Neither line below compiles in GCC 11, independently of the other
+  return invoke_tag(foo_tag{}, 2) + invoke_tag(bar_tag{}, 2);
+}
+
+}
+
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=99599#c22
+namespace GCC_99599_2 {
+
+template<typename T> class indirect {
+public:
+  template<typename U> requires
+    requires (const T& t, const U& u) { t == u; }
+  friend constexpr bool operator==(const indirect&, const U&) { return false; }
+
+private:
+  T* _M_ptr{};
+};
+
+indirect<int> i;
+bool b = i == 1;
+
+}
+
+namespace GCC_99599_3 {
+
+template<typename T>
+struct S { T t; };
+
+template<typename T>
+concept C = sizeof(S<T>) > 0;
+
+struct I;
+
+struct from_range_t {
+    explicit from_range_t() = default;
+};
+inline constexpr from_range_t from_range;
+
+template<typename T>
+concept FromRange = __is_same_as (T, from_range_t);
+
+//#define WORKAROUND
+#ifdef WORKAROUND
+template<FromRange U, C T>
+void f(U, T*);
+#else
+template<C T>
+void f(from_range_t, T*);
+#endif
+
+void f(...);
+
+void g(I* p) {
+  f(0, p);
+}
+
+}
+
+namespace GCC_99599_4 {
+
+struct A {
+  A(...);
+};
+
+template <class T> void f(A, T) { }
+
+int main()
+{
+  f(42, 24);
+}
+
+}
+
+namespace FAILED_GCC_110160 {
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110160
+// Current heuristic FAILED; GCC trunk also failed
+// https://godbolt.org/z/r3Pz9Tehz
+#if 0
+#include <sstream>
+#include <string>
+
+template <class T>
+concept StreamCanReceiveString = requires(T& t, std::string s) {
+    { t << s };
+};
+
+struct NotAStream {};
+struct UnrelatedType {};
+
+template <StreamCanReceiveString S>
+S& operator<<(S& s, UnrelatedType) {
+    return s;
+}
+
+static_assert(!StreamCanReceiveString<NotAStream>);
+
+static_assert(StreamCanReceiveString<std::stringstream>);
+#endif
+}
+}
diff --git a/external/llvm-project/clang/test/SemaTemplate/concepts.cpp b/external/llvm-project/clang/test/SemaTemplate/concepts.cpp
index f335ca3bd22b..62a4f95d79c7 100644
--- a/external/llvm-project/clang/test/SemaTemplate/concepts.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/concepts.cpp
@@ -814,11 +814,7 @@ static_assert(invalid<int> also here ; // expected-error{{use of undeclared iden
 
 int foo() {
     bool b;
-    b = invalid<int> not just in declarations; // expected-error{{expected ';' after expression}}
-                                               // expected-error@-1{{use of undeclared identifier 'invalid'}}
-                                               // expected-error@-2{{expected ';' after expression}}
-                                               // expected-error@-3{{use of undeclared identifier 'just'}}
-                                               // expected-error@-4{{unknown type name 'in'}}
+    b = invalid<int> not just in declarations; // expected-error{{use of undeclared identifier 'invalid'}}
     return b;
 }
 } // namespace GH48182
@@ -1177,3 +1173,80 @@ template <has_member_difference_type Tp>
 struct incrementable_traits<Tp>; // expected-error {{not more specialized than the primary}}
 
 }
+
+namespace InjectedClassNameType {
+
+template <class, class _Err> class expected {
+public:
+  template <class...>
+  expected(...);
+
+  template <class _T2, class _E2>
+  friend bool operator==(expected x, expected<_T2, _E2>)
+    requires requires {
+      { x };
+    }
+  {
+    return true;
+  }
+};
+
+bool test_val_types() {
+  return expected<void, int>() == 1;
+}
+
+}
+
+namespace CWG2369_Regression {
+
+enum class KindEnum {
+  Unknown = 0,
+  Foo = 1,
+};
+
+template <typename T>
+concept KnownKind = T::kind() != KindEnum::Unknown;
+
+template <KnownKind T> struct KnownType;
+
+struct Type {
+  KindEnum kind() const;
+
+  static Type f(Type t);
+
+  template <KnownKind T> static KnownType<T> f(T t);
+
+  static void g() {
+    Type t;
+    f(t);
+  }
+};
+
+template <KnownKind T> struct KnownType {
+  static constexpr KindEnum kind() { return KindEnum::Foo; }
+};
+
+}
+
+namespace GH115838 {
+
+template<typename T> concept has_x = requires(T t) {{ t.x };};
+
+class Publ { public:    int x = 0; };
+class Priv { private:   int x = 0; };
+class Prot { protected: int x = 0; };
+class Same { protected: int x = 0; };
+
+template<typename T> class D;
+template<typename T> requires ( has_x<T>) class D<T>: public T { public: static constexpr bool has = 1; };
+template<typename T> requires (!has_x<T>) class D<T>: public T { public: static constexpr bool has = 0; };
+
+// "Same" is identical to "Prot" but queried before used.
+static_assert(!has_x<Same>,  "Protected should be invisible.");
+static_assert(!D<Same>::has, "Protected should be invisible.");
+
+static_assert( D<Publ>::has, "Public should be visible.");
+static_assert(!D<Priv>::has, "Private should be invisible.");
+static_assert(!D<Prot>::has, "Protected should be invisible.");
+
+}
diff --git a/external/llvm-project/clang/test/SemaTemplate/cxx2a-constraint-exprs.cpp b/external/llvm-project/clang/test/SemaTemplate/cxx2a-constraint-exprs.cpp
index f4403587a625..5809ef684bbf 100644
--- a/external/llvm-project/clang/test/SemaTemplate/cxx2a-constraint-exprs.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/cxx2a-constraint-exprs.cpp
@@ -34,7 +34,7 @@ namespace constant_evaluated {
      expected-note@-1{{candidate template ignored}}
   int a = (foo<int>(), 0);
   // expected-note@-1 {{while checking}} expected-error@-1{{no matching function}} \
-     expected-note@-1 {{in instantiation}}
+     expected-note@-1 {{while substituting}}
   template<typename T> void bar() requires requires { requires f<int[2]>; } { };
   // expected-note@-1{{in instantiation}} \
      expected-note@-1{{while substituting}} \
diff --git a/external/llvm-project/clang/test/SemaTemplate/deduction-guide.cpp b/external/llvm-project/clang/test/SemaTemplate/deduction-guide.cpp
index faabba553950..0953f647426f 100644
--- a/external/llvm-project/clang/test/SemaTemplate/deduction-guide.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/deduction-guide.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2a -verify -ast-dump -ast-dump-decl-types -ast-dump-filter "deduction guide" %s | FileCheck %s --strict-whitespace -dump-input=always
+// RUN: %clang_cc1 -std=c++2a -verify -ast-dump -ast-dump-decl-types -ast-dump-filter "deduction guide" %s | FileCheck %s --strict-whitespace
 
 template<auto ...> struct X {};
 template<template<typename X, X> typename> struct Y {};
@@ -234,7 +234,7 @@ F s(0);
 // CHECK: | `-CXXBoolLiteralExpr {{.*}} 'bool' false
 // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (U) -> F<value-parameter-0-0>'
 // CHECK: | `-ParmVarDecl {{.*}} 'U'
-// CHECK: `-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (int) -> F<>'
+// CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for F> 'auto (int) -> F<>'
 // CHECK:   |-TemplateArgument integral ''x''
 // CHECK:   |-TemplateArgument type 'int'
 // CHECK:   | `-BuiltinType {{.*}} 'int'
diff --git a/external/llvm-project/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp b/external/llvm-project/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
index af3e3358f613..5c7a90273d0e 100644
--- a/external/llvm-project/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
@@ -38,7 +38,7 @@ template<typename A, typename T>
 concept True = true;
 
 template<typename T>
-concept False = false;
+concept False = false; // #False
 
 template<class X> struct concepts {
     template<class Y> struct B {
@@ -68,7 +68,7 @@ template<typename X> struct nested_init_list {
         Y y;
     };
 
-    template<False F>
+    template<False F>  // #INIT_LIST_INNER_INVALID_HEADER
     struct concept_fail { // #INIT_LIST_INNER_INVALID
         X x;
         F f;
@@ -81,7 +81,9 @@ using NIL = nested_init_list<int>::B<int>;
 
 // expected-error@+1 {{no viable constructor or deduction guide for deduction of template arguments of 'nested_init_list<int>::concept_fail'}}
 nested_init_list<int>::concept_fail nil_invalid{1, ""};
-// expected-note@#INIT_LIST_INNER_INVALID {{candidate template ignored: substitution failure [with F = const char *]: constraints not satisfied for class template 'concept_fail' [with F = const char *]}}
+// expected-note@#INIT_LIST_INNER_INVALID {{candidate template ignored: constraints not satisfied [with F = const char *]}}
+// expected-note@#INIT_LIST_INNER_INVALID_HEADER {{because 'const char *' does not satisfy 'False'}}
+// expected-note@#False {{because 'false' evaluated to false}}
 // expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail(int, F) -> concept_fail<F>'}}
 // expected-note@#INIT_LIST_INNER_INVALID {{candidate function template not viable: requires 1 argument, but 2 were provided}}
 // expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail(concept_fail<F>) -> concept_fail<F>'}}
diff --git a/external/llvm-project/clang/test/SemaTemplate/typo-variadic.cpp b/external/llvm-project/clang/test/SemaTemplate/typo-variadic.cpp
index c9b777aebbe9..48306fb9ce80 100644
--- a/external/llvm-project/clang/test/SemaTemplate/typo-variadic.cpp
+++ b/external/llvm-project/clang/test/SemaTemplate/typo-variadic.cpp
@@ -1,2 +1,2 @@
 // RUN: %clang_cc1 -fsyntax-only %s -verify
-int x = m(s...); // expected-error{{pack expansion does not}} expected-error{{undeclared identifier}}
+int x = m(s...); // expected-error{{undeclared identifier}}
diff --git a/external/llvm-project/clang/tools/cir-opt/cir-opt.cpp b/external/llvm-project/clang/tools/cir-opt/cir-opt.cpp
index 0e20b97feced..3dad3b18f708 100644
--- a/external/llvm-project/clang/tools/cir-opt/cir-opt.cpp
+++ b/external/llvm-project/clang/tools/cir-opt/cir-opt.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
+#include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -32,7 +33,8 @@ int main(int argc, char **argv) {
   // TODO: register needed MLIR passes for CIR?
   mlir::DialectRegistry registry;
   registry.insert<mlir::BuiltinDialect, cir::CIRDialect,
-                  mlir::memref::MemRefDialect, mlir::LLVM::LLVMDialect>();
+                  mlir::memref::MemRefDialect, mlir::LLVM::LLVMDialect,
+                  mlir::DLTIDialect>();
 
   ::mlir::registerPass([]() -> std::unique_ptr<::mlir::Pass> {
     return mlir::createCIRCanonicalizePass();
diff --git a/external/llvm-project/clang/tools/clang-format/ClangFormat.cpp b/external/llvm-project/clang/tools/clang-format/ClangFormat.cpp
index b22d3aaf3183..c0efbb7588cc 100644
--- a/external/llvm-project/clang/tools/clang-format/ClangFormat.cpp
+++ b/external/llvm-project/clang/tools/clang-format/ClangFormat.cpp
@@ -244,17 +244,17 @@ static bool fillRanges(MemoryBuffer *Code,
   DiagnosticsEngine Diagnostics(
       IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs), DiagOpts);
   SourceManager Sources(Diagnostics, Files);
-  FileID ID = createInMemoryFile("<irrelevant>", *Code, Sources, Files,
-                                 InMemoryFileSystem.get());
+  const auto ID = createInMemoryFile("<irrelevant>", *Code, Sources, Files,
+                                     InMemoryFileSystem.get());
   if (!LineRanges.empty()) {
     if (!Offsets.empty() || !Lengths.empty()) {
       errs() << "error: cannot use -lines with -offset/-length\n";
       return true;
     }
 
-    for (unsigned i = 0, e = LineRanges.size(); i < e; ++i) {
+    for (const auto &LineRange : LineRanges) {
       unsigned FromLine, ToLine;
-      if (parseLineRange(LineRanges[i], FromLine, ToLine)) {
+      if (parseLineRange(LineRange, FromLine, ToLine)) {
         errs() << "error: invalid <start line>:<end line> pair\n";
         return true;
       }
@@ -266,12 +266,12 @@ static bool fillRanges(MemoryBuffer *Code,
         errs() << "error: start line should not exceed end line\n";
         return true;
       }
-      SourceLocation Start = Sources.translateLineCol(ID, FromLine, 1);
-      SourceLocation End = Sources.translateLineCol(ID, ToLine, UINT_MAX);
+      const auto Start = Sources.translateLineCol(ID, FromLine, 1);
+      const auto End = Sources.translateLineCol(ID, ToLine, UINT_MAX);
       if (Start.isInvalid() || End.isInvalid())
         return true;
-      unsigned Offset = Sources.getFileOffset(Start);
-      unsigned Length = Sources.getFileOffset(End) - Offset;
+      const auto Offset = Sources.getFileOffset(Start);
+      const auto Length = Sources.getFileOffset(End) - Offset;
       Ranges.push_back(tooling::Range(Offset, Length));
     }
     return false;
@@ -279,33 +279,33 @@ static bool fillRanges(MemoryBuffer *Code,
 
   if (Offsets.empty())
     Offsets.push_back(0);
-  if (Offsets.size() != Lengths.size() &&
-      !(Offsets.size() == 1 && Lengths.empty())) {
-    errs() << "error: number of -offset and -length arguments must match.\n";
+  const bool EmptyLengths = Lengths.empty();
+  unsigned Length = 0;
+  if (Offsets.size() == 1 && EmptyLengths) {
+    Length = Sources.getFileOffset(Sources.getLocForEndOfFile(ID)) - Offsets[0];
+  } else if (Offsets.size() != Lengths.size()) {
+    errs() << "error: number of -offset and -length arguments must match\n";
     return true;
   }
-  for (unsigned i = 0, e = Offsets.size(); i != e; ++i) {
-    if (Offsets[i] >= Code->getBufferSize()) {
-      errs() << "error: offset " << Offsets[i] << " is outside the file\n";
+  for (unsigned I = 0, E = Offsets.size(), CodeSize = Code->getBufferSize();
+       I < E; ++I) {
+    const auto Offset = Offsets[I];
+    if (Offset >= CodeSize) {
+      errs() << "error: offset " << Offset << " is outside the file\n";
       return true;
     }
-    SourceLocation Start =
-        Sources.getLocForStartOfFile(ID).getLocWithOffset(Offsets[i]);
-    SourceLocation End;
-    if (i < Lengths.size()) {
-      if (Offsets[i] + Lengths[i] > Code->getBufferSize()) {
-        errs() << "error: invalid length " << Lengths[i]
-               << ", offset + length (" << Offsets[i] + Lengths[i]
-               << ") is outside the file.\n";
-        return true;
-      }
-      End = Start.getLocWithOffset(Lengths[i]);
-    } else {
-      End = Sources.getLocForEndOfFile(ID);
+    if (!EmptyLengths)
+      Length = Lengths[I];
+    if (Length == 0) {
+      errs() << "error: length should be at least 1\n";
+      return true;
+    }
+    if (Offset + Length > CodeSize) {
+      errs() << "error: invalid length " << Length << ", offset + length ("
+             << Offset + Length << ") is outside the file\n";
+      return true;
     }
-    unsigned Offset = Sources.getFileOffset(Start);
-    unsigned Length = Sources.getFileOffset(End) - Offset;
-    Ranges.push_back(tooling::Range(Offset, Length));
+    Ranges.push_back(tooling::Range(Offset, Length - 1));
   }
   return false;
 }
diff --git a/external/llvm-project/clang/tools/clang-installapi/Options.cpp b/external/llvm-project/clang/tools/clang-installapi/Options.cpp
index 73f5470b8248..52b48a34d361 100644
--- a/external/llvm-project/clang/tools/clang-installapi/Options.cpp
+++ b/external/llvm-project/clang/tools/clang-installapi/Options.cpp
@@ -763,7 +763,6 @@ getInterfaceFile(const StringRef Filename) {
     return errorCodeToError(std::move(Err));
 
   auto Buffer = std::move(*BufferOrErr);
-  std::unique_ptr<InterfaceFile> IF;
   switch (identify_magic(Buffer->getBuffer())) {
   case file_magic::macho_dynamically_linked_shared_lib:
   case file_magic::macho_dynamically_linked_shared_lib_stub:
@@ -1084,10 +1083,10 @@ void Options::addConditionalCC1Args(std::vector<std::string> &ArgStrings,
   // Add specific to platform arguments.
   PathSeq PlatformSearchPaths =
       getPathsForPlatform(FEOpts.SystemFwkPaths, mapToPlatformType(Targ));
-  llvm::for_each(PlatformSearchPaths, [&ArgStrings](const StringRef Path) {
+  for (StringRef Path : PlatformSearchPaths) {
     ArgStrings.push_back("-iframework");
     ArgStrings.push_back(Path.str());
-  });
+  }
 
   // Add specific to header type arguments.
   if (Type == HeaderType::Project)
diff --git a/external/llvm-project/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/external/llvm-project/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index ed33449cff65..b8f45925d2a4 100644
--- a/external/llvm-project/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/external/llvm-project/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -310,22 +310,21 @@ Error relocateOffloadSection(const ArgList &Args, StringRef Output) {
   // Remove the old .llvm.offloading section to prevent further linking.
   ObjcopyArgs.emplace_back("--remove-section");
   ObjcopyArgs.emplace_back(".llvm.offloading");
-  for (StringRef Prefix : {"omp", "cuda", "hip"}) {
-    auto Section = (Prefix + "_offloading_entries").str();
-    // Rename the offloading entires to make them private to this link unit.
-    ObjcopyArgs.emplace_back("--rename-section");
-    ObjcopyArgs.emplace_back(
-        Args.MakeArgString(Section + "=" + Section + Suffix));
-
-    // Rename the __start_ / __stop_ symbols appropriately to iterate over the
-    // newly renamed section containing the offloading entries.
-    ObjcopyArgs.emplace_back("--redefine-sym");
-    ObjcopyArgs.emplace_back(Args.MakeArgString("__start_" + Section + "=" +
-                                                "__start_" + Section + Suffix));
-    ObjcopyArgs.emplace_back("--redefine-sym");
-    ObjcopyArgs.emplace_back(Args.MakeArgString("__stop_" + Section + "=" +
-                                                "__stop_" + Section + Suffix));
-  }
+  StringRef Prefix = "llvm";
+  auto Section = (Prefix + "_offload_entries").str();
+  // Rename the offloading entires to make them private to this link unit.
+  ObjcopyArgs.emplace_back("--rename-section");
+  ObjcopyArgs.emplace_back(
+      Args.MakeArgString(Section + "=" + Section + Suffix));
+
+  // Rename the __start_ / __stop_ symbols appropriately to iterate over the
+  // newly renamed section containing the offloading entries.
+  ObjcopyArgs.emplace_back("--redefine-sym");
+  ObjcopyArgs.emplace_back(Args.MakeArgString("__start_" + Section + "=" +
+                                              "__start_" + Section + Suffix));
+  ObjcopyArgs.emplace_back("--redefine-sym");
+  ObjcopyArgs.emplace_back(Args.MakeArgString("__stop_" + Section + "=" +
+                                              "__stop_" + Section + Suffix));
 
   if (Error Err = executeCommands(*ObjcopyPath, ObjcopyArgs))
     return Err;
diff --git a/external/llvm-project/clang/tools/clang-offload-packager/ClangOffloadPackager.cpp b/external/llvm-project/clang/tools/clang-offload-packager/ClangOffloadPackager.cpp
index a20e680b1631..ff60e336116c 100644
--- a/external/llvm-project/clang/tools/clang-offload-packager/ClangOffloadPackager.cpp
+++ b/external/llvm-project/clang/tools/clang-offload-packager/ClangOffloadPackager.cpp
@@ -111,7 +111,6 @@ static Error bundleImages() {
     // Permit using multiple instances of `file` in a single string.
     for (auto &File : llvm::split(Args["file"], ",")) {
       OffloadBinary::OffloadingImage ImageBinary{};
-      std::unique_ptr<llvm::MemoryBuffer> DeviceImage;
 
       llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> ObjectOrErr =
           llvm::MemoryBuffer::getFileOrSTDIN(File);
diff --git a/external/llvm-project/clang/tools/driver/driver.cpp b/external/llvm-project/clang/tools/driver/driver.cpp
index d494628f57f4..ee4863243527 100644
--- a/external/llvm-project/clang/tools/driver/driver.cpp
+++ b/external/llvm-project/clang/tools/driver/driver.cpp
@@ -312,7 +312,7 @@ int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContext) {
   if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
     // FIXME: Driver shouldn't take extra initial argument.
     driver::applyOverrideOptions(Args, OverrideStr, SavedStrings,
-                                 &llvm::errs());
+                                 "CCC_OVERRIDE_OPTIONS", &llvm::errs());
   }
 
   std::string Path = GetExecutablePath(ToolContext.Path, CanonicalPrefixes);
diff --git a/external/llvm-project/clang/tools/libclang/CXCursor.cpp b/external/llvm-project/clang/tools/libclang/CXCursor.cpp
index 635d03a88d10..a6301daa672c 100644
--- a/external/llvm-project/clang/tools/libclang/CXCursor.cpp
+++ b/external/llvm-project/clang/tools/libclang/CXCursor.cpp
@@ -598,7 +598,6 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::SubstNonTypeTemplateParmPackExprClass:
   case Stmt::FunctionParmPackExprClass:
   case Stmt::UnresolvedLookupExprClass:
-  case Stmt::TypoExprClass: // A typo could actually be a DeclRef or a MemberRef
     K = CXCursor_DeclRefExpr;
     break;
 
diff --git a/external/llvm-project/clang/tools/libclang/CXType.cpp b/external/llvm-project/clang/tools/libclang/CXType.cpp
index 586d7edf9334..e7864e6d62e4 100644
--- a/external/llvm-project/clang/tools/libclang/CXType.cpp
+++ b/external/llvm-project/clang/tools/libclang/CXType.cpp
@@ -732,8 +732,8 @@ CXCallingConv clang_getFunctionTypeCallingConv(CXType X) {
       TCALLINGCONV(RISCVVLSCall_32768);
       TCALLINGCONV(RISCVVLSCall_65536);
     case CC_SpirFunction: return CXCallingConv_Unexposed;
-    case CC_AMDGPUKernelCall: return CXCallingConv_Unexposed;
-    case CC_OpenCLKernel: return CXCallingConv_Unexposed;
+    case CC_DeviceKernel:
+      return CXCallingConv_Unexposed;
       break;
     }
 #undef TCALLINGCONV
diff --git a/external/llvm-project/clang/tools/scan-build/bin/scan-build b/external/llvm-project/clang/tools/scan-build/bin/scan-build
index b90e635d3175..da7fe94cf4c0 100755
--- a/external/llvm-project/clang/tools/scan-build/bin/scan-build
+++ b/external/llvm-project/clang/tools/scan-build/bin/scan-build
@@ -1898,7 +1898,7 @@ if ($Clang !~ /\+\+(\.exe)?$/) {
   # Determine operating system under which this copy of Perl was built.
   my $IsWinBuild = ($^O =~/msys|cygwin|MSWin32/);
   if($IsWinBuild) {
-    $ClangCXX =~ s/.exe$/++.exe/;
+    $ClangCXX =~ s/\-\d+(\.\d+)?.exe$/++.exe/;
   }
   else {
     $ClangCXX =~ s/\-\d+(\.\d+)?$//;
diff --git a/external/llvm-project/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp b/external/llvm-project/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
index 4ad6f8025674..4cb260268b95 100644
--- a/external/llvm-project/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
+++ b/external/llvm-project/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
@@ -138,7 +138,7 @@ struct VerifyingConsumer {
 
   void consumeInitial(DirectoryWatcher::Event E) {
     std::unique_lock<std::mutex> L(Mtx);
-    auto It = std::find(ExpectedInitial.begin(), ExpectedInitial.end(), E);
+    auto It = llvm::find(ExpectedInitial, E);
     if (It == ExpectedInitial.end()) {
       UnexpectedInitial.push_back(E);
     } else {
@@ -152,11 +152,9 @@ struct VerifyingConsumer {
 
   void consumeNonInitial(DirectoryWatcher::Event E) {
     std::unique_lock<std::mutex> L(Mtx);
-    auto It =
-        std::find(ExpectedNonInitial.begin(), ExpectedNonInitial.end(), E);
+    auto It = llvm::find(ExpectedNonInitial, E);
     if (It == ExpectedNonInitial.end()) {
-      auto OptIt =
-          std::find(OptionalNonInitial.begin(), OptionalNonInitial.end(), E);
+      auto OptIt = llvm::find(OptionalNonInitial, E);
       if (OptIt != OptionalNonInitial.end()) {
         OptionalNonInitial.erase(OptIt);
       } else {
diff --git a/external/llvm-project/clang/unittests/Driver/MultilibBuilderTest.cpp b/external/llvm-project/clang/unittests/Driver/MultilibBuilderTest.cpp
index 0e5385c91353..38c9344d2966 100644
--- a/external/llvm-project/clang/unittests/Driver/MultilibBuilderTest.cpp
+++ b/external/llvm-project/clang/unittests/Driver/MultilibBuilderTest.cpp
@@ -12,9 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/MultilibBuilder.h"
-#include "../../lib/Driver/ToolChains/CommonArgs.h"
 #include "SimpleDiagnosticConsumer.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Driver/CommonArgs.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/external/llvm-project/clang/unittests/Driver/MultilibTest.cpp b/external/llvm-project/clang/unittests/Driver/MultilibTest.cpp
index c03e117d9930..4c11e6a32d7c 100644
--- a/external/llvm-project/clang/unittests/Driver/MultilibTest.cpp
+++ b/external/llvm-project/clang/unittests/Driver/MultilibTest.cpp
@@ -11,10 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/Multilib.h"
-#include "../../lib/Driver/ToolChains/CommonArgs.h"
 #include "SimpleDiagnosticConsumer.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Version.h"
+#include "clang/Driver/CommonArgs.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/external/llvm-project/clang/unittests/Format/FormatTestRawStrings.cpp b/external/llvm-project/clang/unittests/Format/FormatTestRawStrings.cpp
index 3f09c7b6086e..5e8737d65666 100644
--- a/external/llvm-project/clang/unittests/Format/FormatTestRawStrings.cpp
+++ b/external/llvm-project/clang/unittests/Format/FormatTestRawStrings.cpp
@@ -1002,9 +1002,11 @@ TEST_F(FormatTestRawStrings, Json) {
   };
 
   EXPECT_EQ("json = R\"json({\n"
+            "                \"foo\": \"bar\",\n"
             "                \"str\": \"test\"\n"
             "              })json\";",
             format("json = R\"json({\n"
+                   "  \"foo\": \"bar\",\n"
                    "  \"str\": \"test\"\n"
                    "})json\";",
                    Style));
diff --git a/external/llvm-project/clang/unittests/Format/TokenAnnotatorTest.cpp b/external/llvm-project/clang/unittests/Format/TokenAnnotatorTest.cpp
index ba6a9f813f05..a1285e4bc9bf 100644
--- a/external/llvm-project/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/external/llvm-project/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -1453,6 +1453,15 @@ TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
   ASSERT_EQ(Tokens.size(), 17u) << Tokens;
   EXPECT_TOKEN(Tokens[4], tok::ampamp, TT_PointerOrReference);
   EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
+
+  Tokens = annotate("auto foo() -> auto *\n"
+                    "  requires(not bar)\n"
+                    "{\n"
+                    "  return baz;\n"
+                    "}");
+  ASSERT_EQ(Tokens.size(), 18u) << Tokens;
+  EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
+  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
@@ -2257,12 +2266,17 @@ TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {
   EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
   EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
 
-  Tokens = annotate("#define FUNC(foo, bar) \\\n"
-                    "  auto foo##bar() -> Type {}");
-  ASSERT_EQ(Tokens.size(), 19u) << Tokens;
+  Tokens = annotate("#define FUNC(foo, bar, baz) \\\n"
+                    "  auto foo##bar##baz() -> Type {}");
+  ASSERT_EQ(Tokens.size(), 23u) << Tokens;
   EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
-  EXPECT_TOKEN(Tokens[12], tok::l_paren, TT_FunctionDeclarationLParen);
-  EXPECT_TOKEN(Tokens[14], tok::arrow, TT_TrailingReturnArrow);
+  EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_FunctionDeclarationLParen);
+  EXPECT_TOKEN(Tokens[18], tok::arrow, TT_TrailingReturnArrow);
+
+  Tokens = annotate("void __stdcall f();");
+  ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::identifier, TT_FunctionDeclarationName);
+  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
 
   Tokens = annotate("int iso_time(time_t);");
   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
@@ -3740,6 +3754,13 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
   EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
   EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
+
+  Tokens = annotate("auto f1{&T::operator()};");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_BRACE_KIND(Tokens[2], BK_BracedInit);
+  // Not TT_FunctionDeclarationName.
+  EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);
+  EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) {
@@ -4091,6 +4112,20 @@ TEST_F(TokenAnnotatorTest, BitFieldColon) {
   EXPECT_TOKEN(Tokens[5], tok::colon, TT_BitFieldColon);
 }
 
+TEST_F(TokenAnnotatorTest, JsonCodeInRawString) {
+  auto Tokens = annotate("{\n"
+                         "  \"foo\": \"bar\",\n"
+                         "  \"str\": \"test\"\n"
+                         "}",
+                         getLLVMStyle(FormatStyle::LK_Json));
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_DictLiteral);
+  EXPECT_TOKEN(Tokens[1], tok::string_literal, TT_SelectorName);
+  EXPECT_TOKEN(Tokens[2], tok::colon, TT_DictLiteral);
+  EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_SelectorName);
+  EXPECT_TOKEN(Tokens[6], tok::colon, TT_DictLiteral);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang
diff --git a/external/llvm-project/clang/unittests/Frontend/CompilerInstanceTest.cpp b/external/llvm-project/clang/unittests/Frontend/CompilerInstanceTest.cpp
index a7b258d5e537..459a3864887e 100644
--- a/external/llvm-project/clang/unittests/Frontend/CompilerInstanceTest.cpp
+++ b/external/llvm-project/clang/unittests/Frontend/CompilerInstanceTest.cpp
@@ -9,9 +9,12 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "gtest/gtest.h"
@@ -97,4 +100,52 @@ TEST(CompilerInstance, AllowDiagnosticLogWithUnownedDiagnosticConsumer) {
   ASSERT_EQ(DiagnosticOutput, "error: expected no crash\n");
 }
 
+TEST(CompilerInstance, MultipleInputsCleansFileIDs) {
+  auto VFS = makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  VFS->addFile("a.cc", /*ModificationTime=*/{},
+               MemoryBuffer::getMemBuffer(R"cpp(
+      #include "a.h"
+      )cpp"));
+  // Paddings of `void foo();` in the sources below are "important". We're
+  // testing against source locations from previous compilations colliding.
+  // Hence the `unused` variable in `b.h` needs to be within `#pragma clang
+  // diagnostic` block from `a.h`.
+  VFS->addFile("a.h", /*ModificationTime=*/{}, MemoryBuffer::getMemBuffer(R"cpp(
+      #include "b.h"
+      #pragma clang diagnostic push
+      #pragma clang diagnostic warning "-Wunused"
+      void foo();
+      #pragma clang diagnostic pop
+      )cpp"));
+  VFS->addFile("b.h", /*ModificationTime=*/{}, MemoryBuffer::getMemBuffer(R"cpp(
+      void foo(); void foo(); void foo(); void foo();
+      inline void foo() { int unused = 2; }
+      )cpp"));
+
+  DiagnosticOptions DiagOpts;
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
+      CompilerInstance::createDiagnostics(*VFS, DiagOpts);
+
+  CreateInvocationOptions CIOpts;
+  CIOpts.Diags = Diags;
+
+  const char *Args[] = {"clang", "-xc++", "a.cc"};
+  std::shared_ptr<CompilerInvocation> CInvok =
+      createInvocation(Args, std::move(CIOpts));
+  ASSERT_TRUE(CInvok) << "could not create compiler invocation";
+
+  CompilerInstance Instance(std::move(CInvok));
+  Instance.setDiagnostics(Diags.get());
+  Instance.createFileManager(VFS);
+
+  // Run once for `a.cc` and then for `a.h`. This makes sure we get the same
+  // file ID for `b.h` in the second run as `a.h` from first run.
+  const auto &OrigInputKind = Instance.getFrontendOpts().Inputs[0].getKind();
+  Instance.getFrontendOpts().Inputs.emplace_back("a.h", OrigInputKind);
+
+  SyntaxOnlyAction Act;
+  EXPECT_TRUE(Instance.ExecuteAction(Act)) << "Failed to execute action";
+  EXPECT_FALSE(Diags->hasErrorOccurred());
+  EXPECT_EQ(Diags->getNumWarnings(), 0u);
+}
 } // anonymous namespace
diff --git a/external/llvm-project/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp b/external/llvm-project/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
index bdb5e2351011..61f74929c1e9 100644
--- a/external/llvm-project/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
+++ b/external/llvm-project/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
@@ -677,13 +677,28 @@ TEST(MinimizeSourceToDependencyDirectivesTest, EmptyIncludesAndImports) {
                Out.data());
 }
 
-TEST(MinimizeSourceToDependencyDirectivesTest, AtImportFailures) {
+TEST(MinimizeSourceToDependencyDirectivesTest, ImportFailures) {
   SmallVector<char, 128> Out;
 
   ASSERT_TRUE(minimizeSourceToDependencyDirectives("@import A\n", Out));
   ASSERT_FALSE(
       minimizeSourceToDependencyDirectives("@import MACRO(A);\n", Out));
   ASSERT_FALSE(minimizeSourceToDependencyDirectives("@import \" \";\n", Out));
+
+  ASSERT_FALSE(minimizeSourceToDependencyDirectives("import <Foo.h>\n"
+                                                    "@import Foo;",
+                                                    Out));
+  EXPECT_STREQ("@import Foo;\n", Out.data());
+
+  ASSERT_FALSE(
+      minimizeSourceToDependencyDirectives("import <Foo.h>\n"
+                                           "#import <Foo.h>\n"
+                                           "@;\n"
+                                           "#pragma clang module import Foo",
+                                           Out));
+  EXPECT_STREQ("#import <Foo.h>\n"
+               "#pragma clang module import Foo\n",
+               Out.data());
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest, RawStringLiteral) {
@@ -774,6 +789,97 @@ TEST(MinimizeSourceToDependencyDirectivesTest,
                Out.data());
 }
 
+TEST(MinimizeSourceToDependencyDirectivesTest,
+     WhitespaceAfterLineContinuationSlashLineComment) {
+  SmallVector<char, 128> Out;
+
+  ASSERT_FALSE(minimizeSourceToDependencyDirectives("// some comment \\  \n"
+                                                    "module A;\n",
+                                                    Out));
+  EXPECT_STREQ("", Out.data());
+}
+
+TEST(MinimizeSourceToDependencyDirectivesTest,
+     WhitespaceAfterLineContinuationSlashAllDirectives) {
+  SmallVector<char, 512> Out;
+  SmallVector<dependency_directives_scan::Token, 16> Tokens;
+  SmallVector<Directive, 16> Directives;
+
+  StringRef Input = "#define \\   \n"
+                    "A\n"
+                    "#undef\t\\   \n"
+                    "A\n"
+                    "#endif \\\t\t\n"
+                    "\n"
+                    "#if \\     \t\n"
+                    "A\n"
+                    "#ifdef\t\\   \n"
+                    "A\n"
+                    "#ifndef \\ \t\n"
+                    "A\n"
+                    "#elifdef \\  \n"
+                    "A\n"
+                    "#elifndef \\ \n"
+                    "A\n"
+                    "#elif \\\t\t \n"
+                    "A\n"
+                    "#else \\\t \t\n"
+                    "\n"
+                    "#include \\  \n"
+                    "<A>\n"
+                    "#include_next \\    \n"
+                    "<A>\n"
+                    "#__include_macros\\ \n"
+                    "<A>\n"
+                    "#import \\ \t\n"
+                    "<A>\n"
+                    "@import \\\t \n"
+                    "A;\n"
+                    "#pragma clang \\   \n"
+                    "module \\    \n"
+                    "import A\n"
+                    "#pragma \\   \n"
+                    "push_macro(A)\n"
+                    "#pragma \\\t \n"
+                    "pop_macro(A)\n"
+                    "#pragma \\   \n"
+                    "include_alias(<A>,\\ \n"
+                    "<B>)\n"
+                    "export \\    \n"
+                    "module m;\n"
+                    "import\t\\\t \n"
+                    "m;\n"
+                    "#pragma\t\\  \n"
+                    "clang\t\\  \t\n"
+                    "system_header\n";
+  ASSERT_FALSE(
+      minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives));
+
+  EXPECT_EQ(pp_define, Directives[0].Kind);
+  EXPECT_EQ(pp_undef, Directives[1].Kind);
+  EXPECT_EQ(pp_endif, Directives[2].Kind);
+  EXPECT_EQ(pp_if, Directives[3].Kind);
+  EXPECT_EQ(pp_ifdef, Directives[4].Kind);
+  EXPECT_EQ(pp_ifndef, Directives[5].Kind);
+  EXPECT_EQ(pp_elifdef, Directives[6].Kind);
+  EXPECT_EQ(pp_elifndef, Directives[7].Kind);
+  EXPECT_EQ(pp_elif, Directives[8].Kind);
+  EXPECT_EQ(pp_else, Directives[9].Kind);
+  EXPECT_EQ(pp_include, Directives[10].Kind);
+  EXPECT_EQ(pp_include_next, Directives[11].Kind);
+  EXPECT_EQ(pp___include_macros, Directives[12].Kind);
+  EXPECT_EQ(pp_import, Directives[13].Kind);
+  EXPECT_EQ(decl_at_import, Directives[14].Kind);
+  EXPECT_EQ(pp_pragma_import, Directives[15].Kind);
+  EXPECT_EQ(pp_pragma_push_macro, Directives[16].Kind);
+  EXPECT_EQ(pp_pragma_pop_macro, Directives[17].Kind);
+  EXPECT_EQ(pp_pragma_include_alias, Directives[18].Kind);
+  EXPECT_EQ(cxx_export_module_decl, Directives[19].Kind);
+  EXPECT_EQ(cxx_import_decl, Directives[20].Kind);
+  EXPECT_EQ(pp_pragma_system_header, Directives[21].Kind);
+  EXPECT_EQ(pp_eof, Directives[22].Kind);
+}
+
 TEST(MinimizeSourceToDependencyDirectivesTest, PoundWarningAndError) {
   SmallVector<char, 128> Out;
 
diff --git a/external/llvm-project/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp b/external/llvm-project/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp
index 943a31a08604..b5bd233c5255 100644
--- a/external/llvm-project/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp
+++ b/external/llvm-project/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp
@@ -136,7 +136,7 @@ TEST_F(LexHLSLRootSignatureTest, ValidLexAllTokensTest) {
     space visibility flags
     numDescriptors offset
 
-    mipLODBias addressU addressV addressW
+    filter mipLODBias addressU addressV addressW
     maxAnisotropy comparisonFunc borderColor
     minLOD maxLOD
 
@@ -171,6 +171,43 @@ TEST_F(LexHLSLRootSignatureTest, ValidLexAllTokensTest) {
     shader_visibility_amplification
     shader_visibility_mesh
 
+    FILTER_MIN_MAG_MIP_POINT
+    FILTER_MIN_MAG_POINT_MIP_LINEAR
+    FILTER_MIN_POINT_MAG_LINEAR_MIP_POINT
+    FILTER_MIN_POINT_MAG_MIP_LINEAR
+    FILTER_MIN_LINEAR_MAG_MIP_POINT
+    FILTER_MIN_LINEAR_MAG_POINT_MIP_LINEAR
+    FILTER_MIN_MAG_LINEAR_MIP_POINT
+    FILTER_MIN_MAG_MIP_LINEAR
+    FILTER_ANISOTROPIC
+    FILTER_COMPARISON_MIN_MAG_MIP_POINT
+    FILTER_COMPARISON_MIN_MAG_POINT_MIP_LINEAR
+    FILTER_COMPARISON_MIN_POINT_MAG_LINEAR_MIP_POINT
+    FILTER_COMPARISON_MIN_POINT_MAG_MIP_LINEAR
+    FILTER_COMPARISON_MIN_LINEAR_MAG_MIP_POINT
+    FILTER_COMPARISON_MIN_LINEAR_MAG_POINT_MIP_LINEAR
+    FILTER_COMPARISON_MIN_MAG_LINEAR_MIP_POINT
+    FILTER_COMPARISON_MIN_MAG_MIP_LINEAR
+    FILTER_COMPARISON_ANISOTROPIC
+    FILTER_MINIMUM_MIN_MAG_MIP_POINT
+    FILTER_MINIMUM_MIN_MAG_POINT_MIP_LINEAR
+    FILTER_MINIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT
+    FILTER_MINIMUM_MIN_POINT_MAG_MIP_LINEAR
+    FILTER_MINIMUM_MIN_LINEAR_MAG_MIP_POINT
+    FILTER_MINIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR
+    FILTER_MINIMUM_MIN_MAG_LINEAR_MIP_POINT
+    FILTER_MINIMUM_MIN_MAG_MIP_LINEAR
+    FILTER_MINIMUM_ANISOTROPIC
+    FILTER_MAXIMUM_MIN_MAG_MIP_POINT
+    FILTER_MAXIMUM_MIN_MAG_POINT_MIP_LINEAR
+    FILTER_MAXIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT
+    FILTER_MAXIMUM_MIN_POINT_MAG_MIP_LINEAR
+    FILTER_MAXIMUM_MIN_LINEAR_MAG_MIP_POINT
+    FILTER_MAXIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR
+    FILTER_MAXIMUM_MIN_MAG_LINEAR_MIP_POINT
+    FILTER_MAXIMUM_MIN_MAG_MIP_LINEAR
+    FILTER_MAXIMUM_ANISOTROPIC
+
     TEXTURE_ADDRESS_WRAP
     TEXTURE_ADDRESS_MIRROR
     TEXTURE_ADDRESS_CLAMP
diff --git a/external/llvm-project/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/external/llvm-project/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index d45f99e06c63..1e46ee35d5d4 100644
--- a/external/llvm-project/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/external/llvm-project/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -231,6 +231,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
       minLOD = 4.2f, mipLODBias = 0.23e+3,
       addressW = TEXTURE_ADDRESS_CLAMP,
       addressV = TEXTURE_ADDRESS_BORDER,
+      filter = FILTER_MAXIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT,
       maxLOD = 9000, addressU = TEXTURE_ADDRESS_MIRROR,
       comparisonFunc = COMPARISON_NOT_EQUAL,
       borderColor = STATIC_BORDER_COLOR_OPAQUE_BLACK_UINT
@@ -257,6 +258,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
   ASSERT_TRUE(std::holds_alternative<StaticSampler>(Elem));
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.ViewType, RegisterType::SReg);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.Number, 0u);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Filter, SamplerFilter::Anisotropic);
   ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU, TextureAddressMode::Wrap);
   ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV, TextureAddressMode::Wrap);
   ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW, TextureAddressMode::Wrap);
@@ -275,6 +277,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
   ASSERT_TRUE(std::holds_alternative<StaticSampler>(Elem));
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.ViewType, RegisterType::SReg);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.Number, 0u);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Filter,
+            SamplerFilter::MaximumMinPointMagLinearMipPoint);
   ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU, TextureAddressMode::Mirror);
   ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV, TextureAddressMode::Border);
   ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW, TextureAddressMode::Clamp);
diff --git a/external/llvm-project/clang/unittests/Sema/ExternalSemaSourceTest.cpp b/external/llvm-project/clang/unittests/Sema/ExternalSemaSourceTest.cpp
index 2b271d4bf782..cc9dd4175af5 100644
--- a/external/llvm-project/clang/unittests/Sema/ExternalSemaSourceTest.cpp
+++ b/external/llvm-project/clang/unittests/Sema/ExternalSemaSourceTest.cpp
@@ -268,20 +268,6 @@ TEST(ExternalSemaSource, ExternalTypoCorrectionOrdering) {
   ASSERT_EQ(1, Watcher.SeenCount);
 }
 
-TEST(ExternalSemaSource, ExternalDelayedTypoCorrection) {
-  auto Installer = std::make_unique<ExternalSemaSourceInstaller>();
-  auto Provider = makeIntrusiveRefCnt<FunctionTypoProvider>("aaa", "bbb");
-  DiagnosticWatcher Watcher("aaa", "bbb");
-  Installer->PushSource(Provider.get());
-  Installer->PushWatcher(&Watcher);
-  std::vector<std::string> Args(1, "-std=c++11");
-  ASSERT_TRUE(clang::tooling::runToolOnCodeWithArgs(
-      std::move(Installer), "namespace AAA { } void foo() { AAA::aaa(); }",
-      Args));
-  ASSERT_LE(0, Provider->CallCount);
-  ASSERT_EQ(1, Watcher.SeenCount);
-}
-
 // We should only try MaybeDiagnoseMissingCompleteType if we can't otherwise
 // solve the problem.
 TEST(ExternalSemaSource, TryOtherTacticsBeforeDiagnosing) {
diff --git a/external/llvm-project/clang/unittests/StaticAnalyzer/BlockEntranceCallbackTest.cpp b/external/llvm-project/clang/unittests/StaticAnalyzer/BlockEntranceCallbackTest.cpp
index 9f62d8b87d96..0f05c39df93e 100644
--- a/external/llvm-project/clang/unittests/StaticAnalyzer/BlockEntranceCallbackTest.cpp
+++ b/external/llvm-project/clang/unittests/StaticAnalyzer/BlockEntranceCallbackTest.cpp
@@ -111,10 +111,10 @@ llvm::SmallVector<StringRef> parseEachDiag(StringRef Diags) {
   llvm::SmallVector<StringRef> Fragments;
   llvm::SplitString(Diags, Fragments, "\n");
   // Drop the prefix like "test.BlockEntranceTester: " from each fragment.
-  llvm::for_each(Fragments, [](StringRef &Fragment) {
+  for (StringRef &Fragment : Fragments) {
     Fragment = Fragment.drop_until([](char Ch) { return Ch == ' '; });
     Fragment.consume_front(" ");
-  });
+  }
   llvm::sort(Fragments);
   return Fragments;
 }
diff --git a/external/llvm-project/clang/unittests/Tooling/ToolingTest.cpp b/external/llvm-project/clang/unittests/Tooling/ToolingTest.cpp
index 562f87d430e6..32af4b6b3b35 100644
--- a/external/llvm-project/clang/unittests/Tooling/ToolingTest.cpp
+++ b/external/llvm-project/clang/unittests/Tooling/ToolingTest.cpp
@@ -771,25 +771,6 @@ TEST(ClangToolTest, BaseVirtualFileSystemUsage) {
   EXPECT_EQ(0, Tool.run(Action.get()));
 }
 
-// Check -cc1 command doesn't fail.
-TEST(ClangToolTest, CC1Arg) {
-  FixedCompilationDatabase Compilations("/", {"-cc1"});
-  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFileSystem(
-      new llvm::vfs::OverlayFileSystem(llvm::vfs::getRealFileSystem()));
-  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem(
-      new llvm::vfs::InMemoryFileSystem);
-  OverlayFileSystem->pushOverlay(InMemoryFileSystem);
-
-  InMemoryFileSystem->addFile(
-      "a.cpp", 0, llvm::MemoryBuffer::getMemBuffer("int main() {}"));
-
-  ClangTool Tool(Compilations, std::vector<std::string>(1, "a.cpp"),
-                 std::make_shared<PCHContainerOperations>(), OverlayFileSystem);
-  std::unique_ptr<FrontendActionFactory> Action(
-      newFrontendActionFactory<SyntaxOnlyAction>());
-  EXPECT_EQ(0, Tool.run(Action.get()));
-}
-
 // Check getClangStripDependencyFileAdjuster doesn't strip args after -MD/-MMD.
 TEST(ClangToolTest, StripDependencyFileAdjuster) {
   FixedCompilationDatabase Compilations("/", {"-MD", "-c", "-MMD", "-w"});
diff --git a/external/llvm-project/clang/utils/TableGen/ClangAttrEmitter.cpp b/external/llvm-project/clang/utils/TableGen/ClangAttrEmitter.cpp
index 21d76c12a3cc..f892626a447e 100644
--- a/external/llvm-project/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/external/llvm-project/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -3739,7 +3739,8 @@ static void GenerateHasAttrSpellingStringSwitch(
                       : '(' + itostr(Version) + ')';
 
     if (Scope.empty() || Scope == Spelling.nameSpace()) {
-      if (TestStringMap.contains(Spelling.name()))
+      if (TestStringMap.contains(Spelling.name()) &&
+          TestStringMap[Spelling.name()] != TestStr)
         TestStringMap[Spelling.name()] += " || " + TestStr;
       else
         TestStringMap[Spelling.name()] = TestStr;
@@ -5405,7 +5406,7 @@ void EmitClangAttrDocs(const RecordKeeper &Records, raw_ostream &OS) {
       // Handle Undocumented category separately - no content merging
       if (Cat == "Undocumented" && UndocumentedCategory) {
         UndocumentedDocs.push_back(
-            DocumentationData(Doc, Attr, HeadingAndSpellings));
+            DocumentationData(Doc, Attr, std::move(HeadingAndSpellings)));
         continue;
       }
 
diff --git a/external/llvm-project/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp b/external/llvm-project/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
index f15e30cd3f8f..161dd425fbc7 100644
--- a/external/llvm-project/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
+++ b/external/llvm-project/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
@@ -78,10 +78,10 @@ void clang::EmitClangCommentCommandInfo(const RecordKeeper &Records,
 
 static std::string MangleName(StringRef Str) {
   std::string Mangled;
-  for (unsigned i = 0, e = Str.size(); i != e; ++i) {
-    switch (Str[i]) {
+  for (char C : Str) {
+    switch (C) {
     default:
-      Mangled += Str[i];
+      Mangled += C;
       break;
     case '(':
       Mangled += "lparen";
@@ -122,9 +122,8 @@ void clang::EmitClangCommentCommandList(const RecordKeeper &Records,
      << "#endif\n";
 
   ArrayRef<const Record *> Tags = Records.getAllDerivedDefinitions("Command");
-  for (size_t i = 0, e = Tags.size(); i != e; ++i) {
-    const Record &Tag = *Tags[i];
-    std::string MangledName = MangleName(Tag.getValueAsString("Name"));
+  for (const Record *Tag : Tags) {
+    std::string MangledName = MangleName(Tag->getValueAsString("Name"));
 
     OS << "COMMENT_COMMAND(" << MangledName << ")\n";
   }
diff --git a/external/llvm-project/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp b/external/llvm-project/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
index b8d8ac853a5c..e5eec5e7ca8d 100644
--- a/external/llvm-project/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
+++ b/external/llvm-project/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
@@ -37,9 +37,9 @@ static bool translateCodePointToUTF8(unsigned CodePoint,
 
   raw_svector_ostream OS(CLiteral);
   OS << "\"";
-  for (size_t i = 0, e = UTF8.size(); i != e; ++i) {
+  for (char C : UTF8) {
     OS << "\\x";
-    OS.write_hex(static_cast<unsigned char>(UTF8[i]));
+    OS.write_hex(static_cast<unsigned char>(C));
   }
   OS << "\"";
 
diff --git a/external/llvm-project/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp b/external/llvm-project/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index e347b89a85d4..bfc60f485cd3 100644
--- a/external/llvm-project/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/external/llvm-project/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -1794,8 +1794,8 @@ static std::string getDiagCategoryEnum(StringRef name) {
   if (name.empty())
     return "DiagCat_None";
   SmallString<256> enumName = StringRef("DiagCat_");
-  for (StringRef::iterator I = name.begin(), E = name.end(); I != E; ++I)
-    enumName += isalnum(*I) ? *I : '_';
+  for (char C : name)
+    enumName += isalnum(C) ? C : '_';
   return std::string(enumName);
 }
 
diff --git a/external/llvm-project/clang/utils/TableGen/ClangOpcodesEmitter.cpp b/external/llvm-project/clang/utils/TableGen/ClangOpcodesEmitter.cpp
index 5d6d90994cf3..9d0773e1aff8 100644
--- a/external/llvm-project/clang/utils/TableGen/ClangOpcodesEmitter.cpp
+++ b/external/llvm-project/clang/utils/TableGen/ClangOpcodesEmitter.cpp
@@ -224,8 +224,7 @@ void ClangOpcodesEmitter::EmitProto(raw_ostream &OS, StringRef N,
   auto Args = R->getValueAsListOfDefs("Args");
   Enumerate(R, N, [&OS, &Args](ArrayRef<const Record *> TS, const Twine &ID) {
     OS << "bool emit" << ID << "(";
-    for (size_t I = 0, N = Args.size(); I < N; ++I) {
-      const auto *Arg = Args[I];
+    for (const Record *Arg : Args) {
       bool AsRef = Arg->getValueAsBit("AsRef");
       auto Name = Arg->getValueAsString("Name");
 
diff --git a/external/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp b/external/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp
index c58f66568a31..17f0c8280bde 100644
--- a/external/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp
+++ b/external/llvm-project/clang/utils/TableGen/RISCVVEmitter.cpp
@@ -45,7 +45,7 @@ struct SemaRecord {
   unsigned Log2LMULMask;
 
   // Required extensions for this intrinsic.
-  RequiredExtensionBits RequiredExtensions;
+  std::string RequiredExtensions;
 
   // Prototype for this intrinsic.
   SmallVector<PrototypeDescriptor> Prototype;
@@ -768,35 +768,9 @@ void RVVEmitter::createRVVIntrinsics(
       Log2LMULMask |= 1 << (Log2LMUL + 3);
 
     SR.Log2LMULMask = Log2LMULMask;
-
-    for (auto RequiredFeature : RequiredFeatures) {
-      unsigned RequireExt =
-          StringSwitch<RVVRequire>(RequiredFeature)
-              .Case("RV64", RVV_REQ_RV64)
-              .Case("Zvfhmin", RVV_REQ_Zvfhmin)
-              .Case("Xandesvpackfph", RVV_REQ_Xandesvpackfph)
-              .Case("Xandesvdot", RVV_REQ_Xandesvdot)
-              .Case("Xsfvcp", RVV_REQ_Xsfvcp)
-              .Case("Xsfvfnrclipxfqf", RVV_REQ_Xsfvfnrclipxfqf)
-              .Case("Xsfvfwmaccqqq", RVV_REQ_Xsfvfwmaccqqq)
-              .Case("Xsfvqmaccdod", RVV_REQ_Xsfvqmaccdod)
-              .Case("Xsfvqmaccqoq", RVV_REQ_Xsfvqmaccqoq)
-              .Case("Zvbb", RVV_REQ_Zvbb)
-              .Case("Zvbc", RVV_REQ_Zvbc)
-              .Case("Zvkb", RVV_REQ_Zvkb)
-              .Case("Zvkg", RVV_REQ_Zvkg)
-              .Case("Zvkned", RVV_REQ_Zvkned)
-              .Case("Zvknha", RVV_REQ_Zvknha)
-              .Case("Zvknhb", RVV_REQ_Zvknhb)
-              .Case("Zvksed", RVV_REQ_Zvksed)
-              .Case("Zvksh", RVV_REQ_Zvksh)
-              .Case("Zvfbfwma", RVV_REQ_Zvfbfwma)
-              .Case("Zvfbfmin", RVV_REQ_Zvfbfmin)
-              .Case("Zvfh", RVV_REQ_Zvfh)
-              .Case("Experimental", RVV_REQ_Experimental);
-      SR.RequiredExtensions.set(RequireExt);
-    }
-
+    std::string RFs =
+        join(RequiredFeatures.begin(), RequiredFeatures.end(), ",");
+    SR.RequiredExtensions = RFs;
     SR.NF = NF;
     SR.HasMasked = HasMasked;
     SR.HasVL = HasVL;
@@ -838,7 +812,7 @@ void RVVEmitter::createRVVIntrinsicRecords(std::vector<RVVIntrinsicRecord> &Out,
     R.PrototypeLength = SR.Prototype.size();
     R.SuffixLength = SR.Suffix.size();
     R.OverloadedSuffixSize = SR.OverloadedSuffix.size();
-    R.RequiredExtensions = SR.RequiredExtensions;
+    R.RequiredExtensions = SR.RequiredExtensions.c_str();
     R.TypeRangeMask = SR.TypeRangeMask;
     R.Log2LMULMask = SR.Log2LMULMask;
     R.NF = SR.NF;
diff --git a/external/llvm-project/clang/utils/analyzer/requirements.txt b/external/llvm-project/clang/utils/analyzer/requirements.txt
index 8ae8bc88ac19..ed09161e5902 100644
--- a/external/llvm-project/clang/utils/analyzer/requirements.txt
+++ b/external/llvm-project/clang/utils/analyzer/requirements.txt
@@ -1,6 +1,6 @@
 graphviz
 humanize
 matplotlib
-pandas
-psutil
+pandas>=1.0.4
+psutil>=5.6.6
 seaborn
diff --git a/external/llvm-project/clang/utils/perf-training/perf-helper.py b/external/llvm-project/clang/utils/perf-training/perf-helper.py
index 80c6356d0497..29904aded5ab 100644
--- a/external/llvm-project/clang/utils/perf-training/perf-helper.py
+++ b/external/llvm-project/clang/utils/perf-training/perf-helper.py
@@ -237,6 +237,7 @@ def get_cc1_command_for_args(cmd, env):
             or ln.startswith("InstalledDir:")
             or ln.startswith("LLVM Profile Note")
             or ln.startswith(" (in-process)")
+            or ln.startswith("Configuration file:")
             or " version " in ln
         ):
             continue
diff --git a/external/llvm-project/clang/www/c_status.html b/external/llvm-project/clang/www/c_status.html
index e47466e3273f..dcff2fc2b1a3 100644
--- a/external/llvm-project/clang/www/c_status.html
+++ b/external/llvm-project/clang/www/c_status.html
@@ -864,7 +864,7 @@ <h2 id="c2x">C23 implementation status</h2>
     <tr>
       <td>Underspecified object definitions</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3006.htm">N3006</a></td>
-      <td class="none" align="center">No</td>
+      <td class="unreleased" align="center">Yes</td>
     </tr>
     <tr>
       <td>Type inference for object declarations</td>
diff --git a/external/llvm-project/clang/www/cxx_dr_status.html b/external/llvm-project/clang/www/cxx_dr_status.html
index 52c9cb7d37c2..ad82d7ab0cec 100755
--- a/external/llvm-project/clang/www/cxx_dr_status.html
+++ b/external/llvm-project/clang/www/cxx_dr_status.html
@@ -14049,7 +14049,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2369.html">2369</a></td>
     <td>CD6</td>
     <td>Ordering between constraints and substitution</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="partial" align="center">Partial</td>
   </tr>
   <tr id="2370">
     <td><a href="https://cplusplus.github.io/CWG/issues/2370.html">2370</a></td>
@@ -14811,7 +14811,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2496.html">2496</a></td>
     <td>CD6</td>
     <td><I>ref-qualifier</I>s and virtual overriding</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 21</td>
   </tr>
   <tr class="open" id="2497">
     <td><a href="https://cplusplus.github.io/CWG/issues/2497.html">2497</a></td>
@@ -14937,7 +14937,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2517.html">2517</a></td>
     <td>C++23</td>
     <td>Useless restriction on use of parameter in <I>constraint-expression</I></td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 21</td>
   </tr>
   <tr id="2518">
     <td><a href="https://cplusplus.github.io/CWG/issues/2518.html">2518</a></td>
@@ -16468,7 +16468,11 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2770.html">2770</a></td>
     <td>open</td>
     <td>Trailing <I>requires-clause</I> can refer to function parameters before they are substituted into</td>
-    <td align="center">Not resolved</td>
+    <td align="center">
+      <details>
+        <summary>Not resolved</summary>
+        Clang 20 implements 2023-07-14 resolution
+      </details></td>
   </tr>
   <tr id="2771">
     <td><a href="https://cplusplus.github.io/CWG/issues/2771.html">2771</a></td>
diff --git a/external/llvm-project/compiler-rt/cmake/builtin-config-ix.cmake b/external/llvm-project/compiler-rt/cmake/builtin-config-ix.cmake
index 8c9c84ad64bc..c62855835512 100644
--- a/external/llvm-project/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/external/llvm-project/compiler-rt/cmake/builtin-config-ix.cmake
@@ -50,6 +50,24 @@ void foo(void)  __arm_streaming_compatible {
 }
 ")
 
+builtin_check_c_compiler_source(COMPILER_RT_HAS_ARM_UNALIGNED
+"
+void foo() {
+#ifndef __ARM_FEATURE_UNALIGNED
+#error \"Unaligned accesses unsupported\"
+#endif
+}
+")
+
+builtin_check_c_compiler_source(COMPILER_RT_HAS_ARM_FP
+"
+void foo() {
+#ifndef __ARM_FP
+#error \"No floating-point support\"
+#endif
+}
+")
+
 check_include_files("sys/auxv.h"    COMPILER_RT_HAS_AUXV)
 
 if(ANDROID)
diff --git a/external/llvm-project/compiler-rt/include/profile/InstrProfData.inc b/external/llvm-project/compiler-rt/include/profile/InstrProfData.inc
index 6d18680f8916..0496f240dc82 100644
--- a/external/llvm-project/compiler-rt/include/profile/InstrProfData.inc
+++ b/external/llvm-project/compiler-rt/include/profile/InstrProfData.inc
@@ -405,14 +405,14 @@ typedef struct ValueProfRecord {
   /*!
    * Read data from this record and save it to Record.
    */
-  void deserializeTo(InstrProfRecord &Record,
+  LLVM_ABI void deserializeTo(InstrProfRecord &Record,
                      InstrProfSymtab *SymTab);
   /*
    * In-place byte swap:
    * Do byte swap for this instance. \c Old is the original order before
    * the swap, and \c New is the New byte order.
    */
-  void swapBytes(llvm::endianness Old, llvm::endianness New);
+  LLVM_ABI void swapBytes(llvm::endianness Old, llvm::endianness New);
 #endif
 } ValueProfRecord;
 
@@ -449,41 +449,41 @@ typedef struct ValueProfData {
    * Return the total size in bytes of the on-disk value profile data
    * given the data stored in Record.
    */
-  static uint32_t getSize(const InstrProfRecord &Record);
+  LLVM_ABI static uint32_t getSize(const InstrProfRecord &Record);
   /*!
    * Return a pointer to \c ValueProfData instance ready to be streamed.
    */
-  static std::unique_ptr<ValueProfData>
+  LLVM_ABI static std::unique_ptr<ValueProfData>
   serializeFrom(const InstrProfRecord &Record);
   /*!
    * Check the integrity of the record.
    */
-  Error checkIntegrity();
+  LLVM_ABI Error checkIntegrity();
   /*!
    * Return a pointer to \c ValueProfileData instance ready to be read.
    * All data in the instance are properly byte swapped. The input
    * data is assumed to be in little endian order.
    */
-  static Expected<std::unique_ptr<ValueProfData>>
+  LLVM_ABI static Expected<std::unique_ptr<ValueProfData>>
   getValueProfData(const unsigned char *SrcBuffer,
                    const unsigned char *const SrcBufferEnd,
                    llvm::endianness SrcDataEndianness);
   /*!
    * Swap byte order from \c Endianness order to host byte order.
    */
-  void swapBytesToHost(llvm::endianness Endianness);
+  LLVM_ABI void swapBytesToHost(llvm::endianness Endianness);
   /*!
    * Swap byte order from host byte order to \c Endianness order.
    */
-  void swapBytesFromHost(llvm::endianness Endianness);
+  LLVM_ABI void swapBytesFromHost(llvm::endianness Endianness);
   /*!
    * Return the total size of \c ValueProfileData.
    */
-  uint32_t getSize() const { return TotalSize; }
+  LLVM_ABI uint32_t getSize() const { return TotalSize; }
   /*!
    * Read data from this data and save it to \c Record.
    */
-  void deserializeTo(InstrProfRecord &Record,
+  LLVM_ABI void deserializeTo(InstrProfRecord &Record,
                      InstrProfSymtab *SymTab);
   void operator delete(void *ptr) { ::operator delete(ptr); }
 #endif
diff --git a/external/llvm-project/compiler-rt/lib/builtins/CMakeLists.txt b/external/llvm-project/compiler-rt/lib/builtins/CMakeLists.txt
index 072e01146e13..680b00097ca9 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/external/llvm-project/compiler-rt/lib/builtins/CMakeLists.txt
@@ -600,9 +600,18 @@ if (COMPILER_RT_HAS_AARCH64_SME)
     set_source_files_properties(aarch64/arm_apple_sme_abi.s PROPERTIES COMPILE_FLAGS -march=armv8a+sme)
     message(STATUS "AArch64 Apple SME ABI routines enabled")
   elseif (NOT COMPILER_RT_DISABLE_AARCH64_FMV AND COMPILER_RT_HAS_FNO_BUILTIN_FLAG AND COMPILER_RT_AARCH64_FMV_USES_GLOBAL_CONSTRUCTOR)
-    list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-libc-mem-routines.S aarch64/sme-abi-assert.c aarch64/sme-libc-routines.c)
+    list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-abi-assert.c)
+    if(COMPILER_RT_HAS_ARM_UNALIGNED AND COMPILER_RT_HAS_ARM_FP)
+      list(APPEND aarch64_SOURCES aarch64/sme-libc-opt-memset-memchr.S aarch64/sme-libc-opt-memcpy-memmove.S)
+    elseif(COMPILER_RT_HAS_ARM_UNALIGNED)
+      list(APPEND aarch64_SOURCES aarch64/sme-libc-memset-memchr.c aarch64/sme-libc-opt-memcpy-memmove.S)
+      message(WARNING "AArch64 SME ABI assembly-optimized memset/memchr disabled: target does not have hardware floating-point support.")
+    else()
+      list(APPEND aarch64_SOURCES aarch64/sme-libc-memset-memchr.c aarch64/sme-libc-memcpy-memmove.c)
+      message(WARNING "AArch64 SME ABI assembly-optimized routines disabled: target does not support unaligned accesses.")
+    endif()
     message(STATUS "AArch64 SME ABI routines enabled")
-    set_source_files_properties(aarch64/sme-libc-routines.c PROPERTIES COMPILE_FLAGS "-fno-builtin")
+    set_source_files_properties(aarch64/sme-libc-memset-memchr.c aarch64/sme-libc-memcpy-memmove.c PROPERTIES COMPILE_FLAGS "-fno-builtin")
   else()
     if(COMPILER_RT_DISABLE_AARCH64_FMV)
       message(WARNING "AArch64 SME ABI routines require function multiversioning support.")
diff --git a/external/llvm-project/compiler-rt/lib/builtins/README.txt b/external/llvm-project/compiler-rt/lib/builtins/README.txt
index 19f26c92a0f9..2d213d95f333 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/README.txt
+++ b/external/llvm-project/compiler-rt/lib/builtins/README.txt
@@ -272,11 +272,6 @@ switch32
 switch8
 switchu8
 
-// This function generates a custom trampoline function with the specific
-// realFunc and localsPtr values.
-void __trampoline_setup(uint32_t* trampOnStack, int trampSizeAllocated,
-                        const void* realFunc, void* localsPtr);
-
 // There is no C interface to the *_vfp_d8_d15_regs functions.  There are
 // called in the prolog and epilog of Thumb1 functions.  When the C++ ABI use
 // SJLJ for exceptions, each function with a catch clause or destructors needs
diff --git a/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-memcpy-memmove.c b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-memcpy-memmove.c
new file mode 100644
index 000000000000..9792079964d5
--- /dev/null
+++ b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-memcpy-memmove.c
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains basic implementations of Scalable Matrix Extension (SME)
+/// compatible memcpy and memmove functions to be used when their assembly-
+/// optimized counterparts can't.
+///
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+static void *__arm_sc_memcpy_fwd(void *dest, const void *src,
+                                 size_t n) __arm_streaming_compatible {
+  unsigned char *destp = (unsigned char *)dest;
+  const unsigned char *srcp = (const unsigned char *)src;
+
+  for (size_t i = 0; i < n; ++i)
+    destp[i] = srcp[i];
+  return dest;
+}
+
+static void *__arm_sc_memcpy_rev(void *dest, const void *src,
+                                 size_t n) __arm_streaming_compatible {
+  unsigned char *destp = (unsigned char *)dest;
+  const unsigned char *srcp = (const unsigned char *)src;
+
+  while (n > 0) {
+    --n;
+    destp[n] = srcp[n];
+  }
+  return dest;
+}
+
+extern void *__arm_sc_memcpy(void *__restrict dest, const void *__restrict src,
+                             size_t n) __arm_streaming_compatible {
+  return __arm_sc_memcpy_fwd(dest, src, n);
+}
+
+extern void *__arm_sc_memmove(void *dest, const void *src,
+                              size_t n) __arm_streaming_compatible {
+  unsigned char *destp = (unsigned char *)dest;
+  const unsigned char *srcp = (const unsigned char *)src;
+
+  if ((srcp > (destp + n)) || (destp > (srcp + n)))
+    return __arm_sc_memcpy(dest, src, n);
+  if (srcp > destp)
+    return __arm_sc_memcpy_fwd(dest, src, n);
+  return __arm_sc_memcpy_rev(dest, src, n);
+}
diff --git a/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-memset-memchr.c b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-memset-memchr.c
new file mode 100644
index 000000000000..85873c3554fb
--- /dev/null
+++ b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-memset-memchr.c
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains basic implementations of Scalable Matrix Extension (SME)
+/// compatible memset and memchr functions to be used when their assembly-
+/// optimized counterparts can't.
+///
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+extern void *__arm_sc_memset(void *dest, int c,
+                             size_t n) __arm_streaming_compatible {
+  unsigned char *destp = (unsigned char *)dest;
+  unsigned char c8 = (unsigned char)c;
+  for (size_t i = 0; i < n; ++i)
+    destp[i] = c8;
+
+  return dest;
+}
+
+extern const void *__arm_sc_memchr(const void *src, int c,
+                                   size_t n) __arm_streaming_compatible {
+  const unsigned char *srcp = (const unsigned char *)src;
+  unsigned char c8 = (unsigned char)c;
+  for (size_t i = 0; i < n; ++i)
+    if (srcp[i] == c8)
+      return &srcp[i];
+
+  return NULL;
+}
\ No newline at end of file
diff --git a/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memcpy-memmove.S
similarity index 70%
rename from external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
rename to external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memcpy-memmove.S
index 73b1ab2c76aa..8bc759a29a31 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memcpy-memmove.S
@@ -1,8 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-// Routines taken from libc/AOR_v20.02/string/aarch64
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains assembly-optimized implementations of Scalable Matrix
+/// Extension (SME) compatible memcpy and memmove functions.
+///
+/// These implementations depend on unaligned access support.
+///
+/// Routines taken from libc/AOR_v20.02/string/aarch64.
+///
+//===----------------------------------------------------------------------===//
 
 #include "../assembly.h"
 
@@ -234,116 +246,3 @@ END_COMPILERRT_FUNCTION(__arm_sc_memcpy)
 
 DEFINE_COMPILERRT_FUNCTION_ALIAS(__arm_sc_memmove, __arm_sc_memcpy)
 
-// This version uses FP registers. Use this only on targets with them
-#if (defined(__aarch64__) && __ARM_FP != 0) || defined(__arm64ec__)
-//
-//  __arm_sc_memset
-//
-
-#define dstin    x0
-#define val      x1
-#define valw     w1
-#define count    x2
-#define dst      x3
-#define dstend2  x4
-#define zva_val  x5
-
-DEFINE_COMPILERRT_FUNCTION(__arm_sc_memset)
-#ifdef __ARM_FEATURE_SVE
-        mov     z0.b, valw
-#else
-        bfi valw, valw, #8, #8
-        bfi valw, valw, #16, #16
-        bfi val, val, #32, #32
-        fmov d0, val
-        fmov v0.d[1], val
-#endif
-        add     dstend2, dstin, count
-
-        cmp     count, 96
-        b.hi    7f  // set_long
-        cmp     count, 16
-        b.hs    4f  // set_medium
-        mov     val, v0.D[0]
-
-        /* Set 0..15 bytes.  */
-        tbz     count, 3, 1f
-        str     val, [dstin]
-        str     val, [dstend2, -8]
-        ret
-        nop
-1:      tbz     count, 2, 2f
-        str     valw, [dstin]
-        str     valw, [dstend2, -4]
-        ret
-2:      cbz     count, 3f
-        strb    valw, [dstin]
-        tbz     count, 1, 3f
-        strh    valw, [dstend2, -2]
-3:      ret
-
-        /* Set 17..96 bytes.  */
-4:  // set_medium
-        str     q0, [dstin]
-        tbnz    count, 6, 6f  // set96
-        str     q0, [dstend2, -16]
-        tbz     count, 5, 5f
-        str     q0, [dstin, 16]
-        str     q0, [dstend2, -32]
-5:      ret
-
-        .p2align 4
-        /* Set 64..96 bytes.  Write 64 bytes from the start and
-           32 bytes from the end.  */
-6:  // set96
-        str     q0, [dstin, 16]
-        stp     q0, q0, [dstin, 32]
-        stp     q0, q0, [dstend2, -32]
-        ret
-
-        .p2align 4
-7:  // set_long
-        and     valw, valw, 255
-        bic     dst, dstin, 15
-        str     q0, [dstin]
-        cmp     count, 160
-        ccmp    valw, 0, 0, hs
-        b.ne    9f  // no_zva
-
-#ifndef SKIP_ZVA_CHECK
-        mrs     zva_val, dczid_el0
-        and     zva_val, zva_val, 31
-        cmp     zva_val, 4              /* ZVA size is 64 bytes.  */
-        b.ne    9f  // no_zva
-#endif
-        str     q0, [dst, 16]
-        stp     q0, q0, [dst, 32]
-        bic     dst, dst, 63
-        sub     count, dstend2, dst      /* Count is now 64 too large.  */
-        sub     count, count, 128       /* Adjust count and bias for loop.  */
-
-        .p2align 4
-8:  // zva_loop
-        add     dst, dst, 64
-        dc      zva, dst
-        subs    count, count, 64
-        b.hi    8b  // zva_loop
-        stp     q0, q0, [dstend2, -64]
-        stp     q0, q0, [dstend2, -32]
-        ret
-
-9:  // no_zva
-        sub     count, dstend2, dst      /* Count is 16 too large.  */
-        sub     dst, dst, 16            /* Dst is biased by -32.  */
-        sub     count, count, 64 + 16   /* Adjust count and bias for loop.  */
-10: // no_zva_loop
-        stp     q0, q0, [dst, 32]
-        stp     q0, q0, [dst, 64]!
-        subs    count, count, 64
-        b.hi    10b  // no_zva_loop
-        stp     q0, q0, [dstend2, -64]
-        stp     q0, q0, [dstend2, -32]
-        ret
-END_COMPILERRT_FUNCTION(__arm_sc_memset)
-
-#endif // __aarch64__
diff --git a/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memset-memchr.S b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memset-memchr.S
new file mode 100644
index 000000000000..b6939f114cba
--- /dev/null
+++ b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memset-memchr.S
@@ -0,0 +1,261 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains assembly-optimized implementations of Scalable Matrix
+/// Extension (SME) compatible memset and memchr functions.
+///
+/// These implementations depend on unaligned access and floating-point support.
+///
+/// Routines taken from libc/AOR_v20.02/string/aarch64.
+///
+//===----------------------------------------------------------------------===//
+
+#include "../assembly.h"
+
+//
+//  __arm_sc_memset
+//
+
+#define dstin    x0
+#define val      x1
+#define valw     w1
+#define count    x2
+#define dst      x3
+#define dstend2  x4
+#define zva_val  x5
+
+DEFINE_COMPILERRT_FUNCTION(__arm_sc_memset)
+#ifdef __ARM_FEATURE_SVE
+        mov     z0.b, valw
+#else
+        bfi valw, valw, #8, #8
+        bfi valw, valw, #16, #16
+        bfi val, val, #32, #32
+        fmov d0, val
+        fmov v0.d[1], val
+#endif
+        add     dstend2, dstin, count
+
+        cmp     count, 96
+        b.hi    7f  // set_long
+        cmp     count, 16
+        b.hs    4f  // set_medium
+        mov     val, v0.D[0]
+
+        /* Set 0..15 bytes.  */
+        tbz     count, 3, 1f
+        str     val, [dstin]
+        str     val, [dstend2, -8]
+        ret
+        nop
+1:      tbz     count, 2, 2f
+        str     valw, [dstin]
+        str     valw, [dstend2, -4]
+        ret
+2:      cbz     count, 3f
+        strb    valw, [dstin]
+        tbz     count, 1, 3f
+        strh    valw, [dstend2, -2]
+3:      ret
+
+        /* Set 17..96 bytes.  */
+4:  // set_medium
+        str     q0, [dstin]
+        tbnz    count, 6, 6f  // set96
+        str     q0, [dstend2, -16]
+        tbz     count, 5, 5f
+        str     q0, [dstin, 16]
+        str     q0, [dstend2, -32]
+5:      ret
+
+        .p2align 4
+        /* Set 64..96 bytes.  Write 64 bytes from the start and
+           32 bytes from the end.  */
+6:  // set96
+        str     q0, [dstin, 16]
+        stp     q0, q0, [dstin, 32]
+        stp     q0, q0, [dstend2, -32]
+        ret
+
+        .p2align 4
+7:  // set_long
+        and     valw, valw, 255
+        bic     dst, dstin, 15
+        str     q0, [dstin]
+        cmp     count, 160
+        ccmp    valw, 0, 0, hs
+        b.ne    9f  // no_zva
+
+#ifndef SKIP_ZVA_CHECK
+        mrs     zva_val, dczid_el0
+        and     zva_val, zva_val, 31
+        cmp     zva_val, 4              /* ZVA size is 64 bytes.  */
+        b.ne    9f  // no_zva
+#endif
+        str     q0, [dst, 16]
+        stp     q0, q0, [dst, 32]
+        bic     dst, dst, 63
+        sub     count, dstend2, dst      /* Count is now 64 too large.  */
+        sub     count, count, 128       /* Adjust count and bias for loop.  */
+
+        .p2align 4
+8:  // zva_loop
+        add     dst, dst, 64
+        dc      zva, dst
+        subs    count, count, 64
+        b.hi    8b  // zva_loop
+        stp     q0, q0, [dstend2, -64]
+        stp     q0, q0, [dstend2, -32]
+        ret
+
+9:  // no_zva
+        sub     count, dstend2, dst      /* Count is 16 too large.  */
+        sub     dst, dst, 16            /* Dst is biased by -32.  */
+        sub     count, count, 64 + 16   /* Adjust count and bias for loop.  */
+10: // no_zva_loop
+        stp     q0, q0, [dst, 32]
+        stp     q0, q0, [dst, 64]!
+        subs    count, count, 64
+        b.hi    10b  // no_zva_loop
+        stp     q0, q0, [dstend2, -64]
+        stp     q0, q0, [dstend2, -32]
+        ret
+END_COMPILERRT_FUNCTION(__arm_sc_memset)
+
+//
+//  __arm_sc_memchr
+//
+
+#define srcin		x0
+#define chrin		w1
+#define cntin		x2
+
+#define result		x0
+
+#define src		x3
+#define	tmp		x4
+#define wtmp2		w5
+#define synd		x6
+#define soff		x9
+#define cntrem		x10
+
+#define vrepchr		v0
+#define vdata1		v1
+#define vdata2		v2
+#define vhas_chr1	v3
+#define vhas_chr2	v4
+#define vrepmask	v5
+#define vend		v6
+
+/*
+ * Core algorithm:
+ *
+ * For each 32-byte chunk we calculate a 64-bit syndrome value, with two bits
+ * per byte. For each tuple, bit 0 is set if the relevant byte matched the
+ * requested character and bit 1 is not used (faster than using a 32bit
+ * syndrome). Since the bits in the syndrome reflect exactly the order in which
+ * things occur in the original string, counting trailing zeros allows to
+ * identify exactly which byte has matched.
+ */
+
+DEFINE_COMPILERRT_FUNCTION(__arm_sc_memchr)
+	/* Do not dereference srcin if no bytes to compare.  */
+	cbz	cntin, 4f
+	/*
+	 * Magic constant 0x40100401 allows us to identify which lane matches
+	 * the requested byte.
+	 */
+	mov	wtmp2, #0x0401
+	movk	wtmp2, #0x4010, lsl #16
+	dup	vrepchr.16b, chrin
+	/* Work with aligned 32-byte chunks */
+	bic	src, srcin, #31
+	dup	vrepmask.4s, wtmp2
+	ands	soff, srcin, #31
+	and	cntrem, cntin, #31
+	b.eq	0f
+
+	/*
+	 * Input string is not 32-byte aligned. We calculate the syndrome
+	 * value for the aligned 32 bytes block containing the first bytes
+	 * and mask the irrelevant part.
+	 */
+
+	ld1	{vdata1.16b, vdata2.16b}, [src], #32
+	sub	tmp, soff, #32
+	adds	cntin, cntin, tmp
+	cmeq	vhas_chr1.16b, vdata1.16b, vrepchr.16b
+	cmeq	vhas_chr2.16b, vdata2.16b, vrepchr.16b
+	and	vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b
+	and	vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b
+	addp	vend.16b, vhas_chr1.16b, vhas_chr2.16b		/* 256->128 */
+	addp	vend.16b, vend.16b, vend.16b			/* 128->64 */
+	mov	synd, vend.d[0]
+	/* Clear the soff*2 lower bits */
+	lsl	tmp, soff, #1
+	lsr	synd, synd, tmp
+	lsl	synd, synd, tmp
+	/* The first block can also be the last */
+	b.ls	2f
+	/* Have we found something already? */
+	cbnz	synd, 3f
+
+0: // loop
+	ld1	{vdata1.16b, vdata2.16b}, [src], #32
+	subs	cntin, cntin, #32
+	cmeq	vhas_chr1.16b, vdata1.16b, vrepchr.16b
+	cmeq	vhas_chr2.16b, vdata2.16b, vrepchr.16b
+	/* If we're out of data we finish regardless of the result */
+	b.ls	1f
+	/* Use a fast check for the termination condition */
+	orr	vend.16b, vhas_chr1.16b, vhas_chr2.16b
+	addp	vend.2d, vend.2d, vend.2d
+	mov	synd, vend.d[0]
+	/* We're not out of data, loop if we haven't found the character */
+	cbz	synd, 0b
+
+1: // end
+	/* Termination condition found, let's calculate the syndrome value */
+	and	vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b
+	and	vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b
+	addp	vend.16b, vhas_chr1.16b, vhas_chr2.16b		/* 256->128 */
+	addp	vend.16b, vend.16b, vend.16b			/* 128->64 */
+	mov	synd, vend.d[0]
+	/* Only do the clear for the last possible block */
+	b.hi	3f
+
+2: // masklast
+	/* Clear the (32 - ((cntrem + soff) % 32)) * 2 upper bits */
+	add	tmp, cntrem, soff
+	and	tmp, tmp, #31
+	sub	tmp, tmp, #32
+	neg	tmp, tmp, lsl #1
+	lsl	synd, synd, tmp
+	lsr	synd, synd, tmp
+
+3: // tail
+	/* Count the trailing zeros using bit reversing */
+	rbit	synd, synd
+	/* Compensate the last post-increment */
+	sub	src, src, #32
+	/* Check that we have found a character */
+	cmp	synd, #0
+	/* And count the leading zeros */
+	clz	synd, synd
+	/* Compute the potential result */
+	add	result, src, synd, lsr #1
+	/* Select result or NULL */
+	csel	result, xzr, result, eq
+	ret
+
+4: // zero_length
+	mov	result, #0
+	ret
+END_COMPILERRT_FUNCTION(__arm_sc_memchr)
+
diff --git a/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c b/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
deleted file mode 100644
index 07d668148555..000000000000
--- a/external/llvm-project/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
+++ /dev/null
@@ -1,24 +0,0 @@
-#include <stddef.h>
-
-/* The asm version uses FP registers. Use this on targets without them */
-#if __ARM_FP == 0
-void *__arm_sc_memset(void *dest, int c, size_t n) __arm_streaming_compatible {
-  unsigned char *destp = (unsigned char *)dest;
-  unsigned char c8 = (unsigned char)c;
-  for (size_t i = 0; i < n; ++i)
-    destp[i] = c8;
-
-  return dest;
-}
-#endif
-
-const void *__arm_sc_memchr(const void *src, int c,
-                            size_t n) __arm_streaming_compatible {
-  const unsigned char *srcp = (const unsigned char *)src;
-  unsigned char c8 = (unsigned char)c;
-  for (size_t i = 0; i < n; ++i)
-    if (srcp[i] == c8)
-      return &srcp[i];
-
-  return NULL;
-}
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
index c7abdb003a68..19311396b201 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
@@ -22,7 +22,12 @@
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq)
+        PACBTI_LANDING
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push {r0-r3, r12, lr}
+#else
         push {r0-r3, lr}
+#endif
         bl __aeabi_cdcmpeq_check_nan
         cmp r0, #1
 #if defined(USE_THUMB_1)
@@ -37,8 +42,13 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq)
         movs r0, #0xF
         lsls r0, r0, #31
         pop {r0-r3, pc}
+#else
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop {r0-r3, r12, lr}
+        aut r12, lr, sp
 #else
         pop {r0-r3, lr}
+#endif
 
         // NaN has been ruled out, so __aeabi_cdcmple can't trap
         // Use "it ne" + unconditional branch to guarantee a supported relocation if
@@ -47,8 +57,8 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq)
         bne __aeabi_cdcmple
 
 #if defined(USE_THUMB_2)
-        mov ip, #APSR_C
-        msr APSR_nzcvq, ip
+        mov r12, #APSR_C
+        msr APSR_nzcvq, r12
 #else
         msr APSR_nzcvq, #APSR_C
 #endif
@@ -70,9 +80,14 @@ END_COMPILERRT_FUNCTION(__aeabi_cdcmpeq)
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple)
+        PACBTI_LANDING
         // Per the RTABI, this function must preserve r0-r11.
         // Save lr in the same instruction for compactness
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push {r0-r3, r12, lr}
+#else
         push {r0-r3, lr}
+#endif
 
         bl __aeabi_dcmplt
         cmp r0, #1
@@ -111,9 +126,14 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple)
 
 1:
         msr APSR_nzcvq, ip
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop {r0-r3, r12, lr}
+        bxaut r12, lr, sp
+#else
         pop {r0-r3}
         POP_PC()
 #endif
+#endif
 END_COMPILERRT_FUNCTION(__aeabi_cdcmple)
 
 // int __aeabi_cdrcmple(double a, double b) {
@@ -123,6 +143,9 @@ END_COMPILERRT_FUNCTION(__aeabi_cdcmple)
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_cdrcmple)
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+        bti
+#endif
         // Swap r0 and r2
         mov ip, r0
         mov r0, r2
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
index 81c47661c8b5..ca382968cf5d 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
@@ -22,7 +22,12 @@
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq)
+        PACBTI_LANDING
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push {r0-r3, r12, lr}
+#else
         push {r0-r3, lr}
+#endif
         bl __aeabi_cfcmpeq_check_nan
         cmp r0, #1
 #if defined(USE_THUMB_1)
@@ -37,8 +42,13 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq)
         movs r0, #0xF
         lsls r0, r0, #31
         pop {r0-r3, pc}
+#else
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop {r0-r3, r12, lr}
+        aut r12, lr, sp
 #else
         pop {r0-r3, lr}
+#endif
 
         // NaN has been ruled out, so __aeabi_cfcmple can't trap
         // Use "it ne" + unconditional branch to guarantee a supported relocation if
@@ -47,8 +57,8 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq)
         bne __aeabi_cfcmple
 
 #if defined(USE_THUMB_2)
-        mov ip, #APSR_C
-        msr APSR_nzcvq, ip
+        mov r12, #APSR_C
+        msr APSR_nzcvq, r12
 #else
         msr APSR_nzcvq, #APSR_C
 #endif
@@ -70,9 +80,14 @@ END_COMPILERRT_FUNCTION(__aeabi_cfcmpeq)
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple)
+        PACBTI_LANDING
         // Per the RTABI, this function must preserve r0-r11.
         // Save lr in the same instruction for compactness
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push {r0-r3, r12, lr}
+#else
         push {r0-r3, lr}
+#endif
 
         bl __aeabi_fcmplt
         cmp r0, #1
@@ -111,9 +126,14 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple)
 
 1:
         msr APSR_nzcvq, ip
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop {r0-r3, r12, lr}
+        bxaut r12, lr, sp
+#else
         pop {r0-r3}
         POP_PC()
 #endif
+#endif
 END_COMPILERRT_FUNCTION(__aeabi_cfcmple)
 
 // int __aeabi_cfrcmple(float a, float b) {
@@ -123,6 +143,9 @@ END_COMPILERRT_FUNCTION(__aeabi_cfcmple)
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_cfrcmple)
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+        bti
+#endif
         // Swap r0 and r1
         mov ip, r0
         mov r0, r1
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_dcmp.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_dcmp.S
index bee14b3ff8af..4d1cc77d9a28 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_dcmp.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_dcmp.S
@@ -25,20 +25,34 @@
 #  define CONVERT_DCMP_ARGS_TO_DF2_ARGS
 #endif
 
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+#  define PROLOGUE PACBTI_LANDING        SEPARATOR \
+                   push      { r12, lr }
+#  define EPILOGUE pop       { r12, lr } SEPARATOR \
+                   bxaut     r12, lr, sp
+#elif defined(__ARM_FEATURE_BTI_DEFAULT)
+#  define PROLOGUE PACBTI_LANDING        SEPARATOR \
+                   push      { r4, lr }
+#  define EPILOGUE pop       { r4, pc }
+#else
+#  define PROLOGUE push      { r4, lr }
+#  define EPILOGUE pop       { r4, pc }
+#endif
+
 #define DEFINE_AEABI_DCMP(cond)                            \
         .syntax unified                          SEPARATOR \
         .p2align 2                               SEPARATOR \
 DEFINE_COMPILERRT_FUNCTION(__aeabi_dcmp ## cond)           \
-        push      { r4, lr }                     SEPARATOR \
+        PROLOGUE                                 SEPARATOR \
         CONVERT_DCMP_ARGS_TO_DF2_ARGS            SEPARATOR \
         bl        SYMBOL_NAME(__ ## cond ## df2) SEPARATOR \
         cmp       r0, #0                         SEPARATOR \
         b ## cond 1f                             SEPARATOR \
         movs      r0, #0                         SEPARATOR \
-        pop       { r4, pc }                     SEPARATOR \
+        EPILOGUE                                 SEPARATOR \
 1:                                               SEPARATOR \
         movs      r0, #1                         SEPARATOR \
-        pop       { r4, pc }                     SEPARATOR \
+        EPILOGUE                                 SEPARATOR \
 END_COMPILERRT_FUNCTION(__aeabi_dcmp ## cond)
 
 DEFINE_AEABI_DCMP(eq)
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_fcmp.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_fcmp.S
index cd311b417096..190a111d3447 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_fcmp.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_fcmp.S
@@ -25,20 +25,34 @@
 #  define CONVERT_FCMP_ARGS_TO_SF2_ARGS
 #endif
 
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+#  define PROLOGUE PACBTI_LANDING        SEPARATOR \
+                   push      { r12, lr }
+#  define EPILOGUE pop       { r12, lr } SEPARATOR \
+                   bxaut     r12, lr, sp
+#elif defined(__ARM_FEATURE_BTI_DEFAULT)
+#  define PROLOGUE PACBTI_LANDING        SEPARATOR \
+                   push      { r4, lr }
+#  define EPILOGUE pop       { r4, pc }
+#else
+#  define PROLOGUE push      { r4, lr }
+#  define EPILOGUE pop       { r4, pc }
+#endif
+
 #define DEFINE_AEABI_FCMP(cond)                            \
         .syntax unified                          SEPARATOR \
         .p2align 2                               SEPARATOR \
 DEFINE_COMPILERRT_FUNCTION(__aeabi_fcmp ## cond)           \
-        push      { r4, lr }                     SEPARATOR \
+        PROLOGUE                                 SEPARATOR \
         CONVERT_FCMP_ARGS_TO_SF2_ARGS            SEPARATOR \
         bl        SYMBOL_NAME(__ ## cond ## sf2) SEPARATOR \
         cmp       r0, #0                         SEPARATOR \
         b ## cond 1f                             SEPARATOR \
         movs      r0, #0                         SEPARATOR \
-        pop       { r4, pc }                     SEPARATOR \
+        EPILOGUE                                 SEPARATOR \
 1:                                               SEPARATOR \
         movs      r0, #1                         SEPARATOR \
-        pop       { r4, pc }                     SEPARATOR \
+        EPILOGUE                                 SEPARATOR \
 END_COMPILERRT_FUNCTION(__aeabi_fcmp ## cond)
 
 DEFINE_AEABI_FCMP(eq)
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_idivmod.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
index bb80e4b96fc1..4b82c946b1eb 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
@@ -31,7 +31,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_idivmod)
         subs    r1, r1, r2
         JMP     (r3)
 #else  // defined(USE_THUMB_1)
+        PACBTI_LANDING
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push    { r12, lr }
+#else
         push    { lr }
+#endif
         sub     sp, sp, #4
         mov     r2, sp
 #if defined(__MINGW32__)
@@ -42,7 +47,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_idivmod)
         bl      SYMBOL_NAME(__divmodsi4)
         ldr     r1, [sp]
         add     sp, sp, #4
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop     { r12, lr }
+        bxaut   r12, lr, sp
+#else
         pop     { pc }
+#endif
 #endif //  defined(USE_THUMB_1)
 END_COMPILERRT_FUNCTION(__aeabi_idivmod)
 
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S
index d0d06be6f329..c11ead68ff8f 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S
@@ -22,7 +22,12 @@
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_ldivmod)
+        PACBTI_LANDING
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push    {r6, r12, lr}
+#else
         push    {r6, lr}
+#endif
         sub     sp, sp, #16
         add     r6, sp, #8
         str     r6, [sp]
@@ -38,7 +43,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_ldivmod)
         ldr     r2, [sp, #8]
         ldr     r3, [sp, #12]
         add     sp, sp, #16
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop     {r6, r12, lr}
+        bxaut   r12, lr, sp
+#else
         pop     {r6, pc}
+#endif
 END_COMPILERRT_FUNCTION(__aeabi_ldivmod)
 
 NO_EXEC_STACK_DIRECTIVE
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memcpy.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memcpy.S
index 93e1b05d54ce..fa6a58640179 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memcpy.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memcpy.S
@@ -18,6 +18,9 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_memcpy)
         bl      memcpy
         pop     {r7, pc}
 #else
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+        bti
+#endif
         b       memcpy
 #endif
 END_COMPILERRT_FUNCTION(__aeabi_memcpy)
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memmove.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memmove.S
index c2f0fa4cd8ff..a72d52a397b9 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memmove.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memmove.S
@@ -17,6 +17,9 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_memmove)
         bl      memmove
         pop     {r7, pc}
 #else
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+        bti
+#endif
         b       memmove
 #endif
 END_COMPILERRT_FUNCTION(__aeabi_memmove)
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memset.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memset.S
index 2aa8ec0c4679..950be84c5307 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memset.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_memset.S
@@ -14,6 +14,9 @@
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_memset)
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+        bti
+#endif
         mov     r3, r1
         mov     r1, r2
         mov     r2, r3
@@ -31,6 +34,9 @@ DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset8, __aeabi_memset)
 
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_memclr)
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+        bti
+#endif
         mov     r2, r1
         movs    r1, #0
 #ifdef USE_THUMB_1
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
index df030769fd40..db2eecb3234f 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
@@ -38,7 +38,12 @@ LOCAL_LABEL(case_denom_larger):
         movs    r0, #0
         JMP     (lr)
 #else // defined(USE_THUMB_1)
+        PACBTI_LANDING
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push    { r12, lr }
+#else
         push    { lr }
+#endif
         sub     sp, sp, #4
         mov     r2, sp
 #if defined(__MINGW32__)
@@ -49,8 +54,13 @@ LOCAL_LABEL(case_denom_larger):
         bl      SYMBOL_NAME(__udivmodsi4)
         ldr     r1, [sp]
         add     sp, sp, #4
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop     { r12, lr }
+        bxaut   r12, lr, sp
+#else
         pop     { pc }
 #endif
+#endif
 END_COMPILERRT_FUNCTION(__aeabi_uidivmod)
 
 NO_EXEC_STACK_DIRECTIVE
diff --git a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S
index 4fc97704d3c5..66da6a39ef5d 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S
@@ -22,7 +22,12 @@
         .syntax unified
         .p2align 2
 DEFINE_COMPILERRT_FUNCTION(__aeabi_uldivmod)
-        push	{r6, lr}
+        PACBTI_LANDING
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        push    {r6, r12, lr}
+#else
+        push    {r6, lr}
+#endif
         sub	sp, sp, #16
         add	r6, sp, #8
         str	r6, [sp]
@@ -38,7 +43,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_uldivmod)
         ldr	r2, [sp, #8]
         ldr	r3, [sp, #12]
         add	sp, sp, #16
-        pop	{r6, pc}
+#if defined(__ARM_FEATURE_PAC_DEFAULT)
+        pop     {r6, r12, lr}
+        bxaut   r12, lr, sp
+#else
+        pop     {r6, pc}
+#endif
 END_COMPILERRT_FUNCTION(__aeabi_uldivmod)
 
 NO_EXEC_STACK_DIRECTIVE
diff --git a/external/llvm-project/compiler-rt/lib/builtins/assembly.h b/external/llvm-project/compiler-rt/lib/builtins/assembly.h
index 385dc190369d..714bcd4d5c1f 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/assembly.h
+++ b/external/llvm-project/compiler-rt/lib/builtins/assembly.h
@@ -194,6 +194,17 @@
 #else
 #define WIDE(op) op
 #endif
+
+#if defined(__ARM_FEATURE_PAC_DEFAULT) && defined(__ARM_FEATURE_BTI_DEFAULT)
+#define PACBTI_LANDING pacbti r12, lr, sp
+#elif defined(__ARM_FEATURE_PAC_DEFAULT)
+#define PACBTI_LANDING pac r12, lr, sp
+#elif defined(__ARM_FEATURE_BTI_DEFAULT)
+#define PACBTI_LANDING bti
+#else
+#define PACBTI_LANDING
+#endif
+
 #else // !defined(__arm)
 #define DECLARE_FUNC_ENCODING
 #define DEFINE_CODE_STATE
diff --git a/external/llvm-project/compiler-rt/lib/builtins/cpu_model/riscv.c b/external/llvm-project/compiler-rt/lib/builtins/cpu_model/riscv.c
index 4d0fda473c87..16d55fcfffe7 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/cpu_model/riscv.c
+++ b/external/llvm-project/compiler-rt/lib/builtins/cpu_model/riscv.c
@@ -24,12 +24,18 @@ struct {
 // TODO: Maybe generate a header from tablegen then include it.
 #define A_GROUPID 0
 #define A_BITMASK (1ULL << 0)
+#define B_GROUPID 0
+#define B_BITMASK (1ULL << 1)
 #define C_GROUPID 0
 #define C_BITMASK (1ULL << 2)
 #define D_GROUPID 0
 #define D_BITMASK (1ULL << 3)
+#define E_GROUPID 0
+#define E_BITMASK (1ULL << 4)
 #define F_GROUPID 0
 #define F_BITMASK (1ULL << 5)
+#define H_GROUPID 0
+#define H_BITMASK (1ULL << 7)
 #define I_GROUPID 0
 #define I_BITMASK (1ULL << 8)
 #define M_GROUPID 0
diff --git a/external/llvm-project/compiler-rt/lib/builtins/riscv/restore.S b/external/llvm-project/compiler-rt/lib/builtins/riscv/restore.S
index d87dfc1ac71d..7816f74ed497 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/riscv/restore.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/riscv/restore.S
@@ -14,9 +14,12 @@
 // them falling through into each other and don't want the linker to
 // accidentally split them up, garbage collect, or reorder them.
 //
-// The entry points are grouped up into 2s for rv64 and 4s for rv32 since this
-// is the minimum grouping which will maintain the required 16-byte stack
-// alignment.
+// For the conventional ABIs, entry points are grouped up into 2s for rv64 and
+// 4s for rv32 since this is the minimum grouping which will maintain the
+// required 16-byte stack alignment.
+//
+// For the ilp32e/lp64e abis, entry points are grouped into 1s, since this is
+// the minimum grouping which will maintain the required 4-byte stack alignment.
 
   .text
 
@@ -92,17 +95,23 @@ __riscv_restore_0:
 
   .globl  __riscv_restore_2
   .type   __riscv_restore_2,@function
+__riscv_restore_2:
+  lw      s1, 0(sp)
+  addi    sp, sp, 4
+  // fallthrough into __riscv_restore_1/0
+
   .globl  __riscv_restore_1
   .type   __riscv_restore_1,@function
+__riscv_restore_1:
+  lw      s0, 0(sp)
+  addi    sp, sp, 4
+  // fallthrough into __riscv_restore_0
+
   .globl  __riscv_restore_0
   .type   __riscv_restore_0,@function
-__riscv_restore_2:
-__riscv_restore_1:
 __riscv_restore_0:
-  lw      s1,  0(sp)
-  lw      s0,  4(sp)
-  lw      ra,  8(sp)
-  addi    sp, sp, 12
+  lw      ra, 0(sp)
+  addi    sp, sp, 4
   ret
 
 #endif
@@ -188,17 +197,23 @@ __riscv_restore_0:
 
   .globl  __riscv_restore_2
   .type   __riscv_restore_2,@function
+__riscv_restore_2:
+  ld      s1, 0(sp)
+  addi    sp, sp, 8
+  // fallthrough into __riscv_restore_1/0
+
   .globl  __riscv_restore_1
   .type   __riscv_restore_1,@function
+__riscv_restore_1:
+  ld      s0, 0(sp)
+  addi    sp, sp, 8
+  // fallthrough into __riscv_restore_0
+
   .globl  __riscv_restore_0
   .type   __riscv_restore_0,@function
-__riscv_restore_2:
-__riscv_restore_1:
 __riscv_restore_0:
-  ld      s1,  0(sp)
-  ld      s0,  8(sp)
-  ld      ra,  16(sp)
-  addi    sp, sp, 24
+  ld      ra, 0(sp)
+  addi    sp, sp, 8
   ret
 
 #endif
diff --git a/external/llvm-project/compiler-rt/lib/builtins/riscv/save.S b/external/llvm-project/compiler-rt/lib/builtins/riscv/save.S
index 6324e05e9719..6b50bce02252 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/riscv/save.S
+++ b/external/llvm-project/compiler-rt/lib/builtins/riscv/save.S
@@ -98,19 +98,28 @@ __riscv_save_0:
 
   .globl  __riscv_save_2
   .type   __riscv_save_2,@function
-  .globl  __riscv_save_1
-  .type   __riscv_save_1,@function
-  .globl  __riscv_save_0
-  .type   __riscv_save_0,@function
 __riscv_save_2:
-__riscv_save_1:
-__riscv_save_0:
   addi    sp, sp, -12
   sw      s1,  0(sp)
   sw      s0,  4(sp)
   sw      ra,  8(sp)
   jr      t0
 
+  .globl  __riscv_save_1
+  .type   __riscv_save_1,@function
+__riscv_save_1:
+  addi    sp, sp, -8
+  sw      s0,  0(sp)
+  sw      ra,  4(sp)
+  jr      t0
+
+  .globl  __riscv_save_0
+  .type   __riscv_save_0,@function
+__riscv_save_0:
+  addi    sp, sp, -4
+  sw      ra,  0(sp)
+  jr      t0
+
 #endif
 
 #elif __riscv_xlen == 64
@@ -208,18 +217,27 @@ __riscv_save_0:
 
   .globl  __riscv_save_2
   .type   __riscv_save_2,@function
+__riscv_save_2:
+  addi    sp, sp, -24
+  sw      s1, 0(sp)
+  sw      s0, 8(sp)
+  sw      ra, 16(sp)
+  jr      t0
+
   .globl  __riscv_save_1
   .type   __riscv_save_1,@function
+__riscv_save_1:
+  addi    sp, sp, -16
+  sw      s0, 0(sp)
+  sw      ra, 8(sp)
+  jr      t0
+
   .globl  __riscv_save_0
   .type   __riscv_save_0,@function
-__riscv_save_2:
-__riscv_save_1:
 __riscv_save_0:
-  addi   sp, sp, -24
-  sd     s1, 0(sp)
-  sd     s0, 8(sp)
-  sd     ra, 16(sp)
-  jr     t0
+  addi    sp, sp, -8
+  sw      ra,  0(sp)
+  jr      t0
 
 #endif
 
diff --git a/external/llvm-project/compiler-rt/lib/builtins/trampoline_setup.c b/external/llvm-project/compiler-rt/lib/builtins/trampoline_setup.c
index 830e25e4c030..844eb2794414 100644
--- a/external/llvm-project/compiler-rt/lib/builtins/trampoline_setup.c
+++ b/external/llvm-project/compiler-rt/lib/builtins/trampoline_setup.c
@@ -41,45 +41,3 @@ COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack,
   __clear_cache(trampOnStack, &trampOnStack[10]);
 }
 #endif // __powerpc__ && !defined(__powerpc64__)
-
-// The AArch64 compiler generates calls to __trampoline_setup() when creating
-// trampoline functions on the stack for use with nested functions.
-// This function creates a custom 36-byte trampoline function on the stack
-// which loads x18 with a pointer to the outer function's locals
-// and then jumps to the target nested function.
-// Note: x18 is a reserved platform register on Windows and macOS.
-
-#if defined(__aarch64__) && defined(__ELF__)
-COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack,
-                                        int trampSizeAllocated,
-                                        const void *realFunc, void *localsPtr) {
-  // This should never happen, but if compiler did not allocate
-  // enough space on stack for the trampoline, abort.
-  if (trampSizeAllocated < 36)
-    compilerrt_abort();
-
-  // create trampoline
-  // Load realFunc into x17. mov/movk 16 bits at a time.
-  trampOnStack[0] =
-      0xd2800000u | ((((uint64_t)realFunc >> 0) & 0xffffu) << 5) | 0x11;
-  trampOnStack[1] =
-      0xf2a00000u | ((((uint64_t)realFunc >> 16) & 0xffffu) << 5) | 0x11;
-  trampOnStack[2] =
-      0xf2c00000u | ((((uint64_t)realFunc >> 32) & 0xffffu) << 5) | 0x11;
-  trampOnStack[3] =
-      0xf2e00000u | ((((uint64_t)realFunc >> 48) & 0xffffu) << 5) | 0x11;
-  // Load localsPtr into x18
-  trampOnStack[4] =
-      0xd2800000u | ((((uint64_t)localsPtr >> 0) & 0xffffu) << 5) | 0x12;
-  trampOnStack[5] =
-      0xf2a00000u | ((((uint64_t)localsPtr >> 16) & 0xffffu) << 5) | 0x12;
-  trampOnStack[6] =
-      0xf2c00000u | ((((uint64_t)localsPtr >> 32) & 0xffffu) << 5) | 0x12;
-  trampOnStack[7] =
-      0xf2e00000u | ((((uint64_t)localsPtr >> 48) & 0xffffu) << 5) | 0x12;
-  trampOnStack[8] = 0xd61f0220; // br x17
-
-  // Clear instruction cache.
-  __clear_cache(trampOnStack, &trampOnStack[9]);
-}
-#endif // defined(__aarch64__) && !defined(__APPLE__) && !defined(_WIN64)
diff --git a/external/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp b/external/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
index 93bf817a857b..c9210c78a063 100644
--- a/external/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
+++ b/external/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
@@ -265,8 +265,6 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
     // we then request tags in [0,Size/2) and [Size/2, Size), and so on.
     // Function number => DFT.
     auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File)));
-    std::unordered_map<size_t, std::vector<uint8_t>> DFTMap;
-    std::unordered_set<std::string> Cov;
     Command Cmd;
     Cmd.addArgument(DFTBinary);
     Cmd.addArgument(F.File);
diff --git a/external/llvm-project/compiler-rt/lib/interception/interception_win.cpp b/external/llvm-project/compiler-rt/lib/interception/interception_win.cpp
index c9a77180014c..246a22c56c31 100644
--- a/external/llvm-project/compiler-rt/lib/interception/interception_win.cpp
+++ b/external/llvm-project/compiler-rt/lib/interception/interception_win.cpp
@@ -754,6 +754,12 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x7B81:  // 81 7B YY XX XX XX XX  cmp DWORD PTR [rbx+YY], XX XX XX XX
     case 0x7981:  // 81 79 YY XX XX XX XX  cmp dword ptr [rcx+YY], XX XX XX XX
       return 7;
+
+    case 0xb848:  // 48 b8 XX XX XX XX XX XX XX XX :
+                  //   movabsq XX XX XX XX XX XX XX XX, rax
+    case 0xba48:  // 48 ba XX XX XX XX XX XX XX XX :
+                  //   movabsq XX XX XX XX XX XX XX XX, rdx
+      return 10;
   }
 
   switch (0x00FFFFFF & *(u32 *)address) {
diff --git a/external/llvm-project/compiler-rt/lib/interception/tests/interception_win_test.cpp b/external/llvm-project/compiler-rt/lib/interception/tests/interception_win_test.cpp
index 893f346d73b8..f11c1d145855 100644
--- a/external/llvm-project/compiler-rt/lib/interception/tests/interception_win_test.cpp
+++ b/external/llvm-project/compiler-rt/lib/interception/tests/interception_win_test.cpp
@@ -1043,6 +1043,8 @@ const struct InstructionSizeData {
     { 8, {0xc7, 0x44, 0x24, 0x73, 0x74, 0x75, 0x76, 0x77}, 0, "C7 44 24 XX YY YY YY YY : mov dword ptr [rsp + XX], YYYYYYYY"},
     { 9, {0x41, 0x81, 0x7c, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78}, 0, "41 81 7c ZZ YY XX XX XX XX : cmp DWORD PTR [reg+reg*n+YY], XX XX XX XX"},
     { 9, {0xA1, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78}, 0, "A1 XX XX XX XX XX XX XX XX : movabs eax, dword ptr ds:[XXXXXXXX]"},
+    {10, {0x48, 0xb8, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79}, 0, "48 b8 XX XX XX XX XX XX XX XX : movabsq XX XX XX XX XX XX XX XX, rax"},
+    {10, {0x48, 0xba, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79}, 0, "48 ba XX XX XX XX XX XX XX XX : movabsq XX XX XX XX XX XX XX XX, rdx"},
 #else
     // sorted list
     { 3, {0x8B, 0x45, 0x72}, 0, "8B 45 XX : mov eax, dword ptr [ebp + XX]"},
diff --git a/external/llvm-project/compiler-rt/lib/lsan/lsan_common.cpp b/external/llvm-project/compiler-rt/lib/lsan/lsan_common.cpp
index 7ab9e4ff2ac9..b17a17e1193b 100644
--- a/external/llvm-project/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/external/llvm-project/compiler-rt/lib/lsan/lsan_common.cpp
@@ -124,7 +124,8 @@ static const char kStdSuppressions[] =
 #  endif
     // TLS leak in some glibc versions, described in
     // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
-    "leak:*tls_get_addr*\n";
+    "leak:*tls_get_addr*\n"
+    "leak:*dlerror*\n";
 
 void InitializeSuppressions() {
   CHECK_EQ(nullptr, suppression_ctx);
diff --git a/external/llvm-project/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/external/llvm-project/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index c2d07400593d..2ee35555c24d 100644
--- a/external/llvm-project/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/external/llvm-project/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -449,12 +449,6 @@ TEST_F(RtsanFileTest, FcntlSetFdDiesWhenRealtime) {
   close(fd);
 }
 
-TEST(TestRtsanInterceptors, CloseDiesWhenRealtime) {
-  auto Func = []() { close(0); };
-  ExpectRealtimeDeath(Func, "close");
-  ExpectNonRealtimeSurvival(Func);
-}
-
 TEST(TestRtsanInterceptors, ChdirDiesWhenRealtime) {
   auto Func = []() { chdir("."); };
   ExpectRealtimeDeath(Func, "chdir");
@@ -606,8 +600,10 @@ class RtsanOpenedFileTest : public RtsanFileTest {
   }
 
   void TearDown() override {
-    if (file != nullptr)
+    const bool is_open = fcntl(fd, F_GETFD) != -1;
+    if (is_open && file != nullptr)
       fclose(file);
+
     RtsanFileTest::TearDown();
   }
 
@@ -620,6 +616,16 @@ class RtsanOpenedFileTest : public RtsanFileTest {
   int fd = -1;
 };
 
+TEST_F(RtsanOpenedFileTest, CloseDiesWhenRealtime) {
+  auto Func = [this]() { close(GetOpenFd()); };
+  ExpectRealtimeDeath(Func, "close");
+}
+
+TEST_F(RtsanOpenedFileTest, CloseSurvivesWhenNotRealtime) {
+  auto Func = [this]() { close(GetOpenFd()); };
+  ExpectNonRealtimeSurvival(Func);
+}
+
 #if SANITIZER_INTERCEPT_FSEEK
 TEST_F(RtsanOpenedFileTest, FgetposDieWhenRealtime) {
   auto Func = [this]() {
diff --git a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 9272e2ab6cbd..2d6cf7fc3282 100644
--- a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -481,7 +481,8 @@ INTERCEPTOR(char*, textdomain, const char *domainname) {
 #endif
 
 #if SANITIZER_INTERCEPT_STRCMP || SANITIZER_INTERCEPT_MEMCMP
-static inline int CharCmpX(unsigned char c1, unsigned char c2) {
+[[maybe_unused]] static inline int CharCmpX(unsigned char c1,
+                                            unsigned char c2) {
   return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1;
 }
 #endif
@@ -1350,7 +1351,8 @@ INTERCEPTOR(unsigned long, time, unsigned long *t) {
 #if SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS
 static void unpoison_tm(void *ctx, __sanitizer_tm *tm) {
   COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tm, sizeof(*tm));
-#if !SANITIZER_SOLARIS
+// AIX tm struct does not have tm_zone field.
+#  if !SANITIZER_SOLARIS && !SANITIZER_AIX
   if (tm->tm_zone) {
     // Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone
     // can point to shared memory and tsan would report a data race.
@@ -1735,10 +1737,12 @@ INTERCEPTOR(int, __vsprintf_chk, char *str, int flag, SIZE_T size_to,
 VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap)
 #endif
 
+#  if SANITIZER_INTERCEPT_VASPRINTF
 INTERCEPTOR(int, vasprintf, char **strp, const char *format, va_list ap)
 VASPRINTF_INTERCEPTOR_IMPL(vasprintf, strp, format, ap)
+#  endif
 
-#if SANITIZER_INTERCEPT_ISOC99_PRINTF
+#  if SANITIZER_INTERCEPT_ISOC99_PRINTF
 INTERCEPTOR(int, __isoc99_vprintf, const char *format, va_list ap)
 VPRINTF_INTERCEPTOR_IMPL(__isoc99_vprintf, format, ap)
 
@@ -1787,10 +1791,12 @@ INTERCEPTOR(int, __snprintf_chk, char *str, SIZE_T size, int flag,
 FORMAT_INTERCEPTOR_IMPL(__snprintf_chk, vsnprintf, str, size, format)
 #endif
 
+#  if SANITIZER_INTERCEPT_ASPRINTF
 INTERCEPTOR(int, asprintf, char **strp, const char *format, ...)
 FORMAT_INTERCEPTOR_IMPL(asprintf, vasprintf, strp, format)
+#  endif
 
-#if SANITIZER_INTERCEPT_ISOC99_PRINTF
+#  if SANITIZER_INTERCEPT_ISOC99_PRINTF
 INTERCEPTOR(int, __isoc99_printf, const char *format, ...)
 FORMAT_INTERCEPTOR_IMPL(__isoc99_printf, __isoc99_vprintf, format)
 
@@ -1811,17 +1817,24 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size,
 #endif  // SANITIZER_INTERCEPT_PRINTF
 
 #if SANITIZER_INTERCEPT_PRINTF
-#define INIT_PRINTF                     \
-  COMMON_INTERCEPT_FUNCTION_LDBL(printf);    \
-  COMMON_INTERCEPT_FUNCTION_LDBL(sprintf);   \
-  COMMON_INTERCEPT_FUNCTION_LDBL(snprintf);  \
-  COMMON_INTERCEPT_FUNCTION_LDBL(asprintf);  \
-  COMMON_INTERCEPT_FUNCTION_LDBL(fprintf);   \
-  COMMON_INTERCEPT_FUNCTION_LDBL(vprintf);   \
-  COMMON_INTERCEPT_FUNCTION_LDBL(vsprintf);  \
-  COMMON_INTERCEPT_FUNCTION_LDBL(vsnprintf); \
-  COMMON_INTERCEPT_FUNCTION_LDBL(vasprintf); \
-  COMMON_INTERCEPT_FUNCTION_LDBL(vfprintf);
+#  define INIT_PRINTF_COMMON                   \
+    COMMON_INTERCEPT_FUNCTION_LDBL(printf);    \
+    COMMON_INTERCEPT_FUNCTION_LDBL(sprintf);   \
+    COMMON_INTERCEPT_FUNCTION_LDBL(snprintf);  \
+    COMMON_INTERCEPT_FUNCTION_LDBL(fprintf);   \
+    COMMON_INTERCEPT_FUNCTION_LDBL(vprintf);   \
+    COMMON_INTERCEPT_FUNCTION_LDBL(vsprintf);  \
+    COMMON_INTERCEPT_FUNCTION_LDBL(vsnprintf); \
+    COMMON_INTERCEPT_FUNCTION_LDBL(vfprintf);
+#  if !SANITIZER_AIX
+// AIX does not have [v]asprintf.
+#    define INIT_PRINTF_EXTRA                   \
+      COMMON_INTERCEPT_FUNCTION_LDBL(asprintf); \
+      COMMON_INTERCEPT_FUNCTION_LDBL(vasprintf);
+#  else
+#    define INIT_PRINTF_EXTRA
+#  endif
+#  define INIT_PRINTF INIT_PRINTF_COMMON INIT_PRINTF_EXTRA
 #else
 #define INIT_PRINTF
 #endif
diff --git a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
index bc8f02826c61..08c2be47f535 100644
--- a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
+++ b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
@@ -79,7 +79,9 @@ static void ioctl_table_fill() {
   _(TIOCMSET, READ, sizeof(int));
   _(TIOCNXCL, NONE, 0);
   _(TIOCOUTQ, WRITE, sizeof(int));
+#  if !SANITIZER_AIX
   _(TIOCSCTTY, NONE, 0);
+#  endif
   _(TIOCSPGRP, READ, pid_t_sz);
   _(TIOCSWINSZ, READ, struct_winsize_sz);
 
diff --git a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
index 1565a494140f..0b6731c89950 100644
--- a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
+++ b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
@@ -33,11 +33,13 @@
 
 // Platform-specific options.
 #if SANITIZER_APPLE
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
+#  define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
 #elif SANITIZER_WINDOWS64
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
+#  define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
+#elif SANITIZER_AIX
+#  define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
 #else
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
+#  define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
 #endif  // SANITIZER_APPLE
 
 #ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
diff --git a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 4bc55d7801db..ccc808b60ca7 100644
--- a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -141,6 +141,12 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SI_SOLARIS 0
 #endif
 
+#if SANITIZER_AIX
+#  define SI_NOT_AIX 0
+#else
+#  define SI_NOT_AIX 1
+#endif
+
 #if SANITIZER_SOLARIS32
 #define SI_SOLARIS32 1
 #else
@@ -161,20 +167,20 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 
 #define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA
 #define SANITIZER_INTERCEPT_STRNLEN (SI_NOT_MAC && SI_NOT_FUCHSIA)
-#define SANITIZER_INTERCEPT_STRCMP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCMP (SI_NOT_FUCHSIA && SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_STRSTR SI_NOT_FUCHSIA
-#define SANITIZER_INTERCEPT_STRCASESTR SI_POSIX
+#define SANITIZER_INTERCEPT_STRCASESTR (SI_POSIX && SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_STRTOK SI_NOT_FUCHSIA
 #define SANITIZER_INTERCEPT_STRCHR SI_NOT_FUCHSIA
-#define SANITIZER_INTERCEPT_STRCHRNUL SI_POSIX_NOT_MAC
+#define SANITIZER_INTERCEPT_STRCHRNUL (SI_POSIX_NOT_MAC && SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_STRRCHR SI_NOT_FUCHSIA
 #define SANITIZER_INTERCEPT_STRSPN SI_NOT_FUCHSIA
 #define SANITIZER_INTERCEPT_STRPBRK SI_NOT_FUCHSIA
 #define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID || SI_SOLARIS
 #define SANITIZER_INTERCEPT_STRCASECMP SI_POSIX
 #define SANITIZER_INTERCEPT_MEMSET 1
-#define SANITIZER_INTERCEPT_MEMMOVE 1
-#define SANITIZER_INTERCEPT_MEMCPY 1
+#define SANITIZER_INTERCEPT_MEMMOVE SI_NOT_AIX
+#define SANITIZER_INTERCEPT_MEMCPY SI_NOT_AIX
 #define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA
 #define SANITIZER_INTERCEPT_BCMP \
   SANITIZER_INTERCEPT_MEMCMP &&  \
@@ -233,9 +239,11 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_ISOC99_SCANF SI_GLIBC
 
 #ifndef SANITIZER_INTERCEPT_PRINTF
-#define SANITIZER_INTERCEPT_PRINTF SI_POSIX
-#define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD)
-#define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_GLIBC
+#  define SANITIZER_INTERCEPT_ASPRINTF SI_NOT_AIX
+#  define SANITIZER_INTERCEPT_VASPRINTF SI_NOT_AIX
+#  define SANITIZER_INTERCEPT_PRINTF SI_POSIX
+#  define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD)
+#  define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_GLIBC
 #endif
 
 #define SANITIZER_INTERCEPT_SETPROCTITLE (SI_FREEBSD || SI_NETBSD)
@@ -243,8 +251,9 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT___PRINTF_CHK \
   (SANITIZER_INTERCEPT_PRINTF && SI_GLIBC)
 
-#define SANITIZER_INTERCEPT_FREXP SI_NOT_FUCHSIA
-#define SANITIZER_INTERCEPT_FREXPF SI_POSIX
+// AIX libc does not export FREXP and FREXPF.
+#define SANITIZER_INTERCEPT_FREXP (SI_NOT_FUCHSIA && SI_NOT_AIX)
+#define SANITIZER_INTERCEPT_FREXPF (SI_POSIX && SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_FREXPL SI_POSIX
 
 #define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_POSIX
@@ -294,7 +303,7 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_ACCEPT4 \
   (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_PACCEPT SI_NETBSD
-#define SANITIZER_INTERCEPT_MODF SI_POSIX
+#define SANITIZER_INTERCEPT_MODF (SI_POSIX && SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
 #define SANITIZER_INTERCEPT_SENDMSG SI_POSIX
 #define SANITIZER_INTERCEPT_RECVMMSG SI_LINUX
@@ -329,8 +338,9 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT___WCSXFRM_L SI_LINUX
 #define SANITIZER_INTERCEPT_WCSNRTOMBS \
   (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_WCRTOMB \
-  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCRTOMB                                           \
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS || \
+   !SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_WCTOMB \
   (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS
@@ -370,7 +380,8 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_STATFS \
   (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_STATFS64 SI_GLIBC && SANITIZER_HAS_STATFS64
+#define SANITIZER_INTERCEPT_STATFS64 \
+  ((SI_GLIBC || !SI_NOT_AIX) && SANITIZER_HAS_STATFS64)
 #define SANITIZER_INTERCEPT_STATVFS \
   (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_STATVFS64 SI_GLIBC
@@ -419,10 +430,10 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX
 #define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX
 #define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS
-#define SANITIZER_INTERCEPT_REMQUO SI_POSIX
-#define SANITIZER_INTERCEPT_REMQUOL (SI_POSIX && !SI_NETBSD)
-#define SANITIZER_INTERCEPT_LGAMMA SI_POSIX
-#define SANITIZER_INTERCEPT_LGAMMAL (SI_POSIX && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_REMQUO (SI_POSIX && SI_NOT_AIX)
+#define SANITIZER_INTERCEPT_REMQUOL (SI_POSIX && !SI_NETBSD && SI_NOT_AIX)
+#define SANITIZER_INTERCEPT_LGAMMA (SI_POSIX && SI_NOT_AIX)
+#define SANITIZER_INTERCEPT_LGAMMAL (SI_POSIX && !SI_NETBSD && SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS
 #define SANITIZER_INTERCEPT_DRAND48_R SI_GLIBC
@@ -505,11 +516,13 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE (SI_LINUX || SI_FREEBSD)
 
 #define SI_STAT_LINUX (SI_LINUX && __GLIBC_PREREQ(2, 33))
-#define SANITIZER_INTERCEPT_STAT                                        \
-  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS ||     \
-   SI_STAT_LINUX)
-#define SANITIZER_INTERCEPT_STAT64 SI_STAT_LINUX && SANITIZER_HAS_STAT64
-#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD || SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT_STAT                                    \
+  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS || \
+   SI_STAT_LINUX || !SI_NOT_AIX)
+#define SANITIZER_INTERCEPT_STAT64 \
+  ((SI_STAT_LINUX || !SI_NOT_AIX) && SANITIZER_HAS_STAT64)
+#define SANITIZER_INTERCEPT_LSTAT \
+  (SI_NETBSD || SI_FREEBSD || SI_STAT_LINUX || !SI_NOT_AIX)
 #define SANITIZER_INTERCEPT___XSTAT \
   ((!SANITIZER_INTERCEPT_STAT && SI_POSIX) || SI_STAT_LINUX)
 #define SANITIZER_INTERCEPT___XSTAT64 SI_GLIBC
@@ -578,7 +591,7 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_PROTOENT_R SI_GLIBC
 #define SANITIZER_INTERCEPT_NETENT (SI_LINUX || SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_SETVBUF \
-  (SI_NETBSD || SI_FREEBSD || SI_LINUX || SI_MAC)
+  (SI_NETBSD || SI_FREEBSD || SI_LINUX || SI_MAC || !SI_NOT_AIX)
 #define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD || SI_FREEBSD || SI_MAC)
 #define SANITIZER_INTERCEPT_MI_VECTOR_HASH SI_NETBSD
 #define SANITIZER_INTERCEPT_GETVFSSTAT SI_NETBSD
diff --git a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
index 80b8158f43db..452b30308be9 100644
--- a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
+++ b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
@@ -9,9 +9,6 @@
 // Information about the process mappings (Solaris-specific parts).
 //===----------------------------------------------------------------------===//
 
-// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
-#undef _FILE_OFFSET_BITS
-
 // Avoid conflict between `_TIME_BITS` defined vs. `_FILE_OFFSET_BITS`
 // undefined in some Linux configurations.
 #undef _TIME_BITS
diff --git a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
index 41e0613d6fc1..bda0f0468769 100644
--- a/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
+++ b/external/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
@@ -15,7 +15,7 @@
 #    define SANITIZER_REDEFINE_BUILTINS_H
 
 // The asm hack only works with GCC and Clang.
-#    if !defined(_WIN32)
+#    if !defined(_WIN32) && !defined(_AIX)
 
 asm(R"(
     .set memcpy, __sanitizer_internal_memcpy
diff --git a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 7c4d23a6a0d7..a4bc3d6cff91 100644
--- a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -3085,6 +3085,10 @@ void InitializeInterceptors() {
 #if !SANITIZER_ANDROID
   TSAN_INTERCEPT(dl_iterate_phdr);
 #endif
+
+  // Symbolization indirectly calls dl_iterate_phdr
+  ready_to_symbolize = true;
+
   TSAN_MAYBE_INTERCEPT_ON_EXIT;
   TSAN_INTERCEPT(__cxa_atexit);
   TSAN_INTERCEPT(_exit);
diff --git a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
index 373acd3d95d0..2c55645a1547 100644
--- a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
+++ b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
@@ -259,7 +259,15 @@ static void ReExecIfNeeded(bool ignore_heap) {
             "WARNING: Program is run with randomized virtual address "
             "space, which wouldn't work with ThreadSanitizer on Android.\n"
             "Re-execing with fixed virtual address space.\n");
-    CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
+
+    if (personality(old_personality | ADDR_NO_RANDOMIZE) == -1) {
+      Printf(
+          "FATAL: ThreadSanitizer: unable to disable ASLR (perhaps "
+          "sandboxing is enabled?).\n");
+      Printf("FATAL: Please rerun without sandboxing and/or ASLR.\n");
+      Die();
+    }
+
     reexec = true;
   }
 #      endif
@@ -287,7 +295,18 @@ static void ReExecIfNeeded(bool ignore_heap) {
               "possibly due to high-entropy ASLR.\n"
               "Re-execing with fixed virtual address space.\n"
               "N.B. reducing ASLR entropy is preferable.\n");
-      CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
+
+      if (personality(old_personality | ADDR_NO_RANDOMIZE) == -1) {
+        Printf(
+            "FATAL: ThreadSanitizer: encountered an incompatible memory "
+            "layout but was unable to disable ASLR (perhaps sandboxing is "
+            "enabled?).\n");
+        Printf(
+            "FATAL: Please rerun with lower ASLR entropy, ASLR disabled, "
+            "and/or sandboxing disabled.\n");
+        Die();
+      }
+
       reexec = true;
     } else {
       Printf(
diff --git a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
index c83efec8eaca..d8be21284b93 100644
--- a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
+++ b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
@@ -679,6 +679,12 @@ void CheckUnwind() {
 
 bool is_initialized;
 
+// Symbolization indirectly calls dl_iterate_phdr. If a CHECK() fails early on
+// (prior to the dl_iterate_phdr interceptor setup), resulting in an attempted
+// symbolization, it will segfault.
+// dl_iterate_phdr is not intercepted for Android.
+bool ready_to_symbolize = SANITIZER_ANDROID;
+
 void Initialize(ThreadState *thr) {
   // Thread safe because done before all threads exist.
   if (is_initialized)
diff --git a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.h
index 4dc5e630c524..0be53599b6a4 100644
--- a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.h
+++ b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl.h
@@ -54,6 +54,8 @@
 
 namespace __tsan {
 
+extern bool ready_to_symbolize;
+
 #if !SANITIZER_GO
 struct MapUnmapCallback;
 #  if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \
diff --git a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
index 51a98e2f2d5e..0820bf1adee4 100644
--- a/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
+++ b/external/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
@@ -846,7 +846,16 @@ ALWAYS_INLINE USED void PrintCurrentStack(uptr pc, bool fast) {
     ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1];
     ptrace->trace_buffer[ptrace->size - i - 1] = tmp;
   }
-  PrintStack(SymbolizeStack(*ptrace));
+
+  if (ready_to_symbolize) {
+    PrintStack(SymbolizeStack(*ptrace));
+  } else {
+    Printf(
+        "WARNING: PrintCurrentStack() has been called too early, before "
+        "symbolization is possible. Printing unsymbolized stack trace:\n");
+    for (unsigned int i = 0; i < ptrace->size; i++)
+      Printf("    #%u: 0x%zx\n", i, ptrace->trace[i]);
+  }
 #endif
 }
 
diff --git a/external/llvm-project/compiler-rt/test/builtins/Unit/trampoline_setup_test.c b/external/llvm-project/compiler-rt/test/builtins/Unit/trampoline_setup_test.c
index d51d35acaa02..da115fe76427 100644
--- a/external/llvm-project/compiler-rt/test/builtins/Unit/trampoline_setup_test.c
+++ b/external/llvm-project/compiler-rt/test/builtins/Unit/trampoline_setup_test.c
@@ -7,7 +7,7 @@
 
 /*
  * Tests nested functions
- * The ppc and aarch64 compilers generates a call to __trampoline_setup
+ * The ppc compiler generates a call to __trampoline_setup
  * The i386 and x86_64 compilers generate a call to ___enable_execute_stack
  */
 
diff --git a/external/llvm-project/compiler-rt/test/cfi/CMakeLists.txt b/external/llvm-project/compiler-rt/test/cfi/CMakeLists.txt
index 98c328fdcf11..a02573dfeb1e 100644
--- a/external/llvm-project/compiler-rt/test/cfi/CMakeLists.txt
+++ b/external/llvm-project/compiler-rt/test/cfi/CMakeLists.txt
@@ -47,7 +47,7 @@ foreach(arch ${CFI_TEST_ARCH})
   else()
     add_cfi_test_suites(False False)
     add_cfi_test_suites(False True)
-    if (COMPILER_RT_HAS_LLD AND NOT arch STREQUAL "i386")
+    if (COMPILER_RT_HAS_LLD)
 	    add_cfi_test_suites(True False)
 	    add_cfi_test_suites(True True)
     endif()
diff --git a/external/llvm-project/compiler-rt/test/hwasan/TestCases/Posix/dlerror.cpp b/external/llvm-project/compiler-rt/test/hwasan/TestCases/Posix/dlerror.cpp
new file mode 100644
index 000000000000..91acd28a1a5f
--- /dev/null
+++ b/external/llvm-project/compiler-rt/test/hwasan/TestCases/Posix/dlerror.cpp
@@ -0,0 +1,33 @@
+// Make sure dlerror is not classified as a leak even if we use dynamic TLS.
+// This is currently not implemented, so this test is XFAIL.
+
+// Android HWAsan does not support LSan.
+// UNSUPPORTED: android
+
+// RUN: %clangxx_hwasan -O0 %s -o %t && HWASAN_OPTIONS=detect_leaks=1 %run %t
+
+#include <assert.h>
+#include <dlfcn.h>
+#include <pthread.h>
+#include <sanitizer/hwasan_interface.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+// musl only has  128 keys
+constexpr auto kKeys = 100;
+
+int main(int argc, char **argv) {
+  __hwasan_enable_allocator_tagging();
+  // Exhaust static TLS slots to force use of dynamic TLS.
+  pthread_key_t keys[kKeys];
+  for (int i = 0; i < kKeys; ++i) {
+    assert(pthread_key_create(&keys[i], nullptr) == 0);
+  }
+  void *o = dlopen("invalid_file_name.so", 0);
+  const char *err = dlerror();
+  for (int i = 0; i < kKeys; ++i) {
+    assert(pthread_key_delete(keys[i]) == 0);
+  }
+  return 0;
+}
diff --git a/external/llvm-project/compiler-rt/test/msan/ifaddrs.cpp b/external/llvm-project/compiler-rt/test/msan/ifaddrs.cpp
index 91730a01f2d8..e06775db3251 100644
--- a/external/llvm-project/compiler-rt/test/msan/ifaddrs.cpp
+++ b/external/llvm-project/compiler-rt/test/msan/ifaddrs.cpp
@@ -16,10 +16,10 @@
 
 #include <sanitizer/msan_interface.h>
 
-#define CHECK_AND_PUSH(addr, size)                                \
-  if (addr) {                                                     \
-    assert(-1 == __msan_test_shadow(addr, sizeof(size)));         \
-    ranges.push_back(std::make_pair((void *)addr, (size_t)size)); \
+#define CHECK_AND_PUSH(addr, size)                                             \
+  if (addr) {                                                                  \
+    assert(-1 == __msan_test_shadow(addr, (size_t)(size)));                    \
+    ranges.push_back(std::make_pair((void *)addr, (size_t)size));              \
   }
 
 int main(int argc, char *argv[]) {
@@ -35,7 +35,7 @@ int main(int argc, char *argv[]) {
   assert(res == 0);
   assert(-1 == __msan_test_shadow(&ifas, sizeof(ifaddrs *)));
 
-  std::vector<std::pair<void *, size_t> > ranges;
+  std::vector<std::pair<void *, size_t>> ranges;
   ifaddrs *p = ifas;
   while (p) {
     CHECK_AND_PUSH(p, sizeof(ifaddrs));
diff --git a/external/llvm-project/compiler-rt/test/msan/qsort.cpp b/external/llvm-project/compiler-rt/test/msan/qsort.cpp
index af287ed64357..93e6845e1ea7 100644
--- a/external/llvm-project/compiler-rt/test/msan/qsort.cpp
+++ b/external/llvm-project/compiler-rt/test/msan/qsort.cpp
@@ -52,7 +52,7 @@ int compar1(const void *a, const void *b) {
   // kind of random
   for (int i = 0; i < kSize2; ++i)
     p[i] = i * 2 + (i % 3 - 1) * 3;
-  qsort(p, kSize1, sizeof(long), compar2);
+  qsort(p, kSize2, sizeof(long), compar2);
   __msan_check_mem_is_initialized(p, sizeof(long) * kSize2);
   delete[] p;
 
diff --git a/external/llvm-project/compiler-rt/test/rtsan/fork_exec.cpp b/external/llvm-project/compiler-rt/test/rtsan/fork_exec.cpp
index 3b2d2e5ca2f5..5890a0936a2f 100644
--- a/external/llvm-project/compiler-rt/test/rtsan/fork_exec.cpp
+++ b/external/llvm-project/compiler-rt/test/rtsan/fork_exec.cpp
@@ -45,7 +45,12 @@ int main() MAYBE_NONBLOCKING {
 }
 
 // CHECK-NOHALT: Intercepted call to {{.*}} `fork` {{.*}}
-// CHECK-NOHALT: Intercepted call to {{.*}} `execve` {{.*}}
+
+// We should also get some other intercepted call. On some systems this
+// is `execve`, on others, it's a lock to set up `execve`. In either
+// case, just check that we get a second intercepted call, don't sweat
+// the name.
+// CHECK-NOHALT: Intercepted call to {{.*}}
 
 // usleep checks that rtsan is still enabled in the parent process
 // See note in our interceptors file for why we don't look for `wait`
diff --git a/external/llvm-project/compiler-rt/test/tysan/CMakeLists.txt b/external/llvm-project/compiler-rt/test/tysan/CMakeLists.txt
index 76f57501e854..ce0afa8769f0 100644
--- a/external/llvm-project/compiler-rt/test/tysan/CMakeLists.txt
+++ b/external/llvm-project/compiler-rt/test/tysan/CMakeLists.txt
@@ -21,9 +21,7 @@ foreach(arch ${TYSAN_TEST_ARCH})
 endforeach()
 
 set(TYSAN_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
-if(NOT COMPILER_RT_STANDALONE_BUILD)
-  list(APPEND TYSAN_TEST_DEPS tysan)
-endif()
+list(APPEND TYSAN_TEST_DEPS tysan)
 
 add_lit_testsuite(check-tysan "Running the TypeSanitizer tests"
   ${TYSAN_TESTSUITES}
diff --git a/external/llvm-project/flang-rt/lib/cuda/descriptor.cpp b/external/llvm-project/flang-rt/lib/cuda/descriptor.cpp
index 7b768f91af29..aa75d4eff051 100644
--- a/external/llvm-project/flang-rt/lib/cuda/descriptor.cpp
+++ b/external/llvm-project/flang-rt/lib/cuda/descriptor.cpp
@@ -54,6 +54,14 @@ void RTDEF(CUFSyncGlobalDescriptor)(
   ((Descriptor *)devAddr, (Descriptor *)hostPtr, sourceFile, sourceLine);
 }
 
+void RTDEF(CUFDescriptorCheckSection)(
+    const Descriptor *desc, const char *sourceFile, int sourceLine) {
+  if (desc && !desc->IsContiguous()) {
+    Terminator terminator{sourceFile, sourceLine};
+    terminator.Crash("device array section argument is not contiguous");
+  }
+}
+
 RT_EXT_API_GROUP_END
 }
 } // namespace Fortran::runtime::cuda
diff --git a/external/llvm-project/flang-rt/lib/cuda/memory.cpp b/external/llvm-project/flang-rt/lib/cuda/memory.cpp
index 766f6847946c..e4e13fa825f3 100644
--- a/external/llvm-project/flang-rt/lib/cuda/memory.cpp
+++ b/external/llvm-project/flang-rt/lib/cuda/memory.cpp
@@ -105,6 +105,11 @@ void RTDECL(CUFDataTransferDescDesc)(Descriptor *dstDesc, Descriptor *srcDesc,
   } else {
     terminator.Crash("host to host copy not supported");
   }
+  // Allocate dst descriptor if not allocated.
+  if (!dstDesc->IsAllocated()) {
+    dstDesc->ApplyMold(*srcDesc, dstDesc->rank());
+    dstDesc->Allocate(/*asyncObject=*/nullptr);
+  }
   if ((srcDesc->rank() > 0) && (dstDesc->Elements() < srcDesc->Elements())) {
     // Special case when rhs is bigger than lhs and both are contiguous arrays.
     // In this case we do a simple ptr to ptr transfer with the size of lhs.
diff --git a/external/llvm-project/flang-rt/lib/runtime/assign.cpp b/external/llvm-project/flang-rt/lib/runtime/assign.cpp
index cf0ac8f71672..b5dc84b884b3 100644
--- a/external/llvm-project/flang-rt/lib/runtime/assign.cpp
+++ b/external/llvm-project/flang-rt/lib/runtime/assign.cpp
@@ -266,7 +266,8 @@ RT_API_ATTRS void Assign(Descriptor &to, const Descriptor &from,
   if (MayAlias(to, from)) {
     if (mustDeallocateLHS) {
       deferDeallocation = &deferredDeallocStatDesc.descriptor();
-      Fortran::runtime::memcpy(deferDeallocation, &to, to.SizeInBytes());
+      Fortran::runtime::memcpy(
+          reinterpret_cast<void *>(deferDeallocation), &to, to.SizeInBytes());
       to.set_base_addr(nullptr);
     } else if (!isSimpleMemmove()) {
       // Handle LHS/RHS aliasing by copying RHS into a temp, then
@@ -274,7 +275,7 @@ RT_API_ATTRS void Assign(Descriptor &to, const Descriptor &from,
       auto descBytes{from.SizeInBytes()};
       StaticDescriptor<maxRank, true, 16> staticDesc;
       Descriptor &newFrom{staticDesc.descriptor()};
-      Fortran::runtime::memcpy(&newFrom, &from, descBytes);
+      Fortran::runtime::memcpy(reinterpret_cast<void *>(&newFrom), &from, descBytes);
       // Pretend the temporary descriptor is for an ALLOCATABLE
       // entity, otherwise, the Deallocate() below will not
       // free the descriptor memory.
@@ -329,6 +330,7 @@ RT_API_ATTRS void Assign(Descriptor &to, const Descriptor &from,
       }
       flags &= ~NeedFinalization;
       toElementBytes = to.ElementBytes(); // may have changed
+      toDerived = toAddendum ? toAddendum->derivedType() : nullptr;
     }
   }
   if (toDerived && (flags & CanBeDefinedAssignment)) {
diff --git a/external/llvm-project/flang-rt/unittests/Runtime/CUDA/Memory.cpp b/external/llvm-project/flang-rt/unittests/Runtime/CUDA/Memory.cpp
index 7915baca6c20..f2e17870f799 100644
--- a/external/llvm-project/flang-rt/unittests/Runtime/CUDA/Memory.cpp
+++ b/external/llvm-project/flang-rt/unittests/Runtime/CUDA/Memory.cpp
@@ -70,3 +70,29 @@ TEST(MemoryCUFTest, CUFDataTransferDescDesc) {
     EXPECT_EQ(*host->ZeroBasedIndexedElement<std::int32_t>(i), (std::int32_t)i);
   }
 }
+
+TEST(MemoryCUFTest, CUFDataTransferDescDescDstNotAllocated) {
+  using Fortran::common::TypeCategory;
+  RTNAME(CUFRegisterAllocator)();
+  // INTEGER(4), DEVICE, ALLOCATABLE :: a(:)
+  auto dev{createAllocatable(TypeCategory::Integer, 4)};
+  dev->SetAllocIdx(kDeviceAllocatorPos);
+  EXPECT_EQ((int)kDeviceAllocatorPos, dev->GetAllocIdx());
+  EXPECT_FALSE(dev->IsAllocated());
+
+  // Create temp array to transfer to device.
+  auto x{MakeArray<TypeCategory::Integer, 4>(std::vector<int>{10},
+      std::vector<int32_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})};
+  RTNAME(CUFDataTransferDescDesc)
+  (dev.get(), x.get(), kHostToDevice, __FILE__, __LINE__);
+
+  // Retrieve data from device.
+  auto host{MakeArray<TypeCategory::Integer, 4>(std::vector<int>{10},
+      std::vector<int32_t>{0, 0, 0, 0, 0, 0, 0, 0, 0, 0})};
+  RTNAME(CUFDataTransferDescDesc)
+  (host.get(), dev.get(), kDeviceToHost, __FILE__, __LINE__);
+
+  for (unsigned i = 0; i < 10; ++i) {
+    EXPECT_EQ(*host->ZeroBasedIndexedElement<std::int32_t>(i), (std::int32_t)i);
+  }
+}
diff --git a/external/llvm-project/flang/Maintainers.md b/external/llvm-project/flang/Maintainers.md
index f4a763538913..b994c300e4e2 100644
--- a/external/llvm-project/flang/Maintainers.md
+++ b/external/llvm-project/flang/Maintainers.md
@@ -79,6 +79,13 @@ clementval@gmail.com (email), clementval (GitHub), clementval (Discourse)
 Abid Qadeer \
 haqadeer@amd.com (email), abidh (GitHub), abidh (Discourse)
 
+### Platform maintainers
+These maintainers are responsible for particular platforms that Flang supports
+
+#### Windows
+David Truby \
+david.truby@arm.com (email), davidtruby (GitHub), davidtruby (Discourse), truby (Discord)
+
 ## Inactive Maintainers
 ### Lead Maintainers
 #### Backend : (Lowering, FIR, Codegen)
diff --git a/external/llvm-project/flang/docs/CMakeLists.txt b/external/llvm-project/flang/docs/CMakeLists.txt
index 92feb059d4ca..a9f5811fcd06 100644
--- a/external/llvm-project/flang/docs/CMakeLists.txt
+++ b/external/llvm-project/flang/docs/CMakeLists.txt
@@ -82,7 +82,7 @@ if (LLVM_ENABLE_DOXYGEN)
 endif()
 endif()
 
-function (gen_rst_file_from_td output_file td_option source docs_target)
+function (gen_rst_file_from_td output_file td_option source)
   if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${source}")
     message(FATAL_ERROR "Cannot find source file: ${source} in ${CMAKE_CURRENT_SOURCE_DIR}")
   endif()
@@ -90,12 +90,8 @@ function (gen_rst_file_from_td output_file td_option source docs_target)
   list(APPEND LLVM_TABLEGEN_FLAGS "-I${TABLEGEN_INCLUDE_DIR}")
   list(APPEND LLVM_TABLEGEN_FLAGS "-I${CMAKE_CURRENT_SOURCE_DIR}/../../clang/include/clang/Driver/")
   clang_tablegen(Source/${output_file} ${td_option} SOURCE ${source} TARGET "gen-${output_file}")
-  add_dependencies(${docs_target} "gen-${output_file}")
-
   # clang_tablegen() does not create the output directory automatically,
-  # so we have to create it explicitly. Note that copy-flang-src-docs below
-  # does create the output directory, but it is not necessarily run
-  # before RST generation.
+  # so we have to create it explicitly. 
   add_custom_target(create-flang-rst-output-dir
     COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/Source
   )
@@ -103,34 +99,70 @@ function (gen_rst_file_from_td output_file td_option source docs_target)
 endfunction()
 
 if (LLVM_ENABLE_SPHINX)
+  set (FLANG_DOCS_HTML_DIR "${CMAKE_CURRENT_BINARY_DIR}/SourceHtml")
+  set (FLANG_DOCS_MAN_DIR "${CMAKE_CURRENT_BINARY_DIR}/SourceMan")
   include(AddSphinxTarget)
   if (SPHINX_FOUND)
-    if (${SPHINX_OUTPUT_HTML})
-      add_sphinx_target(html flang SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/Source")
 
-      add_dependencies(docs-flang-html copy-flang-src-docs)
+    # CLANG_TABLEGEN_EXE variable needs to be set for clang_tablegen to run without error
+    find_program(CLANG_TABLEGEN_EXE "clang-tblgen" ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
 
-      # Copy the flang/docs directory and the generated FIRLangRef.md file to a place in the binary directory.
-      # Having all the files in a single directory makes it possible for Sphinx to process them together.
-      # Add a dependency to the flang-doc target to ensure that the FIRLangRef.md file is generated before the copying happens.
-      add_custom_target(copy-flang-src-docs
-        COMMAND "${CMAKE_COMMAND}" -E copy_directory
-        "${CMAKE_CURRENT_SOURCE_DIR}"
-        "${CMAKE_CURRENT_BINARY_DIR}/Source"
-        DEPENDS flang-doc)
+    # Generate the RST file from TableGen (shared by both HTML and MAN builds)
+    gen_rst_file_from_td(FlangCommandLineReference.rst -gen-opt-docs FlangOptionsDocs.td)
 
-      # Runs a python script prior to HTML generation to prepend a header to FIRLangRef,
-      # Without the header, the page is incorrectly formatted, as it assumes the first entry is the page title.
-      add_custom_command(TARGET copy-flang-src-docs
+    if (${SPHINX_OUTPUT_HTML})
+      message(STATUS "Using index.md for html build")
+
+      # Copy the entire flang/docs directory to the build Source dir,
+      # then remove the index.rst file, to avoid clash with index.md 
+      # which is used for the HTML build.
+      add_custom_target(copy-flang-src-docs-html
+        COMMAND "${CMAKE_COMMAND}" -E copy_directory
+              "${CMAKE_CURRENT_SOURCE_DIR}"
+              "${FLANG_DOCS_HTML_DIR}"
+        COMMAND "${CMAKE_COMMAND}" -E remove
+          "${FLANG_DOCS_HTML_DIR}/CommandGuide/index.rst"
+        COMMAND "${CMAKE_COMMAND}" -E copy
+          "${CMAKE_CURRENT_BINARY_DIR}/Source/FlangCommandLineReference.rst"
+          "${FLANG_DOCS_HTML_DIR}/FlangCommandLineReference.rst"
+        DEPENDS flang-doc gen-FlangCommandLineReference.rst)
+
+      # Run Python preprocessing ONLY for HTML build
+      # This script prepends headers to FIRLangRef.md for proper formatting
+      add_custom_command(TARGET copy-flang-src-docs-html
         COMMAND "${Python3_EXECUTABLE}"
-        ARGS ${CMAKE_CURRENT_BINARY_DIR}/Source/FIR/CreateFIRLangRef.py)
+        ARGS "${FLANG_DOCS_HTML_DIR}/FIR/CreateFIRLangRef.py")
 
-      # CLANG_TABLEGEN_EXE variable needs to be set for clang_tablegen to run without error
-      find_program(CLANG_TABLEGEN_EXE "clang-tblgen" ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
-      gen_rst_file_from_td(FlangCommandLineReference.rst -gen-opt-docs FlangOptionsDocs.td docs-flang-html)
+      add_sphinx_target(html flang SOURCE_DIR "${FLANG_DOCS_HTML_DIR}")
+      add_dependencies(docs-flang-html copy-flang-src-docs-html)
     endif()
+
+    # ----------------------------
+    # MAN BUILD SETUP
+    # ----------------------------
     if (${SPHINX_OUTPUT_MAN})
-      add_sphinx_target(man flang)
+      message(STATUS "NOTE: The Flang man page is currently a placeholder with a TODO. See docs/CommandGuide/index.rst for details")
+
+      # Create minimal Source dir with ONLY the files needed for man build:
+      # - conf.py (Sphinx config)
+      # - index.rst (top-level man page)
+      # - FlangCommandLineReference.rst (generated reference)
+      add_custom_target(copy-flang-src-docs-man
+        COMMAND "${CMAKE_COMMAND}" -E make_directory
+                "${FLANG_DOCS_MAN_DIR}"
+        COMMAND "${CMAKE_COMMAND}" -E copy
+          "${CMAKE_CURRENT_SOURCE_DIR}/conf.py"
+          "${FLANG_DOCS_MAN_DIR}/conf.py"
+        COMMAND "${CMAKE_COMMAND}" -E copy
+          "${CMAKE_CURRENT_BINARY_DIR}/Source/FlangCommandLineReference.rst"
+          "${FLANG_DOCS_MAN_DIR}/FlangCommandLineReference.rst"
+        COMMAND "${CMAKE_COMMAND}" -E copy
+                "${CMAKE_CURRENT_SOURCE_DIR}/CommandGuide/index.rst"
+                "${FLANG_DOCS_MAN_DIR}/index.rst"
+        DEPENDS flang-doc gen-FlangCommandLineReference.rst)
+        
+      add_sphinx_target(man flang SOURCE_DIR "${FLANG_DOCS_MAN_DIR}")
+      add_dependencies(docs-flang-man copy-flang-src-docs-man)
     endif()
   endif()
 endif()
diff --git a/external/llvm-project/flang/docs/CommandGuide/index.rst b/external/llvm-project/flang/docs/CommandGuide/index.rst
new file mode 100644
index 000000000000..093c79fa185b
--- /dev/null
+++ b/external/llvm-project/flang/docs/CommandGuide/index.rst
@@ -0,0 +1,22 @@
+Flang Manual Page (In Progress)
+==================================================
+
+.. note::
+    This man page is under development.
+
+For full documentation, please see the online HTML docs:
+
+https://flang.llvm.org/docs/
+
+..  
+  The placeholder text "FlangCommandLineReference" below should eventually be replaced with the actual man page contents.  
+
+----
+
+Flang Command Line Reference
+----------------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   FlangCommandLineReference
diff --git a/external/llvm-project/flang/docs/Extensions.md b/external/llvm-project/flang/docs/Extensions.md
index 7d5e35a3991d..78d871c593e1 100644
--- a/external/llvm-project/flang/docs/Extensions.md
+++ b/external/llvm-project/flang/docs/Extensions.md
@@ -164,6 +164,12 @@ end
   No other Fortran compiler enforces C7108 (to our knowledge);
   they all resolve the ambiguity by interpreting the call as a function
   reference.  We do the same, with a portability warning.
+* An override for an inaccessible procedure binding works only within
+  the same module; other apparent overrides of inaccessible bindings
+  are actually new bindings of the same name.
+  In the case of `DEFERRED` bindings in an `ABSTRACT` derived type,
+  however, overrides are necessary, so they are permitted for inaccessible
+  bindings with an optional warning.
 
 ## Extensions, deletions, and legacy features supported by default
 
diff --git a/external/llvm-project/flang/docs/FIR/CreateFIRLangRef.py b/external/llvm-project/flang/docs/FIR/CreateFIRLangRef.py
index b6077364cdee..803cd9da3f12 100644
--- a/external/llvm-project/flang/docs/FIR/CreateFIRLangRef.py
+++ b/external/llvm-project/flang/docs/FIR/CreateFIRLangRef.py
@@ -4,9 +4,9 @@
 import os
 
 # These paths are relative to flang/docs in the build directory, not source, as that's where this tool is executed.
-HEADER_PATH = os.path.join("Source", "FIR", "FIRLangRef_Header.md")
+HEADER_PATH = os.path.join("SourceHtml", "FIR", "FIRLangRef_Header.md")
 DOCS_PATH = os.path.join("Dialect", "FIRLangRef.md")
-OUTPUT_PATH = os.path.join("Source", "FIRLangRef.md")
+OUTPUT_PATH = os.path.join("SourceHtml", "FIRLangRef.md")
 
 # 1. Writes line 1 from docs to output, (comment line that the file is autogenerated)
 # 2. Adds a new line
diff --git a/external/llvm-project/flang/docs/FlangDriver.md b/external/llvm-project/flang/docs/FlangDriver.md
index 97744f0bee06..f246163d3ce6 100644
--- a/external/llvm-project/flang/docs/FlangDriver.md
+++ b/external/llvm-project/flang/docs/FlangDriver.md
@@ -614,3 +614,31 @@ nvfortran defines `-fast` as
  - `-Mcache_align`: there is no equivalent flag in Flang or Clang.
  - `-Mflushz`: flush-to-zero mode - when `-ffast-math` is specified, Flang will
    link to `crtfastmath.o` to ensure denormal numbers are flushed to zero.
+
+
+## FCC_OVERRIDE_OPTIONS
+
+The environment variable `FCC_OVERRIDE_OPTIONS` can be used to edit flang's
+command line arguments. The value of this variable is a space-separated list of
+edits to perform. The edits are applied in the order in which they appear in
+`FCC_OVERRIDE_OPTIONS`. Each edit should be one of the following form:
+
+- `#`: Silence information about the changes to the command line arguments.
+
+- `^FOO`: Add `FOO` as a new argument at the beginning of the command line right
+  after the name of the compiler executable.
+
+- `+FOO`: Add `FOO` as a new argument at the end of the command line.
+
+- `s/XXX/YYY/`: Substitute the regular expression `XXX` with `YYY` in the
+  command line.
+
+- `xOPTION`: Removes all instances of the literal argument `OPTION`.
+
+- `XOPTION`: Removes all instances of the literal argument `OPTION`, and the
+  following argument.
+
+- `Ox`: Removes all flags matching `O` or `O[sz0-9]` and adds `Ox` at the end
+  of the command line.
+
+This environment variable does not affect the options added by the config files.
diff --git a/external/llvm-project/flang/docs/OpenMPSupport.md b/external/llvm-project/flang/docs/OpenMPSupport.md
index 7a4f95693a89..c9f19c37fd7f 100644
--- a/external/llvm-project/flang/docs/OpenMPSupport.md
+++ b/external/llvm-project/flang/docs/OpenMPSupport.md
@@ -60,3 +60,16 @@ Note : No distinction is made between the support in Parser/Semantics, MLIR, Low
 | target teams distribute parallel loop construct            | P      | device, reduction and dist_schedule clauses are not supported |
 | teams distribute parallel loop simd construct              | P      | reduction, dist_schedule, and linear clauses are not supported |
 | target teams distribute parallel loop simd construct       | P      | device, reduction, dist_schedule and linear clauses are not supported |
+
+## Extensions
+### ATOMIC construct
+The implementation of the ATOMIC construct follows OpenMP 6.0 with the following extensions:
+- `x = x` is an allowed form of ATOMIC UPDATE.
+This is motivated by the fact that the equivalent forms `x = x+0` or `x = x*1` are allowed.
+- Explicit type conversions are allowed in ATOMIC READ, WRITE or UPDATE constructs, and in the capture statement in ATOMIC UPDATE CAPTURE.
+The OpenMP spec requires intrinsic- or pointer-assignments, which include (as per the Fortran standard) implicit type conversions.  Since such conversions need to be handled, allowing explicit conversions comes at no extra cost.
+- A literal `.true.` or `.false.` is an allowed condition in ATOMIC UPDATE COMPARE. [1]
+- A logical variable is an allowed form of the condition even if its value is not computed within the ATOMIC UPDATE COMPARE construct [1].
+- `expr equalop x` is an allowed condition in ATOMIC UPDATE COMPARE. [1]
+
+[1] Code generation for ATOMIC UPDATE COMPARE is not implemented yet.
diff --git a/external/llvm-project/flang/docs/conf.py b/external/llvm-project/flang/docs/conf.py
index 48f7b69f5d75..a470d8afd718 100644
--- a/external/llvm-project/flang/docs/conf.py
+++ b/external/llvm-project/flang/docs/conf.py
@@ -28,20 +28,22 @@
     "sphinx.ext.autodoc",
 ]
 
-# When building man pages, we do not use the markdown pages,
-# So, we can continue without the myst_parser dependencies.
-# Doing so reduces dependencies of some packaged llvm distributions.
 try:
     import myst_parser
 
     extensions.append("myst_parser")
 except ImportError:
-    if not tags.has("builder-man"):
-        raise
+    raise ImportError(
+        "myst_parser is required to build documentation, including man pages."
+    )
 
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
+source_suffix = {
+    ".rst": "restructuredtext",
+    ".md": "markdown",
+}
 myst_heading_anchors = 6
 
 import sphinx
@@ -227,7 +229,15 @@
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = []
+man_pages = [
+    (
+        "index",
+        "flang",
+        "Flang Documentation (In Progress)",
+        ["Flang Contributors"],
+        1,
+    )
+]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
diff --git a/external/llvm-project/flang/examples/FeatureList/FeatureList.cpp b/external/llvm-project/flang/examples/FeatureList/FeatureList.cpp
index d1407cf0ef23..a36b8719e365 100644
--- a/external/llvm-project/flang/examples/FeatureList/FeatureList.cpp
+++ b/external/llvm-project/flang/examples/FeatureList/FeatureList.cpp
@@ -445,13 +445,6 @@ struct NodeVisitor {
   READ_FEATURE(ObjectDecl)
   READ_FEATURE(OldParameterStmt)
   READ_FEATURE(OmpAlignedClause)
-  READ_FEATURE(OmpAtomic)
-  READ_FEATURE(OmpAtomicCapture)
-  READ_FEATURE(OmpAtomicCapture::Stmt1)
-  READ_FEATURE(OmpAtomicCapture::Stmt2)
-  READ_FEATURE(OmpAtomicRead)
-  READ_FEATURE(OmpAtomicUpdate)
-  READ_FEATURE(OmpAtomicWrite)
   READ_FEATURE(OmpBeginBlockDirective)
   READ_FEATURE(OmpBeginLoopDirective)
   READ_FEATURE(OmpBeginSectionsDirective)
@@ -480,7 +473,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpIterationOffset)
   READ_FEATURE(OmpIterationVector)
   READ_FEATURE(OmpEndAllocators)
-  READ_FEATURE(OmpEndAtomic)
   READ_FEATURE(OmpEndBlockDirective)
   READ_FEATURE(OmpEndCriticalDirective)
   READ_FEATURE(OmpEndLoopDirective)
@@ -566,8 +558,6 @@ struct NodeVisitor {
   READ_FEATURE(OpenMPDeclareTargetConstruct)
   READ_FEATURE(OmpMemoryOrderType)
   READ_FEATURE(OmpMemoryOrderClause)
-  READ_FEATURE(OmpAtomicClause)
-  READ_FEATURE(OmpAtomicClauseList)
   READ_FEATURE(OmpAtomicDefaultMemOrderClause)
   READ_FEATURE(OpenMPFlushConstruct)
   READ_FEATURE(OpenMPLoopConstruct)
diff --git a/external/llvm-project/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp b/external/llvm-project/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
index bf66151d5995..feb7b4eced9e 100644
--- a/external/llvm-project/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
+++ b/external/llvm-project/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
@@ -74,25 +74,19 @@ SourcePosition OpenMPCounterVisitor::getLocation(const OpenMPConstruct &c) {
           // the directive field.
           [&](const auto &c) -> SourcePosition {
             const CharBlock &source{std::get<0>(c.t).source};
-            return (parsing->allCooked().GetSourcePositionRange(source))->first;
+            return parsing->allCooked().GetSourcePositionRange(source)->first;
           },
           [&](const OpenMPAtomicConstruct &c) -> SourcePosition {
-            return std::visit(
-                [&](const auto &o) -> SourcePosition {
-                  const CharBlock &source{std::get<Verbatim>(o.t).source};
-                  return parsing->allCooked()
-                      .GetSourcePositionRange(source)
-                      ->first;
-                },
-                c.u);
+            const CharBlock &source{c.source};
+            return parsing->allCooked().GetSourcePositionRange(source)->first;
           },
           [&](const OpenMPSectionConstruct &c) -> SourcePosition {
             const CharBlock &source{c.source};
-            return (parsing->allCooked().GetSourcePositionRange(source))->first;
+            return parsing->allCooked().GetSourcePositionRange(source)->first;
           },
           [&](const OpenMPUtilityConstruct &c) -> SourcePosition {
             const CharBlock &source{c.source};
-            return (parsing->allCooked().GetSourcePositionRange(source))->first;
+            return parsing->allCooked().GetSourcePositionRange(source)->first;
           },
       },
       c.u);
@@ -157,14 +151,9 @@ std::string OpenMPCounterVisitor::getName(const OpenMPConstruct &c) {
             return normalize_construct_name(source.ToString());
           },
           [&](const OpenMPAtomicConstruct &c) -> std::string {
-            return std::visit(
-                [&](const auto &c) {
-                  // Get source from the verbatim fields
-                  const CharBlock &source{std::get<Verbatim>(c.t).source};
-                  return "atomic-" +
-                      normalize_construct_name(source.ToString());
-                },
-                c.u);
+            auto &dirSpec = std::get<OmpDirectiveSpecification>(c.t);
+            auto &dirName = std::get<OmpDirectiveName>(dirSpec.t);
+            return normalize_construct_name(dirName.source.ToString());
           },
           [&](const OpenMPUtilityConstruct &c) -> std::string {
             const CharBlock &source{c.source};
diff --git a/external/llvm-project/flang/include/flang/Common/enum-class.h b/external/llvm-project/flang/include/flang/Common/enum-class.h
index 41575d45091a..9f9134ee3ff7 100644
--- a/external/llvm-project/flang/include/flang/Common/enum-class.h
+++ b/external/llvm-project/flang/include/flang/Common/enum-class.h
@@ -18,8 +18,8 @@
 #define FORTRAN_COMMON_ENUM_CLASS_H_
 
 #include <array>
-#include <string>
-
+#include <functional>
+#include <string_view>
 namespace Fortran::common {
 
 constexpr std::size_t CountEnumNames(const char *p) {
@@ -62,11 +62,18 @@ constexpr std::array<std::string_view, ITEMS> EnumNames(const char *p) {
   enum class NAME { __VA_ARGS__ }; \
   [[maybe_unused]] static constexpr std::size_t NAME##_enumSize{ \
       ::Fortran::common::CountEnumNames(#__VA_ARGS__)}; \
+  [[maybe_unused]] static inline std::size_t EnumToInt(NAME e) { \
+    return static_cast<std::size_t>(e); \
+  } \
   [[maybe_unused]] static inline std::string_view EnumToString(NAME e) { \
     static const constexpr auto names{ \
         ::Fortran::common::EnumNames<NAME##_enumSize>(#__VA_ARGS__)}; \
     return names[static_cast<std::size_t>(e)]; \
+  } \
+  [[maybe_unused]] inline void ForEach##NAME(std::function<void(NAME)> f) { \
+    for (std::size_t i{0}; i < NAME##_enumSize; ++i) { \
+      f(static_cast<NAME>(i)); \
+    } \
   }
-
 } // namespace Fortran::common
 #endif // FORTRAN_COMMON_ENUM_CLASS_H_
diff --git a/external/llvm-project/flang/include/flang/Common/optional.h b/external/llvm-project/flang/include/flang/Common/optional.h
index c7c81f40cc8c..72991111be57 100644
--- a/external/llvm-project/flang/include/flang/Common/optional.h
+++ b/external/llvm-project/flang/include/flang/Common/optional.h
@@ -28,7 +28,6 @@
 
 #include "api-attrs.h"
 #include <optional>
-#include <type_traits>
 
 #if !defined(STD_OPTIONAL_UNSUPPORTED) && \
     (defined(__CUDACC__) || defined(__CUDA__)) && defined(__CUDA_ARCH__)
diff --git a/external/llvm-project/flang/include/flang/Evaluate/tools.h b/external/llvm-project/flang/include/flang/Evaluate/tools.h
index 4dce1257a650..1959d5f3a589 100644
--- a/external/llvm-project/flang/include/flang/Evaluate/tools.h
+++ b/external/llvm-project/flang/include/flang/Evaluate/tools.h
@@ -490,26 +490,30 @@ template <typename A> std::optional<CoarrayRef> ExtractCoarrayRef(const A &x) {
   }
 }
 
-struct ExtractSubstringHelper {
-  template <typename T> static std::optional<Substring> visit(T &&) {
+template <typename TARGET> struct ExtractFromExprDesignatorHelper {
+  template <typename T> static std::optional<TARGET> visit(T &&) {
     return std::nullopt;
   }
 
-  static std::optional<Substring> visit(const Substring &e) { return e; }
+  static std::optional<TARGET> visit(const TARGET &t) { return t; }
 
   template <typename T>
-  static std::optional<Substring> visit(const Designator<T> &e) {
+  static std::optional<TARGET> visit(const Designator<T> &e) {
     return common::visit([](auto &&s) { return visit(s); }, e.u);
   }
 
-  template <typename T>
-  static std::optional<Substring> visit(const Expr<T> &e) {
+  template <typename T> static std::optional<TARGET> visit(const Expr<T> &e) {
     return common::visit([](auto &&s) { return visit(s); }, e.u);
   }
 };
 
 template <typename A> std::optional<Substring> ExtractSubstring(const A &x) {
-  return ExtractSubstringHelper::visit(x);
+  return ExtractFromExprDesignatorHelper<Substring>::visit(x);
+}
+
+template <typename A>
+std::optional<ComplexPart> ExtractComplexPart(const A &x) {
+  return ExtractFromExprDesignatorHelper<ComplexPart>::visit(x);
 }
 
 // If an expression is simply a whole symbol data designator,
diff --git a/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.def b/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.def
index 65dc5835df71..8204858d068d 100644
--- a/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.def
+++ b/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.def
@@ -24,8 +24,15 @@ CODEGENOPT(OptimizationLevel, 2, 0) ///< The -O[0-3] option specified.
 CODEGENOPT(DebugPassManager, 1, 0) ///< Prints debug information for the new
                                    ///< pass manager.
 
+
+/// Choose profile instrumenation kind or no instrumentation.
+ENUM_CODEGENOPT(ProfileInstr, llvm::driver::ProfileInstrKind, 2, llvm::driver::ProfileInstrKind::ProfileNone)
+/// Choose profile kind for PGO use compilation.
+ENUM_CODEGENOPT(ProfileUse, llvm::driver::ProfileInstrKind, 2, llvm::driver::ProfileInstrKind::ProfileNone)
+
 CODEGENOPT(InstrumentFunctions, 1, 0) ///< Set when -finstrument_functions is
                                       ///< enabled on the compile step.
+
 CODEGENOPT(IsPIE, 1, 0) ///< PIE level is the same as PIC Level.
 CODEGENOPT(PICLevel, 2, 0) ///< PIC level of the LLVM module.
 CODEGENOPT(PrepareForFullLTO , 1, 0) ///< Set when -flto is enabled on the
@@ -42,6 +49,8 @@ CODEGENOPT(AliasAnalysis, 1, 0) ///< Enable alias analysis pass
 
 CODEGENOPT(Underscoring, 1, 1)
 CODEGENOPT(OffloadGlobalFiltering, 1, 0)
+CODEGENOPT(DeferDescriptorMapping, 1, 0) ///< Fortran OpenMP specific optimisation for delaying descriptor mapping until target/target data regions
+
 ENUM_CODEGENOPT(RelocationModel, llvm::Reloc::Model, 3, llvm::Reloc::PIC_) ///< Name of the relocation model to use.
 ENUM_CODEGENOPT(DebugInfo,  llvm::codegenoptions::DebugInfoKind, 4,  llvm::codegenoptions::NoDebugInfo) ///< Level of debug info to generate
 ENUM_CODEGENOPT(VecLib, llvm::driver::VectorLibrary, 4, llvm::driver::VectorLibrary::NoLibrary) ///< Vector functions library to use
diff --git a/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.h b/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.h
index 61e56e51c4bb..bad17c8309eb 100644
--- a/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.h
+++ b/external/llvm-project/flang/include/flang/Frontend/CodeGenOptions.h
@@ -53,9 +53,12 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// The paths to the pass plugins that were registered using -fpass-plugin.
   std::vector<std::string> LLVMPassPlugins;
 
-  // The prefered vector width, if requested by -mprefer-vector-width.
+  /// The prefered vector width, if requested by -mprefer-vector-width.
   std::string PreferVectorWidth;
 
+  /// List of reciprocal estimate sub-options.
+  std::string Reciprocals;
+
   /// List of filenames passed in using the -fembed-offload-object option. These
   /// are offloading binaries containing device images and metadata.
   std::vector<std::string> OffloadObjects;
@@ -151,6 +154,44 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// OpenMP is enabled.
   using DoConcurrentMappingKind = flangomp::DoConcurrentMappingKind;
 
+  /// Name of the profile file to use as output for -fprofile-instr-generate,
+  /// -fprofile-generate, and -fcs-profile-generate.
+  std::string InstrProfileOutput;
+
+  /// Name of the profile file to use as input for -fmemory-profile-use.
+  std::string MemoryProfileUsePath;
+
+  /// Name of the profile file to use as input for -fprofile-instr-use
+  std::string ProfileInstrumentUsePath;
+
+  /// Name of the profile remapping file to apply to the profile data supplied
+  /// by -fprofile-sample-use or -fprofile-instr-use.
+  std::string ProfileRemappingFile;
+
+  /// Check if Clang profile instrumenation is on.
+  bool hasProfileClangInstr() const {
+    return getProfileInstr() == llvm::driver::ProfileClangInstr;
+  }
+
+  /// Check if IR level profile instrumentation is on.
+  bool hasProfileIRInstr() const {
+    return getProfileInstr() == llvm::driver::ProfileIRInstr;
+  }
+
+  /// Check if CS IR level profile instrumentation is on.
+  bool hasProfileCSIRInstr() const {
+    return getProfileInstr() == llvm::driver::ProfileCSIRInstr;
+  }
+  /// Check if IR level profile use is on.
+  bool hasProfileIRUse() const {
+    return getProfileUse() == llvm::driver::ProfileIRInstr ||
+           getProfileUse() == llvm::driver::ProfileCSIRInstr;
+  }
+  /// Check if CSIR profile use is on.
+  bool hasProfileCSIRUse() const {
+    return getProfileUse() == llvm::driver::ProfileCSIRInstr;
+  }
+
   // Define accessors/mutators for code generation options of enumeration type.
 #define CODEGENOPT(Name, Bits, Default)
 #define ENUM_CODEGENOPT(Name, Type, Bits, Default)                             \
diff --git a/external/llvm-project/flang/include/flang/Lower/LoweringOptions.def b/external/llvm-project/flang/include/flang/Lower/LoweringOptions.def
index b062ea1a805a..d97abf4d864b 100644
--- a/external/llvm-project/flang/include/flang/Lower/LoweringOptions.def
+++ b/external/llvm-project/flang/include/flang/Lower/LoweringOptions.def
@@ -63,5 +63,8 @@ ENUM_LOWERINGOPT(StackRepackArrays, unsigned, 1, 0)
 /// in the leading dimension.
 ENUM_LOWERINGOPT(RepackArraysWhole, unsigned, 1, 0)
 
+/// If true, CUDA Fortran runtime check is inserted.
+ENUM_LOWERINGOPT(CUDARuntimeCheck, unsigned, 1, 0)
+
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Builder/DirectivesCommon.h b/external/llvm-project/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
index f71a2ccd07bf..78c1bb359c80 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/Builder/DirectivesCommon.h
@@ -88,6 +88,16 @@ inline AddrAndBoundsInfo getDataOperandBaseAddr(fir::FirOpBuilder &builder,
 
     return AddrAndBoundsInfo(symAddr, rawInput, isPresent, boxTy);
   }
+  // For boxchar references, do the same as what is done above for box
+  // references - Load the boxchar so that it is easier to retrieve the length
+  // of the underlying character and the data pointer.
+  if (auto boxCharType = mlir::dyn_cast<fir::BoxCharType>(
+          fir::unwrapRefType((symAddr.getType())))) {
+    if (!isOptional && mlir::isa<fir::ReferenceType>(symAddr.getType())) {
+      mlir::Value boxChar = builder.create<fir::LoadOp>(loc, symAddr);
+      return AddrAndBoundsInfo(boxChar, rawInput, isPresent);
+    }
+  }
   return AddrAndBoundsInfo(symAddr, rawInput, isPresent);
 }
 
@@ -134,26 +144,64 @@ template <typename BoundsOp, typename BoundsType>
 mlir::Value
 genBoundsOpFromBoxChar(fir::FirOpBuilder &builder, mlir::Location loc,
                        fir::ExtendedValue dataExv, AddrAndBoundsInfo &info) {
-  // TODO: Handle info.isPresent.
-  if (auto boxCharType =
-          mlir::dyn_cast<fir::BoxCharType>(info.addr.getType())) {
-    mlir::Type idxTy = builder.getIndexType();
-    mlir::Type lenType = builder.getCharacterLengthType();
+
+  if (!mlir::isa<fir::BoxCharType>(fir::unwrapRefType(info.addr.getType())))
+    return mlir::Value{};
+
+  mlir::Type idxTy = builder.getIndexType();
+  mlir::Type lenType = builder.getCharacterLengthType();
+  mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
+  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
+  using ExtentAndStride = std::tuple<mlir::Value, mlir::Value>;
+  auto [extent, stride] = [&]() -> ExtentAndStride {
+    if (info.isPresent) {
+      llvm::SmallVector<mlir::Type> resTypes = {idxTy, idxTy};
+      mlir::Operation::result_range ifRes =
+          builder
+              .genIfOp(loc, resTypes, info.isPresent, /*withElseRegion=*/true)
+              .genThen([&]() {
+                mlir::Value boxChar =
+                    fir::isa_ref_type(info.addr.getType())
+                        ? builder.create<fir::LoadOp>(loc, info.addr)
+                        : info.addr;
+                fir::BoxCharType boxCharType =
+                    mlir::cast<fir::BoxCharType>(boxChar.getType());
+                mlir::Type refType = builder.getRefType(boxCharType.getEleTy());
+                auto unboxed = builder.create<fir::UnboxCharOp>(
+                    loc, refType, lenType, boxChar);
+                mlir::SmallVector<mlir::Value> results = {unboxed.getResult(1),
+                                                          one};
+                builder.create<fir::ResultOp>(loc, results);
+              })
+              .genElse([&]() {
+                mlir::SmallVector<mlir::Value> results = {zero, zero};
+                builder.create<fir::ResultOp>(loc, results);
+              })
+              .getResults();
+      return {ifRes[0], ifRes[1]};
+    }
+    // We have already established that info.addr.getType() is a boxchar
+    // or a boxchar address. If an address, load the boxchar.
+    mlir::Value boxChar = fir::isa_ref_type(info.addr.getType())
+                              ? builder.create<fir::LoadOp>(loc, info.addr)
+                              : info.addr;
+    fir::BoxCharType boxCharType =
+        mlir::cast<fir::BoxCharType>(boxChar.getType());
     mlir::Type refType = builder.getRefType(boxCharType.getEleTy());
     auto unboxed =
-        builder.create<fir::UnboxCharOp>(loc, refType, lenType, info.addr);
-    mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
-    mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-    mlir::Value extent = unboxed.getResult(1);
-    mlir::Value stride = one;
-    mlir::Value ub = builder.create<mlir::arith::SubIOp>(loc, extent, one);
-    mlir::Type boundTy = builder.getType<mlir::omp::MapBoundsType>();
-    return builder.create<mlir::omp::MapBoundsOp>(
-        loc, boundTy, /*lower_bound=*/zero,
-        /*upper_bound=*/ub, /*extent=*/extent, /*stride=*/stride,
-        /*stride_in_bytes=*/true, /*start_idx=*/zero);
-  }
-  return mlir::Value{};
+        builder.create<fir::UnboxCharOp>(loc, refType, lenType, boxChar);
+    return {unboxed.getResult(1), one};
+  }();
+
+  mlir::Value ub = builder.create<mlir::arith::SubIOp>(loc, extent, one);
+  mlir::Type boundTy = builder.getType<BoundsType>();
+  return builder.create<BoundsOp>(loc, boundTy,
+                                  /*lower_bound=*/zero,
+                                  /*upper_bound=*/ub,
+                                  /*extent=*/extent,
+                                  /*stride=*/stride,
+                                  /*stride_in_bytes=*/true,
+                                  /*start_idx=*/zero);
 }
 
 /// Generate the bounds operation from the descriptor information.
@@ -293,11 +341,11 @@ genImplicitBoundsOps(fir::FirOpBuilder &builder, AddrAndBoundsInfo &info,
     bounds = genBaseBoundsOps<BoundsOp, BoundsType>(builder, loc, dataExv,
                                                     dataExvIsAssumedSize);
   }
-  if (characterWithDynamicLen(fir::unwrapRefType(baseOp.getType()))) {
+  if (characterWithDynamicLen(fir::unwrapRefType(baseOp.getType())) ||
+      mlir::isa<fir::BoxCharType>(fir::unwrapRefType(info.addr.getType()))) {
     bounds = {genBoundsOpFromBoxChar<BoundsOp, BoundsType>(builder, loc,
                                                            dataExv, info)};
   }
-
   return bounds;
 }
 
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/external/llvm-project/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index dfe2876ab31d..9986f37d7d13 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -271,14 +271,14 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// Sample genDeclare callback for createArrayTemp() below.
   /// It creates fir.declare operation using the given operands.
   /// \p memref is the base of the allocated temporary,
-  /// which may be !fir.ref<!fir.array<>> or !fir.ref<!fir.box/class<>>.
+  /// which may be !fir.ref<!fir.array<>> or !fir.box/class<>.
   static mlir::Value genTempDeclareOp(fir::FirOpBuilder &builder,
                                       mlir::Location loc, mlir::Value memref,
                                       llvm::StringRef name, mlir::Value shape,
                                       llvm::ArrayRef<mlir::Value> typeParams,
                                       fir::FortranVariableFlagsAttr attrs);
 
-  /// Create a temporary array with the given \p arrayType,
+  /// Create a temporary with the given \p baseType,
   /// \p shape, \p extents and \p typeParams. An optional
   /// \p polymorphicMold specifies the entity which dynamic type
   /// has to be used for the allocation.
@@ -291,16 +291,26 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// If \p useStack is true, the function will try to do the allocation
   /// in stack memory (which is not always possible currently).
   /// The first return value is the base of the temporary object,
-  /// which may be !fir.ref<!fir.array<>> or !fir.ref<!fir.box/class<>>.
+  /// which may be !fir.ref<!fir.array<>> or !fir.box/class<>.
   /// The second return value is true, if the actual allocation
   /// was done in heap memory.
+  std::pair<mlir::Value, bool> createAndDeclareTemp(
+      mlir::Location loc, mlir::Type baseType, mlir::Value shape,
+      llvm::ArrayRef<mlir::Value> extents,
+      llvm::ArrayRef<mlir::Value> typeParams,
+      const std::function<decltype(genTempDeclareOp)> &genDeclare,
+      mlir::Value polymorphicMold, bool useStack, llvm::StringRef tmpName);
+  /// Create and declare an array temporary.
   std::pair<mlir::Value, bool>
   createArrayTemp(mlir::Location loc, fir::SequenceType arrayType,
                   mlir::Value shape, llvm::ArrayRef<mlir::Value> extents,
                   llvm::ArrayRef<mlir::Value> typeParams,
                   const std::function<decltype(genTempDeclareOp)> &genDeclare,
                   mlir::Value polymorphicMold, bool useStack = false,
-                  llvm::StringRef tmpName = ".tmp.array");
+                  llvm::StringRef tmpName = ".tmp.array") {
+    return createAndDeclareTemp(loc, arrayType, shape, extents, typeParams,
+                                genDeclare, polymorphicMold, useStack, tmpName);
+  }
 
   /// Create an LLVM stack save intrinsic op. Returns the saved stack pointer.
   /// The stack address space is fetched from the data layout of the current
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/external/llvm-project/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index ed00cec04dc3..2cbad6e268a3 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -374,12 +374,14 @@ struct LoopNest {
 /// loop constructs currently.
 LoopNest genLoopNest(mlir::Location loc, fir::FirOpBuilder &builder,
                      mlir::ValueRange extents, bool isUnordered = false,
-                     bool emitWorkshareLoop = false);
+                     bool emitWorkshareLoop = false,
+                     bool couldVectorize = true);
 inline LoopNest genLoopNest(mlir::Location loc, fir::FirOpBuilder &builder,
                             mlir::Value shape, bool isUnordered = false,
-                            bool emitWorkshareLoop = false) {
+                            bool emitWorkshareLoop = false,
+                            bool couldVectorize = true) {
   return genLoopNest(loc, builder, getIndexExtents(loc, builder, shape),
-                     isUnordered, emitWorkshareLoop);
+                     isUnordered, emitWorkshareLoop, couldVectorize);
 }
 
 /// The type of a callback that generates the body of a reduction
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Builder/MutableBox.h b/external/llvm-project/flang/include/flang/Optimizer/Builder/MutableBox.h
index 39657ddaf6e0..e1cb0f9852ba 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Builder/MutableBox.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/Builder/MutableBox.h
@@ -181,6 +181,18 @@ mlir::Value genIsNotAllocatedOrAssociatedTest(fir::FirOpBuilder &builder,
 mlir::Value genNullBoxStorage(fir::FirOpBuilder &builder, mlir::Location loc,
                               mlir::Type boxTy);
 
+/// Generate an unallocated box of the given \p boxTy with the
+/// bounds, type parameters, and dynamic type set according to the
+/// parameters.
+/// \p shape may be null for scalars, and \p polymorphicMold may be null for
+/// statically typed entities. This box can then be directly passed to the
+/// runtime for allocation.
+mlir::Value getAndEstablishBoxStorage(fir::FirOpBuilder &builder,
+                                      mlir::Location loc,
+                                      fir::BaseBoxType boxTy, mlir::Value shape,
+                                      llvm::ArrayRef<mlir::Value> typeParams,
+                                      mlir::Value polymorphicMold);
+
 } // namespace fir::factory
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_MUTABLEBOX_H
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h b/external/llvm-project/flang/include/flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h
index 14d262bf22a7..bdeb7574012c 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h
@@ -26,6 +26,11 @@ namespace fir::runtime::cuda {
 void genSyncGlobalDescriptor(fir::FirOpBuilder &builder, mlir::Location loc,
                              mlir::Value hostPtr);
 
+/// Generate runtime call to check the section of a descriptor and raise an
+/// error if it is not contiguous.
+void genDescriptorCheckSection(fir::FirOpBuilder &builder, mlir::Location loc,
+                               mlir::Value desc);
+
 } // namespace fir::runtime::cuda
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_CUDA_DESCRIPTOR_H_
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIROps.td b/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIROps.td
index f4b17ef7eed0..90e05ce3d5ca 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3240,11 +3240,17 @@ def fir_BoxOffsetOp : fir_Op<"box_offset", [NoMemoryEffect]> {
     descriptor implementation must have, only the base_addr and derived_type
     descriptor fields can be addressed.
 
+    It also accepts the address of a fir.boxchar and returns
+    address of the data pointer encapsulated by the fir.boxchar.
+
     ```
         %addr = fir.box_offset %box base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
         %tdesc = fir.box_offset %box derived_type : (!fir.ref<!fir.box<!fir.type<t>>>) -> !fir.llvm_ptr<!fir.tdesc<!fir.type<t>>>
 
+        %addr1 = fir.box_offset %boxchar base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<fir.char<1,?>>>
     ```
+
+    The derived_type field cannot be used when the input to this op is a reference to a fir.boxchar.
   }];
 
   let arguments = (ins
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIRType.h b/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIRType.h
index 52b14f15f89b..ecab12de55d6 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -47,6 +47,9 @@ class BaseBoxType : public mlir::Type {
   /// Returns the element type of this box type.
   mlir::Type getEleTy() const;
 
+  /// Get the raw address type of the memory described by the box.
+  mlir::Type getBaseAddressType() const;
+
   /// Unwrap element type from fir.heap, fir.ptr and fir.array.
   mlir::Type unwrapInnerType() const;
 
@@ -56,6 +59,12 @@ class BaseBoxType : public mlir::Type {
   /// Is this a box for a pointer?
   bool isPointer() const;
 
+  /// Does this box for a pointer or allocatable?
+  bool isPointerOrAllocatable() const;
+
+  /// Is this a box describing volatile memory?
+  bool isVolatile() const;
+
   /// Return the same type, except for the shape, that is taken the shape
   /// of shapeMold.
   BaseBoxType getBoxTypeWithNewShape(mlir::Type shapeMold) const;
diff --git a/external/llvm-project/flang/include/flang/Optimizer/HLFIR/Passes.td b/external/llvm-project/flang/include/flang/Optimizer/HLFIR/Passes.td
index d44514011807..04d7aec5fe48 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/HLFIR/Passes.td
+++ b/external/llvm-project/flang/include/flang/Optimizer/HLFIR/Passes.td
@@ -69,6 +69,10 @@ def InlineHLFIRAssign : Pass<"inline-hlfir-assign"> {
   let summary = "Inline hlfir.assign operations";
 }
 
+def InlineHLFIRCopyIn : Pass<"inline-hlfir-copy-in"> {
+  let summary = "Inline hlfir.copy_in operations";
+}
+
 def PropagateFortranVariableAttributes : Pass<"propagate-fortran-attrs"> {
   let summary = "Propagate FortranVariableFlagsAttr attributes through HLFIR";
 }
diff --git a/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.h b/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.h
index f9ba70188bf7..fc55efe71a5e 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.h
@@ -33,6 +33,8 @@ namespace flangomp {
 bool shouldUseWorkshareLowering(mlir::Operation *op);
 
 std::unique_ptr<mlir::Pass> createDoConcurrentConversionPass(bool mapToDevice);
+
+std::unique_ptr<mlir::Pass> createMapInfoFinalizationPass(bool deferDescMap);
 } // namespace flangomp
 
 #endif // FORTRAN_OPTIMIZER_OPENMP_PASSES_H
diff --git a/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.td b/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.td
index 8e75ea4386aa..2b6540cb18d2 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.td
+++ b/external/llvm-project/flang/include/flang/Optimizer/OpenMP/Passes.td
@@ -21,6 +21,12 @@ def MapInfoFinalizationPass
     explicit individual mapping by the OpenMP runtime.
   }];
   let dependentDialects = ["mlir::omp::OpenMPDialect"];
+
+  let options = [Option<"deferDescMapping", "opt-defer-desc-mapping",
+                      "bool", /*default=*/"false",
+                      "Activates or deactivates deferred descriptor mapping, "
+                      "which delays mapping of top-level descriptors to target "
+                      "regions and target data regions">];
 }
 
 def MapsForPrivatizedSymbolsPass
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Passes/Pipelines.h b/external/llvm-project/flang/include/flang/Optimizer/Passes/Pipelines.h
index db6fe2b15c3b..d747edcd87e2 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Passes/Pipelines.h
+++ b/external/llvm-project/flang/include/flang/Optimizer/Passes/Pipelines.h
@@ -134,6 +134,11 @@ struct OpenMPFIRPassPipelineOpts {
   bool isTargetDevice;
   bool enableOffloadGlobalFiltering;
 
+  /// Deactivates or activates MapInfoFinalization passes removal of
+  /// top-level descriptor mapping for non-Target Data/Target region
+  /// directives.
+  bool deferDescMap;
+
   /// Controls how to map `do concurrent` loops; to device, host, or none at
   /// all.
   Fortran::frontend::CodeGenOptions::DoConcurrentMappingKind
diff --git a/external/llvm-project/flang/include/flang/Optimizer/Transforms/Passes.td b/external/llvm-project/flang/include/flang/Optimizer/Transforms/Passes.td
index 1b1970412676..34842f978594 100644
--- a/external/llvm-project/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/external/llvm-project/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -429,6 +429,9 @@ def FunctionAttr : Pass<"function-attr", "mlir::func::FuncOp"> {
               "module.">,
        Option<"unsafeFPMath", "unsafe-fp-math", "bool", /*default=*/"false",
               "Set the unsafe-fp-math attribute on functions in the module.">,
+       Option<"reciprocals", "mrecip", "std::string", /*default=*/"",
+              "Set the reciprocal-estimates attribute on functions in the "
+              "module.">,
        Option<"preferVectorWidth", "prefer-vector-width", "std::string",
               /*default=*/"",
               "Set the prefer-vector-width attribute on functions in the "
diff --git a/external/llvm-project/flang/include/flang/Parser/dump-parse-tree.h b/external/llvm-project/flang/include/flang/Parser/dump-parse-tree.h
index df9278697346..c6a5150a85a4 100644
--- a/external/llvm-project/flang/include/flang/Parser/dump-parse-tree.h
+++ b/external/llvm-project/flang/include/flang/Parser/dump-parse-tree.h
@@ -532,15 +532,6 @@ class ParseTreeDumper {
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
   NODE_ENUM(OmpSeverityClause, Severity)
-  NODE(parser, OmpAtomic)
-  NODE(parser, OmpAtomicCapture)
-  NODE(OmpAtomicCapture, Stmt1)
-  NODE(OmpAtomicCapture, Stmt2)
-  NODE(parser, OmpAtomicCompare)
-  NODE(parser, OmpAtomicCompareIfStmt)
-  NODE(parser, OmpAtomicRead)
-  NODE(parser, OmpAtomicUpdate)
-  NODE(parser, OmpAtomicWrite)
   NODE(parser, OmpBeginBlockDirective)
   NODE(parser, OmpBeginLoopDirective)
   NODE(parser, OmpBeginSectionsDirective)
@@ -587,7 +578,6 @@ class ParseTreeDumper {
   NODE(parser, OmpDoacrossClause)
   NODE(parser, OmpDestroyClause)
   NODE(parser, OmpEndAllocators)
-  NODE(parser, OmpEndAtomic)
   NODE(parser, OmpEndBlockDirective)
   NODE(parser, OmpEndCriticalDirective)
   NODE(parser, OmpEndLoopDirective)
@@ -716,8 +706,6 @@ class ParseTreeDumper {
   NODE(parser, OpenMPDeclareMapperConstruct)
   NODE_ENUM(common, OmpMemoryOrderType)
   NODE(parser, OmpMemoryOrderClause)
-  NODE(parser, OmpAtomicClause)
-  NODE(parser, OmpAtomicClauseList)
   NODE(parser, OmpAtomicDefaultMemOrderClause)
   NODE(parser, OpenMPDepobjConstruct)
   NODE(parser, OpenMPUtilityConstruct)
diff --git a/external/llvm-project/flang/include/flang/Parser/parse-tree.h b/external/llvm-project/flang/include/flang/Parser/parse-tree.h
index c99006f0c1c2..67405f88e09f 100644
--- a/external/llvm-project/flang/include/flang/Parser/parse-tree.h
+++ b/external/llvm-project/flang/include/flang/Parser/parse-tree.h
@@ -4857,94 +4857,37 @@ struct OmpMemoryOrderClause {
   CharBlock source;
 };
 
-// 2.17.7 Atomic construct
-//        atomic-clause -> memory-order-clause | HINT(hint-expression) |
-//        FAIL(memory-order)
-struct OmpAtomicClause {
-  UNION_CLASS_BOILERPLATE(OmpAtomicClause);
-  CharBlock source;
-  std::variant<OmpMemoryOrderClause, OmpFailClause, OmpHintClause> u;
-};
-
-// atomic-clause-list -> [atomic-clause, [atomic-clause], ...]
-struct OmpAtomicClauseList {
-  WRAPPER_CLASS_BOILERPLATE(OmpAtomicClauseList, std::list<OmpAtomicClause>);
-  CharBlock source;
-};
-
-// END ATOMIC
-EMPTY_CLASS(OmpEndAtomic);
-
-// ATOMIC READ
-struct OmpAtomicRead {
-  TUPLE_CLASS_BOILERPLATE(OmpAtomicRead);
-  CharBlock source;
-  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
-      Statement<AssignmentStmt>, std::optional<OmpEndAtomic>>
-      t;
-};
-
-// ATOMIC WRITE
-struct OmpAtomicWrite {
-  TUPLE_CLASS_BOILERPLATE(OmpAtomicWrite);
-  CharBlock source;
-  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
-      Statement<AssignmentStmt>, std::optional<OmpEndAtomic>>
-      t;
-};
-
-// ATOMIC UPDATE
-struct OmpAtomicUpdate {
-  TUPLE_CLASS_BOILERPLATE(OmpAtomicUpdate);
-  CharBlock source;
-  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
-      Statement<AssignmentStmt>, std::optional<OmpEndAtomic>>
-      t;
-};
-
-// ATOMIC CAPTURE
-struct OmpAtomicCapture {
-  TUPLE_CLASS_BOILERPLATE(OmpAtomicCapture);
-  CharBlock source;
-  WRAPPER_CLASS(Stmt1, Statement<AssignmentStmt>);
-  WRAPPER_CLASS(Stmt2, Statement<AssignmentStmt>);
-  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList, Stmt1, Stmt2,
-      OmpEndAtomic>
-      t;
-};
-
-struct OmpAtomicCompareIfStmt {
-  UNION_CLASS_BOILERPLATE(OmpAtomicCompareIfStmt);
-  std::variant<common::Indirection<IfStmt>, common::Indirection<IfConstruct>> u;
-};
-
-// ATOMIC COMPARE (OpenMP 5.1, OPenMP 5.2 spec: 15.8.4)
-struct OmpAtomicCompare {
-  TUPLE_CLASS_BOILERPLATE(OmpAtomicCompare);
+struct OpenMPAtomicConstruct {
+  llvm::omp::Clause GetKind() const;
+  bool IsCapture() const;
+  bool IsCompare() const;
+  TUPLE_CLASS_BOILERPLATE(OpenMPAtomicConstruct);
   CharBlock source;
-  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
-      OmpAtomicCompareIfStmt, std::optional<OmpEndAtomic>>
+  std::tuple<OmpDirectiveSpecification, Block,
+      std::optional<OmpDirectiveSpecification>>
       t;
-};
 
-// ATOMIC
-struct OmpAtomic {
-  TUPLE_CLASS_BOILERPLATE(OmpAtomic);
-  CharBlock source;
-  std::tuple<Verbatim, OmpAtomicClauseList, Statement<AssignmentStmt>,
-      std::optional<OmpEndAtomic>>
-      t;
-};
+  // Information filled out during semantic checks to avoid duplication
+  // of analyses.
+  struct Analysis {
+    static constexpr int None = 0;
+    static constexpr int Read = 1;
+    static constexpr int Write = 2;
+    static constexpr int Update = Read | Write;
+    static constexpr int Action = 3; // Bitmask for None, Read, Write, Update
+    static constexpr int IfTrue = 4;
+    static constexpr int IfFalse = 8;
+    static constexpr int Condition = 12; // Bitmask for IfTrue, IfFalse
+
+    struct Op {
+      int what;
+      AssignmentStmt::TypedAssignment assign;
+    };
+    TypedExpr atom, cond;
+    Op op0, op1;
+  };
 
-// 2.17.7 atomic ->
-//        ATOMIC [atomic-clause-list] atomic-construct [atomic-clause-list] |
-//        ATOMIC [atomic-clause-list]
-//        atomic-construct -> READ | WRITE | UPDATE | CAPTURE | COMPARE
-struct OpenMPAtomicConstruct {
-  UNION_CLASS_BOILERPLATE(OpenMPAtomicConstruct);
-  std::variant<OmpAtomicRead, OmpAtomicWrite, OmpAtomicCapture, OmpAtomicUpdate,
-      OmpAtomicCompare, OmpAtomic>
-      u;
+  mutable Analysis analysis;
 };
 
 // OpenMP directives that associate with loop(s)
diff --git a/external/llvm-project/flang/include/flang/Parser/tools.h b/external/llvm-project/flang/include/flang/Parser/tools.h
index f1ead11734fa..447bccd5d35a 100644
--- a/external/llvm-project/flang/include/flang/Parser/tools.h
+++ b/external/llvm-project/flang/include/flang/Parser/tools.h
@@ -250,5 +250,8 @@ template <typename A> std::optional<CharBlock> GetLastSource(A &x) {
   return GetSourceHelper<false>::GetSource(const_cast<const A &>(x));
 }
 
+// Checks whether the assignment statement has a single variable on the RHS.
+bool CheckForSingleVariableOnRHS(const AssignmentStmt &);
+
 } // namespace Fortran::parser
 #endif // FORTRAN_PARSER_TOOLS_H_
diff --git a/external/llvm-project/flang/include/flang/Runtime/CUDA/descriptor.h b/external/llvm-project/flang/include/flang/Runtime/CUDA/descriptor.h
index 0ee7feca10e4..06e4a4649db1 100644
--- a/external/llvm-project/flang/include/flang/Runtime/CUDA/descriptor.h
+++ b/external/llvm-project/flang/include/flang/Runtime/CUDA/descriptor.h
@@ -37,6 +37,10 @@ void RTDECL(CUFDescriptorSync)(Descriptor *dst, const Descriptor *src,
 void RTDECL(CUFSyncGlobalDescriptor)(
     void *hostPtr, const char *sourceFile = nullptr, int sourceLine = 0);
 
+/// Check descriptor passed to a kernel.
+void RTDECL(CUFDescriptorCheckSection)(
+    const Descriptor *, const char *sourceFile = nullptr, int sourceLine = 0);
+
 } // extern "C"
 
 } // namespace Fortran::runtime::cuda
diff --git a/external/llvm-project/flang/include/flang/Semantics/openmp-dsa.h b/external/llvm-project/flang/include/flang/Semantics/openmp-dsa.h
new file mode 100644
index 000000000000..4b94a679f29e
--- /dev/null
+++ b/external/llvm-project/flang/include/flang/Semantics/openmp-dsa.h
@@ -0,0 +1,20 @@
+//===-- include/flang/Semantics/openmp-dsa.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_OPENMP_DSA_H_
+#define FORTRAN_SEMANTICS_OPENMP_DSA_H_
+
+#include "flang/Semantics/symbol.h"
+
+namespace Fortran::semantics {
+
+Symbol::Flags GetSymbolDSA(const Symbol &symbol);
+
+} // namespace Fortran::semantics
+
+#endif // FORTRAN_SEMANTICS_OPENMP_DSA_H_
diff --git a/external/llvm-project/flang/include/flang/Semantics/symbol.h b/external/llvm-project/flang/include/flang/Semantics/symbol.h
index 4cded64d170c..b319e2c7e5e7 100644
--- a/external/llvm-project/flang/include/flang/Semantics/symbol.h
+++ b/external/llvm-project/flang/include/flang/Semantics/symbol.h
@@ -30,6 +30,8 @@ class raw_ostream;
 }
 namespace Fortran::parser {
 struct Expr;
+struct OpenMPDeclareReductionConstruct;
+struct OmpMetadirectiveDirective;
 }
 
 namespace Fortran::semantics {
@@ -728,6 +730,40 @@ class GenericDetails {
 };
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const GenericDetails &);
 
+// Used for OpenMP DECLARE REDUCTION, it holds the information
+// needed to resolve which declaration (there could be multiple
+// with the same name) to use for a given type.
+class UserReductionDetails {
+public:
+  using TypeVector = std::vector<const DeclTypeSpec *>;
+  using DeclInfo = std::variant<const parser::OpenMPDeclareReductionConstruct *,
+      const parser::OmpMetadirectiveDirective *>;
+  using DeclVector = std::vector<DeclInfo>;
+
+  UserReductionDetails() = default;
+
+  void AddType(const DeclTypeSpec &type) { typeList_.push_back(&type); }
+  const TypeVector &GetTypeList() const { return typeList_; }
+
+  bool SupportsType(const DeclTypeSpec &type) const {
+    // We have to compare the actual type, not the pointer, as some
+    // types are not guaranteed to be the same object.
+    for (auto t : typeList_) {
+      if (*t == type) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  void AddDecl(const DeclInfo &decl) { declList_.emplace_back(decl); }
+  const DeclVector &GetDeclList() const { return declList_; }
+
+private:
+  TypeVector typeList_;
+  DeclVector declList_;
+};
+
 class UnknownDetails {};
 
 using Details = std::variant<UnknownDetails, MainProgramDetails, ModuleDetails,
@@ -735,7 +771,7 @@ using Details = std::variant<UnknownDetails, MainProgramDetails, ModuleDetails,
     ObjectEntityDetails, ProcEntityDetails, AssocEntityDetails,
     DerivedTypeDetails, UseDetails, UseErrorDetails, HostAssocDetails,
     GenericDetails, ProcBindingDetails, NamelistDetails, CommonBlockDetails,
-    TypeParamDetails, MiscDetails>;
+    TypeParamDetails, MiscDetails, UserReductionDetails>;
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const Details &);
 std::string DetailsToString(const Details &);
 
@@ -756,6 +792,7 @@ class Symbol {
       LocalityShared, // named in SHARED locality-spec
       InDataStmt, // initialized in a DATA statement, =>object, or /init/
       InNamelist, // in a Namelist group
+      InCommonBlock, // referenced in a common block
       EntryDummyArgument,
       CompilerCreated, // A compiler created symbol
       // For compiler created symbols that are constant but cannot legally have
@@ -785,8 +822,9 @@ class Symbol {
       OmpAllocate, OmpDeclarativeAllocateDirective,
       OmpExecutableAllocateDirective, OmpDeclareSimd, OmpDeclareTarget,
       OmpThreadprivate, OmpDeclareReduction, OmpFlushed, OmpCriticalLock,
-      OmpIfSpecified, OmpNone, OmpPreDetermined, OmpImplicit, OmpDependObject,
-      OmpInclusiveScan, OmpExclusiveScan, OmpInScanReduction);
+      OmpIfSpecified, OmpNone, OmpPreDetermined, OmpExplicit, OmpImplicit,
+      OmpDependObject, OmpInclusiveScan, OmpExclusiveScan, OmpInScanReduction,
+      OmpUniform);
   using Flags = common::EnumSet<Flag, Flag_enumSize>;
 
   const Scope &owner() const { return *owner_; }
diff --git a/external/llvm-project/flang/include/flang/Semantics/tools.h b/external/llvm-project/flang/include/flang/Semantics/tools.h
index 3839bc1d2a21..b13370512e5c 100644
--- a/external/llvm-project/flang/include/flang/Semantics/tools.h
+++ b/external/llvm-project/flang/include/flang/Semantics/tools.h
@@ -753,29 +753,154 @@ std::string GetCommonBlockObjectName(const Symbol &, bool underscoring);
 // Check for ambiguous USE associations
 bool HadUseError(SemanticsContext &, SourceName at, const Symbol *);
 
-/// Checks if the assignment statement has a single variable on the RHS.
-inline bool checkForSingleVariableOnRHS(
-    const Fortran::parser::AssignmentStmt &assignmentStmt) {
-  const Fortran::parser::Expr &expr{
-      std::get<Fortran::parser::Expr>(assignmentStmt.t)};
-  const Fortran::common::Indirection<Fortran::parser::Designator> *designator =
-      std::get_if<Fortran::common::Indirection<Fortran::parser::Designator>>(
-          &expr.u);
-  return designator != nullptr;
-}
-
-/// Checks if the symbol on the LHS is present in the RHS expression.
-inline bool checkForSymbolMatch(const Fortran::semantics::SomeExpr *lhs,
-    const Fortran::semantics::SomeExpr *rhs) {
-  auto lhsSyms{Fortran::evaluate::GetSymbolVector(*lhs)};
-  const Fortran::semantics::Symbol &lhsSymbol{*lhsSyms.front()};
-  for (const Fortran::semantics::Symbol &symbol :
-      Fortran::evaluate::GetSymbolVector(*rhs)) {
-    if (lhsSymbol == symbol) {
-      return true;
-    }
+// Checks whether the symbol on the LHS is present in the RHS expression.
+bool CheckForSymbolMatch(const SomeExpr *lhs, const SomeExpr *rhs);
+
+namespace operation {
+
+enum class Operator {
+  Unknown,
+  Add,
+  And,
+  Associated,
+  Call,
+  Constant,
+  Convert,
+  Div,
+  Eq,
+  Eqv,
+  False,
+  Ge,
+  Gt,
+  Identity,
+  Intrinsic,
+  Le,
+  Lt,
+  Max,
+  Min,
+  Mul,
+  Ne,
+  Neqv,
+  Not,
+  Or,
+  Pow,
+  Resize, // Convert within the same TypeCategory
+  Sub,
+  True,
+};
+
+std::string ToString(Operator op);
+
+template <typename... Ts, int Kind>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::LogicalOperation<Kind>, Ts...> &op) {
+  switch (op.derived().logicalOperator) {
+  case common::LogicalOperator::And:
+    return Operator::And;
+  case common::LogicalOperator::Or:
+    return Operator::Or;
+  case common::LogicalOperator::Eqv:
+    return Operator::Eqv;
+  case common::LogicalOperator::Neqv:
+    return Operator::Neqv;
+  case common::LogicalOperator::Not:
+    return Operator::Not;
   }
-  return false;
+  return Operator::Unknown;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Relational<T>, Ts...> &op) {
+  switch (op.derived().opr) {
+  case common::RelationalOperator::LT:
+    return Operator::Lt;
+  case common::RelationalOperator::LE:
+    return Operator::Le;
+  case common::RelationalOperator::EQ:
+    return Operator::Eq;
+  case common::RelationalOperator::NE:
+    return Operator::Ne;
+  case common::RelationalOperator::GE:
+    return Operator::Ge;
+  case common::RelationalOperator::GT:
+    return Operator::Gt;
+  }
+  return Operator::Unknown;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(const evaluate::Operation<evaluate::Add<T>, Ts...> &op) {
+  return Operator::Add;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Subtract<T>, Ts...> &op) {
+  return Operator::Sub;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Multiply<T>, Ts...> &op) {
+  return Operator::Mul;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Divide<T>, Ts...> &op) {
+  return Operator::Div;
 }
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Power<T>, Ts...> &op) {
+  return Operator::Pow;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::RealToIntPower<T>, Ts...> &op) {
+  return Operator::Pow;
+}
+
+template <typename T, common::TypeCategory C, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Convert<T, C>, Ts...> &op) {
+  if constexpr (C == T::category) {
+    return Operator::Resize;
+  } else {
+    return Operator::Convert;
+  }
+}
+
+template <typename T> //
+Operator OperationCode(const evaluate::Constant<T> &x) {
+  return Operator::Constant;
+}
+
+template <typename T> //
+Operator OperationCode(const T &) {
+  return Operator::Unknown;
+}
+
+Operator OperationCode(const evaluate::ProcedureDesignator &proc);
+
+} // namespace operation
+
+/// Return information about the top-level operation (ignoring parentheses):
+/// the operation code and the list of arguments.
+std::pair<operation::Operator, std::vector<SomeExpr>> GetTopLevelOperation(
+    const SomeExpr &expr);
+
+/// Check if expr is same as x, or a sequence of Convert operations on x.
+bool IsSameOrConvertOf(const SomeExpr &expr, const SomeExpr &x);
+
+/// Strip away any top-level Convert operations (if any exist) and return
+/// the input value. A ComplexConstructor(x, 0) is also considered as a
+/// convert operation.
+/// If the input is not Operation, Designator, FunctionRef or Constant,
+/// it returns std::nullopt.
+MaybeExpr GetConvertInput(const SomeExpr &x);
 } // namespace Fortran::semantics
 #endif // FORTRAN_SEMANTICS_TOOLS_H_
diff --git a/external/llvm-project/flang/include/flang/Support/Fortran-features.h b/external/llvm-project/flang/include/flang/Support/Fortran-features.h
index ac2b1825d316..5798297761f1 100644
--- a/external/llvm-project/flang/include/flang/Support/Fortran-features.h
+++ b/external/llvm-project/flang/include/flang/Support/Fortran-features.h
@@ -11,8 +11,7 @@
 
 #include "Fortran.h"
 #include "flang/Common/enum-set.h"
-#include "flang/Common/idioms.h"
-#include <optional>
+#include <string_view>
 #include <vector>
 
 namespace Fortran::common {
@@ -54,9 +53,10 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     PolymorphicActualAllocatableOrPointerToMonomorphicDummy, RelaxedPureDummy,
     UndefinableAsynchronousOrVolatileActual, AutomaticInMainProgram, PrintCptr,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
-    IgnoreIrrelevantAttributes, Unsigned, ContiguousOkForSeqAssociation, AmbiguousStructureConstructor,
-    // Maybe not a real language feature, but let's keep it here for now
-    ForwardRefExplicitTypeDummy, AmdMemoryAllocator)
+    IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
+    ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
+    InaccessibleDeferredOverride,
+    AmdMemoryAllocator, CudaWarpMatchFunction)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
@@ -83,9 +83,6 @@ ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
 using LanguageFeatures = EnumSet<LanguageFeature, LanguageFeature_enumSize>;
 using UsageWarnings = EnumSet<UsageWarning, UsageWarning_enumSize>;
 
-std::optional<LanguageFeature> FindLanguageFeature(const char *);
-std::optional<UsageWarning> FindUsageWarning(const char *);
-
 class LanguageFeatureControl {
 public:
   LanguageFeatureControl();
@@ -98,8 +95,10 @@ class LanguageFeatureControl {
   void EnableWarning(UsageWarning w, bool yes = true) {
     warnUsage_.set(w, yes);
   }
-  void WarnOnAllNonstandard(bool yes = true) { warnAllLanguage_ = yes; }
-  void WarnOnAllUsage(bool yes = true) { warnAllUsage_ = yes; }
+  void WarnOnAllNonstandard(bool yes = true);
+  bool IsWarnOnAllNonstandard() const { return warnAllLanguage_; }
+  void WarnOnAllUsage(bool yes = true);
+  bool IsWarnOnAllUsage() const { return warnAllUsage_; }
   void DisableAllNonstandardWarnings() {
     warnAllLanguage_ = false;
     warnLanguage_.clear();
@@ -108,26 +107,59 @@ class LanguageFeatureControl {
     warnAllUsage_ = false;
     warnUsage_.clear();
   }
-
+  void DisableAllWarnings() {
+    disableAllWarnings_ = true;
+    DisableAllNonstandardWarnings();
+    DisableAllUsageWarnings();
+  }
+  bool AreWarningsDisabled() const { return disableAllWarnings_; }
   bool IsEnabled(LanguageFeature f) const { return !disable_.test(f); }
-  bool ShouldWarn(LanguageFeature f) const {
-    return (warnAllLanguage_ && f != LanguageFeature::OpenMP &&
-               f != LanguageFeature::OpenACC && f != LanguageFeature::CUDA) ||
-        warnLanguage_.test(f);
+  bool ShouldWarn(LanguageFeature f) const { return warnLanguage_.test(f); }
+  bool ShouldWarn(UsageWarning w) const { return warnUsage_.test(w); }
+  // Cli options
+  // Take a string from the Cli and apply it to the LanguageFeatureControl.
+  // Return true if the option was recognized (and hence applied).
+  bool ApplyCliOption(std::string input);
+  // The add and replace functions are not currently used but are provided
+  // to allow a flexible many-to-one mapping from Cli spellings to enum values.
+  // Taking a string by value because the functions own this string after the
+  // call.
+  void AddAlternativeCliSpelling(LanguageFeature f, std::string input) {
+    cliOptions_.insert({input, {f}});
   }
-  bool ShouldWarn(UsageWarning w) const {
-    return warnAllUsage_ || warnUsage_.test(w);
+  void AddAlternativeCliSpelling(UsageWarning w, std::string input) {
+    cliOptions_.insert({input, {w}});
   }
+  void ReplaceCliCanonicalSpelling(LanguageFeature f, std::string input);
+  void ReplaceCliCanonicalSpelling(UsageWarning w, std::string input);
+  std::string_view getDefaultCliSpelling(LanguageFeature f) const {
+    return languageFeatureCliCanonicalSpelling_[EnumToInt(f)];
+  };
+  std::string_view getDefaultCliSpelling(UsageWarning w) const {
+    return usageWarningCliCanonicalSpelling_[EnumToInt(w)];
+  };
   // Return all spellings of operators names, depending on features enabled
   std::vector<const char *> GetNames(LogicalOperator) const;
   std::vector<const char *> GetNames(RelationalOperator) const;
 
 private:
+  // Map from Cli syntax of language features and usage warnings to their enum
+  // values.
+  std::unordered_map<std::string, std::variant<LanguageFeature, UsageWarning>>
+      cliOptions_;
+  // These two arrays map the enum values to their cannonical Cli spellings.
+  // Since each of the CanonicalSpelling is a string in the domain of the map
+  // above we just use a view of the string instead of another copy.
+  std::array<std::string_view, LanguageFeature_enumSize>
+      languageFeatureCliCanonicalSpelling_;
+  std::array<std::string_view, UsageWarning_enumSize>
+      usageWarningCliCanonicalSpelling_;
   LanguageFeatures disable_;
   LanguageFeatures warnLanguage_;
   bool warnAllLanguage_{false};
   UsageWarnings warnUsage_;
   bool warnAllUsage_{false};
+  bool disableAllWarnings_{false};
 };
 } // namespace Fortran::common
 #endif // FORTRAN_SUPPORT_FORTRAN_FEATURES_H_
diff --git a/external/llvm-project/flang/include/flang/Tools/CrossToolHelpers.h b/external/llvm-project/flang/include/flang/Tools/CrossToolHelpers.h
index 0f0107e4f34b..a387b9e1a888 100644
--- a/external/llvm-project/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/external/llvm-project/flang/include/flang/Tools/CrossToolHelpers.h
@@ -102,6 +102,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
     UnsafeFPMath = mathOpts.getAssociativeMath() &&
         mathOpts.getReciprocalMath() && NoSignedZerosFPMath &&
         ApproxFuncFPMath && mathOpts.getFPContractEnabled();
+    Reciprocals = opts.Reciprocals;
     PreferVectorWidth = opts.PreferVectorWidth;
     if (opts.InstrumentFunctions) {
       InstrumentFunctionEntry = "__cyg_profile_func_enter";
@@ -127,6 +128,8 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   bool NoSignedZerosFPMath =
       false; ///< Set no-signed-zeros-fp-math attribute for functions.
   bool UnsafeFPMath = false; ///< Set unsafe-fp-math attribute for functions.
+  std::string Reciprocals = ""; ///< Set reciprocal-estimate attribute for
+                                ///< functions.
   std::string PreferVectorWidth = ""; ///< Set prefer-vector-width attribute for
                                       ///< functions.
   bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
diff --git a/external/llvm-project/flang/lib/Evaluate/tools.cpp b/external/llvm-project/flang/lib/Evaluate/tools.cpp
index c70915cfa615..222c32a9c332 100644
--- a/external/llvm-project/flang/lib/Evaluate/tools.cpp
+++ b/external/llvm-project/flang/lib/Evaluate/tools.cpp
@@ -1206,7 +1206,8 @@ parser::Message *AttachDeclaration(
   }
   if (const auto *binding{
           unhosted->detailsIf<semantics::ProcBindingDetails>()}) {
-    if (binding->symbol().name() != symbol.name()) {
+    if (!symbol.attrs().test(semantics::Attr::DEFERRED) &&
+        binding->symbol().name() != symbol.name()) {
       message.Attach(binding->symbol().name(),
           "Procedure '%s' of type '%s' is bound to '%s'"_en_US, symbol.name(),
           symbol.owner().GetName().value(), binding->symbol().name());
diff --git a/external/llvm-project/flang/lib/Frontend/CompilerInvocation.cpp b/external/llvm-project/flang/lib/Frontend/CompilerInvocation.cpp
index 7e0b338ed1f6..9f9a4e48f709 100644
--- a/external/llvm-project/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/external/llvm-project/flang/lib/Frontend/CompilerInvocation.cpp
@@ -20,22 +20,21 @@
 #include "flang/Support/Version.h"
 #include "flang/Tools/TargetSetup.h"
 #include "flang/Version.inc"
-#include "clang/Basic/AllDiagnostics.h"
 #include "clang/Basic/DiagnosticDriver.h"
 #include "clang/Basic/DiagnosticOptions.h"
+#include "clang/Driver/CommonArgs.h"
 #include "clang/Driver/Driver.h"
-#include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Driver/OptionUtils.h"
 #include "clang/Driver/Options.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Frontend/Debug/Options.h"
+#include "llvm/Frontend/Driver/CodeGenOptions.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/FileSystem.h"
-#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/raw_ostream.h"
@@ -309,19 +308,10 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   for (auto *a : args.filtered(clang::driver::options::OPT_fpass_plugin_EQ))
     opts.LLVMPassPlugins.push_back(a->getValue());
 
-  // -mprefer_vector_width option
-  if (const llvm::opt::Arg *a = args.getLastArg(
-          clang::driver::options::OPT_mprefer_vector_width_EQ)) {
-    llvm::StringRef s = a->getValue();
-    unsigned width;
-    if (s == "none")
-      opts.PreferVectorWidth = "none";
-    else if (s.getAsInteger(10, width))
-      diags.Report(clang::diag::err_drv_invalid_value)
-          << a->getAsString(args) << a->getValue();
-    else
-      opts.PreferVectorWidth = s.str();
-  }
+  opts.Reciprocals = clang::driver::tools::parseMRecipOption(diags, args);
+
+  opts.PreferVectorWidth =
+      clang::driver::tools::parseMPreferVectorWidthOption(diags, args);
 
   // -fembed-offload-object option
   for (auto *a :
@@ -452,6 +442,15 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
       opts.IsPIE = 1;
   }
 
+  if (args.hasArg(clang::driver::options::OPT_fprofile_generate)) {
+    opts.setProfileInstr(llvm::driver::ProfileInstrKind::ProfileIRInstr);
+  }
+
+  if (auto A = args.getLastArg(clang::driver::options::OPT_fprofile_use_EQ)) {
+    opts.setProfileUse(llvm::driver::ProfileInstrKind::ProfileIRInstr);
+    opts.ProfileInstrumentUsePath = A->getValue();
+  }
+
   // -mcmodel option.
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_mcmodel_EQ)) {
@@ -488,6 +487,10 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   }
 
   parseDoConcurrentMapping(opts, args, diags);
+
+  opts.DeferDescriptorMapping =
+      args.hasFlag(clang::driver::options::OPT_fdefer_desc_map,
+                   clang::driver::options::OPT_fno_defer_desc_map, false);
 }
 
 /// Parses all target input arguments and populates the target
@@ -997,10 +1000,23 @@ static bool parseSemaArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
 /// Parses all diagnostics related arguments and populates the variables
 /// options accordingly. Returns false if new errors are generated.
+/// FC1 driver entry point for parsing diagnostic arguments.
 static bool parseDiagArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
                           clang::DiagnosticsEngine &diags) {
   unsigned numErrorsBefore = diags.getNumErrors();
 
+  auto &features{res.getFrontendOpts().features};
+  // The order of these flags (-pedantic -W<feature> -w) is important and is
+  // chosen to match clang's behavior.
+
+  // -pedantic
+  if (args.hasArg(clang::driver::options::OPT_pedantic)) {
+    features.WarnOnAllNonstandard();
+    features.WarnOnAllUsage();
+    res.setEnableConformanceChecks();
+    res.setEnableUsageChecks();
+  }
+
   // -Werror option
   // TODO: Currently throws a Diagnostic for anything other than -W<error>,
   // this has to change when other -W<opt>'s are supported.
@@ -1010,22 +1026,26 @@ static bool parseDiagArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
     for (const auto &wArg : wArgs) {
       if (wArg == "error") {
         res.setWarnAsErr(true);
-      } else {
-        const unsigned diagID =
-            diags.getCustomDiagID(clang::DiagnosticsEngine::Error,
-                                  "Only `-Werror` is supported currently.");
-        diags.Report(diagID);
+        // -W(no-)<feature>
+      } else if (!features.ApplyCliOption(wArg)) {
+        const unsigned diagID = diags.getCustomDiagID(
+            clang::DiagnosticsEngine::Error, "Unknown diagnostic option: -W%0");
+        diags.Report(diagID) << wArg;
       }
     }
   }
 
-  // Default to off for `flang -fc1`.
-  res.getFrontendOpts().showColors =
-      parseShowColorsArgs(args, /*defaultDiagColor=*/false);
-
-  // Honor color diagnostics.
-  res.getDiagnosticOpts().ShowColors = res.getFrontendOpts().showColors;
+  // -w
+  if (args.hasArg(clang::driver::options::OPT_w)) {
+    features.DisableAllWarnings();
+    res.setDisableWarnings();
+  }
 
+  // Default to off for `flang -fc1`.
+  bool showColors{parseShowColorsArgs(args, false)};
+  diags.getDiagnosticOptions().ShowColors = showColors;
+  res.getDiagnosticOpts().ShowColors = showColors;
+  res.getFrontendOpts().showColors = showColors;
   return diags.getNumErrors() == numErrorsBefore;
 }
 
@@ -1100,16 +1120,6 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
         Fortran::common::LanguageFeature::OpenACC);
   }
 
-  // -pedantic
-  if (args.hasArg(clang::driver::options::OPT_pedantic)) {
-    res.setEnableConformanceChecks();
-    res.setEnableUsageChecks();
-  }
-
-  // -w
-  if (args.hasArg(clang::driver::options::OPT_w))
-    res.setDisableWarnings();
-
   // -std=f2018
   // TODO: Set proper options when more fortran standards
   // are supported.
@@ -1118,6 +1128,7 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
     // We only allow f2018 as the given standard
     if (standard == "f2018") {
       res.setEnableConformanceChecks();
+      res.getFrontendOpts().features.WarnOnAllNonstandard();
     } else {
       const unsigned diagID =
           diags.getCustomDiagID(clang::DiagnosticsEngine::Error,
@@ -1669,6 +1680,10 @@ void CompilerInvocation::setDefaultPredefinitions() {
       fortranOptions.predefinitions.emplace_back("__64BIT__", "1");
     }
     break;
+  case llvm::Triple::ArchType::aarch64:
+    fortranOptions.predefinitions.emplace_back("__aarch64__", "1");
+    fortranOptions.predefinitions.emplace_back("__aarch64", "1");
+    break;
   }
 }
 
@@ -1721,16 +1736,7 @@ void CompilerInvocation::setFortranOpts() {
   if (frontendOptions.needProvenanceRangeToCharBlockMappings)
     fortranOptions.needProvenanceRangeToCharBlockMappings = true;
 
-  if (getEnableConformanceChecks())
-    fortranOptions.features.WarnOnAllNonstandard();
-
-  if (getEnableUsageChecks())
-    fortranOptions.features.WarnOnAllUsage();
-
-  if (getDisableWarnings()) {
-    fortranOptions.features.DisableAllNonstandardWarnings();
-    fortranOptions.features.DisableAllUsageWarnings();
-  }
+  fortranOptions.features = frontendOptions.features;
 }
 
 std::unique_ptr<Fortran::semantics::SemanticsContext>
diff --git a/external/llvm-project/flang/lib/Frontend/FrontendActions.cpp b/external/llvm-project/flang/lib/Frontend/FrontendActions.cpp
index e86cc9fc9cdb..d7371d0d9948 100644
--- a/external/llvm-project/flang/lib/Frontend/FrontendActions.cpp
+++ b/external/llvm-project/flang/lib/Frontend/FrontendActions.cpp
@@ -56,10 +56,12 @@
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/PGOOptions.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/ToolOutputFile.h"
@@ -67,6 +69,7 @@
 #include "llvm/TargetParser/RISCVISAInfo.h"
 #include "llvm/TargetParser/RISCVTargetParser.h"
 #include "llvm/Transforms/IPO/Internalize.h"
+#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 #include <memory>
 #include <system_error>
@@ -307,6 +310,8 @@ bool CodeGenAction::beginSourceFileAction() {
       ci.getInvocation().getCodeGenOpts().getDoConcurrentMapping();
   opts.enableOffloadGlobalFiltering =
       ci.getInvocation().getCodeGenOpts().OffloadGlobalFiltering;
+  opts.deferDescMap =
+      ci.getInvocation().getCodeGenOpts().DeferDescriptorMapping;
 
   if (opts.doConcurrentMappingKind != DoConcurrentMappingKind::DCMK_None &&
       !isOpenMPEnabled) {
@@ -743,6 +748,7 @@ void CodeGenAction::generateLLVMIR() {
     config.VScaleMax = vsr->second;
   }
 
+  config.Reciprocals = opts.Reciprocals;
   config.PreferVectorWidth = opts.PreferVectorWidth;
 
   if (ci.getInvocation().getFrontendOpts().features.IsEnabled(
@@ -920,6 +926,29 @@ void CodeGenAction::runOptimizationPipeline(llvm::raw_pwrite_stream &os) {
   llvm::PassInstrumentationCallbacks pic;
   llvm::PipelineTuningOptions pto;
   std::optional<llvm::PGOOptions> pgoOpt;
+
+  if (opts.hasProfileIRInstr()) {
+    // -fprofile-generate.
+    pgoOpt = llvm::PGOOptions(opts.InstrProfileOutput.empty()
+                                  ? llvm::driver::getDefaultProfileGenName()
+                                  : opts.InstrProfileOutput,
+                              "", "", opts.MemoryProfileUsePath, nullptr,
+                              llvm::PGOOptions::IRInstr,
+                              llvm::PGOOptions::NoCSAction,
+                              llvm::PGOOptions::ColdFuncOpt::Default, false,
+                              /*PseudoProbeForProfiling=*/false, false);
+  } else if (opts.hasProfileIRUse()) {
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
+        llvm::vfs::getRealFileSystem();
+    // -fprofile-use.
+    auto CSAction = opts.hasProfileCSIRUse() ? llvm::PGOOptions::CSIRUse
+                                             : llvm::PGOOptions::NoCSAction;
+    pgoOpt = llvm::PGOOptions(
+        opts.ProfileInstrumentUsePath, "", opts.ProfileRemappingFile,
+        opts.MemoryProfileUsePath, VFS, llvm::PGOOptions::IRUse, CSAction,
+        llvm::PGOOptions::ColdFuncOpt::Default, false);
+  }
+
   llvm::StandardInstrumentations si(llvmModule->getContext(),
                                     opts.DebugPassManager);
   si.registerCallbacks(pic, &mam);
diff --git a/external/llvm-project/flang/lib/Lower/Allocatable.cpp b/external/llvm-project/flang/lib/Lower/Allocatable.cpp
index c05124c7c9c6..ecc1aca56635 100644
--- a/external/llvm-project/flang/lib/Lower/Allocatable.cpp
+++ b/external/llvm-project/flang/lib/Lower/Allocatable.cpp
@@ -37,6 +37,7 @@
 #include "flang/Runtime/pointer.h"
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "llvm/Support/CommandLine.h"
 
 /// By default fir memory operation fir::AllocMemOp/fir::FreeMemOp are used.
@@ -174,11 +175,34 @@ static void genRuntimeInitCharacter(fir::FirOpBuilder &builder,
   builder.create<fir::CallOp>(loc, callee, convertedArgs);
 }
 
+/// Check if region is nested in omp.target or
+/// region nested in function with declare target
+bool isRegionNestedInOmpTarget(mlir::Region &region) {
+  mlir::Operation *parentOp = region.getParentOp();
+  while (parentOp) {
+    if (auto declareTargetOp =
+            llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(parentOp)) {
+      if (declareTargetOp.isDeclareTarget())
+        return true;
+    }
+    if (llvm::isa<mlir::omp::TargetOp>(parentOp))
+      return true;
+    mlir::Region *parentRegion = parentOp->getParentRegion();
+    if (!parentRegion)
+      break;
+    parentOp = parentRegion->getParentOp();
+  }
+
+  return false;
+}
+
 /// Generate a runtime call to set allocator idx of descriptor for target amd.
 static void genAMDRuntimeDescriptorSetAllocIdx(fir::FirOpBuilder &builder,
                                                mlir::Location loc,
                                                const fir::MutableBoxValue &box,
                                                int allocatorId) {
+  if (isRegionNestedInOmpTarget(builder.getRegion()))
+    return;
   auto *context = builder.getContext();
   mlir::Type descriptorTy = box.getAddr().getType();
   mlir::IntegerType posTy = builder.getI32Type();
diff --git a/external/llvm-project/flang/lib/Lower/Bridge.cpp b/external/llvm-project/flang/lib/Lower/Bridge.cpp
index 92ab0f3b9489..2f2845ee683a 100644
--- a/external/llvm-project/flang/lib/Lower/Bridge.cpp
+++ b/external/llvm-project/flang/lib/Lower/Bridge.cpp
@@ -59,6 +59,7 @@
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Runtime/iostat-consts.h"
+#include "flang/Semantics/openmp-dsa.h"
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
@@ -1391,7 +1392,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     if (isUnordered || sym.has<Fortran::semantics::HostAssocDetails>() ||
         sym.has<Fortran::semantics::UseDetails>()) {
       if (!shallowLookupSymbol(sym) &&
-          !sym.test(Fortran::semantics::Symbol::Flag::OmpShared)) {
+          !GetSymbolDSA(sym).test(
+              Fortran::semantics::Symbol::Flag::OmpShared)) {
         // Do concurrent loop variables are not mapped yet since they are local
         // to the Do concurrent scope (same for OpenMP loops).
         mlir::OpBuilder::InsertPoint insPt = builder->saveInsertionPoint();
diff --git a/external/llvm-project/flang/lib/Lower/ConvertCall.cpp b/external/llvm-project/flang/lib/Lower/ConvertCall.cpp
index 7378118cfef7..864499e6c343 100644
--- a/external/llvm-project/flang/lib/Lower/ConvertCall.cpp
+++ b/external/llvm-project/flang/lib/Lower/ConvertCall.cpp
@@ -26,6 +26,7 @@
 #include "flang/Optimizer/Builder/IntrinsicCall.h"
 #include "flang/Optimizer/Builder/LowLevelIntrinsics.h"
 #include "flang/Optimizer/Builder/MutableBox.h"
+#include "flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h"
 #include "flang/Optimizer/Builder/Runtime/Derived.h"
 #include "flang/Optimizer/Builder/Todo.h"
 #include "flang/Optimizer/Dialect/CUF/CUFOps.h"
@@ -543,6 +544,19 @@ Fortran::lower::genCallOpAndResult(
   fir::FortranProcedureFlagsEnumAttr procAttrs =
       caller.getProcedureAttrs(builder.getContext());
 
+  if (converter.getLoweringOptions().getCUDARuntimeCheck()) {
+    if (caller.getCallDescription().chevrons().empty()) {
+      for (auto [oper, arg] :
+           llvm::zip(operands, caller.getPassedArguments())) {
+        if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(oper.getType())) {
+          const Fortran::semantics::Symbol *sym = caller.getDummySymbol(arg);
+          if (sym && Fortran::evaluate::IsCUDADeviceSymbol(*sym))
+            fir::runtime::cuda::genDescriptorCheckSection(builder, loc, oper);
+        }
+      }
+    }
+  }
+
   if (!caller.getCallDescription().chevrons().empty()) {
     // A call to a CUDA kernel with the chevron syntax.
 
diff --git a/external/llvm-project/flang/lib/Lower/ConvertType.cpp b/external/llvm-project/flang/lib/Lower/ConvertType.cpp
index d45f9e7c0bf1..7a2e8e509518 100644
--- a/external/llvm-project/flang/lib/Lower/ConvertType.cpp
+++ b/external/llvm-project/flang/lib/Lower/ConvertType.cpp
@@ -276,19 +276,23 @@ struct TypeBuilderImpl {
     } else {
       fir::emitFatalError(loc, "symbol must have a type");
     }
-    bool isPolymorphic = (Fortran::semantics::IsPolymorphic(symbol) ||
-                          Fortran::semantics::IsUnlimitedPolymorphic(symbol)) &&
-                         !Fortran::semantics::IsAssumedType(symbol);
-    if (ultimate.IsObjectArray()) {
-      auto shapeExpr =
-          Fortran::evaluate::GetShape(converter.getFoldingContext(), ultimate);
+
+    auto shapeExpr =
+        Fortran::evaluate::GetShape(converter.getFoldingContext(), ultimate);
+
+    if (shapeExpr && !shapeExpr->empty()) {
+      // Statically ranked array.
       fir::SequenceType::Shape shape;
-      // If there is no shapExpr, this is an assumed-rank, and the empty shape
-      // will build the desired fir.array<*:T> type.
-      if (shapeExpr)
-        translateShape(shape, std::move(*shapeExpr));
+      translateShape(shape, std::move(*shapeExpr));
       ty = fir::SequenceType::get(shape, ty);
+    } else if (!shapeExpr) {
+      // Assumed-rank.
+      ty = fir::SequenceType::get(fir::SequenceType::Shape{}, ty);
     }
+
+    bool isPolymorphic = (Fortran::semantics::IsPolymorphic(symbol) ||
+                          Fortran::semantics::IsUnlimitedPolymorphic(symbol)) &&
+                         !Fortran::semantics::IsAssumedType(symbol);
     if (Fortran::semantics::IsPointer(symbol))
       return fir::wrapInClassOrBoxType(fir::PointerType::get(ty),
                                        isPolymorphic);
diff --git a/external/llvm-project/flang/lib/Lower/OpenACC.cpp b/external/llvm-project/flang/lib/Lower/OpenACC.cpp
index 02dba22c29c7..69e9c53baa74 100644
--- a/external/llvm-project/flang/lib/Lower/OpenACC.cpp
+++ b/external/llvm-project/flang/lib/Lower/OpenACC.cpp
@@ -653,8 +653,8 @@ void genAtomicCapture(Fortran::lower::AbstractConverter &converter,
   firOpBuilder.createBlock(&(atomicCaptureOp->getRegion(0)));
   mlir::Block &block = atomicCaptureOp->getRegion(0).back();
   firOpBuilder.setInsertionPointToStart(&block);
-  if (Fortran::semantics::checkForSingleVariableOnRHS(stmt1)) {
-    if (Fortran::semantics::checkForSymbolMatch(
+  if (Fortran::parser::CheckForSingleVariableOnRHS(stmt1)) {
+    if (Fortran::semantics::CheckForSymbolMatch(
             Fortran::semantics::GetExpr(stmt2Var),
             Fortran::semantics::GetExpr(stmt2Expr))) {
       // Atomic capture construct is of the form [capture-stmt, update-stmt]
@@ -2150,6 +2150,70 @@ privatizeIv(Fortran::lower::AbstractConverter &converter,
   ivPrivate.push_back(privateValue);
 }
 
+static void determineDefaultLoopParMode(
+    Fortran::lower::AbstractConverter &converter, mlir::acc::LoopOp &loopOp,
+    llvm::SmallVector<mlir::Attribute> &seqDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &independentDeviceTypes,
+    llvm::SmallVector<mlir::Attribute> &autoDeviceTypes) {
+  auto hasDeviceNone = [](mlir::Attribute attr) -> bool {
+    return mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(attr).getValue() ==
+           mlir::acc::DeviceType::None;
+  };
+  bool hasDefaultSeq = llvm::any_of(seqDeviceTypes, hasDeviceNone);
+  bool hasDefaultIndependent =
+      llvm::any_of(independentDeviceTypes, hasDeviceNone);
+  bool hasDefaultAuto = llvm::any_of(autoDeviceTypes, hasDeviceNone);
+  if (hasDefaultSeq || hasDefaultIndependent || hasDefaultAuto)
+    return; // Default loop par mode is already specified.
+
+  mlir::Region *currentRegion =
+      converter.getFirOpBuilder().getBlock()->getParent();
+  mlir::Operation *parentOp = mlir::acc::getEnclosingComputeOp(*currentRegion);
+  const bool isOrphanedLoop = !parentOp;
+  if (isOrphanedLoop ||
+      mlir::isa_and_present<mlir::acc::ParallelOp>(parentOp)) {
+    // As per OpenACC 3.3 standard section 2.9.6 independent clause:
+    // A loop construct with no auto or seq clause is treated as if it has the
+    // independent clause when it is an orphaned loop construct or its parent
+    // compute construct is a parallel construct.
+    independentDeviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+        converter.getFirOpBuilder().getContext(), mlir::acc::DeviceType::None));
+  } else if (mlir::isa_and_present<mlir::acc::SerialOp>(parentOp)) {
+    // Serial construct implies `seq` clause on loop. However, this
+    // conflicts with parallelism assignment if already set. Therefore check
+    // that first.
+    bool hasDefaultGangWorkerOrVector =
+        loopOp.hasVector() || loopOp.getVectorValue() || loopOp.hasWorker() ||
+        loopOp.getWorkerValue() || loopOp.hasGang() ||
+        loopOp.getGangValue(mlir::acc::GangArgType::Num) ||
+        loopOp.getGangValue(mlir::acc::GangArgType::Dim) ||
+        loopOp.getGangValue(mlir::acc::GangArgType::Static);
+    if (!hasDefaultGangWorkerOrVector)
+      seqDeviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+          converter.getFirOpBuilder().getContext(),
+          mlir::acc::DeviceType::None));
+    // Since the loop has some parallelism assigned - we cannot assign `seq`.
+    // However, the `acc.loop` verifier will check that one of seq, independent,
+    // or auto is marked. Seems reasonable to mark as auto since the OpenACC
+    // spec does say "If not, or if it is unable to make a determination, it
+    // must treat the auto clause as if it is a seq clause, and it must
+    // ignore any gang, worker, or vector clauses on the loop construct"
+    else
+      autoDeviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+          converter.getFirOpBuilder().getContext(),
+          mlir::acc::DeviceType::None));
+  } else {
+    // As per OpenACC 3.3 standard section 2.9.7 auto clause:
+    // When the parent compute construct is a kernels construct, a loop
+    // construct with no independent or seq clause is treated as if it has the
+    // auto clause.
+    assert(mlir::isa_and_present<mlir::acc::KernelsOp>(parentOp) &&
+           "Expected kernels construct");
+    autoDeviceTypes.push_back(mlir::acc::DeviceTypeAttr::get(
+        converter.getFirOpBuilder().getContext(), mlir::acc::DeviceType::None));
+  }
+}
+
 static mlir::acc::LoopOp createLoopOp(
     Fortran::lower::AbstractConverter &converter,
     mlir::Location currentLocation,
@@ -2482,6 +2546,9 @@ static mlir::acc::LoopOp createLoopOp(
     loopOp.setTileOperandsSegmentsAttr(
         builder.getDenseI32ArrayAttr(tileOperandsSegments));
 
+  // Determine the loop's default par mode - either seq, independent, or auto.
+  determineDefaultLoopParMode(converter, loopOp, seqDeviceTypes,
+                              independentDeviceTypes, autoDeviceTypes);
   if (!seqDeviceTypes.empty())
     loopOp.setSeqAttr(builder.getArrayAttr(seqDeviceTypes));
   if (!independentDeviceTypes.empty())
diff --git a/external/llvm-project/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/external/llvm-project/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 28fe57f80b1b..47d9c9083664 100644
--- a/external/llvm-project/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/external/llvm-project/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -927,14 +927,10 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
     for (const omp::Object &object : objects) {
       assert(object.ref() && "Expecting designator");
       mlir::Value dependVar;
+      SomeExpr expr = *object.ref();
 
-      if (evaluate::ExtractSubstring(*object.ref())) {
-        TODO(converter.getCurrentLocation(),
-             "substring not supported for task depend");
-      } else if (evaluate::IsArrayElement(*object.ref())) {
-        // Array Section
-        SomeExpr expr = *object.ref();
-
+      if (evaluate::IsArrayElement(expr) || evaluate::ExtractSubstring(expr)) {
+        // Array Section or character (sub)string
         if (isVectorSubscript(expr)) {
           // OpenMP needs the address of the first indexed element (required by
           // the standard to be the lowest index) to identify the dependency. We
@@ -948,7 +944,8 @@ bool ClauseProcessor::processDepend(lower::SymMap &symMap,
               converter.getCurrentLocation(), converter, expr, symMap, stmtCtx);
           dependVar = entity.getBase();
         }
-      } else if (evaluate::isStructureComponent(*object.ref())) {
+      } else if (evaluate::isStructureComponent(expr) ||
+                 evaluate::ExtractComplexPart(expr)) {
         SomeExpr expr = *object.ref();
         hlfir::EntityWithAttributes entity = convertExprToHLFIR(
             converter.getCurrentLocation(), converter, expr, symMap, stmtCtx);
diff --git a/external/llvm-project/flang/lib/Lower/OpenMP/Clauses.cpp b/external/llvm-project/flang/lib/Lower/OpenMP/Clauses.cpp
index a4838bc24ac9..fd36053c03ca 100644
--- a/external/llvm-project/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/external/llvm-project/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -70,19 +70,18 @@ struct SymbolAndDesignatorExtractor {
 
   static void verify(const SymbolWithDesignator &sd) {
     const semantics::Symbol *symbol = std::get<0>(sd);
-    assert(symbol && "Expecting symbol");
-    auto &maybeDsg = std::get<1>(sd);
+    const std::optional<evaluate::Expr<evaluate::SomeType>> &maybeDsg =
+        std::get<1>(sd);
     if (!maybeDsg)
       return; // Symbol with no designator -> OK
-    std::optional<evaluate::DataRef> maybeRef =
-        evaluate::ExtractDataRef(*maybeDsg);
+    assert(symbol && "Expecting symbol");
+    std::optional<evaluate::DataRef> maybeRef = evaluate::ExtractDataRef(
+        *maybeDsg, /*intoSubstring=*/true, /*intoComplexPart=*/true);
     if (maybeRef) {
       if (&maybeRef->GetLastSymbol() == symbol)
         return; // Symbol with a designator for it -> OK
       llvm_unreachable("Expecting designator for given symbol");
     } else {
-      // This could still be a Substring or ComplexPart, but at least Substring
-      // is not allowed in OpenMP.
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
       maybeDsg->dump();
 #endif
diff --git a/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index cfd62a1b9a61..78e4e64309e7 100644
--- a/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -361,26 +361,26 @@ getSource(const semantics::SemanticsContext &semaCtx,
   const parser::CharBlock *source = nullptr;
 
   auto ompConsVisit = [&](const parser::OpenMPConstruct &x) {
-    std::visit(common::visitors{
-                   [&](const parser::OpenMPSectionsConstruct &x) {
-                     source = &std::get<0>(x.t).source;
-                   },
-                   [&](const parser::OpenMPLoopConstruct &x) {
-                     source = &std::get<0>(x.t).source;
-                   },
-                   [&](const parser::OpenMPBlockConstruct &x) {
-                     source = &std::get<0>(x.t).source;
-                   },
-                   [&](const parser::OpenMPCriticalConstruct &x) {
-                     source = &std::get<0>(x.t).source;
-                   },
-                   [&](const parser::OpenMPAtomicConstruct &x) {
-                     std::visit([&](const auto &x) { source = &x.source; },
-                                x.u);
-                   },
-                   [&](const auto &x) { source = &x.source; },
-               },
-               x.u);
+    std::visit(
+        common::visitors{
+            [&](const parser::OpenMPSectionsConstruct &x) {
+              source = &std::get<0>(x.t).source;
+            },
+            [&](const parser::OpenMPLoopConstruct &x) {
+              source = &std::get<0>(x.t).source;
+            },
+            [&](const parser::OpenMPBlockConstruct &x) {
+              source = &std::get<0>(x.t).source;
+            },
+            [&](const parser::OpenMPCriticalConstruct &x) {
+              source = &std::get<0>(x.t).source;
+            },
+            [&](const parser::OpenMPAtomicConstruct &x) {
+              source = &std::get<parser::OmpDirectiveSpecification>(x.t).source;
+            },
+            [&](const auto &x) { source = &x.source; },
+        },
+        x.u);
   };
 
   eval.visit(common::visitors{
@@ -393,19 +393,43 @@ getSource(const semantics::SemanticsContext &semaCtx,
   return source;
 }
 
+bool DataSharingProcessor::isOpenMPPrivatizingConstruct(
+    const parser::OpenMPConstruct &omp) {
+  return common::visit(
+      [](auto &&s) {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        return std::is_same_v<BareS, parser::OpenMPBlockConstruct> ||
+               std::is_same_v<BareS, parser::OpenMPLoopConstruct> ||
+               std::is_same_v<BareS, parser::OpenMPSectionsConstruct>;
+      },
+      omp.u);
+}
+
+bool DataSharingProcessor::isOpenMPPrivatizingEvaluation(
+    const pft::Evaluation &eval) const {
+  return eval.visit([](auto &&s) {
+    using BareS = llvm::remove_cvref_t<decltype(s)>;
+    if constexpr (std::is_same_v<BareS, parser::OpenMPConstruct>) {
+      return isOpenMPPrivatizingConstruct(s);
+    } else {
+      return false;
+    }
+  });
+}
+
 void DataSharingProcessor::collectSymbolsInNestedRegions(
     lower::pft::Evaluation &eval, semantics::Symbol::Flag flag,
     llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) {
-  for (lower::pft::Evaluation &nestedEval : eval.getNestedEvaluations()) {
-    if (nestedEval.hasNestedEvaluations()) {
-      if (nestedEval.isConstruct())
-        // Recursively look for OpenMP constructs within `nestedEval`'s region
-        collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions);
-      else {
-        converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
-                                   /*collectSymbols=*/true,
-                                   /*collectHostAssociatedSymbols=*/false);
-      }
+  if (!eval.hasNestedEvaluations())
+    return;
+  for (pft::Evaluation &nestedEval : eval.getNestedEvaluations()) {
+    if (isOpenMPPrivatizingEvaluation(nestedEval)) {
+      converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
+                                 /*collectSymbols=*/true,
+                                 /*collectHostAssociatedSymbols=*/false);
+    } else {
+      // Recursively look for OpenMP constructs within `nestedEval`'s region
+      collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions);
     }
   }
 }
diff --git a/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 458ae2db8706..c805d2e25f7a 100644
--- a/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/external/llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -45,20 +45,22 @@ class DataSharingProcessor {
 
     bool Pre(const parser::OpenMPConstruct &omp) {
       // Skip constructs that may not have privatizations.
-      if (!std::holds_alternative<parser::OpenMPCriticalConstruct>(omp.u))
-        currentConstruct = &omp;
+      if (isOpenMPPrivatizingConstruct(omp))
+        constructs.push_back(&omp);
       return true;
     }
 
     void Post(const parser::OpenMPConstruct &omp) {
-      currentConstruct = nullptr;
+      if (isOpenMPPrivatizingConstruct(omp))
+        constructs.pop_back();
     }
 
     void Post(const parser::Name &name) {
-      symDefMap.try_emplace(name.symbol, currentConstruct);
+      auto *current = !constructs.empty() ? constructs.back() : nullptr;
+      symDefMap.try_emplace(name.symbol, current);
     }
 
-    const parser::OpenMPConstruct *currentConstruct = nullptr;
+    llvm::SmallVector<const parser::OpenMPConstruct *> constructs;
     llvm::DenseMap<semantics::Symbol *, const parser::OpenMPConstruct *>
         symDefMap;
 
@@ -114,6 +116,9 @@ class DataSharingProcessor {
                              mlir::OpBuilder::InsertPoint *lastPrivIP);
   void insertDeallocs();
 
+  static bool isOpenMPPrivatizingConstruct(const parser::OpenMPConstruct &omp);
+  bool isOpenMPPrivatizingEvaluation(const pft::Evaluation &eval) const;
+
 public:
   DataSharingProcessor(lower::AbstractConverter &converter,
                        semantics::SemanticsContext &semaCtx,
diff --git a/external/llvm-project/flang/lib/Lower/OpenMP/OpenMP.cpp b/external/llvm-project/flang/lib/Lower/OpenMP/OpenMP.cpp
index 19e85ef068fb..1f8c7e058e4a 100644
--- a/external/llvm-project/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/external/llvm-project/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -41,10 +41,13 @@
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include "llvm/Support/CommandLine.h"
 
 using namespace Fortran::lower::omp;
 using namespace Fortran::common::openmp;
 
+static llvm::cl::opt<bool> DumpAtomicAnalysis("fdebug-dump-atomic-analysis");
+
 //===----------------------------------------------------------------------===//
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
@@ -1122,6 +1125,16 @@ markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
   declareTargetOp.setDeclareTarget(deviceType, captureClause);
 }
 
+static bool isPointerAssignment(const evaluate::Assignment &assign) {
+  return common::visit(
+      common::visitors{
+          [](const evaluate::Assignment::BoundsSpec &) { return true; },
+          [](const evaluate::Assignment::BoundsRemapping &) { return true; },
+          [](const auto &) { return false; },
+      },
+      assign.u);
+}
+
 //===----------------------------------------------------------------------===//
 // Op body generation helper structures and functions
 //===----------------------------------------------------------------------===//
@@ -2587,645 +2600,215 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 //===----------------------------------------------------------------------===//
 // Code generation for atomic operations
 //===----------------------------------------------------------------------===//
+static fir::FirOpBuilder::InsertPoint
+getInsertionPointBefore(mlir::Operation *op) {
+  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
+                                        mlir::Block::iterator(op));
+}
 
-/// Populates \p hint and \p memoryOrder with appropriate clause information
-/// if present on atomic construct.
-static void genOmpAtomicHintAndMemoryOrderClauses(
-    lower::AbstractConverter &converter,
-    const parser::OmpAtomicClauseList &clauseList, mlir::IntegerAttr &hint,
-    mlir::omp::ClauseMemoryOrderKindAttr &memoryOrder) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  for (const parser::OmpAtomicClause &clause : clauseList.v) {
-    common::visit(
-        common::visitors{
-            [&](const parser::OmpMemoryOrderClause &s) {
-              auto kind = common::visit(
-                  common::visitors{
-                      [&](const parser::OmpClause::AcqRel &) {
-                        return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
-                      },
-                      [&](const parser::OmpClause::Acquire &) {
-                        return mlir::omp::ClauseMemoryOrderKind::Acquire;
-                      },
-                      [&](const parser::OmpClause::Relaxed &) {
-                        return mlir::omp::ClauseMemoryOrderKind::Relaxed;
-                      },
-                      [&](const parser::OmpClause::Release &) {
-                        return mlir::omp::ClauseMemoryOrderKind::Release;
-                      },
-                      [&](const parser::OmpClause::SeqCst &) {
-                        return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
-                      },
-                      [&](auto &&) -> mlir::omp::ClauseMemoryOrderKind {
-                        llvm_unreachable("Unexpected clause");
-                      },
-                  },
-                  s.v.u);
-              memoryOrder = mlir::omp::ClauseMemoryOrderKindAttr::get(
-                  firOpBuilder.getContext(), kind);
-            },
-            [&](const parser::OmpHintClause &s) {
-              const auto *expr = semantics::GetExpr(s.v);
-              uint64_t hintExprValue = *evaluate::ToInt64(*expr);
-              hint = firOpBuilder.getI64IntegerAttr(hintExprValue);
-            },
-            [&](const parser::OmpFailClause &) {},
-        },
-        clause.u);
+static fir::FirOpBuilder::InsertPoint
+getInsertionPointAfter(mlir::Operation *op) {
+  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
+                                        ++mlir::Block::iterator(op));
+}
+
+static mlir::IntegerAttr getAtomicHint(lower::AbstractConverter &converter,
+                                       const List<Clause> &clauses) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  for (const Clause &clause : clauses) {
+    if (clause.id != llvm::omp::Clause::OMPC_hint)
+      continue;
+    auto &hint = std::get<clause::Hint>(clause.u);
+    auto maybeVal = evaluate::ToInt64(hint.v);
+    CHECK(maybeVal);
+    return builder.getI64IntegerAttr(*maybeVal);
   }
+  return nullptr;
 }
 
-static void processOmpAtomicTODO(mlir::Type elementType, mlir::Location loc) {
-  if (!elementType)
-    return;
-  assert(fir::isa_trivial(fir::unwrapRefType(elementType)) &&
-         "is supported type for omp atomic");
-}
-
-/// Used to generate atomic.read operation which is created in existing
-/// location set by builder.
-static void genAtomicCaptureStatement(
-    lower::AbstractConverter &converter, mlir::Value fromAddress,
-    mlir::Value toAddress,
-    const parser::OmpAtomicClauseList *leftHandClauseList,
-    const parser::OmpAtomicClauseList *rightHandClauseList,
-    mlir::Type elementType, mlir::Location loc) {
-  // Generate `atomic.read` operation for atomic assigment statements
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+static mlir::omp::ClauseMemoryOrderKindAttr
+getAtomicMemoryOrder(lower::AbstractConverter &converter,
+                     semantics::SemanticsContext &semaCtx,
+                     const List<Clause> &clauses) {
+  std::optional<mlir::omp::ClauseMemoryOrderKind> kind;
+  unsigned version = semaCtx.langOptions().OpenMPVersion;
 
-  processOmpAtomicTODO(elementType, loc);
-
-  // If no hint clause is specified, the effect is as if
-  // hint(omp_sync_hint_none) had been specified.
-  mlir::IntegerAttr hint = nullptr;
-
-  mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr;
-  if (leftHandClauseList)
-    genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, hint,
-                                          memoryOrder);
-  if (rightHandClauseList)
-    genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, hint,
-                                          memoryOrder);
-  firOpBuilder.create<mlir::omp::AtomicReadOp>(loc, fromAddress, toAddress,
-                                               mlir::TypeAttr::get(elementType),
-                                               hint, memoryOrder);
-}
-
-/// Used to generate atomic.write operation which is created in existing
-/// location set by builder.
-static void genAtomicWriteStatement(
-    lower::AbstractConverter &converter, mlir::Value lhsAddr,
-    mlir::Value rhsExpr, const parser::OmpAtomicClauseList *leftHandClauseList,
-    const parser::OmpAtomicClauseList *rightHandClauseList, mlir::Location loc,
-    mlir::Value *evaluatedExprValue = nullptr) {
-  // Generate `atomic.write` operation for atomic assignment statements
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  for (const Clause &clause : clauses) {
+    switch (clause.id) {
+    case llvm::omp::Clause::OMPC_acq_rel:
+      kind = mlir::omp::ClauseMemoryOrderKind::Acq_rel;
+      break;
+    case llvm::omp::Clause::OMPC_acquire:
+      kind = mlir::omp::ClauseMemoryOrderKind::Acquire;
+      break;
+    case llvm::omp::Clause::OMPC_relaxed:
+      kind = mlir::omp::ClauseMemoryOrderKind::Relaxed;
+      break;
+    case llvm::omp::Clause::OMPC_release:
+      kind = mlir::omp::ClauseMemoryOrderKind::Release;
+      break;
+    case llvm::omp::Clause::OMPC_seq_cst:
+      kind = mlir::omp::ClauseMemoryOrderKind::Seq_cst;
+      break;
+    default:
+      break;
+    }
+  }
 
-  mlir::Type varType = fir::unwrapRefType(lhsAddr.getType());
-  // Create a conversion outside the capture block.
-  auto insertionPoint = firOpBuilder.saveInsertionPoint();
-  firOpBuilder.setInsertionPointAfter(rhsExpr.getDefiningOp());
-  rhsExpr = firOpBuilder.createConvert(loc, varType, rhsExpr);
-  firOpBuilder.restoreInsertionPoint(insertionPoint);
-
-  processOmpAtomicTODO(varType, loc);
-
-  // If no hint clause is specified, the effect is as if
-  // hint(omp_sync_hint_none) had been specified.
-  mlir::IntegerAttr hint = nullptr;
-  mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr;
-  if (leftHandClauseList)
-    genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, hint,
-                                          memoryOrder);
-  if (rightHandClauseList)
-    genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, hint,
-                                          memoryOrder);
-  firOpBuilder.create<mlir::omp::AtomicWriteOp>(loc, lhsAddr, rhsExpr, hint,
-                                                memoryOrder);
-}
-
-/// Used to generate atomic.update operation which is created in existing
-/// location set by builder.
-static void genAtomicUpdateStatement(
-    lower::AbstractConverter &converter, mlir::Value lhsAddr,
-    mlir::Type varType, const parser::Variable &assignmentStmtVariable,
-    const parser::Expr &assignmentStmtExpr,
-    const parser::OmpAtomicClauseList *leftHandClauseList,
-    const parser::OmpAtomicClauseList *rightHandClauseList, mlir::Location loc,
-    mlir::Operation *atomicCaptureOp = nullptr,
-    lower::StatementContext *atomicCaptureStmtCtx = nullptr) {
-  // Generate `atomic.update` operation for atomic assignment statements
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  mlir::Location currentLocation = converter.getCurrentLocation();
+  // Starting with 5.1, if no memory-order clause is present, the effect
+  // is as if "relaxed" was present.
+  if (!kind) {
+    if (version <= 50)
+      return nullptr;
+    kind = mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  }
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  return mlir::omp::ClauseMemoryOrderKindAttr::get(builder.getContext(), *kind);
+}
+
+static mlir::Operation * //
+genAtomicRead(lower::AbstractConverter &converter, mlir::Location loc,
+              lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+              const semantics::SomeExpr &atom,
+              const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+              mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+              fir::FirOpBuilder::InsertPoint preAt,
+              fir::FirOpBuilder::InsertPoint atomicAt,
+              fir::FirOpBuilder::InsertPoint postAt) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  mlir::Value storeAddr =
+      fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx, &loc));
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+  mlir::Type storeType = fir::unwrapRefType(storeAddr.getType());
+
+  mlir::Value toAddr = [&]() {
+    if (atomType == storeType)
+      return storeAddr;
+    return builder.createTemporary(loc, atomType, ".tmp.atomval");
+  }();
 
-  //  Create the omp.atomic.update or acc.atomic.update operation
-  //
-  //  func.func @_QPsb() {
-  //    %0 = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFsbEa"}
-  //    %1 = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFsbEb"}
-  //    %2 = fir.load %1 : !fir.ref<i32>
-  //    omp.atomic.update   %0 : !fir.ref<i32> {
-  //    ^bb0(%arg0: i32):
-  //      %3 = arith.addi %arg0, %2 : i32
-  //      omp.yield(%3 : i32)
-  //    }
-  //    return
-  //  }
-
-  auto getArgExpression =
-      [](std::list<parser::ActualArgSpec>::const_iterator it) {
-        const auto &arg{std::get<parser::ActualArg>((*it).t)};
-        const auto *parserExpr{
-            std::get_if<common::Indirection<parser::Expr>>(&arg.u)};
-        return parserExpr;
-      };
+  builder.restoreInsertionPoint(atomicAt);
+  mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
+      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint, memOrder);
+
+  if (atomType != storeType) {
+    lower::ExprToValueMap overrides;
+    // The READ operation could be a part of UPDATE CAPTURE, so make sure
+    // we don't emit extra code into the body of the atomic op.
+    builder.restoreInsertionPoint(postAt);
+    mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
+    overrides.try_emplace(&atom, load);
+
+    converter.overrideExprValues(&overrides);
+    mlir::Value value =
+        fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+    converter.resetExprOverrides();
 
-  // Lower any non atomic sub-expression before the atomic operation, and
-  // map its lowered value to the semantic representation.
-  lower::ExprToValueMap exprValueOverrides;
-  // Max and min intrinsics can have a list of Args. Hence we need a list
-  // of nonAtomicSubExprs to hoist. Currently, only the load is hoisted.
-  llvm::SmallVector<const lower::SomeExpr *> nonAtomicSubExprs;
-  common::visit(
-      common::visitors{
-          [&](const common::Indirection<parser::FunctionReference> &funcRef)
-              -> void {
-            const auto &args{std::get<std::list<parser::ActualArgSpec>>(
-                funcRef.value().v.t)};
-            std::list<parser::ActualArgSpec>::const_iterator beginIt =
-                args.begin();
-            std::list<parser::ActualArgSpec>::const_iterator endIt = args.end();
-            const auto *exprFirst{getArgExpression(beginIt)};
-            if (exprFirst && exprFirst->value().source ==
-                                 assignmentStmtVariable.GetSource()) {
-              // Add everything except the first
-              beginIt++;
-            } else {
-              // Add everything except the last
-              endIt--;
-            }
-            std::list<parser::ActualArgSpec>::const_iterator it;
-            for (it = beginIt; it != endIt; it++) {
-              const common::Indirection<parser::Expr> *expr =
-                  getArgExpression(it);
-              if (expr)
-                nonAtomicSubExprs.push_back(semantics::GetExpr(*expr));
-            }
-          },
-          [&](const auto &op) -> void {
-            using T = std::decay_t<decltype(op)>;
-            if constexpr (std::is_base_of<parser::Expr::IntrinsicBinary,
-                                          T>::value) {
-              const auto &exprLeft{std::get<0>(op.t)};
-              const auto &exprRight{std::get<1>(op.t)};
-              if (exprLeft.value().source == assignmentStmtVariable.GetSource())
-                nonAtomicSubExprs.push_back(semantics::GetExpr(exprRight));
-              else
-                nonAtomicSubExprs.push_back(semantics::GetExpr(exprLeft));
-            }
-          },
-      },
-      assignmentStmtExpr.u);
-  lower::StatementContext nonAtomicStmtCtx;
-  lower::StatementContext *stmtCtxPtr = &nonAtomicStmtCtx;
-  if (!nonAtomicSubExprs.empty()) {
-    // Generate non atomic part before all the atomic operations.
-    auto insertionPoint = firOpBuilder.saveInsertionPoint();
-    if (atomicCaptureOp) {
-      assert(atomicCaptureStmtCtx && "must specify statement context");
-      firOpBuilder.setInsertionPoint(atomicCaptureOp);
-      // Any clean-ups associated with the expression lowering
-      // must also be generated outside of the atomic update operation
-      // and after the atomic capture operation.
-      // The atomicCaptureStmtCtx will be finalized at the end
-      // of the atomic capture operation generation.
-      stmtCtxPtr = atomicCaptureStmtCtx;
-    }
-    mlir::Value nonAtomicVal;
-    for (auto *nonAtomicSubExpr : nonAtomicSubExprs) {
-      nonAtomicVal = fir::getBase(converter.genExprValue(
-          currentLocation, *nonAtomicSubExpr, *stmtCtxPtr));
-      exprValueOverrides.try_emplace(nonAtomicSubExpr, nonAtomicVal);
-    }
-    if (atomicCaptureOp)
-      firOpBuilder.restoreInsertionPoint(insertionPoint);
+    builder.create<fir::StoreOp>(loc, value, storeAddr);
   }
+  return op;
+}
 
-  mlir::Operation *atomicUpdateOp = nullptr;
-  // If no hint clause is specified, the effect is as if
-  // hint(omp_sync_hint_none) had been specified.
-  mlir::IntegerAttr hint = nullptr;
-  mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr;
-  if (leftHandClauseList)
-    genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, hint,
-                                          memoryOrder);
-  if (rightHandClauseList)
-    genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, hint,
-                                          memoryOrder);
-  atomicUpdateOp = firOpBuilder.create<mlir::omp::AtomicUpdateOp>(
-      currentLocation, lhsAddr, hint, memoryOrder);
-
-  processOmpAtomicTODO(varType, loc);
-
-  llvm::SmallVector<mlir::Type> varTys = {varType};
-  llvm::SmallVector<mlir::Location> locs = {currentLocation};
-  firOpBuilder.createBlock(&atomicUpdateOp->getRegion(0), {}, varTys, locs);
-  mlir::Value val =
-      fir::getBase(atomicUpdateOp->getRegion(0).front().getArgument(0));
-
-  exprValueOverrides.try_emplace(semantics::GetExpr(assignmentStmtVariable),
-                                 val);
-  {
-    // statement context inside the atomic block.
-    converter.overrideExprValues(&exprValueOverrides);
-    lower::StatementContext atomicStmtCtx;
-    mlir::Value rhsExpr = fir::getBase(converter.genExprValue(
-        *semantics::GetExpr(assignmentStmtExpr), atomicStmtCtx));
-    mlir::Type exprType = fir::unwrapRefType(rhsExpr.getType());
-    if (fir::isa_complex(exprType) && !fir::isa_complex(varType)) {
-      // Emit an additional `ExtractValueOp` if the expression is of complex
-      // type
-      auto extract = firOpBuilder.create<fir::ExtractValueOp>(
-          currentLocation,
-          mlir::cast<mlir::ComplexType>(exprType).getElementType(), rhsExpr,
-          firOpBuilder.getArrayAttr(
-              firOpBuilder.getIntegerAttr(firOpBuilder.getIndexType(), 0)));
-      mlir::Value convertResult = firOpBuilder.create<fir::ConvertOp>(
-          currentLocation, varType, extract);
-      firOpBuilder.create<mlir::omp::YieldOp>(currentLocation, convertResult);
-    } else {
-      mlir::Value convertResult =
-          firOpBuilder.createConvert(currentLocation, varType, rhsExpr);
-      firOpBuilder.create<mlir::omp::YieldOp>(currentLocation, convertResult);
+static mlir::Operation * //
+genAtomicWrite(lower::AbstractConverter &converter, mlir::Location loc,
+               lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+               const semantics::SomeExpr &atom,
+               const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+               mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+               fir::FirOpBuilder::InsertPoint preAt,
+               fir::FirOpBuilder::InsertPoint atomicAt,
+               fir::FirOpBuilder::InsertPoint postAt) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  mlir::Value value =
+      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+  mlir::Value converted = builder.createConvert(loc, atomType, value);
+
+  builder.restoreInsertionPoint(atomicAt);
+  mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
+      loc, atomAddr, converted, hint, memOrder);
+  return op;
+}
+
+static mlir::Operation *
+genAtomicUpdate(lower::AbstractConverter &converter, mlir::Location loc,
+                lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+                const semantics::SomeExpr &atom,
+                const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+                mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+                fir::FirOpBuilder::InsertPoint preAt,
+                fir::FirOpBuilder::InsertPoint atomicAt,
+                fir::FirOpBuilder::InsertPoint postAt) {
+  lower::ExprToValueMap overrides;
+  lower::StatementContext naCtx;
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+
+  // This must exist by now.
+  SomeExpr input = *semantics::GetConvertInput(assign.rhs);
+  std::vector<SomeExpr> args{semantics::GetTopLevelOperation(input).second};
+  assert(!args.empty() && "Update operation without arguments");
+  for (auto &arg : args) {
+    if (!semantics::IsSameOrConvertOf(arg, atom)) {
+      mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
+      overrides.try_emplace(&arg, val);
     }
-    converter.resetExprOverrides();
   }
-  firOpBuilder.setInsertionPointAfter(atomicUpdateOp);
-}
-
-/// Processes an atomic construct with write clause.
-static void genAtomicWrite(lower::AbstractConverter &converter,
-                           const parser::OmpAtomicWrite &atomicWrite,
-                           mlir::Location loc) {
-  const parser::OmpAtomicClauseList *rightHandClauseList = nullptr;
-  const parser::OmpAtomicClauseList *leftHandClauseList = nullptr;
-  // Get the address of atomic read operands.
-  rightHandClauseList = &std::get<2>(atomicWrite.t);
-  leftHandClauseList = &std::get<0>(atomicWrite.t);
-
-  const parser::AssignmentStmt &stmt =
-      std::get<parser::Statement<parser::AssignmentStmt>>(atomicWrite.t)
-          .statement;
-  const evaluate::Assignment &assign = *stmt.typedAssignment->v;
-  lower::StatementContext stmtCtx;
-  // Get the value and address of atomic write operands.
-  mlir::Value rhsExpr =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx));
-  mlir::Value lhsAddr =
-      fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx));
-  genAtomicWriteStatement(converter, lhsAddr, rhsExpr, leftHandClauseList,
-                          rightHandClauseList, loc);
-}
-
-/*
-    Emit an implicit cast. Different yet compatible types on
-    omp.atomic.read constitute valid Fortran. The OMPIRBuilder will
-    emit atomic instructions (on primitive types) and `__atomic_load`
-    libcall (on complex type) without explicitly converting
-    between such compatible types. The OMPIRBuilder relies on the
-    frontend to resolve such inconsistencies between `omp.atomic.read `
-    operand types. Similar inconsistencies between operand types in
-    `omp.atomic.write` are resolved through implicit casting by use of typed
-    assignment (i.e. `evaluate::Assignment`). However, use of typed
-    assignment in `omp.atomic.read` (of form `v = x`) leads to an unsafe,
-    non-atomic load of `x` into a temporary `alloca`, followed by an atomic
-    read of form `v = alloca`. Hence, it is needed to perform a custom
-    implicit cast.
-
-    An atomic read of form `v = x` would (without implicit casting)
-    lower to `omp.atomic.read %v = %x : !fir.ref<type1>, !fir.ref<type2>,
-    type2`. This implicit casting will rather generate the following FIR:
-
-         %alloca = fir.alloca type2
-         omp.atomic.read %alloca = %x : !fir.ref<type2>, !fir.ref<type2>, type2
-         %load = fir.load %alloca : !fir.ref<type2>
-         %cvt = fir.convert %load : (type2) -> type1
-         fir.store %cvt to %v : !fir.ref<type1>
-
-    These sequence of operations is thread-safe since each thread allocates
-    the `alloca` in its stack, and performs `%alloca = %x` atomically. Once
-    safely read, each thread performs the implicit cast on the local
-    `alloca`, and writes the final result to `%v`.
-
-/// \param builder              : FirOpBuilder
-/// \param loc                  : Location for FIR generation
-/// \param toAddress            : Address of %v
-/// \param toType               : Type of %v
-/// \param fromType             : Type of %x
-/// \param alloca               : Thread scoped `alloca`
-//				  It is the responsibility of the callee
-//				  to position the `alloca` at `AllocaIP`
-//				  through `builder.getAllocaBlock()`
-*/
-
-static void emitAtomicReadImplicitCast(fir::FirOpBuilder &builder,
-                                       mlir::Location loc,
-                                       mlir::Value toAddress, mlir::Type toType,
-                                       mlir::Type fromType,
-                                       mlir::Value alloca) {
-  auto load = builder.create<fir::LoadOp>(loc, alloca);
-  if (fir::isa_complex(fromType) && !fir::isa_complex(toType)) {
-    // Emit an additional `ExtractValueOp` if `fromAddress` is of complex
-    // type, but `toAddress` is not.
-    auto extract = builder.create<fir::ExtractValueOp>(
-        loc, mlir::cast<mlir::ComplexType>(fromType).getElementType(), load,
-        builder.getArrayAttr(
-            builder.getIntegerAttr(builder.getIndexType(), 0)));
-    auto cvt = builder.create<fir::ConvertOp>(loc, toType, extract);
-    builder.create<fir::StoreOp>(loc, cvt, toAddress);
-  } else if (!fir::isa_complex(fromType) && fir::isa_complex(toType)) {
-    // Emit an additional `InsertValueOp` if `toAddress` is of complex
-    // type, but `fromAddress` is not.
-    mlir::Value undef = builder.create<fir::UndefOp>(loc, toType);
-    mlir::Type complexEleTy =
-        mlir::cast<mlir::ComplexType>(toType).getElementType();
-    mlir::Value cvt = builder.create<fir::ConvertOp>(loc, complexEleTy, load);
-    mlir::Value zero = builder.createRealZeroConstant(loc, complexEleTy);
-    mlir::Value idx0 = builder.create<fir::InsertValueOp>(
-        loc, toType, undef, cvt,
-        builder.getArrayAttr(
-            builder.getIntegerAttr(builder.getIndexType(), 0)));
-    mlir::Value idx1 = builder.create<fir::InsertValueOp>(
-        loc, toType, idx0, zero,
-        builder.getArrayAttr(
-            builder.getIntegerAttr(builder.getIndexType(), 1)));
-    builder.create<fir::StoreOp>(loc, idx1, toAddress);
-  } else {
-    auto cvt = builder.create<fir::ConvertOp>(loc, toType, load);
-    builder.create<fir::StoreOp>(loc, cvt, toAddress);
-  }
-}
 
-/// Processes an atomic construct with read clause.
-static void genAtomicRead(lower::AbstractConverter &converter,
-                          const parser::OmpAtomicRead &atomicRead,
-                          mlir::Location loc) {
-  const parser::OmpAtomicClauseList *rightHandClauseList = nullptr;
-  const parser::OmpAtomicClauseList *leftHandClauseList = nullptr;
-  // Get the address of atomic read operands.
-  rightHandClauseList = &std::get<2>(atomicRead.t);
-  leftHandClauseList = &std::get<0>(atomicRead.t);
+  builder.restoreInsertionPoint(atomicAt);
+  auto updateOp =
+      builder.create<mlir::omp::AtomicUpdateOp>(loc, atomAddr, hint, memOrder);
 
-  const auto &assignmentStmtExpr = std::get<parser::Expr>(
-      std::get<parser::Statement<parser::AssignmentStmt>>(atomicRead.t)
-          .statement.t);
-  const auto &assignmentStmtVariable = std::get<parser::Variable>(
-      std::get<parser::Statement<parser::AssignmentStmt>>(atomicRead.t)
-          .statement.t);
+  mlir::Region &region = updateOp->getRegion(0);
+  mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
+  mlir::Value localAtom = fir::getBase(block->getArgument(0));
+  overrides.try_emplace(&atom, localAtom);
 
-  lower::StatementContext stmtCtx;
-  const semantics::SomeExpr &fromExpr = *semantics::GetExpr(assignmentStmtExpr);
-  mlir::Type elementType = converter.genType(fromExpr);
-  mlir::Value fromAddress =
-      fir::getBase(converter.genExprAddr(fromExpr, stmtCtx));
-  mlir::Value toAddress = fir::getBase(converter.genExprAddr(
-      *semantics::GetExpr(assignmentStmtVariable), stmtCtx));
-
-  if (fromAddress.getType() != toAddress.getType()) {
-
-    mlir::Type toType = fir::unwrapRefType(toAddress.getType());
-    mlir::Type fromType = fir::unwrapRefType(fromAddress.getType());
-    fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-    auto oldIP = builder.saveInsertionPoint();
-    builder.setInsertionPointToStart(builder.getAllocaBlock());
-    mlir::Value alloca = builder.create<fir::AllocaOp>(
-        loc, fromType); // Thread scope `alloca` to atomically read `%x`.
-    builder.restoreInsertionPoint(oldIP);
-    genAtomicCaptureStatement(converter, fromAddress, alloca,
-                              leftHandClauseList, rightHandClauseList,
-                              elementType, loc);
-    emitAtomicReadImplicitCast(builder, loc, toAddress, toType, fromType,
-                               alloca);
-  } else
-    genAtomicCaptureStatement(converter, fromAddress, toAddress,
-                              leftHandClauseList, rightHandClauseList,
-                              elementType, loc);
-}
-
-/// Processes an atomic construct with update clause.
-static void genAtomicUpdate(lower::AbstractConverter &converter,
-                            const parser::OmpAtomicUpdate &atomicUpdate,
-                            mlir::Location loc) {
-  const parser::OmpAtomicClauseList *rightHandClauseList = nullptr;
-  const parser::OmpAtomicClauseList *leftHandClauseList = nullptr;
-  // Get the address of atomic read operands.
-  rightHandClauseList = &std::get<2>(atomicUpdate.t);
-  leftHandClauseList = &std::get<0>(atomicUpdate.t);
-
-  const auto &assignmentStmtExpr = std::get<parser::Expr>(
-      std::get<parser::Statement<parser::AssignmentStmt>>(atomicUpdate.t)
-          .statement.t);
-  const auto &assignmentStmtVariable = std::get<parser::Variable>(
-      std::get<parser::Statement<parser::AssignmentStmt>>(atomicUpdate.t)
-          .statement.t);
+  converter.overrideExprValues(&overrides);
+  mlir::Value updated =
+      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+  mlir::Value converted = builder.createConvert(loc, atomType, updated);
+  builder.create<mlir::omp::YieldOp>(loc, converted);
+  converter.resetExprOverrides();
 
-  lower::StatementContext stmtCtx;
-  mlir::Value lhsAddr = fir::getBase(converter.genExprAddr(
-      *semantics::GetExpr(assignmentStmtVariable), stmtCtx));
-  mlir::Type varType = fir::unwrapRefType(lhsAddr.getType());
-  genAtomicUpdateStatement(converter, lhsAddr, varType, assignmentStmtVariable,
-                           assignmentStmtExpr, leftHandClauseList,
-                           rightHandClauseList, loc);
-}
-
-/// Processes an atomic construct with no clause - which implies update clause.
-static void genOmpAtomic(lower::AbstractConverter &converter,
-                         const parser::OmpAtomic &atomicConstruct,
-                         mlir::Location loc) {
-  const parser::OmpAtomicClauseList &atomicClauseList =
-      std::get<parser::OmpAtomicClauseList>(atomicConstruct.t);
-  const auto &assignmentStmtExpr = std::get<parser::Expr>(
-      std::get<parser::Statement<parser::AssignmentStmt>>(atomicConstruct.t)
-          .statement.t);
-  const auto &assignmentStmtVariable = std::get<parser::Variable>(
-      std::get<parser::Statement<parser::AssignmentStmt>>(atomicConstruct.t)
-          .statement.t);
-  lower::StatementContext stmtCtx;
-  mlir::Value lhsAddr = fir::getBase(converter.genExprAddr(
-      *semantics::GetExpr(assignmentStmtVariable), stmtCtx));
-  mlir::Type varType = fir::unwrapRefType(lhsAddr.getType());
-  // If atomic-clause is not present on the construct, the behaviour is as if
-  // the update clause is specified (for both OpenMP and OpenACC).
-  genAtomicUpdateStatement(converter, lhsAddr, varType, assignmentStmtVariable,
-                           assignmentStmtExpr, &atomicClauseList, nullptr, loc);
-}
-
-/// Processes an atomic construct with capture clause.
-static void genAtomicCapture(lower::AbstractConverter &converter,
-                             const parser::OmpAtomicCapture &atomicCapture,
-                             mlir::Location loc) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(postAt); // For naCtx cleanups
+  return updateOp;
+}
 
-  const parser::AssignmentStmt &stmt1 =
-      std::get<parser::OmpAtomicCapture::Stmt1>(atomicCapture.t).v.statement;
-  const evaluate::Assignment &assign1 = *stmt1.typedAssignment->v;
-  const auto &stmt1Var{std::get<parser::Variable>(stmt1.t)};
-  const auto &stmt1Expr{std::get<parser::Expr>(stmt1.t)};
-  const parser::AssignmentStmt &stmt2 =
-      std::get<parser::OmpAtomicCapture::Stmt2>(atomicCapture.t).v.statement;
-  const evaluate::Assignment &assign2 = *stmt2.typedAssignment->v;
-  const auto &stmt2Var{std::get<parser::Variable>(stmt2.t)};
-  const auto &stmt2Expr{std::get<parser::Expr>(stmt2.t)};
-
-  // Pre-evaluate expressions to be used in the various operations inside
-  // `atomic.capture` since it is not desirable to have anything other than
-  // a `atomic.read`, `atomic.write`, or `atomic.update` operation
-  // inside `atomic.capture`
-  lower::StatementContext stmtCtx;
-  // LHS evaluations are common to all combinations of `atomic.capture`
-  mlir::Value stmt1LHSArg =
-      fir::getBase(converter.genExprAddr(assign1.lhs, stmtCtx));
-  mlir::Value stmt2LHSArg =
-      fir::getBase(converter.genExprAddr(assign2.lhs, stmtCtx));
-
-  // Type information used in generation of `atomic.update` operation
-  mlir::Type stmt1VarType =
-      fir::getBase(converter.genExprValue(assign1.lhs, stmtCtx)).getType();
-  mlir::Type stmt2VarType =
-      fir::getBase(converter.genExprValue(assign2.lhs, stmtCtx)).getType();
-
-  mlir::Operation *atomicCaptureOp = nullptr;
-  mlir::IntegerAttr hint = nullptr;
-  mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr;
-  const parser::OmpAtomicClauseList &rightHandClauseList =
-      std::get<2>(atomicCapture.t);
-  const parser::OmpAtomicClauseList &leftHandClauseList =
-      std::get<0>(atomicCapture.t);
-  genOmpAtomicHintAndMemoryOrderClauses(converter, leftHandClauseList, hint,
-                                        memoryOrder);
-  genOmpAtomicHintAndMemoryOrderClauses(converter, rightHandClauseList, hint,
-                                        memoryOrder);
-  atomicCaptureOp =
-      firOpBuilder.create<mlir::omp::AtomicCaptureOp>(loc, hint, memoryOrder);
-
-  firOpBuilder.createBlock(&(atomicCaptureOp->getRegion(0)));
-  mlir::Block &block = atomicCaptureOp->getRegion(0).back();
-  firOpBuilder.setInsertionPointToStart(&block);
-  if (semantics::checkForSingleVariableOnRHS(stmt1)) {
-    if (semantics::checkForSymbolMatch(semantics::GetExpr(stmt2Var),
-                                       semantics::GetExpr(stmt2Expr))) {
-      // Atomic capture construct is of the form [capture-stmt, update-stmt]
-      const semantics::SomeExpr &fromExpr = *semantics::GetExpr(stmt1Expr);
-      mlir::Type elementType = converter.genType(fromExpr);
-      if (stmt1VarType != stmt2VarType) {
-        mlir::Value alloca;
-        mlir::Type toType = fir::unwrapRefType(stmt1LHSArg.getType());
-        mlir::Type fromType = fir::unwrapRefType(stmt2LHSArg.getType());
-        {
-          mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-          firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
-          alloca = firOpBuilder.create<fir::AllocaOp>(loc, fromType);
-        }
-        genAtomicCaptureStatement(converter, stmt2LHSArg, alloca,
-                                  /*leftHandClauseList=*/nullptr,
-                                  /*rightHandClauseList=*/nullptr, elementType,
-                                  loc);
-        {
-          mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-          firOpBuilder.setInsertionPointAfter(atomicCaptureOp);
-          emitAtomicReadImplicitCast(firOpBuilder, loc, stmt1LHSArg, toType,
-                                     fromType, alloca);
-        }
-      } else {
-        genAtomicCaptureStatement(converter, stmt2LHSArg, stmt1LHSArg,
-                                  /*leftHandClauseList=*/nullptr,
-                                  /*rightHandClauseList=*/nullptr, elementType,
-                                  loc);
-      }
-      genAtomicUpdateStatement(
-          converter, stmt2LHSArg, stmt2VarType, stmt2Var, stmt2Expr,
-          /*leftHandClauseList=*/nullptr,
-          /*rightHandClauseList=*/nullptr, loc, atomicCaptureOp, &stmtCtx);
-    } else {
-      // Atomic capture construct is of the form [capture-stmt, write-stmt]
-      firOpBuilder.setInsertionPoint(atomicCaptureOp);
-      mlir::Value stmt2RHSArg =
-          fir::getBase(converter.genExprValue(assign2.rhs, stmtCtx));
-      firOpBuilder.setInsertionPointToStart(&block);
-      const semantics::SomeExpr &fromExpr = *semantics::GetExpr(stmt1Expr);
-      mlir::Type elementType = converter.genType(fromExpr);
-
-      if (stmt1VarType != stmt2VarType) {
-        mlir::Value alloca;
-        mlir::Type toType = fir::unwrapRefType(stmt1LHSArg.getType());
-        mlir::Type fromType = fir::unwrapRefType(stmt2LHSArg.getType());
-        {
-          mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-          firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
-          alloca = firOpBuilder.create<fir::AllocaOp>(loc, fromType);
-        }
-        genAtomicCaptureStatement(converter, stmt2LHSArg, alloca,
-                                  /*leftHandClauseList=*/nullptr,
-                                  /*rightHandClauseList=*/nullptr, elementType,
-                                  loc);
-        {
-          mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-          firOpBuilder.setInsertionPointAfter(atomicCaptureOp);
-          emitAtomicReadImplicitCast(firOpBuilder, loc, stmt1LHSArg, toType,
-                                     fromType, alloca);
-        }
-      } else {
-        genAtomicCaptureStatement(converter, stmt2LHSArg, stmt1LHSArg,
-                                  /*leftHandClauseList=*/nullptr,
-                                  /*rightHandClauseList=*/nullptr, elementType,
-                                  loc);
-      }
-      genAtomicWriteStatement(converter, stmt2LHSArg, stmt2RHSArg,
-                              /*leftHandClauseList=*/nullptr,
-                              /*rightHandClauseList=*/nullptr, loc);
-    }
-  } else {
-    // Atomic capture construct is of the form [update-stmt, capture-stmt]
-    const semantics::SomeExpr &fromExpr = *semantics::GetExpr(stmt2Expr);
-    mlir::Type elementType = converter.genType(fromExpr);
-    genAtomicUpdateStatement(
-        converter, stmt1LHSArg, stmt1VarType, stmt1Var, stmt1Expr,
-        /*leftHandClauseList=*/nullptr,
-        /*rightHandClauseList=*/nullptr, loc, atomicCaptureOp, &stmtCtx);
-
-    if (stmt1VarType != stmt2VarType) {
-      mlir::Value alloca;
-      mlir::Type toType = fir::unwrapRefType(stmt2LHSArg.getType());
-      mlir::Type fromType = fir::unwrapRefType(stmt1LHSArg.getType());
-
-      {
-        mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-        firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
-        alloca = firOpBuilder.create<fir::AllocaOp>(loc, fromType);
-      }
+static mlir::Operation *
+genAtomicOperation(lower::AbstractConverter &converter, mlir::Location loc,
+                   lower::StatementContext &stmtCtx, int action,
+                   mlir::Value atomAddr, const semantics::SomeExpr &atom,
+                   const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+                   mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+                   fir::FirOpBuilder::InsertPoint preAt,
+                   fir::FirOpBuilder::InsertPoint atomicAt,
+                   fir::FirOpBuilder::InsertPoint postAt) {
+  if (isPointerAssignment(assign)) {
+    TODO(loc, "Code generation for pointer assignment is not implemented yet");
+  }
 
-      genAtomicCaptureStatement(converter, stmt1LHSArg, alloca,
-                                /*leftHandClauseList=*/nullptr,
-                                /*rightHandClauseList=*/nullptr, elementType,
-                                loc);
-      {
-        mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
-        firOpBuilder.setInsertionPointAfter(atomicCaptureOp);
-        emitAtomicReadImplicitCast(firOpBuilder, loc, stmt2LHSArg, toType,
-                                   fromType, alloca);
-      }
-    } else {
-      genAtomicCaptureStatement(converter, stmt1LHSArg, stmt2LHSArg,
-                                /*leftHandClauseList=*/nullptr,
-                                /*rightHandClauseList=*/nullptr, elementType,
-                                loc);
-    }
+  // This function and the functions called here do not preserve the
+  // builder's insertion point, or set it to anything specific.
+  switch (action) {
+  case parser::OpenMPAtomicConstruct::Analysis::Read:
+    return genAtomicRead(converter, loc, stmtCtx, atomAddr, atom, assign, hint,
+                         memOrder, preAt, atomicAt, postAt);
+  case parser::OpenMPAtomicConstruct::Analysis::Write:
+    return genAtomicWrite(converter, loc, stmtCtx, atomAddr, atom, assign, hint,
+                          memOrder, preAt, atomicAt, postAt);
+  case parser::OpenMPAtomicConstruct::Analysis::Update:
+    return genAtomicUpdate(converter, loc, stmtCtx, atomAddr, atom, assign,
+                           hint, memOrder, preAt, atomicAt, postAt);
+  default:
+    return nullptr;
   }
-  firOpBuilder.setInsertionPointToEnd(&block);
-  firOpBuilder.create<mlir::omp::TerminatorOp>(loc);
-  // The clean-ups associated with the statements inside the capture
-  // construct must be generated after the AtomicCaptureOp.
-  firOpBuilder.setInsertionPointAfter(atomicCaptureOp);
 }
 
 //===----------------------------------------------------------------------===//
@@ -4134,10 +3717,6 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
       standaloneConstruct.u);
 }
 
-//===----------------------------------------------------------------------===//
-// OpenMPConstruct visitors
-//===----------------------------------------------------------------------===//
-
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
@@ -4145,38 +3724,166 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
 }
 
+//===----------------------------------------------------------------------===//
+// OpenMPConstruct visitors
+//===----------------------------------------------------------------------===//
+
+[[maybe_unused]] static void
+dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
+  auto whatStr = [](int k) {
+    std::string txt = "?";
+    switch (k & parser::OpenMPAtomicConstruct::Analysis::Action) {
+    case parser::OpenMPAtomicConstruct::Analysis::None:
+      txt = "None";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Read:
+      txt = "Read";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Write:
+      txt = "Write";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Update:
+      txt = "Update";
+      break;
+    }
+    switch (k & parser::OpenMPAtomicConstruct::Analysis::Condition) {
+    case parser::OpenMPAtomicConstruct::Analysis::IfTrue:
+      txt += " | IfTrue";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::IfFalse:
+      txt += " | IfFalse";
+      break;
+    }
+    return txt;
+  };
+
+  auto exprStr = [&](const parser::TypedExpr &expr) {
+    if (auto *maybe = expr.get()) {
+      if (maybe->v)
+        return maybe->v->AsFortran();
+    }
+    return "<null>"s;
+  };
+  auto assignStr = [&](const parser::AssignmentStmt::TypedAssignment &assign) {
+    if (auto *maybe = assign.get(); maybe && maybe->v) {
+      std::string str;
+      llvm::raw_string_ostream os(str);
+      maybe->v->AsFortran(os);
+      return str;
+    }
+    return "<null>"s;
+  };
+
+  const SomeExpr &atom = *analysis.atom.get()->v;
+
+  llvm::errs() << "Analysis {\n";
+  llvm::errs() << "  atom: " << atom.AsFortran() << "\n";
+  llvm::errs() << "  cond: " << exprStr(analysis.cond) << "\n";
+  llvm::errs() << "  op0 {\n";
+  llvm::errs() << "    what: " << whatStr(analysis.op0.what) << "\n";
+  llvm::errs() << "    assign: " << assignStr(analysis.op0.assign) << "\n";
+  llvm::errs() << "  }\n";
+  llvm::errs() << "  op1 {\n";
+  llvm::errs() << "    what: " << whatStr(analysis.op1.what) << "\n";
+  llvm::errs() << "    assign: " << assignStr(analysis.op1.assign) << "\n";
+  llvm::errs() << "  }\n";
+  llvm::errs() << "}\n";
+}
+
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
-                   const parser::OpenMPAtomicConstruct &atomicConstruct) {
-  Fortran::common::visit(
-      common::visitors{
-          [&](const parser::OmpAtomicRead &atomicRead) {
-            mlir::Location loc = converter.genLocation(atomicRead.source);
-            genAtomicRead(converter, atomicRead, loc);
-          },
-          [&](const parser::OmpAtomicWrite &atomicWrite) {
-            mlir::Location loc = converter.genLocation(atomicWrite.source);
-            genAtomicWrite(converter, atomicWrite, loc);
-          },
-          [&](const parser::OmpAtomic &atomicConstruct) {
-            mlir::Location loc = converter.genLocation(atomicConstruct.source);
-            genOmpAtomic(converter, atomicConstruct, loc);
-          },
-          [&](const parser::OmpAtomicUpdate &atomicUpdate) {
-            mlir::Location loc = converter.genLocation(atomicUpdate.source);
-            genAtomicUpdate(converter, atomicUpdate, loc);
-          },
-          [&](const parser::OmpAtomicCapture &atomicCapture) {
-            mlir::Location loc = converter.genLocation(atomicCapture.source);
-            genAtomicCapture(converter, atomicCapture, loc);
-          },
-          [&](const parser::OmpAtomicCompare &atomicCompare) {
-            mlir::Location loc = converter.genLocation(atomicCompare.source);
-            TODO(loc, "OpenMP atomic compare");
-          },
-      },
-      atomicConstruct.u);
+                   const parser::OpenMPAtomicConstruct &construct) {
+  auto get = [](auto &&typedWrapper) -> decltype(&*typedWrapper.get()->v) {
+    if (auto *maybe = typedWrapper.get(); maybe && maybe->v) {
+      return &*maybe->v;
+    } else {
+      return nullptr;
+    }
+  };
+
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  auto &dirSpec = std::get<parser::OmpDirectiveSpecification>(construct.t);
+  List<Clause> clauses = makeClauses(dirSpec.Clauses(), semaCtx);
+  lower::StatementContext stmtCtx;
+
+  const parser::OpenMPAtomicConstruct::Analysis &analysis = construct.analysis;
+  if (DumpAtomicAnalysis)
+    dumpAtomicAnalysis(analysis);
+
+  const semantics::SomeExpr &atom = *get(analysis.atom);
+  mlir::Location loc = converter.genLocation(construct.source);
+  mlir::Value atomAddr =
+      fir::getBase(converter.genExprAddr(atom, stmtCtx, &loc));
+  mlir::IntegerAttr hint = getAtomicHint(converter, clauses);
+  mlir::omp::ClauseMemoryOrderKindAttr memOrder =
+      getAtomicMemoryOrder(converter, semaCtx, clauses);
+
+  if (auto *cond = get(analysis.cond)) {
+    (void)cond;
+    TODO(loc, "OpenMP ATOMIC COMPARE");
+  } else {
+    int action0 = analysis.op0.what & analysis.Action;
+    int action1 = analysis.op1.what & analysis.Action;
+    mlir::Operation *captureOp = nullptr;
+    fir::FirOpBuilder::InsertPoint preAt = builder.saveInsertionPoint();
+    fir::FirOpBuilder::InsertPoint atomicAt, postAt;
+
+    if (construct.IsCapture()) {
+      // Capturing operation.
+      assert(action0 != analysis.None && action1 != analysis.None &&
+             "Expexcing two actions");
+      (void)action0;
+      (void)action1;
+      captureOp =
+          builder.create<mlir::omp::AtomicCaptureOp>(loc, hint, memOrder);
+      // Set the non-atomic insertion point to before the atomic.capture.
+      preAt = getInsertionPointBefore(captureOp);
+
+      mlir::Block *block = builder.createBlock(&captureOp->getRegion(0));
+      builder.setInsertionPointToEnd(block);
+      // Set the atomic insertion point to before the terminator inside
+      // atomic.capture.
+      mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
+      atomicAt = getInsertionPointBefore(term);
+      postAt = getInsertionPointAfter(captureOp);
+      hint = nullptr;
+      memOrder = nullptr;
+    } else {
+      // Non-capturing operation.
+      assert(action0 != analysis.None && action1 == analysis.None &&
+             "Expexcing single action");
+      assert(!(analysis.op0.what & analysis.Condition));
+      postAt = atomicAt = preAt;
+    }
+
+    // The builder's insertion point needs to be specifically set before
+    // each call to `genAtomicOperation`.
+    mlir::Operation *firstOp = genAtomicOperation(
+        converter, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
+        *get(analysis.op0.assign), hint, memOrder, preAt, atomicAt, postAt);
+    assert(firstOp && "Should have created an atomic operation");
+    atomicAt = getInsertionPointAfter(firstOp);
+
+    mlir::Operation *secondOp = nullptr;
+    if (analysis.op1.what != analysis.None) {
+      secondOp = genAtomicOperation(converter, loc, stmtCtx, analysis.op1.what,
+                                    atomAddr, atom, *get(analysis.op1.assign),
+                                    hint, memOrder, preAt, atomicAt, postAt);
+    }
+
+    if (construct.IsCapture()) {
+      // If this is a capture operation, the first/second ops will be inside
+      // of it. Set the insertion point to past the capture op itself.
+      builder.restoreInsertionPoint(postAt);
+    } else {
+      if (secondOp) {
+        builder.setInsertionPointAfter(secondOp);
+      } else {
+        builder.setInsertionPointAfter(firstOp);
+      }
+    }
+  }
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
@@ -4239,6 +3946,13 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
           parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
       TODO(clauseLocation, name + " clause is not implemented yet");
     }
+
+    if (std::holds_alternative<clause::Private>(clause.u) &&
+        origDirective == llvm::omp::Directive::OMPD_target_teams)
+      TODO(clauseLocation, "TARGET TEAMS PRIVATE is not implemented yet");
+    if (std::holds_alternative<clause::Private>(clause.u) &&
+        origDirective == llvm::omp::Directive::OMPD_target_parallel)
+      TODO(clauseLocation, "TARGET PARALLEL PRIVATE is not implemented yet");
   }
 
   llvm::omp::Directive directive =
diff --git a/external/llvm-project/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp b/external/llvm-project/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
index b373f75feea8..cc5c6448401b 100644
--- a/external/llvm-project/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
+++ b/external/llvm-project/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
@@ -57,6 +57,7 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
   };
 
   mlir::Type valTy = fir::unwrapRefType(argType);
+  const bool argIsVolatile = fir::isa_volatile_type(argType);
   if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(valTy)) {
     // TODO: what about undoing init of unboxed derived types?
     if (auto recTy = mlir::dyn_cast<fir::RecordType>(
@@ -64,7 +65,7 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
       mlir::Type eleTy = boxTy.getEleTy();
       if (mlir::isa<fir::PointerType, fir::HeapType>(eleTy)) {
         mlir::Type mutableBoxTy =
-            fir::ReferenceType::get(fir::BoxType::get(eleTy));
+            fir::ReferenceType::get(fir::BoxType::get(eleTy), argIsVolatile);
         mlir::Value converted =
             builder.createConvert(loc, mutableBoxTy, block->getArgument(0));
         if (recTy.getNumLenParams() > 0)
diff --git a/external/llvm-project/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/external/llvm-project/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index 9e76223eab1b..fb7a33bc4aa0 100644
--- a/external/llvm-project/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/external/llvm-project/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -387,7 +387,8 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   hlfir::LoopNest nest = hlfir::genLoopNest(
       loc, builder, shapeShift.getExtents(), /*isUnordered=*/true);
   builder.setInsertionPointToStart(nest.body);
-  mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
+  const bool seqIsVolatile = fir::isa_volatile_type(seqTy.getEleTy());
+  mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy(), seqIsVolatile);
   auto lhsEleAddr = builder.create<fir::ArrayCoorOp>(
       loc, refTy, lhs, shapeShift, /*slice=*/mlir::Value{},
       nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});
@@ -657,7 +658,8 @@ void ReductionProcessor::processReductionArguments(
     assert(fir::isa_ref_type(symVal.getType()) &&
            "reduction input var is passed by reference");
     mlir::Type elementType = fir::dyn_cast_ptrEleTy(symVal.getType());
-    mlir::Type refTy = fir::ReferenceType::get(elementType);
+    const bool symIsVolatile = fir::isa_volatile_type(symVal.getType());
+    mlir::Type refTy = fir::ReferenceType::get(elementType, symIsVolatile);
 
     reductionVars.push_back(
         builder.createConvert(currentLocation, refTy, symVal));
diff --git a/external/llvm-project/flang/lib/Lower/Support/Utils.cpp b/external/llvm-project/flang/lib/Lower/Support/Utils.cpp
index 2de9db992e27..f85fec6a9700 100644
--- a/external/llvm-project/flang/lib/Lower/Support/Utils.cpp
+++ b/external/llvm-project/flang/lib/Lower/Support/Utils.cpp
@@ -13,8 +13,14 @@
 #include "flang/Lower/Support/Utils.h"
 
 #include "flang/Common/indirection.h"
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/ConvertVariable.h"
 #include "flang/Lower/IterationSpace.h"
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Semantics/tools.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include <cstdint>
 #include <optional>
 #include <type_traits>
diff --git a/external/llvm-project/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/external/llvm-project/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index d1f930c28a7e..a997a0e83890 100644
--- a/external/llvm-project/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -366,55 +366,29 @@ mlir::Value fir::FirOpBuilder::createHeapTemporary(
                                  name, dynamicLength, dynamicShape, attrs);
 }
 
-std::pair<mlir::Value, bool> fir::FirOpBuilder::createArrayTemp(
-    mlir::Location loc, fir::SequenceType arrayType, mlir::Value shape,
+std::pair<mlir::Value, bool> fir::FirOpBuilder::createAndDeclareTemp(
+    mlir::Location loc, mlir::Type baseType, mlir::Value shape,
     llvm::ArrayRef<mlir::Value> extents, llvm::ArrayRef<mlir::Value> typeParams,
     const std::function<decltype(FirOpBuilder::genTempDeclareOp)> &genDeclare,
     mlir::Value polymorphicMold, bool useStack, llvm::StringRef tmpName) {
   if (polymorphicMold) {
     // Create *allocated* polymorphic temporary using the dynamic type
-    // of the mold and the provided shape/extents. The created temporary
-    // array will be written element per element, that is why it has to be
-    // allocated.
-    mlir::Type boxHeapType = fir::HeapType::get(arrayType);
-    mlir::Value alloc = fir::factory::genNullBoxStorage(
-        *this, loc, fir::ClassType::get(boxHeapType));
-    fir::FortranVariableFlagsAttr declAttrs =
-        fir::FortranVariableFlagsAttr::get(
-            getContext(), fir::FortranVariableFlagsEnum::allocatable);
-
-    mlir::Value base = genDeclare(*this, loc, alloc, tmpName,
-                                  /*shape=*/nullptr, typeParams, declAttrs);
-
-    int rank = extents.size();
-    fir::runtime::genAllocatableApplyMold(*this, loc, alloc, polymorphicMold,
-                                          rank);
-    if (!extents.empty()) {
-      mlir::Type idxTy = getIndexType();
-      mlir::Value one = createIntegerConstant(loc, idxTy, 1);
-      unsigned dim = 0;
-      for (mlir::Value extent : extents) {
-        mlir::Value dimIndex = createIntegerConstant(loc, idxTy, dim++);
-        fir::runtime::genAllocatableSetBounds(*this, loc, alloc, dimIndex, one,
-                                              extent);
-      }
-    }
-    if (!typeParams.empty()) {
-      // We should call AllocatableSetDerivedLength() here.
-      // TODO: does the mold provide the length parameters or
-      // the operation itself or should they be in sync?
-      TODO(loc, "polymorphic type with length parameters");
-    }
-    fir::runtime::genAllocatableAllocate(*this, loc, alloc);
-
+    // of the mold and the provided shape/extents.
+    auto boxType = fir::ClassType::get(fir::HeapType::get(baseType));
+    mlir::Value boxAddress = fir::factory::getAndEstablishBoxStorage(
+        *this, loc, boxType, shape, typeParams, polymorphicMold);
+    fir::runtime::genAllocatableAllocate(*this, loc, boxAddress);
+    mlir::Value box = create<fir::LoadOp>(loc, boxAddress);
+    mlir::Value base =
+        genDeclare(*this, loc, box, tmpName, /*shape=*/mlir::Value{},
+                   typeParams, fir::FortranVariableFlagsAttr{});
     return {base, /*isHeapAllocation=*/true};
   }
   mlir::Value allocmem;
   if (useStack)
-    allocmem = createTemporary(loc, arrayType, tmpName, extents, typeParams);
+    allocmem = createTemporary(loc, baseType, tmpName, extents, typeParams);
   else
-    allocmem =
-        createHeapTemporary(loc, arrayType, tmpName, extents, typeParams);
+    allocmem = createHeapTemporary(loc, baseType, tmpName, extents, typeParams);
   mlir::Value base = genDeclare(*this, loc, allocmem, tmpName, shape,
                                 typeParams, fir::FortranVariableFlagsAttr{});
   return {base, !useStack};
diff --git a/external/llvm-project/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/external/llvm-project/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index f24dc2caeedf..b54cccbef14f 100644
--- a/external/llvm-project/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/IRMapping.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include <mlir/Dialect/LLVMIR/LLVMAttrs.h>
 #include <mlir/Dialect/OpenMP/OpenMPDialect.h>
 #include <optional>
 
@@ -932,7 +933,8 @@ mlir::Value hlfir::inlineElementalOp(
 hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc,
                                    fir::FirOpBuilder &builder,
                                    mlir::ValueRange extents, bool isUnordered,
-                                   bool emitWorkshareLoop) {
+                                   bool emitWorkshareLoop,
+                                   bool couldVectorize) {
   emitWorkshareLoop = emitWorkshareLoop && isUnordered;
   hlfir::LoopNest loopNest;
   assert(!extents.empty() && "must have at least one extent");
@@ -967,6 +969,15 @@ hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc,
       auto ub = builder.createConvert(loc, indexType, extent);
       auto doLoop =
           builder.create<fir::DoLoopOp>(loc, one, ub, one, isUnordered);
+      if (!couldVectorize) {
+        mlir::LLVM::LoopVectorizeAttr va{mlir::LLVM::LoopVectorizeAttr::get(
+            builder.getContext(),
+            /*disable=*/builder.getBoolAttr(true), {}, {}, {}, {}, {}, {})};
+        mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get(
+            builder.getContext(), {}, /*vectorize=*/va, {}, /*unroll*/ {},
+            /*unroll_and_jam*/ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});
+        doLoop.setLoopAnnotationAttr(la);
+      }
       loopNest.body = doLoop.getBody();
       builder.setInsertionPointToStart(loopNest.body);
       // Reverse the indices so they are in column-major order.
@@ -1331,63 +1342,36 @@ bool hlfir::elementalOpMustProduceTemp(hlfir::ElementalOp elemental) {
 std::pair<hlfir::Entity, mlir::Value>
 hlfir::createTempFromMold(mlir::Location loc, fir::FirOpBuilder &builder,
                           hlfir::Entity mold) {
+  assert(!mold.isAssumedRank() &&
+         "cannot create temporary from assumed-rank mold");
   llvm::SmallVector<mlir::Value> lenParams;
   hlfir::genLengthParameters(loc, builder, mold, lenParams);
   llvm::StringRef tmpName{".tmp"};
-  mlir::Value alloc;
-  mlir::Value isHeapAlloc;
-  mlir::Value shape{};
-  fir::FortranVariableFlagsAttr declAttrs;
 
-  if (mold.isPolymorphic()) {
-    // Create unallocated polymorphic temporary using the dynamic type
-    // of the mold. The static type of the temporary matches
-    // the static type of the mold, but then the dynamic type
-    // of the mold is applied to the temporary's descriptor.
-
-    if (mold.isArray())
-      hlfir::genShape(loc, builder, mold);
-
-    // Create polymorphic allocatable box on the stack.
-    mlir::Type boxHeapType = fir::HeapType::get(fir::unwrapRefType(
-        mlir::cast<fir::BaseBoxType>(mold.getType()).getEleTy()));
-    // The box must be initialized, because AllocatableApplyMold
-    // may read its contents (e.g. for checking whether it is allocated).
-    alloc = fir::factory::genNullBoxStorage(builder, loc,
-                                            fir::ClassType::get(boxHeapType));
-    // The temporary is unallocated even after AllocatableApplyMold below.
-    // If the temporary is used as assignment LHS it will be automatically
-    // allocated on the heap, as long as we use Assign family
-    // runtime functions. So set MustFree to true.
-    isHeapAlloc = builder.createBool(loc, true);
-    declAttrs = fir::FortranVariableFlagsAttr::get(
-        builder.getContext(), fir::FortranVariableFlagsEnum::allocatable);
-  } else if (mold.isArray()) {
-    mlir::Type sequenceType =
-        hlfir::getFortranElementOrSequenceType(mold.getType());
+  mlir::Value shape{};
+  llvm::SmallVector<mlir::Value> extents;
+  if (mold.isArray()) {
     shape = hlfir::genShape(loc, builder, mold);
-    auto extents = hlfir::getIndexExtents(loc, builder, shape);
-    alloc = builder.createHeapTemporary(loc, sequenceType, tmpName, extents,
-                                        lenParams);
-    isHeapAlloc = builder.createBool(loc, true);
-  } else {
-    alloc = builder.createTemporary(loc, mold.getFortranElementType(), tmpName,
-                                    /*shape=*/std::nullopt, lenParams);
-    isHeapAlloc = builder.createBool(loc, false);
-  }
-  auto declareOp =
-      builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape, lenParams,
-                                       /*dummy_scope=*/nullptr, declAttrs);
-  if (mold.isPolymorphic()) {
-    int rank = mold.getRank();
-    // TODO: should probably read rank from the mold.
-    if (rank < 0)
-      TODO(loc, "create temporary for assumed rank polymorphic");
-    fir::runtime::genAllocatableApplyMold(builder, loc, alloc,
-                                          mold.getFirBase(), rank);
+    extents = hlfir::getExplicitExtentsFromShape(shape, builder);
   }
 
-  return {hlfir::Entity{declareOp.getBase()}, isHeapAlloc};
+  bool useStack = !mold.isArray() && !mold.isPolymorphic();
+  auto genTempDeclareOp =
+      [](fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value memref,
+         llvm::StringRef name, mlir::Value shape,
+         llvm::ArrayRef<mlir::Value> typeParams,
+         fir::FortranVariableFlagsAttr attrs) -> mlir::Value {
+    auto declareOp =
+        builder.create<hlfir::DeclareOp>(loc, memref, name, shape, typeParams,
+                                         /*dummy_scope=*/nullptr, attrs);
+    return declareOp.getBase();
+  };
+
+  auto [base, isHeapAlloc] = builder.createAndDeclareTemp(
+      loc, mold.getElementOrSequenceType(), shape, extents, lenParams,
+      genTempDeclareOp, mold.isPolymorphic() ? mold.getBase() : nullptr,
+      useStack, tmpName);
+  return {hlfir::Entity{base}, builder.createBool(loc, isHeapAlloc)};
 }
 
 hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
diff --git a/external/llvm-project/flang/lib/Optimizer/Builder/MutableBox.cpp b/external/llvm-project/flang/lib/Optimizer/Builder/MutableBox.cpp
index e6d630412ec3..f20b5e37dffd 100644
--- a/external/llvm-project/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -349,10 +349,7 @@ mlir::Value fir::factory::createUnallocatedBox(
   const bool isAssumedRank = baseBoxType.isAssumedRank();
   if (isAssumedRank)
     baseBoxType = baseBoxType.getBoxTypeWithNewShape(/*rank=*/0);
-  auto baseAddrType = baseBoxType.getEleTy();
-  if (!fir::isa_ref_type(baseAddrType))
-    baseAddrType =
-        builder.getRefType(baseAddrType, fir::isa_volatile_type(baseBoxType));
+  auto baseAddrType = baseBoxType.getBaseAddressType();
   auto type = fir::unwrapRefType(baseAddrType);
   auto eleTy = fir::unwrapSequenceType(type);
   if (auto recTy = mlir::dyn_cast<fir::RecordType>(eleTy))
@@ -982,3 +979,20 @@ mlir::Value fir::factory::genNullBoxStorage(fir::FirOpBuilder &builder,
   builder.create<fir::StoreOp>(loc, nullBox, boxStorage);
   return boxStorage;
 }
+
+mlir::Value fir::factory::getAndEstablishBoxStorage(
+    fir::FirOpBuilder &builder, mlir::Location loc, fir::BaseBoxType boxTy,
+    mlir::Value shape, llvm::ArrayRef<mlir::Value> typeParams,
+    mlir::Value polymorphicMold) {
+  mlir::Value boxStorage = builder.createTemporary(loc, boxTy);
+  mlir::Value nullAddr =
+      builder.createNullConstant(loc, boxTy.getBaseAddressType());
+  mlir::Value box =
+      builder.create<fir::EmboxOp>(loc, boxTy, nullAddr, shape,
+                                   /*emptySlice=*/mlir::Value{},
+                                   fir::factory::elideLengthsAlreadyInType(
+                                       boxTy.unwrapInnerType(), typeParams),
+                                   polymorphicMold);
+  builder.create<fir::StoreOp>(loc, box, boxStorage);
+  return boxStorage;
+}
diff --git a/external/llvm-project/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp b/external/llvm-project/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
index 90662c094c65..a943469a7672 100644
--- a/external/llvm-project/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
@@ -32,3 +32,18 @@ void fir::runtime::cuda::genSyncGlobalDescriptor(fir::FirOpBuilder &builder,
       builder, loc, fTy, hostPtr, sourceFile, sourceLine)};
   builder.create<fir::CallOp>(loc, callee, args);
 }
+
+void fir::runtime::cuda::genDescriptorCheckSection(fir::FirOpBuilder &builder,
+                                                   mlir::Location loc,
+                                                   mlir::Value desc) {
+  mlir::func::FuncOp func =
+      fir::runtime::getRuntimeFunc<mkRTKey(CUFDescriptorCheckSection)>(loc,
+                                                                       builder);
+  auto fTy = func.getFunctionType();
+  mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+  mlir::Value sourceLine =
+      fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
+  llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
+      builder, loc, fTy, desc, sourceFile, sourceLine)};
+  builder.create<fir::CallOp>(loc, func, args);
+}
diff --git a/external/llvm-project/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/external/llvm-project/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index 82b11ad7db32..69bdb48146a5 100644
--- a/external/llvm-project/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -274,12 +274,12 @@ class BoxedProcedurePass
             auto loc = embox.getLoc();
             mlir::Type i8Ty = builder.getI8Type();
             mlir::Type i8Ptr = builder.getRefType(i8Ty);
-            // For AArch64, PPC32 and PPC64, the thunk is populated by a call to
+            // For PPC32 and PPC64, the thunk is populated by a call to
             // __trampoline_setup, which is defined in
             // compiler-rt/lib/builtins/trampoline_setup.c and requires the
-            // thunk size greater than 32 bytes.  For RISCV and x86_64, the
-            // thunk setup doesn't go through __trampoline_setup and fits in 32
-            // bytes.
+            // thunk size greater than 32 bytes.  For AArch64, RISCV and x86_64,
+            // the thunk setup doesn't go through __trampoline_setup and fits in
+            // 32 bytes.
             fir::SequenceType::Extent thunkSize = triple.getTrampolineSize();
             mlir::Type buffTy = SequenceType::get({thunkSize}, i8Ty);
             auto buffer = builder.create<AllocaOp>(loc, buffTy);
diff --git a/external/llvm-project/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/external/llvm-project/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index d1757febd4bf..b6269c11265b 100644
--- a/external/llvm-project/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -3341,6 +3341,30 @@ struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
   }
 };
 
+struct LocalitySpecifierOpConversion
+    : public fir::FIROpConversion<fir::LocalitySpecifierOp> {
+  using FIROpConversion::FIROpConversion;
+  llvm::LogicalResult
+  matchAndRewrite(fir::LocalitySpecifierOp localizer, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+#ifdef EXPENSIVE_CHECKS
+    auto uses = mlir::SymbolTable::getSymbolUses(
+        localizer, localizer->getParentOfType<mlir::ModuleOp>());
+
+    // `fir.local` ops are not supposed to have any uses at this point (i.e.
+    // during lowering to LLVM). In case of serialization, the
+    // `fir.do_concurrent` users are expected to have been lowered to
+    // `fir.do_loop` nests. In case of parallelization, the `fir.do_concurrent`
+    // users are expected to have been lowered to the target parallel model
+    // (e.g. OpenMP).
+    assert(uses && uses->empty());
+#endif
+
+    rewriter.eraseOp(localizer);
+    return mlir::success();
+  }
+};
+
 /// Lower `fir.no_reassoc` to LLVM IR dialect.
 /// TODO: how do we want to enforce this in LLVM-IR? Can we manipulate the fast
 /// math flags?
@@ -3977,12 +4001,25 @@ struct BoxOffsetOpConversion : public fir::FIROpConversion<fir::BoxOffsetOp> {
                   mlir::ConversionPatternRewriter &rewriter) const override {
 
     mlir::Type pty = ::getLlvmPtrType(boxOffset.getContext());
-    mlir::Type boxType = fir::unwrapRefType(boxOffset.getBoxRef().getType());
-    mlir::Type llvmBoxTy =
-        lowerTy().convertBoxTypeAsStruct(mlir::cast<fir::BaseBoxType>(boxType));
-    int fieldId = boxOffset.getField() == fir::BoxFieldAttr::derived_type
-                      ? getTypeDescFieldId(boxType)
-                      : kAddrPosInBox;
+    mlir::Type boxRefType = fir::unwrapRefType(boxOffset.getBoxRef().getType());
+
+    assert((mlir::isa<fir::BaseBoxType>(boxRefType) ||
+            mlir::isa<fir::BoxCharType>(boxRefType)) &&
+           "boxRef should be a reference to either fir.box or fir.boxchar");
+
+    mlir::Type llvmBoxTy;
+    int fieldId;
+    if (auto boxType = mlir::dyn_cast_or_null<fir::BaseBoxType>(boxRefType)) {
+      llvmBoxTy = lowerTy().convertBoxTypeAsStruct(
+          mlir::cast<fir::BaseBoxType>(boxType));
+      fieldId = boxOffset.getField() == fir::BoxFieldAttr::derived_type
+                    ? getTypeDescFieldId(boxType)
+                    : kAddrPosInBox;
+    } else {
+      auto boxCharType = mlir::cast<fir::BoxCharType>(boxRefType);
+      llvmBoxTy = lowerTy().convertType(boxCharType);
+      fieldId = kAddrPosInBox;
+    }
     rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
         boxOffset, pty, llvmBoxTy, adaptor.getBoxRef(),
         llvm::ArrayRef<mlir::LLVM::GEPArg>{0, fieldId});
@@ -4283,15 +4320,15 @@ void fir::populateFIRToLLVMConversionPatterns(
       FieldIndexOpConversion, FirEndOpConversion, FreeMemOpConversion,
       GlobalLenOpConversion, GlobalOpConversion, InsertOnRangeOpConversion,
       IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
-      MulcOpConversion, NegcOpConversion, NoReassocOpConversion,
-      SelectCaseOpConversion, SelectOpConversion, SelectRankOpConversion,
-      SelectTypeOpConversion, ShapeOpConversion, ShapeShiftOpConversion,
-      ShiftOpConversion, SliceOpConversion, StoreOpConversion,
-      StringLitOpConversion, SubcOpConversion, TypeDescOpConversion,
-      TypeInfoOpConversion, UnboxCharOpConversion, UnboxProcOpConversion,
-      UndefOpConversion, UnreachableOpConversion, XArrayCoorOpConversion,
-      XEmboxOpConversion, XReboxOpConversion, ZeroOpConversion>(converter,
-                                                                options);
+      LocalitySpecifierOpConversion, MulcOpConversion, NegcOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
+      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
+      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
+      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
+      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
+      XArrayCoorOpConversion, XEmboxOpConversion, XReboxOpConversion,
+      ZeroOpConversion>(converter, options);
 
   // Patterns that are populated without a type converter do not trigger
   // target materializations for the operands of the root op.
diff --git a/external/llvm-project/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp b/external/llvm-project/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
index 7fb713ff1a6c..de97a0bbc184 100644
--- a/external/llvm-project/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
@@ -190,12 +190,10 @@ mlir::Value PackArrayConversion::allocateTempBuffer(
   if (useStack && canAllocateTempOnStack(origBox))
     assert(!isHeapAllocation && "temp must have been allocated on the stack");
 
-  if (isHeapAllocation)
-    if (auto baseType = mlir::dyn_cast<fir::ReferenceType>(base.getType()))
-      if (mlir::isa<fir::BaseBoxType>(baseType.getEleTy()))
-        return builder.create<fir::LoadOp>(loc, base);
-
   mlir::Type ptrType = base.getType();
+  if (llvm::isa<fir::BaseBoxType>(ptrType))
+    return base;
+
   mlir::Type tempBoxType = fir::BoxType::get(mlir::isa<fir::HeapType>(ptrType)
                                                  ? ptrType
                                                  : fir::unwrapRefType(ptrType));
diff --git a/external/llvm-project/flang/lib/Optimizer/Dialect/FIROps.cpp b/external/llvm-project/flang/lib/Optimizer/Dialect/FIROps.cpp
index cbe93907265f..6181e1fad424 100644
--- a/external/llvm-project/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -4484,15 +4484,24 @@ void fir::IfOp::resultToSourceOps(llvm::SmallVectorImpl<mlir::Value> &results,
 llvm::LogicalResult fir::BoxOffsetOp::verify() {
   auto boxType = mlir::dyn_cast_or_null<fir::BaseBoxType>(
       fir::dyn_cast_ptrEleTy(getBoxRef().getType()));
-  if (!boxType)
-    return emitOpError("box_ref operand must have !fir.ref<!fir.box<T>> type");
+  mlir::Type boxCharType;
+  if (!boxType) {
+    boxCharType = mlir::dyn_cast_or_null<fir::BoxCharType>(
+        fir::dyn_cast_ptrEleTy(getBoxRef().getType()));
+    if (!boxCharType)
+      return emitOpError("box_ref operand must have !fir.ref<!fir.box<T>> or "
+                         "!fir.ref<!fir.boxchar<k>> type");
+    if (getField() == fir::BoxFieldAttr::derived_type)
+      return emitOpError("cannot address derived_type field of a fir.boxchar");
+  }
   if (getField() != fir::BoxFieldAttr::base_addr &&
       getField() != fir::BoxFieldAttr::derived_type)
     return emitOpError("cannot address provided field");
-  if (getField() == fir::BoxFieldAttr::derived_type)
+  if (getField() == fir::BoxFieldAttr::derived_type) {
     if (!fir::boxHasAddendum(boxType))
       return emitOpError("can only address derived_type field of derived type "
                          "or unlimited polymorphic fir.box");
+  }
   return mlir::success();
 }
 
diff --git a/external/llvm-project/flang/lib/Optimizer/Dialect/FIRType.cpp b/external/llvm-project/flang/lib/Optimizer/Dialect/FIRType.cpp
index 1e6e95393c2f..78571f1f4bc2 100644
--- a/external/llvm-project/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -255,7 +255,7 @@ mlir::Type dyn_cast_ptrOrBoxEleTy(mlir::Type t) {
   return llvm::TypeSwitch<mlir::Type, mlir::Type>(t)
       .Case<fir::ReferenceType, fir::PointerType, fir::HeapType,
             fir::LLVMPointerType>([](auto p) { return p.getEleTy(); })
-      .Case<fir::BaseBoxType>(
+      .Case<fir::BaseBoxType, fir::BoxCharType>(
           [](auto p) { return unwrapRefType(p.getEleTy()); })
       .Default([](mlir::Type) { return mlir::Type{}; });
 }
@@ -1407,6 +1407,13 @@ mlir::Type BaseBoxType::getEleTy() const {
           [](auto type) { return type.getEleTy(); });
 }
 
+mlir::Type BaseBoxType::getBaseAddressType() const {
+  mlir::Type eleTy = getEleTy();
+  if (fir::isa_ref_type(eleTy))
+    return eleTy;
+  return fir::ReferenceType::get(eleTy, isVolatile());
+}
+
 mlir::Type BaseBoxType::unwrapInnerType() const {
   return fir::unwrapInnerType(getEleTy());
 }
@@ -1475,10 +1482,12 @@ fir::BaseBoxType fir::BaseBoxType::getBoxTypeWithNewAttr(
     break;
   }
   return llvm::TypeSwitch<fir::BaseBoxType, fir::BaseBoxType>(*this)
-      .Case<fir::BoxType>(
-          [baseType](auto) { return fir::BoxType::get(baseType); })
-      .Case<fir::ClassType>(
-          [baseType](auto) { return fir::ClassType::get(baseType); });
+      .Case<fir::BoxType>([baseType](auto b) {
+        return fir::BoxType::get(baseType, b.isVolatile());
+      })
+      .Case<fir::ClassType>([baseType](auto b) {
+        return fir::ClassType::get(baseType, b.isVolatile());
+      });
 }
 
 bool fir::BaseBoxType::isAssumedRank() const {
@@ -1492,6 +1501,12 @@ bool fir::BaseBoxType::isPointer() const {
   return llvm::isa<fir::PointerType>(getEleTy());
 }
 
+bool fir::BaseBoxType::isPointerOrAllocatable() const {
+  return llvm::isa<fir::PointerType, fir::HeapType>(getEleTy());
+}
+
+bool BaseBoxType::isVolatile() const { return fir::isa_volatile_type(*this); }
+
 //===----------------------------------------------------------------------===//
 // FIROpsDialect
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/external/llvm-project/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index 8cfca59ecdad..df6ce12215d2 100644
--- a/external/llvm-project/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -180,16 +180,26 @@ void hlfir::AssignOp::getEffects(
 // DeclareOp
 //===----------------------------------------------------------------------===//
 
-/// Given a FIR memory type, and information about non default lower bounds, get
-/// the related HLFIR variable type.
-mlir::Type hlfir::DeclareOp::getHLFIRVariableType(mlir::Type inputType,
-                                                  bool hasExplicitLowerBounds) {
+static std::pair<mlir::Type, mlir::Type>
+getDeclareOutputTypes(mlir::Type inputType, bool hasExplicitLowerBounds) {
+  // Drop pointer/allocatable attribute of descriptor values. Only descriptor
+  // addresses are ALLOCATABLE/POINTER. The HLFIR box result of an hlfir.declare
+  // without those attributes should not have these attributes set.
+  if (auto baseBoxType = mlir::dyn_cast<fir::BaseBoxType>(inputType))
+    if (baseBoxType.isPointerOrAllocatable()) {
+      mlir::Type boxWithoutAttributes =
+          baseBoxType.getBoxTypeWithNewAttr(fir::BaseBoxType::Attribute::None);
+      return {boxWithoutAttributes, boxWithoutAttributes};
+    }
   mlir::Type type = fir::unwrapRefType(inputType);
   if (mlir::isa<fir::BaseBoxType>(type))
-    return inputType;
+    return {inputType, inputType};
   if (auto charType = mlir::dyn_cast<fir::CharacterType>(type))
-    if (charType.hasDynamicLen())
-      return fir::BoxCharType::get(charType.getContext(), charType.getFKind());
+    if (charType.hasDynamicLen()) {
+      mlir::Type hlfirType =
+          fir::BoxCharType::get(charType.getContext(), charType.getFKind());
+      return {hlfirType, inputType};
+    }
 
   auto seqType = mlir::dyn_cast<fir::SequenceType>(type);
   bool hasDynamicExtents =
@@ -197,9 +207,19 @@ mlir::Type hlfir::DeclareOp::getHLFIRVariableType(mlir::Type inputType,
   mlir::Type eleType = seqType ? seqType.getEleTy() : type;
   bool hasDynamicLengthParams = fir::characterWithDynamicLen(eleType) ||
                                 fir::isRecordWithTypeParameters(eleType);
-  if (hasExplicitLowerBounds || hasDynamicExtents || hasDynamicLengthParams)
-    return fir::BoxType::get(type, fir::isa_volatile_type(inputType));
-  return inputType;
+  if (hasExplicitLowerBounds || hasDynamicExtents || hasDynamicLengthParams) {
+    mlir::Type boxType =
+        fir::BoxType::get(type, fir::isa_volatile_type(inputType));
+    return {boxType, inputType};
+  }
+  return {inputType, inputType};
+}
+
+/// Given a FIR memory type, and information about non default lower bounds, get
+/// the related HLFIR variable type.
+mlir::Type hlfir::DeclareOp::getHLFIRVariableType(mlir::Type inputType,
+                                                  bool hasExplicitLowerBounds) {
+  return getDeclareOutputTypes(inputType, hasExplicitLowerBounds).first;
 }
 
 static bool hasExplicitLowerBounds(mlir::Value shape) {
@@ -256,17 +276,18 @@ void hlfir::DeclareOp::build(mlir::OpBuilder &builder,
     std::tie(inputType, memref) = updateDeclaredInputTypeWithVolatility(
         inputType, memref, builder, flags);
   }
-  mlir::Type hlfirVariableType =
-      getHLFIRVariableType(inputType, hasExplicitLbs);
-  build(builder, result, {hlfirVariableType, inputType}, memref, shape,
+  auto [hlfirVariableType, firVarType] =
+      getDeclareOutputTypes(inputType, hasExplicitLbs);
+  build(builder, result, {hlfirVariableType, firVarType}, memref, shape,
         typeparams, dummy_scope, nameAttr, fortran_attrs, data_attr);
 }
 
 llvm::LogicalResult hlfir::DeclareOp::verify() {
-  if (getMemref().getType() != getResult(1).getType())
-    return emitOpError("second result type must match input memref type");
-  mlir::Type hlfirVariableType = getHLFIRVariableType(
+  auto [hlfirVariableType, firVarType] = getDeclareOutputTypes(
       getMemref().getType(), hasExplicitLowerBounds(getShape()));
+  if (firVarType != getResult(1).getType())
+    return emitOpError("second result type must match input memref type, "
+                       "unless it is a box with heap or pointer attribute");
   if (hlfirVariableType != getResult(0).getType())
     return emitOpError("first result type is inconsistent with variable "
                        "properties: expected ")
@@ -1586,7 +1607,7 @@ void hlfir::AssociateOp::build(mlir::OpBuilder &builder,
   mlir::Type firVarType;
   auto sourceExprType = mlir::dyn_cast<hlfir::ExprType>(source.getType());
   if (sourceExprType && sourceExprType.isPolymorphic())
-    firVarType = fir::ClassType::get(fir::HeapType::get(dataType));
+    firVarType = fir::ClassType::get(dataType);
   else
     firVarType = fir::ReferenceType::get(dataType);
 
@@ -1608,7 +1629,7 @@ void hlfir::AssociateOp::build(
   mlir::Type firVarType;
   auto sourceExprType = mlir::dyn_cast<hlfir::ExprType>(source.getType());
   if (sourceExprType && sourceExprType.isPolymorphic())
-    firVarType = fir::ClassType::get(fir::HeapType::get(dataType));
+    firVarType = fir::ClassType::get(dataType);
   else
     firVarType = fir::ReferenceType::get(dataType);
 
diff --git a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 8a36214def16..58f2b5771297 100644
--- a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -125,7 +125,9 @@ createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder,
   auto [base, isHeapAlloc] = builder.createArrayTemp(
       loc, sequenceType, shape, extents, lenParams, genTempDeclareOp,
       polymorphicMold ? polymorphicMold->getFirBase() : nullptr);
-  return {hlfir::Entity{base}, builder.createBool(loc, isHeapAlloc)};
+  hlfir::Entity temp = hlfir::Entity{base};
+  assert(!temp.isAllocatable() && "temp must have been allocated");
+  return {temp, builder.createBool(loc, isHeapAlloc)};
 }
 
 /// Copy \p source into a new temporary and package the temporary into a
@@ -134,13 +136,10 @@ static mlir::Value copyInTempAndPackage(mlir::Location loc,
                                         fir::FirOpBuilder &builder,
                                         hlfir::Entity source) {
   auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source);
-  builder.create<hlfir::AssignOp>(loc, source, temp, temp.isAllocatable(),
+  assert(!temp.isAllocatable() && "expect temp to already be allocated");
+  builder.create<hlfir::AssignOp>(loc, source, temp, /*realloc=*/false,
                                   /*keep_lhs_length_if_realloc=*/false,
                                   /*temporary_lhs=*/true);
-  // Dereference allocatable temporary directly to simplify processing
-  // of its uses.
-  if (temp.isAllocatable())
-    temp = hlfir::derefPointersAndAllocatables(loc, builder, temp);
   return packageBufferizedExpr(loc, builder, temp, cleanup);
 }
 
@@ -442,18 +441,10 @@ struct AssociateOpConversion
       // !fir.box<!fir.heap<!fir.type<_T{y:i32}>>> value must be
       // propagated as the box address !fir.ref<!fir.type<_T{y:i32}>>.
       auto adjustVar = [&](mlir::Value sourceVar, mlir::Type assocType) {
-        if (mlir::isa<fir::ReferenceType>(sourceVar.getType()) &&
-            mlir::isa<fir::ClassType>(
-                fir::unwrapRefType(sourceVar.getType()))) {
-          // Association of a polymorphic value.
-          sourceVar = builder.create<fir::LoadOp>(loc, sourceVar);
-          assert(mlir::isa<fir::ClassType>(sourceVar.getType()) &&
-                 fir::isAllocatableType(sourceVar.getType()));
-          assert(sourceVar.getType() == assocType);
-        } else if ((mlir::isa<fir::BaseBoxType>(sourceVar.getType()) &&
-                    !mlir::isa<fir::BaseBoxType>(assocType)) ||
-                   ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
-                     !mlir::isa<fir::BoxCharType>(assocType)))) {
+        if ((mlir::isa<fir::BaseBoxType>(sourceVar.getType()) &&
+             !mlir::isa<fir::BaseBoxType>(assocType)) ||
+            ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
+              !mlir::isa<fir::BoxCharType>(assocType)))) {
           sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
         } else {
           sourceVar = builder.createConvert(loc, assocType, sourceVar);
@@ -549,23 +540,7 @@ static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder,
     // fir::FreeMemOp operand type must be a fir::HeapType.
     mlir::Type heapType = fir::HeapType::get(
         hlfir::getFortranElementOrSequenceType(var.getType()));
-    if (mlir::isa<fir::ReferenceType>(var.getType()) &&
-        mlir::isa<fir::ClassType>(fir::unwrapRefType(var.getType()))) {
-      // A temporary for a polymorphic expression is represented
-      // via an allocatable. Variable type in this case
-      // is !fir.ref<!fir.class<!fir.heap<!fir.type<>>>>.
-      // We need to free the allocatable data, not the box
-      // that is allocated on the stack.
-      var = builder.create<fir::LoadOp>(loc, var);
-      assert(mlir::isa<fir::ClassType>(var.getType()) &&
-             fir::isAllocatableType(var.getType()));
-      addr = builder.create<fir::BoxAddrOp>(loc, heapType, var);
-      // Lowering currently does not produce DestroyOp with 'finalize'
-      // for polymorphic temporaries. It will have to do so, for example,
-      // for MERGE with polymorphic results.
-      if (mustFinalize)
-        TODO(loc, "finalizing polymorphic temporary in HLFIR");
-    } else if (mlir::isa<fir::BaseBoxType, fir::BoxCharType>(var.getType())) {
+    if (mlir::isa<fir::BaseBoxType, fir::BoxCharType>(var.getType())) {
       if (mustFinalize && !mlir::isa<fir::BaseBoxType>(var.getType()))
         fir::emitFatalError(loc, "non-finalizable variable");
 
@@ -757,9 +732,6 @@ struct ElementalOpConversion
                                               adaptor.getTypeparams().end());
     auto [temp, cleanup] = createArrayTemp(loc, builder, elemental.getType(),
                                            shape, extents, typeParams, mold);
-    // If the box load is needed, we'd better place it outside
-    // of the loop nest.
-    temp = derefPointersAndAllocatables(loc, builder, temp);
 
     if (optimizeEmptyElementals)
       extents = fir::factory::updateRuntimeExtentsForEmptyArrays(builder, loc,
diff --git a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
index d959428ebd20..cc74273d9c5d 100644
--- a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
+++ b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
@@ -5,6 +5,7 @@ add_flang_library(HLFIRTransforms
   ConvertToFIR.cpp
   InlineElementals.cpp
   InlineHLFIRAssign.cpp
+  InlineHLFIRCopyIn.cpp
   LowerHLFIRIntrinsics.cpp
   LowerHLFIROrderedAssignments.cpp
   ScheduleOrderedAssignments.cpp
diff --git a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 8f206b5a1ade..f7efaa736a27 100644
--- a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -356,7 +356,7 @@ class DeclareOpConversion : public mlir::OpRewritePattern<hlfir::DeclareOp> {
         // is used for accessing the bounds etc. Using the HLFIR box,
         // that holds the same base_addr at this point, makes
         // the representation a little bit more clear.
-        if (hlfirBase.getType() == firBase.getType())
+        if (hlfirBase.getType() == declareOp.getOriginalBase().getType())
           firBase = hlfirBase;
       } else {
         // Need to conditionally rebox/embox the optional: the input fir.box
diff --git a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
index 6e209cce07ad..6c4a07be52a4 100644
--- a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
@@ -109,9 +109,14 @@ class InlineHLFIRAssignConversion
     builder.setInsertionPoint(assign);
     rhs = hlfir::derefPointersAndAllocatables(loc, builder, rhs);
     lhs = hlfir::derefPointersAndAllocatables(loc, builder, lhs);
-    mlir::Value shape = hlfir::genShape(loc, builder, lhs);
+    mlir::Value lhsShape = hlfir::genShape(loc, builder, lhs);
+    llvm::SmallVector<mlir::Value> lhsExtents =
+        hlfir::getIndexExtents(loc, builder, lhsShape);
+    mlir::Value rhsShape = hlfir::genShape(loc, builder, rhs);
+    llvm::SmallVector<mlir::Value> rhsExtents =
+        hlfir::getIndexExtents(loc, builder, rhsShape);
     llvm::SmallVector<mlir::Value> extents =
-        hlfir::getIndexExtents(loc, builder, shape);
+        fir::factory::deduceOptimalExtents(lhsExtents, rhsExtents);
     hlfir::LoopNest loopNest =
         hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
                            flangomp::shouldUseWorkshareLowering(assign));
diff --git a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
new file mode 100644
index 000000000000..7e8acc515ee2
--- /dev/null
+++ b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
@@ -0,0 +1,183 @@
+//===- InlineHLFIRCopyIn.cpp - Inline hlfir.copy_in ops -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Transform hlfir.copy_in array operations into loop nests performing element
+// per element assignments. For simplicity, the inlining is done for trivial
+// data types when the copy_in does not require a corresponding copy_out and
+// when the input array is not behind a pointer. This may change in the future.
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Optimizer/OpenMP/Passes.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Support/LLVM.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace hlfir {
+#define GEN_PASS_DEF_INLINEHLFIRCOPYIN
+#include "flang/Optimizer/HLFIR/Passes.h.inc"
+} // namespace hlfir
+
+#define DEBUG_TYPE "inline-hlfir-copy-in"
+
+static llvm::cl::opt<bool> noInlineHLFIRCopyIn(
+    "no-inline-hlfir-copy-in",
+    llvm::cl::desc("Do not inline hlfir.copy_in operations"),
+    llvm::cl::init(false));
+
+namespace {
+class InlineCopyInConversion : public mlir::OpRewritePattern<hlfir::CopyInOp> {
+public:
+  using mlir::OpRewritePattern<hlfir::CopyInOp>::OpRewritePattern;
+
+  llvm::LogicalResult
+  matchAndRewrite(hlfir::CopyInOp copyIn,
+                  mlir::PatternRewriter &rewriter) const override;
+};
+
+llvm::LogicalResult
+InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
+                                        mlir::PatternRewriter &rewriter) const {
+  fir::FirOpBuilder builder(rewriter, copyIn.getOperation());
+  mlir::Location loc = copyIn.getLoc();
+  hlfir::Entity inputVariable{copyIn.getVar()};
+  mlir::Type resultAddrType = copyIn.getCopiedIn().getType();
+  if (!fir::isa_trivial(inputVariable.getFortranElementType()))
+    return rewriter.notifyMatchFailure(copyIn,
+                                       "CopyInOp's data type is not trivial");
+
+  // There should be exactly one user of WasCopied - the corresponding
+  // CopyOutOp.
+  if (!copyIn.getWasCopied().hasOneUse())
+    return rewriter.notifyMatchFailure(
+        copyIn, "CopyInOp's WasCopied has no single user");
+  // The copy out should always be present, either to actually copy or just
+  // deallocate memory.
+  auto copyOut = mlir::dyn_cast<hlfir::CopyOutOp>(
+      copyIn.getWasCopied().user_begin().getCurrent().getUser());
+
+  if (!copyOut)
+    return rewriter.notifyMatchFailure(copyIn,
+                                       "CopyInOp has no direct CopyOut");
+
+  if (mlir::cast<fir::BaseBoxType>(resultAddrType).isAssumedRank())
+    return rewriter.notifyMatchFailure(copyIn,
+                                       "The result array is assumed-rank");
+
+  // Only inline the copy_in when copy_out does not need to be done, i.e. in
+  // case of intent(in).
+  if (copyOut.getVar())
+    return rewriter.notifyMatchFailure(copyIn, "CopyIn needs a copy-out");
+
+  inputVariable =
+      hlfir::derefPointersAndAllocatables(loc, builder, inputVariable);
+  mlir::Type sequenceType =
+      hlfir::getFortranElementOrSequenceType(inputVariable.getType());
+  fir::BoxType resultBoxType = fir::BoxType::get(sequenceType);
+  mlir::Value isContiguous =
+      builder.create<fir::IsContiguousBoxOp>(loc, inputVariable);
+  mlir::Operation::result_range results =
+      builder
+          .genIfOp(loc, {resultBoxType, builder.getI1Type()}, isContiguous,
+                   /*withElseRegion=*/true)
+          .genThen([&]() {
+            mlir::Value result = inputVariable;
+            if (fir::isPointerType(inputVariable.getType())) {
+              result = builder.create<fir::ReboxOp>(
+                  loc, resultBoxType, inputVariable, mlir::Value{},
+                  mlir::Value{});
+            }
+            builder.create<fir::ResultOp>(
+                loc, mlir::ValueRange{result, builder.createBool(loc, false)});
+          })
+          .genElse([&] {
+            mlir::Value shape = hlfir::genShape(loc, builder, inputVariable);
+            llvm::SmallVector<mlir::Value> extents =
+                hlfir::getIndexExtents(loc, builder, shape);
+            llvm::StringRef tmpName{".tmp.copy_in"};
+            llvm::SmallVector<mlir::Value> lenParams;
+            mlir::Value alloc = builder.createHeapTemporary(
+                loc, sequenceType, tmpName, extents, lenParams);
+
+            auto declareOp = builder.create<hlfir::DeclareOp>(
+                loc, alloc, tmpName, shape, lenParams,
+                /*dummy_scope=*/nullptr);
+            hlfir::Entity temp{declareOp.getBase()};
+            hlfir::LoopNest loopNest =
+                hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+                                   flangomp::shouldUseWorkshareLowering(copyIn),
+                                   /*couldVectorize=*/false);
+            builder.setInsertionPointToStart(loopNest.body);
+            hlfir::Entity elem = hlfir::getElementAt(
+                loc, builder, inputVariable, loopNest.oneBasedIndices);
+            elem = hlfir::loadTrivialScalar(loc, builder, elem);
+            hlfir::Entity tempElem = hlfir::getElementAt(
+                loc, builder, temp, loopNest.oneBasedIndices);
+            builder.create<hlfir::AssignOp>(loc, elem, tempElem);
+            builder.setInsertionPointAfter(loopNest.outerOp);
+
+            mlir::Value result;
+            // Make sure the result is always a boxed array by boxing it
+            // ourselves if need be.
+            if (mlir::isa<fir::BaseBoxType>(temp.getType())) {
+              result = temp;
+            } else {
+              fir::ReferenceType refTy =
+                  fir::ReferenceType::get(temp.getElementOrSequenceType());
+              mlir::Value refVal = builder.createConvert(loc, refTy, temp);
+              result = builder.create<fir::EmboxOp>(loc, resultBoxType, refVal,
+                                                    shape);
+            }
+
+            builder.create<fir::ResultOp>(
+                loc, mlir::ValueRange{result, builder.createBool(loc, true)});
+          })
+          .getResults();
+
+  mlir::OpResult resultBox = results[0];
+  mlir::OpResult needsCleanup = results[1];
+
+  // Prepare the corresponding copyOut to free the temporary if it is required
+  auto alloca = builder.create<fir::AllocaOp>(loc, resultBox.getType());
+  auto store = builder.create<fir::StoreOp>(loc, resultBox, alloca);
+  rewriter.startOpModification(copyOut);
+  copyOut->setOperand(0, store.getMemref());
+  copyOut->setOperand(1, needsCleanup);
+  rewriter.finalizeOpModification(copyOut);
+
+  rewriter.replaceOp(copyIn, {resultBox, builder.genNot(loc, isContiguous)});
+  return mlir::success();
+}
+
+class InlineHLFIRCopyInPass
+    : public hlfir::impl::InlineHLFIRCopyInBase<InlineHLFIRCopyInPass> {
+public:
+  void runOnOperation() override {
+    mlir::MLIRContext *context = &getContext();
+
+    mlir::GreedyRewriteConfig config;
+    // Prevent the pattern driver from merging blocks.
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
+
+    mlir::RewritePatternSet patterns(context);
+    if (!noInlineHLFIRCopyIn) {
+      patterns.insert<InlineCopyInConversion>(context);
+    }
+
+    if (mlir::failed(mlir::applyPatternsGreedily(
+            getOperation(), std::move(patterns), config))) {
+      mlir::emitError(getOperation()->getLoc(),
+                      "failure in hlfir.copy_in inlining");
+      signalPassFailure();
+    }
+  }
+};
+} // namespace
diff --git a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index e2ca754a1817..91df8672c20d 100644
--- a/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -572,13 +572,11 @@ ElementalAssignBufferization::findMatch(hlfir::ElementalOp elemental) {
   // hlfir.assign, check there are no effects which make this unsafe
 
   // keep track of any values written to in the elemental, as these can't be
-  // read from between the elemental and the assignment
+  // read from or written to between the elemental and the assignment
+  mlir::SmallVector<mlir::Value, 1> notToBeAccessedBeforeAssign;
   // likewise, values read in the elemental cannot be written to between the
   // elemental and the assign
-  mlir::SmallVector<mlir::Value, 1> notToBeAccessedBeforeAssign;
-  // any accesses to the array between the array and the assignment means it
-  // would be unsafe to move the elemental to the assignment
-  notToBeAccessedBeforeAssign.push_back(match.array);
+  mlir::SmallVector<mlir::Value, 1> notToBeWrittenBeforeAssign;
 
   // 1) side effects in the elemental body - it isn't sufficient to just look
   // for ordered elementals because we also cannot support out of order reads
@@ -593,10 +591,12 @@ ElementalAssignBufferization::findMatch(hlfir::ElementalOp elemental) {
   for (const mlir::MemoryEffects::EffectInstance &effect : *effects) {
     mlir::AliasResult res = containsReadOrWriteEffectOn(effect, match.array);
     if (res.isNo()) {
-      if (mlir::isa<mlir::MemoryEffects::Write, mlir::MemoryEffects::Read>(
-              effect.getEffect()))
-        if (effect.getValue())
+      if (effect.getValue()) {
+        if (mlir::isa<mlir::MemoryEffects::Write>(effect.getEffect()))
           notToBeAccessedBeforeAssign.push_back(effect.getValue());
+        else if (mlir::isa<mlir::MemoryEffects::Read>(effect.getEffect()))
+          notToBeWrittenBeforeAssign.push_back(effect.getValue());
+      }
 
       // this is safe in the elemental
       continue;
@@ -669,11 +669,23 @@ ElementalAssignBufferization::findMatch(hlfir::ElementalOp elemental) {
       mlir::AliasResult res = containsReadOrWriteEffectOn(effect, val);
       if (!res.isNo()) {
         LLVM_DEBUG(llvm::dbgs()
-                   << "diasllowed side-effect: " << effect.getValue() << " for "
+                   << "disallowed side-effect: " << effect.getValue() << " for "
                    << elemental.getLoc() << "\n");
         return std::nullopt;
       }
     }
+    // Anything that is read inside the elemental can only be safely read
+    // between the elemental and the assignment.
+    for (mlir::Value val : notToBeWrittenBeforeAssign) {
+      mlir::AliasResult res = containsReadOrWriteEffectOn(effect, val);
+      if (!res.isNo() &&
+          !mlir::isa<mlir::MemoryEffects::Read>(effect.getEffect())) {
+        LLVM_DEBUG(llvm::dbgs()
+                   << "disallowed non-read side-effect: " << effect.getValue()
+                   << " for " << elemental.getLoc() << "\n");
+        return std::nullopt;
+      }
+    }
   }
 
   return match;
@@ -688,10 +700,17 @@ llvm::LogicalResult ElementalAssignBufferization::matchAndRewrite(
 
   mlir::Location loc = elemental->getLoc();
   fir::FirOpBuilder builder(rewriter, elemental.getOperation());
-  auto extents = hlfir::getIndexExtents(loc, builder, elemental.getShape());
+  auto rhsExtents = hlfir::getIndexExtents(loc, builder, elemental.getShape());
 
   // create the loop at the assignment
   builder.setInsertionPoint(match->assign);
+  hlfir::Entity lhs{match->array};
+  lhs = hlfir::derefPointersAndAllocatables(loc, builder, lhs);
+  mlir::Value lhsShape = hlfir::genShape(loc, builder, lhs);
+  llvm::SmallVector<mlir::Value> lhsExtents =
+      hlfir::getIndexExtents(loc, builder, lhsShape);
+  llvm::SmallVector<mlir::Value> extents =
+      fir::factory::deduceOptimalExtents(rhsExtents, lhsExtents);
 
   // Generate a loop nest looping around the hlfir.elemental shape and clone
   // hlfir.elemental region inside the inner loop
@@ -705,8 +724,8 @@ llvm::LogicalResult ElementalAssignBufferization::matchAndRewrite(
   rewriter.eraseOp(yield);
 
   // Assign the element value to the array element for this iteration.
-  auto arrayElement = hlfir::getElementAt(
-      loc, builder, hlfir::Entity{match->array}, loopNest.oneBasedIndices);
+  auto arrayElement =
+      hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
   builder.create<hlfir::AssignOp>(
       loc, elementValue, arrayElement, /*realloc=*/false,
       /*keep_lhs_length_if_realloc=*/false, match->assign.getTemporaryLhs());
diff --git a/external/llvm-project/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp b/external/llvm-project/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
index 2f9f7eed11a3..e9b15e537292 100644
--- a/external/llvm-project/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/OpenMP/DoConcurrentConversion.cpp
@@ -11,12 +11,12 @@
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
 #include "flang/Optimizer/Builder/Todo.h"
-#include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/OpenMP/Passes.h"
+#include "flang/Optimizer/OpenMP/Utils.h"
 #include "flang/Support/OpenMP-utils.h"
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
@@ -451,10 +451,68 @@ class DoConcurrentConversion
               fir::DoConcurrentLoopOp loop, mlir::IRMapping &mapper,
               const mlir::omp::LoopNestOperands &clauseOps,
               bool isComposite) const {
+    mlir::omp::WsloopOperands wsloopClauseOps;
+
+    // For `local` (and `local_init`) opernads, emit corresponding `private`
+    // clauses and attach these clauses to the workshare loop.
+    if (!loop.getLocalOperands().empty())
+      for (auto [op, sym, arg] : llvm::zip_equal(
+               loop.getLocalOperands(),
+               loop.getLocalSymsAttr().getAsRange<mlir::SymbolRefAttr>(),
+               loop.getRegionLocalArgs())) {
+        auto localizer = mlir::SymbolTable::lookupNearestSymbolFrom<
+            fir::LocalitySpecifierOp>(loop, sym);
+        if (localizer.getLocalitySpecifierType() ==
+            fir::LocalitySpecifierType::LocalInit)
+          TODO(localizer.getLoc(),
+               "local_init conversion is not supported yet");
+
+        auto oldIP = rewriter.saveInsertionPoint();
+        rewriter.setInsertionPointAfter(localizer);
+        auto privatizer = rewriter.create<mlir::omp::PrivateClauseOp>(
+            localizer.getLoc(), sym.getLeafReference().str() + ".omp",
+            localizer.getTypeAttr().getValue(),
+            mlir::omp::DataSharingClauseType::Private);
+
+        if (!localizer.getInitRegion().empty()) {
+          rewriter.cloneRegionBefore(localizer.getInitRegion(),
+                                     privatizer.getInitRegion(),
+                                     privatizer.getInitRegion().begin());
+          auto firYield = mlir::cast<fir::YieldOp>(
+              privatizer.getInitRegion().back().getTerminator());
+          rewriter.setInsertionPoint(firYield);
+          rewriter.create<mlir::omp::YieldOp>(firYield.getLoc(),
+                                              firYield.getOperands());
+          rewriter.eraseOp(firYield);
+        }
+
+        if (!localizer.getDeallocRegion().empty()) {
+          rewriter.cloneRegionBefore(localizer.getDeallocRegion(),
+                                     privatizer.getDeallocRegion(),
+                                     privatizer.getDeallocRegion().begin());
+          auto firYield = mlir::cast<fir::YieldOp>(
+              privatizer.getDeallocRegion().back().getTerminator());
+          rewriter.setInsertionPoint(firYield);
+          rewriter.create<mlir::omp::YieldOp>(firYield.getLoc(),
+                                              firYield.getOperands());
+          rewriter.eraseOp(firYield);
+        }
+
+        rewriter.restoreInsertionPoint(oldIP);
 
-    auto wsloopOp = rewriter.create<mlir::omp::WsloopOp>(loop.getLoc());
+        wsloopClauseOps.privateVars.push_back(op);
+        wsloopClauseOps.privateSyms.push_back(
+            mlir::SymbolRefAttr::get(privatizer));
+      }
+
+    auto wsloopOp =
+        rewriter.create<mlir::omp::WsloopOp>(loop.getLoc(), wsloopClauseOps);
     wsloopOp.setComposite(isComposite);
-    rewriter.createBlock(&wsloopOp.getRegion());
+
+    Fortran::common::openmp::EntryBlockArgs wsloopArgs;
+    wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
+    Fortran::common::openmp::genEntryBlock(rewriter, wsloopArgs,
+                                           wsloopOp.getRegion());
 
     auto loopNestOp =
         rewriter.create<mlir::omp::LoopNestOp>(loop.getLoc(), clauseOps);
@@ -467,6 +525,18 @@ class DoConcurrentConversion
     rewriter.setInsertionPointToEnd(&loopNestOp.getRegion().back());
     rewriter.create<mlir::omp::YieldOp>(loop->getLoc());
 
+    // `local` region arguments are transferred/cloned from the `do concurrent`
+    // loop to the loopnest op when the region is cloned above. Instead, these
+    // region arguments should be on the workshare loop's region.
+    for (auto [wsloopArg, loopNestArg] :
+         llvm::zip_equal(wsloopOp.getRegion().getArguments(),
+                         loopNestOp.getRegion().getArguments().drop_front(
+                             clauseOps.loopLowerBounds.size())))
+      rewriter.replaceAllUsesWith(loopNestArg, wsloopArg);
+
+    for (unsigned i = 0; i < loop.getLocalVars().size(); ++i)
+      loopNestOp.getRegion().eraseArgument(clauseOps.loopLowerBounds.size());
+
     return loopNestOp;
   }
 
diff --git a/external/llvm-project/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp b/external/llvm-project/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
index fb5c615e2411..7a6fadf40fef 100644
--- a/external/llvm-project/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
@@ -48,7 +48,7 @@
 #include <numeric>
 
 #define DEBUG_TYPE "omp-map-info-finalization"
-#define PDBGS() (llvm::dbgs() << "[" << DEBUG_TYPE << "]: ")
+
 namespace flangomp {
 #define GEN_PASS_DEF_MAPINFOFINALIZATIONPASS
 #include "flang/Optimizer/OpenMP/Passes.h.inc"
@@ -58,6 +58,13 @@ namespace {
 class MapInfoFinalizationPass
     : public flangomp::impl::MapInfoFinalizationPassBase<
           MapInfoFinalizationPass> {
+public:
+  MapInfoFinalizationPass() = default;
+
+  MapInfoFinalizationPass(
+      const flangomp::MapInfoFinalizationPassOptions &options)
+      : MapInfoFinalizationPassBase(options) {}
+
   /// Helper class tracking a members parent and its
   /// placement in the parents member list
   struct ParentAndPlacement {
@@ -335,6 +342,62 @@ class MapInfoFinalizationPass
     return false;
   }
 
+  mlir::omp::MapInfoOp genBoxcharMemberMap(mlir::omp::MapInfoOp op,
+                                           fir::FirOpBuilder &builder) {
+    if (!op.getMembers().empty())
+      return op;
+    mlir::Location loc = op.getVarPtr().getLoc();
+    mlir::Value boxChar = op.getVarPtr();
+
+    if (mlir::isa<fir::ReferenceType>(op.getVarPtr().getType()))
+      boxChar = builder.create<fir::LoadOp>(loc, op.getVarPtr());
+
+    fir::BoxCharType boxCharType =
+        mlir::dyn_cast<fir::BoxCharType>(boxChar.getType());
+    mlir::Value boxAddr = builder.create<fir::BoxOffsetOp>(
+        loc, op.getVarPtr(), fir::BoxFieldAttr::base_addr);
+
+    uint64_t mapTypeToImplicit = static_cast<
+        std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
+        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
+
+    mlir::ArrayAttr newMembersAttr;
+    llvm::SmallVector<llvm::SmallVector<int64_t>> memberIdx = {{0}};
+    newMembersAttr = builder.create2DI64ArrayAttr(memberIdx);
+
+    mlir::Value varPtr = op.getVarPtr();
+    mlir::omp::MapInfoOp memberMapInfoOp = builder.create<mlir::omp::MapInfoOp>(
+        op.getLoc(), varPtr.getType(), varPtr,
+        mlir::TypeAttr::get(boxCharType.getEleTy()),
+        builder.getIntegerAttr(builder.getIntegerType(64, /*isSigned=*/false),
+                               mapTypeToImplicit),
+        builder.getAttr<mlir::omp::VariableCaptureKindAttr>(
+            mlir::omp::VariableCaptureKind::ByRef),
+        /*varPtrPtr=*/boxAddr,
+        /*members=*/llvm::SmallVector<mlir::Value>{},
+        /*member_index=*/mlir::ArrayAttr{},
+        /*bounds=*/op.getBounds(),
+        /*mapperId=*/mlir::FlatSymbolRefAttr(), /*name=*/op.getNameAttr(),
+        builder.getBoolAttr(false));
+
+    mlir::omp::MapInfoOp newMapInfoOp = builder.create<mlir::omp::MapInfoOp>(
+        op.getLoc(), op.getResult().getType(), varPtr,
+        mlir::TypeAttr::get(
+            llvm::cast<mlir::omp::PointerLikeType>(varPtr.getType())
+                .getElementType()),
+        op.getMapTypeAttr(), op.getMapCaptureTypeAttr(),
+        /*varPtrPtr=*/mlir::Value{},
+        /*members=*/llvm::SmallVector<mlir::Value>{memberMapInfoOp},
+        /*member_index=*/newMembersAttr,
+        /*bounds=*/llvm::SmallVector<mlir::Value>{},
+        /*mapperId=*/mlir::FlatSymbolRefAttr(), op.getNameAttr(),
+        /*partial_map=*/builder.getBoolAttr(false));
+    op.replaceAllUsesWith(newMapInfoOp.getResult());
+    op->erase();
+    return newMapInfoOp;
+  }
+
   mlir::omp::MapInfoOp genDescriptorMemberMaps(mlir::omp::MapInfoOp op,
                                                fir::FirOpBuilder &builder,
                                                mlir::Operation *target) {
@@ -730,6 +793,7 @@ class MapInfoFinalizationPass
         fir::factory::AddrAndBoundsInfo info =
             fir::factory::getDataOperandBaseAddr(
                 builder, varPtr, /*isOptional=*/false, varPtr.getLoc());
+
         fir::ExtendedValue extendedValue =
             hlfir::translateToExtendedValue(varPtr.getLoc(), builder,
                                             hlfir::Entity{info.addr},
@@ -907,6 +971,37 @@ class MapInfoFinalizationPass
       //   return mlir::WalkResult::advance();
       // });
 
+      func->walk([&](mlir::omp::MapInfoOp op) {
+        if (!op.getMembers().empty())
+          return;
+
+        if (!mlir::isa<fir::BoxCharType>(fir::unwrapRefType(op.getVarType())))
+          return;
+
+        // POSSIBLE_HACK_ALERT: If the boxchar has been implicitly mapped then
+        // it is likely that the underlying pointer to the data
+        // (!fir.ref<fir.char<k,?>>) has already been mapped. So, skip such
+        // boxchars. We are primarily interested in boxchars that were mapped
+        // by passes such as MapsForPrivatizedSymbols that map boxchars that
+        // are privatized. At present, such boxchar maps are not marked
+        // implicit. Should they be? I don't know. If they should be then
+        // we need to change this check for early return OR live with
+        // over-mapping.
+        bool hasImplicitMap =
+            (llvm::omp::OpenMPOffloadMappingFlags(op.getMapType()) &
+             llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT) ==
+            llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+        if (hasImplicitMap)
+          return;
+
+        assert(llvm::hasSingleElement(op->getUsers()) &&
+               "OMPMapInfoFinalization currently only supports single users "
+               "of a MapInfoOp");
+
+        builder.setInsertionPoint(op);
+        genBoxcharMemberMap(op, builder);
+      });
+
       func->walk([&](mlir::omp::MapInfoOp op) {
         // TODO: Currently only supports a single user for the MapInfoOp. This
         // is fine for the moment, as the Fortran frontend will generate a
@@ -954,17 +1049,20 @@ class MapInfoFinalizationPass
       // within a target region. At which point we map the relevant descriptor
       // data and the runtime should correctly associate the data with the
       // descriptor and bind together and allow clean mapping and execution.
-      func->walk([&](mlir::omp::MapInfoOp op) {
-        if (fir::isTypeWithDescriptor(op.getVarType()) ||
-            mlir::isa_and_present<fir::BoxAddrOp>(
-                op.getVarPtr().getDefiningOp())) {
-          mlir::Operation *targetUser = getFirstTargetUser(op);
-          assert(targetUser && "expected user of map operation was not found");
-          builder.setInsertionPoint(op);
-          removeTopLevelDescriptor(op, builder, targetUser);
-          addImplictDescriptorMapToTargetDataOp(op, builder, targetUser);
-        }
-      });
+      if (deferDescMapping) {
+        func->walk([&](mlir::omp::MapInfoOp op) {
+          if (fir::isTypeWithDescriptor(op.getVarType()) ||
+              mlir::isa_and_present<fir::BoxAddrOp>(
+                  op.getVarPtr().getDefiningOp())) {
+            mlir::Operation *targetUser = getFirstTargetUser(op);
+            assert(targetUser &&
+                   "expected user of map operation was not found");
+            builder.setInsertionPoint(op);
+            removeTopLevelDescriptor(op, builder, targetUser);
+            addImplictDescriptorMapToTargetDataOp(op, builder, targetUser);
+          }
+        });
+      }
 
       // Wait until after we have generated all of our maps to add them onto
       // the target's block arguments, simplifying the process as there would be
@@ -977,5 +1075,11 @@ class MapInfoFinalizationPass
     });
   }
 };
-
 } // namespace
+
+std::unique_ptr<mlir::Pass>
+flangomp::createMapInfoFinalizationPass(bool deferDescMap) {
+  MapInfoFinalizationPassOptions options;
+  options.deferDescMapping = deferDescMap;
+  return std::make_unique<MapInfoFinalizationPass>(options);
+}
diff --git a/external/llvm-project/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp b/external/llvm-project/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
index dedff59d66aa..19566af11db0 100644
--- a/external/llvm-project/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/OpenMP/MapsForPrivatizedSymbols.cpp
@@ -22,7 +22,9 @@
 // 2. Generalize this for more than just omp.target ops.
 //===----------------------------------------------------------------------===//
 
+#include "flang/Optimizer/Builder/DirectivesCommon.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/HLFIRTools.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
@@ -184,37 +186,23 @@ class MapsForPrivatizedSymbolsPass
     return fir::hasDynamicSize(t);
   }
 
-  // TODO: Remove this in favor of fir::factory::genImplicitBoundsOps
-  // in a subsequent PR.
   void genBoundsOps(fir::FirOpBuilder &builder, mlir::Value var,
                     llvm::SmallVector<mlir::Value> &boundsOps) {
-    if (!fir::isBoxAddress(var.getType()))
-      return;
-
-    unsigned int rank = 0;
-    rank = fir::getBoxRank(fir::unwrapRefType(var.getType()));
     mlir::Location loc = var.getLoc();
-    mlir::Type idxTy = builder.getIndexType();
-    mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-    mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
-    mlir::Type boundTy = builder.getType<omp::MapBoundsType>();
-    mlir::Value box = builder.create<fir::LoadOp>(loc, var);
-    for (unsigned int i = 0; i < rank; ++i) {
-      mlir::Value dimNo = builder.createIntegerConstant(loc, idxTy, i);
-      auto dimInfo =
-          builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, dimNo);
-      mlir::Value lb = dimInfo.getLowerBound();
-      mlir::Value extent = dimInfo.getExtent();
-      mlir::Value byteStride = dimInfo.getByteStride();
-      mlir::Value ub = builder.create<mlir::arith::SubIOp>(loc, extent, one);
-
-      mlir::Value boundsOp = builder.create<omp::MapBoundsOp>(
-          loc, boundTy, /*lower_bound=*/zero,
-          /*upper_bound=*/ub, /*extent=*/extent, /*stride=*/byteStride,
-          /*stride_in_bytes = */ true, /*start_idx=*/lb);
-      LLVM_DEBUG(PDBGS() << "Created BoundsOp " << boundsOp << "\n");
-      boundsOps.push_back(boundsOp);
-    }
+    fir::factory::AddrAndBoundsInfo info =
+        fir::factory::getDataOperandBaseAddr(builder, var,
+                                             /*isOptional=*/false, loc);
+    fir::ExtendedValue extendedValue =
+        hlfir::translateToExtendedValue(loc, builder, hlfir::Entity{info.addr},
+                                        /*continguousHint=*/true)
+            .first;
+    llvm::SmallVector<mlir::Value> boundsOpsVec =
+        fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                           mlir::omp::MapBoundsType>(
+            builder, info, extendedValue,
+            /*dataExvIsAssumedSize=*/false, loc);
+    for (auto bounds : boundsOpsVec)
+      boundsOps.push_back(bounds);
   }
 };
 } // namespace
diff --git a/external/llvm-project/flang/lib/Optimizer/Passes/Pipelines.cpp b/external/llvm-project/flang/lib/Optimizer/Passes/Pipelines.cpp
index b0a1c4155e9f..168eddd909c0 100644
--- a/external/llvm-project/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -260,6 +260,11 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
         pm, hlfir::createOptimizedBufferization);
     addNestedPassToAllTopLevelOperations<PassConstructor>(
         pm, hlfir::createInlineHLFIRAssign);
+
+    if (optLevel == llvm::OptimizationLevel::O3) {
+      addNestedPassToAllTopLevelOperations<PassConstructor>(
+          pm, hlfir::createInlineHLFIRCopyIn);
+    }
   }
   pm.addPass(hlfir::createLowerHLFIROrderedAssignments());
   pm.addPass(hlfir::createLowerHLFIRIntrinsics());
@@ -312,7 +317,7 @@ void createOpenMPFIRPassPipeline(mlir::PassManager &pm,
   // underlying data which is necessary to access the data on the offload
   // target device.
   pm.addPass(flangomp::createMapsForPrivatizedSymbolsPass());
-  pm.addPass(flangomp::createMapInfoFinalizationPass());
+  pm.addPass(flangomp::createMapInfoFinalizationPass(opts.deferDescMap));
   pm.addPass(flangomp::createMarkDeclareTargetPass());
   pm.addPass(flangomp::createGenericLoopConversionPass());
   if (opts.isTargetDevice) {
@@ -368,7 +373,8 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
       {framePointerKind, config.InstrumentFunctionEntry,
        config.InstrumentFunctionExit, config.NoInfsFPMath, config.NoNaNsFPMath,
        config.ApproxFuncFPMath, config.NoSignedZerosFPMath, config.UnsafeFPMath,
-       config.PreferVectorWidth, /*tuneCPU=*/"", setNoCapture, setNoAlias}));
+       config.Reciprocals, config.PreferVectorWidth, /*tuneCPU=*/"",
+       setNoCapture, setNoAlias}));
 
   if (config.EnableOpenMP) {
     pm.addNestedPass<mlir::func::FuncOp>(
diff --git a/external/llvm-project/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/external/llvm-project/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index 23a9bac84b4d..5cfbdc33285f 100644
--- a/external/llvm-project/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -43,13 +43,10 @@ static llvm::cl::opt<bool>
 static llvm::cl::opt<bool>
     enableDirect("direct-tbaa", llvm::cl::init(true), llvm::cl::Hidden,
                  llvm::cl::desc("Add TBAA tags to direct variables"));
-// This is **known unsafe** (misscompare in spec2017/wrf_r). It should
-// not be enabled by default.
-// The code is kept so that these may be tried with new benchmarks to see if
-// this is worth fixing in the future.
-static llvm::cl::opt<bool> enableLocalAllocs(
-    "tbaa-local-alloc", llvm::cl::init(false), llvm::cl::Hidden,
-    llvm::cl::desc("Add TBAA tags to local allocations. UNSAFE."));
+static llvm::cl::opt<bool>
+    enableLocalAllocs("local-alloc-tbaa", llvm::cl::init(true),
+                      llvm::cl::Hidden,
+                      llvm::cl::desc("Add TBAA tags to local allocations."));
 
 namespace {
 
diff --git a/external/llvm-project/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/external/llvm-project/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index f06ad2db90d5..9a0071a6c6ae 100644
--- a/external/llvm-project/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -87,13 +87,48 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     return success();
   }
 };
+
+struct IfConversion : public OpRewritePattern<fir::IfOp> {
+  using OpRewritePattern<fir::IfOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(fir::IfOp ifOp,
+                                PatternRewriter &rewriter) const override {
+    mlir::Location loc = ifOp.getLoc();
+    mlir::detail::TypedValue<mlir::IntegerType> condition = ifOp.getCondition();
+    ValueTypeRange<ResultRange> resultTypes = ifOp.getResultTypes();
+    mlir::scf::IfOp scfIfOp = rewriter.create<scf::IfOp>(
+        loc, resultTypes, condition, !ifOp.getElseRegion().empty());
+    // then region
+    scfIfOp.getThenRegion().takeBody(ifOp.getThenRegion());
+    Block &scfThenBlock = scfIfOp.getThenRegion().front();
+    Operation *scfThenTerminator = scfThenBlock.getTerminator();
+    // fir.result->scf.yield
+    rewriter.setInsertionPointToEnd(&scfThenBlock);
+    rewriter.replaceOpWithNewOp<scf::YieldOp>(scfThenTerminator,
+                                              scfThenTerminator->getOperands());
+
+    // else region
+    if (!ifOp.getElseRegion().empty()) {
+      scfIfOp.getElseRegion().takeBody(ifOp.getElseRegion());
+      mlir::Block &elseBlock = scfIfOp.getElseRegion().front();
+      mlir::Operation *elseTerminator = elseBlock.getTerminator();
+
+      rewriter.setInsertionPointToEnd(&elseBlock);
+      rewriter.replaceOpWithNewOp<scf::YieldOp>(elseTerminator,
+                                                elseTerminator->getOperands());
+    }
+
+    scfIfOp->setAttrs(ifOp->getAttrs());
+    rewriter.replaceOp(ifOp, scfIfOp);
+    return success();
+  }
+};
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
   RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion>(patterns.getContext());
+  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
   ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp>();
+  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/external/llvm-project/flang/lib/Optimizer/Transforms/FunctionAttr.cpp b/external/llvm-project/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
index 041aa8717d20..5ac4ed8a93b6 100644
--- a/external/llvm-project/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Transforms/FunctionAttr.cpp
@@ -107,6 +107,10 @@ void FunctionAttrPass::runOnOperation() {
     func->setAttr(
         mlir::LLVM::LLVMFuncOp::getUnsafeFpMathAttrName(llvmFuncOpName),
         mlir::BoolAttr::get(context, true));
+  if (!reciprocals.empty())
+    func->setAttr(
+        mlir::LLVM::LLVMFuncOp::getReciprocalEstimatesAttrName(llvmFuncOpName),
+        mlir::StringAttr::get(context, reciprocals));
   if (!preferVectorWidth.empty())
     func->setAttr(
         mlir::LLVM::LLVMFuncOp::getPreferVectorWidthAttrName(llvmFuncOpName),
diff --git a/external/llvm-project/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/external/llvm-project/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index cb9e48cced2a..e440852b3103 100644
--- a/external/llvm-project/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/external/llvm-project/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -180,41 +180,50 @@ class DoConcurrentConversion
 
       std::optional<mlir::ArrayAttr> localSyms = loop.getLocalSyms();
 
-      for (auto [localVar, localArg, localizerSym] : llvm::zip_equal(
+      for (auto localInfo : llvm::zip_equal(
                loop.getLocalVars(), loop.getRegionLocalArgs(), *localSyms)) {
+        mlir::Value localVar = std::get<0>(localInfo);
+        mlir::BlockArgument localArg = std::get<1>(localInfo);
+        mlir::Attribute localizerSym = std::get<2>(localInfo);
         mlir::SymbolRefAttr localizerName =
             llvm::cast<mlir::SymbolRefAttr>(localizerSym);
         fir::LocalitySpecifierOp localizer = findLocalizer(loop, localizerName);
 
-        if (!localizer.getInitRegion().empty() ||
-            !localizer.getDeallocRegion().empty())
-          TODO(localizer.getLoc(), "localizers with `init` and `dealloc` "
-                                   "regions are not handled yet.");
-
         // TODO Should this be a heap allocation instead? For now, we allocate
         // on the stack for each loop iteration.
         mlir::Value localAlloc =
             rewriter.create<fir::AllocaOp>(loop.getLoc(), localizer.getType());
 
-        if (localizer.getLocalitySpecifierType() ==
-            fir::LocalitySpecifierType::LocalInit) {
+        auto cloneLocalizerRegion = [&](mlir::Region &region,
+                                        mlir::ValueRange regionArgs,
+                                        mlir::Block::iterator insertionPoint) {
           // It is reasonable to make this assumption since, at this stage,
           // control-flow ops are not converted yet. Therefore, things like `if`
           // conditions will still be represented by their encapsulating `fir`
           // dialect ops.
-          assert(localizer.getCopyRegion().hasOneBlock() &&
-                 "Expected localizer to have a single block.");
-          mlir::Block *beforeLocalInit = rewriter.getInsertionBlock();
-          mlir::Block *afterLocalInit = rewriter.splitBlock(
-              rewriter.getInsertionBlock(), rewriter.getInsertionPoint());
-          rewriter.cloneRegionBefore(localizer.getCopyRegion(), afterLocalInit);
-          mlir::Block *copyRegionBody = beforeLocalInit->getNextNode();
-
-          rewriter.eraseOp(copyRegionBody->getTerminator());
-          rewriter.mergeBlocks(afterLocalInit, copyRegionBody);
-          rewriter.mergeBlocks(copyRegionBody, beforeLocalInit,
-                               {localVar, localArg});
-        }
+          assert(region.hasOneBlock() &&
+                 "Expected localizer region to have a single block.");
+          mlir::OpBuilder::InsertionGuard guard(rewriter);
+          rewriter.setInsertionPoint(rewriter.getInsertionBlock(),
+                                     insertionPoint);
+          mlir::IRMapping mapper;
+          mapper.map(region.getArguments(), regionArgs);
+          for (mlir::Operation &op : region.front().without_terminator())
+            (void)rewriter.clone(op, mapper);
+        };
+
+        if (!localizer.getInitRegion().empty())
+          cloneLocalizerRegion(localizer.getInitRegion(), {localVar, localArg},
+                               rewriter.getInsertionPoint());
+
+        if (localizer.getLocalitySpecifierType() ==
+            fir::LocalitySpecifierType::LocalInit)
+          cloneLocalizerRegion(localizer.getCopyRegion(), {localVar, localArg},
+                               rewriter.getInsertionPoint());
+
+        if (!localizer.getDeallocRegion().empty())
+          cloneLocalizerRegion(localizer.getDeallocRegion(), {localArg},
+                               rewriter.getInsertionBlock()->end());
 
         rewriter.replaceAllUsesWith(localArg, localAlloc);
       }
diff --git a/external/llvm-project/flang/lib/Parser/openmp-parsers.cpp b/external/llvm-project/flang/lib/Parser/openmp-parsers.cpp
index 08326fad8c14..9b112a213391 100644
--- a/external/llvm-project/flang/lib/Parser/openmp-parsers.cpp
+++ b/external/llvm-project/flang/lib/Parser/openmp-parsers.cpp
@@ -24,6 +24,12 @@
 // OpenMP Directives and Clauses
 namespace Fortran::parser {
 
+// Helper function to print the buffer contents starting at the current point.
+[[maybe_unused]] static std::string ahead(const ParseState &state) {
+  return std::string(
+      state.GetLocation(), std::min<size_t>(64, state.BytesRemaining()));
+}
+
 constexpr auto startOmpLine = skipStuffBeforeStatement >> "!$OMP "_sptok;
 constexpr auto endOmpLine = space >> endOfLine;
 
@@ -941,8 +947,10 @@ TYPE_PARSER( //
             parenthesized(Parser<OmpAtomicDefaultMemOrderClause>{}))) ||
     "BIND" >> construct<OmpClause>(construct<OmpClause::Bind>(
                   parenthesized(Parser<OmpBindClause>{}))) ||
+    "CAPTURE" >> construct<OmpClause>(construct<OmpClause::Capture>()) ||
     "COLLAPSE" >> construct<OmpClause>(construct<OmpClause::Collapse>(
                       parenthesized(scalarIntConstantExpr))) ||
+    "COMPARE" >> construct<OmpClause>(construct<OmpClause::Compare>()) ||
     "CONTAINS" >> construct<OmpClause>(construct<OmpClause::Contains>(
                       parenthesized(Parser<OmpContainsClause>{}))) ||
     "COPYIN" >> construct<OmpClause>(construct<OmpClause::Copyin>(
@@ -1062,6 +1070,7 @@ TYPE_PARSER( //
     "TASK_REDUCTION" >>
         construct<OmpClause>(construct<OmpClause::TaskReduction>(
             parenthesized(Parser<OmpTaskReductionClause>{}))) ||
+    "READ" >> construct<OmpClause>(construct<OmpClause::Read>()) ||
     "RELAXED" >> construct<OmpClause>(construct<OmpClause::Relaxed>()) ||
     "RELEASE" >> construct<OmpClause>(construct<OmpClause::Release>()) ||
     "REVERSE_OFFLOAD" >>
@@ -1105,6 +1114,7 @@ TYPE_PARSER( //
                     maybe(Parser<OmpUpdateClause>{}))) ||
     "WHEN" >> construct<OmpClause>(construct<OmpClause::When>(
                   parenthesized(Parser<OmpWhenClause>{}))) ||
+    "WRITE" >> construct<OmpClause>(construct<OmpClause::Write>()) ||
     // Cancellable constructs
     construct<OmpClause>(construct<OmpClause::CancellationConstructType>(
         Parser<OmpCancellationConstructTypeClause>{})))
@@ -1223,6 +1233,155 @@ TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
 TYPE_PARSER(sourced(construct<OmpBeginLoopDirective>(
     sourced(Parser<OmpLoopDirective>{}), Parser<OmpClauseList>{})))
 
+struct OmpEndDirectiveParser {
+  using resultType = OmpDirectiveSpecification;
+
+  constexpr OmpEndDirectiveParser(llvm::omp::Directive dir) : dir_(dir) {}
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    if ((startOmpLine >> "END"_sptok).Parse(state)) {
+      auto &&dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+      if (dirSpec && dirSpec->DirId() == dir_) {
+        return std::move(dirSpec);
+      }
+    }
+    return std::nullopt;
+  }
+
+private:
+  llvm::omp::Directive dir_;
+};
+
+// Parser for an arbitrary OpenMP ATOMIC construct.
+//
+// Depending on circumstances, an ATOMIC construct applies to one or more
+// following statements. In certain cases when a single statement is
+// expected, the end-directive is optional. The specifics depend on both
+// the clauses used, and the form of the executable statement. To emit
+// more meaningful messages in case of errors, the exact analysis of the
+// structure of the construct will be delayed until semantic checks.
+//
+// The parser will first try the case when the end-directive is present,
+// and will parse at most "BodyLimit" (and potentially zero) constructs
+// while looking for the end-directive before it gives up.
+// Then it will assume that no end-directive is present, and will try to
+// parse a single executable construct as the body of the construct.
+//
+// The limit on the number of constructs is there to reduce the amount of
+// unnecessary parsing when the end-directive is absent. It's higher than
+// the maximum number of statements in any valid construct to accept cases
+// when extra statements are present by mistake.
+// A problem can occur when atomic constructs without end-directive follow
+// each other closely, e.g.
+//   !$omp atomic write
+//     x = v
+//   !$omp atomic update
+//     x = x + 1
+//   ...
+// The speculative parsing will become "recursive", and has the potential
+// to take a (practically) infinite amount of time given a sufficiently
+// large number of such constructs in a row. Since atomic constructs cannot
+// contain other OpenMP constructs, guarding against recursive calls to the
+// atomic construct parser solves the problem.
+struct OmpAtomicConstructParser {
+  using resultType = OpenMPAtomicConstruct;
+
+  static constexpr size_t BodyLimit{5};
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    if (recursing_) {
+      return std::nullopt;
+    }
+    recursing_ = true;
+
+    auto dirSpec{Parser<OmpDirectiveSpecification>{}.Parse(state)};
+    if (!dirSpec || dirSpec->DirId() != llvm::omp::Directive::OMPD_atomic) {
+      recursing_ = false;
+      return std::nullopt;
+    }
+
+    auto exec{Parser<ExecutionPartConstruct>{}};
+    auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_atomic}};
+    TailType tail;
+
+    if (ParseOne(exec, end, tail, state)) {
+      if (!tail.first.empty()) {
+        if (auto &&rest{attempt(LimitedTailParser(BodyLimit)).Parse(state)}) {
+          for (auto &&s : rest->first) {
+            tail.first.emplace_back(std::move(s));
+          }
+          assert(!tail.second);
+          tail.second = std::move(rest->second);
+        }
+      }
+      recursing_ = false;
+      return OpenMPAtomicConstruct{
+          std::move(*dirSpec), std::move(tail.first), std::move(tail.second)};
+    }
+
+    recursing_ = false;
+    return std::nullopt;
+  }
+
+private:
+  // Begin-directive + TailType = entire construct.
+  using TailType = std::pair<Block, std::optional<OmpDirectiveSpecification>>;
+
+  // Parse either an ExecutionPartConstruct, or atomic end-directive. When
+  // successful, record the result in the "tail" provided, otherwise fail.
+  static std::optional<Success> ParseOne( //
+      Parser<ExecutionPartConstruct> &exec, OmpEndDirectiveParser &end,
+      TailType &tail, ParseState &state) {
+    auto isRecovery{[](const ExecutionPartConstruct &e) {
+      return std::holds_alternative<ErrorRecovery>(e.u);
+    }};
+    if (auto &&stmt{attempt(exec).Parse(state)}; stmt && !isRecovery(*stmt)) {
+      tail.first.emplace_back(std::move(*stmt));
+    } else if (auto &&dir{attempt(end).Parse(state)}) {
+      tail.second = std::move(*dir);
+    } else {
+      return std::nullopt;
+    }
+    return Success{};
+  }
+
+  struct LimitedTailParser {
+    using resultType = TailType;
+
+    constexpr LimitedTailParser(size_t count) : count_(count) {}
+
+    std::optional<resultType> Parse(ParseState &state) const {
+      auto exec{Parser<ExecutionPartConstruct>{}};
+      auto end{OmpEndDirectiveParser{llvm::omp::Directive::OMPD_atomic}};
+      TailType tail;
+
+      for (size_t i{0}; i != count_; ++i) {
+        if (ParseOne(exec, end, tail, state)) {
+          if (tail.second) {
+            // Return when the end-directive was parsed.
+            return std::move(tail);
+          }
+        } else {
+          break;
+        }
+      }
+      return std::nullopt;
+    }
+
+  private:
+    const size_t count_;
+  };
+
+  // The recursion guard should become thread_local if parsing is ever
+  // parallelized.
+  static bool recursing_;
+};
+
+bool OmpAtomicConstructParser::recursing_{false};
+
+TYPE_PARSER(sourced( //
+    construct<OpenMPAtomicConstruct>(OmpAtomicConstructParser{})))
+
 // 2.17.7 Atomic construct/2.17.8 Flush construct [OpenMP 5.0]
 //        memory-order-clause ->
 //                               acq_rel
@@ -1237,19 +1396,6 @@ TYPE_PARSER(sourced(construct<OmpMemoryOrderClause>(
         "RELEASE" >> construct<OmpClause>(construct<OmpClause::Release>()) ||
         "SEQ_CST" >> construct<OmpClause>(construct<OmpClause::SeqCst>())))))
 
-// 2.17.7 Atomic construct
-//        atomic-clause -> memory-order-clause | HINT(hint-expression)
-TYPE_PARSER(sourced(construct<OmpAtomicClause>(
-    construct<OmpAtomicClause>(Parser<OmpMemoryOrderClause>{}) ||
-    construct<OmpAtomicClause>(
-        "FAIL" >> parenthesized(Parser<OmpFailClause>{})) ||
-    construct<OmpAtomicClause>(
-        "HINT" >> parenthesized(Parser<OmpHintClause>{})))))
-
-// atomic-clause-list -> [atomic-clause, [atomic-clause], ...]
-TYPE_PARSER(sourced(construct<OmpAtomicClauseList>(
-    many(maybe(","_tok) >> sourced(Parser<OmpAtomicClause>{})))))
-
 static bool IsSimpleStandalone(const OmpDirectiveName &name) {
   switch (name.v) {
   case llvm::omp::Directive::OMPD_barrier:
@@ -1421,67 +1567,6 @@ TYPE_PARSER(sourced(
 TYPE_PARSER(construct<OmpReductionCombiner>(Parser<AssignmentStmt>{}) ||
     construct<OmpReductionCombiner>(Parser<FunctionReference>{}))
 
-// 2.17.7 atomic -> ATOMIC [clause [,]] atomic-clause [[,] clause] |
-//                  ATOMIC [clause]
-//       clause -> memory-order-clause | HINT(hint-expression)
-//       memory-order-clause -> SEQ_CST | ACQ_REL | RELEASE | ACQUIRE | RELAXED
-//       atomic-clause -> READ | WRITE | UPDATE | CAPTURE
-
-// OMP END ATOMIC
-TYPE_PARSER(construct<OmpEndAtomic>(startOmpLine >> "END ATOMIC"_tok))
-
-// OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] READ [MEMORY-ORDER-CLAUSE-LIST]
-TYPE_PARSER("ATOMIC" >>
-    sourced(construct<OmpAtomicRead>(
-        Parser<OmpAtomicClauseList>{} / maybe(","_tok), verbatim("READ"_tok),
-        Parser<OmpAtomicClauseList>{} / endOmpLine, statement(assignmentStmt),
-        maybe(Parser<OmpEndAtomic>{} / endOmpLine))))
-
-// OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] CAPTURE [MEMORY-ORDER-CLAUSE-LIST]
-TYPE_PARSER("ATOMIC" >>
-    sourced(construct<OmpAtomicCapture>(
-        Parser<OmpAtomicClauseList>{} / maybe(","_tok), verbatim("CAPTURE"_tok),
-        Parser<OmpAtomicClauseList>{} / endOmpLine, statement(assignmentStmt),
-        statement(assignmentStmt), Parser<OmpEndAtomic>{} / endOmpLine)))
-
-TYPE_PARSER(construct<OmpAtomicCompareIfStmt>(indirect(Parser<IfStmt>{})) ||
-    construct<OmpAtomicCompareIfStmt>(indirect(Parser<IfConstruct>{})))
-
-// OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] COMPARE [MEMORY-ORDER-CLAUSE-LIST]
-TYPE_PARSER("ATOMIC" >>
-    sourced(construct<OmpAtomicCompare>(
-        Parser<OmpAtomicClauseList>{} / maybe(","_tok), verbatim("COMPARE"_tok),
-        Parser<OmpAtomicClauseList>{} / endOmpLine,
-        Parser<OmpAtomicCompareIfStmt>{},
-        maybe(Parser<OmpEndAtomic>{} / endOmpLine))))
-
-// OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] UPDATE [MEMORY-ORDER-CLAUSE-LIST]
-TYPE_PARSER("ATOMIC" >>
-    sourced(construct<OmpAtomicUpdate>(
-        Parser<OmpAtomicClauseList>{} / maybe(","_tok), verbatim("UPDATE"_tok),
-        Parser<OmpAtomicClauseList>{} / endOmpLine, statement(assignmentStmt),
-        maybe(Parser<OmpEndAtomic>{} / endOmpLine))))
-
-// OMP ATOMIC [atomic-clause-list]
-TYPE_PARSER(sourced(construct<OmpAtomic>(verbatim("ATOMIC"_tok),
-    Parser<OmpAtomicClauseList>{} / endOmpLine, statement(assignmentStmt),
-    maybe(Parser<OmpEndAtomic>{} / endOmpLine))))
-
-// OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] WRITE [MEMORY-ORDER-CLAUSE-LIST]
-TYPE_PARSER("ATOMIC" >>
-    sourced(construct<OmpAtomicWrite>(
-        Parser<OmpAtomicClauseList>{} / maybe(","_tok), verbatim("WRITE"_tok),
-        Parser<OmpAtomicClauseList>{} / endOmpLine, statement(assignmentStmt),
-        maybe(Parser<OmpEndAtomic>{} / endOmpLine))))
-
-// Atomic Construct
-TYPE_PARSER(construct<OpenMPAtomicConstruct>(Parser<OmpAtomicRead>{}) ||
-    construct<OpenMPAtomicConstruct>(Parser<OmpAtomicCapture>{}) ||
-    construct<OpenMPAtomicConstruct>(Parser<OmpAtomicCompare>{}) ||
-    construct<OpenMPAtomicConstruct>(Parser<OmpAtomicWrite>{}) ||
-    construct<OpenMPAtomicConstruct>(Parser<OmpAtomicUpdate>{}) ||
-    construct<OpenMPAtomicConstruct>(Parser<OmpAtomic>{}))
-
 // 2.13.2 OMP CRITICAL
 TYPE_PARSER(startOmpLine >>
     sourced(construct<OmpEndCriticalDirective>(
diff --git a/external/llvm-project/flang/lib/Parser/parse-tree.cpp b/external/llvm-project/flang/lib/Parser/parse-tree.cpp
index 3dd87ad9a365..824612e49293 100644
--- a/external/llvm-project/flang/lib/Parser/parse-tree.cpp
+++ b/external/llvm-project/flang/lib/Parser/parse-tree.cpp
@@ -321,6 +321,34 @@ std::string OmpTraitSetSelectorName::ToString() const {
   return std::string(EnumToString(v));
 }
 
+llvm::omp::Clause OpenMPAtomicConstruct::GetKind() const {
+  auto &dirSpec{std::get<OmpDirectiveSpecification>(t)};
+  for (auto &clause : dirSpec.Clauses().v) {
+    switch (clause.Id()) {
+    case llvm::omp::Clause::OMPC_read:
+    case llvm::omp::Clause::OMPC_write:
+    case llvm::omp::Clause::OMPC_update:
+      return clause.Id();
+    default:
+      break;
+    }
+  }
+  return llvm::omp::Clause::OMPC_update;
+}
+
+bool OpenMPAtomicConstruct::IsCapture() const {
+  auto &dirSpec{std::get<OmpDirectiveSpecification>(t)};
+  return llvm::any_of(dirSpec.Clauses().v, [](auto &clause) {
+    return clause.Id() == llvm::omp::Clause::OMPC_capture;
+  });
+}
+
+bool OpenMPAtomicConstruct::IsCompare() const {
+  auto &dirSpec{std::get<OmpDirectiveSpecification>(t)};
+  return llvm::any_of(dirSpec.Clauses().v, [](auto &clause) {
+    return clause.Id() == llvm::omp::Clause::OMPC_compare;
+  });
+}
 } // namespace Fortran::parser
 
 template <typename C> static llvm::omp::Clause getClauseIdForClass(C &&) {
diff --git a/external/llvm-project/flang/lib/Parser/tools.cpp b/external/llvm-project/flang/lib/Parser/tools.cpp
index 6e5f1ed2fc66..264ca520f38b 100644
--- a/external/llvm-project/flang/lib/Parser/tools.cpp
+++ b/external/llvm-project/flang/lib/Parser/tools.cpp
@@ -174,4 +174,9 @@ const CoindexedNamedObject *GetCoindexedNamedObject(
       },
       allocateObject.u);
 }
+
+bool CheckForSingleVariableOnRHS(const AssignmentStmt &assignmentStmt) {
+  return Unwrap<Designator>(std::get<Expr>(assignmentStmt.t)) != nullptr;
+}
+
 } // namespace Fortran::parser
diff --git a/external/llvm-project/flang/lib/Parser/unparse.cpp b/external/llvm-project/flang/lib/Parser/unparse.cpp
index 0784a6703bbd..ed0f227fd5b9 100644
--- a/external/llvm-project/flang/lib/Parser/unparse.cpp
+++ b/external/llvm-project/flang/lib/Parser/unparse.cpp
@@ -2571,83 +2571,22 @@ class UnparseVisitor {
     Word(ToUpperCaseLetters(common::EnumToString(x)));
   }
 
-  void Unparse(const OmpAtomicClauseList &x) { Walk(" ", x.v, " "); }
-
-  void Unparse(const OmpAtomic &x) {
-    BeginOpenMP();
-    Word("!$OMP ATOMIC");
-    Walk(std::get<OmpAtomicClauseList>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AssignmentStmt>>(x.t));
-    BeginOpenMP();
-    Walk(std::get<std::optional<OmpEndAtomic>>(x.t), "!$OMP END ATOMIC\n");
-    EndOpenMP();
-  }
-  void Unparse(const OmpAtomicCapture &x) {
+  void Unparse(const OpenMPAtomicConstruct &x) {
     BeginOpenMP();
-    Word("!$OMP ATOMIC");
-    Walk(std::get<0>(x.t));
-    Word(" CAPTURE");
-    Walk(std::get<2>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<OmpAtomicCapture::Stmt1>(x.t));
-    Put("\n");
-    Walk(std::get<OmpAtomicCapture::Stmt2>(x.t));
-    BeginOpenMP();
-    Word("!$OMP END ATOMIC\n");
-    EndOpenMP();
-  }
-  void Unparse(const OmpAtomicCompare &x) {
-    BeginOpenMP();
-    Word("!$OMP ATOMIC");
-    Walk(std::get<0>(x.t));
-    Word(" COMPARE");
-    Walk(std::get<2>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<OmpAtomicCompareIfStmt>(x.t));
-  }
-  void Unparse(const OmpAtomicRead &x) {
-    BeginOpenMP();
-    Word("!$OMP ATOMIC");
-    Walk(std::get<0>(x.t));
-    Word(" READ");
-    Walk(std::get<2>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AssignmentStmt>>(x.t));
-    BeginOpenMP();
-    Walk(std::get<std::optional<OmpEndAtomic>>(x.t), "!$OMP END ATOMIC\n");
-    EndOpenMP();
-  }
-  void Unparse(const OmpAtomicUpdate &x) {
-    BeginOpenMP();
-    Word("!$OMP ATOMIC");
-    Walk(std::get<0>(x.t));
-    Word(" UPDATE");
-    Walk(std::get<2>(x.t));
-    Put("\n");
-    EndOpenMP();
-    Walk(std::get<Statement<AssignmentStmt>>(x.t));
-    BeginOpenMP();
-    Walk(std::get<std::optional<OmpEndAtomic>>(x.t), "!$OMP END ATOMIC\n");
-    EndOpenMP();
-  }
-  void Unparse(const OmpAtomicWrite &x) {
-    BeginOpenMP();
-    Word("!$OMP ATOMIC");
-    Walk(std::get<0>(x.t));
-    Word(" WRITE");
-    Walk(std::get<2>(x.t));
+    Word("!$OMP ");
+    Walk(std::get<OmpDirectiveSpecification>(x.t));
     Put("\n");
     EndOpenMP();
-    Walk(std::get<Statement<AssignmentStmt>>(x.t));
-    BeginOpenMP();
-    Walk(std::get<std::optional<OmpEndAtomic>>(x.t), "!$OMP END ATOMIC\n");
-    EndOpenMP();
+    Walk(std::get<Block>(x.t), "");
+    if (auto &end{std::get<std::optional<OmpDirectiveSpecification>>(x.t)}) {
+      BeginOpenMP();
+      Word("!$OMP END ");
+      Walk(*end);
+      Put("\n");
+      EndOpenMP();
+    }
   }
+
   void Unparse(const OpenMPExecutableAllocate &x) {
     const auto &fields =
         std::get<std::optional<std::list<parser::OpenMPDeclarativeAllocate>>>(
@@ -2920,23 +2859,8 @@ class UnparseVisitor {
     Put("\n");
     EndOpenMP();
   }
+  void Unparse(const OmpFailClause &x) { Walk(x.v); }
   void Unparse(const OmpMemoryOrderClause &x) { Walk(x.v); }
-  void Unparse(const OmpAtomicClause &x) {
-    common::visit(common::visitors{
-                      [&](const OmpMemoryOrderClause &y) { Walk(y); },
-                      [&](const OmpFailClause &y) {
-                        Word("FAIL(");
-                        Walk(y.v);
-                        Put(")");
-                      },
-                      [&](const OmpHintClause &y) {
-                        Word("HINT(");
-                        Walk(y.v);
-                        Put(")");
-                      },
-                  },
-        x.u);
-  }
   void Unparse(const OmpMetadirectiveDirective &x) {
     BeginOpenMP();
     Word("!$OMP METADIRECTIVE ");
@@ -3368,4 +3292,12 @@ template void Unparse<Program>(llvm::raw_ostream &, const Program &,
 template void Unparse<Expr>(llvm::raw_ostream &, const Expr &,
     const common::LangOptions &, Encoding, bool, bool, preStatementType *,
     AnalyzedObjectsAsFortran *);
+
+template void Unparse<parser::OpenMPDeclareReductionConstruct>(
+    llvm::raw_ostream &, const parser::OpenMPDeclareReductionConstruct &,
+    const common::LangOptions &, Encoding, bool, bool, preStatementType *,
+    AnalyzedObjectsAsFortran *);
+template void Unparse<parser::OmpMetadirectiveDirective>(llvm::raw_ostream &,
+    const parser::OmpMetadirectiveDirective &, const common::LangOptions &,
+    Encoding, bool, bool, preStatementType *, AnalyzedObjectsAsFortran *);
 } // namespace Fortran::parser
diff --git a/external/llvm-project/flang/lib/Semantics/CMakeLists.txt b/external/llvm-project/flang/lib/Semantics/CMakeLists.txt
index bd8cc47365f0..18c89587843a 100644
--- a/external/llvm-project/flang/lib/Semantics/CMakeLists.txt
+++ b/external/llvm-project/flang/lib/Semantics/CMakeLists.txt
@@ -32,6 +32,7 @@ add_flang_library(FortranSemantics
   dump-expr.cpp
   expression.cpp
   mod-file.cpp
+  openmp-dsa.cpp
   openmp-modifiers.cpp
   pointer-assignment.cpp
   program-tree.cpp
diff --git a/external/llvm-project/flang/lib/Semantics/assignment.cpp b/external/llvm-project/flang/lib/Semantics/assignment.cpp
index 6e55d0210ee0..43e23a9d8f60 100644
--- a/external/llvm-project/flang/lib/Semantics/assignment.cpp
+++ b/external/llvm-project/flang/lib/Semantics/assignment.cpp
@@ -43,6 +43,7 @@ class AssignmentContext {
   void Analyze(const parser::PointerAssignmentStmt &);
   void Analyze(const parser::ConcurrentControl &);
   int deviceConstructDepth_{0};
+  SemanticsContext &context() { return context_; }
 
 private:
   bool CheckForPureContext(const SomeExpr &rhs, parser::CharBlock rhsSource);
@@ -218,8 +219,17 @@ void AssignmentContext::PopWhereContext() {
 
 AssignmentChecker::~AssignmentChecker() {}
 
+SemanticsContext &AssignmentChecker::context() {
+  return context_.value().context();
+}
+
 AssignmentChecker::AssignmentChecker(SemanticsContext &context)
     : context_{new AssignmentContext{context}} {}
+
+void AssignmentChecker::Enter(
+    const parser::OpenMPDeclareReductionConstruct &x) {
+  context().set_location(x.source);
+}
 void AssignmentChecker::Enter(const parser::AssignmentStmt &x) {
   context_.value().Analyze(x);
 }
diff --git a/external/llvm-project/flang/lib/Semantics/assignment.h b/external/llvm-project/flang/lib/Semantics/assignment.h
index a67bee4a03df..4a1bb9203711 100644
--- a/external/llvm-project/flang/lib/Semantics/assignment.h
+++ b/external/llvm-project/flang/lib/Semantics/assignment.h
@@ -37,6 +37,7 @@ class AssignmentChecker : public virtual BaseChecker {
 public:
   explicit AssignmentChecker(SemanticsContext &);
   ~AssignmentChecker();
+  void Enter(const parser::OpenMPDeclareReductionConstruct &x);
   void Enter(const parser::AssignmentStmt &);
   void Enter(const parser::PointerAssignmentStmt &);
   void Enter(const parser::WhereStmt &);
@@ -54,6 +55,8 @@ class AssignmentChecker : public virtual BaseChecker {
   void Enter(const parser::OpenACCLoopConstruct &);
   void Leave(const parser::OpenACCLoopConstruct &);
 
+  SemanticsContext &context();
+
 private:
   common::Indirection<AssignmentContext> context_;
 };
diff --git a/external/llvm-project/flang/lib/Semantics/check-call.cpp b/external/llvm-project/flang/lib/Semantics/check-call.cpp
index dfc2ddbacf07..6f2503285013 100644
--- a/external/llvm-project/flang/lib/Semantics/check-call.cpp
+++ b/external/llvm-project/flang/lib/Semantics/check-call.cpp
@@ -1033,6 +1033,13 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
               *actualDataAttr == common::CUDADataAttr::Managed)) {
         actualDataAttr = common::CUDADataAttr::Device;
       }
+      // For device procedures, treat actual arguments with VALUE attribute as
+      // device data
+      if (!actualDataAttr && actualLastSymbol && IsValue(*actualLastSymbol) &&
+          (*procedure.cudaSubprogramAttrs ==
+              common::CUDASubprogramAttrs::Device)) {
+        actualDataAttr = common::CUDADataAttr::Device;
+      }
     }
     if (dummyDataAttr == common::CUDADataAttr::Device &&
         (dummyIsAssumedShape || dummyIsAssumedRank) &&
diff --git a/external/llvm-project/flang/lib/Semantics/check-cuda.cpp b/external/llvm-project/flang/lib/Semantics/check-cuda.cpp
index c024640af122..8decfb014982 100644
--- a/external/llvm-project/flang/lib/Semantics/check-cuda.cpp
+++ b/external/llvm-project/flang/lib/Semantics/check-cuda.cpp
@@ -17,6 +17,7 @@
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/ADT/StringSet.h"
 
 // Once labeled DO constructs have been canonicalized and their parse subtrees
 // transformed into parser::DoConstructs, scan the parser::Blocks of the program
@@ -61,6 +62,11 @@ bool CanonicalizeCUDA(parser::Program &program) {
 
 using MaybeMsg = std::optional<parser::MessageFormattedText>;
 
+static const llvm::StringSet<> warpFunctions_ = {"match_all_syncjj",
+    "match_all_syncjx", "match_all_syncjf", "match_all_syncjd",
+    "match_any_syncjj", "match_any_syncjx", "match_any_syncjf",
+    "match_any_syncjd"};
+
 // Traverses an evaluate::Expr<> in search of unsupported operations
 // on the device.
 
@@ -68,7 +74,7 @@ struct DeviceExprChecker
     : public evaluate::AnyTraverse<DeviceExprChecker, MaybeMsg> {
   using Result = MaybeMsg;
   using Base = evaluate::AnyTraverse<DeviceExprChecker, Result>;
-  DeviceExprChecker() : Base(*this) {}
+  explicit DeviceExprChecker(SemanticsContext &c) : Base(*this), context_{c} {}
   using Base::operator();
   Result operator()(const evaluate::ProcedureDesignator &x) const {
     if (const Symbol * sym{x.GetInterfaceSymbol()}) {
@@ -78,10 +84,17 @@ struct DeviceExprChecker
         if (auto attrs{subp->cudaSubprogramAttrs()}) {
           if (*attrs == common::CUDASubprogramAttrs::HostDevice ||
               *attrs == common::CUDASubprogramAttrs::Device) {
+            if (warpFunctions_.contains(sym->name().ToString()) &&
+                !context_.languageFeatures().IsEnabled(
+                    Fortran::common::LanguageFeature::CudaWarpMatchFunction)) {
+              return parser::MessageFormattedText(
+                  "warp match function disabled"_err_en_US);
+            }
             return {};
           }
         }
       }
+
       const Symbol &ultimate{sym->GetUltimate()};
       const Scope &scope{ultimate.owner()};
       const Symbol *mod{scope.IsModule() ? scope.symbol() : nullptr};
@@ -94,9 +107,12 @@ struct DeviceExprChecker
       // TODO(CUDA): Check for unsupported intrinsics here
       return {};
     }
+
     return parser::MessageFormattedText(
         "'%s' may not be called in device code"_err_en_US, x.GetName());
   }
+
+  SemanticsContext &context_;
 };
 
 struct FindHostArray
@@ -133,9 +149,10 @@ struct FindHostArray
   }
 };
 
-template <typename A> static MaybeMsg CheckUnwrappedExpr(const A &x) {
+template <typename A>
+static MaybeMsg CheckUnwrappedExpr(SemanticsContext &context, const A &x) {
   if (const auto *expr{parser::Unwrap<parser::Expr>(x)}) {
-    return DeviceExprChecker{}(expr->typedExpr);
+    return DeviceExprChecker{context}(expr->typedExpr);
   }
   return {};
 }
@@ -144,104 +161,124 @@ template <typename A>
 static void CheckUnwrappedExpr(
     SemanticsContext &context, SourceName at, const A &x) {
   if (const auto *expr{parser::Unwrap<parser::Expr>(x)}) {
-    if (auto msg{DeviceExprChecker{}(expr->typedExpr)}) {
+    if (auto msg{DeviceExprChecker{context}(expr->typedExpr)}) {
       context.Say(at, std::move(*msg));
     }
   }
 }
 
 template <bool CUF_KERNEL> struct ActionStmtChecker {
-  template <typename A> static MaybeMsg WhyNotOk(const A &x) {
+  template <typename A>
+  static MaybeMsg WhyNotOk(SemanticsContext &context, const A &x) {
     if constexpr (ConstraintTrait<A>) {
-      return WhyNotOk(x.thing);
+      return WhyNotOk(context, x.thing);
     } else if constexpr (WrapperTrait<A>) {
-      return WhyNotOk(x.v);
+      return WhyNotOk(context, x.v);
     } else if constexpr (UnionTrait<A>) {
-      return WhyNotOk(x.u);
+      return WhyNotOk(context, x.u);
     } else if constexpr (TupleTrait<A>) {
-      return WhyNotOk(x.t);
+      return WhyNotOk(context, x.t);
     } else {
       return parser::MessageFormattedText{
           "Statement may not appear in device code"_err_en_US};
     }
   }
   template <typename A>
-  static MaybeMsg WhyNotOk(const common::Indirection<A> &x) {
-    return WhyNotOk(x.value());
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const common::Indirection<A> &x) {
+    return WhyNotOk(context, x.value());
   }
   template <typename... As>
-  static MaybeMsg WhyNotOk(const std::variant<As...> &x) {
-    return common::visit([](const auto &x) { return WhyNotOk(x); }, x);
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const std::variant<As...> &x) {
+    return common::visit(
+        [&context](const auto &x) { return WhyNotOk(context, x); }, x);
   }
   template <std::size_t J = 0, typename... As>
-  static MaybeMsg WhyNotOk(const std::tuple<As...> &x) {
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const std::tuple<As...> &x) {
     if constexpr (J == sizeof...(As)) {
       return {};
-    } else if (auto msg{WhyNotOk(std::get<J>(x))}) {
+    } else if (auto msg{WhyNotOk(context, std::get<J>(x))}) {
       return msg;
     } else {
-      return WhyNotOk<(J + 1)>(x);
+      return WhyNotOk<(J + 1)>(context, x);
     }
   }
-  template <typename A> static MaybeMsg WhyNotOk(const std::list<A> &x) {
+  template <typename A>
+  static MaybeMsg WhyNotOk(SemanticsContext &context, const std::list<A> &x) {
     for (const auto &y : x) {
-      if (MaybeMsg result{WhyNotOk(y)}) {
+      if (MaybeMsg result{WhyNotOk(context, y)}) {
         return result;
       }
     }
     return {};
   }
-  template <typename A> static MaybeMsg WhyNotOk(const std::optional<A> &x) {
+  template <typename A>
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const std::optional<A> &x) {
     if (x) {
-      return WhyNotOk(*x);
+      return WhyNotOk(context, *x);
     } else {
       return {};
     }
   }
   template <typename A>
-  static MaybeMsg WhyNotOk(const parser::UnlabeledStatement<A> &x) {
-    return WhyNotOk(x.statement);
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::UnlabeledStatement<A> &x) {
+    return WhyNotOk(context, x.statement);
   }
   template <typename A>
-  static MaybeMsg WhyNotOk(const parser::Statement<A> &x) {
-    return WhyNotOk(x.statement);
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::Statement<A> &x) {
+    return WhyNotOk(context, x.statement);
   }
-  static MaybeMsg WhyNotOk(const parser::AllocateStmt &) {
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::AllocateStmt &) {
     return {}; // AllocateObjects are checked elsewhere
   }
-  static MaybeMsg WhyNotOk(const parser::AllocateCoarraySpec &) {
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::AllocateCoarraySpec &) {
     return parser::MessageFormattedText(
         "A coarray may not be allocated on the device"_err_en_US);
   }
-  static MaybeMsg WhyNotOk(const parser::DeallocateStmt &) {
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::DeallocateStmt &) {
     return {}; // AllocateObjects are checked elsewhere
   }
-  static MaybeMsg WhyNotOk(const parser::AssignmentStmt &x) {
-    return DeviceExprChecker{}(x.typedAssignment);
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::AssignmentStmt &x) {
+    return DeviceExprChecker{context}(x.typedAssignment);
   }
-  static MaybeMsg WhyNotOk(const parser::CallStmt &x) {
-    return DeviceExprChecker{}(x.typedCall);
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::CallStmt &x) {
+    return DeviceExprChecker{context}(x.typedCall);
+  }
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::ContinueStmt &) {
+    return {};
   }
-  static MaybeMsg WhyNotOk(const parser::ContinueStmt &) { return {}; }
-  static MaybeMsg WhyNotOk(const parser::IfStmt &x) {
-    if (auto result{
-            CheckUnwrappedExpr(std::get<parser::ScalarLogicalExpr>(x.t))}) {
+  static MaybeMsg WhyNotOk(SemanticsContext &context, const parser::IfStmt &x) {
+    if (auto result{CheckUnwrappedExpr(
+            context, std::get<parser::ScalarLogicalExpr>(x.t))}) {
       return result;
     }
-    return WhyNotOk(
+    return WhyNotOk(context,
         std::get<parser::UnlabeledStatement<parser::ActionStmt>>(x.t)
             .statement);
   }
-  static MaybeMsg WhyNotOk(const parser::NullifyStmt &x) {
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::NullifyStmt &x) {
     for (const auto &y : x.v) {
-      if (MaybeMsg result{DeviceExprChecker{}(y.typedExpr)}) {
+      if (MaybeMsg result{DeviceExprChecker{context}(y.typedExpr)}) {
         return result;
       }
     }
     return {};
   }
-  static MaybeMsg WhyNotOk(const parser::PointerAssignmentStmt &x) {
-    return DeviceExprChecker{}(x.typedAssignment);
+  static MaybeMsg WhyNotOk(
+      SemanticsContext &context, const parser::PointerAssignmentStmt &x) {
+    return DeviceExprChecker{context}(x.typedAssignment);
   }
 };
 
@@ -435,12 +472,14 @@ template <bool IsCUFKernelDo> class DeviceContextChecker {
                 ErrorIfHostSymbol(assign->lhs, source);
                 ErrorIfHostSymbol(assign->rhs, source);
               }
-              if (auto msg{ActionStmtChecker<IsCUFKernelDo>::WhyNotOk(x)}) {
+              if (auto msg{ActionStmtChecker<IsCUFKernelDo>::WhyNotOk(
+                      context_, x)}) {
                 context_.Say(source, std::move(*msg));
               }
             },
             [&](const auto &x) {
-              if (auto msg{ActionStmtChecker<IsCUFKernelDo>::WhyNotOk(x)}) {
+              if (auto msg{ActionStmtChecker<IsCUFKernelDo>::WhyNotOk(
+                      context_, x)}) {
                 context_.Say(source, std::move(*msg));
               }
             },
@@ -504,7 +543,7 @@ template <bool IsCUFKernelDo> class DeviceContextChecker {
     Check(DEREF(parser::Unwrap<parser::Expr>(x)));
   }
   void Check(const parser::Expr &expr) {
-    if (MaybeMsg msg{DeviceExprChecker{}(expr.typedExpr)}) {
+    if (MaybeMsg msg{DeviceExprChecker{context_}(expr.typedExpr)}) {
       context_.Say(expr.source, std::move(*msg));
     }
   }
diff --git a/external/llvm-project/flang/lib/Semantics/check-declarations.cpp b/external/llvm-project/flang/lib/Semantics/check-declarations.cpp
index 1d09dea06db5..f9d64485f140 100644
--- a/external/llvm-project/flang/lib/Semantics/check-declarations.cpp
+++ b/external/llvm-project/flang/lib/Semantics/check-declarations.cpp
@@ -2586,9 +2586,12 @@ void CheckHelper::CheckProcBinding(
   }
   if (overridden) {
     if (isInaccessibleDeferred) {
-      SayWithDeclaration(*overridden,
-          "Override of PRIVATE DEFERRED '%s' must appear in its module"_err_en_US,
-          symbol.name());
+      evaluate::AttachDeclaration(
+          Warn(common::LanguageFeature::InaccessibleDeferredOverride,
+              symbol.name(),
+              "Override of PRIVATE DEFERRED '%s' should appear in its module"_warn_en_US,
+              symbol.name()),
+          *overridden);
     }
     if (overridden->attrs().test(Attr::NON_OVERRIDABLE)) {
       SayWithDeclaration(*overridden,
@@ -2955,6 +2958,14 @@ static std::optional<std::string> DefinesGlobalName(const Symbol &symbol) {
   return std::nullopt;
 }
 
+static bool IsSameSymbolFromHermeticModule(
+    const Symbol &symbol, const Symbol &other) {
+  return symbol.name() == other.name() && symbol.owner().IsModule() &&
+      other.owner().IsModule() && symbol.owner() != other.owner() &&
+      symbol.owner().GetName() &&
+      symbol.owner().GetName() == other.owner().GetName();
+}
+
 // 19.2 p2
 void CheckHelper::CheckGlobalName(const Symbol &symbol) {
   if (auto global{DefinesGlobalName(symbol)}) {
@@ -2972,6 +2983,8 @@ void CheckHelper::CheckGlobalName(const Symbol &symbol) {
           (!IsExternalProcedureDefinition(symbol) ||
               !IsExternalProcedureDefinition(other))) {
         // both are procedures/BLOCK DATA, not both definitions
+      } else if (IsSameSymbolFromHermeticModule(symbol, other)) {
+        // Both symbols are the same thing.
       } else if (symbol.has<ModuleDetails>()) {
         Warn(common::LanguageFeature::BenignNameClash, symbol.name(),
             "Module '%s' conflicts with a global name"_port_en_US,
diff --git a/external/llvm-project/flang/lib/Semantics/check-omp-structure.cpp b/external/llvm-project/flang/lib/Semantics/check-omp-structure.cpp
index 46c33bd2d640..c2792c90aa2a 100644
--- a/external/llvm-project/flang/lib/Semantics/check-omp-structure.cpp
+++ b/external/llvm-project/flang/lib/Semantics/check-omp-structure.cpp
@@ -8,18 +8,26 @@
 
 #include "check-omp-structure.h"
 #include "definable.h"
+#include "resolve-names-utils.h"
 #include "flang/Evaluate/check-expression.h"
 #include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/type.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/openmp-modifiers.h"
 #include "flang/Semantics/tools.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
 #include <variant>
 
 namespace Fortran::semantics {
 
+template <typename T, typename U>
+static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
+  return !(e == f);
+}
+
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
   void OmpStructureChecker::Enter(const parser::OmpClause::X &) { \
@@ -78,6 +86,32 @@ static const parser::ArrayElement *GetArrayElementFromObj(
   return nullptr;
 }
 
+static bool IsVarOrFunctionRef(const MaybeExpr &expr) {
+  if (expr) {
+    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
+        evaluate::IsVariable(*expr);
+  } else {
+    return false;
+  }
+}
+
+static std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
+  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
+  // ForwardOwningPointer           typedExpr
+  // `- GenericExprWrapper          ^.get()
+  //    `- std::optional<Expr>      ^->v
+  return typedExpr.get()->v;
+}
+
+static std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr) {
+  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
+    return maybeExpr->GetType();
+  } else {
+    return std::nullopt;
+  }
+}
+
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -594,51 +628,26 @@ void OmpStructureChecker::CheckPredefinedAllocatorRestriction(
   }
 }
 
-template <class D>
-void OmpStructureChecker::CheckHintClause(
-    D *leftOmpClauseList, D *rightOmpClauseList, std::string_view dirName) {
-  bool foundHint{false};
+void OmpStructureChecker::Enter(const parser::OmpClause::Hint &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_hint);
+  auto &dirCtx{GetContext()};
 
-  auto checkForValidHintClause = [&](const D *clauseList) {
-    for (const auto &clause : clauseList->v) {
-      const parser::OmpHintClause *ompHintClause = nullptr;
-      if constexpr (std::is_same_v<D, const parser::OmpAtomicClauseList>) {
-        ompHintClause = std::get_if<parser::OmpHintClause>(&clause.u);
-      } else if constexpr (std::is_same_v<D, const parser::OmpClauseList>) {
-        if (auto *hint{std::get_if<parser::OmpClause::Hint>(&clause.u)}) {
-          ompHintClause = &hint->v;
-        }
-      }
-      if (!ompHintClause)
-        continue;
-      if (foundHint) {
-        context_.Say(clause.source,
-            "At most one HINT clause can appear on the %s directive"_err_en_US,
-            parser::ToUpperCaseLetters(dirName));
-      }
-      foundHint = true;
-      std::optional<std::int64_t> hintValue = GetIntValue(ompHintClause->v);
-      if (hintValue && *hintValue >= 0) {
-        /*`omp_sync_hint_nonspeculative` and `omp_lock_hint_speculative`*/
-        if ((*hintValue & 0xC) == 0xC
-            /*`omp_sync_hint_uncontended` and omp_sync_hint_contended*/
-            || (*hintValue & 0x3) == 0x3)
-          context_.Say(clause.source,
-              "Hint clause value "
-              "is not a valid OpenMP synchronization value"_err_en_US);
-      } else {
-        context_.Say(clause.source,
-            "Hint clause must have non-negative constant "
-            "integer expression"_err_en_US);
+  if (std::optional<int64_t> maybeVal{GetIntValue(x.v.v)}) {
+    int64_t val{*maybeVal};
+    if (val >= 0) {
+      // Check contradictory values.
+      if ((val & 0xC) == 0xC || // omp_sync_hint_speculative and nonspeculative
+          (val & 0x3) == 0x3) { // omp_sync_hint_contended and uncontended
+        context_.Say(dirCtx.clauseSource,
+            "The synchronization hint is not valid"_err_en_US);
       }
+    } else {
+      context_.Say(dirCtx.clauseSource,
+          "Synchronization hint must be non-negative"_err_en_US);
     }
-  };
-
-  if (leftOmpClauseList) {
-    checkForValidHintClause(leftOmpClauseList);
-  }
-  if (rightOmpClauseList) {
-    checkForValidHintClause(rightOmpClauseList);
+  } else {
+    context_.Say(dirCtx.clauseSource,
+        "Synchronization hint must be a constant integer value"_err_en_US);
   }
 }
 
@@ -2382,8 +2391,9 @@ void OmpStructureChecker::Leave(const parser::OpenMPCancelConstruct &) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPCriticalConstruct &x) {
   const auto &dir{std::get<parser::OmpCriticalDirective>(x.t)};
+  const auto &dirSource{std::get<parser::Verbatim>(dir.t).source};
   const auto &endDir{std::get<parser::OmpEndCriticalDirective>(x.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_critical);
+  PushContextAndClauseSets(dirSource, llvm::omp::Directive::OMPD_critical);
   const auto &block{std::get<parser::Block>(x.t)};
   CheckNoBranching(block, llvm::omp::Directive::OMPD_critical, dir.source);
   const auto &dirName{std::get<std::optional<parser::Name>>(dir.t)};
@@ -2416,7 +2426,6 @@ void OmpStructureChecker::Enter(const parser::OpenMPCriticalConstruct &x) {
             "Hint clause other than omp_sync_hint_none cannot be specified for "
             "an unnamed CRITICAL directive"_err_en_US});
   }
-  CheckHintClause<const parser::OmpClauseList>(&ompClause, nullptr, "CRITICAL");
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPCriticalConstruct &) {
@@ -2653,422 +2662,1418 @@ void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
   }
 }
 
-inline void OmpStructureChecker::ErrIfAllocatableVariable(
-    const parser::Variable &var) {
-  // Err out if the given symbol has
-  // ALLOCATABLE attribute
-  if (const auto *e{GetExpr(context_, var)})
-    for (const Symbol &symbol : evaluate::CollectSymbols(*e))
-      if (IsAllocatable(symbol)) {
-        const auto &designator =
-            std::get<common::Indirection<parser::Designator>>(var.u);
-        const auto *dataRef =
-            std::get_if<parser::DataRef>(&designator.value().u);
-        const parser::Name *name =
-            dataRef ? std::get_if<parser::Name>(&dataRef->u) : nullptr;
-        if (name)
-          context_.Say(name->source,
-              "%s must not have ALLOCATABLE "
-              "attribute"_err_en_US,
-              name->ToString());
+/// parser::Block is a list of executable constructs, parser::BlockConstruct
+/// is Fortran's BLOCK/ENDBLOCK construct.
+/// Strip the outermost BlockConstructs, return the reference to the Block
+/// in the executable part of the innermost of the stripped constructs.
+/// Specifically, if the given `block` has a single entry (it's a list), and
+/// the entry is a BlockConstruct, get the Block contained within. Repeat
+/// this step as many times as possible.
+static const parser::Block &GetInnermostExecPart(const parser::Block &block) {
+  const parser::Block *iter{&block};
+  while (iter->size() == 1) {
+    const parser::ExecutionPartConstruct &ep{iter->front()};
+    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
+      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
+      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
+        iter = &std::get<parser::Block>(bc->value().t);
+        continue;
       }
+    }
+    break;
+  }
+  return *iter;
 }
 
-inline void OmpStructureChecker::ErrIfLHSAndRHSSymbolsMatch(
-    const parser::Variable &var, const parser::Expr &expr) {
-  // Err out if the symbol on the LHS is also used on the RHS of the assignment
-  // statement
-  const auto *e{GetExpr(context_, expr)};
-  const auto *v{GetExpr(context_, var)};
-  if (e && v) {
-    auto vSyms{evaluate::GetSymbolVector(*v)};
-    const Symbol &varSymbol = vSyms.front();
-    for (const Symbol &symbol : evaluate::GetSymbolVector(*e)) {
-      if (varSymbol == symbol) {
-        const common::Indirection<parser::Designator> *designator =
-            std::get_if<common::Indirection<parser::Designator>>(&expr.u);
-        if (designator) {
-          auto *z{var.typedExpr.get()};
-          auto *c{expr.typedExpr.get()};
-          if (z->v == c->v) {
-            context_.Say(expr.source,
-                "RHS expression on atomic assignment statement cannot access '%s'"_err_en_US,
-                var.GetSource());
-          }
+// There is no consistent way to get the source of a given ActionStmt, so
+// extract the source information from Statement<ActionStmt> when we can,
+// and keep it around for error reporting in further analyses.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+struct AnalyzedCondStmt {
+  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
+  parser::CharBlock source;
+  SourcedActionStmt ift, iff;
+};
+
+static SourcedActionStmt GetActionStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+static SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
+// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
+// is that the ActionStmt will be either an assignment or a pointer-assignment,
+// otherwise return std::nullopt.
+// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
+// the "typedAssignment" is unset. This can happen if there are semantic errors
+// in the purported assignment.
+static std::optional<evaluate::Assignment> GetEvaluateAssignment(
+    const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
+
+  return common::visit(
+      [](auto &&s) -> std::optional<evaluate::Assignment> {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>) {
+          const TypedAssignment &typed{s.value().typedAssignment};
+          // ForwardOwningPointer                 typedAssignment
+          // `- GenericAssignmentWrapper          ^.get()
+          //    `- std::optional<Assignment>      ^->v
+          return typed.get()->v;
         } else {
-          context_.Say(expr.source,
-              "RHS expression on atomic assignment statement cannot access '%s'"_err_en_US,
-              var.GetSource());
+          return std::nullopt;
+        }
+      },
+      x->u);
+}
+
+// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
+// to separate cases where the source has something that looks like an
+// assignment, but is semantically wrong (diagnosed by general semantic
+// checks), and where the source has some other statement (which we want
+// to report as "should be an assignment").
+static bool IsAssignment(const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return false;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+
+  return common::visit(
+      [](auto &&s) -> bool {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        return std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>;
+      },
+      x->u);
+}
+
+static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  // Extract the evaluate::Expr from ScalarLogicalExpr.
+  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
+    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
+    const parser::Expr &expr{logical.thing.thing.value()};
+    return GetEvaluateExpr(expr);
+  }};
+
+  // Recognize either
+  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
+  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
+
+  if (auto &&action{GetActionStmt(x)}) {
+    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
+            &action.stmt->u)}) {
+      const parser::IfStmt &s{ifs->value()};
+      auto &&maybeCond{
+          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
+      auto &thenStmt{
+          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
+      if (maybeCond) {
+        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
+            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
+            SourcedActionStmt{}};
+      }
+    }
+    return std::nullopt;
+  }
+
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    if (auto *ifc{
+            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
+      using ElseBlock = parser::IfConstruct::ElseBlock;
+      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
+      const parser::IfConstruct &s{ifc->value()};
+
+      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
+        // Not expecting any else-if statements.
+        return std::nullopt;
+      }
+      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
+      auto &&maybeCond{getFromLogical(
+          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
+      if (!maybeCond) {
+        return std::nullopt;
+      }
+
+      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)),
+            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
+        if (result.ift.stmt && result.iff.stmt) {
+          return result;
+        }
+      } else {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
+        if (result.ift.stmt) {
+          return result;
         }
       }
     }
+    return std::nullopt;
   }
+
+  return std::nullopt;
 }
 
-inline void OmpStructureChecker::ErrIfNonScalarAssignmentStmt(
-    const parser::Variable &var, const parser::Expr &expr) {
-  // Err out if either the variable on the LHS or the expression on the RHS of
-  // the assignment statement are non-scalar (i.e. have rank > 0 or is of
-  // CHARACTER type)
-  const auto *e{GetExpr(context_, expr)};
-  const auto *v{GetExpr(context_, var)};
-  if (e && v) {
-    if (e->Rank() != 0 ||
-        (e->GetType().has_value() &&
-            e->GetType().value().category() == common::TypeCategory::Character))
-      context_.Say(expr.source,
-          "Expected scalar expression "
-          "on the RHS of atomic assignment "
-          "statement"_err_en_US);
-    if (v->Rank() != 0 ||
-        (v->GetType().has_value() &&
-            v->GetType()->category() == common::TypeCategory::Character))
-      context_.Say(var.GetSource(),
-          "Expected scalar variable "
-          "on the LHS of atomic assignment "
-          "statement"_err_en_US);
-  }
-}
-
-template <typename T, typename D>
-bool OmpStructureChecker::IsOperatorValid(const T &node, const D &variable) {
-  using AllowedBinaryOperators =
-      std::variant<parser::Expr::Add, parser::Expr::Multiply,
-          parser::Expr::Subtract, parser::Expr::Divide, parser::Expr::AND,
-          parser::Expr::OR, parser::Expr::EQV, parser::Expr::NEQV>;
-  using BinaryOperators = std::variant<parser::Expr::Add,
-      parser::Expr::Multiply, parser::Expr::Subtract, parser::Expr::Divide,
-      parser::Expr::AND, parser::Expr::OR, parser::Expr::EQV,
-      parser::Expr::NEQV, parser::Expr::Power, parser::Expr::Concat,
-      parser::Expr::LT, parser::Expr::LE, parser::Expr::EQ, parser::Expr::NE,
-      parser::Expr::GE, parser::Expr::GT>;
-
-  if constexpr (common::HasMember<T, BinaryOperators>) {
-    const auto &variableName{variable.GetSource().ToString()};
-    const auto &exprLeft{std::get<0>(node.t)};
-    const auto &exprRight{std::get<1>(node.t)};
-    if ((exprLeft.value().source.ToString() != variableName) &&
-        (exprRight.value().source.ToString() != variableName)) {
-      context_.Say(variable.GetSource(),
-          "Atomic update statement should be of form "
-          "`%s = %s operator expr` OR `%s = expr operator %s`"_err_en_US,
-          variableName, variableName, variableName, variableName);
-    }
-    return common::HasMember<T, AllowedBinaryOperators>;
+static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
+    parser::CharBlock source) {
+  // Find => in the range, if not found, find = that is not a part of
+  // <=, >=, ==, or /=.
+  auto trim{[](std::string_view v) {
+    const char *begin{v.data()};
+    const char *end{begin + v.size()};
+    while (*begin == ' ' && begin != end) {
+      ++begin;
+    }
+    while (begin != end && end[-1] == ' ') {
+      --end;
+    }
+    assert(begin != end && "Source should not be empty");
+    return parser::CharBlock(begin, end - begin);
+  }};
+
+  std::string_view sv(source.begin(), source.size());
+
+  if (auto where{sv.find("=>")}; where != sv.npos) {
+    std::string_view lhs(sv.data(), where);
+    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
+    return std::make_pair(trim(lhs), trim(rhs));
   }
-  return false;
+
+  // Go backwards, since all the exclusions above end with a '='.
+  for (size_t next{source.size()}; next > 1; --next) {
+    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
+      std::string_view lhs(sv.data(), next - 1);
+      std::string_view rhs(sv.data() + next, sv.size() - next);
+      return std::make_pair(trim(lhs), trim(rhs));
+    }
+  }
+  llvm_unreachable("Could not find assignment operator");
 }
 
-void OmpStructureChecker::CheckAtomicCaptureStmt(
-    const parser::AssignmentStmt &assignmentStmt) {
-  const auto &var{std::get<parser::Variable>(assignmentStmt.t)};
-  const auto &expr{std::get<parser::Expr>(assignmentStmt.t)};
-  common::visit(
-      common::visitors{
-          [&](const common::Indirection<parser::Designator> &designator) {
-            const auto *dataRef =
-                std::get_if<parser::DataRef>(&designator.value().u);
-            const auto *name =
-                dataRef ? std::get_if<parser::Name>(&dataRef->u) : nullptr;
-            if (name && IsAllocatable(*name->symbol))
-              context_.Say(name->source,
-                  "%s must not have ALLOCATABLE "
-                  "attribute"_err_en_US,
-                  name->ToString());
-          },
-          [&](const auto &) {
-            // Anything other than a `parser::Designator` is not allowed
-            context_.Say(expr.source,
-                "Expected scalar variable "
-                "of intrinsic type on RHS of atomic "
-                "assignment statement"_err_en_US);
-          }},
-      expr.u);
-  ErrIfLHSAndRHSSymbolsMatch(var, expr);
-  ErrIfNonScalarAssignmentStmt(var, expr);
-}
-
-void OmpStructureChecker::CheckAtomicWriteStmt(
-    const parser::AssignmentStmt &assignmentStmt) {
-  const auto &var{std::get<parser::Variable>(assignmentStmt.t)};
-  const auto &expr{std::get<parser::Expr>(assignmentStmt.t)};
-  ErrIfAllocatableVariable(var);
-  ErrIfLHSAndRHSSymbolsMatch(var, expr);
-  ErrIfNonScalarAssignmentStmt(var, expr);
-}
-
-void OmpStructureChecker::CheckAtomicUpdateStmt(
-    const parser::AssignmentStmt &assignment) {
-  const auto &expr{std::get<parser::Expr>(assignment.t)};
-  const auto &var{std::get<parser::Variable>(assignment.t)};
-  bool isIntrinsicProcedure{false};
-  bool isValidOperator{false};
-  common::visit(
-      common::visitors{
-          [&](const common::Indirection<parser::FunctionReference> &x) {
-            isIntrinsicProcedure = true;
-            const auto &procedureDesignator{
-                std::get<parser::ProcedureDesignator>(x.value().v.t)};
-            const parser::Name *name{
-                std::get_if<parser::Name>(&procedureDesignator.u)};
-            if (name &&
-                !(name->source == "max" || name->source == "min" ||
-                    name->source == "iand" || name->source == "ior" ||
-                    name->source == "ieor")) {
-              context_.Say(expr.source,
-                  "Invalid intrinsic procedure name in "
-                  "OpenMP ATOMIC (UPDATE) statement"_err_en_US);
-            }
-          },
-          [&](const auto &x) {
-            if (!IsOperatorValid(x, var)) {
-              context_.Say(expr.source,
-                  "Invalid or missing operator in atomic update "
-                  "statement"_err_en_US);
-            } else
-              isValidOperator = true;
-          },
-      },
-      expr.u);
-  if (const auto *e{GetExpr(context_, expr)}) {
-    const auto *v{GetExpr(context_, var)};
-    if (e->Rank() != 0 ||
-        (e->GetType().has_value() &&
-            e->GetType().value().category() == common::TypeCategory::Character))
-      context_.Say(expr.source,
-          "Expected scalar expression "
-          "on the RHS of atomic update assignment "
-          "statement"_err_en_US);
-    if (v->Rank() != 0 ||
-        (v->GetType().has_value() &&
-            v->GetType()->category() == common::TypeCategory::Character))
-      context_.Say(var.GetSource(),
-          "Expected scalar variable "
-          "on the LHS of atomic update assignment "
-          "statement"_err_en_US);
-    auto vSyms{evaluate::GetSymbolVector(*v)};
-    const Symbol &varSymbol = vSyms.front();
-    int numOfSymbolMatches{0};
-    SymbolVector exprSymbols{evaluate::GetSymbolVector(*e)};
-    for (const Symbol &symbol : exprSymbols) {
-      if (varSymbol == symbol) {
-        numOfSymbolMatches++;
+namespace atomic {
+
+struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
+                                 std::vector<SomeExpr>, false> {
+  using Result = std::vector<SomeExpr>;
+  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
+  DesignatorCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    // Once in a designator, don't traverse it any further (i.e. only
+    // collect top-level designators).
+    auto copy{x};
+    return Result{AsGenericExpr(std::move(copy))};
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
       }
+    }};
+    (moveAppend(v, std::move(results)), ...);
+    return v;
+  }
+};
+
+struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
+  using Base = evaluate::AnyTraverse<VariableFinder>;
+  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
+
+  using Base::operator();
+
+  template <typename T>
+  bool operator()(const evaluate::Designator<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+  template <typename T>
+  bool operator()(const evaluate::FunctionRef<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+private:
+  const SomeExpr &var;
+};
+} // namespace atomic
+
+static bool IsPointerAssignment(const evaluate::Assignment &x) {
+  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
+      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
+}
+
+static bool IsCheckForAssociated(const SomeExpr &cond) {
+  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
+}
+
+static bool HasCommonDesignatorSymbols(
+    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
+  // Compare the designators used in "other" with the designators whose
+  // symbols are given in baseSyms.
+  // This is a part of the check if these two expressions can access the same
+  // storage: if the designators used in them are different enough, then they
+  // will be assumed not to access the same memory.
+  //
+  // Consider an (array element) expression x%y(w%z), the corresponding symbol
+  // vector will be {x, y, w, z} (i.e. the symbols for these names).
+  // Check whether this exact sequence appears anywhere in any the symbol
+  // vector for "other". This will be true for x(y) and x(y+1), so this is
+  // not a sufficient condition, but can be used to eliminate candidates
+  // before doing more exhaustive checks.
+  //
+  // If any of the symbols in this sequence are function names, assume that
+  // there is no storage overlap, mostly because it would be impossible in
+  // general to determine what storage the function will access.
+  // Note: if f is pure, then two calls to f will access the same storage
+  // when called with the same arguments. This check is not done yet.
+
+  if (llvm::any_of(
+          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
+    // If there is a function symbol in the chain then we can't infer much
+    // about the accessed storage.
+    return false;
+  }
+
+  auto isSubsequence{// Is u a subsequence of v.
+      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
+        size_t us{u.size()}, vs{v.size()};
+        if (us > vs) {
+          return false;
+        }
+        for (size_t off{0}; off != vs - us + 1; ++off) {
+          bool same{true};
+          for (size_t i{0}; i != us; ++i) {
+            if (u[i] != v[off + i]) {
+              same = false;
+              break;
+            }
+          }
+          if (same) {
+            return true;
+          }
+        }
+        return false;
+      }};
+
+  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
+  return isSubsequence(baseSyms, otherSyms);
+}
+
+static bool HasCommonTopLevelDesignators(
+    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
+  // Compare designators directly as expressions. This will ensure
+  // that x(y) and x(y+1) are not flagged as overlapping, whereas
+  // the symbol vectors for both of these would be identical.
+  std::vector<SomeExpr> otherDsgs{atomic::DesignatorCollector{}(other)};
+
+  for (auto &s : baseDsgs) {
+    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
+      return true;
     }
-    if (isIntrinsicProcedure) {
-      std::string varName = var.GetSource().ToString();
-      if (numOfSymbolMatches != 1)
-        context_.Say(expr.source,
-            "Intrinsic procedure"
-            " arguments in atomic update statement"
-            " must have exactly one occurence of '%s'"_err_en_US,
-            varName);
-      else if (varSymbol != exprSymbols.front() &&
-          varSymbol != exprSymbols.back())
-        context_.Say(expr.source,
-            "Atomic update statement "
-            "should be of the form `%s = intrinsic_procedure(%s, expr_list)` "
-            "OR `%s = intrinsic_procedure(expr_list, %s)`"_err_en_US,
-            varName, varName, varName, varName);
-    } else if (isValidOperator) {
-      if (numOfSymbolMatches != 1)
-        context_.Say(expr.source,
-            "Exactly one occurence of '%s' "
-            "expected on the RHS of atomic update assignment statement"_err_en_US,
-            var.GetSource().ToString());
+  }
+  return false;
+}
+
+static const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
+  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
+  std::vector<SomeExpr> baseDsgs{atomic::DesignatorCollector{}(base)};
+
+  for (const SomeExpr &expr : exprs) {
+    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
+      continue;
     }
+    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
+      return &expr;
+    }
+  }
+  return nullptr;
+}
+
+static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
+  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
+  // for example.
+  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
+}
+
+static bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
+  return atomic::VariableFinder{sub}(super);
+}
+
+static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
+  if (value) {
+    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
+        evaluate::GenericExprWrapper::Deleter);
   }
+}
 
-  ErrIfAllocatableVariable(var);
+static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
+    std::optional<evaluate::Assignment> value) {
+  if (value) {
+    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
+        evaluate::GenericAssignmentWrapper::Deleter);
+  }
 }
 
-void OmpStructureChecker::CheckAtomicCompareConstruct(
-    const parser::OmpAtomicCompare &atomicCompareConstruct) {
+static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
+    int what,
+    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+  parser::OpenMPAtomicConstruct::Analysis::Op operation;
+  operation.what = what;
+  SetAssignment(operation.assign, maybeAssign);
+  return operation;
+}
 
-  // TODO: Check that the if-stmt is `if (var == expr) var = new`
-  //       [with or without then/end-do]
+static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
+    const SomeExpr &atom, const MaybeExpr &cond,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
+  // Defined in flang/include/flang/Parser/parse-tree.h
+  //
+  // struct Analysis {
+  //   struct Kind {
+  //     static constexpr int None = 0;
+  //     static constexpr int Read = 1;
+  //     static constexpr int Write = 2;
+  //     static constexpr int Update = Read | Write;
+  //     static constexpr int Action = 3; // Bits containing N, R, W, U
+  //     static constexpr int IfTrue = 4;
+  //     static constexpr int IfFalse = 8;
+  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
+  //   };
+  //   struct Op {
+  //     int what;
+  //     TypedAssignment assign;
+  //   };
+  //   TypedExpr atom, cond;
+  //   Op op0, op1;
+  // };
+
+  parser::OpenMPAtomicConstruct::Analysis an;
+  SetExpr(an.atom, atom);
+  SetExpr(an.cond, cond);
+  an.op0 = std::move(op0);
+  an.op1 = std::move(op1);
+  return an;
+}
+
+void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
+    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
+    parser::CharBlock source) {
+  if (auto *expr{HasStorageOverlap(base, exprs)}) {
+    context_.Say(source,
+        "Within atomic operation %s and %s access the same storage"_warn_en_US,
+        base.AsFortran(), expr->AsFortran());
+  }
+}
 
-  unsigned version{context_.langOptions().OpenMPVersion};
-  if (version < 51) {
-    context_.Say(atomicCompareConstruct.source,
-        "%s construct not allowed in %s, %s"_err_en_US,
-        atomicCompareConstruct.source, ThisVersion(version), TryVersion(51));
-  }
-
-  // TODO: More work needed here. Some of the Update restrictions need to
-  // be added, but Update isn't the same either.
-}
-
-// TODO: Allow cond-update-stmt once compare clause is supported.
-void OmpStructureChecker::CheckAtomicCaptureConstruct(
-    const parser::OmpAtomicCapture &atomicCaptureConstruct) {
-  const parser::AssignmentStmt &stmt1 =
-      std::get<parser::OmpAtomicCapture::Stmt1>(atomicCaptureConstruct.t)
-          .v.statement;
-  const auto &stmt1Var{std::get<parser::Variable>(stmt1.t)};
-  const auto &stmt1Expr{std::get<parser::Expr>(stmt1.t)};
-  const auto *v1 = GetExpr(context_, stmt1Var);
-  const auto *e1 = GetExpr(context_, stmt1Expr);
-
-  const parser::AssignmentStmt &stmt2 =
-      std::get<parser::OmpAtomicCapture::Stmt2>(atomicCaptureConstruct.t)
-          .v.statement;
-  const auto &stmt2Var{std::get<parser::Variable>(stmt2.t)};
-  const auto &stmt2Expr{std::get<parser::Expr>(stmt2.t)};
-  const auto *v2 = GetExpr(context_, stmt2Var);
-  const auto *e2 = GetExpr(context_, stmt2Expr);
-
-  if (e1 && v1 && e2 && v2) {
-    if (semantics::checkForSingleVariableOnRHS(stmt1)) {
-      CheckAtomicCaptureStmt(stmt1);
-      if (semantics::checkForSymbolMatch(v2, e2)) {
-        // ATOMIC CAPTURE construct is of the form [capture-stmt, update-stmt]
-        CheckAtomicUpdateStmt(stmt2);
+void OmpStructureChecker::ErrorShouldBeVariable(
+    const MaybeExpr &expr, parser::CharBlock source) {
+  if (expr) {
+    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
+        expr->AsFortran());
+  } else {
+    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
+  }
+}
+
+/// Check if `expr` satisfies the following conditions for x and v:
+///
+/// [6.0:189:10-12]
+/// - x and v (as applicable) are either scalar variables or
+///   function references with scalar data pointer result of non-character
+///   intrinsic type or variables that are non-polymorphic scalar pointers
+///   and any length type parameter must be constant.
+void OmpStructureChecker::CheckAtomicType(
+    SymbolRef sym, parser::CharBlock source, std::string_view name) {
+  const DeclTypeSpec *typeSpec{sym->GetType()};
+  if (!typeSpec) {
+    return;
+  }
+
+  if (!IsPointer(sym)) {
+    using Category = DeclTypeSpec::Category;
+    Category cat{typeSpec->category()};
+    if (cat == Category::Character) {
+      context_.Say(source,
+          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
+    } else if (cat != Category::Numeric && cat != Category::Logical) {
+      context_.Say(source,
+          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
+    }
+    return;
+  }
+
+  // Variable is a pointer.
+  if (typeSpec->IsPolymorphic()) {
+    context_.Say(source,
+        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
+        name);
+    return;
+  }
+
+  // Go over all length parameters, if any, and check if they are
+  // explicit.
+  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
+    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
+          // "entry" is a map entry
+          return entry.second.isLen() && !entry.second.isExplicit();
+        })) {
+      context_.Say(source,
+          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
+          name);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckAtomicVariable(
+    const SomeExpr &atom, parser::CharBlock source) {
+  if (atom.Rank() != 0) {
+    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
+        atom.AsFortran());
+  }
+
+  std::vector<SomeExpr> dsgs{atomic::DesignatorCollector{}(atom)};
+  assert(dsgs.size() == 1 && "Should have a single top-level designator");
+  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
+
+  CheckAtomicType(syms.back(), source, atom.AsFortran());
+
+  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
+    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
+        atom.AsFortran());
+  }
+}
+
+std::pair<const parser::ExecutionPartConstruct *,
+    const parser::ExecutionPartConstruct *>
+OmpStructureChecker::CheckUpdateCapture(
+    const parser::ExecutionPartConstruct *ec1,
+    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
+  // Decide which statement is the atomic update and which is the capture.
+  //
+  // The two allowed cases are:
+  //   x = ...      atomic-var = ...
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  // or
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  //   x = ...      atomic-var = ...
+  //
+  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
+  // (which makes more sense, as it captures the original value of the atomic
+  // variable).
+  //
+  // If the two statements don't fit these criteria, return a pair of default-
+  // constructed values.
+  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
+      const parser::ExecutionPartConstruct *>;
+
+  SourcedActionStmt act1{GetActionStmt(ec1)};
+  SourcedActionStmt act2{GetActionStmt(ec2)};
+  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
+  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
+  if (!maybeAssign1 || !maybeAssign2) {
+    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
+      context_.Say(source,
+          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
+    }
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  auto as1{*maybeAssign1}, as2{*maybeAssign2};
+
+  auto isUpdateCapture{
+      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
+        return IsSameOrConvertOf(c.rhs, u.lhs);
+      }};
+
+  // Do some checks that narrow down the possible choices for the update
+  // and the capture statements. This will help to emit better diagnostics.
+  // 1. An assignment could be an update (cbu) if the left-hand side is a
+  //    subexpression of the right-hand side.
+  // 2. An assignment could be a capture (cbc) if the right-hand side is
+  //    a variable (or a function ref), with potential type conversions.
+  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
+  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
+  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
+  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
+
+  // We want to diagnose cases where both assignments cannot be an update,
+  // or both cannot be a capture, as well as cases where either assignment
+  // cannot be any of these two.
+  //
+  // If we organize these boolean values into a matrix
+  //   |cbu1 cbu2|
+  //   |cbc1 cbc2|
+  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
+  // row or column, including the case where everything is zero. All these
+  // cases correspond to the determinant of the matrix being 0, which suggests
+  // that checking the det may be a convenient diagnostic check. There is only
+  // one additional case where the det is 0, which is when the matrix is all 1
+  // ("true"). The "all true" case represents the situation where both
+  // assignments could be an update as well as a capture. On the other hand,
+  // whenever det != 0, the roles of the update and the capture can be
+  // unambiguously assigned to as1 and as2 [1].
+  //
+  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
+  // det = 0, leaving 6 cases where det != 0:
+  //   0 1   0 1   1 0   1 0   1 1   1 1
+  //   1 0   1 1   0 1   1 1   0 1   1 0
+  // In each case the classification is unambiguous.
+
+  //     |cbu1 cbu2|
+  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
+  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
+
+  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
+                                  const std::string &expr) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
+        expr);
+  }};
+
+  auto errorNeitherWorks{[&]() {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
+  }};
+
+  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
+    // If det != 0, then the checks unambiguously suggest a specific
+    // categorization.
+    // If det == 0, then this function should be called only if the
+    // checks haven't ruled out any possibility, i.e. when both assigments
+    // could still be either updates or captures.
+    if (det > 0) {
+      // as1 is update, as2 is capture
+      if (isUpdateCapture(as1, as2)) {
+        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
+      } else {
+        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else if (det < 0) {
+      // as2 is update, as1 is capture
+      if (isUpdateCapture(as2, as1)) {
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
       } else {
-        // ATOMIC CAPTURE construct is of the form [capture-stmt, write-stmt]
-        CheckAtomicWriteStmt(stmt2);
+        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
       }
-      if (!(*e1 == *v2)) {
-        context_.Say(stmt1Expr.source,
-            "Captured variable/array element/derived-type component %s expected to be assigned in the second statement of ATOMIC CAPTURE construct"_err_en_US,
-            stmt1Expr.source);
+    } else {
+      bool updateFirst{isUpdateCapture(as1, as2)};
+      bool captureFirst{isUpdateCapture(as2, as1)};
+      if (updateFirst && captureFirst) {
+        // If both assignment could be the update and both could be the
+        // capture, emit a warning about the ambiguity.
+        context_.Say(act1.source,
+            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
       }
-    } else if (semantics::checkForSymbolMatch(v1, e1) &&
-        semantics::checkForSingleVariableOnRHS(stmt2)) {
-      // ATOMIC CAPTURE construct is of the form [update-stmt, capture-stmt]
-      CheckAtomicUpdateStmt(stmt1);
-      CheckAtomicCaptureStmt(stmt2);
-      // Variable updated in stmt1 should be captured in stmt2
-      if (!(*v1 == *e2)) {
-        context_.Say(stmt1Var.GetSource(),
-            "Updated variable/array element/derived-type component %s expected to be captured in the second statement of ATOMIC CAPTURE construct"_err_en_US,
-            stmt1Var.GetSource());
+      if (updateFirst != captureFirst) {
+        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
+        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
+        return std::make_pair(upd, cap);
       }
+      assert(!updateFirst && !captureFirst);
+      errorNeitherWorks();
+      return std::make_pair(nullptr, nullptr);
+    }
+  }};
+
+  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
+    return makeSelectionFromDet(det);
+  }
+  assert(det == 0 && "Prior checks should have covered det != 0");
+
+  // If neither of the statements is an RMW update, it could still be a
+  // "write" update. Pretty much any assignment can be a write update, so
+  // recompute det with cbu1 = cbu2 = true.
+  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
+    return makeSelectionFromDet(writeDet);
+  }
+
+  // It's only errors from here on.
+
+  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
+    errorNeitherWorks();
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // The remaining cases are that
+  // - no candidate for update, or for capture,
+  // - one of the assigments cannot be anything.
+
+  if (!cbu1 && !cbu2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  } else if (!cbc1 && !cbc2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
+    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
+    context_.Say(src,
+        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // All cases should have been covered.
+  llvm_unreachable("Unchecked condition");
+}
+
+void OmpStructureChecker::CheckAtomicCaptureAssignment(
+    const evaluate::Assignment &capture, const SomeExpr &atom,
+    parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &cap{capture.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, rsrc);
+    // This part should have been checked prior to calling this function.
+    assert(*GetConvertInput(capture.rhs) == atom &&
+        "This cannot be a capture assignment");
+    CheckStorageOverlap(atom, {cap}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicReadAssignment(
+    const evaluate::Assignment &read, parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+
+  if (auto maybe{GetConvertInput(read.rhs)}) {
+    const SomeExpr &atom{*maybe};
+
+    if (!IsVarOrFunctionRef(atom)) {
+      ErrorShouldBeVariable(atom, rsrc);
     } else {
-      context_.Say(stmt1Expr.source,
-          "Invalid ATOMIC CAPTURE construct statements. Expected one of [update-stmt, capture-stmt], [capture-stmt, update-stmt], or [capture-stmt, write-stmt]"_err_en_US);
+      CheckAtomicVariable(atom, rsrc);
+      CheckStorageOverlap(atom, {read.lhs}, source);
     }
+  } else {
+    ErrorShouldBeVariable(read.rhs, rsrc);
   }
 }
 
-void OmpStructureChecker::CheckAtomicMemoryOrderClause(
-    const parser::OmpAtomicClauseList *leftHandClauseList,
-    const parser::OmpAtomicClauseList *rightHandClauseList) {
-  int numMemoryOrderClause{0};
-  int numFailClause{0};
-  auto checkForValidMemoryOrderClause = [&](const parser::OmpAtomicClauseList
-                                                *clauseList) {
-    for (const auto &clause : clauseList->v) {
-      if (std::get_if<parser::OmpFailClause>(&clause.u)) {
-        numFailClause++;
-        if (numFailClause > 1) {
-          context_.Say(clause.source,
-              "More than one FAIL clause not allowed on OpenMP ATOMIC construct"_err_en_US);
-          return;
+void OmpStructureChecker::CheckAtomicWriteAssignment(
+    const evaluate::Assignment &write, parser::CharBlock source) {
+  // [6.0:190:13-15]
+  // A write structured block is write-statement, a write statement that has
+  // one of the following forms:
+  //   x = expr
+  //   x => expr
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{write.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, lsrc);
+    CheckStorageOverlap(atom, {write.rhs}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateAssignment(
+    const evaluate::Assignment &update, parser::CharBlock source) {
+  // [6.0:191:1-7]
+  // An update structured block is update-statement, an update statement
+  // that has one of the following forms:
+  //   x = x operator expr
+  //   x = expr operator x
+  //   x = intrinsic-procedure-name (x)
+  //   x = intrinsic-procedure-name (x, expr-list)
+  //   x = intrinsic-procedure-name (expr-list, x)
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{update.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, lsrc);
+
+  std::pair<operation::Operator, std::vector<SomeExpr>> top{
+      operation::Operator::Unknown, {}};
+  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
+    top = GetTopLevelOperation(*maybeInput);
+  }
+  switch (top.first) {
+  case operation::Operator::Add:
+  case operation::Operator::Sub:
+  case operation::Operator::Mul:
+  case operation::Operator::Div:
+  case operation::Operator::And:
+  case operation::Operator::Or:
+  case operation::Operator::Eqv:
+  case operation::Operator::Neqv:
+  case operation::Operator::Min:
+  case operation::Operator::Max:
+  case operation::Operator::Identity:
+    break;
+  case operation::Operator::Call:
+    context_.Say(source,
+        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Convert:
+    context_.Say(source,
+        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Intrinsic:
+    context_.Say(source,
+        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Constant:
+  case operation::Operator::Unknown:
+    context_.Say(
+        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(source,
+        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
+        operation::ToString(top.first));
+    return;
+  }
+  // Check if `atom` occurs exactly once in the argument list.
+  std::vector<SomeExpr> nonAtom;
+  auto unique{[&]() { // -> iterator
+    auto found{top.second.end()};
+    for (auto i{top.second.begin()}, e{top.second.end()}; i != e; ++i) {
+      if (IsSameOrConvertOf(*i, atom)) {
+        if (found != top.second.end()) {
+          return top.second.end();
         }
+        found = i;
       } else {
-        if (std::get_if<parser::OmpMemoryOrderClause>(&clause.u)) {
-          numMemoryOrderClause++;
-          if (numMemoryOrderClause > 1) {
-            context_.Say(clause.source,
-                "More than one memory order clause not allowed on OpenMP ATOMIC construct"_err_en_US);
-            return;
-          }
+        nonAtom.push_back(*i);
+      }
+    }
+    return found;
+  }()};
+
+  if (unique == top.second.end()) {
+    if (top.first == operation::Operator::Identity) {
+      // This is "x = y".
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
+          atom.AsFortran());
+    } else {
+      assert(top.first != operation::Operator::Identity &&
+          "Handle this separately");
+      context_.Say(rsrc,
+          "The atomic variable %s should occur exactly once among the arguments of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+    }
+  } else {
+    CheckStorageOverlap(atom, nonAtom, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
+    const SomeExpr &cond, parser::CharBlock condSource,
+    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
+  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
+  const SomeExpr &atom{assign.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, arsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, alsrc);
+
+  auto top{GetTopLevelOperation(cond)};
+  // Missing arguments to operations would have been diagnosed by now.
+
+  switch (top.first) {
+  case operation::Operator::Associated:
+    if (atom != top.second.front()) {
+      context_.Say(assignSource,
+          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
+    }
+    break;
+  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
+  case operation::Operator::Eq:
+  case operation::Operator::Eqv:
+  // x ordop expr | expr ordop x
+  case operation::Operator::Lt:
+  case operation::Operator::Gt: {
+    const SomeExpr &arg0{top.second[0]};
+    const SomeExpr &arg1{top.second[1]};
+    if (IsSameOrConvertOf(arg0, atom)) {
+      CheckStorageOverlap(atom, {arg1}, condSource);
+    } else if (IsSameOrConvertOf(arg1, atom)) {
+      CheckStorageOverlap(atom, {arg0}, condSource);
+    } else {
+      assert(top.first != operation::Operator::Identity &&
+          "Handle this separately");
+      context_.Say(assignSource,
+          "An argument of the %s operator should be the target of the assignment"_err_en_US,
+          operation::ToString(top.first));
+    }
+    break;
+  }
+  case operation::Operator::Identity:
+  case operation::Operator::True:
+  case operation::Operator::False:
+    break;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(condSource,
+        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
+        operation::ToString(top.first));
+    break;
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
+    const AnalyzedCondStmt &update, parser::CharBlock source) {
+  // The condition/statements must be:
+  // - cond: x equalop e      ift: x =  d     iff: -
+  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
+  // - cond: associated(x)    ift: x => expr  iff: -
+  // - cond: associated(x, e) ift: x => expr  iff: -
+
+  // The if-true statement must be present, and must be an assignment.
+  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
+  if (!maybeAssign) {
+    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
+    } else {
+      context_.Say(
+          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
+    }
+    return;
+  }
+  const evaluate::Assignment assign{*maybeAssign};
+  const SomeExpr &atom{assign.lhs};
+
+  CheckAtomicConditionalUpdateAssignment(
+      update.cond, update.source, assign, update.ift.source);
+
+  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
+
+  if (update.iff) {
+    context_.Say(update.iff.source,
+        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateOnly(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeUpdate->lhs};
+      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdate(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // Allowable forms are (single-statement):
+  // - if ...
+  // - x = (... ? ... : x)
+  // and two-statement:
+  // - r = cond ; if (r) ...
+
+  const parser::ExecutionPartConstruct *ust{nullptr}; // update
+  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
+
+  if (body.size() == 1) {
+    ust = &body.front();
+  } else if (body.size() == 2) {
+    cst = &body.front();
+    ust = &body.back();
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
+    return;
+  }
+
+  // Flang doesn't support conditional-expr yet, so all update statements
+  // are if-statements.
+
+  // IfStmt:        if (...) ...
+  // IfConstruct:   if (...) then ... endif
+  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
+  if (!maybeUpdate) {
+    context_.Say(source,
+        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
+    return;
+  }
+
+  AnalyzedCondStmt &update{*maybeUpdate};
+
+  if (SourcedActionStmt action{GetActionStmt(cst)}) {
+    // The "condition" statement must be `r = cond`.
+    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
+      if (maybeCond->lhs != update.cond) {
+        context_.Say(update.source,
+            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
+            maybeCond->lhs.AsFortran());
+      } else {
+        // If it's "r = ...; if (r) ..." then put the original condition
+        // in `update`.
+        update.cond = maybeCond->rhs;
+      }
+    } else {
+      context_.Say(action.source,
+          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
+    }
+  }
+
+  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
+
+  CheckAtomicConditionalUpdateStmt(update, source);
+  if (IsCheckForAssociated(update.cond)) {
+    if (!IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
+    }
+  } else {
+    if (IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
+    }
+  }
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
+      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
+      MakeAtomicAnalysisOp(Analysis::None));
+}
+
+void OmpStructureChecker::CheckAtomicUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() != 2) {
+    context_.Say(source,
+        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
+    return;
+  }
+
+  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
+  if (!uec || !cec) {
+    // Diagnostics already emitted.
+    return;
+  }
+  SourcedActionStmt uact{GetActionStmt(uec)};
+  SourcedActionStmt cact{GetActionStmt(cec)};
+  // The "dereferences" of std::optional are guaranteed to be valid after
+  // CheckUpdateCapture.
+  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
+  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
+
+  const SomeExpr &atom{update.lhs};
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  int action;
+
+  if (IsMaybeAtomicWrite(update)) {
+    action = Analysis::Write;
+    CheckAtomicWriteAssignment(update, uact.source);
+  } else {
+    action = Analysis::Update;
+    CheckAtomicUpdateAssignment(update, uact.source);
+  }
+  CheckAtomicCaptureAssignment(capture, atom, cact.source);
+
+  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
+    context_.Say(cact.source,
+        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+    return;
+  }
+
+  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(action, update),
+        MakeAtomicAnalysisOp(Analysis::Read, capture));
+  } else {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(Analysis::Read, capture),
+        MakeAtomicAnalysisOp(action, update));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // There are two different variants of this:
+  // (1) conditional-update and capture separately:
+  //     This form only allows single-statement updates, i.e. the update
+  //     form "r = cond; if (r) ..." is not allowed.
+  // (2) conditional-update combined with capture in a single statement:
+  //     This form does allow the condition to be calculated separately,
+  //     i.e. "r = cond; if (r) ...".
+  // Regardless of what form it is, the actual update assignment is a
+  // proper write, i.e. "x = d", where d does not depend on x.
+
+  AnalyzedCondStmt update;
+  SourcedActionStmt capture;
+  bool captureAlways{true}, captureFirst{true};
+
+  auto extractCapture{[&]() {
+    capture = update.iff;
+    captureAlways = false;
+    update.iff = SourcedActionStmt{};
+  }};
+
+  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
+    // The non-update statement is either "r = cond" or the capture.
+    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
+      if (update.cond == maybeAssign->lhs) {
+        // If this is "r = cond; if (r) ...", then update the condition.
+        update.cond = maybeAssign->rhs;
+        update.source = action.source;
+        // In this form, the update and the capture are combined into
+        // an IF-THEN-ELSE statement.
+        extractCapture();
+      } else {
+        // Assume this is the capture-statement.
+        capture = action;
+      }
+    }
+  }};
+
+  if (body.size() == 2) {
+    // This could be
+    // - capture; conditional-update (in any order), or
+    // - r = cond; if (r) capture-update
+    const parser::ExecutionPartConstruct *st1{&body.front()};
+    const parser::ExecutionPartConstruct *st2{&body.back()};
+    // In either case, the conditional statement can be analyzed by
+    // AnalyzeConditionalStmt, whereas the other statement cannot.
+    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
+      update = *maybeUpdate1;
+      classifyNonUpdate(GetActionStmt(st2));
+      captureFirst = false;
+    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
+      update = *maybeUpdate2;
+      classifyNonUpdate(GetActionStmt(st1));
+    } else {
+      // None of the statements are conditional, this rules out the
+      // "r = cond; if (r) ..." and the "capture + conditional-update"
+      // variants. This could still be capture + write (which is classified
+      // as conditional-update-capture in the spec).
+      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
+      if (!uec || !cec) {
+        // Diagnostics already emitted.
+        return;
+      }
+      SourcedActionStmt uact{GetActionStmt(uec)};
+      SourcedActionStmt cact{GetActionStmt(cec)};
+      update.ift = uact;
+      capture = cact;
+      if (uec == st1) {
+        captureFirst = false;
+      }
+    }
+  } else if (body.size() == 1) {
+    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
+      update = *maybeUpdate;
+      // This is the form with update and capture combined into an IF-THEN-ELSE
+      // statement. The capture-statement is always the ELSE branch.
+      extractCapture();
+    } else {
+      goto invalid;
+    }
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
+    return;
+  invalid:
+    context_.Say(source,
+        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
+    return;
+  }
+
+  // The update must have a form `x = d` or `x => d`.
+  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
+    const SomeExpr &atom{maybeWrite->lhs};
+    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
+    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
+      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
+
+      if (IsPointerAssignment(*maybeWrite) !=
+          IsPointerAssignment(*maybeCapture)) {
+        context_.Say(capture.source,
+            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+        return;
+      }
+    } else {
+      if (!IsAssignment(capture.stmt)) {
+        context_.Say(capture.source,
+            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
+      }
+      return;
+    }
+  } else {
+    if (!IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
+    }
+    return;
+  }
+
+  // update.iff should be empty here, the capture statement should be
+  // stored in "capture".
+
+  // Fill out the analysis in the AST node.
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  bool condUnused{std::visit(
+      [](auto &&s) {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
+          return true;
+        } else {
+          return false;
         }
+      },
+      update.cond.u)};
+
+  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
+  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
+
+  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
+  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
+
+  if (captureFirst) {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
+  } else {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicRead(
+    const parser::OpenMPAtomicConstruct &x) {
+  // [6.0:190:5-7]
+  // A read structured block is read-statement, a read statement that has one
+  // of the following forms:
+  //   v = x
+  //   v => x
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Read cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
+      CheckAtomicReadAssignment(*maybeRead, action.source);
+
+      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
+        const SomeExpr &atom{*maybe};
+        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
+            MakeAtomicAnalysisOp(Analysis::None));
       }
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
     }
-  };
-  if (leftHandClauseList) {
-    checkForValidMemoryOrderClause(leftHandClauseList);
+  } else {
+    context_.Say(x.source,
+        "ATOMIC READ operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWrite(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Write cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeWrite->lhs};
+      CheckAtomicWriteAssignment(*maybeWrite, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC WRITE operation should have a single statement"_err_en_US);
   }
-  if (rightHandClauseList) {
-    checkForValidMemoryOrderClause(rightHandClauseList);
+}
+
+void OmpStructureChecker::CheckAtomicUpdate(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &block{std::get<parser::Block>(x.t)};
+
+  bool isConditional{x.IsCompare()};
+  bool isCapture{x.IsCapture()};
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (isConditional && isCapture) {
+    CheckAtomicConditionalUpdateCapture(x, body, x.source);
+  } else if (isConditional) {
+    CheckAtomicConditionalUpdate(x, body, x.source);
+  } else if (isCapture) {
+    CheckAtomicUpdateCapture(x, body, x.source);
+  } else { // update-only
+    CheckAtomicUpdateOnly(x, body, x.source);
   }
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
-  common::visit(
-      common::visitors{
-          [&](const parser::OmpAtomic &atomicConstruct) {
-            const auto &dir{std::get<parser::Verbatim>(atomicConstruct.t)};
-            PushContextAndClauseSets(
-                dir.source, llvm::omp::Directive::OMPD_atomic);
-            CheckAtomicUpdateStmt(
-                std::get<parser::Statement<parser::AssignmentStmt>>(
-                    atomicConstruct.t)
-                    .statement);
-            CheckAtomicMemoryOrderClause(
-                &std::get<parser::OmpAtomicClauseList>(atomicConstruct.t),
-                nullptr);
-            CheckHintClause<const parser::OmpAtomicClauseList>(
-                &std::get<parser::OmpAtomicClauseList>(atomicConstruct.t),
-                nullptr, "ATOMIC");
-          },
-          [&](const parser::OmpAtomicUpdate &atomicUpdate) {
-            const auto &dir{std::get<parser::Verbatim>(atomicUpdate.t)};
-            PushContextAndClauseSets(
-                dir.source, llvm::omp::Directive::OMPD_atomic);
-            CheckAtomicUpdateStmt(
-                std::get<parser::Statement<parser::AssignmentStmt>>(
-                    atomicUpdate.t)
-                    .statement);
-            CheckAtomicMemoryOrderClause(
-                &std::get<0>(atomicUpdate.t), &std::get<2>(atomicUpdate.t));
-            CheckHintClause<const parser::OmpAtomicClauseList>(
-                &std::get<0>(atomicUpdate.t), &std::get<2>(atomicUpdate.t),
-                "UPDATE");
-          },
-          [&](const parser::OmpAtomicRead &atomicRead) {
-            const auto &dir{std::get<parser::Verbatim>(atomicRead.t)};
-            PushContextAndClauseSets(
-                dir.source, llvm::omp::Directive::OMPD_atomic);
-            CheckAtomicMemoryOrderClause(
-                &std::get<0>(atomicRead.t), &std::get<2>(atomicRead.t));
-            CheckHintClause<const parser::OmpAtomicClauseList>(
-                &std::get<0>(atomicRead.t), &std::get<2>(atomicRead.t), "READ");
-            CheckAtomicCaptureStmt(
-                std::get<parser::Statement<parser::AssignmentStmt>>(
-                    atomicRead.t)
-                    .statement);
-          },
-          [&](const parser::OmpAtomicWrite &atomicWrite) {
-            const auto &dir{std::get<parser::Verbatim>(atomicWrite.t)};
-            PushContextAndClauseSets(
-                dir.source, llvm::omp::Directive::OMPD_atomic);
-            CheckAtomicMemoryOrderClause(
-                &std::get<0>(atomicWrite.t), &std::get<2>(atomicWrite.t));
-            CheckHintClause<const parser::OmpAtomicClauseList>(
-                &std::get<0>(atomicWrite.t), &std::get<2>(atomicWrite.t),
-                "WRITE");
-            CheckAtomicWriteStmt(
-                std::get<parser::Statement<parser::AssignmentStmt>>(
-                    atomicWrite.t)
-                    .statement);
-          },
-          [&](const parser::OmpAtomicCapture &atomicCapture) {
-            const auto &dir{std::get<parser::Verbatim>(atomicCapture.t)};
-            PushContextAndClauseSets(
-                dir.source, llvm::omp::Directive::OMPD_atomic);
-            CheckAtomicMemoryOrderClause(
-                &std::get<0>(atomicCapture.t), &std::get<2>(atomicCapture.t));
-            CheckHintClause<const parser::OmpAtomicClauseList>(
-                &std::get<0>(atomicCapture.t), &std::get<2>(atomicCapture.t),
-                "CAPTURE");
-            CheckAtomicCaptureConstruct(atomicCapture);
-          },
-          [&](const parser::OmpAtomicCompare &atomicCompare) {
-            const auto &dir{std::get<parser::Verbatim>(atomicCompare.t)};
-            PushContextAndClauseSets(
-                dir.source, llvm::omp::Directive::OMPD_atomic);
-            CheckAtomicMemoryOrderClause(
-                &std::get<0>(atomicCompare.t), &std::get<2>(atomicCompare.t));
-            CheckHintClause<const parser::OmpAtomicClauseList>(
-                &std::get<0>(atomicCompare.t), &std::get<2>(atomicCompare.t),
-                "CAPTURE");
-            CheckAtomicCompareConstruct(atomicCompare);
-          },
-      },
-      x.u);
+  // All of the following groups have the "exclusive" property, i.e. at
+  // most one clause from each group is allowed.
+  // The exclusivity-checking code should eventually be unified for all
+  // clauses, with clause groups defined in OMP.td.
+  std::array atomic{llvm::omp::Clause::OMPC_read,
+      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
+  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
+      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
+      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
+
+  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
+                          std::string_view name,
+                          const parser::OmpClauseList &clauses) {
+    const parser::OmpClause *present{nullptr};
+    for (const parser::OmpClause &clause : clauses.v) {
+      llvm::omp::Clause id{clause.Id()};
+      if (!llvm::is_contained(group, id)) {
+        continue;
+      }
+      if (present == nullptr) {
+        present = &clause;
+        continue;
+      } else if (id == present->Id()) {
+        // Ignore repetitions of the same clause, those will be diagnosed
+        // separately.
+        continue;
+      }
+      parser::MessageFormattedText txt(
+          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
+          name.data());
+      parser::Message message(clause.source, txt);
+      message.Attach(present->source,
+          "Previous clause from this group provided here"_en_US);
+      context_.Say(std::move(message));
+      return;
+    }
+  }};
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
+  llvm::omp::Clause kind{x.GetKind()};
+
+  checkExclusive(atomic, "atomic", dirSpec.Clauses());
+  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
+
+  switch (kind) {
+  case llvm::omp::Clause::OMPC_read:
+    CheckAtomicRead(x);
+    break;
+  case llvm::omp::Clause::OMPC_write:
+    CheckAtomicWrite(x);
+    break;
+  case llvm::omp::Clause::OMPC_update:
+    CheckAtomicUpdate(x);
+    break;
+  default:
+    break;
+  }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
@@ -3318,7 +4323,6 @@ CHECK_SIMPLE_CLAUSE(Final, OMPC_final)
 CHECK_SIMPLE_CLAUSE(Flush, OMPC_flush)
 CHECK_SIMPLE_CLAUSE(Full, OMPC_full)
 CHECK_SIMPLE_CLAUSE(Grainsize, OMPC_grainsize)
-CHECK_SIMPLE_CLAUSE(Hint, OMPC_hint)
 CHECK_SIMPLE_CLAUSE(Holds, OMPC_holds)
 CHECK_SIMPLE_CLAUSE(Inclusive, OMPC_inclusive)
 CHECK_SIMPLE_CLAUSE(Initializer, OMPC_initializer)
@@ -3507,6 +4511,17 @@ bool OmpStructureChecker::CheckReductionOperator(
         break;
       }
     }
+    // User-defined operators are OK if there has been a declared reduction
+    // for that. We mangle those names to store the user details.
+    if (const auto *definedOp{std::get_if<parser::DefinedOpName>(&dOpr.u)}) {
+      std::string mangled{MangleDefinedOperator(definedOp->v.symbol->name())};
+      const Scope &scope{definedOp->v.symbol->owner()};
+      if (const Symbol *symbol{scope.FindSymbol(mangled)}) {
+        if (symbol->detailsIf<UserReductionDetails>()) {
+          return true;
+        }
+      }
+    }
     context_.Say(source, "Invalid reduction operator in %s clause."_err_en_US,
         parser::ToUpperCaseLetters(getClauseName(clauseId).str()));
     return false;
@@ -3520,8 +4535,7 @@ bool OmpStructureChecker::CheckReductionOperator(
       valid =
           llvm::is_contained({"max", "min", "iand", "ior", "ieor"}, realName);
       if (!valid) {
-        auto *misc{name->symbol->detailsIf<MiscDetails>()};
-        valid = misc && misc->kind() == MiscDetails::Kind::ConstructName;
+        valid = name->symbol->detailsIf<UserReductionDetails>();
       }
     }
     if (!valid) {
@@ -3601,8 +4615,20 @@ void OmpStructureChecker::CheckReductionObjects(
   }
 }
 
+static bool CheckSymbolSupportsType(const Scope &scope,
+    const parser::CharBlock &name, const DeclTypeSpec &type) {
+  if (const auto *symbol{scope.FindSymbol(name)}) {
+    if (const auto *reductionDetails{
+            symbol->detailsIf<UserReductionDetails>()}) {
+      return reductionDetails->SupportsType(type);
+    }
+  }
+  return false;
+}
+
 static bool IsReductionAllowedForType(
-    const parser::OmpReductionIdentifier &ident, const DeclTypeSpec &type) {
+    const parser::OmpReductionIdentifier &ident, const DeclTypeSpec &type,
+    const Scope &scope, SemanticsContext &context) {
   auto isLogical{[](const DeclTypeSpec &type) -> bool {
     return type.category() == DeclTypeSpec::Logical;
   }};
@@ -3622,27 +4648,40 @@ static bool IsReductionAllowedForType(
       case parser::DefinedOperator::IntrinsicOperator::Multiply:
       case parser::DefinedOperator::IntrinsicOperator::Add:
       case parser::DefinedOperator::IntrinsicOperator::Subtract:
-        return type.IsNumeric(TypeCategory::Integer) ||
+        if (type.IsNumeric(TypeCategory::Integer) ||
             type.IsNumeric(TypeCategory::Real) ||
-            type.IsNumeric(TypeCategory::Complex);
+            type.IsNumeric(TypeCategory::Complex))
+          return true;
+        break;
 
       case parser::DefinedOperator::IntrinsicOperator::AND:
       case parser::DefinedOperator::IntrinsicOperator::OR:
       case parser::DefinedOperator::IntrinsicOperator::EQV:
       case parser::DefinedOperator::IntrinsicOperator::NEQV:
-        return isLogical(type);
+        if (isLogical(type)) {
+          return true;
+        }
+        break;
 
       // Reduction identifier is not in OMP5.2 Table 5.2
       default:
         DIE("This should have been caught in CheckIntrinsicOperator");
         return false;
       }
-    }
-    return true;
+      parser::CharBlock name{MakeNameFromOperator(*intrinsicOp, context)};
+      return CheckSymbolSupportsType(scope, name, type);
+    } else if (const auto *definedOp{
+                   std::get_if<parser::DefinedOpName>(&dOpr.u)}) {
+      return CheckSymbolSupportsType(
+          scope, MangleDefinedOperator(definedOp->v.symbol->name()), type);
+    }
+    llvm_unreachable(
+        "A DefinedOperator is either a DefinedOpName or an IntrinsicOperator");
   }};
 
   auto checkDesignator{[&](const parser::ProcedureDesignator &procD) {
     const parser::Name *name{std::get_if<parser::Name>(&procD.u)};
+    CHECK(name && name->symbol);
     if (name && name->symbol) {
       const SourceName &realName{name->symbol->GetUltimate().name()};
       // OMP5.2: The type [...] of a list item that appears in a
@@ -3651,18 +4690,35 @@ static bool IsReductionAllowedForType(
         // IAND: arguments must be integers: F2023 16.9.100
         // IEOR: arguments must be integers: F2023 16.9.106
         // IOR: arguments must be integers: F2023 16.9.111
-        return type.IsNumeric(TypeCategory::Integer);
+        if (type.IsNumeric(TypeCategory::Integer)) {
+          return true;
+        }
       } else if (realName == "max" || realName == "min") {
         // MAX: arguments must be integer, real, or character:
         // F2023 16.9.135
         // MIN: arguments must be integer, real, or character:
         // F2023 16.9.141
-        return type.IsNumeric(TypeCategory::Integer) ||
-            type.IsNumeric(TypeCategory::Real) || isCharacter(type);
+        if (type.IsNumeric(TypeCategory::Integer) ||
+            type.IsNumeric(TypeCategory::Real) || isCharacter(type)) {
+          return true;
+        }
+      }
+
+      // If we get here, it may be a user declared reduction, so check
+      // if the symbol has UserReductionDetails, and if so, the type is
+      // supported.
+      if (const auto *reductionDetails{
+              name->symbol->detailsIf<UserReductionDetails>()}) {
+        return reductionDetails->SupportsType(type);
       }
+
+      // We also need to check for mangled names (max, min, iand, ieor and ior)
+      // and then check if the type is there.
+      parser::CharBlock mangledName{MangleSpecialFunctions(name->source)};
+      return CheckSymbolSupportsType(scope, mangledName, type);
     }
-    // TODO: user defined reduction operators. Just allow everything for now.
-    return true;
+    // Everything else is "not matching type".
+    return false;
   }};
 
   return common::visit(
@@ -3677,7 +4733,8 @@ void OmpStructureChecker::CheckReductionObjectTypes(
 
   for (auto &[symbol, source] : symbols) {
     if (auto *type{symbol->GetType()}) {
-      if (!IsReductionAllowedForType(ident, *type)) {
+      const auto &scope{context_.FindScope(symbol->name())};
+      if (!IsReductionAllowedForType(ident, *type, scope, context_)) {
         context_.Say(source,
             "The type of '%s' is incompatible with the reduction operator."_err_en_US,
             symbol->name());
@@ -3947,40 +5004,6 @@ void OmpStructureChecker::CheckIsLoopIvPartOfClause(
     }
   }
 }
-// Following clauses have a separate node in parse-tree.h.
-// Atomic-clause
-CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicRead, OMPC_read)
-CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicWrite, OMPC_write)
-CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicUpdate, OMPC_update)
-CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicCapture, OMPC_capture)
-
-void OmpStructureChecker::Leave(const parser::OmpAtomicRead &) {
-  CheckNotAllowedIfClause(llvm::omp::Clause::OMPC_read,
-      {llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_acq_rel});
-}
-
-void OmpStructureChecker::Leave(const parser::OmpAtomicWrite &) {
-  CheckNotAllowedIfClause(llvm::omp::Clause::OMPC_write,
-      {llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_acq_rel});
-}
-
-void OmpStructureChecker::Leave(const parser::OmpAtomicUpdate &) {
-  CheckNotAllowedIfClause(llvm::omp::Clause::OMPC_update,
-      {llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_acq_rel});
-}
-
-// OmpAtomic node represents atomic directive without atomic-clause.
-// atomic-clause - READ,WRITE,UPDATE,CAPTURE.
-void OmpStructureChecker::Leave(const parser::OmpAtomic &) {
-  if (const auto *clause{FindClause(llvm::omp::Clause::OMPC_acquire)}) {
-    context_.Say(clause->source,
-        "Clause ACQUIRE is not allowed on the ATOMIC directive"_err_en_US);
-  }
-  if (const auto *clause{FindClause(llvm::omp::Clause::OMPC_acq_rel)}) {
-    context_.Say(clause->source,
-        "Clause ACQ_REL is not allowed on the ATOMIC directive"_err_en_US);
-  }
-}
 
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
@@ -4407,6 +5430,13 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
     CheckDoacross(*doaDep);
     CheckDependenceType(doaDep->GetDepType());
   } else {
+    using Modifier = parser::OmpDependClause::TaskDep::Modifier;
+    auto &modifiers{std::get<std::optional<std::list<Modifier>>>(taskDep->t)};
+    if (!modifiers) {
+      context_.Say(GetContext().clauseSource,
+          "A DEPEND clause on a TASK construct must have a valid task dependence type"_err_en_US);
+      return;
+    }
     CheckTaskDependenceType(taskDep->GetTaskDepType());
   }
 
@@ -4952,21 +5982,6 @@ void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
   ExitDirectiveNest(ContextSelectorNest);
 }
 
-std::optional<evaluate::DynamicType> OmpStructureChecker::GetDynamicType(
-    const common::Indirection<parser::Expr> &parserExpr) {
-  // Indirection<parser::Expr>      parserExpr
-  //  `- parser::Expr               ^.value()
-  const parser::TypedExpr &typedExpr{parserExpr.value().typedExpr};
-  // ForwardOwningPointer           typedExpr
-  // `- GenericExprWrapper          ^.get()
-  //    `- std::optional<Expr>      ^->v
-  if (auto maybeExpr{typedExpr.get()->v}) {
-    return maybeExpr->GetType();
-  } else {
-    return std::nullopt;
-  }
-}
-
 const std::list<parser::OmpTraitProperty> &
 OmpStructureChecker::GetTraitPropertyList(
     const parser::OmpTraitSelector &trait) {
@@ -5356,7 +6371,7 @@ void OmpStructureChecker::CheckTraitCondition(
     const parser::OmpTraitProperty &property{properties.front()};
     auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
 
-    auto maybeType{GetDynamicType(scalarExpr.thing)};
+    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
     if (!maybeType || maybeType->category() != TypeCategory::Logical) {
       context_.Say(property.source,
           "%s trait requires a single LOGICAL expression"_err_en_US,
@@ -5497,6 +6512,29 @@ void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
 void OmpStructureChecker::CheckArraySection(
     const parser::ArrayElement &arrayElement, const parser::Name &name,
     const llvm::omp::Clause clause) {
+  // Sometimes substring operations are incorrectly parsed as array accesses.
+  // Detect this by looking for array accesses on character variables which are
+  // not arrays.
+  bool isSubstring{false};
+  evaluate::ExpressionAnalyzer ea{context_};
+  if (MaybeExpr expr = ea.Analyze(arrayElement.base)) {
+    std::optional<evaluate::Shape> shape = evaluate::GetShape(expr);
+    // Not an array: rank 0
+    if (shape && shape->size() == 0) {
+      if (std::optional<evaluate::DynamicType> type = expr->GetType()) {
+        if (type->category() == evaluate::TypeCategory::Character) {
+          // Substrings are explicitly denied by the standard [6.0:163:9-11].
+          // This is supported as an extension. This restriction was added in
+          // OpenMP 5.2.
+          isSubstring = true;
+          context_.Say(GetContext().clauseSource,
+              "The use of substrings in OpenMP argument lists has been disallowed since OpenMP 5.2."_port_en_US);
+        } else {
+          llvm_unreachable("Array indexing on a variable that isn't an array");
+        }
+      }
+    }
+  }
   if (!arrayElement.subscripts.empty()) {
     for (const auto &subscript : arrayElement.subscripts) {
       if (const auto *triplet{
@@ -5514,6 +6552,10 @@ void OmpStructureChecker::CheckArraySection(
                   name.ToString(),
                   parser::ToUpperCaseLetters(getClauseName(clause).str()));
             }
+            if (isSubstring) {
+              context_.Say(GetContext().clauseSource,
+                  "Cannot specify a step for a substring"_err_en_US);
+            }
           }
           const auto &lower{std::get<0>(triplet->t)};
           const auto &upper{std::get<1>(triplet->t)};
@@ -5537,6 +6579,12 @@ void OmpStructureChecker::CheckArraySection(
             }
           }
         }
+      } else if (std::get_if<parser::IntExpr>(&subscript.u)) {
+        // base(n) is valid as an array index but not as a substring operation
+        if (isSubstring) {
+          context_.Say(GetContext().clauseSource,
+              "Substrings must be in the form parent-string(lb:ub)"_err_en_US);
+        }
       }
     }
   }
diff --git a/external/llvm-project/flang/lib/Semantics/check-omp-structure.h b/external/llvm-project/flang/lib/Semantics/check-omp-structure.h
index 1a8059d8548e..2074ec611dc2 100644
--- a/external/llvm-project/flang/lib/Semantics/check-omp-structure.h
+++ b/external/llvm-project/flang/lib/Semantics/check-omp-structure.h
@@ -48,6 +48,7 @@ static const OmpDirectiveSet noWaitClauseNotAllowedSet{
 } // namespace llvm
 
 namespace Fortran::semantics {
+struct AnalyzedCondStmt;
 
 // Mapping from 'Symbol' to 'Source' to keep track of the variables
 // used in multiple clauses
@@ -144,15 +145,6 @@ class OmpStructureChecker
   void Leave(const parser::OmpClauseList &);
   void Enter(const parser::OmpClause &);
 
-  void Enter(const parser::OmpAtomicRead &);
-  void Leave(const parser::OmpAtomicRead &);
-  void Enter(const parser::OmpAtomicWrite &);
-  void Leave(const parser::OmpAtomicWrite &);
-  void Enter(const parser::OmpAtomicUpdate &);
-  void Leave(const parser::OmpAtomicUpdate &);
-  void Enter(const parser::OmpAtomicCapture &);
-  void Leave(const parser::OmpAtomic &);
-
   void Enter(const parser::DoConstruct &);
   void Leave(const parser::DoConstruct &);
 
@@ -192,8 +184,6 @@ class OmpStructureChecker
   void CheckAllowedMapTypes(const parser::OmpMapType::Value &,
       const std::list<parser::OmpMapType::Value> &);
 
-  std::optional<evaluate::DynamicType> GetDynamicType(
-      const common::Indirection<parser::Expr> &);
   const std::list<parser::OmpTraitProperty> &GetTraitPropertyList(
       const parser::OmpTraitSelector &);
   std::optional<llvm::omp::Clause> GetClauseFromProperty(
@@ -265,14 +255,44 @@ class OmpStructureChecker
   void CheckDoWhile(const parser::OpenMPLoopConstruct &x);
   void CheckAssociatedLoopConstraints(const parser::OpenMPLoopConstruct &x);
   template <typename T, typename D> bool IsOperatorValid(const T &, const D &);
-  void CheckAtomicMemoryOrderClause(
-      const parser::OmpAtomicClauseList *, const parser::OmpAtomicClauseList *);
-  void CheckAtomicUpdateStmt(const parser::AssignmentStmt &);
-  void CheckAtomicCaptureStmt(const parser::AssignmentStmt &);
-  void CheckAtomicWriteStmt(const parser::AssignmentStmt &);
-  void CheckAtomicCaptureConstruct(const parser::OmpAtomicCapture &);
-  void CheckAtomicCompareConstruct(const parser::OmpAtomicCompare &);
-  void CheckAtomicConstructStructure(const parser::OpenMPAtomicConstruct &);
+
+  void CheckStorageOverlap(const evaluate::Expr<evaluate::SomeType> &,
+      llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>>, parser::CharBlock);
+  void ErrorShouldBeVariable(const MaybeExpr &expr, parser::CharBlock source);
+  void CheckAtomicType(
+      SymbolRef sym, parser::CharBlock source, std::string_view name);
+  void CheckAtomicVariable(
+      const evaluate::Expr<evaluate::SomeType> &, parser::CharBlock);
+  std::pair<const parser::ExecutionPartConstruct *,
+      const parser::ExecutionPartConstruct *>
+  CheckUpdateCapture(const parser::ExecutionPartConstruct *ec1,
+      const parser::ExecutionPartConstruct *ec2, parser::CharBlock source);
+  void CheckAtomicCaptureAssignment(const evaluate::Assignment &capture,
+      const SomeExpr &atom, parser::CharBlock source);
+  void CheckAtomicReadAssignment(
+      const evaluate::Assignment &read, parser::CharBlock source);
+  void CheckAtomicWriteAssignment(
+      const evaluate::Assignment &write, parser::CharBlock source);
+  void CheckAtomicUpdateAssignment(
+      const evaluate::Assignment &update, parser::CharBlock source);
+  void CheckAtomicConditionalUpdateAssignment(const SomeExpr &cond,
+      parser::CharBlock condSource, const evaluate::Assignment &assign,
+      parser::CharBlock assignSource);
+  void CheckAtomicConditionalUpdateStmt(
+      const AnalyzedCondStmt &update, parser::CharBlock source);
+  void CheckAtomicUpdateOnly(const parser::OpenMPAtomicConstruct &x,
+      const parser::Block &body, parser::CharBlock source);
+  void CheckAtomicConditionalUpdate(const parser::OpenMPAtomicConstruct &x,
+      const parser::Block &body, parser::CharBlock source);
+  void CheckAtomicUpdateCapture(const parser::OpenMPAtomicConstruct &x,
+      const parser::Block &body, parser::CharBlock source);
+  void CheckAtomicConditionalUpdateCapture(
+      const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+      parser::CharBlock source);
+  void CheckAtomicRead(const parser::OpenMPAtomicConstruct &x);
+  void CheckAtomicWrite(const parser::OpenMPAtomicConstruct &x);
+  void CheckAtomicUpdate(const parser::OpenMPAtomicConstruct &x);
+
   void CheckDistLinear(const parser::OpenMPLoopConstruct &x);
   void CheckSIMDNest(const parser::OpenMPConstruct &x);
   void CheckTargetNest(const parser::OpenMPConstruct &x);
@@ -324,7 +344,6 @@ class OmpStructureChecker
   void EnterDirectiveNest(const int index) { directiveNest_[index]++; }
   void ExitDirectiveNest(const int index) { directiveNest_[index]--; }
   int GetDirectiveNest(const int index) { return directiveNest_[index]; }
-  template <typename D> void CheckHintClause(D *, D *, std::string_view);
   inline void ErrIfAllocatableVariable(const parser::Variable &);
   inline void ErrIfLHSAndRHSSymbolsMatch(
       const parser::Variable &, const parser::Expr &);
diff --git a/external/llvm-project/flang/lib/Semantics/expression.cpp b/external/llvm-project/flang/lib/Semantics/expression.cpp
index d68e71f57f14..f4af738284ed 100644
--- a/external/llvm-project/flang/lib/Semantics/expression.cpp
+++ b/external/llvm-project/flang/lib/Semantics/expression.cpp
@@ -2907,7 +2907,7 @@ std::pair<const Symbol *, bool> ExpressionAnalyzer::ResolveGeneric(
                 continue;
               }
               // Matching distance is smaller than the previously matched
-              // specific. Let it go thourgh so the current procedure is picked.
+              // specific. Let it go through so the current procedure is picked.
             } else {
               // 16.9.144(6): a bare NULL() is not allowed as an actual
               // argument to a generic procedure if the specific procedure
@@ -4824,31 +4824,41 @@ bool ArgumentAnalyzer::OkLogicalIntegerAssignment(
 
 std::optional<ProcedureRef> ArgumentAnalyzer::GetDefinedAssignmentProc() {
   const Symbol *proc{nullptr};
+  bool isProcElemental{false};
   std::optional<int> passedObjectIndex;
   std::string oprNameString{"assignment(=)"};
   parser::CharBlock oprName{oprNameString};
   const auto &scope{context_.context().FindScope(source_)};
-  // If multiple resolutions were possible, they will have been already
-  // diagnosed.
   {
     auto restorer{context_.GetContextualMessages().DiscardMessages()};
     if (const Symbol *symbol{scope.FindSymbol(oprName)}) {
       ExpressionAnalyzer::AdjustActuals noAdjustment;
       proc =
           context_.ResolveGeneric(*symbol, actuals_, noAdjustment, true).first;
+      if (proc) {
+        isProcElemental = IsElementalProcedure(*proc);
+      }
     }
-    for (std::size_t i{0}; !proc && i < actuals_.size(); ++i) {
+    for (std::size_t i{0}; (!proc || isProcElemental) && i < actuals_.size();
+        ++i) {
       const Symbol *generic{nullptr};
       if (const Symbol *
           binding{FindBoundOp(oprName, i, generic, /*isSubroutine=*/true)}) {
-        if (CheckAccessibleSymbol(scope, DEREF(generic))) {
-          // ignore inaccessible type-bound ASSIGNMENT(=) generic
-        } else if (const Symbol *
-            resolution{GetBindingResolution(GetType(i), *binding)}) {
-          proc = resolution;
-        } else {
-          proc = binding;
-          passedObjectIndex = i;
+        // ignore inaccessible type-bound ASSIGNMENT(=) generic
+        if (!CheckAccessibleSymbol(scope, DEREF(generic))) {
+          const Symbol *resolution{GetBindingResolution(GetType(i), *binding)};
+          const Symbol &newProc{*(resolution ? resolution : binding)};
+          bool isElemental{IsElementalProcedure(newProc)};
+          if (!proc || !isElemental) {
+            // Non-elemental resolution overrides elemental
+            proc = &newProc;
+            isProcElemental = isElemental;
+            if (resolution) {
+              passedObjectIndex.reset();
+            } else {
+              passedObjectIndex = i;
+            }
+          }
         }
       }
     }
diff --git a/external/llvm-project/flang/lib/Semantics/mod-file.cpp b/external/llvm-project/flang/lib/Semantics/mod-file.cpp
index a1ec95656220..9f9e9f584045 100644
--- a/external/llvm-project/flang/lib/Semantics/mod-file.cpp
+++ b/external/llvm-project/flang/lib/Semantics/mod-file.cpp
@@ -143,18 +143,22 @@ void ModFileWriter::Write(const Symbol &symbol) {
   std::string path{context_.moduleDirectory() + '/' +
       ModFileName(symbol.name(), ancestorName, context_.moduleFileSuffix())};
 
-  UnorderedSymbolSet hermeticModules;
-  hermeticModules.insert(symbol);
+  std::set<std::string> hermeticModuleNames;
+  hermeticModuleNames.insert(symbol.name().ToString());
   UnorderedSymbolSet additionalModules;
   PutSymbols(DEREF(symbol.scope()),
       hermeticModuleFileOutput_ ? &additionalModules : nullptr);
   auto asStr{GetAsString(symbol)};
   while (!additionalModules.empty()) {
-    for (auto ref : UnorderedSymbolSet{std::move(additionalModules)}) {
-      if (hermeticModules.insert(*ref).second &&
-          !ref->owner().IsIntrinsicModules()) {
-        PutSymbols(DEREF(ref->scope()), &additionalModules);
-        asStr += GetAsString(*ref);
+    UnorderedSymbolSet nextPass{std::move(additionalModules)};
+    additionalModules.clear();
+    for (const Symbol &modSym : nextPass) {
+      if (!modSym.owner().IsIntrinsicModules() &&
+          hermeticModuleNames.find(modSym.name().ToString()) ==
+              hermeticModuleNames.end()) {
+        hermeticModuleNames.insert(modSym.name().ToString());
+        PutSymbols(DEREF(modSym.scope()), &additionalModules);
+        asStr += GetAsString(modSym);
       }
     }
   }
@@ -894,6 +898,7 @@ void ModFileWriter::PutEntity(llvm::raw_ostream &os, const Symbol &symbol) {
           [&](const ObjectEntityDetails &) { PutObjectEntity(os, symbol); },
           [&](const ProcEntityDetails &) { PutProcEntity(os, symbol); },
           [&](const TypeParamDetails &) { PutTypeParam(os, symbol); },
+          [&](const UserReductionDetails &) { PutUserReduction(os, symbol); },
           [&](const auto &) {
             common::die("PutEntity: unexpected details: %s",
                 DetailsToString(symbol.details()).c_str());
@@ -1043,6 +1048,28 @@ void ModFileWriter::PutTypeParam(llvm::raw_ostream &os, const Symbol &symbol) {
   os << '\n';
 }
 
+void ModFileWriter::PutUserReduction(
+    llvm::raw_ostream &os, const Symbol &symbol) {
+  const auto &details{symbol.get<UserReductionDetails>()};
+  // The module content for a OpenMP Declare Reduction is the OpenMP
+  // declaration. There may be multiple declarations.
+  // Decls are pointers, so do not use a reference.
+  for (const auto decl : details.GetDeclList()) {
+    common::visit( //
+        common::visitors{//
+            [&](const parser::OpenMPDeclareReductionConstruct *d) {
+              Unparse(os, *d, context_.langOptions());
+            },
+            [&](const parser::OmpMetadirectiveDirective *m) {
+              Unparse(os, *m, context_.langOptions());
+            },
+            [&](const auto &) {
+              DIE("Unknown OpenMP DECLARE REDUCTION content");
+            }},
+        decl);
+  }
+}
+
 void PutInit(llvm::raw_ostream &os, const Symbol &symbol, const MaybeExpr &init,
     const parser::Expr *unanalyzed, SemanticsContext &context) {
   if (IsNamedConstant(symbol) || symbol.owner().IsDerivedType()) {
diff --git a/external/llvm-project/flang/lib/Semantics/mod-file.h b/external/llvm-project/flang/lib/Semantics/mod-file.h
index 82538fb51087..9e5724089b3c 100644
--- a/external/llvm-project/flang/lib/Semantics/mod-file.h
+++ b/external/llvm-project/flang/lib/Semantics/mod-file.h
@@ -80,6 +80,7 @@ class ModFileWriter {
   void PutDerivedType(const Symbol &, const Scope * = nullptr);
   void PutDECStructure(const Symbol &, const Scope * = nullptr);
   void PutTypeParam(llvm::raw_ostream &, const Symbol &);
+  void PutUserReduction(llvm::raw_ostream &, const Symbol &);
   void PutSubprogram(const Symbol &);
   void PutGeneric(const Symbol &);
   void PutUse(const Symbol &);
diff --git a/external/llvm-project/flang/lib/Semantics/openmp-dsa.cpp b/external/llvm-project/flang/lib/Semantics/openmp-dsa.cpp
new file mode 100644
index 000000000000..48aa36febe5c
--- /dev/null
+++ b/external/llvm-project/flang/lib/Semantics/openmp-dsa.cpp
@@ -0,0 +1,29 @@
+//===-- flang/lib/Semantics/openmp-dsa.cpp ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Semantics/openmp-dsa.h"
+
+namespace Fortran::semantics {
+
+Symbol::Flags GetSymbolDSA(const Symbol &symbol) {
+  Symbol::Flags dsaFlags{Symbol::Flag::OmpPrivate,
+      Symbol::Flag::OmpFirstPrivate, Symbol::Flag::OmpLastPrivate,
+      Symbol::Flag::OmpShared, Symbol::Flag::OmpLinear,
+      Symbol::Flag::OmpReduction};
+  Symbol::Flags dsa{symbol.flags() & dsaFlags};
+  if (dsa.any()) {
+    return dsa;
+  }
+  // If no DSA are set use those from the host associated symbol, if any.
+  if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
+    return GetSymbolDSA(details->symbol());
+  }
+  return {};
+}
+
+} // namespace Fortran::semantics
diff --git a/external/llvm-project/flang/lib/Semantics/resolve-directives.cpp b/external/llvm-project/flang/lib/Semantics/resolve-directives.cpp
index 9fa7bc896485..b5f8667fe36f 100644
--- a/external/llvm-project/flang/lib/Semantics/resolve-directives.cpp
+++ b/external/llvm-project/flang/lib/Semantics/resolve-directives.cpp
@@ -19,9 +19,11 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/openmp-dsa.h"
 #include "flang/Semantics/openmp-modifiers.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/Support/Debug.h"
 #include <list>
 #include <map>
 #include <sstream>
@@ -111,10 +113,9 @@ template <typename T> class DirectiveAttributeVisitor {
   const parser::Name *GetLoopIndex(const parser::DoConstruct &);
   const parser::DoConstruct *GetDoConstructIf(
       const parser::ExecutionPartConstruct &);
-  Symbol *DeclareNewPrivateAccessEntity(const Symbol &, Symbol::Flag, Scope &);
-  Symbol *DeclarePrivateAccessEntity(
-      const parser::Name &, Symbol::Flag, Scope &);
-  Symbol *DeclarePrivateAccessEntity(Symbol &, Symbol::Flag, Scope &);
+  Symbol *DeclareNewAccessEntity(const Symbol &, Symbol::Flag, Scope &);
+  Symbol *DeclareAccessEntity(const parser::Name &, Symbol::Flag, Scope &);
+  Symbol *DeclareAccessEntity(Symbol &, Symbol::Flag, Scope &);
   Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
 
   UnorderedSymbolSet dataSharingAttributeObjects_; // on one directive
@@ -550,6 +551,11 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return false;
   }
 
+  bool Pre(const parser::OmpClause::Uniform &x) {
+    ResolveOmpNameList(x.v, Symbol::Flag::OmpUniform);
+    return false;
+  }
+
   bool Pre(const parser::OmpInReductionClause &x) {
     auto &objects{std::get<parser::OmpObjectList>(x.t)};
     ResolveOmpObjectList(objects, Symbol::Flag::OmpInReduction);
@@ -749,10 +755,11 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
 
   Symbol::Flags ompFlagsRequireNewSymbol{Symbol::Flag::OmpPrivate,
       Symbol::Flag::OmpLinear, Symbol::Flag::OmpFirstPrivate,
-      Symbol::Flag::OmpLastPrivate, Symbol::Flag::OmpReduction,
-      Symbol::Flag::OmpCriticalLock, Symbol::Flag::OmpCopyIn,
-      Symbol::Flag::OmpUseDevicePtr, Symbol::Flag::OmpUseDeviceAddr,
-      Symbol::Flag::OmpIsDevicePtr, Symbol::Flag::OmpHasDeviceAddr};
+      Symbol::Flag::OmpLastPrivate, Symbol::Flag::OmpShared,
+      Symbol::Flag::OmpReduction, Symbol::Flag::OmpCriticalLock,
+      Symbol::Flag::OmpCopyIn, Symbol::Flag::OmpUseDevicePtr,
+      Symbol::Flag::OmpUseDeviceAddr, Symbol::Flag::OmpIsDevicePtr,
+      Symbol::Flag::OmpHasDeviceAddr, Symbol::Flag::OmpUniform};
 
   Symbol::Flags ompFlagsRequireMark{Symbol::Flag::OmpThreadprivate,
       Symbol::Flag::OmpDeclareTarget, Symbol::Flag::OmpExclusiveScan,
@@ -829,8 +836,24 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   void IssueNonConformanceWarning(
       llvm::omp::Directive D, parser::CharBlock source);
 
-  void CreateImplicitSymbols(
-      const Symbol *symbol, std::optional<Symbol::Flag> setFlag = std::nullopt);
+  void CreateImplicitSymbols(const Symbol *symbol);
+
+  void AddToContextObjectWithExplicitDSA(Symbol &symbol, Symbol::Flag flag) {
+    AddToContextObjectWithDSA(symbol, flag);
+    if (dataSharingAttributeFlags.test(flag)) {
+      symbol.set(Symbol::Flag::OmpExplicit);
+    }
+  }
+
+  // Clear any previous data-sharing attribute flags and set the new ones.
+  // Needed when setting PreDetermined DSAs, that take precedence over
+  // Implicit ones.
+  void SetSymbolDSA(Symbol &symbol, Symbol::Flags flags) {
+    symbol.flags() &= ~(dataSharingAttributeFlags |
+        Symbol::Flags{Symbol::Flag::OmpExplicit, Symbol::Flag::OmpImplicit,
+            Symbol::Flag::OmpPreDetermined});
+    symbol.flags() |= flags;
+  }
 };
 
 template <typename T>
@@ -867,7 +890,7 @@ const parser::DoConstruct *DirectiveAttributeVisitor<T>::GetDoConstructIf(
 }
 
 template <typename T>
-Symbol *DirectiveAttributeVisitor<T>::DeclareNewPrivateAccessEntity(
+Symbol *DirectiveAttributeVisitor<T>::DeclareNewAccessEntity(
     const Symbol &object, Symbol::Flag flag, Scope &scope) {
   assert(object.owner() != currScope());
   auto &symbol{MakeAssocSymbol(object.name(), object, scope)};
@@ -880,20 +903,20 @@ Symbol *DirectiveAttributeVisitor<T>::DeclareNewPrivateAccessEntity(
 }
 
 template <typename T>
-Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
+Symbol *DirectiveAttributeVisitor<T>::DeclareAccessEntity(
     const parser::Name &name, Symbol::Flag flag, Scope &scope) {
   if (!name.symbol) {
     return nullptr; // not resolved by Name Resolution step, do nothing
   }
-  name.symbol = DeclarePrivateAccessEntity(*name.symbol, flag, scope);
+  name.symbol = DeclareAccessEntity(*name.symbol, flag, scope);
   return name.symbol;
 }
 
 template <typename T>
-Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
+Symbol *DirectiveAttributeVisitor<T>::DeclareAccessEntity(
     Symbol &object, Symbol::Flag flag, Scope &scope) {
   if (object.owner() != currScope()) {
-    return DeclareNewPrivateAccessEntity(object, flag, scope);
+    return DeclareNewAccessEntity(object, flag, scope);
   } else {
     object.set(flag);
     return &object;
@@ -1600,6 +1623,20 @@ void AccAttributeVisitor::CheckMultipleAppearances(
   }
 }
 
+#ifndef NDEBUG
+
+#define DEBUG_TYPE "omp"
+
+static llvm::raw_ostream &operator<<(
+    llvm::raw_ostream &os, const Symbol::Flags &flags);
+
+namespace dbg {
+static void DumpAssocSymbols(llvm::raw_ostream &os, const Symbol &sym);
+static std::string ScopeSourcePos(const Fortran::semantics::Scope &scope);
+} // namespace dbg
+
+#endif
+
 bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
@@ -1792,12 +1829,12 @@ void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
       }
     }
   }
-  // If this symbol is already Private or Firstprivate in the enclosing
-  // OpenMP parallel or task then there is nothing to do here.
+  // If this symbol already has an explicit data-sharing attribute in the
+  // enclosing OpenMP parallel or task then there is nothing to do here.
   if (auto *symbol{targetIt->scope.FindSymbol(iv.source)}) {
     if (symbol->owner() == targetIt->scope) {
-      if (symbol->test(Symbol::Flag::OmpPrivate) ||
-          symbol->test(Symbol::Flag::OmpFirstPrivate)) {
+      if (symbol->test(Symbol::Flag::OmpExplicit) &&
+          (symbol->flags() & dataSharingAttributeFlags).any()) {
         return;
       }
     }
@@ -1806,7 +1843,8 @@ void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
   // parallel or task
   if (auto *symbol{ResolveOmp(iv, Symbol::Flag::OmpPrivate, targetIt->scope)}) {
     targetIt++;
-    symbol->set(Symbol::Flag::OmpPreDetermined);
+    SetSymbolDSA(
+        *symbol, {Symbol::Flag::OmpPreDetermined, Symbol::Flag::OmpPrivate});
     iv.symbol = symbol; // adjust the symbol within region
     for (auto it{dirContext_.rbegin()}; it != targetIt; ++it) {
       AddToContextObjectWithDSA(*symbol, Symbol::Flag::OmpPrivate, *it);
@@ -1918,7 +1956,7 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
       const parser::Name *iv{GetLoopIndex(*loop)};
       if (iv) {
         if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
-          symbol->set(Symbol::Flag::OmpPreDetermined);
+          SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
           iv->symbol = symbol; // adjust the symbol within region
           AddToContextObjectWithDSA(*symbol, ivDSA);
         }
@@ -2035,6 +2073,7 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPDispatchConstruct &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
+  IssueNonConformanceWarning(llvm::omp::Directive::OMPD_allocate, x.source);
   PushContext(x.source, llvm::omp::Directive::OMPD_allocate);
   const auto &list{std::get<std::optional<parser::OmpObjectList>>(x.t)};
   if (list) {
@@ -2178,42 +2217,62 @@ static bool IsPrivatizable(const Symbol *sym) {
               misc->kind() != MiscDetails::Kind::ConstructName));
 }
 
-void OmpAttributeVisitor::CreateImplicitSymbols(
-    const Symbol *symbol, std::optional<Symbol::Flag> setFlag) {
+static bool IsSymbolStaticStorageDuration(const Symbol &symbol) {
+  LLVM_DEBUG(llvm::dbgs() << "IsSymbolStaticStorageDuration(" << symbol.name()
+                          << "):\n");
+  auto ultSym = symbol.GetUltimate();
+  // Module-scope variable
+  return (ultSym.owner().kind() == Scope::Kind::Module) ||
+      // Data statement variable
+      (ultSym.flags().test(Symbol::Flag::InDataStmt)) ||
+      // Save attribute variable
+      (ultSym.attrs().test(Attr::SAVE)) ||
+      // Referenced in a common block
+      (ultSym.flags().test(Symbol::Flag::InCommonBlock));
+}
+
+void OmpAttributeVisitor::CreateImplicitSymbols(const Symbol *symbol) {
   if (!IsPrivatizable(symbol)) {
     return;
   }
 
+  LLVM_DEBUG(llvm::dbgs() << "CreateImplicitSymbols: " << *symbol << '\n');
+
   // Implicitly determined DSAs
   // OMP 5.2 5.1.1 - Variables Referenced in a Construct
   Symbol *lastDeclSymbol = nullptr;
-  std::optional<Symbol::Flag> prevDSA;
+  Symbol::Flags prevDSA;
   for (int dirDepth{0}; dirDepth < (int)dirContext_.size(); ++dirDepth) {
     DirContext &dirContext = dirContext_[dirDepth];
-    std::optional<Symbol::Flag> dsa;
+    Symbol::Flags dsa;
 
-    for (auto symMap : dirContext.objectWithDSA) {
-      // if the `symbol` already has a data-sharing attribute
-      if (symMap.first->name() == symbol->name()) {
-        dsa = symMap.second;
-        break;
+    Scope &scope{context_.FindScope(dirContext.directiveSource)};
+    auto it{scope.find(symbol->name())};
+    if (it != scope.end()) {
+      // There is already a symbol in the current scope, use its DSA.
+      dsa = GetSymbolDSA(*it->second);
+    } else {
+      for (auto symMap : dirContext.objectWithDSA) {
+        if (symMap.first->name() == symbol->name()) {
+          // `symbol` already has a data-sharing attribute in the current
+          // context, use it.
+          dsa.set(symMap.second);
+          break;
+        }
       }
     }
 
     // When handling each implicit rule for a given symbol, one of the
-    // following 3 actions may be taken:
-    // 1. Declare a new private symbol.
-    // 2. Create a new association symbol with no flags, that will represent
-    //    a shared symbol in the current scope. Note that symbols without
-    //    any private flags are considered as shared.
-    // 3. Use the last declared private symbol, by inserting a new symbol
-    //    in the scope being processed, associated with it.
-    //    If no private symbol was declared previously, then no association
-    //    is needed and the symbol from the enclosing scope will be
-    //    inherited by the current one.
+    // following actions may be taken:
+    // 1. Declare a new private or shared symbol.
+    // 2. Use the last declared symbol, by inserting a new symbol in the
+    //    scope being processed, associated with it.
+    //    If no symbol was declared previously, then no association is needed
+    //    and the symbol from the enclosing scope will be inherited by the
+    //    current one.
     //
     // Because of how symbols are collected in lowering, not inserting a new
-    // symbol in the last case could lead to the conclusion that a symbol
+    // symbol in the second case could lead to the conclusion that a symbol
     // from an enclosing construct was declared in the current construct,
     // which would result in wrong privatization code being generated.
     // Consider the following example:
@@ -2231,46 +2290,72 @@ void OmpAttributeVisitor::CreateImplicitSymbols(
     // it would have the private flag set.
     // This would make x appear to be defined in p2, causing it to be
     // privatized in p2 and its privatization in p1 to be skipped.
-    auto makePrivateSymbol = [&](Symbol::Flag flag) {
+    auto makeSymbol = [&](Symbol::Flags flags) {
       const Symbol *hostSymbol =
           lastDeclSymbol ? lastDeclSymbol : &symbol->GetUltimate();
-      lastDeclSymbol = DeclareNewPrivateAccessEntity(
+      assert(flags.LeastElement());
+      Symbol::Flag flag = *flags.LeastElement();
+      lastDeclSymbol = DeclareNewAccessEntity(
           *hostSymbol, flag, context_.FindScope(dirContext.directiveSource));
-      if (setFlag) {
-        lastDeclSymbol->set(*setFlag);
-      }
+      lastDeclSymbol->flags() |= flags;
       return lastDeclSymbol;
     };
-    auto makeSharedSymbol = [&](std::optional<Symbol::Flag> flag = {}) {
-      const Symbol *hostSymbol =
-          lastDeclSymbol ? lastDeclSymbol : &symbol->GetUltimate();
-      Symbol &assocSymbol = MakeAssocSymbol(symbol->name(), *hostSymbol,
-          context_.FindScope(dirContext.directiveSource));
-      if (flag) {
-        assocSymbol.set(*flag);
-      }
-    };
     auto useLastDeclSymbol = [&]() {
       if (lastDeclSymbol) {
-        makeSharedSymbol();
+        const Symbol *hostSymbol =
+            lastDeclSymbol ? lastDeclSymbol : &symbol->GetUltimate();
+        MakeAssocSymbol(symbol->name(), *hostSymbol,
+            context_.FindScope(dirContext.directiveSource));
       }
     };
 
+#ifndef NDEBUG
+    auto printImplicitRule = [&](const char *id) {
+      LLVM_DEBUG(llvm::dbgs() << "\t" << id << ": dsa: " << dsa << '\n');
+      LLVM_DEBUG(
+          llvm::dbgs() << "\t\tScope: " << dbg::ScopeSourcePos(scope) << '\n');
+    };
+#define PRINT_IMPLICIT_RULE(id) printImplicitRule(id)
+#else
+#define PRINT_IMPLICIT_RULE(id)
+#endif
+
     bool taskGenDir = llvm::omp::taskGeneratingSet.test(dirContext.directive);
     bool targetDir = llvm::omp::allTargetSet.test(dirContext.directive);
     bool parallelDir = llvm::omp::allParallelSet.test(dirContext.directive);
     bool teamsDir = llvm::omp::allTeamsSet.test(dirContext.directive);
-
-    if (dsa.has_value()) {
-      if (dsa.value() == Symbol::Flag::OmpShared &&
-          (parallelDir || taskGenDir || teamsDir)) {
-        makeSharedSymbol(Symbol::Flag::OmpShared);
+    bool isStaticStorageDuration = IsSymbolStaticStorageDuration(*symbol);
+
+    if (dsa.any()) {
+      if (parallelDir || taskGenDir || teamsDir) {
+        Symbol *prevDeclSymbol{lastDeclSymbol};
+        // NOTE As `dsa` will match that of the symbol in the current scope
+        //      (if any), we won't override the DSA of any existing symbol.
+        if ((dsa & dataSharingAttributeFlags).any()) {
+          makeSymbol(dsa);
+        }
+        // Fix host association of explicit symbols, as they can be created
+        // before implicit ones in enclosing scope.
+        if (prevDeclSymbol && prevDeclSymbol != lastDeclSymbol &&
+            lastDeclSymbol->test(Symbol::Flag::OmpExplicit)) {
+          const auto *hostAssoc{lastDeclSymbol->detailsIf<HostAssocDetails>()};
+          if (hostAssoc && hostAssoc->symbol() != *prevDeclSymbol) {
+            lastDeclSymbol->set_details(HostAssocDetails{*prevDeclSymbol});
+          }
+        }
       }
-      // Private symbols will have been declared already.
       prevDSA = dsa;
+      PRINT_IMPLICIT_RULE("0) already has DSA");
       continue;
     }
 
+    // NOTE Because of how lowering uses OmpImplicit flag, we can only set it
+    //      for symbols with private DSA.
+    //      Also, as the default clause is handled separately in lowering,
+    //      don't mark its symbols with OmpImplicit either.
+    //      Ideally, lowering should be changed and all implicit symbols
+    //      should be marked with OmpImplicit.
+
     if (dirContext.defaultDSA == Symbol::Flag::OmpPrivate ||
         dirContext.defaultDSA == Symbol::Flag::OmpFirstPrivate ||
         dirContext.defaultDSA == Symbol::Flag::OmpShared) {
@@ -2279,33 +2364,36 @@ void OmpAttributeVisitor::CreateImplicitSymbols(
       if (!parallelDir && !taskGenDir && !teamsDir) {
         return;
       }
-      if (dirContext.defaultDSA != Symbol::Flag::OmpShared) {
-        makePrivateSymbol(dirContext.defaultDSA);
-      } else {
-        makeSharedSymbol();
-      }
-      dsa = dirContext.defaultDSA;
+      dsa = {dirContext.defaultDSA};
+      makeSymbol(dsa);
+      PRINT_IMPLICIT_RULE("1) default");
     } else if (parallelDir) {
       // 2) parallel -> shared
-      makeSharedSymbol();
-      dsa = Symbol::Flag::OmpShared;
+      dsa = {Symbol::Flag::OmpShared};
+      makeSymbol(dsa);
+      PRINT_IMPLICIT_RULE("2) parallel");
     } else if (!taskGenDir && !targetDir) {
       // 3) enclosing context
-      useLastDeclSymbol();
       dsa = prevDSA;
+      useLastDeclSymbol();
+      PRINT_IMPLICIT_RULE("3) enclosing context");
     } else if (targetDir) {
       // TODO 4) not mapped target variable -> firstprivate
       dsa = prevDSA;
     } else if (taskGenDir) {
       // TODO 5) dummy arg in orphaned taskgen construct -> firstprivate
-      if (prevDSA == Symbol::Flag::OmpShared) {
+      if (prevDSA.test(Symbol::Flag::OmpShared) ||
+          (isStaticStorageDuration &&
+              (prevDSA & dataSharingAttributeFlags).none())) {
         // 6) shared in enclosing context -> shared
-        makeSharedSymbol();
-        dsa = Symbol::Flag::OmpShared;
+        dsa = {Symbol::Flag::OmpShared};
+        makeSymbol(dsa);
+        PRINT_IMPLICIT_RULE("6) taskgen: shared");
       } else {
         // 7) firstprivate
-        dsa = Symbol::Flag::OmpFirstPrivate;
-        makePrivateSymbol(*dsa)->set(Symbol::Flag::OmpImplicit);
+        dsa = {Symbol::Flag::OmpFirstPrivate};
+        makeSymbol(dsa)->set(Symbol::Flag::OmpImplicit);
+        PRINT_IMPLICIT_RULE("7) taskgen: firstprivate");
       }
     }
     prevDSA = dsa;
@@ -2371,7 +2459,7 @@ void OmpAttributeVisitor::ResolveOmpName(
   if (ResolveName(&name)) {
     if (auto *resolvedSymbol{ResolveOmp(name, ompFlag, currScope())}) {
       if (dataSharingAttributeFlags.test(ompFlag)) {
-        AddToContextObjectWithDSA(*resolvedSymbol, ompFlag);
+        AddToContextObjectWithExplicitDSA(*resolvedSymbol, ompFlag);
       }
     }
   } else if (ompFlag == Symbol::Flag::OmpCriticalLock) {
@@ -2484,7 +2572,7 @@ void OmpAttributeVisitor::ResolveOmpObject(
                 if (dataCopyingAttributeFlags.test(ompFlag)) {
                   CheckDataCopyingClause(*name, *symbol, ompFlag);
                 } else {
-                  AddToContextObjectWithDSA(*symbol, ompFlag);
+                  AddToContextObjectWithExplicitDSA(*symbol, ompFlag);
                   if (dataSharingAttributeFlags.test(ompFlag)) {
                     CheckMultipleAppearances(*name, *symbol, ompFlag);
                   }
@@ -2588,8 +2676,14 @@ void OmpAttributeVisitor::ResolveOmpObject(
                               GetContext().directive))) {
                     for (Symbol::Flag ompFlag1 : dataMappingAttributeFlags) {
                       for (Symbol::Flag ompFlag2 : dataSharingAttributeFlags) {
-                        checkExclusivelists(
-                            hostAssocSym, ompFlag1, symbol, ompFlag2);
+                        if ((hostAssocSym->test(ompFlag2) &&
+                                hostAssocSym->test(
+                                    Symbol::Flag::OmpExplicit)) ||
+                            (symbol->test(ompFlag2) &&
+                                symbol->test(Symbol::Flag::OmpExplicit))) {
+                          checkExclusivelists(
+                              hostAssocSym, ompFlag1, symbol, ompFlag2);
+                        }
                       }
                     }
                   }
@@ -2624,7 +2718,7 @@ void OmpAttributeVisitor::ResolveOmpObject(
                   if (dataCopyingAttributeFlags.test(ompFlag)) {
                     CheckDataCopyingClause(name, *resolvedObject, ompFlag);
                   } else {
-                    AddToContextObjectWithDSA(*resolvedObject, ompFlag);
+                    AddToContextObjectWithExplicitDSA(*resolvedObject, ompFlag);
                   }
                   details.replace_object(*resolvedObject, index);
                 }
@@ -2643,7 +2737,7 @@ void OmpAttributeVisitor::ResolveOmpObject(
 Symbol *OmpAttributeVisitor::ResolveOmp(
     const parser::Name &name, Symbol::Flag ompFlag, Scope &scope) {
   if (ompFlagsRequireNewSymbol.test(ompFlag)) {
-    return DeclarePrivateAccessEntity(name, ompFlag, scope);
+    return DeclareAccessEntity(name, ompFlag, scope);
   } else {
     return DeclareOrMarkOtherAccessEntity(name, ompFlag);
   }
@@ -2652,7 +2746,7 @@ Symbol *OmpAttributeVisitor::ResolveOmp(
 Symbol *OmpAttributeVisitor::ResolveOmp(
     Symbol &symbol, Symbol::Flag ompFlag, Scope &scope) {
   if (ompFlagsRequireNewSymbol.test(ompFlag)) {
-    return DeclarePrivateAccessEntity(symbol, ompFlag, scope);
+    return DeclareAccessEntity(symbol, ompFlag, scope);
   } else {
     return DeclareOrMarkOtherAccessEntity(symbol, ompFlag);
   }
@@ -2809,20 +2903,6 @@ void ResolveOmpTopLevelParts(
   });
 }
 
-static bool IsSymbolInCommonBlock(const Symbol &symbol) {
-  // TODO Improve the performance of this predicate function.
-  //      Going through all symbols sequentially, in all common blocks, can be
-  //      slow when there are many symbols. A possible optimization is to add
-  //      an OmpInCommonBlock flag to Symbol, to make it possible to quickly
-  //      test if a given symbol is in a common block.
-  for (const auto &cb : symbol.owner().commonBlocks()) {
-    if (IsCommonBlockContaining(cb.second.get(), symbol)) {
-      return true;
-    }
-  }
-  return false;
-}
-
 static bool IsSymbolThreadprivate(const Symbol &symbol) {
   if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
     return details->symbol().test(Symbol::Flag::OmpThreadprivate);
@@ -2831,10 +2911,16 @@ static bool IsSymbolThreadprivate(const Symbol &symbol) {
 }
 
 static bool IsSymbolPrivate(const Symbol &symbol) {
-  if (symbol.test(Symbol::Flag::OmpPrivate) ||
-      symbol.test(Symbol::Flag::OmpFirstPrivate)) {
+  LLVM_DEBUG(llvm::dbgs() << "IsSymbolPrivate(" << symbol.name() << "):\n");
+  LLVM_DEBUG(dbg::DumpAssocSymbols(llvm::dbgs(), symbol));
+
+  if (Symbol::Flags dsa{GetSymbolDSA(symbol)}; dsa.any()) {
+    if (dsa.test(Symbol::Flag::OmpShared)) {
+      return false;
+    }
     return true;
   }
+
   // A symbol that has not gone through constructs that may privatize the
   // original symbol may be predetermined as private.
   // (OMP 5.2 5.1.1 - Variables Referenced in a Construct)
@@ -2845,7 +2931,7 @@ static bool IsSymbolPrivate(const Symbol &symbol) {
     case Scope::Kind::BlockConstruct:
       return !symbol.attrs().test(Attr::SAVE) &&
           !symbol.attrs().test(Attr::PARAMETER) && !IsAssumedShape(symbol) &&
-          !IsSymbolInCommonBlock(symbol);
+          !symbol.flags().test(Symbol::Flag::InCommonBlock);
     default:
       return false;
     }
@@ -2937,10 +3023,14 @@ void OmpAttributeVisitor::CheckSourceLabel(const parser::Label &label) {
 void OmpAttributeVisitor::CheckLabelContext(const parser::CharBlock source,
     const parser::CharBlock target, std::optional<DirContext> sourceContext,
     std::optional<DirContext> targetContext) {
+  auto dirContextsSame = [](DirContext &lhs, DirContext &rhs) -> bool {
+    // Sometimes nested constructs share a scope but are different contexts
+    return (lhs.scope == rhs.scope) && (lhs.directive == rhs.directive);
+  };
   unsigned version{context_.langOptions().OpenMPVersion};
   if (targetContext &&
       (!sourceContext ||
-          (sourceContext->scope != targetContext->scope &&
+          (!dirContextsSame(*targetContext, *sourceContext) &&
               !DoesScopeContain(
                   &targetContext->scope, sourceContext->scope)))) {
     context_
@@ -2952,7 +3042,7 @@ void OmpAttributeVisitor::CheckLabelContext(const parser::CharBlock source,
   }
   if (sourceContext &&
       (!targetContext ||
-          (sourceContext->scope != targetContext->scope &&
+          (!dirContextsSame(*sourceContext, *targetContext) &&
               !DoesScopeContain(
                   &sourceContext->scope, targetContext->scope)))) {
     context_
@@ -3074,10 +3164,69 @@ void OmpAttributeVisitor::IssueNonConformanceWarning(
   case llvm::omp::OMPD_parallel_master_taskloop_simd:
     setAlternativeStr("PARALLEL_MASKED TASKLOOP SIMD");
     break;
+  case llvm::omp::OMPD_allocate:
+    setAlternativeStr("ALLOCATORS");
+    break;
   case llvm::omp::OMPD_target_loop:
   default:;
   }
   context_.Warn(common::UsageWarning::OpenMPUsage, source, "%s"_warn_en_US,
       warnStrOS.str());
 }
+
+#ifndef NDEBUG
+
+static llvm::raw_ostream &operator<<(
+    llvm::raw_ostream &os, const Symbol::Flags &flags) {
+  flags.Dump(os, Symbol::EnumToString);
+  return os;
+}
+
+namespace dbg {
+
+static llvm::raw_ostream &operator<<(
+    llvm::raw_ostream &os, std::optional<parser::SourcePosition> srcPos) {
+  if (srcPos) {
+    os << *srcPos.value().path << ":" << srcPos.value().line << ": ";
+  }
+  return os;
+}
+
+static std::optional<parser::SourcePosition> GetSourcePosition(
+    const Fortran::semantics::Scope &scope,
+    const Fortran::parser::CharBlock &src) {
+  parser::AllCookedSources &allCookedSources{
+      scope.context().allCookedSources()};
+  if (std::optional<parser::ProvenanceRange> prange{
+          allCookedSources.GetProvenanceRange(src)}) {
+    return allCookedSources.allSources().GetSourcePosition(prange->start());
+  }
+  return std::nullopt;
+}
+
+// Returns a string containing the source location of `scope` followed by
+// its first source line.
+static std::string ScopeSourcePos(const Fortran::semantics::Scope &scope) {
+  const parser::CharBlock &sourceRange{scope.sourceRange()};
+  std::string src{sourceRange.ToString()};
+  size_t nl{src.find('\n')};
+  std::string str;
+  llvm::raw_string_ostream ss{str};
+
+  ss << GetSourcePosition(scope, sourceRange) << src.substr(0, nl);
+  return str;
+}
+
+static void DumpAssocSymbols(llvm::raw_ostream &os, const Symbol &sym) {
+  os << '\t' << sym << '\n';
+  os << "\t\tOwner: " << ScopeSourcePos(sym.owner()) << '\n';
+  if (const auto *details{sym.detailsIf<HostAssocDetails>()}) {
+    DumpAssocSymbols(os, details->symbol());
+  }
+}
+
+} // namespace dbg
+
+#endif
+
 } // namespace Fortran::semantics
diff --git a/external/llvm-project/flang/lib/Semantics/resolve-names-utils.h b/external/llvm-project/flang/lib/Semantics/resolve-names-utils.h
index 64784722ff4f..ee8113a3fda5 100644
--- a/external/llvm-project/flang/lib/Semantics/resolve-names-utils.h
+++ b/external/llvm-project/flang/lib/Semantics/resolve-names-utils.h
@@ -146,5 +146,11 @@ struct SymbolAndTypeMappings;
 void MapSubprogramToNewSymbols(const Symbol &oldSymbol, Symbol &newSymbol,
     Scope &newScope, SymbolAndTypeMappings * = nullptr);
 
+parser::CharBlock MakeNameFromOperator(
+    const parser::DefinedOperator::IntrinsicOperator &op,
+    SemanticsContext &context);
+parser::CharBlock MangleSpecialFunctions(const parser::CharBlock &name);
+std::string MangleDefinedOperator(const parser::CharBlock &name);
+
 } // namespace Fortran::semantics
 #endif // FORTRAN_SEMANTICS_RESOLVE_NAMES_H_
diff --git a/external/llvm-project/flang/lib/Semantics/resolve-names.cpp b/external/llvm-project/flang/lib/Semantics/resolve-names.cpp
index 57035c57ee16..e23e91b674a7 100644
--- a/external/llvm-project/flang/lib/Semantics/resolve-names.cpp
+++ b/external/llvm-project/flang/lib/Semantics/resolve-names.cpp
@@ -38,6 +38,7 @@
 #include "flang/Semantics/type.h"
 #include "flang/Support/Fortran.h"
 #include "flang/Support/default-kinds.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 #include <list>
 #include <map>
@@ -1467,11 +1468,15 @@ class OmpVisitor : public virtual DeclarationVisitor {
   static bool NeedsScope(const parser::OpenMPBlockConstruct &);
   static bool NeedsScope(const parser::OmpClause &);
 
-  bool Pre(const parser::OmpMetadirectiveDirective &) {
+  bool Pre(const parser::OmpMetadirectiveDirective &x) { //
+    metaDirective_ = &x;
     ++metaLevel_;
     return true;
   }
-  void Post(const parser::OmpMetadirectiveDirective &) { --metaLevel_; }
+  void Post(const parser::OmpMetadirectiveDirective &) { //
+    metaDirective_ = nullptr;
+    --metaLevel_;
+  }
 
   bool Pre(const parser::OpenMPRequiresConstruct &x) {
     AddOmpSourceRange(x.source);
@@ -1511,13 +1516,18 @@ class OmpVisitor : public virtual DeclarationVisitor {
     return false;
   }
 
+  bool Pre(const parser::OpenMPDeclareSimdConstruct &x) {
+    AddOmpSourceRange(x.source);
+    return true;
+  }
+
   bool Pre(const parser::OmpInitializerProc &x) {
     auto &procDes = std::get<parser::ProcedureDesignator>(x.t);
     auto &name = std::get<parser::Name>(procDes.u);
     auto *symbol{FindSymbol(NonDerivedTypeScope(), name)};
     if (!symbol) {
       context().Say(name.source,
-          "Implicit subroutine declaration '%s' in !$OMP DECLARE REDUCTION"_err_en_US,
+          "Implicit subroutine declaration '%s' in DECLARE REDUCTION"_err_en_US,
           name.source);
     }
     return true;
@@ -1546,7 +1556,7 @@ class OmpVisitor : public virtual DeclarationVisitor {
     AddOmpSourceRange(x.source);
     ProcessReductionSpecifier(
         std::get<Indirection<parser::OmpReductionSpecifier>>(x.t).value(),
-        std::get<std::optional<parser::OmpClauseList>>(x.t));
+        std::get<std::optional<parser::OmpClauseList>>(x.t), x);
     return false;
   }
   bool Pre(const parser::OmpMapClause &);
@@ -1656,9 +1666,14 @@ class OmpVisitor : public virtual DeclarationVisitor {
   }
   bool Pre(const parser::OpenMPDeclarativeConstruct &x) {
     AddOmpSourceRange(x.source);
+    // Without skipping implicit typing, declarative constructs
+    // can implicitly declare variables instead of only using the
+    // ones already declared in the Fortran sources.
+    SkipImplicitTyping(true);
     return true;
   }
   void Post(const parser::OpenMPDeclarativeConstruct &) {
+    SkipImplicitTyping(false);
     messageHandler().set_currStmtSource(std::nullopt);
   }
   bool Pre(const parser::OpenMPDepobjConstruct &x) {
@@ -1669,11 +1684,8 @@ class OmpVisitor : public virtual DeclarationVisitor {
     messageHandler().set_currStmtSource(std::nullopt);
   }
   bool Pre(const parser::OpenMPAtomicConstruct &x) {
-    return common::visit(common::visitors{[&](const auto &u) -> bool {
-      AddOmpSourceRange(u.source);
-      return true;
-    }},
-        x.u);
+    AddOmpSourceRange(x.source);
+    return true;
   }
   void Post(const parser::OpenMPAtomicConstruct &) {
     messageHandler().set_currStmtSource(std::nullopt);
@@ -1702,9 +1714,13 @@ class OmpVisitor : public virtual DeclarationVisitor {
 private:
   void ProcessMapperSpecifier(const parser::OmpMapperSpecifier &spec,
       const parser::OmpClauseList &clauses);
+  template <typename T>
   void ProcessReductionSpecifier(const parser::OmpReductionSpecifier &spec,
-      const std::optional<parser::OmpClauseList> &clauses);
+      const std::optional<parser::OmpClauseList> &clauses,
+      const T &wholeConstruct);
+
   int metaLevel_{0};
+  const parser::OmpMetadirectiveDirective *metaDirective_{nullptr};
 };
 
 bool OmpVisitor::NeedsScope(const parser::OpenMPBlockConstruct &x) {
@@ -1791,14 +1807,92 @@ void OmpVisitor::ProcessMapperSpecifier(const parser::OmpMapperSpecifier &spec,
   PopScope();
 }
 
+parser::CharBlock MakeNameFromOperator(
+    const parser::DefinedOperator::IntrinsicOperator &op,
+    SemanticsContext &context) {
+  switch (op) {
+  case parser::DefinedOperator::IntrinsicOperator::Multiply:
+    return parser::CharBlock{"op.*", 4};
+  case parser::DefinedOperator::IntrinsicOperator::Add:
+    return parser::CharBlock{"op.+", 4};
+  case parser::DefinedOperator::IntrinsicOperator::Subtract:
+    return parser::CharBlock{"op.-", 4};
+
+  case parser::DefinedOperator::IntrinsicOperator::AND:
+    return parser::CharBlock{"op.AND", 6};
+  case parser::DefinedOperator::IntrinsicOperator::OR:
+    return parser::CharBlock{"op.OR", 6};
+  case parser::DefinedOperator::IntrinsicOperator::EQV:
+    return parser::CharBlock{"op.EQV", 7};
+  case parser::DefinedOperator::IntrinsicOperator::NEQV:
+    return parser::CharBlock{"op.NEQV", 8};
+
+  default:
+    context.Say("Unsupported operator in DECLARE REDUCTION"_err_en_US);
+    return parser::CharBlock{"op.?", 4};
+  }
+}
+
+parser::CharBlock MangleSpecialFunctions(const parser::CharBlock &name) {
+  return llvm::StringSwitch<parser::CharBlock>(name.ToString())
+      .Case("max", {"op.max", 6})
+      .Case("min", {"op.min", 6})
+      .Case("iand", {"op.iand", 7})
+      .Case("ior", {"op.ior", 6})
+      .Case("ieor", {"op.ieor", 7})
+      .Default(name);
+}
+
+std::string MangleDefinedOperator(const parser::CharBlock &name) {
+  CHECK(name[0] == '.' && name[name.size() - 1] == '.');
+  return "op" + name.ToString();
+}
+
+template <typename T>
 void OmpVisitor::ProcessReductionSpecifier(
     const parser::OmpReductionSpecifier &spec,
-    const std::optional<parser::OmpClauseList> &clauses) {
+    const std::optional<parser::OmpClauseList> &clauses,
+    const T &wholeOmpConstruct) {
+  const parser::Name *name{nullptr};
+  parser::CharBlock mangledName;
+  UserReductionDetails reductionDetailsTemp;
   const auto &id{std::get<parser::OmpReductionIdentifier>(spec.t)};
-  if (auto procDes{std::get_if<parser::ProcedureDesignator>(&id.u)}) {
-    if (auto *name{std::get_if<parser::Name>(&procDes->u)}) {
-      name->symbol =
-          &MakeSymbol(*name, MiscDetails{MiscDetails::Kind::ConstructName});
+  if (auto *procDes{std::get_if<parser::ProcedureDesignator>(&id.u)}) {
+    name = std::get_if<parser::Name>(&procDes->u);
+    // This shouldn't be a procedure component: this is the name of the
+    // reduction being declared.
+    CHECK(name);
+    // Prevent the symbol from conflicting with the builtin function name
+    mangledName = MangleSpecialFunctions(name->source);
+    // Note: the Name inside the parse tree is not updated because it is const.
+    // All lookups must use MangleSpecialFunctions.
+  } else {
+    const auto &defOp{std::get<parser::DefinedOperator>(id.u)};
+    if (const auto *definedOp{std::get_if<parser::DefinedOpName>(&defOp.u)}) {
+      name = &definedOp->v;
+      mangledName = context().SaveTempName(MangleDefinedOperator(name->source));
+    } else {
+      mangledName = MakeNameFromOperator(
+          std::get<parser::DefinedOperator::IntrinsicOperator>(defOp.u),
+          context());
+    }
+  }
+
+  // Use reductionDetailsTemp if we can't find the symbol (this is
+  // the first, or only, instance with this name). The details then
+  // gets stored in the symbol when it's created.
+  UserReductionDetails *reductionDetails{&reductionDetailsTemp};
+  Symbol *symbol{currScope().FindSymbol(mangledName)};
+  if (symbol) {
+    // If we found a symbol, we append the type info to the
+    // existing reductionDetails.
+    reductionDetails = symbol->detailsIf<UserReductionDetails>();
+
+    if (!reductionDetails) {
+      context().Say(
+          "Duplicate definition of '%s' in DECLARE REDUCTION"_err_en_US,
+          mangledName);
+      return;
     }
   }
 
@@ -1828,19 +1922,31 @@ void OmpVisitor::ProcessReductionSpecifier(
     // We need to walk t.u because Walk(t) does it's own BeginDeclTypeSpec.
     Walk(t.u);
 
-    const DeclTypeSpec *typeSpec{GetDeclTypeSpec()};
-    assert(typeSpec && "We should have a type here");
+    // Only process types we can find. There will be an error later on when
+    // a type isn't found.
+    if (const DeclTypeSpec *typeSpec{GetDeclTypeSpec()}) {
+      reductionDetails->AddType(*typeSpec);
 
-    for (auto &nm : ompVarNames) {
-      ObjectEntityDetails details{};
-      details.set_type(*typeSpec);
-      MakeSymbol(nm, Attrs{}, std::move(details));
+      for (auto &nm : ompVarNames) {
+        ObjectEntityDetails details{};
+        details.set_type(*typeSpec);
+        MakeSymbol(nm, Attrs{}, std::move(details));
+      }
     }
     EndDeclTypeSpec();
     Walk(std::get<std::optional<parser::OmpReductionCombiner>>(spec.t));
     Walk(clauses);
     PopScope();
   }
+
+  reductionDetails->AddDecl(&wholeOmpConstruct);
+
+  if (!symbol) {
+    symbol = &MakeSymbol(mangledName, Attrs{}, std::move(*reductionDetails));
+  }
+  if (name) {
+    name->symbol = symbol;
+  }
 }
 
 bool OmpVisitor::Pre(const parser::OmpDirectiveSpecification &x) {
@@ -1870,7 +1976,8 @@ bool OmpVisitor::Pre(const parser::OmpDirectiveSpecification &x) {
     if (maybeArgs && maybeClauses) {
       const parser::OmpArgument &first{maybeArgs->v.front()};
       if (auto *spec{std::get_if<parser::OmpReductionSpecifier>(&first.u)}) {
-        ProcessReductionSpecifier(*spec, maybeClauses);
+        CHECK(metaDirective_);
+        ProcessReductionSpecifier(*spec, maybeClauses, *metaDirective_);
       }
     }
     break;
@@ -2721,6 +2828,16 @@ Scope &ScopeHandler::NonDerivedTypeScope() {
   return currScope_->IsDerivedType() ? currScope_->parent() : *currScope_;
 }
 
+static void SetImplicitCUDADevice(Symbol &symbol) {
+  if (auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
+    if (!object->cudaDataAttr() && !IsValue(symbol) &&
+        !IsFunctionResult(symbol)) {
+      // Implicitly set device attribute if none is set in device context.
+      object->set_cudaDataAttr(common::CUDADataAttr::Device);
+    }
+  }
+}
+
 void ScopeHandler::PushScope(Scope::Kind kind, Symbol *symbol) {
   PushScope(currScope().MakeScope(kind, symbol));
 }
@@ -2760,9 +2877,35 @@ void ScopeHandler::PopScope() {
   // Entities that are not yet classified as objects or procedures are now
   // assumed to be objects.
   // TODO: Statement functions
+  bool inDeviceSubprogram{false};
+  const Symbol *scopeSym{currScope().GetSymbol()};
+  if (currScope().kind() == Scope::Kind::BlockConstruct) {
+    scopeSym = GetProgramUnitContaining(currScope()).GetSymbol();
+  }
+  if (scopeSym) {
+    if (auto *details{scopeSym->detailsIf<SubprogramDetails>()}) {
+      // Check the current procedure is a device procedure to apply implicit
+      // attribute at the end.
+      if (auto attrs{details->cudaSubprogramAttrs()}) {
+        if (*attrs == common::CUDASubprogramAttrs::Device ||
+            *attrs == common::CUDASubprogramAttrs::Global ||
+            *attrs == common::CUDASubprogramAttrs::Grid_Global) {
+          inDeviceSubprogram = true;
+        }
+      }
+    }
+  }
   for (auto &pair : currScope()) {
     ConvertToObjectEntity(*pair.second);
   }
+
+  // Apply CUDA device attributes if in a device subprogram
+  if (inDeviceSubprogram && currScope().kind() == Scope::Kind::BlockConstruct) {
+    for (auto &pair : currScope()) {
+      SetImplicitCUDADevice(*pair.second);
+    }
+  }
+
   funcResultStack_.Pop();
   // If popping back into a global scope, pop back to the top scope.
   Scope *hermetic{context().currentHermeticModuleFileScope()};
@@ -6810,6 +6953,9 @@ bool DeclarationVisitor::Pre(const parser::CommonBlockObject &) {
 
 void DeclarationVisitor::Post(const parser::CommonBlockObject &x) {
   const auto &name{std::get<parser::Name>(x.t)};
+  if (auto *symbol{FindSymbol(name)}) {
+    symbol->set(Symbol::Flag::InCommonBlock);
+  }
   DeclareObjectEntity(name);
   auto pair{specPartState_.commonBlockObjects.insert(name.source)};
   if (!pair.second) {
@@ -9445,40 +9591,11 @@ void ResolveNamesVisitor::CreateGeneric(const parser::GenericSpec &x) {
   info.Resolve(&MakeSymbol(symbolName, Attrs{}, std::move(genericDetails)));
 }
 
-static void SetImplicitCUDADevice(bool inDeviceSubprogram, Symbol &symbol) {
-  if (inDeviceSubprogram && symbol.has<ObjectEntityDetails>()) {
-    auto *object{symbol.detailsIf<ObjectEntityDetails>()};
-    if (!object->cudaDataAttr() && !IsValue(symbol) &&
-        !IsFunctionResult(symbol)) {
-      // Implicitly set device attribute if none is set in device context.
-      object->set_cudaDataAttr(common::CUDADataAttr::Device);
-    }
-  }
-}
-
 void ResolveNamesVisitor::FinishSpecificationPart(
     const std::list<parser::DeclarationConstruct> &decls) {
   misparsedStmtFuncFound_ = false;
   funcResultStack().CompleteFunctionResultType();
   CheckImports();
-  bool inDeviceSubprogram{false};
-  Symbol *scopeSym{currScope().symbol()};
-  if (currScope().kind() == Scope::Kind::BlockConstruct) {
-    scopeSym = currScope().parent().symbol();
-  }
-  if (scopeSym) {
-    if (auto *details{scopeSym->detailsIf<SubprogramDetails>()}) {
-      // Check the current procedure is a device procedure to apply implicit
-      // attribute at the end.
-      if (auto attrs{details->cudaSubprogramAttrs()}) {
-        if (*attrs == common::CUDASubprogramAttrs::Device ||
-            *attrs == common::CUDASubprogramAttrs::Global ||
-            *attrs == common::CUDASubprogramAttrs::Grid_Global) {
-          inDeviceSubprogram = true;
-        }
-      }
-    }
-  }
   for (auto &pair : currScope()) {
     auto &symbol{*pair.second};
     if (inInterfaceBlock()) {
@@ -9513,11 +9630,6 @@ void ResolveNamesVisitor::FinishSpecificationPart(
         SetBindNameOn(symbol);
       }
     }
-    if (currScope().kind() == Scope::Kind::BlockConstruct) {
-      // Only look for specification in BlockConstruct. Other cases are done in
-      // ResolveSpecificationParts.
-      SetImplicitCUDADevice(inDeviceSubprogram, symbol);
-    }
   }
   currScope().InstantiateDerivedTypes();
   for (const auto &decl : decls) {
@@ -10077,7 +10189,9 @@ void ResolveNamesVisitor::ResolveSpecificationParts(ProgramTree &node) {
     }
     ApplyImplicitRules(symbol);
     // Apply CUDA implicit attributes if needed.
-    SetImplicitCUDADevice(inDeviceSubprogram, symbol);
+    if (inDeviceSubprogram) {
+      SetImplicitCUDADevice(symbol);
+    }
     // Main program local objects usually don't have an implied SAVE attribute,
     // as one might think, but in the exceptional case of a derived type
     // local object that contains a coarray, we have to mark it as an
diff --git a/external/llvm-project/flang/lib/Semantics/rewrite-directives.cpp b/external/llvm-project/flang/lib/Semantics/rewrite-directives.cpp
index 104a77885d27..91b60ea151de 100644
--- a/external/llvm-project/flang/lib/Semantics/rewrite-directives.cpp
+++ b/external/llvm-project/flang/lib/Semantics/rewrite-directives.cpp
@@ -51,23 +51,21 @@ class OmpRewriteMutator : public DirectiveRewriteMutator {
 
 bool OmpRewriteMutator::Pre(parser::OpenMPAtomicConstruct &x) {
   // Find top-level parent of the operation.
-  Symbol *topLevelParent{common::visit(
-      [&](auto &atomic) {
-        Symbol *symbol{nullptr};
-        Scope *scope{
-            &context_.FindScope(std::get<parser::Verbatim>(atomic.t).source)};
-        do {
-          if (Symbol * parent{scope->symbol()}) {
-            symbol = parent;
-          }
-          scope = &scope->parent();
-        } while (!scope->IsGlobal());
-
-        assert(symbol &&
-            "Atomic construct must be within a scope associated with a symbol");
-        return symbol;
-      },
-      x.u)};
+  Symbol *topLevelParent{[&]() {
+    Symbol *symbol{nullptr};
+    Scope *scope{&context_.FindScope(
+        std::get<parser::OmpDirectiveSpecification>(x.t).source)};
+    do {
+      if (Symbol * parent{scope->symbol()}) {
+        symbol = parent;
+      }
+      scope = &scope->parent();
+    } while (!scope->IsGlobal());
+
+    assert(symbol &&
+        "Atomic construct must be within a scope associated with a symbol");
+    return symbol;
+  }()};
 
   // Get the `atomic_default_mem_order` clause from the top-level parent.
   std::optional<common::OmpMemoryOrderType> defaultMemOrder;
@@ -86,66 +84,61 @@ bool OmpRewriteMutator::Pre(parser::OpenMPAtomicConstruct &x) {
     return false;
   }
 
-  auto findMemOrderClause =
-      [](const std::list<parser::OmpAtomicClause> &clauses) {
-        return llvm::any_of(clauses, [](const auto &clause) {
-          return std::get_if<parser::OmpMemoryOrderClause>(&clause.u);
+  auto findMemOrderClause{[](const parser::OmpClauseList &clauses) {
+    return llvm::any_of(
+        clauses.v, [](auto &clause) -> const parser::OmpClause * {
+          switch (clause.Id()) {
+          case llvm::omp::Clause::OMPC_acq_rel:
+          case llvm::omp::Clause::OMPC_acquire:
+          case llvm::omp::Clause::OMPC_relaxed:
+          case llvm::omp::Clause::OMPC_release:
+          case llvm::omp::Clause::OMPC_seq_cst:
+            return &clause;
+          default:
+            return nullptr;
+          }
         });
-      };
-
-  // Get the clause list to which the new memory order clause must be added,
-  // only if there are no other memory order clauses present for this atomic
-  // directive.
-  std::list<parser::OmpAtomicClause> *clauseList = common::visit(
-      common::visitors{[&](parser::OmpAtomic &atomicConstruct) {
-                         // OmpAtomic only has a single list of clauses.
-                         auto &clauses{std::get<parser::OmpAtomicClauseList>(
-                             atomicConstruct.t)};
-                         return !findMemOrderClause(clauses.v) ? &clauses.v
-                                                               : nullptr;
-                       },
-          [&](auto &atomicConstruct) {
-            // All other atomic constructs have two lists of clauses.
-            auto &clausesLhs{std::get<0>(atomicConstruct.t)};
-            auto &clausesRhs{std::get<2>(atomicConstruct.t)};
-            return !findMemOrderClause(clausesLhs.v) &&
-                    !findMemOrderClause(clausesRhs.v)
-                ? &clausesRhs.v
-                : nullptr;
-          }},
-      x.u);
+  }};
 
-  // Add a memory order clause to the atomic directive.
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &clauseList{std::get<std::optional<parser::OmpClauseList>>(dirSpec.t)};
   if (clauseList) {
-    atomicDirectiveDefaultOrderFound_ = true;
-    switch (*defaultMemOrder) {
-    case common::OmpMemoryOrderType::Acq_Rel:
-      clauseList->emplace_back<parser::OmpMemoryOrderClause>(common::visit(
-          common::visitors{[](parser::OmpAtomicRead &) -> parser::OmpClause {
-                             return parser::OmpClause::Acquire{};
-                           },
-              [](parser::OmpAtomicCapture &) -> parser::OmpClause {
-                return parser::OmpClause::AcqRel{};
-              },
-              [](auto &) -> parser::OmpClause {
-                // parser::{OmpAtomic, OmpAtomicUpdate, OmpAtomicWrite}
-                return parser::OmpClause::Release{};
-              }},
-          x.u));
-      break;
-    case common::OmpMemoryOrderType::Relaxed:
-      clauseList->emplace_back<parser::OmpMemoryOrderClause>(
-          parser::OmpClause{parser::OmpClause::Relaxed{}});
-      break;
-    case common::OmpMemoryOrderType::Seq_Cst:
-      clauseList->emplace_back<parser::OmpMemoryOrderClause>(
-          parser::OmpClause{parser::OmpClause::SeqCst{}});
-      break;
-    default:
-      // FIXME: Don't process other values at the moment since their validity
-      // depends on the OpenMP version (which is unavailable here).
-      break;
+    if (findMemOrderClause(*clauseList)) {
+      return false;
+    }
+  } else {
+    clauseList = parser::OmpClauseList(decltype(parser::OmpClauseList::v){});
+  }
+
+  // Add a memory order clause to the atomic directive.
+  atomicDirectiveDefaultOrderFound_ = true;
+  llvm::omp::Clause kind{x.GetKind()};
+  switch (*defaultMemOrder) {
+  case common::OmpMemoryOrderType::Acq_Rel:
+    // FIXME: Implement 5.0 rules, pending clarification on later spec
+    // versions.
+    // [5.0:62:22-26]
+    if (kind == llvm::omp::Clause::OMPC_read) {
+      clauseList->v.emplace_back(
+          parser::OmpClause{parser::OmpClause::Acquire{}});
+    } else if (kind == llvm::omp::Clause::OMPC_update && x.IsCapture()) {
+      clauseList->v.emplace_back(
+          parser::OmpClause{parser::OmpClause::AcqRel{}});
+    } else {
+      clauseList->v.emplace_back(
+          parser::OmpClause{parser::OmpClause::Release{}});
     }
+    break;
+  case common::OmpMemoryOrderType::Relaxed:
+    clauseList->v.emplace_back(parser::OmpClause{parser::OmpClause::Relaxed{}});
+    break;
+  case common::OmpMemoryOrderType::Seq_Cst:
+    clauseList->v.emplace_back(parser::OmpClause{parser::OmpClause::SeqCst{}});
+    break;
+  default:
+    // FIXME: Don't process other values at the moment since their validity
+    // depends on the OpenMP version (which is unavailable here).
+    break;
   }
 
   return false;
diff --git a/external/llvm-project/flang/lib/Semantics/runtime-type-info.cpp b/external/llvm-project/flang/lib/Semantics/runtime-type-info.cpp
index 98295f3705a7..26ae81f97895 100644
--- a/external/llvm-project/flang/lib/Semantics/runtime-type-info.cpp
+++ b/external/llvm-project/flang/lib/Semantics/runtime-type-info.cpp
@@ -1018,9 +1018,11 @@ SymbolVector CollectBindings(const Scope &dtScope) {
       if (overriderIter != localBindings.end()) {
         Symbol &overrider{*overriderIter->second};
         if (symbol.attrs().test(Attr::PRIVATE) &&
+            !symbol.attrs().test(Attr::DEFERRED) &&
             FindModuleContaining(symbol.owner()) !=
                 FindModuleContaining(dtScope)) {
-          // Don't override inaccessible PRIVATE bindings
+          // Don't override inaccessible PRIVATE bindings, unless
+          // they are deferred
           auto &binding{overrider.get<ProcBindingDetails>()};
           binding.set_numPrivatesNotOverridden(
               binding.numPrivatesNotOverridden() + 1);
@@ -1061,7 +1063,7 @@ RuntimeTableBuilder::DescribeSpecialGenerics(const Scope &dtScope,
     specials =
         DescribeSpecialGenerics(*parentScope, thisScope, derivedTypeSpec);
   }
-  for (auto pair : dtScope) {
+  for (const auto &pair : dtScope) {
     const Symbol &symbol{*pair.second};
     if (const auto *generic{symbol.detailsIf<GenericDetails>()}) {
       DescribeSpecialGeneric(*generic, specials, thisScope, derivedTypeSpec);
@@ -1121,10 +1123,10 @@ void RuntimeTableBuilder::DescribeSpecialProc(
     int argThatMightBeDescriptor{0};
     MaybeExpr which;
     if (isAssignment) {
-      // Only type-bound asst's with the same type on both dummy arguments
+      // Only type-bound asst's with compatible types on both dummy arguments
       // are germane to the runtime, which needs only these to implement
       // component assignment as part of intrinsic assignment.
-      // Non-type-bound generic INTERFACEs and assignments from distinct
+      // Non-type-bound generic INTERFACEs and assignments from incompatible
       // types must not be used for component intrinsic assignment.
       CHECK(proc->dummyArguments.size() == 2);
       const auto t1{
@@ -1137,8 +1139,12 @@ void RuntimeTableBuilder::DescribeSpecialProc(
               .type.type()};
       if (!binding || t1.category() != TypeCategory::Derived ||
           t2.category() != TypeCategory::Derived ||
-          t1.IsUnlimitedPolymorphic() || t2.IsUnlimitedPolymorphic() ||
-          t1.GetDerivedTypeSpec() != t2.GetDerivedTypeSpec()) {
+          t1.IsUnlimitedPolymorphic() || t2.IsUnlimitedPolymorphic()) {
+        return;
+      }
+      if (!derivedTypeSpec ||
+          !derivedTypeSpec->MatchesOrExtends(t1.GetDerivedTypeSpec()) ||
+          !derivedTypeSpec->MatchesOrExtends(t2.GetDerivedTypeSpec())) {
         return;
       }
       which = proc->IsElemental() ? elementalAssignmentEnum_
@@ -1235,7 +1241,7 @@ void RuntimeTableBuilder::DescribeSpecialProc(
     AddValue(values, specialSchema_, procCompName,
         SomeExpr{evaluate::ProcedureDesignator{specific}});
     // index might already be present in the case of an override
-    specials.emplace(*index,
+    specials.insert_or_assign(*index,
         evaluate::StructureConstructor{
             DEREF(specialSchema_.AsDerived()), std::move(values)});
   }
diff --git a/external/llvm-project/flang/lib/Semantics/symbol.cpp b/external/llvm-project/flang/lib/Semantics/symbol.cpp
index 2118970a7bf2..0380207927ad 100644
--- a/external/llvm-project/flang/lib/Semantics/symbol.cpp
+++ b/external/llvm-project/flang/lib/Semantics/symbol.cpp
@@ -292,8 +292,7 @@ void GenericDetails::CopyFrom(const GenericDetails &from) {
 // This is primarily for debugging.
 std::string DetailsToString(const Details &details) {
   return common::visit(
-      common::visitors{
-          [](const UnknownDetails &) { return "Unknown"; },
+      common::visitors{[](const UnknownDetails &) { return "Unknown"; },
           [](const MainProgramDetails &) { return "MainProgram"; },
           [](const ModuleDetails &) { return "Module"; },
           [](const SubprogramDetails &) { return "Subprogram"; },
@@ -312,7 +311,7 @@ std::string DetailsToString(const Details &details) {
           [](const TypeParamDetails &) { return "TypeParam"; },
           [](const MiscDetails &) { return "Misc"; },
           [](const AssocEntityDetails &) { return "AssocEntity"; },
-      },
+          [](const UserReductionDetails &) { return "UserReductionDetails"; }},
       details);
 }
 
@@ -346,6 +345,9 @@ bool Symbol::CanReplaceDetails(const Details &details) const {
             [&](const HostAssocDetails &) {
               return this->has<HostAssocDetails>();
             },
+            [&](const UserReductionDetails &) {
+              return this->has<UserReductionDetails>();
+            },
             [](const auto &) { return false; },
         },
         details);
@@ -644,6 +646,11 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Details &details) {
           [&](const MiscDetails &x) {
             os << ' ' << MiscDetails::EnumToString(x.kind());
           },
+          [&](const UserReductionDetails &x) {
+            for (auto &type : x.GetTypeList()) {
+              DumpType(os, type);
+            }
+          },
           [&](const auto &x) { os << x; },
       },
       details);
@@ -842,6 +849,9 @@ std::string Symbol::OmpFlagToClauseName(Symbol::Flag ompFlag) {
   case Symbol::Flag::OmpLinear:
     clauseName = "LINEAR";
     break;
+  case Symbol::Flag::OmpUniform:
+    clauseName = "UNIFORM";
+    break;
   case Symbol::Flag::OmpFirstPrivate:
     clauseName = "FIRSTPRIVATE";
     break;
diff --git a/external/llvm-project/flang/lib/Semantics/tools.cpp b/external/llvm-project/flang/lib/Semantics/tools.cpp
index 1d1e3ac04416..a1445187b1e9 100644
--- a/external/llvm-project/flang/lib/Semantics/tools.cpp
+++ b/external/llvm-project/flang/lib/Semantics/tools.cpp
@@ -17,6 +17,7 @@
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
 #include "flang/Support/Fortran.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <set>
@@ -1756,4 +1757,332 @@ bool HadUseError(
   }
 }
 
-} // namespace Fortran::semantics
+bool CheckForSymbolMatch(const SomeExpr *lhs, const SomeExpr *rhs) {
+  if (lhs && rhs) {
+    if (SymbolVector lhsSymbols{evaluate::GetSymbolVector(*lhs)};
+        !lhsSymbols.empty()) {
+      const Symbol &first{*lhsSymbols.front()};
+      for (const Symbol &symbol : evaluate::GetSymbolVector(*rhs)) {
+        if (first == symbol) {
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+namespace operation {
+template <typename T> //
+SomeExpr asSomeExpr(const T &x) {
+  auto copy{x};
+  return AsGenericExpr(std::move(copy));
+}
+
+template <bool IgnoreResizingConverts> //
+struct ArgumentExtractor
+    : public evaluate::Traverse<ArgumentExtractor<IgnoreResizingConverts>,
+          std::pair<operation::Operator, std::vector<SomeExpr>>, false> {
+  using Arguments = std::vector<SomeExpr>;
+  using Result = std::pair<operation::Operator, Arguments>;
+  using Base = evaluate::Traverse<ArgumentExtractor<IgnoreResizingConverts>,
+      Result, false>;
+  static constexpr auto IgnoreResizes = IgnoreResizingConverts;
+  static constexpr auto Logical = common::TypeCategory::Logical;
+  ArgumentExtractor() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <int Kind> //
+  Result operator()(
+      const evaluate::Constant<evaluate::Type<Logical, Kind>> &x) const {
+    if (const auto &val{x.GetScalarValue()}) {
+      return val->IsTrue()
+          ? std::make_pair(operation::Operator::True, Arguments{})
+          : std::make_pair(operation::Operator::False, Arguments{});
+    }
+    return Default();
+  }
+
+  template <typename R> //
+  Result operator()(const evaluate::FunctionRef<R> &x) const {
+    Result result{operation::OperationCode(x.proc()), {}};
+    for (size_t i{0}, e{x.arguments().size()}; i != e; ++i) {
+      if (auto *e{x.UnwrapArgExpr(i)}) {
+        result.second.push_back(*e);
+      }
+    }
+    return result;
+  }
+
+  template <typename D, typename R, typename... Os>
+  Result operator()(const evaluate::Operation<D, R, Os...> &x) const {
+    if constexpr (std::is_same_v<D, evaluate::Parentheses<R>>) {
+      // Ignore top-level parentheses.
+      return (*this)(x.template operand<0>());
+    }
+    if constexpr (IgnoreResizes &&
+        std::is_same_v<D, evaluate::Convert<R, R::category>>) {
+      // Ignore conversions within the same category.
+      // Atomic operations on int(kind=1) may be implicitly widened
+      // to int(kind=4) for example.
+      return (*this)(x.template operand<0>());
+    } else {
+      return std::make_pair(operation::OperationCode(x),
+          OperationArgs(x, std::index_sequence_for<Os...>{}));
+    }
+  }
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    return {operation::Operator::Identity, {asSomeExpr(x)}};
+  }
+
+  template <typename T> //
+  Result operator()(const evaluate::Constant<T> &x) const {
+    return {operation::Operator::Identity, {asSomeExpr(x)}};
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    // There shouldn't be any combining needed, since we're stopping the
+    // traversal at the top-level operation, but implement one that picks
+    // the first non-empty result.
+    if constexpr (sizeof...(Rs) == 0) {
+      return std::move(result);
+    } else {
+      if (!result.second.empty()) {
+        return std::move(result);
+      } else {
+        return Combine(std::move(results)...);
+      }
+    }
+  }
+
+private:
+  template <typename D, typename R, typename... Os, size_t... Is>
+  Arguments OperationArgs(const evaluate::Operation<D, R, Os...> &x,
+      std::index_sequence<Is...>) const {
+    return Arguments{SomeExpr(x.template operand<Is>())...};
+  }
+};
+} // namespace operation
+
+std::string operation::ToString(operation::Operator op) {
+  switch (op) {
+  case Operator::Unknown:
+    return "??";
+  case Operator::Add:
+    return "+";
+  case Operator::And:
+    return "AND";
+  case Operator::Associated:
+    return "ASSOCIATED";
+  case Operator::Call:
+    return "function-call";
+  case Operator::Constant:
+    return "constant";
+  case Operator::Convert:
+    return "type-conversion";
+  case Operator::Div:
+    return "/";
+  case Operator::Eq:
+    return "==";
+  case Operator::Eqv:
+    return "EQV";
+  case Operator::False:
+    return ".FALSE.";
+  case Operator::Ge:
+    return ">=";
+  case Operator::Gt:
+    return ">";
+  case Operator::Identity:
+    return "identity";
+  case Operator::Intrinsic:
+    return "intrinsic";
+  case Operator::Le:
+    return "<=";
+  case Operator::Lt:
+    return "<";
+  case Operator::Max:
+    return "MAX";
+  case Operator::Min:
+    return "MIN";
+  case Operator::Mul:
+    return "*";
+  case Operator::Ne:
+    return "/=";
+  case Operator::Neqv:
+    return "NEQV/EOR";
+  case Operator::Not:
+    return "NOT";
+  case Operator::Or:
+    return "OR";
+  case Operator::Pow:
+    return "**";
+  case Operator::Resize:
+    return "resize";
+  case Operator::Sub:
+    return "-";
+  case Operator::True:
+    return ".TRUE.";
+  }
+  llvm_unreachable("Unhandler operator");
+}
+
+operation::Operator operation::OperationCode(
+    const evaluate::ProcedureDesignator &proc) {
+  Operator code = llvm::StringSwitch<Operator>(proc.GetName())
+                      .Case("associated", Operator::Associated)
+                      .Case("min", Operator::Min)
+                      .Case("max", Operator::Max)
+                      .Case("iand", Operator::And)
+                      .Case("ior", Operator::Or)
+                      .Case("ieor", Operator::Neqv)
+                      .Default(Operator::Call);
+  if (code == Operator::Call && proc.GetSpecificIntrinsic()) {
+    return Operator::Intrinsic;
+  }
+  return code;
+}
+
+std::pair<operation::Operator, std::vector<SomeExpr>> GetTopLevelOperation(
+    const SomeExpr &expr) {
+  return operation::ArgumentExtractor<true>{}(expr);
+}
+
+namespace operation {
+struct ConvertCollector
+    : public evaluate::Traverse<ConvertCollector,
+          std::pair<MaybeExpr, std::vector<evaluate::DynamicType>>, false> {
+  using Result = std::pair<MaybeExpr, std::vector<evaluate::DynamicType>>;
+  using Base = evaluate::Traverse<ConvertCollector, Result, false>;
+  ConvertCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    return {asSomeExpr(x), {}};
+  }
+
+  template <typename T> //
+  Result operator()(const evaluate::FunctionRef<T> &x) const {
+    return {asSomeExpr(x), {}};
+  }
+
+  template <typename T> //
+  Result operator()(const evaluate::Constant<T> &x) const {
+    return {asSomeExpr(x), {}};
+  }
+
+  template <typename D, typename R, typename... Os>
+  Result operator()(const evaluate::Operation<D, R, Os...> &x) const {
+    if constexpr (std::is_same_v<D, evaluate::Parentheses<R>>) {
+      // Ignore parentheses.
+      return (*this)(x.template operand<0>());
+    } else if constexpr (is_convert_v<D>) {
+      // Convert should always have a typed result, so it should be safe to
+      // dereference x.GetType().
+      return Combine(
+          {std::nullopt, {*x.GetType()}}, (*this)(x.template operand<0>()));
+    } else if constexpr (is_complex_constructor_v<D>) {
+      // This is a conversion iff the imaginary operand is 0.
+      if (IsZero(x.template operand<1>())) {
+        return Combine(
+            {std::nullopt, {*x.GetType()}}, (*this)(x.template operand<0>()));
+      } else {
+        return {asSomeExpr(x.derived()), {}};
+      }
+    } else {
+      return {asSomeExpr(x.derived()), {}};
+    }
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto setValue{[](MaybeExpr &x, MaybeExpr &&y) {
+      assert((!x.has_value() || !y.has_value()) && "Multiple designators");
+      if (!x.has_value()) {
+        x = std::move(y);
+      }
+    }};
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
+      }
+    }};
+    (setValue(v.first, std::move(results).first), ...);
+    (moveAppend(v.second, std::move(results).second), ...);
+    return v;
+  }
+
+private:
+  template <typename T> //
+  static bool IsZero(const T &x) {
+    return false;
+  }
+  template <typename T> //
+  static bool IsZero(const evaluate::Expr<T> &x) {
+    return common::visit([](auto &&s) { return IsZero(s); }, x.u);
+  }
+  template <typename T> //
+  static bool IsZero(const evaluate::Constant<T> &x) {
+    if (auto &&maybeScalar{x.GetScalarValue()}) {
+      return maybeScalar->IsZero();
+    } else {
+      return false;
+    }
+  }
+
+  template <typename T> //
+  struct is_convert {
+    static constexpr bool value{false};
+  };
+  template <typename T, common::TypeCategory C> //
+  struct is_convert<evaluate::Convert<T, C>> {
+    static constexpr bool value{true};
+  };
+  template <int K> //
+  struct is_convert<evaluate::ComplexComponent<K>> {
+    // Conversion from complex to real.
+    static constexpr bool value{true};
+  };
+  template <typename T> //
+  static constexpr bool is_convert_v = is_convert<T>::value;
+
+  template <typename T> //
+  struct is_complex_constructor {
+    static constexpr bool value{false};
+  };
+  template <int K> //
+  struct is_complex_constructor<evaluate::ComplexConstructor<K>> {
+    static constexpr bool value{true};
+  };
+  template <typename T> //
+  static constexpr bool is_complex_constructor_v =
+      is_complex_constructor<T>::value;
+};
+} // namespace operation
+
+MaybeExpr GetConvertInput(const SomeExpr &x) {
+  // This returns SomeExpr(x) when x is a designator/functionref/constant.
+  return operation::ConvertCollector{}(x).first;
+}
+
+bool IsSameOrConvertOf(const SomeExpr &expr, const SomeExpr &x) {
+  // Check if expr is same as x, or a sequence of Convert operations on x.
+  if (expr == x) {
+    return true;
+  } else if (auto maybe{GetConvertInput(expr)}) {
+    return *maybe == x;
+  } else {
+    return false;
+  }
+}
+} // namespace Fortran::semantics
\ No newline at end of file
diff --git a/external/llvm-project/flang/lib/Support/Fortran-features.cpp b/external/llvm-project/flang/lib/Support/Fortran-features.cpp
index b5e66e626f15..e36fe744cea2 100644
--- a/external/llvm-project/flang/lib/Support/Fortran-features.cpp
+++ b/external/llvm-project/flang/lib/Support/Fortran-features.cpp
@@ -8,11 +8,68 @@
 
 #include "flang/Support/Fortran-features.h"
 #include "flang/Common/idioms.h"
+#include "flang/Parser/characters.h"
 #include "flang/Support/Fortran.h"
+#include <string>
+#include <string_view>
 
 namespace Fortran::common {
 
+static std::vector<std::string_view> SplitCamelCase(std::string_view x) {
+  std::vector<std::string_view> result;
+  // NB, we start at 1 because the first character is never a word boundary.
+  size_t xSize{x.size()}, wordStart{0}, wordEnd{1};
+  for (; wordEnd < xSize; ++wordEnd) {
+    // Identify when wordEnd is at the start of a new word.
+    if ((!parser::IsUpperCaseLetter(x[wordEnd - 1]) &&
+            parser::IsUpperCaseLetter(x[wordEnd])) ||
+        // ACCUsage => ACC-Usage, CComment => C-Comment, etc.
+        (parser::IsUpperCaseLetter(x[wordEnd]) && wordEnd + 1 < xSize &&
+            parser::IsLowerCaseLetter(x[wordEnd + 1]))) {
+      result.push_back(x.substr(wordStart, wordEnd - wordStart));
+      wordStart = wordEnd;
+    }
+  }
+  // We went one past the end of the last word.
+  result.push_back(x.substr(wordStart, wordEnd - wordStart));
+  return result;
+}
+
+// Namespace for helper functions for parsing Cli options used instead of static
+// so that there can be unit tests for this function.
+namespace details {
+std::string CamelCaseToLowerCaseHyphenated(std::string_view x) {
+  std::vector<std::string_view> words{SplitCamelCase(x)};
+  std::string result{};
+  result.reserve(x.size() + words.size() + 1);
+  for (size_t i{0}; i < words.size(); ++i) {
+    std::string word{parser::ToLowerCaseLetters(words[i])};
+    result += i == 0 ? "" : "-";
+    result += word;
+  }
+  return result;
+}
+} // namespace details
+
 LanguageFeatureControl::LanguageFeatureControl() {
+  // Initialize the bidirectional maps with the default spellings.
+  cliOptions_.reserve(LanguageFeature_enumSize + UsageWarning_enumSize);
+  ForEachLanguageFeature([&](auto feature) {
+    std::string_view name{Fortran::common::EnumToString(feature)};
+    std::string cliOption{details::CamelCaseToLowerCaseHyphenated(name)};
+    cliOptions_.insert({cliOption, {feature}});
+    languageFeatureCliCanonicalSpelling_[EnumToInt(feature)] =
+        std::string_view{cliOption};
+  });
+
+  ForEachUsageWarning([&](auto warning) {
+    std::string_view name{Fortran::common::EnumToString(warning)};
+    std::string cliOption{details::CamelCaseToLowerCaseHyphenated(name)};
+    cliOptions_.insert({cliOption, {warning}});
+    usageWarningCliCanonicalSpelling_[EnumToInt(warning)] =
+        std::string_view{cliOption};
+  });
+
   // These features must be explicitly enabled by command line options.
   disable_.set(LanguageFeature::OldDebugLines);
   disable_.set(LanguageFeature::OpenACC);
@@ -95,57 +152,40 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnLanguage_.set(LanguageFeature::NullActualForAllocatable);
 }
 
-// Ignore case and any inserted punctuation (like '-'/'_')
-static std::optional<char> GetWarningChar(char ch) {
-  if (ch >= 'a' && ch <= 'z') {
-    return ch;
-  } else if (ch >= 'A' && ch <= 'Z') {
-    return ch - 'A' + 'a';
-  } else if (ch >= '0' && ch <= '9') {
-    return ch;
-  } else {
-    return std::nullopt;
+// Take a string from the Cli and apply it to the LanguageFeatureControl.
+bool LanguageFeatureControl::ApplyCliOption(std::string input) {
+  bool negated{false};
+  if (input.size() > 3 && input.substr(0, 3) == "no-") {
+    negated = true;
+    input = input.substr(3);
   }
-}
-
-static bool WarningNameMatch(const char *a, const char *b) {
-  while (true) {
-    auto ach{GetWarningChar(*a)};
-    while (!ach && *a) {
-      ach = GetWarningChar(*++a);
-    }
-    auto bch{GetWarningChar(*b)};
-    while (!bch && *b) {
-      bch = GetWarningChar(*++b);
-    }
-    if (!ach && !bch) {
+  if (auto it{cliOptions_.find(input)}; it != cliOptions_.end()) {
+    if (std::holds_alternative<LanguageFeature>(it->second)) {
+      EnableWarning(std::get<LanguageFeature>(it->second), !negated);
       return true;
-    } else if (!ach || !bch || *ach != *bch) {
-      return false;
     }
-    ++a, ++b;
-  }
-}
-
-template <typename ENUM, std::size_t N>
-std::optional<ENUM> ScanEnum(const char *name) {
-  if (name) {
-    for (std::size_t j{0}; j < N; ++j) {
-      auto feature{static_cast<ENUM>(j)};
-      if (WarningNameMatch(name, EnumToString(feature).data())) {
-        return feature;
-      }
+    if (std::holds_alternative<UsageWarning>(it->second)) {
+      EnableWarning(std::get<UsageWarning>(it->second), !negated);
+      return true;
     }
   }
-  return std::nullopt;
+  return false;
 }
 
-std::optional<LanguageFeature> FindLanguageFeature(const char *name) {
-  return ScanEnum<LanguageFeature, LanguageFeature_enumSize>(name);
+void LanguageFeatureControl::ReplaceCliCanonicalSpelling(
+    LanguageFeature f, std::string input) {
+  std::string_view &old{languageFeatureCliCanonicalSpelling_[EnumToInt(f)]};
+  cliOptions_.erase(std::string{old});
+  languageFeatureCliCanonicalSpelling_[EnumToInt(f)] = input;
+  cliOptions_.insert({input, {f}});
 }
 
-std::optional<UsageWarning> FindUsageWarning(const char *name) {
-  return ScanEnum<UsageWarning, UsageWarning_enumSize>(name);
+void LanguageFeatureControl::ReplaceCliCanonicalSpelling(
+    UsageWarning w, std::string input) {
+  std::string_view &old{usageWarningCliCanonicalSpelling_[EnumToInt(w)]};
+  cliOptions_.erase(std::string{old});
+  usageWarningCliCanonicalSpelling_[EnumToInt(w)] = input;
+  cliOptions_.insert({input, {w}});
 }
 
 std::vector<const char *> LanguageFeatureControl::GetNames(
@@ -202,4 +242,26 @@ std::vector<const char *> LanguageFeatureControl::GetNames(
   }
 }
 
+void LanguageFeatureControl::WarnOnAllNonstandard(bool yes) {
+  warnAllLanguage_ = yes;
+  warnLanguage_.reset();
+  if (yes) {
+    disableAllWarnings_ = false;
+    warnLanguage_.flip();
+    // These three features do not need to be warned about,
+    // but we do want their feature flags.
+    warnLanguage_.set(LanguageFeature::OpenMP, false);
+    warnLanguage_.set(LanguageFeature::OpenACC, false);
+    warnLanguage_.set(LanguageFeature::CUDA, false);
+  }
+}
+
+void LanguageFeatureControl::WarnOnAllUsage(bool yes) {
+  warnAllUsage_ = yes;
+  warnUsage_.reset();
+  if (yes) {
+    disableAllWarnings_ = false;
+    warnUsage_.flip();
+  }
+}
 } // namespace Fortran::common
diff --git a/external/llvm-project/flang/test/CMakeLists.txt b/external/llvm-project/flang/test/CMakeLists.txt
index a658f6f984fa..8520bec64697 100644
--- a/external/llvm-project/flang/test/CMakeLists.txt
+++ b/external/llvm-project/flang/test/CMakeLists.txt
@@ -73,6 +73,7 @@ if (NOT FLANG_STANDALONE_BUILD)
     not
     llvm-dis
     llvm-objdump
+    llvm-profdata
     llvm-readobj
     split-file
   )
diff --git a/external/llvm-project/flang/test/Driver/Inputs/config-7.cfg b/external/llvm-project/flang/test/Driver/Inputs/config-7.cfg
new file mode 100644
index 000000000000..2f41be663b28
--- /dev/null
+++ b/external/llvm-project/flang/test/Driver/Inputs/config-7.cfg
@@ -0,0 +1 @@
+-Werror
diff --git a/external/llvm-project/flang/test/Driver/darwin-version.f90 b/external/llvm-project/flang/test/Driver/darwin-version.f90
new file mode 100644
index 000000000000..99d19ee44be9
--- /dev/null
+++ b/external/llvm-project/flang/test/Driver/darwin-version.f90
@@ -0,0 +1,107 @@
+! Based on clang's darwin-version.c test with tests for ios watchos and tvos
+! removed
+
+! RUN: %flang -target i686-apple-darwin8 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX4 %s
+! RUN: %flang -target i686-apple-darwin9 -mmacos-version-min=10.4 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX4 %s
+! CHECK-VERSION-OSX4: "i686-apple-macosx10.4.0"
+! RUN: %flang -target i686-apple-darwin9 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX5 %s
+! RUN: %flang -target i686-apple-darwin9 -mmacos-version-min=10.5 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX5 %s
+! CHECK-VERSION-OSX5: "i686-apple-macosx10.5.0"
+! RUN: %flang -target i686-apple-darwin10 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX6 %s
+! RUN: %flang -target i686-apple-darwin9 -mmacos-version-min=10.6 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX6 %s
+! CHECK-VERSION-OSX6: "i686-apple-macosx10.6.0"
+! RUN: %flang -target x86_64-apple-darwin14 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX10 %s
+! RUN: %flang -target x86_64-apple-darwin -mmacos-version-min=10.10 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX10 %s
+! RUN: %flang -target x86_64-apple-darwin -mmacos-version-min=10.10 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX10 %s
+! CHECK-VERSION-OSX10: "x86_64-apple-macosx10.10.0"
+! RUN: not %flang -target x86_64-apple-darwin -mmacos-version-min= -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-MISSING %s
+! RUN: not %flang -target x86_64-apple-darwin -mmacos-version-min= -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-MISSING %s
+! CHECK-VERSION-MISSING: missing version number
+
+! RUN: %flang -target x86_64-apple-driverkit19.0 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-DRIVERKIT190 %s
+! CHECK-VERSION-DRIVERKIT190: "x86_64-apple-driverkit19.0.0"
+
+! Check environment variable gets interpreted correctly
+! RUN: env MACOSX_DEPLOYMENT_TARGET=10.5 IPHONEOS_DEPLOYMENT_TARGET=2.0 \
+! RUN:   %flang -target i686-apple-darwin9 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX5 %s
+
+! RUN: env MACOSX_DEPLOYMENT_TARGET=10.4.10 \
+! RUN:   %flang -target i386-apple-darwin9 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-OSX49 %s
+! CHECK-VERSION-OSX49: "i386-apple-macosx10.4.10"
+! RUN: env IPHONEOS_DEPLOYMENT_TARGET=2.3.1 \
+
+! Target can specify the OS version:
+
+! RUN: %flang -target x86_64-apple-macos10.11.2 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-TMAC2 %s
+! CHECK-VERSION-TMAC2: "x86_64-apple-macosx10.11.2"
+
+! Warn about -m<os>-version-min when it's used with target:
+
+! RUN: %flang -target x86_64-apple-macos10.11.2 -mmacos-version-min=10.6 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-TNO-OSV1 %s
+! CHECK-VERSION-TNO-OSV1: overriding '-mmacos-version-min=10.6' option with '-target x86_64-apple-macos10.11.2'
+
+! RUN: %flang -target x86_64-apple-macos10.6 -mmacos-version-min=10.6 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-TNO-SAME %s
+! CHECK-VERSION-TNO-SAME-NOT: overriding
+! CHECK-VERSION-TNO-SAME-NOT: argument unused during compilation
+
+! Target with OS version is not overridden by -m<os>-version-min variables:
+
+! RUN: %flang -target x86_64-apple-macos10.11.2 -mmacos-version-min=10.6 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-TIGNORE-OSV1 %s
+! CHECK-VERSION-TIGNORE-OSV1: "x86_64-apple-macosx10.11.2"
+
+! Target without OS version includes the OS given by -m<os>-version-min arguments:
+
+! RUN: %flang -target x86_64-apple-macos -mmacos-version-min=10.11 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-USE-OS-ARG1 %s
+! CHECK-VERSION-USE-OS-ARG1: "x86_64-apple-macosx10.11.0"
+
+! Target with OS version is not overridden by environment variables:
+
+! RUN: env MACOSX_DEPLOYMENT_TARGET=10.1 \
+! RUN:   %flang -target i386-apple-macos10.5 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-TMACOS-CMD %s
+! CHECK-VERSION-TMACOS-CMD: "i386-apple-macosx10.5.0"
+
+! Target with OS version is not overridden by arch:
+
+! RUN: %flang -target uknown-apple-macos10.11.2 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-VERSION-TIGNORE-ARCH1 %s
+! CHECK-VERSION-TIGNORE-ARCH1: "unknown-apple-macosx10.11.2"
+
+! Target can be used to specify the environment:
+
+! RUN: %flang -target x86_64-apple-macos11 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
+! RUN: %flang -target x86_64-apple-darwin20 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
+! RUN: %flang -target x86_64-apple-darwin -mmacos-version-min=11 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-MACOS11 %s
+! CHECK-MACOS11: "x86_64-apple-macosx11.0.0"
+
+! RUN: %flang -target arm64-apple-macosx10.16 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-IMPLICIT-MACOS11 %s
+! CHECK-IMPLICIT-MACOS11: warning: overriding deployment version
+! CHECK-IMPLICIT-MACOS11: "arm64-apple-macosx11.0.0"
+
+! RUN: %flang -target arm64-apple-macos999 -c %s -### 2>&1 | \
+! RUN:   FileCheck --check-prefix=CHECK-MACOS999 %s
+
+! CHECK-MACOS999: "arm64-apple-macosx999.0.0"
diff --git a/external/llvm-project/flang/test/Driver/disable-diagnostic.f90 b/external/llvm-project/flang/test/Driver/disable-diagnostic.f90
new file mode 100644
index 000000000000..849489377da1
--- /dev/null
+++ b/external/llvm-project/flang/test/Driver/disable-diagnostic.f90
@@ -0,0 +1,20 @@
+! RUN: %flang -Wknown-bad-implicit-interface %s -c 2>&1 | FileCheck %s --check-prefix=WARN
+! RUN: %flang -pedantic -Wno-known-bad-implicit-interface %s -c 2>&1 | FileCheck %s --allow-empty
+! RUN: not %flang -WKnownBadImplicitInterface %s -c 2>&1 | FileCheck %s --check-prefix=ERROR1
+! RUN: not %flang -WKnown-Bad-Implicit-Interface %s -c 2>&1 | FileCheck %s --check-prefix=ERROR2
+
+! ERROR1: error: Unknown diagnostic option: -WKnownBadImplicitInterface
+! ERROR2: error: Unknown diagnostic option: -WKnown-Bad-Implicit-Interface
+
+program disable_diagnostic
+  REAL :: x
+  INTEGER :: y
+   ! CHECK-NOT: warning
+  ! WARN: warning: If the procedure's interface were explicit, this reference would be in error
+  call sub(x)
+  ! WARN: warning: If the procedure's interface were explicit, this reference would be in error
+  call sub(y)
+end program disable_diagnostic
+
+subroutine sub()
+end subroutine sub
diff --git a/external/llvm-project/flang/test/Driver/fcc_override.f90 b/external/llvm-project/flang/test/Driver/fcc_override.f90
new file mode 100644
index 000000000000..2717d203c2ea
--- /dev/null
+++ b/external/llvm-project/flang/test/Driver/fcc_override.f90
@@ -0,0 +1,17 @@
+! RUN: env FCC_OVERRIDE_OPTIONS="#+-Os +-Oz +-O +-O3 +-Oignore +a +b +c xb Xa Omagic ^-###  " %flang --target=x86_64-unknown-linux-gnu %s -O2 b -O3 2>&1 | FileCheck %s
+! RUN: env FCC_OVERRIDE_OPTIONS="x-Werror +-g" %flang --target=x86_64-unknown-linux-gnu -Werror %s -c -### 2>&1 | FileCheck %s -check-prefix=RM-WERROR
+! RUN: env FCC_OVERRIDE_OPTIONS="x-Werror" %flang --config=%S/Inputs/config-7.cfg -### %s -c  2>&1 | FileCheck %s -check-prefix=CONF
+
+! CHECK: "-fc1"
+! CHECK-NOT: "-Oignore"
+! CHECK: "-Omagic"
+! CHECK-NOT: "-Oignore"
+
+! RM-WERROR: ### FCC_OVERRIDE_OPTIONS: x-Werror +-g
+! RM-WERROR-NEXT: ### Deleting argument -Werror
+! RM-WERROR-NEXT: ### Adding argument -g at end
+! RM-WERROR-NOT: "-Werror"
+
+! Test that FCC_OVERRIDE_OPTIONS does not affect the options from config files.
+! CONF: ### FCC_OVERRIDE_OPTIONS: x-Werror
+! CONF: "-Werror"
diff --git a/external/llvm-project/flang/test/Driver/flang-f-opts.f90 b/external/llvm-project/flang/test/Driver/flang-f-opts.f90
index 4493a519e201..b972b9b7b2a5 100644
--- a/external/llvm-project/flang/test/Driver/flang-f-opts.f90
+++ b/external/llvm-project/flang/test/Driver/flang-f-opts.f90
@@ -8,3 +8,8 @@
 ! CHECK-LABEL: "-fc1"
 ! CHECK: -ffp-contract=off
 ! CHECK: -O3
+
+! RUN: %flang -### -S -fprofile-generate %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-GENERATE-LLVM %s
+! CHECK-PROFILE-GENERATE-LLVM: "-fprofile-generate"
+! RUN: %flang -### -S -fprofile-use=%S %s 2>&1 | FileCheck -check-prefix=CHECK-PROFILE-USE-DIR %s
+! CHECK-PROFILE-USE-DIR: "-fprofile-use={{.*}}"
diff --git a/external/llvm-project/flang/test/Driver/mrecip.f90 b/external/llvm-project/flang/test/Driver/mrecip.f90
new file mode 100644
index 000000000000..aed7cfad4c2a
--- /dev/null
+++ b/external/llvm-project/flang/test/Driver/mrecip.f90
@@ -0,0 +1,52 @@
+! Test that -mrecip[=<list>] works as expected.
+
+! RUN: %flang -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-OMIT
+! RUN: %flang -mrecip -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-NOARG
+! RUN: %flang -mrecip=all -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-ALL
+! RUN: %flang -mrecip=none -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-NONE
+! RUN: %flang -mrecip=default -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-DEF
+! RUN: %flang -mrecip=divd,divf,divh,vec-divd,vec-divf,vec-divh,sqrtd,sqrtf,sqrth,vec-sqrtd,vec-sqrtf,vec-sqrth -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-POS
+! RUN: %flang -mrecip=!divd,!divf,!divh,!vec-divd,!vec-divf,!vec-divh,!sqrtd,!sqrtf,!sqrth,!vec-sqrtd,!vec-sqrtf,!vec-sqrth -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-NEG
+! RUN: %flang -mrecip=!divd,divf,!divh,sqrtd,!sqrtf,sqrth -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG,CHECK-FLANG-MIX
+! RUN: not %flang -mrecip=xxx  -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG-INV
+! RUN: not %flang -mrecip=divd,divd  -### -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FLANG-DUP
+
+! RUN: %flang_fc1 -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-FC1-OMIT
+! RUN: %flang_fc1 -mrecip -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1,CHECK-FC1-NOARG
+! RUN: %flang_fc1 -mrecip=all -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1,CHECK-FC1-ALL
+! RUN: %flang_fc1 -mrecip=none -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1,CHECK-FC1-NONE
+! RUN: %flang_fc1 -mrecip=default -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1,CHECK-FC1-DEF
+! RUN: %flang_fc1 -mrecip=divd,divf,divh,vec-divd,vec-divf,vec-divh,sqrtd,sqrtf,sqrth,vec-sqrtd,vec-sqrtf,vec-sqrth -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1,CHECK-FC1-POS
+! RUN: %flang_fc1 -mrecip=!divd,!divf,!divh,!vec-divd,!vec-divf,!vec-divh,!sqrtd,!sqrtf,!sqrth,!vec-sqrtd,!vec-sqrtf,!vec-sqrth
+! -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1,CHECK-FC1-NEG
+! RUN: %flang_fc1 -mrecip=!divd,divf,!divh,sqrtd,!sqrtf,sqrth -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1,CHECK-FC1-MIX
+! RUN: not %flang_fc1 -mrecip=xxx  -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1-INV
+! RUN: not %flang_fc1 -mrecip=divd,divd  -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-FC1-DUP
+
+subroutine func
+end subroutine func
+
+! CHECK-FLANG: "-fc1"
+! CHECK-FLANG-OMIT-NOT: "-mrecip="
+! CHECK-FLANG-NOARG-SAME: "-mrecip=all"
+! CHECK-FLANG-ALL-SAME: "-mrecip=all"
+! CHECK-FLANG-NONE-SAME: "-mrecip=none"
+! CHECK-FLANG-DEF-SAME: "-mrecip=default"
+! CHECK-FLANG-POS-SAME: "-mrecip=divd,divf,divh,vec-divd,vec-divf,vec-divh,sqrtd,sqrtf,sqrth,vec-sqrtd,vec-sqrtf,vec-sqrth"
+! CHECK-FLANG-NEG-SAME: "-mrecip=!divd,!divf,!divh,!vec-divd,!vec-divf,!vec-divh,!sqrtd,!sqrtf,!sqrth,!vec-sqrtd,!vec-sqrtf,!vec-sqrth"
+! CHECK-FLANG-MIX-SAME: "-mrecip=!divd,divf,!divh,sqrtd,!sqrtf,sqrth"
+! CHECK-FLANG-INV: error: unknown argument: 'xxx'
+! CHECK-FLANG-DUP: error: invalid value 'divd' in 'mrecip='
+
+! CHECK-FC1: define {{.+}} @func{{.*}} #[[ATTRS:[0-9]+]]
+! CHECK-FC1: attributes #[[ATTRS]] =
+! CHECK-FC1-OMIT-NOT: "reciprocal-estimates"
+! CHECK-FC1-NOARG-SAME: "reciprocal-estimates"="all"
+! CHECK-FC1-ALL-SAME: "reciprocal-estimates"="all"
+! CHECK-FC1-NONE-SAME: "reciprocal-estimates"="none"
+! CHECK-FC1-DEF-SAME: "reciprocal-estimates"="default"
+! CHECK-FC1-POS-SAME: "reciprocal-estimates"="divd,divf,divh,vec-divd,vec-divf,vec-divh,sqrtd,sqrtf,sqrth,vec-sqrtd,vec-sqrtf,vec-sqrth"
+! CHECK-FC1-NEG-SAME: "reciprocal-estimates"="!divd,!divf,!divh,!vec-divd,!vec-divf,!vec-divh,!sqrtd,!sqrtf,!sqrth,!vec-sqrtd,!vec-sqrtf,!vec-sqrth"
+! CHECK-FC1-MIX-SAME: "reciprocal-estimates"="!divd,divf,!divh,sqrtd,!sqrtf,sqrth"
+! CHECK-FC1-INV: error: unknown argument: 'xxx'
+! CHECK-FC1-DUP: error: invalid value 'divd' in 'mrecip='
diff --git a/external/llvm-project/flang/test/Driver/pic-flags.f90 b/external/llvm-project/flang/test/Driver/pic-flags.f90
index 2a35e8d02e15..7ddcce94f50c 100644
--- a/external/llvm-project/flang/test/Driver/pic-flags.f90
+++ b/external/llvm-project/flang/test/Driver/pic-flags.f90
@@ -1,7 +1,7 @@
 ! RUN: %if aarch64-registered-target %{ %flang -v -S -emit-llvm -o - %s --target=aarch64-linux-gnu -fno-pie 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-STATIC,CHECK-STATIC-IR %}
 ! REQUIRES: StableDriver
 
-! RUN: %if aarch64-registered-target %{ %flang -v -S -emit-llvm -o - %s --target=aarch64-linux-gnu 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PIE-LEVEL2,CHECK-PIE-LEVEL2-IR %}
+! RUN: %if aarch64-registered-target && clang_default_pie_on_linux %{ %flang -v -S -emit-llvm -o - %s --target=aarch64-linux-gnu 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PIE-LEVEL2,CHECK-PIE-LEVEL2-IR %}
 ! RUN: %if aarch64-registered-target %{ %flang -v -S -emit-llvm -o - %s --target=aarch64-linux-gnu -fpie 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PIE-LEVEL1,CHECK-PIE-LEVEL1-IR %}
 ! RUN: %if aarch64-registered-target %{ %flang -v -S -emit-llvm -o - %s --target=aarch64-linux-gnu -fPIE 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PIE-LEVEL2,CHECK-PIE-LEVEL2-IR %}
 
diff --git a/external/llvm-project/flang/test/Driver/predefined-macros-aarch64.f90 b/external/llvm-project/flang/test/Driver/predefined-macros-aarch64.f90
new file mode 100644
index 000000000000..dabbd1977103
--- /dev/null
+++ b/external/llvm-project/flang/test/Driver/predefined-macros-aarch64.f90
@@ -0,0 +1,16 @@
+! Test predefined macro for AArch64
+
+! REQUIRES: aarch64-registered-target
+
+! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -cpp -E %s | FileCheck %s
+
+! CHECK: integer :: var1 = 1
+! CHECK: integer :: var2 = 1
+
+#if __aarch64__
+  integer :: var1 = __aarch64__
+#endif
+#if __aarch64
+  integer :: var2 = __aarch64
+#endif
+end program
diff --git a/external/llvm-project/flang/test/Driver/prefer-vector-width.f90 b/external/llvm-project/flang/test/Driver/prefer-vector-width.f90
index 8a66a24b88ba..89dc6d35d8ba 100644
--- a/external/llvm-project/flang/test/Driver/prefer-vector-width.f90
+++ b/external/llvm-project/flang/test/Driver/prefer-vector-width.f90
@@ -1,16 +1,18 @@
 ! Test that -mprefer-vector-width works as expected.
 
 ! RUN: %flang_fc1 -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-DEF
-! RUN: %flang_fc1 -mprefer-vector-width=none -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONE
-! RUN: %flang_fc1 -mprefer-vector-width=128 -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-128
-! RUN: %flang_fc1 -mprefer-vector-width=256 -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-256
-! RUN: not %flang_fc1 -mprefer-vector-width=xxx  -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-INVALID
+! RUN: %flang_fc1 -mprefer-vector-width=none -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK,CHECK-NONE
+! RUN: %flang_fc1 -mprefer-vector-width=128 -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK,CHECK-128
+! RUN: %flang_fc1 -mprefer-vector-width=256 -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK,CHECK-256
+! RUN: not %flang_fc1 -mprefer-vector-width=xxx  -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefixes=CHECK-INVALID
 
 subroutine func
 end subroutine func
 
-! CHECK-DEF-NOT: attributes #0 = { "prefer-vector-width"={{.*}}
-! CHECK-NONE: attributes #0 = { "prefer-vector-width"="none"
-! CHECK-128: attributes #0 = { "prefer-vector-width"="128"
-! CHECK-256: attributes #0 = { "prefer-vector-width"="256"
-! CHECK-INVALID:error: invalid value 'xxx' in '-mprefer-vector-width=xxx'
+! CHECK: define {{.+}} @func{{.*}} #[[ATTRS:[0-9]+]]
+! CHECK: attributes #[[ATTRS]] =
+! CHECK-DEF-NOT: "prefer-vector-width"
+! CHECK-NONE-SAME: "prefer-vector-width"="none"
+! CHECK-128-SAME: "prefer-vector-width"="128"
+! CHECK-256-SAME: "prefer-vector-width"="256"
+! CHECK-INVALID: error: invalid value 'xxx' in 'mprefer-vector-width='
diff --git a/external/llvm-project/flang/test/Driver/werror-wrong.f90 b/external/llvm-project/flang/test/Driver/werror-wrong.f90
index 58adf6f745d5..6e3c7cca15bc 100644
--- a/external/llvm-project/flang/test/Driver/werror-wrong.f90
+++ b/external/llvm-project/flang/test/Driver/werror-wrong.f90
@@ -1,6 +1,7 @@
 ! Ensure that only argument -Werror is supported.
 
-! RUN: not %flang_fc1 -fsyntax-only -Wall %s  2>&1 | FileCheck %s --check-prefix=WRONG
-! RUN: not %flang_fc1 -fsyntax-only -WX %s  2>&1 | FileCheck %s --check-prefix=WRONG
+! RUN: not %flang_fc1 -fsyntax-only -Wall %s  2>&1 | FileCheck %s --check-prefix=WRONG1
+! RUN: not %flang_fc1 -fsyntax-only -WX %s  2>&1 | FileCheck %s --check-prefix=WRONG2
 
-! WRONG: Only `-Werror` is supported currently.
+! WRONG1: error: Unknown diagnostic option: -Wall
+! WRONG2: error: Unknown diagnostic option: -WX
diff --git a/external/llvm-project/flang/test/Driver/wextra-ok.f90 b/external/llvm-project/flang/test/Driver/wextra-ok.f90
index 441029aa0af2..db15c7f14aa3 100644
--- a/external/llvm-project/flang/test/Driver/wextra-ok.f90
+++ b/external/llvm-project/flang/test/Driver/wextra-ok.f90
@@ -5,7 +5,7 @@
 ! RUN: not %flang -std=f2018 -Wblah -Wextra %s -c 2>&1 | FileCheck %s --check-prefix=WRONG
 
 ! CHECK-OK: the warning option '-Wextra' is not supported
-! WRONG: Only `-Werror` is supported currently.
+! WRONG: Unknown diagnostic option: -Wblah
 
 program wextra_ok
 end program wextra_ok
diff --git a/external/llvm-project/flang/test/Examples/omp-atomic.f90 b/external/llvm-project/flang/test/Examples/omp-atomic.f90
index dcca34b633a3..934f84f13248 100644
--- a/external/llvm-project/flang/test/Examples/omp-atomic.f90
+++ b/external/llvm-project/flang/test/Examples/omp-atomic.f90
@@ -26,25 +26,31 @@
 ! CHECK:---
 ! CHECK-NEXT:- file:            '{{[^"]*}}omp-atomic.f90'
 ! CHECK-NEXT:  line:            9
-! CHECK-NEXT:  construct:       atomic-read
+! CHECK-NEXT:  construct:       atomic
 ! CHECK-NEXT:  clauses:
-! CHECK-NEXT:    - clause:      seq_cst
+! CHECK-NEXT:    - clause:      read
 ! CHECK-NEXT:      details:     ''
+! CHECK-NEXT:    - clause:      seq_cst
+! CHECK-NEXT:      details:     'name_modifier=atomic;'
 ! CHECK-NEXT:- file:            '{{[^"]*}}omp-atomic.f90'
 ! CHECK-NEXT:  line:            12
-! CHECK-NEXT:  construct:       atomic-write
+! CHECK-NEXT:  construct:       atomic
 ! CHECK-NEXT:  clauses:
 ! CHECK-NEXT:    - clause:      seq_cst
+! CHECK-NEXT:      details:     'name_modifier=atomic;'
+! CHECK-NEXT:    - clause:      write
 ! CHECK-NEXT:      details:     ''
 ! CHECK-NEXT:- file:            '{{[^"]*}}omp-atomic.f90'
 ! CHECK-NEXT:  line:            16
-! CHECK-NEXT:  construct:       atomic-capture
+! CHECK-NEXT:  construct:       atomic
 ! CHECK-NEXT:  clauses:
+! CHECK-NEXT:    - clause:      capture
+! CHECK-NEXT:      details:     'name_modifier=atomic;name_modifier=atomic;'
 ! CHECK-NEXT:    - clause:      seq_cst
 ! CHECK-NEXT:      details:     ''
 ! CHECK-NEXT:- file:            '{{[^"]*}}omp-atomic.f90'
 ! CHECK-NEXT:  line:            21
-! CHECK-NEXT:  construct:       atomic-atomic
+! CHECK-NEXT:  construct:       atomic
 ! CHECK-NEXT:  clauses:         []
 ! CHECK-NEXT:- file:            '{{[^"]*}}omp-atomic.f90'
 ! CHECK-NEXT:  line:            8
diff --git a/external/llvm-project/flang/test/Fir/FirToSCF/if.fir b/external/llvm-project/flang/test/Fir/FirToSCF/if.fir
new file mode 100644
index 000000000000..9e43cf1cd11d
--- /dev/null
+++ b/external/llvm-project/flang/test/Fir/FirToSCF/if.fir
@@ -0,0 +1,56 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK: func.func @test_only(%[[ARG0:.*]]: i1, %[[ARG1:.*]]: i32) {
+// CHECK:   scf.if %[[ARG0:.*]] {
+// CHECK:    %[[VAL_1:.*]] = arith.addi %[[ARG1:.*]], %[[ARG1:.*]] : i32
+// CHECK:   }
+// CHECK:   return
+// CHECK:   }
+func.func @test_only(%arg0 : i1, %arg1 : i32) {
+  fir.if %arg0 {
+    %0 = arith.addi %arg1, %arg1 : i32
+  }
+  return
+}
+
+// CHECK: func.func @test_else()   {
+// CHECK:   %[[VAL_1:.*]] = arith.constant false
+// CHECK:   %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:   scf.if %[[VAL_1:.*]] {
+// CHECK:     %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:   } else {
+// CHECK:     %[[VAL_3:.*]] = arith.constant 3 : i32
+// CHECK:   }
+// CHECK:   return
+// CHECK: }
+func.func @test_else()   {
+  %false = arith.constant false
+  %1 = arith.constant 2 : i32
+  fir.if %false {
+    %2 = arith.constant 3 : i32
+  } else {
+    %3 = arith.constant 3 : i32
+  }
+  return
+}
+
+// CHECK-LABEL:     func.func @test_two_result()  {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[RES:[0-9]+]]:2 = scf.if %[[VAL_2:.*]] -> (f32, f32) {
+// CHECK:            scf.yield %[[VAL_1:.*]], %[[VAL_1:.*]] : f32, f32
+// CHECK:            } else {
+// CHECK:              scf.yield %[[VAL_1:.*]], %[[VAL_1:.*]] : f32, f32
+// CHECK:            }
+// CHECK:            return 
+// CHECK:            }
+func.func @test_two_result() {
+  %1 = arith.constant 2.0 : f32
+  %cmp = arith.constant false
+  %x, %y = fir.if %cmp -> (f32, f32) {
+    fir.result %1, %1 : f32, f32
+  } else {
+    fir.result %1, %1 : f32, f32
+  }
+  return
+}
diff --git a/external/llvm-project/flang/test/Fir/box-offset-codegen.fir b/external/llvm-project/flang/test/Fir/box-offset-codegen.fir
index 11d5750ffc38..9af92a48959d 100644
--- a/external/llvm-project/flang/test/Fir/box-offset-codegen.fir
+++ b/external/llvm-project/flang/test/Fir/box-offset-codegen.fir
@@ -37,3 +37,13 @@ func.func @array_tdesc(%array : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.t
 // CHECK-SAME: ptr {{[^%]*}}%[[BOX:.*]]){{.*}}{
 // CHECK:    %[[VAL_0:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[BOX]], i32 0, i32 8
 // CHECK:    ret ptr %[[VAL_0]]
+
+func.func @boxchar_addr(%boxchar : !fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>> {
+  %addr = fir.box_offset %boxchar base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
+  return %addr : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
+}
+
+// CHECK-LABEL: define ptr @boxchar_addr(
+// CHECK-SAME: ptr {{.*}}%[[BOXCHAR:.*]]){{.*}} {
+// CHECK: %[[VAL_0:.*]] = getelementptr { ptr, i64 }, ptr %[[BOXCHAR]], i32 0, i32 0
+// CHECK: ret ptr %[[VAL_0]]
diff --git a/external/llvm-project/flang/test/Fir/box-offset.fir b/external/llvm-project/flang/test/Fir/box-offset.fir
index 98c2eaefb8d6..181ad51a5dbe 100644
--- a/external/llvm-project/flang/test/Fir/box-offset.fir
+++ b/external/llvm-project/flang/test/Fir/box-offset.fir
@@ -21,6 +21,9 @@ func.func @test_box_offset(%unlimited : !fir.ref<!fir.class<none>>, %type_star :
 
   %addr6 = fir.box_offset %type_star base_addr : (!fir.ref<!fir.box<!fir.array<?xnone>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xnone>>>
   %tdesc6 = fir.box_offset %type_star derived_type : (!fir.ref<!fir.box<!fir.array<?xnone>>>) -> !fir.llvm_ptr<!fir.tdesc<none>>
+
+  %boxchar = fir.alloca !fir.boxchar<1>
+  %addr7 = fir.box_offset %boxchar base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
   return
 }
 // CHECK-LABEL:   func.func @test_box_offset(
@@ -40,3 +43,5 @@ func.func @test_box_offset(%unlimited : !fir.ref<!fir.class<none>>, %type_star :
 // CHECK:           %[[VAL_13:.*]] = fir.box_offset %[[VAL_0]] derived_type : (!fir.ref<!fir.class<none>>) -> !fir.llvm_ptr<!fir.tdesc<none>>
 // CHECK:           %[[VAL_14:.*]] = fir.box_offset %[[VAL_1]] base_addr : (!fir.ref<!fir.box<!fir.array<?xnone>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xnone>>>
 // CHECK:           %[[VAL_15:.*]] = fir.box_offset %[[VAL_1]] derived_type : (!fir.ref<!fir.box<!fir.array<?xnone>>>) -> !fir.llvm_ptr<!fir.tdesc<none>>
+// CHECK:           %[[VAL_16:.*]] = fir.alloca !fir.boxchar<1>
+// CHECK:           %[[VAL_17:.*]] = fir.box_offset %[[VAL_16]] base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
diff --git a/external/llvm-project/flang/test/Fir/boxproc.fir b/external/llvm-project/flang/test/Fir/boxproc.fir
index 5d82522055ad..97d9b38ed6f4 100644
--- a/external/llvm-project/flang/test/Fir/boxproc.fir
+++ b/external/llvm-project/flang/test/Fir/boxproc.fir
@@ -3,7 +3,7 @@
 // RUN: %if powerpc-registered-target %{tco --target=powerpc64le-unknown-linux-gnu %s | FileCheck %s --check-prefixes=CHECK,CHECK-PPC %}
 
 // CHECK-LABEL: define void @_QPtest_proc_dummy()
-// CHECK-AARCH64: %[[VAL_3:.*]] = alloca [36 x i8], i64 1, align 1
+// CHECK-AARCH64: %[[VAL_3:.*]] = alloca [32 x i8], i64 1, align 1
 // CHECK-X86:     %[[VAL_3:.*]] = alloca [32 x i8], i64 1, align 1
 // CHECK-PPC:     %[[VAL_3:.*]] = alloca [4{{[0-8]+}} x i8], i64 1, align 1
 // CHECK:         %[[VAL_1:.*]] = alloca { ptr }, i64 1, align 8
@@ -63,7 +63,7 @@ func.func @_QPtest_proc_dummy_other(%arg0: !fir.boxproc<() -> ()>) {
 }
 
 // CHECK-LABEL: define void @_QPtest_proc_dummy_char()
-// CHECK-AARCH64: %[[VAL_20:.*]] = alloca [36 x i8], i64 1, align 1
+// CHECK-AARCH64: %[[VAL_20:.*]] = alloca [32 x i8], i64 1, align 1
 // CHECK-X86:     %[[VAL_20:.*]] = alloca [32 x i8], i64 1, align 1
 // CHECK-PPC:     %[[VAL_20:.*]] = alloca [4{{[0-8]+}} x i8], i64 1, align 1
 // CHECK:         %[[VAL_2:.*]] = alloca { { ptr, i64 } }, i64 1, align 8
diff --git a/external/llvm-project/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/external/llvm-project/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index b13921f822b4..24e5cad84b70 100644
--- a/external/llvm-project/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/external/llvm-project/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -1288,3 +1288,30 @@ omp.declare_mapper @my_mapper : !fir.type<_QFdeclare_mapperTmy_type{data:i32}> {
   omp.declare_mapper.info map_entries(%4, %3 : !fir.ref<!fir.type<_QFdeclare_mapperTmy_type{data:i32}>>, !fir.ref<i32>)
 // CHECK:         }
 }
+
+// -----
+omp.private {type = firstprivate} @boxchar_privatizer : !fir.boxchar<1> copy {
+  ^bb0(%arg0: !fir.boxchar<1>, %arg1: !fir.boxchar<1>):
+    omp.yield(%arg1 : !fir.boxchar<1>)
+}
+
+func.func @map_privatized_boxchar(%arg0 : !fir.boxchar<1>) {
+    %0 = fir.alloca !fir.boxchar<1>
+    fir.store %arg0 to %0 : !fir.ref<!fir.boxchar<1>>
+    %7 = fir.box_offset %0 base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
+    %8 = omp.map.info var_ptr(%0 : !fir.ref<!fir.boxchar<1>>, !fir.char<1,?>) map_clauses(implicit, to) capture(ByRef) var_ptr_ptr(%7 : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>) -> !fir.ref<!fir.boxchar<1>>
+    %9 = omp.map.info var_ptr(%0 : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(to) capture(ByRef) members(%8 : [0] : !fir.ref<!fir.boxchar<1>>) -> !fir.ref<!fir.boxchar<1>>
+    omp.target map_entries(%9 -> %arg1, %8 -> %arg2 : !fir.ref<!fir.boxchar<1>>, !fir.ref<!fir.boxchar<1>>) private(@boxchar_privatizer %arg0 -> %arg3 [map_idx=0] : !fir.boxchar<1>) {
+      omp.terminator
+    }
+    return
+}
+
+// CHECK-LABEL: llvm.func @map_privatized_boxchar(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.struct<(ptr, i64)>) {
+// CHECK: %[[BOXCHAR_ALLOCA:.*]] = llvm.alloca {{.*}} x !llvm.struct<(ptr, i64)> : (i64) -> !llvm.ptr
+// CHECK: llvm.store %[[ARG0]], %[[BOXCHAR_ALLOCA]] : !llvm.struct<(ptr, i64)>, !llvm.ptr
+// CHECK: %[[BASE_ADDR:.*]] = llvm.getelementptr %[[BOXCHAR_ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64)>
+// CHECK: %[[MAP_BASE_ADDR:.*]] = omp.map.info var_ptr(%[[BOXCHAR_ALLOCA]] : !llvm.ptr, i8) map_clauses(implicit, to) capture(ByRef) var_ptr_ptr(%[[BASE_ADDR]] : !llvm.ptr) -> !llvm.ptr
+// CHECK: %[[MAP_BOXCHAR:.*]] = omp.map.info var_ptr(%[[BOXCHAR_ALLOCA]] : !llvm.ptr, !llvm.struct<(ptr, i64)>) map_clauses(to) capture(ByRef) members(%[[MAP_BASE_ADDR]] : [0] : !llvm.ptr) -> !llvm.ptr
+// CHECK: omp.target map_entries(%[[MAP_BOXCHAR]] -> %arg1, %[[MAP_BASE_ADDR]] -> %arg2 : !llvm.ptr, !llvm.ptr) private(@boxchar_privatizer %[[ARG0]] -> %arg3 [map_idx=0] : !llvm.struct<(ptr, i64)>) {
diff --git a/external/llvm-project/flang/test/Fir/invalid.fir b/external/llvm-project/flang/test/Fir/invalid.fir
index fd607fd9066f..45cae1f82cb8 100644
--- a/external/llvm-project/flang/test/Fir/invalid.fir
+++ b/external/llvm-project/flang/test/Fir/invalid.fir
@@ -972,13 +972,21 @@ func.func @rec_to_rec(%arg0: !fir.type<t1{i:i32, f:f32}>) -> !fir.type<t2{f:f32,
 // -----
 
 func.func @bad_box_offset(%not_a_box : !fir.ref<i32>) {
-  // expected-error@+1{{'fir.box_offset' op box_ref operand must have !fir.ref<!fir.box<T>> type}}
+  // expected-error@+1{{'fir.box_offset' op box_ref operand must have !fir.ref<!fir.box<T>> or !fir.ref<!fir.boxchar<k>> type}}
   %addr1 = fir.box_offset %not_a_box base_addr : (!fir.ref<i32>) -> !fir.llvm_ptr<!fir.ref<i32>>
   return
 }
 
 // -----
 
+func.func @bad_box_offset(%boxchar : !fir.ref<!fir.boxchar<1>>) {
+  // expected-error@+1{{'fir.box_offset' op cannot address derived_type field of a fir.boxchar}}
+  %addr1 = fir.box_offset %boxchar derived_type : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
+  return
+}
+
+// -----
+
 func.func @bad_box_offset(%no_addendum : !fir.ref<!fir.box<i32>>) {
   // expected-error@+1{{'fir.box_offset' op can only address derived_type field of derived type or unlimited polymorphic fir.box}}
   %addr1 = fir.box_offset %no_addendum derived_type : (!fir.ref<!fir.box<i32>>) -> !fir.llvm_ptr<!fir.tdesc<!fir.type<none>>>
diff --git a/external/llvm-project/flang/test/Fir/local.fir b/external/llvm-project/flang/test/Fir/local.fir
new file mode 100644
index 000000000000..006f5ca94467
--- /dev/null
+++ b/external/llvm-project/flang/test/Fir/local.fir
@@ -0,0 +1,10 @@
+// RUN: fir-opt --fir-to-llvm-ir %s | FileCheck %s
+
+// Tests that `fir.local` ops are dropped from the module before LLVM lowering.
+
+fir.local {type = local} @local_privatizer : i32
+func.func @foo() {
+  return
+}
+
+// CHECK-NOT: fir.local
diff --git a/external/llvm-project/flang/test/Fir/tbaa-codegen2.fir b/external/llvm-project/flang/test/Fir/tbaa-codegen2.fir
index 69b36c261150..4907aa03ec5a 100644
--- a/external/llvm-project/flang/test/Fir/tbaa-codegen2.fir
+++ b/external/llvm-project/flang/test/Fir/tbaa-codegen2.fir
@@ -100,7 +100,7 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.targ
 // [...]
 // CHECK:  %[[VAL50:.*]] = getelementptr i32, ptr %{{.*}}, i64 %{{.*}}
 // store to the temporary:
-// CHECK:  store i32 %{{.*}}, ptr %[[VAL50]], align 4, !tbaa ![[DATA_ACCESS_TAG:.*]]
+// CHECK:  store i32 %{{.*}}, ptr %[[VAL50]], align 4, !tbaa ![[TMP_DATA_ACCESS_TAG:.*]]
 // [...]
 
 // CHECK: [[BOX_ACCESS_TAG]] = !{![[BOX_ACCESS_TYPE:.*]], ![[BOX_ACCESS_TYPE]], i64 0}
@@ -111,4 +111,7 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.targ
 // CHECK: ![[A_ACCESS_TYPE]] = !{!"dummy arg data/_QFfuncEa", ![[ARG_ACCESS_TYPE:.*]], i64 0}
 // CHECK: ![[ARG_ACCESS_TYPE]] = !{!"dummy arg data", ![[DATA_ACCESS_TYPE:.*]], i64 0}
 // CHECK: ![[DATA_ACCESS_TYPE]] = !{!"any data access", ![[ANY_ACCESS_TYPE]], i64 0}
-// CHECK: ![[DATA_ACCESS_TAG]] = !{![[DATA_ACCESS_TYPE]], ![[DATA_ACCESS_TYPE]], i64 0}
+// CHECK: ![[TMP_DATA_ACCESS_TAG]] = !{![[TMP_DATA_ACCESS_TYPE:.*]], ![[TMP_DATA_ACCESS_TYPE]], i64 0}
+// CHECK: ![[TMP_DATA_ACCESS_TYPE]] = !{!"allocated data/", ![[TMP_ACCESS_TYPE:.*]], i64 0}
+// CHECK: ![[TMP_ACCESS_TYPE]] = !{!"allocated data", ![[TARGET_ACCESS_TAG:.*]], i64 0}
+// CHECK: ![[TARGET_ACCESS_TAG]] = !{!"target data", ![[DATA_ACCESS_TYPE]], i64 0}
diff --git a/external/llvm-project/flang/test/HLFIR/as_expr-codegen-polymorphic.fir b/external/llvm-project/flang/test/HLFIR/as_expr-codegen-polymorphic.fir
index 30a8da72c3e4..31301697ad22 100644
--- a/external/llvm-project/flang/test/HLFIR/as_expr-codegen-polymorphic.fir
+++ b/external/llvm-project/flang/test/HLFIR/as_expr-codegen-polymorphic.fir
@@ -9,14 +9,13 @@ func.func @as_expr_class(%arg0 : !fir.class<!t>, %arg1: !fir.ref<!t>) {
    return
 }
 // CHECK-LABEL:   func.func @as_expr_class(
+// CHECK:           %[[VAL_6:.*]]:2 =  hlfir.declare %{{.*}} {uniq_name = ".tmp"} : (!fir.class<!fir.heap<!fir.type<t{i:i32}>>>) -> (!fir.class<!fir.type<t{i:i32}>>, !fir.class<!fir.type<t{i:i32}>>)
 // CHECK:           %[[VAL_5:.*]] = arith.constant true
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".tmp"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>>)
 // ... copy ...
-// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>>
-// CHECK:           %[[VAL_12:.*]] = fir.undefined tuple<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>, i1>
-// CHECK:           %[[VAL_13:.*]] = fir.insert_value %[[VAL_12]], %[[VAL_5]], [1 : index] : (tuple<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>, i1>, i1) -> tuple<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>, i1>
-// CHECK:           %[[VAL_14:.*]] = fir.insert_value %[[VAL_13]], %[[VAL_11]], [0 : index] : (tuple<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>, i1>, !fir.class<!fir.heap<!fir.type<t{i:i32}>>>) -> tuple<!fir.class<!fir.heap<!fir.type<t{i:i32}>>>, i1>
-// CHECK:           hlfir.assign %[[VAL_11]] to %{{.*}} : !fir.class<!fir.heap<!fir.type<t{i:i32}>>>, !fir.ref<!fir.type<t{i:i32}>>
+// CHECK:           %[[VAL_12:.*]] = fir.undefined tuple<!fir.class<!fir.type<t{i:i32}>>, i1>
+// CHECK:           %[[VAL_13:.*]] = fir.insert_value %[[VAL_12]], %[[VAL_5]], [1 : index] : (tuple<!fir.class<!fir.type<t{i:i32}>>, i1>, i1) -> tuple<!fir.class<!fir.type<t{i:i32}>>, i1>
+// CHECK:           %[[VAL_14:.*]] = fir.insert_value %[[VAL_13]], %[[VAL_6]]#0, [0 : index] : (tuple<!fir.class<!fir.type<t{i:i32}>>, i1>, !fir.class<!fir.type<t{i:i32}>>) -> tuple<!fir.class<!fir.type<t{i:i32}>>, i1>
+// CHECK:           hlfir.assign %[[VAL_6]]#0 to %{{.*}} : !fir.class<!fir.type<t{i:i32}>>, !fir.ref<!fir.type<t{i:i32}>>
 
 
 func.func @as_expr_class_2(%arg0 : !fir.class<!fir.array<?x!t>>) {
@@ -26,13 +25,11 @@ func.func @as_expr_class_2(%arg0 : !fir.class<!fir.array<?x!t>>) {
    return
 }
 // CHECK-LABEL:   func.func @as_expr_class_2(
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = ".tmp"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>) -> (!fir.class<!fir.array<?x!fir.type<t{i:i32}>>>, !fir.class<!fir.array<?x!fir.type<t{i:i32}>>>)
 // CHECK:           %[[VAL_9:.*]] = arith.constant true
-// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".tmp"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>>)
-// CHECK:           %[[VAL_11:.*]] = arith.constant 1 : i32
 // ... copy ...
-// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>>
-// CHECK:           %[[VAL_16:.*]] = fir.undefined tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>, i1>
-// CHECK:           %[[VAL_17:.*]] = fir.insert_value %[[VAL_16]], %[[VAL_9]], [1 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>, i1>, i1) -> tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>, i1>
-// CHECK:           %[[VAL_18:.*]] = fir.insert_value %[[VAL_17]], %[[VAL_15]], [0 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>, i1>, !fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>) -> tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>, i1>
+// CHECK:           %[[VAL_16:.*]] = fir.undefined tuple<!fir.class<!fir.array<?x!fir.type<t{i:i32}>>>, i1>
+// CHECK:           %[[VAL_17:.*]] = fir.insert_value %[[VAL_16]], %[[VAL_9]], [1 : index] : (tuple<!fir.class<!fir.array<?x!fir.type<t{i:i32}>>>, i1>, i1) -> tuple<!fir.class<!fir.array<?x!fir.type<t{i:i32}>>>, i1>
+// CHECK:           %[[VAL_18:.*]] = fir.insert_value %[[VAL_17]], %[[VAL_10]]#0, [0 : index] : (tuple<!fir.class<!fir.array<?x!fir.type<t{i:i32}>>>, i1>, !fir.class<!fir.array<?x!fir.type<t{i:i32}>>>) -> tuple<!fir.class<!fir.array<?x!fir.type<t{i:i32}>>>, i1>
 // CHECK:           %[[VAL_19:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_20:.*]] = hlfir.designate %[[VAL_15]] (%[[VAL_19]])  : (!fir.class<!fir.heap<!fir.array<?x!fir.type<t{i:i32}>>>>, index) -> !fir.class<!fir.type<t{i:i32}>>
+// CHECK:           %[[VAL_20:.*]] = hlfir.designate %[[VAL_10]]#0 (%[[VAL_19]])  : (!fir.class<!fir.array<?x!fir.type<t{i:i32}>>>, index) -> !fir.class<!fir.type<t{i:i32}>>
diff --git a/external/llvm-project/flang/test/HLFIR/as_expr-codegen.fir b/external/llvm-project/flang/test/HLFIR/as_expr-codegen.fir
index 7123e499e9f7..e4dd2a9a3fc3 100644
--- a/external/llvm-project/flang/test/HLFIR/as_expr-codegen.fir
+++ b/external/llvm-project/flang/test/HLFIR/as_expr-codegen.fir
@@ -12,8 +12,8 @@ func.func @char_expr(%addr: !fir.ref<!fir.char<1,?>>, %len: index) {
 // CHECK-SAME:    %[[VAL_1:.*]]: index) {
 // CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_1]] {uniq_name = "c"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 // CHECK:  %[[VAL_3:.*]] = fir.alloca !fir.char<1,?>(%[[VAL_1]] : index) {bindc_name = ".tmp"}
-// CHECK:  %[[VAL_5:.*]] = arith.constant false
 // CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] typeparams %[[VAL_1]] {uniq_name = ".tmp"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+// CHECK:  %[[VAL_5:.*]] = arith.constant false
 // CHECK:  hlfir.assign %[[VAL_2]]#0 to %[[VAL_4]]#0 temporary_lhs : !fir.boxchar<1>, !fir.boxchar<1>
 // CHECK:  %[[VAL_6:.*]] = fir.undefined tuple<!fir.boxchar<1>, i1>
 // CHECK:  %[[VAL_7:.*]] = fir.insert_value %[[VAL_6]], %[[VAL_5]], [1 : index] : (tuple<!fir.boxchar<1>, i1>, i1) -> tuple<!fir.boxchar<1>, i1>
@@ -29,8 +29,8 @@ func.func @char_expr_2(%addr: !fir.ref<!fir.char<1,10>>, %len: index) {
 // CHECK-SAME:    %[[VAL_1:.*]]: index) {
 // CHECK:  %[[VAL_2:.*]] = fir.alloca !fir.char<1,10> {bindc_name = ".tmp"}
 // CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_1]] {uniq_name = "c"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
-// CHECK:  %[[VAL_5:.*]] = arith.constant false
 // CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[VAL_1]] {uniq_name = ".tmp"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
+// CHECK:  %[[VAL_5:.*]] = arith.constant false
 // CHECK:  hlfir.assign %[[VAL_3]]#0 to %[[VAL_4]]#0 temporary_lhs : !fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>
 // CHECK:  %[[VAL_6:.*]] = fir.undefined tuple<!fir.ref<!fir.char<1,10>>, i1>
 // CHECK:  %[[VAL_7:.*]] = fir.insert_value %[[VAL_6]], %[[VAL_5]], [1 : index] : (tuple<!fir.ref<!fir.char<1,10>>, i1>, i1) -> tuple<!fir.ref<!fir.char<1,10>>, i1>
@@ -46,8 +46,8 @@ func.func @shape_from_type(%arg0 : !fir.ref<!fir.array<10x20xi32>>) {
 // CHECK:    %[[VAL_2:.*]] = arith.constant 20 : index
 // CHECK:    %[[VAL_3:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]] : (index, index) -> !fir.shape<2>
 // CHECK:    %[[VAL_4:.*]] = fir.allocmem !fir.array<10x20xi32> {bindc_name = ".tmp", uniq_name = ""}
-// CHECK:    %[[VAL_5:.*]] = arith.constant true
 // CHECK:    %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_3]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<10x20xi32>>, !fir.shape<2>) -> (!fir.heap<!fir.array<10x20xi32>>, !fir.heap<!fir.array<10x20xi32>>)
+// CHECK:    %[[VAL_5:.*]] = arith.constant true
 // CHECK:    hlfir.assign %[[VAL_0]] to %[[VAL_6]]#0 temporary_lhs : !fir.ref<!fir.array<10x20xi32>>, !fir.heap<!fir.array<10x20xi32>>
 // CHECK:    %[[VAL_7:.*]] = fir.undefined tuple<!fir.heap<!fir.array<10x20xi32>>, i1>
 // CHECK:    %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_5]], [1 : index] : (tuple<!fir.heap<!fir.array<10x20xi32>>, i1>, i1) -> tuple<!fir.heap<!fir.array<10x20xi32>>, i1>
@@ -65,8 +65,8 @@ func.func @shape_from_box(%arg0 : !fir.box<!fir.array<10x?xi32>>) {
 // CHECK:    %[[VAL_3:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<10x?xi32>>, index) -> (index, index, index)
 // CHECK:    %[[VAL_4:.*]] = fir.shape %[[VAL_1]], %[[VAL_3]]#1 : (index, index) -> !fir.shape<2>
 // CHECK:    %[[VAL_5:.*]] = fir.allocmem !fir.array<10x?xi32>, %[[VAL_3]]#1 {bindc_name = ".tmp", uniq_name = ""}
-// CHECK:    %[[VAL_6:.*]] = arith.constant true
 // CHECK:    %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_4]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<10x?xi32>>, !fir.shape<2>) -> (!fir.box<!fir.array<10x?xi32>>, !fir.heap<!fir.array<10x?xi32>>)
+// CHECK:    %[[VAL_6:.*]] = arith.constant true
 // CHECK:    hlfir.assign %[[VAL_0]] to %[[VAL_7]]#0 temporary_lhs : !fir.box<!fir.array<10x?xi32>>, !fir.box<!fir.array<10x?xi32>>
 // CHECK:    %[[VAL_8:.*]] = fir.undefined tuple<!fir.box<!fir.array<10x?xi32>>, i1>
 // CHECK:    %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_6]], [1 : index] : (tuple<!fir.box<!fir.array<10x?xi32>>, i1>, i1) -> tuple<!fir.box<!fir.array<10x?xi32>>, i1>
diff --git a/external/llvm-project/flang/test/HLFIR/assign-side-effects.fir b/external/llvm-project/flang/test/HLFIR/assign-side-effects.fir
index dfd1c5886e4f..cac9530e2277 100644
--- a/external/llvm-project/flang/test/HLFIR/assign-side-effects.fir
+++ b/external/llvm-project/flang/test/HLFIR/assign-side-effects.fir
@@ -2,14 +2,14 @@
 // RUN: fir-opt %s --test-side-effects --verify-diagnostics
 
 func.func @test1(%x: !fir.ref<i32>, %i: i32) {
-  // expected-remark @below {{found an instance of 'write' on a op operand, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'write' on op operand 1, on resource '<Default>'}}
   hlfir.assign %i to %x : i32, !fir.ref<i32>
   return
 }
 
 func.func @test2(%x: !fir.ref<i32>, %y: !fir.ref<i32>) {
-  // expected-remark @below {{found an instance of 'write' on a op operand, on resource '<Default>'}}
-  // expected-remark @below {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'write' on op operand 1, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   hlfir.assign %y to %x : !fir.ref<i32>, !fir.ref<i32>
   return
 }
@@ -22,7 +22,8 @@ func.func @test3(%x: !fir.ref<!fir.type<t>>, %y: !fir.ref<!fir.type<t>>) {
 }
 
 func.func @test4(%x: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, %y: !fir.box<!fir.array<?xi32>>) {
-  // expected-remark @below {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'read' on op operand 1, on resource '<Default>'}}
   // expected-remark @below {{found an instance of 'write' on resource '<Default>'}}
   // expected-remark @below {{found an instance of 'free' on resource '<Default>'}}
   // expected-remark @below {{found an instance of 'allocate' on resource '<Default>'}}
diff --git a/external/llvm-project/flang/test/HLFIR/associate-codegen.fir b/external/llvm-project/flang/test/HLFIR/associate-codegen.fir
index f7ee4fa83c9f..5d0d67854365 100644
--- a/external/llvm-project/flang/test/HLFIR/associate-codegen.fir
+++ b/external/llvm-project/flang/test/HLFIR/associate-codegen.fir
@@ -174,22 +174,22 @@ func.func @test_result_convert(%x : !fir.heap<!fir.array<10xi32>>) {
 
 func.func @test_0dim_box(%x : !fir.ref<!fir.box<!fir.heap<i32>>>) {
   %0 = fir.load %x : !fir.ref<!fir.box<!fir.heap<i32>>>
-  %1:2 = hlfir.declare %0 {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<i32>>) -> (!fir.box<!fir.heap<i32>>, !fir.box<!fir.heap<i32>>)
+  %1:2 = hlfir.declare %0 {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<i32>>) -> (!fir.box<i32>, !fir.box<i32>)
   %true = arith.constant true
-  %2 = hlfir.as_expr %1#0 move %true : (!fir.box<!fir.heap<i32>>, i1) -> !hlfir.expr<i32>
+  %2 = hlfir.as_expr %1#0 move %true : (!fir.box<i32>, i1) -> !hlfir.expr<i32>
   %3:3 = hlfir.associate %2 {adapt.valuebyref} : (!hlfir.expr<i32>) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
   return
 }
 // CHECK-LABEL:   func.func @test_0dim_box(
 // CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<i32>>>) {
 // CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<i32>>>
-// CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<i32>>) -> (!fir.box<!fir.heap<i32>>, !fir.box<!fir.heap<i32>>)
+// CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<i32>>) -> (!fir.box<i32>, !fir.box<i32>)
 // CHECK:           %[[VAL_3:.*]] = arith.constant true
-// CHECK:           %[[VAL_4:.*]] = fir.undefined tuple<!fir.box<!fir.heap<i32>>, i1>
-// CHECK:           %[[VAL_5:.*]] = fir.insert_value %[[VAL_4]], %[[VAL_3]], [1 : index] : (tuple<!fir.box<!fir.heap<i32>>, i1>, i1) -> tuple<!fir.box<!fir.heap<i32>>, i1>
-// CHECK:           %[[VAL_6:.*]] = fir.insert_value %[[VAL_5]], %[[VAL_2]]#0, [0 : index] : (tuple<!fir.box<!fir.heap<i32>>, i1>, !fir.box<!fir.heap<i32>>) -> tuple<!fir.box<!fir.heap<i32>>, i1>
-// CHECK:           %[[VAL_7:.*]] = fir.box_addr %[[VAL_2]]#0 : (!fir.box<!fir.heap<i32>>) -> !fir.ref<i32>
-// CHECK:           %[[VAL_8:.*]] = fir.box_addr %[[VAL_2]]#1 : (!fir.box<!fir.heap<i32>>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_4:.*]] = fir.undefined tuple<!fir.box<i32>, i1>
+// CHECK:           %[[VAL_5:.*]] = fir.insert_value %[[VAL_4]], %[[VAL_3]], [1 : index] : (tuple<!fir.box<i32>, i1>, i1) -> tuple<!fir.box<i32>, i1>
+// CHECK:           %[[VAL_6:.*]] = fir.insert_value %[[VAL_5]], %[[VAL_2]]#0, [0 : index] : (tuple<!fir.box<i32>, i1>, !fir.box<i32>) -> tuple<!fir.box<i32>, i1>
+// CHECK:           %[[VAL_7:.*]] = fir.box_addr %[[VAL_2]]#0 : (!fir.box<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.box_addr %[[VAL_2]]#1 : (!fir.box<i32>) -> !fir.ref<i32>
 // CHECK:           return
 // CHECK:         }
 
@@ -258,8 +258,8 @@ func.func @test_multiple_associations(%arg0: !hlfir.expr<1x2xi32>) {
 // CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_6:.*]] = arith.constant 2 : index
 // CHECK:           %[[VAL_7:.*]] = fir.allocmem !fir.array<1x2xi32> {bindc_name = ".tmp", uniq_name = ""}
-// CHECK:           %[[VAL_8:.*]] = arith.constant true
 // CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_4]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<1x2xi32>>, !fir.shape<2>) -> (!fir.heap<!fir.array<1x2xi32>>, !fir.heap<!fir.array<1x2xi32>>)
+// CHECK:           %[[VAL_8:.*]] = arith.constant true
 // CHECK:           hlfir.assign %[[VAL_0]] to %[[VAL_9]]#0 temporary_lhs : !hlfir.expr<1x2xi32>, !fir.heap<!fir.array<1x2xi32>>
 // CHECK:           %[[VAL_10:.*]] = fir.undefined tuple<!fir.heap<!fir.array<1x2xi32>>, i1>
 // CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [1 : index] : (tuple<!fir.heap<!fir.array<1x2xi32>>, i1>, i1) -> tuple<!fir.heap<!fir.array<1x2xi32>>, i1>
@@ -271,8 +271,8 @@ func.func @test_multiple_associations(%arg0: !hlfir.expr<1x2xi32>) {
 // CHECK:           %[[VAL_16:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_17:.*]] = arith.constant 2 : index
 // CHECK:           %[[VAL_18:.*]] = fir.allocmem !fir.array<1x2xi32> {bindc_name = ".tmp", uniq_name = ""}
-// CHECK:           %[[VAL_19:.*]] = arith.constant true
 // CHECK:           %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_18]](%[[VAL_15]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<1x2xi32>>, !fir.shape<2>) -> (!fir.heap<!fir.array<1x2xi32>>, !fir.heap<!fir.array<1x2xi32>>)
+// CHECK:           %[[VAL_19:.*]] = arith.constant true
 // CHECK:           hlfir.assign %[[VAL_0]] to %[[VAL_20]]#0 temporary_lhs : !hlfir.expr<1x2xi32>, !fir.heap<!fir.array<1x2xi32>>
 // CHECK:           %[[VAL_21:.*]] = fir.undefined tuple<!fir.heap<!fir.array<1x2xi32>>, i1>
 // CHECK:           %[[VAL_22:.*]] = fir.insert_value %[[VAL_21]], %[[VAL_19]], [1 : index] : (tuple<!fir.heap<!fir.array<1x2xi32>>, i1>, i1) -> tuple<!fir.heap<!fir.array<1x2xi32>>, i1>
@@ -302,8 +302,8 @@ func.func @test_get_length(%arg0: !fir.ref<!fir.char<1,2>>) {
 // CHECK-SAME:                               %[[VAL_0:.*]]: !fir.ref<!fir.char<1,2>>) {
 // CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.char<1,2> {bindc_name = ".tmp"}
 // CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant false
 // CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] typeparams %[[VAL_2]] {uniq_name = ".tmp"} : (!fir.ref<!fir.char<1,2>>, index) -> (!fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>)
+// CHECK:           %[[VAL_3:.*]] = arith.constant false
 // CHECK:           hlfir.assign %[[VAL_0]] to %[[VAL_4]]#0 temporary_lhs : !fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>
 // CHECK:           %[[VAL_5:.*]] = fir.undefined tuple<!fir.ref<!fir.char<1,2>>, i1>
 // CHECK:           %[[VAL_6:.*]] = fir.insert_value %[[VAL_5]], %[[VAL_3]], [1 : index] : (tuple<!fir.ref<!fir.char<1,2>>, i1>, i1) -> tuple<!fir.ref<!fir.char<1,2>>, i1>
@@ -354,8 +354,8 @@ func.func @_QPtest_multiple_expr_uses_inside_elemental() {
 // CHECK:           %[[VAL_14:.*]] = arith.constant 1 : index
 // CHECK:           fir.do_loop %[[VAL_15:.*]] = %[[VAL_14]] to %[[VAL_9]] step %[[VAL_14]] unordered {
 // CHECK:             %[[VAL_16:.*]] = fir.alloca !fir.char<1,?>(%[[VAL_4]] : index) {bindc_name = ".tmp"}
-// CHECK:             %[[VAL_17:.*]] = arith.constant false
 // CHECK:             %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_16]] typeparams %[[VAL_4]] {uniq_name = ".tmp"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+// CHECK:             %[[VAL_17:.*]] = arith.constant false
 // CHECK:             hlfir.assign %[[VAL_5]]#0 to %[[VAL_18]]#0 temporary_lhs : !fir.boxchar<1>, !fir.boxchar<1>
 // CHECK:             %[[VAL_19:.*]] = fir.undefined tuple<!fir.boxchar<1>, i1>
 // CHECK:             %[[VAL_20:.*]] = fir.insert_value %[[VAL_19]], %[[VAL_17]], [1 : index] : (tuple<!fir.boxchar<1>, i1>, i1) -> tuple<!fir.boxchar<1>, i1>
@@ -406,8 +406,8 @@ func.func @_QPtest_multitple_associates_for_same_expr() {
 // CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_5]], [1 : index] : (tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>, i1) -> tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>
 // CHECK:           %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_4]]#0, [0 : index] : (tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>, !fir.heap<!fir.array<10x!fir.char<1>>>) -> tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>
 // CHECK:           %[[VAL_13:.*]] = fir.allocmem !fir.array<10x!fir.char<1>> {bindc_name = ".tmp", uniq_name = ""}
-// CHECK:           %[[VAL_14:.*]] = arith.constant true
 // CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]](%[[VAL_2]]) typeparams %[[VAL_0]] {uniq_name = ".tmp"} : (!fir.heap<!fir.array<10x!fir.char<1>>>, !fir.shape<1>, index) -> (!fir.heap<!fir.array<10x!fir.char<1>>>, !fir.heap<!fir.array<10x!fir.char<1>>>)
+// CHECK:           %[[VAL_14:.*]] = arith.constant true
 // CHECK:           hlfir.assign %[[VAL_4]]#0 to %[[VAL_15]]#0 temporary_lhs : !fir.heap<!fir.array<10x!fir.char<1>>>, !fir.heap<!fir.array<10x!fir.char<1>>>
 // CHECK:           %[[VAL_16:.*]] = fir.undefined tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>
 // CHECK:           %[[VAL_17:.*]] = fir.insert_value %[[VAL_16]], %[[VAL_14]], [1 : index] : (tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>, i1) -> tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>
@@ -417,8 +417,8 @@ func.func @_QPtest_multitple_associates_for_same_expr() {
 // CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.array<10x!fir.char<1>>>) -> !fir.heap<!fir.array<10x!fir.char<1>>>
 // CHECK:           fir.freemem %[[VAL_21]] : !fir.heap<!fir.array<10x!fir.char<1>>>
 // CHECK:           %[[VAL_22:.*]] = fir.allocmem !fir.array<10x!fir.char<1>> {bindc_name = ".tmp", uniq_name = ""}
-// CHECK:           %[[VAL_23:.*]] = arith.constant true
 // CHECK:           %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_22]](%[[VAL_2]]) typeparams %[[VAL_0]] {uniq_name = ".tmp"} : (!fir.heap<!fir.array<10x!fir.char<1>>>, !fir.shape<1>, index) -> (!fir.heap<!fir.array<10x!fir.char<1>>>, !fir.heap<!fir.array<10x!fir.char<1>>>)
+// CHECK:           %[[VAL_23:.*]] = arith.constant true
 // CHECK:           hlfir.assign %[[VAL_4]]#0 to %[[VAL_24]]#0 temporary_lhs : !fir.heap<!fir.array<10x!fir.char<1>>>, !fir.heap<!fir.array<10x!fir.char<1>>>
 // CHECK:           %[[VAL_25:.*]] = fir.undefined tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>
 // CHECK:           %[[VAL_26:.*]] = fir.insert_value %[[VAL_25]], %[[VAL_23]], [1 : index] : (tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>, i1) -> tuple<!fir.heap<!fir.array<10x!fir.char<1>>>, i1>
diff --git a/external/llvm-project/flang/test/HLFIR/bufferize-poly-expr.fir b/external/llvm-project/flang/test/HLFIR/bufferize-poly-expr.fir
index 49c2347b2b26..54cdfa38f81a 100644
--- a/external/llvm-project/flang/test/HLFIR/bufferize-poly-expr.fir
+++ b/external/llvm-project/flang/test/HLFIR/bufferize-poly-expr.fir
@@ -4,8 +4,8 @@ func.func @test_poly_expr_without_associate() {
   %5 = fir.alloca !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>> {bindc_name = "r", uniq_name = "_QFtestEr"}
   %8:2 = hlfir.declare %5 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtestEr"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>)
   %26 = fir.undefined !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>
-  %27:2 = hlfir.declare %26 {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>)
-  %28 = hlfir.as_expr %27#0 : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> !hlfir.expr<!fir.type<_QFtestTt{c:i32}>?>
+  %27:2 = hlfir.declare %26 {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> (!fir.class<!fir.type<_QFtestTt{c:i32}>>, !fir.class<!fir.type<_QFtestTt{c:i32}>>)
+  %28 = hlfir.as_expr %27#0 : (!fir.class<!fir.type<_QFtestTt{c:i32}>>) -> !hlfir.expr<!fir.type<_QFtestTt{c:i32}>?>
   hlfir.assign %28 to %8#0 realloc : !hlfir.expr<!fir.type<_QFtestTt{c:i32}>?>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>
   hlfir.destroy %28 : !hlfir.expr<!fir.type<_QFtestTt{c:i32}>?>
   return
@@ -15,24 +15,28 @@ func.func @test_poly_expr_without_associate() {
 // CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>> {bindc_name = "r", uniq_name = "_QFtestEr"}
 // CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtestEr"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>)
 // CHECK:           %[[VAL_3:.*]] = fir.undefined !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>
-// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>)
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> (!fir.class<!fir.type<_QFtestTt{c:i32}>>, !fir.class<!fir.type<_QFtestTt{c:i32}>>)
 // CHECK:           %[[VAL_5:.*]] = fir.zero_bits !fir.heap<!fir.type<_QFtestTt{c:i32}>>
-// CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]] : (!fir.heap<!fir.type<_QFtestTt{c:i32}>>) -> !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>
+// CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]] source_box %[[VAL_4]]#0 : (!fir.heap<!fir.type<_QFtestTt{c:i32}>>, !fir.class<!fir.type<_QFtestTt{c:i32}>>) -> !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>
 // CHECK:           fir.store %[[VAL_6]] to %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>
-// CHECK:           %[[VAL_7:.*]] = arith.constant true
-// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".tmp"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>)
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_4]]#1 : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> !fir.box<none>
-// CHECK:           fir.call @_FortranAAllocatableApplyMold(%[[VAL_10]], %[[VAL_11]], %[[VAL_9]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:           hlfir.assign %[[VAL_4]]#0 to %[[VAL_8]]#0 realloc temporary_lhs : !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>
-// CHECK:           %[[VAL_8B:.*]] = fir.load %[[VAL_8]]#0
-// CHECK:           %[[VAL_13:.*]] = fir.undefined tuple<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, i1>
-// CHECK:           %[[VAL_14:.*]] = fir.insert_value %[[VAL_13]], %[[VAL_7]], [1 : index] : (tuple<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, i1>, i1) -> tuple<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, i1>
-// CHECK:           %[[VAL_15:.*]] = fir.insert_value %[[VAL_14]], %[[VAL_8B]], [0 : index] : (tuple<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, i1>, !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> tuple<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, i1>
-// CHECK:           hlfir.assign %[[VAL_8B]] to %[[VAL_2]]#0 realloc : !fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>
-// CHECK:           %[[VAL_17:.*]] = fir.box_addr %[[VAL_8B]] : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> !fir.heap<!fir.type<_QFtestTt{c:i32}>>
-// CHECK:           fir.freemem %[[VAL_17]] : !fir.heap<!fir.type<_QFtestTt{c:i32}>>
+// CHECK:           %[[VAL_7:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:           %[[VAL_11:.*]] = arith.constant false
+// CHECK:           %[[VAL_12:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:           %[[VAL_16:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_13]], %[[VAL_14]], %[[VAL_11]], %[[VAL_12:.*]], {{.*}}
+// CHECK:           %[[VAL_17:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>
+// CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = ".tmp"} : (!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>) -> (!fir.class<!fir.type<_QFtestTt{c:i32}>>, !fir.class<!fir.type<_QFtestTt{c:i32}>>)
+// CHECK:           %[[VAL_19:.*]] = arith.constant true
+// CHECK:           hlfir.assign %[[VAL_4]]#0 to %[[VAL_18]]#0 temporary_lhs : !fir.class<!fir.type<_QFtestTt{c:i32}>>, !fir.class<!fir.type<_QFtestTt{c:i32}>>
+// CHECK:           %[[VAL_20:.*]] = fir.undefined tuple<!fir.class<!fir.type<_QFtestTt{c:i32}>>, i1>
+// CHECK:           %[[VAL_21:.*]] = fir.insert_value %[[VAL_20]], %[[VAL_19]], [1 : index] : (tuple<!fir.class<!fir.type<_QFtestTt{c:i32}>>, i1>, i1) -> tuple<!fir.class<!fir.type<_QFtestTt{c:i32}>>, i1>
+// CHECK:           %[[VAL_22:.*]] = fir.insert_value %[[VAL_21]], %[[VAL_18]]#0, [0 : index] : (tuple<!fir.class<!fir.type<_QFtestTt{c:i32}>>, i1>, !fir.class<!fir.type<_QFtestTt{c:i32}>>) -> tuple<!fir.class<!fir.type<_QFtestTt{c:i32}>>, i1>
+// CHECK:           hlfir.assign %[[VAL_18]]#0 to %[[VAL_2]]#0 realloc : !fir.class<!fir.type<_QFtestTt{c:i32}>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtestTt{c:i32}>>>>
+// CHECK:           %[[VAL_23:.*]] = fir.box_addr %[[VAL_18]]#0 : (!fir.class<!fir.type<_QFtestTt{c:i32}>>) -> !fir.heap<!fir.type<_QFtestTt{c:i32}>>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_18]]#0 : (!fir.class<!fir.type<_QFtestTt{c:i32}>>) -> !fir.box<none>
+// CHECK:           fir.call @_FortranADestroyWithoutFinalization(%[[VAL_24]]) : (!fir.box<none>) -> ()
+// CHECK:           fir.freemem %[[VAL_23]] : !fir.heap<!fir.type<_QFtestTt{c:i32}>>
 // CHECK:           return
 // CHECK:         }
 
@@ -45,7 +49,7 @@ func.func @test_poly_expr_with_associate(%arg1: !fir.class<!fir.array<3x!fir.typ
   %c0 = arith.constant 0 : index
   %7:3 = fir.box_dims %5, %c0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, index) -> (index, index, index)
   %8 = fir.shape %7#1 : (index) -> !fir.shape<1>
-  %9:3 = hlfir.associate %6(%8) {uniq_name = ".tmp.assign"} : (!hlfir.expr<?x!fir.type<_QMtest_typeTt1{i:i32}>?>, !fir.shape<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, i1)
+  %9:3 = hlfir.associate %6(%8) {uniq_name = ".tmp.assign"} : (!hlfir.expr<?x!fir.type<_QMtest_typeTt1{i:i32}>?>, !fir.shape<1>) -> (!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, i1)
   %10 = fir.convert %0 : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> !fir.box<none>
   fir.call @_FortranADestroy(%10) fastmath<contract> : (!fir.box<none>) -> ()
   %c3 = arith.constant 3 : index
@@ -53,52 +57,54 @@ func.func @test_poly_expr_with_associate(%arg1: !fir.class<!fir.array<3x!fir.typ
   %c1 = arith.constant 1 : index
   fir.do_loop %arg2 = %c1 to %c3 step %c1 {
     %13 = hlfir.designate %2#0 (%arg2)  : (!fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>, index) -> !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>
-    %14 = hlfir.designate %9#0 (%arg2)  : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, index) -> !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>
+    %14 = hlfir.designate %9#0 (%arg2)  : (!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, index) -> !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>
     fir.dispatch "assign"(%13 : !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>) (%13, %14 : !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>, !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>) {pass_arg_pos = 0 : i32}
   }
-  hlfir.end_associate %9#1, %9#2 : !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, i1
+  hlfir.end_associate %9#1, %9#2 : !fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, i1
   return
 }
 // CHECK-LABEL:   func.func @test_poly_expr_with_associate(
-// CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>> {fir.bindc_name = "v2"}) {
-// CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>
-// CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>> {bindc_name = ".result"}
-// CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEv2"} : (!fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>) -> (!fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>, !fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>)
-// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>)
-// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>
-// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_6]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_7]]#1 : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_9:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_9]](%[[VAL_11]]) : (!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, !fir.shape<1>) -> !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>
-// CHECK:           fir.store %[[VAL_12]] to %[[VAL_1]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>
-// CHECK:           %[[VAL_13:.*]] = arith.constant true
-// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".tmp"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>)
-// CHECK:           %[[VAL_15:.*]] = arith.constant 1 : i32
-// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_5]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>) -> !fir.box<none>
-// CHECK:           fir.call @_FortranAAllocatableApplyMold(%[[VAL_16]], %[[VAL_17]], %[[VAL_15]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_14]]#0 realloc temporary_lhs : !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>
-// CHECK:           %[[VAL_14B:.*]] = fir.load %[[VAL_14]]#0
-// CHECK:           %[[VAL_19:.*]] = fir.undefined tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, i1>
-// CHECK:           %[[VAL_20:.*]] = fir.insert_value %[[VAL_19]], %[[VAL_13]], [1 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, i1>, i1) -> tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, i1>
-// CHECK:           %[[VAL_21:.*]] = fir.insert_value %[[VAL_20]], %[[VAL_14B]], [0 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, i1>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>) -> tuple<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, i1>
-// CHECK:           %[[VAL_22:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_22]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_24:.*]] = fir.shape %[[VAL_23]]#1 : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> !fir.box<none>
-// CHECK:           fir.call @_FortranADestroy(%[[VAL_27]]) fastmath<contract> : (!fir.box<none>) -> ()
-// CHECK:           %[[VAL_29:.*]] = arith.constant 3 : index
-// CHECK:           %[[VAL_30:.*]] = fir.shape %[[VAL_29]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_31:.*]] = arith.constant 1 : index
-// CHECK:           fir.do_loop %[[VAL_32:.*]] = %[[VAL_31]] to %[[VAL_29]] step %[[VAL_31]] {
-// CHECK:             %[[VAL_33:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_32]])  : (!fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>, index) -> !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>
-// CHECK:             %[[VAL_34:.*]] = hlfir.designate %[[VAL_14B]] (%[[VAL_32]])  : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, index) -> !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>
-// CHECK:             fir.dispatch "assign"(%[[VAL_33]] : !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>) (%[[VAL_33]], %[[VAL_34]] : !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>, !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>) {pass_arg_pos = 0 : i32}
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>> {fir.bindc_name = "v2"}) {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>
+// CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>> {bindc_name = ".result"}
+// CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFtestEv2"} : (!fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>) -> (!fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>, !fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>)
+// CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>)
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_5]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_8:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>
+// CHECK:           %[[VAL_9:.*]] = fir.embox %[[VAL_8]](%[[VAL_7]]) source_box %[[VAL_4]] : (!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, !fir.shape<1>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>) -> !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>
+// CHECK:           fir.store %[[VAL_9]] to %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>
+// CHECK:           %[[VAL_10:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:           %[[VAL_14:.*]] = arith.constant false
+// CHECK:           %[[VAL_15:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_10]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:           %[[VAL_19:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_16]], %[[VAL_17]], %[[VAL_14]], %[[VAL_15]], {{.*}}
+// CHECK:           %[[VAL_20:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>
+// CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = ".tmp"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>) -> (!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>)
+// CHECK:           %[[VAL_22:.*]] = arith.constant true
+// CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_21]]#0 temporary_lhs : !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, !fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>
+// CHECK:           %[[VAL_23:.*]] = fir.undefined tuple<!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, i1>
+// CHECK:           %[[VAL_24:.*]] = fir.insert_value %[[VAL_23]], %[[VAL_22]], [1 : index] : (tuple<!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, i1>, i1) -> tuple<!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, i1>
+// CHECK:           %[[VAL_25:.*]] = fir.insert_value %[[VAL_24]], %[[VAL_21]]#0, [0 : index] : (tuple<!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, i1>, !fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>) -> tuple<!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, i1>
+// CHECK:           %[[VAL_26:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_26]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_28:.*]] = fir.shape %[[VAL_27]]#1 : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_29:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>>) -> !fir.box<none>
+// CHECK:           fir.call @_FortranADestroy(%[[VAL_29]]) fastmath<contract> : (!fir.box<none>) -> ()
+// CHECK:           %[[VAL_30:.*]] = arith.constant 3 : index
+// CHECK:           %[[VAL_31:.*]] = fir.shape %[[VAL_30]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_32:.*]] = arith.constant 1 : index
+// CHECK:           fir.do_loop %[[VAL_33:.*]] = %[[VAL_32]] to %[[VAL_30]] step %[[VAL_32]] {
+// CHECK:             %[[VAL_34:.*]] = hlfir.designate %[[VAL_2]]#0 (%[[VAL_33]])  : (!fir.class<!fir.array<3x!fir.type<_QMtest_typeTt1{i:i32}>>>, index) -> !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>
+// CHECK:             %[[VAL_35:.*]] = hlfir.designate %[[VAL_21]]#0 (%[[VAL_33]])  : (!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>, index) -> !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>
+// CHECK:             fir.dispatch "assign"(%[[VAL_34]] : !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>) (%[[VAL_34]], %[[VAL_35]] : !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>, !fir.class<!fir.type<_QMtest_typeTt1{i:i32}>>) {pass_arg_pos = 0 : i32}
 // CHECK:           }
-// CHECK:           %[[VAL_35:.*]] = fir.box_addr %[[VAL_14B]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>>) -> !fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>
-// CHECK:           fir.freemem %[[VAL_35]] : !fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>
+// CHECK:           %[[VAL_36:.*]] = fir.box_addr %[[VAL_21]]#1 : (!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>) -> !fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>
+// CHECK:           %[[VAL_37:.*]] = fir.convert %[[VAL_21]]#1 : (!fir.class<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>) -> !fir.box<none>
+// CHECK:           fir.call @_FortranADestroyWithoutFinalization(%[[VAL_37]]) : (!fir.box<none>) -> ()
+// CHECK:           fir.freemem %[[VAL_36]] : !fir.heap<!fir.array<?x!fir.type<_QMtest_typeTt1{i:i32}>>>
 // CHECK:           return
 // CHECK:         }
diff --git a/external/llvm-project/flang/test/HLFIR/declare-alloc-target.fir b/external/llvm-project/flang/test/HLFIR/declare-alloc-target.fir
new file mode 100644
index 000000000000..7cc362f6fafb
--- /dev/null
+++ b/external/llvm-project/flang/test/HLFIR/declare-alloc-target.fir
@@ -0,0 +1,21 @@
+// // RUN: fir-opt %s -convert-hlfir-to-fir | FileCheck %s
+// Check that the fir.box allocatable attribute is dropped when assigning to
+// the target of an allocatable (and not to the allocatable itself directly).
+// Otherwise, the runtime may trigger reallocation instead of trimming/padding.
+
+func.func @repro(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>, %arg1: !fir.ref<!fir.array<1x!fir.char<1,7>>>) {
+  %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
+  %1:2 = hlfir.declare %0 {uniq_name = "_QFreproEx"} : (!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
+  hlfir.assign %arg1 to %1#0 : !fir.ref<!fir.array<1x!fir.char<1,7>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+  return
+}
+// CHECK-LABEL:   func.func @repro(
+// CHECK-SAME:                     %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>,
+// CHECK-SAME:                     %[[ARG1:.*]]: !fir.ref<!fir.array<1x!fir.char<1,7>>>) {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_1:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
+// CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]] {uniq_name = "_QFreproEx"} : (!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>) -> !fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>
+// CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           fir.store %[[VAL_3]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?x!fir.char<1,?>>>>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.box<!fir.array<?x!fir.char<1,?>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           fir.call @_FortranAAssign(%[[VAL_11]],
diff --git a/external/llvm-project/flang/test/HLFIR/element-codegen-issue-118922.fir b/external/llvm-project/flang/test/HLFIR/element-codegen-issue-118922.fir
index 8fc0aa92a6b3..da998cc7430e 100644
--- a/external/llvm-project/flang/test/HLFIR/element-codegen-issue-118922.fir
+++ b/external/llvm-project/flang/test/HLFIR/element-codegen-issue-118922.fir
@@ -36,8 +36,8 @@ func.func @_QMmPbug(%val: !fir.char<1>, %var: !fir.ref<!fir.array<10x!fir.char<1
 // CHECK:             fir.store %[[VAL_0]] to %[[VAL_11]] : !fir.heap<!fir.char<1>>
 // CHECK:             %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (!fir.heap<!fir.char<1>>) -> !fir.ref<!fir.char<1>>
 // CHECK:             %[[VAL_13:.*]] = arith.constant 1 : index
-// CHECK:             %[[VAL_14:.*]] = arith.constant false
 // CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[VAL_13]] {uniq_name = ".tmp"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+// CHECK:             %[[VAL_14:.*]] = arith.constant false
 // CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]]#0 temporary_lhs : !fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>
 // CHECK:             %[[VAL_16:.*]] = fir.undefined tuple<!fir.ref<!fir.char<1>>, i1>
 // CHECK:             %[[VAL_17:.*]] = fir.insert_value %[[VAL_16]], %[[VAL_14]], [1 : index] : (tuple<!fir.ref<!fir.char<1>>, i1>, i1) -> tuple<!fir.ref<!fir.char<1>>, i1>
diff --git a/external/llvm-project/flang/test/HLFIR/elemental-codegen.fir b/external/llvm-project/flang/test/HLFIR/elemental-codegen.fir
index 67af4261470f..ef8110291c9c 100644
--- a/external/llvm-project/flang/test/HLFIR/elemental-codegen.fir
+++ b/external/llvm-project/flang/test/HLFIR/elemental-codegen.fir
@@ -153,70 +153,41 @@ func.func @test_polymorphic(%arg0: !fir.class<!fir.type<_QMtypesTt>> {fir.bindc_
   return
 }
 // CHECK-LABEL:   func.func @test_polymorphic(
-// CHECK-SAME:        %[[VAL_0:.*]]: !fir.class<!fir.type<_QMtypesTt>> {fir.bindc_name = "x"},
-// CHECK-SAME:        %[[VAL_1:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "y"},
-// CHECK-SAME:        %[[EX0:.*]]: index,
-// CHECK-SAME:        %[[EX1:.*]]: index) {
-// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
-// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEx"} : (!fir.class<!fir.type<_QMtypesTt>>) -> (!fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>)
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEy"} : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>)
-// CHECK:           %[[VAL_7:.*]] = fir.shape %[[EX0]], %[[EX1]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_8:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_9]], %[[VAL_9]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_8]](%[[VAL_10]]) : (!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.shape<2>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
-// CHECK:           fir.store %[[VAL_11]] to %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
-// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_4]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".tmp.array"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>)
-// CHECK:           %[[RANK:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_5]]#1 : (!fir.class<!fir.type<_QMtypesTt>>) -> !fir.box<none>
-// CHECK:           fir.call @_FortranAAllocatableApplyMold(%[[VAL_15]], %[[VAL_16]], %[[RANK]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:           %[[VAL_18:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_19:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (index) -> i32
-// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_18]] : (index) -> i64
-// CHECK:           %[[VAL_23:.*]] = fir.convert %[[EX0]] : (index) -> i64
-// CHECK:           fir.call @_FortranAAllocatableSetBounds(%[[VAL_20]], %[[VAL_21]], %[[VAL_22]], %[[VAL_23]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:           %[[VAL_25:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_25]] : (index) -> i32
-// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_18]] : (index) -> i64
-// CHECK:           %[[VAL_29:.*]] = fir.convert %[[EX1]] : (index) -> i64
-// CHECK:           fir.call @_FortranAAllocatableSetBounds(%[[VAL_26]], %[[VAL_27]], %[[VAL_28]], %[[VAL_29]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:           %[[VAL_31:.*]] = fir.address_of(@_QQclX
-// CHECK:           %[[VAL_32:.*]] = arith.constant {{.*}} : index
-// CHECK:           %[[VAL_33:.*]] = arith.constant {{.*}} : i32
-// CHECK:           %[[VAL_34:.*]] = arith.constant false
-// CHECK:           %[[VAL_35:.*]] = fir.absent !fir.box<none>
-// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_37:.*]] = fir.convert %[[VAL_31]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:           %[[VAL_38:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_36]], %{{.*}}, %[[VAL_34]], %[[VAL_35]], %[[VAL_37]], %[[VAL_33]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-// CHECK:           %[[VAL_12:.*]] = arith.constant true
-// CHECK:           %[[VAL_39:.*]] = fir.load %[[VAL_13]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
-// CHECK:           %[[VAL_40:.*]] = arith.constant 1 : index
-// CHECK:           fir.do_loop %[[VAL_41:.*]] = %[[VAL_40]] to %[[EX1]] step %[[VAL_40]] unordered {
-// CHECK:             fir.do_loop %[[VAL_42:.*]] = %[[VAL_40]] to %[[EX0]] step %[[VAL_40]] unordered {
-// CHECK:               %[[VAL_43:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_42]], %[[VAL_41]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
-// CHECK:               %[[VAL_44:.*]] = arith.constant 0 : index
-// CHECK:               %[[VAL_45:.*]]:3 = fir.box_dims %[[VAL_39]], %[[VAL_44]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_46:.*]] = arith.constant 1 : index
-// CHECK:               %[[VAL_47:.*]]:3 = fir.box_dims %[[VAL_39]], %[[VAL_46]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_48:.*]] = arith.constant 1 : index
-// CHECK:               %[[VAL_49:.*]] = arith.subi %[[VAL_45]]#0, %[[VAL_48]] : index
-// CHECK:               %[[VAL_50:.*]] = arith.addi %[[VAL_42]], %[[VAL_49]] : index
-// CHECK:               %[[VAL_51:.*]] = arith.subi %[[VAL_47]]#0, %[[VAL_48]] : index
-// CHECK:               %[[VAL_52:.*]] = arith.addi %[[VAL_41]], %[[VAL_51]] : index
-// CHECK:               %[[VAL_53:.*]] = hlfir.designate %[[VAL_39]] (%[[VAL_50]], %[[VAL_52]])  : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
-// CHECK:               hlfir.assign %[[VAL_43]] to %[[VAL_53]] temporary_lhs : !fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.class<!fir.type<_QMtypesTt>> {fir.bindc_name = "x"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "y"},
+// CHECK-SAME:      %[[ARG2:.*]]: index,
+// CHECK-SAME:      %[[ARG3:.*]]: index) {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEx"} : (!fir.class<!fir.type<_QMtypesTt>>) -> (!fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>)
+// CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEy"} : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_3:.*]] = fir.shape %[[ARG2]], %[[ARG3]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_4:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_5:.*]] = fir.embox %[[VAL_4]](%[[VAL_3]]) source_box %[[VAL_1]]#1 : (!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.shape<2>, !fir.class<!fir.type<_QMtypesTt>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_6:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:           %[[VAL_10:.*]] = arith.constant false
+// CHECK:           %[[VAL_11:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:           %[[VAL_15:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_12]], %[[VAL_13]], %[[VAL_10]], %[[VAL_11]],
+// CHECK:           %[[VAL_16:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = ".tmp.array"} : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>) -> (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_18:.*]] = arith.constant true
+// CHECK:           %[[VAL_19:.*]] = arith.constant 1 : index
+// CHECK:           fir.do_loop %[[VAL_20:.*]] = %[[VAL_19]] to %[[ARG3]] step %[[VAL_19]] unordered {
+// CHECK:             fir.do_loop %[[VAL_21:.*]] = %[[VAL_19]] to %[[ARG2]] step %[[VAL_19]] unordered {
+// CHECK:               %[[VAL_22:.*]] = hlfir.designate %[[VAL_2]]#0 (%[[VAL_21]], %[[VAL_20]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:               %[[VAL_23:.*]] = hlfir.designate %[[VAL_17]]#0 (%[[VAL_21]], %[[VAL_20]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:               hlfir.assign %[[VAL_22]] to %[[VAL_23]] temporary_lhs : !fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>
 // CHECK:             }
 // CHECK:           }
-// CHECK:           %[[VAL_54:.*]] = fir.undefined tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
-// CHECK:           %[[VAL_55:.*]] = fir.insert_value %[[VAL_54]], %[[VAL_12]], [1 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>, i1) -> tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
-// CHECK:           %[[VAL_56:.*]] = fir.insert_value %[[VAL_55]], %[[VAL_39]], [0 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>, !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>) -> tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
+// CHECK:           %[[VAL_24:.*]] = fir.undefined tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
+// CHECK:           %[[VAL_25:.*]] = fir.insert_value %[[VAL_24]], %[[VAL_18]], [1 : index] : (tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>, i1) -> tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
+// CHECK:           %[[VAL_26:.*]] = fir.insert_value %[[VAL_25]], %[[VAL_17]]#0, [0 : index] : (tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
 // CHECK:           return
 // CHECK:         }
 
+
 // Test that hlfir.expr mold is properly applied for the second hlfir.elemental.
 func.func @test_polymorphic_expr(%arg0: !fir.class<!fir.type<_QMtypesTt>> {fir.bindc_name = "x"}, %arg1: !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "y"}, %ex0 : index, %ex1 : index) {
   %1:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEx"} : (!fir.class<!fir.type<_QMtypesTt>>) -> (!fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>)
@@ -236,124 +207,67 @@ func.func @test_polymorphic_expr(%arg0: !fir.class<!fir.type<_QMtypesTt>> {fir.b
   return
 }
 // CHECK-LABEL:   func.func @test_polymorphic_expr(
-// CHECK-SAME:        %[[VAL_0:.*]]: !fir.class<!fir.type<_QMtypesTt>> {fir.bindc_name = "x"},
-// CHECK-SAME:        %[[VAL_1:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "y"},
-// CHECK-SAME:        %[[VAL_2:.*]]: index,
-// CHECK-SAME:        %[[VAL_3:.*]]: index) {
-// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
-// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEx"} : (!fir.class<!fir.type<_QMtypesTt>>) -> (!fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>)
-// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEy"} : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>)
-// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_9:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_10]], %[[VAL_10]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_9]](%[[VAL_11]]) : (!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.shape<2>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
-// CHECK:           fir.store %[[VAL_12]] to %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
-// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_5]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".tmp.array"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>)
-// CHECK:           %[[VAL_15:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_6]]#1 : (!fir.class<!fir.type<_QMtypesTt>>) -> !fir.box<none>
-// CHECK:           fir.call @_FortranAAllocatableApplyMold(%[[VAL_16]], %[[VAL_17]], %[[VAL_15]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:           %[[VAL_19:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_20:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (index) -> i32
-// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_19]] : (index) -> i64
-// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (index) -> i64
-// CHECK:           fir.call @_FortranAAllocatableSetBounds(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_24]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:           %[[VAL_26:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_26]] : (index) -> i32
-// CHECK:           %[[VAL_29:.*]] = fir.convert %[[VAL_19]] : (index) -> i64
-// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_3]] : (index) -> i64
-// CHECK:           fir.call @_FortranAAllocatableSetBounds(%[[VAL_27]], %[[VAL_28]], %[[VAL_29]], %[[VAL_30]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:           %[[VAL_32:.*]] = fir.address_of(@_QQcl
-// CHECK:           %[[VAL_33:.*]] = arith.constant {{.*}} : index
-// CHECK:           %[[VAL_34:.*]] = arith.constant {{.*}} : i32
-// CHECK:           %[[VAL_35:.*]] = arith.constant false
-// CHECK:           %[[VAL_36:.*]] = fir.absent !fir.box<none>
-// CHECK:           %[[VAL_37:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_38:.*]] = fir.convert %[[VAL_32]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:           %[[VAL_39:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_37]], %{{.*}}, %[[VAL_35]], %[[VAL_36]], %[[VAL_38]], %[[VAL_34]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-// CHECK:           %[[VAL_13:.*]] = arith.constant true
-// CHECK:           %[[VAL_40:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
-// CHECK:           %[[VAL_41:.*]] = arith.constant 1 : index
-// CHECK:           fir.do_loop %[[VAL_42:.*]] = %[[VAL_41]] to %[[VAL_3]] step %[[VAL_41]] unordered {
-// CHECK:             fir.do_loop %[[VAL_43:.*]] = %[[VAL_41]] to %[[VAL_2]] step %[[VAL_41]] unordered {
-// CHECK:               %[[VAL_44:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_43]], %[[VAL_42]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
-// CHECK:               %[[VAL_45:.*]] = arith.constant 0 : index
-// CHECK:               %[[VAL_46:.*]]:3 = fir.box_dims %[[VAL_40]], %[[VAL_45]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_47:.*]] = arith.constant 1 : index
-// CHECK:               %[[VAL_48:.*]]:3 = fir.box_dims %[[VAL_40]], %[[VAL_47]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_49:.*]] = arith.constant 1 : index
-// CHECK:               %[[VAL_50:.*]] = arith.subi %[[VAL_46]]#0, %[[VAL_49]] : index
-// CHECK:               %[[VAL_51:.*]] = arith.addi %[[VAL_43]], %[[VAL_50]] : index
-// CHECK:               %[[VAL_52:.*]] = arith.subi %[[VAL_48]]#0, %[[VAL_49]] : index
-// CHECK:               %[[VAL_53:.*]] = arith.addi %[[VAL_42]], %[[VAL_52]] : index
-// CHECK:               %[[VAL_54:.*]] = hlfir.designate %[[VAL_40]] (%[[VAL_51]], %[[VAL_53]])  : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
-// CHECK:               hlfir.assign %[[VAL_44]] to %[[VAL_54]] temporary_lhs : !fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.class<!fir.type<_QMtypesTt>> {fir.bindc_name = "x"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>> {fir.bindc_name = "y"},
+// CHECK-SAME:      %[[ARG2:.*]]: index,
+// CHECK-SAME:      %[[ARG3:.*]]: index) {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEx"} : (!fir.class<!fir.type<_QMtypesTt>>) -> (!fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>)
+// CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[ARG1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtestEy"} : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_4:.*]] = fir.shape %[[ARG2]], %[[ARG3]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_5:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]](%[[VAL_4]]) source_box %[[VAL_2]]#1 : (!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.shape<2>, !fir.class<!fir.type<_QMtypesTt>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_6]] to %[[VAL_1]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_7:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:           %[[VAL_11:.*]] = arith.constant false
+// CHECK:           %[[VAL_12:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:           %[[VAL_16:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_13]], %[[VAL_14]], %[[VAL_11]], %[[VAL_12]],
+// CHECK:           %[[VAL_17:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = ".tmp.array"} : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>) -> (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_19:.*]] = arith.constant true
+// CHECK:           %[[VAL_20:.*]] = arith.constant 1 : index
+// CHECK:           fir.do_loop %[[VAL_21:.*]] = %[[VAL_20]] to %[[ARG3]] step %[[VAL_20]] unordered {
+// CHECK:             fir.do_loop %[[VAL_22:.*]] = %[[VAL_20]] to %[[ARG2]] step %[[VAL_20]] unordered {
+// CHECK:               %[[VAL_23:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_22]], %[[VAL_21]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_18]]#0 (%[[VAL_22]], %[[VAL_21]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:               hlfir.assign %[[VAL_23]] to %[[VAL_24]] temporary_lhs : !fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>
 // CHECK:             }
 // CHECK:           }
-// CHECK:           %[[VAL_55:.*]] = fir.undefined tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
-// CHECK:           %[[VAL_56:.*]] = fir.insert_value %[[VAL_55]], %[[VAL_13]], [1 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>, i1) -> tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
-// CHECK:           %[[VAL_57:.*]] = fir.insert_value %[[VAL_56]], %[[VAL_40]], [0 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>, !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>) -> tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
-// CHECK:           %[[VAL_58:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>
-// CHECK:           %[[VAL_59:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_60:.*]] = fir.shape %[[VAL_59]], %[[VAL_59]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_61:.*]] = fir.embox %[[VAL_58]](%[[VAL_60]]) : (!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.shape<2>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
-// CHECK:           fir.store %[[VAL_61]] to %[[VAL_4]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
-// CHECK:           %[[VAL_63:.*]]:2 = hlfir.declare %[[VAL_4]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".tmp.array"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>)
-// CHECK:           %[[VAL_64:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_65:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_66:.*]] = fir.convert %[[VAL_40]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>) -> !fir.box<none>
-// CHECK:           fir.call @_FortranAAllocatableApplyMold(%[[VAL_65]], %[[VAL_66]], %[[VAL_64]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:           %[[VAL_68:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_69:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_70:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_71:.*]] = fir.convert %[[VAL_69]] : (index) -> i32
-// CHECK:           %[[VAL_72:.*]] = fir.convert %[[VAL_68]] : (index) -> i64
-// CHECK:           %[[VAL_73:.*]] = fir.convert %[[VAL_2]] : (index) -> i64
-// CHECK:           fir.call @_FortranAAllocatableSetBounds(%[[VAL_70]], %[[VAL_71]], %[[VAL_72]], %[[VAL_73]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:           %[[VAL_75:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_76:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_77:.*]] = fir.convert %[[VAL_75]] : (index) -> i32
-// CHECK:           %[[VAL_78:.*]] = fir.convert %[[VAL_68]] : (index) -> i64
-// CHECK:           %[[VAL_79:.*]] = fir.convert %[[VAL_3]] : (index) -> i64
-// CHECK:           fir.call @_FortranAAllocatableSetBounds(%[[VAL_76]], %[[VAL_77]], %[[VAL_78]], %[[VAL_79]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:           %[[VAL_81:.*]] = fir.address_of(@_QQcl
-// CHECK:           %[[VAL_82:.*]] = arith.constant {{.*}} : index
-// CHECK:           %[[VAL_83:.*]] = arith.constant {{.*}} : i32
-// CHECK:           %[[VAL_84:.*]] = arith.constant false
-// CHECK:           %[[VAL_85:.*]] = fir.absent !fir.box<none>
-// CHECK:           %[[VAL_86:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:           %[[VAL_87:.*]] = fir.convert %[[VAL_81]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:           %[[VAL_88:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_86]], %{{.*}}, %[[VAL_84]], %[[VAL_85]], %[[VAL_87]], %[[VAL_83]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-// CHECK:           %[[VAL_62:.*]] = arith.constant true
-// CHECK:           %[[VAL_89:.*]] = fir.load %[[VAL_63]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
-// CHECK:           %[[VAL_90:.*]] = arith.constant 1 : index
-// CHECK:           fir.do_loop %[[VAL_91:.*]] = %[[VAL_90]] to %[[VAL_3]] step %[[VAL_90]] unordered {
-// CHECK:             fir.do_loop %[[VAL_92:.*]] = %[[VAL_90]] to %[[VAL_2]] step %[[VAL_90]] unordered {
-// CHECK:               %[[VAL_93:.*]] = hlfir.designate %[[VAL_40]] (%[[VAL_92]], %[[VAL_91]])  : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
-// CHECK:               %[[VAL_94:.*]] = arith.constant false
-// CHECK:               %[[VAL_95:.*]] = fir.undefined tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>
-// CHECK:               %[[VAL_96:.*]] = fir.insert_value %[[VAL_95]], %[[VAL_94]], [1 : index] : (tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>, i1) -> tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>
-// CHECK:               %[[VAL_97:.*]] = fir.insert_value %[[VAL_96]], %[[VAL_93]], [0 : index] : (tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>, !fir.class<!fir.type<_QMtypesTt>>) -> tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>
-// CHECK:               %[[VAL_98:.*]] = arith.constant 0 : index
-// CHECK:               %[[VAL_99:.*]]:3 = fir.box_dims %[[VAL_89]], %[[VAL_98]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_100:.*]] = arith.constant 1 : index
-// CHECK:               %[[VAL_101:.*]]:3 = fir.box_dims %[[VAL_89]], %[[VAL_100]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_102:.*]] = arith.constant 1 : index
-// CHECK:               %[[VAL_103:.*]] = arith.subi %[[VAL_99]]#0, %[[VAL_102]] : index
-// CHECK:               %[[VAL_104:.*]] = arith.addi %[[VAL_92]], %[[VAL_103]] : index
-// CHECK:               %[[VAL_105:.*]] = arith.subi %[[VAL_101]]#0, %[[VAL_102]] : index
-// CHECK:               %[[VAL_106:.*]] = arith.addi %[[VAL_91]], %[[VAL_105]] : index
-// CHECK:               %[[VAL_107:.*]] = hlfir.designate %[[VAL_89]] (%[[VAL_104]], %[[VAL_106]])  : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
-// CHECK:               hlfir.assign %[[VAL_93]] to %[[VAL_107]] temporary_lhs : !fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:           %[[VAL_25:.*]] = fir.undefined tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
+// CHECK:           %[[VAL_26:.*]] = fir.insert_value %[[VAL_25]], %[[VAL_19]], [1 : index] : (tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>, i1) -> tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
+// CHECK:           %[[VAL_27:.*]] = fir.insert_value %[[VAL_26]], %[[VAL_18]]#0, [0 : index] : (tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
+// CHECK:           %[[VAL_28:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_29:.*]] = fir.embox %[[VAL_28]](%[[VAL_4]]) source_box %[[VAL_18]]#1 : (!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.shape<2>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_29]] to %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_30:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:           %[[VAL_34:.*]] = arith.constant false
+// CHECK:           %[[VAL_35:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_37:.*]] = fir.convert %[[VAL_30]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:           %[[VAL_39:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_36]], %[[VAL_37]], %[[VAL_34]], %[[VAL_35]],
+// CHECK:           %[[VAL_40:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[VAL_41:.*]]:2 = hlfir.declare %[[VAL_40]] {uniq_name = ".tmp.array"} : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>) -> (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_42:.*]] = arith.constant true
+// CHECK:           %[[VAL_43:.*]] = arith.constant 1 : index
+// CHECK:           fir.do_loop %[[VAL_44:.*]] = %[[VAL_43]] to %[[ARG3]] step %[[VAL_43]] unordered {
+// CHECK:             fir.do_loop %[[VAL_45:.*]] = %[[VAL_43]] to %[[ARG2]] step %[[VAL_43]] unordered {
+// CHECK:               %[[VAL_46:.*]] = hlfir.designate %[[VAL_18]]#0 (%[[VAL_45]], %[[VAL_44]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:               %[[VAL_47:.*]] = arith.constant false
+// CHECK:               %[[VAL_48:.*]] = fir.undefined tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>
+// CHECK:               %[[VAL_49:.*]] = fir.insert_value %[[VAL_48]], %[[VAL_47]], [1 : index] : (tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>, i1) -> tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>
+// CHECK:               %[[VAL_50:.*]] = fir.insert_value %[[VAL_49]], %[[VAL_46]], [0 : index] : (tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>, !fir.class<!fir.type<_QMtypesTt>>) -> tuple<!fir.class<!fir.type<_QMtypesTt>>, i1>
+// CHECK:               %[[VAL_51:.*]] = hlfir.designate %[[VAL_41]]#0 (%[[VAL_45]], %[[VAL_44]])  : (!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, index, index) -> !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:               hlfir.assign %[[VAL_46]] to %[[VAL_51]] temporary_lhs : !fir.class<!fir.type<_QMtypesTt>>, !fir.class<!fir.type<_QMtypesTt>>
+// CHECK:               %[[VAL_52:.*]] = fir.box_addr %[[VAL_46]] : (!fir.class<!fir.type<_QMtypesTt>>) -> !fir.heap<!fir.type<_QMtypesTt>>
+// CHECK:               %[[VAL_53:.*]] = fir.convert %[[VAL_46]] : (!fir.class<!fir.type<_QMtypesTt>>) -> !fir.box<none>
+// CHECK:               fir.call @_FortranADestroyWithoutFinalization(%[[VAL_53]]) : (!fir.box<none>) -> ()
 // CHECK:             }
 // CHECK:           }
-// CHECK:           %[[VAL_108:.*]] = fir.undefined tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
-// CHECK:           %[[VAL_109:.*]] = fir.insert_value %[[VAL_108]], %[[VAL_62]], [1 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>, i1) -> tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
-// CHECK:           %[[VAL_110:.*]] = fir.insert_value %[[VAL_109]], %[[VAL_89]], [0 : index] : (tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>, !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>) -> tuple<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMtypesTt>>>>, i1>
+// CHECK:           %[[VAL_54:.*]] = fir.undefined tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
+// CHECK:           %[[VAL_55:.*]] = fir.insert_value %[[VAL_54]], %[[VAL_42]], [1 : index] : (tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>, i1) -> tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
+// CHECK:           %[[VAL_56:.*]] = fir.insert_value %[[VAL_55]], %[[VAL_41]]#0, [0 : index] : (tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>, !fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>) -> tuple<!fir.class<!fir.array<?x?x!fir.type<_QMtypesTt>>>, i1>
 // CHECK:           return
 // CHECK:         }
diff --git a/external/llvm-project/flang/test/HLFIR/inline-hlfir-assign.fir b/external/llvm-project/flang/test/HLFIR/inline-hlfir-assign.fir
index f834e7971e3d..db71720119e7 100644
--- a/external/llvm-project/flang/test/HLFIR/inline-hlfir-assign.fir
+++ b/external/llvm-project/flang/test/HLFIR/inline-hlfir-assign.fir
@@ -145,18 +145,16 @@ func.func @_QPtest3(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x"})
 // CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_10]]#1, %[[VAL_1]] : index
 // CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_12]], %[[VAL_14]] : (index, index) -> !fir.shape<2>
 // CHECK:           %[[VAL_16:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_2]]:%[[VAL_9]]#1:%[[VAL_2]], %[[VAL_2]]:%[[VAL_10]]#1:%[[VAL_2]])  shape %[[VAL_15]] : (!fir.box<!fir.array<?x?xf32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
-// CHECK:           fir.do_loop %[[VAL_17:.*]] = %[[VAL_2]] to %[[VAL_14]] step %[[VAL_2]] unordered {
-// CHECK:             fir.do_loop %[[VAL_18:.*]] = %[[VAL_2]] to %[[VAL_12]] step %[[VAL_2]] unordered {
+// CHECK:           fir.do_loop %[[VAL_17:.*]] = %[[VAL_2]] to %[[VAL_3]] step %[[VAL_2]] unordered {
+// CHECK:             fir.do_loop %[[VAL_18:.*]] = %[[VAL_2]] to %[[VAL_3]] step %[[VAL_2]] unordered {
 // CHECK:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.ref<!fir.array<3x3xf32>>, index, index) -> !fir.ref<f32>
 // CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<f32>
 // CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_16]] (%[[VAL_18]], %[[VAL_17]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
 // CHECK:               hlfir.assign %[[VAL_20]] to %[[VAL_21]] : f32, !fir.ref<f32>
 // CHECK:             }
 // CHECK:           }
-// CHECK:           %[[VAL_22:.*]]:3 = fir.box_dims %[[VAL_4]]#0, %[[VAL_1]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// CHECK:           %[[VAL_23:.*]]:3 = fir.box_dims %[[VAL_4]]#0, %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
-// CHECK:           fir.do_loop %[[VAL_24:.*]] = %[[VAL_2]] to %[[VAL_23]]#1 step %[[VAL_2]] unordered {
-// CHECK:             fir.do_loop %[[VAL_25:.*]] = %[[VAL_2]] to %[[VAL_22]]#1 step %[[VAL_2]] unordered {
+// CHECK:           fir.do_loop %[[VAL_24:.*]] = %[[VAL_2]] to %[[VAL_3]] step %[[VAL_2]] unordered {
+// CHECK:             fir.do_loop %[[VAL_25:.*]] = %[[VAL_2]] to %[[VAL_3]] step %[[VAL_2]] unordered {
 // CHECK:               %[[VAL_26:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_25]], %[[VAL_24]])  : (!fir.ref<!fir.array<3x3xf32>>, index, index) -> !fir.ref<f32>
 // CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_26]] : !fir.ref<f32>
 // CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_25]], %[[VAL_24]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
diff --git a/external/llvm-project/flang/test/HLFIR/inline-hlfir-copy-in.fir b/external/llvm-project/flang/test/HLFIR/inline-hlfir-copy-in.fir
new file mode 100644
index 000000000000..f3c4b38962a0
--- /dev/null
+++ b/external/llvm-project/flang/test/HLFIR/inline-hlfir-copy-in.fir
@@ -0,0 +1,206 @@
+// Test inlining of hlfir.copy_in
+// RUN: fir-opt --inline-hlfir-copy-in %s | FileCheck %s
+
+// Test inlining of hlfir.copy_in that does not require the array to be copied out
+func.func private @_test_inline_copy_in(%arg0: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "i"}, %arg2: !fir.ref<i32> {fir.bindc_name = "j"}) {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>>
+  %1 = fir.dummy_scope : !fir.dscope
+  %2:2 = hlfir.declare %arg1 dummy_scope %1 {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3:2 = hlfir.declare %arg2 dummy_scope %1 {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:2 = hlfir.declare %arg0 dummy_scope %1 {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+  %5 = fir.load %2#0 : !fir.ref<i32>
+  %6 = fir.convert %5 : (i32) -> i64
+  %c1 = arith.constant 1 : index
+  %c1_0 = arith.constant 1 : index
+  %7:3 = fir.box_dims %4#1, %c1_0 : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+  %c1_1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %8 = arith.subi %7#1, %c1 : index
+  %9 = arith.addi %8, %c1_1 : index
+  %10 = arith.divsi %9, %c1_1 : index
+  %11 = arith.cmpi sgt, %10, %c0 : index
+  %12 = arith.select %11, %10, %c0 : index
+  %13 = fir.load %3#0 : !fir.ref<i32>
+  %14 = fir.convert %13 : (i32) -> i64
+  %15 = fir.shape %12 : (index) -> !fir.shape<1>
+  %16 = hlfir.designate %4#0 (%6, %c1:%7#1:%c1_1, %14)  shape %15 : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+  %c100_i32 = arith.constant 100 : i32
+  %17:2 = hlfir.copy_in %16 to %0 : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.box<!fir.array<?xf64>>, i1)
+  %18 = fir.box_addr %17#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+  %19:3 = hlfir.associate %c100_i32 {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+  fir.call @_QFPsb(%18, %19#0) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+  hlfir.copy_out %0, %17#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1) -> ()
+  hlfir.end_associate %19#1, %19#2 : !fir.ref<i32>, i1
+  return
+}
+
+// CHECK: #loop_vectorize = #llvm.loop_vectorize<disable = true>
+// CHECK: #loop_annotation = #llvm.loop_annotation<vectorize = #loop_vectorize>
+// CHECK-LABEL:   func.func private @_test_inline_copy_in(
+// CHECK-SAME:                                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"},
+// CHECK-SAME:                                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "i"},
+// CHECK-SAME:                                          %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "j"}) {
+// CHECK:    %[[VAL_3:.*]] = arith.constant true
+// CHECK:    %[[VAL_4:.*]] = arith.constant false
+// CHECK:    %[[VAL_5:.*]] = arith.constant 100 : i32
+// CHECK:    %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK:    %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:    %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:    %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_1:.*]] dummy_scope %[[VAL_8:.*]] {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:    %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2:.*]] dummy_scope %[[VAL_8:.*]] {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:    %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %[[VAL_8:.*]] {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+// CHECK:    %[[VAL_11:.*]] = fir.load %[[VAL_22:.*]]#0 : !fir.ref<i32>
+// CHECK:    %[[VAL_12:.*]] = fir.convert %[[VAL_11:.*]] : (i32) -> i64
+// CHECK:    %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_10:.*]]#1, %[[VAL_7:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+// CHECK:    %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13:.*]]#1, %[[VAL_6:.*]] : index
+// CHECK:    %[[VAL_15:.*]] = arith.select %[[VAL_14:.*]], %[[VAL_13:.*]]#1, %[[VAL_6:.*]] : index
+// CHECK:    %[[VAL_16:.*]] = fir.load %[[VAL_9:.*]]#0 : !fir.ref<i32>
+// CHECK:    %[[VAL_17:.*]] = fir.convert %[[VAL_16:.*]] : (i32) -> i64
+// CHECK:    %[[VAL_18:.*]] = fir.shape %[[VAL_15:.*]] : (index) -> !fir.shape<1>
+// CHECK:    %[[VAL_19:.*]] = hlfir.designate %[[VAL_10:.*]]#0 (%[[VAL_12:.*]], %[[VAL_7:.*]]:%[[VAL_13:.*]]#1:%[[VAL_7:.*]], %[[VAL_17:.*]])  shape %[[VAL_18:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+// CHECK:    %[[VAL_20:.*]] = fir.is_contiguous_box %[[VAL_19:.*]] whole : (!fir.box<!fir.array<?xf64>>) -> i1
+// CHECK:    %[[VAL_21:.*]]:2 = fir.if %[[VAL_20:.*]] -> (!fir.box<!fir.array<?xf64>>, i1) {
+// CHECK:      fir.result %[[VAL_19:.*]], %[[VAL_4:.*]] : !fir.box<!fir.array<?xf64>>, i1
+// CHECK:    } else {
+// CHECK:      %[[VAL_24:.*]] = fir.allocmem !fir.array<?xf64>, %[[VAL_15:.*]] {bindc_name = ".tmp.copy_in", uniq_name = ""}
+// CHECK:      %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_24:.*]](%[[VAL_18:.*]]) {uniq_name = ".tmp.copy_in"} : (!fir.heap<!fir.array<?xf64>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.heap<!fir.array<?xf64>>)
+// CHECK:      fir.do_loop %arg3 = %[[VAL_7:.*]] to %[[VAL_15:.*]] step %[[VAL_7:.*]] unordered attributes {loopAnnotation = #loop_annotation} {
+// CHECK:        %[[VAL_26:.*]] = hlfir.designate %[[VAL_19:.*]] (%arg3)  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:        %[[VAL_27:.*]] = fir.load %[[VAL_26:.*]] : !fir.ref<f64>
+// CHECK:        %[[VAL_28:.*]] = hlfir.designate %[[VAL_25:.*]]#0 (%arg3)  : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK:        hlfir.assign %[[VAL_27:.*]] to %[[VAL_28:.*]] : f64, !fir.ref<f64>
+// CHECK:      }
+// CHECK:      fir.result %[[VAL_25:.*]]#0, %[[VAL_3:.*]] : !fir.box<!fir.array<?xf64>>, i1
+// CHECK:    }
+// CHECK:    %[[VAL_22:.*]] = fir.box_addr %[[VAL_21:.*]]#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+// CHECK:    %[[VAL_23:.*]]:3 = hlfir.associate %[[VAL_5:.*]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+// CHECK:    fir.call @_QFPsb(%[[VAL_22:.*]], %[[VAL_23:.*]]#0) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+// CHECK:    hlfir.copy_out %16, %15#1 : (!fir.ref<!fir.box<!fir.array<?xf64>>>, i1) -> ()
+// CHECK:    hlfir.end_associate %[[VAL_23:.*]]#1, %[[VAL_23:.*]]#2 : !fir.ref<i32>, i1
+// CHECK:    return
+// CHECK:  }
+
+// Test not inlining of hlfir.copy_in that requires the array to be copied out
+func.func private @_test_no_inline_copy_in(%arg0: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "i"}, %arg2: !fir.ref<i32> {fir.bindc_name = "j"}) {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>>
+  %1 = fir.dummy_scope : !fir.dscope
+  %2:2 = hlfir.declare %arg1 dummy_scope %1 {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3:2 = hlfir.declare %arg2 dummy_scope %1 {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:2 = hlfir.declare %arg0 dummy_scope %1 {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+  %5 = fir.load %2#0 : !fir.ref<i32>
+  %6 = fir.convert %5 : (i32) -> i64
+  %c1 = arith.constant 1 : index
+  %c1_0 = arith.constant 1 : index
+  %7:3 = fir.box_dims %4#1, %c1_0 : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+  %c1_1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %8 = arith.subi %7#1, %c1 : index
+  %9 = arith.addi %8, %c1_1 : index
+  %10 = arith.divsi %9, %c1_1 : index
+  %11 = arith.cmpi sgt, %10, %c0 : index
+  %12 = arith.select %11, %10, %c0 : index
+  %13 = fir.load %3#0 : !fir.ref<i32>
+  %14 = fir.convert %13 : (i32) -> i64
+  %15 = fir.shape %12 : (index) -> !fir.shape<1>
+  %16 = hlfir.designate %4#0 (%6, %c1:%7#1:%c1_1, %14)  shape %15 : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+  %c100_i32 = arith.constant 100 : i32
+  %17:2 = hlfir.copy_in %16 to %0 : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.box<!fir.array<?xf64>>, i1)
+  %18 = fir.box_addr %17#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+  %19:3 = hlfir.associate %c100_i32 {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+  fir.call @_QFPsb(%18, %19#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+  hlfir.copy_out %0, %17#1 to %16 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1, !fir.box<!fir.array<?xf64>>) -> ()
+  hlfir.end_associate %19#1, %19#2 : !fir.ref<i32>, i1
+  return
+}
+
+// CHECK-LABEL:  func.func private @_test_no_inline_copy_in(
+// CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"},
+// CHECK-SAME:                                             %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "i"},
+// CHECK-SAME:                                             %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "j"}) {
+// CHECK:    %[[VAL_3:.*]] = arith.constant 100 : i32
+// CHECK:    %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:    %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:    %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>>
+// CHECK:    %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:    %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_1:.*]] dummy_scope %[[VAL_7:.*]] {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:    %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2:.*]] dummy_scope %[[VAL_7:.*]] {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:    %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %[[VAL_7:.*]] {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+// CHECK:    %[[VAL_11:.*]] = fir.load %[[VAL_8:.*]]#0 : !fir.ref<i32>
+// CHECK:    %[[VAL_12:.*]] = fir.convert %[[VAL_11:.*]] : (i32) -> i64
+// CHECK:    %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_10:.*]]#1, %[[VAL_5:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+// CHECK:    %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13:.*]]#1, %[[VAL_4:.*]] : index
+// CHECK:    %[[VAL_15:.*]] = arith.select %[[VAL_14:.*]], %[[VAL_13:.*]]#1, %[[VAL_4:.*]] : index
+// CHECK:    %[[VAL_16:.*]] = fir.load %[[VAL_9:.*]]#0 : !fir.ref<i32>
+// CHECK:    %[[VAL_17:.*]] = fir.convert %[[VAL_16:.*]] : (i32) -> i64
+// CHECK:    %[[VAL_18:.*]] = fir.shape %[[VAL_15:.*]] : (index) -> !fir.shape<1>
+// CHECK:    %[[VAL_19:.*]] = hlfir.designate %[[VAL_10:.*]]#0 (%[[VAL_12:.*]], %[[VAL_5:.*]]:%[[VAL_13:.*]]#1:%[[VAL_5:.*]], %[[VAL_17:.*]])  shape %[[VAL_18:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+// CHECK:    %[[VAL_20:.*]]:2 = hlfir.copy_in %[[VAL_19:.*]] to %[[VAL_6:.*]] : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.box<!fir.array<?xf64>>, i1)
+// CHECK:    %[[VAL_21:.*]] = fir.box_addr %[[VAL_20:.*]]#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+// CHECK:    %[[VAL_22:.*]]:3 = hlfir.associate %[[VAL_3:.*]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+// CHECK:    fir.call @_QFPsb(%[[VAL_21:.*]], %[[VAL_22:.*]]#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+// CHECK:    hlfir.copy_out %[[VAL_6:.*]], %[[VAL_20:.*]]#1 to %[[VAL_19:.*]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1, !fir.box<!fir.array<?xf64>>) -> ()
+// CHECK:    hlfir.end_associate %[[VAL_22:.*]]#1, %[[VAL_22:.*]]#2 : !fir.ref<i32>, i1
+// CHECK:    return
+// CHECK:  }
+
+// Test not inlining optional dummy arguments (no direct copy-out)
+func.func @_QPoptional_copy_in_out(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.optional}) {
+  %false = arith.constant false
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+  %1 = fir.dummy_scope : !fir.dscope
+  %2:2 = hlfir.declare %arg0 dummy_scope %1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %3 = fir.is_present %2#0 : (!fir.box<!fir.array<?xf32>>) -> i1
+  %4:2 = fir.if %3 -> (!fir.ref<!fir.array<?xf32>>, i1) {
+    %5:2 = hlfir.copy_in %2#0 to %0 : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.box<!fir.array<?xf32>>, i1)
+    %6 = fir.box_addr %5#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+    fir.result %6, %5#1 : !fir.ref<!fir.array<?xf32>>, i1
+  } else {
+    %5 = fir.absent !fir.ref<!fir.array<?xf32>>
+    fir.result %5, %false : !fir.ref<!fir.array<?xf32>>, i1
+  }
+  fir.call @_QPtakes_optional_explicit(%4#0) fastmath<contract> : (!fir.ref<!fir.array<?xf32>>) -> ()
+  hlfir.copy_out %0, %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, i1) -> ()
+  return
+}
+
+// CHECK-LABEL:  func.func @_QPoptional_copy_in_out(
+// CHECK-SAME: %[[ARG_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.optional}) {
+// CHECK:    %false = arith.constant false
+// CHECK:    %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK:    %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG_0:.*]] dummy_scope %[[VAL_1:.*]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:    %[[VAL_3:.*]] = fir.is_present %[[VAL_2:.*]]#0 : (!fir.box<!fir.array<?xf32>>) -> i1
+// CHECK:    %[[VAL_4:.*]]:2 = fir.if %[[VAL_3:.*]] -> (!fir.ref<!fir.array<?xf32>>, i1) {
+// CHECK:      %[[VAL_5:.*]]:2 = hlfir.copy_in %[[VAL_2:.*]]#0 to %[[VAL_0:.*]] : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.box<!fir.array<?xf32>>, i1)
+// CHECK:      %[[VAL_6:.*]] = fir.box_addr %[[VAL_5:.*]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+// CHECK:      fir.result %[[VAL_6:.*]], %[[VAL_5:.*]]#1 : !fir.ref<!fir.array<?xf32>>, i1
+// CHECK:    } else {
+// CHECK:      %[[VAL_5:.*]] = fir.absent !fir.ref<!fir.array<?xf32>>
+// CHECK:      fir.result %[[VAL_5:.*]], %false : !fir.ref<!fir.array<?xf32>>, i1
+// CHECK:    }
+// CHECK:    fir.call @_QPtakes_optional_explicit(%[[VAL_4:.*]]#0) fastmath<contract> : (!fir.ref<!fir.array<?xf32>>) -> ()
+// CHECK:    hlfir.copy_out %[[VAL_0:.*]], %[[VAL_4:.*]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, i1) -> ()
+// CHECK:    return
+// CHECK:  }
+
+// Test not inlining of assumed-rank arrays
+func.func @_QPtest_copy_in_out_2(%arg0: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x"}) {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<*:f32>>>
+  %1 = fir.dummy_scope : !fir.dscope
+  %2:2 = hlfir.declare %arg0 dummy_scope %1 {uniq_name = "_QFtest_copy_in_out_2Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+  %3:2 = hlfir.copy_in %2#0 to %0 : (!fir.box<!fir.array<*:f32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> (!fir.box<!fir.array<*:f32>>, i1)
+  fir.call @_QPtakes_contiguous_intentin(%3#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
+  hlfir.copy_out %0, %3#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, i1) -> ()
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest_copy_in_out_2(
+// CHECK-SAME: %[[ARG_0]]: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x"}) {
+// CHECK:   %[[VAL_0]] = fir.alloca !fir.box<!fir.heap<!fir.array<*:f32>>>
+// CHECK:   %[[VAL_1]] = fir.dummy_scope : !fir.dscope
+// CHECK:   %[[VAL_2]]:2 = hlfir.declare %[[ARG_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_copy_in_out_2Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+// CHECK:   %[[VAL_3]]:2 = hlfir.copy_in %[[VAL_2]]#0 to %[[VAL_0]] : (!fir.box<!fir.array<*:f32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> (!fir.box<!fir.array<*:f32>>, i1)
+// CHECK:   fir.call @_QPtakes_contiguous_intentin(%[[VAL_3]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
+// CHECK:   hlfir.copy_out %[[VAL_0]], %[[VAL_3]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, i1) -> ()
+// CHECK:   return
+// CHECK: }
diff --git a/external/llvm-project/flang/test/HLFIR/memory-effects.fir b/external/llvm-project/flang/test/HLFIR/memory-effects.fir
index cac887ebe67d..6c791f1260be 100644
--- a/external/llvm-project/flang/test/HLFIR/memory-effects.fir
+++ b/external/llvm-project/flang/test/HLFIR/memory-effects.fir
@@ -3,8 +3,9 @@
 func.func @concat(%arg0: !fir.ref<!fir.char<1,10>>, %arg1: !fir.ref<!fir.char<1, 20>>) {
 // expected-remark@+1 {{operation has no memory effects}}
   %c30 = arith.constant 30 : index
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+3 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 1, on resource '<Default>'}}
   %0 = hlfir.concat %arg0, %arg1 len %c30 : (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,20>>, index) -> (!hlfir.expr<!fir.char<1,30>>)
   return
 }
@@ -16,8 +17,8 @@ func.func @all_no_effects(%arg0: !hlfir.expr<2x!fir.logical<4>>) {
 }
 
 func.func @all_effects(%arg0: !fir.ref<!fir.array<2x10x!fir.logical<4>>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %all = hlfir.all %arg0 dim %arg1 : (!fir.ref<!fir.array<2x10x!fir.logical<4>>>, i32) -> !hlfir.expr<?x!fir.logical<4>>
   return
 }
@@ -29,8 +30,8 @@ func.func @any_no_effects(%arg0: !hlfir.expr<2x!fir.logical<4>>) {
 }
 
 func.func @any_effects(%arg0: !fir.ref<!fir.array<2x10x!fir.logical<4>>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %all = hlfir.any %arg0 dim %arg1 : (!fir.ref<!fir.array<2x10x!fir.logical<4>>>, i32) -> !hlfir.expr<?x!fir.logical<4>>
   return
 }
@@ -42,7 +43,7 @@ func.func @count_no_effects(%arg0: !hlfir.expr<2x!fir.logical<4>>) {
 }
 
 func.func @count_effects(%arg0: !fir.ref<!fir.array<2x10x!fir.logical<4>>>, %arg1: i32) {
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %all = hlfir.count %arg0 dim %arg1 : (!fir.ref<!fir.array<2x10x!fir.logical<4>>>, i32) -> i32
   return
 }
@@ -54,15 +55,15 @@ func.func @product_no_effects(%arg0: !hlfir.expr<?xf32>) {
 }
 
 func.func @product_effects(%arg0: !fir.ref<!fir.array<2x2xf32>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %product = hlfir.product %arg0 dim %arg1 : (!fir.ref<!fir.array<2x2xf32>>, i32) -> !hlfir.expr<2xf32>
   return
 }
 
 func.func @set_length_read(%arg0: !fir.ref<!fir.char<1,10>>, %arg1: index) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %0 = hlfir.set_length %arg0 len %arg1 : (!fir.ref<!fir.char<1,10>>, index) -> !hlfir.expr<!fir.char<1,?>>
   return
 }
@@ -74,8 +75,8 @@ func.func @sum_no_effects(%arg0: !hlfir.expr<?xf32>) {
 }
 
 func.func @sum_effects(%arg0: !fir.ref<!fir.array<2x2xf32>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %sum = hlfir.sum %arg0 dim %arg1 : (!fir.ref<!fir.array<2x2xf32>>, i32) -> !hlfir.expr<2xf32>
   return
 }
@@ -87,8 +88,8 @@ func.func @maxval_no_effects(%arg0: !hlfir.expr<?xf32>) {
 }
 
 func.func @maxval_effects(%arg0: !fir.ref<!fir.array<2x2xf32>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %maxval = hlfir.maxval %arg0 dim %arg1 : (!fir.ref<!fir.array<2x2xf32>>, i32) -> !hlfir.expr<2xf32>
   return
 }
@@ -100,34 +101,34 @@ func.func @minval_no_effects(%arg0: !hlfir.expr<?xf32>) {
 }
 
 func.func @minval_effects(%arg0: !fir.ref<!fir.array<2x2xf32>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %minval = hlfir.minval %arg0 dim %arg1 : (!fir.ref<!fir.array<2x2xf32>>, i32) -> !hlfir.expr<2xf32>
   return
 }
 
 func.func @minloc_effects_simple(%arg0: !hlfir.expr<?xf32>) {
-// expected-remark@+1 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
   %minloc = hlfir.minloc %arg0 : (!hlfir.expr<?xf32>) -> !hlfir.expr<?xi32>
   return
 }
 
 func.func @minloc_effects(%arg0: !fir.ref<!fir.array<2x2xf32>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %minloc = hlfir.minloc %arg0 dim %arg1 : (!fir.ref<!fir.array<2x2xf32>>, i32) -> !hlfir.expr<2xi32>
   return
 }
 
 func.func @maxloc_effects_simple(%arg0: !hlfir.expr<?xf32>) {
-// expected-remark@+1 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
   %maxloc = hlfir.maxloc %arg0 : (!hlfir.expr<?xf32>) -> !hlfir.expr<?xi32>
   return
 }
 
 func.func @maxloc_effects(%arg0: !fir.ref<!fir.array<2x2xf32>>, %arg1: i32) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %maxloc = hlfir.maxloc %arg0 dim %arg1 : (!fir.ref<!fir.array<2x2xf32>>, i32) -> !hlfir.expr<2xi32>
   return
 }
@@ -139,49 +140,49 @@ func.func @dot_product_no_effects(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.expr<
 }
 
 func.func @dot_product_effects(%arg0: !fir.ref<!fir.array<10xf32>>, %arg1: !fir.ref<!fir.array<10xf32>>) {
-// there are read effects on both arguments - the diagnostic verification just doesn't register duplicate identical diagnostics
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 1, on resource '<Default>'}}
   %0 = hlfir.dot_product %arg0 %arg1 : (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) -> f32
   return
 }
 
 func.func @matmul_no_reads(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.expr<?x?xf32>) {
-// expected-remark@+1 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
   %0 = hlfir.matmul %arg0 %arg1 : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?xf32>) -> !hlfir.expr<?x?xf32>
   return
 }
 
 func.func @matmul_reads(%arg0: !fir.ref<!fir.array<10x5xf32>>, %arg1: !fir.ref<!fir.array<5x10xf32>>) {
-// expected-remark@+3 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// there are read effects on both arguments - the diagnostic verification just doesn't register duplicate identical diagnostics
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+3 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 1, on resource '<Default>'}}
   %0 = hlfir.matmul %arg0 %arg1 : (!fir.ref<!fir.array<10x5xf32>>, !fir.ref<!fir.array<5x10xf32>>) -> !hlfir.expr<10x10xf32>
   return
 }
 
 func.func @transpose_no_reads(%arg0: !hlfir.expr<?x?xf32>) {
-// expected-remark@+1 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
   %0 = hlfir.transpose %arg0 : (!hlfir.expr<?x?xf32>) -> !hlfir.expr<?x?xf32>
   return
 }
 
 func.func @transpose_read(%arg0: !fir.ref<!fir.array<10x5xf32>>) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %0 = hlfir.transpose %arg0 : (!fir.ref<!fir.array<10x5xf32>>) -> !hlfir.expr<5x10xf32>
   return
 }
 
 func.func @matmul_transpose_no_reads(%arg0: !hlfir.expr<?x?xf32>, %arg1: !hlfir.expr<?x?xf32>) {
-// expected-remark@+1 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
   %0 = hlfir.matmul_transpose %arg0 %arg1 : (!hlfir.expr<?x?xf32>, !hlfir.expr<?x?xf32>) -> !hlfir.expr<?x?xf32>
   return
 }
 
 func.func @matmul_transpose_reads(%arg0: !fir.ref<!fir.array<5x10xf32>>, %arg1: !fir.ref<!fir.array<5x10xf32>>) {
-// expected-remark@+3 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// there are read effects on both arguments - the diagnostic verification just doesn't register duplicate identical diagnostics
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+3 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 1, on resource '<Default>'}}
   %0 = hlfir.matmul_transpose %arg0 %arg1 : (!fir.ref<!fir.array<5x10xf32>>, !fir.ref<!fir.array<5x10xf32>>) -> !hlfir.expr<10x10xf32>
   return
 }
@@ -195,8 +196,8 @@ func.func @associate(%arg0: i32) {
 }
 
 func.func @as_expr_read(%arg0: !fir.ref<!fir.array<2xi32>>) {
-// expected-remark@+2 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   %0 = hlfir.as_expr %arg0 : (!fir.ref<!fir.array<2xi32>>) -> !hlfir.expr<?xi32>
 // expected-remark@+1 {{found an instance of 'free' on resource '<Default>'}}
   hlfir.destroy %0 : !hlfir.expr<?xi32>
@@ -204,28 +205,28 @@ func.func @as_expr_read(%arg0: !fir.ref<!fir.array<2xi32>>) {
 }
 
 func.func @char_extremum(%arg0: !fir.ref<!fir.char<1,10>>, %arg1: !fir.ref<!fir.char<1,20>>) {
-// expected-remark@+3 {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
-// there are read effects on both arguments - the diagnostic verification just doesn't register duplicate identical diagnostics
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+3 {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 1, on resource '<Default>'}}
   %0 = hlfir.char_extremum min, %arg0, %arg1 : (!fir.ref<!fir.char<1, 10>>, !fir.ref<!fir.char<1,20>>) -> !hlfir.expr<!fir.char<1,10>>
   return
 }
 
 func.func @copy_in(%box: !fir.box<!fir.array<?xf64>>, %temp: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, %is_present: i1) {
 // expected-remark@+3 {{found an instance of 'allocate' on resource '<Default>'}}
-// expected-remark@+2 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'write' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'write' on op operand 1, on resource '<Default>'}}
   %0:2 = hlfir.copy_in %box to %temp : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.box<!fir.array<?xf64>>, i1)
   return
 }
 
 func.func @copy_out(%box: !fir.box<!fir.array<?xf64>>, %temp: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, %was_copied: i1) {
 // expected-remark@+2 {{found an instance of 'free' on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
   hlfir.copy_out %temp, %was_copied : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1) -> ()
 // expected-remark@+3 {{found an instance of 'free' on resource '<Default>'}}
-// expected-remark@+2 {{found an instance of 'read' on a op operand, on resource '<Default>'}}
-// expected-remark@+1 {{found an instance of 'write' on a op operand, on resource '<Default>'}}
+// expected-remark@+2 {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+// expected-remark@+1 {{found an instance of 'write' on op operand 2, on resource '<Default>'}}
   hlfir.copy_out %temp, %was_copied to %box : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1, !fir.box<!fir.array<?xf64>>) -> ()
   return
 }
diff --git a/external/llvm-project/flang/test/HLFIR/opt-bufferization-elemental-assign-shape.fir b/external/llvm-project/flang/test/HLFIR/opt-bufferization-elemental-assign-shape.fir
new file mode 100644
index 000000000000..b55848225a41
--- /dev/null
+++ b/external/llvm-project/flang/test/HLFIR/opt-bufferization-elemental-assign-shape.fir
@@ -0,0 +1,54 @@
+// RUN: fir-opt --opt-bufferization %s | FileCheck %s
+
+// Check that the elemental+assign are rewritten into a loop
+// with "optimal" loop bounds, e.g. that we use constants
+// when possible.
+
+// CHECK-LABEL:   func.func @_QPtest1(
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
+// CHECK:           fir.do_loop %[[VAL_6:.*]] = %[[VAL_0]] to %[[VAL_1]] step %[[VAL_0]] unordered {
+// CHECK-NOT: hlfir.assign{{.*}}array
+func.func @_QPtest1(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<3xf32>> {fir.bindc_name = "y"}) {
+  %c0 = arith.constant 0 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtest1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %arg1(%2) dummy_scope %0 {uniq_name = "_QFtest1Ey"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+  ^bb0(%arg2: index):
+    %7 = hlfir.designate %1#0 (%arg2)  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+    %8 = fir.load %7 : !fir.ref<f32>
+    %9 = arith.addf %8, %8 fastmath<contract> : f32
+    hlfir.yield_element %9 : f32
+  }
+  hlfir.assign %6 to %3#0 : !hlfir.expr<?xf32>, !fir.ref<!fir.array<3xf32>>
+  hlfir.destroy %6 : !hlfir.expr<?xf32>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest2(
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
+// CHECK:           fir.do_loop %[[VAL_6:.*]] = %[[VAL_0]] to %[[VAL_1]] step %[[VAL_0]] unordered {
+// CHECK-NOT: hlfir.assign{{.*}}array
+func.func @_QPtest2(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.array<3xf32>> {fir.bindc_name = "y"}) {
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtest2Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %arg1(%2) dummy_scope %0 {uniq_name = "_QFtest2Ey"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
+  %4 = hlfir.elemental %2 unordered : (!fir.shape<1>) -> !hlfir.expr<3xf32> {
+  ^bb0(%arg2: index):
+    %5 = hlfir.designate %3#0 (%arg2)  : (!fir.ref<!fir.array<3xf32>>, index) -> !fir.ref<f32>
+    %6 = fir.load %5 : !fir.ref<f32>
+    %7 = arith.addf %6, %6 fastmath<contract> : f32
+    hlfir.yield_element %7 : f32
+  }
+  hlfir.assign %4 to %1#0 : !hlfir.expr<3xf32>, !fir.box<!fir.array<?xf32>>
+  hlfir.destroy %4 : !hlfir.expr<3xf32>
+  return
+}
diff --git a/external/llvm-project/flang/test/HLFIR/opt-bufferization-tonto.fir b/external/llvm-project/flang/test/HLFIR/opt-bufferization-tonto.fir
new file mode 100644
index 000000000000..df30a90bc6ec
--- /dev/null
+++ b/external/llvm-project/flang/test/HLFIR/opt-bufferization-tonto.fir
@@ -0,0 +1,59 @@
+// RUN: fir-opt --opt-bufferization %s | FileCheck %s
+
+// tonto case where optimized bufferization conservatively
+// did not pull elemental into the assignment due to
+// the designate op in between:
+// module test
+//   type my_type
+//      real, dimension(:,:), pointer :: p => null()
+//   end type my_type
+// contains
+//   subroutine assign1(self, i)
+//     type(my_type) :: self
+//     real, dimension(3) :: ct, q
+//     integer :: i
+//     self%p(:,i) = q - ct
+//   end subroutine assign1
+// end module test
+
+// CHECK-LABEL:   func.func @_QMtestPassign1(
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.assign{{.*}}array
+func.func @_QMtestPassign1(%arg0: !fir.ref<!fir.type<_QMtestTmy_type{p:!fir.box<!fir.ptr<!fir.array<?x?xf32>>>}>> {fir.bindc_name = "self"}, %arg1: !fir.ref<i32> {fir.bindc_name = "i"}) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca !fir.array<3xf32> {bindc_name = "ct", uniq_name = "_QMtestFassign1Ect"}
+  %2 = fir.shape %c3 : (index) -> !fir.shape<1>
+  %3:2 = hlfir.declare %1(%2) {uniq_name = "_QMtestFassign1Ect"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
+  %4:2 = hlfir.declare %arg1 dummy_scope %0 {uniq_name = "_QMtestFassign1Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %5 = fir.alloca !fir.array<3xf32> {bindc_name = "q", uniq_name = "_QMtestFassign1Eq"}
+  %6:2 = hlfir.declare %5(%2) {uniq_name = "_QMtestFassign1Eq"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
+  %7:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QMtestFassign1Eself"} : (!fir.ref<!fir.type<_QMtestTmy_type{p:!fir.box<!fir.ptr<!fir.array<?x?xf32>>>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMtestTmy_type{p:!fir.box<!fir.ptr<!fir.array<?x?xf32>>>}>>, !fir.ref<!fir.type<_QMtestTmy_type{p:!fir.box<!fir.ptr<!fir.array<?x?xf32>>>}>>)
+  %8 = hlfir.elemental %2 unordered : (!fir.shape<1>) -> !hlfir.expr<3xf32> {
+  ^bb0(%arg2: index):
+    %22 = hlfir.designate %6#0 (%arg2)  : (!fir.ref<!fir.array<3xf32>>, index) -> !fir.ref<f32>
+    %23 = hlfir.designate %3#0 (%arg2)  : (!fir.ref<!fir.array<3xf32>>, index) -> !fir.ref<f32>
+    %24 = fir.load %22 : !fir.ref<f32>
+    %25 = fir.load %23 : !fir.ref<f32>
+    %26 = arith.subf %24, %25 fastmath<contract> : f32
+    hlfir.yield_element %26 : f32
+  }
+  %9 = hlfir.designate %7#0{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QMtestTmy_type{p:!fir.box<!fir.ptr<!fir.array<?x?xf32>>>}>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  %10 = fir.load %9 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
+  %11:3 = fir.box_dims %10, %c0 : (!fir.box<!fir.ptr<!fir.array<?x?xf32>>>, index) -> (index, index, index)
+  %12 = arith.addi %11#0, %11#1 : index
+  %13 = arith.subi %12, %c1 : index
+  %14 = arith.subi %13, %11#0 : index
+  %15 = arith.addi %14, %c1 : index
+  %16 = arith.cmpi sgt, %15, %c0 : index
+  %17 = arith.select %16, %15, %c0 : index
+  %18 = fir.load %4#0 : !fir.ref<i32>
+  %19 = fir.convert %18 : (i32) -> i64
+  %20 = fir.shape %17 : (index) -> !fir.shape<1>
+  %21 = hlfir.designate %10 (%11#0:%13:%c1, %19)  shape %20 : (!fir.box<!fir.ptr<!fir.array<?x?xf32>>>, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+  hlfir.assign %8 to %21 : !hlfir.expr<3xf32>, !fir.box<!fir.array<?xf32>>
+  hlfir.destroy %8 : !hlfir.expr<3xf32>
+  return
+}
diff --git a/external/llvm-project/flang/test/HLFIR/reshape-lowering.fir b/external/llvm-project/flang/test/HLFIR/reshape-lowering.fir
index 7ffe38d64eee..a76e73ef8a5a 100644
--- a/external/llvm-project/flang/test/HLFIR/reshape-lowering.fir
+++ b/external/llvm-project/flang/test/HLFIR/reshape-lowering.fir
@@ -261,8 +261,8 @@ func.func @_QPreshape5(%arg0: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir
 // CHECK:           %[[VAL_25:.*]] = fir.load %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
 // CHECK:           %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_25]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
 // CHECK:           %[[VAL_27:.*]] = fir.shift %[[VAL_26]]#0 : (index) -> !fir.shift<1>
-// CHECK:           %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_25]](%[[VAL_27]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
-// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_28]]#0 move %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
+// CHECK:           %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_25]](%[[VAL_27]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_29:.*]] = hlfir.as_expr %[[VAL_28]]#0 move %[[VAL_2]] : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
 // CHECK:           hlfir.assign %[[VAL_29]] to %[[VAL_8]]#0 : !hlfir.expr<?x!fir.type<_QMtypesTt>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
 // CHECK:           hlfir.destroy %[[VAL_29]] : !hlfir.expr<?x!fir.type<_QMtypesTt>>
 // CHECK:           return
@@ -318,8 +318,8 @@ func.func @_QPreshape6(%arg0: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.ty
 // CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_6]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
 // CHECK:           %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_4]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
 // CHECK:           %[[VAL_28:.*]] = fir.shift %[[VAL_27]]#0 : (index) -> !fir.shift<1>
-// CHECK:           %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_26]](%[[VAL_28]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
-// CHECK:           %[[VAL_30:.*]] = hlfir.as_expr %[[VAL_29]]#0 move %[[VAL_2]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+// CHECK:           %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_26]](%[[VAL_28]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_30:.*]] = hlfir.as_expr %[[VAL_29]]#0 move %[[VAL_2]] : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
 // CHECK:           hlfir.assign %[[VAL_30]] to %[[VAL_8]]#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
 // CHECK:           hlfir.destroy %[[VAL_30]] : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
 // CHECK:           return
@@ -375,8 +375,8 @@ func.func @_QPreshape7(%arg0: !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>> {fir
 // CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
 // CHECK:           %[[VAL_27:.*]]:3 = fir.box_dims %[[VAL_26]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
 // CHECK:           %[[VAL_28:.*]] = fir.shift %[[VAL_27]]#0 : (index) -> !fir.shift<1>
-// CHECK:           %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_26]](%[[VAL_28]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
-// CHECK:           %[[VAL_30:.*]] = hlfir.as_expr %[[VAL_29]]#0 move %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
+// CHECK:           %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_26]](%[[VAL_28]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_30:.*]] = hlfir.as_expr %[[VAL_29]]#0 move %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>>
 // CHECK:           hlfir.assign %[[VAL_30]] to %[[VAL_10]]#0 : !hlfir.expr<?x!fir.type<_QMtypesTt>>, !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
 // CHECK:           hlfir.destroy %[[VAL_30]] : !hlfir.expr<?x!fir.type<_QMtypesTt>>
 // CHECK:           return
@@ -435,8 +435,8 @@ func.func @_QPreshape8(%arg0: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.ty
 // CHECK:           %[[VAL_27:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
 // CHECK:           %[[VAL_28:.*]]:3 = fir.box_dims %[[VAL_27]], %[[VAL_5]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, index) -> (index, index, index)
 // CHECK:           %[[VAL_29:.*]] = fir.shift %[[VAL_28]]#0 : (index) -> !fir.shift<1>
-// CHECK:           %[[VAL_30:.*]]:2 = hlfir.declare %[[VAL_27]](%[[VAL_29]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>)
-// CHECK:           %[[VAL_31:.*]] = hlfir.as_expr %[[VAL_30]]#0 move %[[VAL_3]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+// CHECK:           %[[VAL_30:.*]]:2 = hlfir.declare %[[VAL_27]](%[[VAL_29]]) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.shift<1>) -> (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>)
+// CHECK:           %[[VAL_31:.*]] = hlfir.as_expr %[[VAL_30]]#0 move %[[VAL_3]] : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i1) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
 // CHECK:           hlfir.assign %[[VAL_31]] to %[[VAL_10]]#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
 // CHECK:           hlfir.destroy %[[VAL_31]] : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
 // CHECK:           return
diff --git a/external/llvm-project/flang/test/Integration/amdgpu/debug-target-var.f90 b/external/llvm-project/flang/test/Integration/amdgpu/debug-target-var.f90
index efed9a33e28d..f47cf1e4405d 100644
--- a/external/llvm-project/flang/test/Integration/amdgpu/debug-target-var.f90
+++ b/external/llvm-project/flang/test/Integration/amdgpu/debug-target-var.f90
@@ -19,8 +19,8 @@ end subroutine fff
 ! CHECK-DAG: store ptr %[[ARG2]], ptr %[[CAST2:[0-9]+]]{{.*}}
 ! CHECK-DAG: %[[CAST2]] = addrspacecast ptr addrspace(5) %[[AL2:[0-9]+]]
 ! CHECK-DAG: %[[AL2]] = alloca{{.*}}
-! CHECK-DAG: #dbg_declare(ptr %[[CAST1]], ![[X:[0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(ptr), DIOpDeref(ptr)), {{.*}})
-! CHECK-DAG: #dbg_declare(ptr %[[CAST2]], ![[Y:[0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(ptr), DIOpDeref(ptr)), {{.*}})
+! CHECK-DAG: #dbg_declare(ptr {{.*}}%[[AL1]], ![[X:[0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(ptr), DIOpDeref(ptr)), {{.*}})
+! CHECK-DAG: #dbg_declare(ptr {{.*}}%[[AL2]], ![[Y:[0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(ptr), DIOpDeref(ptr)), {{.*}})
 ! CHECK: }
 
 ! CHECK-DAG: ![[SP]] = {{.*}}!DISubprogram(name: "[[FN]]"{{.*}})
diff --git a/external/llvm-project/flang/test/Integration/debug-local-var-2.f90 b/external/llvm-project/flang/test/Integration/debug-local-var-2.f90
index 468bb0c5a128..b97be141cc8d 100644
--- a/external/llvm-project/flang/test/Integration/debug-local-var-2.f90
+++ b/external/llvm-project/flang/test/Integration/debug-local-var-2.f90
@@ -1,6 +1,5 @@
-! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -mllvm --experimental-debuginfo-iterators=true -o - | FileCheck %s --check-prefixes=BOTH,RECORDS
-! RUN: %flang_fc1 -emit-llvm -debug-info-kind=line-tables-only %s -mllvm --experimental-debuginfo-iterators=false -o - | FileCheck --check-prefix=LINEONLY %s
-! RUN: %flang_fc1 -emit-llvm -debug-info-kind=line-tables-only %s -mllvm --experimental-debuginfo-iterators=true -o - | FileCheck --check-prefix=LINEONLY %s
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck %s --check-prefixes=BOTH,RECORDS
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=line-tables-only %s -o - | FileCheck --check-prefix=LINEONLY %s
 
 ! This tests checks the debug information for local variables in llvm IR.
 
diff --git a/external/llvm-project/flang/test/Integration/no-malloc-private.f90 b/external/llvm-project/flang/test/Integration/no-malloc-private.f90
index 25aeb1dd7c47..176af39c2de2 100644
--- a/external/llvm-project/flang/test/Integration/no-malloc-private.f90
+++ b/external/llvm-project/flang/test/Integration/no-malloc-private.f90
@@ -1,5 +1,4 @@
 ! RUN: %flang_fc1 -emit-llvm -fopenmp -o - -x f95 %s | FileCheck %s
-
 subroutine foo(state,ilast,jlast,vals)
   real, intent(in) :: state(:,:)
   integer, intent(in) :: ilast, jlast
diff --git a/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_declare_target.f90 b/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_declare_target.f90
new file mode 100644
index 000000000000..6addd21d9a45
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_declare_target.f90
@@ -0,0 +1,24 @@
+! RUN: %flang -ffast-amd-memory-allocator -S -emit-llvm -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa --offload-arch=gfx90a -o - %s | FileCheck %s --check-prefix=CHECK-OMP
+! RUN: %flang -ffast-amd-memory-allocator -S -emit-llvm -target amdgcn-- -o - %s | FileCheck %s --check-prefix=CHECK
+!REQUIRES: AFAR
+subroutine func_t_device()
+  !$omp declare target enter(func_t_device) device_type(nohost)
+    integer, ALLOCATABLE :: poly
+
+! CHECK-OMP-NOT: call void @_FortranAAMDAllocatableSetAllocIdx({{.*}}, i32 1)
+! CHECK: call void @_FortranAAMDAllocatableSetAllocIdx({{.*}}, i32 1)
+! CHECK-OMP: call i32 @_FortranAAllocatableAllocate
+! CHECK: call i32 @_FortranAAllocatableAllocate   
+    ALLOCATE(poly)
+
+! CHECK-OMP: call i32 @_FortranAAllocatableDeallocate
+! CHECK: call i32 @_FortranAAllocatableDeallocate
+    DEALLOCATE(poly)
+end subroutine func_t_device
+
+program main
+  implicit none
+  !$omp target
+    call func_t_device()
+  !$omp end target
+end program
diff --git a/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_declare_target_nested.f90 b/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_declare_target_nested.f90
new file mode 100644
index 000000000000..fbd9ba1ac4a0
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_declare_target_nested.f90
@@ -0,0 +1,25 @@
+! RUN: %flang -ffast-amd-memory-allocator -S -emit-llvm -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa --offload-arch=gfx90a -o - %s | FileCheck %s --check-prefix=CHECK-OMP
+! RUN: %flang -ffast-amd-memory-allocator -S -emit-llvm -target amdgcn-- -o - %s | FileCheck %s --check-prefix=CHECK
+!REQUIRES: AFAR
+subroutine func_t_device()
+  !$omp declare target enter(func_t_device) device_type(nohost)
+    integer, ALLOCATABLE :: poly
+    do j=1,10
+! CHECK-OMP-NOT: call void @_FortranAAMDAllocatableSetAllocIdx({{.*}}, i32 1)
+! CHECK: call void @_FortranAAMDAllocatableSetAllocIdx({{.*}}, i32 1)
+! CHECK-OMP: call i32 @_FortranAAllocatableAllocate
+! CHECK: call i32 @_FortranAAllocatableAllocate
+      ALLOCATE(poly)
+
+! CHECK-OMP: call i32 @_FortranAAllocatableDeallocate
+! CHECK: call i32 @_FortranAAllocatableDeallocate
+      DEALLOCATE(poly)
+    end do
+end subroutine func_t_device
+
+program main
+  implicit none
+  !$omp target
+    call func_t_device()
+  !$omp end target
+end program
diff --git a/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_target.f90 b/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_target.f90
new file mode 100644
index 000000000000..d4b8ad28120f
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/AMDGPU/allocate_deallocate_omp_target.f90
@@ -0,0 +1,24 @@
+! RUN: %flang -ffast-amd-memory-allocator -S -emit-llvm -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa --offload-arch=gfx90a -o - %s | FileCheck %s --check-prefix=CHECK-OMP
+! RUN: %flang -ffast-amd-memory-allocator -S -emit-llvm -target amdgcn-- -o - %s | FileCheck %s --check-prefix=CHECK
+!REQUIRES: AFAR
+program main
+   implicit none
+   !$omp requires unified_shared_memory
+   REAL, DIMENSION(:), ALLOCATABLE :: poly
+   integer,parameter :: n = 10
+   integer :: i,j
+     !$omp target teams distribute parallel do private(poly)
+     do j=1,n
+
+! CHECK-OMP-NOT: call void @_FortranAAMDAllocatableSetAllocIdx({{.*}}, i32 1)
+! CHECK: call void @_FortranAAMDAllocatableSetAllocIdx({{.*}}, i32 1)
+! CHECK-OMP: call i32 @_FortranAAllocatableAllocate
+! CHECK: call i32 @_FortranAAllocatableAllocate
+       ALLOCATE(poly(1:3))
+       poly = 2.0_8
+! CHECK-OMP: call i32 @_FortranAAllocatableDeallocate
+! CHECK: call i32 @_FortranAAllocatableDeallocate
+       DEALLOCATE(poly)
+     enddo
+     !$omp end target teams distribute parallel do
+end program
diff --git a/external/llvm-project/flang/test/Lower/CUDA/cuda-runtime-check.cuf b/external/llvm-project/flang/test/Lower/CUDA/cuda-runtime-check.cuf
new file mode 100644
index 000000000000..f26d372769ca
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/CUDA/cuda-runtime-check.cuf
@@ -0,0 +1,22 @@
+! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
+
+! Check insertion of runtime checks
+
+interface
+  subroutine foo(a)
+    real, device, dimension(:,:) :: a
+  end subroutine
+end interface
+
+  real, device, allocatable, dimension(:,:) :: a
+  allocate(a(10,10))
+  call foo(a(1:10,1:10:2))
+end
+
+subroutine foo(a)
+  real, device, dimension(:,:) :: a
+end subroutine
+
+! CHECK-LABEL: func.func @_QQmain()
+! CHECK: fir.call @_FortranACUFDescriptorCheckSection
+! CHECK: fir.call @_QPfoo
diff --git a/external/llvm-project/flang/test/Lower/HLFIR/assumed-rank-entry.f90 b/external/llvm-project/flang/test/Lower/HLFIR/assumed-rank-entry.f90
index 9ac56c4282f8..d2e470ab85a4 100644
--- a/external/llvm-project/flang/test/Lower/HLFIR/assumed-rank-entry.f90
+++ b/external/llvm-project/flang/test/Lower/HLFIR/assumed-rank-entry.f90
@@ -25,4 +25,4 @@ subroutine some_proc(x)
 ! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.box<!fir.heap<f32>>) -> !fir.box<!fir.heap<!fir.array<*:f32>>>
 ! CHECK:           fir.store %[[VAL_3]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFtest_main_entryEx"} : (!fir.box<!fir.heap<!fir.array<*:f32>>>) -> (!fir.box<!fir.heap<!fir.array<*:f32>>>, !fir.box<!fir.heap<!fir.array<*:f32>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFtest_main_entryEx"} : (!fir.box<!fir.heap<!fir.array<*:f32>>>) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
diff --git a/external/llvm-project/flang/test/Lower/HLFIR/function-return-as-expr.f90 b/external/llvm-project/flang/test/Lower/HLFIR/function-return-as-expr.f90
index f7e7c161e81b..3b43b6e261db 100644
--- a/external/llvm-project/flang/test/Lower/HLFIR/function-return-as-expr.f90
+++ b/external/llvm-project/flang/test/Lower/HLFIR/function-return-as-expr.f90
@@ -67,8 +67,8 @@ end function inner
 end subroutine test4
 ! CHECK-LABEL:   func.func @_QPtest4() {
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_0:.*]] : !fir.ref<!fir.class<!fir.heap<none>>>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = ".tmp.func_result"} : (!fir.class<!fir.heap<none>>) -> (!fir.class<!fir.heap<none>>, !fir.class<!fir.heap<none>>)
-! CHECK:           hlfir.assign %[[VAL_7]]#0 to %{{.*}}#0 realloc : !fir.class<!fir.heap<none>>, !fir.ref<!fir.class<!fir.heap<none>>>
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = ".tmp.func_result"} : (!fir.class<!fir.heap<none>>) -> (!fir.class<none>, !fir.class<none>)
+! CHECK:           hlfir.assign %[[VAL_7]]#0 to %{{.*}}#0 realloc : !fir.class<none>, !fir.ref<!fir.class<!fir.heap<none>>>
 ! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<none>>>) -> !fir.box<none>
 ! CHECK:           fir.call @_FortranADestroy(%[[VAL_10]]) fastmath<contract> : (!fir.box<none>) -> ()
 
@@ -82,8 +82,8 @@ end function inner
 end subroutine test4b
 ! CHECK-LABEL:   func.func @_QPtest4b() {
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_0:.*]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = ".tmp.func_result"} : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> (!fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.class<!fir.heap<!fir.array<?x?xnone>>>)
-! CHECK:           hlfir.assign %[[VAL_7]]#0 to %{{.*}}#0 realloc : !fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = ".tmp.func_result"} : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
+! CHECK:           hlfir.assign %[[VAL_7]]#0 to %{{.*}}#0 realloc : !fir.class<!fir.array<?x?xnone>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
 ! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.box<none>
 ! CHECK:           fir.call @_FortranADestroy(%[[VAL_10]]) fastmath<contract> : (!fir.box<none>) -> ()
 
diff --git a/external/llvm-project/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90 b/external/llvm-project/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90
index 26d19c308fea..822a0bcc20e5 100644
--- a/external/llvm-project/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90
+++ b/external/llvm-project/flang/test/Lower/HLFIR/poly_expr_for_nonpoly_dummy.f90
@@ -20,12 +20,12 @@ end subroutine test1
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_23:.*]] = hlfir.cshift %{{.*}} %[[VAL_4]] : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i32) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
 ! CHECK:           %[[VAL_26:.*]] = hlfir.shape_of %[[VAL_23]] : (!hlfir.expr<?x!fir.type<_QMtypesTt>?>) -> !fir.shape<1>
-! CHECK:           %[[VAL_27:.*]]:3 = hlfir.associate %[[VAL_23]](%[[VAL_26]]) {adapt.valuebyref} : (!hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.shape<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1)
-! CHECK:           %[[VAL_28:.*]] = fir.rebox %[[VAL_27]]#0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
+! CHECK:           %[[VAL_27:.*]]:3 = hlfir.associate %[[VAL_23]](%[[VAL_26]]) {adapt.valuebyref} : (!hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.shape<1>) -> (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i1)
+! CHECK:           %[[VAL_28:.*]] = fir.rebox %[[VAL_27]]#0 : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
 ! CHECK:           %[[VAL_29:.*]]:2 = hlfir.copy_in %[[VAL_28]] to %[[TMP_BOX:.*]] : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, i1)
 ! CHECK:           fir.call @_QMtypesPcallee(%[[VAL_29]]#0) fastmath<contract> : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>) -> ()
 ! CHECK:           hlfir.copy_out %[[TMP_BOX]], %[[VAL_29]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, i1) -> ()
-! CHECK:           hlfir.end_associate %[[VAL_27]]#0, %[[VAL_27]]#2 : !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1
+! CHECK:           hlfir.end_associate %[[VAL_27]]#0, %[[VAL_27]]#2 : !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i1
 ! CHECK:           hlfir.destroy %[[VAL_23]] : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
 
 subroutine test2(x)
@@ -35,10 +35,10 @@ subroutine test2(x)
 end subroutine test2
 ! CHECK-LABEL:   func.func @_QPtest2(
 ! CHECK:           %[[VAL_5:.*]] = hlfir.elemental %{{.*}} mold %{{.*}} unordered : (!fir.shape<1>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?> {
-! CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_5]](%{{.*}}) {adapt.valuebyref} : (!hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.shape<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1)
-! CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_9]]#0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
+! CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_5]](%{{.*}}) {adapt.valuebyref} : (!hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.shape<1>) -> (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i1)
+! CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_9]]#0 : (!fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>) -> !fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.copy_in %[[VAL_10]] to %[[TMP_BOX:.*]] : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>, i1)
 ! CHECK:           fir.call @_QMtypesPcallee(%[[VAL_11]]#0) fastmath<contract> : (!fir.box<!fir.array<?x!fir.type<_QMtypesTt>>>) -> ()
 ! CHECK:           hlfir.copy_out %[[TMP_BOX]], %[[VAL_11]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, i1) -> ()
-! CHECK:           hlfir.end_associate %[[VAL_9]]#0, %[[VAL_9]]#2 : !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i1
+! CHECK:           hlfir.end_associate %[[VAL_9]]#0, %[[VAL_9]]#2 : !fir.class<!fir.array<?x!fir.type<_QMtypesTt>>>, i1
 ! CHECK:           hlfir.destroy %[[VAL_5]] : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
diff --git a/external/llvm-project/flang/test/Lower/HLFIR/polymorphic-expressions.f90 b/external/llvm-project/flang/test/Lower/HLFIR/polymorphic-expressions.f90
index d6934e6ae6bd..0731190f465d 100644
--- a/external/llvm-project/flang/test/Lower/HLFIR/polymorphic-expressions.f90
+++ b/external/llvm-project/flang/test/Lower/HLFIR/polymorphic-expressions.f90
@@ -20,14 +20,13 @@ end subroutine callee
   call callee(spread(a, 1, 2))
 end subroutine test1
 ! CHECK-LABEL:   func.func @_QPtest1(
-! CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, !fir.shift<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>)
+! CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = ".tmp.intrinsic_result"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, !fir.shift<1>) -> (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>)
 ! CHECK:           %[[VAL_22:.*]] = arith.constant true
-! CHECK:           %[[VAL_23:.*]] = hlfir.as_expr %[[VAL_21]]#0 move %[[VAL_22]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, i1) -> !hlfir.expr<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>?>
+! CHECK:           %[[VAL_23:.*]] = hlfir.as_expr %[[VAL_21]]#0 move %[[VAL_22]] : (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>, i1) -> !hlfir.expr<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>?>
 ! CHECK:           %[[VAL_24:.*]] = arith.constant 0 : index
-! CHECK:           %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_21]]#0, %[[VAL_24]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_25:.*]]:3 = fir.box_dims %[[VAL_21]]#0, %[[VAL_24]] : (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_26:.*]] = fir.shape %[[VAL_25]]#1 : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_27:.*]]:3 = hlfir.associate %[[VAL_23]](%[[VAL_26]]) {adapt.valuebyref} : (!hlfir.expr<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>?>, !fir.shape<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, i1)
-! CHECK:           %[[VAL_28:.*]] = fir.rebox %[[VAL_27]]#0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>) -> !fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>
-! CHECK:           fir.call @_QPcallee(%[[VAL_28]]) fastmath<contract> : (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>) -> ()
-! CHECK:           hlfir.end_associate %[[VAL_27]]#0, %[[VAL_27]]#2 : !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>>, i1
+! CHECK:           %[[VAL_27:.*]]:3 = hlfir.associate %[[VAL_23]](%[[VAL_26]]) {adapt.valuebyref} : (!hlfir.expr<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>?>, !fir.shape<1>) -> (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>, i1)
+! CHECK:           fir.call @_QPcallee(%[[VAL_27]]#0) fastmath<contract> : (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_27]]#0, %[[VAL_27]]#2 : !fir.class<!fir.array<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>>>, i1
 ! CHECK:           hlfir.destroy %[[VAL_23]] : !hlfir.expr<?x!fir.type<_QMpolymorphic_expressions_typesTt{c:i32}>?>
diff --git a/external/llvm-project/flang/test/Lower/HLFIR/select-type-selector.f90 b/external/llvm-project/flang/test/Lower/HLFIR/select-type-selector.f90
index fb8a04f6896a..603b154a9ddd 100644
--- a/external/llvm-project/flang/test/Lower/HLFIR/select-type-selector.f90
+++ b/external/llvm-project/flang/test/Lower/HLFIR/select-type-selector.f90
@@ -45,9 +45,9 @@ end subroutine test
 ! CHECK:         ^bb1:
 ! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.box<!fir.heap<!fir.array<?x!fir.type<_QFtestTt>>>>
 ! CHECK:           %[[VAL_13:.*]] = fir.shift %[[VAL_11]]#0 : (index) -> !fir.shift<1>
-! CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]](%[[VAL_13]]) {uniq_name = "_QFtestEx"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QFtestTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QFtestTt>>>>, !fir.box<!fir.heap<!fir.array<?x!fir.type<_QFtestTt>>>>)
+! CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]](%[[VAL_13]]) {uniq_name = "_QFtestEx"} : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QFtestTt>>>>, !fir.shift<1>) -> (!fir.box<!fir.array<?x!fir.type<_QFtestTt>>>, !fir.box<!fir.array<?x!fir.type<_QFtestTt>>>)
 ! CHECK:           %[[VAL_15:.*]] = arith.constant 0 : index
-! CHECK:           %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_14]]#0, %[[VAL_15]] : (!fir.box<!fir.heap<!fir.array<?x!fir.type<_QFtestTt>>>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_14]]#0, %[[VAL_15]] : (!fir.box<!fir.array<?x!fir.type<_QFtestTt>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]]#1 : (index) -> i64
 ! CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_11]]#0 : (index) -> i64
 ! CHECK:           %[[VAL_19:.*]] = arith.addi %[[VAL_17]], %[[VAL_18]] : i64
diff --git a/external/llvm-project/flang/test/Lower/HLFIR/vector-subscript-as-value.f90 b/external/llvm-project/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
index 31be1293fa74..6b693692b4d2 100644
--- a/external/llvm-project/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
+++ b/external/llvm-project/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
@@ -209,10 +209,9 @@ subroutine do_something(x)
 ! CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_4]], %[[VAL_11]])  : (!fir.class<!fir.array<?x?xnone>>, index, i64) -> !fir.class<none>
 ! CHECK:             hlfir.yield_element %[[VAL_12]] : !fir.class<none>
 ! CHECK:           }
-! CHECK:           %[[VAL_13:.*]]:3 = hlfir.associate %[[VAL_8]](%[[VAL_7]]) {adapt.valuebyref} : (!hlfir.expr<?xnone?>, !fir.shape<1>) -> (!fir.class<!fir.heap<!fir.array<?xnone>>>, !fir.class<!fir.heap<!fir.array<?xnone>>>, i1)
-! CHECK:           %[[VAL_14:.*]] = fir.rebox %[[VAL_13]]#0 : (!fir.class<!fir.heap<!fir.array<?xnone>>>) -> !fir.class<!fir.array<?xnone>>
-! CHECK:           fir.call @_QPdo_something(%[[VAL_14]]) fastmath<contract> : (!fir.class<!fir.array<?xnone>>) -> ()
-! CHECK:           hlfir.end_associate %[[VAL_13]]#0, %[[VAL_13]]#2 : !fir.class<!fir.heap<!fir.array<?xnone>>>, i1
+! CHECK:           %[[VAL_13:.*]]:3 = hlfir.associate %[[VAL_8]](%[[VAL_7]]) {adapt.valuebyref} : (!hlfir.expr<?xnone?>, !fir.shape<1>) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>, i1)
+! CHECK:           fir.call @_QPdo_something(%[[VAL_13]]#0) fastmath<contract> : (!fir.class<!fir.array<?xnone>>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_13]]#0, %[[VAL_13]]#2 : !fir.class<!fir.array<?xnone>>, i1
 ! CHECK:           hlfir.destroy %[[VAL_8]] : !hlfir.expr<?xnone?>
 ! CHECK:           return
 ! CHECK:         }
diff --git a/external/llvm-project/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/external/llvm-project/flang/test/Lower/OpenACC/acc-kernels-loop.f90
index 8608b0ad98ce..4e968144399a 100644
--- a/external/llvm-project/flang/test/Lower/OpenACC/acc-kernels-loop.f90
+++ b/external/llvm-project/flang/test/Lower/OpenACC/acc-kernels-loop.f90
@@ -47,7 +47,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels {
 ! CHECK:        acc.loop private{{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>]{{.*}}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -59,7 +59,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels combined(loop) {
 ! CHECK:        acc.loop combined(kernels) private{{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>]{{.*}}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -490,7 +490,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} gang {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -503,7 +503,7 @@ subroutine acc_kernels_loop
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
 ! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -516,7 +516,7 @@ subroutine acc_kernels_loop
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -528,7 +528,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} gang({num=%{{.*}} : i32, static=%{{.*}} : i32})
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -540,7 +540,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} vector {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -553,7 +553,7 @@ subroutine acc_kernels_loop
 ! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
 ! CHECK:        acc.loop {{.*}} vector([[CONSTANT128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -566,7 +566,7 @@ subroutine acc_kernels_loop
 ! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:        acc.loop {{.*}} vector([[VECTORLENGTH]] : i32) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -578,7 +578,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} worker {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -591,7 +591,7 @@ subroutine acc_kernels_loop
 ! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
 ! CHECK:        acc.loop {{.*}} worker([[WORKER128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -605,7 +605,7 @@ subroutine acc_kernels_loop
 ! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true>}
+! CHECK-NEXT:   } attributes {{{.*}}collapse = [2], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
@@ -621,9 +621,9 @@ subroutine acc_kernels_loop
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:            acc.loop {{.*}} {
 ! CHECK:              acc.yield
-! CHECK-NEXT:     }{{$}}
+! CHECK-NEXT:     } attributes {auto_ = [#acc.device_type<none>]{{.*}}}
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>]{{.*}}}
 ! CHECK:        acc.terminator
 ! CHECK-NEXT: }{{$}}
 
diff --git a/external/llvm-project/flang/test/Lower/OpenACC/acc-loop.f90 b/external/llvm-project/flang/test/Lower/OpenACC/acc-loop.f90
index 0246f6070589..5baa485534b2 100644
--- a/external/llvm-project/flang/test/Lower/OpenACC/acc-loop.f90
+++ b/external/llvm-project/flang/test/Lower/OpenACC/acc-loop.f90
@@ -29,7 +29,7 @@ program acc_loop
 
 ! CHECK: acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}{{$}}
 
  !$acc loop seq
   DO i = 1, n
@@ -65,7 +65,7 @@ program acc_loop
 
 ! CHECK:      acc.loop gang private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop gang(num: 8)
   DO i = 1, n
@@ -75,7 +75,7 @@ program acc_loop
 ! CHECK:      [[GANGNUM1:%.*]] = arith.constant 8 : i32
 ! CHECK:      acc.loop gang({num=[[GANGNUM1]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop gang(num: gangNum)
   DO i = 1, n
@@ -85,7 +85,7 @@ program acc_loop
 ! CHECK:      [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:      acc.loop gang({num=[[GANGNUM2]] : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
  !$acc loop gang(num: gangNum, static: gangStatic)
   DO i = 1, n
@@ -94,7 +94,7 @@ program acc_loop
 
 ! CHECK: acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop vector
   DO i = 1, n
@@ -103,7 +103,7 @@ program acc_loop
 
 ! CHECK:      acc.loop vector private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop vector(128)
   DO i = 1, n
@@ -113,7 +113,7 @@ program acc_loop
 ! CHECK: [[CONSTANT128:%.*]] = arith.constant 128 : i32
 ! CHECK:      acc.loop vector([[CONSTANT128]] : i32) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: }{{$}}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop vector(vectorLength)
   DO i = 1, n
@@ -123,7 +123,7 @@ program acc_loop
 ! CHECK:      [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:      acc.loop vector([[VECTORLENGTH]] : i32) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
 !$acc loop worker
   DO i = 1, n
@@ -132,7 +132,7 @@ program acc_loop
 
 ! CHECK:      acc.loop worker private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop worker(128)
   DO i = 1, n
@@ -142,7 +142,7 @@ program acc_loop
 ! CHECK: [[WORKER128:%.*]] = arith.constant 128 : i32
 ! CHECK:      acc.loop worker([[WORKER128]] : i32) private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop private(c)
   DO i = 1, n
@@ -151,7 +151,7 @@ program acc_loop
 
 ! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   ! When the induction variable is explicitly private - only a single private entry should be created.
   !$acc loop private(i)
@@ -161,7 +161,7 @@ program acc_loop
 
 ! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop private(c, d)
   DO i = 1, n
@@ -170,7 +170,7 @@ program acc_loop
 
 ! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop private(c) private(d)
   DO i = 1, n
@@ -179,7 +179,7 @@ program acc_loop
 
 ! CHECK:      acc.loop private(@privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_10x10xf32 -> %{{.*}} : !fir.ref<!fir.array<10x10xf32>>, @privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop tile(2)
   DO i = 1, n
@@ -189,7 +189,7 @@ program acc_loop
 ! CHECK:      [[TILESIZE:%.*]] = arith.constant 2 : i32
 ! CHECK:      acc.loop {{.*}} tile({[[TILESIZE]] : i32}) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
  !$acc loop tile(*)
   DO i = 1, n
@@ -198,7 +198,7 @@ program acc_loop
 ! CHECK:      [[TILESIZEM1:%.*]] = arith.constant -1 : i32
 ! CHECK:      acc.loop {{.*}} tile({[[TILESIZEM1]] : i32}) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop tile(2, 2)
   DO i = 1, n
@@ -211,7 +211,7 @@ program acc_loop
 ! CHECK:      [[TILESIZE2:%.*]] = arith.constant 2 : i32
 ! CHECK:      acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop tile(tileSize)
   DO i = 1, n
@@ -220,7 +220,7 @@ program acc_loop
 
 ! CHECK:      acc.loop {{.*}} tile({%{{.*}} : i32}) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop tile(tileSize, tileSize)
   DO i = 1, n
@@ -231,7 +231,7 @@ program acc_loop
 
 ! CHECK:      acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop collapse(2)
   DO i = 1, n
@@ -244,7 +244,7 @@ program acc_loop
 ! CHECK:        fir.store %arg0 to %{{.*}} : !fir.ref<i32>
 ! CHECK:        fir.store %arg1 to %{{.*}} : !fir.ref<i32>
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true>}
+! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
 
   !$acc loop
   DO i = 1, n
@@ -257,9 +257,9 @@ program acc_loop
 ! CHECK:      acc.loop {{.*}} control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:          acc.loop {{.*}} control(%arg1 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:            acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop reduction(+:reduction_r) reduction(*:reduction_i)
   do i = 1, n
@@ -269,7 +269,7 @@ program acc_loop
 
 ! CHECK:      acc.loop private(@privatization_ref_i32 -> %{{.*}} : !fir.ref<i32>) reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
  !$acc loop gang(dim: gangDim, static: gangStatic)
   DO i = 1, n
@@ -278,7 +278,7 @@ program acc_loop
 
 ! CHECK: acc.loop gang({dim=%{{.*}}, static=%{{.*}} : i32}) {{.*}} control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop gang(dim: 1)
   DO i = 1, n
@@ -287,7 +287,7 @@ program acc_loop
 
 ! CHECK:      acc.loop gang({dim={{.*}} : i32}) {{.*}} control(%arg0 : i32) = (%{{.*}} : i32) to (%{{.*}} : i32) step (%{{.*}} : i32) {
 ! CHECK:        acc.yield
-! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>}
+! CHECK-NEXT: } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
   !$acc loop
   DO i = 1, n
@@ -335,4 +335,4 @@ subroutine sub1(i, j, k)
 ! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "j"}
 ! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = "k"}
 ! CHECK: acc.loop combined(parallel) private(@privatization_ref_i32 -> %[[P_I]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_J]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_K]] : !fir.ref<i32>) control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32)  step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
-! CHECK: } attributes {inclusiveUpperbound = array<i1: true, true, true>}
+! CHECK: } attributes {inclusiveUpperbound = array<i1: true, true, true>, independent = [#acc.device_type<none>]}
diff --git a/external/llvm-project/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/external/llvm-project/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index 4cf268d2517f..32060179acdf 100644
--- a/external/llvm-project/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/external/llvm-project/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -49,7 +49,7 @@ subroutine acc_parallel_loop
 ! CHECK:      acc.parallel {
 ! CHECK:        acc.loop private{{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {{{.*}}independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -61,7 +61,7 @@ subroutine acc_parallel_loop
 ! CHECK:      acc.parallel combined(loop) {
 ! CHECK:        acc.loop combined(parallel) private{{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {{{.*}}independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -505,7 +505,7 @@ subroutine acc_parallel_loop
 ! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} gang
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -518,7 +518,7 @@ subroutine acc_parallel_loop
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
 ! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32})
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -531,7 +531,7 @@ subroutine acc_parallel_loop
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32})
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -543,7 +543,7 @@ subroutine acc_parallel_loop
 ! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} gang({num=%{{.*}} : i32, static=%{{.*}} : i32})
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -555,7 +555,7 @@ subroutine acc_parallel_loop
 ! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} vector
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -568,7 +568,7 @@ subroutine acc_parallel_loop
 ! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
 ! CHECK:        acc.loop {{.*}} vector([[CONSTANT128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -582,7 +582,7 @@ subroutine acc_parallel_loop
 ! CHECK:        acc.loop {{.*}} vector([[VECTORLENGTH]] : i32) {{.*}} {
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -595,7 +595,7 @@ subroutine acc_parallel_loop
 ! CHECK:        acc.loop {{.*}} worker {{.*}} {
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -609,7 +609,7 @@ subroutine acc_parallel_loop
 ! CHECK:        acc.loop {{.*}} worker([[WORKER128]] : i32) {{.*}} {
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -623,7 +623,7 @@ subroutine acc_parallel_loop
 ! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true>}
+! CHECK-NEXT:   } attributes {{{.*}}collapse = [2], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -639,9 +639,9 @@ subroutine acc_parallel_loop
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:            acc.loop {{.*}} {
 ! CHECK:              acc.yield
-! CHECK-NEXT:     }{{$}}
+! CHECK-NEXT:     } attributes {{{.*}}independent = [#acc.device_type<none>]}
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {{{.*}}independent = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
diff --git a/external/llvm-project/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/external/llvm-project/flang/test/Lower/OpenACC/acc-serial-loop.f90
index 34391f78ae70..af7bb0fac158 100644
--- a/external/llvm-project/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/external/llvm-project/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -68,7 +68,7 @@ subroutine acc_serial_loop
 ! CHECK:      acc.serial {
 ! CHECK:        acc.loop private{{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {{{.*}}seq = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -80,7 +80,7 @@ subroutine acc_serial_loop
 ! CHECK:      acc.serial combined(loop) {
 ! CHECK:        acc.loop combined(serial) private{{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {{{.*}}seq = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -446,7 +446,7 @@ subroutine acc_serial_loop
 ! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} gang {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -459,7 +459,7 @@ subroutine acc_serial_loop
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
 ! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -472,7 +472,7 @@ subroutine acc_serial_loop
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:        acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -484,7 +484,7 @@ subroutine acc_serial_loop
 ! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -496,7 +496,7 @@ subroutine acc_serial_loop
 ! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} vector {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -509,7 +509,7 @@ subroutine acc_serial_loop
 ! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
 ! CHECK:        acc.loop {{.*}} vector([[CONSTANT128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -522,7 +522,7 @@ subroutine acc_serial_loop
 ! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:        acc.loop {{.*}} vector([[VECTORLENGTH]] : i32) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -534,7 +534,7 @@ subroutine acc_serial_loop
 ! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} worker {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -547,7 +547,7 @@ subroutine acc_serial_loop
 ! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
 ! CHECK:        acc.loop {{.*}} worker([[WORKER128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -562,7 +562,7 @@ subroutine acc_serial_loop
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK-NOT:            fir.do_loop
 ! CHECK:          acc.yield
-! CHECK-NEXT:   } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true>}
+! CHECK-NEXT:   } attributes {{{.*}}collapse = [2], collapseDeviceType = [#acc.device_type<none>]{{.*}}}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
@@ -578,9 +578,9 @@ subroutine acc_serial_loop
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:            acc.loop {{.*}} {
 ! CHECK:              acc.yield
-! CHECK-NEXT:     }{{$}}
+! CHECK-NEXT:     } attributes {{{.*}}seq = [#acc.device_type<none>]}
 ! CHECK:          acc.yield
-! CHECK-NEXT:   }{{$}}
+! CHECK-NEXT:   } attributes {{{.*}}seq = [#acc.device_type<none>]}
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/external/llvm-project/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
index 202e29321bcf..6d3a9a091521 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -1,3 +1,4 @@
+! XFAIL: *
 ! Tests delayed privatization for `targets ... private(..)` for allocatables.
 
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --enable-delayed-privatization-staging \
@@ -64,8 +65,9 @@ end subroutine target_allocatable
 ! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : index
 ! CHECK-NEXT:   %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[MOLD]], %[[C0]]
 ! CHECK-NEXT:   %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1
-! CHECK-NEXT:   %[[DATA_ALLOC:.*]] = fir.alloca !fir.array<?xf32>, %[[BOX_DIMS]]#1
+! CHECK-NEXT:   %[[DATA_ALLOC:.*]] = fir.allocmem !fir.array<?xf32>, %[[BOX_DIMS]]#1
 ! CHECK-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[DATA_ALLOC:.*]](%[[SHAPE]])
+! CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
 ! CHECK-NEXT:   %[[C0_2:.*]] = arith.constant 0 : index
 ! CHECK-NEXT:   %[[BOX_DIMS_2:.*]]:3 = fir.box_dims %[[MOLD]], %[[C0_2]]
 ! CHECK-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS_2]]#0, %[[BOX_DIMS_2]]#1
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare-fail.f90 b/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare-fail.f90
index b82bd1362276..6f58e0939a78 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare-fail.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare-fail.f90
@@ -1,6 +1,6 @@
 ! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
 
-! CHECK: not yet implemented: OpenMP atomic compare
+! CHECK: not yet implemented: OpenMP ATOMIC COMPARE
 program p
   integer :: x
   logical :: r
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare.f90 b/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare.f90
index 88ec6fe910b9..6729be6e5cf8 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/Todo/atomic-compare.f90
@@ -1,6 +1,6 @@
 ! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1 | FileCheck %s
 
-! CHECK: not yet implemented: OpenMP atomic compare
+! CHECK: not yet implemented: OpenMP ATOMIC COMPARE
 program p
   integer :: x
   logical :: r
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/Todo/omp-declarative-allocate.f90 b/external/llvm-project/flang/test/Lower/OpenMP/Todo/omp-declarative-allocate.f90
index 425ccbc5dd56..e83b433d0fda 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/Todo/omp-declarative-allocate.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/Todo/omp-declarative-allocate.f90
@@ -1,10 +1,10 @@
 ! This test checks lowering of OpenMP allocate Directive.
 
-// RUN: not flang -fc1 -emit-fir -fopenmp %s 2>&1 | FileCheck %s
+! RUN: not flang -fc1 -emit-fir -fopenmp %s 2>&1 | FileCheck %s
 
 program main
   integer :: x, y
 
-  // CHECK: not yet implemented: OpenMPDeclarativeAllocate
+  ! CHECK: not yet implemented: OpenMPDeclarativeAllocate
   !$omp allocate(x, y)
 end
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/Todo/target-parallel-private.f90 b/external/llvm-project/flang/test/Lower/OpenMP/Todo/target-parallel-private.f90
new file mode 100644
index 000000000000..e820143021f9
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/Todo/target-parallel-private.f90
@@ -0,0 +1,13 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+
+!===============================================================================
+! `private` clause on `target parallel`
+!===============================================================================
+
+! CHECK: not yet implemented: TARGET PARALLEL PRIVATE is not implemented yet
+subroutine target_teams_private()
+integer, dimension(3) :: i
+!$omp target parallel private(i)
+!$omp end target parallel
+end subroutine
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/Todo/target-teams-private.f90 b/external/llvm-project/flang/test/Lower/OpenMP/Todo/target-teams-private.f90
new file mode 100644
index 000000000000..c8d998a5cbf9
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/Todo/target-teams-private.f90
@@ -0,0 +1,13 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+
+!===============================================================================
+! `private` clause on `target teams`
+!===============================================================================
+
+! CHECK: not yet implemented: TARGET TEAMS PRIVATE is not implemented yet
+subroutine target_teams_private()
+integer, dimension(3) :: i
+!$omp target teams private(i)
+!$omp end target teams
+end subroutine
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/atomic-capture.f90 b/external/llvm-project/flang/test/Lower/OpenMP/atomic-capture.f90
index 2f800d534dc3..14fd0c942a9b 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/atomic-capture.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/atomic-capture.f90
@@ -79,16 +79,16 @@ subroutine pointers_in_atomic_capture()
 !CHECK: %[[VAL_A_BOX_ADDR:.*]] = fir.box_addr %[[VAL_A_LOADED]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
 !CHECK: %[[VAL_B_LOADED:.*]] = fir.load %[[VAL_B_DECLARE]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK: %[[VAL_B_BOX_ADDR:.*]] = fir.box_addr %[[VAL_B_LOADED]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK: %[[VAL_B:.*]] = fir.load %[[VAL_B_BOX_ADDR]] : !fir.ptr<i32>
 !CHECK: %[[VAL_B_LOADED_2:.*]] = fir.load %[[VAL_B_DECLARE]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK: %[[VAL_B_BOX_ADDR_2:.*]] = fir.box_addr %[[VAL_B_LOADED_2]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
-!CHECK: %[[VAL_B:.*]] = fir.load %[[VAL_B_BOX_ADDR_2]] : !fir.ptr<i32>
 !CHECK: omp.atomic.capture {
 !CHECK: omp.atomic.update %[[VAL_A_BOX_ADDR]] : !fir.ptr<i32> {
 !CHECK: ^bb0(%[[ARG:.*]]: i32):
 !CHECK: %[[TEMP:.*]] = arith.addi %[[ARG]], %[[VAL_B]] : i32
 !CHECK: omp.yield(%[[TEMP]] : i32)
 !CHECK: }
-!CHECK: omp.atomic.read %[[VAL_B_BOX_ADDR]] = %[[VAL_A_BOX_ADDR]] : !fir.ptr<i32>, !fir.ptr<i32>, i32
+!CHECK: omp.atomic.read %[[VAL_B_BOX_ADDR_2]] = %[[VAL_A_BOX_ADDR]] : !fir.ptr<i32>, !fir.ptr<i32>, i32
 !CHECK: }
 !CHECK: return
 !CHECK: }
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/atomic-implicit-cast.f90 b/external/llvm-project/flang/test/Lower/OpenMP/atomic-implicit-cast.f90
index 4c1be1ca91ac..dab18c43d220 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/atomic-implicit-cast.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/atomic-implicit-cast.f90
@@ -1,7 +1,4 @@
-! REQUIRES : openmp_runtime
-
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-
 ! CHECK: func.func @_QPatomic_implicit_cast_read() {
 subroutine atomic_implicit_cast_read
 ! CHECK: %[[ALLOCA7:.*]] = fir.alloca complex<f64>
@@ -29,21 +26,21 @@ subroutine atomic_implicit_cast_read
     complex :: w
     complex(8) :: m
 
-! CHECK: omp.atomic.read %[[ALLOCA0:.*]] = %[[Y_DECL]]#0 : !fir.ref<f32>, !fir.ref<f32>, f32
+! CHECK: omp.atomic.read %[[ALLOCA0:.*]] = %[[Y_DECL]]#0 memory_order(relaxed) : !fir.ref<f32>, !fir.ref<f32>, f32
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ALLOCA0]] : !fir.ref<f32>
 ! CHECK: %[[CVT:.*]] = fir.convert %[[LOAD]] : (f32) -> i32
 ! CHECK: fir.store %[[CVT]] to %[[X_DECL]]#0 : !fir.ref<i32>
     !$omp atomic read
         x = y
 
-! CHECK: omp.atomic.read %[[ALLOCA1:.*]] = %[[X_DECL]]#0 : !fir.ref<i32>, !fir.ref<i32>, i32
+! CHECK: omp.atomic.read %[[ALLOCA1:.*]] = %[[X_DECL]]#0 memory_order(relaxed) : !fir.ref<i32>, !fir.ref<i32>, i32
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ALLOCA1]] : !fir.ref<i32>
 ! CHECK: %[[CVT:.*]] = fir.convert %[[LOAD]] : (i32) -> f64
 ! CHECK: fir.store %[[CVT]] to %[[Z_DECL]]#0 : !fir.ref<f64>
     !$omp atomic read
         z = x
 
-! CHECK: omp.atomic.read %[[ALLOCA2:.*]] = %[[W_DECL]]#0 : !fir.ref<complex<f32>>, !fir.ref<complex<f32>>, complex<f32>
+! CHECK: omp.atomic.read %[[ALLOCA2:.*]] = %[[W_DECL]]#0 memory_order(relaxed) : !fir.ref<complex<f32>>, !fir.ref<complex<f32>>, complex<f32>
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ALLOCA2]] : !fir.ref<complex<f32>>
 ! CHECK: %[[EXTRACT:.*]] = fir.extract_value %[[LOAD]], [0 : index] : (complex<f32>) -> f32
 ! CHECK: %[[CVT:.*]] = fir.convert %[[EXTRACT]] : (f32) -> i32
@@ -51,7 +48,7 @@ subroutine atomic_implicit_cast_read
     !$omp atomic read
         x = w
 
-! CHECK: omp.atomic.read %[[ALLOCA3:.*]] = %[[W_DECL]]#0 : !fir.ref<complex<f32>>, !fir.ref<complex<f32>>, complex<f32>
+! CHECK: omp.atomic.read %[[ALLOCA3:.*]] = %[[W_DECL]]#0 memory_order(relaxed) : !fir.ref<complex<f32>>, !fir.ref<complex<f32>>, complex<f32>
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ALLOCA3]] : !fir.ref<complex<f32>>
 ! CHECK: %[[CVT:.*]] = fir.convert %[[LOAD]] : (complex<f32>) -> complex<f64>
 ! CHECK: fir.store %[[CVT]] to %[[M_DECL]]#0 : !fir.ref<complex<f64>>
@@ -59,7 +56,7 @@ subroutine atomic_implicit_cast_read
         m = w
 
 ! CHECK: %[[CONST:.*]] = arith.constant 1 : i32
-! CHECK: omp.atomic.capture {
+! CHECK: omp.atomic.capture memory_order(relaxed) {
 ! CHECK: omp.atomic.read %[[ALLOCA4]] = %[[X_DECL]]#0 : !fir.ref<i32>, !fir.ref<i32>, i32
 ! CHECK: omp.atomic.update %[[X_DECL]]#0 : !fir.ref<i32> {
 ! CHECK: ^bb0(%[[ARG:.*]]: i32):
@@ -76,7 +73,7 @@ subroutine atomic_implicit_cast_read
      !$omp end atomic
 
 ! CHECK: %[[CONST:.*]] = arith.constant 10 : i32
-! CHECK: omp.atomic.capture {
+! CHECK: omp.atomic.capture memory_order(relaxed) {
 ! CHECK: omp.atomic.read %[[ALLOCA5:.*]] = %[[X_DECL]]#0 : !fir.ref<i32>, !fir.ref<i32>, i32
 ! CHECK: omp.atomic.write %[[X_DECL]]#0 = %[[CONST]] : !fir.ref<i32>, i32
 ! CHECK: }
@@ -89,7 +86,7 @@ subroutine atomic_implicit_cast_read
      !$omp end atomic
 
 ! CHECK: %[[CONST:.*]] = arith.constant 1 : i32
-! CHECK: omp.atomic.capture {
+! CHECK: omp.atomic.capture memory_order(relaxed) {
 ! CHECK: omp.atomic.update %[[X_DECL]]#0 : !fir.ref<i32> {
 ! CHECK: ^bb0(%[[ARG:.*]]: i32):
 ! CHECK: %[[RESULT:.*]] = arith.addi %[[ARG]], %[[CONST]] : i32
@@ -97,9 +94,9 @@ subroutine atomic_implicit_cast_read
 ! CHECK: }
 ! CHECK: omp.atomic.read %[[ALLOCA6]] = %[[X_DECL]]#0 : !fir.ref<i32>, !fir.ref<i32>, i32
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ALLOCA6]] : !fir.ref<i32>
-! CHECK: %[[UNDEF:.*]] = fir.undefined complex<f32>
 ! CHECK: %[[CVT:.*]] = fir.convert %[[LOAD]] : (i32) -> f32
 ! CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK: %[[UNDEF:.*]] = fir.undefined complex<f32>
 ! CHECK: %[[IDX1:.*]] = fir.insert_value %[[UNDEF]], %[[CVT]], [0 : index] : (complex<f32>, f32) -> complex<f32>
 ! CHECK: %[[IDX2:.*]] = fir.insert_value %[[IDX1]], %[[CST]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 ! CHECK: fir.store %[[IDX2]] to %[[W_DECL]]#0 : !fir.ref<complex<f32>>
@@ -109,14 +106,14 @@ subroutine atomic_implicit_cast_read
      !$omp end atomic
 
 
-! CHECK: omp.atomic.capture {
-! CHECK: omp.atomic.update %[[M_DECL]]#0 : !fir.ref<complex<f64>> {
-! CHECK: ^bb0(%[[ARG:.*]]: complex<f64>):
 ! CHECK: %[[CST1:.*]] = arith.constant 1.000000e+00 : f64
 ! CHECK: %[[CST2:.*]] = arith.constant 0.000000e+00 : f64
 ! CHECK: %[[UNDEF:.*]] = fir.undefined complex<f64>
 ! CHECK: %[[IDX1:.*]] = fir.insert_value %[[UNDEF]], %[[CST1]], [0 : index] : (complex<f64>, f64) -> complex<f64>
 ! CHECK: %[[IDX2:.*]] = fir.insert_value %[[IDX1]], %[[CST2]], [1 : index] : (complex<f64>, f64) -> complex<f64>
+! CHECK: omp.atomic.capture memory_order(relaxed) {
+! CHECK: omp.atomic.update %[[M_DECL]]#0 : !fir.ref<complex<f64>> {
+! CHECK: ^bb0(%[[ARG:.*]]: complex<f64>):
 ! CHECK: %[[RESULT:.*]] = fir.addc %[[ARG]], %[[IDX2]] {fastmath = #arith.fastmath<contract>} : complex<f64>
 ! CHECK: omp.yield(%[[RESULT]] : complex<f64>)
 ! CHECK: }
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/atomic-privatize.f90 b/external/llvm-project/flang/test/Lower/OpenMP/atomic-privatize.f90
new file mode 100644
index 000000000000..c876266cf018
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/atomic-privatize.f90
@@ -0,0 +1,25 @@
+! Testcase adopted from the Fujitsu test suite:
+! https://github.com/fujitsu/compiler-test-suite/blob/main/Fortran/0407/0407_0006.f90
+
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=60 %s -o - | FileCheck %s
+
+! Make sure that the variable in the atomic construct is privatized at
+! the task level
+
+!CHECK: omp.task private(@_QFfredEprv_firstprivate_i32 %{{[0-9]+}}#0 -> %arg0
+!CHECK: %[[DECL:[0-9]+]]:2 = hlfir.declare %arg0 {uniq_name = "_QFfredEprv"}
+!CHECK: omp.atomic.update memory_order(relaxed) %[[DECL]]#0
+
+integer function fred
+  integer :: prv
+
+  prv = 1
+  !$omp parallel shared(prv)
+  !$omp task default(firstprivate)
+  !$omp atomic
+  prv = prv + 1
+  !$omp end task
+  !$omp end parallel
+  fred = prv
+end
+
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/atomic-write.f90 b/external/llvm-project/flang/test/Lower/OpenMP/atomic-write.f90
index 13392ad76471..6eded49b0b15 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/atomic-write.f90
@@ -44,9 +44,9 @@ end program OmpAtomicWrite
 !CHECK-LABEL: func.func @_QPatomic_write_pointer() {
 !CHECK:    %[[X_REF:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "x", uniq_name = "_QFatomic_write_pointerEx"}
 !CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFatomic_write_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
-!CHECK:    %[[C1:.*]] = arith.constant 1 : i32
 !CHECK:    %[[X_ADDR_BOX:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK:    %[[X_POINTEE_ADDR:.*]] = fir.box_addr %[[X_ADDR_BOX]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK:    %[[C1:.*]] = arith.constant 1 : i32
 !CHECK:    omp.atomic.write %[[X_POINTEE_ADDR]] = %[[C1]]   : !fir.ptr<i32>, i32
 !CHECK:    %[[C2:.*]] = arith.constant 2 : i32
 !CHECK:    %[[X_ADDR_BOX:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/common-atomic-lowering.f90 b/external/llvm-project/flang/test/Lower/OpenMP/common-atomic-lowering.f90
index a53cc101024c..43edf2064060 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/common-atomic-lowering.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/common-atomic-lowering.f90
@@ -1,5 +1,4 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-
 !CHECK: func.func @_QQmain() attributes {fir.bindc_name = "sample"} {
 !CHECK: %[[val_0:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK: %[[val_1:.*]]:2 = hlfir.declare %[[val_0]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -17,7 +16,7 @@
 !CHECK: %[[val_10:.*]] = fir.load %[[val_5]]#0 : !fir.ref<i32>
 !CHECK: %[[val_11:.*]] = arith.addi %[[val_c8]], %[[val_10]] : i32
 !CHECK: %[[val_12:.*]] = hlfir.no_reassoc %[[val_11]] : i32
-!CHECK: omp.atomic.update %[[val_9]] : !fir.ref<i32> {
+!CHECK: omp.atomic.update memory_order(relaxed) %[[val_9]] : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG:.*]]: i32):
 !CHECK:     %[[val_18:.*]] = arith.muli %[[val_12]], %[[ARG]] : i32
 !CHECK:     omp.yield(%[[val_18]] : i32)
@@ -25,7 +24,7 @@
 !CHECK: %[[val_c2_0:.*]] = arith.constant 2 : index
 !CHECK: %[[val_13:.*]] = hlfir.designate %[[val_8]]#0 (%[[val_c2_0]])  : (!fir.ref<!fir.array<5xi32>>, index) -> !fir.ref<i32>
 !CHECK: %[[val_c8_1:.*]] = arith.constant 8 : i32
-!CHECK: omp.atomic.update %[[val_13:.*]] : !fir.ref<i32> {
+!CHECK: omp.atomic.update memory_order(relaxed)  %[[val_13:.*]] : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG:.*]]: i32):
 !CHECK:     %[[val_18:.*]] = arith.divsi %[[ARG]], %[[val_c8_1]] : i32
 !CHECK:     omp.yield(%[[val_18]] : i32)
@@ -36,13 +35,13 @@
 !CHECK: %[[val_15:.*]] = fir.load %[[val_14]] : !fir.ref<i32>
 !CHECK: %[[val_16:.*]] = arith.addi %[[val_c8_2]], %[[val_15]] : i32
 !CHECK: %[[val_17:.*]] = hlfir.no_reassoc %[[val_16]] : i32
-!CHECK: omp.atomic.update %[[val_5]]#0 : !fir.ref<i32> {
+!CHECK: omp.atomic.update memory_order(relaxed)  %[[val_5]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG:.*]]: i32):
 !CHECK:      %[[val_18:.*]] = arith.addi %[[val_17]], %[[ARG]] : i32
 !CHECK:      omp.yield(%[[val_18]] : i32)
 !CHECK: }
 !CHECK: %[[val_c8_3:.*]] = arith.constant 8 : i32
-!CHECK: omp.atomic.update %[[val_5]]#0 : !fir.ref<i32> {
+!CHECK: omp.atomic.update memory_order(relaxed) %[[val_5]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG]]: i32):
 !CHECK:     %[[val_18:.*]] = arith.subi %[[val_c8_3]], %[[ARG]] : i32
 !CHECK:     omp.yield(%[[val_18]] : i32)
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90 b/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90
index d1c7167546b4..8446e99126f2 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-allocatable-array.f90
@@ -36,8 +36,8 @@ subroutine delayed_privatization_private(var1, l1)
 ! CHECK-NEXT:     %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0]]
 ! CHECK-NEXT:     %[[SHAPE:.*]] = fir.shape %[[DIMS]]#1
 ! CHECK-NEXT:     %[[MEM:.*]] = fir.allocmem !fir.array<?xi32>, %[[DIMS]]#1
-! CHECK-NEXT:     %[[TRUE:.*]] = arith.constant true
 ! CHECK-NEXT:     %[[DECL:.*]]:2 = hlfir.declare %[[MEM]](%[[SHAPE]])
+! CHECK-NEXT:     %[[TRUE:.*]] = arith.constant true
 ! CHECK-NEXT:     %[[C0_2:.*]] = arith.constant 0 : index
 ! CHECK-NEXT:     %[[DIMS_2:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_2]]
 ! CHECK-NEXT:     %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS_2]]#0, %[[DIMS_2]]#1
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-array.f90 b/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-array.f90
index 32f310fe8121..50f6761a927f 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-array.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/delayed-privatization-array.f90
@@ -1,4 +1,6 @@
 ! Test delayed privatization for arrays.
+! XFAIL: *
+
 ! RUN: split-file %s %t
 
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --enable-delayed-privatization \
@@ -36,8 +38,9 @@ subroutine delayed_privatization_private_1d(var1, l1, u1)
 ! ONE_DIM-NEXT:   %[[C0:.*]] = arith.constant 0 : index
 ! ONE_DIM-NEXT:   %[[DIMS:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0]]
 ! ONE_DIM-NEXT:   %[[SHAPE:.*]] = fir.shape %[[DIMS]]#1
-! ONE_DIM-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.alloca !fir.array<?xi32>, %[[DIMS]]#1
+! ONE_DIM-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.allocmem !fir.array<?xi32>, %[[DIMS]]#1
 ! ONE_DIM-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[ARRAY_ALLOC]](%[[SHAPE]])
+! ONE_DIM-NEXT:   %[[TRUE:.*]] = arith.constant true
 ! ONE_DIM-NEXT:   %[[C0_0:.*]] = arith.constant 0
 ! ONE_DIM-NEXT:   %[[DIMS2:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_0]]
 ! ONE_DIM-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS2]]#0, %[[DIMS2]]#1
@@ -73,8 +76,9 @@ subroutine delayed_privatization_private_2d(var1, l1, u1, l2, u2)
 ! TWO_DIM-NEXT:   %[[C1:.*]] = arith.constant 1 : index
 ! TWO_DIM-NEXT:   %[[DIMS_1:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C1]]
 ! TWO_DIM-NEXT:   %[[SHAPE:.*]] = fir.shape %[[DIMS_0]]#1, %[[DIMS_1]]#1
-! TWO_DIM-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.alloca !fir.array<?x?xi32>, %[[DIMS_0]]#1, %[[DIMS_1]]#1
+! TWO_DIM-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.allocmem !fir.array<?x?xi32>, %[[DIMS_0]]#1, %[[DIMS_1]]#1
 ! TWO_DIM-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[ARRAY_ALLOC]](%[[SHAPE]])
+! TWO_DIM-NEXT:   %[[TRUE:.*]] = arith.constant true
 ! TWO_DIM-NEXT:   %[[C0_0:.*]] = arith.constant 0
 ! TWO_DIM-NEXT:   %[[DIMS2_0:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_0]]
 ! TWO_DIM-NEXT:   %[[C1_0:.*]] = arith.constant 1
@@ -105,14 +109,14 @@ program main
 ! ONE_DIM_DEFAULT_LB-SAME: @[[PRIVATIZER_SYM:.*]] : [[BOX_TYPE:!fir.box<!fir.array<10xi32>>]] init {
 
 ! ONE_DIM_DEFAULT_LB-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[TYPE:!fir.ref<!fir.box<!fir.array<10xi32>>>]], %[[PRIV_BOX_ALLOC:.*]]: [[TYPE]]):
-! ONE_DIM_DEFAULT_LB-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.alloca !fir.array<10xi32>
-! ONE_DIM_DEFAULT_LB-NEXT:   %[[PRIV_ARG_VAL:.*]] = fir.load %[[PRIV_ARG]]
 ! ONE_DIM_DEFAULT_LB-NEXT:   %[[C10:.*]] = arith.constant 10 : index
 ! ONE_DIM_DEFAULT_LB-NEXT:   %[[SHAPE:.*]] = fir.shape %[[C10]]
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[ARRAY_ALLOC:.*]] = fir.allocmem !fir.array<10xi32>
 ! ONE_DIM_DEFAULT_LB-NEXT:   %[[DECL:.*]]:2 = hlfir.declare %[[ARRAY_ALLOC]](%[[SHAPE]])
-! ONE_DIM_DEFAULT_LB-NEXT:   %[[C0_0:.*]] = arith.constant 0
-! ONE_DIM_DEFAULT_LB-NEXT:   %[[DIMS2:.*]]:3 = fir.box_dims %[[PRIV_ARG_VAL]], %[[C0_0]]
-! ONE_DIM_DEFAULT_LB-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[DIMS2]]#0, %[[DIMS2]]#1
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[TRUE:.*]] = arith.constant true
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[ONE:.*]] = arith.constant 1 : index
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[TEN:.*]] = arith.constant 10 : index
+! ONE_DIM_DEFAULT_LB-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[ONE]], %[[TEN]]
 ! ONE_DIM_DEFAULT_LB-NEXT:   %[[EMBOX:.*]] = fir.embox %[[DECL]]#0(%[[SHAPE_SHIFT]])
 ! ONE_DIM_DEFAULT_LB-NEXT:   fir.store %[[EMBOX]] to %[[PRIV_BOX_ALLOC]]
 ! ONE_DIM_DEFAULT_LB-NEXT:   omp.yield(%[[PRIV_BOX_ALLOC]] : [[TYPE]])
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/depend-complex.f90 b/external/llvm-project/flang/test/Lower/OpenMP/depend-complex.f90
new file mode 100644
index 000000000000..488696b56507
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/depend-complex.f90
@@ -0,0 +1,22 @@
+! RUN: %flang_fc1 -fopenmp -emit-hlfir -o - %s | FileCheck %s
+
+subroutine depend_complex(z)
+! CHECK-LABEL:   func.func @_QPdepend_complex(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "z"}) {
+  complex :: z
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {uniq_name = "_QFdepend_complexEz"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+  !$omp task depend(in:z%re)
+! CHECK:           %[[VAL_2:.*]] = hlfir.designate %[[VAL_1]]#0  real : (!fir.ref<complex<f32>>) -> !fir.ref<f32>
+! CHECK:           omp.task depend(taskdependin -> %[[VAL_2]] : !fir.ref<f32>) {
+! CHECK:             omp.terminator
+! CHECK:           }
+  !$omp end task
+  !$omp task depend(in:z%im)
+! CHECK:           %[[VAL_3:.*]] = hlfir.designate %[[VAL_1]]#0  imag : (!fir.ref<complex<f32>>) -> !fir.ref<f32>
+! CHECK:           omp.task depend(taskdependin -> %[[VAL_3]] : !fir.ref<f32>) {
+! CHECK:             omp.terminator
+! CHECK:           }
+  !$omp end task
+end subroutine
+
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/depend-substring.f90 b/external/llvm-project/flang/test/Lower/OpenMP/depend-substring.f90
new file mode 100644
index 000000000000..5de11e06cc10
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/depend-substring.f90
@@ -0,0 +1,108 @@
+! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+subroutine substring_0(c)
+  character(:), pointer :: c
+  !$omp task depend(out:c(:))
+  !$omp end task
+end
+! CHECK-LABEL:   func.func @_QPsubstring_0(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>> {fir.bindc_name = "c"}) {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsubstring_0Ec"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> !fir.ptr<!fir.char<1,?>>
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_5:.*]] = fir.box_elesize %[[VAL_4]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> index
+! CHECK:           %[[VAL_6:.*]] = fir.emboxchar %[[VAL_3]], %[[VAL_5]] : (!fir.ptr<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_9:.*]] = fir.box_elesize %[[VAL_8]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> index
+! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (index) -> i64
+! CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (i64) -> index
+! CHECK:           %[[VAL_12:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_13:.*]] = arith.subi %[[VAL_11]], %[[VAL_7]] : index
+! CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_13]], %[[VAL_12]] : index
+! CHECK:           %[[VAL_15:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_16:.*]] = arith.cmpi sgt, %[[VAL_14]], %[[VAL_15]] : index
+! CHECK:           %[[VAL_17:.*]] = arith.select %[[VAL_16]], %[[VAL_14]], %[[VAL_15]] : index
+! CHECK:           %[[VAL_18:.*]] = hlfir.designate %[[VAL_6]]  substr %[[VAL_7]], %[[VAL_11]]  typeparams %[[VAL_17]] : (!fir.boxchar<1>, index, index, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_19:.*]] = fir.box_addr %[[VAL_18]] : (!fir.boxchar<1>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           omp.task depend(taskdependout -> %[[VAL_19]] : !fir.ref<!fir.char<1,?>>) {
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
+
+subroutine substring_1(c)
+  character(:), pointer :: c
+  !$omp task depend(out:c(2:))
+  !$omp end task
+end
+! CHECK-LABEL:   func.func @_QPsubstring_1(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>> {fir.bindc_name = "c"}) {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsubstring_1Ec"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> !fir.ptr<!fir.char<1,?>>
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_5:.*]] = fir.box_elesize %[[VAL_4]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> index
+! CHECK:           %[[VAL_6:.*]] = fir.emboxchar %[[VAL_3]], %[[VAL_5]] : (!fir.ptr<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_7:.*]] = arith.constant 2 : index
+! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_9:.*]] = fir.box_elesize %[[VAL_8]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> index
+! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (index) -> i64
+! CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (i64) -> index
+! CHECK:           %[[VAL_12:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_13:.*]] = arith.subi %[[VAL_11]], %[[VAL_7]] : index
+! CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_13]], %[[VAL_12]] : index
+! CHECK:           %[[VAL_15:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_16:.*]] = arith.cmpi sgt, %[[VAL_14]], %[[VAL_15]] : index
+! CHECK:           %[[VAL_17:.*]] = arith.select %[[VAL_16]], %[[VAL_14]], %[[VAL_15]] : index
+! CHECK:           %[[VAL_18:.*]] = hlfir.designate %[[VAL_6]]  substr %[[VAL_7]], %[[VAL_11]]  typeparams %[[VAL_17]] : (!fir.boxchar<1>, index, index, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_19:.*]] = fir.box_addr %[[VAL_18]] : (!fir.boxchar<1>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           omp.task depend(taskdependout -> %[[VAL_19]] : !fir.ref<!fir.char<1,?>>) {
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
+
+subroutine substring_2(c)
+  character(:), pointer :: c
+  !$omp task depend(out:c(:2))
+  !$omp end task
+end
+! CHECK-LABEL:   func.func @_QPsubstring_2(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>> {fir.bindc_name = "c"}) {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsubstring_2Ec"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> !fir.ptr<!fir.char<1,?>>
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_5:.*]] = fir.box_elesize %[[VAL_4]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> index
+! CHECK:           %[[VAL_6:.*]] = fir.emboxchar %[[VAL_3]], %[[VAL_5]] : (!fir.ptr<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_8:.*]] = arith.constant 2 : index
+! CHECK:           %[[VAL_9:.*]] = arith.constant 2 : index
+! CHECK:           %[[VAL_10:.*]] = hlfir.designate %[[VAL_6]]  substr %[[VAL_7]], %[[VAL_8]]  typeparams %[[VAL_9]] : (!fir.boxchar<1>, index, index, index) -> !fir.ref<!fir.char<1,2>>
+! CHECK:           omp.task depend(taskdependout -> %[[VAL_10]] : !fir.ref<!fir.char<1,2>>) {
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
+
+subroutine substring_4(c)
+  character(:), pointer :: c
+  !$omp task depend(out:c)
+  !$omp end task
+end
+! CHECK-LABEL:   func.func @_QPsubstring_4(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>> {fir.bindc_name = "c"}) {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsubstring_4Ec"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> !fir.ptr<!fir.char<1,?>>
+! CHECK:           omp.task depend(taskdependout -> %[[VAL_3]] : !fir.ptr<!fir.char<1,?>>) {
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/different_vars_lastprivate_barrier.f90 b/external/llvm-project/flang/test/Lower/OpenMP/different_vars_lastprivate_barrier.f90
index 7729e6894165..5bf634c86652 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/different_vars_lastprivate_barrier.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/different_vars_lastprivate_barrier.f90
@@ -1,8 +1,8 @@
 ! RUN: %flang_fc1 -fopenmp -mmlir --enable-delayed-privatization-staging=true -emit-hlfir %s -o - | FileCheck %s
 
-subroutine first_and_lastprivate
+subroutine first_and_lastprivate(var)
   integer i
-  integer, dimension(3) :: var
+  integer, dimension(:) :: var
 
   !$omp parallel do lastprivate(i) private(var)
   do i=1,1
@@ -10,19 +10,20 @@ subroutine first_and_lastprivate
   !$omp end parallel do
 end subroutine
 
-! CHECK:      omp.private {type = private} @[[VAR_PRIVATIZER:.*Evar_private_box_3xi32]] : [[BOX_TYPE:!fir\.box<!fir\.array<3xi32>>]] init {
+! CHECK:      omp.private {type = private} @[[VAR_PRIVATIZER:.*Evar_private_box_Uxi32]] : [[BOX_TYPE:!fir\.box<!fir\.array<\?xi32>>]] init {
 ! CHECK-NEXT: ^bb0(%[[ORIG_REF:.*]]: {{.*}}, %[[PRIV_REF:.*]]: {{.*}}):
 ! CHECK:        %[[ORIG_VAL:.*]] = fir.load %[[ORIG_REF]]
+! CHECK:        %[[BOX_DIMS_0:.*]]:3 = fir.box_dims %[[ORIG_VAL]], %{{.*}} : ([[BOX_TYPE]], index) -> (index, index, index)
 ! CHECK:        %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ORIG_VAL]], %{{.*}} : ([[BOX_TYPE]], index) -> (index, index, index)
 ! CHECK:        %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS]]#0, %[[BOX_DIMS]]#1
-! CHECK:        %[[EMBOX:.*]] = fir.embox %{{.*}}(%[[SHAPE_SHIFT]]) : {{.*}} -> [[BOX_TYPE]]
+! CHECK:        %[[EMBOX:.*]] = fir.rebox %{{.*}}(%[[SHAPE_SHIFT]]) : {{.*}} -> [[BOX_TYPE]]
 ! CHECK:        fir.store %[[EMBOX]] to %[[PRIV_REF]]
 ! CHECK:        omp.yield(%[[PRIV_REF]] : !fir.ref<[[BOX_TYPE]]>)
 ! CHECK:      }
 
 ! CHECK: omp.private {type = private} @[[I_PRIVATIZER:.*Ei_private_i32]] : i32
 
-! CHECK:     func.func @{{.*}}first_and_lastprivate()
+! CHECK:     func.func @{{.*}}first_and_lastprivate({{.*}})
 ! CHECK:       %[[ORIG_I_DECL:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "{{.*}}Ei"}
 ! CHECK:       omp.parallel {
 ! CHECK-NOT:      omp.barrier
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/dump-atomic-analysis.f90 b/external/llvm-project/flang/test/Lower/OpenMP/dump-atomic-analysis.f90
new file mode 100644
index 000000000000..cbaf7bc9f2d8
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/dump-atomic-analysis.f90
@@ -0,0 +1,82 @@
+!RUN: %flang_fc1 -fopenmp -fopenmp-version=60 -emit-hlfir -mmlir -fdebug-dump-atomic-analysis %s -o /dev/null 2>&1 | FileCheck %s
+
+subroutine f00(x)
+  integer :: x, v
+  !$omp atomic read
+    v = x
+end
+
+!CHECK: Analysis {
+!CHECK-NEXT:   atom: x
+!CHECK-NEXT:   cond: <null>
+!CHECK-NEXT:   op0 {
+!CHECK-NEXT:     what: Read
+!CHECK-NEXT:     assign: v=x
+!CHECK-NEXT:   }
+!CHECK-NEXT:   op1 {
+!CHECK-NEXT:     what: None
+!CHECK-NEXT:     assign: <null>
+!CHECK-NEXT:   }
+!CHECK-NEXT: }
+
+
+subroutine f01(v)
+  integer :: x, v
+  !$omp atomic write
+    x = v
+end
+
+!CHECK: Analysis {
+!CHECK-NEXT:   atom: x
+!CHECK-NEXT:   cond: <null>
+!CHECK-NEXT:   op0 {
+!CHECK-NEXT:     what: Write
+!CHECK-NEXT:     assign: x=v
+!CHECK-NEXT:   }
+!CHECK-NEXT:   op1 {
+!CHECK-NEXT:     what: None
+!CHECK-NEXT:     assign: <null>
+!CHECK-NEXT:   }
+!CHECK-NEXT: }
+
+
+subroutine f02(x, v)
+  integer :: x, v
+  !$omp atomic update
+    x = x + v
+end
+
+!CHECK: Analysis {
+!CHECK-NEXT:   atom: x
+!CHECK-NEXT:   cond: <null>
+!CHECK-NEXT:   op0 {
+!CHECK-NEXT:     what: Update
+!CHECK-NEXT:     assign: x=x+v
+!CHECK-NEXT:   }
+!CHECK-NEXT:   op1 {
+!CHECK-NEXT:     what: None
+!CHECK-NEXT:     assign: <null>
+!CHECK-NEXT:   }
+!CHECK-NEXT: }
+
+
+subroutine f03(x, v)
+  integer :: x, v, t
+  !$omp atomic update capture
+    t = x
+    x = x + v
+  !$omp end atomic
+end
+
+!CHECK: Analysis {
+!CHECK-NEXT:   atom: x
+!CHECK-NEXT:   cond: <null>
+!CHECK-NEXT:   op0 {
+!CHECK-NEXT:     what: Read
+!CHECK-NEXT:     assign: t=x
+!CHECK-NEXT:   }
+!CHECK-NEXT:   op1 {
+!CHECK-NEXT:     what: Update
+!CHECK-NEXT:     assign: x=x+v
+!CHECK-NEXT:   }
+!CHECK-NEXT: }
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/flush02.f90 b/external/llvm-project/flang/test/Lower/OpenMP/flush02.f90
new file mode 100644
index 000000000000..b372e700e1a1
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/flush02.f90
@@ -0,0 +1,32 @@
+! This test checks lowering of OpenMP Flush Directive.
+
+!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+module flush02_mod
+    type t1
+       integer(kind=4) :: x = 4
+    end type t1
+
+    type :: t2
+       type(t1) :: y = t1(2)
+    end type t2
+
+
+contains
+
+    subroutine sub01(pt)
+        class(t1), intent(inout) :: pt
+        type(t2)                 :: dt
+        integer, allocatable     :: a(:)
+        integer, pointer         :: b(:)
+
+        ! CHECK: omp.flush({{.*}} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+        ! CHECK: omp.flush({{.*}} : !fir.ref<f32>)
+        ! CHECK: omp.flush({{.*}} : !fir.ref<!fir.type<_QMflush02_modTt2{y:!fir.type<_QMflush02_modTt1{x:i32}>}>>)
+        ! CHECK: omp.flush({{.*}} : !fir.class<!fir.type<_QMflush02_modTt1{x:i32}>>)
+        !$omp flush(a)
+        !$omp flush(p)
+        !$omp flush(dt)
+        !$omp flush(pt)
+    end subroutine
+end module flush02_mod
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/map-character.f90 b/external/llvm-project/flang/test/Lower/OpenMP/map-character.f90
index 232c0a6361cb..cefd3ac0e54f 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/map-character.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/map-character.f90
@@ -18,25 +18,38 @@ end subroutine TestOfCharacter
 !CHECK:  %[[A0_DECL:.*]]:2 = hlfir.declare %[[UNBOXED_ARG0]]#0 typeparams %[[UNBOXED_ARG0]]#1 dummy_scope {{.*}} -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 !CHECK:  %[[UNBOXED_ARG1:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 !CHECK:  %[[A1_DECL:.*]]:2 = hlfir.declare %[[UNBOXED_ARG1]]#0 typeparams %[[UNBOXED_ARG1]]#1 dummy_scope {{.*}} -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
-!CHECK:  %[[UNBOXED_A0_DECL:.*]]:2 = fir.unboxchar %[[A0_DECL]]#0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 !CHECK:  %[[A0_LB:.*]] = arith.constant 0 : index
 !CHECK:  %[[A0_STRIDE:.*]] = arith.constant 1 : index
+!CHECK:  %[[UNBOXED_A0_DECL:.*]]:2 = fir.unboxchar %[[A0_DECL]]#0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 !CHECK:  %[[A0_UB:.*]] = arith.subi %[[UNBOXED_A0_DECL]]#1, %[[A0_STRIDE]] : index
 !CHECK:  %[[A0_BOUNDS:.*]] = omp.map.bounds lower_bound(%[[A0_LB]] : index) upper_bound(%[[A0_UB]] : index) extent(%[[UNBOXED_A0_DECL]]#1 : index)
 !CHECK-SAME:  stride(%[[A0_STRIDE]] : index) start_idx(%[[A0_LB]] : index) {stride_in_bytes = true}
 !CHECK:  %[[A0_MAP:.*]] = omp.map.info var_ptr(%[[A0_DECL]]#1 : !fir.ref<!fir.char<1,?>>, !fir.char<1,?>) map_clauses(to) capture(ByRef) bounds(%[[A0_BOUNDS]]) -> !fir.ref<!fir.char<1,?>> {name = "a0"}
-!CHECK:  %[[UNBOXED_A1_DECL:.*]]:2 = fir.unboxchar %[[A1_DECL]]#0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 !CHECK:  %[[A1_LB:.*]] = arith.constant 0 : index
 !CHECK:  %[[A1_STRIDE:.*]] = arith.constant 1 : index
+!CHECK:  %[[UNBOXED_A1_DECL:.*]]:2 = fir.unboxchar %[[A1_DECL]]#0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 !CHECK:  %[[A1_UB:.*]] = arith.subi %[[UNBOXED_A1_DECL]]#1, %[[A1_STRIDE]] : index
 !CHECK:  %[[A1_BOUNDS:.*]] = omp.map.bounds lower_bound(%[[A1_LB]] : index) upper_bound(%[[A1_UB]] : index) extent(%[[UNBOXED_A1_DECL]]#1 : index)
 !CHECKL-SAME: stride(%[[A1_STRIDE]] : index) start_idx(%[[A1_LB]] : index) {stride_in_bytes = true}
 !CHECK:  %[[A1_MAP:.*]] = omp.map.info var_ptr(%[[A1_DECL]]#1 : !fir.ref<!fir.char<1,?>>, !fir.char<1,?>) map_clauses(from) capture(ByRef) bounds(%[[A1_BOUNDS]]) -> !fir.ref<!fir.char<1,?>> {name = "a1"}
 !CHECK:  fir.store %[[ARG1]] to %[[A1_BOXCHAR_ALLOCA]] : !fir.ref<!fir.boxchar<1>>
-!CHECK:  %[[A1_BOXCHAR_MAP:.*]] = omp.map.info var_ptr(%[[A1_BOXCHAR_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(implicit, to) capture(ByRef) -> !fir.ref<!fir.boxchar<1>> {name = ""}
+!CHECK:  %[[CONST_ZERO:.*]] = arith.constant 0 : index
+!CHECK:  %[[CONST_ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[UNBOXED_ARG1:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+!CHECK:  %[[A1_UB:.*]] = arith.subi %[[UNBOXED_ARG1]]#1, %[[CONST_ONE]] : index
+!CHECK:  %[[BOUNDS_A1_BOXCHAR:.*]] = omp.map.bounds lower_bound(%[[CONST_ZERO]] : index) upper_bound(%[[A1_UB]] : index) extent(%[[UNBOXED_ARG1]]#1 : index)
+!CHECK-SAME: stride(%[[CONST_ONE]] : index) start_idx(%[[CONST_ZERO]] : index) {stride_in_bytes = true}
+!CHECK:  %[[A1_BOXCHAR_MAP:.*]] = omp.map.info var_ptr(%[[A1_BOXCHAR_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(implicit, to)
+!CHECK-SAME: capture(ByRef) bounds(%[[BOUNDS_A1_BOXCHAR]]) -> !fir.ref<!fir.boxchar<1>> {name = ""}
 !CHECK:  fir.store %[[ARG0]] to %[[A0_BOXCHAR_ALLOCA]] : !fir.ref<!fir.boxchar<1>>
-!CHECK:  %[[A0_BOXCHAR_MAP:.*]] = omp.map.info var_ptr(%[[A0_BOXCHAR_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(implicit, to) capture(ByRef) -> !fir.ref<!fir.boxchar<1>> {name = ""}
-
+!CHECK:  %[[CONST_ZERO:.*]] = arith.constant 0 : index
+!CHECK:  %[[CONST_ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[UNBOXED_ARG0:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+!CHECK:  %[[A0_UB:.*]] = arith.subi %[[UNBOXED_ARG0]]#1, %[[CONST_ONE]] : index
+!CHECK:  %[[BOUNDS_A0_BOXCHAR:.*]] = omp.map.bounds lower_bound(%[[CONST_ZERO]] : index) upper_bound(%[[A0_UB]] : index) extent(%[[UNBOXED_ARG0]]#1 : index)
+!CHECK-SAME: stride(%[[CONST_ONE]] : index) start_idx(%[[CONST_ZERO]] : index) {stride_in_bytes = true}
+!CHECK:  %[[A0_BOXCHAR_MAP:.*]] = omp.map.info var_ptr(%[[A0_BOXCHAR_ALLOCA]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(implicit, to)
+!CHECK-SAME: capture(ByRef) bounds(%[[BOUNDS_A0_BOXCHAR]]) -> !fir.ref<!fir.boxchar<1>> {name = ""}
 !CHECK:  omp.target map_entries(%[[A0_MAP]] -> %[[TGT_A0:.*]], %[[A1_MAP]] -> %[[TGT_A1:.*]], %[[A1_BOXCHAR_MAP]] -> %[[TGT_A1_BOXCHAR:.*]], %[[A0_BOXCHAR_MAP]] -> %[[TGT_A0_BOXCHAR:.*]] : !fir.ref<!fir.char<1,?>>, !fir.ref<!fir.char<1,?>>, !fir.ref<!fir.boxchar<1>>, !fir.ref<!fir.boxchar<1>>) {
 !CHECK:    %[[TGT_A0_BC_LD:.*]] = fir.load %[[TGT_A0_BOXCHAR]] : !fir.ref<!fir.boxchar<1>>
 !CHECK:    %[[TGT_A1_BC_LD:.*]] = fir.load %[[TGT_A1_BOXCHAR]] : !fir.ref<!fir.boxchar<1>>
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/optional-argument-map-2.f90 b/external/llvm-project/flang/test/Lower/OpenMP/optional-argument-map-2.f90
index 3b629cfc06d3..e713d71cfbec 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/optional-argument-map-2.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/optional-argument-map-2.f90
@@ -3,24 +3,31 @@
 module mod
   implicit none
 contains
-  subroutine routine(a)
+  subroutine routine_box(a)
     implicit none
     real(4), allocatable, optional, intent(inout) :: a(:)
     integer(4) :: i
 
     !$omp target teams distribute parallel do shared(a)
-        do i=1,10
-            a(i) = i + a(i)
-        end do
+    do i=1,10
+       a(i) = i + a(i)
+    end do
 
-  end subroutine routine
+  end subroutine routine_box
+  subroutine routine_boxchar(a)
+    character(len=*), optional, intent(in) :: a
+    character(len=4) :: b
+    !$omp target map(from: b)
+    b = a
+    !$omp end target
+  end subroutine routine_boxchar
 end module mod
 
-! CHECK-LABEL:   func.func @_QMmodProutine(
+! CHECK-LABEL:   func.func @_QMmodProutine_box(
 ! CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "a", fir.optional}) {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
 ! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable, intent_inout, optional>, uniq_name = "_QMmodFroutineEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable, intent_inout, optional>, uniq_name = "_QMmodFroutine_boxEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.is_present %[[VAL_2]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> i1
 ! CHECK:           %[[VAL_9:.*]]:5 = fir.if %[[VAL_8]] -> (index, index, index, index, index) {
 ! CHECK:             %[[VAL_10:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
@@ -44,3 +51,45 @@ end module mod
 ! CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK:             fir.store %[[VAL_24]] to %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK:           }
+
+
+! CHECK-LABEL:   func.func @_QMmodProutine_boxchar(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.boxchar<1> {fir.bindc_name = "a", fir.optional}) {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.boxchar<1>
+! CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_2:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %[[VAL_1]] {fortran_attrs = #fir.var_attrs<intent_in, optional>, uniq_name = "_QMmodFroutine_boxcharEa"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:           %[[VAL_4:.*]] = arith.constant 4 : index
+! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.char<1,4> {bindc_name = "b", uniq_name = "_QMmodFroutine_boxcharEb"}
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] typeparams %[[VAL_4]] {uniq_name = "_QMmodFroutine_boxcharEb"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:           %[[VAL_7:.*]] = omp.map.info var_ptr(%[[VAL_6]]#1 : !fir.ref<!fir.char<1,4>>, !fir.char<1,4>) map_clauses(from) capture(ByRef) -> !fir.ref<!fir.char<1,4>> {name = "b"}
+! CHECK:           %[[VAL_8:.*]] = fir.is_present %[[VAL_3]]#1 : (!fir.ref<!fir.char<1,?>>) -> i1
+! CHECK:           %[[VAL_9:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_10:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_11:.*]]:2 = fir.if %[[VAL_8]] -> (index, index) {
+! CHECK:             %[[VAL_12:.*]]:2 = fir.unboxchar %[[VAL_3]]#0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:             fir.result %[[VAL_12]]#1, %[[VAL_10]] : index, index
+! CHECK:           } else {
+! CHECK:             fir.result %[[VAL_9]], %[[VAL_9]] : index, index
+! CHECK:           }
+! CHECK:           %[[VAL_13:.*]] = arith.subi %[[VAL_14:.*]]#0, %[[VAL_10]] : index
+! CHECK:           %[[VAL_15:.*]] = omp.map.bounds lower_bound(%[[VAL_9]] : index) upper_bound(%[[VAL_13]] : index) extent(%[[VAL_14]]#0 : index) stride(%[[VAL_14]]#1 : index) start_idx(%[[VAL_9]] : index) {stride_in_bytes = true}
+! CHECK:           %[[VAL_16:.*]] = omp.map.info var_ptr(%[[VAL_3]]#1 : !fir.ref<!fir.char<1,?>>, !fir.char<1,?>) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) bounds(%[[VAL_15]]) -> !fir.ref<!fir.char<1,?>> {name = "a"}
+! CHECK:           fir.store %[[ARG0]] to %[[VAL_0]] : !fir.ref<!fir.boxchar<1>>
+! CHECK:           %[[VAL_17:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_18:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_19:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_20:.*]] = arith.subi %[[VAL_19]]#1, %[[VAL_18]] : index
+! CHECK:           %[[VAL_21:.*]] = omp.map.bounds lower_bound(%[[VAL_17]] : index) upper_bound(%[[VAL_20]] : index) extent(%[[VAL_19]]#1 : index) stride(%[[VAL_18]] : index) start_idx(%[[VAL_17]] : index) {stride_in_bytes = true}
+! CHECK:           %[[VAL_22:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(implicit, to) capture(ByRef) bounds(%[[VAL_21]]) -> !fir.ref<!fir.boxchar<1>> {name = ""}
+! CHECK:           omp.target map_entries(%[[VAL_7]] -> %[[VAL_23:.*]], %[[VAL_16]] -> %[[VAL_24:.*]], %[[VAL_22]] -> %[[VAL_25:.*]] : !fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,?>>, !fir.ref<!fir.boxchar<1>>) {
+! CHECK:             %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.boxchar<1>>
+! CHECK:             %[[VAL_27:.*]]:2 = fir.unboxchar %[[VAL_26]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:             %[[VAL_28:.*]] = arith.constant 4 : index
+! CHECK:             %[[VAL_29:.*]]:2 = hlfir.declare %[[VAL_23]] typeparams %[[VAL_28]] {uniq_name = "_QMmodFroutine_boxcharEb"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:             %[[VAL_30:.*]]:2 = hlfir.declare %[[VAL_24]] typeparams %[[VAL_27]]#1 {fortran_attrs = #fir.var_attrs<intent_in, optional>, uniq_name = "_QMmodFroutine_boxcharEa"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:             hlfir.assign %[[VAL_30]]#0 to %[[VAL_29]]#0 : !fir.boxchar<1>, !fir.ref<!fir.char<1,4>>
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index 96e719faf912..4bfd5d8d1926 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -39,8 +39,8 @@ program reduce
 ! CHECK:             %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_5:.*]] = fir.shape %[[VAL_4]]#1 : (index) -> !fir.shape<1>
 ! CHECK:             %[[VAL_6:.*]] = fir.allocmem !fir.array<?xi32>, %[[VAL_4]]#1 {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:             %[[VAL_7:.*]] = arith.constant true
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:             %[[VAL_7:.*]] = arith.constant true
 ! CHECK:             %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:             %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index 82a70aa8ac2f..060de40d0b75 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! XFAIL: *
 
 program reduce
 integer, dimension(2:4, 2) :: i = 0
@@ -17,19 +18,20 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_15]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>)
 ! CHECK-LABEL:   } init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3x2xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3x2xi32>>>):
-! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.array<3x2xi32> {bindc_name = ".tmp"}
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i32
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_3]], %[[VAL_4]] : (index, index) -> !fir.shape<2>
-! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<3x2xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x2xi32>>, !fir.ref<!fir.array<3x2xi32>>)
+! CHECK:           %[[VAL_6:.*]] = fir.allocmem !fir.array<3x2xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3x2xi32>>, !fir.shape<2>) -> (!fir.heap<!fir.array<3x2xi32>>, !fir.heap<!fir.array<3x2xi32>>)
+! CHECK:           %[[VAL_7:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_9]] : (!fir.box<!fir.array<3x2xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_11:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_11]] : (!fir.box<!fir.array<3x2xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_13:.*]] = fir.shape_shift %[[VAL_10]]#0, %[[VAL_10]]#1, %[[VAL_12]]#0, %[[VAL_12]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
-! CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_8]]#0(%[[VAL_13]]) : (!fir.ref<!fir.array<3x2xi32>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<3x2xi32>>
+! CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_8]]#0(%[[VAL_13]]) : (!fir.heap<!fir.array<3x2xi32>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           hlfir.assign %[[VAL_1]] to %[[VAL_14]] : i32, !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           fir.store %[[VAL_14]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
 ! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>)
@@ -53,6 +55,18 @@ program reduce
 ! CHECK:             }
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>)
+! CHECK:         }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3x2xi32>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3x2xi32>>) -> !fir.ref<!fir.array<3x2xi32>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3x2xi32>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3x2xi32>>) -> !fir.heap<!fir.array<3x2xi32>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3x2xi32>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index de29e257f846..8ad32a636f40 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! XFAIL: *
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -18,17 +19,18 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
 ! CHECK-LABEL:   } init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32> {bindc_name = ".tmp"}
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<3xi32>>,
-!fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
+! CHECK:           %[[TRUE:.*]]  = arith.constant true
+!fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
 ! CHECK:           %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
 ! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
 ! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
 ! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
@@ -49,6 +51,18 @@ program reduce
 ! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CHECK:         }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain()
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
index e3ba768c0cc6..14213fa25cf0 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! XFAIL: *
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -18,16 +19,17 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
 ! CHECK-LABEL:   } init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32>
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32>
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
+! CHECK:           %[[TRUE:.*]]  = arith.constant true
 ! CHECK:           %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
 ! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
 ! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
 ! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
@@ -48,6 +50,18 @@ program reduce
 ! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CHECK:         }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90 b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
index 72360bdc43a6..f638688bc2cc 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
@@ -40,8 +40,8 @@ program reduce
 ! CHECK:             %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_9]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_11:.*]] = fir.shape %[[VAL_10]]#1 : (index) -> !fir.shape<1>
 ! CHECK:             %[[VAL_12:.*]] = fir.allocmem !fir.array<?xi32>, %[[VAL_10]]#1 {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:             %[[VAL_13:.*]] = arith.constant true
 ! CHECK:             %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]](%[[VAL_11]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:             %[[VAL_13:.*]] = arith.constant true
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 0 : index
 ! CHECK:             %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_15]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_17:.*]] = fir.shape_shift %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index) -> !fir.shapeshift<1>
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction3.f90 b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction3.f90
index 208616aa4d5b..6279fac65932 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction3.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/parallel-reduction3.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! XFAIL: *
 
 ! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>> alloc {
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
@@ -11,8 +12,9 @@
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_3]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]]#1 : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.array<?xi32>, %[[VAL_4]]#1 {bindc_name = ".tmp"}
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_6:.*]] = fir.allocmem !fir.array<?xi32>, %[[VAL_4]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:           %[[TRUE:.*]]  = arith.constant true
 ! CHECK:           %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_2]], %[[C0]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
@@ -38,6 +40,18 @@
 ! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>)
+! CHECK:         }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<?xi32>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<?xi32>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<?xi32>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPs(
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90 b/external/llvm-project/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
index 4a6993f2f574..1aa785faa172 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! XFAIL: *
 
 subroutine max_array_reduction(l, r)
   integer :: l(:), r(:)
@@ -19,8 +20,9 @@ subroutine max_array_reduction(l, r)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_4]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<?xi32>, %[[VAL_5]]#1 {bindc_name = ".tmp"}
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_6]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_7:.*]] = fir.allocmem !fir.array<?xi32>, %[[VAL_5]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_6]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_10]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_12:.*]] = fir.shape_shift %[[VAL_11]]#0, %[[VAL_11]]#1 : (index, index) -> !fir.shapeshift<1>
@@ -46,6 +48,18 @@ subroutine max_array_reduction(l, r)
 ! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>)
+! CHECK-LABEL:   }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<?xi32>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<?xi32>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<?xi32>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPmax_array_reduction(
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/requires-atomic-default-mem-order.f90 b/external/llvm-project/flang/test/Lower/OpenMP/requires-atomic-default-mem-order.f90
new file mode 100644
index 000000000000..91cb654aeeb3
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/OpenMP/requires-atomic-default-mem-order.f90
@@ -0,0 +1,22 @@
+!RUN: %flang_fc1 -emit-hlfir %openmp_flags -fopenmp-version=50 %s -o - | FileCheck %s
+
+module m
+!$omp requires atomic_default_mem_order(acq_rel)
+
+contains
+
+!CHECK: %[[V:[0-9]+]]:2 = hlfir.declare {{.*}} {uniq_name = "_QMmFf00Ev"}
+!CHECK: %[[X:[0-9]+]]:2 = hlfir.declare {{.*}} {uniq_name = "_QMmFf00Ex"}
+!CHECK: omp.atomic.read %[[V]]#0 = %[[X]]#0 memory_order(acquire)
+!CHECK: omp.atomic.write %[[X]]#0 = %{{[0-9]+}} memory_order(release)
+
+subroutine f00(x, v)
+  integer :: x, v
+  !$omp atomic read
+    v = x
+
+  !$omp atomic write
+    x = v
+end
+
+end module
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/sections-predetermined-private.f90 b/external/llvm-project/flang/test/Lower/OpenMP/sections-predetermined-private.f90
index 9c2e2e127aa7..3ca3b2219c91 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/sections-predetermined-private.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/sections-predetermined-private.f90
@@ -11,7 +11,7 @@
 end
 ! CHECK-LABEL:   func.func @_QQmain() {
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "i", pinned}
+! CHECK:             %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "i", pinned, uniq_name = "_QFEi"}
 ! CHECK:             %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             omp.sections {
 ! CHECK:               omp.section {
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90 b/external/llvm-project/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90
index 63bbe7a93e77..0a0db7c6ba36 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90
@@ -1,7 +1,6 @@
 ! The "use_device_addr" was added to the "target data" directive in OpenMP 5.0.
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
-! RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
-
+! RUN: %flang_fc1 -fdefer-desc-map -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
+! RUN: bbc -fdefer-desc-map -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
 ! This tests primary goal is to check the promotion of non-CPTR arguments from
 ! use_device_ptr to use_device_addr works, without breaking any functionality.
 
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index eaa388a6f2f5..5b4c5e65ffcc 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -53,8 +53,8 @@ program reduce15
 ! CHECK:             %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_9]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_11:.*]] = fir.shape %[[VAL_10]]#1 : (index) -> !fir.shape<1>
 ! CHECK:             %[[VAL_12:.*]] = fir.allocmem !fir.array<?xi32>, %[[VAL_10]]#1 {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:             %[[VAL_13:.*]] = arith.constant true
 ! CHECK:             %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]](%[[VAL_11]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:             %[[VAL_13:.*]] = arith.constant true
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 0 : index
 ! CHECK:             %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_15]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_17:.*]] = fir.shape_shift %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index) -> !fir.shapeshift<1>
@@ -115,8 +115,8 @@ program reduce15
 ! CHECK:             %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_9]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_11:.*]] = fir.shape %[[VAL_10]]#1 : (index) -> !fir.shape<1>
 ! CHECK:             %[[VAL_12:.*]] = fir.allocmem !fir.array<?xi32>, %[[VAL_10]]#1 {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:             %[[VAL_13:.*]] = arith.constant true
 ! CHECK:             %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]](%[[VAL_11]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.heap<!fir.array<?xi32>>)
+! CHECK:             %[[VAL_13:.*]] = arith.constant true
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 0 : index
 ! CHECK:             %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_15]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_17:.*]] = fir.shape_shift %[[VAL_16]]#0, %[[VAL_16]]#1 : (index, index) -> !fir.shapeshift<1>
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 6255d6e7f3e9..ab1137e2986b 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+! XFAIL: *
 
 program reduce_assumed_shape
 real(8), dimension(2) :: r
@@ -32,8 +33,9 @@ subroutine reduce(r)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_3]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]]#1 : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.array<?xf64>, %[[VAL_4]]#1 {bindc_name = ".tmp"}
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<?xf64>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
+! CHECK:           %[[VAL_6:.*]] = fir.allocmem !fir.array<?xf64>, %[[VAL_4]]#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf64>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.heap<!fir.array<?xf64>>)
+! CHECK:           %[[TRUE:.*]]  = arith.constant true
 ! CHECK:           %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_2]], %[[C0]] : (!fir.box<!fir.array<?xf64>>, index) -> (index, index, index)
 ! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
@@ -59,6 +61,18 @@ subroutine reduce(r)
 ! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<f64>
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf64>>>)
+! CHECK:         }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<?xf64>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<?xf64>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<?xf64>>) -> !fir.heap<!fir.array<?xf64>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<?xf64>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func private @_QFPreduce(
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index da32f65924cb..6cf99a16a795 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! XFAIL: *
 program reduce
 integer :: i = 0
 integer, dimension(2) :: r = 0
@@ -19,16 +20,17 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK-LABEL:  } init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<2xi32> {bindc_name = ".tmp"}
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<2xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<2xi32>>, !fir.heap<!fir.array<2xi32>>)
+! CHECK:           %[[TRUE:.*]]  = arith.constant true
 ! CHECK:           %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<2xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.ref<!fir.array<2xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<2xi32>>
+! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<2xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<2xi32>>
 ! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<2xi32>>
 ! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
 ! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
@@ -50,6 +52,18 @@ program reduce
 ! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
+! CHECK:         }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<2xi32>>) -> !fir.ref<!fir.array<2xi32>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<2xi32>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<2xi32>>) -> !fir.heap<!fir.array<2xi32>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<2xi32>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index 4d32b1ad297d..ebf138745ec6 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! XFAIL: *
 program reduce
 integer :: i = 0
 integer, dimension(2) :: r = 0
@@ -19,16 +20,17 @@ program reduce
 ! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK-LABEL:  } init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.array<2xi32> {bindc_name = ".tmp"}
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<2xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<2xi32>>, !fir.heap<!fir.array<2xi32>>)
+! CHECK:           %[[TRUE:.*]]  = arith.constant true
 ! CHECK:           %[[C0:.*]] = arith.constant 0 : index
 ! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<2xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.ref<!fir.array<2xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<2xi32>>
+! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<2xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<2xi32>>
 ! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<2xi32>>
 ! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
 ! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
@@ -50,6 +52,18 @@ program reduce
 ! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>)
+! CHECK:         }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<2xi32>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<2xi32>>) -> !fir.ref<!fir.array<2xi32>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<2xi32>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<2xi32>>) -> !fir.heap<!fir.array<2xi32>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<2xi32>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
diff --git a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index fc99bdb411e3..9fddf4112af0 100644
--- a/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/external/llvm-project/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-hlfir -fopenmp -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+! XFAIL: *
 
 program main
   implicit none
@@ -29,19 +30,20 @@ program main
 ! CHECK:           omp.yield(%[[VAL_3]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>)
 ! CHECK-LABEL:   } init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3x3xf64>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3x3xf64>>>):
-! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<3x3xf64> {bindc_name = ".tmp"}
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_4]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_6]]) {uniq_name = ".tmp"} : (!fir.ref<!fir.array<3x3xf64>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x3xf64>>, !fir.ref<!fir.array<3x3xf64>>)
+! CHECK:           %[[VAL_7:.*]] = fir.allocmem !fir.array<3x3xf64> {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_6]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3x3xf64>>, !fir.shape<2>) -> (!fir.heap<!fir.array<3x3xf64>>, !fir.heap<!fir.array<3x3xf64>>)
+! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_10]] : (!fir.box<!fir.array<3x3xf64>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_12:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_12]] : (!fir.box<!fir.array<3x3xf64>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_14:.*]] = fir.shape_shift %[[VAL_11]]#0, %[[VAL_11]]#1, %[[VAL_13]]#0, %[[VAL_13]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
-! CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_9]]#0(%[[VAL_14]]) : (!fir.ref<!fir.array<3x3xf64>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<3x3xf64>>
+! CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_9]]#0(%[[VAL_14]]) : (!fir.heap<!fir.array<3x3xf64>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<3x3xf64>>
 ! CHECK:           hlfir.assign %[[VAL_1]] to %[[VAL_15]] : f64, !fir.box<!fir.array<3x3xf64>>
 ! CHECK:           fir.store %[[VAL_15]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
 ! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>)
@@ -66,6 +68,19 @@ program main
 ! CHECK:             }
 ! CHECK:           }
 ! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>)
+
+! CHECK-LABEL:   }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3x3xf64>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3x3xf64>>) -> !fir.ref<!fir.array<3x3xf64>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3x3xf64>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3x3xf64>>) -> !fir.heap<!fir.array<3x3xf64>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3x3xf64>>
+! CHECK:           }
+! CHECK:           omp.yield
 ! CHECK:         }
 
 ! CHECK-LABEL:   omp.declare_reduction @add_reduction_f64 : f64 init {
diff --git a/external/llvm-project/flang/test/Lower/do_concurrent_local_assoc_entity.f90 b/external/llvm-project/flang/test/Lower/do_concurrent_local_assoc_entity.f90
new file mode 100644
index 000000000000..d0a523180b0c
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/do_concurrent_local_assoc_entity.f90
@@ -0,0 +1,38 @@
+! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s
+! XFAIL: *
+subroutine local_assoc
+  implicit none
+  integer  i
+  real, dimension(2:11) :: aa
+
+  associate(a => aa(4:))
+    do concurrent (i = 4:11) local(a)
+      a(i) = 0
+    end do
+  end associate
+end subroutine local_assoc
+
+! CHECK: fir.local {type = local} @[[LOCALIZER:.*local_assocEa.*]] : !fir.box<!fir.array<8xf32>> init {
+! CHECK-NEXT: ^{{.*}}(%{{.*}}: !{{.*}}, %[[LOCAL_ARG:.*]]: !{{.*}}):
+! CHECK-NEXT:   %[[C8:.*]] = arith.constant 8 : index
+! CHECK-NEXT:   %[[SHAPE:.*]] = fir.shape %[[C8]]
+! CHECK-NEXT:   %[[TMP_ALLOC:.*]] = fir.allocmem !{{.*}} {bindc_name = ".tmp", {{.*}}}
+! CHECK:        %[[TMP_DECL:.*]]:2 = hlfir.declare %[[TMP_ALLOC]](%[[SHAPE]])
+! CHECK:        %[[C1:.*]] = arith.constant 1 : index
+! CHECK-NEXT:   %[[C8:.*]] = arith.constant 8 : index
+! CHECK-NEXT:   %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C8]]
+! CHECK-NEXT:   %[[TMP_BOX:.*]] = fir.embox %[[TMP_DECL]]#0(%[[SHAPE_SHIFT]])
+! CHECK-NEXT:   fir.store %[[TMP_BOX]] to %[[LOCAL_ARG]]
+! CHECK-NEXT:   fir.yield(%[[LOCAL_ARG]] : !fir.ref<!fir.box<!fir.array<8xf32>>>)
+! CHECK-NEXT: }
+
+! CHECK: func.func @_QPlocal_assoc()
+! CHECK: %[[BOX_REF:.*]] = fir.alloca !fir.box<!fir.array<8xf32>>
+! CHECK: %[[ASSOC_DECL:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "{{.*}}local_assocEa"}
+! CHECK: %[[ASSOC_BOX:.*]] = fir.embox %[[ASSOC_DECL]]#0(%{{.*}})
+! CHECK: fir.store %[[ASSOC_BOX]] to %[[BOX_REF]]
+! CHECK: fir.do_concurrent.loop {{.*}} local(@[[LOCALIZER]] %[[BOX_REF]] -> %[[LOCAL_ARG:.*]] : !fir.ref<!fir.box<!fir.array<8xf32>>>) {
+! CHECK:   %[[LOCAL_DECL:.*]]:2 = hlfir.declare %[[LOCAL_ARG]]
+! CHECK:   %[[LOCAL_LD:.*]] = fir.load %[[LOCAL_DECL]]#0 : !fir.ref<!fir.box<!fir.array<8xf32>>>
+! CHECK:   hlfir.designate %[[LOCAL_LD]] (%{{.*}})
+! CHECK: }
diff --git a/external/llvm-project/flang/test/Lower/do_concurrent_local_default_init.f90 b/external/llvm-project/flang/test/Lower/do_concurrent_local_default_init.f90
index 207704ac1a99..7f97dc437345 100644
--- a/external/llvm-project/flang/test/Lower/do_concurrent_local_default_init.f90
+++ b/external/llvm-project/flang/test/Lower/do_concurrent_local_default_init.f90
@@ -1,6 +1,6 @@
 ! Test default initialization of DO CONCURRENT LOCAL() entities.
-! RUN: bbc -emit-hlfir -I nowhere -o - %s | FileCheck %s
-
+! RUN: bbc -emit-hlfir --enable-delayed-privatization-staging=true -I nowhere -o - %s | FileCheck %s
+! XFAIL: *
 subroutine test_ptr(p)
   interface
     pure subroutine takes_ptr(p)
@@ -25,28 +25,39 @@ subroutine test_default_init()
   end do
   call something(res)
 end subroutine
+
+! CHECK: fir.local {type = local} @[[DEFAULT_INIT_LOCALIZER:.*test_default_init.*]] : !fir.type<{{.*}}test_default_initTt{i:i32}> init {
+! CHECK-NEXT: ^{{.*}}(%{{.*}}: !{{.*}}, %[[LOCAL_ARG:.*]]: !fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>):
+! CHECK-NEXT:   %[[LOCAL_ARG_BOX:.*]] = fir.embox %[[LOCAL_ARG]]
+! CHECK:        %[[LOCAL_ARG_BOX_CVT:.*]] = fir.convert %[[LOCAL_ARG_BOX]]
+! CHECK:        fir.call @_FortranAInitialize(%[[LOCAL_ARG_BOX_CVT]], {{.*}})
+! CHECK-NEXT:   fir.yield(%[[LOCAL_ARG]] : {{.*}})
+! CHECK-NEXT: }
+
+! CHECK: fir.local {type = local} @[[PTR_LOCALIZER:.*test_ptrEp_private_box.*]] : !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>> init {
+! CHECK-NEXT: ^{{.*}}(%[[ORIG_ARG:.*]]: !{{.*}}, %[[LOCAL_ARG:.*]]: !{{.*}}):
+! CHECK-NEXT:   %[[ORIG_ARG_LD:.*]] = fir.load %[[ORIG_ARG]]
+! CHECK-NEXT:   %[[ELEM_SIZE:.*]] = fir.box_elesize %[[ORIG_ARG_LD]]
+! CHECK-NEXT:   %[[C0:.*]] = arith.constant 0 : index
+! CHECK-NEXT:   %[[SHAPE:.*]] = fir.shape %[[C0]]
+! CHECK-NEXT:   %[[ZERO_BITS:.*]] = fir.zero_bits
+! CHECK-NEXT:   %[[LOCAL_BOX:.*]] = fir.embox %[[ZERO_BITS]](%[[SHAPE]]) typeparams %[[ELEM_SIZE]]
+! CHECK-NEXT:   fir.store %[[LOCAL_BOX]] to %[[LOCAL_ARG]]
+! CHECK-NEXT:   fir.yield(%[[LOCAL_ARG]] : {{.*}})
+! CHECK-NEXT: }
+
 ! CHECK-LABEL:   func.func @_QPtest_ptr(
 ! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>> {fir.bindc_name = "p"}) {
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>
 ! CHECK:           %[[VAL_7:.*]] = fir.box_elesize %[[VAL_6]] : (!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>) -> index
-! CHECK:           fir.do_concurrent.loop
-! CHECK:             %[[VAL_16:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>> {bindc_name = "p", pinned, uniq_name = "_QFtest_ptrEp"}
-! CHECK:             %[[VAL_17:.*]] = fir.zero_bits !fir.ptr<!fir.array<?x!fir.char<1,?>>>
-! CHECK:             %[[VAL_18:.*]] = arith.constant 0 : index
-! CHECK:             %[[VAL_19:.*]] = fir.shape %[[VAL_18]] : (index) -> !fir.shape<1>
-! CHECK:             %[[VAL_20:.*]] = fir.embox %[[VAL_17]](%[[VAL_19]]) typeparams %[[VAL_7]] : (!fir.ptr<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index) -> !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>
-! CHECK:             fir.store %[[VAL_20]] to %[[VAL_16]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>
-! CHECK:             %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_16]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptrEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>)
+! CHECK:           fir.do_concurrent.loop {{.*}} local(@[[PTR_LOCALIZER]] %{{.*}}#0 -> %[[LOCAL_ARG:.*]] : {{.*}})
+! CHECK:             %[[VAL_21:.*]]:2 = hlfir.declare %[[LOCAL_ARG]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptrEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>)
 ! CHECK:             fir.call @_QPtakes_ptr(%[[VAL_21]]#0) proc_attrs<pure> fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) -> ()
 ! CHECK:           }
 ! CHECK:           return
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPtest_default_init(
-! CHECK:           fir.do_concurrent.loop
-! CHECK:             %[[VAL_26:.*]] = fir.alloca !fir.type<_QFtest_default_initTt{i:i32}> {bindc_name = "a", pinned, uniq_name = "_QFtest_default_initEa"}
-! CHECK:             %[[VAL_27:.*]] = fir.embox %[[VAL_26]] : (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>
-! CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_27]] : (!fir.box<!fir.type<_QFtest_default_initTt{i:i32}>>) -> !fir.box<none>
-! CHECK:             fir.call @_FortranAInitialize(%[[VAL_30]], {{.*}}
-! CHECK:             %[[VAL_33:.*]]:2 = hlfir.declare %[[VAL_26]] {uniq_name = "_QFtest_default_initEa"} : (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>) -> (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>, !fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>)
+! CHECK:           fir.do_concurrent.loop {{.*}} local(@[[DEFAULT_INIT_LOCALIZER]] %{{.*}}#0 -> %[[LOCAL_ARG:.*]] : {{.*}})
+! CHECK:             %[[VAL_33:.*]]:2 = hlfir.declare %[[LOCAL_ARG]] {uniq_name = "_QFtest_default_initEa"} : (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>) -> (!fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>, !fir.ref<!fir.type<_QFtest_default_initTt{i:i32}>>)
 ! CHECK:           }
diff --git a/external/llvm-project/flang/test/Lower/namelist.f90 b/external/llvm-project/flang/test/Lower/namelist.f90
index 9f9bbec13be9..94b0ef11cb10 100644
--- a/external/llvm-project/flang/test/Lower/namelist.f90
+++ b/external/llvm-project/flang/test/Lower/namelist.f90
@@ -130,15 +130,15 @@ module mmm
 
 ! CHECK-LABEL: c.func @_QPrename_sub
 subroutine rename_sub
-  use mmm, zzz => aaa
+  use mmm, ggg => aaa
   rrr = 3.
   ! CHECK:     %[[V_4:[0-9]+]] = fir.call @_FortranAioBeginExternalListOutput
   ! CHECK:     %[[V_5:[0-9]+]] = fir.address_of(@_QMmmmNaaa) : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
   ! CHECK:     %[[V_6:[0-9]+]] = fir.convert %[[V_5]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
   ! CHECK:     %[[V_7:[0-9]+]] = fir.call @_FortranAioOutputNamelist(%[[V_4]], %[[V_6]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<tuple<>>) -> i1
   ! CHECK:     %[[V_8:[0-9]+]] = fir.call @_FortranAioEndIoStatement(%[[V_4]]) fastmath<contract> : (!fir.ref<i8>) -> i32
-  write(*,zzz)
+  write(*,ggg)
 end
 
-! CHECK-NOT:   zzz
+! CHECK-NOT:   ggg
 ! CHECK:       fir.string_lit "aaa\00"(4) : !fir.char<1,4>
diff --git a/external/llvm-project/flang/test/Lower/volatile-allocatable.f90 b/external/llvm-project/flang/test/Lower/volatile-allocatable.f90
index 59e724bce846..c33d368c5858 100644
--- a/external/llvm-project/flang/test/Lower/volatile-allocatable.f90
+++ b/external/llvm-project/flang/test/Lower/volatile-allocatable.f90
@@ -127,8 +127,8 @@ subroutine test_unlimited_polymorphic()
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableAllocate(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK:           fir.call @_FortranAAllocatableInitDerivedForAllocate(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<none>, i32, i32) -> ()
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableAllocate(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}} {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFtest_scalar_volatileEv1"} : (!fir.box<!fir.heap<!fir.type<{{.*}}>>, volatile>) -> (!fir.box<!fir.heap<!fir.type<{{.*}}>>, volatile>, !fir.box<!fir.heap<!fir.type<{{.*}}>>, volatile>)
-! CHECK:           %{{.+}} = hlfir.designate %{{.+}}#0{"j"}   : (!fir.box<!fir.heap<!fir.type<{{.*}}>>, volatile>) -> !fir.ref<i32, volatile>
+! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}} {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFtest_scalar_volatileEv1"} : (!fir.box<!fir.heap<!fir.type<{{.*}}>>, volatile>) -> (!fir.box<!fir.type<{{.*}}>, volatile>, !fir.box<!fir.type<{{.*}}>, volatile>)
+! CHECK:           %{{.+}} = hlfir.designate %{{.+}}#0{"j"}   : (!fir.box<!fir.type<{{.*}}>, volatile>) -> !fir.ref<i32, volatile>
 ! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}} {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQro._QMderived_typesText_type.0"} : (!fir.ref<!fir.type<{{.*}}>>) -> (!fir.ref<!fir.type<{{.*}}>>, !fir.ref<!fir.type<{{.*}}>>)
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableAllocateSource(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}} typeparams %{{.+}} {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQclX766F6C6174696C6520636861726163746572"} : (!fir.ref<!fir.char<1,18>>, index) -> (!fir.ref<!fir.char<1,18>>, !fir.ref<!fir.char<1,18>>)
@@ -156,8 +156,8 @@ subroutine test_unlimited_polymorphic()
 ! CHECK:           fir.call @_FortranAAllocatableSetBounds(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableAllocate(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK:           %{{.+}} = fir.call @_FortranAClassIs(%{{.+}}, %{{.+}}) : (!fir.box<none>, !fir.ref<none>) -> i1
-! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}}(%{{.+}}) {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFtest_select_base_type_volatileEv"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, !fir.shift<1>) -> (!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, !fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>)
-! CHECK:           %{{.+}} = hlfir.designate %{{.+}}#0 (%{{.+}})  : (!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, index) -> !fir.class<!fir.type<{{.*}}>, volatile>
+! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}}(%{{.+}}) {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFtest_select_base_type_volatileEv"} : (!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, !fir.shift<1>) -> (!fir.class<!fir.array<?x!fir.type<{{.*}}>>, volatile>, !fir.class<!fir.array<?x!fir.type<{{.*}}>>, volatile>)
+! CHECK:           %{{.+}} = hlfir.designate %{{.+}}#0 (%{{.+}})  : (!fir.class<!fir.array<?x!fir.type<{{.*}}>>, volatile>, index) -> !fir.class<!fir.type<{{.*}}>, volatile>
 ! CHECK:           %{{.+}} = hlfir.designate %{{.+}}{"i"}   : (!fir.class<!fir.type<{{.*}}>, volatile>) -> !fir.ref<i32, volatile>
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableDeallocatePolymorphic(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableDeallocatePolymorphic(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
@@ -186,7 +186,7 @@ subroutine test_unlimited_polymorphic()
 ! CHECK:           fir.call @_FortranAAllocatableInitIntrinsicForAllocate(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, i32, i32, i32, i32) -> ()
 ! CHECK:           fir.call @_FortranAAllocatableSetBounds(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableAllocate(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}}(%{{.+}}) {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFtest_unlimited_polymorphicEupa"} : (!fir.box<!fir.heap<!fir.array<?xf32>>, volatile>, !fir.shift<1>) -> (!fir.box<!fir.heap<!fir.array<?xf32>>, volatile>, !fir.box<!fir.heap<!fir.array<?xf32>>, volatile>)
+! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}}(%{{.+}}) {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFtest_unlimited_polymorphicEupa"} : (!fir.box<!fir.heap<!fir.array<?xf32>>, volatile>, !fir.shift<1>) -> (!fir.box<!fir.array<?xf32>, volatile>, !fir.box<!fir.array<?xf32>, volatile>)
 ! CHECK:           %{{.+}}:2 = hlfir.declare %{{.+}}(%{{.+}}) {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQro.3xr4.3"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableDeallocatePolymorphic(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK:           %{{.+}} = fir.call @_FortranAAllocatableDeallocatePolymorphic(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.ref<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
diff --git a/external/llvm-project/flang/test/Lower/volatile-openmp.f90 b/external/llvm-project/flang/test/Lower/volatile-openmp.f90
index bcea21ebd10b..4b2d9e1b552b 100644
--- a/external/llvm-project/flang/test/Lower/volatile-openmp.f90
+++ b/external/llvm-project/flang/test/Lower/volatile-openmp.f90
@@ -1,4 +1,4 @@
-! RUN: bbc --strict-fir-volatile-verifier -fopenmp %s -o - | FileCheck %s
+! RUN: bbc -fdefer-desc-map --strict-fir-volatile-verifier -fopenmp %s -o - | FileCheck %s
 type t
 integer, pointer :: array(:)
 end type
diff --git a/external/llvm-project/flang/test/Lower/volatile-openmp1.f90 b/external/llvm-project/flang/test/Lower/volatile-openmp1.f90
new file mode 100644
index 000000000000..163db953b6b8
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/volatile-openmp1.f90
@@ -0,0 +1,46 @@
+! RUN: bbc --strict-fir-volatile-verifier -fopenmp %s -o - | FileCheck %s
+program main
+implicit none
+integer,volatile::a
+integer::n,i
+a=0
+n=1000
+!$omp parallel 
+!$omp do reduction(+:a)
+  do i=1,n
+    a=a+1
+  end do
+!$omp end parallel
+end program
+
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_1:.*]] = arith.constant 1000 : i32
+! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
+! CHECK:           %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
+! CHECK:           %[[VAL_5:.*]] = fir.volatile_cast %[[VAL_4]] : (!fir.ref<i32>) -> !fir.ref<i32, volatile>
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFEa"} : (!fir.ref<i32, volatile>) -> (!fir.ref<i32, volatile>, !fir.ref<i32, volatile>)
+! CHECK:           %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_9:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFEn"}
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {uniq_name = "_QFEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_6]]#0 : i32, !fir.ref<i32, volatile>
+! CHECK:           hlfir.assign %[[VAL_1]] to %[[VAL_10]]#0 : i32, !fir.ref<i32>
+! CHECK:           omp.parallel {
+! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<i32>
+! CHECK:             omp.wsloop private(@_QFEi_private_i32 %[[VAL_8]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) reduction(@add_reduction_i32 %[[VAL_6]]#0 -> %[[VAL_13:.*]] : !fir.ref<i32, volatile>) {
+! CHECK:               omp.loop_nest (%[[VAL_14:.*]]) : i32 = (%[[VAL_0]]) to (%[[VAL_11]]) inclusive step (%[[VAL_0]]) {
+! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:                 %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_13]] {fortran_attrs = #fir.var_attrs<volatile>, uniq_name = "_QFEa"} : (!fir.ref<i32, volatile>) -> (!fir.ref<i32, volatile>, !fir.ref<i32, volatile>)
+! CHECK:                 hlfir.assign %[[VAL_14]] to %[[VAL_15]]#0 : i32, !fir.ref<i32>
+! CHECK:                 %[[VAL_17:.*]] = fir.load %[[VAL_16]]#0 : !fir.ref<i32, volatile>
+! CHECK:                 %[[VAL_18:.*]] = arith.addi %[[VAL_17]], %[[VAL_0]] : i32
+! CHECK:                 hlfir.assign %[[VAL_18]] to %[[VAL_16]]#0 : i32, !fir.ref<i32, volatile>
+! CHECK:                 omp.yield
+! CHECK:               }
+! CHECK:             }
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
diff --git a/external/llvm-project/flang/test/Lower/volatile-openmp2.f03 b/external/llvm-project/flang/test/Lower/volatile-openmp2.f03
new file mode 100644
index 000000000000..4df14ea52ff9
--- /dev/null
+++ b/external/llvm-project/flang/test/Lower/volatile-openmp2.f03
@@ -0,0 +1,34 @@
+! RUN: bbc -fopenmp --strict-fir-volatile-verifier %s -o - | FileCheck %s
+type x
+    integer::x1=1
+end type x
+class(x),allocatable,volatile,asynchronous::v(:)
+!$omp parallel private(v)
+!$omp end parallel
+end
+
+! CHECK-LABEL:   func.func @_QQmain() {
+! CHECK:           %[[VAL_0:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+! CHECK:           %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_4:.*]] = fir.address_of(@_QFE.n.x1) : !fir.ref<!fir.char<1,2>>
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] typeparams %[[VAL_2]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.n.x1"} : (!fir.ref<!fir.char<1,2>>, index) -> (!fir.ref<!fir.char<1,2>>, !fir.ref<!fir.char<1,2>>)
+! CHECK:           %[[VAL_6:.*]] = fir.address_of(@_QFE.di.x.x1) : !fir.ref<i32>
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.di.x.x1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_8:.*]] = fir.address_of(@_QFE.n.x) : !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] typeparams %[[VAL_1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.n.x"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+! CHECK:           %[[VAL_10:.*]] = fir.address_of(@_QFEv) : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>>>
+! CHECK:           %[[VAL_11:.*]] = fir.volatile_cast %[[VAL_10]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>>>) -> !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>
+! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {fortran_attrs = #fir.var_attrs<allocatable, asynchronous, volatile>, uniq_name = "_QFEv"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>)
+! CHECK:           %[[VAL_13:.*]] = fir.address_of(@_QFE.c.x) : !fir.ref<!fir.array<1x!fir.type<{{.*}}>>>
+! CHECK:           %[[VAL_14:.*]] = fir.shape_shift %[[VAL_0]], %[[VAL_1]] : (index, index) -> !fir.shapeshift<1>
+! CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]](%[[VAL_14]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.c.x"} : (!fir.ref<!fir.array<1x!fir.type<{{.*}}>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<1x!fir.type<{{.*}}>>>, !fir.ref<!fir.array<1x!fir.type<{{.*}}>>>)
+! CHECK:           %[[VAL_16:.*]] = fir.address_of(@_QFE.dt.x) : !fir.ref<!fir.type<{{.*}}>>
+! CHECK:           %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_16]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFE.dt.x"} : (!fir.ref<!fir.type<{{.*}}>>) -> (!fir.ref<!fir.type<{{.*}}>>, !fir.ref<!fir.type<{{.*}}>>)
+! CHECK:           omp.parallel private(@_QFEv_private_class_heap_Uxrec__QFTx %[[VAL_12]]#0 -> %[[VAL_18:.*]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>) {
+! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {fortran_attrs = #fir.var_attrs<allocatable, asynchronous, volatile>, uniq_name = "_QFEv"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<{{.*}}>>>, volatile>, volatile>)
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
diff --git a/external/llvm-project/flang/test/Parser/OpenMP/atomic-compare.f90 b/external/llvm-project/flang/test/Parser/OpenMP/atomic-compare.f90
index 5cd02698ff48..e09da4a359fc 100644
--- a/external/llvm-project/flang/test/Parser/OpenMP/atomic-compare.f90
+++ b/external/llvm-project/flang/test/Parser/OpenMP/atomic-compare.f90
@@ -1,16 +1,290 @@
-! RUN: not %flang_fc1  -fopenmp-version=51 -fopenmp %s 2>&1 | FileCheck %s
-! OpenMP version for documentation purposes only - it isn't used until Sema.
-! This is testing for Parser errors that bail out before Sema. 
-program main
-   implicit none
-   integer :: i, j = 10
-   logical :: r
-
-  !CHECK: error: expected OpenMP construct
-  !$omp atomic compare write
-  r =  i .eq. j + 1
-
-  !CHECK: error: expected end of line
-  !$omp atomic compare num_threads(4)
-  r = i .eq. j
-end program main
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(a, b)
+  integer :: a, b
+  integer :: x
+  !$omp atomic update compare
+  if (x < a) x = b
+end
+
+!UNPARSE: SUBROUTINE f00 (a, b)
+!UNPARSE:  INTEGER a, b
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP ATOMIC UPDATE COMPARE
+!UNPARSE:  IF (x<a)  x=b
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Update ->
+!PARSE-TREE: | | OmpClause -> Compare
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt
+!PARSE-TREE: | | | Scalar -> Logical -> Expr = 'x<a'
+!PARSE-TREE: | | | | LT
+!PARSE-TREE: | | | | | Expr = 'x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'a'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'a'
+!PARSE-TREE: | | | ActionStmt -> AssignmentStmt = 'x=b'
+!PARSE-TREE: | | | | Variable = 'x'
+!PARSE-TREE: | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | Expr = 'b'
+!PARSE-TREE: | | | | | Designator -> DataRef -> Name = 'b'
+
+subroutine f01(a, b)
+  integer :: a, b
+  integer :: x
+  !$omp atomic update compare
+  if (x < a) then
+    x = b
+  endif
+end
+
+!UNPARSE: SUBROUTINE f01 (a, b)
+!UNPARSE:  INTEGER a, b
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP ATOMIC UPDATE COMPARE
+!UNPARSE:  IF (x<a) THEN
+!UNPARSE:    x=b
+!UNPARSE:  END IF
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Update ->
+!PARSE-TREE: | | OmpClause -> Compare
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct
+!PARSE-TREE: | | | IfThenStmt
+!PARSE-TREE: | | | | Scalar -> Logical -> Expr = 'x<a'
+!PARSE-TREE: | | | | | LT
+!PARSE-TREE: | | | | | | Expr = 'x'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | | Expr = 'a'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'a'
+!PARSE-TREE: | | | Block
+!PARSE-TREE: | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=b'
+!PARSE-TREE: | | | | | Variable = 'x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'b'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'b'
+!PARSE-TREE: | | | EndIfStmt ->
+
+subroutine f02(a, b)
+  integer :: a, b
+  integer :: x
+  logical :: c
+  c = x < a
+  !$omp atomic update compare
+  if (c) then
+    x = b
+  endif
+end
+
+!UNPARSE: SUBROUTINE f02 (a, b)
+!UNPARSE:  INTEGER a, b
+!UNPARSE:  INTEGER x
+!UNPARSE:  LOGICAL c
+!UNPARSE:   c=x<a
+!UNPARSE: !$OMP ATOMIC UPDATE COMPARE
+!UNPARSE:  IF (c) THEN
+!UNPARSE:    x=b
+!UNPARSE:  END IF
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'c=x<a'
+!PARSE-TREE: | Variable = 'c'
+!PARSE-TREE: | | Designator -> DataRef -> Name = 'c'
+!PARSE-TREE: | Expr = 'x<a'
+!PARSE-TREE: | | LT
+!PARSE-TREE: | | | Expr = 'x'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | Expr = 'a'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'a'
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Update ->
+!PARSE-TREE: | | OmpClause -> Compare
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct
+!PARSE-TREE: | | | IfThenStmt
+!PARSE-TREE: | | | | Scalar -> Logical -> Expr = 'c'
+!PARSE-TREE: | | | | | Designator -> DataRef -> Name = 'c'
+!PARSE-TREE: | | | Block
+!PARSE-TREE: | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=b'
+!PARSE-TREE: | | | | | Variable = 'x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'b'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'b'
+!PARSE-TREE: | | | EndIfStmt ->
+
+subroutine g00(a, b)
+  integer :: a, b
+  integer :: x, v
+  !$omp atomic update capture compare
+  v = x
+  if (x < a) x = b
+  !$omp end atomic
+end
+
+!UNPARSE: SUBROUTINE g00 (a, b)
+!UNPARSE:  INTEGER a, b
+!UNPARSE:  INTEGER x, v
+!UNPARSE: !$OMP ATOMIC UPDATE CAPTURE COMPARE
+!UNPARSE:   v=x
+!UNPARSE:  IF (x<a)  x=b
+!UNPARSE: !$OMP END ATOMIC
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Update ->
+!PARSE-TREE: | | OmpClause -> Capture
+!PARSE-TREE: | | OmpClause -> Compare
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'v=x'
+!PARSE-TREE: | | | Variable = 'v'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'v'
+!PARSE-TREE: | | | Expr = 'x'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt
+!PARSE-TREE: | | | Scalar -> Logical -> Expr = 'x<a'
+!PARSE-TREE: | | | | LT
+!PARSE-TREE: | | | | | Expr = 'x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'a'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'a'
+!PARSE-TREE: | | | ActionStmt -> AssignmentStmt = 'x=b'
+!PARSE-TREE: | | | | Variable = 'x'
+!PARSE-TREE: | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | Expr = 'b'
+!PARSE-TREE: | | | | | Designator -> DataRef -> Name = 'b'
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList ->
+!PARSE-TREE: | | Flags = None
+
+subroutine g01(a, b)
+  integer :: a, b
+  integer :: x, v
+  !$omp atomic update capture compare
+  v = x
+  if (x < a) then
+    x = b
+  endif
+  !$omp end atomic
+end
+
+!UNPARSE: SUBROUTINE g01 (a, b)
+!UNPARSE:  INTEGER a, b
+!UNPARSE:  INTEGER x, v
+!UNPARSE: !$OMP ATOMIC UPDATE CAPTURE COMPARE
+!UNPARSE:   v=x
+!UNPARSE:  IF (x<a) THEN
+!UNPARSE:    x=b
+!UNPARSE:  END IF
+!UNPARSE: !$OMP END ATOMIC
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Update ->
+!PARSE-TREE: | | OmpClause -> Capture
+!PARSE-TREE: | | OmpClause -> Compare
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'v=x'
+!PARSE-TREE: | | | Variable = 'v'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'v'
+!PARSE-TREE: | | | Expr = 'x'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct
+!PARSE-TREE: | | | IfThenStmt
+!PARSE-TREE: | | | | Scalar -> Logical -> Expr = 'x<a'
+!PARSE-TREE: | | | | | LT
+!PARSE-TREE: | | | | | | Expr = 'x'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | | Expr = 'a'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'a'
+!PARSE-TREE: | | | Block
+!PARSE-TREE: | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=b'
+!PARSE-TREE: | | | | | Variable = 'x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'b'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'b'
+!PARSE-TREE: | | | EndIfStmt ->
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList ->
+!PARSE-TREE: | | Flags = None
+
+subroutine g02(a, b)
+  integer :: a, b
+  integer :: x, v
+  !$omp atomic update capture compare
+  if (x < a) then
+    x = b
+  else
+    v = x
+  endif
+  !$omp end atomic
+end
+
+!UNPARSE: SUBROUTINE g02 (a, b)
+!UNPARSE:  INTEGER a, b
+!UNPARSE:  INTEGER x, v
+!UNPARSE: !$OMP ATOMIC UPDATE CAPTURE COMPARE
+!UNPARSE:  IF (x<a) THEN
+!UNPARSE:    x=b
+!UNPARSE:  ELSE
+!UNPARSE:    v=x
+!UNPARSE:  END IF
+!UNPARSE: !$OMP END ATOMIC
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Update ->
+!PARSE-TREE: | | OmpClause -> Capture
+!PARSE-TREE: | | OmpClause -> Compare
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct
+!PARSE-TREE: | | | IfThenStmt
+!PARSE-TREE: | | | | Scalar -> Logical -> Expr = 'x<a'
+!PARSE-TREE: | | | | | LT
+!PARSE-TREE: | | | | | | Expr = 'x'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | | Expr = 'a'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'a'
+!PARSE-TREE: | | | Block
+!PARSE-TREE: | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=b'
+!PARSE-TREE: | | | | | Variable = 'x'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | | | Expr = 'b'
+!PARSE-TREE: | | | | | | Designator -> DataRef -> Name = 'b'
+!PARSE-TREE: | | | ElseBlock
+!PARSE-TREE: | | | | ElseStmt ->
+!PARSE-TREE: | | | | Block
+!PARSE-TREE: | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'v=x'
+!PARSE-TREE: | | | | | | Variable = 'v'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'v'
+!PARSE-TREE: | | | | | | Expr = 'x'
+!PARSE-TREE: | | | | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | EndIfStmt ->
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList ->
+!PARSE-TREE: | | Flags = None
diff --git a/external/llvm-project/flang/test/Parser/OpenMP/atomic-end.f90 b/external/llvm-project/flang/test/Parser/OpenMP/atomic-end.f90
new file mode 100644
index 000000000000..e5eac87517b1
--- /dev/null
+++ b/external/llvm-project/flang/test/Parser/OpenMP/atomic-end.f90
@@ -0,0 +1,63 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00
+  integer :: x, v
+  !$omp atomic read
+  v = x
+  !$omp end atomic
+end
+
+!UNPARSE: SUBROUTINE f00
+!UNPARSE:  INTEGER x, v
+!UNPARSE: !$OMP ATOMIC READ
+!UNPARSE:   v=x
+!UNPARSE: !$OMP END ATOMIC
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Read
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'v=x'
+!PARSE-TREE: | | | Variable = 'v'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'v'
+!PARSE-TREE: | | | Expr = 'x'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList ->
+!PARSE-TREE: | | Flags = None
+
+
+subroutine f01
+  integer :: x, v
+  !$omp atomic read
+  v = x
+  !$omp endatomic
+end
+
+!UNPARSE: SUBROUTINE f01
+!UNPARSE:  INTEGER x, v
+!UNPARSE: !$OMP ATOMIC READ
+!UNPARSE:   v=x
+!UNPARSE: !$OMP END ATOMIC
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Read
+!PARSE-TREE: | | Flags = None
+!PARSE-TREE: | Block
+!PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'v=x'
+!PARSE-TREE: | | | Variable = 'v'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'v'
+!PARSE-TREE: | | | Expr = 'x'
+!PARSE-TREE: | | | | Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpDirectiveSpecification
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = atomic
+!PARSE-TREE: | | OmpClauseList ->
+!PARSE-TREE: | | Flags = None
diff --git a/external/llvm-project/flang/test/Parser/OpenMP/declare-reduction-multi.f90 b/external/llvm-project/flang/test/Parser/OpenMP/declare-reduction-multi.f90
new file mode 100644
index 000000000000..0e1adcc9958d
--- /dev/null
+++ b/external/llvm-project/flang/test/Parser/OpenMP/declare-reduction-multi.f90
@@ -0,0 +1,134 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+!! Test multiple declarations for the same type, with different operations.
+module mymod
+  type :: tt
+     real r
+  end type tt
+contains
+  function mymax(a, b)
+    type(tt) :: a, b, mymax
+    if (a%r > b%r) then
+       mymax = a
+    else
+       mymax = b
+    end if
+  end function mymax
+end module mymod
+
+program omp_examples
+!CHECK-LABEL: PROGRAM omp_examples
+  use mymod
+  implicit none
+  integer, parameter :: n = 100
+  integer :: i
+  type(tt) :: values(n), sum, prod, big, small
+
+  !$omp declare reduction(+:tt:omp_out%r = omp_out%r + omp_in%r) initializer(omp_priv%r = 0)
+!CHECK: !$OMP DECLARE REDUCTION (+:tt: omp_out%r=omp_out%r+omp_in%r
+!CHECK-NEXT: ) INITIALIZER(omp_priv%r=0_4)
+!PARSE-TREE:  DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE:  Verbatim
+!PARSE-TREE: OmpReductionSpecifier
+!PARSE-TREE-NEXT: OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE-NEXT:  Name = 'tt'
+!PARSE-TREE: OmpReductionCombiner -> AssignmentStmt = 'omp_out%r=omp_out%r+omp_in%r'
+!PARSE-TREE: OmpClauseList -> OmpClause -> Initializer -> OmpInitializerClause -> AssignmentStmt = 'omp_priv%r=0._4
+  !$omp declare reduction(*:tt:omp_out%r = omp_out%r * omp_in%r) initializer(omp_priv%r = 1)
+!CHECK-NEXT: !$OMP DECLARE REDUCTION (*:tt: omp_out%r=omp_out%r*omp_in%r
+!CHECK-NEXT: ) INITIALIZER(omp_priv%r=1_4)
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE: Verbatim
+!PARSE-TREE: OmpReductionSpecifier
+!PARSE-TREE: OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Multiply
+!PARSE-TREE: OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE-NEXT:  Name = 'tt'
+!PARSE-TREE: OmpReductionCombiner -> AssignmentStmt = 'omp_out%r=omp_out%r*omp_in%r'
+!PARSE-TREE: OmpClauseList -> OmpClause -> Initializer -> OmpInitializerClause -> AssignmentStmt = 'omp_priv%r=1._4'
+  !$omp declare reduction(max:tt:omp_out = mymax(omp_out, omp_in)) initializer(omp_priv%r = 0)
+!CHECK-NEXT: !$OMP DECLARE REDUCTION (max:tt: omp_out=mymax(omp_out,omp_in)
+!CHECK-NEXT: ) INITIALIZER(omp_priv%r=0_4)
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE: Verbatim
+!PARSE-TREE: OmpReductionSpecifier
+!PARSE-TREE: OmpReductionIdentifier -> ProcedureDesignator -> Name = 'max'
+!PARSE-TREE: OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: Name = 'tt'
+!PARSE-TREE: OmpReductionCombiner -> AssignmentStmt = 'omp_out=mymax(omp_out,omp_in)'
+!PARSE-TREE: OmpClauseList -> OmpClause -> Initializer -> OmpInitializerClause -> AssignmentStmt = 'omp_priv%r=0._4'
+  !$omp declare reduction(min:tt:omp_out%r = min(omp_out%r, omp_in%r)) initializer(omp_priv%r = 1)
+!CHECK-NEXT: !$OMP DECLARE REDUCTION (min:tt: omp_out%r=min(omp_out%r,omp_in%r)
+!CHECK-NEXT: ) INITIALIZER(omp_priv%r=1_4)
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE: Verbatim
+!PARSE-TREE: OmpReductionSpecifier
+!PARSE-TREE: OmpReductionIdentifier -> ProcedureDesignator -> Name = 'min'
+!PARSE-TREE: OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
+!PARSE-TREE: Name = 'tt'
+!PARSE-TREE: OmpReductionCombiner -> AssignmentStmt = 'omp_out%r=min(omp_out%r,omp_in%r)'
+!PARSE-TREE: OmpClauseList -> OmpClause -> Initializer -> OmpInitializerClause -> AssignmentStmt = 'omp_priv%r=1._4'
+  call random_number(values%r)
+
+  sum%r = 0
+  !$omp parallel do reduction(+:sum)
+!CHECK: !$OMP PARALLEL DO  REDUCTION(+: sum) 
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!PARSE-TREE: OmpBeginLoopDirective
+!PARSE-TREE: OmpLoopDirective -> llvm::omp::Directive = parallel do
+!PARSE-TREE: OmpClauseList -> OmpClause -> Reduction -> OmpReductionClause
+!PARSE-TREE: Modifier -> OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'sum
+!PARSE-TREE: DoConstruct
+  do i = 1, n
+     sum%r = sum%r + values(i)%r
+  end do
+
+  prod%r = 1
+  !$omp parallel do reduction(*:prod)
+!CHECK: !$OMP PARALLEL DO  REDUCTION(*: prod)
+!PARSE-TREE:  ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!PARSE-TREE: OmpBeginLoopDirective
+!PARSE-TREE: OmpLoopDirective -> llvm::omp::Directive = parallel do
+!PARSE-TREE: OmpClauseList -> OmpClause -> Reduction -> OmpReductionClause
+!PARSE-TREE: Modifier -> OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Multiply
+!PARSE-TREE: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'prod'
+!PARSE-TREE: DoConstruct
+  do i = 1, n
+     prod%r = prod%r * (values(i)%r+0.6)
+  end do
+
+  big%r = 0
+  !$omp parallel do reduction(max:big)
+!CHECK:  $OMP PARALLEL DO  REDUCTION(max: big) 
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!PARSE-TREE: OmpBeginLoopDirective
+!PARSE-TREE: OmpLoopDirective -> llvm::omp::Directive = parallel do
+!PARSE-TREE: OmpClauseList -> OmpClause -> Reduction -> OmpReductionClause
+!PARSE-TREE: Modifier -> OmpReductionIdentifier -> ProcedureDesignator -> Name = 'max'
+!PARSE-TREE: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'big'
+!PARSE-TREE: DoConstruct
+  do i = 1, n
+     big = mymax(values(i), big)
+  end do
+
+  small%r = 1
+  !$omp parallel do reduction(min:small)
+!CHECK: !$OMP PARALLEL DO  REDUCTION(min: small)
+!CHECK-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-TREE: OmpBeginLoopDirective
+!CHECK-TREE: OmpLoopDirective -> llvm::omp::Directive = parallel do
+!CHECK-TREE: OmpClauseList -> OmpClause -> Reduction -> OmpReductionClause
+!CHECK-TREE: Modifier -> OmpReductionIdentifier -> ProcedureDesignator -> Name = 'min'
+!CHECK-TREE: OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'small'
+!CHECK-TREE: DoConstruct
+  do i = 1, n
+     small%r = min(values(i)%r, small%r)
+  end do
+  
+  print *, values%r
+  print *, "sum=", sum%r
+  print *, "prod=", prod%r
+  print *, "small=", small%r, " big=", big%r
+end program omp_examples
diff --git a/external/llvm-project/flang/test/Parser/OpenMP/declare-reduction-operator.f90 b/external/llvm-project/flang/test/Parser/OpenMP/declare-reduction-operator.f90
new file mode 100644
index 000000000000..7bfb78115b10
--- /dev/null
+++ b/external/llvm-project/flang/test/Parser/OpenMP/declare-reduction-operator.f90
@@ -0,0 +1,59 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+!CHECK-LABEL: SUBROUTINE reduce_1 (n, tts)
+subroutine reduce_1 ( n, tts )
+  type :: tt
+    integer :: x
+    integer :: y
+ end type tt
+  type :: tt2
+    real(8) :: x
+    real(8) :: y
+  end type
+ 
+  integer :: n
+  type(tt) :: tts(n)
+  type(tt2) :: tts2(n)
+
+!CHECK: !$OMP DECLARE REDUCTION (+:tt: omp_out=tt(x=omp_out%x-omp_in%x,y=omp_out%y-omp_in%y)
+!CHECK: ) INITIALIZER(omp_priv=tt(x=0_4,y=0_4))
+!PARSE-TREE:  DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE: Verbatim
+!PARSE-TREE: OmpReductionSpecifier
+!PARSE-TREE: OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: OmpReductionCombiner -> AssignmentStmt = 'omp_out=tt(x=omp_out%x-omp_in%x,y=omp_out%y-omp_in%y)'
+!PARSE-TREE:    OmpInitializerClause -> AssignmentStmt = 'omp_priv=tt(x=0_4,y=0_4)'
+  
+  !$omp declare reduction(+ : tt :  omp_out = tt(omp_out%x - omp_in%x , omp_out%y - omp_in%y)) initializer(omp_priv = tt(0,0))
+
+  
+!CHECK: !$OMP DECLARE REDUCTION (+:tt2: omp_out=tt2(x=omp_out%x-omp_in%x,y=omp_out%y-omp_in%y)
+!CHECK: ) INITIALIZER(omp_priv=tt2(x=0._8,y=0._8)
+!PARSE-TREE:  DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareReductionConstruct
+!PARSE-TREE: Verbatim
+!PARSE-TREE: OmpReductionSpecifier
+!PARSE-TREE: OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
+!PARSE-TREE: OmpReductionCombiner -> AssignmentStmt = 'omp_out=tt2(x=omp_out%x-omp_in%x,y=omp_out%y-omp_in%y)'
+!PARSE-TREE:    OmpInitializerClause -> AssignmentStmt = 'omp_priv=tt2(x=0._8,y=0._8)'
+  
+  !$omp declare reduction(+ :tt2 :  omp_out = tt2(omp_out%x - omp_in%x , omp_out%y - omp_in%y)) initializer(omp_priv = tt2(0,0))
+  
+  type(tt) :: diffp = tt( 0, 0 )
+  type(tt2) :: diffp2 = tt2( 0, 0 )
+  integer :: i
+
+  !$omp parallel do reduction(+ : diffp)
+  do i = 1, n
+     diffp%x = diffp%x + tts(i)%x
+     diffp%y = diffp%y + tts(i)%y
+  end do
+
+  !$omp parallel do reduction(+ : diffp2)
+  do i = 1, n
+     diffp2%x = diffp2%x + tts2(i)%x
+     diffp2%y = diffp2%y + tts2(i)%y
+  end do
+
+end subroutine reduce_1
+!CHECK: END SUBROUTINE reduce_1
diff --git a/external/llvm-project/flang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext b/external/llvm-project/flang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
new file mode 100644
index 000000000000..2650fb5ebfd3
--- /dev/null
+++ b/external/llvm-project/flang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
@@ -0,0 +1,18 @@
+# IR level Instrumentation Flag
+:ir
+_QQmain
+# Func Hash:
+146835646621254984
+# Num Counters:
+2
+# Counter Values:
+100
+1
+
+main
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+1
\ No newline at end of file
diff --git a/external/llvm-project/flang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext b/external/llvm-project/flang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
new file mode 100644
index 000000000000..c4a2a26557e8
--- /dev/null
+++ b/external/llvm-project/flang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
@@ -0,0 +1,11 @@
+# IR level Instrumentation Flag
+:ir
+:entry_first
+_QQmain
+# Func Hash:
+146835646621254984
+# Num Counters:
+2
+# Counter Values:
+100
+1
\ No newline at end of file
diff --git a/external/llvm-project/flang/test/Profile/gcc-flag-compatibility.f90 b/external/llvm-project/flang/test/Profile/gcc-flag-compatibility.f90
new file mode 100644
index 000000000000..4490c45232d2
--- /dev/null
+++ b/external/llvm-project/flang/test/Profile/gcc-flag-compatibility.f90
@@ -0,0 +1,32 @@
+! Tests for -fprofile-generate and -fprofile-use flag compatibility. These two
+! flags behave similarly to their GCC counterparts:
+!
+! -fprofile-generate         Generates the profile file ./default.profraw
+! -fprofile-use=<dir>/file   Uses the profile file <dir>/file
+
+! On AIX, -flto used to be required with -fprofile-generate. gcc-flag-compatibility-aix.c is used to do the testing on AIX with -flto
+! RUN: %flang %s -c -S -o - -emit-llvm -fprofile-generate | FileCheck -check-prefix=PROFILE-GEN %s
+! PROFILE-GEN: @__profc_{{_?}}main = {{(private|internal)}} global [1 x i64] zeroinitializer, section
+! PROFILE-GEN: @__profd_{{_?}}main =
+
+! Check that -fprofile-use=some/path/file.prof reads some/path/file.prof
+! This uses LLVM IR format profile.
+! RUN: rm -rf %t.dir
+! RUN: mkdir -p %t.dir/some/path
+! RUN: llvm-profdata merge %S/Inputs/gcc-flag-compatibility_IR.proftext -o %t.dir/some/path/file.prof
+! RUN: %flang %s -o - -emit-llvm -S -fprofile-use=%t.dir/some/path/file.prof | FileCheck -check-prefix=PROFILE-USE-IR1 %s
+! RUN: llvm-profdata merge %S/Inputs/gcc-flag-compatibility_IR_entry.proftext -o %t.dir/some/path/file.prof
+! RUN: %flang %s -o - -emit-llvm -S -fprofile-use=%t.dir/some/path/file.prof | FileCheck -check-prefix=PROFILE-USE-IR2 %s
+! PROFILE-USE-IR1: = !{!"branch_weights", i32 100, i32 1}
+! PROFILE-USE-IR2: = !{!"branch_weights", i32 1, i32 100}
+
+program main
+  implicit none
+  integer :: i
+  integer :: X = 0
+
+  do i = 0, 99
+     X = X + i
+  end do
+
+end program main
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/allocate-align01.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/allocate-align01.f90
index ba0776cf46a6..4974f5e18397 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/allocate-align01.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/allocate-align01.f90
@@ -13,6 +13,7 @@ program allocate_align_tree
     z = 3
     !ERROR: The alignment value should be a constant positive integer
 !$omp allocate(j) align(xx)
+    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
     !ERROR: The alignment value should be a constant positive integer
 !$omp allocate(xarray) align(-32) allocator(omp_large_cap_mem_alloc)
     allocate(j(z), xarray(t))
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/allocate01.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/allocate01.f90
index 6ccb8bb09e83..8a680eee743e 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/allocate01.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/allocate01.f90
@@ -19,6 +19,7 @@ subroutine sema()
     !$omp allocate(y)
         print *, a
 
+    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
     !ERROR: List items must be declared in the same scoping unit in which the ALLOCATE directive appears
     !$omp allocate(x) allocator(omp_default_mem_alloc)
       allocate ( x(a), darray(a, b) )
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/allocate02.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/allocate02.f90
index 8f0579e810bb..80ef60b31e70 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/allocate02.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/allocate02.f90
@@ -16,9 +16,11 @@ subroutine allocate()
   !ERROR: At most one ALLOCATOR clause can appear on the ALLOCATE directive
   !$omp allocate(x, y) allocator(omp_default_mem_alloc) allocator(omp_default_mem_alloc)
 
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(darray) allocator(omp_default_mem_alloc)
       allocate ( darray(a, b) )
 
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !ERROR: At most one ALLOCATOR clause can appear on the ALLOCATE directive
   !$omp allocate(darray) allocator(omp_default_mem_alloc) allocator(omp_default_mem_alloc)
       allocate ( darray(a, b) )
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/allocate03.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/allocate03.f90
index e35115f3897c..b8c6b8e5dee7 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/allocate03.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/allocate03.f90
@@ -18,6 +18,7 @@ subroutine allocate()
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the ALLOCATE directive
   !$omp allocate(my_var%array)
 
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the ALLOCATE directive
   !$omp allocate(darray, my_var%array) allocator(omp_default_mem_alloc)
     allocate ( darray(a, b) )
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/allocate05.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/allocate05.f90
index a787e8bb32a4..2c81c4dbc82c 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/allocate05.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/allocate05.f90
@@ -13,11 +13,13 @@ subroutine allocate()
   real, dimension (:,:), allocatable :: darray
 
   !$omp target
+      !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
       !$omp allocate allocator(omp_default_mem_alloc)
           allocate ( darray(a, b) )
   !$omp end target
 
   !$omp target
+      !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
       !ERROR: ALLOCATE directives that appear in a TARGET region must specify an allocator clause
       !$omp allocate
           allocate ( darray(a, b) )
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/allocate06.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/allocate06.f90
index e14134cd0730..7196bcac2b9b 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/allocate06.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/allocate06.f90
@@ -14,6 +14,7 @@ subroutine allocate()
   !ERROR: List items specified in the ALLOCATE directive must not have the ALLOCATABLE attribute unless the directive is associated with an ALLOCATE statement
   !$omp allocate(darray) allocator(omp_default_mem_alloc)
 
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(darray) allocator(omp_default_mem_alloc)
     allocate(darray(a, b))
 
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/allocate09.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/allocate09.f90
index 0f93a340fe1e..645e97a3a33f 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/allocate09.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/allocate09.f90
@@ -12,23 +12,28 @@ subroutine allocate()
   integer, dimension(:), allocatable :: a, b, c, d, e, f, &
                                         g, h, i, j, k, l
 
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(a) allocator(omp_default_mem_alloc)
     allocate(a(1), b(2))
 
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(c, d) allocator(omp_default_mem_alloc)
     allocate(c(3), d(4))
 
   !$omp allocate(e) allocator(omp_default_mem_alloc)
   !$omp allocate(f, g) allocator(omp_default_mem_alloc)
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate
     allocate(e(5), f(6), g(7))
 
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !ERROR: Object 'i' in ALLOCATE directive not found in corresponding ALLOCATE statement
   !$omp allocate(h, i) allocator(omp_default_mem_alloc)
     allocate(h(8))
 
   !ERROR: Object 'j' in ALLOCATE directive not found in corresponding ALLOCATE statement
   !$omp allocate(j, k) allocator(omp_default_mem_alloc)
+  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(l) allocator(omp_default_mem_alloc)
     allocate(k(9), l(10))
 
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-compare.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-compare.f90
index 54492bf6a22a..11e23e062bce 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-compare.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-compare.f90
@@ -44,46 +44,37 @@
   !$omp end atomic
 
   ! Check for error conditions:
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the COMPARE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst seq_cst compare
   if (b .eq. c) b = a
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the COMPARE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic compare seq_cst seq_cst
   if (b .eq. c) b = a
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the COMPARE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst compare seq_cst
   if (b .eq. c) b = a
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the COMPARE directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic acquire acquire compare
   if (b .eq. c) b = a
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the COMPARE directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic compare acquire acquire
   if (b .eq. c) b = a
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the COMPARE directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic acquire compare acquire
   if (b .eq. c) b = a
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the COMPARE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed relaxed compare
   if (b .eq. c) b = a
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the COMPARE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic compare relaxed relaxed
   if (b .eq. c) b = a
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the COMPARE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed compare relaxed
   if (b .eq. c) b = a
 
-  !ERROR: More than one FAIL clause not allowed on OpenMP ATOMIC construct
+  !ERROR: At most one FAIL clause can appear on the ATOMIC directive
   !$omp atomic fail(release) compare fail(release)
   if (c .eq. a) a = b
   !$omp end atomic
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-hint-clause.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
index c13a11a8dd5d..8adb0f1a6740 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
@@ -16,20 +16,21 @@ program sample
     !$omp atomic read hint(2)
         y = x    
      
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
     !$omp atomic hint(3)
         y = y + 10
     
     !$omp atomic update hint(5)
         y = x + y
     
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
     !$omp atomic hint(7) capture
+    !WARNING: In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement
         y = x
         x = y
     !$omp end atomic
    
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp atomic update hint(x)
         y = y * 1
@@ -46,7 +47,7 @@ program sample
     !$omp atomic hint(omp_lock_hint_speculative)
         x = y + x
     
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp atomic hint(omp_sync_hint_uncontended + omp_sync_hint) read
         y = x 
@@ -69,36 +70,36 @@ program sample
     !$omp atomic hint(omp_lock_hint_contended + omp_sync_hint_nonspeculative)
         x = y + x
 
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
     !$omp atomic hint(omp_sync_hint_uncontended + omp_sync_hint_contended) read
         y = x 
 
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
     !$omp atomic hint(omp_sync_hint_nonspeculative + omp_lock_hint_speculative)
         y = y * 9
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must have INTEGER type, but is REAL(4)
     !$omp atomic hint(1.0) read
         y = x
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Operands of + must be numeric; have LOGICAL(4) and INTEGER(4)
     !$omp atomic hint(z + omp_sync_hint_nonspeculative) read
         y = x
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp atomic hint(k + omp_sync_hint_speculative) read
         y = x
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp atomic hint(p(1) + omp_sync_hint_uncontended) write
         x = 10 * y
 
     !$omp atomic write hint(a)
-    !ERROR: RHS expression on atomic assignment statement cannot access 'x'
+    !ERROR: Within atomic operation x and y+x access the same storage
         x = y + x
 
     !$omp atomic hint(abs(-1)) write
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-read.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-read.f90
new file mode 100644
index 000000000000..06c301cb78b7
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-read.f90
@@ -0,0 +1,118 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+
+subroutine f00
+  integer :: x, v
+  ! The end-directive is optional in ATOMIC READ. Expect no diagnostics.
+  !$omp atomic read
+  v = x
+
+  !$omp atomic read
+  v = x
+  !$omp end atomic
+end
+
+subroutine f01
+  integer, pointer :: x, v
+  ! Intrinsic assignment and pointer assignment are both ok. Expect no
+  ! diagnostics.
+  !$omp atomic read
+  v = x
+
+  !$omp atomic read
+  v => x
+end
+
+subroutine f02(i)
+  integer :: i, v
+  interface
+    function p(i)
+      integer, pointer :: p
+      integer :: i
+    end
+  end interface
+
+  ! Atomic variable can be a function reference. Expect no diagostics.
+  !$omp atomic read
+  v = p(i)
+end
+
+subroutine f03
+  integer :: x(3), y(5), v(3)
+
+  !$omp atomic read
+  !ERROR: Atomic variable x should be a scalar
+  v = x
+
+  !$omp atomic read
+  !ERROR: Atomic variable y(2_8:4_8:1_8) should be a scalar
+  v = y(2:4)
+end
+
+subroutine f04
+  integer :: x, y(3), v
+
+  !$omp atomic read
+  !ERROR: Within atomic operation x and x access the same storage
+  x = x
+
+  ! Accessing same array, but not the same storage. Expect no diagnostics.
+  !$omp atomic read
+  y(1) = y(2)
+end
+
+subroutine f05
+  integer :: x, v
+
+  !$omp atomic read
+  !ERROR: Atomic expression x+1_4 should be a variable
+  v = x + 1
+end
+
+subroutine f06
+  character :: x, v
+
+  !$omp atomic read
+  !ERROR: Atomic variable x cannot have CHARACTER type
+  v = x
+end
+
+subroutine f07
+  integer, allocatable :: x
+  integer :: v
+
+  allocate(x)
+
+  !$omp atomic read
+  !ERROR: Atomic variable x cannot be ALLOCATABLE
+  v = x
+end
+
+subroutine f08
+  type :: struct
+    integer :: m
+  end type
+  type(struct) :: x, v
+
+  !$omp atomic read
+  !ERROR: Atomic variable x should have an intrinsic type
+  v = x
+end
+
+subroutine f09(x, v)
+  class(*), pointer :: x, v
+
+  !$omp atomic read
+  !ERROR: Atomic variable x cannot be a pointer to a polymorphic type
+  v => x
+end
+
+subroutine f10(x, v)
+  type struct(length)
+    integer, len :: length
+  end type
+  type(struct(*)), pointer :: x, v
+
+  !$omp atomic read
+  !ERROR: Atomic variable x is a pointer to a type with non-constant length parameter
+  v => x
+end
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-capture.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-capture.f90
new file mode 100644
index 000000000000..f808ed916fb7
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-capture.f90
@@ -0,0 +1,77 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+
+subroutine f00
+  integer :: x, y, v
+
+  !ERROR: ATOMIC UPDATE operation with CAPTURE should contain two statements
+  !$omp atomic update capture
+  x = v
+  x = x + 1
+  y = x
+  !$omp end atomic
+end
+
+subroutine f01
+  integer :: x, y, v
+
+  !ERROR: ATOMIC UPDATE operation with CAPTURE should contain two assignments
+  !$omp atomic update capture
+  x = v
+  block
+    x = x + 1
+    y = x
+  end block
+  !$omp end atomic
+end
+
+subroutine f02
+  integer :: x, y
+
+  ! The update and capture statements can be inside of a single BLOCK.
+  ! The end-directive is then optional. Expect no diagnostics.
+  !$omp atomic update capture
+  block
+    x = x + 1
+    y = x
+  end block
+end
+
+subroutine f03
+  integer :: x
+
+  !ERROR: In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture
+  !$omp atomic update capture
+  x = x + 1
+  x = x + 2
+  !$omp end atomic
+end
+
+subroutine f04
+  integer :: x, v
+
+  !$omp atomic update capture
+  !WARNING: In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement
+  v = x
+  x = v
+  !$omp end atomic
+end
+
+subroutine f05
+  integer :: x, v, z
+
+  !$omp atomic update capture
+  !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read z
+  v = x
+  z = x + 1
+  !$omp end atomic
+end
+
+subroutine f06
+  integer :: x, v, z
+
+  !$omp atomic update capture
+  z = x + 1
+  !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read z
+  v = x
+  !$omp end atomic
+end
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-only.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-only.f90
new file mode 100644
index 000000000000..28d0e264359c
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-only.f90
@@ -0,0 +1,83 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+
+subroutine f00
+  integer :: x, y
+
+  ! The x is a direct argument of the + operator. Expect no diagnostics.
+  !$omp atomic update
+  x = x + (y - 1)
+end
+
+subroutine f01
+  integer :: x
+
+  ! x + 0 is unusual, but legal. Expect no diagnostics.
+  !$omp atomic update
+  x = x + 0
+end
+
+subroutine f02
+  integer :: x
+
+  ! This is formally not allowed by the syntax restrictions of the spec,
+  ! but it's equivalent to either x+0 or x*1, both of which are legal.
+  ! Allow this case. Expect no diagnostics.
+  !$omp atomic update
+  x = x
+end
+
+subroutine f03
+  integer :: x, y
+
+  !$omp atomic update
+  !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
+  x = (x + y) + 1
+end
+
+subroutine f04
+  integer :: x
+  real :: y
+
+  !$omp atomic update
+  !ERROR: This intrinsic function is not a valid ATOMIC UPDATE operation
+  x = floor(x + y)
+end
+
+subroutine f05
+  integer :: x
+  real :: y
+
+  ! An explicit conversion is accepted as an extension.
+  !$omp atomic update
+  x = int(x + y)
+end
+
+subroutine f06
+  integer :: x, y
+  interface
+    function f(i, j)
+      integer :: f, i, j
+    end
+  end interface
+
+  !$omp atomic update
+  !ERROR: A call to this function is not a valid ATOMIC UPDATE operation
+  x = f(x, y)
+end
+
+subroutine f07
+  real :: x
+  integer :: y
+
+  !$omp atomic update
+  !ERROR: The ** operator is not a valid ATOMIC UPDATE operation
+  x = x ** y
+end
+
+subroutine f08
+  integer :: x, y
+
+  !$omp atomic update
+  !ERROR: The atomic variable x should appear as an argument in the update operation
+  x = y
+end
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90
index 21a9b87d2634..3084376b4275 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90
@@ -22,10 +22,10 @@ program sample
         x = x / y
      
     !$omp atomic update
-    !ERROR: Invalid or missing operator in atomic update statement
+    !ERROR: A call to this function is not a valid ATOMIC UPDATE operation
         x = x .MYOPERATOR. y
 
     !$omp atomic
-    !ERROR: Invalid or missing operator in atomic update statement
+    !ERROR: A call to this function is not a valid ATOMIC UPDATE operation
         x = x .MYOPERATOR. y
 end program
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic-write.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-write.f90
new file mode 100644
index 000000000000..7965ad2dc7db
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic-write.f90
@@ -0,0 +1,81 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+
+subroutine f00
+  integer :: x, v
+  ! The end-directive is optional in ATOMIC WRITE. Expect no diagnostics.
+  !$omp atomic write
+  x = v + 1
+
+  !$omp atomic write
+  x = v + 3
+  !$omp end atomic
+end
+
+subroutine f01
+  integer, pointer :: x, v
+  ! Intrinsic assignment and pointer assignment are both ok. Expect no
+  ! diagnostics.
+  !$omp atomic write
+  x = 2 * v + 3
+
+  !$omp atomic write
+  x => v
+end
+
+subroutine f02(i)
+  integer :: i, v
+  interface
+    function p(i)
+      integer, pointer :: p
+      integer :: i
+    end
+  end interface
+
+  ! Atomic variable can be a function reference. Expect no diagostics.
+  !$omp atomic write
+  p(i) = v
+end
+
+subroutine f03
+  integer :: x(3), y(5), v(3)
+
+  !$omp atomic write
+  !ERROR: Atomic variable x should be a scalar
+  x = v
+
+  !$omp atomic write
+  !ERROR: Atomic variable y(2_8:4_8:1_8) should be a scalar
+  y(2:4) = v
+end
+
+subroutine f04
+  integer :: x, y(3), v
+
+  !$omp atomic write
+  !ERROR: Within atomic operation x and x+1_4 access the same storage
+  x = x + 1
+
+  ! Accessing same array, but not the same storage. Expect no diagnostics.
+  !$omp atomic write
+  y(1) = y(2)
+end
+
+subroutine f06
+  character :: x, v
+
+  !$omp atomic write
+  !ERROR: Atomic variable x cannot have CHARACTER type
+  x = v
+end
+
+subroutine f07
+  integer, allocatable :: x
+  integer :: v
+
+  allocate(x)
+
+  !$omp atomic write
+  !ERROR: Atomic variable x cannot be ALLOCATABLE
+  x = v
+end
+
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic.f90
index 0e100871ea9b..e4be810acc62 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic.f90
@@ -1,4 +1,6 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! REQUIRES: openmp_runtime
+
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp %openmp_flags -fopenmp-version=31
 use omp_lib
 ! Check OpenMP 2.13.6 atomic Construct
 
@@ -11,9 +13,13 @@
   a = b
   !$omp end atomic
 
+  !ERROR: ACQUIRE clause is not allowed on directive ATOMIC in OpenMP v3.1, try -fopenmp-version=50
+  !ERROR: HINT clause is not allowed on directive ATOMIC in OpenMP v3.1, try -fopenmp-version=50
   !$omp atomic read acquire hint(OMP_LOCK_HINT_CONTENDED)
   a = b
 
+  !ERROR: RELEASE clause is not allowed on directive ATOMIC in OpenMP v3.1, try -fopenmp-version=50
+  !ERROR: HINT clause is not allowed on directive ATOMIC in OpenMP v3.1, try -fopenmp-version=50
   !$omp atomic release hint(OMP_LOCK_HINT_UNCONTENDED) write
   a = b
 
@@ -22,39 +28,32 @@
   a = a + 1
   !$omp end atomic
 
+  !ERROR: HINT clause is not allowed on directive ATOMIC in OpenMP v3.1, try -fopenmp-version=50
+  !ERROR: ACQ_REL clause is not allowed on directive ATOMIC in OpenMP v3.1, try -fopenmp-version=50
   !$omp atomic hint(1) acq_rel capture
   b = a
   a = a + 1
   !$omp end atomic
 
-  !ERROR: expected end of line
+  !ERROR: At most one clause from the 'atomic' group is allowed on ATOMIC construct
   !$omp atomic read write
+  !ERROR: Atomic expression a+1._4 should be a variable
   a = a + 1
 
   !$omp atomic
   a = a + 1
-  !ERROR: expected 'UPDATE'
-  !ERROR: expected 'WRITE'
-  !ERROR: expected 'COMPARE'
-  !ERROR: expected 'CAPTURE'
-  !ERROR: expected 'READ'
+  !ERROR: NUM_THREADS clause is not allowed on the ATOMIC directive
   !$omp atomic num_threads(4)
   a = a + 1
 
-  !ERROR: expected end of line
+  !ERROR: ATOMIC UPDATE operation with CAPTURE should contain two statements
+  !ERROR: NUM_THREADS clause is not allowed on the ATOMIC directive
   !$omp atomic capture num_threads(4)
   a = a + 1
 
+  !ERROR: RELAXED clause is not allowed on directive ATOMIC in OpenMP v3.1, try -fopenmp-version=50
   !$omp atomic relaxed
   a = a + 1
 
-  !ERROR: expected 'UPDATE'
-  !ERROR: expected 'WRITE'
-  !ERROR: expected 'COMPARE'
-  !ERROR: expected 'CAPTURE'
-  !ERROR: expected 'READ'
-  !$omp atomic num_threads write
-  a = a + 1
-
   !$omp end parallel
 end
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic01.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic01.f90
index 173effe86b69..f700c381cadd 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic01.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic01.f90
@@ -14,322 +14,277 @@
 ! At most one memory-order-clause may appear on the construct.
 
 !READ
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the READ directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst seq_cst read
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the READ directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic read seq_cst seq_cst
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the READ directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst read seq_cst
     i = j
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the READ directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic acquire acquire read
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the READ directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic read acquire acquire
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the READ directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic acquire read acquire
     i = j
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the READ directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed relaxed read
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the READ directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic read relaxed relaxed
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the READ directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed read relaxed
     i = j
 
 !UPDATE
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the UPDATE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst seq_cst update
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the UPDATE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic update seq_cst seq_cst
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the UPDATE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst update seq_cst
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the UPDATE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic release release update
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the UPDATE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic update release release
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the UPDATE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic release update release
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the UPDATE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed relaxed update
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the UPDATE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic update relaxed relaxed
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the UPDATE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed update relaxed
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
 !CAPTURE
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the CAPTURE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst seq_cst capture
     i = j
     j = k
   !$omp end atomic
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the CAPTURE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic capture seq_cst seq_cst
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the CAPTURE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst capture seq_cst
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the CAPTURE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic release release capture
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the CAPTURE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic capture release release
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the CAPTURE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic release capture release
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the CAPTURE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed relaxed capture
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the CAPTURE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic capture relaxed relaxed
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the CAPTURE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed capture relaxed
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQ_REL clause can appear on the CAPTURE directive
+  !ERROR: At most one ACQ_REL clause can appear on the ATOMIC directive
   !$omp atomic acq_rel acq_rel capture
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQ_REL clause can appear on the CAPTURE directive
+  !ERROR: At most one ACQ_REL clause can appear on the ATOMIC directive
   !$omp atomic capture acq_rel acq_rel
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQ_REL clause can appear on the CAPTURE directive
+  !ERROR: At most one ACQ_REL clause can appear on the ATOMIC directive
   !$omp atomic acq_rel capture acq_rel
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the CAPTURE directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic acquire acquire capture
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the CAPTURE directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic capture acquire acquire
     i = j
     j = k
   !$omp end atomic
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one ACQUIRE clause can appear on the CAPTURE directive
+  !ERROR: At most one ACQUIRE clause can appear on the ATOMIC directive
   !$omp atomic acquire capture acquire
     i = j
     j = k
   !$omp end atomic
 
 !WRITE
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the WRITE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst seq_cst write
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the WRITE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic write seq_cst seq_cst
     i = j
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one SEQ_CST clause can appear on the WRITE directive
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst write seq_cst
     i = j
 
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the WRITE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic release release write
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the WRITE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic write release release
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELEASE clause can appear on the WRITE directive
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic release write release
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the WRITE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed relaxed write
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the WRITE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic write relaxed relaxed
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
-  !ERROR: At most one RELAXED clause can appear on the WRITE directive
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed write relaxed
     i = j
 
 !No atomic-clause
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
   !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
   !$omp atomic relaxed relaxed
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
   !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
   !$omp atomic seq_cst seq_cst
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
   !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
   !$omp atomic release release
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
 ! 2.17.7.3
 ! At most one hint clause may appear on the construct.
 
-  !ERROR: At most one HINT clause can appear on the READ directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_speculative) hint(omp_sync_hint_speculative) read
     i = j
-  !ERROR: At most one HINT clause can appear on the READ directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_nonspeculative) read hint(omp_sync_hint_nonspeculative)
     i = j
-  !ERROR: At most one HINT clause can appear on the READ directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic read hint(omp_sync_hint_uncontended) hint (omp_sync_hint_uncontended)
     i = j
-  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) write
     i = j
-  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_nonspeculative) write hint(omp_sync_hint_nonspeculative)
     i = j
-  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic write hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
     i = j
-  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) write
     i = j
-  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_nonspeculative) write hint(omp_sync_hint_nonspeculative)
     i = j
-  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic write hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
     i = j
-  !ERROR: At most one HINT clause can appear on the UPDATE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) update
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: At most one HINT clause can appear on the UPDATE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_nonspeculative) update hint(omp_sync_hint_nonspeculative)
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: At most one HINT clause can appear on the UPDATE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic update hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
   !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative)
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
   !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_none) hint(omp_sync_hint_nonspeculative)
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
   !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
-  !ERROR: At most one HINT clause can appear on the CAPTURE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) capture
     i = j
     j = k
   !$omp end atomic
-  !ERROR: At most one HINT clause can appear on the CAPTURE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic hint(omp_sync_hint_nonspeculative) capture hint(omp_sync_hint_nonspeculative)
     i = j
     j = k
   !$omp end atomic
-  !ERROR: At most one HINT clause can appear on the CAPTURE directive
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
   !$omp atomic capture hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
     i = j
     j = k
@@ -337,34 +292,26 @@
 ! 2.17.7.4
 ! If atomic-clause is read then memory-order-clause must not be acq_rel or release.
 
-  !ERROR: Clause ACQ_REL is not allowed if clause READ appears on the ATOMIC directive
   !$omp atomic acq_rel read
     i = j
-  !ERROR: Clause ACQ_REL is not allowed if clause READ appears on the ATOMIC directive
   !$omp atomic read acq_rel
     i = j
 
-  !ERROR: Clause RELEASE is not allowed if clause READ appears on the ATOMIC directive
   !$omp atomic release read
     i = j
-  !ERROR: Clause RELEASE is not allowed if clause READ appears on the ATOMIC directive
   !$omp atomic read release
     i = j
 
 ! 2.17.7.5
 ! If atomic-clause is write then memory-order-clause must not be acq_rel or acquire.
 
-  !ERROR: Clause ACQ_REL is not allowed if clause WRITE appears on the ATOMIC directive
   !$omp atomic acq_rel write
     i = j
-  !ERROR: Clause ACQ_REL is not allowed if clause WRITE appears on the ATOMIC directive
   !$omp atomic write acq_rel
     i = j
 
-  !ERROR: Clause ACQUIRE is not allowed if clause WRITE appears on the ATOMIC directive
   !$omp atomic acquire write
     i = j
-  !ERROR: Clause ACQUIRE is not allowed if clause WRITE appears on the ATOMIC directive
   !$omp atomic write acquire
     i = j
 
@@ -372,33 +319,27 @@
 ! 2.17.7.6
 ! If atomic-clause is update or not present then memory-order-clause must not be acq_rel or acquire.
 
-  !ERROR: Clause ACQ_REL is not allowed if clause UPDATE appears on the ATOMIC directive
   !$omp atomic acq_rel update
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
-  !ERROR: Clause ACQ_REL is not allowed if clause UPDATE appears on the ATOMIC directive
   !$omp atomic update acq_rel
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
-  !ERROR: Clause ACQUIRE is not allowed if clause UPDATE appears on the ATOMIC directive
   !$omp atomic acquire update
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
-  !ERROR: Clause ACQUIRE is not allowed if clause UPDATE appears on the ATOMIC directive
   !$omp atomic update acquire
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
-  !ERROR: Clause ACQ_REL is not allowed on the ATOMIC directive
   !$omp atomic acq_rel
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 
-  !ERROR: Clause ACQUIRE is not allowed on the ATOMIC directive
   !$omp atomic acquire
-  !ERROR: Invalid or missing operator in atomic update statement
+  !ERROR: The atomic variable i should appear as an argument in the update operation
     i = j
 end program
 
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic02.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic02.f90
index c66085d00f15..45e41f255296 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic02.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic02.f90
@@ -28,36 +28,29 @@ program OmpAtomic
    !$omp atomic
    a = a/(b + 1)
    !$omp atomic
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The ** operator is not a valid ATOMIC UPDATE operation
    a = a**4
    !$omp atomic 
-   !ERROR: Expected scalar variable on the LHS of atomic update assignment statement
-   !ERROR: Invalid or missing operator in atomic update statement
-   !ERROR: Expected scalar expression on the RHS of atomic update assignment statement
+   !ERROR: Atomic variable c cannot have CHARACTER type
+   !ERROR: The atomic variable c should appear as an argument in the update operation
    c = d 
    !$omp atomic
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The < operator is not a valid ATOMIC UPDATE operation
    l = a .LT. b
    !$omp atomic
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The <= operator is not a valid ATOMIC UPDATE operation
    l = a .LE. b
    !$omp atomic
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The == operator is not a valid ATOMIC UPDATE operation
    l = a .EQ. b
    !$omp atomic
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The /= operator is not a valid ATOMIC UPDATE operation
    l = a .NE. b
    !$omp atomic
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The >= operator is not a valid ATOMIC UPDATE operation
    l = a .GE. b
    !$omp atomic
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The > operator is not a valid ATOMIC UPDATE operation
    l = a .GT. b
    !$omp atomic
    m = m .AND. n
@@ -76,32 +69,26 @@ program OmpAtomic
    !$omp atomic update
    a = a/(b + 1)
    !$omp atomic update
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The ** operator is not a valid ATOMIC UPDATE operation
    a = a**4
    !$omp atomic update
-   !ERROR: Expected scalar variable on the LHS of atomic update assignment statement
-   !ERROR: Invalid or missing operator in atomic update statement
-   !ERROR: Expected scalar expression on the RHS of atomic update assignment statement
+   !ERROR: Atomic variable c cannot have CHARACTER type
+   !ERROR: This is not a valid ATOMIC UPDATE operation
    c = c//d
    !$omp atomic update
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The < operator is not a valid ATOMIC UPDATE operation
    l = a .LT. b
    !$omp atomic update
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The <= operator is not a valid ATOMIC UPDATE operation
    l = a .LE. b
    !$omp atomic update
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The == operator is not a valid ATOMIC UPDATE operation
    l = a .EQ. b
    !$omp atomic update
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The >= operator is not a valid ATOMIC UPDATE operation
    l = a .GE. b
    !$omp atomic update
-   !ERROR: Atomic update statement should be of form `l = l operator expr` OR `l = expr operator l`
-   !ERROR: Invalid or missing operator in atomic update statement
+   !ERROR: The > operator is not a valid ATOMIC UPDATE operation
    l = a .GT. b
    !$omp atomic update
    m = m .AND. n
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic03.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic03.f90
index 76367495b986..b3a3c0d5e7a1 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic03.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic03.f90
@@ -25,28 +25,26 @@ program OmpAtomic
    y = MIN(y, 8)
 
 !$omp atomic
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level AND operator
    z = IAND(y, 4)
 !$omp atomic
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level OR operator
    z = IOR(y, 5)
 !$omp atomic
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level NEQV/EOR operator
    z = IEOR(y, 6)
 !$omp atomic
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MAX operator
    z = MAX(y, 7, b, c)
 !$omp atomic
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MIN operator
    z = MIN(y, 8, a, d)
 
 !$omp atomic
-   !ERROR: Invalid intrinsic procedure name in OpenMP ATOMIC (UPDATE) statement
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'y'
+   !ERROR: This intrinsic function is not a valid ATOMIC UPDATE operation
    y = FRACTION(x)
 !$omp atomic
-   !ERROR: Invalid intrinsic procedure name in OpenMP ATOMIC (UPDATE) statement
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'y'
+   !ERROR: The atomic variable y should appear as an argument in the update operation
    y = REAL(x)
 !$omp atomic update
    y = IAND(y, 4)
@@ -60,26 +58,26 @@ program OmpAtomic
    y = MIN(y, 8)
 
 !$omp atomic update
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level AND operator
    z = IAND(y, 4)
 !$omp atomic update 
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level OR operator
    z = IOR(y, 5)
 !$omp atomic update
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level NEQV/EOR operator
    z = IEOR(y, 6)
 !$omp atomic update
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MAX operator
    z = MAX(y, 7)
 !$omp atomic update
-   !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MIN operator
    z = MIN(y, 8)
 
 !$omp atomic update
-   !ERROR: Invalid intrinsic procedure name in OpenMP ATOMIC (UPDATE) statement
+  !ERROR: This intrinsic function is not a valid ATOMIC UPDATE operation
    y = MOD(y, 9)
 !$omp atomic update
-   !ERROR: Invalid intrinsic procedure name in OpenMP ATOMIC (UPDATE) statement
+  !ERROR: This intrinsic function is not a valid ATOMIC UPDATE operation
    x = ABS(x)
 end program OmpAtomic
 
@@ -92,7 +90,7 @@ subroutine conflicting_types()
     type(simple) ::s
     z = 1
     !$omp atomic
-    !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'z'
+    !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level AND operator
     z = IAND(s%z, 4)
 end subroutine
 
@@ -105,40 +103,37 @@ subroutine more_invalid_atomic_update_stmts()
     type(some_type) :: s
  
     !$omp atomic update
-    !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'a'
+    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level MIN operator
         a = min(a, a, b)
      
     !$omp atomic
-    !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'a'
+    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level MAX operator
         a = max(b, a, b, a)
 
     !$omp atomic
-    !ERROR: Atomic update statement should be of the form `a = intrinsic_procedure(a, expr_list)` OR `a = intrinsic_procedure(expr_list, a)`
         a = min(b, a, b)
 
     !$omp atomic
-    !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'a'
+    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level MAX operator
         a = max(b, a, b, a, b)
     
     !$omp atomic update
-    !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'y'
+    !ERROR: The atomic variable y should occur exactly once among the arguments of the top-level MIN operator
         y = min(z, x)
      
     !$omp atomic
         z = max(z, y)
 
     !$omp atomic update
-    !ERROR: Expected scalar variable on the LHS of atomic update assignment statement
-    !ERROR: Intrinsic procedure arguments in atomic update statement must have exactly one occurence of 'k'
+    !ERROR: Atomic variable k should be a scalar
+    !ERROR: The atomic variable k should occur exactly once among the arguments of the top-level MAX operator
         k = max(x, y)
-    
+
     !$omp atomic
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar REAL(4) and rank 1 array of REAL(4)
-    !ERROR: Expected scalar expression on the RHS of atomic update assignment statement
         x = min(x, k)
 
     !$omp atomic
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar REAL(4) and rank 1 array of REAL(4)
-    !ERROR: Expected scalar expression on the RHS of atomic update assignment statement
-        z =z + s%m
+        z = z + s%m
 end subroutine
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic04.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic04.f90
index a9644ad95aa3..0f69befed141 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic04.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic04.f90
@@ -1,5 +1,3 @@
-! REQUIRES: openmp_runtime
-
 ! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 
 ! OpenMP Atomic construct
@@ -7,7 +5,6 @@
 ! Update assignment must be 'var = var op expr' or 'var = expr op var'
 
 program OmpAtomic
-   use omp_lib
    real x
    integer y
    logical m, n, l
@@ -20,12 +17,10 @@ program OmpAtomic
 !$omp atomic
    x = 1 + x
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
    x = y + 1
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
    x = 1 + y
 
 !$omp atomic
@@ -33,12 +28,10 @@ program OmpAtomic
 !$omp atomic
    x = 1 - x
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
    x = y - 1
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
    x = 1 - y
 
 !$omp atomic
@@ -46,12 +39,10 @@ program OmpAtomic
 !$omp atomic
    x = 1*x
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should appear as an argument in the update operation
    x = y*1
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should appear as an argument in the update operation
    x = 1*y
 
 !$omp atomic
@@ -59,12 +50,10 @@ program OmpAtomic
 !$omp atomic
    x = 1/x
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
    x = y/1
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
    x = 1/y
 
 !$omp atomic
@@ -72,8 +61,7 @@ program OmpAtomic
 !$omp atomic
    m = n .AND. m
 !$omp atomic 
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level AND operator
    m = n .AND. l
 
 !$omp atomic
@@ -81,8 +69,7 @@ program OmpAtomic
 !$omp atomic
    m = n .OR. m
 !$omp atomic 
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level OR operator
    m = n .OR. l
 
 !$omp atomic
@@ -90,8 +77,7 @@ program OmpAtomic
 !$omp atomic
    m = n .EQV. m
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level EQV operator
    m = n .EQV. l
 
 !$omp atomic
@@ -99,8 +85,7 @@ program OmpAtomic
 !$omp atomic
    m = n .NEQV. m
 !$omp atomic
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level NEQV/EOR operator
    m = n .NEQV. l
 
 !$omp atomic update
@@ -108,12 +93,10 @@ program OmpAtomic
 !$omp atomic update
    x = 1 + x
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
    x = y + 1
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
    x = 1 + y
 
 !$omp atomic update
@@ -121,12 +104,10 @@ program OmpAtomic
 !$omp atomic update
    x = 1 - x
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
    x = y - 1
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
    x = 1 - y
 
 !$omp atomic update
@@ -134,12 +115,10 @@ program OmpAtomic
 !$omp atomic update
    x = 1*x
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should appear as an argument in the update operation
    x = y*1
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should appear as an argument in the update operation
    x = 1*y
 
 !$omp atomic update
@@ -147,12 +126,10 @@ program OmpAtomic
 !$omp atomic update
    x = 1/x
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
    x = y/1
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-   !ERROR: Exactly one occurence of 'x' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
    x = 1/y
 
 !$omp atomic update
@@ -160,8 +137,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .AND. m
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level AND operator
    m = n .AND. l
 
 !$omp atomic update
@@ -169,8 +145,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .OR. m
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level OR operator
    m = n .OR. l
 
 !$omp atomic update
@@ -178,8 +153,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .EQV. m
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level EQV operator
    m = n .EQV. l
 
 !$omp atomic update
@@ -187,8 +161,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .NEQV. m
 !$omp atomic update
-   !ERROR: Atomic update statement should be of form `m = m operator expr` OR `m = expr operator m`
-   !ERROR: Exactly one occurence of 'm' expected on the RHS of atomic update assignment statement
+   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level NEQV/EOR operator
    m = n .NEQV. l
 
 end program OmpAtomic
@@ -204,35 +177,34 @@ subroutine more_invalid_atomic_update_stmts()
     type(some_type) p
     
     !$omp atomic
-    !ERROR: Invalid or missing operator in atomic update statement
         x = x
 
     !$omp atomic update
-    !ERROR: Invalid or missing operator in atomic update statement
+    !ERROR: The atomic variable x should appear as an argument in the update operation
         x = 1    
 
     !$omp atomic update
-    !ERROR: Exactly one occurence of 'a' expected on the RHS of atomic update assignment statement
+    !ERROR: Within atomic operation a and a*b access the same storage
         a = a * b + a
 
     !$omp atomic
-    !ERROR: Atomic update statement should be of form `a = a operator expr` OR `a = expr operator a`
+    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level * operator
         a = b * (a + 9)
 
     !$omp atomic update
-    !ERROR: Exactly one occurence of 'a' expected on the RHS of atomic update assignment statement
+    !ERROR: Within atomic operation a and (a+b) access the same storage
         a = a * (a + b)
 
     !$omp atomic
-    !ERROR: Exactly one occurence of 'a' expected on the RHS of atomic update assignment statement
+    !ERROR: Within atomic operation a and (b+a) access the same storage
         a = (b + a) * a
 
     !$omp atomic
-    !ERROR: Atomic update statement should be of form `a = a operator expr` OR `a = expr operator a`
+    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level + operator
         a = a * b + c
 
     !$omp atomic update
-    !ERROR: Atomic update statement should be of form `a = a operator expr` OR `a = expr operator a`
+    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level + operator
         a = a + b + c
 
     !$omp atomic
@@ -243,23 +215,18 @@ subroutine more_invalid_atomic_update_stmts()
 
     !$omp atomic
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar INTEGER(4) and rank 1 array of INTEGER(4)
-    !ERROR: Expected scalar expression on the RHS of atomic update assignment statement
         a = a + d
 
     !$omp atomic update
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar REAL(4) and rank 1 array of REAL(4)
-    !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
-    !ERROR: Expected scalar expression on the RHS of atomic update assignment statement
+    !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
         x = x * y / z
 
     !$omp atomic
-    !ERROR: Atomic update statement should be of form `p%m = p%m operator expr` OR `p%m = expr operator p%m`
-    !ERROR: Exactly one occurence of 'p%m' expected on the RHS of atomic update assignment statement
+    !ERROR: The atomic variable p%m should occur exactly once among the arguments of the top-level + operator
         p%m = x + y
 
     !$omp atomic update
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar REAL(4) and rank 1 array of REAL(4)
-    !ERROR: Expected scalar expression on the RHS of atomic update assignment statement
-    !ERROR: Exactly one occurence of 'p%m' expected on the RHS of atomic update assignment statement
         p%m = p%m + p%n
 end subroutine
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/atomic05.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/atomic05.f90
index 266268a21244..77ffc6e57f1a 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/atomic05.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/atomic05.f90
@@ -8,20 +8,20 @@ program OmpAtomic
     use omp_lib
     integer :: g, x
 
-    !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
+    !ERROR: At most one clause from the 'memory-order' group is allowed on ATOMIC construct
     !$omp atomic relaxed, seq_cst
         x = x + 1
-    !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
+    !ERROR: At most one clause from the 'memory-order' group is allowed on ATOMIC construct
     !$omp atomic read seq_cst, relaxed
         x = g
-    !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
+    !ERROR: At most one clause from the 'memory-order' group is allowed on ATOMIC construct
     !$omp atomic write relaxed, release
         x = 2 * 4
-    !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
+    !ERROR: At most one clause from the 'memory-order' group is allowed on ATOMIC construct
     !$omp atomic update release, seq_cst
-    !ERROR: Invalid or missing operator in atomic update statement
+    !ERROR: The atomic variable x should appear as an argument in the update operation
         x = 10
-    !ERROR: More than one memory order clause not allowed on OpenMP ATOMIC construct
+    !ERROR: At most one clause from the 'memory-order' group is allowed on ATOMIC construct
     !$omp atomic capture release, seq_cst
         x = g
         g = x * 10
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/common-block.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/common-block.f90
index e1ddd120da85..adf77b016ecd 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/common-block.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/common-block.f90
@@ -1,18 +1,18 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
 
 program main
-  !CHECK: a size=4 offset=0: ObjectEntity type: REAL(4)
-  !CHECK: b size=8 offset=4: ObjectEntity type: INTEGER(4) shape: 1_8:2_8
-  !CHECK: c size=4 offset=12: ObjectEntity type: REAL(4)
+  !CHECK: a (InCommonBlock) size=4 offset=0: ObjectEntity type: REAL(4)
+  !CHECK: b (InCommonBlock) size=8 offset=4: ObjectEntity type: INTEGER(4) shape: 1_8:2_8
+  !CHECK: c (InCommonBlock) size=4 offset=12: ObjectEntity type: REAL(4)
   !CHECK: blk size=16 offset=0: CommonBlockDetails alignment=4: a b c
   real :: a, c
   integer :: b(2)
   common /blk/ a, b, c
   !$omp parallel private(/blk/)
     !CHECK: OtherConstruct scope: size=0 alignment=1
-    !CHECK:   a (OmpPrivate): HostAssoc
-    !CHECK:   b (OmpPrivate): HostAssoc
-    !CHECK:   c (OmpPrivate): HostAssoc
+    !CHECK:   a (OmpPrivate, OmpExplicit): HostAssoc
+    !CHECK:   b (OmpPrivate, OmpExplicit): HostAssoc
+    !CHECK:   c (OmpPrivate, OmpExplicit): HostAssoc
     call sub(a, b, c)
   !$omp end parallel
 end program
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/copyprivate03.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/copyprivate03.f90
index 9d39fdb6b13c..249b27e4b2b8 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/copyprivate03.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/copyprivate03.f90
@@ -6,6 +6,8 @@
 
 program omp_copyprivate
   integer :: a(10), b(10)
+  real, dimension(:), allocatable :: c
+  real, dimension(:), pointer :: d
   integer, save :: k
 
   !$omp threadprivate(k)
@@ -43,4 +45,16 @@ program omp_copyprivate
 
   print *, a, b
 
+  !$omp task
+    !$omp parallel private(c, d)
+      allocate(c(5))
+      allocate(d(10))
+      !$omp single
+        c = 22
+        d = 33
+      !Check that 'c' and 'd' inherit PRIVATE DSA from the enclosing PARALLEL
+      !and no error occurs.
+      !$omp end single copyprivate(c, d)
+    !$omp end parallel
+  !$omp end task
 end program omp_copyprivate
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/critical-hint-clause.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/critical-hint-clause.f90
index 7ca8c858239f..e9cfa49bf934 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/critical-hint-clause.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/critical-hint-clause.f90
@@ -18,7 +18,7 @@ program sample
         y = 2
     !$omp end critical (name)
      
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
     !$omp critical (name) hint(3)
         y = 2
     !$omp end critical (name)
@@ -27,12 +27,12 @@ program sample
         y = 2
     !$omp end critical (name)
     
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
     !$omp critical (name) hint(7)
         y = 2
     !$omp end critical (name)
    
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp critical (name) hint(x)
         y = 2
@@ -54,7 +54,7 @@ program sample
         y = 2
     !$omp end critical (name)
     
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp critical (name) hint(omp_sync_hint_uncontended + omp_sync_hint) 
         y = 2
@@ -84,35 +84,35 @@ program sample
         y = 2
     !$omp end critical (name)
 
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
      !$omp critical (name) hint(omp_sync_hint_uncontended + omp_sync_hint_contended)
         y = 2
     !$omp end critical (name)
 
-    !ERROR: Hint clause value is not a valid OpenMP synchronization value
+    !ERROR: The synchronization hint is not valid
     !$omp critical (name) hint(omp_sync_hint_nonspeculative + omp_lock_hint_speculative)
         y = 2
     !$omp end critical (name)
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must have INTEGER type, but is REAL(4)
     !$omp critical (name) hint(1.0) 
         y = 2
     !$omp end critical (name)
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Operands of + must be numeric; have LOGICAL(4) and INTEGER(4)
     !$omp critical (name) hint(z + omp_sync_hint_nonspeculative)
         y = 2
     !$omp end critical (name)
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp critical (name) hint(k + omp_sync_hint_speculative)
         y = 2
     !$omp end critical (name)
 
-    !ERROR: Hint clause must have non-negative constant integer expression
+    !ERROR: Synchronization hint must be a constant integer value
     !ERROR: Must be a constant value
     !$omp critical (name) hint(p(1) + omp_sync_hint_uncontended)
         y = 2
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-bad-operator.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-bad-operator.f90
new file mode 100644
index 000000000000..1d1d2903a278
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-bad-operator.f90
@@ -0,0 +1,6 @@
+! RUN: not %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s 2>&1 | FileCheck %s
+
+function func(n)
+    !$omp declare reduction(/:integer:omp_out=omp_out+omp_in)
+!CHECK: error: Unsupported operator in DECLARE REDUCTION
+end function func
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-bad-operator2.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-bad-operator2.f90
new file mode 100644
index 000000000000..9ee223c1c71f
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-bad-operator2.f90
@@ -0,0 +1,28 @@
+! RUN: not %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s 2>&1 | FileCheck %s
+
+module m1
+  interface operator(.fluffy.)
+    procedure my_mul
+  end interface
+  type t1
+    integer :: val = 1
+  end type
+!$omp declare reduction(.fluffy.:t1:omp_out=omp_out.fluffy.omp_in)
+contains
+  function my_mul(x, y)
+    type (t1), intent (in) :: x, y
+    type (t1) :: my_mul
+    my_mul%val = x%val * y%val
+  end function my_mul
+
+  subroutine subr(a, r)
+    implicit none
+    integer, intent(in), dimension(10) :: a
+    integer, intent(out) :: r
+    integer :: i
+    !$omp do parallel reduction(.fluffy.:r)
+!CHECK: error: The type of 'r' is incompatible with the reduction operator.
+    do i=1,10
+    end do
+  end subroutine subr
+end module m1
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-dupsym.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-dupsym.f90
new file mode 100644
index 000000000000..83f8f85299dc
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-dupsym.f90
@@ -0,0 +1,15 @@
+! RUN: not %flang_fc1 -fopenmp -fopenmp-version=50 %s 2>&1 | FileCheck %s
+
+!! Check for duplicate symbol use.
+subroutine dup_symbol()
+  type :: loc
+     integer :: x
+     integer :: y
+  end type loc
+ 
+  integer :: my_red
+
+!CHECK: error: Duplicate definition of 'my_red' in DECLARE REDUCTION
+  !$omp declare reduction(my_red : loc :  omp_out%x = omp_out%x + omp_in%x) initializer(omp_priv%x = 0)
+  
+end subroutine dup_symbol
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-error.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-error.f90
index c22cf106ea50..21f5cc186e03 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-error.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-error.f90
@@ -7,5 +7,5 @@ end subroutine initme
 
 subroutine subr
   !$omp declare reduction(red_add:integer(4):omp_out=omp_out+omp_in) initializer(initme(omp_priv,0))
-  !CHECK: error: Implicit subroutine declaration 'initme' in !$OMP DECLARE REDUCTION
+  !CHECK: error: Implicit subroutine declaration 'initme' in DECLARE REDUCTION
 end subroutine subr
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-functions.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-functions.f90
new file mode 100644
index 000000000000..000d323f522c
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-functions.f90
@@ -0,0 +1,203 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+module mm
+  implicit none
+  type two
+     integer(4) :: a, b
+  end type two
+
+  type three
+     integer(8) :: a, b, c
+  end type three
+
+  type twothree
+     type(two) t2
+     type(three) t3
+  end type twothree
+
+contains
+!CHECK-LABEL: Subprogram scope: inittwo
+  subroutine inittwo(x,n)
+    integer :: n
+    type(two) :: x
+    x%a=n
+    x%b=n
+  end subroutine inittwo
+  
+  subroutine initthree(x,n)
+    integer :: n
+    type(three) :: x
+    x%a=n
+    x%b=n
+  end subroutine initthree
+
+  function add_two(x, y)
+    type(two) add_two, x, y, res
+    res%a = x%a + y%a
+    res%b = x%b + y%b
+    add_two = res
+  end function add_two
+
+  function add_three(x, y)
+    type(three) add_three, x, y, res
+    res%a = x%a + y%a
+    res%b = x%b + y%b
+    res%c = x%c + y%c
+    add_three = res
+  end function add_three
+  
+!CHECK-LABEL: Subprogram scope: functwo
+  function functwo(x, n)
+    type(two) functwo
+    integer :: n
+    type(two) ::  x(n)
+    type(two) :: res
+    integer :: i
+    !$omp declare reduction(adder:two:omp_out=add_two(omp_out,omp_in)) initializer(inittwo(omp_priv,0))
+!CHECK: adder: UserReductionDetails TYPE(two)
+!CHECK OtherConstruct scope
+!CHECK: omp_in size=8 offset=0: ObjectEntity type: TYPE(two)
+!CHECK: omp_orig size=8 offset=8: ObjectEntity type: TYPE(two)
+!CHECK: omp_out size=8 offset=16: ObjectEntity type: TYPE(two)
+!CHECK: omp_priv size=8 offset=24: ObjectEntity type: TYPE(two)
+    
+  
+    !$omp simd reduction(adder:res)
+    do i=1,n
+       res=add_two(res,x(i))
+    enddo
+    functwo=res
+  end function functwo
+
+  function functhree(x, n)
+    implicit none
+    type(three) :: functhree
+    type(three) :: x(n)
+    type(three) :: res
+    integer :: i
+    integer :: n
+    !$omp declare reduction(adder:three:omp_out=add_three(omp_out,omp_in)) initializer(initthree(omp_priv,1))
+    
+    !$omp simd reduction(adder:res)
+    do i=1,n
+       res=add_three(res,x(i))
+    enddo
+    functhree=res
+  end function functhree
+  
+  function functwothree(x, n)
+    type(twothree) :: functwothree
+    type(twothree) :: x(n)
+    type(twothree) :: res
+    type(two) :: res2
+    type(three) :: res3
+    integer :: n
+    integer :: i
+
+    !$omp declare reduction(adder:two:omp_out=add_two(omp_out,omp_in)) initializer(inittwo(omp_priv,0))
+    
+    !$omp declare reduction(adder:three:omp_out=add_three(omp_out,omp_in)) initializer(initthree(omp_priv,1))
+    
+!CHECK: adder: UserReductionDetails TYPE(two) TYPE(three)
+!CHECK OtherConstruct scope
+!CHECK: omp_in size=8 offset=0: ObjectEntity type: TYPE(two)
+!CHECK: omp_orig size=8 offset=8: ObjectEntity type: TYPE(two)
+!CHECK: omp_out size=8 offset=16: ObjectEntity type: TYPE(two)
+!CHECK: omp_priv size=8 offset=24: ObjectEntity type: TYPE(two)
+!CHECK OtherConstruct scope
+!CHECK: omp_in size=24 offset=0: ObjectEntity type: TYPE(three)
+!CHECK: omp_orig size=24 offset=24: ObjectEntity type: TYPE(three)
+!CHECK: omp_out size=24 offset=48: ObjectEntity type: TYPE(three)
+!CHECK: omp_priv size=24 offset=72: ObjectEntity type: TYPE(three)
+
+    !$omp simd reduction(adder:res3)
+    do i=1,n
+       res3=add_three(res%t3,x(i)%t3)
+    enddo
+
+    !$omp simd reduction(adder:res2)
+    do i=1,n
+       res2=add_two(res2,x(i)%t2)
+    enddo
+    res%t2 = res2
+    res%t3 = res3
+    functwothree=res
+  end function functwothree
+
+!CHECK-LABEL: Subprogram scope: funcbtwo
+  function funcBtwo(x, n)
+    type(two) funcBtwo
+    integer :: n
+    type(two) ::  x(n)
+    type(two) :: res
+    integer :: i
+    !$omp declare reduction(+:two:omp_out=add_two(omp_out,omp_in)) initializer(inittwo(omp_priv,0))
+!CHECK: op.+: UserReductionDetails TYPE(two)
+!CHECK OtherConstruct scope
+!CHECK: omp_in size=8 offset=0: ObjectEntity type: TYPE(two)
+!CHECK: omp_orig size=8 offset=8: ObjectEntity type: TYPE(two)
+!CHECK: omp_out size=8 offset=16: ObjectEntity type: TYPE(two)
+!CHECK: omp_priv size=8 offset=24: ObjectEntity type: TYPE(two)
+    
+  
+    !$omp simd reduction(+:res)
+    do i=1,n
+       res=add_two(res,x(i))
+    enddo
+    funcBtwo=res
+  end function funcBtwo
+
+  function funcBtwothree(x, n)
+    type(twothree) :: funcBtwothree
+    type(twothree) :: x(n)
+    type(twothree) :: res
+    type(two) :: res2
+    type(three) :: res3
+    integer :: n
+    integer :: i
+
+    !$omp declare reduction(+:two:omp_out=add_two(omp_out,omp_in)) initializer(inittwo(omp_priv,0))
+    
+    !$omp declare reduction(+:three:omp_out=add_three(omp_out,omp_in)) initializer(initthree(omp_priv,1))
+    
+!CHECK: op.+: UserReductionDetails TYPE(two) TYPE(three)
+!CHECK OtherConstruct scope
+!CHECK: omp_in size=8 offset=0: ObjectEntity type: TYPE(two)
+!CHECK: omp_orig size=8 offset=8: ObjectEntity type: TYPE(two)
+!CHECK: omp_out size=8 offset=16: ObjectEntity type: TYPE(two)
+!CHECK: omp_priv size=8 offset=24: ObjectEntity type: TYPE(two)
+!CHECK: OtherConstruct scope
+!CHECK: omp_in size=24 offset=0: ObjectEntity type: TYPE(three)
+!CHECK: omp_orig size=24 offset=24: ObjectEntity type: TYPE(three)
+!CHECK: omp_out size=24 offset=48: ObjectEntity type: TYPE(three)
+!CHECK: omp_priv size=24 offset=72: ObjectEntity type: TYPE(three)
+
+    !$omp simd reduction(+:res3)
+    do i=1,n
+       res3=add_three(res%t3,x(i)%t3)
+    enddo
+
+    !$omp simd reduction(+:res2)
+    do i=1,n
+       res2=add_two(res2,x(i)%t2)
+    enddo
+    res%t2 = res2
+    res%t3 = res3
+  end function funcBtwothree
+
+  !! This is checking a special case, where a reduction is declared inside a
+  !! pure function
+
+  pure logical function reduction()
+!CHECK: reduction size=4 offset=0: ObjectEntity funcResult type: LOGICAL(4)
+!CHECK: rr: UserReductionDetails INTEGER(4)
+!CHECK: OtherConstruct scope: size=16 alignment=4 sourceRange=0 bytes
+!CHECK: omp_in size=4 offset=0: ObjectEntity type: INTEGER(4)
+!CHECK: omp_orig size=4 offset=4: ObjectEntity type: INTEGER(4)
+!CHECK: omp_out size=4 offset=8: ObjectEntity type: INTEGER(4)
+!CHECK: omp_priv size=4 offset=12: ObjectEntity type: INTEGER(4)
+    !$omp declare reduction (rr : integer : omp_out = omp_out + omp_in) initializer (omp_priv = 0)
+    reduction = .false.
+  end function reduction
+  
+end module mm
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-logical.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-logical.f90
new file mode 100644
index 000000000000..7ab7cad473ac
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-logical.f90
@@ -0,0 +1,32 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+module mm
+  implicit none
+  type logicalwrapper
+     logical b
+  end type logicalwrapper
+
+contains
+!CHECK-LABEL: Subprogram scope: func
+  function func(x, n)
+    logical func
+    integer :: n
+    type(logicalwrapper) ::  x(n)
+    type(logicalwrapper) :: res
+    integer :: i
+    !$omp declare reduction(.AND.:type(logicalwrapper):omp_out%b=omp_out%b .AND. omp_in%b) initializer(omp_priv%b=.true.)
+!CHECK: op.AND: UserReductionDetails TYPE(logicalwrapper)
+!CHECK OtherConstruct scope
+!CHECK: omp_in size=4 offset=0: ObjectEntity type: TYPE(logicalwrapper)
+!CHECK: omp_orig size=4 offset=4: ObjectEntity type: TYPE(logicalwrapper)
+!CHECK: omp_out size=4 offset=8: ObjectEntity type: TYPE(logicalwrapper)
+!CHECK: omp_priv size=4 offset=12: ObjectEntity type: TYPE(logicalwrapper)
+  
+    !$omp simd reduction(.AND.:res)
+    do i=1,n
+       res%b=res%b .and. x(i)%b
+    enddo
+    
+    func=res%b
+  end function func
+end module mm
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
new file mode 100644
index 000000000000..9d0a097fb199
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-mangled.f90
@@ -0,0 +1,51 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+!! Test that the name mangling for min & max (also used for iand, ieor and ior).
+module mymod
+  type :: tt
+     real r
+  end type tt
+contains
+  function mymax(a, b)
+    type(tt) :: a, b, mymax
+    if (a%r > b%r) then
+       mymax = a
+    else
+       mymax = b
+    end if
+  end function mymax
+end module mymod
+
+program omp_examples
+!CHECK-LABEL: MainProgram scope: omp_examples
+  use mymod
+  implicit none
+  integer, parameter :: n = 100
+  integer :: i
+  type(tt) :: values(n), big, small
+
+  !$omp declare reduction(max:tt:omp_out = mymax(omp_out, omp_in)) initializer(omp_priv%r = 0)
+  !$omp declare reduction(min:tt:omp_out%r = min(omp_out%r, omp_in%r)) initializer(omp_priv%r = 1)
+
+!CHECK: min, ELEMENTAL, INTRINSIC, PURE (Function): ProcEntity
+!CHECK: mymax (Function): Use from mymax in mymod
+!CHECK: op.max: UserReductionDetails TYPE(tt)
+!CHECK: op.min: UserReductionDetails TYPE(tt)
+
+  big%r = 0
+  !$omp parallel do reduction(max:big)
+!CHECK: big (OmpReduction, OmpExplicit): HostAssoc
+!CHECK: max, INTRINSIC: ProcEntity  
+  do i = 1, n
+     big = mymax(values(i), big)
+  end do
+
+  small%r = 1
+  !$omp parallel do reduction(min:small)
+!CHECK: small (OmpReduction, OmpExplicit): HostAssoc
+  do i = 1, n
+     small%r = min(values(i)%r, small%r)
+  end do
+  
+  print *, "small=", small%r, " big=", big%r
+end program omp_examples
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-modfile.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-modfile.f90
new file mode 100644
index 000000000000..f80eb1097e18
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-modfile.f90
@@ -0,0 +1,65 @@
+! RUN: %python %S/../test_modfile.py %s %flang_fc1 -fopenmp -fopenmp-version=52
+! Check correct modfile generation for OpenMP DECLARE REDUCTION construct.
+
+!Expect: drm.mod
+!module drm
+!type::t1
+!integer(4)::val
+!endtype
+!!$OMP DECLARE REDUCTION (*:t1:omp_out = omp_out*omp_in) INITIALIZER(omp_priv=t&
+!!$OMP&1(1))
+!!$OMP METADIRECTIVE OTHERWISE(DECLARE REDUCTION(+:INTEGER))
+!!$OMP DECLARE REDUCTION (.fluffy.:t1:omp_out = omp_out.fluffy.omp_in) INITIALI&
+!!$OMP&ZER(omp_priv=t1(0))
+!!$OMP DECLARE REDUCTION (.mul.:t1:omp_out = omp_out.mul.omp_in) INITIALIZER(om&
+!!$OMP&p_priv=t1(1))
+!interface operator(.mul.)
+!procedure::mul
+!end interface
+!interface operator(.fluffy.)
+!procedure::add
+!end interface
+!interface operator(*)
+!procedure::mul
+!end interface
+!contains
+!function mul(v1,v2)
+!type(t1),intent(in)::v1
+!type(t1),intent(in)::v2
+!type(t1)::mul
+!end
+!function add(v1,v2)
+!type(t1),intent(in)::v1
+!type(t1),intent(in)::v2
+!type(t1)::add
+!end
+!end
+
+module drm
+  type t1
+    integer :: val
+  end type t1
+  interface operator(.mul.)
+    procedure mul
+  end interface
+  interface operator(.fluffy.)
+    procedure add
+  end interface
+  interface operator(*)
+    module procedure mul
+  end interface
+!$omp declare reduction(*:t1:omp_out=omp_out*omp_in) initializer(omp_priv=t1(1))
+!$omp declare reduction(.mul.:t1:omp_out=omp_out.mul.omp_in) initializer(omp_priv=t1(1))
+!$omp declare reduction(.fluffy.:t1:omp_out=omp_out.fluffy.omp_in) initializer(omp_priv=t1(0))
+!$omp metadirective otherwise(declare reduction(+: integer))
+contains
+  type(t1) function mul(v1, v2)
+    type(t1), intent (in):: v1, v2
+    mul%val = v1%val * v2%val
+  end function
+  type(t1) function add(v1, v2)
+    type(t1), intent (in):: v1, v2
+    add%val = v1%val + v2%val
+  end function
+end module drm
+
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-operator.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-operator.f90
new file mode 100644
index 000000000000..dc12332b80ba
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-operator.f90
@@ -0,0 +1,36 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+module m1
+  interface operator(.fluffy.)
+!CHECK: .fluffy., PUBLIC (Function): Generic DefinedOp procs: my_mul
+    procedure my_mul
+  end interface
+  type t1
+    integer :: val = 1
+  end type
+!$omp declare reduction(.fluffy.:t1:omp_out=omp_out.fluffy.omp_in)
+!CHECK: op.fluffy., PUBLIC: UserReductionDetails TYPE(t1)
+!CHECK: t1, PUBLIC: DerivedType components: val
+!CHECK: OtherConstruct scope: size=16 alignment=4 sourceRange=0 bytes
+!CHECK: omp_in size=4 offset=0: ObjectEntity type: TYPE(t1)
+!CHECK: omp_orig size=4 offset=4: ObjectEntity type: TYPE(t1)
+!CHECK: omp_out size=4 offset=8: ObjectEntity type: TYPE(t1)
+!CHECK: omp_priv size=4 offset=12: ObjectEntity type: TYPE(t1)
+contains
+  function my_mul(x, y)
+    type (t1), intent (in) :: x, y
+    type (t1) :: my_mul
+    my_mul%val = x%val * y%val
+  end function my_mul
+
+  subroutine subr(a, r)
+    implicit none
+    type(t1), intent(in), dimension(10) :: a
+    type(t1), intent(out) :: r
+    integer :: i
+    !$omp parallel do reduction(.fluffy.:r)
+    do i=1,10
+       r = r .fluffy. a(i)
+    end do
+  end subroutine subr
+end module m1
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-operators.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
new file mode 100644
index 000000000000..d7a9f2fc0a36
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-operators.f90
@@ -0,0 +1,84 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+module vector_mod
+  implicit none
+  type :: Vector
+    real :: x, y, z
+  contains
+    procedure :: add_vectors
+    generic :: operator(+) => add_vectors
+  end type Vector
+contains
+  ! Function implementing vector addition
+  function add_vectors(a, b) result(res)
+    class(Vector), intent(in) :: a, b
+    type(Vector) :: res
+    res%x = a%x + b%x
+    res%y = a%y + b%y
+    res%z = a%z + b%z
+  end function add_vectors
+end module vector_mod
+
+!! Test user-defined operators. Two different varieties, using conventional and
+!! unconventional names.
+module m1
+  interface operator(.mul.)
+    procedure my_mul
+  end interface
+  interface operator(.fluffy.)
+    procedure my_add
+  end interface
+  type t1
+    integer :: val = 1
+  end type
+!$omp declare reduction(.mul.:t1:omp_out=omp_out.mul.omp_in)
+!$omp declare reduction(.fluffy.:t1:omp_out=omp_out.fluffy.omp_in)
+!CHECK: op.fluffy., PUBLIC: UserReductionDetails TYPE(t1)
+!CHECK: op.mul., PUBLIC: UserReductionDetails TYPE(t1)   
+contains
+  function my_mul(x, y)
+    type (t1), intent (in) :: x, y
+    type (t1) :: my_mul
+    my_mul%val = x%val * y%val
+  end function
+  function my_add(x, y)
+    type (t1), intent (in) :: x, y
+    type (t1) :: my_add
+    my_add%val = x%val + y%val
+  end function
+end module m1
+
+program test_vector
+!CHECK-LABEL: MainProgram scope: test_vector
+  use vector_mod
+!CHECK: add_vectors (Function): Use from add_vectors in vector_mod
+  implicit none
+  integer :: i
+  type(Vector) :: v1(100), v2(100)
+
+  !$OMP declare reduction(+:vector:omp_out=omp_out+omp_in) initializer(omp_priv=Vector(0,0,0))
+!CHECK: op.+: UserReductionDetails TYPE(vector)
+!CHECK: v1 size=1200 offset=4: ObjectEntity type: TYPE(vector) shape: 1_8:100_8
+!CHECK: v2 size=1200 offset=1204: ObjectEntity type: TYPE(vector) shape: 1_8:100_8
+!CHECK: vector: Use from vector in vector_mod
+
+!CHECK: OtherConstruct scope:
+!CHECK: omp_in size=12 offset=0: ObjectEntity type: TYPE(vector)
+!CHECK: omp_orig size=12 offset=12: ObjectEntity type: TYPE(vector)
+!CHECK: omp_out size=12 offset=24: ObjectEntity type: TYPE(vector)
+!CHECK: omp_priv size=12 offset=36: ObjectEntity type: TYPE(vector)
+
+  v2 = Vector(0.0, 0.0, 0.0)
+  v1 = Vector(1.0, 2.0, 3.0)
+  !$OMP parallel do reduction(+:v2)
+!CHECK: OtherConstruct scope
+!CHECK: i (OmpPrivate, OmpPreDetermined): HostAssoc
+!CHECK: v1 (OmpShared): HostAssoc
+!CHECK: v2 (OmpReduction, OmpExplicit): HostAssoc
+
+  do i = 1, 100
+     v2(i) = v2(i) + v1(i)  ! Invokes add_vectors
+  end do
+  
+  print *, 'v2 components:', v2%x, v2%y, v2%z
+end program test_vector
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
new file mode 100644
index 000000000000..12e80cbf7b32
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-renamedop.f90
@@ -0,0 +1,47 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s | FileCheck %s
+
+!! Test that we can "rename" an operator when using a module's operator.
+module module1
+!CHECK:  Module scope: module1 size=0
+  implicit none
+  type :: t1
+     real :: value
+  end type t1
+  interface operator(.mul.)
+     module procedure my_mul
+  end interface operator(.mul.)
+!CHECK: .mul., PUBLIC (Function): Generic DefinedOp procs: my_mul
+!CHECK: my_mul, PUBLIC (Function): Subprogram result:TYPE(t1) r (TYPE(t1) x,TYPE(t1)
+!CHECK: t1, PUBLIC: DerivedType components: value
+contains
+    function my_mul(x, y) result(r)
+      type(t1), intent(in) :: x, y
+      type(t1) :: r
+      r%value = x%value * y%value
+    end function my_mul
+end module module1
+
+program test_omp_reduction
+!CHECK: MainProgram scope: test_omp_reduction
+  use module1, only: t1, operator(.modmul.) => operator(.mul.)
+
+!CHECK: .modmul. (Function): Use from .mul. in module1
+  implicit none
+
+  type(t1) :: result
+  integer :: i
+  !$omp declare reduction (.modmul. : t1 : omp_out = omp_out .modmul. omp_in) initializer(omp_priv = t1(1.0))
+!CHECK: op.modmul.: UserReductionDetails TYPE(t1)
+!CHECK: t1: Use from t1 in module1
+!CHECK: OtherConstruct scope: size=16 alignment=4 sourceRange=0 bytes
+!CHECK: omp_in size=4 offset=0: ObjectEntity type: TYPE(t1)
+!CHECK: omp_orig size=4 offset=4: ObjectEntity type: TYPE(t1)
+!CHECK: omp_out size=4 offset=8: ObjectEntity type: TYPE(t1)
+!CHECK: omp_priv size=4 offset=12: ObjectEntity type: TYPE(t1)
+  result = t1(1.0)
+  !$omp parallel do reduction(.modmul.:result)
+  do i = 1, 10
+     result = result .modmul. t1(real(i))
+  end do
+  !$omp end parallel do
+end program test_omp_reduction
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-typeerror.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-typeerror.f90
new file mode 100644
index 000000000000..b8ede55aa0ed
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction-typeerror.f90
@@ -0,0 +1,34 @@
+! RUN: not %flang_fc1 -fdebug-dump-symbols -fopenmp -fopenmp-version=50 %s 2>&1 | FileCheck %s
+
+module mm
+  implicit none
+  type two
+     integer(4) :: a, b
+  end type two
+
+  type three
+     integer(8) :: a, b, c
+  end type three
+contains
+  function add_two(x, y)
+    type(two) add_two, x, y, res
+    add_two = res
+  end function add_two
+
+  function func(n)
+    type(three) :: func
+    type(three) :: res3
+    integer :: n
+    integer :: i
+
+    !$omp declare reduction(dummy:kerflunk:omp_out=omp_out+omp_in)
+!CHECK: error: Derived type 'kerflunk' not found
+    
+    !$omp declare reduction(adder:two:omp_out=add_two(omp_out,omp_in))
+    !$omp simd reduction(adder:res3)
+!CHECK: error: The type of 'res3' is incompatible with the reduction operator.
+    do i=1,n
+    enddo
+    func = res3
+  end function func
+end module mm
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction.f90
index 11612f01f0f2..ddca38fd5781 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-reduction.f90
@@ -17,7 +17,7 @@ subroutine initme(x,n)
      end subroutine initme
   end interface
   !$omp declare reduction(red_add:integer(4):omp_out=omp_out+omp_in) initializer(initme(omp_priv,0))
-!CHECK: red_add: Misc ConstructName
+!CHECK: red_add: UserReductionDetails
 !CHECK: Subprogram scope: initme
 !CHECK: omp_in size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK: omp_orig size=4 offset=4: ObjectEntity type: INTEGER(4)
@@ -35,7 +35,7 @@ program main
 
   !$omp declare reduction (my_add_red : integer : omp_out = omp_out + omp_in) initializer (omp_priv=0)
 
-!CHECK: my_add_red: Misc ConstructName
+!CHECK: my_add_red: UserReductionDetails
 !CHECK: omp_in size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK: omp_orig size=4 offset=4: ObjectEntity type: INTEGER(4)
 !CHECK: omp_out size=4 offset=8: ObjectEntity type: INTEGER(4)
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-simd-linear.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-simd-linear.f90
new file mode 100644
index 000000000000..2a6ef0081996
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-simd-linear.f90
@@ -0,0 +1,12 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! Test declare simd with linear clause does not cause an implicit declaration of i
+
+module mod
+contains
+subroutine test(i)
+!$omp declare simd linear(i:1)
+  implicit none
+  integer*8 i
+  i=i+2
+end subroutine
+end module
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-simd-uniform.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-simd-uniform.f90
new file mode 100644
index 000000000000..8d986300cd75
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-simd-uniform.f90
@@ -0,0 +1,22 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
+! Test declare simd with uniform clause
+
+function add2(a,b,i,fact,alc) result(c)
+  !$omp declare simd(add2) uniform(a,b,fact)
+  integer :: i
+  integer,pointer::alc
+  double precision :: a(*),b(*),fact,c
+  c = a(i) + b(i) + fact
+end function
+
+end
+
+! CHECK-LABEL: Subprogram scope: add2 size=48 alignment=8 sourceRange=189 bytes
+! CHECK-NEXT:    a (OmpUniform): ObjectEntity dummy type: REAL(8) shape: 1_8:*
+! CHECK-NEXT:    add2 (Function): HostAssoc
+! CHECK-NEXT:    alc, POINTER size=24 offset=8: ObjectEntity dummy type: INTEGER(4)
+! CHECK-NEXT:    b (OmpUniform): ObjectEntity dummy type: REAL(8) shape: 1_8:*
+! CHECK-NEXT:    c size=8 offset=40: ObjectEntity funcResult type: REAL(8)
+! CHECK-NEXT:    fact (OmpUniform) size=8 offset=32: ObjectEntity dummy type: REAL(8)
+! CHECK-NEXT:    i size=4 offset=0: ObjectEntity dummy type: INTEGER(4)
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/declare-target-common-block.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/declare-target-common-block.f90
index 33a093a03a22..ccdedaa52848 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/declare-target-common-block.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/declare-target-common-block.f90
@@ -1,8 +1,8 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
 
 PROGRAM main
-    !CHECK: one (OmpDeclareTarget) size=4 offset=0: ObjectEntity type: REAL(4)
-    !CHECK: two (OmpDeclareTarget) size=4 offset=4: ObjectEntity type: REAL(4)
+    !CHECK: one (InCommonBlock, OmpDeclareTarget) size=4 offset=0: ObjectEntity type: REAL(4)
+    !CHECK: two (InCommonBlock, OmpDeclareTarget) size=4 offset=4: ObjectEntity type: REAL(4)
     !CHECK: numbers size=8 offset=0: CommonBlockDetails alignment=4: one two
     REAL :: one, two
     COMMON /numbers/ one, two
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/default-clause.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/default-clause.f90
index 9cde77be2bab..d4c38ea56de5 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/default-clause.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/default-clause.f90
@@ -15,8 +15,8 @@ program sample
         !CHECK: OtherConstruct scope: size=0 alignment=1
         !CHECK: a (OmpPrivate): HostAssoc
         !CHECK: k (OmpPrivate): HostAssoc
-        !CHECK: x (OmpFirstPrivate): HostAssoc
-        !CHECK: y (OmpPrivate): HostAssoc
+        !CHECK: x (OmpFirstPrivate, OmpExplicit): HostAssoc
+        !CHECK: y (OmpPrivate, OmpExplicit): HostAssoc
         !CHECK: z (OmpPrivate): HostAssoc
         !$omp parallel default(private)
             !CHECK: OtherConstruct scope: size=0 alignment=1
@@ -34,7 +34,7 @@ program sample
         !$omp parallel default(firstprivate) shared(y) private(w)
             !CHECK: OtherConstruct scope: size=0 alignment=1
             !CHECK: k (OmpFirstPrivate): HostAssoc
-            !CHECK: w (OmpPrivate): HostAssoc
+            !CHECK: w (OmpPrivate, OmpExplicit): HostAssoc
             !CHECK: z (OmpFirstPrivate): HostAssoc
             y = 30
             w = 40 
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/depend-substring.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/depend-substring.f90
new file mode 100644
index 000000000000..23d6bb4c0b7b
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/depend-substring.f90
@@ -0,0 +1,65 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! Test for parsing confusion between array indexing and string subscripts
+
+! This is okay: selects the whole substring
+subroutine substring_0(c)
+  character(:), pointer :: c
+  !PORTABILITY: The use of substrings in OpenMP argument lists has been disallowed since OpenMP 5.2.
+  !$omp task depend(out:c(:))
+  !$omp end task
+end
+
+! This is okay: selects from the second character onwards
+subroutine substring_1(c)
+  character(:), pointer :: c
+  !PORTABILITY: The use of substrings in OpenMP argument lists has been disallowed since OpenMP 5.2.
+  !$omp task depend(out:c(2:))
+  !$omp end task
+end
+
+! This is okay: selects the first 2 characters
+subroutine substring_2(c)
+  character(:), pointer :: c
+  !PORTABILITY: The use of substrings in OpenMP argument lists has been disallowed since OpenMP 5.2.
+  !$omp task depend(out:c(:2))
+  !$omp end task
+end
+
+! Error
+subroutine substring_3(c)
+  character(:), pointer :: c
+  !PORTABILITY: The use of substrings in OpenMP argument lists has been disallowed since OpenMP 5.2.
+  !ERROR: Substrings must be in the form parent-string(lb:ub)
+  !$omp task depend(out:c(2))
+  !$omp end task
+end
+
+! This is okay: interpreted as indexing into the array not as a substring
+subroutine substring_3b(c)
+  character(:), pointer :: c(:)
+  !$omp task depend(out:c(2))
+  !$omp end task
+end
+
+! This is okay: no indexing or substring at all
+subroutine substring_4(c)
+  character(:), pointer :: c
+  !$omp task depend(out:c)
+  !$omp end task
+end
+
+! This is not okay: substrings can't have a stride
+subroutine substring_5(c)
+  character(:), pointer :: c
+  !PORTABILITY: The use of substrings in OpenMP argument lists has been disallowed since OpenMP 5.2.
+  !ERROR: Cannot specify a step for a substring
+  !$omp task depend(out:c(1:20:5))
+  !$omp end task
+end
+
+! This is okay: interpreted as indexing the array
+subroutine substring_5b(c)
+  character(:), pointer :: c(:)
+  !$omp task depend(out:c(1:20:5))
+  !$omp end task
+end
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/do05-positivecase.f90
index 5e1b1b86f72f..8481cb2fc2ca 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -20,12 +20,12 @@ program omp_do
   !$omp parallel  default(shared)
   !$omp do
   !DEF: /omp_do/OtherConstruct2/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !DEF: /omp_do/OtherConstruct2/n HostAssoc INTEGER(4)
+  !DEF: /omp_do/OtherConstruct2/OtherConstruct1/n HostAssoc INTEGER(4)
   do i=1,n
     !$omp parallel
     !$omp single
     !DEF: /work EXTERNAL (Subroutine) ProcEntity
-    !DEF: /omp_do/OtherConstruct2/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
+    !DEF: /omp_do/OtherConstruct2/OtherConstruct1/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
     call work(i, 1)
     !$omp end single
     !$omp end parallel
@@ -34,7 +34,7 @@ program omp_do
   !$omp end parallel
 
   !$omp parallel private(i)
-  !DEF: /omp_do/OtherConstruct3/i (OmpPrivate) HostAssoc INTEGER(4)
+  !DEF: /omp_do/OtherConstruct3/i (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
   do i=1,10
      !$omp single
      print *, "hello"
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/do20.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/do20.f90
index 040a82079590..ee305ad1a34c 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/do20.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/do20.f90
@@ -10,7 +10,7 @@ subroutine shared_iv
 
   !$omp parallel shared(i)
     !$omp single
-      !DEF: /shared_iv/OtherConstruct1/i (OmpShared) HostAssoc INTEGER(4)
+      !DEF: /shared_iv/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
       do i = 0, 1
       end do
     !$omp end single
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/forall.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/forall.f90
index 58492664a4e8..b862b4b27641 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/forall.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/forall.f90
@@ -18,8 +18,8 @@
 
   !$omp parallel
     !DEF: /MainProgram1/OtherConstruct1/Forall1/i (Implicit) ObjectEntity INTEGER(4)
-    !DEF: /MainProgram1/OtherConstruct1/a HostAssoc INTEGER(4)
-    !DEF: /MainProgram1/OtherConstruct1/b HostAssoc INTEGER(4)
+    !DEF: /MainProgram1/OtherConstruct1/a (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /MainProgram1/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
     forall(i = 1:5) a(i) = b(i) * 2
   !$omp end parallel
 
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/implicit-dsa.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/implicit-dsa.f90
index a7ed834b0f1c..4a07e256e2bb 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/implicit-dsa.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/implicit-dsa.f90
@@ -14,15 +14,15 @@ subroutine implicit_dsa_test1
 
   !$omp task private(y) shared(z)
     !DEF: /implicit_dsa_test1/OtherConstruct1/x (OmpFirstPrivate, OmpImplicit) HostAssoc INTEGER(4)
-    !DEF: /implicit_dsa_test1/OtherConstruct1/y (OmpPrivate) HostAssoc INTEGER(4)
-    !DEF: /implicit_dsa_test1/OtherConstruct1/z (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /implicit_dsa_test1/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
+    !DEF: /implicit_dsa_test1/OtherConstruct1/z (OmpShared, OmpExplicit) HostAssoc INTEGER(4)
     x = y + z
   !$omp end task
 
   !$omp task default(shared)
-    !DEF: /implicit_dsa_test1/OtherConstruct2/x HostAssoc INTEGER(4)
-    !DEF: /implicit_dsa_test1/OtherConstruct2/y HostAssoc INTEGER(4)
-    !DEF: /implicit_dsa_test1/OtherConstruct2/z HostAssoc INTEGER(4)
+    !DEF: /implicit_dsa_test1/OtherConstruct2/x (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /implicit_dsa_test1/OtherConstruct2/y (OmpShared) HostAssoc INTEGER(4)
+    !DEF: /implicit_dsa_test1/OtherConstruct2/z (OmpShared) HostAssoc INTEGER(4)
     x = y + z
   !$omp end task
 
@@ -61,16 +61,16 @@ subroutine implicit_dsa_test3
 
   !$omp parallel
     !$omp task
-      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct1/x HostAssoc INTEGER(4)
+      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct1/x (OmpShared) HostAssoc INTEGER(4)
       x = 1
-      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct1/y HostAssoc INTEGER(4)
+      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct1/y (OmpShared) HostAssoc INTEGER(4)
       y = 1
     !$omp end task
 
     !$omp task firstprivate(x)
-      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct2/x (OmpFirstPrivate) HostAssoc INTEGER(4)
+      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct2/x (OmpFirstPrivate, OmpExplicit) HostAssoc INTEGER(4)
       x = 1
-      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct2/z HostAssoc INTEGER(4)
+      !DEF: /implicit_dsa_test3/OtherConstruct1/OtherConstruct2/z (OmpShared) HostAssoc INTEGER(4)
       z = 1
     !$omp end task
   !$omp end parallel
@@ -110,7 +110,7 @@ subroutine implicit_dsa_test5
   !$omp parallel default(private)
     !$omp task
       !$omp parallel
-        !DEF: /implicit_dsa_test5/OtherConstruct1/OtherConstruct1/OtherConstruct1/x HostAssoc INTEGER(4)
+        !DEF: /implicit_dsa_test5/OtherConstruct1/OtherConstruct1/OtherConstruct1/x (OmpShared) HostAssoc INTEGER(4)
         x = 1
       !$omp end parallel
     !$omp end task
@@ -133,7 +133,7 @@ subroutine implicit_dsa_test6
     !$omp end parallel
 
     !$omp parallel default(firstprivate) shared(y)
-      !DEF: /implicit_dsa_test6/OtherConstruct1/OtherConstruct2/y (OmpShared) HostAssoc INTEGER(4)
+      !DEF: /implicit_dsa_test6/OtherConstruct1/OtherConstruct2/y (OmpShared, OmpExplicit) HostAssoc INTEGER(4)
       !DEF: /implicit_dsa_test6/OtherConstruct1/OtherConstruct2/x (OmpFirstPrivate) HostAssocINTEGER(4)
       !DEF: /implicit_dsa_test6/OtherConstruct1/OtherConstruct2/z (OmpFirstPrivate) HostAssocINTEGER(4)
       y = x + z
@@ -156,3 +156,113 @@ subroutine implicit_dsa_test7
     !$omp end taskgroup
   !$omp end task
 end subroutine
+
+! Predetermined loop iteration variable.
+!DEF: /implicit_dsa_test8 (Subroutine) Subprogram
+subroutine implicit_dsa_test8
+  !DEF: /implicit_dsa_test8/i ObjectEntity INTEGER(4)
+  integer i
+
+  !$omp task
+    !DEF: /implicit_dsa_test8/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+    do i = 1, 10
+    end do
+  !$omp end task
+end subroutine
+
+! Test variables defined in modules default to shared DSA
+!DEF: /implicit_dsa_test9_mod Module
+module implicit_dsa_test9_mod
+ !DEF: /implicit_dsa_test9_mod/tm3a PUBLIC (InDataStmt) ObjectEntity COMPLEX(4)
+  complex tm3a/(0,0)/
+ !DEF: /implicit_dsa_test9_mod/tm4a PUBLIC ObjectEntity COMPLEX(4)
+  complex tm4a
+contains
+ !DEF: /implicit_dsa_test9_mod/implict_dsa_test9 PUBLIC (Subroutine) Subprogram
+  subroutine implict_dsa_test9
+    !$omp task
+      !$omp task
+        !DEF: /implicit_dsa_test9_mod/implict_dsa_test9/OtherConstruct1/OtherConstruct1/tm3a (OmpShared) HostAssoc COMPLEX(4)
+        tm3a = (1, 2)
+        !DEF: /implicit_dsa_test9_mod/implict_dsa_test9/OtherConstruct1/OtherConstruct1/tm4a (OmpShared) HostAssoc COMPLEX(4)
+        tm4a = (3, 4)
+      !$omp end task
+    !$omp end task
+  !$omp taskwait
+  !REF: /implicit_dsa_test9_mod/tm3a
+  print *,tm3a
+  end subroutine
+end module
+
+! Test variables in data statement default to shared DSA
+!DEF: /implicit_dsa_test10 (Subroutine) Subprogram
+subroutine implicit_dsa_test10
+ !DEF: /implicit_dsa_test10/tm3a (Implicit, InDataStmt) ObjectEntity REAL(4)
+data tm3a /3/
+!$omp task
+  !$omp task
+ !DEF: /implicit_dsa_test10/OtherConstruct1/OtherConstruct1/tm3a (OmpShared) HostAssoc REAL(4)
+    tm3a = 5
+  !$omp end task
+!$omp end task
+!$omp taskwait
+ !REF: /implicit_dsa_test10/tm3a
+print *,tm3a
+end subroutine
+
+! Test variables with the SAVE attrtibute default to shared DSA
+!DEF: /implicit_dsa_test_11 (Subroutine) Subprogram
+subroutine implicit_dsa_test_11
+ !DEF: /implicit_dsa_test_11/tm3a SAVE ObjectEntity COMPLEX(4)
+complex, save :: tm3a
+!$omp task
+  !$omp task
+    !DEF: /implicit_dsa_test_11/OtherConstruct1/OtherConstruct1/tm3a (OmpShared) HostAssoc COMPLEX(4)
+    tm3a = (1, 2)
+  !$omp end task
+!$omp end task
+!$omp taskwait
+!REF: /implicit_dsa_test_11/tm3a
+print *,tm3a
+end subroutine
+
+! Test variables referenced in a common block default to shared DSA
+!DEF: /implicit_dsa_test_12 (Subroutine) Subprogram
+subroutine implicit_dsa_test_12
+ !DEF: /implicit_dsa_test_12/tm3a (InCommonBlock) ObjectEntity COMPLEX(4)
+complex tm3a
+ !DEF: /implicit_dsa_test_12/tcom CommonBlockDetails
+ !REF: /implicit_dsa_test_12/tm3a
+common /tcom/ tm3a
+!$omp task
+  !$omp task
+    !DEF: /implicit_dsa_test_12/OtherConstruct1/OtherConstruct1/tm3a (OmpShared) HostAssoc COMPLEX(4)
+    tm3a = (1, 2)
+  !$omp end task
+!$omp end task
+!$omp taskwait
+!REF: /implicit_dsa_test_12/tm3a
+print *,tm3a
+end subroutine
+
+! Test static duration variables with DSA set in the enclosing scope do not default to shared DSA
+!DEF: /implicit_dsa_test_13_mod Module
+module implicit_dsa_test_13_mod
+  !DEF: /implicit_dsa_test_13_mod/a PUBLIC ObjectEntity INTEGER(4)
+  integer::a=5
+contains
+  !DEF: /implicit_dsa_test_13_mod/implicit_dsa_test_13 PUBLIC (Subroutine) Subprogram
+  subroutine implicit_dsa_test_13
+    !DEF: /implicit_dsa_test_13_mod/implicit_dsa_test_13/i ObjectEntity INTEGER(4)
+    integer i
+    !$omp do private(a)
+      !DEF: /implicit_dsa_test_13_mod/implicit_dsa_test_13/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+      do i=0,10
+        !$omp task
+        !DEF: /implicit_dsa_test_13_mod/implicit_dsa_test_13/OtherConstruct1/OtherConstruct1/a (OmpFirstPrivate, OmpImplicit) HostAssoc INTEGER(4)
+        !DEF: /implicit_dsa_test_13_mod/implicit_dsa_test_13/OtherConstruct1/OtherConstruct1/i (OmpFirstPrivate, OmpImplicit) HostAssoc INTEGER(4)
+        a=a+i
+        !$omp end task
+      end do
+  end subroutine implicit_dsa_test_13
+end module implicit_dsa_test_13_mod
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/linear-clause01.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/linear-clause01.f90
index 343d633a5181..973f4904d325 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/linear-clause01.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/linear-clause01.f90
@@ -27,12 +27,10 @@ subroutine linear_clause_02(arg_01, arg_02)
     integer, value, intent(in) :: arg_02
 
     !WARNING: The 'modifier(<list>)' syntax is deprecated in OpenMP v5.2, use '<list> : modifier' instead
-    !ERROR: The list item 'var' specified without the REF 'linear-modifier' must be of INTEGER type
     !ERROR: If the `linear-modifier` is REF or UVAL, the list item 'var' must be a dummy argument without the VALUE attribute
     !ERROR: The list item `var` must be a dummy argument
     !ERROR: The list item `var` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute
     !$omp declare simd linear(uval(var))
-    !ERROR: The type of 'var' has already been implicitly declared
     integer, pointer :: var
 end subroutine linear_clause_02
 
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
index 505cbc48fef9..8fdd2aed3ec1 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
@@ -20,70 +20,64 @@ program sample
 
     !$omp atomic read
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar INTEGER(4) and rank 1 array of INTEGER(4)
-    !ERROR: Expected scalar expression on the RHS of atomic assignment statement
+    !ERROR: Atomic variable y(1_8:3_8:1_8) should be a scalar
         v = y(1:3)
 
     !$omp atomic read
-    !ERROR: Expected scalar variable of intrinsic type on RHS of atomic assignment statement
+    !ERROR: Atomic expression x*(10_4+x) should be a variable
         v = x * (10 + x)
 
     !$omp atomic read
-    !ERROR: Expected scalar variable of intrinsic type on RHS of atomic assignment statement
+    !ERROR: Atomic expression 4_4 should be a variable
         v = 4
 
     !$omp atomic read
-    !ERROR: k must not have ALLOCATABLE attribute
+    !ERROR: Atomic variable k cannot be ALLOCATABLE
         v = k
 
     !$omp atomic write
-    !ERROR: k must not have ALLOCATABLE attribute
+    !ERROR: Atomic variable k cannot be ALLOCATABLE
         k = x
 
     !$omp atomic update
-    !ERROR: k must not have ALLOCATABLE attribute
+    !ERROR: Atomic variable k cannot be ALLOCATABLE
         k = k + x * (v * x)
 
     !$omp atomic
-    !ERROR: k must not have ALLOCATABLE attribute
+    !ERROR: Atomic variable k cannot be ALLOCATABLE
         k = v * k  
          
     !$omp atomic write
-    !ERROR: RHS expression on atomic assignment statement cannot access 'z%y'
+    !ERROR: Within atomic operation z%y and x+z%y access the same storage
        z%y = x + z%y
 
     !$omp atomic write
-    !ERROR: RHS expression on atomic assignment statement cannot access 'x'
+    !ERROR: Within atomic operation x and x access the same storage
         x = x
 
     !$omp atomic write
-    !ERROR: RHS expression on atomic assignment statement cannot access 'm'
+    !ERROR: Within atomic operation m and min(m,x,z%m)+k access the same storage
         m = min(m, x, z%m) + k
  
     !$omp atomic read
-    !ERROR: RHS expression on atomic assignment statement cannot access 'x'
+    !ERROR: Within atomic operation x and x access the same storage
         x = x
 
     !$omp atomic read
-    !ERROR: Expected scalar variable of intrinsic type on RHS of atomic assignment statement
-    !ERROR: RHS expression on atomic assignment statement cannot access 'm'
+    !ERROR: Atomic expression min(m,x,z%m)+k should be a variable
         m = min(m, x, z%m) + k
 
     !$omp atomic read
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar INTEGER(4) and rank 1 array of INTEGER(4)
-    !ERROR: Expected scalar expression on the RHS of atomic assignment statement
+    !ERROR: Atomic variable a should be a scalar
         x = a
 
-    !$omp atomic read
-    !ERROR: Expected scalar variable on the LHS of atomic assignment statement
-        a = x
-
     !$omp atomic write
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar INTEGER(4) and rank 1 array of INTEGER(4)
-    !ERROR: Expected scalar expression on the RHS of atomic assignment statement
         x = a
 
     !$omp atomic write
-    !ERROR: Expected scalar variable on the LHS of atomic assignment statement
+    !ERROR: Atomic variable a should be a scalar
         a = x
 
     !$omp atomic capture
@@ -93,7 +87,7 @@ program sample
 
     !$omp atomic release capture
         v = x
-    !ERROR: Atomic update statement should be of form `x = x operator expr` OR `x = expr operator x`
+    ! This ends up being "x = b + x".
         x = b + (x*1)
     !$omp end atomic
 
@@ -103,60 +97,58 @@ program sample
     !$omp end atomic
 
     !$omp atomic capture
-    !ERROR: Captured variable/array element/derived-type component x expected to be assigned in the second statement of ATOMIC CAPTURE construct
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read b
         v = x
         b = b + 1
     !$omp end atomic
 
     !$omp atomic capture
-    !ERROR: Captured variable/array element/derived-type component x expected to be assigned in the second statement of ATOMIC CAPTURE construct
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read b
         v = x
         b = 10
     !$omp end atomic
 
     !$omp atomic capture
-    !ERROR: Updated variable/array element/derived-type component x expected to be captured in the second statement of ATOMIC CAPTURE construct
         x = x + 10
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read x
         v = b
     !$omp end atomic
 
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture
     !$omp atomic capture
-    !ERROR: Invalid ATOMIC CAPTURE construct statements. Expected one of [update-stmt, capture-stmt], [capture-stmt, update-stmt], or [capture-stmt, write-stmt]
         v = 1
         x = 4
     !$omp end atomic
 
     !$omp atomic capture
-    !ERROR: Captured variable/array element/derived-type component z%y expected to be assigned in the second statement of ATOMIC CAPTURE construct
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read z%m
         x = z%y
         z%m = z%m + 1.0
     !$omp end atomic
 
     !$omp atomic capture
-    !ERROR: Updated variable/array element/derived-type component z%m expected to be captured in the second statement of ATOMIC CAPTURE construct
         z%m = z%m + 1.0
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read z%m
         x = z%y
     !$omp end atomic
 
     !$omp atomic capture
-    !ERROR: Captured variable/array element/derived-type component y(2) expected to be assigned in the second statement of ATOMIC CAPTURE construct
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read y(1_8)
         x = y(2)
         y(1) = y(1) + 1
     !$omp end atomic
 
     !$omp atomic capture
-    !ERROR: Updated variable/array element/derived-type component y(1) expected to be captured in the second statement of ATOMIC CAPTURE construct
         y(1) = y(1) + 1
+    !ERROR: In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read y(1_8)
         x = y(2)
     !$omp end atomic
 
     !$omp atomic read
-    !ERROR: Expected scalar variable on the LHS of atomic assignment statement
-    !ERROR: Expected scalar expression on the RHS of atomic assignment statement
+    !ERROR: Atomic variable r cannot have CHARACTER type
         l = r
 
     !$omp atomic write
-    !ERROR: Expected scalar variable on the LHS of atomic assignment statement
-    !ERROR: Expected scalar expression on the RHS of atomic assignment statement
+    !ERROR: Atomic variable l cannot have CHARACTER type
         l = r
 end program
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/parallel-master-goto.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/parallel-master-goto.f90
new file mode 100644
index 000000000000..72c8002ab4c5
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/parallel-master-goto.f90
@@ -0,0 +1,15 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! Regression test for #143229
+
+!$omp parallel
+do i = 1, 2
+!ERROR: invalid branch into an OpenMP structured block
+!ERROR: invalid branch leaving an OpenMP structured block
+  goto 10
+end do
+!WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+!$omp master
+10 print *, i
+!$omp end master
+!$omp end parallel
+end
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/reduction08.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/reduction08.f90
index 9442fbd4d597..01a06eb7d741 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/reduction08.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/reduction08.f90
@@ -13,9 +13,9 @@ program omp_reduction
   !$omp parallel do  reduction(max:k)
   !DEF: /omp_reduction/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     !DEF: /omp_reduction/max ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct1/m HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct1/m (OmpShared) HostAssoc INTEGER(4)
     k = max(k, m)
   end do
   !$omp end parallel do
@@ -23,9 +23,9 @@ program omp_reduction
   !$omp parallel do  reduction(min:k)
   !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct2/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     !DEF: /omp_reduction/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct2/m HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct2/m (OmpShared) HostAssoc INTEGER(4)
     k = min(k, m)
   end do
   !$omp end parallel do
@@ -33,9 +33,9 @@ program omp_reduction
   !$omp parallel do  reduction(iand:k)
   !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct3/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     !DEF: /omp_reduction/iand ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct3/m HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct3/m (OmpShared) HostAssoc INTEGER(4)
     k = iand(k, m)
   end do
   !$omp end parallel do
@@ -43,9 +43,9 @@ program omp_reduction
   !$omp parallel do  reduction(ior:k)
   !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct4/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     !DEF: /omp_reduction/ior ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct4/m HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct4/m (OmpShared) HostAssoc INTEGER(4)
     k = ior(k, m)
   end do
   !$omp end parallel do
@@ -53,9 +53,9 @@ program omp_reduction
   !$omp parallel do  reduction(ieor:k)
   !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct5/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     !DEF: /omp_reduction/ieor ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-    !DEF: /omp_reduction/OtherConstruct5/m HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct5/m (OmpShared) HostAssoc INTEGER(4)
     k = ieor(k,m)
   end do
   !$omp end parallel do
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/reduction09.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/reduction09.f90
index 1af2fc4fd969..d6c71c30d283 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/reduction09.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/reduction09.f90
@@ -16,7 +16,7 @@ program omp_reduction
   !$omp do  reduction(+:k)
   !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct1/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
@@ -26,7 +26,7 @@ program omp_reduction
   !$omp parallel do  reduction(+:a(10))
   !DEF: /omp_reduction/OtherConstruct2/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct2/k HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct2/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
@@ -35,7 +35,7 @@ program omp_reduction
   !$omp parallel do  reduction(+:a(1:10:1))
   !DEF: /omp_reduction/OtherConstruct3/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct3/k HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct3/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
@@ -43,7 +43,7 @@ program omp_reduction
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2))
   !DEF: /omp_reduction/OtherConstruct4/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct4/k HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct4/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
@@ -51,7 +51,7 @@ program omp_reduction
   !$omp parallel do  reduction(+:b(1:10:1,1:5,2:5:1))
   !DEF: /omp_reduction/OtherConstruct5/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct5/k HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct5/k (OmpShared) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end parallel do
@@ -60,7 +60,7 @@ program omp_reduction
   !$omp do reduction(+:k) reduction(+:j)
   !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct6/OtherConstruct1/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
@@ -69,7 +69,7 @@ program omp_reduction
   !$omp do reduction(+:k) reduction(*:j) reduction(+:l)
   !DEF: /omp_reduction/OtherConstruct7/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-    !DEF: /omp_reduction/OtherConstruct7/k (OmpReduction) HostAssoc INTEGER(4)
+    !DEF: /omp_reduction/OtherConstruct7/k (OmpReduction, OmpExplicit) HostAssoc INTEGER(4)
     k = k+1
   end do
   !$omp end do
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/reduction11.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/reduction11.f90
index 3893fe70b407..b2ad0f6a6ee1 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/reduction11.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/reduction11.f90
@@ -12,7 +12,7 @@ program omp_reduction
 
   ! CHECK: OtherConstruct scope
   ! CHECK: i (OmpPrivate, OmpPreDetermined): HostAssoc
-  ! CHECK: k (OmpReduction): HostAssoc
+  ! CHECK: k (OmpReduction, OmpExplicit): HostAssoc
   ! CHECK: max, INTRINSIC: ProcEntity
   !$omp parallel do  reduction(max:k)
   do i=1,10
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic01.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic01.f90
index ae9fd086015d..e8817c3f5ef6 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic01.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic01.f90
@@ -10,20 +10,23 @@ program requires
   ! READ
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Read
+  ! CHECK: OmpClause -> SeqCst
   !$omp atomic read
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Read
   !$omp atomic relaxed read
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Read
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic read relaxed
   i = j
   
@@ -31,20 +34,23 @@ program requires
   ! WRITE
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Write
+  ! CHECK: OmpClause -> SeqCst
   !$omp atomic write
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Write
   !$omp atomic relaxed write
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Write
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic write relaxed
   i = j
 
@@ -52,31 +58,34 @@ program requires
   ! UPDATE
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Update
+  ! CHECK: OmpClause -> SeqCst
   !$omp atomic update
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Update
   !$omp atomic relaxed update
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Update
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic update relaxed
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> SeqCst
   !$omp atomic
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic relaxed
   i = i + j
 
@@ -84,24 +93,27 @@ program requires
   ! CAPTURE
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Capture
+  ! CHECK: OmpClause -> SeqCst
   !$omp atomic capture
   i = j
   j = j + 1
   !$omp end atomic
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Capture
   !$omp atomic relaxed capture
   i = j
   j = j + 1
   !$omp end atomic
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Capture
+  ! CHECK-NOT: OmpClause -> SeqCst
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic capture relaxed
   i = j
   j = j + 1
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic02.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic02.f90
index 4976a9667eb7..04a9b7a09aa9 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic02.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/requires-atomic02.f90
@@ -10,20 +10,23 @@ program requires
   ! READ
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Acquire
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Read
+  ! CHECK: OmpClause -> Acquire
   !$omp atomic read
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Acquire
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Read
   !$omp atomic relaxed read
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Acquire
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Read
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic read relaxed
   i = j
   
@@ -31,20 +34,23 @@ program requires
   ! WRITE
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Write
+  ! CHECK: OmpClause -> Release
   !$omp atomic write
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Write
   !$omp atomic relaxed write
   i = j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Write
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic write relaxed
   i = j
 
@@ -52,31 +58,34 @@ program requires
   ! UPDATE
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Update
+  ! CHECK: OmpClause -> Release
   !$omp atomic update
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Update
   !$omp atomic relaxed update
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Update
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic update relaxed
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Release
   !$omp atomic
   i = i + j
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic relaxed
   i = i + j
 
@@ -84,24 +93,27 @@ program requires
   ! CAPTURE
   ! ----------------------------------------------------------------------------
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> AcqRel
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK: OmpClause -> Capture
+  ! CHECK: OmpClause -> AcqRel
   !$omp atomic capture
   i = j
   j = j + 1
   !$omp end atomic
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> AcqRel
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Relaxed
+  ! CHECK: OmpClause -> Capture
   !$omp atomic relaxed capture
   i = j
   j = j + 1
   !$omp end atomic
 
-  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
-  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> AcqRel
-  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  ! CHECK-LABEL: OpenMPAtomicConstruct
+  ! CHECK-NOT: OmpClause -> AcqRel
+  ! CHECK: OmpClause -> Capture
+  ! CHECK: OmpClause -> Relaxed
   !$omp atomic capture relaxed
   i = j
   j = j + 1
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/scan2.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/scan2.f90
index 5232e63aa6b4..ffe84910f88a 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/scan2.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/scan2.f90
@@ -12,13 +12,13 @@ program omp_reduction
 
   ! CHECK: OtherConstruct scope
   ! CHECK: i (OmpPrivate, OmpPreDetermined): HostAssoc
-  ! CHECK: k (OmpReduction, OmpInclusiveScan, OmpInScanReduction): HostAssoc
+  ! CHECK: k (OmpReduction, OmpExplicit, OmpInclusiveScan, OmpInScanReduction): HostAssoc
   !$omp parallel do  reduction(inscan, +:k)
   do i=1,10
    !$omp scan inclusive(k)
   end do
   !$omp end parallel do
-  ! CHECK: m (OmpReduction, OmpExclusiveScan, OmpInScanReduction): HostAssoc
+  ! CHECK: m (OmpReduction, OmpExplicit, OmpExclusiveScan, OmpInScanReduction): HostAssoc
   !$omp parallel do  reduction(inscan, +:m)
   do i=1,10
    !$omp scan exclusive(m)
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol01.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol01.f90
index a40a8563fde1..fbd9a0286c79 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol01.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol01.f90
@@ -21,8 +21,8 @@ program mm
  !REF: /md
  use :: md
  !DEF: /mm/c CommonBlockDetails
- !DEF: /mm/x ObjectEntity REAL(4)
- !DEF: /mm/y ObjectEntity REAL(4)
+ !DEF: /mm/x (InCommonBlock) ObjectEntity REAL(4)
+ !DEF: /mm/y (InCommonBlock) ObjectEntity REAL(4)
  common /c/x, y
  !REF: /mm/x
  !REF: /mm/y
@@ -47,22 +47,22 @@ program mm
 !$omp parallel do  private(a,t,/c/) shared(c)
  !DEF: /mm/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,10
-  !DEF: /mm/OtherConstruct1/a (OmpPrivate) HostAssoc REAL(4)
-  !DEF: /mm/OtherConstruct1/b HostAssoc INTEGER(4)
+  !DEF: /mm/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
+  !DEF: /mm/OtherConstruct1/b (OmpShared) HostAssoc INTEGER(4)
   !REF: /mm/OtherConstruct1/i
   a = a+b(i)
-  !DEF: /mm/OtherConstruct1/t (OmpPrivate) HostAssoc TYPE(myty)
+  !DEF: /mm/OtherConstruct1/t (OmpPrivate, OmpExplicit) HostAssoc TYPE(myty)
   !REF: /md/myty/a
   !REF: /mm/OtherConstruct1/i
   t%a = i
-  !DEF: /mm/OtherConstruct1/y (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /mm/OtherConstruct1/y (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   y = 0.
-  !DEF: /mm/OtherConstruct1/x (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /mm/OtherConstruct1/x (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   !REF: /mm/OtherConstruct1/a
   !REF: /mm/OtherConstruct1/i
   !REF: /mm/OtherConstruct1/y
   x = a+i+y
-  !DEF: /mm/OtherConstruct1/c (OmpShared) HostAssoc REAL(4)
+  !DEF: /mm/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
   c = 3.0
  end do
 end program
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol02.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol02.f90
index 31d9cb2e46ba..9007da042845 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol02.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol02.f90
@@ -11,13 +11,13 @@
   !DEF: /MainProgram1/c (Implicit) ObjectEntity REAL(4)
   c = 0
   !$omp parallel  private(a,b) shared(c,d)
-  !DEF: /MainProgram1/OtherConstruct1/a (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   a = 3.
-  !DEF: /MainProgram1/OtherConstruct1/b (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/b (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   b = 4
-  !DEF: /MainProgram1/OtherConstruct1/c (OmpShared) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/c (OmpShared, OmpExplicit) HostAssoc REAL(4)
   c = 5
-  !DEF: /MainProgram1/OtherConstruct1/d (OmpShared) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/d (OmpShared, OmpExplicit) HostAssoc REAL(4)
   d = 6
   !$omp end parallel
   !DEF: /MainProgram1/a (Implicit) ObjectEntity REAL(4)
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol03.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol03.f90
index 08defb40e56a..d67c1fdf333c 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol03.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol03.f90
@@ -7,14 +7,14 @@
   !DEF: /MainProgram1/b (Implicit) ObjectEntity REAL(4)
   b = 2
   !$omp parallel  private(a) shared(b)
-  !DEF: /MainProgram1/OtherConstruct1/a (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   a = 3.
-  !DEF: /MainProgram1/OtherConstruct1/b (OmpShared) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/b (OmpShared, OmpExplicit) HostAssoc REAL(4)
   b = 4
   !$omp parallel  private(b) shared(a)
-  !DEF: /MainProgram1/OtherConstruct1/OtherConstruct1/a (OmpShared) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/OtherConstruct1/a (OmpShared, OmpExplicit) HostAssoc REAL(4)
   a = 5.
-  !DEF: /MainProgram1/OtherConstruct1/OtherConstruct1/b (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /MainProgram1/OtherConstruct1/OtherConstruct1/b (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   b = 6
   !$omp end parallel
   !$omp end parallel
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol04.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol04.f90
index 808d1e0dd09b..834b16626637 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol04.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol04.f90
@@ -9,12 +9,12 @@
   !REF: /MainProgram1/a
   a = 3.14
   !$omp parallel  private(a)
-  !DEF: /MainProgram1/OtherConstruct1/a (OmpPrivate) HostAssoc REAL(8)
+  !DEF: /MainProgram1/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(8)
   a = 2.
   !$omp do  private(a)
   !DEF: /MainProgram1/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-     !DEF: /MainProgram1/OtherConstruct1/OtherConstruct1/a (OmpPrivate) HostAssoc REAL(8)
+     !DEF: /MainProgram1/OtherConstruct1/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(8)
      a = 1.
   end do
   !$omp end parallel
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol05.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol05.f90
index 1ad0c10a4013..fe01f15d20aa 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol05.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol05.f90
@@ -15,7 +15,7 @@ subroutine foo
     !DEF: /mm/foo/a ObjectEntity INTEGER(4)
     integer :: a = 3
     !$omp parallel
-    !DEF: /mm/foo/OtherConstruct1/a HostAssoc INTEGER(4)
+    !DEF: /mm/foo/OtherConstruct1/a (OmpShared) HostAssoc INTEGER(4)
     a = 1
     !DEF: /mm/i PUBLIC (Implicit, OmpThreadprivate) ObjectEntity INTEGER(4)
     !REF: /mm/foo/OtherConstruct1/a
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol06.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol06.f90
index 906264eb1264..daf3874b79af 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol06.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol06.f90
@@ -10,7 +10,7 @@
   !$omp parallel do  firstprivate(a) lastprivate(a)
   !DEF: /MainProgram1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do i=1,10
-     !DEF: /MainProgram1/OtherConstruct1/a (OmpFirstPrivate, OmpLastPrivate) HostAssoc REAL(4)
+     !DEF: /MainProgram1/OtherConstruct1/a (OmpFirstPrivate, OmpLastPrivate, OmpExplicit) HostAssoc REAL(4)
      a = 2.
   end do
 end program
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol07.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol07.f90
index a375942ebb1d..86b730541134 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol07.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol07.f90
@@ -21,9 +21,9 @@ subroutine function_call_in_region
   !DEF: /function_call_in_region/b ObjectEntity REAL(4)
   real :: b = 5.
   !$omp parallel  default(none) private(a) shared(b)
-  !DEF: /function_call_in_region/OtherConstruct1/a (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /function_call_in_region/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   !REF: /function_call_in_region/foo
-  !DEF: /function_call_in_region/OtherConstruct1/b (OmpShared) HostAssoc REAL(4)
+  !DEF: /function_call_in_region/OtherConstruct1/b (OmpShared, OmpExplicit) HostAssoc REAL(4)
   a = foo(b)
   !$omp end parallel
   !REF: /function_call_in_region/a
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol08.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol08.f90
index 80ae1c6d2242..545bccc86b06 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol08.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol08.f90
@@ -28,19 +28,19 @@ subroutine test_do
  !DEF: /test_do/k ObjectEntity INTEGER(4)
  integer i, j, k
 !$omp parallel
- !DEF: /test_do/OtherConstruct1/i HostAssoc INTEGER(4)
+ !DEF: /test_do/OtherConstruct1/i (OmpShared) HostAssoc INTEGER(4)
  i = 99
 !$omp do  collapse(2)
  !DEF: /test_do/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,5
   !DEF: /test_do/OtherConstruct1/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   do j=6,10
-   !DEF: /test_do/OtherConstruct1/a HostAssoc REAL(4)
+   !DEF: /test_do/OtherConstruct1/OtherConstruct1/a HostAssoc REAL(4)
    a(1,1,1) = 0.
    !DEF: /test_do/OtherConstruct1/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
    do k=11,15
-    !REF: /test_do/OtherConstruct1/a
-    !REF: /test_do/OtherConstruct1/k
+    !REF: /test_do/OtherConstruct1/OtherConstruct1/a
+    !DEF: /test_do/OtherConstruct1/OtherConstruct1/k HostAssoc INTEGER(4)
     !REF: /test_do/OtherConstruct1/OtherConstruct1/j
     !REF: /test_do/OtherConstruct1/OtherConstruct1/i
     a(k,j,i) = 1.
@@ -65,9 +65,9 @@ subroutine test_pardo
  do i=1,5
    !DEF: /test_pardo/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
     do j=6,10
-   !DEF: /test_pardo/OtherConstruct1/a HostAssoc REAL(4)
+   !DEF: /test_pardo/OtherConstruct1/a (OmpShared) HostAssoc REAL(4)
    a(1,1,1) = 0.
-   !DEF: /test_pardo/OtherConstruct1/k (OmpPrivate) HostAssoc INTEGER(4)
+   !DEF: /test_pardo/OtherConstruct1/k (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
    do k=11,15
     !REF: /test_pardo/OtherConstruct1/a
     !REF: /test_pardo/OtherConstruct1/k
@@ -91,7 +91,7 @@ subroutine test_taskloop
 !$omp taskloop  private(j)
  !DEF: /test_taskloop/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,5
-  !DEF: /test_taskloop/OtherConstruct1/j (OmpPrivate) HostAssoc INTEGER(4)
+  !DEF: /test_taskloop/OtherConstruct1/j (OmpPrivate, OmpExplicit) HostAssoc INTEGER(4)
   !REF: /test_taskloop/OtherConstruct1/i
   do j=1,i
    !DEF: /test_taskloop/OtherConstruct1/a (OmpFirstPrivate, OmpImplicit) HostAssoc REAL(4)
@@ -139,15 +139,15 @@ subroutine dotprod (b, c, n, block_size, num_teams, block_threads)
  do i0=1,n,block_size
 !$omp parallel do  reduction(+: sum)
   !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
-  !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/i0 HostAssoc INTEGER(4)
+  !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/i0 (OmpShared) HostAssoc INTEGER(4)
   !DEF: /dotprod/min ELEMENTAL, INTRINSIC, PURE (Function) ProcEntity
-  !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/block_size HostAssoc INTEGER(4)
-  !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/n HostAssoc INTEGER(4)
+  !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/block_size (OmpShared) HostAssoc INTEGER(4)
+  !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/n (OmpShared) HostAssoc INTEGER(4)
   do i=i0,min(i0+block_size, n)
-   !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/sum (OmpReduction) HostAssoc REAL(4)
-   !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/b HostAssoc REAL(4)
+   !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/sum (OmpReduction, OmpExplicit) HostAssoc REAL(4)
+   !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/b (OmpShared) HostAssoc REAL(4)
    !REF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/i
-   !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/c HostAssoc REAL(4)
+   !DEF: /dotprod/OtherConstruct1/OtherConstruct1/OtherConstruct1/OtherConstruct1/c (OmpShared) HostAssoc REAL(4)
    sum = sum+b(i)*c(i)
   end do
  end do
@@ -175,7 +175,7 @@ subroutine test_simd
   do j=6,10
    !DEF: /test_simd/OtherConstruct1/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
    do k=11,15
-    !DEF: /test_simd/OtherConstruct1/a HostAssoc REAL(4)
+    !DEF: /test_simd/OtherConstruct1/a (OmpShared) HostAssoc REAL(4)
     !REF: /test_simd/OtherConstruct1/k
     !REF: /test_simd/OtherConstruct1/j
     !REF: /test_simd/OtherConstruct1/i
@@ -202,7 +202,7 @@ subroutine test_simd_multi
   do j=6,10
    !DEF: /test_simd_multi/OtherConstruct1/k (OmpLastPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
    do k=11,15
-    !DEF: /test_simd_multi/OtherConstruct1/a HostAssoc REAL(4)
+    !DEF: /test_simd_multi/OtherConstruct1/a (OmpShared) HostAssoc REAL(4)
     !REF: /test_simd_multi/OtherConstruct1/k
     !REF: /test_simd_multi/OtherConstruct1/j
     !REF: /test_simd_multi/OtherConstruct1/i
@@ -224,11 +224,11 @@ subroutine test_seq_loop
   !REF: /test_seq_loop/j
   j = -1
   !$omp parallel
-  !DEF: /test_seq_loop/OtherConstruct1/i HostAssoc INTEGER(4)
-  !DEF: /test_seq_loop/OtherConstruct1/j HostAssoc INTEGER(4)
+  !DEF: /test_seq_loop/OtherConstruct1/i (OmpShared) HostAssoc INTEGER(4)
+  !DEF: /test_seq_loop/OtherConstruct1/j (OmpShared) HostAssoc INTEGER(4)
   print *, i, j
   !$omp parallel
-  !DEF: /test_seq_loop/OtherConstruct1/OtherConstruct1/i HostAssoc INTEGER(4)
+  !DEF: /test_seq_loop/OtherConstruct1/OtherConstruct1/i (OmpShared) HostAssoc INTEGER(4)
   !DEF: /test_seq_loop/OtherConstruct1/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
   print *, i, j
   !$omp do
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/symbol09.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/symbol09.f90
index a375942ebb1d..86b730541134 100644
--- a/external/llvm-project/flang/test/Semantics/OpenMP/symbol09.f90
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/symbol09.f90
@@ -21,9 +21,9 @@ subroutine function_call_in_region
   !DEF: /function_call_in_region/b ObjectEntity REAL(4)
   real :: b = 5.
   !$omp parallel  default(none) private(a) shared(b)
-  !DEF: /function_call_in_region/OtherConstruct1/a (OmpPrivate) HostAssoc REAL(4)
+  !DEF: /function_call_in_region/OtherConstruct1/a (OmpPrivate, OmpExplicit) HostAssoc REAL(4)
   !REF: /function_call_in_region/foo
-  !DEF: /function_call_in_region/OtherConstruct1/b (OmpShared) HostAssoc REAL(4)
+  !DEF: /function_call_in_region/OtherConstruct1/b (OmpShared, OmpExplicit) HostAssoc REAL(4)
   a = foo(b)
   !$omp end parallel
   !REF: /function_call_in_region/a
diff --git a/external/llvm-project/flang/test/Semantics/OpenMP/task-depend.f90 b/external/llvm-project/flang/test/Semantics/OpenMP/task-depend.f90
new file mode 100644
index 000000000000..69a37e014bde
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/OpenMP/task-depend.f90
@@ -0,0 +1,8 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+
+program test
+! ERROR: A DEPEND clause on a TASK construct must have a valid task dependence type
+!$omp task depend(ii)
+!$omp end task
+end
+
diff --git a/external/llvm-project/flang/test/Semantics/bug141807.f90 b/external/llvm-project/flang/test/Semantics/bug141807.f90
new file mode 100644
index 000000000000..48539f19927c
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/bug141807.f90
@@ -0,0 +1,32 @@
+!RUN: %flang_fc1 -fdebug-unparse %s | FileCheck %s
+!Ensure that non-elemental specific takes precedence over elemental
+!defined assignment, even with non-default PASS argument.
+module m
+  type base
+    integer :: n = -999
+   contains
+    procedure, pass(from) :: array_assign_scalar
+    procedure :: elemental_assign
+    generic :: assignment(=) => array_assign_scalar, elemental_assign
+  end type
+ contains
+  subroutine array_assign_scalar(to, from)
+    class(base), intent(out) :: to(:)
+    class(base), intent(in) :: from
+    to%n = from%n
+  end
+  impure elemental subroutine elemental_assign(to, from)
+    class(base), intent(out) :: to
+    class(base), intent(in) :: from
+    to%n = from%n
+  end
+end
+
+use m
+type(base) :: array(1), scalar
+scalar%n = 1
+!CHECK: CALL array_assign_scalar(array,(scalar))
+array = scalar
+!CHECK: CALL elemental_assign(array,[base::scalar])
+array = [scalar]
+end
diff --git a/external/llvm-project/flang/test/Semantics/cuf21.cuf b/external/llvm-project/flang/test/Semantics/cuf21.cuf
index b8b99a8d1d9b..db32f1dbd0e7 100644
--- a/external/llvm-project/flang/test/Semantics/cuf21.cuf
+++ b/external/llvm-project/flang/test/Semantics/cuf21.cuf
@@ -9,19 +9,25 @@ module mlocModule
   end interface maxlocUpdate
 contains
 
-  attributes(global) subroutine maxlocPartialMaskR_32F1D()
+  attributes(global) subroutine maxlocPartialMaskR_32F1D(back)
     implicit none
+    logical, intent(in), value :: back
     real(4) :: mval
-
-    call maxlocUpdate(mval)
-
+  block
+    integer(8) :: xloc
+    call maxlocUpdate(mval, xloc, back)
+  end block
   end subroutine maxlocPartialMaskR_32F1D
 
-  attributes(device) subroutine maxlocUpdateR_32F(mval)
+  attributes(device) subroutine maxlocUpdateR_32F(mval, xloc, back)
     real(4) :: mval
+    integer(8) :: xloc
+    logical :: back
   end subroutine maxlocUpdateR_32F
 
-  attributes(device) subroutine maxlocUpdateR_64F(mval)
+  attributes(device) subroutine maxlocUpdateR_64F(mval, xloc, back)
     real(8) :: mval
+    integer(8) :: xloc
+    logical :: back
   end subroutine maxlocUpdateR_64F
 end module
diff --git a/external/llvm-project/flang/test/Semantics/cuf22.cuf b/external/llvm-project/flang/test/Semantics/cuf22.cuf
new file mode 100644
index 000000000000..36e0f0b2502d
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/cuf22.cuf
@@ -0,0 +1,8 @@
+! RUN: not bbc -fcuda -fcuda-disable-warp-function %s -o - 2>&1 | FileCheck %s
+
+attributes(device) subroutine testMatch()
+  integer :: a, ipred, mask, v32
+  a = match_all_sync(mask, v32, ipred)
+end subroutine
+
+! CHECK:  warp match function disabled
diff --git a/external/llvm-project/flang/test/Semantics/deferred01.f90 b/external/llvm-project/flang/test/Semantics/deferred01.f90
index 87818c10bd39..ce406a72b8fa 100644
--- a/external/llvm-project/flang/test/Semantics/deferred01.f90
+++ b/external/llvm-project/flang/test/Semantics/deferred01.f90
@@ -1,6 +1,7 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic -Werror
 ! Deferred TBPs must be overridden, but when they are private, those
-! overrides must appear in the same module.
+! overrides are required to appear in the same module.  We allow overrides
+! elsewhere as an extension.
 module m1
   type, abstract :: absBase
    contains
@@ -18,7 +19,7 @@ module m2
   use m1
   type, extends(absBase) :: ext
    contains
-    !ERROR: Override of PRIVATE DEFERRED 'deferredtbp' must appear in its module
+    !WARNING: Override of PRIVATE DEFERRED 'deferredtbp' should appear in its module
     procedure :: deferredTbp => implTbp
   end type
  contains
diff --git a/external/llvm-project/flang/test/Semantics/modfile76.F90 b/external/llvm-project/flang/test/Semantics/modfile76.F90
new file mode 100644
index 000000000000..50ee9a088e11
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/modfile76.F90
@@ -0,0 +1,24 @@
+!RUN: %flang_fc1 -fsyntax-only -fhermetic-module-files -DSTEP=1 %s
+!RUN: %flang_fc1 -fsyntax-only %s
+
+! Tests that a BIND(C) variable in a module A captured in a hermetic module
+! file USE'd in a module B is not creating bogus complaints about BIND(C) name
+! conflict when both module A and B are later accessed.
+
+#if STEP == 1
+module modfile75a
+  integer, bind(c) :: x
+end
+
+module modfile75b
+  use modfile75a ! capture hermetically
+end
+
+#else
+subroutine test
+  use modfile75a
+  use modfile75b
+  implicit none
+  print *, x
+end subroutine
+#endif
diff --git a/external/llvm-project/flang/test/Semantics/modfile77.F90 b/external/llvm-project/flang/test/Semantics/modfile77.F90
new file mode 100644
index 000000000000..a82904ebbcc2
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/modfile77.F90
@@ -0,0 +1,37 @@
+!RUN: %flang -c -fhermetic-module-files -DWHICH=1 %s && %flang -c -fhermetic-module-files -DWHICH=2 %s && %flang -c -fhermetic-module-files %s && cat modfile77c.mod | FileCheck %s
+
+#if WHICH == 1
+module modfile77a
+  interface gen
+    procedure proc
+  end interface
+ contains
+  subroutine proc
+    print *, 'ok'
+  end
+end
+#elif WHICH == 2
+module modfile77b
+  use modfile77a
+end
+#else
+module modfile77c
+  use modfile77a
+  use modfile77b
+end
+#endif
+
+!CHECK: module modfile77c
+!CHECK: use modfile77a,only:proc
+!CHECK: use modfile77a,only:gen
+!CHECK: interface gen
+!CHECK: end interface
+!CHECK: end
+!CHECK: module modfile77a
+!CHECK: interface gen
+!CHECK: procedure::proc
+!CHECK: end interface
+!CHECK: contains
+!CHECK: subroutine proc()
+!CHECK: end
+!CHECK: end
diff --git a/external/llvm-project/flang/test/Semantics/modfile78.F90 b/external/llvm-project/flang/test/Semantics/modfile78.F90
new file mode 100644
index 000000000000..cb3eccd9a410
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/modfile78.F90
@@ -0,0 +1,33 @@
+!RUN: %flang -c -fhermetic-module-files -DWHICH=1 %s && %flang -c -fhermetic-module-files -DWHICH=2 %s && %flang -c -fhermetic-module-files %s && cat modfile78c.mod | FileCheck %s
+
+#if WHICH == 1
+module modfile78a
+  integer :: global_variable = 0
+end
+#elif WHICH == 2
+module modfile78b
+  use modfile78a
+ contains
+  subroutine test
+  end
+end
+#else
+module modfile78c
+  use modfile78a
+  use modfile78b
+end
+#endif
+
+!CHECK: module modfile78c
+!CHECK: use modfile78a,only:global_variable
+!CHECK: use modfile78b,only:test
+!CHECK: end
+!CHECK: module modfile78a
+!CHECK: integer(4)::global_variable
+!CHECK: end
+!CHECK: module modfile78b
+!CHECK: use modfile78a,only:global_variable
+!CHECK: contains
+!CHECK: subroutine test()
+!CHECK: end
+!CHECK: end
diff --git a/external/llvm-project/flang/test/Semantics/offsets03.f90 b/external/llvm-project/flang/test/Semantics/offsets03.f90
index c8c1abebb5d3..75bc43d9f4cc 100644
--- a/external/llvm-project/flang/test/Semantics/offsets03.f90
+++ b/external/llvm-project/flang/test/Semantics/offsets03.f90
@@ -30,10 +30,10 @@ subroutine mc         !CHECK: Subprogram scope: mc size=12 alignment=1
 ! Common block: objects are in order from COMMON statement and not part of module
 module md                   !CHECK: Module scope: md size=1 alignment=1
   integer(1) :: i
-  integer(2) :: d1          !CHECK: d1, PUBLIC size=2 offset=8:
-  integer(4) :: d2          !CHECK: d2, PUBLIC size=4 offset=4:
-  integer(1) :: d3          !CHECK: d3, PUBLIC size=1 offset=0:
-  real(2) :: d4             !CHECK: d4, PUBLIC size=2 offset=0:
+  integer(2) :: d1          !CHECK: d1, PUBLIC (InCommonBlock) size=2 offset=8:
+  integer(4) :: d2          !CHECK: d2, PUBLIC (InCommonBlock) size=4 offset=4:
+  integer(1) :: d3          !CHECK: d3, PUBLIC (InCommonBlock) size=1 offset=0:
+  real(2) :: d4             !CHECK: d4, PUBLIC (InCommonBlock) size=2 offset=0:
   common /common1/ d3,d2,d1 !CHECK: common1 size=10 offset=0: CommonBlockDetails alignment=4:
   common /common2/ d4       !CHECK: common2 size=2 offset=0: CommonBlockDetails alignment=2:
 end
@@ -71,7 +71,7 @@ module me
 subroutine host1
  contains
   subroutine internal
-    common /b/ x(4)  ! CHECK: x (Implicit) size=16 offset=0: ObjectEntity type: REAL(4) shape: 1_8:4_8
+    common /b/ x(4)  ! CHECK: x (Implicit, InCommonBlock) size=16 offset=0: ObjectEntity type: REAL(4) shape: 1_8:4_8
     equivalence(x,y) ! CHECK: y (Implicit) size=4 offset=0: ObjectEntity type: REAL(4)
   end
 end
diff --git a/external/llvm-project/flang/test/Semantics/resolve121.f90 b/external/llvm-project/flang/test/Semantics/resolve121.f90
index d84bc53a50f7..fa54a0924ff7 100644
--- a/external/llvm-project/flang/test/Semantics/resolve121.f90
+++ b/external/llvm-project/flang/test/Semantics/resolve121.f90
@@ -25,7 +25,7 @@ subroutine test3()
   ! CHECK-LABEL: Subprogram scope: test3
   ! CHECK: i1, SAVE size=4 offset=0: ObjectEntity type: INTEGER(4)
   ! CHECK: j1, SAVE size=4 offset=0: ObjectEntity type: INTEGER(4)
-  ! CHECK: k1, SAVE size=4 offset=0: ObjectEntity type: INTEGER(4)
+  ! CHECK: k1, SAVE (InCommonBlock) size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer :: i1
   integer :: j1, k1
   common /blk/ k1
@@ -37,7 +37,7 @@ subroutine test4()
   ! CHECK-LABEL: Subprogram scope: test4
   ! CHECK: i1, SAVE size=4 offset=0: ObjectEntity type: INTEGER(4) init:1_4
   ! CHECK: j1, SAVE size=4 offset=0: ObjectEntity type: INTEGER(4)
-  ! CHECK: k1, SAVE size=4 offset=0: ObjectEntity type: INTEGER(4)
+  ! CHECK: k1, SAVE (InCommonBlock) size=4 offset=0: ObjectEntity type: INTEGER(4)
   integer :: i1 = 1
   integer :: j1, k1
   common /blk/ k1
diff --git a/external/llvm-project/flang/test/Semantics/symbol33.f90 b/external/llvm-project/flang/test/Semantics/symbol33.f90
index fbb5321b8854..ccc52b6ef5e6 100644
--- a/external/llvm-project/flang/test/Semantics/symbol33.f90
+++ b/external/llvm-project/flang/test/Semantics/symbol33.f90
@@ -3,7 +3,7 @@
 ! array element reference still applies implicit typing, &c.
 !DEF: /subr (Subroutine) Subprogram
 subroutine subr
-  !DEF: /subr/moo (Implicit) ObjectEntity INTEGER(4)
+  !DEF: /subr/moo (Implicit, InCommonBlock) ObjectEntity INTEGER(4)
   common //moo(1)
   !DEF: /subr/a ObjectEntity REAL(4)
   !REF: /subr/moo
diff --git a/external/llvm-project/flang/test/Semantics/typeinfo01.f90 b/external/llvm-project/flang/test/Semantics/typeinfo01.f90
index c1427f28753c..d228cd2a84ca 100644
--- a/external/llvm-project/flang/test/Semantics/typeinfo01.f90
+++ b/external/llvm-project/flang/test/Semantics/typeinfo01.f90
@@ -73,7 +73,7 @@ module m06
   end type
   type, extends(t) :: t2
    contains
-    procedure :: s1 => s2 ! override
+    procedure :: s1 => s2
   end type
  contains
   subroutine s1(x, y)
@@ -86,8 +86,37 @@ subroutine s2(x, y)
   end subroutine
 !CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
 !CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1)
-!CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=NULL(),specialbitset=0_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1)
+!CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1)
 !CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
+!CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
+!CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
+end module
+
+module m06a
+  type :: t
+   contains
+    procedure, pass(y) :: s1
+    generic :: assignment(=) => s1
+  end type
+  type, extends(t) :: t2
+   contains
+    procedure, pass(y) :: s1 => s2
+  end type
+ contains
+  subroutine s1(x, y)
+    class(t), intent(out) :: x
+    class(t), intent(in) :: y
+  end subroutine
+  subroutine s2(x, y)
+    class(t), intent(out) :: x
+    class(t2), intent(in) :: y
+  end subroutine
+!CHECK: .c.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=6_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
+!CHECK: .dt.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t,specialbitset=2_4,hasparent=0_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1)
+!CHECK: .dt.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=.s.t2,specialbitset=2_4,hasparent=1_1,noinitializationneeded=1_1,nodestructionneeded=1_1,nofinalizationneeded=1_1)
+!CHECK: .s.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s1)]
+!CHECK: .s.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=1_1,isargdescriptorset=3_1,istypebound=1_1,isargcontiguousset=0_1,proc=s2)]
 !CHECK: .v.t, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
 !CHECK: .v.t2, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
 end module
diff --git a/external/llvm-project/flang/test/Semantics/typeinfo13.f90 b/external/llvm-project/flang/test/Semantics/typeinfo13.f90
new file mode 100644
index 000000000000..cf4abf9e3818
--- /dev/null
+++ b/external/llvm-project/flang/test/Semantics/typeinfo13.f90
@@ -0,0 +1,26 @@
+!RUN: %flang_fc1 -fdebug-dump-symbols %s | FileCheck %s
+!Ensure ASSIGNMENT(=) overrides are applied to the special procedures table.
+module m
+  type base
+   contains
+    procedure :: baseAssign
+    generic :: assignment(=) => baseAssign
+  end type
+  type, extends(base) :: child
+   contains
+    procedure :: override
+    generic :: assignment(=) => override
+  end type
+ contains
+  impure elemental subroutine baseAssign(to, from)
+    class(base), intent(out) :: to
+    type(base), intent(in) :: from
+  end
+  impure elemental subroutine override(to, from)
+    class(child), intent(out) :: to
+    type(child), intent(in) :: from
+  end
+end
+
+!CHECK: .s.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=2_1,isargdescriptorset=1_1,istypebound=1_1,isargcontiguousset=0_1,proc=override)]
+!CHECK: .v.child, SAVE, TARGET (CompilerCreated, ReadOnly): ObjectEntity type: TYPE(binding) shape: 0_8:1_8 init:[binding::binding(proc=baseassign,name=.n.baseassign),binding(proc=override,name=.n.override)]
diff --git a/external/llvm-project/flang/test/Transforms/DoConcurrent/locality_specifiers_init_dealloc.mlir b/external/llvm-project/flang/test/Transforms/DoConcurrent/locality_specifiers_init_dealloc.mlir
new file mode 100644
index 000000000000..1659c7bdf6d3
--- /dev/null
+++ b/external/llvm-project/flang/test/Transforms/DoConcurrent/locality_specifiers_init_dealloc.mlir
@@ -0,0 +1,51 @@
+// Tests mapping `local` locality specifier to `private` clauses for non-empty
+// `init` and `dealloc` regions.
+
+// RUN: fir-opt --omp-do-concurrent-conversion="map-to=host" %s | FileCheck %s
+
+func.func @my_allocator(%arg0: !fir.ref<!fir.box<!fir.array<10xf32>>>, %arg1: !fir.ref<!fir.box<!fir.array<10xf32>>>) {
+  return
+}
+
+func.func @my_deallocator(%arg0: !fir.ref<!fir.box<!fir.array<10xf32>>>) {
+  return
+}
+
+fir.local {type = local} @_QFlocal_assocEaa_private_box_10xf32 : !fir.box<!fir.array<10xf32>> init {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.array<10xf32>>>, %arg1: !fir.ref<!fir.box<!fir.array<10xf32>>>):
+  fir.call @my_allocator(%arg0, %arg1) : (!fir.ref<!fir.box<!fir.array<10xf32>>>, !fir.ref<!fir.box<!fir.array<10xf32>>>) -> ()
+  fir.yield(%arg1 : !fir.ref<!fir.box<!fir.array<10xf32>>>)
+} dealloc {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.array<10xf32>>>):
+  fir.call @my_deallocator(%arg0) : (!fir.ref<!fir.box<!fir.array<10xf32>>>) -> ()
+  fir.yield
+}
+
+func.func @_QPlocal_assoc() {
+  %0 = fir.alloca !fir.box<!fir.array<10xf32>>
+  %c1 = arith.constant 1 : index
+
+  fir.do_concurrent {
+    %9 = fir.alloca i32 {bindc_name = "i"}
+    %10:2 = hlfir.declare %9 {uniq_name = "_QFlocal_assocEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) local(@_QFlocal_assocEaa_private_box_10xf32 %0 -> %arg1 : !fir.ref<!fir.box<!fir.array<10xf32>>>) {
+      %11 = fir.convert %arg0 : (index) -> i32
+      fir.store %11 to %10#0 : !fir.ref<i32>
+    }
+  }
+
+  return
+}
+
+// CHECK:      omp.private {type = private} @[[PRIVATIZER:.*]] : !fir.box<!fir.array<10xf32>> init {
+// CHECK-NEXT: ^bb0(%[[ORIG_ARG:.*]]: !{{.*}}, %[[PRIV_ARG:.*]]: !{{.*}}):
+// CHECK-NEXT:   fir.call @my_allocator(%[[ORIG_ARG]], %[[PRIV_ARG]]) : ({{.*}}) -> ()
+// CHECK-NEXT:   omp.yield(%[[PRIV_ARG]] : {{.*}})
+// CHECK-NEXT: } dealloc {
+// CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: !{{.*}}):
+// CHECK-NEXT:   fir.call @my_deallocator(%[[PRIV_ARG]]) : ({{.*}}) -> ()
+// CHECK-NEXT:   omp.yield
+// CHECK-NEXT: }
+
+// CHECK: %[[LOCAL_ALLOC:.*]] = fir.alloca !fir.box<!fir.array<10xf32>>
+// CHECK: omp.wsloop private(@[[PRIVATIZER]] %[[LOCAL_ALLOC]] -> %{{.*}} : !{{.*}})
diff --git a/external/llvm-project/flang/test/Transforms/DoConcurrent/locality_specifiers_simple.mlir b/external/llvm-project/flang/test/Transforms/DoConcurrent/locality_specifiers_simple.mlir
new file mode 100644
index 000000000000..160c1df04068
--- /dev/null
+++ b/external/llvm-project/flang/test/Transforms/DoConcurrent/locality_specifiers_simple.mlir
@@ -0,0 +1,48 @@
+// Tests mapping `local` locality specifier to `private` clauses for a simple
+// case (not `init` or `copy` regions).
+
+// RUN: fir-opt --omp-do-concurrent-conversion="map-to=host" %s | FileCheck %s
+
+fir.local {type = local} @_QFlocal_spec_translationElocal_var_private_f32 : f32
+
+func.func @_QPlocal_spec_translation() {
+  %3 = fir.alloca f32 {bindc_name = "local_var", uniq_name = "_QFlocal_spec_translationElocal_var"}
+  %4:2 = hlfir.declare %3 {uniq_name = "_QFlocal_spec_translationElocal_var"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+
+  %c4_i32 = arith.constant 4 : index
+  %c11_i32 = arith.constant 11 : index
+  %c1 = arith.constant 1 : index
+
+  fir.do_concurrent {
+    %7 = fir.alloca i32 {bindc_name = "i"}
+    %8:2 = hlfir.declare %7 {uniq_name = "_QFlocal_spec_translationEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+    fir.do_concurrent.loop (%arg0) = (%c4_i32) to (%c11_i32) step (%c1)
+      local(@_QFlocal_spec_translationElocal_var_private_f32 %4#0 -> %arg1 : !fir.ref<f32>) {
+      %9 = fir.convert %arg0 : (index) -> i32
+      fir.store %9 to %8#0 : !fir.ref<i32>
+
+      %10:2 = hlfir.declare %arg1 {uniq_name = "_QFlocal_spec_translationElocal_var"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+      %cst = arith.constant 4.200000e+01 : f32
+      hlfir.assign %cst to %10#0 : f32, !fir.ref<f32>
+    }
+  }
+  return
+}
+
+// CHECK: omp.private {type = private} @[[PRIVATIZER:.*local_spec_translationElocal_var.*.omp]] : f32
+
+// CHECK: func.func @_QPlocal_spec_translation
+// CHECK:   %[[LOCAL_VAR:.*]] = fir.alloca f32 {bindc_name = "local_var", {{.*}}}
+// CHECK:   %[[LOCAL_VAR_DECL:.*]]:2 = hlfir.declare %[[LOCAL_VAR]]
+// CHECK:   omp.parallel {
+// CHECK:     omp.wsloop private(@[[PRIVATIZER]] %[[LOCAL_VAR_DECL]]#0 -> %[[LOCAL_ARG:.*]] : !fir.ref<f32>) {
+// CHECK:       omp.loop_nest {{.*}} {
+// CHECK:       %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[LOCAL_ARG]]
+// CHECK:       %[[C42:.*]] = arith.constant
+// CHECK:       hlfir.assign %[[C42]] to %[[PRIV_DECL]]#0
+// CHECK:       omp.yield
+// CHECK:     }
+// CHECK:   }
+// CHECK:   omp.terminator
+// CHECK: }
diff --git a/external/llvm-project/flang/test/Transforms/do-concurrent-localizer-dealloc-region.fir b/external/llvm-project/flang/test/Transforms/do-concurrent-localizer-dealloc-region.fir
new file mode 100644
index 000000000000..b59ffdfb34ad
--- /dev/null
+++ b/external/llvm-project/flang/test/Transforms/do-concurrent-localizer-dealloc-region.fir
@@ -0,0 +1,126 @@
+// Tests converting `fir.local` ops that have `dealloc` regions.
+
+// RUN: fir-opt --split-input-file --simplify-fir-operations %s | FileCheck %s
+
+fir.local {type = local} @_QFlocalizer_with_dealloc_regionEa_private_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.array<?xi32>>>, %arg1: !fir.ref<!fir.box<!fir.array<?xi32>>>):
+  %c0 = arith.constant 0 : index
+  %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
+  %1:3 = fir.box_dims %0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %2 = fir.shape %1#1 : (index) -> !fir.shape<1>
+  %3 = fir.allocmem !fir.array<?xi32>, %1#1 {bindc_name = ".tmp", uniq_name = ""}
+  %4 = fir.declare %3(%2) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.heap<!fir.array<?xi32>>
+  %5 = fir.embox %4(%2) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+  %6 = fir.shape_shift %1#0, %1#1 : (index, index) -> !fir.shapeshift<1>
+  %7 = fir.rebox %5(%6) : (!fir.box<!fir.array<?xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<?xi32>>
+  fir.store %7 to %arg1 : !fir.ref<!fir.box<!fir.array<?xi32>>>
+  fir.yield(%arg1 : !fir.ref<!fir.box<!fir.array<?xi32>>>)
+} dealloc {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.array<?xi32>>>):
+  %c0_i64 = arith.constant 0 : i64
+  %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
+  %1 = fir.box_addr %0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+  %2 = fir.convert %1 : (!fir.ref<!fir.array<?xi32>>) -> i64
+  %3 = arith.cmpi ne, %2, %c0_i64 : i64
+  fir.if %3 {
+    %4 = fir.convert %1 : (!fir.ref<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>>
+    fir.freemem %4 : !fir.heap<!fir.array<?xi32>>
+  }
+  fir.yield
+}
+
+func.func @_QPlocalizer_with_dealloc_region(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}) {
+  %c42_i32 = arith.constant 42 : i32
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.alloca !fir.box<!fir.array<?xi32>>
+  %1 = fir.dummy_scope : !fir.dscope
+  %2 = fir.declare %arg0 dummy_scope %1 {uniq_name = "_QFlocalizer_with_dealloc_regionEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+  %3 = fir.load %2 : !fir.ref<i32>
+  %4 = fir.convert %3 : (i32) -> index
+  %5 = arith.cmpi sgt, %4, %c0 : index
+  %6 = arith.select %5, %4, %c0 : index
+  %7 = fir.alloca !fir.array<?xi32>, %6 {bindc_name = "a", uniq_name = "_QFlocalizer_with_dealloc_regionEa"}
+  %8 = fir.shape %6 : (index) -> !fir.shape<1>
+  %9 = fir.declare %7(%8) {uniq_name = "_QFlocalizer_with_dealloc_regionEa"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.ref<!fir.array<?xi32>>
+  %10 = fir.embox %9(%8) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+  fir.store %10 to %0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
+  fir.do_concurrent {
+    %11 = fir.alloca i32 {bindc_name = "i"}
+    %12 = fir.declare %11 {uniq_name = "_QFlocalizer_with_dealloc_regionEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    fir.do_concurrent.loop (%arg1) = (%c1) to (%4) step (%c1) local(@_QFlocalizer_with_dealloc_regionEa_private_box_Uxi32 %0 -> %arg2 : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
+      %13 = fir.convert %arg1 : (index) -> i32
+      fir.store %13 to %12 : !fir.ref<i32>
+      %14 = fir.declare %arg2 {uniq_name = "_QFlocalizer_with_dealloc_regionEa"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+      %15 = fir.load %14 : !fir.ref<!fir.box<!fir.array<?xi32>>>
+      %16 = fir.load %12 : !fir.ref<i32>
+      %17 = fir.convert %16 : (i32) -> i64
+      %18:3 = fir.box_dims %15, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+      %19 = fir.shift %18#0 : (index) -> !fir.shift<1>
+      %20 = fir.array_coor %15(%19) %17 : (!fir.box<!fir.array<?xi32>>, !fir.shift<1>, i64) -> !fir.ref<i32>
+      fir.store %c42_i32 to %20 : !fir.ref<i32>
+    }
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPlocalizer_with_dealloc_region(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 42 : i32
+// CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "i"}
+// CHECK:           %[[VAL_5:.*]] = fir.declare %[[VAL_4]] {uniq_name = "_QFlocalizer_with_dealloc_regionEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]] = fir.declare %[[ARG0]] dummy_scope %[[VAL_7]] {uniq_name = "_QFlocalizer_with_dealloc_regionEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+// CHECK:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_10]], %[[VAL_1]] : index
+// CHECK:           %[[VAL_12:.*]] = arith.select %[[VAL_11]], %[[VAL_10]], %[[VAL_1]] : index
+// CHECK:           %[[VAL_13:.*]] = fir.alloca !fir.array<?xi32>, %[[VAL_12]] {bindc_name = "a", uniq_name = "_QFlocalizer_with_dealloc_regionEa"}
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_12]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_15:.*]] = fir.declare %[[VAL_13]](%[[VAL_14]]) {uniq_name = "_QFlocalizer_with_dealloc_regionEa"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.ref<!fir.array<?xi32>>
+// CHECK:           %[[VAL_16:.*]] = fir.embox %[[VAL_15]](%[[VAL_14]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           fir.store %[[VAL_16]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK:           fir.do_loop %[[VAL_17:.*]] = %[[VAL_2]] to %[[VAL_10]] step %[[VAL_2]] unordered {
+
+// Local allocation
+// CHECK:             %[[VAL_18:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+
+// `init` region body
+// CHECK:             %[[VAL_19:.*]] = fir.load %[[VAL_6]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK:             %[[VAL_20:.*]]:3 = fir.box_dims %[[VAL_19]], %[[VAL_1]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_21:.*]] = fir.shape %[[VAL_20]]#1 : (index) -> !fir.shape<1>
+// CHECK:             %[[VAL_22:.*]] = fir.allocmem !fir.array<?xi32>, %[[VAL_20]]#1 {bindc_name = ".tmp", uniq_name = ""}
+// CHECK:             %[[VAL_23:.*]] = fir.declare %[[VAL_22]](%[[VAL_21]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.heap<!fir.array<?xi32>>
+// CHECK:             %[[VAL_24:.*]] = fir.embox %[[VAL_23]](%[[VAL_21]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:             %[[VAL_25:.*]] = fir.shape_shift %[[VAL_20]]#0, %[[VAL_20]]#1 : (index, index) -> !fir.shapeshift<1>
+// CHECK:             %[[VAL_26:.*]] = fir.rebox %[[VAL_24]](%[[VAL_25]]) : (!fir.box<!fir.array<?xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:             fir.store %[[VAL_26]] to %[[VAL_18]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+
+// Loop body
+// CHECK:             %[[VAL_27:.*]] = fir.convert %[[VAL_17]] : (index) -> i32
+// CHECK:             fir.store %[[VAL_27]] to %[[VAL_5]] : !fir.ref<i32>
+// CHECK:             %[[VAL_28:.*]] = fir.declare %[[VAL_18]] {uniq_name = "_QFlocalizer_with_dealloc_regionEa"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK:             %[[VAL_29:.*]] = fir.load %[[VAL_28]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK:             %[[VAL_30:.*]] = fir.load %[[VAL_5]] : !fir.ref<i32>
+// CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (i32) -> i64
+// CHECK:             %[[VAL_32:.*]]:3 = fir.box_dims %[[VAL_29]], %[[VAL_1]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_33:.*]] = fir.shift %[[VAL_32]]#0 : (index) -> !fir.shift<1>
+// CHECK:             %[[VAL_34:.*]] = fir.array_coor %[[VAL_29]](%[[VAL_33]]) %[[VAL_31]] : (!fir.box<!fir.array<?xi32>>, !fir.shift<1>, i64) -> !fir.ref<i32>
+// CHECK:             fir.store %[[VAL_3]] to %[[VAL_34]] : !fir.ref<i32>
+
+// `dealloc` region
+// CHECK:             %[[VAL_35:.*]] = fir.load %[[VAL_18]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK:             %[[VAL_36:.*]] = fir.box_addr %[[VAL_35]] : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+// CHECK:             %[[VAL_37:.*]] = fir.convert %[[VAL_36]] : (!fir.ref<!fir.array<?xi32>>) -> i64
+// CHECK:             %[[VAL_38:.*]] = arith.cmpi ne, %[[VAL_37]], %[[VAL_0]] : i64
+// CHECK:             fir.if %[[VAL_38]] {
+// CHECK:               %[[VAL_39:.*]] = fir.convert %[[VAL_36]] : (!fir.ref<!fir.array<?xi32>>) -> !fir.heap<!fir.array<?xi32>>
+// CHECK:               fir.freemem %[[VAL_39]] : !fir.heap<!fir.array<?xi32>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
diff --git a/external/llvm-project/flang/test/Transforms/do-concurrent-localizer-init-region.fir b/external/llvm-project/flang/test/Transforms/do-concurrent-localizer-init-region.fir
new file mode 100644
index 000000000000..ebb56aec278f
--- /dev/null
+++ b/external/llvm-project/flang/test/Transforms/do-concurrent-localizer-init-region.fir
@@ -0,0 +1,102 @@
+// Tests converting `fir.local` ops that have `init` regions.
+
+// RUN: fir-opt --split-input-file --simplify-fir-operations %s | FileCheck %s
+
+fir.local {type = local_init} @_QFlocalizer_with_init_regionEp_firstprivate_box_ptr_Uxi32 : !fir.box<!fir.ptr<!fir.array<?xi32>>> init {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, %arg1: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>):
+  %c0 = arith.constant 0 : index
+  %0 = fir.shape %c0 : (index) -> !fir.shape<1>
+  %1 = fir.zero_bits !fir.ptr<!fir.array<?xi32>>
+  %2 = fir.embox %1(%0) : (!fir.ptr<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
+  fir.store %2 to %arg1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+  fir.yield(%arg1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+} copy {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, %arg1: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>):
+  %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+  fir.store %0 to %arg1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+  fir.yield(%arg1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+}
+
+func.func @_QPlocalizer_with_init_region() {
+  %c42_i32 = arith.constant 42 : i32
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFlocalizer_with_init_regionEn"}
+  %2 = fir.declare %1 {uniq_name = "_QFlocalizer_with_init_regionEn"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  %3 = fir.alloca !fir.box<!fir.ptr<!fir.array<?xi32>>> {bindc_name = "p", uniq_name = "_QFlocalizer_with_init_regionEp"}
+  %4 = fir.zero_bits !fir.ptr<!fir.array<?xi32>>
+  %5 = fir.shape %c0 : (index) -> !fir.shape<1>
+  %6 = fir.embox %4(%5) : (!fir.ptr<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
+  fir.store %6 to %3 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+  %7 = fir.declare %3 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFlocalizer_with_init_regionEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+  %8 = fir.load %2 : !fir.ref<i32>
+  %9 = fir.convert %8 : (i32) -> index
+
+  fir.do_concurrent {
+    %10 = fir.alloca i32 {bindc_name = "i"}
+    %11 = fir.declare %10 {uniq_name = "_QFlocalizer_with_init_regionEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%9) step (%c1) local(@_QFlocalizer_with_init_regionEp_firstprivate_box_ptr_Uxi32 %7 -> %arg1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) {
+      %12 = fir.convert %arg0 : (index) -> i32
+      fir.store %12 to %11 : !fir.ref<i32>
+      %13 = fir.declare %arg1 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFlocalizer_with_init_regionEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+      %14 = fir.load %13 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+      %15 = fir.load %11 : !fir.ref<i32>
+      %16 = fir.convert %15 : (i32) -> i64
+      %17:3 = fir.box_dims %14, %c0 : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+      %18 = fir.shift %17#0 : (index) -> !fir.shift<1>
+      %19 = fir.array_coor %14(%18) %16 : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, !fir.shift<1>, i64) -> !fir.ref<i32>
+      fir.store %c42_i32 to %19 : !fir.ref<i32>
+    }
+  }
+
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPlocalizer_with_init_region() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 42 : i32
+// CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "i"}
+// CHECK:           %[[VAL_4:.*]] = fir.declare %[[VAL_3]] {uniq_name = "_QFlocalizer_with_init_regionEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFlocalizer_with_init_regionEn"}
+// CHECK:           %[[VAL_7:.*]] = fir.declare %[[VAL_6]] {uniq_name = "_QFlocalizer_with_init_regionEn"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xi32>>> {bindc_name = "p", uniq_name = "_QFlocalizer_with_init_regionEp"}
+// CHECK:           %[[VAL_9:.*]] = fir.zero_bits !fir.ptr<!fir.array<?xi32>>
+// CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_9]](%[[VAL_10]]) : (!fir.ptr<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
+// CHECK:           fir.store %[[VAL_11]] to %[[VAL_8]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+// CHECK:           %[[VAL_12:.*]] = fir.declare %[[VAL_8]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFlocalizer_with_init_regionEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+// CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i32) -> index
+// CHECK:           fir.do_loop %[[VAL_15:.*]] = %[[VAL_1]] to %[[VAL_14]] step %[[VAL_1]] unordered {
+
+// Local allocation
+// CHECK:             %[[VAL_16:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xi32>>>
+
+// `init` region body
+// CHECK:             %[[VAL_17:.*]] = fir.shape %[[VAL_0]] : (index) -> !fir.shape<1>
+// CHECK:             %[[VAL_18:.*]] = fir.zero_bits !fir.ptr<!fir.array<?xi32>>
+// CHECK:             %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_17]]) : (!fir.ptr<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
+// CHECK:             fir.store %[[VAL_19]] to %[[VAL_16]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+
+// `copy` region body
+// CHECK:             %[[VAL_20:.*]] = fir.load %[[VAL_12]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+// CHECK:             fir.store %[[VAL_20]] to %[[VAL_16]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+
+// loop body
+// CHECK:             %[[VAL_21:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
+// CHECK:             fir.store %[[VAL_21]] to %[[VAL_4]] : !fir.ref<i32>
+// CHECK:             %[[VAL_22:.*]] = fir.declare %[[VAL_16]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFlocalizer_with_init_regionEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+// CHECK:             %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_4]] : !fir.ref<i32>
+// CHECK:             %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (i32) -> i64
+// CHECK:             %[[VAL_26:.*]]:3 = fir.box_dims %[[VAL_23]], %[[VAL_0]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_27:.*]] = fir.shift %[[VAL_26]]#0 : (index) -> !fir.shift<1>
+// CHECK:             %[[VAL_28:.*]] = fir.array_coor %[[VAL_23]](%[[VAL_27]]) %[[VAL_25]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, !fir.shift<1>, i64) -> !fir.ref<i32>
+// CHECK:             fir.store %[[VAL_2]] to %[[VAL_28]] : !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
diff --git a/external/llvm-project/flang/test/Transforms/lower-repack-arrays.fir b/external/llvm-project/flang/test/Transforms/lower-repack-arrays.fir
index 0b323b1bb069..012e957173ac 100644
--- a/external/llvm-project/flang/test/Transforms/lower-repack-arrays.fir
+++ b/external/llvm-project/flang/test/Transforms/lower-repack-arrays.fir
@@ -799,79 +799,67 @@ func.func @_QPtest6(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bi
   return
 }
 // CHECK-LABEL:   func.func @_QPtest6(
-// CHECK-SAME:                        %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bindc_name = "x"}) {
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant
 // CHECK:           %[[VAL_1:.*]] = arith.constant
-// CHECK:           %[[VAL_2:.*]] = arith.constant
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant false
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
-// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = fir.if %[[VAL_9]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
-// CHECK:             %[[VAL_11:.*]] = fir.is_contiguous_box %[[VAL_0]] innermost : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
-// CHECK:             %[[VAL_12:.*]] = arith.cmpi eq, %[[VAL_11]], %[[VAL_6]] : i1
-// CHECK:             %[[VAL_13:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>
-// CHECK:             %[[VAL_14:.*]] = fir.is_present %[[VAL_13]] : (!fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
-// CHECK:             %[[VAL_15:.*]] = arith.andi %[[VAL_12]], %[[VAL_14]] : i1
-// CHECK:             %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               %[[VAL_20:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_19]](%[[VAL_20]]) : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
-// CHECK:               fir.store %[[VAL_21]] to %[[VAL_7]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
-// CHECK:               %[[VAL_22:.*]] = fir.declare %[[VAL_7]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".repacked"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
-// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               fir.call @_FortranAAllocatableApplyMold(%[[VAL_23]], %[[VAL_24]], %[[VAL_3]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
-// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_17]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_25]], %[[VAL_26]], %[[VAL_27]], %[[VAL_28]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
-// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_18]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_29]], %[[VAL_30]], %[[VAL_31]], %[[VAL_32]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_34:.*]] = fir.absent !fir.box<none>
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_33]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               %[[VAL_37:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_35]], %{{.*}}, %[[VAL_6]], %[[VAL_34]], %[[VAL_36]], %[[VAL_2]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-// CHECK:               %[[VAL_38:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
-// CHECK:               %[[VAL_39:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_41:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_39]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_40]], %[[VAL_41]], %[[VAL_42]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_43:.*]] = fir.shift %[[VAL_17]]#0, %[[VAL_18]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_44:.*]] = fir.rebox %[[VAL_38]](%[[VAL_43]]) : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               fir.result %[[VAL_44]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
+// CHECK:           %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:             %[[VAL_9:.*]] = fir.is_contiguous_box %[[ARG0]] innermost : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
+// CHECK:             %[[VAL_10:.*]] = arith.cmpi eq, %[[VAL_9]], %[[VAL_4]] : i1
+// CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
+// CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:               %[[VAL_18:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_17]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>, !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:               fir.store %[[VAL_19]] to %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
+// CHECK:               %[[VAL_20:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_22:.*]] = fir.absent !fir.box<none>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
+// CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             } else {
-// CHECK:               fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             }
-// CHECK:             fir.result %[[VAL_16]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             fir.result %[[VAL_14]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:           } else {
-// CHECK:             fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             fir.result %[[ARG0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:           }
-// CHECK:           %[[VAL_45:.*]] = fir.declare %[[VAL_10]] dummy_scope %[[VAL_8]] {uniq_name = "_QFtest6Ex"} : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.dscope) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:           %[[VAL_46:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
-// CHECK:           fir.if %[[VAL_46]] {
-// CHECK:             %[[VAL_47:.*]] = fir.box_addr %[[VAL_10]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:             %[[VAL_49:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_47]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
-// CHECK:             %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
-// CHECK:             %[[VAL_51:.*]] = arith.cmpi ne, %[[VAL_48]], %[[VAL_50]] : index
-// CHECK:             fir.if %[[VAL_51]] {
-// CHECK:               %[[VAL_52:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_53:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_54:.*]] = fir.convert %[[VAL_10]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_55:.*]] = fir.convert %[[VAL_52]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_53]], %[[VAL_54]], %[[VAL_55]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               fir.freemem %[[VAL_47]] : !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:           %[[VAL_35:.*]] = fir.declare %[[VAL_8]] dummy_scope %[[VAL_6]] {uniq_name = "_QFtest6Ex"} : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.dscope) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:           %[[VAL_36:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
+// CHECK:           fir.if %[[VAL_36]] {
+// CHECK:             %[[VAL_37:.*]] = fir.box_addr %[[VAL_8]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             %[[VAL_38:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
+// CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
+// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_45:.*]] = fir.convert %[[VAL_42]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_43]], %[[VAL_44]], %[[VAL_45]], %[[VAL_0]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               fir.freemem %[[VAL_37]] : !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             }
 // CHECK:           }
 // CHECK:           return
@@ -887,79 +875,67 @@ func.func @_QPtest6_stack(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {
   return
 }
 // CHECK-LABEL:   func.func @_QPtest6_stack(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bindc_name = "x"}) {
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant
 // CHECK:           %[[VAL_1:.*]] = arith.constant
-// CHECK:           %[[VAL_2:.*]] = arith.constant
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant false
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
-// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = fir.if %[[VAL_9]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
-// CHECK:             %[[VAL_11:.*]] = fir.is_contiguous_box %[[VAL_0]] innermost : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
-// CHECK:             %[[VAL_12:.*]] = arith.cmpi eq, %[[VAL_11]], %[[VAL_6]] : i1
-// CHECK:             %[[VAL_13:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>
-// CHECK:             %[[VAL_14:.*]] = fir.is_present %[[VAL_13]] : (!fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
-// CHECK:             %[[VAL_15:.*]] = arith.andi %[[VAL_12]], %[[VAL_14]] : i1
-// CHECK:             %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               %[[VAL_20:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_19]](%[[VAL_20]]) : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
-// CHECK:               fir.store %[[VAL_21]] to %[[VAL_7]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
-// CHECK:               %[[VAL_22:.*]] = fir.declare %[[VAL_7]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".repacked"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
-// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               fir.call @_FortranAAllocatableApplyMold(%[[VAL_23]], %[[VAL_24]], %[[VAL_3]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
-// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_17]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_25]], %[[VAL_26]], %[[VAL_27]], %[[VAL_28]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
-// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_18]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_29]], %[[VAL_30]], %[[VAL_31]], %[[VAL_32]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_34:.*]] = fir.absent !fir.box<none>
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_33]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               %[[VAL_37:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_35]], %{{.*}}, %[[VAL_6]], %[[VAL_34]], %[[VAL_36]], %[[VAL_2]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-// CHECK:               %[[VAL_38:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
-// CHECK:               %[[VAL_39:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_41:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_39]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_40]], %[[VAL_41]], %[[VAL_42]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_43:.*]] = fir.shift %[[VAL_17]]#0, %[[VAL_18]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_44:.*]] = fir.rebox %[[VAL_38]](%[[VAL_43]]) : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:               fir.result %[[VAL_44]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
+// CHECK:           %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:             %[[VAL_9:.*]] = fir.is_contiguous_box %[[ARG0]] innermost : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
+// CHECK:             %[[VAL_10:.*]] = arith.cmpi eq, %[[VAL_9]], %[[VAL_4]] : i1
+// CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
+// CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:               %[[VAL_18:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_17]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.shape<2>, !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:               fir.store %[[VAL_19]] to %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
+// CHECK:               %[[VAL_20:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_22:.*]] = fir.absent !fir.box<none>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>>
+// CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>
+// CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             } else {
-// CHECK:               fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             }
-// CHECK:             fir.result %[[VAL_16]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             fir.result %[[VAL_14]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:           } else {
-// CHECK:             fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             fir.result %[[ARG0]] : !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:           }
-// CHECK:           %[[VAL_45:.*]] = fir.declare %[[VAL_10]] dummy_scope %[[VAL_8]] {uniq_name = "_QFtest6_stackEx"} : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.dscope) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:           %[[VAL_46:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
-// CHECK:           fir.if %[[VAL_46]] {
-// CHECK:             %[[VAL_47:.*]] = fir.box_addr %[[VAL_10]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:             %[[VAL_49:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
-// CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_47]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
-// CHECK:             %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
-// CHECK:             %[[VAL_51:.*]] = arith.cmpi ne, %[[VAL_48]], %[[VAL_50]] : index
-// CHECK:             fir.if %[[VAL_51]] {
-// CHECK:               %[[VAL_52:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_53:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_54:.*]] = fir.convert %[[VAL_10]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_55:.*]] = fir.convert %[[VAL_52]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_53]], %[[VAL_54]], %[[VAL_55]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               fir.freemem %[[VAL_47]] : !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:           %[[VAL_35:.*]] = fir.declare %[[VAL_8]] dummy_scope %[[VAL_6]] {uniq_name = "_QFtest6_stackEx"} : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, !fir.dscope) -> !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:           %[[VAL_36:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> i1
+// CHECK:           fir.if %[[VAL_36]] {
+// CHECK:             %[[VAL_37:.*]] = fir.box_addr %[[VAL_8]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             %[[VAL_38:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
+// CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
+// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_45:.*]] = fir.convert %[[VAL_42]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_43]], %[[VAL_44]], %[[VAL_45]], %[[VAL_0]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               fir.freemem %[[VAL_37]] : !fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>
 // CHECK:             }
 // CHECK:           }
 // CHECK:           return
@@ -974,79 +950,67 @@ func.func @_QPtest7(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x
   return
 }
 // CHECK-LABEL:   func.func @_QPtest7(
-// CHECK-SAME:                        %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x"}) {
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant
 // CHECK:           %[[VAL_1:.*]] = arith.constant
-// CHECK:           %[[VAL_2:.*]] = arith.constant
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant false
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?xnone>>>
-// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = fir.if %[[VAL_9]] -> (!fir.class<!fir.array<?x?xnone>>) {
-// CHECK:             %[[VAL_11:.*]] = fir.is_contiguous_box %[[VAL_0]] innermost : (!fir.class<!fir.array<?x?xnone>>) -> i1
-// CHECK:             %[[VAL_12:.*]] = arith.cmpi eq, %[[VAL_11]], %[[VAL_6]] : i1
-// CHECK:             %[[VAL_13:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.ref<!fir.class<!fir.array<?x?xnone>>>
-// CHECK:             %[[VAL_14:.*]] = fir.is_present %[[VAL_13]] : (!fir.ref<!fir.class<!fir.array<?x?xnone>>>) -> i1
-// CHECK:             %[[VAL_15:.*]] = arith.andi %[[VAL_12]], %[[VAL_14]] : i1
-// CHECK:             %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (!fir.class<!fir.array<?x?xnone>>) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xnone>>
-// CHECK:               %[[VAL_20:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_19]](%[[VAL_20]]) : (!fir.heap<!fir.array<?x?xnone>>, !fir.shape<2>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
-// CHECK:               fir.store %[[VAL_21]] to %[[VAL_7]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
-// CHECK:               %[[VAL_22:.*]] = fir.declare %[[VAL_7]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".repacked"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
-// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               fir.call @_FortranAAllocatableApplyMold(%[[VAL_23]], %[[VAL_24]], %[[VAL_3]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
-// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_17]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_25]], %[[VAL_26]], %[[VAL_27]], %[[VAL_28]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
-// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_18]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_29]], %[[VAL_30]], %[[VAL_31]], %[[VAL_32]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_34:.*]] = fir.absent !fir.box<none>
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_33]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               %[[VAL_37:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_35]], %{{.*}}, %[[VAL_6]], %[[VAL_34]], %[[VAL_36]], %[[VAL_2]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-// CHECK:               %[[VAL_38:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
-// CHECK:               %[[VAL_39:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_41:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_39]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_40]], %[[VAL_41]], %[[VAL_42]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_43:.*]] = fir.shift %[[VAL_17]]#0, %[[VAL_18]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_44:.*]] = fir.rebox %[[VAL_38]](%[[VAL_43]]) : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?xnone>>
-// CHECK:               fir.result %[[VAL_44]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
+// CHECK:           %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (!fir.class<!fir.array<?x?xnone>>) {
+// CHECK:             %[[VAL_9:.*]] = fir.is_contiguous_box %[[ARG0]] innermost : (!fir.class<!fir.array<?x?xnone>>) -> i1
+// CHECK:             %[[VAL_10:.*]] = arith.cmpi eq, %[[VAL_9]], %[[VAL_4]] : i1
+// CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.ref<!fir.class<!fir.array<?x?xnone>>>
+// CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?xnone>>>) -> i1
+// CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?xnone>>) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:               %[[VAL_18:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xnone>>
+// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_17]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?xnone>>, !fir.shape<2>, !fir.class<!fir.array<?x?xnone>>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:               fir.store %[[VAL_19]] to %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
+// CHECK:               %[[VAL_20:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_22:.*]] = fir.absent !fir.box<none>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
+// CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?xnone>>
+// CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:             } else {
-// CHECK:               fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:             }
-// CHECK:             fir.result %[[VAL_16]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:             fir.result %[[VAL_14]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:           } else {
-// CHECK:             fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:             fir.result %[[ARG0]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:           }
-// CHECK:           %[[VAL_45:.*]] = fir.declare %[[VAL_10]] dummy_scope %[[VAL_8]] {uniq_name = "_QFtest7Ex"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> !fir.class<!fir.array<?x?xnone>>
-// CHECK:           %[[VAL_46:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
-// CHECK:           fir.if %[[VAL_46]] {
-// CHECK:             %[[VAL_47:.*]] = fir.box_addr %[[VAL_10]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
-// CHECK:             %[[VAL_49:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
-// CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_47]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
-// CHECK:             %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
-// CHECK:             %[[VAL_51:.*]] = arith.cmpi ne, %[[VAL_48]], %[[VAL_50]] : index
-// CHECK:             fir.if %[[VAL_51]] {
-// CHECK:               %[[VAL_52:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_53:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               %[[VAL_54:.*]] = fir.convert %[[VAL_10]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               %[[VAL_55:.*]] = fir.convert %[[VAL_52]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_53]], %[[VAL_54]], %[[VAL_55]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               fir.freemem %[[VAL_47]] : !fir.heap<!fir.array<?x?xnone>>
+// CHECK:           %[[VAL_35:.*]] = fir.declare %[[VAL_8]] dummy_scope %[[VAL_6]] {uniq_name = "_QFtest7Ex"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> !fir.class<!fir.array<?x?xnone>>
+// CHECK:           %[[VAL_36:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
+// CHECK:           fir.if %[[VAL_36]] {
+// CHECK:             %[[VAL_37:.*]] = fir.box_addr %[[VAL_8]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
+// CHECK:             %[[VAL_38:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
+// CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
+// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_45:.*]] = fir.convert %[[VAL_42]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_43]], %[[VAL_44]], %[[VAL_45]], %[[VAL_0]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               fir.freemem %[[VAL_37]] : !fir.heap<!fir.array<?x?xnone>>
 // CHECK:             }
 // CHECK:           }
 // CHECK:           return
@@ -1062,79 +1026,67 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
   return
 }
 // CHECK-LABEL:   func.func @_QPtest7_stack(
-// CHECK-SAME:                              %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x"}) {
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant
 // CHECK:           %[[VAL_1:.*]] = arith.constant
-// CHECK:           %[[VAL_2:.*]] = arith.constant
-// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
-// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_6:.*]] = arith.constant false
-// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?xnone>>>
-// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
-// CHECK:           %[[VAL_9:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
-// CHECK:           %[[VAL_10:.*]] = fir.if %[[VAL_9]] -> (!fir.class<!fir.array<?x?xnone>>) {
-// CHECK:             %[[VAL_11:.*]] = fir.is_contiguous_box %[[VAL_0]] innermost : (!fir.class<!fir.array<?x?xnone>>) -> i1
-// CHECK:             %[[VAL_12:.*]] = arith.cmpi eq, %[[VAL_11]], %[[VAL_6]] : i1
-// CHECK:             %[[VAL_13:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.ref<!fir.class<!fir.array<?x?xnone>>>
-// CHECK:             %[[VAL_14:.*]] = fir.is_present %[[VAL_13]] : (!fir.ref<!fir.class<!fir.array<?x?xnone>>>) -> i1
-// CHECK:             %[[VAL_15:.*]] = arith.andi %[[VAL_12]], %[[VAL_14]] : i1
-// CHECK:             %[[VAL_16:.*]] = fir.if %[[VAL_15]] -> (!fir.class<!fir.array<?x?xnone>>) {
-// CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_18:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
-// CHECK:               %[[VAL_19:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xnone>>
-// CHECK:               %[[VAL_20:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
-// CHECK:               %[[VAL_21:.*]] = fir.embox %[[VAL_19]](%[[VAL_20]]) : (!fir.heap<!fir.array<?x?xnone>>, !fir.shape<2>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
-// CHECK:               fir.store %[[VAL_21]] to %[[VAL_7]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
-// CHECK:               %[[VAL_22:.*]] = fir.declare %[[VAL_7]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = ".repacked"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
-// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               fir.call @_FortranAAllocatableApplyMold(%[[VAL_23]], %[[VAL_24]], %[[VAL_3]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32) -> ()
-// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
-// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_17]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_25]], %[[VAL_26]], %[[VAL_27]], %[[VAL_28]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
-// CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
-// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_18]]#1 : (index) -> i64
-// CHECK:               fir.call @_FortranAAllocatableSetBounds(%[[VAL_29]], %[[VAL_30]], %[[VAL_31]], %[[VAL_32]]) : (!fir.ref<!fir.box<none>>, i32, i64, i64) -> ()
-// CHECK:               %[[VAL_33:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_34:.*]] = fir.absent !fir.box<none>
-// CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
-// CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_33]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               %[[VAL_37:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_35]], %{{.*}}, %[[VAL_6]], %[[VAL_34]], %[[VAL_36]], %[[VAL_2]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
-// CHECK:               %[[VAL_38:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
-// CHECK:               %[[VAL_39:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.box<none>
-// CHECK:               %[[VAL_41:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_39]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_40]], %[[VAL_41]], %[[VAL_42]], %[[VAL_2]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               %[[VAL_43:.*]] = fir.shift %[[VAL_17]]#0, %[[VAL_18]]#0 : (index, index) -> !fir.shift<2>
-// CHECK:               %[[VAL_44:.*]] = fir.rebox %[[VAL_38]](%[[VAL_43]]) : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?xnone>>
-// CHECK:               fir.result %[[VAL_44]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant false
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
+// CHECK:           %[[VAL_8:.*]] = fir.if %[[VAL_7]] -> (!fir.class<!fir.array<?x?xnone>>) {
+// CHECK:             %[[VAL_9:.*]] = fir.is_contiguous_box %[[ARG0]] innermost : (!fir.class<!fir.array<?x?xnone>>) -> i1
+// CHECK:             %[[VAL_10:.*]] = arith.cmpi eq, %[[VAL_9]], %[[VAL_4]] : i1
+// CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.ref<!fir.class<!fir.array<?x?xnone>>>
+// CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?xnone>>>) -> i1
+// CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?xnone>>) {
+// CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:               %[[VAL_18:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xnone>>
+// CHECK:               %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_17]]) source_box %[[ARG0]] : (!fir.heap<!fir.array<?x?xnone>>, !fir.shape<2>, !fir.class<!fir.array<?x?xnone>>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:               fir.store %[[VAL_19]] to %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
+// CHECK:               %[[VAL_20:.*]] = fir.zero_bits !fir.ref<none>
+// CHECK:               %[[VAL_21:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_22:.*]] = fir.absent !fir.box<none>
+// CHECK:               %[[VAL_23:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:               %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<none>) -> !fir.ref<i64>
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               %[[VAL_26:.*]] = fir.call @_FortranAAllocatableAllocate(%[[VAL_23]], %[[VAL_24]], %[[VAL_4]], %[[VAL_22]], %[[VAL_25]], %[[VAL_1]]) : (!fir.ref<!fir.box<none>>, !fir.ref<i64>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
+// CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
+// CHECK:               %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = ".repacked"} : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.class<!fir.heap<!fir.array<?x?xnone>>>
+// CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_28]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.box<none>
+// CHECK:               %[[VAL_31:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_29]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_30]], %[[VAL_31]], %[[VAL_32]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               %[[VAL_33:.*]] = fir.shift %[[VAL_15]]#0, %[[VAL_16]]#0 : (index, index) -> !fir.shift<2>
+// CHECK:               %[[VAL_34:.*]] = fir.rebox %[[VAL_28]](%[[VAL_33]]) : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>, !fir.shift<2>) -> !fir.class<!fir.array<?x?xnone>>
+// CHECK:               fir.result %[[VAL_34]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:             } else {
-// CHECK:               fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:               fir.result %[[ARG0]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:             }
-// CHECK:             fir.result %[[VAL_16]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:             fir.result %[[VAL_14]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:           } else {
-// CHECK:             fir.result %[[VAL_0]] : !fir.class<!fir.array<?x?xnone>>
+// CHECK:             fir.result %[[ARG0]] : !fir.class<!fir.array<?x?xnone>>
 // CHECK:           }
-// CHECK:           %[[VAL_45:.*]] = fir.declare %[[VAL_10]] dummy_scope %[[VAL_8]] {uniq_name = "_QFtest7Ex"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> !fir.class<!fir.array<?x?xnone>>
-// CHECK:           %[[VAL_46:.*]] = fir.is_present %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
-// CHECK:           fir.if %[[VAL_46]] {
-// CHECK:             %[[VAL_47:.*]] = fir.box_addr %[[VAL_10]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
-// CHECK:             %[[VAL_49:.*]] = fir.box_addr %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
-// CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_47]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
-// CHECK:             %[[VAL_50:.*]] = fir.convert %[[VAL_49]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
-// CHECK:             %[[VAL_51:.*]] = arith.cmpi ne, %[[VAL_48]], %[[VAL_50]] : index
-// CHECK:             fir.if %[[VAL_51]] {
-// CHECK:               %[[VAL_52:.*]] = fir.address_of(@{{_QQcl.*}}
-// CHECK:               %[[VAL_53:.*]] = fir.convert %[[VAL_0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               %[[VAL_54:.*]] = fir.convert %[[VAL_10]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-// CHECK:               %[[VAL_55:.*]] = fir.convert %[[VAL_52]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
-// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_53]], %[[VAL_54]], %[[VAL_55]], %[[VAL_1]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
-// CHECK:               fir.freemem %[[VAL_47]] : !fir.heap<!fir.array<?x?xnone>>
+// CHECK:           %[[VAL_35:.*]] = fir.declare %[[VAL_8]] dummy_scope %[[VAL_6]] {uniq_name = "_QFtest7Ex"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> !fir.class<!fir.array<?x?xnone>>
+// CHECK:           %[[VAL_36:.*]] = fir.is_present %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> i1
+// CHECK:           fir.if %[[VAL_36]] {
+// CHECK:             %[[VAL_37:.*]] = fir.box_addr %[[VAL_8]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
+// CHECK:             %[[VAL_38:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.heap<!fir.array<?x?xnone>>
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
+// CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
+// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
+// CHECK:               %[[VAL_45:.*]] = fir.convert %[[VAL_42]] : (!fir.ref<!fir.char<1,{{.*}}>>) -> !fir.ref<i8>
+// CHECK:               fir.call @_FortranAShallowCopyDirect(%[[VAL_43]], %[[VAL_44]], %[[VAL_45]], %[[VAL_0]]) : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+// CHECK:               fir.freemem %[[VAL_37]] : !fir.heap<!fir.array<?x?xnone>>
 // CHECK:             }
 // CHECK:           }
 // CHECK:           return
diff --git a/external/llvm-project/flang/test/Transforms/omp-map-info-finalization.fir b/external/llvm-project/flang/test/Transforms/omp-map-info-finalization.fir
index e753c3d66c7f..017609914f0b 100644
--- a/external/llvm-project/flang/test/Transforms/omp-map-info-finalization.fir
+++ b/external/llvm-project/flang/test/Transforms/omp-map-info-finalization.fir
@@ -1,4 +1,5 @@
-// RUN: fir-opt --split-input-file --omp-map-info-finalization %s | FileCheck %s
+// RUN: fir-opt  --split-input-file --omp-map-info-finalization %s | FileCheck %s
+// XFAIL: *
 func.func @test_descriptor_expansion_pass(%arg0: !fir.box<!fir.array<?xi32>>) {
   %0 = fir.alloca !fir.box<!fir.heap<i32>>
   %1 = fir.zero_bits !fir.heap<i32>
@@ -333,3 +334,62 @@ func.func @_QPreuse_alloca(%arg0: !fir.box<!fir.array<?xf64>> {fir.bindc_name =
 // CHECK:       }
 // CHECK:       return
 
+
+
+omp.private {type = firstprivate} @boxchar.privatizer : !fir.boxchar<1> copy {
+  ^bb0(%arg0: !fir.boxchar<1>, %arg1: !fir.boxchar<1>):
+    omp.yield(%arg0 : !fir.boxchar<1>)
+  }
+func.func @_QPrealtest(%arg0: !fir.boxchar<1>) {
+  %0 = fir.alloca !fir.boxchar<1>
+  %1 = fir.dummy_scope : !fir.dscope
+  %2:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3:2 = hlfir.declare %2#0 typeparams %2#1 dummy_scope %1 {uniq_name = "a0"}: (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+  fir.store %3#0 to %0 : !fir.ref<!fir.boxchar<1>>
+  %4 = fir.load %0 : !fir.ref<!fir.boxchar<1>>
+  %5:2 = fir.unboxchar %4 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %6:2 = fir.unboxchar %4 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %7 = arith.subi %6#1, %c1 : index
+  %8 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%7 : index) extent(%6#1 : index) stride(%c1 : index) start_idx(%c0 : index) {stride_in_bytes = true}
+  %9 = omp.map.info var_ptr(%0 : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(to) capture(ByRef) bounds(%8) -> !fir.ref<!fir.boxchar<1>>
+  omp.target map_entries(%9 -> %arg1 : !fir.ref<!fir.boxchar<1>>) private(@boxchar.privatizer %3#0 -> %arg2 [map_idx=0] : !fir.boxchar<1>) {
+    %10:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+    %11:2 = hlfir.declare %10#0 typeparams %10#1 {uniq_name = "tgt_a0"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+    omp.terminator
+  }
+  return
+}
+
+
+// CHECK-LABEL:   omp.private {type = firstprivate} @boxchar.privatizer : !fir.boxchar<1> copy {
+// CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.boxchar<1>, %[[VAL_1:.*]]: !fir.boxchar<1>):
+// CHECK:           omp.yield(%[[VAL_0]] : !fir.boxchar<1>)
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @_QPrealtest(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.boxchar<1>) {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.boxchar<1>
+// CHECK:           %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_2:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %[[VAL_1]] {uniq_name = "a0"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+// CHECK:           fir.store %[[VAL_3]]#0 to %[[VAL_0]] : !fir.ref<!fir.boxchar<1>>
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.boxchar<1>>
+// CHECK:           %[[VAL_5:.*]]:2 = fir.unboxchar %[[VAL_4]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_8:.*]]:2 = fir.unboxchar %[[VAL_4]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_9:.*]] = arith.subi %[[VAL_8]]#1, %[[VAL_7]] : index
+// CHECK:           %[[VAL_10:.*]] = omp.map.bounds lower_bound(%[[VAL_6]] : index) upper_bound(%[[VAL_9]] : index) extent(%[[VAL_8]]#1 : index) stride(%[[VAL_7]] : index) start_idx(%[[VAL_6]] : index) {stride_in_bytes = true}
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.boxchar<1>>
+// CHECK:           %[[VAL_12:.*]] = fir.box_offset %[[VAL_0]] base_addr : (!fir.ref<!fir.boxchar<1>>) -> !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>
+// CHECK:           %[[VAL_13:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.char<1,?>) map_clauses(implicit, to) capture(ByRef) var_ptr_ptr(%[[VAL_12]] : !fir.llvm_ptr<!fir.ref<!fir.char<1,?>>>) bounds(%[[VAL_10]]) -> !fir.ref<!fir.boxchar<1>>
+// CHECK:           %[[VAL_14:.*]] = omp.map.info var_ptr(%[[VAL_0]] : !fir.ref<!fir.boxchar<1>>, !fir.boxchar<1>) map_clauses(to) capture(ByRef) members(%[[VAL_13]] : [0] : !fir.ref<!fir.boxchar<1>>) -> !fir.ref<!fir.boxchar<1>>
+// CHECK:           omp.target map_entries(%[[VAL_14]] -> %[[VAL_15:.*]], %[[VAL_13]] -> %[[VAL_16:.*]] : !fir.ref<!fir.boxchar<1>>, !fir.ref<!fir.boxchar<1>>) private(@boxchar.privatizer %[[VAL_3]]#0 -> %[[VAL_17:.*]] [map_idx=0] : !fir.boxchar<1>) {
+// CHECK:             %[[VAL_18:.*]]:2 = fir.unboxchar %[[VAL_17]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]]#0 typeparams %[[VAL_18]]#1 {uniq_name = "tgt_a0"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+// CHECK:             omp.terminator
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
diff --git a/external/llvm-project/flang/test/Transforms/tbaa-with-dummy-scope2.fir b/external/llvm-project/flang/test/Transforms/tbaa-with-dummy-scope2.fir
index c8f419fbee65..249471de458d 100644
--- a/external/llvm-project/flang/test/Transforms/tbaa-with-dummy-scope2.fir
+++ b/external/llvm-project/flang/test/Transforms/tbaa-with-dummy-scope2.fir
@@ -43,12 +43,15 @@ func.func @_QPtest1() attributes {noinline} {
 // CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest1">
 // CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_0]], 0>}>
 // CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_1]], 0>}>
+// CHECK: #[[$TARGETDATA:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_2]], 0>}>
 // CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_2]], 0>}>
-// CHECK: #[[TARGETDATA:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_2]], 0>}>
+// CHECK: #[[$LOCAL_ATTR_0:.+]] = #llvm.tbaa_type_desc<id = "allocated data", members = {<#[[$TARGETDATA]], 0>}>
 // CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtest1FinnerEx", members = {<#[[$ATTR_3]], 0>}>
-// CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[TARGETDATA]], 0>}>
+// CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$TARGETDATA]], 0>}>
 // CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_5]], access_type = #[[$ATTR_5]], offset = 0>
+// CHECK: #[[$LOCAL_ATTR_1:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QFtest1FinnerEy", members = {<#[[$LOCAL_ATTR_0]], 0>}>
 // CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmEglob", members = {<#[[$ATTR_4]], 0>}>
+// CHECK: #[[$LOCAL_ATTR_2:.+]] = #llvm.tbaa_tag<base_type = #[[$LOCAL_ATTR_1]], access_type = #[[$LOCAL_ATTR_1]], offset = 0>
 // CHECK: #[[$ATTR_8:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_6]], access_type = #[[$ATTR_6]], offset = 0>
 // CHECK-LABEL:   func.func @_QPtest1() attributes {noinline} {
 // CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFtest1FinnerEy"}
@@ -57,8 +60,8 @@ func.func @_QPtest1() attributes {noinline} {
 // CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
 // CHECK:           %[[VAL_6:.*]] = fir.declare %[[VAL_4]] dummy_scope %[[VAL_5]] {uniq_name = "_QFtest1FinnerEx"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 // CHECK:           %[[VAL_7:.*]] = fir.declare %[[VAL_2]] {uniq_name = "_QFtest1FinnerEy"} : (!fir.ref<i32>) -> !fir.ref<i32>
-// CHECK:           fir.store %{{.*}} to %[[VAL_7]] : !fir.ref<i32>
-// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           fir.store %{{.*}} to %[[VAL_7]] {tbaa = [#[[$LOCAL_ATTR_2]]]} : !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] {tbaa = [#[[$LOCAL_ATTR_2]]]}  : !fir.ref<i32>
 // CHECK:           fir.store %[[VAL_8]] to %[[VAL_6]] {tbaa = [#[[$ATTR_7]]]} : !fir.ref<i32>
 // CHECK:           fir.store %{{.*}} to %[[VAL_4]] {tbaa = [#[[$ATTR_8]]]} : !fir.ref<i32>
 
@@ -87,12 +90,16 @@ func.func @_QPtest2() attributes {noinline} {
 // CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_1]], 0>}>
 // CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_2]], 0>}>
 // CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_3]], 0>}>
+// CHECK: #[[$TARGETDATA_0:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_4]], 0>}>
 // CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_4]], 0>}>
-// CHECK: #[[TARGETDATA:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_5]], 0>}>
+// CHECK: #[[$TARGETDATA_1:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_5]], 0>}>
+// CHECK: #[[$LOCAL_ATTR_0:.+]] = #llvm.tbaa_type_desc<id = "allocated data", members = {<#[[$TARGETDATA_0]], 0>}>
 // CHECK: #[[$ATTR_8:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtest2FinnerEx", members = {<#[[$ATTR_6]], 0>}>
-// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[TARGETDATA]], 0>}>
+// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$TARGETDATA_1]], 0>}>
 // CHECK: #[[$ATTR_10:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_8]], access_type = #[[$ATTR_8]], offset = 0>
+// CHECK: #[[$LOCAL_ATTR_1:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QFtest2FinnerEy", members = {<#[[$LOCAL_ATTR_0]], 0>}>
 // CHECK: #[[$ATTR_9:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmEglob", members = {<#[[$ATTR_7]], 0>}>
+// CHECK: #[[$LOCAL_ATTR_2:.+]] = #llvm.tbaa_tag<base_type = #[[$LOCAL_ATTR_1]], access_type = #[[$LOCAL_ATTR_1]], offset = 0>
 // CHECK: #[[$ATTR_11:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_9]], access_type = #[[$ATTR_9]], offset = 0>
 // CHECK-LABEL:   func.func @_QPtest2() attributes {noinline} {
 // CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFtest2FinnerEy"}
@@ -102,7 +109,7 @@ func.func @_QPtest2() attributes {noinline} {
 // CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
 // CHECK:           %[[VAL_7:.*]] = fir.declare %[[VAL_5]] dummy_scope %[[VAL_6]] {uniq_name = "_QFtest2FinnerEx"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 // CHECK:           %[[VAL_8:.*]] = fir.declare %[[VAL_2]] {uniq_name = "_QFtest2FinnerEy"} : (!fir.ref<i32>) -> !fir.ref<i32>
-// CHECK:           fir.store %{{.*}} to %[[VAL_8]] : !fir.ref<i32>
-// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+// CHECK:           fir.store %{{.*}} to %[[VAL_8]] {tbaa = [#[[$LOCAL_ATTR_2]]]}  : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_8]] {tbaa = [#[[$LOCAL_ATTR_2]]]} : !fir.ref<i32>
 // CHECK:           fir.store %[[VAL_9]] to %[[VAL_7]] {tbaa = [#[[$ATTR_10]]]} : !fir.ref<i32>
 // CHECK:           fir.store %{{.*}} to %[[VAL_5]] {tbaa = [#[[$ATTR_11]]]} : !fir.ref<i32>
diff --git a/external/llvm-project/flang/test/Transforms/tbaa2.fir b/external/llvm-project/flang/test/Transforms/tbaa2.fir
index 4678a1cd4a68..1429d0b42076 100644
--- a/external/llvm-project/flang/test/Transforms/tbaa2.fir
+++ b/external/llvm-project/flang/test/Transforms/tbaa2.fir
@@ -50,6 +50,7 @@
 // CHECK: #[[TARGETDATA:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[ANY_DATA]], 0>}>
 // CHECK: #[[ANY_ARG:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[ANY_DATA]], 0>}>
 // CHECK: #[[ANY_GLBL:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[TARGETDATA]], 0>}>
+// CHECK: #[[ANY_LOCAL:.+]] = #llvm.tbaa_type_desc<id = "allocated data", members = {<#[[TARGETDATA]], 0>}>
 // CHECK: #[[ARG_LOW:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QMmodFcalleeElow", members = {<#[[ANY_ARG]], 0>}>
 // CHECK: #[[ANY_DIRECT:.+]] = #llvm.tbaa_type_desc<id = "direct data", members = {<#[[TARGETDATA]], 0>}>
 // CHECK: #[[ARG_Z:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QMmodFcalleeEz", members = {<#[[ANY_ARG]], 0>}>
@@ -61,21 +62,31 @@
 
 // CHECK: #[[GLBL_ZSTART:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmodEzstart", members = {<#[[ANY_GLBL]], 0>}>
 // CHECK: #[[GLBL_ZSTOP:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmodEzstop", members = {<#[[ANY_GLBL]], 0>}>
+// CHECK: #[[LOCAL1_ALLOC:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QMmodFcalleeEk", members = {<#[[ANY_LOCAL]], 0>}>
 // CHECK: #[[GLBL_YSTART:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmodEystart", members = {<#[[ANY_GLBL]], 0>}>
 // CHECK: #[[GLBL_YSTOP:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmodEystop", members = {<#[[ANY_GLBL]], 0>}>
+// CHECK: #[[LOCAL2_ALLOC:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QMmodFcalleeEj", members = {<#[[ANY_LOCAL]], 0>}>
 // CHECK: #[[GLBL_XSTART:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmodExstart", members = {<#[[ANY_GLBL]], 0>}>
+// CHECK: #[[LOCAL3_ALLOC:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QMmodFcalleeEi", members = {<#[[ANY_LOCAL]], 0>}>
+// CHECK: #[[LOCAL4_ALLOC:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QMmodFcalleeEdxold", members = {<#[[ANY_LOCAL]], 0>}>
 // CHECK: #[[DIRECT_A:.+]] = #llvm.tbaa_type_desc<id = "direct data/_QMmodEa", members = {<#[[ANY_DIRECT]], 0>}>
 // CHECK: #[[DIRECT_B:.+]] = #llvm.tbaa_type_desc<id = "direct data/_QMmodEb", members = {<#[[ANY_DIRECT]], 0>}>
 // CHECK: #[[GLBL_DYINV:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmodEdyinv", members = {<#[[ANY_GLBL]], 0>}>
+// CHECK: #[[LOCAL5_ALLOC:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QMmodFcalleeEdzinv", members = {<#[[ANY_LOCAL]], 0>}>
 
 // CHECK: #[[GLBL_ZSTART_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[GLBL_ZSTART]], access_type = #[[GLBL_ZSTART]], offset = 0>
 // CHECK: #[[GLBL_ZSTOP_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[GLBL_ZSTOP]], access_type = #[[GLBL_ZSTOP]], offset = 0>
+// CHECK: #[[LOCAL1_ALLOC_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[LOCAL1_ALLOC]], access_type = #[[LOCAL1_ALLOC]], offset = 0>
 // CHECK: #[[GLBL_YSTART_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[GLBL_YSTART]], access_type = #[[GLBL_YSTART]], offset = 0>
 // CHECK: #[[GLBL_YSTOP_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[GLBL_YSTOP]], access_type = #[[GLBL_YSTOP]], offset = 0>
+// CHECK: #[[LOCAL2_ALLOC_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[LOCAL2_ALLOC]], access_type = #[[LOCAL2_ALLOC]], offset = 0>
 // CHECK: #[[GLBL_XSTART_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[GLBL_XSTART]], access_type = #[[GLBL_XSTART]], offset = 0>
+// CHECK: #[[LOCAL3_ALLOC_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[LOCAL3_ALLOC]], access_type = #[[LOCAL3_ALLOC]], offset = 0>
+// CHECK: #[[LOCAL4_ALLOC_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[LOCAL4_ALLOC]], access_type = #[[LOCAL4_ALLOC]], offset = 0>
 // CHECK: #[[DIRECT_A_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[DIRECT_A]], access_type = #[[DIRECT_A]], offset = 0>
 // CHECK: #[[DIRECT_B_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[DIRECT_B]], access_type = #[[DIRECT_B]], offset = 0>
 // CHECK: #[[GLBL_DYINV_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[GLBL_DYINV]], access_type = #[[GLBL_DYINV]], offset = 0>
+// CHECK: #[[LOCAL5_ALLOC_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[LOCAL5_ALLOC]], access_type = #[[LOCAL5_ALLOC]], offset = 0>
 
   func.func @_QMmodPcallee(%arg0: !fir.box<!fir.array<?x?x?xf32>> {fir.bindc_name = "z"}, %arg1: !fir.box<!fir.array<?x?x?xf32>> {fir.bindc_name = "y"}, %arg2: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>> {fir.bindc_name = "low"}) {
     %c2 = arith.constant 2 : index
@@ -277,7 +288,7 @@
 // CHECK:           %[[VAL_44:.*]] = fir.convert %[[VAL_43]] : (i32) -> index
 // CHECK:           %[[VAL_45:.*]] = fir.convert %[[VAL_42]] : (index) -> i32
 // CHECK:           %[[VAL_46:.*]]:2 = fir.do_loop %[[VAL_47:.*]] = %[[VAL_42]] to %[[VAL_44]] step %[[VAL_5]] iter_args(%[[VAL_48:.*]] = %[[VAL_45]]) -> (index, i32) {
-// CHECK:             fir.store %[[VAL_48]] to %[[VAL_34]] : !fir.ref<i32>
+// CHECK:             fir.store %[[VAL_48]] to %[[VAL_34]] {tbaa = [#[[LOCAL1_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:             %[[VAL_49:.*]] = fir.load %[[VAL_18]] {tbaa = [#[[GLBL_YSTART_TAG]]]} : !fir.ref<i32>
 // CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_49]], %[[VAL_6]] : i32
 // CHECK:             %[[VAL_51:.*]] = fir.convert %[[VAL_50]] : (i32) -> index
@@ -285,24 +296,20 @@
 // CHECK:             %[[VAL_53:.*]] = fir.convert %[[VAL_52]] : (i32) -> index
 // CHECK:             %[[VAL_54:.*]] = fir.convert %[[VAL_51]] : (index) -> i32
 // CHECK:             %[[VAL_55:.*]]:2 = fir.do_loop %[[VAL_56:.*]] = %[[VAL_51]] to %[[VAL_53]] step %[[VAL_5]] iter_args(%[[VAL_57:.*]] = %[[VAL_54]]) -> (index, i32) {
-// CHECK:               fir.store %[[VAL_57]] to %[[VAL_32]] : !fir.ref<i32>
+// CHECK:               fir.store %[[VAL_57]] to %[[VAL_32]] {tbaa = [#[[LOCAL2_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:               %[[VAL_58:.*]] = fir.load %[[VAL_16]] {tbaa = [#[[GLBL_XSTART_TAG]]]} : !fir.ref<i32>
 // CHECK:               %[[VAL_59:.*]] = arith.addi %[[VAL_58]], %[[VAL_6]] : i32
 // CHECK:               %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (i32) -> index
 // CHECK:               %[[VAL_61:.*]] = fir.convert %[[VAL_60]] : (index) -> i32
 // CHECK:               %[[VAL_62:.*]]:2 = fir.do_loop %[[VAL_63:.*]] = %[[VAL_60]] to %[[VAL_4]] step %[[VAL_5]] iter_args(%[[VAL_64:.*]] = %[[VAL_61]]) -> (index, i32) {
-// TODO: local allocation assumed to always alias
-// CHECK:                 fir.store %[[VAL_64]] to %[[VAL_30]] : !fir.ref<i32>
+// CHECK:                 fir.store %[[VAL_64]] to %[[VAL_30]] {tbaa = [#[[LOCAL3_ALLOC_TAG]]]} : !fir.ref<i32>
 // load from box tagged in CodeGen
 // CHECK:                 %[[VAL_65:.*]] = fir.load %[[VAL_35]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
-// TODO: local allocation assumed to always alias
-// CHECK:                 %[[VAL_66:.*]] = fir.load %[[VAL_30]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_66:.*]] = fir.load %[[VAL_30]] {tbaa = [#[[LOCAL3_ALLOC_TAG]]]}  : !fir.ref<i32>
 // CHECK:                 %[[VAL_67:.*]] = fir.convert %[[VAL_66]] : (i32) -> i64
-// TODO: local allocation assumed to always alias
-// CHECK:                 %[[VAL_68:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_68:.*]] = fir.load %[[VAL_32]] {tbaa = [#[[LOCAL2_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:                 %[[VAL_69:.*]] = fir.convert %[[VAL_68]] : (i32) -> i64
-// TODO: local allocation assumed to always alias
-// CHECK:                 %[[VAL_70:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_70:.*]] = fir.load %[[VAL_34]]  {tbaa = [#[[LOCAL1_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:                 %[[VAL_71:.*]] = fir.convert %[[VAL_70]] : (i32) -> i64
 // CHECK:                 %[[VAL_72:.*]] = fir.box_addr %[[VAL_65]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>) -> !fir.heap<!fir.array<?x?x?xf32>>
 // CHECK:                 %[[VAL_73:.*]]:3 = fir.box_dims %[[VAL_65]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
@@ -311,11 +318,10 @@
 // CHECK:                 %[[VAL_76:.*]] = fir.shape_shift %[[VAL_73]]#0, %[[VAL_73]]#1, %[[VAL_74]]#0, %[[VAL_74]]#1, %[[VAL_75]]#0, %[[VAL_75]]#1 : (index, index, index, index, index, index) -> !fir.shapeshift<3>
 // CHECK:                 %[[VAL_77:.*]] = fir.array_coor %[[VAL_72]](%[[VAL_76]]) %[[VAL_67]], %[[VAL_69]], %[[VAL_71]] : (!fir.heap<!fir.array<?x?x?xf32>>, !fir.shapeshift<3>, i64, i64, i64) -> !fir.ref<f32>
 // CHECK:                 %[[VAL_78:.*]] = fir.load %[[VAL_77]] {tbaa = [#[[ARG_LOW_TAG]]]} : !fir.ref<f32>
-// CHECK:                 fir.store %[[VAL_78]] to %[[VAL_26]] : !fir.ref<f32>
+// CHECK:                 fir.store %[[VAL_78]] to %[[VAL_26]]  {tbaa = [#[[LOCAL4_ALLOC_TAG]]]} : !fir.ref<f32>
 // load from box tagged in CodeGen
 // CHECK:                 %[[VAL_79:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
-// TODO: local allocation assumed to always alias
-// CHECK:                 %[[VAL_80:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_80:.*]] = fir.load %[[VAL_32]]  {tbaa = [#[[LOCAL2_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:                 %[[VAL_81:.*]] = fir.convert %[[VAL_80]] : (i32) -> i64
 // CHECK:                 %[[VAL_82:.*]] = fir.box_addr %[[VAL_79]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
 // CHECK:                 %[[VAL_83:.*]]:3 = fir.box_dims %[[VAL_79]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
@@ -324,11 +330,9 @@
 // CHECK:                 %[[VAL_86:.*]] = fir.load %[[VAL_85]] {tbaa = [#[[DIRECT_A_TAG]]]} : !fir.ref<f32>
 // load from box
 // CHECK:                 %[[VAL_87:.*]] = fir.load %[[VAL_35]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
-// load from local allocation
-// CHECK:                 %[[VAL_88:.*]] = fir.load %[[VAL_30]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_88:.*]] = fir.load %[[VAL_30]] {tbaa = [#[[LOCAL3_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:                 %[[VAL_89:.*]] = fir.convert %[[VAL_88]] : (i32) -> i64
-// load from local allocation
-// CHECK:                 %[[VAL_90:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_90:.*]] = fir.load %[[VAL_34]] {tbaa = [#[[LOCAL1_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:                 %[[VAL_91:.*]] = fir.convert %[[VAL_90]] : (i32) -> i64
 // CHECK:                 %[[VAL_92:.*]] = fir.box_addr %[[VAL_87]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>) -> !fir.heap<!fir.array<?x?x?xf32>>
 // CHECK:                 %[[VAL_93:.*]]:3 = fir.box_dims %[[VAL_87]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
@@ -363,8 +367,7 @@
 // CHECK:                 %[[VAL_121:.*]] = fir.load %[[VAL_120]] {tbaa = [#[[ARG_Y_TAG]]]} : !fir.ref<f32>
 // CHECK:                 %[[VAL_122:.*]] = arith.subf %[[VAL_119]], %[[VAL_121]] fastmath<contract> : f32
 // CHECK:                 %[[VAL_123:.*]] = fir.no_reassoc %[[VAL_122]] : f32
-// load from local allocation
-// CHECK:                 %[[VAL_124:.*]] = fir.load %[[VAL_28]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_124:.*]] = fir.load %[[VAL_28]] {tbaa = [#[[LOCAL5_ALLOC_TAG]]]} : !fir.ref<f32>
 // CHECK:                 %[[VAL_125:.*]] = arith.mulf %[[VAL_123]], %[[VAL_124]] fastmath<contract> : f32
 // CHECK:                 %[[VAL_126:.*]] = arith.addf %[[VAL_115]], %[[VAL_125]] fastmath<contract> : f32
 // CHECK:                 %[[VAL_127:.*]] = fir.no_reassoc %[[VAL_126]] : f32
@@ -373,30 +376,24 @@
 // CHECK:                 fir.store %[[VAL_129]] to %[[VAL_97]] {tbaa = [#[[ARG_LOW_TAG]]]} : !fir.ref<f32>
 // CHECK:                 %[[VAL_130:.*]] = arith.addi %[[VAL_63]], %[[VAL_5]] : index
 // CHECK:                 %[[VAL_131:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
-// load from local allocation
-// CHECK:                 %[[VAL_132:.*]] = fir.load %[[VAL_30]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_132:.*]] = fir.load %[[VAL_30]] {tbaa = [#[[LOCAL3_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:                 %[[VAL_133:.*]] = arith.addi %[[VAL_132]], %[[VAL_131]] : i32
 // CHECK:                 fir.result %[[VAL_130]], %[[VAL_133]] : index, i32
 // CHECK:               }
-// store to local allocation
-// CHECK:               fir.store %[[VAL_134:.*]]#1 to %[[VAL_30]] : !fir.ref<i32>
+// CHECK:               fir.store %[[VAL_134:.*]]#1 to %[[VAL_30]] {tbaa = [#[[LOCAL3_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:               %[[VAL_135:.*]] = arith.addi %[[VAL_56]], %[[VAL_5]] : index
 // CHECK:               %[[VAL_136:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
-// local allocation:
-// CHECK:               %[[VAL_137:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
+// CHECK:               %[[VAL_137:.*]] = fir.load %[[VAL_32]] {tbaa = [#[[LOCAL2_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:               %[[VAL_138:.*]] = arith.addi %[[VAL_137]], %[[VAL_136]] : i32
 // CHECK:               fir.result %[[VAL_135]], %[[VAL_138]] : index, i32
 // CHECK:             }
-// local allocation:
-// CHECK:             fir.store %[[VAL_139:.*]]#1 to %[[VAL_32]] : !fir.ref<i32>
+// CHECK:             fir.store %[[VAL_139:.*]]#1 to %[[VAL_32]] {tbaa = [#[[LOCAL2_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:             %[[VAL_140:.*]] = arith.addi %[[VAL_47]], %[[VAL_5]] : index
 // CHECK:             %[[VAL_141:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
-// local allocation:
-// CHECK:             %[[VAL_142:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
+// CHECK:             %[[VAL_142:.*]] = fir.load %[[VAL_34]] {tbaa = [#[[LOCAL1_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:             %[[VAL_143:.*]] = arith.addi %[[VAL_142]], %[[VAL_141]] : i32
 // CHECK:             fir.result %[[VAL_140]], %[[VAL_143]] : index, i32
 // CHECK:           }
-// local allocation:
-// CHECK:           fir.store %[[VAL_144:.*]]#1 to %[[VAL_34]] : !fir.ref<i32>
+// CHECK:           fir.store %[[VAL_144:.*]]#1 to %[[VAL_34]] {tbaa = [#[[LOCAL1_ALLOC_TAG]]]} : !fir.ref<i32>
 // CHECK:           return
 // CHECK:         }
diff --git a/external/llvm-project/flang/test/Transforms/tbaa3.fir b/external/llvm-project/flang/test/Transforms/tbaa3.fir
index 32596080ca4a..97bf69da1b99 100644
--- a/external/llvm-project/flang/test/Transforms/tbaa3.fir
+++ b/external/llvm-project/flang/test/Transforms/tbaa3.fir
@@ -1,5 +1,5 @@
 // RUN: fir-opt --fir-add-alias-tags %s | FileCheck --check-prefixes=ALL,DEFAULT %s
-// RUN: fir-opt --fir-add-alias-tags --tbaa-local-alloc %s | FileCheck --check-prefixes=ALL,LOCAL %s
+// RUN: fir-opt --fir-add-alias-tags --local-alloc-tbaa %s | FileCheck --check-prefixes=ALL,LOCAL %s
 
 // Test AddAliasTagsPass creating sub-tree for TARGET/POINTER variables.
 
@@ -263,12 +263,12 @@ module {
     fir.store %cst to %67 : !fir.ref<f32>
     %68 = fir.array_coor %20(%5) %c1 : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
 // real :: local(10)
-// DEFAULT-NOT: fir.store{{.*}}tbaa
+// DEFAULT: fir.store{{.*}}tbaa
 // LOCAL: fir.store{{.*}}{tbaa = [#[[LOCALTAG]]]} : !fir.ref<f32>
     fir.store %cst to %68 : !fir.ref<f32>
     %69 = fir.array_coor %33(%5) %c1 : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, index) -> !fir.ref<f32>
 // real, target :: localt(10)
-// DEFAULT-NOT: fir.store{{.*}}tbaa
+// DEFAULT: fir.store{{.*}}tbaa
 // LOCAL: fir.store{{.*}}{tbaa = [#[[LOCALTTAG]]]} : !fir.ref<f32>
     fir.store %cst to %69 : !fir.ref<f32>
 // ALL-NOT: fir.load{{.*}}tbaa
@@ -278,7 +278,7 @@ module {
     %73 = fir.shape_shift %72#0, %72#1 : (index, index) -> !fir.shapeshift<1>
     %74 = fir.array_coor %71(%73) %c1 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
 // real, allocatable :: locala(:)
-// DEFAULT-NOT: fir.store{{.*}}tbaa
+// DEFAULT: fir.store{{.*}}tbaa
 // LOCAL: fir.store{{.*}}{tbaa = [#[[LOCALATAG]]]} : !fir.ref<f32>
     fir.store %cst to %74 : !fir.ref<f32>
 // ALL-NOT: fir.load{{.*}}tbaa
@@ -288,7 +288,7 @@ module {
     %78 = fir.shape_shift %77#0, %77#1 : (index, index) -> !fir.shapeshift<1>
     %79 = fir.array_coor %76(%78) %c1 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, index) -> !fir.ref<f32>
 // real, allocatable, target :: localat(:)
-// DEFAULT-NOT: fir.store{{.*}}tbaa
+// DEFAULT: fir.store{{.*}}tbaa
 // LOCAL: fir.store{{.*}}{tbaa = [#[[LOCALATTAG]]]} : !fir.ref<f32>
     fir.store %cst to %79 : !fir.ref<f32>
 // ALL-NOT: fir.load{{.*}}tbaa
@@ -297,7 +297,7 @@ module {
     %82 = fir.shift %81#0 : (index) -> !fir.shift<1>
     %83 = fir.array_coor %80(%82) %c1 : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shift<1>, index) -> !fir.ref<f32>
 // real, pointer :: localp(:)
-// DEFAULT-NOT: fir.store{{.*}}tbaa
+// DEFAULT: fir.store{{.*}}tbaa
 // LOCAL: fir.store{{.*}}{tbaa = [#[[TARGETTAG]]]} : !fir.ref<f32>
     fir.store %cst to %83 : !fir.ref<f32>
 // ALL-NOT: fir.load{{.*}}tbaa
diff --git a/external/llvm-project/flang/test/Transforms/tbaa4.fir b/external/llvm-project/flang/test/Transforms/tbaa4.fir
index 8b1f16b23a74..6fa8fff02b6a 100644
--- a/external/llvm-project/flang/test/Transforms/tbaa4.fir
+++ b/external/llvm-project/flang/test/Transforms/tbaa4.fir
@@ -1,6 +1,6 @@
 // Test TBAA tags for common and equivalence.
 // RUN: fir-opt --fir-add-alias-tags --split-input-file %s | FileCheck --check-prefixes=ALL,DEFAULT %s
-// RUN: fir-opt --fir-add-alias-tags --tbaa-local-alloc --split-input-file %s | FileCheck --check-prefixes=ALL,LOCAL %s
+// RUN: fir-opt --fir-add-alias-tags --local-alloc-tbaa --split-input-file %s | FileCheck --check-prefixes=ALL,LOCAL %s
 
 // ALL: #[[ROOT:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest_common">
 // ALL: #[[ANY:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
diff --git a/external/llvm-project/flang/tools/bbc/bbc.cpp b/external/llvm-project/flang/tools/bbc/bbc.cpp
index e1549362ad18..10d8979dd1f4 100644
--- a/external/llvm-project/flang/tools/bbc/bbc.cpp
+++ b/external/llvm-project/flang/tools/bbc/bbc.cpp
@@ -142,6 +142,12 @@ static llvm::cl::opt<bool>
                        llvm::cl::desc("enable openmp device compilation"),
                        llvm::cl::init(false));
 
+static llvm::cl::opt<bool>
+    deferDescMap("fdefer-desc-map",
+                 llvm::cl::desc("disable or enable OpenMP deference of mapping "
+                                "for top-level descriptors"),
+                 llvm::cl::init(false));
+
 static llvm::cl::opt<std::string> enableDoConcurrentToOpenMPConversion(
     "fdo-concurrent-to-openmp",
     llvm::cl::desc(
@@ -223,6 +229,11 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
+static llvm::cl::opt<bool>
+    disableCUDAWarpFunction("fcuda-disable-warp-function",
+                            llvm::cl::desc("Disable CUDA Warp Function"),
+                            llvm::cl::init(false));
+
 static llvm::cl::opt<std::string>
     enableGPUMode("gpu", llvm::cl::desc("Enable GPU Mode managed|unified"),
                   llvm::cl::init(""));
@@ -329,6 +340,7 @@ static llvm::LogicalResult runOpenMPPasses(mlir::ModuleOp mlirModule) {
           .Case("host", DoConcurrentMappingKind::DCMK_Host)
           .Case("device", DoConcurrentMappingKind::DCMK_Device)
           .Default(DoConcurrentMappingKind::DCMK_None);
+  opts.deferDescMap = deferDescMap;
 
   fir::createOpenMPFIRPassPipeline(pm, opts);
   (void)mlir::applyPassManagerCLOptions(pm);
@@ -429,6 +441,8 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setStackRepackArrays(stackRepackArrays);
   loweringOptions.setRepackArrays(repackArrays);
   loweringOptions.setRepackArraysWhole(repackArraysWhole);
+  if (enableCUDA)
+    loweringOptions.setCUDARuntimeCheck(true);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   Fortran::frontend::TargetOptions targetOpts;
   Fortran::frontend::CodeGenOptions cgOpts;
@@ -600,6 +614,11 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
+  if (disableCUDAWarpFunction) {
+    options.features.Enable(
+        Fortran::common::LanguageFeature::CudaWarpMatchFunction, false);
+  }
+
   if (enableGPUMode == "managed") {
     options.features.Enable(Fortran::common::LanguageFeature::CudaManaged);
   } else if (enableGPUMode == "unified") {
diff --git a/external/llvm-project/flang/tools/flang-driver/driver.cpp b/external/llvm-project/flang/tools/flang-driver/driver.cpp
index 35cc2efc0ac0..3a2dffc66428 100644
--- a/external/llvm-project/flang/tools/flang-driver/driver.cpp
+++ b/external/llvm-project/flang/tools/flang-driver/driver.cpp
@@ -111,6 +111,13 @@ int main(int argc, const char **argv) {
     }
   }
 
+  llvm::StringSet<> savedStrings;
+  // Handle FCC_OVERRIDE_OPTIONS, used for editing a command line behind the
+  // scenes.
+  if (const char *overrideStr = ::getenv("FCC_OVERRIDE_OPTIONS"))
+    clang::driver::applyOverrideOptions(args, overrideStr, savedStrings,
+                                        "FCC_OVERRIDE_OPTIONS", &llvm::errs());
+
   // Not in the frontend mode - continue in the compiler driver mode.
 
   // Create DiagnosticsEngine for the compiler driver
diff --git a/external/llvm-project/flang/unittests/Common/CMakeLists.txt b/external/llvm-project/flang/unittests/Common/CMakeLists.txt
index bda02ed29a5e..3149cb9f7bc4 100644
--- a/external/llvm-project/flang/unittests/Common/CMakeLists.txt
+++ b/external/llvm-project/flang/unittests/Common/CMakeLists.txt
@@ -1,3 +1,6 @@
 add_flang_unittest(FlangCommonTests
+  EnumClassTests.cpp
   FastIntSetTest.cpp
+  FortranFeaturesTest.cpp
 )
+target_link_libraries(FlangCommonTests PRIVATE FortranSupport)
diff --git a/external/llvm-project/flang/unittests/Common/EnumClassTests.cpp b/external/llvm-project/flang/unittests/Common/EnumClassTests.cpp
new file mode 100644
index 000000000000..eafba542a3b8
--- /dev/null
+++ b/external/llvm-project/flang/unittests/Common/EnumClassTests.cpp
@@ -0,0 +1,34 @@
+//===-- flang/unittests/Common/FastIntSetTest.cpp ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+#include "flang/Common/enum-class.h"
+
+namespace Fortran::common {
+
+ENUM_CLASS(TestEnum, One, Two, Three)
+
+TEST(EnumClassTest, EnumToString) {
+  ASSERT_EQ(EnumToString(TestEnum::One), "One");
+  ASSERT_EQ(EnumToString(TestEnum::Two), "Two");
+  ASSERT_EQ(EnumToString(TestEnum::Three), "Three");
+}
+
+TEST(EnumClassTest, EnumClassForEach) {
+  std::string result;
+  bool first{true};
+  ForEachTestEnum([&](auto e) {
+    if (!first) {
+      result += ", ";
+    }
+    result += EnumToString(e);
+    first = false;
+  });
+  ASSERT_EQ(result, "One, Two, Three");
+}
+} // namespace Fortran::common
diff --git a/external/llvm-project/flang/unittests/Common/FortranFeaturesTest.cpp b/external/llvm-project/flang/unittests/Common/FortranFeaturesTest.cpp
new file mode 100644
index 000000000000..afd677f69a16
--- /dev/null
+++ b/external/llvm-project/flang/unittests/Common/FortranFeaturesTest.cpp
@@ -0,0 +1,561 @@
+//===-- flang/unittests/Common/FastIntSetTest.cpp ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+#include "flang/Support/Fortran-features.h"
+
+namespace Fortran::common::details {
+
+std::string CamelCaseToLowerCaseHyphenated(std::string_view x);
+
+TEST(FortranFeaturesTest, CamelCaseToLowerCaseHyphenated) {
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BackslashEscapes)),
+      "backslash-escapes");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::OldDebugLines)),
+      "old-debug-lines");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                LanguageFeature::FixedFormContinuationWithColumn1Ampersand)),
+      "fixed-form-continuation-with-column1-ampersand");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::LogicalAbbreviations)),
+      "logical-abbreviations");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::XOROperator)),
+      "xor-operator");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::PunctuationInNames)),
+      "punctuation-in-names");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::OptionalFreeFormSpace)),
+      "optional-free-form-space");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BOZExtensions)),
+      "boz-extensions");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::EmptyStatement)),
+      "empty-statement");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AlternativeNE)),
+      "alternative-ne");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ExecutionPartNamelist)),
+      "execution-part-namelist");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::DECStructures)),
+      "dec-structures");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::DoubleComplex)),
+      "double-complex");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::Byte)),
+      "byte");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::StarKind)),
+      "star-kind");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ExponentMatchingKindParam)),
+      "exponent-matching-kind-param");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::QuadPrecision)),
+      "quad-precision");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::SlashInitialization)),
+      "slash-initialization");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::TripletInArrayConstructor)),
+      "triplet-in-array-constructor");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::MissingColons)),
+      "missing-colons");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::SignedComplexLiteral)),
+      "signed-complex-literal");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::OldStyleParameter)),
+      "old-style-parameter");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ComplexConstructor)),
+      "complex-constructor");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::PercentLOC)),
+      "percent-loc");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::SignedMultOperand)),
+      "signed-mult-operand");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::FileName)),
+      "file-name");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::Carriagecontrol)),
+      "carriagecontrol");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::Convert)),
+      "convert");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::Dispose)),
+      "dispose");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::IOListLeadingComma)),
+      "io-list-leading-comma");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AbbreviatedEditDescriptor)),
+      "abbreviated-edit-descriptor");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ProgramParentheses)),
+      "program-parentheses");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::PercentRefAndVal)),
+      "percent-ref-and-val");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::OmitFunctionDummies)),
+      "omit-function-dummies");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::CrayPointer)),
+      "cray-pointer");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::Hollerith)),
+      "hollerith");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ArithmeticIF)),
+      "arithmetic-if");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::Assign)),
+      "assign");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AssignedGOTO)),
+      "assigned-goto");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::Pause)),
+      "pause");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::OpenACC)),
+      "open-acc");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::OpenMP)),
+      "open-mp");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::CUDA)),
+      "cuda");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::CruftAfterAmpersand)),
+      "cruft-after-ampersand");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ClassicCComments)),
+      "classic-c-comments");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AdditionalFormats)),
+      "additional-formats");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BigIntLiterals)),
+      "big-int-literals");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::RealDoControls)),
+      "real-do-controls");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::EquivalenceNumericWithCharacter)),
+      "equivalence-numeric-with-character");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::EquivalenceNonDefaultNumeric)),
+      "equivalence-non-default-numeric");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::EquivalenceSameNonSequence)),
+      "equivalence-same-non-sequence");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AdditionalIntrinsics)),
+      "additional-intrinsics");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AnonymousParents)),
+      "anonymous-parents");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::OldLabelDoEndStatements)),
+      "old-label-do-end-statements");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::LogicalIntegerAssignment)),
+      "logical-integer-assignment");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::EmptySourceFile)),
+      "empty-source-file");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ProgramReturn)),
+      "program-return");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ImplicitNoneTypeNever)),
+      "implicit-none-type-never");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ImplicitNoneTypeAlways)),
+      "implicit-none-type-always");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ImplicitNoneExternal)),
+      "implicit-none-external");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ForwardRefImplicitNone)),
+      "forward-ref-implicit-none");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::OpenAccessAppend)),
+      "open-access-append");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BOZAsDefaultInteger)),
+      "boz-as-default-integer");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::DistinguishableSpecifics)),
+      "distinguishable-specifics");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::DefaultSave)),
+      "default-save");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::PointerInSeqType)),
+      "pointer-in-seq-type");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::NonCharacterFormat)),
+      "non-character-format");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::SaveMainProgram)),
+      "save-main-program");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::SaveBigMainProgramVariables)),
+      "save-big-main-program-variables");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::DistinctArrayConstructorLengths)),
+      "distinct-array-constructor-lengths");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::PPCVector)),
+      "ppc-vector");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::RelaxedIntentInChecking)),
+      "relaxed-intent-in-checking");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ForwardRefImplicitNoneData)),
+      "forward-ref-implicit-none-data");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::NullActualForAllocatable)),
+      "null-actual-for-allocatable");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                LanguageFeature::ActualIntegerConvertedToSmallerKind)),
+      "actual-integer-converted-to-smaller-kind");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::HollerithOrCharacterAsBOZ)),
+      "hollerith-or-character-as-boz");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BindingAsProcedure)),
+      "binding-as-procedure");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::StatementFunctionExtensions)),
+      "statement-function-extensions");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                LanguageFeature::UseGenericIntrinsicWhenSpecificDoesntMatch)),
+      "use-generic-intrinsic-when-specific-doesnt-match");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::DataStmtExtensions)),
+      "data-stmt-extensions");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::RedundantContiguous)),
+      "redundant-contiguous");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::RedundantAttribute)),
+      "redundant-attribute");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::InitBlankCommon)),
+      "init-blank-common");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::EmptyBindCDerivedType)),
+      "empty-bind-c-derived-type");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::MiscSourceExtensions)),
+      "misc-source-extensions");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AllocateToOtherLength)),
+      "allocate-to-other-length");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::LongNames)),
+      "long-names");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::IntrinsicAsSpecific)),
+      "intrinsic-as-specific");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BenignNameClash)),
+      "benign-name-clash");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BenignRedundancy)),
+      "benign-redundancy");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                LanguageFeature::NullMoldAllocatableComponentValue)),
+      "null-mold-allocatable-component-value");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::NopassScalarBase)),
+      "nopass-scalar-base");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::MiscUseExtensions)),
+      "misc-use-extensions");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ImpliedDoIndexScope)),
+      "implied-do-index-scope");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::DistinctCommonSizes)),
+      "distinct-common-sizes");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::OddIndexVariableRestrictions)),
+      "odd-index-variable-restrictions");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::IndistinguishableSpecifics)),
+      "indistinguishable-specifics");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::SubroutineAndFunctionSpecifics)),
+      "subroutine-and-function-specifics");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::EmptySequenceType)),
+      "empty-sequence-type");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::NonSequenceCrayPointee)),
+      "non-sequence-cray-pointee");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BranchIntoConstruct)),
+      "branch-into-construct");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::BadBranchTarget)),
+      "bad-branch-target");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::HollerithPolymorphic)),
+      "hollerith-polymorphic");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ListDirectedSize)),
+      "list-directed-size");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::NonBindCInteroperability)),
+      "non-bind-c-interoperability");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::CudaManaged)),
+      "cuda-managed");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::CudaUnified)),
+      "cuda-unified");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::
+                    PolymorphicActualAllocatableOrPointerToMonomorphicDummy)),
+      "polymorphic-actual-allocatable-or-pointer-to-monomorphic-dummy");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::RelaxedPureDummy)),
+      "relaxed-pure-dummy");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                LanguageFeature::UndefinableAsynchronousOrVolatileActual)),
+      "undefinable-asynchronous-or-volatile-actual");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AutomaticInMainProgram)),
+      "automatic-in-main-program");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::PrintCptr)),
+      "print-cptr");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::SavedLocalInSpecExpr)),
+      "saved-local-in-spec-expr");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::PrintNamelist)),
+      "print-namelist");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                LanguageFeature::AssumedRankPassedToNonAssumedRank)),
+      "assumed-rank-passed-to-non-assumed-rank");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::IgnoreIrrelevantAttributes)),
+      "ignore-irrelevant-attributes");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(LanguageFeature::Unsigned)),
+      "unsigned");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::AmbiguousStructureConstructor)),
+      "ambiguous-structure-constructor");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ContiguousOkForSeqAssociation)),
+      "contiguous-ok-for-seq-association");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(LanguageFeature::ForwardRefExplicitTypeDummy)),
+      "forward-ref-explicit-type-dummy");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::Portability)),
+      "portability");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::PointerToUndefinable)),
+      "pointer-to-undefinable");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::NonTargetPassedToTarget)),
+      "non-target-passed-to-target");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::PointerToPossibleNoncontiguous)),
+      "pointer-to-possible-noncontiguous");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ShortCharacterActual)),
+      "short-character-actual");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ShortArrayActual)),
+      "short-array-actual");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ImplicitInterfaceActual)),
+      "implicit-interface-actual");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::PolymorphicTransferArg)),
+      "polymorphic-transfer-arg");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::PointerComponentTransferArg)),
+      "pointer-component-transfer-arg");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::TransferSizePresence)),
+      "transfer-size-presence");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::F202XAllocatableBreakingChange)),
+      "f202-x-allocatable-breaking-change");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::OptionalMustBePresent)),
+      "optional-must-be-present");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::CommonBlockPadding)),
+      "common-block-padding");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::LogicalVsCBool)),
+      "logical-vs-c-bool");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::BindCCharLength)),
+      "bind-c-char-length");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ProcDummyArgShapes)),
+      "proc-dummy-arg-shapes");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ExternalNameConflict)),
+      "external-name-conflict");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::FoldingException)),
+      "folding-exception");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::FoldingAvoidsRuntimeCrash)),
+      "folding-avoids-runtime-crash");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::FoldingValueChecks)),
+      "folding-value-checks");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::FoldingFailure)),
+      "folding-failure");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::FoldingLimit)),
+      "folding-limit");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::Interoperability)),
+      "interoperability");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::CharacterInteroperability)),
+      "character-interoperability");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::Bounds)),
+      "bounds");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::Preprocessing)),
+      "preprocessing");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::Scanning)),
+      "scanning");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::OpenAccUsage)),
+      "open-acc-usage");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ProcPointerCompatibility)),
+      "proc-pointer-compatibility");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::VoidMold)),
+      "void-mold");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::KnownBadImplicitInterface)),
+      "known-bad-implicit-interface");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::EmptyCase)),
+      "empty-case");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::CaseOverflow)),
+      "case-overflow");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::CUDAUsage)),
+      "cuda-usage");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::IgnoreTKRUsage)),
+      "ignore-tkr-usage");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ExternalInterfaceMismatch)),
+      "external-interface-mismatch");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::DefinedOperatorArgs)),
+      "defined-operator-args");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::Final)),
+      "final");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::ZeroDoStep)),
+      "zero-do-step");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::UnusedForallIndex)),
+      "unused-forall-index");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::OpenMPUsage)),
+      "open-mp-usage");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::DataLength)),
+      "data-length");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::IgnoredDirective)),
+      "ignored-directive");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::HomonymousSpecific)),
+      "homonymous-specific");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::HomonymousResult)),
+      "homonymous-result");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::IgnoredIntrinsicFunctionType)),
+      "ignored-intrinsic-function-type");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::PreviousScalarUse)),
+      "previous-scalar-use");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::RedeclaredInaccessibleComponent)),
+      "redeclared-inaccessible-component");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::ImplicitShared)),
+      "implicit-shared");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::IndexVarRedefinition)),
+      "index-var-redefinition");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::IncompatibleImplicitInterfaces)),
+      "incompatible-implicit-interfaces");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::VectorSubscriptFinalization)),
+      "vector-subscript-finalization");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::UndefinedFunctionResult)),
+      "undefined-function-result");
+  EXPECT_EQ(
+      CamelCaseToLowerCaseHyphenated(EnumToString(UsageWarning::UselessIomsg)),
+      "useless-iomsg");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::MismatchingDummyProcedure)),
+      "mismatching-dummy-procedure");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::SubscriptedEmptyArray)),
+      "subscripted-empty-array");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::UnsignedLiteralTruncation)),
+      "unsigned-literal-truncation");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                UsageWarning::CompatibleDeclarationsFromDistinctModules)),
+      "compatible-declarations-from-distinct-modules");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                UsageWarning::NullActualForDefaultIntentAllocatable)),
+      "null-actual-for-default-intent-allocatable");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(EnumToString(
+                UsageWarning::UseAssociationIntoSameNameSubprogram)),
+      "use-association-into-same-name-subprogram");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::HostAssociatedIntentOutInSpecExpr)),
+      "host-associated-intent-out-in-spec-expr");
+  EXPECT_EQ(CamelCaseToLowerCaseHyphenated(
+                EnumToString(UsageWarning::NonVolatilePointerToVolatile)),
+      "non-volatile-pointer-to-volatile");
+}
+
+} // namespace Fortran::common::details
diff --git a/external/llvm-project/libc/CMakeLists.txt b/external/llvm-project/libc/CMakeLists.txt
index f21fc2fba730..9907adfc55a5 100644
--- a/external/llvm-project/libc/CMakeLists.txt
+++ b/external/llvm-project/libc/CMakeLists.txt
@@ -135,6 +135,7 @@ endif()
 option(LLVM_LIBC_FULL_BUILD "Build and test LLVM libc as if it is the full libc" ${default_to_full_build})
 option(LLVM_LIBC_IMPLEMENTATION_DEFINED_TEST_BEHAVIOR "Build LLVM libc tests assuming our implementation-defined behavior" ON)
 option(LLVM_LIBC_ENABLE_LINTING "Enables linting of libc source files" OFF)
+option(LLVM_LIBC_ALL_HEADERS "Outputs all functions in header files, regardless of whether they are enabled on this target" OFF)
 
 option(LIBC_CONFIG_PATH "The path to user provided folder that configures the build for the target system." OFF)
 
diff --git a/external/llvm-project/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/external/llvm-project/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 0facb0b9be0c..a98e7276bef8 100644
--- a/external/llvm-project/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/external/llvm-project/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -106,6 +106,10 @@ function(_get_compile_options_from_config output_var)
     list(APPEND config_options "-DLIBC_MATH=${LIBC_CONF_MATH_OPTIMIZATIONS}")
   endif()
 
+  if(LIBC_CONF_ERRNO_MODE)
+    set(APPEND config_options "-DLIBC_ERRNO_MODE=${LIBC_CONF_ERRNO_MODE}")
+  endif()
+
   set(${output_var} ${config_options} PARENT_SCOPE)
 endfunction(_get_compile_options_from_config)
 
diff --git a/external/llvm-project/libc/cmake/modules/LLVMLibCHeaderRules.cmake b/external/llvm-project/libc/cmake/modules/LLVMLibCHeaderRules.cmake
index 99f90244e013..01c288f0b919 100644
--- a/external/llvm-project/libc/cmake/modules/LLVMLibCHeaderRules.cmake
+++ b/external/llvm-project/libc/cmake/modules/LLVMLibCHeaderRules.cmake
@@ -97,8 +97,13 @@ function(add_gen_header target_name)
   set(out_file ${LIBC_INCLUDE_DIR}/${relative_path})
   set(dep_file "${out_file}.d")
   set(yaml_file ${CMAKE_SOURCE_DIR}/${ADD_GEN_HDR_YAML_FILE})
+  
+  if(LLVM_LIBC_ALL_HEADERS)
+    set(entry_points "")
+  else()
+    set(entry_points "${TARGET_ENTRYPOINT_NAME_LIST}")
+  endif()
 
-  set(entry_points "${TARGET_ENTRYPOINT_NAME_LIST}")
   list(TRANSFORM entry_points PREPEND "--entry-point=")
 
   add_custom_command(
diff --git a/external/llvm-project/libc/cmake/modules/LLVMLibCLibraryRules.cmake b/external/llvm-project/libc/cmake/modules/LLVMLibCLibraryRules.cmake
index 80351792fb18..b464c189420b 100644
--- a/external/llvm-project/libc/cmake/modules/LLVMLibCLibraryRules.cmake
+++ b/external/llvm-project/libc/cmake/modules/LLVMLibCLibraryRules.cmake
@@ -19,8 +19,7 @@ function(collect_object_file_deps target result)
     return()
   endif()
 
-  if(${target_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE} OR
-     ${target_type} STREQUAL ${ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE})
+  if(${target_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE})
     set(entrypoint_target ${target})
     get_target_property(is_alias ${entrypoint_target} "IS_ALIAS")
     if(is_alias)
@@ -55,8 +54,7 @@ function(get_all_object_file_deps result fq_deps_list)
   foreach(dep ${fq_deps_list})
     get_target_property(dep_type ${dep} "TARGET_TYPE")
     if(NOT ((${dep_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE}) OR
-            (${dep_type} STREQUAL ${ENTRYPOINT_EXT_TARGET_TYPE}) OR
-            (${dep_type} STREQUAL ${ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE})))
+            (${dep_type} STREQUAL ${ENTRYPOINT_EXT_TARGET_TYPE})))
       message(FATAL_ERROR "Dependency '${dep}' of 'add_entrypoint_collection' is "
                           "not an 'add_entrypoint_object' or 'add_entrypoint_external' target.")
     endif()
@@ -64,8 +62,7 @@ function(get_all_object_file_deps result fq_deps_list)
     list(APPEND all_deps ${recursive_deps})
     # Add the entrypoint object target explicitly as collect_object_file_deps
     # only collects object files from non-entrypoint targets.
-    if(${dep_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE} OR
-       ${dep_type} STREQUAL ${ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE})
+    if(${dep_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE})
       set(entrypoint_target ${dep})
       get_target_property(is_alias ${entrypoint_target} "IS_ALIAS")
       if(is_alias)
diff --git a/external/llvm-project/libc/cmake/modules/LLVMLibCObjectRules.cmake b/external/llvm-project/libc/cmake/modules/LLVMLibCObjectRules.cmake
index 142778d9ea1c..805da91284ce 100644
--- a/external/llvm-project/libc/cmake/modules/LLVMLibCObjectRules.cmake
+++ b/external/llvm-project/libc/cmake/modules/LLVMLibCObjectRules.cmake
@@ -111,7 +111,6 @@ function(add_object_library target_name)
 endfunction(add_object_library)
 
 set(ENTRYPOINT_OBJ_TARGET_TYPE "ENTRYPOINT_OBJ")
-set(ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE "ENTRYPOINT_OBJ_VENDOR")
 
 # A rule for entrypoint object targets.
 # Usage:
@@ -129,20 +128,13 @@ set(ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE "ENTRYPOINT_OBJ_VENDOR")
 function(create_entrypoint_object fq_target_name)
   cmake_parse_arguments(
     "ADD_ENTRYPOINT_OBJ"
-    "ALIAS;REDIRECTED;VENDOR" # Optional argument
+    "ALIAS;REDIRECTED" # Optional argument
     "NAME;CXX_STANDARD" # Single value arguments
     "SRCS;HDRS;DEPENDS;COMPILE_OPTIONS;FLAGS"  # Multi value arguments
     ${ARGN}
   )
 
   set(entrypoint_target_type ${ENTRYPOINT_OBJ_TARGET_TYPE})
-  if(${ADD_ENTRYPOINT_OBJ_VENDOR})
-    # TODO: We currently rely on external definitions of certain math functions
-    # provided by GPU vendors like AMD or Nvidia. We need to mark these so we
-    # don't end up running tests on these. In the future all of these should be
-    # implemented and this can be removed.
-    set(entrypoint_target_type ${ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE})
-  endif()
   list(FIND TARGET_ENTRYPOINT_NAME_LIST ${ADD_ENTRYPOINT_OBJ_NAME} entrypoint_name_index)
   if(${entrypoint_name_index} EQUAL -1)
     add_custom_target(${fq_target_name})
@@ -186,8 +178,8 @@ function(create_entrypoint_object fq_target_name)
     endif()
 
     get_target_property(obj_type ${fq_dep_name} "TARGET_TYPE")
-    if((NOT obj_type) OR (NOT (${obj_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE} OR
-                               ${obj_type} STREQUAL ${ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE})))
+    if((NOT obj_type) OR (NOT ${obj_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE}))
+                              
       message(FATAL_ERROR "The aliasee of an entrypoint alias should be an entrypoint.")
     endif()
 
diff --git a/external/llvm-project/libc/cmake/modules/LLVMLibCTestRules.cmake b/external/llvm-project/libc/cmake/modules/LLVMLibCTestRules.cmake
index 54f337ecf099..1cd09816e223 100644
--- a/external/llvm-project/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/external/llvm-project/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -168,10 +168,6 @@ function(get_object_files_for_test result skipped_entrypoints_list)
           endif()
           list(APPEND dep_obj ${object_file_raw})
         endif()
-      elseif(${dep_type} STREQUAL ${ENTRYPOINT_OBJ_VENDOR_TARGET_TYPE})
-        # Skip tests for externally implemented entrypoints.
-        list(APPEND dep_skip ${dep})
-        list(REMOVE_ITEM dep_obj ${dep})
       endif()
 
       set_target_properties(${dep} PROPERTIES
diff --git a/external/llvm-project/libc/config/config.json b/external/llvm-project/libc/config/config.json
index bfe956855cb5..d53b2936edb0 100644
--- a/external/llvm-project/libc/config/config.json
+++ b/external/llvm-project/libc/config/config.json
@@ -2,7 +2,7 @@
   "errno": {
     "LIBC_CONF_ERRNO_MODE": {
       "value": "LIBC_ERRNO_MODE_DEFAULT",
-      "doc": "The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, and LIBC_ERRNO_MODE_SYSTEM."
+      "doc": "The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, LIBC_ERRNO_MODE_SYSTEM, and LIBC_ERRNO_MODE_SYSTEM_INLINE."
     }
   },
   "printf": {
diff --git a/external/llvm-project/libc/config/gpu/amdgpu/entrypoints.txt b/external/llvm-project/libc/config/gpu/amdgpu/entrypoints.txt
index 15c4b51cb241..f41ebdc59f6b 100644
--- a/external/llvm-project/libc/config/gpu/amdgpu/entrypoints.txt
+++ b/external/llvm-project/libc/config/gpu/amdgpu/entrypoints.txt
@@ -282,18 +282,14 @@ set(TARGET_LIBM_ENTRYPOINTS
     # math.h entrypoints
     libc.src.math.acos
     libc.src.math.acosf
-    libc.src.math.acosh
-    libc.src.math.acoshf
     libc.src.math.asin
     libc.src.math.asinf
-    libc.src.math.asinh
     libc.src.math.asinhf
     libc.src.math.atan
     libc.src.math.atan2
     libc.src.math.atan2f
     libc.src.math.atan2l
     libc.src.math.atanf
-    libc.src.math.atanh
     libc.src.math.atanhf
     libc.src.math.canonicalize
     libc.src.math.canonicalizef
@@ -308,14 +304,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.copysignl
     libc.src.math.cos
     libc.src.math.cosf
-    libc.src.math.cosh
     libc.src.math.coshf
     libc.src.math.cospif
     libc.src.math.ddivl
     libc.src.math.dfmal
     libc.src.math.dmull
     libc.src.math.dsqrtl
-    libc.src.math.erf
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
@@ -456,8 +450,6 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.nextupl
     libc.src.math.pow
     libc.src.math.powf
-    libc.src.math.powi
-    libc.src.math.powif
     libc.src.math.remainder
     libc.src.math.remainderf
     libc.src.math.remainderl
@@ -489,7 +481,6 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.sincos
     libc.src.math.sincosf
     libc.src.math.sinf
-    libc.src.math.sinh
     libc.src.math.sinhf
     libc.src.math.sinpif
     libc.src.math.sqrt
@@ -497,7 +488,6 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.sqrtl
     libc.src.math.tan
     libc.src.math.tanf
-    libc.src.math.tanh
     libc.src.math.tanhf
     libc.src.math.tgamma
     libc.src.math.tgammaf
diff --git a/external/llvm-project/libc/config/gpu/nvptx/entrypoints.txt b/external/llvm-project/libc/config/gpu/nvptx/entrypoints.txt
index 36c5ce193a57..2b8c29fda3dc 100644
--- a/external/llvm-project/libc/config/gpu/nvptx/entrypoints.txt
+++ b/external/llvm-project/libc/config/gpu/nvptx/entrypoints.txt
@@ -280,20 +280,16 @@ set(TARGET_LIBC_ENTRYPOINTS
 
 set(TARGET_LIBM_ENTRYPOINTS
     # math.h entrypoints
-    libc.src.math.acos
     libc.src.math.acosf
-    libc.src.math.acosh
     libc.src.math.acoshf
     libc.src.math.asin
     libc.src.math.asinf
-    libc.src.math.asinh
     libc.src.math.asinhf
     libc.src.math.atan
     libc.src.math.atan2
     libc.src.math.atan2f
     libc.src.math.atan2l
     libc.src.math.atanf
-    libc.src.math.atanh
     libc.src.math.atanhf
     libc.src.math.canonicalize
     libc.src.math.canonicalizef
@@ -308,14 +304,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.copysignl
     libc.src.math.cos
     libc.src.math.cosf
-    libc.src.math.cosh
     libc.src.math.coshf
     libc.src.math.cospif
     libc.src.math.ddivl
     libc.src.math.dfmal
     libc.src.math.dmull
     libc.src.math.dsqrtl
-    libc.src.math.erf
     libc.src.math.erff
     libc.src.math.exp
     libc.src.math.exp10
@@ -457,8 +451,6 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.nextupl
     libc.src.math.pow
     libc.src.math.powf
-    libc.src.math.powi
-    libc.src.math.powif
     libc.src.math.remainder
     libc.src.math.remainderf
     libc.src.math.remainderl
@@ -490,7 +482,6 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.sincos
     libc.src.math.sincosf
     libc.src.math.sinf
-    libc.src.math.sinh
     libc.src.math.sinhf
     libc.src.math.sinpif
     libc.src.math.sqrt
@@ -498,7 +489,6 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.sqrtl
     libc.src.math.tan
     libc.src.math.tanf
-    libc.src.math.tanh
     libc.src.math.tanhf
     libc.src.math.tgamma
     libc.src.math.tgammaf
diff --git a/external/llvm-project/libc/config/linux/aarch64/entrypoints.txt b/external/llvm-project/libc/config/linux/aarch64/entrypoints.txt
index 520046f768b5..9e042cd4a8ac 100644
--- a/external/llvm-project/libc/config/linux/aarch64/entrypoints.txt
+++ b/external/llvm-project/libc/config/linux/aarch64/entrypoints.txt
@@ -245,6 +245,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     # https://github.com/llvm/llvm-project/issues/80060
     # libc.src.sys.epoll.epoll_pwait2
 
+    # sys/ioctl.h entrypoints
+    libc.src.sys.ioctl.ioctl
+
     # sys/mman.h entrypoints
     libc.src.sys.mman.madvise
     libc.src.sys.mman.mincore
@@ -969,6 +972,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdio.getc_unlocked
     libc.src.stdio.getchar
     libc.src.stdio.getchar_unlocked
+    libc.src.stdio.perror
     libc.src.stdio.putc
     libc.src.stdio.putchar
     libc.src.stdio.puts
diff --git a/external/llvm-project/libc/config/linux/arm/entrypoints.txt b/external/llvm-project/libc/config/linux/arm/entrypoints.txt
index 7432a7e912e8..1161ae260be2 100644
--- a/external/llvm-project/libc/config/linux/arm/entrypoints.txt
+++ b/external/llvm-project/libc/config/linux/arm/entrypoints.txt
@@ -172,6 +172,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdlib.free
     libc.src.stdlib.malloc
 
+    # sys/ioctl.h entrypoints
+    libc.src.sys.ioctl.ioctl
+
     # sys/mman.h entrypoints
     libc.src.sys.mman.mmap
     libc.src.sys.mman.munmap
diff --git a/external/llvm-project/libc/config/linux/riscv/entrypoints.txt b/external/llvm-project/libc/config/linux/riscv/entrypoints.txt
index 0b645a2d2fb8..db8f8a7cf0b7 100644
--- a/external/llvm-project/libc/config/linux/riscv/entrypoints.txt
+++ b/external/llvm-project/libc/config/linux/riscv/entrypoints.txt
@@ -246,6 +246,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     # https://github.com/llvm/llvm-project/issues/80060
     # libc.src.sys.epoll.epoll_pwait2
 
+    # sys/ioctl.h entrypoints
+    libc.src.sys.ioctl.ioctl
+
     # sys/mman.h entrypoints
     libc.src.sys.mman.madvise
     libc.src.sys.mman.mincore
@@ -1095,6 +1098,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdio.getc_unlocked
     libc.src.stdio.getchar
     libc.src.stdio.getchar_unlocked
+    libc.src.stdio.perror
     libc.src.stdio.putc
     libc.src.stdio.putchar
     libc.src.stdio.puts
diff --git a/external/llvm-project/libc/config/linux/x86_64/entrypoints.txt b/external/llvm-project/libc/config/linux/x86_64/entrypoints.txt
index 545b9227349f..aa2079faed40 100644
--- a/external/llvm-project/libc/config/linux/x86_64/entrypoints.txt
+++ b/external/llvm-project/libc/config/linux/x86_64/entrypoints.txt
@@ -246,6 +246,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     # https://github.com/llvm/llvm-project/issues/80060
     # libc.src.sys.epoll.epoll_pwait2
 
+    # sys/ioctl.h entrypoints
+    libc.src.sys.ioctl.ioctl
+
     # sys/mman.h entrypoints
     libc.src.sys.mman.madvise
     libc.src.sys.mman.mincore
@@ -364,12 +367,23 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.btowc
     libc.src.wchar.wcslen
     libc.src.wchar.wctob
+    libc.src.wchar.wmemmove
     libc.src.wchar.wmemset
     libc.src.wchar.wcschr
+    libc.src.wchar.wcsncmp
+    libc.src.wchar.wcscmp
     libc.src.wchar.wcspbrk
+    libc.src.wchar.wcsrchr
     libc.src.wchar.wcsspn
     libc.src.wchar.wmemcmp
+    libc.src.wchar.wmempcpy
     libc.src.wchar.wmemcpy
+    libc.src.wchar.wcsncpy
+    libc.src.wchar.wcscat
+    libc.src.wchar.wcsstr
+    libc.src.wchar.wcsncat
+    libc.src.wchar.wcscpy
+    libc.src.wchar.wmemchr
 
     # sys/uio.h entrypoints
     libc.src.sys.uio.writev
@@ -1103,6 +1117,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.stdio.getc_unlocked
     libc.src.stdio.getchar
     libc.src.stdio.getchar_unlocked
+    libc.src.stdio.perror
     libc.src.stdio.putc
     libc.src.stdio.putchar
     libc.src.stdio.puts
diff --git a/external/llvm-project/libc/docs/configure.rst b/external/llvm-project/libc/docs/configure.rst
index 8d53390ae19b..109412225634 100644
--- a/external/llvm-project/libc/docs/configure.rst
+++ b/external/llvm-project/libc/docs/configure.rst
@@ -29,7 +29,7 @@ to learn about the defaults for your platform and target.
     - ``LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR``: Enable -fstack-protector-strong to defend against stack smashing attack.
     - ``LIBC_CONF_KEEP_FRAME_POINTER``: Keep frame pointer in functions for better debugging experience.
 * **"errno" options**
-    - ``LIBC_CONF_ERRNO_MODE``: The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, and LIBC_ERRNO_MODE_SYSTEM.
+    - ``LIBC_CONF_ERRNO_MODE``: The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, LIBC_ERRNO_MODE_SYSTEM, and LIBC_ERRNO_MODE_SYSTEM_INLINE.
 * **"general" options**
     - ``LIBC_ADD_NULL_CHECKS``: Add nullptr checks in the library's implementations to some functions for which passing nullptr is undefined behavior.
 * **"math" options**
diff --git a/external/llvm-project/libc/docs/dev/code_style.rst b/external/llvm-project/libc/docs/dev/code_style.rst
index 0bd3a69ae3ff..86247966552f 100644
--- a/external/llvm-project/libc/docs/dev/code_style.rst
+++ b/external/llvm-project/libc/docs/dev/code_style.rst
@@ -101,7 +101,7 @@ test infrastructure itself can be affected. To avoid perturbing the unit test
 infrastructure around the setting of ``errno``, the following rules are to be
 followed:
 
-#. A special macro named ``libc_errno`` defined in ``src/errno/libc_errno.h``
+#. A special macro named ``libc_errno`` defined in ``src/__support/libc_errno.h``
    should be used when setting ``errno`` from libc runtime code. For example,
    code to set ``errno`` to ``EINVAL`` should be:
 
@@ -117,7 +117,7 @@ followed:
    `ErrorOr <https://github.com/llvm/llvm-project/blob/main/libc/src/__support/error_or.h>`_
    to return error values.
 
-#. The header file ``src/errno/libc_errno.h`` is shipped as part of the target
+#. The header file ``src/__support/libc_errno.h`` is shipped as part of the target
    corresponding to the ``errno`` entrypoint ``libc.src.errno.errno``. We do
    not in general allow dependencies between entrypoints. However, the ``errno``
    entrypoint is the only exceptional entrypoint on which other entrypoints
diff --git a/external/llvm-project/libc/docs/headers/time.rst b/external/llvm-project/libc/docs/headers/time.rst
index de82d80a2bec..9733a176fbb2 100644
--- a/external/llvm-project/libc/docs/headers/time.rst
+++ b/external/llvm-project/libc/docs/headers/time.rst
@@ -42,72 +42,74 @@ Implementation Status
 
 * To check date and time functions enabled for embedded system:
 
+  - `barebone-aarch64 <https://github.com/llvm/llvm-project/tree/main/libc/config/baremetal/aarch64/entrypoints.txt>`_
+
   - `barebone-aarch32 <https://github.com/llvm/llvm-project/tree/main/libc/config/baremetal/arm/entrypoints.txt>`_
 
   - barebone-riscv32 - to be added
 
-+---------------------+-----------------------------------------------+-------------------+-------------------+-------------------+-------------------+
-| <Func>              |  Linux                                        | Windows           | MacOS             | Embedded          | GPU               |
-|                     +---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-|                     | x86_64  | aarch64 | aarch32 | riscv64/riscv32 | x86_64  | aarch64 | x86_64  | aarch64 | aarch32 | riscv32 | AMD     | nVidia  |
-+=====================+=========+=========+=========+=================+=========+=========+=========+=========+=========+=========+=========+=========+
-| asctime             | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| asctime_r           | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ctime               | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ctime_r             | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| clock               | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| clock_getcpuclockid |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| clock_getres        |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| clock_gettime       | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| clock_nanosleep     |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| clock_settime       |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ctime               |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ctime_r             |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| difftime            | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| getdate             |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| gettimeofday        | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| gmtime              | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| gmtime_r            | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| localtime           |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| localtime_r         |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| mktime              | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nanosleep           | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| strftime            |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| strptime            |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| time                | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| timer_create        |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| timer_delete        |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| timer_gettime       |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| timer_getoverrun    |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| timer_settime       |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
-| tzset               |         |         |         |                 |         |         |         |         |         |         |         |         |
-+---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
++---------------------+-----------------------------------------------+-------------------+-------------------+-----------------------------+---------+---------+
+| <Func>              |  Linux                                        | Windows           | MacOS             | Embedded                    | GPU               |
+|                     +---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+|                     | x86_64  | aarch64 | aarch32 | riscv64/riscv32 | x86_64  | aarch64 | x86_64  | aarch64 | aarch64 | aarch32 | riscv32 | AMD     | nVidia  |
++=====================+=========+=========+=========+=================+=========+=========+=========+=========+=========+=========+=========+=========+=========+
+| asctime             | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| asctime_r           | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ctime               | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ctime_r             | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| clock               | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| clock_getcpuclockid |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| clock_getres        |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| clock_gettime       | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| clock_nanosleep     |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| clock_settime       |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ctime               |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ctime_r             |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| difftime            | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| getdate             |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| gettimeofday        | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| gmtime              | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| gmtime_r            | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| localtime           |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| localtime_r         |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| mktime              | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nanosleep           | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| strftime            |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| strptime            |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| time                | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| timer_create        |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| timer_delete        |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| timer_gettime       |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| timer_getoverrun    |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| timer_settime       |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| tzset               |         |         |         |                 |         |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
\ No newline at end of file
diff --git a/external/llvm-project/libc/fuzzing/stdio/CMakeLists.txt b/external/llvm-project/libc/fuzzing/stdio/CMakeLists.txt
index 8f89baa70200..401785a30469 100644
--- a/external/llvm-project/libc/fuzzing/stdio/CMakeLists.txt
+++ b/external/llvm-project/libc/fuzzing/stdio/CMakeLists.txt
@@ -4,6 +4,7 @@ add_libc_fuzzer(
     printf_parser_fuzz.cpp
   DEPENDS
     libc.src.stdio.printf_core.parser
+    libc.src.errno.errno # needed for the strerror conversion
 )
 
 add_libc_fuzzer(
diff --git a/external/llvm-project/libc/hdr/CMakeLists.txt b/external/llvm-project/libc/hdr/CMakeLists.txt
index 209fcb965242..052a773a4fce 100644
--- a/external/llvm-project/libc/hdr/CMakeLists.txt
+++ b/external/llvm-project/libc/hdr/CMakeLists.txt
@@ -126,6 +126,15 @@ add_proxy_header_library(
     libc.include.llvm-libc-macros.sys_epoll_macros
 )
 
+add_proxy_header_library(
+  sys_ioctl_macros
+  HDRS
+    sys_ioctl_macros.h
+  FULL_BUILD_DEPENDS
+    libc.include.sys_ioctl
+    libc.include.llvm-libc-macros.sys_ioctl_macros
+)
+
 add_proxy_header_library(
   sys_stat_macros
   HDRS
@@ -212,6 +221,8 @@ add_proxy_header_library(
 
 add_header_library(wchar_overlay HDRS wchar_overlay.h)
 
+add_header_library(uchar_overlay HDRS uchar_overlay.h)
+
 add_proxy_header_library(
   wchar_macros
   HDRS
diff --git a/external/llvm-project/libc/hdr/stdlib_overlay.h b/external/llvm-project/libc/hdr/stdlib_overlay.h
index f095cafe5e0b..53c32ec0ae33 100644
--- a/external/llvm-project/libc/hdr/stdlib_overlay.h
+++ b/external/llvm-project/libc/hdr/stdlib_overlay.h
@@ -19,6 +19,11 @@
 // functions, causing external alias errors.  They are guarded by
 // `__USE_FORTIFY_LEVEL`, which will be temporarily disabled.
 
+#ifdef _FORTIFY_SOURCE
+#define LIBC_OLD_FORTIFY_SOURCE _FORTIFY_SOURCE
+#undef _FORTIFY_SOURCE
+#endif
+
 #ifdef __USE_FORTIFY_LEVEL
 #define LIBC_OLD_USE_FORTIFY_LEVEL __USE_FORTIFY_LEVEL
 #undef __USE_FORTIFY_LEVEL
@@ -27,6 +32,11 @@
 
 #include <stdlib.h>
 
+#ifdef LIBC_OLD_FORTIFY_SOURCE
+#define _FORTIFY_SOURCE LIBC_OLD_FORTIFY_SOURCE
+#undef LIBC_OLD_FORTIFY_SOURCE
+#endif
+
 #ifdef LIBC_OLD_USE_FORTIFY_LEVEL
 #undef __USE_FORTIFY_LEVEL
 #define __USE_FORTIFY_LEVEL LIBC_OLD_USE_FORTIFY_LEVEL
diff --git a/external/llvm-project/libc/hdr/sys_ioctl_macros.h b/external/llvm-project/libc/hdr/sys_ioctl_macros.h
new file mode 100644
index 000000000000..935d43627346
--- /dev/null
+++ b/external/llvm-project/libc/hdr/sys_ioctl_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from sys/ioctl.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_SYS_IOCTL_MACROS_H
+#define LLVM_LIBC_HDR_SYS_IOCTL_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/sys-ioctl-macros.h"
+
+#else // Overlay mode
+
+#include <sys/ioctl.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_SYS_IOCTL_MACROS_H
diff --git a/external/llvm-project/libc/hdr/types/CMakeLists.txt b/external/llvm-project/libc/hdr/types/CMakeLists.txt
index 5f6197c93d44..c88c35700907 100644
--- a/external/llvm-project/libc/hdr/types/CMakeLists.txt
+++ b/external/llvm-project/libc/hdr/types/CMakeLists.txt
@@ -1,3 +1,25 @@
+add_proxy_header_library(
+  char8_t 
+  HDRS
+    char8_t.h
+  DEPENDS
+    libc.hdr.uchar_overlay
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.char8_t
+    libc.include.uchar
+)
+
+add_proxy_header_library(
+  char32_t 
+  HDRS
+    char32_t.h
+  DEPENDS
+    libc.hdr.uchar_overlay
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.char32_t
+    libc.include.uchar
+)
+
 add_proxy_header_library(
   div_t
   HDRS
diff --git a/external/llvm-project/libc/hdr/types/char32_t.h b/external/llvm-project/libc/hdr/types/char32_t.h
new file mode 100644
index 000000000000..94fe5747d341
--- /dev/null
+++ b/external/llvm-project/libc/hdr/types/char32_t.h
@@ -0,0 +1,22 @@
+//===-- Definition of char32_t.h ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CHAR32_T_H
+#define LLVM_LIBC_HDR_TYPES_CHAR32_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/char32_t.h"
+
+#else // overlay mode
+
+#include "hdr/uchar_overlay.h"
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CHAR32_T_H
diff --git a/external/llvm-project/libc/hdr/types/char8_t.h b/external/llvm-project/libc/hdr/types/char8_t.h
new file mode 100644
index 000000000000..31de764658f9
--- /dev/null
+++ b/external/llvm-project/libc/hdr/types/char8_t.h
@@ -0,0 +1,22 @@
+//===-- Definition of char8_t.h -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CHAR8_T_H
+#define LLVM_LIBC_HDR_TYPES_CHAR8_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/char8_t.h"
+
+#else // overlay mode
+
+#include "hdr/uchar_overlay.h"
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CHAR8_T_H
diff --git a/external/llvm-project/libc/hdr/uchar_overlay.h b/external/llvm-project/libc/hdr/uchar_overlay.h
new file mode 100644
index 000000000000..44ed3d48c6c1
--- /dev/null
+++ b/external/llvm-project/libc/hdr/uchar_overlay.h
@@ -0,0 +1,69 @@
+//===-- Including uchar.h in overlay mode ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_UCHAR_OVERLAY_H
+#define LLVM_LIBC_HDR_UCHAR_OVERLAY_H
+
+#ifdef LIBC_FULL_BUILD
+#error "This header should only be included in overlay mode"
+#endif
+
+// Overlay mode
+
+// glibc <uchar.h> header might provide extern inline definitions for few
+// functions, causing external alias errors.  They are guarded by
+// `__USE_EXTERN_INLINES` macro.  We temporarily disable `__USE_EXTERN_INLINES`
+// macro by defining `__NO_INLINE__` before including <uchar.h>.
+// And the same with `__USE_FORTIFY_LEVEL`, which will be temporarily disabled
+// with `_FORTIFY_SOURCE`.
+
+#ifdef _FORTIFY_SOURCE
+#define LIBC_OLD_FORTIFY_SOURCE _FORTIFY_SOURCE
+#undef _FORTIFY_SOURCE
+#endif
+
+#ifndef __NO_INLINE__
+#define __NO_INLINE__ 1
+#define LIBC_SET_NO_INLINE
+#endif
+
+#ifdef __USE_EXTERN_INLINES
+#define LIBC_OLD_USE_EXTERN_INLINES
+#undef __USE_EXTERN_INLINES
+#endif
+
+#ifdef __USE_FORTIFY_LEVEL
+#define LIBC_OLD_USE_FORTIFY_LEVEL __USE_FORTIFY_LEVEL
+#undef __USE_FORTIFY_LEVEL
+#define __USE_FORTIFY_LEVEL 0
+#endif
+
+#include <uchar.h>
+
+#ifdef LIBC_OLD_FORTIFY_SOURCE
+#define _FORTIFY_SOURCE LIBC_OLD_FORTIFY_SOURCE
+#undef LIBC_OLD_FORTIFY_SOURCE
+#endif
+
+#ifdef LIBC_SET_NO_INLINE
+#undef __NO_INLINE__
+#undef LIBC_SET_NO_INLINE
+#endif
+
+#ifdef LIBC_OLD_USE_FORTIFY_LEVEL
+#undef __USE_FORTIFY_LEVEL
+#define __USE_FORTIFY_LEVEL LIBC_OLD_USE_FORTIFY_LEVEL
+#undef LIBC_OLD_USE_FORTIFY_LEVEL
+#endif
+
+#ifdef LIBC_OLD_USE_EXTERN_INLINES
+#define __USE_EXTERN_INLINES
+#undef LIBC_OLD_USE_EXTERN_INLINES
+#endif
+
+#endif // LLVM_LIBC_HDR_UCHAR_OVERLAY_H
diff --git a/external/llvm-project/libc/include/CMakeLists.txt b/external/llvm-project/libc/include/CMakeLists.txt
index 7209e10c68b8..55268d19529c 100644
--- a/external/llvm-project/libc/include/CMakeLists.txt
+++ b/external/llvm-project/libc/include/CMakeLists.txt
@@ -255,6 +255,7 @@ add_header_macro(
   time.h
   DEPENDS
     .llvm_libc_common_h
+    .llvm-libc-macros.null_macro
     .llvm-libc-macros.time_macros
     .llvm-libc-types.clock_t
     .llvm-libc-types.time_t
@@ -329,6 +330,7 @@ add_header_macro(
   stdio.h
   DEPENDS
     .llvm-libc-macros.file_seek_macros
+    .llvm-libc-macros.null_macro
     .llvm-libc-macros.stdio_macros
     .llvm-libc-types.FILE
     .llvm-libc-types.cookie_io_functions_t
@@ -343,6 +345,7 @@ add_header_macro(
   ../libc/include/stdlib.yaml
   stdlib.h
   DEPENDS
+    .llvm-libc-macros.null_macro
     .llvm-libc-macros.stdlib_macros
     .llvm-libc-types.__atexithandler_t
     .llvm-libc-types.__qsortcompare_t
@@ -709,6 +712,7 @@ add_header_macro(
   wchar.h
   DEPENDS
     .llvm_libc_common_h
+    .llvm-libc-macros.null_macro
     .llvm-libc-macros.wchar_macros
     .llvm-libc-types.mbstate_t
     .llvm-libc-types.size_t
@@ -723,6 +727,7 @@ add_header_macro(
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.locale_macros
+    .llvm-libc-macros.null_macro
     .llvm-libc-types.locale_t
     .llvm-libc-types.struct_lconv
 )
diff --git a/external/llvm-project/libc/include/llvm-libc-macros/baremetal/CMakeLists.txt b/external/llvm-project/libc/include/llvm-libc-macros/baremetal/CMakeLists.txt
new file mode 100644
index 000000000000..ea08c63c0030
--- /dev/null
+++ b/external/llvm-project/libc/include/llvm-libc-macros/baremetal/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_header(
+  time_macros
+  HDR
+    time-macros.h
+)
diff --git a/external/llvm-project/libc/include/llvm-libc-macros/baremetal/time-macros.h b/external/llvm-project/libc/include/llvm-libc-macros/baremetal/time-macros.h
new file mode 100644
index 000000000000..3537376c4bca
--- /dev/null
+++ b/external/llvm-project/libc/include/llvm-libc-macros/baremetal/time-macros.h
@@ -0,0 +1,26 @@
+//===-- Definition of macros from time.h ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_BAREMETAL_TIME_MACROS_H
+#define LLVM_LIBC_MACROS_BAREMETAL_TIME_MACROS_H
+
+#ifdef __CLK_TCK
+#define CLOCKS_PER_SEC __CLK_TCK
+#else
+#if defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) ||             \
+    defined(__arm64__) || defined(_M_ARM64)
+// This default implementation of this function shall use semihosting
+// Semihosting measures time in centiseconds
+// https://github.com/ARM-software/abi-aa/blob/main/semihosting/semihosting.rst#sys-clock-0x10
+#define CLOCKS_PER_SEC 100
+#else
+#define CLOCKS_PER_SEC 1000000
+#endif
+#endif
+
+#endif // LLVM_LIBC_MACROS_BAREMETAL_TIME_MACROS_H
diff --git a/external/llvm-project/libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h b/external/llvm-project/libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h
index 5eb779aeeca5..41226080084c 100644
--- a/external/llvm-project/libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h
+++ b/external/llvm-project/libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h
@@ -15,5 +15,6 @@
 // around the definitions of macros like _IO, _IOR, _IOW, and _IOWR that I don't
 // think is worth digging into right now.
 #define TIOCGETD 0x5424
+#define FIONREAD 0x541B
 
 #endif // LLVM_LIBC_MACROS_LINUX_SYS_IOCTL_MACROS_H
diff --git a/external/llvm-project/libc/include/llvm-libc-macros/time-macros.h b/external/llvm-project/libc/include/llvm-libc-macros/time-macros.h
index 445d8b3e837e..30e0a310a548 100644
--- a/external/llvm-project/libc/include/llvm-libc-macros/time-macros.h
+++ b/external/llvm-project/libc/include/llvm-libc-macros/time-macros.h
@@ -5,8 +5,14 @@
 #include "gpu/time-macros.h"
 #elif defined(__linux__)
 #include "linux/time-macros.h"
+#elif defined(__ELF__)
+#include "baremetal/time-macros.h"
+#else
+#define CLOCKS_PER_SEC 1000000
 #endif
 
+#define CLK_TCK CLOCKS_PER_SEC
+
 #define TIME_UTC 1
 #define TIME_MONOTONIC 2
 #define TIME_ACTIVE 3
diff --git a/external/llvm-project/libc/include/llvm-libc-types/char8_t.h b/external/llvm-project/libc/include/llvm-libc-types/char8_t.h
index ddadab1afa21..a343be77d810 100644
--- a/external/llvm-project/libc/include/llvm-libc-types/char8_t.h
+++ b/external/llvm-project/libc/include/llvm-libc-types/char8_t.h
@@ -9,8 +9,7 @@
 #ifndef LLVM_LIBC_TYPES_CHAR8_T_H
 #define LLVM_LIBC_TYPES_CHAR8_T_H
 
-#if !defined(__cplusplus) && defined(__STDC_VERSION__) &&                      \
-    __STDC_VERSION__ >= 202311L
+#if !(defined(__cplusplus) && defined(__cpp_char8_t))
 typedef unsigned char char8_t;
 #endif
 
diff --git a/external/llvm-project/libc/include/llvm-libc-types/size_t.h b/external/llvm-project/libc/include/llvm-libc-types/size_t.h
index 3b31b0820f23..26ae68abe0ee 100644
--- a/external/llvm-project/libc/include/llvm-libc-types/size_t.h
+++ b/external/llvm-project/libc/include/llvm-libc-types/size_t.h
@@ -9,11 +9,6 @@
 #ifndef LLVM_LIBC_TYPES_SIZE_T_H
 #define LLVM_LIBC_TYPES_SIZE_T_H
 
-// Since __need_size_t is defined, we get the definition of size_t from the
-// standalone C header stddef.h. Also, because __need_size_t is defined,
-// including stddef.h will pull only the type size_t and nothing else.
-#define __need_size_t
-#include <stddef.h>
-#undef __need_size_t
+typedef __SIZE_TYPE__ size_t;
 
 #endif // LLVM_LIBC_TYPES_SIZE_T_H
diff --git a/external/llvm-project/libc/include/llvm-libc-types/ssize_t.h b/external/llvm-project/libc/include/llvm-libc-types/ssize_t.h
index 41e4b6d2c500..8f579e2749ba 100644
--- a/external/llvm-project/libc/include/llvm-libc-types/ssize_t.h
+++ b/external/llvm-project/libc/include/llvm-libc-types/ssize_t.h
@@ -9,6 +9,6 @@
 #ifndef LLVM_LIBC_TYPES_SSIZE_T_H
 #define LLVM_LIBC_TYPES_SSIZE_T_H
 
-typedef __INT64_TYPE__ ssize_t;
+typedef __PTRDIFF_TYPE__ ssize_t;
 
 #endif // LLVM_LIBC_TYPES_SSIZE_T_H
diff --git a/external/llvm-project/libc/include/locale.yaml b/external/llvm-project/libc/include/locale.yaml
index 6c71b70e59f0..4566984ad83a 100644
--- a/external/llvm-project/libc/include/locale.yaml
+++ b/external/llvm-project/libc/include/locale.yaml
@@ -1,5 +1,8 @@
 header: locale.h
 header_template: locale.h.def
+macros:
+  - macro_name: NULL
+    macro_header: null-macro.h
 types:
   - type_name: locale_t
   - type_name: struct_lconv
diff --git a/external/llvm-project/libc/include/math.yaml b/external/llvm-project/libc/include/math.yaml
index 466c08ade6fc..11bead074595 100644
--- a/external/llvm-project/libc/include/math.yaml
+++ b/external/llvm-project/libc/include/math.yaml
@@ -734,7 +734,7 @@ functions:
       - type: float128
       - type: float128
       - type: float128
-    guards: LIBC_TYPES_HAS_FLOAT128
+    guard: LIBC_TYPES_HAS_FLOAT128
   - name: ffmal
     standards:
       - stdc
diff --git a/external/llvm-project/libc/include/stdio.yaml b/external/llvm-project/libc/include/stdio.yaml
index 2619984cca26..2a0c56370998 100644
--- a/external/llvm-project/libc/include/stdio.yaml
+++ b/external/llvm-project/libc/include/stdio.yaml
@@ -1,6 +1,8 @@
 header: stdio.h
 header_template: stdio.h.def
 macros:
+  - macro_name: NULL
+    macro_header: null-macro.h
   - macro_name: stdout
     macro_value: stdout
   - macro_name: stdin
@@ -247,6 +249,12 @@ functions:
       - POSIX
     return_type: int
     arguments: []
+  - name: perror
+    standards:
+      - stdc
+    return_type: void
+    arguments:
+      - type: const char *
   - name: printf
     standards:
       - stdc
diff --git a/external/llvm-project/libc/include/stdlib.yaml b/external/llvm-project/libc/include/stdlib.yaml
index f7155ba27a16..3b2ff13c684b 100644
--- a/external/llvm-project/libc/include/stdlib.yaml
+++ b/external/llvm-project/libc/include/stdlib.yaml
@@ -4,7 +4,9 @@ standards:
   - stdc
 merge_yaml_files:
   - stdlib-malloc.yaml
-macros: []
+macros:
+  - macro_name: NULL
+    macro_header: null-macro.h
 types:
   - type_name: __atexithandler_t
   - type_name: __qsortcompare_t
diff --git a/external/llvm-project/libc/include/string.h.def b/external/llvm-project/libc/include/string.h.def
index 1bd2687db2be..339d005e43a4 100644
--- a/external/llvm-project/libc/include/string.h.def
+++ b/external/llvm-project/libc/include/string.h.def
@@ -11,8 +11,6 @@
 
 #include "__llvm-libc-common.h"
 
-#include "llvm-libc-macros/null-macro.h"
-
 %%public_api()
 
 #endif // LLVM_LIBC_STRING_H
diff --git a/external/llvm-project/libc/include/string.yaml b/external/llvm-project/libc/include/string.yaml
index 9f72b8db6c1e..736deceb453d 100644
--- a/external/llvm-project/libc/include/string.yaml
+++ b/external/llvm-project/libc/include/string.yaml
@@ -1,6 +1,8 @@
 header: string.h
 header_template: string.h.def
-macros: []
+macros:
+  - macro_name: NULL
+    macro_header: null-macro.h
 types:
   - type_name: locale_t
   - type_name: size_t
diff --git a/external/llvm-project/libc/include/time.yaml b/external/llvm-project/libc/include/time.yaml
index 7bb25dbe85ac..3b9d77c0aaae 100644
--- a/external/llvm-project/libc/include/time.yaml
+++ b/external/llvm-project/libc/include/time.yaml
@@ -1,6 +1,8 @@
 header: time.h
 header_template: time.h.def
-macros: []
+macros:
+  - macro_name: NULL
+    macro_header: null-macro.h
 types:
   - type_name: struct_timeval
   - type_name: clockid_t
diff --git a/external/llvm-project/libc/include/wchar.yaml b/external/llvm-project/libc/include/wchar.yaml
index bfd9a1034201..84db73d8f01e 100644
--- a/external/llvm-project/libc/include/wchar.yaml
+++ b/external/llvm-project/libc/include/wchar.yaml
@@ -1,6 +1,8 @@
 header: wchar.h
 header_template: wchar.h.def
-macros: []
+macros:
+  - macro_name: NULL
+    macro_header: null-macro.h
 types:
   - type_name: size_t
   - type_name: wint_t
@@ -39,9 +41,24 @@ functions:
     standards:
       - stdc
     return_type: const wchar_t *
-    arguments: 
+    arguments:
       - type: const wchar_t *
       - type: wchar_t
+  - name: wcsncmp
+    standards:
+      - stdc
+    return_type: int
+    arguments:
+      - type: const wchar_t *
+      - type: const wchar_t *
+      - type: size_t
+  - name: wcscmp
+    standards:
+      - stdc
+    return_type: int
+    arguments:
+      - type: const wchar_t *
+      - type: const wchar_t *
   - name: wcspbrk
     standards:
       - stdc
@@ -49,26 +66,94 @@ functions:
     arguments:
       - type: const wchar_t *
       - type: const wchar_t *
+  - name: wcsrchr
+    standards:
+      - stdc
+    return_type: const wchar_t *
+    arguments:
+      - type: const wchar_t *
+      - type: wchar_t
   - name: wcsspn
     standards:
       - stdc
     return_type: size_t
-    arguments: 
+    arguments:
       - type: const wchar_t *
       - type: const wchar_t *
   - name: wmemcmp
     standards:
       - stdc
     return_type: int
-    arguments: 
+    arguments:
       - type: const wchar_t *
       - type: const wchar_t *
       - type: size_t
+  - name: wmemchr
+    standards:
+      - stdc
+    return_type: const wchar_t *
+    arguments:
+      - type: const wchar_t *
+      - type: wchar_t
+      - type: size_t
+  - name: wmempcpy
+    standards:
+      - gnu
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *
+      - type: const wchar_t *
+      - type: size_t
   - name: wmemcpy
     standards:
       - stdc
     return_type: wchar_t *
     arguments: 
-      - type: __restrict wchar_t *
-      - type: const __restrict wchar_t *
+      - type: wchar_t *__restrict 
+      - type: const wchar_t *__restrict
+      - type: size_t
+  - name: wmemmove
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *
+      - type: const wchar_t *
       - type: size_t
+  - name: wcsncpy
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *__restrict
+      - type: const wchar_t *__restrict
+      - type: size_t
+  - name: wcscat
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments: 
+      - type: wchar_t *__restrict
+      - type: const wchar_t *__restrict
+  - name: wcsstr
+    standards:
+      - stdc
+    return_type: const wchar_t *
+    arguments:
+      - type: const wchar_t *
+      - type: const wchar_t *
+  - name: wcsncat
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *__restrict
+      - type: const wchar_t *__restrict
+      - type: size_t
+  - name: wcscpy
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *__restrict
+      - type: const wchar_t *__restrict
diff --git a/external/llvm-project/libc/shared/fp_bits.h b/external/llvm-project/libc/shared/fp_bits.h
index 2898c508b777..e6bb1e17b80c 100644
--- a/external/llvm-project/libc/shared/fp_bits.h
+++ b/external/llvm-project/libc/shared/fp_bits.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_SHARED_FP_BITS_H
 #define LLVM_LIBC_SHARED_FP_BITS_H
 
+#include "libc_common.h"
 #include "src/__support/FPUtil/FPBits.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/shared/libc_common.h b/external/llvm-project/libc/shared/libc_common.h
new file mode 100644
index 000000000000..c4560bbb0276
--- /dev/null
+++ b/external/llvm-project/libc/shared/libc_common.h
@@ -0,0 +1,26 @@
+//===-- Common defines for sharing LLVM libc with LLVM projects -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_LIBC_COMMON_H
+#define LLVM_LIBC_SHARED_LIBC_COMMON_H
+
+// Use system errno.
+#ifdef LIBC_ERRNO_MODE
+#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE
+#error                                                                         \
+    "LIBC_ERRNO_MODE was set to something different from LIBC_ERRNO_MODE_SYSTEM_INLINE."
+#endif // LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE
+#else
+#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM_INLINE
+#endif // LIBC_ERRNO_MODE
+
+#ifndef LIBC_NAMESPACE
+#define LIBC_NAMESPACE __llvm_libc
+#endif // LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SHARED_LIBC_COMMON_H
diff --git a/external/llvm-project/libc/shared/math.h b/external/llvm-project/libc/shared/math.h
new file mode 100644
index 000000000000..4ddc29c7ae83
--- /dev/null
+++ b/external/llvm-project/libc/shared/math.h
@@ -0,0 +1,16 @@
+//===-- Floating point math functions ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_H
+#define LLVM_LIBC_SHARED_MATH_H
+
+#include "libc_common.h"
+
+#include "math/expf.h"
+
+#endif // LLVM_LIBC_SHARED_MATH_H
diff --git a/external/llvm-project/libc/shared/math/expf.h b/external/llvm-project/libc/shared/math/expf.h
new file mode 100644
index 000000000000..a4e8b0751bb4
--- /dev/null
+++ b/external/llvm-project/libc/shared/math/expf.h
@@ -0,0 +1,23 @@
+//===-- Shared expf function ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_EXPF_H
+#define LLVM_LIBC_SHARED_MATH_EXPF_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/expf.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::expf;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_MATH_EXPF_H
diff --git a/external/llvm-project/libc/shared/rpc_server.h b/external/llvm-project/libc/shared/rpc_server.h
index 5509094b944a..46e35f13f0ea 100644
--- a/external/llvm-project/libc/shared/rpc_server.h
+++ b/external/llvm-project/libc/shared/rpc_server.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_SHARED_RPC_SERVER_H
 #define LLVM_LIBC_SHARED_RPC_SERVER_H
 
+#include "libc_common.h"
 #include "src/__support/RPC/rpc_server.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/shared/str_to_float.h b/external/llvm-project/libc/shared/str_to_float.h
index b133a28e26ef..dcc6027d6c77 100644
--- a/external/llvm-project/libc/shared/str_to_float.h
+++ b/external/llvm-project/libc/shared/str_to_float.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_SHARED_STR_TO_FLOAT_H
 #define LLVM_LIBC_SHARED_STR_TO_FLOAT_H
 
+#include "libc_common.h"
 #include "src/__support/str_to_float.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/shared/str_to_integer.h b/external/llvm-project/libc/shared/str_to_integer.h
index 15bee698d5a6..6ed38c932662 100644
--- a/external/llvm-project/libc/shared/str_to_integer.h
+++ b/external/llvm-project/libc/shared/str_to_integer.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_SHARED_STR_TO_INTEGER_H
 #define LLVM_LIBC_SHARED_STR_TO_INTEGER_H
 
+#include "libc_common.h"
 #include "src/__support/str_to_integer.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/__support/CMakeLists.txt b/external/llvm-project/libc/src/__support/CMakeLists.txt
index f92499fdbf45..7e85136c0885 100644
--- a/external/llvm-project/libc/src/__support/CMakeLists.txt
+++ b/external/llvm-project/libc/src/__support/CMakeLists.txt
@@ -1,6 +1,15 @@
 add_subdirectory(CPP)
 add_subdirectory(macros)
 
+add_header_library(
+  libc_errno
+  HDRS
+    libc_errno.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.src.__support.macros.config
+)
+
 add_header_library(
   block
   HDRS
@@ -381,3 +390,11 @@ add_subdirectory(HashTable)
 add_subdirectory(fixed_point)
 
 add_subdirectory(time)
+
+# Requires access to uchar header which is not on macos
+# Therefore, cannot currently build this on macos in overlay mode
+if(NOT(LIBC_TARGET_OS_IS_DARWIN))
+  add_subdirectory(wchar)
+endif()
+
+add_subdirectory(math)
diff --git a/external/llvm-project/libc/src/__support/FPUtil/FEnvImpl.h b/external/llvm-project/libc/src/__support/FPUtil/FEnvImpl.h
index 4c8f34a435bd..50a101f833c5 100644
--- a/external/llvm-project/libc/src/__support/FPUtil/FEnvImpl.h
+++ b/external/llvm-project/libc/src/__support/FPUtil/FEnvImpl.h
@@ -12,10 +12,10 @@
 #include "hdr/fenv_macros.h"
 #include "hdr/math_macros.h"
 #include "hdr/types/fenv_t.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/attributes.h" // LIBC_INLINE
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/architectures.h"
-#include "src/errno/libc_errno.h"
 
 #if defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP)
 #if defined(__APPLE__)
diff --git a/external/llvm-project/libc/src/__support/File/dir.cpp b/external/llvm-project/libc/src/__support/File/dir.cpp
index 21b0106f7010..aea8862c15f7 100644
--- a/external/llvm-project/libc/src/__support/File/dir.cpp
+++ b/external/llvm-project/libc/src/__support/File/dir.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/CPP/mutex.h" // lock_guard
 #include "src/__support/CPP/new.h"
 #include "src/__support/error_or.h"
+#include "src/__support/libc_errno.h" // For error macros
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h" // For error macros
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/__support/File/file.cpp b/external/llvm-project/libc/src/__support/File/file.cpp
index 528542cccf32..303852dbbb71 100644
--- a/external/llvm-project/libc/src/__support/File/file.cpp
+++ b/external/llvm-project/libc/src/__support/File/file.cpp
@@ -13,8 +13,8 @@
 #include "hdr/types/off_t.h"
 #include "src/__support/CPP/new.h"
 #include "src/__support/CPP/span.h"
+#include "src/__support/libc_errno.h" // For error macros
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h" // For error macros
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/__support/File/linux/file.cpp b/external/llvm-project/libc/src/__support/File/linux/file.cpp
index 824c1f200e8c..4594dadf1ccd 100644
--- a/external/llvm-project/libc/src/__support/File/linux/file.cpp
+++ b/external/llvm-project/libc/src/__support/File/linux/file.cpp
@@ -15,12 +15,12 @@
 #include "src/__support/File/linux/lseekImpl.h"
 #include "src/__support/OSUtil/fcntl.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/libc_errno.h"     // For error macros
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h" // For error macros
 
 #include "hdr/fcntl_macros.h" // For mode_t and other flags to the open syscall
-#include <sys/stat.h>    // For S_IS*, S_IF*, and S_IR* flags.
-#include <sys/syscall.h> // For syscall numbers
+#include <sys/stat.h>         // For S_IS*, S_IF*, and S_IR* flags.
+#include <sys/syscall.h>      // For syscall numbers
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -128,10 +128,11 @@ ErrorOr<LinuxFile *> create_file_from_fd(int fd, const char *mode) {
     return Error(EINVAL);
   }
 
-  int fd_flags = internal::fcntl(fd, F_GETFL);
-  if (fd_flags == -1) {
+  auto result = internal::fcntl(fd, F_GETFL);
+  if (!result.has_value()) {
     return Error(EBADF);
   }
+  int fd_flags = result.value();
 
   using OpenMode = File::OpenMode;
   if (((fd_flags & O_ACCMODE) == O_RDONLY &&
@@ -145,8 +146,9 @@ ErrorOr<LinuxFile *> create_file_from_fd(int fd, const char *mode) {
   if ((modeflags & static_cast<ModeFlags>(OpenMode::APPEND)) &&
       !(fd_flags & O_APPEND)) {
     do_seek = true;
-    if (internal::fcntl(fd, F_SETFL,
-                        reinterpret_cast<void *>(fd_flags | O_APPEND)) == -1) {
+    if (!internal::fcntl(fd, F_SETFL,
+                         reinterpret_cast<void *>(fd_flags | O_APPEND))
+             .has_value()) {
       return Error(EBADF);
     }
   }
diff --git a/external/llvm-project/libc/src/__support/File/linux/lseekImpl.h b/external/llvm-project/libc/src/__support/File/linux/lseekImpl.h
index a034913d9f6e..300e5c5dd55b 100644
--- a/external/llvm-project/libc/src/__support/File/linux/lseekImpl.h
+++ b/external/llvm-project/libc/src/__support/File/linux/lseekImpl.h
@@ -13,8 +13,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/__support/error_or.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <stdint.h>      // For uint64_t.
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/__support/GPU/allocator.cpp b/external/llvm-project/libc/src/__support/GPU/allocator.cpp
index ca68cbcedd48..66ab155e5c29 100644
--- a/external/llvm-project/libc/src/__support/GPU/allocator.cpp
+++ b/external/llvm-project/libc/src/__support/GPU/allocator.cpp
@@ -129,6 +129,14 @@ static inline constexpr T round_up(const T x) {
   return (x + N) & ~(N - 1);
 }
 
+// Perform a lane parallel memset on a uint32_t pointer.
+void uniform_memset(uint32_t *s, uint32_t c, uint32_t n, uint64_t uniform) {
+  uint64_t mask = gpu::get_lane_mask();
+  uint32_t workers = cpp::popcount(uniform);
+  for (uint32_t i = impl::lane_count(mask & uniform); i < n; i += workers)
+    s[i] = c;
+}
+
 } // namespace impl
 
 /// A slab allocator used to hand out identically sized slabs of memory.
@@ -157,10 +165,15 @@ struct Slab {
     Header *header = reinterpret_cast<Header *>(memory);
     header->chunk_size = chunk_size;
     header->global_index = global_index;
+  }
 
-    // This memset is expensive and likely not necessary for the current 'kfd'
-    // driver. Until zeroed pages are exposed by the API we must be careful.
-    __builtin_memset(get_bitfield(), 0, bitfield_bytes(chunk_size));
+  // Set the necessary bitfield bytes to zero in parallel using many lanes. This
+  // must be called before the bitfield can be accessed safely, memory is not
+  // guaranteed to be zero initialized in the current implementation.
+  void initialize(uint64_t uniform) {
+    uint32_t size = (bitfield_bytes(get_chunk_size()) + sizeof(uint32_t) - 1) /
+                    sizeof(uint32_t);
+    impl::uniform_memset(get_bitfield(), 0, size, uniform);
   }
 
   // Get the number of chunks that can theoretically fit inside this slab.
@@ -229,24 +242,34 @@ struct Slab {
 
     // The uniform mask represents which lanes contain a uniform target pointer.
     // We attempt to place these next to each other.
-    // TODO: We should coalesce these bits and use the result of `fetch_or` to
-    //       search for free bits in parallel.
     void *result = nullptr;
     for (uint64_t mask = lane_mask; mask;
          mask = gpu::ballot(lane_mask, !result)) {
-      uint32_t id = impl::lane_count(uniform & mask);
-      uint32_t index =
-          (gpu::broadcast_value(lane_mask, impl::xorshift32(state)) + id) %
-          usable_bits(chunk_size);
+      if (result)
+        continue;
+
+      uint32_t start = gpu::broadcast_value(lane_mask, impl::xorshift32(state));
 
+      uint32_t id = impl::lane_count(uniform & mask);
+      uint32_t index = (start + id) % usable_bits(chunk_size);
       uint32_t slot = index / BITS_IN_WORD;
       uint32_t bit = index % BITS_IN_WORD;
-      if (!result) {
-        uint32_t before = cpp::AtomicRef(get_bitfield()[slot])
-                              .fetch_or(1u << bit, cpp::MemoryOrder::RELAXED);
-        if (~before & (1 << bit))
-          result = ptr_from_index(index, chunk_size);
-      }
+
+      // Get the mask of bits destined for the same slot and coalesce it.
+      uint64_t match = uniform & gpu::match_any(mask, slot);
+      uint32_t length = cpp::popcount(match);
+      uint32_t bitmask = static_cast<uint32_t>((uint64_t(1) << length) - 1)
+                         << bit;
+
+      uint32_t before = 0;
+      if (gpu::get_lane_id() == static_cast<uint32_t>(cpp::countr_zero(match)))
+        before = cpp::AtomicRef(get_bitfield()[slot])
+                     .fetch_or(bitmask, cpp::MemoryOrder::RELAXED);
+      before = gpu::shuffle(mask, cpp::countr_zero(match), before);
+      if (~before & (1 << bit))
+        result = ptr_from_index(index, chunk_size);
+      else
+        sleep_briefly();
     }
 
     cpp::atomic_thread_fence(cpp::MemoryOrder::ACQUIRE);
@@ -273,7 +296,7 @@ struct Slab {
 
 /// A wait-free guard around a pointer resource to be created dynamically if
 /// space is available and freed once there are no more users.
-template <typename T> struct GuardPtr {
+struct GuardPtr {
 private:
   struct RefCounter {
     // Indicates that the object is in its deallocation phase and thus invalid.
@@ -329,32 +352,25 @@ template <typename T> struct GuardPtr {
     cpp::Atomic<uint64_t> counter{0};
   };
 
-  cpp::Atomic<T *> ptr{nullptr};
+  cpp::Atomic<Slab *> ptr{nullptr};
   RefCounter ref{};
 
   // Should be called be a single lane for each different pointer.
   template <typename... Args>
-  T *try_lock_impl(uint32_t n, uint64_t &count, Args &&...args) {
-    T *expected = ptr.load(cpp::MemoryOrder::RELAXED);
+  Slab *try_lock_impl(uint32_t n, uint64_t &count, Args &&...args) {
+    Slab *expected = ptr.load(cpp::MemoryOrder::RELAXED);
     if (!expected &&
-        ptr.compare_exchange_strong(expected, reinterpret_cast<T *>(SENTINEL),
-                                    cpp::MemoryOrder::RELAXED,
-                                    cpp::MemoryOrder::RELAXED)) {
+        ptr.compare_exchange_strong(
+            expected, reinterpret_cast<Slab *>(SENTINEL),
+            cpp::MemoryOrder::RELAXED, cpp::MemoryOrder::RELAXED)) {
       count = cpp::numeric_limits<uint64_t>::max();
-      void *raw = impl::rpc_allocate(sizeof(T));
+      void *raw = impl::rpc_allocate(sizeof(Slab));
       if (!raw)
         return nullptr;
-      T *mem = new (raw) T(cpp::forward<Args>(args)...);
-
-      cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
-      ptr.store(mem, cpp::MemoryOrder::RELAXED);
-      cpp::atomic_thread_fence(cpp::MemoryOrder::ACQUIRE);
-      if (!ref.acquire(n, count))
-        ref.reset(n, count);
-      return mem;
+      return new (raw) Slab(cpp::forward<Args>(args)...);
     }
 
-    if (!expected || expected == reinterpret_cast<T *>(SENTINEL))
+    if (!expected || expected == reinterpret_cast<Slab *>(SENTINEL))
       return nullptr;
 
     if (!ref.acquire(n, count))
@@ -364,15 +380,25 @@ template <typename T> struct GuardPtr {
     return ptr.load(cpp::MemoryOrder::RELAXED);
   }
 
+  // Finalize the associated memory and signal that it is ready to use by
+  // resetting the counter.
+  void finalize(Slab *mem, uint32_t n, uint64_t &count) {
+    cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
+    ptr.store(mem, cpp::MemoryOrder::RELAXED);
+    cpp::atomic_thread_fence(cpp::MemoryOrder::ACQUIRE);
+    if (!ref.acquire(n, count))
+      ref.reset(n, count);
+  }
+
 public:
   // Attempt to lock access to the pointer, potentially creating it if empty.
   // The uniform mask represents which lanes share the same pointer. For each
   // uniform value we elect a leader to handle it on behalf of the other lanes.
   template <typename... Args>
-  T *try_lock(uint64_t lane_mask, uint64_t uniform, uint64_t &count,
-              Args &&...args) {
+  Slab *try_lock(uint64_t lane_mask, uint64_t uniform, uint64_t &count,
+                 Args &&...args) {
     count = 0;
-    T *result = nullptr;
+    Slab *result = nullptr;
     if (gpu::get_lane_id() == uint32_t(cpp::countr_zero(uniform)))
       result = try_lock_impl(cpp::popcount(uniform), count,
                              cpp::forward<Args>(args)...);
@@ -382,6 +408,14 @@ template <typename T> struct GuardPtr {
     if (!result)
       return nullptr;
 
+    // We defer storing the newly allocated slab until now so that we can use
+    // multiple lanes to initialize it and release it for use.
+    if (count == cpp::numeric_limits<uint64_t>::max()) {
+      result->initialize(uniform);
+      if (gpu::get_lane_id() == uint32_t(cpp::countr_zero(uniform)))
+        finalize(result, cpp::popcount(uniform), count);
+    }
+
     if (count != cpp::numeric_limits<uint64_t>::max())
       count = count - cpp::popcount(uniform) + impl::lane_count(uniform) + 1;
 
@@ -393,8 +427,8 @@ template <typename T> struct GuardPtr {
     cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
     if (gpu::get_lane_id() == uint32_t(cpp::countr_zero(mask)) &&
         ref.release(cpp::popcount(mask))) {
-      T *p = ptr.load(cpp::MemoryOrder::RELAXED);
-      p->~T();
+      Slab *p = ptr.load(cpp::MemoryOrder::RELAXED);
+      p->~Slab();
       impl::rpc_free(p);
       cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
       ptr.store(nullptr, cpp::MemoryOrder::RELAXED);
@@ -407,7 +441,7 @@ template <typename T> struct GuardPtr {
 };
 
 // The global array used to search for a valid slab to allocate from.
-static GuardPtr<Slab> slots[ARRAY_SIZE] = {};
+static GuardPtr slots[ARRAY_SIZE] = {};
 
 // Tries to find a slab in the table that can support the given chunk size.
 static Slab *find_slab(uint32_t chunk_size) {
diff --git a/external/llvm-project/libc/src/__support/HashTable/randomness.h b/external/llvm-project/libc/src/__support/HashTable/randomness.h
index 244dd41be3ee..6b58a4125f78 100644
--- a/external/llvm-project/libc/src/__support/HashTable/randomness.h
+++ b/external/llvm-project/libc/src/__support/HashTable/randomness.h
@@ -14,7 +14,7 @@
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
 #if defined(LIBC_HASHTABLE_USE_GETRANDOM)
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sys/random/getrandom.h"
 #endif
 
diff --git a/external/llvm-project/libc/src/__support/OSUtil/fcntl.h b/external/llvm-project/libc/src/__support/OSUtil/fcntl.h
index 46f7d2813239..3983d78f7f89 100644
--- a/external/llvm-project/libc/src/__support/OSUtil/fcntl.h
+++ b/external/llvm-project/libc/src/__support/OSUtil/fcntl.h
@@ -8,12 +8,18 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_FCNTL_H
 #define LLVM_LIBC_SRC___SUPPORT_OSUTIL_FCNTL_H
 
+#include "hdr/types/mode_t.h"
+#include "src/__support/error_or.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 
-int fcntl(int fd, int cmd, void *arg = nullptr);
+ErrorOr<int> fcntl(int fd, int cmd, void *arg = nullptr);
+
+ErrorOr<int> open(const char *path, int flags, mode_t mode_flags = 0);
+
+ErrorOr<int> close(int fd);
 
 } // namespace internal
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/__support/OSUtil/linux/CMakeLists.txt b/external/llvm-project/libc/src/__support/OSUtil/linux/CMakeLists.txt
index b9704d42cd33..4681d8c2bb73 100644
--- a/external/llvm-project/libc/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/external/llvm-project/libc/src/__support/OSUtil/linux/CMakeLists.txt
@@ -16,7 +16,6 @@ add_object_library(
     .${LIBC_TARGET_ARCHITECTURE}.linux_${LIBC_TARGET_ARCHITECTURE}_util
     libc.src.__support.common
     libc.src.__support.CPP.string_view
-    libc.src.errno.errno
     libc.hdr.fcntl_macros
     libc.hdr.types.struct_flock
     libc.hdr.types.struct_flock64
diff --git a/external/llvm-project/libc/src/__support/OSUtil/linux/fcntl.cpp b/external/llvm-project/libc/src/__support/OSUtil/linux/fcntl.cpp
index 4742b2a00220..bb76eee90efd 100644
--- a/external/llvm-project/libc/src/__support/OSUtil/linux/fcntl.cpp
+++ b/external/llvm-project/libc/src/__support/OSUtil/linux/fcntl.cpp
@@ -8,23 +8,24 @@
 
 #include "src/__support/OSUtil/fcntl.h"
 
+#include "hdr/errno_macros.h"
 #include "hdr/fcntl_macros.h"
+#include "hdr/types/mode_t.h"
 #include "hdr/types/off_t.h"
 #include "hdr/types/struct_f_owner_ex.h"
 #include "hdr/types/struct_flock.h"
 #include "hdr/types/struct_flock64.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/error_or.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
-#include <stdarg.h>
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 
-int fcntl(int fd, int cmd, void *arg) {
+ErrorOr<int> fcntl(int fd, int cmd, void *arg) {
 #if SYS_fcntl
   constexpr auto FCNTL_SYSCALL_ID = SYS_fcntl;
 #elif defined(SYS_fcntl64)
@@ -33,8 +34,7 @@ int fcntl(int fd, int cmd, void *arg) {
 #error "fcntl and fcntl64 syscalls not available."
 #endif
 
-  int new_cmd = cmd;
-  switch (new_cmd) {
+  switch (cmd) {
   case F_OFD_SETLKW: {
     struct flock *flk = reinterpret_cast<struct flock *>(arg);
     // convert the struct to a flock64
@@ -45,8 +45,11 @@ int fcntl(int fd, int cmd, void *arg) {
     flk64.l_len = flk->l_len;
     flk64.l_pid = flk->l_pid;
     // create a syscall
-    return LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, new_cmd,
-                                             &flk64);
+    int ret =
+        LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, cmd, &flk64);
+    if (ret < 0)
+      return Error(-ret);
+    return ret;
   }
   case F_OFD_GETLK:
   case F_OFD_SETLK: {
@@ -59,60 +62,80 @@ int fcntl(int fd, int cmd, void *arg) {
     flk64.l_len = flk->l_len;
     flk64.l_pid = flk->l_pid;
     // create a syscall
-    int retVal = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd,
-                                                   new_cmd, &flk64);
+    int ret =
+        LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, cmd, &flk64);
     // On failure, return
-    if (retVal == -1)
-      return -1;
+    if (ret < 0)
+      return Error(-1);
     // Check for overflow, i.e. the offsets are not the same when cast
     // to off_t from off64_t.
     if (static_cast<off_t>(flk64.l_len) != flk64.l_len ||
-        static_cast<off_t>(flk64.l_start) != flk64.l_start) {
-      libc_errno = EOVERFLOW;
-      return -1;
-    }
+        static_cast<off_t>(flk64.l_start) != flk64.l_start)
+      return Error(EOVERFLOW);
+
     // Now copy back into flk, in case flk64 got modified
     flk->l_type = flk64.l_type;
     flk->l_whence = flk64.l_whence;
     flk->l_start = static_cast<decltype(flk->l_start)>(flk64.l_start);
     flk->l_len = static_cast<decltype(flk->l_len)>(flk64.l_len);
     flk->l_pid = flk64.l_pid;
-    return retVal;
+    return ret;
   }
   case F_GETOWN: {
     struct f_owner_ex fex;
     int ret = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd,
                                                 F_GETOWN_EX, &fex);
-    if (ret >= 0)
-      return fex.type == F_OWNER_PGRP ? -fex.pid : fex.pid;
-    libc_errno = -ret;
-    return -1;
+    if (ret < 0)
+      return Error(-ret);
+    return fex.type == F_OWNER_PGRP ? -fex.pid : fex.pid;
   }
 #ifdef SYS_fcntl64
   case F_GETLK: {
     if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
-      new_cmd = F_GETLK64;
+      cmd = F_GETLK64;
     break;
   }
   case F_SETLK: {
     if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
-      new_cmd = F_SETLK64;
+      cmd = F_SETLK64;
     break;
   }
   case F_SETLKW: {
     if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64)
-      new_cmd = F_SETLKW64;
+      cmd = F_SETLKW64;
     break;
   }
 #endif
   }
-  int retVal = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, new_cmd,
-                                                 reinterpret_cast<void *>(arg));
-  if (retVal >= 0) {
-    return retVal;
-  }
-  libc_errno = -retVal;
-  return -1;
+
+  // default, but may use rewritten cmd from above.
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(FCNTL_SYSCALL_ID, fd, cmd,
+                                              reinterpret_cast<void *>(arg));
+  if (ret < 0)
+    return Error(-ret);
+  return ret;
+}
+
+ErrorOr<int> open(const char *path, int flags, mode_t mode_flags) {
+#ifdef SYS_open
+  int fd = LIBC_NAMESPACE::syscall_impl<int>(SYS_open, path, flags, mode_flags);
+#else
+  int fd = LIBC_NAMESPACE::syscall_impl<int>(SYS_openat, AT_FDCWD, path, flags,
+                                             mode_flags);
+#endif
+  if (fd < 0)
+    return Error(-fd);
+
+  return fd;
+}
+
+ErrorOr<int> close(int fd) {
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_close, fd);
+
+  if (ret < 0)
+    return Error(-ret);
+
+  return ret;
 }
 
 } // namespace internal
diff --git a/external/llvm-project/libc/src/__support/OSUtil/linux/vdso.cpp b/external/llvm-project/libc/src/__support/OSUtil/linux/vdso.cpp
index 8c9bd3e1bcc7..e4e53c3c2a0f 100644
--- a/external/llvm-project/libc/src/__support/OSUtil/linux/vdso.cpp
+++ b/external/llvm-project/libc/src/__support/OSUtil/linux/vdso.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/string_view.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/threads/callonce.h"
 #include "src/__support/threads/linux/futex_word.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/auxv/getauxval.h"
 #include <linux/auxvec.h>
 
diff --git a/external/llvm-project/libc/src/__support/StringUtil/tables/linux_extension_errors.h b/external/llvm-project/libc/src/__support/StringUtil/tables/linux_extension_errors.h
index 425590f6e91c..de637d60bea9 100644
--- a/external/llvm-project/libc/src/__support/StringUtil/tables/linux_extension_errors.h
+++ b/external/llvm-project/libc/src/__support/StringUtil/tables/linux_extension_errors.h
@@ -10,8 +10,8 @@
 #define LLVM_LIBC_SRC___SUPPORT_STRINGUTIL_TABLES_LINUX_EXTENSION_ERRORS_H
 
 #include "src/__support/StringUtil/message_mapper.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/__support/libc_errno.h b/external/llvm-project/libc/src/__support/libc_errno.h
new file mode 100644
index 000000000000..ab5f6a9c4b9d
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/libc_errno.h
@@ -0,0 +1,108 @@
+//===-- Implementation header for libc_errno --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_LIBC_ERRNO_H
+#define LLVM_LIBC_SRC___SUPPORT_LIBC_ERRNO_H
+
+// This header is to be consumed by internal implementations, in which all of
+// them should refer to `libc_errno` instead of using `errno` directly from
+// <errno.h> header.
+
+// Unit and hermetic tests should:
+// - #include "src/__support/libc_errno.h"
+// - NOT #include <errno.h>
+// - Only use `libc_errno` in the code
+// - Depend on libc.src.errno.errno
+
+// Integration tests should:
+// - NOT #include "src/__support/libc_errno.h"
+// - #include <errno.h>
+// - Use regular `errno` in the code
+// - Still depend on libc.src.errno.errno
+
+// libc uses a fallback default value, either system or thread local.
+#define LIBC_ERRNO_MODE_DEFAULT 0
+// libc never stores a value; `errno` macro uses get link-time failure.
+#define LIBC_ERRNO_MODE_UNDEFINED 1
+// libc maintains per-thread state (requires C++ `thread_local` support).
+#define LIBC_ERRNO_MODE_THREAD_LOCAL 2
+// libc maintains shared state used by all threads, contrary to standard C
+// semantics unless always single-threaded; nothing prevents data races.
+#define LIBC_ERRNO_MODE_SHARED 3
+// libc doesn't maintain any internal state, instead the embedder must define
+// `int *__llvm_libc_errno(void);` C function.
+#define LIBC_ERRNO_MODE_EXTERNAL 4
+// libc uses system `<errno.h>` `errno` macro directly in the overlay mode; in
+// fullbuild mode, effectively the same as `LIBC_ERRNO_MODE_EXTERNAL`.
+// In this mode, the public C++ symbol `LIBC_NAMESPACE::libc_errno ` is still
+// exported and get redirected to the system `errno` inside its implementation.
+
+// TODO: Investigate deprecating LIBC_ERRNO_MODE_SYSTEM in favor of
+//       LIBC_ERRNO_MODE_SYSTEM_INLINE.
+//       https://github.com/llvm/llvm-project/issues/143454
+#define LIBC_ERRNO_MODE_SYSTEM 5
+// In this mode, the libc_errno is simply a macro resolved to `errno` from the
+// system header <errno.h>.  There is no need to link against the
+// `libc.src.errno.errno` object.
+#define LIBC_ERRNO_MODE_SYSTEM_INLINE 6
+
+#if !defined(LIBC_ERRNO_MODE) || LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_DEFAULT
+#undef LIBC_ERRNO_MODE
+#if defined(LIBC_FULL_BUILD) || !defined(LIBC_COPT_PUBLIC_PACKAGING)
+#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_THREAD_LOCAL
+#else
+#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM
+#endif
+#endif // LIBC_ERRNO_MODE
+
+#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_DEFAULT &&                              \
+    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_UNDEFINED &&                            \
+    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_THREAD_LOCAL &&                         \
+    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SHARED &&                               \
+    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_EXTERNAL &&                             \
+    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM &&                               \
+    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE
+#error LIBC_ERRNO_MODE must be one of the following values: \
+LIBC_ERRNO_MODE_DEFAULT, \
+LIBC_ERRNO_MODE_UNDEFINED, \
+LIBC_ERRNO_MODE_THREAD_LOCAL, \
+LIBC_ERRNO_MODE_SHARED, \
+LIBC_ERRNO_MODE_EXTERNAL, \
+LIBC_ERRNO_MODE_SYSTEM, \
+LIBC_ERRNO_MODE_SYSTEM_INLINE.
+#endif
+
+#if LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_SYSTEM_INLINE
+
+#include <errno.h>
+
+#define libc_errno errno
+
+#else // !LIBC_ERRNO_MODE_SYSTEM_INLINE
+
+#include "hdr/errno_macros.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+extern "C" int *__llvm_libc_errno() noexcept;
+
+struct Errno {
+  void operator=(int);
+  operator int();
+};
+
+extern Errno libc_errno;
+
+} // namespace LIBC_NAMESPACE_DECL
+
+using LIBC_NAMESPACE::libc_errno;
+
+#endif // LIBC_ERRNO_MODE_SYSTEM_INLINE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_LIBC_ERRNO_H
diff --git a/external/llvm-project/libc/src/__support/math/CMakeLists.txt b/external/llvm-project/libc/src/__support/math/CMakeLists.txt
new file mode 100644
index 000000000000..66c1d19a1cab
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/math/CMakeLists.txt
@@ -0,0 +1,24 @@
+add_header_library(
+  exp_float_constants
+  HDRS
+    exp_float_constants.h
+  DEPENDS
+    libc.src.__support.macros.config
+)
+
+add_header_library(
+  expf
+  HDRS
+    expf.h
+  DEPENDS
+    .exp_float_constants
+    libc.src.__support.common
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.nearest_integer
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.FPUtil.rounding_mode
+    libc.src.__support.macros.config
+    libc.src.__support.macros.optimization
+)
diff --git a/external/llvm-project/libc/src/__support/math/exp_float_constants.h b/external/llvm-project/libc/src/__support/math/exp_float_constants.h
new file mode 100644
index 000000000000..cabb227a034b
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/math/exp_float_constants.h
@@ -0,0 +1,145 @@
+//===-- Look-up tables for exp*f functions ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXP_FLOAT_CONSTANTS_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_EXP_FLOAT_CONSTANTS_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+// Lookup table for exp(m) with m = -104, ..., 89.
+//   -104 = floor(log(single precision's min denormal))
+//     89 = ceil(log(single precision's max normal))
+// Table is generated with Sollya as follow:
+// > display = hexadecimal;
+// > for i from -104 to 89 do { D(exp(i)); };
+static constexpr double EXP_M1[195] = {
+    0x1.f1e6b68529e33p-151, 0x1.525be4e4e601dp-149, 0x1.cbe0a45f75eb1p-148,
+    0x1.3884e838aea68p-146, 0x1.a8c1f14e2af5dp-145, 0x1.20a717e64a9bdp-143,
+    0x1.8851d84118908p-142, 0x1.0a9bdfb02d240p-140, 0x1.6a5bea046b42ep-139,
+    0x1.ec7f3b269efa8p-138, 0x1.4eafb87eab0f2p-136, 0x1.c6e2d05bbc000p-135,
+    0x1.35208867c2683p-133, 0x1.a425b317eeacdp-132, 0x1.1d8508fa8246ap-130,
+    0x1.840fbc08fdc8ap-129, 0x1.07b7112bc1ffep-127, 0x1.666d0dad2961dp-126,
+    0x1.e726c3f64d0fep-125, 0x1.4b0dc07cabf98p-123, 0x1.c1f2daf3b6a46p-122,
+    0x1.31c5957a47de2p-120, 0x1.9f96445648b9fp-119, 0x1.1a6baeadb4fd1p-117,
+    0x1.7fd974d372e45p-116, 0x1.04da4d1452919p-114, 0x1.62891f06b3450p-113,
+    0x1.e1dd273aa8a4ap-112, 0x1.4775e0840bfddp-110, 0x1.bd109d9d94bdap-109,
+    0x1.2e73f53fba844p-107, 0x1.9b138170d6bfep-106, 0x1.175af0cf60ec5p-104,
+    0x1.7baee1bffa80bp-103, 0x1.02057d1245cebp-101, 0x1.5eafffb34ba31p-100,
+    0x1.dca23bae16424p-99,  0x1.43e7fc88b8056p-97,  0x1.b83bf23a9a9ebp-96,
+    0x1.2b2b8dd05b318p-94,  0x1.969d47321e4ccp-93,  0x1.1452b7723aed2p-91,
+    0x1.778fe2497184cp-90,  0x1.fe7116182e9ccp-89,  0x1.5ae191a99585ap-87,
+    0x1.d775d87da854dp-86,  0x1.4063f8cc8bb98p-84,  0x1.b374b315f87c1p-83,
+    0x1.27ec458c65e3cp-81,  0x1.923372c67a074p-80,  0x1.1152eaeb73c08p-78,
+    0x1.737c5645114b5p-77,  0x1.f8e6c24b5592ep-76,  0x1.571db733a9d61p-74,
+    0x1.d257d547e083fp-73,  0x1.3ce9b9de78f85p-71,  0x1.aebabae3a41b5p-70,
+    0x1.24b6031b49bdap-68,  0x1.8dd5e1bb09d7ep-67,  0x1.0e5b73d1ff53dp-65,
+    0x1.6f741de1748ecp-64,  0x1.f36bd37f42f3ep-63,  0x1.536452ee2f75cp-61,
+    0x1.cd480a1b74820p-60,  0x1.39792499b1a24p-58,  0x1.aa0de4bf35b38p-57,
+    0x1.2188ad6ae3303p-55,  0x1.898471fca6055p-54,  0x1.0b6c3afdde064p-52,
+    0x1.6b7719a59f0e0p-51,  0x1.ee001eed62aa0p-50,  0x1.4fb547c775da8p-48,
+    0x1.c8464f7616468p-47,  0x1.36121e24d3bbap-45,  0x1.a56e0c2ac7f75p-44,
+    0x1.1e642baeb84a0p-42,  0x1.853f01d6d53bap-41,  0x1.0885298767e9ap-39,
+    0x1.67852a7007e42p-38,  0x1.e8a37a45fc32ep-37,  0x1.4c1078fe9228ap-35,
+    0x1.c3527e433fab1p-34,  0x1.32b48bf117da2p-32,  0x1.a0db0d0ddb3ecp-31,
+    0x1.1b48655f37267p-29,  0x1.81056ff2c5772p-28,  0x1.05a628c699fa1p-26,
+    0x1.639e3175a689dp-25,  0x1.e355bbaee85cbp-24,  0x1.4875ca227ec38p-22,
+    0x1.be6c6fdb01612p-21,  0x1.2f6053b981d98p-19,  0x1.9c54c3b43bc8bp-18,
+    0x1.18354238f6764p-16,  0x1.7cd79b5647c9bp-15,  0x1.02cf22526545ap-13,
+    0x1.5fc21041027adp-12,  0x1.de16b9c24a98fp-11,  0x1.44e51f113d4d6p-9,
+    0x1.b993fe00d5376p-8,   0x1.2c155b8213cf4p-6,   0x1.97db0ccceb0afp-5,
+    0x1.152aaa3bf81ccp-3,   0x1.78b56362cef38p-2,   0x1.0000000000000p+0,
+    0x1.5bf0a8b145769p+1,   0x1.d8e64b8d4ddaep+2,   0x1.415e5bf6fb106p+4,
+    0x1.b4c902e273a58p+5,   0x1.28d389970338fp+7,   0x1.936dc5690c08fp+8,
+    0x1.122885aaeddaap+10,  0x1.749ea7d470c6ep+11,  0x1.fa7157c470f82p+12,
+    0x1.5829dcf950560p+14,  0x1.d3c4488ee4f7fp+15,  0x1.3de1654d37c9ap+17,
+    0x1.b00b5916ac955p+18,  0x1.259ac48bf05d7p+20,  0x1.8f0ccafad2a87p+21,
+    0x1.0f2ebd0a80020p+23,  0x1.709348c0ea4f9p+24,  0x1.f4f22091940bdp+25,
+    0x1.546d8f9ed26e1p+27,  0x1.ceb088b68e804p+28,  0x1.3a6e1fd9eecfdp+30,
+    0x1.ab5adb9c43600p+31,  0x1.226af33b1fdc1p+33,  0x1.8ab7fb5475fb7p+34,
+    0x1.0c3d3920962c9p+36,  0x1.6c932696a6b5dp+37,  0x1.ef822f7f6731dp+38,
+    0x1.50bba3796379ap+40,  0x1.c9aae4631c056p+41,  0x1.370470aec28edp+43,
+    0x1.a6b765d8cdf6dp+44,  0x1.1f43fcc4b662cp+46,  0x1.866f34a725782p+47,
+    0x1.0953e2f3a1ef7p+49,  0x1.689e221bc8d5bp+50,  0x1.ea215a1d20d76p+51,
+    0x1.4d13fbb1a001ap+53,  0x1.c4b334617cc67p+54,  0x1.33a43d282a519p+56,
+    0x1.a220d397972ebp+57,  0x1.1c25c88df6862p+59,  0x1.8232558201159p+60,
+    0x1.0672a3c9eb871p+62,  0x1.64b41c6d37832p+63,  0x1.e4cf766fe49bep+64,
+    0x1.49767bc0483e3p+66,  0x1.bfc951eb8bb76p+67,  0x1.304d6aeca254bp+69,
+    0x1.9d97010884251p+70,  0x1.19103e4080b45p+72,  0x1.7e013cd114461p+73,
+    0x1.03996528e074cp+75,  0x1.60d4f6fdac731p+76,  0x1.df8c5af17ba3bp+77,
+    0x1.45e3076d61699p+79,  0x1.baed16a6e0da7p+80,  0x1.2cffdfebde1a1p+82,
+    0x1.9919cabefcb69p+83,  0x1.160345c9953e3p+85,  0x1.79dbc9dc53c66p+86,
+    0x1.00c810d464097p+88,  0x1.5d009394c5c27p+89,  0x1.da57de8f107a8p+90,
+    0x1.425982cf597cdp+92,  0x1.b61e5ca3a5e31p+93,  0x1.29bb825dfcf87p+95,
+    0x1.94a90db0d6fe2p+96,  0x1.12fec759586fdp+98,  0x1.75c1dc469e3afp+99,
+    0x1.fbfd219c43b04p+100, 0x1.5936d44e1a146p+102, 0x1.d531d8a7ee79cp+103,
+    0x1.3ed9d24a2d51bp+105, 0x1.b15cfe5b6e17bp+106, 0x1.268038c2c0e00p+108,
+    0x1.9044a73545d48p+109, 0x1.1002ab6218b38p+111, 0x1.71b3540cbf921p+112,
+    0x1.f6799ea9c414ap+113, 0x1.55779b984f3ebp+115, 0x1.d01a210c44aa4p+116,
+    0x1.3b63da8e91210p+118, 0x1.aca8d6b0116b8p+119, 0x1.234de9e0c74e9p+121,
+    0x1.8bec7503ca477p+122, 0x1.0d0eda9796b90p+124, 0x1.6db0118477245p+125,
+    0x1.f1056dc7bf22dp+126, 0x1.51c2cc3433801p+128, 0x1.cb108ffbec164p+129,
+};
+
+// Lookup table for exp(m * 2^(-7)) with m = 0, ..., 127.
+// Table is generated with Sollya as follow:
+// > display = hexadecimal;
+// > for i from 0 to 127 do { D(exp(i / 128)); };
+static constexpr double EXP_M2[128] = {
+    0x1.0000000000000p0, 0x1.0202015600446p0, 0x1.04080ab55de39p0,
+    0x1.06122436410ddp0, 0x1.08205601127edp0, 0x1.0a32a84e9c1f6p0,
+    0x1.0c49236829e8cp0, 0x1.0e63cfa7ab09dp0, 0x1.1082b577d34edp0,
+    0x1.12a5dd543ccc5p0, 0x1.14cd4fc989cd6p0, 0x1.16f9157587069p0,
+    0x1.192937074e0cdp0, 0x1.1b5dbd3f68122p0, 0x1.1d96b0eff0e79p0,
+    0x1.1fd41afcba45ep0, 0x1.2216045b6f5cdp0, 0x1.245c7613b8a9bp0,
+    0x1.26a7793f60164p0, 0x1.28f7170a755fdp0, 0x1.2b4b58b372c79p0,
+    0x1.2da4478b620c7p0, 0x1.3001ecf601af7p0, 0x1.32645269ea829p0,
+    0x1.34cb8170b5835p0, 0x1.373783a722012p0, 0x1.39a862bd3c106p0,
+    0x1.3c1e2876834aap0, 0x1.3e98deaa11dccp0, 0x1.41188f42c3e32p0,
+    0x1.439d443f5f159p0, 0x1.462707b2bac21p0, 0x1.48b5e3c3e8186p0,
+    0x1.4b49e2ae5ac67p0, 0x1.4de30ec211e60p0, 0x1.50817263c13cdp0,
+    0x1.5325180cfacf7p0, 0x1.55ce0a4c58c7cp0, 0x1.587c53c5a7af0p0,
+    0x1.5b2fff3210fd9p0, 0x1.5de9176045ff5p0, 0x1.60a7a734ab0e8p0,
+    0x1.636bb9a983258p0, 0x1.663559cf1bc7cp0, 0x1.690492cbf9433p0,
+    0x1.6bd96fdd034a2p0, 0x1.6eb3fc55b1e76p0, 0x1.719443a03acb9p0,
+    0x1.747a513dbef6ap0, 0x1.776630c678bc1p0, 0x1.7a57ede9ea23ep0,
+    0x1.7d4f946f0ba8dp0, 0x1.804d30347b546p0, 0x1.8350cd30ac390p0,
+    0x1.865a7772164c5p0, 0x1.896a3b1f66a0ep0, 0x1.8c802477b0010p0,
+    0x1.8f9c3fd29beafp0, 0x1.92be99a09bf00p0, 0x1.95e73e6b1b75ep0,
+    0x1.99163ad4b1dccp0, 0x1.9c4b9b995509bp0, 0x1.9f876d8e8c566p0,
+    0x1.a2c9bda3a3e78p0, 0x1.a61298e1e069cp0, 0x1.a9620c6cb3374p0,
+    0x1.acb82581eee54p0, 0x1.b014f179fc3b8p0, 0x1.b3787dc80f95fp0,
+    0x1.b6e2d7fa5eb18p0, 0x1.ba540dba56e56p0, 0x1.bdcc2cccd3c85p0,
+    0x1.c14b431256446p0, 0x1.c4d15e873c193p0, 0x1.c85e8d43f7cd0p0,
+    0x1.cbf2dd7d490f2p0, 0x1.cf8e5d84758a9p0, 0x1.d3311bc7822b4p0,
+    0x1.d6db26d16cd67p0, 0x1.da8c8d4a66969p0, 0x1.de455df80e3c0p0,
+    0x1.e205a7bdab73ep0, 0x1.e5cd799c6a54ep0, 0x1.e99ce2b397649p0,
+    0x1.ed73f240dc142p0, 0x1.f152b7a07bb76p0, 0x1.f539424d90f5ep0,
+    0x1.f927a1e24bb76p0, 0x1.fd1de6182f8c9p0, 0x1.008e0f64294abp1,
+    0x1.02912df5ce72ap1, 0x1.049856cd84339p1, 0x1.06a39207f0a09p1,
+    0x1.08b2e7d2035cfp1, 0x1.0ac6606916501p1, 0x1.0cde041b0e9aep1,
+    0x1.0ef9db467dcf8p1, 0x1.1119ee5ac36b6p1, 0x1.133e45d82e952p1,
+    0x1.1566ea50201d7p1, 0x1.1793e4652cc50p1, 0x1.19c53ccb3fc6bp1,
+    0x1.1bfafc47bda73p1, 0x1.1e352bb1a74adp1, 0x1.2073d3f1bd518p1,
+    0x1.22b6fe02a3b9cp1, 0x1.24feb2f105cb8p1, 0x1.274afbdbba4a6p1,
+    0x1.299be1f3e7f1cp1, 0x1.2bf16e7d2a38cp1, 0x1.2e4baacdb6614p1,
+    0x1.30aaa04e80d05p1, 0x1.330e587b62b28p1, 0x1.3576dce33feadp1,
+    0x1.37e437282d4eep1, 0x1.3a5670ff972edp1, 0x1.3ccd9432682b4p1,
+    0x1.3f49aa9d30590p1, 0x1.41cabe304cb34p1, 0x1.4450d8f00edd4p1,
+    0x1.46dc04f4e5338p1, 0x1.496c4c6b832dap1, 0x1.4c01b9950a111p1,
+    0x1.4e9c56c731f5dp1, 0x1.513c2e6c731d7p1, 0x1.53e14b042f9cap1,
+    0x1.568bb722dd593p1, 0x1.593b7d72305bbp1,
+};
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXP_FLOAT_CONSTANTS_H
diff --git a/external/llvm-project/libc/src/__support/math/expf.h b/external/llvm-project/libc/src/__support/math/expf.h
new file mode 100644
index 000000000000..88c151492a04
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/math/expf.h
@@ -0,0 +1,116 @@
+//===-- Implementation header for expf --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXPF_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_EXPF_H
+
+#include "exp_float_constants.h" // Lookup tables EXP_M1 and EXP_M2.
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/nearest_integer.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+static constexpr float expf(float x) {
+  using FPBits = typename fputil::FPBits<float>;
+  FPBits xbits(x);
+
+  uint32_t x_u = xbits.uintval();
+  uint32_t x_abs = x_u & 0x7fff'ffffU;
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  // Exceptional values
+  if (LIBC_UNLIKELY(x_u == 0xc236'bd8cU)) { // x = -0x1.6d7b18p+5f
+    return 0x1.108a58p-66f - x * 0x1.0p-95f;
+  }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
+  // When |x| >= 89, |x| < 2^-25, or x is nan
+  if (LIBC_UNLIKELY(x_abs >= 0x42b2'0000U || x_abs <= 0x3280'0000U)) {
+    // |x| < 2^-25
+    if (xbits.get_biased_exponent() <= 101) {
+      return 1.0f + x;
+    }
+
+    // When x < log(2^-150) or nan
+    if (xbits.uintval() >= 0xc2cf'f1b5U) {
+      // exp(-Inf) = 0
+      if (xbits.is_inf())
+        return 0.0f;
+      // exp(nan) = nan
+      if (xbits.is_nan())
+        return x;
+      if (fputil::fenv_is_round_up())
+        return FPBits::min_subnormal().get_val();
+      fputil::set_errno_if_required(ERANGE);
+      fputil::raise_except_if_required(FE_UNDERFLOW);
+      return 0.0f;
+    }
+    // x >= 89 or nan
+    if (xbits.is_pos() && (xbits.uintval() >= 0x42b2'0000)) {
+      // x is finite
+      if (xbits.uintval() < 0x7f80'0000U) {
+        int rounding = fputil::quick_get_round();
+        if (rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO)
+          return FPBits::max_normal().get_val();
+
+        fputil::set_errno_if_required(ERANGE);
+        fputil::raise_except_if_required(FE_OVERFLOW);
+      }
+      // x is +inf or nan
+      return x + FPBits::inf().get_val();
+    }
+  }
+  // For -104 < x < 89, to compute exp(x), we perform the following range
+  // reduction: find hi, mid, lo such that:
+  //   x = hi + mid + lo, in which
+  //     hi is an integer,
+  //     mid * 2^7 is an integer
+  //     -2^(-8) <= lo < 2^-8.
+  // In particular,
+  //   hi + mid = round(x * 2^7) * 2^(-7).
+  // Then,
+  //   exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo).
+  // We store exp(hi) and exp(mid) in the lookup tables EXP_M1 and EXP_M2
+  // respectively.  exp(lo) is computed using a degree-4 minimax polynomial
+  // generated by Sollya.
+
+  // x_hi = (hi + mid) * 2^7 = round(x * 2^7).
+  float kf = fputil::nearest_integer(x * 0x1.0p7f);
+  // Subtract (hi + mid) from x to get lo.
+  double xd = static_cast<double>(fputil::multiply_add(kf, -0x1.0p-7f, x));
+  int x_hi = static_cast<int>(kf);
+  x_hi += 104 << 7;
+  // hi = x_hi >> 7
+  double exp_hi = EXP_M1[x_hi >> 7];
+  // mid * 2^7 = x_hi & 0x0000'007fU;
+  double exp_mid = EXP_M2[x_hi & 0x7f];
+  // Degree-4 minimax polynomial generated by Sollya with the following
+  // commands:
+  //   > display = hexadecimal;
+  //   > Q = fpminimax(expm1(x)/x, 3, [|D...|], [-2^-8, 2^-8]);
+  //   > Q;
+  double exp_lo =
+      fputil::polyeval(xd, 0x1p0, 0x1.ffffffffff777p-1, 0x1.000000000071cp-1,
+                       0x1.555566668e5e7p-3, 0x1.55555555ef243p-5);
+  return static_cast<float>(exp_hi * exp_mid * exp_lo);
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXPF_H
diff --git a/external/llvm-project/libc/src/__support/threads/linux/thread.cpp b/external/llvm-project/libc/src/__support/threads/linux/thread.cpp
index c531d74c5335..baad26aed685 100644
--- a/external/llvm-project/libc/src/__support/threads/linux/thread.cpp
+++ b/external/llvm-project/libc/src/__support/threads/linux/thread.cpp
@@ -14,9 +14,9 @@
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
 #include "src/__support/common.h"
 #include "src/__support/error_or.h"
+#include "src/__support/libc_errno.h" // For error macros
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/linux/futex_utils.h" // For FutexWordType
-#include "src/errno/libc_errno.h"                    // For error macros
 
 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
 #include <arm_acle.h>
diff --git a/external/llvm-project/libc/src/__support/wchar/CMakeLists.txt b/external/llvm-project/libc/src/__support/wchar/CMakeLists.txt
new file mode 100644
index 000000000000..5cca58400ff4
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/wchar/CMakeLists.txt
@@ -0,0 +1,26 @@
+add_header_library(
+  mbstate
+  HDRS
+    mbstate.h
+  DEPENDS
+    libc.hdr.types.char32_t    
+)
+
+add_object_library(
+  character_converter
+  HDRS
+    character_converter.h
+  SRCS 
+    character_converter.cpp
+  DEPENDS
+    libc.hdr.types.char8_t
+    libc.hdr.types.char32_t
+    .mbstate
+    .utf_ret
+)
+
+add_header_library(
+  utf_ret
+  HDRS
+    utf_ret.h
+)
diff --git a/external/llvm-project/libc/src/__support/wchar/character_converter.cpp b/external/llvm-project/libc/src/__support/wchar/character_converter.cpp
new file mode 100644
index 000000000000..f09c7815a6cc
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/wchar/character_converter.cpp
@@ -0,0 +1,34 @@
+//===-- Implementation of a class for conversion --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/char32_t.h"
+#include "hdr/types/char8_t.h"
+#include "src/__support/wchar/mbstate.h"
+#include "src/__support/wchar/utf_ret.h"
+
+#include "character_converter.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+CharacterConverter::CharacterConverter(mbstate *mbstate) { state = mbstate; }
+
+bool CharacterConverter::isComplete() {
+  return state->bytes_processed == state->total_bytes;
+}
+
+int CharacterConverter::push(char8_t utf8_byte) {}
+
+int CharacterConverter::push(char32_t utf32) {}
+
+utf_ret<char8_t> CharacterConverter::pop_utf8() {}
+
+utf_ret<char32_t> CharacterConverter::pop_utf32() {}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/__support/wchar/character_converter.h b/external/llvm-project/libc/src/__support/wchar/character_converter.h
new file mode 100644
index 000000000000..d0602d2defe2
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/wchar/character_converter.h
@@ -0,0 +1,39 @@
+//===-- Definition of a class for mbstate_t and conversion -----*-- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
+#define LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
+
+#include "hdr/types/char32_t.h"
+#include "hdr/types/char8_t.h"
+#include "src/__support/wchar/mbstate.h"
+#include "src/__support/wchar/utf_ret.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+class CharacterConverter {
+private:
+  mbstate *state;
+
+public:
+  CharacterConverter(mbstate *mbstate);
+
+  bool isComplete();
+
+  int push(char8_t utf8_byte);
+  int push(char32_t utf32);
+
+  utf_ret<char8_t> pop_utf8();
+  utf_ret<char32_t> pop_utf32();
+};
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
diff --git a/external/llvm-project/libc/src/__support/wchar/mbstate.h b/external/llvm-project/libc/src/__support/wchar/mbstate.h
new file mode 100644
index 000000000000..d33ee354a544
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/wchar/mbstate.h
@@ -0,0 +1,28 @@
+//===-- Definition of mbstate-----------------------------------*-- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MBSTATE_H
+#define LLVM_LIBC_SRC___SUPPORT_MBSTATE_H
+
+#include "hdr/types/char32_t.h"
+#include "src/__support/common.h"
+#include <stdint.h>
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+struct mbstate {
+  char32_t partial;
+  uint8_t bytes_processed;
+  uint8_t total_bytes;
+};
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MBSTATE_H
diff --git a/external/llvm-project/libc/src/__support/wchar/utf_ret.h b/external/llvm-project/libc/src/__support/wchar/utf_ret.h
new file mode 100644
index 000000000000..fa99b76159bd
--- /dev/null
+++ b/external/llvm-project/libc/src/__support/wchar/utf_ret.h
@@ -0,0 +1,24 @@
+//===-- Definition of utf_ret ----------------------------------*-- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_UTF_RET_H
+#define LLVM_LIBC_SRC___SUPPORT_UTF_RET_H
+
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+template <typename T> struct utf_ret {
+  T out;
+  int error;
+};
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_UTF_RET_H
diff --git a/external/llvm-project/libc/src/dirent/closedir.cpp b/external/llvm-project/libc/src/dirent/closedir.cpp
index 1249ef94cf41..2f8f6f0c044d 100644
--- a/external/llvm-project/libc/src/dirent/closedir.cpp
+++ b/external/llvm-project/libc/src/dirent/closedir.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/File/dir.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <dirent.h>
 
diff --git a/external/llvm-project/libc/src/dirent/opendir.cpp b/external/llvm-project/libc/src/dirent/opendir.cpp
index fee14ef0f558..bf47d0edac18 100644
--- a/external/llvm-project/libc/src/dirent/opendir.cpp
+++ b/external/llvm-project/libc/src/dirent/opendir.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/File/dir.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <dirent.h>
 
diff --git a/external/llvm-project/libc/src/dirent/readdir.cpp b/external/llvm-project/libc/src/dirent/readdir.cpp
index ad460b5e80b8..f95f7c1ae864 100644
--- a/external/llvm-project/libc/src/dirent/readdir.cpp
+++ b/external/llvm-project/libc/src/dirent/readdir.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/File/dir.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <dirent.h>
 
diff --git a/external/llvm-project/libc/src/errno/CMakeLists.txt b/external/llvm-project/libc/src/errno/CMakeLists.txt
index 1d78a5eedff9..2852044e9416 100644
--- a/external/llvm-project/libc/src/errno/CMakeLists.txt
+++ b/external/llvm-project/libc/src/errno/CMakeLists.txt
@@ -1,28 +1,16 @@
 # If we are in full build mode, we will provide the errno definition ourselves,
 # and if we are in overlay mode, we will just re-use the system's errno.
-# We are passing LIBC_FULL_BUILD flag in full build mode so that the
-# implementation of libc_errno will know if we are in full build mode or not.
-
-# TODO: Move LIBC_FULL_BUILD flag to _get_common_compile_options.
-set(full_build_flag "")
-if(LLVM_LIBC_FULL_BUILD)
-  set(full_build_flag "-DLIBC_FULL_BUILD")
-endif()
-
-if(LIBC_CONF_ERRNO_MODE)
-  set(errno_config_copts "-DLIBC_ERRNO_MODE=${LIBC_CONF_ERRNO_MODE}")
-endif()
 
 add_entrypoint_object(
   errno
   SRCS
     libc_errno.cpp
   HDRS
-    libc_errno.h     # Include this
-  COMPILE_OPTIONS
-    ${full_build_flag}
-    ${errno_config_copts}
+    ../__support/libc_errno.h
   DEPENDS
     libc.hdr.errno_macros
     libc.src.__support.common
+    libc.src.__support.libc_errno
+    libc.src.__support.macros.attributes
+    libc.src.__support.macros.config
 )
diff --git a/external/llvm-project/libc/src/errno/libc_errno.cpp b/external/llvm-project/libc/src/errno/libc_errno.cpp
index d1600d1b050e..8ff1eec1b103 100644
--- a/external/llvm-project/libc/src/errno/libc_errno.cpp
+++ b/external/llvm-project/libc/src/errno/libc_errno.cpp
@@ -6,51 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "libc_errno.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
 
-// libc uses a fallback default value, either system or thread local.
-#define LIBC_ERRNO_MODE_DEFAULT 0
-// libc never stores a value; `errno` macro uses get link-time failure.
-#define LIBC_ERRNO_MODE_UNDEFINED 1
-// libc maintains per-thread state (requires C++ `thread_local` support).
-#define LIBC_ERRNO_MODE_THREAD_LOCAL 2
-// libc maintains shared state used by all threads, contrary to standard C
-// semantics unless always single-threaded; nothing prevents data races.
-#define LIBC_ERRNO_MODE_SHARED 3
-// libc doesn't maintain any internal state, instead the embedder must define
-// `int *__llvm_libc_errno(void);` C function.
-#define LIBC_ERRNO_MODE_EXTERNAL 4
-// libc uses system `<errno.h>` `errno` macro directly in the overlay mode; in
-// fullbuild mode, effectively the same as `LIBC_ERRNO_MODE_EXTERNAL`.
-#define LIBC_ERRNO_MODE_SYSTEM 5
-
-#if !defined(LIBC_ERRNO_MODE) || LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_DEFAULT
-#undef LIBC_ERRNO_MODE
-#if defined(LIBC_FULL_BUILD) || !defined(LIBC_COPT_PUBLIC_PACKAGING)
-#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_THREAD_LOCAL
-#else
-#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM
-#endif
-#endif // LIBC_ERRNO_MODE
-
-#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_DEFAULT &&                              \
-    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_UNDEFINED &&                            \
-    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_THREAD_LOCAL &&                         \
-    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SHARED &&                               \
-    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_EXTERNAL &&                             \
-    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM
-#error LIBC_ERRNO_MODE must be one of the following values: \
-LIBC_ERRNO_MODE_DEFAULT, \
-LIBC_ERRNO_MODE_UNDEFINED, \
-LIBC_ERRNO_MODE_THREAD_LOCAL, \
-LIBC_ERRNO_MODE_SHARED, \
-LIBC_ERRNO_MODE_EXTERNAL, \
-LIBC_ERRNO_MODE_SYSTEM
-#endif
-
 namespace LIBC_NAMESPACE_DECL {
 
+#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE
+
 #if LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_UNDEFINED
 
 void Errno::operator=(int) {}
@@ -93,4 +56,6 @@ Errno::operator int() { return errno; }
 // Define the global `libc_errno` instance.
 Errno libc_errno;
 
+#endif // LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/errno/libc_errno.h b/external/llvm-project/libc/src/errno/libc_errno.h
deleted file mode 100644
index 44ee2714843b..000000000000
--- a/external/llvm-project/libc/src/errno/libc_errno.h
+++ /dev/null
@@ -1,47 +0,0 @@
-//===-- Implementation header for libc_errno --------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H
-#define LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H
-
-#include "src/__support/macros/attributes.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/properties/architectures.h"
-
-#include "hdr/errno_macros.h"
-
-// This header is to be consumed by internal implementations, in which all of
-// them should refer to `libc_errno` instead of using `errno` directly from
-// <errno.h> header.
-
-// Unit and hermetic tests should:
-// - #include "src/errno/libc_errno.h"
-// - NOT #include <errno.h>
-// - Only use `libc_errno` in the code
-// - Depend on libc.src.errno.errno
-
-// Integration tests should:
-// - NOT #include "src/errno/libc_errno.h"
-// - #include <errno.h>
-// - Use regular `errno` in the code
-// - Still depend on libc.src.errno.errno
-
-namespace LIBC_NAMESPACE_DECL {
-
-extern "C" int *__llvm_libc_errno() noexcept;
-
-struct Errno {
-  void operator=(int);
-  operator int();
-};
-
-extern Errno libc_errno;
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H
diff --git a/external/llvm-project/libc/src/fcntl/linux/CMakeLists.txt b/external/llvm-project/libc/src/fcntl/linux/CMakeLists.txt
index 580db16cd413..c31eb3f438c1 100644
--- a/external/llvm-project/libc/src/fcntl/linux/CMakeLists.txt
+++ b/external/llvm-project/libc/src/fcntl/linux/CMakeLists.txt
@@ -19,6 +19,7 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.fcntl_macros
     libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
 )
 
 add_entrypoint_object(
diff --git a/external/llvm-project/libc/src/fcntl/linux/creat.cpp b/external/llvm-project/libc/src/fcntl/linux/creat.cpp
index 23abae243aed..71412a8e68c5 100644
--- a/external/llvm-project/libc/src/fcntl/linux/creat.cpp
+++ b/external/llvm-project/libc/src/fcntl/linux/creat.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/fcntl/linux/fcntl.cpp b/external/llvm-project/libc/src/fcntl/linux/fcntl.cpp
index a0c8459ced34..fd9c48eb562f 100644
--- a/external/llvm-project/libc/src/fcntl/linux/fcntl.cpp
+++ b/external/llvm-project/libc/src/fcntl/linux/fcntl.cpp
@@ -10,6 +10,7 @@
 
 #include "src/__support/OSUtil/fcntl.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 
 #include <stdarg.h>
@@ -22,7 +23,14 @@ LLVM_LIBC_FUNCTION(int, fcntl, (int fd, int cmd, ...)) {
   va_start(varargs, cmd);
   arg = va_arg(varargs, void *);
   va_end(varargs);
-  return LIBC_NAMESPACE::internal::fcntl(fd, cmd, arg);
+
+  auto result = LIBC_NAMESPACE::internal::fcntl(fd, cmd, arg);
+
+  if (!result.has_value()) {
+    libc_errno = result.error();
+    return -1;
+  }
+  return result.value();
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/fcntl/linux/open.cpp b/external/llvm-project/libc/src/fcntl/linux/open.cpp
index 8b699ecdd204..3a56d1055419 100644
--- a/external/llvm-project/libc/src/fcntl/linux/open.cpp
+++ b/external/llvm-project/libc/src/fcntl/linux/open.cpp
@@ -8,15 +8,13 @@
 
 #include "src/fcntl/open.h"
 
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
-
 #include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
+#include "src/__support/OSUtil/fcntl.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/config.h"
 #include <stdarg.h>
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -31,17 +29,13 @@ LLVM_LIBC_FUNCTION(int, open, (const char *path, int flags, ...)) {
     va_end(varargs);
   }
 
-#ifdef SYS_open
-  int fd = LIBC_NAMESPACE::syscall_impl<int>(SYS_open, path, flags, mode_flags);
-#else
-  int fd = LIBC_NAMESPACE::syscall_impl<int>(SYS_openat, AT_FDCWD, path, flags,
-                                             mode_flags);
-#endif
-  if (fd > 0)
-    return fd;
+  auto result = internal::open(path, flags, mode_flags);
 
-  libc_errno = -fd;
-  return -1;
+  if (!result.has_value()) {
+    libc_errno = result.error();
+    return -1;
+  }
+  return result.value();
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/fcntl/linux/openat.cpp b/external/llvm-project/libc/src/fcntl/linux/openat.cpp
index 6063d9c00ad6..b47ad1fb3bb0 100644
--- a/external/llvm-project/libc/src/fcntl/linux/openat.cpp
+++ b/external/llvm-project/libc/src/fcntl/linux/openat.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/types/mode_t.h"
 #include <stdarg.h>
diff --git a/external/llvm-project/libc/src/inttypes/strtoimax.cpp b/external/llvm-project/libc/src/inttypes/strtoimax.cpp
index 85f197c75d90..6e55a4b56aac 100644
--- a/external/llvm-project/libc/src/inttypes/strtoimax.cpp
+++ b/external/llvm-project/libc/src/inttypes/strtoimax.cpp
@@ -8,9 +8,9 @@
 
 #include "src/inttypes/strtoimax.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/inttypes/strtoumax.cpp b/external/llvm-project/libc/src/inttypes/strtoumax.cpp
index 2e9cbc9acba7..ce5a0a782d97 100644
--- a/external/llvm-project/libc/src/inttypes/strtoumax.cpp
+++ b/external/llvm-project/libc/src/inttypes/strtoumax.cpp
@@ -8,9 +8,9 @@
 
 #include "src/inttypes/strtoumax.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/math/amdgpu/CMakeLists.txt b/external/llvm-project/libc/src/math/amdgpu/CMakeLists.txt
index 23fd3169fddf..e2cd3b99c303 100644
--- a/external/llvm-project/libc/src/math/amdgpu/CMakeLists.txt
+++ b/external/llvm-project/libc/src/math/amdgpu/CMakeLists.txt
@@ -1,21 +1,3 @@
-# Math functions not yet available in the libc project, or those not yet tuned
-# for GPU workloads are provided as wrappers over vendor libraries. If we find
-# them ahead of time we will import them statically. Otherwise, we will keep
-# them as external references and expect them to be resolved by the user when
-# they compile. In the future,we will use implementations from the 'libc'
-# project and not provide these wrappers.
-find_package(AMDDeviceLibs QUIET HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
-if(AMDDeviceLibs_FOUND)
-  message(STATUS "Found the ROCm device library. Implementations falling back "
-                 "to the vendor libraries will be resolved statically.")
-  get_target_property(ocml_path ocml IMPORTED_LOCATION)
-  set(bitcode_link_flags
-      "SHELL:-Xclang -mlink-builtin-bitcode -Xclang ${ocml_path}")
-else()
-  message(STATUS "Could not find the ROCm device library. Unimplemented "
-                 "functions will be an external reference to the vendor libraries.")
-endif()
-
 add_entrypoint_object(
   ceil
   SRCS
@@ -346,152 +328,6 @@ add_entrypoint_object(
     -O2
 )
 
-# The following functions currently are not implemented natively and borrow from
-# existing implementations. This will be removed in the future.
-add_entrypoint_object(
-  acos
-  SRCS
-    acos.cpp
-  HDRS
-    ../acos.h
-  VENDOR
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-)
-
-add_entrypoint_object(
-  acosh
-  SRCS
-    acosh.cpp
-  HDRS
-    ../acosh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  asin
-  SRCS
-    asin.cpp
-  HDRS
-    ../asin.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  asinh
-  SRCS
-    asinh.cpp
-  HDRS
-    ../asinh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  atan
-  SRCS
-    atan.cpp
-  HDRS
-    ../atan.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  atanh
-  SRCS
-    atanh.cpp
-  HDRS
-    ../atanh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  cosh
-  SRCS
-    cosh.cpp
-  HDRS
-    ../cosh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  erf
-  SRCS
-    erf.cpp
-  HDRS
-    ../erf.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  powi
-  SRCS
-    powi.cpp
-  HDRS
-    ../powi.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  powif
-  SRCS
-    powif.cpp
-  HDRS
-    ../powif.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  sinh
-  SRCS
-    sinh.cpp
-  HDRS
-    ../sinh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  tanh
-  SRCS
-    tanh.cpp
-  HDRS
-    ../tanh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
 add_entrypoint_object(
   tgamma
   SRCS
@@ -501,7 +337,6 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
 
 add_entrypoint_object(
@@ -513,7 +348,6 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
 
 add_entrypoint_object(
@@ -525,7 +359,6 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
 
 add_entrypoint_object(
@@ -537,5 +370,4 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
diff --git a/external/llvm-project/libc/src/math/amdgpu/asin.cpp b/external/llvm-project/libc/src/math/amdgpu/asin.cpp
deleted file mode 100644
index a79641e1977c..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/asin.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU asin function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asin.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, asin, (double x)) { return __ocml_asin_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/asinf.cpp b/external/llvm-project/libc/src/math/amdgpu/asinf.cpp
deleted file mode 100644
index e70944a4d989..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/asinf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the asinf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asinf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, asinf, (float x)) { return __ocml_asin_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/asinh.cpp b/external/llvm-project/libc/src/math/amdgpu/asinh.cpp
deleted file mode 100644
index 642368592a99..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/asinh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU asinh function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asinh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, asinh, (double x)) { return __ocml_asinh_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/asinhf.cpp b/external/llvm-project/libc/src/math/amdgpu/asinhf.cpp
deleted file mode 100644
index bafa77f94600..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/asinhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the asinhf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asinhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, asinhf, (float x)) { return __ocml_asinh_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/atan.cpp b/external/llvm-project/libc/src/math/amdgpu/atan.cpp
deleted file mode 100644
index 49941e97096f..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/atan.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU atan function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atan.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, atan, (double x)) { return __ocml_atan_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/atanf.cpp b/external/llvm-project/libc/src/math/amdgpu/atanf.cpp
deleted file mode 100644
index ab1837dd026b..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/atanf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the atanf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atanf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, atanf, (float x)) { return __ocml_atan_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/atanh.cpp b/external/llvm-project/libc/src/math/amdgpu/atanh.cpp
deleted file mode 100644
index 091c155190f9..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/atanh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU atanh function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atanh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, atanh, (double x)) { return __ocml_atanh_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/atanhf.cpp b/external/llvm-project/libc/src/math/amdgpu/atanhf.cpp
deleted file mode 100644
index fa9cf3938396..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/atanhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the atanhf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atanhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, atanhf, (float x)) { return __ocml_atanh_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/cos.cpp b/external/llvm-project/libc/src/math/amdgpu/cos.cpp
deleted file mode 100644
index a4d4c94dd7ac..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/cos.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the cos function for GPU ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/cos.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, cos, (double x)) { return __ocml_cos_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/cosf.cpp b/external/llvm-project/libc/src/math/amdgpu/cosf.cpp
deleted file mode 100644
index 99ec1185a834..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/cosf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the cosf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/cosf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, cosf, (float x)) { return __ocml_cos_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/cosh.cpp b/external/llvm-project/libc/src/math/amdgpu/cosh.cpp
deleted file mode 100644
index d94d7af25126..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/cosh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the cosh function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/cosh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, cosh, (double x)) { return __ocml_cosh_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/coshf.cpp b/external/llvm-project/libc/src/math/amdgpu/coshf.cpp
deleted file mode 100644
index 5b641be27818..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/coshf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the coshf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/coshf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, coshf, (float x)) { return __ocml_cosh_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/declarations.h b/external/llvm-project/libc/src/math/amdgpu/declarations.h
deleted file mode 100644
index 88e2201521b6..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/declarations.h
+++ /dev/null
@@ -1,91 +0,0 @@
-//===-- AMDGPU specific declarations for math support ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_MATH_AMDGPU_DECLARATIONS_H
-#define LLVM_LIBC_SRC_MATH_AMDGPU_DECLARATIONS_H
-
-#include "platform.h"
-
-#include "src/__support/GPU/utils.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-extern "C" {
-float __ocml_acos_f32(float);
-double __ocml_acos_f64(double);
-float __ocml_acosh_f32(float);
-double __ocml_acosh_f64(double);
-float __ocml_asin_f32(float);
-double __ocml_asin_f64(double);
-float __ocml_asinh_f32(float);
-double __ocml_asinh_f64(double);
-float __ocml_atan_f32(float);
-double __ocml_atan_f64(double);
-float __ocml_atan2_f32(float, float);
-double __ocml_atan2_f64(double, double);
-float __ocml_atanh_f32(float);
-double __ocml_atanh_f64(double);
-float __ocml_cos_f32(float);
-double __ocml_cos_f64(double);
-float __ocml_cosh_f32(float);
-double __ocml_cosh_f64(double);
-float __ocml_erf_f32(float);
-double __ocml_erf_f64(double);
-float __ocml_exp_f32(float);
-double __ocml_exp_f64(double);
-float __ocml_exp2_f32(float);
-double __ocml_exp2_f64(double);
-float __ocml_exp10_f32(float);
-double __ocml_exp10_f64(double);
-double __ocml_exp2_f64(double);
-float __ocml_expm1_f32(float);
-double __ocml_expm1_f64(double);
-float __ocml_fdim_f32(float, float);
-double __ocml_fdim_f64(double, double);
-float __ocml_hypot_f32(float, float);
-double __ocml_hypot_f64(double, double);
-int __ocml_ilogb_f64(double);
-int __ocml_ilogb_f32(float);
-float __ocml_ldexp_f32(float, int);
-double __ocml_ldexp_f64(double, int);
-float __ocml_log10_f32(float);
-double __ocml_log10_f64(double);
-float __ocml_log1p_f32(float);
-double __ocml_log1p_f64(double);
-float __ocml_log2_f32(float);
-double __ocml_log2_f64(double);
-float __ocml_log_f32(float);
-double __ocml_log_f64(double);
-float __ocml_nextafter_f32(float, float);
-double __ocml_nextafter_f64(double, double);
-float __ocml_pow_f32(float, float);
-double __ocml_pow_f64(double, double);
-float __ocml_pown_f32(float, int);
-double __ocml_pown_f64(double, int);
-float __ocml_sin_f32(float);
-double __ocml_sin_f64(double);
-float __ocml_sincos_f32(float, float *);
-double __ocml_sincos_f64(double, double *);
-float __ocml_sinh_f32(float);
-double __ocml_sinh_f64(double);
-float __ocml_tan_f32(float);
-double __ocml_tan_f64(double);
-float __ocml_tanh_f32(float);
-double __ocml_tanh_f64(double);
-float __ocml_remquo_f32(float, float, gpu::Private<int> *);
-double __ocml_remquo_f64(double, double, gpu::Private<int> *);
-double __ocml_tgamma_f64(double);
-float __ocml_tgamma_f32(float);
-double __ocml_lgamma_f64(double);
-double __ocml_lgamma_r_f64(double, gpu::Private<int> *);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC_MATH_AMDGPU_DECLARATIONS_H
diff --git a/external/llvm-project/libc/src/math/amdgpu/erf.cpp b/external/llvm-project/libc/src/math/amdgpu/erf.cpp
deleted file mode 100644
index 07ae268ff2cd..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/erf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU erf function ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/erf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, erf, (double x)) { return __ocml_erf_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/erff.cpp b/external/llvm-project/libc/src/math/amdgpu/erff.cpp
deleted file mode 100644
index a4b7b275cfcc..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/erff.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU erff function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/erff.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, erff, (float x)) { return __ocml_erf_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/exp.cpp b/external/llvm-project/libc/src/math/amdgpu/exp.cpp
deleted file mode 100644
index dae79bec5599..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/exp.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU exp function ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, exp, (double x)) { return __ocml_exp_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/exp10.cpp b/external/llvm-project/libc/src/math/amdgpu/exp10.cpp
deleted file mode 100644
index f13d218ca88c..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/exp10.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU exp10 function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp10.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, exp10, (double x)) { return __ocml_exp10_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/exp10f.cpp b/external/llvm-project/libc/src/math/amdgpu/exp10f.cpp
deleted file mode 100644
index 883e734e0687..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/exp10f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the exp10f function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp10f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, exp10f, (float x)) { return __ocml_exp10_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/exp2.cpp b/external/llvm-project/libc/src/math/amdgpu/exp2.cpp
deleted file mode 100644
index fb336cf48f15..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/exp2.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU exp2 function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp2.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, exp2, (double x)) { return __ocml_exp2_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/exp2f.cpp b/external/llvm-project/libc/src/math/amdgpu/exp2f.cpp
deleted file mode 100644
index 77b4a9c3b86e..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/exp2f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the exp2f function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp2f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, exp2f, (float x)) { return __ocml_exp2_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/expf.cpp b/external/llvm-project/libc/src/math/amdgpu/expf.cpp
deleted file mode 100644
index 6c44aad164bf..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/expf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the expf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/expf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, expf, (float x)) { return __ocml_exp_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/expm1.cpp b/external/llvm-project/libc/src/math/amdgpu/expm1.cpp
deleted file mode 100644
index df3643f9c78b..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/expm1.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU expm1 function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/expm1.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, expm1, (double x)) { return __ocml_expm1_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/expm1f.cpp b/external/llvm-project/libc/src/math/amdgpu/expm1f.cpp
deleted file mode 100644
index 2409997f479c..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/expm1f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the expm1f function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/expm1f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, expm1f, (float x)) { return __ocml_expm1_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/frexp.cpp b/external/llvm-project/libc/src/math/amdgpu/frexp.cpp
index 00e5187c0c71..4ae2b00198ec 100644
--- a/external/llvm-project/libc/src/math/amdgpu/frexp.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/frexp.cpp
@@ -9,7 +9,6 @@
 #include "src/math/frexp.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/frexpf.cpp b/external/llvm-project/libc/src/math/amdgpu/frexpf.cpp
index 2799e542efae..fd53f65701f5 100644
--- a/external/llvm-project/libc/src/math/amdgpu/frexpf.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/frexpf.cpp
@@ -9,7 +9,6 @@
 #include "src/math/frexpf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/hypot.cpp b/external/llvm-project/libc/src/math/amdgpu/hypot.cpp
deleted file mode 100644
index dcf1152a7516..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/hypot.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the hypot function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/hypot.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, hypot, (double x, double y)) {
-  return __ocml_hypot_f64(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/hypotf.cpp b/external/llvm-project/libc/src/math/amdgpu/hypotf.cpp
deleted file mode 100644
index 68ec659d12e5..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/hypotf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the hypotf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/hypotf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) {
-  return __ocml_hypot_f32(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/ilogb.cpp b/external/llvm-project/libc/src/math/amdgpu/ilogb.cpp
deleted file mode 100644
index 37f24dfd8c39..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/ilogb.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the ilogb function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/ilogb.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(int, ilogb, (double x)) { return __ocml_ilogb_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/ilogbf.cpp b/external/llvm-project/libc/src/math/amdgpu/ilogbf.cpp
deleted file mode 100644
index 56e74e1575f1..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/ilogbf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the ilogbf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/ilogbf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(int, ilogbf, (float x)) { return __ocml_ilogb_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/ldexp.cpp b/external/llvm-project/libc/src/math/amdgpu/ldexp.cpp
index 393eabb84e61..5b786a9b21e7 100644
--- a/external/llvm-project/libc/src/math/amdgpu/ldexp.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/ldexp.cpp
@@ -9,7 +9,6 @@
 #include "src/math/ldexp.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/ldexpf.cpp b/external/llvm-project/libc/src/math/amdgpu/ldexpf.cpp
index 970603dd170e..d3aa77fe592c 100644
--- a/external/llvm-project/libc/src/math/amdgpu/ldexpf.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/ldexpf.cpp
@@ -9,7 +9,6 @@
 #include "src/math/ldexpf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/lgamma.cpp b/external/llvm-project/libc/src/math/amdgpu/lgamma.cpp
index acff4c76734e..9666927e60c4 100644
--- a/external/llvm-project/libc/src/math/amdgpu/lgamma.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/lgamma.cpp
@@ -9,11 +9,11 @@
 #include "src/math/lgamma.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, lgamma, (double x)) { return __ocml_lgamma_f64(x); }
+// TODO: Implement this.
+LLVM_LIBC_FUNCTION(double, lgamma, (double)) { return 0.0; }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/lgamma_r.cpp b/external/llvm-project/libc/src/math/amdgpu/lgamma_r.cpp
index 0a79988d66e3..8b69ca349456 100644
--- a/external/llvm-project/libc/src/math/amdgpu/lgamma_r.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/lgamma_r.cpp
@@ -9,16 +9,13 @@
 #include "src/math/lgamma_r.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, lgamma_r, (double x, int *signp)) {
-  int tmp = *signp;
-  double r = __ocml_lgamma_r_f64(x, (gpu::Private<int> *)&tmp);
-  *signp = tmp;
-  return r;
+LLVM_LIBC_FUNCTION(double, lgamma_r, (double, int *signp)) {
+  *signp = 0;
+  return 0;
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/llrint.cpp b/external/llvm-project/libc/src/math/amdgpu/llrint.cpp
index 21129fe80a0c..6e0f57a5a038 100644
--- a/external/llvm-project/libc/src/math/amdgpu/llrint.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/llrint.cpp
@@ -9,7 +9,6 @@
 #include "src/math/llrint.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/llrintf.cpp b/external/llvm-project/libc/src/math/amdgpu/llrintf.cpp
index a6f9f4387601..d8de23fac3c8 100644
--- a/external/llvm-project/libc/src/math/amdgpu/llrintf.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/llrintf.cpp
@@ -9,7 +9,6 @@
 #include "src/math/llrintf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/log.cpp b/external/llvm-project/libc/src/math/amdgpu/log.cpp
deleted file mode 100644
index bd01adfa08a4..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/log.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log function ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log, (double x)) { return __ocml_log_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/log10.cpp b/external/llvm-project/libc/src/math/amdgpu/log10.cpp
deleted file mode 100644
index 75957c9aeb50..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/log10.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log10 function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log10.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log10, (double x)) { return __ocml_log10_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/log10f.cpp b/external/llvm-project/libc/src/math/amdgpu/log10f.cpp
deleted file mode 100644
index 9c12d6b11bcf..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/log10f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log10f function -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log10f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, log10f, (float x)) { return __ocml_log10_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/log1p.cpp b/external/llvm-project/libc/src/math/amdgpu/log1p.cpp
deleted file mode 100644
index fc275196830f..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/log1p.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log1p function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log1p.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log1p, (double x)) { return __ocml_log1p_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/log1pf.cpp b/external/llvm-project/libc/src/math/amdgpu/log1pf.cpp
deleted file mode 100644
index b2d26fb1166f..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/log1pf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log1pf function -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log1pf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { return __ocml_log1p_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/log2.cpp b/external/llvm-project/libc/src/math/amdgpu/log2.cpp
deleted file mode 100644
index 73f34b65a32f..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/log2.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log2 function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log2.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log2, (double x)) { return __ocml_log2_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/log2f.cpp b/external/llvm-project/libc/src/math/amdgpu/log2f.cpp
deleted file mode 100644
index 3b62edacfa65..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/log2f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log2f function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log2f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, log2f, (float x)) { return __ocml_log2_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/logb.cpp b/external/llvm-project/libc/src/math/amdgpu/logb.cpp
deleted file mode 100644
index 4b68e281be87..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/logb.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU logb function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/logb.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, logb, (double x)) { return __ocml_logb_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/logbf.cpp b/external/llvm-project/libc/src/math/amdgpu/logbf.cpp
deleted file mode 100644
index bc7c462c7f92..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/logbf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU logbf function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/logbf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, logbf, (float x)) { return __ocml_logb_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/logf.cpp b/external/llvm-project/libc/src/math/amdgpu/logf.cpp
deleted file mode 100644
index 17925672ef32..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/logf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU logf function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/logf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, logf, (float x)) { return __ocml_log_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/lrint.cpp b/external/llvm-project/libc/src/math/amdgpu/lrint.cpp
index 715b55257578..5ba70ec890b2 100644
--- a/external/llvm-project/libc/src/math/amdgpu/lrint.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/lrint.cpp
@@ -9,7 +9,6 @@
 #include "src/math/lrint.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/lrintf.cpp b/external/llvm-project/libc/src/math/amdgpu/lrintf.cpp
index 387063803f27..1c985b01c2dc 100644
--- a/external/llvm-project/libc/src/math/amdgpu/lrintf.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/lrintf.cpp
@@ -9,7 +9,6 @@
 #include "src/math/lrintf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/nextafter.cpp b/external/llvm-project/libc/src/math/amdgpu/nextafter.cpp
deleted file mode 100644
index 226b8a5a1254..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/nextafter.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the nextafter function for GPU ------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/nextafter.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, nextafter, (double x, double y)) {
-  return __ocml_nextafter_f64(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/nextafterf.cpp b/external/llvm-project/libc/src/math/amdgpu/nextafterf.cpp
deleted file mode 100644
index 7bed2c17b6e1..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/nextafterf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the nextafterf function for GPU -----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/nextafterf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, nextafterf, (float x, float y)) {
-  return __ocml_nextafter_f32(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/platform.h b/external/llvm-project/libc/src/math/amdgpu/platform.h
deleted file mode 100644
index 472a9830e8bb..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/platform.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- AMDGPU specific platform definitions for math support -------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_MATH_AMDGPU_PLATFORM_H
-#define LLVM_LIBC_SRC_MATH_AMDGPU_PLATFORM_H
-
-#include "src/__support/macros/attributes.h"
-#include "src/__support/macros/config.h"
-
-#include <stdint.h>
-
-namespace LIBC_NAMESPACE_DECL {
-
-// The ROCm device library uses control globals to alter codegen for the
-// different targets. To avoid needing to link them in manually we simply
-// define them here.
-extern "C" {
-
-// Disable unsafe math optimizations in the implementation.
-extern const LIBC_INLINE_VAR uint8_t __oclc_unsafe_math_opt = 0;
-
-// Disable denormalization at zero optimizations in the implementation.
-extern const LIBC_INLINE_VAR uint8_t __oclc_daz_opt = 0;
-
-// Disable rounding optimizations for 32-bit square roots.
-extern const LIBC_INLINE_VAR uint8_t __oclc_correctly_rounded_sqrt32 = 1;
-
-// Disable finite math optimizations.
-extern const LIBC_INLINE_VAR uint8_t __oclc_finite_only_opt = 0;
-
-// Set the ISA value to a high enough value that the ROCm device library math
-// functions will assume we have fast FMA operations among other features. This
-// is determined to be safe on all targets by looking at the source code.
-// https://github.com/ROCm/ROCm-Device-Libs/blob/amd-stg-open/ocml/src/opts.h
-extern const LIBC_INLINE_VAR uint32_t __oclc_ISA_version = 9000;
-}
-
-// These aliases cause clang to emit the control constants with ODR linkage.
-// This allows us to link against the symbols without preventing them from being
-// optimized out or causing symbol collisions.
-[[gnu::alias("__oclc_unsafe_math_opt")]] const uint8_t __oclc_unsafe_math_opt__;
-[[gnu::alias("__oclc_daz_opt")]] const uint8_t __oclc_daz_opt__;
-[[gnu::alias("__oclc_correctly_rounded_sqrt32")]] const uint8_t
-    __oclc_correctly_rounded_sqrt32__;
-[[gnu::alias("__oclc_finite_only_opt")]] const uint8_t __oclc_finite_only_opt__;
-[[gnu::alias("__oclc_ISA_version")]] const uint32_t __oclc_ISA_version__;
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC_MATH_AMDGPU_PLATFORM_H
diff --git a/external/llvm-project/libc/src/math/amdgpu/powf.cpp b/external/llvm-project/libc/src/math/amdgpu/powf.cpp
deleted file mode 100644
index 6931934d4c11..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/powf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the powf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/powf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
-  return __ocml_pow_f32(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/powi.cpp b/external/llvm-project/libc/src/math/amdgpu/powi.cpp
deleted file mode 100644
index 6b31b471e4b2..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/powi.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the powi function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/powi.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-
-#include "declarations.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, powi, (double x, int y)) {
-  return __ocml_pown_f64(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/powif.cpp b/external/llvm-project/libc/src/math/amdgpu/powif.cpp
deleted file mode 100644
index 94f8a91d9c03..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/powif.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the powi function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/powif.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-
-#include "declarations.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, powif, (float x, int y)) {
-  return __ocml_pown_f32(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/remquof.cpp b/external/llvm-project/libc/src/math/amdgpu/remquof.cpp
deleted file mode 100644
index 854d3bfe842f..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/remquof.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- Implementation of the GPU remquof function ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/remquof.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, remquof, (float x, float y, int *quo)) {
-  int tmp;
-  float r = __ocml_remquo_f32(x, y, (gpu::Private<int> *)&tmp);
-  *quo = tmp;
-  return r;
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/scalbn.cpp b/external/llvm-project/libc/src/math/amdgpu/scalbn.cpp
index 05bbbc69b47a..3c3c19ea0d9d 100644
--- a/external/llvm-project/libc/src/math/amdgpu/scalbn.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/scalbn.cpp
@@ -9,7 +9,6 @@
 #include "src/math/scalbn.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/scalbnf.cpp b/external/llvm-project/libc/src/math/amdgpu/scalbnf.cpp
index f0e9e475674c..c348aa7c3be2 100644
--- a/external/llvm-project/libc/src/math/amdgpu/scalbnf.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/scalbnf.cpp
@@ -9,7 +9,6 @@
 #include "src/math/scalbnf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/amdgpu/sin.cpp b/external/llvm-project/libc/src/math/amdgpu/sin.cpp
deleted file mode 100644
index f3d88af009ce..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/sin.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sin function for GPU ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sin.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, sin, (double x)) { return __ocml_sin_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/sincos.cpp b/external/llvm-project/libc/src/math/amdgpu/sincos.cpp
deleted file mode 100644
index 304ac0c7c633..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/sincos.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the sincos function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sincos.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sinptr, double *cosptr)) {
-  *sinptr = __ocml_sincos_f64(x, cosptr);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/sincosf.cpp b/external/llvm-project/libc/src/math/amdgpu/sincosf.cpp
deleted file mode 100644
index 1c4e9c6a2463..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/sincosf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the sincosf function for GPU --------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sincosf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinptr, float *cosptr)) {
-  *sinptr = __ocml_sincos_f32(x, cosptr);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/sinf.cpp b/external/llvm-project/libc/src/math/amdgpu/sinf.cpp
deleted file mode 100644
index c6d64a63c755..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/sinf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sinf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sinf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, sinf, (float x)) { return __ocml_sin_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/sinh.cpp b/external/llvm-project/libc/src/math/amdgpu/sinh.cpp
deleted file mode 100644
index 26314f46d0e9..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/sinh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sinh function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sinh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, sinh, (double x)) { return __ocml_sinh_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/sinhf.cpp b/external/llvm-project/libc/src/math/amdgpu/sinhf.cpp
deleted file mode 100644
index a4eb8e1a6528..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/sinhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sinhf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sinhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, sinhf, (float x)) { return __ocml_sinh_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/tan.cpp b/external/llvm-project/libc/src/math/amdgpu/tan.cpp
deleted file mode 100644
index c946dc2a3db6..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/tan.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tan function for GPU ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tan.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, tan, (double x)) { return __ocml_tan_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/tanf.cpp b/external/llvm-project/libc/src/math/amdgpu/tanf.cpp
deleted file mode 100644
index 8c93fc4f6236..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/tanf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tanf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tanf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, tanf, (float x)) { return __ocml_tan_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/tanh.cpp b/external/llvm-project/libc/src/math/amdgpu/tanh.cpp
deleted file mode 100644
index 834353e122ab..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/tanh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tanh function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tanh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, tanh, (double x)) { return __ocml_tanh_f64(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/tanhf.cpp b/external/llvm-project/libc/src/math/amdgpu/tanhf.cpp
deleted file mode 100644
index 5029596a8e6f..000000000000
--- a/external/llvm-project/libc/src/math/amdgpu/tanhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tanhf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tanhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, tanhf, (float x)) { return __ocml_tanh_f32(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/tgamma.cpp b/external/llvm-project/libc/src/math/amdgpu/tgamma.cpp
index 485a6a3def43..0dfb0fd9ee13 100644
--- a/external/llvm-project/libc/src/math/amdgpu/tgamma.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/tgamma.cpp
@@ -9,11 +9,10 @@
 #include "src/math/tgamma.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, tgamma, (double x)) { return __ocml_tgamma_f64(x); }
+LLVM_LIBC_FUNCTION(double, tgamma, (double)) { return 0.0; }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/tgammaf.cpp b/external/llvm-project/libc/src/math/amdgpu/tgammaf.cpp
index e48a486c9c5e..9dd17ab8de91 100644
--- a/external/llvm-project/libc/src/math/amdgpu/tgammaf.cpp
+++ b/external/llvm-project/libc/src/math/amdgpu/tgammaf.cpp
@@ -9,11 +9,10 @@
 #include "src/math/tgammaf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, tgammaf, (float x)) { return __ocml_tgamma_f32(x); }
+LLVM_LIBC_FUNCTION(float, tgammaf, (float)) { return 0.0f; }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/generic/CMakeLists.txt b/external/llvm-project/libc/src/math/generic/CMakeLists.txt
index 27fe481cc64c..fd1e6c0d648a 100644
--- a/external/llvm-project/libc/src/math/generic/CMakeLists.txt
+++ b/external/llvm-project/libc/src/math/generic/CMakeLists.txt
@@ -1275,14 +1275,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
-add_object_library(
-  exp_utils
-  HDRS
-    exp_utils.h
-  SRCS
-    exp_utils.cpp
-)
-
 add_entrypoint_object(
   erff
   SRCS
@@ -1329,15 +1321,7 @@ add_entrypoint_object(
   HDRS
     ../expf.h
   DEPENDS
-    .common_constants
-    libc.src.__support.FPUtil.basic_operations
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.nearest_integer
-    libc.src.__support.FPUtil.polyeval
-    libc.src.__support.FPUtil.rounding_mode
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.expf
     libc.src.errno.errno
 )
 
@@ -3978,7 +3962,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-    libc.src.__support.macros.properties.types 
+    libc.src.__support.macros.properties.types
 )
 
 add_entrypoint_object(
@@ -4046,7 +4030,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
-    libc.src.__support.macros.properties.types 
+    libc.src.__support.macros.properties.types
 )
 
 add_object_library(
@@ -4093,7 +4077,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-    libc.src.__support.macros.properties.types  
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
@@ -4162,7 +4146,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-    libc.src.__support.macros.properties.types  
+    libc.src.__support.macros.properties.types
 )
 
 add_entrypoint_object(
@@ -4181,7 +4165,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-    libc.src.__support.macros.properties.types  
+    libc.src.__support.macros.properties.types
     libc.src.__support.macros.properties.cpu_features
 )
 
@@ -4201,7 +4185,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.macros.optimization
-    libc.src.__support.macros.properties.types  
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
diff --git a/external/llvm-project/libc/src/math/generic/exp10m1f.cpp b/external/llvm-project/libc/src/math/generic/exp10m1f.cpp
index e973b2921c2e..27729104e038 100644
--- a/external/llvm-project/libc/src/math/generic/exp10m1f.cpp
+++ b/external/llvm-project/libc/src/math/generic/exp10m1f.cpp
@@ -14,9 +14,9 @@
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/FPUtil/rounding_mode.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
-#include "src/errno/libc_errno.h"
 
 #include "explogxf.h"
 
diff --git a/external/llvm-project/libc/src/math/generic/exp2m1f.cpp b/external/llvm-project/libc/src/math/generic/exp2m1f.cpp
index 4913a5e4277e..127c6eaa494d 100644
--- a/external/llvm-project/libc/src/math/generic/exp2m1f.cpp
+++ b/external/llvm-project/libc/src/math/generic/exp2m1f.cpp
@@ -14,10 +14,10 @@
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/FPUtil/rounding_mode.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/cpu_features.h"
-#include "src/errno/libc_errno.h"
 
 #include "explogxf.h"
 
diff --git a/external/llvm-project/libc/src/math/generic/exp_utils.cpp b/external/llvm-project/libc/src/math/generic/exp_utils.cpp
deleted file mode 100644
index cc21637e4c7c..000000000000
--- a/external/llvm-project/libc/src/math/generic/exp_utils.cpp
+++ /dev/null
@@ -1,128 +0,0 @@
-//===-- Implemention of exp and friends' utils ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "exp_utils.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-const Exp2fDataTable exp2f_data = {
-    // :tab[i] = uint(2^(i/N)) - (i << 52-BITS)
-    //    used for computing 2^(k/N) for an int |k| < 150 N as
-    //    double(tab[k%N] + (k << 52-BITS))
-    {
-// tab
-#if N == 8
-        0x3ff0000000000000,
-        0x3fef72b83c7d517b,
-        0x3fef06fe0a31b715,
-        0x3feebfdad5362a27,
-        0x3feea09e667f3bcd,
-        0x3feeace5422aa0db,
-        0x3feee89f995ad3ad,
-        0x3fef5818dcfba487,
-#elif N == 16
-        0x3ff0000000000000,
-        0x3fefb5586cf9890f,
-        0x3fef72b83c7d517b,
-        0x3fef387a6e756238,
-        0x3fef06fe0a31b715,
-        0x3feedea64c123422,
-        0x3feebfdad5362a27,
-        0x3feeab07dd485429,
-        0x3feea09e667f3bcd,
-        0x3feea11473eb0187,
-        0x3feeace5422aa0db,
-        0x3feec49182a3f090,
-        0x3feee89f995ad3ad,
-        0x3fef199bdd85529c,
-        0x3fef5818dcfba487,
-        0x3fefa4afa2a490da,
-#elif N == 32
-        0x3ff0000000000000, 0x3fefd9b0d3158574, 0x3fefb5586cf9890f,
-        0x3fef9301d0125b51, 0x3fef72b83c7d517b, 0x3fef54873168b9aa,
-        0x3fef387a6e756238, 0x3fef1e9df51fdee1, 0x3fef06fe0a31b715,
-        0x3feef1a7373aa9cb, 0x3feedea64c123422, 0x3feece086061892d,
-        0x3feebfdad5362a27, 0x3feeb42b569d4f82, 0x3feeab07dd485429,
-        0x3feea47eb03a5585, 0x3feea09e667f3bcd, 0x3fee9f75e8ec5f74,
-        0x3feea11473eb0187, 0x3feea589994cce13, 0x3feeace5422aa0db,
-        0x3feeb737b0cdc5e5, 0x3feec49182a3f090, 0x3feed503b23e255d,
-        0x3feee89f995ad3ad, 0x3feeff76f2fb5e47, 0x3fef199bdd85529c,
-        0x3fef3720dcef9069, 0x3fef5818dcfba487, 0x3fef7c97337b9b5f,
-        0x3fefa4afa2a490da, 0x3fefd0765b6e4540,
-#elif N == 64
-        0x3ff0000000000000, 0x3fefec9a3e778061, 0x3fefd9b0d3158574,
-        0x3fefc74518759bc8, 0x3fefb5586cf9890f, 0x3fefa3ec32d3d1a2,
-        0x3fef9301d0125b51, 0x3fef829aaea92de0, 0x3fef72b83c7d517b,
-        0x3fef635beb6fcb75, 0x3fef54873168b9aa, 0x3fef463b88628cd6,
-        0x3fef387a6e756238, 0x3fef2b4565e27cdd, 0x3fef1e9df51fdee1,
-        0x3fef1285a6e4030b, 0x3fef06fe0a31b715, 0x3feefc08b26416ff,
-        0x3feef1a7373aa9cb, 0x3feee7db34e59ff7, 0x3feedea64c123422,
-        0x3feed60a21f72e2a, 0x3feece086061892d, 0x3feec6a2b5c13cd0,
-        0x3feebfdad5362a27, 0x3feeb9b2769d2ca7, 0x3feeb42b569d4f82,
-        0x3feeaf4736b527da, 0x3feeab07dd485429, 0x3feea76f15ad2148,
-        0x3feea47eb03a5585, 0x3feea23882552225, 0x3feea09e667f3bcd,
-        0x3fee9fb23c651a2f, 0x3fee9f75e8ec5f74, 0x3fee9feb564267c9,
-        0x3feea11473eb0187, 0x3feea2f336cf4e62, 0x3feea589994cce13,
-        0x3feea8d99b4492ed, 0x3feeace5422aa0db, 0x3feeb1ae99157736,
-        0x3feeb737b0cdc5e5, 0x3feebd829fde4e50, 0x3feec49182a3f090,
-        0x3feecc667b5de565, 0x3feed503b23e255d, 0x3feede6b5579fdbf,
-        0x3feee89f995ad3ad, 0x3feef3a2b84f15fb, 0x3feeff76f2fb5e47,
-        0x3fef0c1e904bc1d2, 0x3fef199bdd85529c, 0x3fef27f12e57d14b,
-        0x3fef3720dcef9069, 0x3fef472d4a07897c, 0x3fef5818dcfba487,
-        0x3fef69e603db3285, 0x3fef7c97337b9b5f, 0x3fef902ee78b3ff6,
-        0x3fefa4afa2a490da, 0x3fefba1bee615a27, 0x3fefd0765b6e4540,
-        0x3fefe7c1819e90d8,
-#endif
-    },
-    as_double(0x4338000000000000) / N, // shift_scaled
-    {
-// poly
-#if N == 8
-        as_double(0x3fac6a00335106e2),
-        as_double(0x3fcec0c313449f55),
-        as_double(0x3fe62e431111f69f),
-#elif N == 16
-        as_double(0x3fac6ac6aa313963),
-        as_double(0x3fcebfff4532d9ba),
-        as_double(0x3fe62e43001bc49f),
-#elif N == 32
-        as_double(0x3fac6af84b912394),
-        as_double(0x3fcebfce50fac4f3),
-        as_double(0x3fe62e42ff0c52d6),
-#elif N == 64
-        as_double(0x3fac6b04b4221b2a),
-        as_double(0x3fcebfc213e184d7),
-        as_double(0x3fe62e42fefb5b7f),
-#endif
-    },
-    as_double(0x4338000000000000),     // shift
-    as_double(0x3ff71547652b82fe) * N, // invln2_scaled
-    {
-// poly_scaled
-#if N == 8
-        as_double(0x3fac6a00335106e2) / N / N / N,
-        as_double(0x3fcec0c313449f55) / N / N,
-        as_double(0x3fe62e431111f69f) / N,
-#elif N == 16
-        as_double(0x3fac6ac6aa313963) / N / N / N,
-        as_double(0x3fcebfff4532d9ba) / N / N,
-        as_double(0x3fe62e43001bc49f) / N,
-#elif N == 32
-        as_double(0x3fac6af84b912394) / N / N / N,
-        as_double(0x3fcebfce50fac4f3) / N / N,
-        as_double(0x3fe62e42ff0c52d6) / N,
-#elif N == 64
-        as_double(0x3fac6b04b4221b2a) / N / N / N,
-        as_double(0x3fcebfc213e184d7) / N / N,
-        as_double(0x3fe62e42fefb5b7f) / N,
-#endif
-    },
-};
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/generic/exp_utils.h b/external/llvm-project/libc/src/math/generic/exp_utils.h
deleted file mode 100644
index dca9eb7c2d71..000000000000
--- a/external/llvm-project/libc/src/math/generic/exp_utils.h
+++ /dev/null
@@ -1,34 +0,0 @@
-//===-- Collection of utils for exp and friends -----------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_EXP_UTILS_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_EXP_UTILS_H
-
-#include "src/__support/macros/config.h"
-#include <stdint.h>
-
-#define EXP2F_TABLE_BITS 5
-#define EXP2F_POLY_ORDER 3
-#define N (1 << EXP2F_TABLE_BITS)
-
-namespace LIBC_NAMESPACE_DECL {
-
-struct Exp2fDataTable {
-  uint64_t tab[1 << EXP2F_TABLE_BITS];
-  double shift_scaled;
-  double poly[EXP2F_POLY_ORDER];
-  double shift;
-  double invln2_scaled;
-  double poly_scaled[EXP2F_POLY_ORDER];
-};
-
-extern const Exp2fDataTable exp2f_data;
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_EXP_UTILS_H
diff --git a/external/llvm-project/libc/src/math/generic/expf.cpp b/external/llvm-project/libc/src/math/generic/expf.cpp
index fa507d4d9322..de11f51ac64a 100644
--- a/external/llvm-project/libc/src/math/generic/expf.cpp
+++ b/external/llvm-project/libc/src/math/generic/expf.cpp
@@ -7,103 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/expf.h"
-#include "common_constants.h" // Lookup tables EXP_M1 and EXP_M2.
-#include "src/__support/FPUtil/BasicOperations.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/PolyEval.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/FPUtil/nearest_integer.h"
-#include "src/__support/FPUtil/rounding_mode.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/__support/math/expf.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, expf, (float x)) {
-  using FPBits = typename fputil::FPBits<float>;
-  FPBits xbits(x);
-
-  uint32_t x_u = xbits.uintval();
-  uint32_t x_abs = x_u & 0x7fff'ffffU;
-
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-  // Exceptional values
-  if (LIBC_UNLIKELY(x_u == 0xc236'bd8cU)) { // x = -0x1.6d7b18p+5f
-    return 0x1.108a58p-66f - x * 0x1.0p-95f;
-  }
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-
-  // When |x| >= 89, |x| < 2^-25, or x is nan
-  if (LIBC_UNLIKELY(x_abs >= 0x42b2'0000U || x_abs <= 0x3280'0000U)) {
-    // |x| < 2^-25
-    if (xbits.get_biased_exponent() <= 101) {
-      return 1.0f + x;
-    }
-
-    // When x < log(2^-150) or nan
-    if (xbits.uintval() >= 0xc2cf'f1b5U) {
-      // exp(-Inf) = 0
-      if (xbits.is_inf())
-        return 0.0f;
-      // exp(nan) = nan
-      if (xbits.is_nan())
-        return x;
-      if (fputil::fenv_is_round_up())
-        return FPBits::min_subnormal().get_val();
-      fputil::set_errno_if_required(ERANGE);
-      fputil::raise_except_if_required(FE_UNDERFLOW);
-      return 0.0f;
-    }
-    // x >= 89 or nan
-    if (xbits.is_pos() && (xbits.uintval() >= 0x42b2'0000)) {
-      // x is finite
-      if (xbits.uintval() < 0x7f80'0000U) {
-        int rounding = fputil::quick_get_round();
-        if (rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO)
-          return FPBits::max_normal().get_val();
-
-        fputil::set_errno_if_required(ERANGE);
-        fputil::raise_except_if_required(FE_OVERFLOW);
-      }
-      // x is +inf or nan
-      return x + FPBits::inf().get_val();
-    }
-  }
-  // For -104 < x < 89, to compute exp(x), we perform the following range
-  // reduction: find hi, mid, lo such that:
-  //   x = hi + mid + lo, in which
-  //     hi is an integer,
-  //     mid * 2^7 is an integer
-  //     -2^(-8) <= lo < 2^-8.
-  // In particular,
-  //   hi + mid = round(x * 2^7) * 2^(-7).
-  // Then,
-  //   exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo).
-  // We store exp(hi) and exp(mid) in the lookup tables EXP_M1 and EXP_M2
-  // respectively.  exp(lo) is computed using a degree-4 minimax polynomial
-  // generated by Sollya.
-
-  // x_hi = (hi + mid) * 2^7 = round(x * 2^7).
-  float kf = fputil::nearest_integer(x * 0x1.0p7f);
-  // Subtract (hi + mid) from x to get lo.
-  double xd = static_cast<double>(fputil::multiply_add(kf, -0x1.0p-7f, x));
-  int x_hi = static_cast<int>(kf);
-  x_hi += 104 << 7;
-  // hi = x_hi >> 7
-  double exp_hi = EXP_M1[x_hi >> 7];
-  // mid * 2^7 = x_hi & 0x0000'007fU;
-  double exp_mid = EXP_M2[x_hi & 0x7f];
-  // Degree-4 minimax polynomial generated by Sollya with the following
-  // commands:
-  //   > display = hexadecimal;
-  //   > Q = fpminimax(expm1(x)/x, 3, [|D...|], [-2^-8, 2^-8]);
-  //   > Q;
-  double exp_lo =
-      fputil::polyeval(xd, 0x1p0, 0x1.ffffffffff777p-1, 0x1.000000000071cp-1,
-                       0x1.555566668e5e7p-3, 0x1.55555555ef243p-5);
-  return static_cast<float>(exp_hi * exp_mid * exp_lo);
-}
+LLVM_LIBC_FUNCTION(float, expf, (float x)) { return math::expf(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/generic/nan.cpp b/external/llvm-project/libc/src/math/generic/nan.cpp
index f92cd3ff5eb5..829a2ea435ac 100644
--- a/external/llvm-project/libc/src/math/generic/nan.cpp
+++ b/external/llvm-project/libc/src/math/generic/nan.cpp
@@ -8,9 +8,9 @@
 
 #include "src/math/nan.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/math/generic/nanf.cpp b/external/llvm-project/libc/src/math/generic/nanf.cpp
index 7287182406ac..1cb66160e736 100644
--- a/external/llvm-project/libc/src/math/generic/nanf.cpp
+++ b/external/llvm-project/libc/src/math/generic/nanf.cpp
@@ -8,9 +8,9 @@
 
 #include "src/math/nanf.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/math/generic/nanf128.cpp b/external/llvm-project/libc/src/math/generic/nanf128.cpp
index 3d8581afa037..4155c5333a9c 100644
--- a/external/llvm-project/libc/src/math/generic/nanf128.cpp
+++ b/external/llvm-project/libc/src/math/generic/nanf128.cpp
@@ -8,9 +8,9 @@
 
 #include "src/math/nanf128.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/math/generic/nanf16.cpp b/external/llvm-project/libc/src/math/generic/nanf16.cpp
index 27d9d165f4a8..7b166400601b 100644
--- a/external/llvm-project/libc/src/math/generic/nanf16.cpp
+++ b/external/llvm-project/libc/src/math/generic/nanf16.cpp
@@ -8,9 +8,9 @@
 
 #include "src/math/nanf16.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/math/generic/nanl.cpp b/external/llvm-project/libc/src/math/generic/nanl.cpp
index 4f698cb3c88d..58d638c4b531 100644
--- a/external/llvm-project/libc/src/math/generic/nanl.cpp
+++ b/external/llvm-project/libc/src/math/generic/nanl.cpp
@@ -8,9 +8,9 @@
 
 #include "src/math/nanl.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/math/nvptx/CMakeLists.txt b/external/llvm-project/libc/src/math/nvptx/CMakeLists.txt
index e85e17b6701f..fcb2870b4bb1 100644
--- a/external/llvm-project/libc/src/math/nvptx/CMakeLists.txt
+++ b/external/llvm-project/libc/src/math/nvptx/CMakeLists.txt
@@ -1,22 +1,3 @@
-# Math functions not yet available in the libc project, or those not yet tuned
-# for GPU workloads are provided as wrappers over vendor libraries. If we find
-# them ahead of time we will import them statically. Otherwise, we will keep
-# them as external references and expect them to be resolved by the user when
-# they compile. In the future,we will use implementations from the 'libc'
-# project and not provide these wrappers.
-if(CUDAToolkit_FOUND)
-  set(libdevice_path ${CUDAToolkit_BIN_DIR}/../nvvm/libdevice/libdevice.10.bc)
-  if (EXISTS ${libdevice_path})
-    message(STATUS "Found the CUDA device library. Implementations falling back "
-                   "to the vendor libraries will be resolved statically.")
-    set(bitcode_link_flags
-        "SHELL:-Xclang -mlink-builtin-bitcode -Xclang ${libdevice_path}")
-  endif()
-else()
-  message(STATUS "Could not find the CUDA device library. Unimplemented "
-                 "functions will be an external reference to the vendor libraries.")
-endif()
-
 add_entrypoint_object(
   ceil
   SRCS
@@ -287,164 +268,6 @@ add_entrypoint_object(
     -O2
 )
 
-# The following functions currently are not implemented natively and borrow from
-# existing implementations. This will be removed in the future.
-add_entrypoint_object(
-  acos
-  SRCS
-    acos.cpp
-  HDRS
-    ../acos.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  acosh
-  SRCS
-    acosh.cpp
-  HDRS
-    ../acosh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  asin
-  SRCS
-    asin.cpp
-  HDRS
-    ../asin.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  asinh
-  SRCS
-    asinh.cpp
-  HDRS
-    ../asinh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  atan
-  SRCS
-    atan.cpp
-  HDRS
-    ../atan.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  atanh
-  SRCS
-    atanh.cpp
-  HDRS
-    ../atanh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  cos
-  SRCS
-    cos.cpp
-  HDRS
-    ../cos.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  cosh
-  SRCS
-    cosh.cpp
-  HDRS
-    ../cosh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  erf
-  SRCS
-    erf.cpp
-  HDRS
-    ../erf.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  powi
-  SRCS
-    powi.cpp
-  HDRS
-    ../powi.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  powif
-  SRCS
-    powif.cpp
-  HDRS
-    ../powif.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  sinh
-  SRCS
-    sinh.cpp
-  HDRS
-    ../sinh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
-add_entrypoint_object(
-  tanh
-  SRCS
-    tanh.cpp
-  HDRS
-    ../tanh.h
-  COMPILE_OPTIONS
-    ${bitcode_link_flags}
-    -O2
-  VENDOR
-)
-
 add_entrypoint_object(
   tgamma
   SRCS
@@ -454,7 +277,6 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
 
 add_entrypoint_object(
@@ -466,7 +288,6 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
 
 add_entrypoint_object(
@@ -478,7 +299,6 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
 
 add_entrypoint_object(
@@ -490,5 +310,4 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
-  VENDOR
 )
diff --git a/external/llvm-project/libc/src/math/nvptx/acos.cpp b/external/llvm-project/libc/src/math/nvptx/acos.cpp
deleted file mode 100644
index 7049f9fb31c5..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/acos.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU acos function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/acos.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, acos, (double x)) { return __nv_acos(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/acosf.cpp b/external/llvm-project/libc/src/math/nvptx/acosf.cpp
deleted file mode 100644
index cf70a0f3b923..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/acosf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the acosf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/acosf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, acosf, (float x)) { return __nv_acosf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/acosh.cpp b/external/llvm-project/libc/src/math/nvptx/acosh.cpp
deleted file mode 100644
index 2628aa997d0b..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/acosh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU acosh function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/acosh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, acosh, (double x)) { return __nv_acosh(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/acoshf.cpp b/external/llvm-project/libc/src/math/nvptx/acoshf.cpp
deleted file mode 100644
index b8f57fdc05c2..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/acoshf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the acoshf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/acoshf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, acoshf, (float x)) { return __nv_acoshf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/asin.cpp b/external/llvm-project/libc/src/math/nvptx/asin.cpp
deleted file mode 100644
index 417b67ac7da0..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/asin.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU asin function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asin.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, asin, (double x)) { return __nv_asin(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/asinf.cpp b/external/llvm-project/libc/src/math/nvptx/asinf.cpp
deleted file mode 100644
index ea819bdfd00e..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/asinf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the asinf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asinf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, asinf, (float x)) { return __nv_asinf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/asinh.cpp b/external/llvm-project/libc/src/math/nvptx/asinh.cpp
deleted file mode 100644
index 49dcd227bb4b..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/asinh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU asinh function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asinh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, asinh, (double x)) { return __nv_asinh(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/asinhf.cpp b/external/llvm-project/libc/src/math/nvptx/asinhf.cpp
deleted file mode 100644
index af8afe7590ed..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/asinhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the asinhf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/asinhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, asinhf, (float x)) { return __nv_asinhf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/atan.cpp b/external/llvm-project/libc/src/math/nvptx/atan.cpp
deleted file mode 100644
index e94ef5787ca4..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/atan.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU atan function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atan.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, atan, (double x)) { return __nv_atan(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/atan2.cpp b/external/llvm-project/libc/src/math/nvptx/atan2.cpp
deleted file mode 100644
index d12c4cb205d5..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/atan2.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the GPU atan2 function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atan2.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, atan2, (double x, double y)) {
-  return __nv_atan2(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/atan2f.cpp b/external/llvm-project/libc/src/math/nvptx/atan2f.cpp
deleted file mode 100644
index f39f3227b1d5..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/atan2f.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the GPU atan2f function -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atan2f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, atan2f, (float x, float y)) {
-  return __nv_atan2f(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/atanf.cpp b/external/llvm-project/libc/src/math/nvptx/atanf.cpp
deleted file mode 100644
index f66ade7a438f..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/atanf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the atanf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atanf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, atanf, (float x)) { return __nv_atanf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/atanh.cpp b/external/llvm-project/libc/src/math/nvptx/atanh.cpp
deleted file mode 100644
index a206d517af11..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/atanh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU atanh function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atanh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, atanh, (double x)) { return __nv_atanh(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/atanhf.cpp b/external/llvm-project/libc/src/math/nvptx/atanhf.cpp
deleted file mode 100644
index 06f68833b3d2..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/atanhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the atanhf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/atanhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, atanhf, (float x)) { return __nv_atanhf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/cos.cpp b/external/llvm-project/libc/src/math/nvptx/cos.cpp
deleted file mode 100644
index 873fd50041b1..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/cos.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the cos function for GPU ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/cos.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, cos, (double x)) { return __nv_cos(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/cosf.cpp b/external/llvm-project/libc/src/math/nvptx/cosf.cpp
deleted file mode 100644
index 82b3bb759ba0..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/cosf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the cosf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/cosf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, cosf, (float x)) { return __nv_cosf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/cosh.cpp b/external/llvm-project/libc/src/math/nvptx/cosh.cpp
deleted file mode 100644
index 434e8c56bbf7..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/cosh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the cosh function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/cosh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, cosh, (double x)) { return __nv_cosh(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/coshf.cpp b/external/llvm-project/libc/src/math/nvptx/coshf.cpp
deleted file mode 100644
index 87d6b16ba8a3..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/coshf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the coshf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/coshf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, coshf, (float x)) { return __nv_coshf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/declarations.h b/external/llvm-project/libc/src/math/nvptx/declarations.h
deleted file mode 100644
index 6f0bcfe7dce0..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/declarations.h
+++ /dev/null
@@ -1,94 +0,0 @@
-//===-- NVPTX specific declarations for math support ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_MATH_NVPTX_DECLARATIONS_H
-#define LLVM_LIBC_SRC_MATH_NVPTX_DECLARATIONS_H
-
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-extern "C" {
-double __nv_acos(double);
-float __nv_acosf(float);
-double __nv_acosh(double);
-float __nv_acoshf(float);
-double __nv_asin(double);
-float __nv_asinf(float);
-double __nv_asinh(double);
-float __nv_asinhf(float);
-double __nv_atan(double);
-float __nv_atanf(float);
-double __nv_atan2(double, double);
-float __nv_atan2f(float, float);
-double __nv_atanh(double);
-float __nv_atanhf(float);
-double __nv_cos(double);
-float __nv_cosf(float);
-double __nv_cosh(double);
-float __nv_coshf(float);
-double __nv_erf(double);
-float __nv_erff(float);
-double __nv_exp(double);
-float __nv_expf(float);
-double __nv_exp2(double);
-float __nv_exp2f(float);
-double __nv_exp10(double);
-float __nv_exp10f(float);
-double __nv_expm1(double);
-float __nv_expm1f(float);
-double __nv_fdim(double, double);
-float __nv_fdimf(float, float);
-double __nv_hypot(double, double);
-float __nv_hypotf(float, float);
-int __nv_ilogb(double);
-int __nv_ilogbf(float);
-double __nv_ldexp(double, int);
-float __nv_ldexpf(float, int);
-long long __nv_llrint(double);
-long long __nv_llrintf(float);
-long __nv_lrint(double);
-long __nv_lrintf(float);
-double __nv_log10(double);
-float __nv_log10f(float);
-double __nv_log1p(double);
-float __nv_log1pf(float);
-double __nv_log2(double);
-float __nv_log2f(float);
-double __nv_log(double);
-float __nv_logf(float);
-double __nv_nextafter(double, double);
-float __nv_nextafterf(float, float);
-double __nv_pow(double, double);
-float __nv_powf(float, float);
-double __nv_powi(double, int);
-float __nv_powif(float, int);
-double __nv_sin(double);
-float __nv_sinf(float);
-void __nv_sincos(double, double *, double *);
-void __nv_sincosf(float, float *, float *);
-double __nv_sinh(double);
-float __nv_sinhf(float);
-double __nv_tan(double);
-float __nv_tanf(float);
-double __nv_tanh(double);
-float __nv_tanhf(float);
-double __nv_frexp(double, int *);
-float __nv_frexpf(float, int *);
-double __nv_scalbn(double, int);
-float __nv_scalbnf(float, int);
-double __nv_remquo(double, double, int *);
-float __nv_remquof(float, float, int *);
-double __nv_tgamma(double);
-float __nv_tgammaf(float);
-float __nv_lgamma(double);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC_MATH_NVPTX_DECLARATIONS_H
diff --git a/external/llvm-project/libc/src/math/nvptx/erf.cpp b/external/llvm-project/libc/src/math/nvptx/erf.cpp
deleted file mode 100644
index c7ca7d42fe8a..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/erf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU erf function ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/erf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, erf, (double x)) { return __nv_erf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/erff.cpp b/external/llvm-project/libc/src/math/nvptx/erff.cpp
deleted file mode 100644
index 1c64e07c3370..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/erff.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU erff function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/erff.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, erff, (float x)) { return __nv_erff(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/exp10.cpp b/external/llvm-project/libc/src/math/nvptx/exp10.cpp
deleted file mode 100644
index 0972a3c4ae46..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/exp10.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU exp10 function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp10.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, exp10, (double x)) { return __nv_exp10(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/exp10f.cpp b/external/llvm-project/libc/src/math/nvptx/exp10f.cpp
deleted file mode 100644
index 0709771fc99a..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/exp10f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the exp10f function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp10f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, exp10f, (float x)) { return __nv_exp10f(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/exp2.cpp b/external/llvm-project/libc/src/math/nvptx/exp2.cpp
deleted file mode 100644
index 3465e5b608fa..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/exp2.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU exp2 function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp2.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, exp2, (double x)) { return __nv_exp2(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/exp2f.cpp b/external/llvm-project/libc/src/math/nvptx/exp2f.cpp
deleted file mode 100644
index 05257648c2e0..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/exp2f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the exp2f function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/exp2f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, exp2f, (float x)) { return __nv_exp2f(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/expf.cpp b/external/llvm-project/libc/src/math/nvptx/expf.cpp
deleted file mode 100644
index 2b802e5c3555..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/expf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the expf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/expf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, expf, (float x)) { return __nv_expf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/expm1.cpp b/external/llvm-project/libc/src/math/nvptx/expm1.cpp
deleted file mode 100644
index 91845cffc5fd..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/expm1.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU expm1 function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/expm1.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, expm1, (double x)) { return __nv_expm1(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/expm1f.cpp b/external/llvm-project/libc/src/math/nvptx/expm1f.cpp
deleted file mode 100644
index 31e67af084fd..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/expm1f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the expm1f function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/expm1f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, expm1f, (float x)) { return __nv_expm1f(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/fdim.cpp b/external/llvm-project/libc/src/math/nvptx/fdim.cpp
deleted file mode 100644
index edf400927323..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/fdim.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the fdim function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/fdim.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, fdim, (double x, double y)) {
-  return __nv_fdim(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/fdimf.cpp b/external/llvm-project/libc/src/math/nvptx/fdimf.cpp
deleted file mode 100644
index 017143aaebc8..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/fdimf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the fdimf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/fdimf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, fdimf, (float x, float y)) {
-  return __nv_fdimf(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/frexp.cpp b/external/llvm-project/libc/src/math/nvptx/frexp.cpp
deleted file mode 100644
index edba8d712aed..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/frexp.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the frexp function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/frexp.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, frexp, (double x, int *p)) {
-  return __nv_frexp(x, p);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/frexpf.cpp b/external/llvm-project/libc/src/math/nvptx/frexpf.cpp
deleted file mode 100644
index 04619412c160..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/frexpf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the frexpf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/frexpf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, frexpf, (float x, int *p)) {
-  return __nv_frexpf(x, p);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/hypot.cpp b/external/llvm-project/libc/src/math/nvptx/hypot.cpp
deleted file mode 100644
index a7bacd6b1a85..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/hypot.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the hypot function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/hypot.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, hypot, (double x, double y)) {
-  return __nv_hypot(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/hypotf.cpp b/external/llvm-project/libc/src/math/nvptx/hypotf.cpp
deleted file mode 100644
index 49e85a11a0c1..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/hypotf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the hypotf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/hypotf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) {
-  return __nv_hypotf(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/ilogb.cpp b/external/llvm-project/libc/src/math/nvptx/ilogb.cpp
deleted file mode 100644
index 3d552c2130af..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/ilogb.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the ilogb function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/ilogb.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(int, ilogb, (double x)) { return __nv_ilogb(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/ilogbf.cpp b/external/llvm-project/libc/src/math/nvptx/ilogbf.cpp
deleted file mode 100644
index a78926f95d6d..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/ilogbf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the ilogbf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/ilogbf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(int, ilogbf, (float x)) { return __nv_ilogbf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/ldexp.cpp b/external/llvm-project/libc/src/math/nvptx/ldexp.cpp
deleted file mode 100644
index 0adc7c3f57f0..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/ldexp.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the ldexp function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/ldexp.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, ldexp, (double x, int y)) {
-  return __nv_ldexp(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/ldexpf.cpp b/external/llvm-project/libc/src/math/nvptx/ldexpf.cpp
deleted file mode 100644
index eb7dd4285315..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/ldexpf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the ldexpf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/ldexpf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, ldexpf, (float x, int y)) {
-  return __nv_ldexpf(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/lgamma.cpp b/external/llvm-project/libc/src/math/nvptx/lgamma.cpp
index 0447a97eeae7..0dc3111c6ae1 100644
--- a/external/llvm-project/libc/src/math/nvptx/lgamma.cpp
+++ b/external/llvm-project/libc/src/math/nvptx/lgamma.cpp
@@ -9,11 +9,10 @@
 #include "src/math/lgamma.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, lgamma, (double x)) { return __nv_lgamma(x); }
+LLVM_LIBC_FUNCTION(double, lgamma, (double)) { return 0.0; }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/lgamma_r.cpp b/external/llvm-project/libc/src/math/nvptx/lgamma_r.cpp
index 85f66c8ea5d2..92018da4f860 100644
--- a/external/llvm-project/libc/src/math/nvptx/lgamma_r.cpp
+++ b/external/llvm-project/libc/src/math/nvptx/lgamma_r.cpp
@@ -9,15 +9,13 @@
 #include "src/math/lgamma_r.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, lgamma_r, (double x, int *signp)) {
-  double result = __nv_lgamma(x);
-  *signp = (result < 0.0) ? -1 : 1;
-  return result;
+LLVM_LIBC_FUNCTION(double, lgamma_r, (double, int *signp)) {
+  *signp = 0.0;
+  return 0.0;
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/llrint.cpp b/external/llvm-project/libc/src/math/nvptx/llrint.cpp
index 21129fe80a0c..6e0f57a5a038 100644
--- a/external/llvm-project/libc/src/math/nvptx/llrint.cpp
+++ b/external/llvm-project/libc/src/math/nvptx/llrint.cpp
@@ -9,7 +9,6 @@
 #include "src/math/llrint.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/nvptx/llrintf.cpp b/external/llvm-project/libc/src/math/nvptx/llrintf.cpp
index a6f9f4387601..d8de23fac3c8 100644
--- a/external/llvm-project/libc/src/math/nvptx/llrintf.cpp
+++ b/external/llvm-project/libc/src/math/nvptx/llrintf.cpp
@@ -9,7 +9,6 @@
 #include "src/math/llrintf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/nvptx/log.cpp b/external/llvm-project/libc/src/math/nvptx/log.cpp
deleted file mode 100644
index fd556ac1e165..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/log.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log function ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log, (double x)) { return __nv_log(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/log10.cpp b/external/llvm-project/libc/src/math/nvptx/log10.cpp
deleted file mode 100644
index fbbf214167c1..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/log10.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log10 function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log10.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log10, (double x)) { return __nv_log10(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/log10f.cpp b/external/llvm-project/libc/src/math/nvptx/log10f.cpp
deleted file mode 100644
index c2f24df043ab..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/log10f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log10f function -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log10f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, log10f, (float x)) { return __nv_log10f(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/log1p.cpp b/external/llvm-project/libc/src/math/nvptx/log1p.cpp
deleted file mode 100644
index 2ffd0fee5717..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/log1p.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log1p function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log1p.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log1p, (double x)) { return __nv_log1p(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/log1pf.cpp b/external/llvm-project/libc/src/math/nvptx/log1pf.cpp
deleted file mode 100644
index 2de4f27ff20c..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/log1pf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log1pf function -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log1pf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { return __nv_log1pf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/log2.cpp b/external/llvm-project/libc/src/math/nvptx/log2.cpp
deleted file mode 100644
index 54e2da0c9efa..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/log2.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log2 function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log2.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, log2, (double x)) { return __nv_log2(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/log2f.cpp b/external/llvm-project/libc/src/math/nvptx/log2f.cpp
deleted file mode 100644
index c2a6054c797f..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/log2f.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU log2f function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/log2f.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, log2f, (float x)) { return __nv_log2f(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/logb.cpp b/external/llvm-project/libc/src/math/nvptx/logb.cpp
deleted file mode 100644
index f6998bd19488..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/logb.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU logb function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/logb.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, logb, (double x)) { return __nv_logb(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/logbf.cpp b/external/llvm-project/libc/src/math/nvptx/logbf.cpp
deleted file mode 100644
index f72c04f988b6..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/logbf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU logbf function --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/logbf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, logbf, (float x)) { return __nv_logbf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/logf.cpp b/external/llvm-project/libc/src/math/nvptx/logf.cpp
deleted file mode 100644
index 1a5bc1e8fc1b..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/logf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the GPU logf function ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/logf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, logf, (float x)) { return __nv_logf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/lrint.cpp b/external/llvm-project/libc/src/math/nvptx/lrint.cpp
index 715b55257578..5ba70ec890b2 100644
--- a/external/llvm-project/libc/src/math/nvptx/lrint.cpp
+++ b/external/llvm-project/libc/src/math/nvptx/lrint.cpp
@@ -9,7 +9,6 @@
 #include "src/math/lrint.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/nvptx/lrintf.cpp b/external/llvm-project/libc/src/math/nvptx/lrintf.cpp
deleted file mode 100644
index 3ed05f6ca34c..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/lrintf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the lrintf function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/lrintf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(long, lrintf, (float x)) { return __nv_lrintf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/nextafter.cpp b/external/llvm-project/libc/src/math/nvptx/nextafter.cpp
deleted file mode 100644
index 3dc910042192..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/nextafter.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the nextafter function for GPU ------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/nextafter.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, nextafter, (double x, double y)) {
-  return __nv_nextafter(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/nextafterf.cpp b/external/llvm-project/libc/src/math/nvptx/nextafterf.cpp
deleted file mode 100644
index 162b85ef76c1..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/nextafterf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the nextafterf function for GPU -----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/nextafterf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, nextafterf, (float x, float y)) {
-  return __nv_nextafterf(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/nvptx.h b/external/llvm-project/libc/src/math/nvptx/nvptx.h
deleted file mode 100644
index 2035b766e622..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/nvptx.h
+++ /dev/null
@@ -1,103 +0,0 @@
-//===-- NVPTX specific definitions for math support -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_MATH_NVPTX_NVPTX_H
-#define LLVM_LIBC_SRC_MATH_NVPTX_NVPTX_H
-
-#include "declarations.h"
-
-#include "src/__support/macros/attributes.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace internal {
-LIBC_INLINE double acos(double x) { return __nv_acos(x); }
-LIBC_INLINE float acosf(float x) { return __nv_acosf(x); }
-LIBC_INLINE double acosh(double x) { return __nv_acosh(x); }
-LIBC_INLINE float acoshf(float x) { return __nv_acoshf(x); }
-LIBC_INLINE double asin(double x) { return __nv_asin(x); }
-LIBC_INLINE float asinf(float x) { return __nv_asinf(x); }
-LIBC_INLINE double asinh(double x) { return __nv_asinh(x); }
-LIBC_INLINE float asinhf(float x) { return __nv_asinhf(x); }
-LIBC_INLINE double atan2(double x, double y) { return __nv_atan2(x, y); }
-LIBC_INLINE float atan2f(float x, float y) { return __nv_atan2f(x, y); }
-LIBC_INLINE double atan(double x) { return __nv_atan(x); }
-LIBC_INLINE float atanf(float x) { return __nv_atanf(x); }
-LIBC_INLINE double atanh(double x) { return __nv_atanh(x); }
-LIBC_INLINE float atanhf(float x) { return __nv_atanhf(x); }
-LIBC_INLINE double cos(double x) { return __nv_cos(x); }
-LIBC_INLINE float cosf(float x) { return __nv_cosf(x); }
-LIBC_INLINE double cosh(double x) { return __nv_cosh(x); }
-LIBC_INLINE float coshf(float x) { return __nv_coshf(x); }
-LIBC_INLINE double erf(double x) { return __nv_erf(x); }
-LIBC_INLINE float erff(float x) { return __nv_erff(x); }
-LIBC_INLINE double exp(double x) { return __nv_exp(x); }
-LIBC_INLINE float expf(float x) { return __nv_expf(x); }
-LIBC_INLINE double exp2(double x) { return __nv_exp2(x); }
-LIBC_INLINE float exp2f(float x) { return __nv_exp2f(x); }
-LIBC_INLINE double exp10(double x) { return __nv_exp10(x); }
-LIBC_INLINE float exp10f(float x) { return __nv_exp10f(x); }
-LIBC_INLINE double expm1(double x) { return __nv_expm1(x); }
-LIBC_INLINE float expm1f(float x) { return __nv_expm1f(x); }
-LIBC_INLINE double fdim(double x, double y) { return __nv_fdim(x, y); }
-LIBC_INLINE float fdimf(float x, float y) { return __nv_fdimf(x, y); }
-LIBC_INLINE double hypot(double x, double y) { return __nv_hypot(x, y); }
-LIBC_INLINE float hypotf(float x, float y) { return __nv_hypotf(x, y); }
-LIBC_INLINE int ilogb(double x) { return __nv_ilogb(x); }
-LIBC_INLINE int ilogbf(float x) { return __nv_ilogbf(x); }
-LIBC_INLINE double ldexp(double x, int i) { return __nv_ldexp(x, i); }
-LIBC_INLINE float ldexpf(float x, int i) { return __nv_ldexpf(x, i); }
-LIBC_INLINE long long llrint(double x) { return __nv_llrint(x); }
-LIBC_INLINE long long llrintf(float x) { return __nv_llrintf(x); }
-LIBC_INLINE double log10(double x) { return __nv_log10(x); }
-LIBC_INLINE float log10f(float x) { return __nv_log10f(x); }
-LIBC_INLINE double log1p(double x) { return __nv_log1p(x); }
-LIBC_INLINE float log1pf(float x) { return __nv_log1pf(x); }
-LIBC_INLINE double log2(double x) { return __nv_log2(x); }
-LIBC_INLINE float log2f(float x) { return __nv_log2f(x); }
-LIBC_INLINE double log(double x) { return __nv_log(x); }
-LIBC_INLINE float logf(float x) { return __nv_logf(x); }
-LIBC_INLINE long lrint(double x) { return __nv_lrint(x); }
-LIBC_INLINE long lrintf(float x) { return __nv_lrintf(x); }
-LIBC_INLINE double nextafter(double x, double y) {
-  return __nv_nextafter(x, y);
-}
-LIBC_INLINE float nextafterf(float x, float y) { return __nv_nextafterf(x, y); }
-LIBC_INLINE double pow(double x, double y) { return __nv_pow(x, y); }
-LIBC_INLINE float powf(float x, float y) { return __nv_powf(x, y); }
-LIBC_INLINE double sin(double x) { return __nv_sin(x); }
-LIBC_INLINE float sinf(float x) { return __nv_sinf(x); }
-LIBC_INLINE void sincos(double x, double *sinptr, double *cosptr) {
-  return __nv_sincos(x, sinptr, cosptr);
-}
-LIBC_INLINE void sincosf(float x, float *sinptr, float *cosptr) {
-  return __nv_sincosf(x, sinptr, cosptr);
-}
-LIBC_INLINE double sinh(double x) { return __nv_sinh(x); }
-LIBC_INLINE float sinhf(float x) { return __nv_sinhf(x); }
-LIBC_INLINE double tan(double x) { return __nv_tan(x); }
-LIBC_INLINE float tanf(float x) { return __nv_tanf(x); }
-LIBC_INLINE double tanh(double x) { return __nv_tanh(x); }
-LIBC_INLINE float tanhf(float x) { return __nv_tanhf(x); }
-LIBC_INLINE double scalbn(double x, int i) { return __nv_scalbn(x, i); }
-LIBC_INLINE float scalbnf(float x, int i) { return __nv_scalbnf(x, i); }
-LIBC_INLINE double frexp(double x, int *i) { return __nv_frexp(x, i); }
-LIBC_INLINE float frexpf(float x, int *i) { return __nv_frexpf(x, i); }
-LIBC_INLINE double remquo(double x, double y, int *i) {
-  return __nv_remquo(x, y, i);
-}
-LIBC_INLINE float remquof(float x, float y, int *i) {
-  return __nv_remquof(x, y, i);
-}
-LIBC_INLINE double tgamma(double x) { return __nv_tgamma(x); }
-LIBC_INLINE float tgammaf(float x) { return __nv_tgammaf(x); }
-
-} // namespace internal
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC_MATH_NVPTX_NVPTX_H
diff --git a/external/llvm-project/libc/src/math/nvptx/powf.cpp b/external/llvm-project/libc/src/math/nvptx/powf.cpp
deleted file mode 100644
index 9c577c78a4a8..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/powf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the powf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/powf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { return __nv_powf(x, y); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/powi.cpp b/external/llvm-project/libc/src/math/nvptx/powi.cpp
deleted file mode 100644
index 16214bd9cc27..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/powi.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the powi function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/powi.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-
-#include "declarations.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, powi, (double x, int y)) { return __nv_powi(x, y); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/powif.cpp b/external/llvm-project/libc/src/math/nvptx/powif.cpp
deleted file mode 100644
index 5d26fdc87dac..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/powif.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the powif function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/powif.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-
-#include "declarations.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, powif, (float x, int y)) { return __nv_powif(x, y); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/remquo.cpp b/external/llvm-project/libc/src/math/nvptx/remquo.cpp
deleted file mode 100644
index aae1831bce61..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/remquo.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the GPU remquo function -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/remquo.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, remquo, (double x, double y, int *quo)) {
-  return __nv_remquo(x, y, quo);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/remquof.cpp b/external/llvm-project/libc/src/math/nvptx/remquof.cpp
deleted file mode 100644
index d7ce3c0f03b5..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/remquof.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the GPU remquof function ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/remquof.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, remquof, (float x, float y, int *quo)) {
-  return __nv_remquof(x, y, quo);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/scalbn.cpp b/external/llvm-project/libc/src/math/nvptx/scalbn.cpp
deleted file mode 100644
index ecadc345a353..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/scalbn.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the GPU scalbn function -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/scalbn.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, scalbn, (double x, int y)) {
-  return __nv_scalbn(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/scalbnf.cpp b/external/llvm-project/libc/src/math/nvptx/scalbnf.cpp
deleted file mode 100644
index 35ff69964fff..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/scalbnf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the GPU scalbnf function ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/scalbnf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, scalbnf, (float x, int y)) {
-  return __nv_scalbnf(x, y);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/sin.cpp b/external/llvm-project/libc/src/math/nvptx/sin.cpp
deleted file mode 100644
index 0e86f9c77e59..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/sin.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sin function for GPU ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sin.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, sin, (double x)) { return __nv_sin(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/sincos.cpp b/external/llvm-project/libc/src/math/nvptx/sincos.cpp
deleted file mode 100644
index 5a77234e2d74..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/sincos.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the sincos function for GPU ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sincos.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sinptr, double *cosptr)) {
-  return __nv_sincos(x, sinptr, cosptr);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/sincosf.cpp b/external/llvm-project/libc/src/math/nvptx/sincosf.cpp
deleted file mode 100644
index e4039ada837b..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/sincosf.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation of the sincosf function for GPU --------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sincosf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinptr, float *cosptr)) {
-  return __nv_sincosf(x, sinptr, cosptr);
-}
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/sinf.cpp b/external/llvm-project/libc/src/math/nvptx/sinf.cpp
deleted file mode 100644
index 14e722fbad19..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/sinf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sinf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sinf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, sinf, (float x)) { return __nv_sinf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/sinh.cpp b/external/llvm-project/libc/src/math/nvptx/sinh.cpp
deleted file mode 100644
index 701811d9a992..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/sinh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sinh function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sinh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, sinh, (double x)) { return __nv_sinh(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/sinhf.cpp b/external/llvm-project/libc/src/math/nvptx/sinhf.cpp
deleted file mode 100644
index 2c6ac21fffa1..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/sinhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the sinhf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/sinhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, sinhf, (float x)) { return __nv_sinhf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/tan.cpp b/external/llvm-project/libc/src/math/nvptx/tan.cpp
deleted file mode 100644
index 2d3f1fe51ea4..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/tan.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tan function for GPU ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tan.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, tan, (double x)) { return __nv_tan(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/tanf.cpp b/external/llvm-project/libc/src/math/nvptx/tanf.cpp
deleted file mode 100644
index bdf51d9a619a..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/tanf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tanf function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tanf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, tanf, (float x)) { return __nv_tanf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/tanh.cpp b/external/llvm-project/libc/src/math/nvptx/tanh.cpp
deleted file mode 100644
index 8255889e7d2b..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/tanh.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tanh function for GPU -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tanh.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(double, tanh, (double x)) { return __nv_tanh(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/tanhf.cpp b/external/llvm-project/libc/src/math/nvptx/tanhf.cpp
deleted file mode 100644
index a22524a0fa42..000000000000
--- a/external/llvm-project/libc/src/math/nvptx/tanhf.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- Implementation of the tanhf function for GPU ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/math/tanhf.h"
-#include "src/__support/common.h"
-
-#include "declarations.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-
-LLVM_LIBC_FUNCTION(float, tanhf, (float x)) { return __nv_tanhf(x); }
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/tgamma.cpp b/external/llvm-project/libc/src/math/nvptx/tgamma.cpp
index 29f0334f7274..0dfb0fd9ee13 100644
--- a/external/llvm-project/libc/src/math/nvptx/tgamma.cpp
+++ b/external/llvm-project/libc/src/math/nvptx/tgamma.cpp
@@ -9,11 +9,10 @@
 #include "src/math/tgamma.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, tgamma, (double x)) { return __nv_tgamma(x); }
+LLVM_LIBC_FUNCTION(double, tgamma, (double)) { return 0.0; }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/nvptx/tgammaf.cpp b/external/llvm-project/libc/src/math/nvptx/tgammaf.cpp
index d7de80b36d29..bb77cf56600b 100644
--- a/external/llvm-project/libc/src/math/nvptx/tgammaf.cpp
+++ b/external/llvm-project/libc/src/math/nvptx/tgammaf.cpp
@@ -9,11 +9,10 @@
 #include "src/math/tgammaf.h"
 #include "src/__support/common.h"
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, tgammaf, (float x)) { return __nv_tgammaf(x); }
+LLVM_LIBC_FUNCTION(float, tgammaf, (float)) { return 0.0; }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/poll/linux/poll.cpp b/external/llvm-project/libc/src/poll/linux/poll.cpp
index f82fcbcc6577..4cac75b9687c 100644
--- a/external/llvm-project/libc/src/poll/linux/poll.cpp
+++ b/external/llvm-project/libc/src/poll/linux/poll.cpp
@@ -13,8 +13,8 @@
 #include "hdr/types/struct_timespec.h"
 #include "src/__support/OSUtil/syscall.h" // syscall_impl
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // SYS_poll, SYS_ppoll
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_atfork.cpp b/external/llvm-project/libc/src/pthread/pthread_atfork.cpp
index b2c67c78e5d9..4cad16a02de7 100644
--- a/external/llvm-project/libc/src/pthread/pthread_atfork.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_atfork.cpp
@@ -9,9 +9,9 @@
 #include "pthread_atfork.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/fork_callbacks.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h> // For pthread_* type definitions.
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_attr_setdetachstate.cpp b/external/llvm-project/libc/src/pthread/pthread_attr_setdetachstate.cpp
index 872f694e01f3..c482d25610c2 100644
--- a/external/llvm-project/libc/src/pthread/pthread_attr_setdetachstate.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_attr_setdetachstate.cpp
@@ -9,8 +9,8 @@
 #include "pthread_attr_setdetachstate.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_attr_setguardsize.cpp b/external/llvm-project/libc/src/pthread/pthread_attr_setguardsize.cpp
index fa4375e915ab..c996210a61d8 100644
--- a/external/llvm-project/libc/src/pthread/pthread_attr_setguardsize.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_attr_setguardsize.cpp
@@ -9,8 +9,8 @@
 #include "pthread_attr_setguardsize.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <linux/param.h> // For EXEC_PAGESIZE.
 #include <pthread.h>
diff --git a/external/llvm-project/libc/src/pthread/pthread_attr_setstack.cpp b/external/llvm-project/libc/src/pthread/pthread_attr_setstack.cpp
index 1154055a63a7..767f959b1400 100644
--- a/external/llvm-project/libc/src/pthread/pthread_attr_setstack.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_attr_setstack.cpp
@@ -10,9 +10,9 @@
 #include "pthread_attr_setstacksize.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h" // For STACK_ALIGNMENT
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 #include <stdint.h>
diff --git a/external/llvm-project/libc/src/pthread/pthread_attr_setstacksize.cpp b/external/llvm-project/libc/src/pthread/pthread_attr_setstacksize.cpp
index 0a5d1af661ab..38c77ca761d6 100644
--- a/external/llvm-project/libc/src/pthread/pthread_attr_setstacksize.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_attr_setstacksize.cpp
@@ -9,8 +9,8 @@
 #include "pthread_attr_setstacksize.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_condattr_setclock.cpp b/external/llvm-project/libc/src/pthread/pthread_condattr_setclock.cpp
index 5e825d5ecea6..2f63d5e9d194 100644
--- a/external/llvm-project/libc/src/pthread/pthread_condattr_setclock.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_condattr_setclock.cpp
@@ -9,8 +9,8 @@
 #include "pthread_condattr_setclock.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/time_macros.h" // CLOCK_MONOTONIC, CLOCK_REALTIME
 #include <pthread.h>         // pthread_condattr_t
diff --git a/external/llvm-project/libc/src/pthread/pthread_condattr_setpshared.cpp b/external/llvm-project/libc/src/pthread/pthread_condattr_setpshared.cpp
index 433b2dc1d2d9..9c117499a559 100644
--- a/external/llvm-project/libc/src/pthread/pthread_condattr_setpshared.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_condattr_setpshared.cpp
@@ -9,8 +9,8 @@
 #include "pthread_condattr_setpshared.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h> // pthread_condattr_t, PTHREAD_PROCESS_SHARED, PTHREAD_PROCESS_PRIVATE
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_create.cpp b/external/llvm-project/libc/src/pthread/pthread_create.cpp
index e1b1f3b325d1..45be2807fa83 100644
--- a/external/llvm-project/libc/src/pthread/pthread_create.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_create.cpp
@@ -16,10 +16,10 @@
 #include "pthread_attr_getstack.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/threads/thread.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h> // For pthread_* type definitions.
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_key_create.cpp b/external/llvm-project/libc/src/pthread/pthread_key_create.cpp
index 383762f273e7..7253de14cc0d 100644
--- a/external/llvm-project/libc/src/pthread/pthread_key_create.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_key_create.cpp
@@ -9,9 +9,9 @@
 #include "pthread_key_create.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_key_delete.cpp b/external/llvm-project/libc/src/pthread/pthread_key_delete.cpp
index b54db821ab05..2b14d874fe31 100644
--- a/external/llvm-project/libc/src/pthread/pthread_key_delete.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_key_delete.cpp
@@ -9,9 +9,9 @@
 #include "pthread_key_delete.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_mutexattr_setpshared.cpp b/external/llvm-project/libc/src/pthread/pthread_mutexattr_setpshared.cpp
index deeae15be230..a87a08259c4b 100644
--- a/external/llvm-project/libc/src/pthread/pthread_mutexattr_setpshared.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_mutexattr_setpshared.cpp
@@ -10,8 +10,8 @@
 #include "pthread_mutexattr.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_mutexattr_setrobust.cpp b/external/llvm-project/libc/src/pthread/pthread_mutexattr_setrobust.cpp
index 9fd46f4c928d..fd7a8d7ce1d1 100644
--- a/external/llvm-project/libc/src/pthread/pthread_mutexattr_setrobust.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_mutexattr_setrobust.cpp
@@ -10,8 +10,8 @@
 #include "pthread_mutexattr.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_mutexattr_settype.cpp b/external/llvm-project/libc/src/pthread/pthread_mutexattr_settype.cpp
index c7e78271f9c3..5a65f031045d 100644
--- a/external/llvm-project/libc/src/pthread/pthread_mutexattr_settype.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_mutexattr_settype.cpp
@@ -10,8 +10,8 @@
 #include "pthread_mutexattr.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_rwlock_timedrdlock.cpp b/external/llvm-project/libc/src/pthread/pthread_rwlock_timedrdlock.cpp
index 112ff5c9cdad..fcddfed22490 100644
--- a/external/llvm-project/libc/src/pthread/pthread_rwlock_timedrdlock.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_rwlock_timedrdlock.cpp
@@ -9,11 +9,11 @@
 #include "src/pthread/pthread_rwlock_timedrdlock.h"
 #include "src/__support/common.h"
 #include "src/__support/libc_assert.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/threads/linux/rwlock.h"
 #include "src/__support/time/linux/abs_timeout.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_rwlock_trywrlock.cpp b/external/llvm-project/libc/src/pthread/pthread_rwlock_trywrlock.cpp
index a63dc893e716..660c15a87b36 100644
--- a/external/llvm-project/libc/src/pthread/pthread_rwlock_trywrlock.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_rwlock_trywrlock.cpp
@@ -9,9 +9,9 @@
 #include "src/pthread/pthread_rwlock_trywrlock.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/linux/rwlock.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_rwlock_unlock.cpp b/external/llvm-project/libc/src/pthread/pthread_rwlock_unlock.cpp
index e61290179bd6..5496bea929c5 100644
--- a/external/llvm-project/libc/src/pthread/pthread_rwlock_unlock.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_rwlock_unlock.cpp
@@ -9,9 +9,9 @@
 #include "src/pthread/pthread_rwlock_unlock.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/linux/rwlock.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp b/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
index 80d34a35c717..e6800311b858 100644
--- a/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
@@ -9,8 +9,8 @@
 #include "pthread_rwlockattr_setkind_np.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h> // pthread_rwlockattr_t
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setpshared.cpp b/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setpshared.cpp
index 5a7191aefd3d..4fbd095ac2b4 100644
--- a/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setpshared.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_rwlockattr_setpshared.cpp
@@ -9,8 +9,8 @@
 #include "pthread_rwlockattr_setpshared.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h> // pthread_rwlockattr_t, PTHREAD_PROCESS_SHARED, PTHREAD_PROCESS_PRIVATE
 
diff --git a/external/llvm-project/libc/src/pthread/pthread_setspecific.cpp b/external/llvm-project/libc/src/pthread/pthread_setspecific.cpp
index 70c29c167084..b147a66d2fad 100644
--- a/external/llvm-project/libc/src/pthread/pthread_setspecific.cpp
+++ b/external/llvm-project/libc/src/pthread/pthread_setspecific.cpp
@@ -9,9 +9,9 @@
 #include "pthread_setspecific.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h"
-#include "src/errno/libc_errno.h"
 
 #include <pthread.h>
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_get_priority_max.cpp b/external/llvm-project/libc/src/sched/linux/sched_get_priority_max.cpp
index 77a82c77405f..fb30b1e319e7 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_get_priority_max.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_get_priority_max.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_get_priority_min.cpp b/external/llvm-project/libc/src/sched/linux/sched_get_priority_min.cpp
index fca66a15edb5..54f67e915fc1 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_get_priority_min.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_get_priority_min.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_getaffinity.cpp b/external/llvm-project/libc/src/sched/linux/sched_getaffinity.cpp
index 7b1fd8c5aa2a..e005819e2a97 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_getaffinity.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_getaffinity.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sched.h>
 #include <stdint.h>
diff --git a/external/llvm-project/libc/src/sched/linux/sched_getparam.cpp b/external/llvm-project/libc/src/sched/linux/sched_getparam.cpp
index 75756a65f0ed..b0576c3ac65b 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_getparam.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_getparam.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_getscheduler.cpp b/external/llvm-project/libc/src/sched/linux/sched_getscheduler.cpp
index 545cda8e7484..d8e02967a633 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_getscheduler.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_getscheduler.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_rr_get_interval.cpp b/external/llvm-project/libc/src/sched/linux/sched_rr_get_interval.cpp
index 1f0ef69dfc89..5668d596bce1 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_rr_get_interval.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_rr_get_interval.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_setaffinity.cpp b/external/llvm-project/libc/src/sched/linux/sched_setaffinity.cpp
index cad48c26bf93..93e930dcf2e3 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_setaffinity.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_setaffinity.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sched.h>
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/sched/linux/sched_setparam.cpp b/external/llvm-project/libc/src/sched/linux/sched_setparam.cpp
index e78e78a707e0..7875d9e2f19b 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_setparam.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_setparam.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_setscheduler.cpp b/external/llvm-project/libc/src/sched/linux/sched_setscheduler.cpp
index b6b6f667b3f9..232e5a59b185 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_setscheduler.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_setscheduler.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sched/linux/sched_yield.cpp b/external/llvm-project/libc/src/sched/linux/sched_yield.cpp
index 3de9d0ba3571..c1e9168f34d0 100644
--- a/external/llvm-project/libc/src/sched/linux/sched_yield.cpp
+++ b/external/llvm-project/libc/src/sched/linux/sched_yield.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/search/hcreate.cpp b/external/llvm-project/libc/src/search/hcreate.cpp
index ac816a902e22..68bdb29e51df 100644
--- a/external/llvm-project/libc/src/search/hcreate.cpp
+++ b/external/llvm-project/libc/src/search/hcreate.cpp
@@ -9,8 +9,8 @@
 #include "src/search/hcreate.h"
 #include "src/__support/HashTable/randomness.h"
 #include "src/__support/HashTable/table.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/search/hsearch/global.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/search/hcreate_r.cpp b/external/llvm-project/libc/src/search/hcreate_r.cpp
index 17acd808c19a..c89be803b4e1 100644
--- a/external/llvm-project/libc/src/search/hcreate_r.cpp
+++ b/external/llvm-project/libc/src/search/hcreate_r.cpp
@@ -9,8 +9,8 @@
 #include "src/search/hcreate_r.h"
 #include "src/__support/HashTable/randomness.h"
 #include "src/__support/HashTable/table.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(int, hcreate_r,
diff --git a/external/llvm-project/libc/src/search/hdestroy_r.cpp b/external/llvm-project/libc/src/search/hdestroy_r.cpp
index 7eff5bb6fff9..ba5476098be2 100644
--- a/external/llvm-project/libc/src/search/hdestroy_r.cpp
+++ b/external/llvm-project/libc/src/search/hdestroy_r.cpp
@@ -8,8 +8,8 @@
 
 #include "src/search/hdestroy_r.h"
 #include "src/__support/HashTable/table.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(void, hdestroy_r, (struct hsearch_data * htab)) {
diff --git a/external/llvm-project/libc/src/search/hsearch.cpp b/external/llvm-project/libc/src/search/hsearch.cpp
index c18b5d3d7f54..034333d17057 100644
--- a/external/llvm-project/libc/src/search/hsearch.cpp
+++ b/external/llvm-project/libc/src/search/hsearch.cpp
@@ -9,8 +9,8 @@
 #include "src/search/hsearch.h"
 #include "src/__support/HashTable/randomness.h"
 #include "src/__support/HashTable/table.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/search/hsearch/global.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/search/hsearch_r.cpp b/external/llvm-project/libc/src/search/hsearch_r.cpp
index f93e608a190b..323001e1b103 100644
--- a/external/llvm-project/libc/src/search/hsearch_r.cpp
+++ b/external/llvm-project/libc/src/search/hsearch_r.cpp
@@ -8,8 +8,8 @@
 
 #include "src/search/hsearch_r.h"
 #include "src/__support/HashTable/table.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(int, hsearch_r,
diff --git a/external/llvm-project/libc/src/signal/linux/kill.cpp b/external/llvm-project/libc/src/signal/linux/kill.cpp
index ed117858f51e..0f5e88757acb 100644
--- a/external/llvm-project/libc/src/signal/linux/kill.cpp
+++ b/external/llvm-project/libc/src/signal/linux/kill.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 #include <signal.h>
diff --git a/external/llvm-project/libc/src/signal/linux/sigaction.cpp b/external/llvm-project/libc/src/signal/linux/sigaction.cpp
index 65ec36741683..43a3e195474e 100644
--- a/external/llvm-project/libc/src/signal/linux/sigaction.cpp
+++ b/external/llvm-project/libc/src/signal/linux/sigaction.cpp
@@ -10,8 +10,8 @@
 
 #include "hdr/types/sigset_t.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/signal/linux/sigaddset.cpp b/external/llvm-project/libc/src/signal/linux/sigaddset.cpp
index 628883e13b88..2091e8b51453 100644
--- a/external/llvm-project/libc/src/signal/linux/sigaddset.cpp
+++ b/external/llvm-project/libc/src/signal/linux/sigaddset.cpp
@@ -10,8 +10,8 @@
 
 #include "hdr/types/sigset_t.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/signal/linux/sigaltstack.cpp b/external/llvm-project/libc/src/signal/linux/sigaltstack.cpp
index c19394cd1791..990b841c6d90 100644
--- a/external/llvm-project/libc/src/signal/linux/sigaltstack.cpp
+++ b/external/llvm-project/libc/src/signal/linux/sigaltstack.cpp
@@ -8,8 +8,8 @@
 
 #include "src/signal/sigaltstack.h"
 #include "hdr/types/stack_t.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 #include "src/__support/common.h"
diff --git a/external/llvm-project/libc/src/signal/linux/sigdelset.cpp b/external/llvm-project/libc/src/signal/linux/sigdelset.cpp
index 2e964051ebde..6fce0d7a6e14 100644
--- a/external/llvm-project/libc/src/signal/linux/sigdelset.cpp
+++ b/external/llvm-project/libc/src/signal/linux/sigdelset.cpp
@@ -10,8 +10,8 @@
 
 #include "hdr/types/sigset_t.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/signal/linux/sigemptyset.cpp b/external/llvm-project/libc/src/signal/linux/sigemptyset.cpp
index d347477695e6..034a9e2cbe15 100644
--- a/external/llvm-project/libc/src/signal/linux/sigemptyset.cpp
+++ b/external/llvm-project/libc/src/signal/linux/sigemptyset.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/signal/sigemptyset.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 #include "src/__support/common.h"
diff --git a/external/llvm-project/libc/src/signal/linux/sigfillset.cpp b/external/llvm-project/libc/src/signal/linux/sigfillset.cpp
index 3e9897a03bb7..f0b499093b31 100644
--- a/external/llvm-project/libc/src/signal/linux/sigfillset.cpp
+++ b/external/llvm-project/libc/src/signal/linux/sigfillset.cpp
@@ -10,8 +10,8 @@
 
 #include "hdr/types/sigset_t.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/signal/linux/sigprocmask.cpp b/external/llvm-project/libc/src/signal/linux/sigprocmask.cpp
index 8838379ae5d3..af3c424c5f34 100644
--- a/external/llvm-project/libc/src/signal/linux/sigprocmask.cpp
+++ b/external/llvm-project/libc/src/signal/linux/sigprocmask.cpp
@@ -11,8 +11,8 @@
 #include "hdr/types/sigset_t.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addclose.cpp b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addclose.cpp
index bb8504f655c4..9a575bd59163 100644
--- a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addclose.cpp
+++ b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addclose.cpp
@@ -11,8 +11,8 @@
 #include "file_actions.h"
 #include "src/__support/CPP/new.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <spawn.h>
 
diff --git a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_adddup2.cpp b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_adddup2.cpp
index 710063d52e74..1ad45ed942bb 100644
--- a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_adddup2.cpp
+++ b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_adddup2.cpp
@@ -11,8 +11,8 @@
 #include "file_actions.h"
 #include "src/__support/CPP/new.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <spawn.h>
 
diff --git a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addopen.cpp b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addopen.cpp
index 028d6e895f3c..9977fc2d0a21 100644
--- a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addopen.cpp
+++ b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_addopen.cpp
@@ -11,8 +11,8 @@
 #include "file_actions.h"
 #include "src/__support/CPP/new.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <spawn.h>
 
diff --git a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_destroy.cpp b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_destroy.cpp
index 168118da249d..affd338005cf 100644
--- a/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_destroy.cpp
+++ b/external/llvm-project/libc/src/spawn/posix_spawn_file_actions_destroy.cpp
@@ -12,8 +12,8 @@
 
 #include "src/__support/CPP/new.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <spawn.h>
 
diff --git a/external/llvm-project/libc/src/stdio/CMakeLists.txt b/external/llvm-project/libc/src/stdio/CMakeLists.txt
index 63f6ed8a11f1..b0a6ef1e291b 100644
--- a/external/llvm-project/libc/src/stdio/CMakeLists.txt
+++ b/external/llvm-project/libc/src/stdio/CMakeLists.txt
@@ -221,6 +221,7 @@ add_stdio_entrypoint_object(fopen)
 add_stdio_entrypoint_object(fclose)
 add_stdio_entrypoint_object(fread_unlocked)
 add_stdio_entrypoint_object(fread)
+add_stdio_entrypoint_object(perror)
 add_stdio_entrypoint_object(puts)
 add_stdio_entrypoint_object(fputs)
 add_stdio_entrypoint_object(fwrite_unlocked)
diff --git a/external/llvm-project/libc/src/stdio/fopencookie.cpp b/external/llvm-project/libc/src/stdio/fopencookie.cpp
index 9f5694e8e058..da8a132a4db6 100644
--- a/external/llvm-project/libc/src/stdio/fopencookie.cpp
+++ b/external/llvm-project/libc/src/stdio/fopencookie.cpp
@@ -14,8 +14,8 @@
 #include "src/__support/CPP/new.h"
 #include "src/__support/File/file.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/CMakeLists.txt b/external/llvm-project/libc/src/stdio/generic/CMakeLists.txt
index e1f4ed5c1949..6361822b6199 100644
--- a/external/llvm-project/libc/src/stdio/generic/CMakeLists.txt
+++ b/external/llvm-project/libc/src/stdio/generic/CMakeLists.txt
@@ -206,6 +206,21 @@ add_generic_entrypoint_object(
     libc.src.__support.File.platform_file
 )
 
+add_generic_entrypoint_object(
+  perror
+  SRCS
+    perror.cpp
+  HDRS
+    ../perror.h
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.__support.StringUtil.error_to_string
+    libc.src.__support.CPP.string_view
+    libc.src.__support.File.file
+    libc.src.__support.File.platform_file
+    libc.src.__support.File.platform_stderr
+)
+
 add_generic_entrypoint_object(
   fputs
   SRCS
diff --git a/external/llvm-project/libc/src/stdio/generic/fclose.cpp b/external/llvm-project/libc/src/stdio/generic/fclose.cpp
index 388407a58d41..902b4cf97237 100644
--- a/external/llvm-project/libc/src/stdio/generic/fclose.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fclose.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/fflush.cpp b/external/llvm-project/libc/src/stdio/generic/fflush.cpp
index 5bdf71ad3594..d0271d9154c8 100644
--- a/external/llvm-project/libc/src/stdio/generic/fflush.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fflush.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/fgetc.cpp b/external/llvm-project/libc/src/stdio/generic/fgetc.cpp
index aa6660ca180c..e65ce2fda49b 100644
--- a/external/llvm-project/libc/src/stdio/generic/fgetc.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fgetc.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fgetc_unlocked.cpp b/external/llvm-project/libc/src/stdio/generic/fgetc_unlocked.cpp
index 34a27f1d1c42..5c07d4feb513 100644
--- a/external/llvm-project/libc/src/stdio/generic/fgetc_unlocked.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fgetc_unlocked.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fgets.cpp b/external/llvm-project/libc/src/stdio/generic/fgets.cpp
index de6474087a14..e0ad9b6e2f56 100644
--- a/external/llvm-project/libc/src/stdio/generic/fgets.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fgets.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fopen.cpp b/external/llvm-project/libc/src/stdio/generic/fopen.cpp
index d6e418bacf37..57c85c2e54e1 100644
--- a/external/llvm-project/libc/src/stdio/generic/fopen.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fopen.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/fputc.cpp b/external/llvm-project/libc/src/stdio/generic/fputc.cpp
index 54a38aeb2f1e..6639f0687c87 100644
--- a/external/llvm-project/libc/src/stdio/generic/fputc.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fputc.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fputs.cpp b/external/llvm-project/libc/src/stdio/generic/fputs.cpp
index 8aef7683b3ce..621b40f63c91 100644
--- a/external/llvm-project/libc/src/stdio/generic/fputs.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fputs.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fread.cpp b/external/llvm-project/libc/src/stdio/generic/fread.cpp
index 3a04094ea8b4..1b576ec34688 100644
--- a/external/llvm-project/libc/src/stdio/generic/fread.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fread.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fread_unlocked.cpp b/external/llvm-project/libc/src/stdio/generic/fread_unlocked.cpp
index 151f43c6bbeb..257f1a212add 100644
--- a/external/llvm-project/libc/src/stdio/generic/fread_unlocked.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fread_unlocked.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fseek.cpp b/external/llvm-project/libc/src/stdio/generic/fseek.cpp
index 21820da18542..99191e7c4194 100644
--- a/external/llvm-project/libc/src/stdio/generic/fseek.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fseek.cpp
@@ -9,8 +9,8 @@
 #include "src/stdio/fseek.h"
 #include "src/__support/File/file.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/fseeko.cpp b/external/llvm-project/libc/src/stdio/generic/fseeko.cpp
index 7456b4a21907..afcfc71c7c09 100644
--- a/external/llvm-project/libc/src/stdio/generic/fseeko.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fseeko.cpp
@@ -9,8 +9,8 @@
 #include "src/stdio/fseeko.h"
 #include "src/__support/File/file.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/ftell.cpp b/external/llvm-project/libc/src/stdio/generic/ftell.cpp
index ec15ca4e96ca..b55a806007af 100644
--- a/external/llvm-project/libc/src/stdio/generic/ftell.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/ftell.cpp
@@ -9,8 +9,8 @@
 #include "src/stdio/ftell.h"
 #include "src/__support/File/file.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/ftello.cpp b/external/llvm-project/libc/src/stdio/generic/ftello.cpp
index e3d0726ec484..91031cb7fad7 100644
--- a/external/llvm-project/libc/src/stdio/generic/ftello.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/ftello.cpp
@@ -9,8 +9,8 @@
 #include "src/stdio/ftello.h"
 #include "src/__support/File/file.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/fwrite.cpp b/external/llvm-project/libc/src/stdio/generic/fwrite.cpp
index 66eb9a3c7185..b44ecb283811 100644
--- a/external/llvm-project/libc/src/stdio/generic/fwrite.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fwrite.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/fwrite_unlocked.cpp b/external/llvm-project/libc/src/stdio/generic/fwrite_unlocked.cpp
index a0d9014cd68d..2f9ec26f2f80 100644
--- a/external/llvm-project/libc/src/stdio/generic/fwrite_unlocked.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/fwrite_unlocked.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/getc.cpp b/external/llvm-project/libc/src/stdio/generic/getc.cpp
index e988468898c5..0ac010ebc599 100644
--- a/external/llvm-project/libc/src/stdio/generic/getc.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/getc.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/getc_unlocked.cpp b/external/llvm-project/libc/src/stdio/generic/getc_unlocked.cpp
index 92d5092623ac..eee23a18d05d 100644
--- a/external/llvm-project/libc/src/stdio/generic/getc_unlocked.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/getc_unlocked.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/getchar.cpp b/external/llvm-project/libc/src/stdio/generic/getchar.cpp
index 371fc70eb214..87d24a2b1f09 100644
--- a/external/llvm-project/libc/src/stdio/generic/getchar.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/getchar.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/getchar_unlocked.cpp b/external/llvm-project/libc/src/stdio/generic/getchar_unlocked.cpp
index b898f5cb2596..f321969483e3 100644
--- a/external/llvm-project/libc/src/stdio/generic/getchar_unlocked.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/getchar_unlocked.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/generic/perror.cpp b/external/llvm-project/libc/src/stdio/generic/perror.cpp
new file mode 100644
index 000000000000..68b4ad644caa
--- /dev/null
+++ b/external/llvm-project/libc/src/stdio/generic/perror.cpp
@@ -0,0 +1,81 @@
+//===-- Implementation of perror ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdio/perror.h"
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/File/file.h"
+#include "src/__support/StringUtil/error_to_string.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+static int write_out(cpp::string_view str_view, File *f) {
+  if (str_view.size() > 0) {
+    auto result = f->write_unlocked(str_view.data(), str_view.size());
+    if (result.has_error())
+      return result.error;
+  }
+  return 0;
+}
+
+// separate function so that we can return early on error but still get the
+// unlock. This function sets errno and should not be called elsewhere.
+static void write_sequence(cpp::string_view str_view,
+                           cpp::string_view err_str) {
+  int write_err;
+  // TODO: this seems like there should be some sort of queue system to
+  // deduplicate this code.
+
+  // FORMAT:
+  // if str != nullptr and doesn't start with a null byte:
+  //   "[str]: [strerror(errno)]\n"
+  // else
+  //   "[strerror(errno)]\n"
+  if (str_view.size() > 0) {
+    write_err = write_out(str_view, LIBC_NAMESPACE::stderr);
+    if (write_err != 0) {
+      libc_errno = write_err;
+      return;
+    }
+
+    write_err = write_out(": ", LIBC_NAMESPACE::stderr);
+    if (write_err != 0) {
+      libc_errno = write_err;
+      return;
+    }
+  }
+
+  write_err = write_out(err_str, LIBC_NAMESPACE::stderr);
+  if (write_err != 0) {
+    libc_errno = write_err;
+    return;
+  }
+
+  write_err = write_out("\n", LIBC_NAMESPACE::stderr);
+  if (write_err != 0) {
+    libc_errno = write_err;
+    return;
+  }
+}
+
+LLVM_LIBC_FUNCTION(void, perror, (const char *str)) {
+  const char empty_str[1] = {'\0'};
+  if (str == nullptr)
+    str = empty_str;
+  cpp::string_view str_view(str);
+
+  cpp::string_view err_str = get_error_string(libc_errno);
+
+  // We need to lock the stream to ensure the newline is always appended.
+  LIBC_NAMESPACE::stderr->lock();
+  write_sequence(str_view, err_str);
+  LIBC_NAMESPACE::stderr->unlock();
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/stdio/generic/putc.cpp b/external/llvm-project/libc/src/stdio/generic/putc.cpp
index b5f008fdce44..83bc3d4131e7 100644
--- a/external/llvm-project/libc/src/stdio/generic/putc.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/putc.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/putchar.cpp b/external/llvm-project/libc/src/stdio/generic/putchar.cpp
index e86df23d6716..2b3509e5e414 100644
--- a/external/llvm-project/libc/src/stdio/generic/putchar.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/putchar.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/generic/puts.cpp b/external/llvm-project/libc/src/stdio/generic/puts.cpp
index 7dbe2c79f920..4267dd546c4d 100644
--- a/external/llvm-project/libc/src/stdio/generic/puts.cpp
+++ b/external/llvm-project/libc/src/stdio/generic/puts.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/gpu/CMakeLists.txt b/external/llvm-project/libc/src/stdio/gpu/CMakeLists.txt
index c4ad333e2516..bea113409263 100644
--- a/external/llvm-project/libc/src/stdio/gpu/CMakeLists.txt
+++ b/external/llvm-project/libc/src/stdio/gpu/CMakeLists.txt
@@ -59,6 +59,7 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.FILE
     libc.src.__support.RPC.rpc_client
+    .gpu_file
 )
 
 add_entrypoint_object(
@@ -70,6 +71,7 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.FILE
     libc.src.__support.RPC.rpc_client
+    .gpu_file
 )
 
 add_entrypoint_object(
@@ -114,6 +116,7 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.FILE
     libc.src.__support.RPC.rpc_client
+    .gpu_file
 )
 
 add_entrypoint_object(
@@ -124,6 +127,7 @@ add_entrypoint_object(
     ../fopen.h
   DEPENDS
     libc.hdr.types.FILE
+    .gpu_file
 )
 
 add_entrypoint_object(
@@ -134,6 +138,8 @@ add_entrypoint_object(
     ../fclose.h
   DEPENDS
     libc.hdr.types.FILE
+    libc.hdr.stdio_macros
+    .gpu_file
 )
 
 add_entrypoint_object(
@@ -144,6 +150,7 @@ add_entrypoint_object(
     ../fread.h
   DEPENDS
     libc.hdr.types.FILE
+    .gpu_file
 )
 
 add_entrypoint_object(
@@ -254,8 +261,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.FILE
     .gpu_file
-    .feof
-    .ferror
 )
 
 add_entrypoint_object(
diff --git a/external/llvm-project/libc/src/stdio/gpu/clearerr.cpp b/external/llvm-project/libc/src/stdio/gpu/clearerr.cpp
index 4d843e4be33e..5a0ca52e33fa 100644
--- a/external/llvm-project/libc/src/stdio/gpu/clearerr.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/clearerr.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/clearerr.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fclose.cpp b/external/llvm-project/libc/src/stdio/gpu/fclose.cpp
index 847d6b846172..1e00515e0650 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fclose.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fclose.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fclose.h"
-#include "src/__support/macros/config.h"
-#include "src/stdio/gpu/file.h"
 
 #include "hdr/stdio_macros.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
+#include "src/stdio/gpu/file.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/feof.cpp b/external/llvm-project/libc/src/stdio/gpu/feof.cpp
index a15e4874d0e2..3ae308bad60b 100644
--- a/external/llvm-project/libc/src/stdio/gpu/feof.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/feof.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/feof.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/ferror.cpp b/external/llvm-project/libc/src/stdio/gpu/ferror.cpp
index 827382037067..64d62e706525 100644
--- a/external/llvm-project/libc/src/stdio/gpu/ferror.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/ferror.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/ferror.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fflush.cpp b/external/llvm-project/libc/src/stdio/gpu/fflush.cpp
index 5a5137be6a4a..0b6ef92f346a 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fflush.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fflush.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fflush.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fgetc.cpp b/external/llvm-project/libc/src/stdio/gpu/fgetc.cpp
index cb42f3100a9f..aaeb1591d3fa 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fgetc.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fgetc.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fgetc.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/stdio_macros.h" // for EOF.
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fgets.cpp b/external/llvm-project/libc/src/stdio/gpu/fgets.cpp
index d90b7aa4aa66..5447e86d1c76 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fgets.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fgets.cpp
@@ -7,14 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fgets.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
-#include "src/stdio/feof.h"
-#include "src/stdio/ferror.h"
 
+#include "file.h"
 #include "hdr/stdio_macros.h" // for EOF.
 #include "hdr/types/FILE.h"
-#include <stddef.h>
+#include "src/__support/common.h"
+
+#include <stdint.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/file.h b/external/llvm-project/libc/src/stdio/gpu/file.h
index 437ee9982438..102c6c601b30 100644
--- a/external/llvm-project/libc/src/stdio/gpu/file.h
+++ b/external/llvm-project/libc/src/stdio/gpu/file.h
@@ -6,12 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/__support/RPC/rpc_client.h"
-#include "src/__support/macros/config.h"
-#include "src/string/string_utils.h"
-
 #include "hdr/stdio_macros.h" // For stdin/out/err
 #include "hdr/types/FILE.h"
+#include "src/__support/RPC/rpc_client.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/attributes.h"
 
 namespace LIBC_NAMESPACE_DECL {
 namespace file {
diff --git a/external/llvm-project/libc/src/stdio/gpu/fopen.cpp b/external/llvm-project/libc/src/stdio/gpu/fopen.cpp
index 18dd71953778..eee3edaac1c1 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fopen.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fopen.cpp
@@ -7,11 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fopen.h"
-#include "src/__support/CPP/string_view.h"
-#include "src/__support/macros/config.h"
-#include "src/stdio/gpu/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
+#include "src/stdio/gpu/file.h"
+#include "src/string/memory_utils/inline_memcpy.h"
+#include "src/string/string_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fprintf.cpp b/external/llvm-project/libc/src/stdio/gpu/fprintf.cpp
index 46196d7d2b10..9877817d9209 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fprintf.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fprintf.cpp
@@ -11,7 +11,8 @@
 #include "hdr/types/FILE.h"
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/arg_list.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/stdio/gpu/vfprintf_utils.h"
 
 #include <stdarg.h>
diff --git a/external/llvm-project/libc/src/stdio/gpu/fputc.cpp b/external/llvm-project/libc/src/stdio/gpu/fputc.cpp
index ad2db03df0ca..8a8959b04d20 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fputc.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fputc.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fputc.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/stdio_macros.h" // for EOF.
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fputs.cpp b/external/llvm-project/libc/src/stdio/gpu/fputs.cpp
index 7a08244bc26e..780a40d1eb05 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fputs.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fputs.cpp
@@ -7,13 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fputs.h"
-#include "src/__support/CPP/string_view.h"
-#include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
-#include "src/stdio/gpu/file.h"
 
 #include "hdr/stdio_macros.h" // for EOF.
 #include "hdr/types/FILE.h"
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/common.h"
+#include "src/stdio/gpu/file.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fread.cpp b/external/llvm-project/libc/src/stdio/gpu/fread.cpp
index 57fa5b6111ff..5619b54b8890 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fread.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fread.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fread.h"
-#include "src/__support/macros/config.h"
-#include "src/stdio/gpu/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
+#include "src/stdio/gpu/file.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fseek.cpp b/external/llvm-project/libc/src/stdio/gpu/fseek.cpp
index 5ada5de50db8..468d65ff77e2 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fseek.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fseek.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fseek.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/ftell.cpp b/external/llvm-project/libc/src/stdio/gpu/ftell.cpp
index 04bb0ddfa06a..7ee33d7b05d9 100644
--- a/external/llvm-project/libc/src/stdio/gpu/ftell.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/ftell.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/ftell.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/fwrite.cpp b/external/llvm-project/libc/src/stdio/gpu/fwrite.cpp
index 6c548690039a..73c265158103 100644
--- a/external/llvm-project/libc/src/stdio/gpu/fwrite.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/fwrite.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fwrite.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/getc.cpp b/external/llvm-project/libc/src/stdio/gpu/getc.cpp
index 9c329749ee1f..6dfb5cef0ad9 100644
--- a/external/llvm-project/libc/src/stdio/gpu/getc.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/getc.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/getc.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/stdio_macros.h" // for EOF.
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/getchar.cpp b/external/llvm-project/libc/src/stdio/gpu/getchar.cpp
index d99b97b5c5a0..7bf561fe6a90 100644
--- a/external/llvm-project/libc/src/stdio/gpu/getchar.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/getchar.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/getchar.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/stdio_macros.h" // for EOF and stdin.
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/printf.cpp b/external/llvm-project/libc/src/stdio/gpu/printf.cpp
index be1885fd6801..8a9174d7397a 100644
--- a/external/llvm-project/libc/src/stdio/gpu/printf.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/printf.cpp
@@ -10,7 +10,8 @@
 
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/arg_list.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/stdio/gpu/vfprintf_utils.h"
 
 #include <stdarg.h>
diff --git a/external/llvm-project/libc/src/stdio/gpu/putc.cpp b/external/llvm-project/libc/src/stdio/gpu/putc.cpp
index f8ae98d26cf0..da80d6c35a53 100644
--- a/external/llvm-project/libc/src/stdio/gpu/putc.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/putc.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/putc.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/stdio_macros.h" // for EOF.
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/putchar.cpp b/external/llvm-project/libc/src/stdio/gpu/putchar.cpp
index c49b02e2f1f1..da2d0a837619 100644
--- a/external/llvm-project/libc/src/stdio/gpu/putchar.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/putchar.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/putchar.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/stdio_macros.h" // for EOF and stdout.
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/puts.cpp b/external/llvm-project/libc/src/stdio/gpu/puts.cpp
index fc252abe52d1..9b4ae661b148 100644
--- a/external/llvm-project/libc/src/stdio/gpu/puts.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/puts.cpp
@@ -7,19 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/puts.h"
-#include "src/__support/CPP/string_view.h"
-#include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
-#include "src/stdio/gpu/file.h"
 
 #include "hdr/stdio_macros.h" // for EOF and stdout.
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/common.h"
+#include "src/stdio/gpu/file.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, puts, (const char *__restrict str)) {
   cpp::string_view str_view(str);
-  auto written = file::write_impl<LIBC_WRITE_TO_STDOUT_NEWLINE>(stdout, str,
-                                                               str_view.size());
+  auto written = file::write_impl<LIBC_WRITE_TO_STDOUT_NEWLINE>(
+      stdout, str, str_view.size());
   if (written != str_view.size() + 1)
     return EOF;
   return 0;
diff --git a/external/llvm-project/libc/src/stdio/gpu/remove.cpp b/external/llvm-project/libc/src/stdio/gpu/remove.cpp
index 4bfb5d3fe8ed..3cd72745cb6d 100644
--- a/external/llvm-project/libc/src/stdio/gpu/remove.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/remove.cpp
@@ -7,10 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/remove.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
+#include "src/string/string_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/rename.cpp b/external/llvm-project/libc/src/stdio/gpu/rename.cpp
index 589bf2456c8d..d7c71fc82682 100644
--- a/external/llvm-project/libc/src/stdio/gpu/rename.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/rename.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/rename.h"
-#include "src/__support/CPP/string_view.h"
-#include "src/__support/macros/config.h"
-#include "src/stdio/gpu/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
+#include "src/stdio/gpu/file.h"
+#include "src/string/string_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/stderr.cpp b/external/llvm-project/libc/src/stdio/gpu/stderr.cpp
index efbc3b407d60..d0fa1aeb54ee 100644
--- a/external/llvm-project/libc/src/stdio/gpu/stderr.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/stderr.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/types/FILE.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 static struct {
diff --git a/external/llvm-project/libc/src/stdio/gpu/stdin.cpp b/external/llvm-project/libc/src/stdio/gpu/stdin.cpp
index 66618fd0b4db..fef4c3e8f2df 100644
--- a/external/llvm-project/libc/src/stdio/gpu/stdin.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/stdin.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/types/FILE.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 static struct {
diff --git a/external/llvm-project/libc/src/stdio/gpu/stdout.cpp b/external/llvm-project/libc/src/stdio/gpu/stdout.cpp
index e3869f9e26b7..e458dccf917d 100644
--- a/external/llvm-project/libc/src/stdio/gpu/stdout.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/stdout.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/types/FILE.h"
-#include "src/__support/macros/config.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 static struct {
diff --git a/external/llvm-project/libc/src/stdio/gpu/ungetc.cpp b/external/llvm-project/libc/src/stdio/gpu/ungetc.cpp
index fd1db467975e..fadd1d7db13c 100644
--- a/external/llvm-project/libc/src/stdio/gpu/ungetc.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/ungetc.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/ungetc.h"
-#include "file.h"
-#include "src/__support/macros/config.h"
 
+#include "file.h"
 #include "hdr/types/FILE.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/vfprintf.cpp b/external/llvm-project/libc/src/stdio/gpu/vfprintf.cpp
index c92685f48c72..16a5ed4e47d1 100644
--- a/external/llvm-project/libc/src/stdio/gpu/vfprintf.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/vfprintf.cpp
@@ -11,7 +11,7 @@
 #include "hdr/types/FILE.h"
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/arg_list.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/common.h"
 #include "src/stdio/gpu/vfprintf_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/gpu/vfprintf_utils.h b/external/llvm-project/libc/src/stdio/gpu/vfprintf_utils.h
index df157d36c516..6df46489d6c9 100644
--- a/external/llvm-project/libc/src/stdio/gpu/vfprintf_utils.h
+++ b/external/llvm-project/libc/src/stdio/gpu/vfprintf_utils.h
@@ -10,7 +10,6 @@
 #include "src/__support/GPU/utils.h"
 #include "src/__support/RPC/rpc_client.h"
 #include "src/__support/arg_list.h"
-#include "src/__support/macros/config.h"
 #include "src/stdio/gpu/file.h"
 #include "src/string/string_utils.h"
 
diff --git a/external/llvm-project/libc/src/stdio/gpu/vprintf.cpp b/external/llvm-project/libc/src/stdio/gpu/vprintf.cpp
index 54012f307184..65f5dbfe7c76 100644
--- a/external/llvm-project/libc/src/stdio/gpu/vprintf.cpp
+++ b/external/llvm-project/libc/src/stdio/gpu/vprintf.cpp
@@ -10,7 +10,7 @@
 
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/arg_list.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/common.h"
 #include "src/stdio/gpu/vfprintf_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/linux/fdopen.cpp b/external/llvm-project/libc/src/stdio/linux/fdopen.cpp
index 7d72fdc88e9f..5623f06b7cff 100644
--- a/external/llvm-project/libc/src/stdio/linux/fdopen.cpp
+++ b/external/llvm-project/libc/src/stdio/linux/fdopen.cpp
@@ -9,8 +9,8 @@
 #include "src/stdio/fdopen.h"
 
 #include "src/__support/File/linux/file.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/linux/remove.cpp b/external/llvm-project/libc/src/stdio/linux/remove.cpp
index dbb4491d0e6c..ac755db0bc78 100644
--- a/external/llvm-project/libc/src/stdio/linux/remove.cpp
+++ b/external/llvm-project/libc/src/stdio/linux/remove.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h" // For AT_* macros.
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/linux/rename.cpp b/external/llvm-project/libc/src/stdio/linux/rename.cpp
index fbcb29be48f4..426c8698e557 100644
--- a/external/llvm-project/libc/src/stdio/linux/rename.cpp
+++ b/external/llvm-project/libc/src/stdio/linux/rename.cpp
@@ -10,8 +10,8 @@
 #include "hdr/fcntl_macros.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/math/nvptx/exp.cpp b/external/llvm-project/libc/src/stdio/perror.h
similarity index 63%
rename from external/llvm-project/libc/src/math/nvptx/exp.cpp
rename to external/llvm-project/libc/src/stdio/perror.h
index dcbadcf2ae8d..bf8d0af1df5d 100644
--- a/external/llvm-project/libc/src/math/nvptx/exp.cpp
+++ b/external/llvm-project/libc/src/stdio/perror.h
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU exp function ----------------------------===//
+//===-- Implementation header of perror -------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/exp.h"
-#include "src/__support/common.h"
+#ifndef LLVM_LIBC_SRC_STDIO_PERROR_H
+#define LLVM_LIBC_SRC_STDIO_PERROR_H
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, exp, (double x)) { return __nv_exp(x); }
+void perror(const char *s);
 
 } // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STDIO_PERROR_H
diff --git a/external/llvm-project/libc/src/stdio/printf_core/parser.h b/external/llvm-project/libc/src/stdio/printf_core/parser.h
index 89556f1a9e5f..cef9b1ae58fa 100644
--- a/external/llvm-project/libc/src/stdio/printf_core/parser.h
+++ b/external/llvm-project/libc/src/stdio/printf_core/parser.h
@@ -25,7 +25,7 @@
 #include "src/__support/fixed_point/fx_rep.h"
 #endif // LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT
 #ifndef LIBC_COPT_PRINTF_DISABLE_STRERROR
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #endif // LIBC_COPT_PRINTF_DISABLE_STRERROR
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdio/setbuf.cpp b/external/llvm-project/libc/src/stdio/setbuf.cpp
index f3db97de5837..fcc6df12ddb0 100644
--- a/external/llvm-project/libc/src/stdio/setbuf.cpp
+++ b/external/llvm-project/libc/src/stdio/setbuf.cpp
@@ -9,8 +9,8 @@
 #include "src/stdio/setbuf.h"
 #include "hdr/stdio_macros.h"
 #include "src/__support/File/file.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdio/setvbuf.cpp b/external/llvm-project/libc/src/stdio/setvbuf.cpp
index 0a6b8cacb59c..9fc6cb040233 100644
--- a/external/llvm-project/libc/src/stdio/setvbuf.cpp
+++ b/external/llvm-project/libc/src/stdio/setvbuf.cpp
@@ -10,8 +10,8 @@
 #include "src/__support/File/file.h"
 
 #include "hdr/types/FILE.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/stdlib/atof.cpp b/external/llvm-project/libc/src/stdlib/atof.cpp
index 18a65c67705d..d0d8d211dea8 100644
--- a/external/llvm-project/libc/src/stdlib/atof.cpp
+++ b/external/llvm-project/libc/src/stdlib/atof.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/atof.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/atoi.cpp b/external/llvm-project/libc/src/stdlib/atoi.cpp
index 9e46b53b1aa0..420bbc8143d5 100644
--- a/external/llvm-project/libc/src/stdlib/atoi.cpp
+++ b/external/llvm-project/libc/src/stdlib/atoi.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/atoi.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/atol.cpp b/external/llvm-project/libc/src/stdlib/atol.cpp
index 7f3414a4afdd..e1110ffa449b 100644
--- a/external/llvm-project/libc/src/stdlib/atol.cpp
+++ b/external/llvm-project/libc/src/stdlib/atol.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/atol.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/atoll.cpp b/external/llvm-project/libc/src/stdlib/atoll.cpp
index 4f1a02ad8315..063e817f9b79 100644
--- a/external/llvm-project/libc/src/stdlib/atoll.cpp
+++ b/external/llvm-project/libc/src/stdlib/atoll.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/atoll.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtod.cpp b/external/llvm-project/libc/src/stdlib/strtod.cpp
index 2c6819163aa4..deb2390c7fcd 100644
--- a/external/llvm-project/libc/src/stdlib/strtod.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtod.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtod.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtod_l.cpp b/external/llvm-project/libc/src/stdlib/strtod_l.cpp
index 247314398315..ad333b32d240 100644
--- a/external/llvm-project/libc/src/stdlib/strtod_l.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtod_l.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtod_l.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtof.cpp b/external/llvm-project/libc/src/stdlib/strtof.cpp
index 351bf64ad4f7..fc52dc85ffc5 100644
--- a/external/llvm-project/libc/src/stdlib/strtof.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtof.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtof.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtof_l.cpp b/external/llvm-project/libc/src/stdlib/strtof_l.cpp
index d54efa66e084..c6e03ff51fa2 100644
--- a/external/llvm-project/libc/src/stdlib/strtof_l.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtof_l.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtof_l.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtol.cpp b/external/llvm-project/libc/src/stdlib/strtol.cpp
index 77f8712d7c13..42db36b2052b 100644
--- a/external/llvm-project/libc/src/stdlib/strtol.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtol.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtol.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtol_l.cpp b/external/llvm-project/libc/src/stdlib/strtol_l.cpp
index f94aff1a0d7b..497a4403eff4 100644
--- a/external/llvm-project/libc/src/stdlib/strtol_l.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtol_l.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtol_l.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtold.cpp b/external/llvm-project/libc/src/stdlib/strtold.cpp
index 88d29c9f3627..44046c2c6f61 100644
--- a/external/llvm-project/libc/src/stdlib/strtold.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtold.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtold.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtold_l.cpp b/external/llvm-project/libc/src/stdlib/strtold_l.cpp
index d0c57f50246b..c3af30a1b9ec 100644
--- a/external/llvm-project/libc/src/stdlib/strtold_l.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtold_l.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtold_l.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtoll.cpp b/external/llvm-project/libc/src/stdlib/strtoll.cpp
index 8d1b3efdcf87..c1dca13112e0 100644
--- a/external/llvm-project/libc/src/stdlib/strtoll.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtoll.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtoll.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtoll_l.cpp b/external/llvm-project/libc/src/stdlib/strtoll_l.cpp
index e82971d59c48..6f30d7794c5c 100644
--- a/external/llvm-project/libc/src/stdlib/strtoll_l.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtoll_l.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtoll_l.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtoul.cpp b/external/llvm-project/libc/src/stdlib/strtoul.cpp
index 1d832318c448..d26ca5e5a10a 100644
--- a/external/llvm-project/libc/src/stdlib/strtoul.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtoul.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtoul.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtoul_l.cpp b/external/llvm-project/libc/src/stdlib/strtoul_l.cpp
index 74fce00a0ac3..9a875ddee902 100644
--- a/external/llvm-project/libc/src/stdlib/strtoul_l.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtoul_l.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtoul_l.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtoull.cpp b/external/llvm-project/libc/src/stdlib/strtoull.cpp
index dba22611cfb0..8f929f577311 100644
--- a/external/llvm-project/libc/src/stdlib/strtoull.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtoull.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtoull.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/stdlib/strtoull_l.cpp b/external/llvm-project/libc/src/stdlib/strtoull_l.cpp
index 2ea8a43a40ef..9eb056b0e59b 100644
--- a/external/llvm-project/libc/src/stdlib/strtoull_l.cpp
+++ b/external/llvm-project/libc/src/stdlib/strtoull_l.cpp
@@ -8,9 +8,9 @@
 
 #include "src/stdlib/strtoull_l.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/string/CMakeLists.txt b/external/llvm-project/libc/src/string/CMakeLists.txt
index 2c607bf8ea89..8784bc3750cb 100644
--- a/external/llvm-project/libc/src/string/CMakeLists.txt
+++ b/external/llvm-project/libc/src/string/CMakeLists.txt
@@ -15,10 +15,8 @@ add_header_library(
   HDRS
     string_utils.h
   DEPENDS
-    .memory_utils.inline_bzero
-    .memory_utils.inline_memcpy
     libc.hdr.types.size_t
-    libc.include.stdlib
+    libc.hdr.limits_macros
     libc.src.__support.CPP.bitset
     libc.src.__support.CPP.type_traits
     libc.src.__support.common
@@ -89,7 +87,6 @@ add_entrypoint_object(
   HDRS
     stpcpy.h
   DEPENDS
-    .mempcpy
     .string_utils
 )
 
@@ -110,7 +107,6 @@ add_entrypoint_object(
   HDRS
     strcat.h
   DEPENDS
-    .strcpy
     .string_utils
     libc.include.llvm-libc-types.size_t
 )
@@ -267,7 +263,6 @@ add_entrypoint_object(
   HDRS
     strncat.h
   DEPENDS
-    .strncpy
     .string_utils
     libc.include.llvm-libc-types.size_t
 )
@@ -442,7 +437,7 @@ function(add_memcmp memcmp_name)
   )
 endfunction()
 
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86_64})
   add_memcmp(memcmp_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_memcmp(memcmp_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_memcmp(memcmp_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
@@ -474,7 +469,7 @@ function(add_memcpy memcpy_name)
   )
 endfunction()
 
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86_64})
   add_memcpy(memcpy_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_memcpy(memcpy_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_memcpy(memcpy_x86_64_opt_avx    COMPILE_OPTIONS -march=sandybridge    REQUIRE AVX)
@@ -510,7 +505,7 @@ function(add_memmove memmove_name)
   )
 endfunction()
 
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86_64})
   add_memmove(memmove_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_memmove(memmove_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_memmove(memmove_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
@@ -544,7 +539,7 @@ function(add_memset memset_name)
   )
 endfunction()
 
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86_64})
   add_memset(memset_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_memset(memset_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_memset(memset_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
diff --git a/external/llvm-project/libc/src/string/memccpy.cpp b/external/llvm-project/libc/src/string/memccpy.cpp
index ae90cf9370d4..d5654fc5e46a 100644
--- a/external/llvm-project/libc/src/string/memccpy.cpp
+++ b/external/llvm-project/libc/src/string/memccpy.cpp
@@ -10,6 +10,7 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include <stddef.h> // For size_t.
 
 namespace LIBC_NAMESPACE_DECL {
@@ -17,6 +18,10 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(void *, memccpy,
                    (void *__restrict dest, const void *__restrict src, int c,
                     size_t count)) {
+  if (count) {
+    LIBC_CRASH_ON_NULLPTR(dest);
+    LIBC_CRASH_ON_NULLPTR(src);
+  }
   unsigned char end = static_cast<unsigned char>(c);
   const unsigned char *uc_src = static_cast<const unsigned char *>(src);
   unsigned char *uc_dest = static_cast<unsigned char *>(dest);
diff --git a/external/llvm-project/libc/src/string/memchr.cpp b/external/llvm-project/libc/src/string/memchr.cpp
index ba52f14afa9d..ccdc262837cd 100644
--- a/external/llvm-project/libc/src/string/memchr.cpp
+++ b/external/llvm-project/libc/src/string/memchr.cpp
@@ -8,6 +8,7 @@
 
 #include "src/string/memchr.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/string_utils.h"
 
 #include "src/__support/common.h"
@@ -17,6 +18,8 @@ namespace LIBC_NAMESPACE_DECL {
 
 // TODO: Look at performance benefits of comparing words.
 LLVM_LIBC_FUNCTION(void *, memchr, (const void *src, int c, size_t n)) {
+  if (n)
+    LIBC_CRASH_ON_NULLPTR(src);
   return internal::find_first_character(
       reinterpret_cast<const unsigned char *>(src),
       static_cast<unsigned char>(c), n);
diff --git a/external/llvm-project/libc/src/string/memcmp.cpp b/external/llvm-project/libc/src/string/memcmp.cpp
index 68996fb78723..d2f67f0478ef 100644
--- a/external/llvm-project/libc/src/string/memcmp.cpp
+++ b/external/llvm-project/libc/src/string/memcmp.cpp
@@ -8,6 +8,7 @@
 
 #include "src/string/memcmp.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_memcmp.h"
 
 #include <stddef.h> // size_t
@@ -16,6 +17,10 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, memcmp,
                    (const void *lhs, const void *rhs, size_t count)) {
+  if (count) {
+    LIBC_CRASH_ON_NULLPTR(lhs);
+    LIBC_CRASH_ON_NULLPTR(rhs);
+  }
   return inline_memcmp(lhs, rhs, count);
 }
 
diff --git a/external/llvm-project/libc/src/string/memcpy.cpp b/external/llvm-project/libc/src/string/memcpy.cpp
index 0eb7f2c170e0..4d4ff4dd3872 100644
--- a/external/llvm-project/libc/src/string/memcpy.cpp
+++ b/external/llvm-project/libc/src/string/memcpy.cpp
@@ -9,6 +9,7 @@
 #include "src/string/memcpy.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_memcpy.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -16,6 +17,10 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(void *, memcpy,
                    (void *__restrict dst, const void *__restrict src,
                     size_t size)) {
+  if (size) {
+    LIBC_CRASH_ON_NULLPTR(dst);
+    LIBC_CRASH_ON_NULLPTR(src);
+  }
   inline_memcpy(dst, src, size);
   return dst;
 }
diff --git a/external/llvm-project/libc/src/string/memmove.cpp b/external/llvm-project/libc/src/string/memmove.cpp
index 26a8c4187f3d..04ed51b84f8f 100644
--- a/external/llvm-project/libc/src/string/memmove.cpp
+++ b/external/llvm-project/libc/src/string/memmove.cpp
@@ -8,6 +8,7 @@
 
 #include "src/string/memmove.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_memcpy.h"
 #include "src/string/memory_utils/inline_memmove.h"
 #include <stddef.h> // size_t
@@ -16,6 +17,10 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void *, memmove,
                    (void *dst, const void *src, size_t count)) {
+  if (count) {
+    LIBC_CRASH_ON_NULLPTR(dst);
+    LIBC_CRASH_ON_NULLPTR(src);
+  }
   // Memmove may handle some small sizes as efficiently as inline_memcpy.
   // For these sizes we may not do is_disjoint check.
   // This both avoids additional code for the most frequent smaller sizes
diff --git a/external/llvm-project/libc/src/string/mempcpy.cpp b/external/llvm-project/libc/src/string/mempcpy.cpp
index 09392ceb966d..b6a9721960ce 100644
--- a/external/llvm-project/libc/src/string/mempcpy.cpp
+++ b/external/llvm-project/libc/src/string/mempcpy.cpp
@@ -8,6 +8,7 @@
 
 #include "src/string/mempcpy.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_memcpy.h"
 
 #include "src/__support/common.h"
@@ -18,6 +19,10 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(void *, mempcpy,
                    (void *__restrict dst, const void *__restrict src,
                     size_t count)) {
+  if (count) {
+    LIBC_CRASH_ON_NULLPTR(dst);
+    LIBC_CRASH_ON_NULLPTR(src);
+  }
   inline_memcpy(dst, src, count);
   return reinterpret_cast<char *>(dst) + count;
 }
diff --git a/external/llvm-project/libc/src/string/memrchr.cpp b/external/llvm-project/libc/src/string/memrchr.cpp
index d665e225bbb7..d5c843c733b3 100644
--- a/external/llvm-project/libc/src/string/memrchr.cpp
+++ b/external/llvm-project/libc/src/string/memrchr.cpp
@@ -9,11 +9,16 @@
 #include "src/string/memrchr.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void *, memrchr, (const void *src, int c, size_t n)) {
+
+  if (n)
+    LIBC_CRASH_ON_NULLPTR(src);
+
   const unsigned char *str = reinterpret_cast<const unsigned char *>(src);
   const unsigned char ch = static_cast<unsigned char>(c);
   for (; n != 0; --n) {
diff --git a/external/llvm-project/libc/src/string/memset.cpp b/external/llvm-project/libc/src/string/memset.cpp
index c2868afa9103..a0b96b3cf88b 100644
--- a/external/llvm-project/libc/src/string/memset.cpp
+++ b/external/llvm-project/libc/src/string/memset.cpp
@@ -9,11 +9,15 @@
 #include "src/string/memset.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_memset.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) {
+  if (count)
+    LIBC_CRASH_ON_NULLPTR(dst);
+
   inline_memset(dst, static_cast<uint8_t>(value), count);
   return dst;
 }
diff --git a/external/llvm-project/libc/src/string/stpcpy.cpp b/external/llvm-project/libc/src/string/stpcpy.cpp
index 979edd72c1f1..48c0db950ace 100644
--- a/external/llvm-project/libc/src/string/stpcpy.cpp
+++ b/external/llvm-project/libc/src/string/stpcpy.cpp
@@ -8,7 +8,6 @@
 
 #include "src/string/stpcpy.h"
 #include "src/__support/macros/config.h"
-#include "src/string/mempcpy.h"
 #include "src/string/string_utils.h"
 
 #include "src/__support/common.h"
@@ -18,8 +17,8 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(char *, stpcpy,
                    (char *__restrict dest, const char *__restrict src)) {
   size_t size = internal::string_length(src) + 1;
-  char *result =
-      reinterpret_cast<char *>(LIBC_NAMESPACE::mempcpy(dest, src, size));
+  __builtin_memcpy(dest, src, size);
+  char *result = dest + size;
 
   if (result != nullptr)
     return result - 1;
diff --git a/external/llvm-project/libc/src/string/stpncpy.cpp b/external/llvm-project/libc/src/string/stpncpy.cpp
index d2a6e0474982..47bf4c6e86fb 100644
--- a/external/llvm-project/libc/src/string/stpncpy.cpp
+++ b/external/llvm-project/libc/src/string/stpncpy.cpp
@@ -8,6 +8,7 @@
 
 #include "src/string/stpncpy.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_bzero.h"
 
 #include "src/__support/common.h"
@@ -17,6 +18,10 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(char *, stpncpy,
                    (char *__restrict dest, const char *__restrict src,
                     size_t n)) {
+  if (n) {
+    LIBC_CRASH_ON_NULLPTR(dest);
+    LIBC_CRASH_ON_NULLPTR(src);
+  }
   size_t i;
   // Copy up until \0 is found.
   for (i = 0; i < n && src[i] != '\0'; ++i)
diff --git a/external/llvm-project/libc/src/string/strcasestr.cpp b/external/llvm-project/libc/src/string/strcasestr.cpp
index 1da1e3f02507..de8e4bec7fe0 100644
--- a/external/llvm-project/libc/src/string/strcasestr.cpp
+++ b/external/llvm-project/libc/src/string/strcasestr.cpp
@@ -11,6 +11,7 @@
 #include "src/__support/common.h"
 #include "src/__support/ctype_utils.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_strstr.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -23,6 +24,9 @@ LLVM_LIBC_FUNCTION(char *, strcasestr,
     return LIBC_NAMESPACE::internal::tolower(a) -
            LIBC_NAMESPACE::internal::tolower(b);
   };
+
+  LIBC_CRASH_ON_NULLPTR(haystack);
+  LIBC_CRASH_ON_NULLPTR(needle);
   return inline_strstr(haystack, needle, case_cmp);
 }
 
diff --git a/external/llvm-project/libc/src/string/strcat.cpp b/external/llvm-project/libc/src/string/strcat.cpp
index 0eb189ce204f..7dce6d15a65c 100644
--- a/external/llvm-project/libc/src/string/strcat.cpp
+++ b/external/llvm-project/libc/src/string/strcat.cpp
@@ -8,7 +8,7 @@
 
 #include "src/string/strcat.h"
 #include "src/__support/macros/config.h"
-#include "src/string/strcpy.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/string_utils.h"
 
 #include "src/__support/common.h"
@@ -17,10 +17,14 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(char *, strcat,
                    (char *__restrict dest, const char *__restrict src)) {
+  LIBC_CRASH_ON_NULLPTR(dest);
+  LIBC_CRASH_ON_NULLPTR(src);
   size_t dest_length = internal::string_length(dest);
-  size_t src_length = internal::string_length(src);
-  LIBC_NAMESPACE::strcpy(dest + dest_length, src);
-  dest[dest_length + src_length] = '\0';
+  size_t i;
+  for (i = 0; src[i] != '\0'; ++i)
+    dest[dest_length + i] = src[i];
+
+  dest[dest_length + i] = '\0';
   return dest;
 }
 
diff --git a/external/llvm-project/libc/src/string/strcoll.cpp b/external/llvm-project/libc/src/string/strcoll.cpp
index eeb2c79e3807..aa08f7194c9e 100644
--- a/external/llvm-project/libc/src/string/strcoll.cpp
+++ b/external/llvm-project/libc/src/string/strcoll.cpp
@@ -10,11 +10,14 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 // TODO: Add support for locales.
 LLVM_LIBC_FUNCTION(int, strcoll, (const char *left, const char *right)) {
+  LIBC_CRASH_ON_NULLPTR(left);
+  LIBC_CRASH_ON_NULLPTR(right);
   for (; *left && *left == *right; ++left, ++right)
     ;
   return static_cast<int>(*left) - static_cast<int>(*right);
diff --git a/external/llvm-project/libc/src/string/strcoll_l.cpp b/external/llvm-project/libc/src/string/strcoll_l.cpp
index f664a3c7c03f..e820efa564a3 100644
--- a/external/llvm-project/libc/src/string/strcoll_l.cpp
+++ b/external/llvm-project/libc/src/string/strcoll_l.cpp
@@ -10,12 +10,15 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 // TODO: Add support for locales.
 LLVM_LIBC_FUNCTION(int, strcoll_l,
                    (const char *left, const char *right, locale_t)) {
+  LIBC_CRASH_ON_NULLPTR(left);
+  LIBC_CRASH_ON_NULLPTR(right);
   for (; *left && *left == *right; ++left, ++right)
     ;
   return static_cast<int>(*left) - static_cast<int>(*right);
diff --git a/external/llvm-project/libc/src/string/strcpy.cpp b/external/llvm-project/libc/src/string/strcpy.cpp
index 60b73ab3aa82..2013593db24a 100644
--- a/external/llvm-project/libc/src/string/strcpy.cpp
+++ b/external/llvm-project/libc/src/string/strcpy.cpp
@@ -8,6 +8,7 @@
 
 #include "src/string/strcpy.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_memcpy.h"
 #include "src/string/string_utils.h"
 
@@ -17,6 +18,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(char *, strcpy,
                    (char *__restrict dest, const char *__restrict src)) {
+  LIBC_CRASH_ON_NULLPTR(dest);
   size_t size = internal::string_length(src) + 1;
   inline_memcpy(dest, src, size);
   return dest;
diff --git a/external/llvm-project/libc/src/string/strdup.cpp b/external/llvm-project/libc/src/string/strdup.cpp
index 4cf4173a27bf..dab0ab4288c9 100644
--- a/external/llvm-project/libc/src/string/strdup.cpp
+++ b/external/llvm-project/libc/src/string/strdup.cpp
@@ -8,8 +8,8 @@
 
 #include "src/string/strdup.h"
 #include "hdr/stdlib_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/string/allocating_string_utils.h"
 #include "src/string/memory_utils/inline_memcpy.h"
 
diff --git a/external/llvm-project/libc/src/string/string_utils.h b/external/llvm-project/libc/src/string/string_utils.h
index e4659f65c93e..4f56263fce8e 100644
--- a/external/llvm-project/libc/src/string/string_utils.h
+++ b/external/llvm-project/libc/src/string/string_utils.h
@@ -14,19 +14,19 @@
 #ifndef LLVM_LIBC_SRC_STRING_STRING_UTILS_H
 #define LLVM_LIBC_SRC_STRING_STRING_UTILS_H
 
+#include "hdr/limits_macros.h"
 #include "hdr/types/size_t.h"
 #include "src/__support/CPP/bitset.h"
 #include "src/__support/CPP/type_traits.h" // cpp::is_same_v
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#include "src/string/memory_utils/inline_bzero.h"
-#include "src/string/memory_utils/inline_memcpy.h"
 
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 
 template <typename Word> LIBC_INLINE constexpr Word repeat_byte(Word byte) {
-  constexpr size_t BITS_IN_BYTE = 8;
+  static_assert(CHAR_BIT == 8, "repeat_byte assumes a byte is 8 bits.");
+  constexpr size_t BITS_IN_BYTE = CHAR_BIT;
   constexpr size_t BYTE_MASK = 0xff;
   Word result = 0;
   byte = byte & BYTE_MASK;
@@ -90,12 +90,11 @@ template <typename T> LIBC_INLINE size_t string_length(const T *src) {
   // string a block at a time.
   if constexpr (cpp::is_same_v<T, char>)
     return string_length_wide_read<unsigned int>(src);
-#else
+#endif
   size_t length;
   for (length = 0; *src; ++src, ++length)
     ;
   return length;
-#endif
 }
 
 template <typename Word>
@@ -189,8 +188,7 @@ LIBC_INLINE char *string_token(char *__restrict src,
   if (LIBC_UNLIKELY(src == nullptr && ((src = *saveptr) == nullptr)))
     return nullptr;
 
-  static_assert(sizeof(char) == sizeof(cpp::byte),
-                "bitset of 256 assumes char is 8 bits");
+  static_assert(CHAR_BIT == 8, "bitset of 256 assumes char is 8 bits");
   cpp::bitset<256> delimiter_set;
   for (; *delimiter_string != '\0'; ++delimiter_string)
     delimiter_set.set(static_cast<size_t>(*delimiter_string));
@@ -220,7 +218,7 @@ LIBC_INLINE size_t strlcpy(char *__restrict dst, const char *__restrict src,
   if (!size)
     return len;
   size_t n = len < size - 1 ? len : size - 1;
-  inline_memcpy(dst, src, n);
+  __builtin_memcpy(dst, src, n);
   dst[n] = '\0';
   return len;
 }
diff --git a/external/llvm-project/libc/src/string/strlen.cpp b/external/llvm-project/libc/src/string/strlen.cpp
index ff7ab14dd313..234edb81d4c8 100644
--- a/external/llvm-project/libc/src/string/strlen.cpp
+++ b/external/llvm-project/libc/src/string/strlen.cpp
@@ -8,6 +8,7 @@
 
 #include "src/string/strlen.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/string_utils.h"
 
 #include "src/__support/common.h"
@@ -17,6 +18,7 @@ namespace LIBC_NAMESPACE_DECL {
 // TODO: investigate the performance of this function.
 // There might be potential for compiler optimization.
 LLVM_LIBC_FUNCTION(size_t, strlen, (const char *src)) {
+  LIBC_CRASH_ON_NULLPTR(src);
   return internal::string_length(src);
 }
 
diff --git a/external/llvm-project/libc/src/string/strncat.cpp b/external/llvm-project/libc/src/string/strncat.cpp
index 221881f93c47..6d8bb6960748 100644
--- a/external/llvm-project/libc/src/string/strncat.cpp
+++ b/external/llvm-project/libc/src/string/strncat.cpp
@@ -8,8 +8,8 @@
 
 #include "src/string/strncat.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/string_utils.h"
-#include "src/string/strncpy.h"
 
 #include "src/__support/common.h"
 
@@ -18,11 +18,16 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(char *, strncat,
                    (char *__restrict dest, const char *__restrict src,
                     size_t count)) {
-  size_t src_length = internal::string_length(src);
-  size_t copy_amount = src_length > count ? count : src_length;
+  if (count) {
+    LIBC_CRASH_ON_NULLPTR(dest);
+    LIBC_CRASH_ON_NULLPTR(src);
+  }
   size_t dest_length = internal::string_length(dest);
-  LIBC_NAMESPACE::strncpy(dest + dest_length, src, copy_amount);
-  dest[dest_length + copy_amount] = '\0';
+  size_t i;
+  for (i = 0; i < count && src[i] != '\0'; ++i)
+    dest[dest_length + i] = src[i];
+
+  dest[dest_length + i] = '\0';
   return dest;
 }
 
diff --git a/external/llvm-project/libc/src/string/strncmp.cpp b/external/llvm-project/libc/src/string/strncmp.cpp
index 16d4601fe845..f21fd769f394 100644
--- a/external/llvm-project/libc/src/string/strncmp.cpp
+++ b/external/llvm-project/libc/src/string/strncmp.cpp
@@ -10,6 +10,7 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_strcmp.h"
 
 #include <stddef.h>
@@ -18,6 +19,10 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, strncmp,
                    (const char *left, const char *right, size_t n)) {
+  if (n) {
+    LIBC_CRASH_ON_NULLPTR(left);
+    LIBC_CRASH_ON_NULLPTR(right);
+  }
   auto comp = [](char l, char r) -> int { return l - r; };
   return inline_strncmp(left, right, n, comp);
 }
diff --git a/external/llvm-project/libc/src/string/strncpy.cpp b/external/llvm-project/libc/src/string/strncpy.cpp
index 4976ad94708c..e271009502f2 100644
--- a/external/llvm-project/libc/src/string/strncpy.cpp
+++ b/external/llvm-project/libc/src/string/strncpy.cpp
@@ -10,6 +10,7 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include <stddef.h> // For size_t.
 
 namespace LIBC_NAMESPACE_DECL {
@@ -17,6 +18,10 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(char *, strncpy,
                    (char *__restrict dest, const char *__restrict src,
                     size_t n)) {
+  if (n) {
+    LIBC_CRASH_ON_NULLPTR(dest);
+    LIBC_CRASH_ON_NULLPTR(src);
+  }
   size_t i = 0;
   // Copy up until \0 is found.
   for (; i < n && src[i] != '\0'; ++i)
diff --git a/external/llvm-project/libc/src/string/strsep.cpp b/external/llvm-project/libc/src/string/strsep.cpp
index 4c275122de52..41874b6af226 100644
--- a/external/llvm-project/libc/src/string/strsep.cpp
+++ b/external/llvm-project/libc/src/string/strsep.cpp
@@ -9,14 +9,19 @@
 #include "src/string/strsep.h"
 
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/string_utils.h"
 
+#include "src/__support/common.h"
+
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(char *, strsep,
                    (char **__restrict stringp, const char *__restrict delim)) {
+  LIBC_CRASH_ON_NULLPTR(stringp);
   if (!*stringp)
     return nullptr;
+  LIBC_CRASH_ON_NULLPTR(delim);
   return internal::string_token<false>(*stringp, delim, stringp);
 }
 
diff --git a/external/llvm-project/libc/src/string/strspn.cpp b/external/llvm-project/libc/src/string/strspn.cpp
index 66bb39903ab3..b205bedf80ca 100644
--- a/external/llvm-project/libc/src/string/strspn.cpp
+++ b/external/llvm-project/libc/src/string/strspn.cpp
@@ -11,11 +11,14 @@
 #include "src/__support/CPP/bitset.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include <stddef.h>
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(size_t, strspn, (const char *src, const char *segment)) {
+  LIBC_CRASH_ON_NULLPTR(src);
+  LIBC_CRASH_ON_NULLPTR(segment);
   const char *initial = src;
   cpp::bitset<256> bitset;
 
diff --git a/external/llvm-project/libc/src/string/strstr.cpp b/external/llvm-project/libc/src/string/strstr.cpp
index 5132f06ef53f..44797ef670b9 100644
--- a/external/llvm-project/libc/src/string/strstr.cpp
+++ b/external/llvm-project/libc/src/string/strstr.cpp
@@ -10,6 +10,7 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/memory_utils/inline_strstr.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -18,6 +19,8 @@ namespace LIBC_NAMESPACE_DECL {
 // improved upon using well known string matching algorithms.
 LLVM_LIBC_FUNCTION(char *, strstr, (const char *haystack, const char *needle)) {
   auto comp = [](char l, char r) -> int { return l - r; };
+  LIBC_CRASH_ON_NULLPTR(haystack);
+  LIBC_CRASH_ON_NULLPTR(needle);
   return inline_strstr(haystack, needle, comp);
 }
 
diff --git a/external/llvm-project/libc/src/strings/CMakeLists.txt b/external/llvm-project/libc/src/strings/CMakeLists.txt
index d312495ae8d9..ea1c932be048 100644
--- a/external/llvm-project/libc/src/strings/CMakeLists.txt
+++ b/external/llvm-project/libc/src/strings/CMakeLists.txt
@@ -8,7 +8,7 @@ function(add_bcmp bcmp_name)
   )
 endfunction()
 
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86_64})
   add_bcmp(bcmp_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_bcmp(bcmp_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_bcmp(bcmp_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
@@ -32,7 +32,7 @@ function(add_bzero bzero_name)
   )
 endfunction()
 
-if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
+if(${LIBC_TARGET_ARCHITECTURE_IS_X86_64})
   add_bzero(bzero_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_bzero(bzero_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_bzero(bzero_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
diff --git a/external/llvm-project/libc/src/strings/rindex.cpp b/external/llvm-project/libc/src/strings/rindex.cpp
index 1242e0faf85f..2540222ffb0e 100644
--- a/external/llvm-project/libc/src/strings/rindex.cpp
+++ b/external/llvm-project/libc/src/strings/rindex.cpp
@@ -10,11 +10,13 @@
 
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 #include "src/string/string_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(char *, rindex, (const char *src, int c)) {
+  LIBC_CRASH_ON_NULLPTR(src);
   return internal::strrchr_implementation(src, c);
 }
 
diff --git a/external/llvm-project/libc/src/sys/CMakeLists.txt b/external/llvm-project/libc/src/sys/CMakeLists.txt
index 9a73b80d35d2..0fa11e9eee69 100644
--- a/external/llvm-project/libc/src/sys/CMakeLists.txt
+++ b/external/llvm-project/libc/src/sys/CMakeLists.txt
@@ -13,3 +13,4 @@ add_subdirectory(utsname)
 add_subdirectory(wait)
 add_subdirectory(prctl)
 add_subdirectory(uio)
+add_subdirectory(ioctl)
diff --git a/external/llvm-project/libc/src/sys/auxv/linux/getauxval.cpp b/external/llvm-project/libc/src/sys/auxv/linux/getauxval.cpp
index 236fd25698f6..b50c5845bcc2 100644
--- a/external/llvm-project/libc/src/sys/auxv/linux/getauxval.cpp
+++ b/external/llvm-project/libc/src/sys/auxv/linux/getauxval.cpp
@@ -8,23 +8,29 @@
 
 #include "src/sys/auxv/getauxval.h"
 #include "config/app.h"
+#include "hdr/fcntl_macros.h"
+#include "src/__support/OSUtil/fcntl.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <linux/auxvec.h>
 
 // for guarded initialization
 #include "src/__support/threads/callonce.h"
 #include "src/__support/threads/linux/futex_word.h"
 
+// -----------------------------------------------------------------------------
+// TODO: This file should not include other public libc functions. Calling other
+// public libc functions is an antipattern within LLVM-libc. This needs to be
+// cleaned up. DO NOT COPY THIS.
+// -----------------------------------------------------------------------------
+
 // for mallocing the global auxv
 #include "src/sys/mman/mmap.h"
 #include "src/sys/mman/munmap.h"
 
 // for reading /proc/self/auxv
-#include "src/fcntl/open.h"
 #include "src/sys/prctl/prctl.h"
-#include "src/unistd/close.h"
 #include "src/unistd/read.h"
 
 // getauxval will work either with or without __cxa_atexit support.
@@ -60,17 +66,18 @@ class AuxvMMapGuard {
   constexpr static size_t AUXV_MMAP_SIZE = sizeof(AuxEntry) * MAX_AUXV_ENTRIES;
 
   AuxvMMapGuard()
-      : ptr(mmap(nullptr, AUXV_MMAP_SIZE, PROT_READ | PROT_WRITE,
-                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) {}
+      : ptr(LIBC_NAMESPACE::mmap(nullptr, AUXV_MMAP_SIZE,
+                                 PROT_READ | PROT_WRITE,
+                                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) {}
   ~AuxvMMapGuard() {
     if (ptr != MAP_FAILED)
-      munmap(ptr, AUXV_MMAP_SIZE);
+      LIBC_NAMESPACE::munmap(ptr, AUXV_MMAP_SIZE);
   }
   void submit_to_global() {
     // atexit may fail, we do not set it to global in that case.
     int ret = __cxa_atexit(
         [](void *) {
-          munmap(auxv, AUXV_MMAP_SIZE);
+          LIBC_NAMESPACE::munmap(auxv, AUXV_MMAP_SIZE);
           auxv = nullptr;
         },
         nullptr, nullptr);
@@ -90,10 +97,16 @@ class AuxvMMapGuard {
 
 class AuxvFdGuard {
 public:
-  AuxvFdGuard() : fd(open("/proc/self/auxv", O_RDONLY | O_CLOEXEC)) {}
+  AuxvFdGuard() {
+    auto result = internal::open("/proc/self/auxv", O_RDONLY | O_CLOEXEC);
+    if (!result.has_value())
+      fd = -1;
+
+    fd = result.value();
+  }
   ~AuxvFdGuard() {
     if (fd != -1)
-      close(fd);
+      internal::close(fd);
   }
   bool valid() const { return fd != -1; }
   int get() const { return fd; }
@@ -135,7 +148,8 @@ static void initialize_auxv_once(void) {
   bool error_detected = false;
   // Read until we use up all the available space or we finish reading the file.
   while (available_size != 0) {
-    ssize_t bytes_read = read(fd_guard.get(), buf, available_size);
+    ssize_t bytes_read =
+        LIBC_NAMESPACE::read(fd_guard.get(), buf, available_size);
     if (bytes_read <= 0) {
       if (libc_errno == EINTR)
         continue;
@@ -158,7 +172,7 @@ static AuxEntry read_entry(int fd) {
   size_t size = sizeof(AuxEntry);
   char *ptr = reinterpret_cast<char *>(&buf);
   while (size > 0) {
-    ssize_t ret = read(fd, ptr, size);
+    ssize_t ret = LIBC_NAMESPACE::read(fd, ptr, size);
     if (ret < 0) {
       if (libc_errno == EINTR)
         continue;
@@ -195,7 +209,8 @@ LLVM_LIBC_FUNCTION(unsigned long, getauxval, (unsigned long id)) {
     return search_auxv(app.auxv_ptr, id);
 
   static FutexWordType once_flag;
-  callonce(reinterpret_cast<CallOnceFlag *>(&once_flag), initialize_auxv_once);
+  LIBC_NAMESPACE::callonce(reinterpret_cast<CallOnceFlag *>(&once_flag),
+                           initialize_auxv_once);
   if (auxv != nullptr)
     return search_auxv(auxv, id);
 
diff --git a/external/llvm-project/libc/src/sys/epoll/linux/epoll_create.cpp b/external/llvm-project/libc/src/sys/epoll/linux/epoll_create.cpp
index 7196ac7410c3..2e44e883ddf0 100644
--- a/external/llvm-project/libc/src/sys/epoll/linux/epoll_create.cpp
+++ b/external/llvm-project/libc/src/sys/epoll/linux/epoll_create.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/epoll/linux/epoll_create1.cpp b/external/llvm-project/libc/src/sys/epoll/linux/epoll_create1.cpp
index efff282e2714..3c60090fb7b4 100644
--- a/external/llvm-project/libc/src/sys/epoll/linux/epoll_create1.cpp
+++ b/external/llvm-project/libc/src/sys/epoll/linux/epoll_create1.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/epoll/linux/epoll_ctl.cpp b/external/llvm-project/libc/src/sys/epoll/linux/epoll_ctl.cpp
index 5f7dbb77b1e5..079bd60403b0 100644
--- a/external/llvm-project/libc/src/sys/epoll/linux/epoll_ctl.cpp
+++ b/external/llvm-project/libc/src/sys/epoll/linux/epoll_ctl.cpp
@@ -11,8 +11,8 @@
 #include "hdr/types/struct_epoll_event.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait.cpp b/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait.cpp
index d7836549928c..24fd1dbdc467 100644
--- a/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait.cpp
+++ b/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait.cpp
@@ -13,9 +13,9 @@
 #include "hdr/types/struct_epoll_event.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/sanitizer.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait2.cpp b/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait2.cpp
index 14b419399fe9..219984528efd 100644
--- a/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait2.cpp
+++ b/external/llvm-project/libc/src/sys/epoll/linux/epoll_pwait2.cpp
@@ -14,9 +14,9 @@
 #include "hdr/types/struct_timespec.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/sanitizer.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/epoll/linux/epoll_wait.cpp b/external/llvm-project/libc/src/sys/epoll/linux/epoll_wait.cpp
index 1a63be5e260f..7fae7b55992f 100644
--- a/external/llvm-project/libc/src/sys/epoll/linux/epoll_wait.cpp
+++ b/external/llvm-project/libc/src/sys/epoll/linux/epoll_wait.cpp
@@ -13,9 +13,9 @@
 #include "hdr/types/struct_epoll_event.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/sanitizer.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/ioctl/CMakeLists.txt b/external/llvm-project/libc/src/sys/ioctl/CMakeLists.txt
new file mode 100644
index 000000000000..099a1b96389f
--- /dev/null
+++ b/external/llvm-project/libc/src/sys/ioctl/CMakeLists.txt
@@ -0,0 +1,10 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+endif()
+
+add_entrypoint_object(
+  ioctl
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.ioctl
+)
diff --git a/external/llvm-project/libc/src/math/amdgpu/acosf.cpp b/external/llvm-project/libc/src/sys/ioctl/ioctl.h
similarity index 61%
rename from external/llvm-project/libc/src/math/amdgpu/acosf.cpp
rename to external/llvm-project/libc/src/sys/ioctl/ioctl.h
index 0a72a70a3ee9..62323ba7dd4d 100644
--- a/external/llvm-project/libc/src/math/amdgpu/acosf.cpp
+++ b/external/llvm-project/libc/src/sys/ioctl/ioctl.h
@@ -1,4 +1,4 @@
-//===-- Implementation of the acosf function for GPU ----------------------===//
+//===-- Implementation header for ioctl ---------------------------*-C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/acosf.h"
-#include "src/__support/common.h"
+#ifndef LLVM_LIBC_SRC_SYS_IOCTL_IOCTL_H
+#define LLVM_LIBC_SRC_SYS_IOCTL_IOCTL_H
 
-#include "declarations.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, acosf, (float x)) { return __ocml_acos_f32(x); }
+int ioctl(int fd, unsigned long request, ...);
 
 } // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_SYS_IOCTL_IOCTL_H
diff --git a/external/llvm-project/libc/src/sys/ioctl/linux/CMakeLists.txt b/external/llvm-project/libc/src/sys/ioctl/linux/CMakeLists.txt
new file mode 100644
index 000000000000..876f35aaee66
--- /dev/null
+++ b/external/llvm-project/libc/src/sys/ioctl/linux/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_entrypoint_object(
+  ioctl
+  SRCS
+    ioctl.cpp
+  HDRS
+    ../ioctl.h
+  DEPENDS
+    libc.include.sys_ioctl
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
diff --git a/external/llvm-project/libc/src/sys/ioctl/linux/ioctl.cpp b/external/llvm-project/libc/src/sys/ioctl/linux/ioctl.cpp
new file mode 100644
index 000000000000..9bb669c6a6f6
--- /dev/null
+++ b/external/llvm-project/libc/src/sys/ioctl/linux/ioctl.cpp
@@ -0,0 +1,36 @@
+//===---------- Linux implementation of the ioctl function ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/ioctl/ioctl.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+#include <stdarg.h>
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, ioctl, (int fd, unsigned long request, ...)) {
+  va_list vargs;
+  va_start(vargs, request);
+  void *data_pointer = va_arg(vargs, void *);
+  int ret =
+      LIBC_NAMESPACE::syscall_impl<int>(SYS_ioctl, fd, request, data_pointer);
+  va_end(vargs);
+
+  // Some ioctls can be expected to return positive values
+  if (ret >= 0)
+    return ret;
+
+  // If there is an error, errno is set and -1 is returned.
+  libc_errno = -ret;
+  return -1;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/sys/mman/linux/madvise.cpp b/external/llvm-project/libc/src/sys/mman/linux/madvise.cpp
index 332d6c2db4ac..1bb284f62b89 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/madvise.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/madvise.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/mincore.cpp b/external/llvm-project/libc/src/sys/mman/linux/mincore.cpp
index b5436fda3853..d583f1ef85f3 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/mincore.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/mincore.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/mlock.cpp b/external/llvm-project/libc/src/sys/mman/linux/mlock.cpp
index be7eb28e29c4..8582eb7c0063 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/mlock.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/mlock.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/mlock2.cpp b/external/llvm-project/libc/src/sys/mman/linux/mlock2.cpp
index 7bc557f9bf58..955cfe128de7 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/mlock2.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/mlock2.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/mlockall.cpp b/external/llvm-project/libc/src/sys/mman/linux/mlockall.cpp
index eae3a9ea0a18..c3502fbb3af3 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/mlockall.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/mlockall.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/mmap.cpp b/external/llvm-project/libc/src/sys/mman/linux/mmap.cpp
index ee9a0a32e8f5..33f9fe8ff370 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/mmap.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/mmap.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <linux/param.h> // For EXEC_PAGESIZE.
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/mman/linux/mprotect.cpp b/external/llvm-project/libc/src/sys/mman/linux/mprotect.cpp
index e2351028e2c7..6b14915b60c9 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/mprotect.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/mprotect.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/mremap.cpp b/external/llvm-project/libc/src/sys/mman/linux/mremap.cpp
index 38bcfce833d3..6cdda9435bb6 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/mremap.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/mremap.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <linux/param.h> // For EXEC_PAGESIZE.
 #include <stdarg.h>
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/sys/mman/linux/msync.cpp b/external/llvm-project/libc/src/sys/mman/linux/msync.cpp
index e2b4f81d616a..650678bcb36e 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/msync.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/msync.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/munlock.cpp b/external/llvm-project/libc/src/sys/mman/linux/munlock.cpp
index 93c25f844c6e..9638949f5fcb 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/munlock.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/munlock.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/munlockall.cpp b/external/llvm-project/libc/src/sys/mman/linux/munlockall.cpp
index f5911cb01bc2..f47eaece178e 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/munlockall.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/munlockall.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/munmap.cpp b/external/llvm-project/libc/src/sys/mman/linux/munmap.cpp
index 9c01b15ac8dc..61b1f1549dd1 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/munmap.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/munmap.cpp
@@ -11,9 +11,9 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
-#include <sys/syscall.h>          // For syscall numbers.
+#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/sys/mman/linux/remap_file_pages.cpp b/external/llvm-project/libc/src/sys/mman/linux/remap_file_pages.cpp
index f616e1915ecc..58ae4017f628 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/remap_file_pages.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/remap_file_pages.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/mman/linux/shm_common.h b/external/llvm-project/libc/src/sys/mman/linux/shm_common.h
index ce75c2b5b699..29d1401821e4 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/shm_common.h
+++ b/external/llvm-project/libc/src/sys/mman/linux/shm_common.h
@@ -9,10 +9,15 @@
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/string_view.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/string/memory_utils/inline_memcpy.h"
 
+// TODO: clean this up.
+//  1. Change from optional to ErrorOr, and return the errno instead of setting
+//    it here.
+//  2. Replace inline memcpy with __builtin_memcpy
+
 // TODO: Get PATH_MAX via https://github.com/llvm/llvm-project/issues/85121
 #include <linux/limits.h>
 
diff --git a/external/llvm-project/libc/src/sys/mman/linux/shm_open.cpp b/external/llvm-project/libc/src/sys/mman/linux/shm_open.cpp
index 11de482272d0..3099062eace9 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/shm_open.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/shm_open.cpp
@@ -7,9 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/sys/mman/shm_open.h"
+#include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
+#include "src/__support/OSUtil/fcntl.h"
 #include "src/__support/macros/config.h"
-#include "src/fcntl/open.h"
 #include "src/sys/mman/linux/shm_common.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -17,9 +18,16 @@ namespace LIBC_NAMESPACE_DECL {
 static constexpr int DEFAULT_OFLAGS = O_NOFOLLOW | O_CLOEXEC | O_NONBLOCK;
 
 LLVM_LIBC_FUNCTION(int, shm_open, (const char *name, int oflags, mode_t mode)) {
-  using namespace shm_common;
-  if (cpp::optional<SHMPath> buffer = translate_name(name))
-    return open(buffer->data(), oflags | DEFAULT_OFLAGS, mode);
+  if (cpp::optional<shm_common::SHMPath> buffer =
+          shm_common::translate_name(name)) {
+    auto result = internal::open(buffer->data(), oflags | DEFAULT_OFLAGS, mode);
+
+    if (!result.has_value()) {
+      libc_errno = result.error();
+      return -1;
+    }
+    return result.value();
+  }
   return -1;
 }
 
diff --git a/external/llvm-project/libc/src/sys/mman/linux/shm_unlink.cpp b/external/llvm-project/libc/src/sys/mman/linux/shm_unlink.cpp
index 6a7630151220..4c61c7cd16ba 100644
--- a/external/llvm-project/libc/src/sys/mman/linux/shm_unlink.cpp
+++ b/external/llvm-project/libc/src/sys/mman/linux/shm_unlink.cpp
@@ -13,10 +13,13 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+// TODO: stop calling the public unlink function. It should be calling an
+// internal shared utility.
+
 LLVM_LIBC_FUNCTION(int, shm_unlink, (const char *name)) {
-  using namespace shm_common;
-  if (cpp::optional<SHMPath> buffer = translate_name(name))
-    return unlink(buffer->data());
+  if (cpp::optional<shm_common::SHMPath> buffer =
+          shm_common::translate_name(name))
+    return LIBC_NAMESPACE::unlink(buffer->data());
   return -1;
 }
 
diff --git a/external/llvm-project/libc/src/sys/prctl/linux/prctl.cpp b/external/llvm-project/libc/src/sys/prctl/linux/prctl.cpp
index 5d4e9046b877..c726b0a53959 100644
--- a/external/llvm-project/libc/src/sys/prctl/linux/prctl.cpp
+++ b/external/llvm-project/libc/src/sys/prctl/linux/prctl.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/random/linux/getrandom.cpp b/external/llvm-project/libc/src/sys/random/linux/getrandom.cpp
index 9a8869a2d6d3..0b8471ed8b37 100644
--- a/external/llvm-project/libc/src/sys/random/linux/getrandom.cpp
+++ b/external/llvm-project/libc/src/sys/random/linux/getrandom.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/resource/linux/getrlimit.cpp b/external/llvm-project/libc/src/sys/resource/linux/getrlimit.cpp
index 30c2e91b036d..d27213419494 100644
--- a/external/llvm-project/libc/src/sys/resource/linux/getrlimit.cpp
+++ b/external/llvm-project/libc/src/sys/resource/linux/getrlimit.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/resource.h> // For struct rlimit
 #include <sys/syscall.h>  // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/resource/linux/setrlimit.cpp b/external/llvm-project/libc/src/sys/resource/linux/setrlimit.cpp
index 85f07900aaef..300bad75baa6 100644
--- a/external/llvm-project/libc/src/sys/resource/linux/setrlimit.cpp
+++ b/external/llvm-project/libc/src/sys/resource/linux/setrlimit.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/resource.h> // For struct rlimit
 #include <sys/syscall.h>  // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/select/linux/select.cpp b/external/llvm-project/libc/src/sys/select/linux/select.cpp
index 9ccb1e95f275..6c434eb58459 100644
--- a/external/llvm-project/libc/src/sys/select/linux/select.cpp
+++ b/external/llvm-project/libc/src/sys/select/linux/select.cpp
@@ -13,8 +13,8 @@
 #include "src/__support/CPP/limits.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <stddef.h>      // For size_t
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/sys/sendfile/linux/sendfile.cpp b/external/llvm-project/libc/src/sys/sendfile/linux/sendfile.cpp
index 9d4174cb8c91..ec892323def5 100644
--- a/external/llvm-project/libc/src/sys/sendfile/linux/sendfile.cpp
+++ b/external/llvm-project/libc/src/sys/sendfile/linux/sendfile.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/sendfile.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/socket/linux/bind.cpp b/external/llvm-project/libc/src/sys/socket/linux/bind.cpp
index 72a3307a91dd..83a3d06f5380 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/bind.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/bind.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <linux/net.h>   // For SYS_SOCKET socketcall number.
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/sys/socket/linux/recv.cpp b/external/llvm-project/libc/src/sys/socket/linux/recv.cpp
index 5e9f2d3233fc..baf4de1b5eb5 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/recv.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/recv.cpp
@@ -16,8 +16,8 @@
 #include "hdr/types/struct_sockaddr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/sanitizer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/sys/socket/linux/recvfrom.cpp b/external/llvm-project/libc/src/sys/socket/linux/recvfrom.cpp
index 574e65f64a54..3d8397b478cc 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/recvfrom.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/recvfrom.cpp
@@ -16,8 +16,8 @@
 #include "hdr/types/struct_sockaddr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/sanitizer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/sys/socket/linux/recvmsg.cpp b/external/llvm-project/libc/src/sys/socket/linux/recvmsg.cpp
index e42b6346f330..bc6d072dbf9a 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/recvmsg.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/recvmsg.cpp
@@ -15,8 +15,8 @@
 #include "hdr/types/struct_msghdr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/sanitizer.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/sys/socket/linux/send.cpp b/external/llvm-project/libc/src/sys/socket/linux/send.cpp
index cb3b4d5a9ece..43b01e7e6e0f 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/send.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/send.cpp
@@ -16,7 +16,7 @@
 #include "hdr/types/struct_sockaddr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/sys/socket/linux/sendmsg.cpp b/external/llvm-project/libc/src/sys/socket/linux/sendmsg.cpp
index b4d9c9deda02..b04783ebfe7e 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/sendmsg.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/sendmsg.cpp
@@ -15,7 +15,7 @@
 #include "hdr/types/struct_msghdr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/sys/socket/linux/sendto.cpp b/external/llvm-project/libc/src/sys/socket/linux/sendto.cpp
index 2fada192b086..9dda127f872d 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/sendto.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/sendto.cpp
@@ -16,7 +16,7 @@
 #include "hdr/types/struct_sockaddr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/sys/socket/linux/socket.cpp b/external/llvm-project/libc/src/sys/socket/linux/socket.cpp
index 3e6df4d487a5..69eb6cfa01ce 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/socket.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/socket.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <linux/net.h>   // For SYS_SOCKET socketcall number.
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/sys/socket/linux/socketpair.cpp b/external/llvm-project/libc/src/sys/socket/linux/socketpair.cpp
index 60612ac04d61..7ea8ca46cee5 100644
--- a/external/llvm-project/libc/src/sys/socket/linux/socketpair.cpp
+++ b/external/llvm-project/libc/src/sys/socket/linux/socketpair.cpp
@@ -10,9 +10,9 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/sanitizer.h"
-#include "src/errno/libc_errno.h"
 #include <linux/net.h>   // For SYS_SOCKET socketcall number.
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/stat/linux/chmod.cpp b/external/llvm-project/libc/src/sys/stat/linux/chmod.cpp
index 1b787e47e7c6..2bd0788ec1df 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/chmod.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/chmod.cpp
@@ -13,8 +13,8 @@
 
 #include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/stat/linux/fchmod.cpp b/external/llvm-project/libc/src/sys/stat/linux/fchmod.cpp
index 0d6fd359169a..3dadfdd1d943 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/fchmod.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/fchmod.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/types/mode_t.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/stat/linux/fchmodat.cpp b/external/llvm-project/libc/src/sys/stat/linux/fchmodat.cpp
index e76db4d160fb..add2192a558a 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/fchmodat.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/fchmodat.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/stat/linux/fstat.cpp b/external/llvm-project/libc/src/sys/stat/linux/fstat.cpp
index 35cf8f08f782..dea002c5e12a 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/fstat.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/fstat.cpp
@@ -8,8 +8,8 @@
 
 #include "src/sys/stat/fstat.h"
 #include "kernel_statx.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "src/__support/common.h"
 
diff --git a/external/llvm-project/libc/src/sys/stat/linux/lstat.cpp b/external/llvm-project/libc/src/sys/stat/linux/lstat.cpp
index 354c5b6e029a..5601dd5d78a9 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/lstat.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/lstat.cpp
@@ -8,8 +8,8 @@
 
 #include "src/sys/stat/lstat.h"
 #include "kernel_statx.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
diff --git a/external/llvm-project/libc/src/sys/stat/linux/mkdir.cpp b/external/llvm-project/libc/src/sys/stat/linux/mkdir.cpp
index b319b5c8393d..0829ff4f9432 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/mkdir.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/mkdir.cpp
@@ -13,8 +13,8 @@
 
 #include "hdr/fcntl_macros.h"
 #include "hdr/types/mode_t.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/stat/linux/mkdirat.cpp b/external/llvm-project/libc/src/sys/stat/linux/mkdirat.cpp
index 097fc158010d..8f4194dc3275 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/mkdirat.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/mkdirat.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/stat.h>
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/sys/stat/linux/stat.cpp b/external/llvm-project/libc/src/sys/stat/linux/stat.cpp
index de9cdb197d68..5553eaf00be2 100644
--- a/external/llvm-project/libc/src/sys/stat/linux/stat.cpp
+++ b/external/llvm-project/libc/src/sys/stat/linux/stat.cpp
@@ -8,8 +8,8 @@
 
 #include "src/sys/stat/stat.h"
 #include "kernel_statx.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "src/__support/common.h"
 
diff --git a/external/llvm-project/libc/src/sys/statvfs/linux/statfs_utils.h b/external/llvm-project/libc/src/sys/statvfs/linux/statfs_utils.h
index 1e5be5153101..8ee4de288ef6 100644
--- a/external/llvm-project/libc/src/sys/statvfs/linux/statfs_utils.h
+++ b/external/llvm-project/libc/src/sys/statvfs/linux/statfs_utils.h
@@ -12,9 +12,9 @@
 #include "include/llvm-libc-types/struct_statvfs.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/OSUtil/syscall.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <asm/statfs.h>
 #include <sys/syscall.h>
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/time/linux/getitimer.cpp b/external/llvm-project/libc/src/sys/time/linux/getitimer.cpp
index fec06aa4086e..b87406679694 100644
--- a/external/llvm-project/libc/src/sys/time/linux/getitimer.cpp
+++ b/external/llvm-project/libc/src/sys/time/linux/getitimer.cpp
@@ -10,7 +10,7 @@
 #include "hdr/types/struct_itimerval.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <sys/syscall.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/time/linux/setitimer.cpp b/external/llvm-project/libc/src/sys/time/linux/setitimer.cpp
index def04a474011..1de0d4329776 100644
--- a/external/llvm-project/libc/src/sys/time/linux/setitimer.cpp
+++ b/external/llvm-project/libc/src/sys/time/linux/setitimer.cpp
@@ -9,7 +9,7 @@
 #include "hdr/types/struct_itimerval.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <sys/syscall.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/time/linux/utimes.cpp b/external/llvm-project/libc/src/sys/time/linux/utimes.cpp
index 76b69937a5f4..ed37b42aedf6 100644
--- a/external/llvm-project/libc/src/sys/time/linux/utimes.cpp
+++ b/external/llvm-project/libc/src/sys/time/linux/utimes.cpp
@@ -15,7 +15,7 @@
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 #include <sys/syscall.h>
 
diff --git a/external/llvm-project/libc/src/sys/uio/linux/readv.cpp b/external/llvm-project/libc/src/sys/uio/linux/readv.cpp
index f1393a9749be..c9d8d87ddc72 100644
--- a/external/llvm-project/libc/src/sys/uio/linux/readv.cpp
+++ b/external/llvm-project/libc/src/sys/uio/linux/readv.cpp
@@ -10,7 +10,7 @@
 #include "hdr/types/struct_iovec.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <sys/syscall.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/uio/linux/writev.cpp b/external/llvm-project/libc/src/sys/uio/linux/writev.cpp
index 8992bed95c98..b0b9e1520792 100644
--- a/external/llvm-project/libc/src/sys/uio/linux/writev.cpp
+++ b/external/llvm-project/libc/src/sys/uio/linux/writev.cpp
@@ -10,7 +10,7 @@
 #include "hdr/types/struct_iovec.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <sys/syscall.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/sys/utsname/linux/uname.cpp b/external/llvm-project/libc/src/sys/utsname/linux/uname.cpp
index 7bb227e801e3..b47ba964faf0 100644
--- a/external/llvm-project/libc/src/sys/utsname/linux/uname.cpp
+++ b/external/llvm-project/libc/src/sys/utsname/linux/uname.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 #include <sys/utsname.h>
 
diff --git a/external/llvm-project/libc/src/sys/wait/wait4Impl.h b/external/llvm-project/libc/src/sys/wait/wait4Impl.h
index f2bdeb02f866..77ed3ad22f14 100644
--- a/external/llvm-project/libc/src/sys/wait/wait4Impl.h
+++ b/external/llvm-project/libc/src/sys/wait/wait4Impl.h
@@ -12,8 +12,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/__support/error_or.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <signal.h>
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/termios/linux/cfsetispeed.cpp b/external/llvm-project/libc/src/termios/linux/cfsetispeed.cpp
index 9656b714a8ed..47b19974d21b 100644
--- a/external/llvm-project/libc/src/termios/linux/cfsetispeed.cpp
+++ b/external/llvm-project/libc/src/termios/linux/cfsetispeed.cpp
@@ -9,8 +9,8 @@
 #include "src/termios/cfsetispeed.h"
 
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <termios.h>
 
diff --git a/external/llvm-project/libc/src/termios/linux/cfsetospeed.cpp b/external/llvm-project/libc/src/termios/linux/cfsetospeed.cpp
index 6130d266dbff..d2f138257a47 100644
--- a/external/llvm-project/libc/src/termios/linux/cfsetospeed.cpp
+++ b/external/llvm-project/libc/src/termios/linux/cfsetospeed.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/termios/cfsetospeed.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "src/__support/common.h"
 
diff --git a/external/llvm-project/libc/src/termios/linux/tcdrain.cpp b/external/llvm-project/libc/src/termios/linux/tcdrain.cpp
index 116e3f0e0cbc..570b15c24fe7 100644
--- a/external/llvm-project/libc/src/termios/linux/tcdrain.cpp
+++ b/external/llvm-project/libc/src/termios/linux/tcdrain.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <asm/ioctls.h> // Safe to include without the risk of name pollution.
 #include <sys/syscall.h> // For syscall numbers
diff --git a/external/llvm-project/libc/src/termios/linux/tcflow.cpp b/external/llvm-project/libc/src/termios/linux/tcflow.cpp
index d229230b5d13..714ef6aa7129 100644
--- a/external/llvm-project/libc/src/termios/linux/tcflow.cpp
+++ b/external/llvm-project/libc/src/termios/linux/tcflow.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <asm/ioctls.h> // Safe to include without the risk of name pollution.
 #include <sys/syscall.h> // For syscall numbers
diff --git a/external/llvm-project/libc/src/termios/linux/tcflush.cpp b/external/llvm-project/libc/src/termios/linux/tcflush.cpp
index 028a5414b196..4c7b9fadc446 100644
--- a/external/llvm-project/libc/src/termios/linux/tcflush.cpp
+++ b/external/llvm-project/libc/src/termios/linux/tcflush.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <asm/ioctls.h> // Safe to include without the risk of name pollution.
 #include <sys/syscall.h> // For syscall numbers
diff --git a/external/llvm-project/libc/src/termios/linux/tcgetattr.cpp b/external/llvm-project/libc/src/termios/linux/tcgetattr.cpp
index 63c096ff88eb..2e768269c874 100644
--- a/external/llvm-project/libc/src/termios/linux/tcgetattr.cpp
+++ b/external/llvm-project/libc/src/termios/linux/tcgetattr.cpp
@@ -11,8 +11,8 @@
 
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <asm/ioctls.h> // Safe to include without the risk of name pollution.
 #include <sys/syscall.h> // For syscall numbers
diff --git a/external/llvm-project/libc/src/termios/linux/tcgetsid.cpp b/external/llvm-project/libc/src/termios/linux/tcgetsid.cpp
index c283d0e4fda9..7487816cf274 100644
--- a/external/llvm-project/libc/src/termios/linux/tcgetsid.cpp
+++ b/external/llvm-project/libc/src/termios/linux/tcgetsid.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <asm/ioctls.h> // Safe to include without the risk of name pollution.
 #include <sys/syscall.h> // For syscall numbers
diff --git a/external/llvm-project/libc/src/termios/linux/tcsendbreak.cpp b/external/llvm-project/libc/src/termios/linux/tcsendbreak.cpp
index 30bc91cf3de0..1d546c1d5953 100644
--- a/external/llvm-project/libc/src/termios/linux/tcsendbreak.cpp
+++ b/external/llvm-project/libc/src/termios/linux/tcsendbreak.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <asm/ioctls.h> // Safe to include without the risk of name pollution.
 #include <sys/syscall.h> // For syscall numbers
diff --git a/external/llvm-project/libc/src/termios/linux/tcsetattr.cpp b/external/llvm-project/libc/src/termios/linux/tcsetattr.cpp
index 8aa1e5c57b34..8a2c7290217b 100644
--- a/external/llvm-project/libc/src/termios/linux/tcsetattr.cpp
+++ b/external/llvm-project/libc/src/termios/linux/tcsetattr.cpp
@@ -11,8 +11,8 @@
 
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <asm/ioctls.h> // Safe to include without the risk of name pollution.
 #include <sys/syscall.h> // For syscall numbers
diff --git a/external/llvm-project/libc/src/threads/thrd_create.cpp b/external/llvm-project/libc/src/threads/thrd_create.cpp
index 4680944c2eee..67e22e72fd0e 100644
--- a/external/llvm-project/libc/src/threads/thrd_create.cpp
+++ b/external/llvm-project/libc/src/threads/thrd_create.cpp
@@ -8,9 +8,9 @@
 
 #include "src/threads/thrd_create.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/threads/thread.h"
-#include "src/errno/libc_errno.h"
 
 #include <threads.h> // For thrd_* type definitions.
 
diff --git a/external/llvm-project/libc/src/time/linux/clock.cpp b/external/llvm-project/libc/src/time/linux/clock.cpp
index ee4fa82b4f89..c38697cd0668 100644
--- a/external/llvm-project/libc/src/time/linux/clock.cpp
+++ b/external/llvm-project/libc/src/time/linux/clock.cpp
@@ -10,10 +10,10 @@
 #include "hdr/time_macros.h"
 #include "src/__support/CPP/limits.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/time/clock_gettime.h"
 #include "src/__support/time/units.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/time/linux/clock_gettime.cpp b/external/llvm-project/libc/src/time/linux/clock_gettime.cpp
index 743c644d65d0..b3fcd2b22f9d 100644
--- a/external/llvm-project/libc/src/time/linux/clock_gettime.cpp
+++ b/external/llvm-project/libc/src/time/linux/clock_gettime.cpp
@@ -8,9 +8,9 @@
 
 #include "src/time/clock_gettime.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/time/clock_gettime.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/time/linux/gettimeofday.cpp b/external/llvm-project/libc/src/time/linux/gettimeofday.cpp
index e8ddf482fc98..237b05903c70 100644
--- a/external/llvm-project/libc/src/time/linux/gettimeofday.cpp
+++ b/external/llvm-project/libc/src/time/linux/gettimeofday.cpp
@@ -10,10 +10,10 @@
 #include "hdr/time_macros.h"
 #include "hdr/types/suseconds_t.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/time/clock_gettime.h"
 #include "src/__support/time/units.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/time/linux/nanosleep.cpp b/external/llvm-project/libc/src/time/linux/nanosleep.cpp
index 7a856376ffb2..6b9704126a0a 100644
--- a/external/llvm-project/libc/src/time/linux/nanosleep.cpp
+++ b/external/llvm-project/libc/src/time/linux/nanosleep.cpp
@@ -10,8 +10,8 @@
 #include "hdr/time_macros.h"
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <stdint.h>      // For int64_t.
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/time/linux/timespec_get.cpp b/external/llvm-project/libc/src/time/linux/timespec_get.cpp
index cf5174523aa4..a4d437233273 100644
--- a/external/llvm-project/libc/src/time/linux/timespec_get.cpp
+++ b/external/llvm-project/libc/src/time/linux/timespec_get.cpp
@@ -9,9 +9,9 @@
 #include "src/time/timespec_get.h"
 #include "hdr/time_macros.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/time/clock_gettime.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/time/time.cpp b/external/llvm-project/libc/src/time/time.cpp
index 860909af7488..2a81f0182c31 100644
--- a/external/llvm-project/libc/src/time/time.cpp
+++ b/external/llvm-project/libc/src/time/time.cpp
@@ -10,9 +10,9 @@
 
 #include "hdr/time_macros.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/time/clock_gettime.h"
-#include "src/errno/libc_errno.h"
 
 namespace LIBC_NAMESPACE_DECL {
 // avoid inconsitent clang-format behavior
diff --git a/external/llvm-project/libc/src/time/time_utils.h b/external/llvm-project/libc/src/time/time_utils.h
index bbbb1c08a475..0541c24ece82 100644
--- a/external/llvm-project/libc/src/time/time_utils.h
+++ b/external/llvm-project/libc/src/time/time_utils.h
@@ -15,8 +15,8 @@
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "time_constants.h"
 
 #include <stdint.h>
diff --git a/external/llvm-project/libc/src/time/windows/clock_getres.cpp b/external/llvm-project/libc/src/time/windows/clock_getres.cpp
index b8c0c82aa641..969bb66be2d2 100644
--- a/external/llvm-project/libc/src/time/windows/clock_getres.cpp
+++ b/external/llvm-project/libc/src/time/windows/clock_getres.cpp
@@ -13,10 +13,10 @@
 
 #include "src/__support/CPP/limits.h"
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/time/units.h"
 #include "src/__support/time/windows/performance_counter.h"
-#include "src/errno/libc_errno.h"
 #include "src/time/clock_getres.h"
 
 #define WIN32_LEAN_AND_MEAN
diff --git a/external/llvm-project/libc/src/unistd/linux/access.cpp b/external/llvm-project/libc/src/unistd/linux/access.cpp
index 2f7ebbcdf9e8..55cd6adca779 100644
--- a/external/llvm-project/libc/src/unistd/linux/access.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/access.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/chdir.cpp b/external/llvm-project/libc/src/unistd/linux/chdir.cpp
index a30d1dc883be..04ba509b49a5 100644
--- a/external/llvm-project/libc/src/unistd/linux/chdir.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/chdir.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/close.cpp b/external/llvm-project/libc/src/unistd/linux/close.cpp
index 58d42a9673fb..6ef3a3c6d63f 100644
--- a/external/llvm-project/libc/src/unistd/linux/close.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/close.cpp
@@ -8,22 +8,22 @@
 
 #include "src/unistd/close.h"
 
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/OSUtil/fcntl.h"
 #include "src/__support/common.h"
-
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, close, (int fd)) {
-  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_close, fd);
-  if (ret < 0) {
-    libc_errno = -ret;
+  auto result = internal::close(fd);
+
+  if (!result.has_value()) {
+    libc_errno = result.error();
     return -1;
   }
-  return ret;
+  return result.value();
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/unistd/linux/dup.cpp b/external/llvm-project/libc/src/unistd/linux/dup.cpp
index c1710a37f611..81d30c6cdbc4 100644
--- a/external/llvm-project/libc/src/unistd/linux/dup.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/dup.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/dup2.cpp b/external/llvm-project/libc/src/unistd/linux/dup2.cpp
index 7ffc151a053c..0a0e86573b34 100644
--- a/external/llvm-project/libc/src/unistd/linux/dup2.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/dup2.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/dup3.cpp b/external/llvm-project/libc/src/unistd/linux/dup3.cpp
index c096ba73c96b..770fb73515b2 100644
--- a/external/llvm-project/libc/src/unistd/linux/dup3.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/dup3.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/execv.cpp b/external/llvm-project/libc/src/unistd/linux/execv.cpp
index a3f2525ed7ca..d4f2bd9a5165 100644
--- a/external/llvm-project/libc/src/unistd/linux/execv.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/execv.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/execve.cpp b/external/llvm-project/libc/src/unistd/linux/execve.cpp
index 37162c412178..2214b6df493b 100644
--- a/external/llvm-project/libc/src/unistd/linux/execve.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/execve.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/fchdir.cpp b/external/llvm-project/libc/src/unistd/linux/fchdir.cpp
index 8196dc63ab1e..f7a7422363e6 100644
--- a/external/llvm-project/libc/src/unistd/linux/fchdir.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/fchdir.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/fork.cpp b/external/llvm-project/libc/src/unistd/linux/fork.cpp
index 8aa0477a15d5..75a76fdea50b 100644
--- a/external/llvm-project/libc/src/unistd/linux/fork.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/fork.cpp
@@ -15,7 +15,7 @@
 #include "src/__support/threads/identifier.h"
 #include "src/__support/threads/thread.h" // For thread self object
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <signal.h>      // For SIGCHLD
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/unistd/linux/fsync.cpp b/external/llvm-project/libc/src/unistd/linux/fsync.cpp
index ae3895bab15f..fe08aed61e25 100644
--- a/external/llvm-project/libc/src/unistd/linux/fsync.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/fsync.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/ftruncate.cpp b/external/llvm-project/libc/src/unistd/linux/ftruncate.cpp
index ccbb0634664a..f6aa6f8b48cc 100644
--- a/external/llvm-project/libc/src/unistd/linux/ftruncate.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/ftruncate.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/unistd_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stdint.h>      // For uint64_t.
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/unistd/linux/getcwd.cpp b/external/llvm-project/libc/src/unistd/linux/getcwd.cpp
index 1bb11a7c8e7b..c0e475dd3e8f 100644
--- a/external/llvm-project/libc/src/unistd/linux/getcwd.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/getcwd.cpp
@@ -13,7 +13,7 @@
 #include "src/__support/macros/config.h"
 #include "src/string/allocating_string_utils.h" // For strdup.
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <linux/limits.h> // This is safe to include without any name pollution.
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/unistd/linux/getentropy.cpp b/external/llvm-project/libc/src/unistd/linux/getentropy.cpp
index 168a1197734e..65bcbf27601d 100644
--- a/external/llvm-project/libc/src/unistd/linux/getentropy.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/getentropy.cpp
@@ -10,7 +10,7 @@
 #include "hdr/errno_macros.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/unistd/linux/getsid.cpp b/external/llvm-project/libc/src/unistd/linux/getsid.cpp
index 5977c5bf10e9..025b8d1691ac 100644
--- a/external/llvm-project/libc/src/unistd/linux/getsid.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/getsid.cpp
@@ -11,8 +11,8 @@
 #include "hdr/types/pid_t.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/isatty.cpp b/external/llvm-project/libc/src/unistd/linux/isatty.cpp
index e6ea22a714c7..a4d17912b57b 100644
--- a/external/llvm-project/libc/src/unistd/linux/isatty.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/isatty.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/ioctl.h>   // For ioctl numbers.
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/unistd/linux/link.cpp b/external/llvm-project/libc/src/unistd/linux/link.cpp
index 477806a70df7..205cf8a84a5c 100644
--- a/external/llvm-project/libc/src/unistd/linux/link.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/link.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/unistd/linux/linkat.cpp b/external/llvm-project/libc/src/unistd/linux/linkat.cpp
index 40f68cc90c48..ea5bc48cbedc 100644
--- a/external/llvm-project/libc/src/unistd/linux/linkat.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/linkat.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/lseek.cpp b/external/llvm-project/libc/src/unistd/linux/lseek.cpp
index 0e957498da74..26a08269fd8d 100644
--- a/external/llvm-project/libc/src/unistd/linux/lseek.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/lseek.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/unistd/lseek.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "src/__support/File/linux/lseekImpl.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
diff --git a/external/llvm-project/libc/src/unistd/linux/pathconf.cpp b/external/llvm-project/libc/src/unistd/linux/pathconf.cpp
index ca1c10bb9f7f..7dde857c1cfd 100644
--- a/external/llvm-project/libc/src/unistd/linux/pathconf.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/pathconf.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/unistd/pathconf.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/statvfs/linux/statfs_utils.h"
 #include "src/unistd/linux/pathconf_utils.h"
 
diff --git a/external/llvm-project/libc/src/unistd/linux/pathconf_utils.cpp b/external/llvm-project/libc/src/unistd/linux/pathconf_utils.cpp
index 035e628dff25..9a62e31fd188 100644
--- a/external/llvm-project/libc/src/unistd/linux/pathconf_utils.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/pathconf_utils.cpp
@@ -14,8 +14,8 @@
 #include "hdr/unistd_macros.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/statvfs/linux/statfs_utils.h"
 
 // other linux specific includes
diff --git a/external/llvm-project/libc/src/unistd/linux/pipe.cpp b/external/llvm-project/libc/src/unistd/linux/pipe.cpp
index dfcd5bfdaf53..b9943c833805 100644
--- a/external/llvm-project/libc/src/unistd/linux/pipe.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/pipe.cpp
@@ -10,10 +10,10 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/sanitizer.h" // for MSAN_UNPOISON
-#include "src/errno/libc_errno.h"
-#include <sys/syscall.h> // For syscall numbers.
+#include <sys/syscall.h>                    // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/unistd/linux/pipe2.cpp b/external/llvm-project/libc/src/unistd/linux/pipe2.cpp
index ebe7e0114ae9..d30f3b37a1ad 100644
--- a/external/llvm-project/libc/src/unistd/linux/pipe2.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/pipe2.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/pread.cpp b/external/llvm-project/libc/src/unistd/linux/pread.cpp
index 3e27857f9a2b..2f86e397feef 100644
--- a/external/llvm-project/libc/src/unistd/linux/pread.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/pread.cpp
@@ -10,11 +10,11 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/sanitizer.h" // for MSAN_UNPOISON
-#include "src/errno/libc_errno.h"
-#include <stdint.h>      // For uint64_t.
-#include <sys/syscall.h> // For syscall numbers.
+#include <stdint.h>                         // For uint64_t.
+#include <sys/syscall.h>                    // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/unistd/linux/pwrite.cpp b/external/llvm-project/libc/src/unistd/linux/pwrite.cpp
index 1b81b2a05949..f4cf8e16d766 100644
--- a/external/llvm-project/libc/src/unistd/linux/pwrite.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/pwrite.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stdint.h>      // For uint64_t.
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/unistd/linux/read.cpp b/external/llvm-project/libc/src/unistd/linux/read.cpp
index 4419900f2330..55676f3f7010 100644
--- a/external/llvm-project/libc/src/unistd/linux/read.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/read.cpp
@@ -10,10 +10,10 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/sanitizer.h" // for MSAN_UNPOISON
-#include "src/errno/libc_errno.h"
-#include <sys/syscall.h> // For syscall numbers.
+#include <sys/syscall.h>                    // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/external/llvm-project/libc/src/unistd/linux/readlink.cpp b/external/llvm-project/libc/src/unistd/linux/readlink.cpp
index 2055e6b3400f..b297a41ca37b 100644
--- a/external/llvm-project/libc/src/unistd/linux/readlink.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/readlink.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/readlinkat.cpp b/external/llvm-project/libc/src/unistd/linux/readlinkat.cpp
index e5e4d0d39bc9..cd0dcb8e0ff0 100644
--- a/external/llvm-project/libc/src/unistd/linux/readlinkat.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/readlinkat.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/rmdir.cpp b/external/llvm-project/libc/src/unistd/linux/rmdir.cpp
index 075af12af64c..eca6e954ef89 100644
--- a/external/llvm-project/libc/src/unistd/linux/rmdir.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/rmdir.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/unistd/linux/symlink.cpp b/external/llvm-project/libc/src/unistd/linux/symlink.cpp
index 9e1b2886ea0f..3f43de19d2f4 100644
--- a/external/llvm-project/libc/src/unistd/linux/symlink.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/symlink.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/unistd/linux/symlinkat.cpp b/external/llvm-project/libc/src/unistd/linux/symlinkat.cpp
index bcf2d0f8cc05..8cee172f39df 100644
--- a/external/llvm-project/libc/src/unistd/linux/symlinkat.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/symlinkat.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/syscall.cpp b/external/llvm-project/libc/src/unistd/linux/syscall.cpp
index 5394bff46adf..0f7b3da88d62 100644
--- a/external/llvm-project/libc/src/unistd/linux/syscall.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/syscall.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <stdarg.h>
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/sysconf.cpp b/external/llvm-project/libc/src/unistd/linux/sysconf.cpp
index f785ff321c7d..03f224b15027 100644
--- a/external/llvm-project/libc/src/unistd/linux/sysconf.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/sysconf.cpp
@@ -11,8 +11,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/unistd_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/auxv/getauxval.h"
 #include <sys/auxv.h>
 
diff --git a/external/llvm-project/libc/src/unistd/linux/truncate.cpp b/external/llvm-project/libc/src/unistd/linux/truncate.cpp
index 8236edb480d1..6103d4b51350 100644
--- a/external/llvm-project/libc/src/unistd/linux/truncate.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/truncate.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/unistd_macros.h"
 #include <stdint.h>      // For uint64_t.
diff --git a/external/llvm-project/libc/src/unistd/linux/unlink.cpp b/external/llvm-project/libc/src/unistd/linux/unlink.cpp
index 72d8e2398e3d..5fde2600937b 100644
--- a/external/llvm-project/libc/src/unistd/linux/unlink.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/unlink.cpp
@@ -12,8 +12,8 @@
 #include "src/__support/common.h"
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/external/llvm-project/libc/src/unistd/linux/unlinkat.cpp b/external/llvm-project/libc/src/unistd/linux/unlinkat.cpp
index 4ed20f542f17..b2012c52b885 100644
--- a/external/llvm-project/libc/src/unistd/linux/unlinkat.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/unlinkat.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include "hdr/fcntl_macros.h"
 #include <sys/syscall.h> // For syscall numbers.
diff --git a/external/llvm-project/libc/src/unistd/linux/write.cpp b/external/llvm-project/libc/src/unistd/linux/write.cpp
index 99d5ab7e480b..eecb74429182 100644
--- a/external/llvm-project/libc/src/unistd/linux/write.cpp
+++ b/external/llvm-project/libc/src/unistd/linux/write.cpp
@@ -10,8 +10,8 @@
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 
 #include <sys/syscall.h> // For syscall numbers.
 
diff --git a/external/llvm-project/libc/src/unistd/windows/getentropy.cpp b/external/llvm-project/libc/src/unistd/windows/getentropy.cpp
index bfaec723ac63..e25a7a8fed40 100644
--- a/external/llvm-project/libc/src/unistd/windows/getentropy.cpp
+++ b/external/llvm-project/libc/src/unistd/windows/getentropy.cpp
@@ -9,7 +9,7 @@
 #include "src/unistd/getentropy.h"
 #include "hdr/errno_macros.h"
 #include "src/__support/common.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 #define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
diff --git a/external/llvm-project/libc/src/wchar/CMakeLists.txt b/external/llvm-project/libc/src/wchar/CMakeLists.txt
index 9db121762348..491dd5b34340 100644
--- a/external/llvm-project/libc/src/wchar/CMakeLists.txt
+++ b/external/llvm-project/libc/src/wchar/CMakeLists.txt
@@ -43,7 +43,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.size_t
     libc.hdr.types.wchar_t
-    libc.src.__support.wctype_utils
 )
 
 add_entrypoint_object(
@@ -54,7 +53,16 @@ add_entrypoint_object(
     wcschr.h
   DEPENDS
     libc.hdr.wchar_macros
-    libc.src.__support.wctype_utils
+)
+
+add_entrypoint_object(
+  wcscmp
+  SRCS
+    wcscmp.cpp
+  HDRS
+    wcscmp.h
+  DEPENDS
+    libc.hdr.wchar_macros
 )
 
 add_entrypoint_object(
@@ -65,7 +73,28 @@ add_entrypoint_object(
     wcspbrk.h
   DEPENDS
     libc.hdr.wchar_macros
-    libc.src.__support.wctype_utils
+    libc.src.__support.macros.null_check
+)
+
+add_entrypoint_object(
+  wcsrchr
+  SRCS
+    wcsrchr.cpp
+  HDRS
+    wcsrchr.h
+  DEPENDS
+    libc.hdr.wchar_macros
+)
+
+add_entrypoint_object(
+  wcsncmp
+  SRCS
+    wcsncmp.cpp
+  HDRS
+    wcsncmp.h
+  DEPENDS
+    libc.hdr.wchar_macros
+    libc.hdr.types.size_t
 )
 
 add_entrypoint_object(
@@ -77,7 +106,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.wchar_macros
     libc.hdr.types.size_t
-    libc.src.__support.wctype_utils
 )
 
 add_entrypoint_object(
@@ -89,7 +117,6 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.size_t
     libc.hdr.wchar_macros
-    libc.src.__support.wctype_utils
     libc.src.__support.macros.null_check
 )
 
@@ -102,5 +129,98 @@ add_entrypoint_object(
   DEPENDS
     libc.hdr.types.size_t
     libc.hdr.wchar_macros
-    libc.src.__support.wctype_utils
+)
+
+add_entrypoint_object(
+  wmemmove
+  SRCS
+    wmemmove.cpp
+  HDRS
+    wmemmove.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+    libc.src.__support.macros.null_check
+)
+
+add_entrypoint_object(
+  wcsncpy
+  SRCS
+    wcsncpy.cpp
+  HDRS
+    wcsncpy.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+    libc.src.string.string_utils
+)
+
+add_entrypoint_object(
+  wcscat
+  SRCS
+    wcscat.cpp
+  HDRS
+    wcscat.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+    libc.src.string.string_utils
+)
+
+add_entrypoint_object(
+  wcsstr
+  SRCS
+    wcsstr.cpp
+  HDRS
+    wcsstr.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+    libc.src.string.string_utils
+)
+
+add_entrypoint_object(
+  wcsncat
+  SRCS
+    wcsncat.cpp
+  HDRS
+    wcsncat.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+    libc.src.string.string_utils
+)
+
+add_entrypoint_object(
+  wmemchr
+  SRCS
+    wmemchr.cpp
+  HDRS
+    wmemchr.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+)
+
+add_entrypoint_object(
+  wmempcpy
+  SRCS
+    wmempcpy.cpp
+  HDRS
+    wmempcpy.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+)
+
+add_entrypoint_object(
+  wcscpy
+  SRCS
+    wcscpy.cpp
+  HDRS
+    wcscpy.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.wchar_macros
+    libc.src.string.string_utils
 )
diff --git a/external/llvm-project/libc/src/wchar/wcscat.cpp b/external/llvm-project/libc/src/wchar/wcscat.cpp
new file mode 100644
index 000000000000..50d90c0941c7
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcscat.cpp
@@ -0,0 +1,29 @@
+//===-- Implementation of wcscat ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcscat.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/string/string_utils.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(wchar_t *, wcscat,
+                   (wchar_t *__restrict s1, const wchar_t *__restrict s2)) {
+  size_t size_1 = internal::string_length(s1);
+  size_t i = 0;
+  for (; s2[i] != L'\0'; i++)
+    s1[size_1 + i] = s2[i];
+  s1[size_1 + i] = L'\0';
+  return s1;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/acosh.cpp b/external/llvm-project/libc/src/wchar/wcscat.h
similarity index 59%
rename from external/llvm-project/libc/src/math/amdgpu/acosh.cpp
rename to external/llvm-project/libc/src/wchar/wcscat.h
index 15c973449983..b0b4455983b4 100644
--- a/external/llvm-project/libc/src/math/amdgpu/acosh.cpp
+++ b/external/llvm-project/libc/src/wchar/wcscat.h
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU acosh function --------------------------===//
+//===-- Implementation header for wcscat ----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/acosh.h"
-#include "src/__support/common.h"
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSCAT_H
+#define LLVM_LIBC_SRC_WCHAR_WCSCAT_H
 
-#include "declarations.h"
+#include "hdr/types/wchar_t.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, acosh, (double x)) { return __ocml_acosh_f64(x); }
+wchar_t *wcscat(wchar_t *__restrict s1, const wchar_t *__restrict s2);
 
 } // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSCAT_H
diff --git a/external/llvm-project/libc/src/wchar/wcscmp.cpp b/external/llvm-project/libc/src/wchar/wcscmp.cpp
new file mode 100644
index 000000000000..48c8b3be6c70
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcscmp.cpp
@@ -0,0 +1,28 @@
+//===-- Implementation of wcscmp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcscmp.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/null_check.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, wcscmp, (const wchar_t *left, const wchar_t *right)) {
+  LIBC_CRASH_ON_NULLPTR(left);
+  LIBC_CRASH_ON_NULLPTR(right);
+
+  for (; *left && (*left == *right); ++left, ++right)
+    ;
+
+  return *left - *right;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/wchar/wcscmp.h b/external/llvm-project/libc/src/wchar/wcscmp.h
new file mode 100644
index 000000000000..af82d063f017
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcscmp.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcscmp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSCMP_H
+#define LLVM_LIBC_SRC_WCHAR_WCSCMP_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int wcscmp(const wchar_t *left, const wchar_t *right);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSCMP_H
diff --git a/external/llvm-project/libc/src/math/amdgpu/atan2f.cpp b/external/llvm-project/libc/src/wchar/wcscpy.cpp
similarity index 51%
rename from external/llvm-project/libc/src/math/amdgpu/atan2f.cpp
rename to external/llvm-project/libc/src/wchar/wcscpy.cpp
index 736c77d1cbce..01ba994cecbb 100644
--- a/external/llvm-project/libc/src/math/amdgpu/atan2f.cpp
+++ b/external/llvm-project/libc/src/wchar/wcscpy.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU atan2f function -------------------------===//
+//===-- Implementation of wcscpy ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,16 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/atan2f.h"
-#include "src/__support/common.h"
+#include "src/wchar/wcscpy.h"
 
-#include "declarations.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/string/string_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, atan2f, (float x, float y)) {
-  return __ocml_atan2_f32(x, y);
+LLVM_LIBC_FUNCTION(wchar_t *, wcscpy,
+                   (wchar_t *__restrict s1, const wchar_t *__restrict s2)) {
+  size_t size = internal::string_length(s2) + 1;
+  __builtin_memcpy(s1, s2, size * sizeof(wchar_t));
+  return s1;
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/wchar/wcscpy.h b/external/llvm-project/libc/src/wchar/wcscpy.h
new file mode 100644
index 000000000000..c3f0e5f2211c
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcscpy.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for wcscpy ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSCPY_H
+#define LLVM_LIBC_SRC_WCHAR_WCSCPY_H
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+wchar_t *wcscpy(wchar_t *__restrict s1, const wchar_t *__restrict s2);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSCPY_H
diff --git a/external/llvm-project/libc/src/wchar/wcsncat.cpp b/external/llvm-project/libc/src/wchar/wcsncat.cpp
new file mode 100644
index 000000000000..62595b4b5418
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcsncat.cpp
@@ -0,0 +1,31 @@
+//===-- Implementation of wcsncat -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcsncat.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/string/string_utils.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(wchar_t *, wcsncat,
+                   (wchar_t *__restrict s1, const wchar_t *__restrict s2,
+                    size_t n)) {
+  size_t size = internal::string_length(s1);
+  size_t i = 0;
+  for (; s2[i] && i < n; ++i)
+    s1[size + i] = s2[i];
+  // Appending null character to the end of the result.
+  s1[size + i] = L'\0';
+  return s1;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/remquo.cpp b/external/llvm-project/libc/src/wchar/wcsncat.h
similarity index 55%
rename from external/llvm-project/libc/src/math/amdgpu/remquo.cpp
rename to external/llvm-project/libc/src/wchar/wcsncat.h
index 42c908e91c6c..978645e90c6b 100644
--- a/external/llvm-project/libc/src/math/amdgpu/remquo.cpp
+++ b/external/llvm-project/libc/src/wchar/wcsncat.h
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU remquo function -------------------------===//
+//===-- Implementation header for wcsncat ---------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,19 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/remquo.h"
-#include "src/__support/common.h"
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSNCAT_H
+#define LLVM_LIBC_SRC_WCHAR_WCSNCAT_H
 
-#include "declarations.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, remquo, (double x, double y, int *quo)) {
-  int tmp;
-  double r = __ocml_remquo_f64(x, y, (gpu::Private<int> *)&tmp);
-  *quo = tmp;
-  return r;
-}
+wchar_t *wcsncat(wchar_t *__restrict s1, const wchar_t *__restrict s2,
+                 size_t n);
 
 } // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSNCAT_H
diff --git a/external/llvm-project/libc/src/wchar/wcsncmp.cpp b/external/llvm-project/libc/src/wchar/wcsncmp.cpp
new file mode 100644
index 000000000000..f2e052b3c9fe
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcsncmp.cpp
@@ -0,0 +1,37 @@
+//===-- Implementation of wcsncmp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcsncmp.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/null_check.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, wcsncmp,
+                   (const wchar_t *left, const wchar_t *right, size_t n)) {
+  LIBC_CRASH_ON_NULLPTR(left);
+  LIBC_CRASH_ON_NULLPTR(right);
+
+  if (n == 0)
+    return 0;
+
+  auto comp = [](wchar_t l, wchar_t r) -> int { return l - r; };
+
+  for (; n > 1; --n, ++left, ++right) {
+    wchar_t lc = *left;
+    if (!comp(lc, '\0') || comp(lc, *right))
+      break;
+  }
+  return comp(*reinterpret_cast<const wchar_t *>(left),
+              *reinterpret_cast<const wchar_t *>(right));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/wchar/wcsncmp.h b/external/llvm-project/libc/src/wchar/wcsncmp.h
new file mode 100644
index 000000000000..0b4187e730e2
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcsncmp.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcsncmp ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSNCMP_H
+#define LLVM_LIBC_SRC_WCHAR_WCSNCMP_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int wcsncmp(const wchar_t *left, const wchar_t *right, size_t n);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSNCMP_H
diff --git a/external/llvm-project/libc/src/wchar/wcsncpy.cpp b/external/llvm-project/libc/src/wchar/wcsncpy.cpp
new file mode 100644
index 000000000000..7ad6730cd776
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcsncpy.cpp
@@ -0,0 +1,31 @@
+//===-- Implementation of wcsncpy -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcsncpy.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(wchar_t *, wcsncpy,
+                   (wchar_t *__restrict s1, const wchar_t *__restrict s2,
+                    size_t n)) {
+  size_t i = 0;
+  // Copy up until \0 is found.
+  for (; i < n && s2[i] != L'\0'; ++i)
+    s1[i] = s2[i];
+  // When s2 is shorter than n, append \0.
+  for (; i < n; ++i)
+    s1[i] = L'\0';
+  return s1;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/wchar/wcsncpy.h b/external/llvm-project/libc/src/wchar/wcsncpy.h
new file mode 100644
index 000000000000..06c23f24bcde
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcsncpy.h
@@ -0,0 +1,23 @@
+//===-- Implementation header for wcsncpy ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSNCPY_H
+#define LLVM_LIBC_SRC_WCHAR_WCSNCPY_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+wchar_t *wcsncpy(wchar_t *__restrict s1, const wchar_t *__restrict s2,
+                 size_t n);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSNCPY_H
diff --git a/external/llvm-project/libc/src/wchar/wcspbrk.cpp b/external/llvm-project/libc/src/wchar/wcspbrk.cpp
index bf305a5dbe12..a00ba9979a48 100644
--- a/external/llvm-project/libc/src/wchar/wcspbrk.cpp
+++ b/external/llvm-project/libc/src/wchar/wcspbrk.cpp
@@ -10,6 +10,7 @@
 
 #include "hdr/types/wchar_t.h"
 #include "src/__support/common.h"
+#include "src/__support/macros/null_check.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -23,6 +24,9 @@ bool contains_char(const wchar_t *str, wchar_t target) {
 
 LLVM_LIBC_FUNCTION(const wchar_t *, wcspbrk,
                    (const wchar_t *src, const wchar_t *breakset)) {
+  LIBC_CRASH_ON_NULLPTR(src);
+  LIBC_CRASH_ON_NULLPTR(breakset);
+
   // currently O(n * m), can be further optimized to O(n + m) with a hash set
   for (int src_idx = 0; src[src_idx] != 0; src_idx++)
     if (contains_char(breakset, src[src_idx]))
diff --git a/external/llvm-project/libc/src/math/amdgpu/fdim.cpp b/external/llvm-project/libc/src/wchar/wcsrchr.cpp
similarity index 50%
rename from external/llvm-project/libc/src/math/amdgpu/fdim.cpp
rename to external/llvm-project/libc/src/wchar/wcsrchr.cpp
index 8ade0b21503b..bb4e373bb1ce 100644
--- a/external/llvm-project/libc/src/math/amdgpu/fdim.cpp
+++ b/external/llvm-project/libc/src/wchar/wcsrchr.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the fdim function for GPU -----------------------===//
+//===-- Implementation of wcsrchr -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,16 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/fdim.h"
-#include "src/__support/common.h"
+#include "src/wchar/wcsrchr.h"
 
-#include "declarations.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, fdim, (double x, double y)) {
-  return __ocml_fdim_f64(x, y);
+LLVM_LIBC_FUNCTION(const wchar_t *, wcsrchr, (const wchar_t *s, wchar_t c)) {
+  LIBC_CRASH_ON_NULLPTR(s);
+
+  const wchar_t *last_occurrence = nullptr;
+  while (true) {
+    if (*s == c)
+      last_occurrence = s;
+    if (*s == L'\0')
+      return last_occurrence;
+    ++s;
+  }
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/acos.cpp b/external/llvm-project/libc/src/wchar/wcsrchr.h
similarity index 61%
rename from external/llvm-project/libc/src/math/amdgpu/acos.cpp
rename to external/llvm-project/libc/src/wchar/wcsrchr.h
index de870f207326..5e9d8c3c2db9 100644
--- a/external/llvm-project/libc/src/math/amdgpu/acos.cpp
+++ b/external/llvm-project/libc/src/wchar/wcsrchr.h
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU acos function ---------------------------===//
+//===-- Implementation header for wcsrchr ---------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/acos.h"
-#include "src/__support/common.h"
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSRCHR_H
+#define LLVM_LIBC_SRC_WCHAR_WCSRCHR_H
 
-#include "declarations.h"
+#include "hdr/types/wchar_t.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, acos, (double x)) { return __ocml_acos_f64(x); }
+const wchar_t *wcsrchr(const wchar_t *s, wchar_t c);
 
 } // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSRCHR_H
diff --git a/external/llvm-project/libc/src/wchar/wcsstr.cpp b/external/llvm-project/libc/src/wchar/wcsstr.cpp
new file mode 100644
index 000000000000..961835ae3965
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wcsstr.cpp
@@ -0,0 +1,38 @@
+//===-- Implementation of wcsstr ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcsstr.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/string/string_utils.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(const wchar_t *, wcsstr,
+                   (const wchar_t *s1, const wchar_t *s2)) {
+  size_t s1_len = internal::string_length(s1);
+  size_t s2_len = internal::string_length(s2);
+  if (s2_len == 0)
+    return s1;
+  if (s2_len > s1_len)
+    return nullptr;
+  for (size_t i = 0; i <= (s1_len - s2_len); ++i) {
+    size_t j = 0;
+    // j will increment until the characters don't match or end of string.
+    for (; j < s2_len && s1[i + j] == s2[j]; ++j)
+      ;
+    if (j == s2_len)
+      return (s1 + i);
+  }
+  return nullptr;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/math/amdgpu/acoshf.cpp b/external/llvm-project/libc/src/wchar/wcsstr.h
similarity index 60%
rename from external/llvm-project/libc/src/math/amdgpu/acoshf.cpp
rename to external/llvm-project/libc/src/wchar/wcsstr.h
index 79e71b02e719..af054d8495a4 100644
--- a/external/llvm-project/libc/src/math/amdgpu/acoshf.cpp
+++ b/external/llvm-project/libc/src/wchar/wcsstr.h
@@ -1,4 +1,4 @@
-//===-- Implementation of the acoshf function for GPU ---------------------===//
+//===-- Implementation header for wcsstr ----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/acoshf.h"
-#include "src/__support/common.h"
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSSTR_H
+#define LLVM_LIBC_SRC_WCHAR_WCSSTR_H
 
-#include "declarations.h"
+#include "hdr/types/wchar_t.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, acoshf, (float x)) { return __ocml_acosh_f32(x); }
+const wchar_t *wcsstr(const wchar_t *s1, const wchar_t *s2);
 
 } // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSSTR_H
diff --git a/external/llvm-project/libc/src/math/amdgpu/atan2.cpp b/external/llvm-project/libc/src/wchar/wmemchr.cpp
similarity index 55%
rename from external/llvm-project/libc/src/math/amdgpu/atan2.cpp
rename to external/llvm-project/libc/src/wchar/wmemchr.cpp
index f5907504cb36..8a1cd6ce4389 100644
--- a/external/llvm-project/libc/src/math/amdgpu/atan2.cpp
+++ b/external/llvm-project/libc/src/wchar/wmemchr.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU atan2 function --------------------------===//
+//===-- Implementation of wmemchr -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,16 +6,22 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/atan2.h"
-#include "src/__support/common.h"
+#include "src/wchar/wmemchr.h"
 
-#include "declarations.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(double, atan2, (double x, double y)) {
-  return __ocml_atan2_f64(x, y);
+LLVM_LIBC_FUNCTION(const wchar_t *, wmemchr,
+                   (const wchar_t *s, wchar_t c, size_t n)) {
+  size_t i = 0;
+  for (; i < n; ++i)
+    if (s[i] == c)
+      return (s + i);
+  return nullptr;
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/wchar/wmemchr.h b/external/llvm-project/libc/src/wchar/wmemchr.h
new file mode 100644
index 000000000000..55412907ddca
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wmemchr.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wmemchr ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WMEMCHR_H
+#define LLVM_LIBC_SRC_WCHAR_WMEMCHR_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+const wchar_t *wmemchr(const wchar_t *s, wchar_t c, size_t n);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WMEMCHR_H
diff --git a/external/llvm-project/libc/src/wchar/wmemcpy.cpp b/external/llvm-project/libc/src/wchar/wmemcpy.cpp
index 56708d6cee49..bf92309b2094 100644
--- a/external/llvm-project/libc/src/wchar/wmemcpy.cpp
+++ b/external/llvm-project/libc/src/wchar/wmemcpy.cpp
@@ -12,14 +12,13 @@
 #include "hdr/types/wchar_t.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "src/string/memory_utils/inline_memcpy.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(wchar_t *, wmemcpy,
                    (wchar_t *__restrict s1, const wchar_t *__restrict s2,
                     size_t n)) {
-  inline_memcpy(s1, s2, n * sizeof(wchar_t));
+  __builtin_memcpy(s1, s2, n * sizeof(wchar_t));
   return s1;
 }
 
diff --git a/external/llvm-project/libc/src/wchar/wmemmove.cpp b/external/llvm-project/libc/src/wchar/wmemmove.cpp
new file mode 100644
index 000000000000..3282077003bd
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wmemmove.cpp
@@ -0,0 +1,27 @@
+//===-- Implementation of wmemmove ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wmemmove.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/null_check.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(wchar_t *, wmemmove,
+                   (wchar_t * dest, const wchar_t *src, size_t n)) {
+  LIBC_CRASH_ON_NULLPTR(dest);
+  LIBC_CRASH_ON_NULLPTR(src);
+
+  __builtin_memmove(dest, src, n * sizeof(wchar_t));
+  return dest;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/wchar/wmemmove.h b/external/llvm-project/libc/src/wchar/wmemmove.h
new file mode 100644
index 000000000000..b4c31ac7b397
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wmemmove.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wmemmove --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WMEMMOVE_H
+#define LLVM_LIBC_SRC_WCHAR_WMEMMOVE_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+wchar_t *wmemmove(wchar_t *dest, const wchar_t *src, size_t n);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WMEMMOVE_H
diff --git a/external/llvm-project/libc/src/math/amdgpu/fdimf.cpp b/external/llvm-project/libc/src/wchar/wmempcpy.cpp
similarity index 50%
rename from external/llvm-project/libc/src/math/amdgpu/fdimf.cpp
rename to external/llvm-project/libc/src/wchar/wmempcpy.cpp
index ed3855e5ce2d..21e16210a757 100644
--- a/external/llvm-project/libc/src/math/amdgpu/fdimf.cpp
+++ b/external/llvm-project/libc/src/wchar/wmempcpy.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the fdimf function for GPU ----------------------===//
+//===-- Implementation of wmempcpy ----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,16 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/fdimf.h"
-#include "src/__support/common.h"
+#include "src/wchar/wmempcpy.h"
 
-#include "declarations.h"
-#include "src/__support/macros/config.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, fdimf, (float x, float y)) {
-  return __ocml_fdim_f32(x, y);
+LLVM_LIBC_FUNCTION(wchar_t *, wmempcpy,
+                   (wchar_t *__restrict to, const wchar_t *__restrict from,
+                    size_t size)) {
+  __builtin_memcpy(to, from, size * sizeof(wchar_t));
+  return reinterpret_cast<wchar_t *>(to) + size;
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/src/wchar/wmempcpy.h b/external/llvm-project/libc/src/wchar/wmempcpy.h
new file mode 100644
index 000000000000..580674b1be80
--- /dev/null
+++ b/external/llvm-project/libc/src/wchar/wmempcpy.h
@@ -0,0 +1,23 @@
+//===-- Implementation header for wmempcpy---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WMEMPCPY_H
+#define LLVM_LIBC_SRC_WCHAR_WMEMPCPY_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+wchar_t *wmempcpy(wchar_t *__restrict from, const wchar_t *__restrict s2,
+                  size_t n);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WMEMPCPY_H
diff --git a/external/llvm-project/libc/test/IntegrationTest/test.h b/external/llvm-project/libc/test/IntegrationTest/test.h
index 5be66d9edff0..24c007d2e12e 100644
--- a/external/llvm-project/libc/test/IntegrationTest/test.h
+++ b/external/llvm-project/libc/test/IntegrationTest/test.h
@@ -68,12 +68,9 @@
 ////////////////////////////////////////////////////////////////////////////////
 // Errno checks.
 
-#define ASSERT_ERRNO_EQ(VAL)                                                   \
-  ASSERT_EQ(VAL, static_cast<int>(LIBC_NAMESPACE::libc_errno))
-#define ASSERT_ERRNO_SUCCESS()                                                 \
-  ASSERT_EQ(0, static_cast<int>(LIBC_NAMESPACE::libc_errno))
-#define ASSERT_ERRNO_FAILURE()                                                 \
-  ASSERT_NE(0, static_cast<int>(LIBC_NAMESPACE::libc_errno))
+#define ASSERT_ERRNO_EQ(VAL) ASSERT_EQ(VAL, static_cast<int>(libc_errno))
+#define ASSERT_ERRNO_SUCCESS() ASSERT_EQ(0, static_cast<int>(libc_errno))
+#define ASSERT_ERRNO_FAILURE() ASSERT_NE(0, static_cast<int>(libc_errno))
 
 // Integration tests are compiled with -ffreestanding which stops treating
 // the main function as a non-overloadable special function. Hence, we use a
diff --git a/external/llvm-project/libc/test/UnitTest/CMakeLists.txt b/external/llvm-project/libc/test/UnitTest/CMakeLists.txt
index b0a3a7431c22..c32809da577d 100644
--- a/external/llvm-project/libc/test/UnitTest/CMakeLists.txt
+++ b/external/llvm-project/libc/test/UnitTest/CMakeLists.txt
@@ -35,7 +35,7 @@ function(add_unittest_framework_library name)
   else()
     _get_common_test_compile_options(compile_options "" "")
     target_compile_options(${name}.unit PRIVATE ${compile_options})
-endif()
+  endif()
 
   _get_hermetic_test_compile_options(compile_options "")
   target_include_directories(${name}.hermetic PRIVATE ${LIBC_INCLUDE_DIR})
diff --git a/external/llvm-project/libc/test/UnitTest/ErrnoCheckingTest.h b/external/llvm-project/libc/test/UnitTest/ErrnoCheckingTest.h
index 3d3b72f80544..4b7ff452f409 100644
--- a/external/llvm-project/libc/test/UnitTest/ErrnoCheckingTest.h
+++ b/external/llvm-project/libc/test/UnitTest/ErrnoCheckingTest.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIBC_TEST_UNITTEST_ERRNOCHECKINGTEST_H
 #define LLVM_LIBC_TEST_UNITTEST_ERRNOCHECKINGTEST_H
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "test/UnitTest/Test.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -25,7 +25,7 @@ class ErrnoCheckingTest : public Test {
 public:
   void SetUp() override {
     Test::SetUp();
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
   }
 
   void TearDown() override {
diff --git a/external/llvm-project/libc/test/UnitTest/ErrnoSetterMatcher.h b/external/llvm-project/libc/test/UnitTest/ErrnoSetterMatcher.h
index c6eadd25858e..212b7a8f83e7 100644
--- a/external/llvm-project/libc/test/UnitTest/ErrnoSetterMatcher.h
+++ b/external/llvm-project/libc/test/UnitTest/ErrnoSetterMatcher.h
@@ -12,9 +12,9 @@
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/fpbits_str.h"
 #include "src/__support/StringUtil/error_to_string.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/architectures.h"
-#include "src/errno/libc_errno.h"
 #include "test/UnitTest/Test.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -114,8 +114,8 @@ template <typename T> class ErrnoSetterMatcher : public Matcher<T> {
 
   bool match(T got) {
     actual_return = got;
-    actual_errno = LIBC_NAMESPACE::libc_errno;
-    LIBC_NAMESPACE::libc_errno = 0;
+    actual_errno = libc_errno;
+    libc_errno = 0;
     if constexpr (ignore_errno())
       return return_cmp.compare(actual_return);
     else
diff --git a/external/llvm-project/libc/test/UnitTest/FPMatcher.h b/external/llvm-project/libc/test/UnitTest/FPMatcher.h
index 21b8a45b0726..da15cf2907f7 100644
--- a/external/llvm-project/libc/test/UnitTest/FPMatcher.h
+++ b/external/llvm-project/libc/test/UnitTest/FPMatcher.h
@@ -279,8 +279,8 @@ struct ModifyMXCSR {
 #define EXPECT_MATH_ERRNO(expected)                                            \
   do {                                                                         \
     if (math_errhandling & MATH_ERRNO) {                                       \
-      int actual = LIBC_NAMESPACE::libc_errno;                                 \
-      LIBC_NAMESPACE::libc_errno = 0;                                          \
+      int actual = libc_errno;                                                 \
+      libc_errno = 0;                                                          \
       EXPECT_EQ(actual, expected);                                             \
     }                                                                          \
   } while (0)
@@ -288,8 +288,8 @@ struct ModifyMXCSR {
 #define ASSERT_MATH_ERRNO(expected)                                            \
   do {                                                                         \
     if (math_errhandling & MATH_ERRNO) {                                       \
-      int actual = LIBC_NAMESPACE::libc_errno;                                 \
-      LIBC_NAMESPACE::libc_errno = 0;                                          \
+      int actual = libc_errno;                                                 \
+      libc_errno = 0;                                                          \
       ASSERT_EQ(actual, expected);                                             \
     }                                                                          \
   } while (0)
diff --git a/external/llvm-project/libc/test/UnitTest/Test.h b/external/llvm-project/libc/test/UnitTest/Test.h
index 95d48f40914e..a5a2a3c7cf58 100644
--- a/external/llvm-project/libc/test/UnitTest/Test.h
+++ b/external/llvm-project/libc/test/UnitTest/Test.h
@@ -42,15 +42,14 @@
 
 #define ASSERT_ERRNO_EQ(VAL)                                                   \
   do {                                                                         \
-    ASSERT_EQ(VAL, static_cast<int>(LIBC_NAMESPACE::libc_errno));              \
-    LIBC_NAMESPACE::libc_errno = 0;                                            \
+    ASSERT_EQ(VAL, static_cast<int>(libc_errno));                              \
+    libc_errno = 0;                                                            \
   } while (0)
-#define ASSERT_ERRNO_SUCCESS()                                                 \
-  ASSERT_EQ(0, static_cast<int>(LIBC_NAMESPACE::libc_errno))
+#define ASSERT_ERRNO_SUCCESS() ASSERT_EQ(0, static_cast<int>(libc_errno))
 #define ASSERT_ERRNO_FAILURE()                                                 \
   do {                                                                         \
-    ASSERT_NE(0, static_cast<int>(LIBC_NAMESPACE::libc_errno));                \
-    LIBC_NAMESPACE::libc_errno = 0;                                            \
+    ASSERT_NE(0, static_cast<int>(libc_errno));                                \
+    libc_errno = 0;                                                            \
   } while (0)
 
 #endif // LLVM_LIBC_TEST_UNITTEST_TEST_H
diff --git a/external/llvm-project/libc/test/integration/src/pthread/pthread_create_test.cpp b/external/llvm-project/libc/test/integration/src/pthread/pthread_create_test.cpp
index 29da4d5c3c8d..aecbad6514aa 100644
--- a/external/llvm-project/libc/test/integration/src/pthread/pthread_create_test.cpp
+++ b/external/llvm-project/libc/test/integration/src/pthread/pthread_create_test.cpp
@@ -29,7 +29,7 @@
 #include "src/__support/CPP/new.h"
 #include "src/__support/threads/thread.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 #include "test/IntegrationTest/test.h"
 
@@ -332,7 +332,7 @@ static void run_failure_tests() {
 }
 
 TEST_MAIN() {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   run_success_tests();
   run_failure_tests();
   return 0;
diff --git a/external/llvm-project/libc/test/integration/src/pthread/pthread_join_test.cpp b/external/llvm-project/libc/test/integration/src/pthread/pthread_join_test.cpp
index 994fa57a6b33..5d0bcd8e2365 100644
--- a/external/llvm-project/libc/test/integration/src/pthread/pthread_join_test.cpp
+++ b/external/llvm-project/libc/test/integration/src/pthread/pthread_join_test.cpp
@@ -9,7 +9,7 @@
 #include "src/pthread/pthread_create.h"
 #include "src/pthread/pthread_join.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 #include "test/IntegrationTest/test.h"
 #include <pthread.h>
@@ -25,7 +25,7 @@ static void nullJoinTest() {
 }
 
 TEST_MAIN() {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   nullJoinTest();
   return 0;
 }
diff --git a/external/llvm-project/libc/test/integration/src/pthread/pthread_name_test.cpp b/external/llvm-project/libc/test/integration/src/pthread/pthread_name_test.cpp
index 37ceceee880d..35dd3b165e0e 100644
--- a/external/llvm-project/libc/test/integration/src/pthread/pthread_name_test.cpp
+++ b/external/llvm-project/libc/test/integration/src/pthread/pthread_name_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/CPP/string_view.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/pthread/pthread_create.h"
 #include "src/pthread/pthread_getname_np.h"
 #include "src/pthread/pthread_join.h"
diff --git a/external/llvm-project/libc/test/integration/src/unistd/getcwd_test.cpp b/external/llvm-project/libc/test/integration/src/unistd/getcwd_test.cpp
index 551768187bf0..1b321b01e931 100644
--- a/external/llvm-project/libc/test/integration/src/unistd/getcwd_test.cpp
+++ b/external/llvm-project/libc/test/integration/src/unistd/getcwd_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/CPP/string_view.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/stdlib/getenv.h"
 #include "src/unistd/getcwd.h"
 
@@ -31,12 +31,12 @@ TEST_MAIN(int argc, char **argv, char **envp) {
   cwd = LIBC_NAMESPACE::getcwd(buffer, 0);
   ASSERT_TRUE(cwd == nullptr);
   ASSERT_ERRNO_EQ(EINVAL);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   // Insufficient size
   cwd = LIBC_NAMESPACE::getcwd(buffer, 2);
   ASSERT_TRUE(cwd == nullptr);
-  int err = LIBC_NAMESPACE::libc_errno;
+  int err = libc_errno;
   ASSERT_EQ(err, ERANGE);
 
   return 0;
diff --git a/external/llvm-project/libc/test/integration/startup/gpu/rpc_stream_test.cpp b/external/llvm-project/libc/test/integration/startup/gpu/rpc_stream_test.cpp
index aba5b0b60d16..3e8132862914 100644
--- a/external/llvm-project/libc/test/integration/startup/gpu/rpc_stream_test.cpp
+++ b/external/llvm-project/libc/test/integration/startup/gpu/rpc_stream_test.cpp
@@ -12,6 +12,7 @@
 #include "src/__support/integer_to_string.h"
 #include "src/string/memory_utils/inline_memcmp.h"
 #include "src/string/memory_utils/inline_memcpy.h"
+#include "src/string/memory_utils/inline_memset.h"
 #include "src/string/string_utils.h"
 #include "test/IntegrationTest/test.h"
 
diff --git a/external/llvm-project/libc/test/integration/startup/linux/tls_test.cpp b/external/llvm-project/libc/test/integration/startup/linux/tls_test.cpp
index ef9fd9fcb7ff..de3bd06c39cf 100644
--- a/external/llvm-project/libc/test/integration/startup/linux/tls_test.cpp
+++ b/external/llvm-project/libc/test/integration/startup/linux/tls_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sys/mman/mmap.h"
 #include "test/IntegrationTest/test.h"
 
diff --git a/external/llvm-project/libc/test/src/__support/CMakeLists.txt b/external/llvm-project/libc/test/src/__support/CMakeLists.txt
index c1736c8fe59e..4fb0dae86e5c 100644
--- a/external/llvm-project/libc/test/src/__support/CMakeLists.txt
+++ b/external/llvm-project/libc/test/src/__support/CMakeLists.txt
@@ -127,7 +127,6 @@ add_libc_test(
     libc.src.__support.integer_literals
     libc.src.__support.str_to_float
     libc.src.__support.uint128
-    libc.src.errno.errno
 )
 
 
@@ -140,7 +139,6 @@ add_libc_test(
   DEPENDS
     libc.src.__support.integer_literals
     libc.src.__support.str_to_integer
-    libc.src.errno.errno
 )
 
 add_libc_test(
diff --git a/external/llvm-project/libc/test/src/__support/str_to_double_test.cpp b/external/llvm-project/libc/test/src/__support/str_to_double_test.cpp
index ccfa44f12d8e..dc503aa16f08 100644
--- a/external/llvm-project/libc/test/src/__support/str_to_double_test.cpp
+++ b/external/llvm-project/libc/test/src/__support/str_to_double_test.cpp
@@ -99,7 +99,6 @@ TEST(LlvmLibcStrToDblTest, SimpleDecimalConversionExtraTypes) {
   uint64_t double_output_mantissa = 0;
   uint32_t output_exp2 = 0;
 
-  LIBC_NAMESPACE::libc_errno = 0;
   auto double_result =
       internal::simple_decimal_conversion<double>("123456789012345678900");
 
diff --git a/external/llvm-project/libc/test/src/__support/str_to_float_test.cpp b/external/llvm-project/libc/test/src/__support/str_to_float_test.cpp
index 66f7db742eb4..03ae80fc2ee3 100644
--- a/external/llvm-project/libc/test/src/__support/str_to_float_test.cpp
+++ b/external/llvm-project/libc/test/src/__support/str_to_float_test.cpp
@@ -55,7 +55,6 @@ TEST(LlvmLibcStrToFltTest, SimpleDecimalConversionExtraTypes) {
   uint32_t float_output_mantissa = 0;
   uint32_t output_exp2 = 0;
 
-  LIBC_NAMESPACE::libc_errno = 0;
   auto float_result =
       internal::simple_decimal_conversion<float>("123456789012345678900");
   float_output_mantissa = float_result.num.mantissa;
diff --git a/external/llvm-project/libc/test/src/__support/str_to_fp_test.h b/external/llvm-project/libc/test/src/__support/str_to_fp_test.h
index c7bc57b845fe..9b4844d410db 100644
--- a/external/llvm-project/libc/test/src/__support/str_to_fp_test.h
+++ b/external/llvm-project/libc/test/src/__support/str_to_fp_test.h
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_float.h"
 #include "src/__support/uint128.h"
-#include "src/errno/libc_errno.h"
 
 #include "test/UnitTest/Test.h"
 
@@ -67,7 +67,6 @@ template <typename T> struct LlvmLibcStrToFloatTest : public testing::Test {
                                       const int expectedErrno = 0) {
     StorageType actual_output_mantissa = 0;
     uint32_t actual_output_exp2 = 0;
-    LIBC_NAMESPACE::libc_errno = 0;
 
     auto result = internal::simple_decimal_conversion<T>(numStart);
 
diff --git a/external/llvm-project/libc/test/src/__support/str_to_integer_test.cpp b/external/llvm-project/libc/test/src/__support/str_to_integer_test.cpp
index 34b645b4b38c..40cb76a8bd6a 100644
--- a/external/llvm-project/libc/test/src/__support/str_to_integer_test.cpp
+++ b/external/llvm-project/libc/test/src/__support/str_to_integer_test.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 #include <stddef.h>
 
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/dirent/dirent_test.cpp b/external/llvm-project/libc/test/src/dirent/dirent_test.cpp
index 41f522a6a75f..3f0095ca5ebe 100644
--- a/external/llvm-project/libc/test/src/dirent/dirent_test.cpp
+++ b/external/llvm-project/libc/test/src/dirent/dirent_test.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/CPP/string_view.h"
+#include "src/__support/libc_errno.h"
 #include "src/dirent/closedir.h"
 #include "src/dirent/dirfd.h"
 #include "src/dirent/opendir.h"
 #include "src/dirent/readdir.h"
-#include "src/errno/libc_errno.h"
 
 #include "test/UnitTest/Test.h"
 
@@ -55,17 +55,17 @@ TEST(LlvmLibcDirentTest, SimpleOpenAndRead) {
 }
 
 TEST(LlvmLibcDirentTest, OpenNonExistentDir) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ::DIR *dir = LIBC_NAMESPACE::opendir("___xyz123__.non_existent__");
   ASSERT_TRUE(dir == nullptr);
   ASSERT_ERRNO_EQ(ENOENT);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 }
 
 TEST(LlvmLibcDirentTest, OpenFile) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ::DIR *dir = LIBC_NAMESPACE::opendir("testdata/file1.txt");
   ASSERT_TRUE(dir == nullptr);
   ASSERT_ERRNO_EQ(ENOTDIR);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 }
diff --git a/external/llvm-project/libc/test/src/errno/errno_test.cpp b/external/llvm-project/libc/test/src/errno/errno_test.cpp
index b0db22a85f3b..de82b0077f17 100644
--- a/external/llvm-project/libc/test/src/errno/errno_test.cpp
+++ b/external/llvm-project/libc/test/src/errno/errno_test.cpp
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcErrnoTest, Basic) {
   int test_val = 123;
-  LIBC_NAMESPACE::libc_errno = test_val;
+  libc_errno = test_val;
   ASSERT_ERRNO_EQ(test_val);
 }
diff --git a/external/llvm-project/libc/test/src/fcntl/creat_test.cpp b/external/llvm-project/libc/test/src/fcntl/creat_test.cpp
index 4c9d2cbc33f4..d60c98493470 100644
--- a/external/llvm-project/libc/test/src/fcntl/creat_test.cpp
+++ b/external/llvm-project/libc/test/src/fcntl/creat_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/fcntl/creat.h"
 #include "src/fcntl/open.h"
 #include "src/unistd/close.h"
diff --git a/external/llvm-project/libc/test/src/fcntl/fcntl_test.cpp b/external/llvm-project/libc/test/src/fcntl/fcntl_test.cpp
index 1a21afe51085..082c42481777 100644
--- a/external/llvm-project/libc/test/src/fcntl/fcntl_test.cpp
+++ b/external/llvm-project/libc/test/src/fcntl/fcntl_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/fcntl_macros.h"
 #include "hdr/stdio_macros.h"
 #include "hdr/types/struct_flock.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/fcntl/fcntl.h"
 #include "src/fcntl/open.h"
 #include "src/unistd/close.h"
@@ -166,7 +166,7 @@ TEST(LlvmLibcFcntlTest, UseAfterClose) {
 }
 
 TEST(LlvmLibcFcntlTest, SetGetOwnerTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   pid_t pid = LIBC_NAMESPACE::getpid();
   ASSERT_GT(pid, -1);
diff --git a/external/llvm-project/libc/test/src/fcntl/openat_test.cpp b/external/llvm-project/libc/test/src/fcntl/openat_test.cpp
index 213b074799c8..1997476f16a6 100644
--- a/external/llvm-project/libc/test/src/fcntl/openat_test.cpp
+++ b/external/llvm-project/libc/test/src/fcntl/openat_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/fcntl/open.h"
 #include "src/fcntl/openat.h"
 #include "src/unistd/close.h"
diff --git a/external/llvm-project/libc/test/src/math/RoundToIntegerTest.h b/external/llvm-project/libc/test/src/math/RoundToIntegerTest.h
index 77b465a3a0e6..6af9cfea0e0a 100644
--- a/external/llvm-project/libc/test/src/math/RoundToIntegerTest.h
+++ b/external/llvm-project/libc/test/src/math/RoundToIntegerTest.h
@@ -55,7 +55,7 @@ class RoundToIntegerTestTemplate
 
   void test_one_input(RoundToIntegerFunc func, FloatType input,
                       IntType expected, bool expectError) {
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
 
     ASSERT_EQ(func(input), expected);
diff --git a/external/llvm-project/libc/test/src/math/acosf_test.cpp b/external/llvm-project/libc/test/src/math/acosf_test.cpp
index 2e4c8eb2ab96..aa0128fee999 100644
--- a/external/llvm-project/libc/test/src/math/acosf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/acosf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acosf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LlvmLibcAcosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAcosfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acosf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/acoshf16_test.cpp b/external/llvm-project/libc/test/src/math/acoshf16_test.cpp
index 7348018396bd..2eb95215e4e8 100644
--- a/external/llvm-project/libc/test/src/math/acoshf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/acoshf16_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acoshf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/acoshf_test.cpp b/external/llvm-project/libc/test/src/math/acoshf_test.cpp
index 18ed5a11d50a..3d3b827411a4 100644
--- a/external/llvm-project/libc/test/src/math/acoshf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/acoshf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acoshf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ using LlvmLibcAcoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcAcoshfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acoshf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/asin_test.cpp b/external/llvm-project/libc/test/src/math/asin_test.cpp
index 385e341318ae..03ae963e9f92 100644
--- a/external/llvm-project/libc/test/src/math/asin_test.cpp
+++ b/external/llvm-project/libc/test/src/math/asin_test.cpp
@@ -38,7 +38,7 @@ TEST_F(LlvmLibcAsinTest, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf())
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::asin(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/asinf_test.cpp b/external/llvm-project/libc/test/src/math/asinf_test.cpp
index 5197810d8bd5..1eaa6b8a5135 100644
--- a/external/llvm-project/libc/test/src/math/asinf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/asinf_test.cpp
@@ -9,7 +9,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/asinf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -22,7 +22,7 @@ using LlvmLibcAsinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcAsinfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::asinf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/asinhf_test.cpp b/external/llvm-project/libc/test/src/math/asinhf_test.cpp
index ac125c3520c4..8c78f939cabf 100644
--- a/external/llvm-project/libc/test/src/math/asinhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/asinhf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/asinhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ using LlvmLibcAsinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcAsinhfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::asinhf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/atan2f_test.cpp b/external/llvm-project/libc/test/src/math/atan2f_test.cpp
index 331f4281af83..50ab38208089 100644
--- a/external/llvm-project/libc/test/src/math/atan2f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/atan2f_test.cpp
@@ -81,7 +81,7 @@ TEST_F(LlvmLibcAtan2fTest, InFloatRange) {
         if (FPBits(w).is_nan() || FPBits(w).is_inf())
           continue;
 
-        LIBC_NAMESPACE::libc_errno = 0;
+        libc_errno = 0;
         float result = LIBC_NAMESPACE::atan2f(x, y);
         ++total_count;
         if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/atan_test.cpp b/external/llvm-project/libc/test/src/math/atan_test.cpp
index 7f52578b9efe..7fa0dffd607e 100644
--- a/external/llvm-project/libc/test/src/math/atan_test.cpp
+++ b/external/llvm-project/libc/test/src/math/atan_test.cpp
@@ -39,7 +39,7 @@ TEST_F(LlvmLibcAtanTest, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf())
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::atan(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/atanf_test.cpp b/external/llvm-project/libc/test/src/math/atanf_test.cpp
index 575ec89bd493..a4bdf1867c39 100644
--- a/external/llvm-project/libc/test/src/math/atanf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/atanf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/atanf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -23,7 +23,7 @@ namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 // TODO: This test needs to have its checks for exceptions, errno
 // tightened
 TEST_F(LlvmLibcAtanfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atanf(aNaN));
   // TODO: Uncomment these checks later, RoundingMode affects running
diff --git a/external/llvm-project/libc/test/src/math/atanhf_test.cpp b/external/llvm-project/libc/test/src/math/atanhf_test.cpp
index 8b9db1dfdd97..32272ef482ab 100644
--- a/external/llvm-project/libc/test/src/math/atanhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/atanhf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/atanhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -25,7 +25,7 @@ namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 // tightened https://github.com/llvm/llvm-project/issues/88819.
 TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) {
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atanhf(aNaN));
   // TODO: Uncomment these checks later, RoundingMode affects running
diff --git a/external/llvm-project/libc/test/src/math/cosf_test.cpp b/external/llvm-project/libc/test/src/math/cosf_test.cpp
index 2143c36f3d30..90dc8ff6a0ea 100644
--- a/external/llvm-project/libc/test/src/math/cosf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/cosf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/cosf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -23,7 +23,7 @@ using LlvmLibcCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcCosfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cosf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/coshf_test.cpp b/external/llvm-project/libc/test/src/math/coshf_test.cpp
index 0d1c322b8e62..bdaba50f1f14 100644
--- a/external/llvm-project/libc/test/src/math/coshf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/coshf_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/math_macros.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/coshf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -22,7 +22,7 @@ using LlvmLibcCoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcCoshfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::coshf(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -41,7 +41,7 @@ TEST_F(LlvmLibcCoshfTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcCoshfTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::coshf(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/cospif_test.cpp b/external/llvm-project/libc/test/src/math/cospif_test.cpp
index 37ec2516f6a3..cb88bfcade0d 100644
--- a/external/llvm-project/libc/test/src/math/cospif_test.cpp
+++ b/external/llvm-project/libc/test/src/math/cospif_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/cospif.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/src/math/sdcomp26094.h"
@@ -19,7 +19,7 @@ using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcCospifTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cospif(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/exp10_test.cpp b/external/llvm-project/libc/test/src/math/exp10_test.cpp
index 6fb1d2d9d925..6126e5f211ff 100644
--- a/external/llvm-project/libc/test/src/math/exp10_test.cpp
+++ b/external/llvm-project/libc/test/src/math/exp10_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -105,7 +105,7 @@ TEST_F(LlvmLibcExp10Test, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::exp10(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/exp10f_test.cpp b/external/llvm-project/libc/test/src/math/exp10f_test.cpp
index 001b37809d93..89915961c9b9 100644
--- a/external/llvm-project/libc/test/src/math/exp10f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/exp10f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ using LlvmLibcExp10fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcExp10fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp10f(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExp10fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp10fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp10f(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
@@ -55,7 +55,7 @@ TEST_F(LlvmLibcExp10fTest, Overflow) {
 }
 
 TEST_F(LlvmLibcExp10fTest, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       0.0f, LIBC_NAMESPACE::exp10f(FPBits(0xff7fffffU).get_val()),
       FE_UNDERFLOW);
@@ -97,7 +97,7 @@ TEST_F(LlvmLibcExp10fTest, TrickyInputs) {
       0x41200000, // x = 10.0f
   };
   for (int i = 0; i < N; ++i) {
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float x = FPBits(INPUTS[i]).get_val();
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp10, x,
                                    LIBC_NAMESPACE::exp10f(x), 0.5);
@@ -113,15 +113,14 @@ TEST_F(LlvmLibcExp10fTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_nan() || FPBits(v).is_inf())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float result = LIBC_NAMESPACE::exp10f(x);
 
     // If the computation resulted in an error or did not produce valid result
     // in the single-precision floating point range, then ignore comparing with
     // MPFR result as MPFR can still produce valid results because of its
     // wider precision.
-    if (FPBits(result).is_nan() || FPBits(result).is_inf() ||
-        LIBC_NAMESPACE::libc_errno != 0)
+    if (FPBits(result).is_nan() || FPBits(result).is_inf() || libc_errno != 0)
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp10, x,
                                    LIBC_NAMESPACE::exp10f(x), 0.5);
diff --git a/external/llvm-project/libc/test/src/math/exp10m1f_test.cpp b/external/llvm-project/libc/test/src/math/exp10m1f_test.cpp
index aee273384f1a..01802bd68f7e 100644
--- a/external/llvm-project/libc/test/src/math/exp10m1f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/exp10m1f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/CPP/array.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10m1f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -69,7 +69,7 @@ TEST_F(LlvmLibcExp10m1fTest, TrickyInputs) {
   };
 
   for (float x : INPUTS) {
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp10m1, x,
                                    LIBC_NAMESPACE::exp10m1f(x), 0.5);
   }
@@ -82,14 +82,14 @@ TEST_F(LlvmLibcExp10m1fTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_inf_or_nan())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float result = LIBC_NAMESPACE::exp10m1f(x);
 
     // If the computation resulted in an error or did not produce valid result
     // in the single-precision floating point range, then ignore comparing with
     // MPFR result as MPFR can still produce valid results because of its
     // wider precision.
-    if (FPBits(result).is_inf_or_nan() || LIBC_NAMESPACE::libc_errno != 0)
+    if (FPBits(result).is_inf_or_nan() || libc_errno != 0)
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp10m1, x,
                                    LIBC_NAMESPACE::exp10m1f(x), 0.5);
diff --git a/external/llvm-project/libc/test/src/math/exp2_test.cpp b/external/llvm-project/libc/test/src/math/exp2_test.cpp
index adfceceeef4b..4cd95dd5486e 100644
--- a/external/llvm-project/libc/test/src/math/exp2_test.cpp
+++ b/external/llvm-project/libc/test/src/math/exp2_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -80,7 +80,7 @@ TEST_F(LlvmLibcExp2Test, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::exp2(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/exp2f_test.cpp b/external/llvm-project/libc/test/src/math/exp2f_test.cpp
index 0c4c82153439..aeecb3e74b07 100644
--- a/external/llvm-project/libc/test/src/math/exp2f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/exp2f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ using LlvmLibcExp2fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcExp2fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp2f(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExp2fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp2fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp2f(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
@@ -71,7 +71,7 @@ TEST_F(LlvmLibcExp2fTest, TrickyInputs) {
       0xc3150000U, /*-0x1.2ap+7f*/
   };
   for (int i = 0; i < N; ++i) {
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float x = FPBits(INPUTS[i]).get_val();
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp2, x,
                                    LIBC_NAMESPACE::exp2f(x), 0.5);
@@ -80,7 +80,7 @@ TEST_F(LlvmLibcExp2fTest, TrickyInputs) {
 }
 
 TEST_F(LlvmLibcExp2fTest, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       0.0f, LIBC_NAMESPACE::exp2f(FPBits(0xff7fffffU).get_val()), FE_UNDERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
@@ -108,15 +108,14 @@ TEST_F(LlvmLibcExp2fTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_nan() || FPBits(v).is_inf())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float result = LIBC_NAMESPACE::exp2f(x);
 
     // If the computation resulted in an error or did not produce valid result
     // in the single-precision floating point range, then ignore comparing with
     // MPFR result as MPFR can still produce valid results because of its
     // wider precision.
-    if (FPBits(result).is_nan() || FPBits(result).is_inf() ||
-        LIBC_NAMESPACE::libc_errno != 0)
+    if (FPBits(result).is_nan() || FPBits(result).is_inf() || libc_errno != 0)
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp2, x,
                                    LIBC_NAMESPACE::exp2f(x), 0.5);
diff --git a/external/llvm-project/libc/test/src/math/exp2m1f_test.cpp b/external/llvm-project/libc/test/src/math/exp2m1f_test.cpp
index 793cf0cc2cbb..0c87657abc08 100644
--- a/external/llvm-project/libc/test/src/math/exp2m1f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/exp2m1f_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/math_macros.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2m1f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -38,7 +38,7 @@ TEST_F(LlvmLibcExp2m1fTest, TrickyInputs) {
   };
 
   for (float x : INPUTS) {
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp2m1, x,
                                    LIBC_NAMESPACE::exp2m1f(x), 0.5);
   }
@@ -51,15 +51,14 @@ TEST_F(LlvmLibcExp2m1fTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_nan() || FPBits(v).is_inf())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float result = LIBC_NAMESPACE::exp2m1f(x);
 
     // If the computation resulted in an error or did not produce valid result
     // in the single-precision floating point range, then ignore comparing with
     // MPFR result as MPFR can still produce valid results because of its
     // wider precision.
-    if (FPBits(result).is_nan() || FPBits(result).is_inf() ||
-        LIBC_NAMESPACE::libc_errno != 0)
+    if (FPBits(result).is_nan() || FPBits(result).is_inf() || libc_errno != 0)
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp2m1, x,
                                    LIBC_NAMESPACE::exp2m1f(x), 0.5);
diff --git a/external/llvm-project/libc/test/src/math/exp_test.cpp b/external/llvm-project/libc/test/src/math/exp_test.cpp
index 0ab3a4e54346..83addaeb943d 100644
--- a/external/llvm-project/libc/test/src/math/exp_test.cpp
+++ b/external/llvm-project/libc/test/src/math/exp_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -78,7 +78,7 @@ TEST_F(LlvmLibcExpTest, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::exp(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/expf_test.cpp b/external/llvm-project/libc/test/src/math/expf_test.cpp
index 26a0bca4ce25..3c10812ff5bc 100644
--- a/external/llvm-project/libc/test/src/math/expf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/expf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ using LlvmLibcExpfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcExpfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::expf(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExpfTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExpfTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expf(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
@@ -55,7 +55,7 @@ TEST_F(LlvmLibcExpfTest, Overflow) {
 }
 
 TEST_F(LlvmLibcExpfTest, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       0.0f, LIBC_NAMESPACE::expf(FPBits(0xff7fffffU).get_val()), FE_UNDERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
@@ -76,7 +76,7 @@ TEST_F(LlvmLibcExpfTest, Underflow) {
 TEST_F(LlvmLibcExpfTest, Borderline) {
   float x;
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   x = FPBits(0x42affff8U).get_val();
   ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp, x,
                                  LIBC_NAMESPACE::expf(x), 0.5);
@@ -110,15 +110,14 @@ TEST_F(LlvmLibcExpfTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_nan() || FPBits(v).is_inf())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float result = LIBC_NAMESPACE::expf(x);
 
     // If the computation resulted in an error or did not produce valid result
     // in the single-precision floating point range, then ignore comparing with
     // MPFR result as MPFR can still produce valid results because of its
     // wider precision.
-    if (FPBits(result).is_nan() || FPBits(result).is_inf() ||
-        LIBC_NAMESPACE::libc_errno != 0)
+    if (FPBits(result).is_nan() || FPBits(result).is_inf() || libc_errno != 0)
       continue;
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp, x,
                                    LIBC_NAMESPACE::expf(x), 0.5);
diff --git a/external/llvm-project/libc/test/src/math/expm1_test.cpp b/external/llvm-project/libc/test/src/math/expm1_test.cpp
index 9720773d9f96..0cf07e2e4973 100644
--- a/external/llvm-project/libc/test/src/math/expm1_test.cpp
+++ b/external/llvm-project/libc/test/src/math/expm1_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expm1.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -64,7 +64,7 @@ TEST_F(LlvmLibcExpm1Test, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::expm1(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/expm1f_test.cpp b/external/llvm-project/libc/test/src/math/expm1f_test.cpp
index 274fe3bb7afb..cf3fe9c26ae1 100644
--- a/external/llvm-project/libc/test/src/math/expm1f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/expm1f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expm1f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ using LlvmLibcExpm1fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcExpm1fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::expm1f(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExpm1fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExpm1fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expm1f(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
@@ -55,7 +55,7 @@ TEST_F(LlvmLibcExpm1fTest, Overflow) {
 }
 
 TEST_F(LlvmLibcExpm1fTest, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ(-1.0f, LIBC_NAMESPACE::expm1f(FPBits(0xff7fffffU).get_val()));
 
   float x = FPBits(0xc2cffff8U).get_val();
@@ -70,7 +70,7 @@ TEST_F(LlvmLibcExpm1fTest, Underflow) {
 TEST_F(LlvmLibcExpm1fTest, Borderline) {
   float x;
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   x = FPBits(0x42affff8U).get_val();
   ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
                                  LIBC_NAMESPACE::expm1f(x), 0.5);
@@ -119,15 +119,14 @@ TEST_F(LlvmLibcExpm1fTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_nan() || FPBits(v).is_inf())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float result = LIBC_NAMESPACE::expm1f(x);
 
     // If the computation resulted in an error or did not produce valid result
     // in the single-precision floating point range, then ignore comparing with
     // MPFR result as MPFR can still produce valid results because of its
     // wider precision.
-    if (FPBits(result).is_nan() || FPBits(result).is_inf() ||
-        LIBC_NAMESPACE::libc_errno != 0)
+    if (FPBits(result).is_nan() || FPBits(result).is_inf() || libc_errno != 0)
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
                                    LIBC_NAMESPACE::expm1f(x), 0.5);
diff --git a/external/llvm-project/libc/test/src/math/log10_test.cpp b/external/llvm-project/libc/test/src/math/log10_test.cpp
index 01aa1f82ae5d..e9529d87c388 100644
--- a/external/llvm-project/libc/test/src/math/log10_test.cpp
+++ b/external/llvm-project/libc/test/src/math/log10_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log10.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -101,7 +101,7 @@ TEST_F(LlvmLibcLog10Test, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::log10(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/log1p_test.cpp b/external/llvm-project/libc/test/src/math/log1p_test.cpp
index 107e965a0d3a..e5747b7e5ec0 100644
--- a/external/llvm-project/libc/test/src/math/log1p_test.cpp
+++ b/external/llvm-project/libc/test/src/math/log1p_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log1p.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -102,7 +102,7 @@ TEST_F(LlvmLibcLog1pTest, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::log1p(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/log1pf_test.cpp b/external/llvm-project/libc/test/src/math/log1pf_test.cpp
index bb181dc5e43b..ffe2dd2c33dd 100644
--- a/external/llvm-project/libc/test/src/math/log1pf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/log1pf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log1pf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -75,7 +75,7 @@ TEST_F(LlvmLibcLog1pfTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_nan() || FPBits(v).is_inf())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Log1p, x,
                                    LIBC_NAMESPACE::log1pf(x), 0.5);
   }
diff --git a/external/llvm-project/libc/test/src/math/log2_test.cpp b/external/llvm-project/libc/test/src/math/log2_test.cpp
index 8a07991a6888..fc440c09b42b 100644
--- a/external/llvm-project/libc/test/src/math/log2_test.cpp
+++ b/external/llvm-project/libc/test/src/math/log2_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log2.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -100,7 +100,7 @@ TEST_F(LlvmLibcLog2Test, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::log2(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/log2f_test.cpp b/external/llvm-project/libc/test/src/math/log2f_test.cpp
index 83691fb75300..92226c763f45 100644
--- a/external/llvm-project/libc/test/src/math/log2f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/log2f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log2f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -52,14 +52,13 @@ TEST_F(LlvmLibcLog2fTest, InFloatRange) {
     float x = FPBits(v).get_val();
     if (FPBits(v).is_nan() || FPBits(v).is_inf())
       continue;
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     float result = LIBC_NAMESPACE::log2f(x);
     // If the computation resulted in an error or did not produce valid result
     // in the single-precision floating point range, then ignore comparing with
     // MPFR result as MPFR can still produce valid results because of its
     // wider precision.
-    if (FPBits(result).is_nan() || FPBits(result).is_inf() ||
-        LIBC_NAMESPACE::libc_errno != 0)
+    if (FPBits(result).is_nan() || FPBits(result).is_inf() || libc_errno != 0)
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Log2, x,
                                    LIBC_NAMESPACE::log2f(x), 0.5);
diff --git a/external/llvm-project/libc/test/src/math/log_test.cpp b/external/llvm-project/libc/test/src/math/log_test.cpp
index 969a469b2e1c..54afaa33d135 100644
--- a/external/llvm-project/libc/test/src/math/log_test.cpp
+++ b/external/llvm-project/libc/test/src/math/log_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -99,7 +99,7 @@ TEST_F(LlvmLibcLogTest, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf() || x < 0.0)
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::log(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/powf_test.cpp b/external/llvm-project/libc/test/src/math/powf_test.cpp
index 448dcc0035e9..4d189d813e58 100644
--- a/external/llvm-project/libc/test/src/math/powf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/powf_test.cpp
@@ -78,7 +78,7 @@ TEST_F(LlvmLibcPowfTest, InFloatRange) {
         if (FPBits(w).is_nan() || FPBits(w).is_inf())
           continue;
 
-        LIBC_NAMESPACE::libc_errno = 0;
+        libc_errno = 0;
         float result = LIBC_NAMESPACE::powf(x, y);
         ++cc;
         if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/sin_test.cpp b/external/llvm-project/libc/test/src/math/sin_test.cpp
index d4c6bd416a40..4d5d9ddf464b 100644
--- a/external/llvm-project/libc/test/src/math/sin_test.cpp
+++ b/external/llvm-project/libc/test/src/math/sin_test.cpp
@@ -71,7 +71,7 @@ TEST_F(LlvmLibcSinTest, InDoubleRange) {
       double x = FPBits(v).get_val();
       if (FPBits(v).is_nan() || FPBits(v).is_inf())
         continue;
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
       double result = LIBC_NAMESPACE::sin(x);
       ++cc;
       if (FPBits(result).is_nan() || FPBits(result).is_inf())
diff --git a/external/llvm-project/libc/test/src/math/sincosf_test.cpp b/external/llvm-project/libc/test/src/math/sincosf_test.cpp
index 2823110331f3..ad2155f329cd 100644
--- a/external/llvm-project/libc/test/src/math/sincosf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/sincosf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sincosf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -24,7 +24,7 @@ using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcSinCosfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   float sin, cos;
 
   LIBC_NAMESPACE::sincosf(aNaN, &sin, &cos);
diff --git a/external/llvm-project/libc/test/src/math/sinf_test.cpp b/external/llvm-project/libc/test/src/math/sinf_test.cpp
index 8fd3ed1577ce..e0357e6157fd 100644
--- a/external/llvm-project/libc/test/src/math/sinf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/sinf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -24,7 +24,7 @@ using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcSinfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/sinhf_test.cpp b/external/llvm-project/libc/test/src/math/sinhf_test.cpp
index 6867c7aec57d..74f906ebaa98 100644
--- a/external/llvm-project/libc/test/src/math/sinhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/sinhf_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/math_macros.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -22,7 +22,7 @@ using LlvmLibcSinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcSinhfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinhf(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -65,7 +65,7 @@ TEST_F(LlvmLibcSinhfTest, SmallValues) {
 }
 
 TEST_F(LlvmLibcSinhfTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::sinhf(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/sinpif_test.cpp b/external/llvm-project/libc/test/src/math/sinpif_test.cpp
index d00fd77d288c..986c676761f0 100644
--- a/external/llvm-project/libc/test/src/math/sinpif_test.cpp
+++ b/external/llvm-project/libc/test/src/math/sinpif_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinpif.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/src/math/sdcomp26094.h"
@@ -21,7 +21,7 @@ using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcSinpifTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinpif(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/FModTest.h b/external/llvm-project/libc/test/src/math/smoke/FModTest.h
index 8fbcc2a27654..04cbc659ece5 100644
--- a/external/llvm-project/libc/test/src/math/smoke/FModTest.h
+++ b/external/llvm-project/libc/test/src/math/smoke/FModTest.h
@@ -10,7 +10,7 @@
 #define LLVM_LIBC_TEST_SRC_MATH_FMODTEST_H
 
 #include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/RoundToIntegerTest.h b/external/llvm-project/libc/test/src/math/smoke/RoundToIntegerTest.h
index 6ae97ce35a0d..745ccbc748ec 100644
--- a/external/llvm-project/libc/test/src/math/smoke/RoundToIntegerTest.h
+++ b/external/llvm-project/libc/test/src/math/smoke/RoundToIntegerTest.h
@@ -40,7 +40,7 @@ class RoundToIntegerTestTemplate
 
   void test_one_input(RoundToIntegerFunc func, F input, I expected,
                       bool expectError) {
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
     LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
 
     ASSERT_EQ(func(input), expected);
diff --git a/external/llvm-project/libc/test/src/math/smoke/acos_test.cpp b/external/llvm-project/libc/test/src/math/smoke/acos_test.cpp
index 3a59bce26407..fe2caefb52ab 100644
--- a/external/llvm-project/libc/test/src/math/smoke/acos_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/acos_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/fenv_macros.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acos.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ TEST_F(LlvmLibcAcosTest, SpecialNumbers) {
   EXPECT_FP_EQ(0x1.921fb54442d18p0, LIBC_NAMESPACE::acos(zero));
   EXPECT_FP_EQ(0x1.921fb54442d18p0, LIBC_NAMESPACE::acos(neg_zero));
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acos(inf),
                                            FE_INVALID);
   EXPECT_MATH_ERRNO(EDOM);
diff --git a/external/llvm-project/libc/test/src/math/smoke/acosf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/acosf16_test.cpp
index c4274b824509..7103dc33fec3 100644
--- a/external/llvm-project/libc/test/src/math/smoke/acosf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/acosf16_test.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acosf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -15,7 +15,7 @@
 using LlvmLibcAcosf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcAcosf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::acosf16(aNaN));
   EXPECT_MATH_ERRNO(0);
 
diff --git a/external/llvm-project/libc/test/src/math/smoke/acosf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/acosf_test.cpp
index 74f68e00011a..257c6a3d1d22 100644
--- a/external/llvm-project/libc/test/src/math/smoke/acosf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/acosf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acosf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcAcosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAcosfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::acosf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/acoshf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/acoshf16_test.cpp
index 7681c2a4e7fb..6b9c995cf992 100644
--- a/external/llvm-project/libc/test/src/math/smoke/acoshf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/acoshf16_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acoshf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -15,7 +15,7 @@
 using LlvmLibcAcoshf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcAcoshf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::acoshf16(aNaN));
   EXPECT_MATH_ERRNO(0);
 
diff --git a/external/llvm-project/libc/test/src/math/smoke/acoshf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/acoshf_test.cpp
index c5ba88055ac5..b6abfab99929 100644
--- a/external/llvm-project/libc/test/src/math/smoke/acoshf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/acoshf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acoshf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcAcoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAcoshfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::acoshf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/acospif16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/acospif16_test.cpp
index 66b94706eab9..4b2f6de3f7e3 100644
--- a/external/llvm-project/libc/test/src/math/smoke/acospif16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/acospif16_test.cpp
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/acospif16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using LlvmLibcAcospif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 TEST_F(LlvmLibcAcospif16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::acospif16(aNaN));
   EXPECT_MATH_ERRNO(0);
 
diff --git a/external/llvm-project/libc/test/src/math/smoke/asinf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/asinf16_test.cpp
index 9f675b08319c..b03f0a420a49 100644
--- a/external/llvm-project/libc/test/src/math/smoke/asinf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/asinf16_test.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/asinf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -15,7 +15,7 @@
 using LlvmLibcAsinf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcAsinf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::asinf16(aNaN));
   EXPECT_MATH_ERRNO(0);
 
diff --git a/external/llvm-project/libc/test/src/math/smoke/asinf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/asinf_test.cpp
index d817d2b36619..2615a8ddd16b 100644
--- a/external/llvm-project/libc/test/src/math/smoke/asinf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/asinf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/asinf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcAsinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAsinfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::asinf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/asinhf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/asinhf16_test.cpp
index dcaab217331c..7f612ce3c467 100644
--- a/external/llvm-project/libc/test/src/math/smoke/asinhf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/asinhf16_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/asinhf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -14,7 +14,7 @@
 using LlvmLibcAsinhf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcAsinhf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::asinhf16(aNaN));
   EXPECT_MATH_ERRNO(0);
 
diff --git a/external/llvm-project/libc/test/src/math/smoke/asinhf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/asinhf_test.cpp
index 4a8743c50075..d812a2dffe8a 100644
--- a/external/llvm-project/libc/test/src/math/smoke/asinhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/asinhf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/asinhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcAsinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAsinhfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::asinhf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/atan2f_test.cpp b/external/llvm-project/libc/test/src/math/smoke/atan2f_test.cpp
index 1fbcfbe96b2d..7f8cfb9830d2 100644
--- a/external/llvm-project/libc/test/src/math/smoke/atan2f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/atan2f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/atan2f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcAtan2fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAtan2fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::atan2f(sNaN, sNaN),
                               FE_INVALID);
diff --git a/external/llvm-project/libc/test/src/math/smoke/atanf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/atanf16_test.cpp
index af50287d9b22..ba1e3b2fc8be 100644
--- a/external/llvm-project/libc/test/src/math/smoke/atanf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/atanf16_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/atanf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -14,7 +14,7 @@
 using LlvmLibcAtanf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcAtanf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::atanf16(aNaN));
   EXPECT_MATH_ERRNO(0);
 
diff --git a/external/llvm-project/libc/test/src/math/smoke/atanf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/atanf_test.cpp
index 7d09a28beaa3..b56b9d0162b9 100644
--- a/external/llvm-project/libc/test/src/math/smoke/atanf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/atanf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/atanf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcAtanfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAtanfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::atanf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
 
diff --git a/external/llvm-project/libc/test/src/math/smoke/atanhf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/atanhf16_test.cpp
index 81df6da8cee2..c2a520f7638f 100644
--- a/external/llvm-project/libc/test/src/math/smoke/atanhf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/atanhf16_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/atanhf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -15,7 +15,7 @@
 using LlvmLibcAtanhf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcAtanhf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atanhf16(sNaN),
                                            FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/atanhf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/atanhf_test.cpp
index 73a5b81b0240..038cb30d89a4 100644
--- a/external/llvm-project/libc/test/src/math/smoke/atanhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/atanhf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/atanhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -20,7 +20,7 @@ using LIBC_NAMESPACE::Sign;
 using LlvmLibcAtanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::atanhf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
   // TODO: Strengthen errno,exception checks and remove these assert macros
diff --git a/external/llvm-project/libc/test/src/math/smoke/cosf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/cosf16_test.cpp
index 2638551fb1d1..4362a5a3a4bd 100644
--- a/external/llvm-project/libc/test/src/math/smoke/cosf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/cosf16_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/cosf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -14,7 +14,7 @@
 using LlvmLibcCosf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcCosf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::cosf16(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/cosf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/cosf_test.cpp
index 99773583dcb1..470a876c63a7 100644
--- a/external/llvm-project/libc/test/src/math/smoke/cosf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/cosf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/cosf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcCosfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::cosf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/coshf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/coshf16_test.cpp
index 08d05ecce86b..7bf62afa24c4 100644
--- a/external/llvm-project/libc/test/src/math/smoke/coshf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/coshf16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/coshf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcCoshf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcCoshf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::coshf16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcCoshf16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcCoshf16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::coshf16(max_normal),
                               FE_OVERFLOW | FE_INEXACT);
diff --git a/external/llvm-project/libc/test/src/math/smoke/coshf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/coshf_test.cpp
index 1611ea1b9292..ee8f0199df3b 100644
--- a/external/llvm-project/libc/test/src/math/smoke/coshf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/coshf_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/math_macros.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/coshf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -19,7 +19,7 @@
 using LlvmLibcCoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcCoshfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::coshf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -41,7 +41,7 @@ TEST_F(LlvmLibcCoshfTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcCoshfTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::coshf(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/smoke/cospif16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/cospif16_test.cpp
index edd8ed97b30f..fcde0cc79e35 100644
--- a/external/llvm-project/libc/test/src/math/smoke/cospif16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/cospif16_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/cospif16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -15,7 +15,7 @@
 using LlvmLibcCospif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcCospif16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::cospif16(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/cospif_test.cpp b/external/llvm-project/libc/test/src/math/smoke/cospif_test.cpp
index 20153897dc45..3d48909cca93 100644
--- a/external/llvm-project/libc/test/src/math/smoke/cospif_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/cospif_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/cospif.h"
 #include "test/UnitTest/FPMatcher.h"
 
@@ -15,7 +15,7 @@
 using LlvmLibcCospifTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcCospifTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::cospif(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp10_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp10_test.cpp
index baf8a7681097..50d3de0c7fe7 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp10_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp10_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp10f16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp10f16_test.cpp
index 1c4ef2aa08a7..bda40348f883 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp10f16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp10f16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10f16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcExp10f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcExp10f16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::exp10f16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExp10f16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp10f16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::exp10f16(max_normal),
                               FE_OVERFLOW);
@@ -53,7 +53,7 @@ TEST_F(LlvmLibcExp10f16Test, Overflow) {
 }
 
 TEST_F(LlvmLibcExp10f16Test, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(zero, LIBC_NAMESPACE::exp10f16(neg_max_normal),
                               FE_UNDERFLOW | FE_INEXACT);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp10f_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp10f_test.cpp
index bf39e2cc12d0..fcd334bb9e36 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp10f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp10f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcExp10fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcExp10fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::exp10f(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -44,7 +44,7 @@ TEST_F(LlvmLibcExp10fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp10fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp10f(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp10m1f16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp10m1f16_test.cpp
index dfa7fa477d3d..ed2d5a48b316 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp10m1f16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp10m1f16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10m1f16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcExp10m1f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcExp10m1f16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::exp10m1f16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExp10m1f16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp10m1f16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::exp10m1f16(max_normal),
                               FE_OVERFLOW | FE_INEXACT);
@@ -67,7 +67,7 @@ TEST_F(LlvmLibcExp10m1f16Test, Overflow) {
 }
 
 TEST_F(LlvmLibcExp10m1f16Test, ResultNearNegOne) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(LIBC_NAMESPACE::fputil::cast<float16>(-1.0),
                               LIBC_NAMESPACE::exp10m1f16(neg_max_normal),
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp10m1f_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp10m1f_test.cpp
index 2c2cfdbb08a3..19369a897aaa 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp10m1f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp10m1f_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp10m1f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -14,7 +14,7 @@
 using LlvmLibcExp10m1fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcExp10m1fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::exp10m1f(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -34,7 +34,7 @@ TEST_F(LlvmLibcExp10m1fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp10m1fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::exp10m1f(0x1.fffffep+127f),
                               FE_OVERFLOW);
@@ -50,7 +50,7 @@ TEST_F(LlvmLibcExp10m1fTest, Overflow) {
 }
 
 TEST_F(LlvmLibcExp10m1fTest, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(-1.0f, LIBC_NAMESPACE::exp10m1f(-max_normal),
                               FE_UNDERFLOW);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp2_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp2_test.cpp
index 9ab9129416da..aebf80835072 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp2_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp2_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp2f16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp2f16_test.cpp
index f69b33a3cf37..1eb7343dcd22 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp2f16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp2f16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2f16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcExp2f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcExp2f16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::exp2f16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExp2f16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp2f16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::exp2f16(max_normal),
                               FE_OVERFLOW);
@@ -53,7 +53,7 @@ TEST_F(LlvmLibcExp2f16Test, Overflow) {
 }
 
 TEST_F(LlvmLibcExp2f16Test, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(zero, LIBC_NAMESPACE::exp2f16(neg_max_normal),
                               FE_UNDERFLOW | FE_INEXACT);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp2f_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp2f_test.cpp
index a928389cc41b..c5243273d9ed 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp2f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp2f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcExp2fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcExp2fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::exp2f(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -45,7 +45,7 @@ TEST_F(LlvmLibcExp2fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp2fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp2f(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp2m1f16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp2m1f16_test.cpp
index f423196a7036..635b7a6e187d 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp2m1f16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp2m1f16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2m1f16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcExp2m1f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcExp2m1f16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::exp2m1f16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -39,7 +39,7 @@ TEST_F(LlvmLibcExp2m1f16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp2m1f16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::exp2m1f16(max_normal),
                               FE_OVERFLOW | FE_INEXACT);
@@ -65,7 +65,7 @@ TEST_F(LlvmLibcExp2m1f16Test, Overflow) {
 }
 
 TEST_F(LlvmLibcExp2m1f16Test, ResultNearNegOne) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(-1.0, LIBC_NAMESPACE::exp2m1f16(neg_max_normal),
                               FE_INEXACT);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp2m1f_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp2m1f_test.cpp
index 99bdf0035df0..63852e11655a 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp2m1f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp2m1f_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp2m1f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@ using LIBC_NAMESPACE::fputil::testing::ForceRoundingMode;
 using LIBC_NAMESPACE::fputil::testing::RoundingMode;
 
 TEST_F(LlvmLibcExp2m1fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::exp2m1f(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -34,7 +34,7 @@ TEST_F(LlvmLibcExp2m1fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExp2m1fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::exp2m1f(0x1.fffffep+127),
                               FE_OVERFLOW);
@@ -50,7 +50,7 @@ TEST_F(LlvmLibcExp2m1fTest, Overflow) {
 }
 
 TEST_F(LlvmLibcExp2m1fTest, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(-1.0f, LIBC_NAMESPACE::exp2m1f(-0x1.fffffep+127),
                               FE_UNDERFLOW);
diff --git a/external/llvm-project/libc/test/src/math/smoke/exp_test.cpp b/external/llvm-project/libc/test/src/math/smoke/exp_test.cpp
index f86243092f1f..c3b2ae70e1d9 100644
--- a/external/llvm-project/libc/test/src/math/smoke/exp_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/exp_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/exp.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/expf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/expf16_test.cpp
index ab745a3cf6f5..863f694ffc41 100644
--- a/external/llvm-project/libc/test/src/math/smoke/expf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/expf16_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -17,7 +17,7 @@
 using LlvmLibcExpf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcExpf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::expf16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -41,7 +41,7 @@ TEST_F(LlvmLibcExpf16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExpf16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::expf16(max_normal),
                               FE_OVERFLOW);
@@ -54,7 +54,7 @@ TEST_F(LlvmLibcExpf16Test, Overflow) {
 }
 
 TEST_F(LlvmLibcExpf16Test, Underflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(zero, LIBC_NAMESPACE::expf16(neg_max_normal),
                               FE_UNDERFLOW | FE_INEXACT);
diff --git a/external/llvm-project/libc/test/src/math/smoke/expf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/expf_test.cpp
index eee830499927..d34151735afa 100644
--- a/external/llvm-project/libc/test/src/math/smoke/expf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/expf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcExpfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcExpfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::expf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExpfTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExpfTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expf(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/smoke/expm1_test.cpp b/external/llvm-project/libc/test/src/math/smoke/expm1_test.cpp
index bc71c53abc7a..c842fe3c45fe 100644
--- a/external/llvm-project/libc/test/src/math/smoke/expm1_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/expm1_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expm1.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/expm1f16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/expm1f16_test.cpp
index f297c5dfc3c7..4d19a9bac5eb 100644
--- a/external/llvm-project/libc/test/src/math/smoke/expm1f16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/expm1f16_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expm1f16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -17,7 +17,7 @@
 using LlvmLibcExpm1f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcExpm1f16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::expm1f16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExpm1f16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExpm1f16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::expm1f16(max_normal),
                               FE_OVERFLOW | FE_INEXACT);
@@ -67,7 +67,7 @@ TEST_F(LlvmLibcExpm1f16Test, Overflow) {
 }
 
 TEST_F(LlvmLibcExpm1f16Test, ResultNearNegOne) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(LIBC_NAMESPACE::fputil::cast<float16>(-1.0),
                               LIBC_NAMESPACE::expm1f16(neg_max_normal),
diff --git a/external/llvm-project/libc/test/src/math/smoke/expm1f_test.cpp b/external/llvm-project/libc/test/src/math/smoke/expm1f_test.cpp
index dfb474d70fb6..214bfe8abd4d 100644
--- a/external/llvm-project/libc/test/src/math/smoke/expm1f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/expm1f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/expm1f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcExpm1fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcExpm1fTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::expm1f(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcExpm1fTest, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcExpm1fTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expm1f(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/smoke/log10_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log10_test.cpp
index ff73850c5210..49cfda85111a 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log10_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log10_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log10.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/log10f16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log10f16_test.cpp
index 471e19893332..53f5ac46aa60 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log10f16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log10f16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log10f16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcLog10f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcLog10f16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::log10f16(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/log1p_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log1p_test.cpp
index 631c24b8abcf..61c56cd2c6dd 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log1p_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log1p_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log1p.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/log1pf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log1pf_test.cpp
index bd828ad58c4c..dc3489fddf99 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log1pf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log1pf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log1pf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/log2_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log2_test.cpp
index 9993d442967c..0534d00b1f40 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log2_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log2_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log2.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/log2f16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log2f16_test.cpp
index 6d98482aa449..fd20652d2f00 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log2f16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log2f16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log2f16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcLog2f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcLog2f16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::log2f16(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/log2f_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log2f_test.cpp
index 8648b75b88b8..53d54ac36763 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log2f_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log2f_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log2f.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/log_test.cpp b/external/llvm-project/libc/test/src/math/smoke/log_test.cpp
index d31eb0c1db73..09e9ab0a9a4d 100644
--- a/external/llvm-project/libc/test/src/math/smoke/log_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/log_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/log.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/math/smoke/logf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/logf16_test.cpp
index c7232aa1c1e3..2784f3d5fa54 100644
--- a/external/llvm-project/libc/test/src/math/smoke/logf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/logf16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/logf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcLogf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcLogf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::logf16(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/sincosf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/sincosf_test.cpp
index 5f66868f12a1..8ba0d04347bb 100644
--- a/external/llvm-project/libc/test/src/math/smoke/sincosf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/sincosf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sincosf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcSinCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcSinCosfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   float sin, cos;
 
   LIBC_NAMESPACE::sincosf(sNaN, &sin, &cos);
diff --git a/external/llvm-project/libc/test/src/math/smoke/sinf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/sinf16_test.cpp
index a0e7a7ba321f..6b168ac040db 100644
--- a/external/llvm-project/libc/test/src/math/smoke/sinf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/sinf16_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -14,7 +14,7 @@
 using LlvmLibcSinf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcSinf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::sinf16(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/sinf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/sinf_test.cpp
index de504b4f5335..8173969fb256 100644
--- a/external/llvm-project/libc/test/src/math/smoke/sinf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/sinf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcSinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcSinfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::sinf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/sinhf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/sinhf16_test.cpp
index 4f21d33ba78e..d52739a9adb3 100644
--- a/external/llvm-project/libc/test/src/math/smoke/sinhf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/sinhf16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinhf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcSinhf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcSinhf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::sinhf16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -38,7 +38,7 @@ TEST_F(LlvmLibcSinhf16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcSinhf16Test, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::sinhf16(max_normal),
                               FE_OVERFLOW | FE_INEXACT);
diff --git a/external/llvm-project/libc/test/src/math/smoke/sinhf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/sinhf_test.cpp
index e22cfc7ea14d..ea6a4474a780 100644
--- a/external/llvm-project/libc/test/src/math/smoke/sinhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/sinhf_test.cpp
@@ -9,7 +9,7 @@
 #include "hdr/math_macros.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -19,7 +19,7 @@
 using LlvmLibcSinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcSinhfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::sinhf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
@@ -52,7 +52,7 @@ TEST_F(LlvmLibcSinhfTest, SmallValues) {
 }
 
 TEST_F(LlvmLibcSinhfTest, Overflow) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::sinhf(FPBits(0x7f7fffffU).get_val()), FE_OVERFLOW);
   EXPECT_MATH_ERRNO(ERANGE);
diff --git a/external/llvm-project/libc/test/src/math/smoke/sinpif16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/sinpif16_test.cpp
index b2db6fb9f862..9edf2cc663d4 100644
--- a/external/llvm-project/libc/test/src/math/smoke/sinpif16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/sinpif16_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinpif16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -15,7 +15,7 @@
 using LlvmLibcSinpif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcSinpif16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::sinpif16(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/sinpif_test.cpp b/external/llvm-project/libc/test/src/math/smoke/sinpif_test.cpp
index 1ba5c1d2b720..b840f3980eda 100644
--- a/external/llvm-project/libc/test/src/math/smoke/sinpif_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/sinpif_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/sinpif.h"
 #include "test/UnitTest/FPMatcher.h"
 
@@ -15,7 +15,7 @@
 using LlvmLibcSinpifTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcSinpifTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::sinpif(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/tanf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/tanf16_test.cpp
index f65b9fced72c..95d200cf5591 100644
--- a/external/llvm-project/libc/test/src/math/smoke/tanf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/tanf16_test.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/tanf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -15,7 +15,7 @@
 using LlvmLibcTanf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcTanf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::tanf16(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/tanf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/tanf_test.cpp
index 178e9065f430..12deca5cf941 100644
--- a/external/llvm-project/libc/test/src/math/smoke/tanf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/tanf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/tanf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcTanfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcTanfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::tanf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/tanhf16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/tanhf16_test.cpp
index fa6328e9ef0a..eb90f02a8d7c 100644
--- a/external/llvm-project/libc/test/src/math/smoke/tanhf16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/tanhf16_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/cast.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/tanhf16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -16,7 +16,7 @@
 using LlvmLibcTanhf16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcTanhf16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::tanhf16(aNaN));
   EXPECT_MATH_ERRNO(0);
@@ -40,7 +40,7 @@ TEST_F(LlvmLibcTanhf16Test, SpecialNumbers) {
 }
 
 TEST_F(LlvmLibcTanhf16Test, ResultNearBounds) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(LIBC_NAMESPACE::fputil::cast<float16>(1.0),
                               LIBC_NAMESPACE::tanhf16(max_normal), FE_INEXACT);
diff --git a/external/llvm-project/libc/test/src/math/smoke/tanhf_test.cpp b/external/llvm-project/libc/test/src/math/smoke/tanhf_test.cpp
index c09761ef531f..b12a331b3190 100644
--- a/external/llvm-project/libc/test/src/math/smoke/tanhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/tanhf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/tanhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -18,7 +18,7 @@
 using LlvmLibcTanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcTanhfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::tanhf(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/smoke/tanpif16_test.cpp b/external/llvm-project/libc/test/src/math/smoke/tanpif16_test.cpp
index 74797d1649b1..ea896d7bb3e5 100644
--- a/external/llvm-project/libc/test/src/math/smoke/tanpif16_test.cpp
+++ b/external/llvm-project/libc/test/src/math/smoke/tanpif16_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/tanpif16.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -14,7 +14,7 @@
 using LlvmLibcTanpif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
 
 TEST_F(LlvmLibcTanpif16Test, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::tanpif16(sNaN), FE_INVALID);
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/tanf_test.cpp b/external/llvm-project/libc/test/src/math/tanf_test.cpp
index 9061cf6fb30b..ecc70194b649 100644
--- a/external/llvm-project/libc/test/src/math/tanf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/tanf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/tanf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -24,7 +24,7 @@ using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcTanfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::tanf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/math/tanhf_test.cpp b/external/llvm-project/libc/test/src/math/tanhf_test.cpp
index 389abe4d8589..966ce649e2b3 100644
--- a/external/llvm-project/libc/test/src/math/tanhf_test.cpp
+++ b/external/llvm-project/libc/test/src/math/tanhf_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/math_macros.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/math/tanhf.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ using LlvmLibcTanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcTanhfTest, SpecialNumbers) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::tanhf(aNaN));
   EXPECT_MATH_ERRNO(0);
diff --git a/external/llvm-project/libc/test/src/poll/poll_test.cpp b/external/llvm-project/libc/test/src/poll/poll_test.cpp
index 30f5e41c61ec..97b7b0271817 100644
--- a/external/llvm-project/libc/test/src/poll/poll_test.cpp
+++ b/external/llvm-project/libc/test/src/poll/poll_test.cpp
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/limits_macros.h" // UINT_MAX
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/poll/poll.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcPollTest, SmokeTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   int ret = LIBC_NAMESPACE::poll(nullptr, 0, 0);
   ASSERT_ERRNO_SUCCESS();
   ASSERT_EQ(0, ret);
 }
 TEST(LlvmLibcPollTest, SmokeFailureTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   int ret = LIBC_NAMESPACE::poll(nullptr, UINT_MAX, 0);
   ASSERT_ERRNO_EQ(EINVAL);
   ASSERT_EQ(-1, ret);
diff --git a/external/llvm-project/libc/test/src/sched/affinity_test.cpp b/external/llvm-project/libc/test/src/sched/affinity_test.cpp
index b5085203e5ce..b77f22f8e60d 100644
--- a/external/llvm-project/libc/test/src/sched/affinity_test.cpp
+++ b/external/llvm-project/libc/test/src/sched/affinity_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/OSUtil/syscall.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sched/sched_getaffinity.h"
 #include "src/sched/sched_setaffinity.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
@@ -17,7 +17,7 @@
 
 TEST(LlvmLibcSchedAffinityTest, SmokeTest) {
   cpu_set_t mask;
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   pid_t tid = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid);
   ASSERT_GT(tid, pid_t(0));
@@ -32,15 +32,15 @@ TEST(LlvmLibcSchedAffinityTest, BadMask) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
   pid_t tid = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid);
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(
       LIBC_NAMESPACE::sched_getaffinity(tid, sizeof(cpu_set_t), nullptr),
       Fails(EFAULT));
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(
       LIBC_NAMESPACE::sched_setaffinity(tid, sizeof(cpu_set_t), nullptr),
       Fails(EFAULT));
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 }
diff --git a/external/llvm-project/libc/test/src/sched/cpu_count_test.cpp b/external/llvm-project/libc/test/src/sched/cpu_count_test.cpp
index 5250368a2616..919f1475e1d4 100644
--- a/external/llvm-project/libc/test/src/sched/cpu_count_test.cpp
+++ b/external/llvm-project/libc/test/src/sched/cpu_count_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/OSUtil/syscall.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sched/sched_getaffinity.h"
 #include "src/sched/sched_getcpucount.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
@@ -17,7 +17,7 @@
 
 TEST(LlvmLibcSchedCpuCountTest, SmokeTest) {
   cpu_set_t mask;
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   pid_t tid = LIBC_NAMESPACE::syscall_impl<pid_t>(SYS_gettid);
   ASSERT_GT(tid, pid_t(0));
diff --git a/external/llvm-project/libc/test/src/sched/get_priority_test.cpp b/external/llvm-project/libc/test/src/sched/get_priority_test.cpp
index 59205c51e4a1..bb41dc0be201 100644
--- a/external/llvm-project/libc/test/src/sched/get_priority_test.cpp
+++ b/external/llvm-project/libc/test/src/sched/get_priority_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sched/sched_get_priority_max.h"
 #include "src/sched/sched_get_priority_min.h"
 #include "test/UnitTest/Test.h"
@@ -58,7 +58,7 @@ TEST(LlvmLibcSchedGetPriorityTest, HandleBadPolicyTest) {
 }
 
 TEST(LlvmLibcSchedGetPriorityTest, SmokeTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   // We Test:
   // SCHED_OTHER, SCHED_FIFO, SCHED_RR
diff --git a/external/llvm-project/libc/test/src/sched/param_and_scheduler_test.cpp b/external/llvm-project/libc/test/src/sched/param_and_scheduler_test.cpp
index 747c7e3409e4..4f2b6e412a4b 100644
--- a/external/llvm-project/libc/test/src/sched/param_and_scheduler_test.cpp
+++ b/external/llvm-project/libc/test/src/sched/param_and_scheduler_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sched/sched_get_priority_max.h"
 #include "src/sched/sched_get_priority_min.h"
 #include "src/sched/sched_getparam.h"
@@ -37,7 +37,7 @@
 class SchedTest : public LIBC_NAMESPACE::testing::Test {
 public:
   void testSched(int policy, bool is_mandatory) {
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     int init_policy = LIBC_NAMESPACE::sched_getscheduler(0);
     ASSERT_GE(init_policy, 0);
@@ -55,30 +55,29 @@ class SchedTest : public LIBC_NAMESPACE::testing::Test {
     // Negative pid
     ASSERT_EQ(LIBC_NAMESPACE::sched_setscheduler(-1, policy, &param), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     ASSERT_EQ(LIBC_NAMESPACE::sched_getscheduler(-1), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     // Invalid Policy
     ASSERT_EQ(LIBC_NAMESPACE::sched_setscheduler(0, policy | 128, &param), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     // Out of bounds priority
     param.sched_priority = min_priority - 1;
     ASSERT_EQ(LIBC_NAMESPACE::sched_setscheduler(0, policy, &param), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     param.sched_priority = max_priority + 1;
     ASSERT_EQ(LIBC_NAMESPACE::sched_setscheduler(0, policy, &param), -1);
     // A bit hard to test as depending on user privileges we can run into
     // different issues.
-    ASSERT_TRUE(LIBC_NAMESPACE::libc_errno == EINVAL ||
-                LIBC_NAMESPACE::libc_errno == EPERM);
-    LIBC_NAMESPACE::libc_errno = 0;
+    ASSERT_TRUE(libc_errno == EINVAL || libc_errno == EPERM);
+    libc_errno = 0;
 
     param.sched_priority = min_priority;
     // Success/unsupported policy/missing permissions.
@@ -87,10 +86,9 @@ class SchedTest : public LIBC_NAMESPACE::testing::Test {
     ASSERT_TRUE(setscheduler_result == 0 || setscheduler_result == -1);
     ASSERT_TRUE(
         setscheduler_result != -1
-            ? (LIBC_NAMESPACE::libc_errno == 0)
-            : ((!is_mandatory && LIBC_NAMESPACE::libc_errno == EINVAL) ||
-               LIBC_NAMESPACE::libc_errno == EPERM));
-    LIBC_NAMESPACE::libc_errno = 0;
+            ? (libc_errno == 0)
+            : ((!is_mandatory && libc_errno == EINVAL) || libc_errno == EPERM));
+    libc_errno = 0;
 
     ASSERT_EQ(LIBC_NAMESPACE::sched_getscheduler(0),
               setscheduler_result != -1 ? policy : init_policy);
@@ -100,12 +98,12 @@ class SchedTest : public LIBC_NAMESPACE::testing::Test {
     param.sched_priority = -1;
     ASSERT_EQ(LIBC_NAMESPACE::sched_setparam(0, &param), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     param.sched_priority = max_priority + 1;
     ASSERT_EQ(LIBC_NAMESPACE::sched_setparam(0, &param), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     for (int priority = min_priority; priority <= max_priority; ++priority) {
       ASSERT_EQ(LIBC_NAMESPACE::sched_getparam(0, &param), 0);
@@ -117,21 +115,20 @@ class SchedTest : public LIBC_NAMESPACE::testing::Test {
       // Negative pid
       ASSERT_EQ(LIBC_NAMESPACE::sched_setparam(-1, &param), -1);
       ASSERT_ERRNO_EQ(EINVAL);
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
 
       ASSERT_EQ(LIBC_NAMESPACE::sched_getparam(-1, &param), -1);
       ASSERT_ERRNO_EQ(EINVAL);
-      LIBC_NAMESPACE::libc_errno = 0;
+      libc_errno = 0;
 
       // Success/unsupported policy/missing permissions
       int setparam_result = LIBC_NAMESPACE::sched_setparam(0, &param);
       ASSERT_TRUE(setparam_result == 0 || setparam_result == -1);
       ASSERT_TRUE(setparam_result != -1
-                      ? (LIBC_NAMESPACE::libc_errno == 0)
-                      : ((setscheduler_result == -1 &&
-                          LIBC_NAMESPACE::libc_errno == EINVAL) ||
-                         LIBC_NAMESPACE::libc_errno == EPERM));
-      LIBC_NAMESPACE::libc_errno = 0;
+                      ? (libc_errno == 0)
+                      : ((setscheduler_result == -1 && libc_errno == EINVAL) ||
+                         libc_errno == EPERM));
+      libc_errno = 0;
 
       ASSERT_EQ(LIBC_NAMESPACE::sched_getparam(0, &param), 0);
       ASSERT_ERRNO_SUCCESS();
@@ -143,7 +140,7 @@ class SchedTest : public LIBC_NAMESPACE::testing::Test {
     // Null test
     ASSERT_EQ(LIBC_NAMESPACE::sched_setscheduler(0, policy, nullptr), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
   }
 };
 
@@ -161,13 +158,13 @@ LIST_SCHED_TESTS(SCHED_BATCH, true)
 LIST_SCHED_TESTS(SCHED_IDLE, true)
 
 TEST(LlvmLibcSchedParamAndSchedulerTest, NullParamTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   ASSERT_EQ(LIBC_NAMESPACE::sched_setparam(0, nullptr), -1);
   ASSERT_ERRNO_EQ(EINVAL);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   ASSERT_EQ(LIBC_NAMESPACE::sched_getparam(0, nullptr), -1);
   ASSERT_ERRNO_EQ(EINVAL);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 }
diff --git a/external/llvm-project/libc/test/src/sched/sched_rr_get_interval_test.cpp b/external/llvm-project/libc/test/src/sched/sched_rr_get_interval_test.cpp
index c22a2c76d743..a0fe5edbe014 100644
--- a/external/llvm-project/libc/test/src/sched/sched_rr_get_interval_test.cpp
+++ b/external/llvm-project/libc/test/src/sched/sched_rr_get_interval_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sched/sched_get_priority_min.h"
 #include "src/sched/sched_getscheduler.h"
 #include "src/sched/sched_rr_get_interval.h"
@@ -17,7 +17,7 @@
 #include <sched.h>
 
 TEST(LlvmLibcSchedRRGetIntervalTest, SmokeTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   auto SetSched = [&](int policy) {
     int min_priority = LIBC_NAMESPACE::sched_get_priority_min(policy);
     ASSERT_GE(min_priority, 0);
@@ -58,19 +58,19 @@ TEST(LlvmLibcSchedRRGetIntervalTest, SmokeTest) {
     // Null timespec
     ASSERT_EQ(LIBC_NAMESPACE::sched_rr_get_interval(0, nullptr), -1);
     ASSERT_ERRNO_EQ(EFAULT);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     // Negative pid
     ASSERT_EQ(LIBC_NAMESPACE::sched_rr_get_interval(-1, &ts), -1);
     ASSERT_ERRNO_EQ(EINVAL);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
   }
 
   // Negative tests don't have SCHED_RR set
   SetSched(SCHED_OTHER);
   ASSERT_EQ(LIBC_NAMESPACE::sched_rr_get_interval(0, &ts), 0);
   ASSERT_ERRNO_SUCCESS();
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   // TODO: Missing unkown pid -> ESRCH. This is read only so safe to try a few
   //       unlikely values.
diff --git a/external/llvm-project/libc/test/src/sched/yield_test.cpp b/external/llvm-project/libc/test/src/sched/yield_test.cpp
index f1627a71fa9a..4d13d50e25eb 100644
--- a/external/llvm-project/libc/test/src/sched/yield_test.cpp
+++ b/external/llvm-project/libc/test/src/sched/yield_test.cpp
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sched/sched_yield.h"
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcSchedYieldTest, SmokeTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   // sched_yield() always succeeds, just do a basic test that errno/ret are
   // properly 0.
   ASSERT_EQ(LIBC_NAMESPACE::sched_yield(), 0);
diff --git a/external/llvm-project/libc/test/src/signal/sigaltstack_test.cpp b/external/llvm-project/libc/test/src/signal/sigaltstack_test.cpp
index cc392da8f473..ce4dfddae248 100644
--- a/external/llvm-project/libc/test/src/signal/sigaltstack_test.cpp
+++ b/external/llvm-project/libc/test/src/signal/sigaltstack_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/signal_macros.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/signal/linux/signal_utils.h"
 #include "src/signal/raise.h"
 #include "src/signal/sigaction.h"
@@ -46,7 +46,7 @@ static void handler(int) {
 
 TEST(LlvmLibcSignalTest, SigaltstackRunOnAltStack) {
   struct sigaction action;
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(LIBC_NAMESPACE::sigaction(SIGUSR1, nullptr, &action),
               Succeeds(0));
   action.sa_handler = handler;
diff --git a/external/llvm-project/libc/test/src/signal/signal_test.cpp b/external/llvm-project/libc/test/src/signal/signal_test.cpp
index bac9c3b8b68b..62b86bf44029 100644
--- a/external/llvm-project/libc/test/src/signal/signal_test.cpp
+++ b/external/llvm-project/libc/test/src/signal/signal_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/signal/raise.h"
 #include "src/signal/signal.h"
 
@@ -17,7 +17,7 @@ using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
 TEST(LlvmLibcSignal, Invalid) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   auto *valid = +[](int) {};
   EXPECT_THAT((void *)LIBC_NAMESPACE::signal(0, valid),
               Fails(EINVAL, (void *)SIG_ERR));
diff --git a/external/llvm-project/libc/test/src/signal/sigprocmask_test.cpp b/external/llvm-project/libc/test/src/signal/sigprocmask_test.cpp
index 12403f68b593..891eac0f5bf7 100644
--- a/external/llvm-project/libc/test/src/signal/sigprocmask_test.cpp
+++ b/external/llvm-project/libc/test/src/signal/sigprocmask_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/signal/raise.h"
 #include "src/signal/sigaddset.h"
 #include "src/signal/sigemptyset.h"
@@ -33,7 +33,7 @@ using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
 // This tests for invalid input.
 TEST_F(LlvmLibcSignalTest, SigprocmaskInvalid) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   sigset_t valid;
   // 17 and -4 are out of the range for sigprocmask's how paramater.
diff --git a/external/llvm-project/libc/test/src/spawn/posix_spawn_file_actions_test.cpp b/external/llvm-project/libc/test/src/spawn/posix_spawn_file_actions_test.cpp
index c1edf56bdbd8..01ccb8218ee2 100644
--- a/external/llvm-project/libc/test/src/spawn/posix_spawn_file_actions_test.cpp
+++ b/external/llvm-project/libc/test/src/spawn/posix_spawn_file_actions_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/spawn/file_actions.h"
 #include "src/spawn/posix_spawn_file_actions_addclose.h"
 #include "src/spawn/posix_spawn_file_actions_adddup2.h"
diff --git a/external/llvm-project/libc/test/src/stdio/CMakeLists.txt b/external/llvm-project/libc/test/src/stdio/CMakeLists.txt
index 01904a30504e..ce2171f19597 100644
--- a/external/llvm-project/libc/test/src/stdio/CMakeLists.txt
+++ b/external/llvm-project/libc/test/src/stdio/CMakeLists.txt
@@ -357,6 +357,18 @@ add_libc_test(
     libc.src.stdio.puts
 )
 
+add_libc_test(
+  perror_test
+  HERMETIC_TEST_ONLY # writes to libc's stderr
+  SUITE
+    libc_stdio_unittests
+  SRCS
+    perror_test.cpp
+  DEPENDS
+    libc.src.stdio.perror
+    libc.src.errno.errno
+)
+
 add_libc_test(
   fputs_test
   HERMETIC_TEST_ONLY # writes to libc's stdout and stderr
diff --git a/external/llvm-project/libc/test/src/stdio/fdopen_test.cpp b/external/llvm-project/libc/test/src/stdio/fdopen_test.cpp
index ef36cff2ffbd..104fc478b100 100644
--- a/external/llvm-project/libc/test/src/stdio/fdopen_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/fdopen_test.cpp
@@ -9,7 +9,7 @@
 #include "src/stdio/fdopen.h"
 
 #include "hdr/fcntl_macros.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/fcntl/open.h"
 #include "src/stdio/fclose.h"
 #include "src/stdio/fgets.h"
@@ -22,7 +22,7 @@
 
 TEST(LlvmLibcStdioFdopenTest, WriteAppendRead) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   constexpr const char *TEST_FILE_NAME = "testdata/write_read_append.test";
   auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
   int fd = LIBC_NAMESPACE::open(TEST_FILE, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU);
@@ -53,7 +53,7 @@ TEST(LlvmLibcStdioFdopenTest, WriteAppendRead) {
 }
 
 TEST(LlvmLibcStdioFdopenTest, InvalidFd) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   constexpr const char *TEST_FILE_NAME = "testdata/invalid_fd.test";
   auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
   int fd = LIBC_NAMESPACE::open(TEST_FILE, O_CREAT | O_TRUNC);
@@ -65,7 +65,7 @@ TEST(LlvmLibcStdioFdopenTest, InvalidFd) {
 }
 
 TEST(LlvmLibcStdioFdopenTest, InvalidMode) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   constexpr const char *TEST_FILE_NAME = "testdata/invalid_mode.test";
   auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
   int fd = LIBC_NAMESPACE::open(TEST_FILE, O_CREAT | O_RDONLY, S_IRWXU);
@@ -83,7 +83,7 @@ TEST(LlvmLibcStdioFdopenTest, InvalidMode) {
   auto *fp2 = LIBC_NAMESPACE::fdopen(fd, "w");
   ASSERT_ERRNO_EQ(EINVAL);
   ASSERT_TRUE(nullptr == fp2);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   LIBC_NAMESPACE::close(fd);
   ASSERT_ERRNO_SUCCESS();
 }
diff --git a/external/llvm-project/libc/test/src/stdio/fgetc_test.cpp b/external/llvm-project/libc/test/src/stdio/fgetc_test.cpp
index 2cc8436bd66f..56bde5f0099a 100644
--- a/external/llvm-project/libc/test/src/stdio/fgetc_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/fgetc_test.cpp
@@ -17,7 +17,7 @@
 #include "test/UnitTest/Test.h"
 
 #include "hdr/stdio_macros.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 class LlvmLibcGetcTest : public LIBC_NAMESPACE::testing::Test {
 public:
@@ -33,7 +33,7 @@ class LlvmLibcGetcTest : public LIBC_NAMESPACE::testing::Test {
     // This is an error and not a real EOF.
     ASSERT_EQ(LIBC_NAMESPACE::feof(file), 0);
     ASSERT_NE(LIBC_NAMESPACE::ferror(file), 0);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     ASSERT_EQ(0, LIBC_NAMESPACE::fclose(file));
 
diff --git a/external/llvm-project/libc/test/src/stdio/fgetc_unlocked_test.cpp b/external/llvm-project/libc/test/src/stdio/fgetc_unlocked_test.cpp
index 46cf12c2c253..90429ecf4e82 100644
--- a/external/llvm-project/libc/test/src/stdio/fgetc_unlocked_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/fgetc_unlocked_test.cpp
@@ -20,7 +20,7 @@
 #include "test/UnitTest/Test.h"
 
 #include "hdr/stdio_macros.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 class LlvmLibcGetcTest : public LIBC_NAMESPACE::testing::Test {
 public:
@@ -36,7 +36,7 @@ class LlvmLibcGetcTest : public LIBC_NAMESPACE::testing::Test {
     // This is an error and not a real EOF.
     ASSERT_EQ(LIBC_NAMESPACE::feof(file), 0);
     ASSERT_NE(LIBC_NAMESPACE::ferror(file), 0);
-    LIBC_NAMESPACE::libc_errno = 0;
+    libc_errno = 0;
 
     ASSERT_EQ(0, LIBC_NAMESPACE::fclose(file));
 
diff --git a/external/llvm-project/libc/test/src/stdio/fgets_test.cpp b/external/llvm-project/libc/test/src/stdio/fgets_test.cpp
index a8a2c62f07b5..abed3d405293 100644
--- a/external/llvm-project/libc/test/src/stdio/fgets_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/fgets_test.cpp
@@ -14,7 +14,7 @@
 #include "src/stdio/fwrite.h"
 #include "test/UnitTest/Test.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 TEST(LlvmLibcFgetsTest, WriteAndReadCharacters) {
   constexpr char FILENAME[] = "testdata/fgets.test";
@@ -35,7 +35,7 @@ TEST(LlvmLibcFgetsTest, WriteAndReadCharacters) {
   // This is an error and not a real EOF.
   ASSERT_EQ(LIBC_NAMESPACE::feof(file), 0);
   ASSERT_NE(LIBC_NAMESPACE::ferror(file), 0);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   ASSERT_EQ(0, LIBC_NAMESPACE::fclose(file));
 
diff --git a/external/llvm-project/libc/test/src/stdio/fileop_test.cpp b/external/llvm-project/libc/test/src/stdio/fileop_test.cpp
index a0368d701a67..e624181c795b 100644
--- a/external/llvm-project/libc/test/src/stdio/fileop_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/fileop_test.cpp
@@ -21,7 +21,7 @@
 #include "test/UnitTest/Test.h"
 
 #include "hdr/stdio_macros.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::EQ;
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::NE;
@@ -41,7 +41,7 @@ TEST(LlvmLibcFILETest, SimpleFileOperations) {
   ASSERT_THAT(LIBC_NAMESPACE::fread(read_data, 1, sizeof(CONTENT), file),
               returns(EQ(size_t(0))).with_errno(NE(0)));
   ASSERT_NE(LIBC_NAMESPACE::ferror(file), 0);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr(file);
   ASSERT_EQ(LIBC_NAMESPACE::ferror(file), 0);
@@ -72,7 +72,7 @@ TEST(LlvmLibcFILETest, SimpleFileOperations) {
   ASSERT_THAT(LIBC_NAMESPACE::fwrite(CONTENT, 1, sizeof(CONTENT), file),
               returns(EQ(size_t(0))).with_errno(NE(0)));
   ASSERT_NE(LIBC_NAMESPACE::ferror(file), 0);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr(file);
 
@@ -80,15 +80,15 @@ TEST(LlvmLibcFILETest, SimpleFileOperations) {
   ASSERT_THAT(LIBC_NAMESPACE::fputs(CONTENT, file),
               returns(EQ(EOF)).with_errno(NE(0)));
   ASSERT_NE(LIBC_NAMESPACE::ferror(file), 0);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr(file);
   ASSERT_EQ(LIBC_NAMESPACE::ferror(file), 0);
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(LIBC_NAMESPACE::fwrite("nothing", 1, 1, file),
               returns(EQ(size_t(0))).with_errno(NE(0)));
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   ASSERT_EQ(LIBC_NAMESPACE::fclose(file), 0);
 
@@ -103,10 +103,10 @@ TEST(LlvmLibcFILETest, SimpleFileOperations) {
   ASSERT_EQ(LIBC_NAMESPACE::ferror(file), 0);
 
   // This is not a readable file.
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(LIBC_NAMESPACE::fread(data, 1, 1, file),
               returns(EQ(0)).with_errno(NE(0)));
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   ASSERT_EQ(0, LIBC_NAMESPACE::fclose(file));
 
@@ -121,15 +121,15 @@ TEST(LlvmLibcFILETest, SimpleFileOperations) {
 
   // Check that the other functions correctly set libc_errno.
 
-  // LIBC_NAMESPACE::libc_errno = 0;
+  // libc_errno = 0;
   // ASSERT_NE(LIBC_NAMESPACE::fseek(file, 0, SEEK_SET), 0);
   // ASSERT_ERRNO_FAILURE();
 
-  // LIBC_NAMESPACE::libc_errno = 0;
+  // libc_errno = 0;
   // ASSERT_NE(LIBC_NAMESPACE::fclose(file), 0);
   // ASSERT_ERRNO_FAILURE();
 
-  // LIBC_NAMESPACE::libc_errno = 0;
+  // libc_errno = 0;
   // ASSERT_EQ(LIBC_NAMESPACE::fopen("INVALID FILE NAME", "r"),
   //           static_cast<FILE *>(nullptr));
   // ASSERT_ERRNO_FAILURE();
@@ -165,7 +165,7 @@ TEST(LlvmLibcFILETest, FOpenFWriteSizeGreaterThanOne) {
   constexpr size_t WRITE_NMEMB = sizeof(WRITE_DATA) / sizeof(MyStruct);
   constexpr char FILENAME[] = "testdata/fread_fwrite.test";
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   FILE *file = LIBC_NAMESPACE::fopen(FILENAME, "w");
   ASSERT_FALSE(file == nullptr);
   ASSERT_EQ(size_t(0), LIBC_NAMESPACE::fwrite(WRITE_DATA, 0, 1, file));
diff --git a/external/llvm-project/libc/test/src/stdio/fopencookie_test.cpp b/external/llvm-project/libc/test/src/stdio/fopencookie_test.cpp
index 61ce2a207fa1..03e1ac286b64 100644
--- a/external/llvm-project/libc/test/src/stdio/fopencookie_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/fopencookie_test.cpp
@@ -20,7 +20,7 @@
 
 #include "hdr/stdio_macros.h"
 #include "hdr/types/size_t.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 using MemoryView = LIBC_NAMESPACE::testing::MemoryView;
 
@@ -67,7 +67,7 @@ int seek_ss(void *cookie, off64_t *offset, int whence) {
   } else if (whence == SEEK_END) {
     new_offset = *offset + ss->endpos;
   } else {
-    LIBC_NAMESPACE::libc_errno = EINVAL;
+    libc_errno = EINVAL;
     return -1;
   }
   if (new_offset < 0 || size_t(new_offset) > ss->bufsize)
@@ -115,7 +115,7 @@ TEST(LlvmLibcFOpenCookie, ReadOnlyCookieTest) {
   ASSERT_EQ(size_t(0), LIBC_NAMESPACE::fwrite(CONTENT, 1, sizeof(CONTENT), f));
   ASSERT_NE(LIBC_NAMESPACE::ferror(f), 0);
   ASSERT_ERRNO_FAILURE();
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr(f);
   ASSERT_EQ(LIBC_NAMESPACE::ferror(f), 0);
@@ -149,7 +149,7 @@ TEST(LlvmLibcFOpenCookie, WriteOnlyCookieTest) {
             LIBC_NAMESPACE::fread(read_data, 1, sizeof(WRITE_DATA), f));
   ASSERT_NE(LIBC_NAMESPACE::ferror(f), 0);
   ASSERT_ERRNO_EQ(EBADF);
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr(f);
   ASSERT_EQ(LIBC_NAMESPACE::ferror(f), 0);
@@ -178,7 +178,7 @@ TEST(LlvmLibcFOpenCookie, AppendOnlyCookieTest) {
   ASSERT_EQ(LIBC_NAMESPACE::fread(read_data, 1, READ_SIZE, f), size_t(0));
   ASSERT_NE(LIBC_NAMESPACE::ferror(f), 0);
   ASSERT_ERRNO_FAILURE();
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr(f);
   ASSERT_EQ(LIBC_NAMESPACE::ferror(f), 0);
diff --git a/external/llvm-project/libc/test/src/stdio/perror_test.cpp b/external/llvm-project/libc/test/src/stdio/perror_test.cpp
new file mode 100644
index 000000000000..9a97be2eff21
--- /dev/null
+++ b/external/llvm-project/libc/test/src/stdio/perror_test.cpp
@@ -0,0 +1,32 @@
+//===-- Unittests for perror ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdio/perror.h"
+
+#include "src/__support/libc_errno.h"
+#include "test/UnitTest/Test.h"
+
+// The standard says perror prints directly to stderr and returns nothing. This
+// makes it rather difficult to test automatically.
+
+// TODO: figure out redirecting stderr so this test can check correctness.
+TEST(LlvmLibcPerrorTest, PrintOut) {
+  LIBC_NAMESPACE::libc_errno = 0;
+  constexpr char simple[] = "A simple string";
+  LIBC_NAMESPACE::perror(simple);
+
+  // stick to stdc errno values, specifically 0, EDOM, ERANGE, and EILSEQ.
+  LIBC_NAMESPACE::libc_errno = EDOM;
+  LIBC_NAMESPACE::perror("Print this and an error");
+
+  LIBC_NAMESPACE::libc_errno = EILSEQ;
+  LIBC_NAMESPACE::perror("\0 shouldn't print this.");
+
+  LIBC_NAMESPACE::libc_errno = ERANGE;
+  LIBC_NAMESPACE::perror(nullptr);
+}
diff --git a/external/llvm-project/libc/test/src/stdio/printf_core/converter_test.cpp b/external/llvm-project/libc/test/src/stdio/printf_core/converter_test.cpp
index 96a00ae598ec..bf088937e410 100644
--- a/external/llvm-project/libc/test/src/stdio/printf_core/converter_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/printf_core/converter_test.cpp
@@ -124,7 +124,7 @@ TEST_F(LlvmLibcPrintfConverterTest, StringConversionSimple) {
 TEST_F(LlvmLibcPrintfConverterTest, StringConversionPrecisionHigh) {
   LIBC_NAMESPACE::printf_core::FormatSection high_precision_conv;
   high_precision_conv.has_conv = true;
-  high_precision_conv.raw_string = "%4s";
+  high_precision_conv.raw_string = "%.4s";
   high_precision_conv.conv_name = 's';
   high_precision_conv.precision = 4;
   high_precision_conv.conv_val_ptr = const_cast<char *>("456");
diff --git a/external/llvm-project/libc/test/src/stdio/remove_test.cpp b/external/llvm-project/libc/test/src/stdio/remove_test.cpp
index 72875600903a..84984e26398c 100644
--- a/external/llvm-project/libc/test/src/stdio/remove_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/remove_test.cpp
@@ -14,13 +14,13 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include <unistd.h>
 
 TEST(LlvmLibcRemoveTest, CreateAndRemoveFile) {
   // The test strategy is to create a file and remove it, and also verify that
   // it was removed.
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
@@ -39,7 +39,7 @@ TEST(LlvmLibcRemoveTest, CreateAndRemoveFile) {
 TEST(LlvmLibcRemoveTest, CreateAndRemoveDir) {
   // The test strategy is to create a dir and remove it, and also verify that
   // it was removed.
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *FILENAME = "remove.test.dir";
diff --git a/external/llvm-project/libc/test/src/stdio/rename_test.cpp b/external/llvm-project/libc/test/src/stdio/rename_test.cpp
index a5dd734c6361..ac494a4ecaf8 100644
--- a/external/llvm-project/libc/test/src/stdio/rename_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/rename_test.cpp
@@ -8,7 +8,7 @@
 
 #include "include/llvm-libc-macros/linux/sys-stat-macros.h"
 #include "include/llvm-libc-macros/linux/unistd-macros.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/fcntl/open.h"
 #include "src/stdio/rename.h"
 #include "src/unistd/access.h"
@@ -19,7 +19,7 @@
 TEST(LlvmLibcRenameTest, CreateAndRenameFile) {
   // The test strategy is to create a file and rename it, and also verify that
   // it was renamed.
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
diff --git a/external/llvm-project/libc/test/src/stdio/setvbuf_test.cpp b/external/llvm-project/libc/test/src/stdio/setvbuf_test.cpp
index a1e1fee25db3..5872943c1bb4 100644
--- a/external/llvm-project/libc/test/src/stdio/setvbuf_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/setvbuf_test.cpp
@@ -14,7 +14,7 @@
 #include "test/UnitTest/Test.h"
 
 #include "hdr/stdio_macros.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 TEST(LlvmLibcSetvbufTest, SetNBFBuffer) {
   // The idea in this test is that we open a file for writing and reading, and
@@ -102,6 +102,6 @@ TEST(LlvmLibcSetbufTest, InvalidBufferMode) {
             0);
   ASSERT_ERRNO_EQ(EINVAL);
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_EQ(0, LIBC_NAMESPACE::fclose(f));
 }
diff --git a/external/llvm-project/libc/test/src/stdio/sprintf_test.cpp b/external/llvm-project/libc/test/src/stdio/sprintf_test.cpp
index f6af6ad3e364..f1b545ba546f 100644
--- a/external/llvm-project/libc/test/src/stdio/sprintf_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/sprintf_test.cpp
@@ -10,7 +10,7 @@
 #include "src/stdio/sprintf.h"
 
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "test/UnitTest/RoundingModeUtils.h"
 #include "test/UnitTest/Test.h"
 #include <inttypes.h>
@@ -3228,46 +3228,46 @@ TEST(LlvmLibcSPrintfTest, StrerrorConv) {
   char buff[1000];
   int written;
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%m");
   ASSERT_STREQ_LEN(written, buff, "Success");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%m");
   ASSERT_STREQ_LEN(written, buff, "Numerical result out of range");
 
   // Check that it correctly consumes no arguments.
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%m %d", 1);
   ASSERT_STREQ_LEN(written, buff, "Success 1");
 
   // Width Tests
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%10m");
   ASSERT_STREQ_LEN(written, buff, "   Success");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%10m");
   ASSERT_STREQ_LEN(written, buff, "Numerical result out of range");
 
   // Precision Tests
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%.10m");
   ASSERT_STREQ_LEN(written, buff, "Success");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%.10m");
   ASSERT_STREQ_LEN(written, buff, "Numerical ");
 
   // Flag Tests (Only '-' since the others only affect ints)
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%-10m");
   ASSERT_STREQ_LEN(written, buff, "Success   ");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%-10m");
   ASSERT_STREQ_LEN(written, buff, "Numerical result out of range");
 
@@ -3275,93 +3275,93 @@ TEST(LlvmLibcSPrintfTest, StrerrorConv) {
   // Since alt mode here is effectively a completely separate conversion, it
   // gets separate tests.
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%#m");
   ASSERT_STREQ_LEN(written, buff, "0");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#m");
   ASSERT_STREQ_LEN(written, buff, "-9999");
 
   // Alt Mode Width
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%#10m");
   ASSERT_STREQ_LEN(written, buff, "         0");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#10m");
   ASSERT_STREQ_LEN(written, buff, "    ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#10m");
   ASSERT_STREQ_LEN(written, buff, "     -9999");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#3m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#3m");
   ASSERT_STREQ_LEN(written, buff, "-9999");
 
   // Alt Mode Precision
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#.10m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#.10m");
   ASSERT_STREQ_LEN(written, buff, "-0000009999");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#.3m");
   ASSERT_STREQ_LEN(written, buff, "ERA");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#.3m");
   ASSERT_STREQ_LEN(written, buff, "-9999");
 
   // We don't test precision (or int flags) on errno = 0 because it behaves
   // weirdly, see the docs for more information.
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%#.1m");
   ASSERT_STREQ_LEN(written, buff, "0");
 
   // Alt Mode Flags
 
   // '-' flag
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   written = LIBC_NAMESPACE::sprintf(buff, "%#-10m");
   ASSERT_STREQ_LEN(written, buff, "0         ");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#-10m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE    ");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#-10m");
   ASSERT_STREQ_LEN(written, buff, "-9999     ");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#-3m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#-3m");
   ASSERT_STREQ_LEN(written, buff, "-9999");
 
   // '+' flag
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#+m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#+m");
   ASSERT_STREQ_LEN(written, buff, "-9999");
 
@@ -3370,38 +3370,38 @@ TEST(LlvmLibcSPrintfTest, StrerrorConv) {
   // come up, but I've avoided it for the other %m tests for ease of
   // refactoring if necessary. Here it needs to be positive to test that the
   // flags that only affect positive signed integers are properly passed along.
-  LIBC_NAMESPACE::libc_errno = 9999;
+  libc_errno = 9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#+m");
   ASSERT_STREQ_LEN(written, buff, "+9999");
 
   // ' ' flag
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%# m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%# m");
   ASSERT_STREQ_LEN(written, buff, "-9999");
 
-  LIBC_NAMESPACE::libc_errno = 9999;
+  libc_errno = 9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%# m");
   ASSERT_STREQ_LEN(written, buff, " 9999");
 
   // '0' flag
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#010m");
   ASSERT_STREQ_LEN(written, buff, "    ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#010m");
   ASSERT_STREQ_LEN(written, buff, "-000009999");
 
-  LIBC_NAMESPACE::libc_errno = ERANGE;
+  libc_errno = ERANGE;
   written = LIBC_NAMESPACE::sprintf(buff, "%#03m");
   ASSERT_STREQ_LEN(written, buff, "ERANGE");
 
-  LIBC_NAMESPACE::libc_errno = -9999;
+  libc_errno = -9999;
   written = LIBC_NAMESPACE::sprintf(buff, "%#03m");
   ASSERT_STREQ_LEN(written, buff, "-9999");
 }
diff --git a/external/llvm-project/libc/test/src/stdio/unlocked_fileop_test.cpp b/external/llvm-project/libc/test/src/stdio/unlocked_fileop_test.cpp
index 67f1b0ff513b..5d482b70064b 100644
--- a/external/llvm-project/libc/test/src/stdio/unlocked_fileop_test.cpp
+++ b/external/llvm-project/libc/test/src/stdio/unlocked_fileop_test.cpp
@@ -17,7 +17,7 @@
 #include "src/stdio/fwrite_unlocked.h"
 #include "test/UnitTest/Test.h"
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 
 TEST(LlvmLibcFILETest, UnlockedReadAndWrite) {
   constexpr char fNAME[] = "testdata/unlocked_read_and_write.test";
@@ -36,7 +36,7 @@ TEST(LlvmLibcFILETest, UnlockedReadAndWrite) {
             LIBC_NAMESPACE::fread_unlocked(data, 1, sizeof(READ_SIZE), f));
   ASSERT_NE(LIBC_NAMESPACE::ferror_unlocked(f), 0);
   ASSERT_ERRNO_FAILURE();
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr_unlocked(f);
   ASSERT_EQ(LIBC_NAMESPACE::ferror_unlocked(f), 0);
@@ -57,7 +57,7 @@ TEST(LlvmLibcFILETest, UnlockedReadAndWrite) {
             LIBC_NAMESPACE::fwrite_unlocked(CONTENT, 1, sizeof(CONTENT), f));
   ASSERT_NE(LIBC_NAMESPACE::ferror_unlocked(f), 0);
   ASSERT_ERRNO_FAILURE();
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   LIBC_NAMESPACE::clearerr_unlocked(f);
   ASSERT_EQ(LIBC_NAMESPACE::ferror_unlocked(f), 0);
diff --git a/external/llvm-project/libc/test/src/stdlib/CMakeLists.txt b/external/llvm-project/libc/test/src/stdlib/CMakeLists.txt
index 302971a078c1..45fd49b6d352 100644
--- a/external/llvm-project/libc/test/src/stdlib/CMakeLists.txt
+++ b/external/llvm-project/libc/test/src/stdlib/CMakeLists.txt
@@ -9,6 +9,7 @@ add_libc_test(
   DEPENDS
     libc.src.errno.errno
     libc.src.stdlib.atof
+    libc.test.UnitTest.ErrnoCheckingTest
 )
 
 add_header_library(
@@ -64,6 +65,7 @@ add_fp_unittest(
     libc.src.errno.errno
     libc.src.stdlib.strtod
     libc.src.__support.FPUtil.fenv_impl
+    libc.test.UnitTest.ErrnoCheckingTest
 )
 
 add_fp_unittest(
@@ -76,6 +78,7 @@ add_fp_unittest(
     libc.src.errno.errno
     libc.src.stdlib.strtof
     libc.src.__support.FPUtil.fenv_impl
+    libc.test.UnitTest.ErrnoCheckingTest
 )
 
 add_header_library(
@@ -86,6 +89,7 @@ add_header_library(
     libc.src.__support.CPP.limits
     libc.src.__support.CPP.type_traits
     libc.src.errno.errno
+    libc.test.UnitTest.ErrnoCheckingTest
 )
 
 add_libc_test(
@@ -133,6 +137,7 @@ add_libc_test(
     libc.src.errno.errno
     libc.src.__support.uint128
     libc.src.stdlib.strtold
+    libc.test.UnitTest.ErrnoCheckingTest
 )
 
 add_libc_test(
diff --git a/external/llvm-project/libc/test/src/stdlib/StrtolTest.h b/external/llvm-project/libc/test/src/stdlib/StrtolTest.h
index ed302f14d03e..3eeccc5727e7 100644
--- a/external/llvm-project/libc/test/src/stdlib/StrtolTest.h
+++ b/external/llvm-project/libc/test/src/stdlib/StrtolTest.h
@@ -9,8 +9,9 @@
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/ctype_utils.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/properties/architectures.h"
-#include "src/errno/libc_errno.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/Test.h"
 
 #include <stddef.h>
@@ -18,7 +19,7 @@
 using LIBC_NAMESPACE::cpp::is_signed_v;
 
 template <typename ReturnT>
-struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
+struct StrtoTest : public LIBC_NAMESPACE::testing::ErrnoCheckingTest {
   using FunctionT = ReturnT (*)(const char *, char **, int);
 
   static constexpr ReturnT T_MAX =
@@ -28,7 +29,6 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
 
   void InvalidBase(FunctionT func) {
     const char *ten = "10";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(ten, nullptr, -1), ReturnT(0));
     ASSERT_ERRNO_EQ(EINVAL);
   }
@@ -38,23 +38,19 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
 
     // TODO: Look into collapsing these repeated segments.
     const char *ten = "10";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(ten, &str_end, 10), ReturnT(10));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - ten, ptrdiff_t(2));
 
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(ten, nullptr, 10), ReturnT(10));
     ASSERT_ERRNO_SUCCESS();
 
     const char *hundred = "100";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(hundred, &str_end, 10), ReturnT(100));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - hundred, ptrdiff_t(3));
 
     const char *big_number = "1234567890";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(big_number, &str_end, 10), ReturnT(1234567890));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - big_number, ptrdiff_t(10));
@@ -62,7 +58,6 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     // This number is larger than 2^32, meaning that if long is only 32 bits
     // wide, strtol will return LONG_MAX.
     const char *bigger_number = "12345678900";
-    LIBC_NAMESPACE::libc_errno = 0;
     if constexpr (sizeof(ReturnT) < 8) {
       ASSERT_EQ(func(bigger_number, &str_end, 10), T_MAX);
       ASSERT_ERRNO_EQ(ERANGE);
@@ -73,14 +68,12 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     EXPECT_EQ(str_end - bigger_number, ptrdiff_t(11));
 
     const char *too_big_number = "123456789012345678901";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(too_big_number, &str_end, 10), T_MAX);
     ASSERT_ERRNO_EQ(ERANGE);
     EXPECT_EQ(str_end - too_big_number, ptrdiff_t(21));
 
     const char *long_number_range_test =
         "10000000000000000000000000000000000000000000000000";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(long_number_range_test, &str_end, 10), T_MAX);
     ASSERT_ERRNO_EQ(ERANGE);
     EXPECT_EQ(str_end - long_number_range_test, ptrdiff_t(50));
@@ -88,19 +81,16 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     // For most negative numbers, the unsigned functions treat it the same as
     // casting a negative variable to an unsigned type.
     const char *negative = "-100";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(negative, &str_end, 10), ReturnT(-100));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - negative, ptrdiff_t(4));
 
     const char *big_negative_number = "-1234567890";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(big_negative_number, &str_end, 10), ReturnT(-1234567890));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - big_negative_number, ptrdiff_t(11));
 
     const char *too_big_negative_number = "-123456789012345678901";
-    LIBC_NAMESPACE::libc_errno = 0;
     // If the number is signed, it should return the smallest negative number
     // for the current type, but if it's unsigned it should max out and return
     // the largest positive number for the current type. From the standard:
@@ -118,73 +108,61 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     char *str_end = nullptr;
 
     const char *spaces_before = "     10";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(spaces_before, &str_end, 10), ReturnT(10));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - spaces_before, ptrdiff_t(7));
 
     const char *spaces_after = "10      ";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(spaces_after, &str_end, 10), ReturnT(10));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - spaces_after, ptrdiff_t(2));
 
     const char *word_before = "word10";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(word_before, &str_end, 10), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - word_before, ptrdiff_t(0));
 
     const char *word_after = "10word";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(word_after, &str_end, 10), ReturnT(10));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - word_after, ptrdiff_t(2));
 
     const char *two_numbers = "10 999";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(two_numbers, &str_end, 10), ReturnT(10));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - two_numbers, ptrdiff_t(2));
 
     const char *two_signs = "--10 999";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(two_signs, &str_end, 10), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - two_signs, ptrdiff_t(0));
 
     const char *sign_before = "+2=4";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(sign_before, &str_end, 10), ReturnT(2));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - sign_before, ptrdiff_t(2));
 
     const char *sign_after = "2+2=4";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(sign_after, &str_end, 10), ReturnT(2));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - sign_after, ptrdiff_t(1));
 
     const char *tab_before = "\t10";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(tab_before, &str_end, 10), ReturnT(10));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - tab_before, ptrdiff_t(3));
 
     const char *all_together = "\t  -12345and+67890";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(all_together, &str_end, 10), ReturnT(-12345));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - all_together, ptrdiff_t(9));
 
     const char *just_spaces = "  ";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(just_spaces, &str_end, 10), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - just_spaces, ptrdiff_t(0));
 
     const char *just_space_and_sign = " +";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(just_space_and_sign, &str_end, 10), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - just_space_and_sign, ptrdiff_t(0));
@@ -203,12 +181,10 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
         small_string[0] = static_cast<char>(
             LIBC_NAMESPACE::internal::int_to_b36_char(first_digit));
         if (first_digit < base) {
-          LIBC_NAMESPACE::libc_errno = 0;
           ASSERT_EQ(func(small_string, nullptr, base),
                     static_cast<ReturnT>(first_digit));
           ASSERT_ERRNO_SUCCESS();
         } else {
-          LIBC_NAMESPACE::libc_errno = 0;
           ASSERT_EQ(func(small_string, nullptr, base), ReturnT(0));
           ASSERT_ERRNO_SUCCESS();
         }
@@ -223,18 +199,15 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
           small_string[1] = static_cast<char>(
               LIBC_NAMESPACE::internal::int_to_b36_char(second_digit));
           if (first_digit < base && second_digit < base) {
-            LIBC_NAMESPACE::libc_errno = 0;
             ASSERT_EQ(
                 func(small_string, nullptr, base),
                 static_cast<ReturnT>(second_digit + (first_digit * base)));
             ASSERT_ERRNO_SUCCESS();
           } else if (first_digit < base) {
-            LIBC_NAMESPACE::libc_errno = 0;
             ASSERT_EQ(func(small_string, nullptr, base),
                       static_cast<ReturnT>(first_digit));
             ASSERT_ERRNO_SUCCESS();
           } else {
-            LIBC_NAMESPACE::libc_errno = 0;
             ASSERT_EQ(func(small_string, nullptr, base), ReturnT(0));
             ASSERT_ERRNO_SUCCESS();
           }
@@ -255,14 +228,12 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
 
             if (first_digit < base && second_digit < base &&
                 third_digit < base) {
-              LIBC_NAMESPACE::libc_errno = 0;
               ASSERT_EQ(func(small_string, nullptr, base),
                         static_cast<ReturnT>(third_digit +
                                              (second_digit * base) +
                                              (first_digit * base * base)));
               ASSERT_ERRNO_SUCCESS();
             } else if (first_digit < base && second_digit < base) {
-              LIBC_NAMESPACE::libc_errno = 0;
               ASSERT_EQ(
                   func(small_string, nullptr, base),
                   static_cast<ReturnT>(second_digit + (first_digit * base)));
@@ -272,23 +243,19 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
               // The number is treated as a one digit hexadecimal.
               if (base == 16 && first_digit == 0 && second_digit == 33) {
                 if (third_digit < base) {
-                  LIBC_NAMESPACE::libc_errno = 0;
                   ASSERT_EQ(func(small_string, nullptr, base),
                             static_cast<ReturnT>(third_digit));
                   ASSERT_ERRNO_SUCCESS();
                 } else {
-                  LIBC_NAMESPACE::libc_errno = 0;
                   ASSERT_EQ(func(small_string, nullptr, base), ReturnT(0));
                   ASSERT_ERRNO_SUCCESS();
                 }
               } else {
-                LIBC_NAMESPACE::libc_errno = 0;
                 ASSERT_EQ(func(small_string, nullptr, base),
                           static_cast<ReturnT>(first_digit));
                 ASSERT_ERRNO_SUCCESS();
               }
             } else {
-              LIBC_NAMESPACE::libc_errno = 0;
               ASSERT_EQ(func(small_string, nullptr, base), ReturnT(0));
               ASSERT_ERRNO_SUCCESS();
             }
@@ -302,19 +269,16 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     char *str_end = nullptr;
 
     const char *no_prefix = "123abc";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(no_prefix, &str_end, 16), ReturnT(0x123abc));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - no_prefix, ptrdiff_t(6));
 
     const char *yes_prefix = "0x456def";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(yes_prefix, &str_end, 16), ReturnT(0x456def));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - yes_prefix, ptrdiff_t(8));
 
     const char *letter_after_prefix = "0xabc123";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(letter_after_prefix, &str_end, 16), ReturnT(0xabc123));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - letter_after_prefix, ptrdiff_t(8));
@@ -325,7 +289,6 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     // Max size for unsigned 32 bit numbers
 
     const char *max_32_bit_value = "0xFFFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(max_32_bit_value, &str_end, 0),
               ((is_signed_v<ReturnT> && sizeof(ReturnT) == 4)
                    ? T_MAX
@@ -334,7 +297,6 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     EXPECT_EQ(str_end - max_32_bit_value, ptrdiff_t(10));
 
     const char *negative_max_32_bit_value = "-0xFFFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(negative_max_32_bit_value, &str_end, 0),
               ((is_signed_v<ReturnT> && sizeof(ReturnT) == 4)
                    ? T_MIN
@@ -345,13 +307,11 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     // Max size for signed 32 bit numbers
 
     const char *max_31_bit_value = "0x7FFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(max_31_bit_value, &str_end, 0), ReturnT(0x7FFFFFFF));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - max_31_bit_value, ptrdiff_t(10));
 
     const char *negative_max_31_bit_value = "-0x7FFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(negative_max_31_bit_value, &str_end, 0),
               -ReturnT(0x7FFFFFFF));
     ASSERT_ERRNO_SUCCESS();
@@ -360,7 +320,6 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     // Max size for unsigned 64 bit numbers
 
     const char *max_64_bit_value = "0xFFFFFFFFFFFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(max_64_bit_value, &str_end, 0),
               (is_signed_v<ReturnT> || sizeof(ReturnT) < 8
                    ? T_MAX
@@ -371,7 +330,6 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     // See the end of CleanBase10Decode for an explanation of how this large
     // negative number can end up as T_MAX.
     const char *negative_max_64_bit_value = "-0xFFFFFFFFFFFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(
         func(negative_max_64_bit_value, &str_end, 0),
         (is_signed_v<ReturnT>
@@ -383,14 +341,12 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     // Max size for signed 64 bit numbers
 
     const char *max_63_bit_value = "0x7FFFFFFFFFFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(max_63_bit_value, &str_end, 0),
               (sizeof(ReturnT) < 8 ? T_MAX : ReturnT(0x7FFFFFFFFFFFFFFF)));
     ASSERT_ERRNO_EQ(sizeof(ReturnT) < 8 ? ERANGE : 0);
     EXPECT_EQ(str_end - max_63_bit_value, ptrdiff_t(18));
 
     const char *negative_max_63_bit_value = "-0x7FFFFFFFFFFFFFFF";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(negative_max_63_bit_value, &str_end, 0),
               (sizeof(ReturnT) >= 8 ? -ReturnT(0x7FFFFFFFFFFFFFFF)
                                     : (is_signed_v<ReturnT> ? T_MIN : T_MAX)));
@@ -402,23 +358,19 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     char *str_end = nullptr;
 
     const char *just_prefix = "0x";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(just_prefix, &str_end, 16), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - just_prefix, ptrdiff_t(1));
 
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(just_prefix, &str_end, 0), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - just_prefix, ptrdiff_t(1));
 
     const char *prefix_with_x_after = "0xx";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(prefix_with_x_after, &str_end, 16), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - prefix_with_x_after, ptrdiff_t(1));
 
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(prefix_with_x_after, &str_end, 0), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - prefix_with_x_after, ptrdiff_t(1));
@@ -428,43 +380,36 @@ struct StrtoTest : public LIBC_NAMESPACE::testing::Test {
     char *str_end = nullptr;
 
     const char *base_ten = "12345";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(base_ten, &str_end, 0), ReturnT(12345));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - base_ten, ptrdiff_t(5));
 
     const char *base_sixteen_no_prefix = "123abc";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(base_sixteen_no_prefix, &str_end, 0), ReturnT(123));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - base_sixteen_no_prefix, ptrdiff_t(3));
 
     const char *base_sixteen_with_prefix = "0x456def";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(base_sixteen_with_prefix, &str_end, 0), ReturnT(0x456def));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - base_sixteen_with_prefix, ptrdiff_t(8));
 
     const char *base_eight_with_prefix = "012345";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(base_eight_with_prefix, &str_end, 0), ReturnT(012345));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - base_eight_with_prefix, ptrdiff_t(6));
 
     const char *just_zero = "0";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(just_zero, &str_end, 0), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - just_zero, ptrdiff_t(1));
 
     const char *just_zero_x = "0x";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(just_zero_x, &str_end, 0), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - just_zero_x, ptrdiff_t(1));
 
     const char *just_zero_eight = "08";
-    LIBC_NAMESPACE::libc_errno = 0;
     ASSERT_EQ(func(just_zero_eight, &str_end, 0), ReturnT(0));
     ASSERT_ERRNO_SUCCESS();
     EXPECT_EQ(str_end - just_zero_eight, ptrdiff_t(1));
diff --git a/external/llvm-project/libc/test/src/stdlib/atof_test.cpp b/external/llvm-project/libc/test/src/stdlib/atof_test.cpp
index 1e4259b792d7..92b904ecad94 100644
--- a/external/llvm-project/libc/test/src/stdlib/atof_test.cpp
+++ b/external/llvm-project/libc/test/src/stdlib/atof_test.cpp
@@ -7,29 +7,28 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
 #include "src/stdlib/atof.h"
 
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 #include <stddef.h>
 
+using LlvmLibcAToFTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
 // This is just a simple test to make sure that this function works at all. It's
 // functionally identical to strtod so the bulk of the testing is there.
-TEST(LlvmLibcAToFTest, SimpleTest) {
+TEST_F(LlvmLibcAToFTest, SimpleTest) {
   LIBC_NAMESPACE::fputil::FPBits<double> expected_fp =
       LIBC_NAMESPACE::fputil::FPBits<double>(uint64_t(0x405ec00000000000));
 
-  LIBC_NAMESPACE::libc_errno = 0;
   EXPECT_THAT(LIBC_NAMESPACE::atof("123"),
               Succeeds<double>(expected_fp.get_val()));
 }
 
-TEST(LlvmLibcAToFTest, FailedParsingTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+TEST_F(LlvmLibcAToFTest, FailedParsingTest) {
   // atof does not flag errors.
   EXPECT_THAT(LIBC_NAMESPACE::atof("???"), Succeeds<double>(0.0));
 }
diff --git a/external/llvm-project/libc/test/src/stdlib/strtod_test.cpp b/external/llvm-project/libc/test/src/stdlib/strtod_test.cpp
index 92d14640e653..db3c1d73bd22 100644
--- a/external/llvm-project/libc/test/src/stdlib/strtod_test.cpp
+++ b/external/llvm-project/libc/test/src/stdlib/strtod_test.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
 #include "src/stdlib/strtod.h"
 
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/RoundingModeUtils.h"
 #include "test/UnitTest/Test.h"
@@ -22,7 +22,7 @@ using LIBC_NAMESPACE::fputil::testing::RoundingMode;
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
-class LlvmLibcStrToDTest : public LIBC_NAMESPACE::testing::Test,
+class LlvmLibcStrToDTest : public LIBC_NAMESPACE::testing::ErrnoCheckingTest,
                            ForceRoundingModeTest<RoundingMode::Nearest> {
 public:
   void run_test(const char *inputString, const ptrdiff_t expectedStrLen,
@@ -46,7 +46,6 @@ class LlvmLibcStrToDTest : public LIBC_NAMESPACE::testing::Test,
     LIBC_NAMESPACE::fputil::FPBits<double> expected_fp =
         LIBC_NAMESPACE::fputil::FPBits<double>(expectedRawData);
 
-    LIBC_NAMESPACE::libc_errno = 0;
     double result = LIBC_NAMESPACE::strtod(inputString, &str_end);
     if (expectedErrno == 0)
       EXPECT_THAT(result, Succeeds<double>(expected_fp.get_val()));
diff --git a/external/llvm-project/libc/test/src/stdlib/strtof_test.cpp b/external/llvm-project/libc/test/src/stdlib/strtof_test.cpp
index 6a716c956291..6df1ddda93bf 100644
--- a/external/llvm-project/libc/test/src/stdlib/strtof_test.cpp
+++ b/external/llvm-project/libc/test/src/stdlib/strtof_test.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/errno/libc_errno.h"
 #include "src/stdlib/strtof.h"
 
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/RoundingModeUtils.h"
 #include "test/UnitTest/Test.h"
@@ -19,7 +19,7 @@
 using LIBC_NAMESPACE::fputil::testing::ForceRoundingModeTest;
 using LIBC_NAMESPACE::fputil::testing::RoundingMode;
 
-class LlvmLibcStrToFTest : public LIBC_NAMESPACE::testing::Test,
+class LlvmLibcStrToFTest : public LIBC_NAMESPACE::testing::ErrnoCheckingTest,
                            ForceRoundingModeTest<RoundingMode::Nearest> {
 public:
   void run_test(const char *inputString, const ptrdiff_t expectedStrLen,
@@ -43,7 +43,6 @@ class LlvmLibcStrToFTest : public LIBC_NAMESPACE::testing::Test,
     LIBC_NAMESPACE::fputil::FPBits<float> expected_fp =
         LIBC_NAMESPACE::fputil::FPBits<float>(expectedRawData);
 
-    LIBC_NAMESPACE::libc_errno = 0;
     float result = LIBC_NAMESPACE::strtof(inputString, &str_end);
 
     EXPECT_EQ(str_end - inputString, expectedStrLen);
diff --git a/external/llvm-project/libc/test/src/stdlib/strtoint32_test.cpp b/external/llvm-project/libc/test/src/stdlib/strtoint32_test.cpp
index 17df432fc8e6..e6da692714d2 100644
--- a/external/llvm-project/libc/test/src/stdlib/strtoint32_test.cpp
+++ b/external/llvm-project/libc/test/src/stdlib/strtoint32_test.cpp
@@ -8,9 +8,9 @@
 
 #include <stdint.h>
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 #include "StrtolTest.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ int32_t strtoint32(const char *__restrict str, char **__restrict str_end,
                    int base) {
   auto result = internal::strtointeger<int32_t>(str, base);
   if (result.has_error())
-    LIBC_NAMESPACE::libc_errno = result.error;
+    libc_errno = result.error;
 
   if (str_end != nullptr)
     *str_end = const_cast<char *>(str + result.parsed_len);
@@ -33,7 +33,7 @@ uint32_t strtouint32(const char *__restrict str, char **__restrict str_end,
                      int base) {
   auto result = internal::strtointeger<uint32_t>(str, base);
   if (result.has_error())
-    LIBC_NAMESPACE::libc_errno = result.error;
+    libc_errno = result.error;
 
   if (str_end != nullptr)
     *str_end = const_cast<char *>(str + result.parsed_len);
diff --git a/external/llvm-project/libc/test/src/stdlib/strtoint64_test.cpp b/external/llvm-project/libc/test/src/stdlib/strtoint64_test.cpp
index b5fe69dfaa70..2c5d948f5fae 100644
--- a/external/llvm-project/libc/test/src/stdlib/strtoint64_test.cpp
+++ b/external/llvm-project/libc/test/src/stdlib/strtoint64_test.cpp
@@ -8,9 +8,9 @@
 
 #include <stdint.h>
 
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/str_to_integer.h"
-#include "src/errno/libc_errno.h"
 
 #include "StrtolTest.h"
 #include "test/UnitTest/Test.h"
@@ -21,7 +21,7 @@ int64_t strtoint64(const char *__restrict str, char **__restrict str_end,
                    int base) {
   auto result = internal::strtointeger<int64_t>(str, base);
   if (result.has_error())
-    LIBC_NAMESPACE::libc_errno = result.error;
+    libc_errno = result.error;
 
   if (str_end != nullptr)
     *str_end = const_cast<char *>(str + result.parsed_len);
@@ -33,7 +33,7 @@ uint64_t strtouint64(const char *__restrict str, char **__restrict str_end,
                      int base) {
   auto result = internal::strtointeger<uint64_t>(str, base);
   if (result.has_error())
-    LIBC_NAMESPACE::libc_errno = result.error;
+    libc_errno = result.error;
 
   if (str_end != nullptr)
     *str_end = const_cast<char *>(str + result.parsed_len);
diff --git a/external/llvm-project/libc/test/src/stdlib/strtold_test.cpp b/external/llvm-project/libc/test/src/stdlib/strtold_test.cpp
index b209c85b88e3..c2f2b9c9a11c 100644
--- a/external/llvm-project/libc/test/src/stdlib/strtold_test.cpp
+++ b/external/llvm-project/libc/test/src/stdlib/strtold_test.cpp
@@ -7,10 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/uint128.h"
-#include "src/errno/libc_errno.h"
 #include "src/stdlib/strtold.h"
 
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/Test.h"
 
 #include <stddef.h>
@@ -25,7 +26,7 @@
 #error "Unknown long double type"
 #endif
 
-class LlvmLibcStrToLDTest : public LIBC_NAMESPACE::testing::Test {
+class LlvmLibcStrToLDTest : public LIBC_NAMESPACE::testing::ErrnoCheckingTest {
 public:
 #if defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
   void run_test(const char *inputString, const ptrdiff_t expectedStrLen,
@@ -80,7 +81,6 @@ class LlvmLibcStrToLDTest : public LIBC_NAMESPACE::testing::Test {
         FPBits(static_cast<FPBits::StorageType>(expectedRawData));
     const int expected_errno = expectedErrno;
 
-    LIBC_NAMESPACE::libc_errno = 0;
     long double result = LIBC_NAMESPACE::strtold(inputString, &str_end);
 
     LIBC_NAMESPACE::fputil::FPBits<long double> actual_fp =
diff --git a/external/llvm-project/libc/test/src/string/CMakeLists.txt b/external/llvm-project/libc/test/src/string/CMakeLists.txt
index a675373938e9..ced60750a45c 100644
--- a/external/llvm-project/libc/test/src/string/CMakeLists.txt
+++ b/external/llvm-project/libc/test/src/string/CMakeLists.txt
@@ -168,6 +168,7 @@ add_libc_test(
   DEPENDS
     libc.src.string.strdup
     libc.src.errno.errno
+    libc.test.UnitTest.ErrnoCheckingTest
 )
 
 # FIXME: This is failing on the bot for some reason, disable for now.
diff --git a/external/llvm-project/libc/test/src/string/memchr_test.cpp b/external/llvm-project/libc/test/src/string/memchr_test.cpp
index 343958234edc..1455183e4646 100644
--- a/external/llvm-project/libc/test/src/string/memchr_test.cpp
+++ b/external/llvm-project/libc/test/src/string/memchr_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/string/memchr.h"
 #include "test/UnitTest/Test.h"
 #include <stddef.h>
@@ -120,3 +121,12 @@ TEST(LlvmLibcMemChrTest, SignedCharacterFound) {
   // Should find the first character 'c'.
   ASSERT_EQ(actual[0], c);
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcMemChrTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::memchr(nullptr, 1, 1); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/memcmp_test.cpp b/external/llvm-project/libc/test/src/string/memcmp_test.cpp
index 9f85a6d4f222..3dfbceda5953 100644
--- a/external/llvm-project/libc/test/src/string/memcmp_test.cpp
+++ b/external/llvm-project/libc/test/src/string/memcmp_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "memory_utils/memory_check_utils.h"
 #include "src/__support/macros/config.h"
 #include "src/string/memcmp.h"
@@ -65,4 +66,13 @@ TEST(LlvmLibcMemcmpTest, SizeSweep) {
   }
 }
 
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcMemcmpTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::memcmp(nullptr, nullptr, 1); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/test/src/string/memcpy_test.cpp b/external/llvm-project/libc/test/src/string/memcpy_test.cpp
index ce267d17591f..8c43ac8b0e60 100644
--- a/external/llvm-project/libc/test/src/string/memcpy_test.cpp
+++ b/external/llvm-project/libc/test/src/string/memcpy_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "memory_utils/memory_check_utils.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/os.h" // LIBC_TARGET_OS_IS_LINUX
@@ -72,4 +73,12 @@ TEST(LlvmLibcMemcpyTest, CheckAccess) {
 
 #endif // !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
 
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcMemcpyTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::memcpy(nullptr, nullptr, 1); },
+               WITH_SIGNAL(-1));
+}
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/test/src/string/memmove_test.cpp b/external/llvm-project/libc/test/src/string/memmove_test.cpp
index 1e225e58ca1e..0d476555333d 100644
--- a/external/llvm-project/libc/test/src/string/memmove_test.cpp
+++ b/external/llvm-project/libc/test/src/string/memmove_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/__support/macros/config.h"
 #include "src/string/memmove.h"
 
@@ -103,4 +104,13 @@ TEST(LlvmLibcMemmoveTest, SizeSweep) {
     }
 }
 
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcMemmoveTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::memmove(nullptr, nullptr, 2); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // LIBC_TARGET_OS_IS_LINUX
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/test/src/string/mempcpy_test.cpp b/external/llvm-project/libc/test/src/string/mempcpy_test.cpp
index 877ee8104880..24482a81352c 100644
--- a/external/llvm-project/libc/test/src/string/mempcpy_test.cpp
+++ b/external/llvm-project/libc/test/src/string/mempcpy_test.cpp
@@ -5,7 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-
+#include "hdr/signal_macros.h"
 #include "src/string/mempcpy.h"
 #include "test/UnitTest/Test.h"
 
@@ -26,3 +26,12 @@ TEST(LlvmLibcMempcpyTest, ZeroCount) {
   void *result = LIBC_NAMESPACE::mempcpy(dest, src, 0);
   ASSERT_EQ(static_cast<char *>(result), dest + 0);
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcMempcpyTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::mempcpy(nullptr, nullptr, 1); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/memrchr_test.cpp b/external/llvm-project/libc/test/src/string/memrchr_test.cpp
index 421cb9b98516..c73a47917dd5 100644
--- a/external/llvm-project/libc/test/src/string/memrchr_test.cpp
+++ b/external/llvm-project/libc/test/src/string/memrchr_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/string/memrchr.h"
 #include "test/UnitTest/Test.h"
 #include <stddef.h>
@@ -112,3 +113,12 @@ TEST(LlvmLibcMemRChrTest, ZeroLengthShouldReturnNullptr) {
   // This will iterate over exactly zero characters, so should return nullptr.
   ASSERT_STREQ(call_memrchr(src, 'd', 0), nullptr);
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcMemRChrTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::memrchr(nullptr, 'd', 1); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/memset_test.cpp b/external/llvm-project/libc/test/src/string/memset_test.cpp
index 46d6ce747877..9562d2d2e37a 100644
--- a/external/llvm-project/libc/test/src/string/memset_test.cpp
+++ b/external/llvm-project/libc/test/src/string/memset_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "memory_utils/memory_check_utils.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/os.h" // LIBC_TARGET_OS_IS_LINUX
@@ -59,4 +60,13 @@ TEST(LlvmLibcMemsetTest, CheckAccess) {
 
 #endif // !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
 
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcMemsetTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::memset(nullptr, 0, 1); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/external/llvm-project/libc/test/src/string/stpncpy_test.cpp b/external/llvm-project/libc/test/src/string/stpncpy_test.cpp
index 247fa92a0c7b..f5c61e277f66 100644
--- a/external/llvm-project/libc/test/src/string/stpncpy_test.cpp
+++ b/external/llvm-project/libc/test/src/string/stpncpy_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/__support/CPP/span.h"
 #include "src/string/stpncpy.h"
 #include "test/UnitTest/Test.h"
@@ -71,3 +72,12 @@ TEST_F(LlvmLibcStpncpyTest, CopyTwoWithNull) {
   const char expected[] = {'x', '\0'};
   check_stpncpy(dst, src, 2, expected, 1);
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST_F(LlvmLibcStpncpyTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::stpncpy(nullptr, nullptr, 1); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/strcat_test.cpp b/external/llvm-project/libc/test/src/string/strcat_test.cpp
index e4f6c1ee7599..20f8d11d239b 100644
--- a/external/llvm-project/libc/test/src/string/strcat_test.cpp
+++ b/external/llvm-project/libc/test/src/string/strcat_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/string/strcat.h"
 #include "test/UnitTest/Test.h"
 
@@ -35,3 +36,12 @@ TEST(LlvmLibcStrCatTest, NonEmptyDest) {
   ASSERT_STREQ(dest, result);
   ASSERT_STREQ(dest, "xyzabc");
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcStrCatTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::strcat(nullptr, nullptr); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/strcoll_test.cpp b/external/llvm-project/libc/test/src/string/strcoll_test.cpp
index a10f98f1ca4d..268e232a9a98 100644
--- a/external/llvm-project/libc/test/src/string/strcoll_test.cpp
+++ b/external/llvm-project/libc/test/src/string/strcoll_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/string/strcoll.h"
 #include "test/UnitTest/Test.h"
 
@@ -28,3 +29,12 @@ TEST(LlvmLibcStrcollTest, SimpleTest) {
   result = LIBC_NAMESPACE::strcoll(s3, s1);
   ASSERT_GT(result, 0);
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcStrcollTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::strcoll(nullptr, nullptr); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/strcpy_test.cpp b/external/llvm-project/libc/test/src/string/strcpy_test.cpp
index 1a1227aac5d2..ead60be1267d 100644
--- a/external/llvm-project/libc/test/src/string/strcpy_test.cpp
+++ b/external/llvm-project/libc/test/src/string/strcpy_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/string/strcpy.h"
 #include "test/UnitTest/Test.h"
 
@@ -42,3 +43,12 @@ TEST(LlvmLibcStrCpyTest, OffsetDest) {
   ASSERT_STREQ(dest + 3, result);
   ASSERT_STREQ(dest, "xyzabc");
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcStrCpyTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::strcpy(nullptr, nullptr); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/strdup_test.cpp b/external/llvm-project/libc/test/src/string/strdup_test.cpp
index 20b85c37637d..4b18fc7f1bde 100644
--- a/external/llvm-project/libc/test/src/string/strdup_test.cpp
+++ b/external/llvm-project/libc/test/src/string/strdup_test.cpp
@@ -6,14 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
 #include "src/string/strdup.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/Test.h"
 
-TEST(LlvmLibcStrDupTest, EmptyString) {
+using LlvmLibcStrDupTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+
+TEST_F(LlvmLibcStrDupTest, EmptyString) {
   const char *empty = "";
 
-  LIBC_NAMESPACE::libc_errno = 0;
   char *result = LIBC_NAMESPACE::strdup(empty);
   ASSERT_ERRNO_SUCCESS();
 
@@ -23,10 +24,9 @@ TEST(LlvmLibcStrDupTest, EmptyString) {
   ::free(result);
 }
 
-TEST(LlvmLibcStrDupTest, AnyString) {
+TEST_F(LlvmLibcStrDupTest, AnyString) {
   const char *abc = "abc";
 
-  LIBC_NAMESPACE::libc_errno = 0;
   char *result = LIBC_NAMESPACE::strdup(abc);
   ASSERT_ERRNO_SUCCESS();
 
@@ -36,8 +36,7 @@ TEST(LlvmLibcStrDupTest, AnyString) {
   ::free(result);
 }
 
-TEST(LlvmLibcStrDupTest, NullPtr) {
-  LIBC_NAMESPACE::libc_errno = 0;
+TEST_F(LlvmLibcStrDupTest, NullPtr) {
   char *result = LIBC_NAMESPACE::strdup(nullptr);
   ASSERT_ERRNO_SUCCESS();
 
diff --git a/external/llvm-project/libc/test/src/string/strlcpy_test.cpp b/external/llvm-project/libc/test/src/string/strlcpy_test.cpp
index 0914257ecc1f..b42954ca6137 100644
--- a/external/llvm-project/libc/test/src/string/strlcpy_test.cpp
+++ b/external/llvm-project/libc/test/src/string/strlcpy_test.cpp
@@ -14,8 +14,8 @@ TEST(LlvmLibcStrlcpyTest, TooBig) {
   char buf[2];
   EXPECT_EQ(LIBC_NAMESPACE::strlcpy(buf, str, 2), size_t(3));
   EXPECT_STREQ(buf, "a");
-
-  EXPECT_EQ(LIBC_NAMESPACE::strlcpy(nullptr, str, 0), size_t(3));
+  char dst[] = "";
+  EXPECT_EQ(LIBC_NAMESPACE::strlcpy(dst, str, 0), size_t(3));
 }
 
 TEST(LlvmLibcStrlcpyTest, Smaller) {
diff --git a/external/llvm-project/libc/test/src/string/strsep_test.cpp b/external/llvm-project/libc/test/src/string/strsep_test.cpp
index 0daa29f6a7ad..6f02ce36edf2 100644
--- a/external/llvm-project/libc/test/src/string/strsep_test.cpp
+++ b/external/llvm-project/libc/test/src/string/strsep_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/string/strsep.h"
 #include "test/UnitTest/Test.h"
 
@@ -51,3 +52,12 @@ TEST(LlvmLibcStrsepTest, DelimitersShouldNotBeIncludedInToken) {
     ASSERT_STREQ(LIBC_NAMESPACE::strsep(&string, "_:"), expected[i]);
   }
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcStrsepTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::strsep(nullptr, nullptr); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/string/strspn_test.cpp b/external/llvm-project/libc/test/src/string/strspn_test.cpp
index cdd12af07ee7..adf9a4564a1f 100644
--- a/external/llvm-project/libc/test/src/string/strspn_test.cpp
+++ b/external/llvm-project/libc/test/src/string/strspn_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/signal_macros.h"
 #include "src/string/strspn.h"
 
 #include "test/UnitTest/Test.h"
@@ -83,3 +84,12 @@ TEST(LlvmLibcStrSpnTest, DuplicatedCharactersToBeSearchedForShouldStillMatch) {
   EXPECT_EQ(LIBC_NAMESPACE::strspn("aaa", "aa"), size_t{3});
   EXPECT_EQ(LIBC_NAMESPACE::strspn("aaaa", "aa"), size_t{4});
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+
+TEST(LlvmLibcStrSpnTest, CrashOnNullPtr) {
+  ASSERT_DEATH([]() { LIBC_NAMESPACE::strspn(nullptr, nullptr); },
+               WITH_SIGNAL(-1));
+}
+
+#endif // defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
diff --git a/external/llvm-project/libc/test/src/sys/CMakeLists.txt b/external/llvm-project/libc/test/src/sys/CMakeLists.txt
index 224cc7905ad3..13bf91eef04b 100644
--- a/external/llvm-project/libc/test/src/sys/CMakeLists.txt
+++ b/external/llvm-project/libc/test/src/sys/CMakeLists.txt
@@ -13,3 +13,4 @@ add_subdirectory(auxv)
 add_subdirectory(epoll)
 add_subdirectory(uio)
 add_subdirectory(time)
+add_subdirectory(ioctl)
diff --git a/external/llvm-project/libc/test/src/sys/ioctl/CMakeLists.txt b/external/llvm-project/libc/test/src/sys/ioctl/CMakeLists.txt
new file mode 100644
index 000000000000..b4bbe81c92ff
--- /dev/null
+++ b/external/llvm-project/libc/test/src/sys/ioctl/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/external/llvm-project/libc/test/src/sys/ioctl/linux/CMakeLists.txt b/external/llvm-project/libc/test/src/sys/ioctl/linux/CMakeLists.txt
new file mode 100644
index 000000000000..2df67e9d9cbd
--- /dev/null
+++ b/external/llvm-project/libc/test/src/sys/ioctl/linux/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_custom_target(libc_sys_ioctl_unittests)
+
+add_libc_unittest(
+  ioctl_test
+  SUITE
+    libc_sys_ioctl_unittests
+  SRCS
+    ioctl_test.cpp
+  DEPENDS
+    libc.hdr.sys_ioctl_macros
+    libc.src.sys.ioctl.ioctl
+    libc.src.errno.errno
+    libc.src.fcntl.open
+    libc.src.unistd.close
+    libc.src.unistd.read
+    libc.src.unistd.write
+)
+
diff --git a/external/llvm-project/libc/test/src/sys/ioctl/linux/ioctl_test.cpp b/external/llvm-project/libc/test/src/sys/ioctl/linux/ioctl_test.cpp
new file mode 100644
index 000000000000..b76dc14824c9
--- /dev/null
+++ b/external/llvm-project/libc/test/src/sys/ioctl/linux/ioctl_test.cpp
@@ -0,0 +1,75 @@
+//===-- Unittests for ioctl -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/libc_errno.h"
+#include "src/fcntl/open.h"
+#include "src/sys/ioctl/ioctl.h"
+#include "src/unistd/close.h"
+#include "src/unistd/read.h"
+#include "src/unistd/write.h"
+
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+#include "hdr/sys_stat_macros.h"
+
+#include "hdr/sys_ioctl_macros.h"
+
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
+
+TEST(LlvmLibcSysIoctlTest, InvalidCommandAndFIONREAD) {
+  LIBC_NAMESPACE::libc_errno = 0;
+
+  // Setup the test file
+  constexpr const char *TEST_FILE_NAME = "ioctl.test";
+  constexpr const char TEST_MSG[] = "ioctl test";
+  constexpr int TEST_MSG_SIZE = sizeof(TEST_MSG) - 1;
+  auto TEST_FILE = libc_make_test_file_path(TEST_FILE_NAME);
+  int new_test_file_fd = LIBC_NAMESPACE::open(
+      TEST_FILE, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+  ASSERT_THAT(
+      (int)LIBC_NAMESPACE::write(new_test_file_fd, TEST_MSG, TEST_MSG_SIZE),
+      Succeeds(TEST_MSG_SIZE));
+  ASSERT_ERRNO_SUCCESS();
+  ASSERT_THAT(LIBC_NAMESPACE::close(new_test_file_fd), Succeeds(0));
+  ASSERT_ERRNO_SUCCESS();
+
+  // Reopen the file for testing
+  int fd = LIBC_NAMESPACE::open(TEST_FILE, O_RDONLY);
+  ASSERT_ERRNO_SUCCESS();
+  ASSERT_GT(fd, 0);
+
+  // FIONREAD reports the number of available bytes to read for the passed fd
+  // This will report the full size of the file, as we haven't read anything yet
+  int n = -1;
+  int ret = LIBC_NAMESPACE::ioctl(fd, FIONREAD, &n);
+  ASSERT_ERRNO_SUCCESS();
+  ASSERT_GT(ret, -1);
+  ASSERT_EQ(n, TEST_MSG_SIZE);
+
+  // But if we read some bytes...
+  constexpr int READ_COUNT = 5;
+  char read_buffer[READ_COUNT];
+  ASSERT_THAT((int)LIBC_NAMESPACE::read(fd, read_buffer, READ_COUNT),
+              Succeeds(READ_COUNT));
+
+  // ... n should have decreased by the number of bytes we've read
+  int n_after_reading = -1;
+  ret = LIBC_NAMESPACE::ioctl(fd, FIONREAD, &n_after_reading);
+  ASSERT_ERRNO_SUCCESS();
+  ASSERT_GT(ret, -1);
+  ASSERT_EQ(n - READ_COUNT, n_after_reading);
+
+  // 0xDEADBEEF is just a random nonexistent command;
+  // calling this should always fail with ENOTTY
+  ret = LIBC_NAMESPACE::ioctl(fd, 0xDEADBEEF, NULL);
+  ASSERT_ERRNO_EQ(ENOTTY);
+  ASSERT_EQ(ret, -1);
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0));
+}
diff --git a/external/llvm-project/libc/test/src/sys/mman/linux/mlock_test.cpp b/external/llvm-project/libc/test/src/sys/mman/linux/mlock_test.cpp
index 88abacad554e..6b81411ca604 100644
--- a/external/llvm-project/libc/test/src/sys/mman/linux/mlock_test.cpp
+++ b/external/llvm-project/libc/test/src/sys/mman/linux/mlock_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/sys/mman/madvise.h"
 #include "src/sys/mman/mincore.h"
 #include "src/sys/mman/mlock.h"
@@ -149,9 +149,8 @@ TEST_F(LlvmLibcMlockTest, MLockAll) {
         Succeeds());
     auto retval = LIBC_NAMESPACE::mlockall(MCL_CURRENT);
     if (retval == -1) {
-      EXPECT_TRUE(LIBC_NAMESPACE::libc_errno == ENOMEM ||
-                  LIBC_NAMESPACE::libc_errno == EPERM);
-      LIBC_NAMESPACE::libc_errno = 0;
+      EXPECT_TRUE(libc_errno == ENOMEM || libc_errno == EPERM);
+      libc_errno = 0;
       return;
     }
     unsigned char vec;
@@ -163,9 +162,8 @@ TEST_F(LlvmLibcMlockTest, MLockAll) {
   {
     auto retval = LIBC_NAMESPACE::mlockall(MCL_FUTURE);
     if (retval == -1) {
-      EXPECT_TRUE(LIBC_NAMESPACE::libc_errno == ENOMEM ||
-                  LIBC_NAMESPACE::libc_errno == EPERM);
-      LIBC_NAMESPACE::libc_errno = 0;
+      EXPECT_TRUE(libc_errno == ENOMEM || libc_errno == EPERM);
+      libc_errno = 0;
       return;
     }
     PageHolder holder;
@@ -180,9 +178,8 @@ TEST_F(LlvmLibcMlockTest, MLockAll) {
   {
     auto retval = LIBC_NAMESPACE::mlockall(MCL_FUTURE | MCL_ONFAULT);
     if (retval == -1) {
-      EXPECT_TRUE(LIBC_NAMESPACE::libc_errno == ENOMEM ||
-                  LIBC_NAMESPACE::libc_errno == EPERM);
-      LIBC_NAMESPACE::libc_errno = 0;
+      EXPECT_TRUE(libc_errno == ENOMEM || libc_errno == EPERM);
+      libc_errno = 0;
       return;
     }
     PageHolder holder;
diff --git a/external/llvm-project/libc/test/src/sys/prctl/linux/CMakeLists.txt b/external/llvm-project/libc/test/src/sys/prctl/linux/CMakeLists.txt
index b06e1c808700..d02900e1857a 100644
--- a/external/llvm-project/libc/test/src/sys/prctl/linux/CMakeLists.txt
+++ b/external/llvm-project/libc/test/src/sys/prctl/linux/CMakeLists.txt
@@ -1,5 +1,10 @@
 add_custom_target(libc_sys_prctl_unittests)
 
+# Temporarily disable this test while setting up arm and riscv buildbots
+# using qemu, since PR_GET_THP_DISABLE is not supported on qemu.
+if (NOT (LIBC_TARGET_ARCHITECTURE_IS_ARM OR
+	 LIBC_TARGET_ARCHITECTURE_IS_RISCV32 OR
+	 LIBC_TARGET_ARCHITECTURE_IS_RISCV64))
 add_libc_unittest(
   prctl_test
   SUITE
@@ -13,3 +18,4 @@ add_libc_unittest(
     libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.ErrnoSetterMatcher
 )
+endif()
diff --git a/external/llvm-project/libc/test/src/sys/prctl/linux/prctl_test.cpp b/external/llvm-project/libc/test/src/sys/prctl/linux/prctl_test.cpp
index 374c905e0ef8..76b829c82d1b 100644
--- a/external/llvm-project/libc/test/src/sys/prctl/linux/prctl_test.cpp
+++ b/external/llvm-project/libc/test/src/sys/prctl/linux/prctl_test.cpp
@@ -34,6 +34,7 @@ TEST_F(LlvmLibcSysPrctlTest, GetSetName) {
 TEST_F(LlvmLibcSysPrctlTest, GetTHPDisable) {
   // Manually check errno since the return value logic here is not
   // covered in ErrnoSetterMatcher.
+  // Note that PR_GET_THP_DISABLE is not supported by QEMU.
   int ret = LIBC_NAMESPACE::prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0);
   ASSERT_ERRNO_SUCCESS();
   // PR_GET_THP_DISABLE return (as the function result) the current
diff --git a/external/llvm-project/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp b/external/llvm-project/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
index 455a82678e18..ba0ee4f09109 100644
--- a/external/llvm-project/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
+++ b/external/llvm-project/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/fcntl/open.h"
 #include "src/sys/stat/mkdirat.h"
 #include "src/sys/statvfs/fstatvfs.h"
@@ -41,7 +41,7 @@ TEST_F(LlvmLibcSysFStatvfsTest, FStatvfsInvalidPath) {
 
   // Always delete the folder so that we start in a consistent state.
   LIBC_NAMESPACE::rmdir(TEST_DIR);
-  LIBC_NAMESPACE::libc_errno = 0; // Reset errno
+  libc_errno = 0; // Reset errno
 
   ASSERT_THAT(LIBC_NAMESPACE::mkdirat(AT_FDCWD, TEST_DIR, S_IRWXU),
               Succeeds(0));
diff --git a/external/llvm-project/libc/test/src/sys/statvfs/linux/statvfs_test.cpp b/external/llvm-project/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
index f356bb3d277b..327dec07a1b7 100644
--- a/external/llvm-project/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
+++ b/external/llvm-project/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/fcntl_macros.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/stat/mkdirat.h"
 #include "src/sys/statvfs/statvfs.h"
 #include "src/unistd/rmdir.h"
@@ -37,7 +37,7 @@ TEST_F(LlvmLibcSysStatvfsTest, StatvfsInvalidPath) {
 
   // Always delete the folder so that we start in a consistent state.
   LIBC_NAMESPACE::rmdir(TEST_DIR);
-  LIBC_NAMESPACE::libc_errno = 0; // Reset errno
+  libc_errno = 0; // Reset errno
 
   ASSERT_THAT(LIBC_NAMESPACE::mkdirat(AT_FDCWD, TEST_DIR, S_IRWXU),
               Succeeds(0));
diff --git a/external/llvm-project/libc/test/src/sys/time/setitimer_test.cpp b/external/llvm-project/libc/test/src/sys/time/setitimer_test.cpp
index 16d33fdf1e4f..115f9e662ed4 100644
--- a/external/llvm-project/libc/test/src/sys/time/setitimer_test.cpp
+++ b/external/llvm-project/libc/test/src/sys/time/setitimer_test.cpp
@@ -24,7 +24,7 @@ static bool timer_fired(false);
 extern "C" void handle_sigalrm(int) { timer_fired = true; }
 
 TEST_F(LlvmLibcSysTimeSetitimerTest, SmokeTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   struct sigaction sa;
   sa.sa_handler = handle_sigalrm;
   LIBC_NAMESPACE::sigemptyset(&sa.sa_mask);
diff --git a/external/llvm-project/libc/test/src/termios/termios_test.cpp b/external/llvm-project/libc/test/src/termios/termios_test.cpp
index f8fc09a8bbf0..5ec169a886b1 100644
--- a/external/llvm-project/libc/test/src/termios/termios_test.cpp
+++ b/external/llvm-project/libc/test/src/termios/termios_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/fcntl/open.h"
 #include "src/termios/cfgetispeed.h"
 #include "src/termios/cfgetospeed.h"
@@ -30,21 +30,21 @@ using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
 TEST(LlvmLibcTermiosTest, SpeedSmokeTest) {
   struct termios t;
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(LIBC_NAMESPACE::cfsetispeed(&t, B50), Succeeds(0));
   ASSERT_EQ(LIBC_NAMESPACE::cfgetispeed(&t), speed_t(B50));
   ASSERT_THAT(LIBC_NAMESPACE::cfsetospeed(&t, B75), Succeeds(0));
   ASSERT_EQ(LIBC_NAMESPACE::cfgetospeed(&t), speed_t(B75));
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(LIBC_NAMESPACE::cfsetispeed(&t, ~CBAUD), Fails(EINVAL));
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   ASSERT_THAT(LIBC_NAMESPACE::cfsetospeed(&t, ~CBAUD), Fails(EINVAL));
 }
 
 TEST(LlvmLibcTermiosTest, GetAttrSmokeTest) {
   struct termios t;
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   int fd = LIBC_NAMESPACE::open("/dev/tty", O_RDONLY);
   if (fd < 0)
     return; // When /dev/tty is not available, no point continuing.
@@ -54,7 +54,7 @@ TEST(LlvmLibcTermiosTest, GetAttrSmokeTest) {
 }
 
 TEST(LlvmLibcTermiosTest, TcGetSidSmokeTest) {
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   int fd = LIBC_NAMESPACE::open("/dev/tty", O_RDONLY);
   if (fd < 0)
     return; // When /dev/tty is not available, no point continuing.
diff --git a/external/llvm-project/libc/test/src/time/asctime_r_test.cpp b/external/llvm-project/libc/test/src/time/asctime_r_test.cpp
index b595cfe02486..d840248b7df4 100644
--- a/external/llvm-project/libc/test/src/time/asctime_r_test.cpp
+++ b/external/llvm-project/libc/test/src/time/asctime_r_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/time/asctime_r.h"
 #include "src/time/time_constants.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/time/asctime_test.cpp b/external/llvm-project/libc/test/src/time/asctime_test.cpp
index 169a7463a303..cad25fffc65a 100644
--- a/external/llvm-project/libc/test/src/time/asctime_test.cpp
+++ b/external/llvm-project/libc/test/src/time/asctime_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/time/asctime.h"
 #include "test/UnitTest/Test.h"
 #include "test/src/time/TmHelper.h"
diff --git a/external/llvm-project/libc/test/src/time/ctime_r_test.cpp b/external/llvm-project/libc/test/src/time/ctime_r_test.cpp
index 27011b7e0fbd..fe43877aa499 100644
--- a/external/llvm-project/libc/test/src/time/ctime_r_test.cpp
+++ b/external/llvm-project/libc/test/src/time/ctime_r_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/time/ctime_r.h"
 #include "src/time/time_constants.h"
 #include "test/UnitTest/Test.h"
diff --git a/external/llvm-project/libc/test/src/time/ctime_test.cpp b/external/llvm-project/libc/test/src/time/ctime_test.cpp
index 6f1168f0b668..5ff69f6619b4 100644
--- a/external/llvm-project/libc/test/src/time/ctime_test.cpp
+++ b/external/llvm-project/libc/test/src/time/ctime_test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/time/ctime.h"
 #include "test/UnitTest/Test.h"
 #include "test/src/time/TmHelper.h"
diff --git a/external/llvm-project/libc/test/src/time/gmtime_test.cpp b/external/llvm-project/libc/test/src/time/gmtime_test.cpp
index 6af5a18d3699..41236665d2ea 100644
--- a/external/llvm-project/libc/test/src/time/gmtime_test.cpp
+++ b/external/llvm-project/libc/test/src/time/gmtime_test.cpp
@@ -8,7 +8,7 @@
 
 #include "hdr/types/struct_tm.h"
 #include "src/__support/CPP/limits.h" // INT_MAX, INT_MIN
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/time/gmtime.h"
 #include "src/time/time_constants.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
@@ -30,7 +30,7 @@ TEST(LlvmLibcGmTime, OutOfRange) {
   EXPECT_TRUE(tm_data == nullptr);
   ASSERT_ERRNO_EQ(EOVERFLOW);
 
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
   seconds =
       INT_MIN *
           static_cast<int64_t>(
diff --git a/external/llvm-project/libc/test/src/time/nanosleep_test.cpp b/external/llvm-project/libc/test/src/time/nanosleep_test.cpp
index d4f98e29bd98..e0200ff3aaa2 100644
--- a/external/llvm-project/libc/test/src/time/nanosleep_test.cpp
+++ b/external/llvm-project/libc/test/src/time/nanosleep_test.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "hdr/types/struct_timespec.h"
-#include "src/errno/libc_errno.h"
+#include "src/__support/libc_errno.h"
 #include "src/time/nanosleep.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -17,7 +17,7 @@ namespace cpp = LIBC_NAMESPACE::cpp;
 TEST(LlvmLibcNanosleep, SmokeTest) {
   // TODO: When we have the code to read clocks, test that time has passed.
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
-  LIBC_NAMESPACE::libc_errno = 0;
+  libc_errno = 0;
 
   struct timespec tim = {1, 500};
   struct timespec tim2 = {0, 0};
diff --git a/external/llvm-project/libc/test/src/wchar/CMakeLists.txt b/external/llvm-project/libc/test/src/wchar/CMakeLists.txt
index 9bc230e0bddf..4990b6953348 100644
--- a/external/llvm-project/libc/test/src/wchar/CMakeLists.txt
+++ b/external/llvm-project/libc/test/src/wchar/CMakeLists.txt
@@ -55,6 +55,26 @@ add_libc_test(
     libc.src.wchar.wcschr
 )
 
+add_libc_test(
+  wcsncmp_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcsncmp_test.cpp
+  DEPENDS
+    libc.src.wchar.wcsncmp
+)
+
+add_libc_test(
+  wcscmp_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcscmp_test.cpp
+  DEPENDS
+    libc.src.wchar.wcscmp
+)
+
 add_libc_test(
   wcspbrk_test
   SUITE
@@ -65,6 +85,16 @@ add_libc_test(
     libc.src.wchar.wcspbrk
 )
 
+add_libc_test(
+  wcsrchr_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcsrchr_test.cpp
+  DEPENDS
+    libc.src.wchar.wcsrchr
+)
+
 add_libc_test(
   wcsspn_test
   SUITE
@@ -75,6 +105,16 @@ add_libc_test(
     libc.src.wchar.wcsspn
 )
 
+add_libc_test(
+  wmemchr_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wmemchr_test.cpp
+  DEPENDS
+    libc.src.wchar.wmemchr
+)
+
 add_libc_test(
   wmemcmp_test
   SUITE
@@ -85,6 +125,16 @@ add_libc_test(
     libc.src.wchar.wmemcmp
 )
 
+add_libc_test(
+  wmempcpy_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wmempcpy_test.cpp
+  DEPENDS
+    libc.src.wchar.wmempcpy
+)
+
 add_libc_test(
   wmemcpy_test
   SUITE
@@ -94,3 +144,63 @@ add_libc_test(
   DEPENDS
     libc.src.wchar.wmemcpy
 )
+
+add_libc_test(
+  wmemmove_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wmemmove_test.cpp
+  DEPENDS
+    libc.src.wchar.wmemmove
+)
+
+add_libc_test(
+  wcsncpy_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcsncpy_test.cpp
+  DEPENDS
+    libc.src.wchar.wcsncpy
+)
+
+add_libc_test(
+  wcscat_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcscat_test.cpp
+  DEPENDS
+    libc.src.wchar.wcscat
+)
+  
+add_libc_test(
+  wcsstr_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcsstr_test.cpp
+  DEPENDS
+    libc.src.wchar.wcsstr
+)
+
+add_libc_test(
+  wcsncat_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcsncat_test.cpp
+  DEPENDS
+    libc.src.wchar.wcsncat
+)
+
+add_libc_test(
+  wcscpy_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcscpy_test.cpp
+  DEPENDS
+    libc.src.wchar.wcscpy
+)
diff --git a/external/llvm-project/libc/test/src/wchar/wcscat_test.cpp b/external/llvm-project/libc/test/src/wchar/wcscat_test.cpp
new file mode 100644
index 000000000000..e91f79613ca9
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcscat_test.cpp
@@ -0,0 +1,47 @@
+//===-- Unittests for wcscat ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcscat.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSCatTest, EmptyDest) {
+  // Dest should be fully replaced with src.
+  wchar_t dest[4] = {L'\0'};
+  const wchar_t *src = L"abc";
+  LIBC_NAMESPACE::wcscat(dest, src);
+  ASSERT_TRUE(dest[0] == L'a');
+  ASSERT_TRUE(dest[1] == L'b');
+  ASSERT_TRUE(dest[2] == L'c');
+  ASSERT_TRUE(dest[3] == L'\0');
+}
+
+TEST(LlvmLibcWCSCatTest, NonEmptyDest) {
+  // Src should be appended on to dest.
+  wchar_t dest[7] = {L'x', L'y', L'z', L'\0'};
+  const wchar_t *src = L"abc";
+  LIBC_NAMESPACE::wcscat(dest, src);
+  ASSERT_TRUE(dest[0] == L'x');
+  ASSERT_TRUE(dest[1] == L'y');
+  ASSERT_TRUE(dest[2] == L'z');
+  ASSERT_TRUE(dest[3] == L'a');
+  ASSERT_TRUE(dest[4] == L'b');
+  ASSERT_TRUE(dest[5] == L'c');
+  ASSERT_TRUE(dest[6] == L'\0');
+}
+
+TEST(LlvmLibcWCSCatTest, EmptySrc) {
+  // Dest should remain intact.
+  wchar_t dest[4] = {L'x', L'y', L'z', L'\0'};
+  const wchar_t *src = L"";
+  LIBC_NAMESPACE::wcscat(dest, src);
+  ASSERT_TRUE(dest[0] == L'x');
+  ASSERT_TRUE(dest[1] == L'y');
+  ASSERT_TRUE(dest[2] == L'z');
+  ASSERT_TRUE(dest[3] == L'\0');
+}
diff --git a/external/llvm-project/libc/test/src/wchar/wcscmp_test.cpp b/external/llvm-project/libc/test/src/wchar/wcscmp_test.cpp
new file mode 100644
index 000000000000..6572aadb066a
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcscmp_test.cpp
@@ -0,0 +1,97 @@
+//===-- Unittests for wcscmp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcscmp.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWcscmpTest, EmptyStringsShouldReturnZero) {
+  const wchar_t *s1 = L"";
+  const wchar_t *s2 = L"";
+  int result = LIBC_NAMESPACE::wcscmp(s1, s2);
+  ASSERT_EQ(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcscmp(s2, s1);
+  ASSERT_EQ(result, 0);
+}
+
+TEST(LlvmLibcWcscmpTest, EmptyStringShouldNotEqualNonEmptyString) {
+  const wchar_t *empty = L"";
+  const wchar_t *s2 = L"abc";
+  int result = LIBC_NAMESPACE::wcscmp(empty, s2);
+  ASSERT_LT(result, 0);
+
+  // Similar case if empty string is second argument.
+  const wchar_t *s3 = L"123";
+  result = LIBC_NAMESPACE::wcscmp(s3, empty);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcscmpTest, EqualStringsShouldReturnZero) {
+  const wchar_t *s1 = L"abc";
+  const wchar_t *s2 = L"abc";
+  int result = LIBC_NAMESPACE::wcscmp(s1, s2);
+  ASSERT_EQ(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcscmp(s2, s1);
+  ASSERT_EQ(result, 0);
+}
+
+TEST(LlvmLibcWcscmpTest, ShouldReturnResultOfFirstDifference) {
+  const wchar_t *s1 = L"___B42__";
+  const wchar_t *s2 = L"___C55__";
+  int result = LIBC_NAMESPACE::wcscmp(s1, s2);
+  ASSERT_LT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcscmp(s2, s1);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcscmpTest, CapitalizedLetterShouldNotBeEqual) {
+  const wchar_t *s1 = L"abcd";
+  const wchar_t *s2 = L"abCd";
+  int result = LIBC_NAMESPACE::wcscmp(s1, s2);
+  ASSERT_GT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcscmp(s2, s1);
+  ASSERT_LT(result, 0);
+}
+
+TEST(LlvmLibcWcscmpTest, UnequalLengthStringsShouldNotReturnZero) {
+  const wchar_t *s1 = L"abc";
+  const wchar_t *s2 = L"abcd";
+  int result = LIBC_NAMESPACE::wcscmp(s1, s2);
+  ASSERT_LT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcscmp(s2, s1);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcscmpTest, StringArgumentSwapChangesSign) {
+  const wchar_t *a = L"a";
+  const wchar_t *b = L"b";
+  int result = LIBC_NAMESPACE::wcscmp(b, a);
+  ASSERT_GT(result, 0);
+
+  result = LIBC_NAMESPACE::wcscmp(a, b);
+  ASSERT_LT(result, 0);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+TEST(LlvmLibcWcscmpTest, NullptrCrash) {
+  // Passing in a nullptr should crash the program.
+  EXPECT_DEATH([] { LIBC_NAMESPACE::wcscmp(L"aaaaaaaaaaaaaa", nullptr); },
+               WITH_SIGNAL(-1));
+  EXPECT_DEATH([] { LIBC_NAMESPACE::wcscmp(nullptr, L"aaaaaaaaaaaaaa"); },
+               WITH_SIGNAL(-1));
+}
+#endif // LIBC_HAS_ADDRESS_SANITIZER
diff --git a/external/llvm-project/libc/test/src/wchar/wcscpy_test.cpp b/external/llvm-project/libc/test/src/wchar/wcscpy_test.cpp
new file mode 100644
index 000000000000..7b71b2b30b6a
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcscpy_test.cpp
@@ -0,0 +1,48 @@
+//===-- Unittests for wcscpy ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcscpy.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSCpyTest, EmptySrc) {
+  // Empty src should lead to empty destination.
+  wchar_t dest[4] = {L'a', L'b', L'c', L'\0'};
+  const wchar_t *src = L"";
+  LIBC_NAMESPACE::wcscpy(dest, src);
+  ASSERT_TRUE(dest[0] == src[0]);
+  ASSERT_TRUE(dest[0] == L'\0');
+}
+
+TEST(LlvmLibcWCSCpyTest, EmptyDest) {
+  // Empty dest should result in src
+  const wchar_t *src = L"abc";
+  wchar_t dest[4];
+  LIBC_NAMESPACE::wcscpy(dest, src);
+  ASSERT_TRUE(dest[0] == L'a');
+  ASSERT_TRUE(dest[1] == L'b');
+  ASSERT_TRUE(dest[2] == L'c');
+  ASSERT_TRUE(dest[3] == L'\0');
+}
+
+TEST(LlvmLibcWCSCpyTest, OffsetDest) {
+  // Offsetting should result in a concatenation.
+  const wchar_t *src = L"abc";
+  wchar_t dest[7];
+  dest[0] = L'x';
+  dest[1] = L'y';
+  dest[2] = L'z';
+  LIBC_NAMESPACE::wcscpy(dest + 3, src);
+  ASSERT_TRUE(dest[0] == L'x');
+  ASSERT_TRUE(dest[1] == L'y');
+  ASSERT_TRUE(dest[2] == L'z');
+  ASSERT_TRUE(dest[3] == src[0]);
+  ASSERT_TRUE(dest[4] == src[1]);
+  ASSERT_TRUE(dest[5] == src[2]);
+  ASSERT_TRUE(dest[6] == src[3]);
+}
diff --git a/external/llvm-project/libc/test/src/wchar/wcsncat_test.cpp b/external/llvm-project/libc/test/src/wchar/wcsncat_test.cpp
new file mode 100644
index 000000000000..47359f88cec9
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcsncat_test.cpp
@@ -0,0 +1,82 @@
+//===-- Unittests for wcscat ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcsncat.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSNCatTest, EmptyDest) {
+  wchar_t dest[4] = {L'\0'};
+  const wchar_t *src = L"abc";
+
+  // Start by copying nothing
+  LIBC_NAMESPACE::wcsncat(dest, src, 0);
+  ASSERT_TRUE(dest[0] == L'\0');
+
+  // Copying part of it.
+  LIBC_NAMESPACE::wcsncat(dest, src, 1);
+  ASSERT_TRUE(dest[0] == L'a');
+  ASSERT_TRUE(dest[1] == L'\0');
+
+  // Resetting for the last test.
+  dest[0] = '\0';
+
+  // Copying all of it.
+  LIBC_NAMESPACE::wcsncat(dest, src, 3);
+  ASSERT_TRUE(dest[0] == L'a');
+  ASSERT_TRUE(dest[1] == L'b');
+  ASSERT_TRUE(dest[2] == L'c');
+  ASSERT_TRUE(dest[3] == L'\0');
+}
+
+TEST(LlvmLibcWCSNCatTest, NonEmptyDest) {
+  wchar_t dest[7] = {L'x', L'y', L'z', L'\0'};
+  const wchar_t *src = L"abc";
+
+  // Adding on only part of the string
+  LIBC_NAMESPACE::wcsncat(dest, src, 1);
+  ASSERT_TRUE(dest[0] == L'x');
+  ASSERT_TRUE(dest[1] == L'y');
+  ASSERT_TRUE(dest[2] == L'z');
+  ASSERT_TRUE(dest[3] == L'a');
+  ASSERT_TRUE(dest[4] == L'\0');
+
+  // Copying more without resetting
+  LIBC_NAMESPACE::wcsncat(dest, src, 2);
+  ASSERT_TRUE(dest[0] == L'x');
+  ASSERT_TRUE(dest[1] == L'y');
+  ASSERT_TRUE(dest[2] == L'z');
+  ASSERT_TRUE(dest[3] == L'a');
+  ASSERT_TRUE(dest[4] == L'a');
+  ASSERT_TRUE(dest[5] == L'b');
+  ASSERT_TRUE(dest[6] == L'\0');
+
+  // Setting end marker to make sure it overwrites properly.
+  dest[3] = L'\0';
+
+  // Copying all of it.
+  LIBC_NAMESPACE::wcsncat(dest, src, 3);
+  ASSERT_TRUE(dest[0] == L'x');
+  ASSERT_TRUE(dest[1] == L'y');
+  ASSERT_TRUE(dest[2] == L'z');
+  ASSERT_TRUE(dest[3] == L'a');
+  ASSERT_TRUE(dest[4] == L'b');
+  ASSERT_TRUE(dest[5] == L'c');
+  ASSERT_TRUE(dest[6] == L'\0');
+
+  // Check that copying still works when count > src length.
+  dest[0] = L'\0';
+  // And that it doesn't write beyond what is necessary.
+  dest[4] = L'Z';
+  LIBC_NAMESPACE::wcsncat(dest, src, 4);
+  ASSERT_TRUE(dest[0] == L'a');
+  ASSERT_TRUE(dest[1] == L'b');
+  ASSERT_TRUE(dest[2] == L'c');
+  ASSERT_TRUE(dest[3] == L'\0');
+  ASSERT_TRUE(dest[4] == L'Z');
+}
diff --git a/external/llvm-project/libc/test/src/wchar/wcsncmp_test.cpp b/external/llvm-project/libc/test/src/wchar/wcsncmp_test.cpp
new file mode 100644
index 000000000000..28bbb5264822
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcsncmp_test.cpp
@@ -0,0 +1,169 @@
+//===-- Unittests for wcsncmp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcsncmp.h"
+#include "test/UnitTest/Test.h"
+
+// This group is just copies of the wcscmp tests, since all the same cases still
+// need to be tested.
+
+TEST(LlvmLibcWcsncmpTest, EmptyStringsShouldReturnZeroWithSufficientLength) {
+  const wchar_t *s1 = L"";
+  const wchar_t *s2 = L"";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 1);
+  ASSERT_EQ(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 1);
+  ASSERT_EQ(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest,
+     EmptyStringShouldNotEqualNonEmptyStringWithSufficientLength) {
+  const wchar_t *empty = L"";
+  const wchar_t *s2 = L"abc";
+  int result = LIBC_NAMESPACE::wcsncmp(empty, s2, 3);
+  ASSERT_LT(result, 0);
+
+  // Similar case if empty string is second argument.
+  const wchar_t *s3 = L"123";
+  result = LIBC_NAMESPACE::wcsncmp(s3, empty, 3);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest, EqualStringsShouldReturnZeroWithSufficientLength) {
+  const wchar_t *s1 = L"abc";
+  const wchar_t *s2 = L"abc";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 3);
+  ASSERT_EQ(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 3);
+  ASSERT_EQ(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest,
+     ShouldReturnResultOfFirstDifferenceWithSufficientLength) {
+  const wchar_t *s1 = L"___B42__";
+  const wchar_t *s2 = L"___C55__";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 8);
+  ASSERT_LT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 8);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest,
+     CapitalizedLetterShouldNotBeEqualWithSufficientLength) {
+  const wchar_t *s1 = L"abcd";
+  const wchar_t *s2 = L"abCd";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 4);
+  ASSERT_GT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 4);
+  ASSERT_LT(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest,
+     UnequalLengthStringsShouldNotReturnZeroWithSufficientLength) {
+  const wchar_t *s1 = L"abc";
+  const wchar_t *s2 = L"abcd";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 4);
+  ASSERT_LT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 4);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest, StringArgumentSwapChangesSignWithSufficientLength) {
+  const wchar_t *a = L"a";
+  const wchar_t *b = L"b";
+  int result = LIBC_NAMESPACE::wcsncmp(b, a, 1);
+  ASSERT_GT(result, 0);
+
+  result = LIBC_NAMESPACE::wcsncmp(a, b, 1);
+  ASSERT_LT(result, 0);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+TEST(LlvmLibcWcsncmpTest, NullptrCrash) {
+  // Passing in a nullptr should crash the program.
+  EXPECT_DEATH([] { LIBC_NAMESPACE::wcsncmp(L"aaaaaaaaaaaaaa", nullptr, 3); },
+               WITH_SIGNAL(-1));
+  EXPECT_DEATH([] { LIBC_NAMESPACE::wcsncmp(nullptr, L"aaaaaaaaaaaaaa", 3); },
+               WITH_SIGNAL(-1));
+}
+#endif // LIBC_HAS_ADDRESS_SANITIZER
+
+// This group is actually testing wcsncmp functionality
+
+TEST(LlvmLibcWcsncmpTest, NonEqualStringsEqualWithLengthZero) {
+  const wchar_t *s1 = L"abc";
+  const wchar_t *s2 = L"def";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 0);
+  ASSERT_EQ(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 0);
+  ASSERT_EQ(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest, NonEqualStringsNotEqualWithLengthOne) {
+  const wchar_t *s1 = L"abc";
+  const wchar_t *s2 = L"def";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 1);
+  ASSERT_LT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 1);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest, NonEqualStringsEqualWithShorterLength) {
+  const wchar_t *s1 = L"___B42__";
+  const wchar_t *s2 = L"___C55__";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 3);
+  ASSERT_EQ(result, 0);
+
+  // This should return 'B' - 'C' = -1.
+  result = LIBC_NAMESPACE::wcsncmp(s1, s2, 4);
+  ASSERT_LT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 3);
+  ASSERT_EQ(result, 0);
+
+  // This should return 'C' - 'B' = 1.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 4);
+  ASSERT_GT(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest, StringComparisonEndsOnNullByteEvenWithLongerLength) {
+  const wchar_t *s1 = L"abc\0def";
+  const wchar_t *s2 = L"abc\0abc";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 7);
+  ASSERT_EQ(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 7);
+  ASSERT_EQ(result, 0);
+}
+
+TEST(LlvmLibcWcsncmpTest, Case) {
+  const wchar_t *s1 = L"aB";
+  const wchar_t *s2 = L"ab";
+  int result = LIBC_NAMESPACE::wcsncmp(s1, s2, 2);
+  ASSERT_LT(result, 0);
+
+  // Verify operands reversed.
+  result = LIBC_NAMESPACE::wcsncmp(s2, s1, 2);
+  ASSERT_GT(result, 0);
+}
diff --git a/external/llvm-project/libc/test/src/wchar/wcsncpy_test.cpp b/external/llvm-project/libc/test/src/wchar/wcsncpy_test.cpp
new file mode 100644
index 000000000000..9b5ffbe20b4a
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcsncpy_test.cpp
@@ -0,0 +1,66 @@
+//===-- Unittests for wcsncpy ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcsncpy.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSNCpyTest, CopyZero) {
+  // Dest should remain unchanged.
+  wchar_t dest[3] = {L'a', L'b', L'\0'};
+  const wchar_t *src = L"x";
+  LIBC_NAMESPACE::wcsncpy(dest, src, 0);
+  ASSERT_TRUE(dest[0] == L'a');
+  ASSERT_TRUE(dest[1] == L'b');
+  ASSERT_TRUE(dest[2] == L'\0');
+}
+
+TEST(LlvmLibcWCSNCpyTest, CopyFullIntoEmpty) {
+  // Dest should be the exact same as src.
+  wchar_t dest[15];
+  const wchar_t *src = L"aaaaabbbbccccc";
+  LIBC_NAMESPACE::wcsncpy(dest, src, 15);
+  for (int i = 0; i < 15; i++)
+    ASSERT_TRUE(dest[i] == src[i]);
+}
+
+TEST(LlvmLibcWCSNCpyTest, CopyPartial) {
+  // First two characters of dest should be the first two characters of src.
+  wchar_t dest[] = {L'a', L'b', L'c', L'd', L'\0'};
+  const wchar_t *src = L"1234";
+  LIBC_NAMESPACE::wcsncpy(dest, src, 2);
+  ASSERT_TRUE(dest[0] == L'1');
+  ASSERT_TRUE(dest[1] == L'2');
+  ASSERT_TRUE(dest[2] == L'c');
+  ASSERT_TRUE(dest[3] == L'd');
+  ASSERT_TRUE(dest[4] == L'\0');
+}
+
+TEST(LlvmLibcWCSNCpyTest, CopyNullTerminator) {
+  // Null terminator should copy into dest.
+  wchar_t dest[] = {L'a', L'b', L'c', L'd', L'\0'};
+  const wchar_t src[] = {L'\0', L'y'};
+  LIBC_NAMESPACE::wcsncpy(dest, src, 1);
+  ASSERT_TRUE(dest[0] == L'\0');
+  ASSERT_TRUE(dest[1] == L'b');
+  ASSERT_TRUE(dest[2] == L'c');
+  ASSERT_TRUE(dest[3] == L'd');
+  ASSERT_TRUE(dest[4] == L'\0');
+}
+
+TEST(LlvmLibcWCSNCpyTest, CopyPastSrc) {
+  // Copying past src should fill with null terminator.
+  wchar_t dest[] = {L'a', L'b', L'c', L'd', L'\0'};
+  const wchar_t src[] = {L'x', L'\0'};
+  LIBC_NAMESPACE::wcsncpy(dest, src, 4);
+  ASSERT_TRUE(dest[0] == L'x');
+  ASSERT_TRUE(dest[1] == L'\0');
+  ASSERT_TRUE(dest[2] == L'\0');
+  ASSERT_TRUE(dest[3] == L'\0');
+  ASSERT_TRUE(dest[4] == L'\0');
+}
diff --git a/external/llvm-project/libc/test/src/wchar/wcspbrk_test.cpp b/external/llvm-project/libc/test/src/wchar/wcspbrk_test.cpp
index f7754c0b324e..bca9bffcf85f 100644
--- a/external/llvm-project/libc/test/src/wchar/wcspbrk_test.cpp
+++ b/external/llvm-project/libc/test/src/wchar/wcspbrk_test.cpp
@@ -60,3 +60,13 @@ TEST(LlvmLibcWCSPBrkTest, FindsFirstInBreakset) {
   EXPECT_EQ(LIBC_NAMESPACE::wcspbrk(src, L"34"), src + 2);
   EXPECT_EQ(LIBC_NAMESPACE::wcspbrk(src, L"43"), src + 2);
 }
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+TEST(LlvmLibcWCSPBrkTest, NullptrCrash) {
+  // Passing in a nullptr should crash the program.
+  EXPECT_DEATH([] { LIBC_NAMESPACE::wcspbrk(L"aaaaaaaaaaaaaa", nullptr); },
+               WITH_SIGNAL(-1));
+  EXPECT_DEATH([] { LIBC_NAMESPACE::wcspbrk(nullptr, L"aaaaaaaaaaaaaa"); },
+               WITH_SIGNAL(-1));
+}
+#endif // LIBC_HAS_ADDRESS_SANITIZER
diff --git a/external/llvm-project/libc/test/src/wchar/wcsrchr_test.cpp b/external/llvm-project/libc/test/src/wchar/wcsrchr_test.cpp
new file mode 100644
index 000000000000..707dfb6756ee
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcsrchr_test.cpp
@@ -0,0 +1,68 @@
+//===-- Unittests for wcsrchr ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcsrchr.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSRChrTest, FindsFirstCharacter) {
+  // Should return pointer to original string since 'a' is the first character.
+  const wchar_t *src = L"abcde";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'a'), src);
+}
+
+TEST(LlvmLibcWCSRChrTest, FindsMiddleCharacter) {
+  // Should return pointer to 'c'.
+  const wchar_t *src = L"abcde";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'c'), (src + 2));
+}
+
+TEST(LlvmLibcWCSRChrTest, FindsLastCharacterThatIsNotNullTerminator) {
+  // Should return pointer to 'e'.
+  const wchar_t *src = L"abcde";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'e'), (src + 4));
+}
+
+TEST(LlvmLibcWCSRChrTest, FindsNullTerminator) {
+  // Should return pointer to null terminator.
+  const wchar_t *src = L"abcde";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'\0'), (src + 5));
+}
+
+TEST(LlvmLibcWCSRChrTest, CharacterNotWithinStringShouldReturnNullptr) {
+  // Since 'z' is not within the string, should return nullptr.
+  const wchar_t *src = L"abcde";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'z'), nullptr);
+}
+
+TEST(LlvmLibcWCSRChrTest, ShouldFindLastOfDuplicates) {
+  // Should return pointer to the last '1'.
+  const wchar_t *src = L"abc1def1ghi";
+  ASSERT_EQ((int)(LIBC_NAMESPACE::wcsrchr(src, L'1') - src), 7);
+
+  // Should return pointer to the last 'X'
+  const wchar_t *dups = L"XXXXX";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(dups, L'X'), dups + 4);
+}
+
+TEST(LlvmLibcWCSRChrTest, EmptyStringShouldOnlyMatchNullTerminator) {
+  // Null terminator should match
+  const wchar_t *src = L"";
+  ASSERT_EQ(src, LIBC_NAMESPACE::wcsrchr(src, L'\0'));
+  // All other characters should not match
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'Z'), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'3'), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wcsrchr(src, L'*'), nullptr);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+TEST(LlvmLibcWCSRChrTest, NullptrCrash) {
+  // Passing in a nullptr should crash the program.
+  EXPECT_DEATH([] { LIBC_NAMESPACE::wcsrchr(nullptr, L'a'); }, WITH_SIGNAL(-1));
+}
+#endif // LIBC_HAS_ADDRESS_SANITIZER
diff --git a/external/llvm-project/libc/test/src/wchar/wcsstr_test.cpp b/external/llvm-project/libc/test/src/wchar/wcsstr_test.cpp
new file mode 100644
index 000000000000..c1448bbaf1ab
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wcsstr_test.cpp
@@ -0,0 +1,113 @@
+//===-- Unittests for wcsstr ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcsstr.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSStrTest, NeedleNotInHaystack) {
+  // Should return nullptr if string is not found.
+  const wchar_t *haystack = L"12345";
+  const wchar_t *needle = L"a";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), nullptr);
+}
+
+TEST(LlvmLibcWCSStrTest, NeedleIsEmptyString) {
+  // Should return pointer to first character if needle is empty.
+  const wchar_t *haystack = L"12345";
+  const wchar_t *needle = L"";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack);
+}
+
+TEST(LlvmLibcWCSStrTest, HaystackIsEmptyString) {
+  // Should return nullptr since haystack is empty.
+  const wchar_t *needle = L"12345";
+  const wchar_t *haystack = L"";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), nullptr);
+}
+
+TEST(LlvmLibcWCSStrTest, HaystackAndNeedleAreEmptyStrings) {
+  // Should point to haystack since needle is empty.
+  const wchar_t *needle = L"";
+  const wchar_t *haystack = L"";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack);
+}
+
+TEST(LlvmLibcWCSStrTest, HaystackAndNeedleAreSingleCharacters) {
+  const wchar_t *haystack = L"a";
+  // Should point to haystack.
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"a"), haystack);
+  // Should return nullptr.
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"b"), nullptr);
+}
+
+TEST(LlvmLibcWCSStrTest, NeedleEqualToHaystack) {
+  const wchar_t *haystack = L"12345";
+  // Should point to haystack.
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"12345"), haystack);
+}
+
+TEST(LlvmLibcWCSStrTest, NeedleLargerThanHaystack) {
+  const wchar_t *haystack = L"123";
+  // Should return nullptr.
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"12345"), nullptr);
+}
+
+TEST(LlvmLibcWCSStrTest, NeedleAtBeginning) {
+  const wchar_t *haystack = L"12345";
+  const wchar_t *needle = L"12";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack);
+}
+
+TEST(LlvmLibcWCSStrTest, NeedleInMiddle) {
+  const wchar_t *haystack = L"abcdefghi";
+  const wchar_t *needle = L"def";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack + 3);
+}
+
+TEST(LlvmLibcWCSStrTest, NeedleDirectlyBeforeNullTerminator) {
+  const wchar_t *haystack = L"abcdefghi";
+  const wchar_t *needle = L"ghi";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack + 6);
+}
+
+TEST(LlvmLibcWCSStrTest, NeedlePastNullTerminator) {
+  const wchar_t haystack[5] = {L'1', L'2', L'\0', L'3', L'4'};
+  // Shouldn't find anything after the null terminator.
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, /*needle=*/L"3"), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, /*needle=*/L"4"), nullptr);
+}
+
+TEST(LlvmLibcWCSStrTest, PartialNeedle) {
+  const wchar_t *haystack = L"la_ap_lap";
+  const wchar_t *needle = L"lap";
+  // Shouldn't find la or ap.
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack + 6);
+}
+
+TEST(LlvmLibcWCSStrTest, MisspelledNeedle) {
+  const wchar_t *haystack = L"atalloftwocities...wait, tale";
+  const wchar_t *needle = L"tale";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack + 25);
+}
+
+TEST(LlvmLibcWCSStrTest, AnagramNeedle) {
+  const wchar_t *haystack = L"dgo_ogd_god_odg_gdo_dog";
+  const wchar_t *needle = L"dog";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, needle), haystack + 20);
+}
+
+TEST(LlvmLibcWCSStrTest, MorphedNeedle) {
+  // Changes a single letter in the needle to mismatch with the haystack.
+  const wchar_t *haystack = L"once upon a time";
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"time"), haystack + 12);
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"lime"), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"tome"), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"tire"), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wcsstr(haystack, L"timo"), nullptr);
+}
diff --git a/external/llvm-project/libc/test/src/wchar/wmemchr_test.cpp b/external/llvm-project/libc/test/src/wchar/wmemchr_test.cpp
new file mode 100644
index 000000000000..6b25bd83b16e
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wmemchr_test.cpp
@@ -0,0 +1,92 @@
+//===-- Unittests for wmemchr ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wmemchr.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWMemChrTest, FindsCharacterAfterNullTerminator) {
+  // wmemchr should continue searching after a null terminator.
+  const size_t size = 5;
+  const wchar_t src[size] = {L'a', L'\0', L'b', L'c', L'\0'};
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'b', size), (src + 2));
+}
+
+TEST(LlvmLibcWMemChrTest, FindsCharacterInNonNullTerminatedCollection) {
+  const size_t size = 3;
+  const wchar_t src[size] = {L'a', L'b', L'c'};
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'b', size), (src + 1));
+}
+
+TEST(LlvmLibcWMemChrTest, FindsFirstCharacter) {
+  const size_t size = 6;
+  const wchar_t *src = L"abcde";
+  // Should return original array since 'a' is the first character.
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'a', size), (src));
+}
+
+TEST(LlvmLibcWMemChrTest, FindsMiddleCharacter) {
+  const size_t size = 6;
+  const wchar_t *src = L"abcde";
+  // Should return characters after and including 'c'.
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'c', size), (src + 2));
+}
+
+TEST(LlvmLibcWMemChrTest, FindsLastCharacterThatIsNotNullTerminator) {
+  const size_t size = 6;
+  const wchar_t *src = L"abcde";
+  // Should return 'e' and null terminator.
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'e', size), (src + 4));
+}
+
+TEST(LlvmLibcWMemChrTest, FindsNullTerminator) {
+  const size_t size = 6;
+  const wchar_t *src = L"abcde";
+  // Should return null terminator.
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'\0', size), (src + 5));
+}
+
+TEST(LlvmLibcWMemChrTest, CharacterNotWithinStringShouldReturnNullptr) {
+  const size_t size = 6;
+  const wchar_t *src = L"abcde";
+  // Should return nullptr.
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'z', size), nullptr);
+}
+
+TEST(LlvmLibcWMemChrTest, CharacterNotWithinSizeShouldReturnNullptr) {
+  const size_t size = 3;
+  const wchar_t *src = L"abcde";
+  // Should return nullptr.
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'd', size), nullptr);
+}
+
+TEST(LlvmLibcWMemChrTest, TheSourceShouldNotChange) {
+  const size_t size = 3;
+  const wchar_t *src = L"ab";
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'a', size), src);
+  ASSERT_TRUE(src[0] == L'a');
+  ASSERT_TRUE(src[1] == L'b');
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'c', size), nullptr);
+  ASSERT_TRUE(src[0] == L'a');
+  ASSERT_TRUE(src[1] == L'b');
+}
+
+TEST(LlvmLibcWMemChrTest, EmptyStringShouldOnlyMatchNullTerminator) {
+  const size_t size = 1;
+  const wchar_t *src = L"";
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'\0', size), src);
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'c', size), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'1', size), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'?', size), nullptr);
+}
+
+TEST(LlvmLibcWMemChrTest, SingleRepeatedCharacterShouldReturnFirst) {
+  const size_t size = 6;
+  const wchar_t *src = L"XXXXX";
+  ASSERT_EQ(LIBC_NAMESPACE::wmemchr(src, L'X', size), src);
+}
diff --git a/external/llvm-project/libc/test/src/wchar/wmemmove_test.cpp b/external/llvm-project/libc/test/src/wchar/wmemmove_test.cpp
new file mode 100644
index 000000000000..d23aa0f0b3af
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wmemmove_test.cpp
@@ -0,0 +1,111 @@
+//===-- Unittests for wmemmove --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wmemmove.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWMemmoveTest, MoveZeroByte) {
+  wchar_t buffer[] = {L'a', L'b', L'y', L'z'};
+
+  wchar_t *ret = LIBC_NAMESPACE::wmemmove(buffer, buffer + 2, 0);
+  EXPECT_EQ(ret, buffer);
+
+  const wchar_t expected[] = {L'a', L'b', L'y', L'z'};
+  EXPECT_TRUE(buffer[0] == expected[0]);
+  EXPECT_TRUE(buffer[1] == expected[1]);
+  EXPECT_TRUE(buffer[2] == expected[2]);
+  EXPECT_TRUE(buffer[3] == expected[3]);
+}
+
+TEST(LlvmLibcWMemmoveTest, DstAndSrcPointToSameAddress) {
+  wchar_t buffer[] = {L'a', L'b'};
+
+  wchar_t *ret = LIBC_NAMESPACE::wmemmove(buffer, buffer, 1);
+  EXPECT_EQ(ret, buffer);
+
+  const wchar_t expected[] = {L'a', L'b'};
+  EXPECT_TRUE(buffer[0] == expected[0]);
+  EXPECT_TRUE(buffer[1] == expected[1]);
+}
+
+TEST(LlvmLibcWMemmoveTest, DstStartsBeforeSrc) {
+  // Set boundary at beginning and end for not overstepping when
+  // copy forward or backward.
+  wchar_t buffer[] = {L'z', L'a', L'b', L'c', L'z'};
+
+  wchar_t *dst = buffer + 1;
+  wchar_t *ret = LIBC_NAMESPACE::wmemmove(dst, buffer + 2, 2);
+  EXPECT_EQ(ret, dst);
+
+  const wchar_t expected[] = {L'z', L'b', L'c', L'c', L'z'};
+  EXPECT_TRUE(buffer[0] == expected[0]);
+  EXPECT_TRUE(buffer[1] == expected[1]);
+  EXPECT_TRUE(buffer[2] == expected[2]);
+  EXPECT_TRUE(buffer[3] == expected[3]);
+  EXPECT_TRUE(buffer[4] == expected[4]);
+}
+
+TEST(LlvmLibcWMemmoveTest, DstStartsAfterSrc) {
+  wchar_t buffer[] = {L'z', L'a', L'b', L'c', L'z'};
+
+  wchar_t *dst = buffer + 2;
+  wchar_t *ret = LIBC_NAMESPACE::wmemmove(dst, buffer + 1, 2);
+  EXPECT_EQ(ret, dst);
+
+  const wchar_t expected[] = {L'z', L'a', L'a', L'b', L'z'};
+  EXPECT_TRUE(buffer[0] == expected[0]);
+  EXPECT_TRUE(buffer[1] == expected[1]);
+  EXPECT_TRUE(buffer[2] == expected[2]);
+  EXPECT_TRUE(buffer[3] == expected[3]);
+  EXPECT_TRUE(buffer[4] == expected[4]);
+}
+
+// e.g. `Dst` follow `src`.
+// str: [abcdefghij]
+//      [__src_____]
+//      [_____Dst__]
+TEST(LlvmLibcWMemmoveTest, SrcFollowDst) {
+  wchar_t buffer[] = {L'z', L'a', L'b', L'z'};
+
+  wchar_t *dst = buffer + 1;
+  wchar_t *ret = LIBC_NAMESPACE::wmemmove(dst, buffer + 2, 1);
+  EXPECT_EQ(ret, dst);
+
+  const char expected[] = {L'z', L'b', L'b', L'z'};
+  EXPECT_TRUE(buffer[0] == expected[0]);
+  EXPECT_TRUE(buffer[1] == expected[1]);
+  EXPECT_TRUE(buffer[2] == expected[2]);
+  EXPECT_TRUE(buffer[3] == expected[3]);
+}
+
+TEST(LlvmLibcWMemmoveTest, DstFollowSrc) {
+  wchar_t buffer[] = {L'z', L'a', L'b', L'z'};
+
+  wchar_t *dst = buffer + 2;
+  wchar_t *ret = LIBC_NAMESPACE::wmemmove(dst, buffer + 1, 1);
+  EXPECT_EQ(ret, dst);
+
+  const char expected[] = {L'z', L'a', L'a', L'z'};
+  EXPECT_TRUE(buffer[0] == expected[0]);
+  EXPECT_TRUE(buffer[1] == expected[1]);
+  EXPECT_TRUE(buffer[2] == expected[2]);
+  EXPECT_TRUE(buffer[3] == expected[3]);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+TEST(LlvmLibcWMemmoveTest, NullptrCrash) {
+  wchar_t buffer[] = {L'a', L'b'};
+  // Passing in a nullptr should crash the program.
+  EXPECT_DEATH([&buffer] { LIBC_NAMESPACE::wmemmove(buffer, nullptr, 2); },
+               WITH_SIGNAL(-1));
+  EXPECT_DEATH([&buffer] { LIBC_NAMESPACE::wmemmove(nullptr, buffer, 2); },
+               WITH_SIGNAL(-1));
+}
+#endif // LIBC_HAS_ADDRESS_SANITIZER
diff --git a/external/llvm-project/libc/test/src/wchar/wmempcpy_test.cpp b/external/llvm-project/libc/test/src/wchar/wmempcpy_test.cpp
new file mode 100644
index 000000000000..000e4b389909
--- /dev/null
+++ b/external/llvm-project/libc/test/src/wchar/wmempcpy_test.cpp
@@ -0,0 +1,50 @@
+//===-- Unittests for wmempcpy --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wmempcpy.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWMempcpyTest, Simple) {
+  const wchar_t *src = L"12345";
+  wchar_t dest[10] = {};
+  void *result = LIBC_NAMESPACE::wmempcpy(dest, src, 6);
+  ASSERT_EQ(static_cast<wchar_t *>(result), dest + 6);
+
+  ASSERT_TRUE(dest[0] == src[0]);
+  ASSERT_TRUE(dest[1] == src[1]);
+  ASSERT_TRUE(dest[2] == src[2]);
+  ASSERT_TRUE(dest[3] == src[3]);
+  ASSERT_TRUE(dest[4] == src[4]);
+  ASSERT_TRUE(dest[5] == src[5]);
+}
+
+TEST(LlvmLibcWmempcpyTest, ZeroCount) {
+  const wchar_t *src = L"12345";
+  wchar_t dest[5] = {};
+  void *result = LIBC_NAMESPACE::wmempcpy(dest, src, 0);
+  ASSERT_EQ(static_cast<wchar_t *>(result), dest);
+
+  ASSERT_TRUE(dest[0] == 0);
+  ASSERT_TRUE(dest[1] == 0);
+  ASSERT_TRUE(dest[2] == 0);
+  ASSERT_TRUE(dest[3] == 0);
+  ASSERT_TRUE(dest[4] == 0);
+}
+
+TEST(LlvmLibcWMempcpyTest, BoundaryCheck) {
+  const wchar_t *src = L"12345";
+  wchar_t dest[4] = {};
+  void *result = LIBC_NAMESPACE::wmempcpy(dest + 1, src + 1, 2);
+
+  ASSERT_TRUE(dest[0] == 0);
+  ASSERT_TRUE(dest[1] == src[1]);
+  ASSERT_TRUE(dest[2] == src[2]);
+  ASSERT_TRUE(dest[3] == 0);
+
+  ASSERT_EQ(static_cast<wchar_t *>(result), dest + 3);
+}
diff --git a/external/llvm-project/libc/test/utils/FPUtil/CMakeLists.txt b/external/llvm-project/libc/test/utils/FPUtil/CMakeLists.txt
index 7b6c294506b1..c4ade937d09a 100644
--- a/external/llvm-project/libc/test/utils/FPUtil/CMakeLists.txt
+++ b/external/llvm-project/libc/test/utils/FPUtil/CMakeLists.txt
@@ -1,4 +1,5 @@
-if((${LIBC_TARGET_OS} STREQUAL "linux") AND (${LIBC_TARGET_ARCHITECTURE_IS_X86}))
+if((${LIBC_TARGET_OS} STREQUAL "linux") AND ((${LIBC_TARGET_ARCHITECTURE_IS_X86})
+   OR (${LIBC_TARGET_ARCHITECTURE_IS_X86_64})))
   add_libc_unittest(
     x86_long_double_test
     SRCS
diff --git a/external/llvm-project/libclc/CMakeLists.txt b/external/llvm-project/libclc/CMakeLists.txt
index 0d9a21c6d63a..c98e2043464d 100644
--- a/external/llvm-project/libclc/CMakeLists.txt
+++ b/external/llvm-project/libclc/CMakeLists.txt
@@ -72,7 +72,7 @@ else()
   # Note that we check this later (for both build types) but we can provide a
   # more useful error message when built in-tree. We assume that LLVM tools are
   # always available so don't warn here.
-  if( NOT clang IN_LIST LLVM_ENABLE_PROJECTS )
+  if( NOT LLVM_RUNTIMES_BUILD AND NOT clang IN_LIST LLVM_ENABLE_PROJECTS )
     message(FATAL_ERROR "Clang is not enabled, but is required to build libclc in-tree")
   endif()
 
@@ -445,8 +445,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
     if( ARCH STREQUAL amdgcn OR ARCH STREQUAL r600 OR ARCH STREQUAL amdgcn-amdhsa )
       set ( private_addrspace_val 5 )
     endif()
-    if( ARCH STREQUAL spirv OR ARCH STREQUAL spirv64
-        OR ARCH STREQUAL clspv OR ARCH STREQUAL clspv64 )
+    if( ARCH STREQUAL spirv OR ARCH STREQUAL spirv64)
       set ( generic_addrspace_val 4 )
     endif()
     list( APPEND build_flags
diff --git a/external/llvm-project/libclc/clc/include/clc/geometric/clc_fast_normalize.h b/external/llvm-project/libclc/clc/include/clc/geometric/clc_fast_normalize.h
new file mode 100644
index 000000000000..66eed8b83ab1
--- /dev/null
+++ b/external/llvm-project/libclc/clc/include/clc/geometric/clc_fast_normalize.h
@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
+#define __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
+
+#define __FLOAT_ONLY
+#define __CLC_GEOMETRIC_RET_GENTYPE
+#define __CLC_FUNCTION __clc_fast_normalize
+#define __CLC_BODY <clc/geometric/unary_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_FUNCTION
+#undef __CLC_GEOMETRIC_RET_GENTYPE
+#undef __FLOAT_ONLY
+
+#endif // __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
diff --git a/external/llvm-project/libclc/clc/include/clc/geometric/clc_normalize.h b/external/llvm-project/libclc/clc/include/clc/geometric/clc_normalize.h
new file mode 100644
index 000000000000..3058a72b2bbb
--- /dev/null
+++ b/external/llvm-project/libclc/clc/include/clc/geometric/clc_normalize.h
@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_GEOMETRIC_CLC_NORMALIZE_H__
+#define __CLC_GEOMETRIC_CLC_NORMALIZE_H__
+
+#define __CLC_GEOMETRIC_RET_GENTYPE
+#define __CLC_FUNCTION __clc_normalize
+#define __CLC_BODY <clc/geometric/unary_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_FUNCTION
+#undef __CLC_GEOMETRIC_RET_GENTYPE
+
+#endif // __CLC_GEOMETRIC_CLC_NORMALIZE_H__
diff --git a/external/llvm-project/libclc/clc/lib/generic/SOURCES b/external/llvm-project/libclc/clc/lib/generic/SOURCES
index 0b5a805cfc33..d285bbba3dd2 100644
--- a/external/llvm-project/libclc/clc/lib/generic/SOURCES
+++ b/external/llvm-project/libclc/clc/lib/generic/SOURCES
@@ -9,7 +9,9 @@ geometric/clc_distance.cl
 geometric/clc_dot.cl
 geometric/clc_fast_distance.cl
 geometric/clc_fast_length.cl
+geometric/clc_fast_normalize.cl
 geometric/clc_length.cl
+geometric/clc_normalize.cl
 integer/clc_abs.cl
 integer/clc_abs_diff.cl
 integer/clc_add_sat.cl
diff --git a/external/llvm-project/libclc/clc/lib/generic/geometric/clc_dot.cl b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_dot.cl
index 2c82798d71a5..a30f9bc6e301 100644
--- a/external/llvm-project/libclc/clc/lib/generic/geometric/clc_dot.cl
+++ b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_dot.cl
@@ -7,59 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/internal/clc.h>
+#include <clc/math/clc_fma.h>
 
-_CLC_OVERLOAD _CLC_DEF float __clc_dot(float p0, float p1) { return p0 * p1; }
-
-_CLC_OVERLOAD _CLC_DEF float __clc_dot(float2 p0, float2 p1) {
-  return p0.x * p1.x + p0.y * p1.y;
-}
-
-_CLC_OVERLOAD _CLC_DEF float __clc_dot(float3 p0, float3 p1) {
-  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z;
-}
-
-_CLC_OVERLOAD _CLC_DEF float __clc_dot(float4 p0, float4 p1) {
-  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z + p0.w * p1.w;
-}
-
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_OVERLOAD _CLC_DEF double __clc_dot(double p0, double p1) {
-  return p0 * p1;
-}
-
-_CLC_OVERLOAD _CLC_DEF double __clc_dot(double2 p0, double2 p1) {
-  return p0.x * p1.x + p0.y * p1.y;
-}
-
-_CLC_OVERLOAD _CLC_DEF double __clc_dot(double3 p0, double3 p1) {
-  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z;
-}
-
-_CLC_OVERLOAD _CLC_DEF double __clc_dot(double4 p0, double4 p1) {
-  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z + p0.w * p1.w;
-}
-
-#endif
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_OVERLOAD _CLC_DEF half __clc_dot(half p0, half p1) { return p0 * p1; }
-
-_CLC_OVERLOAD _CLC_DEF half __clc_dot(half2 p0, half2 p1) {
-  return p0.x * p1.x + p0.y * p1.y;
-}
-
-_CLC_OVERLOAD _CLC_DEF half __clc_dot(half3 p0, half3 p1) {
-  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z;
-}
-
-_CLC_OVERLOAD _CLC_DEF half __clc_dot(half4 p0, half4 p1) {
-  return p0.x * p1.x + p0.y * p1.y + p0.z * p1.z + p0.w * p1.w;
-}
-
-#endif
+#define __CLC_BODY <clc_dot.inc>
+#include <clc/math/gentype.inc>
diff --git a/external/llvm-project/libclc/clc/lib/generic/geometric/clc_dot.inc b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_dot.inc
new file mode 100644
index 000000000000..0c89493bf733
--- /dev/null
+++ b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_dot.inc
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if (__CLC_VECSIZE_OR_1 == 1 || __CLC_VECSIZE_OR_1 == 2 ||                     \
+     __CLC_VECSIZE_OR_1 == 3 || __CLC_VECSIZE_OR_1 == 4)
+
+_CLC_OVERLOAD _CLC_DEF __CLC_SCALAR_GENTYPE __clc_dot(__CLC_GENTYPE x,
+                                                      __CLC_GENTYPE y) {
+#if __CLC_VECSIZE_OR_1 == 1
+  return x * y;
+#elif __CLC_VECSIZE_OR_1 == 2
+  return x.s0 * y.s0 + x.s1 * y.s1;
+#elif __CLC_VECSIZE_OR_1 == 3
+  return x.s0 * y.s0 + x.s1 * y.s1 + x.s2 * y.s2;
+#else
+  return x.s0 * y.s0 + x.s1 * y.s1 + x.s2 * y.s2 + x.s3 * y.s3;
+#endif
+}
+
+#endif
diff --git a/external/llvm-project/libclc/clc/lib/generic/geometric/clc_fast_normalize.cl b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_fast_normalize.cl
new file mode 100644
index 000000000000..85684d0f49bc
--- /dev/null
+++ b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_fast_normalize.cl
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/geometric/clc_dot.h>
+#include <clc/geometric/clc_normalize.h>
+#include <clc/math/clc_half_rsqrt.h>
+
+#define __FLOAT_ONLY
+#define __CLC_BODY <clc_fast_normalize.inc>
+#include <clc/math/gentype.inc>
diff --git a/external/llvm-project/libclc/clc/lib/generic/geometric/clc_fast_normalize.inc b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_fast_normalize.inc
new file mode 100644
index 000000000000..e4c3ab2c5a65
--- /dev/null
+++ b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_fast_normalize.inc
@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if __CLC_VECSIZE_OR_1 == 1
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_fast_normalize(__CLC_GENTYPE p) {
+  return __clc_normalize(p);
+}
+
+#elif (__CLC_VECSIZE_OR_1 == 2 || __CLC_VECSIZE_OR_1 == 3 ||                   \
+       __CLC_VECSIZE_OR_1 == 4)
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_fast_normalize(__CLC_GENTYPE p) {
+  __CLC_SCALAR_GENTYPE l2 = __clc_dot(p, p);
+  return l2 == 0.0f ? p : p * __clc_half_rsqrt(l2);
+}
+
+#endif
diff --git a/external/llvm-project/libclc/clc/lib/generic/geometric/clc_normalize.cl b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_normalize.cl
new file mode 100644
index 000000000000..ec4a67647c13
--- /dev/null
+++ b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_normalize.cl
@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/clc_convert.h>
+#include <clc/common/clc_sign.h>
+#include <clc/float/definitions.h>
+#include <clc/geometric/clc_dot.h>
+#include <clc/geometric/clc_normalize.h>
+#include <clc/internal/clc.h>
+#include <clc/math/clc_copysign.h>
+#include <clc/math/clc_rsqrt.h>
+#include <clc/relational/clc_all.h>
+#include <clc/relational/clc_isinf.h>
+#include <clc/relational/clc_select.h>
+
+#define __CLC_BODY <clc_normalize.inc>
+#include <clc/math/gentype.inc>
diff --git a/external/llvm-project/libclc/clc/lib/generic/geometric/clc_normalize.inc b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_normalize.inc
new file mode 100644
index 000000000000..8a47c6d4826e
--- /dev/null
+++ b/external/llvm-project/libclc/clc/lib/generic/geometric/clc_normalize.inc
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if (__CLC_VECSIZE_OR_1 == 1 || __CLC_VECSIZE_OR_1 == 2 ||                     \
+     __CLC_VECSIZE_OR_1 == 3 || __CLC_VECSIZE_OR_1 == 4)
+
+// Until we have a native FP16 implementation, go via FP32
+#if __CLC_FPSIZE == 16
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_normalize(__CLC_GENTYPE p) {
+  return __CLC_CONVERT_GENTYPE(__clc_normalize(__CLC_CONVERT_FLOATN(p)));
+}
+
+// Scalar normalize
+#elif defined(__CLC_SCALAR)
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_normalize(__CLC_GENTYPE p) {
+  return __clc_sign(p);
+}
+
+// Vector normalize
+#else
+
+#if __CLC_FPSIZE == 32
+
+#define MIN_VAL FLT_MIN
+#define MAX_SQRT 0x1.0p+86F
+#if __CLC_VECSIZE_OR_1 == 2
+#define MIN_SQRT 0x1.0p-65F
+#else
+#define MIN_SQRT 0x1.0p-66F
+#endif
+
+#elif __CLC_FPSIZE == 64
+
+#define MIN_VAL DBL_MIN
+#define MAX_SQRT 0x1.0p+563
+#if __CLC_VECSIZE_OR_1 == 2
+#define MIN_SQRT 0x1.0p-513
+#else
+#define MIN_SQRT 0x1.0p-514
+#endif
+
+#else
+#error "Invalid FP size"
+#endif
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_normalize(__CLC_GENTYPE p) {
+  if (__clc_all(p == __CLC_FP_LIT(0.0))) {
+    return p;
+  }
+
+  __CLC_SCALAR_GENTYPE l2 = __clc_dot(p, p);
+
+  if (l2 < MIN_VAL) {
+    p *= MAX_SQRT;
+    l2 = __clc_dot(p, p);
+  } else if (l2 == INFINITY) {
+    p *= MIN_SQRT;
+    l2 = __clc_dot(p, p);
+    if (l2 == INFINITY) {
+      p = __clc_copysign(__clc_select((__CLC_GENTYPE)__CLC_FP_LIT(0.0),
+                                      (__CLC_GENTYPE)__CLC_FP_LIT(1.0),
+                                      __clc_isinf(p)),
+                         p);
+      l2 = __clc_dot(p, p);
+    }
+  }
+  return p * __clc_rsqrt(l2);
+}
+
+#undef MIN_VAL
+#undef MIN_SQRT
+#undef MAX_SQRT
+
+#endif
+
+#endif
diff --git a/external/llvm-project/libclc/opencl/lib/generic/geometric/dot.cl b/external/llvm-project/libclc/opencl/lib/generic/geometric/dot.cl
index 72d30ea53f8c..785767f27aac 100644
--- a/external/llvm-project/libclc/opencl/lib/generic/geometric/dot.cl
+++ b/external/llvm-project/libclc/opencl/lib/generic/geometric/dot.cl
@@ -9,60 +9,6 @@
 #include <clc/geometric/clc_dot.h>
 #include <clc/opencl/clc.h>
 
-_CLC_OVERLOAD _CLC_DEF float dot(float p0, float p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF float dot(float2 p0, float2 p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF float dot(float3 p0, float3 p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF float dot(float4 p0, float4 p1) {
-  return __clc_dot(p0, p1);
-}
-
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_OVERLOAD _CLC_DEF double dot(double p0, double p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF double dot(double2 p0, double2 p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF double dot(double3 p0, double3 p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF double dot(double4 p0, double4 p1) {
-  return __clc_dot(p0, p1);
-}
-
-#endif
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_OVERLOAD _CLC_DEF half dot(half p0, half p1) { return __clc_dot(p0, p1); }
-
-_CLC_OVERLOAD _CLC_DEF half dot(half2 p0, half2 p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF half dot(half3 p0, half3 p1) {
-  return __clc_dot(p0, p1);
-}
-
-_CLC_OVERLOAD _CLC_DEF half dot(half4 p0, half4 p1) {
-  return __clc_dot(p0, p1);
-}
-
-#endif
+#define FUNCTION dot
+#define __CLC_BODY <clc/geometric/binary_def.inc>
+#include <clc/math/gentype.inc>
diff --git a/external/llvm-project/libclc/opencl/lib/generic/geometric/fast_normalize.cl b/external/llvm-project/libclc/opencl/lib/generic/geometric/fast_normalize.cl
index d6d140f3c406..938b30093b8d 100644
--- a/external/llvm-project/libclc/opencl/lib/generic/geometric/fast_normalize.cl
+++ b/external/llvm-project/libclc/opencl/lib/generic/geometric/fast_normalize.cl
@@ -6,11 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/geometric/clc_fast_normalize.h>
 #include <clc/opencl/clc.h>
 
-_CLC_OVERLOAD _CLC_DEF float fast_normalize(float p) { return normalize(p); }
-
-#define __CLC_BODY <fast_normalize.inc>
+#define FUNCTION fast_normalize
 #define __FLOAT_ONLY
+#define __CLC_GEOMETRIC_RET_GENTYPE
+#define __CLC_BODY <clc/geometric/unary_def.inc>
+
 #include <clc/math/gentype.inc>
-#undef __FLOAT_ONLY
diff --git a/external/llvm-project/libclc/opencl/lib/generic/geometric/fast_normalize.inc b/external/llvm-project/libclc/opencl/lib/generic/geometric/fast_normalize.inc
deleted file mode 100644
index ec1be94efc27..000000000000
--- a/external/llvm-project/libclc/opencl/lib/generic/geometric/fast_normalize.inc
+++ /dev/null
@@ -1,19 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// Geometric functions are only defined for scalar, vec2, vec3 and vec4
-// Only handle vector implementations
-#if (__CLC_VECSIZE_OR_1 == 2 || __CLC_VECSIZE_OR_1 == 3 ||                     \
-     __CLC_VECSIZE_OR_1 == 4)
-
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE fast_normalize(__CLC_GENTYPE p) {
-  __CLC_SCALAR_GENTYPE l2 = dot(p, p);
-  return l2 == 0.0f ? p : p * half_rsqrt(l2);
-}
-
-#endif
diff --git a/external/llvm-project/libclc/opencl/lib/generic/geometric/normalize.cl b/external/llvm-project/libclc/opencl/lib/generic/geometric/normalize.cl
index e1e51b306a1b..914f493f2cb2 100644
--- a/external/llvm-project/libclc/opencl/lib/generic/geometric/normalize.cl
+++ b/external/llvm-project/libclc/opencl/lib/generic/geometric/normalize.cl
@@ -6,134 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/geometric/clc_normalize.h>
 #include <clc/opencl/clc.h>
 
-_CLC_OVERLOAD _CLC_DEF float normalize(float p) { return sign(p); }
+#define FUNCTION normalize
+#define __CLC_GEOMETRIC_RET_GENTYPE
+#define __CLC_BODY <clc/geometric/unary_def.inc>
 
-_CLC_OVERLOAD _CLC_DEF float2 normalize(float2 p) {
-  if (all(p == (float2)0.0F))
-    return p;
-
-  float l2 = dot(p, p);
-
-  if (l2 < FLT_MIN) {
-    p *= 0x1.0p+86F;
-    l2 = dot(p, p);
-  } else if (l2 == INFINITY) {
-    p *= 0x1.0p-65f;
-    l2 = dot(p, p);
-    if (l2 == INFINITY) {
-      p = copysign(select((float2)0.0F, (float2)1.0F, isinf(p)), p);
-      l2 = dot(p, p);
-    }
-  }
-  return p * rsqrt(l2);
-}
-
-_CLC_OVERLOAD _CLC_DEF float3 normalize(float3 p) {
-  if (all(p == (float3)0.0F))
-    return p;
-
-  float l2 = dot(p, p);
-
-  if (l2 < FLT_MIN) {
-    p *= 0x1.0p+86F;
-    l2 = dot(p, p);
-  } else if (l2 == INFINITY) {
-    p *= 0x1.0p-66f;
-    l2 = dot(p, p);
-    if (l2 == INFINITY) {
-      p = copysign(select((float3)0.0F, (float3)1.0F, isinf(p)), p);
-      l2 = dot(p, p);
-    }
-  }
-  return p * rsqrt(l2);
-}
-
-_CLC_OVERLOAD _CLC_DEF float4 normalize(float4 p) {
-  if (all(p == (float4)0.0F))
-    return p;
-
-  float l2 = dot(p, p);
-
-  if (l2 < FLT_MIN) {
-    p *= 0x1.0p+86F;
-    l2 = dot(p, p);
-  } else if (l2 == INFINITY) {
-    p *= 0x1.0p-66f;
-    l2 = dot(p, p);
-    if (l2 == INFINITY) {
-      p = copysign(select((float4)0.0F, (float4)1.0F, isinf(p)), p);
-      l2 = dot(p, p);
-    }
-  }
-  return p * rsqrt(l2);
-}
-
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_OVERLOAD _CLC_DEF double normalize(double p) { return sign(p); }
-
-_CLC_OVERLOAD _CLC_DEF double2 normalize(double2 p) {
-  if (all(p == (double2)0.0))
-    return p;
-
-  double l2 = dot(p, p);
-
-  if (l2 < DBL_MIN) {
-    p *= 0x1.0p+563;
-    l2 = dot(p, p);
-  } else if (l2 == INFINITY) {
-    p *= 0x1.0p-513;
-    l2 = dot(p, p);
-    if (l2 == INFINITY) {
-      p = copysign(select((double2)0.0, (double2)1.0, isinf(p)), p);
-      l2 = dot(p, p);
-    }
-  }
-  return p * rsqrt(l2);
-}
-
-_CLC_OVERLOAD _CLC_DEF double3 normalize(double3 p) {
-  if (all(p == (double3)0.0))
-    return p;
-
-  double l2 = dot(p, p);
-
-  if (l2 < DBL_MIN) {
-    p *= 0x1.0p+563;
-    l2 = dot(p, p);
-  } else if (l2 == INFINITY) {
-    p *= 0x1.0p-514;
-    l2 = dot(p, p);
-    if (l2 == INFINITY) {
-      p = copysign(select((double3)0.0, (double3)1.0, isinf(p)), p);
-      l2 = dot(p, p);
-    }
-  }
-  return p * rsqrt(l2);
-}
-
-_CLC_OVERLOAD _CLC_DEF double4 normalize(double4 p) {
-  if (all(p == (double4)0.0))
-    return p;
-
-  double l2 = dot(p, p);
-
-  if (l2 < DBL_MIN) {
-    p *= 0x1.0p+563;
-    l2 = dot(p, p);
-  } else if (l2 == INFINITY) {
-    p *= 0x1.0p-514;
-    l2 = dot(p, p);
-    if (l2 == INFINITY) {
-      p = copysign(select((double4)0.0, (double4)1.0, isinf(p)), p);
-      l2 = dot(p, p);
-    }
-  }
-  return p * rsqrt(l2);
-}
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/external/llvm-project/libcxx/cmake/caches/AndroidNDK.cmake b/external/llvm-project/libcxx/cmake/caches/AndroidNDK.cmake
index 298518781e9b..1a04b7fbb217 100644
--- a/external/llvm-project/libcxx/cmake/caches/AndroidNDK.cmake
+++ b/external/llvm-project/libcxx/cmake/caches/AndroidNDK.cmake
@@ -33,5 +33,5 @@ set(CMAKE_CXX_COMPILER_WORKS ON CACHE BOOL "")
 
 # Use adb to push tests to a locally-connected device (e.g. emulator) and run
 # them.
-set(LIBCXX_TEST_CONFIG "llvm-libc++-android-ndk.cfg.in" CACHE STRING "")
-set(LIBCXXABI_TEST_CONFIG "llvm-libc++abi-android-ndk.cfg.in" CACHE STRING "")
+set(LIBCXX_TEST_CONFIG "llvm-libc++-android.cfg.in" CACHE STRING "")
+set(LIBCXXABI_TEST_CONFIG "llvm-libc++abi-android.cfg.in" CACHE STRING "")
diff --git a/external/llvm-project/libcxx/docs/ABIGuarantees.rst b/external/llvm-project/libcxx/docs/ABIGuarantees.rst
new file mode 100644
index 000000000000..e6ac4f2b5b23
--- /dev/null
+++ b/external/llvm-project/libcxx/docs/ABIGuarantees.rst
@@ -0,0 +1,210 @@
+.. _ABIGuarantees:
+
+=======================
+libc++'s ABI Guarantees
+=======================
+
+libc++ provides multiple types of ABI guarantees. These include stability of the layout of structs, the linking of TUs
+built against different versions and configurations of the library, and more. This document describes what guarantees
+libc++ provides in these different areas as well as what options exist for vendors to affect these guarantees.
+
+Note that all of the guarantees listed below come with an asterisk that there may be circumstances where we deem it
+worth it to break that guarantee. These breaks are communicated to vendors by CCing #libcxx-vendors on GitHub. If you
+are a vendor, please ask to be added to that group to be notified about changes that potentially affect you.
+
+ABI flags
+=========
+All the ABI flags listed below can be added to the ``__config_site`` header by the vendor to opt in to an ABI breaking
+change. These flags should never be set by the user. When porting libc++ to a new platform, vendord should consider
+which flags to enable, assuming that ABI stability is relevant to them. Please contact the libc++ team on Discord or
+through other means to be able to make an informed decision on which flags make sense to enable, and to avoid enabling
+flags which may not be stable. Flags can be enabled via the ``LIBCXX_ABI_DEFINES`` CMake option.
+
+
+Stability of the Layout of Structs
+==================================
+
+The layout of any user-observable struct is kept stable across versions of the library and any user-facing options
+documented :ref:`here <libcxx-configuration-macros>`. There are a lot of structs that have internal names, but are none
+the less observable by users; for example through public aliases to these types or because they affect the layout of
+other types.
+
+There are multiple ABI flags which affect the layout of certain structs:
+
+``_LIBCPP_ABI_ALTERNATE_STRING_LAYOUT``
+---------------------------------------
+This changes the internal layout of ``basic_string`` to move the section that is used for the internal buffer to the
+front, making it eight byte aligned instead of being unaligned, improving the performance of some operations
+significantly.
+
+``_LIBCPP_ABI_NO_ITERATOR_BASES``
+---------------------------------
+This removes the ``iterator`` base class from ``back_insert_iterator``, ``front_insert_iterator``, ``insert_iterator``,
+``istream_iterator``, ``ostream_iterator``, ``ostreambuf_iterator``, ``reverse_iterator``, and ``raw_storage_iterator``.
+This doesn't directly affect the layout of these types in most cases, but may result in more padding being used when
+they are used in combination, for example ``reverse_iterator<reverse_iterator<T>>``.
+
+``_LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION``
+-------------------------------------------------
+This changes the index type used inside ``variant`` to the smallest required type to reduce the datasize of variants in
+most cases.
+
+``_LIBCPP_ABI_OPTIMIZED_FUNCTION``
+----------------------------------
+This significantly restructures how ``function`` is written to provide better performance, but is currently not ABI
+stable.
+
+``_LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT``
+-----------------------------------------------------
+This changes the layout of ``random_device`` to only holds state with an implementation that gets entropy from a file
+(see ``_LIBCPP_USING_DEV_RANDOM``). When switching from this implementation to another one on a platform that has
+already shipped ``random_device``, one needs to retain the same object layout to remain ABI compatible. This flag
+removes these workarounds for platforms that don't care about ABI compatibility.
+
+``_LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING``
+------------------------------------------
+This removes artificial padding from ``_LIBCPP_COMPRESSED_PAIR`` and ``_LIBCPP_COMPRESSED_TRIPLE``.
+
+These macros are used inside the associative and unordered containers, ``deque``, ``forward_list``, ``future``,
+``list``, ``basic_string``, ``function``, ``shared_ptr``, ``unique_ptr``, and ``vector`` to stay ABI compatible with the
+legacy ``__compressed_pair`` type. ``__compressed_pair`` had historically been used to reduce storage requirements in
+the case of empty types, but has been replaced by ``[[no_unique_address]]``. ``[[no_unique_address]]`` is significantly
+lighter in terms of compile time and debug information, and also improves the layout of structs further. However, to
+keep ABI stability, the additional improvements in layout had to be reverted by introducing artificial padding. This
+flag removes that artificial padding.
+
+``_LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE``
+----------------------------------------------
+``basic_ios`` uses ``WEOF`` to indicate that the fill value is uninitialized. However, on platforms where the size of
+``char_type`` is equal to or greater than the size of ``int_type`` and ``char_type`` is unsigned,
+``char_traits<char_type>::eq_int_type()`` cannot distinguish between ``WEOF`` and ``WCHAR_MAX``. This flag changes
+``basic_ios`` to instead track whether the fill value has been initialized using a separate boolean.
+
+
+Linking TUs which have been compiled against different releases of libc++
+=========================================================================
+libc++ supports linking TUs which have been compiled against different releases of libc++ by marking symbols with
+hidden visibility and changing the mangling of header-only functions in every release.
+
+
+Linking TUs which have been compiled with different flags affecting code gen
+============================================================================
+There are a lot of compiler (and library) flags which change the code generated for functions. This includes flags like
+``-O1``, which are guaranteed by the compiler to not change the observable behaviour of a correct program, as well as
+flags like ``-fexceptions``, which **do** change the observable behaviour. libc++ allows linking of TUs which have been
+compiled whith specific flags only and makes no guarantees for any of the flags not listed below.
+
+The flags allowed (in any combination) are:
+- ``-f[no-]exceptions``
+- ``-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE{_FAST,_EXTENSIVE,_DEBUG,_NONE}``
+
+Note that this does not provide any guarantees about user-defined functions, but only that the libc++ functions linked
+behave as the flags say.
+
+
+Availability of symbols in the built library (both static and shared)
+=====================================================================
+In general, libc++ does not make any guarantees about forwards-compatibility. That is, a TU compiled against new headers
+may not work with an older library. Vendors who require such support can leverage availability markup. On the other
+hand, backwards compatibility is generally guaranteed.
+
+There are multiple ABI flags that change the symbols exported from the built library:
+
+``_LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON``
+-------------------------------------------------
+This removes ``__basic_string_common<true>::__throw_length_error()`` and
+``__basic_string_common<true>::__throw_out_of_range()``. These symbols have been used by ``basic_string`` in the past,
+but are not referenced from the headers anymore.
+
+``_LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON``
+------------------------------------------------
+This removes ``__vector_base_common<true>::__throw_length_error()`` and
+``__vector_base_common<true>::__throw_out_of_range()``. These symbols have been used by ``vector`` in the past, but are
+not referenced from the headers anymore.
+
+``_LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10``
+----------------------------------------------
+This removes ``__itoa::__u32toa()`` and ``__iota::__u64toa``. These symbols have been used by ``to_chars`` in the past,
+but are not referenced from the headers anymore.
+
+``_LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION``
+-------------------------------------------------------
+This replaces the symbols that are exported for ``basic_string`` to avoid exporting functions which are likely to be
+inlined as well as explicitly moving paths to the built library which are slow, improving fast-path inlining of multiple
+functions. This flag is currently unstable.
+
+
+Stability of the traits of a type
+=================================
+Whether a particular trait of a type is kept stable depends heavily on the type in question and the trait. The most
+important trait of a type to keep stable is the triviality for the purpose of calls, since that directly affects the
+function call ABI. Which types are considered non-trivial for the purpose of calls is defined in the
+`Itanium ABI <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#definitions>`_.
+``is_trivially_copyable`` should also be kept stable usually, since many programs depend on this trait for their own
+layouting. This isn't as rigid as the previous requirement though.
+
+There are multiple ABI flags that change traits of a struct:
+
+``_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI``
+---------------------------------------------
+This flag adds ``[[clang::trivial_abi]]`` to ``unique_ptr``, which makes it trivial for the purpose of calls.
+
+``_LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI``
+---------------------------------------------
+This flag adds ``[[clang::trivial_abi]]`` to ``shared_ptr``, which makes it trivial for the purpose of calls.
+
+
+Types that public aliases reference
+===================================
+There are a lot of aliases that reference types with library internal names. For example, containers contain an
+``iterator`` alias to a type with a library internal name. These have to always reference the same type, since the
+mangling of user-defined function overloads would change otherwise. A notable exception to this are the alias templates
+to type traits. There doesn't seem to be anybody who relies on these names staying the same, so it is OK to change what
+these aliases actually reference.
+
+There are multiple ABI flags which change which type an alias references:
+
+``_LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE``
+-----------------------------------------
+This changes ``deque::iterator`` to avoid requiring complete types for ``deque``.
+
+``_LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE``
+-------------------------------------------------
+This changes the unordered container's ``size_types`` aliases.
+
+``_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY`` and ``_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW``
+-----------------------------------------------------------------------------------------------
+This changes the ``iterator`` and ``const_iterator`` of ``array`` and ``string_view`` respectively to reference
+``__wrap_iter`` instead, which makes it less likely for users to depend on non-portable implementation details. This is
+especially useful because enabling bounded iterators hardening requires code not to make these assumptions.
+
+``_LIBCPP_ABI_BOUNDED_ITERATORS``, ``_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING``, ``_LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR``, and ``_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY``
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+These flags change the ``iterator`` member of various classes to reference hardened iterators instead. See the
+:ref:`hardening documentation <hardening>` for more details.
+
+
+Meaning of values
+=================
+The meaning of specific values can usually not be changed, since programs compiled against older versions of the headers
+may check for these values. These specific values don't have to be hard-coded, but can also depend on user input.
+
+There are multiple ABI flags that change the meaning of particular values:
+
+``_LIBCPP_ABI_REGEX_CONSTANTS_NONZERO``
+---------------------------------------
+This changes the value of ``regex_constants::syntax_option-type::ECMAScript`` to be standards-conforming.
+
+``_LIBCPP_ABI_FIX_CITYHASH_IMPLEMENTATION``
+-------------------------------------------
+This flag fixes the implementation of CityHash used for ``hash<fundamental-type>``. The incorrect implementation of
+CityHash has the problem that it drops some bits on the floor. Fixing the implementation changes the hash of values,
+resulting in an ABI break.
+
+inline namespaces
+=================
+Inline namespaces which contain types that are observable by the user need to be kept the same, since they affect
+mangling. Almost all of libc++'s symbols are inside an inline namespace. By default that namespace is ``__1``, but can
+be changed by the vendor by setting `LIBCXX_ABI_NAMESPACE` during CMake configuration. There is also
+``_LIBCPP_ABI_NO_FILESYSTEM_INLINE_NAMESPACE`` to remove the ``__fs`` namespace from surrounding the ``filesystem``
+namespace. This shortens the mangling of the filesystem symbols a bit.
diff --git a/external/llvm-project/libcxx/docs/FeatureTestMacroTable.rst b/external/llvm-project/libcxx/docs/FeatureTestMacroTable.rst
index 9b57b7c8eeb5..3e6fd643f620 100644
--- a/external/llvm-project/libcxx/docs/FeatureTestMacroTable.rst
+++ b/external/llvm-project/libcxx/docs/FeatureTestMacroTable.rst
@@ -420,8 +420,12 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_algorithms``                         ``202306L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_forward_list``                       ``202502L``
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_new``                                ``202406L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_queue``                              ``202502L``
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_constrained_equality``                         *unimplemented*
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_copyable_function``                            *unimplemented*
diff --git a/external/llvm-project/libcxx/docs/ReleaseNotes/21.rst b/external/llvm-project/libcxx/docs/ReleaseNotes/21.rst
index d0383a705190..2a5b90750eaf 100644
--- a/external/llvm-project/libcxx/docs/ReleaseNotes/21.rst
+++ b/external/llvm-project/libcxx/docs/ReleaseNotes/21.rst
@@ -97,6 +97,13 @@ Potentially breaking changes
 - The implementation of ``num_put::do_put`` has been replaced to improve the performance, which can lead to different
   output when printing pointers.
 
+- User-defined specializations of ``std::common_reference`` are diagnosed now. To customize the common reference type, ``std::basic_common_reference`` should be specialized instead.
+
+- ``std::function`` used to have allocator support, which was removed from the Standard by `http://wg21.link/p0302r1`
+  due to issues with its design and inconsistent support from implementations. Previously, libc++ would provide
+  allocator-aware APIs in ``std::function`` in C++11 and C++14, but ignores the allocator argument in all places but
+  one. Starting in this release, the allocator argument is always ignored.
+
 Announcements About Future Releases
 -----------------------------------
 
diff --git a/external/llvm-project/libcxx/docs/Status/Cxx2cIssues.csv b/external/llvm-project/libcxx/docs/Status/Cxx2cIssues.csv
index 9bf31c417f3c..fdf381862d87 100644
--- a/external/llvm-project/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/external/llvm-project/libcxx/docs/Status/Cxx2cIssues.csv
@@ -107,7 +107,7 @@
 "`LWG4153 <https://wg21.link/LWG4153>`__","Fix extra ""-1"" for ``philox_engine::max()``","2024-11 (Wrocław)","","",""
 "`LWG4154 <https://wg21.link/LWG4154>`__","The Mandates for ``std::packaged_task``'s constructor from a callable entity should consider decaying","2024-11 (Wrocław)","","",""
 "`LWG4157 <https://wg21.link/LWG4157>`__","The resolution of LWG3465 was damaged by P2167R3","2024-11 (Wrocław)","","20",""
-"`LWG4164 <https://wg21.link/LWG4164>`__","Missing guarantees for ``forward_list`` modifiers","2024-11 (Wrocław)","","",""
+"`LWG4164 <https://wg21.link/LWG4164>`__","Missing guarantees for ``forward_list`` modifiers","2024-11 (Wrocław)","|Complete|","21",""
 "`LWG4169 <https://wg21.link/LWG4169>`__","``std::atomic<T>``'s default constructor should be constrained","2024-11 (Wrocław)","","",""
 "`LWG4170 <https://wg21.link/LWG4170>`__","``contiguous_iterator`` should require ``to_address(I{})``","2024-11 (Wrocław)","","",""
 "","","","","",""
diff --git a/external/llvm-project/libcxx/docs/Status/Cxx2cPapers.csv b/external/llvm-project/libcxx/docs/Status/Cxx2cPapers.csv
index 3809446a5789..8a0417e120d7 100644
--- a/external/llvm-project/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/external/llvm-project/libcxx/docs/Status/Cxx2cPapers.csv
@@ -59,7 +59,7 @@
 "`P2248R8 <https://wg21.link/P2248R8>`__","Enabling list-initialization for algorithms","2024-03 (Tokyo)","","",""
 "`P2810R4 <https://wg21.link/P2810R4>`__","``is_debugger_present`` ``is_replaceable``","2024-03 (Tokyo)","","",""
 "`P1068R11 <https://wg21.link/P1068R11>`__","Vector API for random number generation","2024-03 (Tokyo)","","",""
-"`P2944R3 <https://wg21.link/P2944R3>`__","Comparisons for ``reference_wrapper``","2024-03 (Tokyo)","|Partial|","","Implemented changes to ``reference_wrapper`` and ``pair``"
+"`P2944R3 <https://wg21.link/P2944R3>`__","Comparisons for ``reference_wrapper``","2024-03 (Tokyo)","|Partial|","","The changes to ``optional``, ``tuple`` and ``variant`` are not yet implemented"
 "`P2642R6 <https://wg21.link/P2642R6>`__","Padded ``mdspan`` layouts","2024-03 (Tokyo)","","",""
 "`P3029R1 <https://wg21.link/P3029R1>`__","Better ``mdspan``'s CTAD","2024-03 (Tokyo)","|Complete|","19",""
 "","","","","",""
diff --git a/external/llvm-project/libcxx/docs/UserDocumentation.rst b/external/llvm-project/libcxx/docs/UserDocumentation.rst
index dbacb44735d0..f82554a4869f 100644
--- a/external/llvm-project/libcxx/docs/UserDocumentation.rst
+++ b/external/llvm-project/libcxx/docs/UserDocumentation.rst
@@ -82,6 +82,8 @@ when ``-fexperimental-library`` is passed:
       version has shipped. The full policy is explained :ref:`here <experimental features>`.
 
 
+.. _libcxx-configuration-macros:
+
 Libc++ Configuration Macros
 ===========================
 
diff --git a/external/llvm-project/libcxx/docs/VendorDocumentation.rst b/external/llvm-project/libcxx/docs/VendorDocumentation.rst
index 959a28607d75..aede8f9a81dd 100644
--- a/external/llvm-project/libcxx/docs/VendorDocumentation.rst
+++ b/external/llvm-project/libcxx/docs/VendorDocumentation.rst
@@ -582,3 +582,8 @@ situations will give the same result:
   $ clang++ -stdlib=libc++ helloworld.cpp -lcxxrt
 
 .. _`libcxxrt`: https://github.com/libcxxrt/libcxxrt
+
+libc++'s ABI guarantees
+=======================
+
+Libc++ provides several ABI guarantees, which are documented :ref:`here <ABIGuarantees>`.
diff --git a/external/llvm-project/libcxx/docs/index.rst b/external/llvm-project/libcxx/docs/index.rst
index a8b0d5ce1ee9..ae9cc87c797f 100644
--- a/external/llvm-project/libcxx/docs/index.rst
+++ b/external/llvm-project/libcxx/docs/index.rst
@@ -37,6 +37,7 @@ Getting Started with libc++
    ReleaseNotes
    UserDocumentation
    VendorDocumentation
+   ABIGuarantees
    Contributing
    CodingGuidelines
    TestingLibcxx
@@ -134,7 +135,7 @@ Compiler     Versions            Restrictions               Support policy
 Clang        19, 20, 21-git                                 latest two stable releases per `LLVM's release page <https://releases.llvm.org>`_ and the development version
 AppleClang   15                                             latest stable release per `Xcode's release page <https://developer.apple.com/documentation/xcode-release-notes>`_
 Open XL      17.1.3 (AIX)                                   latest stable release per `Open XL's documentation page <https://www.ibm.com/docs/en/openxl-c-and-cpp-aix>`_
-GCC          14                  In C++11 or later only     latest stable release per `GCC's release page <https://gcc.gnu.org/releases.html>`_
+GCC          15                  In C++11 or later only     latest stable release per `GCC's release page <https://gcc.gnu.org/releases.html>`_
 ============ =================== ========================== =====================
 
 Libc++ also supports common platforms and architectures:
diff --git a/external/llvm-project/libcxx/include/CMakeLists.txt b/external/llvm-project/libcxx/include/CMakeLists.txt
index 43cefd560064..8931a1b35f6d 100644
--- a/external/llvm-project/libcxx/include/CMakeLists.txt
+++ b/external/llvm-project/libcxx/include/CMakeLists.txt
@@ -800,6 +800,7 @@ set(files
   __type_traits/extent.h
   __type_traits/has_unique_object_representation.h
   __type_traits/has_virtual_destructor.h
+  __type_traits/integer_traits.h
   __type_traits/integral_constant.h
   __type_traits/invoke.h
   __type_traits/is_abstract.h
@@ -850,7 +851,6 @@ set(files
   __type_traits/is_same.h
   __type_traits/is_scalar.h
   __type_traits/is_signed.h
-  __type_traits/is_signed_integer.h
   __type_traits/is_specialization.h
   __type_traits/is_standard_layout.h
   __type_traits/is_swappable.h
@@ -864,7 +864,6 @@ set(files
   __type_traits/is_unbounded_array.h
   __type_traits/is_union.h
   __type_traits/is_unsigned.h
-  __type_traits/is_unsigned_integer.h
   __type_traits/is_valid_expansion.h
   __type_traits/is_void.h
   __type_traits/is_volatile.h
@@ -1706,6 +1705,11 @@ add_dependencies(cxx-headers generate-cxx-headers)
 target_include_directories(cxx-headers INTERFACE ${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}
                                                  ${LIBCXX_GENERATED_INCLUDE_DIR})
 
+# Make sure to map the generated include directory back to libc++'s actual source directory when generating
+# debug information. Otherwise, the debug information will refer to generated headers which are created during
+# the build and generally not persistent.
+target_add_compile_flags_if_supported(cxx-headers INTERFACE "-fdebug-prefix-map=${LIBCXX_GENERATED_INCLUDE_DIR}=${LIBCXX_SOURCE_DIR}/include")
+
 if (LIBCXX_INSTALL_HEADERS)
   foreach(file ${files})
     get_filename_component(dir ${file} DIRECTORY)
diff --git a/external/llvm-project/libcxx/include/__bit/bit_ceil.h b/external/llvm-project/libcxx/include/__bit/bit_ceil.h
index cfd792dc2e2a..99881a853829 100644
--- a/external/llvm-project/libcxx/include/__bit/bit_ceil.h
+++ b/external/llvm-project/libcxx/include/__bit/bit_ceil.h
@@ -11,8 +11,8 @@
 
 #include <__assert>
 #include <__bit/countl.h>
-#include <__concepts/arithmetic.h>
 #include <__config>
+#include <__type_traits/integer_traits.h>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -41,7 +41,7 @@ template <class _Tp>
 
 #  if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp bit_ceil(_Tp __t) noexcept {
   return std::__bit_ceil(__t);
 }
diff --git a/external/llvm-project/libcxx/include/__bit/bit_floor.h b/external/llvm-project/libcxx/include/__bit/bit_floor.h
index 6bcbc53fb497..799a064130b4 100644
--- a/external/llvm-project/libcxx/include/__bit/bit_floor.h
+++ b/external/llvm-project/libcxx/include/__bit/bit_floor.h
@@ -10,8 +10,8 @@
 #define _LIBCPP___BIT_BIT_FLOOR_H
 
 #include <__bit/bit_log2.h>
-#include <__concepts/arithmetic.h>
 #include <__config>
+#include <__type_traits/integer_traits.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -21,7 +21,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp bit_floor(_Tp __t) noexcept {
   return __t == 0 ? 0 : _Tp{1} << std::__bit_log2(__t);
 }
diff --git a/external/llvm-project/libcxx/include/__bit/bit_log2.h b/external/llvm-project/libcxx/include/__bit/bit_log2.h
index b22e1ce1f84e..8077cd91d6fd 100644
--- a/external/llvm-project/libcxx/include/__bit/bit_log2.h
+++ b/external/llvm-project/libcxx/include/__bit/bit_log2.h
@@ -11,7 +11,7 @@
 
 #include <__bit/countl.h>
 #include <__config>
-#include <__type_traits/is_unsigned_integer.h>
+#include <__type_traits/integer_traits.h>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -22,7 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __bit_log2(_Tp __t) _NOEXCEPT {
-  static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__bit_log2 requires an unsigned integer type");
+  static_assert(__is_unsigned_integer_v<_Tp>, "__bit_log2 requires an unsigned integer type");
   return numeric_limits<_Tp>::digits - 1 - std::__countl_zero(__t);
 }
 
diff --git a/external/llvm-project/libcxx/include/__bit/bit_width.h b/external/llvm-project/libcxx/include/__bit/bit_width.h
index 853e481776f7..75050acabbe8 100644
--- a/external/llvm-project/libcxx/include/__bit/bit_width.h
+++ b/external/llvm-project/libcxx/include/__bit/bit_width.h
@@ -10,8 +10,8 @@
 #define _LIBCPP___BIT_BIT_WIDTH_H
 
 #include <__bit/bit_log2.h>
-#include <__concepts/arithmetic.h>
 #include <__config>
+#include <__type_traits/integer_traits.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -21,7 +21,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int bit_width(_Tp __t) noexcept {
   return __t == 0 ? 0 : std::__bit_log2(__t) + 1;
 }
diff --git a/external/llvm-project/libcxx/include/__bit/countl.h b/external/llvm-project/libcxx/include/__bit/countl.h
index 9499bf9b458e..075914020879 100644
--- a/external/llvm-project/libcxx/include/__bit/countl.h
+++ b/external/llvm-project/libcxx/include/__bit/countl.h
@@ -9,9 +9,8 @@
 #ifndef _LIBCPP___BIT_COUNTL_H
 #define _LIBCPP___BIT_COUNTL_H
 
-#include <__concepts/arithmetic.h>
 #include <__config>
-#include <__type_traits/is_unsigned_integer.h>
+#include <__type_traits/integer_traits.h>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -25,18 +24,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countl_zero(_Tp __t) _NOEXCEPT {
-  static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__countl_zero requires an unsigned integer type");
+  static_assert(__is_unsigned_integer_v<_Tp>, "__countl_zero requires an unsigned integer type");
   return __builtin_clzg(__t, numeric_limits<_Tp>::digits);
 }
 
 #if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countl_zero(_Tp __t) noexcept {
   return std::__countl_zero(__t);
 }
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countl_one(_Tp __t) noexcept {
   return __t != numeric_limits<_Tp>::max() ? std::countl_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;
 }
diff --git a/external/llvm-project/libcxx/include/__bit/countr.h b/external/llvm-project/libcxx/include/__bit/countr.h
index 7b311b83853c..f6c98695d3d0 100644
--- a/external/llvm-project/libcxx/include/__bit/countr.h
+++ b/external/llvm-project/libcxx/include/__bit/countr.h
@@ -9,9 +9,8 @@
 #ifndef _LIBCPP___BIT_COUNTR_H
 #define _LIBCPP___BIT_COUNTR_H
 
-#include <__concepts/arithmetic.h>
 #include <__config>
-#include <__type_traits/is_unsigned.h>
+#include <__type_traits/integer_traits.h>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -25,18 +24,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __countr_zero(_Tp __t) _NOEXCEPT {
-  static_assert(is_unsigned<_Tp>::value, "__countr_zero only works with unsigned types");
+  static_assert(__is_unsigned_integer_v<_Tp>, "__countr_zero only works with unsigned types");
   return __builtin_ctzg(__t, numeric_limits<_Tp>::digits);
 }
 
 #if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept {
   return std::__countr_zero(__t);
 }
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept {
   return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;
 }
diff --git a/external/llvm-project/libcxx/include/__bit/has_single_bit.h b/external/llvm-project/libcxx/include/__bit/has_single_bit.h
index 52f5853a1bc8..b43e69323e77 100644
--- a/external/llvm-project/libcxx/include/__bit/has_single_bit.h
+++ b/external/llvm-project/libcxx/include/__bit/has_single_bit.h
@@ -9,8 +9,8 @@
 #ifndef _LIBCPP___BIT_HAS_SINGLE_BIT_H
 #define _LIBCPP___BIT_HAS_SINGLE_BIT_H
 
-#include <__concepts/arithmetic.h>
 #include <__config>
+#include <__type_traits/integer_traits.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -23,7 +23,7 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool has_single_bit(_Tp __t) noexcept {
   return __t != 0 && (((__t & (__t - 1)) == 0));
 }
diff --git a/external/llvm-project/libcxx/include/__bit/popcount.h b/external/llvm-project/libcxx/include/__bit/popcount.h
index 9ae572d466ba..8d9ba0993848 100644
--- a/external/llvm-project/libcxx/include/__bit/popcount.h
+++ b/external/llvm-project/libcxx/include/__bit/popcount.h
@@ -9,9 +9,8 @@
 #ifndef _LIBCPP___BIT_POPCOUNT_H
 #define _LIBCPP___BIT_POPCOUNT_H
 
-#include <__concepts/arithmetic.h>
 #include <__config>
-#include <__type_traits/is_unsigned.h>
+#include <__type_traits/integer_traits.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -24,13 +23,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __popcount(_Tp __t) _NOEXCEPT {
-  static_assert(is_unsigned<_Tp>::value, "__popcount only works with unsigned types");
+  static_assert(__is_unsigned_integer_v<_Tp>, "__popcount only works with unsigned types");
   return __builtin_popcountg(__t);
 }
 
 #if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int popcount(_Tp __t) noexcept {
   return std::__popcount(__t);
 }
diff --git a/external/llvm-project/libcxx/include/__bit/rotate.h b/external/llvm-project/libcxx/include/__bit/rotate.h
index d79d98de296a..c6f34bdaf6e6 100644
--- a/external/llvm-project/libcxx/include/__bit/rotate.h
+++ b/external/llvm-project/libcxx/include/__bit/rotate.h
@@ -9,9 +9,8 @@
 #ifndef _LIBCPP___BIT_ROTATE_H
 #define _LIBCPP___BIT_ROTATE_H
 
-#include <__concepts/arithmetic.h>
 #include <__config>
-#include <__type_traits/is_unsigned_integer.h>
+#include <__type_traits/integer_traits.h>
 #include <limits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -25,7 +24,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // the rotr function becomes the ROR instruction.
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotl(_Tp __x, int __s) _NOEXCEPT {
-  static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__rotl requires an unsigned integer type");
+  static_assert(__is_unsigned_integer_v<_Tp>, "__rotl requires an unsigned integer type");
   const int __n = numeric_limits<_Tp>::digits;
   int __r       = __s % __n;
 
@@ -40,7 +39,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotl(_Tp __x, int __s)
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotr(_Tp __x, int __s) _NOEXCEPT {
-  static_assert(__libcpp_is_unsigned_integer<_Tp>::value, "__rotr requires an unsigned integer type");
+  static_assert(__is_unsigned_integer_v<_Tp>, "__rotr requires an unsigned integer type");
   const int __n = numeric_limits<_Tp>::digits;
   int __r       = __s % __n;
 
@@ -55,12 +54,12 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotr(_Tp __x, int __s)
 
 #if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotl(_Tp __t, int __cnt) noexcept {
   return std::__rotl(__t, __cnt);
 }
 
-template <__libcpp_unsigned_integer _Tp>
+template <__unsigned_integer _Tp>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotr(_Tp __t, int __cnt) noexcept {
   return std::__rotr(__t, __cnt);
 }
diff --git a/external/llvm-project/libcxx/include/__chrono/time_point.h b/external/llvm-project/libcxx/include/__chrono/time_point.h
index 6b866b882f89..fc4408d23dbf 100644
--- a/external/llvm-project/libcxx/include/__chrono/time_point.h
+++ b/external/llvm-project/libcxx/include/__chrono/time_point.h
@@ -58,6 +58,19 @@ class time_point {
 
   // arithmetic
 
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr time_point& operator++() {
+    ++__d_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr time_point operator++(int) { return time_point{__d_++}; }
+  _LIBCPP_HIDE_FROM_ABI constexpr time_point& operator--() {
+    --__d_;
+    return *this;
+  }
+  _LIBCPP_HIDE_FROM_ABI constexpr time_point operator--(int) { return time_point{__d_--}; }
+#endif // _LIBCPP_STD_VER >= 20
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 time_point& operator+=(const duration& __d) {
     __d_ += __d;
     return *this;
diff --git a/external/llvm-project/libcxx/include/__concepts/arithmetic.h b/external/llvm-project/libcxx/include/__concepts/arithmetic.h
index 0c44f117805f..64c0200783df 100644
--- a/external/llvm-project/libcxx/include/__concepts/arithmetic.h
+++ b/external/llvm-project/libcxx/include/__concepts/arithmetic.h
@@ -13,8 +13,6 @@
 #include <__type_traits/is_floating_point.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/is_signed.h>
-#include <__type_traits/is_signed_integer.h>
-#include <__type_traits/is_unsigned_integer.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -38,17 +36,6 @@ concept unsigned_integral = integral<_Tp> && !signed_integral<_Tp>;
 template <class _Tp>
 concept floating_point = is_floating_point_v<_Tp>;
 
-// Concept helpers for the internal type traits for the fundamental types.
-
-template <class _Tp>
-concept __libcpp_unsigned_integer = __libcpp_is_unsigned_integer<_Tp>::value;
-
-template <class _Tp>
-concept __libcpp_signed_integer = __libcpp_is_signed_integer<_Tp>::value;
-
-template <class _Tp>
-concept __libcpp_integer = __libcpp_unsigned_integer<_Tp> || __libcpp_signed_integer<_Tp>;
-
 #endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/__config b/external/llvm-project/libcxx/include/__config
index 110450f6e9c5..af8a297fdf3f 100644
--- a/external/llvm-project/libcxx/include/__config
+++ b/external/llvm-project/libcxx/include/__config
@@ -319,37 +319,12 @@ typedef __char32_t char32_t;
 
 #  define _LIBCPP_PREFERRED_ALIGNOF(_Tp) __alignof(_Tp)
 
-// Objective-C++ features (opt-in)
-#  if __has_feature(objc_arc)
-#    define _LIBCPP_HAS_OBJC_ARC 1
-#  else
-#    define _LIBCPP_HAS_OBJC_ARC 0
-#  endif
-
-#  if __has_feature(objc_arc_weak)
-#    define _LIBCPP_HAS_OBJC_ARC_WEAK 1
-#  else
-#    define _LIBCPP_HAS_OBJC_ARC_WEAK 0
-#  endif
-
-#  if __has_extension(blocks)
-#    define _LIBCPP_HAS_EXTENSION_BLOCKS 1
-#  else
-#    define _LIBCPP_HAS_EXTENSION_BLOCKS 0
-#  endif
-
-#  if _LIBCPP_HAS_EXTENSION_BLOCKS && defined(__APPLE__)
+#  if __has_extension(blocks) && defined(__APPLE__)
 #    define _LIBCPP_HAS_BLOCKS_RUNTIME 1
 #  else
 #    define _LIBCPP_HAS_BLOCKS_RUNTIME 0
 #  endif
 
-#  if __has_feature(address_sanitizer)
-#    define _LIBCPP_HAS_ASAN 1
-#  else
-#    define _LIBCPP_HAS_ASAN 0
-#  endif
-
 #  define _LIBCPP_ALWAYS_INLINE __attribute__((__always_inline__))
 
 #  if defined(_LIBCPP_OBJECT_FORMAT_COFF)
@@ -530,13 +505,6 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI
 #  endif
 
-// TODO: Remove this workaround once we drop support for Clang 16
-#  if __has_warning("-Wc++23-extensions")
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++23-extensions")
-#  else
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions")
-#  endif
-
 // Clang modules take a significant compile time hit when pushing and popping diagnostics.
 // Since all the headers are marked as system headers unless _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER is defined, we can
 // simply disable this pushing and popping when _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER isn't defined.
@@ -547,7 +515,7 @@ typedef __char32_t char32_t;
       _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                                                           \
       _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                                                           \
       _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                                                           \
-      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION                                                                 \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++23-extensions")                                                           \
       _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                                                             \
       _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                                                             \
       _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                                                             \
diff --git a/external/llvm-project/libcxx/include/__configuration/abi.h b/external/llvm-project/libcxx/include/__configuration/abi.h
index cc4b930b3cf4..a75cd0a67533 100644
--- a/external/llvm-project/libcxx/include/__configuration/abi.h
+++ b/external/llvm-project/libcxx/include/__configuration/abi.h
@@ -38,84 +38,47 @@
 #endif
 
 #if _LIBCPP_ABI_VERSION >= 2
-// Change short string representation so that string data starts at offset 0,
-// improving its alignment in some cases.
-#  define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
-// Fix deque iterator type in order to support incomplete types.
-#  define _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE
-// Fix undefined behavior in how std::list stores its linked nodes.
+// TODO: Move the description of the remaining ABI flags to ABIGuarantees.rst or remove them.
+
+// Override the default return value of exception::what() for bad_function_call::what()
+// with a string that is specific to bad_function_call (see http://wg21.link/LWG2233).
+// This is an ABI break on platforms that sign and authenticate vtable function pointers
+// because it changes the mangling of the virtual function located in the vtable, which
+// changes how it gets signed.
+#  define _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE
+// According to the Standard, `bitset::operator[] const` returns bool
+#  define _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
+
+// In LLVM 20, we've changed to take these ABI breaks unconditionally. These flags only exist in case someone is running
+// into the static_asserts we added to catch the ABI break and don't care that it is one.
+// TODO(LLVM 22): Remove these flags
 #  define _LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB
-// Fix undefined behavior in  how __tree stores its end and parent nodes.
 #  define _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB
-// Fix undefined behavior in how __hash_table stores its pointer types.
 #  define _LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB
 #  define _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
+
+// These flags are documented in ABIGuarantees.rst
+#  define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#  define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
+#  define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
+#  define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
+#  define _LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI
+#  define _LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
+#  define _LIBCPP_ABI_FIX_CITYHASH_IMPLEMENTATION
 #  define _LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE
-// Give reverse_iterator<T> one data member of type T, not two.
-// Also, in C++17 and later, don't derive iterator types from std::iterator.
+#  define _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE
+#  define _LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE
+#  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
+#  define _LIBCPP_ABI_NO_FILESYSTEM_INLINE_NAMESPACE
 #  define _LIBCPP_ABI_NO_ITERATOR_BASES
-// Use the smallest possible integer type to represent the index of the variant.
-// Previously libc++ used "unsigned int" exclusively.
-#  define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
-// Unstable attempt to provide a more optimized std::function
+#  define _LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT
 #  define _LIBCPP_ABI_OPTIMIZED_FUNCTION
-// All the regex constants must be distinct and nonzero.
 #  define _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
-// Re-worked external template instantiations for std::string with a focus on
-// performance and fast-path inlining.
 #  define _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
-// Enable clang::trivial_abi on std::unique_ptr.
-#  define _LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
-// Enable clang::trivial_abi on std::shared_ptr and std::weak_ptr
-#  define _LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI
-// std::random_device holds some state when it uses an implementation that gets
-// entropy from a file (see _LIBCPP_USING_DEV_RANDOM). When switching from this
-// implementation to another one on a platform that has already shipped
-// std::random_device, one needs to retain the same object layout to remain ABI
-// compatible. This switch removes these workarounds for platforms that don't care
-// about ABI compatibility.
-#  define _LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT
-// Don't export the legacy __basic_string_common class and its methods from the built library.
-#  define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
-// Don't export the legacy __vector_base_common class and its methods from the built library.
-#  define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
-// According to the Standard, `bitset::operator[] const` returns bool
-#  define _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
-// Fix the implementation of CityHash used for std::hash<fundamental-type>.
-// This is an ABI break because `std::hash` will return a different result,
-// which means that hashing the same object in translation units built against
-// different versions of libc++ can return inconsistent results. This is especially
-// tricky since std::hash is used in the implementation of unordered containers.
-//
-// The incorrect implementation of CityHash has the problem that it drops some
-// bits on the floor.
-#  define _LIBCPP_ABI_FIX_CITYHASH_IMPLEMENTATION
-// Remove the base 10 implementation of std::to_chars from the dylib.
-// The implementation moved to the header, but we still export the symbols from
-// the dylib for backwards compatibility.
-#  define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
-// Define std::array/std::string_view iterators to be __wrap_iters instead of raw
-// pointers, which prevents people from relying on a non-portable implementation
-// detail. This is especially useful because enabling bounded iterators hardening
-// requires code not to make these assumptions.
 #  define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY
 #  define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW
-// Dont' add an inline namespace for `std::filesystem`
-#  define _LIBCPP_ABI_NO_FILESYSTEM_INLINE_NAMESPACE
-// std::basic_ios uses WEOF to indicate that the fill value is
-// uninitialized. However, on platforms where the size of char_type is
-// equal to or greater than the size of int_type and char_type is unsigned,
-// std::char_traits<char_type>::eq_int_type() cannot distinguish between WEOF
-// and WCHAR_MAX. This ABI setting determines whether we should instead track whether the fill
-// value has been initialized using a separate boolean, which changes the ABI.
-#  define _LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE
-// Historically, libc++ used a type called `__compressed_pair` to reduce storage needs in cases of empty types (e.g. an
-// empty allocator in std::vector). We switched to using `[[no_unique_address]]`. However, for ABI compatibility reasons
-// we had to add artificial padding in a few places.
-//
-// This setting disables the addition of such artificial padding, leading to a more optimal
-// representation for several types.
-#  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
+#  define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+
 #elif _LIBCPP_ABI_VERSION == 1
 #  if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
 // Enable compiling copies of now inline methods into the dylib to support
@@ -145,35 +108,6 @@
 // The macro below is used for all classes whose ABI have changed as part of fixing these bugs.
 #define _LIBCPP_ABI_LLVM18_NO_UNIQUE_ADDRESS __attribute__((__abi_tag__("llvm18_nua")))
 
-// Changes the iterator type of select containers (see below) to a bounded iterator that keeps track of whether it's
-// within the bounds of the original container and asserts it on every dereference.
-//
-// ABI impact: changes the iterator type of the relevant containers.
-//
-// Supported containers:
-// - `span`;
-// - `string_view`.
-// #define _LIBCPP_ABI_BOUNDED_ITERATORS
-
-// Changes the iterator type of `basic_string` to a bounded iterator that keeps track of whether it's within the bounds
-// of the original container and asserts it on every dereference and when performing iterator arithmetics.
-//
-// ABI impact: changes the iterator type of `basic_string` and its specializations, such as `string` and `wstring`.
-// #define _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
-
-// Changes the iterator type of `vector` to a bounded iterator that keeps track of whether it's within the bounds of the
-// original container and asserts it on every dereference and when performing iterator arithmetics. Note: this doesn't
-// yet affect `vector<bool>`.
-//
-// ABI impact: changes the iterator type of `vector` (except `vector<bool>`).
-// #define _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
-
-// Changes the iterator type of `array` to a bounded iterator that keeps track of whether it's within the bounds of the
-// container and asserts it on every dereference and when performing iterator arithmetic.
-//
-// ABI impact: changes the iterator type of `array`, its size and its layout.
-// #define _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY
-
 // [[msvc::no_unique_address]] seems to mostly affect empty classes, so the padding scheme for Itanium doesn't work.
 #if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING)
 #  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
diff --git a/external/llvm-project/libcxx/include/__configuration/availability.h b/external/llvm-project/libcxx/include/__configuration/availability.h
index bbcdfdc784e9..ae58e36b508b 100644
--- a/external/llvm-project/libcxx/include/__configuration/availability.h
+++ b/external/llvm-project/libcxx/include/__configuration/availability.h
@@ -116,9 +116,6 @@
 #  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH /* nothing */
 #  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP  /* nothing */
 
-#  define _LIBCPP_INTRODUCED_IN_LLVM_4 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE /* nothing */
-
 #elif defined(__APPLE__)
 
 // clang-format off
@@ -258,14 +255,6 @@
     _Pragma("clang attribute pop") \
     _Pragma("clang attribute pop")
 
-// LLVM 4
-#  if defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000
-#    define _LIBCPP_INTRODUCED_IN_LLVM_4 0
-#  else
-#    define _LIBCPP_INTRODUCED_IN_LLVM_4 1
-#  endif
-#  define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE __attribute__((availability(watchos, strict, introduced = 5.0)))
-
 // clang-format on
 
 #else
@@ -277,23 +266,6 @@
 
 #endif
 
-// These macros control the availability of std::bad_optional_access and
-// other exception types. These were put in the shared library to prevent
-// code bloat from every user program defining the vtable for these exception
-// types.
-//
-// Note that when exceptions are disabled, the methods that normally throw
-// these exceptions can be used even on older deployment targets, but those
-// methods will abort instead of throwing.
-#define _LIBCPP_AVAILABILITY_HAS_BAD_OPTIONAL_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4
-#define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
-
-#define _LIBCPP_AVAILABILITY_HAS_BAD_VARIANT_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4
-#define _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
-
-#define _LIBCPP_AVAILABILITY_HAS_BAD_ANY_CAST _LIBCPP_INTRODUCED_IN_LLVM_4
-#define _LIBCPP_AVAILABILITY_BAD_ANY_CAST _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
-
 // These macros control the availability of all parts of <filesystem> that
 // depend on something in the dylib.
 #define _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9
@@ -383,19 +355,6 @@
 #define _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE _LIBCPP_INTRODUCED_IN_LLVM_21
 // No attribute, since we've had bad_function_call::what() in the headers before
 
-// Define availability attributes that depend on _LIBCPP_HAS_EXCEPTIONS.
-// Those are defined in terms of the availability attributes above, and
-// should not be vendor-specific.
-#if !_LIBCPP_HAS_EXCEPTIONS
-#  define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST
-#  define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS
-#  define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
-#else
-#  define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _LIBCPP_AVAILABILITY_BAD_ANY_CAST
-#  define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS
-#  define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS
-#endif
-
 // Define availability attributes that depend on both
 // _LIBCPP_HAS_EXCEPTIONS and _LIBCPP_HAS_RTTI.
 #if !_LIBCPP_HAS_EXCEPTIONS || !_LIBCPP_HAS_RTTI
diff --git a/external/llvm-project/libcxx/include/__debug_utils/sanitizers.h b/external/llvm-project/libcxx/include/__debug_utils/sanitizers.h
index 73d192711eab..058feab0269e 100644
--- a/external/llvm-project/libcxx/include/__debug_utils/sanitizers.h
+++ b/external/llvm-project/libcxx/include/__debug_utils/sanitizers.h
@@ -17,7 +17,7 @@
 #  pragma GCC system_header
 #endif
 
-#if _LIBCPP_HAS_ASAN
+#if __has_feature(address_sanitizer)
 
 extern "C" {
 _LIBCPP_EXPORTED_FROM_ABI void
@@ -28,12 +28,12 @@ _LIBCPP_EXPORTED_FROM_ABI int
 __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, const void*, const void*);
 }
 
-#endif // _LIBCPP_HAS_ASAN
+#endif // __has_feature(address_sanitizer)
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 // ASan choices
-#if _LIBCPP_HAS_ASAN
+#if __has_feature(address_sanitizer)
 #  define _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS 1
 #endif
 
@@ -57,7 +57,7 @@ _LIBCPP_HIDE_FROM_ABI void __annotate_double_ended_contiguous_container(
     const void* __last_old_contained,
     const void* __first_new_contained,
     const void* __last_new_contained) {
-#if !_LIBCPP_HAS_ASAN
+#if !__has_feature(address_sanitizer)
   (void)__first_storage;
   (void)__last_storage;
   (void)__first_old_contained;
@@ -86,7 +86,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __annotate_contiguous_c
     const void* __last_storage,
     const void* __old_last_contained,
     const void* __new_last_contained) {
-#if !_LIBCPP_HAS_ASAN
+#if !__has_feature(address_sanitizer)
   (void)__first_storage;
   (void)__last_storage;
   (void)__old_last_contained;
diff --git a/external/llvm-project/libcxx/include/__flat_map/flat_map.h b/external/llvm-project/libcxx/include/__flat_map/flat_map.h
index f5e9756ff6a6..8f01882934b7 100644
--- a/external/llvm-project/libcxx/include/__flat_map/flat_map.h
+++ b/external/llvm-project/libcxx/include/__flat_map/flat_map.h
@@ -11,16 +11,15 @@
 #define _LIBCPP___FLAT_MAP_FLAT_MAP_H
 
 #include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/lower_bound.h>
 #include <__algorithm/min.h>
 #include <__algorithm/ranges_adjacent_find.h>
 #include <__algorithm/ranges_equal.h>
 #include <__algorithm/ranges_inplace_merge.h>
-#include <__algorithm/ranges_lower_bound.h>
-#include <__algorithm/ranges_partition_point.h>
 #include <__algorithm/ranges_sort.h>
 #include <__algorithm/ranges_unique.h>
-#include <__algorithm/ranges_upper_bound.h>
 #include <__algorithm/remove_if.h>
+#include <__algorithm/upper_bound.h>
 #include <__assert>
 #include <__compare/synth_three_way.h>
 #include <__concepts/swappable.h>
@@ -863,6 +862,13 @@ class flat_map {
     __containers_.values.erase(__containers_.values.begin() + __dist, __containers_.values.end());
   }
 
+  template <class _Self, class _KeyIter>
+  _LIBCPP_HIDE_FROM_ABI static auto __corresponding_mapped_it(_Self&& __self, _KeyIter&& __key_iter) {
+    return __self.__containers_.values.begin() +
+           static_cast<ranges::range_difference_t<mapped_container_type>>(
+               ranges::distance(__self.__containers_.keys.begin(), __key_iter));
+  }
+
   template <bool _WasSorted, class _InputIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI void __append_sort_merge_unique(_InputIterator __first, _Sentinel __last) {
     auto __on_failure        = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
@@ -903,7 +909,8 @@ class flat_map {
 
   template <class _Self, class _Kp>
   _LIBCPP_HIDE_FROM_ABI static auto __key_equal_range(_Self&& __self, const _Kp& __key) {
-    auto __it   = ranges::lower_bound(__self.__containers_.keys, __key, __self.__compare_);
+    auto __it =
+        std::lower_bound(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __key, __self.__compare_);
     auto __last = __self.__containers_.keys.end();
     if (__it == __last || __self.__compare_(__key, *__it)) {
       return std::make_pair(__it, __it);
@@ -914,42 +921,30 @@ class flat_map {
   template <class _Self, class _Kp>
   _LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
     auto [__key_first, __key_last] = __key_equal_range(__self, __key);
-
-    const auto __make_mapped_iter = [&](const auto& __key_iter) {
-      return __self.__containers_.values.begin() +
-             static_cast<ranges::range_difference_t<mapped_container_type>>(
-                 ranges::distance(__self.__containers_.keys.begin(), __key_iter));
-    };
-
-    using __iterator_type = ranges::iterator_t<decltype(__self)>;
-    return std::make_pair(__iterator_type(__key_first, __make_mapped_iter(__key_first)),
-                          __iterator_type(__key_last, __make_mapped_iter(__key_last)));
+    using __iterator_type          = ranges::iterator_t<decltype(__self)>;
+    return std::make_pair(__iterator_type(__key_first, __corresponding_mapped_it(__self, __key_first)),
+                          __iterator_type(__key_last, __corresponding_mapped_it(__self, __key_last)));
   }
 
   template <class _Res, class _Self, class _Kp>
   _LIBCPP_HIDE_FROM_ABI static _Res __lower_bound(_Self&& __self, _Kp& __x) {
-    return __binary_search<_Res>(__self, ranges::lower_bound, __x);
+    auto __key_iter =
+        std::lower_bound(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __x, __self.__compare_);
+    auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
+    return _Res(std::move(__key_iter), std::move(__mapped_iter));
   }
 
   template <class _Res, class _Self, class _Kp>
   _LIBCPP_HIDE_FROM_ABI static _Res __upper_bound(_Self&& __self, _Kp& __x) {
-    return __binary_search<_Res>(__self, ranges::upper_bound, __x);
-  }
-
-  template <class _Res, class _Self, class _Fn, class _Kp>
-  _LIBCPP_HIDE_FROM_ABI static _Res __binary_search(_Self&& __self, _Fn __search_fn, _Kp& __x) {
-    auto __key_iter = __search_fn(__self.__containers_.keys, __x, __self.__compare_);
-    auto __mapped_iter =
-        __self.__containers_.values.begin() +
-        static_cast<ranges::range_difference_t<mapped_container_type>>(
-            ranges::distance(__self.__containers_.keys.begin(), __key_iter));
-
+    auto __key_iter =
+        std::upper_bound(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __x, __self.__compare_);
+    auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
     return _Res(std::move(__key_iter), std::move(__mapped_iter));
   }
 
   template <class _KeyArg, class... _MArgs>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __try_emplace(_KeyArg&& __key, _MArgs&&... __mapped_args) {
-    auto __key_it    = ranges::lower_bound(__containers_.keys, __key, __compare_);
+    auto __key_it    = std::lower_bound(__containers_.keys.begin(), __containers_.keys.end(), __key, __compare_);
     auto __mapped_it = __containers_.values.begin() + ranges::distance(__containers_.keys.begin(), __key_it);
 
     if (__key_it == __containers_.keys.end() || __compare_(__key, *__key_it)) {
@@ -1017,11 +1012,11 @@ class flat_map {
   }
 
   _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
-    if constexpr (requires { __containers_.keys.reserve(__size); }) {
+    if constexpr (__container_traits<_KeyContainer>::__reservable) {
       __containers_.keys.reserve(__size);
     }
 
-    if constexpr (requires { __containers_.values.reserve(__size); }) {
+    if constexpr (__container_traits<_MappedContainer>::__reservable) {
       __containers_.values.reserve(__size);
     }
   }
diff --git a/external/llvm-project/libcxx/include/__flat_map/flat_multimap.h b/external/llvm-project/libcxx/include/__flat_map/flat_multimap.h
index 15fcd7995ad0..0af6aac00c38 100644
--- a/external/llvm-project/libcxx/include/__flat_map/flat_multimap.h
+++ b/external/llvm-project/libcxx/include/__flat_map/flat_multimap.h
@@ -10,18 +10,16 @@
 #ifndef _LIBCPP___FLAT_MAP_FLAT_MULTIMAP_H
 #define _LIBCPP___FLAT_MAP_FLAT_MULTIMAP_H
 
+#include <__algorithm/equal_range.h>
 #include <__algorithm/lexicographical_compare_three_way.h>
+#include <__algorithm/lower_bound.h>
 #include <__algorithm/min.h>
 #include <__algorithm/ranges_equal.h>
-#include <__algorithm/ranges_equal_range.h>
 #include <__algorithm/ranges_inplace_merge.h>
 #include <__algorithm/ranges_is_sorted.h>
-#include <__algorithm/ranges_lower_bound.h>
-#include <__algorithm/ranges_partition_point.h>
 #include <__algorithm/ranges_sort.h>
-#include <__algorithm/ranges_unique.h>
-#include <__algorithm/ranges_upper_bound.h>
 #include <__algorithm/remove_if.h>
+#include <__algorithm/upper_bound.h>
 #include <__assert>
 #include <__compare/synth_three_way.h>
 #include <__concepts/convertible_to.h>
@@ -443,7 +441,7 @@ class flat_multimap {
              is_move_constructible_v<mapped_type>
   _LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
     std::pair<key_type, mapped_type> __pair(std::forward<_Args>(__args)...);
-    auto __key_it    = ranges::upper_bound(__containers_.keys, __pair.first, __compare_);
+    auto __key_it    = std::upper_bound(__containers_.keys.begin(), __containers_.keys.end(), __pair.first, __compare_);
     auto __mapped_it = __corresponding_mapped_it(*this, __key_it);
 
     return __flat_map_utils::__emplace_exact_pos(
@@ -473,7 +471,7 @@ class flat_multimap {
       //                   |
       //                  hint
       // We want to insert "2" after the last existing "2"
-      __key_iter    = ranges::upper_bound(__containers_.keys.begin(), __key_iter, __pair.first, __compare_);
+      __key_iter    = std::upper_bound(__containers_.keys.begin(), __key_iter, __pair.first, __compare_);
       __mapped_iter = __corresponding_mapped_it(*this, __key_iter);
     } else {
       _LIBCPP_ASSERT_INTERNAL(!__prev_larger && __next_smaller, "this means that the multimap is not sorted");
@@ -485,7 +483,7 @@ class flat_multimap {
       //  |
       // hint
       // We want to insert "2" before the first existing "2"
-      __key_iter    = ranges::lower_bound(__key_iter, __containers_.keys.end(), __pair.first, __compare_);
+      __key_iter    = std::lower_bound(__key_iter, __containers_.keys.end(), __pair.first, __compare_);
       __mapped_iter = __corresponding_mapped_it(*this, __key_iter);
     }
     return __flat_map_utils::__emplace_exact_pos(
@@ -804,7 +802,8 @@ class flat_multimap {
 
   template <class _Self, class _Kp>
   _LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
-    auto [__key_first, __key_last] = ranges::equal_range(__self.__containers_.keys, __key, __self.__compare_);
+    auto [__key_first, __key_last] =
+        std::equal_range(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __key, __self.__compare_);
 
     using __iterator_type = ranges::iterator_t<decltype(__self)>;
     return std::make_pair(__iterator_type(__key_first, __corresponding_mapped_it(__self, __key_first)),
@@ -813,24 +812,26 @@ class flat_multimap {
 
   template <class _Res, class _Self, class _Kp>
   _LIBCPP_HIDE_FROM_ABI static _Res __lower_bound(_Self&& __self, _Kp& __x) {
-    auto __key_iter    = ranges::lower_bound(__self.__containers_.keys, __x, __self.__compare_);
+    auto __key_iter =
+        std::lower_bound(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __x, __self.__compare_);
     auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
     return _Res(std::move(__key_iter), std::move(__mapped_iter));
   }
 
   template <class _Res, class _Self, class _Kp>
   _LIBCPP_HIDE_FROM_ABI static _Res __upper_bound(_Self&& __self, _Kp& __x) {
-    auto __key_iter    = ranges::upper_bound(__self.__containers_.keys, __x, __self.__compare_);
+    auto __key_iter =
+        std::upper_bound(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __x, __self.__compare_);
     auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
     return _Res(std::move(__key_iter), std::move(__mapped_iter));
   }
 
   _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
-    if constexpr (requires { __containers_.keys.reserve(__size); }) {
+    if constexpr (__container_traits<_KeyContainer>::__reservable) {
       __containers_.keys.reserve(__size);
     }
 
-    if constexpr (requires { __containers_.values.reserve(__size); }) {
+    if constexpr (__container_traits<_MappedContainer>::__reservable) {
       __containers_.values.reserve(__size);
     }
   }
diff --git a/external/llvm-project/libcxx/include/__flat_set/flat_multiset.h b/external/llvm-project/libcxx/include/__flat_set/flat_multiset.h
index 0fed377b25e5..44d8af05a56a 100644
--- a/external/llvm-project/libcxx/include/__flat_set/flat_multiset.h
+++ b/external/llvm-project/libcxx/include/__flat_set/flat_multiset.h
@@ -667,7 +667,7 @@ class flat_multiset {
   }
 
   _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
-    if constexpr (requires { __keys_.reserve(__size); }) {
+    if constexpr (__container_traits<_KeyContainer>::__reservable) {
       __keys_.reserve(__size);
     }
   }
diff --git a/external/llvm-project/libcxx/include/__flat_set/flat_set.h b/external/llvm-project/libcxx/include/__flat_set/flat_set.h
index a87496bb9916..b8edd8eee0cf 100644
--- a/external/llvm-project/libcxx/include/__flat_set/flat_set.h
+++ b/external/llvm-project/libcxx/include/__flat_set/flat_set.h
@@ -16,7 +16,6 @@
 #include <__algorithm/ranges_adjacent_find.h>
 #include <__algorithm/ranges_equal.h>
 #include <__algorithm/ranges_inplace_merge.h>
-#include <__algorithm/ranges_partition_point.h>
 #include <__algorithm/ranges_sort.h>
 #include <__algorithm/ranges_unique.h>
 #include <__algorithm/remove_if.h>
@@ -699,7 +698,7 @@ class flat_set {
   }
 
   _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
-    if constexpr (requires { __keys_.reserve(__size); }) {
+    if constexpr (__container_traits<_KeyContainer>::__reservable) {
       __keys_.reserve(__size);
     }
   }
diff --git a/external/llvm-project/libcxx/include/__format/format_arg_store.h b/external/llvm-project/libcxx/include/__format/format_arg_store.h
index 87557aa4da7b..fbb4cad21b23 100644
--- a/external/llvm-project/libcxx/include/__format/format_arg_store.h
+++ b/external/llvm-project/libcxx/include/__format/format_arg_store.h
@@ -14,7 +14,6 @@
 #  pragma GCC system_header
 #endif
 
-#include <__concepts/arithmetic.h>
 #include <__concepts/same_as.h>
 #include <__config>
 #include <__cstddef/size_t.h>
@@ -22,6 +21,7 @@
 #include <__format/format_arg.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/extent.h>
+#include <__type_traits/integer_traits.h>
 #include <__type_traits/remove_const.h>
 #include <cstdint>
 #include <string>
@@ -65,7 +65,7 @@ consteval __arg_t __determine_arg_t() {
 #  endif
 
 // Signed integers
-template <class, __libcpp_signed_integer _Tp>
+template <class, __signed_integer _Tp>
 consteval __arg_t __determine_arg_t() {
   if constexpr (sizeof(_Tp) <= sizeof(int))
     return __arg_t::__int;
@@ -80,7 +80,7 @@ consteval __arg_t __determine_arg_t() {
 }
 
 // Unsigned integers
-template <class, __libcpp_unsigned_integer _Tp>
+template <class, __unsigned_integer _Tp>
 consteval __arg_t __determine_arg_t() {
   if constexpr (sizeof(_Tp) <= sizeof(unsigned))
     return __arg_t::__unsigned;
diff --git a/external/llvm-project/libcxx/include/__functional/function.h b/external/llvm-project/libcxx/include/__functional/function.h
index e33c5ab9b886..e71c778386fd 100644
--- a/external/llvm-project/libcxx/include/__functional/function.h
+++ b/external/llvm-project/libcxx/include/__functional/function.h
@@ -17,13 +17,7 @@
 #include <__functional/binary_function.h>
 #include <__functional/invoke.h>
 #include <__functional/unary_function.h>
-#include <__iterator/iterator_traits.h>
 #include <__memory/addressof.h>
-#include <__memory/allocator.h>
-#include <__memory/allocator_destructor.h>
-#include <__memory/allocator_traits.h>
-#include <__memory/compressed_pair.h>
-#include <__memory/unique_ptr.h>
 #include <__type_traits/aligned_storage.h>
 #include <__type_traits/decay.h>
 #include <__type_traits/is_core_convertible.h>
@@ -34,9 +28,7 @@
 #include <__type_traits/strip_signature.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
-#include <__utility/piecewise_construct.h>
 #include <__utility/swap.h>
-#include <__verbose_abort>
 #include <tuple>
 #include <typeinfo>
 
@@ -122,7 +114,7 @@ _LIBCPP_HIDE_FROM_ABI bool __not_null(function<_Fp> const& __f) {
   return !!__f;
 }
 
-#  if _LIBCPP_HAS_EXTENSION_BLOCKS
+#  if __has_extension(blocks)
 template <class _Rp, class... _Args>
 _LIBCPP_HIDE_FROM_ABI bool __not_null(_Rp (^__p)(_Args...)) {
   return __p;
@@ -133,71 +125,9 @@ _LIBCPP_HIDE_FROM_ABI bool __not_null(_Rp (^__p)(_Args...)) {
 
 namespace __function {
 
-// __alloc_func holds a functor and an allocator.
-
-template <class _Fp, class _Ap, class _FB>
-class __alloc_func;
 template <class _Fp, class _FB>
 class __default_alloc_func;
 
-template <class _Fp, class _Ap, class _Rp, class... _ArgTypes>
-class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> {
-  _LIBCPP_COMPRESSED_PAIR(_Fp, __func_, _Ap, __alloc_);
-
-public:
-  using _Target _LIBCPP_NODEBUG = _Fp;
-  using _Alloc _LIBCPP_NODEBUG  = _Ap;
-
-  _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __func_; }
-
-  // WIN32 APIs may define __allocator, so use __get_allocator instead.
-  _LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __alloc_; }
-
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f) : __func_(std::move(__f)), __alloc_() {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a) : __func_(__f), __alloc_(__a) {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, _Alloc&& __a)
-      : __func_(__f), __alloc_(std::move(__a)) {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f, _Alloc&& __a)
-      : __func_(std::move(__f)), __alloc_(std::move(__a)) {}
-
-  _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) {
-    return std::__invoke_r<_Rp>(__func_, std::forward<_ArgTypes>(__arg)...);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI __alloc_func* __clone() const {
-    typedef allocator_traits<_Alloc> __alloc_traits;
-    typedef __rebind_alloc<__alloc_traits, __alloc_func> _AA;
-    _AA __a(__alloc_);
-    typedef __allocator_destructor<_AA> _Dp;
-    unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
-    ::new ((void*)__hold.get()) __alloc_func(__func_, _Alloc(__a));
-    return __hold.release();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT {
-    __func_.~_Fp();
-    __alloc_.~_Alloc();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__alloc_func* __f) {
-    typedef allocator_traits<_Alloc> __alloc_traits;
-    typedef __rebind_alloc<__alloc_traits, __alloc_func> _FunAlloc;
-    _FunAlloc __a(__f->__get_allocator());
-    __f->destroy();
-    __a.deallocate(__f, 1);
-  }
-};
-
-template <class _Tp>
-struct __deallocating_deleter {
-  _LIBCPP_HIDE_FROM_ABI void operator()(void* __p) const {
-    std::__libcpp_deallocate<_Tp>(static_cast<_Tp*>(__p), __element_count(1));
-  }
-};
-
 template <class _Fp, class _Rp, class... _ArgTypes>
 class __default_alloc_func<_Fp, _Rp(_ArgTypes...)> {
   _Fp __f_;
@@ -215,20 +145,9 @@ class __default_alloc_func<_Fp, _Rp(_ArgTypes...)> {
     return std::__invoke_r<_Rp>(__f_, std::forward<_ArgTypes>(__arg)...);
   }
 
-  _LIBCPP_HIDE_FROM_ABI __default_alloc_func* __clone() const {
-    using _Self = __default_alloc_func;
-    unique_ptr<_Self, __deallocating_deleter<_Self>> __hold(std::__libcpp_allocate<_Self>(__element_count(1)));
-    _Self* __res = ::new ((void*)__hold.get()) _Self(__f_);
-    (void)__hold.release();
-    return __res;
-  }
+  _LIBCPP_HIDE_FROM_ABI __default_alloc_func* __clone() const { return new __default_alloc_func(__f_); }
 
   _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~_Target(); }
-
-  _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__default_alloc_func* __f) {
-    __f->destroy();
-    std::__libcpp_deallocate<__default_alloc_func>(__f, __element_count(1));
-  }
 };
 
 // __base provides an abstract interface for copyable functors.
@@ -257,84 +176,38 @@ class __base<_Rp(_ArgTypes...)> {
 
 // __func implements __base for a given functor type.
 
-template <class _FD, class _Alloc, class _FB>
+template <class _FD, class _FB>
 class __func;
 
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-class __func<_Fp, _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> {
-  __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> __f_;
+template <class _Fp, class _Rp, class... _ArgTypes>
+class __func<_Fp, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> {
+  _Fp __func_;
 
 public:
-  _LIBCPP_HIDE_FROM_ABI explicit __func(_Fp&& __f) : __f_(std::move(__f)) {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit __func(const _Fp& __f, const _Alloc& __a) : __f_(__f, __a) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __func(_Fp&& __f) : __func_(std::move(__f)) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __func(const _Fp& __f) : __func_(__f) {}
 
-  _LIBCPP_HIDE_FROM_ABI explicit __func(const _Fp& __f, _Alloc&& __a) : __f_(__f, std::move(__a)) {}
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL __base<_Rp(_ArgTypes...)>* __clone() const override { return new __func(__func_); }
 
-  _LIBCPP_HIDE_FROM_ABI explicit __func(_Fp&& __f, _Alloc&& __a) : __f_(std::move(__f), std::move(__a)) {}
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __clone(__base<_Rp(_ArgTypes...)>* __p) const override {
+    ::new ((void*)__p) __func(__func_);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual __base<_Rp(_ArgTypes...)>* __clone() const;
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void __clone(__base<_Rp(_ArgTypes...)>*) const;
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT;
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy_deallocate() _NOEXCEPT;
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual _Rp operator()(_ArgTypes&&... __arg);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void destroy() _NOEXCEPT override { __func_.~_Fp(); }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void destroy_deallocate() _NOEXCEPT override { delete this; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL _Rp operator()(_ArgTypes&&... __arg) override {
+    return std::__invoke_r<_Rp>(__func_, std::forward<_ArgTypes>(__arg)...);
+  }
 #  if _LIBCPP_HAS_RTTI
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const void* target(const type_info&) const _NOEXCEPT;
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const std::type_info& target_type() const _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const void* target(const type_info& __ti) const _NOEXCEPT override {
+    if (__ti == typeid(_Fp))
+      return std::addressof(__func_);
+    return nullptr;
+  }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const std::type_info& target_type() const _NOEXCEPT override { return typeid(_Fp); }
 #  endif // _LIBCPP_HAS_RTTI
 };
 
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-__base<_Rp(_ArgTypes...)>* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone() const {
-  typedef allocator_traits<_Alloc> __alloc_traits;
-  typedef __rebind_alloc<__alloc_traits, __func> _Ap;
-  _Ap __a(__f_.__get_allocator());
-  typedef __allocator_destructor<_Ap> _Dp;
-  unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
-  ::new ((void*)__hold.get()) __func(__f_.__target(), _Alloc(__a));
-  return __hold.release();
-}
-
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__base<_Rp(_ArgTypes...)>* __p) const {
-  ::new ((void*)__p) __func(__f_.__target(), __f_.__get_allocator());
-}
-
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT {
-  __f_.destroy();
-}
-
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() _NOEXCEPT {
-  typedef allocator_traits<_Alloc> __alloc_traits;
-  typedef __rebind_alloc<__alloc_traits, __func> _Ap;
-  _Ap __a(__f_.__get_allocator());
-  __f_.destroy();
-  __a.deallocate(this, 1);
-}
-
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-_Rp __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&&... __arg) {
-  return __f_(std::forward<_ArgTypes>(__arg)...);
-}
-
-#  if _LIBCPP_HAS_RTTI
-
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-const void* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT {
-  if (__ti == typeid(_Fp))
-    return std::addressof(__f_.__target());
-  return nullptr;
-}
-
-template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
-const std::type_info& __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() const _NOEXCEPT {
-  return typeid(_Fp);
-}
-
-#  endif // _LIBCPP_HAS_RTTI
-
 // __value_func creates a value-type from a __func.
 
 template <class _Fp>
@@ -354,29 +227,19 @@ class __value_func<_Rp(_ArgTypes...)> {
 public:
   _LIBCPP_HIDE_FROM_ABI __value_func() _NOEXCEPT : __f_(nullptr) {}
 
-  template <class _Fp, class _Alloc>
-  _LIBCPP_HIDE_FROM_ABI __value_func(_Fp&& __f, const _Alloc& __a) : __f_(nullptr) {
-    typedef allocator_traits<_Alloc> __alloc_traits;
-    typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun;
-    typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc;
+  template <class _Fp, __enable_if_t<!is_same<__decay_t<_Fp>, __value_func>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI explicit __value_func(_Fp&& __f) : __f_(nullptr) {
+    typedef __function::__func<_Fp, _Rp(_ArgTypes...)> _Fun;
 
     if (__function::__not_null(__f)) {
-      _FunAlloc __af(__a);
-      if (sizeof(_Fun) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value &&
-          is_nothrow_copy_constructible<_FunAlloc>::value) {
-        __f_ = ::new ((void*)&__buf_) _Fun(std::move(__f), _Alloc(__af));
+      if (sizeof(_Fun) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value) {
+        __f_ = ::new (std::addressof(__buf_)) _Fun(std::move(__f));
       } else {
-        typedef __allocator_destructor<_FunAlloc> _Dp;
-        unique_ptr<__func, _Dp> __hold(__af.allocate(1), _Dp(__af, 1));
-        ::new ((void*)__hold.get()) _Fun(std::move(__f), _Alloc(__a));
-        __f_ = __hold.release();
+        __f_ = new _Fun(std::move(__f));
       }
     }
   }
 
-  template <class _Fp, __enable_if_t<!is_same<__decay_t<_Fp>, __value_func>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI explicit __value_func(_Fp&& __f) : __value_func(std::forward<_Fp>(__f), allocator<_Fp>()) {}
-
   _LIBCPP_HIDE_FROM_ABI __value_func(const __value_func& __f) {
     if (__f.__f_ == nullptr)
       __f_ = nullptr;
@@ -544,7 +407,7 @@ struct __policy {
 
   template <typename _Fun>
   _LIBCPP_HIDE_FROM_ABI static void __large_destroy(void* __s) {
-    _Fun::__destroy_and_delete(static_cast<_Fun*>(__s));
+    delete static_cast<_Fun*>(__s);
   }
 
   template <typename _Fun>
@@ -583,7 +446,7 @@ struct __policy {
 template <typename _Tp>
 using __fast_forward _LIBCPP_NODEBUG = __conditional_t<is_scalar<_Tp>::value, _Tp, _Tp&&>;
 
-// __policy_invoker calls an instance of __alloc_func held in __policy_storage.
+// __policy_invoker calls an instance of __default_alloc_func held in __policy_storage.
 
 template <class _Fp>
 struct __policy_invoker;
@@ -641,28 +504,6 @@ class __policy_func<_Rp(_ArgTypes...)> {
 public:
   _LIBCPP_HIDE_FROM_ABI __policy_func() : __policy_(__policy::__create_empty()) {}
 
-  template <class _Fp, class _Alloc>
-  _LIBCPP_HIDE_FROM_ABI __policy_func(_Fp&& __f, const _Alloc& __a) : __policy_(__policy::__create_empty()) {
-    typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun;
-    typedef allocator_traits<_Alloc> __alloc_traits;
-    typedef __rebind_alloc<__alloc_traits, _Fun> _FunAlloc;
-
-    if (__function::__not_null(__f)) {
-      __invoker_ = __invoker::template __create<_Fun>();
-      __policy_  = __policy::__create<_Fun>();
-
-      _FunAlloc __af(__a);
-      if (__use_small_storage<_Fun>()) {
-        ::new ((void*)&__buf_.__small) _Fun(std::move(__f), _Alloc(__af));
-      } else {
-        typedef __allocator_destructor<_FunAlloc> _Dp;
-        unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1));
-        ::new ((void*)__hold.get()) _Fun(std::move(__f), _Alloc(__af));
-        __buf_.__large = __hold.release();
-      }
-    }
-  }
-
   template <class _Fp, __enable_if_t<!is_same<__decay_t<_Fp>, __policy_func>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI explicit __policy_func(_Fp&& __f) : __policy_(__policy::__create_empty()) {
     typedef __default_alloc_func<_Fp, _Rp(_ArgTypes...)> _Fun;
@@ -673,9 +514,7 @@ class __policy_func<_Rp(_ArgTypes...)> {
       if (__use_small_storage<_Fun>()) {
         ::new ((void*)&__buf_.__small) _Fun(std::move(__f));
       } else {
-        unique_ptr<_Fun, __deallocating_deleter<_Fun>> __hold(std::__libcpp_allocate<_Fun>(__element_count(1)));
-        __buf_.__large = ::new ((void*)__hold.get()) _Fun(std::move(__f));
-        (void)__hold.release();
+        __buf_.__large = ::new _Fun(std::move(__f));
       }
     }
   }
@@ -750,14 +589,14 @@ class __policy_func<_Rp(_ArgTypes...)> {
 extern "C" void* _Block_copy(const void*);
 extern "C" void _Block_release(const void*);
 
-template <class _Rp1, class... _ArgTypes1, class _Alloc, class _Rp, class... _ArgTypes>
-class __func<_Rp1 (^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> {
+template <class _Rp1, class... _ArgTypes1, class _Rp, class... _ArgTypes>
+class __func<_Rp1 (^)(_ArgTypes1...), _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> {
   typedef _Rp1 (^__block_type)(_ArgTypes1...);
   __block_type __f_;
 
 public:
   _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type const& __f)
-#    if _LIBCPP_HAS_OBJC_ARC
+#    if __has_feature(objc_arc)
       : __f_(__f)
 #    else
       : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
@@ -767,15 +606,6 @@ class __func<_Rp1 (^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> : public __base
 
   // [TODO] add && to save on a retain
 
-  _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type __f, const _Alloc& /* unused */)
-#    if _LIBCPP_HAS_OBJC_ARC
-      : __f_(__f)
-#    else
-      : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
-#    endif
-  {
-  }
-
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual __base<_Rp(_ArgTypes...)>* __clone() const {
     _LIBCPP_ASSERT_INTERNAL(
         false,
@@ -790,7 +620,7 @@ class __func<_Rp1 (^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> : public __base
   }
 
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT {
-#    if !_LIBCPP_HAS_OBJC_ARC
+#    if !__has_feature(objc_arc)
     if (__f_)
       _Block_release(__f_);
 #    endif
@@ -822,7 +652,7 @@ class __func<_Rp1 (^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> : public __base
 #    endif // _LIBCPP_HAS_RTTI
 };
 
-#  endif // _LIBCPP_HAS_EXTENSION_BLOCKS
+#  endif // _LIBCPP_HAS_BLOCKS_RUNTIME
 
 } // namespace __function
 
@@ -954,7 +784,7 @@ function<_Rp(_ArgTypes...)>::function(_Fp __f) : __f_(std::move(__f)) {}
 #  if _LIBCPP_STD_VER <= 14
 template <class _Rp, class... _ArgTypes>
 template <class _Fp, class _Alloc, class>
-function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a, _Fp __f) : __f_(std::move(__f), __a) {}
+function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, _Fp __f) : __f_(std::move(__f)) {}
 #  endif
 
 template <class _Rp, class... _ArgTypes>
diff --git a/external/llvm-project/libcxx/include/__functional/hash.h b/external/llvm-project/libcxx/include/__functional/hash.h
index f9f7d2c767ca..489a6f00b8a3 100644
--- a/external/llvm-project/libcxx/include/__functional/hash.h
+++ b/external/llvm-project/libcxx/include/__functional/hash.h
@@ -19,6 +19,8 @@
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_constructible.h>
 #include <__type_traits/is_enum.h>
+#include <__type_traits/is_floating_point.h>
+#include <__type_traits/is_integral.h>
 #include <__type_traits/underlying_type.h>
 #include <__utility/pair.h>
 #include <__utility/swap.h>
@@ -345,122 +347,43 @@ struct hash<_Tp*> : public __unary_function<_Tp*, size_t> {
   }
 };
 
-template <>
-struct hash<bool> : public __unary_function<bool, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(bool __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<char> : public __unary_function<char, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(char __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<signed char> : public __unary_function<signed char, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(signed char __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<unsigned char> : public __unary_function<unsigned char, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned char __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-#if _LIBCPP_HAS_CHAR8_T
-template <>
-struct hash<char8_t> : public __unary_function<char8_t, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(char8_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-#endif // _LIBCPP_HAS_CHAR8_T
-
-template <>
-struct hash<char16_t> : public __unary_function<char16_t, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(char16_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<char32_t> : public __unary_function<char32_t, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(char32_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-#if _LIBCPP_HAS_WIDE_CHARACTERS
-template <>
-struct hash<wchar_t> : public __unary_function<wchar_t, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(wchar_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-#endif // _LIBCPP_HAS_WIDE_CHARACTERS
-
-template <>
-struct hash<short> : public __unary_function<short, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(short __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<unsigned short> : public __unary_function<unsigned short, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned short __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+template <class _Tp, class = void>
+struct __hash_impl {
+  __hash_impl()                              = delete;
+  __hash_impl(__hash_impl const&)            = delete;
+  __hash_impl& operator=(__hash_impl const&) = delete;
 };
 
-template <>
-struct hash<int> : public __unary_function<int, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(int __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<unsigned int> : public __unary_function<unsigned int, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned int __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<long> : public __unary_function<long, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(long __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
-};
-
-template <>
-struct hash<unsigned long> : public __unary_function<unsigned long, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned long __v) const _NOEXCEPT {
-    static_assert(sizeof(size_t) >= sizeof(unsigned long),
-                  "This would be a terrible hash function on a platform where size_t is smaller than unsigned long");
-    return static_cast<size_t>(__v);
+template <class _Tp>
+struct __hash_impl<_Tp, __enable_if_t<is_enum<_Tp>::value> > : __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
+    using type = __underlying_type_t<_Tp>;
+    return hash<type>()(static_cast<type>(__v));
   }
 };
 
-template <>
-struct hash<long long> : public __scalar_hash<long long> {};
-
-template <>
-struct hash<unsigned long long> : public __scalar_hash<unsigned long long> {};
-
-#if _LIBCPP_HAS_INT128
-
-template <>
-struct hash<__int128_t> : public __scalar_hash<__int128_t> {};
-
-template <>
-struct hash<__uint128_t> : public __scalar_hash<__uint128_t> {};
+template <class _Tp>
+struct __hash_impl<_Tp, __enable_if_t<is_integral<_Tp>::value && (sizeof(_Tp) <= sizeof(size_t))> >
+    : __unary_function<_Tp, size_t> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
+};
 
-#endif
+template <class _Tp>
+struct __hash_impl<_Tp, __enable_if_t<is_integral<_Tp>::value && (sizeof(_Tp) > sizeof(size_t))> >
+    : __scalar_hash<_Tp> {};
 
-template <>
-struct hash<float> : public __scalar_hash<float> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(float __v) const _NOEXCEPT {
+template <class _Tp>
+struct __hash_impl<_Tp, __enable_if_t<is_floating_point<_Tp>::value> > : __scalar_hash<_Tp> {
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
     // -0.0 and 0.0 should return same hash
     if (__v == 0.0f)
       return 0;
-    return __scalar_hash<float>::operator()(__v);
-  }
-};
-
-template <>
-struct hash<double> : public __scalar_hash<double> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(double __v) const _NOEXCEPT {
-    // -0.0 and 0.0 should return same hash
-    if (__v == 0.0)
-      return 0;
-    return __scalar_hash<double>::operator()(__v);
+    return __scalar_hash<_Tp>::operator()(__v);
   }
 };
 
 template <>
-struct hash<long double> : public __scalar_hash<long double> {
+struct __hash_impl<long double> : __scalar_hash<long double> {
   _LIBCPP_HIDE_FROM_ABI size_t operator()(long double __v) const _NOEXCEPT {
     // -0.0 and 0.0 should return same hash
     if (__v == 0.0L)
@@ -501,22 +424,8 @@ struct hash<long double> : public __scalar_hash<long double> {
   }
 };
 
-template <class _Tp, bool = is_enum<_Tp>::value>
-struct __enum_hash : public __unary_function<_Tp, size_t> {
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(_Tp __v) const _NOEXCEPT {
-    using type = __underlying_type_t<_Tp>;
-    return hash<type>()(static_cast<type>(__v));
-  }
-};
-template <class _Tp>
-struct __enum_hash<_Tp, false> {
-  __enum_hash()                              = delete;
-  __enum_hash(__enum_hash const&)            = delete;
-  __enum_hash& operator=(__enum_hash const&) = delete;
-};
-
 template <class _Tp>
-struct hash : public __enum_hash<_Tp> {};
+struct hash : public __hash_impl<_Tp> {};
 
 #if _LIBCPP_STD_VER >= 17
 
diff --git a/external/llvm-project/libcxx/include/__functional/reference_wrapper.h b/external/llvm-project/libcxx/include/__functional/reference_wrapper.h
index b409ad7511f6..c46203a4ca9a 100644
--- a/external/llvm-project/libcxx/include/__functional/reference_wrapper.h
+++ b/external/llvm-project/libcxx/include/__functional/reference_wrapper.h
@@ -11,7 +11,6 @@
 #define _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
 
 #include <__compare/synth_three_way.h>
-#include <__concepts/boolean_testable.h>
 #include <__config>
 #include <__functional/weak_result_type.h>
 #include <__memory/addressof.h>
@@ -19,6 +18,7 @@
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_const.h>
+#include <__type_traits/is_core_convertible.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
@@ -75,7 +75,7 @@ class reference_wrapper : public __weak_result_type<_Tp> {
 
   _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, reference_wrapper __y)
     requires requires {
-      { __x.get() == __y.get() } -> __boolean_testable;
+      { __x.get() == __y.get() } -> __core_convertible_to<bool>;
     }
   {
     return __x.get() == __y.get();
@@ -83,7 +83,7 @@ class reference_wrapper : public __weak_result_type<_Tp> {
 
   _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, const _Tp& __y)
     requires requires {
-      { __x.get() == __y } -> __boolean_testable;
+      { __x.get() == __y } -> __core_convertible_to<bool>;
     }
   {
     return __x.get() == __y;
@@ -91,7 +91,7 @@ class reference_wrapper : public __weak_result_type<_Tp> {
 
   _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, reference_wrapper<const _Tp> __y)
     requires(!is_const_v<_Tp>) && requires {
-      { __x.get() == __y.get() } -> __boolean_testable;
+      { __x.get() == __y.get() } -> __core_convertible_to<bool>;
     }
   {
     return __x.get() == __y.get();
diff --git a/external/llvm-project/libcxx/include/__fwd/variant.h b/external/llvm-project/libcxx/include/__fwd/variant.h
index 3808268df174..fef7071b3264 100644
--- a/external/llvm-project/libcxx/include/__fwd/variant.h
+++ b/external/llvm-project/libcxx/include/__fwd/variant.h
@@ -38,37 +38,28 @@ using variant_alternative_t = typename variant_alternative<_Ip, _Tp>::type;
 inline constexpr size_t variant_npos = static_cast<size_t>(-1);
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>&
-get(variant<_Types...>&);
+_LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>& get(variant<_Types...>&);
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>&&
-get(variant<_Types...>&&);
+_LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>&& get(variant<_Types...>&&);
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>&
-get(const variant<_Types...>&);
+_LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>& get(const variant<_Types...>&);
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>&&
-get(const variant<_Types...>&&);
+_LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>&& get(const variant<_Types...>&&);
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp& get(variant<_Types...>&);
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp& get(variant<_Types...>&);
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp&& get(variant<_Types...>&&);
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp&& get(variant<_Types...>&&);
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp& get(const variant<_Types...>&);
+_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& get(const variant<_Types...>&);
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp&&
-get(const variant<_Types...>&&);
+_LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& get(const variant<_Types...>&&);
 
 #endif // _LIBCPP_STD_VER >= 17
 
diff --git a/external/llvm-project/libcxx/include/__iterator/concepts.h b/external/llvm-project/libcxx/include/__iterator/concepts.h
index ddfc84ed710c..20a1ab469110 100644
--- a/external/llvm-project/libcxx/include/__iterator/concepts.h
+++ b/external/llvm-project/libcxx/include/__iterator/concepts.h
@@ -29,16 +29,19 @@
 #include <__iterator/incrementable_traits.h>
 #include <__iterator/iter_move.h>
 #include <__iterator/iterator_traits.h>
-#include <__iterator/readable_traits.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/add_pointer.h>
 #include <__type_traits/common_reference.h>
+#include <__type_traits/conditional.h>
+#include <__type_traits/disjunction.h>
+#include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_pointer.h>
 #include <__type_traits/is_primary_template.h>
 #include <__type_traits/is_reference.h>
 #include <__type_traits/is_referenceable.h>
+#include <__type_traits/is_valid_expansion.h>
 #include <__type_traits/remove_cv.h>
 #include <__type_traits/remove_cvref.h>
 #include <__utility/forward.h>
@@ -151,6 +154,42 @@ concept sized_sentinel_for =
       { __i - __s } -> same_as<iter_difference_t<_Ip>>;
     };
 
+template <class _Iter>
+struct __iter_traits_cache {
+  using type _LIBCPP_NODEBUG =
+      _If<__is_primary_template<iterator_traits<_Iter> >::value, _Iter, iterator_traits<_Iter> >;
+};
+template <class _Iter>
+using _ITER_TRAITS _LIBCPP_NODEBUG = typename __iter_traits_cache<_Iter>::type;
+
+struct __iter_concept_concept_test {
+  template <class _Iter>
+  using _Apply _LIBCPP_NODEBUG = typename _ITER_TRAITS<_Iter>::iterator_concept;
+};
+struct __iter_concept_category_test {
+  template <class _Iter>
+  using _Apply _LIBCPP_NODEBUG = typename _ITER_TRAITS<_Iter>::iterator_category;
+};
+struct __iter_concept_random_fallback {
+  template <class _Iter>
+  using _Apply _LIBCPP_NODEBUG =
+      __enable_if_t<__is_primary_template<iterator_traits<_Iter> >::value, random_access_iterator_tag>;
+};
+
+template <class _Iter, class _Tester>
+struct __test_iter_concept : _IsValidExpansion<_Tester::template _Apply, _Iter>, _Tester {};
+
+template <class _Iter>
+struct __iter_concept_cache {
+  using type _LIBCPP_NODEBUG =
+      _Or<__test_iter_concept<_Iter, __iter_concept_concept_test>,
+          __test_iter_concept<_Iter, __iter_concept_category_test>,
+          __test_iter_concept<_Iter, __iter_concept_random_fallback> >;
+};
+
+template <class _Iter>
+using _ITER_CONCEPT _LIBCPP_NODEBUG = typename __iter_concept_cache<_Iter>::type::template _Apply<_Iter>;
+
 // [iterator.concept.input]
 template <class _Ip>
 concept input_iterator = input_or_output_iterator<_Ip> && indirectly_readable<_Ip> && requires {
diff --git a/external/llvm-project/libcxx/include/__iterator/iterator_traits.h b/external/llvm-project/libcxx/include/__iterator/iterator_traits.h
index 8256580acb29..221d36614db0 100644
--- a/external/llvm-project/libcxx/include/__iterator/iterator_traits.h
+++ b/external/llvm-project/libcxx/include/__iterator/iterator_traits.h
@@ -27,14 +27,12 @@
 #include <__type_traits/conditional.h>
 #include <__type_traits/detected_or.h>
 #include <__type_traits/disjunction.h>
-#include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_convertible.h>
 #include <__type_traits/is_object.h>
 #include <__type_traits/is_primary_template.h>
 #include <__type_traits/is_reference.h>
 #include <__type_traits/is_referenceable.h>
-#include <__type_traits/is_valid_expansion.h>
 #include <__type_traits/nat.h>
 #include <__type_traits/remove_const.h>
 #include <__type_traits/remove_cv.h>
@@ -73,42 +71,6 @@ struct random_access_iterator_tag : public bidirectional_iterator_tag {};
 struct contiguous_iterator_tag : public random_access_iterator_tag {};
 #endif
 
-template <class _Iter>
-struct __iter_traits_cache {
-  using type _LIBCPP_NODEBUG =
-      _If<__is_primary_template<iterator_traits<_Iter> >::value, _Iter, iterator_traits<_Iter> >;
-};
-template <class _Iter>
-using _ITER_TRAITS _LIBCPP_NODEBUG = typename __iter_traits_cache<_Iter>::type;
-
-struct __iter_concept_concept_test {
-  template <class _Iter>
-  using _Apply _LIBCPP_NODEBUG = typename _ITER_TRAITS<_Iter>::iterator_concept;
-};
-struct __iter_concept_category_test {
-  template <class _Iter>
-  using _Apply _LIBCPP_NODEBUG = typename _ITER_TRAITS<_Iter>::iterator_category;
-};
-struct __iter_concept_random_fallback {
-  template <class _Iter>
-  using _Apply _LIBCPP_NODEBUG =
-      __enable_if_t<__is_primary_template<iterator_traits<_Iter> >::value, random_access_iterator_tag>;
-};
-
-template <class _Iter, class _Tester>
-struct __test_iter_concept : _IsValidExpansion<_Tester::template _Apply, _Iter>, _Tester {};
-
-template <class _Iter>
-struct __iter_concept_cache {
-  using type _LIBCPP_NODEBUG =
-      _Or<__test_iter_concept<_Iter, __iter_concept_concept_test>,
-          __test_iter_concept<_Iter, __iter_concept_category_test>,
-          __test_iter_concept<_Iter, __iter_concept_random_fallback> >;
-};
-
-template <class _Iter>
-using _ITER_CONCEPT _LIBCPP_NODEBUG = typename __iter_concept_cache<_Iter>::type::template _Apply<_Iter>;
-
 template <class _Tp>
 struct __has_iterator_typedefs {
 private:
@@ -195,16 +157,6 @@ concept __specifies_members = requires {
   requires __has_member_iterator_category<_Ip>;
 };
 
-template <class>
-struct __iterator_traits_member_pointer_or_void {
-  using type _LIBCPP_NODEBUG = void;
-};
-
-template <__has_member_pointer _Tp>
-struct __iterator_traits_member_pointer_or_void<_Tp> {
-  using type _LIBCPP_NODEBUG = typename _Tp::pointer;
-};
-
 template <class _Tp>
 concept __cpp17_iterator_missing_members = !__specifies_members<_Tp> && __iterator_traits_detail::__cpp17_iterator<_Tp>;
 
@@ -304,6 +256,9 @@ struct __iterator_traits_difference_type<_Ip> {
 template <class>
 struct __iterator_traits {};
 
+template <class _Tp>
+using __pointer_member _LIBCPP_NODEBUG = typename _Tp::pointer;
+
 // [iterator.traits]/3.1
 // If `I` has valid ([temp.deduct]) member types `difference-type`, `value-type`, `reference`, and
 // `iterator-category`, then `iterator-traits<I>` has the following publicly accessible members:
@@ -312,7 +267,7 @@ struct __iterator_traits<_Ip> {
   using iterator_category = typename _Ip::iterator_category;
   using value_type        = typename _Ip::value_type;
   using difference_type   = typename _Ip::difference_type;
-  using pointer           = typename __iterator_traits_member_pointer_or_void<_Ip>::type;
+  using pointer           = __detected_or_t<void, __pointer_member, _Ip>;
   using reference         = typename _Ip::reference;
 };
 
diff --git a/external/llvm-project/libcxx/include/__mdspan/extents.h b/external/llvm-project/libcxx/include/__mdspan/extents.h
index 00454004851d..99b54badf893 100644
--- a/external/llvm-project/libcxx/include/__mdspan/extents.h
+++ b/external/llvm-project/libcxx/include/__mdspan/extents.h
@@ -21,11 +21,10 @@
 #include <__config>
 
 #include <__concepts/arithmetic.h>
-#include <__cstddef/byte.h>
 #include <__type_traits/common_type.h>
+#include <__type_traits/integer_traits.h>
 #include <__type_traits/is_convertible.h>
 #include <__type_traits/is_nothrow_constructible.h>
-#include <__type_traits/is_same.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/integer_sequence.h>
 #include <__utility/unreachable.h>
@@ -283,7 +282,8 @@ class extents {
   using size_type  = make_unsigned_t<index_type>;
   using rank_type  = size_t;
 
-  static_assert(__libcpp_integer<index_type>, "extents::index_type must be a signed or unsigned integer type");
+  static_assert(__signed_or_unsigned_integer<index_type>,
+                "extents::index_type must be a signed or unsigned integer type");
   static_assert(((__mdspan_detail::__is_representable_as<index_type>(_Extents) || (_Extents == dynamic_extent)) && ...),
                 "extents ctor: arguments must be representable as index_type and nonnegative");
 
diff --git a/external/llvm-project/libcxx/include/__memory/addressof.h b/external/llvm-project/libcxx/include/__memory/addressof.h
index 98b08958a6a9..667071dfc663 100644
--- a/external/llvm-project/libcxx/include/__memory/addressof.h
+++ b/external/llvm-project/libcxx/include/__memory/addressof.h
@@ -23,7 +23,7 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_NO_CFI _LIBCPP_HIDE_FROM_ABI _Tp* a
   return __builtin_addressof(__x);
 }
 
-#if _LIBCPP_HAS_OBJC_ARC
+#if __has_feature(objc_arc)
 // Objective-C++ Automatic Reference Counting uses qualified pointers
 // that require special addressof() signatures.
 template <class _Tp>
@@ -31,7 +31,7 @@ inline _LIBCPP_HIDE_FROM_ABI __strong _Tp* addressof(__strong _Tp& __x) _NOEXCEP
   return &__x;
 }
 
-#  if _LIBCPP_HAS_OBJC_ARC_WEAK
+#  if __has_feature(objc_arc_weak)
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI __weak _Tp* addressof(__weak _Tp& __x) _NOEXCEPT {
   return &__x;
diff --git a/external/llvm-project/libcxx/include/__memory/allocation_guard.h b/external/llvm-project/libcxx/include/__memory/allocation_guard.h
index 66edcd92ed61..016e1a3a429b 100644
--- a/external/llvm-project/libcxx/include/__memory/allocation_guard.h
+++ b/external/llvm-project/libcxx/include/__memory/allocation_guard.h
@@ -49,24 +49,26 @@ struct __allocation_guard {
   using _Size _LIBCPP_NODEBUG    = typename allocator_traits<_Alloc>::size_type;
 
   template <class _AllocT> // we perform the allocator conversion inside the constructor
-  _LIBCPP_HIDE_FROM_ABI explicit __allocation_guard(_AllocT __alloc, _Size __n)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __allocation_guard(_AllocT __alloc, _Size __n)
       : __alloc_(std::move(__alloc)),
         __n_(__n),
         __ptr_(allocator_traits<_Alloc>::allocate(__alloc_, __n_)) // initialization order is important
   {}
 
-  _LIBCPP_HIDE_FROM_ABI ~__allocation_guard() _NOEXCEPT { __destroy(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI ~__allocation_guard() _NOEXCEPT { __destroy(); }
 
-  _LIBCPP_HIDE_FROM_ABI __allocation_guard(const __allocation_guard&) = delete;
-  _LIBCPP_HIDE_FROM_ABI __allocation_guard(__allocation_guard&& __other) _NOEXCEPT
+  __allocation_guard(const __allocation_guard&)                    = delete;
+  __allocation_guard& operator=(const __allocation_guard& __other) = delete;
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __allocation_guard(__allocation_guard&& __other) _NOEXCEPT
       : __alloc_(std::move(__other.__alloc_)),
         __n_(__other.__n_),
         __ptr_(__other.__ptr_) {
     __other.__ptr_ = nullptr;
   }
 
-  _LIBCPP_HIDE_FROM_ABI __allocation_guard& operator=(const __allocation_guard& __other) = delete;
-  _LIBCPP_HIDE_FROM_ABI __allocation_guard& operator=(__allocation_guard&& __other) _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __allocation_guard&
+  operator=(__allocation_guard&& __other) _NOEXCEPT {
     if (std::addressof(__other) != this) {
       __destroy();
 
@@ -79,17 +81,17 @@ struct __allocation_guard {
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI _Pointer
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI _Pointer
   __release_ptr() _NOEXCEPT { // not called __release() because it's a keyword in objective-c++
     _Pointer __tmp = __ptr_;
     __ptr_         = nullptr;
     return __tmp;
   }
 
-  _LIBCPP_HIDE_FROM_ABI _Pointer __get() const _NOEXCEPT { return __ptr_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI _Pointer __get() const _NOEXCEPT { return __ptr_; }
 
 private:
-  _LIBCPP_HIDE_FROM_ABI void __destroy() _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __destroy() _NOEXCEPT {
     if (__ptr_ != nullptr) {
       allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __n_);
     }
diff --git a/external/llvm-project/libcxx/include/__memory/compressed_pair.h b/external/llvm-project/libcxx/include/__memory/compressed_pair.h
index 38798a21fa3c..fb7b7b7afcc8 100644
--- a/external/llvm-project/libcxx/include/__memory/compressed_pair.h
+++ b/external/llvm-project/libcxx/include/__memory/compressed_pair.h
@@ -15,7 +15,6 @@
 #include <__type_traits/datasizeof.h>
 #include <__type_traits/is_empty.h>
 #include <__type_traits/is_final.h>
-#include <__type_traits/is_reference.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -63,9 +62,17 @@ inline const size_t __compressed_pair_alignment = _LIBCPP_ALIGNOF(_Tp);
 template <class _Tp>
 inline const size_t __compressed_pair_alignment<_Tp&> = _LIBCPP_ALIGNOF(void*);
 
-template <class _ToPad,
-          bool _Empty = ((is_empty<_ToPad>::value && !__libcpp_is_final<_ToPad>::value) ||
-                         is_reference<_ToPad>::value || sizeof(_ToPad) == __datasizeof_v<_ToPad>)>
+template <class _ToPad>
+inline const bool __is_reference_or_unpadded_object =
+    (is_empty<_ToPad>::value && !__libcpp_is_final<_ToPad>::value) || sizeof(_ToPad) == __datasizeof_v<_ToPad>;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp&> = true;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp&&> = true;
+
+template <class _ToPad, bool _Empty = __is_reference_or_unpadded_object<_ToPad> >
 class __compressed_pair_padding {
   char __padding_[sizeof(_ToPad) - __datasizeof_v<_ToPad>] = {};
 };
diff --git a/external/llvm-project/libcxx/include/__memory/pointer_traits.h b/external/llvm-project/libcxx/include/__memory/pointer_traits.h
index 4ba50898fb37..879b387b9ad1 100644
--- a/external/llvm-project/libcxx/include/__memory/pointer_traits.h
+++ b/external/llvm-project/libcxx/include/__memory/pointer_traits.h
@@ -245,8 +245,8 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr auto to_address(_Tp* __p) noexcept {
 }
 
 template <class _Pointer>
-inline _LIBCPP_HIDE_FROM_ABI constexpr auto
-to_address(const _Pointer& __p) noexcept -> decltype(std::__to_address(__p)) {
+inline _LIBCPP_HIDE_FROM_ABI constexpr auto to_address(const _Pointer& __p) noexcept
+    -> decltype(std::__to_address(__p)) {
   return std::__to_address(__p);
 }
 #endif
@@ -302,6 +302,18 @@ concept __resettable_smart_pointer_with_args = requires(_Smart __s, _Pointer __p
 
 #endif
 
+// This function ensures safe conversions between fancy pointers at compile-time, where we avoid casts from/to
+// `__void_pointer` by obtaining the underlying raw pointer from the fancy pointer using `std::to_address`,
+// then dereferencing it to retrieve the pointed-to object, and finally constructing the target fancy pointer
+// to that object using the `std::pointer_traits<>::pinter_to` function.
+template <class _PtrTo, class _PtrFrom>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _PtrTo __static_fancy_pointer_cast(const _PtrFrom& __p) {
+  using __ptr_traits   = pointer_traits<_PtrTo>;
+  using __element_type = typename __ptr_traits::element_type;
+  return __p ? __ptr_traits::pointer_to(*static_cast<__element_type*>(std::addressof(*__p)))
+             : static_cast<_PtrTo>(nullptr);
+}
+
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
diff --git a/external/llvm-project/libcxx/include/__numeric/saturation_arithmetic.h b/external/llvm-project/libcxx/include/__numeric/saturation_arithmetic.h
index 4110a8cb142a..9bd3af12c957 100644
--- a/external/llvm-project/libcxx/include/__numeric/saturation_arithmetic.h
+++ b/external/llvm-project/libcxx/include/__numeric/saturation_arithmetic.h
@@ -11,9 +11,9 @@
 #define _LIBCPP___NUMERIC_SATURATION_ARITHMETIC_H
 
 #include <__assert>
-#include <__concepts/arithmetic.h>
 #include <__config>
 #include <__memory/addressof.h>
+#include <__type_traits/integer_traits.h>
 #include <__utility/cmp.h>
 #include <limits>
 
@@ -28,12 +28,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp __add_sat(_Tp __x, _Tp __y) noexcept {
   if (_Tp __sum; !__builtin_add_overflow(__x, __y, std::addressof(__sum)))
     return __sum;
   // Handle overflow
-  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+  if constexpr (__unsigned_integer<_Tp>) {
     return std::numeric_limits<_Tp>::max();
   } else {
     // Signed addition overflow
@@ -46,12 +46,12 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __add_sat(_Tp __x, _Tp __y) noexcept {
   }
 }
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp __sub_sat(_Tp __x, _Tp __y) noexcept {
   if (_Tp __sub; !__builtin_sub_overflow(__x, __y, std::addressof(__sub)))
     return __sub;
   // Handle overflow
-  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+  if constexpr (__unsigned_integer<_Tp>) {
     // Overflows if (x < y)
     return std::numeric_limits<_Tp>::min();
   } else {
@@ -65,12 +65,12 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __sub_sat(_Tp __x, _Tp __y) noexcept {
   }
 }
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp __mul_sat(_Tp __x, _Tp __y) noexcept {
   if (_Tp __mul; !__builtin_mul_overflow(__x, __y, std::addressof(__mul)))
     return __mul;
   // Handle overflow
-  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+  if constexpr (__unsigned_integer<_Tp>) {
     return std::numeric_limits<_Tp>::max();
   } else {
     // Signed multiplication overflow
@@ -81,10 +81,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __mul_sat(_Tp __x, _Tp __y) noexcept {
   }
 }
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp __div_sat(_Tp __x, _Tp __y) noexcept {
   _LIBCPP_ASSERT_UNCATEGORIZED(__y != 0, "Division by 0 is undefined");
-  if constexpr (__libcpp_unsigned_integer<_Tp>) {
+  if constexpr (__unsigned_integer<_Tp>) {
     return __x / __y;
   } else {
     // Handle signed division overflow
@@ -94,7 +94,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __div_sat(_Tp __x, _Tp __y) noexcept {
   }
 }
 
-template <__libcpp_integer _Rp, __libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Rp, __signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Rp __saturate_cast(_Tp __x) noexcept {
   // Saturation is impossible edge case when ((min _Rp) < (min _Tp) && (max _Rp) > (max _Tp)) and it is expected to be
   // optimized out by the compiler.
@@ -112,27 +112,27 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Rp __saturate_cast(_Tp __x) noexcept {
 
 #if _LIBCPP_STD_VER >= 26
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp add_sat(_Tp __x, _Tp __y) noexcept {
   return std::__add_sat(__x, __y);
 }
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp sub_sat(_Tp __x, _Tp __y) noexcept {
   return std::__sub_sat(__x, __y);
 }
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp mul_sat(_Tp __x, _Tp __y) noexcept {
   return std::__mul_sat(__x, __y);
 }
 
-template <__libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp div_sat(_Tp __x, _Tp __y) noexcept {
   return std::__div_sat(__x, __y);
 }
 
-template <__libcpp_integer _Rp, __libcpp_integer _Tp>
+template <__signed_or_unsigned_integer _Rp, __signed_or_unsigned_integer _Tp>
 _LIBCPP_HIDE_FROM_ABI constexpr _Rp saturate_cast(_Tp __x) noexcept {
   return std::__saturate_cast<_Rp>(__x);
 }
diff --git a/external/llvm-project/libcxx/include/__type_traits/common_reference.h b/external/llvm-project/libcxx/include/__type_traits/common_reference.h
index c27da5251b9b..7df55f7fecfd 100644
--- a/external/llvm-project/libcxx/include/__type_traits/common_reference.h
+++ b/external/llvm-project/libcxx/include/__type_traits/common_reference.h
@@ -109,11 +109,18 @@ struct __common_ref {};
 // Note C: For the common_reference trait applied to a parameter pack [...]
 
 template <class...>
-struct common_reference;
+struct _LIBCPP_NO_SPECIALIZATIONS common_reference;
 
 template <class... _Types>
 using common_reference_t = typename common_reference<_Types...>::type;
 
+template <class, class, template <class> class, template <class> class>
+struct basic_common_reference {};
+
+_LIBCPP_DIAGNOSTIC_PUSH
+#  if __has_warning("-Winvalid-specialization")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winvalid-specialization")
+#  endif
 // bullet 1 - sizeof...(T) == 0
 template <>
 struct common_reference<> {};
@@ -145,9 +152,6 @@ struct __common_reference_sub_bullet1<_Tp, _Up> {
 
 // sub-bullet 2 - Otherwise, if basic_common_reference<remove_cvref_t<T1>, remove_cvref_t<T2>, XREF(T1), XREF(T2)>::type
 // is well-formed, then the member typedef `type` denotes that type.
-template <class, class, template <class> class, template <class> class>
-struct basic_common_reference {};
-
 template <class _Tp, class _Up>
 using __basic_common_reference_t _LIBCPP_NODEBUG =
     typename basic_common_reference<remove_cvref_t<_Tp>,
@@ -180,10 +184,11 @@ struct __common_reference_sub_bullet3 : common_type<_Tp, _Up> {};
 template <class _Tp, class _Up, class _Vp, class... _Rest>
   requires requires { typename common_reference_t<_Tp, _Up>; }
 struct common_reference<_Tp, _Up, _Vp, _Rest...> : common_reference<common_reference_t<_Tp, _Up>, _Vp, _Rest...> {};
+_LIBCPP_DIAGNOSTIC_POP
 
 // bullet 5 - Otherwise, there shall be no member `type`.
 template <class...>
-struct common_reference {};
+struct _LIBCPP_NO_SPECIALIZATIONS common_reference {};
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/external/llvm-project/libcxx/include/__type_traits/container_traits.h b/external/llvm-project/libcxx/include/__type_traits/container_traits.h
index 5262cef94c61..ed4e19c67503 100644
--- a/external/llvm-project/libcxx/include/__type_traits/container_traits.h
+++ b/external/llvm-project/libcxx/include/__type_traits/container_traits.h
@@ -36,6 +36,9 @@ struct __container_traits {
   // `insert(...)` or `emplace(...)` has strong exception guarantee, that is, if the function
   // exits via an exception, the original container is unaffected
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = false;
+
+  // A trait that tells whether a container supports `reserve(n)` member function.
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/__type_traits/integer_traits.h b/external/llvm-project/libcxx/include/__type_traits/integer_traits.h
new file mode 100644
index 000000000000..fad502c44e30
--- /dev/null
+++ b/external/llvm-project/libcxx/include/__type_traits/integer_traits.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_INTEGER_TRAITS_H
+#define _LIBCPP___TYPE_TRAITS_INTEGER_TRAITS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// This trait is to determine whether a type is a /signed integer type/
+// See [basic.fundamental]/p1
+template <class _Tp>
+inline const bool __is_signed_integer_v = false;
+template <>
+inline const bool __is_signed_integer_v<signed char> = true;
+template <>
+inline const bool __is_signed_integer_v<signed short> = true;
+template <>
+inline const bool __is_signed_integer_v<signed int> = true;
+template <>
+inline const bool __is_signed_integer_v<signed long> = true;
+template <>
+inline const bool __is_signed_integer_v<signed long long> = true;
+#if _LIBCPP_HAS_INT128
+template <>
+inline const bool __is_signed_integer_v<__int128_t> = true;
+#endif
+
+// This trait is to determine whether a type is an /unsigned integer type/
+// See [basic.fundamental]/p2
+template <class _Tp>
+inline const bool __is_unsigned_integer_v = false;
+template <>
+inline const bool __is_unsigned_integer_v<unsigned char> = true;
+template <>
+inline const bool __is_unsigned_integer_v<unsigned short> = true;
+template <>
+inline const bool __is_unsigned_integer_v<unsigned int> = true;
+template <>
+inline const bool __is_unsigned_integer_v<unsigned long> = true;
+template <>
+inline const bool __is_unsigned_integer_v<unsigned long long> = true;
+#if _LIBCPP_HAS_INT128
+template <>
+inline const bool __is_unsigned_integer_v<__uint128_t> = true;
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+concept __signed_integer = __is_signed_integer_v<_Tp>;
+
+template <class _Tp>
+concept __unsigned_integer = __is_unsigned_integer_v<_Tp>;
+
+// This isn't called __integer, because an integer type according to [basic.fundamental]/p11 is the same as an integral
+// type. An integral type is _not_ the same set of types as signed and unsigned integer types combined.
+template <class _Tp>
+concept __signed_or_unsigned_integer = __signed_integer<_Tp> || __unsigned_integer<_Tp>;
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_INTEGER_TRAITS_H
diff --git a/external/llvm-project/libcxx/include/__type_traits/is_integral.h b/external/llvm-project/libcxx/include/__type_traits/is_integral.h
index 7f7ac26beb77..5a340965f038 100644
--- a/external/llvm-project/libcxx/include/__type_traits/is_integral.h
+++ b/external/llvm-project/libcxx/include/__type_traits/is_integral.h
@@ -19,6 +19,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#if __has_builtin(__is_integral)
+
+template <class _Tp>
+struct _LIBCPP_NO_SPECIALIZATIONS is_integral : _BoolConstant<__is_integral(_Tp)> {};
+
+#  if _LIBCPP_STD_VER >= 17
+template <class _Tp>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_integral_v = __is_integral(_Tp);
+#  endif
+
+#else
+
 // clang-format off
 template <class _Tp> struct __libcpp_is_integral                     { enum { value = 0 }; };
 template <>          struct __libcpp_is_integral<bool>               { enum { value = 1 }; };
@@ -47,18 +59,6 @@ template <>          struct __libcpp_is_integral<__uint128_t>        { enum { va
 #endif
 // clang-format on
 
-#if __has_builtin(__is_integral)
-
-template <class _Tp>
-struct _LIBCPP_NO_SPECIALIZATIONS is_integral : _BoolConstant<__is_integral(_Tp)> {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_integral_v = __is_integral(_Tp);
-#  endif
-
-#else
-
 template <class _Tp>
 struct is_integral : public _BoolConstant<__libcpp_is_integral<__remove_cv_t<_Tp> >::value> {};
 
diff --git a/external/llvm-project/libcxx/include/__type_traits/is_pointer.h b/external/llvm-project/libcxx/include/__type_traits/is_pointer.h
index 7bc78a601866..3c58656e0e61 100644
--- a/external/llvm-project/libcxx/include/__type_traits/is_pointer.h
+++ b/external/llvm-project/libcxx/include/__type_traits/is_pointer.h
@@ -40,7 +40,7 @@ template <class _Tp>
 struct __libcpp_remove_objc_qualifiers {
   typedef _Tp type;
 };
-#  if _LIBCPP_HAS_OBJC_ARC
+#  if __has_feature(objc_arc)
 // clang-format off
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __strong> { typedef _Tp type; };
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __weak> { typedef _Tp type; };
diff --git a/external/llvm-project/libcxx/include/__type_traits/is_scalar.h b/external/llvm-project/libcxx/include/__type_traits/is_scalar.h
index 102a0ca6f2fd..4cf8c3948d9f 100644
--- a/external/llvm-project/libcxx/include/__type_traits/is_scalar.h
+++ b/external/llvm-project/libcxx/include/__type_traits/is_scalar.h
@@ -37,7 +37,7 @@ _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_scalar_v = __is_scalar(_Tp);
 
 template <class _Tp>
 struct __is_block : false_type {};
-#  if _LIBCPP_HAS_EXTENSION_BLOCKS
+#  if __has_extension(blocks)
 template <class _Rp, class... _Args>
 struct __is_block<_Rp (^)(_Args...)> : true_type {};
 #  endif
diff --git a/external/llvm-project/libcxx/include/__type_traits/is_signed_integer.h b/external/llvm-project/libcxx/include/__type_traits/is_signed_integer.h
deleted file mode 100644
index 62943902a183..000000000000
--- a/external/llvm-project/libcxx/include/__type_traits/is_signed_integer.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
-#define _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
-
-#include <__config>
-#include <__type_traits/integral_constant.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-// clang-format off
-template <class _Tp> struct __libcpp_is_signed_integer                   : false_type {};
-template <>          struct __libcpp_is_signed_integer<signed char>      : true_type {};
-template <>          struct __libcpp_is_signed_integer<signed short>     : true_type {};
-template <>          struct __libcpp_is_signed_integer<signed int>       : true_type {};
-template <>          struct __libcpp_is_signed_integer<signed long>      : true_type {};
-template <>          struct __libcpp_is_signed_integer<signed long long> : true_type {};
-#if _LIBCPP_HAS_INT128
-template <>          struct __libcpp_is_signed_integer<__int128_t>       : true_type {};
-#endif
-// clang-format on
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___TYPE_TRAITS_IS_SIGNED_INTEGER_H
diff --git a/external/llvm-project/libcxx/include/__type_traits/is_unsigned_integer.h b/external/llvm-project/libcxx/include/__type_traits/is_unsigned_integer.h
deleted file mode 100644
index 74414a831e79..000000000000
--- a/external/llvm-project/libcxx/include/__type_traits/is_unsigned_integer.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
-#define _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
-
-#include <__config>
-#include <__type_traits/integral_constant.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-// clang-format off
-template <class _Tp> struct __libcpp_is_unsigned_integer                     : false_type {};
-template <>          struct __libcpp_is_unsigned_integer<unsigned char>      : true_type {};
-template <>          struct __libcpp_is_unsigned_integer<unsigned short>     : true_type {};
-template <>          struct __libcpp_is_unsigned_integer<unsigned int>       : true_type {};
-template <>          struct __libcpp_is_unsigned_integer<unsigned long>      : true_type {};
-template <>          struct __libcpp_is_unsigned_integer<unsigned long long> : true_type {};
-#if _LIBCPP_HAS_INT128
-template <>          struct __libcpp_is_unsigned_integer<__uint128_t>        : true_type {};
-#endif
-// clang-format on
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___TYPE_TRAITS_IS_UNSIGNED_INTEGER_H
diff --git a/external/llvm-project/libcxx/include/__type_traits/reference_constructs_from_temporary.h b/external/llvm-project/libcxx/include/__type_traits/reference_constructs_from_temporary.h
index 1c62e58cc4e1..2ff549b4e15c 100644
--- a/external/llvm-project/libcxx/include/__type_traits/reference_constructs_from_temporary.h
+++ b/external/llvm-project/libcxx/include/__type_traits/reference_constructs_from_temporary.h
@@ -30,6 +30,15 @@ _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool reference_constructs_from_tempo
 
 #endif
 
+#if __has_builtin(__reference_constructs_from_temporary)
+template <class _Tp, class _Up>
+inline const bool __reference_constructs_from_temporary_v = __reference_constructs_from_temporary(_Tp, _Up);
+#else
+// TODO(LLVM 22): Remove this as all supported compilers should have __reference_constructs_from_temporary implemented.
+template <class _Tp, class _Up>
+inline const bool __reference_constructs_from_temporary_v = __reference_binds_to_temporary(_Tp, _Up);
+#endif
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_REFERENCE_CONSTRUCTS_FROM_TEMPORARY_H
diff --git a/external/llvm-project/libcxx/include/__utility/cmp.h b/external/llvm-project/libcxx/include/__utility/cmp.h
index b7c1ed614dfc..14dc0c154c04 100644
--- a/external/llvm-project/libcxx/include/__utility/cmp.h
+++ b/external/llvm-project/libcxx/include/__utility/cmp.h
@@ -9,8 +9,8 @@
 #ifndef _LIBCPP___UTILITY_CMP_H
 #define _LIBCPP___UTILITY_CMP_H
 
-#include <__concepts/arithmetic.h>
 #include <__config>
+#include <__type_traits/integer_traits.h>
 #include <__type_traits/is_signed.h>
 #include <__type_traits/make_unsigned.h>
 #include <limits>
@@ -26,7 +26,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 20
 
-template <__libcpp_integer _Tp, __libcpp_integer _Up>
+template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_equal(_Tp __t, _Up __u) noexcept {
   if constexpr (is_signed_v<_Tp> == is_signed_v<_Up>)
     return __t == __u;
@@ -36,12 +36,12 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_equal(_Tp __t, _Up __u) noexcept {
     return __u < 0 ? false : __t == make_unsigned_t<_Up>(__u);
 }
 
-template <__libcpp_integer _Tp, __libcpp_integer _Up>
+template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_not_equal(_Tp __t, _Up __u) noexcept {
   return !std::cmp_equal(__t, __u);
 }
 
-template <__libcpp_integer _Tp, __libcpp_integer _Up>
+template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less(_Tp __t, _Up __u) noexcept {
   if constexpr (is_signed_v<_Tp> == is_signed_v<_Up>)
     return __t < __u;
@@ -51,22 +51,22 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less(_Tp __t, _Up __u) noexcept {
     return __u < 0 ? false : __t < make_unsigned_t<_Up>(__u);
 }
 
-template <__libcpp_integer _Tp, __libcpp_integer _Up>
+template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater(_Tp __t, _Up __u) noexcept {
   return std::cmp_less(__u, __t);
 }
 
-template <__libcpp_integer _Tp, __libcpp_integer _Up>
+template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less_equal(_Tp __t, _Up __u) noexcept {
   return !std::cmp_greater(__t, __u);
 }
 
-template <__libcpp_integer _Tp, __libcpp_integer _Up>
+template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater_equal(_Tp __t, _Up __u) noexcept {
   return !std::cmp_less(__t, __u);
 }
 
-template <__libcpp_integer _Tp, __libcpp_integer _Up>
+template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
 _LIBCPP_HIDE_FROM_ABI constexpr bool in_range(_Up __u) noexcept {
   return std::cmp_less_equal(__u, numeric_limits<_Tp>::max()) &&
          std::cmp_greater_equal(__u, numeric_limits<_Tp>::min());
diff --git a/external/llvm-project/libcxx/include/__utility/pair.h b/external/llvm-project/libcxx/include/__utility/pair.h
index ab390aafa0d9..dbacbce04476 100644
--- a/external/llvm-project/libcxx/include/__utility/pair.h
+++ b/external/llvm-project/libcxx/include/__utility/pair.h
@@ -209,21 +209,12 @@ struct pair
 #  endif
 
 #  if _LIBCPP_STD_VER >= 23
-  // TODO: Remove this workaround in LLVM 20. The bug got fixed in Clang 18.
-  // This is a workaround for http://llvm.org/PR60710. We should be able to remove it once Clang is fixed.
-  template <class _PairLike>
-  _LIBCPP_HIDE_FROM_ABI static constexpr bool __pair_like_explicit_wknd() {
-    if constexpr (__pair_like_no_subrange<_PairLike>) {
-      return !is_convertible_v<decltype(std::get<0>(std::declval<_PairLike&&>())), first_type> ||
-             !is_convertible_v<decltype(std::get<1>(std::declval<_PairLike&&>())), second_type>;
-    }
-    return false;
-  }
-
   template <__pair_like_no_subrange _PairLike>
     requires(is_constructible_v<first_type, decltype(std::get<0>(std::declval<_PairLike &&>()))> &&
              is_constructible_v<second_type, decltype(std::get<1>(std::declval<_PairLike &&>()))>)
-  _LIBCPP_HIDE_FROM_ABI constexpr explicit(__pair_like_explicit_wknd<_PairLike>()) pair(_PairLike&& __p)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(
+      !is_convertible_v<decltype(std::get<0>(std::declval<_PairLike&&>())), first_type> ||
+      !is_convertible_v<decltype(std::get<1>(std::declval<_PairLike&&>())), second_type>) pair(_PairLike&& __p)
       : first(std::get<0>(std::forward<_PairLike>(__p))), second(std::get<1>(std::forward<_PairLike>(__p))) {}
 #  endif
 
diff --git a/external/llvm-project/libcxx/include/__vector/container_traits.h b/external/llvm-project/libcxx/include/__vector/container_traits.h
index 337b9c5c8368..20f2b9ef042c 100644
--- a/external/llvm-project/libcxx/include/__vector/container_traits.h
+++ b/external/llvm-project/libcxx/include/__vector/container_traits.h
@@ -32,6 +32,8 @@ struct __container_traits<vector<_Tp, _Allocator> > {
   //  the effects are unspecified.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
       is_nothrow_move_constructible<_Tp>::value || __is_cpp17_copy_insertable_v<_Allocator>;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = true;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/any b/external/llvm-project/libcxx/include/any
index 940cd2939748..89bf3cf1f7df 100644
--- a/external/llvm-project/libcxx/include/any
+++ b/external/llvm-project/libcxx/include/any
@@ -120,7 +120,7 @@ _LIBCPP_PUSH_MACROS
 #  include <__undef_macros>
 
 _LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_ANY_CAST bad_any_cast : public bad_cast {
+class _LIBCPP_EXPORTED_FROM_ABI bad_any_cast : public bad_cast {
 public:
   const char* what() const _NOEXCEPT override;
 };
@@ -130,7 +130,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #  if _LIBCPP_STD_VER >= 17
 
-[[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST void __throw_bad_any_cast() {
+[[noreturn]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_any_cast() {
 #    if _LIBCPP_HAS_EXCEPTIONS
   throw bad_any_cast();
 #    else
@@ -519,7 +519,7 @@ inline _LIBCPP_HIDE_FROM_ABI any make_any(initializer_list<_Up> __il, _Args&&...
 }
 
 template <class _ValueType>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any const& __v) {
+inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any const& __v) {
   using _RawValueType = __remove_cvref_t<_ValueType>;
   static_assert(is_constructible<_ValueType, _RawValueType const&>::value,
                 "ValueType is required to be a const lvalue reference "
@@ -531,7 +531,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType
 }
 
 template <class _ValueType>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any& __v) {
+inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any& __v) {
   using _RawValueType = __remove_cvref_t<_ValueType>;
   static_assert(is_constructible<_ValueType, _RawValueType&>::value,
                 "ValueType is required to be an lvalue reference "
@@ -543,7 +543,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType
 }
 
 template <class _ValueType>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST _ValueType any_cast(any&& __v) {
+inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any&& __v) {
   using _RawValueType = __remove_cvref_t<_ValueType>;
   static_assert(is_constructible<_ValueType, _RawValueType>::value,
                 "ValueType is required to be an rvalue reference "
diff --git a/external/llvm-project/libcxx/include/bitset b/external/llvm-project/libcxx/include/bitset
index 88dc0e08c995..6be476e2b69d 100644
--- a/external/llvm-project/libcxx/include/bitset
+++ b/external/llvm-project/libcxx/include/bitset
@@ -645,16 +645,13 @@ public:
   template <class _CharT, __enable_if_t<_IsCharLikeType<_CharT>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit bitset(
       const _CharT* __str,
-#  if _LIBCPP_STD_VER >= 26
-      typename basic_string_view<_CharT>::size_type __n = basic_string_view<_CharT>::npos,
-#  else
-      typename basic_string<_CharT>::size_type __n = basic_string<_CharT>::npos,
-#  endif
+      size_t __n    = basic_string<_CharT>::npos,
       _CharT __zero = _CharT('0'),
       _CharT __one  = _CharT('1')) {
-
-    size_t __rlen = std::min(__n, char_traits<_CharT>::length(__str));
-    __init_from_string_view(basic_string_view<_CharT>(__str, __rlen), __zero, __one);
+    if (__n == basic_string<_CharT>::npos)
+      __init_from_string_view(basic_string_view<_CharT>(__str), __zero, __one);
+    else
+      __init_from_string_view(basic_string_view<_CharT>(__str, __n), __zero, __one);
   }
 #  if _LIBCPP_STD_VER >= 26
   template <class _CharT, class _Traits>
diff --git a/external/llvm-project/libcxx/include/chrono b/external/llvm-project/libcxx/include/chrono
index cd9b98872083..82e99a31bcc9 100644
--- a/external/llvm-project/libcxx/include/chrono
+++ b/external/llvm-project/libcxx/include/chrono
@@ -132,6 +132,11 @@ public:
 
     // arithmetic
 
+    constexpr time_point& operator++();    // C++20
+    constexpr time_point  operator++(int); // C++20
+    constexpr time_point& operator--();    // C++20
+    constexpr time_point  operator--(int); // C++20
+
     time_point& operator+=(const duration& d); // constexpr in C++17
     time_point& operator-=(const duration& d); // constexpr in C++17
 
diff --git a/external/llvm-project/libcxx/include/deque b/external/llvm-project/libcxx/include/deque
index d8645d06ae59..e33e7d31500a 100644
--- a/external/llvm-project/libcxx/include/deque
+++ b/external/llvm-project/libcxx/include/deque
@@ -930,7 +930,7 @@ private:
     (void)__end;
     (void)__annotation_type;
     (void)__place;
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     // __beg - index of the first item to annotate
     // __end - index behind the last item to annotate (so last item + 1)
     // __annotation_type - __asan_unposion or __asan_poison
@@ -1023,23 +1023,23 @@ private:
       std::__annotate_double_ended_contiguous_container<_Allocator>(
           __mem_beg, __mem_end, __old_beg, __old_end, __new_beg, __new_end);
     }
-#  endif // _LIBCPP_HAS_ASAN
+#  endif // __has_feature(address_sanitizer)
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
     (void)__current_size;
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     if (__current_size == 0)
       __annotate_from_to(0, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
     else {
       __annotate_from_to(0, __start_, __asan_poison, __asan_front_moved);
       __annotate_from_to(__start_ + __current_size, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
     }
-#  endif // _LIBCPP_HAS_ASAN
+#  endif // __has_feature(address_sanitizer)
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     if (empty()) {
       for (size_t __i = 0; __i < __map_.size(); ++__i) {
         __annotate_whole_block(__i, __asan_unposion);
@@ -1048,19 +1048,19 @@ private:
       __annotate_from_to(0, __start_, __asan_unposion, __asan_front_moved);
       __annotate_from_to(__start_ + size(), __map_.size() * __block_size, __asan_unposion, __asan_back_moved);
     }
-#  endif // _LIBCPP_HAS_ASAN
+#  endif // __has_feature(address_sanitizer)
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_increase_front(size_type __n) const _NOEXCEPT {
     (void)__n;
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     __annotate_from_to(__start_ - __n, __start_, __asan_unposion, __asan_front_moved);
 #  endif
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_increase_back(size_type __n) const _NOEXCEPT {
     (void)__n;
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     __annotate_from_to(__start_ + size(), __start_ + size() + __n, __asan_unposion, __asan_back_moved);
 #  endif
   }
@@ -1068,7 +1068,7 @@ private:
   _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_front(size_type __old_size, size_type __old_start) const _NOEXCEPT {
     (void)__old_size;
     (void)__old_start;
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     __annotate_from_to(__old_start, __old_start + (__old_size - size()), __asan_poison, __asan_front_moved);
 #  endif
   }
@@ -1076,7 +1076,7 @@ private:
   _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_back(size_type __old_size, size_type __old_start) const _NOEXCEPT {
     (void)__old_size;
     (void)__old_start;
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     __annotate_from_to(__old_start + size(), __old_start + __old_size, __asan_poison, __asan_back_moved);
 #  endif
   }
@@ -1089,7 +1089,7 @@ private:
   __annotate_whole_block(size_t __block_index, __asan_annotation_type __annotation_type) const _NOEXCEPT {
     (void)__block_index;
     (void)__annotation_type;
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
     __map_const_iterator __block_it = __map_.begin() + __block_index;
     const void* __block_start       = std::__to_address(*__block_it);
     const void* __block_end         = std::__to_address(*__block_it + __block_size);
@@ -1102,7 +1102,7 @@ private:
     }
 #  endif
   }
-#  if _LIBCPP_HAS_ASAN
+#  if __has_feature(address_sanitizer)
 
 public:
   _LIBCPP_HIDE_FROM_ABI bool __verify_asan_annotations() const _NOEXCEPT {
@@ -1164,7 +1164,7 @@ public:
   }
 
 private:
-#  endif // _LIBCPP_HAS_ASAN
+#  endif // __has_feature(address_sanitizer)
   _LIBCPP_HIDE_FROM_ABI bool __maybe_remove_front_spare(bool __keep_one = true) {
     if (__front_spare_blocks() >= 2 || (!__keep_one && __front_spare_blocks())) {
       __annotate_whole_block(0, __asan_unposion);
@@ -2631,6 +2631,8 @@ struct __container_traits<deque<_Tp, _Allocator> > {
   //  non-Cpp17CopyInsertable T, the effects are unspecified.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
       is_nothrow_move_constructible<_Tp>::value || __is_cpp17_copy_insertable_v<_Allocator>;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/forward_list b/external/llvm-project/libcxx/include/forward_list
index f264f567c9bb..e9b2c860b89c 100644
--- a/external/llvm-project/libcxx/include/forward_list
+++ b/external/llvm-project/libcxx/include/forward_list
@@ -295,8 +295,8 @@ struct __forward_node_traits {
                 "the _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB macro to silence this diagnostic.");
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI static __begin_node_pointer __as_iter_node(__node_pointer __p) {
-    return static_cast<__begin_node_pointer>(static_cast<__void_pointer>(__p));
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI static __begin_node_pointer __as_iter_node(__node_pointer __p) {
+    return std::__static_fancy_pointer_cast<__begin_node_pointer>(__p);
   }
 };
 
@@ -307,11 +307,11 @@ struct __forward_begin_node {
 
   pointer __next_;
 
-  _LIBCPP_HIDE_FROM_ABI __forward_begin_node() : __next_(nullptr) {}
-  _LIBCPP_HIDE_FROM_ABI explicit __forward_begin_node(pointer __n) : __next_(__n) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_begin_node() : __next_(nullptr) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_begin_node(pointer __n) : __next_(__n) {}
 
-  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __next_as_begin() const {
-    return static_cast<__begin_node_pointer>(__next_);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __next_as_begin() const {
+    return std::__static_fancy_pointer_cast<__begin_node_pointer>(__next_);
   }
 };
 
@@ -335,7 +335,7 @@ private:
   };
 
 public:
-  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
 #  else
 
 private:
@@ -345,8 +345,8 @@ public:
   _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_node(_NodePtr __next) : _Base(__next) {}
-  _LIBCPP_HIDE_FROM_ABI ~__forward_list_node() {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_node(_NodePtr __next) : _Base(__next) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI ~__forward_list_node() {}
 };
 
 template <class _Tp, class _Alloc = allocator<_Tp> >
@@ -357,24 +357,26 @@ class __forward_list_const_iterator;
 template <class _NodePtr>
 class __forward_list_iterator {
   typedef __forward_node_traits<_NodePtr> __traits;
+  typedef typename __traits::__node_type __node_type;
+  typedef typename __traits::__begin_node __begin_node_type;
   typedef typename __traits::__node_pointer __node_pointer;
   typedef typename __traits::__begin_node_pointer __begin_node_pointer;
   typedef typename __traits::__void_pointer __void_pointer;
 
   __begin_node_pointer __ptr_;
 
-  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const {
-    return static_cast<__begin_node_pointer>(static_cast<__void_pointer>(__ptr_));
-  }
-  _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
-    return static_cast<__node_pointer>(static_cast<__void_pointer>(__ptr_));
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const { return __ptr_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
+    return std::__static_fancy_pointer_cast<__node_pointer>(__ptr_);
   }
 
-  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(nullptr_t) _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(nullptr_t) _NOEXCEPT
+      : __ptr_(nullptr) {}
 
+  _LIBCPP_CONSTEXPR_SINCE_CXX26
   _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(__begin_node_pointer __p) _NOEXCEPT : __ptr_(__p) {}
 
-  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(__node_pointer __p) _NOEXCEPT
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(__node_pointer __p) _NOEXCEPT
       : __ptr_(__traits::__as_iter_node(__p)) {}
 
   template <class, class>
@@ -389,27 +391,31 @@ public:
   typedef typename pointer_traits<__node_pointer>::difference_type difference_type;
   typedef __rebind_pointer_t<__node_pointer, value_type> pointer;
 
-  _LIBCPP_HIDE_FROM_ABI __forward_list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_unsafe_node_pointer()->__get_value(); }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    return __get_unsafe_node_pointer()->__get_value();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
     return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__get_value());
   }
 
-  _LIBCPP_HIDE_FROM_ABI __forward_list_iterator& operator++() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_iterator& operator++() {
     __ptr_ = __traits::__as_iter_node(__ptr_->__next_);
     return *this;
   }
-  _LIBCPP_HIDE_FROM_ABI __forward_list_iterator operator++(int) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_iterator operator++(int) {
     __forward_list_iterator __t(*this);
     ++(*this);
     return __t;
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __forward_list_iterator& __x, const __forward_list_iterator& __y) {
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __forward_list_iterator& __x, const __forward_list_iterator& __y) {
     return __x.__ptr_ == __y.__ptr_;
   }
-  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __forward_list_iterator& __x, const __forward_list_iterator& __y) {
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const __forward_list_iterator& __x, const __forward_list_iterator& __y) {
     return !(__x == __y);
   }
 };
@@ -421,23 +427,25 @@ class __forward_list_const_iterator {
 
   typedef __forward_node_traits<_NodePtr> __traits;
   typedef typename __traits::__node_type __node_type;
+  typedef typename __traits::__begin_node __begin_node_type;
   typedef typename __traits::__node_pointer __node_pointer;
   typedef typename __traits::__begin_node_pointer __begin_node_pointer;
   typedef typename __traits::__void_pointer __void_pointer;
 
   __begin_node_pointer __ptr_;
 
-  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const {
-    return static_cast<__begin_node_pointer>(static_cast<__void_pointer>(__ptr_));
-  }
-  _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
-    return static_cast<__node_pointer>(static_cast<__void_pointer>(__ptr_));
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const { return __ptr_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
+    return std::__static_fancy_pointer_cast<__node_pointer>(__ptr_);
   }
 
-  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(nullptr_t) _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(nullptr_t) _NOEXCEPT
+      : __ptr_(nullptr) {}
 
+  _LIBCPP_CONSTEXPR_SINCE_CXX26
   _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(__begin_node_pointer __p) _NOEXCEPT : __ptr_(__p) {}
 
+  _LIBCPP_CONSTEXPR_SINCE_CXX26
   _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(__node_pointer __p) _NOEXCEPT
       : __ptr_(__traits::__as_iter_node(__p)) {}
 
@@ -451,30 +459,32 @@ public:
   typedef typename pointer_traits<__node_pointer>::difference_type difference_type;
   typedef __rebind_pointer_t<__node_pointer, const value_type> pointer;
 
-  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
-  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator(__forward_list_iterator<__node_pointer> __p) _NOEXCEPT
-      : __ptr_(__p.__ptr_) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  __forward_list_const_iterator(__forward_list_iterator<__node_pointer> __p) _NOEXCEPT : __ptr_(__p.__ptr_) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_unsafe_node_pointer()->__get_value(); }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    return __get_unsafe_node_pointer()->__get_value();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
     return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__get_value());
   }
 
-  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator& operator++() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator& operator++() {
     __ptr_ = __traits::__as_iter_node(__ptr_->__next_);
     return *this;
   }
-  _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator operator++(int) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator operator++(int) {
     __forward_list_const_iterator __t(*this);
     ++(*this);
     return __t;
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI bool
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
   operator==(const __forward_list_const_iterator& __x, const __forward_list_const_iterator& __y) {
     return __x.__ptr_ == __y.__ptr_;
   }
-  friend _LIBCPP_HIDE_FROM_ABI bool
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
   operator!=(const __forward_list_const_iterator& __x, const __forward_list_const_iterator& __y) {
     return !(__x == __y);
   }
@@ -498,48 +508,53 @@ protected:
 
   _LIBCPP_COMPRESSED_PAIR(__begin_node, __before_begin_, __node_allocator, __alloc_);
 
-  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __before_begin() _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __before_begin() _NOEXCEPT {
     return pointer_traits<__begin_node_pointer>::pointer_to(__before_begin_);
   }
-  _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __before_begin() const _NOEXCEPT {
-    return pointer_traits<__begin_node_pointer>::pointer_to(const_cast<__begin_node&>(__before_begin_));
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __before_begin() const _NOEXCEPT {
+    return pointer_traits<__begin_node_pointer>::pointer_to(
+        *const_cast<__begin_node*>(std::addressof(__before_begin_)));
   }
 
   typedef __forward_list_iterator<__node_pointer> iterator;
   typedef __forward_list_const_iterator<__node_pointer> const_iterator;
 
-  _LIBCPP_HIDE_FROM_ABI __forward_list_base() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_base()
+      _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value)
       : __before_begin_(__begin_node()) {}
-  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_base(const allocator_type& __a)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_base(const allocator_type& __a)
       : __before_begin_(__begin_node()), __alloc_(__node_allocator(__a)) {}
-  _LIBCPP_HIDE_FROM_ABI explicit __forward_list_base(const __node_allocator& __a)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_base(const __node_allocator& __a)
       : __before_begin_(__begin_node()), __alloc_(__a) {}
 
 public:
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
   __forward_list_base(__forward_list_base&& __x) noexcept(is_nothrow_move_constructible<__node_allocator>::value);
-  _LIBCPP_HIDE_FROM_ABI __forward_list_base(__forward_list_base&& __x, const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  __forward_list_base(__forward_list_base&& __x, const allocator_type& __a);
 #  endif // _LIBCPP_CXX03_LANG
 
   __forward_list_base(const __forward_list_base&)            = delete;
   __forward_list_base& operator=(const __forward_list_base&) = delete;
 
-  _LIBCPP_HIDE_FROM_ABI ~__forward_list_base();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI ~__forward_list_base();
 
 protected:
-  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base& __x) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base& __x) {
     __copy_assign_alloc(__x, integral_constant<bool, __node_traits::propagate_on_container_copy_assignment::value>());
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base& __x)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base& __x)
       _NOEXCEPT_(!__node_traits::propagate_on_container_move_assignment::value ||
                  is_nothrow_move_assignable<__node_allocator>::value) {
     __move_assign_alloc(__x, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
   }
 
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__node_pointer __next, _Args&&... __args) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __node_pointer
+  __create_node(__node_pointer __next, _Args&&... __args) {
     __allocation_guard<__node_allocator> __guard(__alloc_, 1);
     // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
     // held inside the node, since we need to use the allocator's construct() method for that.
@@ -554,7 +569,7 @@ protected:
     return __guard.__release_ptr();
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
     // For the same reason as above, we use the allocator's destroy() method for the value_type,
     // but not for the node itself.
     __node_traits::destroy(__alloc_, std::addressof(__node->__get_value()));
@@ -563,7 +578,7 @@ protected:
   }
 
 public:
-  _LIBCPP_HIDE_FROM_ABI void swap(__forward_list_base& __x)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void swap(__forward_list_base& __x)
 #  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT;
 #  else
@@ -571,18 +586,21 @@ public:
 #  endif
 
 protected:
-  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
 
 private:
-  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base&, false_type) {}
-  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base& __x, true_type) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __forward_list_base&, false_type) {
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+  __copy_assign_alloc(const __forward_list_base& __x, true_type) {
     if (__alloc_ != __x.__alloc_)
       clear();
     __alloc_ = __x.__alloc_;
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base&, false_type) _NOEXCEPT {}
-  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base& __x, true_type)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+  __move_assign_alloc(__forward_list_base&, false_type) _NOEXCEPT {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__forward_list_base& __x, true_type)
       _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value) {
     __alloc_ = std::move(__x.__alloc_);
   }
@@ -591,14 +609,15 @@ private:
 #  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base&& __x) noexcept(
-    is_nothrow_move_constructible<__node_allocator>::value)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(
+    __forward_list_base&& __x) noexcept(is_nothrow_move_constructible<__node_allocator>::value)
     : __before_begin_(std::move(__x.__before_begin_)), __alloc_(std::move(__x.__alloc_)) {
   __x.__before_begin()->__next_ = nullptr;
 }
 
 template <class _Tp, class _Alloc>
-inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base&& __x, const allocator_type& __a)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(
+    __forward_list_base&& __x, const allocator_type& __a)
     : __before_begin_(__begin_node()), __alloc_(__node_allocator(__a)) {
   if (__alloc_ == __x.__alloc_) {
     __before_begin()->__next_     = __x.__before_begin()->__next_;
@@ -609,12 +628,12 @@ inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base
 #  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-__forward_list_base<_Tp, _Alloc>::~__forward_list_base() {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 __forward_list_base<_Tp, _Alloc>::~__forward_list_base() {
   clear();
 }
 
 template <class _Tp, class _Alloc>
-inline void __forward_list_base<_Tp, _Alloc>::swap(__forward_list_base& __x)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void __forward_list_base<_Tp, _Alloc>::swap(__forward_list_base& __x)
 #  if _LIBCPP_STD_VER >= 14
     _NOEXCEPT
 #  else
@@ -627,7 +646,7 @@ inline void __forward_list_base<_Tp, _Alloc>::swap(__forward_list_base& __x)
 }
 
 template <class _Tp, class _Alloc>
-void __forward_list_base<_Tp, _Alloc>::clear() _NOEXCEPT {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void __forward_list_base<_Tp, _Alloc>::clear() _NOEXCEPT {
   for (__node_pointer __p = __before_begin()->__next_; __p != nullptr;) {
     __node_pointer __next = __p->__next_;
     __delete_node(__p);
@@ -672,105 +691,123 @@ public:
   typedef void __remove_return_type;
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI forward_list() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {
-  } // = default;
-  _LIBCPP_HIDE_FROM_ABI explicit forward_list(const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list()
+      _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {} // = default;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit forward_list(const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n);
 #  if _LIBCPP_STD_VER >= 14
-  _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n, const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit forward_list(size_type __n, const allocator_type& __a);
 #  endif
-  _LIBCPP_HIDE_FROM_ABI forward_list(size_type __n, const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list(size_type __n, const value_type& __v);
 
   template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI forward_list(size_type __n, const value_type& __v, const allocator_type& __a) : __base(__a) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  forward_list(size_type __n, const value_type& __v, const allocator_type& __a)
+      : __base(__a) {
     insert_after(cbefore_begin(), __n, __v);
   }
 
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI forward_list(_InputIterator __f, _InputIterator __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list(_InputIterator __f, _InputIterator __l);
 
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI forward_list(_InputIterator __f, _InputIterator __l, const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  forward_list(_InputIterator __f, _InputIterator __l, const allocator_type& __a);
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI forward_list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  forward_list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
       : __base(__a) {
     prepend_range(std::forward<_Range>(__range));
   }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI forward_list(const forward_list& __x);
-  _LIBCPP_HIDE_FROM_ABI forward_list(const forward_list& __x, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list(const forward_list& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  forward_list(const forward_list& __x, const __type_identity_t<allocator_type>& __a);
 
-  _LIBCPP_HIDE_FROM_ABI forward_list& operator=(const forward_list& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list& operator=(const forward_list& __x);
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI forward_list(forward_list&& __x) noexcept(is_nothrow_move_constructible<__base>::value)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  forward_list(forward_list&& __x) noexcept(is_nothrow_move_constructible<__base>::value)
       : __base(std::move(__x)) {}
-  _LIBCPP_HIDE_FROM_ABI forward_list(forward_list&& __x, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  forward_list(forward_list&& __x, const __type_identity_t<allocator_type>& __a);
 
-  _LIBCPP_HIDE_FROM_ABI forward_list(initializer_list<value_type> __il);
-  _LIBCPP_HIDE_FROM_ABI forward_list(initializer_list<value_type> __il, const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list(initializer_list<value_type> __il);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  forward_list(initializer_list<value_type> __il, const allocator_type& __a);
 
-  _LIBCPP_HIDE_FROM_ABI forward_list& operator=(forward_list&& __x) noexcept(
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list& operator=(forward_list&& __x) noexcept(
       (__node_traits::propagate_on_container_move_assignment::value &&
        is_nothrow_move_assignable<allocator_type>::value) ||
       allocator_traits<allocator_type>::is_always_equal::value);
 
-  _LIBCPP_HIDE_FROM_ABI forward_list& operator=(initializer_list<value_type> __il);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list& operator=(initializer_list<value_type> __il);
 
-  _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il);
 #  endif // _LIBCPP_CXX03_LANG
 
   // ~forward_list() = default;
 
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
-  void _LIBCPP_HIDE_FROM_ABI assign(_InputIterator __f, _InputIterator __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 void _LIBCPP_HIDE_FROM_ABI assign(_InputIterator __f, _InputIterator __l);
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
     __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __v);
 
-  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return allocator_type(this->__alloc_); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
+    return allocator_type(this->__alloc_);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__base::__before_begin()->__next_); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
+    return iterator(__base::__before_begin()->__next_);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
     return const_iterator(__base::__before_begin()->__next_);
   }
-  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(nullptr); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return const_iterator(nullptr); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(nullptr); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
+    return const_iterator(nullptr);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
     return const_iterator(__base::__before_begin()->__next_);
   }
-  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return const_iterator(nullptr); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
+    return const_iterator(nullptr);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI iterator before_begin() _NOEXCEPT { return iterator(__base::__before_begin()); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator before_begin() const _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator before_begin() _NOEXCEPT {
+    return iterator(__base::__before_begin());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator before_begin() const _NOEXCEPT {
     return const_iterator(__base::__before_begin());
   }
-  _LIBCPP_HIDE_FROM_ABI const_iterator cbefore_begin() const _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbefore_begin() const _NOEXCEPT {
     return const_iterator(__base::__before_begin());
   }
 
-  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
+  [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
     return __base::__before_begin()->__next_ == nullptr;
   }
-  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
     return std::min<size_type>(__node_traits::max_size(this->__alloc_), numeric_limits<difference_type>::max());
   }
 
-  _LIBCPP_HIDE_FROM_ABI reference front() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference front() {
     _LIBCPP_ASSERT_NON_NULL(!empty(), "forward_list::front called on an empty list");
     return __base::__before_begin()->__next_->__get_value();
   }
-  _LIBCPP_HIDE_FROM_ABI const_reference front() const {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference front() const {
     _LIBCPP_ASSERT_NON_NULL(!empty(), "forward_list::front called on an empty list");
     return __base::__before_begin()->__next_->__get_value();
   }
@@ -778,52 +815,59 @@ public:
 #  ifndef _LIBCPP_CXX03_LANG
 #    if _LIBCPP_STD_VER >= 17
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
 #    else
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
 #    endif
-  _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __v);
 #  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __v);
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
     insert_range_after(cbefore_begin(), std::forward<_Range>(__range));
   }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI void pop_front();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void pop_front();
 
 #  ifndef _LIBCPP_CXX03_LANG
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI iterator emplace_after(const_iterator __p, _Args&&... __args);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator emplace_after(const_iterator __p, _Args&&... __args);
 
-  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, value_type&& __v);
-  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, initializer_list<value_type> __il) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, value_type&& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  insert_after(const_iterator __p, initializer_list<value_type> __il) {
     return insert_after(__p, __il.begin(), __il.end());
   }
 #  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, const value_type& __v);
-  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, size_type __n, const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  insert_after(const_iterator __p, size_type __n, const value_type& __v) {
+    return __insert_after(__p, __n, __v);
+  }
   template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI iterator insert_after(const_iterator __p, _InputIterator __f, _InputIterator __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  insert_after(const_iterator __p, _InputIterator __f, _InputIterator __l);
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI iterator insert_range_after(const_iterator __position, _Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  insert_range_after(const_iterator __position, _Range&& __range) {
     return __insert_after_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 
   template <class _InputIterator, class _Sentinel>
-  _LIBCPP_HIDE_FROM_ABI iterator __insert_after_with_sentinel(const_iterator __p, _InputIterator __f, _Sentinel __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_after_with_sentinel(const_iterator __p, _InputIterator __f, _Sentinel __l);
 
-  _LIBCPP_HIDE_FROM_ABI iterator erase_after(const_iterator __p);
-  _LIBCPP_HIDE_FROM_ABI iterator erase_after(const_iterator __f, const_iterator __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator erase_after(const_iterator __p);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator erase_after(const_iterator __f, const_iterator __l);
 
-  _LIBCPP_HIDE_FROM_ABI void swap(forward_list& __x)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void swap(forward_list& __x)
 #  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT
 #  else
@@ -833,55 +877,63 @@ public:
     __base::swap(__x);
   }
 
-  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
-  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __v);
-  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __base::clear(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __base::clear(); }
 
-  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list&& __x);
-  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list&& __x, const_iterator __i);
-  _LIBCPP_HIDE_FROM_ABI void
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list&& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+  splice_after(const_iterator __p, forward_list&& __x, const_iterator __i);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
   splice_after(const_iterator __p, forward_list&& __x, const_iterator __f, const_iterator __l);
-  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list& __x);
-  _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list& __x, const_iterator __i);
-  _LIBCPP_HIDE_FROM_ABI void
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void splice_after(const_iterator __p, forward_list& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+  splice_after(const_iterator __p, forward_list& __x, const_iterator __i);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
   splice_after(const_iterator __p, forward_list& __x, const_iterator __f, const_iterator __l);
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove(const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type remove(const value_type& __v);
   template <class _Predicate>
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove_if(_Predicate __pred);
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique() { return unique(__equal_to()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type remove_if(_Predicate __pred);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type unique() { return unique(__equal_to()); }
   template <class _BinaryPredicate>
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPredicate __binary_pred);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPredicate __binary_pred);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x) { merge(__x, __less<>()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x) { merge(__x, __less<>()); }
   template <class _Compare>
-  _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x, _Compare __comp) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(forward_list&& __x, _Compare __comp) {
     merge(__x, std::move(__comp));
   }
 #  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x) { merge(__x, __less<>()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x) { merge(__x, __less<>()); }
   template <class _Compare>
-  _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x, _Compare __comp);
-  _LIBCPP_HIDE_FROM_ABI void sort() { sort(__less<>()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(forward_list& __x, _Compare __comp);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void sort() { sort(__less<>()); }
   template <class _Compare>
-  _LIBCPP_HIDE_FROM_ABI void sort(_Compare __comp);
-  _LIBCPP_HIDE_FROM_ABI void reverse() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void sort(_Compare __comp);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void reverse() _NOEXCEPT;
 
 private:
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, true_type)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, true_type)
       _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value);
-  _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, false_type);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign(forward_list& __x, false_type);
 #  endif // _LIBCPP_CXX03_LANG
 
   template <class _Iter, class _Sent>
-  _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iter __f, _Sent __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iter __f, _Sent __l);
+
+  template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_after(const_iterator __p, size_type __n, _Args&&... __args);
 
   template <class _Compare>
-  static _LIBCPP_HIDE_FROM_ABI __node_pointer __merge(__node_pointer __f1, __node_pointer __f2, _Compare& __comp);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 static _LIBCPP_HIDE_FROM_ABI __node_pointer
+  __merge(__node_pointer __f1, __node_pointer __f2, _Compare& __comp);
 
   // TODO: Make this _LIBCPP_HIDE_FROM_ABI
   template <class _Compare>
-  static _LIBCPP_HIDDEN __node_pointer __sort(__node_pointer __f, difference_type __sz, _Compare& __comp);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 static _LIBCPP_HIDDEN __node_pointer
+  __sort(__node_pointer __f, difference_type __sz, _Compare& __comp);
 };
 
 #  if _LIBCPP_STD_VER >= 17
@@ -906,10 +958,10 @@ forward_list(from_range_t, _Range&&, _Alloc = _Alloc()) -> forward_list<ranges::
 #  endif
 
 template <class _Tp, class _Alloc>
-inline forward_list<_Tp, _Alloc>::forward_list(const allocator_type& __a) : __base(__a) {}
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline forward_list<_Tp, _Alloc>::forward_list(const allocator_type& __a) : __base(__a) {}
 
 template <class _Tp, class _Alloc>
-forward_list<_Tp, _Alloc>::forward_list(size_type __n) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(size_type __n) {
   if (__n > 0) {
     for (__begin_node_pointer __p = __base::__before_begin(); __n > 0; --__n, __p = __p->__next_as_begin()) {
       __p->__next_ = this->__create_node(/* next = */ nullptr);
@@ -919,7 +971,8 @@ forward_list<_Tp, _Alloc>::forward_list(size_type __n) {
 
 #  if _LIBCPP_STD_VER >= 14
 template <class _Tp, class _Alloc>
-forward_list<_Tp, _Alloc>::forward_list(size_type __n, const allocator_type& __base_alloc) : __base(__base_alloc) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(size_type __n, const allocator_type& __base_alloc)
+    : __base(__base_alloc) {
   if (__n > 0) {
     for (__begin_node_pointer __p = __base::__before_begin(); __n > 0; --__n, __p = __p->__next_as_begin()) {
       __p->__next_ = this->__create_node(/* next = */ nullptr);
@@ -929,37 +982,39 @@ forward_list<_Tp, _Alloc>::forward_list(size_type __n, const allocator_type& __b
 #  endif
 
 template <class _Tp, class _Alloc>
-forward_list<_Tp, _Alloc>::forward_list(size_type __n, const value_type& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(size_type __n, const value_type& __v) {
   insert_after(cbefore_begin(), __n, __v);
 }
 
 template <class _Tp, class _Alloc>
 template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
-forward_list<_Tp, _Alloc>::forward_list(_InputIterator __f, _InputIterator __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(_InputIterator __f, _InputIterator __l) {
   insert_after(cbefore_begin(), __f, __l);
 }
 
 template <class _Tp, class _Alloc>
 template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
+_LIBCPP_CONSTEXPR_SINCE_CXX26
 forward_list<_Tp, _Alloc>::forward_list(_InputIterator __f, _InputIterator __l, const allocator_type& __a)
     : __base(__a) {
   insert_after(cbefore_begin(), __f, __l);
 }
 
 template <class _Tp, class _Alloc>
-forward_list<_Tp, _Alloc>::forward_list(const forward_list& __x)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(const forward_list& __x)
     : __base(__node_traits::select_on_container_copy_construction(__x.__alloc_)) {
   insert_after(cbefore_begin(), __x.begin(), __x.end());
 }
 
 template <class _Tp, class _Alloc>
+_LIBCPP_CONSTEXPR_SINCE_CXX26
 forward_list<_Tp, _Alloc>::forward_list(const forward_list& __x, const __type_identity_t<allocator_type>& __a)
     : __base(__a) {
   insert_after(cbefore_begin(), __x.begin(), __x.end());
 }
 
 template <class _Tp, class _Alloc>
-forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(const forward_list& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(const forward_list& __x) {
   if (this != std::addressof(__x)) {
     __base::__copy_assign_alloc(__x);
     assign(__x.begin(), __x.end());
@@ -969,6 +1024,7 @@ forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(const forward_li
 
 #  ifndef _LIBCPP_CXX03_LANG
 template <class _Tp, class _Alloc>
+_LIBCPP_CONSTEXPR_SINCE_CXX26
 forward_list<_Tp, _Alloc>::forward_list(forward_list&& __x, const __type_identity_t<allocator_type>& __a)
     : __base(std::move(__x), __a) {
   if (this->__alloc_ != __x.__alloc_) {
@@ -978,17 +1034,19 @@ forward_list<_Tp, _Alloc>::forward_list(forward_list&& __x, const __type_identit
 }
 
 template <class _Tp, class _Alloc>
-forward_list<_Tp, _Alloc>::forward_list(initializer_list<value_type> __il) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(initializer_list<value_type> __il) {
   insert_after(cbefore_begin(), __il.begin(), __il.end());
 }
 
 template <class _Tp, class _Alloc>
-forward_list<_Tp, _Alloc>::forward_list(initializer_list<value_type> __il, const allocator_type& __a) : __base(__a) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26
+forward_list<_Tp, _Alloc>::forward_list(initializer_list<value_type> __il, const allocator_type& __a)
+    : __base(__a) {
   insert_after(cbefore_begin(), __il.begin(), __il.end());
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, true_type)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, true_type)
     _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
   clear();
   __base::__move_assign_alloc(__x);
@@ -997,7 +1055,7 @@ void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, true_type)
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, false_type) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, false_type) {
   if (this->__alloc_ == __x.__alloc_)
     __move_assign(__x, true_type());
   else {
@@ -1007,7 +1065,8 @@ void forward_list<_Tp, _Alloc>::__move_assign(forward_list& __x, false_type) {
 }
 
 template <class _Tp, class _Alloc>
-inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(forward_list&& __x) noexcept(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline forward_list<_Tp, _Alloc>&
+forward_list<_Tp, _Alloc>::operator=(forward_list&& __x) noexcept(
     (__node_traits::propagate_on_container_move_assignment::value &&
      is_nothrow_move_assignable<allocator_type>::value) ||
     allocator_traits<allocator_type>::is_always_equal::value) {
@@ -1016,7 +1075,8 @@ inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(forward_l
 }
 
 template <class _Tp, class _Alloc>
-inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(initializer_list<value_type> __il) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline forward_list<_Tp, _Alloc>&
+forward_list<_Tp, _Alloc>::operator=(initializer_list<value_type> __il) {
   assign(__il.begin(), __il.end());
   return *this;
 }
@@ -1025,13 +1085,14 @@ inline forward_list<_Tp, _Alloc>& forward_list<_Tp, _Alloc>::operator=(initializ
 
 template <class _Tp, class _Alloc>
 template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
-void forward_list<_Tp, _Alloc>::assign(_InputIterator __f, _InputIterator __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::assign(_InputIterator __f, _InputIterator __l) {
   __assign_with_sentinel(__f, __l);
 }
 
 template <class _Tp, class _Alloc>
 template <class _Iter, class _Sent>
-_LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::__assign_with_sentinel(_Iter __f, _Sent __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+forward_list<_Tp, _Alloc>::__assign_with_sentinel(_Iter __f, _Sent __l) {
   iterator __i = before_begin();
   iterator __j = std::next(__i);
   iterator __e = end();
@@ -1044,7 +1105,7 @@ _LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::__assign_with_sentinel(_It
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::assign(size_type __n, const value_type& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::assign(size_type __n, const value_type& __v) {
   iterator __i = before_begin();
   iterator __j = std::next(__i);
   iterator __e = end();
@@ -1059,18 +1120,19 @@ void forward_list<_Tp, _Alloc>::assign(size_type __n, const value_type& __v) {
 #  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-inline void forward_list<_Tp, _Alloc>::assign(initializer_list<value_type> __il) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void forward_list<_Tp, _Alloc>::assign(initializer_list<value_type> __il) {
   assign(__il.begin(), __il.end());
 }
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX26
 #    if _LIBCPP_STD_VER >= 17
-typename forward_list<_Tp, _Alloc>::reference
+    typename forward_list<_Tp, _Alloc>::reference
 #    else
-void
+    void
 #    endif
-forward_list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
+    forward_list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
   __base::__before_begin()->__next_ =
       this->__create_node(/* next = */ __base::__before_begin()->__next_, std::forward<_Args>(__args)...);
 #    if _LIBCPP_STD_VER >= 17
@@ -1079,7 +1141,7 @@ forward_list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::push_front(value_type&& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::push_front(value_type&& __v) {
   __base::__before_begin()->__next_ =
       this->__create_node(/* next = */ __base::__before_begin()->__next_, std::move(__v));
 }
@@ -1087,12 +1149,12 @@ void forward_list<_Tp, _Alloc>::push_front(value_type&& __v) {
 #  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::push_front(const value_type& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::push_front(const value_type& __v) {
   __base::__before_begin()->__next_ = this->__create_node(/* next = */ __base::__before_begin()->__next_, __v);
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::pop_front() {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::pop_front() {
   _LIBCPP_ASSERT_NON_NULL(!empty(), "forward_list::pop_front called on an empty list");
   __node_pointer __p                = __base::__before_begin()->__next_;
   __base::__before_begin()->__next_ = __p->__next_;
@@ -1103,7 +1165,7 @@ void forward_list<_Tp, _Alloc>::pop_front() {
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
-typename forward_list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::emplace_after(const_iterator __p, _Args&&... __args) {
   __begin_node_pointer const __r = __p.__get_begin();
   __r->__next_                   = this->__create_node(/* next = */ __r->__next_, std::forward<_Args>(__args)...);
@@ -1111,7 +1173,7 @@ forward_list<_Tp, _Alloc>::emplace_after(const_iterator __p, _Args&&... __args)
 }
 
 template <class _Tp, class _Alloc>
-typename forward_list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, value_type&& __v) {
   __begin_node_pointer const __r = __p.__get_begin();
   __r->__next_                   = this->__create_node(/* next = */ __r->__next_, std::move(__v));
@@ -1121,7 +1183,7 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, value_type&& __v) {
 #  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-typename forward_list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, const value_type& __v) {
   __begin_node_pointer const __r = __p.__get_begin();
   __r->__next_                   = this->__create_node(/* next = */ __r->__next_, __v);
@@ -1129,17 +1191,18 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, const value_type& __
 }
 
 template <class _Tp, class _Alloc>
-typename forward_list<_Tp, _Alloc>::iterator
-forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, size_type __n, const value_type& __v) {
+template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::__insert_after(const_iterator __p, size_type __n, _Args&&... __args) {
   __begin_node_pointer __r = __p.__get_begin();
   if (__n > 0) {
-    __node_pointer __first = this->__create_node(/* next = */ nullptr, __v);
+    __node_pointer __first = this->__create_node(/* next = */ nullptr, std::forward<_Args>(__args)...);
     __node_pointer __last  = __first;
 #  if _LIBCPP_HAS_EXCEPTIONS
     try {
 #  endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, __last = __last->__next_) {
-        __last->__next_ = this->__create_node(/* next = */ nullptr, __v);
+        __last->__next_ = this->__create_node(/* next = */ nullptr, std::forward<_Args>(__args)...);
       }
 #  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
@@ -1153,21 +1216,21 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, size_type __n, const
 #  endif // _LIBCPP_HAS_EXCEPTIONS
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
-    __r             = static_cast<__begin_node_pointer>(__last);
+    __r             = __forward_node_traits<__node_pointer>::__as_iter_node(__last);
   }
   return iterator(__r);
 }
 
 template <class _Tp, class _Alloc>
 template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> >
-typename forward_list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, _InputIterator __f, _InputIterator __l) {
   return __insert_after_with_sentinel(__p, std::move(__f), std::move(__l));
 }
 
 template <class _Tp, class _Alloc>
 template <class _InputIterator, class _Sentinel>
-_LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _InputIterator __f, _Sentinel __l) {
   __begin_node_pointer __r = __p.__get_begin();
 
@@ -1194,14 +1257,15 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
 
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
-    __r             = static_cast<__begin_node_pointer>(__last);
+    __r             = __forward_node_traits<__node_pointer>::__as_iter_node(__last);
   }
 
   return iterator(__r);
 }
 
 template <class _Tp, class _Alloc>
-typename forward_list<_Tp, _Alloc>::iterator forward_list<_Tp, _Alloc>::erase_after(const_iterator __f) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
+forward_list<_Tp, _Alloc>::erase_after(const_iterator __f) {
   __begin_node_pointer __p = __f.__get_begin();
   __node_pointer __n       = __p->__next_;
   __p->__next_             = __n->__next_;
@@ -1210,7 +1274,7 @@ typename forward_list<_Tp, _Alloc>::iterator forward_list<_Tp, _Alloc>::erase_af
 }
 
 template <class _Tp, class _Alloc>
-typename forward_list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::erase_after(const_iterator __f, const_iterator __l) {
   __node_pointer __e = __l.__get_unsafe_node_pointer();
   if (__f != __l) {
@@ -1230,7 +1294,7 @@ forward_list<_Tp, _Alloc>::erase_after(const_iterator __f, const_iterator __l) {
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::resize(size_type __n) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::resize(size_type __n) {
   size_type __sz = 0;
   iterator __p   = before_begin();
   iterator __i   = begin();
@@ -1239,18 +1303,12 @@ void forward_list<_Tp, _Alloc>::resize(size_type __n) {
     ;
   if (__i != __e)
     erase_after(__p, __e);
-  else {
-    __n -= __sz;
-    if (__n > 0) {
-      for (__begin_node_pointer __ptr = __p.__get_begin(); __n > 0; --__n, __ptr = __ptr->__next_as_begin()) {
-        __ptr->__next_ = this->__create_node(/* next = */ nullptr);
-      }
-    }
-  }
+  else
+    __insert_after(__p, __n - __sz);
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::resize(size_type __n, const value_type& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::resize(size_type __n, const value_type& __v) {
   size_type __sz = 0;
   iterator __p   = before_begin();
   iterator __i   = begin();
@@ -1259,18 +1317,12 @@ void forward_list<_Tp, _Alloc>::resize(size_type __n, const value_type& __v) {
     ;
   if (__i != __e)
     erase_after(__p, __e);
-  else {
-    __n -= __sz;
-    if (__n > 0) {
-      for (__begin_node_pointer __ptr = __p.__get_begin(); __n > 0; --__n, __ptr = __ptr->__next_as_begin()) {
-        __ptr->__next_ = this->__create_node(/* next = */ nullptr, __v);
-      }
-    }
-  }
+  else
+    __insert_after(__p, __n - __sz, __v);
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& __x) {
   if (!__x.empty()) {
     if (__p.__get_begin()->__next_ != nullptr) {
       const_iterator __lm1 = __x.before_begin();
@@ -1284,7 +1336,8 @@ void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& _
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& /*__other*/, const_iterator __i) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void
+forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& /*__other*/, const_iterator __i) {
   const_iterator __lm1 = std::next(__i);
   if (__p != __i && __p != __lm1) {
     __i.__get_begin()->__next_   = __lm1.__get_begin()->__next_;
@@ -1294,7 +1347,7 @@ void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& /
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::splice_after(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::splice_after(
     const_iterator __p, forward_list& /*__other*/, const_iterator __f, const_iterator __l) {
   if (__f != __l && __p != __f) {
     const_iterator __lm1 = __f;
@@ -1309,24 +1362,26 @@ void forward_list<_Tp, _Alloc>::splice_after(
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list&& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI void
+forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list&& __x) {
   splice_after(__p, __x);
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI void
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI void
 forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list&& __x, const_iterator __i) {
   splice_after(__p, __x, __i);
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::splice_after(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI void forward_list<_Tp, _Alloc>::splice_after(
     const_iterator __p, forward_list&& __x, const_iterator __f, const_iterator __l) {
   splice_after(__p, __x, __f, __l);
 }
 
 template <class _Tp, class _Alloc>
-typename forward_list<_Tp, _Alloc>::__remove_return_type forward_list<_Tp, _Alloc>::remove(const value_type& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::__remove_return_type
+forward_list<_Tp, _Alloc>::remove(const value_type& __v) {
   forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
   const iterator __e                                            = end();
@@ -1349,7 +1404,8 @@ typename forward_list<_Tp, _Alloc>::__remove_return_type forward_list<_Tp, _Allo
 
 template <class _Tp, class _Alloc>
 template <class _Predicate>
-typename forward_list<_Tp, _Alloc>::__remove_return_type forward_list<_Tp, _Alloc>::remove_if(_Predicate __pred) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::__remove_return_type
+forward_list<_Tp, _Alloc>::remove_if(_Predicate __pred) {
   forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
   const iterator __e                                            = end();
@@ -1372,7 +1428,7 @@ typename forward_list<_Tp, _Alloc>::__remove_return_type forward_list<_Tp, _Allo
 
 template <class _Tp, class _Alloc>
 template <class _BinaryPredicate>
-typename forward_list<_Tp, _Alloc>::__remove_return_type
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::__remove_return_type
 forward_list<_Tp, _Alloc>::unique(_BinaryPredicate __binary_pred) {
   forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
@@ -1390,7 +1446,7 @@ forward_list<_Tp, _Alloc>::unique(_BinaryPredicate __binary_pred) {
 
 template <class _Tp, class _Alloc>
 template <class _Compare>
-void forward_list<_Tp, _Alloc>::merge(forward_list& __x, _Compare __comp) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::merge(forward_list& __x, _Compare __comp) {
   if (this != std::addressof(__x)) {
     __base::__before_begin()->__next_ =
         __merge(__base::__before_begin()->__next_, __x.__before_begin()->__next_, __comp);
@@ -1400,7 +1456,7 @@ void forward_list<_Tp, _Alloc>::merge(forward_list& __x, _Compare __comp) {
 
 template <class _Tp, class _Alloc>
 template <class _Compare>
-typename forward_list<_Tp, _Alloc>::__node_pointer
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::__node_pointer
 forward_list<_Tp, _Alloc>::__merge(__node_pointer __f1, __node_pointer __f2, _Compare& __comp) {
   if (__f1 == nullptr)
     return __f2;
@@ -1437,13 +1493,13 @@ forward_list<_Tp, _Alloc>::__merge(__node_pointer __f1, __node_pointer __f2, _Co
 
 template <class _Tp, class _Alloc>
 template <class _Compare>
-inline void forward_list<_Tp, _Alloc>::sort(_Compare __comp) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void forward_list<_Tp, _Alloc>::sort(_Compare __comp) {
   __base::__before_begin()->__next_ = __sort(__base::__before_begin()->__next_, std::distance(begin(), end()), __comp);
 }
 
 template <class _Tp, class _Alloc>
 template <class _Compare>
-typename forward_list<_Tp, _Alloc>::__node_pointer
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::__node_pointer
 forward_list<_Tp, _Alloc>::__sort(__node_pointer __f1, difference_type __sz, _Compare& __comp) {
   switch (__sz) {
   case 0:
@@ -1467,7 +1523,7 @@ forward_list<_Tp, _Alloc>::__sort(__node_pointer __f1, difference_type __sz, _Co
 }
 
 template <class _Tp, class _Alloc>
-void forward_list<_Tp, _Alloc>::reverse() _NOEXCEPT {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::reverse() _NOEXCEPT {
   __node_pointer __p = __base::__before_begin()->__next_;
   if (__p != nullptr) {
     __node_pointer __f = __p->__next_;
@@ -1483,7 +1539,8 @@ void forward_list<_Tp, _Alloc>::reverse() _NOEXCEPT {
 }
 
 template <class _Tp, class _Alloc>
-_LIBCPP_HIDE_FROM_ABI bool operator==(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
+operator==(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
   typedef forward_list<_Tp, _Alloc> _Cp;
   typedef typename _Cp::const_iterator _Ip;
   _Ip __ix = __x.begin();
@@ -1499,31 +1556,31 @@ _LIBCPP_HIDE_FROM_ABI bool operator==(const forward_list<_Tp, _Alloc>& __x, cons
 #  if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
 operator!=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
   return !(__x == __y);
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
 operator<(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
   return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
 operator>(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
   return __y < __x;
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
 operator>=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
   return !(__x < __y);
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
 operator<=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>& __y) {
   return !(__y < __x);
 }
@@ -1531,7 +1588,7 @@ operator<=(const forward_list<_Tp, _Alloc>& __x, const forward_list<_Tp, _Alloc>
 #  else // #if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Allocator>
-_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
+_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
 operator<=>(const forward_list<_Tp, _Allocator>& __x, const forward_list<_Tp, _Allocator>& __y) {
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
@@ -1539,20 +1596,20 @@ operator<=>(const forward_list<_Tp, _Allocator>& __x, const forward_list<_Tp, _A
 #  endif // #if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI void swap(forward_list<_Tp, _Alloc>& __x, forward_list<_Tp, _Alloc>& __y)
-    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI void
+swap(forward_list<_Tp, _Alloc>& __x, forward_list<_Tp, _Alloc>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
   __x.swap(__y);
 }
 
 #  if _LIBCPP_STD_VER >= 20
 template <class _Tp, class _Allocator, class _Predicate>
-inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
 erase_if(forward_list<_Tp, _Allocator>& __c, _Predicate __pred) {
   return __c.remove_if(__pred);
 }
 
 template <class _Tp, class _Allocator, class _Up>
-inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Allocator>::size_type
 erase(forward_list<_Tp, _Allocator>& __c, const _Up& __v) {
   return std::erase_if(__c, [&](const auto& __elem) -> bool { return __elem == __v; });
 }
@@ -1567,6 +1624,8 @@ struct __container_traits<forward_list<_Tp, _Allocator> > {
   // - If an exception is thrown by an insert() or emplace() function while inserting a single element, that
   // function has no effects.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/ios b/external/llvm-project/libcxx/include/ios
index 9e48ec88ce59..78fc9d610987 100644
--- a/external/llvm-project/libcxx/include/ios
+++ b/external/llvm-project/libcxx/include/ios
@@ -216,6 +216,11 @@ storage-class-specifier const error_category& iostream_category() noexcept;
 #else
 #  include <__config>
 
+// standard-mandated includes
+
+// [ios.syn]
+#  include <iosfwd>
+
 #  if _LIBCPP_HAS_LOCALIZATION
 
 #    include <__fwd/ios.h>
@@ -230,11 +235,6 @@ storage-class-specifier const error_category& iostream_category() noexcept;
 #    include <__verbose_abort>
 #    include <version>
 
-// standard-mandated includes
-
-// [ios.syn]
-#    include <iosfwd>
-
 #    if _LIBCPP_HAS_ATOMIC_HEADER
 #      include <__atomic/atomic.h> // for __xindex_
 #    endif
diff --git a/external/llvm-project/libcxx/include/list b/external/llvm-project/libcxx/include/list
index d1da347b9bd1..98610f59ed74 100644
--- a/external/llvm-project/libcxx/include/list
+++ b/external/llvm-project/libcxx/include/list
@@ -1718,6 +1718,8 @@ struct __container_traits<list<_Tp, _Allocator> > {
   // - If an exception is thrown by an insert() or emplace() function while inserting a single element, that
   // function has no effects.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/map b/external/llvm-project/libcxx/include/map
index 24eadbd15422..73bf672d3a08 100644
--- a/external/llvm-project/libcxx/include/map
+++ b/external/llvm-project/libcxx/include/map
@@ -750,21 +750,9 @@ public:
   }
 };
 
-template <class _TreeIterator>
-class __map_const_iterator;
-
 template <class _Key, class _Tp>
 struct __value_type;
 
-template <class _Tp>
-struct __extract_key_value_types;
-
-template <class _Key, class _Tp>
-struct __extract_key_value_types<__value_type<_Key, _Tp> > {
-  typedef _Key const __key_type;
-  typedef _Tp __mapped_type;
-};
-
 template <class _TreeIterator>
 class __map_iterator {
   typedef typename _TreeIterator::_NodeTypes _NodeTypes;
@@ -1560,6 +1548,8 @@ struct __container_traits<map<_Key, _Tp, _Compare, _Allocator> > {
   // For associative containers, if an exception is thrown by any operation from within
   // an insert or emplace function inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
@@ -2083,6 +2073,8 @@ struct __container_traits<multimap<_Key, _Tp, _Compare, _Allocator> > {
   // For associative containers, if an exception is thrown by any operation from within
   // an insert or emplace function inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/mdspan b/external/llvm-project/libcxx/include/mdspan
index 43982e0f339b..5aeec1bcbfc3 100644
--- a/external/llvm-project/libcxx/include/mdspan
+++ b/external/llvm-project/libcxx/include/mdspan
@@ -450,7 +450,12 @@ namespace std {
 #  include <__config>
 
 #  if _LIBCPP_STD_VER >= 23
-#    include <__fwd/mdspan.h>
+#    include <__fwd/mdspan.h> // TODO(boomanaiden154): This is currently a
+                              // non-standard extension to include
+                              // std::dynamic_extent tracked by LWG issue 4275.
+                              // This comment should be deleted or the include
+                              // deleted upon resolution.
+#    include <__fwd/span.h>
 #    include <__mdspan/default_accessor.h>
 #    include <__mdspan/extents.h>
 #    include <__mdspan/layout_left.h>
diff --git a/external/llvm-project/libcxx/include/module.modulemap.in b/external/llvm-project/libcxx/include/module.modulemap.in
index 7f625cefed1c..f5fd970934e9 100644
--- a/external/llvm-project/libcxx/include/module.modulemap.in
+++ b/external/llvm-project/libcxx/include/module.modulemap.in
@@ -94,6 +94,7 @@ module std_core [system] {
     module extent                                     { header "__type_traits/extent.h" }
     module has_unique_object_representation           { header "__type_traits/has_unique_object_representation.h" }
     module has_virtual_destructor                     { header "__type_traits/has_virtual_destructor.h" }
+    module integer_traits                             { header "__type_traits/integer_traits.h" }
     module integral_constant                          { header "__type_traits/integral_constant.h" }
     module invoke                                     { header "__type_traits/invoke.h" }
     module is_abstract {
@@ -284,10 +285,6 @@ module std_core [system] {
       header "__type_traits/is_scalar.h"
       export std_core.type_traits.integral_constant
     }
-    module is_signed_integer {
-      header "__type_traits/is_signed_integer.h"
-      export std_core.type_traits.integral_constant
-    }
     module is_signed {
       header "__type_traits/is_signed.h"
       export std_core.type_traits.integral_constant
@@ -340,10 +337,6 @@ module std_core [system] {
       header "__type_traits/is_union.h"
       export std_core.type_traits.integral_constant
     }
-    module is_unsigned_integer {
-      header "__type_traits/is_unsigned_integer.h"
-      export std_core.type_traits.integral_constant
-    }
     module is_unsigned {
       header "__type_traits/is_unsigned.h"
       export std_core.type_traits.integral_constant
diff --git a/external/llvm-project/libcxx/include/optional b/external/llvm-project/libcxx/include/optional
index 2153efb2ab89..fa32d75ef86d 100644
--- a/external/llvm-project/libcxx/include/optional
+++ b/external/llvm-project/libcxx/include/optional
@@ -247,7 +247,7 @@ _LIBCPP_PUSH_MACROS
 namespace std // purposefully not using versioning namespace
 {
 
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS bad_optional_access : public exception {
+class _LIBCPP_EXPORTED_FROM_ABI bad_optional_access : public exception {
 public:
   _LIBCPP_HIDE_FROM_ABI bad_optional_access() _NOEXCEPT                                      = default;
   _LIBCPP_HIDE_FROM_ABI bad_optional_access(const bad_optional_access&) _NOEXCEPT            = default;
@@ -263,8 +263,7 @@ public:
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-[[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS void
-__throw_bad_optional_access() {
+[[noreturn]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_optional_access() {
 #    if _LIBCPP_HAS_EXCEPTIONS
   throw bad_optional_access();
 #    else
@@ -827,25 +826,25 @@ public:
   using __base::__get;
   using __base::has_value;
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type const& value() const& {
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type const& value() const& {
     if (!this->has_value())
       std::__throw_bad_optional_access();
     return this->__get();
   }
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type& value() & {
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type& value() & {
     if (!this->has_value())
       std::__throw_bad_optional_access();
     return this->__get();
   }
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type&& value() && {
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type&& value() && {
     if (!this->has_value())
       std::__throw_bad_optional_access();
     return std::move(this->__get());
   }
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr value_type const&& value() const&& {
+  _LIBCPP_HIDE_FROM_ABI constexpr value_type const&& value() const&& {
     if (!this->has_value())
       std::__throw_bad_optional_access();
     return std::move(this->__get());
@@ -867,7 +866,7 @@ public:
 
 #    if _LIBCPP_STD_VER >= 23
   template <class _Func>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto and_then(_Func&& __f) & {
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
     using _Up = invoke_result_t<_Func, value_type&>;
     static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
                   "Result of f(value()) must be a specialization of std::optional");
@@ -877,7 +876,7 @@ public:
   }
 
   template <class _Func>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto and_then(_Func&& __f) const& {
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
     using _Up = invoke_result_t<_Func, const value_type&>;
     static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
                   "Result of f(value()) must be a specialization of std::optional");
@@ -887,7 +886,7 @@ public:
   }
 
   template <class _Func>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto and_then(_Func&& __f) && {
+  _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
     using _Up = invoke_result_t<_Func, value_type&&>;
     static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
                   "Result of f(std::move(value())) must be a specialization of std::optional");
@@ -907,7 +906,7 @@ public:
   }
 
   template <class _Func>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) & {
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
     using _Up = remove_cv_t<invoke_result_t<_Func, value_type&>>;
     static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
     static_assert(!is_same_v<_Up, in_place_t>, "Result of f(value()) should not be std::in_place_t");
@@ -919,7 +918,7 @@ public:
   }
 
   template <class _Func>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) const& {
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
     using _Up = remove_cv_t<invoke_result_t<_Func, const value_type&>>;
     static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
     static_assert(!is_same_v<_Up, in_place_t>, "Result of f(value()) should not be std::in_place_t");
@@ -931,7 +930,7 @@ public:
   }
 
   template <class _Func>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) && {
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
     using _Up = remove_cv_t<invoke_result_t<_Func, value_type&&>>;
     static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
     static_assert(!is_same_v<_Up, in_place_t>, "Result of f(std::move(value())) should not be std::in_place_t");
@@ -943,7 +942,7 @@ public:
   }
 
   template <class _Func>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS constexpr auto transform(_Func&& __f) const&& {
+  _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
     using _Up = remove_cvref_t<invoke_result_t<_Func, const value_type&&>>;
     static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
     static_assert(!is_same_v<_Up, in_place_t>, "Result of f(std::move(value())) should not be std::in_place_t");
diff --git a/external/llvm-project/libcxx/include/queue b/external/llvm-project/libcxx/include/queue
index 7043a84390d0..c33afc892dda 100644
--- a/external/llvm-project/libcxx/include/queue
+++ b/external/llvm-project/libcxx/include/queue
@@ -458,14 +458,12 @@ template <class _InputIterator,
           class _Alloc,
           __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0,
           __enable_if_t<__is_allocator<_Alloc>::value, int>                        = 0>
-queue(_InputIterator,
-      _InputIterator,
-      _Alloc) -> queue<__iter_value_type<_InputIterator>, deque<__iter_value_type<_InputIterator>, _Alloc>>;
+queue(_InputIterator, _InputIterator, _Alloc)
+    -> queue<__iter_value_type<_InputIterator>, deque<__iter_value_type<_InputIterator>, _Alloc>>;
 
 template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
-queue(from_range_t,
-      _Range&&,
-      _Alloc) -> queue<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
+queue(from_range_t, _Range&&, _Alloc)
+    -> queue<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
 #  endif
 
 template <class _Tp, class _Container>
@@ -533,24 +531,25 @@ protected:
   value_compare comp;
 
 public:
-  _LIBCPP_HIDE_FROM_ABI priority_queue() _NOEXCEPT_(
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue() _NOEXCEPT_(
       is_nothrow_default_constructible<container_type>::value&& is_nothrow_default_constructible<value_compare>::value)
       : c(), comp() {}
 
-  _LIBCPP_HIDE_FROM_ABI priority_queue(const priority_queue& __q) : c(__q.c), comp(__q.comp) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(const priority_queue& __q)
+      : c(__q.c), comp(__q.comp) {}
 
-  _LIBCPP_HIDE_FROM_ABI priority_queue& operator=(const priority_queue& __q) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue& operator=(const priority_queue& __q) {
     c    = __q.c;
     comp = __q.comp;
     return *this;
   }
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q) noexcept(
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q) noexcept(
       is_nothrow_move_constructible<container_type>::value && is_nothrow_move_constructible<value_compare>::value)
       : c(std::move(__q.c)), comp(std::move(__q.comp)) {}
 
-  _LIBCPP_HIDE_FROM_ABI priority_queue& operator=(priority_queue&& __q) noexcept(
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue& operator=(priority_queue&& __q) noexcept(
       is_nothrow_move_assignable<container_type>::value && is_nothrow_move_assignable<value_compare>::value) {
     c    = std::move(__q.c);
     comp = std::move(__q.comp);
@@ -558,50 +557,56 @@ public:
   }
 #  endif // _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const value_compare& __comp) : c(), comp(__comp) {}
-  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const container_type& __c);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const value_compare& __comp)
+      : c(), comp(__comp) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  priority_queue(const value_compare& __comp, const container_type& __c);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, container_type&& __c);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, container_type&& __c);
 #  endif
   template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp = value_compare());
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp = value_compare());
 
   template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
   priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c);
 
 #  ifndef _LIBCPP_CXX03_LANG
   template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
   priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c);
 #  endif // _LIBCPP_CXX03_LANG
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const value_compare& __comp = value_compare())
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  priority_queue(from_range_t, _Range&& __range, const value_compare& __comp = value_compare())
       : c(from_range, std::forward<_Range>(__range)), comp(__comp) {
     std::make_heap(c.begin(), c.end(), comp);
   }
 #  endif
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit priority_queue(const _Alloc& __a);
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const _Alloc& __a);
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, const container_type& __c, const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  priority_queue(const value_compare& __comp, const container_type& __c, const _Alloc& __a);
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(const priority_queue& __q, const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(const priority_queue& __q, const _Alloc& __a);
 
 #  ifndef _LIBCPP_CXX03_LANG
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(const value_compare& __comp, container_type&& __c, const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  priority_queue(const value_compare& __comp, container_type&& __c, const _Alloc& __a);
 
   template <class _Alloc, __enable_if_t<uses_allocator<container_type, _Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q, const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(priority_queue&& __q, const _Alloc& __a);
 #  endif // _LIBCPP_CXX03_LANG
 
   template <
@@ -609,21 +614,22 @@ public:
       class _Alloc,
       __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
                     int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const _Alloc& __a);
 
   template <
       class _InputIter,
       class _Alloc,
       __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
                     int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, const _Alloc& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, const _Alloc& __a);
 
   template <
       class _InputIter,
       class _Alloc,
       __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
                     int> = 0>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(
       _InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c, const _Alloc& __a);
 
 #  ifndef _LIBCPP_CXX03_LANG
@@ -632,7 +638,7 @@ public:
       class _Alloc,
       __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<container_type, _Alloc>::value,
                     int> = 0>
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
   priority_queue(_InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c, const _Alloc& __a);
 #  endif // _LIBCPP_CXX03_LANG
 
@@ -641,7 +647,8 @@ public:
   template <_ContainerCompatibleRange<_Tp> _Range,
             class _Alloc,
             class = enable_if_t<uses_allocator<_Container, _Alloc>::value>>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const value_compare& __comp, const _Alloc& __a)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  priority_queue(from_range_t, _Range&& __range, const value_compare& __comp, const _Alloc& __a)
       : c(from_range, std::forward<_Range>(__range), __a), comp(__comp) {
     std::make_heap(c.begin(), c.end(), comp);
   }
@@ -649,24 +656,24 @@ public:
   template <_ContainerCompatibleRange<_Tp> _Range,
             class _Alloc,
             class = enable_if_t<uses_allocator<_Container, _Alloc>::value>>
-  _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const _Alloc& __a)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI priority_queue(from_range_t, _Range&& __range, const _Alloc& __a)
       : c(from_range, std::forward<_Range>(__range), __a), comp() {
     std::make_heap(c.begin(), c.end(), comp);
   }
 
 #  endif
 
-  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
-  _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
-  _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.front(); }
+  [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.front(); }
 
-  _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push(value_type&& __v);
 
 #    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push_range(_Range&& __range) {
     if constexpr (requires(container_type& __c) { __c.append_range(std::forward<_Range>(__range)); }) {
       c.append_range(std::forward<_Range>(__range));
     } else {
@@ -678,14 +685,16 @@ public:
 #    endif
 
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI void emplace(_Args&&... __args);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void emplace(_Args&&... __args);
 #  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void pop();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void pop();
 
-  _LIBCPP_HIDE_FROM_ABI void swap(priority_queue& __q)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void swap(priority_queue& __q)
       _NOEXCEPT_(__is_nothrow_swappable_v<container_type>&& __is_nothrow_swappable_v<value_compare>);
 
-  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const _Container& __get_container() const { return c; }
+  [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const _Container& __get_container() const {
+    return c;
+  }
 };
 
 #  if _LIBCPP_STD_VER >= 17
@@ -767,7 +776,8 @@ priority_queue(from_range_t, _Range&&, _Alloc)
 #  endif
 
 template <class _Tp, class _Container, class _Compare>
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Compare& __comp, const container_type& __c)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    const _Compare& __comp, const container_type& __c)
     : c(__c), comp(__comp) {
   std::make_heap(c.begin(), c.end(), comp);
 }
@@ -775,7 +785,8 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Compare&
 #  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const value_compare& __comp, container_type&& __c)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    const value_compare& __comp, container_type&& __c)
     : c(std::move(__c)), comp(__comp) {
   std::make_heap(c.begin(), c.end(), comp);
 }
@@ -784,7 +795,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const value_com
 
 template <class _Tp, class _Container, class _Compare>
 template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     _InputIter __f, _InputIter __l, const value_compare& __comp)
     : c(__f, __l), comp(__comp) {
   std::make_heap(c.begin(), c.end(), comp);
@@ -792,7 +803,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
 
 template <class _Tp, class _Container, class _Compare>
 template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     _InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c)
     : c(__c), comp(__comp) {
   c.insert(c.end(), __f, __l);
@@ -803,7 +814,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
 
 template <class _Tp, class _Container, class _Compare>
 template <class _InputIter, __enable_if_t<__has_input_iterator_category<_InputIter>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     _InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c)
     : c(std::move(__c)), comp(__comp) {
   c.insert(c.end(), __f, __l);
@@ -814,16 +825,18 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Alloc& __a) : c(__a) {}
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const _Alloc& __a)
+    : c(__a) {}
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const value_compare& __comp, const _Alloc& __a)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    const value_compare& __comp, const _Alloc& __a)
     : c(__a), comp(__comp) {}
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     const value_compare& __comp, const container_type& __c, const _Alloc& __a)
     : c(__c, __a), comp(__comp) {
   std::make_heap(c.begin(), c.end(), comp);
@@ -831,14 +844,15 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(const priority_queue& __q, const _Alloc& __a)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    const priority_queue& __q, const _Alloc& __a)
     : c(__q.c, __a), comp(__q.comp) {}
 
 #  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     const value_compare& __comp, container_type&& __c, const _Alloc& __a)
     : c(std::move(__c), __a), comp(__comp) {
   std::make_heap(c.begin(), c.end(), comp);
@@ -846,7 +860,8 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
 
 template <class _Tp, class _Container, class _Compare>
 template <class _Alloc, __enable_if_t<uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(priority_queue&& __q, const _Alloc& __a)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    priority_queue&& __q, const _Alloc& __a)
     : c(std::move(__q.c), __a), comp(std::move(__q.comp)) {}
 
 #  endif // _LIBCPP_CXX03_LANG
@@ -856,7 +871,8 @@ template <
     class _InputIter,
     class _Alloc,
     __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(_InputIter __f, _InputIter __l, const _Alloc& __a)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+    _InputIter __f, _InputIter __l, const _Alloc& __a)
     : c(__f, __l, __a), comp() {
   std::make_heap(c.begin(), c.end(), comp);
 }
@@ -866,7 +882,7 @@ template <
     class _InputIter,
     class _Alloc,
     __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     _InputIter __f, _InputIter __l, const value_compare& __comp, const _Alloc& __a)
     : c(__f, __l, __a), comp(__comp) {
   std::make_heap(c.begin(), c.end(), comp);
@@ -877,7 +893,7 @@ template <
     class _InputIter,
     class _Alloc,
     __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     _InputIter __f, _InputIter __l, const value_compare& __comp, const container_type& __c, const _Alloc& __a)
     : c(__c, __a), comp(__comp) {
   c.insert(c.end(), __f, __l);
@@ -890,7 +906,7 @@ template <
     class _InputIter,
     class _Alloc,
     __enable_if_t<__has_input_iterator_category<_InputIter>::value && uses_allocator<_Container, _Alloc>::value, int> >
-inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
     _InputIter __f, _InputIter __l, const value_compare& __comp, container_type&& __c, const _Alloc& __a)
     : c(std::move(__c), __a), comp(__comp) {
   c.insert(c.end(), __f, __l);
@@ -899,7 +915,7 @@ inline priority_queue<_Tp, _Container, _Compare>::priority_queue(
 #  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
-inline void priority_queue<_Tp, _Container, _Compare>::push(const value_type& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void priority_queue<_Tp, _Container, _Compare>::push(const value_type& __v) {
   c.push_back(__v);
   std::push_heap(c.begin(), c.end(), comp);
 }
@@ -907,14 +923,14 @@ inline void priority_queue<_Tp, _Container, _Compare>::push(const value_type& __
 #  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
-inline void priority_queue<_Tp, _Container, _Compare>::push(value_type&& __v) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void priority_queue<_Tp, _Container, _Compare>::push(value_type&& __v) {
   c.push_back(std::move(__v));
   std::push_heap(c.begin(), c.end(), comp);
 }
 
 template <class _Tp, class _Container, class _Compare>
 template <class... _Args>
-inline void priority_queue<_Tp, _Container, _Compare>::emplace(_Args&&... __args) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void priority_queue<_Tp, _Container, _Compare>::emplace(_Args&&... __args) {
   c.emplace_back(std::forward<_Args>(__args)...);
   std::push_heap(c.begin(), c.end(), comp);
 }
@@ -922,13 +938,13 @@ inline void priority_queue<_Tp, _Container, _Compare>::emplace(_Args&&... __args
 #  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Container, class _Compare>
-inline void priority_queue<_Tp, _Container, _Compare>::pop() {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void priority_queue<_Tp, _Container, _Compare>::pop() {
   std::pop_heap(c.begin(), c.end(), comp);
   c.pop_back();
 }
 
 template <class _Tp, class _Container, class _Compare>
-inline void priority_queue<_Tp, _Container, _Compare>::swap(priority_queue& __q)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void priority_queue<_Tp, _Container, _Compare>::swap(priority_queue& __q)
     _NOEXCEPT_(__is_nothrow_swappable_v<container_type>&& __is_nothrow_swappable_v<value_compare>) {
   using std::swap;
   swap(c, __q.c);
@@ -939,7 +955,7 @@ template <class _Tp,
           class _Container,
           class _Compare,
           __enable_if_t<__is_swappable_v<_Container> && __is_swappable_v<_Compare>, int> = 0>
-inline _LIBCPP_HIDE_FROM_ABI void
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI void
 swap(priority_queue<_Tp, _Container, _Compare>& __x, priority_queue<_Tp, _Container, _Compare>& __y)
     _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
   __x.swap(__y);
diff --git a/external/llvm-project/libcxx/include/regex b/external/llvm-project/libcxx/include/regex
index 12fc0787dcd5..bbc21e244dd1 100644
--- a/external/llvm-project/libcxx/include/regex
+++ b/external/llvm-project/libcxx/include/regex
@@ -794,6 +794,19 @@ typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
 #else
 #  include <__config>
 
+// standard-mandated includes
+
+// [iterator.range]
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
+
+// [re.syn]
+#  include <compare>
+#  include <initializer_list>
+
 #  if _LIBCPP_HAS_LOCALIZATION
 
 #    include <__algorithm/find.h>
@@ -817,19 +830,6 @@ typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
 #    include <vector>
 #    include <version>
 
-// standard-mandated includes
-
-// [iterator.range]
-#    include <__iterator/access.h>
-#    include <__iterator/data.h>
-#    include <__iterator/empty.h>
-#    include <__iterator/reverse_access.h>
-#    include <__iterator/size.h>
-
-// [re.syn]
-#    include <compare>
-#    include <initializer_list>
-
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #      pragma GCC system_header
 #    endif
diff --git a/external/llvm-project/libcxx/include/set b/external/llvm-project/libcxx/include/set
index 83c8ed59065c..aeea98adf582 100644
--- a/external/llvm-project/libcxx/include/set
+++ b/external/llvm-project/libcxx/include/set
@@ -1030,6 +1030,8 @@ struct __container_traits<set<_Key, _Compare, _Allocator> > {
   // For associative containers, if an exception is thrown by any operation from within
   // an insert or emplace function inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 template <class _Key, class _Compare, class _Allocator>
@@ -1499,6 +1501,8 @@ struct __container_traits<multiset<_Key, _Compare, _Allocator> > {
   // For associative containers, if an exception is thrown by any operation from within
   // an insert or emplace function inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = false;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/string b/external/llvm-project/libcxx/include/string
index 4f05e211919f..c450ddcb4191 100644
--- a/external/llvm-project/libcxx/include/string
+++ b/external/llvm-project/libcxx/include/string
@@ -678,7 +678,7 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
 _LIBCPP_PUSH_MACROS
 #  include <__undef_macros>
 
-#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if __has_feature(address_sanitizer) && _LIBCPP_INSTRUMENTED_WITH_ASAN
 #    define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS __attribute__((__no_sanitize__("address")))
 // This macro disables AddressSanitizer (ASan) instrumentation for a specific function,
 // allowing memory accesses that would normally trigger ASan errors to proceed without crashing.
@@ -749,7 +749,7 @@ public:
   //
   // This string implementation doesn't contain any references into itself. It only contains a bit that says whether
   // it is in small or large string mode, so the entire structure is trivially relocatable if its members are.
-#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if __has_feature(address_sanitizer) && _LIBCPP_INSTRUMENTED_WITH_ASAN
   // When compiling with AddressSanitizer (ASan), basic_string cannot be trivially
   // relocatable. Because the object's memory might be poisoned when its content
   // is kept inside objects memory (short string optimization), instead of in allocated
@@ -771,7 +771,7 @@ public:
                       basic_string,
                       void>;
 
-#  if _LIBCPP_HAS_ASAN && _LIBCPP_INSTRUMENTED_WITH_ASAN
+#  if __has_feature(address_sanitizer) && _LIBCPP_INSTRUMENTED_WITH_ASAN
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __asan_volatile_wrapper(pointer const& __ptr) const {
     if (__libcpp_is_constant_evaluated())
       return __ptr;
@@ -3434,8 +3434,8 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocat
 
   if (__fits_in_sso(__target_capacity)) {
     __annotation_guard __g(*this);
-    traits_type::copy(std::__to_address(__get_short_pointer()), std::__to_address(__ptr), __size + 1);
     __set_short_size(__size);
+    traits_type::copy(std::__to_address(__get_short_pointer()), std::__to_address(__ptr), __size + 1);
     __alloc_traits::deallocate(__alloc_, __ptr, __cap);
     return;
   }
diff --git a/external/llvm-project/libcxx/include/syncstream b/external/llvm-project/libcxx/include/syncstream
index ffa0ed8001ef..1f7605e06aa2 100644
--- a/external/llvm-project/libcxx/include/syncstream
+++ b/external/llvm-project/libcxx/include/syncstream
@@ -122,6 +122,11 @@ namespace std {
 #else
 #  include <__config>
 
+// standard-mandated includes
+
+// [syncstream.syn]
+#  include <ostream>
+
 #  if _LIBCPP_HAS_LOCALIZATION
 
 #    include <__mutex/lock_guard.h>
@@ -130,17 +135,11 @@ namespace std {
 #    include <iosfwd> // required for declaration of default arguments
 #    include <streambuf>
 #    include <string>
-
 #    if _LIBCPP_HAS_THREADS
 #      include <map>
 #      include <shared_mutex>
 #    endif
 
-// standard-mandated includes
-
-// [syncstream.syn]
-#    include <ostream>
-
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #      pragma GCC system_header
 #    endif
diff --git a/external/llvm-project/libcxx/include/tuple b/external/llvm-project/libcxx/include/tuple
index 8dd62ae624f5..6e7a430d219e 100644
--- a/external/llvm-project/libcxx/include/tuple
+++ b/external/llvm-project/libcxx/include/tuple
@@ -258,6 +258,7 @@ template <class... Types>
 #  include <__type_traits/maybe_const.h>
 #  include <__type_traits/nat.h>
 #  include <__type_traits/negation.h>
+#  include <__type_traits/reference_constructs_from_temporary.h>
 #  include <__type_traits/remove_cv.h>
 #  include <__type_traits/remove_cvref.h>
 #  include <__type_traits/remove_reference.h>
@@ -308,15 +309,6 @@ template <size_t _Ip, class _Hp, bool>
 class __tuple_leaf {
   _Hp __value_;
 
-  template <class _Tp>
-  static _LIBCPP_HIDE_FROM_ABI constexpr bool __can_bind_reference() {
-#    if __has_keyword(__reference_binds_to_temporary)
-    return !__reference_binds_to_temporary(_Hp, _Tp);
-#    else
-    return true;
-#    endif
-  }
-
 public:
   _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_leaf& operator=(const __tuple_leaf&) = delete;
 
@@ -346,7 +338,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_leaf(_Tp&& __t) noexcept(is_nothrow_constructible<_Hp, _Tp>::value)
       : __value_(std::forward<_Tp>(__t)) {
-    static_assert(__can_bind_reference<_Tp&&>(),
+    static_assert(!__reference_constructs_from_temporary_v<_Hp, _Tp&&>,
                   "Attempted construction of reference element binds to a temporary whose lifetime has ended");
   }
 
@@ -354,7 +346,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_leaf(integral_constant<int, 0>, const _Alloc&, _Tp&& __t)
       : __value_(std::forward<_Tp>(__t)) {
-    static_assert(__can_bind_reference<_Tp&&>(),
+    static_assert(!__reference_constructs_from_temporary_v<_Hp, _Tp&&>,
                   "Attempted construction of reference element binds to a temporary whose lifetime has ended");
   }
 
diff --git a/external/llvm-project/libcxx/include/unordered_map b/external/llvm-project/libcxx/include/unordered_map
index 61c89a0ca73b..b7f333e5f117 100644
--- a/external/llvm-project/libcxx/include/unordered_map
+++ b/external/llvm-project/libcxx/include/unordered_map
@@ -1843,6 +1843,8 @@ struct __container_traits<unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc> > {
   //  inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
       __is_nothrow_invocable_v<_Hash, const _Key&>;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = true;
 };
 
 template <class _Key,
@@ -2543,6 +2545,8 @@ struct __container_traits<unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc> >
   //  inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
       __is_nothrow_invocable_v<_Hash, const _Key&>;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = true;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/unordered_set b/external/llvm-project/libcxx/include/unordered_set
index e97e6e8042e6..1412badbe37f 100644
--- a/external/llvm-project/libcxx/include/unordered_set
+++ b/external/llvm-project/libcxx/include/unordered_set
@@ -1196,6 +1196,8 @@ struct __container_traits<unordered_set<_Value, _Hash, _Pred, _Alloc> > {
   //  inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
       __is_nothrow_invocable_v<_Hash, const _Value&>;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = true;
 };
 
 template <class _Value, class _Hash = hash<_Value>, class _Pred = equal_to<_Value>, class _Alloc = allocator<_Value> >
@@ -1816,6 +1818,8 @@ struct __container_traits<unordered_multiset<_Value, _Hash, _Pred, _Alloc> > {
   //  inserting a single element, the insertion has no effect.
   static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
       __is_nothrow_invocable_v<_Hash, const _Value&>;
+
+  static _LIBCPP_CONSTEXPR const bool __reservable = true;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/external/llvm-project/libcxx/include/variant b/external/llvm-project/libcxx/include/variant
index 23d876f9a60a..dac6f786cc19 100644
--- a/external/llvm-project/libcxx/include/variant
+++ b/external/llvm-project/libcxx/include/variant
@@ -286,7 +286,7 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
 
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS bad_variant_access : public exception {
+class _LIBCPP_EXPORTED_FROM_ABI bad_variant_access : public exception {
 public:
   const char* what() const _NOEXCEPT override;
 };
@@ -307,8 +307,7 @@ struct __farray {
   _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator[](size_t __n) const noexcept { return __buf_[__n]; }
 };
 
-[[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS void
-__throw_bad_variant_access() {
+[[noreturn]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_variant_access() {
 #    if _LIBCPP_HAS_EXCEPTIONS
   throw bad_variant_access();
 #    else
@@ -1142,16 +1141,14 @@ using __best_match_t _LIBCPP_NODEBUG = typename invoke_result_t<_MakeOverloads<_
 } // namespace __variant_detail
 
 template <class _Visitor, class... _Vs, typename = void_t<decltype(std::__as_variant(std::declval<_Vs>()))...>>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr decltype(auto)
-visit(_Visitor&& __visitor, _Vs&&... __vs);
+_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs);
 
 #    if _LIBCPP_STD_VER >= 20
 template <class _Rp,
           class _Visitor,
           class... _Vs,
           typename = void_t<decltype(std::__as_variant(std::declval<_Vs>()))...>>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Rp
-visit(_Visitor&& __visitor, _Vs&&... __vs);
+_LIBCPP_HIDE_FROM_ABI constexpr _Rp visit(_Visitor&& __visitor, _Vs&&... __vs);
 #    endif
 
 template <class... _Types>
@@ -1338,7 +1335,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool holds_alternative(const variant<_Types...>&
 }
 
 template <size_t _Ip, class _Vp>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr auto&& __generic_get(_Vp&& __v) {
+_LIBCPP_HIDE_FROM_ABI constexpr auto&& __generic_get(_Vp&& __v) {
   using __variant_detail::__access::__variant;
   if (!std::__holds_alternative<_Ip>(__v)) {
     std::__throw_bad_variant_access();
@@ -1347,26 +1344,21 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr au
 }
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>&
-get(variant<_Types...>& __v) {
+_LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>& get(variant<_Types...>& __v) {
   static_assert(_Ip < sizeof...(_Types));
   static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
   return std::__generic_get<_Ip>(__v);
 }
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr variant_alternative_t<_Ip, variant<_Types...>>&&
-get(variant<_Types...>&& __v) {
+_LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>&& get(variant<_Types...>&& __v) {
   static_assert(_Ip < sizeof...(_Types));
   static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
   return std::__generic_get<_Ip>(std::move(__v));
 }
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>&
+_LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>&
 get(const variant<_Types...>& __v) {
   static_assert(_Ip < sizeof...(_Types));
   static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
@@ -1374,8 +1366,7 @@ get(const variant<_Types...>& __v) {
 }
 
 template <size_t _Ip, class... _Types>
-_LIBCPP_HIDE_FROM_ABI
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const variant_alternative_t<_Ip, variant<_Types...>>&&
+_LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>&&
 get(const variant<_Types...>&& __v) {
   static_assert(_Ip < sizeof...(_Types));
   static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
@@ -1383,27 +1374,25 @@ get(const variant<_Types...>&& __v) {
 }
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp& get(variant<_Types...>& __v) {
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp& get(variant<_Types...>& __v) {
   static_assert(!is_void_v<_Tp>);
   return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
 }
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Tp&& get(variant<_Types...>&& __v) {
+_LIBCPP_HIDE_FROM_ABI constexpr _Tp&& get(variant<_Types...>&& __v) {
   static_assert(!is_void_v<_Tp>);
   return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(std::move(__v));
 }
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp&
-get(const variant<_Types...>& __v) {
+_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& get(const variant<_Types...>& __v) {
   static_assert(!is_void_v<_Tp>);
   return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
 }
 
 template <class _Tp, class... _Types>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr const _Tp&&
-get(const variant<_Types...>&& __v) {
+_LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& get(const variant<_Types...>&& __v) {
   static_assert(!is_void_v<_Tp>);
   return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(std::move(__v));
 }
@@ -1551,7 +1540,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const variant<_Types...>& __lhs,
 }
 
 template <class... _Vs>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr void __throw_if_valueless(_Vs&&... __vs) {
+_LIBCPP_HIDE_FROM_ABI constexpr void __throw_if_valueless(_Vs&&... __vs) {
   const bool __valueless = (... || std::__as_variant(__vs).valueless_by_exception());
   if (__valueless) {
     std::__throw_bad_variant_access();
@@ -1559,8 +1548,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr vo
 }
 
 template < class _Visitor, class... _Vs, typename>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr decltype(auto)
-visit(_Visitor&& __visitor, _Vs&&... __vs) {
+_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs) {
   using __variant_detail::__visitation::__variant;
   std::__throw_if_valueless(std::forward<_Vs>(__vs)...);
   return __variant::__visit_value(std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs)...);
@@ -1568,8 +1556,7 @@ visit(_Visitor&& __visitor, _Vs&&... __vs) {
 
 #    if _LIBCPP_STD_VER >= 20
 template < class _Rp, class _Visitor, class... _Vs, typename>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Rp
-visit(_Visitor&& __visitor, _Vs&&... __vs) {
+_LIBCPP_HIDE_FROM_ABI constexpr _Rp visit(_Visitor&& __visitor, _Vs&&... __vs) {
   using __variant_detail::__visitation::__variant;
   std::__throw_if_valueless(std::forward<_Vs>(__vs)...);
   return __variant::__visit_value<_Rp>(std::forward<_Visitor>(__visitor), std::forward<_Vs>(__vs)...);
diff --git a/external/llvm-project/libcxx/include/version b/external/llvm-project/libcxx/include/version
index 77d97b93adc6..87c4ede9a7e5 100644
--- a/external/llvm-project/libcxx/include/version
+++ b/external/llvm-project/libcxx/include/version
@@ -68,12 +68,14 @@ __cpp_lib_constexpr_charconv                            202207L <charconv>
 __cpp_lib_constexpr_cmath                               202202L <cmath> <cstdlib>
 __cpp_lib_constexpr_complex                             201711L <complex>
 __cpp_lib_constexpr_dynamic_alloc                       201907L <memory>
+__cpp_lib_constexpr_forward_list                        202502L <forward_list>
 __cpp_lib_constexpr_functional                          201907L <functional>
 __cpp_lib_constexpr_iterator                            201811L <iterator>
 __cpp_lib_constexpr_memory                              202202L <memory>
                                                         201811L // C++20
 __cpp_lib_constexpr_new                                 202406L <new>
 __cpp_lib_constexpr_numeric                             201911L <numeric>
+__cpp_lib_constexpr_queue                               202502L <queue>
 __cpp_lib_constexpr_string                              201907L <string>
 __cpp_lib_constexpr_string_view                         201811L <string_view>
 __cpp_lib_constexpr_tuple                               201811L <tuple>
@@ -542,9 +544,11 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_bitset                               202306L
 # undef  __cpp_lib_constexpr_algorithms
 # define __cpp_lib_constexpr_algorithms                 202306L
+# define __cpp_lib_constexpr_forward_list               202502L
 # if !defined(_LIBCPP_ABI_VCRUNTIME)
 #   define __cpp_lib_constexpr_new                      202406L
 # endif
+# define __cpp_lib_constexpr_queue                      202502L
 // # define __cpp_lib_constrained_equality                 202403L
 // # define __cpp_lib_copyable_function                    202306L
 // # define __cpp_lib_debugging                            202311L
diff --git a/external/llvm-project/libcxx/modules/std.compat.cppm.in b/external/llvm-project/libcxx/modules/std.compat.cppm.in
index 95931447ccdc..dd7385bf33a4 100644
--- a/external/llvm-project/libcxx/modules/std.compat.cppm.in
+++ b/external/llvm-project/libcxx/modules/std.compat.cppm.in
@@ -24,9 +24,7 @@ module;
 #include <cfloat>
 #include <cinttypes>
 #include <climits>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <clocale>
-#endif
+#include <clocale>
 #include <cmath>
 #include <csetjmp>
 #include <csignal>
diff --git a/external/llvm-project/libcxx/modules/std.cppm.in b/external/llvm-project/libcxx/modules/std.cppm.in
index 5c523691bff4..984b18321923 100644
--- a/external/llvm-project/libcxx/modules/std.cppm.in
+++ b/external/llvm-project/libcxx/modules/std.cppm.in
@@ -35,13 +35,9 @@ module;
 #include <chrono>
 #include <cinttypes>
 #include <climits>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <clocale>
-#endif
+#include <clocale>
 #include <cmath>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <codecvt>
-#endif
+#include <codecvt>
 #include <compare>
 #include <complex>
 #include <concepts>
@@ -68,32 +64,20 @@ module;
 #include <flat_set>
 #include <format>
 #include <forward_list>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <fstream>
-#endif
+#include <fstream>
 #include <functional>
 #include <future>
 #include <initializer_list>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <iomanip>
-#endif
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <ios>
-#endif
+#include <iomanip>
+#include <ios>
 #include <iosfwd>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <iostream>
-#endif
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <istream>
-#endif
+#include <iostream>
+#include <istream>
 #include <iterator>
 #include <latch>
 #include <limits>
 #include <list>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <locale>
-#endif
+#include <locale>
 #include <map>
 #include <mdspan>
 #include <memory>
@@ -103,40 +87,28 @@ module;
 #include <numbers>
 #include <numeric>
 #include <optional>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <ostream>
-#endif
+#include <ostream>
 #include <print>
 #include <queue>
 #include <random>
 #include <ranges>
 #include <ratio>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <regex>
-#endif
+#include <regex>
 #include <scoped_allocator>
 #include <semaphore>
 #include <set>
 #include <shared_mutex>
 #include <source_location>
 #include <span>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <sstream>
-#endif
+#include <sstream>
 #include <stack>
 #include <stdexcept>
 #include <stop_token>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <streambuf>
-#endif
+#include <streambuf>
 #include <string>
 #include <string_view>
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <strstream>
-#endif
-#if _LIBCPP_HAS_LOCALIZATION
-#  include <syncstream>
-#endif
+#include <strstream>
+#include <syncstream>
 #include <system_error>
 #include <thread>
 #include <tuple>
diff --git a/external/llvm-project/libcxx/src/any.cpp b/external/llvm-project/libcxx/src/any.cpp
index eaca2dd23976..f3fc715d517f 100644
--- a/external/llvm-project/libcxx/src/any.cpp
+++ b/external/llvm-project/libcxx/src/any.cpp
@@ -18,7 +18,7 @@ const char* bad_any_cast::what() const noexcept { return "bad any cast"; }
 //  Even though it no longer exists in a header file
 _LIBCPP_BEGIN_NAMESPACE_LFTS
 
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_ANY_CAST bad_any_cast : public bad_cast {
+class _LIBCPP_EXPORTED_FROM_ABI bad_any_cast : public bad_cast {
 public:
   virtual const char* what() const noexcept;
 };
diff --git a/external/llvm-project/libcxx/src/experimental/time_zone.cpp b/external/llvm-project/libcxx/src/experimental/time_zone.cpp
index 289164ab1203..a735800b6031 100644
--- a/external/llvm-project/libcxx/src/experimental/time_zone.cpp
+++ b/external/llvm-project/libcxx/src/experimental/time_zone.cpp
@@ -29,6 +29,15 @@
 // These quirks often use a 12h interval; this is the scan interval of zdump,
 // which implies there are no sys_info objects with a duration of less than 12h.
 
+// Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120502
+
+#include <__config>
+
+// TODO(LLVM 23): When upgrading to GCC 16 this can be removed
+#ifdef _LIBCPP_COMPILER_GCC
+#  pragma GCC optimize("-O0")
+#endif
+
 #include <algorithm>
 #include <cctype>
 #include <chrono>
diff --git a/external/llvm-project/libcxx/src/hash.cpp b/external/llvm-project/libcxx/src/hash.cpp
index 41c4eb480a5f..e1e6d2b4c2bd 100644
--- a/external/llvm-project/libcxx/src/hash.cpp
+++ b/external/llvm-project/libcxx/src/hash.cpp
@@ -9,7 +9,6 @@
 #include <__hash_table>
 #include <algorithm>
 #include <stdexcept>
-#include <type_traits>
 
 _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wtautological-constant-out-of-range-compare")
 
@@ -52,16 +51,14 @@ const unsigned indices[] = {
 // are fewer potential primes to search, and fewer potential primes to divide
 // against.
 
-template <size_t _Sz = sizeof(size_t)>
-inline _LIBCPP_HIDE_FROM_ABI typename enable_if<_Sz == 4, void>::type __check_for_overflow(size_t N) {
-  if (N > 0xFFFFFFFB)
-    std::__throw_overflow_error("__next_prime overflow");
-}
-
-template <size_t _Sz = sizeof(size_t)>
-inline _LIBCPP_HIDE_FROM_ABI typename enable_if<_Sz == 8, void>::type __check_for_overflow(size_t N) {
-  if (N > 0xFFFFFFFFFFFFFFC5ull)
-    std::__throw_overflow_error("__next_prime overflow");
+inline void __check_for_overflow(size_t N) {
+  if constexpr (sizeof(size_t) == 4) {
+    if (N > 0xFFFFFFFB)
+      std::__throw_overflow_error("__next_prime overflow");
+  } else {
+    if (N > 0xFFFFFFFFFFFFFFC5ull)
+      std::__throw_overflow_error("__next_prime overflow");
+  }
 }
 
 size_t __next_prime(size_t n) {
diff --git a/external/llvm-project/libcxx/src/locale.cpp b/external/llvm-project/libcxx/src/locale.cpp
index b3ec28950008..30a7a54e1c01 100644
--- a/external/llvm-project/libcxx/src/locale.cpp
+++ b/external/llvm-project/libcxx/src/locale.cpp
@@ -34,10 +34,6 @@
 #  define _CTYPE_DISABLE_MACROS
 #endif
 
-#if __has_include("<langinfo.h>")
-#  include <langinfo.h>
-#endif
-
 #include "include/atomic_support.h"
 #include "include/sso_allocator.h"
 
diff --git a/external/llvm-project/libcxx/src/optional.cpp b/external/llvm-project/libcxx/src/optional.cpp
index 4e7e28898f88..faabe66cfcfc 100644
--- a/external/llvm-project/libcxx/src/optional.cpp
+++ b/external/llvm-project/libcxx/src/optional.cpp
@@ -23,7 +23,7 @@ const char* bad_optional_access::what() const noexcept { return "bad_optional_ac
 //  Even though it no longer exists in a header file
 _LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
 
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS bad_optional_access : public std::logic_error {
+class _LIBCPP_EXPORTED_FROM_ABI bad_optional_access : public std::logic_error {
 public:
   bad_optional_access() : std::logic_error("Bad optional Access") {}
 
diff --git a/external/llvm-project/libcxx/test/configs/llvm-libc++-android-ndk.cfg.in b/external/llvm-project/libcxx/test/configs/llvm-libc++-android.cfg.in
similarity index 83%
rename from external/llvm-project/libcxx/test/configs/llvm-libc++-android-ndk.cfg.in
rename to external/llvm-project/libcxx/test/configs/llvm-libc++-android.cfg.in
index 31a07f647165..9362c68e8f7a 100644
--- a/external/llvm-project/libcxx/test/configs/llvm-libc++-android-ndk.cfg.in
+++ b/external/llvm-project/libcxx/test/configs/llvm-libc++-android.cfg.in
@@ -1,5 +1,5 @@
 # This testing configuration handles running the test suite against LLVM's
-# libc++ using adb and a libc++_shared.so library on Android.
+# libc++ using adb on Android.
 
 lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
@@ -27,11 +27,11 @@ if re.match(r'i686-linux-android(21|22|23)$', config.target_triple):
     compile_flags += ' -mstackrealign'
 config.substitutions.append(('%{compile_flags}', compile_flags))
 
-# The NDK library is called "libc++_shared.so". Use LD_LIBRARY_PATH to find
-# libc++_shared.so because older Bionic dynamic loaders don't support rpath
-# lookup.
+# The platform library is called "libc++.so" and the NDK library is called "libc++_shared.so". 
+# Use LD_LIBRARY_PATH to find the libcxx shared object because older Bionic dynamic loaders 
+# don't support rpath lookup.
 config.substitutions.append(('%{link_flags}',
-    '-nostdlib++ -L %{lib-dir} -lc++_shared'
+    '-nostdlib++ -L %{lib-dir} -l@LIBCXX_SHARED_OUTPUT_NAME@'
 ))
 config.substitutions.append(('%{exec}',
     '%{executor}' +
diff --git a/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp b/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp
index 563580b68795..4958a258137a 100644
--- a/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp
@@ -11,9 +11,9 @@
 // Concept helpers for the internal type traits for the fundamental types.
 
 // template <class _Tp>
-// concept __libcpp_integer;
+// concept __signed_or_unsigned_integer;
 
-#include <concepts>
+#include <__type_traits/integer_traits.h>
 
 #include "test_macros.h"
 
@@ -24,40 +24,40 @@ enum SomeEnum {};
 enum class SomeScopedEnum {};
 
 // Unsigned
-static_assert(std::__libcpp_integer<unsigned char>);
-static_assert(std::__libcpp_integer<unsigned short int>);
-static_assert(std::__libcpp_integer<unsigned int>);
-static_assert(std::__libcpp_integer<unsigned long int>);
-static_assert(std::__libcpp_integer<unsigned long long int>);
-static_assert(std::__libcpp_integer<unsigned short int>);
+static_assert(std::__signed_or_unsigned_integer<unsigned char>);
+static_assert(std::__signed_or_unsigned_integer<unsigned short int>);
+static_assert(std::__signed_or_unsigned_integer<unsigned int>);
+static_assert(std::__signed_or_unsigned_integer<unsigned long int>);
+static_assert(std::__signed_or_unsigned_integer<unsigned long long int>);
+static_assert(std::__signed_or_unsigned_integer<unsigned short int>);
 #if _LIBCPP_HAS_INT128
-static_assert(std::__libcpp_integer<__uint128_t>);
+static_assert(std::__signed_or_unsigned_integer<__uint128_t>);
 #endif
 // Signed
-static_assert(std::__libcpp_integer<signed char>);
-static_assert(std::__libcpp_integer<short int>);
-static_assert(std::__libcpp_integer<int>);
-static_assert(std::__libcpp_integer<long int>);
-static_assert(std::__libcpp_integer<long long int>);
-static_assert(std::__libcpp_integer<short int>);
+static_assert(std::__signed_or_unsigned_integer<signed char>);
+static_assert(std::__signed_or_unsigned_integer<short int>);
+static_assert(std::__signed_or_unsigned_integer<int>);
+static_assert(std::__signed_or_unsigned_integer<long int>);
+static_assert(std::__signed_or_unsigned_integer<long long int>);
+static_assert(std::__signed_or_unsigned_integer<short int>);
 #if _LIBCPP_HAS_INT128
-static_assert(std::__libcpp_integer<__int128_t>);
+static_assert(std::__signed_or_unsigned_integer<__int128_t>);
 #endif
 // Non-integer
-static_assert(!std::__libcpp_integer<bool>);
-static_assert(!std::__libcpp_integer<char>);
+static_assert(!std::__signed_or_unsigned_integer<bool>);
+static_assert(!std::__signed_or_unsigned_integer<char>);
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-static_assert(!std::__libcpp_integer<wchar_t>);
+static_assert(!std::__signed_or_unsigned_integer<wchar_t>);
 #endif
-static_assert(!std::__libcpp_integer<char8_t>);
-static_assert(!std::__libcpp_integer<char16_t>);
-static_assert(!std::__libcpp_integer<char32_t>);
-static_assert(!std::__libcpp_integer<float>);
-static_assert(!std::__libcpp_integer<double>);
-static_assert(!std::__libcpp_integer<long double>);
-static_assert(!std::__libcpp_integer<void>);
-static_assert(!std::__libcpp_integer<int*>);
-static_assert(!std::__libcpp_integer<unsigned int*>);
-static_assert(!std::__libcpp_integer<SomeObject>);
-static_assert(!std::__libcpp_integer<SomeEnum>);
-static_assert(!std::__libcpp_integer<SomeScopedEnum>);
+static_assert(!std::__signed_or_unsigned_integer<char8_t>);
+static_assert(!std::__signed_or_unsigned_integer<char16_t>);
+static_assert(!std::__signed_or_unsigned_integer<char32_t>);
+static_assert(!std::__signed_or_unsigned_integer<float>);
+static_assert(!std::__signed_or_unsigned_integer<double>);
+static_assert(!std::__signed_or_unsigned_integer<long double>);
+static_assert(!std::__signed_or_unsigned_integer<void>);
+static_assert(!std::__signed_or_unsigned_integer<int*>);
+static_assert(!std::__signed_or_unsigned_integer<unsigned int*>);
+static_assert(!std::__signed_or_unsigned_integer<SomeObject>);
+static_assert(!std::__signed_or_unsigned_integer<SomeEnum>);
+static_assert(!std::__signed_or_unsigned_integer<SomeScopedEnum>);
diff --git a/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp b/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp
index d1e21ee96b07..3fa342685770 100644
--- a/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp
@@ -11,9 +11,9 @@
 // Concept helpers for the internal type traits for the fundamental types.
 
 // template <class _Tp>
-// concept __libcpp_signed_integer;
+// concept __signed_integer;
 
-#include <concepts>
+#include <__type_traits/integer_traits.h>
 
 #include "test_macros.h"
 
@@ -24,40 +24,40 @@ enum SomeEnum {};
 enum class SomeScopedEnum {};
 
 // Unsigned
-static_assert(!std::__libcpp_signed_integer<unsigned char>);
-static_assert(!std::__libcpp_signed_integer<unsigned short int>);
-static_assert(!std::__libcpp_signed_integer<unsigned int>);
-static_assert(!std::__libcpp_signed_integer<unsigned long int>);
-static_assert(!std::__libcpp_signed_integer<unsigned long long int>);
-static_assert(!std::__libcpp_signed_integer<unsigned short int>);
+static_assert(!std::__signed_integer<unsigned char>);
+static_assert(!std::__signed_integer<unsigned short int>);
+static_assert(!std::__signed_integer<unsigned int>);
+static_assert(!std::__signed_integer<unsigned long int>);
+static_assert(!std::__signed_integer<unsigned long long int>);
+static_assert(!std::__signed_integer<unsigned short int>);
 #if _LIBCPP_HAS_INT128
-static_assert(!std::__libcpp_signed_integer<__uint128_t>);
+static_assert(!std::__signed_integer<__uint128_t>);
 #endif
 // Signed
-static_assert(std::__libcpp_signed_integer<signed char>);
-static_assert(std::__libcpp_signed_integer<short int>);
-static_assert(std::__libcpp_signed_integer<int>);
-static_assert(std::__libcpp_signed_integer<long int>);
-static_assert(std::__libcpp_signed_integer<long long int>);
-static_assert(std::__libcpp_signed_integer<short int>);
+static_assert(std::__signed_integer<signed char>);
+static_assert(std::__signed_integer<short int>);
+static_assert(std::__signed_integer<int>);
+static_assert(std::__signed_integer<long int>);
+static_assert(std::__signed_integer<long long int>);
+static_assert(std::__signed_integer<short int>);
 #if _LIBCPP_HAS_INT128
-static_assert(std::__libcpp_signed_integer<__int128_t>);
+static_assert(std::__signed_integer<__int128_t>);
 #endif
 // Non-integer
-static_assert(!std::__libcpp_signed_integer<bool>);
-static_assert(!std::__libcpp_signed_integer<char>);
+static_assert(!std::__signed_integer<bool>);
+static_assert(!std::__signed_integer<char>);
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-static_assert(!std::__libcpp_signed_integer<wchar_t>);
+static_assert(!std::__signed_integer<wchar_t>);
 #endif
-static_assert(!std::__libcpp_signed_integer<char8_t>);
-static_assert(!std::__libcpp_signed_integer<char16_t>);
-static_assert(!std::__libcpp_signed_integer<char32_t>);
-static_assert(!std::__libcpp_signed_integer<float>);
-static_assert(!std::__libcpp_signed_integer<double>);
-static_assert(!std::__libcpp_signed_integer<long double>);
-static_assert(!std::__libcpp_signed_integer<void>);
-static_assert(!std::__libcpp_signed_integer<int*>);
-static_assert(!std::__libcpp_signed_integer<unsigned int*>);
-static_assert(!std::__libcpp_signed_integer<SomeObject>);
-static_assert(!std::__libcpp_signed_integer<SomeEnum>);
-static_assert(!std::__libcpp_signed_integer<SomeScopedEnum>);
+static_assert(!std::__signed_integer<char8_t>);
+static_assert(!std::__signed_integer<char16_t>);
+static_assert(!std::__signed_integer<char32_t>);
+static_assert(!std::__signed_integer<float>);
+static_assert(!std::__signed_integer<double>);
+static_assert(!std::__signed_integer<long double>);
+static_assert(!std::__signed_integer<void>);
+static_assert(!std::__signed_integer<int*>);
+static_assert(!std::__signed_integer<unsigned int*>);
+static_assert(!std::__signed_integer<SomeObject>);
+static_assert(!std::__signed_integer<SomeEnum>);
+static_assert(!std::__signed_integer<SomeScopedEnum>);
diff --git a/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp b/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp
index c671f03cbfce..ff60f3231917 100644
--- a/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp
@@ -11,9 +11,9 @@
 // Concept helpers for the internal type traits for the fundamental types.
 
 // template <class _Tp>
-// concept __libcpp_unsigned_integer;
+// concept __unsigned_integer;
 
-#include <concepts>
+#include <__type_traits/integer_traits.h>
 
 #include "test_macros.h"
 
@@ -24,40 +24,40 @@ enum SomeEnum {};
 enum class SomeScopedEnum {};
 
 // Unsigned
-static_assert(std::__libcpp_unsigned_integer<unsigned char>);
-static_assert(std::__libcpp_unsigned_integer<unsigned short int>);
-static_assert(std::__libcpp_unsigned_integer<unsigned int>);
-static_assert(std::__libcpp_unsigned_integer<unsigned long int>);
-static_assert(std::__libcpp_unsigned_integer<unsigned long long int>);
-static_assert(std::__libcpp_unsigned_integer<unsigned short int>);
+static_assert(std::__unsigned_integer<unsigned char>);
+static_assert(std::__unsigned_integer<unsigned short int>);
+static_assert(std::__unsigned_integer<unsigned int>);
+static_assert(std::__unsigned_integer<unsigned long int>);
+static_assert(std::__unsigned_integer<unsigned long long int>);
+static_assert(std::__unsigned_integer<unsigned short int>);
 #if _LIBCPP_HAS_INT128
-static_assert(std::__libcpp_unsigned_integer<__uint128_t>);
+static_assert(std::__unsigned_integer<__uint128_t>);
 #endif
 // Signed
-static_assert(!std::__libcpp_unsigned_integer<signed char>);
-static_assert(!std::__libcpp_unsigned_integer<short int>);
-static_assert(!std::__libcpp_unsigned_integer<int>);
-static_assert(!std::__libcpp_unsigned_integer<long int>);
-static_assert(!std::__libcpp_unsigned_integer<long long int>);
-static_assert(!std::__libcpp_unsigned_integer<short int>);
+static_assert(!std::__unsigned_integer<signed char>);
+static_assert(!std::__unsigned_integer<short int>);
+static_assert(!std::__unsigned_integer<int>);
+static_assert(!std::__unsigned_integer<long int>);
+static_assert(!std::__unsigned_integer<long long int>);
+static_assert(!std::__unsigned_integer<short int>);
 #if _LIBCPP_HAS_INT128
-static_assert(!std::__libcpp_unsigned_integer<__int128_t>);
+static_assert(!std::__unsigned_integer<__int128_t>);
 #endif
 // Non-integer
-static_assert(!std::__libcpp_unsigned_integer<bool>);
-static_assert(!std::__libcpp_unsigned_integer<char>);
+static_assert(!std::__unsigned_integer<bool>);
+static_assert(!std::__unsigned_integer<char>);
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-static_assert(!std::__libcpp_unsigned_integer<wchar_t>);
+static_assert(!std::__unsigned_integer<wchar_t>);
 #endif
-static_assert(!std::__libcpp_unsigned_integer<char8_t>);
-static_assert(!std::__libcpp_unsigned_integer<char16_t>);
-static_assert(!std::__libcpp_unsigned_integer<char32_t>);
-static_assert(!std::__libcpp_unsigned_integer<float>);
-static_assert(!std::__libcpp_unsigned_integer<double>);
-static_assert(!std::__libcpp_unsigned_integer<long double>);
-static_assert(!std::__libcpp_unsigned_integer<void>);
-static_assert(!std::__libcpp_unsigned_integer<int*>);
-static_assert(!std::__libcpp_unsigned_integer<unsigned int*>);
-static_assert(!std::__libcpp_unsigned_integer<SomeObject>);
-static_assert(!std::__libcpp_unsigned_integer<SomeEnum>);
-static_assert(!std::__libcpp_unsigned_integer<SomeScopedEnum>);
+static_assert(!std::__unsigned_integer<char8_t>);
+static_assert(!std::__unsigned_integer<char16_t>);
+static_assert(!std::__unsigned_integer<char32_t>);
+static_assert(!std::__unsigned_integer<float>);
+static_assert(!std::__unsigned_integer<double>);
+static_assert(!std::__unsigned_integer<long double>);
+static_assert(!std::__unsigned_integer<void>);
+static_assert(!std::__unsigned_integer<int*>);
+static_assert(!std::__unsigned_integer<unsigned int*>);
+static_assert(!std::__unsigned_integer<SomeObject>);
+static_assert(!std::__unsigned_integer<SomeEnum>);
+static_assert(!std::__unsigned_integer<SomeScopedEnum>);
diff --git a/external/llvm-project/libcxx/test/libcxx/containers/container.adaptors/flat.set/scary.compile.pass.cpp b/external/llvm-project/libcxx/test/libcxx/containers/container.adaptors/flat.set/scary.compile.pass.cpp
new file mode 100644
index 000000000000..99e93fc3b08b
--- /dev/null
+++ b/external/llvm-project/libcxx/test/libcxx/containers/container.adaptors/flat.set/scary.compile.pass.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <flat_set>
+
+// class flat_set
+// class flat_multiset
+
+// Extension: SCARY/N2913 iterator compatibility between flat_set and flat_multiset
+// Test for the absence of this feature
+
+#include <flat_set>
+#include <type_traits>
+
+#include "test_macros.h"
+
+void test() {
+  typedef std::flat_set<int, int> M1;
+  typedef std::flat_multiset<int, int> M2;
+
+  static_assert(!std::is_convertible_v<M1::iterator, M2::iterator>);
+  static_assert(!std::is_convertible_v<M2::iterator, M1::iterator>);
+
+  static_assert(!std::is_convertible_v<M1::const_iterator, M2::const_iterator>);
+  static_assert(!std::is_convertible_v<M2::const_iterator, M1::const_iterator>);
+}
diff --git a/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp b/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp
index 5bf514359018..650fecf62128 100644
--- a/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp
@@ -28,8 +28,9 @@
 //       for every rank index r.
 //
 
-#include <mdspan>
 #include <cassert>
+#include <mdspan>
+#include <span>
 
 #include "check_assertion.h"
 
diff --git a/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp b/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
index 6782a9789f89..860849ded2de 100644
--- a/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
@@ -30,8 +30,9 @@
 //
 // Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_), direct-non-list-initializes strides_[d] with as_const(s[d]).
 
-#include <mdspan>
 #include <cassert>
+#include <mdspan>
+#include <span>
 
 #include "check_assertion.h"
 
diff --git a/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp b/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
index 74e4793c9137..70ab0616ec01 100644
--- a/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
@@ -30,8 +30,9 @@
 //
 // Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_), direct-non-list-initializes strides_[d] with as_const(s[d]).
 
-#include <mdspan>
 #include <cassert>
+#include <mdspan>
+#include <span>
 
 #include "check_assertion.h"
 
diff --git a/external/llvm-project/libcxx/test/libcxx/feature_test_macro/generate_header_test.sh.py b/external/llvm-project/libcxx/test/libcxx/feature_test_macro/generate_header_test.sh.py
index 98acfeacd018..789a0bc25a87 100644
--- a/external/llvm-project/libcxx/test/libcxx/feature_test_macro/generate_header_test.sh.py
+++ b/external/llvm-project/libcxx/test/libcxx/feature_test_macro/generate_header_test.sh.py
@@ -7,6 +7,7 @@
 # ===----------------------------------------------------------------------===##
 
 # RUN: %{python} %s %{libcxx-dir}/utils %{libcxx-dir}/test/libcxx/feature_test_macro/test_data.json %t/tests
+# END.
 
 import os
 import sys
diff --git a/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_concepts.pass.cpp b/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_concepts.compile.pass.cpp
similarity index 52%
rename from external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_concepts.pass.cpp
rename to external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_concepts.compile.pass.cpp
index d551a551795f..ffe473c97fed 100644
--- a/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_concepts.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_concepts.compile.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03
+// REQUIRES: std-at-least-c++20
 
 // ITER_TRAITS(I)
 
@@ -19,6 +19,7 @@
 // random_access_iterator_tag.
 // (1.4) -- Otherwise, ITER_CONCEPT(I) does not denote a type.
 
+#include <__iterator/concepts.h>
 #include <__type_traits/is_valid_expansion.h>
 #include <cstddef>
 #include <iterator>
@@ -30,19 +31,19 @@ struct OtherTagTwo : std::output_iterator_tag {};
 
 struct MyIter {
   using iterator_category = std::random_access_iterator_tag;
-  using iterator_concept = int;
-  using value_type = char;
-  using difference_type = std::ptrdiff_t;
-  using pointer = char*;
-  using reference = char&;
+  using iterator_concept  = int;
+  using value_type        = char;
+  using difference_type   = std::ptrdiff_t;
+  using pointer           = char*;
+  using reference         = char&;
 };
 
 struct MyIter2 {
   using iterator_category = OtherTag;
-  using value_type = char;
-  using difference_type = std::ptrdiff_t;
-  using pointer = char*;
-  using reference = char&;
+  using value_type        = char;
+  using difference_type   = std::ptrdiff_t;
+  using pointer           = char*;
+  using reference         = char&;
 };
 
 struct MyIter3 {};
@@ -52,10 +53,10 @@ struct EmptyWithSpecial {};
 template <>
 struct std::iterator_traits<MyIter3> {
   using iterator_category = OtherTagTwo;
-  using value_type = char;
-  using difference_type = std::ptrdiff_t;
-  using pointer = char*;
-  using reference = char&;
+  using value_type        = char;
+  using difference_type   = std::ptrdiff_t;
+  using pointer           = char*;
+  using reference         = char&;
 };
 
 template <>
@@ -63,29 +64,18 @@ struct std::iterator_traits<EmptyWithSpecial> {
   // empty non-default.
 };
 
-int main(int, char**) {
-  // If the qualified-id ITER_TRAITS(I)::iterator_concept is valid and names a type,
-  // then ITER_CONCEPT(I) denotes that type.
-  {
-#if TEST_STD_VER > 17
-    ASSERT_SAME_TYPE(std::_ITER_CONCEPT<char*>, std::contiguous_iterator_tag);
-#endif
-    ASSERT_SAME_TYPE(std::_ITER_CONCEPT<MyIter>, int);
-  }
-  // Otherwise, if the qualified-id ITER_TRAITS(I)::iterator_category is valid
-  // and names a type, then ITER_CONCEPT(I) denotes that type.
-  {
-    ASSERT_SAME_TYPE(std::_ITER_CONCEPT<MyIter2>, OtherTag);
-    ASSERT_SAME_TYPE(std::_ITER_CONCEPT<MyIter3>, OtherTagTwo);
-  }
-  // FIXME - This requirement makes no sense to me. Why does an empty type with
-  // an empty default iterator_traits get a category of random?
-  {
-    ASSERT_SAME_TYPE(std::_ITER_CONCEPT<Empty>, std::random_access_iterator_tag);
-  }
-  {
-    static_assert(!std::_IsValidExpansion<std::_ITER_CONCEPT, EmptyWithSpecial>::value, "");
-  }
+// If the qualified-id ITER_TRAITS(I)::iterator_concept is valid and names a type,
+// then ITER_CONCEPT(I) denotes that type.
+ASSERT_SAME_TYPE(std::_ITER_CONCEPT<char*>, std::contiguous_iterator_tag);
+ASSERT_SAME_TYPE(std::_ITER_CONCEPT<MyIter>, int);
 
-  return 0;
-}
+// Otherwise, if the qualified-id ITER_TRAITS(I)::iterator_category is valid
+// and names a type, then ITER_CONCEPT(I) denotes that type.
+ASSERT_SAME_TYPE(std::_ITER_CONCEPT<MyIter2>, OtherTag);
+ASSERT_SAME_TYPE(std::_ITER_CONCEPT<MyIter3>, OtherTagTwo);
+
+// FIXME - This requirement makes no sense to me. Why does an empty type with
+// an empty default iterator_traits get a category of random?
+ASSERT_SAME_TYPE(std::_ITER_CONCEPT<Empty>, std::random_access_iterator_tag);
+
+static_assert(!std::_IsValidExpansion<std::_ITER_CONCEPT, EmptyWithSpecial>::value);
diff --git a/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_traits.compile.pass.cpp b/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_traits.compile.pass.cpp
index 50047c1b9051..066587d68cd3 100644
--- a/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_traits.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/iterators/iterator.requirements/iterator.concepts/cpp20_iter_traits.compile.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03
+// REQUIRES: std-at-least-c++20
 
 // ITER_TRAITS(I)
 
@@ -14,7 +14,7 @@
 // a specialization generated from the primary template. Otherwise,
 // ITER_TRAITS(I) denotes iterator_traits<I>.
 
-#include <iterator>
+#include <__iterator/concepts.h>
 #include <type_traits>
 
 #include "test_iterators.h"
@@ -27,8 +27,8 @@ template<> struct std::iterator_traits<B> {};
 template<> struct std::iterator_traits<C> : std::iterator_traits<A> {};
 template<> struct std::iterator_traits<D> : std::iterator_traits<int*> {};
 
-static_assert(std::is_same<std::_ITER_TRAITS<int*>, std::iterator_traits<int*>>::value, "");
-static_assert(std::is_same<std::_ITER_TRAITS<A>, A>::value, "");
-static_assert(std::is_same<std::_ITER_TRAITS<B>, std::iterator_traits<B>>::value, "");
-static_assert(std::is_same<std::_ITER_TRAITS<C>, std::iterator_traits<C>>::value, "");
-static_assert(std::is_same<std::_ITER_TRAITS<D>, std::iterator_traits<D>>::value, "");
+static_assert(std::is_same<std::_ITER_TRAITS<int*>, std::iterator_traits<int*>>::value);
+static_assert(std::is_same<std::_ITER_TRAITS<A>, A>::value);
+static_assert(std::is_same<std::_ITER_TRAITS<B>, std::iterator_traits<B>>::value);
+static_assert(std::is_same<std::_ITER_TRAITS<C>, std::iterator_traits<C>>::value);
+static_assert(std::is_same<std::_ITER_TRAITS<D>, std::iterator_traits<D>>::value);
diff --git a/external/llvm-project/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp b/external/llvm-project/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
index 213d06d314a0..8066925f2900 100644
--- a/external/llvm-project/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
@@ -87,7 +87,7 @@ static_assert(!std::__libcpp_is_trivially_relocatable<std::array<NotTriviallyCop
 static_assert(std::__libcpp_is_trivially_relocatable<std::array<std::unique_ptr<int>, 1> >::value, "");
 
 // basic_string
-#if !_LIBCPP_HAS_ASAN || !_LIBCPP_INSTRUMENTED_WITH_ASAN
+#if !__has_feature(address_sanitizer) || !_LIBCPP_INSTRUMENTED_WITH_ASAN
 struct MyChar {
   char c;
 };
diff --git a/external/llvm-project/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp b/external/llvm-project/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp
index 38560161f162..897ae8936501 100644
--- a/external/llvm-project/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/type_traits/no_specializations.verify.cpp
@@ -186,5 +186,12 @@ struct std::enable_if<true, S>; // expected-error {{cannot be specialized}}
 #  if TEST_STD_VER >= 20
 template <>
 struct std::integral_constant<S, {}>; // expected-error {{cannot be specialized}}
+
+template <>
+struct std::common_reference<S>; // expected-error {{cannot be specialized}}
+template <>
+struct std::common_reference<S, S>; // expected-error {{cannot be specialized}}
+template <>
+struct std::common_reference<S, S, S>; // expected-error {{cannot be specialized}}
 #  endif
 #endif
diff --git a/external/llvm-project/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.verify.cpp b/external/llvm-project/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.verify.cpp
index 4a6e3095c101..a1a80483c448 100644
--- a/external/llvm-project/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.verify.cpp
+++ b/external/llvm-project/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.verify.cpp
@@ -38,9 +38,7 @@ template <class T> struct CannotDeduce {
 template <class ...Args>
 void F(typename CannotDeduce<std::tuple<Args...>>::type const&) {}
 
-
 void f() {
-#if TEST_HAS_BUILTIN_IDENTIFIER(__reference_binds_to_temporary)
   // Test that we emit our diagnostic from the library.
   // expected-error@tuple:* 8 {{Attempted construction of reference element binds to a temporary whose lifetime has ended}}
 
@@ -73,8 +71,4 @@ void f() {
     std::tuple<std::string &&> t2("hello"); // expected-note {{requested here}}
     std::tuple<std::string &&> t3(std::allocator_arg, alloc, "hello"); // expected-note {{requested here}}
   }
-#else
-#error force failure
-// expected-error@-1 {{force failure}}
-#endif
 }
diff --git a/external/llvm-project/libcxx/test/std/algorithms/alg.nonmodifying/alg.contains/ranges.contains.pass.cpp b/external/llvm-project/libcxx/test/std/algorithms/alg.nonmodifying/alg.contains/ranges.contains.pass.cpp
index 08d8e119a4d2..1e89cd272e64 100644
--- a/external/llvm-project/libcxx/test/std/algorithms/alg.nonmodifying/alg.contains/ranges.contains.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/algorithms/alg.nonmodifying/alg.contains/ranges.contains.pass.cpp
@@ -195,7 +195,7 @@ constexpr bool test() {
       std::string a[] = {str1, str1, str, str1, str1};
       auto whole =
           std::ranges::subrange(forward_iterator(std::move_iterator(a)), forward_iterator(std::move_iterator(a + 5)));
-      bool ret = std::ranges::contains(whole.begin(), whole.end(), "hello world", [&](const std::string i) {
+      bool ret = std::ranges::contains(whole.begin(), whole.end(), +"hello world", [&](const std::string i) {
         ++projection_count;
         return i;
       });
@@ -207,7 +207,7 @@ constexpr bool test() {
       std::string a[] = {str1, str1, str, str1, str1};
       auto whole =
           std::ranges::subrange(forward_iterator(std::move_iterator(a)), forward_iterator(std::move_iterator(a + 5)));
-      bool ret = std::ranges::contains(whole, "hello world", [&](const std::string i) {
+      bool ret = std::ranges::contains(whole, +"hello world", [&](const std::string i) {
         ++projection_count;
         return i;
       });
diff --git a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
index ca0f40eb77d4..0531c0e096a1 100644
--- a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
@@ -26,6 +26,7 @@
 #include <vector>
 
 #include "compare_types.h"
+#include "test_macros.h"
 
 namespace fundamentals {
 static_assert(std::equality_comparable<int>);
@@ -43,7 +44,12 @@ static_assert(std::equality_comparable<unsigned char&&>);
 static_assert(std::equality_comparable<unsigned short const&&>);
 static_assert(std::equality_comparable<unsigned int volatile&&>);
 static_assert(std::equality_comparable<unsigned long const volatile&&>);
+// Array comparisons are ill-formed in C++26, but Clang doesn't implement this yet.
+#if TEST_STD_VER <= 23 || defined(TEST_COMPILER_CLANG)
 static_assert(std::equality_comparable<int[5]>);
+#else
+static_assert(!std::equality_comparable<int[5]>);
+#endif
 static_assert(std::equality_comparable<int (*)(int)>);
 static_assert(std::equality_comparable<int (&)(int)>);
 static_assert(std::equality_comparable<int (*)(int) noexcept>);
diff --git a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
index 0afbe582ba89..2f8d7862c0f4 100644
--- a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
@@ -107,7 +107,12 @@ static_assert(!check_equality_comparable_with < int,
               int (S::*)() const volatile&& noexcept > ());
 
 static_assert(check_equality_comparable_with<int*, int*>());
+// Array comparisons are ill-formed in C++26, but Clang doesn't implement this yet.
+#if TEST_STD_VER <= 23 || defined(TEST_COMPILER_CLANG)
 static_assert(check_equality_comparable_with<int*, int[5]>());
+#else
+static_assert(!check_equality_comparable_with<int*, int[5]>());
+#endif
 static_assert(!check_equality_comparable_with<int*, int (*)()>());
 static_assert(!check_equality_comparable_with<int*, int (&)()>());
 static_assert(!check_equality_comparable_with<int*, int (S::*)()>());
@@ -148,7 +153,12 @@ static_assert(
 static_assert(!check_equality_comparable_with < int*,
               int (S::*)() const volatile&& noexcept > ());
 
+// Array comparisons are ill-formed in C++26, but Clang doesn't implement this yet.
+#if TEST_STD_VER <= 23 || defined(TEST_COMPILER_CLANG)
 static_assert(check_equality_comparable_with<int[5], int[5]>());
+#else
+static_assert(!check_equality_comparable_with<int[5], int[5]>());
+#endif
 static_assert(!check_equality_comparable_with<int[5], int (*)()>());
 static_assert(!check_equality_comparable_with<int[5], int (&)()>());
 static_assert(!check_equality_comparable_with<int[5], int (S::*)()>());
@@ -942,7 +952,12 @@ static_assert(
 
 static_assert(!check_equality_comparable_with<std::nullptr_t, int>());
 static_assert(check_equality_comparable_with<std::nullptr_t, int*>());
+// Array comparisons are ill-formed in C++26, but Clang doesn't implement this yet.
+#if TEST_STD_VER <= 23 || defined(TEST_COMPILER_CLANG)
 static_assert(check_equality_comparable_with<std::nullptr_t, int[5]>());
+#else
+static_assert(!check_equality_comparable_with<std::nullptr_t, int[5]>());
+#endif
 static_assert(check_equality_comparable_with<std::nullptr_t, int (*)()>());
 static_assert(check_equality_comparable_with<std::nullptr_t, int (&)()>());
 static_assert(check_equality_comparable_with<std::nullptr_t, int (S::*)()>());
diff --git a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp
index 6f8324eaf764..5959f70cf396 100644
--- a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered.compile.pass.cpp
@@ -55,7 +55,10 @@ static_assert(models_totally_ordered<unsigned char&&>());
 static_assert(models_totally_ordered<unsigned short const&&>());
 static_assert(models_totally_ordered<unsigned int volatile&&>());
 static_assert(models_totally_ordered<unsigned long const volatile&&>());
+// Array comparisons are ill-formed in C++26
+#if TEST_STD_VER <= 23
 static_assert(models_totally_ordered<int[5]>());
+#endif
 static_assert(models_totally_ordered<int (*)(int)>());
 static_assert(models_totally_ordered<int (&)(int)>());
 static_assert(models_totally_ordered<int (*)(int) noexcept>());
diff --git a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp
index dffc33265aeb..398ef445baf9 100644
--- a/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/concepts/concepts.compare/concepts.totallyordered/totally_ordered_with.compile.pass.cpp
@@ -89,7 +89,12 @@ static_assert(!check_totally_ordered_with<int, int (S::*)() const volatile&&>())
 static_assert(!check_totally_ordered_with < int, int (S::*)() const volatile&& noexcept > ());
 
 static_assert(check_totally_ordered_with<int*, int*>());
+// Array comparisons are ill-formed in C++26, but Clang doesn't implement this yet.
+#if TEST_STD_VER <= 23 || defined(TEST_COMPILER_CLANG)
 static_assert(check_totally_ordered_with<int*, int[5]>());
+#else
+static_assert(!check_totally_ordered_with<int*, int[5]>());
+#endif
 static_assert(!check_totally_ordered_with<int*, int (*)()>());
 static_assert(!check_totally_ordered_with<int*, int (&)()>());
 static_assert(!check_totally_ordered_with<int*, int (S::*)()>());
@@ -117,7 +122,12 @@ static_assert(!check_totally_ordered_with < int*, int (S::*)() volatile&& noexce
 static_assert(!check_totally_ordered_with<int*, int (S::*)() const volatile&&>());
 static_assert(!check_totally_ordered_with < int*, int (S::*)() const volatile&& noexcept > ());
 
+// Array comparisons are ill-formed in C++26, but Clang doesn't implement this yet.
+#if TEST_STD_VER <= 23 || defined(TEST_COMPILER_CLANG)
 static_assert(check_totally_ordered_with<int[5], int[5]>());
+#else
+static_assert(!check_totally_ordered_with<int[5], int[5]>());
+#endif
 static_assert(!check_totally_ordered_with<int[5], int (*)()>());
 static_assert(!check_totally_ordered_with<int[5], int (&)()>());
 static_assert(!check_totally_ordered_with<int[5], int (S::*)()>());
diff --git a/external/llvm-project/libcxx/test/std/containers/Emplaceable.h b/external/llvm-project/libcxx/test/std/containers/Emplaceable.h
index afc4e2e38b0e..246d5b255d6b 100644
--- a/external/llvm-project/libcxx/test/std/containers/Emplaceable.h
+++ b/external/llvm-project/libcxx/test/std/containers/Emplaceable.h
@@ -15,20 +15,20 @@
 #if TEST_STD_VER >= 11
 
 class Emplaceable {
-  Emplaceable(const Emplaceable&);
-  Emplaceable& operator=(const Emplaceable&);
+  TEST_CONSTEXPR Emplaceable(const Emplaceable&);
+  TEST_CONSTEXPR_CXX14 Emplaceable& operator=(const Emplaceable&);
 
   int int_;
   double double_;
 
 public:
-  Emplaceable() : int_(0), double_(0) {}
-  Emplaceable(int i, double d) : int_(i), double_(d) {}
-  Emplaceable(Emplaceable&& x) : int_(x.int_), double_(x.double_) {
+  TEST_CONSTEXPR Emplaceable() : int_(0), double_(0) {}
+  TEST_CONSTEXPR Emplaceable(int i, double d) : int_(i), double_(d) {}
+  TEST_CONSTEXPR_CXX14 Emplaceable(Emplaceable&& x) : int_(x.int_), double_(x.double_) {
     x.int_    = 0;
     x.double_ = 0;
   }
-  Emplaceable& operator=(Emplaceable&& x) {
+  TEST_CONSTEXPR_CXX14 Emplaceable& operator=(Emplaceable&& x) {
     int_      = x.int_;
     x.int_    = 0;
     double_   = x.double_;
@@ -36,10 +36,12 @@ class Emplaceable {
     return *this;
   }
 
-  bool operator==(const Emplaceable& x) const { return int_ == x.int_ && double_ == x.double_; }
-  bool operator<(const Emplaceable& x) const { return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_); }
+  TEST_CONSTEXPR bool operator==(const Emplaceable& x) const { return int_ == x.int_ && double_ == x.double_; }
+  TEST_CONSTEXPR bool operator<(const Emplaceable& x) const {
+    return int_ < x.int_ || (int_ == x.int_ && double_ < x.double_);
+  }
 
-  int get() const { return int_; }
+  TEST_CONSTEXPR int get() const { return int_; }
 };
 
 template <>
@@ -47,7 +49,7 @@ struct std::hash<Emplaceable> {
   typedef Emplaceable argument_type;
   typedef std::size_t result_type;
 
-  std::size_t operator()(const Emplaceable& x) const { return static_cast<std::size_t>(x.get()); }
+  TEST_CONSTEXPR std::size_t operator()(const Emplaceable& x) const { return static_cast<std::size_t>(x.get()); }
 };
 
 #endif // TEST_STD_VER >= 11
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
index 456f12e0c0d2..bc3fbfca5762 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp
@@ -106,6 +106,13 @@ int main(int, char**) {
     TEST_IGNORE_NODISCARD m.at(Transparent<int>{3});
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    int& x = m.at("alpha");
+    assert(x == 1);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp
index 24c08464f315..760ec69ae878 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp
@@ -94,6 +94,13 @@ int main(int, char**) {
     m[ConvertibleTransparent<int>{3}];
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    int& x = m["alpha"];
+    assert(x == 1);
+  }
   {
     auto index_func = [](auto& m, auto key_arg, auto value_arg) {
       using FlatMap                             = std::decay_t<decltype(m)>;
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp
index 3ba30757bf2c..ecc71a65caa8 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key_transparent.pass.cpp
@@ -139,6 +139,13 @@ int main(int, char**) {
     };
     test_erase_exception_guarantee(erase_transparent);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto n = m.erase("beta");
+    assert(n == 1);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
index 8455b19475fe..ccce117c90fc 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
@@ -13,6 +13,7 @@
 // template <class InputIterator>
 //   void insert(InputIterator first, InputIterator last);
 
+#include <algorithm>
 #include <flat_map>
 #include <cassert>
 #include <functional>
@@ -75,6 +76,7 @@ void test() {
   M expected2{{0, 1}, {1, 1}, {2, 1}, {3, 1}, {4, 1}};
   assert(m == expected2);
 }
+
 int main(int, char**) {
   test<std::vector<int>, std::vector<double>>();
   test<std::deque<int>, std::vector<double>>();
@@ -85,5 +87,12 @@ int main(int, char**) {
     auto insert_func = [](auto& m, const auto& newValues) { m.insert(newValues.begin(), newValues.end()); };
     test_insert_range_exception_guarantee(insert_func);
   }
+  {
+    std::flat_map<int, int, std::less<int>, SillyReserveVector<int>, SillyReserveVector<int>> m{{1, 1}, {2, 2}};
+    std::vector<std::pair<int, int>> v{{3, 3}, {4, 4}};
+    m.insert(v.begin(), v.end());
+    assert(std::ranges::equal(m, std::vector<std::pair<int, int>>{{1, 1}, {2, 2}, {3, 3}, {4, 4}}));
+  }
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp
index 636c4edfe551..bd6d176e26e2 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp
@@ -254,6 +254,18 @@ int main(int, char**) {
     };
     test_emplace_exception_guarantee(insert_or_assign_iter);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto [it, inserted] = m.insert_or_assign("alpha", 2);
+    assert(!inserted);
+    assert(it == m.begin());
+    assert(it->second == 2);
+    auto it2 = m.insert_or_assign(m.begin(), "beta2", 2);
+    assert(it2 == m.begin() + 2);
+    assert(it2->second == 2);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp
index 75cabb70630f..c24c8373f51e 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp
@@ -163,5 +163,15 @@ int main(int, char**) {
     };
     test_emplace_exception_guarantee(insert_func_iter);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto [it, inserted] = m.insert({"alpha", 1});
+    assert(!inserted);
+    assert(it == m.begin());
+    auto it2 = m.insert(m.begin(), {"beta2", 2});
+    assert(it2 == m.begin() + 2);
+  }
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp
index 21fda4378096..c301f07937ea 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp
@@ -160,6 +160,17 @@ int main(int, char**) {
     assert(p->second == 3);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto [it1, inserted] = m.try_emplace("charlie", 4);
+    assert(it1 == m.begin() + 2);
+    assert(inserted);
+
+    auto it2 = m.try_emplace(m.begin(), "beta2", 2);
+    assert(it2 == m.begin() + 2);
+  }
   {
     auto try_emplace = [](auto& m, auto key_arg, auto value_arg) {
       using M   = std::decay_t<decltype(m)>;
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp
index 0493538ab6da..e28e94c3d8fb 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/contains_transparent.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <cassert>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 #include <deque>
@@ -67,5 +68,12 @@ int main(int, char**) {
     assert(b);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    assert(m.contains("beta") == true);
+    assert(m.contains("charlie") == false);
+  }
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp
index cd195ff1fa8b..37d815b2ce98 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/count_transparent.pass.cpp
@@ -15,6 +15,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -67,6 +68,13 @@ int main(int, char**) {
     assert(n == 1);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    assert(m.count("alpha") == 1);
+    assert(m.count("charlie") == 0);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp
index 0198f433bdc4..bff70a320928 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/equal_range_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -95,6 +96,14 @@ int main(int, char**) {
     assert(p.first != p.second);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto [first, last] = m.equal_range("beta");
+    assert(first == m.begin() + 1);
+    assert(last == m.begin() + 2);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp
index 291577a89fc8..b1d1301fce63 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/find_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -83,6 +84,13 @@ int main(int, char**) {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto it = m.find("beta");
+    assert(it == m.begin() + 1);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp
index 6a923c197e91..9e2561631196 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/lower_bound_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -90,6 +91,13 @@ int main(int, char**) {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto it = m.lower_bound("charlie");
+    assert(it == m.begin() + 2);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp
index 4e83f920835d..c87113f67018 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -89,6 +90,13 @@ int main(int, char**) {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_map<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"epsilon", 1}, {"eta", 3}, {"gamma", 3}};
+    auto it = m.upper_bound("charlie");
+    assert(it == m.begin() + 2);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp
index 75a2d205b8f8..5627a67b29e9 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp
@@ -156,6 +156,13 @@ int main(int, char**) {
     };
     test_erase_exception_guarantee(erase_transparent);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    auto n = m.erase("beta");
+    assert(n == 2);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp
index ae031bd010f7..30cb89dadbfe 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp
@@ -16,6 +16,7 @@
 //   void insert(InputIterator first, InputIterator last);
 
 #include <flat_map>
+#include <algorithm>
 #include <cassert>
 #include <functional>
 #include <deque>
@@ -105,5 +106,11 @@ int main(int, char**) {
     auto insert_func = [](auto& m, const auto& newValues) { m.insert(newValues.begin(), newValues.end()); };
     test_insert_range_exception_guarantee(insert_func);
   }
+  {
+    std::flat_multimap<int, int, std::less<int>, SillyReserveVector<int>, SillyReserveVector<int>> m{{1, 1}, {2, 2}};
+    std::vector<std::pair<int, int>> v{{3, 3}, {4, 4}};
+    m.insert(v.begin(), v.end());
+    assert(std::ranges::equal(m, std::vector<std::pair<int, int>>{{1, 1}, {2, 2}, {3, 3}, {4, 4}}));
+  }
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp
index 33ca4d4e3046..760479ade76e 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp
@@ -131,5 +131,14 @@ int main(int, char**) {
     };
     test_emplace_exception_guarantee(insert_func_iter);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    auto it = m.insert({"beta", 1});
+    assert(it == m.begin() + 3);
+    auto it2 = m.insert(m.begin(), {"beta2", 2});
+    assert(it2 == m.begin() + 4);
+  }
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp
index 8a66ec63768d..42feeb0c9341 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp
@@ -16,6 +16,7 @@
 
 #include <cassert>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 #include <deque>
@@ -69,5 +70,12 @@ int main(int, char**) {
     assert(b);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    assert(m.contains("beta") == true);
+    assert(m.contains("charlie") == false);
+  }
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp
index 41f71065b2f7..6811be5de1f0 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp
@@ -17,6 +17,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -78,6 +79,13 @@ int main(int, char**) {
     assert(n == 2);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    assert(m.count("beta") == 2);
+    assert(m.count("charlie") == 0);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp
index 3666492bb921..d80c37feecc0 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp
@@ -18,6 +18,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -105,6 +106,14 @@ int main(int, char**) {
     assert(p.second == m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    auto [first, last] = m.equal_range("beta");
+    assert(first == m.begin() + 1);
+    assert(last == m.begin() + 3);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp
index be8c6f2e3544..dff63560c3cb 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp
@@ -18,6 +18,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -94,6 +95,13 @@ int main(int, char**) {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    auto it = m.find("beta");
+    assert(it == m.begin() + 1);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp
index b757af132e67..3161ca01e557 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp
@@ -18,6 +18,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -102,6 +103,13 @@ int main(int, char**) {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    auto it = m.lower_bound("charlie");
+    assert(it == m.begin() + 3);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp
index 969489d0fe61..d51d87ce7788 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp
@@ -18,6 +18,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_map>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -101,6 +102,13 @@ int main(int, char**) {
     assert(it == m.begin() + 3);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_multimap<std::string, int, std::less<>>;
+    M m{{"alpha", 1}, {"beta", 2}, {"beta", 1}, {"eta", 3}, {"gamma", 3}};
+    auto it = m.upper_bound("charlie");
+    assert(it == m.begin() + 3);
+  }
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_iter.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_iter.pass.cpp
index 3505e097cca6..93815686787c 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_iter.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_iter.pass.cpp
@@ -14,6 +14,7 @@
 //   void insert(InputIterator first, InputIterator last);
 
 #include <flat_set>
+#include <algorithm>
 #include <cassert>
 #include <functional>
 #include <deque>
@@ -79,6 +80,12 @@ void test() {
   test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+  {
+    std::flat_multiset<int, std::less<int>, SillyReserveVector<int>> m{1, 2};
+    std::vector<int> v{3, 4};
+    m.insert(v.begin(), v.end());
+    assert(std::ranges::equal(m, std::vector<int>{1, 2, 3, 4}));
+  }
 }
 
 void test_exception() {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
index 9dd91628d1c2..f4b373364cab 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
@@ -138,6 +138,13 @@ void test() {
     assert(n == 1);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    auto n = m.erase("beta");
+    assert(n == 1);
+  }
 }
 
 void test_exception() {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
index 8063686c960e..18bdb798de01 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
@@ -14,6 +14,7 @@
 //   void insert(InputIterator first, InputIterator last);
 
 #include <flat_set>
+#include <algorithm>
 #include <cassert>
 #include <functional>
 #include <deque>
@@ -79,6 +80,12 @@ void test() {
   test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+  {
+    std::flat_set<int, std::less<int>, SillyReserveVector<int>> m{1, 2};
+    std::vector<int> v{3, 4};
+    m.insert(v.begin(), v.end());
+    assert(std::ranges::equal(m, std::vector<int>{1, 2, 3, 4}));
+  }
 }
 
 void test_exception() {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
index f60fd78008a3..4b5779a10ee7 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
@@ -123,6 +123,14 @@ void test() {
     ASSERT_SAME_TYPE(decltype(m.insert(m.begin(), Evil())), M::iterator);
     ASSERT_SAME_TYPE(decltype(m.insert(m.begin(), m.end())), void);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    auto [iter, inserted] = m.insert("beta");
+    assert(!inserted);
+    assert(iter == m.begin() + 1);
+  }
 }
 
 void test_exception() {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
index cae8eea89471..91f49305cd71 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <cassert>
 #include <flat_set>
+#include <functional>
 #include <string>
 #include <utility>
 #include <deque>
@@ -74,6 +75,13 @@ void test() {
     assert(b);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    assert(m.contains("beta"));
+    assert(!m.contains("eta2"));
+  }
 }
 
 int main(int, char**) {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
index b24b6fa712fa..84e7b9a18b83 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
@@ -15,6 +15,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_set>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -73,6 +74,13 @@ void test() {
     assert(n == 1);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    assert(m.count("beta") == 1);
+    assert(m.count("eta2") == 0);
+  }
 }
 
 int main(int, char**) {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
index 8a270ffa1591..31081eb97039 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_set>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -101,6 +102,14 @@ void test() {
     assert(p.first != p.second);
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    auto [first, last] = m.equal_range("beta");
+    assert(first == m.begin() + 1);
+    assert(last == m.begin() + 2);
+  }
 }
 
 int main(int, char**) {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
index 6ba91bd5ebe4..b8d454c58f54 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_set>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -91,6 +92,15 @@ void test() {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    auto it = m.find("beta");
+    assert(it == m.begin() + 1);
+    auto it2 = m.find("beta2");
+    assert(it2 == m.end());
+  }
 }
 
 int main(int, char**) {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
index 8fcfe4d9d432..0c3e25f795f3 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_set>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -97,6 +98,15 @@ void test() {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    auto it = m.lower_bound("beta");
+    assert(it == m.begin() + 1);
+    auto it2 = m.lower_bound("beta2");
+    assert(it2 == m.begin() + 2);
+  }
 }
 
 int main(int, char**) {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
index db89614dd815..36f2a2094649 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
@@ -16,6 +16,7 @@
 #include <cassert>
 #include <deque>
 #include <flat_set>
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -97,6 +98,15 @@ void test() {
     assert(it != m.end());
     assert(transparent_used);
   }
+  {
+    // LWG4239 std::string and C string literal
+    using M = std::flat_set<std::string, std::less<>>;
+    M m{"alpha", "beta", "epsilon", "eta", "gamma"};
+    auto it = m.upper_bound("beta");
+    assert(it == m.begin() + 2);
+    auto it2 = m.upper_bound("beta2");
+    assert(it2 == m.begin() + 2);
+  }
 }
 
 int main(int, char**) {
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat_helpers.h b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat_helpers.h
index 9cd408ef960a..19d637052203 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat_helpers.h
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/flat_helpers.h
@@ -25,6 +25,13 @@ struct CopyOnlyVector : std::vector<T> {
   CopyOnlyVector& operator=(CopyOnlyVector& other) { return this->operator=(other); }
 };
 
+template <class T>
+struct SillyReserveVector : std::vector<T> {
+  using std::vector<T>::vector;
+
+  void reserve(size_t) { this->clear(); }
+};
+
 template <class T, bool ConvertibleToT = false>
 struct Transparent {
   T t;
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp
index 903a4ca2c2b4..07d694fd399a 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_alloc.pass.cpp
@@ -18,29 +18,38 @@
 #include "test_allocator.h"
 
 template <class T>
-struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+struct Test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
   typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
   typedef typename base::container_type container_type;
   typedef typename base::value_compare value_compare;
 
-  explicit test(const test_allocator<int>& a) : base(a) {}
-  test(const value_compare& comp, const test_allocator<int>& a) : base(comp, c, a) {}
-  test(const value_compare& comp, const container_type& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 explicit Test(const test_allocator<int>& a) : base(a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& comp, const test_allocator<int>& a) : base(comp, c, a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& comp, const container_type& container, const test_allocator<int>& a)
       : base(comp, container, a) {}
 #if TEST_STD_VER >= 11
-  test(const value_compare& comp, container_type&& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& comp, container_type&& container, const test_allocator<int>& a)
       : base(comp, std::move(container), a) {}
-  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  TEST_CONSTEXPR_CXX26 Test(Test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-  test_allocator<int> get_allocator() { return c.get_allocator(); }
+  TEST_CONSTEXPR_CXX26 test_allocator<int> get_allocator() { return c.get_allocator(); }
 
   using base::c;
 };
 
-int main(int, char**) {
-  test<int> q((test_allocator<int>(3)));
+TEST_CONSTEXPR_CXX26 bool test() {
+  Test<int> q((test_allocator<int>(3)));
   assert(q.c.get_allocator() == test_allocator<int>(3));
   assert(q.c.size() == 0);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp
index 439b19377aba..cf2c5c46f509 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_alloc.pass.cpp
@@ -18,29 +18,38 @@
 #include "test_allocator.h"
 
 template <class T>
-struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+struct Test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
   typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
   typedef typename base::container_type container_type;
   typedef typename base::value_compare value_compare;
 
-  explicit test(const test_allocator<int>& a) : base(a) {}
-  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
-  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 explicit Test(const test_allocator<int>& a) : base(a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
       : base(compare, container, a) {}
 #if TEST_STD_VER >= 11
-  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
       : base(compare, std::move(container), a) {}
-  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  TEST_CONSTEXPR_CXX26 Test(Test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-  test_allocator<int> get_allocator() { return c.get_allocator(); }
+  TEST_CONSTEXPR_CXX26 test_allocator<int> get_allocator() { return c.get_allocator(); }
 
   using base::c;
 };
 
-int main(int, char**) {
-  test<int> q(std::less<int>(), test_allocator<int>(3));
+TEST_CONSTEXPR_CXX26 bool test() {
+  Test<int> q(std::less<int>(), test_allocator<int>(3));
   assert(q.c.get_allocator() == test_allocator<int>(3));
   assert(q.c.size() == 0);
 
-  return 0;
+  return true;
 }
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
+  return 0;
+}
\ No newline at end of file
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp
index dddc11c88689..653845a2b6d7 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_cont_alloc.pass.cpp
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(i);
@@ -27,32 +27,41 @@ C make(int n) {
 }
 
 template <class T>
-struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+struct Test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
   typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
   typedef typename base::container_type container_type;
   typedef typename base::value_compare value_compare;
 
-  explicit test(const test_allocator<int>& a) : base(a) {}
-  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
-  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 explicit Test(const test_allocator<int>& a) : base(a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
       : base(compare, container, a) {}
 #if TEST_STD_VER >= 11 // testing rvalue constructor
-  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
       : base(compare, std::move(container), a) {}
-  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  TEST_CONSTEXPR_CXX26 Test(Test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-  test_allocator<int> get_allocator() { return c.get_allocator(); }
+  TEST_CONSTEXPR_CXX26 test_allocator<int> get_allocator() { return c.get_allocator(); }
 
   using base::c;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   typedef std::vector<int, test_allocator<int> > C;
   C v = make<C>(5);
-  test<int> q(std::less<int>(), v, test_allocator<int>(3));
+  Test<int> q(std::less<int>(), v, test_allocator<int>(3));
   assert(q.c.get_allocator() == test_allocator<int>(3));
   assert(q.size() == 5);
   assert(q.top() == 4);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp
index bdb84a5f274d..1e1da8db8568 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_comp_rcont_alloc.pass.cpp
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(i);
@@ -27,31 +27,40 @@ C make(int n) {
 }
 
 template <class T>
-struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+struct Test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
   typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
   typedef typename base::container_type container_type;
   typedef typename base::value_compare value_compare;
 
-  explicit test(const test_allocator<int>& a) : base(a) {}
-  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
-  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 explicit Test(const test_allocator<int>& a) : base(a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const test_allocator<int>& a) : base(compare, a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
       : base(compare, container, a) {}
 #if TEST_STD_VER >= 11 // testing rvalue ctor
-  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
       : base(compare, std::move(container), a) {}
-  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  TEST_CONSTEXPR_CXX26 Test(Test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
 #endif
-  test_allocator<int> get_allocator() { return c.get_allocator(); }
+  TEST_CONSTEXPR_CXX26 test_allocator<int> get_allocator() { return c.get_allocator(); }
 
   using base::c;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   typedef std::vector<int, test_allocator<int> > C;
-  test<int> q(std::less<int>(), make<C>(5), test_allocator<int>(3));
+  Test<int> q(std::less<int>(), make<C>(5), test_allocator<int>(3));
   assert(q.c.get_allocator() == test_allocator<int>(3));
   assert(q.size() == 5);
   assert(q.top() == 4);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp
index e658d6ce874f..c7784221ec21 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_copy_alloc.pass.cpp
@@ -14,8 +14,10 @@
 #include <queue>
 #include <cassert>
 
+#include "test_macros.h"
+
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(i);
@@ -26,27 +28,36 @@ C make(int n) {
 #include "test_allocator.h"
 
 template <class T>
-struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+struct Test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
   typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
   typedef typename base::container_type container_type;
   typedef typename base::value_compare value_compare;
 
-  explicit test(const test_allocator<int>& a) : base(a) {}
-  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, c, a) {}
-  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 explicit Test(const test_allocator<int>& a) : base(a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const test_allocator<int>& a) : base(compare, c, a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
       : base(compare, container, a) {}
-  test(const test& q, const test_allocator<int>& a) : base(q, a) {}
-  test_allocator<int> get_allocator() { return c.get_allocator(); }
+  TEST_CONSTEXPR_CXX26 Test(const Test& q, const test_allocator<int>& a) : base(q, a) {}
+  TEST_CONSTEXPR_CXX26 test_allocator<int> get_allocator() { return c.get_allocator(); }
 
   using base::c;
 };
 
-int main(int, char**) {
-  test<int> qo(std::less<int>(), make<std::vector<int, test_allocator<int> > >(5), test_allocator<int>(2));
-  test<int> q(qo, test_allocator<int>(6));
+TEST_CONSTEXPR_CXX26 bool test() {
+  Test<int> qo(std::less<int>(), make<std::vector<int, test_allocator<int> > >(5), test_allocator<int>(2));
+  Test<int> q(qo, test_allocator<int>(6));
   assert(q.size() == 5);
   assert(q.c.get_allocator() == test_allocator<int>(6));
   assert(q.top() == int(4));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp
index 4786629d46a4..ee1af396314b 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_alloc.pass.cpp
@@ -23,12 +23,12 @@ struct PQ : std::priority_queue<T, Cont, Comp> {
   typedef std::priority_queue<T, Cont, Comp> base;
 
   template <class It, class Alloc>
-  explicit PQ(It first, It last, const Alloc& a) : base(first, last, a) {}
+  TEST_CONSTEXPR_CXX26 explicit PQ(It first, It last, const Alloc& a) : base(first, last, a) {}
 
   using base::c;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a[] = {3, 5, 2, 0, 6, 8, 1};
   typedef test_allocator<int> Alloc;
   PQ<int, std::vector<int, Alloc> > q(a, a + 7, Alloc(2));
@@ -36,5 +36,14 @@ int main(int, char**) {
   assert(q.top() == 8);
   assert(q.c.get_allocator() == Alloc(2));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp
index d3e11b30b1f1..31a72ad0185c 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_alloc.pass.cpp
@@ -24,12 +24,13 @@ struct PQ : std::priority_queue<T, Cont, Comp> {
   typedef std::priority_queue<T, Cont, Comp> base;
 
   template <class It, class Alloc>
-  explicit PQ(It first, It last, const Comp& compare, const Alloc& a) : base(first, last, compare, a) {}
+  TEST_CONSTEXPR_CXX26 explicit PQ(It first, It last, const Comp& compare, const Alloc& a)
+      : base(first, last, compare, a) {}
 
   using base::c;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a[] = {3, 5, 2, 0, 6, 8, 1};
   typedef test_allocator<int> Alloc;
   PQ<int, std::vector<int, Alloc>, std::greater<int> > q(a, a + 7, std::greater<int>(), Alloc(2));
@@ -37,5 +38,14 @@ int main(int, char**) {
   assert(q.top() == 0);
   assert(q.c.get_allocator() == Alloc(2));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp
index a989ca155963..25b1e0ecb9fc 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_cont_alloc.pass.cpp
@@ -23,13 +23,13 @@ struct PQ : std::priority_queue<T, Cont, Comp> {
   typedef std::priority_queue<T, Cont, Comp> base;
 
   template <class It, class Alloc>
-  explicit PQ(It first, It last, const Comp& compare, const Cont& v, const Alloc& a)
+  TEST_CONSTEXPR_CXX26 explicit PQ(It first, It last, const Comp& compare, const Cont& v, const Alloc& a)
       : base(first, last, compare, v, a) {}
 
   using base::c;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   typedef test_allocator<int> Alloc;
   int a[] = {3, 5, 2, 0, 6, 8, 1};
   std::vector<int, Alloc> v(a, a + 3);
@@ -38,5 +38,14 @@ int main(int, char**) {
   assert(q.top() == 8);
   assert(q.c.get_allocator() == Alloc(2));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp
index bd373629955e..ab1b1e129d8c 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_iter_iter_comp_rcont_alloc.pass.cpp
@@ -26,13 +26,13 @@ struct PQ : std::priority_queue<T, Cont, Comp> {
   typedef std::priority_queue<T, Cont, Comp> base;
 
   template <class It, class Alloc>
-  explicit PQ(It first, It last, const Comp& compare, Cont&& v, const Alloc& a)
+  TEST_CONSTEXPR_CXX26 explicit PQ(It first, It last, const Comp& compare, Cont&& v, const Alloc& a)
       : base(first, last, compare, std::move(v), a) {}
 
   using base::c;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   using Alloc = test_allocator<MoveOnly>;
   int a[]     = {3, 5, 2, 0, 6, 8, 1};
   PQ<MoveOnly, std::vector<MoveOnly, Alloc>> q(
@@ -41,5 +41,14 @@ int main(int, char**) {
   assert(q.top() == MoveOnly(8));
   assert(q.c.get_allocator() == Alloc(2));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp
index 95c88cf877a5..8bb3b1754fc4 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons.alloc/ctor_move_alloc.pass.cpp
@@ -20,7 +20,7 @@
 #include "MoveOnly.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(MoveOnly(i));
@@ -30,30 +30,39 @@ C make(int n) {
 #include "test_allocator.h"
 
 template <class T>
-struct test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
+struct Test : public std::priority_queue<T, std::vector<T, test_allocator<T> > > {
   typedef std::priority_queue<T, std::vector<T, test_allocator<T> > > base;
   typedef typename base::container_type container_type;
   typedef typename base::value_compare value_compare;
 
-  explicit test(const test_allocator<int>& a) : base(a) {}
-  test(const value_compare& compare, const test_allocator<int>& a) : base(compare, c, a) {}
-  test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 explicit Test(const test_allocator<int>& a) : base(a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const test_allocator<int>& a) : base(compare, c, a) {}
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, const container_type& container, const test_allocator<int>& a)
       : base(compare, container, a) {}
-  test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
+  TEST_CONSTEXPR_CXX26 Test(const value_compare& compare, container_type&& container, const test_allocator<int>& a)
       : base(compare, std::move(container), a) {}
-  test(test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
-  test_allocator<int> get_allocator() { return c.get_allocator(); }
+  TEST_CONSTEXPR_CXX26 Test(Test&& q, const test_allocator<int>& a) : base(std::move(q), a) {}
+  TEST_CONSTEXPR_CXX26 test_allocator<int> get_allocator() { return c.get_allocator(); }
 
   using base::c;
 };
 
-int main(int, char**) {
-  test<MoveOnly> qo(
+TEST_CONSTEXPR_CXX26 bool test() {
+  Test<MoveOnly> qo(
       std::less<MoveOnly>(), make<std::vector<MoveOnly, test_allocator<MoveOnly> > >(5), test_allocator<MoveOnly>(2));
-  test<MoveOnly> q(std::move(qo), test_allocator<MoveOnly>(6));
+  Test<MoveOnly> q(std::move(qo), test_allocator<MoveOnly>(6));
   assert(q.size() == 5);
   assert(q.c.get_allocator() == test_allocator<MoveOnly>(6));
   assert(q.top() == MoveOnly(4));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp
index 6acc8aef3456..100591d96a66 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_copy.pass.cpp
@@ -17,14 +17,14 @@
 #include "test_macros.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(i);
   return c;
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::vector<int> v = make<std::vector<int> >(5);
   std::priority_queue<int, std::vector<int>, std::greater<int> > qo(std::greater<int>(), v);
   std::priority_queue<int, std::vector<int>, std::greater<int> > q;
@@ -32,5 +32,14 @@ int main(int, char**) {
   assert(q.size() == 5);
   assert(q.top() == 0);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp
index fbdc07f53066..5c9e554e3f93 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/assign_move.pass.cpp
@@ -19,19 +19,28 @@
 #include "MoveOnly.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(MoveOnly(i));
   return c;
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<MoveOnly> qo(std::less<MoveOnly>(), make<std::vector<MoveOnly> >(5));
   std::priority_queue<MoveOnly> q;
   q = std::move(qo);
   assert(q.size() == 5);
   assert(q.top() == MoveOnly(4));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp
index 8e56ea6234e6..a3491efab3ea 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp.pass.cpp
@@ -19,7 +19,7 @@
 #  include "test_convertible.h"
 #endif
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   typedef std::vector<int, limited_allocator<int, 10> > Container;
   typedef std::less<int> Compare;
   typedef std::priority_queue<int, Container> Q;
@@ -34,5 +34,14 @@ int main(int, char**) {
   static_assert(!test_convertible<Q, const Compare&>(), "");
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp
index 08e74a790f90..a08153efbeb7 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_container.pass.cpp
@@ -20,14 +20,14 @@
 #endif
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(i);
   return c;
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   typedef std::vector<int> Container;
   typedef std::greater<int> Compare;
   typedef std::priority_queue<int, Container, Compare> Q;
@@ -41,5 +41,14 @@ int main(int, char**) {
   static_assert(test_convertible<Q, const Compare&, const Container&>(), "");
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp
index f0b0eaef2f3d..6b5c0205ce8a 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_comp_rcontainer.pass.cpp
@@ -21,22 +21,30 @@
 #include "test_convertible.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(MoveOnly(i));
   return c;
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   typedef std::vector<MoveOnly> Container;
   typedef std::less<MoveOnly> Compare;
   typedef std::priority_queue<MoveOnly> Q;
   Q q(Compare(), make<Container>(5));
   assert(q.size() == 5);
   assert(q.top() == MoveOnly(4));
-
   static_assert(test_convertible<Q, const Compare&, Container&&>(), "");
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp
index 0e9f58ec8b9c..591487b47c73 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_copy.pass.cpp
@@ -17,19 +17,28 @@
 #include "test_macros.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(i);
   return c;
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::vector<int> v = make<std::vector<int> >(5);
   std::priority_queue<int, std::vector<int>, std::greater<int> > qo(std::greater<int>(), v);
   std::priority_queue<int, std::vector<int>, std::greater<int> > q = qo;
   assert(q.size() == 5);
   assert(q.top() == 0);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp
index c25caee07ff4..5aa2c00691a2 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_default.pass.cpp
@@ -21,7 +21,7 @@
 #  include "test_convertible.h"
 #endif
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   typedef std::vector<int, limited_allocator<int, 10> > Container;
   typedef std::priority_queue<int, Container> Q;
   Q q;
@@ -36,5 +36,14 @@ int main(int, char**) {
   static_assert(test_convertible<Q>(), "");
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp
index 9566c18193fb..a9de31993ce6 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter.pass.cpp
@@ -17,12 +17,21 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a[] = {3, 5, 2, 0, 6, 8, 1};
   int* an = a + sizeof(a) / sizeof(a[0]);
   std::priority_queue<int> q(a, an);
   assert(q.size() == static_cast<std::size_t>(an - a));
   assert(q.top() == 8);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp
index aa3e31b91957..a65ca970d2e7 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp.pass.cpp
@@ -18,12 +18,21 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a[] = {3, 5, 2, 0, 6, 8, 1};
   int* an = a + sizeof(a) / sizeof(a[0]);
   std::priority_queue<int, std::vector<int>, std::greater<int> > q(a, an, std::greater<int>());
   assert(q.size() == static_cast<std::size_t>(an - a));
   assert(q.top() == 0);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp
index 49518488f6c9..a2818f13e03b 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_cont.pass.cpp
@@ -17,7 +17,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a[]     = {3, 5, 2, 0, 6, 8, 1};
   const int n = sizeof(a) / sizeof(a[0]);
   std::vector<int> v(a, a + n / 2);
@@ -25,5 +25,14 @@ int main(int, char**) {
   assert(q.size() == n);
   assert(q.top() == 8);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp
index 85e31054babc..72cf9e51bc68 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_iter_iter_comp_rcont.pass.cpp
@@ -20,12 +20,21 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a[]     = {3, 5, 2, 0, 6, 8, 1};
   const int n = sizeof(a) / sizeof(a[0]);
   std::priority_queue<MoveOnly> q(a + n / 2, a + n, std::less<MoveOnly>(), std::vector<MoveOnly>(a, a + n / 2));
   assert(q.size() == n);
   assert(q.top() == MoveOnly(8));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp
index 74213881e98f..b45d30ecb502 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.cons/ctor_move.pass.cpp
@@ -19,18 +19,27 @@
 #include "MoveOnly.h"
 
 template <class C>
-C make(int n) {
+TEST_CONSTEXPR_CXX26 C make(int n) {
   C c;
   for (int i = 0; i < n; ++i)
     c.push_back(MoveOnly(i));
   return c;
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<MoveOnly> qo(std::less<MoveOnly>(), make<std::vector<MoveOnly> >(5));
   std::priority_queue<MoveOnly> q = std::move(qo);
   assert(q.size() == 5);
   assert(q.top() == MoveOnly(4));
 
-  return 0;
+  return true;
 }
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
+  return 0;
+}
\ No newline at end of file
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp
index f9f88fd84308..9c1fdb1cf951 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/emplace.pass.cpp
@@ -20,7 +20,7 @@
 #include "test_macros.h"
 #include "../../../Emplaceable.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<Emplaceable> q;
   q.emplace(1, 2.5);
   assert(q.top() == Emplaceable(1, 2.5));
@@ -29,5 +29,14 @@ int main(int, char**) {
   q.emplace(2, 3.5);
   assert(q.top() == Emplaceable(3, 4.5));
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp
index 758c5354aea4..265b559e04bb 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.pass.cpp
@@ -17,7 +17,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<int> q;
   assert(q.empty());
   q.push(1);
@@ -25,5 +25,14 @@ int main(int, char**) {
   q.pop();
   assert(q.empty());
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp
index dba6614ad5b6..d715cf378a3d 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/pop.pass.cpp
@@ -17,7 +17,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<int> q;
   q.push(1);
   assert(q.top() == 1);
@@ -32,5 +32,14 @@ int main(int, char**) {
   q.pop();
   assert(q.empty());
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp
index 36ba7d241c5b..a242942caef6 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push.pass.cpp
@@ -17,7 +17,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<int> q;
   q.push(1);
   assert(q.top() == 1);
@@ -26,5 +26,14 @@ int main(int, char**) {
   q.push(2);
   assert(q.top() == 3);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp
index 7700b641a429..ce0399b64706 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/push_rvalue.pass.cpp
@@ -20,7 +20,7 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<MoveOnly> q;
   q.push(1);
   assert(q.top() == 1);
@@ -29,5 +29,14 @@ int main(int, char**) {
   q.push(2);
   assert(q.top() == 3);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp
index 2c6b00b87c6b..a8e0002b9808 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/size.pass.cpp
@@ -17,7 +17,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<int> q;
   assert(q.size() == 0);
   q.push(1);
@@ -25,5 +25,14 @@ int main(int, char**) {
   q.pop();
   assert(q.size() == 0);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp
index 87e6722d79e3..8c6ad7f905c0 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/swap.pass.cpp
@@ -17,7 +17,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<int> q1;
   std::priority_queue<int> q2;
   q1.push(1);
@@ -28,5 +28,14 @@ int main(int, char**) {
   assert(q2.size() == 3);
   assert(q2.top() == 3);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp
index 8b046bc3ccca..623c54561e29 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/top.pass.cpp
@@ -17,7 +17,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<int> q;
   q.push(1);
   assert(q.top() == 1);
@@ -26,5 +26,14 @@ int main(int, char**) {
   q.push(2);
   assert(q.top() == 3);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp
index c280601baf17..8e7351742305 100644
--- a/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.special/swap.pass.cpp
@@ -19,7 +19,7 @@
 
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   std::priority_queue<int> q1;
   std::priority_queue<int> q2;
   q1.push(1);
@@ -30,5 +30,14 @@ int main(int, char**) {
   assert(q2.size() == 3);
   assert(q2.top() == 3);
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/exception_safety_helpers.h b/external/llvm-project/libcxx/test/std/containers/exception_safety_helpers.h
index da247571d303..2d26735d9bb4 100644
--- a/external/llvm-project/libcxx/test/std/containers/exception_safety_helpers.h
+++ b/external/llvm-project/libcxx/test/std/containers/exception_safety_helpers.h
@@ -11,6 +11,7 @@
 
 #include <cassert>
 #include <cstddef>
+#include <forward_list>
 #include <functional>
 #include <utility>
 #include "test_macros.h"
@@ -53,6 +54,33 @@ int ThrowingCopy<N>::created_by_copying = 0;
 template <int N>
 int ThrowingCopy<N>::destroyed = 0;
 
+template <int N>
+struct ThrowingDefault {
+  static bool throwing_enabled;
+  static int default_constructed;
+  static int destroyed;
+  int x = 0;
+
+  ThrowingDefault() {
+    ++default_constructed;
+    if (throwing_enabled && default_constructed == N) {
+      throw -1;
+    }
+  }
+
+  ThrowingDefault(int value) : x(value) {}
+  ThrowingDefault(const ThrowingDefault& other) = default;
+  friend bool operator==(const ThrowingDefault& lhs, const ThrowingDefault& rhs) { return lhs.x == rhs.x; }
+  friend bool operator<(const ThrowingDefault& lhs, const ThrowingDefault& rhs) { return lhs.x < rhs.x; }
+
+  static void reset() { default_constructed = destroyed = 0; }
+};
+
+template <int N>
+bool ThrowingDefault<N>::throwing_enabled = true;
+template <int N>
+int ThrowingDefault<N>::default_constructed = 0;
+
 template <int N>
 struct std::hash<ThrowingCopy<N>> {
   std::size_t operator()(const ThrowingCopy<N>& value) const { return value.x; }
@@ -95,6 +123,29 @@ void test_exception_safety_throwing_copy_container(Func&& func) {
   }
 }
 
+template <int ThrowOn, int Size, class Func>
+void test_strong_exception_safety_throwing_copy(Func&& func) {
+  using T             = ThrowingCopy<ThrowOn>;
+  T::throwing_enabled = false;
+
+  std::forward_list<T> c0(Size);
+  for (int i = 0; i < Size; ++i)
+    c0.emplace_front(i);
+  std::forward_list<T> c = c0;
+  T in[Size];
+  T::reset();
+  T::throwing_enabled = true;
+  try {
+    func(c, in, in + Size);
+    assert(false); // The function call above should throw.
+
+  } catch (int) {
+    assert(T::created_by_copying == ThrowOn);
+    assert(T::destroyed == ThrowOn - 1); // No destructor call for the partially-constructed element.
+    assert(c == c0);                     // Strong exception guarantee
+  }
+}
+
 #endif // !defined(TEST_HAS_NO_EXCEPTIONS)
 
 #endif // SUPPORT_EXCEPTION_SAFETY_HELPERS_H
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/array/compare.three_way.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/array/compare.three_way.pass.cpp
index 01be1db73041..671747f89a82 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/array/compare.three_way.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/array/compare.three_way.pass.cpp
@@ -26,7 +26,6 @@ constexpr std::size_t N{1};
 static_assert(std::three_way_comparable<std::array<int, N>>);
 
 // Thanks to SFINAE, the following is not a compiler error but returns `false`
-struct NonComparable {};
 static_assert(!std::three_way_comparable<std::array<NonComparable, N>>);
 
 // Implementation detail of `test_sequence_container_array_spaceship`
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/compare.three_way.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/compare.three_way.pass.cpp
index 52adfc4d8598..a9ef855e9a73 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/compare.three_way.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/compare.three_way.pass.cpp
@@ -11,7 +11,7 @@
 
 // template<class T, class Allocator>
 //   synth-three-way-result<T> operator<=>(const forward_list<T, Allocator>& x,
-//                                         const forward_list<T, Allocator>& y);
+//                                         const forward_list<T, Allocator>& y); // constexpr since C++26
 
 #include <cassert>
 #include <forward_list>
@@ -20,6 +20,9 @@
 
 int main(int, char**) {
   assert(test_sequence_container_spaceship<std::forward_list>());
-  // `std::forward_list` is not constexpr, so no `static_assert` test here.
+#if TEST_STD_VER >= 26
+  static_assert(test_sequence_container_spaceship<std::forward_list>());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp
index dbc0631d1193..4482d26f308a 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/empty.pass.cpp
@@ -10,7 +10,7 @@
 
 // class forward_list
 
-// bool empty() const noexcept;
+// bool empty() const noexcept; // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef std::forward_list<int> C;
     C c;
@@ -42,5 +42,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp
index faf42202e951..a6579e144e6d 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/exception_safety.pass.cpp
@@ -35,18 +35,23 @@
 //     void assign_range(R&& rg); // C++23
 //
 // void push_front(const value_type& v);
-//  template<container-compatible-range<T> R>
+// template <class... Args> reference emplace_front(Args&&... args);  // reference in C++17
+// template<container-compatible-range<T> R>
 //    void prepend_range(R&& rg); // C++23
 //
 // iterator insert_after(const_iterator p, const value_type& v);
+// iterator insert_after(const_iterator p, value_type&& v);
 // iterator insert_after(const_iterator p, size_type n, const value_type& v);
+// iterator insert_after(const_iterator p, initializer_list<value_type> il);
 // template <class InputIterator>
 //     iterator insert_after(const_iterator p,
 //                           InputIterator first, InputIterator last);
-//  template<container-compatible-range<T> R>
+// template<container-compatible-range<T> R>
 //     iterator insert_range_after(const_iterator position, R&& rg); // C++23
-//
+// template <class... Args>
+//     iterator emplace_after(const_iterator p, Args&&... args);
 // void resize(size_type n, const value_type& v);
+// void resize(size_type n);
 
 #include <forward_list>
 
@@ -65,16 +70,33 @@ int main(int, char**) {
     using T               = ThrowingCopy<ThrowOn>;
 
     // void push_front(const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
-      std::forward_list<T> c;
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
+      c.push_front(*from);
+    });
+
+    // template <class... Args> reference emplace_front(Args&&... args);
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
       c.push_front(*from);
     });
 
     // iterator insert_after(const_iterator p, const value_type& v);
-    test_exception_safety_throwing_copy</*ThrowOn=*/1, Size>([](T* from, T*) {
-      std::forward_list<T> c;
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
       c.insert_after(c.before_begin(), *from);
     });
+
+    // iterator insert_after(const_iterator p, value_type&& v);
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
+      c.insert_after(c.before_begin(), std::move(*from));
+    });
+
+    // template <class... Args>
+    //     iterator emplace_after(const_iterator p, Args&&... args);
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
+      c.emplace_after(c.before_begin(), *from);
+    });
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
+      c.emplace_after(c.before_begin(), std::move(*from));
+    });
   }
 
   {
@@ -169,40 +191,53 @@ int main(int, char**) {
 #if TEST_STD_VER >= 23
     // template<container-compatible-range<T> R>
     //   void prepend_range(R&& rg); // C++23
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
-      std::forward_list<T> c;
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T* to) {
       c.prepend_range(std::ranges::subrange(from, to));
     });
 #endif
 
     // iterator insert_after(const_iterator p, size_type n, const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
-      std::forward_list<T> c;
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
       c.insert_after(c.before_begin(), Size, *from);
     });
 
     // template <class InputIterator>
     //     iterator insert_after(const_iterator p,
     //                           InputIterator first, InputIterator last);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
-      std::forward_list<T> c;
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T* to) {
       c.insert_after(c.before_begin(), from, to);
     });
 
+    // iterator insert_after(const_iterator p, initializer_list<value_type> il);
+    std::initializer_list<T> il{1, 2, 3, 4, 5};
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([&](std::forward_list<T>& c, T*, T*) {
+      c.insert_after(c.before_begin(), il);
+    });
+
 #if TEST_STD_VER >= 23
     // template<container-compatible-range<T> R>
     //     iterator insert_range_after(const_iterator position, R&& rg); // C++23
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T* to) {
-      std::forward_list<T> c;
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T* to) {
       c.insert_range_after(c.before_begin(), std::ranges::subrange(from, to));
     });
 #endif
 
     // void resize(size_type n, const value_type& v);
-    test_exception_safety_throwing_copy<ThrowOn, Size>([](T* from, T*) {
-      std::forward_list<T> c;
-      c.resize(Size, *from);
-    });
+    test_strong_exception_safety_throwing_copy<ThrowOn, Size>([](std::forward_list<T>& c, T* from, T*) {
+      c.resize(Size * 3, *from);
+    });
+
+    { // void resize(size_type n);
+      using X = ThrowingDefault<ThrowOn>;
+      std::forward_list<X> c0{X{1}, X{2}, X{3}};
+      std::forward_list<X> c = c0;
+      try {
+        c.resize(3 * ThrowOn);
+        assert(false);
+      } catch (int) {
+        assert(c == c0);
+      }
+    }
   }
 
   return 0;
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp
index 757db7d957f5..50b549f17d56 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.access/front.pass.cpp
@@ -8,17 +8,18 @@
 
 // <forward_list>
 
-// reference       front();
-// const_reference front() const;
+// reference       front();       // constexpr since C++26
+// const_reference front() const; // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
 #include <iterator>
 
+#include "test_allocator.h"
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -58,5 +59,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp
index 31893a1b9599..4645560048cf 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.compile.fail.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// explicit forward_list(const allocator_type& a);
+// explicit forward_list(const allocator_type& a); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_allocator.h"
 #include "../../../NotConstructible.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef test_allocator<NotConstructible> A;
     typedef A::value_type T;
@@ -26,5 +26,14 @@ int main(int, char**) {
     assert(c.empty());
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp
index bfb330fdaf9f..ffc6d37f2816 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/alloc.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// explicit forward_list(const allocator_type& a);
+// explicit forward_list(const allocator_type& a); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "../../../NotConstructible.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef test_allocator<NotConstructible> A;
     typedef A::value_type T;
@@ -46,5 +46,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp
index 27d450c63dca..b99af4ccb79e 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_copy.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// forward_list& operator=(const forward_list& x);
+// forward_list& operator=(const forward_list& x); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef test_allocator<int> A;
@@ -143,5 +143,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp
index 1cdcca82d335..ea2802b323a9 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_init.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// void assign(initializer_list<value_type> il);
+// void assign(initializer_list<value_type> il); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -65,5 +65,14 @@ int main(int, char**) {
     assert(n == 4);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp
index 998a7e11ef34..9c88db6166ba 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_move.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// forward_list& operator=(forward_list&& x);
+// forward_list& operator=(forward_list&& x); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -21,7 +21,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef MoveOnly T;
     typedef test_allocator<T> A;
@@ -194,5 +194,14 @@ int main(int, char**) {
     assert(c0.empty());
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp
index a22d6c4985bc..d21898dc4663 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_op_init.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// forward_list& operator=(initializer_list<value_type> il);
+// forward_list& operator=(initializer_list<value_type> il); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -65,5 +65,14 @@ int main(int, char**) {
     assert(n == 4);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp
index 9a3532874079..1601b4b47acd 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_range.pass.cpp
@@ -9,7 +9,7 @@
 // <forward_list>
 
 // template <class InputIterator>
-//     void assign(InputIterator first, InputIterator last);
+//     void assign(InputIterator first, InputIterator last); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -75,5 +75,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp
index b0fbfa3249e5..75626b47c527 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/assign_size_value.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void assign(size_type n, const value_type& v);
+// void assign(size_type n, const value_type& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -65,5 +65,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp
index 22d5054b9ae1..12d701bff4b6 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// forward_list(const forward_list& x);
+// forward_list(const forward_list& x); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef test_allocator<int> A;
@@ -64,5 +64,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp
index a61233e4b5d2..fc3ff485b066 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/copy_alloc.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// forward_list(const forward_list& x, const allocator_type& a);
+// forward_list(const forward_list& x, const allocator_type& a); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef test_allocator<int> A;
@@ -64,5 +64,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp
index b493a89b7800..e0ea8bf66cb3 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/default.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// forward_list();
+// forward_list(); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -38,5 +38,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/from_range.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/from_range.pass.cpp
index 312f6dbad355..d1e1734e86f9 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/from_range.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/from_range.pass.cpp
@@ -9,14 +9,14 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   forward_list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
+//   forward_list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23; constexpr since C++26
 
 #include <forward_list>
 
 #include "../../from_range_sequence_containers.h"
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   for_all_iterators_and_allocators<int>([]<class Iter, class Sent, class Alloc>() {
     test_sequence_container<std::forward_list, int, Iter, Sent, Alloc>([](const auto&) {
       // No additional validation to do.
@@ -26,8 +26,19 @@ int main(int, char**) {
 
   static_assert(test_constraints<std::forward_list, int, double>());
 
-  test_exception_safety_throwing_copy<std::forward_list>();
-  test_exception_safety_throwing_allocator<std::forward_list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_exception_safety_throwing_copy<std::forward_list>();
+    test_exception_safety_throwing_allocator<std::forward_list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp
index b42242b0a83d..b7acf60aa70c 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// forward_list(initializer_list<value_type> il);
+// forward_list(initializer_list<value_type> il); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -38,5 +38,14 @@ int main(int, char**) {
     assert(n == 10);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp
index 0b29cbfa9254..33d569c921a9 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/init_alloc.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// forward_list(initializer_list<value_type> il, const allocator_type& a);
+// forward_list(initializer_list<value_type> il, const allocator_type& a); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef test_allocator<T> A;
@@ -43,5 +43,14 @@ int main(int, char**) {
     assert(c.get_allocator() == A());
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp
index 762e252ca76f..20575479f735 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// forward_list(forward_list&& x);
+// forward_list(forward_list&& x); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -21,7 +21,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef MoveOnly T;
     typedef test_allocator<T> A;
@@ -68,5 +68,14 @@ int main(int, char**) {
     assert(c.get_allocator() == A());
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp
index a9bc2cb12f28..219505bf4fd1 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/move_alloc.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// forward_list(forward_list&& x, const allocator_type& a);
+// forward_list(forward_list&& x, const allocator_type& a); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -21,7 +21,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef MoveOnly T;
     typedef test_allocator<T> A;
@@ -68,5 +68,14 @@ int main(int, char**) {
     assert(c.get_allocator() == A());
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp
index ebd0e6a5bd1e..61393eb28938 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range.pass.cpp
@@ -9,7 +9,7 @@
 // <forward_list>
 
 // template <class InputIterator>
-//     forward_list(InputIterator first, InputIterator last);
+//     forward_list(InputIterator first, InputIterator last); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -45,5 +45,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp
index 4a28041ad2cb..c0637420e328 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/range_alloc.pass.cpp
@@ -10,7 +10,7 @@
 
 // template <class InputIterator>
 //     forward_list(InputIterator first, InputIterator last,
-//                  const allocator_type& a);
+//                  const allocator_type& a);                 // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -21,7 +21,7 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef test_allocator<T> A;
@@ -51,5 +51,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp
index 81b128d2149e..206854560c19 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size.pass.cpp
@@ -8,8 +8,8 @@
 
 // <forward_list>
 
-// explicit forward_list(size_type n);
-// explicit forward_list(size_type n, const Alloc& a);
+// explicit forward_list(size_type n);                 // constexpr since C++26
+// explicit forward_list(size_type n, const Alloc& a); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp
index 663422d1c3c3..85d11e3f40a2 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// forward_list(size_type n, const value_type& v);
+// forward_list(size_type n, const value_type& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -42,5 +42,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp
index af7f7471d4c9..abcdf62452b8 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.cons/size_value_alloc.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// forward_list(size_type n, const value_type& v, const allocator_type& a);
+// forward_list(size_type n, const value_type& v, const allocator_type& a); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef test_allocator<int> A;
     typedef A::value_type T;
@@ -47,5 +47,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp
index 1044d779220e..86d7769fe16e 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase.pass.cpp
@@ -11,7 +11,7 @@
 
 // template <class T, class Allocator, class U>
 //   typename forward_list<T, Allocator>::size_type
-//   erase(forward_list<T, Allocator>& c, const U& value);
+//   erase(forward_list<T, Allocator>& c, const U& value); // constexpr since C++26
 
 #include <forward_list>
 #include <optional>
@@ -21,14 +21,14 @@
 #include "min_allocator.h"
 
 template <class S, class U>
-void test0(S s, U val, S expected, std::size_t expected_erased_count) {
+TEST_CONSTEXPR_CXX26 void test0(S s, U val, S expected, std::size_t expected_erased_count) {
   ASSERT_SAME_TYPE(typename S::size_type, decltype(std::erase(s, val)));
   assert(expected_erased_count == std::erase(s, val));
   assert(s == expected);
 }
 
 template <class S>
-void test() {
+TEST_CONSTEXPR_CXX26 void test() {
   test0(S(), 1, S(), 0);
 
   test0(S({1}), 1, S(), 1);
@@ -62,13 +62,21 @@ void test() {
   test0(S({1, 2, 1}), opt(3), S({1, 2, 1}), 0);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   test<std::forward_list<int>>();
   test<std::forward_list<int, min_allocator<int>>>();
   test<std::forward_list<int, test_allocator<int>>>();
-
   test<std::forward_list<long>>();
   test<std::forward_list<double>>();
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp
index c4f45a1069a2..c665f9cccbf0 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.erasure/erase_if.pass.cpp
@@ -11,7 +11,7 @@
 
 // template <class T, class Allocator, class Predicate>
 //   typename forward_list<T, Allocator>::size_type
-//   erase_if(forward_list<T, Allocator>& c, Predicate pred);
+//   erase_if(forward_list<T, Allocator>& c, Predicate pred); // constexpr since C++26
 
 #include <forward_list>
 
@@ -20,14 +20,14 @@
 #include "min_allocator.h"
 
 template <class S, class Pred>
-void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
+TEST_CONSTEXPR_CXX26 void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
   ASSERT_SAME_TYPE(typename S::size_type, decltype(std::erase_if(s, p)));
   assert(expected_erased_count == std::erase_if(s, p));
   assert(s == expected);
 }
 
 template <typename S>
-void test() {
+TEST_CONSTEXPR_CXX26 void test() {
   auto is1   = [](auto v) { return v == 1; };
   auto is2   = [](auto v) { return v == 2; };
   auto is3   = [](auto v) { return v == 3; };
@@ -64,13 +64,21 @@ void test() {
   test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   test<std::forward_list<int>>();
   test<std::forward_list<int, min_allocator<int>>>();
   test<std::forward_list<int, test_allocator<int>>>();
-
   test<std::forward_list<long>>();
   test<std::forward_list<double>>();
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp
index d66d2cd87951..52b5d87860aa 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/before_begin.pass.cpp
@@ -8,9 +8,9 @@
 
 // <forward_list>
 
-// iterator       before_begin();
-// const_iterator before_begin() const;
-// const_iterator cbefore_begin() const;
+// iterator       before_begin();        // constexpr since C++26
+// const_iterator before_begin() const;  // constexpr since C++26
+// const_iterator cbefore_begin() const; // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -101,5 +101,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp
index 135689b2321c..560c47b17958 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.iter/iterators.pass.cpp
@@ -8,12 +8,12 @@
 
 // <forward_list>
 
-// iterator       begin();
-// iterator       end();
-// const_iterator begin()  const;
-// const_iterator end()    const;
-// const_iterator cbegin() const;
-// const_iterator cend()   const;
+// iterator       begin();        // constexpr since C++26
+// iterator       end();          // constexpr since C++26
+// const_iterator begin()  const; // constexpr since C++26
+// const_iterator end()    const; // constexpr since C++26
+// const_iterator cbegin() const; // constexpr since C++26
+// const_iterator cend()   const; // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -22,7 +22,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -69,6 +69,8 @@ int main(int, char**) {
     typedef std::forward_list<T> C;
     C::iterator i;
     C::const_iterator j;
+    (void)i;
+    (void)j;
   }
 #if TEST_STD_VER >= 11
   {
@@ -117,6 +119,8 @@ int main(int, char**) {
     typedef std::forward_list<T, min_allocator<T>> C;
     C::iterator i;
     C::const_iterator j;
+    (void)i;
+    (void)j;
   }
 #endif
 #if TEST_STD_VER > 11
@@ -142,5 +146,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/assign_range.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/assign_range.pass.cpp
index a27cc757025b..9a3adec1d975 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/assign_range.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/assign_range.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   constexpr void prepend_range(R&& rg); // C++23
+//   constexpr void prepend_range(R&& rg); // C++23; constexpr since C++26
 
 #include <forward_list>
 
@@ -21,7 +21,7 @@
 //   {empty/one-element/full} container);
 // - prepending move-only elements;
 // - an exception is thrown when copying the elements or when allocating new elements.
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   static_assert(test_constraints_assign_range<std::forward_list, int, double>());
 
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
@@ -31,8 +31,19 @@ int main(int, char**) {
   });
   test_sequence_prepend_range_move_only<std::forward_list>();
 
-  test_prepend_range_exception_safety_throwing_copy<std::forward_list>();
-  test_prepend_range_exception_safety_throwing_allocator<std::forward_list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_prepend_range_exception_safety_throwing_copy<std::forward_list>();
+    test_prepend_range_exception_safety_throwing_allocator<std::forward_list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp
index 9f6d34b701df..2e1768cf8bad 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/clear.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void clear() noexcept;
+// void clear() noexcept; // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "../../../NotConstructible.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef NotConstructible T;
     typedef std::forward_list<T> C;
@@ -64,5 +64,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp
index f77d47ee7c74..6433607af9b3 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_after.pass.cpp
@@ -11,7 +11,7 @@
 // <forward_list>
 
 // template <class... Args>
-//     iterator emplace_after(const_iterator p, Args&&... args);
+//     iterator emplace_after(const_iterator p, Args&&... args); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -20,7 +20,7 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef Emplaceable T;
     typedef std::forward_list<T> C;
@@ -84,5 +84,14 @@ int main(int, char**) {
     assert(std::distance(c.begin(), c.end()) == 4);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp
index cd3bb20c52ae..46ae27b43622 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/emplace_front.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// template <class... Args> reference emplace_front(Args&&... args);
+// template <class... Args> reference emplace_front(Args&&... args); // constexpr since C++26
 // return type is 'reference' in C++17; 'void' before
 
 #include <forward_list>
@@ -21,7 +21,7 @@
 #include "../../../Emplaceable.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef Emplaceable T;
     typedef std::forward_list<T> C;
@@ -67,5 +67,14 @@ int main(int, char**) {
     assert(std::distance(c.begin(), c.end()) == 2);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp
index e85951798526..73cb03c2cb7d 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_many.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// iterator erase_after(const_iterator first, const_iterator last);
+// iterator erase_after(const_iterator first, const_iterator last); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -153,5 +153,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp
index 892228e76def..12997f1dad3b 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/erase_after_one.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// iterator erase_after(const_iterator p);
+// iterator erase_after(const_iterator p); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -95,5 +95,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp
index 8443158413e7..d93789dd6bb5 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_const.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// iterator insert_after(const_iterator p, const value_type& v);
+// iterator insert_after(const_iterator p, const value_type& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -84,5 +84,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp
index de924a10c18f..54be47f4264f 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_init.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// iterator insert_after(const_iterator p, initializer_list<value_type> il);
+// iterator insert_after(const_iterator p, initializer_list<value_type> il); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -70,5 +70,14 @@ int main(int, char**) {
     assert(*std::next(c.begin(), 4) == 2);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp
index af810d0f6961..f89fbd7619da 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_range.pass.cpp
@@ -10,7 +10,7 @@
 
 // template <class InputIterator>
 //     iterator insert_after(const_iterator p,
-//                           InputIterator first, InputIterator last);
+//                           InputIterator first, InputIterator last); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_iterators.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -77,5 +77,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp
index acd4bc73f724..01b76f5cd64f 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_rv.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// iterator insert_after(const_iterator p, value_type&& v);
+// iterator insert_after(const_iterator p, value_type&& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef MoveOnly T;
     typedef std::forward_list<T> C;
@@ -85,5 +85,14 @@ int main(int, char**) {
     assert(std::distance(c.begin(), c.end()) == 4);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp
index 2506f04311e0..f4f0521ad237 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_after_size_value.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// iterator insert_after(const_iterator p, size_type n, const value_type& v);
+// iterator insert_after(const_iterator p, size_type n, const value_type& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -70,5 +70,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp
index 25f4c43f3848..71a291430b43 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/insert_range_after.pass.cpp
@@ -8,8 +8,10 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
+// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=20000000
+
 // template<container-compatible-range<T> R>
-//   constexpr iterator insert_range_after(const_iterator position, R&& rg); // C++23
+//   constexpr iterator insert_range_after(const_iterator position, R&& rg); // C++23; constexpr since C++26
 
 #include <forward_list>
 
@@ -321,7 +323,7 @@ constexpr void test_sequence_insert_range_after() {
   }
 }
 
-void test_sequence_insert_range_after_move_only() {
+TEST_CONSTEXPR_CXX26 void test_sequence_insert_range_after_move_only() {
   MoveOnly input[5];
   std::ranges::subrange in(std::move_iterator{input}, std::move_iterator{input + 5});
 
@@ -366,7 +368,7 @@ void test_insert_range_after_exception_safety_throwing_allocator() {
 #endif
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   static_assert(test_constraints_insert_range_after<std::forward_list, int, double>());
 
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
@@ -374,8 +376,19 @@ int main(int, char**) {
   });
   test_sequence_insert_range_after_move_only();
 
-  test_insert_range_after_exception_safety_throwing_copy();
-  test_insert_range_after_exception_safety_throwing_allocator<int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_insert_range_after_exception_safety_throwing_copy();
+    test_insert_range_after_exception_safety_throwing_allocator<int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp
index 98c7a2634117..9fcade7ff6bb 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/pop_front.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void pop_front();
+// void pop_front(); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -71,5 +71,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/prepend_range.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/prepend_range.pass.cpp
index 418aa72052ba..c4b9cd9bdfc4 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/prepend_range.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/prepend_range.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   constexpr void prepend_range(R&& rg); // C++23
+//   constexpr void prepend_range(R&& rg); // C++23; constexpr since C++26
 
 #include <forward_list>
 
@@ -21,7 +21,7 @@
 //   {empty/one-element/full} container);
 // - prepending move-only elements;
 // - an exception is thrown when copying the elements or when allocating new elements.
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   static_assert(test_constraints_prepend_range<std::forward_list, int, double>());
 
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
@@ -31,8 +31,19 @@ int main(int, char**) {
   });
   test_sequence_prepend_range_move_only<std::forward_list>();
 
-  test_prepend_range_exception_safety_throwing_copy<std::forward_list>();
-  test_prepend_range_exception_safety_throwing_allocator<std::forward_list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_prepend_range_exception_safety_throwing_copy<std::forward_list>();
+    test_prepend_range_exception_safety_throwing_allocator<std::forward_list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp
index f99c40fa0c1a..61c5dcac0545 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_const.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void push_front(const value_type& v);
+// void push_front(const value_type& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -44,5 +44,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp
index 467037465eed..cd24d6ff6af0 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_exception_safety.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: no-exceptions
 // <forward_list>
 
-// void push_front(const value_type& x);
+// void push_front(const value_type& x); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp
index d3156c5fdd38..b30ff7a0189e 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/push_front_rv.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// void push_front(value_type&& v);
+// void push_front(value_type&& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef MoveOnly T;
     typedef std::forward_list<T> C;
@@ -45,5 +45,14 @@ int main(int, char**) {
     assert(std::distance(c.begin(), c.end()) == 2);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp
index 2dacf458d7d9..f80886113bf2 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void resize(size_type n);
+// void resize(size_type n); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,8 +18,8 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
-  {
+TEST_CONSTEXPR_CXX26 bool test() {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     typedef DefaultOnly T;
     typedef std::forward_list<T> C;
     C c;
@@ -65,7 +65,7 @@ int main(int, char**) {
     assert(*std::next(c.begin(), 5) == 0);
   }
 #if TEST_STD_VER >= 11
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     typedef DefaultOnly T;
     typedef std::forward_list<T, min_allocator<T>> C;
     C c;
@@ -112,5 +112,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp
index a6af763e6937..4ec859b36336 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.modifiers/resize_size_value.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void resize(size_type n, const value_type& v);
+// void resize(size_type n, const value_type& v); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -22,7 +22,7 @@
 #  include "container_test_types.h"
 #endif
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -84,7 +84,7 @@ int main(int, char**) {
     assert(*std::next(c.begin(), 4) == 10);
     assert(*std::next(c.begin(), 5) == 10);
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // Test that the allocator's construct method is being used to
     // construct the new elements and that it's called exactly N times.
     typedef std::forward_list<int, ContainerTestAllocator<int, int>> Container;
@@ -99,5 +99,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue.pass.cpp
index 9a162789569d..d8e80c56bf39 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void merge(forward_list& x);
+// void merge(forward_list& x); // constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -30,11 +30,11 @@ struct value {
   int a;
   int b;
 
-  friend bool operator<(const value& lhs, const value& rhs) { return lhs.a < rhs.a; }
-  friend bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
+  friend TEST_CONSTEXPR bool operator<(const value& lhs, const value& rhs) { return lhs.a < rhs.a; }
+  friend TEST_CONSTEXPR bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   { // Basic merge operation.
     typedef int T;
     typedef std::forward_list<T> C;
@@ -116,5 +116,14 @@ int main(int, char**) {
     assert(c == std::forward_list<int>(std::begin(a), std::end(a)));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue_pred.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue_pred.pass.cpp
index 4e1814044808..0adadb2dd092 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue_pred.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_lvalue_pred.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// template <class Compare> void merge(forward_list& x, Compare comp);
+// template <class Compare> void merge(forward_list& x, Compare comp); // constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -30,11 +30,11 @@ struct value {
   int a;
   int b;
 
-  friend bool operator>(const value& lhs, const value& rhs) { return lhs.a > rhs.a; }
-  friend bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
+  friend TEST_CONSTEXPR bool operator>(const value& lhs, const value& rhs) { return lhs.a > rhs.a; }
+  friend TEST_CONSTEXPR bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   { // Basic merge operation.
     typedef int T;
     typedef std::forward_list<T> C;
@@ -117,5 +117,14 @@ int main(int, char**) {
     assert(c == std::forward_list<int>(std::begin(a), std::end(a)));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp
index acfa014fe254..906748ec2702 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// void merge(forward_list&& x);
+// void merge(forward_list&& x); // constexpr since C++26
 
 #include <forward_list>
 #include <functional>
@@ -29,11 +29,11 @@ struct value {
   int a;
   int b;
 
-  friend bool operator<(const value& lhs, const value& rhs) { return lhs.a < rhs.a; }
-  friend bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
+  friend TEST_CONSTEXPR bool operator<(const value& lhs, const value& rhs) { return lhs.a < rhs.a; }
+  friend TEST_CONSTEXPR bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   { // Basic merge operation.
     typedef int T;
     typedef std::forward_list<T> C;
@@ -109,5 +109,14 @@ int main(int, char**) {
     assert(c == std::forward_list<int>(std::begin(a), std::end(a)));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp
index 41b56ce7a288..2ced0b1596e4 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/merge_rvalue_pred.pass.cpp
@@ -10,7 +10,7 @@
 
 // <forward_list>
 
-// template <class Compare> void merge(forward_list&& x, Compare comp);
+// template <class Compare> void merge(forward_list&& x, Compare comp); // constexpr since C++26
 
 #include <forward_list>
 #include <functional>
@@ -29,11 +29,11 @@ struct value {
   int a;
   int b;
 
-  friend bool operator>(const value& lhs, const value& rhs) { return lhs.a > rhs.a; }
-  friend bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
+  friend TEST_CONSTEXPR bool operator>(const value& lhs, const value& rhs) { return lhs.a > rhs.a; }
+  friend TEST_CONSTEXPR bool operator==(const value& lhs, const value& rhs) { return lhs.a == rhs.a && lhs.b == rhs.b; }
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   { // Basic merge operation.
     typedef int T;
     typedef std::forward_list<T> C;
@@ -110,5 +110,14 @@ int main(int, char**) {
     assert(c == std::forward_list<int>(std::begin(a), std::end(a)));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp
index ec3bf845dcc5..b17708ba60ee 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove.pass.cpp
@@ -9,7 +9,7 @@
 // <forward_list>
 
 // void remove(const value_type& v);      // C++17 and before
-// size_type remove(const value_type& v); // C++20 and after
+// size_type remove(const value_type& v); // C++20 and after; // constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -19,7 +19,7 @@
 #include "min_allocator.h"
 
 template <class L>
-void do_remove(L& l, const typename L::value_type& value, typename L::size_type expected) {
+TEST_CONSTEXPR_CXX26 void do_remove(L& l, const typename L::value_type& value, typename L::size_type expected) {
   typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
   ASSERT_SAME_TYPE(decltype(l.remove(value)), typename L::size_type);
@@ -32,22 +32,22 @@ void do_remove(L& l, const typename L::value_type& value, typename L::size_type
 }
 
 struct S {
-  S(int i) : i_(new int(i)) {}
-  S(const S& rhs) : i_(new int(*rhs.i_)) {}
-  S& operator=(const S& rhs) {
+  TEST_CONSTEXPR_CXX20 S(int i) : i_(new int(i)) {}
+  TEST_CONSTEXPR_CXX20 S(const S& rhs) : i_(new int(*rhs.i_)) {}
+  TEST_CONSTEXPR_CXX20 S& operator=(const S& rhs) {
     *i_ = *rhs.i_;
     return *this;
   }
-  ~S() {
+  TEST_CONSTEXPR_CXX20 ~S() {
     delete i_;
     i_ = NULL;
   }
-  bool operator==(const S& rhs) const { return *i_ == *rhs.i_; }
-  int get() const { return *i_; }
+  TEST_CONSTEXPR bool operator==(const S& rhs) const { return *i_ == *rhs.i_; }
+  TEST_CONSTEXPR int get() const { return *i_; }
   int* i_;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -171,5 +171,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp
index c6325baea259..f26205d03f64 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/remove_if.pass.cpp
@@ -9,7 +9,7 @@
 // <forward_list>
 
 // template <class Predicate> void      remove_if(Predicate pred); // C++17 and before
-// template <class Predicate> size_type remove_if(Predicate pred); // C++20 and after
+// template <class Predicate> size_type remove_if(Predicate pred); // C++20 and after; constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -22,7 +22,7 @@
 #include "counting_predicates.h"
 
 template <class L, class Predicate>
-void do_remove_if(L& l, Predicate pred, typename L::size_type expected) {
+TEST_CONSTEXPR_CXX26 void do_remove_if(L& l, Predicate pred, typename L::size_type expected) {
   typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
   ASSERT_SAME_TYPE(decltype(l.remove_if(pred)), typename L::size_type);
@@ -34,18 +34,18 @@ void do_remove_if(L& l, Predicate pred, typename L::size_type expected) {
   assert(old_size - std::distance(l.begin(), l.end()) == expected);
 }
 
-bool g(int i) { return i < 3; }
+TEST_CONSTEXPR bool g(int i) { return i < 3; }
 
 struct PredLWG526 {
-  PredLWG526(int i) : i_(i) {}
-  ~PredLWG526() { i_ = -32767; }
-  bool operator()(const PredLWG526& p) const { return p.i_ == i_; }
+  TEST_CONSTEXPR_CXX20 PredLWG526(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 ~PredLWG526() { i_ = -32767; }
+  TEST_CONSTEXPR bool operator()(const PredLWG526& p) const { return p.i_ == i_; }
 
-  bool operator==(int i) const { return i == i_; }
+  TEST_CONSTEXPR bool operator==(int i) const { return i == i_; }
   int i_;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef unary_counting_predicate<bool (*)(T), T> Predicate;
@@ -187,5 +187,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp
index 0d0656897f34..38f0e74f6632 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/reverse.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void reverse();
+// void reverse(); // constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -19,7 +19,7 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(int N) {
+TEST_CONSTEXPR_CXX26 void test1(int N) {
   C c;
   for (int i = 0; i < N; ++i)
     c.push_front(i);
@@ -30,12 +30,21 @@ void test(int N) {
     assert(*j == i);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   for (int i = 0; i < 10; ++i)
-    test<std::forward_list<int> >(i);
+    test1<std::forward_list<int> >(i);
 #if TEST_STD_VER >= 11
   for (int i = 0; i < 10; ++i)
-    test<std::forward_list<int, min_allocator<int>> >(i);
+    test1<std::forward_list<int, min_allocator<int>> >(i);
+#endif
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp
index 4c91d7397adf..f8787d70784d 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_flist.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void splice_after(const_iterator p, forward_list&& x);
+// void splice_after(const_iterator p, forward_list&& x); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,13 +19,13 @@
 #include "min_allocator.h"
 
 typedef int T;
-const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
-const T t2[]                 = {10, 11, 12, 13, 14, 15};
-const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
-const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
+TEST_CONSTEXPR const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
+TEST_CONSTEXPR const T t2[]                 = {10, 11, 12, 13, 14, 15};
+TEST_CONSTEXPR const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
+TEST_CONSTEXPR const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
 
 template <class C>
-void testd(const C& c, int p, int l) {
+TEST_CONSTEXPR_CXX26 void testd(const C& c, int p, int l) {
   typename C::const_iterator i = c.begin();
   int n1                       = 0;
   for (; n1 < p; ++n1, ++i)
@@ -37,7 +37,7 @@ void testd(const C& c, int p, int l) {
   assert(std::distance(c.begin(), c.end()) == size_t1 + l);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     // splicing different containers
     typedef std::forward_list<T> C;
@@ -67,5 +67,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp
index bb8bdea63254..7202b0e15362 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_one.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void splice_after(const_iterator p, forward_list&& x, const_iterator i);
+// void splice_after(const_iterator p, forward_list&& x, const_iterator i); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,13 +19,13 @@
 #include "min_allocator.h"
 
 typedef int T;
-const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
-const T t2[]                 = {10, 11, 12};
-const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
-const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
+TEST_CONSTEXPR const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
+TEST_CONSTEXPR const T t2[]                 = {10, 11, 12};
+TEST_CONSTEXPR const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
+TEST_CONSTEXPR const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
 
 template <class C>
-void testd(const C& c, int p, int f) {
+TEST_CONSTEXPR_CXX26 void testd(const C& c, int p, int f) {
   typename C::const_iterator i = c.begin();
   int n1                       = 0;
   for (; n1 < p; ++n1, ++i)
@@ -38,7 +38,7 @@ void testd(const C& c, int p, int f) {
 }
 
 template <class C>
-void tests(const C& c, int p, int f) {
+TEST_CONSTEXPR_CXX26 void tests(const C& c, int p, int f) {
   typename C::const_iterator i = c.begin();
   int n                        = 0;
   if (p == f || p == f + 1) {
@@ -67,7 +67,7 @@ void tests(const C& c, int p, int f) {
   assert(std::distance(c.begin(), c.end()) == size_t1);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     // splicing different containers
     typedef std::forward_list<T> C;
@@ -117,5 +117,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp
index 99b3ed1c7836..18da6f12b28d 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/splice_after_range.pass.cpp
@@ -8,8 +8,10 @@
 
 // <forward_list>
 
+// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=3000000
+
 // void splice_after(const_iterator p, forward_list&& x,
-//                   const_iterator first, const_iterator last);
+//                   const_iterator first, const_iterator last); // constexpr since C++26
 
 #include <stddef.h>
 #include <forward_list>
@@ -20,13 +22,13 @@
 #include "min_allocator.h"
 
 typedef std::ptrdiff_t T;
-const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
-const T t2[]                 = {10, 11, 12, 13, 14, 15};
-const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
-const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
+TEST_CONSTEXPR const T t1[]                 = {0, 1, 2, 3, 4, 5, 6, 7};
+TEST_CONSTEXPR const T t2[]                 = {10, 11, 12, 13, 14, 15};
+TEST_CONSTEXPR const std::ptrdiff_t size_t1 = std::end(t1) - std::begin(t1);
+TEST_CONSTEXPR const std::ptrdiff_t size_t2 = std::end(t2) - std::begin(t2);
 
 template <class C>
-void testd(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
+TEST_CONSTEXPR_CXX26 void testd(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
   typename C::const_iterator i = c.begin();
   std::ptrdiff_t n1            = 0;
   for (; n1 < p; ++n1, ++i)
@@ -39,7 +41,7 @@ void testd(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
 }
 
 template <class C>
-void tests(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
+TEST_CONSTEXPR_CXX26 void tests(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
   typename C::const_iterator i = c.begin();
   std::ptrdiff_t n             = 0;
   std::ptrdiff_t d             = l > f + 1 ? l - 1 - f : 0;
@@ -69,7 +71,7 @@ void tests(const C& c, std::ptrdiff_t p, ptrdiff_t f, ptrdiff_t l) {
   assert(std::distance(c.begin(), c.end()) == size_t1);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     // splicing different containers
     typedef std::forward_list<T> C;
@@ -157,5 +159,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp
index ebd1a79cdb4b..28efff3849e6 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique.pass.cpp
@@ -9,7 +9,7 @@
 // <forward_list>
 
 // void unique();      // C++17 and before
-// size_type unique(); // C++20 and after
+// size_type unique(); // C++20 and after; constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -19,7 +19,7 @@
 #include "min_allocator.h"
 
 template <class L>
-void do_unique(L& l, typename L::size_type expected) {
+TEST_CONSTEXPR_CXX26 void do_unique(L& l, typename L::size_type expected) {
   typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
   ASSERT_SAME_TYPE(decltype(l.unique()), typename L::size_type);
@@ -31,7 +31,7 @@ void do_unique(L& l, typename L::size_type expected) {
   assert(old_size - std::distance(l.begin(), l.end()) == expected);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -131,5 +131,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp
index 408cbf6ae9c2..f07142dffe9d 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.ops/unique_pred.pass.cpp
@@ -9,7 +9,7 @@
 // <forward_list>
 
 // template <class BinaryPredicate> void unique(BinaryPredicate binary_pred);      // C++17 and before
-// template <class BinaryPredicate> size_type unique(BinaryPredicate binary_pred); // C++20 and after
+// template <class BinaryPredicate> size_type unique(BinaryPredicate binary_pred); // C++20 and after; constexpr since C++26
 
 #include <cassert>
 #include <forward_list>
@@ -20,7 +20,7 @@
 #include "min_allocator.h"
 
 template <class L, class Predicate>
-void do_unique(L& l, Predicate pred, typename L::size_type expected) {
+TEST_CONSTEXPR_CXX26 void do_unique(L& l, Predicate pred, typename L::size_type expected) {
   typename L::size_type old_size = std::distance(l.begin(), l.end());
 #if TEST_STD_VER > 17
   ASSERT_SAME_TYPE(decltype(l.unique(pred)), typename L::size_type);
@@ -33,17 +33,17 @@ void do_unique(L& l, Predicate pred, typename L::size_type expected) {
 }
 
 struct PredLWG526 {
-  PredLWG526(int i) : i_(i) {}
-  ~PredLWG526() { i_ = -32767; }
-  bool operator()(const PredLWG526& lhs, const PredLWG526& rhs) const { return lhs.i_ == rhs.i_; }
+  TEST_CONSTEXPR_CXX20 PredLWG526(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 ~PredLWG526() { i_ = -32767; }
+  TEST_CONSTEXPR bool operator()(const PredLWG526& lhs, const PredLWG526& rhs) const { return lhs.i_ == rhs.i_; }
 
-  bool operator==(int i) const { return i == i_; }
+  TEST_CONSTEXPR bool operator==(int i) const { return i == i_; }
   int i_;
 };
 
-bool g(int x, int y) { return x == y; }
+TEST_CONSTEXPR bool g(int x, int y) { return x == y; }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::forward_list<T> C;
@@ -157,5 +157,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp
index ef6b72ee360a..cb57b094a077 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/equal.pass.cpp
@@ -10,11 +10,11 @@
 
 // template <class T, class Allocator>
 //     bool operator==(const forward_list<T, Allocator>& x,
-//                     const forward_list<T, Allocator>& y);
+//                     const forward_list<T, Allocator>& y); // constexpr since C++26
 //
 // template <class T, class Allocator>
 //     bool operator!=(const forward_list<T, Allocator>& x,
-//                     const forward_list<T, Allocator>& y);
+//                     const forward_list<T, Allocator>& y); // constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -25,7 +25,7 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(int N, int M) {
+TEST_CONSTEXPR_CXX26 void test(int N, int M) {
   C c1;
   for (int i = 0; i < N; ++i)
     c1.push_front(i);
@@ -44,7 +44,7 @@ void test(int N, int M) {
   }
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   for (int i = 0; i < 10; ++i)
     for (int j = 0; j < 10; ++j)
       test<std::forward_list<int> >(i, j);
@@ -54,5 +54,14 @@ int main(int, char**) {
       test<std::forward_list<int, min_allocator<int>> >(i, j);
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp
index e50f9e6e9e47..f4f7c6d1f7e5 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/member_swap.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// void swap(forward_list& x);
+// void swap(forward_list& x); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef test_allocator<T> A;
@@ -257,5 +257,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp
index cae6950436de..ce2547978154 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/non_member_swap.pass.cpp
@@ -9,7 +9,7 @@
 // <forward_list>
 
 // template <class T, class Allocator>
-//     void swap(forward_list<T, Allocator>& x, forward_list<T, Allocator>& y);
+//     void swap(forward_list<T, Allocator>& x, forward_list<T, Allocator>& y); // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef test_allocator<T> A;
@@ -258,5 +258,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp
index d16acadaeb89..7bf80ca026e8 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/relational.pass.cpp
@@ -10,19 +10,19 @@
 
 // template <class T, class Allocator>
 //     bool operator< (const forward_list<T, Allocator>& x,
-//                     const forward_list<T, Allocator>& y);
+//                     const forward_list<T, Allocator>& y); // constexpr since C++26
 //
 // template <class T, class Allocator>
 //     bool operator> (const forward_list<T, Allocator>& x,
-//                     const forward_list<T, Allocator>& y);
+//                     const forward_list<T, Allocator>& y); // constexpr since C++26
 //
 // template <class T, class Allocator>
 //     bool operator>=(const forward_list<T, Allocator>& x,
-//                     const forward_list<T, Allocator>& y);
+//                     const forward_list<T, Allocator>& y); // constexpr since C++26
 //
 // template <class T, class Allocator>
 //     bool operator<=(const forward_list<T, Allocator>& x,
-//                     const forward_list<T, Allocator>& y);
+//                     const forward_list<T, Allocator>& y); // constexpr since C++26
 
 #include <forward_list>
 #include <iterator>
@@ -33,7 +33,7 @@
 #include "min_allocator.h"
 
 template <class C>
-void test(int N, int M) {
+TEST_CONSTEXPR_CXX26 void test(int N, int M) {
   C c1;
   for (int i = 0; i < N; ++i)
     c1.push_front(i);
@@ -50,7 +50,7 @@ void test(int N, int M) {
     assert(c1 > c2);
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   for (int i = 0; i < 10; ++i)
     for (int j = 0; j < 10; ++j)
       test<std::forward_list<int> >(i, j);
@@ -60,5 +60,14 @@ int main(int, char**) {
       test<std::forward_list<int, min_allocator<int>> >(i, j);
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.compile.pass.cpp
similarity index 92%
rename from external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.pass.cpp
rename to external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.compile.pass.cpp
index b83ec3c3c121..02b7b471a1ae 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/forwardlist.spec/swap_noexcept.compile.pass.cpp
@@ -12,10 +12,10 @@
 
 // void swap(forward_list& c)
 //     noexcept(!allocator_type::propagate_on_container_swap::value ||
-//              __is_nothrow_swappable<allocator_type>::value);
+//              __is_nothrow_swappable<allocator_type>::value);          // constexpr since C++26
 //
 //  In C++17, the standard says that swap shall have:
-//     noexcept(is_always_equal<allocator_type>::value);
+//     noexcept(is_always_equal<allocator_type>::value);                 // constexpr since C++26
 
 // This tests a conforming extension
 
@@ -52,7 +52,7 @@ struct some_alloc2 {
   typedef std::true_type is_always_equal;
 };
 
-int main(int, char**) {
+void f() {
   {
     typedef std::forward_list<MoveOnly> C;
     static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
@@ -83,6 +83,4 @@ int main(int, char**) {
     static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
   }
 #endif
-
-  return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp
index f37f5c2f513b..624eeb17799c 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/get_allocator.pass.cpp
@@ -10,7 +10,7 @@
 
 // class forward_list
 
-// allocator_type get_allocator() const
+// allocator_type get_allocator() const // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::allocator<int> alloc;
     const std::forward_list<int> fl(alloc);
@@ -30,5 +30,14 @@ int main(int, char**) {
     assert(fl.get_allocator() == alloc);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp
index b7be03f1062d..16c6f0b90f96 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/incomplete.pass.cpp
@@ -8,9 +8,9 @@
 
 // <forward_list>
 
-// forward_list()
-// forward_list::iterator()
-// forward_list::const_iterator()
+// forward_list()                 // constexpr since C++26
+// forward_list::iterator()       // constexpr since C++26
+// forward_list::const_iterator() // constexpr since C++26
 
 #include <forward_list>
 #include <cassert>
@@ -33,7 +33,7 @@ struct B {
 };
 #endif
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     A a;
     assert(a.d.empty());
@@ -49,5 +49,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp
index 5ba0d61f104e..aab53351f00e 100644
--- a/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/sequences/forwardlist/max_size.pass.cpp
@@ -8,7 +8,7 @@
 
 // <forward_list>
 
-// size_type max_size() const;
+// size_type max_size() const; // constexpr since C++26
 
 #include <cassert>
 #include <forward_list>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef limited_allocator<int, 10> A;
     typedef std::forward_list<int, A> C;
@@ -42,5 +42,14 @@ int main(int, char**) {
     assert(c.max_size() <= alloc_max_size(c.get_allocator()));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h b/external/llvm-project/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
index 7cd42139758e..588a5e9774a5 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
@@ -25,7 +25,6 @@
 #include <cstddef>
 #include <limits>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 #include <utility>
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp
index 1d713044e60f..65bb5475353d 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/comparison.pass.cpp
@@ -20,7 +20,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp
index 7baaa7ec9898..4b6b1a63b80f 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/conversion.pass.cpp
@@ -32,7 +32,6 @@
 #include <cstddef>
 #include <limits>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 template <class To, class From>
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
index 82f111153ce6..b72c3e050b1f 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
@@ -19,7 +19,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "../ConvertibleToIntegral.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dextents.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dextents.pass.cpp
index 2adfa49d3bc4..a9fc8f3bed07 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dextents.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dextents.pass.cpp
@@ -18,7 +18,6 @@
 
 #include <mdspan>
 #include <cstddef>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dims.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dims.pass.cpp
index 0476c11efdb6..e74bc0e66fca 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dims.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/dims.pass.cpp
@@ -18,7 +18,6 @@
 
 #include <mdspan>
 #include <cstddef>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/index_type.verify.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/index_type.verify.cpp
index cdc046425141..ba6941a1ab4c 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/index_type.verify.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/index_type.verify.cpp
@@ -19,7 +19,6 @@
 #include <cstddef>
 #include <climits>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 void invalid_index_types() {
   // expected-error@*:* {{static assertion failed: extents::index_type must be a signed or unsigned integer type}}
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp
index 1d5f61d02e4f..12d5e5a85f64 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/obs_static.pass.cpp
@@ -29,7 +29,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <utility>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp
index 10bc76947395..5b28a58e06c4 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/extents/types.pass.cpp
@@ -26,7 +26,6 @@
 #include <concepts>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp
index 151da5ba6174..6db8fa48f5de 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/comparison.pass.cpp
@@ -18,7 +18,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp
index 12c59a4caf10..7fb657e9debb 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.default.pass.cpp
@@ -18,7 +18,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp
index 299012dc5af8..c13775c8ba75 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.extents.pass.cpp
@@ -21,7 +21,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp
index 1e8f8fb54d5e..f7ea39acf017 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_right.pass.cpp
@@ -24,7 +24,6 @@
 #include <cstddef>
 #include <limits>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
index 1668c26a697d..cbd759eaf72c 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
@@ -27,7 +27,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp
index 737e5f3d2572..28749327033a 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/ctor.mapping.pass.cpp
@@ -22,7 +22,6 @@
 #include <cstddef>
 #include <limits>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
index 75fd37055ad6..55d9539d8c63 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
@@ -27,7 +27,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp
index 32442ecd5a0e..e252e0ec73a7 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/properties.pass.cpp
@@ -29,7 +29,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <utility>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp
index 9ad61b0799c1..174c9633c267 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp
@@ -18,7 +18,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp
index 6410fecdab59..7b42e5045854 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/static_requirements.pass.cpp
@@ -79,7 +79,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 #include <utility>
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/stride.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/stride.pass.cpp
index 064c279bcc49..0b1f9f57ecab 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/stride.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_left/stride.pass.cpp
@@ -23,7 +23,6 @@
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp
index a65d7d39db8e..bcf26e0f5386 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/comparison.pass.cpp
@@ -18,7 +18,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp
index d644b0ff18d8..1ef377191404 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.default.pass.cpp
@@ -18,7 +18,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp
index cd0cff838fac..5579e5a0d303 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.extents.pass.cpp
@@ -21,7 +21,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp
index 994d98a80321..8f49b3773261 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_left.pass.cpp
@@ -24,7 +24,6 @@
 #include <cstddef>
 #include <limits>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
index 89321f860dc3..5886eba9d15e 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
@@ -27,7 +27,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp
index ee00c688301e..7a08f694f310 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/ctor.mapping.pass.cpp
@@ -22,7 +22,6 @@
 #include <cstddef>
 #include <limits>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
index e9362ed50df2..4623af7fb5b7 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
@@ -27,7 +27,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp
index 857ec3cb0583..29204775c5f5 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/properties.pass.cpp
@@ -29,7 +29,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <utility>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp
index 2ffd1f41f963..d19dccfe23aa 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp
@@ -18,7 +18,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp
index b7e01d14532d..b9add4da4282 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/static_requirements.pass.cpp
@@ -79,7 +79,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 #include <utility>
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/stride.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/stride.pass.cpp
index c04f07847c0b..7784c572e9cf 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/stride.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_right/stride.pass.cpp
@@ -23,7 +23,6 @@
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp
index 7c9b4a4ded34..37e20a3061f5 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp
@@ -25,7 +25,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
index 055986d80c72..27843805d91b 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
@@ -24,7 +24,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
index bbda6d4a7363..380c5eb90c03 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
@@ -32,7 +32,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp
index 6ba67ea2d012..66946d2b77b6 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp
@@ -38,7 +38,6 @@
 #include <mdspan>
 #include <cassert>
 #include <limits>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp
index 5669991b8a13..b483885049a7 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp
@@ -27,7 +27,6 @@
 #include <array>
 #include <cassert>
 #include <cstdint>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
index 9f51cc01cf9d..900adb42096a 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
@@ -23,7 +23,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 template <class E>
 constexpr void
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
index eac1029882e3..21ce56fcc65f 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
@@ -43,7 +43,6 @@
 #include <concepts>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
index 629849646bb2..cf1685f6676f 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
@@ -24,7 +24,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <mdspan>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
index 8131ecde5d76..1c31421424b1 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
@@ -79,7 +79,6 @@
 #include <cassert>
 #include <cstddef>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 #include <utility>
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp
index 2f16b1f6ec9a..8c462af54e00 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp
@@ -23,7 +23,6 @@
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
-#include <span> // dynamic_extent
 
 #include "test_macros.h"
 
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
index e0577aeb2826..0dc789cc09be 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
@@ -14,7 +14,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
index 1adab199b0ca..63a673f87b41 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
@@ -41,7 +41,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
index b35a4fb832a1..d00d275b222d 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
@@ -16,7 +16,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
index f77dbc46285a..ac7918b331e7 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
@@ -24,7 +24,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
index f9540fd99c99..f4ceb862d66b 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
@@ -32,7 +32,6 @@
 #include <cassert>
 #include <concepts>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp
index 4d9f91f63d54..4b94dcfed169 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp
@@ -26,7 +26,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp
index 381b2d3a8bc2..ebed92da7413 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp
@@ -33,7 +33,6 @@
 #include <cassert>
 #include <concepts>
 #include <mdspan>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp
index f5e9c1b7e2a4..2260e679aa28 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp
@@ -24,7 +24,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp
index 3239c1d65deb..8ca1bf5eafb5 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp
@@ -21,7 +21,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp
index 46ba2b2096c4..71b119567cd3 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp
@@ -16,7 +16,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp
index 5ce2d06712bf..70ca0d580ac1 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp
@@ -16,7 +16,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp
index 6368acd0b0f4..504af3378372 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp
@@ -54,7 +54,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp
index 47f2abecade8..3d5de0c49b81 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp
@@ -19,7 +19,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
index 69bce593bd36..b576419828b7 100644
--- a/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
@@ -30,7 +30,6 @@
 #include <mdspan>
 #include <cassert>
 #include <concepts>
-#include <span> // dynamic_extent
 #include <type_traits>
 
 #include "test_macros.h"
diff --git a/external/llvm-project/libcxx/test/std/input.output/iostream.forward/iosfwd.pass.cpp b/external/llvm-project/libcxx/test/std/input.output/iostream.forward/iosfwd.pass.cpp
index 1a28a5985a01..85fc5de128ce 100644
--- a/external/llvm-project/libcxx/test/std/input.output/iostream.forward/iosfwd.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/input.output/iostream.forward/iosfwd.pass.cpp
@@ -33,91 +33,76 @@ int main(int, char**)
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_ios<wchar_t>*       >();
 #endif
-    test<std::basic_ios<unsigned short>*>();
 
     test<std::basic_streambuf<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_streambuf<wchar_t>*       >();
 #endif
-    test<std::basic_streambuf<unsigned short>*>();
 
     test<std::basic_istream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_istream<wchar_t>*       >();
 #endif
-    test<std::basic_istream<unsigned short>*>();
 
     test<std::basic_ostream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_ostream<wchar_t>*       >();
 #endif
-    test<std::basic_ostream<unsigned short>*>();
 
     test<std::basic_iostream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_iostream<wchar_t>*       >();
 #endif
-    test<std::basic_iostream<unsigned short>*>();
 
     test<std::basic_stringbuf<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_stringbuf<wchar_t>*       >();
 #endif
-    test<std::basic_stringbuf<unsigned short>*>();
 
     test<std::basic_istringstream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_istringstream<wchar_t>*       >();
 #endif
-    test<std::basic_istringstream<unsigned short>*>();
 
     test<std::basic_ostringstream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_ostringstream<wchar_t>*       >();
 #endif
-    test<std::basic_ostringstream<unsigned short>*>();
 
     test<std::basic_stringstream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_stringstream<wchar_t>*       >();
 #endif
-    test<std::basic_stringstream<unsigned short>*>();
 
     test<std::basic_filebuf<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_filebuf<wchar_t>*       >();
 #endif
-    test<std::basic_filebuf<unsigned short>*>();
 
     test<std::basic_ifstream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_ifstream<wchar_t>*       >();
 #endif
-    test<std::basic_ifstream<unsigned short>*>();
 
     test<std::basic_ofstream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_ofstream<wchar_t>*       >();
 #endif
-    test<std::basic_ofstream<unsigned short>*>();
 
     test<std::basic_fstream<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::basic_fstream<wchar_t>*       >();
 #endif
-    test<std::basic_fstream<unsigned short>*>();
 
     test<std::istreambuf_iterator<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::istreambuf_iterator<wchar_t>*       >();
 #endif
-    test<std::istreambuf_iterator<unsigned short>*>();
 
     test<std::ostreambuf_iterator<char>*          >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     test<std::ostreambuf_iterator<wchar_t>*       >();
 #endif
-    test<std::ostreambuf_iterator<unsigned short>*>();
 
     test<std::ios* >();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
diff --git a/external/llvm-project/libcxx/test/std/iterators/iterator.requirements/iterator.concepts/iterator.concept.readable/indirectly_readable.compile.pass.cpp b/external/llvm-project/libcxx/test/std/iterators/iterator.requirements/iterator.concepts/iterator.concept.readable/indirectly_readable.compile.pass.cpp
index 5fc6ffd37caf..086e79b799c3 100644
--- a/external/llvm-project/libcxx/test/std/iterators/iterator.requirements/iterator.concepts/iterator.concept.readable/indirectly_readable.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/iterators/iterator.requirements/iterator.concepts/iterator.concept.readable/indirectly_readable.compile.pass.cpp
@@ -78,16 +78,10 @@ struct missing_iter_value_t {
 };
 static_assert(!check_indirectly_readable<missing_iter_value_t>());
 
-struct unrelated_lvalue_ref_and_rvalue_ref {};
-
-struct iter_ref1 {};
-template <>
-struct std::common_reference<iter_ref1&, iter_ref1&&> {};
-
-template <>
-struct std::common_reference<iter_ref1&&, iter_ref1&> {};
-
-static_assert(!std::common_reference_with<iter_ref1&, iter_ref1&&>);
+struct iter_ref1 {
+  iter_ref1(const iter_ref1&) = delete;
+  iter_ref1(iter_ref1&&)      = delete;
+};
 
 struct bad_iter_reference_t {
   using value_type = int;
@@ -128,24 +122,9 @@ struct different_reference_types_with_common_reference {
 static_assert(check_indirectly_readable<different_reference_types_with_common_reference>());
 
 struct iter_ref4 {
-  operator iter_rvalue_ref() const;
+  operator iter_rvalue_ref();
 };
 
-template <template <class> class XQual, template <class> class YQual>
-struct std::basic_common_reference<iter_ref4, iter_rvalue_ref, XQual, YQual> {
-  using type = iter_rvalue_ref;
-};
-template <template <class> class XQual, template <class> class YQual>
-struct std::basic_common_reference<iter_rvalue_ref, iter_ref4, XQual, YQual> {
-  using type = iter_rvalue_ref;
-};
-
-// FIXME: This is UB according to [meta.rqmts], and there is no exception for common_reference.
-template <>
-struct std::common_reference<iter_ref4 const&, iter_rvalue_ref&&> {};
-template <>
-struct std::common_reference<iter_rvalue_ref&&, iter_ref4 const&> {};
-
 static_assert(std::common_reference_with<iter_ref4&&, iter_rvalue_ref&&>);
 static_assert(!std::common_reference_with<iter_ref4 const&, iter_rvalue_ref&&>);
 
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.except.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.except.pass.cpp
index 6a2b098c1b57..9ee32b841783 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.except.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.except.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-exceptions
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 #include <new>
 #include <cassert>
 #include <limits>
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.pass.cpp
index 437d06430773..4fdcc3b535a8 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.pass.cpp
@@ -11,6 +11,9 @@
 // asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 #include <new>
 #include <cstddef>
 #include <cassert>
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.except.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.except.pass.cpp
index 4e34ebcb46c7..4dfaf7a30d7a 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.except.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.except.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-exceptions
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 // Libc++ when built for z/OS doesn't contain the aligned allocation functions,
 // nor does the dynamic library shipped with z/OS.
 // XFAIL: target={{.+}}-zos{{.*}}
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp
index c9b59ecaff39..a1b8466340a2 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_align.pass.cpp
@@ -13,6 +13,9 @@
 // asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 // Libc++ when built for z/OS doesn't contain the aligned allocation functions,
 // nor does the dynamic library shipped with z/OS.
 // XFAIL: target={{.+}}-zos{{.*}}
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.except.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.except.pass.cpp
index 6a515555e6db..346e881d016b 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.except.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.except.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-exceptions
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 #include <new>
 #include <cassert>
 #include <limits>
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.pass.cpp
index 729ef3ec46b0..0013dd3d0cbc 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.pass.cpp
@@ -11,6 +11,9 @@
 // asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 #include <new>
 #include <cstddef>
 #include <cassert>
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.except.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.except.pass.cpp
index 7694314c87bf..fbeb880c83d8 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.except.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.except.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-exceptions
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 // Libc++ when built for z/OS doesn't contain the aligned allocation functions,
 // nor does the dynamic library shipped with z/OS.
 // XFAIL: target={{.+}}-zos{{.*}}
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp
index 5d321f08282b..59ecbe205513 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align.pass.cpp
@@ -13,6 +13,9 @@
 // asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
 
+// GCC warns about allocating numeric_limits<size_t>::max() being too large (which we test here)
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 // Libc++ when built for z/OS doesn't contain the aligned allocation functions,
 // nor does the dynamic library shipped with z/OS.
 // XFAIL: target={{.+}}-zos{{.*}}
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/forward_list.version.compile.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/forward_list.version.compile.pass.cpp
index 31b3e900aabc..05f903dccafe 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/forward_list.version.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/forward_list.version.compile.pass.cpp
@@ -24,6 +24,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -54,6 +58,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -87,6 +95,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++17"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -126,6 +138,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -171,6 +187,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should be defined in c++23"
 #  endif
@@ -219,6 +239,13 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should be defined in c++26"
+#  endif
+#  if __cpp_lib_constexpr_forward_list != 202502L
+#    error "__cpp_lib_constexpr_forward_list should have the value 202502L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should be defined in c++26"
 #  endif
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/queue.version.compile.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/queue.version.compile.pass.cpp
index 6aff89ae3f95..0ebfc6de8410 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/queue.version.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/queue.version.compile.pass.cpp
@@ -24,6 +24,10 @@
 #    error "__cpp_lib_adaptor_iterator_pair_constructor should not be defined before c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -34,6 +38,10 @@
 #    error "__cpp_lib_adaptor_iterator_pair_constructor should not be defined before c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -44,6 +52,10 @@
 #    error "__cpp_lib_adaptor_iterator_pair_constructor should not be defined before c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -54,6 +66,10 @@
 #    error "__cpp_lib_adaptor_iterator_pair_constructor should not be defined before c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -67,6 +83,10 @@
 #    error "__cpp_lib_adaptor_iterator_pair_constructor should have the value 202106L in c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should be defined in c++23"
 #  endif
@@ -83,6 +103,13 @@
 #    error "__cpp_lib_adaptor_iterator_pair_constructor should have the value 202106L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should be defined in c++26"
+#  endif
+#  if __cpp_lib_constexpr_queue != 202502L
+#    error "__cpp_lib_constexpr_queue should have the value 202502L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should be defined in c++26"
 #  endif
diff --git a/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index aa33a2788f1e..a13edacd1e46 100644
--- a/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -196,6 +196,10 @@
 #    error "__cpp_lib_constexpr_dynamic_alloc should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -216,6 +220,10 @@
 #    error "__cpp_lib_constexpr_numeric should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_string
 #    error "__cpp_lib_constexpr_string should not be defined before c++20"
 #  endif
@@ -1080,6 +1088,10 @@
 #    error "__cpp_lib_constexpr_dynamic_alloc should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -1100,6 +1112,10 @@
 #    error "__cpp_lib_constexpr_numeric should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_string
 #    error "__cpp_lib_constexpr_string should not be defined before c++20"
 #  endif
@@ -2066,6 +2082,10 @@
 #    error "__cpp_lib_constexpr_dynamic_alloc should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -2086,6 +2106,10 @@
 #    error "__cpp_lib_constexpr_numeric should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_string
 #    error "__cpp_lib_constexpr_string should not be defined before c++20"
 #  endif
@@ -3292,6 +3316,10 @@
 #    error "__cpp_lib_constexpr_dynamic_alloc should have the value 201907L in c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++20"
 #  endif
@@ -3324,6 +3352,10 @@
 #    error "__cpp_lib_constexpr_numeric should have the value 201911L in c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_string
 #    error "__cpp_lib_constexpr_string should be defined in c++20"
 #  endif
@@ -4740,6 +4772,10 @@
 #    error "__cpp_lib_constexpr_dynamic_alloc should have the value 201907L in c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++23"
 #  endif
@@ -4772,6 +4808,10 @@
 #    error "__cpp_lib_constexpr_numeric should have the value 201911L in c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_string
 #    error "__cpp_lib_constexpr_string should be defined in c++23"
 #  endif
@@ -6407,6 +6447,13 @@
 #    error "__cpp_lib_constexpr_dynamic_alloc should have the value 201907L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_constexpr_forward_list
+#    error "__cpp_lib_constexpr_forward_list should be defined in c++26"
+#  endif
+#  if __cpp_lib_constexpr_forward_list != 202502L
+#    error "__cpp_lib_constexpr_forward_list should have the value 202502L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++26"
 #  endif
@@ -6448,6 +6495,13 @@
 #    error "__cpp_lib_constexpr_numeric should have the value 201911L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_constexpr_queue
+#    error "__cpp_lib_constexpr_queue should be defined in c++26"
+#  endif
+#  if __cpp_lib_constexpr_queue != 202502L
+#    error "__cpp_lib_constexpr_queue should have the value 202502L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_string
 #    error "__cpp_lib_constexpr_string should be defined in c++26"
 #  endif
diff --git a/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/ctor_func.pass.cpp b/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/ctor_func.pass.cpp
index c3a88af92d36..c05a9434175a 100644
--- a/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/ctor_func.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/ctor_func.pass.cpp
@@ -15,6 +15,9 @@
 //     discrete_distribution(size_t nw, double xmin, double xmax,
 //                           UnaryOperation fw);
 
+// There is a bogus diagnostic about a too large allocation
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 #include <random>
 
 #include <cassert>
diff --git a/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/param_ctor_func.pass.cpp b/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/param_ctor_func.pass.cpp
index 7ef936b7fc35..206bf5a0eb8a 100644
--- a/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/param_ctor_func.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/param_ctor_func.pass.cpp
@@ -15,6 +15,9 @@
 //     param_type(size_t nw, double xmin, double xmax,
 //                           UnaryOperation fw);
 
+// There is a bogus diagnostic about a too large allocation
+// ADDITIONAL_COMPILE_FLAGS(gcc): -Wno-alloc-size-larger-than
+
 #include <random>
 
 #include <cassert>
diff --git a/external/llvm-project/libcxx/test/std/ranges/range.adaptors/range.lazy.split/general.pass.cpp b/external/llvm-project/libcxx/test/std/ranges/range.adaptors/range.lazy.split/general.pass.cpp
index f4e87bb47399..521c0b1610bc 100644
--- a/external/llvm-project/libcxx/test/std/ranges/range.adaptors/range.lazy.split/general.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/ranges/range.adaptors/range.lazy.split/general.pass.cpp
@@ -312,7 +312,10 @@ constexpr bool main_test() {
   // Leading separator.
   {
     std::array expected = {""sv, "abc"sv, "def"sv};
+// FIXME: Why does GCC complain here?
+#ifndef TEST_COMPILER_GCC
     test_one(" abc def"sv, short_sep, expected);
+#endif
     test_one("12abc12def"sv, long_sep, expected);
   }
 
@@ -326,7 +329,10 @@ constexpr bool main_test() {
   // Input consisting of a single separator.
   {
     std::array expected = {""sv, ""sv};
+// FIXME: Why does GCC complain here?
+#ifndef TEST_COMPILER_GCC
     test_one(" "sv, short_sep, expected);
+#endif
     test_one("12"sv, long_sep, expected);
   }
 
@@ -354,7 +360,10 @@ constexpr bool main_test() {
   // Separators after every character.
   {
     std::array expected = {""sv, "a"sv, "b"sv, "c"sv, ""sv};
+// FIXME: Why does GCC complain here?
+#ifndef TEST_COMPILER_GCC
     test_one(" a b c "sv, short_sep, expected);
+#endif
     test_one("12a12b12c12"sv, long_sep, expected);
   }
 
@@ -383,7 +392,10 @@ constexpr bool main_test() {
   // Terminating null as a separator.
   {
     std::array expected = {"abc"sv, "def"sv};
+// FIXME: Why does GCC complain here?
+#ifndef TEST_COMPILER_GCC
     test_one("abc\0def"sv, '\0', expected);
+#endif
     test_one("abc\0\0def"sv, "\0\0"sv, expected);
   }
 
diff --git a/external/llvm-project/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp b/external/llvm-project/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
index 9ca7825a4ba9..a3b16c8da16c 100644
--- a/external/llvm-project/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
@@ -19,7 +19,7 @@
 #include "test_macros.h"
 
 template <class S>
-TEST_CONSTEXPR_CXX20 void test(S s) {
+TEST_CONSTEXPR_CXX20 void test(S& s) {
   typename S::size_type old_cap = s.capacity();
   S s0                          = s;
   s.shrink_to_fit();
diff --git a/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_++.pass.cpp b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_++.pass.cpp
new file mode 100644
index 000000000000..e035d7ef4fa0
--- /dev/null
+++ b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_++.pass.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: std-at-least-c++20
+
+// <chrono>
+
+// time_point
+
+// constexpr time_point& operator++();
+
+#include <cassert>
+#include <chrono>
+
+#include "test_macros.h"
+
+constexpr bool test() {
+  using Clock    = std::chrono::system_clock;
+  using Duration = std::chrono::milliseconds;
+  std::chrono::time_point<Clock, Duration> t{Duration{5}};
+  std::chrono::time_point<Clock, Duration>& tref{++t};
+  assert(&tref == &t);
+  assert(tref.time_since_epoch() == Duration{6});
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_++int.pass.cpp b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_++int.pass.cpp
new file mode 100644
index 000000000000..5304d37d5c36
--- /dev/null
+++ b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_++int.pass.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: std-at-least-c++20
+
+// <chrono>
+
+// time_point
+
+// constexpr time_point operator++(int);
+
+#include <cassert>
+#include <chrono>
+
+#include "test_macros.h"
+
+constexpr bool test() {
+  using Clock    = std::chrono::system_clock;
+  using Duration = std::chrono::milliseconds;
+  std::chrono::time_point<Clock, Duration> t1{Duration{3}};
+  std::chrono::time_point<Clock, Duration> t2{t1++};
+  assert(t1.time_since_epoch() == Duration{4});
+  assert(t2.time_since_epoch() == Duration{3});
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_--.pass.cpp b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_--.pass.cpp
new file mode 100644
index 000000000000..915156fcc6b8
--- /dev/null
+++ b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_--.pass.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: std-at-least-c++20
+
+// <chrono>
+
+// time_point
+
+// constexpr time_point& operator--();
+
+#include <cassert>
+#include <chrono>
+
+#include "test_macros.h"
+
+constexpr bool test() {
+  using Clock    = std::chrono::system_clock;
+  using Duration = std::chrono::milliseconds;
+  std::chrono::time_point<Clock, Duration> t{Duration{5}};
+  std::chrono::time_point<Clock, Duration>& tref{--t};
+  assert(&tref == &t);
+  assert(tref.time_since_epoch() == Duration{4});
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_--int.pass.cpp b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_--int.pass.cpp
new file mode 100644
index 000000000000..cc5f462106bb
--- /dev/null
+++ b/external/llvm-project/libcxx/test/std/time/time.point/time.point.arithmetic/op_--int.pass.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// REQUIRES: std-at-least-c++20
+
+// <chrono>
+
+// time_point
+
+// constexpr time_point operator--(int);
+
+#include <cassert>
+#include <chrono>
+
+#include "test_macros.h"
+
+constexpr bool test() {
+  using Clock    = std::chrono::system_clock;
+  using Duration = std::chrono::milliseconds;
+  std::chrono::time_point<Clock, Duration> t1{Duration{3}};
+  std::chrono::time_point<Clock, Duration> t2{t1--};
+  assert(t1.time_since_epoch() == Duration{2});
+  assert(t2.time_since_epoch() == Duration{3});
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp
index cbd54d623c0f..97c1e4a40f35 100644
--- a/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform.pass.cpp
@@ -9,8 +9,8 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // GCC has a issue for `Guaranteed copy elision for potentially-overlapping non-static data members`,
-// please refer to: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108333
-// XFAIL: gcc-14
+// please refer to: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98995
+// XFAIL: gcc-14, gcc-15
 
 // <expected>
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp
index a19e17b01f6a..9570b2faac69 100644
--- a/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/expected/expected.expected/monadic/transform_error.pass.cpp
@@ -9,8 +9,8 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // GCC has a issue for `Guaranteed copy elision for potentially-overlapping non-static data members`,
-// please refer to: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108333.
-// XFAIL: gcc-14
+// please refer to: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98995.
+// XFAIL: gcc-14, gcc-15
 
 // <expected>
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/expected/expected.void/monadic/transform_error.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/expected/expected.void/monadic/transform_error.pass.cpp
index f6d3011d1ea9..2ec15b51d11e 100644
--- a/external/llvm-project/libcxx/test/std/utilities/expected/expected.void/monadic/transform_error.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/expected/expected.void/monadic/transform_error.pass.cpp
@@ -9,8 +9,8 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // GCC has a issue for `Guaranteed copy elision for potentially-overlapping non-static data members`,
-// please refer to: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108333
-// XFAIL: gcc-14
+// please refer to: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98995
+// XFAIL: gcc-14, gcc-15
 
 // <expected>
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.char_array.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.char_array.pass.cpp
index bc056db9e254..8c4f3000ec1e 100644
--- a/external/llvm-project/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.char_array.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.char_array.pass.cpp
@@ -8,7 +8,7 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // TODO FMT __builtin_memcpy isn't constexpr in GCC
-// UNSUPPORTED: gcc-14
+// UNSUPPORTED: gcc-14, gcc-15
 
 // <format>
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp
index 85106c18ec35..4a2ae963e3bd 100644
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
 
 // <functional>
 
@@ -23,16 +23,13 @@
 #include "test_comparisons.h"
 #include "test_macros.h"
 
-#include "helper_concepts.h"
-#include "helper_types.h"
-
 // Test SFINAE.
 
-static_assert(HasSpaceshipOperatorWithInt<std::reference_wrapper<StrongOrder>>);
-static_assert(HasSpaceshipOperatorWithInt<std::reference_wrapper<WeakOrder>>);
-static_assert(HasSpaceshipOperatorWithInt<std::reference_wrapper<PartialOrder>>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<StrongOrder>, int>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<WeakOrder>, int>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<PartialOrder>, int>);
 
-static_assert(!HasSpaceshipOperatorWithInt<std::reference_wrapper<NonComparable>>);
+static_assert(!HasOperatorSpaceship<std::reference_wrapper<NonComparable>, int>);
 
 // Test comparisons.
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp
index 794fac00de8a..3d72459bc5a1 100644
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
 
 // <functional>
 
@@ -22,17 +22,13 @@
 
 #include "test_comparisons.h"
 #include "test_macros.h"
-
-#include "helper_concepts.h"
-#include "helper_types.h"
-
 // Test SFINAE.
 
-static_assert(std::three_way_comparable<std::reference_wrapper<StrongOrder>>);
-static_assert(std::three_way_comparable<std::reference_wrapper<WeakOrder>>);
-static_assert(std::three_way_comparable<std::reference_wrapper<PartialOrder>>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<StrongOrder>>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<WeakOrder>>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<PartialOrder>>);
 
-static_assert(!std::three_way_comparable<std::reference_wrapper<NonComparable>>);
+static_assert(!HasOperatorSpaceship<std::reference_wrapper<NonComparable>>);
 
 // Test comparisons.
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp
index 9b1302affa85..1ae22b4ac58e 100644
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
 
 // <functional>
 
@@ -23,18 +23,15 @@
 #include "test_comparisons.h"
 #include "test_macros.h"
 
-#include "helper_concepts.h"
-#include "helper_types.h"
-
 // Test SFINAE.
 
-static_assert(std::three_way_comparable_with<std::reference_wrapper<StrongOrder>, const StrongOrder>);
-static_assert(std::three_way_comparable_with<std::reference_wrapper<WeakOrder>, const WeakOrder>);
-static_assert(std::three_way_comparable_with<std::reference_wrapper<PartialOrder>, const PartialOrder>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<StrongOrder>, std::reference_wrapper<const StrongOrder>>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<WeakOrder>, std::reference_wrapper<const WeakOrder>>);
+static_assert(HasOperatorSpaceship<std::reference_wrapper<PartialOrder>, std::reference_wrapper<const PartialOrder>>);
 
-static_assert(!std::three_way_comparable_with<std::reference_wrapper<StrongOrder>, const NonComparable>);
-static_assert(!std::three_way_comparable_with<std::reference_wrapper<WeakOrder>, const NonComparable>);
-static_assert(!std::three_way_comparable_with<std::reference_wrapper<PartialOrder>, const NonComparable>);
+static_assert(!HasOperatorSpaceship<std::reference_wrapper<StrongOrder>, std::reference_wrapper<const NonComparable>>);
+static_assert(!HasOperatorSpaceship<std::reference_wrapper<WeakOrder>, std::reference_wrapper<const NonComparable>>);
+static_assert(!HasOperatorSpaceship<std::reference_wrapper<PartialOrder>, std::reference_wrapper<const NonComparable>>);
 
 // Test comparisons.
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp
index 465326818f17..316ff7c30331 100644
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
 
 // <functional>
 
@@ -23,14 +23,13 @@
 #include "test_comparisons.h"
 #include "test_macros.h"
 
-#include "helper_concepts.h"
-#include "helper_types.h"
-
 // Test SFINAE.
 
-static_assert(HasEqualityOperatorWithInt<std::reference_wrapper<EqualityComparable>>);
+static_assert(HasOperatorEqual<std::reference_wrapper<EqualityComparable>>);
+static_assert(HasOperatorEqual<std::reference_wrapper<EqualityComparable>, int>);
 
-static_assert(!HasEqualityOperatorWithInt<std::reference_wrapper<NonComparable>>);
+static_assert(!HasOperatorEqual<std::reference_wrapper<NonComparable>>);
+static_assert(!HasOperatorEqual<std::reference_wrapper<NonComparable>, int>);
 
 // Test equality.
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp
index a50b530bbc6e..70e79d399861 100644
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
 
 // <functional>
 
@@ -22,14 +22,11 @@
 #include "test_comparisons.h"
 #include "test_macros.h"
 
-#include "helper_concepts.h"
-#include "helper_types.h"
-
 // Test SFINAE.
 
-static_assert(std::equality_comparable<std::reference_wrapper<EqualityComparable>>);
+static_assert(HasOperatorEqual<std::reference_wrapper<EqualityComparable>>);
 
-static_assert(!std::equality_comparable<std::reference_wrapper<NonComparable>>);
+static_assert(!HasOperatorEqual<std::reference_wrapper<NonComparable>>);
 
 // Test equality.
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp
index 10f017742a87..c68ad5c4aa52 100644
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
 
 // <functional>
 
@@ -23,16 +23,13 @@
 #include "test_comparisons.h"
 #include "test_macros.h"
 
-#include "helper_concepts.h"
-#include "helper_types.h"
-
 // Test SFINAE.
 
-static_assert(std::equality_comparable_with<std::reference_wrapper<EqualityComparable>,
-                                            std::reference_wrapper<const EqualityComparable>>);
+static_assert(
+    HasOperatorEqual<std::reference_wrapper<EqualityComparable>, std::reference_wrapper<const EqualityComparable>>);
 
-static_assert(!std::equality_comparable_with<std::reference_wrapper<EqualityComparable>,
-                                             std::reference_wrapper<const NonComparable>>);
+static_assert(
+    !HasOperatorEqual<std::reference_wrapper<EqualityComparable>, std::reference_wrapper<const NonComparable>>);
 
 // Test equality.
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_concepts.h b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_concepts.h
deleted file mode 100644
index 2dbb304f8af6..000000000000
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_concepts.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_CONCEPTS_H
-#define TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_CONCEPTS_H
-
-#include <concepts>
-#include <utility>
-
-// Equality
-
-template <typename T>
-concept HasEqualityOperatorWithInt = requires(T t, int i) {
-  { t.get() == i } -> std::convertible_to<bool>;
-};
-
-// Spaceship
-
-template <class T>
-concept BooleanTestableImpl = std::convertible_to<T, bool>;
-
-template <class T>
-concept BooleanTestable = BooleanTestableImpl<T> && requires(T&& t) {
-  { !std::forward<T>(t) } -> BooleanTestableImpl;
-};
-
-template <typename T>
-concept HasSpaceshipOperatorWithInt = requires(T t, int i) {
-  { t < i } -> BooleanTestable;
-  { i < t } -> BooleanTestable;
-};
-
-#endif // TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_CONCEPTS_H
diff --git a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_types.h b/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_types.h
deleted file mode 100644
index cf5e568dbf93..000000000000
--- a/external/llvm-project/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_types.h
+++ /dev/null
@@ -1,30 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_TYPES_H
-#define TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_TYPES_H
-
-#include <concepts>
-
-struct EqualityComparable {
-  constexpr EqualityComparable(int value) : value_{value} {};
-
-  friend constexpr bool operator==(const EqualityComparable&, const EqualityComparable&) noexcept = default;
-
-  int value_;
-};
-
-static_assert(std::equality_comparable<EqualityComparable>);
-static_assert(EqualityComparable{94} == EqualityComparable{94});
-static_assert(EqualityComparable{94} != EqualityComparable{82});
-
-struct NonComparable {};
-
-static_assert(!std::three_way_comparable<NonComparable>);
-
-#endif // TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_TYPES_H
diff --git a/external/llvm-project/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp
index f443d2030961..47c95c64a085 100644
--- a/external/llvm-project/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/meta/meta.rel/is_virtual_base_of.pass.cpp
@@ -18,6 +18,8 @@
 #include <type_traits>
 #include <cassert>
 
+#include "test_macros.h"
+
 template <bool expected, class Base, class Derived>
 void test() {
   // Test the type of the variables
@@ -98,8 +100,13 @@ int main(int, char**) {
 
   // Test with virtual inheritance
   {
+#ifdef TEST_COMPILER_GCC // FIXME: Is this a GCC or Clang bug? Or is the standards wording ambiguous?
+    test<true, Base, Derived3Virtual>();
+    test<true, Derived, Derived3Virtual>();
+#else
     test<false, Base, Derived3Virtual>();
     test<false, Derived, Derived3Virtual>();
+#endif
     test<true, Derived2b, Derived3Virtual>();
     test<true, Derived2a, Derived3Virtual>();
     test<true, Base, DerivedPrivate>();
diff --git a/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/dependent_return_type.compile.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/dependent_return_type.pass.cpp
similarity index 94%
rename from external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/dependent_return_type.compile.pass.cpp
rename to external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/dependent_return_type.pass.cpp
index 935a6e3db001..37d66831c7ce 100644
--- a/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/dependent_return_type.compile.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/dependent_return_type.pass.cpp
@@ -168,3 +168,7 @@ void instantiate() {
   void_t<int>();
 #endif
 }
+
+// This is not a .compile.pass.cpp because we want to ensure that GCC doesn't complain about incorrect builtins usage,
+// which only happens during CodeGen.
+int main(int, char**) { return 0; }
diff --git a/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
index 681ad13a07df..afd76e65060e 100644
--- a/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // These compilers don't support __builtin_is_implicit_lifetime yet.
-// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16, apple-clang-17
+// UNSUPPORTED: clang-18, clang-19, gcc-14, gcc-15, apple-clang-15, apple-clang-16, apple-clang-17
 
 // <type_traits>
 
diff --git a/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit.pass.cpp
index e7931e07e31d..23f131d2fc49 100644
--- a/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit.pass.cpp
@@ -12,6 +12,9 @@
 // template <class T, class... Args>
 //   constexpr optional<T> make_optional(Args&&... args);
 
+// GCC crashes on this file, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120577
+// XFAIL: gcc-15
+
 #include <optional>
 #include <string>
 #include <memory>
diff --git a/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit_initializer_list.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit_initializer_list.pass.cpp
index 80371d633371..5ddb229ad926 100644
--- a/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit_initializer_list.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/optional/optional.specalg/make_optional_explicit_initializer_list.pass.cpp
@@ -12,6 +12,9 @@
 // template <class T, class U, class... Args>
 //   constexpr optional<T> make_optional(initializer_list<U> il, Args&&... args);
 
+// GCC crashes on this file, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120577
+// XFAIL: gcc-15
+
 #include <cassert>
 #include <memory>
 #include <optional>
diff --git a/external/llvm-project/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer_deleter.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer_deleter.pass.cpp
index a91abc856fb1..a438bfb58ce4 100644
--- a/external/llvm-project/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer_deleter.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer_deleter.pass.cpp
@@ -32,6 +32,8 @@ bool my_free_called = false;
 
 void my_free(void*) { my_free_called = true; }
 
+TEST_CONSTEXPR_CXX23 void deleter_function(A*) {}
+
 #if TEST_STD_VER >= 11
 struct DeleterBase {
   TEST_CONSTEXPR_CXX23 void operator()(void*) const {}
@@ -325,6 +327,21 @@ TEST_CONSTEXPR_CXX23 void test_nullptr() {
 #endif
 }
 
+template <bool IsArray>
+TEST_CONSTEXPR_CXX23 void test_function_reference() {
+  typedef typename std::conditional<!IsArray, A, A[]>::type VT;
+  {
+    std::unique_ptr<VT, void (&)(A*)> u(nullptr, deleter_function);
+    assert(u.get() == nullptr);
+    assert(u.get_deleter() == deleter_function);
+  }
+  {
+    std::unique_ptr<VT, void (&)(A*)> u(nullptr, deleter_function);
+    assert(u.get() == nullptr);
+    assert(u.get_deleter() == deleter_function);
+  }
+}
+
 TEST_CONSTEXPR_CXX23 bool test() {
   {
     test_basic</*IsArray*/ false>();
@@ -332,6 +349,7 @@ TEST_CONSTEXPR_CXX23 bool test() {
     test_basic_single();
     test_sfinae<false>();
     test_noexcept<false>();
+    test_function_reference<false>();
   }
   {
     test_basic</*IsArray*/ true>();
@@ -339,6 +357,7 @@ TEST_CONSTEXPR_CXX23 bool test() {
     test_sfinae<true>();
     test_sfinae_runtime();
     test_noexcept<true>();
+    test_function_reference<true>();
   }
 
   return true;
diff --git a/external/llvm-project/libcxx/test/std/utilities/template.bitset/bitset.cons/char_ptr_ctor.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/template.bitset/bitset.cons/char_ptr_ctor.pass.cpp
index 86b144ed87b7..4f9cdaeb38c0 100644
--- a/external/llvm-project/libcxx/test/std/utilities/template.bitset/bitset.cons/char_ptr_ctor.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/template.bitset/bitset.cons/char_ptr_ctor.pass.cpp
@@ -72,6 +72,35 @@ TEST_CONSTEXPR_CXX23 void test_char_pointer_ctor()
     for (std::size_t i = 10; i < v.size(); ++i)
         assert(v[i] == false);
   }
+  // Verify that this constructor doesn't read over the given bound.
+  // See https://github.com/llvm/llvm-project/issues/143684
+  {
+    const char not_null_terminated[] = {'1', '0', '1', '0', '1', '0', '1', '0', '1', '0'};
+    std::bitset<N> v(not_null_terminated, 10);
+    std::size_t M = std::min<std::size_t>(v.size(), 10);
+    for (std::size_t i = 0; i < M; ++i)
+      assert(v[i] == (not_null_terminated[M - 1 - i] == '1'));
+    for (std::size_t i = 10; i < v.size(); ++i)
+      assert(!v[i]);
+  }
+  {
+    const char not_null_terminated[] = {'1', 'a', '1', 'a', '1', 'a', '1', 'a', '1', 'a'};
+    std::bitset<N> v(not_null_terminated, 10, 'a');
+    std::size_t M = std::min<std::size_t>(v.size(), 10);
+    for (std::size_t i = 0; i < M; ++i)
+      assert(v[i] == (not_null_terminated[M - 1 - i] == '1'));
+    for (std::size_t i = 10; i < v.size(); ++i)
+      assert(!v[i]);
+  }
+  {
+    const char not_null_terminated[] = {'b', 'a', 'b', 'a', 'b', 'a', 'b', 'a', 'b', 'a'};
+    std::bitset<N> v(not_null_terminated, 10, 'a', 'b');
+    std::size_t M = std::min<std::size_t>(v.size(), 10);
+    for (std::size_t i = 0; i < M; ++i)
+      assert(v[i] == (not_null_terminated[M - 1 - i] == 'b'));
+    for (std::size_t i = 10; i < v.size(); ++i)
+      assert(!v[i]);
+  }
 }
 
 TEST_CONSTEXPR_CXX23 bool test() {
diff --git a/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.pass.cpp
index 463816929353..d78de0eec8e5 100644
--- a/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.pass.cpp
@@ -18,12 +18,14 @@
 #include <cassert>
 #include "test_macros.h"
 
-#if TEST_HAS_BUILTIN_IDENTIFIER(__reference_binds_to_temporary)
-# define ASSERT_REFERENCE_BINDS_TEMPORARY(...) static_assert(__reference_binds_to_temporary(__VA_ARGS__), "")
-# define ASSERT_NOT_REFERENCE_BINDS_TEMPORARY(...) static_assert(!__reference_binds_to_temporary(__VA_ARGS__), "")
+#if TEST_HAS_BUILTIN(__reference_constructs_from_temporary)
+#  define ASSERT_REFERENCE_BINDS_TEMPORARY(...) static_assert(__reference_constructs_from_temporary(__VA_ARGS__), "")
+#  define ASSERT_NOT_REFERENCE_BINDS_TEMPORARY(...)                                                                    \
+    static_assert(!__reference_constructs_from_temporary(__VA_ARGS__), "")
 #else
-# define ASSERT_REFERENCE_BINDS_TEMPORARY(...) static_assert(true, "")
-# define ASSERT_NOT_REFERENCE_BINDS_TEMPORARY(...) static_assert(true, "")
+// TODO(LLVM 22): Remove this as all support compilers should have __reference_constructs_from_temporary implemented.
+#  define ASSERT_REFERENCE_BINDS_TEMPORARY(...) static_assert(__reference_binds_to_temporary(__VA_ARGS__), "")
+#  define ASSERT_NOT_REFERENCE_BINDS_TEMPORARY(...) static_assert(!__reference_binds_to_temporary(__VA_ARGS__), "")
 #endif
 
 template <class Tp>
diff --git a/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR31384.pass.cpp b/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR31384.pass.cpp
index e6812e9a3a30..ae5984c15530 100644
--- a/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR31384.pass.cpp
+++ b/external/llvm-project/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/PR31384.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03
 
 // FIXME: Why does this start to fail with GCC 14?
-// XFAIL: !(c++11 || c++14) && gcc-14
+// XFAIL: !(c++11 || c++14) && (gcc-14 || gcc-15)
 
 // See https://llvm.org/PR31384.
 
diff --git a/external/llvm-project/libcxx/test/support/MinSequenceContainer.h b/external/llvm-project/libcxx/test/support/MinSequenceContainer.h
index 6e61aff06344..ccc17b79288b 100644
--- a/external/llvm-project/libcxx/test/support/MinSequenceContainer.h
+++ b/external/llvm-project/libcxx/test/support/MinSequenceContainer.h
@@ -28,6 +28,13 @@ struct MinSequenceContainer {
   template <class It>
   explicit MinSequenceContainer(It first, It last) : data_(first, last) {}
   MinSequenceContainer(std::initializer_list<T> il) : data_(il) {}
+
+  template <class It>
+  void assign(It first, It last) {
+    data_.assign(first, last);
+  }
+  void assign(std::initializer_list<T> il) { data_.assign(il); }
+  void assign(size_type n, value_type t) { data_.assign(n, t); }
   iterator begin() { return iterator(data_.data()); }
   const_iterator begin() const { return const_iterator(data_.data()); }
   const_iterator cbegin() const { return const_iterator(data_.data()); }
@@ -47,6 +54,11 @@ struct MinSequenceContainer {
     return from_vector_iterator(data_.insert(to_vector_iterator(p), std::move(value)));
   }
 
+  template <class Range>
+  iterator insert_range(const_iterator p, Range&& rg) {
+    return from_vector_iterator(data_.insert_range(to_vector_iterator(p), std::forward<Range>(rg)));
+  }
+
   iterator erase(const_iterator first, const_iterator last) {
     return from_vector_iterator(data_.erase(to_vector_iterator(first), to_vector_iterator(last)));
   }
diff --git a/external/llvm-project/libcxx/test/support/any_helpers.h b/external/llvm-project/libcxx/test/support/any_helpers.h
index dae3ca3c90f5..0e5563142d58 100644
--- a/external/llvm-project/libcxx/test/support/any_helpers.h
+++ b/external/llvm-project/libcxx/test/support/any_helpers.h
@@ -66,32 +66,26 @@ template <class> constexpr bool has_value_member(long) { return false; }
 
 // Assert that an 'any' object stores the specified 'Type' and 'value'.
 template <class Type>
-std::enable_if_t<has_value_member<Type>(0)>
-_LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST
-assertContains(std::any const& a, int value) {
-    assert(a.has_value());
-    assert(containsType<Type>(a));
-    assert(std::any_cast<Type const &>(a).value == value);
+std::enable_if_t<has_value_member<Type>(0)> assertContains(std::any const& a, int value) {
+  assert(a.has_value());
+  assert(containsType<Type>(a));
+  assert(std::any_cast<Type const&>(a).value == value);
 }
 
 template <class Type, class Value>
-std::enable_if_t<!has_value_member<Type>(0)>
-_LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST
-assertContains(std::any const& a, Value value) {
-    assert(a.has_value());
-    assert(containsType<Type>(a));
-    assert(std::any_cast<Type const &>(a) == value);
+std::enable_if_t<!has_value_member<Type>(0)> assertContains(std::any const& a, Value value) {
+  assert(a.has_value());
+  assert(containsType<Type>(a));
+  assert(std::any_cast<Type const&>(a) == value);
 }
 
-
 // Modify the value of a "test type" stored within an any to the specified
 // 'value'.
 template <class Type>
-_LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST
 void modifyValue(std::any& a, int value) {
-    assert(a.has_value());
-    assert(containsType<Type>(a));
-    std::any_cast<Type&>(a).value = value;
+  assert(a.has_value());
+  assert(containsType<Type>(a));
+  std::any_cast<Type&>(a).value = value;
 }
 
 // A test type that will trigger the small object optimization within 'any'.
diff --git a/external/llvm-project/libcxx/test/support/counting_predicates.h b/external/llvm-project/libcxx/test/support/counting_predicates.h
index 6f34ce76302a..8fb2db1af70d 100644
--- a/external/llvm-project/libcxx/test/support/counting_predicates.h
+++ b/external/llvm-project/libcxx/test/support/counting_predicates.h
@@ -16,42 +16,44 @@
 template <typename Predicate, typename Arg>
 struct unary_counting_predicate {
 public:
-    typedef Arg argument_type;
-    typedef bool result_type;
+  typedef Arg argument_type;
+  typedef bool result_type;
 
-    unary_counting_predicate(Predicate p) : p_(p), count_(0) {}
-    unary_counting_predicate(const unary_counting_predicate&) = default;
-    unary_counting_predicate& operator=(const unary_counting_predicate&) = default;
-    ~unary_counting_predicate() {}
+  TEST_CONSTEXPR_CXX20 unary_counting_predicate(Predicate p) : p_(p), count_(0) {}
+  unary_counting_predicate(const unary_counting_predicate&)            = default;
+  unary_counting_predicate& operator=(const unary_counting_predicate&) = default;
+  TEST_CONSTEXPR_CXX20 ~unary_counting_predicate() {}
 
-    bool operator () (const Arg &a) const { ++count_; return p_(a); }
-    std::size_t count() const { return count_; }
-    void reset() { count_ = 0; }
+  TEST_CONSTEXPR_CXX14 bool operator()(const Arg& a) const {
+    ++count_;
+    return p_(a);
+  }
+  TEST_CONSTEXPR std::size_t count() const { return count_; }
+  TEST_CONSTEXPR_CXX14 void reset() { count_ = 0; }
 
 private:
-    Predicate p_;
-    mutable std::size_t count_;
+  Predicate p_;
+  mutable std::size_t count_;
 };
 
-
-template <typename Predicate, typename Arg1, typename Arg2=Arg1>
+template <typename Predicate, typename Arg1, typename Arg2 = Arg1>
 struct binary_counting_predicate {
 public:
-    typedef Arg1 first_argument_type;
-    typedef Arg2 second_argument_type;
-    typedef bool result_type;
-
-    TEST_CONSTEXPR binary_counting_predicate(Predicate p) : p_(p), count_(0) {}
-    TEST_CONSTEXPR_CXX14 bool operator()(const Arg1& a1, const Arg2& a2) const {
-      ++count_;
-      return p_(a1, a2);
-    }
-    TEST_CONSTEXPR std::size_t count() const { return count_; }
-    TEST_CONSTEXPR_CXX14 void reset() { count_ = 0; }
-
-  private:
-    Predicate p_;
-    mutable std::size_t count_;
+  typedef Arg1 first_argument_type;
+  typedef Arg2 second_argument_type;
+  typedef bool result_type;
+
+  TEST_CONSTEXPR binary_counting_predicate(Predicate p) : p_(p), count_(0) {}
+  TEST_CONSTEXPR_CXX14 bool operator()(const Arg1& a1, const Arg2& a2) const {
+    ++count_;
+    return p_(a1, a2);
+  }
+  TEST_CONSTEXPR std::size_t count() const { return count_; }
+  TEST_CONSTEXPR_CXX14 void reset() { count_ = 0; }
+
+private:
+  Predicate p_;
+  mutable std::size_t count_;
 };
 
 #if TEST_STD_VER > 14
@@ -66,13 +68,13 @@ class counting_predicate {
   constexpr counting_predicate(Predicate pred, int& count) : pred_(std::move(pred)), count_(&count) {}
 
   template <class... Args>
-  constexpr decltype(auto) operator()(Args&& ...args) {
+  constexpr decltype(auto) operator()(Args&&... args) {
     ++(*count_);
     return pred_(std::forward<Args>(args)...);
   }
 
   template <class... Args>
-  constexpr decltype(auto) operator()(Args&& ...args) const {
+  constexpr decltype(auto) operator()(Args&&... args) const {
     ++(*count_);
     return pred_(std::forward<Args>(args)...);
   }
diff --git a/external/llvm-project/libcxx/test/support/msvc_stdlib_force_include.h b/external/llvm-project/libcxx/test/support/msvc_stdlib_force_include.h
index 785670224c3b..2ea449c962cb 100644
--- a/external/llvm-project/libcxx/test/support/msvc_stdlib_force_include.h
+++ b/external/llvm-project/libcxx/test/support/msvc_stdlib_force_include.h
@@ -102,6 +102,4 @@ const AssertionDialogAvoider assertion_dialog_avoider{};
 #define TEST_SHORT_WCHAR
 #define TEST_ABI_MICROSOFT
 
-#define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST
-
 #endif // SUPPORT_MSVC_STDLIB_FORCE_INCLUDE_H
diff --git a/external/llvm-project/libcxx/test/support/test_comparisons.h b/external/llvm-project/libcxx/test/support/test_comparisons.h
index db6977a96a2f..d9729e0451b4 100644
--- a/external/llvm-project/libcxx/test/support/test_comparisons.h
+++ b/external/llvm-project/libcxx/test/support/test_comparisons.h
@@ -268,6 +268,29 @@ struct PartialOrder {
   }
 };
 
-#endif
+template <typename T1, typename T2 = T1>
+concept HasOperatorEqual = requires(T1 t1, T2 t2) { t1 == t2; };
+
+template <typename T1, typename T2 = T1>
+concept HasOperatorSpaceship = requires(T1 t1, T2 t2) { t1 <=> t2; };
+
+struct NonComparable {};
+static_assert(!std::equality_comparable<NonComparable>);
+static_assert(!HasOperatorEqual<NonComparable>);
+static_assert(!HasOperatorSpaceship<NonComparable>);
+
+class EqualityComparable {
+public:
+  constexpr EqualityComparable(int value) : value_{value} {};
+
+  friend constexpr bool operator==(const EqualityComparable&, const EqualityComparable&) noexcept = default;
+
+private:
+  int value_;
+};
+static_assert(std::equality_comparable<EqualityComparable>);
+static_assert(HasOperatorEqual<EqualityComparable>);
+
+#endif // TEST_STD_VER >= 20
 
 #endif // TEST_COMPARISONS_H
diff --git a/external/llvm-project/libcxx/test/support/test_container_comparisons.h b/external/llvm-project/libcxx/test/support/test_container_comparisons.h
index f7bf78e48a1f..53db5ba99ce4 100644
--- a/external/llvm-project/libcxx/test/support/test_container_comparisons.h
+++ b/external/llvm-project/libcxx/test/support/test_container_comparisons.h
@@ -88,7 +88,6 @@ constexpr bool test_sequence_container_spaceship() {
                                               std::weak_ordering>();
 
   // Thanks to SFINAE, the following is not a compiler error but returns `false`
-  struct NonComparable {};
   static_assert(!std::three_way_comparable<Container<NonComparable>>);
 
   return true;
@@ -163,7 +162,6 @@ constexpr void test_sequence_container_adaptor_spaceship_with_type() {
 template <template <typename...> typename ContainerAdaptor, template <typename...> typename Container>
 constexpr bool test_sequence_container_adaptor_spaceship() {
   // Thanks to SFINAE, the following is not a compiler error but returns `false`
-  struct NonComparable {};
   static_assert(!std::three_way_comparable<ContainerAdaptor<NonComparable>>);
 
   // The container should fulfill `std::three_way_comparable`
@@ -301,7 +299,6 @@ constexpr void test_ordered_map_container_spaceship_with_type(Compare comp) {
 template <template <typename...> typename Container>
 constexpr bool test_ordered_map_container_spaceship() {
   // Thanks to SFINAE, the following is not a compiler error but returns `false`
-  struct NonComparable {};
   static_assert(!std::three_way_comparable<Container<int, NonComparable>>);
 
   // The container should fulfill `std::three_way_comparable`
@@ -444,7 +441,6 @@ constexpr void test_ordered_set_spaceship_with_type(Compare comp) {
 template <template <typename...> typename Container>
 constexpr bool test_ordered_set_container_spaceship() {
   // Thanks to SFINAE, the following is not a compiler error but returns `false`
-  struct NonComparable {};
   static_assert(!std::three_way_comparable<Container<NonComparable>>);
 
   // The container should fulfill `std::three_way_comparable`
diff --git a/external/llvm-project/libcxx/utils/gdb/libcxx/printers.py b/external/llvm-project/libcxx/utils/gdb/libcxx/printers.py
index e3d5d87aca32..90bc54d987ee 100644
--- a/external/llvm-project/libcxx/utils/gdb/libcxx/printers.py
+++ b/external/llvm-project/libcxx/utils/gdb/libcxx/printers.py
@@ -14,6 +14,7 @@
 
 import re
 import gdb
+import gdb.printing
 
 # One under-documented feature of the gdb pretty-printer API
 # is that clients can call any other member of the API
diff --git a/external/llvm-project/libcxx/utils/generate_feature_test_macro_components.py b/external/llvm-project/libcxx/utils/generate_feature_test_macro_components.py
old mode 100755
new mode 100644
index 2b7f6fa8a48a..b59c7fdaf0a3
--- a/external/llvm-project/libcxx/utils/generate_feature_test_macro_components.py
+++ b/external/llvm-project/libcxx/utils/generate_feature_test_macro_components.py
@@ -357,6 +357,11 @@ def add_version_header(tc):
             "values": {"c++20": 201907},
             "headers": ["memory"],
         },
+        {
+            "name": "__cpp_lib_constexpr_forward_list",
+            "values": {"c++26": 202502},
+            "headers": ["forward_list"],
+        },
         {
             "name": "__cpp_lib_constexpr_functional",
             "values": {"c++20": 201907},
@@ -384,6 +389,11 @@ def add_version_header(tc):
             "values": {"c++20": 201911},
             "headers": ["numeric"],
         },
+        {
+            "name": "__cpp_lib_constexpr_queue",
+            "values": {"c++26": 202502},
+            "headers": ["queue"],
+        },
         {
             "name": "__cpp_lib_constexpr_string",
             "values": {"c++20": 201907},
diff --git a/external/llvm-project/libcxx/utils/libcxx/header_information.py b/external/llvm-project/libcxx/utils/libcxx/header_information.py
index a505d37b65b8..d06271a7908c 100644
--- a/external/llvm-project/libcxx/utils/libcxx/header_information.py
+++ b/external/llvm-project/libcxx/utils/libcxx/header_information.py
@@ -179,29 +179,10 @@ def __hash__(self) -> int:
     # headers with #error directives
     "atomic": "_LIBCPP_HAS_ATOMIC_HEADER",
     "stdatomic.h": "_LIBCPP_HAS_ATOMIC_HEADER",
-
-    # headers with #error directives
-    "ios": "_LIBCPP_HAS_LOCALIZATION",
-    # transitive includers of the above headers
-    "clocale": "_LIBCPP_HAS_LOCALIZATION",
-    "codecvt": "_LIBCPP_HAS_LOCALIZATION",
-    "fstream": "_LIBCPP_HAS_LOCALIZATION",
-    "iomanip": "_LIBCPP_HAS_LOCALIZATION",
-    "iostream": "_LIBCPP_HAS_LOCALIZATION",
-    "istream": "_LIBCPP_HAS_LOCALIZATION",
-    "locale": "_LIBCPP_HAS_LOCALIZATION",
-    "ostream": "_LIBCPP_HAS_LOCALIZATION",
-    "regex": "_LIBCPP_HAS_LOCALIZATION",
-    "sstream": "_LIBCPP_HAS_LOCALIZATION",
-    "streambuf": "_LIBCPP_HAS_LOCALIZATION",
-    "strstream": "_LIBCPP_HAS_LOCALIZATION",
-    "syncstream": "_LIBCPP_HAS_LOCALIZATION",
 }
 
 lit_header_restrictions = {
     "barrier": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17",
-    "clocale": "// UNSUPPORTED: no-localization",
-    "codecvt": "// UNSUPPORTED: no-localization",
     "coroutine": "// UNSUPPORTED: c++03, c++11, c++14, c++17",
     "cwchar": "// UNSUPPORTED: no-wide-characters",
     "cwctype": "// UNSUPPORTED: no-wide-characters",
@@ -211,26 +192,14 @@ def __hash__(self) -> int:
     "experimental/type_traits": "// UNSUPPORTED: c++03",
     "experimental/utility": "// UNSUPPORTED: c++03",
     "filesystem": "// UNSUPPORTED: no-filesystem, c++03, c++11, c++14",
-    "fstream": "// UNSUPPORTED: no-localization, no-filesystem",
     "future": "// UNSUPPORTED: no-threads, c++03",
-    "iomanip": "// UNSUPPORTED: no-localization",
-    "ios": "// UNSUPPORTED: no-localization",
-    "iostream": "// UNSUPPORTED: no-localization",
-    "istream": "// UNSUPPORTED: no-localization",
     "latch": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17",
-    "locale": "// UNSUPPORTED: no-localization",
     "mutex": "// UNSUPPORTED: no-threads, c++03",
-    "ostream": "// UNSUPPORTED: no-localization",
     "print": "// UNSUPPORTED: no-filesystem, c++03, c++11, c++14, c++17, c++20, availability-fp_to_chars-missing", # TODO PRINT investigate
-    "regex": "// UNSUPPORTED: no-localization",
     "semaphore": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17",
     "shared_mutex": "// UNSUPPORTED: no-threads, c++03, c++11",
-    "sstream": "// UNSUPPORTED: no-localization",
     "stdatomic.h": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17, c++20",
     "stop_token": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17",
-    "streambuf": "// UNSUPPORTED: no-localization",
-    "strstream": "// UNSUPPORTED: no-localization",
-    "syncstream": "// UNSUPPORTED: no-localization",
     "thread": "// UNSUPPORTED: no-threads, c++03",
     "wchar.h": "// UNSUPPORTED: no-wide-characters",
     "wctype.h": "// UNSUPPORTED: no-wide-characters",
diff --git a/external/llvm-project/libcxxabi/test/catch_member_function_pointer_02.pass.cpp b/external/llvm-project/libcxxabi/test/catch_member_function_pointer_02.pass.cpp
index 5d702031ce35..ec400713620c 100644
--- a/external/llvm-project/libcxxabi/test/catch_member_function_pointer_02.pass.cpp
+++ b/external/llvm-project/libcxxabi/test/catch_member_function_pointer_02.pass.cpp
@@ -12,7 +12,7 @@
 
 // GCC supports noexcept function types but this test still fails.
 // This is likely a bug in their implementation. Investigation needed.
-// XFAIL: gcc-14
+// XFAIL: gcc-14, gcc-15
 
 #include <cassert>
 
diff --git a/external/llvm-project/libcxxabi/test/configs/llvm-libc++abi-android-ndk.cfg.in b/external/llvm-project/libcxxabi/test/configs/llvm-libc++abi-android.cfg.in
similarity index 72%
rename from external/llvm-project/libcxxabi/test/configs/llvm-libc++abi-android-ndk.cfg.in
rename to external/llvm-project/libcxxabi/test/configs/llvm-libc++abi-android.cfg.in
index f2cb62a32d4e..bc5844661536 100644
--- a/external/llvm-project/libcxxabi/test/configs/llvm-libc++abi-android-ndk.cfg.in
+++ b/external/llvm-project/libcxxabi/test/configs/llvm-libc++abi-android.cfg.in
@@ -1,5 +1,5 @@
 # This testing configuration handles running the test suite against LLVM's
-# libc++abi using adb and a libc++_shared.so library on Android.
+# libc++abi using adb on Android.
 
 lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
@@ -19,12 +19,12 @@ config.substitutions.append(('%{compile_flags}',
     '-nostdinc++ -I %{include} -I %{cxx-include} -I %{cxx-target-include} %{maybe-include-libunwind} -I %{libcxx}/test/support -I %{libcxx}/src -D_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS'
 ))
 
-# The NDK library is called "libc++_shared.so". Use LD_LIBRARY_PATH to find
-# libc++_shared.so because older Bionic dynamic loaders don't support rpath
-# lookup. The Android libc++ shared library exports libc++abi, so we don't need
-# to link with -lc++abi.
+# The platform library is called "libc++.so" and the NDK library is called "libc++_shared.so". 
+# Use LD_LIBRARY_PATH to find the libcxx shared object because older Bionic dynamic loaders 
+# don't support rpath lookup. The Android libc++ shared library exports libc++abi, so we 
+# don't need to link with -lc++abi.
 config.substitutions.append(('%{link_flags}',
-    '-nostdlib++ -L %{lib} -lc++_shared'
+    '-nostdlib++ -L %{lib} -l@LIBCXX_SHARED_OUTPUT_NAME@'
 ))
 config.substitutions.append(('%{exec}',
     '%{executor}' +
diff --git a/external/llvm-project/libunwind/cmake/config-ix.cmake b/external/llvm-project/libunwind/cmake/config-ix.cmake
index 126c872f0d48..d42ceffb1f63 100644
--- a/external/llvm-project/libunwind/cmake/config-ix.cmake
+++ b/external/llvm-project/libunwind/cmake/config-ix.cmake
@@ -26,62 +26,6 @@ if (NOT LIBUNWIND_USE_COMPILER_RT)
   endif ()
 endif()
 
-# libunwind is using -nostdlib++ at the link step when available,
-# otherwise -nodefaultlibs is used. We want all our checks to also
-# use one of these options, otherwise we may end up with an inconsistency between
-# the flags we think we require during configuration (if the checks are
-# performed without one of those options) and the flags that are actually
-# required during compilation (which has the -nostdlib++ or -nodefaultlibs). libc is
-# required for the link to go through. We remove sanitizers from the
-# configuration checks to avoid spurious link errors.
-
-llvm_check_compiler_linker_flag(CXX "-nostdlib++" CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-  set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
-else()
-  llvm_check_compiler_linker_flag(C "-nodefaultlibs" C_SUPPORTS_NODEFAULTLIBS_FLAG)
-  if (C_SUPPORTS_NODEFAULTLIBS_FLAG)
-    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nodefaultlibs")
-  endif()
-endif()
-
-# Only link against compiler-rt manually if we use -nodefaultlibs, since
-# otherwise the compiler will do the right thing on its own.
-if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
-  if (LIBUNWIND_HAS_C_LIB)
-    list(APPEND CMAKE_REQUIRED_LIBRARIES c)
-  endif ()
-  if (LIBUNWIND_HAS_ROOT_LIB)
-    list(APPEND CMAKE_REQUIRED_LIBRARIES root)
-  endif ()
-  if (LIBUNWIND_USE_COMPILER_RT)
-    include(HandleCompilerRT)
-    find_compiler_rt_library(builtins LIBUNWIND_BUILTINS_LIBRARY
-                             FLAGS ${LIBUNWIND_COMPILE_FLAGS})
-    list(APPEND CMAKE_REQUIRED_LIBRARIES "${LIBUNWIND_BUILTINS_LIBRARY}")
-  else ()
-    if (LIBUNWIND_HAS_GCC_S_LIB)
-      list(APPEND CMAKE_REQUIRED_LIBRARIES gcc_s)
-    endif ()
-    if (LIBUNWIND_HAS_GCC_LIB)
-      list(APPEND CMAKE_REQUIRED_LIBRARIES gcc)
-    endif ()
-  endif ()
-  if (MINGW)
-    # Mingw64 requires quite a few "C" runtime libraries in order for basic
-    # programs to link successfully with -nodefaultlibs.
-    if (LIBUNWIND_USE_COMPILER_RT)
-      set(MINGW_RUNTIME ${LIBUNWIND_BUILTINS_LIBRARY})
-    else ()
-      set(MINGW_RUNTIME gcc_s gcc)
-    endif()
-    set(MINGW_LIBRARIES mingw32 ${MINGW_RUNTIME} moldname mingwex msvcrt advapi32
-                        shell32 user32 kernel32 mingw32 ${MINGW_RUNTIME}
-                        moldname mingwex msvcrt)
-    list(APPEND CMAKE_REQUIRED_LIBRARIES ${MINGW_LIBRARIES})
-  endif()
-endif()
-
 if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
diff --git a/external/llvm-project/libunwind/src/CMakeLists.txt b/external/llvm-project/libunwind/src/CMakeLists.txt
index 70bd3a017cda..03818b1bb251 100644
--- a/external/llvm-project/libunwind/src/CMakeLists.txt
+++ b/external/llvm-project/libunwind/src/CMakeLists.txt
@@ -71,18 +71,6 @@ set(LIBUNWIND_SOURCES
     ${LIBUNWIND_ASM_SOURCES})
 
 # Generate library list.
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-  add_link_flags_if_supported(-nostdlib++)
-else()
-  if (LIBUNWIND_USE_COMPILER_RT)
-    add_library_flags("${LIBUNWIND_BUILTINS_LIBRARY}")
-  else()
-    add_library_flags_if(LIBUNWIND_HAS_GCC_S_LIB gcc_s)
-    add_library_flags_if(LIBUNWIND_HAS_GCC_LIB gcc)
-  endif()
-  add_library_flags_if(LIBUNWIND_HAS_C_LIB c)
-endif()
-
 if (NOT APPLE)
   add_library_flags_if(LIBUNWIND_HAS_DL_LIB dl)
 endif()
diff --git a/external/llvm-project/libunwind/src/UnwindLevel1.c b/external/llvm-project/libunwind/src/UnwindLevel1.c
index a258a832a9c3..f3b451ad9b73 100644
--- a/external/llvm-project/libunwind/src/UnwindLevel1.c
+++ b/external/llvm-project/libunwind/src/UnwindLevel1.c
@@ -188,10 +188,11 @@ extern int __unw_step_stage2(unw_cursor_t *);
 
 #if defined(_LIBUNWIND_USE_GCS)
 // Enable the GCS target feature to permit gcspop instructions to be used.
-__attribute__((target("gcs")))
+__attribute__((target("+gcs")))
 #endif
 static _Unwind_Reason_Code
-unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *exception_object) {
+unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor,
+              _Unwind_Exception *exception_object) {
   __unw_init_local(cursor, uc);
 
   _LIBUNWIND_TRACE_UNWINDING("unwind_phase2(ex_obj=%p)",
@@ -332,12 +333,12 @@ unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *except
 
 #if defined(_LIBUNWIND_USE_GCS)
 // Enable the GCS target feature to permit gcspop instructions to be used.
-__attribute__((target("gcs")))
+__attribute__((target("+gcs")))
 #endif
 static _Unwind_Reason_Code
 unwind_phase2_forced(unw_context_t *uc, unw_cursor_t *cursor,
-                     _Unwind_Exception *exception_object,
-                     _Unwind_Stop_Fn stop, void *stop_parameter) {
+                     _Unwind_Exception *exception_object, _Unwind_Stop_Fn stop,
+                     void *stop_parameter) {
   __unw_init_local(cursor, uc);
 
   // uc is initialized by __unw_getcontext in the parent frame. The first stack
@@ -443,7 +444,6 @@ unwind_phase2_forced(unw_context_t *uc, unw_cursor_t *cursor,
   return _URC_FATAL_PHASE2_ERROR;
 }
 
-
 /// Called by __cxa_throw.  Only returns if there is a fatal error.
 _LIBUNWIND_EXPORT _Unwind_Reason_Code
 _Unwind_RaiseException(_Unwind_Exception *exception_object) {
diff --git a/external/llvm-project/lld/COFF/MinGW.cpp b/external/llvm-project/lld/COFF/MinGW.cpp
index ca9ae403f181..e7117cbea2e8 100644
--- a/external/llvm-project/lld/COFF/MinGW.cpp
+++ b/external/llvm-project/lld/COFF/MinGW.cpp
@@ -46,6 +46,8 @@ AutoExporter::AutoExporter(
       "libclang_rt.profile-arm",
       "libclang_rt.profile-i386",
       "libclang_rt.profile-x86_64",
+      "libcygwin",
+      "libmsys-2.0",
       "libc++",
       "libc++abi",
       "libflang_rt.runtime",
diff --git a/external/llvm-project/lld/COFF/TypeMerger.h b/external/llvm-project/lld/COFF/TypeMerger.h
index b4e3d6e7d9fd..effb6d603a59 100644
--- a/external/llvm-project/lld/COFF/TypeMerger.h
+++ b/external/llvm-project/lld/COFF/TypeMerger.h
@@ -9,6 +9,7 @@
 #ifndef LLD_COFF_TYPEMERGER_H
 #define LLD_COFF_TYPEMERGER_H
 
+#include "COFFLinkerContext.h"
 #include "Config.h"
 #include "DebugTypes.h"
 #include "lld/Common/Timer.h"
diff --git a/external/llvm-project/lld/COFF/Writer.cpp b/external/llvm-project/lld/COFF/Writer.cpp
index cb9d0001015b..5f1da5e79dac 100644
--- a/external/llvm-project/lld/COFF/Writer.cpp
+++ b/external/llvm-project/lld/COFF/Writer.cpp
@@ -2495,22 +2495,16 @@ void Writer::setECSymbols() {
               offsetof(data_directory, Size),
           ctx.symtab.edataEnd->getRVA() - ctx.symtab.edataStart->getRVA() +
               ctx.symtab.edataEnd->getSize());
-    if (hybridPdata.first) {
+    if (hybridPdata.first)
       ctx.dynamicRelocs->set(
           dataDirOffset64 + EXCEPTION_TABLE * sizeof(data_directory) +
               offsetof(data_directory, Size),
           hybridPdata.last->getRVA() - hybridPdata.first->getRVA() +
               hybridPdata.last->getSize());
-      if (chpeSym) {
-        size_t size = 0;
-        if (pdata.first)
-          size = pdata.last->getRVA() + pdata.last->getSize() -
-                 pdata.first->getRVA();
-        ctx.dynamicRelocs->set(chpeSym->getRVA() +
-                                   offsetof(chpe_metadata, ExtraRFETableSize),
-                               size);
-      }
-    }
+    if (chpeSym && pdata.first)
+      ctx.dynamicRelocs->set(
+          chpeSym->getRVA() + offsetof(chpe_metadata, ExtraRFETableSize),
+          pdata.last->getRVA() + pdata.last->getSize() - pdata.first->getRVA());
   }
 }
 
diff --git a/external/llvm-project/lld/Common/CMakeLists.txt b/external/llvm-project/lld/Common/CMakeLists.txt
index 4f503d04f784..a9e8d72fb5ec 100644
--- a/external/llvm-project/lld/Common/CMakeLists.txt
+++ b/external/llvm-project/lld/Common/CMakeLists.txt
@@ -34,6 +34,7 @@ add_lld_library(lldCommon
   Strings.cpp
   TargetOptionsCommandFlags.cpp
   Timer.cpp
+  Utils.cpp
   VCSVersion.inc
   Version.cpp
 
diff --git a/external/llvm-project/lld/Common/Utils.cpp b/external/llvm-project/lld/Common/Utils.cpp
new file mode 100644
index 000000000000..56f6b5d90924
--- /dev/null
+++ b/external/llvm-project/lld/Common/Utils.cpp
@@ -0,0 +1,22 @@
+//===- Utils.cpp ------------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//===----------------------------------------------------------------------===//
+//
+// The file defines utils functions that can be shared across archs.
+//
+//===----------------------------------------------------------------------===//
+
+#include "lld/Common/Utils.h"
+
+using namespace llvm;
+using namespace lld;
+
+StringRef lld::utils::getRootSymbol(StringRef name) {
+  name.consume_back(".Tgm");
+  auto [P0, S0] = name.rsplit(".llvm.");
+  auto [P1, S1] = P0.rsplit(".__uniq.");
+  return P1;
+}
diff --git a/external/llvm-project/lld/ELF/Arch/AArch64.cpp b/external/llvm-project/lld/ELF/Arch/AArch64.cpp
index 2b5d5e90573f..8a225ed103ee 100644
--- a/external/llvm-project/lld/ELF/Arch/AArch64.cpp
+++ b/external/llvm-project/lld/ELF/Arch/AArch64.cpp
@@ -1043,8 +1043,7 @@ AArch64BtiPac::AArch64BtiPac(Ctx &ctx) : AArch64(ctx) {
   // instructions.
 
   if (ctx.arg.zPacPlt) {
-    if (llvm::any_of(ctx.aarch64PauthAbiCoreInfo,
-                     [](uint8_t c) { return c != 0; }))
+    if (ctx.aarch64PauthAbiCoreInfo && ctx.aarch64PauthAbiCoreInfo->isValid())
       pacEntryKind = PEK_Auth;
     else
       pacEntryKind = PEK_AuthHint;
diff --git a/external/llvm-project/lld/ELF/Arch/ARM.cpp b/external/llvm-project/lld/ELF/Arch/ARM.cpp
index aa90fecc533e..91a673f13d68 100644
--- a/external/llvm-project/lld/ELF/Arch/ARM.cpp
+++ b/external/llvm-project/lld/ELF/Arch/ARM.cpp
@@ -1317,11 +1317,11 @@ void elf::processArmCmseSymbols(Ctx &ctx) {
   // with its corresponding special symbol __acle_se_<sym>.
   parallelForEach(ctx.objectFiles, [&](InputFile *file) {
     MutableArrayRef<Symbol *> syms = file->getMutableSymbols();
-    for (size_t i = 0, e = syms.size(); i != e; ++i) {
-      StringRef symName = syms[i]->getName();
+    for (Symbol *&sym : syms) {
+      StringRef symName = sym->getName();
       auto it = ctx.symtab->cmseSymMap.find(symName);
       if (it != ctx.symtab->cmseSymMap.end())
-        syms[i] = it->second.acleSeSym;
+        sym = it->second.acleSeSym;
     }
   });
 }
diff --git a/external/llvm-project/lld/ELF/Arch/RISCV.cpp b/external/llvm-project/lld/ELF/Arch/RISCV.cpp
index abe8876668a4..72d83159ad8a 100644
--- a/external/llvm-project/lld/ELF/Arch/RISCV.cpp
+++ b/external/llvm-project/lld/ELF/Arch/RISCV.cpp
@@ -740,19 +740,23 @@ static void relaxCall(Ctx &ctx, const InputSection &sec, size_t i, uint64_t loc,
       (r.expr == R_PLT_PC ? sym.getPltVA(ctx) : sym.getVA(ctx)) + r.addend;
   const int64_t displace = dest - loc;
 
-  if (rvc && isInt<12>(displace) && rd == 0) {
+  // When the caller specifies the old value of `remove`, disallow its
+  // increment.
+  if (remove >= 6 && rvc && isInt<12>(displace) && rd == 0) {
     sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
     sec.relaxAux->writes.push_back(0xa001); // c.j
     remove = 6;
-  } else if (rvc && isInt<12>(displace) && rd == X_RA &&
+  } else if (remove >= 6 && rvc && isInt<12>(displace) && rd == X_RA &&
              !ctx.arg.is64) { // RV32C only
     sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
     sec.relaxAux->writes.push_back(0x2001); // c.jal
     remove = 6;
-  } else if (isInt<21>(displace)) {
+  } else if (remove >= 4 && isInt<21>(displace)) {
     sec.relaxAux->relocTypes[i] = R_RISCV_JAL;
     sec.relaxAux->writes.push_back(0x6f | rd << 7); // jal
     remove = 4;
+  } else {
+    remove = 0;
   }
 }
 
@@ -828,7 +832,7 @@ static void relaxHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i,
   }
 }
 
-static bool relax(Ctx &ctx, InputSection &sec) {
+static bool relax(Ctx &ctx, int pass, InputSection &sec) {
   const uint64_t secAddr = sec.getVA();
   const MutableArrayRef<Relocation> relocs = sec.relocs();
   auto &aux = *sec.relaxAux;
@@ -862,8 +866,13 @@ static bool relax(Ctx &ctx, InputSection &sec) {
     }
     case R_RISCV_CALL:
     case R_RISCV_CALL_PLT:
-      if (relaxable(relocs, i))
+      // Prevent oscillation between states by disallowing the increment of
+      // `remove` after a few passes. The previous `remove` value is
+      // `cur-delta`.
+      if (relaxable(relocs, i)) {
+        remove = pass < 4 ? 6 : cur - delta;
         relaxCall(ctx, sec, i, loc, r, remove);
+      }
       break;
     case R_RISCV_TPREL_HI20:
     case R_RISCV_TPREL_ADD:
@@ -945,7 +954,7 @@ bool RISCV::relaxOnce(int pass) const {
     if (!(osec->flags & SHF_EXECINSTR))
       continue;
     for (InputSection *sec : getInputSections(*osec, storage))
-      changed |= relax(ctx, *sec);
+      changed |= relax(ctx, pass, *sec);
   }
   return changed;
 }
diff --git a/external/llvm-project/lld/ELF/BPSectionOrderer.cpp b/external/llvm-project/lld/ELF/BPSectionOrderer.cpp
index 793176c7725a..f464b1d4518a 100644
--- a/external/llvm-project/lld/ELF/BPSectionOrderer.cpp
+++ b/external/llvm-project/lld/ELF/BPSectionOrderer.cpp
@@ -81,7 +81,8 @@ DenseMap<const InputSectionBase *, int> elf::runBalancedPartitioning(
     if (!sec || sec->size == 0 || !sec->isLive() || sec->repl != sec ||
         !orderer.secToSym.try_emplace(sec, d).second)
       return;
-    rootSymbolToSectionIdxs[CachedHashStringRef(getRootSymbol(sym.getName()))]
+    rootSymbolToSectionIdxs[CachedHashStringRef(
+                                lld::utils::getRootSymbol(sym.getName()))]
         .insert(sections.size());
     sections.emplace_back(sec);
   };
diff --git a/external/llvm-project/lld/ELF/Config.h b/external/llvm-project/lld/ELF/Config.h
index f0e9592d85dd..2b72d54ba410 100644
--- a/external/llvm-project/lld/ELF/Config.h
+++ b/external/llvm-project/lld/ELF/Config.h
@@ -136,9 +136,32 @@ enum LtoKind : uint8_t {UnifiedThin, UnifiedRegular, Default};
 // For -z gcs=
 enum class GcsPolicy { Implicit, Never, Always };
 
+// For -z zicfilp=
+enum class ZicfilpPolicy { Implicit, Never, Unlabeled, FuncSig };
+
+// For -z zicfiss=
+enum class ZicfissPolicy { Implicit, Never, Always };
+
 // For some options that resemble -z bti-report={none,warning,error}
 enum class ReportPolicy { None, Warning, Error };
 
+// Describes the signing schema for a file using the PAuth ABI extension.
+// Two files are considered compatible when both `platform` and `version` match.
+// The pair (0, 0) is reserved to indicate incompatibility with the PAuth ABI.
+struct AArch64PauthAbiCoreInfo {
+  uint64_t platform;
+  uint64_t version;
+  // Returns true if the core info is not the reserved (0, 0) value.
+  bool isValid() const { return platform || version; }
+  static constexpr size_t size() { return sizeof(platform) + sizeof(version); }
+  bool operator==(const AArch64PauthAbiCoreInfo &other) const {
+    return platform == other.platform && version == other.version;
+  }
+  bool operator!=(const AArch64PauthAbiCoreInfo &other) const {
+    return !(*this == other);
+  }
+};
+
 struct SymbolVersion {
   llvm::StringRef name;
   bool isExternCpp;
@@ -235,6 +258,9 @@ struct Config {
   ReportPolicy zGcsReport = ReportPolicy::None;
   ReportPolicy zGcsReportDynamic = ReportPolicy::None;
   ReportPolicy zExecuteOnlyReport = ReportPolicy::None;
+  ReportPolicy zZicfilpUnlabeledReport = ReportPolicy::None;
+  ReportPolicy zZicfilpFuncSigReport = ReportPolicy::None;
+  ReportPolicy zZicfissReport = ReportPolicy::None;
   bool ltoBBAddrMap;
   llvm::StringRef ltoBasicBlockSections;
   std::pair<llvm::StringRef, llvm::StringRef> thinLTOObjectSuffixReplace;
@@ -365,6 +391,7 @@ struct Config {
   bool writeAddends;
   bool zCombreloc;
   bool zCopyreloc;
+  bool zDynamicUndefined;
   bool zForceBti;
   bool zForceIbt;
   bool zGlobal;
@@ -390,6 +417,8 @@ struct Config {
   bool zText;
   bool zRetpolineplt;
   bool zWxneeded;
+  ZicfilpPolicy zZicfilp;
+  ZicfissPolicy zZicfiss;
   DiscardPolicy discard;
   GnuStackKind zGnustack;
   ICFLevel icf;
@@ -695,7 +724,7 @@ struct Ctx : CommonLinkerContext {
 
   llvm::raw_fd_ostream openAuxiliaryFile(llvm::StringRef, std::error_code &);
 
-  ArrayRef<uint8_t> aarch64PauthAbiCoreInfo;
+  std::optional<AArch64PauthAbiCoreInfo> aarch64PauthAbiCoreInfo;
 };
 
 // The first two elements of versionDefinitions represent VER_NDX_LOCAL and
diff --git a/external/llvm-project/lld/ELF/DWARF.h b/external/llvm-project/lld/ELF/DWARF.h
index 64c25c706c34..19b17647378b 100644
--- a/external/llvm-project/lld/ELF/DWARF.h
+++ b/external/llvm-project/lld/ELF/DWARF.h
@@ -10,6 +10,7 @@
 #define LLD_ELF_DWARF_H
 
 #include "InputFiles.h"
+#include "InputSection.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
diff --git a/external/llvm-project/lld/ELF/Driver.cpp b/external/llvm-project/lld/ELF/Driver.cpp
index 4194e032c634..0d47d39bb5d2 100644
--- a/external/llvm-project/lld/ELF/Driver.cpp
+++ b/external/llvm-project/lld/ELF/Driver.cpp
@@ -419,8 +419,22 @@ static void checkOptions(Ctx &ctx) {
           << "--pcrel-optimize is only supported on PowerPC64 targets";
   }
 
-  if (ctx.arg.relaxGP && ctx.arg.emachine != EM_RISCV)
-    ErrAlways(ctx) << "--relax-gp is only supported on RISC-V targets";
+  if (ctx.arg.emachine != EM_RISCV) {
+    if (ctx.arg.relaxGP)
+      ErrAlways(ctx) << "--relax-gp is only supported on RISC-V targets";
+    if (ctx.arg.zZicfilpUnlabeledReport != ReportPolicy::None)
+      ErrAlways(ctx) << "-z zicfilip-unlabeled-report is only supported on "
+                        "RISC-V targets";
+    if (ctx.arg.zZicfilpFuncSigReport != ReportPolicy::None)
+      ErrAlways(ctx) << "-z zicfilip-func-sig-report is only supported on "
+                        "RISC-V targets";
+    if (ctx.arg.zZicfissReport != ReportPolicy::None)
+      ErrAlways(ctx) << "-z zicfiss-report is only supported on RISC-V targets";
+    if (ctx.arg.zZicfilp != ZicfilpPolicy::Implicit)
+      ErrAlways(ctx) << "-z zicfilp is only supported on RISC-V targets";
+    if (ctx.arg.zZicfiss != ZicfissPolicy::Implicit)
+      ErrAlways(ctx) << "-z zicfiss is only supported on RISC-V targets";
+  }
 
   if (ctx.arg.emachine != EM_386 && ctx.arg.emachine != EM_X86_64 &&
       ctx.arg.zCetReport != ReportPolicy::None)
@@ -574,6 +588,46 @@ static GcsPolicy getZGcs(Ctx &ctx, opt::InputArgList &args) {
   return ret;
 }
 
+static ZicfilpPolicy getZZicfilp(Ctx &ctx, opt::InputArgList &args) {
+  auto ret = ZicfilpPolicy::Implicit;
+  for (auto *arg : args.filtered(OPT_z)) {
+    std::pair<StringRef, StringRef> kv = StringRef(arg->getValue()).split('=');
+    if (kv.first == "zicfilp") {
+      arg->claim();
+      if (kv.second == "unlabeled")
+        ret = ZicfilpPolicy::Unlabeled;
+      else if (kv.second == "func-sig")
+        ret = ZicfilpPolicy::FuncSig;
+      else if (kv.second == "never")
+        ret = ZicfilpPolicy::Never;
+      else if (kv.second == "implicit")
+        ret = ZicfilpPolicy::Implicit;
+      else
+        ErrAlways(ctx) << "unknown -z zicfilp= value: " << kv.second;
+    }
+  }
+  return ret;
+}
+
+static ZicfissPolicy getZZicfiss(Ctx &ctx, opt::InputArgList &args) {
+  auto ret = ZicfissPolicy::Implicit;
+  for (auto *arg : args.filtered(OPT_z)) {
+    std::pair<StringRef, StringRef> kv = StringRef(arg->getValue()).split('=');
+    if (kv.first == "zicfiss") {
+      arg->claim();
+      if (kv.second == "always")
+        ret = ZicfissPolicy::Always;
+      else if (kv.second == "never")
+        ret = ZicfissPolicy::Never;
+      else if (kv.second == "implicit")
+        ret = ZicfissPolicy::Implicit;
+      else
+        ErrAlways(ctx) << "unknown -z zicfiss= value: " << kv.second;
+    }
+  }
+  return ret;
+}
+
 // Report a warning for an unknown -z option.
 static void checkZOptions(Ctx &ctx, opt::InputArgList &args) {
   // This function is called before getTarget(), when certain options are not
@@ -581,6 +635,7 @@ static void checkZOptions(Ctx &ctx, opt::InputArgList &args) {
   args::getZOptionValue(args, OPT_z, "max-page-size", 0);
   args::getZOptionValue(args, OPT_z, "common-page-size", 0);
   getZFlag(args, "rel", "rela", false);
+  getZFlag(args, "dynamic-undefined-weak", "nodynamic-undefined-weak", false);
   for (auto *arg : args.filtered(OPT_z))
     if (!arg->isClaimed())
       Warn(ctx) << "unknown -z value: " << StringRef(arg->getValue());
@@ -1569,6 +1624,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
   ctx.arg.zCopyreloc = getZFlag(args, "copyreloc", "nocopyreloc", true);
   ctx.arg.zForceBti = hasZOption(args, "force-bti");
   ctx.arg.zForceIbt = hasZOption(args, "force-ibt");
+  ctx.arg.zZicfilp = getZZicfilp(ctx, args);
+  ctx.arg.zZicfiss = getZZicfiss(ctx, args);
   ctx.arg.zGcs = getZGcs(ctx, args);
   ctx.arg.zGlobal = hasZOption(args, "global");
   ctx.arg.zGnustack = getZGnuStack(args);
@@ -1648,7 +1705,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       std::make_pair("execute-only-report", &ctx.arg.zExecuteOnlyReport),
       std::make_pair("gcs-report", &ctx.arg.zGcsReport),
       std::make_pair("gcs-report-dynamic", &ctx.arg.zGcsReportDynamic),
-      std::make_pair("pauth-report", &ctx.arg.zPauthReport)};
+      std::make_pair("pauth-report", &ctx.arg.zPauthReport),
+      std::make_pair("zicfilp-unlabeled-report",
+                     &ctx.arg.zZicfilpUnlabeledReport),
+      std::make_pair("zicfilp-func-sig-report", &ctx.arg.zZicfilpFuncSigReport),
+      std::make_pair("zicfiss-report", &ctx.arg.zZicfissReport)};
   bool hasGcsReportDynamic = false;
   for (opt::Arg *arg : args.filtered(OPT_z)) {
     std::pair<StringRef, StringRef> option =
@@ -1727,14 +1788,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       ErrAlways(ctx) << errPrefix << pat.takeError() << ": " << kv.first;
   }
 
-  // This was added by https://reviews.llvm.org/D88461 to address a problem
-  // we rocMlir do not care about in cmd options. However, the side effect
-  // of this is it will reset and corrupt global static cl::cmd options,
-  // causing issues in rocm and xmir runner. Since there is no easy way to
-  // back up the cl::cmd hidden state in JitRunner, the most straightforward
-  // fix is to undo this global reset in lld invokation.
-  // cl::ResetAllOptionOccurrences();
-  
+  cl::ResetAllOptionOccurrences();
+
   // Parse LTO options.
   if (auto *arg = args.getLastArg(OPT_plugin_opt_mcpu_eq))
     parseClangOption(ctx, ctx.saver.save("-mcpu=" + StringRef(arg->getValue())),
@@ -2848,9 +2903,12 @@ static void redirectSymbols(Ctx &ctx, ArrayRef<WrappedSymbol> wrapped) {
 // For AArch64 PAuth-enabled object files, the core info of all of them must
 // match. Missing info for some object files with matching info for remaining
 // ones can be allowed (see -z pauth-report).
+//
+// RISC-V Zicfilp/Zicfiss extension also use the same mechanism to record
+// enabled features in the GNU_PROPERTY_RISCV_FEATURE_1_AND bit mask.
 static void readSecurityNotes(Ctx &ctx) {
   if (ctx.arg.emachine != EM_386 && ctx.arg.emachine != EM_X86_64 &&
-      ctx.arg.emachine != EM_AARCH64)
+      ctx.arg.emachine != EM_AARCH64 && ctx.arg.emachine != EM_RISCV)
     return;
 
   ctx.arg.andFeatures = -1;
@@ -2858,15 +2916,15 @@ static void readSecurityNotes(Ctx &ctx) {
   StringRef referenceFileName;
   if (ctx.arg.emachine == EM_AARCH64) {
     auto it = llvm::find_if(ctx.objectFiles, [](const ELFFileBase *f) {
-      return !f->aarch64PauthAbiCoreInfo.empty();
+      return f->aarch64PauthAbiCoreInfo.has_value();
     });
     if (it != ctx.objectFiles.end()) {
       ctx.aarch64PauthAbiCoreInfo = (*it)->aarch64PauthAbiCoreInfo;
       referenceFileName = (*it)->getName();
     }
   }
-  bool hasValidPauthAbiCoreInfo = llvm::any_of(
-      ctx.aarch64PauthAbiCoreInfo, [](uint8_t c) { return c != 0; });
+  bool hasValidPauthAbiCoreInfo =
+      ctx.aarch64PauthAbiCoreInfo && ctx.aarch64PauthAbiCoreInfo->isValid();
 
   auto report = [&](ReportPolicy policy) -> ELFSyncStream {
     return {ctx, toDiagLevel(policy)};
@@ -2902,6 +2960,45 @@ static void readSecurityNotes(Ctx &ctx) {
         << ": -z cet-report: file does not have "
            "GNU_PROPERTY_X86_FEATURE_1_SHSTK property";
 
+    if (ctx.arg.emachine == EM_RISCV) {
+      reportUnless(ctx.arg.zZicfilpUnlabeledReport,
+                   features & GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED)
+          << f
+          << ": -z zicfilp-unlabeled-report: file does not have "
+             "GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED property";
+
+      reportUnless(ctx.arg.zZicfilpFuncSigReport,
+                   features & GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG)
+          << f
+          << ": -z zicfilp-func-sig-report: file does not have "
+             "GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG property";
+
+      if ((features & GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED) &&
+          (features & GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG))
+        Err(ctx) << f
+                 << ": file has conflicting properties: "
+                    "GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED and "
+                    "GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG";
+
+      reportUnless(ctx.arg.zZicfissReport,
+                   features & GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS)
+          << f
+          << ": -z zicfiss-report: file does not have "
+             "GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS property";
+
+      if (ctx.arg.zZicfilp == ZicfilpPolicy::Unlabeled &&
+          (features & GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG))
+        Warn(ctx) << f
+                  << ": -z zicfilp=unlabeled: file has conflicting property: "
+                     "GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG";
+
+      if (ctx.arg.zZicfilp == ZicfilpPolicy::FuncSig &&
+          (features & GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED))
+        Warn(ctx) << f
+                  << ": -z zicfilp=func-sig: file has conflicting property: "
+                     "GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED";
+    }
+
     if (ctx.arg.zForceBti && !(features & GNU_PROPERTY_AARCH64_FEATURE_1_BTI)) {
       features |= GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
       if (ctx.arg.zBtiReport == ReportPolicy::None)
@@ -2926,10 +3023,10 @@ static void readSecurityNotes(Ctx &ctx) {
     }
     ctx.arg.andFeatures &= features;
 
-    if (ctx.aarch64PauthAbiCoreInfo.empty())
+    if (!ctx.aarch64PauthAbiCoreInfo)
       continue;
 
-    if (f->aarch64PauthAbiCoreInfo.empty()) {
+    if (!f->aarch64PauthAbiCoreInfo) {
       report(ctx.arg.zPauthReport)
           << f
           << ": -z pauth-report: file does not have AArch64 "
@@ -2939,11 +3036,18 @@ static void readSecurityNotes(Ctx &ctx) {
     }
 
     if (ctx.aarch64PauthAbiCoreInfo != f->aarch64PauthAbiCoreInfo)
-      Err(ctx) << "incompatible values of AArch64 PAuth core info found\n>>> "
-               << referenceFileName << ": 0x"
-               << toHex(ctx.aarch64PauthAbiCoreInfo, /*LowerCase=*/true)
-               << "\n>>> " << f << ": 0x"
-               << toHex(f->aarch64PauthAbiCoreInfo, /*LowerCase=*/true);
+      Err(ctx)
+          << "incompatible values of AArch64 PAuth core info found\n"
+          << "platform:\n"
+          << ">>> " << referenceFileName << ": 0x"
+          << toHex(ctx.aarch64PauthAbiCoreInfo->platform, /*LowerCase=*/true)
+          << "\n>>> " << f << ": 0x"
+          << toHex(f->aarch64PauthAbiCoreInfo->platform, /*LowerCase=*/true)
+          << "\nversion:\n"
+          << ">>> " << referenceFileName << ": 0x"
+          << toHex(ctx.aarch64PauthAbiCoreInfo->version, /*LowerCase=*/true)
+          << "\n>>> " << f << ": 0x"
+          << toHex(f->aarch64PauthAbiCoreInfo->version, /*LowerCase=*/true);
   }
 
   // Force enable Shadow Stack.
@@ -2956,6 +3060,25 @@ static void readSecurityNotes(Ctx &ctx) {
   else if (ctx.arg.zGcs == GcsPolicy::Never)
     ctx.arg.andFeatures &= ~GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
 
+  if (ctx.arg.emachine == EM_RISCV) {
+    // Force enable/disable Zicfilp.
+    if (ctx.arg.zZicfilp == ZicfilpPolicy::Unlabeled) {
+      ctx.arg.andFeatures |= GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED;
+      ctx.arg.andFeatures &= ~GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG;
+    } else if (ctx.arg.zZicfilp == ZicfilpPolicy::FuncSig) {
+      ctx.arg.andFeatures |= GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG;
+      ctx.arg.andFeatures &= ~GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED;
+    } else if (ctx.arg.zZicfilp == ZicfilpPolicy::Never)
+      ctx.arg.andFeatures &= ~(GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED |
+                               GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG);
+
+    // Force enable/disable Zicfiss.
+    if (ctx.arg.zZicfiss == ZicfissPolicy::Always)
+      ctx.arg.andFeatures |= GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS;
+    else if (ctx.arg.zZicfiss == ZicfissPolicy::Never)
+      ctx.arg.andFeatures &= ~GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS;
+  }
+
   // If we are utilising GCS at any stage, the sharedFiles should be checked to
   // ensure they also support this feature. The gcs-report-dynamic option is
   // used to indicate if the user wants information relating to this, and will
@@ -3033,6 +3156,13 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   ctx.hasDynsym = !ctx.sharedFiles.empty() || ctx.arg.isPic;
   ctx.arg.exportDynamic &= ctx.hasDynsym;
 
+  // Preemptibility of undefined symbols when ctx.hasDynsym is true. Default is
+  // true for dynamic linking.
+  ctx.arg.zDynamicUndefined =
+      getZFlag(args, "dynamic-undefined-weak", "nodynamic-undefined-weak",
+               ctx.sharedFiles.size() || ctx.arg.shared) &&
+      ctx.hasDynsym;
+
   // If an entry symbol is in a static archive, pull out that file now.
   if (Symbol *sym = ctx.symtab->find(ctx.arg.entry))
     handleUndefined(ctx, sym, "--entry");
diff --git a/external/llvm-project/lld/ELF/InputFiles.cpp b/external/llvm-project/lld/ELF/InputFiles.cpp
index 12a77736aba7..71e72e7184b9 100644
--- a/external/llvm-project/lld/ELF/InputFiles.cpp
+++ b/external/llvm-project/lld/ELF/InputFiles.cpp
@@ -948,7 +948,7 @@ static void parseGnuPropertyNote(Ctx &ctx, ELFFileBase &f,
     } else if (ctx.arg.emachine == EM_AARCH64 &&
                type == GNU_PROPERTY_AARCH64_FEATURE_PAUTH) {
       ArrayRef<uint8_t> contents = data ? *data : desc;
-      if (!f.aarch64PauthAbiCoreInfo.empty()) {
+      if (f.aarch64PauthAbiCoreInfo) {
         return void(
             err(contents.data())
             << "multiple GNU_PROPERTY_AARCH64_FEATURE_PAUTH entries are "
@@ -959,7 +959,9 @@ static void parseGnuPropertyNote(Ctx &ctx, ELFFileBase &f,
                        "is invalid: expected 16 bytes, but got "
                     << size);
       }
-      f.aarch64PauthAbiCoreInfo = desc;
+      f.aarch64PauthAbiCoreInfo = {
+          support::endian::read64<ELFT::Endianness>(&desc[0]),
+          support::endian::read64<ELFT::Endianness>(&desc[8])};
     }
 
     // Padding is present in the note descriptor, if necessary.
@@ -968,7 +970,7 @@ static void parseGnuPropertyNote(Ctx &ctx, ELFFileBase &f,
 }
 // Read the following info from the .note.gnu.property section and write it to
 // the corresponding fields in `ObjFile`:
-// - Feature flags (32 bits) representing x86 or AArch64 features for
+// - Feature flags (32 bits) representing x86, AArch64 or RISC-V features for
 //   hardware-assisted call flow control;
 // - AArch64 PAuth ABI core info (16 bytes).
 template <class ELFT>
@@ -977,6 +979,22 @@ static void readGnuProperty(Ctx &ctx, const InputSection &sec,
   using Elf_Nhdr = typename ELFT::Nhdr;
   using Elf_Note = typename ELFT::Note;
 
+  uint32_t featureAndType;
+  switch (ctx.arg.emachine) {
+  case EM_386:
+  case EM_X86_64:
+    featureAndType = GNU_PROPERTY_X86_FEATURE_1_AND;
+    break;
+  case EM_AARCH64:
+    featureAndType = GNU_PROPERTY_AARCH64_FEATURE_1_AND;
+    break;
+  case EM_RISCV:
+    featureAndType = GNU_PROPERTY_RISCV_FEATURE_1_AND;
+    break;
+  default:
+    return;
+  }
+
   ArrayRef<uint8_t> data = sec.content();
   auto err = [&](const uint8_t *place) -> ELFSyncStream {
     auto diag = Err(ctx);
@@ -997,10 +1015,6 @@ static void readGnuProperty(Ctx &ctx, const InputSection &sec,
       continue;
     }
 
-    uint32_t featureAndType = ctx.arg.emachine == EM_AARCH64
-                                  ? GNU_PROPERTY_AARCH64_FEATURE_1_AND
-                                  : GNU_PROPERTY_X86_FEATURE_1_AND;
-
     // Read a body of a NOTE record, which consists of type-length-value fields.
     ArrayRef<uint8_t> desc = note.getDesc(sec.addralign);
     const uint8_t *base = sec.content().data();
@@ -1064,9 +1078,9 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx,
     }
 
     // Object files that use processor features such as Intel Control-Flow
-    // Enforcement (CET) or AArch64 Branch Target Identification BTI, use a
-    // .note.gnu.property section containing a bitfield of feature bits like the
-    // GNU_PROPERTY_X86_FEATURE_1_IBT flag. Read a bitmap containing the flag.
+    // Enforcement (CET), AArch64 Branch Target Identification BTI or RISC-V
+    // Zicfilp/Zicfiss extensions, use a .note.gnu.property section containing
+    // a bitfield of feature bits like the GNU_PROPERTY_X86_FEATURE_1_IBT flag.
     //
     // Since we merge bitmaps from multiple object files to create a new
     // .note.gnu.property containing a single AND'ed bitmap, we discard an input
diff --git a/external/llvm-project/lld/ELF/InputFiles.h b/external/llvm-project/lld/ELF/InputFiles.h
index 808cb5d24079..ba844ad18f63 100644
--- a/external/llvm-project/lld/ELF/InputFiles.h
+++ b/external/llvm-project/lld/ELF/InputFiles.h
@@ -241,7 +241,7 @@ class ELFFileBase : public InputFile {
   StringRef sourceFile;
   uint32_t andFeatures = 0;
   bool hasCommonSyms = false;
-  ArrayRef<uint8_t> aarch64PauthAbiCoreInfo;
+  std::optional<AArch64PauthAbiCoreInfo> aarch64PauthAbiCoreInfo;
 };
 
 // .o file.
diff --git a/external/llvm-project/lld/ELF/Relocations.cpp b/external/llvm-project/lld/ELF/Relocations.cpp
index 8413d8bb2437..6c4209a2b81e 100644
--- a/external/llvm-project/lld/ELF/Relocations.cpp
+++ b/external/llvm-project/lld/ELF/Relocations.cpp
@@ -161,7 +161,7 @@ static RelType getMipsPairType(RelType type, bool isLocal) {
     // symbol, the R_MIPS_GOT16 relocation creates a GOT entry to hold
     // the high 16 bits of the symbol's value. A paired R_MIPS_LO16
     // relocations handle low 16 bits of the address. That allows
-    // to allocate only one GOT entry for every 64 KBytes of local data.
+    // to allocate only one GOT entry for every 64 KiB of local data.
     return isLocal ? R_MIPS_LO16 : R_MIPS_NONE;
   case R_MICROMIPS_GOT16:
     return isLocal ? R_MICROMIPS_LO16 : R_MIPS_NONE;
@@ -990,10 +990,17 @@ bool RelocationScanner::isStaticLinkTimeConstant(RelExpr e, RelType type,
   // only the low bits are used.
   if (e == R_GOT || e == R_PLT)
     return ctx.target->usesOnlyLowPageBits(type) || !ctx.arg.isPic;
-
   // R_AARCH64_AUTH_ABS64 requires a dynamic relocation.
-  if (sym.isPreemptible || e == RE_AARCH64_AUTH)
+  if (e == RE_AARCH64_AUTH)
     return false;
+
+  // The behavior of an undefined weak reference is implementation defined.
+  // (We treat undefined non-weak the same as undefined weak.) For static
+  // -no-pie linking, dynamic relocations are generally avoided (except
+  // IRELATIVE). Emitting dynamic relocations for -shared aligns with its -z
+  // undefs default. Dynamic -no-pie linking and -pie allow flexibility.
+  if (sym.isPreemptible)
+    return sym.isUndefined() && !ctx.arg.isPic;
   if (!ctx.arg.isPic)
     return true;
 
@@ -1113,19 +1120,7 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
   // If the relocation is known to be a link-time constant, we know no dynamic
   // relocation will be created, pass the control to relocateAlloc() or
   // relocateNonAlloc() to resolve it.
-  //
-  // The behavior of an undefined weak reference is implementation defined. For
-  // non-link-time constants, we resolve relocations statically (let
-  // relocate{,Non}Alloc() resolve them) for -no-pie and try producing dynamic
-  // relocations for -pie and -shared.
-  //
-  // The general expectation of -no-pie static linking is that there is no
-  // dynamic relocation (except IRELATIVE). Emitting dynamic relocations for
-  // -shared matches the spirit of its -z undefs default. -pie has freedom on
-  // choices, and we choose dynamic relocations to be consistent with the
-  // handling of GOT-generating relocations.
-  if (isStaticLinkTimeConstant(expr, type, sym, offset) ||
-      (!ctx.arg.isPic && sym.isUndefWeak())) {
+  if (isStaticLinkTimeConstant(expr, type, sym, offset)) {
     sec->addReloc({expr, type, offset, addend, &sym});
     return;
   }
diff --git a/external/llvm-project/lld/ELF/Symbols.cpp b/external/llvm-project/lld/ELF/Symbols.cpp
index c461dfed0d74..de839795c50d 100644
--- a/external/llvm-project/lld/ELF/Symbols.cpp
+++ b/external/llvm-project/lld/ELF/Symbols.cpp
@@ -333,10 +333,13 @@ bool elf::computeIsPreemptible(Ctx &ctx, const Symbol &sym) {
   if (sym.visibility() != STV_DEFAULT)
     return false;
 
-  // At this point copy relocations have not been created yet, so any
-  // symbol that is not defined locally is preemptible.
+  // At this point copy relocations have not been created yet.
+  // Shared symbols are preemptible. Undefined symbols are preemptible
+  // when zDynamicUndefined (default in dynamic linking). Weakness is not
+  // checked, though undefined non-weak would typically trigger relocation
+  // errors unless options like -z undefs are used.
   if (!sym.isDefined())
-    return true;
+    return !sym.isUndefined() || ctx.arg.zDynamicUndefined;
 
   if (!ctx.arg.shared)
     return false;
@@ -360,7 +363,6 @@ void elf::parseVersionAndComputeIsPreemptible(Ctx &ctx) {
   // can contain versions in the form of <name>@<version>.
   // Let them parse and update their names to exclude version suffix.
   // In addition, compute isExported and isPreemptible.
-  bool maybePreemptible = ctx.sharedFiles.size() || ctx.arg.shared;
   for (Symbol *sym : ctx.symtab->getSymbols()) {
     if (sym->hasVersionSuffix)
       sym->parseSymbolVersion(ctx);
@@ -369,11 +371,11 @@ void elf::parseVersionAndComputeIsPreemptible(Ctx &ctx) {
       continue;
     }
     if (!sym->isDefined() && !sym->isCommon()) {
-      sym->isPreemptible = maybePreemptible && computeIsPreemptible(ctx, *sym);
+      sym->isPreemptible = computeIsPreemptible(ctx, *sym);
     } else if (ctx.arg.exportDynamic &&
                (sym->isUsedInRegularObj || !sym->ltoCanOmit)) {
       sym->isExported = true;
-      sym->isPreemptible = maybePreemptible && computeIsPreemptible(ctx, *sym);
+      sym->isPreemptible = computeIsPreemptible(ctx, *sym);
     }
   }
 }
diff --git a/external/llvm-project/lld/ELF/Symbols.h b/external/llvm-project/lld/ELF/Symbols.h
index 64f2f6eaa8d0..c117e3b716c1 100644
--- a/external/llvm-project/lld/ELF/Symbols.h
+++ b/external/llvm-project/lld/ELF/Symbols.h
@@ -343,7 +343,7 @@ class Symbol {
     flags.fetch_or(bits, std::memory_order_relaxed);
   }
   bool hasFlag(uint16_t bit) const {
-    assert(bit && (bit & (bit - 1)) == 0 && "bit must be a power of 2");
+    assert(llvm::has_single_bit(bit) && "bit must be a power of 2");
     return flags.load(std::memory_order_relaxed) & bit;
   }
 
diff --git a/external/llvm-project/lld/ELF/SyntheticSections.cpp b/external/llvm-project/lld/ELF/SyntheticSections.cpp
index 6f513b89d8f3..efec41a737b6 100644
--- a/external/llvm-project/lld/ELF/SyntheticSections.cpp
+++ b/external/llvm-project/lld/ELF/SyntheticSections.cpp
@@ -322,15 +322,28 @@ GnuPropertySection::GnuPropertySection(Ctx &ctx)
                        ctx.arg.wordsize) {}
 
 void GnuPropertySection::writeTo(uint8_t *buf) {
+  uint32_t featureAndType;
+  switch (ctx.arg.emachine) {
+  case EM_386:
+  case EM_X86_64:
+    featureAndType = GNU_PROPERTY_X86_FEATURE_1_AND;
+    break;
+  case EM_AARCH64:
+    featureAndType = GNU_PROPERTY_AARCH64_FEATURE_1_AND;
+    break;
+  case EM_RISCV:
+    featureAndType = GNU_PROPERTY_RISCV_FEATURE_1_AND;
+    break;
+  default:
+    llvm_unreachable(
+        "target machine does not support .note.gnu.property section");
+  }
+
   write32(ctx, buf, 4);                          // Name size
   write32(ctx, buf + 4, getSize() - 16);         // Content size
   write32(ctx, buf + 8, NT_GNU_PROPERTY_TYPE_0); // Type
   memcpy(buf + 12, "GNU", 4);               // Name string
 
-  uint32_t featureAndType = ctx.arg.emachine == EM_AARCH64
-                                ? GNU_PROPERTY_AARCH64_FEATURE_1_AND
-                                : GNU_PROPERTY_X86_FEATURE_1_AND;
-
   unsigned offset = 16;
   if (ctx.arg.andFeatures != 0) {
     write32(ctx, buf + offset + 0, featureAndType);      // Feature type
@@ -341,11 +354,11 @@ void GnuPropertySection::writeTo(uint8_t *buf) {
     offset += 16;
   }
 
-  if (!ctx.aarch64PauthAbiCoreInfo.empty()) {
+  if (ctx.aarch64PauthAbiCoreInfo) {
     write32(ctx, buf + offset + 0, GNU_PROPERTY_AARCH64_FEATURE_PAUTH);
-    write32(ctx, buf + offset + 4, ctx.aarch64PauthAbiCoreInfo.size());
-    memcpy(buf + offset + 8, ctx.aarch64PauthAbiCoreInfo.data(),
-           ctx.aarch64PauthAbiCoreInfo.size());
+    write32(ctx, buf + offset + 4, AArch64PauthAbiCoreInfo::size());
+    write64(ctx, buf + offset + 8, ctx.aarch64PauthAbiCoreInfo->platform);
+    write64(ctx, buf + offset + 16, ctx.aarch64PauthAbiCoreInfo->version);
   }
 }
 
@@ -353,8 +366,8 @@ size_t GnuPropertySection::getSize() const {
   uint32_t contentSize = 0;
   if (ctx.arg.andFeatures != 0)
     contentSize += ctx.arg.is64 ? 16 : 12;
-  if (!ctx.aarch64PauthAbiCoreInfo.empty())
-    contentSize += 4 + 4 + ctx.aarch64PauthAbiCoreInfo.size();
+  if (ctx.aarch64PauthAbiCoreInfo)
+    contentSize += 4 + 4 + AArch64PauthAbiCoreInfo::size();
   assert(contentSize != 0);
   return contentSize + 16;
 }
@@ -668,7 +681,8 @@ void GotSection::addEntry(const Symbol &sym) {
 }
 
 void GotSection::addAuthEntry(const Symbol &sym) {
-  authEntries.push_back({(numEntries - 1) * ctx.arg.wordsize, sym.isFunc()});
+  authEntries.push_back(
+      {(numEntries - 1) * ctx.target->gotEntrySize, sym.isFunc()});
 }
 
 bool GotSection::addTlsDescEntry(const Symbol &sym) {
@@ -679,8 +693,8 @@ bool GotSection::addTlsDescEntry(const Symbol &sym) {
 }
 
 void GotSection::addTlsDescAuthEntry() {
-  authEntries.push_back({(numEntries - 2) * ctx.arg.wordsize, true});
-  authEntries.push_back({(numEntries - 1) * ctx.arg.wordsize, false});
+  authEntries.push_back({(numEntries - 2) * ctx.target->gotEntrySize, true});
+  authEntries.push_back({(numEntries - 1) * ctx.target->gotEntrySize, false});
 }
 
 bool GotSection::addDynTlsEntry(const Symbol &sym) {
@@ -696,13 +710,13 @@ bool GotSection::addDynTlsEntry(const Symbol &sym) {
 bool GotSection::addTlsIndex() {
   if (tlsIndexOff != uint32_t(-1))
     return false;
-  tlsIndexOff = numEntries * ctx.arg.wordsize;
+  tlsIndexOff = numEntries * ctx.target->gotEntrySize;
   numEntries += 2;
   return true;
 }
 
 uint32_t GotSection::getTlsDescOffset(const Symbol &sym) const {
-  return sym.getTlsDescIdx(ctx) * ctx.arg.wordsize;
+  return sym.getTlsDescIdx(ctx) * ctx.target->gotEntrySize;
 }
 
 uint64_t GotSection::getTlsDescAddr(const Symbol &sym) const {
@@ -710,11 +724,11 @@ uint64_t GotSection::getTlsDescAddr(const Symbol &sym) const {
 }
 
 uint64_t GotSection::getGlobalDynAddr(const Symbol &b) const {
-  return this->getVA() + b.getTlsGdIdx(ctx) * ctx.arg.wordsize;
+  return this->getVA() + b.getTlsGdIdx(ctx) * ctx.target->gotEntrySize;
 }
 
 uint64_t GotSection::getGlobalDynOffset(const Symbol &b) const {
-  return b.getTlsGdIdx(ctx) * ctx.arg.wordsize;
+  return b.getTlsGdIdx(ctx) * ctx.target->gotEntrySize;
 }
 
 void GotSection::finalizeContents() {
@@ -723,7 +737,7 @@ void GotSection::finalizeContents() {
       !ctx.sym.globalOffsetTable)
     size = 0;
   else
-    size = numEntries * ctx.arg.wordsize;
+    size = numEntries * ctx.target->gotEntrySize;
 }
 
 bool GotSection::isNeeded() const {
@@ -1199,7 +1213,7 @@ void MipsGotSection::writeTo(uint8_t *buf) {
 // consistent across both 64-bit PowerPC ABIs as well as the 32-bit PowerPC ABI.
 GotPltSection::GotPltSection(Ctx &ctx)
     : SyntheticSection(ctx, ".got.plt", SHT_PROGBITS, SHF_ALLOC | SHF_WRITE,
-                       ctx.arg.wordsize) {
+                       ctx.target->gotEntrySize) {
   if (ctx.arg.emachine == EM_PPC) {
     name = ".plt";
   } else if (ctx.arg.emachine == EM_PPC64) {
@@ -1925,11 +1939,8 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize(Ctx &ctx) {
   // For Rela, we also want to sort by r_addend when r_info is the same. This
   // enables us to group by r_addend as well.
   llvm::sort(nonRelatives, [](const Elf_Rela &a, const Elf_Rela &b) {
-    if (a.r_info != b.r_info)
-      return a.r_info < b.r_info;
-    if (a.r_addend != b.r_addend)
-      return a.r_addend < b.r_addend;
-    return a.r_offset < b.r_offset;
+    return std::tie(a.r_info, a.r_addend, a.r_offset) <
+           std::tie(b.r_info, b.r_addend, b.r_offset);
   });
 
   // Group relocations with the same r_info. Note that each group emits a group
@@ -4015,10 +4026,9 @@ void MergeNoTailSection::finalizeContents() {
   // So far, section pieces have offsets from beginning of shards, but
   // we want offsets from beginning of the whole section. Fix them.
   parallelForEach(sections, [&](MergeInputSection *sec) {
-    for (size_t i = 0, e = sec->pieces.size(); i != e; ++i)
-      if (sec->pieces[i].live)
-        sec->pieces[i].outputOff +=
-            shardOffsets[getShardId(sec->pieces[i].hash)];
+    for (SectionPiece &piece : sec->pieces)
+      if (piece.live)
+        piece.outputOff += shardOffsets[getShardId(piece.hash)];
   });
 }
 
@@ -4956,7 +4966,7 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
   ctx.in.iplt = std::make_unique<IpltSection>(ctx);
   add(*ctx.in.iplt);
 
-  if (ctx.arg.andFeatures || !ctx.aarch64PauthAbiCoreInfo.empty()) {
+  if (ctx.arg.andFeatures || ctx.aarch64PauthAbiCoreInfo) {
     ctx.in.gnuProperty = std::make_unique<GnuPropertySection>(ctx);
     add(*ctx.in.gnuProperty);
   }
diff --git a/external/llvm-project/lld/ELF/Writer.cpp b/external/llvm-project/lld/ELF/Writer.cpp
index ec1f87a47479..15909daf51ab 100644
--- a/external/llvm-project/lld/ELF/Writer.cpp
+++ b/external/llvm-project/lld/ELF/Writer.cpp
@@ -285,7 +285,6 @@ static void demoteDefined(Defined &sym, DenseMap<SectionBase *, size_t> &map) {
 static void demoteSymbolsAndComputeIsPreemptible(Ctx &ctx) {
   llvm::TimeTraceScope timeScope("Demote symbols");
   DenseMap<InputFile *, DenseMap<SectionBase *, size_t>> sectionIndexMap;
-  bool maybePreemptible = ctx.sharedFiles.size() || ctx.arg.shared;
   for (Symbol *sym : ctx.symtab->getSymbols()) {
     if (auto *d = dyn_cast<Defined>(sym)) {
       if (d->section && !d->section->isLive())
@@ -301,9 +300,8 @@ static void demoteSymbolsAndComputeIsPreemptible(Ctx &ctx) {
       }
     }
 
-    if (maybePreemptible)
-      sym->isPreemptible = (sym->isUndefined() || sym->isExported) &&
-                           computeIsPreemptible(ctx, *sym);
+    sym->isPreemptible = (sym->isUndefined() || sym->isExported) &&
+                         computeIsPreemptible(ctx, *sym);
   }
 }
 
@@ -653,15 +651,17 @@ enum RankFlags {
   RF_NOT_ADDR_SET = 1 << 27,
   RF_NOT_ALLOC = 1 << 26,
   RF_PARTITION = 1 << 18, // Partition number (8 bits)
+  RF_LARGE_EXEC_WRITE = 1 << 16,
   RF_LARGE_ALT = 1 << 15,
   RF_WRITE = 1 << 14,
   RF_EXEC_WRITE = 1 << 13,
   RF_EXEC = 1 << 12,
   RF_RODATA = 1 << 11,
-  RF_LARGE = 1 << 10,
-  RF_NOT_RELRO = 1 << 9,
-  RF_NOT_TLS = 1 << 8,
-  RF_BSS = 1 << 7,
+  RF_LARGE_EXEC = 1 << 10,
+  RF_LARGE = 1 << 9,
+  RF_NOT_RELRO = 1 << 8,
+  RF_NOT_TLS = 1 << 7,
+  RF_BSS = 1 << 6,
 };
 
 unsigned elf::getSectionRank(Ctx &ctx, OutputSection &osec) {
@@ -691,6 +691,7 @@ unsigned elf::getSectionRank(Ctx &ctx, OutputSection &osec) {
   // places.
   bool isExec = osec.flags & SHF_EXECINSTR;
   bool isWrite = osec.flags & SHF_WRITE;
+  bool isLarge = osec.flags & SHF_X86_64_LARGE && ctx.arg.emachine == EM_X86_64;
 
   if (!isWrite && !isExec) {
     // Among PROGBITS sections, place .lrodata further from .text.
@@ -698,7 +699,7 @@ unsigned elf::getSectionRank(Ctx &ctx, OutputSection &osec) {
     // layout has one extra PT_LOAD, but alleviates relocation overflow
     // pressure for absolute relocations referencing small data from -fno-pic
     // relocatable files.
-    if (osec.flags & SHF_X86_64_LARGE && ctx.arg.emachine == EM_X86_64)
+    if (isLarge)
       rank |= ctx.arg.zLrodataAfterBss ? RF_LARGE_ALT : 0;
     else
       rank |= ctx.arg.zLrodataAfterBss ? 0 : RF_LARGE;
@@ -722,7 +723,13 @@ unsigned elf::getSectionRank(Ctx &ctx, OutputSection &osec) {
     else
       rank |= RF_RODATA;
   } else if (isExec) {
-    rank |= isWrite ? RF_EXEC_WRITE : RF_EXEC;
+    // Place readonly .ltext before .lrodata and writable .ltext after .lbss to
+    // keep writable and readonly segments separate.
+    if (isLarge) {
+      rank |= isWrite ? RF_LARGE_EXEC_WRITE : RF_LARGE_EXEC;
+    } else {
+      rank |= isWrite ? RF_EXEC_WRITE : RF_EXEC;
+    }
   } else {
     rank |= RF_WRITE;
     // The TLS initialization block needs to be a single contiguous block. Place
@@ -737,7 +744,7 @@ unsigned elf::getSectionRank(Ctx &ctx, OutputSection &osec) {
     // alleviates relocation overflow pressure.
     // For -z lrodata-after-bss, place .lbss/.lrodata/.ldata after .bss.
     // .bss/.lbss being adjacent reuses the NOBITS size optimization.
-    if (osec.flags & SHF_X86_64_LARGE && ctx.arg.emachine == EM_X86_64) {
+    if (isLarge) {
       rank |= ctx.arg.zLrodataAfterBss
                   ? (osec.type == SHT_NOBITS ? 1 : RF_LARGE_ALT)
                   : RF_LARGE;
@@ -1030,7 +1037,7 @@ findOrphanPos(Ctx &ctx, SmallVectorImpl<SectionCommand *>::iterator b,
   // This matches bfd's behavior and is convenient when the linker script fully
   // specifies the start of the file, but doesn't care about the end (the non
   // alloc sections for example).
-  if (std::find_if(i, e, isOutputSecWithInputSections) == e)
+  if (std::none_of(i, e, isOutputSecWithInputSections))
     return e;
 
   while (i != e && shouldSkip(*i))
diff --git a/external/llvm-project/lld/MachO/BPSectionOrderer.cpp b/external/llvm-project/lld/MachO/BPSectionOrderer.cpp
index 6c13a5557e10..1f07ea9caf5d 100644
--- a/external/llvm-project/lld/MachO/BPSectionOrderer.cpp
+++ b/external/llvm-project/lld/MachO/BPSectionOrderer.cpp
@@ -124,7 +124,7 @@ DenseMap<const InputSection *, int> lld::macho::runBalancedPartitioning(
         size_t idx = sections.size();
         sections.emplace_back(isec);
         for (auto *sym : BPOrdererMachO::getSymbols(*isec)) {
-          auto rootName = getRootSymbol(sym->getName());
+          auto rootName = lld::utils::getRootSymbol(sym->getName());
           rootSymbolToSectionIdxs[CachedHashStringRef(rootName)].insert(idx);
           if (auto linkageName =
                   BPOrdererMachO::getResolvedLinkageName(rootName))
diff --git a/external/llvm-project/lld/MachO/DriverUtils.cpp b/external/llvm-project/lld/MachO/DriverUtils.cpp
index 0f17ee7804cf..f7f6be049f0e 100644
--- a/external/llvm-project/lld/MachO/DriverUtils.cpp
+++ b/external/llvm-project/lld/MachO/DriverUtils.cpp
@@ -227,7 +227,12 @@ static DenseMap<CachedHashStringRef, DylibFile *> loadedDylibs;
 
 DylibFile *macho::loadDylib(MemoryBufferRef mbref, DylibFile *umbrella,
                             bool isBundleLoader, bool explicitlyLinked) {
-  CachedHashStringRef path(mbref.getBufferIdentifier());
+  // Frameworks can be found from different symlink paths, so resolve
+  // symlinks before looking up in the dylib cache.
+  SmallString<128> realPath;
+  std::error_code err = fs::real_path(mbref.getBufferIdentifier(), realPath);
+  CachedHashStringRef path(!err ? uniqueSaver().save(StringRef(realPath))
+                                : mbref.getBufferIdentifier());
   DylibFile *&file = loadedDylibs[path];
   if (file) {
     if (explicitlyLinked)
@@ -235,23 +240,6 @@ DylibFile *macho::loadDylib(MemoryBufferRef mbref, DylibFile *umbrella,
     return file;
   }
 
-  // Frameworks can be found from different symlink paths, so resolve
-  // symlinks and look up in the dylib cache.
-  DylibFile *&realfile = file;
-  SmallString<128> realPath;
-  std::error_code err = fs::real_path(mbref.getBufferIdentifier(), realPath);
-  if (!err) {
-    CachedHashStringRef resolvedPath(uniqueSaver().save(StringRef(realPath)));
-    realfile = loadedDylibs[resolvedPath];
-    if (realfile) {
-      if (explicitlyLinked)
-        realfile->setExplicitlyLinked();
-
-      file = realfile;
-      return realfile;
-    }
-  }
-
   DylibFile *newFile;
   file_magic magic = identify_magic(mbref.getBuffer());
   if (magic == file_magic::tapi_file) {
@@ -263,7 +251,6 @@ DylibFile *macho::loadDylib(MemoryBufferRef mbref, DylibFile *umbrella,
     }
     file =
         make<DylibFile>(**result, umbrella, isBundleLoader, explicitlyLinked);
-    realfile = file;
 
     // parseReexports() can recursively call loadDylib(). That's fine since
     // we wrote the DylibFile we just loaded to the loadDylib cache via the
@@ -279,7 +266,6 @@ DylibFile *macho::loadDylib(MemoryBufferRef mbref, DylibFile *umbrella,
            magic == file_magic::macho_executable ||
            magic == file_magic::macho_bundle);
     file = make<DylibFile>(mbref, umbrella, isBundleLoader, explicitlyLinked);
-    realfile = file;
 
     // parseLoadCommands() can also recursively call loadDylib(). See comment
     // in previous block for why this means we must copy `file` here.
diff --git a/external/llvm-project/lld/MachO/SectionPriorities.cpp b/external/llvm-project/lld/MachO/SectionPriorities.cpp
index 7a4a5d8465f6..cf657aad5d14 100644
--- a/external/llvm-project/lld/MachO/SectionPriorities.cpp
+++ b/external/llvm-project/lld/MachO/SectionPriorities.cpp
@@ -21,6 +21,7 @@
 #include "lld/Common/Args.h"
 #include "lld/Common/CommonLinkerContext.h"
 #include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Utils.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Support/Path.h"
@@ -246,15 +247,13 @@ DenseMap<const InputSection *, int> CallGraphSort::run() {
 }
 
 std::optional<int>
-macho::PriorityBuilder::getSymbolPriority(const Defined *sym) {
-  if (sym->isAbsolute())
-    return std::nullopt;
+macho::PriorityBuilder::getSymbolOrCStringPriority(const StringRef key,
+                                                   InputFile *f) {
 
-  auto it = priorities.find(sym->getName());
+  auto it = priorities.find(key);
   if (it == priorities.end())
     return std::nullopt;
   const SymbolPriorityEntry &entry = it->second;
-  const InputFile *f = sym->isec()->getFile();
   if (!f)
     return entry.anyObjectFile;
   // We don't use toString(InputFile *) here because it returns the full path
@@ -268,6 +267,14 @@ macho::PriorityBuilder::getSymbolPriority(const Defined *sym) {
   return std::min(entry.objectFiles.lookup(filename), entry.anyObjectFile);
 }
 
+std::optional<int>
+macho::PriorityBuilder::getSymbolPriority(const Defined *sym) {
+  if (sym->isAbsolute())
+    return std::nullopt;
+  return getSymbolOrCStringPriority(utils::getRootSymbol(sym->getName()),
+                                    sym->isec()->getFile());
+}
+
 void macho::PriorityBuilder::extractCallGraphProfile() {
   TimeTraceScope timeScope("Extract call graph profile");
   bool hasOrderFile = !priorities.empty();
@@ -300,7 +307,7 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) {
   int prio = std::numeric_limits<int>::min();
   MemoryBufferRef mbref = *buffer;
   for (StringRef line : args::getLines(mbref)) {
-    StringRef objectFile, symbol;
+    StringRef objectFile, symbolOrCStrHash;
     line = line.take_until([](char c) { return c == '#'; }); // ignore comments
     line = line.ltrim();
 
@@ -315,7 +322,6 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) {
 
     if (cpuType != CPU_TYPE_ANY && cpuType != target->cpuType)
       continue;
-
     // Drop the CPU type as well as the colon
     if (cpuType != CPU_TYPE_ANY)
       line = line.drop_until([](char c) { return c == ':'; }).drop_front();
@@ -330,10 +336,20 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) {
         break;
       }
     }
-    symbol = line.trim();
 
-    if (!symbol.empty()) {
-      SymbolPriorityEntry &entry = priorities[symbol];
+    // The rest of the line is either <symbol name> or
+    // CStringEntryPrefix<cstring hash>
+    line = line.trim();
+    if (line.starts_with(CStringEntryPrefix)) {
+      StringRef possibleHash = line.drop_front(CStringEntryPrefix.size());
+      uint32_t hash = 0;
+      if (to_integer(possibleHash, hash))
+        symbolOrCStrHash = possibleHash;
+    } else
+      symbolOrCStrHash = utils::getRootSymbol(line);
+
+    if (!symbolOrCStrHash.empty()) {
+      SymbolPriorityEntry &entry = priorities[symbolOrCStrHash];
       if (!objectFile.empty())
         entry.objectFiles.insert(std::make_pair(objectFile, prio));
       else
@@ -388,3 +404,41 @@ macho::PriorityBuilder::buildInputSectionPriorities() {
 
   return sectionPriorities;
 }
+
+std::vector<StringPiecePair> macho::PriorityBuilder::buildCStringPriorities(
+    ArrayRef<CStringInputSection *> inputs) {
+  // Split the input strings into hold and cold sets.
+  // Order hot set based on -order_file_cstring for performance improvement;
+  // TODO: Order cold set of cstrings for compression via BP.
+  std::vector<std::pair<int, StringPiecePair>>
+      hotStringPrioritiesAndStringPieces;
+  std::vector<StringPiecePair> coldStringPieces;
+  std::vector<StringPiecePair> orderedStringPieces;
+
+  for (CStringInputSection *isec : inputs) {
+    for (const auto &[stringPieceIdx, piece] : llvm::enumerate(isec->pieces)) {
+      if (!piece.live)
+        continue;
+
+      std::optional<int> priority = getSymbolOrCStringPriority(
+          std::to_string(piece.hash), isec->getFile());
+      if (!priority)
+        coldStringPieces.emplace_back(isec, stringPieceIdx);
+      else
+        hotStringPrioritiesAndStringPieces.emplace_back(
+            *priority, std::make_pair(isec, stringPieceIdx));
+    }
+  }
+
+  // Order hot set for perf
+  llvm::stable_sort(hotStringPrioritiesAndStringPieces);
+  for (auto &[priority, stringPiecePair] : hotStringPrioritiesAndStringPieces)
+    orderedStringPieces.push_back(stringPiecePair);
+
+  // TODO: Order cold set for compression
+
+  orderedStringPieces.insert(orderedStringPieces.end(),
+                             coldStringPieces.begin(), coldStringPieces.end());
+
+  return orderedStringPieces;
+}
diff --git a/external/llvm-project/lld/MachO/SectionPriorities.h b/external/llvm-project/lld/MachO/SectionPriorities.h
index 44fb101990c5..cc4e30fffc60 100644
--- a/external/llvm-project/lld/MachO/SectionPriorities.h
+++ b/external/llvm-project/lld/MachO/SectionPriorities.h
@@ -16,6 +16,7 @@
 namespace lld::macho {
 
 using SectionPair = std::pair<const InputSection *, const InputSection *>;
+using StringPiecePair = std::pair<CStringInputSection *, size_t>;
 
 class PriorityBuilder {
 public:
@@ -28,17 +29,28 @@ class PriorityBuilder {
   //
   // An order file has one entry per line, in the following format:
   //
-  //   <cpu>:<object file>:<symbol name>
+  //   <cpu>:<object file>:[<symbol name> | CStringEntryPrefix <cstring hash>]
   //
-  // <cpu> and <object file> are optional. If not specified, then that entry
-  // matches any symbol of that name. Parsing this format is not quite
-  // straightforward because the symbol name itself can contain colons, so when
-  // encountering a colon, we consider the preceding characters to decide if it
-  // can be a valid CPU type or file path.
+  // <cpu> and <object file> are optional.
+  // If not specified, then that entry tries to match either,
   //
+  // 1) any symbol of the <symbol name>;
+  // Parsing this format is not quite straightforward because the symbol name
+  // itself can contain colons, so when encountering a colon, we consider the
+  // preceding characters to decide if it can be a valid CPU type or file path.
   // If a symbol is matched by multiple entries, then it takes the
   // lowest-ordered entry (the one nearest to the front of the list.)
   //
+  // or 2) any cstring literal with the given hash, if the entry has the
+  // CStringEntryPrefix prefix defined below in the file. <cstring hash> is the
+  // hash of cstring literal content.
+  //
+  // Cstring literals are not symbolized, we can't identify them by name
+  // However, cstrings are deduplicated, hence unique, so we use the hash of
+  // the content of cstring literals to identify them and assign priority to it.
+  // We use the same hash as used in StringPiece, i.e. 31 bit:
+  // xxh3_64bits(string) & 0x7fffffff
+  //
   // The file can also have line comments that start with '#'.
   void parseOrderFile(StringRef path);
 
@@ -54,6 +66,8 @@ class PriorityBuilder {
   // Each section gets assigned the priority of the highest-priority symbol it
   // contains.
   llvm::DenseMap<const InputSection *, int> buildInputSectionPriorities();
+  std::vector<StringPiecePair>
+      buildCStringPriorities(ArrayRef<CStringInputSection *>);
 
 private:
   // The symbol with the smallest priority should be ordered first in the output
@@ -65,8 +79,11 @@ class PriorityBuilder {
     // The priority given to a matching symbol from a particular object file.
     llvm::DenseMap<llvm::StringRef, int> objectFiles;
   };
+  const llvm::StringRef CStringEntryPrefix = "CSTR;";
 
   std::optional<int> getSymbolPriority(const Defined *sym);
+  std::optional<int> getSymbolOrCStringPriority(const StringRef key,
+                                                InputFile *f);
   llvm::DenseMap<llvm::StringRef, SymbolPriorityEntry> priorities;
   llvm::MapVector<SectionPair, uint64_t> callGraphProfile;
 };
diff --git a/external/llvm-project/lld/MachO/SyntheticSections.cpp b/external/llvm-project/lld/MachO/SyntheticSections.cpp
index 2113aa88d97a..979a4ee6d813 100644
--- a/external/llvm-project/lld/MachO/SyntheticSections.cpp
+++ b/external/llvm-project/lld/MachO/SyntheticSections.cpp
@@ -14,6 +14,7 @@
 #include "InputFiles.h"
 #include "ObjC.h"
 #include "OutputSegment.h"
+#include "SectionPriorities.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 
@@ -946,9 +947,7 @@ uint64_t ObjCStubsSection::getSize() const {
 
 void ObjCStubsSection::writeTo(uint8_t *buf) const {
   uint64_t stubOffset = 0;
-  for (size_t i = 0, n = symbols.size(); i < n; ++i) {
-    Defined *sym = symbols[i];
-
+  for (Defined *sym : symbols) {
     auto methname = getMethname(sym);
     InputSection *selRef = ObjCSelRefsHelper::getSelRef(methname);
     assert(selRef != nullptr && "no selref for methname");
@@ -1760,26 +1759,25 @@ void DeduplicatedCStringSection::finalizeContents() {
     }
   }
 
-  // Assign an offset for each string and save it to the corresponding
+  // Sort the strings for performance and compression size win, and then
+  // assign an offset for each string and save it to the corresponding
   // StringPieces for easy access.
-  for (CStringInputSection *isec : inputs) {
-    for (const auto &[i, piece] : llvm::enumerate(isec->pieces)) {
-      if (!piece.live)
-        continue;
-      auto s = isec->getCachedHashStringRef(i);
-      auto it = stringOffsetMap.find(s);
-      assert(it != stringOffsetMap.end());
-      StringOffset &offsetInfo = it->second;
-      if (offsetInfo.outSecOff == UINT64_MAX) {
-        offsetInfo.outSecOff =
-            alignToPowerOf2(size, 1ULL << offsetInfo.trailingZeros);
-        size =
-            offsetInfo.outSecOff + s.size() + 1; // account for null terminator
-      }
-      piece.outSecOff = offsetInfo.outSecOff;
+  for (auto &[isec, i] : priorityBuilder.buildCStringPriorities(inputs)) {
+    auto &piece = isec->pieces[i];
+    auto s = isec->getCachedHashStringRef(i);
+    auto it = stringOffsetMap.find(s);
+    assert(it != stringOffsetMap.end());
+    lld::macho::DeduplicatedCStringSection::StringOffset &offsetInfo =
+        it->second;
+    if (offsetInfo.outSecOff == UINT64_MAX) {
+      offsetInfo.outSecOff =
+          alignToPowerOf2(size, 1ULL << offsetInfo.trailingZeros);
+      size = offsetInfo.outSecOff + s.size() + 1; // account for null terminator
     }
-    isec->isFinal = true;
+    piece.outSecOff = offsetInfo.outSecOff;
   }
+  for (CStringInputSection *isec : inputs)
+    isec->isFinal = true;
 }
 
 void DeduplicatedCStringSection::writeTo(uint8_t *buf) const {
diff --git a/external/llvm-project/lld/MachO/UnwindInfoSection.cpp b/external/llvm-project/lld/MachO/UnwindInfoSection.cpp
index 624464e41d77..6e9f6c2aba74 100644
--- a/external/llvm-project/lld/MachO/UnwindInfoSection.cpp
+++ b/external/llvm-project/lld/MachO/UnwindInfoSection.cpp
@@ -535,11 +535,9 @@ void UnwindInfoSectionImpl::finalize() {
   llvm::sort(commonEncodings,
              [](const std::pair<compact_unwind_encoding_t, size_t> &a,
                 const std::pair<compact_unwind_encoding_t, size_t> &b) {
-               if (a.second == b.second)
-                 // When frequencies match, secondarily sort on encoding
-                 // to maintain parity with validate-unwind-info.py
-                 return a.first > b.first;
-               return a.second > b.second;
+               // When frequencies match, secondarily sort on encoding
+               // to maintain parity with validate-unwind-info.py
+               return std::tie(a.second, a.first) > std::tie(b.second, b.first);
              });
 
   // Truncate the vector to 127 elements.
diff --git a/external/llvm-project/lld/MinGW/Driver.cpp b/external/llvm-project/lld/MinGW/Driver.cpp
index 8996293fdfa1..98d48bdfcf31 100644
--- a/external/llvm-project/lld/MinGW/Driver.cpp
+++ b/external/llvm-project/lld/MinGW/Driver.cpp
@@ -138,8 +138,9 @@ static std::optional<std::string> findFile(StringRef path1,
 }
 
 // This is for -lfoo. We'll look for libfoo.dll.a or libfoo.a from search paths.
-static std::string
-searchLibrary(StringRef name, ArrayRef<StringRef> searchPaths, bool bStatic) {
+static std::string searchLibrary(StringRef name,
+                                 ArrayRef<StringRef> searchPaths, bool bStatic,
+                                 StringRef prefix) {
   if (name.starts_with(":")) {
     for (StringRef dir : searchPaths)
       if (std::optional<std::string> s = findFile(dir, name.substr(1)))
@@ -160,7 +161,7 @@ searchLibrary(StringRef name, ArrayRef<StringRef> searchPaths, bool bStatic) {
     if (std::optional<std::string> s = findFile(dir, name + ".lib"))
       return *s;
     if (!bStatic) {
-      if (std::optional<std::string> s = findFile(dir, "lib" + name + ".dll"))
+      if (std::optional<std::string> s = findFile(dir, prefix + name + ".dll"))
         return *s;
       if (std::optional<std::string> s = findFile(dir, name + ".dll"))
         return *s;
@@ -554,6 +555,10 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
     add("-libpath:" + StringRef(a->getValue()));
   }
 
+  StringRef dllPrefix = "lib";
+  if (auto *arg = args.getLastArg(OPT_dll_search_prefix))
+    dllPrefix = arg->getValue();
+
   StringRef prefix = "";
   bool isStatic = false;
   for (auto *a : args) {
@@ -565,7 +570,8 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
         add(prefix + StringRef(a->getValue()));
       break;
     case OPT_l:
-      add(prefix + searchLibrary(a->getValue(), searchPaths, isStatic));
+      add(prefix +
+          searchLibrary(a->getValue(), searchPaths, isStatic, dllPrefix));
       break;
     case OPT_whole_archive:
       prefix = "-wholearchive:";
diff --git a/external/llvm-project/lld/MinGW/Options.td b/external/llvm-project/lld/MinGW/Options.td
index 01b01972112a..e6cf48e685b7 100644
--- a/external/llvm-project/lld/MinGW/Options.td
+++ b/external/llvm-project/lld/MinGW/Options.td
@@ -79,6 +79,8 @@ defm exclude_symbols: Eq<"exclude-symbols",
     "Exclude symbols from automatic export">, MetaVarName<"<symbol[,symbol,...]>">;
 def export_all_symbols: F<"export-all-symbols">,
     HelpText<"Export all symbols even if a def file or dllexport attributes are used">;
+defm dll_search_prefix:Eq<"dll-search-prefix", "Specify DLL prefix instead of 'lib'">,
+    MetaVarName<"<dll_search_prefix>">;
 defm fatal_warnings: B<"fatal-warnings",
     "Treat warnings as errors",
     "Do not treat warnings as errors (default)">;
diff --git a/external/llvm-project/lld/docs/ReleaseNotes.rst b/external/llvm-project/lld/docs/ReleaseNotes.rst
index 5c180fd8fbee..064ed0828c31 100644
--- a/external/llvm-project/lld/docs/ReleaseNotes.rst
+++ b/external/llvm-project/lld/docs/ReleaseNotes.rst
@@ -25,6 +25,10 @@ Non-comprehensive list of changes in this release
 
 ELF Improvements
 ----------------
+* Added ``-z dynamic-undefined-weak`` to make undefined weak symbols dynamic
+  when the dynamic symbol table is present.
+  (`#143831 <https://github.com/llvm/llvm-project/pull/143831>`_)
+
 * For AArch64, added support for ``-zgcs-report-dynamic``, enabling checks for
   GNU GCS Attribute Flags in Dynamic Objects when GCS is enabled. Inherits value
   from ``-zgcs-report`` (capped at ``warning`` level) unless user-defined,
diff --git a/external/llvm-project/lld/docs/ld.lld.1 b/external/llvm-project/lld/docs/ld.lld.1
index 86c821bc775e..f59be56b5cba 100644
--- a/external/llvm-project/lld/docs/ld.lld.1
+++ b/external/llvm-project/lld/docs/ld.lld.1
@@ -793,6 +793,14 @@ Specify how to report the missing GNU_PROPERTY_X86_FEATURE_1_IBT or GNU_PROPERTY
 .Cm none
 is the default, linker will not report the missing property otherwise will be reported as a warning or an error.
 .Pp
+.It Cm dynamic-undefined-weak
+Make undefined weak symbols dynamic when the dynamic symbol table is present, if they are referenced from
+relocatable object files and not forced local by symbol visibility or versioning. Do not make them dynamic when
+.Cm nodynamic-undefined-weak
+is specified.
+.Cm dynamic-undefined-weak
+is the default when building a shared object, or when an input shared object is present.
+.Pp
 .It Cm pauth-report Ns = Ns Ar [none|warning|error]
 Specify how to report the missing GNU_PROPERTY_AARCH64_FEATURE_PAUTH property.
 .Cm none
diff --git a/external/llvm-project/lld/include/lld/Common/BPSectionOrdererBase.inc b/external/llvm-project/lld/include/lld/Common/BPSectionOrdererBase.inc
index 51dfb6471644..bb2e55af1eb3 100644
--- a/external/llvm-project/lld/include/lld/Common/BPSectionOrdererBase.inc
+++ b/external/llvm-project/lld/include/lld/Common/BPSectionOrdererBase.inc
@@ -20,6 +20,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Utils.h"
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
@@ -147,19 +148,6 @@ static SmallVector<std::pair<unsigned, UtilityNodes>> getUnsForCompression(
   return sectionUns;
 }
 
-/// Symbols can be appended with "(.__uniq.xxxx)?(.llvm.yyyy)?(.Tgm)?" where
-/// "xxxx" and "yyyy" are numbers that could change between builds, and .Tgm is
-/// the global merge functions suffix
-/// (see GlobalMergeFunc::MergingInstanceSuffix). We need to use the root symbol
-/// name before this suffix so these symbols can be matched with profiles which
-/// may have different suffixes.
-inline StringRef getRootSymbol(StringRef name) {
-  name.consume_back(".Tgm");
-  auto [P0, S0] = name.rsplit(".llvm.");
-  auto [P1, S1] = P0.rsplit(".__uniq.");
-  return P1;
-}
-
 template <class D>
 auto BPOrderer<D>::computeOrder(
     StringRef profilePath, bool forFunctionCompression, bool forDataCompression,
@@ -197,7 +185,7 @@ auto BPOrderer<D>::computeOrder(
       for (size_t timestamp = 0; timestamp < trace.size(); timestamp++) {
         auto [_, parsedFuncName] = getParsedIRPGOName(
             reader->getSymtab().getFuncOrVarName(trace[timestamp]));
-        parsedFuncName = getRootSymbol(parsedFuncName);
+        parsedFuncName = lld::utils::getRootSymbol(parsedFuncName);
 
         auto sectionIdxsIt =
             rootSymbolToSectionIdxs.find(CachedHashStringRef(parsedFuncName));
@@ -375,7 +363,7 @@ auto BPOrderer<D>::computeOrder(
           // 4?
           uint64_t lastPage = endAddress / pageSize;
           StringRef rootSymbol = D::getSymName(*sym);
-          rootSymbol = getRootSymbol(rootSymbol);
+          rootSymbol = lld::utils::getRootSymbol(rootSymbol);
           symbolToPageNumbers.try_emplace(rootSymbol, firstPage, lastPage);
           if (auto resolvedLinkageName = D::getResolvedLinkageName(rootSymbol))
             symbolToPageNumbers.try_emplace(resolvedLinkageName.value(),
@@ -393,7 +381,7 @@ auto BPOrderer<D>::computeOrder(
           auto traceId = trace.FunctionNameRefs[step];
           auto [Filename, ParsedFuncName] =
               getParsedIRPGOName(reader->getSymtab().getFuncOrVarName(traceId));
-          ParsedFuncName = getRootSymbol(ParsedFuncName);
+          ParsedFuncName = lld::utils::getRootSymbol(ParsedFuncName);
           auto it = symbolToPageNumbers.find(ParsedFuncName);
           if (it != symbolToPageNumbers.end()) {
             auto &[firstPage, lastPage] = it->getValue();
diff --git a/external/llvm-project/lld/include/lld/Common/Utils.h b/external/llvm-project/lld/include/lld/Common/Utils.h
new file mode 100644
index 000000000000..86bcfa13d00f
--- /dev/null
+++ b/external/llvm-project/lld/include/lld/Common/Utils.h
@@ -0,0 +1,30 @@
+//===- Utils.h ------------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//===----------------------------------------------------------------------===//
+//
+// The file declares utils functions that can be shared across archs.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_UTILS_H
+#define LLD_UTILS_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace lld {
+namespace utils {
+
+/// Symbols can be appended with "(.__uniq.xxxx)?(.llvm.yyyy)?(.Tgm)?" where
+/// "xxxx" and "yyyy" are numbers that could change between builds, and .Tgm is
+/// the global merge functions suffix
+/// (see GlobalMergeFunc::MergingInstanceSuffix). We need to use the root symbol
+/// name before this suffix so these symbols can be matched with profiles which
+/// may have different suffixes.
+llvm::StringRef getRootSymbol(llvm::StringRef Name);
+} // namespace utils
+} // namespace lld
+
+#endif
diff --git a/external/llvm-project/lld/test/COFF/lto-late-arm.ll b/external/llvm-project/lld/test/COFF/lto-late-arm.ll
index 0e2f148ef74c..1070fc52a013 100644
--- a/external/llvm-project/lld/test/COFF/lto-late-arm.ll
+++ b/external/llvm-project/lld/test/COFF/lto-late-arm.ll
@@ -2,10 +2,11 @@
 
 ;; A bitcode file can generate undefined references to symbols that weren't
 ;; listed as undefined on the bitcode file itself, when lowering produces
-;; calls to e.g. builtin helper functions. If these functions are provided
-;; as LTO bitcode, the linker would hit an unhandled state. (In practice,
-;; compiler-rt builtins are always compiled with -fno-lto, so this shouldn't
-;; happen.)
+;; calls to e.g. builtin helper functions. Ideally all those functions are
+;; listed by lto::LTO::getRuntimeLibcallSymbols(), then we successfully
+;; can link cases when the helper functions are provided as bitcode too.
+;; (In practice, compiler-rt builtins are always compiled with -fno-lto, so
+;; this shouldn't really happen anyway.)
 
 ; RUN: rm -rf %t.dir
 ; RUN: split-file %s %t.dir
@@ -13,9 +14,7 @@
 ; RUN: llvm-as %t.dir/sdiv.ll -o %t.sdiv.obj
 ; RUN: llvm-ar rcs %t.sdiv.lib %t.sdiv.obj
 
-; RUN: env LLD_IN_TEST=1 not lld-link /entry:entry %t.main.obj %t.sdiv.lib /out:%t.exe /subsystem:console 2>&1 | FileCheck %s
-
-; CHECK: error: LTO object file lto-late-arm.ll.tmp.sdiv.lib(lto-late-arm.ll.tmp.sdiv.obj) linked in after doing LTO compilation.
+; RUN: lld-link /entry:entry %t.main.obj %t.sdiv.lib /out:%t.exe /subsystem:console
 
 ;--- main.ll
 target datalayout = "e-m:w-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
diff --git a/external/llvm-project/lld/test/COFF/pdata-arm64ec.test b/external/llvm-project/lld/test/COFF/pdata-arm64ec.test
index cf59330b2354..6bdcc5c5682b 100644
--- a/external/llvm-project/lld/test/COFF/pdata-arm64ec.test
+++ b/external/llvm-project/lld/test/COFF/pdata-arm64ec.test
@@ -80,10 +80,63 @@ DIR3-NEXT:     ExtraRFETableSize: 0x10
 DIR3:        ]
 DIR3:      }
 
+arm64x with no x86 .pdata:
+
 RUN: llvm-objdump -s --section=.pdata test4.dll | FileCheck -check-prefix=DATA4 %s
 DATA4: 180006000 00100000 11000001 00200000 11000001  ......... ......
 DATA4: 180006010 00300000 0e300000
 
+RUN: lld-link -out:testx2.dll -machine:arm64x arm64-func-sym.obj arm64ec-func-sym.obj \
+RUN:          loadconfig-arm64.obj loadconfig-arm64ec.obj -dll -noentry
+
+RUN: llvm-readobj --headers --coff-load-config testx2.dll | FileCheck -check-prefix=DIR4 %s
+DIR4:      ImageOptionalHeader {
+DIR4:        DataDirectory {
+DIR4:          ExceptionTableRVA: 0x5000
+DIR4-NEXT:     ExceptionTableSize: 0x10
+DIR4:        }
+DIR4:      }
+DIR4:      CHPEMetadata [
+DIR4:        ExtraRFETable: 0x0
+DIR4-NEXT:   ExtraRFETableSize: 0x0
+DIR4:      ]
+DIR4:      HybridObject {
+DIR4:        ImageOptionalHeader {
+DIR4:          ExceptionTableRVA: 0x0
+DIR4-NEXT:     ExceptionTableSize: 0x0
+DIR4:        }
+DIR4:        CHPEMetadata [
+DIR4:          ExtraRFETable: 0x5000
+DIR4-NEXT:     ExtraRFETableSize: 0x10
+DIR4:        ]
+DIR4:      }
+
+arm64x with no ARM .pdata:
+
+RUN: lld-link -out:testx3.dll -machine:arm64x x86_64-func-sym.obj loadconfig-arm64.obj loadconfig-arm64ec.obj -dll -noentry
+
+RUN: llvm-readobj --headers --coff-load-config testx3.dll | FileCheck -check-prefix=DIR5 %s
+DIR5:      ImageOptionalHeader {
+DIR5:        DataDirectory {
+DIR5:          ExceptionTableRVA: 0x0
+DIR5-NEXT:     ExceptionTableSize: 0x0
+DIR5:        }
+DIR5:      }
+DIR5:      CHPEMetadata [
+DIR5:        ExtraRFETable: 0x4000
+DIR5-NEXT:   ExtraRFETableSize: 0xC
+DIR5:      ]
+DIR5:      HybridObject {
+DIR5:        ImageOptionalHeader {
+DIR5:          ExceptionTableRVA: 0x4000
+DIR5-NEXT:     ExceptionTableSize: 0xC
+DIR5:        }
+DIR5:        CHPEMetadata [
+DIR5:          ExtraRFETable: 0x0
+DIR5-NEXT:     ExtraRFETableSize: 0x0
+DIR5:        ]
+DIR5:      }
+
 Order of inputs doesn't matter, the data is sorted by type and RVA:
 
 RUN: lld-link -out:test5.dll -machine:arm64ec x86_64-func-sym.obj arm64ec-func-sym.obj \
diff --git a/external/llvm-project/lld/test/ELF/aarch64-feature-pauth.s b/external/llvm-project/lld/test/ELF/aarch64-feature-pauth.s
index bc58f69d32f2..e8c900b9cb13 100644
--- a/external/llvm-project/lld/test/ELF/aarch64-feature-pauth.s
+++ b/external/llvm-project/lld/test/ELF/aarch64-feature-pauth.s
@@ -13,8 +13,12 @@
 # RUN: not ld.lld tag1.o tag1a.o tag2.o -o /dev/null 2>&1 | FileCheck --check-prefix ERR1 %s
 
 # ERR1:      error: incompatible values of AArch64 PAuth core info found
-# ERR1-NEXT: >>> tag1.o: 0x2a000000000000000{{1|2}}00000000000000
-# ERR1-NEXT: >>> tag2.o: 0x2a000000000000000{{1|2}}00000000000000
+# ERR1-NEXT: platform:
+# ERR1-NEXT: >>> tag1.o: 0x2a
+# ERR1-NEXT: >>> tag2.o: 0x2a
+# ERR1-NEXT: version:
+# ERR1-NEXT: >>> tag1.o: 0x01
+# ERR1-NEXT: >>> tag2.o: 0x02
 
 # RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu abi-tag-short.s -o short.o
 # RUN: not ld.lld short.o -o /dev/null 2>&1 | FileCheck --check-prefix ERR2 %s
diff --git a/external/llvm-project/lld/test/ELF/driver.test b/external/llvm-project/lld/test/ELF/driver.test
index 45d73607c8ac..6d5761212cc3 100644
--- a/external/llvm-project/lld/test/ELF/driver.test
+++ b/external/llvm-project/lld/test/ELF/driver.test
@@ -47,7 +47,8 @@
 # ERR9: error: cannot open output file utput=/no/such/file
 
 # RUN: ld.lld %t -z foo -o /dev/null 2>&1 | FileCheck -check-prefix=ERR10 %s --implicit-check-not=warning:
-# RUN: ld.lld %t -z foo -z rel -z rela -z max-page-size=1 -z common-page-size=1 -o /dev/null --version 2>&1 | \
+# RUN: ld.lld %t -z foo -z rel -z rela -z max-page-size=1 -z common-page-size=1 -z dynamic-undefined-weak \
+# RUN:   -z nodynamic-undefined-weak -o /dev/null --version 2>&1 | \
 # RUN:   FileCheck -check-prefix=ERR10 %s --implicit-check-not=warning:
 # ERR10: warning: unknown -z value: foo
 
diff --git a/external/llvm-project/lld/test/ELF/ppc64-long-branch-pi.s b/external/llvm-project/lld/test/ELF/ppc64-long-branch-pi.s
index 4c8c4cebf274..f97eda9632fc 100644
--- a/external/llvm-project/lld/test/ELF/ppc64-long-branch-pi.s
+++ b/external/llvm-project/lld/test/ELF/ppc64-long-branch-pi.s
@@ -15,19 +15,20 @@
 # RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck %s
 
 # SEC-PIE:    Name       Type     Address          Off     Size   ES Flg Lk Inf Al
-# SEC-PIE:    .got       PROGBITS 0000000002002110 2012110 000008 00  WA  0   0  8
-# SEC-PIE:    .branch_lt NOBITS   0000000002002120 2012120 000020 00  WA  0   0  8
+# SEC-PIE:    .got       PROGBITS 0000000002002120 2012120 000010 00  WA  0   0  8
+# SEC-PIE:    .branch_lt NOBITS   0000000002002138 2012138 000020 00  WA  0   0  8
 
 # SEC-SHARED: Name       Type     Address          Off     Size   ES Flg Lk Inf Al
-# SEC-SHARED: .got       PROGBITS 00000000020020f0 20120f0 000008 00  WA  0   0  8
-# SEC-SHARED: .branch_lt NOBITS   0000000002002100 2012100 000020 00  WA  0   0  8
+# SEC-SHARED: .got       PROGBITS 0000000002002100 2012100 000010 00  WA  0   0  8
+# SEC-SHARED: .branch_lt NOBITS   0000000002002118 2012118 000020 00  WA  0   0  8
 
 # RELOC:      .rela.dyn {
-# RELOC-NEXT:   0x2002118 R_PPC64_RELATIVE - 0x2012110
-# RELOC-NEXT:   0x2002120 R_PPC64_RELATIVE - 0x2002000
-# RELOC-NEXT:   0x2002128 R_PPC64_RELATIVE - 0x2002008
-# RELOC-NEXT:   0x2002130 R_PPC64_RELATIVE - 0x200200C
-# RELOC-NEXT:   0x2002138 R_PPC64_RELATIVE - 0x2000
+# RELOC-NEXT:   0x2002128 R_PPC64_RELATIVE - 0x2002000
+# RELOC-NEXT:   0x2002130 R_PPC64_RELATIVE - 0x2012120
+# RELOC-NEXT:   0x2002138 R_PPC64_RELATIVE - 0x2002000
+# RELOC-NEXT:   0x2002140 R_PPC64_RELATIVE - 0x2002008
+# RELOC-NEXT:   0x2002148 R_PPC64_RELATIVE - 0x200200C
+# RELOC-NEXT:   0x2002150 R_PPC64_RELATIVE - 0x2000
 # RELOC-NEXT: }
 
 # CHECK:      <_start>:
@@ -36,24 +37,24 @@
 # CHECK-NEXT:                 bl 0x2030
 # CHECK-NEXT:                 bl 0x2050
 
-## &.branch_lt[0] - .TOC. = .branch_lt - (.got+0x8000) = -32752
+## &.branch_lt[0] - .TOC. = .branch_lt - (.got+0x8000) = -32744
 # CHECK:      <__long_branch_>:
 # CHECK-NEXT:     2010:       addis 12, 2, 0
-# CHECK-NEXT:                 ld 12, -32752(12)
+# CHECK-NEXT:                 ld 12, -32744(12)
 # CHECK-NEXT:                 mtctr 12
 # CHECK-NEXT:                 bctr
 
-## &.branch_lt[1] - .TOC. = .branch_lt - (.got+0x8000) = -32744
+## &.branch_lt[1] - .TOC. = .branch_lt - (.got+0x8000) = -32736
 # CHECK:      <__long_branch_>:
 # CHECK-NEXT:     2030:       addis 12, 2, 0
-# CHECK-NEXT:                 ld 12, -32744(12)
+# CHECK-NEXT:                 ld 12, -32736(12)
 # CHECK-NEXT:                 mtctr 12
 # CHECK-NEXT:                 bctr
 
-## &.branch_lt[2] - .TOC. = .branch_lt - (.got+0x8000) = -32736
+## &.branch_lt[2] - .TOC. = .branch_lt - (.got+0x8000) = -32728
 # CHECK:      <__long_branch_>:
 # CHECK-NEXT:     2050:       addis 12, 2, 0
-# CHECK-NEXT:                 ld 12, -32736(12)
+# CHECK-NEXT:                 ld 12, -32728(12)
 # CHECK-NEXT:                 mtctr 12
 # CHECK-NEXT:                 bctr
 
@@ -68,12 +69,12 @@ bl .text_high+0xc # Need a thunk
 # CHECK:      <high_target>:
 # CHECK-NEXT:  2002000:   bl 0x2004
 # CHECK-NEXT:             bl 0x2004
-# CHECK-NEXT:             bl 0x2002010
+# CHECK-NEXT:             bl 0x2002020
 
-## &.branch_lt[3] - .TOC. = .branch_lt - (.got+0x8000) = -32728
+## &.branch_lt[3] - .TOC. = .branch_lt - (.got+0x8000) = -32720
 # CHECK:      <__long_branch_>:
-# CHECK-NEXT:  2002010:       addis 12, 2, 0
-# CHECK-NEXT:                 ld 12, -32728(12)
+# CHECK-NEXT:  2002020:       addis 12, 2, 0
+# CHECK-NEXT:                 ld 12, -32720(12)
 # CHECK-NEXT:                 mtctr 12
 # CHECK-NEXT:                 bctr
 
@@ -85,6 +86,8 @@ bl .text_low      # Need a thunk
 blr
 
 ## Force creation of .got
+addis 3, 2, .text_high@got@ha
+
 ## The R_PPC64_RELATIVE makes sure .rela.dyn survives removeUnusedSyntheticSections.
 .section .data
 .quad .TOC.@tocbase
diff --git a/external/llvm-project/lld/test/ELF/ppc64-relocs.s b/external/llvm-project/lld/test/ELF/ppc64-relocs.s
index 0b0be487521e..97018ddd25d4 100644
--- a/external/llvm-project/lld/test/ELF/ppc64-relocs.s
+++ b/external/llvm-project/lld/test/ELF/ppc64-relocs.s
@@ -2,12 +2,12 @@
 
 # RUN: llvm-mc -filetype=obj -triple=powerpc64le-unknown-linux %s -o %t.o
 # RUN: ld.lld --no-toc-optimize %t.o -o %t
-# RUN: llvm-readelf -x .rodata -x .R_PPC64_TOC -x .eh_frame %t | FileCheck %s --check-prefix=DATALE
+# RUN: llvm-readelf -x .rodata -x .R_PPC64_TOC %t | FileCheck %s --check-prefix=DATALE
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s
 
 # RUN: llvm-mc -filetype=obj -triple=powerpc64-unknown-linux %s -o %t.o
 # RUN: ld.lld --no-toc-optimize %t.o -o %t
-# RUN: llvm-readelf -x .rodata -x .R_PPC64_TOC -x .eh_frame %t | FileCheck %s --check-prefix=DATABE
+# RUN: llvm-readelf -x .rodata -x .R_PPC64_TOC %t | FileCheck %s --check-prefix=DATABE
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s
 
 .text
diff --git a/external/llvm-project/lld/test/ELF/riscv-branch.s b/external/llvm-project/lld/test/ELF/riscv-branch.s
index 1a2b446b5a43..5f73fd19f9aa 100644
--- a/external/llvm-project/lld/test/ELF/riscv-branch.s
+++ b/external/llvm-project/lld/test/ELF/riscv-branch.s
@@ -1,7 +1,7 @@
 # REQUIRES: riscv
 
-# RUN: llvm-mc -filetype=obj -triple=riscv32-unknown-elf -mattr=-relax -riscv-asm-relax-branches=0 %s -o %t.rv32.o
-# RUN: llvm-mc -filetype=obj -triple=riscv64-unknown-elf -mattr=-relax -riscv-asm-relax-branches=0 %s -o %t.rv64.o
+# RUN: llvm-mc -filetype=obj -triple=riscv32-unknown-elf %s -o %t.rv32.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64-unknown-elf %s -o %t.rv64.o
 
 # RUN: ld.lld %t.rv32.o --defsym foo=_start+4 --defsym bar=_start -o %t.rv32
 # RUN: ld.lld %t.rv64.o --defsym foo=_start+4 --defsym bar=_start -o %t.rv64
@@ -30,6 +30,8 @@
 # RUN: not ld.lld %t.rv64.o --defsym foo=_start+1 --defsym bar=_start-1 -o /dev/null 2>&1 | FileCheck --check-prefix=ERROR-ALIGN %s
 # ERROR-ALIGN: improper alignment for relocation R_RISCV_BRANCH: 0x1 is not aligned to 2 bytes
 
+.option exact
+
 .global _start
 _start:
      beq x0, x0, foo
diff --git a/external/llvm-project/lld/test/ELF/riscv-feature-zicfilp-func-sig.s b/external/llvm-project/lld/test/ELF/riscv-feature-zicfilp-func-sig.s
new file mode 100644
index 000000000000..c5818dd33978
--- /dev/null
+++ b/external/llvm-project/lld/test/ELF/riscv-feature-zicfilp-func-sig.s
@@ -0,0 +1,236 @@
+# REQUIRES: riscv
+## Test the ZICFILP func-sig feature.
+## To lift maintenance burden, most tests are conducted only with 64-bit RISC-V
+## Naming convention: *-s.s files enables ZICFILP func-sig.
+## Naming convention: *-u.s files enables ZICFILP unlabeled.
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f1-s.s -o rv32-f1-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f2-s.s -o rv32-f2-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f3-s.s -o rv32-f3-s.o
+
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f1-s.s -o f1-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f2.s   -o f2.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f2-s.s -o f2-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3.s   -o f3.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3-s.s -o f3-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3-u.s -o f3-u.o
+
+## ZICFILP-func-sig should be enabled when it's enabled in all inputs or when
+## it's forced on.
+# RUN: ld.lld rv32-f1-s.o rv32-f2-s.o rv32-f3-s.o -o out.rv32 --fatal-warnings
+# RUN: llvm-readelf -n out.rv32 | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -o out --fatal-warnings
+# RUN: llvm-readelf -n out | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f1-s.o f3-s.o --shared -o out.so --fatal-warnings
+# RUN: llvm-readelf -n out.so | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -o out.force -z zicfilp=func-sig --fatal-warnings
+# RUN: llvm-readelf -n out.force | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f2-s.o f3.o --shared -o out.force.so -z zicfilp=never -z zicfilp=func-sig --fatal-warnings
+# RUN: llvm-readelf -n out.force.so | FileCheck --check-prefix=ZICFILP %s
+# ZICFILP: Properties: RISC-V feature: ZICFILP-func-sig
+
+## ZICFILP-func-sig should not be enabled if it's not enabled in at least one
+## input
+# RUN: ld.lld f1-s.o f2.o f3-s.o -o out.no --fatal-warnings
+# RUN: llvm-readelf -n out.no | count 0
+# RUN: ld.lld f2-s.o f3.o --shared -o out.no.so --fatal-warnings
+# RUN: llvm-readelf -n out.no.so | count 0
+
+## ZICFILP-func-sig should be disabled with zicfilp=never, even if
+## ZICFILP-func-sig is present in all inputs.
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -z zicfilp=func-sig -z zicfilp=never -o out.never --fatal-warnings
+# RUN: llvm-readelf -n out.never | count 0
+
+## zicfilp-func-sig-report should report any input files that don't have the
+## ZICFILP-func-sig property
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfilp-func-sig-report=warning 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfilp-func-sig-report=warning -z zicfilp=func-sig 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfilp-func-sig-report=warning -z zicfilp=never 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: not ld.lld f2-s.o f3.o --shared -z zicfilp-func-sig-report=error 2>&1 | FileCheck --check-prefix=REPORT-ERROR %s
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -z zicfilp-func-sig-report=warning -z zicfilp=func-sig 2>&1 | count 0
+# REPORT-WARN: warning: f2.o: -z zicfilp-func-sig-report: file does not have GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG property
+# REPORT-ERROR: error: f3.o: -z zicfilp-func-sig-report: file does not have GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG property
+
+## An invalid -z zicfilp-func-sig-report option should give an error
+# RUN: not ld.lld f2-s.o -z zicfilp-func-sig-report=x 2>&1 | FileCheck --check-prefix=INVALID %s
+# INVALID: error: unknown -z zicfilp-func-sig-report= value: x
+
+## ZICFILP-unlabeled and ZICFILP-func-sig should conflict with each other.
+# RUN: ld.lld f3-u.o -o out.override -z zicfilp=func-sig 2>&1 | FileCheck --check-prefix=FORCE-CONFLICT %s
+# FORCE-CONFLICT: warning: f3-u.o: -z zicfilp=func-sig: file has conflicting property: GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+
+## -z zicfilp=func-sig should override and disable ZICFILP-unlabeled.
+# RUN: llvm-readelf -n out.override | FileCheck --check-prefixes=ZICFILP,OVERRIDE %s
+# OVERRIDE-NOT: ZICFILP-unlabeled
+
+#--- rv32-f1-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 4          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 4
+ndesc_end:
+
+.text
+.globl _start
+.type f1,%function
+f1:
+  call f2
+  ret
+
+#--- f1-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 4          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 8
+ndesc_end:
+
+.text
+.globl _start
+.type f1,%function
+f1:
+  call f2
+  ret
+
+#--- f2.s
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- rv32-f2-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 4          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 4
+ndesc_end:
+
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- f2-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 4          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 8
+ndesc_end:
+
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- f3.s
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- rv32-f3-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 4          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 4
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- f3-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 4          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 8
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- f3-u.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 1          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+.balign 8
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
diff --git a/external/llvm-project/lld/test/ELF/riscv-feature-zicfilp-unlabeled.s b/external/llvm-project/lld/test/ELF/riscv-feature-zicfilp-unlabeled.s
new file mode 100644
index 000000000000..20491f057c8e
--- /dev/null
+++ b/external/llvm-project/lld/test/ELF/riscv-feature-zicfilp-unlabeled.s
@@ -0,0 +1,263 @@
+# REQUIRES: riscv
+## Test the ZICFILP unlabeled feature.
+## To lift maintenance burden, most tests are conducted only with 64-bit RISC-V
+## Naming convention: *-s.s files enables ZICFILP unlabeled.
+## Naming convention: *-f.s files enables ZICFILP func-sig.
+## Naming convention: *-c.s files enables both of the conflicting ZICFILP unlabeled and ZICFILP func-sig features.
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f1-s.s -o rv32-f1-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f2-s.s -o rv32-f2-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f3-s.s -o rv32-f3-s.o
+
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f1-s.s -o f1-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f1-c.s -o f1-c.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f2.s   -o f2.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f2-s.s -o f2-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3.s   -o f3.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3-s.s -o f3-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3-f.s -o f3-f.o
+
+## ZICFILP-unlabeled should be enabled when it's enabled in all inputs or when
+## it's forced on.
+# RUN: ld.lld rv32-f1-s.o rv32-f2-s.o rv32-f3-s.o -o out.rv32 --fatal-warnings
+# RUN: llvm-readelf -n out.rv32 | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -o out --fatal-warnings
+# RUN: llvm-readelf -n out | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f1-s.o f3-s.o --shared -o out.so --fatal-warnings
+# RUN: llvm-readelf -n out.so | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -o out.force -z zicfilp=unlabeled --fatal-warnings
+# RUN: llvm-readelf -n out.force | FileCheck --check-prefix=ZICFILP %s
+# RUN: ld.lld f2-s.o f3.o --shared -o out.force.so -z zicfilp=never -z zicfilp=unlabeled --fatal-warnings
+# RUN: llvm-readelf -n out.force.so | FileCheck --check-prefix=ZICFILP %s
+# ZICFILP: Properties: RISC-V feature: ZICFILP-unlabeled
+
+## ZICFILP-unlabeled should not be enabled if it's not enabled in at least one
+## input
+# RUN: ld.lld f1-s.o f2.o f3-s.o -o out.no --fatal-warnings
+# RUN: llvm-readelf -n out.no | count 0
+# RUN: ld.lld f2-s.o f3.o --shared -o out.no.so --fatal-warnings
+# RUN: llvm-readelf -n out.no.so | count 0
+
+## ZICFILP-unlabeled should be disabled with zicfilp=never, even if
+## ZICFILP-unlabeled is present in all inputs.
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -z zicfilp=unlabeled -z zicfilp=never -o out.never --fatal-warnings
+# RUN: llvm-readelf -n out.never | count 0
+
+## zicfilp-unlabeled-report should report any input files that don't have the
+## ZICFILP-unlabeled property
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfilp-unlabeled-report=warning 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfilp-unlabeled-report=warning -z zicfilp=unlabeled 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfilp-unlabeled-report=warning -z zicfilp=never 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: not ld.lld f2-s.o f3.o --shared -z zicfilp-unlabeled-report=error 2>&1 | FileCheck --check-prefix=REPORT-ERROR %s
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -z zicfilp-unlabeled-report=warning -z zicfilp=never 2>&1 | count 0
+# REPORT-WARN: warning: f2.o: -z zicfilp-unlabeled-report: file does not have GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED property
+# REPORT-ERROR: error: f3.o: -z zicfilp-unlabeled-report: file does not have GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED property
+
+## An invalid -z zicfilp-unlabeled-report option should give an error
+# RUN: not ld.lld f2-s.o -z zicfilp=x -z zicfilp-unlabeled-report=x 2>&1 | FileCheck --check-prefix=INVALID %s
+# INVALID: error: unknown -z zicfilp= value: x
+# INVALID: error: unknown -z zicfilp-unlabeled-report= value: x
+
+## ZICFILP-unlabeled and ZICFILP-func-sig should conflict with each other
+# RUN: not ld.lld f1-c.o 2>&1 | FileCheck --check-prefix=CONFLICT %s
+# RUN: ld.lld f3-f.o -o out.override -z zicfilp=unlabeled 2>&1 | FileCheck --check-prefix=FORCE-CONFLICT %s
+# CONFLICT: error: f1-c.o: file has conflicting properties: GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED and GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+# FORCE-CONFLICT: warning: f3-f.o: -z zicfilp=unlabeled: file has conflicting property: GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+
+## -z zicfilp=unlabeled should override and disable ZICFILP-func-sig.
+# RUN: llvm-readelf -n out.override | FileCheck --check-prefixes=ZICFILP,OVERRIDE %s
+# OVERRIDE-NOT: ZICFILP-func-sig
+
+#--- rv32-f1-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 1          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+.balign 4
+ndesc_end:
+
+.text
+.globl _start
+.type f1,%function
+f1:
+  call f2
+  ret
+
+#--- f1-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 1          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+.balign 8
+ndesc_end:
+
+.text
+.globl _start
+.type f1,%function
+f1:
+  call f2
+  ret
+
+#--- f1-c.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 5          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED | GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 8
+ndesc_end:
+
+.text
+.globl _start
+.type f1,%function
+f1:
+  call f2
+  ret
+
+#--- f2.s
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- rv32-f2-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 1          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+.balign 4
+ndesc_end:
+
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- f2-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 1          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+.balign 8
+ndesc_end:
+
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- f3.s
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- rv32-f3-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 1          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+.balign 4
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- f3-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 1          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_UNLABELED
+.balign 8
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- f3-f.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 4          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_LP_FUNC_SIG
+.balign 8
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
diff --git a/external/llvm-project/lld/test/ELF/riscv-feature-zicfiss.s b/external/llvm-project/lld/test/ELF/riscv-feature-zicfiss.s
new file mode 100644
index 000000000000..7b208ddd9b8e
--- /dev/null
+++ b/external/llvm-project/lld/test/ELF/riscv-feature-zicfiss.s
@@ -0,0 +1,205 @@
+# REQUIRES: riscv
+## Test the ZICFISS feature.
+## To lift maintenance burden, most tests are conducted only with 64-bit RISC-V
+## Naming convention: *-s.s files enable ZICFISS.
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f1-s.s -o rv32-f1-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f2-s.s -o rv32-f2-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv32 rv32-f3-s.s -o rv32-f3-s.o
+
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f1-s.s -o f1-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f2.s   -o f2.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f2-s.s -o f2-s.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3.s   -o f3.o
+# RUN: llvm-mc --filetype=obj --triple=riscv64 f3-s.s -o f3-s.o
+
+## ZICFISS should be enabled when it's enabled in all inputs or when it's forced on.
+# RUN: ld.lld rv32-f1-s.o rv32-f2-s.o rv32-f3-s.o -o out.rv32 --fatal-warnings
+# RUN: llvm-readelf -n out.rv32 | FileCheck --check-prefix=ZICFISS %s
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -o out --fatal-warnings
+# RUN: llvm-readelf -n out | FileCheck --check-prefix=ZICFISS %s
+# RUN: ld.lld f1-s.o f3-s.o --shared -o out.so --fatal-warnings
+# RUN: llvm-readelf -n out.so | FileCheck --check-prefix=ZICFISS %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -o out.force -z zicfiss=always --fatal-warnings
+# RUN: llvm-readelf -n out.force | FileCheck --check-prefix=ZICFISS %s
+# RUN: ld.lld f2-s.o f3.o --shared -o out.force.so -z zicfiss=never -z zicfiss=always --fatal-warnings
+# RUN: llvm-readelf -n out.force.so | FileCheck --check-prefix=ZICFISS %s
+# ZICFISS: Properties: RISC-V feature: ZICFISS
+
+## ZICFISS should not be enabled if it's not enabled in at least one input
+# RUN: ld.lld f1-s.o f2.o f3-s.o -o out.no --fatal-warnings
+# RUN: llvm-readelf -n out.no | count 0
+# RUN: ld.lld f2-s.o f3.o --shared -o out.no.so --fatal-warnings
+# RUN: llvm-readelf -n out.no.so | count 0
+
+## ZICFISS should be disabled with zicfiss=never, even if ZICFISS is present in
+## all inputs.
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -z zicfiss=always -z zicfiss=never -o out.never --fatal-warnings
+# RUN: llvm-readelf -n out.never | count 0
+
+## zicfiss-report should report any input files that don't have the zicfiss
+## property
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfiss-report=warning 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfiss-report=warning -z zicfiss=always 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: ld.lld f1-s.o f2.o f3-s.o -z zicfiss-report=warning -z zicfiss=never 2>&1 | FileCheck --check-prefix=REPORT-WARN %s
+# RUN: not ld.lld f2-s.o f3.o --shared -z zicfiss-report=error 2>&1 | FileCheck --check-prefix=REPORT-ERROR %s
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -z zicfiss-report=warning -z zicfiss=always 2>&1 | count 0
+# RUN: ld.lld f1-s.o f2-s.o f3-s.o -z zicfiss-report=error -z zicfiss=always 2>&1 | count 0
+# REPORT-WARN: warning: f2.o: -z zicfiss-report: file does not have GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS property
+# REPORT-ERROR: error: f3.o: -z zicfiss-report: file does not have GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS property
+
+## An invalid -z zicfiss-report option should give an error
+# RUN: not ld.lld f2-s.o f3-s.o -z zicfiss=x -z zicfiss-report=x 2>&1 | FileCheck --check-prefix=INVALID %s
+# INVALID: error: unknown -z zicfiss= value: x
+# INVALID: error: unknown -z zicfiss-report= value: x
+
+#--- rv32-f1-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 2          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS
+.balign 4
+ndesc_end:
+
+.text
+.globl _start
+.type f1,%function
+f1:
+  call f2
+  ret
+
+#--- f1-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 2          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS
+.balign 8
+ndesc_end:
+
+.text
+.globl _start
+.type f1,%function
+f1:
+  call f2
+  ret
+
+#--- f2.s
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- rv32-f2-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 2          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS
+.balign 4
+ndesc_end:
+
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- f2-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 2          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS
+.balign 8
+ndesc_end:
+
+.text
+.globl f2
+.type f2,@function
+f2:
+  .globl f3
+  .type f3, @function
+  call f3
+  ret
+
+#--- f3.s
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- rv32-f3-s.s
+.section ".note.gnu.property", "a"
+.balign 4
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 4
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 2          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS
+.balign 4
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
+
+#--- f3-s.s
+.section ".note.gnu.property", "a"
+.balign 8
+.4byte 4
+.4byte (ndesc_end - ndesc_begin)
+.4byte 0x5        // NT_GNU_PROPERTY_TYPE_0
+.asciz "GNU"
+ndesc_begin:
+.balign 8
+.4byte 0xc0000000 // GNU_PROPERTY_RISCV_FEATURE_1_AND
+.4byte 4
+.4byte 2          // GNU_PROPERTY_RISCV_FEATURE_1_CFI_SS
+.balign 8
+ndesc_end:
+
+.text
+.globl f3
+.type f3,@function
+f3:
+  ret
diff --git a/external/llvm-project/lld/test/ELF/riscv-relax-call-stress.s b/external/llvm-project/lld/test/ELF/riscv-relax-call-stress.s
new file mode 100644
index 000000000000..27b557eddc67
--- /dev/null
+++ b/external/llvm-project/lld/test/ELF/riscv-relax-call-stress.s
@@ -0,0 +1,50 @@
+# REQUIRES: riscv
+## The regression test led to oscillation between two states ("address assignment did not converge" error).
+## First jump (~2^11 bytes away): alternated between 4 and 8 bytes.
+## Second jump (~2^20 bytes away): alternated between 2 and 8 bytes.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax a.s -o a.o
+# RUN: ld.lld -T lds a.o -o out
+# RUN: llvm-objdump -td --no-show-raw-insn -M no-aliases out | FileCheck %s
+
+# CHECK-LABEL: <_start>:
+# CHECK-NEXT:                  jal     zero, {{.*}} <low>
+# CHECK-EMPTY:
+# CHECK-NEXT:  <jump_high>:
+# CHECK-NEXT:      1004:       auipc   t0, 0x100
+# CHECK-NEXT:                  jalr    zero, -0x2(t0) <high>
+# CHECK-NEXT:                  ...
+# CHECK:       <low>:
+# CHECK:           1802:       c.jr    ra
+
+# CHECK:       <high>:
+# CHECK-NEXT:    101002:       jal     ra, 0x1004 <jump_high>
+# CHECK-NEXT:                  auipc   ra, 0xfff00
+# CHECK-NEXT:                  jalr    ra, -0x2(ra) <jump_high>
+
+#--- a.s
+## At the beginning of state1, low-_start = 0x800-2, reachable by a c.j.
+## This increases high-jump_high from 0x100000-2 to 0x100000, unreachable by a jal.
+## In the next iteration, low-_start increases to 0x800, unreachable by a c.j.
+.global _start
+_start:
+  jump low, t0    # state0: 4 bytes; state1: 2 bytes
+jump_high:
+  jump high, t0   # state0: 4 bytes; state1: 8 bytes
+
+  .space 0x800-10
+low:
+  ret
+
+.section .high,"ax",@progbits
+  nop
+high:
+  call jump_high
+  call jump_high
+
+#--- lds
+SECTIONS {
+  .text 0x1000 : { *(.text) }
+  .high 0x101000 : { *(.high) }
+}
diff --git a/external/llvm-project/lld/test/ELF/riscv-undefined-weak.s b/external/llvm-project/lld/test/ELF/riscv-undefined-weak.s
index 57841127792f..c89ac1fe1022 100644
--- a/external/llvm-project/lld/test/ELF/riscv-undefined-weak.s
+++ b/external/llvm-project/lld/test/ELF/riscv-undefined-weak.s
@@ -1,7 +1,7 @@
 # REQUIRES: riscv
 # RUN: llvm-mc -filetype=obj -triple=riscv64 /dev/null -o %t2.o
 # RUN: ld.lld -shared -soname=t2 %t2.o -o %t2.so
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -riscv-asm-relax-branches=0 %s -o %t.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64 %s -o %t.o
 # RUN: llvm-readobj -r %t.o | FileCheck --check-prefix=RELOC %s
 
 # RUN: ld.lld -e absolute %t.o -o %t
@@ -12,6 +12,8 @@
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefixes=CHECK,PLT %s
 # RUN: llvm-readelf -x .data %t | FileCheck --check-prefixes=HEX,HEX-WITH-PLT %s
 
+.option exact
+
 .weak target
 .global absolute, relative, branch
 
diff --git a/external/llvm-project/lld/test/ELF/weak-undef-got-pie.s b/external/llvm-project/lld/test/ELF/weak-undef-got-pie.s
deleted file mode 100644
index 2301400f4e0b..000000000000
--- a/external/llvm-project/lld/test/ELF/weak-undef-got-pie.s
+++ /dev/null
@@ -1,22 +0,0 @@
-# REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %p/Inputs/dummy-shared.s -o %t1.o
-# RUN: ld.lld %t1.o -shared -o %t1.so
-# RUN: llvm-mc -filetype=obj -x86-relax-relocations=false -triple=x86_64 %s -o %t.o
-
-# RUN: ld.lld -pie %t.o %t1.so -o %t
-# RUN: llvm-readobj -r %t | FileCheck --check-prefix=RELOCS %s
-# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=DISASM %s
-
-# RELOCS:      Relocations [
-# RELOCS-NEXT:   Section ({{.*}}) .rela.dyn {
-# RELOCS-NEXT:     R_X86_64_GLOB_DAT foo 0x0
-# RELOCS-NEXT:   }
-# RELOCS-NEXT: ]
-
-.weak foo
-
-.globl _start
-_start:
-# DISASM: <_start>:
-# DISASM-NEXT: movq {{.*}}(%rip), %rax
-mov foo@gotpcrel(%rip), %rax
diff --git a/external/llvm-project/lld/test/ELF/weak-undef-got-plt.s b/external/llvm-project/lld/test/ELF/weak-undef-got-plt.s
new file mode 100644
index 000000000000..48a7914e5b98
--- /dev/null
+++ b/external/llvm-project/lld/test/ELF/weak-undef-got-plt.s
@@ -0,0 +1,40 @@
+# REQUIRES: x86
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 -x86-relax-relocations=false a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %S/Inputs/shared.s -o s.o
+# RUN: ld.lld -shared s.o -o s.so
+
+# RUN: ld.lld a.o -o a
+# RUN: llvm-readelf -r a | FileCheck %s --check-prefix=NORELOC
+# RUN: ld.lld a.o -o a -z dynamic-undefined-weak
+# RUN: llvm-readelf -r a | FileCheck %s --check-prefix=NORELOC
+# RUN: ld.lld a.o s.so -o as
+# RUN: llvm-objdump -dR as | FileCheck %s
+# RUN: ld.lld a.o s.so -o as -z nodynamic-undefined-weak
+# RUN: llvm-readelf -r a | FileCheck %s --check-prefix=NORELOC
+
+# RUN: ld.lld -pie a.o s.so -o as.pie
+# RUN: llvm-objdump -dR as.pie | FileCheck %s
+# RUN: ld.lld -pie a.o s.so -o as.pie -z nodynamic-undefined-weak
+# RUN: llvm-readelf -r as.pie | FileCheck --check-prefix=NORELOC %s
+
+# RUN: ld.lld -shared a.o -o a.so
+# RUN: llvm-objdump -dR a.so | FileCheck %s
+
+# NORELOC:    no relocation
+
+# CHECK:      TYPE                     VALUE
+# CHECK-NEXT: R_X86_64_GLOB_DAT        foo{{$}}
+# CHECK-NEXT: R_X86_64_JUMP_SLOT       foo{{$}}
+# CHECK-EMPTY:
+# CHECK:      <_start>:
+# CHECK-NEXT:   movq {{.*}}(%rip), %rax
+# CHECK-NEXT:   callq {{.*}} <foo@plt>
+
+#--- a.s
+.weak foo
+
+.globl _start
+_start:
+mov foo@gotpcrel(%rip), %rax
+call foo
diff --git a/external/llvm-project/lld/test/ELF/weak-undef-hidden.s b/external/llvm-project/lld/test/ELF/weak-undef-hidden.s
index 2baad5738c36..ad2ba29ec27a 100644
--- a/external/llvm-project/lld/test/ELF/weak-undef-hidden.s
+++ b/external/llvm-project/lld/test/ELF/weak-undef-hidden.s
@@ -5,6 +5,10 @@
 // RUN: ld.lld %t.o -o %t -pie
 // RUN: llvm-readobj -r -S --section-data %t | FileCheck %s
 
+/// -z dynamic-undefined-weak does not affect hidden undefined symbols.
+// RUN: ld.lld %t.o -o %t.so -shared -z dynamic-undefined-weak
+// RUN: llvm-readobj -r -S --section-data %t.so | FileCheck %s
+
 /// This is usually guarded with a comparison. Don't report an error.
 call g
 
diff --git a/external/llvm-project/lld/test/ELF/weak-undef-rw.s b/external/llvm-project/lld/test/ELF/weak-undef-rw.s
index bbc37ba49304..8d777669b7e1 100644
--- a/external/llvm-project/lld/test/ELF/weak-undef-rw.s
+++ b/external/llvm-project/lld/test/ELF/weak-undef-rw.s
@@ -3,51 +3,80 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64 b.s -o b.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64 c.s -o c.o
-# RUN: ld.lld a.o -o nopie --export-dynamic
-# RUN: llvm-readelf -r --hex-dump=.data nopie | FileCheck %s --check-prefix=STATIC
-# RUN: ld.lld a.o -o out.pie -pie
-# RUN: llvm-readelf -r --hex-dump=.data out.pie | FileCheck %s --check-prefix=STATIC
-# RUN: ld.lld a.o -o out.so -shared
-# RUN: llvm-readobj -r out.so | FileCheck %s --check-prefix=PIC
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %S/Inputs/shared.s -o s.o
+# RUN: ld.lld -shared s.o -o s.so
+
+# RUN: ld.lld a.o -o a --export-dynamic
+# RUN: llvm-readelf -r --hex-dump=.data a | FileCheck %s --check-prefix=STATIC
+# RUN: ld.lld a.o s.so -o as
+# RUN: llvm-readelf -r --hex-dump=.data as | FileCheck %s --check-prefix=STATIC
+# RUN: ld.lld a.o -o a.pie -pie
+# RUN: llvm-readelf -r --hex-dump=.data a.pie | FileCheck %s --check-prefix=STATIC
+# RUN: ld.lld a.o -o a.so -shared
+# RUN: llvm-readelf -r a.so | FileCheck %s --check-prefix=DYN
 
 ## gABI leaves the behavior of weak undefined references implementation defined.
 ## We choose to resolve them statically for static linking and produce dynamic relocations
 ## for dynamic linking (-shared or at least one input DSO).
-##
-## Note: Some ports of GNU ld support -z nodynamic-undefined-weak that we don't
-## implement.
+
+## -z dynamic-undefined-weak is ignored if .dynsym is absent (-no-pie without DSO)
+# RUN: ld.lld a.o -o a.d -z dynamic-undefined-weak 2>&1 | count 0
+# RUN: llvm-readelf -r --hex-dump=.data a.d | FileCheck %s --check-prefix=STATIC
+
+## Currently no effect for S+A relocations.
+# RUN: ld.lld a.o s.so -o as.d -z dynamic-undefined-weak
+# RUN: llvm-readelf -r --hex-dump=.data as.d | FileCheck %s --check-prefix=STATIC
+
+## -z dynamic-undefined-weak forces dynamic relocations if .dynsym is present.
+# RUN: ld.lld a.o -o a.pie.d -pie -z dynamic-undefined-weak
+# RUN: llvm-readelf -r a.pie.d | FileCheck %s --check-prefix=DYN
+
+## -z nodynamic-undefined-weak suppresses dynamic relocations.
+# RUN: ld.lld a.o -o a.so.n -shared -z dynamic-undefined-weak -z nodynamic-undefined-weak
+# RUN: llvm-readelf -r --hex-dump=.data a.so.n | FileCheck %s --check-prefix=STATIC
 
 # STATIC:      no relocations
 # STATIC:      Hex dump of section '.data':
-# STATIC-NEXT: {{.*}} 00000000 00000000 .
+# STATIC-NEXT: {{.*}} 00000000 00000000 03000000 00000000 .
 # STATIC-EMPTY:
 
-# PIC:      .rela.dyn {
-# PIC-NEXT:   R_X86_64_64 foobar 0x0
-# PIC-NEXT: }
+# DYN:        Relocation section '.rela.dyn' {{.*}} contains 2
+# DYN:        R_X86_64_64 0000000000000000 foobar + 0{{$}}
 
-# RUN: ld.lld a.o b.o -o out1 -z undefs
-# RUN: llvm-readelf -r -x .data out1 | FileCheck %s --check-prefix=STATIC1
-# RUN: ld.lld a.o b.o -o out1.pie -pie -z undefs
-# RUN: llvm-readelf -r -x .data out1.pie | FileCheck %s --check-prefix=STATIC1
+# RUN: ld.lld a.o b.o -o ab -z undefs
+# RUN: llvm-readelf -r -x .data ab | FileCheck %s --check-prefix=STATIC1
+# RUN: ld.lld a.o b.o s.so -o abs -z undefs
+# RUN: llvm-readelf -r -x .data abs | FileCheck %s --check-prefix=STATIC1
+# RUN: ld.lld a.o b.o -o ab.pie -pie -z undefs
+# RUN: llvm-readelf -r -x .data ab.pie | FileCheck %s --check-prefix=STATIC1
+# RUN: ld.lld a.o b.o s.so -o abs.pie -pie -z undefs
+# RUN: llvm-readelf -r -x .data abs.pie | FileCheck %s --check-prefix=DYN1
 
 # STATIC1:      no relocations
 # STATIC1:      Hex dump of section '.data':
-# STATIC1-NEXT: {{.*}} 00000000 00000000 00000000 00000000 .
+# STATIC1-NEXT: {{.*}} 00000000 00000000 03000000 00000000 .
+# STATIC1-NEXT: {{.*}} 05000000 00000000                   .
 # STATIC1-EMPTY:
 
+# DYN1:        Relocation section '.rela.dyn' {{.*}} contains 3
+# DYN1:        Hex dump of section '.data':
+# DYN1-NEXT:   {{.*}} 00000000 00000000 00000000 00000000 .
+# DYN1-NEXT:   {{.*}} 00000000 00000000                   .
+# DYN1-EMPTY:
+
 # RUN: ld.lld a.o b.o c.o -pie -z undefs 2>&1 | count 0
 
 #--- a.s
-        .global _start
+.global _start
 _start:
-        .data
-        .weak foobar
-        .quad foobar
+.data
+.weak foobar
+.quad foobar
+.quad foobar+3
 
 #--- b.s
 .data
-.quad undef
+.quad undef+5
 
 #--- c.s
 call undef
diff --git a/external/llvm-project/lld/test/ELF/weak-undef.s b/external/llvm-project/lld/test/ELF/weak-undef.s
deleted file mode 100644
index 21488023a79e..000000000000
--- a/external/llvm-project/lld/test/ELF/weak-undef.s
+++ /dev/null
@@ -1,31 +0,0 @@
-# REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
-# RUN: ld.lld %t.o -o %t --export-dynamic
-# RUN: llvm-readelf -r --dyn-syms --hex-dump=.data %t | \
-# RUN:   FileCheck %s --check-prefixes=NORELOC,COMMON
-
-# NORELOC: There are no relocations in this file.
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %p/Inputs/dummy-shared.s -o %t1.o
-# RUN: ld.lld %t1.o -shared -o %t1.so
-# RUN: ld.lld %t.o -o %t %t1.so -pie
-# RUN: llvm-readelf -r --dyn-syms --hex-dump=.data %t | \
-# RUN:   FileCheck %s --check-prefixes=RELOC,COMMON
-
-# RELOC:      Relocation section '.rela.dyn' at offset {{.*}} contains 1 entries:
-# RELOC-NEXT: Offset Info Type Symbol's Value Symbol's Name + Addend
-# RELOC-NEXT: {{.*}} 0000000100000001 R_X86_64_64 0000000000000000 foo + 0
-
-# NORELOC-NOT: Symbol table '.dynsym'
-# RELOC:       Symbol table '.dynsym' contains 2 entries:
-# RELOC-NEXT:  Num: Value Size Type Bind Vis Ndx Name
-# RELOC-NEXT:  0: 0000000000000000 0 NOTYPE LOCAL DEFAULT UND
-# RELOC-NEXT:  1: 0000000000000000 0 NOTYPE WEAK DEFAULT UND foo
-# COMMON:      Hex dump of section '.data':
-# COMMON-NEXT: {{.*}} 00000000 00000000 
-# COMMON-EMPTY:
-
-.weak foo
-
-.data
-  .dc.a foo
diff --git a/external/llvm-project/lld/test/ELF/x86-64-section-layout.s b/external/llvm-project/lld/test/ELF/x86-64-section-layout.s
index b03d3e6c2b99..1432271b885a 100644
--- a/external/llvm-project/lld/test/ELF/x86-64-section-layout.s
+++ b/external/llvm-project/lld/test/ELF/x86-64-section-layout.s
@@ -18,6 +18,10 @@
 # RUN: ld.lld --section-start=.note=0x200300 a.o -z lrodata-after-bss -o a3
 # RUN: llvm-readelf -S -l -sX a3 | FileCheck %s --check-prefix=CHECK3
 
+# RUN: llvm-mc -filetype=obj -triple=x86_64 c.s -o c.o
+# RUN: ld.lld c.o -o c
+# RUN: llvm-readelf -S -l c | FileCheck %s --check-prefix=CHECK4
+
 # CHECK:       Name              Type            Address          Off    Size   ES Flg Lk Inf Al
 # CHECK-NEXT:                    NULL            0000000000000000 000000 000000 00      0   0  0
 # CHECK-NEXT:  .note             NOTE            0000000000200300 000300 000001 00   A  0   0  1
@@ -116,6 +120,18 @@
 # CHECK3-NEXT:  0000000000203307     0 NOTYPE  GLOBAL DEFAULT [[#]] (.data)   _edata
 # CHECK3-NEXT:  0000000000207d0d     0 NOTYPE  GLOBAL DEFAULT [[#]] (.ldata2) _end
 
+# CHECK4:      .note      NOTE
+# CHECK4-NEXT: .ltext     PROGBITS
+# CHECK4-NEXT: .lrodata   PROGBITS
+# CHECK4-NEXT: .rodata    PROGBITS
+# CHECK4-NEXT: .text      PROGBITS
+# CHECK4-NEXT: .data      PROGBITS
+# CHECK4-NEXT: .bss       NOBITS
+# CHECK4-NEXT: .ldata     PROGBITS
+# CHECK4-NEXT: .lbss      NOBITS
+# CHECK4-NEXT: .ltext_w   PROGBITS
+# CHECK4-NEXT: .comment   PROGBITS
+
 #--- a.s
 .globl _start, _etext, _edata, _end
 _start:
@@ -155,3 +171,28 @@ SECTIONS {
   .ldata2 : {}
   .lbss : { *(.lbss .lbss.*) }
 }
+
+#--- c.s
+## Test .ltext layout
+.section .ltext,"axl",@progbits
+.globl f
+f:
+  ret
+
+.section .ltext_w,"awxl",@progbits
+.globl g
+g:
+  ret
+
+.section .text,"ax",@progbits
+.globl h
+h:
+  ret
+
+.section .note,"a",@note; .space 1
+.section .rodata,"a",@progbits; .space 1
+.section .data,"aw",@progbits; .space 1
+.section .bss,"aw",@nobits; .space 1
+.section .lrodata,"al"; .space 1
+.section .ldata,"awl"; .space 1
+.section .lbss,"awl",@nobits; .space 1
diff --git a/external/llvm-project/lld/test/MachO/order-file-strip-hashes.s b/external/llvm-project/lld/test/MachO/order-file-strip-hashes.s
new file mode 100644
index 000000000000..42d1e1b98f49
--- /dev/null
+++ b/external/llvm-project/lld/test/MachO/order-file-strip-hashes.s
@@ -0,0 +1,96 @@
+# REQUIRES: aarch64
+
+# RUN: rm -rf %t && split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/a.s -o %t/a.o
+
+# RUN: %lld -arch arm64 -lSystem -e _main -o %t/a.out %t/a.o -order_file %t/ord-1
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/a.out | FileCheck %s
+	
+#--- a.s
+.text
+.globl _main, A, _B, C.__uniq.111111111111111111111111111111111111111.llvm.2222222222222222222
+
+_main:
+  ret
+A:
+  ret
+F:
+  add w0, w0, #3
+  bl C.__uniq.111111111111111111111111111111111111111.llvm.2222222222222222222
+  ret
+C.__uniq.111111111111111111111111111111111111111.llvm.2222222222222222222:
+  add w0, w0, #2
+  bl  A
+  ret
+D:
+  add w0, w0, #2
+  bl B
+  ret
+B:
+  add w0, w0, #1
+  bl  A
+  ret
+E:
+  add w0, w0, #2
+  bl C.__uniq.111111111111111111111111111111111111111.llvm.2222222222222222222
+  ret
+
+.section __DATA,__objc_const
+# test multiple symbols at the same address, which will be alphabetic sorted based symbol names
+_OBJC_$_CATEGORY_CLASS_METHODS_Foo_$_Cat2:
+  .quad 789
+
+_OBJC_$_CATEGORY_SOME_$_FOLDED:
+_OBJC_$_CATEGORY_Foo_$_Cat1:
+_ALPHABETIC_SORT_FIRST:
+ .quad 123
+
+_OBJC_$_CATEGORY_Foo_$_Cat2:
+ .quad 222
+
+_OBJC_$_CATEGORY_INSTANCE_METHODS_Foo_$_Cat1:
+  .quad 456
+
+.section __DATA,__objc_data
+_OBJC_CLASS_$_Foo:
+ .quad 123
+
+_OBJC_CLASS_$_Bar.llvm.1234:
+ .quad 456
+
+_OBJC_CLASS_$_Baz:
+ .quad 789
+
+_OBJC_CLASS_$_Baz2:
+ .quad 999
+
+.section __DATA,__objc_classrefs
+.quad _OBJC_CLASS_$_Foo
+.quad _OBJC_CLASS_$_Bar.llvm.1234
+.quad _OBJC_CLASS_$_Baz
+
+.subsections_via_symbols
+
+
+#--- ord-1
+# change order, parital covered
+A
+B
+C.__uniq.555555555555555555555555555555555555555.llvm.6666666666666666666
+_OBJC_CLASS_$_Baz
+_OBJC_CLASS_$_Bar.__uniq.12345
+_OBJC_CLASS_$_Foo.__uniq.123.llvm.123456789
+_OBJC_$_CATEGORY_INSTANCE_METHODS_Foo_$_Cat1
+_OBJC_$_CATEGORY_Foo_$_Cat1.llvm.1234567
+
+# .text
+# CHECK: A
+# CHECK: B
+# CHECK: C
+# .section __DATA,__objc_const
+# CHECK: _OBJC_$_CATEGORY_INSTANCE_METHODS_Foo_$_Cat1
+# CHECK: _OBJC_$_CATEGORY_Foo_$_Cat1
+# .section __DATA,__objc_data
+# CHECK: _OBJC_CLASS_$_Baz
+# CHECK: _OBJC_CLASS_$_Bar
+# CHECK: _OBJC_CLASS_$_Foo
diff --git a/external/llvm-project/lld/test/MachO/ordre-file-cstring.s b/external/llvm-project/lld/test/MachO/ordre-file-cstring.s
new file mode 100644
index 000000000000..3c6d2a377dc3
--- /dev/null
+++ b/external/llvm-project/lld/test/MachO/ordre-file-cstring.s
@@ -0,0 +1,228 @@
+# REQUIRES: aarch64
+
+# RUN: rm -rf %t; split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin  %t/test.s -o %t/test.o
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/more-cstrings.s -o %t/more-cstrings.o
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-0 %t/test.o %t/more-cstrings.o
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-0 | FileCheck %s --check-prefix=ORIGIN_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-0 | FileCheck %s --check-prefix=ORIGIN_SEC
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-1 %t/test.o %t/more-cstrings.o -order_file %t/ord-1
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-1 | FileCheck %s --check-prefix=ONE_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-1 | FileCheck %s --check-prefix=ONE_SEC
+
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-2 %t/test.o %t/more-cstrings.o -order_file %t/ord-2
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-2 | FileCheck %s --check-prefix=TWO_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-2 | FileCheck %s --check-prefix=TWO_SEC
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-3 %t/test.o %t/more-cstrings.o -order_file %t/ord-3
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-3 | FileCheck %s --check-prefix=THREE_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-3 | FileCheck %s --check-prefix=THREE_SEC
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-4 %t/test.o %t/more-cstrings.o -order_file %t/ord-4
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-4 | FileCheck %s --check-prefix=FOUR_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-4 | FileCheck %s --check-prefix=FOUR_SEC
+# RUN: llvm-readobj --string-dump=__cstring %t/test-4 | FileCheck %s --check-prefix=FOUR_SEC_ESCAPE
+
+
+# We expect:
+# 1) Covered cstring symbols are reordered
+# 2) the rest of the cstring symbols remain original relative order within the cstring section
+
+# ORIGIN_SYM: _local_foo1
+# ORIGIN_SYM: _globl_foo2
+# ORIGIN_SYM: _local_foo2
+# ORIGIN_SYM: _bar
+# ORIGIN_SYM: _baz
+# ORIGIN_SYM: _baz_dup
+# ORIGIN_SYM: _bar2
+# ORIGIN_SYM: _globl_foo3
+
+# ORIGIN_SEC: foo1
+# ORIGIN_SEC: foo2
+# ORIGIN_SEC: bar
+# ORIGIN_SEC: baz
+# ORIGIN_SEC: bar2
+# ORIGIN_SEC: foo3
+
+# original order, but only parital covered
+#--- ord-1
+#foo2
+CSTR;1433942677
+#bar
+CSTR;540201826
+#bar2
+CSTR;1496286555
+#foo3
+CSTR;1343999025
+
+# ONE_SYM: _globl_foo2
+# ONE_SYM: _local_foo2
+# ONE_SYM: _bar
+# ONE_SYM: _bar2
+# ONE_SYM: _globl_foo3
+# ONE_SYM: _local_foo1
+# ONE_SYM: _baz
+# ONE_SYM: _baz_dup
+
+# ONE_SEC: foo2
+# ONE_SEC: bar
+# ONE_SEC: bar2
+# ONE_SEC: foo3
+# ONE_SEC: foo1
+# ONE_SEC: baz
+
+
+# TWO_SYM: _globl_foo2
+# TWO_SYM: _local_foo2
+# TWO_SYM: _local_foo1
+# TWO_SYM: _baz
+# TWO_SYM: _baz_dup
+# TWO_SYM: _bar
+# TWO_SYM: _bar2
+# TWO_SYM: _globl_foo3
+
+# TWO_SEC: foo2
+# TWO_SEC: foo1
+# TWO_SEC: baz
+# TWO_SEC: bar
+# TWO_SEC: bar2
+# TWO_SEC: foo3
+
+
+# THREE_SYM: _local_foo1
+# THREE_SYM: _baz
+# THREE_SYM: _baz_dup
+# THREE_SYM: _bar
+# THREE_SYM: _bar2
+# THREE_SYM: _globl_foo2
+# THREE_SYM: _local_foo2
+# THREE_SYM: _globl_foo3
+
+# THREE_SEC: foo1
+# THREE_SEC: baz
+# THREE_SEC: bar
+# THREE_SEC: bar2
+# THREE_SEC: foo2
+# THREE_SEC: foo3
+
+
+# FOUR_SYM: _local_escape_white_space
+# FOUR_SYM: _globl_foo2
+# FOUR_SYM: _local_foo2
+# FOUR_SYM: _local_escape
+# FOUR_SYM: _globl_foo3
+# FOUR_SYM: _bar
+# FOUR_SYM: _local_foo1
+# FOUR_SYM: _baz
+# FOUR_SYM: _baz_dup
+# FOUR_SYM: _bar2
+
+# FOUR_SEC: \t\n
+# FOUR_SEC: foo2
+# FOUR_SEC: @\"NSDictionary\"
+# FOUR_SEC: foo3
+# FOUR_SEC: bar
+# FOUR_SEC: foo1
+# FOUR_SEC: baz
+# FOUR_SEC: bar2
+
+# FOUR_SEC_ESCAPE: ..
+# FOUR_SEC_ESCAPE: foo2
+# FOUR_SEC_ESCAPE: @"NSDictionary"
+# FOUR_SEC_ESCAPE: foo3
+# FOUR_SEC_ESCAPE: bar
+# FOUR_SEC_ESCAPE: foo1
+# FOUR_SEC_ESCAPE: baz
+# FOUR_SEC_ESCAPE: bar2
+
+
+# change order, parital covered
+#--- ord-2
+#foo2
+CSTR;1433942677
+#foo1
+CSTR;1663475769
+#baz
+CSTR;862947621
+#bar
+CSTR;540201826
+#bar2
+CSTR;1496286555
+
+# change order, parital covered, with mismatches, duplicates
+#--- ord-3
+foo2222
+CSTR;0x11111111
+#bar (mismatched cpu and file name)
+fakeCPU:fake-file-name.o:CSTR;540201826
+#not a hash
+CSTR;xxx
+#foo1
+CSTR;1663475769
+#baz
+CSTR;862947621
+#bar
+CSTR;540201826
+#bar2
+CSTR;1496286555
+#baz
+CSTR;862947621
+
+# test escape strings
+#--- ord-4
+#\t\n
+CSTR;1035903177
+#foo2
+CSTR;1433942677
+#@\"NSDictionary\"
+CSTR;1202669430
+#foo3
+CSTR;1343999025
+#bar
+CSTR;540201826
+
+
+#--- test.s
+.text
+.globl _main
+
+_main:
+  ret
+
+.cstring
+.p2align 2
+_local_foo1:
+  .asciz "foo1"
+_local_foo2:
+  .asciz "foo2"
+L_.foo1_dup:
+  .asciz "foo1"
+L_.foo2_dup:
+  .asciz "foo2"
+_local_escape:
+  .asciz "@\"NSDictionary\""
+_local_escape_white_space:
+  .asciz "\t\n"
+
+_bar:
+  .asciz "bar"
+_baz:
+  .asciz "baz"
+_bar2:
+  .asciz "bar2"
+_baz_dup:
+  .asciz "baz"
+
+.subsections_via_symbols
+
+#--- more-cstrings.s
+.globl _globl_foo1, _globl_foo3
+.cstring
+.p2align 4
+_globl_foo3:
+  .asciz "foo3"
+_globl_foo2:
+  .asciz "foo2"
diff --git a/external/llvm-project/lld/test/MinGW/lib.test b/external/llvm-project/lld/test/MinGW/lib.test
index 8bd8a0e9304d..ac002e7549b4 100644
--- a/external/llvm-project/lld/test/MinGW/lib.test
+++ b/external/llvm-project/lld/test/MinGW/lib.test
@@ -5,6 +5,7 @@ LIB1: unable to find library -lfoo
 
 RUN: echo > %t/lib/libfoo.dll.a
 RUN: ld.lld -### -m i386pep -lfoo -L%t/lib 2>&1 | FileCheck -check-prefix=LIB2 %s
+RUN: ld.lld -### -m i386pep -lfoo --dll-search-prefix=cyg -L%t/lib 2>&1 | FileCheck -check-prefix=LIB2 %s
 LIB2: libfoo.dll.a
 
 RUN: not ld.lld -### -m i386pep -l:barefilename -L%t/lib 2>&1 | FileCheck -check-prefix=LIB-LITERAL-FAIL %s
@@ -22,6 +23,7 @@ LIB3: unable to find library -lfoo
 
 RUN: echo > %t/lib/libfoo.a
 RUN: ld.lld -### -m i386pep -Bstatic -lfoo -L%t/lib 2>&1 | FileCheck -check-prefix=LIB4 %s
+RUN: ld.lld -### -m i386pep -Bstatic -lfoo --dll-search-prefix=cyg -L%t/lib 2>&1 | FileCheck -check-prefix=LIB4 %s
 LIB4: libfoo.a
 
 RUN: echo > %t/lib/libbar.dll.a
@@ -46,12 +48,17 @@ MSVCSTYLE: msvcstyle.lib
 
 RUN: echo > %t/lib/libnoimplib.dll
 RUN: echo > %t/lib/noprefix_noimplib.dll
+RUN: echo > %t/lib/cygnoimplib2.dll
 RUN: ld.lld -### -m i386pep -L%t/lib -lnoimplib 2>&1 | FileCheck -check-prefix=DLL1 %s
 RUN: ld.lld -### -m i386pep -L%t/lib -lnoprefix_noimplib 2>&1 | FileCheck -check-prefix=DLL2 %s
+RUN: ld.lld -### -m i386pep -L%t/lib -lnoimplib2 --dll-search-prefix=cyg 2>&1 | FileCheck -check-prefix=DLL3 %s
 DLL1: libnoimplib.dll
 DLL2: noprefix_noimplib.dll
+DLL3: cygnoimplib2.dll
 
 RUN: not ld.lld -### -m i386pep -L%t/lib -static -lnoimplib 2>&1 | FileCheck -check-prefix=ERROR-NOIMPLIB %s
 RUN: not ld.lld -### -m i386pep -L%t/lib -static -lnoprefix_noimplib 2>&1 | FileCheck -check-prefix=ERROR-NOPREFIX-NOIMPLIB %s
+RUN: not ld.lld -### -m i386pep -L%t/lib -static -lnoimplib2 --dll-search-prefix=cyg 2>&1 | FileCheck -check-prefix=ERROR-CYG-NOIMPLIB %s
 ERROR-NOIMPLIB: unable to find library -lnoimplib
 ERROR-NOPREFIX-NOIMPLIB: unable to find library -lnoprefix_noimplib
+ERROR-CYG-NOIMPLIB: unable to find library -lnoimplib2
diff --git a/external/llvm-project/lld/test/wasm/lto/Inputs/libcall-return-addr.ll b/external/llvm-project/lld/test/wasm/lto/Inputs/libcall-return-addr.ll
deleted file mode 100644
index 271bdae11e49..000000000000
--- a/external/llvm-project/lld/test/wasm/lto/Inputs/libcall-return-addr.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-f128:64-n32:64-S128-ni:1:10:20"
-target triple = "wasm32-unknown-emscripten"
-
-define ptr @emscripten_return_address() {
-  ret ptr null
-}
diff --git a/external/llvm-project/lld/test/wasm/lto/libcall-return-addr.ll b/external/llvm-project/lld/test/wasm/lto/libcall-return-addr.ll
deleted file mode 100644
index 74eba74f9701..000000000000
--- a/external/llvm-project/lld/test/wasm/lto/libcall-return-addr.ll
+++ /dev/null
@@ -1,18 +0,0 @@
-; RUN: llvm-as %s -o %t.o
-; RUN: llvm-as %p/Inputs/libcall-return-addr.ll -o %t.return-addr.o
-; RUN: rm -f %t.a
-; RUN: llvm-ar rcs %t.a %t.return-addr.o
-; RUN: not wasm-ld --export-all %t.o %t.a -o %t.wasm 2>&1 | FileCheck %s
-
-target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-f128:64-n32:64-S128-ni:1:10:20"
-target triple = "wasm32-unknown-emscripten"
-
-@g_ptr = global ptr null
-
-define void @_start() {
-  %addr = call ptr @llvm.returnaddress(i32 1)
-  store ptr %addr, ptr @g_ptr
-  ret void
-}
-
-; CHECK: wasm-ld: error: {{.*}}return-addr.o): attempt to add bitcode file after LTO (emscripten_return_address)
diff --git a/external/llvm-project/lld/wasm/Driver.cpp b/external/llvm-project/lld/wasm/Driver.cpp
index 2b1fb945f41c..1c5d21c06f5a 100644
--- a/external/llvm-project/lld/wasm/Driver.cpp
+++ b/external/llvm-project/lld/wasm/Driver.cpp
@@ -1226,9 +1226,9 @@ static void wrapSymbols(ArrayRef<WrappedSymbol> wrapped) {
   // Update pointers in input files.
   parallelForEach(ctx.objectFiles, [&](InputFile *file) {
     MutableArrayRef<Symbol *> syms = file->getMutableSymbols();
-    for (size_t i = 0, e = syms.size(); i != e; ++i)
-      if (Symbol *s = map.lookup(syms[i]))
-        syms[i] = s;
+    for (Symbol *&sym : syms)
+      if (Symbol *s = map.lookup(sym))
+        sym = s;
   });
 
   // Update pointers in the symbol table.
diff --git a/external/llvm-project/lldb/CMakeLists.txt b/external/llvm-project/lldb/CMakeLists.txt
index 85ba4fde1741..2aaf75dd87bc 100644
--- a/external/llvm-project/lldb/CMakeLists.txt
+++ b/external/llvm-project/lldb/CMakeLists.txt
@@ -38,6 +38,8 @@ endif()
 include(LLDBConfig)
 include(AddLLDB)
 
+set(LLDB_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
+
 # Define the LLDB_CONFIGURATION_xxx matching the build type.
 if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" )
   add_definitions(-DLLDB_CONFIGURATION_DEBUG)
diff --git a/external/llvm-project/lldb/bindings/interface/SBBreakpointExtensions.i b/external/llvm-project/lldb/bindings/interface/SBBreakpointExtensions.i
index 6bc781a32777..82fc3ab0e7c0 100644
--- a/external/llvm-project/lldb/bindings/interface/SBBreakpointExtensions.i
+++ b/external/llvm-project/lldb/bindings/interface/SBBreakpointExtensions.i
@@ -50,6 +50,15 @@ STRING_EXTENSION_OUTSIDE(SBBreakpoint)
         enabled = property(IsEnabled, SetEnabled, doc='''A read/write property that configures whether this breakpoint is enabled or not.''')
         one_shot = property(IsOneShot, SetOneShot, doc='''A read/write property that configures whether this breakpoint is one-shot (deleted when hit) or not.''')
         num_locations = property(GetNumLocations, None, doc='''A read only property that returns the count of locations of this breakpoint.''')
+        auto_continue = property(GetAutoContinue, SetAutoContinue, doc='A read/write property that configures the auto-continue property of this breakpoint.')
+        condition = property(GetCondition, SetCondition, doc='A read/write property that configures the condition of this breakpoint.')
+        hit_count = property(GetHitCount, doc='A read only property that returns the hit count of this breakpoint.')
+        ignore_count = property(GetIgnoreCount, SetIgnoreCount, doc='A read/write property that configures the ignore count of this breakpoint.')
+        queue_name = property(GetQueueName, SetQueueName, doc='A read/write property that configures the queue name criteria of this breakpoint.')
+        target = property(GetTarget, doc='A read only property that returns the target of this breakpoint.')
+        thread_id = property(GetThreadID, SetThreadID, doc='A read/write property that configures the thread id criteria of this breakpoint.')
+        thread_index = property(GetThreadIndex, SetThreadIndex, doc='A read/write property that configures the thread index criteria of this breakpoint.')
+        thread_name = property(GetThreadName, SetThreadName, doc='A read/write property that configures the thread name criteria of this breakpoint.')
     %}
 #endif
 }
diff --git a/external/llvm-project/lldb/bindings/interface/SBBreakpointLocationExtensions.i b/external/llvm-project/lldb/bindings/interface/SBBreakpointLocationExtensions.i
index 40027790a5e8..12c87eb2c0fe 100644
--- a/external/llvm-project/lldb/bindings/interface/SBBreakpointLocationExtensions.i
+++ b/external/llvm-project/lldb/bindings/interface/SBBreakpointLocationExtensions.i
@@ -7,6 +7,19 @@ STRING_EXTENSION_LEVEL_OUTSIDE(SBBreakpointLocation, lldb::eDescriptionLevelFull
     # our own equality operator here
     def __eq__(self, other):
       return not self.__ne__(other)
+
+    addr = property(GetAddress, doc='A read only property that returns the address of this breakpoint location.')
+    auto_continue = property(GetAutoContinue, SetAutoContinue, doc='A read/write property that configures the auto-continue property of this breakpoint location.')
+    breakpoint = property(GetBreakpoint, doc='A read only property that returns the parent breakpoint of this breakpoint location.')
+    condition = property(GetCondition, SetCondition, doc='A read/write property that configures the condition of this breakpoint location.')
+    hit_count = property(GetHitCount, doc='A read only property that returns the hit count of this breakpoint location.')
+    id = property(GetID, doc='A read only property that returns the id of this breakpoint location.')
+    ignore_count = property(GetIgnoreCount, SetIgnoreCount, doc='A read/write property that configures the ignore count of this breakpoint location.')
+    load_addr = property(GetLoadAddress, doc='A read only property that returns the load address of this breakpoint location.')
+    queue_name = property(GetQueueName, SetQueueName, doc='A read/write property that configures the queue name criteria of this breakpoint location.')
+    thread_id = property(GetThreadID, SetThreadID, doc='A read/write property that configures the thread id criteria of this breakpoint location.')
+    thread_index = property(GetThreadIndex, SetThreadIndex, doc='A read/write property that configures the thread index criteria of this breakpoint location.')
+    thread_name = property(GetThreadName, SetThreadName, doc='A read/write property that configures the thread name criteria of this breakpoint location.')
     %}
 #endif
 }
diff --git a/external/llvm-project/lldb/bindings/interface/SBBreakpointNameExtensions.i b/external/llvm-project/lldb/bindings/interface/SBBreakpointNameExtensions.i
index 2cd47a4a3223..022f4c969e5c 100644
--- a/external/llvm-project/lldb/bindings/interface/SBBreakpointNameExtensions.i
+++ b/external/llvm-project/lldb/bindings/interface/SBBreakpointNameExtensions.i
@@ -7,6 +7,16 @@ STRING_EXTENSION_OUTSIDE(SBBreakpointName)
     # our own equality operator here
     def __eq__(self, other):
       return not self.__ne__(other)
+
+    auto_continue = property(GetAutoContinue, SetAutoContinue, doc='A read/write property that configures the auto-continue property of this breakpoint name.')
+    condition = property(GetCondition, SetCondition, doc='A read/write property that configures the condition of this breakpoint name.')
+    enabled = property(IsEnabled, SetEnabled, doc='''A read/write property that configures whether this breakpoint name is enabled or not.''')
+    ignore_count = property(GetIgnoreCount, SetIgnoreCount, doc='A read/write property that configures the ignore count of this breakpoint name.')
+    one_shot = property(IsOneShot, SetOneShot, doc='''A read/write property that configures whether this breakpoint name is one-shot (deleted when hit) or not.''')
+    queue_name = property(GetQueueName, SetQueueName, doc='A read/write property that configures the queue name criteria of this breakpoint name.')
+    thread_id = property(GetThreadID, SetThreadID, doc='A read/write property that configures the thread id criteria of this breakpoint name.')
+    thread_index = property(GetThreadIndex, SetThreadIndex, doc='A read/write property that configures the thread index criteria of this breakpoint name.')
+    thread_name = property(GetThreadName, SetThreadName, doc='A read/write property that configures the thread name criteria of this breakpoint name.')
     %}
 #endif
 }
diff --git a/external/llvm-project/lldb/cmake/modules/AddLLDB.cmake b/external/llvm-project/lldb/cmake/modules/AddLLDB.cmake
index 3a9dcb79629b..28bf8d816d89 100644
--- a/external/llvm-project/lldb/cmake/modules/AddLLDB.cmake
+++ b/external/llvm-project/lldb/cmake/modules/AddLLDB.cmake
@@ -38,14 +38,11 @@ function(add_lldb_library name)
     ${CMAKE_CURRENT_BINARY_DIR}
 )
 
-  # only supported parameters to this macro are the optional
-  # MODULE;SHARED;STATIC library type and source files
   cmake_parse_arguments(PARAM
     "MODULE;SHARED;STATIC;OBJECT;PLUGIN;FRAMEWORK;NO_INTERNAL_DEPENDENCIES;NO_PLUGIN_DEPENDENCIES"
-    "INSTALL_PREFIX;ENTITLEMENTS"
-    "EXTRA_CXXFLAGS;DEPENDS;LINK_LIBS;LINK_COMPONENTS;CLANG_LIBS"
+    "INSTALL_PREFIX"
+    "LINK_LIBS;CLANG_LIBS"
     ${ARGN})
-  list(APPEND LLVM_LINK_COMPONENTS ${PARAM_LINK_COMPONENTS})
 
   if(PARAM_NO_INTERNAL_DEPENDENCIES)
     foreach(link_lib ${PARAM_LINK_LIBS})
@@ -71,12 +68,6 @@ function(add_lldb_library name)
     set_property(GLOBAL APPEND PROPERTY LLDB_PLUGINS ${name})
   endif()
 
-  if (MSVC_IDE OR XCODE)
-    string(REGEX MATCHALL "/[^/]+" split_path ${CMAKE_CURRENT_SOURCE_DIR})
-    list(GET split_path -1 dir)
-    file(GLOB_RECURSE headers
-      ../../include/lldb${dir}/*.h)
-  endif()
   if (PARAM_MODULE)
     set(libkind MODULE)
   elseif (PARAM_SHARED)
@@ -91,19 +82,13 @@ function(add_lldb_library name)
     set(libkind STATIC)
   endif()
 
-  if(PARAM_ENTITLEMENTS)
-    set(pass_ENTITLEMENTS ENTITLEMENTS ${PARAM_ENTITLEMENTS})
-  endif()
-
   if(LLDB_NO_INSTALL_DEFAULT_RPATH)
     set(pass_NO_INSTALL_RPATH NO_INSTALL_RPATH)
   endif()
 
-  llvm_add_library(${name} ${libkind} ${headers}
+  llvm_add_library(${name} ${libkind}
     ${PARAM_UNPARSED_ARGUMENTS}
     LINK_LIBS ${PARAM_LINK_LIBS}
-    DEPENDS ${PARAM_DEPENDS}
-    ${pass_ENTITLEMENTS}
     ${pass_NO_INSTALL_RPATH}
   )
 
@@ -145,9 +130,6 @@ function(add_lldb_library name)
     add_dependencies(${name} clang-tablegen-targets)
   endif()
 
-  # Add in any extra C++ compilation flags for this library.
-  target_compile_options(${name} PRIVATE ${PARAM_EXTRA_CXXFLAGS})
-
   if(PARAM_PLUGIN)
     get_property(parent_dir DIRECTORY PROPERTY PARENT_DIRECTORY)
     if(EXISTS ${parent_dir})
@@ -169,22 +151,17 @@ endfunction(add_lldb_library)
 function(add_lldb_executable name)
   cmake_parse_arguments(ARG
     "GENERATE_INSTALL"
-    "INSTALL_PREFIX;ENTITLEMENTS"
+    "INSTALL_PREFIX"
     "LINK_LIBS;CLANG_LIBS;LINK_COMPONENTS;BUILD_RPATH;INSTALL_RPATH"
     ${ARGN}
     )
 
-  if(ARG_ENTITLEMENTS)
-    set(pass_ENTITLEMENTS ENTITLEMENTS ${ARG_ENTITLEMENTS})
-  endif()
-
   if(LLDB_NO_INSTALL_DEFAULT_RPATH)
     set(pass_NO_INSTALL_RPATH NO_INSTALL_RPATH)
   endif()
 
   list(APPEND LLVM_LINK_COMPONENTS ${ARG_LINK_COMPONENTS})
   add_llvm_executable(${name}
-    ${pass_ENTITLEMENTS}
     ${pass_NO_INSTALL_RPATH}
     ${ARG_UNPARSED_ARGUMENTS}
   )
diff --git a/external/llvm-project/lldb/cmake/modules/LLDBFramework.cmake b/external/llvm-project/lldb/cmake/modules/LLDBFramework.cmake
index 471aeaaad3c0..70010ffbf738 100644
--- a/external/llvm-project/lldb/cmake/modules/LLDBFramework.cmake
+++ b/external/llvm-project/lldb/cmake/modules/LLDBFramework.cmake
@@ -68,24 +68,17 @@ if(NOT APPLE_EMBEDDED)
   )
 endif()
 
-# At configuration time, collect headers for the framework bundle and copy them
-# into a staging directory. Later we can copy over the entire folder.
-file(GLOB public_headers ${LLDB_SOURCE_DIR}/include/lldb/API/*.h)
-set(generated_public_headers ${LLDB_OBJ_DIR}/include/lldb/API/SBLanguages.h)
-file(GLOB root_public_headers ${LLDB_SOURCE_DIR}/include/lldb/lldb-*.h)
-file(GLOB root_private_headers ${LLDB_SOURCE_DIR}/include/lldb/lldb-private*.h)
-list(REMOVE_ITEM root_public_headers ${root_private_headers})
-
 find_program(unifdef_EXECUTABLE unifdef)
 
-set(lldb_header_staging ${CMAKE_CURRENT_BINARY_DIR}/FrameworkHeaders)
-foreach(header
-    ${public_headers}
-    ${generated_public_headers}
-    ${root_public_headers})
+# All necessary header files will be staged in the include directory in the build directory,
+# so just copy the files from there into the framework's staging directory.
+set(lldb_build_dir_header_staging "${CMAKE_BINARY_DIR}/include/lldb")
+set(lldb_framework_header_staging "${CMAKE_CURRENT_BINARY_DIR}/FrameworkHeaders")
+file(GLOB lldb_build_dir_header_staging_list ${lldb_build_dir_header_staging}/*)
+foreach(header ${lldb_build_dir_header_staging_list})
 
   get_filename_component(basename ${header} NAME)
-  set(staged_header ${lldb_header_staging}/${basename})
+  set(staged_header ${lldb_framework_header_staging}/${basename})
 
   if(unifdef_EXECUTABLE)
     # unifdef returns 0 when the file is unchanged and 1 if something was changed.
@@ -107,15 +100,25 @@ endforeach()
 # Wrap output in a target, so lldb-framework can depend on it.
 add_custom_target(liblldb-resource-headers DEPENDS lldb-sbapi-dwarf-enums ${lldb_staged_headers})
 set_target_properties(liblldb-resource-headers PROPERTIES FOLDER "LLDB/Resources")
+# We're taking the header files from where they've been staged in the build directory's include folder,
+# so create a dependency on the build step that creates that directory.
+add_dependencies(liblldb-resource-headers liblldb-header-staging)
 add_dependencies(liblldb liblldb-resource-headers)
 
-# At build time, copy the staged headers into the framework bundle (and do
-# some post-processing in-place).
-add_custom_command(TARGET liblldb POST_BUILD
-  COMMAND ${CMAKE_COMMAND} -E copy_directory ${lldb_header_staging} $<TARGET_FILE_DIR:liblldb>/Headers
-  COMMAND ${LLDB_SOURCE_DIR}/scripts/framework-header-fix.sh $<TARGET_FILE_DIR:liblldb>/Headers ${LLDB_VERSION}
-  COMMENT "LLDB.framework: copy framework headers"
-)
+# Take the headers from the staging directory and fix up their includes for the framework.
+# Then write them to the output directory.
+# Also, run unifdef to remove any specified guards from the header files.
+file(GLOB lldb_framework_header_staging_list ${lldb_framework_header_staging}/*)
+foreach(header ${lldb_framework_header_staging_list})
+
+  set(input_header ${header})
+  set(output_header $<TARGET_FILE_DIR:liblldb>/Headers/${input_header})
+
+  add_custom_command(TARGET liblldb POST_BUILD
+    COMMAND ${LLDB_SOURCE_DIR}/scripts/framework-header-fix.py -f lldb_main -i ${input_header} -o ${output_header} -p ${unifdef_EXECUTABLE} USWIG
+    COMMENT "LLDB.framework: Fix up and copy framework headers"
+  )
+endforeach()
 
 # Copy vendor-specific headers from clang (without staging).
 if(NOT APPLE_EMBEDDED)
diff --git a/external/llvm-project/lldb/docs/use/formatting.rst b/external/llvm-project/lldb/docs/use/formatting.rst
index e71b08780eb3..21b3ca1912b0 100644
--- a/external/llvm-project/lldb/docs/use/formatting.rst
+++ b/external/llvm-project/lldb/docs/use/formatting.rst
@@ -122,6 +122,8 @@ A complete list of currently supported format string variables is listed below:
 +---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``function.initial-function``                     | Will evaluate to true if this is the start of the first function, as opposed to a change of functions (may be used in ``disassembly-format`` to print the function name for the first function being disassembled)                                                                          |
 +---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| ``function.is-inlined``                           | Will evaluate to true if this function was inlined                                                                                                                                                                                                                                          |
++---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``line.file.basename``                            | The line table entry basename to the file for the current line entry in the current frame.                                                                                                                                                                                                  |
 +---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``line.file.fullpath``                            | The line table entry fullpath to the file for the current line entry in the current frame.                                                                                                                                                                                                  |
diff --git a/external/llvm-project/lldb/include/lldb/Core/Debugger.h b/external/llvm-project/lldb/include/lldb/Core/Debugger.h
index c9e5310cded1..d73aba1e3ce5 100644
--- a/external/llvm-project/lldb/include/lldb/Core/Debugger.h
+++ b/external/llvm-project/lldb/include/lldb/Core/Debugger.h
@@ -243,17 +243,17 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   bool GetAutoConfirm() const;
 
-  const FormatEntity::Entry *GetDisassemblyFormat() const;
+  FormatEntity::Entry GetDisassemblyFormat() const;
 
-  const FormatEntity::Entry *GetFrameFormat() const;
+  FormatEntity::Entry GetFrameFormat() const;
 
-  const FormatEntity::Entry *GetFrameFormatUnique() const;
+  FormatEntity::Entry GetFrameFormatUnique() const;
 
   uint64_t GetStopDisassemblyMaxSize() const;
 
-  const FormatEntity::Entry *GetThreadFormat() const;
+  FormatEntity::Entry GetThreadFormat() const;
 
-  const FormatEntity::Entry *GetThreadStopFormat() const;
+  FormatEntity::Entry GetThreadStopFormat() const;
 
   lldb::ScriptLanguage GetScriptLanguage() const;
 
@@ -297,7 +297,7 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   bool GetShowStatusline() const;
 
-  const FormatEntity::Entry *GetStatuslineFormat() const;
+  FormatEntity::Entry GetStatuslineFormat() const;
   bool SetStatuslineFormat(const FormatEntity::Entry &format);
 
   llvm::StringRef GetSeparator() const;
diff --git a/external/llvm-project/lldb/include/lldb/Core/FormatEntity.h b/external/llvm-project/lldb/include/lldb/Core/FormatEntity.h
index 1aed3c6ff9e9..17fee068230b 100644
--- a/external/llvm-project/lldb/include/lldb/Core/FormatEntity.h
+++ b/external/llvm-project/lldb/include/lldb/Core/FormatEntity.h
@@ -104,6 +104,7 @@ struct Entry {
     FunctionInitial,
     FunctionChanged,
     FunctionIsOptimized,
+    FunctionIsInlined,
     LineEntryFile,
     LineEntryLineNumber,
     LineEntryColumn,
@@ -205,6 +206,8 @@ struct Entry {
     return true;
   }
 
+  operator bool() const { return type != Type::Invalid; }
+
   std::vector<Entry> &GetChildren();
 
   std::string string;
@@ -217,7 +220,7 @@ struct Entry {
   size_t level = 0;
   /// @}
 
-  Type type;
+  Type type = Type::Invalid;
   lldb::Format fmt = lldb::eFormatDefault;
   lldb::addr_t number = 0;
   bool deref = false;
diff --git a/external/llvm-project/lldb/include/lldb/Core/PluginManager.h b/external/llvm-project/lldb/include/lldb/Core/PluginManager.h
index e2f709ecd2ff..e7b169103111 100644
--- a/external/llvm-project/lldb/include/lldb/Core/PluginManager.h
+++ b/external/llvm-project/lldb/include/lldb/Core/PluginManager.h
@@ -19,10 +19,13 @@
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-private-interfaces.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
 
 #include <cstddef>
 #include <cstdint>
+#include <functional>
 #include <vector>
 
 #define LLDB_PLUGIN_DEFINE_ADV(ClassName, PluginName)                          \
@@ -55,12 +58,67 @@ struct RegisteredPluginInfo {
   bool enabled = false;
 };
 
+// Define some data structures to describe known plugin "namespaces".
+// The PluginManager is organized into a series of static functions
+// that operate on different types of plugins. For example SystemRuntime
+// and ObjectFile plugins.
+//
+// The namespace name is used a prefix when matching plugin names. For example,
+// if we have an "macosx" plugin in the "system-runtime" namespace then we will
+// match a plugin name pattern against the "system-runtime.macosx" name.
+//
+// The plugin namespace here is used so we can operate on all the plugins
+// of a given type so it is easy to enable or disable them as a group.
+using GetPluginInfo = std::function<std::vector<RegisteredPluginInfo>()>;
+using SetPluginEnabled = std::function<bool(llvm::StringRef, bool)>;
+struct PluginNamespace {
+  llvm::StringRef name;
+  GetPluginInfo get_info;
+  SetPluginEnabled set_enabled;
+};
+
 class PluginManager {
 public:
   static void Initialize();
 
   static void Terminate();
 
+  // Support for enabling and disabling plugins.
+
+  // Return the plugins that can be enabled or disabled by the user.
+  static llvm::ArrayRef<PluginNamespace> GetPluginNamespaces();
+
+  // Generate a json object that describes the plugins that are available.
+  // This is a json representation of the plugin info returned by
+  // GetPluginNamespaces().
+  //
+  //    {
+  //       <plugin-namespace>: [
+  //           {
+  //               "enabled": <bool>,
+  //               "name": <plugin-name>,
+  //           },
+  //           ...
+  //       ],
+  //       ...
+  //    }
+  //
+  // If pattern is given it will be used to filter the plugins that are
+  // are returned. The pattern filters the plugin names using the
+  // PluginManager::MatchPluginName() function.
+  static llvm::json::Object GetJSON(llvm::StringRef pattern = "");
+
+  // Return true if the pattern matches the plugin name.
+  //
+  // The pattern matches the name if it is exactly equal to the namespace name
+  // or if it is equal to the qualified name, which is the namespace name
+  // followed by a dot and the plugin name (e.g. "system-runtime.foo").
+  //
+  // An empty pattern matches all plugins.
+  static bool MatchPluginName(llvm::StringRef pattern,
+                              const PluginNamespace &plugin_ns,
+                              const RegisteredPluginInfo &plugin);
+
   // ABI
   static bool RegisterPlugin(llvm::StringRef name, llvm::StringRef description,
                              ABICreateInstance create_callback);
@@ -491,6 +549,12 @@ class PluginManager {
   static InstrumentationRuntimeCreateInstance
   GetInstrumentationRuntimeCreateCallbackAtIndex(uint32_t idx);
 
+  static std::vector<RegisteredPluginInfo>
+  GetInstrumentationRuntimePluginInfo();
+
+  static bool SetInstrumentationRuntimePluginEnabled(llvm::StringRef name,
+                                                     bool enabled);
+
   // TypeSystem
   static bool RegisterPlugin(llvm::StringRef name, llvm::StringRef description,
                              TypeSystemCreateInstance create_callback,
diff --git a/external/llvm-project/lldb/include/lldb/Core/Telemetry.h b/external/llvm-project/lldb/include/lldb/Core/Telemetry.h
index 7889cda40e75..1db69723b470 100644
--- a/external/llvm-project/lldb/include/lldb/Core/Telemetry.h
+++ b/external/llvm-project/lldb/include/lldb/Core/Telemetry.h
@@ -102,6 +102,17 @@ struct ClientInfo : public LLDBBaseTelemetryInfo {
   std::string client_data;
   std::optional<std::string> error_msg;
 
+  // For dyn_cast, isa, etc operations.
+  llvm::telemetry::KindType getKind() const override {
+    return LLDBEntryKind::ClientInfo;
+  }
+
+  static bool classof(const llvm::telemetry::TelemetryInfo *t) {
+    // Subclasses of this is also acceptable.
+    return (t->getKind() & LLDBEntryKind::ClientInfo) ==
+           LLDBEntryKind::ClientInfo;
+  }
+
   void serialize(llvm::telemetry::Serializer &serializer) const override;
 };
 
@@ -243,6 +254,8 @@ class TelemetryManager : public llvm::telemetry::Manager {
 protected:
   TelemetryManager(std::unique_ptr<LLDBConfig> config);
 
+  inline const std::string &GetSessionId() const { return m_id; }
+
   static void SetInstance(std::unique_ptr<TelemetryManager> manger);
 
 private:
diff --git a/external/llvm-project/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/external/llvm-project/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index 37f02fb8f7ce..11a4ca2cd833 100644
--- a/external/llvm-project/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/external/llvm-project/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -92,7 +92,7 @@ class SyntheticChildrenFrontEnd {
   lldb::ValueObjectSP
   CreateValueObjectFromAddress(llvm::StringRef name, uint64_t address,
                                const ExecutionContext &exe_ctx,
-                               CompilerType type);
+                               CompilerType type, bool do_deref = true);
 
   lldb::ValueObjectSP CreateValueObjectFromData(llvm::StringRef name,
                                                 const DataExtractor &data,
diff --git a/external/llvm-project/lldb/include/lldb/Host/JSONTransport.h b/external/llvm-project/lldb/include/lldb/Host/JSONTransport.h
new file mode 100644
index 000000000000..4087cdf2b42f
--- /dev/null
+++ b/external/llvm-project/lldb/include/lldb/Host/JSONTransport.h
@@ -0,0 +1,141 @@
+//===-- JSONTransport.h ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Transport layer for encoding and decoding JSON protocol messages.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_HOST_JSONTRANSPORT_H
+#define LLDB_HOST_JSONTRANSPORT_H
+
+#include "lldb/lldb-forward.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
+#include <chrono>
+#include <system_error>
+
+namespace lldb_private {
+
+class TransportEOFError : public llvm::ErrorInfo<TransportEOFError> {
+public:
+  static char ID;
+
+  TransportEOFError() = default;
+
+  void log(llvm::raw_ostream &OS) const override {
+    OS << "transport end of file reached";
+  }
+  std::error_code convertToErrorCode() const override {
+    return llvm::inconvertibleErrorCode();
+  }
+};
+
+class TransportTimeoutError : public llvm::ErrorInfo<TransportTimeoutError> {
+public:
+  static char ID;
+
+  TransportTimeoutError() = default;
+
+  void log(llvm::raw_ostream &OS) const override {
+    OS << "transport operation timed out";
+  }
+  std::error_code convertToErrorCode() const override {
+    return std::make_error_code(std::errc::timed_out);
+  }
+};
+
+class TransportInvalidError : public llvm::ErrorInfo<TransportInvalidError> {
+public:
+  static char ID;
+
+  TransportInvalidError() = default;
+
+  void log(llvm::raw_ostream &OS) const override {
+    OS << "transport IO object invalid";
+  }
+  std::error_code convertToErrorCode() const override {
+    return std::make_error_code(std::errc::not_connected);
+  }
+};
+
+/// A transport class that uses JSON for communication.
+class JSONTransport {
+public:
+  JSONTransport(lldb::IOObjectSP input, lldb::IOObjectSP output);
+  virtual ~JSONTransport() = default;
+
+  /// Transport is not copyable.
+  /// @{
+  JSONTransport(const JSONTransport &rhs) = delete;
+  void operator=(const JSONTransport &rhs) = delete;
+  /// @}
+
+  /// Writes a message to the output stream.
+  template <typename T> llvm::Error Write(const T &t) {
+    const std::string message = llvm::formatv("{0}", toJSON(t)).str();
+    return WriteImpl(message);
+  }
+
+  /// Reads the next message from the input stream.
+  template <typename T>
+  llvm::Expected<T> Read(const std::chrono::microseconds &timeout) {
+    llvm::Expected<std::string> message = ReadImpl(timeout);
+    if (!message)
+      return message.takeError();
+    return llvm::json::parse<T>(/*JSON=*/*message);
+  }
+
+protected:
+  virtual void Log(llvm::StringRef message);
+
+  virtual llvm::Error WriteImpl(const std::string &message) = 0;
+  virtual llvm::Expected<std::string>
+  ReadImpl(const std::chrono::microseconds &timeout) = 0;
+
+  lldb::IOObjectSP m_input;
+  lldb::IOObjectSP m_output;
+};
+
+/// A transport class for JSON with a HTTP header.
+class HTTPDelimitedJSONTransport : public JSONTransport {
+public:
+  HTTPDelimitedJSONTransport(lldb::IOObjectSP input, lldb::IOObjectSP output)
+      : JSONTransport(input, output) {}
+  virtual ~HTTPDelimitedJSONTransport() = default;
+
+protected:
+  virtual llvm::Error WriteImpl(const std::string &message) override;
+  virtual llvm::Expected<std::string>
+  ReadImpl(const std::chrono::microseconds &timeout) override;
+
+  // FIXME: Support any header.
+  static constexpr llvm::StringLiteral kHeaderContentLength =
+      "Content-Length: ";
+  static constexpr llvm::StringLiteral kHeaderSeparator = "\r\n\r\n";
+};
+
+/// A transport class for JSON RPC.
+class JSONRPCTransport : public JSONTransport {
+public:
+  JSONRPCTransport(lldb::IOObjectSP input, lldb::IOObjectSP output)
+      : JSONTransport(input, output) {}
+  virtual ~JSONRPCTransport() = default;
+
+protected:
+  virtual llvm::Error WriteImpl(const std::string &message) override;
+  virtual llvm::Expected<std::string>
+  ReadImpl(const std::chrono::microseconds &timeout) override;
+
+  static constexpr llvm::StringLiteral kMessageSeparator = "\n";
+};
+
+} // namespace lldb_private
+
+#endif
diff --git a/external/llvm-project/lldb/include/lldb/Host/aix/Support.h b/external/llvm-project/lldb/include/lldb/Host/aix/Support.h
new file mode 100644
index 000000000000..f02a1904b09f
--- /dev/null
+++ b/external/llvm-project/lldb/include/lldb/Host/aix/Support.h
@@ -0,0 +1,23 @@
+//===-- Support.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_HOST_AIX_SUPPORT_H
+#define LLDB_HOST_AIX_SUPPORT_H
+
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <memory>
+
+namespace lldb_private {
+
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+getProcFile(::pid_t pid, ::pid_t tid, const llvm::Twine &file);
+
+} // namespace lldb_private
+
+#endif // #ifndef LLDB_HOST_AIX_SUPPORT_H
diff --git a/external/llvm-project/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h b/external/llvm-project/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h
index 1875ff6a048d..8535dfcf46da 100644
--- a/external/llvm-project/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h
+++ b/external/llvm-project/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h
@@ -314,6 +314,7 @@ static constexpr CommandObject::ArgumentTableEntry g_argument_table[] = {
     { lldb::eArgTypeModule, "module", lldb::CompletionType::eModuleCompletion, {}, { nullptr, false }, "The name of a module loaded into the current target." },
     { lldb::eArgTypeCPUName, "cpu-name", lldb::CompletionType::eNoCompletion, {}, { nullptr, false }, "The name of a CPU." },
     { lldb::eArgTypeCPUFeatures, "cpu-features", lldb::CompletionType::eNoCompletion, {}, { nullptr, false }, "The CPU feature string." },
+    { lldb::eArgTypeManagedPlugin, "managed-plugin", lldb::CompletionType::eNoCompletion, {}, { nullptr, false }, "Plugins managed by the PluginManager" },
     // clang-format on
 };
 
diff --git a/external/llvm-project/lldb/include/lldb/Interpreter/OptionValue.h b/external/llvm-project/lldb/include/lldb/Interpreter/OptionValue.h
index e3c139155b0e..f293a3a33bfa 100644
--- a/external/llvm-project/lldb/include/lldb/Interpreter/OptionValue.h
+++ b/external/llvm-project/lldb/include/lldb/Interpreter/OptionValue.h
@@ -284,6 +284,8 @@ class OptionValue {
       return GetStringValue();
     if constexpr (std::is_same_v<T, ArchSpec>)
       return GetArchSpecValue();
+    if constexpr (std::is_same_v<T, FormatEntity::Entry>)
+      return GetFormatEntityValue();
     if constexpr (std::is_enum_v<T>)
       if (std::optional<int64_t> value = GetEnumerationValue())
         return static_cast<T>(*value);
@@ -295,11 +297,9 @@ class OptionValue {
                 typename std::remove_pointer<T>::type>::type,
             std::enable_if_t<std::is_pointer_v<T>, bool> = true>
   T GetValueAs() const {
-    if constexpr (std::is_same_v<U, FormatEntity::Entry>)
-      return GetFormatEntity();
-    if constexpr (std::is_same_v<U, RegularExpression>)
-      return GetRegexValue();
-    return {};
+    static_assert(std::is_same_v<U, RegularExpression>,
+                  "only for RegularExpression");
+    return GetRegexValue();
   }
 
   bool SetValueAs(bool v) { return SetBooleanValue(v); }
@@ -382,7 +382,7 @@ class OptionValue {
   std::optional<UUID> GetUUIDValue() const;
   bool SetUUIDValue(const UUID &uuid);
 
-  const FormatEntity::Entry *GetFormatEntity() const;
+  FormatEntity::Entry GetFormatEntityValue() const;
   bool SetFormatEntityValue(const FormatEntity::Entry &entry);
 
   const RegularExpression *GetRegexValue() const;
diff --git a/external/llvm-project/lldb/include/lldb/Symbol/ObjectFile.h b/external/llvm-project/lldb/include/lldb/Symbol/ObjectFile.h
index 7fca6383fa9f..43567592dd44 100644
--- a/external/llvm-project/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/external/llvm-project/lldb/include/lldb/Symbol/ObjectFile.h
@@ -709,6 +709,13 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
       llvm::StringRef name,
       lldb::SymbolType symbol_type_hint = lldb::eSymbolTypeUndefined);
 
+  /// Parses the section type from a section name for DWARF sections.
+  ///
+  /// The \a name must be stripped of the default prefix (e.g. ".debug_" or
+  /// "__debug_"). If there's no matching section type, \a eSectionTypeOther
+  /// will be returned.
+  static lldb::SectionType GetDWARFSectionTypeFromName(llvm::StringRef name);
+
   /// Loads this objfile to memory.
   ///
   /// Loads the bits needed to create an executable image to the memory. It is
diff --git a/external/llvm-project/lldb/include/lldb/Symbol/SymbolFile.h b/external/llvm-project/lldb/include/lldb/Symbol/SymbolFile.h
index df2f263b18e1..75c7f230ddf3 100644
--- a/external/llvm-project/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/external/llvm-project/lldb/include/lldb/Symbol/SymbolFile.h
@@ -296,7 +296,7 @@ class SymbolFile : public PluginInterface {
                        lldb::SymbolContextItem resolve_scope,
                        SymbolContextList &sc_list);
 
-  virtual void DumpClangAST(Stream &s) {}
+  virtual void DumpClangAST(Stream &s, llvm::StringRef filter) {}
   virtual void FindGlobalVariables(ConstString name,
                                    const CompilerDeclContext &parent_decl_ctx,
                                    uint32_t max_matches,
diff --git a/external/llvm-project/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/external/llvm-project/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
index 6ed389a48880..ba4a7f09afea 100644
--- a/external/llvm-project/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
+++ b/external/llvm-project/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
@@ -127,7 +127,7 @@ class SymbolFileOnDemand : public lldb_private::SymbolFile {
       lldb_private::SymbolContextList &sc_list) override;
 
   void Dump(lldb_private::Stream &s) override;
-  void DumpClangAST(lldb_private::Stream &s) override;
+  void DumpClangAST(lldb_private::Stream &s, llvm::StringRef filter) override;
 
   void
   FindGlobalVariables(lldb_private::ConstString name,
diff --git a/external/llvm-project/lldb/include/lldb/Symbol/TypeSystem.h b/external/llvm-project/lldb/include/lldb/Symbol/TypeSystem.h
index 1f1a3ac4bc56..cb1f0130b548 100644
--- a/external/llvm-project/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/external/llvm-project/lldb/include/lldb/Symbol/TypeSystem.h
@@ -443,7 +443,11 @@ class TypeSystem : public PluginInterface,
   /// given stream.
   ///
   /// This should not modify the state of the TypeSystem if possible.
-  virtual void Dump(llvm::raw_ostream &output) = 0;
+  ///
+  /// \param[out] output Stream to dup the AST into.
+  /// \param[in] filter If empty, dump whole AST. If non-empty, will only
+  /// dump decls whose names contain \c filter.
+  virtual void Dump(llvm::raw_ostream &output, llvm::StringRef filter) = 0;
 
   /// This is used by swift.
   virtual bool IsRuntimeGeneratedType(lldb::opaque_compiler_type_t type) = 0;
diff --git a/external/llvm-project/lldb/include/lldb/Symbol/Variable.h b/external/llvm-project/lldb/include/lldb/Symbol/Variable.h
index c437624d1ea6..5b9c709c8b86 100644
--- a/external/llvm-project/lldb/include/lldb/Symbol/Variable.h
+++ b/external/llvm-project/lldb/include/lldb/Symbol/Variable.h
@@ -89,6 +89,9 @@ class Variable : public UserID, public std::enable_shared_from_this<Variable> {
 
   bool IsInScope(StackFrame *frame);
 
+  /// Returns true if this variable is in scope at `addr` inside `block`.
+  bool IsInScope(const Block &block, const Address &addr);
+
   bool LocationIsValidForFrame(StackFrame *frame);
 
   bool LocationIsValidForAddress(const Address &address);
diff --git a/external/llvm-project/lldb/include/lldb/Target/Language.h b/external/llvm-project/lldb/include/lldb/Target/Language.h
index d62871bd7ed7..3d0aa326d5a6 100644
--- a/external/llvm-project/lldb/include/lldb/Target/Language.h
+++ b/external/llvm-project/lldb/include/lldb/Target/Language.h
@@ -495,9 +495,7 @@ class Language : public PluginInterface {
   /// Python uses \b except. Defaults to \b catch.
   virtual llvm::StringRef GetCatchKeyword() const { return "catch"; }
 
-  virtual const FormatEntity::Entry *GetFunctionNameFormat() const {
-    return nullptr;
-  }
+  virtual FormatEntity::Entry GetFunctionNameFormat() const { return {}; }
 
 protected:
   // Classes that inherit from Language can see and modify these
diff --git a/external/llvm-project/lldb/include/lldb/Target/RegisterContextUnwind.h b/external/llvm-project/lldb/include/lldb/Target/RegisterContextUnwind.h
index 044a387fe5aa..b10a364823b8 100644
--- a/external/llvm-project/lldb/include/lldb/Target/RegisterContextUnwind.h
+++ b/external/llvm-project/lldb/include/lldb/Target/RegisterContextUnwind.h
@@ -151,6 +151,9 @@ class RegisterContextUnwind : public lldb_private::RegisterContext {
       uint32_t lldb_regnum,
       lldb_private::UnwindLLDB::ConcreteRegisterLocation &regloc);
 
+  std::optional<UnwindPlan::Row::AbstractRegisterLocation>
+  GetAbstractRegisterLocation(uint32_t lldb_regnum, lldb::RegisterKind &kind);
+
   bool ReadRegisterValueFromRegisterLocation(
       lldb_private::UnwindLLDB::ConcreteRegisterLocation regloc,
       const lldb_private::RegisterInfo *reg_info,
diff --git a/external/llvm-project/lldb/include/lldb/Target/Statistics.h b/external/llvm-project/lldb/include/lldb/Target/Statistics.h
index 9ac32172e800..2d0d25cd3c75 100644
--- a/external/llvm-project/lldb/include/lldb/Target/Statistics.h
+++ b/external/llvm-project/lldb/include/lldb/Target/Statistics.h
@@ -196,12 +196,21 @@ struct StatisticsOptions {
     return !GetSummaryOnly();
   }
 
+  void SetIncludePlugins(bool value) { m_include_plugins = value; }
+  bool GetIncludePlugins() const {
+    if (m_include_plugins.has_value())
+      return m_include_plugins.value();
+    // Default to true in both default mode and summary mode.
+    return true;
+  }
+
 private:
   std::optional<bool> m_summary_only;
   std::optional<bool> m_load_all_debug_info;
   std::optional<bool> m_include_targets;
   std::optional<bool> m_include_modules;
   std::optional<bool> m_include_transcript;
+  std::optional<bool> m_include_plugins;
 };
 
 /// A class that represents statistics about a TypeSummaryProviders invocations
diff --git a/external/llvm-project/lldb/include/lldb/ValueObject/DILAST.h b/external/llvm-project/lldb/include/lldb/ValueObject/DILAST.h
index 6c7838e05c93..709f0639135f 100644
--- a/external/llvm-project/lldb/include/lldb/ValueObject/DILAST.h
+++ b/external/llvm-project/lldb/include/lldb/ValueObject/DILAST.h
@@ -19,6 +19,7 @@ namespace lldb_private::dil {
 /// The various types DIL AST nodes (used by the DIL parser).
 enum class NodeKind {
   eArraySubscriptNode,
+  eBitExtractionNode,
   eErrorNode,
   eIdentifierNode,
   eMemberOfNode,
@@ -153,6 +154,30 @@ class ArraySubscriptNode : public ASTNode {
   int64_t m_index;
 };
 
+class BitFieldExtractionNode : public ASTNode {
+public:
+  BitFieldExtractionNode(uint32_t location, ASTNodeUP base, int64_t first_index,
+                         int64_t last_index)
+      : ASTNode(location, NodeKind::eBitExtractionNode),
+        m_base(std::move(base)), m_first_index(first_index),
+        m_last_index(last_index) {}
+
+  llvm::Expected<lldb::ValueObjectSP> Accept(Visitor *v) const override;
+
+  ASTNode *GetBase() const { return m_base.get(); }
+  int64_t GetFirstIndex() const { return m_first_index; }
+  int64_t GetLastIndex() const { return m_last_index; }
+
+  static bool classof(const ASTNode *node) {
+    return node->GetKind() == NodeKind::eBitExtractionNode;
+  }
+
+private:
+  ASTNodeUP m_base;
+  int64_t m_first_index;
+  int64_t m_last_index;
+};
+
 /// This class contains one Visit method for each specialized type of
 /// DIL AST node. The Visit methods are used to dispatch a DIL AST node to
 /// the correct function in the DIL expression evaluator for evaluating that
@@ -168,6 +193,8 @@ class Visitor {
   Visit(const UnaryOpNode *node) = 0;
   virtual llvm::Expected<lldb::ValueObjectSP>
   Visit(const ArraySubscriptNode *node) = 0;
+  virtual llvm::Expected<lldb::ValueObjectSP>
+  Visit(const BitFieldExtractionNode *node) = 0;
 };
 
 } // namespace lldb_private::dil
diff --git a/external/llvm-project/lldb/include/lldb/ValueObject/DILEval.h b/external/llvm-project/lldb/include/lldb/ValueObject/DILEval.h
index 9d0fa53c6622..2a0cb548a810 100644
--- a/external/llvm-project/lldb/include/lldb/ValueObject/DILEval.h
+++ b/external/llvm-project/lldb/include/lldb/ValueObject/DILEval.h
@@ -54,6 +54,8 @@ class Interpreter : Visitor {
   llvm::Expected<lldb::ValueObjectSP> Visit(const UnaryOpNode *node) override;
   llvm::Expected<lldb::ValueObjectSP>
   Visit(const ArraySubscriptNode *node) override;
+  llvm::Expected<lldb::ValueObjectSP>
+  Visit(const BitFieldExtractionNode *node) override;
 
   // Used by the interpreter to create objects, perform casts, etc.
   lldb::TargetSP m_target;
diff --git a/external/llvm-project/lldb/include/lldb/ValueObject/DILLexer.h b/external/llvm-project/lldb/include/lldb/ValueObject/DILLexer.h
index 7d70f88f9a71..9c1ba9768025 100644
--- a/external/llvm-project/lldb/include/lldb/ValueObject/DILLexer.h
+++ b/external/llvm-project/lldb/include/lldb/ValueObject/DILLexer.h
@@ -31,6 +31,7 @@ class Token {
     identifier,
     l_paren,
     l_square,
+    minus,
     numeric_constant,
     period,
     r_paren,
diff --git a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObject.h b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObject.h
index 0add8ebeccdc..3c62f3c17619 100644
--- a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObject.h
+++ b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObject.h
@@ -573,10 +573,14 @@ class ValueObject {
   /// child as well.
   void SetName(ConstString name) { m_name = name; }
 
-  virtual lldb::addr_t GetAddressOf(bool scalar_is_load_address = true,
-                                    AddressType *address_type = nullptr);
+  struct AddrAndType {
+    lldb::addr_t address = LLDB_INVALID_ADDRESS;
+    AddressType type = eAddressTypeInvalid;
+  };
+
+  virtual AddrAndType GetAddressOf(bool scalar_is_load_address = true);
 
-  lldb::addr_t GetPointerValue(AddressType *address_type = nullptr);
+  AddrAndType GetPointerValue();
 
   lldb::ValueObjectSP GetSyntheticChild(ConstString key) const;
 
diff --git a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResult.h b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResult.h
index 2ee531f5858e..1e4b81c4dc7f 100644
--- a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResult.h
+++ b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResult.h
@@ -86,8 +86,7 @@ class ValueObjectConstResult : public ValueObject {
 
   lldb::ValueObjectSP AddressOf(Status &error) override;
 
-  lldb::addr_t GetAddressOf(bool scalar_is_load_address = true,
-                            AddressType *address_type = nullptr) override;
+  AddrAndType GetAddressOf(bool scalar_is_load_address = true) override;
 
   size_t GetPointeeData(DataExtractor &data, uint32_t item_idx = 0,
                         uint32_t item_count = 1) override;
diff --git a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultChild.h b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultChild.h
index ad97b885684e..c502ec41dfe9 100644
--- a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultChild.h
+++ b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultChild.h
@@ -48,8 +48,7 @@ class ValueObjectConstResultChild : public ValueObjectChild {
 
   lldb::ValueObjectSP AddressOf(Status &error) override;
 
-  lldb::addr_t GetAddressOf(bool scalar_is_load_address = true,
-                            AddressType *address_type = nullptr) override;
+  AddrAndType GetAddressOf(bool scalar_is_load_address = true) override;
 
   size_t GetPointeeData(DataExtractor &data, uint32_t item_idx = 0,
                         uint32_t item_count = 1) override;
diff --git a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
index 5509886a8965..0d4c976b193d 100644
--- a/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
+++ b/external/llvm-project/lldb/include/lldb/ValueObject/ValueObjectConstResultImpl.h
@@ -10,6 +10,7 @@
 #define LLDB_VALUEOBJECT_VALUEOBJECTCONSTRESULTIMPL_H
 
 #include "lldb/Utility/ConstString.h"
+#include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-private-enumerations.h"
@@ -21,7 +22,6 @@ namespace lldb_private {
 class CompilerType;
 class DataExtractor;
 class Status;
-class ValueObject;
 } // namespace lldb_private
 
 namespace lldb_private {
@@ -58,8 +58,8 @@ class ValueObjectConstResultImpl {
     m_live_address_type = address_type;
   }
 
-  virtual lldb::addr_t GetAddressOf(bool scalar_is_load_address = true,
-                                    AddressType *address_type = nullptr);
+  virtual ValueObject::AddrAndType
+  GetAddressOf(bool scalar_is_load_address = true);
 
   virtual size_t GetPointeeData(DataExtractor &data, uint32_t item_idx = 0,
                                 uint32_t item_count = 1);
diff --git a/external/llvm-project/lldb/include/lldb/lldb-enumerations.h b/external/llvm-project/lldb/include/lldb/lldb-enumerations.h
index 6d10cc8bcffc..eeb7299a354e 100644
--- a/external/llvm-project/lldb/include/lldb/lldb-enumerations.h
+++ b/external/llvm-project/lldb/include/lldb/lldb-enumerations.h
@@ -663,6 +663,7 @@ enum CommandArgumentType {
   eArgTypeModule,
   eArgTypeCPUName,
   eArgTypeCPUFeatures,
+  eArgTypeManagedPlugin,
   eArgTypeLastArg // Always keep this entry as the last entry in this
                   // enumeration!!
 };
diff --git a/external/llvm-project/lldb/packages/Python/lldbsuite/test/decorators.py b/external/llvm-project/lldb/packages/Python/lldbsuite/test/decorators.py
index 868e9f7e5eca..a391319ca9b0 100644
--- a/external/llvm-project/lldb/packages/Python/lldbsuite/test/decorators.py
+++ b/external/llvm-project/lldb/packages/Python/lldbsuite/test/decorators.py
@@ -9,6 +9,7 @@
 import sys
 import tempfile
 import subprocess
+import json
 
 # Third-party modules
 import unittest
@@ -451,24 +452,67 @@ def apple_simulator_test(platform):
     """
     Decorate the test as a test requiring a simulator for a specific platform.
 
-    Consider that a simulator is available if you have the corresponding SDK installed.
-    The SDK identifiers for simulators are iphonesimulator, appletvsimulator, watchsimulator
+    Consider that a simulator is available if you have the corresponding SDK
+    and runtime installed.
+
+    The SDK identifiers for simulators are iphonesimulator, appletvsimulator,
+    watchsimulator
     """
 
     def should_skip_simulator_test():
         if lldbplatformutil.getHostPlatform() not in ["darwin", "macosx"]:
             return "simulator tests are run only on darwin hosts."
+
+        # Make sure we recognize the platform.
+        mapping = {
+            "iphone": "ios",
+            "appletv": "tvos",
+            "watch": "watchos",
+        }
+        if platform not in mapping:
+            return "unknown simulator platform: {}".format(platform)
+
+        # Make sure we have an SDK.
         try:
             output = subprocess.check_output(
                 ["xcodebuild", "-showsdks"], stderr=subprocess.DEVNULL
             ).decode("utf-8")
-            if re.search("%ssimulator" % platform, output):
-                return None
-            else:
+            if not re.search("%ssimulator" % platform, output):
                 return "%s simulator is not supported on this system." % platform
         except subprocess.CalledProcessError:
             return "Simulators are unsupported on this system (xcodebuild failed)"
 
+        # Make sure we a simulator runtime.
+        try:
+            sim_devices_str = subprocess.check_output(
+                ["xcrun", "simctl", "list", "-j", "devices"]
+            ).decode("utf-8")
+
+            sim_devices = json.loads(sim_devices_str)["devices"]
+            for simulator in sim_devices:
+                if isinstance(simulator, dict):
+                    runtime = simulator["name"]
+                    devices = simulator["devices"]
+                else:
+                    runtime = simulator
+                    devices = sim_devices[simulator]
+
+                if not mapping[platform] in runtime.lower():
+                    continue
+
+                for device in devices:
+                    if (
+                        "availability" in device
+                        and device["availability"] == "(available)"
+                    ):
+                        return None
+                    if "isAvailable" in device and device["isAvailable"]:
+                        return None
+
+            return "{} simulator is not supported on this system.".format(platform)
+        except (subprocess.CalledProcessError, json.decoder.JSONDecodeError):
+            return "Simulators are unsupported on this system (simctl failed)"
+
     return skipTestIfFn(should_skip_simulator_test)
 
 
diff --git a/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 6b41aef2bb5b..9786678aa53f 100644
--- a/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -135,6 +135,10 @@ def as_dict(self):
         return source_dict
 
 
+class NotSupportedError(KeyError):
+    """Raised if a feature is not supported due to its capabilities."""
+
+
 class DebugCommunication(object):
     def __init__(
         self,
@@ -153,7 +157,7 @@ def __init__(
         self.recv_thread = threading.Thread(target=self._read_packet_thread)
         self.process_event_body = None
         self.exit_status: Optional[int] = None
-        self.initialize_body = None
+        self.capabilities: dict[str, Any] = {}
         self.progress_events: list[Event] = []
         self.reverse_requests = []
         self.sequence = 1
@@ -300,6 +304,9 @@ def _handle_recv_packet(self, packet: Optional[ProtocolMessage]) -> bool:
             elif event == "breakpoint":
                 # Breakpoint events are sent when a breakpoint is resolved
                 self._update_verified_breakpoints([body["breakpoint"]])
+            elif event == "capabilities":
+                # Update the capabilities with new ones from the event.
+                self.capabilities.update(body["capabilities"])
 
         elif packet_type == "response":
             if packet["command"] == "disconnect":
@@ -308,7 +315,6 @@ def _handle_recv_packet(self, packet: Optional[ProtocolMessage]) -> bool:
         return keepGoing
 
     def _process_continued(self, all_threads_continued: bool):
-        self.threads = None
         self.frame_scopes = {}
         if all_threads_continued:
             self.thread_stop_reasons = {}
@@ -488,13 +494,13 @@ def wait_for_terminated(self, timeout: Optional[float] = None):
             raise ValueError("didn't get terminated event")
         return event_dict
 
-    def get_initialize_value(self, key):
+    def get_capability(self, key):
         """Get a value for the given key if it there is a key/value pair in
-        the "initialize" request response body.
+        the capabilities reported by the adapter.
         """
-        if self.initialize_body and key in self.initialize_body:
-            return self.initialize_body[key]
-        return None
+        if key in self.capabilities:
+            return self.capabilities[key]
+        raise NotSupportedError(key)
 
     def get_threads(self):
         if self.threads is None:
@@ -760,6 +766,9 @@ def request_continue(self, threadId=None, singleThread=False):
         return response
 
     def request_restart(self, restartArguments=None):
+        if self.exit_status is not None:
+            raise ValueError("request_restart called after process exited")
+        self.get_capability("supportsRestartRequest")
         command_dict = {
             "command": "restart",
             "type": "request",
@@ -867,7 +876,7 @@ def request_initialize(self, sourceInitFile=False):
         response = self.send_recv(command_dict)
         if response:
             if "body" in response:
-                self.initialize_body = response["body"]
+                self.capabilities = response["body"]
         return response
 
     def request_launch(
@@ -972,6 +981,7 @@ def request_stepIn(self, threadId, targetId, granularity="statement"):
     def request_stepInTargets(self, frameId):
         if self.exit_status is not None:
             raise ValueError("request_stepInTargets called after process exited")
+        self.get_capability("supportsStepInTargetsRequest")
         args_dict = {"frameId": frameId}
         command_dict = {
             "command": "stepInTargets",
@@ -1180,6 +1190,9 @@ def request_threads(self):
         with information about all threads"""
         command_dict = {"command": "threads", "type": "request", "arguments": {}}
         response = self.send_recv(command_dict)
+        if not response["success"]:
+            self.threads = None
+            return response
         body = response["body"]
         # Fill in "self.threads" correctly so that clients that call
         # self.get_threads() or self.get_thread_id(...) can get information
diff --git a/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py b/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
index e26dbf6ae514..3b54d598c350 100644
--- a/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
+++ b/external/llvm-project/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
@@ -128,6 +128,16 @@ def waitUntil(self, condition_callback):
             time.sleep(0.5)
         return False
 
+    def assertCapabilityIsSet(self, key: str, msg: Optional[str] = None) -> None:
+        """Assert that given capability is set in the client."""
+        self.assertIn(key, self.dap_server.capabilities, msg)
+        self.assertEqual(self.dap_server.capabilities[key], True, msg)
+
+    def assertCapabilityIsNotSet(self, key: str, msg: Optional[str] = None) -> None:
+        """Assert that given capability is not set in the client."""
+        if key in self.dap_server.capabilities:
+            self.assertEqual(self.dap_server.capabilities[key], False, msg)
+
     def verify_breakpoint_hit(self, breakpoint_ids, timeout=DEFAULT_TIMEOUT):
         """Wait for the process we are debugging to stop, and verify we hit
         any breakpoint location in the "breakpoint_ids" array.
diff --git a/external/llvm-project/lldb/scripts/convert-lldb-header-to-rpc-header.py b/external/llvm-project/lldb/scripts/convert-lldb-header-to-rpc-header.py
new file mode 100755
index 000000000000..d7734280076f
--- /dev/null
+++ b/external/llvm-project/lldb/scripts/convert-lldb-header-to-rpc-header.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""
+Usage: convert-lldb-header-to-rpc-header.py <path/to/input-header.h> <path/to/output-header.h>
+
+This scripts takes common LLDB headers (such as lldb-defines.h) and replaces references to LLDB
+with those for RPC. This happens for:
+- namespace definitions
+- namespace usage
+- version string macros
+- ifdef/ifndef lines
+"""
+
+import argparse
+import os
+import re
+
+
+INCLUDES_TO_REMOVE_REGEX = re.compile(
+    r'#include "lldb/lldb-forward.h"|#include "lldb/lldb-versioning.h"'
+)
+LLDB_GUARD_REGEX = re.compile(r"(?P<guard_type>#.+)LLDB_LLDB_\s*", re.M)
+LLDB_API_GUARD_REGEX = re.compile(r"(?P<guard_type>#.+)LLDB_API_\s*", re.M)
+LLDB_VERSION_REGEX = re.compile(r"#define LLDB_VERSION", re.M)
+LLDB_REVISION_REGEX = re.compile(r"#define LLDB_REVISION", re.M)
+LLDB_VERSION_STRING_REGEX = re.compile(r"#define LLDB_VERSION_STRING", re.M)
+LLDB_LOCAL_INCLUDE_REGEX = re.compile(r'#include "lldb/lldb-\s*', re.M)
+LLDB_NAMESPACE_DEFINITION_REGEX = re.compile(
+    r"(?P<comment_marker>//\s*){,1}namespace lldb\s{1}", re.M
+)
+LLDB_NAMESPACE_REGEX = re.compile(r"\s*.+lldb::\s*", re.M)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input")
+    parser.add_argument("output")
+    args = parser.parse_args()
+    input_path = str(args.input)
+    output_path = str(args.output)
+    with open(input_path, "r") as input_file:
+        lines = input_file.readlines()
+        file_buffer = "".join(lines)
+
+    with open(output_path, "w") as output_file:
+        # NOTE: We do not use lldb-forward.h or lldb-versioning.h in RPC, so remove
+        # all includes that are found for these files.
+        file_buffer = re.sub(INCLUDES_TO_REMOVE_REGEX, r"", file_buffer)
+
+        # For lldb-rpc-defines.h, replace the ifndef LLDB_LLDB_ portion with LLDB_RPC_ as we're not
+        # using LLDB private definitions in RPC.
+        lldb_guard_matches = LLDB_GUARD_REGEX.finditer(file_buffer)
+        for match in lldb_guard_matches:
+            file_buffer = re.sub(
+                match.group(),
+                r"{0}LLDB_RPC_".format(match.group("guard_type")),
+                file_buffer,
+            )
+
+        # Similarly to lldb-rpc-defines.h, replace the ifndef for LLDB_API in SBDefines.h to LLDB_RPC_API_ for the same reason.
+        lldb_api_guard_matches = LLDB_API_GUARD_REGEX.finditer(file_buffer)
+        for match in lldb_api_guard_matches:
+            file_buffer = re.sub(
+                match.group(),
+                r"{0}LLDB_RPC_API_".format(match.group("guard_type")),
+                file_buffer,
+            )
+
+        # Replace the references for the macros that define the versioning strings in
+        # lldb-rpc-defines.h.
+        # NOTE: Here we assume that the versioning info has already been uncommented and
+        # populated from the original lldb-defines.h.
+        file_buffer = re.sub(
+            LLDB_VERSION_REGEX, r"#define LLDB_RPC_VERSION", file_buffer
+        )
+        file_buffer = re.sub(
+            LLDB_REVISION_REGEX, r"#define LLDB_RPC_REVISION", file_buffer
+        )
+        file_buffer = re.sub(
+            LLDB_VERSION_STRING_REGEX, r"#define LLDB_RPC_VERSION_STRING", file_buffer
+        )
+
+        # For local #includes
+        file_buffer = re.sub(
+            LLDB_LOCAL_INCLUDE_REGEX, r'#include "lldb-rpc-', file_buffer
+        )
+
+        # Rename the lldb namespace definition to lldb-rpc.
+        lldb_rpc_namespace_definition_matches = (
+            LLDB_NAMESPACE_DEFINITION_REGEX.finditer(file_buffer)
+        )
+        for match in lldb_rpc_namespace_definition_matches:
+            comment_marker = (
+                match.group("comment_marker") if match.group("comment_marker") else ""
+            )
+            file_buffer = re.sub(
+                match.group(),
+                r"{0}namespace lldb_rpc ".format(comment_marker),
+                file_buffer,
+            )
+
+        # Rename the lldb namespace definition to lldb-rpc.
+        file_buffer = re.sub(LLDB_NAMESPACE_REGEX, r"lldb_rpc::", file_buffer)
+
+        output_file.write(file_buffer)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/external/llvm-project/lldb/scripts/framework-header-fix.py b/external/llvm-project/lldb/scripts/framework-header-fix.py
new file mode 100755
index 000000000000..9528fdb7e30b
--- /dev/null
+++ b/external/llvm-project/lldb/scripts/framework-header-fix.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+"""
+Usage: <path/to/input-directory> <path/to/output-directory>
+
+This script is used when building LLDB.framework or LLDBRPC.framework. For each framework, local includes are converted to their respective framework includes.
+
+This script is used in 2 ways:
+1. It is used on header files that are copied into LLDB.framework. For these files, local LLDB includes are converted into framework includes, e.g. #include "lldb/API/SBDefines.h" -> #include <LLDB/SBDefines.h>.
+
+2. It is used on header files for LLDBRPC.framework. For these files, includes of RPC common files will be converted to framework includes, e.g. #include <lldb-rpc/common/RPCCommon.h> -> #include <LLDBRPC/RPCCommon.h>. It will also change local includes to framework includes, e.g. #include "SBAddress.h" -> #include <LLDBRPC/SBAddress.h>
+"""
+
+import argparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# Main header regexes
+INCLUDE_FILENAME_REGEX = re.compile(
+    r'#include "lldb/API/(?P<include_filename>.*){0,1}"'
+)
+
+# RPC header regexes
+RPC_COMMON_REGEX = re.compile(r"#include <lldb-rpc/common/(?P<include_filename>.*)>")
+RPC_INCLUDE_FILENAME_REGEX = re.compile(r'#include "(?P<include_filename>.*)"')
+
+
+def modify_rpc_includes(input_file_path, output_file_path):
+    with open(input_file_path, "r") as input_file:
+        lines = input_file.readlines()
+        file_buffer = "".join(lines)
+        with open(output_file_path, "w") as output_file:
+            # Local includes must be changed to RPC framework level includes.
+            # e.g. #include "SBDefines.h" -> #include <LLDBRPC/SBDefines.h>
+            # Also, RPC common code includes must change to RPC framework level includes.
+            # e.g. #include "lldb-rpc/common/RPCPublic.h" -> #include <LLDBRPC/RPCPublic.h>
+            rpc_common_matches = RPC_COMMON_REGEX.finditer(file_buffer)
+            rpc_include_filename_matches = RPC_INCLUDE_FILENAME_REGEX.finditer(
+                file_buffer
+            )
+            for match in rpc_common_matches:
+                file_buffer = re.sub(
+                    match.group(),
+                    r"#include <LLDBRPC/" + match.group("include_filename") + ">",
+                    file_buffer,
+                )
+            for match in rpc_include_filename_matches:
+                file_buffer = re.sub(
+                    match.group(),
+                    r"#include <LLDBRPC/" + match.group("include_filename") + ">",
+                    file_buffer,
+                )
+            output_file.write(file_buffer)
+
+
+def modify_main_includes(input_file_path, output_file_path):
+    with open(input_file_path, "r") as input_file:
+        lines = input_file.readlines()
+        file_buffer = "".join(lines)
+        with open(output_file_path, "w") as output_file:
+            # Local includes must be changed to framework level includes.
+            # e.g. #include "lldb/API/SBDefines.h" -> #include <LLDB/SBDefines.h>
+            regex_matches = INCLUDE_FILENAME_REGEX.finditer(file_buffer)
+            for match in regex_matches:
+                file_buffer = re.sub(
+                    match.group(),
+                    r"#include <LLDB/" + match.group("include_filename") + ">",
+                    file_buffer,
+                )
+                output_file.write(file_buffer)
+
+
+def remove_guards(output_file_path, unifdef_path, unifdef_guards):
+    # The unifdef path should be passed in from CMake. If it wasn't there in CMake or is incorrect,
+    # find it using shutil. If shutil can't find it, then exit.
+    if not shutil.which(unifdef_path):
+        unifdef_path = shutil.which("unifdef")
+    if not unifdef_path:
+        print(
+            "Unable to find unifdef executable. Guards will not be removed from input files. Exiting..."
+        )
+        sys.exit()
+
+    subprocess_command = (
+        [unifdef_path, "-o", output_file_path] + unifdef_guards + [output_file_path]
+    )
+    subprocess.run(subprocess_command)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--framework", choices=["lldb_main", "lldb_rpc"])
+    parser.add_argument("-i", "--input_file")
+    parser.add_argument("-o", "--output_file")
+    parser.add_argument("-p", "--unifdef_path")
+    parser.add_argument(
+        "unifdef_guards",
+        nargs="+",
+        type=str,
+        help="Guards to be removed with unifdef. These must be specified in the same way as they would be when passed directly into unifdef.",
+    )
+    args = parser.parse_args()
+    input_file_path = str(args.input_file)
+    output_file_path = str(args.output_file)
+    framework_version = args.framework
+    unifdef_path = str(args.unifdef_path)
+    # Prepend dashes to the list of guards passed in from the command line.
+    # unifdef takes the guards to remove as arguments in their own right (e.g. -USWIG)
+    # but passing them in with dashes for this script causes argparse to think that they're
+    # arguments in and of themself, so they need to passed in without dashes.
+    unifdef_guards = ["-" + guard for guard in args.unifdef_guards]
+
+    if framework_version == "lldb_main":
+        modify_main_includes(input_file_path, output_file_path)
+    if framework_version == "lldb_rpc":
+        modify_rpc_includes(input_file_path, output_file_path)
+    # After the incldues have been modified, run unifdef on the headers to remove any guards
+    # specified at the command line.
+    remove_guards(output_file_path, unifdef_path, unifdef_guards)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/external/llvm-project/lldb/scripts/framework-header-fix.sh b/external/llvm-project/lldb/scripts/framework-header-fix.sh
index 3459dd91c9ec..345579c80cdf 100755
--- a/external/llvm-project/lldb/scripts/framework-header-fix.sh
+++ b/external/llvm-project/lldb/scripts/framework-header-fix.sh
@@ -7,11 +7,5 @@ for file in `find $1 -name "*.h"`
 do
   /usr/bin/sed -i.bak 's/\(#include\)[ ]*"lldb\/\(API\/\)\{0,1\}\(.*\)"/\1 <LLDB\/\3>/1' "$file"
   /usr/bin/sed -i.bak 's|<LLDB/Utility|<LLDB|' "$file"
-  LLDB_VERSION=`echo $2 | /usr/bin/sed -E 's/^([0-9]+).([0-9]+).([0-9]+)(.[0-9]+)?$/\\1/g'`
-  LLDB_REVISION=`echo $2 | /usr/bin/sed -E 's/^([0-9]+).([0-9]+).([0-9]+)(.[0-9]+)?$/\\3/g'`
-  LLDB_VERSION_STRING=`echo $2`
-  /usr/bin/sed -i.bak "s|//#define LLDB_VERSION$|#define LLDB_VERSION $LLDB_VERSION |" "$file"
-  /usr/bin/sed -i.bak "s|//#define LLDB_REVISION|#define LLDB_REVISION $LLDB_REVISION |" "$file"
-  /usr/bin/sed -i.bak "s|//#define LLDB_VERSION_STRING|#define LLDB_VERSION_STRING \"$LLDB_VERSION_STRING\" |" "$file"
   rm -f "$file.bak"
 done
diff --git a/external/llvm-project/lldb/scripts/version-header-fix.py b/external/llvm-project/lldb/scripts/version-header-fix.py
new file mode 100755
index 000000000000..fb26ee1579e6
--- /dev/null
+++ b/external/llvm-project/lldb/scripts/version-header-fix.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Usage: <path/to/input-header.h> <path/to/output-header.h> LLDB_MAJOR_VERSION LLDB_MINOR_VERSION LLDB_PATCH_VERSION
+
+This script uncomments and populates the versioning information in lldb-defines.h
+"""
+
+import argparse
+import os
+import re
+
+LLDB_VERSION_REGEX = re.compile(r"//\s*#define LLDB_VERSION\s*$", re.M)
+LLDB_REVISION_REGEX = re.compile(r"//\s*#define LLDB_REVISION\s*$", re.M)
+LLDB_VERSION_STRING_REGEX = re.compile(r"//\s*#define LLDB_VERSION_STRING\s*$", re.M)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input_path")
+    parser.add_argument("output_path")
+    parser.add_argument("lldb_version_major")
+    parser.add_argument("lldb_version_minor")
+    parser.add_argument("lldb_version_patch")
+    args = parser.parse_args()
+    input_path = str(args.input_path)
+    output_path = str(args.output_path)
+    lldb_version_major = args.lldb_version_major
+    lldb_version_minor = args.lldb_version_minor
+    lldb_version_patch = args.lldb_version_patch
+
+    with open(input_path, "r") as input_file:
+        lines = input_file.readlines()
+        file_buffer = "".join(lines)
+
+    with open(output_path, "w") as output_file:
+        # For the defines in lldb-defines.h that define the major, minor and version string
+        # uncomment each define and populate its value using the arguments passed in.
+        # e.g. //#define LLDB_VERSION -> #define LLDB_VERSION <LLDB_MAJOR_VERSION>
+        file_buffer = re.sub(
+            LLDB_VERSION_REGEX,
+            r"#define LLDB_VERSION " + lldb_version_major,
+            file_buffer,
+        )
+
+        file_buffer = re.sub(
+            LLDB_REVISION_REGEX,
+            r"#define LLDB_REVISION " + lldb_version_patch,
+            file_buffer,
+        )
+        file_buffer = re.sub(
+            LLDB_VERSION_STRING_REGEX,
+            r'#define LLDB_VERSION_STRING "{0}.{1}.{2}"'.format(
+                lldb_version_major, lldb_version_minor, lldb_version_patch
+            ),
+            file_buffer,
+        )
+        output_file.write(file_buffer)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/external/llvm-project/lldb/source/API/CMakeLists.txt b/external/llvm-project/lldb/source/API/CMakeLists.txt
index 3bc569608e45..2772aa920411 100644
--- a/external/llvm-project/lldb/source/API/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/API/CMakeLists.txt
@@ -125,9 +125,13 @@ add_lldb_library(liblldb SHARED ${option_framework}
   ${lldb_python_wrapper}
   ${lldb_lua_wrapper}
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/API
   DEPENDS
     lldb-sbapi-dwarf-enums
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -142,8 +146,6 @@ add_lldb_library(liblldb SHARED ${option_framework}
     lldbValueObject
     lldbVersion
     ${LLDB_ALL_PLUGINS}
-  LINK_COMPONENTS
-    Support
 
   ${option_install_prefix}
 )
@@ -290,6 +292,45 @@ else()
   endif()
 endif()
 
+# Stage all headers in the include directory in the build dir.
+file(GLOB public_headers ${LLDB_SOURCE_DIR}/include/lldb/API/*.h)
+set(lldb_header_staging_dir ${CMAKE_BINARY_DIR}/include/lldb)
+file(GLOB root_public_headers ${LLDB_SOURCE_DIR}/include/lldb/lldb-*.h)
+file(GLOB root_private_headers ${LLDB_SOURCE_DIR}/include/lldb/lldb-private*.h)
+list(REMOVE_ITEM root_public_headers ${root_private_headers})
+
+find_program(unifdef_EXECUTABLE unifdef)
+
+foreach(header
+    ${public_headers}
+    ${generated_public_headers}
+    ${root_public_headers})
+  get_filename_component(basename ${header} NAME)
+  set(staged_header ${lldb_header_staging_dir}/${basename})
+
+  if(unifdef_EXECUTABLE)
+    # unifdef returns 0 when the file is unchanged and 1 if something was changed.
+    # That means if we successfully remove SWIG code, the build system believes
+    # that the command has failed and stops. This is undesirable.
+    set(copy_command ${unifdef_EXECUTABLE} -USWIG ${header} > ${staged_header} || (exit 0))
+  else()
+    set(copy_command ${CMAKE_COMMAND} -E copy ${header} ${staged_header})
+  endif()
+
+  add_custom_command(
+    DEPENDS ${header} OUTPUT ${staged_header}
+    COMMAND ${copy_command}
+    COMMENT "LLDB headers: stage LLDB headers in include directory")
+
+  list(APPEND lldb_staged_headers ${staged_header})
+endforeach()
+
+add_custom_command(TARGET liblldb POST_BUILD
+  COMMAND ${LLDB_SOURCE_DIR}/scripts/version-header-fix.py ${LLDB_SOURCE_DIR}/include/lldb/lldb-defines.h ${lldb_header_staging_dir}/lldb-defines.h ${LLDB_VERSION_MAJOR} ${LLDB_VERSION_MINOR} ${LLDB_VERSION_PATCH}
+)
+add_custom_target(liblldb-header-staging DEPENDS ${lldb_staged_headers})
+add_dependencies(liblldb liblldb-header-staging)
+
 if(LLDB_BUILD_FRAMEWORK)
   include(LLDBFramework)
 
diff --git a/external/llvm-project/lldb/source/API/SBValue.cpp b/external/llvm-project/lldb/source/API/SBValue.cpp
index 88c86a548291..d878eb427a18 100644
--- a/external/llvm-project/lldb/source/API/SBValue.cpp
+++ b/external/llvm-project/lldb/source/API/SBValue.cpp
@@ -1336,10 +1336,8 @@ lldb::SBAddress SBValue::GetAddress() {
   if (value_sp) {
     TargetSP target_sp(value_sp->GetTargetSP());
     if (target_sp) {
-      lldb::addr_t value = LLDB_INVALID_ADDRESS;
-      const bool scalar_is_load_address = true;
-      AddressType addr_type;
-      value = value_sp->GetAddressOf(scalar_is_load_address, &addr_type);
+      auto [value, addr_type] =
+          value_sp->GetAddressOf(/*scalar_is_load_address=*/true);
       if (addr_type == eAddressTypeFile) {
         ModuleSP module_sp(value_sp->GetModule());
         if (module_sp)
diff --git a/external/llvm-project/lldb/source/Breakpoint/CMakeLists.txt b/external/llvm-project/lldb/source/Breakpoint/CMakeLists.txt
index 6cd3c396a2c5..de23c8737b32 100644
--- a/external/llvm-project/lldb/source/Breakpoint/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Breakpoint/CMakeLists.txt
@@ -26,6 +26,10 @@ add_lldb_library(lldbBreakpoint NO_PLUGIN_DEPENDENCIES
   WatchpointOptions.cpp
   WatchpointResource.cpp
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Breakpoint
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbExpression
@@ -34,7 +38,4 @@ add_lldb_library(lldbBreakpoint NO_PLUGIN_DEPENDENCIES
     lldbTarget
     lldbUtility
     lldbValueObject
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Breakpoint/WatchpointResource.cpp b/external/llvm-project/lldb/source/Breakpoint/WatchpointResource.cpp
index 49d9d12aadfc..c2f510f03638 100644
--- a/external/llvm-project/lldb/source/Breakpoint/WatchpointResource.cpp
+++ b/external/llvm-project/lldb/source/Breakpoint/WatchpointResource.cpp
@@ -56,8 +56,7 @@ void WatchpointResource::AddConstituent(const WatchpointSP &wp_sp) {
 
 void WatchpointResource::RemoveConstituent(WatchpointSP &wp_sp) {
   std::lock_guard<std::mutex> guard(m_constituents_mutex);
-  const auto &it =
-      std::find(m_constituents.begin(), m_constituents.end(), wp_sp);
+  auto it = llvm::find(m_constituents, wp_sp);
   if (it != m_constituents.end())
     m_constituents.erase(it);
 }
diff --git a/external/llvm-project/lldb/source/Commands/CMakeLists.txt b/external/llvm-project/lldb/source/Commands/CMakeLists.txt
index 71339ccd3a6e..1ea51acec5f1 100644
--- a/external/llvm-project/lldb/source/Commands/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Commands/CMakeLists.txt
@@ -43,6 +43,8 @@ add_lldb_library(lldbCommands NO_PLUGIN_DEPENDENCIES
   CommandOptionsProcessAttach.cpp
   CommandOptionsProcessLaunch.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -55,9 +57,6 @@ add_lldb_library(lldbCommands NO_PLUGIN_DEPENDENCIES
     lldbUtility
     lldbValueObject
     lldbVersion
-
-  LINK_COMPONENTS
-    Support
   )
 
 add_dependencies(lldbCommands LLDBOptionsGen)
diff --git a/external/llvm-project/lldb/source/Commands/CommandObjectBreakpoint.cpp b/external/llvm-project/lldb/source/Commands/CommandObjectBreakpoint.cpp
index 494d6c50e94a..a0c39cf670d4 100644
--- a/external/llvm-project/lldb/source/Commands/CommandObjectBreakpoint.cpp
+++ b/external/llvm-project/lldb/source/Commands/CommandObjectBreakpoint.cpp
@@ -1094,7 +1094,6 @@ class CommandObjectBreakpointList : public CommandObjectParsed {
             interpreter, "breakpoint list",
             "List some or all breakpoints at configurable levels of detail.",
             nullptr) {
-    CommandArgumentEntry arg;
     CommandArgumentData bp_id_arg;
 
     // Define the first (and only) variant of this arg.
diff --git a/external/llvm-project/lldb/source/Commands/CommandObjectMemory.cpp b/external/llvm-project/lldb/source/Commands/CommandObjectMemory.cpp
index 7140333bb3cd..5792c13373c1 100644
--- a/external/llvm-project/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/external/llvm-project/lldb/source/Commands/CommandObjectMemory.cpp
@@ -885,6 +885,61 @@ class CommandObjectMemoryRead : public CommandObjectParsed {
 #define LLDB_OPTIONS_memory_find
 #include "CommandOptions.inc"
 
+static llvm::Error CopyExpressionResult(ValueObject &result,
+                                        DataBufferHeap &buffer,
+                                        ExecutionContextScope *scope) {
+  uint64_t value = result.GetValueAsUnsigned(0);
+  auto size_or_err = result.GetCompilerType().GetByteSize(scope);
+  if (!size_or_err)
+    return size_or_err.takeError();
+
+  switch (*size_or_err) {
+  case 1: {
+    uint8_t byte = (uint8_t)value;
+    buffer.CopyData(&byte, 1);
+  } break;
+  case 2: {
+    uint16_t word = (uint16_t)value;
+    buffer.CopyData(&word, 2);
+  } break;
+  case 4: {
+    uint32_t lword = (uint32_t)value;
+    buffer.CopyData(&lword, 4);
+  } break;
+  case 8: {
+    buffer.CopyData(&value, 8);
+  } break;
+  default:
+    return llvm::createStringError(
+        "Only expressions resulting in 1, 2, 4, or 8-byte-sized values are "
+        "supported. For other pattern sizes the --string (-s) option may be "
+        "used.");
+  }
+
+  return llvm::Error::success();
+}
+
+static llvm::Expected<ValueObjectSP>
+EvaluateExpression(llvm::StringRef expression, StackFrame &frame,
+                   Process &process) {
+  ValueObjectSP result_sp;
+  auto status =
+      process.GetTarget().EvaluateExpression(expression, &frame, result_sp);
+  if (!result_sp)
+    return llvm::createStringError(
+        "No result returned from expression. Exit status: %d", status);
+
+  if (status != eExpressionCompleted)
+    return result_sp->GetError().ToError();
+
+  result_sp = result_sp->GetQualifiedRepresentationIfAvailable(
+      result_sp->GetDynamicValueType(), /*synthValue=*/true);
+  if (!result_sp)
+    return llvm::createStringError("failed to get dynamic result type");
+
+  return result_sp;
+}
+
 // Find the specified data in memory
 class CommandObjectMemoryFind : public CommandObjectParsed {
 public:
@@ -1026,49 +1081,20 @@ class CommandObjectMemoryFind : public CommandObjectParsed {
       }
       buffer.CopyData(str);
     } else if (m_memory_options.m_expr.OptionWasSet()) {
-      StackFrame *frame = m_exe_ctx.GetFramePtr();
-      ValueObjectSP result_sp;
-      if ((eExpressionCompleted ==
-           process->GetTarget().EvaluateExpression(
-               m_memory_options.m_expr.GetValueAs<llvm::StringRef>().value_or(
-                   ""),
-               frame, result_sp)) &&
-          result_sp) {
-        uint64_t value = result_sp->GetValueAsUnsigned(0);
-        std::optional<uint64_t> size = llvm::expectedToOptional(
-            result_sp->GetCompilerType().GetByteSize(nullptr));
-        if (!size)
-          return;
-        switch (*size) {
-        case 1: {
-          uint8_t byte = (uint8_t)value;
-          buffer.CopyData(&byte, 1);
-        } break;
-        case 2: {
-          uint16_t word = (uint16_t)value;
-          buffer.CopyData(&word, 2);
-        } break;
-        case 4: {
-          uint32_t lword = (uint32_t)value;
-          buffer.CopyData(&lword, 4);
-        } break;
-        case 8: {
-          buffer.CopyData(&value, 8);
-        } break;
-        case 3:
-        case 5:
-        case 6:
-        case 7:
-          result.AppendError("unknown type. pass a string instead");
-          return;
-        default:
-          result.AppendError(
-              "result size larger than 8 bytes. pass a string instead");
-          return;
-        }
-      } else {
-        result.AppendError(
-            "expression evaluation failed. pass a string instead");
+      auto result_or_err = EvaluateExpression(
+          m_memory_options.m_expr.GetValueAs<llvm::StringRef>().value_or(""),
+          m_exe_ctx.GetFrameRef(), *process);
+      if (!result_or_err) {
+        result.AppendError("Expression evaluation failed: ");
+        result.AppendError(llvm::toString(result_or_err.takeError()));
+        return;
+      }
+
+      ValueObjectSP result_sp = *result_or_err;
+
+      if (auto err = CopyExpressionResult(*result_sp, buffer,
+                                          m_exe_ctx.GetFramePtr())) {
+        result.AppendError(llvm::toString(std::move(err)));
         return;
       }
     } else {
diff --git a/external/llvm-project/lldb/source/Commands/CommandObjectPlugin.cpp b/external/llvm-project/lldb/source/Commands/CommandObjectPlugin.cpp
index f3108b8a768d..cdc9006bf526 100644
--- a/external/llvm-project/lldb/source/Commands/CommandObjectPlugin.cpp
+++ b/external/llvm-project/lldb/source/Commands/CommandObjectPlugin.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "CommandObjectPlugin.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Host/OptionParser.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 
@@ -46,12 +48,287 @@ class CommandObjectPluginLoad : public CommandObjectParsed {
   }
 };
 
+namespace {
+// Helper function to perform an action on each matching plugin.
+// The action callback is given the containing namespace along with plugin info
+// for each matching plugin.
+static int ActOnMatchingPlugins(
+    const llvm::StringRef pattern,
+    std::function<void(const PluginNamespace &plugin_namespace,
+                       const std::vector<RegisteredPluginInfo> &plugin_info)>
+        action) {
+  int num_matching = 0;
+
+  for (const PluginNamespace &plugin_namespace :
+       PluginManager::GetPluginNamespaces()) {
+
+    std::vector<RegisteredPluginInfo> matching_plugins;
+    for (const RegisteredPluginInfo &plugin_info :
+         plugin_namespace.get_info()) {
+      if (PluginManager::MatchPluginName(pattern, plugin_namespace,
+                                         plugin_info))
+        matching_plugins.push_back(plugin_info);
+    }
+
+    if (!matching_plugins.empty()) {
+      num_matching += matching_plugins.size();
+      action(plugin_namespace, matching_plugins);
+    }
+  }
+
+  return num_matching;
+}
+
+// Call the "SetEnable" function for each matching plugins.
+// Used to share the majority of the code between the enable
+// and disable commands.
+int SetEnableOnMatchingPlugins(const llvm::StringRef &pattern,
+                               CommandReturnObject &result, bool enabled) {
+  return ActOnMatchingPlugins(
+      pattern, [&](const PluginNamespace &plugin_namespace,
+                   const std::vector<RegisteredPluginInfo> &plugins) {
+        result.AppendMessage(plugin_namespace.name);
+        for (const auto &plugin : plugins) {
+          if (!plugin_namespace.set_enabled(plugin.name, enabled)) {
+            result.AppendErrorWithFormat("failed to enable plugin %s.%s",
+                                         plugin_namespace.name.data(),
+                                         plugin.name.data());
+            continue;
+          }
+
+          result.AppendMessageWithFormat(
+              "  %s %-30s %s\n", enabled ? "[+]" : "[-]", plugin.name.data(),
+              plugin.description.data());
+        }
+      });
+}
+
+static std::string ConvertJSONToPrettyString(const llvm::json::Value &json) {
+  std::string str;
+  llvm::raw_string_ostream os(str);
+  os << llvm::formatv("{0:2}", json).str();
+  os.flush();
+  return str;
+}
+
+#define LLDB_OPTIONS_plugin_list
+#include "CommandOptions.inc"
+
+// These option definitions are used by the plugin list command.
+class PluginListCommandOptions : public Options {
+public:
+  PluginListCommandOptions() = default;
+
+  ~PluginListCommandOptions() override = default;
+
+  Status SetOptionValue(uint32_t option_idx, llvm::StringRef option_arg,
+                        ExecutionContext *execution_context) override {
+    Status error;
+    const int short_option = m_getopt_table[option_idx].val;
+
+    switch (short_option) {
+    case 'j':
+      m_json_format = true;
+      break;
+    default:
+      llvm_unreachable("Unimplemented option");
+    }
+
+    return error;
+  }
+
+  void OptionParsingStarting(ExecutionContext *execution_context) override {
+    m_json_format = false;
+  }
+
+  llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
+    return llvm::ArrayRef(g_plugin_list_options);
+  }
+
+  // Instance variables to hold the values for command options.
+  bool m_json_format = false;
+};
+} // namespace
+
+class CommandObjectPluginList : public CommandObjectParsed {
+public:
+  CommandObjectPluginList(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "plugin list",
+                            "Report info about registered LLDB plugins.",
+                            nullptr) {
+    AddSimpleArgumentList(eArgTypeManagedPlugin);
+    SetHelpLong(R"(
+Display information about registered plugins.
+The plugin information is formatted as shown below:
+
+    <plugin-namespace>
+      [+] <plugin-name>                  Plugin #1 description
+      [-] <plugin-name>                  Plugin #2 description
+
+An enabled plugin is marked with [+] and a disabled plugin is marked with [-].
+
+Plugins can be listed by namespace and name with:
+
+  plugin list <plugin-namespace>[.<plugin-name>]
+
+Plugins can be listed by namespace alone or with a fully qualified name. When listed
+with just a namespace all plugins in that namespace are listed.  When no arguments
+are given all plugins are listed.
+
+Examples:
+List all plugins
+
+  (lldb) plugin list
+
+List all plugins in the system-runtime namespace
+
+  (lldb) plugin list system-runtime
+
+List only the plugin 'foo' matching a fully qualified name exactly
+
+  (lldb) plugin list system-runtime.foo
+)");
+  }
+
+  ~CommandObjectPluginList() override = default;
+
+  Options *GetOptions() override { return &m_options; }
+
+protected:
+  void DoExecute(Args &command, CommandReturnObject &result) override {
+    size_t argc = command.GetArgumentCount();
+    result.SetStatus(eReturnStatusSuccessFinishResult);
+
+    // Create a temporary vector to hold the patterns to simplify the logic
+    // for the case when the user passes no patterns
+    std::vector<llvm::StringRef> patterns;
+    patterns.reserve(argc == 0 ? 1 : argc);
+    if (argc == 0)
+      patterns.push_back("");
+    else
+      for (size_t i = 0; i < argc; ++i)
+        patterns.push_back(command[i].ref());
+
+    if (m_options.m_json_format)
+      OutputJsonFormat(patterns, result);
+    else
+      OutputTextFormat(patterns, result);
+  }
+
+private:
+  void OutputJsonFormat(const std::vector<llvm::StringRef> &patterns,
+                        CommandReturnObject &result) {
+    llvm::json::Object obj;
+    bool found_empty = false;
+    for (const llvm::StringRef pattern : patterns) {
+      llvm::json::Object pat_obj = PluginManager::GetJSON(pattern);
+      if (pat_obj.empty()) {
+        found_empty = true;
+        result.AppendErrorWithFormat(
+            "Found no matching plugins for pattern '%s'", pattern.data());
+        break;
+      }
+      for (auto &entry : pat_obj) {
+        obj[entry.first] = std::move(entry.second);
+      }
+    }
+    if (!found_empty) {
+      result.AppendMessage(ConvertJSONToPrettyString(std::move(obj)));
+    }
+  }
+
+  void OutputTextFormat(const std::vector<llvm::StringRef> &patterns,
+                        CommandReturnObject &result) {
+    for (const llvm::StringRef pattern : patterns) {
+      int num_matching = ActOnMatchingPlugins(
+          pattern, [&](const PluginNamespace &plugin_namespace,
+                       const std::vector<RegisteredPluginInfo> &plugins) {
+            result.AppendMessage(plugin_namespace.name);
+            for (auto &plugin : plugins) {
+              result.AppendMessageWithFormat(
+                  "  %s %-30s %s\n", plugin.enabled ? "[+]" : "[-]",
+                  plugin.name.data(), plugin.description.data());
+            }
+          });
+      if (num_matching == 0) {
+        result.AppendErrorWithFormat(
+            "Found no matching plugins for pattern '%s'", pattern.data());
+        break;
+      }
+    }
+  }
+
+  PluginListCommandOptions m_options;
+};
+
+static void DoPluginEnableDisable(Args &command, CommandReturnObject &result,
+                                  bool enable) {
+  const char *name = enable ? "enable" : "disable";
+  size_t argc = command.GetArgumentCount();
+  if (argc == 0) {
+    result.AppendErrorWithFormat("'plugin %s' requires one or more arguments",
+                                 name);
+    return;
+  }
+  result.SetStatus(eReturnStatusSuccessFinishResult);
+
+  for (size_t i = 0; i < argc; ++i) {
+    llvm::StringRef pattern = command[i].ref();
+    int num_matching = SetEnableOnMatchingPlugins(pattern, result, enable);
+
+    if (num_matching == 0) {
+      result.AppendErrorWithFormat(
+          "Found no matching plugins to %s for pattern '%s'", name,
+          pattern.data());
+      break;
+    }
+  }
+}
+
+class CommandObjectPluginEnable : public CommandObjectParsed {
+public:
+  CommandObjectPluginEnable(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "plugin enable",
+                            "Enable registered LLDB plugins.", nullptr) {
+    AddSimpleArgumentList(eArgTypeManagedPlugin);
+  }
+
+  ~CommandObjectPluginEnable() override = default;
+
+protected:
+  void DoExecute(Args &command, CommandReturnObject &result) override {
+    DoPluginEnableDisable(command, result, /*enable=*/true);
+  }
+};
+
+class CommandObjectPluginDisable : public CommandObjectParsed {
+public:
+  CommandObjectPluginDisable(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "plugin disable",
+                            "Disable registered LLDB plugins.", nullptr) {
+    AddSimpleArgumentList(eArgTypeManagedPlugin);
+  }
+
+  ~CommandObjectPluginDisable() override = default;
+
+protected:
+  void DoExecute(Args &command, CommandReturnObject &result) override {
+    DoPluginEnableDisable(command, result, /*enable=*/false);
+  }
+};
+
 CommandObjectPlugin::CommandObjectPlugin(CommandInterpreter &interpreter)
     : CommandObjectMultiword(interpreter, "plugin",
                              "Commands for managing LLDB plugins.",
                              "plugin <subcommand> [<subcommand-options>]") {
   LoadSubCommand("load",
                  CommandObjectSP(new CommandObjectPluginLoad(interpreter)));
+  LoadSubCommand("list",
+                 CommandObjectSP(new CommandObjectPluginList(interpreter)));
+  LoadSubCommand("enable",
+                 CommandObjectSP(new CommandObjectPluginEnable(interpreter)));
+  LoadSubCommand("disable",
+                 CommandObjectSP(new CommandObjectPluginDisable(interpreter)));
 }
 
 CommandObjectPlugin::~CommandObjectPlugin() = default;
diff --git a/external/llvm-project/lldb/source/Commands/CommandObjectProcess.cpp b/external/llvm-project/lldb/source/Commands/CommandObjectProcess.cpp
index d0f5eaf2dfd9..b1f243c9e277 100644
--- a/external/llvm-project/lldb/source/Commands/CommandObjectProcess.cpp
+++ b/external/llvm-project/lldb/source/Commands/CommandObjectProcess.cpp
@@ -1303,7 +1303,7 @@ class CommandObjectProcessSaveCore : public CommandObjectParsed {
         llvm_unreachable("Unimplemented option");
       }
 
-      return {};
+      return error;
     }
 
     void OptionParsingStarting(ExecutionContext *execution_context) override {
diff --git a/external/llvm-project/lldb/source/Commands/CommandObjectStats.cpp b/external/llvm-project/lldb/source/Commands/CommandObjectStats.cpp
index 7d333afc231b..b77c44bdf5d0 100644
--- a/external/llvm-project/lldb/source/Commands/CommandObjectStats.cpp
+++ b/external/llvm-project/lldb/source/Commands/CommandObjectStats.cpp
@@ -103,6 +103,13 @@ class CommandObjectStatsDump : public CommandObjectParsed {
         else
           error = Status::FromError(bool_or_error.takeError());
         break;
+      case 'p':
+        if (llvm::Expected<bool> bool_or_error =
+                OptionArgParser::ToBoolean("--plugins", option_arg))
+          m_stats_options.SetIncludePlugins(*bool_or_error);
+        else
+          error = Status::FromError(bool_or_error.takeError());
+        break;
       default:
         llvm_unreachable("Unimplemented option");
       }
diff --git a/external/llvm-project/lldb/source/Commands/CommandObjectTarget.cpp b/external/llvm-project/lldb/source/Commands/CommandObjectTarget.cpp
index e2ba0008badd..a4ced37649ea 100644
--- a/external/llvm-project/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/external/llvm-project/lldb/source/Commands/CommandObjectTarget.cpp
@@ -2235,11 +2235,23 @@ class CommandObjectTargetModulesDumpClangAST
       : CommandObjectTargetModulesModuleAutoComplete(
             interpreter, "target modules dump ast",
             "Dump the clang ast for a given module's symbol file.",
-            //"target modules dump ast [<file1> ...]")
-            nullptr, eCommandRequiresTarget) {}
+            "target modules dump ast [--filter <name>] [<file1> ...]",
+            eCommandRequiresTarget),
+        m_filter(LLDB_OPT_SET_1, false, "filter", 'f', 0, eArgTypeName,
+                 "Dump only the decls whose names contain the specified filter "
+                 "string.",
+                 /*default_value=*/"") {
+    m_option_group.Append(&m_filter, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
+    m_option_group.Finalize();
+  }
+
+  Options *GetOptions() override { return &m_option_group; }
 
   ~CommandObjectTargetModulesDumpClangAST() override = default;
 
+  OptionGroupOptions m_option_group;
+  OptionGroupString m_filter;
+
 protected:
   void DoExecute(Args &command, CommandReturnObject &result) override {
     Target &target = GetTarget();
@@ -2251,6 +2263,8 @@ class CommandObjectTargetModulesDumpClangAST
       return;
     }
 
+    llvm::StringRef filter = m_filter.GetOptionValue().GetCurrentValueAsRef();
+
     if (command.GetArgumentCount() == 0) {
       // Dump all ASTs for all modules images
       result.GetOutputStream().Format("Dumping clang ast for {0} modules.\n",
@@ -2259,7 +2273,7 @@ class CommandObjectTargetModulesDumpClangAST
         if (INTERRUPT_REQUESTED(GetDebugger(), "Interrupted dumping clang ast"))
           break;
         if (SymbolFile *sf = module_sp->GetSymbolFile())
-          sf->DumpClangAST(result.GetOutputStream());
+          sf->DumpClangAST(result.GetOutputStream(), filter);
       }
       result.SetStatus(eReturnStatusSuccessFinishResult);
       return;
@@ -2288,7 +2302,7 @@ class CommandObjectTargetModulesDumpClangAST
 
         Module *m = module_list.GetModulePointerAtIndex(i);
         if (SymbolFile *sf = m->GetSymbolFile())
-          sf->DumpClangAST(result.GetOutputStream());
+          sf->DumpClangAST(result.GetOutputStream(), filter);
       }
     }
     result.SetStatus(eReturnStatusSuccessFinishResult);
@@ -4871,9 +4885,9 @@ class CommandObjectTargetStopHookAdd : public CommandObjectParsed,
 Command Based stop-hooks:
 -------------------------
   Stop hooks can run a list of lldb commands by providing one or more
-  --one-line-command options.  The commands will get run in the order they are
-  added.  Or you can provide no commands, in which case you will enter a
-  command editor where you can enter the commands to be run.
+  --one-liner options.  The commands will get run in the order they are added.
+  Or you can provide no commands, in which case you will enter a command editor
+  where you can enter the commands to be run.
 
 Python Based Stop Hooks:
 ------------------------
@@ -5272,7 +5286,7 @@ class CommandObjectTargetDumpTypesystem : public CommandObjectParsed {
     // Go over every scratch TypeSystem and dump to the command output.
     for (lldb::TypeSystemSP ts : GetTarget().GetScratchTypeSystems())
       if (ts)
-        ts->Dump(result.GetOutputStream().AsRawOstream());
+        ts->Dump(result.GetOutputStream().AsRawOstream(), "");
 
     result.SetStatus(eReturnStatusSuccessFinishResult);
   }
diff --git a/external/llvm-project/lldb/source/Commands/CommandObjectWatchpoint.cpp b/external/llvm-project/lldb/source/Commands/CommandObjectWatchpoint.cpp
index 20f4b91f1534..e79c3b8939fa 100644
--- a/external/llvm-project/lldb/source/Commands/CommandObjectWatchpoint.cpp
+++ b/external/llvm-project/lldb/source/Commands/CommandObjectWatchpoint.cpp
@@ -822,7 +822,6 @@ corresponding to the byte size of the data type.");
 
     // We passed the sanity check for the command. Proceed to set the
     // watchpoint now.
-    lldb::addr_t addr = 0;
     size_t size = 0;
 
     VariableSP var_sp;
@@ -861,19 +860,7 @@ corresponding to the byte size of the data type.");
 
     CompilerType compiler_type;
 
-    if (valobj_sp) {
-      AddressType addr_type;
-      addr = valobj_sp->GetAddressOf(false, &addr_type);
-      if (addr_type == eAddressTypeLoad) {
-        // We're in business.
-        // Find out the size of this variable.
-        size =
-            m_option_watchpoint.watch_size.GetCurrentValue() == 0
-                ? llvm::expectedToOptional(valobj_sp->GetByteSize()).value_or(0)
-                : m_option_watchpoint.watch_size.GetCurrentValue();
-      }
-      compiler_type = valobj_sp->GetCompilerType();
-    } else {
+    if (!valobj_sp) {
       const char *error_cstr = error.AsCString(nullptr);
       if (error_cstr)
         result.AppendError(error_cstr);
@@ -883,6 +870,16 @@ corresponding to the byte size of the data type.");
                                      command.GetArgumentAtIndex(0));
       return;
     }
+    auto [addr, addr_type] = valobj_sp->GetAddressOf(false);
+    if (addr_type == eAddressTypeLoad) {
+      // We're in business.
+      // Find out the size of this variable.
+      size =
+          m_option_watchpoint.watch_size.GetCurrentValue() == 0
+              ? llvm::expectedToOptional(valobj_sp->GetByteSize()).value_or(0)
+              : m_option_watchpoint.watch_size.GetCurrentValue();
+    }
+    compiler_type = valobj_sp->GetCompilerType();
 
     // Now it's time to create the watchpoint.
     uint32_t watch_type = 0;
diff --git a/external/llvm-project/lldb/source/Commands/Options.td b/external/llvm-project/lldb/source/Commands/Options.td
index 848d42daeb2a..75bdffe16c52 100644
--- a/external/llvm-project/lldb/source/Commands/Options.td
+++ b/external/llvm-project/lldb/source/Commands/Options.td
@@ -683,6 +683,11 @@ let Command = "platform shell" in {
     Desc<"Shell interpreter path. This is the binary used to run the command.">;
 }
 
+let Command = "plugin list" in {
+  def plugin_list_json : Option<"json", "j">,
+                         Desc<"Output the plugin list in json format.">;
+}
+
 let Command = "process launch" in {
   def process_launch_stop_at_entry : Option<"stop-at-entry", "s">,
     Desc<"Stop at the entry point of the program when launching a process.">;
@@ -1484,5 +1489,11 @@ let Command = "statistics dump" in {
     "scripts executed during a debug session. "
     "Defaults to true, unless the '--summary' mode is enabled, in which case "
     "this is turned off unless specified.">;
-
+  def statistics_dump_plugins
+      : Option<"plugins", "p">,
+        Group<1>,
+        Arg<"Boolean">,
+        Desc<"Dump statistics for known plugins including name, order, and "
+             "enabled state. Defaults to true for both summary and default "
+             "mode.">;
 }
diff --git a/external/llvm-project/lldb/source/Core/CMakeLists.txt b/external/llvm-project/lldb/source/Core/CMakeLists.txt
index c4c442dcb204..d6b75bca7f2d 100644
--- a/external/llvm-project/lldb/source/Core/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Core/CMakeLists.txt
@@ -57,9 +57,16 @@ add_lldb_library(lldbCore NO_PLUGIN_DEPENDENCIES
   ThreadedCommunication.cpp
   UserSettingsController.cpp
   Value.cpp
+
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Core
   DEPENDS
     clang-tablegen-targets
-
+  LINK_COMPONENTS
+    Support
+    Demangle
+    TargetParser
+    Telemetry
   LINK_LIBS
     lldbBreakpoint
     lldbDataFormatters
@@ -75,12 +82,6 @@ add_lldb_library(lldbCore NO_PLUGIN_DEPENDENCIES
 
   CLANG_LIBS
     clangDriver
-
-  LINK_COMPONENTS
-    Support
-    Demangle
-    TargetParser
-    Telemetry
   )
 
 add_dependencies(lldbCore
diff --git a/external/llvm-project/lldb/source/Core/CoreProperties.td b/external/llvm-project/lldb/source/Core/CoreProperties.td
index 78988ce5b732..4d1ea5dfec2e 100644
--- a/external/llvm-project/lldb/source/Core/CoreProperties.td
+++ b/external/llvm-project/lldb/source/Core/CoreProperties.td
@@ -59,7 +59,7 @@ let Definition = "debugger" in {
     Desc<"The default disassembly format string to use when disassembling instruction sequences.">;
   def FrameFormat: Property<"frame-format", "FormatEntity">,
     Global,
-    DefaultStringValue<"frame #${frame.index}: ${ansi.fg.cyan}${frame.pc}${ansi.normal}{ ${module.file.basename}{`${function.name-with-args}{${frame.no-debug}${function.pc-offset}}}}{ at ${ansi.fg.cyan}${line.file.basename}${ansi.normal}:${ansi.fg.yellow}${line.number}${ansi.normal}{:${ansi.fg.yellow}${line.column}${ansi.normal}}}{${function.is-optimized} [opt]}{${frame.is-artificial} [artificial]}\\\\n">,
+    DefaultStringValue<"frame #${frame.index}: ${ansi.fg.cyan}${frame.pc}${ansi.normal}{ ${module.file.basename}{`${function.name-with-args}{${frame.no-debug}${function.pc-offset}}}}{ at ${ansi.fg.cyan}${line.file.basename}${ansi.normal}:${ansi.fg.yellow}${line.number}${ansi.normal}{:${ansi.fg.yellow}${line.column}${ansi.normal}}}{${function.is-optimized} [opt]}{${function.is-inlined} [inlined]}{${frame.is-artificial} [artificial]}\\\\n">,
     Desc<"The default frame format string to use when displaying stack frame information for threads.">;
   def NotiftVoid: Property<"notify-void", "Boolean">,
     Global,
@@ -217,7 +217,7 @@ let Definition = "debugger" in {
     Desc<"If true, LLDB will automatically escape non-printable and escape characters when formatting strings.">;
   def FrameFormatUnique: Property<"frame-format-unique", "FormatEntity">,
     Global,
-    DefaultStringValue<"frame #${frame.index}: ${ansi.fg.cyan}${frame.pc}${ansi.normal}{ ${module.file.basename}{`${function.name-without-args}{${frame.no-debug}${function.pc-offset}}}}{ at ${ansi.fg.cyan}${line.file.basename}${ansi.normal}:${ansi.fg.yellow}${line.number}${ansi.normal}{:${ansi.fg.yellow}${line.column}${ansi.normal}}}{${function.is-optimized} [opt]}{${frame.is-artificial} [artificial]}\\\\n">,
+    DefaultStringValue<"frame #${frame.index}: ${ansi.fg.cyan}${frame.pc}${ansi.normal}{ ${module.file.basename}{`${function.name-without-args}{${frame.no-debug}${function.pc-offset}}}}{ at ${ansi.fg.cyan}${line.file.basename}${ansi.normal}:${ansi.fg.yellow}${line.number}${ansi.normal}{:${ansi.fg.yellow}${line.column}${ansi.normal}}}{${function.is-optimized} [opt]}{${function.is-inlined} [inlined]}{${frame.is-artificial} [artificial]}\\\\n">,
     Desc<"The default frame format string to use when displaying stack frame information for threads from thread backtrace unique.">;
   def ShowAutosuggestion: Property<"show-autosuggestion", "Boolean">,
     Global,
diff --git a/external/llvm-project/lldb/source/Core/Debugger.cpp b/external/llvm-project/lldb/source/Core/Debugger.cpp
index 519a2528ca7e..81037d3def81 100644
--- a/external/llvm-project/lldb/source/Core/Debugger.cpp
+++ b/external/llvm-project/lldb/source/Core/Debugger.cpp
@@ -294,19 +294,19 @@ bool Debugger::GetAutoConfirm() const {
       idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
-const FormatEntity::Entry *Debugger::GetDisassemblyFormat() const {
+FormatEntity::Entry Debugger::GetDisassemblyFormat() const {
   constexpr uint32_t idx = ePropertyDisassemblyFormat;
-  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
+  return GetPropertyAtIndexAs<FormatEntity::Entry>(idx, {});
 }
 
-const FormatEntity::Entry *Debugger::GetFrameFormat() const {
+FormatEntity::Entry Debugger::GetFrameFormat() const {
   constexpr uint32_t idx = ePropertyFrameFormat;
-  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
+  return GetPropertyAtIndexAs<FormatEntity::Entry>(idx, {});
 }
 
-const FormatEntity::Entry *Debugger::GetFrameFormatUnique() const {
+FormatEntity::Entry Debugger::GetFrameFormatUnique() const {
   constexpr uint32_t idx = ePropertyFrameFormatUnique;
-  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
+  return GetPropertyAtIndexAs<FormatEntity::Entry>(idx, {});
 }
 
 uint64_t Debugger::GetStopDisassemblyMaxSize() const {
@@ -350,14 +350,14 @@ void Debugger::SetPrompt(llvm::StringRef p) {
   GetCommandInterpreter().UpdatePrompt(new_prompt);
 }
 
-const FormatEntity::Entry *Debugger::GetThreadFormat() const {
+FormatEntity::Entry Debugger::GetThreadFormat() const {
   constexpr uint32_t idx = ePropertyThreadFormat;
-  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
+  return GetPropertyAtIndexAs<FormatEntity::Entry>(idx, {});
 }
 
-const FormatEntity::Entry *Debugger::GetThreadStopFormat() const {
+FormatEntity::Entry Debugger::GetThreadStopFormat() const {
   constexpr uint32_t idx = ePropertyThreadStopFormat;
-  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
+  return GetPropertyAtIndexAs<FormatEntity::Entry>(idx, {});
 }
 
 lldb::ScriptLanguage Debugger::GetScriptLanguage() const {
@@ -492,9 +492,9 @@ bool Debugger::GetShowStatusline() const {
       idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
-const FormatEntity::Entry *Debugger::GetStatuslineFormat() const {
+FormatEntity::Entry Debugger::GetStatuslineFormat() const {
   constexpr uint32_t idx = ePropertyStatuslineFormat;
-  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
+  return GetPropertyAtIndexAs<FormatEntity::Entry>(idx, {});
 }
 
 bool Debugger::SetStatuslineFormat(const FormatEntity::Entry &format) {
@@ -1536,8 +1536,11 @@ bool Debugger::FormatDisassemblerAddress(const FormatEntity::Entry *format,
   FormatEntity::Entry format_entry;
 
   if (format == nullptr) {
-    if (exe_ctx != nullptr && exe_ctx->HasTargetScope())
-      format = exe_ctx->GetTargetRef().GetDebugger().GetDisassemblyFormat();
+    if (exe_ctx != nullptr && exe_ctx->HasTargetScope()) {
+      format_entry =
+          exe_ctx->GetTargetRef().GetDebugger().GetDisassemblyFormat();
+      format = &format_entry;
+    }
     if (format == nullptr) {
       FormatEntity::Parse("${addr}: ", format_entry);
       format = &format_entry;
diff --git a/external/llvm-project/lldb/source/Core/Disassembler.cpp b/external/llvm-project/lldb/source/Core/Disassembler.cpp
index dce3d59457c0..833e327579a2 100644
--- a/external/llvm-project/lldb/source/Core/Disassembler.cpp
+++ b/external/llvm-project/lldb/source/Core/Disassembler.cpp
@@ -310,8 +310,8 @@ void Disassembler::PrintInstructions(Debugger &debugger, const ArchSpec &arch,
   const FormatEntity::Entry *disassembly_format = nullptr;
   FormatEntity::Entry format;
   if (exe_ctx.HasTargetScope()) {
-    disassembly_format =
-        exe_ctx.GetTargetRef().GetDebugger().GetDisassemblyFormat();
+    format = exe_ctx.GetTargetRef().GetDebugger().GetDisassemblyFormat();
+    disassembly_format = &format;
   } else {
     FormatEntity::Parse("${addr}: ", format);
     disassembly_format = &format;
@@ -1037,8 +1037,8 @@ void InstructionList::Dump(Stream *s, bool show_address, bool show_bytes,
   const FormatEntity::Entry *disassembly_format = nullptr;
   FormatEntity::Entry format;
   if (exe_ctx && exe_ctx->HasTargetScope()) {
-    disassembly_format =
-        exe_ctx->GetTargetRef().GetDebugger().GetDisassemblyFormat();
+    format = exe_ctx->GetTargetRef().GetDebugger().GetDisassemblyFormat();
+    disassembly_format = &format;
   } else {
     FormatEntity::Parse("${addr}: ", format);
     disassembly_format = &format;
diff --git a/external/llvm-project/lldb/source/Core/FormatEntity.cpp b/external/llvm-project/lldb/source/Core/FormatEntity.cpp
index 4dcfa43a7bb0..8e3c3c18863f 100644
--- a/external/llvm-project/lldb/source/Core/FormatEntity.cpp
+++ b/external/llvm-project/lldb/source/Core/FormatEntity.cpp
@@ -124,6 +124,7 @@ constexpr Definition g_function_child_entries[] = {
     Definition("initial-function", EntryType::FunctionInitial),
     Definition("changed", EntryType::FunctionChanged),
     Definition("is-optimized", EntryType::FunctionIsOptimized),
+    Definition("is-inlined", EntryType::FunctionIsInlined),
     Definition("prefix", EntryType::FunctionPrefix),
     Definition("scope", EntryType::FunctionScope),
     Definition("basename", EntryType::FunctionBasename),
@@ -402,6 +403,7 @@ const char *FormatEntity::Entry::TypeToCString(Type t) {
     ENUM_TO_CSTR(FunctionInitial);
     ENUM_TO_CSTR(FunctionChanged);
     ENUM_TO_CSTR(FunctionIsOptimized);
+    ENUM_TO_CSTR(FunctionIsInlined);
     ENUM_TO_CSTR(LineEntryFile);
     ENUM_TO_CSTR(LineEntryLineNumber);
     ENUM_TO_CSTR(LineEntryColumn);
@@ -1279,13 +1281,13 @@ static bool FormatFunctionNameForLanguage(Stream &s,
   if (!language_plugin)
     return false;
 
-  const auto *format = language_plugin->GetFunctionNameFormat();
+  FormatEntity::Entry format = language_plugin->GetFunctionNameFormat();
   if (!format)
     return false;
 
   StreamString name_stream;
   const bool success =
-      FormatEntity::Format(*format, name_stream, sc, exe_ctx, /*addr=*/nullptr,
+      FormatEntity::Format(format, name_stream, sc, exe_ctx, /*addr=*/nullptr,
                            /*valobj=*/nullptr, /*function_changed=*/false,
                            /*initial_function=*/false);
   if (success)
@@ -1928,6 +1930,10 @@ bool FormatEntity::Format(const Entry &entry, Stream &s,
     return is_optimized;
   }
 
+  case Entry::Type::FunctionIsInlined: {
+    return sc && sc->block && sc->block->GetInlinedFunctionInfo();
+  }
+
   case Entry::Type::FunctionInitial:
     return initial_function;
 
diff --git a/external/llvm-project/lldb/source/Core/PluginManager.cpp b/external/llvm-project/lldb/source/Core/PluginManager.cpp
index de815e630883..5d44434033c5 100644
--- a/external/llvm-project/lldb/source/Core/PluginManager.cpp
+++ b/external/llvm-project/lldb/source/Core/PluginManager.cpp
@@ -181,6 +181,56 @@ void PluginManager::Terminate() {
   plugin_map.clear();
 }
 
+llvm::ArrayRef<PluginNamespace> PluginManager::GetPluginNamespaces() {
+  // Currently supported set of plugin namespaces. This will be expanded
+  // over time.
+  static PluginNamespace PluginNamespaces[] = {
+      {"system-runtime", PluginManager::GetSystemRuntimePluginInfo,
+       PluginManager::SetSystemRuntimePluginEnabled},
+      {"instrumentation-runtime",
+       PluginManager::GetInstrumentationRuntimePluginInfo,
+       PluginManager::SetInstrumentationRuntimePluginEnabled}};
+
+  return PluginNamespaces;
+}
+
+llvm::json::Object PluginManager::GetJSON(llvm::StringRef pattern) {
+  llvm::json::Object plugin_stats;
+
+  for (const PluginNamespace &plugin_ns : GetPluginNamespaces()) {
+    llvm::json::Array namespace_stats;
+
+    for (const RegisteredPluginInfo &plugin : plugin_ns.get_info()) {
+      if (MatchPluginName(pattern, plugin_ns, plugin)) {
+        llvm::json::Object plugin_json;
+        plugin_json.try_emplace("name", plugin.name);
+        plugin_json.try_emplace("enabled", plugin.enabled);
+        namespace_stats.emplace_back(std::move(plugin_json));
+      }
+    }
+    if (!namespace_stats.empty())
+      plugin_stats.try_emplace(plugin_ns.name, std::move(namespace_stats));
+  }
+
+  return plugin_stats;
+}
+
+bool PluginManager::MatchPluginName(llvm::StringRef pattern,
+                                    const PluginNamespace &plugin_ns,
+                                    const RegisteredPluginInfo &plugin_info) {
+  // The empty pattern matches all plugins.
+  if (pattern.empty())
+    return true;
+
+  // Check if the pattern matches the namespace.
+  if (pattern == plugin_ns.name)
+    return true;
+
+  // Check if the pattern matches the qualified name.
+  std::string qualified_name = (plugin_ns.name + "." + plugin_info.name).str();
+  return pattern == qualified_name;
+}
+
 template <typename Callback> struct PluginInstance {
   typedef Callback CallbackType;
 
@@ -1513,6 +1563,16 @@ PluginManager::GetInstrumentationRuntimeCreateCallbackAtIndex(uint32_t idx) {
   return GetInstrumentationRuntimeInstances().GetCallbackAtIndex(idx);
 }
 
+std::vector<RegisteredPluginInfo>
+PluginManager::GetInstrumentationRuntimePluginInfo() {
+  return GetInstrumentationRuntimeInstances().GetPluginInfoForAllInstances();
+}
+
+bool PluginManager::SetInstrumentationRuntimePluginEnabled(llvm::StringRef name,
+                                                           bool enable) {
+  return GetInstrumentationRuntimeInstances().SetInstanceEnabled(name, enable);
+}
+
 #pragma mark TypeSystem
 
 struct TypeSystemInstance : public PluginInstance<TypeSystemCreateInstance> {
diff --git a/external/llvm-project/lldb/source/Core/Statusline.cpp b/external/llvm-project/lldb/source/Core/Statusline.cpp
index 8b3c8d1ccfa8..8a8640805cac 100644
--- a/external/llvm-project/lldb/source/Core/Statusline.cpp
+++ b/external/llvm-project/lldb/source/Core/Statusline.cpp
@@ -24,8 +24,8 @@
 #define ANSI_SAVE_CURSOR ESCAPE "7"
 #define ANSI_RESTORE_CURSOR ESCAPE "8"
 #define ANSI_CLEAR_BELOW ESCAPE "[J"
-#define ANSI_SET_SCROLL_ROWS ESCAPE "[0;%ur"
-#define ANSI_TO_START_OF_ROW ESCAPE "[%u;0f"
+#define ANSI_SET_SCROLL_ROWS ESCAPE "[1;%ur"
+#define ANSI_TO_START_OF_ROW ESCAPE "[%u;1f"
 #define ANSI_REVERSE_VIDEO ESCAPE "[7m"
 #define ANSI_UP_ROWS ESCAPE "[%dA"
 
@@ -144,9 +144,9 @@ void Statusline::Redraw(bool update) {
   }
 
   StreamString stream;
-  if (auto *format = m_debugger.GetStatuslineFormat())
-    FormatEntity::Format(*format, stream, &symbol_ctx, &exe_ctx, nullptr,
-                         nullptr, false, false);
+  FormatEntity::Entry format = m_debugger.GetStatuslineFormat();
+  FormatEntity::Format(format, stream, &symbol_ctx, &exe_ctx, nullptr, nullptr,
+                       false, false);
 
-  Draw(std::string(stream.GetString()));
+  Draw(stream.GetString().str());
 }
diff --git a/external/llvm-project/lldb/source/DataFormatters/CMakeLists.txt b/external/llvm-project/lldb/source/DataFormatters/CMakeLists.txt
index 91b10ba9e0ac..5f84e4ef43d6 100644
--- a/external/llvm-project/lldb/source/DataFormatters/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/DataFormatters/CMakeLists.txt
@@ -18,6 +18,10 @@ add_lldb_library(lldbDataFormatters NO_PLUGIN_DEPENDENCIES
   ValueObjectPrinter.cpp
   VectorType.cpp
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/DataFormatters
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbInterpreter
@@ -25,7 +29,4 @@ add_lldb_library(lldbDataFormatters NO_PLUGIN_DEPENDENCIES
     lldbTarget
     lldbUtility
     lldbValueObject
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/DataFormatters/CXXFunctionPointer.cpp b/external/llvm-project/lldb/source/DataFormatters/CXXFunctionPointer.cpp
index d79fb1d678eb..ef0d432a32f7 100644
--- a/external/llvm-project/lldb/source/DataFormatters/CXXFunctionPointer.cpp
+++ b/external/llvm-project/lldb/source/DataFormatters/CXXFunctionPointer.cpp
@@ -24,8 +24,7 @@ using namespace lldb_private::formatters;
 bool lldb_private::formatters::CXXFunctionPointerSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   StreamString sstr;
-  AddressType func_ptr_address_type = eAddressTypeInvalid;
-  addr_t func_ptr_address = valobj.GetPointerValue(&func_ptr_address_type);
+  auto [func_ptr_address, func_ptr_address_type] = valobj.GetPointerValue();
   if (func_ptr_address != 0 && func_ptr_address != LLDB_INVALID_ADDRESS) {
     switch (func_ptr_address_type) {
     case eAddressTypeInvalid:
diff --git a/external/llvm-project/lldb/source/DataFormatters/FormattersHelpers.cpp b/external/llvm-project/lldb/source/DataFormatters/FormattersHelpers.cpp
index 5f5541c35262..d7b058d91c4a 100644
--- a/external/llvm-project/lldb/source/DataFormatters/FormattersHelpers.cpp
+++ b/external/llvm-project/lldb/source/DataFormatters/FormattersHelpers.cpp
@@ -113,15 +113,16 @@ lldb_private::formatters::ExtractIndexFromString(const char *item_name) {
 
 Address
 lldb_private::formatters::GetArrayAddressOrPointerValue(ValueObject &valobj) {
-  lldb::addr_t data_addr = LLDB_INVALID_ADDRESS;
-  AddressType type;
+  ValueObject::AddrAndType data_addr;
 
   if (valobj.IsPointerType())
-    data_addr = valobj.GetPointerValue(&type);
+    data_addr = valobj.GetPointerValue();
   else if (valobj.IsArrayType())
-    data_addr = valobj.GetAddressOf(/*scalar_is_load_address=*/true, &type);
-  if (data_addr != LLDB_INVALID_ADDRESS && type == eAddressTypeFile)
-    return Address(data_addr, valobj.GetModule()->GetSectionList());
+    data_addr = valobj.GetAddressOf(/*scalar_is_load_address=*/true);
 
-  return data_addr;
+  if (data_addr.address != LLDB_INVALID_ADDRESS &&
+      data_addr.type == eAddressTypeFile)
+    return Address(data_addr.address, valobj.GetModule()->GetSectionList());
+
+  return data_addr.address;
 }
diff --git a/external/llvm-project/lldb/source/DataFormatters/TypeFormat.cpp b/external/llvm-project/lldb/source/DataFormatters/TypeFormat.cpp
index f4cb8b46d272..f563ab076ae7 100644
--- a/external/llvm-project/lldb/source/DataFormatters/TypeFormat.cpp
+++ b/external/llvm-project/lldb/source/DataFormatters/TypeFormat.cpp
@@ -80,7 +80,7 @@ bool TypeFormatImpl_Format::FormatObject(ValueObject *valobj,
               Status error;
               WritableDataBufferSP buffer_sp(
                   new DataBufferHeap(max_len + 1, 0));
-              Address address(valobj->GetPointerValue());
+              Address address(valobj->GetPointerValue().address);
               target_sp->ReadCStringFromMemory(
                   address, (char *)buffer_sp->GetBytes(), max_len, error);
               if (error.Success())
diff --git a/external/llvm-project/lldb/source/DataFormatters/TypeSynthetic.cpp b/external/llvm-project/lldb/source/DataFormatters/TypeSynthetic.cpp
index 57009b07dc55..33af0ad63077 100644
--- a/external/llvm-project/lldb/source/DataFormatters/TypeSynthetic.cpp
+++ b/external/llvm-project/lldb/source/DataFormatters/TypeSynthetic.cpp
@@ -138,9 +138,9 @@ lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromExpression(
 
 lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromAddress(
     llvm::StringRef name, uint64_t address, const ExecutionContext &exe_ctx,
-    CompilerType type) {
-  ValueObjectSP valobj_sp(
-      ValueObject::CreateValueObjectFromAddress(name, address, exe_ctx, type));
+    CompilerType type, bool do_deref) {
+  ValueObjectSP valobj_sp(ValueObject::CreateValueObjectFromAddress(
+      name, address, exe_ctx, type, do_deref));
   if (valobj_sp)
     valobj_sp->SetSyntheticChildrenGenerated(true);
   return valobj_sp;
diff --git a/external/llvm-project/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/external/llvm-project/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index 5e04a621bbda..40493df8aec3 100644
--- a/external/llvm-project/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/external/llvm-project/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -542,8 +542,7 @@ bool ValueObjectPrinter::ShouldPrintChildren(
   if (is_ptr || is_ref) {
     // We have a pointer or reference whose value is an address. Make sure
     // that address is not NULL
-    AddressType ptr_address_type;
-    if (valobj.GetPointerValue(&ptr_address_type) == 0)
+    if (valobj.GetPointerValue().address == 0)
       return false;
 
     const bool is_root_level = m_curr_depth == 0;
diff --git a/external/llvm-project/lldb/source/Expression/CMakeLists.txt b/external/llvm-project/lldb/source/Expression/CMakeLists.txt
index 9e1c341947e9..08dc536781f9 100644
--- a/external/llvm-project/lldb/source/Expression/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Expression/CMakeLists.txt
@@ -17,9 +17,15 @@ add_lldb_library(lldbExpression NO_PLUGIN_DEPENDENCIES
   UserExpression.cpp
   UtilityFunction.cpp
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Expression
   DEPENDS
   intrinsics_gen
 
+  LINK_COMPONENTS
+    Core
+    ExecutionEngine
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -28,9 +34,4 @@ add_lldb_library(lldbExpression NO_PLUGIN_DEPENDENCIES
     lldbTarget
     lldbUtility
     lldbValueObject
-
-  LINK_COMPONENTS
-    Core
-    ExecutionEngine
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Expression/DWARFExpression.cpp b/external/llvm-project/lldb/source/Expression/DWARFExpression.cpp
index 4b2b111e08e6..661324338e80 100644
--- a/external/llvm-project/lldb/source/Expression/DWARFExpression.cpp
+++ b/external/llvm-project/lldb/source/Expression/DWARFExpression.cpp
@@ -80,8 +80,8 @@ void DWARFExpression::DumpLocation(Stream *s, lldb::DescriptionLevel level,
   };
   llvm::DIDumpOptions DumpOpts;
   DumpOpts.GetNameForDWARFReg = GetRegName;
-  llvm::DWARFExpression(m_data.GetAsLLVM(), m_data.GetAddressByteSize())
-      .print(s->AsRawOstream(), DumpOpts, nullptr);
+  llvm::DWARFExpression E(m_data.GetAsLLVM(), m_data.GetAddressByteSize());
+  llvm::DWARFExpressionPrinter::print(&E, s->AsRawOstream(), DumpOpts, nullptr);
 }
 
 RegisterKind DWARFExpression::GetRegisterKind() const { return m_reg_kind; }
@@ -155,6 +155,7 @@ GetOpcodeDataSize(const DataExtractor &data, const lldb::offset_t data_offset,
   case DW_OP_APPLE_uninit:
   case DW_OP_PGI_omp_thread_num:
   case DW_OP_hi_user:
+  case DW_OP_GNU_implicit_pointer:
     break;
 
   case DW_OP_addr:
diff --git a/external/llvm-project/lldb/source/Expression/FunctionCaller.cpp b/external/llvm-project/lldb/source/Expression/FunctionCaller.cpp
index 83cac130ba72..6c93d94f691c 100644
--- a/external/llvm-project/lldb/source/Expression/FunctionCaller.cpp
+++ b/external/llvm-project/lldb/source/Expression/FunctionCaller.cpp
@@ -323,8 +323,7 @@ bool FunctionCaller::FetchFunctionResults(ExecutionContext &exe_ctx,
 void FunctionCaller::DeallocateFunctionResults(ExecutionContext &exe_ctx,
                                                lldb::addr_t args_addr) {
   std::list<lldb::addr_t>::iterator pos;
-  pos = std::find(m_wrapper_args_addrs.begin(), m_wrapper_args_addrs.end(),
-                  args_addr);
+  pos = llvm::find(m_wrapper_args_addrs, args_addr);
   if (pos != m_wrapper_args_addrs.end())
     m_wrapper_args_addrs.erase(pos);
 
diff --git a/external/llvm-project/lldb/source/Expression/Materializer.cpp b/external/llvm-project/lldb/source/Expression/Materializer.cpp
index 8d48b5e50041..79c804c6c421 100644
--- a/external/llvm-project/lldb/source/Expression/Materializer.cpp
+++ b/external/llvm-project/lldb/source/Expression/Materializer.cpp
@@ -508,10 +508,8 @@ class EntityVariableBase : public Materializer::Entity {
         return;
       }
     } else {
-      AddressType address_type = eAddressTypeInvalid;
-      const bool scalar_is_load_address = false;
       lldb::addr_t addr_of_valobj =
-          valobj_sp->GetAddressOf(scalar_is_load_address, &address_type);
+          valobj_sp->GetAddressOf(/*scalar_is_load_address=*/false).address;
       if (addr_of_valobj != LLDB_INVALID_ADDRESS) {
         Status write_error;
         map.WritePointerToMemory(load_addr, addr_of_valobj, write_error);
diff --git a/external/llvm-project/lldb/source/Host/CMakeLists.txt b/external/llvm-project/lldb/source/Host/CMakeLists.txt
index 90814b1bed4c..b15d72e61b6e 100644
--- a/external/llvm-project/lldb/source/Host/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Host/CMakeLists.txt
@@ -27,8 +27,9 @@ add_host_subdirectory(common
   common/HostNativeThreadBase.cpp
   common/HostProcess.cpp
   common/HostThread.cpp
-  common/LockFileBase.cpp
+  common/JSONTransport.cpp
   common/LZMA.cpp
+  common/LockFileBase.cpp
   common/MainLoopBase.cpp
   common/MemoryMonitor.cpp
   common/MonitoringProcessLauncher.cpp
@@ -142,6 +143,7 @@ else()
     add_host_subdirectory(aix
       aix/Host.cpp
       aix/HostInfoAIX.cpp
+      aix/Support.cpp
       )
   endif()
 endif()
@@ -176,14 +178,15 @@ endif()
 add_lldb_library(lldbHost NO_PLUGIN_DEPENDENCIES
   ${HOST_SOURCES}
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Host
+  LINK_COMPONENTS
+    Object
+    Support
   LINK_LIBS
     lldbUtility
     ${EXTRA_LIBS}
     ${LLDBObjCLibs}
     ${LLDB_LIBEDIT_LIBS}
-
-  LINK_COMPONENTS
-    Object
-    Support
   )
 
diff --git a/external/llvm-project/lldb/source/Host/aix/Support.cpp b/external/llvm-project/lldb/source/Host/aix/Support.cpp
new file mode 100644
index 000000000000..afd975565ff2
--- /dev/null
+++ b/external/llvm-project/lldb/source/Host/aix/Support.cpp
@@ -0,0 +1,24 @@
+//===-- Support.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/aix/Support.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+lldb_private::getProcFile(::pid_t pid, ::pid_t tid, const llvm::Twine &file) {
+  Log *log = GetLog(LLDBLog::Host);
+  std::string File =
+      ("/proc/" + llvm::Twine(pid) + "/lwp/" + llvm::Twine(tid) + "/" + file)
+          .str();
+  auto Ret = llvm::MemoryBuffer::getFileAsStream(File);
+  if (!Ret)
+    LLDB_LOG(log, "Failed to open {0}: {1}", File, Ret.getError().message());
+  return Ret;
+}
diff --git a/external/llvm-project/lldb/source/Host/common/JSONTransport.cpp b/external/llvm-project/lldb/source/Host/common/JSONTransport.cpp
new file mode 100644
index 000000000000..1a0851d5c436
--- /dev/null
+++ b/external/llvm-project/lldb/source/Host/common/JSONTransport.cpp
@@ -0,0 +1,176 @@
+//===-- JSONTransport.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/JSONTransport.h"
+#include "lldb/Utility/IOObject.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "lldb/Utility/SelectHelper.h"
+#include "lldb/Utility/Status.h"
+#include "lldb/lldb-forward.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include <optional>
+#include <string>
+#include <utility>
+
+using namespace llvm;
+using namespace lldb;
+using namespace lldb_private;
+
+/// ReadFull attempts to read the specified number of bytes. If EOF is
+/// encountered, an empty string is returned.
+static Expected<std::string>
+ReadFull(IOObject &descriptor, size_t length,
+         std::optional<std::chrono::microseconds> timeout = std::nullopt) {
+  if (!descriptor.IsValid())
+    return llvm::make_error<TransportInvalidError>();
+
+  bool timeout_supported = true;
+  // FIXME: SelectHelper does not work with NativeFile on Win32.
+#if _WIN32
+  timeout_supported = descriptor.GetFdType() == IOObject::eFDTypeSocket;
+#endif
+
+  if (timeout && timeout_supported) {
+    SelectHelper sh;
+    sh.SetTimeout(*timeout);
+    sh.FDSetRead(descriptor.GetWaitableHandle());
+    Status status = sh.Select();
+    if (status.Fail()) {
+      // Convert timeouts into a specific error.
+      if (status.GetType() == lldb::eErrorTypePOSIX &&
+          status.GetError() == ETIMEDOUT)
+        return make_error<TransportTimeoutError>();
+      return status.takeError();
+    }
+  }
+
+  std::string data;
+  data.resize(length);
+  Status status = descriptor.Read(data.data(), length);
+  if (status.Fail())
+    return status.takeError();
+
+  // Read returns '' on EOF.
+  if (length == 0)
+    return make_error<TransportEOFError>();
+
+  // Return the actual number of bytes read.
+  return data.substr(0, length);
+}
+
+static Expected<std::string>
+ReadUntil(IOObject &descriptor, StringRef delimiter,
+          std::optional<std::chrono::microseconds> timeout = std::nullopt) {
+  std::string buffer;
+  buffer.reserve(delimiter.size() + 1);
+  while (!llvm::StringRef(buffer).ends_with(delimiter)) {
+    Expected<std::string> next =
+        ReadFull(descriptor, buffer.empty() ? delimiter.size() : 1, timeout);
+    if (auto Err = next.takeError())
+      return std::move(Err);
+    buffer += *next;
+  }
+  return buffer.substr(0, buffer.size() - delimiter.size());
+}
+
+JSONTransport::JSONTransport(IOObjectSP input, IOObjectSP output)
+    : m_input(std::move(input)), m_output(std::move(output)) {}
+
+void JSONTransport::Log(llvm::StringRef message) {
+  LLDB_LOG(GetLog(LLDBLog::Host), "{0}", message);
+}
+
+Expected<std::string>
+HTTPDelimitedJSONTransport::ReadImpl(const std::chrono::microseconds &timeout) {
+  if (!m_input || !m_input->IsValid())
+    return llvm::make_error<TransportInvalidError>();
+
+  IOObject *input = m_input.get();
+  Expected<std::string> message_header =
+      ReadFull(*input, kHeaderContentLength.size(), timeout);
+  if (!message_header)
+    return message_header.takeError();
+  if (*message_header != kHeaderContentLength)
+    return createStringError(formatv("expected '{0}' and got '{1}'",
+                                     kHeaderContentLength, *message_header)
+                                 .str());
+
+  Expected<std::string> raw_length = ReadUntil(*input, kHeaderSeparator);
+  if (!raw_length)
+    return handleErrors(raw_length.takeError(),
+                        [&](const TransportEOFError &E) -> llvm::Error {
+                          return createStringError(
+                              "unexpected EOF while reading header separator");
+                        });
+
+  size_t length;
+  if (!to_integer(*raw_length, length))
+    return createStringError(
+        formatv("invalid content length {0}", *raw_length).str());
+
+  Expected<std::string> raw_json = ReadFull(*input, length);
+  if (!raw_json)
+    return handleErrors(
+        raw_json.takeError(), [&](const TransportEOFError &E) -> llvm::Error {
+          return createStringError("unexpected EOF while reading JSON");
+        });
+
+  Log(llvm::formatv("--> {0}", *raw_json).str());
+
+  return raw_json;
+}
+
+Error HTTPDelimitedJSONTransport::WriteImpl(const std::string &message) {
+  if (!m_output || !m_output->IsValid())
+    return llvm::make_error<TransportInvalidError>();
+
+  Log(llvm::formatv("<-- {0}", message).str());
+
+  std::string Output;
+  raw_string_ostream OS(Output);
+  OS << kHeaderContentLength << message.length() << kHeaderSeparator << message;
+  size_t num_bytes = Output.size();
+  return m_output->Write(Output.data(), num_bytes).takeError();
+}
+
+Expected<std::string>
+JSONRPCTransport::ReadImpl(const std::chrono::microseconds &timeout) {
+  if (!m_input || !m_input->IsValid())
+    return make_error<TransportInvalidError>();
+
+  IOObject *input = m_input.get();
+  Expected<std::string> raw_json =
+      ReadUntil(*input, kMessageSeparator, timeout);
+  if (!raw_json)
+    return raw_json.takeError();
+
+  Log(llvm::formatv("--> {0}", *raw_json).str());
+
+  return *raw_json;
+}
+
+Error JSONRPCTransport::WriteImpl(const std::string &message) {
+  if (!m_output || !m_output->IsValid())
+    return llvm::make_error<TransportInvalidError>();
+
+  Log(llvm::formatv("<-- {0}", message).str());
+
+  std::string Output;
+  llvm::raw_string_ostream OS(Output);
+  OS << message << kMessageSeparator;
+  size_t num_bytes = Output.size();
+  return m_output->Write(Output.data(), num_bytes).takeError();
+}
+
+char TransportEOFError::ID;
+char TransportTimeoutError::ID;
+char TransportInvalidError::ID;
diff --git a/external/llvm-project/lldb/source/Host/macosx/objcxx/CMakeLists.txt b/external/llvm-project/lldb/source/Host/macosx/objcxx/CMakeLists.txt
index cda8269ca9ef..1d7573335b8e 100644
--- a/external/llvm-project/lldb/source/Host/macosx/objcxx/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Host/macosx/objcxx/CMakeLists.txt
@@ -8,13 +8,14 @@ add_lldb_library(lldbHostMacOSXObjCXX NO_PLUGIN_DEPENDENCIES
   HostThreadMacOSX.mm
   MemoryMonitorMacOSX.mm
 
-  LINK_LIBS
-    lldbUtility
-    ${EXTRA_LIBS}
-
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Host/macosx/objcxx
   LINK_COMPONENTS
     Support
     TargetParser
+  LINK_LIBS
+    lldbUtility
+    ${EXTRA_LIBS}
   )
 
 target_compile_options(lldbHostMacOSXObjCXX PRIVATE
diff --git a/external/llvm-project/lldb/source/Host/windows/ProcessRunLock.cpp b/external/llvm-project/lldb/source/Host/windows/ProcessRunLock.cpp
index 9f144b4c918f..ac97c3292a32 100644
--- a/external/llvm-project/lldb/source/Host/windows/ProcessRunLock.cpp
+++ b/external/llvm-project/lldb/source/Host/windows/ProcessRunLock.cpp
@@ -28,10 +28,6 @@ static bool WriteLock(lldb::rwlock_t rwlock) {
   return true;
 }
 
-static bool WriteTryLock(lldb::rwlock_t rwlock) {
-  return !!::TryAcquireSRWLockExclusive(GetLock(rwlock));
-}
-
 static bool WriteUnlock(lldb::rwlock_t rwlock) {
   ::ReleaseSRWLockExclusive(GetLock(rwlock));
   return true;
diff --git a/external/llvm-project/lldb/source/Initialization/CMakeLists.txt b/external/llvm-project/lldb/source/Initialization/CMakeLists.txt
index b6282e162aa1..26343f93ecfe 100644
--- a/external/llvm-project/lldb/source/Initialization/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Initialization/CMakeLists.txt
@@ -11,12 +11,14 @@ add_lldb_library(lldbInitialization
   SystemInitializer.cpp
   SystemLifetimeManager.cpp
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Initialization
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbPluginProcessGDBRemote
     ${EXTRA_PLUGINS}
     ${LLDB_SYSTEM_LIBS}
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Interpreter/CMakeLists.txt b/external/llvm-project/lldb/source/Interpreter/CMakeLists.txt
index 642263a8bda7..8af7373702c3 100644
--- a/external/llvm-project/lldb/source/Interpreter/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Interpreter/CMakeLists.txt
@@ -55,6 +55,10 @@ add_lldb_library(lldbInterpreter NO_PLUGIN_DEPENDENCIES
   Property.cpp
   ScriptInterpreter.cpp
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Interpreter
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbInterpreterInterfaces
     lldbCommands
@@ -63,9 +67,6 @@ add_lldb_library(lldbInterpreter NO_PLUGIN_DEPENDENCIES
     lldbHost
     lldbTarget
     lldbUtility
-
-  LINK_COMPONENTS
-    Support
   )
 
 add_dependencies(lldbInterpreter
diff --git a/external/llvm-project/lldb/source/Interpreter/Interfaces/CMakeLists.txt b/external/llvm-project/lldb/source/Interpreter/Interfaces/CMakeLists.txt
index f44672aa50b7..14fcc71fd9a7 100644
--- a/external/llvm-project/lldb/source/Interpreter/Interfaces/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Interpreter/Interfaces/CMakeLists.txt
@@ -1,10 +1,11 @@
 add_lldb_library(lldbInterpreterInterfaces NO_PLUGIN_DEPENDENCIES
   ScriptedInterfaceUsages.cpp
 
-  LINK_LIBS
-    lldbUtility
-
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Interpreter/Interfaces
   LINK_COMPONENTS
     Support
+  LINK_LIBS
+    lldbUtility
   )
 
diff --git a/external/llvm-project/lldb/source/Interpreter/OptionValue.cpp b/external/llvm-project/lldb/source/Interpreter/OptionValue.cpp
index 28bc57a07ac7..aa118107f27c 100644
--- a/external/llvm-project/lldb/source/Interpreter/OptionValue.cpp
+++ b/external/llvm-project/lldb/source/Interpreter/OptionValue.cpp
@@ -380,11 +380,11 @@ bool OptionValue::SetLanguageValue(lldb::LanguageType new_language) {
   return false;
 }
 
-const FormatEntity::Entry *OptionValue::GetFormatEntity() const {
+FormatEntity::Entry OptionValue::GetFormatEntityValue() const {
   std::lock_guard<std::mutex> lock(m_mutex);
   if (const OptionValueFormatEntity *option_value = GetAsFormatEntity())
-    return &option_value->GetCurrentValue();
-  return nullptr;
+    return option_value->GetCurrentValue();
+  return {};
 }
 
 const RegularExpression *OptionValue::GetRegexValue() const {
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp b/external/llvm-project/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp
index b9e7c698cdec..e40d2c5fc121 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp
@@ -12,6 +12,7 @@
 #include "ABIMacOSX_arm64.h"
 #include "ABISysV_arm64.h"
 #include "Utility/ARM64_DWARF_Registers.h"
+#include "Utility/ARM64_ehframe_Registers.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Target/Process.h"
 
@@ -69,9 +70,9 @@ lldb::addr_t ABIAArch64::FixDataAddress(lldb::addr_t pc) {
 std::pair<uint32_t, uint32_t>
 ABIAArch64::GetEHAndDWARFNums(llvm::StringRef name) {
   if (name == "pc")
-    return {LLDB_INVALID_REGNUM, arm64_dwarf::pc};
+    return {arm64_ehframe::pc, arm64_dwarf::pc};
   if (name == "cpsr")
-    return {LLDB_INVALID_REGNUM, arm64_dwarf::cpsr};
+    return {arm64_ehframe::cpsr, arm64_dwarf::cpsr};
   return MCBasedABI::GetEHAndDWARFNums(name);
 }
 
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt
index e1e555249450..f3bc7bcdc420 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/AArch64/CMakeLists.txt
@@ -3,12 +3,12 @@ add_lldb_library(lldbPluginABIAArch64 PLUGIN
   ABIMacOSX_arm64.cpp
   ABISysV_arm64.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/ARC/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/ARC/CMakeLists.txt
index 8654461a6e7b..bcaa3fffc30b 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/ARC/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/ARC/CMakeLists.txt
@@ -1,13 +1,13 @@
 add_lldb_library(lldbPluginABIARC PLUGIN
   ABISysV_arc.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/ARM/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/ARM/CMakeLists.txt
index f73848947e5d..e2ae014157f7 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/ARM/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/ARM/CMakeLists.txt
@@ -3,13 +3,13 @@ add_lldb_library(lldbPluginABIARM PLUGIN
   ABIMacOSX_arm.cpp
   ABISysV_arm.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt
index 2bfc08d62a88..63b32c2c68f2 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/Hexagon/CMakeLists.txt
@@ -1,12 +1,12 @@
 add_lldb_library(lldbPluginABIHexagon PLUGIN
   ABISysV_hexagon.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/LoongArch/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/LoongArch/CMakeLists.txt
index 11ac433f8600..a9a116549f80 100755
--- a/external/llvm-project/lldb/source/Plugins/ABI/LoongArch/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/LoongArch/CMakeLists.txt
@@ -1,12 +1,12 @@
 add_lldb_library(lldbPluginABILoongArch PLUGIN
   ABISysV_loongarch.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt
index 6aff7a2f7bf2..fd7bdb892c71 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/MSP430/CMakeLists.txt
@@ -1,13 +1,13 @@
 add_lldb_library(lldbPluginABIMSP430 PLUGIN
   ABISysV_msp430.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
 
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/Mips/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/Mips/CMakeLists.txt
index c7d0cc69d727..42f2a9772771 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/Mips/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/Mips/CMakeLists.txt
@@ -3,12 +3,12 @@ add_lldb_library(lldbPluginABIMips PLUGIN
   ABISysV_mips.cpp
   ABISysV_mips64.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt
index eff8cd664e4c..bcb19bcf6dd4 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/PowerPC/CMakeLists.txt
@@ -3,13 +3,13 @@ add_lldb_library(lldbPluginABIPowerPC PLUGIN
   ABISysV_ppc.cpp
   ABISysV_ppc64.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginTypeSystemClang
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt
index 5b25bf7c6af8..68518eaaf9ee 100755
--- a/external/llvm-project/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/RISCV/CMakeLists.txt
@@ -1,13 +1,13 @@
 add_lldb_library(lldbPluginABIRISCV PLUGIN
   ABISysV_riscv.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt
index b6164ed87d9d..c281fcef3c8f 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/SystemZ/CMakeLists.txt
@@ -1,12 +1,12 @@
 add_lldb_library(lldbPluginABISystemZ PLUGIN
   ABISysV_s390x.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ABI/X86/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ABI/X86/CMakeLists.txt
index eeab6297e6f2..3f4537e8d058 100644
--- a/external/llvm-project/lldb/source/Plugins/ABI/X86/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ABI/X86/CMakeLists.txt
@@ -6,12 +6,12 @@ add_lldb_library(lldbPluginABIX86 PLUGIN
   ABISysV_x86_64.cpp
   ABIWindows_x86_64.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Architecture/AArch64/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Architecture/AArch64/CMakeLists.txt
index 9bcf99318622..a76239019671 100644
--- a/external/llvm-project/lldb/source/Plugins/Architecture/AArch64/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Architecture/AArch64/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginArchitectureAArch64 PLUGIN
   ArchitectureAArch64.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbPluginProcessUtility
     lldbCore
     lldbTarget
     lldbUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Architecture/Arm/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Architecture/Arm/CMakeLists.txt
index 60bbe69a99fb..b47b285487b3 100644
--- a/external/llvm-project/lldb/source/Plugins/Architecture/Arm/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Architecture/Arm/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginArchitectureArm PLUGIN
   ArchitectureArm.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbPluginProcessUtility
     lldbCore
     lldbTarget
     lldbUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Architecture/Mips/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Architecture/Mips/CMakeLists.txt
index 9734edc6b12b..d4b67766ddc0 100644
--- a/external/llvm-project/lldb/source/Plugins/Architecture/Mips/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Architecture/Mips/CMakeLists.txt
@@ -1,10 +1,10 @@
 add_lldb_library(lldbPluginArchitectureMips PLUGIN
   ArchitectureMips.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbTarget
     lldbUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Architecture/PPC64/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Architecture/PPC64/CMakeLists.txt
index 2cba112cf882..1ca05281265a 100644
--- a/external/llvm-project/lldb/source/Plugins/Architecture/PPC64/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Architecture/PPC64/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginArchitecturePPC64 PLUGIN
   ArchitecturePPC64.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbPluginProcessUtility
     lldbCore
     lldbTarget
     lldbUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Disassembler/LLVMC/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Disassembler/LLVMC/CMakeLists.txt
index 60a091efe416..b498389c2106 100644
--- a/external/llvm-project/lldb/source/Plugins/Disassembler/LLVMC/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Disassembler/LLVMC/CMakeLists.txt
@@ -1,10 +1,6 @@
 add_lldb_library(lldbPluginDisassemblerLLVMC PLUGIN
   DisassemblerLLVMC.cpp
 
-  LINK_LIBS
-    lldbCore
-    lldbSymbol
-    lldbTarget
   LINK_COMPONENTS
     ${LLVM_TARGETS_TO_BUILD}
     MC
@@ -12,4 +8,8 @@ add_lldb_library(lldbPluginDisassemblerLLVMC PLUGIN
     RuntimeDyld
     Support
     TargetParser
+  LINK_LIBS
+    lldbCore
+    lldbSymbol
+    lldbTarget
   )
diff --git a/external/llvm-project/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/CMakeLists.txt
index 7308374c8bfb..20fafb2816dc 100644
--- a/external/llvm-project/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/CMakeLists.txt
@@ -3,6 +3,9 @@ add_lldb_library(lldbPluginDynamicLoaderMacOSXDYLD PLUGIN
   DynamicLoaderMacOS.cpp
   DynamicLoaderDarwin.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -12,7 +15,4 @@ add_lldb_library(lldbPluginDynamicLoaderMacOSXDYLD PLUGIN
     lldbTarget
     lldbUtility
     lldbPluginTypeSystemClang
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/CMakeLists.txt
index c1e00b2dd444..0ee25d7b38d1 100644
--- a/external/llvm-project/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/CMakeLists.txt
@@ -2,6 +2,8 @@ add_lldb_library(lldbPluginDynamicLoaderPosixDYLD PLUGIN
   DYLDRendezvous.cpp
   DynamicLoaderPOSIXDYLD.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -10,6 +12,4 @@ add_lldb_library(lldbPluginDynamicLoaderPosixDYLD PLUGIN
     lldbTarget
     lldbPluginProcessElfCore
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/DynamicLoader/Windows-DYLD/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/DynamicLoader/Windows-DYLD/CMakeLists.txt
index 044d67c6619a..4c2145714081 100644
--- a/external/llvm-project/lldb/source/Plugins/DynamicLoader/Windows-DYLD/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/DynamicLoader/Windows-DYLD/CMakeLists.txt
@@ -1,10 +1,10 @@
 add_lldb_library(lldbPluginDynamicLoaderWindowsDYLD PLUGIN
   DynamicLoaderWindowsDYLD.cpp
 
-  LINK_LIBS
-    lldbCore
-    lldbTarget
   LINK_COMPONENTS
     Support
     TargetParser
+  LINK_LIBS
+    lldbCore
+    lldbTarget
   )
diff --git a/external/llvm-project/lldb/source/Plugins/DynamicLoader/wasm-DYLD/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/DynamicLoader/wasm-DYLD/CMakeLists.txt
index a4a4ac7b44e7..0adbcd081acb 100644
--- a/external/llvm-project/lldb/source/Plugins/DynamicLoader/wasm-DYLD/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/DynamicLoader/wasm-DYLD/CMakeLists.txt
@@ -1,9 +1,9 @@
 add_lldb_library(lldbPluginDynamicLoaderWasmDYLD PLUGIN
   DynamicLoaderWasmDYLD.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbTarget
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
index 675cd066507b..2aae7d13449d 100644
--- a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
@@ -29,6 +29,13 @@ add_lldb_library(lldbPluginExpressionParserClang
   DEPENDS
   intrinsics_gen
 
+  LINK_COMPONENTS
+    Core
+    ExecutionEngine
+    ipo
+    MCJIT
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbExpression
@@ -54,11 +61,4 @@ add_lldb_library(lldbPluginExpressionParserClang
     clangRewriteFrontend
     clangSema
     clangSerialization
-  LINK_COMPONENTS
-    Core
-    ExecutionEngine
-    ipo
-    MCJIT
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index fec8d29248c2..9f77fbc1d243 100644
--- a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -1978,10 +1978,10 @@ void ClangExpressionDeclMap::AddContextClassType(NameSearchContext &context,
       copied_clang_type.GetCompleteType()) {
     CompilerType void_clang_type =
         m_clang_ast_context->GetBasicType(eBasicTypeVoid);
-    CompilerType void_ptr_clang_type = void_clang_type.GetPointerType();
+    std::array<CompilerType, 1> args{void_clang_type.GetPointerType()};
 
     CompilerType method_type = m_clang_ast_context->CreateFunctionType(
-        void_clang_type, &void_ptr_clang_type, 1, false, 0);
+        void_clang_type, args, false, 0);
 
     const bool is_virtual = false;
     const bool is_static = false;
diff --git a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
index c99ed9dd0a68..284e5bc6fb1e 100644
--- a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
@@ -382,6 +382,13 @@ bool ClangModulesDeclVendorImpl::AddModule(const SourceModule &module,
     }
   }
 
+  // If we didn't make the submodule visible here, Clang wouldn't allow LLDB to
+  // pick any of the decls in the submodules during C++ name lookup.
+  if (submodule)
+    m_compiler_instance->makeModuleVisible(
+        submodule, clang::Module::NameVisibilityKind::AllVisible,
+        /*ImportLoc=*/{});
+
   clang::Module *requested_module = DoGetModule(clang_path, true);
 
   if (requested_module != nullptr) {
diff --git a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
index ae34a983612f..6b743e29e21f 100644
--- a/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
@@ -897,12 +897,13 @@ bool ClangUserExpression::AddArguments(ExecutionContext &exe_ctx,
     Status object_ptr_error;
 
     if (m_ctx_obj) {
-      AddressType address_type;
-      object_ptr = m_ctx_obj->GetAddressOf(false, &address_type);
-      if (object_ptr == LLDB_INVALID_ADDRESS ||
-          address_type != eAddressTypeLoad)
+      ValueObject::AddrAndType address = m_ctx_obj->GetAddressOf(false);
+      if (address.address == LLDB_INVALID_ADDRESS ||
+          address.type != eAddressTypeLoad)
         object_ptr_error = Status::FromErrorString("Can't get context object's "
                                                    "debuggee address");
+      else
+        object_ptr = address.address;
     } else {
       if (m_in_cplusplus_method) {
         object_ptr =
diff --git a/external/llvm-project/lldb/source/Plugins/Instruction/ARM/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Instruction/ARM/CMakeLists.txt
index 24f803db3d0e..f1e7646c3be2 100644
--- a/external/llvm-project/lldb/source/Plugins/Instruction/ARM/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Instruction/ARM/CMakeLists.txt
@@ -2,6 +2,8 @@ add_lldb_library(lldbPluginInstructionARM PLUGIN
   EmulateInstructionARM.cpp
   EmulationStateARM.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -9,6 +11,4 @@ add_lldb_library(lldbPluginInstructionARM PLUGIN
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Instruction/ARM64/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Instruction/ARM64/CMakeLists.txt
index effe0e3bfe66..975ff7f526e3 100644
--- a/external/llvm-project/lldb/source/Plugins/Instruction/ARM64/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Instruction/ARM64/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginInstructionARM64 PLUGIN
   EmulateInstructionARM64.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbInterpreter
     lldbSymbol
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Instruction/LoongArch/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Instruction/LoongArch/CMakeLists.txt
index 59802ee8fa9a..41dc0f12cf5e 100644
--- a/external/llvm-project/lldb/source/Plugins/Instruction/LoongArch/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Instruction/LoongArch/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginInstructionLoongArch PLUGIN
   EmulateInstructionLoongArch.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbInterpreter
     lldbPluginProcessUtility
     lldbSymbol
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Instruction/MIPS/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Instruction/MIPS/CMakeLists.txt
index d25735dd86c8..193493745d11 100644
--- a/external/llvm-project/lldb/source/Plugins/Instruction/MIPS/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Instruction/MIPS/CMakeLists.txt
@@ -5,13 +5,13 @@ endif()
 add_lldb_library(lldbPluginInstructionMIPS PLUGIN
   EmulateInstructionMIPS.cpp
 
+  LINK_COMPONENTS
+    MC
+    Support
+    ${mips_target}
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    MC
-    Support
-    ${mips_target}
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Instruction/MIPS64/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Instruction/MIPS64/CMakeLists.txt
index ddb964bf50a5..b0711ebae152 100644
--- a/external/llvm-project/lldb/source/Plugins/Instruction/MIPS64/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Instruction/MIPS64/CMakeLists.txt
@@ -5,14 +5,14 @@ endif()
 add_lldb_library(lldbPluginInstructionMIPS64 PLUGIN
   EmulateInstructionMIPS64.cpp
 
+  LINK_COMPONENTS
+    MC
+    Support
+    ${mips_target}
   LINK_LIBS
     lldbCore
     lldbHost
     lldbInterpreter
     lldbSymbol
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    MC
-    Support
-    ${mips_target}
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Instruction/PPC64/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Instruction/PPC64/CMakeLists.txt
index 0926433fc77c..c97211dc8ec0 100644
--- a/external/llvm-project/lldb/source/Plugins/Instruction/PPC64/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Instruction/PPC64/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginInstructionPPC64 PLUGIN
   EmulateInstructionPPC64.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbInterpreter
     lldbSymbol
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Instruction/RISCV/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Instruction/RISCV/CMakeLists.txt
index ed05d4a3132a..4b613f9888b7 100644
--- a/external/llvm-project/lldb/source/Plugins/Instruction/RISCV/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Instruction/RISCV/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginInstructionRISCV PLUGIN
   EmulateInstructionRISCV.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbInterpreter
     lldbSymbol
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/CMakeLists.txt
index 8095658452d1..7f035ad467ee 100644
--- a/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_lldb_library(lldbPluginInstrumentationRuntimeMainThreadChecker PLUGIN
   InstrumentationRuntimeMainThreadChecker.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -8,6 +10,4 @@ add_lldb_library(lldbPluginInstrumentationRuntimeMainThreadChecker PLUGIN
     lldbInterpreter
     lldbSymbol
     lldbTarget
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt
index 156bd3ac454e..5cd59d773028 100644
--- a/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/InstrumentationRuntime/UBSan/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_lldb_library(lldbPluginInstrumentationRuntimeUBSan PLUGIN
   InstrumentationRuntimeUBSan.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -9,6 +11,4 @@ add_lldb_library(lldbPluginInstrumentationRuntimeUBSan PLUGIN
     lldbSymbol
     lldbTarget
     lldbValueObject
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/JITLoader/GDB/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/JITLoader/GDB/CMakeLists.txt
index ecdff2823d47..b66be412a10d 100644
--- a/external/llvm-project/lldb/source/Plugins/JITLoader/GDB/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/JITLoader/GDB/CMakeLists.txt
@@ -9,6 +9,8 @@ lldb_tablegen(JITLoaderGDBPropertiesEnum.inc -gen-lldb-property-enum-defs
 add_lldb_library(lldbPluginJITLoaderGDB PLUGIN
   JITLoaderGDB.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -17,8 +19,6 @@ add_lldb_library(lldbPluginJITLoaderGDB PLUGIN
     lldbTarget
     lldbUtility
     lldbPluginObjectFileMachO
-  LINK_COMPONENTS
-    Support
   )
 
 add_dependencies(lldbPluginJITLoaderGDB
diff --git a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
index 9bb10c2a792a..5ba2567c80cc 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
@@ -34,6 +34,8 @@ add_lldb_library(lldbPluginCPlusPlusLanguage PLUGIN
   LibStdcppUniquePointer.cpp
   MSVCUndecoratedNameParser.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbDataFormatters
@@ -45,9 +47,6 @@ add_lldb_library(lldbPluginCPlusPlusLanguage PLUGIN
     lldbPluginClangCommon
     lldbPluginCPPRuntime
     lldbPluginTypeSystemClang
-
-  LINK_COMPONENTS
-    Support
 )
 
 add_dependencies(lldbPluginCPlusPlusLanguage
diff --git a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 7485577ae67e..0f18abb47591 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -2066,9 +2066,9 @@ class PluginProperties : public Properties {
     m_collection_sp->Initialize(g_language_cplusplus_properties);
   }
 
-  const FormatEntity::Entry *GetFunctionNameFormat() const {
-    return GetPropertyAtIndexAs<const FormatEntity::Entry *>(
-        ePropertyFunctionNameFormat);
+  FormatEntity::Entry GetFunctionNameFormat() const {
+    return GetPropertyAtIndexAs<FormatEntity::Entry>(
+        ePropertyFunctionNameFormat, {});
   }
 };
 } // namespace
@@ -2078,7 +2078,7 @@ static PluginProperties &GetGlobalPluginProperties() {
   return g_settings;
 }
 
-const FormatEntity::Entry *CPlusPlusLanguage::GetFunctionNameFormat() const {
+FormatEntity::Entry CPlusPlusLanguage::GetFunctionNameFormat() const {
   return GetGlobalPluginProperties().GetFunctionNameFormat();
 }
 
diff --git a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h
index 575f76c3101e..22acdf3e8efe 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h
+++ b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h
@@ -93,8 +93,8 @@ class CPlusPlusLanguage : public Language {
   static llvm::StringRef GetPluginNameStatic() { return "cplusplus"; }
 
   bool SymbolNameFitsToLanguage(Mangled mangled) const override;
-  
-  bool DemangledNameContainsPath(llvm::StringRef path, 
+
+  bool DemangledNameContainsPath(llvm::StringRef path,
                                  ConstString demangled) const override;
 
   ConstString
@@ -136,7 +136,7 @@ class CPlusPlusLanguage : public Language {
 
   llvm::StringRef GetInstanceVariableName() override { return "this"; }
 
-  const FormatEntity::Entry *GetFunctionNameFormat() const override;
+  FormatEntity::Entry GetFunctionNameFormat() const override;
 
   // PluginInterface protocol
   llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
diff --git a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
index d3cdb231fbb0..e8c2db188633 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
@@ -11,8 +11,6 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/VariableList.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -32,8 +30,7 @@ static lldb::addr_t GetCoroFramePtrFromHandle(ValueObjectSP valobj_sp) {
   if (!ptr_sp->GetCompilerType().IsPointerType())
     return LLDB_INVALID_ADDRESS;
 
-  AddressType addr_type;
-  lldb::addr_t frame_ptr_addr = ptr_sp->GetPointerValue(&addr_type);
+  auto [frame_ptr_addr, addr_type] = ptr_sp->GetPointerValue();
   if (!frame_ptr_addr || frame_ptr_addr == LLDB_INVALID_ADDRESS)
     return LLDB_INVALID_ADDRESS;
   lldbassert(addr_type == AddressType::eAddressTypeLoad);
@@ -62,19 +59,23 @@ static Function *ExtractDestroyFunction(lldb::TargetSP target_sp,
   return destroy_func_address.CalculateSymbolContextFunction();
 }
 
-static CompilerType InferPromiseType(Function &destroy_func) {
-  Block &block = destroy_func.GetBlock(true);
+// clang generates aritifical `__promise` and `__coro_frame` variables inside
+// the destroy function. Look for those variables and extract their type.
+static CompilerType InferArtificialCoroType(Function *destroy_func,
+                                            ConstString var_name) {
+  if (!destroy_func)
+    return {};
+
+  Block &block = destroy_func->GetBlock(true);
   auto variable_list = block.GetBlockVariableList(true);
 
-  // clang generates an artificial `__promise` variable inside the
-  // `destroy` function. Look for it.
-  auto promise_var = variable_list->FindVariable(ConstString("__promise"));
-  if (!promise_var)
+  auto var = variable_list->FindVariable(var_name);
+  if (!var)
     return {};
-  if (!promise_var->IsArtificial())
+  if (!var->IsArtificial())
     return {};
 
-  Type *promise_type = promise_var->GetType();
+  Type *promise_type = var->GetType();
   if (!promise_type)
     return {};
   return promise_type->GetForwardCompilerType();
@@ -108,30 +109,17 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
 
 llvm::Expected<uint32_t> lldb_private::formatters::
     StdlibCoroutineHandleSyntheticFrontEnd::CalculateNumChildren() {
-  if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
-    return 0;
-
-  return m_promise_ptr_sp ? 3 : 2;
+  return m_children.size();
 }
 
 lldb::ValueObjectSP lldb_private::formatters::
     StdlibCoroutineHandleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  switch (idx) {
-  case 0:
-    return m_resume_ptr_sp;
-  case 1:
-    return m_destroy_ptr_sp;
-  case 2:
-    return m_promise_ptr_sp;
-  }
-  return lldb::ValueObjectSP();
+  return idx < m_children.size() ? m_children[idx] : lldb::ValueObjectSP();
 }
 
 lldb::ChildCacheState
 lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::Update() {
-  m_resume_ptr_sp.reset();
-  m_destroy_ptr_sp.reset();
-  m_promise_ptr_sp.reset();
+  m_children.clear();
 
   ValueObjectSP valobj_sp = m_backend.GetNonSyntheticValue();
   if (!valobj_sp)
@@ -141,59 +129,66 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::Update() {
   if (frame_ptr_addr == 0 || frame_ptr_addr == LLDB_INVALID_ADDRESS)
     return lldb::ChildCacheState::eRefetch;
 
-  auto ast_ctx = valobj_sp->GetCompilerType().GetTypeSystem<TypeSystemClang>();
-  if (!ast_ctx)
-    return lldb::ChildCacheState::eRefetch;
-
-  // Create the `resume` and `destroy` children.
   lldb::TargetSP target_sp = m_backend.GetTargetSP();
   auto &exe_ctx = m_backend.GetExecutionContextRef();
   lldb::ProcessSP process_sp = target_sp->GetProcessSP();
   auto ptr_size = process_sp->GetAddressByteSize();
-  CompilerType void_type = ast_ctx->GetBasicType(lldb::eBasicTypeVoid);
-  CompilerType coro_func_type = ast_ctx->CreateFunctionType(
-      /*result_type=*/void_type, /*args=*/&void_type, /*num_args=*/1,
-      /*is_variadic=*/false, /*qualifiers=*/0);
-  CompilerType coro_func_ptr_type = coro_func_type.GetPointerType();
-  m_resume_ptr_sp = CreateValueObjectFromAddress(
-      "resume", frame_ptr_addr + 0 * ptr_size, exe_ctx, coro_func_ptr_type);
-  lldbassert(m_resume_ptr_sp);
-  m_destroy_ptr_sp = CreateValueObjectFromAddress(
-      "destroy", frame_ptr_addr + 1 * ptr_size, exe_ctx, coro_func_ptr_type);
-  lldbassert(m_destroy_ptr_sp);
-
-  // Get the `promise_type` from the template argument
-  CompilerType promise_type(
-      valobj_sp->GetCompilerType().GetTypeTemplateArgument(0));
-  if (!promise_type)
+  auto ast_ctx = valobj_sp->GetCompilerType().GetTypeSystem<TypeSystemClang>();
+  if (!ast_ctx)
     return lldb::ChildCacheState::eRefetch;
 
-  // Try to infer the promise_type if it was type-erased
+  // Determine the coroutine frame type and the promise type. Fall back
+  // to `void`, since even the pointer itself might be useful, even if the
+  // type inference failed.
+  Function *destroy_func = ExtractDestroyFunction(target_sp, frame_ptr_addr);
+  CompilerType void_type = ast_ctx->GetBasicType(lldb::eBasicTypeVoid);
+  CompilerType promise_type;
+  if (CompilerType template_arg =
+          valobj_sp->GetCompilerType().GetTypeTemplateArgument(0))
+    promise_type = std::move(template_arg);
   if (promise_type.IsVoidType()) {
-    if (Function *destroy_func =
-            ExtractDestroyFunction(target_sp, frame_ptr_addr)) {
-      if (CompilerType inferred_type = InferPromiseType(*destroy_func)) {
+    // Try to infer the promise_type if it was type-erased
+    if (destroy_func) {
+      if (CompilerType inferred_type =
+              InferArtificialCoroType(destroy_func, ConstString("__promise"))) {
         promise_type = inferred_type;
       }
     }
   }
+  CompilerType coro_frame_type =
+      InferArtificialCoroType(destroy_func, ConstString("__coro_frame"));
+  if (!coro_frame_type)
+    coro_frame_type = void_type;
 
-  // If we don't know the promise type, we don't display the `promise` member.
-  // `CreateValueObjectFromAddress` below would fail for `void` types.
-  if (promise_type.IsVoidType()) {
-    return lldb::ChildCacheState::eRefetch;
-  }
-
-  // Add the `promise` member. We intentionally add `promise` as a pointer type
-  // instead of a value type, and don't automatically dereference this pointer.
-  // We do so to avoid potential very deep recursion in case there is a cycle
-  // formed between `std::coroutine_handle`s and their promises.
-  lldb::ValueObjectSP promise = CreateValueObjectFromAddress(
-      "promise", frame_ptr_addr + 2 * ptr_size, exe_ctx, promise_type);
-  Status error;
-  lldb::ValueObjectSP promisePtr = promise->AddressOf(error);
-  if (error.Success())
-    m_promise_ptr_sp = promisePtr->Clone(ConstString("promise"));
+  // Create the `resume` and `destroy` children.
+  std::array<CompilerType, 1> args{coro_frame_type};
+  CompilerType coro_func_type = ast_ctx->CreateFunctionType(
+      /*result_type=*/void_type, args,
+      /*is_variadic=*/false, /*qualifiers=*/0);
+  CompilerType coro_func_ptr_type = coro_func_type.GetPointerType();
+  ValueObjectSP resume_ptr_sp = CreateValueObjectFromAddress(
+      "resume", frame_ptr_addr + 0 * ptr_size, exe_ctx, coro_func_ptr_type);
+  assert(resume_ptr_sp);
+  m_children.push_back(std::move(resume_ptr_sp));
+  ValueObjectSP destroy_ptr_sp = CreateValueObjectFromAddress(
+      "destroy", frame_ptr_addr + 1 * ptr_size, exe_ctx, coro_func_ptr_type);
+  assert(destroy_ptr_sp);
+  m_children.push_back(std::move(destroy_ptr_sp));
+
+  // Add promise and coro_frame
+  // Add the `promise` and `coro_frame` member. We intentionally add them as
+  // pointer types instead of a value type, and don't automatically dereference
+  // those pointers. We do so to avoid potential very deep recursion in case
+  // there is a cycle formed between `std::coroutine_handle`s and their
+  // promises.
+  ValueObjectSP promise_ptr_sp = CreateValueObjectFromAddress(
+      "promise", frame_ptr_addr + 2 * ptr_size, exe_ctx,
+      promise_type.GetPointerType(), /*do_deref=*/false);
+  m_children.push_back(std::move(promise_ptr_sp));
+  ValueObjectSP coroframe_ptr_sp = CreateValueObjectFromAddress(
+      "coro_frame", frame_ptr_addr, exe_ctx, coro_frame_type.GetPointerType(),
+      /*do_deref=*/false);
+  m_children.push_back(std::move(coroframe_ptr_sp));
 
   return lldb::ChildCacheState::eRefetch;
 }
@@ -201,16 +196,10 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::Update() {
 llvm::Expected<size_t>
 StdlibCoroutineHandleSyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
-  if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
-    return llvm::createStringError("Type has no child named '%s'",
-                                   name.AsCString());
-
-  if (name == ConstString("resume"))
-    return 0;
-  if (name == ConstString("destroy"))
-    return 1;
-  if (name == ConstString("promise_ptr") && m_promise_ptr_sp)
-    return 2;
+  for (const auto &[idx, child_sp] : llvm::enumerate(m_children)) {
+    if (child_sp->GetName() == name)
+      return idx;
+  }
 
   return llvm::createStringError("Type has no child named '%s'",
                                  name.AsCString());
diff --git a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index fd9445d46e6a..520d8e0b3c79 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -43,9 +43,7 @@ class StdlibCoroutineHandleSyntheticFrontEnd
   llvm::Expected<size_t> GetIndexOfChildWithName(ConstString name) override;
 
 private:
-  lldb::ValueObjectSP m_resume_ptr_sp;
-  lldb::ValueObjectSP m_destroy_ptr_sp;
-  lldb::ValueObjectSP m_promise_ptr_sp;
+  std::vector<lldb::ValueObjectSP> m_children;
 };
 
 SyntheticChildrenFrontEnd *
diff --git a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index 30db5f15c388..826e6ab090e1 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -388,7 +388,7 @@ lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) {
       return lldb::ValueObjectSP();
 
     // if we grabbed the __next_ pointer, then the child is one pointer deep-er
-    lldb::addr_t addr = current_sp->GetParent()->GetPointerValue();
+    lldb::addr_t addr = current_sp->GetParent()->GetPointerValue().address;
     addr = addr + 2 * process_sp->GetAddressByteSize();
     ExecutionContext exe_ctx(process_sp);
     current_sp =
diff --git a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 08cfcd4a26b8..77e0755607a0 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -240,16 +240,10 @@ VectorIteratorSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
 bool lldb_private::formatters::LibStdcppStringSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   const bool scalar_is_load_addr = true;
-  AddressType addr_type;
-  lldb::addr_t addr_of_string = LLDB_INVALID_ADDRESS;
-  if (valobj.IsPointerOrReferenceType()) {
-    Status error;
-    ValueObjectSP pointee_sp = valobj.Dereference(error);
-    if (pointee_sp && error.Success())
-      addr_of_string = pointee_sp->GetAddressOf(scalar_is_load_addr, &addr_type);
-  } else
-    addr_of_string =
-        valobj.GetAddressOf(scalar_is_load_addr, &addr_type);
+  auto [addr_of_string, addr_type] =
+      valobj.IsPointerOrReferenceType()
+          ? valobj.GetPointerValue()
+          : valobj.GetAddressOf(scalar_is_load_addr);
   if (addr_of_string != LLDB_INVALID_ADDRESS) {
     switch (addr_type) {
     case eAddressTypeLoad: {
@@ -296,9 +290,7 @@ bool lldb_private::formatters::LibStdcppStringSummaryProvider(
 bool lldb_private::formatters::LibStdcppWStringSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   const bool scalar_is_load_addr = true;
-  AddressType addr_type;
-  lldb::addr_t addr_of_string =
-      valobj.GetAddressOf(scalar_is_load_addr, &addr_type);
+  auto [addr_of_string, addr_type] = valobj.GetAddressOf(scalar_is_load_addr);
   if (addr_of_string != LLDB_INVALID_ADDRESS) {
     switch (addr_type) {
     case eAddressTypeLoad: {
diff --git a/external/llvm-project/lldb/source/Plugins/Language/ClangCommon/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Language/ClangCommon/CMakeLists.txt
index 980d788afc72..470dbc203054 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/ClangCommon/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Language/ClangCommon/CMakeLists.txt
@@ -1,9 +1,9 @@
 add_lldb_library(lldbPluginClangCommon
   ClangHighlighter.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbUtility
-  LINK_COMPONENTS
-    Support
 )
diff --git a/external/llvm-project/lldb/source/Plugins/Language/ObjC/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Language/ObjC/CMakeLists.txt
index 93c23fd32524..b9fc5ce754c4 100644
--- a/external/llvm-project/lldb/source/Plugins/Language/ObjC/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Language/ObjC/CMakeLists.txt
@@ -1,13 +1,3 @@
-set(EXTRA_CXXFLAGS "")
-
-if (CXX_SUPPORTS_NO_GNU_ANONYMOUS_STRUCT)
-  set(EXTRA_CXXFLAGS ${EXTRA_CXXFLAGS} -Wno-gnu-anonymous-struct)
-endif ()
-
-if (CXX_SUPPORTS_NO_NESTED_ANON_TYPES)
-  set(EXTRA_CXXFLAGS ${EXTRA_CXXFLAGS} -Wno-nested-anon-types)
-endif ()
-
 add_lldb_library(lldbPluginObjCLanguage PLUGIN
   ObjCLanguage.cpp
   CF.cpp
@@ -36,6 +26,14 @@ add_lldb_library(lldbPluginObjCLanguage PLUGIN
     lldbPluginTypeSystemClang
   CLANG_LIBS
     clangAST
-
-  EXTRA_CXXFLAGS ${EXTRA_CXXFLAGS}
 )
+
+if (CXX_SUPPORTS_NO_GNU_ANONYMOUS_STRUCT)
+  target_compile_options(lldbPluginObjCLanguage
+    PRIVATE -Wno-gnu-anonymous-struct)
+endif ()
+
+if (CXX_SUPPORTS_NO_NESTED_ANON_TYPES)
+  target_compile_options(lldbPluginObjCLanguage
+    PRIVATE -Wno-nested-anon-types)
+endif ()
diff --git a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
index 0d068ed5950d..75b00518aac5 100644
--- a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
+++ b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
@@ -69,9 +69,8 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo(
       LLDB_LOGF(log,
                 "0x%16.16" PRIx64
                 ": static-type = '%s' has vtable symbol '%s'\n",
-                in_value.GetPointerValue(),
-                in_value.GetTypeName().GetCString(),
-                symbol_name.str().c_str());
+                in_value.GetPointerValue().address,
+                in_value.GetTypeName().GetCString(), symbol_name.str().c_str());
       // We are a C++ class, that's good.  Get the class name and look it
       // up:
       llvm::StringRef class_name = symbol_name;
@@ -111,7 +110,7 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo(
       lldb::TypeSP type_sp;
       if (class_types.Empty()) {
         LLDB_LOGF(log, "0x%16.16" PRIx64 ": is not dynamic\n",
-                  in_value.GetPointerValue());
+                  in_value.GetPointerValue().address);
         return TypeAndOrName();
       }
       if (class_types.GetSize() == 1) {
@@ -119,13 +118,13 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo(
         if (type_sp) {
           if (TypeSystemClang::IsCXXClassType(
                   type_sp->GetForwardCompilerType())) {
-            LLDB_LOGF(
-                log,
-                "0x%16.16" PRIx64
-                ": static-type = '%s' has dynamic type: uid={0x%" PRIx64
-                "}, type-name='%s'\n",
-                in_value.GetPointerValue(), in_value.GetTypeName().AsCString(),
-                type_sp->GetID(), type_sp->GetName().GetCString());
+            LLDB_LOGF(log,
+                      "0x%16.16" PRIx64
+                      ": static-type = '%s' has dynamic type: uid={0x%" PRIx64
+                      "}, type-name='%s'\n",
+                      in_value.GetPointerValue().address,
+                      in_value.GetTypeName().AsCString(), type_sp->GetID(),
+                      type_sp->GetName().GetCString());
             type_info.SetTypeSP(type_sp);
           }
         }
@@ -135,14 +134,13 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo(
           for (i = 0; i < class_types.GetSize(); i++) {
             type_sp = class_types.GetTypeAtIndex(i);
             if (type_sp) {
-              LLDB_LOGF(
-                  log,
-                  "0x%16.16" PRIx64
-                  ": static-type = '%s' has multiple matching dynamic "
-                  "types: uid={0x%" PRIx64 "}, type-name='%s'\n",
-                  in_value.GetPointerValue(),
-                  in_value.GetTypeName().AsCString(),
-                  type_sp->GetID(), type_sp->GetName().GetCString());
+              LLDB_LOGF(log,
+                        "0x%16.16" PRIx64
+                        ": static-type = '%s' has multiple matching dynamic "
+                        "types: uid={0x%" PRIx64 "}, type-name='%s'\n",
+                        in_value.GetPointerValue().address,
+                        in_value.GetTypeName().AsCString(), type_sp->GetID(),
+                        type_sp->GetName().GetCString());
             }
           }
         }
@@ -152,14 +150,13 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo(
           if (type_sp) {
             if (TypeSystemClang::IsCXXClassType(
                     type_sp->GetForwardCompilerType())) {
-              LLDB_LOGF(
-                  log,
-                  "0x%16.16" PRIx64 ": static-type = '%s' has multiple "
-                  "matching dynamic types, picking "
-                  "this one: uid={0x%" PRIx64 "}, type-name='%s'\n",
-                  in_value.GetPointerValue(),
-                  in_value.GetTypeName().AsCString(),
-                  type_sp->GetID(), type_sp->GetName().GetCString());
+              LLDB_LOGF(log,
+                        "0x%16.16" PRIx64 ": static-type = '%s' has multiple "
+                        "matching dynamic types, picking "
+                        "this one: uid={0x%" PRIx64 "}, type-name='%s'\n",
+                        in_value.GetPointerValue().address,
+                        in_value.GetTypeName().AsCString(), type_sp->GetID(),
+                        type_sp->GetName().GetCString());
               type_info.SetTypeSP(type_sp);
             }
           }
@@ -170,7 +167,7 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo(
                     "0x%16.16" PRIx64
                     ": static-type = '%s' has multiple matching dynamic "
                     "types, didn't find a C++ match\n",
-                    in_value.GetPointerValue(),
+                    in_value.GetPointerValue().address,
                     in_value.GetTypeName().AsCString());
         }
       }
@@ -230,13 +227,10 @@ llvm::Expected<LanguageRuntime::VTableInfo>
     return llvm::createStringError(std::errc::invalid_argument,
                                    "invalid process");
 
-  AddressType address_type;
-  lldb::addr_t original_ptr = LLDB_INVALID_ADDRESS;
-  if (type.IsPointerOrReferenceType())
-    original_ptr = in_value.GetPointerValue(&address_type);
-  else
-    original_ptr = in_value.GetAddressOf(/*scalar_is_load_address=*/true,
-                                         &address_type);
+  auto [original_ptr, address_type] =
+      type.IsPointerOrReferenceType()
+          ? in_value.GetPointerValue()
+          : in_value.GetAddressOf(/*scalar_is_load_address=*/true);
   if (original_ptr == LLDB_INVALID_ADDRESS || address_type != eAddressTypeLoad)
     return llvm::createStringError(std::errc::invalid_argument,
                                    "failed to get the address of the value");
@@ -357,7 +351,8 @@ bool ItaniumABILanguageRuntime::GetDynamicTypeAndAddress(
     return false;
   // So the dynamic type is a value that starts at offset_to_top above
   // the original address.
-  lldb::addr_t dynamic_addr = in_value.GetPointerValue() + offset_to_top;
+  lldb::addr_t dynamic_addr =
+      in_value.GetPointerValue().address + offset_to_top;
   if (!m_process->GetTarget().ResolveLoadAddress(
           dynamic_addr, dynamic_address)) {
     dynamic_address.SetRawAddress(dynamic_addr);
diff --git a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
index db1317d70d06..24a73717266a 100644
--- a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
+++ b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
@@ -55,7 +55,7 @@ bool AppleObjCRuntimeV1::GetDynamicTypeAndAddress(
     auto class_descriptor(GetClassDescriptor(in_value));
     if (class_descriptor && class_descriptor->IsValid() &&
         class_descriptor->GetClassName()) {
-      const addr_t object_ptr = in_value.GetPointerValue();
+      const addr_t object_ptr = in_value.GetPointerValue().address;
       address.SetRawAddress(object_ptr);
       class_type_or_name.SetName(class_descriptor->GetClassName());
     }
diff --git a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index f458357d948d..e459cb281793 100644
--- a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -801,7 +801,7 @@ bool AppleObjCRuntimeV2::GetDynamicTypeAndAddress(
     // be the ISA pointer.
     ClassDescriptorSP objc_class_sp(GetNonKVOClassDescriptor(in_value));
     if (objc_class_sp) {
-      const addr_t object_ptr = in_value.GetPointerValue();
+      const addr_t object_ptr = in_value.GetPointerValue().address;
       address.SetRawAddress(object_ptr);
 
       ConstString class_name(objc_class_sp->GetClassName());
@@ -1521,7 +1521,7 @@ AppleObjCRuntimeV2::GetClassDescriptor(ValueObject &valobj) {
   // ObjC object)
   if (!valobj.GetCompilerType().IsValid())
     return objc_class_sp;
-  addr_t isa_pointer = valobj.GetPointerValue();
+  addr_t isa_pointer = valobj.GetPointerValue().address;
 
   // tagged pointer
   if (IsTaggedPointer(isa_pointer))
diff --git a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt
index 53320d3da4cf..0b2dfe12c5c4 100644
--- a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/CMakeLists.txt
@@ -8,6 +8,8 @@ add_lldb_library(lldbPluginAppleObjCRuntime PLUGIN
   AppleObjCClassDescriptorV2.cpp
   AppleObjCTypeEncodingParser.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -23,6 +25,4 @@ add_lldb_library(lldbPluginAppleObjCRuntime PLUGIN
     lldbPluginTypeSystemClang
   CLANG_LIBS
     clangAST
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt
index 0c7bff701dc0..05caad3e7d22 100644
--- a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_lldb_library(lldbPluginGNUstepObjCRuntime PLUGIN
   GNUstepObjCRuntime.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -15,6 +17,4 @@ add_lldb_library(lldbPluginGNUstepObjCRuntime PLUGIN
     lldbPluginTypeSystemClang
   CLANG_LIBS
     clangAST
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
index cb745135b024..c33760eccd12 100644
--- a/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
+++ b/external/llvm-project/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
@@ -270,7 +270,7 @@ ObjCLanguageRuntime::GetClassDescriptor(ValueObject &valobj) {
   // pointers returned by the expression parser, don't consider this a valid
   // ObjC object)
   if (valobj.GetCompilerType().IsValid()) {
-    addr_t isa_pointer = valobj.GetPointerValue();
+    addr_t isa_pointer = valobj.GetPointerValue().address;
     if (isa_pointer != LLDB_INVALID_ADDRESS) {
       ExecutionContext exe_ctx(valobj.GetExecutionContextRef());
 
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectContainer/BSD-Archive/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectContainer/BSD-Archive/CMakeLists.txt
index acc13d31ca25..1040e3d3b9c0 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectContainer/BSD-Archive/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectContainer/BSD-Archive/CMakeLists.txt
@@ -1,10 +1,10 @@
 add_lldb_library(lldbPluginObjectContainerBSDArchive PLUGIN
   ObjectContainerBSDArchive.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/Breakpad/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/Breakpad/CMakeLists.txt
index 705e2d471392..87f3b1df57dd 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/Breakpad/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/Breakpad/CMakeLists.txt
@@ -2,12 +2,12 @@ add_lldb_library(lldbPluginObjectFileBreakpad PLUGIN
   BreakpadRecords.cpp
   ObjectFileBreakpad.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbUtility
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/CMakeLists.txt
index e65b7ea50121..c7341d34bb14 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/CMakeLists.txt
@@ -1,13 +1,13 @@
 add_lldb_library(lldbPluginObjectFileCOFF PLUGIN
   ObjectFileCOFF.cpp
 
+  LINK_COMPONENTS
+    BinaryFormat
+    Object
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbTarget
-
-  LINK_COMPONENTS
-    BinaryFormat
-    Object
-    Support)
+  )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/ObjectFileCOFF.cpp b/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/ObjectFileCOFF.cpp
index a7ad5d27b237..1121f696637b 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/ObjectFileCOFF.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/COFF/ObjectFileCOFF.cpp
@@ -191,19 +191,15 @@ void ObjectFileCOFF::CreateSections(lldb_private::SectionList &sections) {
 
   auto SectionType = [](StringRef Name,
                         const coff_section *Section) -> lldb::SectionType {
-    lldb::SectionType type =
-        StringSwitch<lldb::SectionType>(Name)
-            // DWARF Debug Sections
-            .Case(".debug_abbrev", eSectionTypeDWARFDebugAbbrev)
-            .Case(".debug_info", eSectionTypeDWARFDebugInfo)
-            .Case(".debug_line", eSectionTypeDWARFDebugLine)
-            .Case(".debug_pubnames", eSectionTypeDWARFDebugPubNames)
-            .Case(".debug_pubtypes", eSectionTypeDWARFDebugPubTypes)
-            .Case(".debug_str", eSectionTypeDWARFDebugStr)
-            // CodeView Debug Sections: .debug$S, .debug$T
-            .StartsWith(".debug$", eSectionTypeDebug)
-            .Case("clangast", eSectionTypeOther)
-            .Default(eSectionTypeInvalid);
+    // DWARF Debug Sections
+    if (Name.consume_front(".debug_"))
+      return GetDWARFSectionTypeFromName(Name);
+
+    lldb::SectionType type = StringSwitch<lldb::SectionType>(Name)
+                                 // CodeView Debug Sections: .debug$S, .debug$T
+                                 .StartsWith(".debug$", eSectionTypeDebug)
+                                 .Case("clangast", eSectionTypeOther)
+                                 .Default(eSectionTypeInvalid);
     if (type != eSectionTypeInvalid)
       return type;
 
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/CMakeLists.txt
index 45a4edcbb1c9..90aed254035a 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/CMakeLists.txt
@@ -2,13 +2,13 @@ add_lldb_library(lldbPluginObjectFileELF PLUGIN
   ELFHeader.cpp
   ObjectFileELF.cpp
 
+  LINK_COMPONENTS
+    BinaryFormat
+    Object
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbTarget
-  LINK_COMPONENTS
-    BinaryFormat
-    Object
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 13e1198516f7..f69358de6a28 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -1653,39 +1653,9 @@ lldb::user_id_t ObjectFileELF::GetSectionIndexByName(const char *name) {
 }
 
 static SectionType GetSectionTypeFromName(llvm::StringRef Name) {
-  if (Name.consume_front(".debug_")) {
-    return llvm::StringSwitch<SectionType>(Name)
-        .Case("abbrev", eSectionTypeDWARFDebugAbbrev)
-        .Case("abbrev.dwo", eSectionTypeDWARFDebugAbbrevDwo)
-        .Case("addr", eSectionTypeDWARFDebugAddr)
-        .Case("aranges", eSectionTypeDWARFDebugAranges)
-        .Case("cu_index", eSectionTypeDWARFDebugCuIndex)
-        .Case("frame", eSectionTypeDWARFDebugFrame)
-        .Case("info", eSectionTypeDWARFDebugInfo)
-        .Case("info.dwo", eSectionTypeDWARFDebugInfoDwo)
-        .Cases("line", "line.dwo", eSectionTypeDWARFDebugLine)
-        .Cases("line_str", "line_str.dwo", eSectionTypeDWARFDebugLineStr)
-        .Case("loc", eSectionTypeDWARFDebugLoc)
-        .Case("loc.dwo", eSectionTypeDWARFDebugLocDwo)
-        .Case("loclists", eSectionTypeDWARFDebugLocLists)
-        .Case("loclists.dwo", eSectionTypeDWARFDebugLocListsDwo)
-        .Case("macinfo", eSectionTypeDWARFDebugMacInfo)
-        .Cases("macro", "macro.dwo", eSectionTypeDWARFDebugMacro)
-        .Case("names", eSectionTypeDWARFDebugNames)
-        .Case("pubnames", eSectionTypeDWARFDebugPubNames)
-        .Case("pubtypes", eSectionTypeDWARFDebugPubTypes)
-        .Case("ranges", eSectionTypeDWARFDebugRanges)
-        .Case("rnglists", eSectionTypeDWARFDebugRngLists)
-        .Case("rnglists.dwo", eSectionTypeDWARFDebugRngListsDwo)
-        .Case("str", eSectionTypeDWARFDebugStr)
-        .Case("str.dwo", eSectionTypeDWARFDebugStrDwo)
-        .Case("str_offsets", eSectionTypeDWARFDebugStrOffsets)
-        .Case("str_offsets.dwo", eSectionTypeDWARFDebugStrOffsetsDwo)
-        .Case("tu_index", eSectionTypeDWARFDebugTuIndex)
-        .Case("types", eSectionTypeDWARFDebugTypes)
-        .Case("types.dwo", eSectionTypeDWARFDebugTypesDwo)
-        .Default(eSectionTypeOther);
-  }
+  if (Name.consume_front(".debug_"))
+    return ObjectFile::GetDWARFSectionTypeFromName(Name);
+
   return llvm::StringSwitch<SectionType>(Name)
       .Case(".ARM.exidx", eSectionTypeARMexidx)
       .Case(".ARM.extab", eSectionTypeARMextab)
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/JSON/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/JSON/CMakeLists.txt
index 73a8697d13d1..af358793ea27 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/JSON/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/JSON/CMakeLists.txt
@@ -1,12 +1,12 @@
 add_lldb_library(lldbPluginObjectFileJSON PLUGIN
   ObjectFileJSON.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbUtility
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/CMakeLists.txt
index d39b93768ae7..a16fcd8043a4 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_lldb_library(lldbPluginObjectFileMachO PLUGIN
   ObjectFileMachO.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -8,6 +10,4 @@ add_lldb_library(lldbPluginObjectFileMachO PLUGIN
     lldbTarget
     lldbUtility
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 3950454b7c90..b1741926b74a 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -1595,34 +1595,8 @@ static lldb::SectionType GetSectionType(uint32_t flags,
   static ConstString g_sect_name_objc_classlist("__objc_classlist");
   static ConstString g_sect_name_cfstring("__cfstring");
 
-  static ConstString g_sect_name_dwarf_debug_abbrev("__debug_abbrev");
-  static ConstString g_sect_name_dwarf_debug_abbrev_dwo("__debug_abbrev.dwo");
-  static ConstString g_sect_name_dwarf_debug_addr("__debug_addr");
-  static ConstString g_sect_name_dwarf_debug_aranges("__debug_aranges");
-  static ConstString g_sect_name_dwarf_debug_cu_index("__debug_cu_index");
-  static ConstString g_sect_name_dwarf_debug_frame("__debug_frame");
-  static ConstString g_sect_name_dwarf_debug_info("__debug_info");
-  static ConstString g_sect_name_dwarf_debug_info_dwo("__debug_info.dwo");
-  static ConstString g_sect_name_dwarf_debug_line("__debug_line");
-  static ConstString g_sect_name_dwarf_debug_line_dwo("__debug_line.dwo");
-  static ConstString g_sect_name_dwarf_debug_line_str("__debug_line_str");
-  static ConstString g_sect_name_dwarf_debug_loc("__debug_loc");
-  static ConstString g_sect_name_dwarf_debug_loclists("__debug_loclists");
-  static ConstString g_sect_name_dwarf_debug_loclists_dwo("__debug_loclists.dwo");
-  static ConstString g_sect_name_dwarf_debug_macinfo("__debug_macinfo");
-  static ConstString g_sect_name_dwarf_debug_macro("__debug_macro");
-  static ConstString g_sect_name_dwarf_debug_macro_dwo("__debug_macro.dwo");
-  static ConstString g_sect_name_dwarf_debug_names("__debug_names");
-  static ConstString g_sect_name_dwarf_debug_pubnames("__debug_pubnames");
-  static ConstString g_sect_name_dwarf_debug_pubtypes("__debug_pubtypes");
-  static ConstString g_sect_name_dwarf_debug_ranges("__debug_ranges");
-  static ConstString g_sect_name_dwarf_debug_rnglists("__debug_rnglists");
-  static ConstString g_sect_name_dwarf_debug_str("__debug_str");
-  static ConstString g_sect_name_dwarf_debug_str_dwo("__debug_str.dwo");
   static ConstString g_sect_name_dwarf_debug_str_offs("__debug_str_offs");
   static ConstString g_sect_name_dwarf_debug_str_offs_dwo("__debug_str_offs.dwo");
-  static ConstString g_sect_name_dwarf_debug_tu_index("__debug_tu_index");
-  static ConstString g_sect_name_dwarf_debug_types("__debug_types");
   static ConstString g_sect_name_dwarf_apple_names("__apple_names");
   static ConstString g_sect_name_dwarf_apple_types("__apple_types");
   static ConstString g_sect_name_dwarf_apple_namespaces("__apple_namespac");
@@ -1637,62 +1611,15 @@ static lldb::SectionType GetSectionType(uint32_t flags,
   static ConstString g_sect_name_lldb_formatters("__lldbformatters");
   static ConstString g_sect_name_swift_ast("__swift_ast");
 
-  if (section_name == g_sect_name_dwarf_debug_abbrev)
-    return eSectionTypeDWARFDebugAbbrev;
-  if (section_name == g_sect_name_dwarf_debug_abbrev_dwo)
-    return eSectionTypeDWARFDebugAbbrevDwo;
-  if (section_name == g_sect_name_dwarf_debug_addr)
-    return eSectionTypeDWARFDebugAddr;
-  if (section_name == g_sect_name_dwarf_debug_aranges)
-    return eSectionTypeDWARFDebugAranges;
-  if (section_name == g_sect_name_dwarf_debug_cu_index)
-    return eSectionTypeDWARFDebugCuIndex;
-  if (section_name == g_sect_name_dwarf_debug_frame)
-    return eSectionTypeDWARFDebugFrame;
-  if (section_name == g_sect_name_dwarf_debug_info)
-    return eSectionTypeDWARFDebugInfo;
-  if (section_name == g_sect_name_dwarf_debug_info_dwo)
-    return eSectionTypeDWARFDebugInfoDwo;
-  if (section_name == g_sect_name_dwarf_debug_line)
-    return eSectionTypeDWARFDebugLine;
-  if (section_name == g_sect_name_dwarf_debug_line_dwo)
-    return eSectionTypeDWARFDebugLine; // Same as debug_line.
-  if (section_name == g_sect_name_dwarf_debug_line_str)
-    return eSectionTypeDWARFDebugLineStr;
-  if (section_name == g_sect_name_dwarf_debug_loc)
-    return eSectionTypeDWARFDebugLoc;
-  if (section_name == g_sect_name_dwarf_debug_loclists)
-    return eSectionTypeDWARFDebugLocLists;
-  if (section_name == g_sect_name_dwarf_debug_loclists_dwo)
-    return eSectionTypeDWARFDebugLocListsDwo;
-  if (section_name == g_sect_name_dwarf_debug_macinfo)
-    return eSectionTypeDWARFDebugMacInfo;
-  if (section_name == g_sect_name_dwarf_debug_macro)
-    return eSectionTypeDWARFDebugMacro;
-  if (section_name == g_sect_name_dwarf_debug_macro_dwo)
-    return eSectionTypeDWARFDebugMacInfo; // Same as debug_macro.
-  if (section_name == g_sect_name_dwarf_debug_names)
-    return eSectionTypeDWARFDebugNames;
-  if (section_name == g_sect_name_dwarf_debug_pubnames)
-    return eSectionTypeDWARFDebugPubNames;
-  if (section_name == g_sect_name_dwarf_debug_pubtypes)
-    return eSectionTypeDWARFDebugPubTypes;
-  if (section_name == g_sect_name_dwarf_debug_ranges)
-    return eSectionTypeDWARFDebugRanges;
-  if (section_name == g_sect_name_dwarf_debug_rnglists)
-    return eSectionTypeDWARFDebugRngLists;
-  if (section_name == g_sect_name_dwarf_debug_str)
-    return eSectionTypeDWARFDebugStr;
-  if (section_name == g_sect_name_dwarf_debug_str_dwo)
-    return eSectionTypeDWARFDebugStrDwo;
   if (section_name == g_sect_name_dwarf_debug_str_offs)
     return eSectionTypeDWARFDebugStrOffsets;
   if (section_name == g_sect_name_dwarf_debug_str_offs_dwo)
     return eSectionTypeDWARFDebugStrOffsetsDwo;
-  if (section_name == g_sect_name_dwarf_debug_tu_index)
-    return eSectionTypeDWARFDebugTuIndex;
-  if (section_name == g_sect_name_dwarf_debug_types)
-    return eSectionTypeDWARFDebugTypes;
+
+  llvm::StringRef stripped_name = section_name.GetStringRef();
+  if (stripped_name.consume_front("__debug_"))
+    return ObjectFile::GetDWARFSectionTypeFromName(stripped_name);
+
   if (section_name == g_sect_name_dwarf_apple_names)
     return eSectionTypeDWARFAppleNames;
   if (section_name == g_sect_name_dwarf_apple_types)
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/Minidump/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/Minidump/CMakeLists.txt
index ac5fba200f35..5be4f40921ad 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/Minidump/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/Minidump/CMakeLists.txt
@@ -2,6 +2,8 @@ add_lldb_library(lldbPluginObjectFileMinidump PLUGIN
   ObjectFileMinidump.cpp
   MinidumpFileBuilder.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -9,6 +11,4 @@ add_lldb_library(lldbPluginObjectFileMinidump PLUGIN
     lldbTarget
     lldbUtility
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/PDB/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/PDB/CMakeLists.txt
index d5ade2b93ad8..b693b4346f9c 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/PDB/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/PDB/CMakeLists.txt
@@ -1,10 +1,10 @@
 add_lldb_library(lldbPluginObjectFilePDB PLUGIN
   ObjectFilePDB.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt
index 26a764152a78..97ba4032e271 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt
@@ -18,16 +18,16 @@ add_lldb_library(lldbPluginObjectFilePECOFF PLUGIN
   PECallFrameInfo.cpp
   WindowsMiniDump.cpp
 
+  LINK_COMPONENTS
+    BinaryFormat
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbTarget
     ${DBGHELP_LINK_FILES}
-  LINK_COMPONENTS
-    BinaryFormat
-    Support
-    TargetParser
   )
 
 add_dependencies(lldbPluginObjectFilePECOFF
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index 389339a1bc62..4984445dcbab 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -976,25 +976,14 @@ SectionType ObjectFilePECOFF::GetSectionType(llvm::StringRef sect_name,
       return eSectionTypeData;
   }
 
+  if (sect_name.consume_front(".debug_"))
+    return GetDWARFSectionTypeFromName(sect_name);
+
   SectionType section_type =
       llvm::StringSwitch<SectionType>(sect_name)
           .Case(".debug", eSectionTypeDebug)
           .Case(".stabstr", eSectionTypeDataCString)
           .Case(".reloc", eSectionTypeOther)
-          .Case(".debug_abbrev", eSectionTypeDWARFDebugAbbrev)
-          .Case(".debug_aranges", eSectionTypeDWARFDebugAranges)
-          .Case(".debug_frame", eSectionTypeDWARFDebugFrame)
-          .Case(".debug_info", eSectionTypeDWARFDebugInfo)
-          .Case(".debug_line", eSectionTypeDWARFDebugLine)
-          .Case(".debug_loc", eSectionTypeDWARFDebugLoc)
-          .Case(".debug_loclists", eSectionTypeDWARFDebugLocLists)
-          .Case(".debug_macinfo", eSectionTypeDWARFDebugMacInfo)
-          .Case(".debug_names", eSectionTypeDWARFDebugNames)
-          .Case(".debug_pubnames", eSectionTypeDWARFDebugPubNames)
-          .Case(".debug_pubtypes", eSectionTypeDWARFDebugPubTypes)
-          .Case(".debug_ranges", eSectionTypeDWARFDebugRanges)
-          .Case(".debug_str", eSectionTypeDWARFDebugStr)
-          .Case(".debug_types", eSectionTypeDWARFDebugTypes)
           // .eh_frame can be truncated to 8 chars.
           .Cases(".eh_frame", ".eh_fram", eSectionTypeEHFrame)
           .Case(".gosymtab", eSectionTypeGoSymtab)
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/Placeholder/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/Placeholder/CMakeLists.txt
index d7f876258101..f4b66a20f0e9 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/Placeholder/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/Placeholder/CMakeLists.txt
@@ -1,12 +1,12 @@
 add_lldb_library(lldbPluginObjectFilePlaceholder PLUGIN
   ObjectFilePlaceholder.cpp
 
+  LINK_COMPONENTS
+    Object
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbTarget
-  LINK_COMPONENTS
-    Object
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/CMakeLists.txt
index 8840248574c8..f36008875e96 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/CMakeLists.txt
@@ -1,13 +1,13 @@
 add_lldb_library(lldbPluginObjectFileXCOFF PLUGIN
   ObjectFileXCOFF.cpp
 
+  LINK_COMPONENTS
+    BinaryFormat
+    Object
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbTarget
-  LINK_COMPONENTS
-    BinaryFormat
-    Object
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/ObjectFileXCOFF.cpp b/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/ObjectFileXCOFF.cpp
index e629355cd40b..84d05e173f83 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/ObjectFileXCOFF.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/XCOFF/ObjectFileXCOFF.cpp
@@ -239,6 +239,7 @@ void ObjectFileXCOFF::CreateSectionsWithBitness(
                          .Case(".dwinfo", eSectionTypeDWARFDebugInfo)
                          .Case(".dwline", eSectionTypeDWARFDebugLine)
                          .Case(".dwabrev", eSectionTypeDWARFDebugAbbrev)
+                         .Case(".dwrnges", eSectionTypeDWARFDebugRanges)
                          .Default(eSectionTypeInvalid);
     }
 
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/CMakeLists.txt
index 5069b6b19b95..9a46c914f498 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/CMakeLists.txt
@@ -1,11 +1,11 @@
 add_lldb_library(lldbPluginObjectFileWasm PLUGIN
   ObjectFileWasm.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp b/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp
index 06eb6ff9cafb..67963a790a4f 100644
--- a/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp
+++ b/external/llvm-project/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp
@@ -252,37 +252,7 @@ void ObjectFileWasm::ParseSymtab(Symtab &symtab) {}
 
 static SectionType GetSectionTypeFromName(llvm::StringRef Name) {
   if (Name.consume_front(".debug_") || Name.consume_front(".zdebug_")) {
-    return llvm::StringSwitch<SectionType>(Name)
-        .Case("abbrev", eSectionTypeDWARFDebugAbbrev)
-        .Case("abbrev.dwo", eSectionTypeDWARFDebugAbbrevDwo)
-        .Case("addr", eSectionTypeDWARFDebugAddr)
-        .Case("aranges", eSectionTypeDWARFDebugAranges)
-        .Case("cu_index", eSectionTypeDWARFDebugCuIndex)
-        .Case("frame", eSectionTypeDWARFDebugFrame)
-        .Case("info", eSectionTypeDWARFDebugInfo)
-        .Case("info.dwo", eSectionTypeDWARFDebugInfoDwo)
-        .Cases("line", "line.dwo", eSectionTypeDWARFDebugLine)
-        .Cases("line_str", "line_str.dwo", eSectionTypeDWARFDebugLineStr)
-        .Case("loc", eSectionTypeDWARFDebugLoc)
-        .Case("loc.dwo", eSectionTypeDWARFDebugLocDwo)
-        .Case("loclists", eSectionTypeDWARFDebugLocLists)
-        .Case("loclists.dwo", eSectionTypeDWARFDebugLocListsDwo)
-        .Case("macinfo", eSectionTypeDWARFDebugMacInfo)
-        .Cases("macro", "macro.dwo", eSectionTypeDWARFDebugMacro)
-        .Case("names", eSectionTypeDWARFDebugNames)
-        .Case("pubnames", eSectionTypeDWARFDebugPubNames)
-        .Case("pubtypes", eSectionTypeDWARFDebugPubTypes)
-        .Case("ranges", eSectionTypeDWARFDebugRanges)
-        .Case("rnglists", eSectionTypeDWARFDebugRngLists)
-        .Case("rnglists.dwo", eSectionTypeDWARFDebugRngListsDwo)
-        .Case("str", eSectionTypeDWARFDebugStr)
-        .Case("str.dwo", eSectionTypeDWARFDebugStrDwo)
-        .Case("str_offsets", eSectionTypeDWARFDebugStrOffsets)
-        .Case("str_offsets.dwo", eSectionTypeDWARFDebugStrOffsetsDwo)
-        .Case("tu_index", eSectionTypeDWARFDebugTuIndex)
-        .Case("types", eSectionTypeDWARFDebugTypes)
-        .Case("types.dwo", eSectionTypeDWARFDebugTypesDwo)
-        .Default(eSectionTypeOther);
+    return ObjectFile::GetDWARFSectionTypeFromName(Name);
   }
   return eSectionTypeOther;
 }
diff --git a/external/llvm-project/lldb/source/Plugins/Platform/Android/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Platform/Android/CMakeLists.txt
index 7ffccc30a19f..71108371daf0 100644
--- a/external/llvm-project/lldb/source/Plugins/Platform/Android/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Platform/Android/CMakeLists.txt
@@ -11,14 +11,14 @@ add_lldb_library(lldbPluginPlatformAndroid PLUGIN
   PlatformAndroid.cpp
   PlatformAndroidRemoteGDBServer.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbValueObject
     lldbPluginPlatformLinux
     lldbPluginPlatformGDB
-  LINK_COMPONENTS
-    Support
   )
 
 add_dependencies(lldbPluginPlatformAndroid
diff --git a/external/llvm-project/lldb/source/Plugins/Platform/FreeBSD/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Platform/FreeBSD/CMakeLists.txt
index 530144592ae4..b9366da5ec0f 100644
--- a/external/llvm-project/lldb/source/Plugins/Platform/FreeBSD/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Platform/FreeBSD/CMakeLists.txt
@@ -1,13 +1,12 @@
 add_lldb_library(lldbPluginPlatformFreeBSD PLUGIN
   PlatformFreeBSD.cpp
 
+  LINK_COMPONENTS
+    TargetParser
    LINK_LIBS
     lldbBreakpoint
     lldbCore
     lldbHost
     lldbTarget
     lldbPluginPlatformPOSIX
-
-  LINK_COMPONENTS
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt
index 8c1e9f4013fb..598472430a9e 100644
--- a/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt
@@ -37,6 +37,9 @@ endif()
 add_lldb_library(lldbPluginPlatformMacOSX PLUGIN
   ${PLUGIN_PLATFORM_MACOSX_SOURCES}
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -51,9 +54,6 @@ add_lldb_library(lldbPluginPlatformMacOSX PLUGIN
     ${OBJC_LIBS}
   CLANG_LIBS
     clangBasic
-  LINK_COMPONENTS
-    Support
-    TargetParser
     )
 
 add_dependencies(lldbPluginPlatformMacOSX
diff --git a/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/objcxx/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/objcxx/CMakeLists.txt
index 7d094a5865c4..8640aac0862b 100644
--- a/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/objcxx/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Platform/MacOSX/objcxx/CMakeLists.txt
@@ -4,16 +4,15 @@ include_directories(.)
 add_lldb_library(lldbPluginPlatformMacOSXObjCXX
   PlatformiOSSimulatorCoreSimulatorSupport.mm
 
+  LINK_COMPONENTS
+    Object
+    Support
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbUtility
     ${EXTRA_LIBS}
-
-  LINK_COMPONENTS
-    Object
-    Support
   )
 
 
diff --git a/external/llvm-project/lldb/source/Plugins/Platform/QemuUser/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Platform/QemuUser/CMakeLists.txt
index 03a5ba17044f..dc3ff75079cc 100644
--- a/external/llvm-project/lldb/source/Plugins/Platform/QemuUser/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Platform/QemuUser/CMakeLists.txt
@@ -9,10 +9,10 @@ lldb_tablegen(PlatformQemuUserPropertiesEnum.inc -gen-lldb-property-enum-defs
 add_lldb_library(lldbPluginPlatformQemuUser PLUGIN
   PlatformQemuUser.cpp
 
-  LINK_LIBS
-    lldbUtility
   LINK_COMPONENTS
     Support
+  LINK_LIBS
+    lldbUtility
     )
 
 add_dependencies(lldbPluginPlatformQemuUser
diff --git a/external/llvm-project/lldb/source/Plugins/Platform/Windows/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Platform/Windows/CMakeLists.txt
index 2820ae77e749..7f0042522491 100644
--- a/external/llvm-project/lldb/source/Plugins/Platform/Windows/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Platform/Windows/CMakeLists.txt
@@ -1,13 +1,12 @@
 add_lldb_library(lldbPluginPlatformWindows PLUGIN
   PlatformWindows.cpp
 
+   LINK_COMPONENTS
+    Support
    LINK_LIBS
     lldbBreakpoint
     lldbCore
     lldbHost
     lldbTarget
     lldbPluginPlatformGDB
-
-   LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp b/external/llvm-project/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
index 26ca6ed12897..4a77423bac52 100644
--- a/external/llvm-project/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
@@ -800,7 +800,10 @@ std::string PlatformRemoteGDBServer::MakeUrl(const char *scheme,
                                              const char *hostname,
                                              uint16_t port, const char *path) {
   StreamString result;
-  result.Printf("%s://[%s]", scheme, hostname);
+  result.Printf("%s://", scheme);
+  if (strlen(hostname) > 0)
+    result.Printf("[%s]", hostname);
+
   if (port != 0)
     result.Printf(":%u", port);
   if (path)
diff --git a/external/llvm-project/lldb/source/Plugins/Process/AIX/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/AIX/CMakeLists.txt
index 911f30349ef5..6b3151edbd1e 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/AIX/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/AIX/CMakeLists.txt
@@ -2,6 +2,8 @@ add_lldb_library(lldbPluginProcessAIX
   NativeProcessAIX.cpp
   NativeThreadAIX.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -10,8 +12,6 @@ add_lldb_library(lldbPluginProcessAIX
     lldbUtility
     lldbPluginProcessPOSIX
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
 
 target_compile_definitions(lldbPluginProcessAIX PRIVATE "-D_ALL_SOURCE")
diff --git a/external/llvm-project/lldb/source/Plugins/Process/FreeBSD/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/FreeBSD/CMakeLists.txt
index 598911ce8c95..4c11d21b3e5a 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/FreeBSD/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/FreeBSD/CMakeLists.txt
@@ -8,6 +8,8 @@ add_lldb_library(lldbPluginProcessFreeBSD
   NativeRegisterContextFreeBSD_x86_64.cpp
   NativeThreadFreeBSD.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbHost
     lldbSymbol
@@ -15,6 +17,4 @@ add_lldb_library(lldbPluginProcessFreeBSD
     lldbUtility
     lldbPluginProcessPOSIX
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt
index ca89ab0b6dc9..b0d1370286a8 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt
@@ -18,10 +18,10 @@ add_lldb_library(lldbPluginProcessFreeBSDKernel PLUGIN
   RegisterContextFreeBSDKernel_x86_64.cpp
   ThreadFreeBSDKernel.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbTarget
     ${FBSDKERNEL_LIBS}
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/Linux/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/Linux/CMakeLists.txt
index 708252f887bd..33af2e24dedd 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/Linux/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/Linux/CMakeLists.txt
@@ -18,6 +18,9 @@ add_lldb_library(lldbPluginProcessLinux
   Procfs.cpp
   SingleStepCheck.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbHost
@@ -26,7 +29,4 @@ add_lldb_library(lldbPluginProcessLinux
     lldbUtility
     lldbPluginProcessPOSIX
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/Linux/Procfs.cpp b/external/llvm-project/lldb/source/Plugins/Process/Linux/Procfs.cpp
index d3bd396fbaea..4349a2b1adb9 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/Linux/Procfs.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Process/Linux/Procfs.cpp
@@ -74,7 +74,7 @@ lldb_private::process_linux::GetAvailableLogicalCoreIDs() {
 llvm::Expected<int> lldb_private::process_linux::GetPtraceScope() {
   ErrorOr<std::unique_ptr<MemoryBuffer>> ptrace_scope_file =
       getProcFile("sys/kernel/yama/ptrace_scope");
-  if (!*ptrace_scope_file)
+  if (!ptrace_scope_file)
     return errorCodeToError(ptrace_scope_file.getError());
   // The contents should be something like "1\n". Trim it so we get "1".
   StringRef buffer = (*ptrace_scope_file)->getBuffer().trim();
diff --git a/external/llvm-project/lldb/source/Plugins/Process/NetBSD/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/NetBSD/CMakeLists.txt
index 13f3790497b9..f48ea86fb0a3 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/NetBSD/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/NetBSD/CMakeLists.txt
@@ -4,6 +4,8 @@ add_lldb_library(lldbPluginProcessNetBSD
   NativeRegisterContextNetBSD_x86_64.cpp
   NativeThreadNetBSD.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbHost
     lldbSymbol
@@ -12,6 +14,4 @@ add_lldb_library(lldbPluginProcessNetBSD
     lldbPluginProcessPOSIX
     lldbPluginProcessUtility
     util
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/POSIX/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/POSIX/CMakeLists.txt
index 81effaaf3558..ee6fa6a3fc23 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/POSIX/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/POSIX/CMakeLists.txt
@@ -3,9 +3,9 @@ add_lldb_library(lldbPluginProcessPOSIX
   NativeProcessELF.cpp
   ProcessPOSIXLog.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbPluginProcessUtility
     lldbUtility
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/Utility/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/Utility/CMakeLists.txt
index 8b36919fda61..fd3019613892 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/Utility/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/Utility/CMakeLists.txt
@@ -63,6 +63,9 @@ add_lldb_library(lldbPluginProcessUtility
   StopInfoMachException.cpp
   ThreadMemory.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -73,7 +76,4 @@ add_lldb_library(lldbPluginProcessUtility
     lldbTarget
     lldbUtility
     lldbValueObject
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/Windows/Common/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/Windows/Common/CMakeLists.txt
index e77c204fd61e..9854b79fbb8d 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/Windows/Common/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/Windows/Common/CMakeLists.txt
@@ -20,6 +20,8 @@ add_lldb_library(lldbPluginProcessWindowsCommon PLUGIN
   x64/RegisterContextWindows_x64.cpp
   x86/RegisterContextWindows_x86.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -30,6 +32,4 @@ add_lldb_library(lldbPluginProcessWindowsCommon PLUGIN
     ws2_32
     rpcrt4
     psapi
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/elf-core/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/elf-core/CMakeLists.txt
index 4b0f5199df71..0bc26bb0efbe 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/elf-core/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/elf-core/CMakeLists.txt
@@ -14,13 +14,13 @@ add_lldb_library(lldbPluginProcessElfCore PLUGIN
   RegisterContextPOSIXCore_loongarch64.cpp
   RegisterUtilities.cpp
 
+  LINK_COMPONENTS
+    BinaryFormat
+    Support
   LINK_LIBS
     lldbCore
     lldbTarget
     lldbPluginDynamicLoaderPosixDYLD
     lldbPluginObjectFileELF
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    BinaryFormat
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
index 6755999b1818..5c39384fa31b 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
@@ -33,6 +33,9 @@ add_lldb_library(lldbPluginProcessGDBRemote PLUGIN
   ProcessGDBRemoteLog.cpp
   ThreadGDBRemote.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -45,9 +48,6 @@ add_lldb_library(lldbPluginProcessGDBRemote PLUGIN
     ${LLDB_SYSTEM_LIBS}
     ${LLDB_PLUGINS}
     ${LIBCOMPRESSION}
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
 
 add_dependencies(lldbPluginProcessGDBRemote
diff --git a/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index 67ba42f33d1d..4a1117222f34 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -1377,7 +1377,7 @@ GDBRemoteCommunicationServerCommon::GetModuleInfo(llvm::StringRef module_path,
 
 std::vector<std::string> GDBRemoteCommunicationServerCommon::HandleFeatures(
     const llvm::ArrayRef<llvm::StringRef> client_features) {
-  // 128KBytes is a reasonable max packet size--debugger can always use less.
+  // 128 KiB is a reasonable max packet size--debugger can always use less.
   constexpr uint32_t max_packet_size = 128 * 1024;
 
   // Features common to platform server and llgs.
diff --git a/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 420c84b496d1..f18bdd5175f2 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/external/llvm-project/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -1728,7 +1728,7 @@ ThreadSP ProcessGDBRemote::SetThreadStopInfo(
 
   reg_ctx_sp->InvalidateIfNeeded(true);
 
-  auto iter = std::find(m_thread_ids.begin(), m_thread_ids.end(), tid);
+  auto iter = llvm::find(m_thread_ids, tid);
   if (iter != m_thread_ids.end())
     SetThreadPc(thread_sp, iter - m_thread_ids.begin());
 
diff --git a/external/llvm-project/lldb/source/Plugins/Process/mach-core/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
index e79cd82c92a6..a1ea85ec4c72 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
@@ -2,6 +2,8 @@ add_lldb_library(lldbPluginProcessMachCore PLUGIN
   ProcessMachCore.cpp
   ThreadMachCore.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -12,6 +14,4 @@ add_lldb_library(lldbPluginProcessMachCore PLUGIN
     lldbPluginDynamicLoaderDarwinKernel
     lldbPluginDynamicLoaderMacOSXDYLD
     lldbPluginObjectFileMachO
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/minidump/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/minidump/CMakeLists.txt
index 9af816884c21..d68534247f6b 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/minidump/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/minidump/CMakeLists.txt
@@ -8,6 +8,10 @@ add_lldb_library(lldbPluginProcessMinidump PLUGIN
   ProcessMinidump.cpp
   ThreadMinidump.cpp
 
+  LINK_COMPONENTS
+    BinaryFormat
+    Object
+    Support
   LINK_LIBS
     lldbCore
     lldbTarget
@@ -15,8 +19,4 @@ add_lldb_library(lldbPluginProcessMinidump PLUGIN
     lldbPluginProcessUtility
     lldbPluginProcessElfCore
     lldbPluginObjectFilePlaceholder
-  LINK_COMPONENTS
-    BinaryFormat
-    Object
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/Process/scripted/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Process/scripted/CMakeLists.txt
index 600ef31d032e..590166591a41 100644
--- a/external/llvm-project/lldb/source/Plugins/Process/scripted/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Process/scripted/CMakeLists.txt
@@ -2,13 +2,13 @@ add_lldb_library(lldbPluginScriptedProcess PLUGIN
   ScriptedProcess.cpp
   ScriptedThread.cpp
 
+  LINK_COMPONENTS
+    BinaryFormat
+    Object
+    Support
   LINK_LIBS
     lldbCore
     lldbTarget
     lldbUtility
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    BinaryFormat
-    Object
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/REPL/Clang/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/REPL/Clang/CMakeLists.txt
index b99507123581..3a7e188d7a29 100644
--- a/external/llvm-project/lldb/source/Plugins/REPL/Clang/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/REPL/Clang/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_lldb_library(lldbPluginClangREPL PLUGIN
   ClangREPL.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbDataFormatters
@@ -11,7 +13,4 @@ add_lldb_library(lldbPluginClangREPL PLUGIN
     lldbPluginClangCommon
     lldbPluginCPPRuntime
     lldbPluginTypeSystemClang
-
-  LINK_COMPONENTS
-    Support
 )
diff --git a/external/llvm-project/lldb/source/Plugins/RegisterTypeBuilder/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/RegisterTypeBuilder/CMakeLists.txt
index 7abd34a5cd0e..336ae9105802 100644
--- a/external/llvm-project/lldb/source/Plugins/RegisterTypeBuilder/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/RegisterTypeBuilder/CMakeLists.txt
@@ -1,10 +1,9 @@
 add_lldb_library(lldbPluginRegisterTypeBuilderClang PLUGIN
   RegisterTypeBuilderClang.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbTarget
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
index fd3118aa9c0a..3d09cea46446 100644
--- a/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
@@ -26,6 +26,8 @@ add_lldb_library(lldbPluginScriptInterpreterPython PLUGIN
   PythonReadline.cpp
   ScriptInterpreterPython.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -37,7 +39,4 @@ add_lldb_library(lldbPluginScriptInterpreterPython PLUGIN
     lldbPluginScriptInterpreterPythonInterfaces
     ${Python3_LIBRARIES}
     ${LLDB_LIBEDIT_LIBS}
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/CMakeLists.txt
index ee5e48ad5cdc..db9e11b73895 100644
--- a/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/CMakeLists.txt
@@ -29,6 +29,8 @@ add_lldb_library(lldbPluginScriptInterpreterPythonInterfaces PLUGIN
   ScriptedThreadPlanPythonInterface.cpp
   ScriptedThreadPythonInterface.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -36,9 +38,6 @@ add_lldb_library(lldbPluginScriptInterpreterPythonInterfaces PLUGIN
     lldbTarget
     ${Python3_LIBRARIES}
     ${LLDB_LIBEDIT_LIBS}
-
-  LINK_COMPONENTS
-    Support
   )
 
 
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/Breakpad/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/SymbolFile/Breakpad/CMakeLists.txt
index e93d0618a0e4..8360a6bff5af 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/Breakpad/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/Breakpad/CMakeLists.txt
@@ -1,12 +1,12 @@
 add_lldb_library(lldbPluginSymbolFileBreakpad PLUGIN
   SymbolFileBreakpad.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbUtility
     lldbPluginObjectFileBreakpad
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp
index c0b931f5c131..f4d032388a88 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp
@@ -489,8 +489,8 @@ SymbolFileCTF::CreateFunction(const CTFFunction &ctf_function) {
         llvm::inconvertibleErrorCode());
 
   CompilerType func_type = m_ast->CreateFunctionType(
-      ret_type->GetFullCompilerType(), arg_types.data(), arg_types.size(),
-      ctf_function.variadic, 0, clang::CallingConv::CC_C);
+      ret_type->GetFullCompilerType(), arg_types, ctf_function.variadic, 0,
+      clang::CallingConv::CC_C);
 
   Declaration decl;
   return MakeType(ctf_function.uid, ConstString(ctf_function.name), 0, nullptr,
@@ -814,8 +814,7 @@ size_t SymbolFileCTF::ParseFunctions(CompileUnit &cu) {
       // Create function type.
       CompilerType func_type = m_ast->CreateFunctionType(
           ret_type ? ret_type->GetFullCompilerType() : CompilerType(),
-          arg_types.data(), arg_types.size(), is_variadic, 0,
-          clang::CallingConv::CC_C);
+          arg_types, is_variadic, 0, clang::CallingConv::CC_C);
       lldb::user_id_t function_type_uid = m_types.size() + 1;
       TypeSP type_sp =
           MakeType(function_type_uid, symbol->GetName(), 0, nullptr,
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
index 2d2eb73f4513..212cc3610acf 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
@@ -37,6 +37,9 @@ add_lldb_library(lldbPluginSymbolFileDWARF PLUGIN
   SymbolFileDWARFDebugMap.cpp
   UniqueDWARFASTType.cpp
 
+  LINK_COMPONENTS
+    DebugInfoDWARF
+    Support
   LINK_LIBS
     lldbCore
     lldbExpression
@@ -53,9 +56,6 @@ add_lldb_library(lldbPluginSymbolFileDWARF PLUGIN
   CLANG_LIBS
     clangAST
     clangBasic
-  LINK_COMPONENTS
-    DebugInfoDWARF
-    Support
   )
 
 add_dependencies(lldbPluginSymbolFileDWARF
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 0c26c276cc53..620501b304e6 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -1309,11 +1309,10 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
 
   // clang_type will get the function prototype clang type after this
   // call
-  CompilerType clang_type =
-      m_ast.CreateFunctionType(return_clang_type, function_param_types.data(),
-                               function_param_types.size(), is_variadic,
-                               GetCXXMethodCVQuals(die, object_parameter),
-                               calling_convention, attrs.ref_qual);
+  CompilerType clang_type = m_ast.CreateFunctionType(
+      return_clang_type, function_param_types, is_variadic,
+      GetCXXMethodCVQuals(die, object_parameter), calling_convention,
+      attrs.ref_qual);
 
   if (attrs.name) {
     bool type_handled = false;
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
index 5196ce89a2c1..8217c85f8601 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
@@ -611,7 +611,11 @@ void DWARFDebugInfoEntry::BuildFunctionAddressRangeTable(
     DWARFUnit *cu, DWARFDebugAranges *debug_aranges) const {
   Log *log = GetLog(DWARFLog::DebugInfo);
   if (m_tag) {
-    if (m_tag == DW_TAG_subprogram) {
+    // Subprogram forward declarations don't have
+    // DW_AT_ranges/DW_AT_low_pc/DW_AT_high_pc attributes, so don't even try
+    // getting address range information for them.
+    if (m_tag == DW_TAG_subprogram &&
+        !GetAttributeValueAsOptionalUnsigned(cu, DW_AT_declaration)) {
       if (llvm::Expected<llvm::DWARFAddressRangesVector> ranges =
               GetAttributeAddressRanges(cu, /*check_hi_lo_pc=*/true)) {
         for (const auto &r : *ranges)
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 24f905a02331..71f204c03a42 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -4127,7 +4127,7 @@ void SymbolFileDWARF::Dump(lldb_private::Stream &s) {
   m_index->Dump(s);
 }
 
-void SymbolFileDWARF::DumpClangAST(Stream &s) {
+void SymbolFileDWARF::DumpClangAST(Stream &s, llvm::StringRef filter) {
   auto ts_or_err = GetTypeSystemForLanguage(eLanguageTypeC_plus_plus);
   if (!ts_or_err)
     return;
@@ -4135,7 +4135,7 @@ void SymbolFileDWARF::DumpClangAST(Stream &s) {
   TypeSystemClang *clang = llvm::dyn_cast_or_null<TypeSystemClang>(ts.get());
   if (!clang)
     return;
-  clang->Dump(s.AsRawOstream());
+  clang->Dump(s.AsRawOstream(), filter);
 }
 
 bool SymbolFileDWARF::GetSeparateDebugInfo(StructuredData::Dictionary &d,
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index c9fdfb7b1cf9..d2d30d7decb1 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -276,7 +276,7 @@ class SymbolFileDWARF : public SymbolFileCommon {
 
   void Dump(Stream &s) override;
 
-  void DumpClangAST(Stream &s) override;
+  void DumpClangAST(Stream &s, llvm::StringRef filter) override;
 
   /// List separate dwo files.
   bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
index 961c212e2e6d..f3a940b2ee39 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
@@ -1267,9 +1267,9 @@ CompilerDeclContext SymbolFileDWARFDebugMap::FindNamespace(
   return matching_namespace;
 }
 
-void SymbolFileDWARFDebugMap::DumpClangAST(Stream &s) {
-  ForEachSymbolFile("Dumping clang AST", [&s](SymbolFileDWARF &oso_dwarf) {
-    oso_dwarf.DumpClangAST(s);
+void SymbolFileDWARFDebugMap::DumpClangAST(Stream &s, llvm::StringRef filter) {
+  ForEachSymbolFile("Dumping clang AST", [&](SymbolFileDWARF &oso_dwarf) {
+    oso_dwarf.DumpClangAST(s, filter);
     // The underlying assumption is that DumpClangAST(...) will obtain the
     // AST from the underlying TypeSystem and therefore we only need to do
     // this once and can stop after the first iteration hence we return true.
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
index 8399a267d785..35cbdbbb1692 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
@@ -129,7 +129,7 @@ class SymbolFileDWARFDebugMap : public SymbolFileCommon {
   std::vector<std::unique_ptr<CallEdge>>
   ParseCallEdgesInFunction(UserID func_id) override;
 
-  void DumpClangAST(Stream &s) override;
+  void DumpClangAST(Stream &s, llvm::StringRef filter) override;
 
   /// List separate oso files.
   bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/CMakeLists.txt
index caef0eef6ba5..21756bb26c85 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/CMakeLists.txt
@@ -10,6 +10,11 @@ add_lldb_library(lldbPluginSymbolFileNativePDB
   SymbolFileNativePDB.cpp
   UdtRecordCompleter.cpp
 
+  LINK_COMPONENTS
+    DebugInfoCodeView
+    DebugInfoPDB
+    Support
+    TargetParser
   LINK_LIBS
     lldbCore
     lldbSymbol
@@ -19,9 +24,4 @@ add_lldb_library(lldbPluginSymbolFileNativePDB
   CLANG_LIBS
     clangAST
     clangLex
-  LINK_COMPONENTS
-    DebugInfoCodeView
-    DebugInfoPDB
-    Support
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
index 92840da74a21..702ec5e5c9ea 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
@@ -1216,8 +1216,8 @@ clang::QualType PdbAstBuilder::CreateFunctionType(
     return {};
 
   CompilerType return_ct = ToCompilerType(return_type);
-  CompilerType func_sig_ast_type = m_clang.CreateFunctionType(
-      return_ct, arg_types.data(), arg_types.size(), is_variadic, 0, *cc);
+  CompilerType func_sig_ast_type =
+      m_clang.CreateFunctionType(return_ct, arg_types, is_variadic, 0, *cc);
 
   return clang::QualType::getFromOpaquePtr(
       func_sig_ast_type.GetOpaqueQualType());
@@ -1449,6 +1449,6 @@ PdbAstBuilder::FromCompilerDeclContext(CompilerDeclContext context) {
   return static_cast<clang::DeclContext *>(context.GetOpaqueDeclContext());
 }
 
-void PdbAstBuilder::Dump(Stream &stream) {
-  m_clang.Dump(stream.AsRawOstream());
+void PdbAstBuilder::Dump(Stream &stream, llvm::StringRef filter) {
+  m_clang.Dump(stream.AsRawOstream(), filter);
 }
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h
index b7cad30c69c0..66a3836fac05 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h
@@ -87,7 +87,7 @@ class PdbAstBuilder {
   TypeSystemClang &clang() { return m_clang; }
   ClangASTImporter &GetClangASTImporter() { return m_importer; }
 
-  void Dump(Stream &stream);
+  void Dump(Stream &stream, llvm::StringRef filter);
 
 private:
   clang::Decl *TryGetDecl(PdbSymUid uid) const;
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
index dadc969c48a3..20d8c1acf9c4 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -1630,7 +1630,7 @@ size_t SymbolFileNativePDB::ParseSymbolArrayInScope(
   return count;
 }
 
-void SymbolFileNativePDB::DumpClangAST(Stream &s) {
+void SymbolFileNativePDB::DumpClangAST(Stream &s, llvm::StringRef filter) {
   auto ts_or_err = GetTypeSystemForLanguage(eLanguageTypeC_plus_plus);
   if (!ts_or_err)
     return;
@@ -1638,7 +1638,7 @@ void SymbolFileNativePDB::DumpClangAST(Stream &s) {
   TypeSystemClang *clang = llvm::dyn_cast_or_null<TypeSystemClang>(ts.get());
   if (!clang)
     return;
-  clang->GetNativePDBParser()->Dump(s);
+  clang->GetNativePDBParser()->Dump(s, filter);
 }
 
 void SymbolFileNativePDB::FindGlobalVariables(
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
index b0e78a243a3c..9891313f11d0 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
@@ -157,7 +157,7 @@ class SymbolFileNativePDB : public SymbolFileCommon {
 
   PdbIndex &GetIndex() { return *m_index; };
 
-  void DumpClangAST(Stream &s) override;
+  void DumpClangAST(Stream &s, llvm::StringRef filter) override;
 
   std::optional<llvm::codeview::TypeIndex>
   GetParentType(llvm::codeview::TypeIndex ti);
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/CMakeLists.txt
index ceeb173a99e1..64590d7977b1 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/CMakeLists.txt
@@ -3,6 +3,9 @@ add_lldb_library(lldbPluginSymbolFilePDB PLUGIN
   PDBLocationToDWARFExpression.cpp
   SymbolFilePDB.cpp
 
+  LINK_COMPONENTS
+    DebugInfoPDB
+    Support
   LINK_LIBS
     lldbCore
     lldbPluginSymbolFileNativePDB
@@ -12,7 +15,4 @@ add_lldb_library(lldbPluginSymbolFilePDB PLUGIN
   CLANG_LIBS
     clangAST
     clangLex
-  LINK_COMPONENTS
-    DebugInfoPDB
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
index c6dd72e22fb4..0090d8ff03ab 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
@@ -653,9 +653,8 @@ lldb::TypeSP PDBASTParser::CreateLLDBTypeFromPDBType(const PDBSymbol &type) {
     if (func_sig->isVolatileType())
       type_quals |= clang::Qualifiers::Volatile;
     auto cc = TranslateCallingConvention(func_sig->getCallingConvention());
-    CompilerType func_sig_ast_type =
-        m_ast.CreateFunctionType(return_ast_type, arg_list.data(),
-                                 arg_list.size(), is_variadic, type_quals, cc);
+    CompilerType func_sig_ast_type = m_ast.CreateFunctionType(
+        return_ast_type, arg_list, is_variadic, type_quals, cc);
 
     AddSourceInfoToDecl(type, decl);
     return m_ast.GetSymbolFile()->MakeType(
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
index f1d98e780994..96519ae0b315 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
@@ -1431,7 +1431,7 @@ void SymbolFilePDB::AddSymbols(lldb_private::Symtab &symtab) {
   symtab.Finalize();
 }
 
-void SymbolFilePDB::DumpClangAST(Stream &s) {
+void SymbolFilePDB::DumpClangAST(Stream &s, llvm::StringRef filter) {
   auto type_system_or_err =
       GetTypeSystemForLanguage(lldb::eLanguageTypeC_plus_plus);
   if (auto err = type_system_or_err.takeError()) {
@@ -1445,7 +1445,7 @@ void SymbolFilePDB::DumpClangAST(Stream &s) {
       llvm::dyn_cast_or_null<TypeSystemClang>(ts.get());
   if (!clang_type_system)
     return;
-  clang_type_system->Dump(s.AsRawOstream());
+  clang_type_system->Dump(s.AsRawOstream(), filter);
 }
 
 void SymbolFilePDB::FindTypesByRegex(
diff --git a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
index ea495c575f1f..c0b25b6ee405 100644
--- a/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
+++ b/external/llvm-project/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
@@ -157,7 +157,7 @@ class SymbolFilePDB : public lldb_private::SymbolFileCommon {
 
   const llvm::pdb::IPDBSession &GetPDBSession() const;
 
-  void DumpClangAST(lldb_private::Stream &s) override;
+  void DumpClangAST(lldb_private::Stream &s, llvm::StringRef filter) override;
 
 private:
   struct SecContribInfo {
diff --git a/external/llvm-project/lldb/source/Plugins/Trace/intel-pt/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/Trace/intel-pt/CMakeLists.txt
index 90f36ed31078..ddb2f384d184 100644
--- a/external/llvm-project/lldb/source/Plugins/Trace/intel-pt/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/Trace/intel-pt/CMakeLists.txt
@@ -35,14 +35,14 @@ add_lldb_library(lldbPluginTraceIntelPT PLUGIN
   TraceIntelPTBundleLoader.cpp
   TraceIntelPTBundleSaver.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginTraceCommon
     ${LIBIPT_LIBRARY}
-  LINK_COMPONENTS
-    Support
   )
 
 
diff --git a/external/llvm-project/lldb/source/Plugins/TraceExporter/ctf/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/TraceExporter/ctf/CMakeLists.txt
index 45a92a4b3e38..ea538616292f 100644
--- a/external/llvm-project/lldb/source/Plugins/TraceExporter/ctf/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/TraceExporter/ctf/CMakeLists.txt
@@ -6,13 +6,13 @@ add_lldb_library(lldbPluginTraceExporterCTF PLUGIN
 CommandObjectThreadTraceExportCTF.cpp
   TraceExporterCTF.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbPluginTraceExporterCommon
-  LINK_COMPONENTS
-    Support
   )
 
 add_dependencies(lldbPluginTraceExporterCTF TraceExporterCTFOptionsGen)
diff --git a/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/CMakeLists.txt
index 37a3142da727..90f173bc442d 100644
--- a/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_lldb_library(lldbPluginTypeSystemClang PLUGIN
   TypeSystemClang.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbSymbol
@@ -15,6 +17,4 @@ add_lldb_library(lldbPluginTypeSystemClang PLUGIN
     clangBasic
     clangFrontend
     clangSema
-  LINK_COMPONENTS
-    Support
 )
diff --git a/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index c1917203b684..29302413cf8f 100644
--- a/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -10,6 +10,7 @@
 
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/Frontend/ASTConsumers.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/FormatAdapters.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -2180,25 +2181,22 @@ FunctionDecl *TypeSystemClang::CreateFunctionDeclaration(
 }
 
 CompilerType TypeSystemClang::CreateFunctionType(
-    const CompilerType &result_type, const CompilerType *args,
-    unsigned num_args, bool is_variadic, unsigned type_quals,
-    clang::CallingConv cc, clang::RefQualifierKind ref_qual) {
+    const CompilerType &result_type, llvm::ArrayRef<CompilerType> args,
+    bool is_variadic, unsigned type_quals, clang::CallingConv cc,
+    clang::RefQualifierKind ref_qual) {
   if (!result_type || !ClangUtil::IsClangType(result_type))
     return CompilerType(); // invalid return type
 
   std::vector<QualType> qual_type_args;
-  if (num_args > 0 && args == nullptr)
-    return CompilerType(); // invalid argument array passed in
-
   // Verify that all arguments are valid and the right type
-  for (unsigned i = 0; i < num_args; ++i) {
-    if (args[i]) {
+  for (const auto &arg : args) {
+    if (arg) {
       // Make sure we have a clang type in args[i] and not a type from another
       // language whose name might match
-      const bool is_clang_type = ClangUtil::IsClangType(args[i]);
+      const bool is_clang_type = ClangUtil::IsClangType(arg);
       lldbassert(is_clang_type);
       if (is_clang_type)
-        qual_type_args.push_back(ClangUtil::GetQualType(args[i]));
+        qual_type_args.push_back(ClangUtil::GetQualType(arg));
       else
         return CompilerType(); //  invalid argument type (must be a clang type)
     } else
@@ -8499,8 +8497,16 @@ TypeSystemClang::dump(lldb::opaque_compiler_type_t type) const {
 }
 #endif
 
-void TypeSystemClang::Dump(llvm::raw_ostream &output) {
-  GetTranslationUnitDecl()->dump(output);
+void TypeSystemClang::Dump(llvm::raw_ostream &output, llvm::StringRef filter) {
+  auto consumer =
+      clang::CreateASTDumper(output, filter,
+                             /*DumpDecls=*/true,
+                             /*Deserialize=*/false,
+                             /*DumpLookups=*/false,
+                             /*DumpDeclTypes=*/false, clang::ADOF_Default);
+  assert(consumer);
+  assert(m_ast_up);
+  consumer->HandleTranslationUnit(*m_ast_up);
 }
 
 void TypeSystemClang::DumpFromSymbolFile(Stream &s,
@@ -9625,10 +9631,11 @@ GetNameForIsolatedASTKind(ScratchTypeSystemClang::IsolatedASTKind kind) {
   llvm_unreachable("Unimplemented IsolatedASTKind?");
 }
 
-void ScratchTypeSystemClang::Dump(llvm::raw_ostream &output) {
+void ScratchTypeSystemClang::Dump(llvm::raw_ostream &output,
+                                  llvm::StringRef filter) {
   // First dump the main scratch AST.
   output << "State of scratch Clang type system:\n";
-  TypeSystemClang::Dump(output);
+  TypeSystemClang::Dump(output, filter);
 
   // Now sort the isolated sub-ASTs.
   typedef std::pair<IsolatedASTKey, TypeSystem *> KeyAndTS;
@@ -9643,7 +9650,7 @@ void ScratchTypeSystemClang::Dump(llvm::raw_ostream &output) {
         static_cast<ScratchTypeSystemClang::IsolatedASTKind>(a.first);
     output << "State of scratch Clang type subsystem "
            << GetNameForIsolatedASTKind(kind) << ":\n";
-    a.second->Dump(output);
+    a.second->Dump(output, filter);
   }
 }
 
diff --git a/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index b9e78cc377f0..63dee9dceded 100644
--- a/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/external/llvm-project/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -480,9 +480,9 @@ class TypeSystemClang : public TypeSystem {
       clang::StorageClass storage, bool is_inline);
 
   CompilerType
-  CreateFunctionType(const CompilerType &result_type, const CompilerType *args,
-                     unsigned num_args, bool is_variadic, unsigned type_quals,
-                     clang::CallingConv cc = clang::CC_C,
+  CreateFunctionType(const CompilerType &result_type,
+                     llvm::ArrayRef<CompilerType> args, bool is_variadic,
+                     unsigned type_quals, clang::CallingConv cc = clang::CC_C,
                      clang::RefQualifierKind ref_qual = clang::RQ_None);
 
   clang::ParmVarDecl *
@@ -1074,7 +1074,7 @@ class TypeSystemClang : public TypeSystem {
 #endif
 
   /// \see lldb_private::TypeSystem::Dump
-  void Dump(llvm::raw_ostream &output) override;
+  void Dump(llvm::raw_ostream &output, llvm::StringRef filter) override;
 
   /// Dump clang AST types from the symbol file.
   ///
@@ -1318,7 +1318,7 @@ class ScratchTypeSystemClang : public TypeSystemClang {
   }
 
   /// \see lldb_private::TypeSystem::Dump
-  void Dump(llvm::raw_ostream &output) override;
+  void Dump(llvm::raw_ostream &output, llvm::StringRef filter) override;
 
   UserExpression *GetUserExpression(llvm::StringRef expr,
                                     llvm::StringRef prefix,
diff --git a/external/llvm-project/lldb/source/Plugins/UnwindAssembly/x86/CMakeLists.txt b/external/llvm-project/lldb/source/Plugins/UnwindAssembly/x86/CMakeLists.txt
index 425ae1190992..199d1da66a7a 100644
--- a/external/llvm-project/lldb/source/Plugins/UnwindAssembly/x86/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Plugins/UnwindAssembly/x86/CMakeLists.txt
@@ -2,13 +2,13 @@ add_lldb_library(lldbPluginUnwindAssemblyX86 PLUGIN
   UnwindAssembly-x86.cpp
   x86AssemblyInspectionEngine.cpp
 
+  LINK_COMPONENTS
+    Support
+    MC
+    MCDisassembler
   LINK_LIBS
     lldbCore
     lldbSymbol
     lldbTarget
     lldbUtility
-  LINK_COMPONENTS
-    Support
-    MC
-    MCDisassembler
   )
diff --git a/external/llvm-project/lldb/source/Symbol/CMakeLists.txt b/external/llvm-project/lldb/source/Symbol/CMakeLists.txt
index 86fa8583972b..28d12b301279 100644
--- a/external/llvm-project/lldb/source/Symbol/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Symbol/CMakeLists.txt
@@ -33,6 +33,10 @@ add_lldb_library(lldbSymbol NO_PLUGIN_DEPENDENCIES
   Variable.cpp
   VariableList.cpp
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Symbol
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbExpression
@@ -40,7 +44,4 @@ add_lldb_library(lldbSymbol NO_PLUGIN_DEPENDENCIES
     lldbTarget
     lldbUtility
     lldbValueObject
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/source/Symbol/ObjectFile.cpp b/external/llvm-project/lldb/source/Symbol/ObjectFile.cpp
index afd0d298e675..21daf7476b52 100644
--- a/external/llvm-project/lldb/source/Symbol/ObjectFile.cpp
+++ b/external/llvm-project/lldb/source/Symbol/ObjectFile.cpp
@@ -635,6 +635,41 @@ ObjectFile::GetSymbolTypeFromName(llvm::StringRef name,
   return symbol_type_hint;
 }
 
+lldb::SectionType
+ObjectFile::GetDWARFSectionTypeFromName(llvm::StringRef name) {
+  return llvm::StringSwitch<SectionType>(name)
+      .Case("abbrev", eSectionTypeDWARFDebugAbbrev)
+      .Case("abbrev.dwo", eSectionTypeDWARFDebugAbbrevDwo)
+      .Case("addr", eSectionTypeDWARFDebugAddr)
+      .Case("aranges", eSectionTypeDWARFDebugAranges)
+      .Case("cu_index", eSectionTypeDWARFDebugCuIndex)
+      .Case("frame", eSectionTypeDWARFDebugFrame)
+      .Case("info", eSectionTypeDWARFDebugInfo)
+      .Case("info.dwo", eSectionTypeDWARFDebugInfoDwo)
+      .Cases("line", "line.dwo", eSectionTypeDWARFDebugLine)
+      .Cases("line_str", "line_str.dwo", eSectionTypeDWARFDebugLineStr)
+      .Case("loc", eSectionTypeDWARFDebugLoc)
+      .Case("loc.dwo", eSectionTypeDWARFDebugLocDwo)
+      .Case("loclists", eSectionTypeDWARFDebugLocLists)
+      .Case("loclists.dwo", eSectionTypeDWARFDebugLocListsDwo)
+      .Case("macinfo", eSectionTypeDWARFDebugMacInfo)
+      .Cases("macro", "macro.dwo", eSectionTypeDWARFDebugMacro)
+      .Case("names", eSectionTypeDWARFDebugNames)
+      .Case("pubnames", eSectionTypeDWARFDebugPubNames)
+      .Case("pubtypes", eSectionTypeDWARFDebugPubTypes)
+      .Case("ranges", eSectionTypeDWARFDebugRanges)
+      .Case("rnglists", eSectionTypeDWARFDebugRngLists)
+      .Case("rnglists.dwo", eSectionTypeDWARFDebugRngListsDwo)
+      .Case("str", eSectionTypeDWARFDebugStr)
+      .Case("str.dwo", eSectionTypeDWARFDebugStrDwo)
+      .Case("str_offsets", eSectionTypeDWARFDebugStrOffsets)
+      .Case("str_offsets.dwo", eSectionTypeDWARFDebugStrOffsetsDwo)
+      .Case("tu_index", eSectionTypeDWARFDebugTuIndex)
+      .Case("types", eSectionTypeDWARFDebugTypes)
+      .Case("types.dwo", eSectionTypeDWARFDebugTypesDwo)
+      .Default(eSectionTypeOther);
+}
+
 std::vector<ObjectFile::LoadableData>
 ObjectFile::GetLoadableData(Target &target) {
   std::vector<LoadableData> loadables;
diff --git a/external/llvm-project/lldb/source/Symbol/SaveCoreOptions.cpp b/external/llvm-project/lldb/source/Symbol/SaveCoreOptions.cpp
index e51ae2795493..d884b00a47b0 100644
--- a/external/llvm-project/lldb/source/Symbol/SaveCoreOptions.cpp
+++ b/external/llvm-project/lldb/source/Symbol/SaveCoreOptions.cpp
@@ -24,7 +24,6 @@ Status SaveCoreOptions::SetPluginName(const char *name) {
   if (!PluginManager::IsRegisteredObjectFilePluginName(name)) {
     return Status::FromErrorStringWithFormat(
         "plugin name '%s' is not a valid ObjectFile plugin name", name);
-    return error;
   }
 
   m_plugin_name = name;
diff --git a/external/llvm-project/lldb/source/Symbol/SymbolFileOnDemand.cpp b/external/llvm-project/lldb/source/Symbol/SymbolFileOnDemand.cpp
index 94979b2fb1c2..807c2124e48d 100644
--- a/external/llvm-project/lldb/source/Symbol/SymbolFileOnDemand.cpp
+++ b/external/llvm-project/lldb/source/Symbol/SymbolFileOnDemand.cpp
@@ -305,13 +305,14 @@ void SymbolFileOnDemand::Dump(lldb_private::Stream &s) {
   return m_sym_file_impl->Dump(s);
 }
 
-void SymbolFileOnDemand::DumpClangAST(lldb_private::Stream &s) {
+void SymbolFileOnDemand::DumpClangAST(lldb_private::Stream &s,
+                                      llvm::StringRef filter) {
   if (!m_debug_info_enabled) {
     LLDB_LOG(GetLog(), "[{0}] {1} is skipped", GetSymbolFileName(),
              __FUNCTION__);
     return;
   }
-  return m_sym_file_impl->DumpClangAST(s);
+  return m_sym_file_impl->DumpClangAST(s, filter);
 }
 
 void SymbolFileOnDemand::FindGlobalVariables(const RegularExpression &regex,
diff --git a/external/llvm-project/lldb/source/Symbol/UnwindPlan.cpp b/external/llvm-project/lldb/source/Symbol/UnwindPlan.cpp
index b1a96b5e2684..e9ac6b6cde29 100644
--- a/external/llvm-project/lldb/source/Symbol/UnwindPlan.cpp
+++ b/external/llvm-project/lldb/source/Symbol/UnwindPlan.cpp
@@ -87,8 +87,10 @@ static void DumpDWARFExpr(Stream &s, llvm::ArrayRef<uint8_t> expr, Thread *threa
   if (auto order_and_width = GetByteOrderAndAddrSize(thread)) {
     llvm::DataExtractor data(expr, order_and_width->first == eByteOrderLittle,
                              order_and_width->second);
-    llvm::DWARFExpression(data, order_and_width->second, llvm::dwarf::DWARF32)
-        .print(s.AsRawOstream(), llvm::DIDumpOptions(), nullptr);
+    llvm::DWARFExpression E(data, order_and_width->second,
+                            llvm::dwarf::DWARF32);
+    llvm::DWARFExpressionPrinter::print(&E, s.AsRawOstream(),
+                                        llvm::DIDumpOptions(), nullptr);
   } else
     s.PutCString("dwarf-expr");
 }
diff --git a/external/llvm-project/lldb/source/Symbol/Variable.cpp b/external/llvm-project/lldb/source/Symbol/Variable.cpp
index 8244725aba54..af32e0e958e5 100644
--- a/external/llvm-project/lldb/source/Symbol/Variable.cpp
+++ b/external/llvm-project/lldb/source/Symbol/Variable.cpp
@@ -290,28 +290,9 @@ bool Variable::IsInScope(StackFrame *frame) {
       // this variable was defined in is currently
       Block *deepest_frame_block =
           frame->GetSymbolContext(eSymbolContextBlock).block;
-      if (deepest_frame_block) {
-        SymbolContext variable_sc;
-        CalculateSymbolContext(&variable_sc);
-
-        // Check for static or global variable defined at the compile unit
-        // level that wasn't defined in a block
-        if (variable_sc.block == nullptr)
-          return true;
-
-        // Check if the variable is valid in the current block
-        if (variable_sc.block != deepest_frame_block &&
-            !variable_sc.block->Contains(deepest_frame_block))
-          return false;
-
-        // If no scope range is specified then it means that the scope is the
-        // same as the scope of the enclosing lexical block.
-        if (m_scope_range.IsEmpty())
-          return true;
-
-        addr_t file_address = frame->GetFrameCodeAddress().GetFileAddress();
-        return m_scope_range.FindEntryThatContains(file_address) != nullptr;
-      }
+      Address frame_addr = frame->GetFrameCodeAddress();
+      if (deepest_frame_block)
+        return IsInScope(*deepest_frame_block, frame_addr);
     }
     break;
 
@@ -321,6 +302,27 @@ bool Variable::IsInScope(StackFrame *frame) {
   return false;
 }
 
+bool Variable::IsInScope(const Block &block, const Address &addr) {
+  SymbolContext variable_sc;
+  CalculateSymbolContext(&variable_sc);
+
+  // Check for static or global variable defined at the compile unit
+  // level that wasn't defined in a block
+  if (variable_sc.block == nullptr)
+    return true;
+
+  // Check if the variable is valid in the current block
+  if (variable_sc.block != &block && !variable_sc.block->Contains(&block))
+    return false;
+
+  // If no scope range is specified then it means that the scope is the
+  // same as the scope of the enclosing lexical block.
+  if (m_scope_range.IsEmpty())
+    return true;
+
+  return m_scope_range.FindEntryThatContains(addr.GetFileAddress()) != nullptr;
+}
+
 Status Variable::GetValuesForVariableExpressionPath(
     llvm::StringRef variable_expr_path, ExecutionContextScope *scope,
     GetVariableCallback callback, void *baton, VariableList &variable_list,
diff --git a/external/llvm-project/lldb/source/Target/CMakeLists.txt b/external/llvm-project/lldb/source/Target/CMakeLists.txt
index d3545913a690..b7788e80eeca 100644
--- a/external/llvm-project/lldb/source/Target/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Target/CMakeLists.txt
@@ -82,6 +82,11 @@ add_lldb_library(lldbTarget
   UnwindLLDB.cpp
   VerboseTrapFrameRecognizer.cpp
 
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Target
+  LINK_COMPONENTS
+    Support
+    MC
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -92,10 +97,6 @@ add_lldb_library(lldbTarget
     lldbUtility
     lldbValueObject
     lldbPluginProcessUtility
-
-  LINK_COMPONENTS
-    Support
-    MC
   )
 
 add_dependencies(lldbTarget
diff --git a/external/llvm-project/lldb/source/Target/Process.cpp b/external/llvm-project/lldb/source/Target/Process.cpp
index c377feec86c1..61a3d05bc374 100644
--- a/external/llvm-project/lldb/source/Target/Process.cpp
+++ b/external/llvm-project/lldb/source/Target/Process.cpp
@@ -80,16 +80,6 @@ using namespace lldb;
 using namespace lldb_private;
 using namespace std::chrono;
 
-// Comment out line below to disable memory caching, overriding the process
-// setting target.process.disable-memory-cache
-#define ENABLE_MEMORY_CACHING
-
-#ifdef ENABLE_MEMORY_CACHING
-#define DISABLE_MEM_CACHE_DEFAULT false
-#else
-#define DISABLE_MEM_CACHE_DEFAULT true
-#endif
-
 class ProcessOptionValueProperties
     : public Cloneable<ProcessOptionValueProperties, OptionValueProperties> {
 public:
@@ -2291,25 +2281,18 @@ size_t Process::WriteMemoryPrivate(addr_t addr, const void *buf, size_t size,
   return bytes_written;
 }
 
-#define USE_ALLOCATE_MEMORY_CACHE 1
 size_t Process::WriteMemory(addr_t addr, const void *buf, size_t size,
                             Status &error) {
   if (ABISP abi_sp = GetABI())
     addr = abi_sp->FixAnyAddress(addr);
 
-#if defined(ENABLE_MEMORY_CACHING)
   m_memory_cache.Flush(addr, size);
-#endif
 
   if (buf == nullptr || size == 0)
     return 0;
 
-#if defined(USE_ALLOCATE_MEMORY_CACHE)
   if (TrackMemoryCacheChanges() || !m_allocated_memory_cache.IsInCache(addr))
     m_mod_id.BumpMemoryID();
-#else
-  m_mod_id.BumpMemoryID();
-#endif
 
   // We need to write any data that would go where any current software traps
   // (enabled software breakpoints) any software traps (breakpoints) that we
@@ -2446,20 +2429,7 @@ addr_t Process::AllocateMemory(size_t size, uint32_t permissions,
     return LLDB_INVALID_ADDRESS;
   }
 
-#if defined(USE_ALLOCATE_MEMORY_CACHE)
   return m_allocated_memory_cache.AllocateMemory(size, permissions, error);
-#else
-  addr_t allocated_addr = DoAllocateMemory(size, permissions, error);
-  Log *log = GetLog(LLDBLog::Process);
-  LLDB_LOGF(log,
-            "Process::AllocateMemory(size=%" PRIu64
-            ", permissions=%s) => 0x%16.16" PRIx64
-            " (m_stop_id = %u m_memory_id = %u)",
-            (uint64_t)size, GetPermissionsAsCString(permissions),
-            (uint64_t)allocated_addr, m_mod_id.GetStopID(),
-            m_mod_id.GetMemoryID());
-  return allocated_addr;
-#endif
 }
 
 addr_t Process::CallocateMemory(size_t size, uint32_t permissions,
@@ -2512,21 +2482,10 @@ void Process::SetCanRunCode(bool can_run_code) {
 
 Status Process::DeallocateMemory(addr_t ptr) {
   Status error;
-#if defined(USE_ALLOCATE_MEMORY_CACHE)
   if (!m_allocated_memory_cache.DeallocateMemory(ptr)) {
     error = Status::FromErrorStringWithFormat(
         "deallocation of memory at 0x%" PRIx64 " failed.", (uint64_t)ptr);
   }
-#else
-  error = DoDeallocateMemory(ptr);
-
-  Log *log = GetLog(LLDBLog::Process);
-  LLDB_LOGF(log,
-            "Process::DeallocateMemory(addr=0x%16.16" PRIx64
-            ") => err = %s (m_stop_id = %u, m_memory_id = %u)",
-            ptr, error.AsCString("SUCCESS"), m_mod_id.GetStopID(),
-            m_mod_id.GetMemoryID());
-#endif
   return error;
 }
 
@@ -5879,7 +5838,7 @@ void Process::ClearPreResumeActions() { m_pre_resume_actions.clear(); }
 void Process::ClearPreResumeAction(PreResumeActionCallback callback, void *baton)
 {
     PreResumeCallbackAndBaton element(callback, baton);
-    auto found_iter = std::find(m_pre_resume_actions.begin(), m_pre_resume_actions.end(), element);
+    auto found_iter = llvm::find(m_pre_resume_actions, element);
     if (found_iter != m_pre_resume_actions.end())
     {
         m_pre_resume_actions.erase(found_iter);
diff --git a/external/llvm-project/lldb/source/Target/RegisterContextUnwind.cpp b/external/llvm-project/lldb/source/Target/RegisterContextUnwind.cpp
index 32ed8e038441..880300d0637f 100644
--- a/external/llvm-project/lldb/source/Target/RegisterContextUnwind.cpp
+++ b/external/llvm-project/lldb/source/Target/RegisterContextUnwind.cpp
@@ -1233,247 +1233,201 @@ bool RegisterContextUnwind::IsTrapHandlerSymbol(
   return false;
 }
 
-// Answer the question: Where did THIS frame save the CALLER frame ("previous"
-// frame)'s register value?
-
-enum UnwindLLDB::RegisterSearchResult
-RegisterContextUnwind::SavedLocationForRegister(
-    uint32_t lldb_regnum,
-    lldb_private::UnwindLLDB::ConcreteRegisterLocation &regloc) {
+// Search this stack frame's UnwindPlans for the AbstractRegisterLocation
+// for this register.
+//
+// \param[in] lldb_regnum
+//     The register number (in the eRegisterKindLLDB register numbering)
+//     we are searching for.
+//
+// \param[out] kind
+//     Set to the RegisterKind of the UnwindPlan which is the basis for
+//     the returned AbstractRegisterLocation; if the location is in terms
+//     of another register number, this Kind is needed to interpret it
+//     correctly.
+//
+// \return
+//     An empty optional indicaTes that there was an error in processing
+//     the request.
+//
+//     If there is no unwind rule for a volatile (caller-preserved) register,
+//     the returned AbstractRegisterLocation will be IsUndefined,
+//     indicating that we should stop searching.
+//
+//     If there is no unwind rule for a non-volatile (callee-preserved)
+//     register, the returned AbstractRegisterLocation will be IsSame.
+//     In frame 0, IsSame means get the value from the live register context.
+//     Else it means to continue descending down the stack to more-live frames
+//     looking for a location/value.
+//
+//     If an AbstractRegisterLocation is found in an UnwindPlan, that will
+//     be returned, with no consideration of the current ABI rules for
+//     registers.  Functions using an alternate ABI calling convention
+//     will work as long as the UnwindPlans are exhaustive about what
+//     registers are volatile/non-volatile.
+std::optional<UnwindPlan::Row::AbstractRegisterLocation>
+RegisterContextUnwind::GetAbstractRegisterLocation(uint32_t lldb_regnum,
+                                                   lldb::RegisterKind &kind) {
   RegisterNumber regnum(m_thread, eRegisterKindLLDB, lldb_regnum);
   Log *log = GetLog(LLDBLog::Unwind);
 
-  // Have we already found this register location?
-  if (!m_registers.empty()) {
-    std::map<uint32_t,
-             lldb_private::UnwindLLDB::ConcreteRegisterLocation>::const_iterator
-        iterator;
-    iterator = m_registers.find(regnum.GetAsKind(eRegisterKindLLDB));
-    if (iterator != m_registers.end()) {
-      regloc = iterator->second;
-      UnwindLogMsg("supplying caller's saved %s (%d)'s location, cached",
-                   regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
-      return UnwindLLDB::RegisterSearchResult::eRegisterFound;
-    }
-  }
-
-  // Look through the available UnwindPlans for the register location.
-
+  kind = eRegisterKindLLDB;
   UnwindPlan::Row::AbstractRegisterLocation unwindplan_regloc;
-  bool have_unwindplan_regloc = false;
-  RegisterKind unwindplan_registerkind = kNumRegisterKinds;
 
+  // First, try to find a register location via the FastUnwindPlan
   if (m_fast_unwind_plan_sp) {
     const UnwindPlan::Row *active_row =
         m_fast_unwind_plan_sp->GetRowForFunctionOffset(m_current_offset);
-    unwindplan_registerkind = m_fast_unwind_plan_sp->GetRegisterKind();
-    if (regnum.GetAsKind(unwindplan_registerkind) == LLDB_INVALID_REGNUM) {
+    if (regnum.GetAsKind(kind) == LLDB_INVALID_REGNUM) {
       UnwindLogMsg("could not convert lldb regnum %s (%d) into %d RegisterKind "
                    "reg numbering scheme",
                    regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB),
-                   (int)unwindplan_registerkind);
-      return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
+                   (int)kind);
+      return {};
     }
-    // The architecture default unwind plan marks unknown registers as
-    // Undefined so that we don't forward them up the stack when a
-    // jitted stack frame may have overwritten them.  But when the
-    // arch default unwind plan is used as the Fast Unwind Plan, we
-    // need to recognize this & switch over to the Full Unwind Plan
-    // to see what unwind rule that (more knoweldgeable, probably)
-    // UnwindPlan has.  If the full UnwindPlan says the register
-    // location is Undefined, then it really is.
-    if (active_row->GetRegisterInfo(regnum.GetAsKind(unwindplan_registerkind),
+    kind = m_fast_unwind_plan_sp->GetRegisterKind();
+    // The Fast UnwindPlan typically only provides fp & pc as we move up
+    // the stack, without requiring additional parsing or memory reads.
+    // It may mark all other registers as IsUndefined() because, indicating
+    // that it doesn't know if they were spilled to stack or not.
+    // If this case, for an IsUndefined register, we should continue on
+    // to the Full UnwindPlan which may have more accurate information
+    // about register locations of all registers.
+    if (active_row &&
+        active_row->GetRegisterInfo(regnum.GetAsKind(kind),
                                     unwindplan_regloc) &&
         !unwindplan_regloc.IsUndefined()) {
       UnwindLogMsg(
           "supplying caller's saved %s (%d)'s location using FastUnwindPlan",
           regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
-      have_unwindplan_regloc = true;
+      return unwindplan_regloc;
     }
   }
 
-  if (!have_unwindplan_regloc) {
-    // m_full_unwind_plan_sp being NULL means that we haven't tried to find a
-    // full UnwindPlan yet
-    bool got_new_full_unwindplan = false;
-    if (!m_full_unwind_plan_sp) {
-      m_full_unwind_plan_sp = GetFullUnwindPlanForFrame();
-      got_new_full_unwindplan = true;
+  // Second, try to find a register location via the FullUnwindPlan.
+  bool got_new_full_unwindplan = false;
+  if (!m_full_unwind_plan_sp) {
+    m_full_unwind_plan_sp = GetFullUnwindPlanForFrame();
+    got_new_full_unwindplan = true;
+  }
+  if (m_full_unwind_plan_sp) {
+    RegisterNumber pc_regnum(m_thread, eRegisterKindGeneric,
+                             LLDB_REGNUM_GENERIC_PC);
+
+    const UnwindPlan::Row *active_row =
+        m_full_unwind_plan_sp->GetRowForFunctionOffset(
+            m_current_offset_backed_up_one);
+    kind = m_full_unwind_plan_sp->GetRegisterKind();
+
+    if (got_new_full_unwindplan && active_row && log) {
+      StreamString active_row_strm;
+      ExecutionContext exe_ctx(m_thread.shared_from_this());
+      active_row->Dump(active_row_strm, m_full_unwind_plan_sp.get(), &m_thread,
+                       m_start_pc.GetLoadAddress(exe_ctx.GetTargetPtr()));
+      UnwindLogMsg("Using full unwind plan '%s'",
+                   m_full_unwind_plan_sp->GetSourceName().AsCString());
+      UnwindLogMsg("active row: %s", active_row_strm.GetData());
     }
 
-    if (m_full_unwind_plan_sp) {
-      RegisterNumber pc_regnum(m_thread, eRegisterKindGeneric,
-                               LLDB_REGNUM_GENERIC_PC);
+    if (regnum.GetAsKind(kind) == LLDB_INVALID_REGNUM) {
+      if (kind == eRegisterKindGeneric)
+        UnwindLogMsg("could not convert lldb regnum %s (%d) into "
+                     "eRegisterKindGeneric reg numbering scheme",
+                     regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
+      else
+        UnwindLogMsg("could not convert lldb regnum %s (%d) into %d "
+                     "RegisterKind reg numbering scheme",
+                     regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB),
+                     (int)kind);
+      return {};
+    }
 
-      const UnwindPlan::Row *active_row =
-          m_full_unwind_plan_sp->GetRowForFunctionOffset(
-              m_current_offset_backed_up_one);
-      unwindplan_registerkind = m_full_unwind_plan_sp->GetRegisterKind();
+    if (regnum.IsValid() && active_row &&
+        active_row->GetRegisterInfo(regnum.GetAsKind(kind),
+                                    unwindplan_regloc)) {
+      UnwindLogMsg(
+          "supplying caller's saved %s (%d)'s location using %s UnwindPlan",
+          regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB),
+          m_full_unwind_plan_sp->GetSourceName().GetCString());
+      return unwindplan_regloc;
+    }
 
-      if (got_new_full_unwindplan && active_row && log) {
-        StreamString active_row_strm;
-        ExecutionContext exe_ctx(m_thread.shared_from_this());
-        active_row->Dump(active_row_strm, m_full_unwind_plan_sp.get(),
-                         &m_thread,
-                         m_start_pc.GetLoadAddress(exe_ctx.GetTargetPtr()));
-        UnwindLogMsg("Using full unwind plan '%s'",
-                     m_full_unwind_plan_sp->GetSourceName().AsCString());
-        UnwindLogMsg("active row: %s", active_row_strm.GetData());
-      }
-      RegisterNumber return_address_reg;
-
-      // If we're fetching the saved pc and this UnwindPlan defines a
-      // ReturnAddress register (e.g. lr on arm), look for the return address
-      // register number in the UnwindPlan's row.
-      if (pc_regnum.IsValid() && pc_regnum == regnum &&
-          m_full_unwind_plan_sp->GetReturnAddressRegister() !=
-              LLDB_INVALID_REGNUM) {
-        // If this is a trap handler frame, we should have access to
-        // the complete register context when the interrupt/async
-        // signal was received, we should fetch the actual saved $pc
-        // value instead of the Return Address register.
-        // If $pc is not available, fall back to the RA reg.
-        UnwindPlan::Row::AbstractRegisterLocation scratch;
-        if (m_frame_type == eTrapHandlerFrame && active_row &&
-            active_row->GetRegisterInfo(
-                pc_regnum.GetAsKind(unwindplan_registerkind), scratch)) {
-          UnwindLogMsg("Providing pc register instead of rewriting to "
-                       "RA reg because this is a trap handler and there is "
-                       "a location for the saved pc register value.");
-        } else {
-          return_address_reg.init(
-              m_thread, m_full_unwind_plan_sp->GetRegisterKind(),
-              m_full_unwind_plan_sp->GetReturnAddressRegister());
-          regnum = return_address_reg;
-          UnwindLogMsg("requested caller's saved PC but this UnwindPlan uses a "
-                       "RA reg; getting %s (%d) instead",
-                       return_address_reg.GetName(),
-                       return_address_reg.GetAsKind(eRegisterKindLLDB));
-        }
+    // When asking for the caller's pc, and did not find a register
+    // location for PC above in the UnwindPlan.  Check if we have a
+    // Return Address register on this target.
+    //
+    // On a Return Address Register architecture like arm/mips/riscv,
+    // the caller's pc is in the RA register, and will be spilled to
+    // stack before any other function is called.  If no function
+    // has been called yet, the return address may still be in the
+    // live RA reg.
+    //
+    // There's a lot of variety of what we might see in an UnwindPlan.
+    // We may have
+    //   ra=IsSame {unncessary}
+    //   ra=StackAddr {caller's return addr spilled to stack}
+    // or no reg location for pc or ra at all, in a frameless function -
+    // the caller's return address is in live ra reg.
+    //
+    // If a function has been interrupted in a non-call way --
+    // async signal/sigtramp, or a hardware exception / interrupt / fault --
+    // then the "pc" and "ra" are two distinct values, and must be
+    // handled separately.  The "pc" is the pc value at the point
+    // the function was interrupted.  The "ra" is the return address
+    // register value at that point.
+    // The UnwindPlan for the sigtramp/trap handler will normally have
+    // register loations for both pc and lr, and so we'll have already
+    // fetched them above.
+    if (pc_regnum.IsValid() && pc_regnum == regnum) {
+      uint32_t return_address_regnum = LLDB_INVALID_REGNUM;
+
+      // Get the return address register number from the UnwindPlan
+      // or the register set definition.
+      if (m_full_unwind_plan_sp->GetReturnAddressRegister() !=
+          LLDB_INVALID_REGNUM) {
+        return_address_regnum =
+            m_full_unwind_plan_sp->GetReturnAddressRegister();
       } else {
-        if (regnum.GetAsKind(unwindplan_registerkind) == LLDB_INVALID_REGNUM) {
-          if (unwindplan_registerkind == eRegisterKindGeneric) {
-            UnwindLogMsg("could not convert lldb regnum %s (%d) into "
-                         "eRegisterKindGeneric reg numbering scheme",
-                         regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
-          } else {
-            UnwindLogMsg("could not convert lldb regnum %s (%d) into %d "
-                         "RegisterKind reg numbering scheme",
-                         regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB),
-                         (int)unwindplan_registerkind);
-          }
-          return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
-        }
-      }
-
-      // Check if the active_row has a register location listed.
-      if (regnum.IsValid() && active_row &&
-          active_row->GetRegisterInfo(regnum.GetAsKind(unwindplan_registerkind),
-                                      unwindplan_regloc)) {
-        have_unwindplan_regloc = true;
-        UnwindLogMsg(
-            "supplying caller's saved %s (%d)'s location using %s UnwindPlan",
-            regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB),
-            m_full_unwind_plan_sp->GetSourceName().GetCString());
+        RegisterNumber arch_default_ra_regnum(m_thread, eRegisterKindGeneric,
+                                              LLDB_REGNUM_GENERIC_RA);
+        return_address_regnum = arch_default_ra_regnum.GetAsKind(kind);
       }
 
-      // This is frame 0 and we're retrieving the PC and it's saved in a Return
-      // Address register and it hasn't been saved anywhere yet -- that is,
-      // it's still live in the actual register. Handle this specially.
-      if (!have_unwindplan_regloc && return_address_reg.IsValid() &&
-          return_address_reg.GetAsKind(eRegisterKindLLDB) !=
-              LLDB_INVALID_REGNUM) {
-        if (IsFrameZero()) {
-          lldb_private::UnwindLLDB::ConcreteRegisterLocation new_regloc;
-          new_regloc.type = UnwindLLDB::ConcreteRegisterLocation::
-              eRegisterInLiveRegisterContext;
-          new_regloc.location.register_number =
-              return_address_reg.GetAsKind(eRegisterKindLLDB);
-          m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = new_regloc;
-          regloc = new_regloc;
-          UnwindLogMsg("supplying caller's register %s (%d) from the live "
-                       "RegisterContext at frame 0, saved in %d",
+      // This system is using a return address register.
+      if (return_address_regnum != LLDB_INVALID_REGNUM) {
+        RegisterNumber return_address_reg;
+        return_address_reg.init(m_thread,
+                                m_full_unwind_plan_sp->GetRegisterKind(),
+                                return_address_regnum);
+        UnwindLogMsg("requested caller's saved PC but this UnwindPlan uses a "
+                     "RA reg; getting %s (%d) instead",
+                     return_address_reg.GetName(),
+                     return_address_reg.GetAsKind(eRegisterKindLLDB));
+
+        // Do we have a location for the ra register?
+        if (active_row &&
+            active_row->GetRegisterInfo(return_address_reg.GetAsKind(kind),
+                                        unwindplan_regloc)) {
+          UnwindLogMsg("supplying caller's saved %s (%d)'s location using "
+                       "%s UnwindPlan",
                        return_address_reg.GetName(),
                        return_address_reg.GetAsKind(eRegisterKindLLDB),
-                       return_address_reg.GetAsKind(eRegisterKindLLDB));
-          return UnwindLLDB::RegisterSearchResult::eRegisterFound;
-        } else if (BehavesLikeZerothFrame()) {
-          // This function was interrupted asynchronously -- it faulted,
-          // an async interrupt, a timer fired, a debugger expression etc.
-          // The caller's pc is in the Return Address register, but the
-          // UnwindPlan for this function may have no location rule for
-          // the RA reg.
-          // This means that the caller's return address is in the RA reg
-          // when the function was interrupted--descend down one stack frame
-          // to retrieve it from the trap handler's saved context.
-          unwindplan_regloc.SetSame();
-          have_unwindplan_regloc = true;
-        }
-      }
-
-      // If this architecture stores the return address in a register (it
-      // defines a Return Address register) and we're on a non-zero stack frame
-      // and the Full UnwindPlan says that the pc is stored in the
-      // RA registers (e.g. lr on arm), then we know that the full unwindplan is
-      // not trustworthy -- this
-      // is an impossible situation and the instruction emulation code has
-      // likely been misled. If this stack frame meets those criteria, we need
-      // to throw away the Full UnwindPlan that the instruction emulation came
-      // up with and fall back to the architecture's Default UnwindPlan so the
-      // stack walk can get past this point.
-
-      // Special note:  If the Full UnwindPlan was generated from the compiler,
-      // don't second-guess it when we're at a call site location.
-
-      // arch_default_ra_regnum is the return address register # in the Full
-      // UnwindPlan register numbering
-      RegisterNumber arch_default_ra_regnum(m_thread, eRegisterKindGeneric,
-                                            LLDB_REGNUM_GENERIC_RA);
-
-      if (arch_default_ra_regnum.GetAsKind(unwindplan_registerkind) !=
-              LLDB_INVALID_REGNUM &&
-          pc_regnum == regnum && unwindplan_regloc.IsInOtherRegister() &&
-          unwindplan_regloc.GetRegisterNumber() ==
-              arch_default_ra_regnum.GetAsKind(unwindplan_registerkind) &&
-          m_full_unwind_plan_sp->GetSourcedFromCompiler() != eLazyBoolYes &&
-          !m_all_registers_available) {
-        UnwindLogMsg("%s UnwindPlan tried to restore the pc from the link "
-                     "register but this is a non-zero frame",
-                     m_full_unwind_plan_sp->GetSourceName().GetCString());
-
-        // Throw away the full unwindplan; install the arch default unwindplan
-        if (ForceSwitchToFallbackUnwindPlan()) {
-          // Update for the possibly new unwind plan
-          unwindplan_registerkind = m_full_unwind_plan_sp->GetRegisterKind();
-          const UnwindPlan::Row *active_row =
-              m_full_unwind_plan_sp->GetRowForFunctionOffset(m_current_offset);
-
-          // Sanity check: Verify that we can fetch a pc value and CFA value
-          // with this unwind plan
-
-          RegisterNumber arch_default_pc_reg(m_thread, eRegisterKindGeneric,
-                                             LLDB_REGNUM_GENERIC_PC);
-          bool can_fetch_pc_value = false;
-          bool can_fetch_cfa = false;
-          addr_t cfa_value;
-          if (active_row) {
-            if (arch_default_pc_reg.GetAsKind(unwindplan_registerkind) !=
-                    LLDB_INVALID_REGNUM &&
-                active_row->GetRegisterInfo(
-                    arch_default_pc_reg.GetAsKind(unwindplan_registerkind),
-                    unwindplan_regloc)) {
-              can_fetch_pc_value = true;
-            }
-            if (ReadFrameAddress(unwindplan_registerkind,
-                                 active_row->GetCFAValue(), cfa_value)) {
-              can_fetch_cfa = true;
-            }
-          }
-
-          have_unwindplan_regloc = can_fetch_pc_value && can_fetch_cfa;
+                       m_full_unwind_plan_sp->GetSourceName().GetCString());
+          // If we have "ra=IsSame", rewrite to "ra=InRegister(ra)" because the
+          // calling function thinks it is fetching "pc" and if we return an
+          // IsSame register location, it will try to read pc.
+          if (unwindplan_regloc.IsSame())
+            unwindplan_regloc.SetInRegister(return_address_reg.GetAsKind(kind));
+          return unwindplan_regloc;
         } else {
-          // We were unable to fall back to another unwind plan
-          have_unwindplan_regloc = false;
+          // No unwind rule for the return address reg on frame 0, or an
+          // interrupted function, means that the caller's address is still in
+          // RA reg (0th frame) or the trap handler below this one (sigtramp
+          // etc) has a save location for the RA reg.
+          if (BehavesLikeZerothFrame()) {
+            unwindplan_regloc.SetInRegister(return_address_reg.GetAsKind(kind));
+            return unwindplan_regloc;
+          }
         }
       }
     }
@@ -1481,55 +1435,86 @@ RegisterContextUnwind::SavedLocationForRegister(
 
   ExecutionContext exe_ctx(m_thread.shared_from_this());
   Process *process = exe_ctx.GetProcessPtr();
-  if (!have_unwindplan_regloc) {
-    // If the UnwindPlan failed to give us an unwind location for this
-    // register, we may be able to fall back to some ABI-defined default.  For
-    // example, some ABIs allow to determine the caller's SP via the CFA. Also,
-    // the ABI may set volatile registers to the undefined state.
-    ABI *abi = process ? process->GetABI().get() : nullptr;
-    if (abi) {
-      const RegisterInfo *reg_info =
-          GetRegisterInfoAtIndex(regnum.GetAsKind(eRegisterKindLLDB));
-      if (reg_info &&
-          abi->GetFallbackRegisterLocation(reg_info, unwindplan_regloc)) {
+
+  // Third, try finding a register location via the ABI
+  // FallbackRegisterLocation.
+  //
+  // If the UnwindPlan failed to give us an unwind location for this
+  // register, we may be able to fall back to some ABI-defined default.  For
+  // example, some ABIs allow to determine the caller's SP via the CFA. Also,
+  // the ABI willset volatile registers to the undefined state.
+  ABI *abi = process ? process->GetABI().get() : nullptr;
+  if (abi) {
+    const RegisterInfo *reg_info =
+        GetRegisterInfoAtIndex(regnum.GetAsKind(eRegisterKindLLDB));
+    if (reg_info &&
+        abi->GetFallbackRegisterLocation(reg_info, unwindplan_regloc)) {
+      if (!unwindplan_regloc.IsUndefined())
         UnwindLogMsg(
             "supplying caller's saved %s (%d)'s location using ABI default",
             regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
-        have_unwindplan_regloc = true;
-      }
+      // ABI defined volatile registers with no register location
+      // will be returned as IsUndefined, stopping the search down
+      // the stack.
+      return unwindplan_regloc;
     }
   }
 
-  if (!have_unwindplan_regloc) {
-    if (IsFrameZero()) {
-      // This is frame 0 - we should return the actual live register context
-      // value
-      lldb_private::UnwindLLDB::ConcreteRegisterLocation new_regloc;
-      new_regloc.type =
-          UnwindLLDB::ConcreteRegisterLocation::eRegisterInLiveRegisterContext;
-      new_regloc.location.register_number = regnum.GetAsKind(eRegisterKindLLDB);
-      m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = new_regloc;
-      regloc = new_regloc;
-      UnwindLogMsg("supplying caller's register %s (%d) from the live "
-                   "RegisterContext at frame 0",
+  // We have no AbstractRegisterLocation, and the ABI says this is a
+  // non-volatile / callee-preserved register.  Continue down the stack
+  // or to frame 0 & the live RegisterContext.
+  std::string unwindplan_name;
+  if (m_full_unwind_plan_sp) {
+    unwindplan_name += "via '";
+    unwindplan_name += m_full_unwind_plan_sp->GetSourceName().AsCString();
+    unwindplan_name += "'";
+  }
+  UnwindLogMsg("no save location for %s (%d) %s", regnum.GetName(),
+               regnum.GetAsKind(eRegisterKindLLDB), unwindplan_name.c_str());
+
+  unwindplan_regloc.SetSame();
+  return unwindplan_regloc;
+}
+
+// Answer the question: Where did THIS frame save the CALLER frame ("previous"
+// frame)'s register value?
+
+enum UnwindLLDB::RegisterSearchResult
+RegisterContextUnwind::SavedLocationForRegister(
+    uint32_t lldb_regnum,
+    lldb_private::UnwindLLDB::ConcreteRegisterLocation &regloc) {
+  RegisterNumber regnum(m_thread, eRegisterKindLLDB, lldb_regnum);
+  Log *log = GetLog(LLDBLog::Unwind);
+
+  // Have we already found this register location?
+  if (!m_registers.empty()) {
+    auto iterator = m_registers.find(regnum.GetAsKind(eRegisterKindLLDB));
+    if (iterator != m_registers.end()) {
+      regloc = iterator->second;
+      UnwindLogMsg("supplying caller's saved %s (%d)'s location, cached",
                    regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
       return UnwindLLDB::RegisterSearchResult::eRegisterFound;
-    } else {
-      std::string unwindplan_name;
-      if (m_full_unwind_plan_sp) {
-        unwindplan_name += "via '";
-        unwindplan_name += m_full_unwind_plan_sp->GetSourceName().AsCString();
-        unwindplan_name += "'";
-      }
-      UnwindLogMsg("no save location for %s (%d) %s", regnum.GetName(),
-                   regnum.GetAsKind(eRegisterKindLLDB),
-                   unwindplan_name.c_str());
     }
+  }
+
+  RegisterKind abs_regkind;
+  std::optional<UnwindPlan::Row::AbstractRegisterLocation> abs_regloc =
+      GetAbstractRegisterLocation(lldb_regnum, abs_regkind);
+
+  if (!abs_regloc)
     return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
+
+  if (abs_regloc->IsUndefined()) {
+    UnwindLogMsg(
+        "did not supply reg location for %s (%d) because it is volatile",
+        regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
+    return UnwindLLDB::RegisterSearchResult::eRegisterIsVolatile;
   }
 
-  // unwindplan_regloc has valid contents about where to retrieve the register
-  if (unwindplan_regloc.IsUnspecified()) {
+  ExecutionContext exe_ctx(m_thread.shared_from_this());
+  Process *process = exe_ctx.GetProcessPtr();
+  // abs_regloc has valid contents about where to retrieve the register
+  if (abs_regloc->IsUnspecified()) {
     lldb_private::UnwindLLDB::ConcreteRegisterLocation new_regloc = {};
     new_regloc.type = UnwindLLDB::ConcreteRegisterLocation::eRegisterNotSaved;
     m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = new_regloc;
@@ -1538,15 +1523,23 @@ RegisterContextUnwind::SavedLocationForRegister(
     return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
   }
 
-  if (unwindplan_regloc.IsUndefined()) {
-    UnwindLogMsg(
-        "did not supply reg location for %s (%d) because it is volatile",
-        regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
-    return UnwindLLDB::RegisterSearchResult::eRegisterIsVolatile;
-  }
-
-  if (unwindplan_regloc.IsSame()) {
-    if (!m_all_registers_available &&
+  if (abs_regloc->IsSame()) {
+    if (IsFrameZero()) {
+      regloc.type =
+          UnwindLLDB::ConcreteRegisterLocation::eRegisterInLiveRegisterContext;
+      regloc.location.register_number = regnum.GetAsKind(eRegisterKindLLDB);
+      m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = regloc;
+      UnwindLogMsg("supplying caller's register %s (%d) from the live "
+                   "RegisterContext at frame 0",
+                   regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
+      return UnwindLLDB::RegisterSearchResult::eRegisterFound;
+    }
+    // PC/RA reg don't follow the usual "callee-saved aka non-volatile" versus
+    // "caller saved aka volatile" system.  A stack frame can provide its caller
+    // return address, but if we don't find a rule for pc/RA mid-stack, we
+    // never want to iterate further down the stack looking for it.
+    // Defensively prevent iterating down the stack for these two.
+    if (!BehavesLikeZerothFrame() &&
         (regnum.GetAsKind(eRegisterKindGeneric) == LLDB_REGNUM_GENERIC_PC ||
          regnum.GetAsKind(eRegisterKindGeneric) == LLDB_REGNUM_GENERIC_RA)) {
       UnwindLogMsg("register %s (%d) is marked as 'IsSame' - it is a pc or "
@@ -1554,20 +1547,19 @@ RegisterContextUnwind::SavedLocationForRegister(
                    "registers available -- treat as if we have no information",
                    regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
       return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
-    } else {
-      regloc.type = UnwindLLDB::ConcreteRegisterLocation::eRegisterInRegister;
-      regloc.location.register_number = regnum.GetAsKind(eRegisterKindLLDB);
-      m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = regloc;
-      UnwindLogMsg(
-          "supplying caller's register %s (%d), saved in register %s (%d)",
-          regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB),
-          regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
-      return UnwindLLDB::RegisterSearchResult::eRegisterFound;
     }
+
+    regloc.type = UnwindLLDB::ConcreteRegisterLocation::eRegisterInRegister;
+    regloc.location.register_number = regnum.GetAsKind(eRegisterKindLLDB);
+    m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = regloc;
+    UnwindLogMsg(
+        "supplying caller's register %s (%d) value is unmodified in this frame",
+        regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
+    return UnwindLLDB::RegisterSearchResult::eRegisterFound;
   }
 
-  if (unwindplan_regloc.IsCFAPlusOffset()) {
-    int offset = unwindplan_regloc.GetOffset();
+  if (abs_regloc->IsCFAPlusOffset()) {
+    int offset = abs_regloc->GetOffset();
     regloc.type = UnwindLLDB::ConcreteRegisterLocation::eRegisterValueInferred;
     regloc.location.inferred_value = m_cfa + offset;
     m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = regloc;
@@ -1578,8 +1570,8 @@ RegisterContextUnwind::SavedLocationForRegister(
     return UnwindLLDB::RegisterSearchResult::eRegisterFound;
   }
 
-  if (unwindplan_regloc.IsAtCFAPlusOffset()) {
-    int offset = unwindplan_regloc.GetOffset();
+  if (abs_regloc->IsAtCFAPlusOffset()) {
+    int offset = abs_regloc->GetOffset();
     regloc.type =
         UnwindLLDB::ConcreteRegisterLocation::eRegisterSavedAtMemoryLocation;
     regloc.location.target_memory_location = m_cfa + offset;
@@ -1591,11 +1583,11 @@ RegisterContextUnwind::SavedLocationForRegister(
     return UnwindLLDB::RegisterSearchResult::eRegisterFound;
   }
 
-  if (unwindplan_regloc.IsAFAPlusOffset()) {
+  if (abs_regloc->IsAFAPlusOffset()) {
     if (m_afa == LLDB_INVALID_ADDRESS)
         return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
 
-    int offset = unwindplan_regloc.GetOffset();
+    int offset = abs_regloc->GetOffset();
     regloc.type = UnwindLLDB::ConcreteRegisterLocation::eRegisterValueInferred;
     regloc.location.inferred_value = m_afa + offset;
     m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = regloc;
@@ -1606,11 +1598,11 @@ RegisterContextUnwind::SavedLocationForRegister(
     return UnwindLLDB::RegisterSearchResult::eRegisterFound;
   }
 
-  if (unwindplan_regloc.IsAtAFAPlusOffset()) {
+  if (abs_regloc->IsAtAFAPlusOffset()) {
     if (m_afa == LLDB_INVALID_ADDRESS)
         return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
 
-    int offset = unwindplan_regloc.GetOffset();
+    int offset = abs_regloc->GetOffset();
     regloc.type =
         UnwindLLDB::ConcreteRegisterLocation::eRegisterSavedAtMemoryLocation;
     regloc.location.target_memory_location = m_afa + offset;
@@ -1622,10 +1614,9 @@ RegisterContextUnwind::SavedLocationForRegister(
     return UnwindLLDB::RegisterSearchResult::eRegisterFound;
   }
 
-  if (unwindplan_regloc.IsInOtherRegister()) {
-    uint32_t unwindplan_regnum = unwindplan_regloc.GetRegisterNumber();
-    RegisterNumber row_regnum(m_thread, unwindplan_registerkind,
-                              unwindplan_regnum);
+  if (abs_regloc->IsInOtherRegister()) {
+    RegisterNumber row_regnum(m_thread, abs_regkind,
+                              abs_regloc->GetRegisterNumber());
     if (row_regnum.GetAsKind(eRegisterKindLLDB) == LLDB_INVALID_REGNUM) {
       UnwindLogMsg("could not supply caller's %s (%d) location - was saved in "
                    "another reg but couldn't convert that regnum",
@@ -1642,16 +1633,14 @@ RegisterContextUnwind::SavedLocationForRegister(
     return UnwindLLDB::RegisterSearchResult::eRegisterFound;
   }
 
-  if (unwindplan_regloc.IsDWARFExpression() ||
-      unwindplan_regloc.IsAtDWARFExpression()) {
-    DataExtractor dwarfdata(unwindplan_regloc.GetDWARFExpressionBytes(),
-                            unwindplan_regloc.GetDWARFExpressionLength(),
+  if (abs_regloc->IsDWARFExpression() || abs_regloc->IsAtDWARFExpression()) {
+    DataExtractor dwarfdata(abs_regloc->GetDWARFExpressionBytes(),
+                            abs_regloc->GetDWARFExpressionLength(),
                             process->GetByteOrder(),
                             process->GetAddressByteSize());
     ModuleSP opcode_ctx;
     DWARFExpressionList dwarfexpr(opcode_ctx, dwarfdata, nullptr);
-    dwarfexpr.GetMutableExpressionAtAddress()->SetRegisterKind(
-        unwindplan_registerkind);
+    dwarfexpr.GetMutableExpressionAtAddress()->SetRegisterKind(abs_regkind);
     Value cfa_val = Scalar(m_cfa);
     cfa_val.SetValueType(Value::ValueType::LoadAddress);
     llvm::Expected<Value> result =
@@ -1662,7 +1651,7 @@ RegisterContextUnwind::SavedLocationForRegister(
     } else {
       addr_t val;
       val = result->GetScalar().ULongLong();
-      if (unwindplan_regloc.IsDWARFExpression()) {
+      if (abs_regloc->IsDWARFExpression()) {
         regloc.type =
             UnwindLLDB::ConcreteRegisterLocation::eRegisterValueInferred;
         regloc.location.inferred_value = val;
@@ -1688,9 +1677,9 @@ RegisterContextUnwind::SavedLocationForRegister(
     return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
   }
 
-  if (unwindplan_regloc.IsConstant()) {
+  if (abs_regloc->IsConstant()) {
     regloc.type = UnwindLLDB::ConcreteRegisterLocation::eRegisterValueInferred;
-    regloc.location.inferred_value = unwindplan_regloc.GetConstant();
+    regloc.location.inferred_value = abs_regloc->GetConstant();
     m_registers[regnum.GetAsKind(eRegisterKindLLDB)] = regloc;
     UnwindLogMsg("supplying caller's register %s (%d) via constant value",
                  regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
diff --git a/external/llvm-project/lldb/source/Target/RegisterNumber.cpp b/external/llvm-project/lldb/source/Target/RegisterNumber.cpp
index e5610bf58854..56dda8d8be8b 100644
--- a/external/llvm-project/lldb/source/Target/RegisterNumber.cpp
+++ b/external/llvm-project/lldb/source/Target/RegisterNumber.cpp
@@ -47,6 +47,7 @@ const RegisterNumber &RegisterNumber::operator=(const RegisterNumber &rhs) {
   m_reg_ctx_sp = rhs.m_reg_ctx_sp;
   m_regnum = rhs.m_regnum;
   m_kind = rhs.m_kind;
+  m_kind_regnum_map.clear();
   for (auto it : rhs.m_kind_regnum_map)
     m_kind_regnum_map[it.first] = it.second;
   m_name = rhs.m_name;
diff --git a/external/llvm-project/lldb/source/Target/StackFrame.cpp b/external/llvm-project/lldb/source/Target/StackFrame.cpp
index ab5cd0b27c78..52a54020bc46 100644
--- a/external/llvm-project/lldb/source/Target/StackFrame.cpp
+++ b/external/llvm-project/lldb/source/Target/StackFrame.cpp
@@ -1937,12 +1937,15 @@ void StackFrame::DumpUsingSettingsFormat(Stream *strm, bool show_unique,
   ExecutionContext exe_ctx(shared_from_this());
 
   const FormatEntity::Entry *frame_format = nullptr;
+  FormatEntity::Entry format_entry;
   Target *target = exe_ctx.GetTargetPtr();
   if (target) {
     if (show_unique) {
-      frame_format = target->GetDebugger().GetFrameFormatUnique();
+      format_entry = target->GetDebugger().GetFrameFormatUnique();
+      frame_format = &format_entry;
     } else {
-      frame_format = target->GetDebugger().GetFrameFormat();
+      format_entry = target->GetDebugger().GetFrameFormat();
+      frame_format = &format_entry;
     }
   }
   if (!DumpUsingFormat(*strm, frame_format, frame_marker)) {
diff --git a/external/llvm-project/lldb/source/Target/Statistics.cpp b/external/llvm-project/lldb/source/Target/Statistics.cpp
index 4cfd0629ea27..6ec8f8963baf 100644
--- a/external/llvm-project/lldb/source/Target/Statistics.cpp
+++ b/external/llvm-project/lldb/source/Target/Statistics.cpp
@@ -10,6 +10,7 @@
 
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
+#include "lldb/Core/PluginManager.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Symbol/SymbolFile.h"
 #include "lldb/Target/DynamicLoader.h"
@@ -294,6 +295,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
   const bool include_targets = options.GetIncludeTargets();
   const bool include_modules = options.GetIncludeModules();
   const bool include_transcript = options.GetIncludeTranscript();
+  const bool include_plugins = options.GetIncludePlugins();
 
   json::Array json_targets;
   json::Array json_modules;
@@ -489,6 +491,10 @@ llvm::json::Value DebuggerStats::ReportStatistics(
     }
   }
 
+  if (include_plugins) {
+    global_stats.try_emplace("plugins", PluginManager::GetJSON());
+  }
+
   return std::move(global_stats);
 }
 
diff --git a/external/llvm-project/lldb/source/Target/Target.cpp b/external/llvm-project/lldb/source/Target/Target.cpp
index 9660fc97970b..45a9e1196a04 100644
--- a/external/llvm-project/lldb/source/Target/Target.cpp
+++ b/external/llvm-project/lldb/source/Target/Target.cpp
@@ -1706,6 +1706,8 @@ bool Target::SetArchitecture(const ArchSpec &arch_spec, bool set_platform,
         if (PlatformSP arch_platform_sp =
                 GetDebugger().GetPlatformList().GetOrCreate(other, {},
                                                             &platform_arch)) {
+          arch_platform_sp->SetLocateModuleCallback(
+              platform_sp->GetLocateModuleCallback());
           SetPlatform(arch_platform_sp);
           if (platform_arch.IsValid())
             other = platform_arch;
diff --git a/external/llvm-project/lldb/source/Target/Thread.cpp b/external/llvm-project/lldb/source/Target/Thread.cpp
index b0e0f1e67c06..c68894808eac 100644
--- a/external/llvm-project/lldb/source/Target/Thread.cpp
+++ b/external/llvm-project/lldb/source/Target/Thread.cpp
@@ -1668,10 +1668,14 @@ void Thread::DumpUsingSettingsFormat(Stream &strm, uint32_t frame_idx,
   ExecutionContext exe_ctx(shared_from_this());
 
   const FormatEntity::Entry *thread_format;
-  if (stop_format)
-    thread_format = exe_ctx.GetTargetRef().GetDebugger().GetThreadStopFormat();
-  else
-    thread_format = exe_ctx.GetTargetRef().GetDebugger().GetThreadFormat();
+  FormatEntity::Entry format_entry;
+  if (stop_format) {
+    format_entry = exe_ctx.GetTargetRef().GetDebugger().GetThreadStopFormat();
+    thread_format = &format_entry;
+  } else {
+    format_entry = exe_ctx.GetTargetRef().GetDebugger().GetThreadFormat();
+    thread_format = &format_entry;
+  }
 
   assert(thread_format);
 
diff --git a/external/llvm-project/lldb/source/Target/ThreadPlanTracer.cpp b/external/llvm-project/lldb/source/Target/ThreadPlanTracer.cpp
index ab63cc7f6c22..c5a9c5b806c3 100644
--- a/external/llvm-project/lldb/source/Target/ThreadPlanTracer.cpp
+++ b/external/llvm-project/lldb/source/Target/ThreadPlanTracer.cpp
@@ -174,11 +174,11 @@ void ThreadPlanAssemblyTracer::Log() {
         const bool show_control_flow_kind = true;
         Instruction *instruction =
             instruction_list.GetInstructionAtIndex(0).get();
-        const FormatEntity::Entry *disassemble_format =
+        FormatEntity::Entry disassemble_format =
             m_process.GetTarget().GetDebugger().GetDisassemblyFormat();
         instruction->Dump(stream_sp.get(), max_opcode_byte_size, show_address,
                           show_bytes, show_control_flow_kind, nullptr, nullptr,
-                          nullptr, disassemble_format, 0);
+                          nullptr, &disassemble_format, 0);
       }
     }
   }
diff --git a/external/llvm-project/lldb/source/Utility/CMakeLists.txt b/external/llvm-project/lldb/source/Utility/CMakeLists.txt
index 6954a2508ffe..1dd4d63f7016 100644
--- a/external/llvm-project/lldb/source/Utility/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Utility/CMakeLists.txt
@@ -81,12 +81,13 @@ add_lldb_library(lldbUtility NO_INTERNAL_DEPENDENCIES
   XcodeSDK.cpp
   ZipFile.cpp
 
-  LINK_LIBS
-    ${LLDB_SYSTEM_LIBS}
-    # lldbUtility does not depend on other LLDB libraries
-
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/Utility
   LINK_COMPONENTS
     BinaryFormat
     Support
     TargetParser
+  LINK_LIBS
+    ${LLDB_SYSTEM_LIBS}
+    # lldbUtility does not depend on other LLDB libraries
   )
diff --git a/external/llvm-project/lldb/source/Utility/Listener.cpp b/external/llvm-project/lldb/source/Utility/Listener.cpp
index 1efaad392502..d4ce3bf25ec5 100644
--- a/external/llvm-project/lldb/source/Utility/Listener.cpp
+++ b/external/llvm-project/lldb/source/Utility/Listener.cpp
@@ -352,8 +352,7 @@ Listener::StartListeningForEventSpec(const BroadcasterManagerSP &manager_sp,
       this->shared_from_this(), event_spec);
   if (bits_acquired) {
     BroadcasterManagerWP manager_wp(manager_sp);
-    auto iter = llvm::find_if(m_broadcaster_managers, manager_matcher);
-    if (iter == m_broadcaster_managers.end())
+    if (llvm::none_of(m_broadcaster_managers, manager_matcher))
       m_broadcaster_managers.push_back(manager_wp);
   }
 
diff --git a/external/llvm-project/lldb/source/ValueObject/CMakeLists.txt b/external/llvm-project/lldb/source/ValueObject/CMakeLists.txt
index 92683916f5a5..db590f757972 100644
--- a/external/llvm-project/lldb/source/ValueObject/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/ValueObject/CMakeLists.txt
@@ -19,7 +19,12 @@ add_lldb_library(lldbValueObject
   ValueObjectVariable.cpp
   ValueObjectVTable.cpp
 
-
+  ADDITIONAL_HEADER_DIRS
+    ${LLDB_INCLUDE_DIR}/lldb/ValueObject
+  LINK_COMPONENTS
+    Support
+    Demangle
+    TargetParser
   LINK_LIBS
     lldbBreakpoint
     lldbDataFormatters
@@ -31,9 +36,4 @@ add_lldb_library(lldbValueObject
     lldbUtility
     lldbPluginCPlusPlusLanguage
     lldbPluginObjCLanguage
-
-  LINK_COMPONENTS
-    Support
-    Demangle
-    TargetParser
   )
diff --git a/external/llvm-project/lldb/source/ValueObject/DILAST.cpp b/external/llvm-project/lldb/source/ValueObject/DILAST.cpp
index 8b5e64ad462c..b1cd824c2299 100644
--- a/external/llvm-project/lldb/source/ValueObject/DILAST.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/DILAST.cpp
@@ -32,4 +32,9 @@ ArraySubscriptNode::Accept(Visitor *v) const {
   return v->Visit(this);
 }
 
+llvm::Expected<lldb::ValueObjectSP>
+BitFieldExtractionNode::Accept(Visitor *v) const {
+  return v->Visit(this);
+}
+
 } // namespace lldb_private::dil
diff --git a/external/llvm-project/lldb/source/ValueObject/DILEval.cpp b/external/llvm-project/lldb/source/ValueObject/DILEval.cpp
index c8cb54aa18a9..b2bb4e20ddc2 100644
--- a/external/llvm-project/lldb/source/ValueObject/DILEval.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/DILEval.cpp
@@ -430,4 +430,36 @@ Interpreter::Visit(const ArraySubscriptNode *node) {
   return base->GetSyntheticArrayMember(signed_child_idx, true);
 }
 
+llvm::Expected<lldb::ValueObjectSP>
+Interpreter::Visit(const BitFieldExtractionNode *node) {
+  auto lhs_or_err = Evaluate(node->GetBase());
+  if (!lhs_or_err)
+    return lhs_or_err;
+  lldb::ValueObjectSP base = *lhs_or_err;
+  int64_t first_index = node->GetFirstIndex();
+  int64_t last_index = node->GetLastIndex();
+
+  // if the format given is [high-low], swap range
+  if (first_index > last_index)
+    std::swap(first_index, last_index);
+
+  Status error;
+  if (base->GetCompilerType().IsReferenceType()) {
+    base = base->Dereference(error);
+    if (error.Fail())
+      return error.ToError();
+  }
+  lldb::ValueObjectSP child_valobj_sp =
+      base->GetSyntheticBitFieldChild(first_index, last_index, true);
+  if (!child_valobj_sp) {
+    std::string message = llvm::formatv(
+        "bitfield range {0}-{1} is not valid for \"({2}) {3}\"", first_index,
+        last_index, base->GetTypeName().AsCString("<invalid type>"),
+        base->GetName().AsCString());
+    return llvm::make_error<DILDiagnosticError>(m_expr, message,
+                                                node->GetLocation());
+  }
+  return child_valobj_sp;
+}
+
 } // namespace lldb_private::dil
diff --git a/external/llvm-project/lldb/source/ValueObject/DILLexer.cpp b/external/llvm-project/lldb/source/ValueObject/DILLexer.cpp
index 99182d2da113..eaefaf484bc1 100644
--- a/external/llvm-project/lldb/source/ValueObject/DILLexer.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/DILLexer.cpp
@@ -34,6 +34,8 @@ llvm::StringRef Token::GetTokenName(Kind kind) {
     return "l_paren";
   case Kind::l_square:
     return "l_square";
+  case Kind::minus:
+    return "minus";
   case Kind::numeric_constant:
     return "numeric_constant";
   case Kind::period:
@@ -113,8 +115,9 @@ llvm::Expected<Token> DILLexer::Lex(llvm::StringRef expr,
 
   constexpr std::pair<Token::Kind, const char *> operators[] = {
       {Token::amp, "&"},     {Token::arrow, "->"},   {Token::coloncolon, "::"},
-      {Token::l_paren, "("}, {Token::l_square, "["}, {Token::period, "."},
-      {Token::r_paren, ")"}, {Token::r_square, "]"}, {Token::star, "*"},
+      {Token::l_paren, "("}, {Token::l_square, "["}, {Token::minus, "-"},
+      {Token::period, "."},  {Token::r_paren, ")"},  {Token::r_square, "]"},
+      {Token::star, "*"},
   };
   for (auto [kind, str] : operators) {
     if (remainder.consume_front(str))
diff --git a/external/llvm-project/lldb/source/ValueObject/DILParser.cpp b/external/llvm-project/lldb/source/ValueObject/DILParser.cpp
index 9667885734f2..32af0820acb9 100644
--- a/external/llvm-project/lldb/source/ValueObject/DILParser.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/DILParser.cpp
@@ -120,6 +120,7 @@ ASTNodeUP DILParser::ParseUnaryExpression() {
 //  postfix_expression:
 //    primary_expression
 //    postfix_expression "[" integer_literal "]"
+//    postfix_expression "[" integer_literal "-" integer_literal "]"
 //    postfix_expression "." id_expression
 //    postfix_expression "->" id_expression
 //
@@ -131,17 +132,30 @@ ASTNodeUP DILParser::ParsePostfixExpression() {
     switch (token.GetKind()) {
     case Token::l_square: {
       m_dil_lexer.Advance();
-      std::optional<int64_t> rhs = ParseIntegerConstant();
-      if (!rhs) {
+      std::optional<int64_t> index = ParseIntegerConstant();
+      if (!index) {
         BailOut(
             llvm::formatv("failed to parse integer constant: {0}", CurToken()),
             CurToken().GetLocation(), CurToken().GetSpelling().length());
         return std::make_unique<ErrorNode>();
       }
+      if (CurToken().GetKind() == Token::minus) {
+        m_dil_lexer.Advance();
+        std::optional<int64_t> last_index = ParseIntegerConstant();
+        if (!last_index) {
+          BailOut(llvm::formatv("failed to parse integer constant: {0}",
+                                CurToken()),
+                  CurToken().GetLocation(), CurToken().GetSpelling().length());
+          return std::make_unique<ErrorNode>();
+        }
+        lhs = std::make_unique<BitFieldExtractionNode>(
+            loc, std::move(lhs), std::move(*index), std::move(*last_index));
+      } else {
+        lhs = std::make_unique<ArraySubscriptNode>(loc, std::move(lhs),
+                                                   std::move(*index));
+      }
       Expect(Token::r_square);
       m_dil_lexer.Advance();
-      lhs = std::make_unique<ArraySubscriptNode>(loc, std::move(lhs),
-                                                 std::move(*rhs));
       break;
     }
     case Token::period:
diff --git a/external/llvm-project/lldb/source/ValueObject/ValueObject.cpp b/external/llvm-project/lldb/source/ValueObject/ValueObject.cpp
index 29ddeba3df18..accfd08b3020 100644
--- a/external/llvm-project/lldb/source/ValueObject/ValueObject.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/ValueObject.cpp
@@ -656,9 +656,7 @@ bool ValueObject::IsCStringContainer(bool check_pointer) {
     return true;
   if (type_flags.Test(eTypeIsArray))
     return true;
-  addr_t cstr_address = LLDB_INVALID_ADDRESS;
-  AddressType cstr_address_type = eAddressTypeInvalid;
-  cstr_address = GetPointerValue(&cstr_address_type);
+  addr_t cstr_address = GetPointerValue().address;
   return (cstr_address != LLDB_INVALID_ADDRESS);
 }
 
@@ -707,9 +705,8 @@ size_t ValueObject::GetPointeeData(DataExtractor &data, uint32_t item_idx,
     lldb::DataBufferSP data_sp(heap_buf_ptr =
                                    new lldb_private::DataBufferHeap());
 
-    AddressType addr_type;
-    lldb::addr_t addr = is_pointer_type ? GetPointerValue(&addr_type)
-                                        : GetAddressOf(true, &addr_type);
+    auto [addr, addr_type] =
+        is_pointer_type ? GetPointerValue() : GetAddressOf(true);
 
     switch (addr_type) {
     case eAddressTypeFile: {
@@ -901,8 +898,7 @@ ValueObject::ReadPointedString(lldb::WritableDataBufferSP &buffer_sp,
   const Flags type_flags(GetTypeInfo(&elem_or_pointee_compiler_type));
   if (type_flags.AnySet(eTypeIsArray | eTypeIsPointer) &&
       elem_or_pointee_compiler_type.IsCharType()) {
-    addr_t cstr_address = LLDB_INVALID_ADDRESS;
-    AddressType cstr_address_type = eAddressTypeInvalid;
+    AddrAndType cstr_address;
 
     size_t cstr_len = 0;
     bool capped_data = false;
@@ -917,14 +913,15 @@ ValueObject::ReadPointedString(lldb::WritableDataBufferSP &buffer_sp,
           cstr_len = max_length;
         }
       }
-      cstr_address = GetAddressOf(true, &cstr_address_type);
+      cstr_address = GetAddressOf(true);
     } else {
       // We have a pointer
-      cstr_address = GetPointerValue(&cstr_address_type);
+      cstr_address = GetPointerValue();
     }
 
-    if (cstr_address == 0 || cstr_address == LLDB_INVALID_ADDRESS) {
-      if (cstr_address_type == eAddressTypeHost && is_array) {
+    if (cstr_address.address == 0 ||
+        cstr_address.address == LLDB_INVALID_ADDRESS) {
+      if (cstr_address.type == eAddressTypeHost && is_array) {
         const char *cstr = GetDataExtractor().PeekCStr(0);
         if (cstr == nullptr) {
           s << "<invalid address>";
@@ -943,7 +940,7 @@ ValueObject::ReadPointedString(lldb::WritableDataBufferSP &buffer_sp,
       }
     }
 
-    Address cstr_so_addr(cstr_address);
+    Address cstr_so_addr(cstr_address.address);
     DataExtractor data;
     if (cstr_len > 0 && honor_array) {
       // I am using GetPointeeData() here to abstract the fact that some
@@ -1198,7 +1195,7 @@ llvm::Expected<bool> ValueObject::GetValueAsBool() {
       return value_or_err->isNonZero();
   }
   if (val_type.IsArrayType())
-    return GetAddressOf() != 0;
+    return GetAddressOf().address != 0;
 
   return llvm::make_error<llvm::StringError>("type cannot be converted to bool",
                                              llvm::inconvertibleErrorCode());
@@ -1599,70 +1596,55 @@ bool ValueObject::DumpPrintableRepresentation(
   return var_success;
 }
 
-addr_t ValueObject::GetAddressOf(bool scalar_is_load_address,
-                                 AddressType *address_type) {
+ValueObject::AddrAndType
+ValueObject::GetAddressOf(bool scalar_is_load_address) {
   // Can't take address of a bitfield
   if (IsBitfield())
-    return LLDB_INVALID_ADDRESS;
+    return {};
 
   if (!UpdateValueIfNeeded(false))
-    return LLDB_INVALID_ADDRESS;
+    return {};
 
   switch (m_value.GetValueType()) {
   case Value::ValueType::Invalid:
-    return LLDB_INVALID_ADDRESS;
+    return {};
   case Value::ValueType::Scalar:
     if (scalar_is_load_address) {
-      if (address_type)
-        *address_type = eAddressTypeLoad;
-      return m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
+      return {m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS),
+              eAddressTypeLoad};
     }
-    break;
+    return {};
 
   case Value::ValueType::LoadAddress:
-  case Value::ValueType::FileAddress: {
-    if (address_type)
-      *address_type = m_value.GetValueAddressType();
-    return m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
-  } break;
-  case Value::ValueType::HostAddress: {
-    if (address_type)
-      *address_type = m_value.GetValueAddressType();
-    return LLDB_INVALID_ADDRESS;
-  } break;
+  case Value::ValueType::FileAddress:
+    return {m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS),
+            m_value.GetValueAddressType()};
+  case Value::ValueType::HostAddress:
+    return {LLDB_INVALID_ADDRESS, m_value.GetValueAddressType()};
   }
-  if (address_type)
-    *address_type = eAddressTypeInvalid;
-  return LLDB_INVALID_ADDRESS;
+  llvm_unreachable("Unhandled value type!");
 }
 
-addr_t ValueObject::GetPointerValue(AddressType *address_type) {
-  addr_t address = LLDB_INVALID_ADDRESS;
-  if (address_type)
-    *address_type = eAddressTypeInvalid;
-
+ValueObject::AddrAndType ValueObject::GetPointerValue() {
   if (!UpdateValueIfNeeded(false))
-    return address;
+    return {};
 
   switch (m_value.GetValueType()) {
   case Value::ValueType::Invalid:
-    return LLDB_INVALID_ADDRESS;
+    return {};
   case Value::ValueType::Scalar:
-    address = m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
-    break;
+    return {m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS),
+            GetAddressTypeOfChildren()};
 
   case Value::ValueType::HostAddress:
   case Value::ValueType::LoadAddress:
   case Value::ValueType::FileAddress: {
     lldb::offset_t data_offset = 0;
-    address = m_data.GetAddress(&data_offset);
-  } break;
+    return {m_data.GetAddress(&data_offset), GetAddressTypeOfChildren()};
+  }
   }
 
-  if (address_type)
-    *address_type = GetAddressTypeOfChildren();
-
-  return address;
+  llvm_unreachable("Unhandled value type!");
 }
 
 static const char *ConvertBoolean(lldb::LanguageType language_type,
@@ -2749,7 +2731,7 @@ ValueObjectSP ValueObject::CreateConstantValue(ConstString name) {
 
     valobj_sp = ValueObjectConstResult::Create(
         exe_ctx.GetBestExecutionContextScope(), GetCompilerType(), name, data,
-        GetAddressOf());
+        GetAddressOf().address);
   }
 
   if (!valobj_sp) {
@@ -2875,9 +2857,7 @@ ValueObjectSP ValueObject::AddressOf(Status &error) {
   if (m_addr_of_valobj_sp)
     return m_addr_of_valobj_sp;
 
-  AddressType address_type = eAddressTypeInvalid;
-  const bool scalar_is_load_address = false;
-  addr_t addr = GetAddressOf(scalar_is_load_address, &address_type);
+  auto [addr, address_type] = GetAddressOf(/*scalar_is_load_address=*/false);
   error.Clear();
   if (addr != LLDB_INVALID_ADDRESS && address_type != eAddressTypeHost) {
     switch (address_type) {
@@ -2964,8 +2944,7 @@ lldb::ValueObjectSP ValueObject::Clone(ConstString new_name) {
 ValueObjectSP ValueObject::CastPointerType(const char *name,
                                            CompilerType &compiler_type) {
   ValueObjectSP valobj_sp;
-  AddressType address_type;
-  addr_t ptr_value = GetPointerValue(&address_type);
+  addr_t ptr_value = GetPointerValue().address;
 
   if (ptr_value != LLDB_INVALID_ADDRESS) {
     Address ptr_addr(ptr_value);
@@ -2978,8 +2957,7 @@ ValueObjectSP ValueObject::CastPointerType(const char *name,
 
 ValueObjectSP ValueObject::CastPointerType(const char *name, TypeSP &type_sp) {
   ValueObjectSP valobj_sp;
-  AddressType address_type;
-  addr_t ptr_value = GetPointerValue(&address_type);
+  addr_t ptr_value = GetPointerValue().address;
 
   if (ptr_value != LLDB_INVALID_ADDRESS) {
     Address ptr_addr(ptr_value);
@@ -2991,11 +2969,9 @@ ValueObjectSP ValueObject::CastPointerType(const char *name, TypeSP &type_sp) {
 }
 
 lldb::addr_t ValueObject::GetLoadAddress() {
-  lldb::addr_t addr_value = LLDB_INVALID_ADDRESS;
   if (auto target_sp = GetTargetSP()) {
     const bool scalar_is_load_address = true;
-    AddressType addr_type;
-    addr_value = GetAddressOf(scalar_is_load_address, &addr_type);
+    auto [addr_value, addr_type] = GetAddressOf(scalar_is_load_address);
     if (addr_type == eAddressTypeFile) {
       lldb::ModuleSP module_sp(GetModule());
       if (!module_sp)
@@ -3008,8 +2984,9 @@ lldb::addr_t ValueObject::GetLoadAddress() {
     } else if (addr_type == eAddressTypeHost ||
                addr_type == eAddressTypeInvalid)
       addr_value = LLDB_INVALID_ADDRESS;
+    return addr_value;
   }
-  return addr_value;
+  return LLDB_INVALID_ADDRESS;
 }
 
 llvm::Expected<lldb::ValueObjectSP> ValueObject::CastDerivedToBaseType(
diff --git a/external/llvm-project/lldb/source/ValueObject/ValueObjectChild.cpp b/external/llvm-project/lldb/source/ValueObject/ValueObjectChild.cpp
index d7f1ad08415e..e29634c2f332 100644
--- a/external/llvm-project/lldb/source/ValueObject/ValueObjectChild.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/ValueObjectChild.cpp
@@ -118,7 +118,7 @@ bool ValueObjectChild::UpdateValue() {
            (parent_type_flags.AnySet(lldb::eTypeInstanceIsPointer)));
 
       if (parent->GetCompilerType().ShouldTreatScalarValueAsAddress()) {
-        m_value.GetScalar() = parent->GetPointerValue();
+        m_value.GetScalar() = parent->GetPointerValue().address;
 
         switch (parent->GetAddressTypeOfChildren()) {
         case eAddressTypeFile: {
diff --git a/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResult.cpp b/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResult.cpp
index 01f870529a14..774749620e0d 100644
--- a/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResult.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResult.cpp
@@ -265,9 +265,9 @@ lldb::ValueObjectSP ValueObjectConstResult::AddressOf(Status &error) {
   return m_impl.AddressOf(error);
 }
 
-lldb::addr_t ValueObjectConstResult::GetAddressOf(bool scalar_is_load_address,
-                                                  AddressType *address_type) {
-  return m_impl.GetAddressOf(scalar_is_load_address, address_type);
+ValueObject::AddrAndType
+ValueObjectConstResult::GetAddressOf(bool scalar_is_load_address) {
+  return m_impl.GetAddressOf(scalar_is_load_address);
 }
 
 size_t ValueObjectConstResult::GetPointeeData(DataExtractor &data,
diff --git a/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultChild.cpp b/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultChild.cpp
index b1c800dfe8c3..768878984e16 100644
--- a/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultChild.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultChild.cpp
@@ -50,10 +50,9 @@ lldb::ValueObjectSP ValueObjectConstResultChild::AddressOf(Status &error) {
   return m_impl.AddressOf(error);
 }
 
-lldb::addr_t
-ValueObjectConstResultChild::GetAddressOf(bool scalar_is_load_address,
-                                          AddressType *address_type) {
-  return m_impl.GetAddressOf(scalar_is_load_address, address_type);
+ValueObject::AddrAndType
+ValueObjectConstResultChild::GetAddressOf(bool scalar_is_load_address) {
+  return m_impl.GetAddressOf(scalar_is_load_address);
 }
 
 size_t ValueObjectConstResultChild::GetPointeeData(DataExtractor &data,
diff --git a/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultImpl.cpp b/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultImpl.cpp
index d79f655b015a..874628774d47 100644
--- a/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultImpl.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/ValueObjectConstResultImpl.cpp
@@ -206,22 +206,16 @@ ValueObjectConstResultImpl::Cast(const CompilerType &compiler_type) {
   return result_cast->GetSP();
 }
 
-lldb::addr_t
-ValueObjectConstResultImpl::GetAddressOf(bool scalar_is_load_address,
-                                         AddressType *address_type) {
+ValueObject::AddrAndType
+ValueObjectConstResultImpl::GetAddressOf(bool scalar_is_load_address) {
 
   if (m_impl_backend == nullptr)
-    return 0;
-
-  if (m_live_address == LLDB_INVALID_ADDRESS) {
-    return m_impl_backend->ValueObject::GetAddressOf(scalar_is_load_address,
-                                                     address_type);
-  }
+    return {0, eAddressTypeInvalid};
 
-  if (address_type)
-    *address_type = m_live_address_type;
+  if (m_live_address == LLDB_INVALID_ADDRESS)
+    return m_impl_backend->ValueObject::GetAddressOf(scalar_is_load_address);
 
-  return m_live_address;
+  return {m_live_address, m_live_address_type};
 }
 
 size_t ValueObjectConstResultImpl::GetPointeeData(DataExtractor &data,
diff --git a/external/llvm-project/lldb/source/ValueObject/ValueObjectVTable.cpp b/external/llvm-project/lldb/source/ValueObject/ValueObjectVTable.cpp
index 92bd086d88ee..6f6ee4579cee 100644
--- a/external/llvm-project/lldb/source/ValueObject/ValueObjectVTable.cpp
+++ b/external/llvm-project/lldb/source/ValueObject/ValueObjectVTable.cpp
@@ -252,7 +252,7 @@ bool ValueObjectVTable::UpdateValue() {
   m_num_vtable_entries = (vtable_end_addr - vtable_start_addr) / m_addr_size;
 
   m_value.SetValueType(Value::ValueType::LoadAddress);
-  m_value.GetScalar() = parent->GetAddressOf();
+  m_value.GetScalar() = parent->GetAddressOf().address;
   auto type_system_or_err =
       target_sp->GetScratchTypeSystemForLanguage(eLanguageTypeC_plus_plus);
   if (type_system_or_err) {
diff --git a/external/llvm-project/lldb/source/Version/CMakeLists.txt b/external/llvm-project/lldb/source/Version/CMakeLists.txt
index 90fe53d52cc5..8b0acb9ef755 100644
--- a/external/llvm-project/lldb/source/Version/CMakeLists.txt
+++ b/external/llvm-project/lldb/source/Version/CMakeLists.txt
@@ -21,9 +21,6 @@ add_custom_command(OUTPUT "${vcs_version_inc}"
                            "-DLLVM_FORCE_VC_REPOSITORY=${LLVM_FORCE_VC_REPOSITORY}"
                            -P "${generate_vcs_version_script}")
 
-set_source_files_properties("${vcs_version_inc}"
-  PROPERTIES GENERATED TRUE
-             HEADER_FILE_ONLY TRUE)
 
 # Configure the Version.inc file.
 set(version_inc "${LLDB_BINARY_DIR}/include/lldb/Version/Version.inc")
@@ -32,13 +29,15 @@ configure_file(
   ${LLDB_SOURCE_DIR}/include/lldb/Version/Version.inc.in
   ${version_inc})
 
-set_source_files_properties("${version_inc}"
+set_source_files_properties(
+  "${vcs_version_inc}" "${version_inc}"
   PROPERTIES GENERATED TRUE
-             HEADER_FILE_ONLY TRUE)
-
+)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_lldb_library(lldbVersion NO_PLUGIN_DEPENDENCIES
   Version.cpp
-  ${vcs_version_inc}
-  ${version_inc})
+  ADDITIONAL_HEADERS
+    ${version_inc}
+    ${vcs_version_inc}
+)
diff --git a/external/llvm-project/lldb/test/API/api/multiple-debuggers/TestMultipleDebuggers.py b/external/llvm-project/lldb/test/API/api/multiple-debuggers/TestMultipleDebuggers.py
index 1fd4806cd74f..7d6fdd444791 100644
--- a/external/llvm-project/lldb/test/API/api/multiple-debuggers/TestMultipleDebuggers.py
+++ b/external/llvm-project/lldb/test/API/api/multiple-debuggers/TestMultipleDebuggers.py
@@ -12,7 +12,9 @@
 class TestMultipleSimultaneousDebuggers(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
-    # Sometimes times out on Linux, see https://github.com/llvm/llvm-project/issues/101162.
+    # Times out on heavily loaded Linux buildbots, don't want to get into tweaking
+    # the timeout per bot. Does work when run alone. See:
+    # https://github.com/llvm/llvm-project/issues/101162
     @skipIfLinux
     @skipIfNoSBHeaders
     @skipIfWindows
diff --git a/external/llvm-project/lldb/test/API/api/multiple-debuggers/multi-process-driver.cpp b/external/llvm-project/lldb/test/API/api/multiple-debuggers/multi-process-driver.cpp
index 64728fb7c29a..68d73f1dee01 100644
--- a/external/llvm-project/lldb/test/API/api/multiple-debuggers/multi-process-driver.cpp
+++ b/external/llvm-project/lldb/test/API/api/multiple-debuggers/multi-process-driver.cpp
@@ -13,10 +13,11 @@
 // that are hit when lldb is being used to debug multiple processes
 // simultaneously.
 
+#include <inttypes.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <inttypes.h>
+#include <unistd.h>
 
 #include "lldb/API/LLDB.h"
 #include "lldb/API/SBCommandInterpreter.h"
@@ -296,6 +297,9 @@ int main (int argc, char **argv)
                  NUMBER_OF_SIMULTANEOUS_DEBUG_SESSIONS);
     }
 
-    SBDebugger::Terminate();
-    exit (1);
+    // We do not call SBDebugger::Terminate() here because it will destroy
+    // data that might be being used by threads that are still running. Which
+    // would change the timeout into an unrelated crash.
+    // _exit instead of exit, to skip more things that could cause a crash.
+    _exit(1);
 }
diff --git a/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py
index 0d91f804ce56..c90e0eaa6363 100644
--- a/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py
+++ b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py
@@ -63,7 +63,7 @@ def test_subscript(self):
         self.expect(
             "frame var 'int_arr[-1]'",
             error=True,
-            substrs=["unrecognized token"],
+            substrs=["failed to parse integer constant"],
         )
 
         # Test for floating point index
diff --git a/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/Makefile b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/TestFrameVarDILBitFieldExtraction.py b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/TestFrameVarDILBitFieldExtraction.py
new file mode 100644
index 000000000000..7b5ef0650b6e
--- /dev/null
+++ b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/TestFrameVarDILBitFieldExtraction.py
@@ -0,0 +1,56 @@
+"""
+Test DIL BifField extraction.
+"""
+
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from lldbsuite.test import lldbutil
+
+
+class TestFrameVarDILBitFieldExtraction(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def expect_var_path(self, expr, compare_to_framevar=False, value=None, type=None):
+        value_dil = super().expect_var_path(expr, value=value, type=type)
+        if compare_to_framevar:
+            self.runCmd("settings set target.experimental.use-DIL false")
+            value_frv = super().expect_var_path(expr, value=value, type=type)
+            self.runCmd("settings set target.experimental.use-DIL true")
+            self.assertEqual(value_dil.GetValue(), value_frv.GetValue())
+
+    def test_bitfield_extraction(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+
+        # Test ranges and type
+        self.expect_var_path("value[0-1]", True, value="3", type="int:2")
+        self.expect_var_path("value[4-7]", True, value="7", type="int:4")
+        self.expect_var_path("value[7-0]", True, value="115", type="int:8")
+
+        # Test reference and dereferenced pointer
+        self.expect_var_path("value_ref[0-1]", value="3", type="int:2")
+        self.expect_var_path("(*value_ptr)[0-1]", value="3", type="int:2")
+
+        # Test array and pointer
+        self.expect(
+            "frame var 'int_arr[0-2]'",
+            error=True,
+            substrs=["bitfield range 0-2 is not valid"],
+        )
+        self.expect(
+            "frame var 'value_ptr[0-1]'",
+            error=True,
+            substrs=["bitfield range 0-1 is not valid"],
+        )
+
+        # Test invalid input
+        self.expect(
+            "frame var 'value[1-]'",
+            error=True,
+            substrs=["failed to parse integer constant"],
+        )
diff --git a/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/main.cpp b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/main.cpp
new file mode 100644
index 000000000000..a35f68a9e30a
--- /dev/null
+++ b/external/llvm-project/lldb/test/API/commands/frame/var-dil/basics/BitFieldExtraction/main.cpp
@@ -0,0 +1,9 @@
+int main(int argc, char **argv) {
+  int value = 0b01110011;
+  int &value_ref = value;
+  int *value_ptr = &value;
+
+  int int_arr[] = {7, 3, 1};
+
+  return 0; // Set a breakpoint here
+}
diff --git a/external/llvm-project/lldb/test/API/commands/platform/connect/TestPlatformConnect.py b/external/llvm-project/lldb/test/API/commands/platform/connect/TestPlatformConnect.py
index fc6c2ee98df4..5df0c16fbd1f 100644
--- a/external/llvm-project/lldb/test/API/commands/platform/connect/TestPlatformConnect.py
+++ b/external/llvm-project/lldb/test/API/commands/platform/connect/TestPlatformConnect.py
@@ -57,3 +57,48 @@ def test_platform_process_connect(self):
         self.assertEqual(frame.GetFunction().GetName(), "main")
         self.assertEqual(frame.FindVariable("argc").GetValueAsSigned(), 2)
         process.Continue()
+
+    @skipIfRemote
+    @expectedFailureAll(hostoslist=["windows"], triple=".*-android")
+    @skipIfDarwin  # lldb-server not found correctly
+    @expectedFailureAll(oslist=["windows"])  # process modules not loaded
+    # lldb-server platform times out waiting for the gdbserver port number to be
+    # written to the pipe, yet it seems the gdbserver already has written it.
+    @expectedFailureAll(
+        archs=["aarch64"],
+        oslist=["freebsd"],
+        bugnumber="https://github.com/llvm/llvm-project/issues/84327",
+    )
+    @add_test_categories(["lldb-server"])
+    def test_platform_process_connect_with_unix_connect(self):
+        self.build()
+        lldb_temp_dir = lldb.SBHostOS.GetLLDBPath(lldb.ePathTypeLLDBTempSystemDir)
+        named_pipe = "%s/platform_server.sock" % lldb_temp_dir
+        port_file = self.getBuildArtifact("port")
+        commandline_args = [
+            "platform",
+            "--listen",
+            named_pipe,
+            "--socket-file",
+            port_file,
+            "--",
+            self.getBuildArtifact("a.out"),
+            "foo",
+        ]
+        self.spawnSubprocess(lldbgdbserverutils.get_lldb_server_exe(), commandline_args)
+
+        socket_file = lldbutil.wait_for_file_on_target(self, port_file)
+        new_platform = lldb.SBPlatform("remote-" + self.getPlatform())
+        self.dbg.SetSelectedPlatform(new_platform)
+        connect_url = "unix-connect://%s" % socket_file
+        self.runCmd("platform connect %s" % connect_url)
+
+        lldbutil.run_break_set_by_symbol(self, "main")
+        process = self.process()
+
+        process.Continue()
+
+        frame = self.frame()
+        self.assertEqual(frame.GetFunction().GetName(), "main")
+        self.assertEqual(frame.FindVariable("argc").GetValueAsSigned(), 2)
+        process.Continue()
diff --git a/external/llvm-project/lldb/test/API/commands/settings/TestSettings.py b/external/llvm-project/lldb/test/API/commands/settings/TestSettings.py
index 4f4fa0e032b5..bc864942055c 100644
--- a/external/llvm-project/lldb/test/API/commands/settings/TestSettings.py
+++ b/external/llvm-project/lldb/test/API/commands/settings/TestSettings.py
@@ -168,8 +168,6 @@ def test_set_term_width(self):
             substrs=["term-width (unsigned) = 60"],
         )
 
-    # rdar://problem/10712130
-    @skipIf(oslist=["windows"], bugnumber="llvm.org/pr44431")
     def test_set_frame_format(self):
         """Test that 'set frame-format' with a backtick char in the format string works as well as fullpath."""
         self.build()
diff --git a/external/llvm-project/lldb/test/API/commands/statistics/basic/TestStats.py b/external/llvm-project/lldb/test/API/commands/statistics/basic/TestStats.py
index a9a7e933a321..83132b40d85d 100644
--- a/external/llvm-project/lldb/test/API/commands/statistics/basic/TestStats.py
+++ b/external/llvm-project/lldb/test/API/commands/statistics/basic/TestStats.py
@@ -1068,3 +1068,89 @@ def test_multiple_targets(self):
         all_targets_stats = self.get_stats("--all-targets")
         self.assertIsNotNone(self.find_module_in_metrics(main_exe, all_targets_stats))
         self.assertIsNotNone(self.find_module_in_metrics(second_exe, all_targets_stats))
+
+    # Return some level of the plugin stats hierarchy.
+    # Will return either the top-level node, the namespace node, or a specific
+    # plugin node based on requested values.
+    #
+    # If any of the requested keys are not found in the stats then return None.
+    #
+    # Plugin stats look like this:
+    #
+    # "plugins": {
+    #     "system-runtime": [
+    #         {
+    #             "enabled": true,
+    #             "name": "systemruntime-macosx"
+    #         }
+    #     ]
+    # },
+    def get_plugin_stats(self, debugger_stats, plugin_namespace=None, plugin_name=None):
+        # Get top level plugin stats.
+        if "plugins" not in debugger_stats:
+            return None
+        plugins = debugger_stats["plugins"]
+        if not plugin_namespace:
+            return plugins
+
+        # Plugin namespace stats.
+        if plugin_namespace not in plugins:
+            return None
+        plugins_for_namespace = plugins[plugin_namespace]
+        if not plugin_name:
+            return plugins_for_namespace
+
+        # Specific plugin stats.
+        for plugin in debugger_stats["plugins"][plugin_namespace]:
+            if plugin["name"] == plugin_name:
+                return plugin
+        return None
+
+    def test_plugin_stats(self):
+        """
+        Test "statistics dump" contains plugin info.
+        """
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debugger_stats = self.get_stats()
+
+        # Verify that the statistics dump contains the plugin information.
+        plugins = self.get_plugin_stats(debugger_stats)
+        self.assertIsNotNone(plugins)
+
+        # Check for a known plugin namespace that should be in the stats.
+        system_runtime_plugins = self.get_plugin_stats(debugger_stats, "system-runtime")
+        self.assertIsNotNone(system_runtime_plugins)
+
+        # Validate the keys exists for the bottom-level plugin stats.
+        plugin_keys_exist = [
+            "name",
+            "enabled",
+        ]
+        for plugin in system_runtime_plugins:
+            self.verify_keys(
+                plugin, 'debugger_stats["plugins"]["system-runtime"]', plugin_keys_exist
+            )
+
+        # Check for a known plugin that is enabled by default.
+        system_runtime_macosx_plugin = self.get_plugin_stats(
+            debugger_stats, "system-runtime", "systemruntime-macosx"
+        )
+        self.assertIsNotNone(system_runtime_macosx_plugin)
+        self.assertTrue(system_runtime_macosx_plugin["enabled"])
+
+        # Now disable the plugin and check the stats again.
+        # The stats should show the plugin is disabled.
+        self.runCmd("plugin disable system-runtime.systemruntime-macosx")
+        debugger_stats = self.get_stats()
+        system_runtime_macosx_plugin = self.get_plugin_stats(
+            debugger_stats, "system-runtime", "systemruntime-macosx"
+        )
+        self.assertIsNotNone(system_runtime_macosx_plugin)
+        self.assertFalse(system_runtime_macosx_plugin["enabled"])
+
+        # Plugins should not show up in the stats when disabled with an option.
+        debugger_stats = self.get_stats("--plugins false")
+        plugins = self.get_plugin_stats(debugger_stats)
+        self.assertIsNone(plugins)
diff --git a/external/llvm-project/lldb/test/API/commands/target/stop-hooks/TestStopHookScripted.py b/external/llvm-project/lldb/test/API/commands/target/stop-hooks/TestStopHookScripted.py
index b71f3421f983..954cac159243 100644
--- a/external/llvm-project/lldb/test/API/commands/target/stop-hooks/TestStopHookScripted.py
+++ b/external/llvm-project/lldb/test/API/commands/target/stop-hooks/TestStopHookScripted.py
@@ -58,19 +58,23 @@ def test_stop_hooks_scripted_no_entry(self):
 
     def test_stop_hooks_scripted_right_func(self):
         """Test that a scripted stop hook fires when there is a function match"""
-        self.stop_hooks_scripted(5, "-n step_out_of_me")
+        self.stop_hooks_scripted(5, "-I 0 -n step_out_of_me")
 
     def test_stop_hooks_scripted_wrong_func(self):
         """Test that a scripted stop hook doesn't fire when the function does not match"""
-        self.stop_hooks_scripted(0, "-n main")
+        self.stop_hooks_scripted(0, "-I 0 -n main")
 
     def test_stop_hooks_scripted_right_lines(self):
         """Test that a scripted stop hook fires when there is a function match"""
-        self.stop_hooks_scripted(5, "-f main.c -l 1 -e %d" % (self.main_start_line))
+        self.stop_hooks_scripted(
+            5, "-I 0 -f main.c -l 1 -e %d" % (self.main_start_line)
+        )
 
     def test_stop_hooks_scripted_wrong_lines(self):
         """Test that a scripted stop hook doesn't fire when the function does not match"""
-        self.stop_hooks_scripted(0, "-f main.c -l %d -e 100" % (self.main_start_line))
+        self.stop_hooks_scripted(
+            0, "-I 0 -f main.c -l %d -e 100" % (self.main_start_line)
+        )
 
     def test_stop_hooks_scripted_auto_continue(self):
         """Test that the --auto-continue flag works"""
@@ -89,9 +93,9 @@ def do_test_auto_continue(self, return_true):
         result = lldb.SBCommandReturnObject()
 
         if return_true:
-            command = "target stop-hook add -P stop_hook.stop_handler -k increment -v 5 -k return_false -v 1 -n step_out_of_me"
+            command = "target stop-hook add -I 0 -P stop_hook.stop_handler -k increment -v 5 -k return_false -v 1 -n step_out_of_me"
         else:
-            command = "target stop-hook add -G 1 -P stop_hook.stop_handler -k increment -v 5 -n step_out_of_me"
+            command = "target stop-hook add -I 0 -G 1 -P stop_hook.stop_handler -k increment -v 5 -n step_out_of_me"
 
         self.interp.HandleCommand(command, result)
         self.assertTrue(result.Succeeded(), "Set the target stop hook")
diff --git a/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py b/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py
index c7b184ea3146..1fa03c71e85a 100644
--- a/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py
+++ b/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py
@@ -175,11 +175,11 @@ def ignore_vrs_condition(self, use_location):
         if use_location:
             loc = bkpt.location[0]
             self.assertTrue(loc.IsValid(), "Got a valid location")
-            loc.SetIgnoreCount(2)
-            loc.SetCondition("i >= 3")
+            loc.ignore_count = 2
+            loc.condition = "i >= 3"
         else:
-            bkpt.SetIgnoreCount(2)
-            bkpt.SetCondition("i >= 3")
+            bkpt.ignore_count = 2
+            bkpt.condition = "i >= 3"
 
         threads = lldbutil.continue_to_breakpoint(process, bkpt)
         self.assertEqual(len(threads), 1, "Hit the breakpoint")
@@ -188,4 +188,4 @@ def ignore_vrs_condition(self, use_location):
         val = var.GetValueAsUnsigned(10000)
         self.assertNotEqual(val, 10000, "Got the fail value for i")
         self.assertEqual(val, 5, "We didn't stop the right number of times")
-        self.assertEqual(bkpt.GetHitCount(), 3, "Hit count is not right")
+        self.assertEqual(bkpt.hit_count, 3, "Hit count is not right")
diff --git a/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py b/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
index 0f9510c4507d..1d457415262b 100644
--- a/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
+++ b/external/llvm-project/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
@@ -216,16 +216,14 @@ def do_check_using_names(self):
         )
 
     def check_option_values(self, bp_object):
-        self.assertEqual(bp_object.IsOneShot(), self.is_one_shot, "IsOneShot")
-        self.assertEqual(bp_object.GetIgnoreCount(), self.ignore_count, "IgnoreCount")
-        self.assertEqual(bp_object.GetCondition(), self.condition, "Condition")
-        self.assertEqual(
-            bp_object.GetAutoContinue(), self.auto_continue, "AutoContinue"
-        )
-        self.assertEqual(bp_object.GetThreadID(), self.tid, "Thread ID")
-        self.assertEqual(bp_object.GetThreadIndex(), self.tidx, "Thread Index")
-        self.assertEqual(bp_object.GetThreadName(), self.thread_name, "Thread Name")
-        self.assertEqual(bp_object.GetQueueName(), self.queue_name, "Queue Name")
+        self.assertEqual(bp_object.one_shot, self.is_one_shot, "IsOneShot")
+        self.assertEqual(bp_object.ignore_count, self.ignore_count, "IgnoreCount")
+        self.assertEqual(bp_object.condition, self.condition, "Condition")
+        self.assertEqual(bp_object.auto_continue, self.auto_continue, "AutoContinue")
+        self.assertEqual(bp_object.thread_id, self.tid, "Thread ID")
+        self.assertEqual(bp_object.thread_index, self.tidx, "Thread Index")
+        self.assertEqual(bp_object.thread_name, self.thread_name, "Thread Name")
+        self.assertEqual(bp_object.queue_name, self.queue_name, "Queue Name")
         set_cmds = lldb.SBStringList()
         bp_object.GetCommandLineCommands(set_cmds)
         self.assertEqual(
diff --git a/external/llvm-project/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py b/external/llvm-project/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
index ecea28c6e1f6..351fae6b2767 100644
--- a/external/llvm-project/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
+++ b/external/llvm-project/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
@@ -91,6 +91,11 @@ def test_single_step(self):
         self.finish_test()
 
     @no_debug_info_test
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/138083",
+    )
     def test_single_step_thread_specific(self):
         """Test that single step stops, even though the second breakpoint is not valid."""
         self.prepare_test()
diff --git a/external/llvm-project/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/TestCoroutineHandle.py b/external/llvm-project/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/TestCoroutineHandle.py
index ae1a0c86b45d..54bb661057cd 100644
--- a/external/llvm-project/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/TestCoroutineHandle.py
+++ b/external/llvm-project/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/coroutine_handle/TestCoroutineHandle.py
@@ -18,8 +18,11 @@ def do_test(self, stdlib_type):
         self.build(dictionary={stdlib_type: "1"})
         is_clang = self.expectedCompiler(["clang"])
 
+        # Clang <= 20 used to also name the resume/destroy functions
+        # as `my_generator_func`.
+        # Never versions of clang name the clones as `.resume`/`.destroy`.
         test_generator_func_ptr_re = re.compile(
-            r"^\(a.out`my_generator_func\(\) at main.cpp:[0-9]*\)$"
+            r"^\(a.out`my_generator_func\(\)( \(\..*\))? at main.cpp:[0-9]*\)$"
         )
 
         # Run until the initial suspension point
@@ -43,11 +46,17 @@ def do_test(self, stdlib_type):
                         ValueCheck(name="current_value", value="-1"),
                     ],
                 ),
+                # We don not check any members inside the `coro_frame`,
+                # as its contents are highly compiler-specific.
+                ValueCheck(name="coro_frame"),
             ],
         )
+
+        # For a type-erased `coroutine_handle<>`, we can still devirtualize
+        # the promise call and display the correctly typed promise. This
+        # currently only works in clang, because gcc is not adding the
+        # artificial `__promise` variable to the destroy function.
         if is_clang:
-            # For a type-erased `coroutine_handle<>`, we can still devirtualize
-            # the promise call and display the correctly typed promise.
             self.expect_expr(
                 "type_erased_hdl",
                 result_summary=re.compile("^coro frame = 0x[0-9a-f]*$"),
@@ -60,23 +69,26 @@ def do_test(self, stdlib_type):
                             ValueCheck(name="current_value", value="-1"),
                         ],
                     ),
+                    ValueCheck(name="coro_frame"),
                 ],
             )
-            # For an incorrectly typed `coroutine_handle`, we use the user-supplied
-            # incorrect type instead of inferring the correct type. Strictly speaking,
-            # incorrectly typed coroutine handles are undefined behavior. However,
-            # it provides probably a better debugging experience if we display the
-            # promise as seen by the program instead of fixing this bug based on
-            # the available debug info.
-            self.expect_expr(
-                "incorrectly_typed_hdl",
-                result_summary=re.compile("^coro frame = 0x[0-9a-f]*$"),
-                result_children=[
-                    ValueCheck(name="resume", summary=test_generator_func_ptr_re),
-                    ValueCheck(name="destroy", summary=test_generator_func_ptr_re),
-                    ValueCheck(name="promise", dereference=ValueCheck(value="-1")),
-                ],
-            )
+
+        # For an incorrectly typed `coroutine_handle`, we use the user-supplied
+        # incorrect type instead of inferring the correct type. Strictly speaking,
+        # incorrectly typed coroutine handles are undefined behavior. However,
+        # it provides probably a better debugging experience if we display the
+        # promise as seen by the program instead of fixing this bug based on
+        # the available debug info.
+        self.expect_expr(
+            "incorrectly_typed_hdl",
+            result_summary=re.compile("^coro frame = 0x[0-9a-f]*$"),
+            result_children=[
+                ValueCheck(name="resume", summary=test_generator_func_ptr_re),
+                ValueCheck(name="destroy", summary=test_generator_func_ptr_re),
+                ValueCheck(name="promise", dereference=ValueCheck(value="-1")),
+                ValueCheck(name="coro_frame"),
+            ],
+        )
 
         process = self.process()
 
@@ -107,6 +119,7 @@ def do_test(self, stdlib_type):
                         ValueCheck(name="current_value", value="42"),
                     ],
                 ),
+                ValueCheck(name="coro_frame"),
             ],
         )
 
@@ -130,6 +143,7 @@ def do_test(self, stdlib_type):
                         ValueCheck(name="current_value", value="42"),
                     ],
                 ),
+                ValueCheck(name="coro_frame"),
             ],
         )
         if is_clang:
@@ -147,6 +161,7 @@ def do_test(self, stdlib_type):
                             ValueCheck(name="current_value", value="42"),
                         ],
                     ),
+                    ValueCheck(name="coro_frame"),
                 ],
             )
 
diff --git a/external/llvm-project/lldb/test/API/functionalities/memory/find/TestMemoryFind.py b/external/llvm-project/lldb/test/API/functionalities/memory/find/TestMemoryFind.py
index 09611cc80877..a06b0d960889 100644
--- a/external/llvm-project/lldb/test/API/functionalities/memory/find/TestMemoryFind.py
+++ b/external/llvm-project/lldb/test/API/functionalities/memory/find/TestMemoryFind.py
@@ -56,10 +56,23 @@ def test_memory_find(self):
         # Invalid expr is an error.
         self.expect(
             'memory find -e "not_a_symbol" `&bytedata[0]` `&bytedata[15]`',
+            substrs=[
+                "Expression evaluation failed:",
+                "use of undeclared identifier 'not_a_symbol'",
+            ],
             error=True,
-            substrs=["error: expression evaluation failed. pass a string instead"],
         )
 
+        self.expect(
+            'memory find -e "" `&bytedata[0]` `&bytedata[2]`',
+            substrs=[
+                "Expression evaluation failed:",
+                "No result returned from expression. Exit status: 1",
+            ],
+            error=True,
+        )
+
+        # Valid expressions/strings
         self.expect(
             'memory find -e "(uint8_t)0x22" `&bytedata[0]` `&bytedata[15]`',
             substrs=["data found at location: 0x", "22 33 44 55 66"],
@@ -79,3 +92,44 @@ def test_memory_find(self):
             'memory find -s "nothere" `stringdata` `stringdata+10`',
             substrs=["data not found within the range."],
         )
+
+        # Expression results with unsupported result types.
+        self.expect(
+            'memory find -e "ThreeBytes{}" `&bytedata[0]` `&bytedata[2]`',
+            substrs=[
+                "Only expressions resulting in 1, 2, 4, or 8-byte-sized values are supported"
+            ],
+            error=True,
+        )
+
+        self.expect(
+            'memory find -e "FiveBytes{}" `&bytedata[0]` `&bytedata[2]`',
+            substrs=[
+                "Only expressions resulting in 1, 2, 4, or 8-byte-sized values are supported"
+            ],
+            error=True,
+        )
+
+        self.expect(
+            'memory find -e "SixBytes{}" `&bytedata[0]` `&bytedata[2]`',
+            substrs=[
+                "Only expressions resulting in 1, 2, 4, or 8-byte-sized values are supported"
+            ],
+            error=True,
+        )
+
+        self.expect(
+            'memory find -e "SevenBytes{}" `&bytedata[0]` `&bytedata[2]`',
+            substrs=[
+                "Only expressions resulting in 1, 2, 4, or 8-byte-sized values are supported"
+            ],
+            error=True,
+        )
+
+        self.expect(
+            'memory find -e "NineBytes{}" `&bytedata[0]` `&bytedata[2]`',
+            substrs=[
+                "Only expressions resulting in 1, 2, 4, or 8-byte-sized values are supported"
+            ],
+            error=True,
+        )
diff --git a/external/llvm-project/lldb/test/API/functionalities/memory/find/main.cpp b/external/llvm-project/lldb/test/API/functionalities/memory/find/main.cpp
index e3dcfc762ee0..15c8df1a9fcf 100644
--- a/external/llvm-project/lldb/test/API/functionalities/memory/find/main.cpp
+++ b/external/llvm-project/lldb/test/API/functionalities/memory/find/main.cpp
@@ -1,9 +1,24 @@
 #include <stdio.h>
 #include <stdint.h>
 
+template <size_t T> struct [[gnu::packed]] Payload {
+  uint8_t data[T];
+};
+
+using ThreeBytes = Payload<3>;
+using FiveBytes = Payload<5>;
+using SixBytes = Payload<5>;
+using SevenBytes = Payload<7>;
+using NineBytes = Payload<9>;
+
 int main (int argc, char const *argv[])
 {
     const char* stringdata = "hello world; I like to write text in const char pointers";
     uint8_t bytedata[] = {0xAA,0xBB,0xCC,0xDD,0xEE,0xFF,0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99};
+    ThreeBytes b1;
+    FiveBytes b2;
+    SixBytes b3;
+    SevenBytes b4;
+    NineBytes b5;
     return 0; // break here
 }
diff --git a/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueBreakpoints.py b/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueBreakpoints.py
index 0a5f2d88fb91..53c8cdd4540a 100644
--- a/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueBreakpoints.py
+++ b/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueBreakpoints.py
@@ -63,10 +63,20 @@ def reverse_continue_breakpoint_internal(self, async_mode):
         self.assertEqual(threads_now, initial_threads)
 
     @skipIfRemote
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/138084",
+    )
     def test_reverse_continue_skip_breakpoint(self):
         self.reverse_continue_skip_breakpoint_internal(async_mode=False)
 
     @skipIfRemote
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/138084",
+    )
     def test_reverse_continue_skip_breakpoint_async(self):
         self.reverse_continue_skip_breakpoint_internal(async_mode=True)
 
diff --git a/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueWatchpoints.py b/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueWatchpoints.py
index a9e1bec5750e..38cf5fced494 100644
--- a/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueWatchpoints.py
+++ b/external/llvm-project/lldb/test/API/functionalities/reverse-execution/TestReverseContinueWatchpoints.py
@@ -64,12 +64,22 @@ def reverse_continue_watchpoint_internal(self, async_mode):
     @skipIfRemote
     # Watchpoints don't work in single-step mode
     @skipIf(macos_version=["<", "15.0"], archs=["arm64"])
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/138084",
+    )
     def test_reverse_continue_skip_watchpoint(self):
         self.reverse_continue_skip_watchpoint_internal(async_mode=False)
 
     @skipIfRemote
     # Watchpoints don't work in single-step mode
     @skipIf(macos_version=["<", "15.0"], archs=["arm64"])
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/138084",
+    )
     def test_reverse_continue_skip_watchpoint_async(self):
         self.reverse_continue_skip_watchpoint_internal(async_mode=True)
 
diff --git a/external/llvm-project/lldb/test/API/functionalities/statusline/TestStatusline.py b/external/llvm-project/lldb/test/API/functionalities/statusline/TestStatusline.py
index 53ac7432f4ba..68c5ad3090cf 100644
--- a/external/llvm-project/lldb/test/API/functionalities/statusline/TestStatusline.py
+++ b/external/llvm-project/lldb/test/API/functionalities/statusline/TestStatusline.py
@@ -44,7 +44,7 @@ def test(self):
         self.expect(
             "set set show-statusline true",
             [
-                "\x1b[0;{}r".format(self.TERMINAL_HEIGHT - 1),
+                "\x1b[1;{}r".format(self.TERMINAL_HEIGHT - 1),
                 "a.out | main.c:2:11 | breakpoint 1.1                        ",
             ],
         )
@@ -66,7 +66,7 @@ def test(self):
 
         # Hide the statusline and check or the control character.
         self.expect(
-            "set set show-statusline false", ["\x1b[0;{}r".format(self.TERMINAL_HEIGHT)]
+            "set set show-statusline false", ["\x1b[1;{}r".format(self.TERMINAL_HEIGHT)]
         )
 
     def test_no_color(self):
diff --git a/external/llvm-project/lldb/test/API/functionalities/stop-on-sharedlibrary-load/TestStopOnSharedlibraryEvents.py b/external/llvm-project/lldb/test/API/functionalities/stop-on-sharedlibrary-load/TestStopOnSharedlibraryEvents.py
index f68eb4196c12..74ad3b2070e1 100644
--- a/external/llvm-project/lldb/test/API/functionalities/stop-on-sharedlibrary-load/TestStopOnSharedlibraryEvents.py
+++ b/external/llvm-project/lldb/test/API/functionalities/stop-on-sharedlibrary-load/TestStopOnSharedlibraryEvents.py
@@ -17,7 +17,7 @@ def test_stopping_breakpoints(self):
     @no_debug_info_test
     def test_auto_continue(self):
         def auto_continue(bkpt):
-            bkpt.SetAutoContinue(True)
+            bkpt.auto_continue = True
 
         self.do_test(auto_continue)
 
@@ -26,7 +26,7 @@ def auto_continue(bkpt):
     @no_debug_info_test
     def test_failing_condition(self):
         def condition(bkpt):
-            bkpt.SetCondition("1 == 2")
+            bkpt.condition = "1 == 2"
 
         self.do_test(condition)
 
diff --git a/external/llvm-project/lldb/test/API/functionalities/thread/jump/TestThreadJump.py b/external/llvm-project/lldb/test/API/functionalities/thread/jump/TestThreadJump.py
index d603580ac6f3..715846c10f23 100644
--- a/external/llvm-project/lldb/test/API/functionalities/thread/jump/TestThreadJump.py
+++ b/external/llvm-project/lldb/test/API/functionalities/thread/jump/TestThreadJump.py
@@ -65,6 +65,7 @@ def test(self):
             substrs=["error"],
         )
 
+    @expectedFailureAll(compiler="clang", compiler_version=["<", "17.0"])
     def test_jump_offset(self):
         """Test Thread Jump by negative or positive offset"""
         exe = self.getBuildArtifact("a.out")
diff --git a/external/llvm-project/lldb/test/API/lang/c/forward/TestForwardDeclaration.py b/external/llvm-project/lldb/test/API/lang/c/forward/TestForwardDeclaration.py
index 5d05f255623f..b6b6f1d6b261 100644
--- a/external/llvm-project/lldb/test/API/lang/c/forward/TestForwardDeclaration.py
+++ b/external/llvm-project/lldb/test/API/lang/c/forward/TestForwardDeclaration.py
@@ -53,7 +53,6 @@ def test(self):
     @skipIfDarwin
     @skipIf(compiler=no_match("clang"))
     @skipIf(compiler_version=["<", "8.0"])
-    @expectedFailureAll(oslist=["windows"])
     def test_debug_names(self):
         """Test that we are able to find complete types when using DWARF v5
         accelerator tables"""
diff --git a/external/llvm-project/lldb/test/API/lang/cpp/decl-from-submodule/TestDeclFromSubmodule.py b/external/llvm-project/lldb/test/API/lang/cpp/decl-from-submodule/TestDeclFromSubmodule.py
index f200e51c7f79..d044ad46637f 100644
--- a/external/llvm-project/lldb/test/API/lang/cpp/decl-from-submodule/TestDeclFromSubmodule.py
+++ b/external/llvm-project/lldb/test/API/lang/cpp/decl-from-submodule/TestDeclFromSubmodule.py
@@ -11,17 +11,11 @@
 class DeclFromSubmoduleTestCase(TestBase):
     # Requires DWARF debug info which is not retained when linking with link.exe.
     @skipIfWindows
+    # Lookup for decls in submodules fails in Linux
+    @expectedFailureAll(oslist=["linux"])
     def test_expr(self):
         self.build()
         lldbutil.run_to_source_breakpoint(self, "return 0", lldb.SBFileSpec("main.cpp"))
 
-        # FIXME: LLDB finds the decl for 'func' in the submodules correctly and hands it to Clang
-        # but Sema rejects using the decl during name lookup because it is not marked "Visible".
-        # However, this assertions still ensures that we at least don't fail to compile the
-        # submodule (which would cause other errors to appear before the expression error, hence
-        # we use "startstr").
-        self.expect(
-            "expr func(1, 2)",
-            error=True,
-            startstr="error: <user expression 0>:1:1: 'func' has unknown return type",
-        )
+        self.expect_expr("func(1, 2)", result_type="int", result_value="3")
+        self.expect_expr("func(1)", result_type="int", result_value="1")
diff --git a/external/llvm-project/lldb/test/API/lang/cpp/forward/TestCPPForwardDeclaration.py b/external/llvm-project/lldb/test/API/lang/cpp/forward/TestCPPForwardDeclaration.py
index 5e9dd9c2227d..e66ab4fa7110 100644
--- a/external/llvm-project/lldb/test/API/lang/cpp/forward/TestCPPForwardDeclaration.py
+++ b/external/llvm-project/lldb/test/API/lang/cpp/forward/TestCPPForwardDeclaration.py
@@ -47,7 +47,6 @@ def test(self):
     @skipIfDarwin
     @skipIf(compiler=no_match("clang"))
     @skipIf(compiler_version=["<", "8.0"])
-    @expectedFailureAll(oslist=["windows"])
     def test_debug_names(self):
         """Test that we are able to find complete types when using DWARF v5
         accelerator tables"""
diff --git a/external/llvm-project/lldb/test/API/lang/objc/objc-po-hint/TestObjcPoHint.py b/external/llvm-project/lldb/test/API/lang/objc/objc-po-hint/TestObjcPoHint.py
index 3073cb07e053..ba922944f4ce 100644
--- a/external/llvm-project/lldb/test/API/lang/objc/objc-po-hint/TestObjcPoHint.py
+++ b/external/llvm-project/lldb/test/API/lang/objc/objc-po-hint/TestObjcPoHint.py
@@ -37,7 +37,7 @@ def test_show_po_hint_disabled(self):
             self, "Set breakpoint here", lldb.SBFileSpec("main.m")
         )
         self.runCmd("setting set show-dont-use-po-hint false")
-        # Make sure the hint is printed the first time
+        # Make sure the hint is not printed
         self.expect(
             "dwim-print -O -- foo",
             substrs=["note: object description"],
diff --git a/external/llvm-project/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py b/external/llvm-project/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
index faf2256b03a0..74ba0ee6c83b 100644
--- a/external/llvm-project/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
+++ b/external/llvm-project/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
@@ -39,15 +39,15 @@ def check_debugserver(self, log, expected_platform, expected_version):
         if expected_version:
             self.assertEqual(aout_info["min_version_os_sdk"], expected_version)
 
-    @skipIf(bugnumber="rdar://76995109")
     def run_with(self, arch, os, vers, env, expected_load_command):
         env_list = [env] if env else []
         triple = "-".join([arch, "apple", os + vers] + env_list)
         sdk = lldbutil.get_xcode_sdk(os, env)
 
-        version_min = ""
         if not vers:
             vers = lldbutil.get_xcode_sdk_version(sdk)
+
+        version_min = ""
         if env == "simulator":
             version_min = "-m{}-simulator-version-min={}".format(os, vers)
         elif os == "macosx":
@@ -56,11 +56,14 @@ def run_with(self, arch, os, vers, env, expected_load_command):
         sdk_root = lldbutil.get_xcode_sdk_root(sdk)
         clang = lldbutil.get_xcode_clang(sdk)
 
+        print(triple)
+
         self.build(
             dictionary={
                 "ARCH": arch,
                 "ARCH_CFLAGS": "-target {} {}".format(triple, version_min),
                 "SDKROOT": sdk_root,
+                "USE_SYSTEM_STDLIB": 1,
             },
             compiler=clang,
         )
@@ -146,6 +149,7 @@ def test_watchos_armv7k(self):
 
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
+    @skipIf(archs=["arm64", "arm64e"])
     def test_lc_version_min_macosx(self):
         """Test running a back-deploying non-simulator MacOS X binary"""
         self.run_with(
@@ -198,7 +202,7 @@ def test_ios_backdeploy_apple_silicon(self):
         self.run_with(
             arch=self.getArchitecture(),
             os="ios",
-            vers="11.0",
+            vers="14.0",
             env="simulator",
             expected_load_command="LC_BUILD_VERSION",
         )
@@ -229,7 +233,7 @@ def test_tvos_backdeploy_apple_silicon(self):
         self.run_with(
             arch=self.getArchitecture(),
             os="tvos",
-            vers="11.0",
+            vers="14.0",
             env="simulator",
             expected_load_command="LC_BUILD_VERSION",
         )
diff --git a/external/llvm-project/lldb/test/API/python_api/value/TestValueAPI.py b/external/llvm-project/lldb/test/API/python_api/value/TestValueAPI.py
index 9eaf2c994d84..0da57346212d 100644
--- a/external/llvm-project/lldb/test/API/python_api/value/TestValueAPI.py
+++ b/external/llvm-project/lldb/test/API/python_api/value/TestValueAPI.py
@@ -269,7 +269,14 @@ def test(self):
             frame0.FindVariable("another_fixed_int_ptr").GetValue(),
             "0xaa",
         )
+        a_null_int_ptr = frame0.FindVariable("a_null_int_ptr")
+        self.assertEqual(a_null_int_ptr.GetValue(), "0x0")
+
+        # Check that dereferencing a null pointer produces reasonable results
+        # (does not crash).
+        self.assertEqual(
+            a_null_int_ptr.Dereference().GetError().GetCString(), "parent is NULL"
+        )
         self.assertEqual(
-            frame0.FindVariable("a_null_int_ptr").GetValue(),
-            "0x0",
+            a_null_int_ptr.Dereference().GetLoadAddress(), lldb.LLDB_INVALID_ADDRESS
         )
diff --git a/external/llvm-project/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py b/external/llvm-project/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
index 7a0e42a4fc27..37fa911b3714 100644
--- a/external/llvm-project/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
+++ b/external/llvm-project/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
@@ -160,6 +160,11 @@ def test_watch_address(self):
     # No size constraint on MIPS for watches
     @skipIf(archs=["mips", "mipsel", "mips64", "mips64el"])
     @skipIf(archs=["s390x"])  # Likewise on SystemZ
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/142196",
+    )
     def test_watch_address_with_invalid_watch_size(self):
         """Exercise SBTarget.WatchpointCreateByAddress() API but pass an invalid watch_size."""
         self.build()
diff --git a/external/llvm-project/lldb/test/API/tools/lldb-dap/attach/TestDAP_attachByPortNum.py b/external/llvm-project/lldb/test/API/tools/lldb-dap/attach/TestDAP_attachByPortNum.py
index 7c2b540195d1..edb87a9314d7 100644
--- a/external/llvm-project/lldb/test/API/tools/lldb-dap/attach/TestDAP_attachByPortNum.py
+++ b/external/llvm-project/lldb/test/API/tools/lldb-dap/attach/TestDAP_attachByPortNum.py
@@ -2,26 +2,16 @@
 Test lldb-dap "port" configuration to "attach" request
 """
 
-import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
 from lldbsuite.test import lldbplatformutil
 from lldbgdbserverutils import Pipe
 import lldbdap_testcase
-import os
-import shutil
-import subprocess
-import tempfile
-import threading
-import sys
-import socket
+import lldb
 
 
-@skip("https://github.com/llvm/llvm-project/issues/138803")
+@skip(bugnumber="https://github.com/llvm/llvm-project/issues/138803")
 class TestDAP_attachByPortNum(lldbdap_testcase.DAPTestCaseBase):
-    default_timeout = 20
-
     def set_and_hit_breakpoint(self, continueToExit=True):
         self.dap_server.wait_for_stopped()
 
@@ -50,7 +40,7 @@ def get_debug_server_command_line_args(self):
     def get_debug_server_pipe(self):
         pipe = Pipe(self.getBuildDir())
         self.addTearDownHook(lambda: pipe.close())
-        pipe.finish_connection(self.default_timeout)
+        pipe.finish_connection(self.DEFAULT_TIMEOUT)
         return pipe
 
     @skipIfWindows
@@ -73,28 +63,33 @@ def test_by_port(self):
         )
 
         # Read the port number from the debug server pipe.
-        port = pipe.read(10, self.default_timeout)
+        port = pipe.read(10, self.DEFAULT_TIMEOUT)
         # Trim null byte, convert to int
         port = int(port[:-1])
         self.assertIsNotNone(
             port, " Failed to read the port number from debug server pipe"
         )
 
-        self.attach(program=program, gdbRemotePort=port, sourceInitFile=True)
+        self.attach(
+            program=program,
+            gdbRemotePort=port,
+            sourceInitFile=True,
+            stopOnEntry=True,
+        )
         self.set_and_hit_breakpoint(continueToExit=True)
-        self.process.terminate()
 
     @skipIfWindows
     @skipIfNetBSD
-    def test_by_port_and_pid(self):
+    def test_fails_if_both_port_and_pid_are_set(self):
         """
         Tests attaching to a process by process ID and port number.
         """
         program = self.build_and_create_debug_adapter_for_attach()
 
-        # It is not necessary to launch "lldb-server" to obtain the actual port and pid for attaching.
-        # However, when providing the port number and pid directly, "lldb-dap" throws an error message, which is expected.
-        # So, used random pid and port numbers here.
+        # It is not necessary to launch "lldb-server" to obtain the actual port
+        # and pid for attaching. However, when providing the port number and pid
+        # directly, "lldb-dap" throws an error message, which is expected. So,
+        # used random pid and port numbers here.
 
         pid = 1354
         port = 1234
@@ -106,10 +101,9 @@ def test_by_port_and_pid(self):
             sourceInitFile=True,
             expectFailure=True,
         )
-        if not (response and response["success"]):
-            self.assertFalse(
-                response["success"], "The user can't specify both pid and port"
-            )
+        self.assertFalse(
+            response["success"], "The user can't specify both pid and port"
+        )
 
     @skipIfWindows
     @skipIfNetBSD
@@ -123,11 +117,10 @@ def test_by_invalid_port(self):
         response = self.attach(
             program=program, gdbRemotePort=port, sourceInitFile=True, expectFailure=True
         )
-        if not (response and response["success"]):
-            self.assertFalse(
-                response["success"],
-                "The user can't attach with invalid port (%s)" % port,
-            )
+        self.assertFalse(
+            response["success"],
+            "The user can't attach with invalid port (%s)" % port,
+        )
 
     @skipIfWindows
     @skipIfNetBSD
@@ -147,9 +140,7 @@ def test_by_illegal_port(self):
         response = self.attach(
             program=program, gdbRemotePort=port, sourceInitFile=True, expectFailure=True
         )
-        if not (response and response["success"]):
-            self.assertFalse(
-                response["success"],
-                "The user can't attach with illegal port (%s)" % port,
-            )
-        self.process.terminate()
+        self.assertFalse(
+            response["success"],
+            "The user can't attach with illegal port (%s)" % port,
+        )
diff --git a/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/TestDAP_disassemble.py b/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/TestDAP_disassemble.py
index a8b51864d118..0562f20335a2 100644
--- a/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/TestDAP_disassemble.py
+++ b/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/TestDAP_disassemble.py
@@ -2,13 +2,9 @@
 Test lldb-dap disassemble request
 """
 
-
-import dap_server
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
+from lldbsuite.test.decorators import skipIfWindows
+from lldbsuite.test.lldbtest import line_number
 import lldbdap_testcase
-import os
 
 
 class TestDAP_disassemble(lldbdap_testcase.DAPTestCaseBase):
@@ -20,18 +16,35 @@ def test_disassemble(self):
         program = self.getBuildArtifact("a.out")
         self.build_and_launch(program)
         source = "main.c"
-        self.set_source_breakpoints(source, [line_number(source, "// breakpoint 1")])
+        bp_line_no = line_number(source, "// breakpoint 1")
+        self.set_source_breakpoints(source, [bp_line_no])
         self.continue_to_next_stop()
 
-        _, pc_assembly = self.disassemble(frameIndex=0)
-        self.assertIn("location", pc_assembly, "Source location missing.")
-        self.assertIn("instruction", pc_assembly, "Assembly instruction missing.")
+        insts_with_bp, pc_with_bp_assembly = self.disassemble(frameIndex=0)
+        self.assertIn("location", pc_with_bp_assembly, "Source location missing.")
+        self.assertEqual(
+            pc_with_bp_assembly["line"], bp_line_no, "Expects the same line number"
+        )
+        no_bp = self.set_source_breakpoints(source, [])
+        self.assertEqual(len(no_bp), 0, "Expects no breakpoints.")
+        self.assertIn(
+            "instruction", pc_with_bp_assembly, "Assembly instruction missing."
+        )
+
+        insts_no_bp, pc_no_bp_assembly = self.disassemble(frameIndex=0)
+        self.assertIn("location", pc_no_bp_assembly, "Source location missing.")
+        self.assertEqual(
+            pc_with_bp_assembly["line"], bp_line_no, "Expects the same line number"
+        )
+        # the disassembly instructions should be the same with breakpoint and no breakpoint;
+        self.assertDictEqual(
+            insts_with_bp,
+            insts_no_bp,
+            "Expects instructions are the same after removing breakpoints.",
+        )
+        self.assertIn("instruction", pc_no_bp_assembly, "Assembly instruction missing.")
 
-        # The calling frame (qsort) is coming from a system library, as a result
-        # we should not have a source location.
-        _, qsort_assembly = self.disassemble(frameIndex=1)
-        self.assertNotIn("location", qsort_assembly, "Source location not expected.")
-        self.assertIn("instruction", pc_assembly, "Assembly instruction missing.")
+        self.continue_to_exit()
 
     @skipIfWindows
     def test_disassemble_backwards(self):
@@ -74,3 +87,7 @@ def test_disassemble_backwards(self):
             backwards_instructions,
             f"requested instruction should be preceeded by {backwards_instructions} instructions. Actual index: {frame_instruction_index}",
         )
+
+        # clear breakpoints
+        self.set_source_breakpoints(source, [])
+        self.continue_to_exit()
diff --git a/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/main.c b/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/main.c
index 9da119ef7026..b4927db10e94 100644
--- a/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/main.c
+++ b/external/llvm-project/lldb/test/API/tools/lldb-dap/disassemble/main.c
@@ -6,9 +6,7 @@ int compare_ints(const void *a, const void *b) {
   int arg1 = *(const int *)a;
   int arg2 = *(const int *)b;
 
-  // breakpoint 1
-
-  if (arg1 < arg2)
+  if (arg1 < arg2) // breakpoint 1
     return -1;
   if (arg1 > arg2)
     return 1;
diff --git a/external/llvm-project/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py b/external/llvm-project/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
index c24df9a9bfcf..ae8142ae4f48 100644
--- a/external/llvm-project/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
+++ b/external/llvm-project/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
@@ -566,7 +566,7 @@ def test_version(self):
         )
         version_eval_output = version_eval_response["body"]["result"]
 
-        version_string = self.dap_server.get_initialize_value("$__lldb_version")
+        version_string = self.dap_server.get_capability("$__lldb_version")
         self.assertEqual(
             version_eval_output.splitlines(),
             version_string.splitlines(),
diff --git a/external/llvm-project/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py b/external/llvm-project/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
index 7a9a4f434e04..ed17044a220d 100644
--- a/external/llvm-project/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
+++ b/external/llvm-project/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
@@ -102,7 +102,7 @@ def test_server_interrupt(self):
         process.send_signal(signal.SIGINT)
 
         self.dap_server.wait_for_terminated()
-        self.assertIsNone(
+        self.assertIsNotNone(
             self.dap_server.exit_status,
             "Process exited before interrupting lldb-dap server",
         )
diff --git a/external/llvm-project/lldb/test/Shell/Commands/command-image-dump-ast.test b/external/llvm-project/lldb/test/Shell/Commands/command-image-dump-ast.test
new file mode 100644
index 000000000000..3204022418cb
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Commands/command-image-dump-ast.test
@@ -0,0 +1,74 @@
+# Test `image dump ast` command.
+
+# DWARF is required to properly create the AST, and on Windows that is discarded
+# and we only have PDB which doesn't work for this test.
+# UNSUPPORTED: system-windows
+
+# RUN: split-file %s %t
+# RUN: %clang_host -g -gdwarf %t/main.cpp -o %t.out
+# RUN: %lldb -x -b -s %t/commands.input %t.out -o exit 2>&1 \
+# RUN:       | FileCheck %s
+
+#--- main.cpp
+
+void A() {}
+void A1() {}
+void BA1() {}
+void AB() {}
+
+int main() {
+  A();
+  A1();
+  BA1();
+  AB();
+}
+
+#--- commands.input
+
+break set -n main
+run
+expr A(); A1(); BA1(); AB()
+
+image dump ast
+
+# CHECK: image dump ast
+# CHECK-DAG: FunctionDecl {{.*}} main
+# CHECK-DAG: FunctionDecl {{.*}} A
+# CHECK-DAG: FunctionDecl {{.*}} A1
+# CHECK-DAG: FunctionDecl {{.*}} BA1
+# CHECK-DAG: FunctionDecl {{.*}} AB
+
+image dump ast --filter A
+
+# CHECK: image dump ast --filter A
+# CHECK: Dumping A
+# CHECK-NOT: FunctionDecl {{.*}} main
+# CHECK-DAG: FunctionDecl {{.*}} A1
+# CHECK-DAG: FunctionDecl {{.*}} BA1
+# CHECK-DAG: FunctionDecl {{.*}} AB
+
+image dump ast --filter A1
+
+# CHECK: image dump ast --filter A1
+# CHECK: Dumping A
+# CHECK-NOT: FunctionDecl {{.*}} main
+# CHECK-NOT: FunctionDecl {{.*}} AB
+# CHECK-DAG: FunctionDecl {{.*}} A1
+# CHECK-DAG: FunctionDecl {{.*}} BA1
+
+image dump ast --filter ""
+
+# CHECK: image dump ast --filter ""
+# CHECK-DAG: FunctionDecl {{.*}} main
+# CHECK-DAG: FunctionDecl {{.*}} AB
+# CHECK-DAG: FunctionDecl {{.*}} A1
+# CHECK-DAG: FunctionDecl {{.*}} BA1
+
+image dump ast -f AB
+
+# CHECK: image dump ast -f AB
+# CHECK: Dumping AB
+# CHECK-NOT: FunctionDecl {{.*}} main
+# CHECK-NOT: FunctionDecl {{.*}} A1
+# CHECK-NOT: FunctionDecl {{.*}} BA1
+# CHECK: FunctionDecl {{.*}} AB
diff --git a/external/llvm-project/lldb/test/Shell/Commands/command-plugin-enable+disable.test b/external/llvm-project/lldb/test/Shell/Commands/command-plugin-enable+disable.test
new file mode 100644
index 000000000000..af646e60abef
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Commands/command-plugin-enable+disable.test
@@ -0,0 +1,87 @@
+# This test validates the plugin enable and disable commands.
+# Currently it works only for system-runtime plugins and we only have one
+# system runtime plugin so testing is a bit limited.
+#
+# Note that commands that return errors will stop running a script, so we
+# have new RUN lines for any command that is expected to return an error.
+
+# RUN: %lldb -s %s -o exit 2>&1 | FileCheck %s
+
+# Test plugin list shows the default state which is expected to be enabled.
+plugin list
+# CHECK-LABEL: plugin list
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Test plugin disable disables a plugin.
+plugin disable system-runtime.systemruntime-macosx
+# CHECK-LABEL: plugin disable system-runtime.systemruntime-macosx
+# CHECK: system-runtime
+# CHECK:  [-] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Make sure plugin list shows it disabled as well.
+plugin list
+# CHECK: system-runtime
+# CHECK:  [-] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Test plugin enable re-enables a plugin.
+plugin enable system-runtime.systemruntime-macosx
+# CHECK-LABEL: plugin enable system-runtime.systemruntime-macosx
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Make sure plugin list shows it enabled as well.
+plugin list
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Test plugin disable with namespace works.
+plugin disable system-runtime
+# CHECK-LABEL: plugin disable system-runtime
+# CHECK: system-runtime
+# CHECK:  [-] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Test plugin enable with namespace works.
+plugin enable system-runtime
+# CHECK-LABEL: plugin enable system-runtime
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Test plugin enable/disable for instrumentation plugin works.
+plugin enable instrumentation-runtime
+# CHECK-LABEL: plugin enable instrumentation-runtime
+# CHECK: instrumentation-runtime
+# CHECK:  [+] AddressSanitizer
+plugin disable instrumentation-runtime
+# CHECK-LABEL: plugin disable instrumentation-runtime
+# CHECK: instrumentation-runtime
+# CHECK:  [-] AddressSanitizer
+
+# Test plugin enable with multiple arguments.
+plugin enable system-runtime instrumentation-runtime
+# CHECK-LABEL: plugin enable system-runtime instrumentation-runtime
+# CHECK: system-runtime
+# CHECK:   [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries.
+# CHECK: instrumentation-runtime
+# CHECK:   [+] AddressSanitizer               AddressSanitizer instrumentation runtime plugin.
+
+# Test plugin disable with multiple arguments.
+plugin disable system-runtime instrumentation-runtime
+# CHECK-LABEL: plugin disable system-runtime instrumentation-runtime
+# CHECK: system-runtime
+# CHECK:   [-] systemruntime-macosx           System runtime plugin for Mac OS X native libraries.
+# CHECK: instrumentation-runtime
+# CHECK:   [-] AddressSanitizer               AddressSanitizer instrumentation runtime plugin.
+
+# Test plugin enable/disable for unknown plugin returns an error.
+# RUN: %lldb -o "plugin enable some-plugin-that-does-not-exist" 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+# RUN: %lldb -o "plugin disable some-plugin-that-does-not-exist" 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+# RUN: %lldb -o "plugin enable system-runtime some-plugin-that-does-not-exist" 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+# ERROR_PLUGIN_NOT_FOUND: error: Found no matching plugins
+
+# Test plugin enable/disable requires a plugin name.
+# RUN: %lldb -o "plugin enable" 2>&1 | FileCheck %s --check-prefix=ERROR_ARGUMENTS_ENABLE
+# ERROR_ARGUMENTS_ENABLE: error: 'plugin enable' requires one or more arguments
+
+# RUN: %lldb -o "plugin disable" 2>&1 | FileCheck %s --check-prefix=ERROR_ARGUMENTS_DISABLE
+# ERROR_ARGUMENTS_DISABLE: error: 'plugin disable' requires one or more arguments
diff --git a/external/llvm-project/lldb/test/Shell/Commands/command-plugin-list.test b/external/llvm-project/lldb/test/Shell/Commands/command-plugin-list.test
new file mode 100644
index 000000000000..9d3680d48cdd
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Commands/command-plugin-list.test
@@ -0,0 +1,82 @@
+# This test validates the plugin list command.
+# Currently it works only for system-runtime plugins and we only have one
+# system runtime plugin so testing is a bit limited.
+#
+# Note that commands that return errors will stop running a script, so we
+# have new RUN lines for any command that is expected to return an error.
+
+# RUN: %lldb -s %s -o exit 2>&1 | FileCheck %s
+
+# Test plugin list without an argument will list all plugins.
+plugin list
+# CHECK-LABEL: plugin list
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+# CHECK: instrumentation-runtime
+# CHECK:  [+] AddressSanitizer               AddressSanitizer instrumentation runtime plugin.
+
+# Test plugin list works with fully qualified name.
+plugin list system-runtime.systemruntime-macosx
+# CHECK-LABEL: plugin list system-runtime.systemruntime-macosx
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Test plugin list on plugin namespace works.
+plugin list system-runtime
+# CHECK-LABEL: plugin list system-runtime
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+
+# Test plugin list on multiple args works.
+plugin list system-runtime instrumentation-runtime.AddressSanitizer
+# CHECK-LABEL: plugin list system-runtime instrumentation-runtime.AddressSanitizer
+# CHECK: system-runtime
+# CHECK:  [+] systemruntime-macosx           System runtime plugin for Mac OS X native libraries
+# CHECK: instrumentation-runtime
+# CHECK:  [+] AddressSanitizer               AddressSanitizer instrumentation runtime plugin.
+
+# Test json output for plugin list.
+plugin list --json
+# CHECK-LABEL plugin list --json
+# CHECK: {
+# CHECK-DAG:  "instrumentation-runtime":
+# CHECK-DAG:  "system-runtime":
+# CHECK: }
+
+# Test json output for plugin list with a namespace
+plugin list system-runtime --json
+# CHECK-LABEL plugin list --json
+# CHECK: {
+# CHECK:  "system-runtime": [
+# CHECK:    {
+# CHECK-DAG:  "enabled": true
+# CHECK-DAG:  "name": "systemruntime-macosx"
+# CHECK:    }
+# CHECK:  ]
+# CHECK: }
+
+# Test json output for listing multiple plugins
+plugin list --json system-runtime instrumentation-runtime.AddressSanitizer
+# CHECK-LABEL plugin list --json system-runtime instrumentation-runtime.AddressSanitizer
+# CHECK: {
+# CHECK-DAG:  "instrumentation-runtime":
+# CHECK-DAG:    "name": "AddressSanitizer"
+# CHECK-DAG:  "system-runtime":
+# CHECK: }
+
+
+# Test plugin list does not match a plugin name by substring.
+# RUN: %lldb -o "plugin list macosx" 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+
+# Test plugin list does not match a plugin namespace by substring.
+# RUN: %lldb -o "plugin list system-runtime." 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+
+# Test plugin list returns an error for unknown second argument
+# RUN: %lldb -o "plugin list system-runtime foo" 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+
+# Test plugin list returns an error for unknown second argument
+# RUN: %lldb -o "plugin list --json system-runtime foo" 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+
+# Test plugin list for unknown plugin returns an error.
+# RUN: %lldb -o "plugin list some-plugin-that-does-not-exist" 2>&1 | FileCheck %s --check-prefix=ERROR_PLUGIN_NOT_FOUND
+# ERROR_PLUGIN_NOT_FOUND: error: Found no matching plugins
diff --git a/external/llvm-project/lldb/test/Shell/Commands/command-process-save-core-not-a-plugin.test b/external/llvm-project/lldb/test/Shell/Commands/command-process-save-core-not-a-plugin.test
new file mode 100644
index 000000000000..c034c8ebbf87
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Commands/command-process-save-core-not-a-plugin.test
@@ -0,0 +1,19 @@
+# This checks that lldb returns an error if process save-core is called
+# with a plugin that does not exist.
+
+# RUN: %clang_host -g %S/Inputs/main.c -o %t
+# RUN: %lldb %t -s %s -o exit 2>&1 | FileCheck %s
+
+b main
+# CHECK-LABEL: b main
+# CHECK: Breakpoint 1: where = {{.*}}`main
+
+run
+# CHECK-LABEL: run
+# CHECK: Process {{.*}} stopped
+# CHECK: stop reason = breakpoint 1
+# CHECK:   frame #0: {{.*}}`main at main.c
+
+process save-core --plugin-name=notaplugin dump
+# CHECK-LABEL: process save-core --plugin-name=notaplugin dump
+# CHECK: error: plugin name 'notaplugin' is not a valid ObjectFile plugin name
diff --git a/external/llvm-project/lldb/test/Shell/Driver/TestWaitFor.test b/external/llvm-project/lldb/test/Shell/Driver/TestWaitFor.test
new file mode 100644
index 000000000000..dde8e747713a
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Driver/TestWaitFor.test
@@ -0,0 +1,2 @@
+# RUN: not %lldb --wait-for 2>&1 | FileCheck %s
+# CHECK: error: --wait-for requires a name (--attach-name)
diff --git a/external/llvm-project/lldb/test/Shell/Expr/TestClangModulesDeclLookup.test b/external/llvm-project/lldb/test/Shell/Expr/TestClangModulesDeclLookup.test
new file mode 100644
index 000000000000..a5eb4d626e52
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Expr/TestClangModulesDeclLookup.test
@@ -0,0 +1,62 @@
+# XFAIL: target-windows
+# XFAIL: system-linux
+# XFAIL: system-windows && remote-linux
+
+# Test that we can successfully locate decls in Clang modules for C++.
+
+# RUN: split-file %s %t
+# RUN: %clang_host -g -gdwarf %t/main.cpp -fmodules -fcxx-modules -o %t.out
+# RUN: %lldb -o "settings set interpreter.stop-command-source-on-error false" \
+# RUN:       -x -b -s %t/commands.input %t.out 2>&1 \
+# RUN:       | FileCheck %s
+
+#--- main.cpp
+
+#include "Module.h"
+
+int main() {
+  foo(10);
+  return 0;
+}
+
+#--- module.modulemap
+
+module Module {
+  header "Module.h"
+  export *
+}
+
+#--- Module.h
+
+// We use nodebug here ensures that LLDB tries to pick the decl out of the module.
+// If debug-info is available, it would use that to construct the decl instead.
+[[gnu::nodebug]] int foo(int x) { return x; }
+
+int bar(int x, int y) { return x + y; }
+
+#--- commands.input
+
+breakpoint set -n foo
+run
+
+expression foo(5)
+
+# FIXME: when we're stopped in a frame without debug-info, the ClangModulesDeclVendor
+# is initialized properly and none of the modules in the CU are compiled (and lookup
+# in the DeclVendor is not enabled).
+# CHECK: expression foo(5)
+# CHECK: error: 'foo' has unknown return type; cast the call to its declared return type
+
+breakpoint set -p return -X main
+continue
+expression foo(50)
+
+# However, once we're back in a frame with debug-info, the ClangModulesDeclVendor infrastructure
+# is back on track.
+
+# CHECK:      expression foo(50)
+# CHECK-NEXT: (int) $0 = 5
+
+target modules dump ast --filter foo
+# CHECK:     (lldb) target modules dump ast --filter foo
+# CHECK-NOT: foo
diff --git a/external/llvm-project/lldb/test/Shell/Expr/TestObjCxxEnumConflict.test b/external/llvm-project/lldb/test/Shell/Expr/TestObjCxxEnumConflict.test
new file mode 100644
index 000000000000..cf1f2c67e388
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Expr/TestObjCxxEnumConflict.test
@@ -0,0 +1,54 @@
+# REQUIRES: system-darwin
+
+# Tests whether the expression evaluator can handle the `NS_ENUM`/`NS_OPTIONS`
+# typedefs from Objective-C `CoreFoundation`.
+# Here `module.h` mimicks the `NS_OPTIONS` typedef from `CoreFoundation`.
+# 
+# The `ClangModulesDeclVendor` currently compiles modules as
+# C++, so the `MyInt` Clang decl in the module will be a `TypedefType`,
+# while the DWARF AST parser will produce an `EnumType` (since that's what
+# the debug-info says). When the `ASTImporter` imports these decls into the
+# scratch AST, it will fail to re-use one or the other decl because they
+# aren't structurally equivalent (one is a typedef, the other an enum),
+# so we end up with two conflicting `MyInt` declarations in the scratch AST
+# and the expression fails to run due to ambiguity in name lookup.
+
+# RUN: split-file %s %t
+# RUN: %clang_host -g %t/main.m -fmodules -fcxx-modules -o %t.out
+# RUN: %lldb -x -b -s %t/commands.input %t.out -o exit 2>&1 \
+# RUN:       | FileCheck %s
+
+#--- main.m
+
+#import "module.h"
+
+int main() {
+    MyInt i;
+    return i;
+}
+
+#--- module.h
+
+#if (__cplusplus)
+typedef int MyInt;
+enum : MyInt { CASE };
+#else
+typedef enum MyInt : int MyInt;
+enum MyInt : int { CASE };
+#endif
+
+#--- module.modulemap
+module Int {
+    header "module.h"
+}
+
+#--- commands.input
+
+break set -n main
+run
+expression -l objective-c -- (MyInt)5
+
+# CHECK: error: reference to 'MyInt' is ambiguous 
+# CHECK: error: reference to 'MyInt' is ambiguous 
+# CHECK: note: note: candidate found by name lookup is 'MyInt' 
+# CHECK: note: note: candidate found by name lookup is 'MyInt' 
diff --git a/external/llvm-project/lldb/test/Shell/ObjectFile/PECOFF/dwarf-clang.yaml b/external/llvm-project/lldb/test/Shell/ObjectFile/PECOFF/dwarf-clang.yaml
new file mode 100644
index 000000000000..1adc6152fb90
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/ObjectFile/PECOFF/dwarf-clang.yaml
@@ -0,0 +1,151 @@
+# Test that LLDB can read executables with DWARF sections generated by Clang
+
+# RUN: yaml2obj %s -o %t
+# RUN: lldb-test object-file %t | FileCheck %s
+
+# CHECK: Name: .debug_abbrev
+# CHECK-NEXT: Type: dwarf-abbrev
+
+# CHECK: Name: .debug_addr
+# CHECK-NEXT: Type: dwarf-addr
+
+# CHECK: Name: .debug_aranges
+# CHECK-NEXT: Type: dwarf-aranges
+
+# CHECK: Name: .debug_info
+# CHECK-NEXT: Type: dwarf-info
+
+# CHECK: Name: .debug_line
+# CHECK-NEXT: Type: dwarf-line
+
+# CHECK: Name: .debug_line_str
+# CHECK-NEXT: Type: dwarf-line-str
+
+# CHECK: Name: .debug_rnglists
+# CHECK-NEXT: Type: dwarf-rnglists
+
+# CHECK: Name: .debug_str
+# CHECK-NEXT: Type: dwarf-str
+
+# CHECK: Name: .debug_str_offsets
+# CHECK-NEXT: Type: dwarf-str-offsets
+
+--- !COFF
+OptionalHeader:
+  AddressOfEntryPoint: 4956
+  ImageBase:       5368709120
+  SectionAlignment: 4096
+  FileAlignment:   512
+  MajorOperatingSystemVersion: 6
+  MinorOperatingSystemVersion: 0
+  MajorImageVersion: 0
+  MinorImageVersion: 0
+  MajorSubsystemVersion: 6
+  MinorSubsystemVersion: 0
+  Subsystem:       IMAGE_SUBSYSTEM_WINDOWS_CUI
+  DLLCharacteristics: [ IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA, IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE, IMAGE_DLL_CHARACTERISTICS_NX_COMPAT, IMAGE_DLL_CHARACTERISTICS_TERMINAL_SERVER_AWARE ]
+  SizeOfStackReserve: 1048576
+  SizeOfStackCommit: 4096
+  SizeOfHeapReserve: 1048576
+  SizeOfHeapCommit: 4096
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [ IMAGE_FILE_EXECUTABLE_IMAGE, IMAGE_FILE_LARGE_ADDRESS_AWARE ]
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  4096
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .rdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  401408
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .data
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
+    VirtualAddress:  479232
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .pdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  491520
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .fptable
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
+    VirtualAddress:  512000
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            _RDATA
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  516096
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            _guard_c
+    Characteristics: [  ]
+    VirtualAddress:  520192
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            _guard_d
+    Characteristics: [  ]
+    VirtualAddress:  524288
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            memcpy_
+    Characteristics: [  ]
+    VirtualAddress:  528384
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .reloc
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  532480
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_abbrev
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  536576
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_addr
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  540672
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_aranges
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  544768
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_info
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  548864
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_line
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  552960
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_line_str
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  557056
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_rnglists
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  561152
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_str
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  565248
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_str_offsets
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  569344
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+symbols:         []
+...
diff --git a/external/llvm-project/lldb/test/Shell/ObjectFile/PECOFF/dwarf-gcc-mingw.yaml b/external/llvm-project/lldb/test/Shell/ObjectFile/PECOFF/dwarf-gcc-mingw.yaml
new file mode 100644
index 000000000000..7f6563783746
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/ObjectFile/PECOFF/dwarf-gcc-mingw.yaml
@@ -0,0 +1,151 @@
+# Test that LLDB can read executables with DWARF sections generated by GCC on MinGW
+
+# RUN: yaml2obj %s -o %t
+# RUN: lldb-test object-file %t | FileCheck %s
+
+# CHECK: Name: .debug_aranges
+# CHECK-NEXT: Type: dwarf-aranges
+
+# CHECK: Name: .debug_info
+# CHECK-NEXT: Type: dwarf-info
+
+# CHECK: Name: .debug_abbrev
+# CHECK-NEXT: Type: dwarf-abbrev
+
+# CHECK: Name: .debug_line
+# CHECK-NEXT: Type: dwarf-line
+
+# CHECK: Name: .debug_frame
+# CHECK-NEXT: Type: dwarf-frame
+
+# CHECK: Name: .debug_str
+# CHECK-NEXT: Type: dwarf-str
+
+# CHECK: Name: .debug_line_str
+# CHECK-NEXT: Type: dwarf-line-str
+
+# CHECK: Name: .debug_loclists
+# CHECK-NEXT: Type: dwarf-loclists
+
+# CHECK: Name: .debug_rnglists
+# CHECK-NEXT: Type: dwarf-rnglists
+
+--- !COFF
+OptionalHeader:
+  AddressOfEntryPoint: 5136
+  ImageBase:       5368709120
+  SectionAlignment: 4096
+  FileAlignment:   512
+  MajorOperatingSystemVersion: 4
+  MinorOperatingSystemVersion: 0
+  MajorImageVersion: 0
+  MinorImageVersion: 0
+  MajorSubsystemVersion: 5
+  MinorSubsystemVersion: 2
+  Subsystem:       IMAGE_SUBSYSTEM_WINDOWS_CUI
+  DLLCharacteristics: [ IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA, IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE, IMAGE_DLL_CHARACTERISTICS_NX_COMPAT ]
+  SizeOfStackReserve: 2097152
+  SizeOfStackCommit: 4096
+  SizeOfHeapReserve: 1048576
+  SizeOfHeapCommit: 4096
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [ IMAGE_FILE_EXECUTABLE_IMAGE, IMAGE_FILE_LINE_NUMS_STRIPPED, IMAGE_FILE_LARGE_ADDRESS_AWARE ]
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  4096
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .data
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
+    VirtualAddress:  12288
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .rdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  16384
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .pdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  20480
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .xdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  24576
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .bss
+    Characteristics: [ IMAGE_SCN_CNT_UNINITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
+    VirtualAddress:  28672
+    VirtualSize:     384
+    SectionData:     ''
+  - Name:            .idata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  32768
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .tls
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
+    VirtualAddress:  36864
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .rsrc
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  40960
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .reloc
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  45056
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_aranges
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  49152
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_info
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  53248
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_abbrev
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  98304
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_line
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  106496
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_frame
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  114688
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_str
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  118784
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_line_str
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  122880
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_loclists
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  131072
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+  - Name:            .debug_rnglists
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  139264
+    VirtualSize:     64
+    SectionData:     DEADBEEFBAADF00D
+symbols:         []
+...
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBDefines.test b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBDefines.test
new file mode 100644
index 000000000000..0d89d627cfed
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBDefines.test
@@ -0,0 +1,22 @@
+RUN: mkdir -p %t/Outputs
+
+# Run the convert script on lldb-defines.h.
+RUN: %python %p/../../../../../scripts/convert-lldb-header-to-rpc-header.py %p/Inputs/lldb-defines.h %t/Outputs/lldb-rpc-defines.h
+
+# Check the output
+RUN: cat %t/Outputs/lldb-rpc-defines.h | FileCheck %s
+
+# The include guards must change from LLDB_LLDB_DEFINES_H to LLDB_RPC_DEFINES_H.
+CHECK: #ifndef LLDB_RPC_DEFINES_H
+CHECK: #define LLDB_RPC_DEFINES_H
+
+# Includes of other lldb headers must begin with "lldb-rpc-".
+CHECK: #include "lldb-rpc-types.h"
+
+# The version info must be changed from LLDB_VERSION to LLDB_RPC_VERSION
+CHECK: #define LLDB_RPC_VERSION 21
+CHECK: #define LLDB_RPC_REVISION 12
+CHECK: #define LLDB_RPC_VERSION_STRING "21.0.12"
+
+# The comment that closes the include guard should match the guard.
+CHECK: #endif // LLDB_RPC_DEFINES_H
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBEnumerations.test b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBEnumerations.test
new file mode 100644
index 000000000000..0fb3c6f73dd0
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBEnumerations.test
@@ -0,0 +1,17 @@
+RUN: mkdir -p %t/Outputs
+
+# Run the convert script on lldb-enumerations.h.
+RUN: %python %p/../../../../../scripts/convert-lldb-header-to-rpc-header.py %p/Inputs/lldb-enumerations.h %t/Outputs/lldb-rpc-enumerations.h
+
+# Check the output
+RUN: cat %t/Outputs/lldb-rpc-enumerations.h | FileCheck %s
+
+# The include guards must change from LLDB_LLDB_ENUMERATIONS_H to LLDB_RPC_ENUMERATIONS_H.
+CHECK: #ifndef LLDB_RPC_ENUMERATIONS_H
+CHECK: #define LLDB_RPC_ENUMERATIONS_H
+
+# Change the namespace to lldb_rpc. Also, the comment that closes the namespace should match the namespace.
+CHECK: namespace lldb_rpc {} // namespace lldb_rpc
+
+# The comment that closes the include guard should match the guard.
+CHECK: #endif // LLDB_RPC_ENUMERATIONS_H
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBTypes.test b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBTypes.test
new file mode 100644
index 000000000000..86f2d290209e
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckLLDBTypes.test
@@ -0,0 +1,24 @@
+RUN: mkdir -p %t/Outputs
+
+# Run the convert script on lldb-types.h.
+RUN: %python %p/../../../../../scripts/convert-lldb-header-to-rpc-header.py %p/Inputs/lldb-types.h %t/Outputs/lldb-rpc-types.h
+
+# Check the output
+RUN: cat %t/Outputs/lldb-rpc-types.h | FileCheck %s
+
+# The include guards must change from LLDB_LLDB_TYPES_H to LLDB_RPC_TYPES_H.
+CHECK: #ifndef LLDB_RPC_TYPES_H
+CHECK: #define LLDB_RPC_TYPES_H
+
+# Includes of other lldb headers must begin with "lldb-rpc-".
+# Also, the includes for lldb-forward.h should be removed.
+CHECK: #include "lldb-rpc-enumerations.h"
+
+# Change the namespace to lldb_rpc.
+CHECK: namespace lldb_rpc
+
+# The comment that closes the namespace should match the namespace.
+CHECK: // namespace lldb_rpc
+
+# The comment that closes the include guard should match the guard.
+CHECK: #endif // LLDB_RPC_TYPES_H
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckSBDefines.test b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckSBDefines.test
new file mode 100644
index 000000000000..72444aaf069a
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/CheckSBDefines.test
@@ -0,0 +1,22 @@
+RUN: mkdir -p %t/Outputs
+
+# Run the convert script on SBDefines.h.
+RUN: %python %p/../../../../../scripts/convert-lldb-header-to-rpc-header.py %p/Inputs/SBDefines.h %t/Outputs/SBDefines.h
+
+# Check the output
+RUN: cat %t/Outputs/SBDefines.h | FileCheck %s
+
+# The include guards must change from LLDB_LLDB_API_SBDEFINES_H to LLDB_RPC_API_SBDEFINES_H.
+CHECK: #ifndef LLDB_RPC_API_SBDEFINES_H
+CHECK: #define LLDB_RPC_API_SBDEFINES_H
+
+# Includes of other lldb headers must begin with "lldb-rpc-".
+# Also, the includes for lldb-forward.h and lldb-versioning.h should be removed.
+CHECK: #include "lldb-rpc-defines.h"
+CHECK-NOT: #include "lldb-rpc-forward.h"
+CHECK: #include "lldb-rpc-enumerations.h"
+CHECK: #include "lldb-rpc-types.h"
+CHECK-NOT: #include "lldb-rpc-versioning.h"
+
+# The comment that closes the include guard should match the guard.
+CHECK: #endif // LLDB_RPC_API_SBDEFINES_H
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/SBDefines.h b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/SBDefines.h
new file mode 100644
index 000000000000..50476c402ba7
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/SBDefines.h
@@ -0,0 +1,22 @@
+// This is a truncated version of SBDefines.h used to test that the script
+// convert-lldb-header-to-rpc-header.py works correctly. The script changes LLDB references in
+// the original file to RPC references.
+
+// The include guard should change from LLDB_LLDB to LLDB_RPC.
+// LLDB_API_SBDEFINES_H -> LLDB_RPC_SBDEFINES_H
+#ifndef LLDB_API_SBDEFINES_H
+#define LLDB_API_SBDEFINES_H
+
+// Includes of public main LLDB headers should change to their RPC equivalents:
+// "lldb/lldb-defines.h" -> "lldb-rpc-defines.h"
+// Also, the includes for lldb-forward.h and lldb-versioning.h should be removed.
+#include "lldb/lldb-defines.h"
+#include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-forward.h"
+#include "lldb/lldb-types.h"
+#include "lldb/lldb-versioning.h"
+
+// The comment that closes the include guard must change in the same way
+// the original guard did.
+// #endif // LLDB_API_SBDEFINES_H -> #endif // LLDB_RPC_API_SBDEFINES_H
+#endif // LLDB_API_SBDEFINES_H
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-defines.h b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-defines.h
new file mode 100644
index 000000000000..32064430b3d0
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-defines.h
@@ -0,0 +1,23 @@
+// This is a truncated version of lldb-defines.h used to test that the script
+// convert-lldb-header-to-rpc-header.py works correctly. The script changes LLDB references in
+// the original file to RPC references.
+
+// The include guard should change from LLDB_LLDB to LLDB_RPC.
+// LLDB_LLDB_DEFINES_H -> LLDB_RPC_DEFINES_H
+#ifndef LLDB_LLDB_DEFINES_H
+#define LLDB_LLDB_DEFINES_H
+
+// Includes of public main LLDB headers should change to their RPC equivalents:
+// "lldb/lldb-types.h" -> "lldb-rpc-types.h"
+#include "lldb/lldb-types.h"
+
+// The LLDB version must change from LLDB to LLDB_RPC
+// LLDB_VERSION -> LLDB_RPC_VERSION
+#define LLDB_VERSION 21
+#define LLDB_REVISION 12
+#define LLDB_VERSION_STRING "21.0.12"
+
+// The comment that closes the include guard must change in the same way
+// the original guard did.
+// #endif // LLDB_LLDB_DEFINES_H -> #endif // LLDB_RPC_DEFINES_H
+#endif // LLDB_LLDB_DEFINES_H
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-enumerations.h b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-enumerations.h
new file mode 100644
index 000000000000..42c4bb277fc4
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-enumerations.h
@@ -0,0 +1,17 @@
+// This is a truncated version of lldb-enumerations.h used to test that the script
+// convert-lldb-header-to-rpc-header.py works correctly. The script changes LLDB references in
+// the original file to RPC references.
+
+// The include guard should change from LLDB_LLDB to LLDB_RPC.
+// LLDB_LLDB_ENUMERATIONS_H -> LLDB_RPC_ENUMERATIONS_H
+#ifndef LLDB_LLDB_ENUMERATIONS_H
+#define LLDB_LLDB_ENUMERATIONS_H
+
+// The namespace definition should change to the lldb_rpc namespace, so should the comment that closes it:
+// namespace lldb -> namespace lldb_rpc
+namespace lldb {} // namespace lldb
+
+// The comment that closes the include guard must change in the same way
+// the original guard did:
+// #endif // LLDB_LLDB_ENUMERATIONS_H -> #endif // LLDB_RPC_ENUMERATIONS_H
+#endif // LLDB_LLDB_ENUMERATIONS_H
diff --git a/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-types.h b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-types.h
new file mode 100644
index 000000000000..5a49920405ec
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/RPC/Scripts/TestConvertScript/Inputs/lldb-types.h
@@ -0,0 +1,23 @@
+// This is a truncated version of lldb-types.h used to test that the script
+// convert-lldb-header-to-rpc-header.py works correctly. The script changes LLDB references in
+// the original file to RPC references.
+
+// The include guard should change from LLDB_LLDB to LLDB_RPC.
+// LLDB_LLDB_TYPES_H -> LLDB_RPC_TYPES_H
+#ifndef LLDB_LLDB_TYPES_H
+#define LLDB_LLDB_TYPES_H
+
+// Includes of public main LLDB headers should change to their RPC equivalents:
+// "lldb/lldb-defines.h" -> "lldb-rpc-defines.h":
+// Also, the includes for lldb-forward.h should be removed.
+#include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-forward.h"
+
+// The namespace definition should change to the lldb_rpc namespace, so should the comment that closes it:
+// namespace lldb -> namespace lldb_rpc
+namespace lldb {} // namespace lldb
+
+// The comment that closes the include guard must change in the same way
+// the original guard did:
+// #endif // LLDB_LLDB_TYPES_H -> #endif // LLDB_RPC_TYPES_H
+#endif // LLDB_LLDB_TYPES_H
diff --git a/external/llvm-project/lldb/test/Shell/Scripts/Inputs/Main/SBAddress.h b/external/llvm-project/lldb/test/Shell/Scripts/Inputs/Main/SBAddress.h
new file mode 100644
index 000000000000..fecc69687cd7
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Scripts/Inputs/Main/SBAddress.h
@@ -0,0 +1,13 @@
+// This is a truncated version of an SB API file
+// used to test framework-header-fix.py to make sure the includes are correctly fixed
+// up for the LLDB.framework.
+
+// Local includes must be changed to framework level includes.
+// e.g. #include "lldb/API/SBDefines.h" -> #include <LLDB/SBDefines.h>
+#include "lldb/API/SBDefines.h"
+#include "lldb/API/SBModule.h"
+
+// Any include guards specified at the command line must be removed.
+#ifndef SWIG
+int a = 10
+#endif
diff --git a/external/llvm-project/lldb/test/Shell/Scripts/Inputs/RPC/RPCSBAddress.h b/external/llvm-project/lldb/test/Shell/Scripts/Inputs/RPC/RPCSBAddress.h
new file mode 100644
index 000000000000..556afa38a922
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Scripts/Inputs/RPC/RPCSBAddress.h
@@ -0,0 +1,9 @@
+// This is a truncated version of an SB API file generated by lldb-rpc-gen
+// used to test framework-header-fix.py to make sure the includes are correctly fixed
+// up for the LLDBRPC.framework.
+
+// Local includes must be changed to framework level includes.
+// e.g. #include "lldb/API/SBDefines.h" -> #include <LLDB/SBDefines.h>
+#include "LLDBRPC.h"
+#include "SBDefines.h"
+#include <lldb-rpc/common/RPCPublic.h>
diff --git a/external/llvm-project/lldb/test/Shell/Scripts/Inputs/lldb-defines.h b/external/llvm-project/lldb/test/Shell/Scripts/Inputs/lldb-defines.h
new file mode 100644
index 000000000000..8d3090e26d26
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Scripts/Inputs/lldb-defines.h
@@ -0,0 +1,7 @@
+// This is a truncated version of lldb-defines.h used to test the script
+// that fixes up its versioning info.
+
+// The script needs to uncomment these lines and populate the info for versioning.
+// #define LLDB_VERSION
+// #define LLDB_REVISION
+// #define LLDB_VERSION_STRING
diff --git a/external/llvm-project/lldb/test/Shell/Scripts/TestFrameworkFixScript.test b/external/llvm-project/lldb/test/Shell/Scripts/TestFrameworkFixScript.test
new file mode 100644
index 000000000000..e90c3bdfc5ad
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Scripts/TestFrameworkFixScript.test
@@ -0,0 +1,11 @@
+# Create a temp dir for output and run the framework fix script on the truncated version of SBAddress.h in the inputs dir.
+RUN: mkdir -p %t/Outputs
+RUN: %python %p/../../../scripts/framework-header-fix.py -f lldb_main -i %p/Inputs/Main/SBAddress.h -o %t/Outputs/SBAddress.h -p /usr/bin/unifdef USWIG
+
+# Check the output
+RUN: cat %t/Outputs/SBAddress.h | FileCheck %s
+
+# Local includes must be changed to framework level includes.
+# e.g. #include "lldb/API/SBDefines.h" -> #include <LLDB/SBDefines.h>
+CHECK: #include <LLDB/SBDefines.h>
+CHECK: #include <LLDB/SBModule.h>
diff --git a/external/llvm-project/lldb/test/Shell/Scripts/TestFrameworkFixUnifdef.test b/external/llvm-project/lldb/test/Shell/Scripts/TestFrameworkFixUnifdef.test
new file mode 100644
index 000000000000..a7e82d2f3640
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Scripts/TestFrameworkFixUnifdef.test
@@ -0,0 +1,12 @@
+# REQUIRES: system-darwin
+# Create a temp dir for output and run the framework fix script on the truncated version of SBAddress.h in the inputs dir.
+RUN: mkdir -p %t/Outputs
+RUN: %python %p/../../../scripts/framework-header-fix.py -f lldb_main -i %p/Inputs/Main/SBAddress.h -o %t/Outputs/SBAddress.h -p /usr/bin/unifdef USWIG
+
+# Check the output
+RUN: cat %t/Outputs/SBAddress.h | FileCheck %s
+
+# Any include guards specified at the command line must be removed.
+CHECK-NOT: #ifndef SWIG
+CHECK: int a = 10
+CHECK-NOT: #endif
diff --git a/external/llvm-project/lldb/test/Shell/Scripts/TestRPCFrameworkFixScript.test b/external/llvm-project/lldb/test/Shell/Scripts/TestRPCFrameworkFixScript.test
new file mode 100644
index 000000000000..d01594265396
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Scripts/TestRPCFrameworkFixScript.test
@@ -0,0 +1,14 @@
+# Create a temp dir for output and run the framework fix script on the truncated version of SBAddress.h in the inputs dir.
+RUN: mkdir -p %t/Outputs
+RUN: %python %p/../../../scripts/framework-header-fix.py -f lldb_rpc -i %p/Inputs/RPC/RPCSBAddress.h -o %t/Outputs/RPCSBAddress.h -p /usr/bin/unifdef USWIG
+
+# Check the output
+RUN: cat %t/Outputs/RPCSBAddress.h | FileCheck %s
+
+# Local includes must be changed to RPC framework level includes.
+# e.g. #include "SBDefines.h" -> #include <LLDBRPC/SBDefines.h>
+# Also, RPC common code includes must change to RPC framework level includes.
+# e.g. #include "lldb-rpc/common/RPCPublic.h" -> #include <LLDBRPC/RPCPublic.h>
+CHECK: #include <LLDBRPC/LLDBRPC.h>
+CHECK: #include <LLDBRPC/SBDefines.h>
+CHECK: #include <LLDBRPC/RPCPublic.h>
diff --git a/external/llvm-project/lldb/test/Shell/Scripts/TestVersionFixScript.test b/external/llvm-project/lldb/test/Shell/Scripts/TestVersionFixScript.test
new file mode 100644
index 000000000000..78cc98726307
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Scripts/TestVersionFixScript.test
@@ -0,0 +1,11 @@
+# Create a temp dir for output and run the version fix script on the truncated version of lldb-defines.h in the inputs dir.
+RUN: mkdir -p %t/Outputs
+RUN: %python %p/../../../scripts/version-header-fix.py %p/Inputs/lldb-defines.h %t/Outputs/lldb-defines.h 21 0 12
+
+# Check the output
+RUN: cat %t/Outputs/lldb-defines.h | FileCheck %s
+
+# The LLDB version defines must be uncommented and filled in with the values passed into the script.
+CHECK: {{^}}#define LLDB_VERSION 21
+CHECK: {{^}}#define LLDB_REVISION 12
+CHECK: {{^}}#define LLDB_VERSION_STRING "21.0.12"
diff --git a/external/llvm-project/lldb/test/Shell/Settings/TestCxxFrameFormatRecursive.test b/external/llvm-project/lldb/test/Shell/Settings/TestCxxFrameFormatRecursive.test
index 887de882886e..465b1bb28e32 100644
--- a/external/llvm-project/lldb/test/Shell/Settings/TestCxxFrameFormatRecursive.test
+++ b/external/llvm-project/lldb/test/Shell/Settings/TestCxxFrameFormatRecursive.test
@@ -1,3 +1,5 @@
+# Flaky on Linux, see https://github.com/llvm/llvm-project/issues/142726
+# UNSUPPORTED: system-linux
 # XFAIL: *
 
 # Test disallowed variables inside the
diff --git a/external/llvm-project/lldb/test/Shell/Settings/TestFrameFormatName.test b/external/llvm-project/lldb/test/Shell/Settings/TestFrameFormatName.test
index 34eb2f497531..8da08a3a8287 100644
--- a/external/llvm-project/lldb/test/Shell/Settings/TestFrameFormatName.test
+++ b/external/llvm-project/lldb/test/Shell/Settings/TestFrameFormatName.test
@@ -1,4 +1,8 @@
+# Not run on Windows because we are not able to break on all the required functions.
+# Even if we could, function.name* formatting also does not work correctly:
+# https://github.com/llvm/llvm-project/issues/143149
 # UNSUPPORTED: system-windows
+
 # Test different name formats.
 
 # RUN: %build %S/Inputs/names.cpp --std c++17 -o %t.out
diff --git a/external/llvm-project/lldb/test/Shell/Settings/TestFrameFunctionInlined.test b/external/llvm-project/lldb/test/Shell/Settings/TestFrameFunctionInlined.test
new file mode 100644
index 000000000000..5db34b416085
--- /dev/null
+++ b/external/llvm-project/lldb/test/Shell/Settings/TestFrameFunctionInlined.test
@@ -0,0 +1,39 @@
+# Test the ${function.is-inlined} frame-format variable.
+
+# Windows' link.exe does not preserve DWARF information, and we cannot get
+# inlining information from PDB at this time, see:
+# https://github.com/llvm/llvm-project/issues/143104
+# REQUIRES: (system-windows && lld) || !system-windows
+
+# RUN: split-file %s %t
+# RUN: %clang_host -g -gdwarf %t/main.cpp -o %t.out %if system-windows %{-fuse-ld=lld%}
+# RUN: %lldb -x -b -s %t/commands.input %t.out -o exit 2>&1 \
+# RUN:       | FileCheck %s
+
+#--- main.cpp
+
+void regular();
+
+[[clang::always_inline]] void inlined1() {
+  regular();
+}
+void regular() {inlined1();}
+[[clang::always_inline]] void inlined2() {regular();}
+
+int main() {
+  inlined2();
+  return 0;
+}
+
+#--- commands.input
+
+settings set frame-format "frame '${function.name}{${function.is-inlined} (Inlined)}'\n"
+breakpoint set -n inlined1
+run
+bt
+
+# CHECK:      (lldb) bt
+# CHECK:      frame '{{.*}}inlined1({{.*}}) (Inlined)'
+# CHECK-NEXT: frame '{{.*}}regular({{.*}})'
+# CHECK-NEXT: frame '{{.*}}inlined2({{.*}}) (Inlined)'
+# CHECK-NEXT: frame 'main'
diff --git a/external/llvm-project/lldb/test/Shell/lit.cfg.py b/external/llvm-project/lldb/test/Shell/lit.cfg.py
index 193639667db5..ab6113767187 100644
--- a/external/llvm-project/lldb/test/Shell/lit.cfg.py
+++ b/external/llvm-project/lldb/test/Shell/lit.cfg.py
@@ -198,3 +198,8 @@ def calculate_arch_features(arch_string):
             config.available_features.add("ld_new-bug")
     except:
         pass
+
+# Some shell tests dynamically link with python.dll and need to know the
+# location of the Python libraries. This ensures that we use the same
+# version of Python that was used to build lldb to run our tests.
+config.environment["PYTHONHOME"] = config.python_root_dir
diff --git a/external/llvm-project/lldb/test/Shell/lit.site.cfg.py.in b/external/llvm-project/lldb/test/Shell/lit.site.cfg.py.in
index 7e03938b12b2..5be535921776 100644
--- a/external/llvm-project/lldb/test/Shell/lit.site.cfg.py.in
+++ b/external/llvm-project/lldb/test/Shell/lit.site.cfg.py.in
@@ -21,6 +21,7 @@ config.enable_remote = not @LLDB_TEST_SHELL_DISABLE_REMOTE@
 config.libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
+config.python_root_dir = "@Python3_ROOT_DIR@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.objc_gnustep_dir = "@LLDB_TEST_OBJC_GNUSTEP_DIR@"
 config.lldb_enable_lzma = @LLDB_ENABLE_LZMA@
diff --git a/external/llvm-project/lldb/test/Unit/lit.cfg.py b/external/llvm-project/lldb/test/Unit/lit.cfg.py
index 8d711f17d858..681e3b19dce3 100644
--- a/external/llvm-project/lldb/test/Unit/lit.cfg.py
+++ b/external/llvm-project/lldb/test/Unit/lit.cfg.py
@@ -33,6 +33,7 @@
     ]
 )
 llvm_config.with_environment("PATH", os.path.dirname(sys.executable), append_path=True)
+config.environment["PYTHONHOME"] = config.python_root_dir
 
 # Enable sanitizer runtime flags.
 if config.llvm_use_sanitizer:
diff --git a/external/llvm-project/lldb/test/Unit/lit.site.cfg.py.in b/external/llvm-project/lldb/test/Unit/lit.site.cfg.py.in
index 2748be229cf1..fb94797bcd7d 100644
--- a/external/llvm-project/lldb/test/Unit/lit.site.cfg.py.in
+++ b/external/llvm-project/lldb/test/Unit/lit.site.cfg.py.in
@@ -11,6 +11,7 @@ config.lldb_src_root = "@LLDB_SOURCE_DIR@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.python_executable = "@Python3_EXECUTABLE@"
+config.python_root_dir = "@Python3_ROOT_DIR@"
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
diff --git a/external/llvm-project/lldb/tools/debugserver/source/CMakeLists.txt b/external/llvm-project/lldb/tools/debugserver/source/CMakeLists.txt
index 8340b5ad8948..a81fe5505cc3 100644
--- a/external/llvm-project/lldb/tools/debugserver/source/CMakeLists.txt
+++ b/external/llvm-project/lldb/tools/debugserver/source/CMakeLists.txt
@@ -265,8 +265,8 @@ target_link_libraries(lldbDebugserverCommon
                       ${ENERGY_LIBRARY})
 add_lldb_tool(debugserver ADD_TO_FRAMEWORK
   debugserver.cpp
-  LINK_LIBS lldbDebugserverCommon
   ENTITLEMENTS ${entitlements}
+  LINK_LIBS lldbDebugserverCommon
 )
 
 # Workaround for Xcode-specific code-signing behavior:
@@ -329,10 +329,9 @@ if(APPLE_EMBEDDED)
   add_lldb_tool(debugserver-nonui
     debugserver.cpp
 
-    LINK_LIBS
-      lldbDebugserverCommon_NonUI
-
     ENTITLEMENTS
       ${entitlements}
+    LINK_LIBS
+      lldbDebugserverCommon_NonUI
     )
 endif()
diff --git a/external/llvm-project/lldb/tools/debugserver/source/RNBRemote.cpp b/external/llvm-project/lldb/tools/debugserver/source/RNBRemote.cpp
index af3c66c71c77..391d1c50168e 100644
--- a/external/llvm-project/lldb/tools/debugserver/source/RNBRemote.cpp
+++ b/external/llvm-project/lldb/tools/debugserver/source/RNBRemote.cpp
@@ -3476,7 +3476,7 @@ static bool GetProcessNameFrom_vAttach(const char *&p,
 }
 
 rnb_err_t RNBRemote::HandlePacket_qSupported(const char *p) {
-  uint32_t max_packet_size = 128 * 1024; // 128KBytes is a reasonable max packet
+  uint32_t max_packet_size = 128 * 1024; // 128 KiB is a reasonable max packet
                                          // size--debugger can always use less
   std::stringstream reply;
   reply << "qXfer:features:read+;PacketSize=" << std::hex << max_packet_size
diff --git a/external/llvm-project/lldb/tools/driver/CMakeLists.txt b/external/llvm-project/lldb/tools/driver/CMakeLists.txt
index 8b4aa92f96f0..5c0cac484cc9 100644
--- a/external/llvm-project/lldb/tools/driver/CMakeLists.txt
+++ b/external/llvm-project/lldb/tools/driver/CMakeLists.txt
@@ -20,14 +20,13 @@ add_lldb_tool(lldb
   Driver.cpp
   Platform.cpp
 
+  LINK_COMPONENTS
+    Option
+    Support
   LINK_LIBS
     liblldb
     lldbHost
     lldbUtility
-
-  LINK_COMPONENTS
-    Option
-    Support
   )
 
 add_dependencies(lldb
diff --git a/external/llvm-project/lldb/tools/driver/Driver.cpp b/external/llvm-project/lldb/tools/driver/Driver.cpp
index e19fded05194..16cc736441b5 100644
--- a/external/llvm-project/lldb/tools/driver/Driver.cpp
+++ b/external/llvm-project/lldb/tools/driver/Driver.cpp
@@ -280,6 +280,12 @@ SBError Driver::ProcessArgs(const opt::InputArgList &args, bool &exiting) {
   }
 
   if (args.hasArg(OPT_wait_for)) {
+    if (!args.hasArg(OPT_attach_name)) {
+      error.SetErrorStringWithFormat(
+          "--wait-for requires a name (--attach-name)");
+      return error;
+    }
+
     m_option_data.m_wait_for = true;
   }
 
diff --git a/external/llvm-project/lldb/tools/driver/Options.td b/external/llvm-project/lldb/tools/driver/Options.td
index a24fb3826b90..1d8372c4aa40 100644
--- a/external/llvm-project/lldb/tools/driver/Options.td
+++ b/external/llvm-project/lldb/tools/driver/Options.td
@@ -19,9 +19,11 @@ def: Separate<["-"], "n">,
   HelpText<"Alias for --attach-name">,
   Group<grp_attach>;
 
-def wait_for: F<"wait-for">,
-  HelpText<"Tells the debugger to wait for a process with the given pid or name to launch before attaching.">,
-  Group<grp_attach>;
+def wait_for
+    : F<"wait-for">,
+      HelpText<"Tells the debugger to wait for the process with the name "
+               "specified by --attach-name to launch before attaching.">,
+      Group<grp_attach>;
 def: Flag<["-"], "w">,
   Alias<wait_for>,
   HelpText<"Alias for --wait-for">,
diff --git a/external/llvm-project/lldb/tools/intel-features/intel-mpx/CMakeLists.txt b/external/llvm-project/lldb/tools/intel-features/intel-mpx/CMakeLists.txt
index ca02ac84c846..701380499b8b 100644
--- a/external/llvm-project/lldb/tools/intel-features/intel-mpx/CMakeLists.txt
+++ b/external/llvm-project/lldb/tools/intel-features/intel-mpx/CMakeLists.txt
@@ -1,10 +1,9 @@
 add_lldb_library(lldbIntelMPX
   cli-wrapper-mpxtable.cpp
 
-  LINK_LIBS
-    liblldb
-
   LINK_COMPONENTS
     Support
     TargetParser
+  LINK_LIBS
+    liblldb
   )
diff --git a/external/llvm-project/lldb/tools/lldb-dap/CMakeLists.txt b/external/llvm-project/lldb/tools/lldb-dap/CMakeLists.txt
index 40cba16c141f..8bbb402fdf78 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/CMakeLists.txt
+++ b/external/llvm-project/lldb/tools/lldb-dap/CMakeLists.txt
@@ -67,16 +67,16 @@ add_lldb_library(lldbDAP
   Handler/VariablesRequestHandler.cpp
 
   Protocol/ProtocolBase.cpp
+  Protocol/ProtocolEvents.cpp
   Protocol/ProtocolTypes.cpp
   Protocol/ProtocolRequests.cpp
 
-  LINK_LIBS
-    liblldb
-    lldbHost
-
   LINK_COMPONENTS
     Option
     Support
+  LINK_LIBS
+    liblldb
+    lldbHost
   )
 
 target_include_directories(lldbDAP
diff --git a/external/llvm-project/lldb/tools/lldb-dap/DAP.cpp b/external/llvm-project/lldb/tools/lldb-dap/DAP.cpp
index 394e8f0e6c85..9fe8227cd2d6 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/DAP.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/DAP.cpp
@@ -70,6 +70,7 @@
 
 using namespace lldb_dap;
 using namespace lldb_dap::protocol;
+using namespace lldb_private;
 
 namespace {
 #ifdef _WIN32
@@ -820,7 +821,7 @@ void DAP::SendTerminatedEvent() {
   });
 }
 
-llvm::Error DAP::Disconnect() { return Disconnect(is_attach); }
+llvm::Error DAP::Disconnect() { return Disconnect(!is_attach); }
 
 llvm::Error DAP::Disconnect(bool terminateDebuggee) {
   lldb::SBError error;
@@ -893,14 +894,14 @@ llvm::Error DAP::Loop() {
 
         while (!disconnecting) {
           llvm::Expected<Message> next =
-              transport.Read(std::chrono::seconds(1));
-          if (next.errorIsA<EndOfFileError>()) {
+              transport.Read<protocol::Message>(std::chrono::seconds(1));
+          if (next.errorIsA<TransportEOFError>()) {
             consumeError(next.takeError());
             break;
           }
 
           // If the read timed out, continue to check if we should disconnect.
-          if (next.errorIsA<TimeoutError>()) {
+          if (next.errorIsA<TransportTimeoutError>()) {
             consumeError(next.takeError());
             continue;
           }
@@ -1240,7 +1241,10 @@ void DAP::EventThread() {
             // automatically restarted.
             if (!lldb::SBProcess::GetRestartedFromEvent(event)) {
               SendStdOutStdErr(*this, process);
-              SendThreadStoppedEvent(*this);
+              if (llvm::Error err = SendThreadStoppedEvent(*this))
+                DAP_LOG_ERROR(log, std::move(err),
+                              "({1}) reporting thread stopped: {0}",
+                              transport.GetClientName());
             }
             break;
           case lldb::eStateRunning:
diff --git a/external/llvm-project/lldb/tools/lldb-dap/DAP.h b/external/llvm-project/lldb/tools/lldb-dap/DAP.h
index 1bd94fab402c..89bc827c1141 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/DAP.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/DAP.h
@@ -152,7 +152,7 @@ struct DAP {
   llvm::DenseSet<ClientFeature> clientFeatures;
 
   /// The initial thread list upon attaching.
-  std::optional<llvm::json::Array> initial_thread_list;
+  std::vector<protocol::Thread> initial_thread_list;
 
   /// Keep track of all the modules our client knows about: either through the
   /// modules request or the module events.
diff --git a/external/llvm-project/lldb/tools/lldb-dap/EventHelper.cpp b/external/llvm-project/lldb/tools/lldb-dap/EventHelper.cpp
index c698084836e2..9641f29698b1 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/EventHelper.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/EventHelper.cpp
@@ -8,10 +8,13 @@
 
 #include "EventHelper.h"
 #include "DAP.h"
-#include "DAPLog.h"
+#include "DAPError.h"
 #include "JSONUtils.h"
 #include "LLDBUtils.h"
+#include "Protocol/ProtocolEvents.h"
+#include "Protocol/ProtocolTypes.h"
 #include "lldb/API/SBFileSpec.h"
+#include "llvm/Support/Error.h"
 
 #if defined(_WIN32)
 #define NOMINMAX
@@ -22,6 +25,8 @@
 #endif
 #endif
 
+using namespace llvm;
+
 namespace lldb_dap {
 
 static void SendThreadExitedEvent(DAP &dap, lldb::tid_t tid) {
@@ -33,6 +38,22 @@ static void SendThreadExitedEvent(DAP &dap, lldb::tid_t tid) {
   dap.SendJSON(llvm::json::Value(std::move(event)));
 }
 
+void SendTargetBasedCapabilities(DAP &dap) {
+  if (!dap.target.IsValid())
+    return;
+
+  protocol::CapabilitiesEventBody body;
+
+  // We only support restarting launch requests not attach requests.
+  if (dap.last_launch_request)
+    body.capabilities.supportedFeatures.insert(
+        protocol::eAdapterFeatureRestartRequest);
+
+  // Only notify the client if supportedFeatures changed.
+  if (!body.capabilities.supportedFeatures.empty())
+    dap.Send(protocol::Event{"capabilities", body});
+}
+
 // "ProcessEvent": {
 //   "allOf": [
 //     { "$ref": "#/definitions/Event" },
@@ -116,78 +137,78 @@ void SendProcessEvent(DAP &dap, LaunchMethod launch_method) {
 
 // Send a thread stopped event for all threads as long as the process
 // is stopped.
-void SendThreadStoppedEvent(DAP &dap) {
+llvm::Error SendThreadStoppedEvent(DAP &dap, bool on_entry) {
+  lldb::SBMutex lock = dap.GetAPIMutex();
+  std::lock_guard<lldb::SBMutex> guard(lock);
+
   lldb::SBProcess process = dap.target.GetProcess();
-  if (process.IsValid()) {
-    auto state = process.GetState();
-    if (state == lldb::eStateStopped) {
-      llvm::DenseSet<lldb::tid_t> old_thread_ids;
-      old_thread_ids.swap(dap.thread_ids);
-      uint32_t stop_id = process.GetStopID();
-      const uint32_t num_threads = process.GetNumThreads();
-
-      // First make a pass through the threads to see if the focused thread
-      // has a stop reason. In case the focus thread doesn't have a stop
-      // reason, remember the first thread that has a stop reason so we can
-      // set it as the focus thread if below if needed.
-      lldb::tid_t first_tid_with_reason = LLDB_INVALID_THREAD_ID;
-      uint32_t num_threads_with_reason = 0;
-      bool focus_thread_exists = false;
-      for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-        lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
-        const lldb::tid_t tid = thread.GetThreadID();
-        const bool has_reason = ThreadHasStopReason(thread);
-        // If the focus thread doesn't have a stop reason, clear the thread ID
-        if (tid == dap.focus_tid) {
-          focus_thread_exists = true;
-          if (!has_reason)
-            dap.focus_tid = LLDB_INVALID_THREAD_ID;
-        }
-        if (has_reason) {
-          ++num_threads_with_reason;
-          if (first_tid_with_reason == LLDB_INVALID_THREAD_ID)
-            first_tid_with_reason = tid;
-        }
-      }
+  if (!process.IsValid())
+    return make_error<DAPError>("invalid process");
 
-      // We will have cleared dap.focus_tid if the focus thread doesn't have
-      // a stop reason, so if it was cleared, or wasn't set, or doesn't exist,
-      // then set the focus thread to the first thread with a stop reason.
-      if (!focus_thread_exists || dap.focus_tid == LLDB_INVALID_THREAD_ID)
-        dap.focus_tid = first_tid_with_reason;
-
-      // If no threads stopped with a reason, then report the first one so
-      // we at least let the UI know we stopped.
-      if (num_threads_with_reason == 0) {
-        lldb::SBThread thread = process.GetThreadAtIndex(0);
-        dap.focus_tid = thread.GetThreadID();
-        dap.SendJSON(CreateThreadStopped(dap, thread, stop_id));
-      } else {
-        for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-          lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
-          dap.thread_ids.insert(thread.GetThreadID());
-          if (ThreadHasStopReason(thread)) {
-            dap.SendJSON(CreateThreadStopped(dap, thread, stop_id));
-          }
-        }
-      }
+  lldb::StateType state = process.GetState();
+  if (!lldb::SBDebugger::StateIsStoppedState(state))
+    return make_error<NotStoppedError>();
 
-      for (auto tid : old_thread_ids) {
-        auto end = dap.thread_ids.end();
-        auto pos = dap.thread_ids.find(tid);
-        if (pos == end)
-          SendThreadExitedEvent(dap, tid);
-      }
-    } else {
-      DAP_LOG(
-          dap.log,
-          "error: SendThreadStoppedEvent() when process isn't stopped ({0})",
-          lldb::SBDebugger::StateAsCString(state));
+  llvm::DenseSet<lldb::tid_t> old_thread_ids;
+  old_thread_ids.swap(dap.thread_ids);
+  uint32_t stop_id = process.GetStopID();
+  const uint32_t num_threads = process.GetNumThreads();
+
+  // First make a pass through the threads to see if the focused thread
+  // has a stop reason. In case the focus thread doesn't have a stop
+  // reason, remember the first thread that has a stop reason so we can
+  // set it as the focus thread if below if needed.
+  lldb::tid_t first_tid_with_reason = LLDB_INVALID_THREAD_ID;
+  uint32_t num_threads_with_reason = 0;
+  bool focus_thread_exists = false;
+  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
+    lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
+    const lldb::tid_t tid = thread.GetThreadID();
+    const bool has_reason = ThreadHasStopReason(thread);
+    // If the focus thread doesn't have a stop reason, clear the thread ID
+    if (tid == dap.focus_tid) {
+      focus_thread_exists = true;
+      if (!has_reason)
+        dap.focus_tid = LLDB_INVALID_THREAD_ID;
+    }
+    if (has_reason) {
+      ++num_threads_with_reason;
+      if (first_tid_with_reason == LLDB_INVALID_THREAD_ID)
+        first_tid_with_reason = tid;
     }
+  }
+
+  // We will have cleared dap.focus_tid if the focus thread doesn't have
+  // a stop reason, so if it was cleared, or wasn't set, or doesn't exist,
+  // then set the focus thread to the first thread with a stop reason.
+  if (!focus_thread_exists || dap.focus_tid == LLDB_INVALID_THREAD_ID)
+    dap.focus_tid = first_tid_with_reason;
+
+  // If no threads stopped with a reason, then report the first one so
+  // we at least let the UI know we stopped.
+  if (num_threads_with_reason == 0) {
+    lldb::SBThread thread = process.GetThreadAtIndex(0);
+    dap.focus_tid = thread.GetThreadID();
+    dap.SendJSON(CreateThreadStopped(dap, thread, stop_id));
   } else {
-    DAP_LOG(dap.log, "error: SendThreadStoppedEvent() invalid process");
+    for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
+      lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
+      dap.thread_ids.insert(thread.GetThreadID());
+      if (ThreadHasStopReason(thread)) {
+        dap.SendJSON(CreateThreadStopped(dap, thread, stop_id));
+      }
+    }
   }
+
+  for (const auto &tid : old_thread_ids) {
+    auto end = dap.thread_ids.end();
+    auto pos = dap.thread_ids.find(tid);
+    if (pos == end)
+      SendThreadExitedEvent(dap, tid);
+  }
+
   dap.RunStopCommands();
+  return Error::success();
 }
 
 // Send a "terminated" event to indicate the process is done being
diff --git a/external/llvm-project/lldb/tools/lldb-dap/EventHelper.h b/external/llvm-project/lldb/tools/lldb-dap/EventHelper.h
index 90b009c73089..72ad5308a2b0 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/EventHelper.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/EventHelper.h
@@ -10,15 +10,19 @@
 #define LLDB_TOOLS_LLDB_DAP_EVENTHELPER_H
 
 #include "DAPForward.h"
+#include "llvm/Support/Error.h"
 
 namespace lldb_dap {
 struct DAP;
 
 enum LaunchMethod { Launch, Attach, AttachForSuspendedLaunch };
 
+/// Update capabilities based on the configured target.
+void SendTargetBasedCapabilities(DAP &dap);
+
 void SendProcessEvent(DAP &dap, LaunchMethod launch_method);
 
-void SendThreadStoppedEvent(DAP &dap);
+llvm::Error SendThreadStoppedEvent(DAP &dap, bool on_entry = false);
 
 void SendTerminatedEvent(DAP &dap);
 
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
index 1281857ef4b6..e7735a705d0a 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
@@ -8,8 +8,9 @@
 
 #include "DAP.h"
 #include "EventHelper.h"
-#include "JSONUtils.h"
+#include "LLDBUtils.h"
 #include "Protocol/ProtocolRequests.h"
+#include "ProtocolUtils.h"
 #include "RequestHandler.h"
 #include "lldb/API/SBDebugger.h"
 
@@ -40,6 +41,11 @@ ConfigurationDoneRequestHandler::Run(const ConfigurationDoneArguments &) const {
         "any debugger command scripts are not resuming the process during the "
         "launch sequence.");
 
+  // Waiting until 'configurationDone' to send target based capabilities in case
+  // the launch or attach scripts adjust the target. The initial dummy target
+  // may have different capabilities than the final target.
+  SendTargetBasedCapabilities(dap);
+
   // Clients can request a baseline of currently existing threads after
   // we acknowledge the configurationDone request.
   // Client requests the baseline of currently existing threads after
@@ -51,11 +57,9 @@ ConfigurationDoneRequestHandler::Run(const ConfigurationDoneArguments &) const {
   SendProcessEvent(dap, dap.is_attach ? Attach : Launch);
 
   if (dap.stop_at_entry)
-    SendThreadStoppedEvent(dap);
-  else
-    process.Continue();
+    return SendThreadStoppedEvent(dap, /*on_entry=*/true);
 
-  return Error::success();
+  return ToError(process.Continue());
 }
 
 } // namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/DisconnectRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/DisconnectRequestHandler.cpp
index 81e94c755183..8314a1011a57 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/DisconnectRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/DisconnectRequestHandler.cpp
@@ -20,7 +20,7 @@ namespace lldb_dap {
 /// Disconnect request; value of command field is 'disconnect'.
 Error DisconnectRequestHandler::Run(
     const std::optional<DisconnectArguments> &arguments) const {
-  bool terminateDebuggee = dap.is_attach ? false : true;
+  bool terminateDebuggee = !dap.is_attach;
 
   if (arguments && arguments->terminateDebuggee)
     terminateDebuggee = *arguments->terminateDebuggee;
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/NextRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/NextRequestHandler.cpp
index 3fa167686d2f..2b48350dfba1 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/NextRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/NextRequestHandler.cpp
@@ -8,6 +8,7 @@
 
 #include "DAP.h"
 #include "EventHelper.h"
+#include "LLDBUtils.h"
 #include "Protocol/ProtocolTypes.h"
 #include "RequestHandler.h"
 #include "llvm/Support/Error.h"
@@ -30,16 +31,21 @@ Error NextRequestHandler::Run(const NextArguments &args) const {
   if (!thread.IsValid())
     return make_error<DAPError>("invalid thread");
 
+  if (!SBDebugger::StateIsStoppedState(dap.target.GetProcess().GetState()))
+    return make_error<NotStoppedError>();
+
   // Remember the thread ID that caused the resume so we can set the
   // "threadCausedFocus" boolean value in the "stopped" events.
   dap.focus_tid = thread.GetThreadID();
+  lldb::SBError error;
   if (args.granularity == eSteppingGranularityInstruction) {
-    thread.StepInstruction(/*step_over=*/true);
+    thread.StepInstruction(/*step_over=*/true, error);
   } else {
-    thread.StepOver(args.singleThread ? eOnlyThisThread : eOnlyDuringStepping);
+    thread.StepOver(args.singleThread ? eOnlyThisThread : eOnlyDuringStepping,
+                    error);
   }
 
-  return Error::success();
+  return ToError(error);
 }
 
 } // namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/RequestHandler.h b/external/llvm-project/lldb/tools/lldb-dap/Handler/RequestHandler.h
index 3a965bcc87a5..d3f231589b54 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/RequestHandler.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/RequestHandler.h
@@ -334,9 +334,6 @@ class RestartRequestHandler : public LegacyRequestHandler {
 public:
   using LegacyRequestHandler::LegacyRequestHandler;
   static llvm::StringLiteral GetCommand() { return "restart"; }
-  FeatureSet GetSupportedFeatures() const override {
-    return {protocol::eAdapterFeatureRestartRequest};
-  }
   void operator()(const llvm::json::Object &request) const override;
 };
 
@@ -522,11 +519,14 @@ class StackTraceRequestHandler : public LegacyRequestHandler {
   }
 };
 
-class ThreadsRequestHandler : public LegacyRequestHandler {
+class ThreadsRequestHandler
+    : public RequestHandler<protocol::ThreadsArguments,
+                            llvm::Expected<protocol::ThreadsResponseBody>> {
 public:
-  using LegacyRequestHandler::LegacyRequestHandler;
+  using RequestHandler::RequestHandler;
   static llvm::StringLiteral GetCommand() { return "threads"; }
-  void operator()(const llvm::json::Object &request) const override;
+  llvm::Expected<protocol::ThreadsResponseBody>
+  Run(const protocol::ThreadsArguments &) const override;
 };
 
 class VariablesRequestHandler : public LegacyRequestHandler {
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/RestartRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/RestartRequestHandler.cpp
index 58091b622f7e..45dd7ddce042 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/RestartRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/RestartRequestHandler.cpp
@@ -69,23 +69,6 @@ void RestartRequestHandler::operator()(
     dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
-  // Check if we were in a "launch" session or an "attach" session.
-  //
-  // Restarting is not well defined when we started the session by attaching to
-  // an existing process, because we don't know how the process was started, so
-  // we don't support it.
-  //
-  // Note that when using runInTerminal we're technically attached, but it's an
-  // implementation detail. The adapter *did* launch the process in response to
-  // a "launch" command, so we can still stop it and re-run it. This is why we
-  // don't just check `dap.is_attach`.
-  if (!dap.last_launch_request) {
-    response["success"] = llvm::json::Value(false);
-    EmplaceSafeString(response, "message",
-                      "Restarting an \"attach\" session is not supported.");
-    dap.SendJSON(llvm::json::Value(std::move(response)));
-    return;
-  }
 
   const llvm::json::Object *arguments = request.getObject("arguments");
   if (arguments) {
@@ -145,7 +128,11 @@ void RestartRequestHandler::operator()(
   // Because we're restarting, configuration has already happened so we can
   // continue the process right away.
   if (dap.stop_at_entry) {
-    SendThreadStoppedEvent(dap);
+    if (llvm::Error err = SendThreadStoppedEvent(dap, /*on_entry=*/true)) {
+      EmplaceSafeString(response, "message", llvm::toString(std::move(err)));
+      dap.SendJSON(llvm::json::Value(std::move(response)));
+      return;
+    }
   } else {
     dap.target.GetProcess().Continue();
   }
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/SetExceptionBreakpointsRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/SetExceptionBreakpointsRequestHandler.cpp
index 09d4fea2a9a2..2214833f8a77 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/SetExceptionBreakpointsRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/SetExceptionBreakpointsRequestHandler.cpp
@@ -76,7 +76,7 @@ void SetExceptionBreakpointsRequestHandler::operator()(
 
   for (const auto &value : *filters) {
     const auto filter = GetAsString(value);
-    auto *exc_bp = dap.GetExceptionBreakpoint(std::string(filter));
+    auto *exc_bp = dap.GetExceptionBreakpoint(filter);
     if (exc_bp) {
       exc_bp->SetBreakpoint();
       unset_filters.erase(std::string(filter));
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/StepInRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/StepInRequestHandler.cpp
index 15f242a9e18f..6742c791a548 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/StepInRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/StepInRequestHandler.cpp
@@ -8,6 +8,7 @@
 
 #include "DAP.h"
 #include "EventHelper.h"
+#include "LLDBUtils.h"
 #include "Protocol/ProtocolRequests.h"
 #include "Protocol/ProtocolTypes.h"
 #include "RequestHandler.h"
@@ -39,9 +40,13 @@ Error StepInRequestHandler::Run(const StepInArguments &args) const {
   // "threadCausedFocus" boolean value in the "stopped" events.
   dap.focus_tid = thread.GetThreadID();
 
+  if (!SBDebugger::StateIsStoppedState(dap.target.GetProcess().GetState()))
+    return make_error<NotStoppedError>();
+
+  lldb::SBError error;
   if (args.granularity == eSteppingGranularityInstruction) {
-    thread.StepInstruction(/*step_over=*/false);
-    return Error::success();
+    thread.StepInstruction(/*step_over=*/false, error);
+    return ToError(error);
   }
 
   std::string step_in_target;
@@ -50,8 +55,9 @@ Error StepInRequestHandler::Run(const StepInArguments &args) const {
     step_in_target = it->second;
 
   RunMode run_mode = args.singleThread ? eOnlyThisThread : eOnlyDuringStepping;
-  thread.StepInto(step_in_target.c_str(), run_mode);
-  return Error::success();
+  thread.StepInto(step_in_target.c_str(), LLDB_INVALID_LINE_NUMBER, error,
+                  run_mode);
+  return ToError(error);
 }
 
 } // namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/StepOutRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/StepOutRequestHandler.cpp
index 6b98582262a6..e896e03720b6 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/StepOutRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/StepOutRequestHandler.cpp
@@ -8,6 +8,7 @@
 
 #include "DAP.h"
 #include "EventHelper.h"
+#include "LLDBUtils.h"
 #include "Protocol/ProtocolRequests.h"
 #include "RequestHandler.h"
 #include "llvm/Support/Error.h"
@@ -32,12 +33,17 @@ Error StepOutRequestHandler::Run(const StepOutArguments &arguments) const {
   if (!thread.IsValid())
     return make_error<DAPError>("invalid thread");
 
+  if (!lldb::SBDebugger::StateIsStoppedState(
+          dap.target.GetProcess().GetState()))
+    return make_error<NotStoppedError>();
+
   // Remember the thread ID that caused the resume so we can set the
   // "threadCausedFocus" boolean value in the "stopped" events.
   dap.focus_tid = thread.GetThreadID();
-  thread.StepOut();
+  lldb::SBError error;
+  thread.StepOut(error);
 
-  return Error::success();
+  return ToError(error);
 }
 
 } // namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp b/external/llvm-project/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp
index 16d797c2ab32..8b328d8348e3 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Handler/ThreadsRequestHandler.cpp
@@ -8,72 +8,45 @@
 
 #include "DAP.h"
 #include "EventHelper.h"
-#include "JSONUtils.h"
+#include "Protocol/ProtocolRequests.h"
+#include "ProtocolUtils.h"
 #include "RequestHandler.h"
+#include "lldb/API/SBDebugger.h"
+#include "lldb/API/SBDefines.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+using namespace lldb_dap::protocol;
 
 namespace lldb_dap {
 
-// "ThreadsRequest": {
-//   "allOf": [ { "$ref": "#/definitions/Request" }, {
-//     "type": "object",
-//     "description": "Thread request; value of command field is 'threads'. The
-//     request retrieves a list of all threads.", "properties": {
-//       "command": {
-//         "type": "string",
-//         "enum": [ "threads" ]
-//       }
-//     },
-//     "required": [ "command" ]
-//   }]
-// },
-// "ThreadsResponse": {
-//   "allOf": [ { "$ref": "#/definitions/Response" }, {
-//     "type": "object",
-//     "description": "Response to 'threads' request.",
-//     "properties": {
-//       "body": {
-//         "type": "object",
-//         "properties": {
-//           "threads": {
-//             "type": "array",
-//             "items": {
-//               "$ref": "#/definitions/Thread"
-//             },
-//             "description": "All threads."
-//           }
-//         },
-//         "required": [ "threads" ]
-//       }
-//     },
-//     "required": [ "body" ]
-//   }]
-// }
-void ThreadsRequestHandler::operator()(
-    const llvm::json::Object &request) const {
-  llvm::json::Object response;
-  FillResponse(request, response);
+/// The request retrieves a list of all threads.
+Expected<ThreadsResponseBody>
+ThreadsRequestHandler::Run(const ThreadsArguments &) const {
+  lldb::SBProcess process = dap.target.GetProcess();
+  std::vector<Thread> threads;
 
-  llvm::json::Array threads;
   // Client requests the baseline of currently existing threads after
   // a successful launch or attach by sending a 'threads' request
   // right after receiving the configurationDone response.
   // If no thread has reported to the client, it prevents something
   // like the pause request from working in the running state.
   // Return the cache of initial threads as the process might have resumed
-  if (dap.initial_thread_list) {
-    threads = dap.initial_thread_list.value();
-    dap.initial_thread_list.reset();
+  if (!dap.initial_thread_list.empty()) {
+    threads = dap.initial_thread_list;
+    dap.initial_thread_list.clear();
   } else {
-    threads = GetThreads(dap.target.GetProcess(), dap.thread_format);
-  }
+    if (!lldb::SBDebugger::StateIsStoppedState(process.GetState()))
+      return make_error<NotStoppedError>();
 
-  if (threads.size() == 0) {
-    response["success"] = llvm::json::Value(false);
+    threads = GetThreads(process, dap.thread_format);
   }
-  llvm::json::Object body;
-  body.try_emplace("threads", std::move(threads));
-  response.try_emplace("body", std::move(body));
-  dap.SendJSON(llvm::json::Value(std::move(response)));
+
+  if (threads.size() == 0)
+    return make_error<DAPError>("failed to retrieve threads from process");
+
+  return ThreadsResponseBody{threads};
 }
 
 } // namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.cpp b/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.cpp
index 573f3eba00f6..6cdde63e9796 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -643,70 +643,6 @@ llvm::json::Value CreateExtendedStackFrameLabel(lldb::SBThread &thread,
                                               {"presentationHint", "label"}});
 }
 
-// "Thread": {
-//   "type": "object",
-//   "description": "A Thread",
-//   "properties": {
-//     "id": {
-//       "type": "integer",
-//       "description": "Unique identifier for the thread."
-//     },
-//     "name": {
-//       "type": "string",
-//       "description": "A name of the thread."
-//     }
-//   },
-//   "required": [ "id", "name" ]
-// }
-llvm::json::Value CreateThread(lldb::SBThread &thread, lldb::SBFormat &format) {
-  llvm::json::Object object;
-  object.try_emplace("id", (int64_t)thread.GetThreadID());
-  std::string thread_str;
-  lldb::SBStream stream;
-  if (format && thread.GetDescriptionWithFormat(format, stream).Success()) {
-    thread_str = stream.GetData();
-  } else {
-    llvm::StringRef thread_name(thread.GetName());
-    llvm::StringRef queue_name(thread.GetQueueName());
-
-    if (!thread_name.empty()) {
-      thread_str = thread_name.str();
-    } else if (!queue_name.empty()) {
-      auto kind = thread.GetQueue().GetKind();
-      std::string queue_kind_label = "";
-      if (kind == lldb::eQueueKindSerial) {
-        queue_kind_label = " (serial)";
-      } else if (kind == lldb::eQueueKindConcurrent) {
-        queue_kind_label = " (concurrent)";
-      }
-
-      thread_str =
-          llvm::formatv("Thread {0} Queue: {1}{2}", thread.GetIndexID(),
-                        queue_name, queue_kind_label)
-              .str();
-    } else {
-      thread_str = llvm::formatv("Thread {0}", thread.GetIndexID()).str();
-    }
-  }
-
-  EmplaceSafeString(object, "name", thread_str);
-
-  return llvm::json::Value(std::move(object));
-}
-
-llvm::json::Array GetThreads(lldb::SBProcess process, lldb::SBFormat &format) {
-  lldb::SBMutex lock = process.GetTarget().GetAPIMutex();
-  std::lock_guard<lldb::SBMutex> guard(lock);
-
-  llvm::json::Array threads;
-  const uint32_t num_threads = process.GetNumThreads();
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
-    threads.emplace_back(CreateThread(thread, format));
-  }
-  return threads;
-}
-
 // "StoppedEvent": {
 //   "allOf": [ { "$ref": "#/definitions/Event" }, {
 //     "type": "object",
diff --git a/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.h b/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.h
index 08699a94bbd8..10dc46b94184 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/JSONUtils.h
@@ -283,30 +283,6 @@ llvm::json::Value CreateStackFrame(lldb::SBFrame &frame,
 llvm::json::Value CreateExtendedStackFrameLabel(lldb::SBThread &thread,
                                                 lldb::SBFormat &format);
 
-/// Create a "Thread" object for a LLDB thread object.
-///
-/// This function will fill in the following keys in the returned
-/// object:
-///   "id" - the thread ID as an integer
-///   "name" - the thread name as a string which combines the LLDB
-///            thread index ID along with the string name of the thread
-///            from the OS if it has a name.
-///
-/// \param[in] thread
-///     The LLDB thread to use when populating out the "Thread"
-///     object.
-///
-/// \param[in] format
-///     The LLDB format to use when populating out the "Thread"
-///     object.
-///
-/// \return
-///     A "Thread" JSON object with that follows the formal JSON
-///     definition outlined by Microsoft.
-llvm::json::Value CreateThread(lldb::SBThread &thread, lldb::SBFormat &format);
-
-llvm::json::Array GetThreads(lldb::SBProcess process, lldb::SBFormat &format);
-
 /// Create a "StoppedEvent" object for a LLDB thread object.
 ///
 /// This function will fill in the following keys in the returned
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolBase.h b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolBase.h
index 724da59b50cd..81496380d412 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolBase.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolBase.h
@@ -17,8 +17,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_DAP_PROTOCOL_H
-#define LLDB_TOOLS_LLDB_DAP_PROTOCOL_H
+#ifndef LLDB_TOOLS_LLDB_DAP_PROTOCOL_PROTOCOL_BASE_H
+#define LLDB_TOOLS_LLDB_DAP_PROTOCOL_PROTOCOL_BASE_H
 
 #include "llvm/Support/JSON.h"
 #include <cstdint>
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolEvents.cpp b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolEvents.cpp
new file mode 100644
index 000000000000..ad6e305d09fe
--- /dev/null
+++ b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolEvents.cpp
@@ -0,0 +1,20 @@
+//===-- ProtocolEvents.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Protocol/ProtocolEvents.h"
+#include "llvm/Support/JSON.h"
+
+using namespace llvm;
+
+namespace lldb_dap::protocol {
+
+json::Value toJSON(const CapabilitiesEventBody &CEB) {
+  return json::Object{{"capabilities", CEB.capabilities}};
+}
+
+} // namespace lldb_dap::protocol
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolEvents.h b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolEvents.h
new file mode 100644
index 000000000000..512106222362
--- /dev/null
+++ b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolEvents.h
@@ -0,0 +1,46 @@
+//===-- ProtocolEvents.h --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains POD structs based on the DAP specification at
+// https://microsoft.github.io/debug-adapter-protocol/specification
+//
+// This is not meant to be a complete implementation, new interfaces are added
+// when they're needed.
+//
+// Each struct has a toJSON and fromJSON function, that converts between
+// the struct and a JSON representation. (See JSON.h)
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_TOOLS_LLDB_DAP_PROTOCOL_PROTOCOL_EVENTS_H
+#define LLDB_TOOLS_LLDB_DAP_PROTOCOL_PROTOCOL_EVENTS_H
+
+#include "Protocol/ProtocolTypes.h"
+#include "llvm/Support/JSON.h"
+
+namespace lldb_dap::protocol {
+
+/// The event indicates that one or more capabilities have changed.
+///
+/// Since the capabilities are dependent on the client and its UI, it might not
+/// be possible to change that at random times (or too late).
+///
+/// Consequently this event has a hint characteristic: a client can only be
+/// expected to make a 'best effort' in honoring individual capabilities but
+/// there are no guarantees.
+///
+/// Only changed capabilities need to be included, all other capabilities keep
+/// their values.
+struct CapabilitiesEventBody {
+  Capabilities capabilities;
+};
+llvm::json::Value toJSON(const CapabilitiesEventBody &);
+
+} // end namespace lldb_dap::protocol
+
+#endif
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
index 4160077d419e..2cb7c47d6020 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
@@ -140,9 +140,8 @@ parseSourceMap(const json::Value &Params,
 
 namespace lldb_dap::protocol {
 
-bool fromJSON(const llvm::json::Value &Params, CancelArguments &CA,
-              llvm::json::Path P) {
-  llvm::json::ObjectMapper O(Params, P);
+bool fromJSON(const json::Value &Params, CancelArguments &CA, json::Path P) {
+  json::ObjectMapper O(Params, P);
   return O && O.map("requestId", CA.requestId) &&
          O.map("progressId", CA.progressId);
 }
@@ -150,9 +149,9 @@ bool fromJSON(const llvm::json::Value &Params, CancelArguments &CA,
 bool fromJSON(const json::Value &Params, DisconnectArguments &DA,
               json::Path P) {
   json::ObjectMapper O(Params, P);
-  return O && O.map("restart", DA.restart) &&
-         O.map("terminateDebuggee", DA.terminateDebuggee) &&
-         O.map("suspendDebuggee", DA.suspendDebuggee);
+  return O && O.mapOptional("restart", DA.restart) &&
+         O.mapOptional("terminateDebuggee", DA.terminateDebuggee) &&
+         O.mapOptional("suspendDebuggee", DA.suspendDebuggee);
 }
 
 bool fromJSON(const json::Value &Params, PathFormat &PF, json::Path P) {
@@ -257,12 +256,8 @@ bool fromJSON(const json::Value &Params, BreakpointLocationsArguments &BLA,
          O.mapOptional("endColumn", BLA.endColumn);
 }
 
-llvm::json::Value toJSON(const BreakpointLocationsResponseBody &BLRB) {
-  llvm::json::Array breakpoints_json;
-  for (const auto &breakpoint : BLRB.breakpoints) {
-    breakpoints_json.push_back(toJSON(breakpoint));
-  }
-  return llvm::json::Object{{"breakpoints", std::move(breakpoints_json)}};
+json::Value toJSON(const BreakpointLocationsResponseBody &BLRB) {
+  return json::Object{{"breakpoints", BLRB.breakpoints}};
 }
 
 bool fromJSON(const json::Value &Params, LaunchRequestArguments &LRA,
@@ -293,27 +288,26 @@ bool fromJSON(const json::Value &Params, AttachRequestArguments &ARA,
          O.mapOptional("coreFile", ARA.coreFile);
 }
 
-bool fromJSON(const llvm::json::Value &Params, ContinueArguments &CA,
-              llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, ContinueArguments &CA, json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("threadId", CA.threadId) &&
          O.mapOptional("singleThread", CA.singleThread);
 }
 
-llvm::json::Value toJSON(const ContinueResponseBody &CRB) {
+json::Value toJSON(const ContinueResponseBody &CRB) {
   json::Object Body{{"allThreadsContinued", CRB.allThreadsContinued}};
   return std::move(Body);
 }
 
-bool fromJSON(const llvm::json::Value &Params, SetVariableArguments &SVA,
-              llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, SetVariableArguments &SVA,
+              json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("variablesReference", SVA.variablesReference) &&
          O.map("name", SVA.name) && O.map("value", SVA.value) &&
          O.mapOptional("format", SVA.format);
 }
 
-llvm::json::Value toJSON(const SetVariableResponseBody &SVR) {
+json::Value toJSON(const SetVariableResponseBody &SVR) {
   json::Object Body{{"value", SVR.value}};
   if (SVR.type.has_value())
     Body.insert({"type", SVR.type});
@@ -333,21 +327,15 @@ llvm::json::Value toJSON(const SetVariableResponseBody &SVR) {
   if (SVR.valueLocationReference.has_value())
     Body.insert({"valueLocationReference", SVR.valueLocationReference});
 
-  return llvm::json::Value(std::move(Body));
+  return json::Value(std::move(Body));
 }
-bool fromJSON(const llvm::json::Value &Params, ScopesArguments &SCA,
-              llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, ScopesArguments &SCA, json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("frameId", SCA.frameId);
 }
 
-llvm::json::Value toJSON(const ScopesResponseBody &SCR) {
-  llvm::json::Array scopes;
-  for (const Scope &scope : SCR.scopes) {
-    scopes.emplace_back(toJSON(scope));
-  }
-
-  return llvm::json::Object{{"scopes", std::move(scopes)}};
+json::Value toJSON(const ScopesResponseBody &SCR) {
+  return json::Object{{"scopes", SCR.scopes}};
 }
 
 bool fromJSON(const json::Value &Params, SourceArguments &SA, json::Path P) {
@@ -365,16 +353,14 @@ json::Value toJSON(const SourceResponseBody &SA) {
   return std::move(Result);
 }
 
-bool fromJSON(const llvm::json::Value &Params, NextArguments &NA,
-              llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, NextArguments &NA, json::Path P) {
   json::ObjectMapper OM(Params, P);
   return OM && OM.map("threadId", NA.threadId) &&
          OM.mapOptional("singleThread", NA.singleThread) &&
          OM.mapOptional("granularity", NA.granularity);
 }
 
-bool fromJSON(const llvm::json::Value &Params, StepInArguments &SIA,
-              llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, StepInArguments &SIA, json::Path P) {
   json::ObjectMapper OM(Params, P);
   return OM && OM.map("threadId", SIA.threadId) &&
          OM.map("targetId", SIA.targetId) &&
@@ -382,54 +368,47 @@ bool fromJSON(const llvm::json::Value &Params, StepInArguments &SIA,
          OM.mapOptional("granularity", SIA.granularity);
 }
 
-bool fromJSON(const llvm::json::Value &Params, StepOutArguments &SOA,
-              llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, StepOutArguments &SOA, json::Path P) {
   json::ObjectMapper OM(Params, P);
   return OM && OM.map("threadId", SOA.threadId) &&
          OM.mapOptional("singleThread", SOA.singleThread) &&
          OM.mapOptional("granularity", SOA.granularity);
 }
 
-bool fromJSON(const llvm::json::Value &Params, SetBreakpointsArguments &SBA,
-              llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, SetBreakpointsArguments &SBA,
+              json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("source", SBA.source) &&
          O.map("breakpoints", SBA.breakpoints) && O.map("lines", SBA.lines) &&
          O.map("sourceModified", SBA.sourceModified);
 }
 
-llvm::json::Value toJSON(const SetBreakpointsResponseBody &SBR) {
-  json::Object result;
-  result["breakpoints"] = SBR.breakpoints;
-  return result;
+json::Value toJSON(const SetBreakpointsResponseBody &SBR) {
+  return json::Object{{"breakpoints", SBR.breakpoints}};
 }
 
-bool fromJSON(const llvm::json::Value &Params,
-              SetFunctionBreakpointsArguments &SFBA, llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, SetFunctionBreakpointsArguments &SFBA,
+              json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("breakpoints", SFBA.breakpoints);
 }
 
-llvm::json::Value toJSON(const SetFunctionBreakpointsResponseBody &SFBR) {
-  json::Object result;
-  result["breakpoints"] = SFBR.breakpoints;
-  return result;
+json::Value toJSON(const SetFunctionBreakpointsResponseBody &SFBR) {
+  return json::Object{{"breakpoints", SFBR.breakpoints}};
 }
 
-bool fromJSON(const llvm::json::Value &Params,
-              SetInstructionBreakpointsArguments &SIBA, llvm::json::Path P) {
+bool fromJSON(const json::Value &Params,
+              SetInstructionBreakpointsArguments &SIBA, json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("breakpoints", SIBA.breakpoints);
 }
 
-llvm::json::Value toJSON(const SetInstructionBreakpointsResponseBody &SIBR) {
-  json::Object result;
-  result["breakpoints"] = SIBR.breakpoints;
-  return result;
+json::Value toJSON(const SetInstructionBreakpointsResponseBody &SIBR) {
+  return json::Object{{"breakpoints", SIBR.breakpoints}};
 }
 
-bool fromJSON(const llvm::json::Value &Params,
-              DataBreakpointInfoArguments &DBIA, llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, DataBreakpointInfoArguments &DBIA,
+              json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("variablesReference", DBIA.variablesReference) &&
          O.map("name", DBIA.name) && O.map("frameId", DBIA.frameId) &&
@@ -437,27 +416,30 @@ bool fromJSON(const llvm::json::Value &Params,
          O.map("mode", DBIA.mode);
 }
 
-llvm::json::Value toJSON(const DataBreakpointInfoResponseBody &DBIRB) {
-  json::Object result;
-  result["dataId"] = DBIRB.dataId ? *DBIRB.dataId : llvm::json::Value(nullptr);
-  result["description"] = DBIRB.description;
+json::Value toJSON(const DataBreakpointInfoResponseBody &DBIRB) {
+  json::Object result{{"dataId", DBIRB.dataId},
+                      {"description", DBIRB.description}};
+
   if (DBIRB.accessTypes)
     result["accessTypes"] = *DBIRB.accessTypes;
   if (DBIRB.canPersist)
     result["canPersist"] = *DBIRB.canPersist;
+
   return result;
 }
 
-bool fromJSON(const llvm::json::Value &Params,
-              SetDataBreakpointsArguments &SDBA, llvm::json::Path P) {
+bool fromJSON(const json::Value &Params, SetDataBreakpointsArguments &SDBA,
+              json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("breakpoints", SDBA.breakpoints);
 }
 
-llvm::json::Value toJSON(const SetDataBreakpointsResponseBody &SDBR) {
-  json::Object result;
-  result["breakpoints"] = SDBR.breakpoints;
-  return result;
+json::Value toJSON(const SetDataBreakpointsResponseBody &SDBR) {
+  return json::Object{{"breakpoints", SDBR.breakpoints}};
+}
+
+json::Value toJSON(const ThreadsResponseBody &TR) {
+  return json::Object{{"threads", TR.threads}};
 }
 
 bool fromJSON(const llvm::json::Value &Params, DisassembleArguments &DA,
@@ -470,12 +452,8 @@ bool fromJSON(const llvm::json::Value &Params, DisassembleArguments &DA,
          O.mapOptional("resolveSymbols", DA.resolveSymbols);
 }
 
-llvm::json::Value toJSON(const DisassembleResponseBody &DRB) {
-  llvm::json::Array instructions;
-  for (const auto &instruction : DRB.instructions) {
-    instructions.push_back(toJSON(instruction));
-  }
-  return llvm::json::Object{{"instructions", std::move(instructions)}};
+json::Value toJSON(const DisassembleResponseBody &DRB) {
+  return json::Object{{"instructions", DRB.instructions}};
 }
 
 } // namespace lldb_dap::protocol
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
index 7c774e50d6e5..d199cc886b11 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
@@ -482,6 +482,16 @@ struct SourceResponseBody {
 };
 llvm::json::Value toJSON(const SourceResponseBody &);
 
+/// Arguments for the `threads` request, no arguments.
+using ThreadsArguments = EmptyArguments;
+
+/// Response to `threads` request.
+struct ThreadsResponseBody {
+  /// All threads.
+  std::vector<Thread> threads;
+};
+llvm::json::Value toJSON(const ThreadsResponseBody &);
+
 /// Arguments for `next` request.
 struct NextArguments {
   /// Specifies the thread for which to resume execution for one step (of the
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
index 3b297a0bd431..085d53bb006e 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
@@ -582,6 +582,15 @@ llvm::json::Value toJSON(const SteppingGranularity &SG) {
   llvm_unreachable("unhandled stepping granularity.");
 }
 
+bool fromJSON(const json::Value &Params, Thread &T, json::Path P) {
+  json::ObjectMapper O(Params, P);
+  return O && O.map("id", T.id) && O.map("name", T.name);
+}
+
+json::Value toJSON(const Thread &T) {
+  return json::Object{{"id", T.id}, {"name", T.name}};
+}
+
 bool fromJSON(const llvm::json::Value &Params, ValueFormat &VF,
               llvm::json::Path P) {
   json::ObjectMapper O(Params, P);
@@ -821,22 +830,20 @@ llvm::json::Value toJSON(const DisassembledInstruction::PresentationHint &PH) {
 
 bool fromJSON(const llvm::json::Value &Params, DisassembledInstruction &DI,
               llvm::json::Path P) {
-  std::optional<llvm::StringRef> raw_address =
-      Params.getAsObject()->getString("address");
-  if (!raw_address) {
-    P.report("missing 'address' field");
+  llvm::json::ObjectMapper O(Params, P);
+  std::string raw_address;
+  if (!O || !O.map("address", raw_address))
     return false;
-  }
 
-  std::optional<lldb::addr_t> address = DecodeMemoryReference(*raw_address);
+  std::optional<lldb::addr_t> address = DecodeMemoryReference(raw_address);
   if (!address) {
-    P.report("invalid 'address'");
+    P.field("address").report("expected string encoded uint64_t");
     return false;
   }
 
   DI.address = *address;
-  llvm::json::ObjectMapper O(Params, P);
-  return O && O.map("instruction", DI.instruction) &&
+
+  return O.map("instruction", DI.instruction) &&
          O.mapOptional("instructionBytes", DI.instructionBytes) &&
          O.mapOptional("symbol", DI.symbol) &&
          O.mapOptional("location", DI.location) &&
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
index f5e21c96fe17..c7acfc482987 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
@@ -414,6 +414,16 @@ bool fromJSON(const llvm::json::Value &, SteppingGranularity &,
               llvm::json::Path);
 llvm::json::Value toJSON(const SteppingGranularity &);
 
+/// A Thread.
+struct Thread {
+  /// Unique identifier for the thread.
+  lldb::tid_t id = LLDB_INVALID_THREAD_ID;
+  /// The name of the thread.
+  std::string name;
+};
+bool fromJSON(const llvm::json::Value &, Thread &, llvm::json::Path);
+llvm::json::Value toJSON(const Thread &);
+
 /// Provides formatting information for a value.
 struct ValueFormat {
   /// Display the value in hex.
@@ -637,7 +647,7 @@ struct DisassembledInstruction {
 
   /// The address of the instruction. Treated as a hex value if prefixed with
   /// `0x`, or as a decimal value otherwise.
-  lldb::addr_t address;
+  lldb::addr_t address = LLDB_INVALID_ADDRESS;
 
   /// Raw bytes representing the instruction and its operands, in an
   /// implementation-defined format.
@@ -677,8 +687,6 @@ struct DisassembledInstruction {
   /// addresses may be presented is 'invalid.'
   /// Values: 'normal', 'invalid'
   std::optional<PresentationHint> presentationHint;
-
-  DisassembledInstruction() : address(0) {}
 };
 bool fromJSON(const llvm::json::Value &,
               DisassembledInstruction::PresentationHint &, llvm::json::Path);
diff --git a/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.cpp b/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.cpp
index 4e47c87b7359..6e0adf5bc8b5 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.cpp
@@ -10,9 +10,15 @@
 #include "LLDBUtils.h"
 
 #include "lldb/API/SBDebugger.h"
+#include "lldb/API/SBFormat.h"
+#include "lldb/API/SBMutex.h"
+#include "lldb/API/SBStream.h"
 #include "lldb/API/SBTarget.h"
+#include "lldb/API/SBThread.h"
 #include "lldb/Host/PosixApi.h" // Adds PATH_MAX for windows
+#include <optional>
 
+using namespace lldb_dap::protocol;
 namespace lldb_dap {
 
 static bool ShouldDisplayAssemblySource(
@@ -110,4 +116,49 @@ std::string GetLoadAddressString(const lldb::addr_t addr) {
   return "0x" + llvm::utohexstr(addr, false, 16);
 }
 
+protocol::Thread CreateThread(lldb::SBThread &thread, lldb::SBFormat &format) {
+  std::string name;
+  lldb::SBStream stream;
+  if (format && thread.GetDescriptionWithFormat(format, stream).Success()) {
+    name = stream.GetData();
+  } else {
+    llvm::StringRef thread_name(thread.GetName());
+    llvm::StringRef queue_name(thread.GetQueueName());
+
+    if (!thread_name.empty()) {
+      name = thread_name.str();
+    } else if (!queue_name.empty()) {
+      auto kind = thread.GetQueue().GetKind();
+      std::string queue_kind_label = "";
+      if (kind == lldb::eQueueKindSerial)
+        queue_kind_label = " (serial)";
+      else if (kind == lldb::eQueueKindConcurrent)
+        queue_kind_label = " (concurrent)";
+
+      name = llvm::formatv("Thread {0} Queue: {1}{2}", thread.GetIndexID(),
+                           queue_name, queue_kind_label)
+                 .str();
+    } else {
+      name = llvm::formatv("Thread {0}", thread.GetIndexID()).str();
+    }
+  }
+  return protocol::Thread{thread.GetThreadID(), name};
+}
+
+std::vector<protocol::Thread> GetThreads(lldb::SBProcess process,
+                                         lldb::SBFormat &format) {
+  lldb::SBMutex lock = process.GetTarget().GetAPIMutex();
+  std::lock_guard<lldb::SBMutex> guard(lock);
+
+  std::vector<protocol::Thread> threads;
+
+  const uint32_t num_threads = process.GetNumThreads();
+  threads.reserve(num_threads);
+  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
+    lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
+    threads.emplace_back(CreateThread(thread, format));
+  }
+  return threads;
+}
+
 } // namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.h b/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.h
index 6e4f07d6e347..2b2ac9e8e35f 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/ProtocolUtils.h
@@ -48,6 +48,32 @@ bool IsAssemblySource(const protocol::Source &source);
 /// Get the address as a 16-digit hex string, e.g. "0x0000000000012345"
 std::string GetLoadAddressString(const lldb::addr_t addr);
 
+/// Create a "Thread" object for a LLDB thread object.
+///
+/// This function will fill in the following keys in the returned
+/// object:
+///   "id" - the thread ID as an integer
+///   "name" - the thread name as a string which combines the LLDB
+///            thread index ID along with the string name of the thread
+///            from the OS if it has a name.
+///
+/// \param[in] thread
+///     The LLDB thread to use when populating out the "Thread"
+///     object.
+///
+/// \param[in] format
+///     The LLDB format to use when populating out the "Thread"
+///     object.
+///
+/// \return
+///     A "Thread" JSON object with that follows the formal JSON
+///     definition outlined by Microsoft.
+protocol::Thread CreateThread(lldb::SBThread &thread, lldb::SBFormat &format);
+
+/// Returns the set of threads associated with the process.
+std::vector<protocol::Thread> GetThreads(lldb::SBProcess process,
+                                         lldb::SBFormat &format);
+
 } // namespace lldb_dap
 
 #endif
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Transport.cpp b/external/llvm-project/lldb/tools/lldb-dap/Transport.cpp
index 4e322e9ff135..d602920da34e 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Transport.cpp
+++ b/external/llvm-project/lldb/tools/lldb-dap/Transport.cpp
@@ -8,152 +8,19 @@
 
 #include "Transport.h"
 #include "DAPLog.h"
-#include "Protocol/ProtocolBase.h"
-#include "lldb/Utility/IOObject.h"
-#include "lldb/Utility/SelectHelper.h"
-#include "lldb/Utility/Status.h"
 #include "lldb/lldb-forward.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/raw_ostream.h"
-#include <optional>
-#include <string>
-#include <utility>
 
 using namespace llvm;
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_dap;
-using namespace lldb_dap::protocol;
 
-/// ReadFull attempts to read the specified number of bytes. If EOF is
-/// encountered, an empty string is returned.
-static Expected<std::string>
-ReadFull(IOObject &descriptor, size_t length,
-         std::optional<std::chrono::microseconds> timeout = std::nullopt) {
-  if (!descriptor.IsValid())
-    return createStringError("transport output is closed");
+Transport::Transport(llvm::StringRef client_name, lldb_dap::Log *log,
+                     lldb::IOObjectSP input, lldb::IOObjectSP output)
+    : HTTPDelimitedJSONTransport(input, output), m_client_name(client_name),
+      m_log(log) {}
 
-  bool timeout_supported = true;
-  // FIXME: SelectHelper does not work with NativeFile on Win32.
-#if _WIN32
-  timeout_supported = descriptor.GetFdType() == IOObject::eFDTypeSocket;
-#endif
-
-  if (timeout && timeout_supported) {
-    SelectHelper sh;
-    sh.SetTimeout(*timeout);
-    sh.FDSetRead(descriptor.GetWaitableHandle());
-    Status status = sh.Select();
-    if (status.Fail()) {
-      // Convert timeouts into a specific error.
-      if (status.GetType() == lldb::eErrorTypePOSIX &&
-          status.GetError() == ETIMEDOUT)
-        return make_error<TimeoutError>();
-      return status.takeError();
-    }
-  }
-
-  std::string data;
-  data.resize(length);
-  Status status = descriptor.Read(data.data(), length);
-  if (status.Fail())
-    return status.takeError();
-
-  // Read returns '' on EOF.
-  if (length == 0)
-    return make_error<EndOfFileError>();
-
-  // Return the actual number of bytes read.
-  return data.substr(0, length);
-}
-
-static Expected<std::string>
-ReadUntil(IOObject &descriptor, StringRef delimiter,
-          std::optional<std::chrono::microseconds> timeout = std::nullopt) {
-  std::string buffer;
-  buffer.reserve(delimiter.size() + 1);
-  while (!llvm::StringRef(buffer).ends_with(delimiter)) {
-    Expected<std::string> next =
-        ReadFull(descriptor, buffer.empty() ? delimiter.size() : 1, timeout);
-    if (auto Err = next.takeError())
-      return std::move(Err);
-    buffer += *next;
-  }
-  return buffer.substr(0, buffer.size() - delimiter.size());
-}
-
-/// DAP message format
-/// ```
-/// Content-Length: (?<length>\d+)\r\n\r\n(?<content>.{\k<length>})
-/// ```
-static constexpr StringLiteral kHeaderContentLength = "Content-Length: ";
-static constexpr StringLiteral kHeaderSeparator = "\r\n\r\n";
-
-namespace lldb_dap {
-
-char EndOfFileError::ID;
-char TimeoutError::ID;
-
-Transport::Transport(StringRef client_name, Log *log, IOObjectSP input,
-                     IOObjectSP output)
-    : m_client_name(client_name), m_log(log), m_input(std::move(input)),
-      m_output(std::move(output)) {}
-
-Expected<Message> Transport::Read(const std::chrono::microseconds &timeout) {
-  if (!m_input || !m_input->IsValid())
-    return createStringError("transport output is closed");
-
-  IOObject *input = m_input.get();
-  Expected<std::string> message_header =
-      ReadFull(*input, kHeaderContentLength.size(), timeout);
-  if (!message_header)
-    return message_header.takeError();
-  if (*message_header != kHeaderContentLength)
-    return createStringError(formatv("expected '{0}' and got '{1}'",
-                                     kHeaderContentLength, *message_header)
-                                 .str());
-
-  Expected<std::string> raw_length = ReadUntil(*input, kHeaderSeparator);
-  if (!raw_length)
-    return handleErrors(raw_length.takeError(),
-                        [&](const EndOfFileError &E) -> llvm::Error {
-                          return createStringError(
-                              "unexpected EOF while reading header separator");
-                        });
-
-  size_t length;
-  if (!to_integer(*raw_length, length))
-    return createStringError(
-        formatv("invalid content length {0}", *raw_length).str());
-
-  Expected<std::string> raw_json = ReadFull(*input, length);
-  if (!raw_json)
-    return handleErrors(
-        raw_json.takeError(), [&](const EndOfFileError &E) -> llvm::Error {
-          return createStringError("unexpected EOF while reading JSON");
-        });
-
-  DAP_LOG(m_log, "--> ({0}) {1}", m_client_name, *raw_json);
-
-  return json::parse<Message>(/*JSON=*/*raw_json,
-                              /*RootName=*/"protocol_message");
+void Transport::Log(llvm::StringRef message) {
+  DAP_LOG(m_log, "({0}) {1}", m_client_name, message);
 }
-
-Error Transport::Write(const Message &message) {
-  if (!m_output || !m_output->IsValid())
-    return createStringError("transport output is closed");
-
-  std::string json = formatv("{0}", toJSON(message)).str();
-
-  DAP_LOG(m_log, "<-- ({0}) {1}", m_client_name, json);
-
-  std::string Output;
-  raw_string_ostream OS(Output);
-  OS << kHeaderContentLength << json.length() << kHeaderSeparator << json;
-  size_t num_bytes = Output.size();
-  return m_output->Write(Output.data(), num_bytes).takeError();
-}
-
-} // end namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/Transport.h b/external/llvm-project/lldb/tools/lldb-dap/Transport.h
index 4e347eaa5131..51f62e718a0d 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/Transport.h
+++ b/external/llvm-project/lldb/tools/lldb-dap/Transport.h
@@ -15,70 +15,21 @@
 #define LLDB_TOOLS_LLDB_DAP_TRANSPORT_H
 
 #include "DAPForward.h"
-#include "Protocol/ProtocolBase.h"
+#include "lldb/Host/JSONTransport.h"
 #include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Error.h"
-#include <chrono>
-#include <system_error>
 
 namespace lldb_dap {
 
-class EndOfFileError : public llvm::ErrorInfo<EndOfFileError> {
-public:
-  static char ID;
-
-  EndOfFileError() = default;
-
-  void log(llvm::raw_ostream &OS) const override {
-    OS << "end of file reached";
-  }
-  std::error_code convertToErrorCode() const override {
-    return llvm::inconvertibleErrorCode();
-  }
-};
-
-class TimeoutError : public llvm::ErrorInfo<TimeoutError> {
-public:
-  static char ID;
-
-  TimeoutError() = default;
-
-  void log(llvm::raw_ostream &OS) const override {
-    OS << "operation timed out";
-  }
-  std::error_code convertToErrorCode() const override {
-    return std::make_error_code(std::errc::timed_out);
-  }
-};
-
 /// A transport class that performs the Debug Adapter Protocol communication
 /// with the client.
-class Transport {
+class Transport : public lldb_private::HTTPDelimitedJSONTransport {
 public:
-  Transport(llvm::StringRef client_name, Log *log, lldb::IOObjectSP input,
-            lldb::IOObjectSP output);
-  ~Transport() = default;
-
-  /// Transport is not copyable.
-  /// @{
-  Transport(const Transport &rhs) = delete;
-  void operator=(const Transport &rhs) = delete;
-  /// @}
-
-  /// Writes a Debug Adater Protocol message to the output stream.
-  llvm::Error Write(const protocol::Message &M);
+  Transport(llvm::StringRef client_name, lldb_dap::Log *log,
+            lldb::IOObjectSP input, lldb::IOObjectSP output);
+  virtual ~Transport() = default;
 
-  /// Reads the next Debug Adater Protocol message from the input stream.
-  ///
-  /// \param timeout[in]
-  ///     A timeout to wait for reading the initial header. Once a message
-  ///     header is recieved, this will block until the full message is
-  ///     read.
-  ///
-  /// \returns Returns the next protocol message.
-  llvm::Expected<protocol::Message>
-  Read(const std::chrono::microseconds &timeout);
+  virtual void Log(llvm::StringRef message) override;
 
   /// Returns the name of this transport client, for example `stdin/stdout` or
   /// `client_1`.
@@ -86,9 +37,7 @@ class Transport {
 
 private:
   llvm::StringRef m_client_name;
-  Log *m_log;
-  lldb::IOObjectSP m_input;
-  lldb::IOObjectSP m_output;
+  lldb_dap::Log *m_log;
 };
 
 } // namespace lldb_dap
diff --git a/external/llvm-project/lldb/tools/lldb-dap/package.json b/external/llvm-project/lldb/tools/lldb-dap/package.json
index 20d418d727bb..0f51c4f935e3 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/package.json
+++ b/external/llvm-project/lldb/tools/lldb-dap/package.json
@@ -1,7 +1,7 @@
 {
   "name": "lldb-dap",
   "displayName": "LLDB DAP",
-  "version": "0.2.14",
+  "version": "0.2.15",
   "publisher": "llvm-vs-code-extensions",
   "homepage": "https://lldb.llvm.org",
   "description": "Debugging with LLDB in Visual Studio Code",
diff --git a/external/llvm-project/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts b/external/llvm-project/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
index d61b81e4c041..b5db45b56d6a 100644
--- a/external/llvm-project/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
+++ b/external/llvm-project/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
@@ -169,7 +169,7 @@ export async function createDebugAdapterExecutable(
   workspaceFolder: vscode.WorkspaceFolder | undefined,
   configuration: vscode.DebugConfiguration,
 ): Promise<vscode.DebugAdapterExecutable> {
-  const config = vscode.workspace.getConfiguration("lldb-dap", workspaceFolder);
+  const config = vscode.workspace.workspaceFile ? vscode.workspace.getConfiguration("lldb-dap") : vscode.workspace.getConfiguration("lldb-dap", workspaceFolder);
   const log_path = config.get<string>("log-path");
   let env: { [key: string]: string } = {};
   if (log_path) {
@@ -184,7 +184,7 @@ export async function createDebugAdapterExecutable(
       ...configEnvironment,
       ...env,
     },
-    cwd: workspaceFolder!!.uri.fsPath,
+    cwd: configuration.cwd ?? workspaceFolder?.uri.fsPath,
   };
   const dbgArgs = await getDAPArguments(workspaceFolder, configuration);
 
diff --git a/external/llvm-project/lldb/tools/lldb-instr/CMakeLists.txt b/external/llvm-project/lldb/tools/lldb-instr/CMakeLists.txt
index 8da453b2894f..17d6674707af 100644
--- a/external/llvm-project/lldb/tools/lldb-instr/CMakeLists.txt
+++ b/external/llvm-project/lldb/tools/lldb-instr/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_lldb_tool(lldb-instr
   Instrument.cpp
 
+  LINK_COMPONENTS
+    Support
   CLANG_LIBS
     clangAST
     clangBasic
@@ -10,7 +12,4 @@ add_lldb_tool(lldb-instr
     clangRewrite
     clangSerialization
     clangTooling
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/tools/lldb-server/CMakeLists.txt b/external/llvm-project/lldb/tools/lldb-server/CMakeLists.txt
index 0135b367fcc2..1d8dc72a3f87 100644
--- a/external/llvm-project/lldb/tools/lldb-server/CMakeLists.txt
+++ b/external/llvm-project/lldb/tools/lldb-server/CMakeLists.txt
@@ -49,6 +49,9 @@ add_lldb_tool(lldb-server
     LLDBServerUtilities.cpp
     SystemInitializerLLGS.cpp
 
+    LINK_COMPONENTS
+      Option
+      Support
     LINK_LIBS
       lldbHost
       lldbInitialization
@@ -60,10 +63,6 @@ add_lldb_tool(lldb-server
       lldbPluginInstructionMIPS64
       lldbPluginInstructionRISCV
       ${LLDB_SYSTEM_LIBS}
-
-    LINK_COMPONENTS
-      Option
-      Support
 )
 
 add_dependencies(lldb-server
diff --git a/external/llvm-project/lldb/tools/lldb-test/CMakeLists.txt b/external/llvm-project/lldb/tools/lldb-test/CMakeLists.txt
index 120e3aaa9d20..9d85cb8f8d16 100644
--- a/external/llvm-project/lldb/tools/lldb-test/CMakeLists.txt
+++ b/external/llvm-project/lldb/tools/lldb-test/CMakeLists.txt
@@ -5,6 +5,8 @@ add_lldb_tool(lldb-test
   lldb-test.cpp
   SystemInitializerTest.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
@@ -18,9 +20,6 @@ add_lldb_tool(lldb-test
     lldbUtility
     lldbVersion
     ${LLDB_ALL_PLUGINS}
-
-  LINK_COMPONENTS
-    Support
   )
 
 if(Python3_RPATH)
diff --git a/external/llvm-project/lldb/unittests/ABI/AArch64/ABIAArch64Test.cpp b/external/llvm-project/lldb/unittests/ABI/AArch64/ABIAArch64Test.cpp
new file mode 100644
index 000000000000..5f9332e3e85b
--- /dev/null
+++ b/external/llvm-project/lldb/unittests/ABI/AArch64/ABIAArch64Test.cpp
@@ -0,0 +1,71 @@
+//===-- ABIAArch64Test.cpp ------------------------------------------------===//
+
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/ABI/AArch64/ABIMacOSX_arm64.h"
+#include "Plugins/ABI/AArch64/ABISysV_arm64.h"
+#include "Utility/ARM64_DWARF_Registers.h"
+#include "Utility/ARM64_ehframe_Registers.h"
+#include "lldb/Target/DynamicRegisterInfo.h"
+#include "lldb/Utility/ArchSpec.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gtest/gtest.h"
+#include <vector>
+
+using namespace lldb_private;
+using namespace lldb;
+
+class ABIAArch64TestFixture : public testing::TestWithParam<llvm::StringRef> {
+public:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+
+  //  virtual void SetUp() override { }
+  //  virtual void TearDown() override { }
+
+protected:
+};
+
+void ABIAArch64TestFixture::SetUpTestCase() {
+  LLVMInitializeAArch64TargetInfo();
+  LLVMInitializeAArch64TargetMC();
+  ABISysV_arm64::Initialize();
+  ABIMacOSX_arm64::Initialize();
+}
+
+void ABIAArch64TestFixture::TearDownTestCase() {
+  ABISysV_arm64::Terminate();
+  ABIMacOSX_arm64::Terminate();
+  llvm::llvm_shutdown();
+}
+
+TEST_P(ABIAArch64TestFixture, AugmentRegisterInfo) {
+  ABISP abi_sp = ABI::FindPlugin(ProcessSP(), ArchSpec(GetParam()));
+  ASSERT_TRUE(abi_sp);
+  using Register = DynamicRegisterInfo::Register;
+
+  Register pc;
+  pc.name = ConstString("pc");
+  pc.alt_name = ConstString();
+  pc.set_name = ConstString("GPR");
+  std::vector<Register> regs{pc};
+
+  abi_sp->AugmentRegisterInfo(regs);
+
+  ASSERT_EQ(regs.size(), 1U);
+  Register new_pc = regs[0];
+  EXPECT_EQ(new_pc.name, pc.name);
+  EXPECT_EQ(new_pc.set_name, pc.set_name);
+  EXPECT_EQ(new_pc.regnum_ehframe, arm64_ehframe::pc);
+  EXPECT_EQ(new_pc.regnum_dwarf, arm64_dwarf::pc);
+}
+
+INSTANTIATE_TEST_SUITE_P(ABIAArch64Tests, ABIAArch64TestFixture,
+                         testing::Values("aarch64-pc-linux",
+                                         "arm64-apple-macosx"));
diff --git a/external/llvm-project/lldb/unittests/ABI/AArch64/CMakeLists.txt b/external/llvm-project/lldb/unittests/ABI/AArch64/CMakeLists.txt
new file mode 100644
index 000000000000..592701fc4075
--- /dev/null
+++ b/external/llvm-project/lldb/unittests/ABI/AArch64/CMakeLists.txt
@@ -0,0 +1,9 @@
+add_lldb_unittest(ABIAArch64Tests
+  ABIAArch64Test.cpp
+  LINK_COMPONENTS
+    Support
+    AArch64
+  LINK_LIBS
+    lldbTarget
+    lldbPluginABIAArch64
+  )
diff --git a/external/llvm-project/lldb/unittests/ABI/CMakeLists.txt b/external/llvm-project/lldb/unittests/ABI/CMakeLists.txt
new file mode 100644
index 000000000000..8ad7474e9444
--- /dev/null
+++ b/external/llvm-project/lldb/unittests/ABI/CMakeLists.txt
@@ -0,0 +1,3 @@
+if ("AArch64" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(AArch64)
+endif()
diff --git a/external/llvm-project/lldb/unittests/Breakpoint/CMakeLists.txt b/external/llvm-project/lldb/unittests/Breakpoint/CMakeLists.txt
index 757c2da1a4d9..3c234a4fea29 100644
--- a/external/llvm-project/lldb/unittests/Breakpoint/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Breakpoint/CMakeLists.txt
@@ -2,9 +2,9 @@ add_lldb_unittest(LLDBBreakpointTests
   BreakpointIDTest.cpp
   WatchpointAlgorithmsTests.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/CMakeLists.txt b/external/llvm-project/lldb/unittests/CMakeLists.txt
index d8f9cc747986..6eaaa4f4c8c9 100644
--- a/external/llvm-project/lldb/unittests/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/CMakeLists.txt
@@ -52,6 +52,7 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
   add_subdirectory(API)
   add_subdirectory(DAP)
 endif()
+add_subdirectory(ABI)
 add_subdirectory(Breakpoint)
 add_subdirectory(Callback)
 add_subdirectory(Core)
diff --git a/external/llvm-project/lldb/unittests/Callback/CMakeLists.txt b/external/llvm-project/lldb/unittests/Callback/CMakeLists.txt
index b9e0ef5a396e..e528663ffa51 100644
--- a/external/llvm-project/lldb/unittests/Callback/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Callback/CMakeLists.txt
@@ -1,12 +1,12 @@
 add_lldb_unittest(LLDBCallbackTests
   TestBreakpointSetCallback.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbBreakpoint
     lldbCore
     LLVMTestingSupport
     lldbUtilityHelpers
     lldbPluginPlatformMacOSX
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/Core/CMakeLists.txt b/external/llvm-project/lldb/unittests/Core/CMakeLists.txt
index 97268b3dbf8f..a6820bdc7d7f 100644
--- a/external/llvm-project/lldb/unittests/Core/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Core/CMakeLists.txt
@@ -16,6 +16,9 @@ add_lldb_unittest(LLDBCoreTests
   TelemetryTest.cpp
   UniqueCStringMapTest.cpp
 
+  LINK_COMPONENTS
+    Support
+    Telemetry
   LINK_LIBS
     lldbCore
     lldbHost
@@ -29,7 +32,4 @@ add_lldb_unittest(LLDBCoreTests
     lldbUtilityHelpers
     lldbHostHelpers
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
-    Telemetry
   )
diff --git a/external/llvm-project/lldb/unittests/Core/DebuggerTest.cpp b/external/llvm-project/lldb/unittests/Core/DebuggerTest.cpp
index df7d99978855..4dccd912c63a 100644
--- a/external/llvm-project/lldb/unittests/Core/DebuggerTest.cpp
+++ b/external/llvm-project/lldb/unittests/Core/DebuggerTest.cpp
@@ -46,7 +46,7 @@ TEST_F(DebuggerTest, TestSettings) {
 
   FormatEntity::Entry format("foo");
   EXPECT_TRUE(debugger_sp->SetStatuslineFormat(format));
-  EXPECT_EQ(debugger_sp->GetStatuslineFormat()->string, "foo");
+  EXPECT_EQ(debugger_sp->GetStatuslineFormat().string, "foo");
 
   Debugger::Destroy(debugger_sp);
 }
diff --git a/external/llvm-project/lldb/unittests/Core/FormatEntityTest.cpp b/external/llvm-project/lldb/unittests/Core/FormatEntityTest.cpp
index 2cddf8ff4e90..e056b6fe7de5 100644
--- a/external/llvm-project/lldb/unittests/Core/FormatEntityTest.cpp
+++ b/external/llvm-project/lldb/unittests/Core/FormatEntityTest.cpp
@@ -129,6 +129,7 @@ constexpr llvm::StringRef lookupStrings[] = {
     "${function.initial-function}",
     "${function.changed}",
     "${function.is-optimized}",
+    "${function.is-inlined}",
     "${line.file.basename}",
     "${line.file.dirname}",
     "${line.file.fullpath}",
diff --git a/external/llvm-project/lldb/unittests/Core/MangledTest.cpp b/external/llvm-project/lldb/unittests/Core/MangledTest.cpp
index dfdc02652137..46adb6272209 100644
--- a/external/llvm-project/lldb/unittests/Core/MangledTest.cpp
+++ b/external/llvm-project/lldb/unittests/Core/MangledTest.cpp
@@ -26,9 +26,31 @@
 
 #include "gtest/gtest.h"
 
+#include <cstdlib>
+#include <memory>
+
 using namespace lldb;
 using namespace lldb_private;
 
+/// Custom deleter to use with unique_ptr.
+///
+/// Usage:
+/// \code{.cpp}
+///
+/// auto OB =
+///     std::unique_ptr<TrackingOutputBuffer, TrackingOutputBufferDeleter>(
+///         new TrackingOutputBuffer());
+///
+/// \endcode
+struct TrackingOutputBufferDeleter {
+  void operator()(TrackingOutputBuffer *TOB) {
+    if (!TOB)
+      return;
+    std::free(TOB->getBuffer());
+    delete TOB;
+  }
+};
+
 TEST(MangledTest, ResultForValidName) {
   ConstString MangledName("_ZN1a1b1cIiiiEEvm");
   Mangled TheMangled(MangledName);
@@ -589,25 +611,25 @@ TEST_P(DemanglingPartsTestFixture, DemanglingParts) {
 
   ASSERT_NE(nullptr, Root);
 
-  TrackingOutputBuffer OB;
-  Root->print(OB);
-  auto demangled = std::string_view(OB);
+  auto OB = std::unique_ptr<TrackingOutputBuffer, TrackingOutputBufferDeleter>(
+      new TrackingOutputBuffer());
+  Root->print(*OB);
+  auto demangled = std::string_view(*OB);
 
-  ASSERT_EQ(OB.NameInfo.hasBasename(), valid_basename);
+  ASSERT_EQ(OB->NameInfo.hasBasename(), valid_basename);
 
-  EXPECT_EQ(OB.NameInfo.BasenameRange, info.BasenameRange);
-  EXPECT_EQ(OB.NameInfo.ScopeRange, info.ScopeRange);
-  EXPECT_EQ(OB.NameInfo.ArgumentsRange, info.ArgumentsRange);
-  EXPECT_EQ(OB.NameInfo.QualifiersRange, info.QualifiersRange);
+  EXPECT_EQ(OB->NameInfo.BasenameRange, info.BasenameRange);
+  EXPECT_EQ(OB->NameInfo.ScopeRange, info.ScopeRange);
+  EXPECT_EQ(OB->NameInfo.ArgumentsRange, info.ArgumentsRange);
+  EXPECT_EQ(OB->NameInfo.QualifiersRange, info.QualifiersRange);
 
   auto get_part = [&](const std::pair<size_t, size_t> &loc) {
     return demangled.substr(loc.first, loc.second - loc.first);
   };
 
-  EXPECT_EQ(get_part(OB.NameInfo.BasenameRange), basename);
-  EXPECT_EQ(get_part(OB.NameInfo.ScopeRange), scope);
-  EXPECT_EQ(get_part(OB.NameInfo.QualifiersRange), qualifiers);
-  std::free(OB.getBuffer());
+  EXPECT_EQ(get_part(OB->NameInfo.BasenameRange), basename);
+  EXPECT_EQ(get_part(OB->NameInfo.ScopeRange), scope);
+  EXPECT_EQ(get_part(OB->NameInfo.QualifiersRange), qualifiers);
 }
 
 INSTANTIATE_TEST_SUITE_P(DemanglingPartsTests, DemanglingPartsTestFixture,
@@ -635,8 +657,9 @@ TEST_P(DemanglingInfoCorrectnessTestFixutre, Correctness) {
 
   ASSERT_NE(nullptr, Root);
 
-  TrackingOutputBuffer OB;
-  Root->print(OB);
+  auto OB = std::unique_ptr<TrackingOutputBuffer, TrackingOutputBufferDeleter>(
+      new TrackingOutputBuffer());
+  Root->print(*OB);
 
   // Filter out cases which would never show up in frames. We only care about
   // function names.
@@ -645,24 +668,24 @@ TEST_P(DemanglingInfoCorrectnessTestFixutre, Correctness) {
       Root->getKind() != llvm::itanium_demangle::Node::Kind::KDotSuffix)
     return;
 
-  ASSERT_TRUE(OB.NameInfo.hasBasename());
-
-  auto tracked_name = llvm::StringRef(OB);
-
-  auto return_left = tracked_name.slice(0, OB.NameInfo.ScopeRange.first);
-  auto scope = tracked_name.slice(OB.NameInfo.ScopeRange.first,
-                                  OB.NameInfo.ScopeRange.second);
-  auto basename = tracked_name.slice(OB.NameInfo.BasenameRange.first,
-                                     OB.NameInfo.BasenameRange.second);
-  auto template_args = tracked_name.slice(OB.NameInfo.BasenameRange.second,
-                                          OB.NameInfo.ArgumentsRange.first);
-  auto args = tracked_name.slice(OB.NameInfo.ArgumentsRange.first,
-                                 OB.NameInfo.ArgumentsRange.second);
-  auto return_right = tracked_name.slice(OB.NameInfo.ArgumentsRange.second,
-                                         OB.NameInfo.QualifiersRange.first);
-  auto qualifiers = tracked_name.slice(OB.NameInfo.QualifiersRange.first,
-                                       OB.NameInfo.QualifiersRange.second);
-  auto suffix = tracked_name.slice(OB.NameInfo.QualifiersRange.second,
+  ASSERT_TRUE(OB->NameInfo.hasBasename());
+
+  auto tracked_name = llvm::StringRef(*OB);
+
+  auto return_left = tracked_name.slice(0, OB->NameInfo.ScopeRange.first);
+  auto scope = tracked_name.slice(OB->NameInfo.ScopeRange.first,
+                                  OB->NameInfo.ScopeRange.second);
+  auto basename = tracked_name.slice(OB->NameInfo.BasenameRange.first,
+                                     OB->NameInfo.BasenameRange.second);
+  auto template_args = tracked_name.slice(OB->NameInfo.BasenameRange.second,
+                                          OB->NameInfo.ArgumentsRange.first);
+  auto args = tracked_name.slice(OB->NameInfo.ArgumentsRange.first,
+                                 OB->NameInfo.ArgumentsRange.second);
+  auto return_right = tracked_name.slice(OB->NameInfo.ArgumentsRange.second,
+                                         OB->NameInfo.QualifiersRange.first);
+  auto qualifiers = tracked_name.slice(OB->NameInfo.QualifiersRange.first,
+                                       OB->NameInfo.QualifiersRange.second);
+  auto suffix = tracked_name.slice(OB->NameInfo.QualifiersRange.second,
                                    llvm::StringRef::npos);
 
   auto reconstructed_name =
diff --git a/external/llvm-project/lldb/unittests/Core/PluginManagerTest.cpp b/external/llvm-project/lldb/unittests/Core/PluginManagerTest.cpp
index 9b0ce2286d27..c7a56b28a42f 100644
--- a/external/llvm-project/lldb/unittests/Core/PluginManagerTest.cpp
+++ b/external/llvm-project/lldb/unittests/Core/PluginManagerTest.cpp
@@ -379,3 +379,93 @@ TEST_F(PluginManagerTest, UnRegisterSystemRuntimePluginChangesOrder) {
   ASSERT_EQ(PluginManager::GetSystemRuntimeCreateCallbackAtIndex(2),
             CreateSystemRuntimePluginB);
 }
+
+TEST_F(PluginManagerTest, MatchPluginName) {
+  PluginNamespace Foo{"foo", nullptr, nullptr};
+  RegisteredPluginInfo Bar{"bar", "bar plugin ", true};
+  RegisteredPluginInfo Baz{"baz", "baz plugin ", true};
+
+  // Empty pattern matches everything.
+  ASSERT_TRUE(PluginManager::MatchPluginName("", Foo, Bar));
+
+  // Plugin namespace matches all plugins in that namespace.
+  ASSERT_TRUE(PluginManager::MatchPluginName("foo", Foo, Bar));
+  ASSERT_TRUE(PluginManager::MatchPluginName("foo", Foo, Baz));
+
+  // Fully qualified plugin name matches only that plugin.
+  ASSERT_TRUE(PluginManager::MatchPluginName("foo.bar", Foo, Bar));
+  ASSERT_FALSE(PluginManager::MatchPluginName("foo.baz", Foo, Bar));
+
+  // Prefix match should not match.
+  ASSERT_FALSE(PluginManager::MatchPluginName("f", Foo, Bar));
+  ASSERT_FALSE(PluginManager::MatchPluginName("foo.", Foo, Bar));
+  ASSERT_FALSE(PluginManager::MatchPluginName("foo.ba", Foo, Bar));
+}
+
+TEST_F(PluginManagerTest, JsonFormat) {
+  RegisterMockSystemRuntimePlugins();
+
+  // We expect the following JSON output:
+  // {
+  //   "system-runtime": [
+  //     {
+  //       "enabled": true,
+  //       "name": "a"
+  //     },
+  //     {
+  //       "enabled": true,
+  //       "name": "b"
+  //     },
+  //     {
+  //       "enabled": true,
+  //       "name": "c"
+  //     }
+  //   ]
+  // }
+  llvm::json::Object obj = PluginManager::GetJSON();
+
+  // We should have a "system-runtime" array in the top-level object.
+  llvm::json::Array *maybe_array = obj.getArray("system-runtime");
+  ASSERT_TRUE(maybe_array != nullptr);
+  auto &array = *maybe_array;
+  ASSERT_EQ(array.size(), 3u);
+
+  // Check plugin "a" info.
+  ASSERT_TRUE(array[0].getAsObject() != nullptr);
+  ASSERT_TRUE(array[0].getAsObject()->getString("name") == "a");
+  ASSERT_TRUE(array[0].getAsObject()->getBoolean("enabled") == true);
+
+  // Check plugin "b" info.
+  ASSERT_TRUE(array[1].getAsObject() != nullptr);
+  ASSERT_TRUE(array[1].getAsObject()->getString("name") == "b");
+  ASSERT_TRUE(array[1].getAsObject()->getBoolean("enabled") == true);
+
+  // Check plugin "c" info.
+  ASSERT_TRUE(array[2].getAsObject() != nullptr);
+  ASSERT_TRUE(array[2].getAsObject()->getString("name") == "c");
+  ASSERT_TRUE(array[2].getAsObject()->getBoolean("enabled") == true);
+
+  // Disabling a plugin should be reflected in the JSON output.
+  ASSERT_TRUE(PluginManager::SetSystemRuntimePluginEnabled("b", false));
+  array = *PluginManager::GetJSON().getArray("system-runtime");
+  ASSERT_TRUE(array[0].getAsObject()->getBoolean("enabled") == true);
+  ASSERT_TRUE(array[1].getAsObject()->getBoolean("enabled") == false);
+  ASSERT_TRUE(array[2].getAsObject()->getBoolean("enabled") == true);
+
+  // Un-registering a plugin should be reflected in the JSON output.
+  ASSERT_TRUE(PluginManager::UnregisterPlugin(CreateSystemRuntimePluginB));
+  array = *PluginManager::GetJSON().getArray("system-runtime");
+  ASSERT_EQ(array.size(), 2u);
+  ASSERT_TRUE(array[0].getAsObject()->getString("name") == "a");
+  ASSERT_TRUE(array[1].getAsObject()->getString("name") == "c");
+
+  // Filtering the JSON output should only include the matching plugins.
+  array =
+      *PluginManager::GetJSON("system-runtime.c").getArray("system-runtime");
+  ASSERT_EQ(array.size(), 1u);
+  ASSERT_TRUE(array[0].getAsObject()->getString("name") == "c");
+
+  // Empty JSON output is allowed if there are no matching plugins.
+  obj = PluginManager::GetJSON("non-existent-plugin");
+  ASSERT_TRUE(obj.empty());
+}
diff --git a/external/llvm-project/lldb/unittests/DAP/CMakeLists.txt b/external/llvm-project/lldb/unittests/DAP/CMakeLists.txt
index d8576ff3f371..ee623d341ec6 100644
--- a/external/llvm-project/lldb/unittests/DAP/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/DAP/CMakeLists.txt
@@ -7,16 +7,15 @@ add_lldb_unittest(DAPTests
   LLDBUtilsTest.cpp
   ProtocolTypesTest.cpp
   TestBase.cpp
-  TransportTest.cpp
   VariablesTest.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     liblldb
     lldbDAP
     lldbUtilityHelpers
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
   )
 
 set(test_inputs
diff --git a/external/llvm-project/lldb/unittests/DAP/DAPTest.cpp b/external/llvm-project/lldb/unittests/DAP/DAPTest.cpp
index 5fb6bf7e564a..40ffaf87c9c4 100644
--- a/external/llvm-project/lldb/unittests/DAP/DAPTest.cpp
+++ b/external/llvm-project/lldb/unittests/DAP/DAPTest.cpp
@@ -32,7 +32,8 @@ TEST_F(DAPTest, SendProtocolMessages) {
       /*transport=*/*to_dap,
   };
   dap.Send(Event{/*event=*/"my-event", /*body=*/std::nullopt});
-  ASSERT_THAT_EXPECTED(from_dap->Read(std::chrono::milliseconds(1)),
-                       HasValue(testing::VariantWith<Event>(testing::FieldsAre(
-                           /*event=*/"my-event", /*body=*/std::nullopt))));
+  ASSERT_THAT_EXPECTED(
+      from_dap->Read<protocol::Message>(std::chrono::milliseconds(1)),
+      HasValue(testing::VariantWith<Event>(testing::FieldsAre(
+          /*event=*/"my-event", /*body=*/std::nullopt))));
 }
diff --git a/external/llvm-project/lldb/unittests/DAP/ProtocolTypesTest.cpp b/external/llvm-project/lldb/unittests/DAP/ProtocolTypesTest.cpp
index 41703f4a071f..adf43c9ac204 100644
--- a/external/llvm-project/lldb/unittests/DAP/ProtocolTypesTest.cpp
+++ b/external/llvm-project/lldb/unittests/DAP/ProtocolTypesTest.cpp
@@ -7,12 +7,26 @@
 //===----------------------------------------------------------------------===//
 
 #include "Protocol/ProtocolTypes.h"
+#include "Protocol/ProtocolEvents.h"
+#include "Protocol/ProtocolRequests.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <optional>
 
+using namespace llvm;
 using namespace lldb;
 using namespace lldb_dap;
 using namespace lldb_dap::protocol;
+using llvm::json::parse;
+using llvm::json::Value;
+
+/// Returns a pretty printed json string of a `llvm::json::Value`.
+static std::string pp(const json::Value &E) {
+  return formatv("{0:2}", E).str();
+}
 
 template <typename T> static llvm::Expected<T> roundtrip(const T &input) {
   llvm::json::Value value = toJSON(input);
@@ -578,27 +592,97 @@ TEST(ProtocolTypesTest, DisassembledInstruction) {
   instruction.presentationHint =
       DisassembledInstruction::eDisassembledInstructionPresentationHintNormal;
 
-  llvm::Expected<DisassembledInstruction> deserialized_instruction =
-      roundtrip(instruction);
-  ASSERT_THAT_EXPECTED(deserialized_instruction, llvm::Succeeded());
-
-  EXPECT_EQ(instruction.address, deserialized_instruction->address);
-  EXPECT_EQ(instruction.instructionBytes,
-            deserialized_instruction->instructionBytes);
-  EXPECT_EQ(instruction.instruction, deserialized_instruction->instruction);
-  EXPECT_EQ(instruction.symbol, deserialized_instruction->symbol);
-  EXPECT_EQ(instruction.location->name,
-            deserialized_instruction->location->name);
-  EXPECT_EQ(instruction.location->path,
-            deserialized_instruction->location->path);
-  EXPECT_EQ(instruction.location->sourceReference,
-            deserialized_instruction->location->sourceReference);
-  EXPECT_EQ(instruction.location->presentationHint,
-            deserialized_instruction->location->presentationHint);
-  EXPECT_EQ(instruction.line, deserialized_instruction->line);
-  EXPECT_EQ(instruction.column, deserialized_instruction->column);
-  EXPECT_EQ(instruction.endLine, deserialized_instruction->endLine);
-  EXPECT_EQ(instruction.endColumn, deserialized_instruction->endColumn);
-  EXPECT_EQ(instruction.presentationHint,
-            deserialized_instruction->presentationHint);
+  StringLiteral json = R"({
+  "address": "0x12345678",
+  "column": 5,
+  "endColumn": 10,
+  "endLine": 15,
+  "instruction": "mov eax, ebx",
+  "instructionBytes": "0F 1F 00",
+  "line": 10,
+  "location": {
+    "name": "test.cpp",
+    "path": "/path/to/test.cpp",
+    "presentationHint": "normal",
+    "sourceReference": 123
+  },
+  "presentationHint": "normal",
+  "symbol": "main"
+})";
+
+  // Validate toJSON
+  EXPECT_EQ(json, pp(instruction));
+
+  // Validate fromJSON
+  EXPECT_THAT_EXPECTED(parse<DisassembledInstruction>(json),
+                       HasValue(Value(instruction)));
+  // Validate parsing errors
+  EXPECT_THAT_EXPECTED(
+      parse<DisassembledInstruction>(R"({"address":1})",
+                                     "disassemblyInstruction"),
+      FailedWithMessage("expected string at disassemblyInstruction.address"));
+  EXPECT_THAT_EXPECTED(parse<DisassembledInstruction>(R"({"address":"-1"})",
+                                                      "disassemblyInstruction"),
+                       FailedWithMessage("expected string encoded uint64_t at "
+                                         "disassemblyInstruction.address"));
+  EXPECT_THAT_EXPECTED(parse<DisassembledInstruction>(
+                           R"({"address":"0xfffffffffffffffffffffffffff"})",
+                           "disassemblyInstruction"),
+                       FailedWithMessage("expected string encoded uint64_t at "
+                                         "disassemblyInstruction.address"));
+}
+
+TEST(ProtocolTypesTest, Thread) {
+  const Thread thread{1, "thr1"};
+  const StringRef json = R"({
+  "id": 1,
+  "name": "thr1"
+})";
+  // Validate toJSON
+  EXPECT_EQ(json, pp(thread));
+  // Validate fromJSON
+  EXPECT_THAT_EXPECTED(parse<Thread>(json), HasValue(Value(thread)));
+  // Validate parsing errors
+  EXPECT_THAT_EXPECTED(parse<Thread>(R"({"id":1})", "thread"),
+                       FailedWithMessage("missing value at thread.name"));
+  EXPECT_THAT_EXPECTED(parse<Thread>(R"({"id":"one"})", "thread"),
+                       FailedWithMessage("expected uint64_t at thread.id"));
+  EXPECT_THAT_EXPECTED(parse<Thread>(R"({"id":1,"name":false})", "thread"),
+                       FailedWithMessage("expected string at thread.name"));
+}
+
+TEST(ProtocolTypesTest, ThreadResponseBody) {
+  const ThreadsResponseBody body{{{1, "thr1"}, {2, "thr2"}}};
+  const StringRef json = R"({
+  "threads": [
+    {
+      "id": 1,
+      "name": "thr1"
+    },
+    {
+      "id": 2,
+      "name": "thr2"
+    }
+  ]
+})";
+  // Validate toJSON
+  EXPECT_EQ(json, pp(body));
+}
+
+TEST(ProtocolTypesTest, CapabilitiesEventBody) {
+  Capabilities capabilities;
+  capabilities.supportedFeatures = {
+      eAdapterFeatureANSIStyling,
+      eAdapterFeatureBreakpointLocationsRequest,
+  };
+  CapabilitiesEventBody body;
+  body.capabilities = capabilities;
+  StringRef json = R"({
+  "capabilities": {
+    "supportsANSIStyling": true,
+    "supportsBreakpointLocationsRequest": true
+  }
+})";
+  // Validate toJSON
+  EXPECT_EQ(json, pp(body));
 }
diff --git a/external/llvm-project/lldb/unittests/DAP/TestBase.cpp b/external/llvm-project/lldb/unittests/DAP/TestBase.cpp
index 388d1b901507..27ad42686fbb 100644
--- a/external/llvm-project/lldb/unittests/DAP/TestBase.cpp
+++ b/external/llvm-project/lldb/unittests/DAP/TestBase.cpp
@@ -28,13 +28,8 @@ using lldb_private::File;
 using lldb_private::NativeFile;
 using lldb_private::Pipe;
 
-void PipeBase::SetUp() {
-  ASSERT_THAT_ERROR(input.CreateNew(false).ToError(), Succeeded());
-  ASSERT_THAT_ERROR(output.CreateNew(false).ToError(), Succeeded());
-}
-
 void TransportBase::SetUp() {
-  PipeBase::SetUp();
+  PipeTest::SetUp();
   to_dap = std::make_unique<Transport>(
       "to_dap", nullptr,
       std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
@@ -122,7 +117,8 @@ std::vector<Message> DAPTestBase::DrainOutput() {
   std::vector<Message> msgs;
   output.CloseWriteFileDescriptor();
   while (true) {
-    Expected<Message> next = from_dap->Read(std::chrono::milliseconds(1));
+    Expected<Message> next =
+        from_dap->Read<protocol::Message>(std::chrono::milliseconds(1));
     if (!next) {
       consumeError(next.takeError());
       break;
diff --git a/external/llvm-project/lldb/unittests/DAP/TestBase.h b/external/llvm-project/lldb/unittests/DAP/TestBase.h
index 70b3985271a9..25d37013954d 100644
--- a/external/llvm-project/lldb/unittests/DAP/TestBase.h
+++ b/external/llvm-project/lldb/unittests/DAP/TestBase.h
@@ -8,26 +8,17 @@
 
 #include "DAP.h"
 #include "Protocol/ProtocolBase.h"
+#include "TestingSupport/Host/PipeTestUtilities.h"
 #include "Transport.h"
-#include "lldb/Host/Pipe.h"
 #include "llvm/ADT/StringRef.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 namespace lldb_dap_tests {
 
-/// A base class for tests that need a pair of pipes for communication.
-class PipeBase : public testing::Test {
-protected:
-  lldb_private::Pipe input;
-  lldb_private::Pipe output;
-
-  void SetUp() override;
-};
-
 /// A base class for tests that need transport configured for communicating DAP
 /// messages.
-class TransportBase : public PipeBase {
+class TransportBase : public PipeTest {
 protected:
   std::unique_ptr<lldb_dap::Transport> to_dap;
   std::unique_ptr<lldb_dap::Transport> from_dap;
diff --git a/external/llvm-project/lldb/unittests/DAP/TransportTest.cpp b/external/llvm-project/lldb/unittests/DAP/TransportTest.cpp
deleted file mode 100644
index e6dab42e3094..000000000000
--- a/external/llvm-project/lldb/unittests/DAP/TransportTest.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-//===-- TransportTest.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Transport.h"
-#include "Protocol/ProtocolBase.h"
-#include "TestBase.h"
-#include "lldb/Host/File.h"
-#include "lldb/Host/Pipe.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Testing/Support/Error.h"
-#include "gtest/gtest.h"
-#include <chrono>
-#include <memory>
-#include <optional>
-
-using namespace llvm;
-using namespace lldb;
-using namespace lldb_dap;
-using namespace lldb_dap_tests;
-using namespace lldb_dap::protocol;
-using lldb_private::File;
-using lldb_private::NativeFile;
-using lldb_private::Pipe;
-
-class TransportTest : public PipeBase {
-protected:
-  std::unique_ptr<Transport> transport;
-
-  void SetUp() override {
-    PipeBase::SetUp();
-    transport = std::make_unique<Transport>(
-        "stdio", nullptr,
-        std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
-                                     File::eOpenOptionReadOnly,
-                                     NativeFile::Unowned),
-        std::make_shared<NativeFile>(output.GetWriteFileDescriptor(),
-                                     File::eOpenOptionWriteOnly,
-                                     NativeFile::Unowned));
-  }
-};
-
-TEST_F(TransportTest, MalformedRequests) {
-  std::string malformed_header = "COnTent-LenGth: -1{}\r\n\r\nnotjosn";
-  ASSERT_THAT_EXPECTED(
-      input.Write(malformed_header.data(), malformed_header.size()),
-      Succeeded());
-  ASSERT_THAT_EXPECTED(
-      transport->Read(std::chrono::milliseconds(1)),
-      FailedWithMessage(
-          "expected 'Content-Length: ' and got 'COnTent-LenGth: '"));
-}
-
-TEST_F(TransportTest, Read) {
-  std::string json =
-      R"json({"seq": 1, "type": "request", "command": "abc"})json";
-  std::string message =
-      formatv("Content-Length: {0}\r\n\r\n{1}", json.size(), json).str();
-  ASSERT_THAT_EXPECTED(input.Write(message.data(), message.size()),
-                       Succeeded());
-  ASSERT_THAT_EXPECTED(
-      transport->Read(std::chrono::milliseconds(1)),
-      HasValue(testing::VariantWith<Request>(testing::FieldsAre(
-          /*seq=*/1, /*command=*/"abc", /*arguments=*/std::nullopt))));
-}
-
-TEST_F(TransportTest, ReadWithTimeout) {
-  ASSERT_THAT_EXPECTED(transport->Read(std::chrono::milliseconds(1)),
-                       Failed<TimeoutError>());
-}
-
-TEST_F(TransportTest, ReadWithEOF) {
-  input.CloseWriteFileDescriptor();
-  ASSERT_THAT_EXPECTED(transport->Read(std::chrono::milliseconds(1)),
-                       Failed<EndOfFileError>());
-}
-
-TEST_F(TransportTest, Write) {
-  ASSERT_THAT_ERROR(transport->Write(Event{"my-event", std::nullopt}),
-                    Succeeded());
-  output.CloseWriteFileDescriptor();
-  char buf[1024];
-  Expected<size_t> bytes_read =
-      output.Read(buf, sizeof(buf), std::chrono::milliseconds(1));
-  ASSERT_THAT_EXPECTED(bytes_read, Succeeded());
-  ASSERT_EQ(
-      StringRef(buf, *bytes_read),
-      StringRef("Content-Length: 43\r\n\r\n"
-                R"json({"event":"my-event","seq":0,"type":"event"})json"));
-}
diff --git a/external/llvm-project/lldb/unittests/DataFormatter/CMakeLists.txt b/external/llvm-project/lldb/unittests/DataFormatter/CMakeLists.txt
index b858db1c7163..3126a285ab5a 100644
--- a/external/llvm-project/lldb/unittests/DataFormatter/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/DataFormatter/CMakeLists.txt
@@ -4,13 +4,12 @@ add_lldb_unittest(LLDBFormatterTests
   FormatterBytecodeTest.cpp
   StringPrinterTests.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbInterpreter
     lldbSymbol
     lldbTarget
     lldbUtility
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/Disassembler/CMakeLists.txt b/external/llvm-project/lldb/unittests/Disassembler/CMakeLists.txt
index 2a76158bf90f..0f6d21178933 100644
--- a/external/llvm-project/lldb/unittests/Disassembler/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Disassembler/CMakeLists.txt
@@ -22,13 +22,14 @@ endif()
 if (disas_srcs)
   add_lldb_unittest(DisassemblerTests
     ${disas_srcs}
+    LINK_COMPONENTS
+      Support
+      ${LLVM_TARGETS_TO_BUILD}
     LINK_LIBS
       lldbCore
       lldbSymbol
       lldbTarget
       lldbPluginDisassemblerLLVMC
       lldbPluginProcessUtility
-    LINK_COMPONENTS
-      Support
-      ${LLVM_TARGETS_TO_BUILD})
+    )
 endif()
diff --git a/external/llvm-project/lldb/unittests/Host/CMakeLists.txt b/external/llvm-project/lldb/unittests/Host/CMakeLists.txt
index 5b8deed00af8..3b20f1d723d1 100644
--- a/external/llvm-project/lldb/unittests/Host/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Host/CMakeLists.txt
@@ -6,6 +6,7 @@ set (FILES
   HostInfoTest.cpp
   HostTest.cpp
   MainLoopTest.cpp
+  JSONTransportTest.cpp
   NativeProcessProtocolTest.cpp
   PipeTest.cpp
   ProcessLaunchInfoTest.cpp
diff --git a/external/llvm-project/lldb/unittests/Host/JSONTransportTest.cpp b/external/llvm-project/lldb/unittests/Host/JSONTransportTest.cpp
new file mode 100644
index 000000000000..4621869887ac
--- /dev/null
+++ b/external/llvm-project/lldb/unittests/Host/JSONTransportTest.cpp
@@ -0,0 +1,181 @@
+//===-- JSONTransportTest.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/JSONTransport.h"
+#include "TestingSupport/Host/PipeTestUtilities.h"
+#include "lldb/Host/File.h"
+
+using namespace llvm;
+using namespace lldb_private;
+
+namespace {
+template <typename T> class JSONTransportTest : public PipeTest {
+protected:
+  std::unique_ptr<JSONTransport> transport;
+
+  void SetUp() override {
+    PipeTest::SetUp();
+    transport = std::make_unique<T>(
+        std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
+                                     File::eOpenOptionReadOnly,
+                                     NativeFile::Unowned),
+        std::make_shared<NativeFile>(output.GetWriteFileDescriptor(),
+                                     File::eOpenOptionWriteOnly,
+                                     NativeFile::Unowned));
+  }
+};
+
+class HTTPDelimitedJSONTransportTest
+    : public JSONTransportTest<HTTPDelimitedJSONTransport> {
+public:
+  using JSONTransportTest::JSONTransportTest;
+};
+
+class JSONRPCTransportTest : public JSONTransportTest<JSONRPCTransport> {
+public:
+  using JSONTransportTest::JSONTransportTest;
+};
+
+struct JSONTestType {
+  std::string str;
+};
+
+llvm::json::Value toJSON(const JSONTestType &T) {
+  return llvm::json::Object{{"str", T.str}};
+}
+
+bool fromJSON(const llvm::json::Value &V, JSONTestType &T, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("str", T.str);
+}
+} // namespace
+
+TEST_F(HTTPDelimitedJSONTransportTest, MalformedRequests) {
+  std::string malformed_header = "COnTent-LenGth: -1{}\r\n\r\nnotjosn";
+  ASSERT_THAT_EXPECTED(
+      input.Write(malformed_header.data(), malformed_header.size()),
+      Succeeded());
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      FailedWithMessage(
+          "expected 'Content-Length: ' and got 'COnTent-LenGth: '"));
+}
+
+TEST_F(HTTPDelimitedJSONTransportTest, Read) {
+  std::string json = R"json({"str": "foo"})json";
+  std::string message =
+      formatv("Content-Length: {0}\r\n\r\n{1}", json.size(), json).str();
+  ASSERT_THAT_EXPECTED(input.Write(message.data(), message.size()),
+                       Succeeded());
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      HasValue(testing::FieldsAre(/*str=*/"foo")));
+}
+
+TEST_F(HTTPDelimitedJSONTransportTest, ReadWithEOF) {
+  input.CloseWriteFileDescriptor();
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      Failed<TransportEOFError>());
+}
+
+
+TEST_F(HTTPDelimitedJSONTransportTest, InvalidTransport) {
+  transport = std::make_unique<HTTPDelimitedJSONTransport>(nullptr, nullptr);
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      Failed<TransportInvalidError>());
+}
+
+TEST_F(HTTPDelimitedJSONTransportTest, Write) {
+  ASSERT_THAT_ERROR(transport->Write(JSONTestType{"foo"}), Succeeded());
+  output.CloseWriteFileDescriptor();
+  char buf[1024];
+  Expected<size_t> bytes_read =
+      output.Read(buf, sizeof(buf), std::chrono::milliseconds(1));
+  ASSERT_THAT_EXPECTED(bytes_read, Succeeded());
+  ASSERT_EQ(StringRef(buf, *bytes_read), StringRef("Content-Length: 13\r\n\r\n"
+                                                   R"json({"str":"foo"})json"));
+}
+
+TEST_F(JSONRPCTransportTest, MalformedRequests) {
+  std::string malformed_header = "notjson\n";
+  ASSERT_THAT_EXPECTED(
+      input.Write(malformed_header.data(), malformed_header.size()),
+      Succeeded());
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      llvm::Failed());
+}
+
+TEST_F(JSONRPCTransportTest, Read) {
+  std::string json = R"json({"str": "foo"})json";
+  std::string message = formatv("{0}\n", json).str();
+  ASSERT_THAT_EXPECTED(input.Write(message.data(), message.size()),
+                       Succeeded());
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      HasValue(testing::FieldsAre(/*str=*/"foo")));
+}
+
+TEST_F(JSONRPCTransportTest, ReadWithEOF) {
+  input.CloseWriteFileDescriptor();
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      Failed<TransportEOFError>());
+}
+
+TEST_F(JSONRPCTransportTest, Write) {
+  ASSERT_THAT_ERROR(transport->Write(JSONTestType{"foo"}), Succeeded());
+  output.CloseWriteFileDescriptor();
+  char buf[1024];
+  Expected<size_t> bytes_read =
+      output.Read(buf, sizeof(buf), std::chrono::milliseconds(1));
+  ASSERT_THAT_EXPECTED(bytes_read, Succeeded());
+  ASSERT_EQ(StringRef(buf, *bytes_read), StringRef(R"json({"str":"foo"})json"
+                                                   "\n"));
+}
+
+TEST_F(JSONRPCTransportTest, InvalidTransport) {
+  transport = std::make_unique<JSONRPCTransport>(nullptr, nullptr);
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      Failed<TransportInvalidError>());
+}
+
+#ifndef _WIN32
+TEST_F(HTTPDelimitedJSONTransportTest, ReadWithTimeout) {
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      Failed<TransportTimeoutError>());
+}
+
+TEST_F(JSONRPCTransportTest, ReadWithTimeout) {
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      Failed<TransportTimeoutError>());
+}
+
+// Windows CRT _read checks that the file descriptor is valid and calls a
+// handler if not. This handler is normally a breakpoint, which looks like a
+// crash when not handled by a debugger.
+// https://learn.microsoft.com/en-us/%20cpp/c-runtime-library/reference/read?view=msvc-170
+TEST_F(HTTPDelimitedJSONTransportTest, ReadAfterClosed) {
+  input.CloseReadFileDescriptor();
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      llvm::Failed());
+}
+
+TEST_F(JSONRPCTransportTest, ReadAfterClosed) {
+  input.CloseReadFileDescriptor();
+  ASSERT_THAT_EXPECTED(
+      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
+      llvm::Failed());
+}
+#endif
diff --git a/external/llvm-project/lldb/unittests/Host/posix/SupportTest.cpp b/external/llvm-project/lldb/unittests/Host/posix/SupportTest.cpp
index f3976db75594..e4d7ba89fece 100644
--- a/external/llvm-project/lldb/unittests/Host/posix/SupportTest.cpp
+++ b/external/llvm-project/lldb/unittests/Host/posix/SupportTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Host/posix/Support.h"
+#include "lldb/Host/aix/Support.h"
 #include "llvm/Support/Threading.h"
 #include "gtest/gtest.h"
 
@@ -19,3 +20,11 @@ TEST(Support, getProcFile_Pid) {
   ASSERT_TRUE(*BufferOrError);
 }
 #endif // #ifndef __APPLE__
+
+#if defined(_AIX) && defined(LLVM_ENABLE_THREADING)
+TEST(Support, getProcFile_Tid) {
+  auto BufferOrError = getProcFile(getpid(), llvm::get_threadid(), "lwpstatus");
+  ASSERT_TRUE(BufferOrError);
+  ASSERT_TRUE(*BufferOrError);
+}
+#endif // #ifdef _AIX && LLVM_ENABLE_THREADING
diff --git a/external/llvm-project/lldb/unittests/Instruction/CMakeLists.txt b/external/llvm-project/lldb/unittests/Instruction/CMakeLists.txt
index 7b1f8afce6b2..10385377923b 100644
--- a/external/llvm-project/lldb/unittests/Instruction/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Instruction/CMakeLists.txt
@@ -3,6 +3,8 @@ add_lldb_unittest(EmulatorTests
   LoongArch/TestLoongArchEmulator.cpp
   RISCV/TestRISCVEmulator.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbSymbol
@@ -10,7 +12,4 @@ add_lldb_unittest(EmulatorTests
     lldbPluginInstructionARM64
     lldbPluginInstructionLoongArch
     lldbPluginInstructionRISCV
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/OperatingSystem/CMakeLists.txt b/external/llvm-project/lldb/unittests/OperatingSystem/CMakeLists.txt
index ae810f37b057..2e85b1ec298e 100644
--- a/external/llvm-project/lldb/unittests/OperatingSystem/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/OperatingSystem/CMakeLists.txt
@@ -6,6 +6,8 @@ add_lldb_unittest(ProcessGdbRemoteTests
   GDBRemoteCommunicationTest.cpp
   GDBRemoteTestUtils.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     LLVMTestingSupport
     lldbCore
@@ -15,7 +17,4 @@ add_lldb_unittest(ProcessGdbRemoteTests
     lldbSymbol
     lldbTarget
     lldbValueObject
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/Platform/CMakeLists.txt b/external/llvm-project/lldb/unittests/Platform/CMakeLists.txt
index 5c0ef5ca6ef2..9a4fd8b3cd0c 100644
--- a/external/llvm-project/lldb/unittests/Platform/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Platform/CMakeLists.txt
@@ -5,13 +5,13 @@ add_lldb_unittest(LLDBPlatformTests
   PlatformSiginfoTest.cpp
   PlatformTest.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbPluginPlatformFreeBSD
     lldbPluginPlatformLinux
     lldbPluginPlatformMacOSX
     lldbPluginPlatformNetBSD
-  LINK_COMPONENTS
-    Support
   )
 
 add_subdirectory(Android)
diff --git a/external/llvm-project/lldb/unittests/Process/Linux/ProcfsTests.cpp b/external/llvm-project/lldb/unittests/Process/Linux/ProcfsTests.cpp
index e7af1f469c2b..534e3e132ec6 100644
--- a/external/llvm-project/lldb/unittests/Process/Linux/ProcfsTests.cpp
+++ b/external/llvm-project/lldb/unittests/Process/Linux/ProcfsTests.cpp
@@ -104,7 +104,7 @@ TEST(Perf, RealLogicalCoreIDs) {
   ASSERT_GT((int)cpu_ids->size(), 0) << "We must see at least one core";
 }
 
-TEST(Perf, RealPtraceScope) {
+TEST(Perf, RealPtraceScopeWhenExist) {
   // We first check we can read /proc/sys/kernel/yama/ptrace_scope
   auto buffer_or_error =
       errorOrToExpected(getProcFile("sys/kernel/yama/ptrace_scope"));
@@ -120,6 +120,21 @@ TEST(Perf, RealPtraceScope) {
       << "Sensible values of ptrace_scope are between 0 and 3";
 }
 
+TEST(Perf, RealPtraceScopeWhenNotExist) {
+  // We first check we can NOT read /proc/sys/kernel/yama/ptrace_scope
+  auto buffer_or_error =
+      errorOrToExpected(getProcFile("sys/kernel/yama/ptrace_scope"));
+  if (buffer_or_error)
+    GTEST_SKIP() << "In order for this test to run, "
+                    "/proc/sys/kernel/yama/ptrace_scope should not exist";
+  consumeError(buffer_or_error.takeError());
+
+  // At this point we should fail parsing the ptrace_scope value.
+  Expected<int> ptrace_scope = GetPtraceScope();
+  ASSERT_FALSE((bool)ptrace_scope);
+  consumeError(ptrace_scope.takeError());
+}
+
 #ifdef LLVM_ENABLE_THREADING
 TEST(Support, getProcFile_Tid) {
   auto BufferOrError = getProcFile(getpid(), llvm::get_threadid(), "comm");
diff --git a/external/llvm-project/lldb/unittests/Process/elf-core/CMakeLists.txt b/external/llvm-project/lldb/unittests/Process/elf-core/CMakeLists.txt
index 68ab6e0683c1..47f8867b8143 100644
--- a/external/llvm-project/lldb/unittests/Process/elf-core/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Process/elf-core/CMakeLists.txt
@@ -4,6 +4,8 @@ include(CMakePushCheckState)
 add_lldb_unittest(ProcessElfCoreTests
   ThreadElfCoreTest.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -12,9 +14,6 @@ add_lldb_unittest(ProcessElfCoreTests
     lldbPluginPlatformLinux
 
     LLVMTestingSupport
-
-  LINK_COMPONENTS
-    Support
   )
 
 cmake_push_check_state()
diff --git a/external/llvm-project/lldb/unittests/Process/gdb-remote/CMakeLists.txt b/external/llvm-project/lldb/unittests/Process/gdb-remote/CMakeLists.txt
index 41bce6f31eaf..50a97e5cebaa 100644
--- a/external/llvm-project/lldb/unittests/Process/gdb-remote/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Process/gdb-remote/CMakeLists.txt
@@ -6,6 +6,8 @@ add_lldb_unittest(ProcessGdbRemoteTests
   GDBRemoteCommunicationTest.cpp
   GDBRemoteTestUtils.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -14,7 +16,4 @@ add_lldb_unittest(ProcessGdbRemoteTests
     lldbPluginProcessGDBRemote
 
     LLVMTestingSupport
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/Process/minidump/CMakeLists.txt b/external/llvm-project/lldb/unittests/Process/minidump/CMakeLists.txt
index ad5f18831471..07ea9c5a4550 100644
--- a/external/llvm-project/lldb/unittests/Process/minidump/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Process/minidump/CMakeLists.txt
@@ -2,6 +2,9 @@ add_lldb_unittest(LLDBMinidumpTests
   MinidumpParserTest.cpp
   RegisterContextMinidumpTest.cpp
 
+  LINK_COMPONENTS
+    ObjectYAML
+    Support
   LINK_LIBS
     lldbCore
     lldbHost
@@ -10,9 +13,6 @@ add_lldb_unittest(LLDBMinidumpTests
     lldbPluginProcessMinidump
     lldbUtilityHelpers
     LLVMTestingSupport
-  LINK_COMPONENTS
-    ObjectYAML
-    Support
   )
 
 set(test_inputs
diff --git a/external/llvm-project/lldb/unittests/ScriptInterpreter/Lua/CMakeLists.txt b/external/llvm-project/lldb/unittests/ScriptInterpreter/Lua/CMakeLists.txt
index e030070a140f..558f76dd5488 100644
--- a/external/llvm-project/lldb/unittests/ScriptInterpreter/Lua/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/ScriptInterpreter/Lua/CMakeLists.txt
@@ -2,11 +2,11 @@ add_lldb_unittest(ScriptInterpreterLuaTests
   LuaTests.cpp
   ScriptInterpreterTests.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbHost
     lldbPluginScriptInterpreterLua
     lldbPluginPlatformLinux
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt b/external/llvm-project/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt
index daaf40dceff1..9b0769616ba6 100644
--- a/external/llvm-project/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt
@@ -8,12 +8,12 @@ add_lldb_unittest(ScriptInterpreterPythonTests
   PythonDataObjectsTests.cpp
   PythonTestSuite.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbHost
     lldbPluginScriptInterpreterPython
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
   )
 
 if(Python3_RPATH)
diff --git a/external/llvm-project/lldb/unittests/Symbol/PostfixExpressionTest.cpp b/external/llvm-project/lldb/unittests/Symbol/PostfixExpressionTest.cpp
index d56df476ebaa..1e437da5133d 100644
--- a/external/llvm-project/lldb/unittests/Symbol/PostfixExpressionTest.cpp
+++ b/external/llvm-project/lldb/unittests/Symbol/PostfixExpressionTest.cpp
@@ -159,8 +159,8 @@ static std::string ParseAndGenerateDWARF(llvm::StringRef expr) {
 
   std::string result;
   llvm::raw_string_ostream os(result);
-  llvm::DWARFExpression(extractor, addr_size, llvm::dwarf::DWARF32)
-      .print(os, llvm::DIDumpOptions(), nullptr);
+  llvm::DWARFExpression E(extractor, addr_size, llvm::dwarf::DWARF32);
+  llvm::DWARFExpressionPrinter::print(&E, os, llvm::DIDumpOptions(), nullptr);
   return result;
 }
 
diff --git a/external/llvm-project/lldb/unittests/Symbol/TestTypeSystemClang.cpp b/external/llvm-project/lldb/unittests/Symbol/TestTypeSystemClang.cpp
index a9b0c87c4fbc..d555d27bef95 100644
--- a/external/llvm-project/lldb/unittests/Symbol/TestTypeSystemClang.cpp
+++ b/external/llvm-project/lldb/unittests/Symbol/TestTypeSystemClang.cpp
@@ -866,8 +866,7 @@ TEST_F(TestTypeSystemClang, TestFunctionTemplateConstruction) {
   clang::TranslationUnitDecl *TU = m_ast->GetTranslationUnitDecl();
 
   // Prepare the declarations/types we need for the template.
-  CompilerType clang_type =
-      m_ast->CreateFunctionType(int_type, nullptr, 0U, false, 0U);
+  CompilerType clang_type = m_ast->CreateFunctionType(int_type, {}, false, 0U);
   FunctionDecl *func = m_ast->CreateFunctionDeclaration(
       TU, OptionalClangModuleID(), "foo", clang_type, StorageClass::SC_None,
       false);
@@ -895,8 +894,7 @@ TEST_F(TestTypeSystemClang, TestFunctionTemplateInRecordConstruction) {
   clang::TagDecl *record = ClangUtil::GetAsTagDecl(record_type);
 
   // Prepare the declarations/types we need for the template.
-  CompilerType clang_type =
-      m_ast->CreateFunctionType(int_type, nullptr, 0U, false, 0U);
+  CompilerType clang_type = m_ast->CreateFunctionType(int_type, {}, false, 0U);
   // We create the FunctionDecl for the template in the TU DeclContext because:
   // 1. FunctionDecls can't be in a Record (only CXXMethodDecls can).
   // 2. It is mirroring the behavior of DWARFASTParserClang::ParseSubroutine.
@@ -930,10 +928,9 @@ TEST_F(TestTypeSystemClang, TestDeletingImplicitCopyCstrDueToMoveCStr) {
 
   // Create a move constructor that will delete the implicit copy constructor.
   CompilerType return_type = m_ast->GetBasicType(lldb::eBasicTypeVoid);
-  CompilerType param_type = t.GetRValueReferenceType();
-  CompilerType function_type =
-      m_ast->CreateFunctionType(return_type, &param_type, /*num_params*/ 1,
-                                /*variadic=*/false, /*quals*/ 0U);
+  std::array<CompilerType, 1> args{t.GetRValueReferenceType()};
+  CompilerType function_type = m_ast->CreateFunctionType(
+      return_type, args, /*variadic=*/false, /*quals*/ 0U);
   bool is_virtual = false;
   bool is_static = false;
   bool is_inline = false;
@@ -974,10 +971,9 @@ TEST_F(TestTypeSystemClang, TestNotDeletingUserCopyCstrDueToMoveCStr) {
   bool is_artificial = false;
   // Create a move constructor.
   {
-    CompilerType param_type = t.GetRValueReferenceType();
-    CompilerType function_type =
-        m_ast->CreateFunctionType(return_type, &param_type, /*num_params*/ 1,
-                                  /*variadic=*/false, /*quals*/ 0U);
+    std::array<CompilerType, 1> args{t.GetRValueReferenceType()};
+    CompilerType function_type = m_ast->CreateFunctionType(
+        return_type, args, /*variadic=*/false, /*quals*/ 0U);
     m_ast->AddMethodToCXXRecordType(
         t.GetOpaqueQualType(), class_name, nullptr, function_type,
         lldb::AccessType::eAccessPublic, is_virtual, is_static, is_inline,
@@ -985,9 +981,10 @@ TEST_F(TestTypeSystemClang, TestNotDeletingUserCopyCstrDueToMoveCStr) {
   }
   // Create a copy constructor.
   {
-    CompilerType param_type = t.GetLValueReferenceType().AddConstModifier();
+    std::array<CompilerType, 1> args{
+        t.GetLValueReferenceType().AddConstModifier()};
     CompilerType function_type =
-        m_ast->CreateFunctionType(return_type, &param_type, /*num_params*/ 1,
+        m_ast->CreateFunctionType(return_type, args,
                                   /*variadic=*/false, /*quals*/ 0U);
     m_ast->AddMethodToCXXRecordType(
         t.GetOpaqueQualType(), class_name, nullptr, function_type,
@@ -1012,10 +1009,9 @@ TEST_F(TestTypeSystemClang, AddMethodToObjCObjectType) {
 
   // Add a method to the interface.
   std::vector<CompilerType> args;
-  CompilerType func_type =
-      m_ast->CreateFunctionType(m_ast->GetBasicType(lldb::eBasicTypeInt),
-                                args.data(), args.size(), /*variadic*/ false,
-                                /*quals*/ 0, clang::CallingConv::CC_C);
+  CompilerType func_type = m_ast->CreateFunctionType(
+      m_ast->GetBasicType(lldb::eBasicTypeInt), args, /*variadic*/ false,
+      /*quals*/ 0, clang::CallingConv::CC_C);
   bool variadic = false;
   bool artificial = false;
   bool objc_direct = false;
@@ -1098,9 +1094,9 @@ TEST_F(TestTypeSystemClang, AddMethodToCXXRecordType_ParmVarDecls) {
   llvm::SmallVector<CompilerType> param_types{
       m_ast->GetBasicType(lldb::eBasicTypeInt),
       m_ast->GetBasicType(lldb::eBasicTypeShort)};
-  CompilerType function_type = m_ast->CreateFunctionType(
-      return_type, param_types.data(), /*num_params*/ param_types.size(),
-      /*variadic=*/false, /*quals*/ 0U);
+  CompilerType function_type =
+      m_ast->CreateFunctionType(return_type, param_types,
+                                /*variadic=*/false, /*quals*/ 0U);
   m_ast->AddMethodToCXXRecordType(
       t.GetOpaqueQualType(), "myFunc", nullptr, function_type,
       lldb::AccessType::eAccessPublic, is_virtual, is_static, is_inline,
diff --git a/external/llvm-project/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt b/external/llvm-project/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
index d5b0be7ea2a2..cf8702209a7c 100644
--- a/external/llvm-project/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
@@ -7,6 +7,9 @@ add_lldb_unittest(SymbolFileDWARFTests
   SymbolFileDWARFTests.cpp
   XcodeSDKModuleTests.cpp
 
+  LINK_COMPONENTS
+    Support
+    DebugInfoPDB
   LINK_LIBS
     lldbCore
     lldbHost
@@ -19,9 +22,6 @@ add_lldb_unittest(SymbolFileDWARFTests
     lldbUtilityHelpers
     lldbSymbolHelpers
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
-    DebugInfoPDB
   )
 
 set(test_inputs
diff --git a/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/CMakeLists.txt b/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/CMakeLists.txt
index 4562c3f07d17..bfd74dd5050b 100644
--- a/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/CMakeLists.txt
@@ -2,13 +2,13 @@ add_lldb_unittest(SymbolFileNativePDBTests
   PdbFPOProgramToDWARFExpressionTests.cpp
   UdtRecordCompleterTests.cpp
 
+  LINK_COMPONENTS
+    Support
+    DebugInfoPDB
   LINK_LIBS
     lldbCore
     lldbHost
     lldbSymbol
     lldbPluginSymbolFileNativePDB
     lldbUtilityHelpers
-  LINK_COMPONENTS
-    Support
-    DebugInfoPDB
   )
diff --git a/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp b/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp
index efb8f720b56e..d746e04f8a9f 100644
--- a/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp
+++ b/external/llvm-project/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp
@@ -39,8 +39,8 @@ CheckValidProgramTranslation(llvm::StringRef fpo_program,
 
   std::string result;
   llvm::raw_string_ostream os(result);
-  llvm::DWARFExpression(extractor, /*AddressSize=*/4, llvm::dwarf::DWARF32)
-      .print(os, llvm::DIDumpOptions(), nullptr);
+  llvm::DWARFExpression E(extractor, /*AddressSize=*/4, llvm::dwarf::DWARF32);
+  llvm::DWARFExpressionPrinter::print(&E, os, llvm::DIDumpOptions(), nullptr);
 
   // actual check
   ASSERT_EQ(expected_dwarf_expression, result);
diff --git a/external/llvm-project/lldb/unittests/SymbolFile/PDB/CMakeLists.txt b/external/llvm-project/lldb/unittests/SymbolFile/PDB/CMakeLists.txt
index 8fc2167ee6ba..8edb352e5a3e 100644
--- a/external/llvm-project/lldb/unittests/SymbolFile/PDB/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/SymbolFile/PDB/CMakeLists.txt
@@ -1,6 +1,9 @@
 add_lldb_unittest(SymbolFilePDBTests
   SymbolFilePDBTests.cpp
 
+  LINK_COMPONENTS
+    Support
+    DebugInfoPDB
   LINK_LIBS
     lldbCore
     lldbHost
@@ -11,9 +14,6 @@ add_lldb_unittest(SymbolFilePDBTests
     lldbPluginTypeSystemClang
     lldbUtilityHelpers
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
-    DebugInfoPDB
   )
 
 set(test_inputs
diff --git a/external/llvm-project/lldb/unittests/Target/CMakeLists.txt b/external/llvm-project/lldb/unittests/Target/CMakeLists.txt
index a64876184de1..3169339ec699 100644
--- a/external/llvm-project/lldb/unittests/Target/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Target/CMakeLists.txt
@@ -14,6 +14,8 @@ add_lldb_unittest(TargetTests
   SummaryStatisticsTest.cpp
   FindFileTest.cpp
 
+  LINK_COMPONENTS
+      Support
   LINK_LIBS
       lldbCore
       lldbHost
@@ -28,8 +30,6 @@ add_lldb_unittest(TargetTests
       lldbSymbol
       lldbUtility
       lldbUtilityHelpers
-    LINK_COMPONENTS
-      Support
   )
 
 set(test_inputs
diff --git a/external/llvm-project/lldb/unittests/TestingSupport/CMakeLists.txt b/external/llvm-project/lldb/unittests/TestingSupport/CMakeLists.txt
index df0a37005358..b9affcf41217 100644
--- a/external/llvm-project/lldb/unittests/TestingSupport/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/TestingSupport/CMakeLists.txt
@@ -3,13 +3,12 @@ add_lldb_library(lldbUtilityHelpers
   MockTildeExpressionResolver.cpp
   TestUtilities.cpp
 
-  LINK_LIBS
-    lldbUtility
-    llvm_gtest
-
   LINK_COMPONENTS
     Support
     ObjectYAML
+  LINK_LIBS
+    lldbUtility
+    llvm_gtest
   )
 
 add_subdirectory(Host)
diff --git a/external/llvm-project/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h b/external/llvm-project/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
new file mode 100644
index 000000000000..50d5d4117c89
--- /dev/null
+++ b/external/llvm-project/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
@@ -0,0 +1,28 @@
+//===-- PipeTestUtilities.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UNITTESTS_TESTINGSUPPORT_PIPETESTUTILITIES_H
+#define LLDB_UNITTESTS_TESTINGSUPPORT_PIPETESTUTILITIES_H
+
+#include "lldb/Host/Pipe.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+/// A base class for tests that need a pair of pipes for communication.
+class PipeTest : public testing::Test {
+protected:
+  lldb_private::Pipe input;
+  lldb_private::Pipe output;
+
+  void SetUp() override {
+    ASSERT_THAT_ERROR(input.CreateNew(false).ToError(), llvm::Succeeded());
+    ASSERT_THAT_ERROR(output.CreateNew(false).ToError(), llvm::Succeeded());
+  }
+};
+
+#endif
diff --git a/external/llvm-project/lldb/unittests/TestingSupport/Symbol/CMakeLists.txt b/external/llvm-project/lldb/unittests/TestingSupport/Symbol/CMakeLists.txt
index c4eef2e453dc..eb4f789a4966 100644
--- a/external/llvm-project/lldb/unittests/TestingSupport/Symbol/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/TestingSupport/Symbol/CMakeLists.txt
@@ -2,6 +2,8 @@ set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL ON)
 add_lldb_library(lldbSymbolHelpers
   YAMLModuleTester.cpp
 
+  LINK_COMPONENTS
+    ObjectYAML
   LINK_LIBS
     lldbCore
     lldbHost
@@ -11,7 +13,4 @@ add_lldb_library(lldbSymbolHelpers
     lldbPluginTypeSystemClang
     lldbUtilityHelpers
     LLVMTestingSupport
-
-  LINK_COMPONENTS
-    ObjectYAML
   )
diff --git a/external/llvm-project/lldb/unittests/UnwindAssembly/ARM64/CMakeLists.txt b/external/llvm-project/lldb/unittests/UnwindAssembly/ARM64/CMakeLists.txt
index f3bbd71df32d..264862ce70f3 100644
--- a/external/llvm-project/lldb/unittests/UnwindAssembly/ARM64/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/UnwindAssembly/ARM64/CMakeLists.txt
@@ -1,5 +1,8 @@
 add_lldb_unittest(Arm64InstEmulationTests
   TestArm64InstEmulation.cpp
+  LINK_COMPONENTS
+    Support
+    ${LLVM_TARGETS_TO_BUILD}
   LINK_LIBS
     lldbCore
     lldbSymbol
@@ -8,6 +11,4 @@ add_lldb_unittest(Arm64InstEmulationTests
     lldbPluginDisassemblerLLVMC
     lldbPluginInstructionARM64
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
-    ${LLVM_TARGETS_TO_BUILD})
+  )
diff --git a/external/llvm-project/lldb/unittests/UnwindAssembly/PPC64/CMakeLists.txt b/external/llvm-project/lldb/unittests/UnwindAssembly/PPC64/CMakeLists.txt
index 679f7664cb87..6310ee5bfe93 100644
--- a/external/llvm-project/lldb/unittests/UnwindAssembly/PPC64/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/UnwindAssembly/PPC64/CMakeLists.txt
@@ -1,5 +1,8 @@
 add_lldb_unittest(PPC64InstEmulationTests
   TestPPC64InstEmulation.cpp
+  LINK_COMPONENTS
+    Support
+    ${LLVM_TARGETS_TO_BUILD}
   LINK_LIBS
     lldbCore
     lldbSymbol
@@ -8,6 +11,4 @@ add_lldb_unittest(PPC64InstEmulationTests
     lldbPluginDisassemblerLLVMC
     lldbPluginInstructionPPC64
     lldbPluginProcessUtility
-  LINK_COMPONENTS
-    Support
-    ${LLVM_TARGETS_TO_BUILD})
+  )
diff --git a/external/llvm-project/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt b/external/llvm-project/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt
index d28e9629a64c..a5dcda8362a9 100644
--- a/external/llvm-project/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt
@@ -1,10 +1,10 @@
 add_lldb_unittest(UnwindAssemblyX86ButNoX86TargetTests
     Testx86AssemblyInspectionEngine.cpp
+    LINK_COMPONENTS
+      Support
+      ${LLVM_TARGETS_TO_BUILD}
     LINK_LIBS
       lldbCore
       lldbSymbol
       lldbPluginUnwindAssemblyX86
-    LINK_COMPONENTS
-      Support
-      ${LLVM_TARGETS_TO_BUILD}
   )
diff --git a/external/llvm-project/lldb/unittests/UnwindAssembly/x86/CMakeLists.txt b/external/llvm-project/lldb/unittests/UnwindAssembly/x86/CMakeLists.txt
index 2b5b31f79063..a1a31c6da528 100644
--- a/external/llvm-project/lldb/unittests/UnwindAssembly/x86/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/UnwindAssembly/x86/CMakeLists.txt
@@ -1,10 +1,10 @@
 add_lldb_unittest(UnwindAssemblyx86Tests
     Testx86AssemblyInspectionEngine.cpp
+    LINK_COMPONENTS
+      Support
+      ${LLVM_TARGETS_TO_BUILD}
     LINK_LIBS
       lldbCore
       lldbSymbol
       lldbPluginUnwindAssemblyX86
-    LINK_COMPONENTS
-      Support
-      ${LLVM_TARGETS_TO_BUILD}
   )
diff --git a/external/llvm-project/lldb/unittests/Utility/CMakeLists.txt b/external/llvm-project/lldb/unittests/Utility/CMakeLists.txt
index 8d4e2ea35f26..aed4177f5ede 100644
--- a/external/llvm-project/lldb/unittests/Utility/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/Utility/CMakeLists.txt
@@ -51,13 +51,13 @@ add_lldb_unittest(UtilityTests
   VMRangeTest.cpp
   XcodeSDKTest.cpp
 
+  LINK_COMPONENTS
+    Support
+    TargetParser
   LINK_LIBS
       lldbUtility
       lldbUtilityHelpers
       LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
-    TargetParser
   )
 
 set(test_inputs
diff --git a/external/llvm-project/lldb/unittests/ValueObject/CMakeLists.txt b/external/llvm-project/lldb/unittests/ValueObject/CMakeLists.txt
index 6ef0091647a5..6e1fc64ce3f6 100644
--- a/external/llvm-project/lldb/unittests/ValueObject/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/ValueObject/CMakeLists.txt
@@ -3,12 +3,11 @@ add_lldb_unittest(LLDBValueObjectTests
   DILLexerTests.cpp
   DynamicValueObjectLocalBuffer.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbValueObject
     lldbPluginPlatformLinux
     lldbPluginScriptInterpreterNone
     LLVMTestingSupport
-
-  LINK_COMPONENTS
-    Support
   )
diff --git a/external/llvm-project/lldb/unittests/debugserver/CMakeLists.txt b/external/llvm-project/lldb/unittests/debugserver/CMakeLists.txt
index fd488c1623b8..c2646eb113be 100644
--- a/external/llvm-project/lldb/unittests/debugserver/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/debugserver/CMakeLists.txt
@@ -9,12 +9,12 @@ add_lldb_unittest(debugserverTests
   RNBSocketTest.cpp
   debugserver_LogCallback.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbDebugserverCommon
     lldbHost
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
   )
 
 target_include_directories(debugserverTests PRIVATE
@@ -33,10 +33,10 @@ if(APPLE_EMBEDDED)
     RNBSocketTest.cpp
     debugserver_LogCallback.cpp
 
+    LINK_COMPONENTS
+      Support
     LINK_LIBS
       lldbDebugserverCommon_NonUI
       lldbHost
-    LINK_COMPONENTS
-      Support
     )
 endif()
diff --git a/external/llvm-project/lldb/unittests/tools/lldb-server/tests/CMakeLists.txt b/external/llvm-project/lldb/unittests/tools/lldb-server/tests/CMakeLists.txt
index 1d5f0c789a36..672cdbbb35b1 100644
--- a/external/llvm-project/lldb/unittests/tools/lldb-server/tests/CMakeLists.txt
+++ b/external/llvm-project/lldb/unittests/tools/lldb-server/tests/CMakeLists.txt
@@ -4,6 +4,8 @@ add_lldb_unittest(LLDBServerTests
   TestBase.cpp
   TestClient.cpp
 
+  LINK_COMPONENTS
+    Support
   LINK_LIBS
     lldbHost
     lldbCore
@@ -14,8 +16,6 @@ add_lldb_unittest(LLDBServerTests
     lldbPluginProcessGDBRemote
 
     LLVMTestingSupport
-  LINK_COMPONENTS
-    Support
   )
 
 add_dependencies(LLDBServerTests lldb-server ${ALL_LLDB_TEST_EXECUTABLES})
diff --git a/external/llvm-project/llvm/CMakeLists.txt b/external/llvm-project/llvm/CMakeLists.txt
index 718f08d8d93a..da928007474c 100644
--- a/external/llvm-project/llvm/CMakeLists.txt
+++ b/external/llvm-project/llvm/CMakeLists.txt
@@ -19,21 +19,6 @@ endif()
 
 include(${LLVM_COMMON_CMAKE_UTILS}/Modules/LLVMVersion.cmake)
 
-# TODO: Remove the following block of llvm versions once build_lightning script
-# has been updated to lookup above LLVMVersion.cmake file instead of this file.
-if(NOT DEFINED LLVM_VERSION_MAJOR)
-  set(LLVM_VERSION_MAJOR 19)
-endif()
-if(NOT DEFINED LLVM_VERSION_MINOR)
-  set(LLVM_VERSION_MINOR 0)
-endif()
-if(NOT DEFINED LLVM_VERSION_PATCH)
-  set(LLVM_VERSION_PATCH 0)
-endif()
-if(NOT DEFINED LLVM_VERSION_SUFFIX)
-  set(LLVM_VERSION_SUFFIX git)
-endif()
-
 set_directory_properties(PROPERTIES LLVM_VERSION_MAJOR "${LLVM_VERSION_MAJOR}")
 
 if (NOT PACKAGE_VERSION)
@@ -124,15 +109,20 @@ endif()
 # LLVM_EXTERNAL_${project}_SOURCE_DIR using LLVM_ALL_PROJECTS
 # This allows an easy way of setting up a build directory for llvm and another
 # one for llvm+clang+... using the same sources.
-set(LLVM_ALL_PROJECTS "bolt;clang;clang-tools-extra;compiler-rt;cross-project-tests;libc;libclc;lld;lldb;mlir;openmp;polly;pstl")
+# These projects will be included when "all" is included in LLVM_ENABLE_PROJECTS.
+set(LLVM_ALL_PROJECTS "bolt;clang;clang-tools-extra;compiler-rt;cross-project-tests;libclc;lld;lldb;mlir;openmp;polly;pstl")
 if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
   # Disallow 'openmp' as a LLVM PROJECT on AIX as the supported way is to use
   # LLVM_ENABLE_RUNTIMES.
   list(REMOVE_ITEM LLVM_ALL_PROJECTS openmp)
 endif()
 
-# The flang project is not yet part of "all" projects (see C++ requirements)
-set(LLVM_EXTRA_PROJECTS "flang")
+# The "libc" project, which is not part of "all" projects, could be included in
+# LLVM_ENABLE_PROJECTS.  It is preferred to include "libc" in
+# LLVM_ENABLE_RUNTIMES instead of LLVM_ENABLE_PROJECTS.
+#
+# The flang project is not yet part of "all" projects (see C++ requirements).
+set(LLVM_EXTRA_PROJECTS "flang" "libc")
 # List of all known projects in the mono repo
 set(LLVM_KNOWN_PROJECTS "${LLVM_ALL_PROJECTS};${LLVM_EXTRA_PROJECTS}")
 set(LLVM_ENABLE_PROJECTS "" CACHE STRING
@@ -152,7 +142,7 @@ endforeach()
 # As we migrate runtimes to using the bootstrapping build, the set of default runtimes
 # should grow as we remove those runtimes from LLVM_ENABLE_PROJECTS above.
 set(LLVM_DEFAULT_RUNTIMES "libcxx;libcxxabi;libunwind")
-set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;pstl;libcxx;compiler-rt;openmp;llvm-libgcc;offload;flang-rt")
+set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;pstl;libcxx;compiler-rt;openmp;llvm-libgcc;offload;flang-rt;libclc")
 set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
   "Semicolon-separated list of runtimes to build, or \"all\" (${LLVM_DEFAULT_RUNTIMES}). Supported runtimes are ${LLVM_SUPPORTED_RUNTIMES}.")
 if(LLVM_ENABLE_RUNTIMES STREQUAL "all")
@@ -183,8 +173,7 @@ if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
 endif()
 
 if ("libc" IN_LIST LLVM_ENABLE_PROJECTS)
-  message(WARNING "Using LLVM_ENABLE_PROJECTS=libc is deprecated now, and will "
-    "become a fatal error in the LLVM 21 release.  Please use "
+  message(WARNING "Using LLVM_ENABLE_PROJECTS=libc is deprecated.  Please use "
     "-DLLVM_ENABLE_RUNTIMES=libc or see the instructions at "
     "https://libc.llvm.org/ for building the runtimes.")
 endif()
@@ -216,6 +205,13 @@ if ("flang-rt" IN_LIST LLVM_ENABLE_RUNTIMES)
   endif ()
 endif ()
 
+if ("libclc" IN_LIST LLVM_ENABLE_PROJECTS)
+  message(WARNING "Using LLVM_ENABLE_PROJECTS=libclc is deprecated now, and will "
+    "become a fatal error in the LLVM 21 release.  Please use "
+    "-DLLVM_ENABLE_RUNTIMES=libclc or see the instructions at "
+    "https://libclc.llvm.org/ for building the runtimes.")
+endif()
+
 # Set a shorthand option to enable the GPU build of the 'libc' project.
 option(LIBC_GPU_BUILD "Enable the 'libc' project targeting the GPU" OFF)
 if(LIBC_GPU_BUILD)
@@ -574,8 +570,8 @@ endif()
 option(LLVM_ENABLE_CRASH_DUMPS "Turn on memory dumps on crashes. Currently only implemented on Windows." OFF)
 
 set(LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING "DISABLED" CACHE STRING
-  "Enhance Debugify's line number coverage tracking; enabling this is ABI-breaking. Can be DISABLED, or COVERAGE.")
-set_property(CACHE LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING PROPERTY STRINGS DISABLED COVERAGE)
+  "Enhance Debugify's line number coverage tracking; enabling this is ABI-breaking. Can be DISABLED, COVERAGE, or COVERAGE_AND_ORIGIN.")
+set_property(CACHE LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING PROPERTY STRINGS DISABLED COVERAGE COVERAGE_AND_ORIGIN)
 
 option(LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS
   "Add additional fields to DILocations to support Key Instructions" OFF)
@@ -1319,6 +1315,7 @@ if( LLVM_INCLUDE_UTILS )
   add_subdirectory(utils/yaml-bench)
   add_subdirectory(utils/split-file)
   add_subdirectory(utils/mlgo-utils)
+  add_subdirectory(utils/llvm-test-mustache-spec)
   if( LLVM_INCLUDE_TESTS )
     set(LLVM_SUBPROJECT_TITLE "Third-Party/Google Test")
     add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest)
diff --git a/external/llvm-project/llvm/cmake/config-ix.cmake b/external/llvm-project/llvm/cmake/config-ix.cmake
index 687f5077cbfd..9895469973e4 100644
--- a/external/llvm-project/llvm/cmake/config-ix.cmake
+++ b/external/llvm-project/llvm/cmake/config-ix.cmake
@@ -1,8 +1,3 @@
-if( WIN32 AND NOT CYGWIN )
-  # We consider Cygwin as another Unix
-  set(PURE_WINDOWS 1)
-endif()
-
 include(CheckIncludeFile)
 include(CheckLibraryExists)
 include(CheckSymbolExists)
@@ -31,7 +26,7 @@ elseif (APPLE)
   set(HAVE_SYS_MMAN_H 1)
   set(HAVE_SYSEXITS_H 1)
   set(HAVE_UNISTD_H 1)
-elseif (PURE_WINDOWS)
+elseif (WIN32)
   set(HAVE_MACH_MACH_H 0)
   set(HAVE_MALLOC_MALLOC_H 0)
   set(HAVE_PTHREAD_H 0)
@@ -132,7 +127,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
 endif()
 
 # library checks
-if( NOT PURE_WINDOWS )
+if(NOT WIN32)
   check_library_exists(pthread pthread_create "" HAVE_LIBPTHREAD)
   if (HAVE_LIBPTHREAD)
     check_library_exists(pthread pthread_rwlock_init "" HAVE_PTHREAD_RWLOCK_INIT)
@@ -275,7 +270,7 @@ endif()
 # party code may call MSan interceptors like strlen, leading to false positives.
 if(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
   # Don't look for these libraries on Windows.
-  if (NOT PURE_WINDOWS)
+  if (NOT WIN32)
     # Skip libedit if using ASan as it contains memory leaks.
     if (LLVM_ENABLE_LIBEDIT AND NOT LLVM_USE_SANITIZER MATCHES ".*Address.*")
       if(LLVM_ENABLE_LIBEDIT STREQUAL FORCE_ON)
@@ -384,7 +379,7 @@ check_symbol_exists(sbrk unistd.h HAVE_SBRK)
 check_symbol_exists(strerror_r string.h HAVE_STRERROR_R)
 check_symbol_exists(strerror_s string.h HAVE_DECL_STRERROR_S)
 check_symbol_exists(setenv stdlib.h HAVE_SETENV)
-if( PURE_WINDOWS )
+if(WIN32)
   check_symbol_exists(_chsize_s io.h HAVE__CHSIZE_S)
 
   check_function_exists(_alloca HAVE__ALLOCA)
@@ -420,8 +415,7 @@ else()
       "sys/types.h;sys/stat.h" HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC)
 endif()
 
-# This check requires _GNU_SOURCE.
-if (NOT PURE_WINDOWS)
+if (NOT WIN32)
   if (LLVM_PTHREAD_LIB)
     list(APPEND CMAKE_REQUIRED_LIBRARIES ${LLVM_PTHREAD_LIB})
   endif()
diff --git a/external/llvm-project/llvm/cmake/modules/CrossCompile.cmake b/external/llvm-project/llvm/cmake/modules/CrossCompile.cmake
index ea5195c9c574..bfbd9cfd4063 100644
--- a/external/llvm-project/llvm/cmake/modules/CrossCompile.cmake
+++ b/external/llvm-project/llvm/cmake/modules/CrossCompile.cmake
@@ -83,9 +83,9 @@ function(llvm_create_cross_target project_name target_name toolchain buildtype)
 
   add_custom_command(OUTPUT ${${project_name}_${target_name}_BUILD}/CMakeCache.txt
     COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}"
-        -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}"
-        -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}"
-        -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}"
+        "-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}"
+        "-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}"
+        "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}"
         ${CROSS_TOOLCHAIN_FLAGS_${target_name}} ${CMAKE_CURRENT_SOURCE_DIR}
         ${CROSS_TOOLCHAIN_FLAGS_${project_name}_${target_name}}
         -DLLVM_TARGET_IS_CROSSCOMPILE_HOST=TRUE
diff --git a/external/llvm-project/llvm/cmake/modules/FindFFI.cmake b/external/llvm-project/llvm/cmake/modules/FindFFI.cmake
index 8e67c5d8c6d1..b1f64522b268 100644
--- a/external/llvm-project/llvm/cmake/modules/FindFFI.cmake
+++ b/external/llvm-project/llvm/cmake/modules/FindFFI.cmake
@@ -23,7 +23,10 @@
 # Additionally, the following import target will be defined:
 # FFI::ffi
 
-find_path(FFI_INCLUDE_DIRS ffi.h PATHS ${FFI_INCLUDE_DIR})
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_LIBFFI QUIET libffi)
+
+find_path(FFI_INCLUDE_DIRS ffi.h PATHS ${FFI_INCLUDE_DIR} ${PC_LIBFFI_INCLUDE_DIRS})
 if( EXISTS "${FFI_INCLUDE_DIRS}/ffi.h" )
   set(FFI_HEADER ffi.h CACHE INTERNAL "")
   set(HAVE_FFI_H 1 CACHE INTERNAL "")
@@ -35,8 +38,8 @@ else()
   endif()
 endif()
 
-find_library(FFI_LIBRARIES NAMES ffi PATHS ${FFI_LIBRARY_DIR})
-find_library(FFI_STATIC_LIBRARIES NAMES libffi.a PATHS ${FFI_LIBRARY_DIR})
+find_library(FFI_LIBRARIES NAMES ffi PATHS ${FFI_LIBRARY_DIR} ${PC_LIBFFI_LIBRARY_DIRS})
+find_library(FFI_STATIC_LIBRARIES NAMES libffi.a PATHS ${FFI_LIBRARY_DIR} ${PC_LIBFFI_LIBRARY_DIRS})
 
 if(FFI_LIBRARIES)
   include(CMakePushCheckState)
diff --git a/external/llvm-project/llvm/cmake/modules/HandleLLVMOptions.cmake b/external/llvm-project/llvm/cmake/modules/HandleLLVMOptions.cmake
index 519e0b95dc78..8004d3571fc8 100644
--- a/external/llvm-project/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/external/llvm-project/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -199,12 +199,18 @@ endif()
 string(TOUPPER "${LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING}" uppercase_LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING)
 
 if( uppercase_LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING STREQUAL "COVERAGE" )
-  set( LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING 1 )
+  set( LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE 1 )
+elseif( uppercase_LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING STREQUAL "COVERAGE_AND_ORIGIN" )
+  set( LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE 1 )
+  set( LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN 1 )
 elseif( uppercase_LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING STREQUAL "DISABLED" OR NOT DEFINED LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING )
-  # The DISABLED setting is default and requires no additional defines.
+  # The DISABLED setting is default.
+  set( LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE 0 )
 else()
   message(FATAL_ERROR "Unknown value for LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING: \"${LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING}\"!")
 endif()
+# LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE (non-cached) is expected to be
+# 1 or 0 here, assuming referenced in #cmakedefine01.
 
 if(LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS)
   add_compile_definitions(EXPERIMENTAL_KEY_INSTRUCTIONS)
@@ -214,15 +220,15 @@ if( LLVM_REVERSE_ITERATION )
   set( LLVM_ENABLE_REVERSE_ITERATION 1 )
 endif()
 
-if(WIN32)
+if(WIN32 OR CYGWIN)
   set(LLVM_HAVE_LINK_VERSION_SCRIPT 0)
   if(CYGWIN)
     set(LLVM_ON_WIN32 0)
     set(LLVM_ON_UNIX 1)
-  else(CYGWIN)
+  else()
     set(LLVM_ON_WIN32 1)
     set(LLVM_ON_UNIX 0)
-  endif(CYGWIN)
+  endif()
 elseif(FUCHSIA OR UNIX)
   set(LLVM_ON_WIN32 0)
   set(LLVM_ON_UNIX 1)
@@ -593,10 +599,6 @@ if( MSVC )
     append("/WX" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
   endif (LLVM_ENABLE_WERROR)
 
-  # FIXME(kzhuravl): Need to check if it affects windows ci builds. If yes,
-  # we might need to upstream this, possibly under a cmake option.
-  append("/Zm20" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
-
   append("/Zc:inline" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
 
   if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
diff --git a/external/llvm-project/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst b/external/llvm-project/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
index ce1aaa59e732..61264f31e8bb 100644
--- a/external/llvm-project/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
+++ b/external/llvm-project/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
@@ -5192,7 +5192,7 @@ D. References
 
     .. _amdgpu-dwarf-AMD-ROCgdb:
 
-2.  [AMD-ROCgdb] `AMD ROCm Debugger (ROCgdb) <https://github.com/ROCm-Developer-Tools/ROCgdb>`__
+2.  [AMD-ROCgdb] `AMD ROCm Debugger (ROCgdb) <https://github.com/ROCm/ROCgdb>`__
 
     .. _amdgpu-dwarf-AMD-ROCm:
 
diff --git a/external/llvm-project/llvm/docs/AMDGPUUsage.rst b/external/llvm-project/llvm/docs/AMDGPUUsage.rst
index 2e56debea8a5..bab21e312c96 100644
--- a/external/llvm-project/llvm/docs/AMDGPUUsage.rst
+++ b/external/llvm-project/llvm/docs/AMDGPUUsage.rst
@@ -1216,12 +1216,12 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    denormalization mode, enabled traps, and floating point exceptions.
                                                    The format is a 64-bit concatenation of the MODE and TRAPSTS registers.
 
-  :ref:`llvm.set.fpenv<int_set_fpenv>`             Sets the floating point environment to the specifies state.
+  :ref:`llvm.set.fpenv<int_set_fpenv>`             Sets the floating point environment to the specified state.
   llvm.amdgcn.load.to.lds.p<1/7>                   Loads values from global memory (either in the form of a global
                                                    a raw fat buffer pointer) to LDS. The size of the data copied can be 1, 2,
                                                    or 4 bytes (and gfx950 also allows 12 or 16 bytes). The LDS pointer
                                                    argument should be wavefront-uniform; the global pointer need not be.
-                                                   The LDS pointer is implicitly offset by 4 * lane_id bytes for sies <= 4 bytes
+                                                   The LDS pointer is implicitly offset by 4 * lane_id bytes for size <= 4 bytes
                                                    and 16 * lane_id bytes for larger sizes. This lowers to `global_load_lds`,
                                                    `buffer_load_* ... lds`, or `global_load__* ... lds` depending on address
                                                    space and architecture. `amdgcn.global.load.lds` has the same semantics as
@@ -18396,8 +18396,8 @@ Additional Documentation
 .. [AMD-RADEON-HD-5000] `AMD Evergreen shader ISA <http://developer.amd.com/wordpress/media/2012/10/AMD_Evergreen-Family_Instruction_Set_Architecture.pdf>`__
 .. [AMD-RADEON-HD-6000] `AMD Cayman/Trinity shader ISA <http://developer.amd.com/wordpress/media/2012/10/AMD_HD_6900_Series_Instruction_Set_Architecture.pdf>`__
 .. [AMD-ROCm] `AMD ROCm™ Platform <https://rocmdocs.amd.com/>`__
-.. [AMD-ROCm-github] `AMD ROCm™ github <http://github.com/RadeonOpenCompute>`__
-.. [AMD-ROCm-Release-Notes] `AMD ROCm Release Notes <https://github.com/RadeonOpenCompute/ROCm>`__
+.. [AMD-ROCm-github] `AMD ROCm™ github <http://github.com/ROCm>`__
+.. [AMD-ROCm-Release-Notes] `AMD ROCm Release Notes <https://github.com/ROCm/ROCm>`__
 .. [CLANG-ATTR] `Attributes in Clang <https://clang.llvm.org/docs/AttributeReference.html>`__
 .. [DWARF] `DWARF Debugging Information Format <http://dwarfstd.org/>`__
 .. [ELF] `Executable and Linkable Format (ELF) <http://www.sco.com/developers/gabi/>`__
diff --git a/external/llvm-project/llvm/docs/CMake.rst b/external/llvm-project/llvm/docs/CMake.rst
index 674e4969c691..72f19fd35392 100644
--- a/external/llvm-project/llvm/docs/CMake.rst
+++ b/external/llvm-project/llvm/docs/CMake.rst
@@ -482,11 +482,14 @@ enabled sub-projects. Nearly all of these variable names begin with
 **LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING**:STRING
   Enhances Debugify's ability to detect line number errors by storing extra
   information inside Instructions, removing false positives from Debugify's
-  results at the cost of performance. Allowed values are `DISABLED` (default)
-  and `COVERAGE`. `COVERAGE` tracks whether and why a line number was
-  intentionally dropped or not generated for an instruction, allowing Debugify
-  to avoid reporting these as errors; this comes with a small performance cost
-  of ~0.1%. `COVERAGE` is an ABI-breaking option.
+  results at the cost of performance. Allowed values are `DISABLED` (default),
+  `COVERAGE`, and `COVERAGE_AND_ORIGIN`. `COVERAGE` tracks whether and why a
+  line number was intentionally dropped or not generated for an instruction,
+  allowing Debugify to avoid reporting these as errors; this comes with a small
+  performance cost of ~0.1%. `COVERAGE_AND_ORIGIN` additionally stores a
+  stacktrace of the point where each DebugLoc is unintentionally dropped,
+  allowing for much easier bug triaging at the cost of a ~10x performance
+  slowdown. `COVERAGE` and `COVERAGE_AND_ORIGIN` are ABI-breaking options.
 
 **LLVM_ENABLE_DIA_SDK**:BOOL
   Enable building with MSVC DIA SDK for PDB debugging support. Available
diff --git a/external/llvm-project/llvm/docs/CommandGuide/index.rst b/external/llvm-project/llvm/docs/CommandGuide/index.rst
index 643951eca2a2..88fc1fd326b7 100644
--- a/external/llvm-project/llvm/docs/CommandGuide/index.rst
+++ b/external/llvm-project/llvm/docs/CommandGuide/index.rst
@@ -87,6 +87,7 @@ Developer Tools
    llvm-exegesis
    llvm-ifs
    llvm-locstats
+   llvm-test-mustache-spec
    llvm-pdbutil
    llvm-profgen
    llvm-tli-checker
diff --git a/external/llvm-project/llvm/docs/CommandGuide/lit.rst b/external/llvm-project/llvm/docs/CommandGuide/lit.rst
index 86879f870e06..938b7f9f8b6f 100644
--- a/external/llvm-project/llvm/docs/CommandGuide/lit.rst
+++ b/external/llvm-project/llvm/docs/CommandGuide/lit.rst
@@ -63,12 +63,12 @@ GENERAL OPTIONS
 
  Show :program:`lit`'s version number and exit.
 
-.. option:: -j N, --workers=N
+.. option:: -j N, --workers N
 
  Run ``N`` tests in parallel.  By default, this is automatically chosen to
  match the number of detected available CPUs.
 
-.. option:: --config-prefix=NAME
+.. option:: --config-prefix NAME
 
  Search for :file:`{NAME}.cfg` and :file:`{NAME}.site.cfg` when searching for
  test suites, instead of :file:`lit.cfg` and :file:`lit.site.cfg`.
@@ -157,7 +157,7 @@ EXECUTION OPTIONS
 
  Disable sharding for GoogleTest format.
 
-.. option:: --path=PATH
+.. option:: --path PATH
 
  Specify an additional ``PATH`` to use when searching for executables in tests.
 
@@ -178,7 +178,7 @@ EXECUTION OPTIONS
  feature that can be used to conditionally disable (or expect failure in)
  certain tests.
 
-.. option:: --vg-arg=ARG
+.. option:: --vg-arg ARG
 
  When :option:`--vg` is used, specify an additional argument to pass to
  :program:`valgrind` itself.
@@ -208,7 +208,7 @@ EXECUTION OPTIONS
  Maximum time to spend running a single test (in seconds). 0 means no time
  limit. [Default: 0]
 
-.. option:: --timeout=N
+.. option:: --timeout N
 
  Spend at most ``N`` seconds (approximately) running each individual test.
  ``0`` means no time limit, and ``0`` is the default. Note that this is not an
@@ -268,17 +268,17 @@ The timing data is stored in the `test_exec_root` in a file named
 `.lit_test_times.txt`. If this file does not exist, then `lit` checks the
 `test_source_root` for the file to optionally accelerate clean builds.
 
-.. option:: --max-tests=N
+.. option:: --max-tests N
 
  Run at most ``N`` tests and then terminate.
 
-.. option:: --max-time=N
+.. option:: --max-time N
 
  Spend at most ``N`` seconds (approximately) running tests and then terminate.
  Note that this is not an alias for :option:`--timeout`; the two are
  different kinds of maximums.
 
-.. option:: --order={lexical,random,smart}
+.. option:: --order {lexical,random,smart}
 
  Define the order in which tests are run. The supported values are:
 
@@ -300,21 +300,21 @@ The timing data is stored in the `test_exec_root` in a file named
 
  Run failed tests first (DEPRECATED: use ``--order=smart``).
 
-.. option:: --filter=REGEXP
+.. option:: --filter REGEXP
 
   Run only those tests whose name matches the regular expression specified in
   ``REGEXP``. The environment variable ``LIT_FILTER`` can be also used in place
   of this option, which is especially useful in environments where the call
   to ``lit`` is issued indirectly.
 
-.. option:: --filter-out=REGEXP
+.. option:: --filter-out REGEXP
 
   Filter out those tests whose name matches the regular expression specified in
   ``REGEXP``. The environment variable ``LIT_FILTER_OUT`` can be also used in
   place of this option, which is especially useful in environments where the
   call to ``lit`` is issued indirectly.
 
-.. option:: --xfail=LIST
+.. option:: --xfail LIST
 
   Treat those tests whose name is in the semicolon separated list ``LIST`` as
   ``XFAIL``. This can be helpful when one does not want to modify the test
@@ -346,7 +346,7 @@ The timing data is stored in the `test_exec_root` in a file named
 
     LIT_XFAIL="affinity/kmp-hw-subset.c;libomptarget :: x86_64-pc-linux-gnu :: offloading/memory_manager.cpp"
 
-.. option:: --xfail-not=LIST
+.. option:: --xfail-not LIST
 
   Do not treat the specified tests as ``XFAIL``.  The environment variable
   ``LIT_XFAIL_NOT`` can also be used in place of this option.  The syntax is the
@@ -356,7 +356,7 @@ The timing data is stored in the `test_exec_root` in a file named
   primary purpose is to suppress an ``XPASS`` result without modifying a test
   case that uses the ``XFAIL`` directive.
 
-.. option:: --num-shards=M
+.. option:: --num-shards M
 
  Divide the set of selected tests into ``M`` equal-sized subsets or
  "shards", and run only one of them.  Must be used with the
@@ -366,7 +366,7 @@ The timing data is stored in the `test_exec_root` in a file named
  testsuites, for parallel execution on separate machines (say in a large
  testing farm).
 
-.. option:: --run-shard=N
+.. option:: --run-shard N
 
  Select which shard to run, assuming the ``--num-shards=M`` option was
  provided. The two options must be used together, and the value of ``N``
@@ -688,6 +688,14 @@ newline.
 The ``<progress info>`` field can be used to report progress information such
 as (1/300) or can be empty, but even when empty the parentheses are required.
 
+Should a test be allowed retries (see ``ALLOW_RETRIES:`` annotation) and it
+needed more than one attempt to succeed, then ``<progress info>`` is extended
+by this information:
+
+.. code-block:: none
+
+  , <num_attempts_made> of <max_allowed_attempts> attempts
+
 Each test result may include additional (multiline) log information in the
 following format:
 
diff --git a/external/llvm-project/llvm/docs/CommandGuide/llvm-addr2line.rst b/external/llvm-project/llvm/docs/CommandGuide/llvm-addr2line.rst
index 646a159cd24a..999b3e0d7e6a 100644
--- a/external/llvm-project/llvm/docs/CommandGuide/llvm-addr2line.rst
+++ b/external/llvm-project/llvm/docs/CommandGuide/llvm-addr2line.rst
@@ -36,6 +36,9 @@ Here are some of those differences:
 -  ``llvm-addr2line`` parses options from the environment variable
    ``LLVM_ADDR2LINE_OPTS`` instead of from ``LLVM_SYMBOLIZER_OPTS``.
 
+- ``llvm-addr2line`` accepts an address with a '+' prefix, e.g. `+0x00777fff`.
+  This is treated as a symbol name by ``llvm-symbolizer``.
+
 SEE ALSO
 --------
 
diff --git a/external/llvm-project/llvm/docs/CommandGuide/llvm-cov.rst b/external/llvm-project/llvm/docs/CommandGuide/llvm-cov.rst
index 968f3c452f55..f4db60cf06fa 100644
--- a/external/llvm-project/llvm/docs/CommandGuide/llvm-cov.rst
+++ b/external/llvm-project/llvm/docs/CommandGuide/llvm-cov.rst
@@ -380,6 +380,11 @@ OPTIONS
  Fail if an object file cannot be found for a binary ID present in the profile,
  neither on the command line nor via binary ID lookup.
 
+.. option:: -empty-profile
+
+ Display the baseline coverage of the binaries with all zero execution counts.
+ Mutually exclusive with -instr-profile.
+
 .. program:: llvm-cov report
 
 .. _llvm-cov-report:
@@ -470,6 +475,11 @@ OPTIONS
  Fail if an object file cannot be found for a binary ID present in the profile,
  neither on the command line nor via binary ID lookup.
 
+.. option:: -empty-profile
+
+ Display the baseline coverage of the binaries with all zero execution counts.
+ Mutually exclusive with -instr-profile.
+
 .. program:: llvm-cov export
 
 .. _llvm-cov-export:
@@ -562,6 +572,11 @@ OPTIONS
  Fail if an object file cannot be found for a binary ID present in the profile,
  neither on the command line nor via binary ID lookup.
 
+.. option:: -empty-profile
+
+ Export the baseline coverage of the binaries with all zero execution counts.
+ Mutually exclusive with -instr-profile.
+
 CONVERT-FOR-TESTING COMMAND
 ---------------------------
 
diff --git a/external/llvm-project/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst b/external/llvm-project/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst
index 7c15e65c1adf..1264f8020661 100644
--- a/external/llvm-project/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst
+++ b/external/llvm-project/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst
@@ -134,12 +134,13 @@ toolchain name, binary file format, etc.
  The following attributes describe the most common information for a
  logical element. They help to identify the lexical scope level; the
  element visibility across modules (global, local); the toolchain name
- that produced the binary file.
+ and source language that produced the binary file.
 
  .. code-block:: text
 
    =global: Element referenced across Compile Units.
    =format: Object file format name.
+   =language: Source language name.
    =level: Lexical scope level (File=0, Compile Unit=1).
    =local: Element referenced only in the Compile Unit.
    =producer: Toolchain identification name.
@@ -231,6 +232,7 @@ toolchain name, binary file format, etc.
    =filename
    =files
    =format
+   =language
    =level
    =producer
    =publics
@@ -674,8 +676,7 @@ INTERNAL
  Typically these kind of options are available only in *debug* builds.
 
  :program:`llvm-debuginfo-analyzer` supports these advanced options in
- both *release* and *debug* builds, with the exception of the unique ID
- that is generated only in *debug* builds.
+ both *release* and *debug* builds.
 
 .. option:: --internal=<value[,value,...]>
 
diff --git a/external/llvm-project/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst b/external/llvm-project/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst
new file mode 100644
index 000000000000..8cd5a349e7e4
--- /dev/null
+++ b/external/llvm-project/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst
@@ -0,0 +1,37 @@
+llvm-test-mustache-spec - LLVM tool to test Mustache library compliance
+=======================================================================
+
+.. program:: llvm-test-mustache-spec
+
+SYNOPSIS
+--------
+
+:program:`llvm-test-mustache-spec` [*inputs...*]
+
+Description
+-----------
+
+``llvm-test-mustache-spec`` tests the mustache spec conformance of the LLVM
+mustache library. The spec can be found here: https://github.com/mustache/spec
+
+To test against the spec, simply download the spec and pass the test JSON files
+to the driver. Each spec file should have a list of tests for compliance with
+the spec. These are loaded as test cases, and rendered with our Mustache
+implementation, which is then compared against the expected output from the
+spec.
+
+The current implementation only supports non-optional parts of the spec, so
+we do not expect any of the dynamic-names, inheritance, or lambda tests to
+pass. Additionally, Triple Mustache is not supported. Unsupported tests are
+marked as XFail and are removed from the XFail list as they are fixed.
+
+The tool prints the number of test failures and successes in each of the test
+files to standard output.
+
+EXAMPLE
+-------
+
+.. code-block:: console
+
+   $ llvm-test-mustache-spec path/to/specs/\*.json
+
diff --git a/external/llvm-project/llvm/docs/InstCombineContributorGuide.md b/external/llvm-project/llvm/docs/InstCombineContributorGuide.md
index b4041f8a5b93..cee0a7ce446a 100644
--- a/external/llvm-project/llvm/docs/InstCombineContributorGuide.md
+++ b/external/llvm-project/llvm/docs/InstCombineContributorGuide.md
@@ -404,11 +404,32 @@ The use of TargetTransformInfo is only allowed for hooks for target-specific
 intrinsics, such as `TargetTransformInfo::instCombineIntrinsic()`. These are
 already inherently target-dependent anyway.
 
+If some canonicalization narrow/widen the integer width of expressions, please
+check `shouldChangeType()` first. Otherwise, we may evaluate the expression 
+in illegal/inefficient types.
+
 For vector-specific transforms that require cost-modelling, the VectorCombine
 pass can be used instead. In very rare circumstances, if there are no other
 alternatives, target-dependent transforms may be accepted into
 AggressiveInstCombine.
 
+Generally, we prefer unsigned operations over signed operations in the middle-end, even
+if signed operations are more efficient on some targets. The following is an incomplete
+list of canonicalizations that are implemented in InstCombine:
+
+| Original Pattern             | Canonical Form             | Condition                     |
+|------------------------------|----------------------------|-------------------------------|
+| `icmp spred X, Y`            | `icmp samesign upred X, Y` | `sign(X) == sign(Y)`          |
+| `smin/smax X, Y`             | `umin/umax X, Y`           | `sign(X) == sign(Y)`          |
+| `sext X`                     | `zext nneg X`              | `X >=s 0`                     |
+| `sitofp X`                   | `uitofp nneg X`            | `X >=s 0`                     |
+| `ashr X, Y`                  | `lshr X, Y`                | `X >=s 0`                     |
+| `sdiv/srem X, Y`             | `udiv/urem X, Y`           | `X >=s 0 && Y >=s 0`          |
+| `add X, Y`                   | `or disjoint X, Y`         | `(X & Y) != 0`                |
+| `mul X, C`                   | `shl X, Log2(C)`           | `isPowerOf2(C)`               |
+| `select Cond1, Cond2, false` | `and Cond1, Cond2`         | `impliesPoison(Cond2, Cond1)` |
+| `select Cond1, true, Cond2`  | `or Cond1, Cond2`          | `impliesPoison(Cond2, Cond1)` |
+
 ### PatternMatch
 
 Many transforms make use of the matching infrastructure defined in
@@ -531,6 +552,19 @@ need to add a one-use check for the inner instruction.
 One-use checks can be performed using the `m_OneUse()` matcher, or the
 `V->hasOneUse()` method.
 
+### Flag handling
+
+When possible, favour propagation of poison-generating flags like `nuw` and `nsw` since they may be
+hard to salvage later. Avoid doing so if it introduces additional complexity (e.g. requires querying `willNotOverflow`
+or KnownBits).
+
+Be careful with in-place operand/predicate changes, as poison-generating flags may not be valid for new
+operands. It is recommended to create a new instruction with careful handling of flags. If not
+applicable, call `Instruction::dropPoisonGeneratingFlags()` to clear flags in a conservative manner.
+
+Do not rely on fcmp's `nsz` flag to perform optimizations. It is meaningless for fcmp so it should not affect
+the optimization.
+
 ### Generalization
 
 Transforms can both be too specific (only handling some odd subset of patterns,
@@ -558,6 +592,11 @@ guidelines.
    use of ValueTracking queries. Whether this makes sense depends on the case,
    but it's usually a good idea to only handle the constant pattern first, and
    then generalize later if it seems useful.
+ * When possible, handle more canonical patterns as well. It is encouraged to avoid
+   potential phase-ordering issues. For example, if the motivating transform holds for
+   `add`, it also holds for `or disjoint`. See the canonicalization list above for details.
+   In most cases, it can be easily implemented with matchers like
+   `m_AddLike/m_SExtLike/m_LogicalAnd/m_LogicalOr`.
 
 ## Guidelines for reviewers
 
diff --git a/external/llvm-project/llvm/docs/LangRef.rst b/external/llvm-project/llvm/docs/LangRef.rst
index 12806ceab348..227bf47e4771 100644
--- a/external/llvm-project/llvm/docs/LangRef.rst
+++ b/external/llvm-project/llvm/docs/LangRef.rst
@@ -1954,6 +1954,10 @@ For example:
     The first three options are mutually exclusive, and the remaining options
     describe more details of how the function behaves. The remaining options
     are invalid for "free"-type functions.
+``"alloc-variant-zeroed"="FUNCTION"``
+    This attribute indicates that another function is equivalent to an allocator function,
+    but returns zeroed memory. The function must have "zeroed" allocation behavior,
+    the same ``alloc-family``, and take exactly the same arguments.
 ``allocsize(<EltSizeParam>[, <NumEltsParam>])``
     This attribute indicates that the annotated function will always return at
     least a given number of bytes (or null). Its arguments are zero-indexed
@@ -4434,7 +4438,8 @@ the type size is smaller than the type's store size.
       < vscale x <# elements> x <elementtype> > ; Scalable vector
 
 The number of elements is a constant integer value larger than 0;
-elementtype may be any integer, floating-point or pointer type. Vectors
+elementtype may be any integer, floating-point, pointer type, or a sized  
+target extension type that has the ``CanBeVectorElement`` property. Vectors
 of size zero are not allowed. For scalable vectors, the total number of
 elements is a constant multiple (called vscale) of the specified number
 of elements; vscale is a positive integer that is unknown at compile time
@@ -9597,6 +9602,14 @@ may not be equal to the address provided for the same block to this
 instruction's ``indirect labels`` operand. The assembly code may only transfer
 control to addresses provided via this instruction's ``indirect labels``.
 
+On target architectures that implement branch target enforcement by requiring
+indirect (register-controlled) branch instructions to jump only to locations
+marked by a special instruction (such as AArch64 ``bti``), the called code is
+expected not to use such an indirect branch to transfer control to the
+locations in ``indirect labels``. Therefore, including a label in the
+``indirect labels`` of a ``callbr`` does not require the compiler to put a
+``bti`` or equivalent instruction at the label.
+
 Arguments:
 """"""""""
 
@@ -20533,6 +20546,9 @@ More update operation types may be added in the future.
 
     declare void @llvm.experimental.vector.histogram.add.v8p0.i32(<8 x ptr> %ptrs, i32 %inc, <8 x i1> %mask)
     declare void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %ptrs, i64 %inc, <vscale x 2 x i1> %mask)
+    declare void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i32(<8 x ptr> %ptrs, i32 %inc, <8 x i1> %mask)
+    declare void @llvm.experimental.vector.histogram.umax.v8p0.i32(<8 x ptr> %ptrs, i32 %val, <8 x i1> %mask)
+    declare void @llvm.experimental.vector.histogram.umin.v8p0.i32(<8 x ptr> %ptrs, i32 %val, <8 x i1> %mask)
 
 Arguments:
 """"""""""
diff --git a/external/llvm-project/llvm/docs/MLGO.rst b/external/llvm-project/llvm/docs/MLGO.rst
index 377c2aec4447..4f8fb3f59ca1 100644
--- a/external/llvm-project/llvm/docs/MLGO.rst
+++ b/external/llvm-project/llvm/docs/MLGO.rst
@@ -469,7 +469,6 @@ embeddings can be computed and accessed via an ``ir2vec::Embedder`` instance.
         return;
       }
       const ir2vec::Vocab &Vocabulary = VocabRes.getVocabulary();
-      unsigned Dimension = VocabRes.getDimension();
 
    Note that ``IR2VecVocabAnalysis`` pass is immutable.
 
@@ -481,7 +480,7 @@ embeddings can be computed and accessed via an ``ir2vec::Embedder`` instance.
       // Assuming F is an llvm::Function&
       // For example, using IR2VecKind::Symbolic:
       Expected<std::unique_ptr<ir2vec::Embedder>> EmbOrErr =
-          ir2vec::Embedder::create(IR2VecKind::Symbolic, F, Vocabulary, Dimension);
+          ir2vec::Embedder::create(IR2VecKind::Symbolic, F, Vocabulary);
 
       if (auto Err = EmbOrErr.takeError()) {
         // Handle error in embedder creation
diff --git a/external/llvm-project/llvm/docs/NVPTXUsage.rst b/external/llvm-project/llvm/docs/NVPTXUsage.rst
index 8bb0f2ed17c3..abd7ca545364 100644
--- a/external/llvm-project/llvm/docs/NVPTXUsage.rst
+++ b/external/llvm-project/llvm/docs/NVPTXUsage.rst
@@ -1016,7 +1016,7 @@ Syntax:
 
 .. code-block:: llvm
 
-  declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch)
+  declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %flag_cta_group)
   declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(..., i32 %d0, i32 %d1, ...)
   declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
   declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
@@ -1034,18 +1034,26 @@ source tensor is preserved at the destination. The dimension of the
 tensor data ranges from 1d to 5d with the coordinates specified
 by the ``i32 %d0 ... i32 %d4`` arguments.
 
-* The last two arguments to these intrinsics are boolean flags
-  indicating support for cache_hint and/or multicast modifiers.
-  These flag arguments must be compile-time constants. The backend
-  looks through these flags and lowers the intrinsics appropriately.
+* The last three arguments to these intrinsics are flags
+  indicating support for multicast, cache_hint and cta_group::1/2
+  modifiers. These flag arguments must be compile-time constants.
+  The backend looks through these flags and lowers the intrinsics
+  appropriately.
 
-* The Nth argument (denoted by ``i1 flag_ch``) when set, indicates
+* The argument denoted by ``i1 %flag_ch`` when set, indicates
   a valid cache_hint (``i64 %ch``) and generates the ``.L2::cache_hint``
   variant of the PTX instruction.
 
-* The [N-1]th argument (denoted by ``i1 flag_mc``) when set, indicates
-  the presence of a multicast mask (``i16 %mc``) and generates the PTX
-  instruction with the ``.multicast::cluster`` modifier.
+* The argument denoted by ``i1 %flag_mc`` when set, indicates
+  the presence of a multicast mask (``i16 %mc``) and generates
+  the PTX instruction with the ``.multicast::cluster`` modifier.
+
+* The argument denoted by ``i32 %flag_cta_group`` takes values within
+  the range [0, 3) i.e. {0,1,2}. When the value of ``%flag_cta_group``
+  is not within the range, it may raise an error from the Verifier.
+  The default value is '0' with no cta_group modifier in the
+  instruction. The values of '1' and '2' lower to ``cta_group::1``
+  and ``cta_group::2`` variants of the PTX instruction respectively.
 
 For more information, refer PTX ISA
 `<https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor>`_.
@@ -1058,7 +1066,7 @@ Syntax:
 
 .. code-block:: llvm
 
-  declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch)
+  declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %flag_cta_group)
   declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, ...)
   declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
 
@@ -1074,8 +1082,8 @@ are unrolled into a single dimensional column at the destination. In this
 mode, the tensor has to be at least three-dimensional. Along with the tensor
 coordinates, im2col offsets are also specified (denoted by
 ``i16 im2col0...i16 %im2col2``). The number of im2col offsets is two less
-than the number of dimensions of the tensor operation. The last two arguments
-to these intrinsics are boolean flags, with the same functionality as described
+than the number of dimensions of the tensor operation. The last three arguments
+to these intrinsics are flags, with the same functionality as described
 in the ``tile`` mode intrinsics above.
 
 For more information, refer PTX ISA
@@ -1868,6 +1876,29 @@ If the request failed, the behavior of these intrinsics is undefined.
 
 For more information, refer `PTX ISA <https://docs.nvidia.com/cuda/parallel-thread-execution/?a#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel>`__.
 
+Perf Monitor Event Intrinsics
+-----------------------------
+
+'``llvm.nvvm.pm.event.mask``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+    declare void @llvm.nvvm.pm.event.mask(i16 immarg %mask_val)
+
+Overview:
+"""""""""
+
+The '``llvm.nvvm.pm.event.mask``' intrinsic triggers one or more
+performance monitor events. Each bit in the 16-bit immediate operand
+``%mask_val`` controls an event.
+
+For more information on the pmevent instructions, refer to the PTX ISA
+`<https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-pmevent>`_.
+
 Other Intrinsics
 ----------------
 
diff --git a/external/llvm-project/llvm/docs/Passes.rst b/external/llvm-project/llvm/docs/Passes.rst
index 435e748199b8..e297d3883c39 100644
--- a/external/llvm-project/llvm/docs/Passes.rst
+++ b/external/llvm-project/llvm/docs/Passes.rst
@@ -554,8 +554,8 @@ variables with initializers are marked as internal.
 An interprocedural variant of :ref:`Sparse Conditional Constant Propagation
 <passes-sccp>`.
 
-``ir-normalizer``: Transforms IR into a normal form that's easier to diff
-----------------------------------------------------------------------------
+``normalize``: Transforms IR into a normal form that's easier to diff
+---------------------------------------------------------------------
 
 This pass aims to transform LLVM Modules into a normal form by reordering and
 renaming instructions while preserving the same semantics. The normalizer makes
diff --git a/external/llvm-project/llvm/docs/RISCVUsage.rst b/external/llvm-project/llvm/docs/RISCVUsage.rst
index 7d0d0cc21a27..64f17f59575e 100644
--- a/external/llvm-project/llvm/docs/RISCVUsage.rst
+++ b/external/llvm-project/llvm/docs/RISCVUsage.rst
@@ -487,7 +487,7 @@ The current vendor extensions supported are:
   LLVM implements `version 0.3 of the Qualcomm uC Large Offset Load Store extension specification <https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.11.0>`__ by Qualcomm. These instructions are only available for riscv32.
 
 ``experimental-Xqcilsm``
-  LLVM implements `version 0.5 of the Qualcomm uC Load Store Multiple extension specification <https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.11.0>`__ by Qualcomm. These instructions are only available for riscv32.
+  LLVM implements `version 0.6 of the Qualcomm uC Load Store Multiple extension specification <https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.13.0>`__ by Qualcomm. These instructions are only available for riscv32.
 
 ``experimental-Xqcisim``
   LLVM implements `version 0.2 of the Qualcomm uC Simulation Hint extension specification <https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.11.0>`__ by Qualcomm. These instructions are only available for riscv32.
diff --git a/external/llvm-project/llvm/docs/ReleaseNotes.md b/external/llvm-project/llvm/docs/ReleaseNotes.md
index 10efc889fcc7..5c9ed181af59 100644
--- a/external/llvm-project/llvm/docs/ReleaseNotes.md
+++ b/external/llvm-project/llvm/docs/ReleaseNotes.md
@@ -70,6 +70,12 @@ Changes to the LLVM IR
   `llvm.load.relative`.
 * Inline asm calls no longer accept ``label`` arguments. Use ``callbr`` instead.
 
+* Updated semantics of the `callbr` instruction to clarify that its
+  'indirect labels' are not expected to be reached by indirect (as in
+  register-controlled) branch instructions, and therefore are not
+  guaranteed to start with a `bti` or `endbr64` instruction, where
+  those exist.
+
 Changes to LLVM infrastructure
 ------------------------------
 
@@ -202,6 +208,8 @@ Changes to the RISC-V Backend
   Extensions.
 * `-mcpu=andes-a25` and `-mcpu=andes-ax25` were added.
 * The `Shlcofideleg` extension was added.
+* `-mcpu=sifive-x390` was added.
+* `-mtune=andes-45-series` was added.
 
 Changes to the WebAssembly Backend
 ----------------------------------
@@ -215,6 +223,8 @@ Changes to the Windows Target
 Changes to the X86 Backend
 --------------------------
 
+* `fp128` will now use `*f128` libcalls on 32-bit GNU targets as well.
+
 Changes to the OCaml bindings
 -----------------------------
 
@@ -256,6 +266,7 @@ Changes to the LLVM tools
 * llvm-strip now supports continuing to process files on encountering an error.
 * In llvm-objcopy/llvm-strip's ELF port, `--discard-locals` and `--discard-all` now allow and preserve symbols referenced by relocations.
   ([#47468](https://github.com/llvm/llvm-project/issues/47468))
+* llvm-addr2line now supports a `+` prefix when specifying an address.
 
 Changes to LLDB
 ---------------------------------
diff --git a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
index bb9e6b8c99d2..73a7b5b38a38 100644
--- a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
+++ b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
@@ -47,7 +47,9 @@ class KaleidoscopeJIT {
                   JITTargetMachineBuilder JTMB, DataLayout DL)
       : ES(std::move(ES)), DL(std::move(DL)), Mangle(*this->ES, this->DL),
         ObjectLayer(*this->ES,
-                    []() { return std::make_unique<SectionMemoryManager>(); }),
+                    [](const MemoryBuffer &) {
+                      return std::make_unique<SectionMemoryManager>();
+                    }),
         CompileLayer(*this->ES, ObjectLayer,
                      std::make_unique<ConcurrentIRCompiler>(std::move(JTMB))),
         MainJD(this->ES->createBareJITDylib("<main>")) {
diff --git a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
index 2f7846e7ee2c..d2bedd2c7270 100644
--- a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
+++ b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
@@ -53,7 +53,9 @@ class KaleidoscopeJIT {
                   JITTargetMachineBuilder JTMB, DataLayout DL)
       : ES(std::move(ES)), DL(std::move(DL)), Mangle(*this->ES, this->DL),
         ObjectLayer(*this->ES,
-                    []() { return std::make_unique<SectionMemoryManager>(); }),
+                    [](const MemoryBuffer &) {
+                      return std::make_unique<SectionMemoryManager>();
+                    }),
         CompileLayer(*this->ES, ObjectLayer,
                      std::make_unique<ConcurrentIRCompiler>(std::move(JTMB))),
         OptimizeLayer(*this->ES, CompileLayer, optimizeModule),
diff --git a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
index fee2d26e5d92..09b87b0181f9 100644
--- a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
+++ b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
@@ -66,7 +66,9 @@ class KaleidoscopeJIT {
       : ES(std::move(ES)), EPCIU(std::move(EPCIU)), DL(std::move(DL)),
         Mangle(*this->ES, this->DL),
         ObjectLayer(*this->ES,
-                    []() { return std::make_unique<SectionMemoryManager>(); }),
+                    [](const MemoryBuffer &) {
+                      return std::make_unique<SectionMemoryManager>();
+                    }),
         CompileLayer(*this->ES, ObjectLayer,
                      std::make_unique<ConcurrentIRCompiler>(std::move(JTMB))),
         OptimizeLayer(*this->ES, CompileLayer, optimizeModule),
diff --git a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
index 136e88505a1d..30c0e239a6cb 100644
--- a/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
+++ b/external/llvm-project/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
@@ -151,7 +151,9 @@ class KaleidoscopeJIT {
       : ES(std::move(ES)), EPCIU(std::move(EPCIU)), DL(std::move(DL)),
         Mangle(*this->ES, this->DL),
         ObjectLayer(*this->ES,
-                    []() { return std::make_unique<SectionMemoryManager>(); }),
+                    [](const MemoryBuffer &) {
+                      return std::make_unique<SectionMemoryManager>();
+                    }),
         CompileLayer(*this->ES, ObjectLayer,
                      std::make_unique<ConcurrentIRCompiler>(std::move(JTMB))),
         OptimizeLayer(*this->ES, CompileLayer, optimizeModule),
diff --git a/external/llvm-project/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h b/external/llvm-project/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
index 778437cd8a3d..65f94ed7a80d 100644
--- a/external/llvm-project/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
+++ b/external/llvm-project/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
@@ -48,7 +48,9 @@ class KaleidoscopeJIT {
                   JITTargetMachineBuilder JTMB, DataLayout DL)
       : ES(std::move(ES)), DL(std::move(DL)), Mangle(*this->ES, this->DL),
         ObjectLayer(*this->ES,
-                    []() { return std::make_unique<SectionMemoryManager>(); }),
+                    [](const MemoryBuffer &) {
+                      return std::make_unique<SectionMemoryManager>();
+                    }),
         CompileLayer(*this->ES, ObjectLayer,
                      std::make_unique<ConcurrentIRCompiler>(std::move(JTMB))),
         MainJD(this->ES->createBareJITDylib("<main>")) {
diff --git a/external/llvm-project/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp b/external/llvm-project/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
index df2f48cef14c..8623834dcf42 100644
--- a/external/llvm-project/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
+++ b/external/llvm-project/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
@@ -63,7 +63,7 @@ int main(int argc, char *argv[]) {
       ExitOnErr(LLJITBuilder()
                     .setJITTargetMachineBuilder(std::move(JTMB))
                     .setObjectLinkingLayerCreator([&](ExecutionSession &ES) {
-                      auto GetMemMgr = []() {
+                      auto GetMemMgr = [](const MemoryBuffer &) {
                         return std::make_unique<SectionMemoryManager>();
                       };
                       auto ObjLinkingLayer =
diff --git a/external/llvm-project/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp b/external/llvm-project/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
index 24ae953eca7a..acdbce4f6e3e 100644
--- a/external/llvm-project/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
+++ b/external/llvm-project/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
@@ -125,7 +125,8 @@ class SpeculativeJIT {
     ExitOnErr(CXXRuntimeoverrides.enable(MainJD, Mangle));
   }
 
-  static std::unique_ptr<SectionMemoryManager> createMemMgr() {
+  static std::unique_ptr<SectionMemoryManager>
+  createMemMgr(const MemoryBuffer &) {
     return std::make_unique<SectionMemoryManager>();
   }
 
diff --git a/external/llvm-project/llvm/include/llvm/ADT/DenseMap.h b/external/llvm-project/llvm/include/llvm/ADT/DenseMap.h
index de41a212c65a..3825f8d52372 100644
--- a/external/llvm-project/llvm/include/llvm/ADT/DenseMap.h
+++ b/external/llvm-project/llvm/include/llvm/ADT/DenseMap.h
@@ -220,7 +220,8 @@ class DenseMapBase : public DebugEpochBase {
   // Return the entry with the specified key, or \p Default. This variant is
   // useful, because `lookup` cannot be used with non-default-constructible
   // values.
-  ValueT lookup_or(const_arg_type_t<KeyT> Val, ValueT &&Default) const {
+  template <typename U = std::remove_cv_t<ValueT>>
+  ValueT lookup_or(const_arg_type_t<KeyT> Val, U &&Default) const {
     if (const BucketT *Bucket = doFind(Val))
       return Bucket->getSecond();
     return Default;
diff --git a/external/llvm-project/llvm/include/llvm/ADT/MapVector.h b/external/llvm-project/llvm/include/llvm/ADT/MapVector.h
index adbcda116dd5..015605fd98df 100644
--- a/external/llvm-project/llvm/include/llvm/ADT/MapVector.h
+++ b/external/llvm-project/llvm/include/llvm/ADT/MapVector.h
@@ -99,8 +99,7 @@ class MapVector {
   }
 
   ValueT &operator[](const KeyT &Key) {
-    std::pair<KeyT, typename MapType::mapped_type> Pair = std::make_pair(Key, 0);
-    std::pair<typename MapType::iterator, bool> Result = Map.insert(Pair);
+    std::pair<typename MapType::iterator, bool> Result = Map.try_emplace(Key);
     auto &I = Result.first->second;
     if (Result.second) {
       Vector.push_back(std::make_pair(Key, ValueT()));
@@ -119,7 +118,7 @@ class MapVector {
 
   template <typename... Ts>
   std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&...Args) {
-    auto [It, Inserted] = Map.insert(std::make_pair(Key, 0));
+    auto [It, Inserted] = Map.try_emplace(Key);
     if (Inserted) {
       It->second = Vector.size();
       Vector.emplace_back(std::piecewise_construct, std::forward_as_tuple(Key),
@@ -130,7 +129,7 @@ class MapVector {
   }
   template <typename... Ts>
   std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&...Args) {
-    auto [It, Inserted] = Map.insert(std::make_pair(Key, 0));
+    auto [It, Inserted] = Map.try_emplace(Key);
     if (Inserted) {
       It->second = Vector.size();
       Vector.emplace_back(std::piecewise_construct,
diff --git a/external/llvm-project/llvm/include/llvm/ADT/STLExtras.h b/external/llvm-project/llvm/include/llvm/ADT/STLExtras.h
index bc4be2b65f2c..a252570c7eb7 100644
--- a/external/llvm-project/llvm/include/llvm/ADT/STLExtras.h
+++ b/external/llvm-project/llvm/include/llvm/ADT/STLExtras.h
@@ -1940,6 +1940,28 @@ template <typename R> bool is_sorted(R &&Range) {
   return std::is_sorted(adl_begin(Range), adl_end(Range));
 }
 
+/// Provide wrappers to std::includes which take ranges instead of having to
+/// pass begin/end explicitly.
+/// This function checks if the sorted range \p R2 is a subsequence of the
+/// sorted range \p R1. The ranges must be sorted in non-descending order.
+template <typename R1, typename R2> bool includes(R1 &&Range1, R2 &&Range2) {
+  assert(is_sorted(Range1) && "Range1 must be sorted in non-descending order");
+  assert(is_sorted(Range2) && "Range2 must be sorted in non-descending order");
+  return std::includes(adl_begin(Range1), adl_end(Range1), adl_begin(Range2),
+                       adl_end(Range2));
+}
+
+/// This function checks if the sorted range \p R2 is a subsequence of the
+/// sorted range \p R1. The ranges must be sorted with respect to a comparator
+/// \p C.
+template <typename R1, typename R2, typename Compare>
+bool includes(R1 &&Range1, R2 &&Range2, Compare &&C) {
+  assert(is_sorted(Range1, C) && "Range1 must be sorted with respect to C");
+  assert(is_sorted(Range2, C) && "Range2 must be sorted with respect to C");
+  return std::includes(adl_begin(Range1), adl_end(Range1), adl_begin(Range2),
+                       adl_end(Range2), std::forward<Compare>(C));
+}
+
 /// Wrapper function around std::count to count the number of times an element
 /// \p Element occurs in the given range \p Range.
 template <typename R, typename E> auto count(R &&Range, const E &Element) {
diff --git a/external/llvm-project/llvm/include/llvm/ADT/bit.h b/external/llvm-project/llvm/include/llvm/ADT/bit.h
index 8544efb5c45d..d6e33c3e6133 100644
--- a/external/llvm-project/llvm/include/llvm/ADT/bit.h
+++ b/external/llvm-project/llvm/include/llvm/ADT/bit.h
@@ -148,36 +148,20 @@ template <typename T, typename = std::enable_if_t<std::is_unsigned_v<T>>>
   return (Value != 0) && ((Value & (Value - 1)) == 0);
 }
 
-namespace detail {
-template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
-  static unsigned count(T Val) {
-    if (!Val)
-      return std::numeric_limits<T>::digits;
-    if (Val & 0x1)
-      return 0;
-
-    // Bisection method.
-    unsigned ZeroBits = 0;
-    T Shift = std::numeric_limits<T>::digits >> 1;
-    T Mask = std::numeric_limits<T>::max() >> Shift;
-    while (Shift) {
-      if ((Val & Mask) == 0) {
-        Val >>= Shift;
-        ZeroBits |= Shift;
-      }
-      Shift >>= 1;
-      Mask >>= Shift;
-    }
-    return ZeroBits;
-  }
-};
-
-#if defined(__GNUC__) || defined(_MSC_VER)
-template <typename T> struct TrailingZerosCounter<T, 4> {
-  static unsigned count(T Val) {
-    if (Val == 0)
-      return 32;
+/// Count number of 0's from the least significant bit to the most
+///   stopping at the first 1.
+///
+/// Only unsigned integral types are allowed.
+///
+/// Returns std::numeric_limits<T>::digits on an input of 0.
+template <typename T> [[nodiscard]] int countr_zero(T Val) {
+  static_assert(std::is_unsigned_v<T>,
+                "Only unsigned integral types are allowed.");
+  if (!Val)
+    return std::numeric_limits<T>::digits;
 
+  // Use the intrinsic if available.
+  if constexpr (sizeof(T) == 4) {
 #if __has_builtin(__builtin_ctz) || defined(__GNUC__)
     return __builtin_ctz(Val);
 #elif defined(_MSC_VER)
@@ -185,65 +169,45 @@ template <typename T> struct TrailingZerosCounter<T, 4> {
     _BitScanForward(&Index, Val);
     return Index;
 #endif
-  }
-};
-
-#if !defined(_MSC_VER) || defined(_M_X64)
-template <typename T> struct TrailingZerosCounter<T, 8> {
-  static unsigned count(T Val) {
-    if (Val == 0)
-      return 64;
-
+  } else if constexpr (sizeof(T) == 8) {
 #if __has_builtin(__builtin_ctzll) || defined(__GNUC__)
     return __builtin_ctzll(Val);
-#elif defined(_MSC_VER)
+#elif defined(_MSC_VER) && defined(_M_X64)
     unsigned long Index;
     _BitScanForward64(&Index, Val);
     return Index;
 #endif
   }
-};
-#endif
-#endif
-} // namespace detail
 
-/// Count number of 0's from the least significant bit to the most
+  // Fall back to the bisection method.
+  unsigned ZeroBits = 0;
+  T Shift = std::numeric_limits<T>::digits >> 1;
+  T Mask = std::numeric_limits<T>::max() >> Shift;
+  while (Shift) {
+    if ((Val & Mask) == 0) {
+      Val >>= Shift;
+      ZeroBits |= Shift;
+    }
+    Shift >>= 1;
+    Mask >>= Shift;
+  }
+  return ZeroBits;
+}
+
+/// Count number of 0's from the most significant bit to the least
 ///   stopping at the first 1.
 ///
 /// Only unsigned integral types are allowed.
 ///
 /// Returns std::numeric_limits<T>::digits on an input of 0.
-template <typename T> [[nodiscard]] int countr_zero(T Val) {
+template <typename T> [[nodiscard]] int countl_zero(T Val) {
   static_assert(std::is_unsigned_v<T>,
                 "Only unsigned integral types are allowed.");
-  return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val);
-}
-
-namespace detail {
-template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
-  static unsigned count(T Val) {
-    if (!Val)
-      return std::numeric_limits<T>::digits;
-
-    // Bisection method.
-    unsigned ZeroBits = 0;
-    for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
-      T Tmp = Val >> Shift;
-      if (Tmp)
-        Val = Tmp;
-      else
-        ZeroBits |= Shift;
-    }
-    return ZeroBits;
-  }
-};
-
-#if defined(__GNUC__) || defined(_MSC_VER)
-template <typename T> struct LeadingZerosCounter<T, 4> {
-  static unsigned count(T Val) {
-    if (Val == 0)
-      return 32;
+  if (!Val)
+    return std::numeric_limits<T>::digits;
 
+  // Use the intrinsic if available.
+  if constexpr (sizeof(T) == 4) {
 #if __has_builtin(__builtin_clz) || defined(__GNUC__)
     return __builtin_clz(Val);
 #elif defined(_MSC_VER)
@@ -251,38 +215,26 @@ template <typename T> struct LeadingZerosCounter<T, 4> {
     _BitScanReverse(&Index, Val);
     return Index ^ 31;
 #endif
-  }
-};
-
-#if !defined(_MSC_VER) || defined(_M_X64)
-template <typename T> struct LeadingZerosCounter<T, 8> {
-  static unsigned count(T Val) {
-    if (Val == 0)
-      return 64;
-
+  } else if constexpr (sizeof(T) == 8) {
 #if __has_builtin(__builtin_clzll) || defined(__GNUC__)
     return __builtin_clzll(Val);
-#elif defined(_MSC_VER)
+#elif defined(_MSC_VER) && defined(_M_X64)
     unsigned long Index;
     _BitScanReverse64(&Index, Val);
     return Index ^ 63;
 #endif
   }
-};
-#endif
-#endif
-} // namespace detail
 
-/// Count number of 0's from the most significant bit to the least
-///   stopping at the first 1.
-///
-/// Only unsigned integral types are allowed.
-///
-/// Returns std::numeric_limits<T>::digits on an input of 0.
-template <typename T> [[nodiscard]] int countl_zero(T Val) {
-  static_assert(std::is_unsigned_v<T>,
-                "Only unsigned integral types are allowed.");
-  return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val);
+  // Fall back to the bisection method.
+  unsigned ZeroBits = 0;
+  for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
+    T Tmp = Val >> Shift;
+    if (Tmp)
+      Val = Tmp;
+    else
+      ZeroBits |= Shift;
+  }
+  return ZeroBits;
 }
 
 /// Count the number of ones from the most significant bit to the first
@@ -348,11 +300,12 @@ template <typename T> [[nodiscard]] T bit_ceil(T Value) {
   return T(1) << llvm::bit_width<T>(Value - 1u);
 }
 
-namespace detail {
-template <typename T, std::size_t SizeOfT> struct PopulationCounter {
-  static int count(T Value) {
-    // Generic version, forward to 32 bits.
-    static_assert(SizeOfT <= 4, "Not implemented!");
+/// Count the number of set bits in a value.
+/// Ex. popcount(0xF000F000) = 8
+/// Returns 0 if the word is zero.
+template <typename T, typename = std::enable_if_t<std::is_unsigned_v<T>>>
+[[nodiscard]] inline int popcount(T Value) noexcept {
+  if constexpr (sizeof(T) <= 4) {
 #if defined(__GNUC__)
     return (int)__builtin_popcount(Value);
 #else
@@ -361,11 +314,7 @@ template <typename T, std::size_t SizeOfT> struct PopulationCounter {
     v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
     return int(((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24);
 #endif
-  }
-};
-
-template <typename T> struct PopulationCounter<T, 8> {
-  static int count(T Value) {
+  } else if constexpr (sizeof(T) <= 8) {
 #if defined(__GNUC__)
     return (int)__builtin_popcountll(Value);
 #else
@@ -375,16 +324,9 @@ template <typename T> struct PopulationCounter<T, 8> {
     v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
     return int((uint64_t)(v * 0x0101010101010101ULL) >> 56);
 #endif
+  } else {
+    static_assert(sizeof(T) == 0, "T must be 8 bytes or less");
   }
-};
-} // namespace detail
-
-/// Count the number of set bits in a value.
-/// Ex. popcount(0xF000F000) = 8
-/// Returns 0 if the word is zero.
-template <typename T, typename = std::enable_if_t<std::is_unsigned_v<T>>>
-[[nodiscard]] inline int popcount(T Value) noexcept {
-  return detail::PopulationCounter<T, sizeof(T)>::count(Value);
 }
 
 // Forward-declare rotr so that rotl can use it.
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/AliasAnalysis.h b/external/llvm-project/llvm/include/llvm/Analysis/AliasAnalysis.h
index d8d88639b85a..b7d1251aeb72 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -149,23 +149,24 @@ LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, AliasResult AR);
 struct LLVM_ABI CaptureAnalysis {
   virtual ~CaptureAnalysis() = 0;
 
-  /// Check whether Object is not captured before instruction I. If OrAt is
-  /// true, captures by instruction I itself are also considered.
+  /// Return how Object may be captured before instruction I, considering only
+  /// provenance captures. If OrAt is true, captures by instruction I itself
+  /// are also considered.
   ///
   /// If I is nullptr, then captures at any point will be considered.
-  virtual bool isNotCapturedBefore(const Value *Object, const Instruction *I,
-                                   bool OrAt) = 0;
+  virtual CaptureComponents
+  getCapturesBefore(const Value *Object, const Instruction *I, bool OrAt) = 0;
 };
 
 /// Context-free CaptureAnalysis provider, which computes and caches whether an
 /// object is captured in the function at all, but does not distinguish whether
 /// it was captured before or after the context instruction.
 class LLVM_ABI SimpleCaptureAnalysis final : public CaptureAnalysis {
-  SmallDenseMap<const Value *, bool, 8> IsCapturedCache;
+  SmallDenseMap<const Value *, CaptureComponents, 8> IsCapturedCache;
 
 public:
-  bool isNotCapturedBefore(const Value *Object, const Instruction *I,
-                           bool OrAt) override;
+  CaptureComponents getCapturesBefore(const Value *Object, const Instruction *I,
+                                      bool OrAt) override;
 };
 
 /// Context-sensitive CaptureAnalysis provider, which computes and caches the
@@ -176,10 +177,12 @@ class LLVM_ABI EarliestEscapeAnalysis final : public CaptureAnalysis {
   const LoopInfo *LI;
 
   /// Map from identified local object to an instruction before which it does
-  /// not escape, or nullptr if it never escapes. The "earliest" instruction
-  /// may be a conservative approximation, e.g. the first instruction in the
-  /// function is always a legal choice.
-  DenseMap<const Value *, Instruction *> EarliestEscapes;
+  /// not escape (or nullptr if it never escapes) and the possible components
+  /// that may be captured (by any instruction, not necessarily the earliest
+  /// one). The "earliest" instruction may be a conservative approximation,
+  /// e.g. the first instruction in the function is always a legal choice.
+  DenseMap<const Value *, std::pair<Instruction *, CaptureComponents>>
+      EarliestEscapes;
 
   /// Reverse map from instruction to the objects it is the earliest escape for.
   /// This is used for cache invalidation purposes.
@@ -189,8 +192,8 @@ class LLVM_ABI EarliestEscapeAnalysis final : public CaptureAnalysis {
   EarliestEscapeAnalysis(DominatorTree &DT, const LoopInfo *LI = nullptr)
       : DT(DT), LI(LI) {}
 
-  bool isNotCapturedBefore(const Value *Object, const Instruction *I,
-                           bool OrAt) override;
+  CaptureComponents getCapturesBefore(const Value *Object, const Instruction *I,
+                                      bool OrAt) override;
 
   void removeInstruction(Instruction *I);
 };
@@ -912,7 +915,7 @@ LLVM_ABI bool isIdentifiedFunctionLocal(const Value *V);
 LLVM_ABI bool isBaseOfObject(const Value *V);
 
 /// Returns true if the pointer is one which would have been considered an
-/// escape by isNonEscapingLocalObject.
+/// escape by isNotCapturedBefore.
 LLVM_ABI bool isEscapeSource(const Value *V);
 
 /// Return true if Object memory is not visible after an unwind, in the sense
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/CaptureTracking.h b/external/llvm-project/llvm/include/llvm/Analysis/CaptureTracking.h
index ed160ca3596e..e652bc5a0a5a 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/CaptureTracking.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/CaptureTracking.h
@@ -95,21 +95,21 @@ namespace llvm {
       function_ref<bool(CaptureComponents)> StopFn = capturesAnything,
       const LoopInfo *LI = nullptr, unsigned MaxUsesToExplore = 0);
 
-  // Returns the 'earliest' instruction that captures \p V in \F. An instruction
-  // A is considered earlier than instruction B, if A dominates B. If 2 escapes
-  // do not dominate each other, the terminator of the common dominator is
-  // chosen. If not all uses can be analyzed, the earliest escape is set to
-  // the first instruction in the function entry block. If \p V does not escape,
-  // nullptr is returned. Note that the caller of the function has to ensure
-  // that the instruction the result value is compared against is not in a
-  // cycle.
+  // Returns the 'earliest' instruction that captures \p V in \F, and which
+  // components may be captured (by any use, not necessarily the earliest one).
+  // An instruction A is considered earlier than instruction B, if A dominates
+  // B. If 2 escapes do not dominate each other, the terminator of the common
+  // dominator is chosen. If not all uses can be analyzed, the earliest escape
+  // is set to the first instruction in the function entry block. If \p V does
+  // not escape, nullptr is returned. Note that the caller of the function has
+  // to ensure that the instruction the result value is compared against is
+  // not in a cycle.
   //
   // Only consider components that are part of \p Mask.
-  LLVM_ABI Instruction *FindEarliestCapture(const Value *V, Function &F,
-                                            bool ReturnCaptures,
-                                            const DominatorTree &DT,
-                                            CaptureComponents Mask,
-                                            unsigned MaxUsesToExplore = 0);
+  LLVM_ABI std::pair<Instruction *, CaptureComponents>
+  FindEarliestCapture(const Value *V, Function &F, bool ReturnCaptures,
+                      const DominatorTree &DT, CaptureComponents Mask,
+                      unsigned MaxUsesToExplore = 0);
 
   /// Capture information for a specific Use.
   struct UseCaptureInfo {
@@ -195,12 +195,6 @@ namespace llvm {
   /// implicit captures such as for external globals.
   LLVM_ABI void PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
                                      unsigned MaxUsesToExplore = 0);
-
-  /// Returns true if the pointer is to a function-local object that never
-  /// escapes from the function.
-  LLVM_ABI bool isNonEscapingLocalObject(
-      const Value *V,
-      SmallDenseMap<const Value *, bool, 8> *IsCapturedCache = nullptr);
 } // end namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/HashRecognize.h b/external/llvm-project/llvm/include/llvm/Analysis/HashRecognize.h
new file mode 100644
index 000000000000..8ab68a5dc2cb
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/Analysis/HashRecognize.h
@@ -0,0 +1,121 @@
+//===- HashRecognize.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface for the HashRecognize analysis, which identifies hash functions
+// that can be optimized using a lookup-table or with target-specific
+// instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_HASHRECOGNIZE_H
+#define LLVM_ANALYSIS_HASHRECOGNIZE_H
+
+#include "llvm/ADT/APInt.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Support/KnownBits.h"
+#include <variant>
+
+namespace llvm {
+
+class LPMUpdater;
+
+/// A tuple of bits that are expected to be zero, number N of them expected to
+/// be zero, with a boolean indicating whether it's the top or bottom N bits
+/// expected to be zero.
+using ErrBits = std::tuple<KnownBits, unsigned, bool>;
+
+/// A custom std::array with 256 entries, that also has a print function.
+struct CRCTable : public std::array<APInt, 256> {
+  void print(raw_ostream &OS) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  LLVM_DUMP_METHOD void dump() const;
+#endif
+};
+
+/// The structure that is returned when a polynomial algorithm was recognized by
+/// the analysis. Currently, only the CRC algorithm is recognized.
+struct PolynomialInfo {
+  // The small constant trip-count of the analyzed loop.
+  unsigned TripCount;
+
+  // The LHS in a polynomial operation, or the initial variable of the
+  // computation, since all polynomial operations must have a constant RHS,
+  // which is the generating polynomial. It is the LHS of the polynomial
+  // division in the case of CRC. Since polynomial division is an XOR in
+  // GF(2^m), this variable must be XOR'ed with RHS in a loop to yield the
+  // ComputedValue.
+  const Value *LHS;
+
+  // The generating polynomial, or the RHS of the polynomial division in the
+  // case of CRC.
+  APInt RHS;
+
+  // The final computed value. This is a remainder of a polynomial division in
+  // the case of CRC, which must be zero.
+  const Value *ComputedValue;
+
+  // Set to true in the case of big-endian.
+  bool ByteOrderSwapped;
+
+  // An optional auxiliary checksum that augments the LHS. In the case of CRC,
+  // it is XOR'ed with the LHS, so that the computation's final remainder is
+  // zero.
+  const Value *LHSAux;
+
+  PolynomialInfo(unsigned TripCount, const Value *LHS, const APInt &RHS,
+                 const Value *ComputedValue, bool ByteOrderSwapped,
+                 const Value *LHSAux = nullptr);
+};
+
+/// The analysis.
+class HashRecognize {
+  const Loop &L;
+  ScalarEvolution &SE;
+
+public:
+  HashRecognize(const Loop &L, ScalarEvolution &SE);
+
+  // The main analysis entry point.
+  std::variant<PolynomialInfo, ErrBits, StringRef> recognizeCRC() const;
+
+  // Auxilary entry point after analysis to interleave the generating polynomial
+  // and return a 256-entry CRC table.
+  CRCTable genSarwateTable(const APInt &GenPoly, bool ByteOrderSwapped) const;
+
+  void print(raw_ostream &OS) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  LLVM_DUMP_METHOD void dump() const;
+#endif
+};
+
+class HashRecognizePrinterPass
+    : public PassInfoMixin<HashRecognizePrinterPass> {
+  raw_ostream &OS;
+
+public:
+  explicit HashRecognizePrinterPass(raw_ostream &OS) : OS(OS) {}
+  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
+                        LoopStandardAnalysisResults &AR, LPMUpdater &);
+};
+
+class HashRecognizeAnalysis : public AnalysisInfoMixin<HashRecognizeAnalysis> {
+  friend AnalysisInfoMixin<HashRecognizeAnalysis>;
+  static AnalysisKey Key;
+
+public:
+  using Result = HashRecognize;
+  Result run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR);
+};
+} // namespace llvm
+
+#endif
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/IR2Vec.h b/external/llvm-project/llvm/include/llvm/Analysis/IR2Vec.h
index 288753b3b3b8..de67955d85d7 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/IR2Vec.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/IR2Vec.h
@@ -31,7 +31,9 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/JSON.h"
 #include <map>
 
 namespace llvm {
@@ -43,6 +45,7 @@ class Function;
 class Type;
 class Value;
 class raw_ostream;
+class LLVMContext;
 
 /// IR2Vec computes two kinds of embeddings: Symbolic and Flow-aware.
 /// Symbolic embeddings capture the "syntactic" and "statistical correlation"
@@ -53,7 +56,68 @@ class raw_ostream;
 enum class IR2VecKind { Symbolic };
 
 namespace ir2vec {
-using Embedding = std::vector<double>;
+
+extern cl::opt<float> OpcWeight;
+extern cl::opt<float> TypeWeight;
+extern cl::opt<float> ArgWeight;
+
+/// Embedding is a datatype that wraps std::vector<double>. It provides
+/// additional functionality for arithmetic and comparison operations.
+/// It is meant to be used *like* std::vector<double> but is more restrictive
+/// in the sense that it does not allow the user to change the size of the
+/// embedding vector. The dimension of the embedding is fixed at the time of
+/// construction of Embedding object. But the elements can be modified in-place.
+struct Embedding {
+private:
+  std::vector<double> Data;
+
+public:
+  Embedding() = default;
+  Embedding(const std::vector<double> &V) : Data(V) {}
+  Embedding(std::vector<double> &&V) : Data(std::move(V)) {}
+  Embedding(std::initializer_list<double> IL) : Data(IL) {}
+
+  explicit Embedding(size_t Size) : Data(Size) {}
+  Embedding(size_t Size, double InitialValue) : Data(Size, InitialValue) {}
+
+  size_t size() const { return Data.size(); }
+  bool empty() const { return Data.empty(); }
+
+  double &operator[](size_t Itr) {
+    assert(Itr < Data.size() && "Index out of bounds");
+    return Data[Itr];
+  }
+
+  const double &operator[](size_t Itr) const {
+    assert(Itr < Data.size() && "Index out of bounds");
+    return Data[Itr];
+  }
+
+  using iterator = typename std::vector<double>::iterator;
+  using const_iterator = typename std::vector<double>::const_iterator;
+
+  iterator begin() { return Data.begin(); }
+  iterator end() { return Data.end(); }
+  const_iterator begin() const { return Data.begin(); }
+  const_iterator end() const { return Data.end(); }
+  const_iterator cbegin() const { return Data.cbegin(); }
+  const_iterator cend() const { return Data.cend(); }
+
+  const std::vector<double> &getData() const { return Data; }
+
+  /// Arithmetic operators
+  Embedding &operator+=(const Embedding &RHS);
+  Embedding &operator-=(const Embedding &RHS);
+
+  /// Adds Src Embedding scaled by Factor with the called Embedding.
+  /// Called_Embedding += Src * Factor
+  Embedding &scaleAndAdd(const Embedding &Src, float Factor);
+
+  /// Returns true if the embedding is approximately equal to the RHS embedding
+  /// within the specified tolerance.
+  bool approximatelyEquals(const Embedding &RHS, double Tolerance = 1e-6) const;
+};
+
 using InstEmbeddingsMap = DenseMap<const Instruction *, Embedding>;
 using BBEmbeddingsMap = DenseMap<const BasicBlock *, Embedding>;
 // FIXME: Current the keys are strings. This can be changed to
@@ -61,8 +125,8 @@ using BBEmbeddingsMap = DenseMap<const BasicBlock *, Embedding>;
 using Vocab = std::map<std::string, Embedding>;
 
 /// Embedder provides the interface to generate embeddings (vector
-/// representations) for instructions, basic blocks, and functions. The vector
-/// representations are generated using IR2Vec algorithms.
+/// representations) for instructions, basic blocks, and functions. The
+/// vector representations are generated using IR2Vec algorithms.
 ///
 /// The Embedder class is an abstract class and it is intended to be
 /// subclassed for different IR2Vec algorithms like Symbolic and Flow-aware.
@@ -84,7 +148,7 @@ class Embedder {
   mutable BBEmbeddingsMap BBVecMap;
   mutable InstEmbeddingsMap InstVecMap;
 
-  Embedder(const Function &F, const Vocab &Vocabulary, unsigned Dimension);
+  Embedder(const Function &F, const Vocab &Vocabulary);
 
   /// Helper function to compute embeddings. It generates embeddings for all
   /// the instructions and basic blocks in the function F. Logic of computing
@@ -99,21 +163,12 @@ class Embedder {
   /// zero vector.
   Embedding lookupVocab(const std::string &Key) const;
 
-  /// Adds two vectors: Dst += Src
-  static void addVectors(Embedding &Dst, const Embedding &Src);
-
-  /// Adds Src vector scaled by Factor to Dst vector: Dst += Src * Factor
-  static void addScaledVector(Embedding &Dst, const Embedding &Src,
-                              float Factor);
-
 public:
   virtual ~Embedder() = default;
 
   /// Factory method to create an Embedder object.
-  static Expected<std::unique_ptr<Embedder>> create(IR2VecKind Mode,
-                                                    const Function &F,
-                                                    const Vocab &Vocabulary,
-                                                    unsigned Dimension);
+  static Expected<std::unique_ptr<Embedder>>
+  create(IR2VecKind Mode, const Function &F, const Vocab &Vocabulary);
 
   /// Returns a map containing instructions and the corresponding embeddings for
   /// the function F if it has been computed. If not, it computes the embeddings
@@ -149,9 +204,8 @@ class SymbolicEmbedder : public Embedder {
   void computeEmbeddings(const BasicBlock &BB) const override;
 
 public:
-  SymbolicEmbedder(const Function &F, const Vocab &Vocabulary,
-                   unsigned Dimension)
-      : Embedder(F, Vocabulary, Dimension) {
+  SymbolicEmbedder(const Function &F, const Vocab &Vocabulary)
+      : Embedder(F, Vocabulary) {
     FuncVector = Embedding(Dimension, 0);
   }
 };
@@ -180,10 +234,13 @@ class IR2VecVocabResult {
 class IR2VecVocabAnalysis : public AnalysisInfoMixin<IR2VecVocabAnalysis> {
   ir2vec::Vocab Vocabulary;
   Error readVocabulary();
+  void emitError(Error Err, LLVMContext &Ctx);
 
 public:
   static AnalysisKey Key;
   IR2VecVocabAnalysis() = default;
+  explicit IR2VecVocabAnalysis(const ir2vec::Vocab &Vocab);
+  explicit IR2VecVocabAnalysis(ir2vec::Vocab &&Vocab);
   using Result = IR2VecVocabResult;
   Result run(Module &M, ModuleAnalysisManager &MAM);
 };
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/IVDescriptors.h b/external/llvm-project/llvm/include/llvm/Analysis/IVDescriptors.h
index d12780cde71d..3b627a514085 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -172,8 +172,7 @@ class RecurrenceDescriptor {
 
   /// Returns a struct describing if the instruction is a
   /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
-  LLVM_ABI static InstDesc isConditionalRdxPattern(RecurKind Kind,
-                                                   Instruction *I);
+  LLVM_ABI static InstDesc isConditionalRdxPattern(Instruction *I);
 
   /// Returns the opcode corresponding to the RecurrenceKind.
   LLVM_ABI static unsigned getOpcode(RecurKind Kind);
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/external/llvm-project/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 7cb96422589f..2a44ec803223 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -194,7 +194,8 @@ class MemoryDepChecker {
   /// of a write access.
   LLVM_ABI void addAccess(LoadInst *LI);
 
-  /// Check whether the dependencies between the accesses are safe.
+  /// Check whether the dependencies between the accesses are safe, and records
+  /// the dependence information in Dependences if so.
   ///
   /// Only checks sets with elements in \p CheckDeps.
   LLVM_ABI bool areDepsSafe(const DepCandidates &AccessSets,
@@ -654,7 +655,8 @@ class RuntimePointerChecking {
 /// For memory dependences that cannot be determined at compile time, it
 /// generates run-time checks to prove independence.  This is done by
 /// AccessAnalysis::canCheckPtrAtRT and the checks are maintained by the
-/// RuntimePointerCheck class.
+/// RuntimePointerCheck class. \p AllowPartial determines whether partial checks
+/// are generated when not all pointers could be analyzed.
 ///
 /// If pointers can wrap or can't be expressed as affine AddRec expressions by
 /// ScalarEvolution, we will generate run-time checks by emitting a
@@ -667,7 +669,8 @@ class LoopAccessInfo {
   LLVM_ABI LoopAccessInfo(Loop *L, ScalarEvolution *SE,
                           const TargetTransformInfo *TTI,
                           const TargetLibraryInfo *TLI, AAResults *AA,
-                          DominatorTree *DT, LoopInfo *LI);
+                          DominatorTree *DT, LoopInfo *LI,
+                          bool AllowPartial = false);
 
   /// Return true we can analyze the memory accesses in the loop and there are
   /// no memory dependence cycles. Note that for dependences between loads &
@@ -682,6 +685,11 @@ class LoopAccessInfo {
   /// not legal to insert them.
   bool hasConvergentOp() const { return HasConvergentOp; }
 
+  /// Return true if, when runtime pointer checking does not have complete
+  /// results, it instead has partial results for those memory accesses that
+  /// could be analyzed.
+  bool hasAllowPartial() const { return AllowPartial; }
+
   const RuntimePointerChecking *getRuntimePointerChecking() const {
     return PtrRtChecking.get();
   }
@@ -784,20 +792,30 @@ class LoopAccessInfo {
 
   /// We need to check that all of the pointers in this list are disjoint
   /// at runtime. Using std::unique_ptr to make using move ctor simpler.
+  /// If AllowPartial is true then this list may contain only partial
+  /// information when we've failed to analyze all the memory accesses in the
+  /// loop, in which case HasCompletePtrRtChecking will be false.
   std::unique_ptr<RuntimePointerChecking> PtrRtChecking;
 
-  /// the Memory Dependence Checker which can determine the
+  /// The Memory Dependence Checker which can determine the
   /// loop-independent and loop-carried dependences between memory accesses.
+  /// This will be empty if we've failed to analyze all the memory access in the
+  /// loop (i.e. CanVecMem is false).
   std::unique_ptr<MemoryDepChecker> DepChecker;
 
   Loop *TheLoop;
 
+  /// Determines whether we should generate partial runtime checks when not all
+  /// memory accesses could be analyzed.
+  bool AllowPartial;
+
   unsigned NumLoads = 0;
   unsigned NumStores = 0;
 
   /// Cache the result of analyzeLoop.
   bool CanVecMem = false;
   bool HasConvergentOp = false;
+  bool HasCompletePtrRtChecking = false;
 
   /// Indicator that there are two non vectorizable stores to the same uniform
   /// address.
@@ -920,7 +938,7 @@ class LoopAccessInfoManager {
                         const TargetLibraryInfo *TLI)
       : SE(SE), AA(AA), DT(DT), LI(LI), TTI(TTI), TLI(TLI) {}
 
-  LLVM_ABI const LoopAccessInfo &getInfo(Loop &L);
+  LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial = false);
 
   LLVM_ABI void clear();
 
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/MemoryProfileInfo.h b/external/llvm-project/llvm/include/llvm/Analysis/MemoryProfileInfo.h
index 8cbb8673b69f..b042a717e4e4 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/MemoryProfileInfo.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/MemoryProfileInfo.h
@@ -24,6 +24,18 @@ class OptimizationRemarkEmitter;
 
 namespace memprof {
 
+/// Whether the alloc memeprof metadata will include context size info for all
+/// MIBs.
+LLVM_ABI bool metadataIncludesAllContextSizeInfo();
+
+/// Whether the alloc memprof metadata may include context size info for some
+/// MIBs (but possibly not all).
+LLVM_ABI bool metadataMayIncludeContextSizeInfo();
+
+/// Whether we need to record the context size info in the alloc trie used to
+/// build metadata.
+LLVM_ABI bool recordContextSizeInfoForAnalysis();
+
 /// Build callstack metadata from the provided list of call stack ids. Returns
 /// the resulting metadata node.
 LLVM_ABI MDNode *buildCallstackMetadata(ArrayRef<uint64_t> CallStack,
@@ -87,6 +99,9 @@ class CallStackTrie {
   // allocations for which we apply non-context sensitive allocation hints.
   OptimizationRemarkEmitter *ORE;
 
+  // The maximum size of a cold allocation context, from the profile summary.
+  uint64_t MaxColdSize;
+
   void deleteTrieNode(CallStackTrieNode *Node) {
     if (!Node)
       return;
@@ -113,7 +128,9 @@ class CallStackTrie {
                      uint64_t &ColdBytes);
 
 public:
-  CallStackTrie(OptimizationRemarkEmitter *ORE = nullptr) : ORE(ORE) {}
+  CallStackTrie(OptimizationRemarkEmitter *ORE = nullptr,
+                uint64_t MaxColdSize = 0)
+      : ORE(ORE), MaxColdSize(MaxColdSize) {}
   ~CallStackTrie() { deleteTrieNode(Alloc); }
 
   bool empty() const { return Alloc == nullptr; }
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/ValueTracking.h b/external/llvm-project/llvm/include/llvm/Analysis/ValueTracking.h
index d81c3f10bba9..e215c90b5a72 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/ValueTracking.h
@@ -45,6 +45,10 @@ template <typename T> class ArrayRef;
 
 constexpr unsigned MaxAnalysisRecursionDepth = 6;
 
+/// The max limit of the search depth in DecomposeGEPExpression() and
+/// getUnderlyingObject().
+constexpr unsigned MaxLookupSearchDepth = 10;
+
 /// Determine which bits of V are known to be either zero or one and return
 /// them in the KnownZero/KnownOne bit sets.
 ///
@@ -54,36 +58,37 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6;
 /// same width as the vector element, and the bit is set only if it is true
 /// for all of the elements in the vector.
 LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known,
-                               const DataLayout &DL, unsigned Depth = 0,
+                               const DataLayout &DL,
                                AssumptionCache *AC = nullptr,
                                const Instruction *CxtI = nullptr,
                                const DominatorTree *DT = nullptr,
-                               bool UseInstrInfo = true);
+                               bool UseInstrInfo = true, unsigned Depth = 0);
 
 /// Returns the known bits rather than passing by reference.
 LLVM_ABI KnownBits computeKnownBits(const Value *V, const DataLayout &DL,
-                                    unsigned Depth = 0,
                                     AssumptionCache *AC = nullptr,
                                     const Instruction *CxtI = nullptr,
                                     const DominatorTree *DT = nullptr,
-                                    bool UseInstrInfo = true);
+                                    bool UseInstrInfo = true,
+                                    unsigned Depth = 0);
 
 /// Returns the known bits rather than passing by reference.
 LLVM_ABI KnownBits computeKnownBits(const Value *V, const APInt &DemandedElts,
-                                    const DataLayout &DL, unsigned Depth = 0,
+                                    const DataLayout &DL,
                                     AssumptionCache *AC = nullptr,
                                     const Instruction *CxtI = nullptr,
                                     const DominatorTree *DT = nullptr,
-                                    bool UseInstrInfo = true);
+                                    bool UseInstrInfo = true,
+                                    unsigned Depth = 0);
 
 LLVM_ABI KnownBits computeKnownBits(const Value *V, const APInt &DemandedElts,
-                                    unsigned Depth, const SimplifyQuery &Q);
+                                    const SimplifyQuery &Q, unsigned Depth = 0);
 
-LLVM_ABI KnownBits computeKnownBits(const Value *V, unsigned Depth,
-                                    const SimplifyQuery &Q);
+LLVM_ABI KnownBits computeKnownBits(const Value *V, const SimplifyQuery &Q,
+                                    unsigned Depth = 0);
 
-LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth,
-                               const SimplifyQuery &Q);
+LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known,
+                               const SimplifyQuery &Q, unsigned Depth = 0);
 
 /// Compute known bits from the range metadata.
 /// \p KnownZero the set of bits that are known to be zero
@@ -93,22 +98,22 @@ LLVM_ABI void computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
 
 /// Merge bits known from context-dependent facts into Known.
 LLVM_ABI void computeKnownBitsFromContext(const Value *V, KnownBits &Known,
-                                          unsigned Depth,
-                                          const SimplifyQuery &Q);
+                                          const SimplifyQuery &Q,
+                                          unsigned Depth = 0);
 
 /// Using KnownBits LHS/RHS produce the known bits for logic op (and/xor/or).
 LLVM_ABI KnownBits analyzeKnownBitsFromAndXorOr(const Operator *I,
                                                 const KnownBits &KnownLHS,
                                                 const KnownBits &KnownRHS,
-                                                unsigned Depth,
-                                                const SimplifyQuery &SQ);
+                                                const SimplifyQuery &SQ,
+                                                unsigned Depth = 0);
 
 /// Adjust \p Known for the given select \p Arm to include information from the
 /// select \p Cond.
 LLVM_ABI void adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
                                           Value *Arm, bool Invert,
-                                          unsigned Depth,
-                                          const SimplifyQuery &Q);
+                                          const SimplifyQuery &Q,
+                                          unsigned Depth = 0);
 
 /// Return true if LHS and RHS have no common bits set.
 LLVM_ABI bool haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
@@ -121,14 +126,16 @@ LLVM_ABI bool haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
 /// vectors of integers. If 'OrZero' is set, then return true if the given
 /// value is either a power of two or zero.
 LLVM_ABI bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
-                                     bool OrZero = false, unsigned Depth = 0,
+                                     bool OrZero = false,
                                      AssumptionCache *AC = nullptr,
                                      const Instruction *CxtI = nullptr,
                                      const DominatorTree *DT = nullptr,
-                                     bool UseInstrInfo = true);
+                                     bool UseInstrInfo = true,
+                                     unsigned Depth = 0);
 
 LLVM_ABI bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
-                                     unsigned Depth, const SimplifyQuery &Q);
+                                     const SimplifyQuery &Q,
+                                     unsigned Depth = 0);
 
 LLVM_ABI bool isOnlyUsedInZeroComparison(const Instruction *CxtI);
 
@@ -196,21 +203,21 @@ LLVM_ABI bool MaskedValueIsZero(const Value *V, const APInt &Mask,
 /// sign bits for the vector element with the mininum number of known sign
 /// bits.
 LLVM_ABI unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL,
-                                     unsigned Depth = 0,
                                      AssumptionCache *AC = nullptr,
                                      const Instruction *CxtI = nullptr,
                                      const DominatorTree *DT = nullptr,
-                                     bool UseInstrInfo = true);
+                                     bool UseInstrInfo = true,
+                                     unsigned Depth = 0);
 
 /// Get the upper bound on bit size for this Value \p Op as a signed integer.
 /// i.e.  x == sext(trunc(x to MaxSignificantBits) to bitwidth(x)).
 /// Similar to the APInt::getSignificantBits function.
 LLVM_ABI unsigned ComputeMaxSignificantBits(const Value *Op,
                                             const DataLayout &DL,
-                                            unsigned Depth = 0,
                                             AssumptionCache *AC = nullptr,
                                             const Instruction *CxtI = nullptr,
-                                            const DominatorTree *DT = nullptr);
+                                            const DominatorTree *DT = nullptr,
+                                            unsigned Depth = 0);
 
 /// Map a call instruction to an intrinsic ID.  Libcalls which have equivalent
 /// intrinsics are treated as-if they were intrinsics.
@@ -236,36 +243,36 @@ LLVM_ABI bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
 LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V,
                                           const APInt &DemandedElts,
                                           FPClassTest InterestedClasses,
-                                          unsigned Depth,
-                                          const SimplifyQuery &SQ);
+                                          const SimplifyQuery &SQ,
+                                          unsigned Depth = 0);
 
 LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V,
                                           FPClassTest InterestedClasses,
-                                          unsigned Depth,
-                                          const SimplifyQuery &SQ);
+                                          const SimplifyQuery &SQ,
+                                          unsigned Depth = 0);
 
 LLVM_ABI KnownFPClass computeKnownFPClass(
     const Value *V, const DataLayout &DL,
-    FPClassTest InterestedClasses = fcAllFlags, unsigned Depth = 0,
+    FPClassTest InterestedClasses = fcAllFlags,
     const TargetLibraryInfo *TLI = nullptr, AssumptionCache *AC = nullptr,
     const Instruction *CxtI = nullptr, const DominatorTree *DT = nullptr,
-    bool UseInstrInfo = true);
+    bool UseInstrInfo = true, unsigned Depth = 0);
 
 /// Wrapper to account for known fast math flags at the use instruction.
 LLVM_ABI KnownFPClass computeKnownFPClass(
     const Value *V, const APInt &DemandedElts, FastMathFlags FMF,
-    FPClassTest InterestedClasses, unsigned Depth, const SimplifyQuery &SQ);
+    FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth = 0);
 
 LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, FastMathFlags FMF,
                                           FPClassTest InterestedClasses,
-                                          unsigned Depth,
-                                          const SimplifyQuery &SQ);
+                                          const SimplifyQuery &SQ,
+                                          unsigned Depth = 0);
 
 /// Return true if we can prove that the specified FP value is never equal to
 /// -0.0. Users should use caution when considering PreserveSign
 /// denormal-fp-math.
-LLVM_ABI bool cannotBeNegativeZero(const Value *V, unsigned Depth,
-                                   const SimplifyQuery &SQ);
+LLVM_ABI bool cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
+                                   unsigned Depth = 0);
 
 /// Return true if we can prove that the specified FP value is either NaN or
 /// never less than -0.0.
@@ -275,30 +282,32 @@ LLVM_ABI bool cannotBeNegativeZero(const Value *V, unsigned Depth,
 ///       -0 --> true
 ///   x > +0 --> true
 ///   x < -0 --> false
-LLVM_ABI bool cannotBeOrderedLessThanZero(const Value *V, unsigned Depth,
-                                          const SimplifyQuery &SQ);
+LLVM_ABI bool cannotBeOrderedLessThanZero(const Value *V,
+                                          const SimplifyQuery &SQ,
+                                          unsigned Depth = 0);
 
 /// Return true if the floating-point scalar value is not an infinity or if
 /// the floating-point vector value has no infinities. Return false if a value
 /// could ever be infinity.
-LLVM_ABI bool isKnownNeverInfinity(const Value *V, unsigned Depth,
-                                   const SimplifyQuery &SQ);
+LLVM_ABI bool isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
+                                   unsigned Depth = 0);
 
 /// Return true if the floating-point value can never contain a NaN or infinity.
-LLVM_ABI bool isKnownNeverInfOrNaN(const Value *V, unsigned Depth,
-                                   const SimplifyQuery &SQ);
+LLVM_ABI bool isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
+                                   unsigned Depth = 0);
 
 /// Return true if the floating-point scalar value is not a NaN or if the
 /// floating-point vector value has no NaN elements. Return false if a value
 /// could ever be NaN.
-LLVM_ABI bool isKnownNeverNaN(const Value *V, unsigned Depth,
-                              const SimplifyQuery &SQ);
+LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
+                              unsigned Depth = 0);
 
 /// Return false if we can prove that the specified FP value's sign bit is 0.
 /// Return true if we can prove that the specified FP value's sign bit is 1.
 /// Otherwise return std::nullopt.
-LLVM_ABI std::optional<bool>
-computeKnownFPSignBit(const Value *V, unsigned Depth, const SimplifyQuery &SQ);
+LLVM_ABI std::optional<bool> computeKnownFPSignBit(const Value *V,
+                                                   const SimplifyQuery &SQ,
+                                                   unsigned Depth = 0);
 
 /// Return true if the sign bit of the FP value can be ignored by the user when
 /// the value is zero.
@@ -427,9 +436,10 @@ LLVM_ABI bool isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
 /// original object being addressed. Note that the returned value has pointer
 /// type if the specified value does. If the \p MaxLookup value is non-zero, it
 /// limits the number of instructions to be stripped off.
-LLVM_ABI const Value *getUnderlyingObject(const Value *V,
-                                          unsigned MaxLookup = 6);
-inline Value *getUnderlyingObject(Value *V, unsigned MaxLookup = 6) {
+LLVM_ABI const Value *
+getUnderlyingObject(const Value *V, unsigned MaxLookup = MaxLookupSearchDepth);
+inline Value *getUnderlyingObject(Value *V,
+                                  unsigned MaxLookup = MaxLookupSearchDepth) {
   // Force const to avoid infinite recursion.
   const Value *VConst = V;
   return const_cast<Value *>(getUnderlyingObject(VConst, MaxLookup));
@@ -470,7 +480,7 @@ LLVM_ABI const Value *getUnderlyingObjectAggressive(const Value *V);
 LLVM_ABI void getUnderlyingObjects(const Value *V,
                                    SmallVectorImpl<const Value *> &Objects,
                                    const LoopInfo *LI = nullptr,
-                                   unsigned MaxLookup = 6);
+                                   unsigned MaxLookup = MaxLookupSearchDepth);
 
 /// This is a wrapper around getUnderlyingObjects and adds support for basic
 /// ptrtoint+arithmetic+inttoptr sequences.
@@ -889,7 +899,8 @@ inline SelectPatternResult matchSelectPattern(const Value *V, const Value *&LHS,
 /// predicate and given values as its true/false operands would match.
 LLVM_ABI SelectPatternResult matchDecomposedSelectPattern(
     CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
-    Instruction::CastOps *CastOp = nullptr, unsigned Depth = 0);
+    FastMathFlags FMF = FastMathFlags(), Instruction::CastOps *CastOp = nullptr,
+    unsigned Depth = 0);
 
 /// Determine the pattern for predicate `X Pred Y ? X : Y`.
 LLVM_ABI SelectPatternResult getSelectPattern(
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/VectorUtils.h b/external/llvm-project/llvm/include/llvm/Analysis/VectorUtils.h
index 52fe6f6cf43f..53ba1e8f7779 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/VectorUtils.h
@@ -176,6 +176,12 @@ LLVM_ABI bool isVectorIntrinsicWithStructReturnOverloadAtField(
 LLVM_ABI Intrinsic::ID
 getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI);
 
+/// Returns the corresponding llvm.vector.interleaveN intrinsic for factor N.
+LLVM_ABI Intrinsic::ID getInterleaveIntrinsicID(unsigned Factor);
+
+/// Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor N.
+LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor);
+
 /// Given a vector and an element number, see if the scalar value is
 /// already around as a register, for example if it were inserted then extracted
 /// from the vector.
diff --git a/external/llvm-project/llvm/include/llvm/Analysis/WithCache.h b/external/llvm-project/llvm/include/llvm/Analysis/WithCache.h
index 82c230a32297..3bf35a889bbf 100644
--- a/external/llvm-project/llvm/include/llvm/Analysis/WithCache.h
+++ b/external/llvm-project/llvm/include/llvm/Analysis/WithCache.h
@@ -22,8 +22,8 @@
 
 namespace llvm {
 struct SimplifyQuery;
-LLVM_ABI KnownBits computeKnownBits(const Value *V, unsigned Depth,
-                                    const SimplifyQuery &Q);
+LLVM_ABI KnownBits computeKnownBits(const Value *V, const SimplifyQuery &Q,
+                                    unsigned Depth);
 
 template <typename Arg> class WithCache {
   static_assert(std::is_pointer_v<Arg>, "WithCache requires a pointer type!");
@@ -45,7 +45,7 @@ template <typename Arg> class WithCache {
   mutable KnownBits Known;
 
   void calculateKnownBits(const SimplifyQuery &Q) const {
-    Known = computeKnownBits(Pointer.getPointer(), 0, Q);
+    Known = computeKnownBits(Pointer.getPointer(), Q, 0);
     Pointer.setInt(true);
   }
 
diff --git a/external/llvm-project/llvm/include/llvm/AsmParser/Parser.h b/external/llvm-project/llvm/include/llvm/AsmParser/Parser.h
index 15cc7b5af837..c900b7966540 100644
--- a/external/llvm-project/llvm/include/llvm/AsmParser/Parser.h
+++ b/external/llvm-project/llvm/include/llvm/AsmParser/Parser.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 #include <optional>
 
@@ -44,9 +45,9 @@ typedef llvm::function_ref<std::optional<std::string>(StringRef, StringRef)>
 /// \param Context Context in which to allocate globals info.
 /// \param Slots The optional slot mapping that will be initialized during
 ///              parsing.
-std::unique_ptr<Module> parseAssemblyFile(StringRef Filename, SMDiagnostic &Err,
-                                          LLVMContext &Context,
-                                          SlotMapping *Slots = nullptr);
+LLVM_ABI std::unique_ptr<Module>
+parseAssemblyFile(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context,
+                  SlotMapping *Slots = nullptr);
 
 /// The function is a secondary interface to the LLVM Assembly Parser. It parses
 /// an ASCII string that (presumably) contains LLVM Assembly code. It returns a
@@ -59,10 +60,9 @@ std::unique_ptr<Module> parseAssemblyFile(StringRef Filename, SMDiagnostic &Err,
 /// \param Context Context in which to allocate globals info.
 /// \param Slots The optional slot mapping that will be initialized during
 ///              parsing.
-std::unique_ptr<Module> parseAssemblyString(StringRef AsmString,
-                                            SMDiagnostic &Err,
-                                            LLVMContext &Context,
-                                            SlotMapping *Slots = nullptr);
+LLVM_ABI std::unique_ptr<Module>
+parseAssemblyString(StringRef AsmString, SMDiagnostic &Err,
+                    LLVMContext &Context, SlotMapping *Slots = nullptr);
 
 /// Holds the Module and ModuleSummaryIndex returned by the interfaces
 /// that parse both.
@@ -84,7 +84,7 @@ struct ParsedModuleAndIndex {
 /// \param Slots The optional slot mapping that will be initialized during
 ///              parsing.
 /// \param DataLayoutCallback Override datalayout in the llvm assembly.
-ParsedModuleAndIndex parseAssemblyFileWithIndex(
+LLVM_ABI ParsedModuleAndIndex parseAssemblyFileWithIndex(
     StringRef Filename, SMDiagnostic &Err, LLVMContext &Context,
     SlotMapping *Slots = nullptr,
     DataLayoutCallbackTy DataLayoutCallback = [](StringRef, StringRef) {
@@ -92,7 +92,7 @@ ParsedModuleAndIndex parseAssemblyFileWithIndex(
     });
 
 /// Only for use in llvm-as for testing; this does not produce a valid module.
-ParsedModuleAndIndex parseAssemblyFileWithIndexNoUpgradeDebugInfo(
+LLVM_ABI ParsedModuleAndIndex parseAssemblyFileWithIndexNoUpgradeDebugInfo(
     StringRef Filename, SMDiagnostic &Err, LLVMContext &Context,
     SlotMapping *Slots, DataLayoutCallbackTy DataLayoutCallback);
 
@@ -104,7 +104,7 @@ ParsedModuleAndIndex parseAssemblyFileWithIndexNoUpgradeDebugInfo(
 /// Parse LLVM Assembly Index from a file
 /// \param Filename The name of the file to parse
 /// \param Err Error result info.
-std::unique_ptr<ModuleSummaryIndex>
+LLVM_ABI std::unique_ptr<ModuleSummaryIndex>
 parseSummaryIndexAssemblyFile(StringRef Filename, SMDiagnostic &Err);
 
 /// The function is a secondary interface to the LLVM Assembly Parser. It parses
@@ -115,7 +115,7 @@ parseSummaryIndexAssemblyFile(StringRef Filename, SMDiagnostic &Err);
 /// Parse LLVM Assembly from a string
 /// \param AsmString The string containing assembly
 /// \param Err Error result info.
-std::unique_ptr<ModuleSummaryIndex>
+LLVM_ABI std::unique_ptr<ModuleSummaryIndex>
 parseSummaryIndexAssemblyString(StringRef AsmString, SMDiagnostic &Err);
 
 /// parseAssemblyFile and parseAssemblyString are wrappers around this function.
@@ -125,7 +125,7 @@ parseSummaryIndexAssemblyString(StringRef AsmString, SMDiagnostic &Err);
 /// \param Slots The optional slot mapping that will be initialized during
 ///              parsing.
 /// \param DataLayoutCallback Override datalayout in the llvm assembly.
-std::unique_ptr<Module> parseAssembly(
+LLVM_ABI std::unique_ptr<Module> parseAssembly(
     MemoryBufferRef F, SMDiagnostic &Err, LLVMContext &Context,
     SlotMapping *Slots = nullptr,
     DataLayoutCallbackTy DataLayoutCallback = [](StringRef, StringRef) {
@@ -140,10 +140,9 @@ std::unique_ptr<Module> parseAssembly(
 ///              parsing.
 ///
 /// parseAssemblyFileWithIndex is a wrapper around this function.
-ParsedModuleAndIndex parseAssemblyWithIndex(MemoryBufferRef F,
-                                            SMDiagnostic &Err,
-                                            LLVMContext &Context,
-                                            SlotMapping *Slots = nullptr);
+LLVM_ABI ParsedModuleAndIndex
+parseAssemblyWithIndex(MemoryBufferRef F, SMDiagnostic &Err,
+                       LLVMContext &Context, SlotMapping *Slots = nullptr);
 
 /// Parse LLVM Assembly for summary index from a MemoryBuffer.
 ///
@@ -151,7 +150,7 @@ ParsedModuleAndIndex parseAssemblyWithIndex(MemoryBufferRef F,
 /// \param Err Error result info.
 ///
 /// parseSummaryIndexAssemblyFile is a wrapper around this function.
-std::unique_ptr<ModuleSummaryIndex>
+LLVM_ABI std::unique_ptr<ModuleSummaryIndex>
 parseSummaryIndexAssembly(MemoryBufferRef F, SMDiagnostic &Err);
 
 /// This function is the low-level interface to the LLVM Assembly Parser.
@@ -167,7 +166,7 @@ parseSummaryIndexAssembly(MemoryBufferRef F, SMDiagnostic &Err);
 ///              parsing.
 /// \return true on error.
 /// \param DataLayoutCallback Override datalayout in the llvm assembly.
-bool parseAssemblyInto(
+LLVM_ABI bool parseAssemblyInto(
     MemoryBufferRef F, Module *M, ModuleSummaryIndex *Index, SMDiagnostic &Err,
     SlotMapping *Slots = nullptr,
     DataLayoutCallbackTy DataLayoutCallback = [](StringRef, StringRef) {
@@ -182,16 +181,17 @@ bool parseAssemblyInto(
 /// \param Slots The optional slot mapping that will restore the parsing state
 /// of the module.
 /// \return null on error.
-Constant *parseConstantValue(StringRef Asm, SMDiagnostic &Err, const Module &M,
-                             const SlotMapping *Slots = nullptr);
+LLVM_ABI Constant *parseConstantValue(StringRef Asm, SMDiagnostic &Err,
+                                      const Module &M,
+                                      const SlotMapping *Slots = nullptr);
 
 /// Parse a type in the given string.
 ///
 /// \param Slots The optional slot mapping that will restore the parsing state
 /// of the module.
 /// \return null on error.
-Type *parseType(StringRef Asm, SMDiagnostic &Err, const Module &M,
-                const SlotMapping *Slots = nullptr);
+LLVM_ABI Type *parseType(StringRef Asm, SMDiagnostic &Err, const Module &M,
+                         const SlotMapping *Slots = nullptr);
 
 /// Parse a string \p Asm that starts with a type.
 /// \p Read[out] gives the number of characters that have been read to parse
@@ -200,13 +200,14 @@ Type *parseType(StringRef Asm, SMDiagnostic &Err, const Module &M,
 /// \param Slots The optional slot mapping that will restore the parsing state
 /// of the module.
 /// \return null on error.
-Type *parseTypeAtBeginning(StringRef Asm, unsigned &Read, SMDiagnostic &Err,
-                           const Module &M, const SlotMapping *Slots = nullptr);
-
-DIExpression *parseDIExpressionBodyAtBeginning(StringRef Asm, unsigned &Read,
-                                               SMDiagnostic &Err,
-                                               const Module &M,
-                                               const SlotMapping *Slots);
+LLVM_ABI Type *parseTypeAtBeginning(StringRef Asm, unsigned &Read,
+                                    SMDiagnostic &Err, const Module &M,
+                                    const SlotMapping *Slots = nullptr);
+
+LLVM_ABI DIExpression *
+parseDIExpressionBodyAtBeginning(StringRef Asm, unsigned &Read,
+                                 SMDiagnostic &Err, const Module &M,
+                                 const SlotMapping *Slots);
 
 } // End llvm namespace
 
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h
index f6268b48e6b2..0102fe655ae2 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MsgPackReader.h"
+#include "llvm/Support/Compiler.h"
 
 #include <cstddef>
 #include <optional>
@@ -69,7 +70,7 @@ class MetadataVerifier {
   /// Verify given HSA metadata.
   ///
   /// \returns True when successful, false when metadata is invalid.
-  bool verify(msgpack::DocNode &HSAMetadataRoot);
+  LLVM_ABI bool verify(msgpack::DocNode &HSAMetadataRoot);
 };
 
 } // end namespace V3
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/COFF.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/COFF.h
index bbc5264d1787..f3b5d5e3f23c 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/COFF.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/COFF.h
@@ -22,6 +22,7 @@
 #ifndef LLVM_BINARYFORMAT_COFF_H
 #define LLVM_BINARYFORMAT_COFF_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
 #include <cassert>
 
@@ -851,7 +852,7 @@ inline bool isReservedSectionNumber(int32_t SectionNumber) {
 
 /// Encode section name based on string table offset.
 /// The size of Out must be at least COFF::NameSize.
-bool encodeSectionName(char *Out, uint64_t Offset);
+LLVM_ABI bool encodeSectionName(char *Out, uint64_t Offset);
 
 } // End namespace COFF.
 } // End namespace llvm.
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/DXContainer.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/DXContainer.h
index d1306c85719a..08949e39716d 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -14,6 +14,7 @@
 #define LLVM_BINARYFORMAT_DXCONTAINER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/TargetParser/Triple.h"
@@ -59,7 +60,7 @@ struct ShaderHash {
   uint32_t Flags; // dxbc::HashFlags
   uint8_t Digest[16];
 
-  bool isPopulated();
+  LLVM_ABI bool isPopulated();
 
   void swapBytes() { sys::swapByteOrder(Flags); }
 };
@@ -173,7 +174,7 @@ enum class RootParameterType : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<RootParameterType>> getRootParameterTypes();
+LLVM_ABI ArrayRef<EnumEntry<RootParameterType>> getRootParameterTypes();
 
 #define DESCRIPTOR_RANGE(Val, Enum) Enum = Val,
 enum class DescriptorRangeType : uint32_t {
@@ -197,7 +198,7 @@ enum class ShaderVisibility : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<ShaderVisibility>> getShaderVisibility();
+LLVM_ABI ArrayRef<EnumEntry<ShaderVisibility>> getShaderVisibility();
 
 #define SHADER_VISIBILITY(Val, Enum)                                           \
   case Val:                                                                    \
@@ -229,7 +230,7 @@ enum class SamplersBorderColor : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-PartType parsePartType(StringRef S);
+LLVM_ABI PartType parsePartType(StringRef S);
 
 struct VertexPSVInfo {
   uint8_t OutputPositionPresent;
@@ -360,35 +361,35 @@ enum class SemanticKind : uint8_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<SemanticKind>> getSemanticKinds();
+LLVM_ABI ArrayRef<EnumEntry<SemanticKind>> getSemanticKinds();
 
 #define COMPONENT_TYPE(Val, Enum) Enum = Val,
 enum class ComponentType : uint8_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<ComponentType>> getComponentTypes();
+LLVM_ABI ArrayRef<EnumEntry<ComponentType>> getComponentTypes();
 
 #define INTERPOLATION_MODE(Val, Enum) Enum = Val,
 enum class InterpolationMode : uint8_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<InterpolationMode>> getInterpolationModes();
+LLVM_ABI ArrayRef<EnumEntry<InterpolationMode>> getInterpolationModes();
 
 #define RESOURCE_TYPE(Val, Enum) Enum = Val,
 enum class ResourceType : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<ResourceType>> getResourceTypes();
+LLVM_ABI ArrayRef<EnumEntry<ResourceType>> getResourceTypes();
 
 #define RESOURCE_KIND(Val, Enum) Enum = Val,
 enum class ResourceKind : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<ResourceKind>> getResourceKinds();
+LLVM_ABI ArrayRef<EnumEntry<ResourceKind>> getResourceKinds();
 
 #define RESOURCE_FLAG(Index, Enum) bool Enum = false;
 struct ResourceFlags {
@@ -555,21 +556,21 @@ enum class SigMinPrecision : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<SigMinPrecision>> getSigMinPrecisions();
+LLVM_ABI ArrayRef<EnumEntry<SigMinPrecision>> getSigMinPrecisions();
 
 #define D3D_SYSTEM_VALUE(Val, Enum) Enum = Val,
 enum class D3DSystemValue : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<D3DSystemValue>> getD3DSystemValues();
+LLVM_ABI ArrayRef<EnumEntry<D3DSystemValue>> getD3DSystemValues();
 
 #define COMPONENT_TYPE(Val, Enum) Enum = Val,
 enum class SigComponentType : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<SigComponentType>> getSigComponentTypes();
+LLVM_ABI ArrayRef<EnumEntry<SigComponentType>> getSigComponentTypes();
 
 struct ProgramSignatureHeader {
   uint32_t ParamCount;
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.def b/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.def
index a606cab3e064..0de59d702213 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -928,6 +928,7 @@ HANDLE_DW_OP(0xed, WASM_location, -1, -1, 0, WASM)
 HANDLE_DW_OP(0xee, WASM_location_int, -1, -1, 0, WASM)
 // Historic and not implemented in LLVM.
 HANDLE_DW_OP(0xf0, APPLE_uninit, -1, -1, 0, APPLE)
+HANDLE_DW_OP(0xf2, GNU_implicit_pointer, 2, 0, 4, GNU)
 // The GNU entry value extension.
 HANDLE_DW_OP(0xf3, GNU_entry_value, 2, 0, 0, GNU)
 HANDLE_DW_OP(0xf8, PGI_omp_thread_num, -1, -1, 0, PGI)
@@ -1201,7 +1202,7 @@ HANDLE_DW_CC(0xc3, LLVM_AAPCS)
 HANDLE_DW_CC(0xc4, LLVM_AAPCS_VFP)
 HANDLE_DW_CC(0xc5, LLVM_IntelOclBicc)
 HANDLE_DW_CC(0xc6, LLVM_SpirFunction)
-HANDLE_DW_CC(0xc7, LLVM_OpenCLKernel)
+HANDLE_DW_CC(0xc7, LLVM_DeviceKernel)
 HANDLE_DW_CC(0xc8, LLVM_Swift)
 HANDLE_DW_CC(0xc9, LLVM_PreserveMost)
 HANDLE_DW_CC(0xca, LLVM_PreserveAll)
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.h
index 60ef752d8502..73075cac14ed 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/Dwarf.h
@@ -522,7 +522,7 @@ toDW_LNAME(SourceLanguage language) {
   return {};
 }
 
-llvm::StringRef LanguageDescription(SourceLanguageName name);
+LLVM_ABI llvm::StringRef LanguageDescription(SourceLanguageName name);
 
 inline bool isCPlusPlus(SourceLanguage S) {
   bool result = false;
@@ -1010,45 +1010,45 @@ enum GDBIndexEntryLinkage { GIEL_EXTERNAL, GIEL_STATIC };
 /// isn't known.
 ///
 /// @{
-StringRef TagString(unsigned Tag);
-StringRef ChildrenString(unsigned Children);
-StringRef AttributeString(unsigned Attribute);
-StringRef FormEncodingString(unsigned Encoding);
-StringRef OperationEncodingString(unsigned Encoding);
-StringRef SubOperationEncodingString(unsigned OpEncoding,
-                                     unsigned SubOpEncoding);
-StringRef AttributeEncodingString(unsigned Encoding);
-StringRef DecimalSignString(unsigned Sign);
-StringRef EndianityString(unsigned Endian);
-StringRef AccessibilityString(unsigned Access);
-StringRef DefaultedMemberString(unsigned DefaultedEncodings);
-StringRef VisibilityString(unsigned Visibility);
-StringRef VirtualityString(unsigned Virtuality);
-StringRef EnumKindString(unsigned EnumKind);
-StringRef LanguageString(unsigned Language);
-StringRef CaseString(unsigned Case);
-StringRef ConventionString(unsigned Convention);
-StringRef InlineCodeString(unsigned Code);
-StringRef ArrayOrderString(unsigned Order);
-StringRef LNStandardString(unsigned Standard);
-StringRef LNExtendedString(unsigned Encoding);
-StringRef MacinfoString(unsigned Encoding);
-StringRef MacroString(unsigned Encoding);
-StringRef GnuMacroString(unsigned Encoding);
-StringRef RangeListEncodingString(unsigned Encoding);
-StringRef LocListEncodingString(unsigned Encoding);
-StringRef CallFrameString(unsigned Encoding, Triple::ArchType Arch);
-StringRef ApplePropertyString(unsigned);
-StringRef UnitTypeString(unsigned);
-StringRef AtomTypeString(unsigned Atom);
-StringRef GDBIndexEntryKindString(GDBIndexEntryKind Kind);
-StringRef GDBIndexEntryLinkageString(GDBIndexEntryLinkage Linkage);
-StringRef IndexString(unsigned Idx);
-StringRef FormatString(DwarfFormat Format);
-StringRef FormatString(bool IsDWARF64);
-StringRef RLEString(unsigned RLE);
-StringRef MemorySpaceString(unsigned MS);
-StringRef AddressSpaceString(unsigned AS, llvm::Triple TT);
+LLVM_ABI StringRef TagString(unsigned Tag);
+LLVM_ABI StringRef ChildrenString(unsigned Children);
+LLVM_ABI StringRef AttributeString(unsigned Attribute);
+LLVM_ABI StringRef FormEncodingString(unsigned Encoding);
+LLVM_ABI StringRef OperationEncodingString(unsigned Encoding);
+LLVM_ABI StringRef SubOperationEncodingString(unsigned OpEncoding,
+                                              unsigned SubOpEncoding);
+LLVM_ABI StringRef AttributeEncodingString(unsigned Encoding);
+LLVM_ABI StringRef DecimalSignString(unsigned Sign);
+LLVM_ABI StringRef EndianityString(unsigned Endian);
+LLVM_ABI StringRef AccessibilityString(unsigned Access);
+LLVM_ABI StringRef DefaultedMemberString(unsigned DefaultedEncodings);
+LLVM_ABI StringRef VisibilityString(unsigned Visibility);
+LLVM_ABI StringRef VirtualityString(unsigned Virtuality);
+LLVM_ABI StringRef EnumKindString(unsigned EnumKind);
+LLVM_ABI StringRef LanguageString(unsigned Language);
+LLVM_ABI StringRef CaseString(unsigned Case);
+LLVM_ABI StringRef ConventionString(unsigned Convention);
+LLVM_ABI StringRef InlineCodeString(unsigned Code);
+LLVM_ABI StringRef ArrayOrderString(unsigned Order);
+LLVM_ABI StringRef LNStandardString(unsigned Standard);
+LLVM_ABI StringRef LNExtendedString(unsigned Encoding);
+LLVM_ABI StringRef MacinfoString(unsigned Encoding);
+LLVM_ABI StringRef MacroString(unsigned Encoding);
+LLVM_ABI StringRef GnuMacroString(unsigned Encoding);
+LLVM_ABI StringRef RangeListEncodingString(unsigned Encoding);
+LLVM_ABI StringRef LocListEncodingString(unsigned Encoding);
+LLVM_ABI StringRef CallFrameString(unsigned Encoding, Triple::ArchType Arch);
+LLVM_ABI StringRef ApplePropertyString(unsigned);
+LLVM_ABI StringRef UnitTypeString(unsigned);
+LLVM_ABI StringRef AtomTypeString(unsigned Atom);
+LLVM_ABI StringRef GDBIndexEntryKindString(GDBIndexEntryKind Kind);
+LLVM_ABI StringRef GDBIndexEntryLinkageString(GDBIndexEntryLinkage Linkage);
+LLVM_ABI StringRef IndexString(unsigned Idx);
+LLVM_ABI StringRef FormatString(DwarfFormat Format);
+LLVM_ABI StringRef FormatString(bool IsDWARF64);
+LLVM_ABI StringRef RLEString(unsigned RLE);
+LLVM_ABI StringRef MemorySpaceString(unsigned MS);
+LLVM_ABI StringRef AddressSpaceString(unsigned AS, llvm::Triple TT);
 /// @}
 
 /// \defgroup DwarfConstantsParsing Dwarf constants parsing functions
@@ -1061,18 +1061,18 @@ StringRef AddressSpaceString(unsigned AS, llvm::Triple TT);
 /// \li \a getMacinfo() returns \a DW_MACINFO_invalid on invalid input.
 ///
 /// @{
-unsigned getTag(StringRef TagString);
-unsigned getOperationEncoding(StringRef OperationEncodingString);
-unsigned getSubOperationEncoding(unsigned OpEncoding,
-                                 StringRef SubOperationEncodingString);
-unsigned getVirtuality(StringRef VirtualityString);
-unsigned getEnumKind(StringRef EnumKindString);
-unsigned getLanguage(StringRef LanguageString);
-unsigned getCallingConvention(StringRef LanguageString);
-unsigned getMemorySpace(StringRef LanguageString);
-unsigned getAttributeEncoding(StringRef EncodingString);
-unsigned getMacinfo(StringRef MacinfoString);
-unsigned getMacro(StringRef MacroString);
+LLVM_ABI unsigned getTag(StringRef TagString);
+LLVM_ABI unsigned getOperationEncoding(StringRef OperationEncodingString);
+LLVM_ABI unsigned getSubOperationEncoding(unsigned OpEncoding,
+                                          StringRef SubOperationEncodingString);
+LLVM_ABI unsigned getVirtuality(StringRef VirtualityString);
+LLVM_ABI unsigned getEnumKind(StringRef EnumKindString);
+LLVM_ABI unsigned getLanguage(StringRef LanguageString);
+LLVM_ABI unsigned getCallingConvention(StringRef LanguageString);
+LLVM_ABI unsigned getMemorySpace(StringRef LanguageString);
+LLVM_ABI unsigned getAttributeEncoding(StringRef EncodingString);
+LLVM_ABI unsigned getMacinfo(StringRef MacinfoString);
+LLVM_ABI unsigned getMacro(StringRef MacroString);
 /// @}
 
 /// \defgroup DwarfConstantsVersioning Dwarf version for constants
@@ -1083,12 +1083,12 @@ unsigned getMacro(StringRef MacroString);
 /// Otherwise returns 0.
 ///
 /// @{
-unsigned TagVersion(Tag T);
-unsigned AttributeVersion(Attribute A);
-unsigned FormVersion(Form F);
-unsigned OperationVersion(LocationAtom O);
-unsigned AttributeEncodingVersion(TypeKind E);
-unsigned LanguageVersion(SourceLanguage L);
+LLVM_ABI unsigned TagVersion(Tag T);
+LLVM_ABI unsigned AttributeVersion(Attribute A);
+LLVM_ABI unsigned FormVersion(Form F);
+LLVM_ABI unsigned OperationVersion(LocationAtom O);
+LLVM_ABI unsigned AttributeEncodingVersion(TypeKind E);
+LLVM_ABI unsigned LanguageVersion(SourceLanguage L);
 /// @}
 
 /// \defgroup DwarfConstantsVendor Dwarf "vendor" for constants
@@ -1097,23 +1097,23 @@ unsigned LanguageVersion(SourceLanguage L);
 /// either the DWARF standard itself or the vendor who defined the extension.
 ///
 /// @{
-unsigned TagVendor(Tag T);
-unsigned AttributeVendor(Attribute A);
-unsigned FormVendor(Form F);
-unsigned OperationVendor(LocationAtom O);
-unsigned AttributeEncodingVendor(TypeKind E);
-unsigned LanguageVendor(SourceLanguage L);
+LLVM_ABI unsigned TagVendor(Tag T);
+LLVM_ABI unsigned AttributeVendor(Attribute A);
+LLVM_ABI unsigned FormVendor(Form F);
+LLVM_ABI unsigned OperationVendor(LocationAtom O);
+LLVM_ABI unsigned AttributeEncodingVendor(TypeKind E);
+LLVM_ABI unsigned LanguageVendor(SourceLanguage L);
 /// @}
 
 /// The number of operands for the given LocationAtom.
-std::optional<unsigned> OperationOperands(LocationAtom O);
+LLVM_ABI std::optional<unsigned> OperationOperands(LocationAtom O);
 
 /// The arity of the given LocationAtom. This is the number of elements on the
 /// stack this operation operates on. Returns -1 if the arity is variable (e.g.
 /// depending on the argument) or unknown.
-std::optional<unsigned> OperationArity(LocationAtom O);
+LLVM_ABI std::optional<unsigned> OperationArity(LocationAtom O);
 
-std::optional<unsigned> LanguageLowerBound(SourceLanguage L);
+LLVM_ABI std::optional<unsigned> LanguageLowerBound(SourceLanguage L);
 
 /// The size of a reference determined by the DWARF 32/64-bit format.
 inline uint8_t getDwarfOffsetByteSize(DwarfFormat Format) {
@@ -1175,20 +1175,21 @@ inline uint8_t getUnitLengthFieldByteSize(DwarfFormat Format) {
 /// \param Params DWARF parameters to help interpret forms.
 /// \returns std::optional<uint8_t> value with the fixed byte size or
 /// std::nullopt if \p Form doesn't have a fixed byte size.
-std::optional<uint8_t> getFixedFormByteSize(dwarf::Form Form,
-                                            FormParams Params);
+LLVM_ABI std::optional<uint8_t> getFixedFormByteSize(dwarf::Form Form,
+                                                     FormParams Params);
 
 /// Tells whether the specified form is defined in the specified version,
 /// or is an extension if extensions are allowed.
-bool isValidFormForVersion(Form F, unsigned Version, bool ExtensionsOk = true);
+LLVM_ABI bool isValidFormForVersion(Form F, unsigned Version,
+                                    bool ExtensionsOk = true);
 
 /// Returns the symbolic string representing Val when used as a value
 /// for attribute Attr.
-StringRef AttributeValueString(uint16_t Attr, unsigned Val);
+LLVM_ABI StringRef AttributeValueString(uint16_t Attr, unsigned Val);
 
 /// Returns the symbolic string representing Val when used as a value
 /// for atom Atom.
-StringRef AtomValueString(uint16_t Atom, unsigned Val);
+LLVM_ABI StringRef AtomValueString(uint16_t Atom, unsigned Val);
 
 /// Describes an entry of the various gnu_pub* debug sections.
 ///
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/ELF.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/ELF.h
index 1f3cea4bd1ae..97f417263aff 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/ELF.h
@@ -20,6 +20,7 @@
 #define LLVM_BINARYFORMAT_ELF_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cstdint>
 #include <cstring>
@@ -2014,20 +2015,20 @@ enum {
 constexpr unsigned CREL_HDR_ADDEND = 4;
 
 /// Convert an architecture name into ELF's e_machine value.
-uint16_t convertArchNameToEMachine(StringRef Arch);
+LLVM_ABI uint16_t convertArchNameToEMachine(StringRef Arch);
 
 /// Convert an ELF's e_machine value into an architecture name.
-StringRef convertEMachineToArchName(uint16_t EMachine);
+LLVM_ABI StringRef convertEMachineToArchName(uint16_t EMachine);
 
 // Convert a triple's architecture to ELF's e_machine value.
-uint16_t convertTripleArchTypeToEMachine(Triple::ArchType ArchType);
+LLVM_ABI uint16_t convertTripleArchTypeToEMachine(Triple::ArchType ArchType);
 
 // Convert a lowercase string identifier into an OSABI value.
-uint8_t convertNameToOSABI(StringRef Name);
+LLVM_ABI uint8_t convertNameToOSABI(StringRef Name);
 
 // Convert an OSABI value into a string that identifies the OS- or ABI-
 // specific ELF extension.
-StringRef convertOSABIToName(uint8_t OSABI);
+LLVM_ABI StringRef convertOSABIToName(uint8_t OSABI);
 
 } // end namespace ELF
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/MachO.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/MachO.h
index 5afe70bffc24..5dbdfb13d1a5 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/MachO.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/MachO.h
@@ -1704,10 +1704,11 @@ enum CPUSubTypeRISCV {
   CPU_SUBTYPE_RISCV_ALL = 0,
 };
 
-Expected<uint32_t> getCPUType(const Triple &T);
-Expected<uint32_t> getCPUSubType(const Triple &T);
-Expected<uint32_t> getCPUSubType(const Triple &T, unsigned PtrAuthABIVersion,
-                                 bool PtrAuthKernelABIVersion);
+LLVM_ABI Expected<uint32_t> getCPUType(const Triple &T);
+LLVM_ABI Expected<uint32_t> getCPUSubType(const Triple &T);
+LLVM_ABI Expected<uint32_t> getCPUSubType(const Triple &T,
+                                          unsigned PtrAuthABIVersion,
+                                          bool PtrAuthKernelABIVersion);
 
 struct x86_thread_state32_t {
   uint32_t eax;
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/Magic.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/Magic.h
index 6978c066bda4..11922d843af9 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/Magic.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/Magic.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_BINARYFORMAT_MAGIC_H
 #define LLVM_BINARYFORMAT_MAGIC_H
 
+#include "llvm/Support/Compiler.h"
 #include <system_error>
 
 namespace llvm {
@@ -72,7 +73,7 @@ struct file_magic {
 };
 
 /// Identify the type of a binary file based on how magical it is.
-file_magic identify_magic(StringRef magic);
+LLVM_ABI file_magic identify_magic(StringRef magic);
 
 /// Get and identify \a path's type based on its content.
 ///
@@ -80,7 +81,7 @@ file_magic identify_magic(StringRef magic);
 /// @param result Set to the type of file, or file_magic::unknown.
 /// @returns errc::success if result has been successfully set, otherwise a
 ///          platform-specific error_code.
-std::error_code identify_magic(const Twine &path, file_magic &result);
+LLVM_ABI std::error_code identify_magic(const Twine &path, file_magic &result);
 } // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackDocument.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackDocument.h
index 09091264cac2..26fff8d5f8d3 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackDocument.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackDocument.h
@@ -18,6 +18,7 @@
 #define LLVM_BINARYFORMAT_MSGPACKDOCUMENT_H
 
 #include "llvm/BinaryFormat/MsgPackReader.h"
+#include "llvm/Support/Compiler.h"
 #include <map>
 
 namespace llvm {
@@ -192,12 +193,12 @@ class DocNode {
   }
 
   /// Convert this node to a string, assuming it is scalar.
-  std::string toString() const;
+  LLVM_ABI std::string toString() const;
 
   /// Convert the StringRef and use it to set this DocNode (assuming scalar). If
   /// it is a string, copy the string into the Document's strings list so we do
   /// not rely on S having a lifetime beyond this call. Tag is "" or a YAML tag.
-  StringRef fromString(StringRef S, StringRef Tag = "");
+  LLVM_ABI StringRef fromString(StringRef S, StringRef Tag = "");
 
   /// Convenience assignment operators. This only works if the destination
   /// DocNode has an associated Document, i.e. it was not constructed using the
@@ -205,20 +206,20 @@ class DocNode {
   /// remain valid for the lifetime of the Document. Use fromString to avoid
   /// that restriction.
   DocNode &operator=(const char *Val) { return *this = StringRef(Val); }
-  DocNode &operator=(StringRef Val);
-  DocNode &operator=(MemoryBufferRef Val);
-  DocNode &operator=(bool Val);
-  DocNode &operator=(int Val);
-  DocNode &operator=(unsigned Val);
-  DocNode &operator=(int64_t Val);
-  DocNode &operator=(uint64_t Val);
+  LLVM_ABI DocNode &operator=(StringRef Val);
+  LLVM_ABI DocNode &operator=(MemoryBufferRef Val);
+  LLVM_ABI DocNode &operator=(bool Val);
+  LLVM_ABI DocNode &operator=(int Val);
+  LLVM_ABI DocNode &operator=(unsigned Val);
+  LLVM_ABI DocNode &operator=(int64_t Val);
+  LLVM_ABI DocNode &operator=(uint64_t Val);
 
 private:
   // Private constructor setting KindAndDoc, used by methods in Document.
   DocNode(const KindAndDocument *KindAndDoc) : KindAndDoc(KindAndDoc) {}
 
-  void convertToArray();
-  void convertToMap();
+  LLVM_ABI void convertToArray();
+  LLVM_ABI void convertToMap();
 };
 
 /// A DocNode that is a map.
@@ -233,7 +234,7 @@ class MapDocNode : public DocNode {
   MapTy::iterator begin() { return Map->begin(); }
   MapTy::iterator end() { return Map->end(); }
   MapTy::iterator find(DocNode Key) { return Map->find(Key); }
-  MapTy::iterator find(StringRef Key);
+  LLVM_ABI MapTy::iterator find(StringRef Key);
   MapTy::iterator erase(MapTy::const_iterator I) { return Map->erase(I); }
   size_t erase(DocNode Key) { return Map->erase(Key); }
   MapTy::iterator erase(MapTy::const_iterator First,
@@ -242,13 +243,13 @@ class MapDocNode : public DocNode {
   }
   /// Member access. The string data must remain valid for the lifetime of the
   /// Document.
-  DocNode &operator[](StringRef S);
+  LLVM_ABI DocNode &operator[](StringRef S);
   /// Member access, with convenience versions for an integer key.
-  DocNode &operator[](DocNode Key);
-  DocNode &operator[](int Key);
-  DocNode &operator[](unsigned Key);
-  DocNode &operator[](int64_t Key);
-  DocNode &operator[](uint64_t Key);
+  LLVM_ABI DocNode &operator[](DocNode Key);
+  LLVM_ABI DocNode &operator[](int Key);
+  LLVM_ABI DocNode &operator[](unsigned Key);
+  LLVM_ABI DocNode &operator[](int64_t Key);
+  LLVM_ABI DocNode &operator[](uint64_t Key);
 };
 
 /// A DocNode that is an array.
@@ -269,7 +270,7 @@ class ArrayDocNode : public DocNode {
   }
 
   /// Element access. This extends the array if necessary, with empty nodes.
-  DocNode &operator[](size_t Index);
+  LLVM_ABI DocNode &operator[](size_t Index);
 };
 
 /// Simple in-memory representation of a document of msgpack objects with
@@ -424,7 +425,7 @@ class Document {
   /// map entry, a nil node otherwise.
   ///
   /// The default for Merger is to disallow any conflict.
-  bool readFromBlob(
+  LLVM_ABI bool readFromBlob(
       StringRef Blob, bool Multi,
       function_ref<int(DocNode *DestNode, DocNode SrcNode, DocNode MapKey)>
           Merger = [](DocNode *DestNode, DocNode SrcNode, DocNode MapKey) {
@@ -432,7 +433,7 @@ class Document {
           });
 
   /// Write a MsgPack document to a binary MsgPack blob.
-  void writeToBlob(std::string &Blob);
+  LLVM_ABI void writeToBlob(std::string &Blob);
 
   /// Copy a string into the Document's strings list, and return the copy that
   /// is owned by the Document.
@@ -449,10 +450,10 @@ class Document {
   bool getHexMode() const { return HexMode; }
 
   /// Convert MsgPack Document to YAML text.
-  void toYAML(raw_ostream &OS);
+  LLVM_ABI void toYAML(raw_ostream &OS);
 
   /// Read YAML text into the MsgPack document. Returns false on failure.
-  bool fromYAML(StringRef S);
+  LLVM_ABI bool fromYAML(StringRef S);
 };
 
 } // namespace msgpack
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackReader.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackReader.h
index 29bab8f9691a..66140e66cba4 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackReader.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackReader.h
@@ -38,6 +38,7 @@
 #ifndef LLVM_BINARYFORMAT_MSGPACKREADER_H
 #define LLVM_BINARYFORMAT_MSGPACKREADER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include <cstdint>
@@ -104,9 +105,9 @@ struct Object {
 class Reader {
 public:
   /// Construct a reader, keeping a reference to the \p InputBuffer.
-  Reader(MemoryBufferRef InputBuffer);
+  LLVM_ABI Reader(MemoryBufferRef InputBuffer);
   /// Construct a reader, keeping a reference to the \p Input.
-  Reader(StringRef Input);
+  LLVM_ABI Reader(StringRef Input);
 
   Reader(const Reader &) = delete;
   Reader &operator=(const Reader &) = delete;
@@ -125,7 +126,7 @@ class Reader {
   ///
   /// \returns true when object successfully read, false when at end of
   /// input (and so \p Obj was not updated), otherwise an error.
-  Expected<bool> read(Object &Obj);
+  LLVM_ABI Expected<bool> read(Object &Obj);
 
 private:
   MemoryBufferRef InputBuffer;
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackWriter.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackWriter.h
index b3d3c3bcdef0..173458b57515 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackWriter.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/MsgPackWriter.h
@@ -28,6 +28,7 @@
 #ifndef LLVM_BINARYFORMAT_MSGPACKWRITER_H
 #define LLVM_BINARYFORMAT_MSGPACKWRITER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/MemoryBufferRef.h"
 
@@ -48,7 +49,7 @@ class Writer {
   ///
   /// \param OS stream to output MessagePack objects to.
   /// \param Compatible when set, write in "Compatibility Mode".
-  Writer(raw_ostream &OS, bool Compatible = false);
+  LLVM_ABI Writer(raw_ostream &OS, bool Compatible = false);
 
   Writer(const Writer &) = delete;
   Writer &operator=(const Writer &) = delete;
@@ -56,41 +57,41 @@ class Writer {
   /// Write a \em Nil to the output stream.
   ///
   /// The output will be the \em nil format.
-  void writeNil();
+  LLVM_ABI void writeNil();
 
   /// Write a \em Boolean to the output stream.
   ///
   /// The output will be a \em bool format.
-  void write(bool b);
+  LLVM_ABI void write(bool b);
 
   /// Write a signed integer to the output stream.
   ///
   /// The output will be in the smallest possible \em int format.
   ///
   /// The format chosen may be for an unsigned integer.
-  void write(int64_t i);
+  LLVM_ABI void write(int64_t i);
 
   /// Write an unsigned integer to the output stream.
   ///
   /// The output will be in the smallest possible \em int format.
-  void write(uint64_t u);
+  LLVM_ABI void write(uint64_t u);
 
   /// Write a floating point number to the output stream.
   ///
   /// The output will be in the smallest possible \em float format.
-  void write(double d);
+  LLVM_ABI void write(double d);
 
   /// Write a string to the output stream.
   ///
   /// The output will be in the smallest possible \em str format.
-  void write(StringRef s);
+  LLVM_ABI void write(StringRef s);
 
   /// Write a memory buffer to the output stream.
   ///
   /// The output will be in the smallest possible \em bin format.
   ///
   /// \warning Do not use this overload if in \c Compatible mode.
-  void write(MemoryBufferRef Buffer);
+  LLVM_ABI void write(MemoryBufferRef Buffer);
 
   /// Write the header for an \em Array of the given size.
   ///
@@ -101,7 +102,7 @@ class Writer {
   ///
   /// N.B. The caller must subsequently call \c Write an additional \p Size
   /// times to complete the array.
-  void writeArraySize(uint32_t Size);
+  LLVM_ABI void writeArraySize(uint32_t Size);
 
   /// Write the header for a \em Map of the given size.
   ///
@@ -113,12 +114,12 @@ class Writer {
   /// N.B. The caller must subsequently call \c Write and additional \c Size*2
   /// times to complete the map. Each even numbered call to \c Write defines a
   /// new key, and each odd numbered call defines the previous key's value.
-  void writeMapSize(uint32_t Size);
+  LLVM_ABI void writeMapSize(uint32_t Size);
 
   /// Write a typed memory buffer (an extension type) to the output stream.
   ///
   /// The output will be in the smallest possible \em ext format.
-  void writeExt(int8_t Type, MemoryBufferRef Buffer);
+  LLVM_ABI void writeExt(int8_t Type, MemoryBufferRef Buffer);
 
 private:
   support::endian::Writer EW;
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/Wasm.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/Wasm.h
index 76cb4bf193b9..604104ff3cca 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/Wasm.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/Wasm.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -547,10 +548,10 @@ inline bool operator==(const WasmTableType &LHS, const WasmTableType &RHS) {
   return LHS.ElemType == RHS.ElemType && LHS.Limits == RHS.Limits;
 }
 
-llvm::StringRef toString(WasmSymbolType type);
-llvm::StringRef relocTypetoString(uint32_t type);
-llvm::StringRef sectionTypeToString(uint32_t type);
-bool relocTypeHasAddend(uint32_t type);
+LLVM_ABI llvm::StringRef toString(WasmSymbolType type);
+LLVM_ABI llvm::StringRef relocTypetoString(uint32_t type);
+LLVM_ABI llvm::StringRef sectionTypeToString(uint32_t type);
+LLVM_ABI bool relocTypeHasAddend(uint32_t type);
 
 } // end namespace wasm
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/BinaryFormat/XCOFF.h b/external/llvm-project/llvm/include/llvm/BinaryFormat/XCOFF.h
index b48976769c0c..24d5c74adb3f 100644
--- a/external/llvm-project/llvm/include/llvm/BinaryFormat/XCOFF.h
+++ b/external/llvm-project/llvm/include/llvm/BinaryFormat/XCOFF.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_BINARYFORMAT_XCOFF_H
 #define LLVM_BINARYFORMAT_XCOFF_H
 
+#include "llvm/Support/Compiler.h"
 #include <stddef.h>
 #include <stdint.h>
 
@@ -371,17 +372,17 @@ enum SymbolAuxType : uint8_t {
   AUX_SECT = 250    ///< Identifies a SECT auxiliary entry.
 };                  // 64-bit XCOFF file only.
 
-StringRef getMappingClassString(XCOFF::StorageMappingClass SMC);
-StringRef getRelocationTypeString(XCOFF::RelocationType Type);
-StringRef getTCPUString(XCOFF::CFileCpuId TCPU);
-Expected<SmallString<32>> parseParmsType(uint32_t Value, unsigned FixedParmsNum,
-                                         unsigned FloatingParmsNum);
-Expected<SmallString<32>> parseParmsTypeWithVecInfo(uint32_t Value,
-                                                    unsigned FixedParmsNum,
-                                                    unsigned FloatingParmsNum,
-                                                    unsigned VectorParmsNum);
-Expected<SmallString<32>> parseVectorParmsType(uint32_t Value,
-                                               unsigned ParmsNum);
+LLVM_ABI StringRef getMappingClassString(XCOFF::StorageMappingClass SMC);
+LLVM_ABI StringRef getRelocationTypeString(XCOFF::RelocationType Type);
+LLVM_ABI StringRef getTCPUString(XCOFF::CFileCpuId TCPU);
+LLVM_ABI Expected<SmallString<32>> parseParmsType(uint32_t Value,
+                                                  unsigned FixedParmsNum,
+                                                  unsigned FloatingParmsNum);
+LLVM_ABI Expected<SmallString<32>>
+parseParmsTypeWithVecInfo(uint32_t Value, unsigned FixedParmsNum,
+                          unsigned FloatingParmsNum, unsigned VectorParmsNum);
+LLVM_ABI Expected<SmallString<32>> parseVectorParmsType(uint32_t Value,
+                                                        unsigned ParmsNum);
 
 struct TracebackTable {
   enum LanguageID : uint8_t {
@@ -490,9 +491,10 @@ enum ExtendedTBTableFlag : uint8_t {
   TB_LONGTBTABLE2 = 0x01 ///< Additional tbtable extension exists.
 };
 
-StringRef getNameForTracebackTableLanguageId(TracebackTable::LanguageID LangId);
-SmallString<32> getExtendedTBTableFlagString(uint8_t Flag);
-XCOFF::CFileCpuId getCpuID(StringRef CPU);
+LLVM_ABI StringRef
+getNameForTracebackTableLanguageId(TracebackTable::LanguageID LangId);
+LLVM_ABI SmallString<32> getExtendedTBTableFlagString(uint8_t Flag);
+LLVM_ABI XCOFF::CFileCpuId getCpuID(StringRef CPU);
 
 struct CsectProperties {
   CsectProperties(StorageMappingClass SMC, SymbolType ST)
diff --git a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h
index c7219a52f976..c8677aa7e811 100644
--- a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h
+++ b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeAnalyzer.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <map>
 #include <optional>
@@ -83,14 +84,15 @@ class BitcodeAnalyzer {
   std::map<unsigned, PerBlockIDStats> BlockIDStats;
 
 public:
+  LLVM_ABI
   BitcodeAnalyzer(StringRef Buffer,
                   std::optional<StringRef> BlockInfoBuffer = std::nullopt);
   /// Analyze the bitcode file.
-  Error analyze(std::optional<BCDumpOptions> O = std::nullopt,
-                std::optional<StringRef> CheckHash = std::nullopt);
+  LLVM_ABI Error analyze(std::optional<BCDumpOptions> O = std::nullopt,
+                         std::optional<StringRef> CheckHash = std::nullopt);
   /// Print stats about the bitcode file.
-  void printStats(BCDumpOptions O,
-                  std::optional<StringRef> Filename = std::nullopt);
+  LLVM_ABI void printStats(BCDumpOptions O,
+                           std::optional<StringRef> Filename = std::nullopt);
 
 private:
   /// Read a block, updating statistics, etc.
diff --git a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeReader.h b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeReader.h
index f204060a3a97..e7451a3f1bac 100644
--- a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeReader.h
+++ b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeReader.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Bitstream/BitCodeEnums.h"
 #include "llvm/IR/GlobalValue.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorOr.h"
@@ -60,7 +61,8 @@ typedef std::function<void(Metadata **, unsigned, GetTypeByIDTy,
 // ErrorOr/std::error_code for compatibility with legacy clients. FIXME:
 // Remove these functions once no longer needed by the C and libLTO APIs.
 
-std::error_code errorToErrorCodeAndEmitErrors(LLVMContext &Ctx, Error Err);
+LLVM_ABI std::error_code errorToErrorCodeAndEmitErrors(LLVMContext &Ctx,
+                                                       Error Err);
 
 template <typename T>
 ErrorOr<T> expectedToErrorOrAndEmitErrors(LLVMContext &Ctx, Expected<T> Val) {
@@ -118,7 +120,7 @@ struct ParserCallbacks {
           IdentificationBit(IdentificationBit), ModuleBit(ModuleBit) {}
 
     // Calls the ctor.
-    friend Expected<BitcodeFileContents>
+    LLVM_ABI_FRIEND friend Expected<BitcodeFileContents>
     getBitcodeFileContents(MemoryBufferRef Buffer);
 
     Expected<std::unique_ptr<Module>>
@@ -139,24 +141,24 @@ struct ParserCallbacks {
     /// bodies. If ShouldLazyLoadMetadata is true, lazily load metadata as well.
     /// If IsImporting is true, this module is being parsed for ThinLTO
     /// importing into another module.
-    Expected<std::unique_ptr<Module>>
+    LLVM_ABI Expected<std::unique_ptr<Module>>
     getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata,
                   bool IsImporting, ParserCallbacks Callbacks = {});
 
     /// Read the entire bitcode module and return it.
-    Expected<std::unique_ptr<Module>>
+    LLVM_ABI Expected<std::unique_ptr<Module>>
     parseModule(LLVMContext &Context, ParserCallbacks Callbacks = {});
 
     /// Returns information about the module to be used for LTO: whether to
     /// compile with ThinLTO, and whether it has a summary.
-    Expected<BitcodeLTOInfo> getLTOInfo();
+    LLVM_ABI Expected<BitcodeLTOInfo> getLTOInfo();
 
     /// Parse the specified bitcode buffer, returning the module summary index.
-    Expected<std::unique_ptr<ModuleSummaryIndex>> getSummary();
+    LLVM_ABI Expected<std::unique_ptr<ModuleSummaryIndex>> getSummary();
 
     /// Parse the specified bitcode buffer and merge its module summary index
     /// into CombinedIndex.
-    Error
+    LLVM_ABI Error
     readSummary(ModuleSummaryIndex &CombinedIndex, StringRef ModulePath,
                 std::function<bool(GlobalValue::GUID)> IsPrevailing = nullptr);
   };
@@ -171,17 +173,18 @@ struct ParserCallbacks {
   /// symbol table should prefer to use irsymtab::read instead of this function
   /// because it creates a reader for the irsymtab and handles upgrading bitcode
   /// files without a symbol table or with an old symbol table.
-  Expected<BitcodeFileContents> getBitcodeFileContents(MemoryBufferRef Buffer);
+  LLVM_ABI Expected<BitcodeFileContents>
+  getBitcodeFileContents(MemoryBufferRef Buffer);
 
   /// Returns a list of modules in the specified bitcode buffer.
-  Expected<std::vector<BitcodeModule>>
+  LLVM_ABI Expected<std::vector<BitcodeModule>>
   getBitcodeModuleList(MemoryBufferRef Buffer);
 
   /// Read the header of the specified bitcode buffer and prepare for lazy
   /// deserialization of function bodies. If ShouldLazyLoadMetadata is true,
   /// lazily load metadata as well. If IsImporting is true, this module is
   /// being parsed for ThinLTO importing into another module.
-  Expected<std::unique_ptr<Module>>
+  LLVM_ABI Expected<std::unique_ptr<Module>>
   getLazyBitcodeModule(MemoryBufferRef Buffer, LLVMContext &Context,
                        bool ShouldLazyLoadMetadata = false,
                        bool IsImporting = false,
@@ -191,7 +194,7 @@ struct ParserCallbacks {
   /// the memory buffer if successful. If successful, this moves Buffer. On
   /// error, this *does not* move Buffer. If IsImporting is true, this module is
   /// being parsed for ThinLTO importing into another module.
-  Expected<std::unique_ptr<Module>> getOwningLazyBitcodeModule(
+  LLVM_ABI Expected<std::unique_ptr<Module>> getOwningLazyBitcodeModule(
       std::unique_ptr<MemoryBuffer> &&Buffer, LLVMContext &Context,
       bool ShouldLazyLoadMetadata = false, bool IsImporting = false,
       ParserCallbacks Callbacks = {});
@@ -199,38 +202,40 @@ struct ParserCallbacks {
   /// Read the header of the specified bitcode buffer and extract just the
   /// triple information. If successful, this returns a string. On error, this
   /// returns "".
-  Expected<std::string> getBitcodeTargetTriple(MemoryBufferRef Buffer);
+  LLVM_ABI Expected<std::string> getBitcodeTargetTriple(MemoryBufferRef Buffer);
 
   /// Return true if \p Buffer contains a bitcode file with ObjC code (category
   /// or class) in it.
-  Expected<bool> isBitcodeContainingObjCCategory(MemoryBufferRef Buffer);
+  LLVM_ABI Expected<bool>
+  isBitcodeContainingObjCCategory(MemoryBufferRef Buffer);
 
   /// Read the header of the specified bitcode buffer and extract just the
   /// producer string information. If successful, this returns a string. On
   /// error, this returns "".
-  Expected<std::string> getBitcodeProducerString(MemoryBufferRef Buffer);
+  LLVM_ABI Expected<std::string>
+  getBitcodeProducerString(MemoryBufferRef Buffer);
 
   /// Read the specified bitcode file, returning the module.
-  Expected<std::unique_ptr<Module>>
+  LLVM_ABI Expected<std::unique_ptr<Module>>
   parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context,
                    ParserCallbacks Callbacks = {});
 
   /// Returns LTO information for the specified bitcode file.
-  Expected<BitcodeLTOInfo> getBitcodeLTOInfo(MemoryBufferRef Buffer);
+  LLVM_ABI Expected<BitcodeLTOInfo> getBitcodeLTOInfo(MemoryBufferRef Buffer);
 
   /// Parse the specified bitcode buffer, returning the module summary index.
-  Expected<std::unique_ptr<ModuleSummaryIndex>>
+  LLVM_ABI Expected<std::unique_ptr<ModuleSummaryIndex>>
   getModuleSummaryIndex(MemoryBufferRef Buffer);
 
   /// Parse the specified bitcode buffer and merge the index into CombinedIndex.
-  Error readModuleSummaryIndex(MemoryBufferRef Buffer,
-                               ModuleSummaryIndex &CombinedIndex);
+  LLVM_ABI Error readModuleSummaryIndex(MemoryBufferRef Buffer,
+                                        ModuleSummaryIndex &CombinedIndex);
 
   /// Parse the module summary index out of an IR file and return the module
   /// summary index object if found, or an empty summary if not. If Path refers
   /// to an empty file and IgnoreEmptyThinLTOIndexFile is true, then
   /// this function will return nullptr.
-  Expected<std::unique_ptr<ModuleSummaryIndex>>
+  LLVM_ABI Expected<std::unique_ptr<ModuleSummaryIndex>>
   getModuleSummaryIndexForFile(StringRef Path,
                                bool IgnoreEmptyThinLTOIndexFile = false);
 
@@ -303,9 +308,9 @@ struct ParserCallbacks {
     return false;
   }
 
-  APInt readWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits);
+  LLVM_ABI APInt readWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits);
 
-  const std::error_category &BitcodeErrorCategory();
+  LLVM_ABI const std::error_category &BitcodeErrorCategory();
   enum class BitcodeError { CorruptedBitcode = 1 };
   inline std::error_code make_error_code(BitcodeError E) {
     return std::error_code(static_cast<int>(E), BitcodeErrorCategory());
diff --git a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriter.h b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriter.h
index 2823b438f80b..e9b573733451 100644
--- a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriter.h
+++ b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriter.h
@@ -17,6 +17,7 @@
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include <map>
 #include <memory>
@@ -46,10 +47,10 @@ class BitcodeWriter {
 
 public:
   /// Create a BitcodeWriter that writes to Buffer.
-  BitcodeWriter(SmallVectorImpl<char> &Buffer);
-  BitcodeWriter(raw_ostream &FS);
+  LLVM_ABI BitcodeWriter(SmallVectorImpl<char> &Buffer);
+  LLVM_ABI BitcodeWriter(raw_ostream &FS);
 
-  ~BitcodeWriter();
+  LLVM_ABI ~BitcodeWriter();
 
   /// Attempt to write a symbol table to the bitcode file. This must be called
   /// at most once after all modules have been written.
@@ -58,15 +59,15 @@ class BitcodeWriter {
   /// the symbol table is needed only to improve link-time performance. So
   /// this function may decide not to write a symbol table. It may so decide
   /// if, for example, the target is unregistered or the IR is malformed.
-  void writeSymtab();
+  LLVM_ABI void writeSymtab();
 
   /// Write the bitcode file's string table. This must be called exactly once
   /// after all modules and the optional symbol table have been written.
-  void writeStrtab();
+  LLVM_ABI void writeStrtab();
 
   /// Copy the string table for another module into this bitcode file. This
   /// should be called after copying the module itself into the bitcode file.
-  void copyStrtab(StringRef Strtab);
+  LLVM_ABI void copyStrtab(StringRef Strtab);
 
   /// Write the specified module to the buffer specified at construction time.
   ///
@@ -86,9 +87,11 @@ class BitcodeWriter {
   /// Can be used to produce the same module hash for a minimized bitcode
   /// used just for the thin link as in the regular full bitcode that will
   /// be used in the backend.
-  void writeModule(const Module &M, bool ShouldPreserveUseListOrder = false,
-                   const ModuleSummaryIndex *Index = nullptr,
-                   bool GenerateHash = false, ModuleHash *ModHash = nullptr);
+  LLVM_ABI void writeModule(const Module &M,
+                            bool ShouldPreserveUseListOrder = false,
+                            const ModuleSummaryIndex *Index = nullptr,
+                            bool GenerateHash = false,
+                            ModuleHash *ModHash = nullptr);
 
   /// Write the specified thin link bitcode file (i.e., the minimized bitcode
   /// file) to the buffer specified at construction time. The thin link
@@ -97,12 +100,14 @@ class BitcodeWriter {
   ///
   /// ModHash is for use in ThinLTO incremental build, generated while the
   /// IR bitcode file writing.
-  void writeThinLinkBitcode(const Module &M, const ModuleSummaryIndex &Index,
-                            const ModuleHash &ModHash);
-
-  void writeIndex(const ModuleSummaryIndex *Index,
-                  const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
-                  const GVSummaryPtrSet *DecSummaries);
+  LLVM_ABI void writeThinLinkBitcode(const Module &M,
+                                     const ModuleSummaryIndex &Index,
+                                     const ModuleHash &ModHash);
+
+  LLVM_ABI void
+  writeIndex(const ModuleSummaryIndex *Index,
+             const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
+             const GVSummaryPtrSet *DecSummaries);
 };
 
 /// Write the specified module to the specified raw output stream.
@@ -126,11 +131,11 @@ class BitcodeWriter {
 /// Can be used to produce the same module hash for a minimized bitcode
 /// used just for the thin link as in the regular full bitcode that will
 /// be used in the backend.
-void WriteBitcodeToFile(const Module &M, raw_ostream &Out,
-                        bool ShouldPreserveUseListOrder = false,
-                        const ModuleSummaryIndex *Index = nullptr,
-                        bool GenerateHash = false,
-                        ModuleHash *ModHash = nullptr);
+LLVM_ABI void WriteBitcodeToFile(const Module &M, raw_ostream &Out,
+                                 bool ShouldPreserveUseListOrder = false,
+                                 const ModuleSummaryIndex *Index = nullptr,
+                                 bool GenerateHash = false,
+                                 ModuleHash *ModHash = nullptr);
 
 /// Write the specified thin link bitcode file (i.e., the minimized bitcode
 /// file) to the given raw output stream, where it will be written in a new
@@ -139,9 +144,9 @@ void WriteBitcodeToFile(const Module &M, raw_ostream &Out,
 ///
 /// ModHash is for use in ThinLTO incremental build, generated while the IR
 /// bitcode file writing.
-void writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
-                                const ModuleSummaryIndex &Index,
-                                const ModuleHash &ModHash);
+LLVM_ABI void writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
+                                         const ModuleSummaryIndex &Index,
+                                         const ModuleHash &ModHash);
 
 /// Write the specified module summary index to the given raw output stream,
 /// where it will be written in a new bitcode block. This is used when
@@ -149,7 +154,7 @@ void writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
 /// index for a distributed backend, provide the \p ModuleToSummariesForIndex
 /// map. \p DecSummaries specifies the set of summaries for which the
 /// corresponding value should be imported as a declaration (prototype).
-void writeIndexToFile(
+LLVM_ABI void writeIndexToFile(
     const ModuleSummaryIndex &Index, raw_ostream &Out,
     const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr,
     const GVSummaryPtrSet *DecSummaries = nullptr);
@@ -163,9 +168,9 @@ void writeIndexToFile(
 /// If EmbedCmdline is set, the command line is also exported in
 /// the corresponding section (__LLVM,_cmdline / .llvmcmd) - even if CmdArgs
 /// were empty.
-void embedBitcodeInModule(Module &M, MemoryBufferRef Buf, bool EmbedBitcode,
-                          bool EmbedCmdline,
-                          const std::vector<uint8_t> &CmdArgs);
+LLVM_ABI void embedBitcodeInModule(Module &M, MemoryBufferRef Buf,
+                                   bool EmbedBitcode, bool EmbedCmdline,
+                                   const std::vector<uint8_t> &CmdArgs);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriterPass.h b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriterPass.h
index 4035c00b6793..3ce5db4ef106 100644
--- a/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriterPass.h
+++ b/external/llvm-project/llvm/include/llvm/Bitcode/BitcodeWriterPass.h
@@ -15,6 +15,7 @@
 #define LLVM_BITCODE_BITCODEWRITERPASS_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Module;
@@ -28,11 +29,12 @@ class raw_ostream;
 ///
 /// If \c ShouldPreserveUseListOrder, encode use-list order so it can be
 /// reproduced when deserialized.
-ModulePass *createBitcodeWriterPass(raw_ostream &Str,
-                                    bool ShouldPreserveUseListOrder = false);
+LLVM_ABI ModulePass *
+createBitcodeWriterPass(raw_ostream &Str,
+                        bool ShouldPreserveUseListOrder = false);
 
 /// Check whether a pass is a BitcodeWriterPass.
-bool isBitcodeWriterPass(Pass *P);
+LLVM_ABI bool isBitcodeWriterPass(Pass *P);
 
 /// Pass for writing a module of IR out to a bitcode file.
 ///
@@ -61,7 +63,7 @@ class BitcodeWriterPass : public PassInfoMixin<BitcodeWriterPass> {
 
   /// Run the bitcode writer pass, and output the module to the selected
   /// output stream.
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
 
   static bool isRequired() { return true; }
 };
diff --git a/external/llvm-project/llvm/include/llvm/Bitstream/BitstreamReader.h b/external/llvm-project/llvm/include/llvm/Bitstream/BitstreamReader.h
index dbc98d1ad725..ff42cda71ca6 100644
--- a/external/llvm-project/llvm/include/llvm/Bitstream/BitstreamReader.h
+++ b/external/llvm-project/llvm/include/llvm/Bitstream/BitstreamReader.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Bitstream/BitCodes.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
@@ -512,7 +513,7 @@ class BitstreamCursor : SimpleBitstreamCursor {
   }
 
   /// Having read the ENTER_SUBBLOCK abbrevid, and enter the block.
-  Error EnterSubBlock(unsigned BlockID, unsigned *NumWordsP = nullptr);
+  LLVM_ABI Error EnterSubBlock(unsigned BlockID, unsigned *NumWordsP = nullptr);
 
   bool ReadBlockEnd() {
     if (BlockScope.empty()) return true;
@@ -548,23 +549,23 @@ class BitstreamCursor : SimpleBitstreamCursor {
   }
 
   /// Read the current record and discard it, returning the code for the record.
-  Expected<unsigned> skipRecord(unsigned AbbrevID);
+  LLVM_ABI Expected<unsigned> skipRecord(unsigned AbbrevID);
 
-  Expected<unsigned> readRecord(unsigned AbbrevID,
-                                SmallVectorImpl<uint64_t> &Vals,
-                                StringRef *Blob = nullptr);
+  LLVM_ABI Expected<unsigned> readRecord(unsigned AbbrevID,
+                                         SmallVectorImpl<uint64_t> &Vals,
+                                         StringRef *Blob = nullptr);
 
   //===--------------------------------------------------------------------===//
   // Abbrev Processing
   //===--------------------------------------------------------------------===//
-  Error ReadAbbrevRecord();
+  LLVM_ABI Error ReadAbbrevRecord();
 
   /// Read and return a block info block from the bitstream. If an error was
   /// encountered, return std::nullopt.
   ///
   /// \param ReadBlockInfoNames Whether to read block/record name information in
   /// the BlockInfo block. Only llvm-bcanalyzer uses this.
-  Expected<std::optional<BitstreamBlockInfo>>
+  LLVM_ABI Expected<std::optional<BitstreamBlockInfo>>
   ReadBlockInfoBlock(bool ReadBlockInfoNames = false);
 
   /// Set the block info to be used by this BitstreamCursor to interpret
diff --git a/external/llvm-project/llvm/include/llvm/CGData/CGDataPatchItem.h b/external/llvm-project/llvm/include/llvm/CGData/CGDataPatchItem.h
new file mode 100644
index 000000000000..d13f89b03254
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/CGData/CGDataPatchItem.h
@@ -0,0 +1,33 @@
+//===- CGDataPatchItem.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for patching codegen data.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CGDATA_CGDATAPATCHITEM_H
+#define LLVM_CGDATA_CGDATAPATCHITEM_H
+
+#include "llvm/ADT/ArrayRef.h"
+
+namespace llvm {
+
+/// A struct to define how the data stream should be patched.
+struct CGDataPatchItem {
+  // Where to patch.
+  uint64_t Pos;
+  // Source data.
+  OwningArrayRef<uint64_t> D;
+
+  CGDataPatchItem(uint64_t Pos, const uint64_t *D, int N)
+      : Pos(Pos), D(ArrayRef<uint64_t>(D, N)) {}
+};
+
+} // namespace llvm
+
+#endif // LLVM_CGDATA_CGDATAPATCHITEM_H
diff --git a/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.h b/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.h
index ad7fa579f750..38b96b72ccac 100644
--- a/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.h
+++ b/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.h
@@ -282,6 +282,9 @@ enum CGDataVersion {
   Version1 = 1,
   // Version 2 supports the stable function merging map.
   Version2 = 2,
+  // Version 3 adds the total size of the Names in the stable function map so
+  // we can skip reading them into the memory for non-assertion builds.
+  Version3 = 3,
   CurrentVersion = CG_DATA_INDEX_VERSION
 };
 const uint64_t Version = CGDataVersion::CurrentVersion;
diff --git a/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.inc b/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.inc
index e0ae7a51024d..94de4c0b017a 100644
--- a/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.inc
+++ b/external/llvm-project/llvm/include/llvm/CGData/CodeGenData.inc
@@ -49,4 +49,4 @@ CG_DATA_SECT_ENTRY(CG_merge, CG_DATA_QUOTE(CG_DATA_MERGE_COMMON),
 #endif
 
 /* Indexed codegen data format version (start from 1). */
-#define CG_DATA_INDEX_VERSION 2
+#define CG_DATA_INDEX_VERSION 3
diff --git a/external/llvm-project/llvm/include/llvm/CGData/CodeGenDataWriter.h b/external/llvm-project/llvm/include/llvm/CGData/CodeGenDataWriter.h
index faef6beb30aa..b9c807c2c1cf 100644
--- a/external/llvm-project/llvm/include/llvm/CGData/CodeGenDataWriter.h
+++ b/external/llvm-project/llvm/include/llvm/CGData/CodeGenDataWriter.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CGDATA_CODEGENDATAWRITER_H
 #define LLVM_CGDATA_CODEGENDATAWRITER_H
 
+#include "llvm/CGData/CGDataPatchItem.h"
 #include "llvm/CGData/CodeGenData.h"
 #include "llvm/CGData/OutlinedHashTreeRecord.h"
 #include "llvm/CGData/StableFunctionMapRecord.h"
@@ -22,21 +23,23 @@
 
 namespace llvm {
 
-/// A struct to define how the data stream should be patched.
-struct CGDataPatchItem {
-  uint64_t Pos; // Where to patch.
-  uint64_t *D;  // Pointer to an array of source data.
-  int N;        // Number of elements in \c D array.
-};
-
 /// A wrapper class to abstract writer stream with support of bytes
 /// back patching.
 class CGDataOStream {
+  enum class OStreamKind {
+    fd,
+    string,
+    svector,
+  };
+
 public:
   CGDataOStream(raw_fd_ostream &FD)
-      : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {}
+      : Kind(OStreamKind::fd), OS(FD), LE(FD, llvm::endianness::little) {}
   CGDataOStream(raw_string_ostream &STR)
-      : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
+      : Kind(OStreamKind::string), OS(STR), LE(STR, llvm::endianness::little) {}
+  CGDataOStream(raw_svector_ostream &SVEC)
+      : Kind(OStreamKind::svector), OS(SVEC),
+        LE(SVEC, llvm::endianness::little) {}
 
   uint64_t tell() { return OS.tell(); }
   void write(uint64_t V) { LE.write<uint64_t>(V); }
@@ -48,9 +51,7 @@ class CGDataOStream {
   // directly and it won't be reflected in the stream's internal buffer.
   LLVM_ABI void patch(ArrayRef<CGDataPatchItem> P);
 
-  // If \c OS is an instance of \c raw_fd_ostream, this field will be
-  // true. Otherwise, \c OS will be an raw_string_ostream.
-  bool IsFDOStream;
+  OStreamKind Kind;
   raw_ostream &OS;
   support::endian::Writer LE;
 };
diff --git a/external/llvm-project/llvm/include/llvm/CGData/StableFunctionMapRecord.h b/external/llvm-project/llvm/include/llvm/CGData/StableFunctionMapRecord.h
index d4d3ffa06ad2..a75cb12a70ba 100644
--- a/external/llvm-project/llvm/include/llvm/CGData/StableFunctionMapRecord.h
+++ b/external/llvm-project/llvm/include/llvm/CGData/StableFunctionMapRecord.h
@@ -16,6 +16,7 @@
 #ifndef LLVM_CGDATA_STABLEFUNCTIONMAPRECORD_H
 #define LLVM_CGDATA_STABLEFUNCTIONMAPRECORD_H
 
+#include "llvm/CGData/CGDataPatchItem.h"
 #include "llvm/CGData/StableFunctionMap.h"
 #include "llvm/ObjectYAML/YAML.h"
 #include "llvm/Support/Compiler.h"
@@ -36,13 +37,16 @@ struct StableFunctionMapRecord {
   /// A static helper function to serialize the stable function map without
   /// owning the stable function map.
   LLVM_ABI static void serialize(raw_ostream &OS,
-                                 const StableFunctionMap *FunctionMap);
+                                 const StableFunctionMap *FunctionMap,
+                                 std::vector<CGDataPatchItem> &PatchItems);
 
   /// Serialize the stable function map to a raw_ostream.
-  LLVM_ABI void serialize(raw_ostream &OS) const;
+  LLVM_ABI void serialize(raw_ostream &OS,
+                          std::vector<CGDataPatchItem> &PatchItems) const;
 
   /// Deserialize the stable function map from a raw_ostream.
-  LLVM_ABI void deserialize(const unsigned char *&Ptr);
+  LLVM_ABI void deserialize(const unsigned char *&Ptr,
+                            bool ReadStableFunctionMapNames = true);
 
   /// Serialize the stable function map to a YAML stream.
   LLVM_ABI void serializeYAML(yaml::Output &YOS) const;
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/external/llvm-project/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 574152e254f1..3b87978fe3fa 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -478,12 +478,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   bool isIndexedLoadLegal(TTI::MemIndexedMode M, Type *Ty) const override {
-    EVT VT = getTLI()->getValueType(DL, Ty);
+    EVT VT = getTLI()->getValueType(DL, Ty, /*AllowUnknown=*/true);
     return getTLI()->isIndexedLoadLegal(getISDIndexedMode(M), VT);
   }
 
   bool isIndexedStoreLegal(TTI::MemIndexedMode M, Type *Ty) const override {
-    EVT VT = getTLI()->getValueType(DL, Ty);
+    EVT VT = getTLI()->getValueType(DL, Ty, /*AllowUnknown=*/true);
     return getTLI()->isIndexedStoreLegal(getISDIndexedMode(M), VT);
   }
 
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index 000797163433..3a2509345b77 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -161,6 +161,12 @@ enum {
   /// - Pred(2) - The predicate to test
   GIM_CheckImmOperandPredicate,
 
+  /// Check a leaf predicate on the specified instruction.
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Pred(2) - The predicate to test
+  GIM_CheckLeafOperandPredicate,
+
   /// Check a memory operation has the specified atomic ordering.
   /// - InsnID(ULEB128) - Instruction ID
   /// - Ordering(ULEB128) - The AtomicOrdering value
@@ -707,6 +713,12 @@ class GIMatchTableExecutor {
         "Subclasses must override this with a tablegen-erated function");
   }
 
+  virtual bool testMOPredicate_MO(unsigned, const MachineOperand &,
+                                  const MatcherState &State) const {
+    llvm_unreachable(
+        "Subclasses must override this with a tablegen-erated function");
+  }
+
   virtual bool testSimplePredicate(unsigned) const {
     llvm_unreachable("Subclass does not implement testSimplePredicate!");
   }
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 6c4f03649149..591cf9c97ae4 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -410,6 +410,26 @@ bool GIMatchTableExecutor::executeMatchTable(
           return false;
       break;
     }
+    case GIM_CheckLeafOperandPredicate: {
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t Predicate = readU16();
+      DEBUG_WITH_TYPE(TgtExecutor::getName(),
+                      dbgs() << CurrentIdx
+                             << ": GIM_CheckLeafOperandPredicate(MIs[" << InsnID
+                             << "]->getOperand(" << OpIdx
+                             << "), Predicate=" << Predicate << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      assert(State.MIs[InsnID]->getOperand(OpIdx).isReg() &&
+             "Expected register operand");
+      assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
+      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
+
+      if (!testMOPredicate_MO(Predicate, MO, State))
+        if (handleReject() == RejectAndGiveUp)
+          return false;
+      break;
+    }
     case GIM_CheckIsBuildVectorAllOnes:
     case GIM_CheckIsBuildVectorAllZeros: {
       uint64_t InsnID = readULEB();
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 3712a7fa06d9..22f6a5fde546 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -100,8 +100,8 @@ class LegalizationArtifactCombiner {
       const LLT DstTy = MRI.getType(DstReg);
       if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
         auto &CstVal = SrcMI->getOperand(1);
-        auto *MergedLocation = DILocation::getMergedLocation(
-            MI.getDebugLoc().get(), SrcMI->getDebugLoc().get());
+        auto MergedLocation =
+            DebugLoc::getMergedLocation(MI.getDebugLoc(), SrcMI->getDebugLoc());
         // Set the debug location to the merged location of the SrcMI and the MI
         // if the aext fold is successful.
         Builder.setDebugLoc(MergedLocation);
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 0df3df5b8e9c..fd72a3898562 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -935,6 +935,16 @@ class LegalizeRuleSet {
                                      Types2);
   }
 
+  /// The instruction is custom when the predicate is true and type indexes 0
+  /// and 1 are all in their respective lists.
+  LegalizeRuleSet &
+  customForCartesianProduct(bool Pred, std::initializer_list<LLT> Types0,
+                            std::initializer_list<LLT> Types1) {
+    if (!Pred)
+      return *this;
+    return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1);
+  }
+
   /// Unconditionally custom lower.
   LegalizeRuleSet &custom() {
     return customIf(always);
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index ae3145677817..66c960fe12c6 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -617,6 +617,10 @@ LLVM_ABI bool isGuaranteedNotToBeUndef(Register Reg,
 /// estimate of the type.
 LLVM_ABI Type *getTypeForLLT(LLT Ty, LLVMContext &C);
 
+/// Returns true if the instruction \p MI is one of the assert
+/// instructions.
+LLVM_ABI bool isAssertMI(const MachineInstr &MI);
+
 /// An integer-like constant.
 ///
 /// It abstracts over scalar, fixed-length vectors, and scalable vectors.
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/ISDOpcodes.h b/external/llvm-project/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 47a1aec3da06..465e4a0a9d0d 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1493,8 +1493,9 @@ enum NodeType {
   VECREDUCE_UMIN,
 
   // PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2)
-  // The partial reduction nodes sign or zero extend Input1 and Input2 to the
-  // element type of Accumulator before multiplying their results.
+  // The partial reduction nodes sign or zero extend Input1 and Input2
+  // (with the extension kind noted below) to the element type of
+  // Accumulator before multiplying their results.
   // This result is concatenated to the Accumulator, and this is then reduced,
   // using addition, to the result type.
   // The output is only expected to either be given to another partial reduction
@@ -1506,8 +1507,9 @@ enum NodeType {
   // multiple of the number of elements in the Accumulator / output type.
   // Input1 and Input2 must have an element type which is the same as or smaller
   // than the element type of the Accumulator and output.
-  PARTIAL_REDUCE_SMLA,
-  PARTIAL_REDUCE_UMLA,
+  PARTIAL_REDUCE_SMLA,  // sext, sext
+  PARTIAL_REDUCE_UMLA,  // zext, zext
+  PARTIAL_REDUCE_SUMLA, // sext, zext
 
   // The `llvm.experimental.stackmap` intrinsic.
   // Operands: input chain, glue, <id>, <numShadowBytes>, [live0[, live1...]]
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/external/llvm-project/llvm/include/llvm/CodeGen/MachineBasicBlock.h
index f208d1c04841..3d2da01f2c85 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/MachineBasicBlock.h
@@ -277,18 +277,33 @@ class MachineBasicBlock
   /// of a terminator, exception-handling target, or jump table. This is
   /// either the result of an IR-level "blockaddress", or some form
   /// of target-specific branch lowering.
+  ///
+  /// The name of this function `hasAddressTaken` implies that the address of
+  /// the block is known and used in a general sense, but not necessarily that
+  /// the address is used by an indirect branch instruction. So branch target
+  /// enforcement need not put a BTI instruction (or equivalent) at the start
+  /// of a block just because this function returns true. The decision about
+  /// whether to add a BTI can be more subtle than that, and depends on the
+  /// more detailed checks that this function aggregates together.
   bool hasAddressTaken() const {
-    return MachineBlockAddressTaken || AddressTakenIRBlock;
+    return MachineBlockAddressTaken || AddressTakenIRBlock ||
+           IsInlineAsmBrIndirectTarget;
   }
 
   /// Test whether this block is used as something other than the target of a
   /// terminator, exception-handling target, jump table, or IR blockaddress.
   /// For example, its address might be loaded into a register, or
   /// stored in some branch table that isn't part of MachineJumpTableInfo.
+  ///
+  /// If this function returns true, it _does_ mean that branch target
+  /// enforcement needs to put a BTI or equivalent at the start of the block.
   bool isMachineBlockAddressTaken() const { return MachineBlockAddressTaken; }
 
   /// Test whether this block is the target of an IR BlockAddress.  (There can
   /// more than one MBB associated with an IR BB where the address is taken.)
+  ///
+  /// If this function returns true, it _does_ mean that branch target
+  /// enforcement needs to put a BTI or equivalent at the start of the block.
   bool isIRBlockAddressTaken() const { return AddressTakenIRBlock; }
 
   /// Retrieves the BasicBlock which corresponds to this MachineBasicBlock.
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/MachinePipeliner.h b/external/llvm-project/llvm/include/llvm/CodeGen/MachinePipeliner.h
index 966ffb7a1fbd..e4e794c434ad 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/MachinePipeliner.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/MachinePipeliner.h
@@ -190,6 +190,38 @@ class SwingSchedulerDDGEdge {
   bool ignoreDependence(bool IgnoreAnti) const;
 };
 
+/// Represents loop-carried dependencies. Because SwingSchedulerDAG doesn't
+/// assume cycle dependencies as the name suggests, such dependencies must be
+/// handled separately. After DAG construction is finished, these dependencies
+/// are added to SwingSchedulerDDG.
+/// TODO: Also handle output-dependencies introduced by physical registers.
+struct LoopCarriedEdges {
+  using OrderDep = SmallSetVector<SUnit *, 8>;
+  using OrderDepsType = DenseMap<SUnit *, OrderDep>;
+
+  OrderDepsType OrderDeps;
+
+  const OrderDep *getOrderDepOrNull(SUnit *Key) const {
+    auto Ite = OrderDeps.find(Key);
+    if (Ite == OrderDeps.end())
+      return nullptr;
+    return &Ite->second;
+  }
+
+  /// Retruns true if the edge from \p From to \p To is a back-edge that should
+  /// be used when scheduling.
+  bool shouldUseWhenScheduling(const SUnit *From, const SUnit *To) const;
+
+  /// Adds some edges to the original DAG that correspond to loop-carried
+  /// dependencies. Historically, loop-carried edges are represented by using
+  /// non-loop-carried edges in the original DAG. This function appends such
+  /// edges to preserve the previous behavior.
+  void modifySUnits(std::vector<SUnit> &SUnits);
+
+  void dump(SUnit *SU, const TargetRegisterInfo *TRI,
+            const MachineRegisterInfo *MRI) const;
+};
+
 /// Represents dependencies between instructions. This class is a wrapper of
 /// `SUnits` and its dependencies to manipulate back-edges in a natural way.
 /// Currently it only supports back-edges via PHI, which are expressed as
@@ -402,7 +434,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
                              const MachineInstr *OtherMI) const;
 
 private:
-  void addLoopCarriedDependences();
+  LoopCarriedEdges addLoopCarriedDependences();
   void updatePhiDependences();
   void changeDependences();
   unsigned calculateResMII();
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/MachineScheduler.h b/external/llvm-project/llvm/include/llvm/CodeGen/MachineScheduler.h
index 696e390e766c..486508e760dc 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -41,7 +41,7 @@
 //
 // ScheduleDAGInstrs *<Target>TargetMachine::
 // createMachineScheduler(MachineSchedContext *C) {
-//   ScheduleDAGMI *DAG = createGenericSchedLive(C);
+//   ScheduleDAGMI *DAG = createSchedLive(C);
 //   DAG->addMutation(new CustomDAGMutation(...));
 //   return DAG;
 // }
@@ -304,10 +304,6 @@ class LLVM_ABI ScheduleDAGMI : public ScheduleDAGInstrs {
   /// The bottom of the unscheduled zone.
   MachineBasicBlock::iterator CurrentBottom;
 
-  /// Record the next node in a scheduled cluster.
-  const SUnit *NextClusterPred = nullptr;
-  const SUnit *NextClusterSucc = nullptr;
-
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
   /// The number of instructions scheduled so far. Used to cut off the
   /// scheduler at the point determined by misched-cutoff.
@@ -368,10 +364,6 @@ class LLVM_ABI ScheduleDAGMI : public ScheduleDAGInstrs {
   /// live ranges and region boundary iterators.
   void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos);
 
-  const SUnit *getNextClusterPred() const { return NextClusterPred; }
-
-  const SUnit *getNextClusterSucc() const { return NextClusterSucc; }
-
   void viewGraph(const Twine &Name, const Twine &Title) override;
   void viewGraph() override;
 
@@ -1295,6 +1287,9 @@ class LLVM_ABI GenericScheduler : public GenericSchedulerBase {
   SchedBoundary Top;
   SchedBoundary Bot;
 
+  ClusterInfo *TopCluster;
+  ClusterInfo *BotCluster;
+
   /// Candidate last picked from Top boundary.
   SchedCandidate TopCand;
   /// Candidate last picked from Bot boundary.
@@ -1335,6 +1330,9 @@ class LLVM_ABI PostGenericScheduler : public GenericSchedulerBase {
   /// Candidate last picked from Bot boundary.
   SchedCandidate BotCand;
 
+  ClusterInfo *TopCluster;
+  ClusterInfo *BotCluster;
+
 public:
   PostGenericScheduler(const MachineSchedContext *C)
       : GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ"),
@@ -1383,14 +1381,6 @@ class LLVM_ABI PostGenericScheduler : public GenericSchedulerBase {
   void pickNodeFromQueue(SchedBoundary &Zone, SchedCandidate &Cand);
 };
 
-/// Create the standard converging machine scheduler. This will be used as the
-/// default scheduler if the target does not set a default.
-/// Adds default DAG mutations.
-LLVM_ABI ScheduleDAGMILive *createGenericSchedLive(MachineSchedContext *C);
-
-/// Create a generic scheduler with no vreg liveness or DAG mutation passes.
-LLVM_ABI ScheduleDAGMI *createGenericSchedPostRA(MachineSchedContext *C);
-
 /// If ReorderWhileClustering is set to true, no attempt will be made to
 /// reduce reordering due to store clustering.
 LLVM_ABI std::unique_ptr<ScheduleDAGMutation>
@@ -1409,6 +1399,41 @@ LLVM_ABI std::unique_ptr<ScheduleDAGMutation>
 createCopyConstrainDAGMutation(const TargetInstrInfo *TII,
                                const TargetRegisterInfo *TRI);
 
+/// Create the standard converging machine scheduler. This will be used as the
+/// default scheduler if the target does not set a default.
+/// Adds default DAG mutations.
+template <typename Strategy = GenericScheduler>
+LLVM_ABI ScheduleDAGMILive *createSchedLive(MachineSchedContext *C) {
+  ScheduleDAGMILive *DAG =
+      new ScheduleDAGMILive(C, std::make_unique<Strategy>(C));
+  // Register DAG post-processors.
+  //
+  // FIXME: extend the mutation API to allow earlier mutations to instantiate
+  // data and pass it to later mutations. Have a single mutation that gathers
+  // the interesting nodes in one pass.
+  DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII, DAG->TRI));
+
+  const TargetSubtargetInfo &STI = C->MF->getSubtarget();
+  // Add MacroFusion mutation if fusions are not empty.
+  const auto &MacroFusions = STI.getMacroFusions();
+  if (!MacroFusions.empty())
+    DAG->addMutation(createMacroFusionDAGMutation(MacroFusions));
+  return DAG;
+}
+
+/// Create a generic scheduler with no vreg liveness or DAG mutation passes.
+template <typename Strategy = PostGenericScheduler>
+LLVM_ABI ScheduleDAGMI *createSchedPostRA(MachineSchedContext *C) {
+  ScheduleDAGMI *DAG = new ScheduleDAGMI(C, std::make_unique<Strategy>(C),
+                                         /*RemoveKillFlags=*/true);
+  const TargetSubtargetInfo &STI = C->MF->getSubtarget();
+  // Add MacroFusion mutation if fusions are not empty.
+  const auto &MacroFusions = STI.getMacroFusions();
+  if (!MacroFusions.empty())
+    DAG->addMutation(createMacroFusionDAGMutation(MacroFusions));
+  return DAG;
+}
+
 class MachineSchedulerPass : public PassInfoMixin<MachineSchedulerPass> {
   // FIXME: Remove this member once RegisterClassInfo is queryable as an
   // analysis.
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/RegisterClassInfo.h b/external/llvm-project/llvm/include/llvm/CodeGen/RegisterClassInfo.h
index 3096f8851516..078ae80915fe 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/RegisterClassInfo.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/RegisterClassInfo.h
@@ -50,6 +50,8 @@ class RegisterClassInfo {
   // entry is valid when its tag matches.
   unsigned Tag = 0;
 
+  bool Reverse = false;
+
   const MachineFunction *MF = nullptr;
   const TargetRegisterInfo *TRI = nullptr;
 
@@ -86,9 +88,11 @@ class RegisterClassInfo {
 public:
   LLVM_ABI RegisterClassInfo();
 
-  /// runOnFunction - Prepare to answer questions about MF. This must be called
+  /// runOnFunction - Prepare to answer questions about MF. Rev indicates to
+  /// use reversed raw order when compute register order. This must be called
   /// before any other methods are used.
-  LLVM_ABI void runOnMachineFunction(const MachineFunction &MF);
+  LLVM_ABI void runOnMachineFunction(const MachineFunction &MF,
+                                     bool Rev = false);
 
   /// getNumAllocatableRegs - Returns the number of actually allocatable
   /// registers in RC in the current function.
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAG.h b/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAG.h
index 7a49e9ffedf8..3a0a31b1930b 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAG.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAG.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -235,6 +236,10 @@ class TargetRegisterInfo;
     LLVM_ABI void dump(const TargetRegisterInfo *TRI = nullptr) const;
   };
 
+  /// Keep record of which SUnit are in the same cluster group.
+  typedef SmallSet<SUnit *, 8> ClusterInfo;
+  constexpr unsigned InvalidClusterId = ~0u;
+
   /// Scheduling unit. This is a node in the scheduling DAG.
   class SUnit {
   private:
@@ -275,6 +280,8 @@ class TargetRegisterInfo;
     unsigned TopReadyCycle = 0; ///< Cycle relative to start when node is ready.
     unsigned BotReadyCycle = 0; ///< Cycle relative to end when node is ready.
 
+    unsigned ParentClusterIdx = InvalidClusterId; ///< The parent cluster id.
+
   private:
     unsigned Depth = 0;  ///< Node depth.
     unsigned Height = 0; ///< Node height.
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
index 7f7b3036af80..ba5b2da64fd8 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -181,6 +181,8 @@ namespace llvm {
     /// case of a huge region that gets reduced).
     SUnit *BarrierChain = nullptr;
 
+    SmallVector<ClusterInfo> Clusters;
+
   public:
     /// A list of SUnits, used in Value2SUsMap, during DAG construction.
     /// Note: to gain speed it might be worth investigating an optimized
@@ -384,6 +386,14 @@ namespace llvm {
     /// equivalent edge already existed (false indicates failure).
     bool addEdge(SUnit *SuccSU, const SDep &PredDep);
 
+    /// Returns the array of the clusters.
+    SmallVector<ClusterInfo> &getClusters() { return Clusters; }
+
+    /// Get the specific cluster, return nullptr for InvalidClusterId.
+    ClusterInfo *getCluster(unsigned Idx) {
+      return Idx != InvalidClusterId ? &Clusters[Idx] : nullptr;
+    }
+
   protected:
     void initSUnits();
     void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx);
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/ScoreboardHazardRecognizer.h b/external/llvm-project/llvm/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
index a70acca98a5c..727cd9abf927 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
@@ -59,7 +59,7 @@ class ScoreboardHazardRecognizer : public ScheduleHazardRecognizer {
 
     InstrStage::FuncUnits& operator[](size_t idx) const {
       // Depth is expected to be a power-of-2.
-      assert(Depth && !(Depth & (Depth - 1)) &&
+      assert(llvm::has_single_bit(Depth) &&
              "Scoreboard was not initialized properly!");
 
       return Data[(Head + idx) & (Depth-1)];
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 7a41e09b6aea..375d2f31e283 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -47,7 +47,7 @@ class SelectionDAGISel {
   TargetMachine &TM;
   const TargetLibraryInfo *LibInfo;
   std::unique_ptr<FunctionLoweringInfo> FuncInfo;
-  SwiftErrorValueTracking *SwiftError;
+  std::unique_ptr<SwiftErrorValueTracking> SwiftError;
   MachineFunction *MF;
   MachineModuleInfo *MMI;
   MachineRegisterInfo *RegInfo;
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 2b778aa9e4fe..06217337a46f 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -724,6 +724,19 @@ END_TWO_BYTE_PACK()
     }
   }
 
+  /// Test if this node is an assert operation.
+  bool isAssert() const {
+    switch (NodeType) {
+    default:
+      return false;
+    case ISD::AssertAlign:
+    case ISD::AssertNoFPClass:
+    case ISD::AssertSext:
+    case ISD::AssertZext:
+      return true;
+    }
+  }
+
   /// Test if this node is a vector predication operation.
   bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); }
 
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h b/external/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h
index 9c453f51e129..4ed81d25e8e2 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -254,20 +254,20 @@ class LLVM_ABI TargetLoweringBase {
   /// support for these atomic instructions, and also have different options
   /// w.r.t. what they should expand to.
   enum class AtomicExpansionKind {
-    None,    // Don't expand the instruction.
-    CastToInteger,    // Cast the atomic instruction to another type, e.g. from
-                      // floating-point to integer type.
+    None,          // Don't expand the instruction.
+    CastToInteger, // Cast the atomic instruction to another type, e.g. from
+                   // floating-point to integer type.
     LLSC,    // Expand the instruction into loadlinked/storeconditional; used
-             // by ARM/AArch64.
+             // by ARM/AArch64/PowerPC.
     LLOnly,  // Expand the (load) instruction into just a load-linked, which has
              // greater atomic guarantees than a normal load.
     CmpXChg, // Expand the instruction into cmpxchg; used by at least X86.
-    MaskedIntrinsic,  // Use a target-specific intrinsic for the LL/SC loop.
-    BitTestIntrinsic, // Use a target-specific intrinsic for special bit
-                      // operations; used by X86.
-    CmpArithIntrinsic,// Use a target-specific intrinsic for special compare
-                      // operations; used by X86.
-    Expand,           // Generic expansion in terms of other atomic operations.
+    MaskedIntrinsic,   // Use a target-specific intrinsic for the LL/SC loop.
+    BitTestIntrinsic,  // Use a target-specific intrinsic for special bit
+                       // operations; used by X86.
+    CmpArithIntrinsic, // Use a target-specific intrinsic for special compare
+                       // operations; used by X86.
+    Expand,            // Generic expansion in terms of other atomic operations.
 
     // Rewrite to a non-atomic form for use in a known non-preemptible
     // environment.
@@ -1661,7 +1661,8 @@ class LLVM_ABI TargetLoweringBase {
   /// target has a custom expander for it.
   LegalizeAction getPartialReduceMLAAction(unsigned Opc, EVT AccVT,
                                            EVT InputVT) const {
-    assert(Opc == ISD::PARTIAL_REDUCE_SMLA || Opc == ISD::PARTIAL_REDUCE_UMLA);
+    assert(Opc == ISD::PARTIAL_REDUCE_SMLA || Opc == ISD::PARTIAL_REDUCE_UMLA ||
+           Opc == ISD::PARTIAL_REDUCE_SUMLA);
     PartialReduceActionTypes Key = {Opc, AccVT.getSimpleVT().SimpleTy,
                                     InputVT.getSimpleVT().SimpleTy};
     auto It = PartialReduceMLAActions.find(Key);
@@ -2759,7 +2760,8 @@ class LLVM_ABI TargetLoweringBase {
   /// sequence, or the target has a custom expander for it.
   void setPartialReduceMLAAction(unsigned Opc, MVT AccVT, MVT InputVT,
                                  LegalizeAction Action) {
-    assert(Opc == ISD::PARTIAL_REDUCE_SMLA || Opc == ISD::PARTIAL_REDUCE_UMLA);
+    assert(Opc == ISD::PARTIAL_REDUCE_SMLA || Opc == ISD::PARTIAL_REDUCE_UMLA ||
+           Opc == ISD::PARTIAL_REDUCE_SUMLA);
     assert(AccVT.isValid() && InputVT.isValid() &&
            "setPartialReduceMLAAction types aren't valid");
     PartialReduceActionTypes Key = {Opc, AccVT.SimpleTy, InputVT.SimpleTy};
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/external/llvm-project/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index fa6cb75338d4..00e681e6bf53 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -117,10 +117,6 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
   MCSection *getStaticDtorSection(unsigned Priority,
                                   const MCSymbol *KeySym) const override;
 
-  virtual const MCExpr *createTargetMCExpr(const MCExpr *Expr,
-                                           uint8_t Specifier) const {
-    return nullptr;
-  }
   const MCExpr *
   lowerSymbolDifference(const MCSymbol *LHS, const MCSymbol *RHS,
                         int64_t Addend,
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/external/llvm-project/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 8a62700d550b..d0ff0648c5a7 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -68,7 +68,7 @@ class TargetRegisterClass {
   const bool CoveredBySubRegs;
   const unsigned *SuperClasses;
   const uint16_t SuperClassesSize;
-  ArrayRef<MCPhysReg> (*OrderFunc)(const MachineFunction&);
+  ArrayRef<MCPhysReg> (*OrderFunc)(const MachineFunction &, bool Rev);
 
   /// Return the register class ID number.
   unsigned getID() const { return MC->getID(); }
@@ -199,8 +199,9 @@ class TargetRegisterClass {
   /// other criteria.
   ///
   /// By default, this method returns all registers in the class.
-  ArrayRef<MCPhysReg> getRawAllocationOrder(const MachineFunction &MF) const {
-    return OrderFunc ? OrderFunc(MF) : getRegisters();
+  ArrayRef<MCPhysReg> getRawAllocationOrder(const MachineFunction &MF,
+                                            bool Rev = false) const {
+    return OrderFunc ? OrderFunc(MF, Rev) : getRegisters();
   }
 
   /// Returns the combination of all lane masks of register in this class.
diff --git a/external/llvm-project/llvm/include/llvm/CodeGen/ValueTypes.td b/external/llvm-project/llvm/include/llvm/CodeGen/ValueTypes.td
index 1e9632357edd..4551e7e4b9b6 100644
--- a/external/llvm-project/llvm/include/llvm/CodeGen/ValueTypes.td
+++ b/external/llvm-project/llvm/include/llvm/CodeGen/ValueTypes.td
@@ -155,194 +155,195 @@ def v256i32  : VTVec<256,  i32, 75>;  //  256 x i32 vector value
 def v512i32  : VTVec<512,  i32, 76>;  //  512 x i32 vector value
 def v1024i32 : VTVec<1024, i32, 77>;  // 1024 x i32 vector value
 def v2048i32 : VTVec<2048, i32, 78>;  // 2048 x i32 vector value
-
-def v1i64   : VTVec<1,   i64, 79>;  //   1 x i64 vector value
-def v2i64   : VTVec<2,   i64, 80>;  //   2 x i64 vector value
-def v3i64   : VTVec<3,   i64, 81>;  //   3 x i64 vector value
-def v4i64   : VTVec<4,   i64, 82>;  //   4 x i64 vector value
-def v8i64   : VTVec<8,   i64, 83>;  //   8 x i64 vector value
-def v16i64  : VTVec<16,  i64, 84>;  //  16 x i64 vector value
-def v32i64  : VTVec<32,  i64, 85>;  //  32 x i64 vector value
-def v64i64  : VTVec<64,  i64, 86>;  //  64 x i64 vector value
-def v128i64 : VTVec<128, i64, 87>;  // 128 x i64 vector value
-def v256i64 : VTVec<256, i64, 88>;  // 256 x i64 vector value
-
-def v1i128  : VTVec<1,  i128, 89>;  //  1 x i128 vector value
-
-def v1f16    : VTVec<1,    f16,  90>;  //    1 x f16 vector value
-def v2f16    : VTVec<2,    f16,  91>;  //    2 x f16 vector value
-def v3f16    : VTVec<3,    f16,  92>;  //    3 x f16 vector value
-def v4f16    : VTVec<4,    f16,  93>;  //    4 x f16 vector value
-def v8f16    : VTVec<8,    f16,  94>;  //    8 x f16 vector value
-def v16f16   : VTVec<16,   f16,  95>;  //   16 x f16 vector value
-def v32f16   : VTVec<32,   f16,  96>;  //   32 x f16 vector value
-def v64f16   : VTVec<64,   f16,  97>;  //   64 x f16 vector value
-def v128f16  : VTVec<128,  f16,  98>;  //  128 x f16 vector value
-def v256f16  : VTVec<256,  f16,  99>;  //  256 x f16 vector value
-def v512f16  : VTVec<512,  f16,  100>;  //  512 x f16 vector value
-def v4096f16 : VTVec<4096, f16,  101>;  // 4096 x f16 vector value
-
-def v1bf16    : VTVec<1,    bf16, 102>;  //    1 x bf16 vector value
-def v2bf16    : VTVec<2,    bf16, 103>;  //    2 x bf16 vector value
-def v3bf16    : VTVec<3,    bf16, 104>;  //    3 x bf16 vector value
-def v4bf16    : VTVec<4,    bf16, 105>;  //    4 x bf16 vector value
-def v8bf16    : VTVec<8,    bf16, 106>;  //    8 x bf16 vector value
-def v16bf16   : VTVec<16,   bf16, 107>;  //   16 x bf16 vector value
-def v32bf16   : VTVec<32,   bf16, 108>;  //   32 x bf16 vector value
-def v64bf16   : VTVec<64,   bf16, 109>;  //   64 x bf16 vector value
-def v128bf16  : VTVec<128,  bf16, 110>;  //  128 x bf16 vector value
-def v4096bf16 : VTVec<4096, bf16, 111>;  // 4096 x bf16 vector value
-
-def v1f32    : VTVec<1,    f32, 112>;  //    1 x f32 vector value
-def v2f32    : VTVec<2,    f32, 113>;  //    2 x f32 vector value
-def v3f32    : VTVec<3,    f32, 114>;  //    3 x f32 vector value
-def v4f32    : VTVec<4,    f32, 115>;  //    4 x f32 vector value
-def v5f32    : VTVec<5,    f32, 116>;  //    5 x f32 vector value
-def v6f32    : VTVec<6,    f32, 117>;  //    6 x f32 vector value
-def v7f32    : VTVec<7,    f32, 118>;  //    7 x f32 vector value
-def v8f32    : VTVec<8,    f32, 119>;  //    8 x f32 vector value
-def v9f32    : VTVec<9,    f32, 120>;  //    9 x f32 vector value
-def v10f32   : VTVec<10,   f32, 121>;  //   10 x f32 vector value
-def v11f32   : VTVec<11,   f32, 122>;  //   11 x f32 vector value
-def v12f32   : VTVec<12,   f32, 123>;  //   12 x f32 vector value
-def v16f32   : VTVec<16,   f32, 124>;  //   16 x f32 vector value
-def v32f32   : VTVec<32,   f32, 125>;  //   32 x f32 vector value
-def v64f32   : VTVec<64,   f32, 126>;  //   64 x f32 vector value
-def v128f32  : VTVec<128,  f32, 127>;  //  128 x f32 vector value
-def v256f32  : VTVec<256,  f32, 128>;  //  256 x f32 vector value
-def v512f32  : VTVec<512,  f32, 129>;  //  512 x f32 vector value
-def v1024f32 : VTVec<1024, f32, 130>;  // 1024 x f32 vector value
-def v2048f32 : VTVec<2048, f32, 131>;  // 2048 x f32 vector value
-
-def v1f64    : VTVec<1,    f64, 132>;  //    1 x f64 vector value
-def v2f64    : VTVec<2,    f64, 133>;  //    2 x f64 vector value
-def v3f64    : VTVec<3,    f64, 134>;  //    3 x f64 vector value
-def v4f64    : VTVec<4,    f64, 135>;  //    4 x f64 vector value
-def v8f64    : VTVec<8,    f64, 136>;  //    8 x f64 vector value
-def v16f64   : VTVec<16,   f64, 137>;  //   16 x f64 vector value
-def v32f64   : VTVec<32,   f64, 138>;  //   32 x f64 vector value
-def v64f64   : VTVec<64,   f64, 139>;  //   64 x f64 vector value
-def v128f64  : VTVec<128,  f64, 140>;  //  128 x f64 vector value
-def v256f64  : VTVec<256,  f64, 141>;  //  256 x f64 vector value
-
-def nxv1i1  : VTScalableVec<1,  i1, 142>;  // n x  1 x i1  vector value
-def nxv2i1  : VTScalableVec<2,  i1, 143>;  // n x  2 x i1  vector value
-def nxv4i1  : VTScalableVec<4,  i1, 144>;  // n x  4 x i1  vector value
-def nxv8i1  : VTScalableVec<8,  i1, 145>;  // n x  8 x i1  vector value
-def nxv16i1 : VTScalableVec<16, i1, 146>;  // n x 16 x i1  vector value
-def nxv32i1 : VTScalableVec<32, i1, 147>;  // n x 32 x i1  vector value
-def nxv64i1 : VTScalableVec<64, i1, 148>;  // n x 64 x i1  vector value
-
-def nxv1i8  : VTScalableVec<1,  i8, 149>;  // n x  1 x i8  vector value
-def nxv2i8  : VTScalableVec<2,  i8, 150>;  // n x  2 x i8  vector value
-def nxv4i8  : VTScalableVec<4,  i8, 151>;  // n x  4 x i8  vector value
-def nxv8i8  : VTScalableVec<8,  i8, 152>;  // n x  8 x i8  vector value
-def nxv16i8 : VTScalableVec<16, i8, 153>;  // n x 16 x i8  vector value
-def nxv32i8 : VTScalableVec<32, i8, 154>;  // n x 32 x i8  vector value
-def nxv64i8 : VTScalableVec<64, i8, 155>;  // n x 64 x i8  vector value
-
-def nxv1i16  : VTScalableVec<1,  i16, 156>;  // n x  1 x i16 vector value
-def nxv2i16  : VTScalableVec<2,  i16, 157>;  // n x  2 x i16 vector value
-def nxv4i16  : VTScalableVec<4,  i16, 158>;  // n x  4 x i16 vector value
-def nxv8i16  : VTScalableVec<8,  i16, 159>;  // n x  8 x i16 vector value
-def nxv16i16 : VTScalableVec<16, i16, 160>;  // n x 16 x i16 vector value
-def nxv32i16 : VTScalableVec<32, i16, 161>;  // n x 32 x i16 vector value
-
-def nxv1i32  : VTScalableVec<1,  i32, 162>;  // n x  1 x i32 vector value
-def nxv2i32  : VTScalableVec<2,  i32, 163>;  // n x  2 x i32 vector value
-def nxv4i32  : VTScalableVec<4,  i32, 164>;  // n x  4 x i32 vector value
-def nxv8i32  : VTScalableVec<8,  i32, 165>;  // n x  8 x i32 vector value
-def nxv16i32 : VTScalableVec<16, i32, 166>;  // n x 16 x i32 vector value
-def nxv32i32 : VTScalableVec<32, i32, 167>;  // n x 32 x i32 vector value
-
-def nxv1i64  : VTScalableVec<1,  i64, 168>;  // n x  1 x i64 vector value
-def nxv2i64  : VTScalableVec<2,  i64, 169>;  // n x  2 x i64 vector value
-def nxv4i64  : VTScalableVec<4,  i64, 170>;  // n x  4 x i64 vector value
-def nxv8i64  : VTScalableVec<8,  i64, 171>;  // n x  8 x i64 vector value
-def nxv16i64 : VTScalableVec<16, i64, 172>;  // n x 16 x i64 vector value
-def nxv32i64 : VTScalableVec<32, i64, 173>;  // n x 32 x i64 vector value
-
-def nxv1f16  : VTScalableVec<1,  f16, 174>;  // n x  1 x  f16 vector value
-def nxv2f16  : VTScalableVec<2,  f16, 175>;  // n x  2 x  f16 vector value
-def nxv4f16  : VTScalableVec<4,  f16, 176>;  // n x  4 x  f16 vector value
-def nxv8f16  : VTScalableVec<8,  f16, 177>;  // n x  8 x  f16 vector value
-def nxv16f16 : VTScalableVec<16, f16, 178>;  // n x 16 x  f16 vector value
-def nxv32f16 : VTScalableVec<32, f16, 179>;  // n x 32 x  f16 vector value
-
-def nxv1bf16  : VTScalableVec<1,  bf16, 180>;  // n x  1 x bf16 vector value
-def nxv2bf16  : VTScalableVec<2,  bf16, 181>;  // n x  2 x bf16 vector value
-def nxv4bf16  : VTScalableVec<4,  bf16, 182>;  // n x  4 x bf16 vector value
-def nxv8bf16  : VTScalableVec<8,  bf16, 183>;  // n x  8 x bf16 vector value
-def nxv16bf16 : VTScalableVec<16, bf16, 184>;  // n x 16 x bf16 vector value
-def nxv32bf16 : VTScalableVec<32, bf16, 185>;  // n x 32 x bf16 vector value
-
-def nxv1f32  : VTScalableVec<1,  f32, 186>;  // n x  1 x  f32 vector value
-def nxv2f32  : VTScalableVec<2,  f32, 187>;  // n x  2 x  f32 vector value
-def nxv4f32  : VTScalableVec<4,  f32, 188>;  // n x  4 x  f32 vector value
-def nxv8f32  : VTScalableVec<8,  f32, 189>;  // n x  8 x  f32 vector value
-def nxv16f32 : VTScalableVec<16, f32, 190>;  // n x 16 x  f32 vector value
-
-def nxv1f64  : VTScalableVec<1,  f64, 191>;  // n x  1 x  f64 vector value
-def nxv2f64  : VTScalableVec<2,  f64, 192>;  // n x  2 x  f64 vector value
-def nxv4f64  : VTScalableVec<4,  f64, 193>;  // n x  4 x  f64 vector value
-def nxv8f64  : VTScalableVec<8,  f64, 194>;  // n x  8 x  f64 vector value
+def v4096i32 : VTVec<4096, i32, 79>;  // 4096 x i32 vector value
+
+def v1i64   : VTVec<1,   i64, 80>;  //   1 x i64 vector value
+def v2i64   : VTVec<2,   i64, 81>;  //   2 x i64 vector value
+def v3i64   : VTVec<3,   i64, 82>;  //   3 x i64 vector value
+def v4i64   : VTVec<4,   i64, 83>;  //   4 x i64 vector value
+def v8i64   : VTVec<8,   i64, 84>;  //   8 x i64 vector value
+def v16i64  : VTVec<16,  i64, 85>;  //  16 x i64 vector value
+def v32i64  : VTVec<32,  i64, 86>;  //  32 x i64 vector value
+def v64i64  : VTVec<64,  i64, 87>;  //  64 x i64 vector value
+def v128i64 : VTVec<128, i64, 88>;  // 128 x i64 vector value
+def v256i64 : VTVec<256, i64, 89>;  // 256 x i64 vector value
+
+def v1i128  : VTVec<1,  i128, 90>;  //  1 x i128 vector value
+
+def v1f16    : VTVec<1,    f16,  91>;  //    1 x f16 vector value
+def v2f16    : VTVec<2,    f16,  92>;  //    2 x f16 vector value
+def v3f16    : VTVec<3,    f16,  93>;  //    3 x f16 vector value
+def v4f16    : VTVec<4,    f16,  94>;  //    4 x f16 vector value
+def v8f16    : VTVec<8,    f16,  95>;  //    8 x f16 vector value
+def v16f16   : VTVec<16,   f16,  96>;  //   16 x f16 vector value
+def v32f16   : VTVec<32,   f16,  97>;  //   32 x f16 vector value
+def v64f16   : VTVec<64,   f16,  98>;  //   64 x f16 vector value
+def v128f16  : VTVec<128,  f16,  99>;  //  128 x f16 vector value
+def v256f16  : VTVec<256,  f16, 100>;  //  256 x f16 vector value
+def v512f16  : VTVec<512,  f16, 101>;  //  512 x f16 vector value
+def v4096f16 : VTVec<4096, f16, 102>;  // 4096 x f16 vector value
+
+def v1bf16    : VTVec<1,    bf16, 103>;  //    1 x bf16 vector value
+def v2bf16    : VTVec<2,    bf16, 104>;  //    2 x bf16 vector value
+def v3bf16    : VTVec<3,    bf16, 105>;  //    3 x bf16 vector value
+def v4bf16    : VTVec<4,    bf16, 106>;  //    4 x bf16 vector value
+def v8bf16    : VTVec<8,    bf16, 107>;  //    8 x bf16 vector value
+def v16bf16   : VTVec<16,   bf16, 108>;  //   16 x bf16 vector value
+def v32bf16   : VTVec<32,   bf16, 109>;  //   32 x bf16 vector value
+def v64bf16   : VTVec<64,   bf16, 110>;  //   64 x bf16 vector value
+def v128bf16  : VTVec<128,  bf16, 111>;  //  128 x bf16 vector value
+def v4096bf16 : VTVec<4096, bf16, 112>;  // 4096 x bf16 vector value
+
+def v1f32    : VTVec<1,    f32, 113>;  //    1 x f32 vector value
+def v2f32    : VTVec<2,    f32, 114>;  //    2 x f32 vector value
+def v3f32    : VTVec<3,    f32, 115>;  //    3 x f32 vector value
+def v4f32    : VTVec<4,    f32, 116>;  //    4 x f32 vector value
+def v5f32    : VTVec<5,    f32, 117>;  //    5 x f32 vector value
+def v6f32    : VTVec<6,    f32, 118>;  //    6 x f32 vector value
+def v7f32    : VTVec<7,    f32, 119>;  //    7 x f32 vector value
+def v8f32    : VTVec<8,    f32, 120>;  //    8 x f32 vector value
+def v9f32    : VTVec<9,    f32, 121>;  //    9 x f32 vector value
+def v10f32   : VTVec<10,   f32, 122>;  //   10 x f32 vector value
+def v11f32   : VTVec<11,   f32, 123>;  //   11 x f32 vector value
+def v12f32   : VTVec<12,   f32, 124>;  //   12 x f32 vector value
+def v16f32   : VTVec<16,   f32, 125>;  //   16 x f32 vector value
+def v32f32   : VTVec<32,   f32, 126>;  //   32 x f32 vector value
+def v64f32   : VTVec<64,   f32, 127>;  //   64 x f32 vector value
+def v128f32  : VTVec<128,  f32, 128>;  //  128 x f32 vector value
+def v256f32  : VTVec<256,  f32, 129>;  //  256 x f32 vector value
+def v512f32  : VTVec<512,  f32, 130>;  //  512 x f32 vector value
+def v1024f32 : VTVec<1024, f32, 131>;  // 1024 x f32 vector value
+def v2048f32 : VTVec<2048, f32, 132>;  // 2048 x f32 vector value
+
+def v1f64    : VTVec<1,    f64, 133>;  //    1 x f64 vector value
+def v2f64    : VTVec<2,    f64, 134>;  //    2 x f64 vector value
+def v3f64    : VTVec<3,    f64, 135>;  //    3 x f64 vector value
+def v4f64    : VTVec<4,    f64, 136>;  //    4 x f64 vector value
+def v8f64    : VTVec<8,    f64, 137>;  //    8 x f64 vector value
+def v16f64   : VTVec<16,   f64, 138>;  //   16 x f64 vector value
+def v32f64   : VTVec<32,   f64, 139>;  //   32 x f64 vector value
+def v64f64   : VTVec<64,   f64, 140>;  //   64 x f64 vector value
+def v128f64  : VTVec<128,  f64, 141>;  //  128 x f64 vector value
+def v256f64  : VTVec<256,  f64, 142>;  //  256 x f64 vector value
+
+def nxv1i1  : VTScalableVec<1,  i1, 143>;  // n x  1 x i1  vector value
+def nxv2i1  : VTScalableVec<2,  i1, 144>;  // n x  2 x i1  vector value
+def nxv4i1  : VTScalableVec<4,  i1, 145>;  // n x  4 x i1  vector value
+def nxv8i1  : VTScalableVec<8,  i1, 146>;  // n x  8 x i1  vector value
+def nxv16i1 : VTScalableVec<16, i1, 147>;  // n x 16 x i1  vector value
+def nxv32i1 : VTScalableVec<32, i1, 148>;  // n x 32 x i1  vector value
+def nxv64i1 : VTScalableVec<64, i1, 149>;  // n x 64 x i1  vector value
+
+def nxv1i8  : VTScalableVec<1,  i8, 150>;  // n x  1 x i8  vector value
+def nxv2i8  : VTScalableVec<2,  i8, 151>;  // n x  2 x i8  vector value
+def nxv4i8  : VTScalableVec<4,  i8, 152>;  // n x  4 x i8  vector value
+def nxv8i8  : VTScalableVec<8,  i8, 153>;  // n x  8 x i8  vector value
+def nxv16i8 : VTScalableVec<16, i8, 154>;  // n x 16 x i8  vector value
+def nxv32i8 : VTScalableVec<32, i8, 155>;  // n x 32 x i8  vector value
+def nxv64i8 : VTScalableVec<64, i8, 156>;  // n x 64 x i8  vector value
+
+def nxv1i16  : VTScalableVec<1,  i16, 157>;  // n x  1 x i16 vector value
+def nxv2i16  : VTScalableVec<2,  i16, 158>;  // n x  2 x i16 vector value
+def nxv4i16  : VTScalableVec<4,  i16, 159>;  // n x  4 x i16 vector value
+def nxv8i16  : VTScalableVec<8,  i16, 160>;  // n x  8 x i16 vector value
+def nxv16i16 : VTScalableVec<16, i16, 161>;  // n x 16 x i16 vector value
+def nxv32i16 : VTScalableVec<32, i16, 162>;  // n x 32 x i16 vector value
+
+def nxv1i32  : VTScalableVec<1,  i32, 163>;  // n x  1 x i32 vector value
+def nxv2i32  : VTScalableVec<2,  i32, 164>;  // n x  2 x i32 vector value
+def nxv4i32  : VTScalableVec<4,  i32, 165>;  // n x  4 x i32 vector value
+def nxv8i32  : VTScalableVec<8,  i32, 166>;  // n x  8 x i32 vector value
+def nxv16i32 : VTScalableVec<16, i32, 167>;  // n x 16 x i32 vector value
+def nxv32i32 : VTScalableVec<32, i32, 168>;  // n x 32 x i32 vector value
+
+def nxv1i64  : VTScalableVec<1,  i64, 169>;  // n x  1 x i64 vector value
+def nxv2i64  : VTScalableVec<2,  i64, 170>;  // n x  2 x i64 vector value
+def nxv4i64  : VTScalableVec<4,  i64, 171>;  // n x  4 x i64 vector value
+def nxv8i64  : VTScalableVec<8,  i64, 172>;  // n x  8 x i64 vector value
+def nxv16i64 : VTScalableVec<16, i64, 173>;  // n x 16 x i64 vector value
+def nxv32i64 : VTScalableVec<32, i64, 174>;  // n x 32 x i64 vector value
+
+def nxv1f16  : VTScalableVec<1,  f16, 175>;  // n x  1 x  f16 vector value
+def nxv2f16  : VTScalableVec<2,  f16, 176>;  // n x  2 x  f16 vector value
+def nxv4f16  : VTScalableVec<4,  f16, 177>;  // n x  4 x  f16 vector value
+def nxv8f16  : VTScalableVec<8,  f16, 178>;  // n x  8 x  f16 vector value
+def nxv16f16 : VTScalableVec<16, f16, 179>;  // n x 16 x  f16 vector value
+def nxv32f16 : VTScalableVec<32, f16, 180>;  // n x 32 x  f16 vector value
+
+def nxv1bf16  : VTScalableVec<1,  bf16, 181>;  // n x  1 x bf16 vector value
+def nxv2bf16  : VTScalableVec<2,  bf16, 182>;  // n x  2 x bf16 vector value
+def nxv4bf16  : VTScalableVec<4,  bf16, 183>;  // n x  4 x bf16 vector value
+def nxv8bf16  : VTScalableVec<8,  bf16, 184>;  // n x  8 x bf16 vector value
+def nxv16bf16 : VTScalableVec<16, bf16, 185>;  // n x 16 x bf16 vector value
+def nxv32bf16 : VTScalableVec<32, bf16, 186>;  // n x 32 x bf16 vector value
+
+def nxv1f32  : VTScalableVec<1,  f32, 187>;  // n x  1 x  f32 vector value
+def nxv2f32  : VTScalableVec<2,  f32, 188>;  // n x  2 x  f32 vector value
+def nxv4f32  : VTScalableVec<4,  f32, 189>;  // n x  4 x  f32 vector value
+def nxv8f32  : VTScalableVec<8,  f32, 190>;  // n x  8 x  f32 vector value
+def nxv16f32 : VTScalableVec<16, f32, 191>;  // n x 16 x  f32 vector value
+
+def nxv1f64  : VTScalableVec<1,  f64, 192>;  // n x  1 x  f64 vector value
+def nxv2f64  : VTScalableVec<2,  f64, 193>;  // n x  2 x  f64 vector value
+def nxv4f64  : VTScalableVec<4,  f64, 194>;  // n x  4 x  f64 vector value
+def nxv8f64  : VTScalableVec<8,  f64, 195>;  // n x  8 x  f64 vector value
 
 // Sz = NF * MinNumElts * 8(bits)
-def riscv_nxv1i8x2   : VTVecTup<16,  2, i8, 195>;  // RISCV vector tuple(min_num_elts=1,  nf=2)
-def riscv_nxv1i8x3   : VTVecTup<24,  3, i8, 196>;  // RISCV vector tuple(min_num_elts=1,  nf=3)
-def riscv_nxv1i8x4   : VTVecTup<32,  4, i8, 197>;  // RISCV vector tuple(min_num_elts=1,  nf=4)
-def riscv_nxv1i8x5   : VTVecTup<40,  5, i8, 198>;  // RISCV vector tuple(min_num_elts=1,  nf=5)
-def riscv_nxv1i8x6   : VTVecTup<48,  6, i8, 199>;  // RISCV vector tuple(min_num_elts=1,  nf=6)
-def riscv_nxv1i8x7   : VTVecTup<56,  7, i8, 200>;  // RISCV vector tuple(min_num_elts=1,  nf=7)
-def riscv_nxv1i8x8   : VTVecTup<64,  8, i8, 201>;  // RISCV vector tuple(min_num_elts=1,  nf=8)
-def riscv_nxv2i8x2   : VTVecTup<32,  2, i8, 202>;  // RISCV vector tuple(min_num_elts=2,  nf=2)
-def riscv_nxv2i8x3   : VTVecTup<48,  3, i8, 203>;  // RISCV vector tuple(min_num_elts=2,  nf=3)
-def riscv_nxv2i8x4   : VTVecTup<64,  4, i8, 204>;  // RISCV vector tuple(min_num_elts=2,  nf=4)
-def riscv_nxv2i8x5   : VTVecTup<80,  5, i8, 205>;  // RISCV vector tuple(min_num_elts=2,  nf=5)
-def riscv_nxv2i8x6   : VTVecTup<96,  6, i8, 206>;  // RISCV vector tuple(min_num_elts=2,  nf=6)
-def riscv_nxv2i8x7   : VTVecTup<112, 7, i8, 207>;  // RISCV vector tuple(min_num_elts=2,  nf=7)
-def riscv_nxv2i8x8   : VTVecTup<128, 8, i8, 208>;  // RISCV vector tuple(min_num_elts=2,  nf=8)
-def riscv_nxv4i8x2   : VTVecTup<64,  2, i8, 209>;  // RISCV vector tuple(min_num_elts=4,  nf=2)
-def riscv_nxv4i8x3   : VTVecTup<96,  3, i8, 210>;  // RISCV vector tuple(min_num_elts=4,  nf=3)
-def riscv_nxv4i8x4   : VTVecTup<128, 4, i8, 211>;  // RISCV vector tuple(min_num_elts=4,  nf=4)
-def riscv_nxv4i8x5   : VTVecTup<160, 5, i8, 212>;  // RISCV vector tuple(min_num_elts=4,  nf=5)
-def riscv_nxv4i8x6   : VTVecTup<192, 6, i8, 213>;  // RISCV vector tuple(min_num_elts=4,  nf=6)
-def riscv_nxv4i8x7   : VTVecTup<224, 7, i8, 214>;  // RISCV vector tuple(min_num_elts=4,  nf=7)
-def riscv_nxv4i8x8   : VTVecTup<256, 8, i8, 215>;  // RISCV vector tuple(min_num_elts=4,  nf=8)
-def riscv_nxv8i8x2   : VTVecTup<128, 2, i8, 216>;  // RISCV vector tuple(min_num_elts=8,  nf=2)
-def riscv_nxv8i8x3   : VTVecTup<192, 3, i8, 217>;  // RISCV vector tuple(min_num_elts=8,  nf=3)
-def riscv_nxv8i8x4   : VTVecTup<256, 4, i8, 218>;  // RISCV vector tuple(min_num_elts=8,  nf=4)
-def riscv_nxv8i8x5   : VTVecTup<320, 5, i8, 219>;  // RISCV vector tuple(min_num_elts=8,  nf=5)
-def riscv_nxv8i8x6   : VTVecTup<384, 6, i8, 220>;  // RISCV vector tuple(min_num_elts=8,  nf=6)
-def riscv_nxv8i8x7   : VTVecTup<448, 7, i8, 221>;  // RISCV vector tuple(min_num_elts=8,  nf=7)
-def riscv_nxv8i8x8   : VTVecTup<512, 8, i8, 222>;  // RISCV vector tuple(min_num_elts=8,  nf=8)
-def riscv_nxv16i8x2  : VTVecTup<256, 2, i8, 223>;  // RISCV vector tuple(min_num_elts=16, nf=2)
-def riscv_nxv16i8x3  : VTVecTup<384, 3, i8, 224>;  // RISCV vector tuple(min_num_elts=16, nf=3)
-def riscv_nxv16i8x4  : VTVecTup<512, 4, i8, 225>;  // RISCV vector tuple(min_num_elts=16, nf=4)
-def riscv_nxv32i8x2  : VTVecTup<512, 2, i8, 226>;  // RISCV vector tuple(min_num_elts=32, nf=2)
-
-def x86mmx    : ValueType<64,   227>;  // X86 MMX value
-def Glue      : ValueType<0,    228>;  // Pre-RA sched glue
-def isVoid    : ValueType<0,    229>;  // Produces no value
-def untyped   : ValueType<8,    230> { // Produces an untyped value
+def riscv_nxv1i8x2   : VTVecTup<16,  2, i8, 196>;  // RISCV vector tuple(min_num_elts=1,  nf=2)
+def riscv_nxv1i8x3   : VTVecTup<24,  3, i8, 197>;  // RISCV vector tuple(min_num_elts=1,  nf=3)
+def riscv_nxv1i8x4   : VTVecTup<32,  4, i8, 198>;  // RISCV vector tuple(min_num_elts=1,  nf=4)
+def riscv_nxv1i8x5   : VTVecTup<40,  5, i8, 199>;  // RISCV vector tuple(min_num_elts=1,  nf=5)
+def riscv_nxv1i8x6   : VTVecTup<48,  6, i8, 200>;  // RISCV vector tuple(min_num_elts=1,  nf=6)
+def riscv_nxv1i8x7   : VTVecTup<56,  7, i8, 201>;  // RISCV vector tuple(min_num_elts=1,  nf=7)
+def riscv_nxv1i8x8   : VTVecTup<64,  8, i8, 202>;  // RISCV vector tuple(min_num_elts=1,  nf=8)
+def riscv_nxv2i8x2   : VTVecTup<32,  2, i8, 203>;  // RISCV vector tuple(min_num_elts=2,  nf=2)
+def riscv_nxv2i8x3   : VTVecTup<48,  3, i8, 204>;  // RISCV vector tuple(min_num_elts=2,  nf=3)
+def riscv_nxv2i8x4   : VTVecTup<64,  4, i8, 205>;  // RISCV vector tuple(min_num_elts=2,  nf=4)
+def riscv_nxv2i8x5   : VTVecTup<80,  5, i8, 206>;  // RISCV vector tuple(min_num_elts=2,  nf=5)
+def riscv_nxv2i8x6   : VTVecTup<96,  6, i8, 207>;  // RISCV vector tuple(min_num_elts=2,  nf=6)
+def riscv_nxv2i8x7   : VTVecTup<112, 7, i8, 208>;  // RISCV vector tuple(min_num_elts=2,  nf=7)
+def riscv_nxv2i8x8   : VTVecTup<128, 8, i8, 209>;  // RISCV vector tuple(min_num_elts=2,  nf=8)
+def riscv_nxv4i8x2   : VTVecTup<64,  2, i8, 210>;  // RISCV vector tuple(min_num_elts=4,  nf=2)
+def riscv_nxv4i8x3   : VTVecTup<96,  3, i8, 211>;  // RISCV vector tuple(min_num_elts=4,  nf=3)
+def riscv_nxv4i8x4   : VTVecTup<128, 4, i8, 212>;  // RISCV vector tuple(min_num_elts=4,  nf=4)
+def riscv_nxv4i8x5   : VTVecTup<160, 5, i8, 213>;  // RISCV vector tuple(min_num_elts=4,  nf=5)
+def riscv_nxv4i8x6   : VTVecTup<192, 6, i8, 214>;  // RISCV vector tuple(min_num_elts=4,  nf=6)
+def riscv_nxv4i8x7   : VTVecTup<224, 7, i8, 215>;  // RISCV vector tuple(min_num_elts=4,  nf=7)
+def riscv_nxv4i8x8   : VTVecTup<256, 8, i8, 216>;  // RISCV vector tuple(min_num_elts=4,  nf=8)
+def riscv_nxv8i8x2   : VTVecTup<128, 2, i8, 217>;  // RISCV vector tuple(min_num_elts=8,  nf=2)
+def riscv_nxv8i8x3   : VTVecTup<192, 3, i8, 218>;  // RISCV vector tuple(min_num_elts=8,  nf=3)
+def riscv_nxv8i8x4   : VTVecTup<256, 4, i8, 219>;  // RISCV vector tuple(min_num_elts=8,  nf=4)
+def riscv_nxv8i8x5   : VTVecTup<320, 5, i8, 220>;  // RISCV vector tuple(min_num_elts=8,  nf=5)
+def riscv_nxv8i8x6   : VTVecTup<384, 6, i8, 221>;  // RISCV vector tuple(min_num_elts=8,  nf=6)
+def riscv_nxv8i8x7   : VTVecTup<448, 7, i8, 222>;  // RISCV vector tuple(min_num_elts=8,  nf=7)
+def riscv_nxv8i8x8   : VTVecTup<512, 8, i8, 223>;  // RISCV vector tuple(min_num_elts=8,  nf=8)
+def riscv_nxv16i8x2  : VTVecTup<256, 2, i8, 224>;  // RISCV vector tuple(min_num_elts=16, nf=2)
+def riscv_nxv16i8x3  : VTVecTup<384, 3, i8, 225>;  // RISCV vector tuple(min_num_elts=16, nf=3)
+def riscv_nxv16i8x4  : VTVecTup<512, 4, i8, 226>;  // RISCV vector tuple(min_num_elts=16, nf=4)
+def riscv_nxv32i8x2  : VTVecTup<512, 2, i8, 227>;  // RISCV vector tuple(min_num_elts=32, nf=2)
+
+def x86mmx    : ValueType<64,   228>;  // X86 MMX value
+def Glue      : ValueType<0,    229>;  // Pre-RA sched glue
+def isVoid    : ValueType<0,    230>;  // Produces no value
+def untyped   : ValueType<8,    231> { // Produces an untyped value
   let LLVMName = "Untyped";
 }
-def funcref   : ValueType<0,    231>;  // WebAssembly's funcref type
-def externref : ValueType<0,    232>;  // WebAssembly's externref type
-def exnref    : ValueType<0,    233>;  // WebAssembly's exnref type
-def x86amx    : ValueType<8192, 234>;  // X86 AMX value
-def i64x8     : ValueType<512,  235>;  // 8 Consecutive GPRs (AArch64)
+def funcref   : ValueType<0,    232>;  // WebAssembly's funcref type
+def externref : ValueType<0,    233>;  // WebAssembly's externref type
+def exnref    : ValueType<0,    234>;  // WebAssembly's exnref type
+def x86amx    : ValueType<8192, 235>;  // X86 AMX value
+def i64x8     : ValueType<512,  236>;  // 8 Consecutive GPRs (AArch64)
 def aarch64svcount
-              : ValueType<16,  236>;  // AArch64 predicate-as-counter
-def spirvbuiltin : ValueType<0, 237>; // SPIR-V's builtin type
+              : ValueType<16,  237>;  // AArch64 predicate-as-counter
+def spirvbuiltin : ValueType<0, 238>; // SPIR-V's builtin type
 // AMDGPU buffer fat pointer, buffer rsrc + offset, rewritten before MIR translation.
 // FIXME: Remove this and the getPointerType() override if MVT::i160 is added.
-def amdgpuBufferFatPointer : ValueType<160, 238>;
+def amdgpuBufferFatPointer : ValueType<160, 239>;
 // AMDGPU buffer strided pointer, buffer rsrc + index + offset, doesn't reach MIR.
 // FIXME: Remove this and the getPointerType() override if MVT::i82 is added.
-def amdgpuBufferStridedPointer : ValueType<192, 239>;
+def amdgpuBufferStridedPointer : ValueType<192, 240>;
 
-def aarch64mfp8 : ValueType<8,  240>;  // 8-bit value in FPR (AArch64)
+def aarch64mfp8 : ValueType<8,  241>;  // 8-bit value in FPR (AArch64)
 
 let isNormalValueType = false in {
 def token      : ValueType<0, 504>;  // TokenTy
diff --git a/external/llvm-project/llvm/include/llvm/Config/llvm-config.h.cmake b/external/llvm-project/llvm/include/llvm/Config/llvm-config.h.cmake
index 1fb3f7cd0257..749aeffe7584 100644
--- a/external/llvm-project/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/external/llvm-project/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -17,7 +17,7 @@
 /* The number of commits in the linear history from the
  * start of the universe up to the latest llvm main commit
  * that has been merged */
-#define LLVM_MAIN_REVISION 539334
+#define LLVM_MAIN_REVISION 540910
 
 /* Define if LLVM_ENABLE_DUMP is enabled */
 #cmakedefine LLVM_ENABLE_DUMP
@@ -136,6 +136,10 @@
 
 /* Define to 1 to enable expensive checks for debug location coverage checking,
    and to 0 otherwise. */
-#cmakedefine01 LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#cmakedefine01 LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
+
+/* Define to 1 to enable expensive tracking of the origin of debug location
+   coverage bugs, and to 0 otherwise. */
+#cmakedefine01 LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFContext.h b/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFContext.h
index e3f7f57749ff..ce2295a8711b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFContext.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFContext.h
@@ -16,10 +16,11 @@
 
 #include "llvm/DebugInfo/BTF/BTFParser.h"
 #include "llvm/DebugInfo/DIContext.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
-class BTFContext final : public DIContext {
+class LLVM_ABI BTFContext final : public DIContext {
   BTFParser BTF;
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFParser.h b/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFParser.h
index 53f39919950e..f8b5b29738b3 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFParser.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/BTF/BTFParser.h
@@ -25,6 +25,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/DebugInfo/BTF/BTF.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 
 namespace llvm {
@@ -74,32 +75,33 @@ class BTFParser {
 public:
   // Looks-up a string in the .BTF section's string table.
   // Offset is relative to string table start.
-  StringRef findString(uint32_t Offset) const;
+  LLVM_ABI StringRef findString(uint32_t Offset) const;
 
   // Search for line information for a specific address,
   // address match is exact (contrary to DWARFContext).
   // Return nullptr if no information found.
   // If information is present, return a pointer to object
   // owned by this class.
-  const BTF::BPFLineInfo *findLineInfo(SectionedAddress Address) const;
+  LLVM_ABI const BTF::BPFLineInfo *findLineInfo(SectionedAddress Address) const;
 
   // Search for CO-RE relocation information for a specific address.
   // Return nullptr if no information found.
   // If information is present, return a pointer to object
   // owned by this class.
-  const BTF::BPFFieldReloc *findFieldReloc(SectionedAddress Address) const;
+  LLVM_ABI const BTF::BPFFieldReloc *
+  findFieldReloc(SectionedAddress Address) const;
 
   // Return a human readable representation of the CO-RE relocation
   // record, this is for display purpose only.
   // See implementation for details.
-  void symbolize(const BTF::BPFFieldReloc *Reloc,
-                 SmallVectorImpl<char> &Result) const;
+  LLVM_ABI void symbolize(const BTF::BPFFieldReloc *Reloc,
+                          SmallVectorImpl<char> &Result) const;
 
   // Lookup BTF type definition with a specific index.
   // Return nullptr if no information found.
   // If information is present, return a pointer to object
   // owned by this class.
-  const BTF::CommonType *findType(uint32_t Id) const;
+  LLVM_ABI const BTF::CommonType *findType(uint32_t Id) const;
 
   // Return total number of known BTF types.
   size_t typesCount() const { return Types.size(); }
@@ -120,11 +122,11 @@ class BTFParser {
   // - state of the BTFParser might be incomplete but is not invalid,
   //   queries might be run against it, but some (or all) information
   //   might be unavailable;
-  Error parse(const ObjectFile &Obj, const ParseOptions &Opts);
+  LLVM_ABI Error parse(const ObjectFile &Obj, const ParseOptions &Opts);
   Error parse(const ObjectFile &Obj) { return parse(Obj, {true, true, true}); }
 
   // Return true if `Obj` has .BTF and .BTF.ext sections.
-  static bool hasBTFSections(const ObjectFile &Obj);
+  LLVM_ABI static bool hasBTFSections(const ObjectFile &Obj);
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h
index fa2277343d5d..3a36863ac7e3 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h
@@ -16,6 +16,7 @@
 #include "llvm/DebugInfo/CodeView/TypeCollection.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -23,7 +24,7 @@ namespace codeview {
 
 class ContinuationRecordBuilder;
 
-class AppendingTypeTableBuilder : public TypeCollection {
+class LLVM_ABI AppendingTypeTableBuilder : public TypeCollection {
 
   BumpPtrAllocator &RecordStorage;
   SimpleTypeSerializer SimpleSerializer;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h
index c629018fd109..7b230efe695a 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_CODEVIEW_CVSYMBOLVISITOR_H
 
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -24,14 +25,15 @@ class CVSymbolVisitor {
     std::optional<uint32_t> ChildRecursiveDepth;
   };
 
-  CVSymbolVisitor(SymbolVisitorCallbacks &Callbacks);
+  LLVM_ABI CVSymbolVisitor(SymbolVisitorCallbacks &Callbacks);
 
-  Error visitSymbolRecord(CVSymbol &Record);
-  Error visitSymbolRecord(CVSymbol &Record, uint32_t Offset);
-  Error visitSymbolStream(const CVSymbolArray &Symbols);
-  Error visitSymbolStream(const CVSymbolArray &Symbols, uint32_t InitialOffset);
-  Error visitSymbolStreamFiltered(const CVSymbolArray &Symbols,
-                                  const FilterOptions &Filter);
+  LLVM_ABI Error visitSymbolRecord(CVSymbol &Record);
+  LLVM_ABI Error visitSymbolRecord(CVSymbol &Record, uint32_t Offset);
+  LLVM_ABI Error visitSymbolStream(const CVSymbolArray &Symbols);
+  LLVM_ABI Error visitSymbolStream(const CVSymbolArray &Symbols,
+                                   uint32_t InitialOffset);
+  LLVM_ABI Error visitSymbolStreamFiltered(const CVSymbolArray &Symbols,
+                                           const FilterOptions &Filter);
 
 private:
   SymbolVisitorCallbacks &Callbacks;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h
index 7780e233cab3..c80cf2d6801a 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -30,24 +31,28 @@ enum VisitorDataSource {
                     // supply the bytes.
 };
 
-Error visitTypeRecord(CVType &Record, TypeIndex Index,
-                      TypeVisitorCallbacks &Callbacks,
-                      VisitorDataSource Source = VDS_BytesPresent);
-Error visitTypeRecord(CVType &Record, TypeVisitorCallbacks &Callbacks,
-                      VisitorDataSource Source = VDS_BytesPresent);
-
-Error visitMemberRecord(CVMemberRecord Record, TypeVisitorCallbacks &Callbacks,
-                        VisitorDataSource Source = VDS_BytesPresent);
-Error visitMemberRecord(TypeLeafKind Kind, ArrayRef<uint8_t> Record,
-                        TypeVisitorCallbacks &Callbacks);
-
-Error visitMemberRecordStream(ArrayRef<uint8_t> FieldList,
-                              TypeVisitorCallbacks &Callbacks);
-
-Error visitTypeStream(const CVTypeArray &Types, TypeVisitorCallbacks &Callbacks,
-                      VisitorDataSource Source = VDS_BytesPresent);
-Error visitTypeStream(CVTypeRange Types, TypeVisitorCallbacks &Callbacks);
-Error visitTypeStream(TypeCollection &Types, TypeVisitorCallbacks &Callbacks);
+LLVM_ABI Error visitTypeRecord(CVType &Record, TypeIndex Index,
+                               TypeVisitorCallbacks &Callbacks,
+                               VisitorDataSource Source = VDS_BytesPresent);
+LLVM_ABI Error visitTypeRecord(CVType &Record, TypeVisitorCallbacks &Callbacks,
+                               VisitorDataSource Source = VDS_BytesPresent);
+
+LLVM_ABI Error visitMemberRecord(CVMemberRecord Record,
+                                 TypeVisitorCallbacks &Callbacks,
+                                 VisitorDataSource Source = VDS_BytesPresent);
+LLVM_ABI Error visitMemberRecord(TypeLeafKind Kind, ArrayRef<uint8_t> Record,
+                                 TypeVisitorCallbacks &Callbacks);
+
+LLVM_ABI Error visitMemberRecordStream(ArrayRef<uint8_t> FieldList,
+                                       TypeVisitorCallbacks &Callbacks);
+
+LLVM_ABI Error visitTypeStream(const CVTypeArray &Types,
+                               TypeVisitorCallbacks &Callbacks,
+                               VisitorDataSource Source = VDS_BytesPresent);
+LLVM_ABI Error visitTypeStream(CVTypeRange Types,
+                               TypeVisitorCallbacks &Callbacks);
+LLVM_ABI Error visitTypeStream(TypeCollection &Types,
+                               TypeVisitorCallbacks &Callbacks);
 
 } // end namespace codeview
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeView.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeView.h
index 5cdff5ff0e82..b769e53d8027 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeView.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeView.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_DEBUGINFO_CODEVIEW_CODEVIEW_H
 #define LLVM_DEBUGINFO_CODEVIEW_CODEVIEW_H
 
+#include "llvm/Support/Compiler.h"
 #include <cinttypes>
 #include <type_traits>
 
@@ -144,36 +145,8 @@ enum class CPUType : uint16_t {
 /// Debug Interface Access SDK, and are documented here:
 /// https://learn.microsoft.com/en-us/visualstudio/debugger/debug-interface-access/cv-cfl-lang
 enum SourceLanguage : uint8_t {
-  C = 0x00,
-  Cpp = 0x01,
-  Fortran = 0x02,
-  Masm = 0x03,
-  Pascal = 0x04,
-  Basic = 0x05,
-  Cobol = 0x06,
-  Link = 0x07,
-  Cvtres = 0x08,
-  Cvtpgd = 0x09,
-  CSharp = 0x0a,
-  VB = 0x0b,
-  ILAsm = 0x0c,
-  Java = 0x0d,
-  JScript = 0x0e,
-  MSIL = 0x0f,
-  HLSL = 0x10,
-  ObjC = 0x11,
-  ObjCpp = 0x12,
-  Swift = 0x13,
-  AliasObj = 0x14,
-  Rust = 0x15,
-  Go = 0x16,
-
-  /// The DMD compiler emits 'D' for the CV source language. Microsoft does not
-  /// have an enumerator for it yet.
-  D = 'D',
-  /// The Swift compiler used to emit 'S' for the CV source language, but
-  /// current versions emit the enumerator defined above.
-  OldSwift = 'S',
+#define CV_LANGUAGE(NAME, ID) NAME = ID,
+#include "CodeViewLanguages.def"
 };
 
 /// These values correspond to the CV_call_e enumeration, and are documented
@@ -555,9 +528,10 @@ enum class EncodedFramePtrReg : uint8_t {
   BasePtr = 3,
 };
 
-RegisterId decodeFramePtrReg(EncodedFramePtrReg EncodedReg, CPUType CPU);
+LLVM_ABI RegisterId decodeFramePtrReg(EncodedFramePtrReg EncodedReg,
+                                      CPUType CPU);
 
-EncodedFramePtrReg encodeFramePtrReg(RegisterId Reg, CPUType CPU);
+LLVM_ABI EncodedFramePtrReg encodeFramePtrReg(RegisterId Reg, CPUType CPU);
 
 /// These values correspond to the THUNK_ORDINAL enumeration.
 enum class ThunkOrdinal : uint8_t {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h
index 17eb06c242a9..bb73a95f9ead 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewError.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_CODEVIEW_CODEVIEWERROR_H
 #define LLVM_DEBUGINFO_CODEVIEW_CODEVIEWERROR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -31,7 +32,7 @@ struct is_error_code_enum<llvm::codeview::cv_error_code> : std::true_type {};
 
 namespace llvm {
 namespace codeview {
-const std::error_category &CVErrorCategory();
+LLVM_ABI const std::error_category &CVErrorCategory();
 
 inline std::error_code make_error_code(cv_error_code E) {
   return std::error_code(static_cast<int>(E), CVErrorCategory());
@@ -43,7 +44,7 @@ class CodeViewError : public ErrorInfo<CodeViewError, StringError> {
   using ErrorInfo<CodeViewError,
                   StringError>::ErrorInfo; // inherit constructors
   CodeViewError(const Twine &S) : ErrorInfo(S, cv_error_code::unspecified) {}
-  static char ID;
+  LLVM_ABI static char ID;
 };
 
 } // namespace codeview
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewLanguages.def b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewLanguages.def
new file mode 100644
index 000000000000..5c6335fffd4d
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewLanguages.def
@@ -0,0 +1,51 @@
+//===-- CodeViewLanguages.def - All CodeView languages ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// These values correspond to the CV_CFL_LANG enumeration in the Microsoft
+// Debug Interface Access SDK, and are documented here:
+// https://learn.microsoft.com/en-us/visualstudio/debugger/debug-interface-access/cv-cfl-lang
+// This should match the constants there.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CV_LANGUAGE
+#define CV_LANGUAGE(NAME, ID)
+#endif
+
+CV_LANGUAGE(C, 0x00)
+CV_LANGUAGE(Cpp, 0x01)
+CV_LANGUAGE(Fortran, 0x02)
+CV_LANGUAGE(Masm, 0x03)
+CV_LANGUAGE(Pascal, 0x04)
+CV_LANGUAGE(Basic, 0x05)
+CV_LANGUAGE(Cobol, 0x06)
+CV_LANGUAGE(Link, 0x07)
+CV_LANGUAGE(Cvtres, 0x08)
+CV_LANGUAGE(Cvtpgd, 0x09)
+CV_LANGUAGE(CSharp, 0x0a)
+CV_LANGUAGE(VB, 0x0b)
+CV_LANGUAGE(ILAsm, 0x0c)
+CV_LANGUAGE(Java, 0x0d)
+CV_LANGUAGE(JScript, 0x0e)
+CV_LANGUAGE(MSIL, 0x0f)
+CV_LANGUAGE(HLSL, 0x10)
+CV_LANGUAGE(ObjC, 0x11)
+CV_LANGUAGE(ObjCpp, 0x12)
+CV_LANGUAGE(Swift, 0x13)
+CV_LANGUAGE(AliasObj, 0x14)
+CV_LANGUAGE(Rust, 0x15)
+CV_LANGUAGE(Go, 0x16)
+
+// The DMD compiler emits 'D' for the CV source language. Microsoft does not
+// have an enumerator for it yet.
+CV_LANGUAGE(D, 'D')
+// The Swift compiler used to emit 'S' for the CV source language, but
+// current versions emit the enumerator defined above.
+CV_LANGUAGE(OldSwift, 'S')
+
+#undef CV_LANGUAGE
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
index 29ba0c3eb785..668428e487e1 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
@@ -14,6 +14,7 @@
 #include "llvm/DebugInfo/CodeView/CodeViewError.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cassert>
 #include <cstdint>
@@ -61,10 +62,10 @@ class CodeViewRecordIO {
   explicit CodeViewRecordIO(CodeViewRecordStreamer &Streamer)
       : Streamer(&Streamer) {}
 
-  Error beginRecord(std::optional<uint32_t> MaxLength);
-  Error endRecord();
+  LLVM_ABI Error beginRecord(std::optional<uint32_t> MaxLength);
+  LLVM_ABI Error endRecord();
 
-  Error mapInteger(TypeIndex &TypeInd, const Twine &Comment = "");
+  LLVM_ABI Error mapInteger(TypeIndex &TypeInd, const Twine &Comment = "");
 
   bool isStreaming() const {
     return (Streamer != nullptr) && (Reader == nullptr) && (Writer == nullptr);
@@ -76,7 +77,7 @@ class CodeViewRecordIO {
     return (Writer != nullptr) && (Streamer == nullptr) && (Reader == nullptr);
   }
 
-  uint32_t maxFieldLength() const;
+  LLVM_ABI uint32_t maxFieldLength() const;
 
   template <typename T> Error mapObject(T &Value) {
     if (isStreaming()) {
@@ -130,14 +131,14 @@ class CodeViewRecordIO {
     return Error::success();
   }
 
-  Error mapEncodedInteger(int64_t &Value, const Twine &Comment = "");
-  Error mapEncodedInteger(uint64_t &Value, const Twine &Comment = "");
-  Error mapEncodedInteger(APSInt &Value, const Twine &Comment = "");
-  Error mapStringZ(StringRef &Value, const Twine &Comment = "");
-  Error mapGuid(GUID &Guid, const Twine &Comment = "");
+  LLVM_ABI Error mapEncodedInteger(int64_t &Value, const Twine &Comment = "");
+  LLVM_ABI Error mapEncodedInteger(uint64_t &Value, const Twine &Comment = "");
+  LLVM_ABI Error mapEncodedInteger(APSInt &Value, const Twine &Comment = "");
+  LLVM_ABI Error mapStringZ(StringRef &Value, const Twine &Comment = "");
+  LLVM_ABI Error mapGuid(GUID &Guid, const Twine &Comment = "");
 
-  Error mapStringZVectorZ(std::vector<StringRef> &Value,
-                          const Twine &Comment = "");
+  LLVM_ABI Error mapStringZVectorZ(std::vector<StringRef> &Value,
+                                   const Twine &Comment = "");
 
   template <typename SizeType, typename T, typename ElementMapper>
   Error mapVectorN(T &Items, const ElementMapper &Mapper,
@@ -197,12 +198,13 @@ class CodeViewRecordIO {
     return Error::success();
   }
 
-  Error mapByteVectorTail(ArrayRef<uint8_t> &Bytes, const Twine &Comment = "");
-  Error mapByteVectorTail(std::vector<uint8_t> &Bytes,
-                          const Twine &Comment = "");
+  LLVM_ABI Error mapByteVectorTail(ArrayRef<uint8_t> &Bytes,
+                                   const Twine &Comment = "");
+  LLVM_ABI Error mapByteVectorTail(std::vector<uint8_t> &Bytes,
+                                   const Twine &Comment = "");
 
-  Error padToAlignment(uint32_t Align);
-  Error skipPadding();
+  LLVM_ABI Error padToAlignment(uint32_t Align);
+  LLVM_ABI Error skipPadding();
 
   uint64_t getStreamedLen() {
     if (isStreaming())
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h
index 84cef520a2f4..805cedf1a879 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/CodeView/TypeRecordMapping.h"
 #include "llvm/Support/BinaryByteStream.h"
 #include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <vector>
 
@@ -38,17 +39,17 @@ class ContinuationRecordBuilder {
                              std::optional<TypeIndex> RefersTo);
 
 public:
-  ContinuationRecordBuilder();
-  ~ContinuationRecordBuilder();
+  LLVM_ABI ContinuationRecordBuilder();
+  LLVM_ABI ~ContinuationRecordBuilder();
 
-  void begin(ContinuationRecordKind RecordKind);
+  LLVM_ABI void begin(ContinuationRecordKind RecordKind);
 
   // This template is explicitly instantiated in the implementation file for all
   // supported types.  The method itself is ugly, so inlining it into the header
   // file clutters an otherwise straightforward interface.
   template <typename RecordType> void writeMemberType(RecordType &Record);
 
-  std::vector<CVType> end(TypeIndex Index);
+  LLVM_ABI std::vector<CVType> end(TypeIndex Index);
 };
 } // namespace codeview
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
index 615fd216e655..8f6b413ad8ae 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
@@ -17,6 +17,7 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <vector>
@@ -42,8 +43,8 @@ template <> struct VarStreamArrayExtractor<codeview::FileChecksumEntry> {
 public:
   using ContextType = void;
 
-  Error operator()(BinaryStreamRef Stream, uint32_t &Len,
-                   codeview::FileChecksumEntry &Item);
+  LLVM_ABI Error operator()(BinaryStreamRef Stream, uint32_t &Len,
+                            codeview::FileChecksumEntry &Item);
 };
 
 namespace codeview {
@@ -62,8 +63,8 @@ class DebugChecksumsSubsectionRef final : public DebugSubsectionRef {
 
   bool valid() const { return Checksums.valid(); }
 
-  Error initialize(BinaryStreamReader Reader);
-  Error initialize(BinaryStreamRef Stream);
+  LLVM_ABI Error initialize(BinaryStreamReader Reader);
+  LLVM_ABI Error initialize(BinaryStreamRef Stream);
 
   Iterator begin() const { return Checksums.begin(); }
   Iterator end() const { return Checksums.end(); }
@@ -74,7 +75,7 @@ class DebugChecksumsSubsectionRef final : public DebugSubsectionRef {
   FileChecksumArray Checksums;
 };
 
-class DebugChecksumsSubsection final : public DebugSubsection {
+class LLVM_ABI DebugChecksumsSubsection final : public DebugSubsection {
 public:
   explicit DebugChecksumsSubsection(DebugStringTableSubsection &Strings);
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h
index e21873a3af8f..d0e391779abe 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h
@@ -13,6 +13,7 @@
 #include "llvm/DebugInfo/CodeView/DebugSubsection.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <map>
@@ -34,8 +35,8 @@ class DebugCrossModuleExportsSubsectionRef final : public DebugSubsectionRef {
     return S->kind() == DebugSubsectionKind::CrossScopeExports;
   }
 
-  Error initialize(BinaryStreamReader Reader);
-  Error initialize(BinaryStreamRef Stream);
+  LLVM_ABI Error initialize(BinaryStreamReader Reader);
+  LLVM_ABI Error initialize(BinaryStreamRef Stream);
 
   Iterator begin() const { return References.begin(); }
   Iterator end() const { return References.end(); }
@@ -44,7 +45,8 @@ class DebugCrossModuleExportsSubsectionRef final : public DebugSubsectionRef {
   FixedStreamArray<CrossModuleExport> References;
 };
 
-class DebugCrossModuleExportsSubsection final : public DebugSubsection {
+class LLVM_ABI DebugCrossModuleExportsSubsection final
+    : public DebugSubsection {
 public:
   DebugCrossModuleExportsSubsection()
       : DebugSubsection(DebugSubsectionKind::CrossScopeExports) {}
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
index 198ce4a8b4e4..49ea04c4ecd6 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/CodeView/DebugSubsection.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
@@ -37,8 +38,8 @@ template <> struct VarStreamArrayExtractor<codeview::CrossModuleImportItem> {
 public:
   using ContextType = void;
 
-  Error operator()(BinaryStreamRef Stream, uint32_t &Len,
-                   codeview::CrossModuleImportItem &Item);
+  LLVM_ABI Error operator()(BinaryStreamRef Stream, uint32_t &Len,
+                            codeview::CrossModuleImportItem &Item);
 };
 
 namespace codeview {
@@ -57,8 +58,8 @@ class DebugCrossModuleImportsSubsectionRef final : public DebugSubsectionRef {
     return S->kind() == DebugSubsectionKind::CrossScopeImports;
   }
 
-  Error initialize(BinaryStreamReader Reader);
-  Error initialize(BinaryStreamRef Stream);
+  LLVM_ABI Error initialize(BinaryStreamReader Reader);
+  LLVM_ABI Error initialize(BinaryStreamRef Stream);
 
   Iterator begin() const { return References.begin(); }
   Iterator end() const { return References.end(); }
@@ -67,7 +68,8 @@ class DebugCrossModuleImportsSubsectionRef final : public DebugSubsectionRef {
   ReferenceArray References;
 };
 
-class DebugCrossModuleImportsSubsection final : public DebugSubsection {
+class LLVM_ABI DebugCrossModuleImportsSubsection final
+    : public DebugSubsection {
 public:
   explicit DebugCrossModuleImportsSubsection(
       DebugStringTableSubsection &Strings)
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h
index f2c5bf9d7c95..878bfa45969d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h
@@ -13,6 +13,7 @@
 #include "llvm/DebugInfo/CodeView/DebugSubsection.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 
@@ -29,8 +30,8 @@ class DebugFrameDataSubsectionRef final : public DebugSubsectionRef {
     return S->kind() == DebugSubsectionKind::FrameData;
   }
 
-  Error initialize(BinaryStreamReader Reader);
-  Error initialize(BinaryStreamRef Stream);
+  LLVM_ABI Error initialize(BinaryStreamReader Reader);
+  LLVM_ABI Error initialize(BinaryStreamRef Stream);
 
   FixedStreamArray<FrameData>::Iterator begin() const { return Frames.begin(); }
   FixedStreamArray<FrameData>::Iterator end() const { return Frames.end(); }
@@ -42,7 +43,7 @@ class DebugFrameDataSubsectionRef final : public DebugSubsectionRef {
   FixedStreamArray<FrameData> Frames;
 };
 
-class DebugFrameDataSubsection final : public DebugSubsection {
+class LLVM_ABI DebugFrameDataSubsection final : public DebugSubsection {
 public:
   DebugFrameDataSubsection(bool IncludeRelocPtr)
       : DebugSubsection(DebugSubsectionKind::FrameData),
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
index f9d1507af5f3..28d2ab3d80de 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
@@ -16,6 +16,7 @@
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
@@ -49,8 +50,8 @@ struct InlineeSourceLine {
 } // end namespace codeview
 
 template <> struct VarStreamArrayExtractor<codeview::InlineeSourceLine> {
-  Error operator()(BinaryStreamRef Stream, uint32_t &Len,
-                   codeview::InlineeSourceLine &Item);
+  LLVM_ABI Error operator()(BinaryStreamRef Stream, uint32_t &Len,
+                            codeview::InlineeSourceLine &Item);
 
   bool HasExtraFiles = false;
 };
@@ -62,19 +63,19 @@ class DebugInlineeLinesSubsectionRef final : public DebugSubsectionRef {
   using Iterator = LinesArray::Iterator;
 
 public:
-  DebugInlineeLinesSubsectionRef();
+  LLVM_ABI DebugInlineeLinesSubsectionRef();
 
   static bool classof(const DebugSubsectionRef *S) {
     return S->kind() == DebugSubsectionKind::InlineeLines;
   }
 
-  Error initialize(BinaryStreamReader Reader);
+  LLVM_ABI Error initialize(BinaryStreamReader Reader);
   Error initialize(BinaryStreamRef Section) {
     return initialize(BinaryStreamReader(Section));
   }
 
   bool valid() const { return Lines.valid(); }
-  bool hasExtraFiles() const;
+  LLVM_ABI bool hasExtraFiles() const;
 
   Iterator begin() const { return Lines.begin(); }
   Iterator end() const { return Lines.end(); }
@@ -84,7 +85,7 @@ class DebugInlineeLinesSubsectionRef final : public DebugSubsectionRef {
   LinesArray Lines;
 };
 
-class DebugInlineeLinesSubsection final : public DebugSubsection {
+class LLVM_ABI DebugInlineeLinesSubsection final : public DebugSubsection {
 public:
   struct Entry {
     std::vector<support::ulittle32_t> ExtraFiles;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
index 68eb9e1af3bd..6791d056e5aa 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/CodeView/Line.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
@@ -70,8 +71,8 @@ struct LineColumnEntry {
 
 class LineColumnExtractor {
 public:
-  Error operator()(BinaryStreamRef Stream, uint32_t &Len,
-                   LineColumnEntry &Item);
+  LLVM_ABI Error operator()(BinaryStreamRef Stream, uint32_t &Len,
+                            LineColumnEntry &Item);
 
   const LineFragmentHeader *Header = nullptr;
 };
@@ -83,27 +84,27 @@ class DebugLinesSubsectionRef final : public DebugSubsectionRef {
   using Iterator = LineInfoArray::Iterator;
 
 public:
-  DebugLinesSubsectionRef();
+  LLVM_ABI DebugLinesSubsectionRef();
 
   static bool classof(const DebugSubsectionRef *S) {
     return S->kind() == DebugSubsectionKind::Lines;
   }
 
-  Error initialize(BinaryStreamReader Reader);
+  LLVM_ABI Error initialize(BinaryStreamReader Reader);
 
   Iterator begin() const { return LinesAndColumns.begin(); }
   Iterator end() const { return LinesAndColumns.end(); }
 
   const LineFragmentHeader *header() const { return Header; }
 
-  bool hasColumnInfo() const;
+  LLVM_ABI bool hasColumnInfo() const;
 
 private:
   const LineFragmentHeader *Header = nullptr;
   LineInfoArray LinesAndColumns;
 };
 
-class DebugLinesSubsection final : public DebugSubsection {
+class LLVM_ABI DebugLinesSubsection final : public DebugSubsection {
   struct Block {
     Block(uint32_t ChecksumBufferOffset)
         : ChecksumBufferOffset(ChecksumBufferOffset) {}
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h
index 6e5b8adddd4a..3f1c8bb791cc 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/DebugSubsection.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 
@@ -30,16 +31,16 @@ namespace codeview {
 /// does not own the underlying storage for the buffer.
 class DebugStringTableSubsectionRef : public DebugSubsectionRef {
 public:
-  DebugStringTableSubsectionRef();
+  LLVM_ABI DebugStringTableSubsectionRef();
 
   static bool classof(const DebugSubsectionRef *S) {
     return S->kind() == DebugSubsectionKind::StringTable;
   }
 
-  Error initialize(BinaryStreamRef Contents);
-  Error initialize(BinaryStreamReader &Reader);
+  LLVM_ABI Error initialize(BinaryStreamRef Contents);
+  LLVM_ABI Error initialize(BinaryStreamReader &Reader);
 
-  Expected<StringRef> getString(uint32_t Offset) const;
+  LLVM_ABI Expected<StringRef> getString(uint32_t Offset) const;
 
   bool valid() const { return Stream.valid(); }
 
@@ -53,7 +54,7 @@ class DebugStringTableSubsectionRef : public DebugSubsectionRef {
 /// DebugStringTableSubsection owns the underlying storage for the table, and is
 /// capable of serializing the string table into a format understood by
 /// DebugStringTableSubsectionRef.
-class DebugStringTableSubsection : public DebugSubsection {
+class LLVM_ABI DebugStringTableSubsection : public DebugSubsection {
 public:
   DebugStringTableSubsection();
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h
index 39413bb73b58..710152fb31c5 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsection.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_CODEVIEW_DEBUGSUBSECTION_H
 
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include <cstdint>
@@ -18,7 +19,7 @@ namespace llvm {
 class BinaryStreamWriter;
 namespace codeview {
 
-class DebugSubsectionRef {
+class LLVM_ABI DebugSubsectionRef {
 public:
   explicit DebugSubsectionRef(DebugSubsectionKind Kind) : Kind(Kind) {}
   virtual ~DebugSubsectionRef();
@@ -31,7 +32,7 @@ class DebugSubsectionRef {
   DebugSubsectionKind Kind;
 };
 
-class DebugSubsection {
+class LLVM_ABI DebugSubsection {
 public:
   explicit DebugSubsection(DebugSubsectionKind Kind) : Kind(Kind) {}
   virtual ~DebugSubsection();
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
index e915d8a5830c..a4e578c021cd 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
@@ -12,6 +12,7 @@
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MathExtras.h"
@@ -34,14 +35,16 @@ struct DebugSubsectionHeader {
 
 class DebugSubsectionRecord {
 public:
-  DebugSubsectionRecord();
-  DebugSubsectionRecord(DebugSubsectionKind Kind, BinaryStreamRef Data);
+  LLVM_ABI DebugSubsectionRecord();
+  LLVM_ABI DebugSubsectionRecord(DebugSubsectionKind Kind,
+                                 BinaryStreamRef Data);
 
-  static Error initialize(BinaryStreamRef Stream, DebugSubsectionRecord &Info);
+  LLVM_ABI static Error initialize(BinaryStreamRef Stream,
+                                   DebugSubsectionRecord &Info);
 
-  uint32_t getRecordLength() const;
-  DebugSubsectionKind kind() const;
-  BinaryStreamRef getRecordData() const;
+  LLVM_ABI uint32_t getRecordLength() const;
+  LLVM_ABI DebugSubsectionKind kind() const;
+  LLVM_ABI BinaryStreamRef getRecordData() const;
 
 private:
   DebugSubsectionKind Kind = DebugSubsectionKind::None;
@@ -50,15 +53,17 @@ class DebugSubsectionRecord {
 
 class DebugSubsectionRecordBuilder {
 public:
+  LLVM_ABI
   DebugSubsectionRecordBuilder(std::shared_ptr<DebugSubsection> Subsection);
 
   /// Use this to copy existing subsections directly from source to destination.
   /// For example, line table subsections in an object file only need to be
   /// relocated before being copied into the PDB.
-  DebugSubsectionRecordBuilder(const DebugSubsectionRecord &Contents);
+  LLVM_ABI DebugSubsectionRecordBuilder(const DebugSubsectionRecord &Contents);
 
-  uint32_t calculateSerializedLength() const;
-  Error commit(BinaryStreamWriter &Writer, CodeViewContainer Container) const;
+  LLVM_ABI uint32_t calculateSerializedLength() const;
+  LLVM_ABI Error commit(BinaryStreamWriter &Writer,
+                        CodeViewContainer Container) const;
 
 private:
   /// The subsection to build. Will be null if Contents is non-empty.
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h
index 51b8523ed969..97b1be31736e 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h
@@ -11,6 +11,7 @@
 
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/CodeView/DebugSubsection.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -24,7 +25,7 @@ class DebugSymbolsSubsectionRef final : public DebugSubsectionRef {
     return S->kind() == DebugSubsectionKind::Symbols;
   }
 
-  Error initialize(BinaryStreamReader Reader);
+  LLVM_ABI Error initialize(BinaryStreamReader Reader);
 
   CVSymbolArray::Iterator begin() const { return Records.begin(); }
   CVSymbolArray::Iterator end() const { return Records.end(); }
@@ -33,7 +34,7 @@ class DebugSymbolsSubsectionRef final : public DebugSubsectionRef {
   CVSymbolArray Records;
 };
 
-class DebugSymbolsSubsection final : public DebugSubsection {
+class LLVM_ABI DebugSymbolsSubsection final : public DebugSubsection {
 public:
   DebugSymbolsSubsection() : DebugSubsection(DebugSubsectionKind::Symbols) {}
   static bool classof(const DebugSubsection *S) {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h
index 2e3be094f8a6..d4b48d932806 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/EnumTables.h
@@ -12,43 +12,44 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
 template <typename T> struct EnumEntry;
 namespace codeview {
 
-ArrayRef<EnumEntry<SymbolKind>> getSymbolTypeNames();
-ArrayRef<EnumEntry<TypeLeafKind>> getTypeLeafNames();
-ArrayRef<EnumEntry<uint16_t>> getRegisterNames(CPUType Cpu);
-ArrayRef<EnumEntry<uint32_t>> getPublicSymFlagNames();
-ArrayRef<EnumEntry<uint8_t>> getProcSymFlagNames();
-ArrayRef<EnumEntry<uint16_t>> getLocalFlagNames();
-ArrayRef<EnumEntry<uint8_t>> getFrameCookieKindNames();
-ArrayRef<EnumEntry<SourceLanguage>> getSourceLanguageNames();
-ArrayRef<EnumEntry<uint32_t>> getCompileSym2FlagNames();
-ArrayRef<EnumEntry<uint32_t>> getCompileSym3FlagNames();
-ArrayRef<EnumEntry<uint32_t>> getFileChecksumNames();
-ArrayRef<EnumEntry<unsigned>> getCPUTypeNames();
-ArrayRef<EnumEntry<uint32_t>> getFrameProcSymFlagNames();
-ArrayRef<EnumEntry<uint16_t>> getExportSymFlagNames();
-ArrayRef<EnumEntry<uint32_t>> getModuleSubstreamKindNames();
-ArrayRef<EnumEntry<uint8_t>> getThunkOrdinalNames();
-ArrayRef<EnumEntry<uint16_t>> getTrampolineNames();
-ArrayRef<EnumEntry<COFF::SectionCharacteristics>>
+LLVM_ABI ArrayRef<EnumEntry<SymbolKind>> getSymbolTypeNames();
+LLVM_ABI ArrayRef<EnumEntry<TypeLeafKind>> getTypeLeafNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getRegisterNames(CPUType Cpu);
+LLVM_ABI ArrayRef<EnumEntry<uint32_t>> getPublicSymFlagNames();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getProcSymFlagNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getLocalFlagNames();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getFrameCookieKindNames();
+LLVM_ABI ArrayRef<EnumEntry<SourceLanguage>> getSourceLanguageNames();
+LLVM_ABI ArrayRef<EnumEntry<uint32_t>> getCompileSym2FlagNames();
+LLVM_ABI ArrayRef<EnumEntry<uint32_t>> getCompileSym3FlagNames();
+LLVM_ABI ArrayRef<EnumEntry<uint32_t>> getFileChecksumNames();
+LLVM_ABI ArrayRef<EnumEntry<unsigned>> getCPUTypeNames();
+LLVM_ABI ArrayRef<EnumEntry<uint32_t>> getFrameProcSymFlagNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getExportSymFlagNames();
+LLVM_ABI ArrayRef<EnumEntry<uint32_t>> getModuleSubstreamKindNames();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getThunkOrdinalNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getTrampolineNames();
+LLVM_ABI ArrayRef<EnumEntry<COFF::SectionCharacteristics>>
 getImageSectionCharacteristicNames();
-ArrayRef<EnumEntry<uint16_t>> getClassOptionNames();
-ArrayRef<EnumEntry<uint8_t>> getMemberAccessNames();
-ArrayRef<EnumEntry<uint16_t>> getMethodOptionNames();
-ArrayRef<EnumEntry<uint16_t>> getMemberKindNames();
-ArrayRef<EnumEntry<uint8_t>> getPtrKindNames();
-ArrayRef<EnumEntry<uint8_t>> getPtrModeNames();
-ArrayRef<EnumEntry<uint16_t>> getPtrMemberRepNames();
-ArrayRef<EnumEntry<uint16_t>> getTypeModifierNames();
-ArrayRef<EnumEntry<uint8_t>> getCallingConventions();
-ArrayRef<EnumEntry<uint8_t>> getFunctionOptionEnum();
-ArrayRef<EnumEntry<uint16_t>> getLabelTypeEnum();
-ArrayRef<EnumEntry<uint16_t>> getJumpTableEntrySizeNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getClassOptionNames();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getMemberAccessNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getMethodOptionNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getMemberKindNames();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getPtrKindNames();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getPtrModeNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getPtrMemberRepNames();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getTypeModifierNames();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getCallingConventions();
+LLVM_ABI ArrayRef<EnumEntry<uint8_t>> getFunctionOptionEnum();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getLabelTypeEnum();
+LLVM_ABI ArrayRef<EnumEntry<uint16_t>> getJumpTableEntrySizeNames();
 
 } // end namespace codeview
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Formatters.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Formatters.h
index 10683c289224..bf0340be901e 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Formatters.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Formatters.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/CodeView/GUID.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/FormatAdapters.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/raw_ostream.h"
@@ -26,7 +27,7 @@ struct GUID;
 
 namespace detail {
 
-class GuidAdapter final : public FormatAdapter<ArrayRef<uint8_t>> {
+class LLVM_ABI GuidAdapter final : public FormatAdapter<ArrayRef<uint8_t>> {
   ArrayRef<uint8_t> Guid;
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GUID.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GUID.h
index 5f807e6f7eeb..b01df28bf10d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GUID.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GUID.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_CODEVIEW_GUID_H
 #define LLVM_DEBUGINFO_CODEVIEW_GUID_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <cstring>
 
@@ -46,7 +47,7 @@ inline bool operator!=(const GUID &LHS, const GUID &RHS) {
   return !(LHS == RHS);
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const GUID &Guid);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const GUID &Guid);
 
 } // namespace codeview
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h
index 18f16bc66a77..a587b3e2ed62 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h
@@ -18,6 +18,7 @@
 #include "llvm/DebugInfo/CodeView/TypeHashing.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 
@@ -26,7 +27,7 @@ namespace codeview {
 
 class ContinuationRecordBuilder;
 
-class GlobalTypeTableBuilder : public TypeCollection {
+class LLVM_ABI GlobalTypeTableBuilder : public TypeCollection {
   /// Storage for records.  These need to outlive the TypeTableBuilder.
   BumpPtrAllocator &RecordStorage;
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h
index 240f7092140c..5b23ac9f862a 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStreamArray.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/StringSaver.h"
 #include <cstdint>
@@ -45,7 +46,7 @@ namespace codeview {
 /// require more than one linear scan.  For a type stream of N elements divided
 /// into M chunks of roughly equal size, this yields a worst case lookup time
 /// of O(N/M) and an amortized time of O(1).
-class LazyRandomTypeCollection : public TypeCollection {
+class LLVM_ABI LazyRandomTypeCollection : public TypeCollection {
   using PartialOffsetArray = FixedStreamArray<TypeIndexOffset>;
 
   struct CacheEntry {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Line.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Line.h
index 6918645b94d2..79a87555cf00 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Line.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/Line.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_CODEVIEW_LINE_H
 #define LLVM_DEBUGINFO_CODEVIEW_LINE_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include <cinttypes>
 
@@ -32,7 +33,7 @@ class LineInfo {
     StatementFlag = 0x80000000u
   };
 
-  LineInfo(uint32_t StartLine, uint32_t EndLine, bool IsStatement);
+  LLVM_ABI LineInfo(uint32_t StartLine, uint32_t EndLine, bool IsStatement);
   LineInfo(uint32_t LineData) : LineData(LineData) {}
 
   uint32_t getStartLine() const { return LineData & StartLineMask; }
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h
index 10bc8f60613c..730278488176 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h
@@ -17,6 +17,7 @@
 #include "llvm/DebugInfo/CodeView/TypeCollection.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -25,7 +26,7 @@ struct LocallyHashedType;
 
 class ContinuationRecordBuilder;
 
-class MergingTypeTableBuilder : public TypeCollection {
+class LLVM_ABI MergingTypeTableBuilder : public TypeCollection {
   /// Storage for records.  These need to outlive the TypeTableBuilder.
   BumpPtrAllocator &RecordStorage;
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordName.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordName.h
index 9078ed38d2f1..a62f5ec385bc 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordName.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordName.h
@@ -11,14 +11,15 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
 namespace codeview {
 class TypeCollection;
 class TypeIndex;
-std::string computeTypeName(TypeCollection &Types, TypeIndex Index);
-StringRef getSymbolName(CVSymbol Sym);
+LLVM_ABI std::string computeTypeName(TypeCollection &Types, TypeIndex Index);
+LLVM_ABI StringRef getSymbolName(CVSymbol Sym);
 } // namespace codeview
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h
index 10248dbf646b..ecd1ece09dfb 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/RecordSerialization.h
@@ -14,6 +14,7 @@
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/CodeViewError.h"
 #include "llvm/Support/BinaryStreamReader.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cinttypes>
@@ -39,8 +40,8 @@ struct RecordPrefix {
 
 /// Reinterpret a byte array as an array of characters. Does not interpret as
 /// a C string, as StringRef has several helpers (split) that make that easy.
-StringRef getBytesAsCharacters(ArrayRef<uint8_t> LeafData);
-StringRef getBytesAsCString(ArrayRef<uint8_t> LeafData);
+LLVM_ABI StringRef getBytesAsCharacters(ArrayRef<uint8_t> LeafData);
+LLVM_ABI StringRef getBytesAsCString(ArrayRef<uint8_t> LeafData);
 
 inline Error consume(BinaryStreamReader &Reader) { return Error::success(); }
 
@@ -48,20 +49,20 @@ inline Error consume(BinaryStreamReader &Reader) { return Error::success(); }
 /// the type stream. If the value is positive and less than LF_NUMERIC (1 <<
 /// 15), it is emitted directly in Data. Otherwise, it has a tag like LF_CHAR
 /// that indicates the bitwidth and sign of the numeric data.
-Error consume(BinaryStreamReader &Reader, APSInt &Num);
+LLVM_ABI Error consume(BinaryStreamReader &Reader, APSInt &Num);
 
 /// Decodes a numeric leaf value that is known to be a particular type.
-Error consume_numeric(BinaryStreamReader &Reader, uint64_t &Value);
+LLVM_ABI Error consume_numeric(BinaryStreamReader &Reader, uint64_t &Value);
 
 /// Decodes signed and unsigned fixed-length integers.
-Error consume(BinaryStreamReader &Reader, uint32_t &Item);
-Error consume(BinaryStreamReader &Reader, int32_t &Item);
+LLVM_ABI Error consume(BinaryStreamReader &Reader, uint32_t &Item);
+LLVM_ABI Error consume(BinaryStreamReader &Reader, int32_t &Item);
 
 /// Decodes a null terminated string.
-Error consume(BinaryStreamReader &Reader, StringRef &Item);
+LLVM_ABI Error consume(BinaryStreamReader &Reader, StringRef &Item);
 
-Error consume(StringRef &Data, APSInt &Num);
-Error consume(StringRef &Data, uint32_t &Item);
+LLVM_ABI Error consume(StringRef &Data, APSInt &Num);
+LLVM_ABI Error consume(StringRef &Data, uint32_t &Item);
 
 /// Decodes an arbitrary object whose layout matches that of the underlying
 /// byte sequence, and returns a pointer to the object.
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h
index fcc0452a6ae9..723eb3c0f4a1 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_CODEVIEW_SIMPLETYPESERIALIZER_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Compiler.h"
 #include <vector>
 
 namespace llvm {
@@ -20,8 +21,8 @@ class SimpleTypeSerializer {
   std::vector<uint8_t> ScratchBuffer;
 
 public:
-  SimpleTypeSerializer();
-  ~SimpleTypeSerializer();
+  LLVM_ABI SimpleTypeSerializer();
+  LLVM_ABI ~SimpleTypeSerializer();
 
   // This template is explicitly instantiated in the implementation file for all
   // supported types.  The method itself is ugly, so inlining it into the header
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h
index 50e745e5c2ab..1de75b8e764b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h
@@ -11,6 +11,7 @@
 
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -23,21 +24,22 @@ class DebugStringTableSubsectionRef;
 class StringsAndChecksumsRef {
 public:
   // If no subsections are known about initially, we find as much as we can.
-  StringsAndChecksumsRef();
+  LLVM_ABI StringsAndChecksumsRef();
 
   // If only a string table subsection is given, we find a checksums subsection.
-  explicit StringsAndChecksumsRef(const DebugStringTableSubsectionRef &Strings);
+  LLVM_ABI explicit StringsAndChecksumsRef(
+      const DebugStringTableSubsectionRef &Strings);
 
   // If both subsections are given, we don't need to find anything.
-  StringsAndChecksumsRef(const DebugStringTableSubsectionRef &Strings,
-                         const DebugChecksumsSubsectionRef &Checksums);
+  LLVM_ABI StringsAndChecksumsRef(const DebugStringTableSubsectionRef &Strings,
+                                  const DebugChecksumsSubsectionRef &Checksums);
 
-  void setStrings(const DebugStringTableSubsectionRef &Strings);
-  void setChecksums(const DebugChecksumsSubsectionRef &CS);
+  LLVM_ABI void setStrings(const DebugStringTableSubsectionRef &Strings);
+  LLVM_ABI void setChecksums(const DebugChecksumsSubsectionRef &CS);
 
-  void reset();
-  void resetStrings();
-  void resetChecksums();
+  LLVM_ABI void reset();
+  LLVM_ABI void resetStrings();
+  LLVM_ABI void resetChecksums();
 
   template <typename T> void initialize(T &&FragmentRange) {
     for (const DebugSubsectionRecord &R : FragmentRange) {
@@ -70,8 +72,8 @@ class StringsAndChecksumsRef {
   bool hasChecksums() const { return Checksums != nullptr; }
 
 private:
-  void initializeStrings(const DebugSubsectionRecord &SR);
-  void initializeChecksums(const DebugSubsectionRecord &FCR);
+  LLVM_ABI void initializeStrings(const DebugSubsectionRecord &SR);
+  LLVM_ABI void initializeChecksums(const DebugSubsectionRecord &FCR);
 
   std::shared_ptr<DebugStringTableSubsectionRef> OwnedStrings;
   std::shared_ptr<DebugChecksumsSubsectionRef> OwnedChecksums;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h
index c674700fac59..2d9d7df114d6 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h
@@ -12,6 +12,7 @@
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/SymbolDumpDelegate.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include <memory>
@@ -38,11 +39,11 @@ class CVSymbolDumper {
   /// and true otherwise.  This should be called in order, since the dumper
   /// maintains state about previous records which are necessary for cross
   /// type references.
-  Error dump(CVRecord<SymbolKind> &Record);
+  LLVM_ABI Error dump(CVRecord<SymbolKind> &Record);
 
   /// Dumps the type records in Data. Returns false if there was a type stream
   /// parse error, and true otherwise.
-  Error dump(const CVSymbolArray &Symbols);
+  LLVM_ABI Error dump(const CVSymbolArray &Symbols);
 
   CPUType getCompilationCPUType() const { return CompilationCPUType; }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
index 3cce40dcf735..5b4f0d31e642 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
@@ -19,6 +19,7 @@
 #include "llvm/DebugInfo/CodeView/RecordSerialization.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/Support/BinaryStreamArray.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include <cstdint>
 #include <vector>
@@ -1023,8 +1024,8 @@ class AnnotationSym : public SymbolRecord {
   uint32_t RecordOffset = 0;
 };
 
-Expected<CVSymbol> readSymbolFromStream(BinaryStreamRef Stream,
-                                        uint32_t Offset);
+LLVM_ABI Expected<CVSymbol> readSymbolFromStream(BinaryStreamRef Stream,
+                                                 uint32_t Offset);
 
 } // end namespace codeview
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h
index 71bc70dde6ed..98002483ba23 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h
@@ -11,6 +11,7 @@
 
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace codeview {
@@ -50,11 +51,11 @@ inline bool symbolEndsScope(SymbolKind Kind) {
 
 /// Given a symbol P for which symbolOpensScope(P) == true, return the
 /// corresponding end offset.
-uint32_t getScopeEndOffset(const CVSymbol &Symbol);
-uint32_t getScopeParentOffset(const CVSymbol &Symbol);
+LLVM_ABI uint32_t getScopeEndOffset(const CVSymbol &Symbol);
+LLVM_ABI uint32_t getScopeParentOffset(const CVSymbol &Symbol);
 
-CVSymbolArray limitSymbolArrayToScope(const CVSymbolArray &Symbols,
-                                      uint32_t ScopeBegin);
+LLVM_ABI CVSymbolArray limitSymbolArrayToScope(const CVSymbolArray &Symbols,
+                                               uint32_t ScopeBegin);
 
 } // namespace codeview
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h
index 30e9c626c205..7fa86310f04f 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h
@@ -11,13 +11,14 @@
 
 #include "llvm/DebugInfo/CodeView/CodeViewRecordIO.h"
 #include "llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class BinaryStreamReader;
 class BinaryStreamWriter;
 
 namespace codeview {
-class SymbolRecordMapping : public SymbolVisitorCallbacks {
+class LLVM_ABI SymbolRecordMapping : public SymbolVisitorCallbacks {
 public:
   explicit SymbolRecordMapping(BinaryStreamReader &Reader,
                                CodeViewContainer Container)
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h
index df52b374f72d..feef969d8dc2 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h
@@ -17,6 +17,7 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryByteStream.h"
 #include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <array>
@@ -25,7 +26,7 @@
 namespace llvm {
 namespace codeview {
 
-class SymbolSerializer : public SymbolVisitorCallbacks {
+class LLVM_ABI SymbolSerializer : public SymbolVisitorCallbacks {
   BumpPtrAllocator &Storage;
   // Since this is a fixed size buffer, use a stack allocated buffer.  This
   // yields measurable performance increase over the repeated heap allocations
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h
index 1fad50343e3a..e2ffee4e9406 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h
@@ -13,6 +13,7 @@
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class ScopedPrinter;
@@ -25,7 +26,7 @@ struct MemberAttributes;
 class TypeCollection;
 
 /// Dumper for CodeView type streams found in COFF object files and PDB files.
-class TypeDumpVisitor : public TypeVisitorCallbacks {
+class LLVM_ABI TypeDumpVisitor : public TypeVisitorCallbacks {
 public:
   TypeDumpVisitor(TypeCollection &TpiTypes, ScopedPrinter *W,
                   bool PrintRecordBytes)
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h
index e92ec4855b25..9b9185312a8f 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/CodeView/TypeCollection.h"
@@ -36,7 +37,7 @@ struct LocallyHashedType {
   ArrayRef<uint8_t> RecordData;
 
   /// Given a type, compute its local hash.
-  static LocallyHashedType hashType(ArrayRef<uint8_t> RecordData);
+  LLVM_ABI static LocallyHashedType hashType(ArrayRef<uint8_t> RecordData);
 
   /// Given a sequence of types, compute all of the local hashes.
   template <typename Range>
@@ -103,9 +104,10 @@ struct GloballyHashedType {
   /// this record.  Due to the nature of global hashes incorporating the hashes
   /// of referenced records, this function requires a list of types and ids
   /// that RecordData might reference, indexable by TypeIndex.
-  static GloballyHashedType hashType(ArrayRef<uint8_t> RecordData,
-                                     ArrayRef<GloballyHashedType> PreviousTypes,
-                                     ArrayRef<GloballyHashedType> PreviousIds);
+  LLVM_ABI static GloballyHashedType
+  hashType(ArrayRef<uint8_t> RecordData,
+           ArrayRef<GloballyHashedType> PreviousTypes,
+           ArrayRef<GloballyHashedType> PreviousIds);
 
   /// Given a sequence of bytes representing a record, compute a global hash for
   /// this record.  Due to the nature of global hashes incorporating the hashes
@@ -181,8 +183,8 @@ static_assert(std::is_trivially_copyable<GloballyHashedType>::value,
 } // namespace codeview
 
 template <> struct DenseMapInfo<codeview::LocallyHashedType> {
-  static codeview::LocallyHashedType Empty;
-  static codeview::LocallyHashedType Tombstone;
+  LLVM_ABI static codeview::LocallyHashedType Empty;
+  LLVM_ABI static codeview::LocallyHashedType Tombstone;
 
   static codeview::LocallyHashedType getEmptyKey() { return Empty; }
 
@@ -201,8 +203,8 @@ template <> struct DenseMapInfo<codeview::LocallyHashedType> {
 };
 
 template <> struct DenseMapInfo<codeview::GloballyHashedType> {
-  static codeview::GloballyHashedType Empty;
-  static codeview::GloballyHashedType Tombstone;
+  LLVM_ABI static codeview::GloballyHashedType Empty;
+  LLVM_ABI static codeview::GloballyHashedType Tombstone;
 
   static codeview::GloballyHashedType getEmptyKey() { return Empty; }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h
index 653eafa04e0a..1430fe83cc64 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_CODEVIEW_TYPEINDEX_H
 
 #include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include <cassert>
 #include <cinttypes>
@@ -270,7 +271,7 @@ class TypeIndex {
     return A.toArrayIndex() - B.toArrayIndex();
   }
 
-  static StringRef simpleTypeName(TypeIndex TI);
+  LLVM_ABI static StringRef simpleTypeName(TypeIndex TI);
 
 private:
   support::ulittle32_t Index;
@@ -284,8 +285,8 @@ struct TypeIndexOffset {
   support::ulittle32_t Offset;
 };
 
-void printTypeIndex(ScopedPrinter &Printer, StringRef FieldName, TypeIndex TI,
-                    TypeCollection &Types);
+LLVM_ABI void printTypeIndex(ScopedPrinter &Printer, StringRef FieldName,
+                             TypeIndex TI, TypeCollection &Types);
 }
 
 template <> struct DenseMapInfo<codeview::TypeIndex> {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h
index 7ef8521604fb..e9122fa6089e 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 template <typename T> class SmallVectorImpl;
@@ -23,23 +24,23 @@ struct TiReference {
   uint32_t Count;
 };
 
-void discoverTypeIndices(ArrayRef<uint8_t> RecordData,
-                         SmallVectorImpl<TiReference> &Refs);
-void discoverTypeIndices(const CVType &Type,
-                         SmallVectorImpl<TiReference> &Refs);
-void discoverTypeIndices(const CVType &Type,
-                         SmallVectorImpl<TypeIndex> &Indices);
-void discoverTypeIndices(ArrayRef<uint8_t> RecordData,
-                         SmallVectorImpl<TypeIndex> &Indices);
+LLVM_ABI void discoverTypeIndices(ArrayRef<uint8_t> RecordData,
+                                  SmallVectorImpl<TiReference> &Refs);
+LLVM_ABI void discoverTypeIndices(const CVType &Type,
+                                  SmallVectorImpl<TiReference> &Refs);
+LLVM_ABI void discoverTypeIndices(const CVType &Type,
+                                  SmallVectorImpl<TypeIndex> &Indices);
+LLVM_ABI void discoverTypeIndices(ArrayRef<uint8_t> RecordData,
+                                  SmallVectorImpl<TypeIndex> &Indices);
 
 /// Discover type indices in symbol records. Returns false if this is an unknown
 /// record.
-bool discoverTypeIndicesInSymbol(const CVSymbol &Symbol,
-                                 SmallVectorImpl<TiReference> &Refs);
-bool discoverTypeIndicesInSymbol(ArrayRef<uint8_t> RecordData,
-                                 SmallVectorImpl<TiReference> &Refs);
-bool discoverTypeIndicesInSymbol(ArrayRef<uint8_t> RecordData,
-                                 SmallVectorImpl<TypeIndex> &Indices);
+LLVM_ABI bool discoverTypeIndicesInSymbol(const CVSymbol &Symbol,
+                                          SmallVectorImpl<TiReference> &Refs);
+LLVM_ABI bool discoverTypeIndicesInSymbol(ArrayRef<uint8_t> RecordData,
+                                          SmallVectorImpl<TiReference> &Refs);
+LLVM_ABI bool discoverTypeIndicesInSymbol(ArrayRef<uint8_t> RecordData,
+                                          SmallVectorImpl<TypeIndex> &Indices);
 }
 }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h
index aa183cd7d19d..4d5737b9bcab 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h
@@ -11,6 +11,7 @@
 
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace codeview {
@@ -18,11 +19,11 @@ namespace codeview {
 /// Given an arbitrary codeview type, determine if it is an LF_STRUCTURE,
 /// LF_CLASS, LF_INTERFACE, LF_UNION, or LF_ENUM with the forward ref class
 /// option.
-bool isUdtForwardRef(CVType CVT);
+LLVM_ABI bool isUdtForwardRef(CVType CVT);
 
 /// Given a CVType which is assumed to be an LF_MODIFIER, return the
 /// TypeIndex of the type that the LF_MODIFIER modifies.
-TypeIndex getModifiedType(const CVType &CVT);
+LLVM_ABI TypeIndex getModifiedType(const CVType &CVT);
 
 /// Return true if this record should be in the IPI stream of a PDB. In an
 /// object file, these record kinds will appear mixed into the .debug$T section.
@@ -56,11 +57,11 @@ inline bool isAggregate(CVType CVT) {
 }
 
 /// Given an arbitrary codeview type index, determine its size.
-uint64_t getSizeInBytesForTypeIndex(TypeIndex TI);
+LLVM_ABI uint64_t getSizeInBytesForTypeIndex(TypeIndex TI);
 
 /// Given an arbitrary codeview type, return the type's size in the case
 /// of aggregate (LF_STRUCTURE, LF_CLASS, LF_INTERFACE, LF_UNION).
-uint64_t getSizeInBytesForTypeRecord(CVType CVT);
+LLVM_ABI uint64_t getSizeInBytesForTypeRecord(CVType CVT);
 
 } // namespace codeview
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h
index 26eb7d221679..44478229f349 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h
@@ -13,6 +13,7 @@
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/CodeViewRecordIO.h"
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <optional>
 
@@ -23,7 +24,7 @@ class BinaryStreamWriter;
 namespace codeview {
 class TypeIndex;
 struct CVMemberRecord;
-class TypeRecordMapping : public TypeVisitorCallbacks {
+class LLVM_ABI TypeRecordMapping : public TypeVisitorCallbacks {
 public:
   explicit TypeRecordMapping(BinaryStreamReader &Reader) : IO(Reader) {}
   explicit TypeRecordMapping(BinaryStreamWriter &Writer) : IO(Writer) {}
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h
index 9bcae7a4b729..0afcd0e63023 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -42,9 +43,9 @@ struct PCHMergerInfo {
 ///
 /// \returns Error::success() if the operation succeeded, otherwise an
 /// appropriate error code.
-Error mergeTypeRecords(MergingTypeTableBuilder &Dest,
-                       SmallVectorImpl<TypeIndex> &SourceToDest,
-                       const CVTypeArray &Types);
+LLVM_ABI Error mergeTypeRecords(MergingTypeTableBuilder &Dest,
+                                SmallVectorImpl<TypeIndex> &SourceToDest,
+                                const CVTypeArray &Types);
 
 /// Merge one set of id records into another.  This method assumes
 /// that all records are id records, and there are no Type records present.
@@ -67,9 +68,10 @@ Error mergeTypeRecords(MergingTypeTableBuilder &Dest,
 ///
 /// \returns Error::success() if the operation succeeded, otherwise an
 /// appropriate error code.
-Error mergeIdRecords(MergingTypeTableBuilder &Dest, ArrayRef<TypeIndex> Types,
-                     SmallVectorImpl<TypeIndex> &SourceToDest,
-                     const CVTypeArray &Ids);
+LLVM_ABI Error mergeIdRecords(MergingTypeTableBuilder &Dest,
+                              ArrayRef<TypeIndex> Types,
+                              SmallVectorImpl<TypeIndex> &SourceToDest,
+                              const CVTypeArray &Ids);
 
 /// Merge a unified set of type and id records, splitting them into
 /// separate output streams.
@@ -86,29 +88,30 @@ Error mergeIdRecords(MergingTypeTableBuilder &Dest, ArrayRef<TypeIndex> Types,
 ///
 /// \returns Error::success() if the operation succeeded, otherwise an
 /// appropriate error code.
-Error mergeTypeAndIdRecords(MergingTypeTableBuilder &DestIds,
-                            MergingTypeTableBuilder &DestTypes,
-                            SmallVectorImpl<TypeIndex> &SourceToDest,
-                            const CVTypeArray &IdsAndTypes,
-                            std::optional<PCHMergerInfo> &PCHInfo);
+LLVM_ABI Error mergeTypeAndIdRecords(MergingTypeTableBuilder &DestIds,
+                                     MergingTypeTableBuilder &DestTypes,
+                                     SmallVectorImpl<TypeIndex> &SourceToDest,
+                                     const CVTypeArray &IdsAndTypes,
+                                     std::optional<PCHMergerInfo> &PCHInfo);
 
-Error mergeTypeAndIdRecords(GlobalTypeTableBuilder &DestIds,
-                            GlobalTypeTableBuilder &DestTypes,
-                            SmallVectorImpl<TypeIndex> &SourceToDest,
-                            const CVTypeArray &IdsAndTypes,
-                            ArrayRef<GloballyHashedType> Hashes,
-                            std::optional<PCHMergerInfo> &PCHInfo);
+LLVM_ABI Error mergeTypeAndIdRecords(GlobalTypeTableBuilder &DestIds,
+                                     GlobalTypeTableBuilder &DestTypes,
+                                     SmallVectorImpl<TypeIndex> &SourceToDest,
+                                     const CVTypeArray &IdsAndTypes,
+                                     ArrayRef<GloballyHashedType> Hashes,
+                                     std::optional<PCHMergerInfo> &PCHInfo);
 
-Error mergeTypeRecords(GlobalTypeTableBuilder &Dest,
-                       SmallVectorImpl<TypeIndex> &SourceToDest,
-                       const CVTypeArray &Types,
-                       ArrayRef<GloballyHashedType> Hashes,
-                       std::optional<PCHMergerInfo> &PCHInfo);
+LLVM_ABI Error mergeTypeRecords(GlobalTypeTableBuilder &Dest,
+                                SmallVectorImpl<TypeIndex> &SourceToDest,
+                                const CVTypeArray &Types,
+                                ArrayRef<GloballyHashedType> Hashes,
+                                std::optional<PCHMergerInfo> &PCHInfo);
 
-Error mergeIdRecords(GlobalTypeTableBuilder &Dest, ArrayRef<TypeIndex> Types,
-                     SmallVectorImpl<TypeIndex> &SourceToDest,
-                     const CVTypeArray &Ids,
-                     ArrayRef<GloballyHashedType> Hashes);
+LLVM_ABI Error mergeIdRecords(GlobalTypeTableBuilder &Dest,
+                              ArrayRef<TypeIndex> Types,
+                              SmallVectorImpl<TypeIndex> &SourceToDest,
+                              const CVTypeArray &Ids,
+                              ArrayRef<GloballyHashedType> Hashes);
 
 } // end namespace codeview
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeTableCollection.h b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeTableCollection.h
index 2a389b9ac34e..c927c2dddf33 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeTableCollection.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/CodeView/TypeTableCollection.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_CODEVIEW_TYPETABLECOLLECTION_H
 
 #include "llvm/DebugInfo/CodeView/TypeCollection.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/StringSaver.h"
 
 #include <vector>
@@ -17,7 +18,7 @@
 namespace llvm {
 namespace codeview {
 
-class TypeTableCollection : public TypeCollection {
+class LLVM_ABI TypeTableCollection : public TypeCollection {
 public:
   explicit TypeTableCollection(ArrayRef<ArrayRef<uint8_t>> Records);
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h
index 02b402e86d23..547d92c6ce00 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -87,11 +88,11 @@ class DWARFAbbreviationDeclaration {
     /// use the DWARFUnit to calculate the size of the Form, like for
     /// DW_AT_address and DW_AT_ref_addr, so this isn't just an accessor for
     /// the ByteSize member.
-    std::optional<int64_t> getByteSize(const DWARFUnit &U) const;
+    LLVM_ABI std::optional<int64_t> getByteSize(const DWARFUnit &U) const;
   };
   using AttributeSpecVector = SmallVector<AttributeSpec, 8>;
 
-  DWARFAbbreviationDeclaration();
+  LLVM_ABI DWARFAbbreviationDeclaration();
 
   uint32_t getCode() const { return Code; }
   uint8_t getCodeByteSize() const { return CodeByteSize; }
@@ -136,7 +137,8 @@ class DWARFAbbreviationDeclaration {
   ///
   /// \param attr DWARF attribute to search for.
   /// \returns Optional index of the attribute if found, std::nullopt otherwise.
-  std::optional<uint32_t> findAttributeIndex(dwarf::Attribute attr) const;
+  LLVM_ABI std::optional<uint32_t>
+  findAttributeIndex(dwarf::Attribute attr) const;
 
   /// Extract a DWARF form value from a DIE specified by DIE offset.
   ///
@@ -148,9 +150,9 @@ class DWARFAbbreviationDeclaration {
   /// \param Attr DWARF attribute to search for.
   /// \param U the DWARFUnit the contains the DIE.
   /// \returns Optional DWARF form value if the attribute was extracted.
-  std::optional<DWARFFormValue> getAttributeValue(const uint64_t DIEOffset,
-                                                  const dwarf::Attribute Attr,
-                                                  const DWARFUnit &U) const;
+  LLVM_ABI std::optional<DWARFFormValue>
+  getAttributeValue(const uint64_t DIEOffset, const dwarf::Attribute Attr,
+                    const DWARFUnit &U) const;
 
   /// Compute an offset from a DIE specified by DIE offset and attribute index.
   ///
@@ -159,8 +161,9 @@ class DWARFAbbreviationDeclaration {
   /// code in the .debug_info data.
   /// \param U the DWARFUnit the contains the DIE.
   /// \returns an offset of the attribute.
-  uint64_t getAttributeOffsetFromIndex(uint32_t AttrIndex, uint64_t DIEOffset,
-                                       const DWARFUnit &U) const;
+  LLVM_ABI uint64_t getAttributeOffsetFromIndex(uint32_t AttrIndex,
+                                                uint64_t DIEOffset,
+                                                const DWARFUnit &U) const;
 
   /// Extract a DWARF form value from a DIE speccified by attribute index and
   /// its offset.
@@ -169,17 +172,19 @@ class DWARFAbbreviationDeclaration {
   /// \param Offset offset of the attribute.
   /// \param U the DWARFUnit the contains the DIE.
   /// \returns Optional DWARF form value if the attribute was extracted.
-  std::optional<DWARFFormValue>
+  LLVM_ABI std::optional<DWARFFormValue>
   getAttributeValueFromOffset(uint32_t AttrIndex, uint64_t Offset,
                               const DWARFUnit &U) const;
 
-  llvm::Expected<ExtractState> extract(DataExtractor Data, uint64_t *OffsetPtr);
-  void dump(raw_ostream &OS) const;
+  LLVM_ABI llvm::Expected<ExtractState> extract(DataExtractor Data,
+                                                uint64_t *OffsetPtr);
+  LLVM_ABI void dump(raw_ostream &OS) const;
 
   // Return an optional byte size of all attribute data in this abbreviation
   // if a constant byte size can be calculated given a DWARFUnit. This allows
   // DWARF parsing to be faster as many DWARF DIEs have a fixed byte size.
-  std::optional<size_t> getFixedAttributesByteSize(const DWARFUnit &U) const;
+  LLVM_ABI std::optional<size_t>
+  getFixedAttributesByteSize(const DWARFUnit &U) const;
 
 private:
   void clear();
@@ -204,7 +209,7 @@ class DWARFAbbreviationDeclaration {
     /// \returns the size in bytes for all attribute data in this abbreviation.
     /// The returned size does not include bytes for the  ULEB128 abbreviation
     /// code
-    size_t getByteSize(const DWARFUnit &U) const;
+    LLVM_ABI size_t getByteSize(const DWARFUnit &U) const;
   };
 
   uint32_t Code;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
index cef5fa1f2ee5..87586eda9068 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
@@ -15,6 +15,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <utility>
 
@@ -29,7 +30,7 @@ class ScopedPrinter;
 /// DWARF 5 accelerator tables.
 /// TODO: Generalize the rest of the AppleAcceleratorTable interface and move it
 /// to this class.
-class DWARFAcceleratorTable {
+class LLVM_ABI DWARFAcceleratorTable {
 protected:
   DWARFDataExtractor AccelSection;
   DataExtractor StringSection;
@@ -97,7 +98,7 @@ class DWARFAcceleratorTable {
 
 /// This implements the Apple accelerator table format, a precursor of the
 /// DWARF 5 accelerator table format.
-class AppleAcceleratorTable : public DWARFAcceleratorTable {
+class LLVM_ABI AppleAcceleratorTable : public DWARFAcceleratorTable {
   struct Header {
     uint32_t Magic;
     uint16_t Version;
@@ -106,7 +107,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable {
     uint32_t HashCount;
     uint32_t HeaderDataLength;
 
-    void dump(ScopedPrinter &W) const;
+    LLVM_ABI void dump(ScopedPrinter &W) const;
   };
 
   struct HeaderData {
@@ -116,7 +117,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable {
     uint64_t DIEOffsetBase;
     SmallVector<std::pair<AtomType, Form>, 3> Atoms;
 
-    std::optional<uint64_t>
+    LLVM_ABI std::optional<uint64_t>
     extractOffset(std::optional<DWARFFormValue> Value) const;
   };
 
@@ -210,7 +211,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable {
 
 public:
   /// Apple-specific implementation of an Accelerator Entry.
-  class Entry final : public DWARFAcceleratorTable::Entry {
+  class LLVM_ABI Entry final : public DWARFAcceleratorTable::Entry {
     const AppleAcceleratorTable &Table;
 
     Entry(const AppleAcceleratorTable &Table);
@@ -244,8 +245,8 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable {
 
   public:
     /// Construct a new iterator for the entries at \p DataOffset.
-    SameNameIterator(const AppleAcceleratorTable &AccelTable,
-                     uint64_t DataOffset);
+    LLVM_ABI SameNameIterator(const AppleAcceleratorTable &AccelTable,
+                              uint64_t DataOffset);
 
     const Entry &operator*() {
       uint64_t OffsetCopy = Offset;
@@ -292,7 +293,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable {
 
     /// Reads the next Entry in the table, populating `Current`.
     /// If not possible (e.g. end of the section), becomes the end iterator.
-    void prepareNextEntryOrEnd();
+    LLVM_ABI void prepareNextEntryOrEnd();
 
     /// Reads the next string pointer and the entry count for that string,
     /// populating `NumEntriesToCome`.
@@ -301,7 +302,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable {
     void prepareNextStringOrEnd();
 
   public:
-    Iterator(const AppleAcceleratorTable &Table, bool SetEnd = false);
+    LLVM_ABI Iterator(const AppleAcceleratorTable &Table, bool SetEnd = false);
 
     Iterator &operator++() {
       prepareNextEntryOrEnd();
@@ -380,7 +381,7 @@ class AppleAcceleratorTable : public DWARFAcceleratorTable {
 /// The last segment consists of a list of entries, which is a 0-terminated list
 /// referenced by the name table and interpreted with the help of the
 /// abbreviation table.
-class DWARFDebugNames : public DWARFAcceleratorTable {
+class LLVM_ABI DWARFDebugNames : public DWARFAcceleratorTable {
 public:
   class NameIndex;
   class NameIterator;
@@ -400,8 +401,8 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
     uint32_t AugmentationStringSize;
     SmallString<8> AugmentationString;
 
-    Error extract(const DWARFDataExtractor &AS, uint64_t *Offset);
-    void dump(ScopedPrinter &W) const;
+    LLVM_ABI Error extract(const DWARFDataExtractor &AS, uint64_t *Offset);
+    LLVM_ABI void dump(ScopedPrinter &W) const;
   };
 
   /// Index attribute and its encoding.
@@ -430,11 +431,11 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
         : AbbrevOffset(AbbrevOffset), Code(Code), Tag(Tag),
           Attributes(std::move(Attributes)) {}
 
-    void dump(ScopedPrinter &W) const;
+    LLVM_ABI void dump(ScopedPrinter &W) const;
   };
 
   /// DWARF v5-specific implementation of an Accelerator Entry.
-  class Entry final : public DWARFAcceleratorTable::Entry {
+  class LLVM_ABI Entry final : public DWARFAcceleratorTable::Entry {
     const NameIndex *NameIdx;
     const Abbrev *Abbr;
 
@@ -510,7 +511,7 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
 
   /// Error returned by NameIndex::getEntry to report it has reached the end of
   /// the entry list.
-  class SentinelError : public ErrorInfo<SentinelError> {
+  class LLVM_ABI SentinelError : public ErrorInfo<SentinelError> {
   public:
     static char ID;
 
@@ -521,8 +522,8 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
 private:
   /// DenseMapInfo for struct Abbrev.
   struct AbbrevMapInfo {
-    static Abbrev getEmptyKey();
-    static Abbrev getTombstoneKey();
+    LLVM_ABI static Abbrev getEmptyKey();
+    LLVM_ABI static Abbrev getTombstoneKey();
     static unsigned getHashValue(uint32_t Code) {
       return DenseMapInfo<uint32_t>::getHashValue(Code);
     }
@@ -633,32 +634,32 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
     DWARFDebugNamesOffsets getOffsets() const { return Offsets; }
 
     /// Reads offset of compilation unit CU. CU is 0-based.
-    uint64_t getCUOffset(uint32_t CU) const;
+    LLVM_ABI uint64_t getCUOffset(uint32_t CU) const;
     uint32_t getCUCount() const { return Hdr.CompUnitCount; }
 
     /// Reads offset of local type unit TU, TU is 0-based.
-    uint64_t getLocalTUOffset(uint32_t TU) const;
+    LLVM_ABI uint64_t getLocalTUOffset(uint32_t TU) const;
     uint32_t getLocalTUCount() const { return Hdr.LocalTypeUnitCount; }
 
     /// Reads signature of foreign type unit TU. TU is 0-based.
-    uint64_t getForeignTUSignature(uint32_t TU) const;
+    LLVM_ABI uint64_t getForeignTUSignature(uint32_t TU) const;
     uint32_t getForeignTUCount() const { return Hdr.ForeignTypeUnitCount; }
 
     /// Reads an entry in the Bucket Array for the given Bucket. The returned
     /// value is a (1-based) index into the Names, StringOffsets and
     /// EntryOffsets arrays. The input Bucket index is 0-based.
-    uint32_t getBucketArrayEntry(uint32_t Bucket) const;
+    LLVM_ABI uint32_t getBucketArrayEntry(uint32_t Bucket) const;
     uint32_t getBucketCount() const { return Hdr.BucketCount; }
 
     /// Reads an entry in the Hash Array for the given Index. The input Index
     /// is 1-based.
-    uint32_t getHashArrayEntry(uint32_t Index) const;
+    LLVM_ABI uint32_t getHashArrayEntry(uint32_t Index) const;
 
     /// Reads an entry in the Name Table for the given Index. The Name Table
     /// consists of two arrays -- String Offsets and Entry Offsets. The returned
     /// offsets are relative to the starts of respective sections. Input Index
     /// is 1-based.
-    NameTableEntry getNameTableEntry(uint32_t Index) const;
+    LLVM_ABI NameTableEntry getNameTableEntry(uint32_t Index) const;
 
     uint32_t getNameCount() const { return Hdr.NameCount; }
 
@@ -666,7 +667,7 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
       return Abbrevs;
     }
 
-    Expected<Entry> getEntry(uint64_t *Offset) const;
+    LLVM_ABI Expected<Entry> getEntry(uint64_t *Offset) const;
 
     /// Returns the Entry at the relative `Offset` from the start of the Entry
     /// pool.
@@ -676,18 +677,18 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
     }
 
     /// Look up all entries in this Name Index matching \c Key.
-    iterator_range<ValueIterator> equal_range(StringRef Key) const;
+    LLVM_ABI iterator_range<ValueIterator> equal_range(StringRef Key) const;
 
     NameIterator begin() const { return NameIterator(this, 1); }
     NameIterator end() const { return NameIterator(this, getNameCount() + 1); }
 
-    Error extract();
+    LLVM_ABI Error extract();
     uint64_t getUnitOffset() const { return Base; }
     uint64_t getNextUnitOffset() const {
       return Base + dwarf::getUnitLengthFieldByteSize(Hdr.Format) +
              Hdr.UnitLength;
     }
-    void dump(ScopedPrinter &W) const;
+    LLVM_ABI void dump(ScopedPrinter &W) const;
 
     friend class DWARFDebugNames;
   };
@@ -719,7 +720,7 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
     std::optional<uint64_t> findEntryOffsetInCurrentIndex();
     bool findInCurrentIndex();
     void searchFromStartOfCurrentIndex();
-    void next();
+    LLVM_ABI void next();
 
     /// Set the iterator to the "end" state.
     void setEnd() { *this = ValueIterator(); }
@@ -728,11 +729,11 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
     /// Create a "begin" iterator for looping over all entries in the
     /// accelerator table matching Key. The iterator will run through all Name
     /// Indexes in the section in sequence.
-    ValueIterator(const DWARFDebugNames &AccelTable, StringRef Key);
+    LLVM_ABI ValueIterator(const DWARFDebugNames &AccelTable, StringRef Key);
 
     /// Create a "begin" iterator for looping over all entries in a specific
     /// Name Index. Other indices in the section will not be visited.
-    ValueIterator(const NameIndex &NI, StringRef Key);
+    LLVM_ABI ValueIterator(const NameIndex &NI, StringRef Key);
 
     /// End marker.
     ValueIterator() = default;
@@ -840,7 +841,7 @@ class DWARFDebugNames : public DWARFAcceleratorTable {
 /// Calculates the starting offsets for various sections within the
 /// .debug_names section.
 namespace dwarf {
-DWARFDebugNames::DWARFDebugNamesOffsets
+LLVM_ABI DWARFDebugNames::DWARFDebugNamesOffsets
 findDebugNamesOffsets(uint64_t EndOfHeaderOffset,
                       const DWARFDebugNames::Header &Hdr);
 }
@@ -849,7 +850,7 @@ findDebugNamesOffsets(uint64_t EndOfHeaderOffset,
 /// parameters, returns a substring of `Name` containing no template
 /// parameters.
 /// E.g.: StripTemplateParameters("foo<int>") = "foo".
-std::optional<StringRef> StripTemplateParameters(StringRef Name);
+LLVM_ABI std::optional<StringRef> StripTemplateParameters(StringRef Name);
 
 struct ObjCSelectorNames {
   /// For "-[A(Category) method:]", this would be "method:"
@@ -865,7 +866,8 @@ struct ObjCSelectorNames {
 /// If `Name` is the AT_name of a DIE which refers to an Objective-C selector,
 /// returns an instance of ObjCSelectorNames. The Selector and ClassName fields
 /// are guaranteed to be non-empty in the result.
-std::optional<ObjCSelectorNames> getObjCNamesIfSelector(StringRef Name);
+LLVM_ABI std::optional<ObjCSelectorNames>
+getObjCNamesIfSelector(StringRef Name);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
index f4d6c451cbe1..3af85cfe9043 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
@@ -11,6 +11,7 @@
 
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -68,8 +69,9 @@ struct DWARFAddressRange {
     return true;
   }
 
-  void dump(raw_ostream &OS, uint32_t AddressSize, DIDumpOptions DumpOpts = {},
-            const DWARFObject *Obj = nullptr) const;
+  LLVM_ABI void dump(raw_ostream &OS, uint32_t AddressSize,
+                     DIDumpOptions DumpOpts = {},
+                     const DWARFObject *Obj = nullptr) const;
 };
 
 inline bool operator<(const DWARFAddressRange &LHS,
@@ -82,7 +84,7 @@ inline bool operator==(const DWARFAddressRange &LHS,
   return std::tie(LHS.SectionIndex, LHS.LowPC, LHS.HighPC) == std::tie(RHS.SectionIndex, RHS.LowPC, RHS.HighPC);
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const DWARFAddressRange &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const DWARFAddressRange &R);
 
 /// DWARFAddressRangesVector - represents a set of absolute address ranges.
 using DWARFAddressRangesVector = std::vector<DWARFAddressRange>;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAttribute.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAttribute.h
index 104ac1b9ddfb..5d4b94b1d709 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAttribute.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFAttribute.h
@@ -11,6 +11,7 @@
 
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -40,11 +41,11 @@ struct DWARFAttribute {
   }
 
   /// Identify DWARF attributes that may contain a pointer to a location list.
-  static bool mayHaveLocationList(dwarf::Attribute Attr);
+  LLVM_ABI static bool mayHaveLocationList(dwarf::Attribute Attr);
 
   /// Identifies DWARF attributes that may contain a reference to a
   /// DWARF expression.
-  static bool mayHaveLocationExpr(dwarf::Attribute Attr);
+  LLVM_ABI static bool mayHaveLocationExpr(dwarf::Attribute Attr);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h
new file mode 100644
index 000000000000..24a0f389470d
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h
@@ -0,0 +1,161 @@
+//===- DWARFCFIProgram.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
+#define LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Error.h"
+#include "llvm/TargetParser/Triple.h"
+#include <map>
+#include <memory>
+#include <vector>
+
+namespace llvm {
+
+namespace dwarf {
+/// Represent a sequence of Call Frame Information instructions that, when read
+/// in order, construct a table mapping PC to frame state. This can also be
+/// referred to as "CFI rules" in DWARF literature to avoid confusion with
+/// computer programs in the broader sense, and in this context each instruction
+/// would be a rule to establish the mapping. Refer to pg. 172 in the DWARF5
+/// manual, "6.4.1 Structure of Call Frame Information".
+class CFIProgram {
+public:
+  static constexpr size_t MaxOperands = 3;
+  typedef SmallVector<uint64_t, MaxOperands> Operands;
+
+  /// An instruction consists of a DWARF CFI opcode and an optional sequence of
+  /// operands. If it refers to an expression, then this expression has its own
+  /// sequence of operations and operands handled separately by DWARFExpression.
+  struct Instruction {
+    Instruction(uint8_t Opcode) : Opcode(Opcode) {}
+
+    uint8_t Opcode;
+    Operands Ops;
+    // Associated DWARF expression in case this instruction refers to one
+    std::optional<DWARFExpression> Expression;
+
+    LLVM_ABI Expected<uint64_t> getOperandAsUnsigned(const CFIProgram &CFIP,
+                                                     uint32_t OperandIdx) const;
+
+    LLVM_ABI Expected<int64_t> getOperandAsSigned(const CFIProgram &CFIP,
+                                                  uint32_t OperandIdx) const;
+  };
+
+  using InstrList = std::vector<Instruction>;
+  using iterator = InstrList::iterator;
+  using const_iterator = InstrList::const_iterator;
+
+  iterator begin() { return Instructions.begin(); }
+  const_iterator begin() const { return Instructions.begin(); }
+  iterator end() { return Instructions.end(); }
+  const_iterator end() const { return Instructions.end(); }
+
+  unsigned size() const { return (unsigned)Instructions.size(); }
+  bool empty() const { return Instructions.empty(); }
+  uint64_t codeAlign() const { return CodeAlignmentFactor; }
+  int64_t dataAlign() const { return DataAlignmentFactor; }
+  Triple::ArchType triple() const { return Arch; }
+
+  CFIProgram(uint64_t CodeAlignmentFactor, int64_t DataAlignmentFactor,
+             Triple::ArchType Arch)
+      : CodeAlignmentFactor(CodeAlignmentFactor),
+        DataAlignmentFactor(DataAlignmentFactor), Arch(Arch) {}
+
+  /// Parse and store a sequence of CFI instructions from Data,
+  /// starting at *Offset and ending at EndOffset. *Offset is updated
+  /// to EndOffset upon successful parsing, or indicates the offset
+  /// where a problem occurred in case an error is returned.
+  LLVM_ABI Error parse(DWARFDataExtractor Data, uint64_t *Offset,
+                       uint64_t EndOffset);
+
+  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
+                     unsigned IndentLevel,
+                     std::optional<uint64_t> InitialLocation) const;
+
+  void addInstruction(const Instruction &I) { Instructions.push_back(I); }
+
+  /// Get a DWARF CFI call frame string for the given DW_CFA opcode.
+  LLVM_ABI StringRef callFrameString(unsigned Opcode) const;
+
+private:
+  std::vector<Instruction> Instructions;
+  const uint64_t CodeAlignmentFactor;
+  const int64_t DataAlignmentFactor;
+  Triple::ArchType Arch;
+
+  /// Convenience method to add a new instruction with the given opcode.
+  void addInstruction(uint8_t Opcode) {
+    Instructions.push_back(Instruction(Opcode));
+  }
+
+  /// Add a new single-operand instruction.
+  void addInstruction(uint8_t Opcode, uint64_t Operand1) {
+    Instructions.push_back(Instruction(Opcode));
+    Instructions.back().Ops.push_back(Operand1);
+  }
+
+  /// Add a new instruction that has two operands.
+  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2) {
+    Instructions.push_back(Instruction(Opcode));
+    Instructions.back().Ops.push_back(Operand1);
+    Instructions.back().Ops.push_back(Operand2);
+  }
+
+  /// Add a new instruction that has three operands.
+  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2,
+                      uint64_t Operand3) {
+    Instructions.push_back(Instruction(Opcode));
+    Instructions.back().Ops.push_back(Operand1);
+    Instructions.back().Ops.push_back(Operand2);
+    Instructions.back().Ops.push_back(Operand3);
+  }
+
+  /// Types of operands to CFI instructions
+  /// In DWARF, this type is implicitly tied to a CFI instruction opcode and
+  /// thus this type doesn't need to be explicitly written to the file (this is
+  /// not a DWARF encoding). The relationship of instrs to operand types can
+  /// be obtained from getOperandTypes() and is only used to simplify
+  /// instruction printing.
+  enum OperandType {
+    OT_Unset,
+    OT_None,
+    OT_Address,
+    OT_Offset,
+    OT_FactoredCodeOffset,
+    OT_SignedFactDataOffset,
+    OT_UnsignedFactDataOffset,
+    OT_Register,
+    OT_AddressSpace,
+    OT_Expression
+  };
+
+  /// Get the OperandType as a "const char *".
+  static const char *operandTypeString(OperandType OT);
+
+  /// Retrieve the array describing the types of operands according to the enum
+  /// above. This is indexed by opcode.
+  static ArrayRef<OperandType[MaxOperands]> getOperandTypes();
+
+  /// Print \p Opcode's operand number \p OperandIdx which has value \p Operand.
+  void printOperand(raw_ostream &OS, DIDumpOptions DumpOpts,
+                    const Instruction &Instr, unsigned OperandIdx,
+                    uint64_t Operand, std::optional<uint64_t> &Address) const;
+};
+
+} // end namespace dwarf
+
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h
index d449b7bed796..49fd8908873b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFCOMPILEUNIT_H
 
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -19,7 +20,7 @@ class raw_ostream;
 struct DIDumpOptions;
 struct DWARFSection;
 
-class DWARFCompileUnit : public DWARFUnit {
+class LLVM_ABI DWARFCompileUnit : public DWARFUnit {
 public:
   DWARFCompileUnit(DWARFContext &Context, const DWARFSection &Section,
                    const DWARFUnitHeader &Header, const DWARFDebugAbbrev *DA,
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
index b39fb6852d70..d0187e885a51 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
@@ -20,6 +20,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Error.h"
 #include "llvm/TargetParser/Host.h"
@@ -45,7 +46,7 @@ class DWARFUnitIndex;
 /// DWARFContext
 /// This data structure is the top level entity that deals with dwarf debug
 /// information parsing. The actual data is supplied through DWARFObj.
-class DWARFContext : public DIContext {
+class LLVM_ABI DWARFContext : public DIContext {
 public:
   /// DWARFContextState
   /// This structure contains all member variables for DWARFContext that need
@@ -100,10 +101,8 @@ class DWARFContext : public DIContext {
     virtual bool isThreadSafe() const = 0;
 
     /// Parse a macro[.dwo] or macinfo[.dwo] section.
-    std::unique_ptr<DWARFDebugMacro>
+    LLVM_ABI std::unique_ptr<DWARFDebugMacro>
     parseMacroOrMacinfo(MacroSecType SectionType);
-
-    virtual Error doWorkThreadSafely(function_ref<Error()> Work) = 0;
   };
   friend class DWARFContextState;
 
@@ -492,10 +491,6 @@ class DWARFContext : public DIContext {
   /// manually only for DWARF5.
   void setParseCUTUIndexManually(bool PCUTU) { ParseCUTUIndexManually = PCUTU; }
 
-  Error doWorkThreadSafely(function_ref<Error()> Work) {
-    return State->doWorkThreadSafely(Work);
-  }
-
 private:
   void addLocalsForDie(DWARFCompileUnit *CU, DWARFDie Subprogram, DWARFDie Die,
                        std::vector<DILocal> &Result);
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
index e94fa7bf5590..005ccd208cdd 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
@@ -10,79 +10,59 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTOR_H
 
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h"
+#include "llvm/DebugInfo/DWARF/DWARFObject.h"
+#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h"
 #include "llvm/DebugInfo/DWARF/DWARFSection.h"
-#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
-class DWARFObject;
 
-/// A DataExtractor (typically for an in-memory copy of an object-file section)
-/// plus a relocation map for that section, if there is one.
-class DWARFDataExtractor : public DataExtractor {
+/// A DWARFDataExtractor (typically for an in-memory copy of an object-file
+/// section) plus a relocation map for that section, if there is one.
+class DWARFDataExtractor : public DWARFDataExtractorBase<DWARFDataExtractor> {
   const DWARFObject *Obj = nullptr;
   const DWARFSection *Section = nullptr;
 
 public:
+  using DWARFDataExtractorBase::DWARFDataExtractorBase;
+
   /// Constructor for the normal case of extracting data from a DWARF section.
   /// The DWARFSection's lifetime must be at least as long as the extractor's.
   DWARFDataExtractor(const DWARFObject &Obj, const DWARFSection &Section,
                      bool IsLittleEndian, uint8_t AddressSize)
-      : DataExtractor(Section.Data, IsLittleEndian, AddressSize), Obj(&Obj),
-        Section(&Section) {}
-
-  /// Constructor for cases when there are no relocations.
-  DWARFDataExtractor(StringRef Data, bool IsLittleEndian, uint8_t AddressSize)
-    : DataExtractor(Data, IsLittleEndian, AddressSize) {}
-  DWARFDataExtractor(ArrayRef<uint8_t> Data, bool IsLittleEndian,
-                     uint8_t AddressSize)
-      : DataExtractor(
-            StringRef(reinterpret_cast<const char *>(Data.data()), Data.size()),
-            IsLittleEndian, AddressSize) {}
+      : DWARFDataExtractorBase(Section.Data, IsLittleEndian, AddressSize),
+        Obj(&Obj), Section(&Section) {}
 
   /// Truncating constructor
   DWARFDataExtractor(const DWARFDataExtractor &Other, size_t Length)
-      : DataExtractor(Other.getData().substr(0, Length), Other.isLittleEndian(),
-                      Other.getAddressSize()),
+      : DWARFDataExtractorBase(Other.getData().substr(0, Length),
+                               Other.isLittleEndian(), Other.getAddressSize()),
         Obj(Other.Obj), Section(Other.Section) {}
 
-  /// Extracts the DWARF "initial length" field, which can either be a 32-bit
-  /// value smaller than 0xfffffff0, or the value 0xffffffff followed by a
-  /// 64-bit length. Returns the actual length, and the DWARF format which is
-  /// encoded in the field. In case of errors, it returns {0, DWARF32} and
-  /// leaves the offset unchanged.
-  std::pair<uint64_t, dwarf::DwarfFormat>
-  getInitialLength(uint64_t *Off, Error *Err = nullptr) const;
-
-  std::pair<uint64_t, dwarf::DwarfFormat> getInitialLength(Cursor &C) const {
-    return getInitialLength(&getOffset(C), &getError(C));
-  }
-
   /// Extracts a value and applies a relocation to the result if
   /// one exists for the given offset.
-  uint64_t getRelocatedValue(uint32_t Size, uint64_t *Off,
-                             uint64_t *SectionIndex = nullptr,
-                             Error *Err = nullptr) const;
-  uint64_t getRelocatedValue(Cursor &C, uint32_t Size,
-                             uint64_t *SectionIndex = nullptr) const {
-    return getRelocatedValue(Size, &getOffset(C), SectionIndex, &getError(C));
-  }
+  LLVM_ABI uint64_t getRelocatedValueImpl(uint32_t Size, uint64_t *Off,
+                                          uint64_t *SecNdx, Error *Err) const {
+    if (SecNdx)
+      *SecNdx = object::SectionedAddress::UndefSection;
+    if (!Section)
+      return getUnsigned(Off, Size, Err);
+    ErrorAsOutParameter ErrAsOut(Err);
+    std::optional<RelocAddrEntry> E = Obj->find(*Section, *Off);
+    uint64_t LocData = getUnsigned(Off, Size, Err);
+    if (!E || (Err && *Err))
+      return LocData;
+    if (SecNdx)
+      *SecNdx = E->SectionIndex;
 
-  /// Extracts an address-sized value and applies a relocation to the result if
-  /// one exists for the given offset.
-  uint64_t getRelocatedAddress(uint64_t *Off, uint64_t *SecIx = nullptr) const {
-    return getRelocatedValue(getAddressSize(), Off, SecIx);
+    uint64_t R = object::resolveRelocation(E->Resolver, E->Reloc,
+                                           E->SymbolValue, LocData);
+    if (E->Reloc2)
+      R = object::resolveRelocation(E->Resolver, *E->Reloc2, E->SymbolValue2,
+                                    R);
+    return R;
   }
-  uint64_t getRelocatedAddress(Cursor &C, uint64_t *SecIx = nullptr) const {
-    return getRelocatedValue(getAddressSize(), &getOffset(C), SecIx,
-                             &getError(C));
-  }
-
-  /// Extracts a DWARF-encoded pointer in \p Offset using \p Encoding.
-  /// There is a DWARF encoding that uses a PC-relative adjustment.
-  /// For these values, \p AbsPosOffset is used to fix them, which should
-  /// reflect the absolute address of this pointer.
-  std::optional<uint64_t> getEncodedPointer(uint64_t *Offset, uint8_t Encoding,
-                                            uint64_t AbsPosOffset = 0) const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h
new file mode 100644
index 000000000000..4c9a81093ca8
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h
@@ -0,0 +1,198 @@
+//===- DWARFDataExtractorSimple.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTORSIMPLE_H
+#define LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTORSIMPLE_H
+
+#include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/MathExtras.h"
+
+namespace llvm {
+
+/// A DataExtractor suitable use for parsing dwarf from memory.  Clients use
+/// Relocator::getRelocatedValueImpl to relocate values as appropriate.
+
+template <typename Relocator>
+class DWARFDataExtractorBase : public DataExtractor {
+
+public:
+  DWARFDataExtractorBase(StringRef Data, bool IsLittleEndian,
+                         uint8_t AddressSize)
+      : DataExtractor(Data, IsLittleEndian, AddressSize) {}
+  DWARFDataExtractorBase(ArrayRef<uint8_t> Data, bool IsLittleEndian,
+                         uint8_t AddressSize)
+      : DataExtractor(
+            StringRef(reinterpret_cast<const char *>(Data.data()), Data.size()),
+            IsLittleEndian, AddressSize) {}
+
+  /// Truncating constructor
+  DWARFDataExtractorBase(const DWARFDataExtractorBase &Other, size_t Length)
+      : DataExtractor(Other.getData().substr(0, Length), Other.isLittleEndian(),
+                      Other.getAddressSize()) {}
+
+  /// Extracts a value and returns it as adjusted by the Relocator
+  LLVM_ABI uint64_t getRelocatedValue(uint32_t Size, uint64_t *Off,
+                                      uint64_t *SectionIndex = nullptr,
+                                      Error *Err = nullptr) const {
+    return static_cast<const Relocator *>(this)->getRelocatedValueImpl(
+        Size, Off, SectionIndex, Err);
+  }
+
+  LLVM_ABI uint64_t getRelocatedValue(Cursor &C, uint32_t Size,
+                                      uint64_t *SectionIndex = nullptr) const {
+    return getRelocatedValue(Size, &getOffset(C), SectionIndex, &getError(C));
+  }
+
+  /// Extracts an address-sized value.
+  LLVM_ABI uint64_t getRelocatedAddress(uint64_t *Off,
+                                        uint64_t *SecIx = nullptr) const {
+    return getRelocatedValue(getAddressSize(), Off, SecIx);
+  }
+
+  LLVM_ABI uint64_t getRelocatedAddress(Cursor &C,
+                                        uint64_t *SecIx = nullptr) const {
+    return getRelocatedValue(getAddressSize(), &getOffset(C), SecIx,
+                             &getError(C));
+  }
+
+  /// Extracts the DWARF "initial length" field, which can either be a 32-bit
+  /// value smaller than 0xfffffff0, or the value 0xffffffff followed by a
+  /// 64-bit length. Returns the actual length, and the DWARF format which is
+  /// encoded in the field. In case of errors, it returns {0, DWARF32} and
+  /// leaves the offset unchanged.
+  LLVM_ABI std::pair<uint64_t, dwarf::DwarfFormat>
+  getInitialLength(uint64_t *Off, Error *Err = nullptr) const {
+    ErrorAsOutParameter ErrAsOut(Err);
+    if (Err && *Err)
+      return {0, dwarf::DWARF32};
+
+    Cursor C(*Off);
+    uint64_t Length = getRelocatedValue(C, 4);
+    dwarf::DwarfFormat Format = dwarf::DWARF32;
+    if (Length == dwarf::DW_LENGTH_DWARF64) {
+      Length = getRelocatedValue(C, 8);
+      Format = dwarf::DWARF64;
+    } else if (Length >= dwarf::DW_LENGTH_lo_reserved) {
+      cantFail(C.takeError());
+      if (Err)
+        *Err = createStringError(
+            std::errc::invalid_argument,
+            "unsupported reserved unit length of value 0x%8.8" PRIx64, Length);
+      return {0, dwarf::DWARF32};
+    }
+
+    if (C) {
+      *Off = C.tell();
+      return {Length, Format};
+    }
+    if (Err)
+      *Err = C.takeError();
+    else
+      consumeError(C.takeError());
+    return {0, dwarf::DWARF32};
+  }
+
+  LLVM_ABI std::pair<uint64_t, dwarf::DwarfFormat>
+  getInitialLength(Cursor &C) const {
+    return getInitialLength(&getOffset(C), &getError(C));
+  }
+
+  /// Extracts a DWARF-encoded pointer in \p Offset using \p Encoding.
+  /// There is a DWARF encoding that uses a PC-relative adjustment.
+  /// For these values, \p AbsPosOffset is used to fix them, which should
+  /// reflect the absolute address of this pointer.
+  LLVM_ABI std::optional<uint64_t>
+  getEncodedPointer(uint64_t *Offset, uint8_t Encoding,
+                    uint64_t PCRelOffset) const {
+    if (Encoding == dwarf::DW_EH_PE_omit)
+      return std::nullopt;
+
+    uint64_t Result = 0;
+    uint64_t OldOffset = *Offset;
+    // First get value
+    switch (Encoding & 0x0F) {
+    case dwarf::DW_EH_PE_absptr:
+      switch (getAddressSize()) {
+      case 2:
+      case 4:
+      case 8:
+        Result = getUnsigned(Offset, getAddressSize());
+        break;
+      default:
+        return std::nullopt;
+      }
+      break;
+    case dwarf::DW_EH_PE_uleb128:
+      Result = getULEB128(Offset);
+      break;
+    case dwarf::DW_EH_PE_sleb128:
+      Result = getSLEB128(Offset);
+      break;
+    case dwarf::DW_EH_PE_udata2:
+      Result = getUnsigned(Offset, 2);
+      break;
+    case dwarf::DW_EH_PE_udata4:
+      Result = getUnsigned(Offset, 4);
+      break;
+    case dwarf::DW_EH_PE_udata8:
+      Result = getUnsigned(Offset, 8);
+      break;
+    case dwarf::DW_EH_PE_sdata2:
+      Result = getSigned(Offset, 2);
+      break;
+    case dwarf::DW_EH_PE_sdata4:
+      Result = SignExtend64<32>(getRelocatedValue(4, Offset));
+      break;
+    case dwarf::DW_EH_PE_sdata8:
+      Result = getRelocatedValue(8, Offset);
+      break;
+    default:
+      return std::nullopt;
+    }
+    // Then add relative offset, if required
+    switch (Encoding & 0x70) {
+    case dwarf::DW_EH_PE_absptr:
+      // do nothing
+      break;
+    case dwarf::DW_EH_PE_pcrel:
+      Result += PCRelOffset;
+      break;
+    case dwarf::DW_EH_PE_datarel:
+    case dwarf::DW_EH_PE_textrel:
+    case dwarf::DW_EH_PE_funcrel:
+    case dwarf::DW_EH_PE_aligned:
+    default:
+      *Offset = OldOffset;
+      return std::nullopt;
+    }
+
+    return Result;
+  }
+};
+
+// Non relocating, low-level dwarf-data extractor. Suitable for use from
+// libraries that cannot have build-time dependencies on relocation providers.
+
+class DWARFDataExtractorSimple
+    : public DWARFDataExtractorBase<DWARFDataExtractorSimple> {
+  using DWARFDataExtractorBase::DWARFDataExtractorBase;
+
+  LLVM_ABI uint64_t getRelocatedValueImpl(uint32_t Size, uint64_t *Off,
+                                          uint64_t *SectionIndex = nullptr,
+                                          Error *Err = nullptr) const {
+    assert(SectionIndex == nullptr &&
+           "DWARFDATAExtractorSimple cannot take section indices.");
+    return getUnsigned(Off, Size, Err);
+  }
+};
+
+} // end namespace llvm
+#endif // LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTOR_H
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h
index 6439827ef70f..28c33c02a69d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGABBREV_H
 
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include <cstdint>
 #include <map>
@@ -30,13 +31,13 @@ class DWARFAbbreviationDeclarationSet {
       std::vector<DWARFAbbreviationDeclaration>::const_iterator;
 
 public:
-  DWARFAbbreviationDeclarationSet();
+  LLVM_ABI DWARFAbbreviationDeclarationSet();
 
   uint64_t getOffset() const { return Offset; }
-  void dump(raw_ostream &OS) const;
-  Error extract(DataExtractor Data, uint64_t *OffsetPtr);
+  LLVM_ABI void dump(raw_ostream &OS) const;
+  LLVM_ABI Error extract(DataExtractor Data, uint64_t *OffsetPtr);
 
-  const DWARFAbbreviationDeclaration *
+  LLVM_ABI const DWARFAbbreviationDeclaration *
   getAbbreviationDeclaration(uint32_t AbbrCode) const;
 
   const_iterator begin() const {
@@ -47,7 +48,7 @@ class DWARFAbbreviationDeclarationSet {
     return Decls.end();
   }
 
-  std::string getCodeRange() const;
+  LLVM_ABI std::string getCodeRange() const;
 
   uint32_t getFirstAbbrCode() const { return FirstAbbrCode; }
 
@@ -64,13 +65,13 @@ class DWARFDebugAbbrev {
   mutable std::optional<DataExtractor> Data;
 
 public:
-  DWARFDebugAbbrev(DataExtractor Data);
+  LLVM_ABI DWARFDebugAbbrev(DataExtractor Data);
 
-  Expected<const DWARFAbbreviationDeclarationSet *>
+  LLVM_ABI Expected<const DWARFAbbreviationDeclarationSet *>
   getAbbreviationDeclarationSet(uint64_t CUAbbrOffset) const;
 
-  void dump(raw_ostream &OS) const;
-  Error parse() const;
+  LLVM_ABI void dump(raw_ostream &OS) const;
+  LLVM_ABI Error parse() const;
 
   DWARFAbbreviationDeclarationSetMap::const_iterator begin() const {
     assert(!Data && "Must call parse before iterating over DWARFDebugAbbrev");
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
index 942de05f6bd8..6631692a93c0 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <vector>
@@ -46,7 +47,7 @@ class DWARFDebugArangeSet {
     uint64_t Length;
 
     uint64_t getEndAddress() const { return Address + Length; }
-    void dump(raw_ostream &OS, uint32_t AddressSize) const;
+    LLVM_ABI void dump(raw_ostream &OS, uint32_t AddressSize) const;
   };
 
 private:
@@ -60,10 +61,10 @@ class DWARFDebugArangeSet {
 public:
   DWARFDebugArangeSet() { clear(); }
 
-  void clear();
-  Error extract(DWARFDataExtractor data, uint64_t *offset_ptr,
-                function_ref<void(Error)> WarningHandler = nullptr);
-  void dump(raw_ostream &OS) const;
+  LLVM_ABI void clear();
+  LLVM_ABI Error extract(DWARFDataExtractor data, uint64_t *offset_ptr,
+                         function_ref<void(Error)> WarningHandler = nullptr);
+  LLVM_ABI void dump(raw_ostream &OS) const;
 
   uint64_t getCompileUnitDIEOffset() const { return HeaderData.CuOffset; }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
index a9a3c7edde69..051ea6e11e35 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
@@ -12,7 +12,9 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/iterator.h"
+#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/TargetParser/Triple.h"
 #include <map>
@@ -92,12 +94,12 @@ class UnwindLocation {
   /// Create a location whose rule is set to Unspecified. This means the
   /// register value might be in the same register but it wasn't specified in
   /// the unwind opcodes.
-  static UnwindLocation createUnspecified();
+  LLVM_ABI static UnwindLocation createUnspecified();
   /// Create a location where the value is undefined and not available. This can
   /// happen when a register is volatile and can't be recovered.
-  static UnwindLocation createUndefined();
+  LLVM_ABI static UnwindLocation createUndefined();
   /// Create a location where the value is known to be in the register itself.
-  static UnwindLocation createSame();
+  LLVM_ABI static UnwindLocation createSame();
   /// Create a location that is in (Deref == false) or at (Deref == true) the
   /// CFA plus an offset. Most registers that are spilled onto the stack use
   /// this rule. The rule for the register will use this rule and specify a
@@ -105,8 +107,8 @@ class UnwindLocation {
   /// relative to a CFA value which is typically defined using the register
   /// plus offset location. \see createRegisterPlusOffset(...) for more
   /// information.
-  static UnwindLocation createIsCFAPlusOffset(int32_t Off);
-  static UnwindLocation createAtCFAPlusOffset(int32_t Off);
+  LLVM_ABI static UnwindLocation createIsCFAPlusOffset(int32_t Off);
+  LLVM_ABI static UnwindLocation createAtCFAPlusOffset(int32_t Off);
   /// Create a location where the saved value is in (Deref == false) or at
   /// (Deref == true) a regiser plus an offset and, optionally, in the specified
   /// address space (used mostly for the CFA).
@@ -115,18 +117,18 @@ class UnwindLocation {
   /// frame pointer as the register, with an offset that accounts for all
   /// spilled registers and all local variables in a function, and Deref ==
   /// false.
-  static UnwindLocation
+  LLVM_ABI static UnwindLocation
   createIsRegisterPlusOffset(uint32_t Reg, int32_t Off,
                              std::optional<uint32_t> AddrSpace = std::nullopt);
-  static UnwindLocation
+  LLVM_ABI static UnwindLocation
   createAtRegisterPlusOffset(uint32_t Reg, int32_t Off,
                              std::optional<uint32_t> AddrSpace = std::nullopt);
   /// Create a location whose value is the result of evaluating a DWARF
   /// expression. This allows complex expressions to be evaluated in order to
   /// unwind a register or CFA value.
-  static UnwindLocation createIsDWARFExpression(DWARFExpression Expr);
-  static UnwindLocation createAtDWARFExpression(DWARFExpression Expr);
-  static UnwindLocation createIsConstant(int32_t Value);
+  LLVM_ABI static UnwindLocation createIsDWARFExpression(DWARFExpression Expr);
+  LLVM_ABI static UnwindLocation createAtDWARFExpression(DWARFExpression Expr);
+  LLVM_ABI static UnwindLocation createIsConstant(int32_t Value);
 
   Location getLocation() const { return Kind; }
   uint32_t getRegister() const { return RegNum; }
@@ -164,12 +166,12 @@ class UnwindLocation {
   /// instead of from .debug_frame. This is needed for register number
   /// conversion because some register numbers differ between the two sections
   /// for certain architectures like x86.
-  void dump(raw_ostream &OS, DIDumpOptions DumpOpts) const;
+  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts) const;
 
-  bool operator==(const UnwindLocation &RHS) const;
+  LLVM_ABI bool operator==(const UnwindLocation &RHS) const;
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const UnwindLocation &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindLocation &R);
 
 /// A class that can track all registers with locations in a UnwindRow object.
 ///
@@ -222,7 +224,7 @@ class RegisterLocations {
   /// instead of from .debug_frame. This is needed for register number
   /// conversion because some register numbers differ between the two sections
   /// for certain architectures like x86.
-  void dump(raw_ostream &OS, DIDumpOptions DumpOpts) const;
+  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts) const;
 
   /// Returns true if we have any register locations in this object.
   bool hasLocations() const { return !Locations.empty(); }
@@ -234,7 +236,7 @@ class RegisterLocations {
   }
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const RegisterLocations &RL);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RegisterLocations &RL);
 
 /// A class that represents a single row in the unwind table that is decoded by
 /// parsing the DWARF Call Frame Information opcodes.
@@ -303,13 +305,12 @@ class UnwindRow {
   ///
   /// \param IndentLevel specify the indent level as an integer. The UnwindRow
   /// will be output to the stream preceded by 2 * IndentLevel number of spaces.
-  void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-            unsigned IndentLevel = 0) const;
+  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
+                     unsigned IndentLevel = 0) const;
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const UnwindRow &Row);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindRow &Row);
 
-class CFIProgram;
 class CIE;
 class FDE;
 
@@ -348,8 +349,8 @@ class UnwindTable {
   ///
   /// \param IndentLevel specify the indent level as an integer. The UnwindRow
   /// will be output to the stream preceded by 2 * IndentLevel number of spaces.
-  void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-            unsigned IndentLevel = 0) const;
+  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
+                     unsigned IndentLevel = 0) const;
 
   /// Create an UnwindTable from a Common Information Entry (CIE).
   ///
@@ -359,7 +360,7 @@ class UnwindTable {
   ///
   /// \returns An error if the DWARF Call Frame Information opcodes have state
   /// machine errors, or a valid UnwindTable otherwise.
-  static Expected<UnwindTable> create(const CIE *Cie);
+  LLVM_ABI static Expected<UnwindTable> create(const CIE *Cie);
 
   /// Create an UnwindTable from a Frame Descriptor Entry (FDE).
   ///
@@ -369,7 +370,7 @@ class UnwindTable {
   ///
   /// \returns An error if the DWARF Call Frame Information opcodes have state
   /// machine errors, or a valid UnwindTable otherwise.
-  static Expected<UnwindTable> create(const FDE *Fde);
+  LLVM_ABI static Expected<UnwindTable> create(const FDE *Fde);
 
 private:
   RowContainer Rows;
@@ -396,136 +397,7 @@ class UnwindTable {
                   const RegisterLocations *InitialLocs);
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const UnwindTable &Rows);
-
-/// Represent a sequence of Call Frame Information instructions that, when read
-/// in order, construct a table mapping PC to frame state. This can also be
-/// referred to as "CFI rules" in DWARF literature to avoid confusion with
-/// computer programs in the broader sense, and in this context each instruction
-/// would be a rule to establish the mapping. Refer to pg. 172 in the DWARF5
-/// manual, "6.4.1 Structure of Call Frame Information".
-class CFIProgram {
-public:
-  static constexpr size_t MaxOperands = 3;
-  typedef SmallVector<uint64_t, MaxOperands> Operands;
-
-  /// An instruction consists of a DWARF CFI opcode and an optional sequence of
-  /// operands. If it refers to an expression, then this expression has its own
-  /// sequence of operations and operands handled separately by DWARFExpression.
-  struct Instruction {
-    Instruction(uint8_t Opcode) : Opcode(Opcode) {}
-
-    uint8_t Opcode;
-    Operands Ops;
-    // Associated DWARF expression in case this instruction refers to one
-    std::optional<DWARFExpression> Expression;
-
-    Expected<uint64_t> getOperandAsUnsigned(const CFIProgram &CFIP,
-                                            uint32_t OperandIdx) const;
-
-    Expected<int64_t> getOperandAsSigned(const CFIProgram &CFIP,
-                                         uint32_t OperandIdx) const;
-  };
-
-  using InstrList = std::vector<Instruction>;
-  using iterator = InstrList::iterator;
-  using const_iterator = InstrList::const_iterator;
-
-  iterator begin() { return Instructions.begin(); }
-  const_iterator begin() const { return Instructions.begin(); }
-  iterator end() { return Instructions.end(); }
-  const_iterator end() const { return Instructions.end(); }
-
-  unsigned size() const { return (unsigned)Instructions.size(); }
-  bool empty() const { return Instructions.empty(); }
-  uint64_t codeAlign() const { return CodeAlignmentFactor; }
-  int64_t dataAlign() const { return DataAlignmentFactor; }
-  Triple::ArchType triple() const { return Arch; }
-
-  CFIProgram(uint64_t CodeAlignmentFactor, int64_t DataAlignmentFactor,
-             Triple::ArchType Arch)
-      : CodeAlignmentFactor(CodeAlignmentFactor),
-        DataAlignmentFactor(DataAlignmentFactor),
-        Arch(Arch) {}
-
-  /// Parse and store a sequence of CFI instructions from Data,
-  /// starting at *Offset and ending at EndOffset. *Offset is updated
-  /// to EndOffset upon successful parsing, or indicates the offset
-  /// where a problem occurred in case an error is returned.
-  Error parse(DWARFDataExtractor Data, uint64_t *Offset, uint64_t EndOffset);
-
-  void dump(raw_ostream &OS, DIDumpOptions DumpOpts, unsigned IndentLevel,
-            std::optional<uint64_t> InitialLocation) const;
-
-  void addInstruction(const Instruction &I) { Instructions.push_back(I); }
-
-  /// Get a DWARF CFI call frame string for the given DW_CFA opcode.
-  StringRef callFrameString(unsigned Opcode) const;
-
-private:
-  std::vector<Instruction> Instructions;
-  const uint64_t CodeAlignmentFactor;
-  const int64_t DataAlignmentFactor;
-  Triple::ArchType Arch;
-
-  /// Convenience method to add a new instruction with the given opcode.
-  void addInstruction(uint8_t Opcode) {
-    Instructions.push_back(Instruction(Opcode));
-  }
-
-  /// Add a new single-operand instruction.
-  void addInstruction(uint8_t Opcode, uint64_t Operand1) {
-    Instructions.push_back(Instruction(Opcode));
-    Instructions.back().Ops.push_back(Operand1);
-  }
-
-  /// Add a new instruction that has two operands.
-  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2) {
-    Instructions.push_back(Instruction(Opcode));
-    Instructions.back().Ops.push_back(Operand1);
-    Instructions.back().Ops.push_back(Operand2);
-  }
-
-  /// Add a new instruction that has three operands.
-  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2,
-                      uint64_t Operand3) {
-    Instructions.push_back(Instruction(Opcode));
-    Instructions.back().Ops.push_back(Operand1);
-    Instructions.back().Ops.push_back(Operand2);
-    Instructions.back().Ops.push_back(Operand3);
-  }
-
-  /// Types of operands to CFI instructions
-  /// In DWARF, this type is implicitly tied to a CFI instruction opcode and
-  /// thus this type doesn't need to be explicitly written to the file (this is
-  /// not a DWARF encoding). The relationship of instrs to operand types can
-  /// be obtained from getOperandTypes() and is only used to simplify
-  /// instruction printing.
-  enum OperandType {
-    OT_Unset,
-    OT_None,
-    OT_Address,
-    OT_Offset,
-    OT_FactoredCodeOffset,
-    OT_SignedFactDataOffset,
-    OT_UnsignedFactDataOffset,
-    OT_Register,
-    OT_AddressSpace,
-    OT_Expression
-  };
-
-  /// Get the OperandType as a "const char *".
-  static const char *operandTypeString(OperandType OT);
-
-  /// Retrieve the array describing the types of operands according to the enum
-  /// above. This is indexed by opcode.
-  static ArrayRef<OperandType[MaxOperands]> getOperandTypes();
-
-  /// Print \p Opcode's operand number \p OperandIdx which has value \p Operand.
-  void printOperand(raw_ostream &OS, DIDumpOptions DumpOpts,
-                    const Instruction &Instr, unsigned OperandIdx,
-                    uint64_t Operand, std::optional<uint64_t> &Address) const;
-};
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindTable &Rows);
 
 /// An entry in either debug_frame or eh_frame. This entry can be a CIE or an
 /// FDE.
@@ -564,7 +436,7 @@ class FrameEntry {
 };
 
 /// DWARF Common Information Entry (CIE)
-class CIE : public FrameEntry {
+class LLVM_ABI CIE : public FrameEntry {
 public:
   // CIEs (and FDEs) are simply container classes, so the only sensible way to
   // create them is by providing the full parsed contents in the constructor.
@@ -626,7 +498,7 @@ class CIE : public FrameEntry {
 };
 
 /// DWARF Frame Description Entry (FDE)
-class FDE : public FrameEntry {
+class LLVM_ABI FDE : public FrameEntry {
 public:
   FDE(bool IsDWARF64, uint64_t Offset, uint64_t Length, uint64_t CIEPointer,
       uint64_t InitialLocation, uint64_t AddressRange, CIE *Cie,
@@ -682,17 +554,17 @@ class DWARFDebugFrame {
   // it is a .debug_frame section. EHFrameAddress should be different
   // than zero for correct parsing of .eh_frame addresses when they
   // use a PC-relative encoding.
-  DWARFDebugFrame(Triple::ArchType Arch,
-                  bool IsEH = false, uint64_t EHFrameAddress = 0);
-  ~DWARFDebugFrame();
+  LLVM_ABI DWARFDebugFrame(Triple::ArchType Arch, bool IsEH = false,
+                           uint64_t EHFrameAddress = 0);
+  LLVM_ABI ~DWARFDebugFrame();
 
   /// Dump the section data into the given stream.
-  void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-            std::optional<uint64_t> Offset) const;
+  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
+                     std::optional<uint64_t> Offset) const;
 
   /// Parse the section from raw data. \p Data is assumed to contain the whole
   /// frame section contents to be parsed.
-  Error parse(DWARFDataExtractor Data);
+  LLVM_ABI Error parse(DWARFDataExtractor Data);
 
   /// Return whether the section has any entries.
   bool empty() const { return Entries.empty(); }
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
index 7d59f4257b2e..56f2fdb8ac44 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
@@ -11,6 +11,7 @@
 
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -38,9 +39,9 @@ class DWARFDebugInfoEntry {
   /// starting at a given offset. If DIE can't be extracted, returns false and
   /// doesn't change OffsetPtr.
   /// High performance extraction should use this call.
-  bool extractFast(const DWARFUnit &U, uint64_t *OffsetPtr,
-                   const DWARFDataExtractor &DebugInfoData, uint64_t UEndOffset,
-                   uint32_t ParentIdx);
+  LLVM_ABI bool extractFast(const DWARFUnit &U, uint64_t *OffsetPtr,
+                            const DWARFDataExtractor &DebugInfoData,
+                            uint64_t UEndOffset, uint32_t ParentIdx);
 
   uint64_t getOffset() const { return Offset; }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
index 61f17a27b3d2..79f1af034ccd 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
@@ -14,6 +14,7 @@
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/Path.h"
 #include <cstdint>
@@ -53,11 +54,11 @@ class DWARFDebugLine {
     bool HasSource = false;
 
     /// Update tracked content types with \p ContentType.
-    void trackContentType(dwarf::LineNumberEntryFormat ContentType);
+    LLVM_ABI void trackContentType(dwarf::LineNumberEntryFormat ContentType);
   };
 
   struct Prologue {
-    Prologue();
+    LLVM_ABI Prologue();
 
     /// The size in bytes of the statement information for this compilation unit
     /// (not including the total_length field itself).
@@ -101,43 +102,43 @@ class DWARFDebugLine {
 
     uint32_t sizeofPrologueLength() const { return isDWARF64() ? 8 : 4; }
 
-    bool totalLengthIsValid() const;
+    LLVM_ABI bool totalLengthIsValid() const;
 
     /// Length of the prologue in bytes.
-    uint64_t getLength() const;
+    LLVM_ABI uint64_t getLength() const;
 
     /// Get DWARF-version aware access to the file name entry at the provided
     /// index.
-    const llvm::DWARFDebugLine::FileNameEntry &
+    LLVM_ABI const llvm::DWARFDebugLine::FileNameEntry &
     getFileNameEntry(uint64_t Index) const;
 
-    bool hasFileAtIndex(uint64_t FileIndex) const;
+    LLVM_ABI bool hasFileAtIndex(uint64_t FileIndex) const;
 
-    std::optional<uint64_t> getLastValidFileIndex() const;
+    LLVM_ABI std::optional<uint64_t> getLastValidFileIndex() const;
 
-    bool
+    LLVM_ABI bool
     getFileNameByIndex(uint64_t FileIndex, StringRef CompDir,
                        DILineInfoSpecifier::FileLineInfoKind Kind,
                        std::string &Result,
                        sys::path::Style Style = sys::path::Style::native) const;
 
-    void clear();
-    void dump(raw_ostream &OS, DIDumpOptions DumpOptions) const;
-    Error parse(DWARFDataExtractor Data, uint64_t *OffsetPtr,
-                function_ref<void(Error)> RecoverableErrorHandler,
-                const DWARFContext &Ctx, const DWARFUnit *U = nullptr);
+    LLVM_ABI void clear();
+    LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOptions) const;
+    LLVM_ABI Error parse(DWARFDataExtractor Data, uint64_t *OffsetPtr,
+                         function_ref<void(Error)> RecoverableErrorHandler,
+                         const DWARFContext &Ctx, const DWARFUnit *U = nullptr);
   };
 
   /// Standard .debug_line state machine structure.
   struct Row {
-    explicit Row(bool DefaultIsStmt = false);
+    LLVM_ABI explicit Row(bool DefaultIsStmt = false);
 
     /// Called after a row is appended to the matrix.
-    void postAppend();
-    void reset(bool DefaultIsStmt);
-    void dump(raw_ostream &OS) const;
+    LLVM_ABI void postAppend();
+    LLVM_ABI void reset(bool DefaultIsStmt);
+    LLVM_ABI void dump(raw_ostream &OS) const;
 
-    static void dumpTableHeader(raw_ostream &OS, unsigned Indent);
+    LLVM_ABI static void dumpTableHeader(raw_ostream &OS, unsigned Indent);
 
     static bool orderByAddress(const Row &LHS, const Row &RHS) {
       return std::tie(LHS.Address.SectionIndex, LHS.Address.Address) <
@@ -195,7 +196,7 @@ class DWARFDebugLine {
   /// each compilation unit may consist of multiple sequences, which are not
   /// guaranteed to be in the order of ascending instruction address.
   struct Sequence {
-    Sequence();
+    LLVM_ABI Sequence();
 
     /// Sequence describes instructions at address range [LowPC, HighPC)
     /// and is described by line table rows [FirstRowIndex, LastRowIndex).
@@ -212,7 +213,7 @@ class DWARFDebugLine {
     /// The offset into the line table where this sequence begins
     uint64_t StmtSeqOffset = UINT64_MAX;
 
-    void reset();
+    LLVM_ABI void reset();
 
     static bool orderByHighPC(const Sequence &LHS, const Sequence &RHS) {
       return std::tie(LHS.SectionIndex, LHS.HighPC) <
@@ -230,7 +231,7 @@ class DWARFDebugLine {
   };
 
   struct LineTable {
-    LineTable();
+    LLVM_ABI LineTable();
 
     /// Represents an invalid row
     const uint32_t UnknownRowIndex = UINT32_MAX;
@@ -243,8 +244,8 @@ class DWARFDebugLine {
 
     /// Returns the index of the row with file/line info for a given address,
     /// or UnknownRowIndex if there is no such row.
-    uint32_t lookupAddress(object::SectionedAddress Address,
-                           bool *IsApproximateLine = nullptr) const;
+    LLVM_ABI uint32_t lookupAddress(object::SectionedAddress Address,
+                                    bool *IsApproximateLine = nullptr) const;
 
     /// Fills the Result argument with the indices of the rows that correspond
     /// to the address range specified by \p Address and \p Size.
@@ -256,7 +257,7 @@ class DWARFDebugLine {
     /// starting at the matching offset will be added to the result.
     ///
     /// Returns true if any rows were found.
-    bool lookupAddressRange(
+    LLVM_ABI bool lookupAddressRange(
         object::SectionedAddress Address, uint64_t Size,
         std::vector<uint32_t> &Result,
         std::optional<uint64_t> StmtSequenceOffset = std::nullopt) const;
@@ -282,24 +283,23 @@ class DWARFDebugLine {
 
     /// Fills the Result argument with the file and line information
     /// corresponding to Address. Returns true on success.
-    bool getFileLineInfoForAddress(object::SectionedAddress Address,
-                                   bool Approximate, const char *CompDir,
-                                   DILineInfoSpecifier::FileLineInfoKind Kind,
-                                   DILineInfo &Result) const;
+    LLVM_ABI bool getFileLineInfoForAddress(
+        object::SectionedAddress Address, bool Approximate, const char *CompDir,
+        DILineInfoSpecifier::FileLineInfoKind Kind, DILineInfo &Result) const;
 
     /// Extracts directory name by its Entry in include directories table
     /// in prologue. Returns true on success.
-    bool getDirectoryForEntry(const FileNameEntry &Entry,
-                              std::string &Directory) const;
+    LLVM_ABI bool getDirectoryForEntry(const FileNameEntry &Entry,
+                                       std::string &Directory) const;
 
-    void dump(raw_ostream &OS, DIDumpOptions DumpOptions) const;
-    void clear();
+    LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOptions) const;
+    LLVM_ABI void clear();
 
     /// Parse prologue and all rows.
-    Error parse(DWARFDataExtractor &DebugLineData, uint64_t *OffsetPtr,
-                const DWARFContext &Ctx, const DWARFUnit *U,
-                function_ref<void(Error)> RecoverableErrorHandler,
-                raw_ostream *OS = nullptr, bool Verbose = false);
+    LLVM_ABI Error parse(DWARFDataExtractor &DebugLineData, uint64_t *OffsetPtr,
+                         const DWARFContext &Ctx, const DWARFUnit *U,
+                         function_ref<void(Error)> RecoverableErrorHandler,
+                         raw_ostream *OS = nullptr, bool Verbose = false);
 
     using RowVector = std::vector<Row>;
     using RowIter = RowVector::const_iterator;
@@ -336,20 +336,20 @@ class DWARFDebugLine {
                            std::optional<uint64_t> StmtSequenceOffset) const;
   };
 
-  const LineTable *getLineTable(uint64_t Offset) const;
-  Expected<const LineTable *>
+  LLVM_ABI const LineTable *getLineTable(uint64_t Offset) const;
+  LLVM_ABI Expected<const LineTable *>
   getOrParseLineTable(DWARFDataExtractor &DebugLineData, uint64_t Offset,
                       const DWARFContext &Ctx, const DWARFUnit *U,
                       function_ref<void(Error)> RecoverableErrorHandler);
-  void clearLineTable(uint64_t Offset);
+  LLVM_ABI void clearLineTable(uint64_t Offset);
 
   /// Helper to allow for parsing of an entire .debug_line section in sequence.
   class SectionParser {
   public:
     using LineToUnitMap = std::map<uint64_t, DWARFUnit *>;
 
-    SectionParser(DWARFDataExtractor &Data, const DWARFContext &C,
-                  DWARFUnitVector::iterator_range Units);
+    LLVM_ABI SectionParser(DWARFDataExtractor &Data, const DWARFContext &C,
+                           DWARFUnitVector::iterator_range Units);
 
     /// Get the next line table from the section. Report any issues via the
     /// handlers.
@@ -362,9 +362,10 @@ class DWARFDebugLine {
     /// table as it parses it.
     /// \param Verbose - if true, the parser will print verbose information when
     /// printing to the output.
-    LineTable parseNext(function_ref<void(Error)> RecoverableErrorHandler,
-                        function_ref<void(Error)> UnrecoverableErrorHandler,
-                        raw_ostream *OS = nullptr, bool Verbose = false);
+    LLVM_ABI LineTable
+    parseNext(function_ref<void(Error)> RecoverableErrorHandler,
+              function_ref<void(Error)> UnrecoverableErrorHandler,
+              raw_ostream *OS = nullptr, bool Verbose = false);
 
     /// Skip the current line table and go to the following line table (if
     /// present) immediately.
@@ -373,8 +374,8 @@ class DWARFDebugLine {
     /// parsing issues via this handler.
     /// \param UnrecoverableErrorHandler - report any unrecoverable prologue
     /// parsing issues via this handler.
-    void skip(function_ref<void(Error)> RecoverableErrorHandler,
-              function_ref<void(Error)> UnrecoverableErrorHandler);
+    LLVM_ABI void skip(function_ref<void(Error)> RecoverableErrorHandler,
+                       function_ref<void(Error)> UnrecoverableErrorHandler);
 
     /// Indicates if the parser has parsed as much as possible.
     ///
@@ -400,11 +401,11 @@ class DWARFDebugLine {
 
 private:
   struct ParsingState {
-    ParsingState(struct LineTable *LT, uint64_t TableOffset,
-                 function_ref<void(Error)> ErrorHandler);
+    LLVM_ABI ParsingState(struct LineTable *LT, uint64_t TableOffset,
+                          function_ref<void(Error)> ErrorHandler);
 
-    void resetRowAndSequence(uint64_t Offset);
-    void appendRowToMatrix();
+    LLVM_ABI void resetRowAndSequence(uint64_t Offset);
+    LLVM_ABI void appendRowToMatrix();
 
     struct AddrOpIndexDelta {
       uint64_t AddrOffset;
@@ -413,8 +414,9 @@ class DWARFDebugLine {
 
     /// Advance the address and op-index by the \p OperationAdvance value.
     /// \returns the amount advanced by.
-    AddrOpIndexDelta advanceAddrOpIndex(uint64_t OperationAdvance,
-                                        uint8_t Opcode, uint64_t OpcodeOffset);
+    LLVM_ABI AddrOpIndexDelta advanceAddrOpIndex(uint64_t OperationAdvance,
+                                                 uint8_t Opcode,
+                                                 uint64_t OpcodeOffset);
 
     struct OpcodeAdvanceResults {
       uint64_t AddrDelta;
@@ -424,8 +426,8 @@ class DWARFDebugLine {
 
     /// Advance the address and op-index as required by the specified \p Opcode.
     /// \returns the amount advanced by and the calculated adjusted opcode.
-    OpcodeAdvanceResults advanceForOpcode(uint8_t Opcode,
-                                          uint64_t OpcodeOffset);
+    LLVM_ABI OpcodeAdvanceResults advanceForOpcode(uint8_t Opcode,
+                                                   uint64_t OpcodeOffset);
 
     struct SpecialOpcodeDelta {
       uint64_t Address;
@@ -435,8 +437,8 @@ class DWARFDebugLine {
 
     /// Advance the line, address and op-index as required by the specified
     /// special \p Opcode. \returns the address, op-index and line delta.
-    SpecialOpcodeDelta handleSpecialOpcode(uint8_t Opcode,
-                                           uint64_t OpcodeOffset);
+    LLVM_ABI SpecialOpcodeDelta handleSpecialOpcode(uint8_t Opcode,
+                                                    uint64_t OpcodeOffset);
 
     /// Line table we're currently parsing.
     struct LineTable *LineTable;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
index 5a82e7bd6289..75ca297b24f2 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Errc.h"
 #include <cstdint>
 
@@ -64,12 +65,13 @@ class DWARFLocationTable {
   /// iff it has successfully reched the end of the list. This means that one
   /// can attempt to parse another list after the current one (\p Offset will be
   /// updated to point past the end of the current list).
-  bool dumpLocationList(uint64_t *Offset, raw_ostream &OS,
-                        std::optional<object::SectionedAddress> BaseAddr,
-                        const DWARFObject &Obj, DWARFUnit *U,
-                        DIDumpOptions DumpOpts, unsigned Indent) const;
+  LLVM_ABI bool
+  dumpLocationList(uint64_t *Offset, raw_ostream &OS,
+                   std::optional<object::SectionedAddress> BaseAddr,
+                   const DWARFObject &Obj, DWARFUnit *U, DIDumpOptions DumpOpts,
+                   unsigned Indent) const;
 
-  Error visitAbsoluteLocationList(
+  LLVM_ABI Error visitAbsoluteLocationList(
       uint64_t Offset, std::optional<object::SectionedAddress> BaseAddr,
       std::function<std::optional<object::SectionedAddress>(uint32_t)>
           LookupAddr,
@@ -85,7 +87,7 @@ class DWARFLocationTable {
                             const DWARFObject &Obj) const = 0;
 };
 
-class DWARFDebugLoc final : public DWARFLocationTable {
+class LLVM_ABI DWARFDebugLoc final : public DWARFLocationTable {
 public:
   /// A list of locations that contain one variable.
   struct LocationList {
@@ -121,7 +123,7 @@ class DWARFDebugLoc final : public DWARFLocationTable {
                     const DWARFObject &Obj) const override;
 };
 
-class DWARFDebugLoclists final : public DWARFLocationTable {
+class LLVM_ABI DWARFDebugLoclists final : public DWARFLocationTable {
 public:
   DWARFDebugLoclists(DWARFDataExtractor Data, uint16_t Version)
       : DWARFLocationTable(std::move(Data)), Version(Version) {}
@@ -143,7 +145,7 @@ class DWARFDebugLoclists final : public DWARFLocationTable {
   uint16_t Version;
 };
 
-class ResolverError : public ErrorInfo<ResolverError> {
+class LLVM_ABI ResolverError : public ErrorInfo<ResolverError> {
 public:
   static char ID;
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
index 6c82bbfe74f7..2fce0959a693 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <vector>
 
@@ -74,10 +75,10 @@ class DWARFDebugPubTable {
 public:
   DWARFDebugPubTable() = default;
 
-  void extract(DWARFDataExtractor Data, bool GnuStyle,
-               function_ref<void(Error)> RecoverableErrorHandler);
+  LLVM_ABI void extract(DWARFDataExtractor Data, bool GnuStyle,
+                        function_ref<void(Error)> RecoverableErrorHandler);
 
-  void dump(raw_ostream &OS) const;
+  LLVM_ABI void dump(raw_ostream &OS) const;
 
   ArrayRef<Set> getData() { return Sets; }
 };
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
index 02bb5f0a3d95..5230808712fa 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGRANGELIST_H
 
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <vector>
 
@@ -51,7 +52,7 @@ class DWARFDebugRangeList {
     /// 2. An address, which defines the appropriate base address for
     /// use in interpreting the beginning and ending address offsets of
     /// subsequent entries of the location list.
-    bool isBaseAddressSelectionEntry(uint8_t AddressSize) const;
+    LLVM_ABI bool isBaseAddressSelectionEntry(uint8_t AddressSize) const;
   };
 
 private:
@@ -63,15 +64,15 @@ class DWARFDebugRangeList {
 public:
   DWARFDebugRangeList() { clear(); }
 
-  void clear();
-  void dump(raw_ostream &OS) const;
-  Error extract(const DWARFDataExtractor &data, uint64_t *offset_ptr);
+  LLVM_ABI void clear();
+  LLVM_ABI void dump(raw_ostream &OS) const;
+  LLVM_ABI Error extract(const DWARFDataExtractor &data, uint64_t *offset_ptr);
   const std::vector<RangeListEntry> &getEntries() { return Entries; }
 
   /// getAbsoluteRanges - Returns absolute address ranges defined by this range
   /// list. Has to be passed base address of the compile unit referencing this
   /// range list.
-  DWARFAddressRangesVector
+  LLVM_ABI DWARFAddressRangesVector
   getAbsoluteRanges(std::optional<object::SectionedAddress> BaseAddr) const;
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h
index 8ddd9cab2364..9ffb56255e7b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h
@@ -13,6 +13,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
 #include "llvm/DebugInfo/DWARF/DWARFListTable.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -35,8 +36,8 @@ struct RangeListEntry : public DWARFListEntryBase {
   uint64_t Value0;
   uint64_t Value1;
 
-  Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr);
-  void
+  LLVM_ABI Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr);
+  LLVM_ABI void
   dump(raw_ostream &OS, uint8_t AddrSize, uint8_t MaxEncodingStringLength,
        uint64_t &CurrentBase, DIDumpOptions DumpOpts,
        llvm::function_ref<std::optional<object::SectionedAddress>(uint32_t)>
@@ -48,15 +49,14 @@ struct RangeListEntry : public DWARFListEntryBase {
 class DWARFDebugRnglist : public DWARFListType<RangeListEntry> {
 public:
   /// Build a DWARFAddressRangesVector from a rangelist.
-  DWARFAddressRangesVector getAbsoluteRanges(
+  LLVM_ABI DWARFAddressRangesVector getAbsoluteRanges(
       std::optional<object::SectionedAddress> BaseAddr, uint8_t AddressByteSize,
       function_ref<std::optional<object::SectionedAddress>(uint32_t)>
           LookupPooledAddress) const;
 
   /// Build a DWARFAddressRangesVector from a rangelist.
-  DWARFAddressRangesVector
-  getAbsoluteRanges(std::optional<object::SectionedAddress> BaseAddr,
-                    DWARFUnit &U) const;
+  LLVM_ABI DWARFAddressRangesVector getAbsoluteRanges(
+      std::optional<object::SectionedAddress> BaseAddr, DWARFUnit &U) const;
 };
 
 class DWARFDebugRnglistTable : public DWARFListTableBase<DWARFDebugRnglist> {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h
index 2e98a4a39714..8a0bd1733808 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h
@@ -18,6 +18,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFAttribute.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h"
 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -85,50 +86,50 @@ class DWARFDie {
   bool isNULL() const { return getAbbreviationDeclarationPtr() == nullptr; }
 
   /// Returns true if DIE represents a subprogram (not inlined).
-  bool isSubprogramDIE() const;
+  LLVM_ABI bool isSubprogramDIE() const;
 
   /// Returns true if DIE represents a subprogram or an inlined subroutine.
-  bool isSubroutineDIE() const;
+  LLVM_ABI bool isSubroutineDIE() const;
 
   /// Get the parent of this DIE object.
   ///
   /// \returns a valid DWARFDie instance if this object has a parent or an
   /// invalid DWARFDie instance if it doesn't.
-  DWARFDie getParent() const;
+  LLVM_ABI DWARFDie getParent() const;
 
   /// Get the sibling of this DIE object.
   ///
   /// \returns a valid DWARFDie instance if this object has a sibling or an
   /// invalid DWARFDie instance if it doesn't.
-  DWARFDie getSibling() const;
+  LLVM_ABI DWARFDie getSibling() const;
 
   /// Get the previous sibling of this DIE object.
   ///
   /// \returns a valid DWARFDie instance if this object has a sibling or an
   /// invalid DWARFDie instance if it doesn't.
-  DWARFDie getPreviousSibling() const;
+  LLVM_ABI DWARFDie getPreviousSibling() const;
 
   /// Get the first child of this DIE object.
   ///
   /// \returns a valid DWARFDie instance if this object has children or an
   /// invalid DWARFDie instance if it doesn't.
-  DWARFDie getFirstChild() const;
+  LLVM_ABI DWARFDie getFirstChild() const;
 
   /// Get the last child of this DIE object.
   ///
   /// \returns a valid null DWARFDie instance if this object has children or an
   /// invalid DWARFDie instance if it doesn't.
-  DWARFDie getLastChild() const;
+  LLVM_ABI DWARFDie getLastChild() const;
 
   /// Dump the DIE and all of its attributes to the supplied stream.
   ///
   /// \param OS the stream to use for output.
   /// \param indent the number of characters to indent each line that is output.
-  void dump(raw_ostream &OS, unsigned indent = 0,
-            DIDumpOptions DumpOpts = DIDumpOptions()) const;
+  LLVM_ABI void dump(raw_ostream &OS, unsigned indent = 0,
+                     DIDumpOptions DumpOpts = DIDumpOptions()) const;
 
   /// Convenience zero-argument overload for debugging.
-  LLVM_DUMP_METHOD void dump() const;
+  LLVM_ABI LLVM_DUMP_METHOD void dump() const;
 
   /// Extract the specified attribute from this DIE.
   ///
@@ -139,7 +140,7 @@ class DWARFDie {
   /// \param Attr the attribute to extract.
   /// \returns an optional DWARFFormValue that will have the form value if the
   /// attribute was successfully extracted.
-  std::optional<DWARFFormValue> find(dwarf::Attribute Attr) const;
+  LLVM_ABI std::optional<DWARFFormValue> find(dwarf::Attribute Attr) const;
 
   /// Extract the first value of any attribute in Attrs from this DIE.
   ///
@@ -152,7 +153,8 @@ class DWARFDie {
   /// \returns an optional that has a valid DWARFFormValue for the first
   /// matching attribute in Attrs, or std::nullopt if none of the attributes in
   /// Attrs exist in this DIE.
-  std::optional<DWARFFormValue> find(ArrayRef<dwarf::Attribute> Attrs) const;
+  LLVM_ABI std::optional<DWARFFormValue>
+  find(ArrayRef<dwarf::Attribute> Attrs) const;
 
   /// Extract the first value of any attribute in Attrs from this DIE and
   /// recurse into any DW_AT_specification or DW_AT_abstract_origin referenced
@@ -163,7 +165,7 @@ class DWARFDie {
   /// matching attribute in Attrs, or std::nullopt if none of the attributes in
   /// Attrs exist in this DIE or in any DW_AT_specification or
   /// DW_AT_abstract_origin DIEs.
-  std::optional<DWARFFormValue>
+  LLVM_ABI std::optional<DWARFFormValue>
   findRecursively(ArrayRef<dwarf::Attribute> Attrs) const;
 
   /// Extract the specified attribute from this DIE as the referenced DIE.
@@ -179,21 +181,23 @@ class DWARFDie {
   /// \param Attr the attribute to extract.
   /// \returns a valid DWARFDie instance if the attribute exists, or an invalid
   /// DWARFDie object if it doesn't.
-  DWARFDie getAttributeValueAsReferencedDie(dwarf::Attribute Attr) const;
-  DWARFDie getAttributeValueAsReferencedDie(const DWARFFormValue &V) const;
+  LLVM_ABI DWARFDie
+  getAttributeValueAsReferencedDie(dwarf::Attribute Attr) const;
+  LLVM_ABI DWARFDie
+  getAttributeValueAsReferencedDie(const DWARFFormValue &V) const;
 
-  DWARFDie resolveTypeUnitReference() const;
+  LLVM_ABI DWARFDie resolveTypeUnitReference() const;
 
-  DWARFDie resolveReferencedType(dwarf::Attribute Attr) const;
-  DWARFDie resolveReferencedType(const DWARFFormValue &V) const;
+  LLVM_ABI DWARFDie resolveReferencedType(dwarf::Attribute Attr) const;
+  LLVM_ABI DWARFDie resolveReferencedType(const DWARFFormValue &V) const;
   /// Extract the range base attribute from this DIE as absolute section offset.
   ///
   /// This is a utility function that checks for either the DW_AT_rnglists_base
   /// or DW_AT_GNU_ranges_base attribute.
   ///
   /// \returns anm optional absolute section offset value for the attribute.
-  std::optional<uint64_t> getRangesBaseAttribute() const;
-  std::optional<uint64_t> getLocBaseAttribute() const;
+  LLVM_ABI std::optional<uint64_t> getRangesBaseAttribute() const;
+  LLVM_ABI std::optional<uint64_t> getLocBaseAttribute() const;
 
   /// Get the DW_AT_high_pc attribute value as an address.
   ///
@@ -205,12 +209,12 @@ class DWARFDie {
   ///
   /// \param LowPC the low PC that might be needed to calculate the high PC.
   /// \returns an optional address value for the attribute.
-  std::optional<uint64_t> getHighPC(uint64_t LowPC) const;
+  LLVM_ABI std::optional<uint64_t> getHighPC(uint64_t LowPC) const;
 
   /// Retrieves DW_AT_low_pc and DW_AT_high_pc from CU.
   /// Returns true if both attributes are present.
-  bool getLowAndHighPC(uint64_t &LowPC, uint64_t &HighPC,
-                       uint64_t &SectionIndex) const;
+  LLVM_ABI bool getLowAndHighPC(uint64_t &LowPC, uint64_t &HighPC,
+                                uint64_t &SectionIndex) const;
 
   /// Get the address ranges for this DIE.
   ///
@@ -222,45 +226,46 @@ class DWARFDie {
   ///
   /// \returns a address range vector that might be empty if no address range
   /// information is available.
-  Expected<DWARFAddressRangesVector> getAddressRanges() const;
+  LLVM_ABI Expected<DWARFAddressRangesVector> getAddressRanges() const;
 
-  bool addressRangeContainsAddress(const uint64_t Address) const;
+  LLVM_ABI bool addressRangeContainsAddress(const uint64_t Address) const;
 
-  std::optional<uint64_t> getLanguage() const;
+  LLVM_ABI std::optional<uint64_t> getLanguage() const;
 
-  Expected<DWARFLocationExpressionsVector>
+  LLVM_ABI Expected<DWARFLocationExpressionsVector>
   getLocations(dwarf::Attribute Attr) const;
 
   /// If a DIE represents a subprogram (or inlined subroutine), returns its
   /// mangled name (or short name, if mangled is missing). This name may be
   /// fetched from specification or abstract origin for this subprogram.
   /// Returns null if no name is found.
-  const char *getSubroutineName(DINameKind Kind) const;
+  LLVM_ABI const char *getSubroutineName(DINameKind Kind) const;
 
   /// Return the DIE name resolving DW_AT_specification or DW_AT_abstract_origin
   /// references if necessary. For the LinkageName case it additionaly searches
   /// for ShortName if LinkageName is not found.
   /// Returns null if no name is found.
-  const char *getName(DINameKind Kind) const;
-  void getFullName(raw_string_ostream &,
-                   std::string *OriginalFullName = nullptr) const;
+  LLVM_ABI const char *getName(DINameKind Kind) const;
+  LLVM_ABI void getFullName(raw_string_ostream &,
+                            std::string *OriginalFullName = nullptr) const;
 
   /// Return the DIE short name resolving DW_AT_specification or
   /// DW_AT_abstract_origin references if necessary. Returns null if no name
   /// is found.
-  const char *getShortName() const;
+  LLVM_ABI const char *getShortName() const;
 
   /// Return the DIE linkage name resolving DW_AT_specification or
   /// DW_AT_abstract_origin references if necessary. Returns null if no name
   /// is found.
-  const char *getLinkageName() const;
+  LLVM_ABI const char *getLinkageName() const;
 
   /// Returns the declaration line (start line) for a DIE, assuming it specifies
   /// a subprogram. This may be fetched from specification or abstract origin
   /// for this subprogram by resolving DW_AT_sepcification or
   /// DW_AT_abstract_origin references if necessary.
-  uint64_t getDeclLine() const;
-  std::string getDeclFile(DILineInfoSpecifier::FileLineInfoKind Kind) const;
+  LLVM_ABI uint64_t getDeclLine() const;
+  LLVM_ABI std::string
+  getDeclFile(DILineInfoSpecifier::FileLineInfoKind Kind) const;
 
   /// Retrieves values of DW_AT_call_file, DW_AT_call_line and DW_AT_call_column
   /// from DIE (or zeroes if they are missing). This function looks for
@@ -274,22 +279,23 @@ class DWARFDie {
   /// no DW_AT_call_column attribute in this DIE.
   /// \param CallDiscriminator filled in with non-zero if successful, zero if
   /// there is no DW_AT_GNU_discriminator attribute in this DIE.
-  void getCallerFrame(uint32_t &CallFile, uint32_t &CallLine,
-                      uint32_t &CallColumn, uint32_t &CallDiscriminator) const;
+  LLVM_ABI void getCallerFrame(uint32_t &CallFile, uint32_t &CallLine,
+                               uint32_t &CallColumn,
+                               uint32_t &CallDiscriminator) const;
 
   class attribute_iterator;
 
   /// Get an iterator range to all attributes in the current DIE only.
   ///
   /// \returns an iterator range for the attributes of the current DIE.
-  iterator_range<attribute_iterator> attributes() const;
+  LLVM_ABI iterator_range<attribute_iterator> attributes() const;
 
   /// Gets the type size (in bytes) for this DIE.
   ///
   /// \param PointerSize the pointer size of the containing CU.
   /// \returns if this is a type DIE, or this DIE contains a DW_AT_type, returns
   /// the size of the type.
-  std::optional<uint64_t> getTypeSize(uint64_t PointerSize);
+  LLVM_ABI std::optional<uint64_t> getTypeSize(uint64_t PointerSize);
 
   class iterator;
 
@@ -324,10 +330,10 @@ class DWARFDie::attribute_iterator
 
 public:
   attribute_iterator() = delete;
-  explicit attribute_iterator(DWARFDie D, bool End);
+  LLVM_ABI explicit attribute_iterator(DWARFDie D, bool End);
 
-  attribute_iterator &operator++();
-  attribute_iterator &operator--();
+  LLVM_ABI attribute_iterator &operator++();
+  LLVM_ABI attribute_iterator &operator--();
   explicit operator bool() const { return AttrValue.isValid(); }
   const DWARFAttribute &operator*() const { return AttrValue; }
 };
@@ -481,9 +487,9 @@ inline std::reverse_iterator<DWARFDie::iterator> DWARFDie::rend() const {
   return std::make_reverse_iterator(begin());
 }
 
-void dumpTypeQualifiedName(const DWARFDie &DIE, raw_ostream &OS);
-void dumpTypeUnqualifiedName(const DWARFDie &DIE, raw_ostream &OS,
-                             std::string *OriginalFullName = nullptr);
+LLVM_ABI void dumpTypeQualifiedName(const DWARFDie &DIE, raw_ostream &OS);
+LLVM_ABI void dumpTypeUnqualifiedName(const DWARFDie &DIE, raw_ostream &OS,
+                                      std::string *OriginalFullName = nullptr);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
index 00228a32173f..44432d321911 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 
 namespace llvm {
@@ -76,6 +77,9 @@ class DWARFExpression {
 
   private:
     friend class DWARFExpression::iterator;
+    friend class DWARFExpressionPrinter;
+    friend class DWARFVerifier;
+
     uint8_t Opcode; ///< The Op Opcode, DW_OP_<something>.
     Description Desc;
     bool Error = false;
@@ -86,7 +90,7 @@ class DWARFExpression {
   public:
     const Description &getDescription() const { return Desc; }
     uint8_t getCode() const { return Opcode; }
-    std::optional<unsigned> getSubCode() const;
+    LLVM_ABI std::optional<unsigned> getSubCode() const;
     uint64_t getNumOperands() const { return Operands.size(); }
     ArrayRef<uint64_t> getRawOperands() const { return Operands; };
     uint64_t getRawOperand(unsigned Idx) const { return Operands[Idx]; }
@@ -98,15 +102,11 @@ class DWARFExpression {
     }
     uint64_t getEndOffset() const { return EndOffset; }
     bool isError() const { return Error; }
-    bool print(raw_ostream &OS, DIDumpOptions DumpOpts,
-               const DWARFExpression *Expr, DWARFUnit *U) const;
-
-    /// Verify \p Op. Does not affect the return of \a isError().
-    static bool verify(const Operation &Op, DWARFUnit *U);
 
   private:
-    bool extract(DataExtractor Data, uint8_t AddressSize, uint64_t Offset,
-                 std::optional<dwarf::DwarfFormat> Format);
+    LLVM_ABI bool extract(DataExtractor Data, uint8_t AddressSize,
+                          uint64_t Offset,
+                          std::optional<dwarf::DwarfFormat> Format);
   };
 
   /// An iterator to go through the expression operations.
@@ -152,26 +152,12 @@ class DWARFExpression {
   iterator begin() const { return iterator(this, 0); }
   iterator end() const { return iterator(this, Data.getData().size()); }
 
-  void print(raw_ostream &OS, DIDumpOptions DumpOpts, DWARFUnit *U,
-             bool IsEH = false) const;
-
-  /// Print the expression in a format intended to be compact and useful to a
-  /// user, but not perfectly unambiguous, or capable of representing every
-  /// valid DWARF expression. Returns true if the expression was sucessfully
-  /// printed.
-  bool printCompact(raw_ostream &OS,
-                    std::function<StringRef(uint64_t RegNum, bool IsEH)>
-                        GetNameForDWARFReg = nullptr);
-
-  bool verify(DWARFUnit *U);
-
-  bool operator==(const DWARFExpression &RHS) const;
+  LLVM_ABI bool operator==(const DWARFExpression &RHS) const;
 
   StringRef getData() const { return Data.getData(); }
 
-  static bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
-                                    DIDumpOptions DumpOpts, uint8_t Opcode,
-                                    const ArrayRef<uint64_t> Operands);
+  friend class DWARFExpressionPrinter;
+  friend class DWARFVerifier;
 
 private:
   DataExtractor Data;
@@ -183,5 +169,63 @@ inline bool operator==(const DWARFExpression::iterator &LHS,
                        const DWARFExpression::iterator &RHS) {
   return LHS.Expr == RHS.Expr && LHS.Offset == RHS.Offset;
 }
-}
-#endif
+
+// This functionality is separated from the main data structure so that nothing
+// in DWARFExpression.cpp needs build-time dependencies on DWARFUnit or other
+// higher-level Dwarf structures. This approach creates better layering and
+// allows DWARFExpression to be used from code which can't have dependencies on
+// those higher-level structures.
+
+class DWARFUnit;
+struct DIDumpOptions;
+class raw_ostream;
+
+class DWARFExpressionPrinter {
+public:
+  /// Print a Dwarf expression/
+  /// \param E to be printed
+  /// \param OS to this stream
+  /// \param GetNameForDWARFReg callback to return dwarf register name
+  static void print(const DWARFExpression *E, raw_ostream &OS,
+                    DIDumpOptions DumpOpts, DWARFUnit *U, bool IsEH = false);
+
+  /// Print the expression in a format intended to be compact and useful to a
+  /// user, but not perfectly unambiguous, or capable of representing every
+  /// valid DWARF expression. Returns true if the expression was sucessfully
+  /// printed.
+  ///
+  /// \param E to be printed
+  /// \param OS to this stream
+  /// \param GetNameForDWARFReg callback to return dwarf register name
+  ///
+  /// \returns true if the expression was successfully printed
+  static bool printCompact(const DWARFExpression *E, raw_ostream &OS,
+                           std::function<StringRef(uint64_t RegNum, bool IsEH)>
+                               GetNameForDWARFReg = nullptr);
+
+  /// Pretty print a register opcode and operands.
+  /// \param U within the context of this Dwarf unit, if any.
+  /// \param OS to this stream
+  /// \param DumpOpts with these options
+  /// \param Opcode to print
+  /// \param Operands to the opcode
+  ///
+  /// returns true if the Op was successfully printed
+  static bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
+                                    DIDumpOptions DumpOpts, uint8_t Opcode,
+                                    ArrayRef<uint64_t> Operands);
+
+private:
+  static bool printOp(const DWARFExpression::Operation *Op, raw_ostream &OS,
+                      DIDumpOptions DumpOpts, const DWARFExpression *Expr,
+                      DWARFUnit *U);
+
+  static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
+                                     DIDumpOptions DumpOpts,
+                                     ArrayRef<uint64_t> Operands,
+                                     unsigned Operand);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
index 563887d1149a..3818a90b909c 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DIContext.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include <cstdint>
 
@@ -66,39 +67,42 @@ class DWARFFormValue {
 public:
   DWARFFormValue(dwarf::Form F = dwarf::Form(0)) : Form(F) {}
 
-  static DWARFFormValue createFromSValue(dwarf::Form F, int64_t V);
-  static DWARFFormValue createFromUValue(dwarf::Form F, uint64_t V);
-  static DWARFFormValue createFromPValue(dwarf::Form F, const char *V);
-  static DWARFFormValue createFromBlockValue(dwarf::Form F,
-                                             ArrayRef<uint8_t> D);
-  static DWARFFormValue createFromUnit(dwarf::Form F, const DWARFUnit *Unit,
-                                       uint64_t *OffsetPtr);
-  static std::optional<object::SectionedAddress>
+  LLVM_ABI static DWARFFormValue createFromSValue(dwarf::Form F, int64_t V);
+  LLVM_ABI static DWARFFormValue createFromUValue(dwarf::Form F, uint64_t V);
+  LLVM_ABI static DWARFFormValue createFromPValue(dwarf::Form F, const char *V);
+  LLVM_ABI static DWARFFormValue createFromBlockValue(dwarf::Form F,
+                                                      ArrayRef<uint8_t> D);
+  LLVM_ABI static DWARFFormValue
+  createFromUnit(dwarf::Form F, const DWARFUnit *Unit, uint64_t *OffsetPtr);
+  LLVM_ABI static std::optional<object::SectionedAddress>
   getAsSectionedAddress(const ValueType &Val, const dwarf::Form Form,
                         const DWARFUnit *U);
 
   dwarf::Form getForm() const { return Form; }
   uint64_t getRawUValue() const { return Value.uval; }
 
-  bool isFormClass(FormClass FC) const;
+  LLVM_ABI bool isFormClass(FormClass FC) const;
   const DWARFUnit *getUnit() const { return U; }
-  void dump(raw_ostream &OS, DIDumpOptions DumpOpts = DIDumpOptions()) const;
-  void dumpSectionedAddress(raw_ostream &OS, DIDumpOptions DumpOpts,
-                            object::SectionedAddress SA) const;
-  void dumpAddress(raw_ostream &OS, uint64_t Address) const;
-  static void dumpAddress(raw_ostream &OS, uint8_t AddressSize,
-                          uint64_t Address);
-  static void dumpAddressSection(const DWARFObject &Obj, raw_ostream &OS,
-                                 DIDumpOptions DumpOpts, uint64_t SectionIndex);
+  LLVM_ABI void dump(raw_ostream &OS,
+                     DIDumpOptions DumpOpts = DIDumpOptions()) const;
+  LLVM_ABI void dumpSectionedAddress(raw_ostream &OS, DIDumpOptions DumpOpts,
+                                     object::SectionedAddress SA) const;
+  LLVM_ABI void dumpAddress(raw_ostream &OS, uint64_t Address) const;
+  LLVM_ABI static void dumpAddress(raw_ostream &OS, uint8_t AddressSize,
+                                   uint64_t Address);
+  LLVM_ABI static void dumpAddressSection(const DWARFObject &Obj,
+                                          raw_ostream &OS,
+                                          DIDumpOptions DumpOpts,
+                                          uint64_t SectionIndex);
 
   /// Extracts a value in \p Data at offset \p *OffsetPtr. The information
   /// in \p FormParams is needed to interpret some forms. The optional
   /// \p Context and \p Unit allows extracting information if the form refers
   /// to other sections (e.g., .debug_str).
-  bool extractValue(const DWARFDataExtractor &Data, uint64_t *OffsetPtr,
-                    dwarf::FormParams FormParams,
-                    const DWARFContext *Context = nullptr,
-                    const DWARFUnit *Unit = nullptr);
+  LLVM_ABI bool extractValue(const DWARFDataExtractor &Data,
+                             uint64_t *OffsetPtr, dwarf::FormParams FormParams,
+                             const DWARFContext *Context = nullptr,
+                             const DWARFUnit *Unit = nullptr);
 
   bool extractValue(const DWARFDataExtractor &Data, uint64_t *OffsetPtr,
                     dwarf::FormParams FormParams, const DWARFUnit *U) {
@@ -107,19 +111,20 @@ class DWARFFormValue {
 
   /// getAsFoo functions below return the extracted value as Foo if only
   /// DWARFFormValue has form class is suitable for representing Foo.
-  std::optional<uint64_t> getAsRelativeReference() const;
-  std::optional<uint64_t> getAsDebugInfoReference() const;
-  std::optional<uint64_t> getAsSignatureReference() const;
-  std::optional<uint64_t> getAsSupplementaryReference() const;
-  std::optional<uint64_t> getAsUnsignedConstant() const;
-  std::optional<int64_t> getAsSignedConstant() const;
-  Expected<const char *> getAsCString() const;
-  std::optional<uint64_t> getAsAddress() const;
-  std::optional<object::SectionedAddress> getAsSectionedAddress() const;
-  std::optional<uint64_t> getAsSectionOffset() const;
-  std::optional<ArrayRef<uint8_t>> getAsBlock() const;
-  std::optional<uint64_t> getAsCStringOffset() const;
-  std::optional<uint64_t> getAsReferenceUVal() const;
+  LLVM_ABI std::optional<uint64_t> getAsRelativeReference() const;
+  LLVM_ABI std::optional<uint64_t> getAsDebugInfoReference() const;
+  LLVM_ABI std::optional<uint64_t> getAsSignatureReference() const;
+  LLVM_ABI std::optional<uint64_t> getAsSupplementaryReference() const;
+  LLVM_ABI std::optional<uint64_t> getAsUnsignedConstant() const;
+  LLVM_ABI std::optional<int64_t> getAsSignedConstant() const;
+  LLVM_ABI Expected<const char *> getAsCString() const;
+  LLVM_ABI std::optional<uint64_t> getAsAddress() const;
+  LLVM_ABI std::optional<object::SectionedAddress>
+  getAsSectionedAddress() const;
+  LLVM_ABI std::optional<uint64_t> getAsSectionOffset() const;
+  LLVM_ABI std::optional<ArrayRef<uint8_t>> getAsBlock() const;
+  LLVM_ABI std::optional<uint64_t> getAsCStringOffset() const;
+  LLVM_ABI std::optional<uint64_t> getAsReferenceUVal() const;
   /// Correctly extract any file paths from a form value.
   ///
   /// These attributes can be in the from DW_AT_decl_file or DW_AT_call_file
@@ -131,7 +136,7 @@ class DWARFFormValue {
   ///
   /// \returns A valid string value on success, or std::nullopt if the form
   /// class is not FC_Constant, or if the file index is not valid.
-  std::optional<std::string>
+  LLVM_ABI std::optional<std::string>
   getAsFile(DILineInfoSpecifier::FileLineInfoKind Kind) const;
 
   /// Skip a form's value in \p DebugInfoData at the offset specified by
@@ -158,9 +163,9 @@ class DWARFFormValue {
   /// \param OffsetPtr A reference to the offset that will be updated.
   /// \param FormParams DWARF parameters to help interpret forms.
   /// \returns true on success, false if the form was not skipped.
-  static bool skipValue(dwarf::Form Form, DataExtractor DebugInfoData,
-                        uint64_t *OffsetPtr,
-                        const dwarf::FormParams FormParams);
+  LLVM_ABI static bool skipValue(dwarf::Form Form, DataExtractor DebugInfoData,
+                                 uint64_t *OffsetPtr,
+                                 const dwarf::FormParams FormParams);
 
 private:
   void dumpString(raw_ostream &OS) const;
@@ -430,8 +435,9 @@ toBlock(const std::optional<DWARFFormValue> &V) {
 /// \param FC an attribute form class to check.
 /// \param DwarfVersion the version of DWARF debug info keeping the attribute.
 /// \returns true if specified \p Form belongs to the \p FC class.
-bool doesFormBelongToClass(dwarf::Form Form, DWARFFormValue::FormClass FC,
-                           uint16_t DwarfVersion);
+LLVM_ABI bool doesFormBelongToClass(dwarf::Form Form,
+                                    DWARFFormValue::FormClass FC,
+                                    uint16_t DwarfVersion);
 
 } // end namespace dwarf
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h
index d739d6c195df..2448ad0871a3 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFListTable.h
@@ -12,6 +12,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
@@ -110,8 +111,8 @@ class DWARFListTableHeader {
     llvm_unreachable("Invalid DWARF format (expected DWARF32 or DWARF64");
   }
 
-  void dump(DataExtractor Data, raw_ostream &OS,
-            DIDumpOptions DumpOpts = {}) const;
+  LLVM_ABI void dump(DataExtractor Data, raw_ostream &OS,
+                     DIDumpOptions DumpOpts = {}) const;
   std::optional<uint64_t> getOffsetEntry(DataExtractor Data,
                                          uint32_t Index) const {
     if (Index >= HeaderData.OffsetEntryCount)
@@ -131,12 +132,12 @@ class DWARFListTableHeader {
   }
 
   /// Extract the table header and the array of offsets.
-  Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr);
+  LLVM_ABI Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr);
 
   /// Returns the length of the table, including the length field, or 0 if the
   /// length has not been determined (e.g. because the table has not yet been
   /// parsed, or there was a problem in parsing).
-  uint64_t length() const;
+  LLVM_ABI uint64_t length() const;
 };
 
 /// A class representing a table of lists as specified in the DWARF v5
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h
index 8b5497f4eeb9..3390bec4ac65 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFLOCATIONEXPRESSION_H
 
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -38,7 +39,8 @@ inline bool operator!=(const DWARFLocationExpression &L,
   return !(L == R);
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const DWARFLocationExpression &Loc);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const DWARFLocationExpression &Loc);
 
 /// Represents a set of absolute location expressions.
 using DWARFLocationExpressionsVector = std::vector<DWARFLocationExpression>;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h
index bd25f6c30ebf..a760f773055d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFTypePrinter.h
@@ -734,13 +734,15 @@ void DWARFTypePrinter<DieType>::appendSubroutineNameAfter(
       OS << " __attribute__((intel_ocl_bicc))";
       break;
     case dwarf::CallingConvention::DW_CC_LLVM_SpirFunction:
-    case dwarf::CallingConvention::DW_CC_LLVM_OpenCLKernel:
-      // These aren't available as attributes, but maybe we should still
-      // render them somehow? (Clang doesn't render them, but that's an issue
+      // This isn't available as an attribute, but maybe we should still
+      // render it somehow? (Clang doesn't render it, but that's an issue
       // for template names too - since then the DWARF names of templates
       // instantiated with function types with these calling conventions won't
       // have distinct names - so we'd need to fix that too)
       break;
+    case dwarf::CallingConvention::DW_CC_LLVM_DeviceKernel:
+      OS << " __attribute__((device_kernel))";
+      break;
     case dwarf::CallingConvention::DW_CC_LLVM_Swift:
       // SwiftAsync missing
       OS << " __attribute__((swiftcall))";
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 0f7958f28065..964ff8e39666 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -21,6 +21,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include <cassert>
 #include <cstddef>
@@ -81,11 +82,12 @@ class DWARFUnitHeader {
   /// Note that \p SectionKind is used as a hint to guess the unit type
   /// for DWARF formats prior to DWARFv5. In DWARFv5 the unit type is
   /// explicitly defined in the header and the hint is ignored.
-  Error extract(DWARFContext &Context, const DWARFDataExtractor &debug_info,
-                uint64_t *offset_ptr, DWARFSectionKind SectionKind);
+  LLVM_ABI Error extract(DWARFContext &Context,
+                         const DWARFDataExtractor &debug_info,
+                         uint64_t *offset_ptr, DWARFSectionKind SectionKind);
   // For units in DWARF Package File, remember the index entry and update
   // the abbreviation offset read by extract().
-  Error applyIndexEntry(const DWARFUnitIndex::Entry *Entry);
+  LLVM_ABI Error applyIndexEntry(const DWARFUnitIndex::Entry *Entry);
   uint64_t getOffset() const { return Offset; }
   const dwarf::FormParams &getFormParams() const { return FormParams; }
   uint16_t getVersion() const { return FormParams.Version; }
@@ -118,8 +120,8 @@ class DWARFUnitHeader {
   }
 };
 
-const DWARFUnitIndex &getDWARFUnitIndex(DWARFContext &Context,
-                                        DWARFSectionKind Kind);
+LLVM_ABI const DWARFUnitIndex &getDWARFUnitIndex(DWARFContext &Context,
+                                                 DWARFSectionKind Kind);
 
 bool isCompileUnit(const std::unique_ptr<DWARFUnit> &U);
 
@@ -140,26 +142,28 @@ class DWARFUnitVector final : public SmallVector<std::unique_ptr<DWARFUnit>, 1>
   using compile_unit_range =
       decltype(make_filter_range(std::declval<iterator_range>(), isCompileUnit));
 
-  DWARFUnit *getUnitForOffset(uint64_t Offset) const;
-  DWARFUnit *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E);
+  LLVM_ABI DWARFUnit *getUnitForOffset(uint64_t Offset) const;
+  LLVM_ABI DWARFUnit *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E);
 
   /// Read units from a .debug_info or .debug_types section.  Calls made
   /// before finishedInfoUnits() are assumed to be for .debug_info sections,
   /// calls after finishedInfoUnits() are for .debug_types sections.  Caller
   /// must not mix calls to addUnitsForSection and addUnitsForDWOSection.
-  void addUnitsForSection(DWARFContext &C, const DWARFSection &Section,
-                          DWARFSectionKind SectionKind);
+  LLVM_ABI void addUnitsForSection(DWARFContext &C, const DWARFSection &Section,
+                                   DWARFSectionKind SectionKind);
   /// Read units from a .debug_info.dwo or .debug_types.dwo section.  Calls
   /// made before finishedInfoUnits() are assumed to be for .debug_info.dwo
   /// sections, calls after finishedInfoUnits() are for .debug_types.dwo
   /// sections.  Caller must not mix calls to addUnitsForSection and
   /// addUnitsForDWOSection.
-  void addUnitsForDWOSection(DWARFContext &C, const DWARFSection &DWOSection,
-                             DWARFSectionKind SectionKind, bool Lazy = false);
+  LLVM_ABI void addUnitsForDWOSection(DWARFContext &C,
+                                      const DWARFSection &DWOSection,
+                                      DWARFSectionKind SectionKind,
+                                      bool Lazy = false);
 
   /// Add an existing DWARFUnit to this UnitVector. This is used by the DWARF
   /// verifier to process unit separately.
-  DWARFUnit *addUnit(std::unique_ptr<DWARFUnit> Unit);
+  LLVM_ABI DWARFUnit *addUnit(std::unique_ptr<DWARFUnit> Unit);
 
   /// Returns number of all units held by this instance.
   unsigned getNumUnits() const { return size(); }
@@ -204,11 +208,11 @@ struct StrOffsetsContributionDescriptor {
   /// Determine whether a contribution to the string offsets table is
   /// consistent with the relevant section size and that its length is
   /// a multiple of the size of one of its entries.
-  Expected<StrOffsetsContributionDescriptor>
+  LLVM_ABI Expected<StrOffsetsContributionDescriptor>
   validateContributionSize(DWARFDataExtractor &DA);
 };
 
-class DWARFUnit {
+class LLVM_ABI DWARFUnit {
   DWARFContext &Context;
   /// Section containing this DWARFUnit.
   const DWARFSection &InfoSection;
@@ -566,9 +570,6 @@ class DWARFUnit {
 
   Error tryExtractDIEsIfNeeded(bool CUDieOnly);
 
-  /// clearDIEs - Clear parsed DIEs to keep memory usage low.
-  void clearDIEs(bool KeepCUDie, bool KeepDWODies = false);
-
 private:
   /// Size in bytes of the .debug_info data associated with this compile unit.
   size_t getDebugInfoSize() const {
@@ -584,6 +585,9 @@ class DWARFUnit {
   void extractDIEsToVector(bool AppendCUDie, bool AppendNonCUDIEs,
                            std::vector<DWARFDebugInfoEntry> &DIEs) const;
 
+  /// clearDIEs - Clear parsed DIEs to keep memory usage low.
+  void clearDIEs(bool KeepCUDie);
+
   /// parseDWO - Parses .dwo file for current compile unit. Returns true if
   /// it was actually constructed.
   /// The \p AlternativeLocation specifies an alternative location to get
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
index e65b193ffc86..e485df0db185 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <memory>
 
@@ -88,13 +89,15 @@ inline const char *toString(DWARFSectionKind Kind) {
 /// The conversion depends on the version of the index section.
 /// IndexVersion is expected to be either 2 for pre-standard GNU proposal
 /// or 5 for DWARFv5 package file.
-uint32_t serializeSectionKind(DWARFSectionKind Kind, unsigned IndexVersion);
+LLVM_ABI uint32_t serializeSectionKind(DWARFSectionKind Kind,
+                                       unsigned IndexVersion);
 
 /// Convert a value read from an index section to the internal representation.
 ///
 /// The conversion depends on the index section version, which is expected
 /// to be either 2 for pre-standard GNU proposal or 5 for DWARFv5 package file.
-DWARFSectionKind deserializeSectionKind(uint32_t Value, unsigned IndexVersion);
+LLVM_ABI DWARFSectionKind deserializeSectionKind(uint32_t Value,
+                                                 unsigned IndexVersion);
 
 class DWARFUnitIndex {
   struct Header {
@@ -103,8 +106,8 @@ class DWARFUnitIndex {
     uint32_t NumUnits;
     uint32_t NumBuckets = 0;
 
-    bool parse(DataExtractor IndexData, uint64_t *OffsetPtr);
-    void dump(raw_ostream &OS) const;
+    LLVM_ABI bool parse(DataExtractor IndexData, uint64_t *OffsetPtr);
+    LLVM_ABI void dump(raw_ostream &OS) const;
   };
 
 public:
@@ -135,9 +138,10 @@ class DWARFUnitIndex {
     friend class DWARFUnitIndex;
 
   public:
-    const SectionContribution *getContribution(DWARFSectionKind Sec) const;
-    const SectionContribution *getContribution() const;
-    SectionContribution &getContribution();
+    LLVM_ABI const SectionContribution *
+    getContribution(DWARFSectionKind Sec) const;
+    LLVM_ABI const SectionContribution *getContribution() const;
+    LLVM_ABI SectionContribution &getContribution();
 
     const SectionContribution *getContributions() const {
       return Contributions.get();
@@ -170,13 +174,13 @@ class DWARFUnitIndex {
 
   explicit operator bool() const { return Header.NumBuckets; }
 
-  bool parse(DataExtractor IndexData);
-  void dump(raw_ostream &OS) const;
+  LLVM_ABI bool parse(DataExtractor IndexData);
+  LLVM_ABI void dump(raw_ostream &OS) const;
 
   uint32_t getVersion() const { return Header.Version; }
 
-  const Entry *getFromOffset(uint64_t Offset) const;
-  const Entry *getFromHash(uint64_t Offset) const;
+  LLVM_ABI const Entry *getFromOffset(uint64_t Offset) const;
+  LLVM_ABI const Entry *getFromHash(uint64_t Offset) const;
 
   ArrayRef<DWARFSectionKind> getColumnKinds() const {
     return ArrayRef(ColumnKinds.get(), Header.NumColumns);
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
index 717f9cedc4ee..8e1b9589a5d7 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
@@ -14,7 +14,9 @@
 #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h"
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <map>
 #include <mutex>
@@ -49,11 +51,13 @@ class OutputCategoryAggregator {
       : IncludeDetail(includeDetail) {}
   void ShowDetail(bool showDetail) { IncludeDetail = showDetail; }
   size_t GetNumCategories() const { return Aggregation.size(); }
-  void Report(StringRef category, std::function<void()> detailCallback);
-  void Report(StringRef category, StringRef sub_category,
-              std::function<void()> detailCallback);
-  void EnumerateResults(std::function<void(StringRef, unsigned)> handleCounts);
-  void EnumerateDetailedResultsFor(
+  LLVM_ABI void Report(StringRef category,
+                       std::function<void()> detailCallback);
+  LLVM_ABI void Report(StringRef category, StringRef sub_category,
+                       std::function<void()> detailCallback);
+  LLVM_ABI void
+  EnumerateResults(std::function<void(StringRef, unsigned)> handleCounts);
+  LLVM_ABI void EnumerateDetailedResultsFor(
       StringRef category,
       std::function<void(StringRef, unsigned)> handleCounts);
   /// Return the number of errors that have been reported.
@@ -91,7 +95,8 @@ class DWARFVerifier {
     /// This is used for finding overlapping ranges in the DW_AT_ranges
     /// attribute of a DIE. It is also used as a set of address ranges that
     /// children address ranges must all be contained in.
-    std::optional<DWARFAddressRange> insert(const DWARFAddressRange &R);
+    LLVM_ABI std::optional<DWARFAddressRange>
+    insert(const DWARFAddressRange &R);
 
     /// Inserts the address range info. If any of its ranges overlaps with a
     /// range in an existing range info, the range info is *not* added and an
@@ -100,14 +105,14 @@ class DWARFVerifier {
     /// and the returned iterator will point to end().
     ///
     /// This is used for finding overlapping children of the same DIE.
-    die_range_info_iterator insert(const DieRangeInfo &RI);
+    LLVM_ABI die_range_info_iterator insert(const DieRangeInfo &RI);
 
     /// Return true if ranges in this object contains all ranges within RHS.
-    bool contains(const DieRangeInfo &RHS) const;
+    LLVM_ABI bool contains(const DieRangeInfo &RHS) const;
 
     /// Return true if any range in this object intersects with any range in
     /// RHS. Identical ranges are not considered to be intersecting.
-    bool intersects(const DieRangeInfo &RHS) const;
+    LLVM_ABI bool intersects(const DieRangeInfo &RHS) const;
   };
 
 private:
@@ -319,7 +324,25 @@ class DWARFVerifier {
   void verifyDebugNames(const DWARFSection &AccelSection,
                         const DataExtractor &StrData);
 
+  /// Verify that the the expression is valid within the context of unit U.
+  ///
+  /// \param E expression to verify.
+  /// \param U containing DWARFUnit, if any.
+  ///
+  /// returns true if E is a valid expression.
+  bool verifyExpression(const DWARFExpression &E, DWARFUnit *U);
+
+  /// Verify that the the expression operation is valid within the context of
+  /// unit U.
+  ///
+  /// \param Op operation to verify
+  /// \param U containing DWARFUnit, if any
+  ///
+  /// returns true if Op is a valid Dwarf operation
+  bool verifyExpressionOp(const DWARFExpression::Operation &Op, DWARFUnit *U);
+
 public:
+  LLVM_ABI
   DWARFVerifier(raw_ostream &S, DWARFContext &D,
                 DIDumpOptions DumpOpts = DIDumpOptions::getForSingleDIE());
 
@@ -331,7 +354,7 @@ class DWARFVerifier {
   ///
   /// \returns true if .debug_abbrev and .debug_abbrev.dwo verify successfully,
   /// false otherwise.
-  bool handleDebugAbbrev();
+  LLVM_ABI bool handleDebugAbbrev();
 
   /// Verify the information in the .debug_info and .debug_types sections.
   ///
@@ -339,7 +362,7 @@ class DWARFVerifier {
   /// constructed with.
   ///
   /// \returns true if all sections verify successfully, false otherwise.
-  bool handleDebugInfo();
+  LLVM_ABI bool handleDebugInfo();
 
   /// Verify the information in the .debug_cu_index section.
   ///
@@ -348,7 +371,7 @@ class DWARFVerifier {
   ///
   /// \returns true if the .debug_cu_index verifies successfully, false
   /// otherwise.
-  bool handleDebugCUIndex();
+  LLVM_ABI bool handleDebugCUIndex();
 
   /// Verify the information in the .debug_tu_index section.
   ///
@@ -357,7 +380,7 @@ class DWARFVerifier {
   ///
   /// \returns true if the .debug_tu_index verifies successfully, false
   /// otherwise.
-  bool handleDebugTUIndex();
+  LLVM_ABI bool handleDebugTUIndex();
 
   /// Verify the information in the .debug_line section.
   ///
@@ -365,7 +388,7 @@ class DWARFVerifier {
   /// constructed with.
   ///
   /// \returns true if the .debug_line verifies successfully, false otherwise.
-  bool handleDebugLine();
+  LLVM_ABI bool handleDebugLine();
 
   /// Verify the information in accelerator tables, if they exist.
   ///
@@ -374,7 +397,7 @@ class DWARFVerifier {
   ///
   /// \returns true if the existing Apple-style accelerator tables verify
   /// successfully, false otherwise.
-  bool handleAccelTables();
+  LLVM_ABI bool handleAccelTables();
 
   /// Verify the information in the .debug_str_offsets[.dwo].
   ///
@@ -382,13 +405,14 @@ class DWARFVerifier {
   /// constructed with.
   ///
   /// \returns true if the .debug_line verifies successfully, false otherwise.
-  bool handleDebugStrOffsets();
-  bool verifyDebugStrOffsets(std::optional<dwarf::DwarfFormat> LegacyFormat,
-                             StringRef SectionName, const DWARFSection &Section,
-                             StringRef StrData);
+  LLVM_ABI bool handleDebugStrOffsets();
+  LLVM_ABI bool
+  verifyDebugStrOffsets(std::optional<dwarf::DwarfFormat> LegacyFormat,
+                        StringRef SectionName, const DWARFSection &Section,
+                        StringRef StrData);
 
   /// Emits any aggregate information collected, depending on the dump options
-  void summarize();
+  LLVM_ABI void summarize();
 };
 
 static inline bool operator<(const DWARFVerifier::DieRangeInfo &LHS,
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/CallSiteInfo.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/CallSiteInfo.h
index b09364c74db0..1a8219669e5b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/CallSiteInfo.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/CallSiteInfo.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <vector>
 
@@ -63,14 +64,14 @@ struct CallSiteInfo {
   /// \param Data The binary stream to read the data from.
   /// \param Offset The current offset within the data stream.
   /// \returns A CallSiteInfo or an error describing the issue.
-  static llvm::Expected<CallSiteInfo> decode(DataExtractor &Data,
-                                             uint64_t &Offset);
+  LLVM_ABI static llvm::Expected<CallSiteInfo> decode(DataExtractor &Data,
+                                                      uint64_t &Offset);
 
   /// Encode this CallSiteInfo object into a FileWriter stream.
   ///
   /// \param O The binary stream to write the data to.
   /// \returns An error object that indicates success or failure.
-  llvm::Error encode(FileWriter &O) const;
+  LLVM_ABI llvm::Error encode(FileWriter &O) const;
 };
 
 struct CallSiteInfoCollection {
@@ -80,13 +81,14 @@ struct CallSiteInfoCollection {
   ///
   /// \param Data The binary stream to read the data from.
   /// \returns A CallSiteInfoCollection or an error describing the issue.
-  static llvm::Expected<CallSiteInfoCollection> decode(DataExtractor &Data);
+  LLVM_ABI static llvm::Expected<CallSiteInfoCollection>
+  decode(DataExtractor &Data);
 
   /// Encode this CallSiteInfoCollection object into a FileWriter stream.
   ///
   /// \param O The binary stream to write the data to.
   /// \returns An error object that indicates success or failure.
-  llvm::Error encode(FileWriter &O) const;
+  LLVM_ABI llvm::Error encode(FileWriter &O) const;
 };
 
 class CallSiteInfoLoader {
@@ -107,7 +109,7 @@ class CallSiteInfoLoader {
   /// file to be loaded.
   /// \returns An `llvm::Error` indicating success or describing any issues
   /// encountered during the loading process.
-  llvm::Error loadYAML(StringRef YAMLFile);
+  LLVM_ABI llvm::Error loadYAML(StringRef YAMLFile);
 
 private:
   /// Builds a map from function names to FunctionInfo pointers based on the
@@ -136,8 +138,9 @@ class CallSiteInfoLoader {
   std::vector<FunctionInfo> &Funcs;
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const CallSiteInfo &CSI);
-raw_ostream &operator<<(raw_ostream &OS, const CallSiteInfoCollection &CSIC);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const CallSiteInfo &CSI);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const CallSiteInfoCollection &CSIC);
 
 } // namespace gsym
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h
index d4f39ec0dc28..77ce052201f7 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/DwarfTransformer.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/GSYM/ExtractRanges.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -57,9 +58,9 @@ class DwarfTransformer {
   ///
   /// \returns An error indicating any fatal issues that happen when parsing
   /// the DWARF, or Error::success() if all goes well.
-  llvm::Error convert(uint32_t NumThreads, OutputAggregator &OS);
+  LLVM_ABI llvm::Error convert(uint32_t NumThreads, OutputAggregator &OS);
 
-  llvm::Error verify(StringRef GsymPath, OutputAggregator &OS);
+  LLVM_ABI llvm::Error verify(StringRef GsymPath, OutputAggregator &OS);
 
 private:
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ExtractRanges.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ExtractRanges.h
index aa04518d57d5..11c872d6fdca 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ExtractRanges.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ExtractRanges.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_GSYM_EXTRACTRANGES_H
 
 #include "llvm/ADT/AddressRanges.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include <stdint.h>
@@ -36,9 +37,10 @@ class FileWriter;
 /// encoded addresses easy to relocate as we just need to relocate one base
 /// address.
 /// @{
-AddressRange decodeRange(DataExtractor &Data, uint64_t BaseAddr,
-                         uint64_t &Offset);
-void encodeRange(const AddressRange &Range, FileWriter &O, uint64_t BaseAddr);
+LLVM_ABI AddressRange decodeRange(DataExtractor &Data, uint64_t BaseAddr,
+                                  uint64_t &Offset);
+LLVM_ABI void encodeRange(const AddressRange &Range, FileWriter &O,
+                          uint64_t BaseAddr);
 /// @}
 
 /// Skip an address range object in the specified data a the specified
@@ -47,16 +49,16 @@ void encodeRange(const AddressRange &Range, FileWriter &O, uint64_t BaseAddr);
 /// \param Data The binary stream to read the data from.
 ///
 /// \param Offset The byte offset within \a Data.
-void skipRange(DataExtractor &Data, uint64_t &Offset);
+LLVM_ABI void skipRange(DataExtractor &Data, uint64_t &Offset);
 
 /// Address ranges are decoded and encoded to be relative to a base address.
 /// See the AddressRange comment for the encode and decode methods for full
 /// details.
 /// @{
-void decodeRanges(AddressRanges &Ranges, DataExtractor &Data, uint64_t BaseAddr,
-                  uint64_t &Offset);
-void encodeRanges(const AddressRanges &Ranges, FileWriter &O,
-                  uint64_t BaseAddr);
+LLVM_ABI void decodeRanges(AddressRanges &Ranges, DataExtractor &Data,
+                           uint64_t BaseAddr, uint64_t &Offset);
+LLVM_ABI void encodeRanges(const AddressRanges &Ranges, FileWriter &O,
+                           uint64_t BaseAddr);
 /// @}
 
 /// Skip an address range object in the specified data a the specified
@@ -67,13 +69,13 @@ void encodeRanges(const AddressRanges &Ranges, FileWriter &O,
 /// \param Offset The byte offset within \a Data.
 ///
 /// \returns The number of address ranges that were skipped.
-uint64_t skipRanges(DataExtractor &Data, uint64_t &Offset);
+LLVM_ABI uint64_t skipRanges(DataExtractor &Data, uint64_t &Offset);
 
 } // namespace gsym
 
-raw_ostream &operator<<(raw_ostream &OS, const AddressRange &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const AddressRange &R);
 
-raw_ostream &operator<<(raw_ostream &OS, const AddressRanges &AR);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const AddressRanges &AR);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FileWriter.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FileWriter.h
index b6bfcb115d8a..59a75259d8e3 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FileWriter.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FileWriter.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_GSYM_FILEWRITER_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 
 #include <stddef.h>
@@ -33,51 +34,51 @@ class FileWriter {
 public:
   FileWriter(llvm::raw_pwrite_stream &S, llvm::endianness B)
       : OS(S), ByteOrder(B) {}
-  ~FileWriter();
+  LLVM_ABI ~FileWriter();
   /// Write a single uint8_t value into the stream at the current file
   /// position.
   ///
   /// \param   Value The value to write into the stream.
-  void writeU8(uint8_t Value);
+  LLVM_ABI void writeU8(uint8_t Value);
 
   /// Write a single uint16_t value into the stream at the current file
   /// position. The value will be byte swapped if needed to match the byte
   /// order specified during construction.
   ///
   /// \param   Value The value to write into the stream.
-  void writeU16(uint16_t Value);
+  LLVM_ABI void writeU16(uint16_t Value);
 
   /// Write a single uint32_t value into the stream at the current file
   /// position. The value will be byte swapped if needed to match the byte
   /// order specified during construction.
   ///
   /// \param   Value The value to write into the stream.
-  void writeU32(uint32_t Value);
+  LLVM_ABI void writeU32(uint32_t Value);
 
   /// Write a single uint64_t value into the stream at the current file
   /// position. The value will be byte swapped if needed to match the byte
   /// order specified during construction.
   ///
   /// \param   Value The value to write into the stream.
-  void writeU64(uint64_t Value);
+  LLVM_ABI void writeU64(uint64_t Value);
 
   /// Write the value into the stream encoded using signed LEB128 at the
   /// current file position.
   ///
   /// \param   Value The value to write into the stream.
-  void writeSLEB(int64_t Value);
+  LLVM_ABI void writeSLEB(int64_t Value);
 
   /// Write the value into the stream encoded using unsigned LEB128 at the
   /// current file position.
   ///
   /// \param   Value The value to write into the stream.
-  void writeULEB(uint64_t Value);
+  LLVM_ABI void writeULEB(uint64_t Value);
 
   /// Write an array of uint8_t values into the stream at the current file
   /// position.
   ///
   /// \param   Data An array of values to write into the stream.
-  void writeData(llvm::ArrayRef<uint8_t> Data);
+  LLVM_ABI void writeData(llvm::ArrayRef<uint8_t> Data);
 
   /// Write a NULL terminated C string into the stream at the current file
   /// position. The entire contents of Str will be written into the steam at
@@ -86,7 +87,7 @@ class FileWriter {
   /// characters unless the additional NULL characters are desired.
   ///
   /// \param   Str The value to write into the stream.
-  void writeNullTerminated(llvm::StringRef Str);
+  LLVM_ABI void writeNullTerminated(llvm::StringRef Str);
 
   /// Fixup a uint32_t value at the specified offset in the stream. This
   /// function will save the current file position, seek to the specified
@@ -95,20 +96,20 @@ class FileWriter {
   ///
   /// \param   Value The value to write into the stream.
   /// \param   Offset The offset at which to write the Value within the stream.
-  void fixup32(uint32_t Value, uint64_t Offset);
+  LLVM_ABI void fixup32(uint32_t Value, uint64_t Offset);
 
   /// Pad with zeroes at the current file position until the current file
   /// position matches the specified alignment.
   ///
   /// \param  Align An integer speciying the desired alignment. This does not
   ///         need to be a power of two.
-  void alignTo(size_t Align);
+  LLVM_ABI void alignTo(size_t Align);
 
   /// Return the current offset within the file.
   ///
   /// \return The unsigned offset from the start of the file of the current
   ///         file position.
-  uint64_t tell();
+  LLVM_ABI uint64_t tell();
 
   llvm::raw_pwrite_stream &get_stream() {
     return OS;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h
index 19ec35c947c1..74cdd4869702 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h
@@ -17,6 +17,7 @@
 #include "llvm/DebugInfo/GSYM/LookupResult.h"
 #include "llvm/DebugInfo/GSYM/MergedFunctionsInfo.h"
 #include "llvm/DebugInfo/GSYM/StringTable.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -138,8 +139,8 @@ struct FunctionInfo {
   ///
   /// \returns An FunctionInfo or an error describing the issue that was
   /// encountered during decoding.
-  static llvm::Expected<FunctionInfo> decode(DataExtractor &Data,
-                                             uint64_t BaseAddr);
+  LLVM_ABI static llvm::Expected<FunctionInfo> decode(DataExtractor &Data,
+                                                      uint64_t BaseAddr);
 
   /// Encode this object into FileWriter stream.
   ///
@@ -155,7 +156,8 @@ struct FunctionInfo {
   ///
   /// \returns An error object that indicates failure or the offset of the
   /// function info that was successfully written into the stream.
-  llvm::Expected<uint64_t> encode(FileWriter &O, bool NoPadding = false) const;
+  LLVM_ABI llvm::Expected<uint64_t> encode(FileWriter &O,
+                                           bool NoPadding = false) const;
 
   /// Encode this function info into the internal byte cache and return the size
   /// in bytes.
@@ -167,7 +169,7 @@ struct FunctionInfo {
   ///
   /// \returns The size in bytes of the FunctionInfo if it were to be encoded
   /// into a byte stream.
-  uint64_t cacheEncoding();
+  LLVM_ABI uint64_t cacheEncoding();
 
   /// Lookup an address within a FunctionInfo object's data stream.
   ///
@@ -194,7 +196,7 @@ struct FunctionInfo {
   /// \returns An LookupResult or an error describing the issue that was
   /// encountered during decoding. An error should only be returned if the
   /// address is not contained in the FunctionInfo or if the data is corrupted.
-  static llvm::Expected<LookupResult>
+  LLVM_ABI static llvm::Expected<LookupResult>
   lookup(DataExtractor &Data, const GsymReader &GR, uint64_t FuncAddr,
          uint64_t Addr,
          std::optional<DataExtractor> *MergedFuncsData = nullptr);
@@ -238,7 +240,7 @@ inline bool operator<(const FunctionInfo &LHS, const FunctionInfo &RHS) {
          std::tie(RHS.Range, RHS.Inline, RHS.OptLineTable);
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const FunctionInfo &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const FunctionInfo &R);
 
 } // namespace gsym
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h
index 0d098da96dd2..679c9cc0dd2e 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_GSYM_GSYMCREATOR_H
 #define LLVM_DEBUGINFO_GSYM_GSYMCREATOR_H
 
+#include "llvm/Support/Compiler.h"
 #include <functional>
 #include <memory>
 #include <mutex>
@@ -292,7 +293,7 @@ class GsymCreator {
   }
 
 public:
-  GsymCreator(bool Quiet = false);
+  LLVM_ABI GsymCreator(bool Quiet = false);
 
   /// Save a GSYM file to a stand alone file.
   ///
@@ -308,14 +309,15 @@ class GsymCreator {
   ///                    a single GSYM file that contains all function
   ///                    information will be created.
   /// \returns An error object that indicates success or failure of the save.
-  llvm::Error save(StringRef Path, llvm::endianness ByteOrder,
-                   std::optional<uint64_t> SegmentSize = std::nullopt) const;
+  LLVM_ABI llvm::Error
+  save(StringRef Path, llvm::endianness ByteOrder,
+       std::optional<uint64_t> SegmentSize = std::nullopt) const;
 
   /// Encode a GSYM into the file writer stream at the current position.
   ///
   /// \param O The stream to save the binary data to
   /// \returns An error object that indicates success or failure of the save.
-  llvm::Error encode(FileWriter &O) const;
+  LLVM_ABI llvm::Error encode(FileWriter &O) const;
 
   /// Insert a string into the GSYM string table.
   ///
@@ -327,7 +329,7 @@ class GsymCreator {
   ///             the string is owned by another object that will stay around
   ///             long enough for the GsymCreator to save the GSYM file.
   /// \returns The unique 32 bit offset into the string table.
-  uint32_t insertString(StringRef S, bool Copy = true);
+  LLVM_ABI uint32_t insertString(StringRef S, bool Copy = true);
 
   /// Retrieve a string from the GSYM string table given its offset.
   ///
@@ -337,7 +339,7 @@ class GsymCreator {
   /// \param Offset The offset of the string to retrieve, previously returned by
   /// insertString.
   /// \returns The string at the given offset in the string table.
-  StringRef getString(uint32_t Offset);
+  LLVM_ABI StringRef getString(uint32_t Offset);
 
   /// Insert a file into this GSYM creator.
   ///
@@ -350,8 +352,8 @@ class GsymCreator {
   /// \param   Path The path to the file to insert.
   /// \param   Style The path style for the "Path" parameter.
   /// \returns The unique file index for the inserted file.
-  uint32_t insertFile(StringRef Path,
-                      sys::path::Style Style = sys::path::Style::native);
+  LLVM_ABI uint32_t
+  insertFile(StringRef Path, sys::path::Style Style = sys::path::Style::native);
 
   /// Add a function info to this GSYM creator.
   ///
@@ -360,7 +362,7 @@ class GsymCreator {
   /// offsets for names and other strings.
   ///
   /// \param   FI The function info object to emplace into our functions list.
-  void addFunctionInfo(FunctionInfo &&FI);
+  LLVM_ABI void addFunctionInfo(FunctionInfo &&FI);
 
   /// Load call site information from a YAML file.
   ///
@@ -369,7 +371,7 @@ class GsymCreator {
   ///
   /// \param YAMLFile The path to the YAML file containing call site
   /// information.
-  llvm::Error loadCallSitesFromYAML(StringRef YAMLFile);
+  LLVM_ABI llvm::Error loadCallSitesFromYAML(StringRef YAMLFile);
 
   /// Organize merged FunctionInfo's
   ///
@@ -378,7 +380,7 @@ class GsymCreator {
   ///
   /// \param  Out Output stream to report information about how merged
   /// FunctionInfo's were handled.
-  void prepareMergedFunctions(OutputAggregator &Out);
+  LLVM_ABI void prepareMergedFunctions(OutputAggregator &Out);
 
   /// Finalize the data in the GSYM creator prior to saving the data out.
   ///
@@ -389,7 +391,7 @@ class GsymCreator {
   ///         function infos, and function infos that were merged or removed.
   /// \returns An error object that indicates success or failure of the
   ///          finalize.
-  llvm::Error finalize(OutputAggregator &OS);
+  LLVM_ABI llvm::Error finalize(OutputAggregator &OS);
 
   /// Set the UUID value.
   ///
@@ -402,19 +404,19 @@ class GsymCreator {
   ///
   /// \param  Callback A callback function that will get called with each
   ///         FunctionInfo. If the callback returns false, stop iterating.
-  void forEachFunctionInfo(
-      std::function<bool(FunctionInfo &)> const &Callback);
+  LLVM_ABI void
+  forEachFunctionInfo(std::function<bool(FunctionInfo &)> const &Callback);
 
   /// Thread safe const iteration over all function infos.
   ///
   /// \param  Callback A callback function that will get called with each
   ///         FunctionInfo. If the callback returns false, stop iterating.
-  void forEachFunctionInfo(
+  LLVM_ABI void forEachFunctionInfo(
       std::function<bool(const FunctionInfo &)> const &Callback) const;
 
   /// Get the current number of FunctionInfo objects contained in this
   /// object.
-  size_t getNumFunctionInfos() const;
+  LLVM_ABI size_t getNumFunctionInfos() const;
 
   /// Set valid .text address ranges that all functions must be contained in.
   void SetValidTextRanges(AddressRanges &TextRanges) {
@@ -445,7 +447,7 @@ class GsymCreator {
   ///
   /// \returns True if the address is in the valid text ranges or if no valid
   ///          text ranges have been set, false otherwise.
-  bool IsValidTextAddress(uint64_t Addr) const;
+  LLVM_ABI bool IsValidTextAddress(uint64_t Addr) const;
 
   /// Set the base address to use for the GSYM file.
   ///
@@ -484,7 +486,7 @@ class GsymCreator {
   /// \returns An expected unique pointer to a GsymCreator or an error. The
   /// returned unique pointer can be NULL if there are no more functions to
   /// encode.
-  llvm::Expected<std::unique_ptr<GsymCreator>>
+  LLVM_ABI llvm::Expected<std::unique_ptr<GsymCreator>>
   createSegment(uint64_t SegmentSize, size_t &FuncIdx) const;
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h
index ee7929ae850f..d13f30e19226 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/GSYM/Header.h"
 #include "llvm/DebugInfo/GSYM/LineEntry.h"
 #include "llvm/DebugInfo/GSYM/StringTable.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorOr.h"
@@ -66,15 +67,15 @@ class GsymReader {
   std::unique_ptr<SwappedData> Swap;
 
 public:
-  GsymReader(GsymReader &&RHS);
-  ~GsymReader();
+  LLVM_ABI GsymReader(GsymReader &&RHS);
+  LLVM_ABI ~GsymReader();
 
   /// Construct a GsymReader from a file on disk.
   ///
   /// \param Path The file path the GSYM file to read.
   /// \returns An expected GsymReader that contains the object or an error
   /// object that indicates reason for failing to read the GSYM.
-  static llvm::Expected<GsymReader> openFile(StringRef Path);
+  LLVM_ABI static llvm::Expected<GsymReader> openFile(StringRef Path);
 
   /// Construct a GsymReader from a buffer.
   ///
@@ -82,11 +83,11 @@ class GsymReader {
   /// returned object on success.
   /// \returns An expected GsymReader that contains the object or an error
   /// object that indicates reason for failing to read the GSYM.
-  static llvm::Expected<GsymReader> copyBuffer(StringRef Bytes);
+  LLVM_ABI static llvm::Expected<GsymReader> copyBuffer(StringRef Bytes);
 
   /// Access the GSYM header.
   /// \returns A native endian version of the GSYM header.
-  const Header &getHeader() const;
+  LLVM_ABI const Header &getHeader() const;
 
   /// Get the full function info for an address.
   ///
@@ -104,7 +105,7 @@ class GsymReader {
   /// \returns An expected FunctionInfo that contains the function info object
   /// or an error object that indicates reason for failing to lookup the
   /// address.
-  llvm::Expected<FunctionInfo> getFunctionInfo(uint64_t Addr) const;
+  LLVM_ABI llvm::Expected<FunctionInfo> getFunctionInfo(uint64_t Addr) const;
 
   /// Get the full function info given an address index.
   ///
@@ -113,7 +114,8 @@ class GsymReader {
   /// \returns An expected FunctionInfo that contains the function info object
   /// or an error object that indicates reason for failing get the function
   /// info object.
-  llvm::Expected<FunctionInfo> getFunctionInfoAtIndex(uint64_t AddrIdx) const;
+  LLVM_ABI llvm::Expected<FunctionInfo>
+  getFunctionInfoAtIndex(uint64_t AddrIdx) const;
 
   /// Lookup an address in the a GSYM.
   ///
@@ -135,7 +137,7 @@ class GsymReader {
   /// \returns An expected LookupResult that contains only the information
   /// needed for the current address, or an error object that indicates reason
   /// for failing to lookup the address.
-  llvm::Expected<LookupResult>
+  LLVM_ABI llvm::Expected<LookupResult>
   lookup(uint64_t Addr,
          std::optional<DataExtractor> *MergedFuncsData = nullptr) const;
 
@@ -149,7 +151,8 @@ class GsymReader {
   ///
   /// \returns A vector of LookupResult objects, where the first element is the
   /// primary result, followed by results for any merged functions
-  llvm::Expected<std::vector<LookupResult>> lookupAll(uint64_t Addr) const;
+  LLVM_ABI llvm::Expected<std::vector<LookupResult>>
+  lookupAll(uint64_t Addr) const;
 
   /// Get a string from the string table.
   ///
@@ -175,7 +178,7 @@ class GsymReader {
   /// Dump the entire Gsym data contained in this object.
   ///
   /// \param  OS The output stream to dump to.
-  void dump(raw_ostream &OS);
+  LLVM_ABI void dump(raw_ostream &OS);
 
   /// Dump a FunctionInfo object.
   ///
@@ -188,7 +191,8 @@ class GsymReader {
   ///
   /// \param Indent The indentation as number of spaces. Used when dumping as an
   /// item within MergedFunctionsInfo.
-  void dump(raw_ostream &OS, const FunctionInfo &FI, uint32_t Indent = 0);
+  LLVM_ABI void dump(raw_ostream &OS, const FunctionInfo &FI,
+                     uint32_t Indent = 0);
 
   /// Dump a MergedFunctionsInfo object.
   ///
@@ -198,7 +202,7 @@ class GsymReader {
   /// \param  OS The output stream to dump to.
   ///
   /// \param MFI The object to dump.
-  void dump(raw_ostream &OS, const MergedFunctionsInfo &MFI);
+  LLVM_ABI void dump(raw_ostream &OS, const MergedFunctionsInfo &MFI);
 
   /// Dump a CallSiteInfo object.
   ///
@@ -208,7 +212,7 @@ class GsymReader {
   /// \param OS The output stream to dump to.
   ///
   /// \param CSI The CallSiteInfo object to dump.
-  void dump(raw_ostream &OS, const CallSiteInfo &CSI);
+  LLVM_ABI void dump(raw_ostream &OS, const CallSiteInfo &CSI);
 
   /// Dump a CallSiteInfoCollection object.
   ///
@@ -221,8 +225,8 @@ class GsymReader {
   ///
   /// \param Indent The indentation as number of spaces. Used when dumping as an
   /// item from within MergedFunctionsInfo.
-  void dump(raw_ostream &OS, const CallSiteInfoCollection &CSIC,
-            uint32_t Indent = 0);
+  LLVM_ABI void dump(raw_ostream &OS, const CallSiteInfoCollection &CSIC,
+                     uint32_t Indent = 0);
 
   /// Dump a LineTable object.
   ///
@@ -236,7 +240,7 @@ class GsymReader {
   ///
   /// \param Indent The indentation as number of spaces. Used when dumping as an
   /// item from within MergedFunctionsInfo.
-  void dump(raw_ostream &OS, const LineTable &LT, uint32_t Indent = 0);
+  LLVM_ABI void dump(raw_ostream &OS, const LineTable &LT, uint32_t Indent = 0);
 
   /// Dump a InlineInfo object.
   ///
@@ -249,7 +253,8 @@ class GsymReader {
   ///
   /// \param Indent The indentation as number of spaces. Used for recurive
   /// dumping.
-  void dump(raw_ostream &OS, const InlineInfo &II, uint32_t Indent = 0);
+  LLVM_ABI void dump(raw_ostream &OS, const InlineInfo &II,
+                     uint32_t Indent = 0);
 
   /// Dump a FileEntry object.
   ///
@@ -259,7 +264,7 @@ class GsymReader {
   /// \param  OS The output stream to dump to.
   ///
   /// \param FE The object to dump.
-  void dump(raw_ostream &OS, std::optional<FileEntry> FE);
+  LLVM_ABI void dump(raw_ostream &OS, std::optional<FileEntry> FE);
 
   /// Get the number of addresses in this Gsym file.
   uint32_t getNumAddresses() const {
@@ -273,7 +278,7 @@ class GsymReader {
   /// \param Index A index into the address table.
   /// \returns A resolved virtual address for adddress in the address table
   /// or std::nullopt if Index is out of bounds.
-  std::optional<uint64_t> getAddress(size_t Index) const;
+  LLVM_ABI std::optional<uint64_t> getAddress(size_t Index) const;
 
 protected:
 
@@ -359,10 +364,9 @@ class GsymReader {
   /// GsymReader.
   /// \returns An expected GsymReader that contains the object or an error
   /// object that indicates reason for failing to read the GSYM.
-  static llvm::Expected<llvm::gsym::GsymReader>
+  LLVM_ABI static llvm::Expected<llvm::gsym::GsymReader>
   create(std::unique_ptr<MemoryBuffer> &MemBuffer);
 
-
   /// Given an address, find the address index.
   ///
   /// Binary search the address table and find the matching address index.
@@ -372,7 +376,7 @@ class GsymReader {
   /// \returns An index into the address table. This index can be used to
   /// extract the FunctionInfo data's offset from the AddrInfoOffsets array.
   /// Returns an error if the address isn't in the GSYM with details of why.
-  Expected<uint64_t> getAddressIndex(const uint64_t Addr) const;
+  LLVM_ABI Expected<uint64_t> getAddressIndex(const uint64_t Addr) const;
 
   /// Given an address index, get the offset for the FunctionInfo.
   ///
@@ -383,7 +387,7 @@ class GsymReader {
   /// \param Index An index into the address table.
   /// \returns An optional GSYM data offset for the offset of the FunctionInfo
   /// that needs to be decoded.
-  std::optional<uint64_t> getAddressInfoOffset(size_t Index) const;
+  LLVM_ABI std::optional<uint64_t> getAddressInfoOffset(size_t Index) const;
 
   /// Given an address, find the correct function info data and function
   /// address.
@@ -404,7 +408,7 @@ class GsymReader {
   ///
   /// \returns An valid data extractor on success, or an error if we fail to
   /// find the address in a function info or corrrectly decode the data
-  llvm::Expected<llvm::DataExtractor>
+  LLVM_ABI llvm::Expected<llvm::DataExtractor>
   getFunctionInfoDataForAddress(uint64_t Addr, uint64_t &FuncStartAddr) const;
 
   /// Get the function data and address given an address index.
@@ -414,7 +418,7 @@ class GsymReader {
   /// \returns An expected FunctionInfo that contains the function info object
   /// or an error object that indicates reason for failing to lookup the
   /// address.
-  llvm::Expected<llvm::DataExtractor>
+  LLVM_ABI llvm::Expected<llvm::DataExtractor>
   getFunctionInfoDataAtIndex(uint64_t AddrIdx, uint64_t &FuncStartAddr) const;
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/Header.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/Header.h
index 9ca32d25985e..317b9bbdca80 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/Header.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/Header.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_GSYM_HEADER_H
 #define LLVM_DEBUGINFO_GSYM_HEADER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include <cstddef>
@@ -98,7 +99,7 @@ struct Header {
   ///
   /// \returns An error if anything is wrong in the header, or Error::success()
   /// if there are no errors.
-  llvm::Error checkForError() const;
+  LLVM_ABI llvm::Error checkForError() const;
 
   /// Decode an object from a binary data stream.
   ///
@@ -108,7 +109,7 @@ struct Header {
   ///
   /// \returns A Header or an error describing the issue that was
   /// encountered during decoding.
-  static llvm::Expected<Header> decode(DataExtractor &Data);
+  LLVM_ABI static llvm::Expected<Header> decode(DataExtractor &Data);
 
   /// Encode this object into FileWriter stream.
   ///
@@ -117,11 +118,11 @@ struct Header {
   ///
   /// \returns An error object that indicates success or failure of the
   /// encoding process.
-  llvm::Error encode(FileWriter &O) const;
+  LLVM_ABI llvm::Error encode(FileWriter &O) const;
 };
 
-bool operator==(const Header &LHS, const Header &RHS);
-raw_ostream &operator<<(raw_ostream &OS, const llvm::gsym::Header &H);
+LLVM_ABI bool operator==(const Header &LHS, const Header &RHS);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const llvm::gsym::Header &H);
 
 } // namespace gsym
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h
index 8957d4176ce1..03b0a8da35a6 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h
@@ -12,6 +12,7 @@
 #include "llvm/DebugInfo/GSYM/ExtractRanges.h"
 #include "llvm/DebugInfo/GSYM/LineEntry.h"
 #include "llvm/DebugInfo/GSYM/LookupResult.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <stdint.h>
 #include <vector>
@@ -117,9 +118,9 @@ struct InlineInfo {
   /// \returns An error if the inline information is corrupt, or
   ///          Error::success() for all other cases, even when no information
   ///          is added to \a SrcLocs.
-  static llvm::Error lookup(const GsymReader &GR, DataExtractor &Data,
-                            uint64_t BaseAddr, uint64_t Addr,
-                            SourceLocations &SrcLocs);
+  LLVM_ABI static llvm::Error lookup(const GsymReader &GR, DataExtractor &Data,
+                                     uint64_t BaseAddr, uint64_t Addr,
+                                     SourceLocations &SrcLocs);
 
   /// Lookup an address in the InlineInfo object
   ///
@@ -132,7 +133,7 @@ struct InlineInfo {
   ///
   /// \returns optional vector of InlineInfo objects that describe the
   /// inline call stack for a given address, false otherwise.
-  std::optional<InlineArray> getInlineStack(uint64_t Addr) const;
+  LLVM_ABI std::optional<InlineArray> getInlineStack(uint64_t Addr) const;
 
   /// Decode an InlineInfo object from a binary data stream.
   ///
@@ -147,8 +148,8 @@ struct InlineInfo {
   /// another InlineInfo object.
   /// \returns An InlineInfo or an error describing the issue that was
   /// encountered during decoding.
-  static llvm::Expected<InlineInfo> decode(DataExtractor &Data,
-                                           uint64_t BaseAddr);
+  LLVM_ABI static llvm::Expected<InlineInfo> decode(DataExtractor &Data,
+                                                    uint64_t BaseAddr);
 
   /// Encode this InlineInfo object into FileWriter stream.
   ///
@@ -163,7 +164,7 @@ struct InlineInfo {
   ///
   /// \returns An error object that indicates success or failure or the
   /// encoding process.
-  llvm::Error encode(FileWriter &O, uint64_t BaseAddr) const;
+  LLVM_ABI llvm::Error encode(FileWriter &O, uint64_t BaseAddr) const;
 
   /// Compare InlineInfo objects.
   ///
@@ -174,7 +175,7 @@ struct InlineInfo {
   /// to the GSYM file. We have seen cases where LTO messes up the inline
   /// function information for the same address range, so this helps ensure we
   /// get the most descriptive information we can for an address range.
-  bool operator<(const InlineInfo &RHS) const;
+  LLVM_ABI bool operator<(const InlineInfo &RHS) const;
 };
 
 inline bool operator==(const InlineInfo &LHS, const InlineInfo &RHS) {
@@ -183,7 +184,7 @@ inline bool operator==(const InlineInfo &LHS, const InlineInfo &RHS) {
          LHS.Children == RHS.Children;
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const InlineInfo &FI);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const InlineInfo &FI);
 
 } // namespace gsym
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LineTable.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LineTable.h
index 7749e5e4fbb3..a375110b0443 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LineTable.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LineTable.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_GSYM_LINETABLE_H
 
 #include "llvm/DebugInfo/GSYM/LineEntry.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <vector>
@@ -136,8 +137,8 @@ class LineTable {
   /// initialize the line table row prior to parsing any opcodes.
   ///
   /// \returns An LineEntry object if a match is found, error otherwise.
-  static Expected<LineEntry> lookup(DataExtractor &Data, uint64_t BaseAddr,
-                                    uint64_t Addr);
+  LLVM_ABI static Expected<LineEntry> lookup(DataExtractor &Data,
+                                             uint64_t BaseAddr, uint64_t Addr);
 
   /// Decode an LineTable object from a binary data stream.
   ///
@@ -151,8 +152,8 @@ class LineTable {
   ///
   /// \returns An LineTable or an error describing the issue that was
   /// encountered during decoding.
-  static llvm::Expected<LineTable> decode(DataExtractor &Data,
-                                          uint64_t BaseAddr);
+  LLVM_ABI static llvm::Expected<LineTable> decode(DataExtractor &Data,
+                                                   uint64_t BaseAddr);
   /// Encode this LineTable object into FileWriter stream.
   ///
   /// \param O The binary stream to write the data to at the current file
@@ -163,7 +164,7 @@ class LineTable {
   ///
   /// \returns An error object that indicates success or failure or the
   /// encoding process.
-  llvm::Error encode(FileWriter &O, uint64_t BaseAddr) const;
+  LLVM_ABI llvm::Error encode(FileWriter &O, uint64_t BaseAddr) const;
   bool empty() const { return Lines.empty(); }
   void clear() { Lines.clear(); }
   /// Return the first line entry if the line table isn't empty.
@@ -225,7 +226,7 @@ class LineTable {
 
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const gsym::LineTable &LT);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const gsym::LineTable &LT);
 
 } // namespace gsym
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h
index 98483805d066..cd0637e53329 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/AddressRanges.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <inttypes.h>
 #include <vector>
 
@@ -31,7 +32,7 @@ inline bool operator==(const SourceLocation &LHS, const SourceLocation &RHS) {
          LHS.Line == RHS.Line && LHS.Offset == RHS.Offset;
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const SourceLocation &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SourceLocation &R);
 
 using SourceLocations = std::vector<SourceLocation>;
 
@@ -76,7 +77,7 @@ struct LookupResult {
   /// - Call site info was not included when creating the GSYM file
   std::vector<StringRef> CallSiteFuncRegex;
 
-  std::string getSourceFile(uint32_t Index) const;
+  LLVM_ABI std::string getSourceFile(uint32_t Index) const;
 };
 
 inline bool operator==(const LookupResult &LHS, const LookupResult &RHS) {
@@ -91,7 +92,7 @@ inline bool operator==(const LookupResult &LHS, const LookupResult &RHS) {
   return LHS.Locations == RHS.Locations;
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const LookupResult &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const LookupResult &R);
 
 } // namespace gsym
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/MergedFunctionsInfo.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/MergedFunctionsInfo.h
index 203fb13cada1..048db33229a8 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/MergedFunctionsInfo.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/MergedFunctionsInfo.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_GSYM_MERGEDFUNCTIONSINFO_H
 
 #include "llvm/DebugInfo/GSYM/ExtractRanges.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <stdint.h>
 #include <vector>
@@ -24,7 +25,7 @@ struct FunctionInfo;
 struct MergedFunctionsInfo {
   std::vector<FunctionInfo> MergedFunctions;
 
-  void clear();
+  LLVM_ABI void clear();
 
   /// Query if a MergedFunctionsInfo object is valid.
   ///
@@ -40,7 +41,7 @@ struct MergedFunctionsInfo {
   ///
   /// \returns An llvm::Expected containing a vector of DataExtractor objects on
   /// success, or an error object if parsing fails.
-  static llvm::Expected<std::vector<DataExtractor>>
+  LLVM_ABI static llvm::Expected<std::vector<DataExtractor>>
   getFuncsDataExtractors(DataExtractor &Data);
 
   /// Decode an MergedFunctionsInfo object from a binary data stream.
@@ -53,8 +54,8 @@ struct MergedFunctionsInfo {
   ///
   /// \returns An MergedFunctionsInfo or an error describing the issue that was
   /// encountered during decoding.
-  static llvm::Expected<MergedFunctionsInfo> decode(DataExtractor &Data,
-                                                    uint64_t BaseAddr);
+  LLVM_ABI static llvm::Expected<MergedFunctionsInfo>
+  decode(DataExtractor &Data, uint64_t BaseAddr);
 
   /// Encode this MergedFunctionsInfo object into FileWriter stream.
   ///
@@ -62,10 +63,11 @@ struct MergedFunctionsInfo {
   /// position.
   /// \returns An error object that indicates success or failure for the
   /// encoding process.
-  llvm::Error encode(FileWriter &O) const;
+  LLVM_ABI llvm::Error encode(FileWriter &O) const;
 };
 
-bool operator==(const MergedFunctionsInfo &LHS, const MergedFunctionsInfo &RHS);
+LLVM_ABI bool operator==(const MergedFunctionsInfo &LHS,
+                         const MergedFunctionsInfo &RHS);
 
 } // namespace gsym
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h
index fe44e82e0dd2..31fdf8da9bea 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_GSYM_OBJECTFILETRANSFORMER_H
 #define LLVM_DEBUGINFO_GSYM_OBJECTFILETRANSFORMER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -39,8 +40,9 @@ class ObjectFileTransformer {
   ///
   /// \returns An error indicating any fatal issues that happen when parsing
   /// the DWARF, or Error::success() if all goes well.
-  static llvm::Error convert(const object::ObjectFile &Obj,
-                             OutputAggregator &Output, GsymCreator &Gsym);
+  LLVM_ABI static llvm::Error convert(const object::ObjectFile &Obj,
+                                      OutputAggregator &Output,
+                                      GsymCreator &Gsym);
 };
 
 } // namespace gsym
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVCompare.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVCompare.h
index 4019ea6f1744..065fef08a819 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVCompare.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVCompare.h
@@ -15,6 +15,7 @@
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVCOMPARE_H
 
 #include "llvm/DebugInfo/LogicalView/Core/LVObject.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace logicalview {
@@ -54,27 +55,27 @@ class LVCompare final {
 
 public:
   LVCompare() = delete;
-  LVCompare(raw_ostream &OS);
+  LLVM_ABI LVCompare(raw_ostream &OS);
   LVCompare(const LVCompare &) = delete;
   LVCompare &operator=(const LVCompare &) = delete;
   ~LVCompare() = default;
 
-  static LVCompare &getInstance();
+  LLVM_ABI static LVCompare &getInstance();
 
   // Scopes stack used during the missing/added reporting.
   void push(LVScope *Scope) { ScopeStack.push_back(Scope); }
   void pop() { ScopeStack.pop_back(); }
 
   // Perform comparison between the 'Reference' and 'Target' scopes tree.
-  Error execute(LVReader *ReferenceReader, LVReader *TargetReader);
+  LLVM_ABI Error execute(LVReader *ReferenceReader, LVReader *TargetReader);
 
   void addPassEntry(LVReader *Reader, LVElement *Element, LVComparePass Pass) {
     PassTable.emplace_back(Reader, Element, Pass);
   }
   const LVPassTable &getPassTable() const & { return PassTable; }
 
-  void printItem(LVElement *Element, LVComparePass Pass);
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void printItem(LVElement *Element, LVComparePass Pass);
+  LLVM_ABI void print(raw_ostream &OS) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const { print(dbgs()); }
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h
index d3a6973bb7f0..b4501db190fe 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h
@@ -15,7 +15,9 @@
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVELEMENT_H
 
 #include "llvm/DebugInfo/LogicalView/Core/LVObject.h"
+#include "llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include <map>
 #include <set>
@@ -70,7 +72,7 @@ using LVElementRequest = std::vector<LVElementGetFunction>;
 // lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp.
 constexpr unsigned int DWARF_CHAR_BIT = 8u;
 
-class LVElement : public LVObject {
+class LLVM_ABI LVElement : public LVObject {
   enum class Property {
     IsLine,   // A logical line.
     IsScope,  // A logical scope.
@@ -220,6 +222,9 @@ class LVElement : public LVObject {
   virtual StringRef getProducer() const { return StringRef(); }
   virtual void setProducer(StringRef ProducerName) {}
 
+  virtual LVSourceLanguage getSourceLanguage() const { return {}; }
+  virtual void setSourceLanguage(LVSourceLanguage SL) {}
+
   virtual bool isCompileUnit() const { return false; }
   virtual bool isRoot() const { return false; }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h
index c335c34e372b..3618ce7b0ecd 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLine.h
@@ -15,6 +15,7 @@
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVLINE_H
 
 #include "llvm/DebugInfo/LogicalView/Core/LVElement.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace logicalview {
@@ -37,7 +38,7 @@ using LVLineDispatch = std::map<LVLineKind, LVLineGetFunction>;
 using LVLineRequest = std::vector<LVLineGetFunction>;
 
 // Class to represent a logical line.
-class LVLine : public LVElement {
+class LLVM_ABI LVLine : public LVElement {
   // Typed bitvector with kinds for this line.
   LVProperties<LVLineKind> Kinds;
   static LVLineDispatch Dispatch;
@@ -104,14 +105,10 @@ class LVLine : public LVElement {
 
   void print(raw_ostream &OS, bool Full = true) const override;
   void printExtra(raw_ostream &OS, bool Full = true) const override {}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump() const override { print(dbgs()); }
-#endif
 };
 
 // Class to represent a DWARF line record object.
-class LVLineDebug final : public LVLine {
+class LLVM_ABI LVLineDebug final : public LVLine {
   // Discriminator value (DW_LNE_set_discriminator). The DWARF standard
   // defines the discriminator as an unsigned LEB128 integer.
   uint32_t Discriminator = 0;
@@ -140,7 +137,7 @@ class LVLineDebug final : public LVLine {
 };
 
 // Class to represent an assembler line extracted from the text section.
-class LVLineAssembler final : public LVLine {
+class LLVM_ABI LVLineAssembler final : public LVLine {
 public:
   LVLineAssembler() : LVLine() { setIsLineAssembler(); }
   LVLineAssembler(const LVLineAssembler &) = delete;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h
index 3b556f992783..0718e33f5645 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h
@@ -15,6 +15,7 @@
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVLOCATION_H
 
 #include "llvm/DebugInfo/LogicalView/Core/LVObject.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace logicalview {
@@ -44,17 +45,17 @@ class LVOperation final {
   ~LVOperation() = default;
 
   LVSmall getOpcode() const { return Opcode; }
-  std::string getOperandsDWARFInfo();
-  std::string getOperandsCodeViewInfo();
+  LLVM_ABI std::string getOperandsDWARFInfo();
+  LLVM_ABI std::string getOperandsCodeViewInfo();
 
-  void print(raw_ostream &OS, bool Full = true) const;
+  LLVM_ABI void print(raw_ostream &OS, bool Full = true) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump() { print(dbgs()); }
+  void dump() const { print(dbgs()); }
 #endif
 };
 
-class LVLocation : public LVObject {
+class LLVM_ABI LVLocation : public LVObject {
   enum class Property {
     IsAddressRange,
     IsBaseClassOffset,
@@ -158,13 +159,9 @@ class LVLocation : public LVObject {
 
   void print(raw_ostream &OS, bool Full = true) const override;
   void printExtra(raw_ostream &OS, bool Full = true) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump() const override { print(dbgs()); }
-#endif
 };
 
-class LVLocationSymbol final : public LVLocation {
+class LLVM_ABI LVLocationSymbol final : public LVLocation {
   // Location descriptors for the active range.
   std::unique_ptr<LVOperations> Entries;
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
index efc8db12a697..be64cdaea3d7 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
@@ -18,6 +18,7 @@
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVSupport.h"
+#include "llvm/Support/Compiler.h"
 #include <limits>
 #include <list>
 #include <string>
@@ -35,7 +36,7 @@ namespace logicalview {
 using LVSectionIndex = uint64_t;
 using LVAddress = uint64_t;
 using LVHalf = uint16_t;
-using LVLevel = uint32_t;
+using LVLevel = uint16_t;
 using LVOffset = uint64_t;
 using LVSigned = int64_t;
 using LVUnsigned = uint64_t;
@@ -54,11 +55,11 @@ class LVType;
 class LVOptions;
 class LVPatterns;
 
-StringRef typeNone();
-StringRef typeVoid();
-StringRef typeInt();
-StringRef typeUnknown();
-StringRef emptyString();
+LLVM_ABI StringRef typeNone();
+LLVM_ABI StringRef typeVoid();
+LLVM_ABI StringRef typeInt();
+LLVM_ABI StringRef typeUnknown();
+LLVM_ABI StringRef emptyString();
 
 using LVElementSetFunction = void (LVElement::*)();
 using LVElementGetFunction = bool (LVElement::*)() const;
@@ -105,7 +106,7 @@ struct LVCounter {
   }
 };
 
-class LVObject {
+class LLVM_ABI LVObject {
   enum class Property {
     IsLocation,          // Location.
     IsGlobalReference,   // This object is being referenced from another CU.
@@ -128,8 +129,6 @@ class LVObject {
     HasCodeViewLocation, // CodeView object with debug location.
     LastEntry
   };
-  // Typed bitvector with properties for this object.
-  LVProperties<Property> Properties;
 
   LVOffset Offset = 0;
   uint32_t LineNumber = 0;
@@ -139,6 +138,14 @@ class LVObject {
     dwarf::Attribute Attr;
     LVSmall Opcode;
   } TagAttrOpcode = {dwarf::DW_TAG_null};
+  // Typed bitvector with properties for this object.
+  LVProperties<Property> Properties;
+
+  // This is an internal ID used for debugging logical elements. It is used
+  // for cases where an unique offset within the binary input file is not
+  // available.
+  static uint32_t GID;
+  uint32_t ID = 0;
 
   // The parent of this object (nullptr if the root scope). For locations,
   // the parent is a symbol object; otherwise it is a scope object.
@@ -154,9 +161,7 @@ class LVObject {
   // copy constructor to create that object; it is used to print a reference
   // to another object and in the case of templates, to print its encoded args.
   LVObject(const LVObject &Object) {
-#ifndef NDEBUG
     incID();
-#endif
     Properties = Object.Properties;
     Offset = Object.Offset;
     LineNumber = Object.LineNumber;
@@ -165,18 +170,10 @@ class LVObject {
     Parent = Object.Parent;
   }
 
-#ifndef NDEBUG
-  // This is an internal ID used for debugging logical elements. It is used
-  // for cases where an unique offset within the binary input file is not
-  // available.
-  static uint64_t GID;
-  uint64_t ID = 0;
-
   void incID() {
     ++GID;
     ID = GID;
   }
-#endif
 
 protected:
   // Get a string representation for the given number and discriminator.
@@ -192,11 +189,7 @@ class LVObject {
   virtual void printFileIndex(raw_ostream &OS, bool Full = true) const {}
 
 public:
-  LVObject() {
-#ifndef NDEBUG
-    incID();
-#endif
-  };
+  LVObject() { incID(); };
   LVObject &operator=(const LVObject &) = delete;
   virtual ~LVObject() = default;
 
@@ -312,17 +305,10 @@ class LVObject {
   virtual void printExtra(raw_ostream &OS, bool Full = true) const {}
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  virtual void dump() const { print(dbgs()); }
+  void dump() const { print(dbgs()); }
 #endif
 
-  uint64_t getID() const {
-    return
-#ifndef NDEBUG
-        ID;
-#else
-        0;
-#endif
-  }
+  uint32_t getID() const { return ID; }
 };
 
 } // end namespace logicalview
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
index 409e4652b240..2afbb013fc62 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVOptions.h
@@ -19,6 +19,7 @@
 #include "llvm/DebugInfo/LogicalView/Core/LVScope.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVSymbol.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVType.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Regex.h"
 #include <set>
 #include <string>
@@ -106,6 +107,7 @@ enum class LVAttributeKind {
   Generated,     // --attribute=generated
   Global,        // --attribute=global
   Inserted,      // --attribute=inserted
+  Language,      // --attribute=language
   Level,         // --attribute=level
   Linkage,       // --attribute=linkage
   Local,         // --attribute=local
@@ -293,8 +295,8 @@ class LVOptions {
   }
 
   // Access to command line options, pattern and printing information.
-  static LVOptions *getOptions();
-  static void setOptions(LVOptions *Options);
+  LLVM_ABI static LVOptions *getOptions();
+  LLVM_ABI static void setOptions(LVOptions *Options);
 
   LVOptions() = default;
   LVOptions(const LVOptions &) = default;
@@ -307,7 +309,7 @@ class LVOptions {
   // In the case of logical view comparison, some options related to
   // attributes must be set or reset for a proper comparison.
   // Resolve any dependencies between command line options.
-  void resolveDependencies();
+  LLVM_ABI void resolveDependencies();
   size_t indentationSize() const { return IndentationSize; }
 
   LVAttribute Attribute;
@@ -337,6 +339,7 @@ class LVOptions {
   ATTRIBUTE_OPTION(Generated);
   ATTRIBUTE_OPTION(Global);
   ATTRIBUTE_OPTION(Inserted);
+  ATTRIBUTE_OPTION(Language);
   ATTRIBUTE_OPTION(Level);
   ATTRIBUTE_OPTION(Linkage);
   ATTRIBUTE_OPTION(Location);
@@ -435,7 +438,7 @@ class LVOptions {
   // General shortcuts to some combinations.
   BOOL_FUNCTION(General, CollectRanges);
 
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const { print(dbgs()); }
@@ -505,7 +508,7 @@ class LVPatterns final {
     }
   }
 
-  void addElement(LVElement *Element);
+  LLVM_ABI void addElement(LVElement *Element);
 
   template <typename T, typename U>
   void resolveGenericPatternMatch(T *Element, const U &Requests) {
@@ -548,7 +551,7 @@ class LVPatterns final {
                          bool IgnoreCase, bool UseRegex);
 
 public:
-  static LVPatterns *getPatterns();
+  LLVM_ABI static LVPatterns *getPatterns();
 
   LVPatterns() {
     ElementDispatch = LVElement::getDispatch();
@@ -592,9 +595,9 @@ class LVPatterns final {
     addRequest(Selection, TypeDispatch, TypeRequest);
   }
 
-  void updateReportOptions();
+  LLVM_ABI void updateReportOptions();
 
-  bool matchPattern(StringRef Input, const LVMatchInfo &MatchInfo);
+  LLVM_ABI bool matchPattern(StringRef Input, const LVMatchInfo &MatchInfo);
   // Match a pattern (--select='pattern').
   bool matchGenericPattern(StringRef Input) {
     return matchPattern(Input, GenericMatchInfo);
@@ -619,20 +622,20 @@ class LVPatterns final {
     resolveGenericPatternMatch(Type, TypeRequest);
   }
 
-  void addPatterns(StringSet<> &Patterns, LVMatchInfo &Filters);
+  LLVM_ABI void addPatterns(StringSet<> &Patterns, LVMatchInfo &Filters);
 
   // Add generic and offset patterns info.
-  void addGenericPatterns(StringSet<> &Patterns);
-  void addOffsetPatterns(const LVOffsetSet &Patterns);
+  LLVM_ABI void addGenericPatterns(StringSet<> &Patterns);
+  LLVM_ABI void addOffsetPatterns(const LVOffsetSet &Patterns);
 
   // Conditions to print an object.
-  bool printElement(const LVLine *Line) const;
-  bool printObject(const LVLocation *Location) const;
-  bool printElement(const LVScope *Scope) const;
-  bool printElement(const LVSymbol *Symbol) const;
-  bool printElement(const LVType *Type) const;
+  LLVM_ABI bool printElement(const LVLine *Line) const;
+  LLVM_ABI bool printObject(const LVLocation *Location) const;
+  LLVM_ABI bool printElement(const LVScope *Scope) const;
+  LLVM_ABI bool printElement(const LVSymbol *Symbol) const;
+  LLVM_ABI bool printElement(const LVType *Type) const;
 
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const { print(dbgs()); }
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h
index 3ec0ccb31168..b5c833330e59 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVRange.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/IntervalTree.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVObject.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace logicalview {
@@ -46,7 +47,7 @@ class LVRangeEntry final {
 // scope; the addresses are stored in ascending order and can overlap.
 using LVRangeEntries = std::vector<LVRangeEntry>;
 
-class LVRange final : public LVObject {
+class LLVM_ABI LVRange final : public LVObject {
   /// Map of where a user value is live, and its location.
   using LVRangesTree = IntervalTree<LVAddress, LVScope *>;
   using LVAllocator = LVRangesTree::Allocator;
@@ -86,10 +87,6 @@ class LVRange final : public LVObject {
 
   void print(raw_ostream &OS, bool Full = true) const override;
   void printExtra(raw_ostream &OS, bool Full = true) const override {}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump() const override { print(dbgs()); }
-#endif
 };
 
 } // end namespace logicalview
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h
index 870b53f6774a..371bffb2ed16 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h
@@ -16,6 +16,7 @@
 
 #include "llvm/DebugInfo/LogicalView/Core/LVOptions.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVRange.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ScopedPrinter.h"
@@ -40,9 +41,9 @@ class LVSplitContext final {
   LVSplitContext &operator=(const LVSplitContext &) = delete;
   ~LVSplitContext() = default;
 
-  Error createSplitFolder(StringRef Where);
-  std::error_code open(std::string Name, std::string Extension,
-                       raw_ostream &OS);
+  LLVM_ABI Error createSplitFolder(StringRef Where);
+  LLVM_ABI std::error_code open(std::string Name, std::string Extension,
+                                raw_ostream &OS);
   void close() {
     if (OutputFile) {
       OutputFile->os().close();
@@ -56,8 +57,8 @@ class LVSplitContext final {
 
 /// The logical reader owns of all the logical elements created during
 /// the debug information parsing. For its creation it uses a specific
-///  bump allocator for each type of logical element.
-class LVReader {
+/// bump allocator for each type of logical element.
+class LLVM_ABI LVReader {
   LVBinaryType BinaryType;
 
   // Context used by '--output=split' command line option.
@@ -122,7 +123,24 @@ class LVReader {
 
 #undef LV_OBJECT_ALLOCATOR
 
+  // Scopes with ranges for current compile unit. It is used to find a line
+  // giving its exact or closest address. To support comdat functions, all
+  // addresses for the same section are recorded in the same map.
+  using LVSectionRanges = std::map<LVSectionIndex, std::unique_ptr<LVRange>>;
+  LVSectionRanges SectionRanges;
+
 protected:
+  // Current elements during the processing of a DIE/MDNode.
+  LVElement *CurrentElement = nullptr;
+  LVScope *CurrentScope = nullptr;
+  LVSymbol *CurrentSymbol = nullptr;
+  LVType *CurrentType = nullptr;
+  LVLine *CurrentLine = nullptr;
+  LVOffset CurrentOffset = 0;
+
+  // Address ranges collected for current DIE/MDNode/AST Node.
+  std::vector<LVAddressRange> CurrentRanges;
+
   LVScopeRoot *Root = nullptr;
   std::string InputFilename;
   std::string FileFormatName;
@@ -133,11 +151,18 @@ class LVReader {
   // Only for ELF format. The CodeView is handled in a different way.
   LVSectionIndex DotTextSectionIndex = UndefinedSectionIndex;
 
+  void addSectionRange(LVSectionIndex SectionIndex, LVScope *Scope);
+  void addSectionRange(LVSectionIndex SectionIndex, LVScope *Scope,
+                       LVAddress LowerAddress, LVAddress UpperAddress);
+  LVRange *getSectionRanges(LVSectionIndex SectionIndex);
+
   // Record Compilation Unit entry.
   void addCompileUnitOffset(LVOffset Offset, LVScopeCompileUnit *CompileUnit) {
     CompileUnits.emplace(Offset, CompileUnit);
   }
 
+  LVElement *createElement(dwarf::Tag Tag);
+
   // Create the Scope Root.
   virtual Error createScopes() {
     Root = createScopeRoot();
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h
index a9bd7eef7b66..5715a37185b2 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h
@@ -18,6 +18,7 @@
 #include "llvm/DebugInfo/LogicalView/Core/LVLocation.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVSort.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include <list>
 #include <map>
 #include <set>
@@ -71,7 +72,7 @@ using LVOffsetSymbolMap = std::map<LVOffset, LVSymbol *>;
 using LVTagOffsetsMap = std::map<dwarf::Tag, LVOffsets>;
 
 // Class to represent a DWARF Scope.
-class LVScope : public LVElement {
+class LLVM_ABI LVScope : public LVElement {
   enum class Property {
     HasDiscriminator,
     CanHaveRanges,
@@ -324,14 +325,10 @@ class LVScope : public LVElement {
   void printExtra(raw_ostream &OS, bool Full = true) const override;
   virtual void printWarnings(raw_ostream &OS, bool Full = true) const {}
   virtual void printMatchedElements(raw_ostream &OS, bool UseMatchedElements) {}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump() const override { print(dbgs()); }
-#endif
 };
 
 // Class to represent a DWARF Union/Structure/Class.
-class LVScopeAggregate final : public LVScope {
+class LLVM_ABI LVScopeAggregate final : public LVScope {
   LVScope *Reference = nullptr; // DW_AT_specification, DW_AT_abstract_origin.
   size_t EncodedArgsIndex = 0;  // Template encoded arguments.
 
@@ -369,7 +366,7 @@ class LVScopeAggregate final : public LVScope {
 };
 
 // Class to represent a DWARF Template alias.
-class LVScopeAlias final : public LVScope {
+class LLVM_ABI LVScopeAlias final : public LVScope {
 public:
   LVScopeAlias() : LVScope() {
     setIsTemplateAlias();
@@ -386,7 +383,7 @@ class LVScopeAlias final : public LVScope {
 };
 
 // Class to represent a DWARF array (DW_TAG_array_type).
-class LVScopeArray final : public LVScope {
+class LLVM_ABI LVScopeArray final : public LVScope {
 public:
   LVScopeArray() : LVScope() { setIsArray(); }
   LVScopeArray(const LVScopeArray &) = delete;
@@ -402,7 +399,7 @@ class LVScopeArray final : public LVScope {
 };
 
 // Class to represent a DWARF Compilation Unit (CU).
-class LVScopeCompileUnit final : public LVScope {
+class LLVM_ABI LVScopeCompileUnit final : public LVScope {
   // Names (files and directories) used by the Compile Unit.
   std::vector<size_t> Filenames;
 
@@ -418,6 +415,9 @@ class LVScopeCompileUnit final : public LVScope {
   // Compilation directory name.
   size_t CompilationDirectoryIndex = 0;
 
+  // Source language.
+  LVSourceLanguage SourceLanguage{};
+
   // Used by the CodeView Reader.
   codeview::CPUType CompilationCPUType = codeview::CPUType::X64;
 
@@ -548,6 +548,9 @@ class LVScopeCompileUnit final : public LVScope {
     ProducerIndex = getStringPool().getIndex(ProducerName);
   }
 
+  LVSourceLanguage getSourceLanguage() const override { return SourceLanguage; }
+  void setSourceLanguage(LVSourceLanguage SL) override { SourceLanguage = SL; }
+
   void setCPUType(codeview::CPUType Type) { CompilationCPUType = Type; }
   codeview::CPUType getCPUType() { return CompilationCPUType; }
 
@@ -622,7 +625,7 @@ class LVScopeCompileUnit final : public LVScope {
 };
 
 // Class to represent a DWARF enumerator (DW_TAG_enumeration_type).
-class LVScopeEnumeration final : public LVScope {
+class LLVM_ABI LVScopeEnumeration final : public LVScope {
 public:
   LVScopeEnumeration() : LVScope() { setIsEnumeration(); }
   LVScopeEnumeration(const LVScopeEnumeration &) = delete;
@@ -637,7 +640,7 @@ class LVScopeEnumeration final : public LVScope {
 
 // Class to represent a DWARF formal parameter pack
 // (DW_TAG_GNU_formal_parameter_pack).
-class LVScopeFormalPack final : public LVScope {
+class LLVM_ABI LVScopeFormalPack final : public LVScope {
 public:
   LVScopeFormalPack() : LVScope() { setIsTemplatePack(); }
   LVScopeFormalPack(const LVScopeFormalPack &) = delete;
@@ -651,7 +654,7 @@ class LVScopeFormalPack final : public LVScope {
 };
 
 // Class to represent a DWARF Function.
-class LVScopeFunction : public LVScope {
+class LLVM_ABI LVScopeFunction : public LVScope {
   LVScope *Reference = nullptr; // DW_AT_specification, DW_AT_abstract_origin.
   size_t LinkageNameIndex = 0;  // Function DW_AT_linkage_name attribute.
   size_t EncodedArgsIndex = 0;  // Template encoded arguments.
@@ -703,7 +706,7 @@ class LVScopeFunction : public LVScope {
 };
 
 // Class to represent a DWARF inlined function.
-class LVScopeFunctionInlined final : public LVScopeFunction {
+class LLVM_ABI LVScopeFunctionInlined final : public LVScopeFunction {
   size_t CallFilenameIndex = 0;
   uint32_t CallLineNumber = 0;
   uint32_t Discriminator = 0;
@@ -746,7 +749,7 @@ class LVScopeFunctionInlined final : public LVScopeFunction {
 };
 
 // Class to represent a DWARF subroutine type.
-class LVScopeFunctionType final : public LVScopeFunction {
+class LLVM_ABI LVScopeFunctionType final : public LVScopeFunction {
 public:
   LVScopeFunctionType() : LVScopeFunction() { setIsFunctionType(); }
   LVScopeFunctionType(const LVScopeFunctionType &) = delete;
@@ -757,7 +760,7 @@ class LVScopeFunctionType final : public LVScopeFunction {
 };
 
 // Class to represent a DWARF Module.
-class LVScopeModule final : public LVScope {
+class LLVM_ABI LVScopeModule final : public LVScope {
 public:
   LVScopeModule() : LVScope() {
     setIsModule();
@@ -774,7 +777,7 @@ class LVScopeModule final : public LVScope {
 };
 
 // Class to represent a DWARF Namespace.
-class LVScopeNamespace final : public LVScope {
+class LLVM_ABI LVScopeNamespace final : public LVScope {
   LVScope *Reference = nullptr; // Reference to DW_AT_extension attribute.
 
 public:
@@ -804,7 +807,7 @@ class LVScopeNamespace final : public LVScope {
 };
 
 // Class to represent the binary file being analyzed.
-class LVScopeRoot final : public LVScope {
+class LLVM_ABI LVScopeRoot final : public LVScope {
   size_t FileFormatNameIndex = 0;
 
 public:
@@ -838,7 +841,7 @@ class LVScopeRoot final : public LVScope {
 
 // Class to represent a DWARF template parameter pack
 // (DW_TAG_GNU_template_parameter_pack).
-class LVScopeTemplatePack final : public LVScope {
+class LLVM_ABI LVScopeTemplatePack final : public LVScope {
 public:
   LVScopeTemplatePack() : LVScope() { setIsTemplatePack(); }
   LVScopeTemplatePack(const LVScopeTemplatePack &) = delete;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSort.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSort.h
index 01f0b1880fbc..c8c731028156 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSort.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSort.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSORT_H
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSORT_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 namespace logicalview {
 
@@ -33,17 +35,17 @@ using LVSortFunction = LVSortValue (*)(const LVObject *LHS,
                                        const LVObject *RHS);
 
 // Get the comparator function, based on the command line options.
-LVSortFunction getSortFunction();
+LLVM_ABI LVSortFunction getSortFunction();
 
 // Comparator functions that can be used for sorting.
-LVSortValue compareKind(const LVObject *LHS, const LVObject *RHS);
-LVSortValue compareLine(const LVObject *LHS, const LVObject *RHS);
-LVSortValue compareName(const LVObject *LHS, const LVObject *RHS);
-LVSortValue compareOffset(const LVObject *LHS, const LVObject *RHS);
-LVSortValue compareRange(const LVObject *LHS, const LVObject *RHS);
-LVSortValue sortByKind(const LVObject *LHS, const LVObject *RHS);
-LVSortValue sortByLine(const LVObject *LHS, const LVObject *RHS);
-LVSortValue sortByName(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue compareKind(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue compareLine(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue compareName(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue compareOffset(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue compareRange(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue sortByKind(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue sortByLine(const LVObject *LHS, const LVObject *RHS);
+LLVM_ABI LVSortValue sortByName(const LVObject *LHS, const LVObject *RHS);
 
 } // end namespace logicalview
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h
new file mode 100644
index 000000000000..52e4aa9701e4
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h
@@ -0,0 +1,69 @@
+//===-- LVSourceLanguage.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the LVSourceLanguage struct, a unified representation of
+// the source language used in a compile unit.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSOURCELANGUAGE_H
+#define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSOURCELANGUAGE_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/DebugInfo/CodeView/CodeView.h"
+
+namespace llvm {
+namespace logicalview {
+
+/// A source language supported by any of the debug info representations.
+struct LVSourceLanguage {
+  static constexpr unsigned TagDwarf = 0x00;
+  static constexpr unsigned TagCodeView = 0x01;
+
+  enum TaggedLanguage : uint32_t {
+    Invalid = -1U,
+
+  // DWARF
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
+  DW_LANG_##NAME = (TagDwarf << 16) | ID,
+#include "llvm/BinaryFormat/Dwarf.def"
+  // CodeView
+#define CV_LANGUAGE(NAME, ID) CV_LANG_##NAME = (TagCodeView << 16) | ID,
+#include "llvm/DebugInfo/CodeView/CodeViewLanguages.def"
+  };
+
+  LVSourceLanguage() = default;
+  LVSourceLanguage(llvm::dwarf::SourceLanguage SL)
+      : LVSourceLanguage(TagDwarf, SL) {}
+  LVSourceLanguage(llvm::codeview::SourceLanguage SL)
+      : LVSourceLanguage(TagCodeView, SL) {}
+  bool operator==(const LVSourceLanguage &SL) const {
+    return get() == SL.get();
+  }
+  bool operator==(const LVSourceLanguage::TaggedLanguage &TL) const {
+    return get() == TL;
+  }
+
+  bool isValid() const { return Language != Invalid; }
+  TaggedLanguage get() const { return Language; }
+  StringRef getName() const;
+
+private:
+  TaggedLanguage Language = Invalid;
+
+  LVSourceLanguage(unsigned Tag, unsigned Lang)
+      : Language(static_cast<TaggedLanguage>((Tag << 16) | Lang)) {}
+  unsigned getTag() const { return Language >> 16; }
+  unsigned getLang() const { return Language & 0xffff; }
+};
+
+} // end namespace logicalview
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSOURCELANGUAGE_H
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h
index 8269344fe6ef..01137f80c0f8 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVStringPool.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/Path.h"
@@ -28,7 +29,7 @@ namespace llvm {
 namespace logicalview {
 
 // Returns the unique string pool instance.
-LVStringPool &getStringPool();
+LLVM_ABI LVStringPool &getStringPool();
 
 using LVStringRefs = std::vector<StringRef>;
 using LVLexicalComponent = std::tuple<StringRef, StringRef>;
@@ -210,8 +211,8 @@ class LVDoubleMap {
 };
 
 // Unified and flattened pathnames.
-std::string transformPath(StringRef Path);
-std::string flattenedFilePath(StringRef Path);
+LLVM_ABI std::string transformPath(StringRef Path);
+LLVM_ABI std::string flattenedFilePath(StringRef Path);
 
 inline std::string formattedKind(StringRef Kind) {
   return (Twine("{") + Twine(Kind) + Twine("}")).str();
@@ -229,10 +230,10 @@ inline std::string formattedNames(StringRef Name1, StringRef Name2) {
 // scopes, such as: name, name<..>, scope::name, scope::..::name, etc.
 // The string can have multiple references to template instantiations.
 // It returns the inner most component.
-LVLexicalComponent getInnerComponent(StringRef Name);
-LVStringRefs getAllLexicalComponents(StringRef Name);
-std::string getScopedName(const LVStringRefs &Components,
-                          StringRef BaseName = {});
+LLVM_ABI LVLexicalComponent getInnerComponent(StringRef Name);
+LLVM_ABI LVStringRefs getAllLexicalComponents(StringRef Name);
+LLVM_ABI std::string getScopedName(const LVStringRefs &Components,
+                                   StringRef BaseName = {});
 
 // These are the values assigned to the debug location record IDs.
 // See DebugInfo/CodeView/CodeViewSymbols.def.
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h
index 25bfa9eb77d8..ec9017e16b65 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h
@@ -15,6 +15,7 @@
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSYMBOL_H
 
 #include "llvm/DebugInfo/LogicalView/Core/LVElement.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace logicalview {
@@ -33,7 +34,7 @@ using LVSymbolKindSet = std::set<LVSymbolKind>;
 using LVSymbolDispatch = std::map<LVSymbolKind, LVSymbolGetFunction>;
 using LVSymbolRequest = std::vector<LVSymbolGetFunction>;
 
-class LVSymbol final : public LVElement {
+class LLVM_ABI LVSymbol final : public LVElement {
   enum class Property { HasLocation, FillGaps, LastEntry };
 
   // Typed bitvector with kinds and properties for this symbol.
@@ -182,10 +183,6 @@ class LVSymbol final : public LVElement {
 
   void print(raw_ostream &OS, bool Full = true) const override;
   void printExtra(raw_ostream &OS, bool Full = true) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump() const override { print(dbgs()); }
-#endif
 };
 
 } // end namespace logicalview
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h
index 58d5bc48c3a7..59e6a92be8ce 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Core/LVType.h
@@ -15,6 +15,7 @@
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVTYPE_H
 
 #include "llvm/DebugInfo/LogicalView/Core/LVElement.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace logicalview {
@@ -48,7 +49,7 @@ using LVTypeDispatch = std::map<LVTypeKind, LVTypeGetFunction>;
 using LVTypeRequest = std::vector<LVTypeGetFunction>;
 
 // Class to represent a DWARF Type.
-class LVType : public LVElement {
+class LLVM_ABI LVType : public LVElement {
   enum class Property { IsSubrangeCount, LastEntry };
 
   // Typed bitvector with kinds and properties for this type.
@@ -145,14 +146,10 @@ class LVType : public LVElement {
 
   void print(raw_ostream &OS, bool Full = true) const override;
   void printExtra(raw_ostream &OS, bool Full = true) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump() const override { print(dbgs()); }
-#endif
 };
 
 // Class to represent DW_TAG_typedef_type.
-class LVTypeDefinition final : public LVType {
+class LLVM_ABI LVTypeDefinition final : public LVType {
 public:
   LVTypeDefinition() : LVType() {
     setIsTypedef();
@@ -175,7 +172,7 @@ class LVTypeDefinition final : public LVType {
 };
 
 // Class to represent a DW_TAG_enumerator.
-class LVTypeEnumerator final : public LVType {
+class LLVM_ABI LVTypeEnumerator final : public LVType {
   // Index in the String pool representing any initial value.
   size_t ValueIndex = 0;
 
@@ -204,7 +201,7 @@ class LVTypeEnumerator final : public LVType {
 };
 
 // Class to represent DW_TAG_imported_module / DW_TAG_imported_declaration.
-class LVTypeImport final : public LVType {
+class LLVM_ABI LVTypeImport final : public LVType {
 public:
   LVTypeImport() : LVType() { setIncludeInPrint(); }
   LVTypeImport(const LVTypeImport &) = delete;
@@ -218,7 +215,7 @@ class LVTypeImport final : public LVType {
 };
 
 // Class to represent a DWARF Template parameter holder (type or param).
-class LVTypeParam final : public LVType {
+class LLVM_ABI LVTypeParam final : public LVType {
   // Index in the String pool representing any initial value.
   size_t ValueIndex = 0;
 
@@ -247,7 +244,7 @@ class LVTypeParam final : public LVType {
 };
 
 // Class to represent a DW_TAG_subrange_type.
-class LVTypeSubrange final : public LVType {
+class LLVM_ABI LVTypeSubrange final : public LVType {
   // Values describing the subrange bounds.
   int64_t LowerBound = 0; // DW_AT_lower_bound or DW_AT_count value.
   int64_t UpperBound = 0; // DW_AT_upper_bound value.
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/LVReaderHandler.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/LVReaderHandler.h
index bf30501d00c1..f6dc65f39c7a 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/LVReaderHandler.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/LVReaderHandler.h
@@ -19,6 +19,7 @@
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include <string>
@@ -52,8 +53,8 @@ class LVReaderHandler {
                       object::Archive &Arch);
   Error handleBuffer(LVReaders &Readers, StringRef Filename,
                      MemoryBufferRef Buffer, StringRef ExePath = {});
-  Error handleFile(LVReaders &Readers, StringRef Filename,
-                   StringRef ExePath = {});
+  LLVM_ABI Error handleFile(LVReaders &Readers, StringRef Filename,
+                            StringRef ExePath = {});
   Error handleMach(LVReaders &Readers, StringRef Filename,
                    object::MachOUniversalBinary &Mach);
   Error handleObject(LVReaders &Readers, StringRef Filename,
@@ -77,7 +78,7 @@ class LVReaderHandler {
   Error createReader(StringRef Filename, LVReaders &Readers) {
     return handleFile(Readers, Filename);
   }
-  Error process();
+  LLVM_ABI Error process();
 
   Expected<std::unique_ptr<LVReader>> createReader(StringRef Pathname) {
     LVReaders Readers;
@@ -86,7 +87,7 @@ class LVReaderHandler {
     return std::move(Readers[0]);
   }
 
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const { print(dbgs()); }
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h
index 9cda64e33ddf..1847fa832348 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h
@@ -93,12 +93,6 @@ class LVBinaryReader : public LVReader {
       SectionAddresses.emplace(Section.getAddress(), Section);
   }
 
-  // Scopes with ranges for current compile unit. It is used to find a line
-  // giving its exact or closest address. To support comdat functions, all
-  // addresses for the same section are recorded in the same map.
-  using LVSectionRanges = std::map<LVSectionIndex, std::unique_ptr<LVRange>>;
-  LVSectionRanges SectionRanges;
-
   // Image base and virtual address for Executable file.
   uint64_t ImageBaseAddress = 0;
   uint64_t VirtualAddress = 0;
@@ -179,11 +173,6 @@ class LVBinaryReader : public LVReader {
   Expected<std::pair<LVSectionIndex, object::SectionRef>>
   getSection(LVScope *Scope, LVAddress Address, LVSectionIndex SectionIndex);
 
-  void addSectionRange(LVSectionIndex SectionIndex, LVScope *Scope);
-  void addSectionRange(LVSectionIndex SectionIndex, LVScope *Scope,
-                       LVAddress LowerAddress, LVAddress UpperAddress);
-  LVRange *getSectionRanges(LVSectionIndex SectionIndex);
-
   void includeInlineeLines(LVSectionIndex SectionIndex, LVScope *Function);
 
   Error createInstructions();
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
index aa47bd9cd2cd..d64723ced2e7 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
@@ -39,12 +39,6 @@ class LVDWARFReader final : public LVBinaryReader {
   LVAddress CUBaseAddress = 0;
   LVAddress CUHighAddress = 0;
 
-  // Current elements during the processing of a DIE.
-  LVElement *CurrentElement = nullptr;
-  LVScope *CurrentScope = nullptr;
-  LVSymbol *CurrentSymbol = nullptr;
-  LVType *CurrentType = nullptr;
-  LVOffset CurrentOffset = 0;
   LVOffset CurrentEndOffset = 0;
 
   // In DWARF v4, the files are 1-indexed.
@@ -52,9 +46,6 @@ class LVDWARFReader final : public LVBinaryReader {
   // The DWARF reader expects the indexes as 1-indexed.
   bool IncrementFileIndex = false;
 
-  // Address ranges collected for current DIE.
-  std::vector<LVAddressRange> CurrentRanges;
-
   // Symbols with locations for current compile unit.
   LVSymbols SymbolsWithLocations;
 
@@ -82,7 +73,6 @@ class LVDWARFReader final : public LVBinaryReader {
 
   void mapRangeAddress(const object::ObjectFile &Obj) override;
 
-  LVElement *createElement(dwarf::Tag Tag);
   void traverseDieAndChildren(DWARFDie &DIE, LVScope *Parent,
                               DWARFDie &SkeletonDie);
   // Process the attributes for the given DIE.
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h
index 2ac18a8efaba..1b32bd07771d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <utility>
@@ -54,67 +55,69 @@ class MSFBuilder {
   /// failed.  Currently the only way this can fail is if an invalid block size
   /// is specified, or `MinBlockCount` does not leave enough room for the
   /// mandatory reserved blocks required by an MSF file.
-  static Expected<MSFBuilder> create(BumpPtrAllocator &Allocator,
-                                     uint32_t BlockSize,
-                                     uint32_t MinBlockCount = 0,
-                                     bool CanGrow = true);
+  LLVM_ABI static Expected<MSFBuilder> create(BumpPtrAllocator &Allocator,
+                                              uint32_t BlockSize,
+                                              uint32_t MinBlockCount = 0,
+                                              bool CanGrow = true);
 
   /// Request the block map to be at a specific block address.  This is useful
   /// when editing a MSF and you want the layout to be as stable as possible.
-  Error setBlockMapAddr(uint32_t Addr);
-  Error setDirectoryBlocksHint(ArrayRef<uint32_t> DirBlocks);
-  void setFreePageMap(uint32_t Fpm);
-  void setUnknown1(uint32_t Unk1);
+  LLVM_ABI Error setBlockMapAddr(uint32_t Addr);
+  LLVM_ABI Error setDirectoryBlocksHint(ArrayRef<uint32_t> DirBlocks);
+  LLVM_ABI void setFreePageMap(uint32_t Fpm);
+  LLVM_ABI void setUnknown1(uint32_t Unk1);
 
   /// Add a stream to the MSF file with the given size, occupying the given
   /// list of blocks.  This is useful when reading a MSF file and you want a
   /// particular stream to occupy the original set of blocks.  If the given
   /// blocks are already allocated, or if the number of blocks specified is
   /// incorrect for the given stream size, this function will return an Error.
-  Expected<uint32_t> addStream(uint32_t Size, ArrayRef<uint32_t> Blocks);
+  LLVM_ABI Expected<uint32_t> addStream(uint32_t Size,
+                                        ArrayRef<uint32_t> Blocks);
 
   /// Add a stream to the MSF file with the given size, occupying any available
   /// blocks that the builder decides to use.  This is useful when building a
   /// new PDB file from scratch and you don't care what blocks a stream occupies
   /// but you just want it to work.
-  Expected<uint32_t> addStream(uint32_t Size);
+  LLVM_ABI Expected<uint32_t> addStream(uint32_t Size);
 
   /// Update the size of an existing stream.  This will allocate or deallocate
   /// blocks as needed to match the requested size.  This can fail if `CanGrow`
   /// was set to false when initializing the `MSFBuilder`.
-  Error setStreamSize(uint32_t Idx, uint32_t Size);
+  LLVM_ABI Error setStreamSize(uint32_t Idx, uint32_t Size);
 
   /// Get the total number of streams in the MSF layout.  This should return 1
   /// for every call to `addStream`.
-  uint32_t getNumStreams() const;
+  LLVM_ABI uint32_t getNumStreams() const;
 
   /// Get the size of a stream by index.
-  uint32_t getStreamSize(uint32_t StreamIdx) const;
+  LLVM_ABI uint32_t getStreamSize(uint32_t StreamIdx) const;
 
   /// Get the list of blocks allocated to a particular stream.
-  ArrayRef<uint32_t> getStreamBlocks(uint32_t StreamIdx) const;
+  LLVM_ABI ArrayRef<uint32_t> getStreamBlocks(uint32_t StreamIdx) const;
 
   /// Get the total number of blocks that will be allocated to actual data in
   /// this MSF file.
-  uint32_t getNumUsedBlocks() const;
+  LLVM_ABI uint32_t getNumUsedBlocks() const;
 
   /// Get the total number of blocks that exist in the MSF file but are not
   /// allocated to any valid data.
-  uint32_t getNumFreeBlocks() const;
+  LLVM_ABI uint32_t getNumFreeBlocks() const;
 
   /// Get the total number of blocks in the MSF file.  In practice this is equal
   /// to `getNumUsedBlocks() + getNumFreeBlocks()`.
-  uint32_t getTotalBlockCount() const;
+  LLVM_ABI uint32_t getTotalBlockCount() const;
 
   /// Check whether a particular block is allocated or free.
-  bool isBlockFree(uint32_t Idx) const;
+  LLVM_ABI bool isBlockFree(uint32_t Idx) const;
 
   /// Finalize the layout and build the headers and structures that describe the
   /// MSF layout and can be written directly to the MSF file.
-  Expected<MSFLayout> generateLayout();
+  LLVM_ABI Expected<MSFLayout> generateLayout();
 
   /// Write the MSF layout to the underlying file.
-  Expected<FileBufferByteStream> commit(StringRef Path, MSFLayout &Layout);
+  LLVM_ABI Expected<FileBufferByteStream> commit(StringRef Path,
+                                                 MSFLayout &Layout);
 
   BumpPtrAllocator &getAllocator() { return Allocator; }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h
index 0520b94ea3dd..65f89b848344 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MathExtras.h"
@@ -83,9 +84,9 @@ class MSFStreamLayout {
 /// Determine the layout of the FPM stream, given the MSF layout.  An FPM
 /// stream spans 1 or more blocks, each at equally spaced intervals throughout
 /// the file.
-MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf,
-                                   bool IncludeUnusedFpmData = false,
-                                   bool AltFpm = false);
+LLVM_ABI MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf,
+                                            bool IncludeUnusedFpmData = false,
+                                            bool AltFpm = false);
 
 inline bool isValidBlockSize(uint32_t Size) {
   switch (Size) {
@@ -176,7 +177,7 @@ inline uint32_t getNumFpmIntervals(const MSFLayout &L,
                             AltFpm ? L.alternateFpmBlock() : L.mainFpmBlock());
 }
 
-Error validateSuperBlock(const SuperBlock &SB);
+LLVM_ABI Error validateSuperBlock(const SuperBlock &SB);
 
 } // end namespace msf
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFError.h b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFError.h
index 0d0a43102a9b..a64db999501e 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFError.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MSFError.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_MSF_MSFERROR_H
 #define LLVM_DEBUGINFO_MSF_MSFERROR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -36,7 +37,7 @@ struct is_error_code_enum<llvm::msf::msf_error_code> : std::true_type {};
 
 namespace llvm {
 namespace msf {
-const std::error_category &MSFErrCategory();
+LLVM_ABI const std::error_category &MSFErrCategory();
 
 inline std::error_code make_error_code(msf_error_code E) {
   return std::error_code(static_cast<int>(E), MSFErrCategory());
@@ -67,7 +68,7 @@ class MSFError : public ErrorInfo<MSFError, StringError> {
     llvm_unreachable("msf error code not implemented");
   }
 
-  static char ID;
+  LLVM_ABI static char ID;
 };
 } // namespace msf
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
index 04a64d28e0cc..a5aeeddffbc2 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
@@ -15,6 +15,7 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStream.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
@@ -34,7 +35,7 @@ namespace msf {
 /// the MSF.  MappedBlockStream provides methods for reading from and writing
 /// to one of these streams transparently, as if it were a contiguous sequence
 /// of bytes.
-class MappedBlockStream : public BinaryStream {
+class LLVM_ABI MappedBlockStream : public BinaryStream {
   friend class WritableMappedBlockStream;
 
 public:
@@ -102,7 +103,7 @@ class MappedBlockStream : public BinaryStream {
   DenseMap<uint32_t, std::vector<CacheEntry>> CacheMap;
 };
 
-class WritableMappedBlockStream : public WritableBinaryStream {
+class LLVM_ABI WritableMappedBlockStream : public WritableBinaryStream {
 public:
   static std::unique_ptr<WritableMappedBlockStream>
   createStream(uint32_t BlockSize, const MSFStreamLayout &Layout,
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/GenericError.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/GenericError.h
index 1121343ed64f..31aaeb634377 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/GenericError.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/GenericError.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_PDB_GENERICERROR_H
 #define LLVM_DEBUGINFO_PDB_GENERICERROR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -32,7 +33,7 @@ struct is_error_code_enum<llvm::pdb::pdb_error_code> : std::true_type {};
 
 namespace llvm {
 namespace pdb {
-const std::error_category &PDBErrCategory();
+LLVM_ABI const std::error_category &PDBErrCategory();
 
 inline std::error_code make_error_code(pdb_error_code E) {
   return std::error_code(static_cast<int>(E), PDBErrCategory());
@@ -43,7 +44,7 @@ class PDBError : public ErrorInfo<PDBError, StringError> {
 public:
   using ErrorInfo<PDBError, StringError>::ErrorInfo; // inherit constructors
   PDBError(const Twine &S) : ErrorInfo(S, pdb_error_code::unspecified) {}
-  static char ID;
+  LLVM_ABI static char ID;
 };
 } // namespace pdb
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h
index 24138b380db4..22ef11d072f4 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBFrameData.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_PDB_IPDBFRAMEDATA_H
 #define LLVM_DEBUGINFO_PDB_IPDBFRAMEDATA_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <string>
 
@@ -17,7 +18,7 @@ namespace pdb {
 
 /// IPDBFrameData defines an interface used to represent a frame data of some
 /// code block.
-class IPDBFrameData {
+class LLVM_ABI IPDBFrameData {
 public:
   virtual ~IPDBFrameData();
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h
index 6ee6c7cc8fc1..5be252a49355 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_PDB_IPDBINJECTEDSOURCE_H
 #define LLVM_DEBUGINFO_PDB_IPDBINJECTEDSOURCE_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <string>
 
@@ -18,7 +19,7 @@ namespace pdb {
 /// which were injected directly into the PDB file during the compilation
 /// process.  This is used, for example, to add natvis files to a PDB, but
 /// in theory could be used to add arbitrary source code.
-class IPDBInjectedSource {
+class LLVM_ABI IPDBInjectedSource {
 public:
   virtual ~IPDBInjectedSource();
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBLineNumber.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBLineNumber.h
index 47b6397099b7..af6afab564c5 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBLineNumber.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBLineNumber.h
@@ -9,11 +9,12 @@
 #ifndef LLVM_DEBUGINFO_PDB_IPDBLINENUMBER_H
 #define LLVM_DEBUGINFO_PDB_IPDBLINENUMBER_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
 namespace pdb {
-class IPDBLineNumber {
+class LLVM_ABI IPDBLineNumber {
 public:
   virtual ~IPDBLineNumber();
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h
index f59e933ca575..7e15433b839e 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -32,17 +33,17 @@ enum class PdbSymbolIdField : uint32_t {
   LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ All)
 };
 
-void dumpSymbolIdField(raw_ostream &OS, StringRef Name, SymIndexId Value,
-                       int Indent, const IPDBSession &Session,
-                       PdbSymbolIdField FieldId, PdbSymbolIdField ShowFlags,
-                       PdbSymbolIdField RecurseFlags);
+LLVM_ABI void
+dumpSymbolIdField(raw_ostream &OS, StringRef Name, SymIndexId Value, int Indent,
+                  const IPDBSession &Session, PdbSymbolIdField FieldId,
+                  PdbSymbolIdField ShowFlags, PdbSymbolIdField RecurseFlags);
 
 /// IPDBRawSymbol defines an interface used to represent an arbitrary symbol.
 /// It exposes a monolithic interface consisting of accessors for the union of
 /// all properties that are valid for any symbol type.  This interface is then
 /// wrapped by a concrete class which exposes only those set of methods valid
 /// for this particular symbol type.  See PDBSymbol.h for more details.
-class IPDBRawSymbol {
+class LLVM_ABI IPDBRawSymbol {
 public:
   virtual ~IPDBRawSymbol();
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h
index 7e38654c6550..a8f2cb3756da 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h
@@ -13,6 +13,7 @@
 #include "PDBTypes.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -22,7 +23,7 @@ class PDBSymbolExe;
 
 /// IPDBSession defines an interface used to provide a context for querying
 /// debug information from a debug data source (for example, a PDB).
-class IPDBSession {
+class LLVM_ABI IPDBSession {
 public:
   virtual ~IPDBSession();
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSourceFile.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSourceFile.h
index d7e49fb70580..a16b7edd1f8c 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSourceFile.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/IPDBSourceFile.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_PDB_IPDBSOURCEFILE_H
 
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 #include <string>
 
@@ -20,7 +21,7 @@ namespace pdb {
 
 /// IPDBSourceFile defines an interface used to represent source files whose
 /// information are stored in the PDB.
-class IPDBSourceFile {
+class LLVM_ABI IPDBSourceFile {
 public:
   virtual ~IPDBSourceFile();
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
index 1ecae5c32509..d42e62fb6fc8 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 
@@ -28,24 +29,25 @@ class DbiModuleDescriptor {
   DbiModuleDescriptor(const DbiModuleDescriptor &Info) = default;
   DbiModuleDescriptor &operator=(const DbiModuleDescriptor &Info) = default;
 
-  static Error initialize(BinaryStreamRef Stream, DbiModuleDescriptor &Info);
+  LLVM_ABI static Error initialize(BinaryStreamRef Stream,
+                                   DbiModuleDescriptor &Info);
 
-  bool hasECInfo() const;
-  uint16_t getTypeServerIndex() const;
-  uint16_t getModuleStreamIndex() const;
-  uint32_t getSymbolDebugInfoByteSize() const;
-  uint32_t getC11LineInfoByteSize() const;
-  uint32_t getC13LineInfoByteSize() const;
-  uint32_t getNumberOfFiles() const;
-  uint32_t getSourceFileNameIndex() const;
-  uint32_t getPdbFilePathNameIndex() const;
+  LLVM_ABI bool hasECInfo() const;
+  LLVM_ABI uint16_t getTypeServerIndex() const;
+  LLVM_ABI uint16_t getModuleStreamIndex() const;
+  LLVM_ABI uint32_t getSymbolDebugInfoByteSize() const;
+  LLVM_ABI uint32_t getC11LineInfoByteSize() const;
+  LLVM_ABI uint32_t getC13LineInfoByteSize() const;
+  LLVM_ABI uint32_t getNumberOfFiles() const;
+  LLVM_ABI uint32_t getSourceFileNameIndex() const;
+  LLVM_ABI uint32_t getPdbFilePathNameIndex() const;
 
-  StringRef getModuleName() const;
-  StringRef getObjFileName() const;
+  LLVM_ABI StringRef getModuleName() const;
+  LLVM_ABI StringRef getObjFileName() const;
 
-  uint32_t getRecordLength() const;
+  LLVM_ABI uint32_t getRecordLength() const;
 
-  const SectionContrib &getSectionContrib() const;
+  LLVM_ABI const SectionContrib &getSectionContrib() const;
 
 private:
   StringRef ModuleName;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
index eda24e33869f..6c69cfabbe29 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h"
 #include "llvm/DebugInfo/PDB/Native/RawTypes.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <string>
@@ -64,16 +65,16 @@ class DbiModuleDescriptorBuilder {
   friend class DbiStreamBuilder;
 
 public:
-  DbiModuleDescriptorBuilder(StringRef ModuleName, uint32_t ModIndex,
-                             msf::MSFBuilder &Msf);
-  ~DbiModuleDescriptorBuilder();
+  LLVM_ABI DbiModuleDescriptorBuilder(StringRef ModuleName, uint32_t ModIndex,
+                                      msf::MSFBuilder &Msf);
+  LLVM_ABI ~DbiModuleDescriptorBuilder();
 
   DbiModuleDescriptorBuilder(const DbiModuleDescriptorBuilder &) = delete;
   DbiModuleDescriptorBuilder &
   operator=(const DbiModuleDescriptorBuilder &) = delete;
 
-  void setPdbFilePathNI(uint32_t NI);
-  void setObjFileName(StringRef Name);
+  LLVM_ABI void setPdbFilePathNI(uint32_t NI);
+  LLVM_ABI void setObjFileName(StringRef Name);
 
   // Callback to merge one source of unmerged symbols.
   using MergeSymbolsCallback = Error (*)(void *Ctx, void *Symbols,
@@ -88,21 +89,21 @@ class DbiModuleDescriptorBuilder {
     StringTableFixups = std::move(Fixups);
   }
 
-  void setFirstSectionContrib(const SectionContrib &SC);
-  void addSymbol(codeview::CVSymbol Symbol);
-  void addSymbolsInBulk(ArrayRef<uint8_t> BulkSymbols);
+  LLVM_ABI void setFirstSectionContrib(const SectionContrib &SC);
+  LLVM_ABI void addSymbol(codeview::CVSymbol Symbol);
+  LLVM_ABI void addSymbolsInBulk(ArrayRef<uint8_t> BulkSymbols);
 
   // Add symbols of known size which will be merged (rewritten) when committing
   // the PDB to disk.
-  void addUnmergedSymbols(void *SymSrc, uint32_t SymLength);
+  LLVM_ABI void addUnmergedSymbols(void *SymSrc, uint32_t SymLength);
 
-  void
+  LLVM_ABI void
   addDebugSubsection(std::shared_ptr<codeview::DebugSubsection> Subsection);
 
-  void
+  LLVM_ABI void
   addDebugSubsection(const codeview::DebugSubsectionRecord &SubsectionContents);
 
-  uint16_t getStreamIndex() const;
+  LLVM_ABI uint16_t getStreamIndex() const;
   StringRef getModuleName() const { return ModuleName; }
   StringRef getObjFileName() const { return ObjFileName; }
 
@@ -110,23 +111,23 @@ class DbiModuleDescriptorBuilder {
 
   ArrayRef<std::string> source_files() const { return SourceFiles; }
 
-  uint32_t calculateSerializedLength() const;
+  LLVM_ABI uint32_t calculateSerializedLength() const;
 
   /// Return the offset within the module symbol stream of the next symbol
   /// record passed to addSymbol. Add four to account for the signature.
   uint32_t getNextSymbolOffset() const { return SymbolByteSize + 4; }
 
-  void finalize();
-  Error finalizeMsfLayout();
+  LLVM_ABI void finalize();
+  LLVM_ABI Error finalizeMsfLayout();
 
   /// Commit the DBI descriptor to the DBI stream.
-  Error commit(BinaryStreamWriter &ModiWriter);
+  LLVM_ABI Error commit(BinaryStreamWriter &ModiWriter);
 
   /// Commit the accumulated symbols to the module symbol stream. Safe to call
   /// in parallel on different DbiModuleDescriptorBuilder objects. Only modifies
   /// the pre-allocated stream in question.
-  Error commitSymbolStream(const msf::MSFLayout &MsfLayout,
-                           WritableBinaryStreamRef MsfBuffer);
+  LLVM_ABI Error commitSymbolStream(const msf::MSFLayout &MsfLayout,
+                                    WritableBinaryStreamRef MsfBuffer);
 
 private:
   uint32_t calculateC13DebugInfoSize() const;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h
index de5b46f21672..8992faead73b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstddef>
@@ -34,22 +35,23 @@ class DbiModuleSourceFilesIterator
   using BaseType = typename DbiModuleSourceFilesIterator::iterator_facade_base;
 
 public:
-  DbiModuleSourceFilesIterator(const DbiModuleList &Modules, uint32_t Modi,
-                               uint16_t Filei);
+  LLVM_ABI DbiModuleSourceFilesIterator(const DbiModuleList &Modules,
+                                        uint32_t Modi, uint16_t Filei);
   DbiModuleSourceFilesIterator() = default;
   DbiModuleSourceFilesIterator(const DbiModuleSourceFilesIterator &R) = default;
   DbiModuleSourceFilesIterator &
   operator=(const DbiModuleSourceFilesIterator &R) = default;
 
-  bool operator==(const DbiModuleSourceFilesIterator &R) const;
+  LLVM_ABI bool operator==(const DbiModuleSourceFilesIterator &R) const;
 
   const StringRef &operator*() const { return ThisValue; }
   StringRef &operator*() { return ThisValue; }
 
-  bool operator<(const DbiModuleSourceFilesIterator &RHS) const;
-  std::ptrdiff_t operator-(const DbiModuleSourceFilesIterator &R) const;
-  DbiModuleSourceFilesIterator &operator+=(std::ptrdiff_t N);
-  DbiModuleSourceFilesIterator &operator-=(std::ptrdiff_t N);
+  LLVM_ABI bool operator<(const DbiModuleSourceFilesIterator &RHS) const;
+  LLVM_ABI std::ptrdiff_t
+  operator-(const DbiModuleSourceFilesIterator &R) const;
+  LLVM_ABI DbiModuleSourceFilesIterator &operator+=(std::ptrdiff_t N);
+  LLVM_ABI DbiModuleSourceFilesIterator &operator-=(std::ptrdiff_t N);
 
 private:
   void setValue();
@@ -68,17 +70,17 @@ class DbiModuleList {
   friend DbiModuleSourceFilesIterator;
 
 public:
-  Error initialize(BinaryStreamRef ModInfo, BinaryStreamRef FileInfo);
+  LLVM_ABI Error initialize(BinaryStreamRef ModInfo, BinaryStreamRef FileInfo);
 
-  Expected<StringRef> getFileName(uint32_t Index) const;
-  uint32_t getModuleCount() const;
-  uint32_t getSourceFileCount() const;
-  uint16_t getSourceFileCount(uint32_t Modi) const;
+  LLVM_ABI Expected<StringRef> getFileName(uint32_t Index) const;
+  LLVM_ABI uint32_t getModuleCount() const;
+  LLVM_ABI uint32_t getSourceFileCount() const;
+  LLVM_ABI uint16_t getSourceFileCount(uint32_t Modi) const;
 
-  iterator_range<DbiModuleSourceFilesIterator>
+  LLVM_ABI iterator_range<DbiModuleSourceFilesIterator>
   source_files(uint32_t Modi) const;
 
-  DbiModuleDescriptor getModuleDescriptor(uint32_t Modi) const;
+  LLVM_ABI DbiModuleDescriptor getModuleDescriptor(uint32_t Modi) const;
 
 private:
   Error initializeModInfo(BinaryStreamRef ModInfo);
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h
index 3f60130f5752..36e08cb0e1bd 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStream.h
@@ -16,6 +16,7 @@
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 
@@ -40,57 +41,59 @@ class DbiStream {
   friend class DbiStreamBuilder;
 
 public:
-  explicit DbiStream(std::unique_ptr<BinaryStream> Stream);
-  ~DbiStream();
-  Error reload(PDBFile *Pdb);
+  LLVM_ABI explicit DbiStream(std::unique_ptr<BinaryStream> Stream);
+  LLVM_ABI ~DbiStream();
+  LLVM_ABI Error reload(PDBFile *Pdb);
 
-  PdbRaw_DbiVer getDbiVersion() const;
-  uint32_t getAge() const;
-  uint16_t getPublicSymbolStreamIndex() const;
-  uint16_t getGlobalSymbolStreamIndex() const;
+  LLVM_ABI PdbRaw_DbiVer getDbiVersion() const;
+  LLVM_ABI uint32_t getAge() const;
+  LLVM_ABI uint16_t getPublicSymbolStreamIndex() const;
+  LLVM_ABI uint16_t getGlobalSymbolStreamIndex() const;
 
-  uint16_t getFlags() const;
-  bool isIncrementallyLinked() const;
-  bool hasCTypes() const;
-  bool isStripped() const;
+  LLVM_ABI uint16_t getFlags() const;
+  LLVM_ABI bool isIncrementallyLinked() const;
+  LLVM_ABI bool hasCTypes() const;
+  LLVM_ABI bool isStripped() const;
 
-  uint16_t getBuildNumber() const;
-  uint16_t getBuildMajorVersion() const;
-  uint16_t getBuildMinorVersion() const;
+  LLVM_ABI uint16_t getBuildNumber() const;
+  LLVM_ABI uint16_t getBuildMajorVersion() const;
+  LLVM_ABI uint16_t getBuildMinorVersion() const;
 
-  uint16_t getPdbDllRbld() const;
-  uint32_t getPdbDllVersion() const;
+  LLVM_ABI uint16_t getPdbDllRbld() const;
+  LLVM_ABI uint32_t getPdbDllVersion() const;
 
-  uint32_t getSymRecordStreamIndex() const;
+  LLVM_ABI uint32_t getSymRecordStreamIndex() const;
 
-  PDB_Machine getMachineType() const;
+  LLVM_ABI PDB_Machine getMachineType() const;
 
   const DbiStreamHeader *getHeader() const { return Header; }
 
-  BinarySubstreamRef getSectionContributionData() const;
-  BinarySubstreamRef getSecMapSubstreamData() const;
-  BinarySubstreamRef getModiSubstreamData() const;
-  BinarySubstreamRef getFileInfoSubstreamData() const;
-  BinarySubstreamRef getTypeServerMapSubstreamData() const;
-  BinarySubstreamRef getECSubstreamData() const;
+  LLVM_ABI BinarySubstreamRef getSectionContributionData() const;
+  LLVM_ABI BinarySubstreamRef getSecMapSubstreamData() const;
+  LLVM_ABI BinarySubstreamRef getModiSubstreamData() const;
+  LLVM_ABI BinarySubstreamRef getFileInfoSubstreamData() const;
+  LLVM_ABI BinarySubstreamRef getTypeServerMapSubstreamData() const;
+  LLVM_ABI BinarySubstreamRef getECSubstreamData() const;
 
   /// If the given stream type is present, returns its stream index. If it is
   /// not present, returns InvalidStreamIndex.
-  uint32_t getDebugStreamIndex(DbgHeaderType Type) const;
+  LLVM_ABI uint32_t getDebugStreamIndex(DbgHeaderType Type) const;
 
-  const DbiModuleList &modules() const;
+  LLVM_ABI const DbiModuleList &modules() const;
 
-  FixedStreamArray<object::coff_section> getSectionHeaders() const;
+  LLVM_ABI FixedStreamArray<object::coff_section> getSectionHeaders() const;
 
-  bool hasOldFpoRecords() const;
-  FixedStreamArray<object::FpoData> getOldFpoRecords() const;
-  bool hasNewFpoRecords() const;
-  const codeview::DebugFrameDataSubsectionRef &getNewFpoRecords() const;
+  LLVM_ABI bool hasOldFpoRecords() const;
+  LLVM_ABI FixedStreamArray<object::FpoData> getOldFpoRecords() const;
+  LLVM_ABI bool hasNewFpoRecords() const;
+  LLVM_ABI const codeview::DebugFrameDataSubsectionRef &
+  getNewFpoRecords() const;
 
-  FixedStreamArray<SecMapEntry> getSectionMap() const;
-  void visitSectionContributions(ISectionContribVisitor &Visitor) const;
+  LLVM_ABI FixedStreamArray<SecMapEntry> getSectionMap() const;
+  LLVM_ABI void
+  visitSectionContributions(ISectionContribVisitor &Visitor) const;
 
-  Expected<StringRef> getECName(uint32_t NI) const;
+  LLVM_ABI Expected<StringRef> getECName(uint32_t NI) const;
 
 private:
   Error initializeSectionContributionData();
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
index 9a84fc3e7c55..f5a4b5e2f1b8 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
@@ -14,6 +14,7 @@
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h"
@@ -39,49 +40,52 @@ class DbiModuleDescriptorBuilder;
 
 class DbiStreamBuilder {
 public:
-  DbiStreamBuilder(msf::MSFBuilder &Msf);
-  ~DbiStreamBuilder();
+  LLVM_ABI DbiStreamBuilder(msf::MSFBuilder &Msf);
+  LLVM_ABI ~DbiStreamBuilder();
 
   DbiStreamBuilder(const DbiStreamBuilder &) = delete;
   DbiStreamBuilder &operator=(const DbiStreamBuilder &) = delete;
 
-  void setVersionHeader(PdbRaw_DbiVer V);
-  void setAge(uint32_t A);
-  void setBuildNumber(uint16_t B);
-  void setBuildNumber(uint8_t Major, uint8_t Minor);
-  void setPdbDllVersion(uint16_t V);
-  void setPdbDllRbld(uint16_t R);
-  void setFlags(uint16_t F);
-  void setMachineType(PDB_Machine M);
-  void setMachineType(COFF::MachineTypes M);
+  LLVM_ABI void setVersionHeader(PdbRaw_DbiVer V);
+  LLVM_ABI void setAge(uint32_t A);
+  LLVM_ABI void setBuildNumber(uint16_t B);
+  LLVM_ABI void setBuildNumber(uint8_t Major, uint8_t Minor);
+  LLVM_ABI void setPdbDllVersion(uint16_t V);
+  LLVM_ABI void setPdbDllRbld(uint16_t R);
+  LLVM_ABI void setFlags(uint16_t F);
+  LLVM_ABI void setMachineType(PDB_Machine M);
+  LLVM_ABI void setMachineType(COFF::MachineTypes M);
 
   // Add given bytes as a new stream.
-  Error addDbgStream(pdb::DbgHeaderType Type, ArrayRef<uint8_t> Data);
+  LLVM_ABI Error addDbgStream(pdb::DbgHeaderType Type, ArrayRef<uint8_t> Data);
 
-  uint32_t addECName(StringRef Name);
+  LLVM_ABI uint32_t addECName(StringRef Name);
 
-  uint32_t calculateSerializedLength() const;
+  LLVM_ABI uint32_t calculateSerializedLength() const;
 
-  void setGlobalsStreamIndex(uint32_t Index);
-  void setPublicsStreamIndex(uint32_t Index);
-  void setSymbolRecordStreamIndex(uint32_t Index);
-  void addNewFpoData(const codeview::FrameData &FD);
-  void addOldFpoData(const object::FpoData &Fpo);
+  LLVM_ABI void setGlobalsStreamIndex(uint32_t Index);
+  LLVM_ABI void setPublicsStreamIndex(uint32_t Index);
+  LLVM_ABI void setSymbolRecordStreamIndex(uint32_t Index);
+  LLVM_ABI void addNewFpoData(const codeview::FrameData &FD);
+  LLVM_ABI void addOldFpoData(const object::FpoData &Fpo);
 
-  Expected<DbiModuleDescriptorBuilder &> addModuleInfo(StringRef ModuleName);
-  Error addModuleSourceFile(DbiModuleDescriptorBuilder &Module, StringRef File);
-  Expected<uint32_t> getSourceFileNameIndex(StringRef FileName);
+  LLVM_ABI Expected<DbiModuleDescriptorBuilder &>
+  addModuleInfo(StringRef ModuleName);
+  LLVM_ABI Error addModuleSourceFile(DbiModuleDescriptorBuilder &Module,
+                                     StringRef File);
+  LLVM_ABI Expected<uint32_t> getSourceFileNameIndex(StringRef FileName);
 
-  Error finalizeMsfLayout();
+  LLVM_ABI Error finalizeMsfLayout();
 
-  Error commit(const msf::MSFLayout &Layout, WritableBinaryStreamRef MsfBuffer);
+  LLVM_ABI Error commit(const msf::MSFLayout &Layout,
+                        WritableBinaryStreamRef MsfBuffer);
 
   void addSectionContrib(const SectionContrib &SC) {
     SectionContribs.emplace_back(SC);
   }
 
   // Populate the Section Map from COFF section headers.
-  void createSectionMap(ArrayRef<llvm::object::coff_section> SecHdrs);
+  LLVM_ABI void createSectionMap(ArrayRef<llvm::object::coff_section> SecHdrs);
 
 private:
   struct DebugStream {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h
index 0adbd25cb369..76a019ddf8f3 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/FormatAdapters.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -38,27 +39,28 @@ template <typename T> std::string formatUnknownEnum(T Value) {
   return formatv("unknown ({0})", llvm::to_underlying(Value)).str();
 }
 
-std::string formatSegmentOffset(uint16_t Segment, uint32_t Offset);
+LLVM_ABI std::string formatSegmentOffset(uint16_t Segment, uint32_t Offset);
 
 enum class CharacteristicStyle {
   HeaderDefinition, // format as windows header definition
   Descriptive,      // format as human readable words
 };
-std::string formatSectionCharacteristics(
+LLVM_ABI std::string formatSectionCharacteristics(
     uint32_t IndentLevel, uint32_t C, uint32_t FlagsPerLine,
     StringRef Separator,
     CharacteristicStyle Style = CharacteristicStyle::HeaderDefinition);
 
-std::string typesetItemList(ArrayRef<std::string> Opts, uint32_t IndentLevel,
-                            uint32_t GroupSize, StringRef Sep);
+LLVM_ABI std::string typesetItemList(ArrayRef<std::string> Opts,
+                                     uint32_t IndentLevel, uint32_t GroupSize,
+                                     StringRef Sep);
 
-std::string typesetStringList(uint32_t IndentLevel,
-                              ArrayRef<StringRef> Strings);
+LLVM_ABI std::string typesetStringList(uint32_t IndentLevel,
+                                       ArrayRef<StringRef> Strings);
 
-std::string formatChunkKind(codeview::DebugSubsectionKind Kind,
-                            bool Friendly = true);
-std::string formatSymbolKind(codeview::SymbolKind K);
-std::string formatTypeLeafKind(codeview::TypeLeafKind K);
+LLVM_ABI std::string formatChunkKind(codeview::DebugSubsectionKind Kind,
+                                     bool Friendly = true);
+LLVM_ABI std::string formatSymbolKind(codeview::SymbolKind K);
+LLVM_ABI std::string formatTypeLeafKind(codeview::TypeLeafKind K);
 
 /// Returns the number of digits in the given integer.
 inline int NumDigits(uint64_t N) {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
index 28a72c887f25..ff61fd7b9cc5 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/PDB/Native/GlobalsStream.h"
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -46,30 +47,31 @@ struct SymbolDenseMapInfo;
 class GSIStreamBuilder {
 
 public:
-  explicit GSIStreamBuilder(msf::MSFBuilder &Msf);
-  ~GSIStreamBuilder();
+  LLVM_ABI explicit GSIStreamBuilder(msf::MSFBuilder &Msf);
+  LLVM_ABI ~GSIStreamBuilder();
 
   GSIStreamBuilder(const GSIStreamBuilder &) = delete;
   GSIStreamBuilder &operator=(const GSIStreamBuilder &) = delete;
 
-  Error finalizeMsfLayout();
+  LLVM_ABI Error finalizeMsfLayout();
 
-  Error commit(const msf::MSFLayout &Layout, WritableBinaryStreamRef Buffer);
+  LLVM_ABI Error commit(const msf::MSFLayout &Layout,
+                        WritableBinaryStreamRef Buffer);
 
   uint32_t getPublicsStreamIndex() const { return PublicsStreamIndex; }
   uint32_t getGlobalsStreamIndex() const { return GlobalsStreamIndex; }
   uint32_t getRecordStreamIndex() const { return RecordStreamIndex; }
 
   // Add public symbols in bulk.
-  void addPublicSymbols(std::vector<BulkPublic> &&PublicsIn);
+  LLVM_ABI void addPublicSymbols(std::vector<BulkPublic> &&PublicsIn);
 
-  void addGlobalSymbol(const codeview::ProcRefSym &Sym);
-  void addGlobalSymbol(const codeview::DataSym &Sym);
-  void addGlobalSymbol(const codeview::ConstantSym &Sym);
+  LLVM_ABI void addGlobalSymbol(const codeview::ProcRefSym &Sym);
+  LLVM_ABI void addGlobalSymbol(const codeview::DataSym &Sym);
+  LLVM_ABI void addGlobalSymbol(const codeview::ConstantSym &Sym);
 
   // Add a pre-serialized global symbol record. The caller must ensure that the
   // symbol data remains alive until the global stream is committed to disk.
-  void addGlobalSymbol(const codeview::CVSymbol &Sym);
+  LLVM_ABI void addGlobalSymbol(const codeview::CVSymbol &Sym);
 
 private:
   void finalizePublicBuckets();
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h
index 2988bef4a75b..730167df019b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h
@@ -13,6 +13,7 @@
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
 #include "llvm/DebugInfo/PDB/Native/RawTypes.h"
 #include "llvm/Support/BinaryStreamArray.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 
@@ -55,7 +56,7 @@ class GSIHashTable {
   FixedStreamArray<support::ulittle32_t> HashBuckets;
   std::array<int32_t, IPHR_HASH + 1> BucketMap;
 
-  Error read(BinaryStreamReader &Reader);
+  LLVM_ABI Error read(BinaryStreamReader &Reader);
 
   uint32_t getVerSignature() const { return HashHdr->VerSignature; }
   uint32_t getVerHeader() const { return HashHdr->VerHdr; }
@@ -69,12 +70,13 @@ class GSIHashTable {
 
 class GlobalsStream {
 public:
-  explicit GlobalsStream(std::unique_ptr<msf::MappedBlockStream> Stream);
-  ~GlobalsStream();
+  LLVM_ABI explicit GlobalsStream(
+      std::unique_ptr<msf::MappedBlockStream> Stream);
+  LLVM_ABI ~GlobalsStream();
   const GSIHashTable &getGlobalsTable() const { return GlobalsTable; }
-  Error reload();
+  LLVM_ABI Error reload();
 
-  std::vector<std::pair<uint32_t, codeview::CVSymbol>>
+  LLVM_ABI std::vector<std::pair<uint32_t, codeview::CVSymbol>>
   findRecordsByName(StringRef Name, const SymbolStream &Symbols) const;
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/Hash.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/Hash.h
index b048d878a12c..ab036ce8c328 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/Hash.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/Hash.h
@@ -11,14 +11,15 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
 namespace pdb {
 
-uint32_t hashStringV1(StringRef Str);
-uint32_t hashStringV2(StringRef Str);
-uint32_t hashBufferV8(ArrayRef<uint8_t> Data);
+LLVM_ABI uint32_t hashStringV1(StringRef Str);
+LLVM_ABI uint32_t hashStringV2(StringRef Str);
+LLVM_ABI uint32_t hashBufferV8(ArrayRef<uint8_t> Data);
 
 } // end namespace pdb
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h
index eb03397ba694..efece7a17676 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h
@@ -14,6 +14,7 @@
 #include "llvm/DebugInfo/PDB/Native/RawError.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
@@ -25,8 +26,10 @@ namespace llvm {
 
 namespace pdb {
 
-Error readSparseBitVector(BinaryStreamReader &Stream, SparseBitVector<> &V);
-Error writeSparseBitVector(BinaryStreamWriter &Writer, SparseBitVector<> &Vec);
+LLVM_ABI Error readSparseBitVector(BinaryStreamReader &Stream,
+                                   SparseBitVector<> &V);
+LLVM_ABI Error writeSparseBitVector(BinaryStreamWriter &Writer,
+                                    SparseBitVector<> &Vec);
 
 template <typename ValueT> class HashTable;
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h
index 625bab6a4378..d30787e55810 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStream.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/Support/BinaryStream.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/Support/Error.h"
 
@@ -25,30 +26,30 @@ class InfoStream {
   friend class InfoStreamBuilder;
 
 public:
-  InfoStream(std::unique_ptr<BinaryStream> Stream);
+  LLVM_ABI InfoStream(std::unique_ptr<BinaryStream> Stream);
 
-  Error reload();
+  LLVM_ABI Error reload();
 
-  uint32_t getStreamSize() const;
+  LLVM_ABI uint32_t getStreamSize() const;
 
   const InfoStreamHeader *getHeader() const { return Header; }
 
-  bool containsIdStream() const;
-  PdbRaw_ImplVer getVersion() const;
-  uint32_t getSignature() const;
-  uint32_t getAge() const;
-  codeview::GUID getGuid() const;
-  uint32_t getNamedStreamMapByteSize() const;
+  LLVM_ABI bool containsIdStream() const;
+  LLVM_ABI PdbRaw_ImplVer getVersion() const;
+  LLVM_ABI uint32_t getSignature() const;
+  LLVM_ABI uint32_t getAge() const;
+  LLVM_ABI codeview::GUID getGuid() const;
+  LLVM_ABI uint32_t getNamedStreamMapByteSize() const;
 
-  PdbRaw_Features getFeatures() const;
-  ArrayRef<PdbRaw_FeatureSig> getFeatureSignatures() const;
+  LLVM_ABI PdbRaw_Features getFeatures() const;
+  LLVM_ABI ArrayRef<PdbRaw_FeatureSig> getFeatureSignatures() const;
 
-  const NamedStreamMap &getNamedStreams() const;
+  LLVM_ABI const NamedStreamMap &getNamedStreams() const;
 
-  BinarySubstreamRef getNamedStreamsBuffer() const;
+  LLVM_ABI BinarySubstreamRef getNamedStreamsBuffer() const;
 
-  Expected<uint32_t> getNamedStreamIndex(llvm::StringRef Name) const;
-  StringMap<uint32_t> named_streams() const;
+  LLVM_ABI Expected<uint32_t> getNamedStreamIndex(llvm::StringRef Name) const;
+  LLVM_ABI StringMap<uint32_t> named_streams() const;
 
 private:
   std::unique_ptr<BinaryStream> Stream;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h
index 4b77e321353f..692bc70f0123 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_PDB_NATIVE_INFOSTREAMBUILDER_H
 #define LLVM_DEBUGINFO_PDB_NATIVE_INFOSTREAMBUILDER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include "llvm/DebugInfo/CodeView/GUID.h"
@@ -26,33 +27,34 @@ class NamedStreamMap;
 
 class InfoStreamBuilder {
 public:
-  InfoStreamBuilder(msf::MSFBuilder &Msf, NamedStreamMap &NamedStreams);
+  LLVM_ABI InfoStreamBuilder(msf::MSFBuilder &Msf,
+                             NamedStreamMap &NamedStreams);
   InfoStreamBuilder(const InfoStreamBuilder &) = delete;
   InfoStreamBuilder &operator=(const InfoStreamBuilder &) = delete;
 
-  void setVersion(PdbRaw_ImplVer V);
-  void addFeature(PdbRaw_FeatureSig Sig);
+  LLVM_ABI void setVersion(PdbRaw_ImplVer V);
+  LLVM_ABI void addFeature(PdbRaw_FeatureSig Sig);
 
   // If this is true, the PDB contents are hashed and this hash is used as
   // PDB GUID and as Signature. The age is always 1.
-  void setHashPDBContentsToGUID(bool B);
+  LLVM_ABI void setHashPDBContentsToGUID(bool B);
 
   // These only have an effect if hashPDBContentsToGUID() is false.
-  void setSignature(uint32_t S);
-  void setAge(uint32_t A);
-  void setGuid(codeview::GUID G);
+  LLVM_ABI void setSignature(uint32_t S);
+  LLVM_ABI void setAge(uint32_t A);
+  LLVM_ABI void setGuid(codeview::GUID G);
 
   bool hashPDBContentsToGUID() const { return HashPDBContentsToGUID; }
   uint32_t getAge() const { return Age; }
   codeview::GUID getGuid() const { return Guid; }
   std::optional<uint32_t> getSignature() const { return Signature; }
 
-  uint32_t finalize();
+  LLVM_ABI uint32_t finalize();
 
-  Error finalizeMsfLayout();
+  LLVM_ABI Error finalizeMsfLayout();
 
-  Error commit(const msf::MSFLayout &Layout,
-               WritableBinaryStreamRef Buffer) const;
+  LLVM_ABI Error commit(const msf::MSFLayout &Layout,
+                        WritableBinaryStreamRef Buffer) const;
 
 private:
   msf::MSFBuilder &Msf;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h
index 834cd96b77b4..0e7b9663f27d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h
@@ -18,6 +18,7 @@
 #include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -57,57 +58,57 @@ class InputFile {
   InputFile(PDBFile *Pdb) { PdbOrObj = Pdb; }
   InputFile(object::COFFObjectFile *Obj) { PdbOrObj = Obj; }
   InputFile(MemoryBuffer *Buffer) { PdbOrObj = Buffer; }
-  ~InputFile();
+  LLVM_ABI ~InputFile();
   InputFile(InputFile &&Other) = default;
 
-  static Expected<InputFile> open(StringRef Path,
-                                  bool AllowUnknownFile = false);
+  LLVM_ABI static Expected<InputFile> open(StringRef Path,
+                                           bool AllowUnknownFile = false);
 
-  PDBFile &pdb();
-  const PDBFile &pdb() const;
-  object::COFFObjectFile &obj();
-  const object::COFFObjectFile &obj() const;
-  MemoryBuffer &unknown();
-  const MemoryBuffer &unknown() const;
+  LLVM_ABI PDBFile &pdb();
+  LLVM_ABI const PDBFile &pdb() const;
+  LLVM_ABI object::COFFObjectFile &obj();
+  LLVM_ABI const object::COFFObjectFile &obj() const;
+  LLVM_ABI MemoryBuffer &unknown();
+  LLVM_ABI const MemoryBuffer &unknown() const;
 
-  StringRef getFilePath() const;
+  LLVM_ABI StringRef getFilePath() const;
 
-  bool hasTypes() const;
-  bool hasIds() const;
+  LLVM_ABI bool hasTypes() const;
+  LLVM_ABI bool hasIds() const;
 
-  codeview::LazyRandomTypeCollection &types();
-  codeview::LazyRandomTypeCollection &ids();
+  LLVM_ABI codeview::LazyRandomTypeCollection &types();
+  LLVM_ABI codeview::LazyRandomTypeCollection &ids();
 
-  iterator_range<SymbolGroupIterator> symbol_groups();
-  SymbolGroupIterator symbol_groups_begin();
-  SymbolGroupIterator symbol_groups_end();
+  LLVM_ABI iterator_range<SymbolGroupIterator> symbol_groups();
+  LLVM_ABI SymbolGroupIterator symbol_groups_begin();
+  LLVM_ABI SymbolGroupIterator symbol_groups_end();
 
-  bool isPdb() const;
-  bool isObj() const;
-  bool isUnknown() const;
+  LLVM_ABI bool isPdb() const;
+  LLVM_ABI bool isObj() const;
+  LLVM_ABI bool isUnknown() const;
 };
 
 class SymbolGroup {
   friend class SymbolGroupIterator;
 
 public:
-  explicit SymbolGroup(InputFile *File, uint32_t GroupIndex = 0);
+  LLVM_ABI explicit SymbolGroup(InputFile *File, uint32_t GroupIndex = 0);
 
-  Expected<StringRef> getNameFromStringTable(uint32_t Offset) const;
-  Expected<StringRef> getNameFromChecksums(uint32_t Offset) const;
+  LLVM_ABI Expected<StringRef> getNameFromStringTable(uint32_t Offset) const;
+  LLVM_ABI Expected<StringRef> getNameFromChecksums(uint32_t Offset) const;
 
-  void formatFromFileName(LinePrinter &Printer, StringRef File,
-                          bool Append = false) const;
+  LLVM_ABI void formatFromFileName(LinePrinter &Printer, StringRef File,
+                                   bool Append = false) const;
 
-  void formatFromChecksumsOffset(LinePrinter &Printer, uint32_t Offset,
-                                 bool Append = false) const;
+  LLVM_ABI void formatFromChecksumsOffset(LinePrinter &Printer, uint32_t Offset,
+                                          bool Append = false) const;
 
-  StringRef name() const;
+  LLVM_ABI StringRef name() const;
 
   codeview::DebugSubsectionArray getDebugSubsections() const {
     return Subsections;
   }
-  const ModuleDebugStreamRef &getPdbModuleStream() const;
+  LLVM_ABI const ModuleDebugStreamRef &getPdbModuleStream() const;
 
   const InputFile &getFile() const { return *File; }
   InputFile &getFile() { return *File; }
@@ -132,16 +133,16 @@ class SymbolGroupIterator
     : public iterator_facade_base<SymbolGroupIterator,
                                   std::forward_iterator_tag, SymbolGroup> {
 public:
-  SymbolGroupIterator();
-  explicit SymbolGroupIterator(InputFile &File);
+  LLVM_ABI SymbolGroupIterator();
+  LLVM_ABI explicit SymbolGroupIterator(InputFile &File);
   SymbolGroupIterator(const SymbolGroupIterator &Other) = default;
   SymbolGroupIterator &operator=(const SymbolGroupIterator &R) = default;
 
-  const SymbolGroup &operator*() const;
-  SymbolGroup &operator*();
+  LLVM_ABI const SymbolGroup &operator*() const;
+  LLVM_ABI SymbolGroup &operator*();
 
-  bool operator==(const SymbolGroupIterator &R) const;
-  SymbolGroupIterator &operator++();
+  LLVM_ABI bool operator==(const SymbolGroupIterator &R) const;
+  LLVM_ABI SymbolGroupIterator &operator++();
 
 private:
   void scanToNextDebugS();
@@ -152,13 +153,13 @@ class SymbolGroupIterator
   SymbolGroup Value;
 };
 
-Expected<ModuleDebugStreamRef>
+LLVM_ABI Expected<ModuleDebugStreamRef>
 getModuleDebugStream(PDBFile &File, StringRef &ModuleName, uint32_t Index);
-Expected<ModuleDebugStreamRef> getModuleDebugStream(PDBFile &File,
-                                                    uint32_t Index);
+LLVM_ABI Expected<ModuleDebugStreamRef> getModuleDebugStream(PDBFile &File,
+                                                             uint32_t Index);
 
-bool shouldDumpSymbolGroup(uint32_t Idx, const SymbolGroup &Group,
-                           const FilterOptions &Filters);
+LLVM_ABI bool shouldDumpSymbolGroup(uint32_t Idx, const SymbolGroup &Group,
+                                    const FilterOptions &Filters);
 
 // TODO: Change these callbacks to be function_refs (de-templatify them).
 template <typename CallbackT>
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h
index bb029e534c74..95293ba0818d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h
@@ -14,6 +14,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/DebugInfo/PDB/Native/FormatUtil.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/raw_ostream.h"
@@ -51,15 +52,15 @@ class LinePrinter {
   friend class WithColor;
 
 public:
-  LinePrinter(int Indent, bool UseColor, raw_ostream &Stream,
-              const FilterOptions &Filters);
+  LLVM_ABI LinePrinter(int Indent, bool UseColor, raw_ostream &Stream,
+                       const FilterOptions &Filters);
 
-  void Indent(uint32_t Amount = 0);
-  void Unindent(uint32_t Amount = 0);
-  void NewLine();
+  LLVM_ABI void Indent(uint32_t Amount = 0);
+  LLVM_ABI void Unindent(uint32_t Amount = 0);
+  LLVM_ABI void NewLine();
 
-  void printLine(const Twine &T);
-  void print(const Twine &T);
+  LLVM_ABI void printLine(const Twine &T);
+  LLVM_ABI void print(const Twine &T);
   template <typename... Ts> void formatLine(const char *Fmt, Ts &&...Items) {
     printLine(formatv(Fmt, std::forward<Ts>(Items)...));
   }
@@ -67,27 +68,28 @@ class LinePrinter {
     print(formatv(Fmt, std::forward<Ts>(Items)...));
   }
 
-  void formatBinary(StringRef Label, ArrayRef<uint8_t> Data,
-                    uint64_t StartOffset);
-  void formatBinary(StringRef Label, ArrayRef<uint8_t> Data, uint64_t BaseAddr,
-                    uint64_t StartOffset);
+  LLVM_ABI void formatBinary(StringRef Label, ArrayRef<uint8_t> Data,
+                             uint64_t StartOffset);
+  LLVM_ABI void formatBinary(StringRef Label, ArrayRef<uint8_t> Data,
+                             uint64_t BaseAddr, uint64_t StartOffset);
 
-  void formatMsfStreamData(StringRef Label, PDBFile &File, uint32_t StreamIdx,
-                           StringRef StreamPurpose, uint64_t Offset,
-                           uint64_t Size);
-  void formatMsfStreamData(StringRef Label, PDBFile &File,
-                           const msf::MSFStreamLayout &Stream,
-                           BinarySubstreamRef Substream);
-  void formatMsfStreamBlocks(PDBFile &File, const msf::MSFStreamLayout &Stream);
+  LLVM_ABI void formatMsfStreamData(StringRef Label, PDBFile &File,
+                                    uint32_t StreamIdx, StringRef StreamPurpose,
+                                    uint64_t Offset, uint64_t Size);
+  LLVM_ABI void formatMsfStreamData(StringRef Label, PDBFile &File,
+                                    const msf::MSFStreamLayout &Stream,
+                                    BinarySubstreamRef Substream);
+  LLVM_ABI void formatMsfStreamBlocks(PDBFile &File,
+                                      const msf::MSFStreamLayout &Stream);
 
   bool hasColor() const { return UseColor; }
   raw_ostream &getStream() { return OS; }
   int getIndentLevel() const { return CurrentIndent; }
 
-  bool IsClassExcluded(const ClassLayout &Class);
-  bool IsTypeExcluded(llvm::StringRef TypeName, uint64_t Size);
-  bool IsSymbolExcluded(llvm::StringRef SymbolName);
-  bool IsCompilandExcluded(llvm::StringRef CompilandName);
+  LLVM_ABI bool IsClassExcluded(const ClassLayout &Class);
+  LLVM_ABI bool IsTypeExcluded(llvm::StringRef TypeName, uint64_t Size);
+  LLVM_ABI bool IsSymbolExcluded(llvm::StringRef SymbolName);
+  LLVM_ABI bool IsCompilandExcluded(llvm::StringRef CompilandName);
 
   const FilterOptions &getFilters() const { return Filters; }
 
@@ -169,8 +171,8 @@ enum class PDB_ColorItem {
 
 class WithColor {
 public:
-  WithColor(LinePrinter &P, PDB_ColorItem C);
-  ~WithColor();
+  LLVM_ABI WithColor(LinePrinter &P, PDB_ColorItem C);
+  LLVM_ABI ~WithColor();
 
   raw_ostream &get() { return OS; }
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h
index 0caf9fffbad6..65ae994e28ab 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h
@@ -14,6 +14,7 @@
 #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h"
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <memory>
@@ -32,42 +33,42 @@ class ModuleDebugStreamRef {
   using DebugSubsectionIterator = codeview::DebugSubsectionArray::Iterator;
 
 public:
-  ModuleDebugStreamRef(const DbiModuleDescriptor &Module,
-                       std::unique_ptr<msf::MappedBlockStream> Stream);
+  LLVM_ABI ModuleDebugStreamRef(const DbiModuleDescriptor &Module,
+                                std::unique_ptr<msf::MappedBlockStream> Stream);
   ModuleDebugStreamRef(ModuleDebugStreamRef &&Other) = default;
   ModuleDebugStreamRef(const ModuleDebugStreamRef &Other) = default;
-  ~ModuleDebugStreamRef();
+  LLVM_ABI ~ModuleDebugStreamRef();
 
-  Error reload();
+  LLVM_ABI Error reload();
 
   uint32_t signature() const { return Signature; }
 
-  iterator_range<codeview::CVSymbolArray::Iterator>
+  LLVM_ABI iterator_range<codeview::CVSymbolArray::Iterator>
   symbols(bool *HadError) const;
 
   const codeview::CVSymbolArray &getSymbolArray() const { return SymbolArray; }
-  const codeview::CVSymbolArray
+  LLVM_ABI const codeview::CVSymbolArray
   getSymbolArrayForScope(uint32_t ScopeBegin) const;
 
-  BinarySubstreamRef getSymbolsSubstream() const;
-  BinarySubstreamRef getC11LinesSubstream() const;
-  BinarySubstreamRef getC13LinesSubstream() const;
-  BinarySubstreamRef getGlobalRefsSubstream() const;
+  LLVM_ABI BinarySubstreamRef getSymbolsSubstream() const;
+  LLVM_ABI BinarySubstreamRef getC11LinesSubstream() const;
+  LLVM_ABI BinarySubstreamRef getC13LinesSubstream() const;
+  LLVM_ABI BinarySubstreamRef getGlobalRefsSubstream() const;
 
   ModuleDebugStreamRef &operator=(ModuleDebugStreamRef &&Other) = delete;
 
-  codeview::CVSymbol readSymbolAtOffset(uint32_t Offset) const;
+  LLVM_ABI codeview::CVSymbol readSymbolAtOffset(uint32_t Offset) const;
 
-  iterator_range<DebugSubsectionIterator> subsections() const;
+  LLVM_ABI iterator_range<DebugSubsectionIterator> subsections() const;
   codeview::DebugSubsectionArray getSubsectionsArray() const {
     return Subsections;
   }
 
-  bool hasDebugSubsections() const;
+  LLVM_ABI bool hasDebugSubsections() const;
 
-  Error commit();
+  LLVM_ABI Error commit();
 
-  Expected<codeview::DebugChecksumsSubsectionRef>
+  LLVM_ABI Expected<codeview::DebugChecksumsSubsectionRef>
   findChecksumsSubsection() const;
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
index 18fbab0dd38c..0a3690b88128 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/PDB/Native/HashTable.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 
@@ -27,31 +28,31 @@ class NamedStreamMap;
 struct NamedStreamMapTraits {
   NamedStreamMap *NS;
 
-  explicit NamedStreamMapTraits(NamedStreamMap &NS);
-  uint16_t hashLookupKey(StringRef S) const;
-  StringRef storageKeyToLookupKey(uint32_t Offset) const;
-  uint32_t lookupKeyToStorageKey(StringRef S);
+  LLVM_ABI explicit NamedStreamMapTraits(NamedStreamMap &NS);
+  LLVM_ABI uint16_t hashLookupKey(StringRef S) const;
+  LLVM_ABI StringRef storageKeyToLookupKey(uint32_t Offset) const;
+  LLVM_ABI uint32_t lookupKeyToStorageKey(StringRef S);
 };
 
 class NamedStreamMap {
   friend class NamedStreamMapBuilder;
 
 public:
-  NamedStreamMap();
+  LLVM_ABI NamedStreamMap();
 
-  Error load(BinaryStreamReader &Stream);
-  Error commit(BinaryStreamWriter &Writer) const;
-  uint32_t calculateSerializedLength() const;
+  LLVM_ABI Error load(BinaryStreamReader &Stream);
+  LLVM_ABI Error commit(BinaryStreamWriter &Writer) const;
+  LLVM_ABI uint32_t calculateSerializedLength() const;
 
-  uint32_t size() const;
-  bool get(StringRef Stream, uint32_t &StreamNo) const;
-  void set(StringRef Stream, uint32_t StreamNo);
+  LLVM_ABI uint32_t size() const;
+  LLVM_ABI bool get(StringRef Stream, uint32_t &StreamNo) const;
+  LLVM_ABI void set(StringRef Stream, uint32_t StreamNo);
 
-  uint32_t appendStringData(StringRef S);
-  StringRef getString(uint32_t Offset) const;
-  uint32_t hashString(uint32_t Offset) const;
+  LLVM_ABI uint32_t appendStringData(StringRef S);
+  LLVM_ABI StringRef getString(uint32_t Offset) const;
+  LLVM_ABI uint32_t hashString(uint32_t Offset) const;
 
-  StringMap<uint32_t> entries() const;
+  LLVM_ABI StringMap<uint32_t> entries() const;
 
 private:
   NamedStreamMapTraits HashTraits;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
index 4133be220713..21a93ae7a13b 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_PDB_NATIVE_NATIVERAWSYMBOL_H
 
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <memory>
 
@@ -18,7 +19,7 @@ namespace pdb {
 
 class NativeSession;
 
-class NativeRawSymbol : public IPDBRawSymbol {
+class LLVM_ABI NativeRawSymbol : public IPDBRawSymbol {
   friend class SymbolCache;
   virtual void initialize() {}
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h
index 95be7d09aae9..2bdc9dc028ad 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/PDB/Native/SymbolCache.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -29,7 +30,7 @@ class PDBSymbolCompiland;
 class PDBSymbolExe;
 template <typename ChildType> class IPDBEnumChildren;
 
-class NativeSession : public IPDBSession {
+class LLVM_ABI NativeSession : public IPDBSession {
   struct PdbSearchOptions {
     StringRef ExePath;
     // FIXME: Add other PDB search options (_NT_SYMBOL_PATH, symsrv)
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h
index c6653368bc0c..0506e644608f 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/NativeSourceFile.h
@@ -12,6 +12,7 @@
 #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h"
 #include "llvm/DebugInfo/PDB/IPDBSourceFile.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace pdb {
@@ -19,7 +20,7 @@ class PDBSymbolCompiland;
 template <typename ChildType> class IPDBEnumChildren;
 class NativeSession;
 
-class NativeSourceFile : public IPDBSourceFile {
+class LLVM_ABI NativeSourceFile : public IPDBSourceFile {
 public:
   explicit NativeSourceFile(NativeSession &Session, uint32_t FileId,
                             const codeview::FileChecksumEntry &Checksum);
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h
index 1ea92ed4bf21..62e2c665e546 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h
@@ -13,6 +13,7 @@
 #include "llvm/DebugInfo/MSF/MSFCommon.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 
@@ -37,7 +38,7 @@ class PublicsStream;
 class SymbolStream;
 class TpiStream;
 
-class PDBFile : public msf::IMSFFile {
+class LLVM_ABI PDBFile : public msf::IMSFFile {
   friend PDBFileBuilder;
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
index c23d958f8ed0..abb1b623c5a8 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/PDB/Native/NamedStreamMap.h"
 #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <memory>
@@ -38,28 +39,29 @@ class TpiStreamBuilder;
 
 class PDBFileBuilder {
 public:
-  explicit PDBFileBuilder(BumpPtrAllocator &Allocator);
-  ~PDBFileBuilder();
+  LLVM_ABI explicit PDBFileBuilder(BumpPtrAllocator &Allocator);
+  LLVM_ABI ~PDBFileBuilder();
   PDBFileBuilder(const PDBFileBuilder &) = delete;
   PDBFileBuilder &operator=(const PDBFileBuilder &) = delete;
 
-  Error initialize(uint32_t BlockSize);
+  LLVM_ABI Error initialize(uint32_t BlockSize);
 
-  msf::MSFBuilder &getMsfBuilder();
-  InfoStreamBuilder &getInfoBuilder();
-  DbiStreamBuilder &getDbiBuilder();
-  TpiStreamBuilder &getTpiBuilder();
-  TpiStreamBuilder &getIpiBuilder();
-  PDBStringTableBuilder &getStringTableBuilder();
-  GSIStreamBuilder &getGsiBuilder();
+  LLVM_ABI msf::MSFBuilder &getMsfBuilder();
+  LLVM_ABI InfoStreamBuilder &getInfoBuilder();
+  LLVM_ABI DbiStreamBuilder &getDbiBuilder();
+  LLVM_ABI TpiStreamBuilder &getTpiBuilder();
+  LLVM_ABI TpiStreamBuilder &getIpiBuilder();
+  LLVM_ABI PDBStringTableBuilder &getStringTableBuilder();
+  LLVM_ABI GSIStreamBuilder &getGsiBuilder();
 
   // If HashPDBContentsToGUID is true on the InfoStreamBuilder, Guid is filled
   // with the computed PDB GUID on return.
-  Error commit(StringRef Filename, codeview::GUID *Guid);
+  LLVM_ABI Error commit(StringRef Filename, codeview::GUID *Guid);
 
-  Expected<uint32_t> getNamedStreamIndex(StringRef Name) const;
-  Error addNamedStream(StringRef Name, StringRef Data);
-  void addInjectedSource(StringRef Name, std::unique_ptr<MemoryBuffer> Buffer);
+  LLVM_ABI Expected<uint32_t> getNamedStreamIndex(StringRef Name) const;
+  LLVM_ABI Error addNamedStream(StringRef Name, StringRef Data);
+  LLVM_ABI void addInjectedSource(StringRef Name,
+                                  std::unique_ptr<MemoryBuffer> Buffer);
 
 private:
   struct InjectedSourceDescriptor {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
index 4336cd398baf..32fbc5914f86 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h"
 #include "llvm/Support/BinaryStreamArray.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
@@ -25,19 +26,20 @@ struct PDBStringTableHeader;
 
 class PDBStringTable {
 public:
-  Error reload(BinaryStreamReader &Reader);
+  LLVM_ABI Error reload(BinaryStreamReader &Reader);
 
-  uint32_t getByteSize() const;
-  uint32_t getNameCount() const;
-  uint32_t getHashVersion() const;
-  uint32_t getSignature() const;
+  LLVM_ABI uint32_t getByteSize() const;
+  LLVM_ABI uint32_t getNameCount() const;
+  LLVM_ABI uint32_t getHashVersion() const;
+  LLVM_ABI uint32_t getSignature() const;
 
-  Expected<StringRef> getStringForID(uint32_t ID) const;
-  Expected<uint32_t> getIDForString(StringRef Str) const;
+  LLVM_ABI Expected<StringRef> getStringForID(uint32_t ID) const;
+  LLVM_ABI Expected<uint32_t> getIDForString(StringRef Str) const;
 
-  FixedStreamArray<support::ulittle32_t> name_ids() const;
+  LLVM_ABI FixedStreamArray<support::ulittle32_t> name_ids() const;
 
-  const codeview::DebugStringTableSubsectionRef &getStringTable() const;
+  LLVM_ABI const codeview::DebugStringTableSubsectionRef &
+  getStringTable() const;
 
 private:
   Error readHeader(BinaryStreamReader &Reader);
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h
index 986bd3645108..4ef583a227c0 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 
@@ -34,25 +35,25 @@ class PDBStringTableBuilder;
 struct StringTableHashTraits {
   PDBStringTableBuilder *Table;
 
-  explicit StringTableHashTraits(PDBStringTableBuilder &Table);
-  uint32_t hashLookupKey(StringRef S) const;
-  StringRef storageKeyToLookupKey(uint32_t Offset) const;
-  uint32_t lookupKeyToStorageKey(StringRef S);
+  LLVM_ABI explicit StringTableHashTraits(PDBStringTableBuilder &Table);
+  LLVM_ABI uint32_t hashLookupKey(StringRef S) const;
+  LLVM_ABI StringRef storageKeyToLookupKey(uint32_t Offset) const;
+  LLVM_ABI uint32_t lookupKeyToStorageKey(StringRef S);
 };
 
 class PDBStringTableBuilder {
 public:
   // If string S does not exist in the string table, insert it.
   // Returns the ID for S.
-  uint32_t insert(StringRef S);
+  LLVM_ABI uint32_t insert(StringRef S);
 
-  uint32_t getIdForString(StringRef S) const;
-  StringRef getStringForId(uint32_t Id) const;
+  LLVM_ABI uint32_t getIdForString(StringRef S) const;
+  LLVM_ABI StringRef getStringForId(uint32_t Id) const;
 
-  uint32_t calculateSerializedSize() const;
-  Error commit(BinaryStreamWriter &Writer) const;
+  LLVM_ABI uint32_t calculateSerializedSize() const;
+  LLVM_ABI Error commit(BinaryStreamWriter &Writer) const;
 
-  void setStrings(const codeview::DebugStringTableSubsection &Strings);
+  LLVM_ABI void setStrings(const codeview::DebugStringTableSubsection &Strings);
 
 private:
   uint32_t calculateHashTableSize() const;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
index a59a752ff911..2cb4bee8ca5d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
@@ -11,6 +11,7 @@
 
 #include "llvm/DebugInfo/PDB/Native/GlobalsStream.h"
 #include "llvm/Support/BinaryStreamArray.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -23,13 +24,13 @@ struct SectionOffset;
 
 class PublicsStream {
 public:
-  PublicsStream(std::unique_ptr<msf::MappedBlockStream> Stream);
-  ~PublicsStream();
-  Error reload();
+  LLVM_ABI PublicsStream(std::unique_ptr<msf::MappedBlockStream> Stream);
+  LLVM_ABI ~PublicsStream();
+  LLVM_ABI Error reload();
 
-  uint32_t getSymHash() const;
-  uint16_t getThunkTableSection() const;
-  uint32_t getThunkTableOffset() const;
+  LLVM_ABI uint32_t getSymHash() const;
+  LLVM_ABI uint16_t getThunkTableSection() const;
+  LLVM_ABI uint32_t getThunkTableOffset() const;
   const GSIHashTable &getPublicsTable() const { return PublicsTable; }
   FixedStreamArray<support::ulittle32_t> getAddressMap() const {
     return AddressMap;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h
index 8836adf9497f..550f32421bde 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/RawError.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_DEBUGINFO_PDB_NATIVE_RAWERROR_H
 #define LLVM_DEBUGINFO_PDB_NATIVE_RAWERROR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -38,7 +39,7 @@ struct is_error_code_enum<llvm::pdb::raw_error_code> : std::true_type {};
 
 namespace llvm {
 namespace pdb {
-const std::error_category &RawErrCategory();
+LLVM_ABI const std::error_category &RawErrCategory();
 
 inline std::error_code make_error_code(raw_error_code E) {
   return std::error_code(static_cast<int>(E), RawErrCategory());
@@ -49,7 +50,7 @@ class RawError : public ErrorInfo<RawError, StringError> {
 public:
   using ErrorInfo<RawError, StringError>::ErrorInfo; // inherit constructors
   RawError(const Twine &S) : ErrorInfo(S, raw_error_code::unspecified) {}
-  static char ID;
+  LLVM_ABI static char ID;
 };
 } // namespace pdb
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
index 7c5b6b9e1bdf..9bbceca14c81 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
@@ -18,6 +18,7 @@
 #include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h"
 #include "llvm/DebugInfo/PDB/Native/NativeSourceFile.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 #include <memory>
 #include <vector>
@@ -121,7 +122,7 @@ class SymbolCache {
                                                           uint32_t Offset);
 
 public:
-  SymbolCache(NativeSession &Session, DbiStream *Dbi);
+  LLVM_ABI SymbolCache(NativeSession &Session, DbiStream *Dbi);
 
   template <typename ConcreteSymbolT, typename... Args>
   SymIndexId createSymbol(Args &&...ConstructorArgs) const {
@@ -142,16 +143,16 @@ class SymbolCache {
     return Id;
   }
 
-  std::unique_ptr<IPDBEnumSymbols>
+  LLVM_ABI std::unique_ptr<IPDBEnumSymbols>
   createTypeEnumerator(codeview::TypeLeafKind Kind);
 
-  std::unique_ptr<IPDBEnumSymbols>
+  LLVM_ABI std::unique_ptr<IPDBEnumSymbols>
   createTypeEnumerator(std::vector<codeview::TypeLeafKind> Kinds);
 
-  std::unique_ptr<IPDBEnumSymbols>
+  LLVM_ABI std::unique_ptr<IPDBEnumSymbols>
   createGlobalsEnumerator(codeview::SymbolKind Kind);
 
-  SymIndexId findSymbolByTypeIndex(codeview::TypeIndex TI) const;
+  LLVM_ABI SymIndexId findSymbolByTypeIndex(codeview::TypeIndex TI) const;
 
   template <typename ConcreteSymbolT, typename... Args>
   SymIndexId getOrCreateFieldListMember(codeview::TypeIndex FieldListTI,
@@ -168,31 +169,34 @@ class SymbolCache {
     return SymId;
   }
 
-  SymIndexId getOrCreateGlobalSymbolByOffset(uint32_t Offset);
-  SymIndexId getOrCreateInlineSymbol(codeview::InlineSiteSym Sym,
-                                     uint64_t ParentAddr, uint16_t Modi,
-                                     uint32_t RecordOffset) const;
+  LLVM_ABI SymIndexId getOrCreateGlobalSymbolByOffset(uint32_t Offset);
+  LLVM_ABI SymIndexId getOrCreateInlineSymbol(codeview::InlineSiteSym Sym,
+                                              uint64_t ParentAddr,
+                                              uint16_t Modi,
+                                              uint32_t RecordOffset) const;
 
-  std::unique_ptr<PDBSymbol>
+  LLVM_ABI std::unique_ptr<PDBSymbol>
   findSymbolBySectOffset(uint32_t Sect, uint32_t Offset, PDB_SymType Type);
 
-  std::unique_ptr<IPDBEnumLineNumbers>
+  LLVM_ABI std::unique_ptr<IPDBEnumLineNumbers>
   findLineNumbersByVA(uint64_t VA, uint32_t Length) const;
 
-  std::unique_ptr<PDBSymbolCompiland> getOrCreateCompiland(uint32_t Index);
-  uint32_t getNumCompilands() const;
+  LLVM_ABI std::unique_ptr<PDBSymbolCompiland>
+  getOrCreateCompiland(uint32_t Index);
+  LLVM_ABI uint32_t getNumCompilands() const;
 
-  std::unique_ptr<PDBSymbol> getSymbolById(SymIndexId SymbolId) const;
+  LLVM_ABI std::unique_ptr<PDBSymbol> getSymbolById(SymIndexId SymbolId) const;
 
-  NativeRawSymbol &getNativeSymbolById(SymIndexId SymbolId) const;
+  LLVM_ABI NativeRawSymbol &getNativeSymbolById(SymIndexId SymbolId) const;
 
   template <typename ConcreteT>
   ConcreteT &getNativeSymbolById(SymIndexId SymbolId) const {
     return static_cast<ConcreteT &>(getNativeSymbolById(SymbolId));
   }
 
-  std::unique_ptr<IPDBSourceFile> getSourceFileById(SymIndexId FileId) const;
-  SymIndexId
+  LLVM_ABI std::unique_ptr<IPDBSourceFile>
+  getSourceFileById(SymIndexId FileId) const;
+  LLVM_ABI SymIndexId
   getOrCreateSourceFile(const codeview::FileChecksumEntry &Checksum) const;
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h
index c2f7eb04d16e..846c8238c95c 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_PDB_NATIVE_SYMBOLSTREAM_H
 
 #include "llvm/DebugInfo/CodeView/CVRecord.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/Support/Error.h"
 
@@ -21,20 +22,20 @@ namespace pdb {
 
 class SymbolStream {
 public:
-  SymbolStream(std::unique_ptr<msf::MappedBlockStream> Stream);
-  ~SymbolStream();
-  Error reload();
+  LLVM_ABI SymbolStream(std::unique_ptr<msf::MappedBlockStream> Stream);
+  LLVM_ABI ~SymbolStream();
+  LLVM_ABI Error reload();
 
   const codeview::CVSymbolArray &getSymbolArray() const {
     return SymbolRecords;
   }
 
-  codeview::CVSymbol readRecord(uint32_t Offset) const;
+  LLVM_ABI codeview::CVSymbol readRecord(uint32_t Offset) const;
 
-  iterator_range<codeview::CVSymbolArray::Iterator>
+  LLVM_ABI iterator_range<codeview::CVSymbolArray::Iterator>
   getSymbols(bool *HadError) const;
 
-  Error commit();
+  LLVM_ABI Error commit();
 
 private:
   codeview::CVSymbolArray SymbolRecords;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h
index 3cdb890d81d3..c76f3d1f2eb2 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiHashing.h
@@ -10,12 +10,13 @@
 #define LLVM_DEBUGINFO_PDB_NATIVE_TPIHASHING_H
 
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace pdb {
 
-Expected<uint32_t> hashTypeRecord(const llvm::codeview::CVType &Type);
+LLVM_ABI Expected<uint32_t> hashTypeRecord(const llvm::codeview::CVType &Type);
 
 struct TagRecordHash {
   explicit TagRecordHash(codeview::ClassRecord CR, uint32_t Full,
@@ -63,7 +64,7 @@ struct TagRecordHash {
 
 /// Given a CVType referring to a class, structure, union, or enum, compute
 /// the hash of its forward decl and full decl.
-Expected<TagRecordHash> hashTagRecord(const codeview::CVType &Type);
+LLVM_ABI Expected<TagRecordHash> hashTagRecord(const codeview::CVType &Type);
 
 } // end namespace pdb
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h
index 4c413abb2bf0..ec8ac8563b00 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h
@@ -14,6 +14,7 @@
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/Support/BinaryStreamArray.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/Support/Error.h"
 
@@ -35,43 +36,46 @@ class TpiStream {
   friend class TpiStreamBuilder;
 
 public:
-  TpiStream(PDBFile &File, std::unique_ptr<msf::MappedBlockStream> Stream);
-  ~TpiStream();
-  Error reload();
-
-  PdbRaw_TpiVer getTpiVersion() const;
-
-  uint32_t TypeIndexBegin() const;
-  uint32_t TypeIndexEnd() const;
-  uint32_t getNumTypeRecords() const;
-  uint16_t getTypeHashStreamIndex() const;
-  uint16_t getTypeHashStreamAuxIndex() const;
-
-  uint32_t getHashKeySize() const;
-  uint32_t getNumHashBuckets() const;
-  FixedStreamArray<support::ulittle32_t> getHashValues() const;
-  FixedStreamArray<codeview::TypeIndexOffset> getTypeIndexOffsets() const;
-  HashTable<support::ulittle32_t> &getHashAdjusters();
-
-  codeview::CVTypeRange types(bool *HadError) const;
+  LLVM_ABI TpiStream(PDBFile &File,
+                     std::unique_ptr<msf::MappedBlockStream> Stream);
+  LLVM_ABI ~TpiStream();
+  LLVM_ABI Error reload();
+
+  LLVM_ABI PdbRaw_TpiVer getTpiVersion() const;
+
+  LLVM_ABI uint32_t TypeIndexBegin() const;
+  LLVM_ABI uint32_t TypeIndexEnd() const;
+  LLVM_ABI uint32_t getNumTypeRecords() const;
+  LLVM_ABI uint16_t getTypeHashStreamIndex() const;
+  LLVM_ABI uint16_t getTypeHashStreamAuxIndex() const;
+
+  LLVM_ABI uint32_t getHashKeySize() const;
+  LLVM_ABI uint32_t getNumHashBuckets() const;
+  LLVM_ABI FixedStreamArray<support::ulittle32_t> getHashValues() const;
+  LLVM_ABI FixedStreamArray<codeview::TypeIndexOffset>
+  getTypeIndexOffsets() const;
+  LLVM_ABI HashTable<support::ulittle32_t> &getHashAdjusters();
+
+  LLVM_ABI codeview::CVTypeRange types(bool *HadError) const;
   const codeview::CVTypeArray &typeArray() const { return TypeRecords; }
 
   codeview::LazyRandomTypeCollection &typeCollection() { return *Types; }
 
-  Expected<codeview::TypeIndex>
+  LLVM_ABI Expected<codeview::TypeIndex>
   findFullDeclForForwardRef(codeview::TypeIndex ForwardRefTI) const;
 
-  std::vector<codeview::TypeIndex> findRecordsByName(StringRef Name) const;
+  LLVM_ABI std::vector<codeview::TypeIndex>
+  findRecordsByName(StringRef Name) const;
 
-  codeview::CVType getType(codeview::TypeIndex Index);
+  LLVM_ABI codeview::CVType getType(codeview::TypeIndex Index);
 
-  BinarySubstreamRef getTypeRecordsSubstream() const;
+  LLVM_ABI BinarySubstreamRef getTypeRecordsSubstream() const;
 
-  Error commit();
+  LLVM_ABI Error commit();
 
-  void buildHashMap();
+  LLVM_ABI void buildHashMap();
 
-  bool supportsTypeLookup() const;
+  LLVM_ABI bool supportsTypeLookup() const;
 
 private:
   PDBFile &Pdb;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h
index 855b5fe1f931..255abb0e3e8d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h
@@ -14,6 +14,7 @@
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include <vector>
@@ -38,24 +39,27 @@ struct TpiStreamHeader;
 
 class TpiStreamBuilder {
 public:
-  explicit TpiStreamBuilder(msf::MSFBuilder &Msf, uint32_t StreamIdx);
-  ~TpiStreamBuilder();
+  LLVM_ABI explicit TpiStreamBuilder(msf::MSFBuilder &Msf, uint32_t StreamIdx);
+  LLVM_ABI ~TpiStreamBuilder();
 
   TpiStreamBuilder(const TpiStreamBuilder &) = delete;
   TpiStreamBuilder &operator=(const TpiStreamBuilder &) = delete;
 
-  void setVersionHeader(PdbRaw_TpiVer Version);
-  void addTypeRecord(ArrayRef<uint8_t> Type, std::optional<uint32_t> Hash);
-  void addTypeRecords(ArrayRef<uint8_t> Types, ArrayRef<uint16_t> Sizes,
-                      ArrayRef<uint32_t> Hashes);
+  LLVM_ABI void setVersionHeader(PdbRaw_TpiVer Version);
+  LLVM_ABI void addTypeRecord(ArrayRef<uint8_t> Type,
+                              std::optional<uint32_t> Hash);
+  LLVM_ABI void addTypeRecords(ArrayRef<uint8_t> Types,
+                               ArrayRef<uint16_t> Sizes,
+                               ArrayRef<uint32_t> Hashes);
 
-  Error finalizeMsfLayout();
+  LLVM_ABI Error finalizeMsfLayout();
 
   uint32_t getRecordCount() const { return TypeRecordCount; }
 
-  Error commit(const msf::MSFLayout &Layout, WritableBinaryStreamRef Buffer);
+  LLVM_ABI Error commit(const msf::MSFLayout &Layout,
+                        WritableBinaryStreamRef Buffer);
 
-  uint32_t calculateSerializedLength();
+  LLVM_ABI uint32_t calculateSerializedLength();
 
 private:
   void updateTypeIndexOffsets(ArrayRef<uint16_t> Sizes);
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDB.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDB.h
index 6d734dc2f243..0116ff9996a3 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDB.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDB.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <memory>
 
@@ -19,11 +20,11 @@ namespace pdb {
 
 class IPDBSession;
 
-Error loadDataForPDB(PDB_ReaderType Type, StringRef Path,
-                     std::unique_ptr<IPDBSession> &Session);
+LLVM_ABI Error loadDataForPDB(PDB_ReaderType Type, StringRef Path,
+                              std::unique_ptr<IPDBSession> &Session);
 
-Error loadDataForEXE(PDB_ReaderType Type, StringRef Path,
-                     std::unique_ptr<IPDBSession> &Session);
+LLVM_ABI Error loadDataForEXE(PDB_ReaderType Type, StringRef Path,
+                              std::unique_ptr<IPDBSession> &Session);
 
 } // end namespace pdb
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h
index 802d18a069ee..d35b162d56a6 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBExtras.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
 #include <unordered_map>
@@ -22,26 +23,29 @@ namespace pdb {
 
 using TagStats = std::unordered_map<PDB_SymType, int>;
 
-raw_ostream &operator<<(raw_ostream &OS, const PDB_VariantType &Value);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_CallingConv &Conv);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_BuiltinType &Type);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_DataKind &Data);
-raw_ostream &operator<<(raw_ostream &OS,
-                        const llvm::codeview::CPURegister &CpuReg);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_LocType &Loc);
-raw_ostream &operator<<(raw_ostream &OS, const codeview::ThunkOrdinal &Thunk);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_Checksum &Checksum);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_Lang &Lang);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_SymType &Tag);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_MemberAccess &Access);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_UdtType &Type);
-raw_ostream &operator<<(raw_ostream &OS, const PDB_Machine &Machine);
-
-raw_ostream &operator<<(raw_ostream &OS, const Variant &Value);
-raw_ostream &operator<<(raw_ostream &OS, const VersionInfo &Version);
-raw_ostream &operator<<(raw_ostream &OS, const TagStats &Stats);
-
-raw_ostream& dumpPDBSourceCompression(raw_ostream& OS, uint32_t Compression);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_VariantType &Value);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_CallingConv &Conv);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_BuiltinType &Type);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_DataKind &Data);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const llvm::codeview::CPURegister &CpuReg);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_LocType &Loc);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const codeview::ThunkOrdinal &Thunk);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_Checksum &Checksum);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_Lang &Lang);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_SymType &Tag);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const PDB_MemberAccess &Access);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_UdtType &Type);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const PDB_Machine &Machine);
+
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const Variant &Value);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const VersionInfo &Version);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const TagStats &Stats);
+
+LLVM_ABI raw_ostream &dumpPDBSourceCompression(raw_ostream &OS,
+                                               uint32_t Compression);
 
 template <typename T>
 void dumpSymbolField(raw_ostream &OS, StringRef Name, T Value, int Indent) {
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymDumper.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymDumper.h
index f81b15f2353d..721229413932 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymDumper.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymDumper.h
@@ -10,13 +10,14 @@
 #define LLVM_DEBUGINFO_PDB_PDBSYMDUMPER_H
 
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 class raw_ostream;
 namespace pdb {
 
-class PDBSymDumper {
+class LLVM_ABI PDBSymDumper {
 public:
   PDBSymDumper(bool ShouldRequireImpl);
   virtual ~PDBSymDumper();
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h
index 4e34b75b6117..6eee1d00a469 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h
@@ -13,6 +13,7 @@
 #include "PDBExtras.h"
 #include "PDBTypes.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 
 #define FORWARD_SYMBOL_METHOD(MethodName)                                      \
   decltype(auto) MethodName() const { return RawSymbol->MethodName(); }
@@ -68,7 +69,7 @@ public:                                                                        \
 /// valid for that particular symbol type, as described in the Microsoft
 /// reference "Lexical and Class Hierarchy of Symbol Types":
 /// https://msdn.microsoft.com/en-us/library/370hs6k4.aspx
-class PDBSymbol {
+class LLVM_ABI PDBSymbol {
   static std::unique_ptr<PDBSymbol> createSymbol(const IPDBSession &PDBSession,
                                                  PDB_SymType Tag);
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h
index ca8b39d03f86..f48e0c4844ad 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h
@@ -10,6 +10,7 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -18,7 +19,7 @@ class raw_ostream;
 
 namespace pdb {
 
-class PDBSymbolCompiland : public PDBSymbol {
+class LLVM_ABI PDBSymbolCompiland : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Compiland)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h
index 61e67d1368a8..f33d1d33d52f 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolData.h
@@ -12,6 +12,7 @@
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -19,7 +20,7 @@ namespace pdb {
 
 class PDBSymDumper;
 
-class PDBSymbolData : public PDBSymbol {
+class LLVM_ABI PDBSymbolData : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Data)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h
index cde66d399243..686b4dc46024 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h
@@ -11,6 +11,7 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -18,7 +19,7 @@ class raw_ostream;
 
 namespace pdb {
 
-class PDBSymbolExe : public PDBSymbol {
+class LLVM_ABI PDBSymbolExe : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Exe)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h
index bfc7f7689718..d797d00cfa12 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_PDB_PDBSYMBOLFUNC_H
 
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
+#include "llvm/Support/Compiler.h"
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
@@ -23,7 +24,7 @@ class PDBSymbolData;
 class PDBSymbolTypeFunctionSig;
 template <typename ChildType> class IPDBEnumChildren;
 
-class PDBSymbolFunc : public PDBSymbol {
+class LLVM_ABI PDBSymbolFunc : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Function)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h
index 09c6f4728960..9a024448c6f2 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolFuncDebugEnd : public PDBSymbol {
+class LLVM_ABI PDBSymbolFuncDebugEnd : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FuncDebugEnd)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h
index 843a8348a2f0..50f9c3de3b5c 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolFuncDebugStart : public PDBSymbol {
+class LLVM_ABI PDBSymbolFuncDebugStart : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FuncDebugStart)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h
index 148802a47cbc..1c906209f22f 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolLabel : public PDBSymbol {
+class LLVM_ABI PDBSymbolLabel : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Label)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h
index a757cc02624b..0b6f14669930 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolPublicSymbol : public PDBSymbol {
+class LLVM_ABI PDBSymbolPublicSymbol : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::PublicSymbol)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h
index 2b81a63995e6..b30141449671 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolThunk : public PDBSymbol {
+class LLVM_ABI PDBSymbolThunk : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Thunk)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h
index 496141e5fa68..8deaf15fda5d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolTypeArray : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeArray : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::ArrayType)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h
index c74ac3fb9cce..69ce0159540a 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h
@@ -11,6 +11,7 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
 
@@ -20,7 +21,7 @@ namespace pdb {
 
 class PDBSymDumper;
 
-class PDBSymbolTypeBaseClass : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeBaseClass : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::BaseClass)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h
index b923983095f3..a054b0c02db8 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolTypeBuiltin : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeBuiltin : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::BuiltinType)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h
index ee1f736c17a0..431bf0dab90d 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h
@@ -11,6 +11,7 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
 
@@ -21,7 +22,7 @@ namespace pdb {
 class PDBSymDumper;
 class PDBSymbolTypeBuiltin;
 
-class PDBSymbolTypeEnum : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeEnum : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Enum)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h
index 71decff722a5..0c8f66ccd506 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolTypeFunctionArg : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeFunctionArg : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FunctionArg)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h
index ceb4bff5b7b4..19cdb7336e16 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h
@@ -11,13 +11,14 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 class raw_ostream;
 namespace pdb {
 
-class PDBSymbolTypeFunctionSig : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeFunctionSig : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::FunctionSig)
 public:
   std::unique_ptr<IPDBEnumSymbols> getArguments() const;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h
index 1b43ef9a21bd..8e02d10fc989 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolTypePointer : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypePointer : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::PointerType)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h
index 3f37730cf1df..6005699ba7b4 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolTypeTypedef : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeTypedef : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::Typedef)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h
index a3a49a4b619a..6696e34bf2e4 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEUDT_H
 
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
+#include "llvm/Support/Compiler.h"
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
@@ -20,7 +21,7 @@ namespace pdb {
 
 class PDBSymDumper;
 
-class PDBSymbolTypeUDT : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeUDT : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::UDT)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h
index 6223bee98670..0960248b1080 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolTypeVTable : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeVTable : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::VTable)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h
index bec0a9970a9f..90177c846c34 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolTypeVTableShape : public PDBSymbol {
+class LLVM_ABI PDBSymbolTypeVTableShape : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::VTableShape)
 public:
   void dump(PDBSymDumper &Dumper) const override;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h
index a53af49bc9e0..27d381f361c7 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h
@@ -10,12 +10,13 @@
 #define LLVM_DEBUGINFO_PDB_PDBSYMBOLUNKNOWN_H
 
 #include "PDBSymbol.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolUnknown : public PDBSymbol {
+class LLVM_ABI PDBSymbolUnknown : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CUSTOM_TYPE(S->getSymTag() == PDB_SymType::None ||
                                  S->getSymTag() >= PDB_SymType::Max)
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h
index dde25a023d00..1c221070670f 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h
@@ -11,12 +11,13 @@
 
 #include "PDBSymbol.h"
 #include "PDBTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace pdb {
 
-class PDBSymbolUsingNamespace : public PDBSymbol {
+class LLVM_ABI PDBSymbolUsingNamespace : public PDBSymbol {
   DECLARE_PDB_SYMBOL_CONCRETE_TYPE(PDB_SymType::UsingNamespace)
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h
index 8631c412f114..e07e2092aa63 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/PDB/UDTLayout.h
@@ -14,10 +14,13 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/PDB/PDBSymbol.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolData.h"
+#include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h"
+#include "llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <memory>
 #include <string>
@@ -30,7 +33,7 @@ class BaseClassLayout;
 class ClassLayout;
 class UDTLayoutBase;
 
-class LayoutItemBase {
+class LLVM_ABI LayoutItemBase {
 public:
   LayoutItemBase(const UDTLayoutBase *Parent, const PDBSymbol *Symbol,
                  const std::string &Name, uint32_t OffsetInParent,
@@ -70,9 +73,9 @@ class LayoutItemBase {
 
 class VBPtrLayoutItem : public LayoutItemBase {
 public:
-  VBPtrLayoutItem(const UDTLayoutBase &Parent,
-                  std::unique_ptr<PDBSymbolTypeBuiltin> Sym, uint32_t Offset,
-                  uint32_t Size);
+  LLVM_ABI VBPtrLayoutItem(const UDTLayoutBase &Parent,
+                           std::unique_ptr<PDBSymbolTypeBuiltin> Sym,
+                           uint32_t Offset, uint32_t Size);
 
   bool isVBPtr() const override { return true; }
 
@@ -82,12 +85,12 @@ class VBPtrLayoutItem : public LayoutItemBase {
 
 class DataMemberLayoutItem : public LayoutItemBase {
 public:
-  DataMemberLayoutItem(const UDTLayoutBase &Parent,
-                       std::unique_ptr<PDBSymbolData> DataMember);
+  LLVM_ABI DataMemberLayoutItem(const UDTLayoutBase &Parent,
+                                std::unique_ptr<PDBSymbolData> DataMember);
 
-  const PDBSymbolData &getDataMember();
-  bool hasUDTLayout() const;
-  const ClassLayout &getUDTLayout() const;
+  LLVM_ABI const PDBSymbolData &getDataMember();
+  LLVM_ABI bool hasUDTLayout() const;
+  LLVM_ABI const ClassLayout &getUDTLayout() const;
 
 private:
   std::unique_ptr<PDBSymbolData> DataMember;
@@ -96,8 +99,8 @@ class DataMemberLayoutItem : public LayoutItemBase {
 
 class VTableLayoutItem : public LayoutItemBase {
 public:
-  VTableLayoutItem(const UDTLayoutBase &Parent,
-                   std::unique_ptr<PDBSymbolTypeVTable> VTable);
+  LLVM_ABI VTableLayoutItem(const UDTLayoutBase &Parent,
+                            std::unique_ptr<PDBSymbolTypeVTable> VTable);
 
   uint32_t getElementSize() const { return ElementSize; }
 
@@ -106,7 +109,7 @@ class VTableLayoutItem : public LayoutItemBase {
   std::unique_ptr<PDBSymbolTypeVTable> VTable;
 };
 
-class UDTLayoutBase : public LayoutItemBase {
+class LLVM_ABI UDTLayoutBase : public LayoutItemBase {
   template <typename T> using UniquePtrVector = std::vector<std::unique_ptr<T>>;
 
 public:
@@ -114,6 +117,10 @@ class UDTLayoutBase : public LayoutItemBase {
                 const std::string &Name, uint32_t OffsetInParent, uint32_t Size,
                 bool IsElided);
 
+  // Explicitly non-copyable.
+  UDTLayoutBase(UDTLayoutBase const &) = delete;
+  UDTLayoutBase &operator=(UDTLayoutBase const &) = delete;
+
   uint32_t tailPadding() const override;
   ArrayRef<LayoutItemBase *> layout_items() const { return LayoutItems; }
   ArrayRef<BaseClassLayout *> bases() const { return AllBases; }
@@ -146,8 +153,9 @@ class UDTLayoutBase : public LayoutItemBase {
 
 class BaseClassLayout : public UDTLayoutBase {
 public:
-  BaseClassLayout(const UDTLayoutBase &Parent, uint32_t OffsetInParent,
-                  bool Elide, std::unique_ptr<PDBSymbolTypeBaseClass> Base);
+  LLVM_ABI BaseClassLayout(const UDTLayoutBase &Parent, uint32_t OffsetInParent,
+                           bool Elide,
+                           std::unique_ptr<PDBSymbolTypeBaseClass> Base);
 
   const PDBSymbolTypeBaseClass &getBase() const { return *Base; }
   bool isVirtualBase() const { return IsVirtualBase; }
@@ -158,13 +166,11 @@ class BaseClassLayout : public UDTLayoutBase {
   bool IsVirtualBase;
 };
 
-class ClassLayout : public UDTLayoutBase {
+class LLVM_ABI ClassLayout : public UDTLayoutBase {
 public:
   explicit ClassLayout(const PDBSymbolTypeUDT &UDT);
   explicit ClassLayout(std::unique_ptr<PDBSymbolTypeUDT> UDT);
 
-  ClassLayout(ClassLayout &&Other) = default;
-
   const PDBSymbolTypeUDT &getClass() const { return UDT; }
   uint32_t immediatePadding() const override;
 
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h
index 828875f1bb0e..3b9bd8915717 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h
@@ -15,6 +15,7 @@
 #define LLVM_DEBUGINFO_SYMBOLIZE_DIPRINTER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/JSON.h"
 #include <memory>
 #include <vector>
@@ -67,7 +68,7 @@ struct PrinterConfig {
 
 using ErrorHandler = std::function<void(const ErrorInfoBase &, StringRef)>;
 
-class PlainPrinterBase : public DIPrinter {
+class LLVM_ABI PlainPrinterBase : public DIPrinter {
 protected:
   raw_ostream &OS;
   ErrorHandler ErrHandler;
@@ -104,7 +105,7 @@ class PlainPrinterBase : public DIPrinter {
   void listEnd() override {}
 };
 
-class LLVMPrinter : public PlainPrinterBase {
+class LLVM_ABI LLVMPrinter : public PlainPrinterBase {
 private:
   void printSimpleLocation(StringRef Filename, const DILineInfo &Info) override;
   void printStartAddress(const DILineInfo &Info) override;
@@ -115,17 +116,16 @@ class LLVMPrinter : public PlainPrinterBase {
       : PlainPrinterBase(OS, EH, Config) {}
 };
 
-class GNUPrinter : public PlainPrinterBase {
+class LLVM_ABI GNUPrinter : public PlainPrinterBase {
 private:
   void printSimpleLocation(StringRef Filename, const DILineInfo &Info) override;
 
 public:
   GNUPrinter(raw_ostream &OS, ErrorHandler EH, PrinterConfig &Config)
       : PlainPrinterBase(OS, EH, Config) {}
-
 };
 
-class JSONPrinter : public DIPrinter {
+class LLVM_ABI JSONPrinter : public DIPrinter {
 private:
   raw_ostream &OS;
   PrinterConfig Config;
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Markup.h b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Markup.h
index 37e1d2748e31..699baf2a1c18 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Markup.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Markup.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Regex.h"
 
 namespace llvm {
@@ -50,7 +51,7 @@ struct MarkupNode {
 /// Parses a log containing symbolizer markup into a sequence of nodes.
 class MarkupParser {
 public:
-  MarkupParser(StringSet<> MultilineTags = {});
+  LLVM_ABI MarkupParser(StringSet<> MultilineTags = {});
 
   /// Parses an individual \p Line of input.
   ///
@@ -64,14 +65,14 @@ class MarkupParser {
   /// either the end or something that cannot be part of an element is
   /// encountered. This may only occur after multiple calls to parseLine(),
   /// corresponding to the lines of the multi-line element.
-  void parseLine(StringRef Line);
+  LLVM_ABI void parseLine(StringRef Line);
 
   /// Inform the parser of that the input stream has ended.
   ///
   /// This allows the parser to finish any deferred processing (e.g., an
   /// in-progress multi-line element) and may cause nextNode() to return
   /// additional nodes.
-  void flush();
+  LLVM_ABI void flush();
 
   /// Returns the next node in the input sequence.
   ///
@@ -79,7 +80,7 @@ class MarkupParser {
   /// previous call.
   ///
   /// \returns the next markup node or std::nullopt if none remain.
-  std::optional<MarkupNode> nextNode();
+  LLVM_ABI std::optional<MarkupNode> nextNode();
 
   bool isSGR(const MarkupNode &Node) const {
     return SGRSyntax.match(Node.Text);
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h
index 10613ec57586..60e2de50cf83 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/DebugInfo/Symbolize/Markup.h"
 #include "llvm/Object/BuildID.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <map>
@@ -31,18 +32,18 @@ class LLVMSymbolizer;
 /// text.
 class MarkupFilter {
 public:
-  MarkupFilter(raw_ostream &OS, LLVMSymbolizer &Symbolizer,
-               std::optional<bool> ColorsEnabled = std::nullopt);
+  LLVM_ABI MarkupFilter(raw_ostream &OS, LLVMSymbolizer &Symbolizer,
+                        std::optional<bool> ColorsEnabled = std::nullopt);
 
   /// Filters a line containing symbolizer markup and writes the human-readable
   /// results to the output stream.
   ///
   /// Invalid or unimplemented markup elements are removed. Some output may be
   /// deferred until future filter() or finish() call.
-  void filter(std::string &&InputLine);
+  LLVM_ABI void filter(std::string &&InputLine);
 
   /// Records that the input stream has ended and writes any deferred output.
-  void finish();
+  LLVM_ABI void finish();
 
 private:
   struct Module {
@@ -58,8 +59,8 @@ class MarkupFilter {
     std::string Mode; // Lowercase
     uint64_t ModuleRelativeAddr;
 
-    bool contains(uint64_t Addr) const;
-    uint64_t getModuleRelativeAddr(uint64_t Addr) const;
+    LLVM_ABI bool contains(uint64_t Addr) const;
+    LLVM_ABI uint64_t getModuleRelativeAddr(uint64_t Addr) const;
   };
 
   // An informational module line currently being constructed. As many mmap
diff --git a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
index 7c6beaa2189b..fb8f3d8af6b1 100644
--- a/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
+++ b/external/llvm-project/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
@@ -19,6 +19,7 @@
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/BuildID.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <algorithm>
 #include <cstdint>
@@ -71,57 +72,59 @@ class LLVMSymbolizer {
             : static_cast<size_t>(4ULL * 1024 * 1024 * 1024) /* 4 GiB */;
   };
 
-  LLVMSymbolizer();
-  LLVMSymbolizer(const Options &Opts);
+  LLVM_ABI LLVMSymbolizer();
+  LLVM_ABI LLVMSymbolizer(const Options &Opts);
 
-  ~LLVMSymbolizer();
+  LLVM_ABI ~LLVMSymbolizer();
 
   // Overloads accepting ObjectFile does not support COFF currently
-  Expected<DILineInfo> symbolizeCode(const ObjectFile &Obj,
-                                     object::SectionedAddress ModuleOffset);
-  Expected<DILineInfo> symbolizeCode(StringRef ModuleName,
-                                     object::SectionedAddress ModuleOffset);
-  Expected<DILineInfo> symbolizeCode(ArrayRef<uint8_t> BuildID,
-                                     object::SectionedAddress ModuleOffset);
-  Expected<DIInliningInfo>
+  LLVM_ABI Expected<DILineInfo>
+  symbolizeCode(const ObjectFile &Obj, object::SectionedAddress ModuleOffset);
+  LLVM_ABI Expected<DILineInfo>
+  symbolizeCode(StringRef ModuleName, object::SectionedAddress ModuleOffset);
+  LLVM_ABI Expected<DILineInfo>
+  symbolizeCode(ArrayRef<uint8_t> BuildID,
+                object::SectionedAddress ModuleOffset);
+  LLVM_ABI Expected<DIInliningInfo>
   symbolizeInlinedCode(const ObjectFile &Obj,
                        object::SectionedAddress ModuleOffset);
-  Expected<DIInliningInfo>
+  LLVM_ABI Expected<DIInliningInfo>
   symbolizeInlinedCode(StringRef ModuleName,
                        object::SectionedAddress ModuleOffset);
-  Expected<DIInliningInfo>
+  LLVM_ABI Expected<DIInliningInfo>
   symbolizeInlinedCode(ArrayRef<uint8_t> BuildID,
                        object::SectionedAddress ModuleOffset);
 
-  Expected<DIGlobal> symbolizeData(const ObjectFile &Obj,
-                                   object::SectionedAddress ModuleOffset);
-  Expected<DIGlobal> symbolizeData(StringRef ModuleName,
-                                   object::SectionedAddress ModuleOffset);
-  Expected<DIGlobal> symbolizeData(ArrayRef<uint8_t> BuildID,
-                                   object::SectionedAddress ModuleOffset);
-  Expected<std::vector<DILocal>>
+  LLVM_ABI Expected<DIGlobal>
+  symbolizeData(const ObjectFile &Obj, object::SectionedAddress ModuleOffset);
+  LLVM_ABI Expected<DIGlobal>
+  symbolizeData(StringRef ModuleName, object::SectionedAddress ModuleOffset);
+  LLVM_ABI Expected<DIGlobal>
+  symbolizeData(ArrayRef<uint8_t> BuildID,
+                object::SectionedAddress ModuleOffset);
+  LLVM_ABI Expected<std::vector<DILocal>>
   symbolizeFrame(const ObjectFile &Obj, object::SectionedAddress ModuleOffset);
-  Expected<std::vector<DILocal>>
+  LLVM_ABI Expected<std::vector<DILocal>>
   symbolizeFrame(StringRef ModuleName, object::SectionedAddress ModuleOffset);
-  Expected<std::vector<DILocal>>
+  LLVM_ABI Expected<std::vector<DILocal>>
   symbolizeFrame(ArrayRef<uint8_t> BuildID,
                  object::SectionedAddress ModuleOffset);
 
-  Expected<std::vector<DILineInfo>>
+  LLVM_ABI Expected<std::vector<DILineInfo>>
   findSymbol(const ObjectFile &Obj, StringRef Symbol, uint64_t Offset);
-  Expected<std::vector<DILineInfo>>
+  LLVM_ABI Expected<std::vector<DILineInfo>>
   findSymbol(StringRef ModuleName, StringRef Symbol, uint64_t Offset);
-  Expected<std::vector<DILineInfo>>
+  LLVM_ABI Expected<std::vector<DILineInfo>>
   findSymbol(ArrayRef<uint8_t> BuildID, StringRef Symbol, uint64_t Offset);
 
-  void flush();
+  LLVM_ABI void flush();
 
   // Evict entries from the binary cache until it is under the maximum size
   // given in the options. Calling this invalidates references in the DI...
   // objects returned by the methods above.
-  void pruneCache();
+  LLVM_ABI void pruneCache();
 
-  static std::string
+  LLVM_ABI static std::string
   DemangleName(StringRef Name, const SymbolizableModule *DbiModuleDescriptor);
 
   void setBuildIDFetcher(std::unique_ptr<BuildIDFetcher> Fetcher) {
@@ -132,7 +135,8 @@ class LLVMSymbolizer {
   /// Only one attempt is made to load a module, and errors during loading are
   /// only reported once. Subsequent calls to get module info for a module that
   /// failed to load will return nullptr.
-  Expected<SymbolizableModule *> getOrCreateModuleInfo(StringRef ModuleName);
+  LLVM_ABI Expected<SymbolizableModule *>
+  getOrCreateModuleInfo(StringRef ModuleName);
 
 private:
   // Bundles together object file with code/data and object file with
@@ -240,7 +244,7 @@ class CachedBinary : public ilist_node<CachedBinary> {
 
   // Add an action to be performed when the binary is evicted, before all
   // previously registered evictors.
-  void pushEvictor(std::function<void()> Evictor);
+  LLVM_ABI void pushEvictor(std::function<void()> Evictor);
 
   // Run all registered evictors in the reverse of the order in which they were
   // added.
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h
index ee6084ada1dd..27399d93fe58 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h
@@ -25,6 +25,7 @@
 #include "llvm/Object/Binary.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Target/TargetMachine.h"
@@ -88,7 +89,7 @@ class ExecutionEngineState {
   /// Erase an entry from the mapping table.
   ///
   /// \returns The address that \p ToUnmap was mapped to.
-  uint64_t RemoveMapping(StringRef Name);
+  LLVM_ABI uint64_t RemoveMapping(StringRef Name);
 };
 
 using FunctionCreator = std::function<void *(const std::string &)>;
@@ -96,7 +97,7 @@ using FunctionCreator = std::function<void *(const std::string &)>;
 /// Abstract interface for implementation execution of LLVM modules,
 /// designed to support both interpreter and just-in-time (JIT) compiler
 /// implementations.
-class ExecutionEngine {
+class LLVM_ABI ExecutionEngine {
   /// The state object holding the global address mapping, which must be
   /// accessed synchronously.
   //
@@ -550,13 +551,13 @@ class EngineBuilder {
 
 public:
   /// Default constructor for EngineBuilder.
-  EngineBuilder();
+  LLVM_ABI EngineBuilder();
 
   /// Constructor for EngineBuilder.
-  EngineBuilder(std::unique_ptr<Module> M);
+  LLVM_ABI EngineBuilder(std::unique_ptr<Module> M);
 
   // Out-of-line since we don't have the def'n of RTDyldMemoryManager here.
-  ~EngineBuilder();
+  LLVM_ABI ~EngineBuilder();
 
   /// setEngineKind - Controls whether the user wants the interpreter, the JIT,
   /// or whichever engine works.  This option defaults to EngineKind::Either.
@@ -571,12 +572,14 @@ class EngineBuilder {
   /// to create anything other than MCJIT will cause a runtime error. If create()
   /// is called and is successful, the created engine takes ownership of the
   /// memory manager. This option defaults to NULL.
-  EngineBuilder &setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager> mcjmm);
+  LLVM_ABI EngineBuilder &
+  setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager> mcjmm);
 
-  EngineBuilder&
+  LLVM_ABI EngineBuilder &
   setMemoryManager(std::unique_ptr<MCJITMemoryManager> MM);
 
-  EngineBuilder &setSymbolResolver(std::unique_ptr<LegacyJITSymbolResolver> SR);
+  LLVM_ABI EngineBuilder &
+  setSymbolResolver(std::unique_ptr<LegacyJITSymbolResolver> SR);
 
   /// setErrorStr - Set the error string to write to on error.  This option
   /// defaults to NULL.
@@ -645,20 +648,19 @@ class EngineBuilder {
     this->EmulatedTLS = EmulatedTLS;
   }
 
-  TargetMachine *selectTarget();
+  LLVM_ABI TargetMachine *selectTarget();
 
   /// selectTarget - Pick a target either via -march or by guessing the native
   /// arch.  Add any CPU features specified via -mcpu or -mattr.
-  TargetMachine *selectTarget(const Triple &TargetTriple,
-                              StringRef MArch,
-                              StringRef MCPU,
-                              const SmallVectorImpl<std::string>& MAttrs);
+  LLVM_ABI TargetMachine *
+  selectTarget(const Triple &TargetTriple, StringRef MArch, StringRef MCPU,
+               const SmallVectorImpl<std::string> &MAttrs);
 
   ExecutionEngine *create() {
     return create(selectTarget());
   }
 
-  ExecutionEngine *create(TargetMachine *TM);
+  LLVM_ABI ExecutionEngine *create(TargetMachine *TM);
 };
 
 // Create wrappers for C Binding types (see CBindingWrapping.h).
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Interpreter.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Interpreter.h
index 0749409766e3..b499cd6e0927 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Interpreter.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Interpreter.h
@@ -15,8 +15,9 @@
 #define LLVM_EXECUTIONENGINE_INTERPRETER_H
 
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/Support/Compiler.h"
 
-extern "C" void LLVMLinkInInterpreter();
+extern "C" LLVM_ABI void LLVMLinkInInterpreter();
 
 namespace {
   struct ForceInterpreterLinking {
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITEventListener.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITEventListener.h
index 7ccf8ff0d503..e13ac1d3e976 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITEventListener.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITEventListener.h
@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -37,7 +38,7 @@ class ObjectFile;
 /// profilers and debuggers that need to know where functions have been emitted.
 ///
 /// The default implementation of each method does nothing.
-class JITEventListener {
+class LLVM_ABI JITEventListener {
 public:
   using ObjectKey = uint64_t;
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h
index 6bfa5506017d..3c6b7bbd1a87 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -36,7 +37,7 @@ class EHFrameCFIBlockInspector {
   /// second to PC-begin, third (if present) to LSDA.
   ///
   /// It is illegal to call this function on a block with four or more edges.
-  static EHFrameCFIBlockInspector FromEdgeScan(Block &B);
+  LLVM_ABI static EHFrameCFIBlockInspector FromEdgeScan(Block &B);
 
   /// Returns true if this frame is an FDE, false for a CIE.
   bool isFDE() const { return CIEEdge != nullptr; }
@@ -85,7 +86,7 @@ class EHFrameCFIBlockInspector {
 
 /// Returns a pointer to the DWARF eh-frame section if the graph contains a
 /// non-empty one, otherwise returns null.
-Section *getEHFrameSection(LinkGraph &G);
+LLVM_ABI Section *getEHFrameSection(LinkGraph &G);
 
 } // end namespace jitlink
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index a604b50c95de..cb5d27b39efb 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -27,6 +27,7 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -49,7 +50,7 @@ class Section;
 
 /// Base class for errors originating in JIT linker, e.g. missing relocation
 /// support.
-class JITLinkError : public ErrorInfo<JITLinkError> {
+class LLVM_ABI JITLinkError : public ErrorInfo<JITLinkError> {
 public:
   static char ID;
 
@@ -105,7 +106,7 @@ class Edge {
 
 /// Returns the string name of the given generic edge kind, or "unknown"
 /// otherwise. Useful for debugging.
-const char *getGenericEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getGenericEdgeKindName(Edge::Kind K);
 
 /// Base class for Addressable entities (externals, absolutes, blocks).
 class Addressable {
@@ -389,7 +390,7 @@ inline orc::ExecutorAddr alignToBlock(orc::ExecutorAddr Addr, const Block &B) {
 // Returns true if the given blocks contains exactly one valid c-string.
 // Zero-fill blocks of size 1 count as valid empty strings. Content blocks
 // must end with a zero, and contain no zeros before the end.
-bool isCStringBlock(Block &B);
+LLVM_ABI bool isCStringBlock(Block &B);
 
 /// Describes symbol linkage. This can be used to resolve definition clashes.
 enum class Linkage : uint8_t {
@@ -401,7 +402,7 @@ enum class Linkage : uint8_t {
 using TargetFlagsType = uint8_t;
 
 /// For errors and debugging output.
-const char *getLinkageName(Linkage L);
+LLVM_ABI const char *getLinkageName(Linkage L);
 
 /// Defines the scope in which this symbol should be visible:
 ///   Default -- Visible in the public interface of the linkage unit.
@@ -412,9 +413,9 @@ const char *getLinkageName(Linkage L);
 enum class Scope : uint8_t { Default, Hidden, SideEffectsOnly, Local };
 
 /// For debugging output.
-const char *getScopeName(Scope S);
+LLVM_ABI const char *getScopeName(Scope S);
 
-raw_ostream &operator<<(raw_ostream &OS, const Block &B);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const Block &B);
 
 /// Symbol representation.
 ///
@@ -708,10 +709,10 @@ class Symbol {
   size_t Size = 0;
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const Symbol &A);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const Symbol &A);
 
-void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
-               StringRef EdgeKindName);
+LLVM_ABI void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
+                        StringRef EdgeKindName);
 
 /// Represents an object file section.
 class Section {
@@ -731,7 +732,7 @@ class Section {
   using block_iterator = BlockSet::iterator;
   using const_block_iterator = BlockSet::const_iterator;
 
-  ~Section();
+  LLVM_ABI ~Section();
 
   // Sections are not movable or copyable.
   Section(const Section &) = delete;
@@ -1039,7 +1040,7 @@ class LinkGraph {
   LinkGraph &operator=(const LinkGraph &) = delete;
   LinkGraph(LinkGraph &&) = delete;
   LinkGraph &operator=(LinkGraph &&) = delete;
-  ~LinkGraph();
+  LLVM_ABI ~LinkGraph();
 
   /// Returns the name of this graph (usually the name of the original
   /// underlying MemoryBuffer).
@@ -1658,11 +1659,11 @@ class LinkGraph {
   orc::shared::AllocActions &allocActions() { return AAs; }
 
   /// Dump the graph.
-  void dump(raw_ostream &OS);
+  LLVM_ABI void dump(raw_ostream &OS);
 
 private:
-  std::vector<Block *> splitBlockImpl(std::vector<Block *> Blocks,
-                                      SplitBlockCache *Cache);
+  LLVM_ABI std::vector<Block *> splitBlockImpl(std::vector<Block *> Blocks,
+                                               SplitBlockCache *Cache);
 
   // Put the BumpPtrAllocator first so that we don't free any of the underlying
   // memory until the Symbol/Addressable destructors have been run.
@@ -1894,7 +1895,7 @@ struct PassConfiguration {
 ///        the two types once we have an OrcSupport library.
 enum class SymbolLookupFlags { RequiredSymbol, WeaklyReferencedSymbol };
 
-raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF);
 
 /// A map of symbol names to resolved addresses.
 using AsyncLookupResult =
@@ -1902,7 +1903,7 @@ using AsyncLookupResult =
 
 /// A function object to call with a resolved symbol map (See AsyncLookupResult)
 /// or an error if resolution failed.
-class JITLinkAsyncLookupContinuation {
+class LLVM_ABI JITLinkAsyncLookupContinuation {
 public:
   virtual ~JITLinkAsyncLookupContinuation() = default;
   virtual void run(Expected<AsyncLookupResult> LR) = 0;
@@ -1929,7 +1930,7 @@ createLookupContinuation(Continuation Cont) {
 }
 
 /// Holds context for a single jitLink invocation.
-class JITLinkContext {
+class LLVM_ABI JITLinkContext {
 public:
   using LookupMap = DenseMap<orc::SymbolStringPtr, SymbolLookupFlags>;
 
@@ -1995,14 +1996,14 @@ class JITLinkContext {
 
 /// Marks all symbols in a graph live. This can be used as a default,
 /// conservative mark-live implementation.
-Error markAllSymbolsLive(LinkGraph &G);
+LLVM_ABI Error markAllSymbolsLive(LinkGraph &G);
 
 /// Create an out of range error for the given edge in the given block.
-Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
-                                const Edge &E);
+LLVM_ABI Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
+                                         const Edge &E);
 
-Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
-                         const Edge &E);
+LLVM_ABI Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value,
+                                  int N, const Edge &E);
 
 /// Creates a new pointer block in the given section and returns an
 /// Anonymous symbol pointing to it.
@@ -2016,7 +2017,7 @@ using AnonymousPointerCreator =
                              Symbol *InitialTarget, uint64_t InitialAddend)>;
 
 /// Get target-specific AnonymousPointerCreator
-AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT);
+LLVM_ABI AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT);
 
 /// Create a jump stub that jumps via the pointer at the given symbol and
 /// an anonymous symbol pointing to it. Return the anonymous symbol.
@@ -2026,7 +2027,7 @@ using PointerJumpStubCreator = unique_function<Symbol &(
     LinkGraph &G, Section &StubSection, Symbol &PointerSymbol)>;
 
 /// Get target-specific PointerJumpStubCreator
-PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT);
+LLVM_ABI PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT);
 
 /// Base case for edge-visitors where the visitor-list is empty.
 inline void visitEdge(LinkGraph &G, Block *B, Edge &E) {}
@@ -2063,17 +2064,18 @@ void visitExistingEdges(LinkGraph &G, VisitorTs &&...Vs) {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
+LLVM_ABI Expected<std::unique_ptr<LinkGraph>>
 createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
                           std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// Create a \c LinkGraph defining the given absolute symbols.
-std::unique_ptr<LinkGraph>
+LLVM_ABI std::unique_ptr<LinkGraph>
 absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,
                          orc::SymbolMap Symbols);
 
 /// Link the given graph.
-void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx);
+LLVM_ABI void link(std::unique_ptr<LinkGraph> G,
+                   std::unique_ptr<JITLinkContext> Ctx);
 
 } // end namespace jitlink
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
index 3242055020f7..c7b1f6e0b2c3 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
 #include "llvm/Support/Memory.h"
@@ -43,7 +44,7 @@ class Section;
 ///
 /// Instances of this class may be accessed concurrently from multiple threads
 /// and their implemetations should include any necessary synchronization.
-class JITLinkMemoryManager {
+class LLVM_ABI JITLinkMemoryManager {
 public:
 
   /// Represents a finalized allocation.
@@ -118,7 +119,7 @@ class JITLinkMemoryManager {
   ///
   /// If abandon is called then working memory and executor memory should both
   /// be freed.
-  class InFlightAlloc {
+  class LLVM_ABI InFlightAlloc {
   public:
     using OnFinalizedFunction = unique_function<void(Expected<FinalizedAlloc>)>;
     using OnAbandonedFunction = unique_function<void(Error)>;
@@ -250,7 +251,7 @@ class BasicLayout {
   using SegmentMap = orc::AllocGroupSmallMap<Segment>;
 
 public:
-  BasicLayout(LinkGraph &G);
+  LLVM_ABI BasicLayout(LinkGraph &G);
 
   /// Return a reference to the graph this allocation was created from.
   LinkGraph &getGraph() { return G; }
@@ -264,7 +265,7 @@ class BasicLayout {
   ///
   /// This function will return an error if any segment has an alignment that
   /// is higher than a page.
-  Expected<ContiguousPageBasedLayoutSizes>
+  LLVM_ABI Expected<ContiguousPageBasedLayoutSizes>
   getContiguousPageBasedLayoutSizes(uint64_t PageSize);
 
   /// Returns an iterator over the segments of the layout.
@@ -273,12 +274,12 @@ class BasicLayout {
   }
 
   /// Apply the layout to the graph.
-  Error apply();
+  LLVM_ABI Error apply();
 
   /// Returns a reference to the AllocActions in the graph.
   /// This convenience function saves callers from having to #include
   /// LinkGraph.h if all they need are allocation actions.
-  orc::shared::AllocActions &graphAllocActions();
+  LLVM_ABI orc::shared::AllocActions &graphAllocActions();
 
 private:
   LinkGraph &G;
@@ -320,22 +321,22 @@ class SimpleSegmentAlloc {
   using OnFinalizedFunction =
       JITLinkMemoryManager::InFlightAlloc::OnFinalizedFunction;
 
-  static void Create(JITLinkMemoryManager &MemMgr,
-                     std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
-                     const JITLinkDylib *JD, SegmentMap Segments,
-                     OnCreatedFunction OnCreated);
+  LLVM_ABI static void Create(JITLinkMemoryManager &MemMgr,
+                              std::shared_ptr<orc::SymbolStringPool> SSP,
+                              Triple TT, const JITLinkDylib *JD,
+                              SegmentMap Segments, OnCreatedFunction OnCreated);
 
-  static Expected<SimpleSegmentAlloc>
+  LLVM_ABI static Expected<SimpleSegmentAlloc>
   Create(JITLinkMemoryManager &MemMgr,
          std::shared_ptr<orc::SymbolStringPool> SSP, Triple TT,
          const JITLinkDylib *JD, SegmentMap Segments);
 
-  SimpleSegmentAlloc(SimpleSegmentAlloc &&);
-  SimpleSegmentAlloc &operator=(SimpleSegmentAlloc &&);
-  ~SimpleSegmentAlloc();
+  LLVM_ABI SimpleSegmentAlloc(SimpleSegmentAlloc &&);
+  LLVM_ABI SimpleSegmentAlloc &operator=(SimpleSegmentAlloc &&);
+  LLVM_ABI ~SimpleSegmentAlloc();
 
   /// Returns the SegmentInfo for the given group.
-  SegmentInfo getSegInfo(orc::AllocGroup AG);
+  LLVM_ABI SegmentInfo getSegInfo(orc::AllocGroup AG);
 
   /// Finalize all groups (async version).
   void finalize(OnFinalizedFunction OnFinalized) {
@@ -359,7 +360,7 @@ class SimpleSegmentAlloc {
 };
 
 /// A JITLinkMemoryManager that allocates in-process memory.
-class InProcessMemoryManager : public JITLinkMemoryManager {
+class LLVM_ABI InProcessMemoryManager : public JITLinkMemoryManager {
 public:
   class IPInFlightAlloc;
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
index b3bf96b8549f..6a499b0f6255 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace jitlink {
@@ -24,7 +25,7 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>>
+LLVM_ABI Expected<std::unique_ptr<LinkGraph>>
 createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer,
                                std::shared_ptr<orc::SymbolStringPool> SSP);
 
@@ -32,8 +33,8 @@ createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer,
 ///
 /// Uses conservative defaults for GOT and stub handling based on the target
 /// platform.
-void link_MachO(std::unique_ptr<LinkGraph> G,
-                std::unique_ptr<JITLinkContext> Ctx);
+LLVM_ABI void link_MachO(std::unique_ptr<LinkGraph> G,
+                         std::unique_ptr<JITLinkContext> Ctx);
 
 /// Get a pointer to the standard MachO data section (creates an empty
 /// section with RW- permissions and standard lifetime if one does not
@@ -56,7 +57,7 @@ inline Section &getMachODefaultTextSection(LinkGraph &G) {
 }
 
 /// Gets or creates a MachO header for the current LinkGraph.
-Expected<Symbol &> getOrCreateLocalMachOHeader(LinkGraph &G);
+LLVM_ABI Expected<Symbol &> getOrCreateLocalMachOHeader(LinkGraph &G);
 
 } // end namespace jitlink
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h
index 31770c2a601e..9c8d6c651db0 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace jitlink {
@@ -23,7 +24,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_arm64(
+LLVM_ABI Expected<std::unique_ptr<LinkGraph>>
+createLinkGraphFromMachOObject_arm64(
     MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given object buffer, which must be a MachO arm64 object file.
@@ -35,16 +37,16 @@ Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_arm64(
 /// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will
 /// be inserted. If PostPrunePasses is not empty then the caller is responsible
 /// for including a pass to insert GOT and stub edges.
-void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
-                      std::unique_ptr<JITLinkContext> Ctx);
+LLVM_ABI void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
+                               std::unique_ptr<JITLinkContext> Ctx);
 
 /// Returns a pass suitable for splitting __eh_frame sections in MachO/x86-64
 /// objects.
-LinkGraphPassFunction createEHFrameSplitterPass_MachO_arm64();
+LLVM_ABI LinkGraphPassFunction createEHFrameSplitterPass_MachO_arm64();
 
 /// Returns a pass suitable for fixing missing edges in an __eh_frame section
 /// in a MachO/x86-64 object.
-LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_arm64();
+LLVM_ABI LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_arm64();
 
 } // end namespace jitlink
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h
index 72e9883ba18b..0e719cc9c8e4 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_JITLINK_MACHO_X86_64_H
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace jitlink {
@@ -23,7 +24,8 @@ namespace jitlink {
 /// Note: The graph does not take ownership of the underlying buffer, nor copy
 /// its contents. The caller is responsible for ensuring that the object buffer
 /// outlives the graph.
-Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_x86_64(
+LLVM_ABI Expected<std::unique_ptr<LinkGraph>>
+createLinkGraphFromMachOObject_x86_64(
     MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP);
 
 /// jit-link the given LinkGraph.
@@ -35,16 +37,16 @@ Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromMachOObject_x86_64(
 /// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will
 /// be inserted. If PostPrunePasses is not empty then the caller is responsible
 /// for including a pass to insert GOT and stub edges.
-void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
-                       std::unique_ptr<JITLinkContext> Ctx);
+LLVM_ABI void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
+                                std::unique_ptr<JITLinkContext> Ctx);
 
 /// Returns a pass suitable for splitting __eh_frame sections in MachO/x86-64
 /// objects.
-LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64();
+LLVM_ABI LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64();
 
 /// Returns a pass suitable for fixing missing edges in an __eh_frame section
 /// in a MachO/x86-64 object.
-LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64();
+LLVM_ABI LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64();
 
 } // end namespace jitlink
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h
index 3b8d4c8d1e24..98170f60f6e4 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h
@@ -17,6 +17,7 @@
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/Support/ARMBuildAttributes.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -24,7 +25,7 @@ namespace jitlink {
 namespace aarch32 {
 
 /// Check whether the given target flags are set for this Symbol.
-bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags);
+LLVM_ABI bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags);
 
 /// JITLink-internal AArch32 fixup kinds
 enum EdgeKind_aarch32 : Edge::Kind {
@@ -117,10 +118,10 @@ enum TargetFlags_aarch32 : TargetFlagsType {
 };
 
 /// Human-readable name for a given CPU architecture kind
-const char *getCPUArchName(ARMBuildAttrs::CPUArch K);
+LLVM_ABI const char *getCPUArchName(ARMBuildAttrs::CPUArch K);
 
 /// Get a human-readable name for the given AArch32 edge kind.
-const char *getEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getEdgeKindName(Edge::Kind K);
 
 /// AArch32 uses stubs for a number of purposes, like branch range extension
 /// or interworking between Arm and Thumb instruction subsets.
@@ -173,7 +174,7 @@ struct HalfWords {
 
 /// FixupInfo base class is required for dynamic lookups.
 struct FixupInfoBase {
-  static const FixupInfoBase *getDynFixupInfo(Edge::Kind K);
+  LLVM_ABI static const FixupInfoBase *getDynFixupInfo(Edge::Kind K);
   virtual ~FixupInfoBase() {}
 };
 
@@ -270,16 +271,18 @@ template <> struct FixupInfo<Thumb_MovwPrelNC> : public FixupInfoThumbMov {
 };
 
 /// Helper function to read the initial addend for Data-class relocations.
-Expected<int64_t> readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset,
-                                 Edge::Kind Kind);
+LLVM_ABI Expected<int64_t>
+readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset, Edge::Kind Kind);
 
 /// Helper function to read the initial addend for Arm-class relocations.
-Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, Edge::OffsetT Offset,
-                                Edge::Kind Kind);
+LLVM_ABI Expected<int64_t> readAddendArm(LinkGraph &G, Block &B,
+                                         Edge::OffsetT Offset, Edge::Kind Kind);
 
 /// Helper function to read the initial addend for Thumb-class relocations.
-Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, Edge::OffsetT Offset,
-                                  Edge::Kind Kind, const ArmConfig &ArmCfg);
+LLVM_ABI Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B,
+                                           Edge::OffsetT Offset,
+                                           Edge::Kind Kind,
+                                           const ArmConfig &ArmCfg);
 
 /// Read the initial addend for a REL-type relocation. It's the value encoded
 /// in the immediate field of the fixup location by the compiler.
@@ -300,14 +303,14 @@ inline Expected<int64_t> readAddend(LinkGraph &G, Block &B,
 }
 
 /// Helper function to apply the fixup for Data-class relocations.
-Error applyFixupData(LinkGraph &G, Block &B, const Edge &E);
+LLVM_ABI Error applyFixupData(LinkGraph &G, Block &B, const Edge &E);
 
 /// Helper function to apply the fixup for Arm-class relocations.
-Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E);
+LLVM_ABI Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E);
 
 /// Helper function to apply the fixup for Thumb-class relocations.
-Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,
-                      const ArmConfig &ArmCfg);
+LLVM_ABI Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,
+                               const ArmConfig &ArmCfg);
 
 /// Apply fixup expression for edge to block content.
 inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
@@ -332,8 +335,8 @@ class GOTBuilder : public TableManager<GOTBuilder> {
 public:
   static StringRef getSectionName() { return "$__GOT"; }
 
-  bool visitEdge(LinkGraph &G, Block *B, Edge &E);
-  Symbol &createEntry(LinkGraph &G, Symbol &Target);
+  LLVM_ABI bool visitEdge(LinkGraph &G, Block *B, Edge &E);
+  LLVM_ABI Symbol &createEntry(LinkGraph &G, Symbol &Target);
 
 private:
   Section *GOTSection = nullptr;
@@ -353,7 +356,7 @@ class StubsManager_prev7 {
   }
 
   /// Implements link-graph traversal via visitExistingEdges()
-  bool visitEdge(LinkGraph &G, Block *B, Edge &E);
+  LLVM_ABI bool visitEdge(LinkGraph &G, Block *B, Edge &E);
 
 private:
   // Each stub uses a single block that can have 2 entryponts, one for Arm and
@@ -387,7 +390,7 @@ class StubsManager_v7 {
   }
 
   /// Implements link-graph traversal via visitExistingEdges().
-  bool visitEdge(LinkGraph &G, Block *B, Edge &E);
+  LLVM_ABI bool visitEdge(LinkGraph &G, Block *B, Edge &E);
 
 private:
   // Two slots per external: Arm and Thumb
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
index e5e8ef1bab18..12bcdd1a79d8 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
@@ -16,6 +16,7 @@
 #include "TableManager.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace jitlink {
@@ -418,7 +419,7 @@ enum EdgeKind_aarch64 : Edge::Kind {
 
 /// Returns a string name for the given aarch64 edge. For debugging purposes
 /// only
-const char *getEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getEdgeKindName(Edge::Kind K);
 
 // Returns whether the Instr is LD/ST (imm12)
 inline bool isLoadStoreImm12(uint32_t Instr) {
@@ -697,7 +698,7 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
 constexpr uint64_t PointerSize = 8;
 
 /// AArch64 null pointer content.
-extern const char NullPointerContent[PointerSize];
+LLVM_ABI extern const char NullPointerContent[PointerSize];
 
 /// AArch64 pointer jump stub content.
 ///
@@ -706,7 +707,7 @@ extern const char NullPointerContent[PointerSize];
 ///   ADRP x16, ptr@page21
 ///   LDR  x16, [x16, ptr@pageoff12]
 ///   BR   x16
-extern const char PointerJumpStubContent[12];
+LLVM_ABI extern const char PointerJumpStubContent[12];
 
 /// Creates a new pointer block in the given section and returns an
 /// Anonymous symbol pointing to it.
@@ -761,7 +762,7 @@ inline Symbol &createAnonymousPointerJumpStub(LinkGraph &G,
 /// address (and stack pointer) on the stack and calls the given reentry symbol:
 ///   STP  x29, x30, [sp, #-16]!
 ///   BL   <reentry-symbol>
-extern const char ReentryTrampolineContent[8];
+LLVM_ABI extern const char ReentryTrampolineContent[8];
 
 /// Create a block of N reentry trampolines.
 inline Block &createReentryTrampolineBlock(LinkGraph &G,
@@ -853,7 +854,7 @@ class GOTTableManager : public TableManager<GOTTableManager> {
     return *GOTSection;
   }
 
-  void registerExistingEntries();
+  LLVM_ABI void registerExistingEntries();
 
   Section *GOTSection = nullptr;
 };
@@ -894,14 +895,14 @@ class PLTTableManager : public TableManager<PLTTableManager> {
     return *StubsSection;
   }
 
-  void registerExistingEntries();
+  LLVM_ABI void registerExistingEntries();
 
   GOTTableManager &GOT;
   Section *StubsSection = nullptr;
 };
 
 /// Returns the name of the pointer signing function section.
-const char *getPointerSigningFunctionSectionName();
+LLVM_ABI const char *getPointerSigningFunctionSectionName();
 
 /// Creates a pointer signing function section, block, and symbol to reserve
 /// space for a signing function for this LinkGraph. Clients should insert this
@@ -911,7 +912,7 @@ const char *getPointerSigningFunctionSectionName();
 /// No new Pointer64Auth edges can be inserted into the graph between when this
 /// pass is run and when the pass below runs (since there will not be sufficient
 /// space reserved in the signing function to write the signing code for them).
-Error createEmptyPointerSigningFunction(LinkGraph &G);
+LLVM_ABI Error createEmptyPointerSigningFunction(LinkGraph &G);
 
 /// Given a LinkGraph containing Pointer64Authenticated edges, transform those
 /// edges to Pointer64 and add signing code to the pointer signing function
@@ -921,7 +922,7 @@ Error createEmptyPointerSigningFunction(LinkGraph &G);
 /// This function will add a $__ptrauth_sign section with finalization-lifetime
 /// containing an anonymous function that will sign all pointers in the graph.
 /// An allocation action will be added to run this function during finalization.
-Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G);
+LLVM_ABI Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G);
 
 } // namespace aarch64
 } // namespace jitlink
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h
index 3c664296e9f3..04c5a67ac4fe 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/LEB128.h"
 
 namespace llvm {
@@ -322,7 +323,7 @@ enum EdgeKind_loongarch : Edge::Kind {
 
 /// Returns a string name for the given loongarch edge. For debugging purposes
 /// only.
-const char *getEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getEdgeKindName(Edge::Kind K);
 
 // Returns extract bits Val[Hi:Lo].
 inline uint32_t extractBits(uint64_t Val, unsigned Hi, unsigned Lo) {
@@ -562,7 +563,7 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
 }
 
 /// loongarch null pointer content.
-extern const char NullPointerContent[8];
+LLVM_ABI extern const char NullPointerContent[8];
 inline ArrayRef<char> getGOTEntryBlockContent(LinkGraph &G) {
   return {reinterpret_cast<const char *>(NullPointerContent),
           G.getPointerSize()};
@@ -576,8 +577,8 @@ inline ArrayRef<char> getGOTEntryBlockContent(LinkGraph &G) {
 ///   ld.[w/d]  $t8, %pageoff12(ptr)
 ///   jr        $t8
 constexpr size_t StubEntrySize = 12;
-extern const uint8_t LA64StubContent[StubEntrySize];
-extern const uint8_t LA32StubContent[StubEntrySize];
+LLVM_ABI extern const uint8_t LA64StubContent[StubEntrySize];
+LLVM_ABI extern const uint8_t LA32StubContent[StubEntrySize];
 inline ArrayRef<char> getStubBlockContent(LinkGraph &G) {
   auto StubContent =
       G.getPointerSize() == 8 ? LA64StubContent : LA32StubContent;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h
index b72391445598..9e375b5a592f 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 
 namespace llvm::jitlink::ppc64 {
@@ -73,11 +74,11 @@ enum PLTCallStubKind {
   LongBranchNoTOC,
 };
 
-extern const char NullPointerContent[8];
-extern const char PointerJumpStubContent_big[20];
-extern const char PointerJumpStubContent_little[20];
-extern const char PointerJumpStubNoTOCContent_big[32];
-extern const char PointerJumpStubNoTOCContent_little[32];
+LLVM_ABI extern const char NullPointerContent[8];
+LLVM_ABI extern const char PointerJumpStubContent_big[20];
+LLVM_ABI extern const char PointerJumpStubContent_little[20];
+LLVM_ABI extern const char PointerJumpStubNoTOCContent_big[32];
+LLVM_ABI extern const char PointerJumpStubNoTOCContent_little[32];
 
 struct PLTCallStubReloc {
   Edge::Kind K;
@@ -259,7 +260,7 @@ class PLTTableManager : public TableManager<PLTTableManager<Endianness>> {
 
 /// Returns a string name for the given ppc64 edge. For debugging purposes
 /// only.
-const char *getEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getEdgeKindName(Edge::Kind K);
 
 inline static uint16_t ha(uint64_t x) { return (x + 0x8000) >> 16; }
 inline static uint64_t lo(uint64_t x) { return x & 0xffff; }
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h
index a31f7d73b099..fcc99787c8e1 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_JITLINK_RISCV_H
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace jitlink {
@@ -224,7 +225,7 @@ enum EdgeKind_riscv : Edge::Kind {
 
 /// Returns a string name for the given riscv edge. For debugging purposes
 /// only
-const char *getEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getEdgeKindName(Edge::Kind K);
 } // namespace riscv
 } // namespace jitlink
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86.h
index eb372db88688..69d065704040 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::jitlink::x86 {
 
@@ -177,7 +178,7 @@ enum EdgeKind_x86 : Edge::Kind {
 
 /// Returns a string name for the given x86 edge. For debugging purposes
 /// only
-const char *getEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getEdgeKindName(Edge::Kind K);
 
 /// Apply fixup expression for edge to block content.
 inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
@@ -254,14 +255,14 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
 constexpr uint32_t PointerSize = 4;
 
 /// x86 null pointer content.
-extern const char NullPointerContent[PointerSize];
+LLVM_ABI extern const char NullPointerContent[PointerSize];
 
 /// x86 pointer jump stub content.
 ///
 /// Contains the instruction sequence for an indirect jump via an in-memory
 /// pointer:
 ///   jmpq *ptr
-extern const char PointerJumpStubContent[6];
+LLVM_ABI extern const char PointerJumpStubContent[6];
 
 /// Creates a new pointer block in the given section and returns an anonymous
 /// symbol pointing to it.
@@ -405,7 +406,7 @@ class PLTTableManager : public TableManager<PLTTableManager> {
 /// 2. BranchPCRel32ToPtrJumpStubRelaxable. For this edge kind, if the target is
 /// in range, replace a indirect jump by plt stub with a direct jump to the
 /// target
-Error optimizeGOTAndStubAccesses(LinkGraph &G);
+LLVM_ABI Error optimizeGOTAndStubAccesses(LinkGraph &G);
 
 } // namespace llvm::jitlink::x86
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
index e10242bb2d42..5ce73fa3cfb8 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace jitlink {
@@ -432,7 +433,7 @@ enum EdgeKind_x86_64 : Edge::Kind {
 
 /// Returns a string name for the given x86-64 edge. For debugging purposes
 /// only.
-const char *getEdgeKindName(Edge::Kind K);
+LLVM_ABI const char *getEdgeKindName(Edge::Kind K);
 
 /// Apply fixup expression for edge to block content.
 inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
@@ -586,14 +587,14 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
 constexpr uint64_t PointerSize = 8;
 
 /// x86-64 null pointer content.
-extern const char NullPointerContent[PointerSize];
+LLVM_ABI extern const char NullPointerContent[PointerSize];
 
 /// x86-64 pointer jump stub content.
 ///
 /// Contains the instruction sequence for an indirect jump via an in-memory
 /// pointer:
 ///   jmpq *ptr(%rip)
-extern const char PointerJumpStubContent[6];
+LLVM_ABI extern const char PointerJumpStubContent[6];
 
 /// Creates a new pointer block in the given section and returns an anonymous
 /// symbol pointing to it.
@@ -646,7 +647,7 @@ inline Symbol &createAnonymousPointerJumpStub(LinkGraph &G,
 /// Contains the instruction sequence for a trampoline that stores its return
 /// address on the stack and calls <reentry-symbol>:
 ///   call  <reentry-symbol>
-extern const char ReentryTrampolineContent[5];
+LLVM_ABI extern const char ReentryTrampolineContent[5];
 
 /// Create a block of N reentry trampolines.
 inline Block &createReentryTrampolineBlock(LinkGraph &G,
@@ -726,7 +727,7 @@ class GOTTableManager : public TableManager<GOTTableManager> {
     return *GOTSection;
   }
 
-  void registerExistingEntries();
+  LLVM_ABI void registerExistingEntries();
 
   Section *GOTSection = nullptr;
 };
@@ -770,7 +771,7 @@ class PLTTableManager : public TableManager<PLTTableManager> {
     return *StubsSection;
   }
 
-  void registerExistingEntries();
+  LLVM_ABI void registerExistingEntries();
 
   GOTTableManager &GOT;
   Section *StubsSection = nullptr;
@@ -782,7 +783,7 @@ class PLTTableManager : public TableManager<PLTTableManager> {
 /// 2. BranchPCRel32ToPtrJumpStubRelaxable. For this edge kind, if the target is
 /// in range, replace a indirect jump by plt stub with a direct jump to the
 /// target
-Error optimizeGOTAndStubAccesses(LinkGraph &G);
+LLVM_ABI Error optimizeGOTAndStubAccesses(LinkGraph &G);
 
 } // namespace x86_64
 } // end namespace jitlink
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITSymbol.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITSymbol.h
index d0a8dd7ce5e8..0c4801e5196e 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITSymbol.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/JITSymbol.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_EXECUTIONENGINE_JITSYMBOL_H
 #define LLVM_EXECUTIONENGINE_JITSYMBOL_H
 
+#include "llvm/Support/Compiler.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
@@ -176,15 +177,15 @@ class JITSymbolFlags {
 
   /// Construct a JITSymbolFlags value based on the flags of the given global
   /// value.
-  static JITSymbolFlags fromGlobalValue(const GlobalValue &GV);
+  LLVM_ABI static JITSymbolFlags fromGlobalValue(const GlobalValue &GV);
 
   /// Construct a JITSymbolFlags value based on the flags of the given global
   /// value summary.
-  static JITSymbolFlags fromSummary(GlobalValueSummary *S);
+  LLVM_ABI static JITSymbolFlags fromSummary(GlobalValueSummary *S);
 
   /// Construct a JITSymbolFlags value based on the flags of the given libobject
   /// symbol.
-  static Expected<JITSymbolFlags>
+  LLVM_ABI static Expected<JITSymbolFlags>
   fromObjectSymbol(const object::SymbolRef &Symbol);
 
 private:
@@ -219,7 +220,8 @@ class ARMJITSymbolFlags {
 
   operator JITSymbolFlags::TargetFlagsType&() { return Flags; }
 
-  static ARMJITSymbolFlags fromObjectSymbol(const object::SymbolRef &Symbol);
+  LLVM_ABI static ARMJITSymbolFlags
+  fromObjectSymbol(const object::SymbolRef &Symbol);
 
 private:
   JITSymbolFlags::TargetFlagsType Flags = 0;
@@ -368,7 +370,7 @@ class JITSymbol {
 /// Symbol queries are done in bulk (i.e. you request resolution of a set of
 /// symbols, rather than a single one) to reduce IPC overhead in the case of
 /// remote JITing, and expose opportunities for parallel compilation.
-class JITSymbolResolver {
+class LLVM_ABI JITSymbolResolver {
 public:
   using LookupSet = std::set<StringRef>;
   using LookupResult = std::map<StringRef, JITEvaluatedSymbol>;
@@ -398,7 +400,7 @@ class JITSymbolResolver {
 };
 
 /// Legacy symbol resolution interface.
-class LegacyJITSymbolResolver : public JITSymbolResolver {
+class LLVM_ABI LegacyJITSymbolResolver : public JITSymbolResolver {
 public:
   /// Performs lookup by, for each symbol, first calling
   ///        findSymbolInLogicalDylib and if that fails calling
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/MCJIT.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/MCJIT.h
index adce98f380c5..c836c06813fc 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/MCJIT.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/MCJIT.h
@@ -15,9 +15,10 @@
 #define LLVM_EXECUTIONENGINE_MCJIT_H
 
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdlib>
 
-extern "C" void LLVMLinkInMCJIT();
+extern "C" LLVM_ABI void LLVMLinkInMCJIT();
 
 namespace {
   struct ForceMCJITLinking {
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/ObjectCache.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/ObjectCache.h
index 1c72ca39f7c1..477ac482564a 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/ObjectCache.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/ObjectCache.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_EXECUTIONENGINE_OBJECTCACHE_H
 #define LLVM_EXECUTIONENGINE_OBJECTCACHE_H
 
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -20,7 +21,7 @@ class Module;
 /// This is the base ObjectCache type which can be provided to an
 /// ExecutionEngine for the purpose of avoiding compilation for Modules that
 /// have already been compiled and an object file is available.
-class ObjectCache {
+class LLVM_ABI ObjectCache {
   virtual void anchor();
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/AbsoluteSymbols.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/AbsoluteSymbols.h
index d58fc8b11f50..eeaa9660458f 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/AbsoluteSymbols.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/AbsoluteSymbols.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_ABSOLUTESYMBOLS_H
 
 #include "llvm/ExecutionEngine/Orc/MaterializationUnit.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::orc {
 
@@ -21,7 +22,7 @@ namespace llvm::orc {
 ///
 /// All symbols will be resolved and marked ready as soon as the unit is
 /// materialized.
-class AbsoluteSymbolsMaterializationUnit : public MaterializationUnit {
+class LLVM_ABI AbsoluteSymbolsMaterializationUnit : public MaterializationUnit {
 public:
   AbsoluteSymbolsMaterializationUnit(SymbolMap Symbols);
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h
index d9a6c41baa3d..633b762fd010 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h
@@ -20,6 +20,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 
 #include <future>
 #include <list>
@@ -31,7 +32,7 @@ namespace llvm {
 namespace orc {
 
 /// Mediates between COFF initialization and ExecutionSession state.
-class COFFPlatform : public Platform {
+class LLVM_ABI COFFPlatform : public Platform {
 public:
   /// A function that will be called with the name of dll file that must be
   /// loaded.
@@ -90,7 +91,7 @@ class COFFPlatform : public Platform {
   // The COFFPlatformPlugin scans/modifies LinkGraphs to support COFF
   // platform features including initializers, exceptions, and language
   // runtime registration.
-  class COFFPlatformPlugin : public ObjectLinkingLayer::Plugin {
+  class LLVM_ABI COFFPlatformPlugin : public ObjectLinkingLayer::Plugin {
   public:
     COFFPlatformPlugin(COFFPlatform &CP) : CP(CP) {}
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFVCRuntimeSupport.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFVCRuntimeSupport.h
index 979ebee79039..cf13fa709249 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFVCRuntimeSupport.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/COFFVCRuntimeSupport.h
@@ -18,6 +18,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 
 #include <future>
 #include <memory>
@@ -45,12 +46,12 @@ class COFFVCRuntimeBootstrapper {
   ///
   /// More information is on:
   /// https://docs.microsoft.com/en-us/cpp/c-runtime-library/crt-library-features
-  static Expected<std::unique_ptr<COFFVCRuntimeBootstrapper>>
+  LLVM_ABI static Expected<std::unique_ptr<COFFVCRuntimeBootstrapper>>
   Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
          const char *RuntimePath = nullptr);
 
   /// Adds symbol definitions of static version of msvc runtime libraries.
-  Expected<std::vector<std::string>>
+  LLVM_ABI Expected<std::vector<std::string>>
   loadStaticVCRuntime(JITDylib &JD, bool DebugVersion = false);
 
   /// Runs the initializer of static version of msvc runtime libraries.
@@ -58,10 +59,10 @@ class COFFVCRuntimeBootstrapper {
   /// printf) within the jit session. Note that proper initialization of vc
   /// runtime requires ability of running static initializers. Cosider setting
   /// up COFFPlatform.
-  Error initializeStaticVCRuntime(JITDylib &JD);
+  LLVM_ABI Error initializeStaticVCRuntime(JITDylib &JD);
 
   /// Adds symbol definitions of dynamic version of msvc runtime libraries.
-  Expected<std::vector<std::string>>
+  LLVM_ABI Expected<std::vector<std::string>>
   loadDynamicVCRuntime(JITDylib &JD, bool DebugVersion = false);
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
index d492dfe29fba..f964d006f4ae 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
@@ -37,6 +37,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <algorithm>
@@ -52,7 +53,7 @@
 namespace llvm {
 namespace orc {
 
-class CompileOnDemandLayer : public IRLayer {
+class LLVM_ABI CompileOnDemandLayer : public IRLayer {
 public:
   /// Builder for IndirectStubsManagers.
   using IndirectStubsManagerBuilder =
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h
index a55aa3d71db2..00d0057f8ef2 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h
@@ -16,6 +16,7 @@
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -27,13 +28,13 @@ class TargetMachine;
 
 namespace orc {
 
-IRSymbolMapper::ManglingOptions
+LLVM_ABI IRSymbolMapper::ManglingOptions
 irManglingOptionsFromTargetOptions(const TargetOptions &Opts);
 
 /// Simple compile functor: Takes a single IR module and returns an ObjectFile.
 /// This compiler supports a single compilation thread and LLVMContext only.
 /// For multithreaded compilation, use ConcurrentIRCompiler below.
-class SimpleCompiler : public IRCompileLayer::IRCompiler {
+class LLVM_ABI SimpleCompiler : public IRCompileLayer::IRCompiler {
 public:
   using CompileResult = std::unique_ptr<MemoryBuffer>;
 
@@ -79,7 +80,7 @@ class TMOwningSimpleCompiler : public SimpleCompiler {
 ///
 /// This class creates a new TargetMachine and SimpleCompiler instance for each
 /// compile.
-class ConcurrentIRCompiler : public IRCompileLayer::IRCompiler {
+class LLVM_ABI ConcurrentIRCompiler : public IRCompileLayer::IRCompiler {
 public:
   ConcurrentIRCompiler(JITTargetMachineBuilder JTMB,
                        ObjectCache *ObjCache = nullptr);
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index b333ef8d9ecd..f407b56817fc 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -26,6 +26,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
 #include "llvm/ExecutionEngine/Orc/TaskDispatch.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ExtensibleRTTI.h"
 
@@ -64,7 +65,7 @@ class SymbolInstance {
   const SymbolStringPtr &getName() const { return Name; }
 
   Expected<ExecutorSymbolDef> lookup() const;
-  void lookupAsync(LookupAsyncOnCompleteFn OnComplete) const;
+  LLVM_ABI void lookupAsync(LookupAsyncOnCompleteFn OnComplete) const;
 
 private:
   JITDylibSP JD;
@@ -86,7 +87,7 @@ class ResourceTracker : public ThreadSafeRefCountedBase<ResourceTracker> {
   ResourceTracker(ResourceTracker &&) = delete;
   ResourceTracker &operator=(ResourceTracker &&) = delete;
 
-  ~ResourceTracker();
+  LLVM_ABI ~ResourceTracker();
 
   /// Return the JITDylib targeted by this tracker.
   JITDylib &getJITDylib() const {
@@ -99,11 +100,11 @@ class ResourceTracker : public ThreadSafeRefCountedBase<ResourceTracker> {
   template <typename Func> Error withResourceKeyDo(Func &&F);
 
   /// Remove all resources associated with this key.
-  Error remove();
+  LLVM_ABI Error remove();
 
   /// Transfer all resources associated with this key to the given
   /// tracker, which must target the same JITDylib as this one.
-  void transferTo(ResourceTracker &DstRT);
+  LLVM_ABI void transferTo(ResourceTracker &DstRT);
 
   /// Return true if this tracker has become defunct.
   bool isDefunct() const { return JDAndFlag.load() & 0x1; }
@@ -122,7 +123,7 @@ class ResourceTracker : public ThreadSafeRefCountedBase<ResourceTracker> {
 };
 
 /// Listens for ResourceTracker operations.
-class ResourceManager {
+class LLVM_ABI ResourceManager {
 public:
   virtual ~ResourceManager();
 
@@ -420,9 +421,10 @@ using RegisterDependenciesFunction =
 
 /// This can be used as the value for a RegisterDependenciesFunction if there
 /// are no dependants to register with.
-extern RegisterDependenciesFunction NoDependenciesToRegister;
+LLVM_ABI extern RegisterDependenciesFunction NoDependenciesToRegister;
 
-class ResourceTrackerDefunct : public ErrorInfo<ResourceTrackerDefunct> {
+class LLVM_ABI ResourceTrackerDefunct
+    : public ErrorInfo<ResourceTrackerDefunct> {
 public:
   static char ID;
 
@@ -436,7 +438,7 @@ class ResourceTrackerDefunct : public ErrorInfo<ResourceTrackerDefunct> {
 
 /// Used to notify a JITDylib that the given set of symbols failed to
 /// materialize.
-class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> {
+class LLVM_ABI FailedToMaterialize : public ErrorInfo<FailedToMaterialize> {
 public:
   static char ID;
 
@@ -453,7 +455,7 @@ class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> {
 };
 
 /// Used to report failure due to unsatisfiable symbol dependencies.
-class UnsatisfiedSymbolDependencies
+class LLVM_ABI UnsatisfiedSymbolDependencies
     : public ErrorInfo<UnsatisfiedSymbolDependencies> {
 public:
   static char ID;
@@ -474,7 +476,7 @@ class UnsatisfiedSymbolDependencies
 };
 
 /// Used to notify clients when symbols can not be found during a lookup.
-class SymbolsNotFound : public ErrorInfo<SymbolsNotFound> {
+class LLVM_ABI SymbolsNotFound : public ErrorInfo<SymbolsNotFound> {
 public:
   static char ID;
 
@@ -492,7 +494,8 @@ class SymbolsNotFound : public ErrorInfo<SymbolsNotFound> {
 };
 
 /// Used to notify clients that a set of symbols could not be removed.
-class SymbolsCouldNotBeRemoved : public ErrorInfo<SymbolsCouldNotBeRemoved> {
+class LLVM_ABI SymbolsCouldNotBeRemoved
+    : public ErrorInfo<SymbolsCouldNotBeRemoved> {
 public:
   static char ID;
 
@@ -512,7 +515,8 @@ class SymbolsCouldNotBeRemoved : public ErrorInfo<SymbolsCouldNotBeRemoved> {
 /// definitions that are claimed by the module's associated
 /// MaterializationResponsibility. If this error is returned it is indicative of
 /// a broken transformation / compiler / object cache.
-class MissingSymbolDefinitions : public ErrorInfo<MissingSymbolDefinitions> {
+class LLVM_ABI MissingSymbolDefinitions
+    : public ErrorInfo<MissingSymbolDefinitions> {
 public:
   static char ID;
 
@@ -535,7 +539,8 @@ class MissingSymbolDefinitions : public ErrorInfo<MissingSymbolDefinitions> {
 /// symbols that are not claimed by the module's associated
 /// MaterializationResponsibility. If this error is returned it is indicative of
 /// a broken transformation / compiler / object cache.
-class UnexpectedSymbolDefinitions : public ErrorInfo<UnexpectedSymbolDefinitions> {
+class LLVM_ABI UnexpectedSymbolDefinitions
+    : public ErrorInfo<UnexpectedSymbolDefinitions> {
 public:
   static char ID;
 
@@ -703,7 +708,7 @@ class MaterializationResponsibility {
 
 /// A materialization unit for symbol aliases. Allows existing symbols to be
 /// aliased with alternate flags.
-class ReExportsMaterializationUnit : public MaterializationUnit {
+class LLVM_ABI ReExportsMaterializationUnit : public MaterializationUnit {
 public:
   /// SourceJD is allowed to be nullptr, in which case the source JITDylib is
   /// taken to be whatever JITDylib these definitions are materialized in (and
@@ -760,7 +765,7 @@ reexports(JITDylib &SourceJD, SymbolAliasMap Aliases,
 
 /// Build a SymbolAliasMap for the common case where you want to re-export
 /// symbols from another JITDylib with the same linkage/flags.
-Expected<SymbolAliasMap>
+LLVM_ABI Expected<SymbolAliasMap>
 buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols);
 
 /// Represents the state that a symbol has reached during materialization.
@@ -788,13 +793,13 @@ class AsynchronousSymbolQuery {
   /// Create a query for the given symbols. The NotifyComplete
   /// callback will be called once all queried symbols reach the given
   /// minimum state.
-  AsynchronousSymbolQuery(const SymbolLookupSet &Symbols,
-                          SymbolState RequiredState,
-                          SymbolsResolvedCallback NotifyComplete);
+  LLVM_ABI AsynchronousSymbolQuery(const SymbolLookupSet &Symbols,
+                                   SymbolState RequiredState,
+                                   SymbolsResolvedCallback NotifyComplete);
 
   /// Notify the query that a requested symbol has reached the required state.
-  void notifySymbolMetRequiredState(const SymbolStringPtr &Name,
-                                    ExecutorSymbolDef Sym);
+  LLVM_ABI void notifySymbolMetRequiredState(const SymbolStringPtr &Name,
+                                             ExecutorSymbolDef Sym);
 
   /// Returns true if all symbols covered by this query have been
   ///        resolved.
@@ -831,14 +836,14 @@ class LookupState {
   friend class ExecutionSession;
 
 public:
-  LookupState();
-  LookupState(LookupState &&);
-  LookupState &operator=(LookupState &&);
-  ~LookupState();
+  LLVM_ABI LookupState();
+  LLVM_ABI LookupState(LookupState &&);
+  LLVM_ABI LookupState &operator=(LookupState &&);
+  LLVM_ABI ~LookupState();
 
   /// Continue the lookup. This can be called by DefinitionGenerators
   /// to re-start a captured query-application operation.
-  void continueLookup(Error Err);
+  LLVM_ABI void continueLookup(Error Err);
 
 private:
   LookupState(std::unique_ptr<InProgressLookupState> IPLS);
@@ -851,7 +856,7 @@ class LookupState {
 
 /// Definition generators can be attached to JITDylibs to generate new
 /// definitions for otherwise unresolved symbols during lookup.
-class DefinitionGenerator {
+class LLVM_ABI DefinitionGenerator {
   friend class ExecutionSession;
 
 public:
@@ -905,7 +910,7 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   JITDylib &operator=(const JITDylib &) = delete;
   JITDylib(JITDylib &&) = delete;
   JITDylib &operator=(JITDylib &&) = delete;
-  ~JITDylib();
+  LLVM_ABI ~JITDylib();
 
   /// Get a reference to the ExecutionSession for this JITDylib.
   ///
@@ -918,7 +923,7 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   /// Dump current JITDylib state to OS.
   ///
   /// It is legal to call this method on a defunct JITDylib.
-  void dump(raw_ostream &OS);
+  LLVM_ABI void dump(raw_ostream &OS);
 
   /// Calls remove on all trackers currently associated with this JITDylib.
   /// Does not run static deinits.
@@ -930,19 +935,19 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  Error clear();
+  LLVM_ABI Error clear();
 
   /// Get the default resource tracker for this JITDylib.
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  ResourceTrackerSP getDefaultResourceTracker();
+  LLVM_ABI ResourceTrackerSP getDefaultResourceTracker();
 
   /// Create a resource tracker for this JITDylib.
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  ResourceTrackerSP createResourceTracker();
+  LLVM_ABI ResourceTrackerSP createResourceTracker();
 
   /// Adds a definition generator to this JITDylib and returns a referenece to
   /// it.
@@ -963,7 +968,7 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  void removeGenerator(DefinitionGenerator &G);
+  LLVM_ABI void removeGenerator(DefinitionGenerator &G);
 
   /// Set the link order to be used when fixing up definitions in JITDylib.
   /// This will replace the previous link order, and apply to any symbol
@@ -986,38 +991,40 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  void setLinkOrder(JITDylibSearchOrder NewSearchOrder,
-                    bool LinkAgainstThisJITDylibFirst = true);
+  LLVM_ABI void setLinkOrder(JITDylibSearchOrder NewSearchOrder,
+                             bool LinkAgainstThisJITDylibFirst = true);
 
   /// Append the given JITDylibSearchOrder to the link order for this
   /// JITDylib (discarding any elements already present in this JITDylib's
   /// link order).
-  void addToLinkOrder(const JITDylibSearchOrder &NewLinks);
+  LLVM_ABI void addToLinkOrder(const JITDylibSearchOrder &NewLinks);
 
   /// Add the given JITDylib to the link order for definitions in this
   /// JITDylib.
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  void addToLinkOrder(JITDylib &JD,
-                      JITDylibLookupFlags JDLookupFlags =
-                          JITDylibLookupFlags::MatchExportedSymbolsOnly);
+  LLVM_ABI void
+  addToLinkOrder(JITDylib &JD,
+                 JITDylibLookupFlags JDLookupFlags =
+                     JITDylibLookupFlags::MatchExportedSymbolsOnly);
 
   /// Replace OldJD with NewJD in the link order if OldJD is present.
   /// Otherwise this operation is a no-op.
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  void replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
-                          JITDylibLookupFlags JDLookupFlags =
-                              JITDylibLookupFlags::MatchExportedSymbolsOnly);
+  LLVM_ABI void
+  replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
+                     JITDylibLookupFlags JDLookupFlags =
+                         JITDylibLookupFlags::MatchExportedSymbolsOnly);
 
   /// Remove the given JITDylib from the link order for this JITDylib if it is
   /// present. Otherwise this operation is a no-op.
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  void removeFromLinkOrder(JITDylib &JD);
+  LLVM_ABI void removeFromLinkOrder(JITDylib &JD);
 
   /// Do something with the link order (run under the session lock).
   ///
@@ -1068,7 +1075,7 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// It is illegal to call this method on a defunct JITDylib and the client
   /// is responsible for ensuring that they do not do so.
-  Error remove(const SymbolNameSet &Names);
+  LLVM_ABI Error remove(const SymbolNameSet &Names);
 
   /// Returns the given JITDylibs and all of their transitive dependencies in
   /// DFS order (based on linkage relationships). Each JITDylib will appear
@@ -1076,7 +1083,7 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// If any JITDylib in the order is defunct then this method will return an
   /// error, otherwise returns the order.
-  static Expected<std::vector<JITDylibSP>>
+  LLVM_ABI static Expected<std::vector<JITDylibSP>>
   getDFSLinkOrder(ArrayRef<JITDylibSP> JDs);
 
   /// Returns the given JITDylibs and all of their transitive dependencies in
@@ -1085,7 +1092,7 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// If any JITDylib in the order is defunct then this method will return an
   /// error, otherwise returns the order.
-  static Expected<std::vector<JITDylibSP>>
+  LLVM_ABI static Expected<std::vector<JITDylibSP>>
   getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs);
 
   /// Return this JITDylib and its transitive dependencies in DFS order
@@ -1093,14 +1100,14 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// If any JITDylib in the order is defunct then this method will return an
   /// error, otherwise returns the order.
-  Expected<std::vector<JITDylibSP>> getDFSLinkOrder();
+  LLVM_ABI Expected<std::vector<JITDylibSP>> getDFSLinkOrder();
 
   /// Rteurn this JITDylib and its transitive dependencies in reverse DFS order
   /// based on linkage relationships.
   ///
   /// If any JITDylib in the order is defunct then this method will return an
   /// error, otherwise returns the order.
-  Expected<std::vector<JITDylibSP>> getReverseDFSLinkOrder();
+  LLVM_ABI Expected<std::vector<JITDylibSP>> getReverseDFSLinkOrder();
 
 private:
   using AsynchronousSymbolQuerySet =
@@ -1150,9 +1157,10 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
     std::shared_ptr<EmissionDepUnit> DefiningEDU;
     DenseSet<EmissionDepUnit *> DependantEDUs;
 
-    void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q);
-    void removeQuery(const AsynchronousSymbolQuery &Q);
-    AsynchronousSymbolQueryList takeQueriesMeeting(SymbolState RequiredState);
+    LLVM_ABI void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q);
+    LLVM_ABI void removeQuery(const AsynchronousSymbolQuery &Q);
+    LLVM_ABI AsynchronousSymbolQueryList
+    takeQueriesMeeting(SymbolState RequiredState);
     AsynchronousSymbolQueryList takeAllPendingQueries() {
       return std::move(PendingQueries);
     }
@@ -1214,10 +1222,11 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
 
   void transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT);
 
-  Error defineImpl(MaterializationUnit &MU);
+  LLVM_ABI Error defineImpl(MaterializationUnit &MU);
 
-  void installMaterializationUnit(std::unique_ptr<MaterializationUnit> MU,
-                                  ResourceTracker &RT);
+  LLVM_ABI void
+  installMaterializationUnit(std::unique_ptr<MaterializationUnit> MU,
+                             ResourceTracker &RT);
 
   void detachQueryHelper(AsynchronousSymbolQuery &Q,
                          const SymbolNameSet &QuerySymbols);
@@ -1270,7 +1279,7 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
 /// initializers (e.g. C++ static constructors) and ExecutionSession state.
 /// Note that Platforms do not automatically run initializers: clients are still
 /// responsible for doing this.
-class Platform {
+class LLVM_ABI Platform {
 public:
   virtual ~Platform();
 
@@ -1310,7 +1319,8 @@ class Platform {
 };
 
 /// A materialization task.
-class MaterializationTask : public RTTIExtends<MaterializationTask, Task> {
+class LLVM_ABI MaterializationTask
+    : public RTTIExtends<MaterializationTask, Task> {
 public:
   static char ID;
 
@@ -1329,7 +1339,7 @@ class MaterializationTask : public RTTIExtends<MaterializationTask, Task> {
 /// Lookups are usually run on the current thread, but in some cases they may
 /// be run as tasks, e.g. if the lookup has been continued from a suspended
 /// state.
-class LookupTask : public RTTIExtends<LookupTask, Task> {
+class LLVM_ABI LookupTask : public RTTIExtends<LookupTask, Task> {
 public:
   static char ID;
 
@@ -1370,15 +1380,15 @@ class ExecutionSession {
 
   /// Construct an ExecutionSession with the given ExecutorProcessControl
   /// object.
-  ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC);
+  LLVM_ABI ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC);
 
   /// Destroy an ExecutionSession. Verifies that endSession was called prior to
   /// destruction.
-  ~ExecutionSession();
+  LLVM_ABI ~ExecutionSession();
 
   /// End the session. Closes all JITDylibs and disconnects from the
   /// executor. Clients must call this method before destroying the session.
-  Error endSession();
+  LLVM_ABI Error endSession();
 
   /// Get the ExecutorProcessControl object associated with this
   /// ExecutionSession.
@@ -1413,15 +1423,15 @@ class ExecutionSession {
 
   /// Register the given ResourceManager with this ExecutionSession.
   /// Managers will be notified of events in reverse order of registration.
-  void registerResourceManager(ResourceManager &RM);
+  LLVM_ABI void registerResourceManager(ResourceManager &RM);
 
   /// Deregister the given ResourceManager with this ExecutionSession.
   /// Manager must have been previously registered.
-  void deregisterResourceManager(ResourceManager &RM);
+  LLVM_ABI void deregisterResourceManager(ResourceManager &RM);
 
   /// Return a pointer to the "name" JITDylib.
   /// Ownership of JITDylib remains within Execution Session
-  JITDylib *getJITDylibByName(StringRef Name);
+  LLVM_ABI JITDylib *getJITDylibByName(StringRef Name);
 
   /// Add a new bare JITDylib to this ExecutionSession.
   ///
@@ -1431,7 +1441,7 @@ class ExecutionSession {
   ///
   /// This call does not install any library code or symbols into the newly
   /// created JITDylib. The client is responsible for all configuration.
-  JITDylib &createBareJITDylib(std::string Name);
+  LLVM_ABI JITDylib &createBareJITDylib(std::string Name);
 
   /// Add a new JITDylib to this ExecutionSession.
   ///
@@ -1442,7 +1452,7 @@ class ExecutionSession {
   /// If a Platform is attached then Platform::setupJITDylib will be called to
   /// install standard platform symbols (e.g. standard library interposes).
   /// If no Platform is attached this call is equivalent to createBareJITDylib.
-  Expected<JITDylib &> createJITDylib(std::string Name);
+  LLVM_ABI Expected<JITDylib &> createJITDylib(std::string Name);
 
   /// Removes the given JITDylibs from the ExecutionSession.
   ///
@@ -1461,7 +1471,7 @@ class ExecutionSession {
   /// depend on it, so the JITDylib containing the ORC runtime must be removed
   /// last. If the client has introduced any other dependencies they should be
   /// accounted for in the removal order too.
-  Error removeJITDylibs(std::vector<JITDylibSP> JDsToRemove);
+  LLVM_ABI Error removeJITDylibs(std::vector<JITDylibSP> JDsToRemove);
 
   /// Calls removeJTIDylibs on the gives JITDylib.
   Error removeJITDylib(JITDylib &JD) {
@@ -1481,14 +1491,15 @@ class ExecutionSession {
 
   /// Search the given JITDylibs to find the flags associated with each of the
   /// given symbols.
-  void lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
-                   SymbolLookupSet Symbols,
-                   unique_function<void(Expected<SymbolFlagsMap>)> OnComplete);
+  LLVM_ABI void
+  lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder,
+              SymbolLookupSet Symbols,
+              unique_function<void(Expected<SymbolFlagsMap>)> OnComplete);
 
   /// Blocking version of lookupFlags.
-  Expected<SymbolFlagsMap> lookupFlags(LookupKind K,
-                                       JITDylibSearchOrder SearchOrder,
-                                       SymbolLookupSet Symbols);
+  LLVM_ABI Expected<SymbolFlagsMap> lookupFlags(LookupKind K,
+                                                JITDylibSearchOrder SearchOrder,
+                                                SymbolLookupSet Symbols);
 
   /// Search the given JITDylibs for the given symbols.
   ///
@@ -1509,10 +1520,10 @@ class ExecutionSession {
   /// dependenant symbols for this query (e.g. it is being made by a top level
   /// client to get an address to call) then the value NoDependenciesToRegister
   /// can be used.
-  void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder,
-              SymbolLookupSet Symbols, SymbolState RequiredState,
-              SymbolsResolvedCallback NotifyComplete,
-              RegisterDependenciesFunction RegisterDependencies);
+  LLVM_ABI void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder,
+                       SymbolLookupSet Symbols, SymbolState RequiredState,
+                       SymbolsResolvedCallback NotifyComplete,
+                       RegisterDependenciesFunction RegisterDependencies);
 
   /// Blocking version of lookup above. Returns the resolved symbol map.
   /// If WaitUntilReady is true (the default), will not return until all
@@ -1521,31 +1532,31 @@ class ExecutionSession {
   /// or an error occurs. If WaitUntilReady is false and an error occurs
   /// after resolution, the function will return a success value, but the
   /// error will be reported via reportErrors.
-  Expected<SymbolMap> lookup(const JITDylibSearchOrder &SearchOrder,
-                             SymbolLookupSet Symbols,
-                             LookupKind K = LookupKind::Static,
-                             SymbolState RequiredState = SymbolState::Ready,
-                             RegisterDependenciesFunction RegisterDependencies =
-                                 NoDependenciesToRegister);
+  LLVM_ABI Expected<SymbolMap>
+  lookup(const JITDylibSearchOrder &SearchOrder, SymbolLookupSet Symbols,
+         LookupKind K = LookupKind::Static,
+         SymbolState RequiredState = SymbolState::Ready,
+         RegisterDependenciesFunction RegisterDependencies =
+             NoDependenciesToRegister);
 
   /// Convenience version of blocking lookup.
   /// Searches each of the JITDylibs in the search order in turn for the given
   /// symbol.
-  Expected<ExecutorSymbolDef>
+  LLVM_ABI Expected<ExecutorSymbolDef>
   lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Symbol,
          SymbolState RequiredState = SymbolState::Ready);
 
   /// Convenience version of blocking lookup.
   /// Searches each of the JITDylibs in the search order in turn for the given
   /// symbol. The search will not find non-exported symbols.
-  Expected<ExecutorSymbolDef>
+  LLVM_ABI Expected<ExecutorSymbolDef>
   lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Symbol,
          SymbolState RequiredState = SymbolState::Ready);
 
   /// Convenience version of blocking lookup.
   /// Searches each of the JITDylibs in the search order in turn for the given
   /// symbol. The search will not find non-exported symbols.
-  Expected<ExecutorSymbolDef>
+  LLVM_ABI Expected<ExecutorSymbolDef>
   lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Symbol,
          SymbolState RequiredState = SymbolState::Ready);
 
@@ -1686,18 +1697,18 @@ class ExecutionSession {
   /// JITDylibLookupFlags::MatchAllSymbols (hidden tags will be found), and
   /// LookupFlags::WeaklyReferencedSymbol. Missing tag definitions will not
   /// cause an error, the handler will simply be dropped.
-  Error registerJITDispatchHandlers(JITDylib &JD,
-                                    JITDispatchHandlerAssociationMap WFs);
+  LLVM_ABI Error registerJITDispatchHandlers(
+      JITDylib &JD, JITDispatchHandlerAssociationMap WFs);
 
   /// Run a registered jit-side wrapper function.
   /// This should be called by the ExecutorProcessControl instance in response
   /// to incoming jit-dispatch requests from the executor.
-  void runJITDispatchHandler(SendResultFunction SendResult,
-                             ExecutorAddr HandlerFnTagAddr,
-                             ArrayRef<char> ArgBuffer);
+  LLVM_ABI void runJITDispatchHandler(SendResultFunction SendResult,
+                                      ExecutorAddr HandlerFnTagAddr,
+                                      ArrayRef<char> ArgBuffer);
 
   /// Dump the state of all the JITDylibs in this session.
-  void dump(raw_ostream &OS);
+  LLVM_ABI void dump(raw_ostream &OS);
 
   /// Check the internal consistency of ExecutionSession data structures.
 #ifdef EXPENSIVE_CHECKS
@@ -1760,11 +1771,12 @@ class ExecutionSession {
       unique_function<void(Expected<SymbolFlagsMap>)> OnComplete);
 
   // State machine functions for MaterializationResponsibility.
-  void OL_destroyMaterializationResponsibility(
-      MaterializationResponsibility &MR);
-  SymbolNameSet OL_getRequestedSymbols(const MaterializationResponsibility &MR);
-  Error OL_notifyResolved(MaterializationResponsibility &MR,
-                          const SymbolMap &Symbols);
+  LLVM_ABI void
+  OL_destroyMaterializationResponsibility(MaterializationResponsibility &MR);
+  LLVM_ABI SymbolNameSet
+  OL_getRequestedSymbols(const MaterializationResponsibility &MR);
+  LLVM_ABI Error OL_notifyResolved(MaterializationResponsibility &MR,
+                                   const SymbolMap &Symbols);
 
   using EDUInfosMap =
       DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo>;
@@ -1790,19 +1802,19 @@ class ExecutionSession {
 
   Expected<JITDylib::AsynchronousSymbolQuerySet>
   IL_emit(MaterializationResponsibility &MR, EDUInfosMap EDUInfos);
-  Error OL_notifyEmitted(MaterializationResponsibility &MR,
-                         ArrayRef<SymbolDependenceGroup> EmittedDeps);
+  LLVM_ABI Error OL_notifyEmitted(MaterializationResponsibility &MR,
+                                  ArrayRef<SymbolDependenceGroup> EmittedDeps);
 
-  Error OL_defineMaterializing(MaterializationResponsibility &MR,
-                               SymbolFlagsMap SymbolFlags);
+  LLVM_ABI Error OL_defineMaterializing(MaterializationResponsibility &MR,
+                                        SymbolFlagsMap SymbolFlags);
 
   std::pair<JITDylib::AsynchronousSymbolQuerySet,
             std::shared_ptr<SymbolDependenceMap>>
   IL_failSymbols(JITDylib &JD, const SymbolNameVector &SymbolsToFail);
-  void OL_notifyFailed(MaterializationResponsibility &MR);
-  Error OL_replace(MaterializationResponsibility &MR,
-                   std::unique_ptr<MaterializationUnit> MU);
-  Expected<std::unique_ptr<MaterializationResponsibility>>
+  LLVM_ABI void OL_notifyFailed(MaterializationResponsibility &MR);
+  LLVM_ABI Error OL_replace(MaterializationResponsibility &MR,
+                            std::unique_ptr<MaterializationUnit> MU);
+  LLVM_ABI Expected<std::unique_ptr<MaterializationResponsibility>>
   OL_delegate(MaterializationResponsibility &MR, const SymbolNameSet &Symbols);
 
 #ifndef NDEBUG
@@ -1954,7 +1966,7 @@ Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU,
 
 /// ReexportsGenerator can be used with JITDylib::addGenerator to automatically
 /// re-export a subset of the source JITDylib's symbols in the target.
-class ReexportsGenerator : public DefinitionGenerator {
+class LLVM_ABI ReexportsGenerator : public DefinitionGenerator {
 public:
   using SymbolPredicate = std::function<bool(SymbolStringPtr)>;
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h
index 70f5230c5fce..254b897bca61 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h
@@ -17,6 +17,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Memory.h"
 #include "llvm/Support/MemoryBufferRef.h"
@@ -45,7 +46,7 @@ class DebugObject;
 /// DebugObjectRegistrar is notified. Ownership of DebugObjects remains with the
 /// plugin.
 ///
-class DebugObjectManagerPlugin : public ObjectLinkingLayer::Plugin {
+class LLVM_ABI DebugObjectManagerPlugin : public ObjectLinkingLayer::Plugin {
 public:
   // DEPRECATED - Please specify options explicitly
   DebugObjectManagerPlugin(ExecutionSession &ES,
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
index fb66bf812f61..dea24c76f849 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
@@ -30,67 +31,78 @@ namespace orc {
 // --raw_ostream operators for ORC types--
 
 /// Render a SymbolNameSet.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols);
 
 /// Render a SymbolNameVector.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolNameVector &Symbols);
 
 /// Render an array of SymbolStringPtrs.
-raw_ostream &operator<<(raw_ostream &OS, ArrayRef<SymbolStringPtr> Symbols);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 ArrayRef<SymbolStringPtr> Symbols);
 
 /// Render JITSymbolFlags.
-raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags);
 
 /// Render a SymbolFlagsMap entry.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolFlagsMap::value_type &KV);
 
 /// Render a SymbolMap entry.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolMap::value_type &KV);
 
 /// Render a SymbolFlagsMap.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolFlagsMap &SymbolFlags);
 
 /// Render a SymbolMap.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols);
 
 /// Render a SymbolDependenceMap entry.
-raw_ostream &operator<<(raw_ostream &OS,
-                        const SymbolDependenceMap::value_type &KV);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolDependenceMap::value_type &KV);
 
 /// Render a SymbolDependendeMap.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolDependenceMap &Deps);
 
 /// Render a MaterializationUnit.
-raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const MaterializationUnit &MU);
 
 //// Render a JITDylibLookupFlags instance.
-raw_ostream &operator<<(raw_ostream &OS,
-                        const JITDylibLookupFlags &JDLookupFlags);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const JITDylibLookupFlags &JDLookupFlags);
 
 /// Render a SymbolLookupFlags instance.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolLookupFlags &LookupFlags);
 
 /// Render a SymbolLookupSet entry.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet::value_type &KV);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolLookupSet::value_type &KV);
 
 /// Render a SymbolLookupSet.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolLookupSet &LookupSet);
 
 /// Render a JITDylibSearchOrder.
-raw_ostream &operator<<(raw_ostream &OS,
-                        const JITDylibSearchOrder &SearchOrder);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const JITDylibSearchOrder &SearchOrder);
 
 /// Render a SymbolAliasMap.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolAliasMap &Aliases);
 
 /// Render a SymbolState.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S);
 
 /// Render a LookupKind.
-raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K);
 
 /// Dump a SymbolStringPool. Useful for debugging dangling-pointer crashes.
-raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPool &SSP);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPool &SSP);
 
 /// A function object that can be used as an ObjectTransformLayer transform
 /// to dump object files to disk at a specified path.
@@ -109,10 +121,11 @@ class DumpObjects {
   /// identifier, the resulting files will be named <ident>.o, <ident>.2.o,
   /// <ident>.3.o, and so on). IdentifierOverride should not contain an
   /// extension, as a .o suffix will be added by DumpObjects.
-  DumpObjects(std::string DumpDir = "", std::string IdentifierOverride = "");
+  LLVM_ABI DumpObjects(std::string DumpDir = "",
+                       std::string IdentifierOverride = "");
 
   /// Dumps the given buffer to disk.
-  Expected<std::unique_ptr<MemoryBuffer>>
+  LLVM_ABI Expected<std::unique_ptr<MemoryBuffer>>
   operator()(std::unique_ptr<MemoryBuffer> Obj);
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h
index 011dccbcf640..460d5e8f9e49 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 
@@ -22,10 +23,10 @@ namespace llvm {
 
 namespace orc {
 
-Error preserveDebugSections(jitlink::LinkGraph &G);
+LLVM_ABI Error preserveDebugSections(jitlink::LinkGraph &G);
 // The backing stringmap is also returned, for memory liftime management.
-Expected<std::pair<std::unique_ptr<DWARFContext>,
-                   StringMap<std::unique_ptr<MemoryBuffer>>>>
+LLVM_ABI Expected<std::pair<std::unique_ptr<DWARFContext>,
+                            StringMap<std::unique_ptr<MemoryBuffer>>>>
 createDWARFContext(jitlink::LinkGraph &G);
 
 // Thin wrapper around preserveDebugSections to be used as a standalone plugin.
@@ -59,4 +60,4 @@ class DebugInfoPreservationPlugin : public ObjectLinkingLayer::Plugin {
 
 } // namespace llvm
 
-#endif
\ No newline at end of file
+#endif
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h
index 1b47d7d41bb9..1fa1bb3ad58a 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORT_H
 #define LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORT_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -20,7 +21,7 @@ namespace orc {
 
 class LLJIT;
 
-Error enableDebuggerSupport(LLJIT &J);
+LLVM_ABI Error enableDebuggerSupport(LLJIT &J);
 
 } // namespace orc
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h
index e21d5823288e..3ca3afa12283 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h
@@ -16,6 +16,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
@@ -25,7 +26,8 @@ namespace orc {
 ///
 /// Currently MachO only. For ELF use DebugObjectManagerPlugin. These two
 /// plugins will be merged in the near future.
-class GDBJITDebugInfoRegistrationPlugin : public ObjectLinkingLayer::Plugin {
+class LLVM_ABI GDBJITDebugInfoRegistrationPlugin
+    : public ObjectLinkingLayer::Plugin {
 public:
   class DebugSectionSynthesizer {
   public:
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h
index 8235de861bef..179fedc3972a 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_PERFSUPPORTPLUGIN_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/PerfSharedStructs.h"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
@@ -24,7 +25,7 @@ namespace orc {
 /// Log perf jitdump events for each object (see
 /// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt).
 /// Currently has support for dumping code load records and unwind info records.
-class PerfSupportPlugin : public ObjectLinkingLayer::Plugin {
+class LLVM_ABI PerfSupportPlugin : public ObjectLinkingLayer::Plugin {
 public:
   PerfSupportPlugin(ExecutorProcessControl &EPC,
                     ExecutorAddr RegisterPerfStartAddr,
@@ -65,4 +66,4 @@ class PerfSupportPlugin : public ObjectLinkingLayer::Plugin {
 } // namespace orc
 } // namespace llvm
 
-#endif // LLVM_EXECUTIONENGINE_ORC_PERFSUPPORTPLUGIN_H
\ No newline at end of file
+#endif // LLVM_EXECUTIONENGINE_ORC_PERFSUPPORTPLUGIN_H
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h
index 9deb38a1a71f..5bdb6a1faed8 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h
@@ -19,12 +19,13 @@
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
 #include "llvm/ExecutionEngine/Orc/Shared/VTuneSharedStructs.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 namespace orc {
 
-class VTuneSupportPlugin : public ObjectLinkingLayer::Plugin {
+class LLVM_ABI VTuneSupportPlugin : public ObjectLinkingLayer::Plugin {
 public:
   VTuneSupportPlugin(ExecutorProcessControl &EPC, ExecutorAddr RegisterImplAddr,
                      ExecutorAddr UnregisterImplAddr, bool EmitDebugInfo)
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DylibManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DylibManager.h
index bbcc2e639a18..4cef8524c147 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DylibManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/DylibManager.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_DYLIBMANAGER_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
 
@@ -25,7 +26,7 @@ namespace llvm::orc {
 
 class SymbolLookupSet;
 
-class DylibManager {
+class LLVM_ABI DylibManager {
 public:
   /// A pair of a dylib and a set of symbols to be looked up.
   struct LookupRequest {
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h
index 6f25b998fb00..7b5d0f0eaba2 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_EHFRAMEREGISTRATIONPLUGIN_H
 
 #include "llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h"
+#include "llvm/Support/Compiler.h"
 
 #include <memory>
 #include <mutex>
@@ -23,7 +24,8 @@ namespace llvm::orc {
 
 /// Adds AllocationActions to register and deregister eh-frame sections in the
 /// absence of native Platform support.
-class EHFrameRegistrationPlugin : public LinkGraphLinkingLayer::Plugin {
+class LLVM_ABI EHFrameRegistrationPlugin
+    : public LinkGraphLinkingLayer::Plugin {
 public:
   static Expected<std::unique_ptr<EHFrameRegistrationPlugin>>
   Create(ExecutionSession &ES);
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
index f19cfce16d4e..6872c1ec2d54 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
@@ -18,6 +18,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 
 #include <future>
 #include <thread>
@@ -65,7 +66,7 @@ using DeferredRuntimeFnMap = std::unordered_map<
     FunctionPairKeyHash, FunctionPairKeyEqual>;
 
 /// Mediates between ELFNix initialization and ExecutionSession state.
-class ELFNixPlatform : public Platform {
+class LLVM_ABI ELFNixPlatform : public Platform {
 public:
   /// Try to create a ELFNixPlatform instance, adding the ORC runtime to the
   /// given JITDylib.
@@ -165,7 +166,7 @@ class ELFNixPlatform : public Platform {
   // The ELFNixPlatformPlugin scans/modifies LinkGraphs to support ELF
   // platform features including initializers, exceptions, TLV, and language
   // runtime registration.
-  class ELFNixPlatformPlugin : public ObjectLinkingLayer::Plugin {
+  class LLVM_ABI ELFNixPlatformPlugin : public ObjectLinkingLayer::Plugin {
   public:
     ELFNixPlatformPlugin(ELFNixPlatform &MP) : MP(MP) {}
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h
index 3d125e81f2e1..3a92eee0430c 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h
@@ -16,6 +16,7 @@
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Memory.h"
 
@@ -37,7 +38,7 @@ class DebugObjectRegistrar {
 
 /// Use ExecutorProcessControl to register debug objects locally or in a remote
 /// executor process.
-class EPCDebugObjectRegistrar : public DebugObjectRegistrar {
+class LLVM_ABI EPCDebugObjectRegistrar : public DebugObjectRegistrar {
 public:
   EPCDebugObjectRegistrar(ExecutionSession &ES, ExecutorAddr RegisterFn)
       : ES(ES), RegisterFn(RegisterFn) {}
@@ -57,7 +58,8 @@ class EPCDebugObjectRegistrar : public DebugObjectRegistrar {
 /// If RegistrationFunctionsDylib is non-None then it will be searched to find
 /// the registration functions. If it is None then the process dylib will be
 /// loaded to find the registration functions.
-Expected<std::unique_ptr<EPCDebugObjectRegistrar>> createJITLoaderGDBRegistrar(
+LLVM_ABI Expected<std::unique_ptr<EPCDebugObjectRegistrar>>
+createJITLoaderGDBRegistrar(
     ExecutionSession &ES,
     std::optional<ExecutorAddr> RegistrationFunctionDylib = std::nullopt);
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h
index cb05d7236a7e..ccb855160d6f 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h
@@ -16,13 +16,14 @@
 
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 
 class ExecutorProcessControl;
 
-class EPCDynamicLibrarySearchGenerator : public DefinitionGenerator {
+class LLVM_ABI EPCDynamicLibrarySearchGenerator : public DefinitionGenerator {
 public:
   using SymbolPredicate = unique_function<bool(const SymbolStringPtr &)>;
   using AddAbsoluteSymbolsFn = unique_function<Error(JITDylib &, SymbolMap)>;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h
index 887147a77026..68bc54b89fe5 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
@@ -38,7 +39,7 @@ class EPCGenericDylibManager {
 
   /// Create an EPCGenericMemoryAccess instance from a given set of
   /// function addrs.
-  static Expected<EPCGenericDylibManager>
+  LLVM_ABI static Expected<EPCGenericDylibManager>
   CreateWithDefaultBootstrapSymbols(ExecutorProcessControl &EPC);
 
   /// Create an EPCGenericMemoryAccess instance from a given set of
@@ -47,7 +48,7 @@ class EPCGenericDylibManager {
       : EPC(EPC), SAs(SAs) {}
 
   /// Loads the dylib with the given name.
-  Expected<tpctypes::DylibHandle> open(StringRef Path, uint64_t Mode);
+  LLVM_ABI Expected<tpctypes::DylibHandle> open(StringRef Path, uint64_t Mode);
 
   /// Looks up symbols within the given dylib.
   Expected<std::vector<ExecutorSymbolDef>>
@@ -71,12 +72,14 @@ class EPCGenericDylibManager {
       unique_function<void(Expected<std::vector<ExecutorSymbolDef>>)>;
 
   /// Looks up symbols within the given dylib.
-  void lookupAsync(tpctypes::DylibHandle H, const SymbolLookupSet &Lookup,
-                   SymbolLookupCompleteFn Complete);
+  LLVM_ABI void lookupAsync(tpctypes::DylibHandle H,
+                            const SymbolLookupSet &Lookup,
+                            SymbolLookupCompleteFn Complete);
 
   /// Looks up symbols within the given dylib.
-  void lookupAsync(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &Lookup,
-                   SymbolLookupCompleteFn Complete);
+  LLVM_ABI void lookupAsync(tpctypes::DylibHandle H,
+                            const RemoteSymbolLookupSet &Lookup,
+                            SymbolLookupCompleteFn Complete);
 
 private:
   ExecutorProcessControl &EPC;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h
index 48713e765deb..f9070afc1a32 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h
@@ -20,11 +20,13 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 
-class EPCGenericJITLinkMemoryManager : public jitlink::JITLinkMemoryManager {
+class LLVM_ABI EPCGenericJITLinkMemoryManager
+    : public jitlink::JITLinkMemoryManager {
 public:
   /// Function addresses for memory access.
   struct SymbolAddrs {
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h
index 2ea3c1b66cdd..faec25d81fac 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
+#include "llvm/Support/Compiler.h"
 
 #define DEBUG_TYPE "orc"
 
@@ -23,7 +24,8 @@ namespace llvm {
 namespace orc {
 
 /// Remote-mapped RuntimeDyld-compatible memory manager.
-class EPCGenericRTDyldMemoryManager : public RuntimeDyld::MemoryManager {
+class LLVM_ABI EPCGenericRTDyldMemoryManager
+    : public RuntimeDyld::MemoryManager {
 public:
   /// Symbol addresses for memory access.
   struct SymbolAddrs {
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index 64ac8cb80fc2..e42c902568a0 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/Object/Archive.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include <algorithm>
 #include <cstdint>
@@ -65,23 +66,23 @@ class CtorDtorIterator {
 
   /// Construct an iterator instance. If End is true then this iterator
   ///        acts as the end of the range, otherwise it is the beginning.
-  CtorDtorIterator(const GlobalVariable *GV, bool End);
+  LLVM_ABI CtorDtorIterator(const GlobalVariable *GV, bool End);
 
   /// Test iterators for equality.
-  bool operator==(const CtorDtorIterator &Other) const;
+  LLVM_ABI bool operator==(const CtorDtorIterator &Other) const;
 
   /// Test iterators for inequality.
-  bool operator!=(const CtorDtorIterator &Other) const;
+  LLVM_ABI bool operator!=(const CtorDtorIterator &Other) const;
 
   /// Pre-increment iterator.
-  CtorDtorIterator& operator++();
+  LLVM_ABI CtorDtorIterator &operator++();
 
   /// Post-increment iterator.
-  CtorDtorIterator operator++(int);
+  LLVM_ABI CtorDtorIterator operator++(int);
 
   /// Dereference iterator. The resulting value provides a read-only view
   ///        of this element of the global_ctors/global_dtors list.
-  Element operator*() const;
+  LLVM_ABI Element operator*() const;
 
 private:
   const ConstantArray *InitList;
@@ -90,11 +91,11 @@ class CtorDtorIterator {
 
 /// Create an iterator range over the entries of the llvm.global_ctors
 ///        array.
-iterator_range<CtorDtorIterator> getConstructors(const Module &M);
+LLVM_ABI iterator_range<CtorDtorIterator> getConstructors(const Module &M);
 
 /// Create an iterator range over the entries of the llvm.global_ctors
 ///        array.
-iterator_range<CtorDtorIterator> getDestructors(const Module &M);
+LLVM_ABI iterator_range<CtorDtorIterator> getDestructors(const Module &M);
 
 /// This iterator provides a convenient way to iterate over GlobalValues that
 /// have initialization effects.
@@ -124,7 +125,7 @@ class StaticInitGVIterator {
   GlobalValue &operator*() { return *I; }
 
 private:
-  bool isStaticInitGlobal(GlobalValue &GV);
+  LLVM_ABI bool isStaticInitGlobal(GlobalValue &GV);
   void moveToNextStaticInitGlobal() {
     ++I;
     while (I != E && !isStaticInitGlobal(*I))
@@ -146,8 +147,8 @@ inline iterator_range<StaticInitGVIterator> getStaticInitGVs(Module &M) {
 class CtorDtorRunner {
 public:
   CtorDtorRunner(JITDylib &JD) : JD(JD) {}
-  void add(iterator_range<CtorDtorIterator> CtorDtors);
-  Error run();
+  LLVM_ABI void add(iterator_range<CtorDtorIterator> CtorDtors);
+  LLVM_ABI Error run();
 
 private:
   using CtorDtorList = std::vector<SymbolStringPtr>;
@@ -176,20 +177,20 @@ class LocalCXXRuntimeOverridesBase {
 public:
   /// Run any destructors recorded by the overriden __cxa_atexit function
   /// (CXAAtExitOverride).
-  void runDestructors();
+  LLVM_ABI void runDestructors();
 
 protected:
   using DestructorPtr = void (*)(void *);
   using CXXDestructorDataPair = std::pair<DestructorPtr, void *>;
   using CXXDestructorDataPairList = std::vector<CXXDestructorDataPair>;
   CXXDestructorDataPairList DSOHandleOverride;
-  static int CXAAtExitOverride(DestructorPtr Destructor, void *Arg,
-                               void *DSOHandle);
+  LLVM_ABI static int CXAAtExitOverride(DestructorPtr Destructor, void *Arg,
+                                        void *DSOHandle);
 };
 
 class LocalCXXRuntimeOverrides : public LocalCXXRuntimeOverridesBase {
 public:
-  Error enable(JITDylib &JD, MangleAndInterner &Mangler);
+  LLVM_ABI Error enable(JITDylib &JD, MangleAndInterner &Mangler);
 };
 
 /// An interface for Itanium __cxa_atexit interposer implementations.
@@ -200,8 +201,8 @@ class ItaniumCXAAtExitSupport {
     void *Ctx;
   };
 
-  void registerAtExit(void (*F)(void *), void *Ctx, void *DSOHandle);
-  void runAtExits(void *DSOHandle);
+  LLVM_ABI void registerAtExit(void (*F)(void *), void *Ctx, void *DSOHandle);
+  LLVM_ABI void runAtExits(void *DSOHandle);
 
 private:
   std::mutex AtExitsMutex;
@@ -213,7 +214,7 @@ class ItaniumCXAAtExitSupport {
 /// If an instance of this class is attached to a JITDylib as a fallback
 /// definition generator, then any symbol found in the given DynamicLibrary that
 /// passes the 'Allow' predicate will be added to the JITDylib.
-class DynamicLibrarySearchGenerator : public DefinitionGenerator {
+class LLVM_ABI DynamicLibrarySearchGenerator : public DefinitionGenerator {
 public:
   using SymbolPredicate = std::function<bool(const SymbolStringPtr &)>;
   using AddAbsoluteSymbolsFn = unique_function<Error(JITDylib &, SymbolMap)>;
@@ -266,7 +267,7 @@ class DynamicLibrarySearchGenerator : public DefinitionGenerator {
 /// If an instance of this class is attached to a JITDylib as a fallback
 /// definition generator, then any symbol found in the archive will result in
 /// the containing object being added to the JITDylib.
-class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
+class LLVM_ABI StaticLibraryDefinitionGenerator : public DefinitionGenerator {
 public:
   /// Interface builder function for objects loaded from this archive.
   using GetObjectFileInterface =
@@ -355,7 +356,7 @@ class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
 /// definition generator, PLT stubs and dllimport __imp_ symbols will be
 /// generated for external symbols found outside the given jitdylib. Currently
 /// only supports x86_64 architecture.
-class DLLImportDefinitionGenerator : public DefinitionGenerator {
+class LLVM_ABI DLLImportDefinitionGenerator : public DefinitionGenerator {
 public:
   /// Creates a DLLImportDefinitionGenerator instance.
   static std::unique_ptr<DLLImportDefinitionGenerator>
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
index 86e98e74b705..e63f5f784252 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
@@ -22,6 +22,7 @@
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h"
 #include "llvm/ExecutionEngine/Orc/TaskDispatch.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
 #include "llvm/TargetParser/Triple.h"
@@ -36,7 +37,7 @@ namespace orc {
 class ExecutionSession;
 
 /// ExecutorProcessControl supports interaction with a JIT target process.
-class ExecutorProcessControl {
+class LLVM_ABI ExecutorProcessControl {
   friend class ExecutionSession;
 public:
 
@@ -99,7 +100,7 @@ class ExecutorProcessControl {
   };
 
   /// APIs for manipulating memory in the target process.
-  class MemoryAccess {
+  class LLVM_ABI MemoryAccess {
   public:
     /// Callback function for asynchronous writes.
     using WriteResultFn = unique_function<void(Error)>;
@@ -398,7 +399,8 @@ class ExecutorProcessControl {
   StringMap<ExecutorAddr> BootstrapSymbols;
 };
 
-class InProcessMemoryAccess : public ExecutorProcessControl::MemoryAccess {
+class LLVM_ABI InProcessMemoryAccess
+    : public ExecutorProcessControl::MemoryAccess {
 public:
   InProcessMemoryAccess(bool IsArch64Bit) : IsArch64Bit(IsArch64Bit) {}
   void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws,
@@ -465,9 +467,9 @@ class UnsupportedExecutorProcessControl : public ExecutorProcessControl,
 };
 
 /// A ExecutorProcessControl implementation targeting the current process.
-class SelfExecutorProcessControl : public ExecutorProcessControl,
-                                   private InProcessMemoryAccess,
-                                   private DylibManager {
+class LLVM_ABI SelfExecutorProcessControl : public ExecutorProcessControl,
+                                            private InProcessMemoryAccess,
+                                            private DylibManager {
 public:
   SelfExecutorProcessControl(
       std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D,
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/GetDylibInterface.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/GetDylibInterface.h
index 317d0e40fce2..077d88d1758a 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/GetDylibInterface.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/GetDylibInterface.h
@@ -16,23 +16,25 @@
 #define LLVM_EXECUTIONENGINE_ORC_GETDYLIBINTERFACE_H
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::orc {
 
 /// Returns a SymbolNameSet containing the exported symbols defined in the
 /// given dylib.
-Expected<SymbolNameSet> getDylibInterfaceFromDylib(ExecutionSession &ES,
-                                                   Twine Path);
+LLVM_ABI Expected<SymbolNameSet>
+getDylibInterfaceFromDylib(ExecutionSession &ES, Twine Path);
 
 /// Returns a SymbolNameSet containing the exported symbols defined in the
 /// relevant slice of the TapiUniversal file.
-Expected<SymbolNameSet> getDylibInterfaceFromTapiFile(ExecutionSession &ES,
-                                                      Twine Path);
+LLVM_ABI Expected<SymbolNameSet>
+getDylibInterfaceFromTapiFile(ExecutionSession &ES, Twine Path);
 
 /// Returns a SymbolNameSet containing the exported symbols defined in the
 /// relevant slice of the given file, which may be either a dylib or a tapi
 /// file.
-Expected<SymbolNameSet> getDylibInterface(ExecutionSession &ES, Twine Path);
+LLVM_ABI Expected<SymbolNameSet> getDylibInterface(ExecutionSession &ES,
+                                                   Twine Path);
 
 } // namespace llvm::orc
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
index f493d63a7c8a..c5b88ddaf613 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <functional>
@@ -28,9 +29,9 @@ class Module;
 
 namespace orc {
 
-class IRCompileLayer : public IRLayer {
+class LLVM_ABI IRCompileLayer : public IRLayer {
 public:
-  class IRCompiler {
+  class LLVM_ABI IRCompiler {
   public:
     IRCompiler(IRSymbolMapper::ManglingOptions MO) : MO(std::move(MO)) {}
     virtual ~IRCompiler();
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRPartitionLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRPartitionLayer.h
index a2981fea61d3..112fe24bb282 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRPartitionLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRPartitionLayer.h
@@ -28,13 +28,14 @@
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 
 /// A layer that breaks up IR modules into smaller submodules that only contains
 /// looked up symbols.
-class IRPartitionLayer : public IRLayer {
+class LLVM_ABI IRPartitionLayer : public IRLayer {
   friend class PartitioningIRMaterializationUnit;
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
index 4f1cde32ff18..eaa44eb45830 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -24,7 +25,7 @@ namespace orc {
 /// A layer that applies a transform to emitted modules.
 /// The transform function is responsible for locking the ThreadSafeContext
 /// before operating on the module.
-class IRTransformLayer : public IRLayer {
+class LLVM_ABI IRTransformLayer : public IRLayer {
 public:
   using TransformFunction = unique_function<Expected<ThreadSafeModule>(
       ThreadSafeModule, MaterializationResponsibility &R)>;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
index a8129d70572c..fecffc2a0bb3 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
 #include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Memory.h"
 #include "llvm/Support/Process.h"
@@ -61,7 +62,7 @@ namespace orc {
 /// before calling a supplied function to return the trampoline landing
 /// address, then restore all state before jumping to that address. They
 /// are used by various ORC APIs to support lazy compilation
-class TrampolinePool {
+class LLVM_ABI TrampolinePool {
 public:
   using NotifyLandingResolvedFunction =
       unique_function<void(ExecutorAddr) const>;
@@ -208,11 +209,11 @@ class JITCompileCallbackManager {
   virtual ~JITCompileCallbackManager() = default;
 
   /// Reserve a compile callback.
-  Expected<ExecutorAddr> getCompileCallback(CompileFunction Compile);
+  LLVM_ABI Expected<ExecutorAddr> getCompileCallback(CompileFunction Compile);
 
   /// Execute the callback for the given trampoline id. Called by the JIT
   ///        to compile functions on demand.
-  ExecutorAddr executeCompileCallback(ExecutorAddr TrampolineAddr);
+  LLVM_ABI ExecutorAddr executeCompileCallback(ExecutorAddr TrampolineAddr);
 
 protected:
   /// Construct a JITCompileCallbackManager.
@@ -279,7 +280,7 @@ class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
 };
 
 /// Base class for managing collections of named indirect stubs.
-class IndirectStubsManager : public RedirectableSymbolManager {
+class LLVM_ABI IndirectStubsManager : public RedirectableSymbolManager {
 public:
   /// Map type for initializing the manager. See init.
   using StubInitsMap = StringMap<std::pair<ExecutorAddr, JITSymbolFlags>>;
@@ -479,14 +480,14 @@ class LocalIndirectStubsManager : public IndirectStubsManager {
 /// The given target triple will determine the ABI, and the given
 /// ErrorHandlerAddress will be used by the resulting compile callback
 /// manager if a compile callback fails.
-Expected<std::unique_ptr<JITCompileCallbackManager>>
+LLVM_ABI Expected<std::unique_ptr<JITCompileCallbackManager>>
 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
                                   ExecutorAddr ErrorHandlerAddress);
 
 /// Create a local indirect stubs manager builder.
 ///
 /// The given target triple will determine the ABI.
-std::function<std::unique_ptr<IndirectStubsManager>()>
+LLVM_ABI std::function<std::unique_ptr<IndirectStubsManager>()>
 createLocalIndirectStubsManagerBuilder(const Triple &T);
 
 /// Build a function pointer of FunctionType with the given constant
@@ -494,16 +495,17 @@ createLocalIndirectStubsManagerBuilder(const Triple &T);
 ///
 ///   Usage example: Turn a trampoline address into a function pointer constant
 /// for use in a stub.
-Constant *createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr);
+LLVM_ABI Constant *createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr);
 
 /// Create a function pointer with the given type, name, and initializer
 ///        in the given Module.
-GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
-                                  Constant *Initializer);
+LLVM_ABI GlobalVariable *createImplPointer(PointerType &PT, Module &M,
+                                           const Twine &Name,
+                                           Constant *Initializer);
 
 /// Turn a function declaration into a stub function that makes an
 ///        indirect call using the given function pointer.
-void makeStub(Function &F, Value &ImplPointer);
+LLVM_ABI void makeStub(Function &F, Value &ImplPointer);
 
 /// Promotes private symbols to global hidden, and renames to prevent clashes
 /// with other promoted symbols. The same SymbolPromoter instance should be
@@ -512,7 +514,7 @@ class SymbolLinkagePromoter {
 public:
   /// Promote symbols in the given module. Returns the set of global values
   /// that have been renamed/promoted.
-  std::vector<GlobalValue *> operator()(Module &M);
+  LLVM_ABI std::vector<GlobalValue *> operator()(Module &M);
 
 private:
   unsigned NextId = 0;
@@ -530,16 +532,18 @@ class SymbolLinkagePromoter {
 /// modules with these utilities, all decls should be cloned (and added to a
 /// single VMap) before any bodies are moved. This will ensure that references
 /// between functions all refer to the versions in the new module.
-Function *cloneFunctionDecl(Module &Dst, const Function &F,
-                            ValueToValueMapTy *VMap = nullptr);
+LLVM_ABI Function *cloneFunctionDecl(Module &Dst, const Function &F,
+                                     ValueToValueMapTy *VMap = nullptr);
 
 /// Clone a global variable declaration into a new module.
-GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
-                                        ValueToValueMapTy *VMap = nullptr);
+LLVM_ABI GlobalVariable *
+cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
+                        ValueToValueMapTy *VMap = nullptr);
 
 /// Clone a global alias declaration into a new module.
-GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
-                                  ValueToValueMapTy &VMap);
+LLVM_ABI GlobalAlias *cloneGlobalAliasDecl(Module &Dst,
+                                           const GlobalAlias &OrigA,
+                                           ValueToValueMapTy &VMap);
 
 /// Introduce relocations to \p Sym in its own definition if there are any
 /// pointers formed via PC-relative address that do not already have a
@@ -563,10 +567,9 @@ GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
 ///
 /// This is based on disassembly and should be considered "best effort". It may
 /// silently fail to add relocations.
-Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
-                                                   jitlink::LinkGraph &G,
-                                                   MCDisassembler &Disassembler,
-                                                   MCInstrAnalysis &MIA);
+LLVM_ABI Error addFunctionPointerRelocationsToCurrentSymbol(
+    jitlink::Symbol &Sym, jitlink::LinkGraph &G, MCDisassembler &Disassembler,
+    MCInstrAnalysis &MIA);
 
 } // end namespace orc
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
index 83339e56cfa5..5fa8f89e7ed0 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/StringSaver.h"
 
 #include <atomic>
@@ -22,7 +23,8 @@
 namespace llvm {
 namespace orc {
 
-class JITLinkRedirectableSymbolManager : public RedirectableSymbolManager {
+class LLVM_ABI JITLinkRedirectableSymbolManager
+    : public RedirectableSymbolManager {
 public:
   /// Create redirection manager that uses JITLink based implementaion.
   static Expected<std::unique_ptr<RedirectableSymbolManager>>
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
index 94d2ff0717eb..c6a67ab391c3 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
@@ -17,6 +17,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/LazyReexports.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm::jitlink {
@@ -42,16 +43,16 @@ class JITLinkReentryTrampolines {
 
   /// Create trampolines using the default reentry trampoline function for
   /// the session triple.
-  static Expected<std::unique_ptr<JITLinkReentryTrampolines>>
+  LLVM_ABI static Expected<std::unique_ptr<JITLinkReentryTrampolines>>
   Create(ObjectLinkingLayer &ObjLinkingLayer);
 
-  JITLinkReentryTrampolines(ObjectLinkingLayer &ObjLinkingLayer,
-                            EmitTrampolineFn EmitTrampoline);
+  LLVM_ABI JITLinkReentryTrampolines(ObjectLinkingLayer &ObjLinkingLayer,
+                                     EmitTrampolineFn EmitTrampoline);
   JITLinkReentryTrampolines(JITLinkReentryTrampolines &&) = delete;
   JITLinkReentryTrampolines &operator=(JITLinkReentryTrampolines &&) = delete;
 
-  void emit(ResourceTrackerSP RT, size_t NumTrampolines,
-            OnTrampolinesReadyFn OnTrampolinesReady);
+  LLVM_ABI void emit(ResourceTrackerSP RT, size_t NumTrampolines,
+                     OnTrampolinesReadyFn OnTrampolinesReady);
 
 private:
   class TrampolineAddrScraperPlugin;
@@ -62,7 +63,7 @@ class JITLinkReentryTrampolines {
   std::atomic<size_t> ReentryGraphIdx{0};
 };
 
-Expected<std::unique_ptr<LazyReexportsManager>>
+LLVM_ABI Expected<std::unique_ptr<LazyReexportsManager>>
 createJITLinkLazyReexportsManager(ObjectLinkingLayer &ObjLinkingLayer,
                                   RedirectableSymbolManager &RSMgr,
                                   JITDylib &PlatformJD,
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h
index 62f779c6ac50..d7e666beff6a 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_JITTARGETMACHINEBUILDER_H
 
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -41,14 +42,14 @@ class JITTargetMachineBuilder {
   /// Note: TargetOptions is default-constructed, then EmulatedTLS is set to
   /// true. If EmulatedTLS is not required, these values should be reset before
   /// calling createTargetMachine.
-  JITTargetMachineBuilder(Triple TT);
+  LLVM_ABI JITTargetMachineBuilder(Triple TT);
 
   /// Create a JITTargetMachineBuilder for the host system.
   ///
   /// Note: TargetOptions is default-constructed, then EmulatedTLS is set to
   /// true. If EmulatedTLS is not required, these values should be reset before
   /// calling createTargetMachine.
-  static Expected<JITTargetMachineBuilder> detectHost();
+  LLVM_ABI static Expected<JITTargetMachineBuilder> detectHost();
 
   /// Create a TargetMachine.
   ///
@@ -57,7 +58,7 @@ class JITTargetMachineBuilder {
   /// the target's AsmPrinter must both be registered. To JIT assembly
   /// (including inline and module level assembly) the target's AsmParser must
   /// also be registered.
-  Expected<std::unique_ptr<TargetMachine>> createTargetMachine();
+  LLVM_ABI Expected<std::unique_ptr<TargetMachine>> createTargetMachine();
 
   /// Get the default DataLayout for the target.
   ///
@@ -111,7 +112,7 @@ class JITTargetMachineBuilder {
   }
 
   /// Add subtarget features.
-  JITTargetMachineBuilder &
+  LLVM_ABI JITTargetMachineBuilder &
   addFeatures(const std::vector<std::string> &FeatureVec);
 
   /// Access subtarget features.
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
index 4ff688e07dfe..cb8fdd650a2a 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
@@ -23,6 +23,7 @@
 #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ThreadPool.h"
 #include <variant>
@@ -38,14 +39,15 @@ class ExecutorProcessControl;
 /// A pre-fabricated ORC JIT stack that can serve as an alternative to MCJIT.
 ///
 /// Create instances using LLJITBuilder.
-class LLJIT {
+class LLVM_ABI LLJIT {
   template <typename, typename, typename> friend class LLJITBuilderSetters;
 
-  friend Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J);
+  LLVM_ABI_FRIEND friend Expected<JITDylibSP>
+  setUpGenericLLVMIRPlatform(LLJIT &J);
 
 public:
   /// Initializer support for LLJIT.
-  class PlatformSupport {
+  class LLVM_ABI PlatformSupport {
   public:
     virtual ~PlatformSupport();
 
@@ -279,7 +281,7 @@ class LLLazyJIT : public LLJIT {
   CompileOnDemandLayer &getCompileOnDemandLayer() { return *CODLayer; }
 
   /// Add a module to be lazily compiled to JITDylib JD.
-  Error addLazyIRModule(JITDylib &JD, ThreadSafeModule M);
+  LLVM_ABI Error addLazyIRModule(JITDylib &JD, ThreadSafeModule M);
 
   /// Add a module to be lazily compiled to the main JITDylib.
   Error addLazyIRModule(ThreadSafeModule M) {
@@ -289,7 +291,7 @@ class LLLazyJIT : public LLJIT {
 private:
 
   // Create a single-threaded LLLazyJIT instance.
-  LLLazyJIT(LLLazyJITBuilderState &S, Error &Err);
+  LLVM_ABI LLLazyJIT(LLLazyJITBuilderState &S, Error &Err);
 
   std::unique_ptr<LazyCallThroughManager> LCTMgr;
   std::unique_ptr<IRPartitionLayer> IPLayer;
@@ -327,7 +329,7 @@ class LLJITBuilderState {
   std::optional<bool> SupportConcurrentCompilation;
 
   /// Called prior to JIT class construcion to fix up defaults.
-  Error prepareForConstruction();
+  LLVM_ABI Error prepareForConstruction();
 };
 
 template <typename JITType, typename SetterImpl, typename State>
@@ -523,7 +525,7 @@ class LLLazyJITBuilderState : public LLJITBuilderState {
   std::unique_ptr<LazyCallThroughManager> LCTMgr;
   IndirectStubsManagerBuilderFunction ISMBuilder;
 
-  Error prepareForConstruction();
+  LLVM_ABI Error prepareForConstruction();
 };
 
 template <typename JITType, typename SetterImpl, typename State>
@@ -567,7 +569,7 @@ class LLLazyJITBuilder
 
 /// Configure the LLJIT instance to use orc runtime support. This overload
 /// assumes that the client has manually configured a Platform object.
-Error setUpOrcPlatformManually(LLJIT &J);
+LLVM_ABI Error setUpOrcPlatformManually(LLJIT &J);
 
 /// Configure the LLJIT instance to use the ORC runtime and the detected
 /// native target for the executor.
@@ -590,7 +592,7 @@ class ExecutorNativePlatform {
     return *this;
   }
 
-  Expected<JITDylibSP> operator()(LLJIT &J);
+  LLVM_ABI Expected<JITDylibSP> operator()(LLJIT &J);
 
 private:
   std::variant<std::string, std::unique_ptr<MemoryBuffer>> OrcRuntime;
@@ -601,17 +603,17 @@ class ExecutorNativePlatform {
 /// llvm.global_dtors variables and (if present) build initialization and
 /// deinitialization functions. Platform specific initialization configurations
 /// should be preferred where available.
-Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J);
+LLVM_ABI Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J);
 
 /// Configure the LLJIT instance to disable platform support explicitly. This is
 /// useful in two cases: for platforms that don't have such requirements and for
 /// platforms, that we have no explicit support yet and that don't work well
 /// with the generic IR platform.
-Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J);
+LLVM_ABI Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J);
 
 /// A Platform-support class that implements initialize / deinitialize by
 /// forwarding to ORC runtime dlopen / dlclose operations.
-class ORCPlatformSupport : public LLJIT::PlatformSupport {
+class LLVM_ABI ORCPlatformSupport : public LLJIT::PlatformSupport {
 public:
   ORCPlatformSupport(orc::LLJIT &J) : J(J) {}
   Error initialize(orc::JITDylib &JD) override;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Layer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Layer.h
index cfeedc2a0bda..8dfc12ee9fad 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Layer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Layer.h
@@ -18,6 +18,7 @@
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ExtensibleRTTI.h"
 #include "llvm/Support/MemoryBuffer.h"
 
@@ -28,7 +29,7 @@ namespace orc {
 /// wrapping LLVM IR. Represents materialization responsibility for all symbols
 /// in the given module. If symbols are overridden by other definitions, then
 /// their linkage is changed to available-externally.
-class IRMaterializationUnit : public MaterializationUnit {
+class LLVM_ABI IRMaterializationUnit : public MaterializationUnit {
 public:
   using SymbolNameToDefinitionMap = std::map<SymbolStringPtr, GlobalValue *>;
 
@@ -64,7 +65,7 @@ class IRMaterializationUnit : public MaterializationUnit {
 };
 
 /// Interface for layers that accept LLVM IR.
-class IRLayer {
+class LLVM_ABI IRLayer {
 public:
   IRLayer(ExecutionSession &ES, const IRSymbolMapper::ManglingOptions *&MO)
       : ES(ES), MO(MO) {}
@@ -117,7 +118,7 @@ class IRLayer {
 
 /// MaterializationUnit that materializes modules by calling the 'emit' method
 /// on the given IRLayer.
-class BasicIRLayerMaterializationUnit : public IRMaterializationUnit {
+class LLVM_ABI BasicIRLayerMaterializationUnit : public IRMaterializationUnit {
 public:
   BasicIRLayerMaterializationUnit(IRLayer &L,
                                   const IRSymbolMapper::ManglingOptions &MO,
@@ -130,7 +131,7 @@ class BasicIRLayerMaterializationUnit : public IRMaterializationUnit {
 };
 
 /// Interface for Layers that accept object files.
-class ObjectLayer : public RTTIExtends<ObjectLayer, RTTIRoot> {
+class LLVM_ABI ObjectLayer : public RTTIExtends<ObjectLayer, RTTIRoot> {
 public:
   static char ID;
 
@@ -172,7 +173,8 @@ class ObjectLayer : public RTTIExtends<ObjectLayer, RTTIRoot> {
 
 /// Materializes the given object file (represented by a MemoryBuffer
 /// instance) by calling 'emit' on the given ObjectLayer.
-class BasicObjectLayerMaterializationUnit : public MaterializationUnit {
+class LLVM_ABI BasicObjectLayerMaterializationUnit
+    : public MaterializationUnit {
 public:
   /// Create using the default object interface builder function.
   static Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>>
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
index 8a0350b59d39..5203ae05e5b7 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
@@ -14,6 +14,7 @@
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::orc {
 
@@ -33,7 +34,7 @@ class RedirectableSymbolManager;
 ///
 /// References to data symbols are not lazy and will trigger immediate linking
 /// (same os ObjectlinkingLayer).
-class LazyObjectLinkingLayer : public ObjectLayer {
+class LLVM_ABI LazyObjectLinkingLayer : public ObjectLayer {
 public:
   LazyObjectLinkingLayer(ObjectLinkingLayer &BaseLayer,
                          LazyReexportsManager &LRMgr);
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
index 635f2b08367e..b8836a029325 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
 #include "llvm/ExecutionEngine/Orc/Speculation.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -41,16 +42,17 @@ class LazyCallThroughManager {
   using NotifyResolvedFunction =
       unique_function<Error(ExecutorAddr ResolvedAddr)>;
 
-  LazyCallThroughManager(ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr,
-                         TrampolinePool *TP);
+  LLVM_ABI LazyCallThroughManager(ExecutionSession &ES,
+                                  ExecutorAddr ErrorHandlerAddr,
+                                  TrampolinePool *TP);
 
   // Return a free call-through trampoline and bind it to look up and call
   // through to the given symbol.
-  Expected<ExecutorAddr>
+  LLVM_ABI Expected<ExecutorAddr>
   getCallThroughTrampoline(JITDylib &SourceJD, SymbolStringPtr SymbolName,
                            NotifyResolvedFunction NotifyResolved);
 
-  void resolveTrampolineLandingAddress(
+  LLVM_ABI void resolveTrampolineLandingAddress(
       ExecutorAddr TrampolineAddr,
       TrampolinePool::NotifyLandingResolvedFunction NotifyLandingResolved);
 
@@ -65,9 +67,10 @@ class LazyCallThroughManager {
     SymbolStringPtr SymbolName;
   };
 
-  ExecutorAddr reportCallThroughError(Error Err);
-  Expected<ReexportsEntry> findReexport(ExecutorAddr TrampolineAddr);
-  Error notifyResolved(ExecutorAddr TrampolineAddr, ExecutorAddr ResolvedAddr);
+  LLVM_ABI ExecutorAddr reportCallThroughError(Error Err);
+  LLVM_ABI Expected<ReexportsEntry> findReexport(ExecutorAddr TrampolineAddr);
+  LLVM_ABI Error notifyResolved(ExecutorAddr TrampolineAddr,
+                                ExecutorAddr ResolvedAddr);
   void setTrampolinePool(TrampolinePool &TP) { this->TP = &TP; }
 
 private:
@@ -129,7 +132,7 @@ class LocalLazyCallThroughManager : public LazyCallThroughManager {
 
 /// Create a LocalLazyCallThroughManager from the given triple and execution
 /// session.
-Expected<std::unique_ptr<LazyCallThroughManager>>
+LLVM_ABI Expected<std::unique_ptr<LazyCallThroughManager>>
 createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,
                                   ExecutorAddr ErrorHandlerAddr);
 
@@ -138,7 +141,7 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,
 /// Unlike a 'true' re-export, the address of the lazy re-export will not
 /// match the address of the re-exported symbol, but calling it will behave
 /// the same as calling the re-exported symbol.
-class LazyReexportsMaterializationUnit : public MaterializationUnit {
+class LLVM_ABI LazyReexportsMaterializationUnit : public MaterializationUnit {
 public:
   LazyReexportsMaterializationUnit(LazyCallThroughManager &LCTManager,
                                    RedirectableSymbolManager &RSManager,
@@ -173,7 +176,7 @@ lazyReexports(LazyCallThroughManager &LCTManager,
       LCTManager, RSManager, SourceJD, std::move(CallableAliases), SrcJDLoc);
 }
 
-class LazyReexportsManager : public ResourceManager {
+class LLVM_ABI LazyReexportsManager : public ResourceManager {
 
   friend std::unique_ptr<MaterializationUnit>
   lazyReexports(LazyReexportsManager &, SymbolAliasMap);
@@ -185,7 +188,7 @@ class LazyReexportsManager : public ResourceManager {
     SymbolStringPtr BodyName;
   };
 
-  class Listener {
+  class LLVM_ABI Listener {
   public:
     using CallThroughInfo = LazyReexportsManager::CallThroughInfo;
 
@@ -268,7 +271,8 @@ lazyReexports(LazyReexportsManager &LRM, SymbolAliasMap Reexports) {
   return LRM.createLazyReexports(std::move(Reexports));
 }
 
-class SimpleLazyReexportsSpeculator : public LazyReexportsManager::Listener {
+class LLVM_ABI SimpleLazyReexportsSpeculator
+    : public LazyReexportsManager::Listener {
 public:
   using RecordExecutionFunction =
       unique_function<void(const CallThroughInfo &CTI)>;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLayer.h
index 771a118efae2..f9d82a31e0bf 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLayer.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 
@@ -23,7 +24,7 @@
 
 namespace llvm::orc {
 
-class LinkGraphLayer {
+class LLVM_ABI LinkGraphLayer {
 public:
   LinkGraphLayer(ExecutionSession &ES) : ES(ES) {}
 
@@ -70,7 +71,7 @@ class LinkGraphLayer {
 };
 
 /// MaterializationUnit for wrapping LinkGraphs.
-class LinkGraphMaterializationUnit : public MaterializationUnit {
+class LLVM_ABI LinkGraphMaterializationUnit : public MaterializationUnit {
 public:
   LinkGraphMaterializationUnit(LinkGraphLayer &LGLayer,
                                std::unique_ptr<jitlink::LinkGraph> G,
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h
index 060fda57bd4f..b3d7500dd37e 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h
@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
 #include "llvm/ExecutionEngine/Orc/LinkGraphLayer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <algorithm>
 #include <cassert>
@@ -40,14 +41,15 @@ namespace orc {
 ///
 /// Clients can use this class to add LinkGraphs to an ExecutionSession, and it
 /// serves as a base for the ObjectLinkingLayer that can link object files.
-class LinkGraphLinkingLayer : public LinkGraphLayer, private ResourceManager {
+class LLVM_ABI LinkGraphLinkingLayer : public LinkGraphLayer,
+                                       private ResourceManager {
   class JITLinkCtx;
 
 public:
   /// Plugin instances can be added to the ObjectLinkingLayer to receive
   /// callbacks when code is loaded or emitted, and when JITLink is being
   /// configured.
-  class Plugin {
+  class LLVM_ABI Plugin {
   public:
     virtual ~Plugin();
     virtual void modifyPassConfig(MaterializationResponsibility &MR,
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LoadLinkableFile.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LoadLinkableFile.h
index 74a6f94e1257..7dfe7e96fcc7 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LoadLinkableFile.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LoadLinkableFile.h
@@ -18,6 +18,7 @@
 #ifndef LLVM_EXECUTIONENGINE_ORC_LOADLINKABLEFILE_H
 #define LLVM_EXECUTIONENGINE_ORC_LOADLINKABLEFILE_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/TargetParser/Triple.h"
@@ -54,7 +55,7 @@ enum class LoadArchives {
 ///
 /// If IdentifierOverride is provided then it will be used as the name of the
 /// resulting buffer, rather than Path.
-Expected<std::pair<std::unique_ptr<MemoryBuffer>, LinkableFileKind>>
+LLVM_ABI Expected<std::pair<std::unique_ptr<MemoryBuffer>, LinkableFileKind>>
 loadLinkableFile(StringRef Path, const Triple &TT, LoadArchives LA,
                  std::optional<StringRef> IdentifierOverride = std::nullopt);
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h
index a598405ee4f6..26a7801f0012 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 
 #include <vector>
 
@@ -41,7 +42,7 @@ namespace orc {
 /// addresses before using them).
 ///
 /// Asynchronous version.
-void lookupAndRecordAddrs(
+LLVM_ABI void lookupAndRecordAddrs(
     unique_function<void(Error)> OnRecorded, ExecutionSession &ES, LookupKind K,
     const JITDylibSearchOrder &SearchOrder,
     std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs,
@@ -50,7 +51,7 @@ void lookupAndRecordAddrs(
 /// Record addresses of the given symbols in the given ExecutorAddrs.
 ///
 /// Blocking version.
-Error lookupAndRecordAddrs(
+LLVM_ABI Error lookupAndRecordAddrs(
     ExecutionSession &ES, LookupKind K, const JITDylibSearchOrder &SearchOrder,
     std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs,
     SymbolLookupFlags LookupFlags = SymbolLookupFlags::RequiredSymbol);
@@ -59,7 +60,7 @@ Error lookupAndRecordAddrs(
 ///
 /// ExecutorProcessControl lookup version. Lookups are always implicitly
 /// weak.
-Error lookupAndRecordAddrs(
+LLVM_ABI Error lookupAndRecordAddrs(
     ExecutorProcessControl &EPC, tpctypes::DylibHandle H,
     std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs,
     SymbolLookupFlags LookupFlags = SymbolLookupFlags::RequiredSymbol);
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachO.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachO.h
index 81f889c4933d..a0342d8d75bc 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachO.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachO.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/Orc/LoadLinkableFile.h"
 #include "llvm/Object/Archive.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/TargetParser/Triple.h"
@@ -37,21 +38,21 @@ class ObjectLayer;
 /// given triple.
 /// ObjIsSlice should be set to true if Obj is a slice of a universal binary
 /// (that fact will then be reported in the error messages).
-Error checkMachORelocatableObject(MemoryBufferRef Obj, const Triple &TT,
-                                  bool ObjIsSlice);
+LLVM_ABI Error checkMachORelocatableObject(MemoryBufferRef Obj,
+                                           const Triple &TT, bool ObjIsSlice);
 
 /// Check that the given buffer contains a MachO object file compatible with the
 /// given triple.
 /// This convenience overload returns the buffer if it passes all checks,
 /// otherwise it returns an error.
-Expected<std::unique_ptr<MemoryBuffer>>
+LLVM_ABI Expected<std::unique_ptr<MemoryBuffer>>
 checkMachORelocatableObject(std::unique_ptr<MemoryBuffer> Obj, const Triple &TT,
                             bool ObjIsSlice);
 
 /// Load a relocatable object compatible with TT from Path.
 /// If Path is a universal binary, this function will return a buffer for the
 /// slice compatible with Triple (if one is present).
-Expected<std::pair<std::unique_ptr<MemoryBuffer>, LinkableFileKind>>
+LLVM_ABI Expected<std::pair<std::unique_ptr<MemoryBuffer>, LinkableFileKind>>
 loadMachOLinkableFile(
     StringRef Path, const Triple &TT, LoadArchives LA,
     std::optional<StringRef> IdentifierOverride = std::nullopt);
@@ -60,7 +61,7 @@ loadMachOLinkableFile(
 /// binary.
 /// Path is only used for error reporting. Identifier will be used to name the
 /// resulting buffer.
-Expected<std::pair<std::unique_ptr<MemoryBuffer>, LinkableFileKind>>
+LLVM_ABI Expected<std::pair<std::unique_ptr<MemoryBuffer>, LinkableFileKind>>
 loadLinkableSliceFromMachOUniversalBinary(sys::fs::file_t FD,
                                           std::unique_ptr<MemoryBuffer> UBBuf,
                                           const Triple &TT, LoadArchives LA,
@@ -69,12 +70,12 @@ loadLinkableSliceFromMachOUniversalBinary(sys::fs::file_t FD,
 
 /// Utility for identifying the file-slice compatible with TT in a universal
 /// binary.
-Expected<std::pair<size_t, size_t>>
+LLVM_ABI Expected<std::pair<size_t, size_t>>
 getMachOSliceRangeForTriple(object::MachOUniversalBinary &UB, const Triple &TT);
 
 /// Utility for identifying the file-slice compatible with TT in a universal
 /// binary.
-Expected<std::pair<size_t, size_t>>
+LLVM_ABI Expected<std::pair<size_t, size_t>>
 getMachOSliceRangeForTriple(MemoryBufferRef UBBuf, const Triple &TT);
 
 /// For use with StaticLibraryDefinitionGenerators.
@@ -83,8 +84,8 @@ class ForceLoadMachOArchiveMembers {
   ForceLoadMachOArchiveMembers(ObjectLayer &L, JITDylib &JD, bool ObjCOnly)
       : L(L), JD(JD), ObjCOnly(ObjCOnly) {}
 
-  Expected<bool> operator()(object::Archive &A, MemoryBufferRef MemberBuf,
-                            size_t Index);
+  LLVM_ABI Expected<bool> operator()(object::Archive &A,
+                                     MemoryBufferRef MemberBuf, size_t Index);
 
 private:
   ObjectLayer &L;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
index e764af267cdf..9c34bf1e7a99 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
@@ -18,6 +18,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 
 #include <future>
 #include <thread>
@@ -27,7 +28,7 @@ namespace llvm {
 namespace orc {
 
 /// Mediates between MachO initialization and ExecutionSession state.
-class MachOPlatform : public Platform {
+class LLVM_ABI MachOPlatform : public Platform {
 public:
   // Used internally by MachOPlatform, but made public to enable serialization.
   struct MachOJITDylibDepInfo {
@@ -68,7 +69,7 @@ class MachOPlatform : public Platform {
     struct BuildVersionOpts {
 
       // Derive platform from triple if possible.
-      static std::optional<BuildVersionOpts>
+      LLVM_ABI static std::optional<BuildVersionOpts>
       fromTriple(const Triple &TT, uint32_t MinOS, uint32_t SDK);
 
       uint32_t Platform; // Platform.
@@ -204,7 +205,7 @@ class MachOPlatform : public Platform {
   // The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO
   // platform features including initializers, exceptions, TLV, and language
   // runtime registration.
-  class MachOPlatformPlugin : public ObjectLinkingLayer::Plugin {
+  class LLVM_ABI MachOPlatformPlugin : public ObjectLinkingLayer::Plugin {
   public:
     MachOPlatformPlugin(MachOPlatform &MP) : MP(MP) {}
 
@@ -382,7 +383,7 @@ class MachOPlatform : public Platform {
 };
 
 // Generates a MachO header.
-class SimpleMachOHeaderMU : public MaterializationUnit {
+class LLVM_ABI SimpleMachOHeaderMU : public MaterializationUnit {
 public:
   SimpleMachOHeaderMU(MachOPlatform &MOP, SymbolStringPtr HeaderStartSymbol,
                       MachOPlatform::HeaderOptions Opts);
@@ -426,7 +427,7 @@ struct MachOHeaderInfo {
   uint32_t CPUType = 0;
   uint32_t CPUSubType = 0;
 };
-MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT);
+LLVM_ABI MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT);
 
 } // end namespace orc
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h
index 77429f4b11ee..bfae136e5203 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h
@@ -16,6 +16,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 namespace llvm {
@@ -25,8 +26,8 @@ namespace orc {
 /// ExecutionSession.
 class MangleAndInterner {
 public:
-  MangleAndInterner(ExecutionSession &ES, const DataLayout &DL);
-  SymbolStringPtr operator()(StringRef Name);
+  LLVM_ABI MangleAndInterner(ExecutionSession &ES, const DataLayout &DL);
+  LLVM_ABI SymbolStringPtr operator()(StringRef Name);
 
 private:
   ExecutionSession &ES;
@@ -50,9 +51,10 @@ class IRSymbolMapper {
   /// necessarily one-to-one: thread-local GlobalValues, for example, may
   /// produce more than one symbol, in which case the map will contain duplicate
   /// values.
-  static void add(ExecutionSession &ES, const ManglingOptions &MO,
-                  ArrayRef<GlobalValue *> GVs, SymbolFlagsMap &SymbolFlags,
-                  SymbolNameToDefinitionMap *SymbolToDefinition = nullptr);
+  LLVM_ABI static void
+  add(ExecutionSession &ES, const ManglingOptions &MO,
+      ArrayRef<GlobalValue *> GVs, SymbolFlagsMap &SymbolFlags,
+      SymbolNameToDefinitionMap *SymbolToDefinition = nullptr);
 };
 
 } // End namespace orc
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h
index b1897d557b12..7a752111b04d 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h
@@ -16,11 +16,13 @@
 #include "llvm/ADT/IntervalMap.h"
 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 
-class MapperJITLinkMemoryManager : public jitlink::JITLinkMemoryManager {
+class LLVM_ABI MapperJITLinkMemoryManager
+    : public jitlink::JITLinkMemoryManager {
 public:
   MapperJITLinkMemoryManager(size_t ReservationGranularity,
                              std::unique_ptr<MemoryMapper> Mapper);
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MaterializationUnit.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MaterializationUnit.h
index 4ac8f6b6ba05..f8ac76d88954 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MaterializationUnit.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MaterializationUnit.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ExecutionEngine/Orc/CoreContainers.h"
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::orc {
 
@@ -29,7 +30,7 @@ class MaterializationResponsibility;
 /// JITDylibs. The JITDylib will call materialize when the address of a symbol
 /// is requested via the lookup method. The JITDylib will call discard if a
 /// stronger definition is added or already present.
-class MaterializationUnit {
+class LLVM_ABI MaterializationUnit {
   friend class ExecutionSession;
   friend class JITDylib;
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
index 8815777f35e0..44ef289fc282 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Process.h"
 
 #include <mutex>
@@ -23,7 +24,7 @@ namespace llvm {
 namespace orc {
 
 /// Manages mapping, content transfer and protections for JIT memory
-class MemoryMapper {
+class LLVM_ABI MemoryMapper {
 public:
   /// Represents a single allocation containing multiple segments and
   /// initialization and deinitialization actions
@@ -79,7 +80,7 @@ class MemoryMapper {
   virtual ~MemoryMapper();
 };
 
-class InProcessMemoryMapper : public MemoryMapper {
+class LLVM_ABI InProcessMemoryMapper : public MemoryMapper {
 public:
   InProcessMemoryMapper(size_t PageSize);
 
@@ -121,7 +122,7 @@ class InProcessMemoryMapper : public MemoryMapper {
   size_t PageSize;
 };
 
-class SharedMemoryMapper final : public MemoryMapper {
+class LLVM_ABI SharedMemoryMapper final : public MemoryMapper {
 public:
   struct SymbolAddrs {
     ExecutorAddr Instance;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h
index 1bf09069163e..2fd9ae5611c9 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h
@@ -15,6 +15,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_OBJECTFILEINTERFACE_H
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 namespace llvm {
@@ -23,13 +24,13 @@ namespace orc {
 /// Adds an initializer symbol to the given MU interface.
 /// The init symbol's name is guaranteed to be unique within I, and will be of
 /// the form $.<ObjFileName>.__inits.<N>, where N is some integer.
-void addInitSymbol(MaterializationUnit::Interface &I, ExecutionSession &ES,
-                   StringRef ObjFileName);
+LLVM_ABI void addInitSymbol(MaterializationUnit::Interface &I,
+                            ExecutionSession &ES, StringRef ObjFileName);
 
 /// Returns a MaterializationUnit::Interface for the object file contained in
 /// the given buffer, or an error if the buffer does not contain a valid object
 /// file.
-Expected<MaterializationUnit::Interface>
+LLVM_ABI Expected<MaterializationUnit::Interface>
 getObjectFileInterface(ExecutionSession &ES, MemoryBufferRef ObjBuffer);
 
 } // End namespace orc
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h
index b392c5bf6714..52361ec22387 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h
@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
 #include "llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include <memory>
@@ -38,8 +39,9 @@ namespace orc {
 /// Clients can use this class to add relocatable object files to an
 /// ExecutionSession, and it typically serves as the base layer (underneath
 /// a compiling layer like IRCompileLayer) for the rest of the JIT.
-class ObjectLinkingLayer : public LinkGraphLinkingLayer,
-                           public RTTIExtends<ObjectLinkingLayer, ObjectLayer> {
+class LLVM_ABI ObjectLinkingLayer
+    : public LinkGraphLinkingLayer,
+      public RTTIExtends<ObjectLinkingLayer, ObjectLayer> {
 private:
   using BaseObjectLayer = RTTIExtends<ObjectLinkingLayer, ObjectLayer>;
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
index 9fc5fc0b902b..f1e3042873d6 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
@@ -15,13 +15,14 @@
 
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/Support/Compiler.h"
 #include <algorithm>
 #include <memory>
 
 namespace llvm {
 namespace orc {
 
-class ObjectTransformLayer
+class LLVM_ABI ObjectTransformLayer
     : public RTTIExtends<ObjectTransformLayer, ObjectLayer> {
 public:
   static char ID;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h
index 5d25a3e85464..16ab4fb56772 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h
@@ -18,6 +18,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_ORCABISUPPORT_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
@@ -101,27 +102,26 @@ class OrcAArch64 {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverWorkingMem,
-                                ExecutorAddr ResolverTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr RentryCtxAddr);
+  LLVM_ABI static void writeResolverCode(char *ResolverWorkingMem,
+                                         ExecutorAddr ResolverTargetAddress,
+                                         ExecutorAddr ReentryFnAddr,
+                                         ExecutorAddr RentryCtxAddr);
 
   /// Write the requested number of trampolines into the given memory,
   /// which must be big enough to hold 1 pointer, plus NumTrampolines
   /// trampolines.
-  static void writeTrampolines(char *TrampolineBlockWorkingMem,
-                               ExecutorAddr TrampolineBlockTargetAddress,
-                               ExecutorAddr ResolverAddr,
-                               unsigned NumTrampolines);
+  LLVM_ABI static void
+  writeTrampolines(char *TrampolineBlockWorkingMem,
+                   ExecutorAddr TrampolineBlockTargetAddress,
+                   ExecutorAddr ResolverAddr, unsigned NumTrampolines);
 
   /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem.
   /// Stubs will be written as if linked at StubsBlockTargetAddress, with the
   /// Nth stub using the Nth pointer in memory starting at
   /// PointersBlockTargetAddress.
-  static void writeIndirectStubsBlock(char *StubsBlockWorkingMem,
-                                      ExecutorAddr StubsBlockTargetAddress,
-                                      ExecutorAddr PointersBlockTargetAddress,
-                                      unsigned MinStubs);
+  LLVM_ABI static void writeIndirectStubsBlock(
+      char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress,
+      ExecutorAddr PointersBlockTargetAddress, unsigned MinStubs);
 };
 
 /// X86_64 code that's common to all ABIs.
@@ -137,19 +137,18 @@ class OrcX86_64_Base {
   /// Write the requested number of trampolines into the given memory,
   /// which must be big enough to hold 1 pointer, plus NumTrampolines
   /// trampolines.
-  static void writeTrampolines(char *TrampolineBlockWorkingMem,
-                               ExecutorAddr TrampolineBlockTargetAddress,
-                               ExecutorAddr ResolverAddr,
-                               unsigned NumTrampolines);
+  LLVM_ABI static void
+  writeTrampolines(char *TrampolineBlockWorkingMem,
+                   ExecutorAddr TrampolineBlockTargetAddress,
+                   ExecutorAddr ResolverAddr, unsigned NumTrampolines);
 
   /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem.
   /// Stubs will be written as if linked at StubsBlockTargetAddress, with the
   /// Nth stub using the Nth pointer in memory starting at
   /// PointersBlockTargetAddress.
-  static void writeIndirectStubsBlock(char *StubsBlockWorkingMem,
-                                      ExecutorAddr StubsBlockTargetAddress,
-                                      ExecutorAddr PointersBlockTargetAddress,
-                                      unsigned NumStubs);
+  LLVM_ABI static void writeIndirectStubsBlock(
+      char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress,
+      ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs);
 };
 
 /// X86_64 support for SysV ABI (Linux, MacOSX).
@@ -166,10 +165,10 @@ class OrcX86_64_SysV : public OrcX86_64_Base {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverWorkingMem,
-                                ExecutorAddr ResolverTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr ReentryCtxAddr);
+  LLVM_ABI static void writeResolverCode(char *ResolverWorkingMem,
+                                         ExecutorAddr ResolverTargetAddress,
+                                         ExecutorAddr ReentryFnAddr,
+                                         ExecutorAddr ReentryCtxAddr);
 };
 
 /// X86_64 support for Win32.
@@ -186,10 +185,10 @@ class OrcX86_64_Win32 : public OrcX86_64_Base {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverWorkingMem,
-                                ExecutorAddr ResolverTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr ReentryCtxAddr);
+  LLVM_ABI static void writeResolverCode(char *ResolverWorkingMem,
+                                         ExecutorAddr ResolverTargetAddress,
+                                         ExecutorAddr ReentryFnAddr,
+                                         ExecutorAddr ReentryCtxAddr);
 };
 
 /// I386 support.
@@ -210,27 +209,26 @@ class OrcI386 {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverWorkingMem,
-                                ExecutorAddr ResolverTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr ReentryCtxAddr);
+  LLVM_ABI static void writeResolverCode(char *ResolverWorkingMem,
+                                         ExecutorAddr ResolverTargetAddress,
+                                         ExecutorAddr ReentryFnAddr,
+                                         ExecutorAddr ReentryCtxAddr);
 
   /// Write the requested number of trampolines into the given memory,
   /// which must be big enough to hold 1 pointer, plus NumTrampolines
   /// trampolines.
-  static void writeTrampolines(char *TrampolineBlockWorkingMem,
-                               ExecutorAddr TrampolineBlockTargetAddress,
-                               ExecutorAddr ResolverAddr,
-                               unsigned NumTrampolines);
+  LLVM_ABI static void
+  writeTrampolines(char *TrampolineBlockWorkingMem,
+                   ExecutorAddr TrampolineBlockTargetAddress,
+                   ExecutorAddr ResolverAddr, unsigned NumTrampolines);
 
   /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem.
   /// Stubs will be written as if linked at StubsBlockTargetAddress, with the
   /// Nth stub using the Nth pointer in memory starting at
   /// PointersBlockTargetAddress.
-  static void writeIndirectStubsBlock(char *StubsBlockWorkingMem,
-                                      ExecutorAddr StubsBlockTargetAddress,
-                                      ExecutorAddr PointersBlockTargetAddress,
-                                      unsigned NumStubs);
+  LLVM_ABI static void writeIndirectStubsBlock(
+      char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress,
+      ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs);
 };
 
 // @brief Mips32 support.
@@ -247,10 +245,10 @@ class OrcMips32_Base {
   /// Write the requested number of trampolines into the given memory,
   /// which must be big enough to hold 1 pointer, plus NumTrampolines
   /// trampolines.
-  static void writeTrampolines(char *TrampolineBlockWorkingMem,
-                               ExecutorAddr TrampolineBlockTargetAddress,
-                               ExecutorAddr ResolverAddr,
-                               unsigned NumTrampolines);
+  LLVM_ABI static void
+  writeTrampolines(char *TrampolineBlockWorkingMem,
+                   ExecutorAddr TrampolineBlockTargetAddress,
+                   ExecutorAddr ResolverAddr, unsigned NumTrampolines);
 
   /// Write the resolver code into the given memory. The user is
   /// responsible for allocating the memory and setting permissions.
@@ -259,18 +257,18 @@ class OrcMips32_Base {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverBlockWorkingMem,
-                                ExecutorAddr ResolverBlockTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr ReentryCtxAddr, bool isBigEndian);
+  LLVM_ABI static void
+  writeResolverCode(char *ResolverBlockWorkingMem,
+                    ExecutorAddr ResolverBlockTargetAddress,
+                    ExecutorAddr ReentryFnAddr, ExecutorAddr ReentryCtxAddr,
+                    bool isBigEndian);
   /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem.
   /// Stubs will be written as if linked at StubsBlockTargetAddress, with the
   /// Nth stub using the Nth pointer in memory starting at
   /// PointersBlockTargetAddress.
-  static void writeIndirectStubsBlock(char *StubsBlockWorkingMem,
-                                      ExecutorAddr StubsBlockTargetAddress,
-                                      ExecutorAddr PointersBlockTargetAddress,
-                                      unsigned NumStubs);
+  LLVM_ABI static void writeIndirectStubsBlock(
+      char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress,
+      ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs);
 };
 
 class OrcMips32Le : public OrcMips32_Base {
@@ -313,26 +311,25 @@ class OrcMips64 {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverWorkingMem,
-                                ExecutorAddr ResolverTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr ReentryCtxAddr);
+  LLVM_ABI static void writeResolverCode(char *ResolverWorkingMem,
+                                         ExecutorAddr ResolverTargetAddress,
+                                         ExecutorAddr ReentryFnAddr,
+                                         ExecutorAddr ReentryCtxAddr);
 
   /// Write the requested number of trampolines into the given memory,
   /// which must be big enough to hold 1 pointer, plus NumTrampolines
   /// trampolines.
-  static void writeTrampolines(char *TrampolineBlockWorkingMem,
-                               ExecutorAddr TrampolineBlockTargetAddress,
-                               ExecutorAddr ResolverFnAddr,
-                               unsigned NumTrampolines);
+  LLVM_ABI static void
+  writeTrampolines(char *TrampolineBlockWorkingMem,
+                   ExecutorAddr TrampolineBlockTargetAddress,
+                   ExecutorAddr ResolverFnAddr, unsigned NumTrampolines);
   /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem.
   /// Stubs will be written as if linked at StubsBlockTargetAddress, with the
   /// Nth stub using the Nth pointer in memory starting at
   /// PointersBlockTargetAddress.
-  static void writeIndirectStubsBlock(char *StubsBlockWorkingMem,
-                                      ExecutorAddr StubsBlockTargetAddress,
-                                      ExecutorAddr PointersBlockTargetAddress,
-                                      unsigned NumStubs);
+  LLVM_ABI static void writeIndirectStubsBlock(
+      char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress,
+      ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs);
 };
 
 // @brief riscv64 support.
@@ -353,26 +350,25 @@ class OrcRiscv64 {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverWorkingMem,
-                                ExecutorAddr ResolverTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr ReentryCtxAddr);
+  LLVM_ABI static void writeResolverCode(char *ResolverWorkingMem,
+                                         ExecutorAddr ResolverTargetAddress,
+                                         ExecutorAddr ReentryFnAddr,
+                                         ExecutorAddr ReentryCtxAddr);
 
   /// Write the requested number of trampolines into the given memory,
   /// which must be big enough to hold 1 pointer, plus NumTrampolines
   /// trampolines.
-  static void writeTrampolines(char *TrampolineBlockWorkingMem,
-                               ExecutorAddr TrampolineBlockTargetAddress,
-                               ExecutorAddr ResolverFnAddr,
-                               unsigned NumTrampolines);
+  LLVM_ABI static void
+  writeTrampolines(char *TrampolineBlockWorkingMem,
+                   ExecutorAddr TrampolineBlockTargetAddress,
+                   ExecutorAddr ResolverFnAddr, unsigned NumTrampolines);
   /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem.
   /// Stubs will be written as if linked at StubsBlockTargetAddress, with the
   /// Nth stub using the Nth pointer in memory starting at
   /// PointersBlockTargetAddress.
-  static void writeIndirectStubsBlock(char *StubsBlockWorkingMem,
-                                      ExecutorAddr StubsBlockTargetAddress,
-                                      ExecutorAddr PointersBlockTargetAddress,
-                                      unsigned NumStubs);
+  LLVM_ABI static void writeIndirectStubsBlock(
+      char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress,
+      ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs);
 };
 
 // @brief loongarch64 support.
@@ -393,27 +389,26 @@ class OrcLoongArch64 {
   /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr
   /// argument of writeResolverCode will be passed as the second argument to
   /// the function at ReentryFnAddr.
-  static void writeResolverCode(char *ResolverWorkingMem,
-                                ExecutorAddr ResolverTargetAddress,
-                                ExecutorAddr ReentryFnAddr,
-                                ExecutorAddr ReentryCtxAddr);
+  LLVM_ABI static void writeResolverCode(char *ResolverWorkingMem,
+                                         ExecutorAddr ResolverTargetAddress,
+                                         ExecutorAddr ReentryFnAddr,
+                                         ExecutorAddr ReentryCtxAddr);
 
   /// Write the requested number of trampolines into the given memory,
   /// which must be big enough to hold 1 pointer, plus NumTrampolines
   /// trampolines.
-  static void writeTrampolines(char *TrampolineBlockWorkingMem,
-                               ExecutorAddr TrampolineBlockTargetAddress,
-                               ExecutorAddr ResolverFnAddr,
-                               unsigned NumTrampolines);
+  LLVM_ABI static void
+  writeTrampolines(char *TrampolineBlockWorkingMem,
+                   ExecutorAddr TrampolineBlockTargetAddress,
+                   ExecutorAddr ResolverFnAddr, unsigned NumTrampolines);
 
   /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem.
   /// Stubs will be written as if linked at StubsBlockTargetAddress, with the
   /// Nth stub using the Nth pointer in memory starting at
   /// PointersBlockTargetAddress.
-  static void writeIndirectStubsBlock(char *StubsBlockWorkingMem,
-                                      ExecutorAddr StubsBlockTargetAddress,
-                                      ExecutorAddr PointersBlockTargetAddress,
-                                      unsigned NumStubs);
+  LLVM_ABI static void writeIndirectStubsBlock(
+      char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress,
+      ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs);
 };
 
 } // end namespace orc
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
index b45e287c2b1a..1fb472a177d6 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/Layer.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <algorithm>
 #include <cassert>
@@ -33,7 +34,7 @@
 namespace llvm {
 namespace orc {
 
-class RTDyldObjectLinkingLayer
+class LLVM_ABI RTDyldObjectLinkingLayer
     : public RTTIExtends<RTDyldObjectLinkingLayer, ObjectLayer>,
       private ResourceManager {
 public:
@@ -49,7 +50,8 @@ class RTDyldObjectLinkingLayer
       MaterializationResponsibility &R, std::unique_ptr<MemoryBuffer>)>;
 
   using GetMemoryManagerFunction =
-      unique_function<std::unique_ptr<RuntimeDyld::MemoryManager>()>;
+      unique_function<std::unique_ptr<RuntimeDyld::MemoryManager>(
+          const MemoryBuffer &)>;
 
   /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
   ///        and NotifyEmitted functors.
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
index cd185d54b2e7..a059223820ca 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
@@ -18,13 +18,14 @@
 #include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 
 namespace llvm {
 namespace orc {
 
-class ReOptimizeLayer : public IRLayer, public ResourceManager {
+class LLVM_ABI ReOptimizeLayer : public IRLayer, public ResourceManager {
 public:
   using ReOptMaterializationUnitID = uint64_t;
 
@@ -123,9 +124,9 @@ class ReOptimizeLayer : public IRLayer, public ResourceManager {
       return CurVersion;
     }
 
-    bool tryStartReoptimize();
-    void reoptimizeSucceeded();
-    void reoptimizeFailed();
+    LLVM_ABI bool tryStartReoptimize();
+    LLVM_ABI void reoptimizeSucceeded();
+    LLVM_ABI void reoptimizeFailed();
 
   private:
     std::mutex Mutex;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
index f3d4c7693877..9064c6af8fdc 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
@@ -14,13 +14,14 @@
 #define LLVM_EXECUTIONENGINE_ORC_REDIRECTIONMANAGER_H
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 
 /// Base class for performing redirection of call to symbol to another symbol in
 /// runtime.
-class RedirectionManager {
+class LLVM_ABI RedirectionManager {
 public:
   virtual ~RedirectionManager() = default;
 
@@ -45,7 +46,8 @@ class RedirectableSymbolManager : public RedirectionManager {
 public:
   /// Create redirectable symbols with given symbol names and initial
   /// desitnation symbol addresses.
-  Error createRedirectableSymbols(ResourceTrackerSP RT, SymbolMap InitialDests);
+  LLVM_ABI Error createRedirectableSymbols(ResourceTrackerSP RT,
+                                           SymbolMap InitialDests);
 
   /// Create a single redirectable symbol with given symbol name and initial
   /// desitnation symbol address.
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SectCreate.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SectCreate.h
index c4254249af69..e6384eb4b6d2 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SectCreate.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SectCreate.h
@@ -15,13 +15,14 @@
 
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/Support/Compiler.h"
 
 #include <utility>
 #include <vector>
 
 namespace llvm::orc {
 
-class SectCreateMaterializationUnit : public MaterializationUnit {
+class LLVM_ABI SectCreateMaterializationUnit : public MaterializationUnit {
 public:
   struct ExtraSymbolInfo {
     JITSymbolFlags Flags;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h
index 1aa87cc41ab6..596cc18208f0 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Memory.h"
 
 #include <vector>
@@ -66,16 +67,16 @@ using OnRunFinalizeActionsCompleteFn =
 /// be returned. The dealloc actions should be run by calling
 /// runDeallocationActions. If this function succeeds then the AA argument will
 /// be cleared before the function returns.
-void runFinalizeActions(AllocActions &AAs,
-                        OnRunFinalizeActionsCompleteFn OnComplete);
+LLVM_ABI void runFinalizeActions(AllocActions &AAs,
+                                 OnRunFinalizeActionsCompleteFn OnComplete);
 
 using OnRunDeallocActionsComeleteFn = unique_function<void(Error)>;
 
 /// Run deallocation actions.
 /// Dealloc actions will be run in reverse order (from last element of DAs to
 /// first).
-void runDeallocActions(ArrayRef<WrapperFunctionCall> DAs,
-                       OnRunDeallocActionsComeleteFn OnComplete);
+LLVM_ABI void runDeallocActions(ArrayRef<WrapperFunctionCall> DAs,
+                                OnRunDeallocActionsComeleteFn OnComplete);
 
 using SPSAllocActionCallPair =
     SPSTuple<SPSWrapperFunctionCall, SPSWrapperFunctionCall>;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
index 8eb8c1b4cc89..5f09752664a3 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_SHARED_MACHOOBJECTFORMAT_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
@@ -22,43 +23,43 @@ namespace orc {
 
 // MachO section names.
 
-extern StringRef MachODataCommonSectionName;
-extern StringRef MachODataDataSectionName;
-extern StringRef MachOEHFrameSectionName;
-extern StringRef MachOCompactUnwindSectionName;
-extern StringRef MachOCStringSectionName;
-extern StringRef MachOModInitFuncSectionName;
-extern StringRef MachOObjCCatListSectionName;
-extern StringRef MachOObjCCatList2SectionName;
-extern StringRef MachOObjCClassListSectionName;
-extern StringRef MachOObjCClassNameSectionName;
-extern StringRef MachOObjCClassRefsSectionName;
-extern StringRef MachOObjCConstSectionName;
-extern StringRef MachOObjCDataSectionName;
-extern StringRef MachOObjCImageInfoSectionName;
-extern StringRef MachOObjCMethNameSectionName;
-extern StringRef MachOObjCMethTypeSectionName;
-extern StringRef MachOObjCNLCatListSectionName;
-extern StringRef MachOObjCNLClassListSectionName;
-extern StringRef MachOObjCProtoListSectionName;
-extern StringRef MachOObjCProtoRefsSectionName;
-extern StringRef MachOObjCSelRefsSectionName;
-extern StringRef MachOSwift5ProtoSectionName;
-extern StringRef MachOSwift5ProtosSectionName;
-extern StringRef MachOSwift5TypesSectionName;
-extern StringRef MachOSwift5TypeRefSectionName;
-extern StringRef MachOSwift5FieldMetadataSectionName;
-extern StringRef MachOSwift5EntrySectionName;
-extern StringRef MachOTextTextSectionName;
-extern StringRef MachOThreadBSSSectionName;
-extern StringRef MachOThreadDataSectionName;
-extern StringRef MachOThreadVarsSectionName;
-extern StringRef MachOUnwindInfoSectionName;
+LLVM_ABI extern StringRef MachODataCommonSectionName;
+LLVM_ABI extern StringRef MachODataDataSectionName;
+LLVM_ABI extern StringRef MachOEHFrameSectionName;
+LLVM_ABI extern StringRef MachOCompactUnwindSectionName;
+LLVM_ABI extern StringRef MachOCStringSectionName;
+LLVM_ABI extern StringRef MachOModInitFuncSectionName;
+LLVM_ABI extern StringRef MachOObjCCatListSectionName;
+LLVM_ABI extern StringRef MachOObjCCatList2SectionName;
+LLVM_ABI extern StringRef MachOObjCClassListSectionName;
+LLVM_ABI extern StringRef MachOObjCClassNameSectionName;
+LLVM_ABI extern StringRef MachOObjCClassRefsSectionName;
+LLVM_ABI extern StringRef MachOObjCConstSectionName;
+LLVM_ABI extern StringRef MachOObjCDataSectionName;
+LLVM_ABI extern StringRef MachOObjCImageInfoSectionName;
+LLVM_ABI extern StringRef MachOObjCMethNameSectionName;
+LLVM_ABI extern StringRef MachOObjCMethTypeSectionName;
+LLVM_ABI extern StringRef MachOObjCNLCatListSectionName;
+LLVM_ABI extern StringRef MachOObjCNLClassListSectionName;
+LLVM_ABI extern StringRef MachOObjCProtoListSectionName;
+LLVM_ABI extern StringRef MachOObjCProtoRefsSectionName;
+LLVM_ABI extern StringRef MachOObjCSelRefsSectionName;
+LLVM_ABI extern StringRef MachOSwift5ProtoSectionName;
+LLVM_ABI extern StringRef MachOSwift5ProtosSectionName;
+LLVM_ABI extern StringRef MachOSwift5TypesSectionName;
+LLVM_ABI extern StringRef MachOSwift5TypeRefSectionName;
+LLVM_ABI extern StringRef MachOSwift5FieldMetadataSectionName;
+LLVM_ABI extern StringRef MachOSwift5EntrySectionName;
+LLVM_ABI extern StringRef MachOTextTextSectionName;
+LLVM_ABI extern StringRef MachOThreadBSSSectionName;
+LLVM_ABI extern StringRef MachOThreadDataSectionName;
+LLVM_ABI extern StringRef MachOThreadVarsSectionName;
+LLVM_ABI extern StringRef MachOUnwindInfoSectionName;
 
-extern StringRef MachOInitSectionNames[22];
+LLVM_ABI extern StringRef MachOInitSectionNames[22];
 
-bool isMachOInitializerSection(StringRef SegName, StringRef SecName);
-bool isMachOInitializerSection(StringRef QualifiedName);
+LLVM_ABI bool isMachOInitializerSection(StringRef SegName, StringRef SecName);
+LLVM_ABI bool isMachOInitializerSection(StringRef QualifiedName);
 
 } // end namespace orc
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h
index 2773758970b5..01c44709a7ce 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h
@@ -15,28 +15,29 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 
 // ELF section names.
-extern StringRef ELFEHFrameSectionName;
+LLVM_ABI extern StringRef ELFEHFrameSectionName;
 
-extern StringRef ELFInitArrayFuncSectionName;
-extern StringRef ELFInitFuncSectionName;
-extern StringRef ELFFiniArrayFuncSectionName;
-extern StringRef ELFFiniFuncSectionName;
-extern StringRef ELFCtorArrayFuncSectionName;
-extern StringRef ELFDtorArrayFuncSectionName;
+LLVM_ABI extern StringRef ELFInitArrayFuncSectionName;
+LLVM_ABI extern StringRef ELFInitFuncSectionName;
+LLVM_ABI extern StringRef ELFFiniArrayFuncSectionName;
+LLVM_ABI extern StringRef ELFFiniFuncSectionName;
+LLVM_ABI extern StringRef ELFCtorArrayFuncSectionName;
+LLVM_ABI extern StringRef ELFDtorArrayFuncSectionName;
 
-extern StringRef ELFInitSectionNames[3];
+LLVM_ABI extern StringRef ELFInitSectionNames[3];
 
-extern StringRef ELFThreadBSSSectionName;
-extern StringRef ELFThreadDataSectionName;
+LLVM_ABI extern StringRef ELFThreadBSSSectionName;
+LLVM_ABI extern StringRef ELFThreadDataSectionName;
 
-bool isELFInitializerSection(StringRef SecName);
+LLVM_ABI bool isELFInitializerSection(StringRef SecName);
 
-bool isCOFFInitializerSection(StringRef Name);
+LLVM_ABI bool isCOFFInitializerSection(StringRef Name);
 
 } // end namespace orc
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcError.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcError.h
index d42fc3e2ef49..8c8b9c7ccc38 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcError.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcError.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_ORCERROR_H
 #define LLVM_EXECUTIONENGINE_ORC_SHARED_ORCERROR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
 #include <string>
@@ -42,9 +43,9 @@ enum class OrcErrorCode : int {
   UnexpectedSymbolDefinitions,
 };
 
-std::error_code orcError(OrcErrorCode ErrCode);
+LLVM_ABI std::error_code orcError(OrcErrorCode ErrCode);
 
-class DuplicateDefinition : public ErrorInfo<DuplicateDefinition> {
+class LLVM_ABI DuplicateDefinition : public ErrorInfo<DuplicateDefinition> {
 public:
   static char ID;
 
@@ -60,7 +61,7 @@ class DuplicateDefinition : public ErrorInfo<DuplicateDefinition> {
   std::optional<std::string> Context;
 };
 
-class JITSymbolNotFound : public ErrorInfo<JITSymbolNotFound> {
+class LLVM_ABI JITSymbolNotFound : public ErrorInfo<JITSymbolNotFound> {
 public:
   static char ID;
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
index 927939e63bbf..017ef70469e2 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
@@ -17,39 +17,42 @@
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 namespace rt {
 
-extern const char *SimpleExecutorDylibManagerInstanceName;
-extern const char *SimpleExecutorDylibManagerOpenWrapperName;
-extern const char *SimpleExecutorDylibManagerLookupWrapperName;
-
-extern const char *SimpleExecutorMemoryManagerInstanceName;
-extern const char *SimpleExecutorMemoryManagerReserveWrapperName;
-extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName;
-extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName;
-
-extern const char *ExecutorSharedMemoryMapperServiceInstanceName;
-extern const char *ExecutorSharedMemoryMapperServiceReserveWrapperName;
-extern const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName;
-extern const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName;
-extern const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName;
-
-extern const char *MemoryWriteUInt8sWrapperName;
-extern const char *MemoryWriteUInt16sWrapperName;
-extern const char *MemoryWriteUInt32sWrapperName;
-extern const char *MemoryWriteUInt64sWrapperName;
-extern const char *MemoryWriteBuffersWrapperName;
-extern const char *MemoryWritePointersWrapperName;
-
-extern const char *RegisterEHFrameSectionAllocActionName;
-extern const char *DeregisterEHFrameSectionAllocActionName;
-
-extern const char *RunAsMainWrapperName;
-extern const char *RunAsVoidFunctionWrapperName;
-extern const char *RunAsIntFunctionWrapperName;
+LLVM_ABI extern const char *SimpleExecutorDylibManagerInstanceName;
+LLVM_ABI extern const char *SimpleExecutorDylibManagerOpenWrapperName;
+LLVM_ABI extern const char *SimpleExecutorDylibManagerLookupWrapperName;
+
+LLVM_ABI extern const char *SimpleExecutorMemoryManagerInstanceName;
+LLVM_ABI extern const char *SimpleExecutorMemoryManagerReserveWrapperName;
+LLVM_ABI extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName;
+LLVM_ABI extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName;
+
+LLVM_ABI extern const char *ExecutorSharedMemoryMapperServiceInstanceName;
+LLVM_ABI extern const char *ExecutorSharedMemoryMapperServiceReserveWrapperName;
+LLVM_ABI extern const char
+    *ExecutorSharedMemoryMapperServiceInitializeWrapperName;
+LLVM_ABI extern const char
+    *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName;
+LLVM_ABI extern const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName;
+
+LLVM_ABI extern const char *MemoryWriteUInt8sWrapperName;
+LLVM_ABI extern const char *MemoryWriteUInt16sWrapperName;
+LLVM_ABI extern const char *MemoryWriteUInt32sWrapperName;
+LLVM_ABI extern const char *MemoryWriteUInt64sWrapperName;
+LLVM_ABI extern const char *MemoryWriteBuffersWrapperName;
+LLVM_ABI extern const char *MemoryWritePointersWrapperName;
+
+LLVM_ABI extern const char *RegisterEHFrameSectionAllocActionName;
+LLVM_ABI extern const char *DeregisterEHFrameSectionAllocActionName;
+
+LLVM_ABI extern const char *RunAsMainWrapperName;
+LLVM_ABI extern const char *RunAsVoidFunctionWrapperName;
+LLVM_ABI extern const char *RunAsIntFunctionWrapperName;
 
 using SPSSimpleExecutorDylibManagerOpenSignature =
     shared::SPSExpected<shared::SPSExecutorAddr>(shared::SPSExecutorAddr,
@@ -90,8 +93,8 @@ using SPSRunAsIntFunctionSignature = int32_t(shared::SPSExecutorAddr, int32_t);
 } // end namespace rt
 
 namespace rt_alt {
-extern const char *UnwindInfoManagerRegisterActionName;
-extern const char *UnwindInfoManagerDeregisterActionName;
+LLVM_ABI extern const char *UnwindInfoManagerRegisterActionName;
+LLVM_ABI extern const char *UnwindInfoManagerDeregisterActionName;
 } // end namespace rt_alt
 } // end namespace orc
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/PerfSharedStructs.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/PerfSharedStructs.h
index f4788bcebc3c..e3f9f78b9e1d 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/PerfSharedStructs.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/PerfSharedStructs.h
@@ -230,4 +230,4 @@ class SPSSerializationTraits<SPSPerfJITRecordBatch, PerfJITRecordBatch> {
 
 } // namespace llvm
 
-#endif
\ No newline at end of file
+#endif
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h
index ee3919c73340..14f16838a015 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include <atomic>
@@ -30,8 +31,8 @@ namespace llvm {
 namespace orc {
 
 namespace SimpleRemoteEPCDefaultBootstrapSymbolNames {
-extern const char *ExecutorSessionObjectName;
-extern const char *DispatchFnName;
+LLVM_ABI extern const char *ExecutorSessionObjectName;
+LLVM_ABI extern const char *DispatchFnName;
 } // end namespace SimpleRemoteEPCDefaultBootstrapSymbolNames
 
 enum class SimpleRemoteEPCOpcode : uint8_t {
@@ -51,7 +52,7 @@ struct SimpleRemoteEPCExecutorInfo {
 
 using SimpleRemoteEPCArgBytesVector = SmallVector<char, 128>;
 
-class SimpleRemoteEPCTransportClient {
+class LLVM_ABI SimpleRemoteEPCTransportClient {
 public:
   enum HandleMessageAction { ContinueSession, EndSession };
 
@@ -75,7 +76,7 @@ class SimpleRemoteEPCTransportClient {
   virtual void handleDisconnect(Error Err) = 0;
 };
 
-class SimpleRemoteEPCTransport {
+class LLVM_ABI SimpleRemoteEPCTransport {
 public:
   virtual ~SimpleRemoteEPCTransport();
 
@@ -100,7 +101,7 @@ class SimpleRemoteEPCTransport {
 };
 
 /// Uses read/write on FileDescriptors for transport.
-class FDSimpleRemoteEPCTransport : public SimpleRemoteEPCTransport {
+class LLVM_ABI FDSimpleRemoteEPCTransport : public SimpleRemoteEPCTransport {
 public:
   /// Create a FDSimpleRemoteEPCTransport using the given FDs for
   /// reading (InFD) and writing (OutFD).
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h
index 195cf80a1f0f..7acb6a4db08c 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h
@@ -20,6 +20,7 @@
 #include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h"
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
 
@@ -28,9 +29,9 @@
 namespace llvm {
 namespace orc {
 
-class SimpleRemoteEPC : public ExecutorProcessControl,
-                        public SimpleRemoteEPCTransportClient,
-                        private DylibManager {
+class LLVM_ABI SimpleRemoteEPC : public ExecutorProcessControl,
+                                 public SimpleRemoteEPCTransportClient,
+                                 private DylibManager {
 public:
   /// A setup object containing callbacks to construct a memory manager and
   /// memory access object. Both are optional. If not specified,
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h
index 88c90f54acbd..ef0fed4f4155 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h
@@ -17,6 +17,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include <mutex>
 #include <type_traits>
@@ -38,7 +39,7 @@ class ImplSymbolMap {
   using AliaseeDetails = std::pair<SymbolStringPtr, JITDylib *>;
   using Alias = SymbolStringPtr;
   using ImapTy = DenseMap<Alias, AliaseeDetails>;
-  void trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD);
+  LLVM_ABI void trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD);
 
 private:
   // FIX ME: find a right way to distinguish the pre-compile Symbols, and update
@@ -129,7 +130,7 @@ class Speculator {
   /// Define symbols for this Speculator object (__orc_speculator) and the
   /// speculation runtime entry point symbol (__orc_speculate_for) in the
   /// given JITDylib.
-  Error addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle);
+  LLVM_ABI Error addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle);
 
   // Speculatively compile likely functions for the given Stub Address.
   // destination of __orc_speculate_for jump
@@ -168,7 +169,7 @@ class Speculator {
   StubAddrLikelies GlobalSpecMap;
 };
 
-class IRSpeculationLayer : public IRLayer {
+class LLVM_ABI IRSpeculationLayer : public IRLayer {
 public:
   using IRlikiesStrRef =
       std::optional<DenseMap<StringRef, DenseSet<StringRef>>>;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
index 85f08b53f74a..d92c029ff84d 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Compiler.h"
 #include <atomic>
 #include <mutex>
 
@@ -35,7 +36,8 @@ class SymbolStringPool {
   friend class SymbolStringPoolEntryUnsafe;
 
   // Implemented in DebugUtils.h.
-  friend raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPool &SSP);
+  LLVM_ABI_FRIEND friend raw_ostream &operator<<(raw_ostream &OS,
+                                                 const SymbolStringPool &SSP);
 
 public:
   /// Destroy a SymbolStringPool.
@@ -92,8 +94,8 @@ class SymbolStringPtrBase {
     return LHS.S < RHS.S;
   }
 
-  friend raw_ostream &operator<<(raw_ostream &OS,
-                                 const SymbolStringPtrBase &Sym);
+  LLVM_ABI_FRIEND friend raw_ostream &
+  operator<<(raw_ostream &OS, const SymbolStringPtrBase &Sym);
 
 #ifndef NDEBUG
   // Returns true if the pool entry's ref count is above zero (or if the entry
@@ -313,6 +315,9 @@ SymbolStringPool::getRefCount(const SymbolStringPtrBase &S) const {
   return S.getRefCount();
 }
 
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const SymbolStringPtrBase &Sym);
+
 } // end namespace orc
 
 template <>
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h
index d3277e61eeb7..c34daf86ca70 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h
@@ -15,11 +15,12 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 #include <vector>
 
 namespace llvm::orc {
 
-void addDefaultBootstrapValuesForHostProcess(
+LLVM_ABI void addDefaultBootstrapValuesForHostProcess(
     StringMap<std::vector<char>> &BootstrapMap,
     StringMap<ExecutorAddr> &BootstrapSymbols);
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h
index 32c127634b25..645ef091571a 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h
@@ -17,11 +17,12 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace orc {
 
-class ExecutorBootstrapService {
+class LLVM_ABI ExecutorBootstrapService {
 public:
   virtual ~ExecutorBootstrapService();
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h
index 4e87057feb9f..85c2d655837b 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h
@@ -13,6 +13,7 @@
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h"
+#include "llvm/Support/Compiler.h"
 
 #include <atomic>
 #include <mutex>
@@ -25,7 +26,7 @@ namespace llvm {
 namespace orc {
 namespace rt_bootstrap {
 
-class ExecutorSharedMemoryMapperService final
+class LLVM_ABI ExecutorSharedMemoryMapperService final
     : public ExecutorBootstrapService {
 public:
   ~ExecutorSharedMemoryMapperService(){};
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h
index fa74c835d64b..1aa187754ffa 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h
@@ -26,4 +26,4 @@ llvm_orc_registerJITLoaderPerfStart(const char *ArgData, size_t ArgSize);
 extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
 llvm_orc_registerJITLoaderPerfEnd(const char *ArgData, size_t ArgSize);
 
-#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERPERF_H
\ No newline at end of file
+#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERPERF_H
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
index 9f84fdc8e7a4..b86c551a7ce3 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
@@ -24,12 +24,12 @@ namespace llvm {
 namespace orc {
 
 /// Register frames in the given eh-frame section with libunwind.
-Error registerEHFrameSection(const void *EHFrameSectionAddr,
-                             size_t EHFrameSectionSize);
+LLVM_ABI Error registerEHFrameSection(const void *EHFrameSectionAddr,
+                                      size_t EHFrameSectionSize);
 
 /// Unregister frames in the given eh-frame section with libunwind.
-Error deregisterEHFrameSection(const void *EHFrameSectionAddr,
-                               size_t EHFrameSectionSize);
+LLVM_ABI Error deregisterEHFrameSection(const void *EHFrameSectionAddr,
+                                        size_t EHFrameSectionSize);
 
 } // end namespace orc
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h
index 00fd84e3ec14..36a6f4bf6e19 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h
@@ -23,6 +23,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Error.h"
 
@@ -33,7 +34,7 @@ namespace orc {
 namespace rt_bootstrap {
 
 /// Simple page-based allocator.
-class SimpleExecutorDylibManager : public ExecutorBootstrapService {
+class LLVM_ABI SimpleExecutorDylibManager : public ExecutorBootstrapService {
 public:
   virtual ~SimpleExecutorDylibManager();
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h
index 97b333c68b63..741f203396fa 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h
@@ -20,6 +20,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 #include <mutex>
@@ -29,7 +30,7 @@ namespace orc {
 namespace rt_bootstrap {
 
 /// Simple page-based allocator.
-class SimpleExecutorMemoryManager : public ExecutorBootstrapService {
+class LLVM_ABI SimpleExecutorMemoryManager : public ExecutorBootstrapService {
 public:
   virtual ~SimpleExecutorMemoryManager();
 
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h
index 07f01ecb68a4..427afa93fdc9 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Error.h"
 
@@ -33,12 +34,12 @@ namespace llvm {
 namespace orc {
 
 /// A simple EPC server implementation.
-class SimpleRemoteEPCServer : public SimpleRemoteEPCTransportClient {
+class LLVM_ABI SimpleRemoteEPCServer : public SimpleRemoteEPCTransportClient {
 public:
   using ReportErrorFunction = unique_function<void(Error)>;
 
   /// Dispatches calls to runWrapper.
-  class Dispatcher {
+  class LLVM_ABI Dispatcher {
   public:
     virtual ~Dispatcher();
     virtual void dispatch(unique_function<void()> Work) = 0;
@@ -46,7 +47,7 @@ class SimpleRemoteEPCServer : public SimpleRemoteEPCTransportClient {
   };
 
 #if LLVM_ENABLE_THREADS
-  class ThreadDispatcher : public Dispatcher {
+  class LLVM_ABI ThreadDispatcher : public Dispatcher {
   public:
     void dispatch(unique_function<void()> Work) override;
     void shutdown() override;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h
index 68acfa750fa0..89a87db7b029 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -28,11 +29,11 @@ namespace orc {
 /// It is legal to have an empty argument list and no program name, however
 /// many main functions will expect a name argument at least, and will fail
 /// if none is provided.
-int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args,
-              std::optional<StringRef> ProgramName = std::nullopt);
+LLVM_ABI int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args,
+                       std::optional<StringRef> ProgramName = std::nullopt);
 
-int runAsVoidFunction(int (*Func)(void));
-int runAsIntFunction(int (*Func)(int), int Arg);
+LLVM_ABI int runAsVoidFunction(int (*Func)(void));
+LLVM_ABI int runAsIntFunction(int (*Func)(int), int Arg);
 
 } // end namespace orc
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h
index 847c340eff17..a101738c135b 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h
@@ -15,6 +15,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_UNWINDINFOMANAGER_H
 
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <map>
 #include <mutex>
@@ -35,22 +36,24 @@ class UnwindInfoManager {
 
   UnwindInfoManager(UnwindInfoManager &&) = delete;
   UnwindInfoManager &operator=(UnwindInfoManager &&) = delete;
-  ~UnwindInfoManager();
+  LLVM_ABI ~UnwindInfoManager();
 
   /// If the libunwind find-dynamic-unwind-info callback registration APIs are
   /// available then this method will instantiate a global UnwindInfoManager
   /// instance suitable for the process and return true. Otherwise it will
   /// return false.
-  static bool TryEnable();
+  LLVM_ABI static bool TryEnable();
 
-  static void addBootstrapSymbols(StringMap<ExecutorAddr> &M);
+  LLVM_ABI static void addBootstrapSymbols(StringMap<ExecutorAddr> &M);
 
-  static Error registerSections(ArrayRef<orc::ExecutorAddrRange> CodeRanges,
-                                orc::ExecutorAddr DSOBase,
-                                orc::ExecutorAddrRange DWARFEHFrame,
-                                orc::ExecutorAddrRange CompactUnwind);
+  LLVM_ABI static Error
+  registerSections(ArrayRef<orc::ExecutorAddrRange> CodeRanges,
+                   orc::ExecutorAddr DSOBase,
+                   orc::ExecutorAddrRange DWARFEHFrame,
+                   orc::ExecutorAddrRange CompactUnwind);
 
-  static Error deregisterSections(ArrayRef<orc::ExecutorAddrRange> CodeRanges);
+  LLVM_ABI static Error
+  deregisterSections(ArrayRef<orc::ExecutorAddrRange> CodeRanges);
 
 private:
   UnwindInfoManager() = default;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h
index 67164679f9e3..9cf6e00ad713 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/TaskDispatch.h
@@ -14,6 +14,7 @@
 #define LLVM_EXECUTIONENGINE_ORC_TASKDISPATCH_H
 
 #include "llvm/Config/llvm-config.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ExtensibleRTTI.h"
 #include "llvm/Support/raw_ostream.h"
@@ -32,7 +33,7 @@ namespace llvm {
 namespace orc {
 
 /// Represents an abstract task for ORC to run.
-class Task : public RTTIExtends<Task, RTTIRoot> {
+class LLVM_ABI Task : public RTTIExtends<Task, RTTIRoot> {
 public:
   static char ID;
 
@@ -51,8 +52,8 @@ class Task : public RTTIExtends<Task, RTTIRoot> {
 /// Base class for generic tasks.
 class GenericNamedTask : public RTTIExtends<GenericNamedTask, Task> {
 public:
-  static char ID;
-  static const char *DefaultDescription;
+  LLVM_ABI static char ID;
+  LLVM_ABI static const char *DefaultDescription;
 };
 
 /// Generic task implementation.
@@ -94,7 +95,7 @@ makeGenericNamedTask(FnT &&Fn, const char *Desc = nullptr) {
 
 /// IdleTask can be used as the basis for low-priority tasks, e.g. speculative
 /// lookup.
-class IdleTask : public RTTIExtends<IdleTask, Task> {
+class LLVM_ABI IdleTask : public RTTIExtends<IdleTask, Task> {
 public:
   static char ID;
 
@@ -103,7 +104,7 @@ class IdleTask : public RTTIExtends<IdleTask, Task> {
 };
 
 /// Abstract base for classes that dispatch ORC Tasks.
-class TaskDispatcher {
+class LLVM_ABI TaskDispatcher {
 public:
   virtual ~TaskDispatcher();
 
@@ -115,7 +116,7 @@ class TaskDispatcher {
 };
 
 /// Runs all tasks on the current thread.
-class InPlaceTaskDispatcher : public TaskDispatcher {
+class LLVM_ABI InPlaceTaskDispatcher : public TaskDispatcher {
 public:
   void dispatch(std::unique_ptr<Task> T) override;
   void shutdown() override;
@@ -123,7 +124,7 @@ class InPlaceTaskDispatcher : public TaskDispatcher {
 
 #if LLVM_ENABLE_THREADS
 
-class DynamicThreadPoolTaskDispatcher : public TaskDispatcher {
+class LLVM_ABI DynamicThreadPoolTaskDispatcher : public TaskDispatcher {
 public:
   DynamicThreadPoolTaskDispatcher(
       std::optional<size_t> MaxMaterializationThreads)
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
index 8ebbf8b3123a..b61c8b8563a1 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
@@ -170,10 +170,9 @@ using GVPredicate = std::function<bool(const GlobalValue &)>;
 using GVModifier = std::function<void(GlobalValue &)>;
 
 /// Clones the given module on to a new context.
-ThreadSafeModule
-cloneToNewContext(const ThreadSafeModule &TSMW,
-                  GVPredicate ShouldCloneDef = GVPredicate(),
-                  GVModifier UpdateClonedDefSource = GVModifier());
+LLVM_ABI ThreadSafeModule cloneToNewContext(
+    const ThreadSafeModule &TSMW, GVPredicate ShouldCloneDef = GVPredicate(),
+    GVModifier UpdateClonedDefSource = GVModifier());
 
 } // End namespace orc
 } // End namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h
index 65f20ad3b216..8b0c403e1222 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h
@@ -14,10 +14,12 @@
 #define LLVM_EXECUTIONENGINE_ORC_UNWINDINFOREGISTRATIONPLUGIN_H
 
 #include "llvm/ExecutionEngine/Orc/LinkGraphLinkingLayer.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::orc {
 
-class UnwindInfoRegistrationPlugin : public LinkGraphLinkingLayer::Plugin {
+class LLVM_ABI UnwindInfoRegistrationPlugin
+    : public LinkGraphLinkingLayer::Plugin {
 public:
   UnwindInfoRegistrationPlugin(ExecutionSession &ES, ExecutorAddr Register,
                                ExecutorAddr Deregister)
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h
index 324521cbe88d..be5c4ce60442 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/RTDyldMemoryManager.h
@@ -17,6 +17,7 @@
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 #include <cstddef>
 #include <cstdint>
 #include <string>
@@ -29,7 +30,7 @@ namespace object {
   class ObjectFile;
 } // end namespace object
 
-class MCJITMemoryManager : public RuntimeDyld::MemoryManager {
+class LLVM_ABI MCJITMemoryManager : public RuntimeDyld::MemoryManager {
 public:
   // Don't hide the notifyObjectLoaded method from RuntimeDyld::MemoryManager.
   using RuntimeDyld::MemoryManager::notifyObjectLoaded;
@@ -57,8 +58,8 @@ class MCJITMemoryManager : public RuntimeDyld::MemoryManager {
 //
 // FIXME: As the RuntimeDyld fills out, additional routines will be needed
 //        for the varying types of objects to be allocated.
-class RTDyldMemoryManager : public MCJITMemoryManager,
-                            public LegacyJITSymbolResolver {
+class LLVM_ABI RTDyldMemoryManager : public MCJITMemoryManager,
+                                     public LegacyJITSymbolResolver {
 public:
   RTDyldMemoryManager() = default;
   RTDyldMemoryManager(const RTDyldMemoryManager&) = delete;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h
index 468296ac1523..c1fab1199f51 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h
@@ -19,6 +19,7 @@
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <algorithm>
 #include <cassert>
@@ -39,7 +40,7 @@ template <typename T> class OwningBinary;
 
 /// Base class for errors originating in RuntimeDyld, e.g. missing relocation
 /// support.
-class RuntimeDyldError : public ErrorInfo<RuntimeDyldError> {
+class LLVM_ABI RuntimeDyldError : public ErrorInfo<RuntimeDyldError> {
 public:
   static char ID;
 
@@ -59,14 +60,14 @@ class RuntimeDyld {
 public:
   // Change the address associated with a section when resolving relocations.
   // Any relocations already associated with the symbol will be re-resolved.
-  void reassignSectionAddress(unsigned SectionID, uint64_t Addr);
+  LLVM_ABI void reassignSectionAddress(unsigned SectionID, uint64_t Addr);
 
   using NotifyStubEmittedFunction = std::function<void(
       StringRef FileName, StringRef SectionName, StringRef SymbolName,
       unsigned SectionID, uint32_t StubOffset)>;
 
   /// Information about the loaded object.
-  class LoadedObjectInfo : public llvm::LoadedObjectInfo {
+  class LLVM_ABI LoadedObjectInfo : public llvm::LoadedObjectInfo {
     friend class RuntimeDyldImpl;
 
   public:
@@ -89,7 +90,7 @@ class RuntimeDyld {
   };
 
   /// Memory Management.
-  class MemoryManager {
+  class LLVM_ABI MemoryManager {
     friend class RuntimeDyld;
 
   public:
@@ -189,49 +190,51 @@ class RuntimeDyld {
   };
 
   /// Construct a RuntimeDyld instance.
-  RuntimeDyld(MemoryManager &MemMgr, JITSymbolResolver &Resolver);
+  LLVM_ABI RuntimeDyld(MemoryManager &MemMgr, JITSymbolResolver &Resolver);
   RuntimeDyld(const RuntimeDyld &) = delete;
   RuntimeDyld &operator=(const RuntimeDyld &) = delete;
-  ~RuntimeDyld();
+  LLVM_ABI ~RuntimeDyld();
 
   /// Add the referenced object file to the list of objects to be loaded and
   /// relocated.
-  std::unique_ptr<LoadedObjectInfo> loadObject(const object::ObjectFile &O);
+  LLVM_ABI std::unique_ptr<LoadedObjectInfo>
+  loadObject(const object::ObjectFile &O);
 
   /// Get the address of our local copy of the symbol. This may or may not
   /// be the address used for relocation (clients can copy the data around
   /// and resolve relocatons based on where they put it).
-  void *getSymbolLocalAddress(StringRef Name) const;
+  LLVM_ABI void *getSymbolLocalAddress(StringRef Name) const;
 
   /// Get the section ID for the section containing the given symbol.
-  unsigned getSymbolSectionID(StringRef Name) const;
+  LLVM_ABI unsigned getSymbolSectionID(StringRef Name) const;
 
   /// Get the target address and flags for the named symbol.
   /// This address is the one used for relocation.
-  JITEvaluatedSymbol getSymbol(StringRef Name) const;
+  LLVM_ABI JITEvaluatedSymbol getSymbol(StringRef Name) const;
 
   /// Returns a copy of the symbol table. This can be used by on-finalized
   /// callbacks to extract the symbol table before throwing away the
   /// RuntimeDyld instance. Because the map keys (StringRefs) are backed by
   /// strings inside the RuntimeDyld instance, the map should be processed
   /// before the RuntimeDyld instance is discarded.
-  std::map<StringRef, JITEvaluatedSymbol> getSymbolTable() const;
+  LLVM_ABI std::map<StringRef, JITEvaluatedSymbol> getSymbolTable() const;
 
   /// Resolve the relocations for all symbols we currently know about.
-  void resolveRelocations();
+  LLVM_ABI void resolveRelocations();
 
   /// Map a section to its target address space value.
   /// Map the address of a JIT section as returned from the memory manager
   /// to the address in the target process as the running code will see it.
   /// This is the address which will be used for relocation resolution.
-  void mapSectionAddress(const void *LocalAddress, uint64_t TargetAddress);
+  LLVM_ABI void mapSectionAddress(const void *LocalAddress,
+                                  uint64_t TargetAddress);
 
   /// Returns the section's working memory.
-  StringRef getSectionContent(unsigned SectionID) const;
+  LLVM_ABI StringRef getSectionContent(unsigned SectionID) const;
 
   /// If the section was loaded, return the section's load address,
   /// otherwise return std::nullopt.
-  uint64_t getSectionLoadAddress(unsigned SectionID) const;
+  LLVM_ABI uint64_t getSectionLoadAddress(unsigned SectionID) const;
 
   /// Set the NotifyStubEmitted callback. This is used for debugging
   /// purposes. A callback is made for each stub that is generated.
@@ -244,12 +247,12 @@ class RuntimeDyld {
   /// for identifying the EH frame and calling the memory manager with the
   /// EH frame section data.  However, the memory manager itself will handle
   /// the actual target-specific EH frame registration.
-  void registerEHFrames();
+  LLVM_ABI void registerEHFrames();
 
-  void deregisterEHFrames();
+  LLVM_ABI void deregisterEHFrames();
 
-  bool hasError();
-  StringRef getErrorString();
+  LLVM_ABI bool hasError();
+  LLVM_ABI StringRef getErrorString();
 
   /// By default, only sections that are "required for execution" are passed to
   /// the RTDyldMemoryManager, and other sections are discarded. Passing 'true'
@@ -281,10 +284,10 @@ class RuntimeDyld {
   ///   address of a symbol owned by some other instance in order to apply
   ///   relocations.
   ///
-  void finalizeWithMemoryManagerLocking();
+  LLVM_ABI void finalizeWithMemoryManagerLocking();
 
 private:
-  friend void jitLinkForORC(
+  LLVM_ABI_FRIEND friend void jitLinkForORC(
       object::OwningBinary<object::ObjectFile> O,
       RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver,
       bool ProcessAllSections,
@@ -310,7 +313,7 @@ class RuntimeDyld {
 // but ORC's RTDyldObjectLinkingLayer2. Internally it constructs a RuntimeDyld
 // instance and uses continuation passing to perform the fix-up and finalize
 // steps asynchronously.
-void jitLinkForORC(
+LLVM_ABI void jitLinkForORC(
     object::OwningBinary<object::ObjectFile> O,
     RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver,
     bool ProcessAllSections,
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
index 92fdd376abd1..ec9bee8edb59 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
 #include "llvm/TargetParser/Triple.h"
@@ -159,23 +160,22 @@ class RuntimeDyldChecker {
   using GetGOTInfoFunction = std::function<Expected<MemoryRegionInfo>(
       StringRef GOTContainer, StringRef TargetName)>;
 
-  RuntimeDyldChecker(IsSymbolValidFunction IsSymbolValid,
-                     GetSymbolInfoFunction GetSymbolInfo,
-                     GetSectionInfoFunction GetSectionInfo,
-                     GetStubInfoFunction GetStubInfo,
-                     GetGOTInfoFunction GetGOTInfo, llvm::endianness Endianness,
-                     Triple TT, StringRef CPU, SubtargetFeatures TF,
-                     raw_ostream &ErrStream);
-  ~RuntimeDyldChecker();
+  LLVM_ABI RuntimeDyldChecker(
+      IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo,
+      GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo,
+      GetGOTInfoFunction GetGOTInfo, llvm::endianness Endianness, Triple TT,
+      StringRef CPU, SubtargetFeatures TF, raw_ostream &ErrStream);
+  LLVM_ABI ~RuntimeDyldChecker();
 
   /// Check a single expression against the attached RuntimeDyld
   ///        instance.
-  bool check(StringRef CheckExpr) const;
+  LLVM_ABI bool check(StringRef CheckExpr) const;
 
   /// Scan the given memory buffer for lines beginning with the string
   ///        in RulePrefix. The remainder of the line is passed to the check
   ///        method to be evaluated as an expression.
-  bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const;
+  LLVM_ABI bool checkAllRulesInBuffer(StringRef RulePrefix,
+                                      MemoryBuffer *MemBuf) const;
 
   /// Returns the address of the requested section (or an error message
   ///        in the second element of the pair if the address cannot be found).
@@ -183,13 +183,13 @@ class RuntimeDyldChecker {
   /// if 'LocalAddress' is true, this returns the address of the section
   /// within the linker's memory. If 'LocalAddress' is false it returns the
   /// address within the target process (i.e. the load address).
-  std::pair<uint64_t, std::string> getSectionAddr(StringRef FileName,
-                                                  StringRef SectionName,
-                                                  bool LocalAddress);
+  LLVM_ABI std::pair<uint64_t, std::string>
+  getSectionAddr(StringRef FileName, StringRef SectionName, bool LocalAddress);
 
   /// If there is a section at the given local address, return its load
   /// address, otherwise return std::nullopt.
-  std::optional<uint64_t> getSectionLoadAddress(void *LocalAddress) const;
+  LLVM_ABI std::optional<uint64_t>
+  getSectionLoadAddress(void *LocalAddress) const;
 
 private:
   std::unique_ptr<RuntimeDyldCheckerImpl> Impl;
diff --git a/external/llvm-project/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h b/external/llvm-project/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h
index fa1b2355528d..dacd100184d6 100644
--- a/external/llvm-project/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h
+++ b/external/llvm-project/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Memory.h"
 #include <cstdint>
 #include <string>
@@ -36,7 +37,7 @@ namespace llvm {
 /// in the JITed object.  Permissions can be applied either by calling
 /// MCJIT::finalizeObject or by calling SectionMemoryManager::finalizeMemory
 /// directly.  Clients of MCJIT should call MCJIT::finalizeObject.
-class SectionMemoryManager : public RTDyldMemoryManager {
+class LLVM_ABI SectionMemoryManager : public RTDyldMemoryManager {
 public:
   /// This enum describes the various reasons to allocate pages from
   /// allocateMappedMemory.
@@ -48,7 +49,7 @@ class SectionMemoryManager : public RTDyldMemoryManager {
 
   /// Implementations of this interface are used by SectionMemoryManager to
   /// request pages from the operating system.
-  class MemoryMapper {
+  class LLVM_ABI MemoryMapper {
   public:
     /// This method attempts to allocate \p NumBytes bytes of virtual memory for
     /// \p Purpose.  \p NearBlock may point to an existing allocation, in which
diff --git a/external/llvm-project/llvm/include/llvm/FileCheck/FileCheck.h b/external/llvm-project/llvm/include/llvm/FileCheck/FileCheck.h
index 72d0b91b27ad..b44ed8ed3f83 100644
--- a/external/llvm-project/llvm/include/llvm/FileCheck/FileCheck.h
+++ b/external/llvm-project/llvm/include/llvm/FileCheck/FileCheck.h
@@ -14,6 +14,7 @@
 #define LLVM_FILECHECK_FILECHECK_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/SMLoc.h"
 #include <bitset>
@@ -91,7 +92,7 @@ class FileCheckType {
   operator FileCheckKind() const { return Kind; }
 
   int getCount() const { return Count; }
-  FileCheckType &setCount(int C);
+  LLVM_ABI FileCheckType &setCount(int C);
 
   bool isLiteralMatch() const {
     return Modifiers[FileCheckKindModifier::ModifierLiteral];
@@ -102,10 +103,10 @@ class FileCheckType {
   }
 
   // \returns a description of \p Prefix.
-  std::string getDescription(StringRef Prefix) const;
+  LLVM_ABI std::string getDescription(StringRef Prefix) const;
 
   // \returns a description of \p Modifiers.
-  std::string getModifiersDescription() const;
+  LLVM_ABI std::string getModifiersDescription() const;
 };
 } // namespace Check
 
@@ -167,9 +168,10 @@ struct FileCheckDiag {
   /// A note to replace the one normally indicated by MatchTy, or the empty
   /// string if none.
   std::string Note;
-  FileCheckDiag(const SourceMgr &SM, const Check::FileCheckType &CheckTy,
-                SMLoc CheckLoc, MatchType MatchTy, SMRange InputRange,
-                StringRef Note = "");
+  LLVM_ABI FileCheckDiag(const SourceMgr &SM,
+                         const Check::FileCheckType &CheckTy, SMLoc CheckLoc,
+                         MatchType MatchTy, SMRange InputRange,
+                         StringRef Note = "");
 };
 
 class FileCheckPatternContext;
@@ -183,8 +185,8 @@ class FileCheck {
   std::vector<FileCheckString> CheckStrings;
 
 public:
-  explicit FileCheck(FileCheckRequest Req);
-  ~FileCheck();
+  LLVM_ABI explicit FileCheck(FileCheckRequest Req);
+  LLVM_ABI ~FileCheck();
 
   /// Reads the check file from \p Buffer and records the expected strings it
   /// contains. Errors are reported against \p SM.
@@ -192,24 +194,24 @@ class FileCheck {
   /// If \p ImpPatBufferIDRange, then the range (inclusive start, exclusive end)
   /// of IDs for source buffers added to \p SM for implicit patterns are
   /// recorded in it.  The range is empty if there are none.
-  bool
+  LLVM_ABI bool
   readCheckFile(SourceMgr &SM, StringRef Buffer,
                 std::pair<unsigned, unsigned> *ImpPatBufferIDRange = nullptr);
 
-  bool ValidateCheckPrefixes();
+  LLVM_ABI bool ValidateCheckPrefixes();
 
   /// Canonicalizes whitespaces in the file. Line endings are replaced with
   /// UNIX-style '\n'.
-  StringRef CanonicalizeFile(MemoryBuffer &MB,
-                             SmallVectorImpl<char> &OutputBuffer);
+  LLVM_ABI StringRef CanonicalizeFile(MemoryBuffer &MB,
+                                      SmallVectorImpl<char> &OutputBuffer);
 
   /// Checks the input to FileCheck provided in the \p Buffer against the
   /// expected strings read from the check file and record diagnostics emitted
   /// in \p Diags. Errors are recorded against \p SM.
   ///
   /// \returns false if the input fails to satisfy the checks.
-  bool checkInput(SourceMgr &SM, StringRef Buffer,
-                  std::vector<FileCheckDiag> *Diags = nullptr);
+  LLVM_ABI bool checkInput(SourceMgr &SM, StringRef Buffer,
+                           std::vector<FileCheckDiag> *Diags = nullptr);
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/Atomic/Atomic.h b/external/llvm-project/llvm/include/llvm/Frontend/Atomic/Atomic.h
index 0231e01344bf..8addcd388d73 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/Atomic/Atomic.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/Atomic/Atomic.h
@@ -11,6 +11,7 @@
 
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class AtomicInfo {
@@ -55,13 +56,13 @@ class AtomicInfo {
 
   LLVMContext &getLLVMContext() const { return Builder->getContext(); }
 
-  bool shouldCastToInt(Type *ValTy, bool CmpXchg);
+  LLVM_ABI bool shouldCastToInt(Type *ValTy, bool CmpXchg);
 
-  Value *EmitAtomicLoadOp(AtomicOrdering AO, bool IsVolatile,
-                          bool CmpXchg = false);
+  LLVM_ABI Value *EmitAtomicLoadOp(AtomicOrdering AO, bool IsVolatile,
+                                   bool CmpXchg = false);
 
-  CallInst *EmitAtomicLibcall(StringRef fnName, Type *ResultType,
-                              ArrayRef<Value *> Args);
+  LLVM_ABI CallInst *EmitAtomicLibcall(StringRef fnName, Type *ResultType,
+                                       ArrayRef<Value *> Args);
 
   Value *getAtomicSizeValue() const {
     LLVMContext &ctx = getLLVMContext();
@@ -73,7 +74,7 @@ class AtomicInfo {
                             AtomicSizeInBits / BitsPerByte);
   }
 
-  std::pair<Value *, Value *>
+  LLVM_ABI std::pair<Value *, Value *>
   EmitAtomicCompareExchangeLibcall(Value *ExpectedVal, Value *DesiredVal,
                                    AtomicOrdering Success,
                                    AtomicOrdering Failure);
@@ -86,19 +87,20 @@ class AtomicInfo {
     return castToAtomicIntPointer(getAtomicPointer());
   }
 
-  std::pair<Value *, Value *>
+  LLVM_ABI std::pair<Value *, Value *>
   EmitAtomicCompareExchangeOp(Value *ExpectedVal, Value *DesiredVal,
                               AtomicOrdering Success, AtomicOrdering Failure,
                               bool IsVolatile = false, bool IsWeak = false);
 
-  std::pair<Value *, Value *>
+  LLVM_ABI std::pair<Value *, Value *>
   EmitAtomicCompareExchange(Value *ExpectedVal, Value *DesiredVal,
                             AtomicOrdering Success, AtomicOrdering Failure,
                             bool IsVolatile, bool IsWeak);
 
-  std::pair<LoadInst *, AllocaInst *> EmitAtomicLoadLibcall(AtomicOrdering AO);
+  LLVM_ABI std::pair<LoadInst *, AllocaInst *>
+  EmitAtomicLoadLibcall(AtomicOrdering AO);
 
-  void EmitAtomicStoreLibcall(AtomicOrdering AO, Value *Source);
+  LLVM_ABI void EmitAtomicStoreLibcall(AtomicOrdering AO, Value *Source);
 };
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/Directive/DirectiveBase.td b/external/llvm-project/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
index 3e2744dea8d1..301a7cce5962 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
+++ b/external/llvm-project/llvm/include/llvm/Frontend/Directive/DirectiveBase.td
@@ -51,9 +51,37 @@ class DirectiveLanguage {
   string flangClauseBaseClass = "";
 }
 
-// Information about values accepted by enum-like clauses
-class ClauseVal<string n, int v, bit uv> {
-  // Name of the clause value.
+// Base class for versioned entities.
+class Versioned<int min = 0, int max = 0x7FFFFFFF> {
+  // Mininum version number where this object is valid.
+  int minVersion = min;
+
+  // Maximum version number where this object is valid.
+  int maxVersion = max;
+}
+
+class Spelling<string s, int min = 0, int max = 0x7FFFFFFF>
+    : Versioned<min, max> {
+  string spelling = s;
+}
+
+// Some clauses take an argument from a predefined list of allowed keyword
+// values. For example, assume a clause "someclause" with an argument from
+// the list "foo", "bar", "baz". In the user source code this would look
+// like "someclause(foo)", whereas in the compiler the values would be
+// represented as
+// enum someclause.enumClauseValue {
+//   Xyz_foo = v_foo,
+//   Xyz_bar = v_bar,
+//   Xyz_baz = v_baz,
+// }
+// The "Xyz_..." are the _record_ names of EnumVal's:
+// def Xyz_foo = EnumVal<"foo", v_foo>;
+// def Xyz_bar = EnumVal<"bar", v_bar>;
+// def Xyz_baz = EnumVal<"baz", v_baz>;
+//
+class EnumVal<string n, int v, bit uv> {
+  // Spelling of the value.
   string name = n;
 
   // Integer value of the clause.
@@ -67,12 +95,9 @@ class ClauseVal<string n, int v, bit uv> {
 }
 
 // Information about a specific clause.
-class Clause<string c> {
-  // Name of the clause.
-  string name = c;
-
-  // Define aliases used in the parser.
-  list<string> aliases = [];
+class Clause<list<Spelling> ss> {
+  // Spellings of the clause.
+  list<Spelling> spellings = ss;
 
   // Optional class holding value of the clause in clang AST.
   string clangClass = "";
@@ -90,7 +115,7 @@ class Clause<string c> {
   string enumClauseValue = "";
 
   // List of allowed clause values
-  list<ClauseVal> allowedClauseValues = [];
+  list<EnumVal> allowedClauseValues = [];
 
   // If set to true, value class is part of a list. Single class by default.
   bit isValueList = false;
@@ -120,15 +145,9 @@ class Clause<string c> {
 }
 
 // Hold information about clause validity by version.
-class VersionedClause<Clause c, int min = 1, int max = 0x7FFFFFFF> {
-  // Actual clause.
+class VersionedClause<Clause c, int min = 1, int max = 0x7FFFFFFF>
+    : Versioned<min, max> {
   Clause clause = c;
-
-  // Mininum version number where this clause is valid.
-  int minVersion = min;
-
-  // Maximum version number where this clause is valid.
-  int maxVersion = max;
 }
 
 // Kinds of directive associations.
@@ -176,15 +195,15 @@ class SourceLanguage<string n> {
   string name = n;  // Name of the enum value in enum class Association.
 }
 
-// The C languages also implies C++ until there is a reason to add C++
+// The C language also implies C++ until there is a reason to add C++
 // separately.
 def L_C : SourceLanguage<"C"> {}
 def L_Fortran : SourceLanguage<"Fortran"> {}
 
 // Information about a specific directive.
-class Directive<string d> {
-  // Name of the directive. Can be composite directive sepearted by whitespace.
-  string name = d;
+class Directive<list<Spelling> ss> {
+  // Spellings of the directive.
+  list<Spelling> spellings = ss;
 
   // Clauses cannot appear twice in the three allowed lists below. Also, since
   // required implies allowed, the same clause cannot appear in both the
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/Directive/Spelling.h b/external/llvm-project/llvm/include/llvm/Frontend/Directive/Spelling.h
new file mode 100644
index 000000000000..a10148960325
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/Frontend/Directive/Spelling.h
@@ -0,0 +1,40 @@
+//===-------------------------------------------------------------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_FRONTEND_DIRECTIVE_SPELLING_H
+#define LLVM_FRONTEND_DIRECTIVE_SPELLING_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/iterator_range.h"
+
+#include <limits>
+#include <tuple>
+
+namespace llvm::directive {
+
+struct VersionRange {
+  static constexpr int MaxValue = std::numeric_limits<int>::max();
+  // The default "Version" value in get<Lang><Enum>Name() is 0, include that
+  // in the maximum range.
+  int Min = 0;
+  int Max = MaxValue;
+
+  bool operator<(const VersionRange &R) const {
+    return std::tie(Min, Max) < std::tie(R.Min, R.Max);
+  }
+};
+
+struct Spelling {
+  StringRef Name;
+  VersionRange Versions;
+};
+
+StringRef FindName(llvm::iterator_range<const Spelling *>, unsigned Version);
+
+} // namespace llvm::directive
+
+#endif // LLVM_FRONTEND_DIRECTIVE_SPELLING_H
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h b/external/llvm-project/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h
index ee52645f2e51..f0168c040788 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h
@@ -13,6 +13,9 @@
 #ifndef LLVM_FRONTEND_DRIVER_CODEGENOPTIONS_H
 #define LLVM_FRONTEND_DRIVER_CODEGENOPTIONS_H
 
+#include "llvm/Support/Compiler.h"
+#include <string>
+
 namespace llvm {
 class Triple;
 class TargetLibraryInfoImpl;
@@ -46,9 +49,21 @@ enum class VectorLibrary {
   AMDLIBM             // AMD vector math library.
 };
 
-TargetLibraryInfoImpl *createTLII(const llvm::Triple &TargetTriple,
-                                  VectorLibrary Veclib);
+LLVM_ABI TargetLibraryInfoImpl *createTLII(const llvm::Triple &TargetTriple,
+                                           VectorLibrary Veclib);
+
+enum ProfileInstrKind {
+  ProfileNone,       // Profile instrumentation is turned off.
+  ProfileClangInstr, // Clang instrumentation to generate execution counts
+                     // to use with PGO.
+  ProfileIRInstr,    // IR level PGO instrumentation in LLVM.
+  ProfileCSIRInstr,  // IR level PGO context sensitive instrumentation in LLVM.
+  ProfileIRSampleColdCov, // IR level sample pgo based cold function coverage
+                          // instrumentation in LLVM.
+};
 
+// Default filename used for profile generation.
+std::string getDefaultProfileGenName();
 } // end namespace llvm::driver
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h b/external/llvm-project/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
index 4d4fb261595a..2f028817b45b 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
@@ -6,28 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 ///
-/// \file This file contains helper objects for working with HLSL Root
-/// Signatures.
+/// \file This file contains structure definitions of HLSL Root Signature
+/// objects.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_FRONTEND_HLSL_HLSLROOTSIGNATURE_H
 #define LLVM_FRONTEND_HLSL_HLSLROOTSIGNATURE_H
 
-#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DXILABI.h"
-#include "llvm/Support/raw_ostream.h"
 #include <variant>
 
 namespace llvm {
-class LLVMContext;
-class MDNode;
-class Metadata;
-
 namespace hlsl {
 namespace rootsig {
 
-// Definition of the various enumerations and flags
+// Definition of the various enumerations and flags. The definitions of all
+// values here correspond to their description in the d3d12.h header and are
+// carried over from their values in DXC. For reference:
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/
 
 enum class RootFlags : uint32_t {
   None = 0,
@@ -76,6 +74,47 @@ enum class ShaderVisibility {
   Mesh = 7,
 };
 
+// D3D12_FILTER enumeration:
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_filter
+enum class SamplerFilter {
+  MinMagMipPoint = 0,
+  MinMagPointMipLinear = 0x1,
+  MinPointMagLinearMipPoint = 0x4,
+  MinPointMagMipLinear = 0x5,
+  MinLinearMagMipPoint = 0x10,
+  MinLinearMagPointMipLinear = 0x11,
+  MinMagLinearMipPoint = 0x14,
+  MinMagMipLinear = 0x15,
+  Anisotropic = 0x55,
+  ComparisonMinMagMipPoint = 0x80,
+  ComparisonMinMagPointMipLinear = 0x81,
+  ComparisonMinPointMagLinearMipPoint = 0x84,
+  ComparisonMinPointMagMipLinear = 0x85,
+  ComparisonMinLinearMagMipPoint = 0x90,
+  ComparisonMinLinearMagPointMipLinear = 0x91,
+  ComparisonMinMagLinearMipPoint = 0x94,
+  ComparisonMinMagMipLinear = 0x95,
+  ComparisonAnisotropic = 0xd5,
+  MinimumMinMagMipPoint = 0x100,
+  MinimumMinMagPointMipLinear = 0x101,
+  MinimumMinPointMagLinearMipPoint = 0x104,
+  MinimumMinPointMagMipLinear = 0x105,
+  MinimumMinLinearMagMipPoint = 0x110,
+  MinimumMinLinearMagPointMipLinear = 0x111,
+  MinimumMinMagLinearMipPoint = 0x114,
+  MinimumMinMagMipLinear = 0x115,
+  MinimumAnisotropic = 0x155,
+  MaximumMinMagMipPoint = 0x180,
+  MaximumMinMagPointMipLinear = 0x181,
+  MaximumMinPointMagLinearMipPoint = 0x184,
+  MaximumMinPointMagMipLinear = 0x185,
+  MaximumMinLinearMagMipPoint = 0x190,
+  MaximumMinLinearMagPointMipLinear = 0x191,
+  MaximumMinMagLinearMipPoint = 0x194,
+  MaximumMinMagMipLinear = 0x195,
+  MaximumAnisotropic = 0x1d5
+};
+
 enum class TextureAddressMode {
   Wrap = 1,
   Mirror = 2,
@@ -150,8 +189,6 @@ struct DescriptorTable {
   uint32_t NumClauses = 0;
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table);
-
 static const uint32_t NumDescriptorsUnbounded = 0xffffffff;
 static const uint32_t DescriptorTableOffsetAppend = 0xffffffff;
 // Models DTClause : CBV | SRV | UAV | Sampler, by collecting like parameters
@@ -180,10 +217,9 @@ struct DescriptorTableClause {
   }
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const DescriptorTableClause &Clause);
-
 struct StaticSampler {
   Register Reg;
+  SamplerFilter Filter = SamplerFilter::Anisotropic;
   TextureAddressMode AddressU = TextureAddressMode::Wrap;
   TextureAddressMode AddressV = TextureAddressMode::Wrap;
   TextureAddressMode AddressW = TextureAddressMode::Wrap;
@@ -217,32 +253,6 @@ using RootElement =
     std::variant<RootFlags, RootConstants, RootDescriptor, DescriptorTable,
                  DescriptorTableClause, StaticSampler>;
 
-void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements);
-
-class MetadataBuilder {
-public:
-  MetadataBuilder(llvm::LLVMContext &Ctx, ArrayRef<RootElement> Elements)
-      : Ctx(Ctx), Elements(Elements) {}
-
-  /// Iterates through the elements and dispatches onto the correct Build method
-  ///
-  /// Accumulates the root signature and returns the Metadata node that is just
-  /// a list of all the elements
-  MDNode *BuildRootSignature();
-
-private:
-  /// Define the various builders for the different metadata types
-  MDNode *BuildRootFlags(const RootFlags &Flags);
-  MDNode *BuildRootConstants(const RootConstants &Constants);
-  MDNode *BuildRootDescriptor(const RootDescriptor &Descriptor);
-  MDNode *BuildDescriptorTable(const DescriptorTable &Table);
-  MDNode *BuildDescriptorTableClause(const DescriptorTableClause &Clause);
-
-  llvm::LLVMContext &Ctx;
-  ArrayRef<RootElement> Elements;
-  SmallVector<Metadata *> GeneratedMetadata;
-};
-
 } // namespace rootsig
 } // namespace hlsl
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h b/external/llvm-project/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
new file mode 100644
index 000000000000..6d959ad5bdc7
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
@@ -0,0 +1,66 @@
+//===- HLSLRootSignatureUtils.h - HLSL Root Signature helpers -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains helper obejcts for working with HLSL Root
+/// Signatures.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
+#define LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+class LLVMContext;
+class MDNode;
+class Metadata;
+
+namespace hlsl {
+namespace rootsig {
+
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const DescriptorTableClause &Clause);
+
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table);
+
+LLVM_ABI void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements);
+
+class MetadataBuilder {
+public:
+  MetadataBuilder(llvm::LLVMContext &Ctx, ArrayRef<RootElement> Elements)
+      : Ctx(Ctx), Elements(Elements) {}
+
+  /// Iterates through the elements and dispatches onto the correct Build method
+  ///
+  /// Accumulates the root signature and returns the Metadata node that is just
+  /// a list of all the elements
+  LLVM_ABI MDNode *BuildRootSignature();
+
+private:
+  /// Define the various builders for the different metadata types
+  MDNode *BuildRootFlags(const RootFlags &Flags);
+  MDNode *BuildRootConstants(const RootConstants &Constants);
+  MDNode *BuildRootDescriptor(const RootDescriptor &Descriptor);
+  MDNode *BuildDescriptorTable(const DescriptorTable &Table);
+  MDNode *BuildDescriptorTableClause(const DescriptorTableClause &Clause);
+  MDNode *BuildStaticSampler(const StaticSampler &Sampler);
+
+  llvm::LLVMContext &Ctx;
+  ArrayRef<RootElement> Elements;
+  SmallVector<Metadata *> GeneratedMetadata;
+};
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h b/external/llvm-project/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h
index 79309251c3b6..6b9da0670726 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace offloading {
@@ -22,11 +23,10 @@ using EntryArrayTy = std::pair<GlobalVariable *, GlobalVariable *>;
 /// \param Suffix An optional suffix appended to the emitted symbols.
 /// \param Relocatable Indicate if we need to change the offloading section to
 /// create a relocatable object.
-llvm::Error wrapOpenMPBinaries(llvm::Module &M,
-                               llvm::ArrayRef<llvm::ArrayRef<char>> Images,
-                               EntryArrayTy EntryArray,
-                               llvm::StringRef Suffix = "",
-                               bool Relocatable = false);
+LLVM_ABI llvm::Error
+wrapOpenMPBinaries(llvm::Module &M, llvm::ArrayRef<llvm::ArrayRef<char>> Images,
+                   EntryArrayTy EntryArray, llvm::StringRef Suffix = "",
+                   bool Relocatable = false);
 
 /// Wraps the input fatbinary image into the module \p M as global symbols and
 /// registers the images with the CUDA runtime.
@@ -35,9 +35,11 @@ llvm::Error wrapOpenMPBinaries(llvm::Module &M,
 /// \param Suffix An optional suffix appended to the emitted symbols.
 /// \param EmitSurfacesAndTextures Whether to emit surface and textures
 /// registration code. It defaults to false.
-llvm::Error wrapCudaBinary(llvm::Module &M, llvm::ArrayRef<char> Images,
-                           EntryArrayTy EntryArray, llvm::StringRef Suffix = "",
-                           bool EmitSurfacesAndTextures = true);
+LLVM_ABI llvm::Error wrapCudaBinary(llvm::Module &M,
+                                    llvm::ArrayRef<char> Images,
+                                    EntryArrayTy EntryArray,
+                                    llvm::StringRef Suffix = "",
+                                    bool EmitSurfacesAndTextures = true);
 
 /// Wraps the input bundled image into the module \p M as global symbols and
 /// registers the images with the HIP runtime.
@@ -46,9 +48,10 @@ llvm::Error wrapCudaBinary(llvm::Module &M, llvm::ArrayRef<char> Images,
 /// \param Suffix An optional suffix appended to the emitted symbols.
 /// \param EmitSurfacesAndTextures Whether to emit surface and textures
 /// registration code. It defaults to false.
-llvm::Error wrapHIPBinary(llvm::Module &M, llvm::ArrayRef<char> Images,
-                          EntryArrayTy EntryArray, llvm::StringRef Suffix = "",
-                          bool EmitSurfacesAndTextures = true);
+LLVM_ABI llvm::Error wrapHIPBinary(llvm::Module &M, llvm::ArrayRef<char> Images,
+                                   EntryArrayTy EntryArray,
+                                   llvm::StringRef Suffix = "",
+                                   bool EmitSurfacesAndTextures = true);
 } // namespace offloading
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/Offloading/Utility.h b/external/llvm-project/llvm/include/llvm/Frontend/Offloading/Utility.h
index 7b717a4733b7..f8a2b1237b5e 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/Offloading/Utility.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/Offloading/Utility.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_FRONTEND_OFFLOADING_UTILITY_H
 #define LLVM_FRONTEND_OFFLOADING_UTILITY_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <memory>
 
@@ -67,7 +68,7 @@ enum OffloadEntryKindFlag : uint32_t {
 
 /// Returns the type of the offloading entry we use to store kernels and
 /// globals that will be registered with the offloading runtime.
-StructType *getEntryTy(Module &M);
+LLVM_ABI StructType *getEntryTy(Module &M);
 
 /// Create an offloading section struct used to register this global at
 /// runtime.
@@ -81,22 +82,23 @@ StructType *getEntryTy(Module &M);
 /// \param Data Extra data storage associated with the entry.
 /// \param SectionName The section this entry will be placed at.
 /// \param AuxAddr An extra pointer if needed.
-void emitOffloadingEntry(Module &M, object::OffloadKind Kind, Constant *Addr,
-                         StringRef Name, uint64_t Size, uint32_t Flags,
-                         uint64_t Data, Constant *AuxAddr = nullptr,
-                         StringRef SectionName = "llvm_offload_entries");
+LLVM_ABI void
+emitOffloadingEntry(Module &M, object::OffloadKind Kind, Constant *Addr,
+                    StringRef Name, uint64_t Size, uint32_t Flags,
+                    uint64_t Data, Constant *AuxAddr = nullptr,
+                    StringRef SectionName = "llvm_offload_entries");
 
 /// Create a constant struct initializer used to register this global at
 /// runtime.
 /// \return the constant struct and the global variable holding the symbol name.
-std::pair<Constant *, GlobalVariable *>
+LLVM_ABI std::pair<Constant *, GlobalVariable *>
 getOffloadingEntryInitializer(Module &M, object::OffloadKind Kind,
                               Constant *Addr, StringRef Name, uint64_t Size,
                               uint32_t Flags, uint64_t Data, Constant *AuxAddr);
 
 /// Creates a pair of globals used to iterate the array of offloading entries by
 /// accessing the section variables provided by the linker.
-std::pair<GlobalVariable *, GlobalVariable *>
+LLVM_ABI std::pair<GlobalVariable *, GlobalVariable *>
 getOffloadEntryArray(Module &M, StringRef SectionName = "llvm_offload_entries");
 
 namespace amdgpu {
@@ -109,8 +111,8 @@ namespace amdgpu {
 /// and is compatible with either '+' or '-'. The HSA runtime returns this
 /// information using the target-id, while we use the ELF header to determine
 /// these features.
-bool isImageCompatibleWithEnv(StringRef ImageArch, uint32_t ImageFlags,
-                              StringRef EnvTargetID);
+LLVM_ABI bool isImageCompatibleWithEnv(StringRef ImageArch, uint32_t ImageFlags,
+                                       StringRef EnvTargetID);
 
 /// Struct for holding metadata related to AMDGPU kernels, for more information
 /// about the metadata and its meaning see:
@@ -149,15 +151,16 @@ struct AMDGPUKernelMetaData {
 
 /// Reads AMDGPU specific metadata from the ELF file and propagates the
 /// KernelInfoMap.
-Error getAMDGPUMetaDataFromImage(MemoryBufferRef MemBuffer,
-                                 StringMap<AMDGPUKernelMetaData> &KernelInfoMap,
-                                 uint16_t &ELFABIVersion);
+LLVM_ABI Error getAMDGPUMetaDataFromImage(
+    MemoryBufferRef MemBuffer, StringMap<AMDGPUKernelMetaData> &KernelInfoMap,
+    uint16_t &ELFABIVersion);
 } // namespace amdgpu
 
 namespace intel {
 /// Containerizes an offloading binary into the ELF binary format expected by
 /// the Intel runtime offload plugin.
-Error containerizeOpenMPSPIRVImage(std::unique_ptr<MemoryBuffer> &Binary);
+LLVM_ABI Error
+containerizeOpenMPSPIRVImage(std::unique_ptr<MemoryBuffer> &Binary);
 } // namespace intel
 } // namespace offloading
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/OpenACC/ACC.td b/external/llvm-project/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 46cba9f2400e..65751839ceb0 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/external/llvm-project/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -32,64 +32,65 @@ def OpenACC : DirectiveLanguage {
 //===----------------------------------------------------------------------===//
 
 // 2.16.1
-def ACCC_Async : Clause<"async"> {
+def ACCC_Async : Clause<[Spelling<"async">]> {
   let flangClass = "ScalarIntExpr";
   let isValueOptional = true;
 }
 
 // 2.9.7
-def ACCC_Auto : Clause<"auto"> {}
+def ACCC_Auto : Clause<[Spelling<"auto">]> {}
 
 // 2.7.12
-def ACCC_Attach : Clause<"attach"> {
+def ACCC_Attach : Clause<[Spelling<"attach">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.15.1
-def ACCC_Bind : Clause<"bind"> {
+def ACCC_Bind : Clause<[Spelling<"bind">]> {
   let flangClass = "AccBindClause";
 }
 
 // 2.12
-def ACCC_Capture : Clause<"capture"> {
+def ACCC_Capture : Clause<[Spelling<"capture">]> {
 }
 
 // 2.9.1
-def ACCC_Collapse : Clause<"collapse"> {
+def ACCC_Collapse : Clause<[Spelling<"collapse">]> {
   let flangClass = "AccCollapseArg";
 }
 
 // 2.7.6
-def ACCC_Copy : Clause<"copy"> {
+def ACCC_Copy
+    : Clause<[Spelling<"copy">, Spelling<"present_or_copy">,
+              Spelling<"pcopy">]> {
   let flangClass = "AccObjectList";
-  let aliases = ["present_or_copy", "pcopy"];
 }
 
 // 2.7.7
-def ACCC_Copyin : Clause<"copyin"> {
+def ACCC_Copyin : Clause<[Spelling<"copyin">, Spelling<"present_or_copyin">,
+                          Spelling<"pcopyin">]> {
   let flangClass = "AccObjectListWithModifier";
   let clangAccSpelling = "CopyIn";
-  let aliases = ["present_or_copyin", "pcopyin"];
 }
 
 // 2.7.8
-def ACCC_Copyout : Clause<"copyout"> {
+def ACCC_Copyout : Clause<[Spelling<"copyout">, Spelling<"present_or_copyout">,
+                           Spelling<"pcopyout">]> {
   let flangClass = "AccObjectListWithModifier";
   let clangAccSpelling = "CopyOut";
-  let aliases = ["present_or_copyout", "pcopyout"];
 }
 
 // 2.7.9
-def ACCC_Create : Clause<"create"> {
+def ACCC_Create : Clause<[Spelling<"create">, Spelling<"present_or_create">,
+                          Spelling<"pcreate">]> {
   let flangClass = "AccObjectListWithModifier";
-  let aliases = ["present_or_create", "pcreate"];
 }
 
 // 2.5.16
-def ACC_Default_none : ClauseVal<"none", 1, 1> { let isDefault = 1; }
-def ACC_Default_present : ClauseVal<"present", 0, 1> {}
+def ACC_Default_none : EnumVal<"none", 1, 1> { let isDefault = 1; }
+def ACC_Default_present : EnumVal<"present", 0, 1> {}
 
-def ACCC_Default : Clause<"default"> {
+def ACCC_Default : Clause<[Spelling<"default">]> {
   let flangClass = "AccDefaultClause";
   let enumClauseValue = "DefaultValue";
   let allowedClauseValues = [
@@ -99,174 +100,173 @@ def ACCC_Default : Clause<"default"> {
 }
 
 // 2.14.3
-def ACCC_DefaultAsync : Clause<"default_async"> {
+def ACCC_DefaultAsync : Clause<[Spelling<"default_async">]> {
   let flangClass = "ScalarIntExpr";
 }
 
 // 2.7.11
-def ACCC_Delete : Clause<"delete"> {
+def ACCC_Delete : Clause<[Spelling<"delete">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.7.13
-def ACCC_Detach : Clause<"detach"> {
+def ACCC_Detach : Clause<[Spelling<"detach">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.14.4
-def ACCC_Device : Clause<"device"> {
+def ACCC_Device : Clause<[Spelling<"device">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.14.1 - 2.14.2
-def ACCC_DeviceNum : Clause<"device_num">  {
+def ACCC_DeviceNum : Clause<[Spelling<"device_num">]>  {
   let flangClass = "ScalarIntExpr";
 }
 
 // 2.7.4
-def ACCC_DevicePtr : Clause<"deviceptr"> {
+def ACCC_DevicePtr : Clause<[Spelling<"deviceptr">]> {
   let flangClass = "AccObjectList";
   let clangAccSpelling = "DevicePtr";
 }
 
 // 2.13.1
-def ACCC_DeviceResident : Clause<"device_resident"> {
+def ACCC_DeviceResident : Clause<[Spelling<"device_resident">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.4
-def ACCC_DeviceType : Clause<"device_type"> {
+def ACCC_DeviceType : Clause<[Spelling<"device_type">, Spelling<"dtype">]> {
   let flangClass = "AccDeviceTypeExprList";
   let defaultValue = "*";
-  let aliases = ["dtype"];
 }
 
 // 2.6.6
-def ACCC_Finalize : Clause<"finalize"> {}
+def ACCC_Finalize : Clause<[Spelling<"finalize">]> {}
 
 // 2.5.14
-def ACCC_FirstPrivate : Clause<"firstprivate"> {
+def ACCC_FirstPrivate : Clause<[Spelling<"firstprivate">]> {
   let flangClass = "AccObjectList";
   let clangAccSpelling = "FirstPrivate";
 }
 
 // 2.9.2
-def ACCC_Gang : Clause<"gang"> {
+def ACCC_Gang : Clause<[Spelling<"gang">]> {
   let flangClass = "AccGangArgList";
   let isValueOptional = true;
 }
 
 // 2.14.4
-def ACCC_Host : Clause<"host"> {
+def ACCC_Host : Clause<[Spelling<"host">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.5.6
-def ACCC_If : Clause <"if"> {
+def ACCC_If : Clause<[Spelling<"if">]> {
   let flangClass = "ScalarExpr";
 }
 
 // 2.14.4
-def ACCC_IfPresent : Clause<"if_present"> {}
+def ACCC_IfPresent : Clause<[Spelling<"if_present">]> {}
 
 // 2.9.6
-def ACCC_Independent : Clause<"independent"> {}
+def ACCC_Independent : Clause<[Spelling<"independent">]> {}
 
 // 2.13.3
-def ACCC_Link : Clause<"link"> {
+def ACCC_Link : Clause<[Spelling<"link">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.7.10
-def ACCC_NoCreate : Clause<"no_create"> {
+def ACCC_NoCreate : Clause<[Spelling<"no_create">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.15.1
-def ACCC_NoHost : Clause<"nohost"> { 
-  let clangAccSpelling = "NoHost"; 
+def ACCC_NoHost : Clause<[Spelling<"nohost">]> {
+  let clangAccSpelling = "NoHost";
 }
 
 // 2.5.10
-def ACCC_NumGangs : Clause<"num_gangs"> {
+def ACCC_NumGangs : Clause<[Spelling<"num_gangs">]> {
   let flangClass = "ScalarIntExpr";
   let isValueList = 1;
 }
 
 // 2.5.11
-def ACCC_NumWorkers : Clause<"num_workers"> {
+def ACCC_NumWorkers : Clause<[Spelling<"num_workers">]> {
   let flangClass = "ScalarIntExpr";
 }
 
 // 2.7.5
-def ACCC_Present : Clause<"present"> {
+def ACCC_Present : Clause<[Spelling<"present">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.5.13
-def ACCC_Private : Clause<"private"> {
+def ACCC_Private : Clause<[Spelling<"private">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.9.8
-def ACCC_Tile : Clause <"tile"> {
+def ACCC_Tile : Clause<[Spelling<"tile">]> {
   let flangClass = "AccTileExprList";
 }
 
 // 2.8.1
-def ACCC_UseDevice : Clause <"use_device"> {
+def ACCC_UseDevice : Clause<[Spelling<"use_device">]> {
   let flangClass = "AccObjectList";
 }
 
 // 2.12
-def ACCC_Read : Clause<"read"> {}
+def ACCC_Read : Clause<[Spelling<"read">]> {}
 
 // 2.5.15
-def ACCC_Reduction : Clause<"reduction"> {
+def ACCC_Reduction : Clause<[Spelling<"reduction">]> {
   let flangClass = "AccObjectListWithReduction";
 }
 
 // 2.5.7
-def ACCC_Self : Clause<"self"> {
+def ACCC_Self : Clause<[Spelling<"self">]> {
   let flangClass = "AccSelfClause";
   let isValueOptional = true;
 }
 
 // 2.9.5
-def ACCC_Seq : Clause<"seq"> {}
+def ACCC_Seq : Clause<[Spelling<"seq">]> {}
 
 // Non-standard extension
-def ACCC_ShortLoop : Clause<"shortloop"> {}
+def ACCC_ShortLoop : Clause<[Spelling<"shortloop">]> {}
 
 // 2.9.4
-def ACCC_Vector : Clause<"vector"> {
+def ACCC_Vector : Clause<[Spelling<"vector">]> {
   let flangClass = "ScalarIntExpr";
   let isValueOptional = true;
   let prefix = "length";
 }
 
 // 2.5.12
-def ACCC_VectorLength : Clause<"vector_length"> {
+def ACCC_VectorLength : Clause<[Spelling<"vector_length">]> {
   let flangClass = "ScalarIntExpr";
 }
 
 // 2.16.2
-def ACCC_Wait : Clause<"wait"> {
+def ACCC_Wait : Clause<[Spelling<"wait">]> {
   let flangClass = "AccWaitArgument";
   let isValueOptional = true;
 }
 
 // 2.9.3
-def ACCC_Worker: Clause<"worker"> {
+def ACCC_Worker: Clause<[Spelling<"worker">]> {
   let flangClass = "ScalarIntExpr";
   let isValueOptional = true;
   let prefix = "num";
 }
 
 // 2.12
-def ACCC_Write : Clause<"write"> {}
+def ACCC_Write : Clause<[Spelling<"write">]> {}
 
-def ACCC_Unknown : Clause<"unknown"> {
+def ACCC_Unknown : Clause<[Spelling<"unknown">]> {
   let isDefault = true;
 }
 
@@ -275,14 +275,14 @@ def ACCC_Unknown : Clause<"unknown"> {
 //===----------------------------------------------------------------------===//
 
 // 2.12
-def ACC_Atomic : Directive<"atomic"> {
+def ACC_Atomic : Directive<[Spelling<"atomic">]> {
   let allowedOnceClauses = [VersionedClause<ACCC_If, 34>];
   let association = AS_Block;
   let category = CA_Executable;
 }
 
 // 2.6.5
-def ACC_Data : Directive<"data"> {
+def ACC_Data : Directive<[Spelling<"data">]> {
   let allowedOnceClauses = [
     VersionedClause<ACCC_If>,
     VersionedClause<ACCC_Default>
@@ -308,7 +308,7 @@ def ACC_Data : Directive<"data"> {
 }
 
 // 2.13
-def ACC_Declare : Directive<"declare"> {
+def ACC_Declare : Directive<[Spelling<"declare">]> {
   let allowedClauses = [
     VersionedClause<ACCC_Copy>,
     VersionedClause<ACCC_Copyin>,
@@ -324,7 +324,7 @@ def ACC_Declare : Directive<"declare"> {
 }
 
 // 2.5.3
-def ACC_Kernels : Directive<"kernels"> {
+def ACC_Kernels : Directive<[Spelling<"kernels">]> {
   let allowedClauses = [VersionedClause<ACCC_Async>,
                         VersionedClause<ACCC_Attach>,
                         VersionedClause<ACCC_Copy>,
@@ -347,7 +347,7 @@ def ACC_Kernels : Directive<"kernels"> {
 }
 
 // 2.5.1
-def ACC_Parallel : Directive<"parallel"> {
+def ACC_Parallel : Directive<[Spelling<"parallel">]> {
   let allowedClauses = [
     VersionedClause<ACCC_Attach>,
     VersionedClause<ACCC_Async>,
@@ -377,7 +377,7 @@ def ACC_Parallel : Directive<"parallel"> {
 }
 
 // 2.5.2
-def ACC_Serial : Directive<"serial"> {
+def ACC_Serial : Directive<[Spelling<"serial">]> {
   // Spec line 950-951: clause is as for the parallel construct except that the
   // num_gangs, num_workers, and vector_length clauses are not permitted.
   let allowedClauses = [VersionedClause<ACCC_Async>,
@@ -402,7 +402,7 @@ def ACC_Serial : Directive<"serial"> {
 }
 
 // 2.9
-def ACC_Loop : Directive<"loop"> {
+def ACC_Loop : Directive<[Spelling<"loop">]> {
   let allowedClauses = [
     VersionedClause<ACCC_DeviceType>,
     VersionedClause<ACCC_Private>,
@@ -424,13 +424,13 @@ def ACC_Loop : Directive<"loop"> {
 }
 
 // 2.10
-def ACC_Cache : Directive<"cache"> {
+def ACC_Cache : Directive<[Spelling<"cache">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
 
 // 2.14.1
-def ACC_Init : Directive<"init"> {
+def ACC_Init : Directive<[Spelling<"init">]> {
   let allowedOnceClauses = [VersionedClause<ACCC_DeviceNum>,
                             VersionedClause<ACCC_If>];
   let allowedClauses = [VersionedClause<ACCC_DeviceType>];
@@ -439,7 +439,7 @@ def ACC_Init : Directive<"init"> {
 }
 
 // 2.15.1
-def ACC_Routine : Directive<"routine"> {
+def ACC_Routine : Directive<[Spelling<"routine">]> {
   let allowedClauses = [
     VersionedClause<ACCC_Bind>,
     VersionedClause<ACCC_DeviceType>,
@@ -456,7 +456,7 @@ def ACC_Routine : Directive<"routine"> {
 }
 
 // 2.14.3
-def ACC_Set : Directive<"set"> {
+def ACC_Set : Directive<[Spelling<"set">]> {
   let allowedOnceClauses = [
     VersionedClause<ACCC_DefaultAsync>,
     VersionedClause<ACCC_DeviceNum>,
@@ -476,7 +476,7 @@ def ACC_Set : Directive<"set"> {
 }
 
 // 2.14.2
-def ACC_Shutdown : Directive<"shutdown"> {
+def ACC_Shutdown : Directive<[Spelling<"shutdown">]> {
   let allowedOnceClauses = [VersionedClause<ACCC_DeviceNum>,
                             VersionedClause<ACCC_If>];
   let allowedClauses = [VersionedClause<ACCC_DeviceType>];
@@ -485,7 +485,7 @@ def ACC_Shutdown : Directive<"shutdown"> {
 }
 
 // 2.14.4
-def ACC_Update : Directive<"update"> {
+def ACC_Update : Directive<[Spelling<"update">]> {
   let allowedClauses = [VersionedClause<ACCC_DeviceType>,
                         VersionedClause<ACCC_IfPresent>,
                         VersionedClause<ACCC_Wait>];
@@ -501,7 +501,7 @@ def ACC_Update : Directive<"update"> {
 }
 
 // 2.16.3
-def ACC_Wait : Directive<"wait"> {
+def ACC_Wait : Directive<[Spelling<"wait">]> {
   let allowedOnceClauses = [
     VersionedClause<ACCC_Async>,
     VersionedClause<ACCC_If>
@@ -511,7 +511,7 @@ def ACC_Wait : Directive<"wait"> {
 }
 
 // 2.14.6
-def ACC_EnterData : Directive<"enter data"> {
+def ACC_EnterData : Directive<[Spelling<"enter data">]> {
   let allowedClauses = [
     VersionedClause<ACCC_Wait>
   ];
@@ -529,7 +529,7 @@ def ACC_EnterData : Directive<"enter data"> {
 }
 
 // 2.14.7
-def ACC_ExitData : Directive<"exit data"> {
+def ACC_ExitData : Directive<[Spelling<"exit data">]> {
   let allowedClauses = [VersionedClause<ACCC_Finalize>,
                         VersionedClause<ACCC_Wait>];
   let allowedOnceClauses = [VersionedClause<ACCC_Async>,
@@ -544,7 +544,7 @@ def ACC_ExitData : Directive<"exit data"> {
 }
 
 // 2.8
-def ACC_HostData : Directive<"host_data"> {
+def ACC_HostData : Directive<[Spelling<"host_data">]> {
   let allowedClauses = [VersionedClause<ACCC_IfPresent>];
   let allowedOnceClauses = [VersionedClause<ACCC_If>];
   let requiredClauses = [
@@ -555,7 +555,7 @@ def ACC_HostData : Directive<"host_data"> {
 }
 
 // 2.11
-def ACC_KernelsLoop : Directive<"kernels loop"> {
+def ACC_KernelsLoop : Directive<[Spelling<"kernels loop">]> {
   let allowedClauses = [VersionedClause<ACCC_Async>,
                         VersionedClause<ACCC_Attach>,
                         VersionedClause<ACCC_Collapse>,
@@ -591,7 +591,7 @@ def ACC_KernelsLoop : Directive<"kernels loop"> {
 }
 
 // 2.11
-def ACC_ParallelLoop : Directive<"parallel loop"> {
+def ACC_ParallelLoop : Directive<[Spelling<"parallel loop">]> {
   let allowedClauses = [VersionedClause<ACCC_Async>,
                         VersionedClause<ACCC_Attach>,
                         VersionedClause<ACCC_Collapse>,
@@ -628,7 +628,7 @@ def ACC_ParallelLoop : Directive<"parallel loop"> {
 }
 
 // 2.11
-def ACC_SerialLoop : Directive<"serial loop"> {
+def ACC_SerialLoop : Directive<[Spelling<"serial loop">]> {
   let allowedClauses = [VersionedClause<ACCC_Async>,
                         VersionedClause<ACCC_Attach>,
                         VersionedClause<ACCC_Collapse>,
@@ -661,7 +661,7 @@ def ACC_SerialLoop : Directive<"serial loop"> {
   let category = CA_Executable;
 }
 
-def ACC_Unknown : Directive<"unknown"> {
+def ACC_Unknown : Directive<[Spelling<"unknown">]> {
   let isDefault = true;
   let association = AS_None;
   let category = CA_Utility;
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.h b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.h
index c3381705093a..35dafc6d246f 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -14,23 +14,24 @@
 #define LLVM_FRONTEND_OPENMP_OMP_H
 
 #include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Support/Compiler.h"
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace llvm::omp {
-ArrayRef<Directive> getLeafConstructs(Directive D);
-ArrayRef<Directive> getLeafConstructsOrSelf(Directive D);
+LLVM_ABI ArrayRef<Directive> getLeafConstructs(Directive D);
+LLVM_ABI ArrayRef<Directive> getLeafConstructsOrSelf(Directive D);
 
-ArrayRef<Directive>
+LLVM_ABI ArrayRef<Directive>
 getLeafOrCompositeConstructs(Directive D, SmallVectorImpl<Directive> &Output);
 
-Directive getCompoundConstruct(ArrayRef<Directive> Parts);
+LLVM_ABI Directive getCompoundConstruct(ArrayRef<Directive> Parts);
 
-bool isLeafConstruct(Directive D);
-bool isCompositeConstruct(Directive D);
-bool isCombinedConstruct(Directive D);
+LLVM_ABI bool isLeafConstruct(Directive D);
+LLVM_ABI bool isCompositeConstruct(Directive D);
+LLVM_ABI bool isCombinedConstruct(Directive D);
 
 /// Can clause C have an iterator-modifier.
 static constexpr inline bool canHaveIterator(Clause C) {
@@ -48,14 +49,15 @@ static constexpr inline bool canHaveIterator(Clause C) {
 }
 
 static constexpr unsigned FallbackVersion = 52;
-ArrayRef<unsigned> getOpenMPVersions();
+LLVM_ABI ArrayRef<unsigned> getOpenMPVersions();
 
 /// Create a nicer version of a function name for humans to look at.
-std::string prettifyFunctionName(StringRef FunctionName);
+LLVM_ABI std::string prettifyFunctionName(StringRef FunctionName);
 
 /// Deconstruct an OpenMP kernel name into the parent function name and the line
 /// number.
-std::string deconstructOpenMPKernelName(StringRef KernelName, unsigned &LineNo);
+LLVM_ABI std::string deconstructOpenMPKernelName(StringRef KernelName,
+                                                 unsigned &LineNo);
 
 } // namespace llvm::omp
 
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.td b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 0af4b436649a..027692275b63 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -32,55 +32,56 @@ def OpenMP : DirectiveLanguage {
 // Sorted alphabetically wrt clause spelling.
 //===----------------------------------------------------------------------===//
 
-def OMPC_Absent : Clause<"absent"> {
+def OMPC_Absent : Clause<[Spelling<"absent">]> {
   let clangClass = "OMPAbsentClause";
   let flangClass = "OmpAbsentClause";
 }
-def OMPC_Acquire : Clause<"acquire"> {
+def OMPC_Acquire : Clause<[Spelling<"acquire">]> {
   let clangClass = "OMPAcquireClause";
 }
-def OMPC_AcqRel : Clause<"acq_rel"> {
+def OMPC_AcqRel : Clause<[Spelling<"acq_rel">]> {
   let clangClass = "OMPAcqRelClause";
 }
-def OMPC_AdjustArgs : Clause<"adjust_args"> {
+def OMPC_AdjustArgs : Clause<[Spelling<"adjust_args">]> {
   let flangClass = "OmpAdjustArgsClause";
 }
-def OMPC_Affinity : Clause<"affinity"> {
+def OMPC_Affinity : Clause<[Spelling<"affinity">]> {
   let clangClass = "OMPAffinityClause";
   let flangClass = "OmpAffinityClause";
 }
-def OMPC_Align : Clause<"align"> {
+def OMPC_Align : Clause<[Spelling<"align">]> {
   let clangClass = "OMPAlignClause";
   let flangClass = "OmpAlignClause";
 }
-def OMPC_Aligned : Clause<"aligned"> {
+def OMPC_Aligned : Clause<[Spelling<"aligned">]> {
   let clangClass = "OMPAlignedClause";
   let flangClass = "OmpAlignedClause";
 }
-def OMPC_Allocate : Clause<"allocate"> {
+def OMPC_Allocate : Clause<[Spelling<"allocate">]> {
   let clangClass = "OMPAllocateClause";
   let flangClass = "OmpAllocateClause";
 }
-def OMPC_Allocator : Clause<"allocator"> {
+def OMPC_Allocator : Clause<[Spelling<"allocator">]> {
   let clangClass = "OMPAllocatorClause";
   let flangClass = "ScalarIntExpr";
 }
-def OMPC_AppendArgs : Clause<"append_args"> {
+def OMPC_AppendArgs : Clause<[Spelling<"append_args">]> {
   let flangClass = "OmpAppendArgsClause";
 }
-def OMPC_At : Clause<"at"> {
+def OMPC_At : Clause<[Spelling<"at">]> {
   let clangClass = "OMPAtClause";
   let flangClass = "OmpAtClause";
 }
-def OMPC_AtomicDefaultMemOrder : Clause<"atomic_default_mem_order"> {
+def OMPC_AtomicDefaultMemOrder
+    : Clause<[Spelling<"atomic_default_mem_order">]> {
   let clangClass = "OMPAtomicDefaultMemOrderClause";
   let flangClass = "OmpAtomicDefaultMemOrderClause";
 }
 
-def OMP_BIND_parallel : ClauseVal<"parallel",1,1> {}
-def OMP_BIND_teams : ClauseVal<"teams",2,1> {}
-def OMP_BIND_thread : ClauseVal<"thread",3,1> { let isDefault = true; }
-def OMPC_Bind : Clause<"bind"> {
+def OMP_BIND_parallel : EnumVal<"parallel",1,1> {}
+def OMP_BIND_teams : EnumVal<"teams",2,1> {}
+def OMP_BIND_thread : EnumVal<"thread",3,1> { let isDefault = true; }
+def OMPC_Bind : Clause<[Spelling<"bind">]> {
   let clangClass = "OMPBindClause";
   let flangClass = "OmpBindClause";
   let enumClauseValue = "BindKind";
@@ -91,14 +92,15 @@ def OMPC_Bind : Clause<"bind"> {
   ];
 }
 
-def OMP_CANCELLATION_CONSTRUCT_Parallel : ClauseVal<"parallel", 1, 1> {}
-def OMP_CANCELLATION_CONSTRUCT_Loop : ClauseVal<"loop", 2, 1> {}
-def OMP_CANCELLATION_CONSTRUCT_Sections : ClauseVal<"sections", 3, 1> {}
-def OMP_CANCELLATION_CONSTRUCT_Taskgroup : ClauseVal<"taskgroup", 4, 1> {}
-def OMP_CANCELLATION_CONSTRUCT_None : ClauseVal<"none", 5, 0> {
+def OMP_CANCELLATION_CONSTRUCT_Parallel : EnumVal<"parallel", 1, 1> {}
+def OMP_CANCELLATION_CONSTRUCT_Loop : EnumVal<"loop", 2, 1> {}
+def OMP_CANCELLATION_CONSTRUCT_Sections : EnumVal<"sections", 3, 1> {}
+def OMP_CANCELLATION_CONSTRUCT_Taskgroup : EnumVal<"taskgroup", 4, 1> {}
+def OMP_CANCELLATION_CONSTRUCT_None : EnumVal<"none", 5, 0> {
   let isDefault = 1;
 }
-def OMPC_CancellationConstructType : Clause<"cancellation_construct_type"> {
+def OMPC_CancellationConstructType
+    : Clause<[Spelling<"cancellation_construct_type">]> {
   let enumClauseValue = "CancellationConstructType";
   let allowedClauseValues = [
     OMP_CANCELLATION_CONSTRUCT_Parallel,
@@ -110,109 +112,109 @@ def OMPC_CancellationConstructType : Clause<"cancellation_construct_type"> {
   let flangClass = "OmpCancellationConstructTypeClause";
   let skipFlangUnparser = true;
 }
-def OMPC_Contains : Clause<"contains"> {
+def OMPC_Contains : Clause<[Spelling<"contains">]> {
   let clangClass = "OMPContainsClause";
   let flangClass = "OmpContainsClause";
 }
-def OMPC_Capture : Clause<"capture"> {
+def OMPC_Capture : Clause<[Spelling<"capture">]> {
   let clangClass = "OMPCaptureClause";
 }
-def OMPC_Collapse : Clause<"collapse"> {
+def OMPC_Collapse : Clause<[Spelling<"collapse">]> {
   let clangClass = "OMPCollapseClause";
   let flangClass = "ScalarIntConstantExpr";
 }
-def OMPC_Compare : Clause<"compare"> {
+def OMPC_Compare : Clause<[Spelling<"compare">]> {
   let clangClass = "OMPCompareClause";
 }
-def OMPC_Copyin : Clause<"copyin"> {
+def OMPC_Copyin : Clause<[Spelling<"copyin">]> {
   let clangClass = "OMPCopyinClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_CopyPrivate : Clause<"copyprivate"> {
+def OMPC_CopyPrivate : Clause<[Spelling<"copyprivate">]> {
   let clangClass = "OMPCopyprivateClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_Default : Clause<"default"> {
+def OMPC_Default : Clause<[Spelling<"default">]> {
   let clangClass = "OMPDefaultClause";
   let flangClass = "OmpDefaultClause";
 }
-def OMPC_DefaultMap : Clause<"defaultmap"> {
+def OMPC_DefaultMap : Clause<[Spelling<"defaultmap">]> {
   let clangClass = "OMPDefaultmapClause";
   let flangClass = "OmpDefaultmapClause";
 }
-def OMPC_Depend : Clause<"depend"> {
+def OMPC_Depend : Clause<[Spelling<"depend">]> {
   let clangClass = "OMPDependClause";
   let flangClass = "OmpDependClause";
 }
-def OMPC_Depobj : Clause<"depobj"> {
+def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let clangClass = "OMPDepobjClause";
   let isImplicit = true;
 }
-def OMPC_Destroy : Clause<"destroy"> {
+def OMPC_Destroy : Clause<[Spelling<"destroy">]> {
   let clangClass = "OMPDestroyClause";
   let flangClass = "OmpDestroyClause";
   let isValueOptional = true;
 }
-def OMPC_Detach : Clause<"detach"> {
+def OMPC_Detach : Clause<[Spelling<"detach">]> {
   let clangClass = "OMPDetachClause";
   let flangClass = "OmpDetachClause";
 }
-def OMPC_Device : Clause<"device"> {
+def OMPC_Device : Clause<[Spelling<"device">]> {
   let clangClass = "OMPDeviceClause";
   let flangClass = "OmpDeviceClause";
 }
-def OMPC_DeviceType : Clause<"device_type"> {
+def OMPC_DeviceType : Clause<[Spelling<"device_type">]> {
   let flangClass = "OmpDeviceTypeClause";
 }
-def OMPC_DistSchedule : Clause<"dist_schedule"> {
+def OMPC_DistSchedule : Clause<[Spelling<"dist_schedule">]> {
   let clangClass = "OMPDistScheduleClause";
   let flangClass = "ScalarIntExpr";
   let isValueOptional = true;
 }
-def OMPC_Doacross : Clause<"doacross"> {
+def OMPC_Doacross : Clause<[Spelling<"doacross">]> {
   let clangClass = "OMPDoacrossClause";
   let flangClass = "OmpDoacrossClause";
 }
-def OMPC_DynamicAllocators : Clause<"dynamic_allocators"> {
+def OMPC_DynamicAllocators : Clause<[Spelling<"dynamic_allocators">]> {
   let clangClass = "OMPDynamicAllocatorsClause";
 }
-def OMPC_Enter : Clause<"enter"> {
+def OMPC_Enter : Clause<[Spelling<"enter">]> {
   let flangClass = "OmpObjectList";
 }
-def OMPC_Exclusive : Clause<"exclusive"> {
+def OMPC_Exclusive : Clause<[Spelling<"exclusive">]> {
   let clangClass = "OMPExclusiveClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_Fail : Clause<"fail"> {
+def OMPC_Fail : Clause<[Spelling<"fail">]> {
   let clangClass = "OMPFailClause";
   let flangClass = "OmpFailClause";
 }
-def OMPC_Filter : Clause<"filter"> {
+def OMPC_Filter : Clause<[Spelling<"filter">]> {
   let clangClass = "OMPFilterClause";
   let flangClass = "ScalarIntExpr";
 }
-def OMPC_Final : Clause<"final"> {
+def OMPC_Final : Clause<[Spelling<"final">]> {
   let clangClass = "OMPFinalClause";
   let flangClass = "ScalarLogicalExpr";
 }
-def OMPC_FirstPrivate : Clause<"firstprivate"> {
+def OMPC_FirstPrivate : Clause<[Spelling<"firstprivate">]> {
   let clangClass = "OMPFirstprivateClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_Flush : Clause<"flush"> {
+def OMPC_Flush : Clause<[Spelling<"flush">]> {
   let clangClass = "OMPFlushClause";
   let isImplicit = true;
 }
-def OMPC_From : Clause<"from"> {
+def OMPC_From : Clause<[Spelling<"from">]> {
   let clangClass = "OMPFromClause";
   let flangClass = "OmpFromClause";
 }
-def OMPC_Full: Clause<"full"> {
+def OMPC_Full: Clause<[Spelling<"full">]> {
   let clangClass = "OMPFullClause";
 }
-def OMP_GRAINSIZE_Strict : ClauseVal<"strict", 1, 1> {}
-def OMP_GRAINSIZE_Unknown : ClauseVal<"unknown", 2, 0> { let isDefault = 1; }
-def OMPC_GrainSize : Clause<"grainsize"> {
+def OMP_GRAINSIZE_Strict : EnumVal<"strict", 1, 1> {}
+def OMP_GRAINSIZE_Unknown : EnumVal<"unknown", 2, 0> { let isDefault = 1; }
+def OMPC_GrainSize : Clause<[Spelling<"grainsize">]> {
   let clangClass = "OMPGrainsizeClause";
   let flangClass = "OmpGrainsizeClause";
   let enumClauseValue = "GrainsizeType";
@@ -221,72 +223,72 @@ def OMPC_GrainSize : Clause<"grainsize"> {
     OMP_GRAINSIZE_Unknown
   ];
 }
-def OMPC_HasDeviceAddr : Clause<"has_device_addr"> {
+def OMPC_HasDeviceAddr : Clause<[Spelling<"has_device_addr">]> {
   let clangClass = "OMPHasDeviceAddrClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_Hint : Clause<"hint"> {
+def OMPC_Hint : Clause<[Spelling<"hint">]> {
   let clangClass = "OMPHintClause";
   let flangClass = "OmpHintClause";
 }
-def OMPC_Holds : Clause<"holds"> {
+def OMPC_Holds : Clause<[Spelling<"holds">]> {
   let clangClass = "OMPHoldsClause";
   let flangClass = "OmpHoldsClause";
 }
-def OMPC_If : Clause<"if"> {
+def OMPC_If : Clause<[Spelling<"if">]> {
   let clangClass = "OMPIfClause";
   let flangClass = "OmpIfClause";
 }
-def OMPC_Inbranch : Clause<"inbranch"> {
+def OMPC_Inbranch : Clause<[Spelling<"inbranch">]> {
 }
-def OMPC_Inclusive : Clause<"inclusive"> {
+def OMPC_Inclusive : Clause<[Spelling<"inclusive">]> {
   let clangClass = "OMPInclusiveClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_Indirect : Clause<"indirect"> {
+def OMPC_Indirect : Clause<[Spelling<"indirect">]> {
 }
-def OMPC_Init : Clause<"init"> {
+def OMPC_Init : Clause<[Spelling<"init">]> {
   let clangClass = "OMPInitClause";
   let flangClass = "OmpInitClause";
 }
-def OMPC_Initializer : Clause<"initializer"> {
+def OMPC_Initializer : Clause<[Spelling<"initializer">]> {
   let flangClass = "OmpInitializerClause";
 }
-def OMPC_InReduction : Clause<"in_reduction"> {
+def OMPC_InReduction : Clause<[Spelling<"in_reduction">]> {
   let clangClass = "OMPInReductionClause";
   let flangClass = "OmpInReductionClause";
 }
-def OMPC_IsDevicePtr : Clause<"is_device_ptr"> {
+def OMPC_IsDevicePtr : Clause<[Spelling<"is_device_ptr">]> {
   let clangClass = "OMPIsDevicePtrClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_LastPrivate : Clause<"lastprivate"> {
+def OMPC_LastPrivate : Clause<[Spelling<"lastprivate">]> {
   let clangClass = "OMPLastprivateClause";
   let flangClass = "OmpLastprivateClause";
 }
-def OMPC_Linear : Clause<"linear"> {
+def OMPC_Linear : Clause<[Spelling<"linear">]> {
   let clangClass = "OMPLinearClause";
   let flangClass = "OmpLinearClause";
 }
-def OMPC_Link : Clause<"link"> {
+def OMPC_Link : Clause<[Spelling<"link">]> {
   let flangClass = "OmpObjectList";
 }
-def OMPC_Map : Clause<"map"> {
+def OMPC_Map : Clause<[Spelling<"map">]> {
   let clangClass = "OMPMapClause";
   let flangClass = "OmpMapClause";
 }
-def OMPC_Match : Clause<"match"> {
+def OMPC_Match : Clause<[Spelling<"match">]> {
   let flangClass = "OmpMatchClause";
 }
-def OMP_MEMORY_ORDER_SeqCst : ClauseVal<"seq_cst", 1, 1> {}
-def OMP_MEMORY_ORDER_AcqRel : ClauseVal<"acq_rel", 2, 1> {}
-def OMP_MEMORY_ORDER_Acquire : ClauseVal<"acquire", 3, 1> {}
-def OMP_MEMORY_ORDER_Release : ClauseVal<"release", 4, 1> {}
-def OMP_MEMORY_ORDER_Relaxed : ClauseVal<"relaxed", 5, 1> {}
-def OMP_MEMORY_ORDER_Default : ClauseVal<"default", 6, 0> {
+def OMP_MEMORY_ORDER_SeqCst : EnumVal<"seq_cst", 1, 1> {}
+def OMP_MEMORY_ORDER_AcqRel : EnumVal<"acq_rel", 2, 1> {}
+def OMP_MEMORY_ORDER_Acquire : EnumVal<"acquire", 3, 1> {}
+def OMP_MEMORY_ORDER_Release : EnumVal<"release", 4, 1> {}
+def OMP_MEMORY_ORDER_Relaxed : EnumVal<"relaxed", 5, 1> {}
+def OMP_MEMORY_ORDER_Default : EnumVal<"default", 6, 0> {
   let isDefault = 1;
 }
-def OMPC_MemoryOrder : Clause<"memory_order"> {
+def OMPC_MemoryOrder : Clause<[Spelling<"memory_order">]> {
   let enumClauseValue = "MemoryOrderKind";
   let allowedClauseValues = [
     OMP_MEMORY_ORDER_SeqCst,
@@ -297,49 +299,49 @@ def OMPC_MemoryOrder : Clause<"memory_order"> {
     OMP_MEMORY_ORDER_Default
   ];
 }
-def OMPC_Mergeable : Clause<"mergeable"> {
+def OMPC_Mergeable : Clause<[Spelling<"mergeable">]> {
   let clangClass = "OMPMergeableClause";
 }
-def OMPC_Message : Clause<"message"> {
+def OMPC_Message : Clause<[Spelling<"message">]> {
   let clangClass = "OMPMessageClause";
   let flangClass = "OmpMessageClause";
 }
-def OMPC_NoOpenMP : Clause<"no_openmp"> {
+def OMPC_NoOpenMP : Clause<[Spelling<"no_openmp">]> {
   let clangClass = "OMPNoOpenMPClause";
 }
-def OMPC_NoOpenMPRoutines : Clause<"no_openmp_routines"> {
+def OMPC_NoOpenMPRoutines : Clause<[Spelling<"no_openmp_routines">]> {
   let clangClass = "OMPNoOpenMPRoutinesClause";
 }
-def OMPC_NoOpenMPConstructs : Clause<"no_openmp_constructs"> {
+def OMPC_NoOpenMPConstructs : Clause<[Spelling<"no_openmp_constructs">]> {
   let clangClass = "OMPNoOpenMPConstructsClause";
 }
-def OMPC_NoParallelism : Clause<"no_parallelism"> {
+def OMPC_NoParallelism : Clause<[Spelling<"no_parallelism">]> {
   let clangClass = "OMPNoParallelismClause";
 }
-def OMPC_Nocontext : Clause<"nocontext"> {
+def OMPC_Nocontext : Clause<[Spelling<"nocontext">]> {
   let clangClass = "OMPNocontextClause";
   let flangClass = "ScalarLogicalExpr";
 }
-def OMPC_NoGroup : Clause<"nogroup"> {
+def OMPC_NoGroup : Clause<[Spelling<"nogroup">]> {
   let clangClass = "OMPNogroupClause";
 }
-def OMPC_NonTemporal : Clause<"nontemporal"> {
+def OMPC_NonTemporal : Clause<[Spelling<"nontemporal">]> {
   let clangClass = "OMPNontemporalClause";
   let flangClass = "Name";
   let isValueList = true;
 }
-def OMPC_Notinbranch : Clause<"notinbranch"> {
+def OMPC_Notinbranch : Clause<[Spelling<"notinbranch">]> {
 }
-def OMPC_Novariants : Clause<"novariants"> {
+def OMPC_Novariants : Clause<[Spelling<"novariants">]> {
   let clangClass = "OMPNovariantsClause";
   let flangClass = "ScalarLogicalExpr";
 }
-def OMPC_NoWait : Clause<"nowait"> {
+def OMPC_NoWait : Clause<[Spelling<"nowait">]> {
   let clangClass = "OMPNowaitClause";
 }
-def OMP_NUMTASKS_Strict : ClauseVal<"strict", 1, 1> {}
-def OMP_NUMTASKS_Unknown : ClauseVal<"unknown", 2, 0> { let isDefault = 1; }
-def OMPC_NumTasks : Clause<"num_tasks"> {
+def OMP_NUMTASKS_Strict : EnumVal<"strict", 1, 1> {}
+def OMP_NUMTASKS_Unknown : EnumVal<"unknown", 2, 0> { let isDefault = 1; }
+def OMPC_NumTasks : Clause<[Spelling<"num_tasks">]> {
   let clangClass = "OMPNumTasksClause";
   let flangClass = "OmpNumTasksClause";
   let enumClauseValue = "NumTasksType";
@@ -348,27 +350,27 @@ def OMPC_NumTasks : Clause<"num_tasks"> {
     OMP_NUMTASKS_Unknown
   ];
 }
-def OMPC_NumTeams : Clause<"num_teams"> {
+def OMPC_NumTeams : Clause<[Spelling<"num_teams">]> {
   let clangClass = "OMPNumTeamsClause";
   let flangClass = "ScalarIntExpr";
 }
-def OMPC_NumThreads : Clause<"num_threads"> {
+def OMPC_NumThreads : Clause<[Spelling<"num_threads">]> {
   let clangClass = "OMPNumThreadsClause";
   let flangClass = "ScalarIntExpr";
 }
-def OMPC_OMPX_Attribute : Clause<"ompx_attribute"> {
+def OMPC_OMPX_Attribute : Clause<[Spelling<"ompx_attribute">]> {
   let clangClass = "OMPXAttributeClause";
 }
-def OMPC_OMPX_Bare : Clause<"ompx_bare"> {
+def OMPC_OMPX_Bare : Clause<[Spelling<"ompx_bare">]> {
   let clangClass = "OMPXBareClause";
 }
-def OMPC_OMPX_DynCGroupMem : Clause<"ompx_dyn_cgroup_mem"> {
+def OMPC_OMPX_DynCGroupMem : Clause<[Spelling<"ompx_dyn_cgroup_mem">]> {
   let clangClass = "OMPXDynCGroupMemClause";
   let flangClass = "ScalarIntExpr";
 }
-def OMP_ORDER_concurrent : ClauseVal<"concurrent",1,1> {}
-def OMP_ORDER_unknown : ClauseVal<"unknown",2,0> { let isDefault = 1; }
-def OMPC_Order : Clause<"order"> {
+def OMP_ORDER_concurrent : EnumVal<"concurrent",1,1> {}
+def OMP_ORDER_unknown : EnumVal<"unknown",2,0> { let isDefault = 1; }
+def OMPC_Order : Clause<[Spelling<"order">]> {
   let clangClass = "OMPOrderClause";
   let flangClass = "OmpOrderClause";
   let enumClauseValue = "OrderKind";
@@ -377,40 +379,40 @@ def OMPC_Order : Clause<"order"> {
     OMP_ORDER_concurrent
   ];
 }
-def OMPC_Ordered : Clause<"ordered"> {
+def OMPC_Ordered : Clause<[Spelling<"ordered">]> {
   let clangClass = "OMPOrderedClause";
   let flangClass = "ScalarIntConstantExpr";
   let isValueOptional = true;
 }
-def OMPC_Otherwise : Clause<"otherwise"> {
+def OMPC_Otherwise : Clause<[Spelling<"otherwise">]> {
   let flangClass = "OmpOtherwiseClause";
   let isValueOptional = true;
 }
-def OMPC_Partial: Clause<"partial"> {
+def OMPC_Partial: Clause<[Spelling<"partial">]> {
   let clangClass = "OMPPartialClause";
   let flangClass = "ScalarIntConstantExpr";
   let isValueOptional = true;
 }
-def OMPC_Permutation: Clause<"permutation"> {
+def OMPC_Permutation: Clause<[Spelling<"permutation">]> {
   let clangClass = "OMPPermutationClause";
   let flangClass = "ScalarIntExpr";
   let isValueList = true;
 }
-def OMPC_Priority : Clause<"priority"> {
+def OMPC_Priority : Clause<[Spelling<"priority">]> {
   let clangClass = "OMPPriorityClause";
   let flangClass = "ScalarIntExpr";
 }
-def OMPC_Private : Clause<"private"> {
+def OMPC_Private : Clause<[Spelling<"private">]> {
   let clangClass = "OMPPrivateClause";
   let flangClass = "OmpObjectList";
 }
-def OMP_PROC_BIND_master : ClauseVal<"master",2,1> {}
-def OMP_PROC_BIND_close : ClauseVal<"close",3,1> {}
-def OMP_PROC_BIND_spread : ClauseVal<"spread",4,1> {}
-def OMP_PROC_BIND_primary : ClauseVal<"primary",5,1> {}
-def OMP_PROC_BIND_default : ClauseVal<"default",6,0> {}
-def OMP_PROC_BIND_unknown : ClauseVal<"unknown",7,0> { let isDefault = true; }
-def OMPC_ProcBind : Clause<"proc_bind"> {
+def OMP_PROC_BIND_master : EnumVal<"master",2,1> {}
+def OMP_PROC_BIND_close : EnumVal<"close",3,1> {}
+def OMP_PROC_BIND_spread : EnumVal<"spread",4,1> {}
+def OMP_PROC_BIND_primary : EnumVal<"primary",5,1> {}
+def OMP_PROC_BIND_default : EnumVal<"default",6,0> {}
+def OMP_PROC_BIND_unknown : EnumVal<"unknown",7,0> { let isDefault = true; }
+def OMPC_ProcBind : Clause<[Spelling<"proc_bind">]> {
   let clangClass = "OMPProcBindClause";
   let flangClass = "OmpProcBindClause";
   let enumClauseValue = "ProcBindKind";
@@ -423,33 +425,33 @@ def OMPC_ProcBind : Clause<"proc_bind"> {
     OMP_PROC_BIND_unknown
   ];
 }
-def OMPC_Read : Clause<"read"> {
+def OMPC_Read : Clause<[Spelling<"read">]> {
   let clangClass = "OMPReadClause";
 }
-def OMPC_Reduction : Clause<"reduction"> {
+def OMPC_Reduction : Clause<[Spelling<"reduction">]> {
   let clangClass = "OMPReductionClause";
   let flangClass = "OmpReductionClause";
 }
-def OMPC_Relaxed : Clause<"relaxed"> {
+def OMPC_Relaxed : Clause<[Spelling<"relaxed">]> {
   let clangClass = "OMPRelaxedClause";
 }
-def OMPC_Release : Clause<"release"> {
+def OMPC_Release : Clause<[Spelling<"release">]> {
   let clangClass = "OMPReleaseClause";
 }
-def OMPC_ReverseOffload : Clause<"reverse_offload"> {
+def OMPC_ReverseOffload : Clause<[Spelling<"reverse_offload">]> {
   let clangClass = "OMPReverseOffloadClause";
 }
-def OMPC_SafeLen : Clause<"safelen"> {
+def OMPC_SafeLen : Clause<[Spelling<"safelen">]> {
   let clangClass = "OMPSafelenClause";
   let flangClass = "ScalarIntConstantExpr";
 }
-def OMP_SCHEDULE_Static : ClauseVal<"static", 2, 1> {}
-def OMP_SCHEDULE_Dynamic : ClauseVal<"dynamic", 3, 1> {}
-def OMP_SCHEDULE_Guided : ClauseVal<"guided", 4, 1> {}
-def OMP_SCHEDULE_Auto : ClauseVal<"auto", 5, 1> {}
-def OMP_SCHEDULE_Runtime : ClauseVal<"runtime", 6, 1> {}
-def OMP_SCHEDULE_Default : ClauseVal<"default", 7, 0> { let isDefault = 1; }
-def OMPC_Schedule : Clause<"schedule"> {
+def OMP_SCHEDULE_Static : EnumVal<"static", 2, 1> {}
+def OMP_SCHEDULE_Dynamic : EnumVal<"dynamic", 3, 1> {}
+def OMP_SCHEDULE_Guided : EnumVal<"guided", 4, 1> {}
+def OMP_SCHEDULE_Auto : EnumVal<"auto", 5, 1> {}
+def OMP_SCHEDULE_Runtime : EnumVal<"runtime", 6, 1> {}
+def OMP_SCHEDULE_Default : EnumVal<"default", 7, 0> { let isDefault = 1; }
+def OMPC_Schedule : Clause<[Spelling<"schedule">]> {
   let clangClass = "OMPScheduleClause";
   let flangClass = "OmpScheduleClause";
   let enumClauseValue = "ScheduleKind";
@@ -462,94 +464,94 @@ def OMPC_Schedule : Clause<"schedule"> {
     OMP_SCHEDULE_Default
   ];
 }
-def OMPC_SeqCst : Clause<"seq_cst"> {
+def OMPC_SeqCst : Clause<[Spelling<"seq_cst">]> {
   let clangClass = "OMPSeqCstClause";
 }
-def OMPC_Severity : Clause<"severity"> {
+def OMPC_Severity : Clause<[Spelling<"severity">]> {
   let clangClass = "OMPSeverityClause";
   let flangClass = "OmpSeverityClause";
 }
-def OMPC_Shared : Clause<"shared"> {
+def OMPC_Shared : Clause<[Spelling<"shared">]> {
   let clangClass = "OMPSharedClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_Simd : Clause<"simd"> {
+def OMPC_Simd : Clause<[Spelling<"simd">]> {
   let clangClass = "OMPSIMDClause";
 }
-def OMPC_SimdLen : Clause<"simdlen"> {
+def OMPC_SimdLen : Clause<[Spelling<"simdlen">]> {
   let clangClass = "OMPSimdlenClause";
   let flangClass = "ScalarIntConstantExpr";
 }
-def OMPC_Sizes: Clause<"sizes"> {
+def OMPC_Sizes: Clause<[Spelling<"sizes">]> {
   let clangClass = "OMPSizesClause";
   let flangClass = "ScalarIntExpr";
   let isValueList = true;
 }
-def OMPC_TaskReduction : Clause<"task_reduction"> {
+def OMPC_TaskReduction : Clause<[Spelling<"task_reduction">]> {
   let clangClass = "OMPTaskReductionClause";
   let flangClass = "OmpTaskReductionClause";
 }
-def OMPC_ThreadLimit : Clause<"thread_limit"> {
+def OMPC_ThreadLimit : Clause<[Spelling<"thread_limit">]> {
   let clangClass = "OMPThreadLimitClause";
   let flangClass = "ScalarIntExpr";
 }
-def OMPC_ThreadPrivate : Clause<"threadprivate"> {
+def OMPC_ThreadPrivate : Clause<[Spelling<"threadprivate">]> {
   let isImplicit = true;
 }
-def OMPC_Threads : Clause<"threads"> {
+def OMPC_Threads : Clause<[Spelling<"threads">]> {
   let clangClass = "OMPThreadsClause";
 }
-def OMPC_To : Clause<"to"> {
+def OMPC_To : Clause<[Spelling<"to">]> {
   let clangClass = "OMPToClause";
   let flangClass = "OmpToClause";
 }
-def OMPC_UnifiedAddress : Clause<"unified_address"> {
+def OMPC_UnifiedAddress : Clause<[Spelling<"unified_address">]> {
   let clangClass = "OMPUnifiedAddressClause";
 }
-def OMPC_UnifiedSharedMemory : Clause<"unified_shared_memory"> {
+def OMPC_UnifiedSharedMemory : Clause<[Spelling<"unified_shared_memory">]> {
   let clangClass = "OMPUnifiedSharedMemoryClause";
 }
-def OMPC_SelfMaps : Clause<"self_maps"> {
+def OMPC_SelfMaps : Clause<[Spelling<"self_maps">]> {
   let clangClass = "OMPSelfMapsClause";
 }
-def OMPC_Uniform : Clause<"uniform"> {
+def OMPC_Uniform : Clause<[Spelling<"uniform">]> {
   let flangClass = "Name";
   let isValueList = true;
 }
-def OMPC_Unknown : Clause<"unknown"> {
+def OMPC_Unknown : Clause<[Spelling<"unknown">]> {
   let isImplicit = true;
   let isDefault = true;
 }
-def OMPC_Untied : Clause<"untied"> {
+def OMPC_Untied : Clause<[Spelling<"untied">]> {
   let clangClass = "OMPUntiedClause";
 }
-def OMPC_Update : Clause<"update"> {
+def OMPC_Update : Clause<[Spelling<"update">]> {
   let clangClass = "OMPUpdateClause";
   let flangClass = "OmpUpdateClause";
   let isValueOptional = true;
 }
-def OMPC_Use : Clause<"use"> {
+def OMPC_Use : Clause<[Spelling<"use">]> {
   let clangClass = "OMPUseClause";
   let flangClass = "OmpUseClause";
 }
-def OMPC_UsesAllocators : Clause<"uses_allocators"> {
+def OMPC_UsesAllocators : Clause<[Spelling<"uses_allocators">]> {
   let clangClass = "OMPUsesAllocatorsClause";
 }
-def OMPC_UseDeviceAddr : Clause<"use_device_addr"> {
+def OMPC_UseDeviceAddr : Clause<[Spelling<"use_device_addr">]> {
   let clangClass = "OMPUseDeviceAddrClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_UseDevicePtr : Clause<"use_device_ptr"> {
+def OMPC_UseDevicePtr : Clause<[Spelling<"use_device_ptr">]> {
   let clangClass = "OMPUseDevicePtrClause";
   let flangClass = "OmpObjectList";
 }
-def OMPC_Weak : Clause<"weak"> {
+def OMPC_Weak : Clause<[Spelling<"weak">]> {
   let clangClass = "OMPWeakClause";
 }
-def OMPC_When: Clause<"when"> {
+def OMPC_When: Clause<[Spelling<"when">]> {
   let flangClass = "OmpWhenClause";
 }
-def OMPC_Write : Clause<"write"> {
+def OMPC_Write : Clause<[Spelling<"write">]> {
   let clangClass = "OMPWriteClause";
 }
 
@@ -559,7 +561,7 @@ def OMPC_Write : Clause<"write"> {
 // follows "xyz".
 //===----------------------------------------------------------------------===//
 
-def OMP_Allocate : Directive<"allocate"> {
+def OMP_Allocate : Directive<[Spelling<"allocate">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Align, 51>,
     VersionedClause<OMPC_Allocator>,
@@ -567,7 +569,7 @@ def OMP_Allocate : Directive<"allocate"> {
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_Allocators : Directive<"allocators"> {
+def OMP_Allocators : Directive<[Spelling<"allocators">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
   ];
@@ -575,7 +577,7 @@ def OMP_Allocators : Directive<"allocators"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_Assumes : Directive<"assumes"> {
+def OMP_Assumes : Directive<[Spelling<"assumes">]> {
   let association = AS_None;
   let category = CA_Informational;
   let allowedOnceClauses = [
@@ -587,7 +589,7 @@ def OMP_Assumes : Directive<"assumes"> {
     VersionedClause<OMPC_NoParallelism, 51>,
   ];
 }
-def OMP_Assume : Directive<"assume"> {
+def OMP_Assume : Directive<[Spelling<"assume">]> {
   let association = AS_Block;
   let category = CA_Informational;
   let allowedOnceClauses = [
@@ -600,7 +602,7 @@ def OMP_Assume : Directive<"assume"> {
     VersionedClause<OMPC_NoOpenMPConstructs, 60>,
   ];
 }
-def OMP_Atomic : Directive<"atomic"> {
+def OMP_Atomic : Directive<[Spelling<"atomic">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_AcqRel, 50>,
     VersionedClause<OMPC_Acquire, 50>,
@@ -619,11 +621,11 @@ def OMP_Atomic : Directive<"atomic"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_Barrier : Directive<"barrier"> {
+def OMP_Barrier : Directive<[Spelling<"barrier">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_BeginAssumes : Directive<"begin assumes"> {
+def OMP_BeginAssumes : Directive<[Spelling<"begin assumes">]> {
   let association = AS_Delimited;
   let category = CA_Informational;
   let allowedOnceClauses = [
@@ -636,12 +638,12 @@ def OMP_BeginAssumes : Directive<"begin assumes"> {
   ];
   let languages = [L_C];
 }
-def OMP_EndAssumes : Directive<"end assumes"> {
+def OMP_EndAssumes : Directive<[Spelling<"end assumes">]> {
   let association = AS_Delimited;
   let category = OMP_BeginAssumes.category;
   let languages = OMP_BeginAssumes.languages;
 }
-def OMP_BeginDeclareTarget : Directive<"begin declare target"> {
+def OMP_BeginDeclareTarget : Directive<[Spelling<"begin declare target">]> {
   let allowedClauses = [
     VersionedClause<OMPC_DeviceType>,
     VersionedClause<OMPC_Indirect>,
@@ -652,22 +654,22 @@ def OMP_BeginDeclareTarget : Directive<"begin declare target"> {
   let category = CA_Declarative;
   let languages = [L_C];
 }
-def OMP_EndDeclareTarget : Directive<"end declare target"> {
+def OMP_EndDeclareTarget : Directive<[Spelling<"end declare target">]> {
   let association = AS_Delimited;
   let category = OMP_BeginDeclareTarget.category;
   let languages = OMP_BeginDeclareTarget.languages;
 }
-def OMP_BeginDeclareVariant : Directive<"begin declare variant"> {
+def OMP_BeginDeclareVariant : Directive<[Spelling<"begin declare variant">]> {
   let association = AS_Delimited;
   let category = CA_Declarative;
   let languages = [L_C];
 }
-def OMP_EndDeclareVariant : Directive<"end declare variant"> {
+def OMP_EndDeclareVariant : Directive<[Spelling<"end declare variant">]> {
   let association = AS_Delimited;
   let category = OMP_BeginDeclareVariant.category;
   let languages = OMP_BeginDeclareVariant.languages;
 }
-def OMP_Cancel : Directive<"cancel"> {
+def OMP_Cancel : Directive<[Spelling<"cancel">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_CancellationConstructType>,
     VersionedClause<OMPC_If>,
@@ -675,35 +677,35 @@ def OMP_Cancel : Directive<"cancel"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_CancellationPoint : Directive<"cancellation point"> {
+def OMP_CancellationPoint : Directive<[Spelling<"cancellation point">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_CancellationConstructType>,
   ];
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_Critical : Directive<"critical"> {
+def OMP_Critical : Directive<[Spelling<"critical">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Hint>,
   ];
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_DeclareMapper : Directive<"declare mapper"> {
+def OMP_DeclareMapper : Directive<[Spelling<"declare mapper">]> {
   let requiredClauses = [
     VersionedClause<OMPC_Map, 45>,
   ];
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_DeclareReduction : Directive<"declare reduction"> {
+def OMP_DeclareReduction : Directive<[Spelling<"declare reduction">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Initializer>,
   ];
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_DeclareSimd : Directive<"declare simd"> {
+def OMP_DeclareSimd : Directive<[Spelling<"declare simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Linear>,
@@ -719,7 +721,7 @@ def OMP_DeclareSimd : Directive<"declare simd"> {
   let association = AS_Declaration;
   let category = CA_Declarative;
 }
-def OMP_DeclareTarget : Directive<"declare target"> {
+def OMP_DeclareTarget : Directive<[Spelling<"declare target">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Enter, 52>,
     VersionedClause<OMPC_Indirect>,
@@ -732,7 +734,7 @@ def OMP_DeclareTarget : Directive<"declare target"> {
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_DeclareVariant : Directive<"declare variant"> {
+def OMP_DeclareVariant : Directive<[Spelling<"declare variant">]> {
   let allowedClauses = [
     VersionedClause<OMPC_AdjustArgs, 51>,
   ];
@@ -744,7 +746,7 @@ def OMP_DeclareVariant : Directive<"declare variant"> {
   let category = CA_Declarative;
   let languages = [L_C];
 }
-def OMP_Depobj : Directive<"depobj"> {
+def OMP_Depobj : Directive<[Spelling<"depobj">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend, 50>,
     // TODO This should ne `none` instead. Comment carried over from
@@ -756,7 +758,7 @@ def OMP_Depobj : Directive<"depobj"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_dispatch : Directive<"dispatch"> {
+def OMP_dispatch : Directive<[Spelling<"dispatch">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
@@ -769,7 +771,7 @@ def OMP_dispatch : Directive<"dispatch"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_Distribute : Directive<"distribute"> {
+def OMP_Distribute : Directive<[Spelling<"distribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_FirstPrivate>,
@@ -784,7 +786,7 @@ def OMP_Distribute : Directive<"distribute"> {
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_Do : Directive<"do"> {
+def OMP_Do : Directive<[Spelling<"do">]> {
   let allowedClauses = [
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_LastPrivate>,
@@ -803,7 +805,7 @@ def OMP_Do : Directive<"do"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_EndDo : Directive<"end do"> {
+def OMP_EndDo : Directive<[Spelling<"end do">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_NoWait>,
   ];
@@ -812,7 +814,7 @@ def OMP_EndDo : Directive<"end do"> {
   let category = OMP_Do.category;
   let languages = OMP_Do.languages;
 }
-def OMP_Error : Directive<"error"> {
+def OMP_Error : Directive<[Spelling<"error">]> {
   let allowedClauses = [
     VersionedClause<OMPC_At, 51>,
     VersionedClause<OMPC_Message, 51>,
@@ -821,7 +823,7 @@ def OMP_Error : Directive<"error"> {
   let association = AS_None;
   let category = CA_Utility;
 }
-def OMP_Flush : Directive<"flush"> {
+def OMP_Flush : Directive<[Spelling<"flush">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_AcqRel, 50>,
     VersionedClause<OMPC_Acquire, 50>,
@@ -834,7 +836,7 @@ def OMP_Flush : Directive<"flush"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_For : Directive<"for"> {
+def OMP_For : Directive<[Spelling<"for">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -852,14 +854,14 @@ def OMP_For : Directive<"for"> {
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_Interchange : Directive<"interchange"> {
+def OMP_Interchange : Directive<[Spelling<"interchange">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Permutation>,
   ];
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_interop : Directive<"interop"> {
+def OMP_interop : Directive<[Spelling<"interop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Destroy>,
@@ -871,7 +873,7 @@ def OMP_interop : Directive<"interop"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_loop : Directive<"loop"> {
+def OMP_loop : Directive<[Spelling<"loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_LastPrivate>,
     VersionedClause<OMPC_Private>,
@@ -885,18 +887,18 @@ def OMP_loop : Directive<"loop"> {
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_masked : Directive<"masked"> {
+def OMP_masked : Directive<[Spelling<"masked">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Filter>,
   ];
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_Master : Directive<"master"> {
+def OMP_Master : Directive<[Spelling<"master">]> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_Metadirective : Directive<"metadirective"> {
+def OMP_Metadirective : Directive<[Spelling<"metadirective">]> {
   let allowedClauses = [
     VersionedClause<OMPC_When>,
   ];
@@ -907,11 +909,11 @@ def OMP_Metadirective : Directive<"metadirective"> {
   let association = AS_None;
   let category = CA_Meta;
 }
-def OMP_Nothing : Directive<"nothing"> {
+def OMP_Nothing : Directive<[Spelling<"nothing">]> {
   let association = AS_None;
   let category = CA_Utility;
 }
-def OMP_Ordered : Directive<"ordered"> {
+def OMP_Ordered : Directive<[Spelling<"ordered">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Doacross, 52>,
@@ -924,7 +926,7 @@ def OMP_Ordered : Directive<"ordered"> {
   // There is also a block-associated "ordered" directive.
   let category = CA_Executable;
 }
-def OMP_Parallel : Directive<"parallel"> {
+def OMP_Parallel : Directive<[Spelling<"parallel">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -943,7 +945,7 @@ def OMP_Parallel : Directive<"parallel"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_Requires : Directive<"requires"> {
+def OMP_Requires : Directive<[Spelling<"requires">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_UnifiedAddress>,
     VersionedClause<OMPC_UnifiedSharedMemory>,
@@ -962,11 +964,11 @@ def OMP_Requires : Directive<"requires"> {
   let association = AS_None;
   let category = CA_Informational;
 }
-def OMP_Reverse : Directive<"reverse"> {
+def OMP_Reverse : Directive<[Spelling<"reverse">]> {
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_Scan : Directive<"scan"> {
+def OMP_Scan : Directive<[Spelling<"scan">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Exclusive>,
     VersionedClause<OMPC_Inclusive>,
@@ -974,7 +976,7 @@ def OMP_Scan : Directive<"scan"> {
   let association = AS_Separating;
   let category = CA_Subsidiary;
 }
-def OMP_Scope : Directive<"scope"> {
+def OMP_Scope : Directive<[Spelling<"scope">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Private, 51>,
     VersionedClause<OMPC_Reduction, 51>,
@@ -987,7 +989,7 @@ def OMP_Scope : Directive<"scope"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_EndScope : Directive<"end scope"> {
+def OMP_EndScope : Directive<[Spelling<"end scope">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_NoWait>,
   ];
@@ -996,11 +998,11 @@ def OMP_EndScope : Directive<"end scope"> {
   let category = OMP_Scope.category;
   let languages = [L_Fortran];
 }
-def OMP_Section : Directive<"section"> {
+def OMP_Section : Directive<[Spelling<"section">]> {
   let association = AS_Separating;
   let category = CA_Subsidiary;
 }
-def OMP_Sections : Directive<"sections"> {
+def OMP_Sections : Directive<[Spelling<"sections">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_FirstPrivate>,
@@ -1012,7 +1014,7 @@ def OMP_Sections : Directive<"sections"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_EndSections : Directive<"end sections"> {
+def OMP_EndSections : Directive<[Spelling<"end sections">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_NoWait>,
   ];
@@ -1021,7 +1023,7 @@ def OMP_EndSections : Directive<"end sections"> {
   let category = OMP_Sections.category;
   let languages = [L_Fortran];
 }
-def OMP_Simd : Directive<"simd"> {
+def OMP_Simd : Directive<[Spelling<"simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1041,7 +1043,7 @@ def OMP_Simd : Directive<"simd"> {
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_Single : Directive<"single"> {
+def OMP_Single : Directive<[Spelling<"single">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_CopyPrivate>,
@@ -1054,7 +1056,7 @@ def OMP_Single : Directive<"single"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_EndSingle : Directive<"end single"> {
+def OMP_EndSingle : Directive<[Spelling<"end single">]> {
   let allowedClauses = [
     VersionedClause<OMPC_CopyPrivate>,
   ];
@@ -1066,7 +1068,7 @@ def OMP_EndSingle : Directive<"end single"> {
   let category = OMP_Single.category;
   let languages = [L_Fortran];
 }
-def OMP_Target : Directive<"target"> {
+def OMP_Target : Directive<[Spelling<"target">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -1091,7 +1093,7 @@ def OMP_Target : Directive<"target"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_TargetData : Directive<"target data"> {
+def OMP_TargetData : Directive<[Spelling<"target data">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Device>,
     VersionedClause<OMPC_If>,
@@ -1104,7 +1106,7 @@ def OMP_TargetData : Directive<"target data"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_TargetEnterData : Directive<"target enter data"> {
+def OMP_TargetEnterData : Directive<[Spelling<"target enter data">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
   ];
@@ -1119,7 +1121,7 @@ def OMP_TargetEnterData : Directive<"target enter data"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_TargetExitData : Directive<"target exit data"> {
+def OMP_TargetExitData : Directive<[Spelling<"target exit data">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
   ];
@@ -1134,7 +1136,7 @@ def OMP_TargetExitData : Directive<"target exit data"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_TargetUpdate : Directive<"target update"> {
+def OMP_TargetUpdate : Directive<[Spelling<"target update">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_From>,
@@ -1148,7 +1150,7 @@ def OMP_TargetUpdate : Directive<"target update"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_Task : Directive<"task"> {
+def OMP_Task : Directive<[Spelling<"task">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Affinity, 50>,
     VersionedClause<OMPC_Allocate>,
@@ -1170,7 +1172,7 @@ def OMP_Task : Directive<"task"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_TaskGroup : Directive<"taskgroup"> {
+def OMP_TaskGroup : Directive<[Spelling<"taskgroup">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate, 50>,
     VersionedClause<OMPC_TaskReduction, 50>,
@@ -1178,7 +1180,7 @@ def OMP_TaskGroup : Directive<"taskgroup"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_TaskLoop : Directive<"taskloop"> {
+def OMP_TaskLoop : Directive<[Spelling<"taskloop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_FirstPrivate>,
@@ -1205,7 +1207,7 @@ def OMP_TaskLoop : Directive<"taskloop"> {
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_TaskWait : Directive<"taskwait"> {
+def OMP_TaskWait : Directive<[Spelling<"taskwait">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend, 50>,
     VersionedClause<OMPC_NoWait, 51>,
@@ -1213,11 +1215,11 @@ def OMP_TaskWait : Directive<"taskwait"> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_TaskYield : Directive<"taskyield"> {
+def OMP_TaskYield : Directive<[Spelling<"taskyield">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_Teams : Directive<"teams"> {
+def OMP_Teams : Directive<[Spelling<"teams">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_FirstPrivate>,
@@ -1235,30 +1237,30 @@ def OMP_Teams : Directive<"teams"> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_ThreadPrivate : Directive<"threadprivate"> {
+def OMP_ThreadPrivate : Directive<[Spelling<"threadprivate">]> {
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_Tile : Directive<"tile"> {
+def OMP_Tile : Directive<[Spelling<"tile">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Sizes, 51>,
   ];
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_Stripe : Directive<"stripe"> {
+def OMP_Stripe : Directive<[Spelling<"stripe">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Sizes, 60>,
   ];
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_Unknown : Directive<"unknown"> {
+def OMP_Unknown : Directive<[Spelling<"unknown">]> {
   let isDefault = true;
   let association = AS_None;
   let category = CA_Utility;
 }
-def OMP_Unroll : Directive<"unroll"> {
+def OMP_Unroll : Directive<[Spelling<"unroll">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Full, 51>,
     VersionedClause<OMPC_Partial, 51>,
@@ -1266,7 +1268,7 @@ def OMP_Unroll : Directive<"unroll"> {
   let association = AS_Loop;
   let category = CA_Executable;
 }
-def OMP_Workshare : Directive<"workshare"> {
+def OMP_Workshare : Directive<[Spelling<"workshare">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_NoWait>,
   ];
@@ -1274,7 +1276,7 @@ def OMP_Workshare : Directive<"workshare"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_EndWorkshare : Directive<"end workshare"> {
+def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_NoWait>,
   ];
@@ -1290,7 +1292,7 @@ def OMP_EndWorkshare : Directive<"end workshare"> {
 // follows "xyz".
 //===----------------------------------------------------------------------===//
 
-def OMP_DistributeParallelDo : Directive<"distribute parallel do"> {
+def OMP_DistributeParallelDo : Directive<[Spelling<"distribute parallel do">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -1315,7 +1317,8 @@ def OMP_DistributeParallelDo : Directive<"distribute parallel do"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_DistributeParallelDoSimd : Directive<"distribute parallel do simd"> {
+def OMP_DistributeParallelDoSimd
+    : Directive<[Spelling<"distribute parallel do simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1342,7 +1345,8 @@ def OMP_DistributeParallelDoSimd : Directive<"distribute parallel do simd"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_DistributeParallelFor : Directive<"distribute parallel for"> {
+def OMP_DistributeParallelFor
+    : Directive<[Spelling<"distribute parallel for">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -1365,7 +1369,8 @@ def OMP_DistributeParallelFor : Directive<"distribute parallel for"> {
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_DistributeParallelForSimd : Directive<"distribute parallel for simd"> {
+def OMP_DistributeParallelForSimd
+    : Directive<[Spelling<"distribute parallel for simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1393,7 +1398,7 @@ def OMP_DistributeParallelForSimd : Directive<"distribute parallel for simd"> {
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_DistributeSimd : Directive<"distribute simd"> {
+def OMP_DistributeSimd : Directive<[Spelling<"distribute simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1420,7 +1425,7 @@ def OMP_DistributeSimd : Directive<"distribute simd"> {
   let leafConstructs = [OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_DoSimd : Directive<"do simd"> {
+def OMP_DoSimd : Directive<[Spelling<"do simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_FirstPrivate>,
@@ -1443,7 +1448,7 @@ def OMP_DoSimd : Directive<"do simd"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_EndDoSimd : Directive<"end do simd"> {
+def OMP_EndDoSimd : Directive<[Spelling<"end do simd">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_NoWait>,
   ];
@@ -1452,7 +1457,7 @@ def OMP_EndDoSimd : Directive<"end do simd"> {
   let category = OMP_DoSimd.category;
   let languages = [L_Fortran];
 }
-def OMP_ForSimd : Directive<"for simd"> {
+def OMP_ForSimd : Directive<[Spelling<"for simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1474,7 +1479,7 @@ def OMP_ForSimd : Directive<"for simd"> {
   let leafConstructs = [OMP_For, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_target_loop : Directive<"target loop"> {
+def OMP_target_loop : Directive<[Spelling<"target loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -1504,7 +1509,7 @@ def OMP_target_loop : Directive<"target loop"> {
   let leafConstructs = [OMP_Target, OMP_loop];
   let category = CA_Executable;
 }
-def OMP_MaskedTaskloop : Directive<"masked taskloop"> {
+def OMP_MaskedTaskloop : Directive<[Spelling<"masked taskloop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -1528,7 +1533,7 @@ def OMP_MaskedTaskloop : Directive<"masked taskloop"> {
   let leafConstructs = [OMP_masked, OMP_TaskLoop];
   let category = CA_Executable;
 }
-def OMP_MaskedTaskloopSimd : Directive<"masked taskloop simd"> {
+def OMP_MaskedTaskloopSimd : Directive<[Spelling<"masked taskloop simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1558,7 +1563,7 @@ def OMP_MaskedTaskloopSimd : Directive<"masked taskloop simd"> {
   let leafConstructs = [OMP_masked, OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_MasterTaskloop : Directive<"master taskloop"> {
+def OMP_MasterTaskloop : Directive<[Spelling<"master taskloop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -1581,7 +1586,7 @@ def OMP_MasterTaskloop : Directive<"master taskloop"> {
   let leafConstructs = [OMP_Master, OMP_TaskLoop];
   let category = CA_Executable;
 }
-def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> {
+def OMP_MasterTaskloopSimd : Directive<[Spelling<"master taskloop simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1610,7 +1615,7 @@ def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> {
   let leafConstructs = [OMP_Master, OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_ParallelDo : Directive<"parallel do"> {
+def OMP_ParallelDo : Directive<[Spelling<"parallel do">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Copyin>,
     VersionedClause<OMPC_Default>,
@@ -1634,7 +1639,7 @@ def OMP_ParallelDo : Directive<"parallel do"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_ParallelDoSimd : Directive<"parallel do simd"> {
+def OMP_ParallelDoSimd : Directive<[Spelling<"parallel do simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1663,7 +1668,7 @@ def OMP_ParallelDoSimd : Directive<"parallel do simd"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_ParallelFor : Directive<"parallel for"> {
+def OMP_ParallelFor : Directive<[Spelling<"parallel for">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -1687,7 +1692,7 @@ def OMP_ParallelFor : Directive<"parallel for"> {
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_ParallelForSimd : Directive<"parallel for simd"> {
+def OMP_ParallelForSimd : Directive<[Spelling<"parallel for simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1715,7 +1720,7 @@ def OMP_ParallelForSimd : Directive<"parallel for simd"> {
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_parallel_loop : Directive<"parallel loop"> {
+def OMP_parallel_loop : Directive<[Spelling<"parallel loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -1738,7 +1743,7 @@ def OMP_parallel_loop : Directive<"parallel loop"> {
   let leafConstructs = [OMP_Parallel, OMP_loop];
   let category = CA_Executable;
 }
-def OMP_ParallelMasked : Directive<"parallel masked"> {
+def OMP_ParallelMasked : Directive<[Spelling<"parallel masked">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -1756,8 +1761,8 @@ def OMP_ParallelMasked : Directive<"parallel masked"> {
   let leafConstructs = [OMP_Parallel, OMP_masked];
   let category = CA_Executable;
 }
-def OMP_ParallelMaskedTaskloop :
-    Directive<"parallel masked taskloop"> {
+def OMP_ParallelMaskedTaskloop
+    : Directive<[Spelling<"parallel masked taskloop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -1784,8 +1789,8 @@ def OMP_ParallelMaskedTaskloop :
   let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop];
   let category = CA_Executable;
 }
-def OMP_ParallelMaskedTaskloopSimd :
-    Directive<"parallel masked taskloop simd"> {
+def OMP_ParallelMaskedTaskloopSimd
+    : Directive<[Spelling<"parallel masked taskloop simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1818,7 +1823,7 @@ def OMP_ParallelMaskedTaskloopSimd :
   let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_ParallelMaster : Directive<"parallel master"> {
+def OMP_ParallelMaster : Directive<[Spelling<"parallel master">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -1835,8 +1840,8 @@ def OMP_ParallelMaster : Directive<"parallel master"> {
   let leafConstructs = [OMP_Parallel, OMP_Master];
   let category = CA_Executable;
 }
-def OMP_ParallelMasterTaskloop :
-    Directive<"parallel master taskloop"> {
+def OMP_ParallelMasterTaskloop
+    : Directive<[Spelling<"parallel master taskloop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -1862,8 +1867,8 @@ def OMP_ParallelMasterTaskloop :
   let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop];
   let category = CA_Executable;
 }
-def OMP_ParallelMasterTaskloopSimd :
-    Directive<"parallel master taskloop simd"> {
+def OMP_ParallelMasterTaskloopSimd
+    : Directive<[Spelling<"parallel master taskloop simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -1895,7 +1900,7 @@ def OMP_ParallelMasterTaskloopSimd :
   let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_ParallelSections : Directive<"parallel sections"> {
+def OMP_ParallelSections : Directive<[Spelling<"parallel sections">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -1915,7 +1920,7 @@ def OMP_ParallelSections : Directive<"parallel sections"> {
   let leafConstructs = [OMP_Parallel, OMP_Sections];
   let category = CA_Executable;
 }
-def OMP_ParallelWorkshare : Directive<"parallel workshare"> {
+def OMP_ParallelWorkshare : Directive<[Spelling<"parallel workshare">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -1934,7 +1939,7 @@ def OMP_ParallelWorkshare : Directive<"parallel workshare"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_TargetParallel : Directive<"target parallel"> {
+def OMP_TargetParallel : Directive<[Spelling<"target parallel">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Default>,
@@ -1962,7 +1967,7 @@ def OMP_TargetParallel : Directive<"target parallel"> {
   let leafConstructs = [OMP_Target, OMP_Parallel];
   let category = CA_Executable;
 }
-def OMP_TargetParallelDo : Directive<"target parallel do"> {
+def OMP_TargetParallelDo : Directive<[Spelling<"target parallel do">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocator>,
     VersionedClause<OMPC_Default>,
@@ -1994,7 +1999,8 @@ def OMP_TargetParallelDo : Directive<"target parallel do"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> {
+def OMP_TargetParallelDoSimd
+    : Directive<[Spelling<"target parallel do simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2028,7 +2034,7 @@ def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> {
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_TargetParallelFor : Directive<"target parallel for"> {
+def OMP_TargetParallelFor : Directive<[Spelling<"target parallel for">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -2063,7 +2069,8 @@ def OMP_TargetParallelFor : Directive<"target parallel for"> {
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> {
+def OMP_TargetParallelForSimd
+    : Directive<[Spelling<"target parallel for simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2102,7 +2109,7 @@ def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> {
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_target_parallel_loop : Directive<"target parallel loop"> {
+def OMP_target_parallel_loop : Directive<[Spelling<"target parallel loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -2134,7 +2141,7 @@ def OMP_target_parallel_loop : Directive<"target parallel loop"> {
   let leafConstructs = [OMP_Target, OMP_Parallel, OMP_loop];
   let category = CA_Executable;
 }
-def OMP_TargetSimd : Directive<"target simd"> {
+def OMP_TargetSimd : Directive<[Spelling<"target simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2170,7 +2177,7 @@ def OMP_TargetSimd : Directive<"target simd"> {
   let leafConstructs = [OMP_Target, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_TargetTeams : Directive<"target teams"> {
+def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -2198,7 +2205,8 @@ def OMP_TargetTeams : Directive<"target teams"> {
   let leafConstructs = [OMP_Target, OMP_Teams];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> {
+def OMP_TargetTeamsDistribute
+    : Directive<[Spelling<"target teams distribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -2229,8 +2237,8 @@ def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> {
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute];
   let category = CA_Executable;
 }
-def OMP_TargetTeamsDistributeParallelDo :
-    Directive<"target teams distribute parallel do"> {
+def OMP_TargetTeamsDistributeParallelDo
+    : Directive<[Spelling<"target teams distribute parallel do">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
@@ -2265,8 +2273,8 @@ def OMP_TargetTeamsDistributeParallelDo :
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_TargetTeamsDistributeParallelDoSimd :
-    Directive<"target teams distribute parallel do simd"> {
+def OMP_TargetTeamsDistributeParallelDoSimd
+    : Directive<[Spelling<"target teams distribute parallel do simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2305,8 +2313,8 @@ def OMP_TargetTeamsDistributeParallelDoSimd :
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_TargetTeamsDistributeParallelFor :
-    Directive<"target teams distribute parallel for"> {
+def OMP_TargetTeamsDistributeParallelFor
+    : Directive<[Spelling<"target teams distribute parallel for">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -2342,8 +2350,8 @@ def OMP_TargetTeamsDistributeParallelFor :
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_TargetTeamsDistributeParallelForSimd :
-    Directive<"target teams distribute parallel for simd"> {
+def OMP_TargetTeamsDistributeParallelForSimd
+    : Directive<[Spelling<"target teams distribute parallel for simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2384,8 +2392,8 @@ def OMP_TargetTeamsDistributeParallelForSimd :
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_TargetTeamsDistributeSimd :
-    Directive<"target teams distribute simd"> {
+def OMP_TargetTeamsDistributeSimd
+    : Directive<[Spelling<"target teams distribute simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2420,7 +2428,7 @@ def OMP_TargetTeamsDistributeSimd :
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_target_teams_loop : Directive<"target teams loop"> {
+def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_DefaultMap>,
@@ -2451,7 +2459,7 @@ def OMP_target_teams_loop : Directive<"target teams loop"> {
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_loop];
   let category = CA_Executable;
 }
-def OMP_TaskLoopSimd : Directive<"taskloop simd"> {
+def OMP_TaskLoopSimd : Directive<[Spelling<"taskloop simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2484,7 +2492,7 @@ def OMP_TaskLoopSimd : Directive<"taskloop simd"> {
   let leafConstructs = [OMP_TaskLoop, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_TeamsDistribute : Directive<"teams distribute"> {
+def OMP_TeamsDistribute : Directive<[Spelling<"teams distribute">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -2506,8 +2514,8 @@ def OMP_TeamsDistribute : Directive<"teams distribute"> {
   let leafConstructs = [OMP_Teams, OMP_Distribute];
   let category = CA_Executable;
 }
-def OMP_TeamsDistributeParallelDo :
-    Directive<"teams distribute parallel do"> {
+def OMP_TeamsDistributeParallelDo
+    : Directive<[Spelling<"teams distribute parallel do">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
@@ -2534,8 +2542,8 @@ def OMP_TeamsDistributeParallelDo :
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_TeamsDistributeParallelDoSimd :
-    Directive<"teams distribute parallel do simd"> {
+def OMP_TeamsDistributeParallelDoSimd
+    : Directive<[Spelling<"teams distribute parallel do simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2566,8 +2574,8 @@ def OMP_TeamsDistributeParallelDoSimd :
   let category = CA_Executable;
   let languages = [L_Fortran];
 }
-def OMP_TeamsDistributeParallelFor :
-    Directive<"teams distribute parallel for"> {
+def OMP_TeamsDistributeParallelFor
+    : Directive<[Spelling<"teams distribute parallel for">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Collapse>,
@@ -2592,8 +2600,8 @@ def OMP_TeamsDistributeParallelFor :
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_TeamsDistributeParallelForSimd :
-    Directive<"teams distribute parallel for simd"> {
+def OMP_TeamsDistributeParallelForSimd
+    : Directive<[Spelling<"teams distribute parallel for simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2623,7 +2631,7 @@ def OMP_TeamsDistributeParallelForSimd :
   let category = CA_Executable;
   let languages = [L_C];
 }
-def OMP_TeamsDistributeSimd : Directive<"teams distribute simd"> {
+def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
@@ -2650,7 +2658,7 @@ def OMP_TeamsDistributeSimd : Directive<"teams distribute simd"> {
   let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
-def OMP_teams_loop : Directive<"teams loop"> {
+def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_FirstPrivate>,
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPContext.h b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
index 9942cbd08aa4..e6c1b4f1f224 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Triple;
@@ -49,69 +50,73 @@ enum class TraitProperty {
 };
 
 /// Parse \p Str and return the trait set it matches or TraitSet::invalid.
-TraitSet getOpenMPContextTraitSetKind(StringRef Str);
+LLVM_ABI TraitSet getOpenMPContextTraitSetKind(StringRef Str);
 
 /// Return the trait set for which \p Selector is a selector.
-TraitSet getOpenMPContextTraitSetForSelector(TraitSelector Selector);
+LLVM_ABI TraitSet getOpenMPContextTraitSetForSelector(TraitSelector Selector);
 
 /// Return the trait set for which \p Property is a property.
-TraitSet getOpenMPContextTraitSetForProperty(TraitProperty Property);
+LLVM_ABI TraitSet getOpenMPContextTraitSetForProperty(TraitProperty Property);
 
 /// Return a textual representation of the trait set \p Kind.
-StringRef getOpenMPContextTraitSetName(TraitSet Kind);
+LLVM_ABI StringRef getOpenMPContextTraitSetName(TraitSet Kind);
 
 /// Parse \p Str and return the trait set it matches or
 /// TraitSelector::invalid.
-TraitSelector getOpenMPContextTraitSelectorKind(StringRef Str, TraitSet Set);
+LLVM_ABI TraitSelector getOpenMPContextTraitSelectorKind(StringRef Str,
+                                                         TraitSet Set);
 
 /// Return the trait selector for which \p Property is a property.
-TraitSelector getOpenMPContextTraitSelectorForProperty(TraitProperty Property);
+LLVM_ABI TraitSelector
+getOpenMPContextTraitSelectorForProperty(TraitProperty Property);
 
 /// Return a textual representation of the trait selector \p Kind.
-StringRef getOpenMPContextTraitSelectorName(TraitSelector Kind);
+LLVM_ABI StringRef getOpenMPContextTraitSelectorName(TraitSelector Kind);
 
 /// Parse \p Str and return the trait property it matches in the set \p Set and
 /// selector \p Selector or TraitProperty::invalid.
-TraitProperty getOpenMPContextTraitPropertyKind(TraitSet Set,
-                                                TraitSelector Selector,
-                                                StringRef Str);
+LLVM_ABI TraitProperty getOpenMPContextTraitPropertyKind(TraitSet Set,
+                                                         TraitSelector Selector,
+                                                         StringRef Str);
 
 /// Return the trait property for a singleton selector \p Selector.
-TraitProperty getOpenMPContextTraitPropertyForSelector(TraitSelector Selector);
+LLVM_ABI TraitProperty
+getOpenMPContextTraitPropertyForSelector(TraitSelector Selector);
 
 /// Return a textual representation of the trait property \p Kind, which might
 /// be the raw string we parsed (\p RawString) if we do not translate the
 /// property into a (distinct) enum.
-StringRef getOpenMPContextTraitPropertyName(TraitProperty Kind,
-                                            StringRef RawString);
+LLVM_ABI StringRef getOpenMPContextTraitPropertyName(TraitProperty Kind,
+                                                     StringRef RawString);
 
 /// Return a textual representation of the trait property \p Kind with selector
 /// and set name included.
-StringRef getOpenMPContextTraitPropertyFullName(TraitProperty Kind);
+LLVM_ABI StringRef getOpenMPContextTraitPropertyFullName(TraitProperty Kind);
 
 /// Return a string listing all trait sets.
-std::string listOpenMPContextTraitSets();
+LLVM_ABI std::string listOpenMPContextTraitSets();
 
 /// Return a string listing all trait selectors for \p Set.
-std::string listOpenMPContextTraitSelectors(TraitSet Set);
+LLVM_ABI std::string listOpenMPContextTraitSelectors(TraitSet Set);
 
 /// Return a string listing all trait properties for \p Set and \p Selector.
-std::string listOpenMPContextTraitProperties(TraitSet Set,
-                                             TraitSelector Selector);
+LLVM_ABI std::string listOpenMPContextTraitProperties(TraitSet Set,
+                                                      TraitSelector Selector);
 ///}
 
 /// Return true if \p Selector can be nested in \p Set. Also sets
 /// \p AllowsTraitScore and \p RequiresProperty to true/false if the user can
 /// specify a score for properties in \p Selector and if the \p Selector
 /// requires at least one property.
-bool isValidTraitSelectorForTraitSet(TraitSelector Selector, TraitSet Set,
-                                     bool &AllowsTraitScore,
-                                     bool &RequiresProperty);
+LLVM_ABI bool isValidTraitSelectorForTraitSet(TraitSelector Selector,
+                                              TraitSet Set,
+                                              bool &AllowsTraitScore,
+                                              bool &RequiresProperty);
 
 /// Return true if \p Property can be nested in \p Selector and \p Set.
-bool isValidTraitPropertyForTraitSetAndSelector(TraitProperty Property,
-                                                TraitSelector Selector,
-                                                TraitSet Set);
+LLVM_ABI bool isValidTraitPropertyForTraitSetAndSelector(TraitProperty Property,
+                                                         TraitSelector Selector,
+                                                         TraitSet Set);
 
 /// Variant match information describes the required traits and how they are
 /// scored (via the ScoresMap). In addition, the required consturct nesting is
@@ -157,8 +162,8 @@ struct VariantMatchInfo {
 /// e.g., device={kind(host)}, and constructs traits which describe the nesting
 /// in OpenMP constructs at the location.
 struct OMPContext {
-  OMPContext(bool IsDeviceCompilation, Triple TargetTriple,
-             Triple TargetOffloadTriple, int DeviceNum);
+  LLVM_ABI OMPContext(bool IsDeviceCompilation, Triple TargetTriple,
+                      Triple TargetOffloadTriple, int DeviceNum);
   virtual ~OMPContext() = default;
 
   void addTrait(TraitProperty Property) {
@@ -184,14 +189,15 @@ struct OMPContext {
 /// \p DeviceOrImplementationSetOnly is true, only the device and implementation
 /// selector set, if present, are checked. Note that we still honor extension
 /// traits provided by the user.
-bool isVariantApplicableInContext(const VariantMatchInfo &VMI,
-                                  const OMPContext &Ctx,
-                                  bool DeviceOrImplementationSetOnly = false);
+LLVM_ABI bool
+isVariantApplicableInContext(const VariantMatchInfo &VMI, const OMPContext &Ctx,
+                             bool DeviceOrImplementationSetOnly = false);
 
 /// Return the index (into \p VMIs) of the variant with the highest score
 /// from the ones applicable in \p Ctx. See llvm::isVariantApplicableInContext.
-int getBestVariantMatchForContext(const SmallVectorImpl<VariantMatchInfo> &VMIs,
-                                  const OMPContext &Ctx);
+LLVM_ABI int
+getBestVariantMatchForContext(const SmallVectorImpl<VariantMatchInfo> &VMIs,
+                              const OMPContext &Ctx);
 
 } // namespace omp
 
diff --git a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index e1deb8cdb5a4..bcd5e1b514fb 100644
--- a/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -22,6 +22,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ValueMap.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/TargetParser/Triple.h"
 #include <forward_list>
@@ -43,14 +44,15 @@ class OpenMPIRBuilder;
 /// \p IP insert block remains degenerate and it is up to the caller to insert a
 /// terminator. \p DL is used as the debug location for the branch instruction
 /// if one is created.
-void spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New, bool CreateBranch,
-              DebugLoc DL);
+LLVM_ABI void spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
+                       bool CreateBranch, DebugLoc DL);
 
 /// Splice a BasicBlock at an IRBuilder's current insertion point. Its new
 /// insert location will stick to after the instruction before the insertion
 /// point (instead of moving with the instruction the InsertPoint stores
 /// internally).
-void spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch);
+LLVM_ABI void spliceBB(IRBuilder<> &Builder, BasicBlock *New,
+                       bool CreateBranch);
 
 /// Split a BasicBlock at an InsertPoint, even if the block is degenerate
 /// (missing the terminator).
@@ -62,25 +64,27 @@ void spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch);
 /// remains degenerate and it is the caller's responsibility to insert a
 /// terminator. \p DL is used as the debug location for the branch instruction
 /// if one is created. Returns the new successor block.
-BasicBlock *splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
-                    DebugLoc DL, llvm::Twine Name = {});
+LLVM_ABI BasicBlock *splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
+                             DebugLoc DL, llvm::Twine Name = {});
 
 /// Split a BasicBlock at \p Builder's insertion point, even if the block is
 /// degenerate (missing the terminator).  Its new insert location will stick to
 /// after the instruction before the insertion point (instead of moving with the
 /// instruction the InsertPoint stores internally).
-BasicBlock *splitBB(IRBuilderBase &Builder, bool CreateBranch,
-                    llvm::Twine Name = {});
+LLVM_ABI BasicBlock *splitBB(IRBuilderBase &Builder, bool CreateBranch,
+                             llvm::Twine Name = {});
 
 /// Split a BasicBlock at \p Builder's insertion point, even if the block is
 /// degenerate (missing the terminator).  Its new insert location will stick to
 /// after the instruction before the insertion point (instead of moving with the
 /// instruction the InsertPoint stores internally).
-BasicBlock *splitBB(IRBuilder<> &Builder, bool CreateBranch, llvm::Twine Name);
+LLVM_ABI BasicBlock *splitBB(IRBuilder<> &Builder, bool CreateBranch,
+                             llvm::Twine Name);
 
 /// Like splitBB, but reuses the current block's name for the new name.
-BasicBlock *splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
-                              llvm::Twine Suffix = ".split");
+LLVM_ABI BasicBlock *splitBBWithSuffix(IRBuilderBase &Builder,
+                                       bool CreateBranch,
+                                       llvm::Twine Suffix = ".split");
 
 /// Captures attributes that affect generating LLVM-IR using the
 /// OpenMPIRBuilder and related classes. Note that not all attributes are
@@ -124,13 +128,13 @@ class OpenMPIRBuilderConfig {
   /// false`), this contains the list of offloading triples associated, if any.
   SmallVector<Triple> TargetTriples;
 
-  OpenMPIRBuilderConfig();
-  OpenMPIRBuilderConfig(bool IsTargetDevice, bool IsGPU,
-                        bool OpenMPOffloadMandatory,
-                        bool HasRequiresReverseOffload,
-                        bool HasRequiresUnifiedAddress,
-                        bool HasRequiresUnifiedSharedMemory,
-                        bool HasRequiresDynamicAllocators);
+  LLVM_ABI OpenMPIRBuilderConfig();
+  LLVM_ABI OpenMPIRBuilderConfig(bool IsTargetDevice, bool IsGPU,
+                                 bool OpenMPOffloadMandatory,
+                                 bool HasRequiresReverseOffload,
+                                 bool HasRequiresUnifiedAddress,
+                                 bool HasRequiresUnifiedSharedMemory,
+                                 bool HasRequiresDynamicAllocators);
 
   // Getters functions that assert if the required values are not present.
   bool isTargetDevice() const {
@@ -155,14 +159,14 @@ class OpenMPIRBuilderConfig {
   }
 
   bool hasRequiresFlags() const { return RequiresFlags; }
-  bool hasRequiresReverseOffload() const;
-  bool hasRequiresUnifiedAddress() const;
-  bool hasRequiresUnifiedSharedMemory() const;
-  bool hasRequiresDynamicAllocators() const;
+  LLVM_ABI bool hasRequiresReverseOffload() const;
+  LLVM_ABI bool hasRequiresUnifiedAddress() const;
+  LLVM_ABI bool hasRequiresUnifiedSharedMemory() const;
+  LLVM_ABI bool hasRequiresDynamicAllocators() const;
 
   /// Returns requires directive clauses as flags compatible with those expected
   /// by libomptarget.
-  int64_t getRequiresFlags() const;
+  LLVM_ABI int64_t getRequiresFlags() const;
 
   // Returns the FirstSeparator if set, otherwise use the default separator
   // depending on isGPU
@@ -192,10 +196,10 @@ class OpenMPIRBuilderConfig {
   void setSeparator(StringRef S) { Separator = S; }
   void setGridValue(omp::GV G) { GridValue = G; }
 
-  void setHasRequiresReverseOffload(bool Value);
-  void setHasRequiresUnifiedAddress(bool Value);
-  void setHasRequiresUnifiedSharedMemory(bool Value);
-  void setHasRequiresDynamicAllocators(bool Value);
+  LLVM_ABI void setHasRequiresReverseOffload(bool Value);
+  LLVM_ABI void setHasRequiresUnifiedAddress(bool Value);
+  LLVM_ABI void setHasRequiresUnifiedSharedMemory(bool Value);
+  LLVM_ABI void setHasRequiresDynamicAllocators(bool Value);
 
 private:
   /// Flags for specifying which requires directive clauses are present.
@@ -220,10 +224,10 @@ struct TargetRegionEntryInfo {
       : ParentName(ParentName), DeviceID(DeviceID), FileID(FileID), Line(Line),
         Count(Count) {}
 
-  static void getTargetRegionEntryFnName(SmallVectorImpl<char> &Name,
-                                         StringRef ParentName,
-                                         unsigned DeviceID, unsigned FileID,
-                                         unsigned Line, unsigned Count);
+  LLVM_ABI static void
+  getTargetRegionEntryFnName(SmallVectorImpl<char> &Name, StringRef ParentName,
+                             unsigned DeviceID, unsigned FileID, unsigned Line,
+                             unsigned Count);
 
   bool operator<(const TargetRegionEntryInfo &RHS) const {
     return std::make_tuple(ParentName, DeviceID, FileID, Line, Count) <
@@ -287,7 +291,7 @@ class OffloadEntriesInfoManager {
   };
 
   /// Return true if a there are no entries defined.
-  bool empty() const;
+  LLVM_ABI bool empty() const;
   /// Return number of entries defined so far.
   unsigned size() const { return OffloadingEntriesNum; }
 
@@ -331,26 +335,28 @@ class OffloadEntriesInfoManager {
 
   /// Initialize target region entry.
   /// This is ONLY needed for DEVICE compilation.
-  void initializeTargetRegionEntryInfo(const TargetRegionEntryInfo &EntryInfo,
-                                       unsigned Order);
+  LLVM_ABI void
+  initializeTargetRegionEntryInfo(const TargetRegionEntryInfo &EntryInfo,
+                                  unsigned Order);
   /// Register target region entry.
-  void registerTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo,
-                                     Constant *Addr, Constant *ID,
-                                     OMPTargetRegionEntryKind Flags);
+  LLVM_ABI void registerTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo,
+                                              Constant *Addr, Constant *ID,
+                                              OMPTargetRegionEntryKind Flags);
   /// Return true if a target region entry with the provided information
   /// exists.
-  bool hasTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo,
-                                bool IgnoreAddressId = false) const;
+  LLVM_ABI bool hasTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo,
+                                         bool IgnoreAddressId = false) const;
 
   // Return the Name based on \a EntryInfo using the next available Count.
-  void getTargetRegionEntryFnName(SmallVectorImpl<char> &Name,
-                                  const TargetRegionEntryInfo &EntryInfo);
+  LLVM_ABI void
+  getTargetRegionEntryFnName(SmallVectorImpl<char> &Name,
+                             const TargetRegionEntryInfo &EntryInfo);
 
   /// brief Applies action \a Action on all registered entries.
   typedef function_ref<void(const TargetRegionEntryInfo &EntryInfo,
                             const OffloadEntryInfoTargetRegion &)>
       OffloadTargetRegionEntryInfoActTy;
-  void
+  LLVM_ABI void
   actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action);
 
   //
@@ -424,15 +430,13 @@ class OffloadEntriesInfoManager {
 
   /// Initialize device global variable entry.
   /// This is ONLY used for DEVICE compilation.
-  void initializeDeviceGlobalVarEntryInfo(StringRef Name,
-                                          OMPTargetGlobalVarEntryKind Flags,
-                                          unsigned Order);
+  LLVM_ABI void initializeDeviceGlobalVarEntryInfo(
+      StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order);
 
   /// Register device global variable entry.
-  void registerDeviceGlobalVarEntryInfo(StringRef VarName, Constant *Addr,
-                                        int64_t VarSize,
-                                        OMPTargetGlobalVarEntryKind Flags,
-                                        GlobalValue::LinkageTypes Linkage);
+  LLVM_ABI void registerDeviceGlobalVarEntryInfo(
+      StringRef VarName, Constant *Addr, int64_t VarSize,
+      OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage);
   /// Checks if the variable with the given name has been registered already.
   bool hasDeviceGlobalVarEntryInfo(StringRef VarName) const {
     return OffloadEntriesDeviceGlobalVar.count(VarName) > 0;
@@ -440,7 +444,7 @@ class OffloadEntriesInfoManager {
   /// Applies action \a Action on all registered entries.
   typedef function_ref<void(StringRef, const OffloadEntryInfoDeviceGlobalVar &)>
       OffloadDeviceGlobalVarEntryInfoActTy;
-  void actOnDeviceGlobalVarEntriesInfo(
+  LLVM_ABI void actOnDeviceGlobalVarEntriesInfo(
       const OffloadDeviceGlobalVarEntryInfoActTy &Action);
 
 private:
@@ -482,7 +486,7 @@ class OpenMPIRBuilder {
   OpenMPIRBuilder(Module &M)
       : M(M), Builder(M.getContext()), OffloadInfoManager(this),
         T(M.getTargetTriple()) {}
-  ~OpenMPIRBuilder();
+  LLVM_ABI ~OpenMPIRBuilder();
 
   class AtomicInfo : public llvm::AtomicInfo {
     llvm::Value *AtomicVar;
@@ -509,17 +513,17 @@ class OpenMPIRBuilder {
   /// potentially other helpers into the underlying module. Must be called
   /// before any other method and only once! This internal state includes types
   /// used in the OpenMPIRBuilder generated from OMPKinds.def.
-  void initialize();
+  LLVM_ABI void initialize();
 
   void setConfig(OpenMPIRBuilderConfig C) { Config = C; }
 
   /// Finalize the underlying module, e.g., by outlining regions.
   /// \param Fn                    The function to be finalized. If not used,
   ///                              all functions are finalized.
-  void finalize(Function *Fn = nullptr);
+  LLVM_ABI void finalize(Function *Fn = nullptr);
 
   /// Add attributes known for \p FnID to \p Fn.
-  void addAttributes(omp::RuntimeFunction FnID, Function &Fn);
+  LLVM_ABI void addAttributes(omp::RuntimeFunction FnID, Function &Fn);
 
   /// Type used throughout for insertion points.
   using InsertPointTy = IRBuilder<>::InsertPoint;
@@ -535,7 +539,8 @@ class OpenMPIRBuilder {
   /// parts: "p1", "p2", "p3", "p4"
   /// The resulting name is "p1$p2.p3.p4"
   /// The separators are retrieved from the OpenMPIRBuilderConfig.
-  std::string createPlatformSpecificName(ArrayRef<StringRef> Parts) const;
+  LLVM_ABI std::string
+  createPlatformSpecificName(ArrayRef<StringRef> Parts) const;
 
   /// Callback type for variable finalization (think destructors).
   ///
@@ -671,10 +676,10 @@ class OpenMPIRBuilder {
   /// \param ThreadID Optional parameter to pass in any existing ThreadID value.
   ///
   /// \returns The insertion point after the barrier.
-  InsertPointOrErrorTy createBarrier(const LocationDescription &Loc,
-                                     omp::Directive Kind,
-                                     bool ForceSimpleCall = false,
-                                     bool CheckCancelFlag = true);
+  LLVM_ABI InsertPointOrErrorTy createBarrier(const LocationDescription &Loc,
+                                              omp::Directive Kind,
+                                              bool ForceSimpleCall = false,
+                                              bool CheckCancelFlag = true);
 
   /// Generator for '#omp cancel'
   ///
@@ -683,9 +688,9 @@ class OpenMPIRBuilder {
   /// \param CanceledDirective The kind of directive that is cancled.
   ///
   /// \returns The insertion point after the barrier.
-  InsertPointOrErrorTy createCancel(const LocationDescription &Loc,
-                                    Value *IfCondition,
-                                    omp::Directive CanceledDirective);
+  LLVM_ABI InsertPointOrErrorTy createCancel(const LocationDescription &Loc,
+                                             Value *IfCondition,
+                                             omp::Directive CanceledDirective);
 
   /// Generator for '#omp cancellation point'
   ///
@@ -693,9 +698,8 @@ class OpenMPIRBuilder {
   /// \param CanceledDirective The kind of directive that is cancled.
   ///
   /// \returns The insertion point after the barrier.
-  InsertPointOrErrorTy
-  createCancellationPoint(const LocationDescription &Loc,
-                          omp::Directive CanceledDirective);
+  LLVM_ABI InsertPointOrErrorTy createCancellationPoint(
+      const LocationDescription &Loc, omp::Directive CanceledDirective);
 
   /// Generator for '#omp parallel'
   ///
@@ -710,12 +714,11 @@ class OpenMPIRBuilder {
   /// \param IsCancellable Flag to indicate a cancellable parallel region.
   ///
   /// \returns The insertion position *after* the parallel.
-  InsertPointOrErrorTy
-  createParallel(const LocationDescription &Loc, InsertPointTy AllocaIP,
-                 BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
-                 FinalizeCallbackTy FiniCB, Value *IfCondition,
-                 Value *NumThreads, omp::ProcBindKind ProcBind,
-                 bool IsCancellable);
+  LLVM_ABI InsertPointOrErrorTy createParallel(
+      const LocationDescription &Loc, InsertPointTy AllocaIP,
+      BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
+      FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
+      omp::ProcBindKind ProcBind, bool IsCancellable);
 
   /// Generator for the control flow structure of an OpenMP canonical loop.
   ///
@@ -736,7 +739,7 @@ class OpenMPIRBuilder {
   ///
   /// \returns An object representing the created control flow structure which
   ///          can be used for loop-associated directives.
-  Expected<CanonicalLoopInfo *>
+  LLVM_ABI Expected<CanonicalLoopInfo *>
   createCanonicalLoop(const LocationDescription &Loc,
                       LoopBodyGenCallbackTy BodyGenCB, Value *TripCount,
                       const Twine &Name = "loop");
@@ -781,10 +784,9 @@ class OpenMPIRBuilder {
   /// \param Name      Base name used to derive instruction names.
   ///
   /// \returns The value holding the calculated trip count.
-  Value *calculateCanonicalLoopTripCount(const LocationDescription &Loc,
-                                         Value *Start, Value *Stop, Value *Step,
-                                         bool IsSigned, bool InclusiveStop,
-                                         const Twine &Name = "loop");
+  LLVM_ABI Value *calculateCanonicalLoopTripCount(
+      const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,
+      bool IsSigned, bool InclusiveStop, const Twine &Name = "loop");
 
   /// Generator for the control flow structure of an OpenMP canonical loop.
   ///
@@ -813,7 +815,7 @@ class OpenMPIRBuilder {
   ///
   /// \returns An object representing the created control flow structure which
   ///          can be used for loop-associated directives.
-  Expected<CanonicalLoopInfo *> createCanonicalLoop(
+  LLVM_ABI Expected<CanonicalLoopInfo *> createCanonicalLoop(
       const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
       Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
       InsertPointTy ComputeIP = {}, const Twine &Name = "loop");
@@ -878,16 +880,17 @@ class OpenMPIRBuilder {
   ///                  loop.
   ///
   /// \returns The CanonicalLoopInfo object representing the collapsed loop.
-  CanonicalLoopInfo *collapseLoops(DebugLoc DL,
-                                   ArrayRef<CanonicalLoopInfo *> Loops,
-                                   InsertPointTy ComputeIP);
+  LLVM_ABI CanonicalLoopInfo *collapseLoops(DebugLoc DL,
+                                            ArrayRef<CanonicalLoopInfo *> Loops,
+                                            InsertPointTy ComputeIP);
 
   /// Get the default alignment value for given target
   ///
   /// \param TargetTriple   Target triple
   /// \param Features       StringMap which describes extra CPU features
-  static unsigned getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
-                                            const StringMap<bool> &Features);
+  LLVM_ABI static unsigned
+  getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
+                            const StringMap<bool> &Features);
 
   /// Retrieve (or create if non-existent) the address of a declare
   /// target variable, used in conjunction with registerTargetGlobalVariable
@@ -923,7 +926,7 @@ class OpenMPIRBuilder {
   /// linkage type, if unspecified and a nullptr is given, it will instead
   /// utilise the linkage stored on the existing global variable in the
   /// LLVMModule.
-  Constant *getAddrOfDeclareTargetVar(
+  LLVM_ABI Constant *getAddrOfDeclareTargetVar(
       OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
       OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
       bool IsDeclaration, bool IsExternallyVisible,
@@ -967,7 +970,7 @@ class OpenMPIRBuilder {
   /// retrieving an address for
   /// \param Addr - the original llvm value (addr) of the variable to be
   /// registered
-  void registerTargetGlobalVariable(
+  LLVM_ABI void registerTargetGlobalVariable(
       OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
       OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
       bool IsDeclaration, bool IsExternallyVisible,
@@ -979,7 +982,7 @@ class OpenMPIRBuilder {
       Type *LlvmPtrTy, Constant *Addr);
 
   /// Get the offset of the OMP_MAP_MEMBER_OF field.
-  unsigned getFlagMemberOffset();
+  LLVM_ABI unsigned getFlagMemberOffset();
 
   /// Get OMP_MAP_MEMBER_OF flag with extra bits reserved based on
   /// the position given.
@@ -988,7 +991,7 @@ class OpenMPIRBuilder {
   /// by the parents position in the combined information vectors used
   /// to generate the structure itself. Multiple children (member's of)
   /// with the same parent will use the same returned member flag.
-  omp::OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position);
+  LLVM_ABI omp::OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position);
 
   /// Given an initial flag set, this function modifies it to contain
   /// the passed in MemberOfFlag generated from the getMemberOfFlag
@@ -999,8 +1002,9 @@ class OpenMPIRBuilder {
   /// \param MemberOfFlag - A modified OMP_MAP_MEMBER_OF flag, adjusted
   /// slightly based on the getMemberOfFlag which adjusts the flag bits
   /// based on the members position in its parent.
-  void setCorrectMemberOfFlag(omp::OpenMPOffloadMappingFlags &Flags,
-                              omp::OpenMPOffloadMappingFlags MemberOfFlag);
+  LLVM_ABI void
+  setCorrectMemberOfFlag(omp::OpenMPOffloadMappingFlags &Flags,
+                         omp::OpenMPOffloadMappingFlags MemberOfFlag);
 
 private:
   /// Modifies the canonical loop to be a statically-scheduled workshare loop
@@ -1144,7 +1148,7 @@ class OpenMPIRBuilder {
   ///                 It corresponds to type of loop workshare OpenMP pragma.
   ///
   /// \returns Point where to insert code after the workshare construct.
-  InsertPointOrErrorTy applyWorkshareLoop(
+  LLVM_ABI InsertPointOrErrorTy applyWorkshareLoop(
       DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
       bool NeedsBarrier,
       llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default,
@@ -1197,7 +1201,7 @@ class OpenMPIRBuilder {
   /// \returns A list of generated loops. Contains twice as many loops as the
   ///          input loop nest; the first half are the floor loops and the
   ///          second half are the tile loops.
-  std::vector<CanonicalLoopInfo *>
+  LLVM_ABI std::vector<CanonicalLoopInfo *>
   tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
             ArrayRef<Value *> TileSizes);
 
@@ -1209,14 +1213,14 @@ class OpenMPIRBuilder {
   ///
   /// \param DL   Debug location for instructions added by unrolling.
   /// \param Loop The loop to unroll. The loop will be invalidated.
-  void unrollLoopFull(DebugLoc DL, CanonicalLoopInfo *Loop);
+  LLVM_ABI void unrollLoopFull(DebugLoc DL, CanonicalLoopInfo *Loop);
 
   /// Fully or partially unroll a loop. How the loop is unrolled is determined
   /// using LLVM's LoopUnrollPass.
   ///
   /// \param DL   Debug location for instructions added by unrolling.
   /// \param Loop The loop to unroll. The loop will be invalidated.
-  void unrollLoopHeuristic(DebugLoc DL, CanonicalLoopInfo *Loop);
+  LLVM_ABI void unrollLoopHeuristic(DebugLoc DL, CanonicalLoopInfo *Loop);
 
   /// Partially unroll a loop.
   ///
@@ -1240,8 +1244,9 @@ class OpenMPIRBuilder {
   /// \param UnrolledCLI If non-null, receives the CanonicalLoopInfo of the
   ///                    partially unrolled loop. Otherwise, uses loop metadata
   ///                    to defer unrolling to the LoopUnrollPass.
-  void unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop, int32_t Factor,
-                         CanonicalLoopInfo **UnrolledCLI);
+  LLVM_ABI void unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
+                                  int32_t Factor,
+                                  CanonicalLoopInfo **UnrolledCLI);
 
   /// Add metadata to simd-ize a loop. If IfCond is not nullptr, the loop
   /// is cloned. The metadata which prevents vectorization is added to
@@ -1256,25 +1261,25 @@ class OpenMPIRBuilder {
   /// \param Order       The enum to map order clause.
   /// \param Simdlen     The Simdlen length to apply to the simd loop.
   /// \param Safelen     The Safelen length to apply to the simd loop.
-  void applySimd(CanonicalLoopInfo *Loop,
-                 MapVector<Value *, Value *> AlignedVars, Value *IfCond,
-                 omp::OrderKind Order, ConstantInt *Simdlen,
-                 ConstantInt *Safelen);
+  LLVM_ABI void applySimd(CanonicalLoopInfo *Loop,
+                          MapVector<Value *, Value *> AlignedVars,
+                          Value *IfCond, omp::OrderKind Order,
+                          ConstantInt *Simdlen, ConstantInt *Safelen);
 
   /// Generator for '#omp flush'
   ///
   /// \param Loc The location where the flush directive was encountered
-  void createFlush(const LocationDescription &Loc);
+  LLVM_ABI void createFlush(const LocationDescription &Loc);
 
   /// Generator for '#omp taskwait'
   ///
   /// \param Loc The location where the taskwait directive was encountered.
-  void createTaskwait(const LocationDescription &Loc);
+  LLVM_ABI void createTaskwait(const LocationDescription &Loc);
 
   /// Generator for '#omp taskyield'
   ///
   /// \param Loc The location where the taskyield directive was encountered.
-  void createTaskyield(const LocationDescription &Loc);
+  LLVM_ABI void createTaskyield(const LocationDescription &Loc);
 
   /// A struct to pack the relevant information for an OpenMP depend clause.
   struct DependData {
@@ -1306,7 +1311,7 @@ class OpenMPIRBuilder {
   /// \param Mergeable	 If the given task is `mergeable`
   /// \param priority `priority-value' specifies the execution order of the
   ///                 tasks that is generated by the construct
-  InsertPointOrErrorTy
+  LLVM_ABI InsertPointOrErrorTy
   createTask(const LocationDescription &Loc, InsertPointTy AllocaIP,
              BodyGenCallbackTy BodyGenCB, bool Tied = true,
              Value *Final = nullptr, Value *IfCondition = nullptr,
@@ -1318,9 +1323,9 @@ class OpenMPIRBuilder {
   /// \param Loc The location where the taskgroup construct was encountered.
   /// \param AllocaIP The insertion point to be used for alloca instructions.
   /// \param BodyGenCB Callback that will generate the region code.
-  InsertPointOrErrorTy createTaskgroup(const LocationDescription &Loc,
-                                       InsertPointTy AllocaIP,
-                                       BodyGenCallbackTy BodyGenCB);
+  LLVM_ABI InsertPointOrErrorTy createTaskgroup(const LocationDescription &Loc,
+                                                InsertPointTy AllocaIP,
+                                                BodyGenCallbackTy BodyGenCB);
 
   using FileIdentifierInfoCallbackTy =
       std::function<std::tuple<std::string, uint64_t>()>;
@@ -1332,7 +1337,7 @@ class OpenMPIRBuilder {
   /// resides in as well as the line number for the target entry
   /// \param ParentName The name of the parent the target entry resides in, if
   /// any.
-  static TargetRegionEntryInfo
+  LLVM_ABI static TargetRegionEntryInfo
   getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
                            StringRef ParentName = "");
 
@@ -1923,7 +1928,7 @@ class OpenMPIRBuilder {
   /// \param ReductionBufNum    Optional OpenMPCUDAReductionBufNumValue to be
   /// used for teams reduction.
   /// \param SrcLocInfo         Source location information global.
-  InsertPointOrErrorTy createReductionsGPU(
+  LLVM_ABI InsertPointOrErrorTy createReductionsGPU(
       const LocationDescription &Loc, InsertPointTy AllocaIP,
       InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
       bool IsNoWait = false, bool IsTeamsReduction = false,
@@ -1996,12 +2001,10 @@ class OpenMPIRBuilder {
   /// or direct value.
   /// \param IsTeamsReduction   Optional flag set if it is a teams
   ///                           reduction.
-  InsertPointOrErrorTy createReductions(const LocationDescription &Loc,
-                                        InsertPointTy AllocaIP,
-                                        ArrayRef<ReductionInfo> ReductionInfos,
-                                        ArrayRef<bool> IsByRef,
-                                        bool IsNoWait = false,
-                                        bool IsTeamsReduction = false);
+  LLVM_ABI InsertPointOrErrorTy createReductions(
+      const LocationDescription &Loc, InsertPointTy AllocaIP,
+      ArrayRef<ReductionInfo> ReductionInfos, ArrayRef<bool> IsByRef,
+      bool IsNoWait = false, bool IsTeamsReduction = false);
 
   ///}
 
@@ -2016,8 +2019,8 @@ class OpenMPIRBuilder {
   }
 
   /// Return the function declaration for the runtime function with \p FnID.
-  FunctionCallee getOrCreateRuntimeFunction(Module &M,
-                                            omp::RuntimeFunction FnID);
+  LLVM_ABI FunctionCallee getOrCreateRuntimeFunction(Module &M,
+                                                     omp::RuntimeFunction FnID);
 
   /// Return the function declaration for atomic CAS runtime function
   /// with name \p FunName. Used for unsigned types as basic .def machinery
@@ -2026,49 +2029,52 @@ class OpenMPIRBuilder {
   /// \param RetType Type of function return parameter
   /// \param AddrTy Type of atomic target pointer
   /// \param UpdateTy Type of atomic update expression
-  FunctionCallee unsignedGetOrCreateAtomicCASRuntimeFunction(
+  LLVM_ABI FunctionCallee unsignedGetOrCreateAtomicCASRuntimeFunction(
       Module &M, const StringRef &FunName, Type *RetType, Type *AddrTy,
       Type *UpdateTy);
 
-  Function *getOrCreateRuntimeFunctionPtr(omp::RuntimeFunction FnID);
+  LLVM_ABI Function *getOrCreateRuntimeFunctionPtr(omp::RuntimeFunction FnID);
 
   /// Return the (LLVM-IR) string describing the source location \p LocStr.
-  Constant *getOrCreateSrcLocStr(StringRef LocStr, uint32_t &SrcLocStrSize);
+  LLVM_ABI Constant *getOrCreateSrcLocStr(StringRef LocStr,
+                                          uint32_t &SrcLocStrSize);
 
   /// Return the (LLVM-IR) string describing the default source location.
-  Constant *getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize);
+  LLVM_ABI Constant *getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize);
 
   /// Return the (LLVM-IR) string describing the source location identified by
   /// the arguments.
-  Constant *getOrCreateSrcLocStr(StringRef FunctionName, StringRef FileName,
-                                 unsigned Line, unsigned Column,
-                                 uint32_t &SrcLocStrSize);
+  LLVM_ABI Constant *getOrCreateSrcLocStr(StringRef FunctionName,
+                                          StringRef FileName, unsigned Line,
+                                          unsigned Column,
+                                          uint32_t &SrcLocStrSize);
 
   /// Return the (LLVM-IR) string describing the DebugLoc \p DL. Use \p F as
   /// fallback if \p DL does not specify the function name.
-  Constant *getOrCreateSrcLocStr(DebugLoc DL, uint32_t &SrcLocStrSize,
-                                 Function *F = nullptr);
+  LLVM_ABI Constant *getOrCreateSrcLocStr(DebugLoc DL, uint32_t &SrcLocStrSize,
+                                          Function *F = nullptr);
 
   /// Return the (LLVM-IR) string describing the source location \p Loc.
-  Constant *getOrCreateSrcLocStr(const LocationDescription &Loc,
-                                 uint32_t &SrcLocStrSize);
+  LLVM_ABI Constant *getOrCreateSrcLocStr(const LocationDescription &Loc,
+                                          uint32_t &SrcLocStrSize);
 
   /// Return an ident_t* encoding the source location \p SrcLocStr and \p Flags.
   /// TODO: Create a enum class for the Reserve2Flags
-  Constant *getOrCreateIdent(Constant *SrcLocStr, uint32_t SrcLocStrSize,
-                             omp::IdentFlag Flags = omp::IdentFlag(0),
-                             unsigned Reserve2Flags = 0);
+  LLVM_ABI Constant *getOrCreateIdent(Constant *SrcLocStr,
+                                      uint32_t SrcLocStrSize,
+                                      omp::IdentFlag Flags = omp::IdentFlag(0),
+                                      unsigned Reserve2Flags = 0);
 
   /// Create a hidden global flag \p Name in the module with initial value \p
   /// Value.
-  GlobalValue *createGlobalFlag(unsigned Value, StringRef Name);
+  LLVM_ABI GlobalValue *createGlobalFlag(unsigned Value, StringRef Name);
 
   /// Emit the llvm.used metadata.
-  void emitUsed(StringRef Name, ArrayRef<llvm::WeakTrackingVH> List);
+  LLVM_ABI void emitUsed(StringRef Name, ArrayRef<llvm::WeakTrackingVH> List);
 
   /// Emit the kernel execution mode.
-  GlobalVariable *emitKernelExecutionMode(StringRef KernelName,
-                                          omp::OMPTgtExecModeFlags Mode);
+  LLVM_ABI GlobalVariable *
+  emitKernelExecutionMode(StringRef KernelName, omp::OMPTgtExecModeFlags Mode);
 
   /// Generate control flow and cleanup for cancellation.
   ///
@@ -2077,9 +2083,9 @@ class OpenMPIRBuilder {
   /// \param ExitCB Extra code to be generated in the exit block.
   ///
   /// \return an error, if any were triggered during execution.
-  Error emitCancelationCheckImpl(Value *CancelFlag,
-                                 omp::Directive CanceledDirective,
-                                 FinalizeCallbackTy ExitCB = {});
+  LLVM_ABI Error emitCancelationCheckImpl(Value *CancelFlag,
+                                          omp::Directive CanceledDirective,
+                                          FinalizeCallbackTy ExitCB = {});
 
   /// Generate a target region entry call.
   ///
@@ -2092,16 +2098,17 @@ class OpenMPIRBuilder {
   /// \param NumThreads Number of threads via the 'thread_limit' clause.
   /// \param HostPtr Pointer to the host-side pointer of the target kernel.
   /// \param KernelArgs Array of arguments to the kernel.
-  InsertPointTy emitTargetKernel(const LocationDescription &Loc,
-                                 InsertPointTy AllocaIP, Value *&Return,
-                                 Value *Ident, Value *DeviceID, Value *NumTeams,
-                                 Value *NumThreads, Value *HostPtr,
-                                 ArrayRef<Value *> KernelArgs);
+  LLVM_ABI InsertPointTy emitTargetKernel(const LocationDescription &Loc,
+                                          InsertPointTy AllocaIP,
+                                          Value *&Return, Value *Ident,
+                                          Value *DeviceID, Value *NumTeams,
+                                          Value *NumThreads, Value *HostPtr,
+                                          ArrayRef<Value *> KernelArgs);
 
   /// Generate a flush runtime call.
   ///
   /// \param Loc The location at which the request originated and is fulfilled.
-  void emitFlush(const LocationDescription &Loc);
+  LLVM_ABI void emitFlush(const LocationDescription &Loc);
 
   /// The finalization stack made up of finalize callbacks currently in-flight,
   /// wrapped into FinalizationInfo objects that reference also the finalization
@@ -2119,17 +2126,17 @@ class OpenMPIRBuilder {
   /// Generate a taskwait runtime call.
   ///
   /// \param Loc The location at which the request originated and is fulfilled.
-  void emitTaskwaitImpl(const LocationDescription &Loc);
+  LLVM_ABI void emitTaskwaitImpl(const LocationDescription &Loc);
 
   /// Generate a taskyield runtime call.
   ///
   /// \param Loc The location at which the request originated and is fulfilled.
-  void emitTaskyieldImpl(const LocationDescription &Loc);
+  LLVM_ABI void emitTaskyieldImpl(const LocationDescription &Loc);
 
   /// Return the current thread ID.
   ///
   /// \param Ident The ident (ident_t*) describing the query origin.
-  Value *getOrCreateThreadID(Value *Ident);
+  LLVM_ABI Value *getOrCreateThreadID(Value *Ident);
 
   /// The OpenMPIRBuilder Configuration
   OpenMPIRBuilderConfig Config;
@@ -2162,8 +2169,8 @@ class OpenMPIRBuilder {
 
     /// Collect all blocks in between EntryBB and ExitBB in both the given
     /// vector and set.
-    void collectBlocks(SmallPtrSetImpl<BasicBlock *> &BlockSet,
-                       SmallVectorImpl<BasicBlock *> &BlockVector);
+    LLVM_ABI void collectBlocks(SmallPtrSetImpl<BasicBlock *> &BlockSet,
+                                SmallVectorImpl<BasicBlock *> &BlockVector);
 
     /// Return the function that contains the region to be outlined.
     Function *getFunction() const { return EntryBB->getParent(); }
@@ -2192,16 +2199,17 @@ class OpenMPIRBuilder {
   StringMap<GlobalVariable *, BumpPtrAllocator> InternalVars;
 
   /// Computes the size of type in bytes.
-  Value *getSizeInBytes(Value *BasePtr);
+  LLVM_ABI Value *getSizeInBytes(Value *BasePtr);
 
   // Emit a branch from the current block to the Target block only if
   // the current block has a terminator.
-  void emitBranch(BasicBlock *Target);
+  LLVM_ABI void emitBranch(BasicBlock *Target);
 
   // If BB has no use then delete it and return. Else place BB after the current
   // block, if possible, or else at the end of the function. Also add a branch
   // from current block to BB if current block does not have a terminator.
-  void emitBlock(BasicBlock *BB, Function *CurFn, bool IsFinished = false);
+  LLVM_ABI void emitBlock(BasicBlock *BB, Function *CurFn,
+                          bool IsFinished = false);
 
   /// Emits code for OpenMP 'if' clause using specified \a BodyGenCallbackTy
   /// Here is the logic:
@@ -2212,15 +2220,17 @@ class OpenMPIRBuilder {
   /// }
   ///
   /// \return an error, if any were triggered during execution.
-  Error emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
-                     BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP = {});
+  LLVM_ABI Error emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
+                              BodyGenCallbackTy ElseGen,
+                              InsertPointTy AllocaIP = {});
 
   /// Create the global variable holding the offload mappings information.
-  GlobalVariable *createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
-                                        std::string VarName);
+  LLVM_ABI GlobalVariable *
+  createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
+                        std::string VarName);
 
   /// Create the global variable holding the offload names information.
-  GlobalVariable *
+  LLVM_ABI GlobalVariable *
   createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
                         std::string VarName);
 
@@ -2231,9 +2241,10 @@ class OpenMPIRBuilder {
   };
 
   /// Create the allocas instruction used in call to mapper functions.
-  void createMapperAllocas(const LocationDescription &Loc,
-                           InsertPointTy AllocaIP, unsigned NumOperands,
-                           struct MapperAllocas &MapperAllocas);
+  LLVM_ABI void createMapperAllocas(const LocationDescription &Loc,
+                                    InsertPointTy AllocaIP,
+                                    unsigned NumOperands,
+                                    struct MapperAllocas &MapperAllocas);
 
   /// Create the call for the target mapper function.
   /// \param Loc The source location description.
@@ -2244,10 +2255,11 @@ class OpenMPIRBuilder {
   /// \param MapperAllocas The AllocaInst used for the call.
   /// \param DeviceID Device ID for the call.
   /// \param NumOperands Number of operands in the call.
-  void emitMapperCall(const LocationDescription &Loc, Function *MapperFunc,
-                      Value *SrcLocInfo, Value *MaptypesArg, Value *MapnamesArg,
-                      struct MapperAllocas &MapperAllocas, int64_t DeviceID,
-                      unsigned NumOperands);
+  LLVM_ABI void emitMapperCall(const LocationDescription &Loc,
+                               Function *MapperFunc, Value *SrcLocInfo,
+                               Value *MaptypesArg, Value *MapnamesArg,
+                               struct MapperAllocas &MapperAllocas,
+                               int64_t DeviceID, unsigned NumOperands);
 
   /// Container for the arguments used to pass data to the runtime library.
   struct TargetDataRTArgs {
@@ -2352,9 +2364,9 @@ class OpenMPIRBuilder {
   /// Create the kernel args vector used by emitTargetKernel. This function
   /// creates various constant values that are used in the resulting args
   /// vector.
-  static void getKernelArgsVector(TargetKernelArgs &KernelArgs,
-                                  IRBuilderBase &Builder,
-                                  SmallVector<Value *> &ArgsVector);
+  LLVM_ABI static void getKernelArgsVector(TargetKernelArgs &KernelArgs,
+                                           IRBuilderBase &Builder,
+                                           SmallVector<Value *> &ArgsVector);
 
   /// Struct that keeps the information that should be kept throughout
   /// a 'target data' region.
@@ -2473,11 +2485,10 @@ class OpenMPIRBuilder {
   /// \param DeviceID Identifier for the device via the 'device' clause.
   /// \param RTLoc Source location identifier
   /// \param AllocaIP The insertion point to be used for alloca instructions.
-  InsertPointOrErrorTy
-  emitKernelLaunch(const LocationDescription &Loc, Value *OutlinedFnID,
-                   EmitFallbackCallbackTy EmitTargetCallFallbackCB,
-                   TargetKernelArgs &Args, Value *DeviceID, Value *RTLoc,
-                   InsertPointTy AllocaIP);
+  LLVM_ABI InsertPointOrErrorTy emitKernelLaunch(
+      const LocationDescription &Loc, Value *OutlinedFnID,
+      EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
+      Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP);
 
   /// Callback type for generating the bodies of device directives that require
   /// outer target tasks (e.g. in case of having `nowait` or `depend` clauses).
@@ -2503,7 +2514,7 @@ class OpenMPIRBuilder {
   ///        dependencies as specified by the 'depend' clause.
   /// \param HasNoWait True if the target construct had 'nowait' on it, false
   ///        otherwise
-  InsertPointOrErrorTy emitTargetTask(
+  LLVM_ABI InsertPointOrErrorTy emitTargetTask(
       TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
       OpenMPIRBuilder::InsertPointTy AllocaIP,
       const SmallVector<llvm::OpenMPIRBuilder::DependData> &Dependencies,
@@ -2513,23 +2524,22 @@ class OpenMPIRBuilder {
   /// arrays of base pointers, pointers, sizes, map types, and mappers.  If
   /// ForEndCall, emit map types to be passed for the end of the region instead
   /// of the beginning.
-  void emitOffloadingArraysArgument(IRBuilderBase &Builder,
-                                    OpenMPIRBuilder::TargetDataRTArgs &RTArgs,
-                                    OpenMPIRBuilder::TargetDataInfo &Info,
-                                    bool ForEndCall = false);
+  LLVM_ABI void emitOffloadingArraysArgument(
+      IRBuilderBase &Builder, OpenMPIRBuilder::TargetDataRTArgs &RTArgs,
+      OpenMPIRBuilder::TargetDataInfo &Info, bool ForEndCall = false);
 
   /// Emit an array of struct descriptors to be assigned to the offload args.
-  void emitNonContiguousDescriptor(InsertPointTy AllocaIP,
-                                   InsertPointTy CodeGenIP,
-                                   MapInfosTy &CombinedInfo,
-                                   TargetDataInfo &Info);
+  LLVM_ABI void emitNonContiguousDescriptor(InsertPointTy AllocaIP,
+                                            InsertPointTy CodeGenIP,
+                                            MapInfosTy &CombinedInfo,
+                                            TargetDataInfo &Info);
 
   /// Emit the arrays used to pass the captures and map information to the
   /// offloading runtime library. If there is no map or capture information,
   /// return nullptr by reference. Accepts a reference to a MapInfosTy object
   /// that contains information generated for mappable clauses,
   /// including base pointers, pointers, sizes, map types, user-defined mappers.
-  Error emitOffloadingArrays(
+  LLVM_ABI Error emitOffloadingArrays(
       InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
       TargetDataInfo &Info, CustomMapperCallbackTy CustomMapperCB,
       bool IsNonContiguous = false,
@@ -2541,7 +2551,7 @@ class OpenMPIRBuilder {
   /// library. In essence, this function is a combination of
   /// emitOffloadingArrays and emitOffloadingArraysArgument and should arguably
   /// be preferred by clients of OpenMPIRBuilder.
-  Error emitOffloadingArraysAndArgs(
+  LLVM_ABI Error emitOffloadingArraysAndArgs(
       InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,
       TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo,
       CustomMapperCallbackTy CustomMapperCB, bool IsNonContiguous = false,
@@ -2550,9 +2560,9 @@ class OpenMPIRBuilder {
 
   /// Creates offloading entry for the provided entry ID \a ID, address \a
   /// Addr, size \a Size, and flags \a Flags.
-  void createOffloadEntry(Constant *ID, Constant *Addr, uint64_t Size,
-                          int32_t Flags, GlobalValue::LinkageTypes,
-                          StringRef Name = "");
+  LLVM_ABI void createOffloadEntry(Constant *ID, Constant *Addr, uint64_t Size,
+                                   int32_t Flags, GlobalValue::LinkageTypes,
+                                   StringRef Name = "");
 
   /// The kind of errors that can occur when emitting the offload entries and
   /// metadata.
@@ -2573,7 +2583,7 @@ class OpenMPIRBuilder {
   // !omp_offload.info = !{!1, ...}
   //
   // We only generate metadata for function that contain target regions.
-  void createOffloadEntriesAndInfoMetadata(
+  LLVM_ABI void createOffloadEntriesAndInfoMetadata(
       EmitMetadataErrorReportFunctionTy &ErrorReportFunction);
 
 public:
@@ -2587,9 +2597,11 @@ class OpenMPIRBuilder {
   ///
   /// \return The insertion position *after* the CopyPrivate call.
 
-  InsertPointTy createCopyPrivate(const LocationDescription &Loc,
-                                  llvm::Value *BufSize, llvm::Value *CpyBuf,
-                                  llvm::Value *CpyFn, llvm::Value *DidIt);
+  LLVM_ABI InsertPointTy createCopyPrivate(const LocationDescription &Loc,
+                                           llvm::Value *BufSize,
+                                           llvm::Value *CpyBuf,
+                                           llvm::Value *CpyFn,
+                                           llvm::Value *DidIt);
 
   /// Generator for '#omp single'
   ///
@@ -2601,11 +2613,11 @@ class OpenMPIRBuilder {
   /// \param CPFuncs copy functions to use for each copyprivate variable.
   ///
   /// \returns The insertion position *after* the single call.
-  InsertPointOrErrorTy createSingle(const LocationDescription &Loc,
-                                    BodyGenCallbackTy BodyGenCB,
-                                    FinalizeCallbackTy FiniCB, bool IsNowait,
-                                    ArrayRef<llvm::Value *> CPVars = {},
-                                    ArrayRef<llvm::Function *> CPFuncs = {});
+  LLVM_ABI InsertPointOrErrorTy
+  createSingle(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
+               FinalizeCallbackTy FiniCB, bool IsNowait,
+               ArrayRef<llvm::Value *> CPVars = {},
+               ArrayRef<llvm::Function *> CPFuncs = {});
 
   /// Generator for '#omp master'
   ///
@@ -2614,9 +2626,9 @@ class OpenMPIRBuilder {
   /// \param FiniCB Callback to finalize variable copies.
   ///
   /// \returns The insertion position *after* the master.
-  InsertPointOrErrorTy createMaster(const LocationDescription &Loc,
-                                    BodyGenCallbackTy BodyGenCB,
-                                    FinalizeCallbackTy FiniCB);
+  LLVM_ABI InsertPointOrErrorTy createMaster(const LocationDescription &Loc,
+                                             BodyGenCallbackTy BodyGenCB,
+                                             FinalizeCallbackTy FiniCB);
 
   /// Generator for '#omp masked'
   ///
@@ -2625,9 +2637,10 @@ class OpenMPIRBuilder {
   /// \param FiniCB Callback to finialize variable copies.
   ///
   /// \returns The insertion position *after* the masked.
-  InsertPointOrErrorTy createMasked(const LocationDescription &Loc,
-                                    BodyGenCallbackTy BodyGenCB,
-                                    FinalizeCallbackTy FiniCB, Value *Filter);
+  LLVM_ABI InsertPointOrErrorTy createMasked(const LocationDescription &Loc,
+                                             BodyGenCallbackTy BodyGenCB,
+                                             FinalizeCallbackTy FiniCB,
+                                             Value *Filter);
 
   /// Generator for '#omp critical'
   ///
@@ -2638,10 +2651,11 @@ class OpenMPIRBuilder {
   /// \param HintInst Hint Instruction for hint clause associated with critical
   ///
   /// \returns The insertion position *after* the critical.
-  InsertPointOrErrorTy createCritical(const LocationDescription &Loc,
-                                      BodyGenCallbackTy BodyGenCB,
-                                      FinalizeCallbackTy FiniCB,
-                                      StringRef CriticalName, Value *HintInst);
+  LLVM_ABI InsertPointOrErrorTy createCritical(const LocationDescription &Loc,
+                                               BodyGenCallbackTy BodyGenCB,
+                                               FinalizeCallbackTy FiniCB,
+                                               StringRef CriticalName,
+                                               Value *HintInst);
 
   /// Generator for '#omp ordered depend (source | sink)'
   ///
@@ -2653,10 +2667,10 @@ class OpenMPIRBuilder {
   /// \param IsDependSource If true, depend source; otherwise, depend sink.
   ///
   /// \return The insertion position *after* the ordered.
-  InsertPointTy createOrderedDepend(const LocationDescription &Loc,
-                                    InsertPointTy AllocaIP, unsigned NumLoops,
-                                    ArrayRef<llvm::Value *> StoreValues,
-                                    const Twine &Name, bool IsDependSource);
+  LLVM_ABI InsertPointTy
+  createOrderedDepend(const LocationDescription &Loc, InsertPointTy AllocaIP,
+                      unsigned NumLoops, ArrayRef<llvm::Value *> StoreValues,
+                      const Twine &Name, bool IsDependSource);
 
   /// Generator for '#omp ordered [threads | simd]'
   ///
@@ -2667,10 +2681,9 @@ class OpenMPIRBuilder {
   /// otherwise, with simd clause;
   ///
   /// \returns The insertion position *after* the ordered.
-  InsertPointOrErrorTy createOrderedThreadsSimd(const LocationDescription &Loc,
-                                                BodyGenCallbackTy BodyGenCB,
-                                                FinalizeCallbackTy FiniCB,
-                                                bool IsThreads);
+  LLVM_ABI InsertPointOrErrorTy createOrderedThreadsSimd(
+      const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
+      FinalizeCallbackTy FiniCB, bool IsThreads);
 
   /// Generator for '#omp sections'
   ///
@@ -2683,7 +2696,7 @@ class OpenMPIRBuilder {
   /// \param IsNowait If true, barrier - to ensure all sections are executed
   /// before moving forward will not be generated.
   /// \returns The insertion position *after* the sections.
-  InsertPointOrErrorTy
+  LLVM_ABI InsertPointOrErrorTy
   createSections(const LocationDescription &Loc, InsertPointTy AllocaIP,
                  ArrayRef<StorableBodyGenCallbackTy> SectionCBs,
                  PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB,
@@ -2695,9 +2708,9 @@ class OpenMPIRBuilder {
   /// \param BodyGenCB Callback that will generate the region body code.
   /// \param FiniCB Callback to finalize variable copies.
   /// \returns The insertion position *after* the section.
-  InsertPointOrErrorTy createSection(const LocationDescription &Loc,
-                                     BodyGenCallbackTy BodyGenCB,
-                                     FinalizeCallbackTy FiniCB);
+  LLVM_ABI InsertPointOrErrorTy createSection(const LocationDescription &Loc,
+                                              BodyGenCallbackTy BodyGenCB,
+                                              FinalizeCallbackTy FiniCB);
 
   /// Generator for `#omp teams`
   ///
@@ -2711,19 +2724,21 @@ class OpenMPIRBuilder {
   ///        contention group created by each team.
   /// \param IfExpr is the integer argument value of the if condition on the
   ///        teams clause.
-  InsertPointOrErrorTy
-  createTeams(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
-              Value *NumTeamsLower = nullptr, Value *NumTeamsUpper = nullptr,
-              Value *ThreadLimit = nullptr, Value *IfExpr = nullptr);
+  LLVM_ABI InsertPointOrErrorTy createTeams(const LocationDescription &Loc,
+                                            BodyGenCallbackTy BodyGenCB,
+                                            Value *NumTeamsLower = nullptr,
+                                            Value *NumTeamsUpper = nullptr,
+                                            Value *ThreadLimit = nullptr,
+                                            Value *IfExpr = nullptr);
 
   /// Generator for `#omp distribute`
   ///
   /// \param Loc The location where the distribute construct was encountered.
   /// \param AllocaIP The insertion points to be used for alloca instructions.
   /// \param BodyGenCB Callback that will generate the region code.
-  InsertPointOrErrorTy createDistribute(const LocationDescription &Loc,
-                                        InsertPointTy AllocaIP,
-                                        BodyGenCallbackTy BodyGenCB);
+  LLVM_ABI InsertPointOrErrorTy createDistribute(const LocationDescription &Loc,
+                                                 InsertPointTy AllocaIP,
+                                                 BodyGenCallbackTy BodyGenCB);
 
   /// Generate conditional branch and relevant BasicBlocks through which private
   /// threads copy the 'copyin' variables from Master copy to threadprivate
@@ -2737,10 +2752,11 @@ class OpenMPIRBuilder {
   //				 and copy.in.end block
   ///
   /// \returns The insertion point where copying operation to be emitted.
-  InsertPointTy createCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr,
-                                         Value *PrivateAddr,
-                                         llvm::IntegerType *IntPtrTy,
-                                         bool BranchtoEnd = true);
+  LLVM_ABI InsertPointTy createCopyinClauseBlocks(InsertPointTy IP,
+                                                  Value *MasterAddr,
+                                                  Value *PrivateAddr,
+                                                  llvm::IntegerType *IntPtrTy,
+                                                  bool BranchtoEnd = true);
 
   /// Create a runtime call for kmpc_Alloc
   ///
@@ -2750,8 +2766,8 @@ class OpenMPIRBuilder {
   /// \param Name Name of call Instruction for OMP_alloc
   ///
   /// \returns CallInst to the OMP_Alloc call
-  CallInst *createOMPAlloc(const LocationDescription &Loc, Value *Size,
-                           Value *Allocator, std::string Name = "");
+  LLVM_ABI CallInst *createOMPAlloc(const LocationDescription &Loc, Value *Size,
+                                    Value *Allocator, std::string Name = "");
 
   /// Create a runtime call for kmpc_free
   ///
@@ -2761,8 +2777,8 @@ class OpenMPIRBuilder {
   /// \param Name Name of call Instruction for OMP_Free
   ///
   /// \returns CallInst to the OMP_Free call
-  CallInst *createOMPFree(const LocationDescription &Loc, Value *Addr,
-                          Value *Allocator, std::string Name = "");
+  LLVM_ABI CallInst *createOMPFree(const LocationDescription &Loc, Value *Addr,
+                                   Value *Allocator, std::string Name = "");
 
   /// Create a runtime call for kmpc_threadprivate_cached
   ///
@@ -2772,10 +2788,10 @@ class OpenMPIRBuilder {
   /// \param Name Name of call Instruction for callinst
   ///
   /// \returns CallInst to the thread private cache call.
-  CallInst *createCachedThreadPrivate(const LocationDescription &Loc,
-                                      llvm::Value *Pointer,
-                                      llvm::ConstantInt *Size,
-                                      const llvm::Twine &Name = Twine(""));
+  LLVM_ABI CallInst *
+  createCachedThreadPrivate(const LocationDescription &Loc,
+                            llvm::Value *Pointer, llvm::ConstantInt *Size,
+                            const llvm::Twine &Name = Twine(""));
 
   /// Create a runtime call for __tgt_interop_init
   ///
@@ -2788,12 +2804,12 @@ class OpenMPIRBuilder {
   /// \param HaveNowaitClause does nowait clause exist
   ///
   /// \returns CallInst to the __tgt_interop_init call
-  CallInst *createOMPInteropInit(const LocationDescription &Loc,
-                                 Value *InteropVar,
-                                 omp::OMPInteropType InteropType, Value *Device,
-                                 Value *NumDependences,
-                                 Value *DependenceAddress,
-                                 bool HaveNowaitClause);
+  LLVM_ABI CallInst *createOMPInteropInit(const LocationDescription &Loc,
+                                          Value *InteropVar,
+                                          omp::OMPInteropType InteropType,
+                                          Value *Device, Value *NumDependences,
+                                          Value *DependenceAddress,
+                                          bool HaveNowaitClause);
 
   /// Create a runtime call for __tgt_interop_destroy
   ///
@@ -2805,11 +2821,11 @@ class OpenMPIRBuilder {
   /// \param HaveNowaitClause does nowait clause exist
   ///
   /// \returns CallInst to the __tgt_interop_destroy call
-  CallInst *createOMPInteropDestroy(const LocationDescription &Loc,
-                                    Value *InteropVar, Value *Device,
-                                    Value *NumDependences,
-                                    Value *DependenceAddress,
-                                    bool HaveNowaitClause);
+  LLVM_ABI CallInst *createOMPInteropDestroy(const LocationDescription &Loc,
+                                             Value *InteropVar, Value *Device,
+                                             Value *NumDependences,
+                                             Value *DependenceAddress,
+                                             bool HaveNowaitClause);
 
   /// Create a runtime call for __tgt_interop_use
   ///
@@ -2821,10 +2837,11 @@ class OpenMPIRBuilder {
   /// \param HaveNowaitClause does nowait clause exist
   ///
   /// \returns CallInst to the __tgt_interop_use call
-  CallInst *createOMPInteropUse(const LocationDescription &Loc,
-                                Value *InteropVar, Value *Device,
-                                Value *NumDependences, Value *DependenceAddress,
-                                bool HaveNowaitClause);
+  LLVM_ABI CallInst *createOMPInteropUse(const LocationDescription &Loc,
+                                         Value *InteropVar, Value *Device,
+                                         Value *NumDependences,
+                                         Value *DependenceAddress,
+                                         bool HaveNowaitClause);
 
   /// The `omp target` interface
   ///
@@ -2838,7 +2855,7 @@ class OpenMPIRBuilder {
   /// \param Loc The insert and source location description.
   /// \param Attrs Structure containing the default attributes, including
   ///        numbers of threads and teams to launch the kernel with.
-  InsertPointTy createTargetInit(
+  LLVM_ABI InsertPointTy createTargetInit(
       const LocationDescription &Loc,
       const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs);
 
@@ -2849,9 +2866,9 @@ class OpenMPIRBuilder {
   ///        for teams reduction.
   /// \param TeamsReductionBufferLength The number of elements (each of up to
   ///        \p TeamsReductionDataSize size), in the teams reduction buffer.
-  void createTargetDeinit(const LocationDescription &Loc,
-                          int32_t TeamsReductionDataSize = 0,
-                          int32_t TeamsReductionBufferLength = 1024);
+  LLVM_ABI void createTargetDeinit(const LocationDescription &Loc,
+                                   int32_t TeamsReductionDataSize = 0,
+                                   int32_t TeamsReductionBufferLength = 1024);
 
   ///}
 
@@ -2861,17 +2878,18 @@ class OpenMPIRBuilder {
 
   /// Read/write a bounds on threads for \p Kernel. Read will return 0 if none
   /// is set.
-  static std::pair<int32_t, int32_t>
+  LLVM_ABI static std::pair<int32_t, int32_t>
   readThreadBoundsForKernel(const Triple &T, Function &Kernel);
-  static void writeThreadBoundsForKernel(const Triple &T, Function &Kernel,
-                                         int32_t LB, int32_t UB);
+  LLVM_ABI static void writeThreadBoundsForKernel(const Triple &T,
+                                                  Function &Kernel, int32_t LB,
+                                                  int32_t UB);
 
   /// Read/write a bounds on teams for \p Kernel. Read will return 0 if none
   /// is set.
-  static std::pair<int32_t, int32_t> readTeamBoundsForKernel(const Triple &T,
-                                                             Function &Kernel);
-  static void writeTeamsForKernel(const Triple &T, Function &Kernel, int32_t LB,
-                                  int32_t UB);
+  LLVM_ABI static std::pair<int32_t, int32_t>
+  readTeamBoundsForKernel(const Triple &T, Function &Kernel);
+  LLVM_ABI static void writeTeamsForKernel(const Triple &T, Function &Kernel,
+                                           int32_t LB, int32_t UB);
   ///}
 
 private:
@@ -2924,10 +2942,10 @@ class OpenMPIRBuilder {
   /// \param GenerateFunctionCallback The callback function to generate the code
   /// \param OutlinedFunction Pointer to the outlined function
   /// \param EntryFnIDName Name of the ID o be created
-  Error emitTargetRegionFunction(TargetRegionEntryInfo &EntryInfo,
-                                 FunctionGenCallback &GenerateFunctionCallback,
-                                 bool IsOffloadEntry, Function *&OutlinedFn,
-                                 Constant *&OutlinedFnID);
+  LLVM_ABI Error emitTargetRegionFunction(
+      TargetRegionEntryInfo &EntryInfo,
+      FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
+      Function *&OutlinedFn, Constant *&OutlinedFnID);
 
   /// Registers the given function and sets up the attribtues of the function
   /// Returns the FunctionID.
@@ -2937,10 +2955,10 @@ class OpenMPIRBuilder {
   /// \param OutlinedFunction Pointer to the outlined function
   /// \param EntryFnName Name of the outlined function
   /// \param EntryFnIDName Name of the ID o be created
-  Constant *registerTargetRegionFunction(TargetRegionEntryInfo &EntryInfo,
-                                         Function *OutlinedFunction,
-                                         StringRef EntryFnName,
-                                         StringRef EntryFnIDName);
+  LLVM_ABI Constant *
+  registerTargetRegionFunction(TargetRegionEntryInfo &EntryInfo,
+                               Function *OutlinedFunction,
+                               StringRef EntryFnName, StringRef EntryFnIDName);
 
   /// Type of BodyGen to use for region codegen
   ///
@@ -3015,7 +3033,7 @@ class OpenMPIRBuilder {
   /// \param FuncName Optional param to specify mapper function name.
   /// \param CustomMapperCB Optional callback to generate code related to
   /// custom mappers.
-  Expected<Function *> emitUserDefinedMapper(
+  LLVM_ABI Expected<Function *> emitUserDefinedMapper(
       function_ref<MapInfosOrErrorTy(
           InsertPointTy CodeGenIP, llvm::Value *PtrPHI, llvm::Value *BeginArg)>
           PrivAndGenMapInfoCB,
@@ -3039,7 +3057,7 @@ class OpenMPIRBuilder {
   /// \param BodyGenCB Optional Callback to generate the region code.
   /// \param DeviceAddrCB Optional callback to generate code related to
   /// use_device_ptr and use_device_addr.
-  InsertPointOrErrorTy createTargetData(
+  LLVM_ABI InsertPointOrErrorTy createTargetData(
       const LocationDescription &Loc, InsertPointTy AllocaIP,
       InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
       TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,
@@ -3082,7 +3100,7 @@ class OpenMPIRBuilder {
   ///        dependency information as passed in the depend clause
   /// \param HasNowait Whether the target construct has a `nowait` clause or
   ///        not.
-  InsertPointOrErrorTy createTarget(
+  LLVM_ABI InsertPointOrErrorTy createTarget(
       const LocationDescription &Loc, bool IsOffloadEntry,
       OpenMPIRBuilder::InsertPointTy AllocaIP,
       OpenMPIRBuilder::InsertPointTy CodeGenIP, TargetDataInfo &Info,
@@ -3098,8 +3116,9 @@ class OpenMPIRBuilder {
   /// Returns __kmpc_for_static_init_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned. Will create a distribute call
   /// __kmpc_distribute_static_init* if \a IsGPUDistribute is set.
-  FunctionCallee createForStaticInitFunction(unsigned IVSize, bool IVSigned,
-                                             bool IsGPUDistribute);
+  LLVM_ABI FunctionCallee createForStaticInitFunction(unsigned IVSize,
+                                                      bool IVSigned,
+                                                      bool IsGPUDistribute);
 
   /// Return the __kmpc_distribute_static_init_multi_device* function.
   FunctionCallee createMDDistributeForStaticInitFunction(unsigned IVSize,
@@ -3107,18 +3126,21 @@ class OpenMPIRBuilder {
 
   /// Returns __kmpc_dispatch_init_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned.
-  FunctionCallee createDispatchInitFunction(unsigned IVSize, bool IVSigned);
+  LLVM_ABI FunctionCallee createDispatchInitFunction(unsigned IVSize,
+                                                     bool IVSigned);
 
   /// Returns __kmpc_dispatch_next_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned.
-  FunctionCallee createDispatchNextFunction(unsigned IVSize, bool IVSigned);
+  LLVM_ABI FunctionCallee createDispatchNextFunction(unsigned IVSize,
+                                                     bool IVSigned);
 
   /// Returns __kmpc_dispatch_fini_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned.
-  FunctionCallee createDispatchFiniFunction(unsigned IVSize, bool IVSigned);
+  LLVM_ABI FunctionCallee createDispatchFiniFunction(unsigned IVSize,
+                                                     bool IVSigned);
 
   /// Returns __kmpc_dispatch_deinit runtime function.
-  FunctionCallee createDispatchDeinitFunction();
+  LLVM_ABI FunctionCallee createDispatchDeinitFunction();
 
   /// Declarations for LLVM-IR types (simple, array, function and structure) are
   /// generated below. Their names are defined and used in OpenMPKinds.def. Here
@@ -3299,9 +3321,10 @@ class OpenMPIRBuilder {
   /// \param AllocaIP           Insert point for allocas
   //
   /// \return Insertion point after generated atomic read IR.
-  InsertPointTy createAtomicRead(const LocationDescription &Loc,
-                                 AtomicOpValue &X, AtomicOpValue &V,
-                                 AtomicOrdering AO, InsertPointTy AllocaIP);
+  LLVM_ABI InsertPointTy createAtomicRead(const LocationDescription &Loc,
+                                          AtomicOpValue &X, AtomicOpValue &V,
+                                          AtomicOrdering AO,
+                                          InsertPointTy AllocaIP);
 
   /// Emit atomic write for : X = Expr --- Only Scalar data types.
   ///
@@ -3313,9 +3336,10 @@ class OpenMPIRBuilder {
   /// \param AllocaIP           Insert point for allocas
   ///
   /// \return Insertion point after generated atomic Write IR.
-  InsertPointTy createAtomicWrite(const LocationDescription &Loc,
-                                  AtomicOpValue &X, Value *Expr,
-                                  AtomicOrdering AO, InsertPointTy AllocaIP);
+  LLVM_ABI InsertPointTy createAtomicWrite(const LocationDescription &Loc,
+                                           AtomicOpValue &X, Value *Expr,
+                                           AtomicOrdering AO,
+                                           InsertPointTy AllocaIP);
 
   /// Emit atomic update for constructs: X = X BinOp Expr ,or X = Expr BinOp X
   /// For complex Operations: X = UpdateOp(X) => CmpExch X, old_X, UpdateOp(X)
@@ -3337,11 +3361,10 @@ class OpenMPIRBuilder {
   ///	                    (e.g. true for X = X BinOp Expr)
   ///
   /// \return Insertion point after generated atomic update IR.
-  InsertPointOrErrorTy
-  createAtomicUpdate(const LocationDescription &Loc, InsertPointTy AllocaIP,
-                     AtomicOpValue &X, Value *Expr, AtomicOrdering AO,
-                     AtomicRMWInst::BinOp RMWOp,
-                     AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr);
+  LLVM_ABI InsertPointOrErrorTy createAtomicUpdate(
+      const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
+      Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
+      AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr);
 
   /// Emit atomic update for constructs: --- Only Scalar data types
   /// V = X; X = X BinOp Expr ,
@@ -3372,12 +3395,11 @@ class OpenMPIRBuilder {
   ///                        'v', not an updated one.
   ///
   /// \return Insertion point after generated atomic capture IR.
-  InsertPointOrErrorTy
-  createAtomicCapture(const LocationDescription &Loc, InsertPointTy AllocaIP,
-                      AtomicOpValue &X, AtomicOpValue &V, Value *Expr,
-                      AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-                      AtomicUpdateCallbackTy &UpdateOp, bool UpdateExpr,
-                      bool IsPostfixUpdate, bool IsXBinopExpr);
+  LLVM_ABI InsertPointOrErrorTy createAtomicCapture(
+      const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
+      AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
+      AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
+      bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr);
 
   /// Emit atomic compare for constructs: --- Only scalar data types
   /// cond-expr-stmt:
@@ -3424,18 +3446,16 @@ class OpenMPIRBuilder {
   ///                     the case the comparison is '=='.
   ///
   /// \return Insertion point after generated atomic capture IR.
-  InsertPointTy
+  LLVM_ABI InsertPointTy
   createAtomicCompare(const LocationDescription &Loc, AtomicOpValue &X,
                       AtomicOpValue &V, AtomicOpValue &R, Value *E, Value *D,
                       AtomicOrdering AO, omp::OMPAtomicCompareOp Op,
                       bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly);
-  InsertPointTy createAtomicCompare(const LocationDescription &Loc,
-                                    AtomicOpValue &X, AtomicOpValue &V,
-                                    AtomicOpValue &R, Value *E, Value *D,
-                                    AtomicOrdering AO,
-                                    omp::OMPAtomicCompareOp Op,
-                                    bool IsXBinopExpr, bool IsPostfixUpdate,
-                                    bool IsFailOnly, AtomicOrdering Failure);
+  LLVM_ABI InsertPointTy createAtomicCompare(
+      const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
+      AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
+      omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
+      bool IsFailOnly, AtomicOrdering Failure);
 
   /// Create the control flow structure of a canonical OpenMP loop.
   ///
@@ -3453,11 +3473,11 @@ class OpenMPIRBuilder {
   ///                  and instruction names.
   ///
   /// \returns The CanonicalLoopInfo that represents the emitted loop.
-  CanonicalLoopInfo *createLoopSkeleton(DebugLoc DL, Value *TripCount,
-                                        Function *F,
-                                        BasicBlock *PreInsertBefore,
-                                        BasicBlock *PostInsertBefore,
-                                        const Twine &Name = {});
+  LLVM_ABI CanonicalLoopInfo *createLoopSkeleton(DebugLoc DL, Value *TripCount,
+                                                 Function *F,
+                                                 BasicBlock *PreInsertBefore,
+                                                 BasicBlock *PostInsertBefore,
+                                                 const Twine &Name = {});
   /// OMP Offload Info Metadata name string
   const std::string ompOffloadInfoName = "omp_offload.info";
 
@@ -3467,7 +3487,7 @@ class OpenMPIRBuilder {
   ///
   /// \param M         Module to load Metadata info from. Module passed maybe
   /// loaded from bitcode file, i.e, different from OpenMPIRBuilder::M module.
-  void loadOffloadInfoMetadata(Module &M);
+  LLVM_ABI void loadOffloadInfoMetadata(Module &M);
 
   /// Loads all the offload entries information from the host IR
   /// metadata read from the file passed in as the HostFilePath argument. This
@@ -3476,7 +3496,7 @@ class OpenMPIRBuilder {
   /// \param HostFilePath The path to the host IR file,
   /// used to load in offload metadata for the device, allowing host and device
   /// to maintain the same metadata mapping.
-  void loadOffloadInfoMetadata(StringRef HostFilePath);
+  LLVM_ABI void loadOffloadInfoMetadata(StringRef HostFilePath);
 
   /// Gets (if variable with the given name already exist) or creates
   /// internal global variable with the specified Name. The created variable has
@@ -3484,8 +3504,9 @@ class OpenMPIRBuilder {
   /// \param Ty Type of the global variable. If it is exist already the type
   /// must be the same.
   /// \param Name Name of the variable.
-  GlobalVariable *getOrCreateInternalVariable(Type *Ty, const StringRef &Name,
-                                              unsigned AddressSpace = 0);
+  LLVM_ABI GlobalVariable *
+  getOrCreateInternalVariable(Type *Ty, const StringRef &Name,
+                              unsigned AddressSpace = 0);
 };
 
 /// Class to represented the control flow structure of an OpenMP canonical loop.
@@ -3636,7 +3657,7 @@ class CanonicalLoopInfo {
   /// Code that must be execute before any loop iteration can be emitted here,
   /// such as computing the loop trip count and begin lifetime markers. Code in
   /// the preheader is not considered part of the canonical loop.
-  BasicBlock *getPreheader() const;
+  LLVM_ABI BasicBlock *getPreheader() const;
 
   /// The header is the entry for each iteration. In the canonical control flow,
   /// it only contains the PHINode for the induction variable.
@@ -3736,11 +3757,11 @@ class CanonicalLoopInfo {
   }
 
   /// Consistency self-check.
-  void assertOK() const;
+  LLVM_ABI void assertOK() const;
 
   /// Invalidate this loop. That is, the underlying IR does not fulfill the
   /// requirements of an OpenMP canonical loop anymore.
-  void invalidate();
+  LLVM_ABI void invalidate();
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/FuzzMutate/FuzzerCLI.h b/external/llvm-project/llvm/include/llvm/FuzzMutate/FuzzerCLI.h
index d6518107b24b..3641b82f37d8 100644
--- a/external/llvm-project/llvm/include/llvm/FuzzMutate/FuzzerCLI.h
+++ b/external/llvm-project/llvm/include/llvm/FuzzMutate/FuzzerCLI.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_FUZZMUTATE_FUZZERCLI_H
 #define LLVM_FUZZMUTATE_FUZZERCLI_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
 #include <stddef.h>
 
@@ -24,7 +25,7 @@ class StringRef;
 /// Parse cl::opts from a fuzz target commandline.
 ///
 /// This handles all arguments after -ignore_remaining_args=1 as cl::opts.
-void parseFuzzerCLOpts(int ArgC, char *ArgV[]);
+LLVM_ABI void parseFuzzerCLOpts(int ArgC, char *ArgV[]);
 
 /// Handle backend options that are encoded in the executable name.
 ///
@@ -35,11 +36,11 @@ void parseFuzzerCLOpts(int ArgC, char *ArgV[]);
 ///
 /// This is meant to be used for environments like OSS-Fuzz that aren't capable
 /// of passing in command line arguments in the normal way.
-void handleExecNameEncodedBEOpts(StringRef ExecName);
+LLVM_ABI void handleExecNameEncodedBEOpts(StringRef ExecName);
 
 /// Handle optimizer options which are encoded in the executable name.
 /// Same semantics as in 'handleExecNameEncodedBEOpts'.
-void handleExecNameEncodedOptimizerOpts(StringRef ExecName);
+LLVM_ABI void handleExecNameEncodedOptimizerOpts(StringRef ExecName);
 
 using FuzzerTestFun = int (*)(const uint8_t *Data, size_t Size);
 using FuzzerInitFun = int (*)(int *argc, char ***argv);
@@ -48,7 +49,7 @@ using FuzzerInitFun = int (*)(int *argc, char ***argv);
 ///
 /// Useful for testing fuzz targets without linking to libFuzzer. Finds inputs
 /// in the argument list in a libFuzzer compatible way.
-int runFuzzerOnInputs(
+LLVM_ABI int runFuzzerOnInputs(
     int ArgC, char *ArgV[], FuzzerTestFun TestOne,
     FuzzerInitFun Init = [](int *, char ***) { return 0; });
 
diff --git a/external/llvm-project/llvm/include/llvm/FuzzMutate/IRMutator.h b/external/llvm-project/llvm/include/llvm/FuzzMutate/IRMutator.h
index dd4534bd9d1a..4b02578792bc 100644
--- a/external/llvm-project/llvm/include/llvm/FuzzMutate/IRMutator.h
+++ b/external/llvm-project/llvm/include/llvm/FuzzMutate/IRMutator.h
@@ -19,6 +19,7 @@
 #define LLVM_FUZZMUTATE_IRMUTATOR_H
 
 #include "llvm/FuzzMutate/OpDescriptor.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <optional>
 
@@ -32,7 +33,7 @@ struct RandomIRBuilder;
 
 /// Base class for describing how to mutate a module. mutation functions for
 /// each IR unit forward to the contained unit.
-class IRMutationStrategy {
+class LLVM_ABI IRMutationStrategy {
 public:
   virtual ~IRMutationStrategy() = default;
 
@@ -75,18 +76,18 @@ class IRMutator {
   ///
   /// \param M module
   /// \return number of objects in module
-  static size_t getModuleSize(const Module &M);
+  LLVM_ABI static size_t getModuleSize(const Module &M);
 
   /// Mutate given module. No change will be made if no strategy is selected.
   ///
   /// \param M  module to mutate
   /// \param Seed seed for random mutation
   /// \param MaxSize max module size (see getModuleSize)
-  void mutateModule(Module &M, int Seed, size_t MaxSize);
+  LLVM_ABI void mutateModule(Module &M, int Seed, size_t MaxSize);
 };
 
 /// Strategy that injects operations into the function.
-class InjectorIRStrategy : public IRMutationStrategy {
+class LLVM_ABI InjectorIRStrategy : public IRMutationStrategy {
   std::vector<fuzzerop::OpDescriptor> Operations;
 
   std::optional<fuzzerop::OpDescriptor> chooseOperation(Value *Src,
@@ -109,7 +110,7 @@ class InjectorIRStrategy : public IRMutationStrategy {
 };
 
 /// Strategy that deletes instructions when the Module is too large.
-class InstDeleterIRStrategy : public IRMutationStrategy {
+class LLVM_ABI InstDeleterIRStrategy : public IRMutationStrategy {
 public:
   uint64_t getWeight(size_t CurrentSize, size_t MaxSize,
                      uint64_t CurrentWeight) override;
@@ -120,7 +121,7 @@ class InstDeleterIRStrategy : public IRMutationStrategy {
 };
 
 /// Strategy that modifies instruction attributes and operands.
-class InstModificationIRStrategy : public IRMutationStrategy {
+class LLVM_ABI InstModificationIRStrategy : public IRMutationStrategy {
 public:
   uint64_t getWeight(size_t CurrentSize, size_t MaxSize,
                      uint64_t CurrentWeight) override {
@@ -134,7 +135,7 @@ class InstModificationIRStrategy : public IRMutationStrategy {
 /// Strategy that generates new function calls and inserts function signatures
 /// to the modules. If any signatures are present in the module it will be
 /// called.
-class InsertFunctionStrategy : public IRMutationStrategy {
+class LLVM_ABI InsertFunctionStrategy : public IRMutationStrategy {
 public:
   uint64_t getWeight(size_t CurrentSize, size_t MaxSize,
                      uint64_t CurrentWeight) override {
@@ -146,7 +147,7 @@ class InsertFunctionStrategy : public IRMutationStrategy {
 };
 
 /// Strategy to split a random block and insert a random CFG in between.
-class InsertCFGStrategy : public IRMutationStrategy {
+class LLVM_ABI InsertCFGStrategy : public IRMutationStrategy {
 private:
   uint64_t MaxNumCases;
   enum CFGToSink { Return, DirectSink, SinkOrSelfLoop, EndOfCFGToLink };
@@ -166,7 +167,7 @@ class InsertCFGStrategy : public IRMutationStrategy {
 };
 
 /// Strategy to insert PHI Nodes at the head of each basic block.
-class InsertPHIStrategy : public IRMutationStrategy {
+class LLVM_ABI InsertPHIStrategy : public IRMutationStrategy {
 public:
   uint64_t getWeight(size_t CurrentSize, size_t MaxSize,
                      uint64_t CurrentWeight) override {
@@ -178,7 +179,7 @@ class InsertPHIStrategy : public IRMutationStrategy {
 
 /// Strategy to select a random instruction and add a new sink (user) to it to
 /// increate data dependency.
-class SinkInstructionStrategy : public IRMutationStrategy {
+class LLVM_ABI SinkInstructionStrategy : public IRMutationStrategy {
 public:
   uint64_t getWeight(size_t CurrentSize, size_t MaxSize,
                      uint64_t CurrentWeight) override {
@@ -191,7 +192,7 @@ class SinkInstructionStrategy : public IRMutationStrategy {
 
 /// Strategy to randomly select a block and shuffle the operations without
 /// affecting data dependency.
-class ShuffleBlockStrategy : public IRMutationStrategy {
+class LLVM_ABI ShuffleBlockStrategy : public IRMutationStrategy {
 public:
   uint64_t getWeight(size_t CurrentSize, size_t MaxSize,
                      uint64_t CurrentWeight) override {
@@ -206,8 +207,8 @@ class ShuffleBlockStrategy : public IRMutationStrategy {
 /// \param Data Bitcode we are going to parse
 /// \param Size Size of the 'Data' in bytes
 /// \return New module or nullptr in case of error
-std::unique_ptr<Module> parseModule(const uint8_t *Data, size_t Size,
-                                    LLVMContext &Context);
+LLVM_ABI std::unique_ptr<Module> parseModule(const uint8_t *Data, size_t Size,
+                                             LLVMContext &Context);
 
 /// Fuzzer friendly interface for the llvm bitcode printer.
 ///
@@ -216,13 +217,13 @@ std::unique_ptr<Module> parseModule(const uint8_t *Data, size_t Size,
 /// \param MaxSize Size of the destination buffer
 /// \return Number of bytes that were written. When module size exceeds MaxSize
 ///         returns 0 and leaves Dest unchanged.
-size_t writeModule(const Module &M, uint8_t *Dest, size_t MaxSize);
+LLVM_ABI size_t writeModule(const Module &M, uint8_t *Dest, size_t MaxSize);
 
 /// Try to parse module and verify it. May output verification errors to the
 /// errs().
 /// \return New module or nullptr in case of error.
-std::unique_ptr<Module> parseAndVerify(const uint8_t *Data, size_t Size,
-                                       LLVMContext &Context);
+LLVM_ABI std::unique_ptr<Module>
+parseAndVerify(const uint8_t *Data, size_t Size, LLVMContext &Context);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/FuzzMutate/OpDescriptor.h b/external/llvm-project/llvm/include/llvm/FuzzMutate/OpDescriptor.h
index 771b711dd1b4..9c449c100ae8 100644
--- a/external/llvm-project/llvm/include/llvm/FuzzMutate/OpDescriptor.h
+++ b/external/llvm-project/llvm/include/llvm/FuzzMutate/OpDescriptor.h
@@ -21,6 +21,7 @@
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <functional>
 
 namespace llvm {
@@ -29,8 +30,8 @@ namespace fuzzerop {
 
 /// @{
 /// Populate a small list of potentially interesting constants of a given type.
-void makeConstantsWithType(Type *T, std::vector<Constant *> &Cs);
-std::vector<Constant *> makeConstantsWithType(Type *T);
+LLVM_ABI void makeConstantsWithType(Type *T, std::vector<Constant *> &Cs);
+LLVM_ABI std::vector<Constant *> makeConstantsWithType(Type *T);
 /// @}
 
 /// A matcher/generator for finding suitable values for the next source in an
diff --git a/external/llvm-project/llvm/include/llvm/FuzzMutate/Operations.h b/external/llvm-project/llvm/include/llvm/FuzzMutate/Operations.h
index 84155730a93c..4f972886d449 100644
--- a/external/llvm-project/llvm/include/llvm/FuzzMutate/Operations.h
+++ b/external/llvm-project/llvm/include/llvm/FuzzMutate/Operations.h
@@ -17,37 +17,44 @@
 #include "llvm/FuzzMutate/OpDescriptor.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// Getters for the default sets of operations, per general category.
 /// @{
-void describeFuzzerIntOps(std::vector<fuzzerop::OpDescriptor> &Ops);
-void describeFuzzerFloatOps(std::vector<fuzzerop::OpDescriptor> &Ops);
-void describeFuzzerControlFlowOps(std::vector<fuzzerop::OpDescriptor> &Ops);
-void describeFuzzerPointerOps(std::vector<fuzzerop::OpDescriptor> &Ops);
-void describeFuzzerAggregateOps(std::vector<fuzzerop::OpDescriptor> &Ops);
-void describeFuzzerVectorOps(std::vector<fuzzerop::OpDescriptor> &Ops);
-void describeFuzzerUnaryOperations(std::vector<fuzzerop::OpDescriptor> &Ops);
-void describeFuzzerOtherOps(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void describeFuzzerIntOps(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void describeFuzzerFloatOps(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void
+describeFuzzerControlFlowOps(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void
+describeFuzzerPointerOps(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void
+describeFuzzerAggregateOps(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void describeFuzzerVectorOps(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void
+describeFuzzerUnaryOperations(std::vector<fuzzerop::OpDescriptor> &Ops);
+LLVM_ABI void describeFuzzerOtherOps(std::vector<fuzzerop::OpDescriptor> &Ops);
 /// @}
 
 namespace fuzzerop {
 
 /// Descriptors for individual operations.
 /// @{
-OpDescriptor selectDescriptor(unsigned Weight);
-OpDescriptor fnegDescriptor(unsigned Weight);
-OpDescriptor binOpDescriptor(unsigned Weight, Instruction::BinaryOps Op);
-OpDescriptor cmpOpDescriptor(unsigned Weight, Instruction::OtherOps CmpOp,
-                             CmpInst::Predicate Pred);
-OpDescriptor splitBlockDescriptor(unsigned Weight);
-OpDescriptor gepDescriptor(unsigned Weight);
-OpDescriptor extractValueDescriptor(unsigned Weight);
-OpDescriptor insertValueDescriptor(unsigned Weight);
-OpDescriptor extractElementDescriptor(unsigned Weight);
-OpDescriptor insertElementDescriptor(unsigned Weight);
-OpDescriptor shuffleVectorDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor selectDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor fnegDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor binOpDescriptor(unsigned Weight,
+                                      Instruction::BinaryOps Op);
+LLVM_ABI OpDescriptor cmpOpDescriptor(unsigned Weight,
+                                      Instruction::OtherOps CmpOp,
+                                      CmpInst::Predicate Pred);
+LLVM_ABI OpDescriptor splitBlockDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor gepDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor extractValueDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor insertValueDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor extractElementDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor insertElementDescriptor(unsigned Weight);
+LLVM_ABI OpDescriptor shuffleVectorDescriptor(unsigned Weight);
 
 /// @}
 
diff --git a/external/llvm-project/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h b/external/llvm-project/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h
index af6d7216b77b..65b1ca686416 100644
--- a/external/llvm-project/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
 #include <random>
 
 namespace llvm {
@@ -49,11 +50,12 @@ struct RandomIRBuilder {
 
   /// Create a stack memory at the head of the function, store \c Init to the
   /// memory if provided.
-  AllocaInst *createStackMemory(Function *F, Type *Ty, Value *Init = nullptr);
+  LLVM_ABI AllocaInst *createStackMemory(Function *F, Type *Ty,
+                                         Value *Init = nullptr);
   /// Find or create a global variable. It will be initialized by random
   /// constants that satisfies \c Pred. It will also report whether this global
   /// variable found or created.
-  std::pair<GlobalVariable *, bool>
+  LLVM_ABI std::pair<GlobalVariable *, bool>
   findOrCreateGlobalVariable(Module *M, ArrayRef<Value *> Srcs,
                              fuzzerop::SourcePred Pred);
   enum SourceType {
@@ -67,19 +69,22 @@ struct RandomIRBuilder {
   /// Find a "source" for some operation, which will be used in one of the
   /// operation's operands. This either selects an instruction in \c Insts or
   /// returns some new arbitrary Value.
-  Value *findOrCreateSource(BasicBlock &BB, ArrayRef<Instruction *> Insts);
+  LLVM_ABI Value *findOrCreateSource(BasicBlock &BB,
+                                     ArrayRef<Instruction *> Insts);
   /// Find a "source" for some operation, which will be used in one of the
   /// operation's operands. This either selects an instruction in \c Insts that
   /// matches \c Pred, or returns some new Value that matches \c Pred. The
   /// values in \c Srcs should be source operands that have already been
   /// selected.
-  Value *findOrCreateSource(BasicBlock &BB, ArrayRef<Instruction *> Insts,
+  LLVM_ABI Value *findOrCreateSource(BasicBlock &BB,
+                                     ArrayRef<Instruction *> Insts,
+                                     ArrayRef<Value *> Srcs,
+                                     fuzzerop::SourcePred Pred,
+                                     bool allowConstant = true);
+  /// Create some Value suitable as a source for some operation.
+  LLVM_ABI Value *newSource(BasicBlock &BB, ArrayRef<Instruction *> Insts,
                             ArrayRef<Value *> Srcs, fuzzerop::SourcePred Pred,
                             bool allowConstant = true);
-  /// Create some Value suitable as a source for some operation.
-  Value *newSource(BasicBlock &BB, ArrayRef<Instruction *> Insts,
-                   ArrayRef<Value *> Srcs, fuzzerop::SourcePred Pred,
-                   bool allowConstant = true);
 
   enum SinkType {
     /// TODO: Also consider pointers in function argument.
@@ -92,17 +97,18 @@ struct RandomIRBuilder {
   };
   /// Find a viable user for \c V in \c Insts, which should all be contained in
   /// \c BB. This may also create some new instruction in \c BB and use that.
-  Instruction *connectToSink(BasicBlock &BB, ArrayRef<Instruction *> Insts,
-                             Value *V);
+  LLVM_ABI Instruction *connectToSink(BasicBlock &BB,
+                                      ArrayRef<Instruction *> Insts, Value *V);
   /// Create a user for \c V in \c BB.
-  Instruction *newSink(BasicBlock &BB, ArrayRef<Instruction *> Insts, Value *V);
-  Value *findPointer(BasicBlock &BB, ArrayRef<Instruction *> Insts);
+  LLVM_ABI Instruction *newSink(BasicBlock &BB, ArrayRef<Instruction *> Insts,
+                                Value *V);
+  LLVM_ABI Value *findPointer(BasicBlock &BB, ArrayRef<Instruction *> Insts);
   /// Return a uniformly choosen type from \c AllowedTypes
-  Type *randomType();
-  Function *createFunctionDeclaration(Module &M, uint64_t ArgNum);
-  Function *createFunctionDeclaration(Module &M);
-  Function *createFunctionDefinition(Module &M, uint64_t ArgNum);
-  Function *createFunctionDefinition(Module &M);
+  LLVM_ABI Type *randomType();
+  LLVM_ABI Function *createFunctionDeclaration(Module &M, uint64_t ArgNum);
+  LLVM_ABI Function *createFunctionDeclaration(Module &M);
+  LLVM_ABI Function *createFunctionDefinition(Module &M, uint64_t ArgNum);
+  LLVM_ABI Function *createFunctionDefinition(Module &M);
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/IR/AbstractCallSite.h b/external/llvm-project/llvm/include/llvm/IR/AbstractCallSite.h
index 50afe016f0d6..9e24ae7d1b43 100644
--- a/external/llvm-project/llvm/include/llvm/IR/AbstractCallSite.h
+++ b/external/llvm-project/llvm/include/llvm/IR/AbstractCallSite.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 
 namespace llvm {
@@ -95,15 +96,16 @@ class AbstractCallSite {
   /// If the use is not a callee use of a call or invoke instruction, the
   /// callback metadata is used to determine the argument <-> parameter mapping
   /// as well as the callee of the abstract call site.
-  AbstractCallSite(const Use *U);
+  LLVM_ABI AbstractCallSite(const Use *U);
 
   /// Add operand uses of \p CB that represent callback uses into
   /// \p CallbackUses.
   ///
   /// All uses added to \p CallbackUses can be used to create abstract call
   /// sites for which AbstractCallSite::isCallbackCall() will return true.
-  static void getCallbackUses(const CallBase &CB,
-                              SmallVectorImpl<const Use *> &CallbackUses);
+  LLVM_ABI static void
+  getCallbackUses(const CallBase &CB,
+                  SmallVectorImpl<const Use *> &CallbackUses);
 
   /// Conversion operator to conveniently check for a valid/initialized ACS.
   explicit operator bool() const { return CB != nullptr; }
diff --git a/external/llvm-project/llvm/include/llvm/IR/Analysis.h b/external/llvm-project/llvm/include/llvm/IR/Analysis.h
index dd43f5ee5f70..8ab70dd9ed59 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Analysis.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Analysis.h
@@ -13,6 +13,7 @@
 #define LLVM_IR_ANALYSIS_H
 
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -56,8 +57,8 @@ template <typename IRUnitT> class AllAnalysesOn {
 
 template <typename IRUnitT> AnalysisSetKey AllAnalysesOn<IRUnitT>::SetKey;
 
-extern template class AllAnalysesOn<Module>;
-extern template class AllAnalysesOn<Function>;
+extern template class LLVM_TEMPLATE_ABI AllAnalysesOn<Module>;
+extern template class LLVM_TEMPLATE_ABI AllAnalysesOn<Function>;
 
 /// Represents analyses that only rely on functions' control flow.
 ///
@@ -74,7 +75,7 @@ class CFGAnalyses {
   static AnalysisSetKey *ID() { return &SetKey; }
 
 private:
-  static AnalysisSetKey SetKey;
+  LLVM_ABI static AnalysisSetKey SetKey;
 };
 
 /// A set of analyses that are preserved following a run of a transformation
@@ -310,7 +311,7 @@ class PreservedAnalyses {
 
 private:
   /// A special key used to indicate all analyses.
-  static AnalysisSetKey AllAnalysesKey;
+  LLVM_ABI static AnalysisSetKey AllAnalysesKey;
 
   /// The IDs of analyses and analysis sets that are preserved.
   SmallPtrSet<void *, 2> PreservedIDs;
diff --git a/external/llvm-project/llvm/include/llvm/IR/Argument.h b/external/llvm-project/llvm/include/llvm/IR/Argument.h
index 5be58d7eca06..60854b17094b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Argument.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Argument.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -37,8 +38,8 @@ class Argument final : public Value {
 
 public:
   /// Argument constructor.
-  explicit Argument(Type *Ty, const Twine &Name = "", Function *F = nullptr,
-                    unsigned ArgNo = 0);
+  LLVM_ABI explicit Argument(Type *Ty, const Twine &Name = "",
+                             Function *F = nullptr, unsigned ArgNo = 0);
 
   inline const Function *getParent() const { return Parent; }
   inline       Function *getParent()       { return Parent; }
@@ -56,133 +57,133 @@ class Argument final : public Value {
   /// addrspace(0).
   /// If AllowUndefOrPoison is true, respect the semantics of nonnull attribute
   /// and return true even if the argument can be undef or poison.
-  bool hasNonNullAttr(bool AllowUndefOrPoison = true) const;
+  LLVM_ABI bool hasNonNullAttr(bool AllowUndefOrPoison = true) const;
 
   /// If this argument has the dereferenceable attribute, return the number of
   /// bytes known to be dereferenceable. Otherwise, zero is returned.
-  uint64_t getDereferenceableBytes() const;
+  LLVM_ABI uint64_t getDereferenceableBytes() const;
 
   /// If this argument has the dereferenceable_or_null attribute, return the
   /// number of bytes known to be dereferenceable. Otherwise, zero is returned.
-  uint64_t getDereferenceableOrNullBytes() const;
+  LLVM_ABI uint64_t getDereferenceableOrNullBytes() const;
 
   /// If this argument has nofpclass attribute, return the mask representing
   /// disallowed floating-point values. Otherwise, fcNone is returned.
-  FPClassTest getNoFPClass() const;
+  LLVM_ABI FPClassTest getNoFPClass() const;
 
   /// If this argument has a range attribute, return the value range of the
   /// argument. Otherwise, std::nullopt is returned.
-  std::optional<ConstantRange> getRange() const;
+  LLVM_ABI std::optional<ConstantRange> getRange() const;
 
   /// Return true if this argument has the byval attribute.
-  bool hasByValAttr() const;
+  LLVM_ABI bool hasByValAttr() const;
 
   /// Return true if this argument has the byref attribute.
-  bool hasByRefAttr() const;
+  LLVM_ABI bool hasByRefAttr() const;
 
   /// Return true if this argument has the swiftself attribute.
-  bool hasSwiftSelfAttr() const;
+  LLVM_ABI bool hasSwiftSelfAttr() const;
 
   /// Return true if this argument has the swifterror attribute.
-  bool hasSwiftErrorAttr() const;
+  LLVM_ABI bool hasSwiftErrorAttr() const;
 
   /// Return true if this argument has the byval, inalloca, or preallocated
   /// attribute. These attributes represent arguments being passed by value,
   /// with an associated copy between the caller and callee
-  bool hasPassPointeeByValueCopyAttr() const;
+  LLVM_ABI bool hasPassPointeeByValueCopyAttr() const;
 
   /// If this argument satisfies has hasPassPointeeByValueAttr, return the
   /// in-memory ABI size copied to the stack for the call. Otherwise, return 0.
-  uint64_t getPassPointeeByValueCopySize(const DataLayout &DL) const;
+  LLVM_ABI uint64_t getPassPointeeByValueCopySize(const DataLayout &DL) const;
 
   /// Return true if this argument has the byval, sret, inalloca, preallocated,
   /// or byref attribute. These attributes represent arguments being passed by
   /// value (which may or may not involve a stack copy)
-  bool hasPointeeInMemoryValueAttr() const;
+  LLVM_ABI bool hasPointeeInMemoryValueAttr() const;
 
   /// If hasPointeeInMemoryValueAttr returns true, the in-memory ABI type is
   /// returned. Otherwise, nullptr.
-  Type *getPointeeInMemoryValueType() const;
+  LLVM_ABI Type *getPointeeInMemoryValueType() const;
 
   /// If this is a byval or inalloca argument, return its alignment.
   /// FIXME: Remove this function once transition to Align is over.
   /// Use getParamAlign() instead.
-  LLVM_DEPRECATED("Use getParamAlign() instead", "getParamAlign")
-  uint64_t getParamAlignment() const;
+  LLVM_ABI LLVM_DEPRECATED("Use getParamAlign() instead",
+                           "getParamAlign") uint64_t getParamAlignment() const;
 
   /// If this is a byval or inalloca argument, return its alignment.
-  MaybeAlign getParamAlign() const;
+  LLVM_ABI MaybeAlign getParamAlign() const;
 
-  MaybeAlign getParamStackAlign() const;
+  LLVM_ABI MaybeAlign getParamStackAlign() const;
 
   /// If this is a byval argument, return its type.
-  Type *getParamByValType() const;
+  LLVM_ABI Type *getParamByValType() const;
 
   /// If this is an sret argument, return its type.
-  Type *getParamStructRetType() const;
+  LLVM_ABI Type *getParamStructRetType() const;
 
   /// If this is a byref argument, return its type.
-  Type *getParamByRefType() const;
+  LLVM_ABI Type *getParamByRefType() const;
 
   /// If this is an inalloca argument, return its type.
-  Type *getParamInAllocaType() const;
+  LLVM_ABI Type *getParamInAllocaType() const;
 
   /// Return true if this argument has the nest attribute.
-  bool hasNestAttr() const;
+  LLVM_ABI bool hasNestAttr() const;
 
   /// Return true if this argument has the noalias attribute.
-  bool hasNoAliasAttr() const;
+  LLVM_ABI bool hasNoAliasAttr() const;
 
   /// Return true if this argument has the nocapture attribute.
-  bool hasNoCaptureAttr() const;
+  LLVM_ABI bool hasNoCaptureAttr() const;
 
   /// Return true if this argument has the nofree attribute.
-  bool hasNoFreeAttr() const;
+  LLVM_ABI bool hasNoFreeAttr() const;
 
   /// Return true if this argument has the sret attribute.
-  bool hasStructRetAttr() const;
+  LLVM_ABI bool hasStructRetAttr() const;
 
   /// Return true if this argument has the inreg attribute.
-  bool hasInRegAttr() const;
+  LLVM_ABI bool hasInRegAttr() const;
 
   /// Return true if this argument has the returned attribute.
-  bool hasReturnedAttr() const;
+  LLVM_ABI bool hasReturnedAttr() const;
 
   /// Return true if this argument has the readonly or readnone attribute.
-  bool onlyReadsMemory() const;
+  LLVM_ABI bool onlyReadsMemory() const;
 
   /// Return true if this argument has the inalloca attribute.
-  bool hasInAllocaAttr() const;
+  LLVM_ABI bool hasInAllocaAttr() const;
 
   /// Return true if this argument has the preallocated attribute.
-  bool hasPreallocatedAttr() const;
+  LLVM_ABI bool hasPreallocatedAttr() const;
 
   /// Return true if this argument has the zext attribute.
-  bool hasZExtAttr() const;
+  LLVM_ABI bool hasZExtAttr() const;
 
   /// Return true if this argument has the sext attribute.
-  bool hasSExtAttr() const;
+  LLVM_ABI bool hasSExtAttr() const;
 
   /// Add attributes to an argument.
-  void addAttrs(AttrBuilder &B);
+  LLVM_ABI void addAttrs(AttrBuilder &B);
 
-  void addAttr(Attribute::AttrKind Kind);
+  LLVM_ABI void addAttr(Attribute::AttrKind Kind);
 
-  void addAttr(Attribute Attr);
+  LLVM_ABI void addAttr(Attribute Attr);
 
   /// Remove attributes from an argument.
-  void removeAttr(Attribute::AttrKind Kind);
+  LLVM_ABI void removeAttr(Attribute::AttrKind Kind);
 
-  void removeAttrs(const AttributeMask &AM);
+  LLVM_ABI void removeAttrs(const AttributeMask &AM);
 
   /// Check if an argument has a given attribute.
-  bool hasAttribute(Attribute::AttrKind Kind) const;
+  LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const;
 
-  bool hasAttribute(StringRef Kind) const;
+  LLVM_ABI bool hasAttribute(StringRef Kind) const;
 
-  Attribute getAttribute(Attribute::AttrKind Kind) const;
+  LLVM_ABI Attribute getAttribute(Attribute::AttrKind Kind) const;
 
-  AttributeSet getAttributes() const;
+  LLVM_ABI AttributeSet getAttributes() const;
 
   /// Method for support type inquiry through isa, cast, and dyn_cast.
   static bool classof(const Value *V) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/AssemblyAnnotationWriter.h b/external/llvm-project/llvm/include/llvm/IR/AssemblyAnnotationWriter.h
index 3fd3c57a6796..d68bcbf85b2b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/AssemblyAnnotationWriter.h
+++ b/external/llvm-project/llvm/include/llvm/IR/AssemblyAnnotationWriter.h
@@ -16,6 +16,8 @@
 #ifndef LLVM_IR_ASSEMBLYANNOTATIONWRITER_H
 #define LLVM_IR_ASSEMBLYANNOTATIONWRITER_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 class Function;
@@ -24,7 +26,7 @@ class Instruction;
 class Value;
 class formatted_raw_ostream;
 
-class AssemblyAnnotationWriter {
+class LLVM_ABI AssemblyAnnotationWriter {
 public:
   virtual ~AssemblyAnnotationWriter();
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/Assumptions.h b/external/llvm-project/llvm/include/llvm/IR/Assumptions.h
index 2d2ecfbde6e6..f1663f3d6d2b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Assumptions.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Assumptions.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -29,7 +30,7 @@ constexpr StringRef AssumptionAttrKey = "llvm.assume";
 
 /// A set of known assumption strings that are accepted without warning and
 /// which can be recommended as typo correction.
-extern StringSet<> KnownAssumptionStrings;
+LLVM_ABI extern StringSet<> KnownAssumptionStrings;
 
 /// Helper that allows to insert a new assumption string in the known assumption
 /// set by creating a (static) object.
@@ -49,25 +50,27 @@ struct KnownAssumptionString {
 };
 
 /// Return true if \p F has the assumption \p AssumptionStr attached.
-bool hasAssumption(const Function &F,
-                   const KnownAssumptionString &AssumptionStr);
+LLVM_ABI bool hasAssumption(const Function &F,
+                            const KnownAssumptionString &AssumptionStr);
 
 /// Return true if \p CB or the callee has the assumption \p AssumptionStr
 /// attached.
-bool hasAssumption(const CallBase &CB,
-                   const KnownAssumptionString &AssumptionStr);
+LLVM_ABI bool hasAssumption(const CallBase &CB,
+                            const KnownAssumptionString &AssumptionStr);
 
 /// Return the set of all assumptions for the function \p F.
-DenseSet<StringRef> getAssumptions(const Function &F);
+LLVM_ABI DenseSet<StringRef> getAssumptions(const Function &F);
 
 /// Return the set of all assumptions for the call \p CB.
-DenseSet<StringRef> getAssumptions(const CallBase &CB);
+LLVM_ABI DenseSet<StringRef> getAssumptions(const CallBase &CB);
 
 /// Appends the set of assumptions \p Assumptions to \F.
-bool addAssumptions(Function &F, const DenseSet<StringRef> &Assumptions);
+LLVM_ABI bool addAssumptions(Function &F,
+                             const DenseSet<StringRef> &Assumptions);
 
 /// Appends the set of assumptions \p Assumptions to \CB.
-bool addAssumptions(CallBase &CB, const DenseSet<StringRef> &Assumptions);
+LLVM_ABI bool addAssumptions(CallBase &CB,
+                             const DenseSet<StringRef> &Assumptions);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/Attributes.h b/external/llvm-project/llvm/include/llvm/IR/Attributes.h
index 7f853efdb47c..5a23aea902be 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Attributes.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Attributes.h
@@ -22,6 +22,7 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ModRef.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include <cassert>
@@ -114,14 +115,14 @@ class Attribute {
            Kind <= LastConstantRangeListAttr;
   }
 
-  static bool canUseAsFnAttr(AttrKind Kind);
-  static bool canUseAsParamAttr(AttrKind Kind);
-  static bool canUseAsRetAttr(AttrKind Kind);
+  LLVM_ABI static bool canUseAsFnAttr(AttrKind Kind);
+  LLVM_ABI static bool canUseAsParamAttr(AttrKind Kind);
+  LLVM_ABI static bool canUseAsRetAttr(AttrKind Kind);
 
-  static bool intersectMustPreserve(AttrKind Kind);
-  static bool intersectWithAnd(AttrKind Kind);
-  static bool intersectWithMin(AttrKind Kind);
-  static bool intersectWithCustom(AttrKind Kind);
+  LLVM_ABI static bool intersectMustPreserve(AttrKind Kind);
+  LLVM_ABI static bool intersectWithAnd(AttrKind Kind);
+  LLVM_ABI static bool intersectWithMin(AttrKind Kind);
+  LLVM_ABI static bool intersectWithCustom(AttrKind Kind);
 
 private:
   AttributeImpl *pImpl = nullptr;
@@ -136,37 +137,49 @@ class Attribute {
   //===--------------------------------------------------------------------===//
 
   /// Return a uniquified Attribute object.
-  static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val = 0);
-  static Attribute get(LLVMContext &Context, StringRef Kind,
-                       StringRef Val = StringRef());
-  static Attribute get(LLVMContext &Context, AttrKind Kind, Type *Ty);
-  static Attribute get(LLVMContext &Context, AttrKind Kind,
-                       const ConstantRange &CR);
-  static Attribute get(LLVMContext &Context, AttrKind Kind,
-                       ArrayRef<ConstantRange> Val);
+  LLVM_ABI static Attribute get(LLVMContext &Context, AttrKind Kind,
+                                uint64_t Val = 0);
+  LLVM_ABI static Attribute get(LLVMContext &Context, StringRef Kind,
+                                StringRef Val = StringRef());
+  LLVM_ABI static Attribute get(LLVMContext &Context, AttrKind Kind, Type *Ty);
+  LLVM_ABI static Attribute get(LLVMContext &Context, AttrKind Kind,
+                                const ConstantRange &CR);
+  LLVM_ABI static Attribute get(LLVMContext &Context, AttrKind Kind,
+                                ArrayRef<ConstantRange> Val);
 
   /// Return a uniquified Attribute object that has the specific
   /// alignment set.
-  static Attribute getWithAlignment(LLVMContext &Context, Align Alignment);
-  static Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment);
-  static Attribute getWithDereferenceableBytes(LLVMContext &Context,
-                                              uint64_t Bytes);
-  static Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context,
-                                                     uint64_t Bytes);
-  static Attribute getWithAllocSizeArgs(
-      LLVMContext &Context, unsigned ElemSizeArg,
-      const std::optional<unsigned> &NumElemsArg);
-  static Attribute getWithVScaleRangeArgs(LLVMContext &Context,
-                                          unsigned MinValue, unsigned MaxValue);
-  static Attribute getWithByValType(LLVMContext &Context, Type *Ty);
-  static Attribute getWithStructRetType(LLVMContext &Context, Type *Ty);
-  static Attribute getWithByRefType(LLVMContext &Context, Type *Ty);
-  static Attribute getWithPreallocatedType(LLVMContext &Context, Type *Ty);
-  static Attribute getWithInAllocaType(LLVMContext &Context, Type *Ty);
-  static Attribute getWithUWTableKind(LLVMContext &Context, UWTableKind Kind);
-  static Attribute getWithMemoryEffects(LLVMContext &Context, MemoryEffects ME);
-  static Attribute getWithNoFPClass(LLVMContext &Context, FPClassTest Mask);
-  static Attribute getWithCaptureInfo(LLVMContext &Context, CaptureInfo CI);
+  LLVM_ABI static Attribute getWithAlignment(LLVMContext &Context,
+                                             Align Alignment);
+  LLVM_ABI static Attribute getWithStackAlignment(LLVMContext &Context,
+                                                  Align Alignment);
+  LLVM_ABI static Attribute getWithDereferenceableBytes(LLVMContext &Context,
+                                                        uint64_t Bytes);
+  LLVM_ABI static Attribute
+  getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes);
+  LLVM_ABI static Attribute
+  getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg,
+                       const std::optional<unsigned> &NumElemsArg);
+  LLVM_ABI static Attribute getWithAllocKind(LLVMContext &Context,
+                                             AllocFnKind Kind);
+  LLVM_ABI static Attribute getWithVScaleRangeArgs(LLVMContext &Context,
+                                                   unsigned MinValue,
+                                                   unsigned MaxValue);
+  LLVM_ABI static Attribute getWithByValType(LLVMContext &Context, Type *Ty);
+  LLVM_ABI static Attribute getWithStructRetType(LLVMContext &Context,
+                                                 Type *Ty);
+  LLVM_ABI static Attribute getWithByRefType(LLVMContext &Context, Type *Ty);
+  LLVM_ABI static Attribute getWithPreallocatedType(LLVMContext &Context,
+                                                    Type *Ty);
+  LLVM_ABI static Attribute getWithInAllocaType(LLVMContext &Context, Type *Ty);
+  LLVM_ABI static Attribute getWithUWTableKind(LLVMContext &Context,
+                                               UWTableKind Kind);
+  LLVM_ABI static Attribute getWithMemoryEffects(LLVMContext &Context,
+                                                 MemoryEffects ME);
+  LLVM_ABI static Attribute getWithNoFPClass(LLVMContext &Context,
+                                             FPClassTest Mask);
+  LLVM_ABI static Attribute getWithCaptureInfo(LLVMContext &Context,
+                                               CaptureInfo CI);
 
   /// For a typed attribute, return the equivalent attribute with the type
   /// changed to \p ReplacementTy.
@@ -175,45 +188,45 @@ class Attribute {
     return get(Context, getKindAsEnum(), ReplacementTy);
   }
 
-  static Attribute::AttrKind getAttrKindFromName(StringRef AttrName);
+  LLVM_ABI static Attribute::AttrKind getAttrKindFromName(StringRef AttrName);
 
-  static StringRef getNameFromAttrKind(Attribute::AttrKind AttrKind);
+  LLVM_ABI static StringRef getNameFromAttrKind(Attribute::AttrKind AttrKind);
 
   /// Return true if the provided string matches the IR name of an attribute.
   /// example: "noalias" return true but not "NoAlias"
-  static bool isExistingAttribute(StringRef Name);
+  LLVM_ABI static bool isExistingAttribute(StringRef Name);
 
   //===--------------------------------------------------------------------===//
   // Attribute Accessors
   //===--------------------------------------------------------------------===//
 
   /// Return true if the attribute is an Attribute::AttrKind type.
-  bool isEnumAttribute() const;
+  LLVM_ABI bool isEnumAttribute() const;
 
   /// Return true if the attribute is an integer attribute.
-  bool isIntAttribute() const;
+  LLVM_ABI bool isIntAttribute() const;
 
   /// Return true if the attribute is a string (target-dependent)
   /// attribute.
-  bool isStringAttribute() const;
+  LLVM_ABI bool isStringAttribute() const;
 
   /// Return true if the attribute is a type attribute.
-  bool isTypeAttribute() const;
+  LLVM_ABI bool isTypeAttribute() const;
 
   /// Return true if the attribute is a ConstantRange attribute.
-  bool isConstantRangeAttribute() const;
+  LLVM_ABI bool isConstantRangeAttribute() const;
 
   /// Return true if the attribute is a ConstantRangeList attribute.
-  bool isConstantRangeListAttribute() const;
+  LLVM_ABI bool isConstantRangeListAttribute() const;
 
   /// Return true if the attribute is any kind of attribute.
   bool isValid() const { return pImpl; }
 
   /// Return true if the attribute is present.
-  bool hasAttribute(AttrKind Val) const;
+  LLVM_ABI bool hasAttribute(AttrKind Val) const;
 
   /// Return true if the target-dependent attribute is present.
-  bool hasAttribute(StringRef Val) const;
+  LLVM_ABI bool hasAttribute(StringRef Val) const;
 
   /// Returns true if the attribute's kind can be represented as an enum (Enum,
   /// Integer, Type, ConstantRange, or ConstantRangeList attribute).
@@ -221,104 +234,105 @@ class Attribute {
 
   /// Return the attribute's kind as an enum (Attribute::AttrKind). This
   /// requires the attribute be representable as an enum (see: `hasKindAsEnum`).
-  Attribute::AttrKind getKindAsEnum() const;
+  LLVM_ABI Attribute::AttrKind getKindAsEnum() const;
 
   /// Return the attribute's value as an integer. This requires that the
   /// attribute be an integer attribute.
-  uint64_t getValueAsInt() const;
+  LLVM_ABI uint64_t getValueAsInt() const;
 
   /// Return the attribute's value as a boolean. This requires that the
   /// attribute be a string attribute.
-  bool getValueAsBool() const;
+  LLVM_ABI bool getValueAsBool() const;
 
   /// Return the attribute's kind as a string. This requires the
   /// attribute to be a string attribute.
-  StringRef getKindAsString() const;
+  LLVM_ABI StringRef getKindAsString() const;
 
   /// Return the attribute's value as a string. This requires the
   /// attribute to be a string attribute.
-  StringRef getValueAsString() const;
+  LLVM_ABI StringRef getValueAsString() const;
 
   /// Return the attribute's value as a Type. This requires the attribute to be
   /// a type attribute.
-  Type *getValueAsType() const;
+  LLVM_ABI Type *getValueAsType() const;
 
   /// Return the attribute's value as a ConstantRange. This requires the
   /// attribute to be a ConstantRange attribute.
-  const ConstantRange &getValueAsConstantRange() const;
+  LLVM_ABI const ConstantRange &getValueAsConstantRange() const;
 
   /// Return the attribute's value as a ConstantRange array. This requires the
   /// attribute to be a ConstantRangeList attribute.
-  ArrayRef<ConstantRange> getValueAsConstantRangeList() const;
+  LLVM_ABI ArrayRef<ConstantRange> getValueAsConstantRangeList() const;
 
   /// Returns the alignment field of an attribute as a byte alignment
   /// value.
-  MaybeAlign getAlignment() const;
+  LLVM_ABI MaybeAlign getAlignment() const;
 
   /// Returns the stack alignment field of an attribute as a byte
   /// alignment value.
-  MaybeAlign getStackAlignment() const;
+  LLVM_ABI MaybeAlign getStackAlignment() const;
 
   /// Returns the number of dereferenceable bytes from the
   /// dereferenceable attribute.
-  uint64_t getDereferenceableBytes() const;
+  LLVM_ABI uint64_t getDereferenceableBytes() const;
 
   /// Returns the number of dereferenceable_or_null bytes from the
   /// dereferenceable_or_null attribute.
-  uint64_t getDereferenceableOrNullBytes() const;
+  LLVM_ABI uint64_t getDereferenceableOrNullBytes() const;
 
   /// Returns the argument numbers for the allocsize attribute.
-  std::pair<unsigned, std::optional<unsigned>> getAllocSizeArgs() const;
+  LLVM_ABI std::pair<unsigned, std::optional<unsigned>>
+  getAllocSizeArgs() const;
 
   /// Returns the minimum value for the vscale_range attribute.
-  unsigned getVScaleRangeMin() const;
+  LLVM_ABI unsigned getVScaleRangeMin() const;
 
   /// Returns the maximum value for the vscale_range attribute or std::nullopt
   /// when unknown.
-  std::optional<unsigned> getVScaleRangeMax() const;
+  LLVM_ABI std::optional<unsigned> getVScaleRangeMax() const;
 
   // Returns the unwind table kind.
-  UWTableKind getUWTableKind() const;
+  LLVM_ABI UWTableKind getUWTableKind() const;
 
   // Returns the allocator function kind.
-  AllocFnKind getAllocKind() const;
+  LLVM_ABI AllocFnKind getAllocKind() const;
 
   /// Returns memory effects.
-  MemoryEffects getMemoryEffects() const;
+  LLVM_ABI MemoryEffects getMemoryEffects() const;
 
   /// Returns information from captures attribute.
-  CaptureInfo getCaptureInfo() const;
+  LLVM_ABI CaptureInfo getCaptureInfo() const;
 
   /// Return the FPClassTest for nofpclass
-  FPClassTest getNoFPClass() const;
+  LLVM_ABI FPClassTest getNoFPClass() const;
 
   /// Return if global variable is instrumented by AddrSanitizer.
   bool isSanitizedPaddedGlobal() const;
 
   /// Returns the value of the range attribute.
-  const ConstantRange &getRange() const;
+  LLVM_ABI const ConstantRange &getRange() const;
 
   /// Returns the value of the initializes attribute.
-  ArrayRef<ConstantRange> getInitializes() const;
+  LLVM_ABI ArrayRef<ConstantRange> getInitializes() const;
 
   /// The Attribute is converted to a string of equivalent mnemonic. This
   /// is, presumably, for writing out the mnemonics for the assembly writer.
-  std::string getAsString(bool InAttrGrp = false) const;
+  LLVM_ABI std::string getAsString(bool InAttrGrp = false) const;
 
   /// Return true if this attribute belongs to the LLVMContext.
-  bool hasParentContext(LLVMContext &C) const;
+  LLVM_ABI bool hasParentContext(LLVMContext &C) const;
 
   /// Equality and non-equality operators.
   bool operator==(Attribute A) const { return pImpl == A.pImpl; }
   bool operator!=(Attribute A) const { return pImpl != A.pImpl; }
 
   /// Used to sort attribute by kind.
-  int cmpKind(Attribute A) const;
+  LLVM_ABI int cmpKind(Attribute A) const;
 
   /// Less-than operator. Useful for sorting the attributes list.
-  bool operator<(Attribute A) const;
+  LLVM_ABI bool operator<(Attribute A) const;
 
-  void Profile(FoldingSetNodeID &ID) const;
+  LLVM_ABI void Profile(FoldingSetNodeID &ID) const;
 
   /// Return a raw pointer that uniquely identifies this attribute.
   void *getRawPointer() const {
@@ -367,94 +381,94 @@ class AttributeSet {
   AttributeSet(const AttributeSet &) = default;
   ~AttributeSet() = default;
 
-  static AttributeSet get(LLVMContext &C, const AttrBuilder &B);
-  static AttributeSet get(LLVMContext &C, ArrayRef<Attribute> Attrs);
+  LLVM_ABI static AttributeSet get(LLVMContext &C, const AttrBuilder &B);
+  LLVM_ABI static AttributeSet get(LLVMContext &C, ArrayRef<Attribute> Attrs);
 
   bool operator==(const AttributeSet &O) const { return SetNode == O.SetNode; }
   bool operator!=(const AttributeSet &O) const { return !(*this == O); }
 
   /// Add an argument attribute. Returns a new set because attribute sets are
   /// immutable.
-  [[nodiscard]] AttributeSet addAttribute(LLVMContext &C,
-                                          Attribute::AttrKind Kind) const;
+  [[nodiscard]] LLVM_ABI AttributeSet
+  addAttribute(LLVMContext &C, Attribute::AttrKind Kind) const;
 
   /// Add a target-dependent attribute. Returns a new set because attribute sets
   /// are immutable.
-  [[nodiscard]] AttributeSet addAttribute(LLVMContext &C, StringRef Kind,
-                                          StringRef Value = StringRef()) const;
+  [[nodiscard]] LLVM_ABI AttributeSet addAttribute(
+      LLVMContext &C, StringRef Kind, StringRef Value = StringRef()) const;
 
   /// Add attributes to the attribute set. Returns a new set because attribute
   /// sets are immutable.
-  [[nodiscard]] AttributeSet addAttributes(LLVMContext &C,
-                                           AttributeSet AS) const;
+  [[nodiscard]] LLVM_ABI AttributeSet addAttributes(LLVMContext &C,
+                                                    AttributeSet AS) const;
 
   /// Remove the specified attribute from this set. Returns a new set because
   /// attribute sets are immutable.
-  [[nodiscard]] AttributeSet removeAttribute(LLVMContext &C,
-                                             Attribute::AttrKind Kind) const;
+  [[nodiscard]] LLVM_ABI AttributeSet
+  removeAttribute(LLVMContext &C, Attribute::AttrKind Kind) const;
 
   /// Remove the specified attribute from this set. Returns a new set because
   /// attribute sets are immutable.
-  [[nodiscard]] AttributeSet removeAttribute(LLVMContext &C,
-                                             StringRef Kind) const;
+  [[nodiscard]] LLVM_ABI AttributeSet removeAttribute(LLVMContext &C,
+                                                      StringRef Kind) const;
 
   /// Remove the specified attributes from this set. Returns a new set because
   /// attribute sets are immutable.
-  [[nodiscard]] AttributeSet
+  [[nodiscard]] LLVM_ABI AttributeSet
   removeAttributes(LLVMContext &C, const AttributeMask &AttrsToRemove) const;
 
   /// Try to intersect this AttributeSet with Other. Returns std::nullopt if
   /// the two lists are inherently incompatible (imply different behavior, not
   /// just analysis).
-  [[nodiscard]] std::optional<AttributeSet>
+  [[nodiscard]] LLVM_ABI std::optional<AttributeSet>
   intersectWith(LLVMContext &C, AttributeSet Other) const;
 
   /// Return the number of attributes in this set.
-  unsigned getNumAttributes() const;
+  LLVM_ABI unsigned getNumAttributes() const;
 
   /// Return true if attributes exists in this set.
   bool hasAttributes() const { return SetNode != nullptr; }
 
   /// Return true if the attribute exists in this set.
-  bool hasAttribute(Attribute::AttrKind Kind) const;
+  LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const;
 
   /// Return true if the attribute exists in this set.
-  bool hasAttribute(StringRef Kind) const;
+  LLVM_ABI bool hasAttribute(StringRef Kind) const;
 
   /// Return the attribute object.
-  Attribute getAttribute(Attribute::AttrKind Kind) const;
+  LLVM_ABI Attribute getAttribute(Attribute::AttrKind Kind) const;
 
   /// Return the target-dependent attribute object.
-  Attribute getAttribute(StringRef Kind) const;
-
-  MaybeAlign getAlignment() const;
-  MaybeAlign getStackAlignment() const;
-  uint64_t getDereferenceableBytes() const;
-  uint64_t getDereferenceableOrNullBytes() const;
-  Type *getByValType() const;
-  Type *getStructRetType() const;
-  Type *getByRefType() const;
-  Type *getPreallocatedType() const;
-  Type *getInAllocaType() const;
-  Type *getElementType() const;
-  std::optional<std::pair<unsigned, std::optional<unsigned>>> getAllocSizeArgs()
-      const;
-  unsigned getVScaleRangeMin() const;
-  std::optional<unsigned> getVScaleRangeMax() const;
-  UWTableKind getUWTableKind() const;
-  AllocFnKind getAllocKind() const;
-  MemoryEffects getMemoryEffects() const;
-  CaptureInfo getCaptureInfo() const;
-  FPClassTest getNoFPClass() const;
-  std::string getAsString(bool InAttrGrp = false) const;
+  LLVM_ABI Attribute getAttribute(StringRef Kind) const;
+
+  LLVM_ABI MaybeAlign getAlignment() const;
+  LLVM_ABI MaybeAlign getStackAlignment() const;
+  LLVM_ABI uint64_t getDereferenceableBytes() const;
+  LLVM_ABI uint64_t getDereferenceableOrNullBytes() const;
+  LLVM_ABI Type *getByValType() const;
+  LLVM_ABI Type *getStructRetType() const;
+  LLVM_ABI Type *getByRefType() const;
+  LLVM_ABI Type *getPreallocatedType() const;
+  LLVM_ABI Type *getInAllocaType() const;
+  LLVM_ABI Type *getElementType() const;
+  LLVM_ABI std::optional<std::pair<unsigned, std::optional<unsigned>>>
+  getAllocSizeArgs() const;
+  LLVM_ABI unsigned getVScaleRangeMin() const;
+  LLVM_ABI std::optional<unsigned> getVScaleRangeMax() const;
+  LLVM_ABI UWTableKind getUWTableKind() const;
+  LLVM_ABI AllocFnKind getAllocKind() const;
+  LLVM_ABI MemoryEffects getMemoryEffects() const;
+  LLVM_ABI CaptureInfo getCaptureInfo() const;
+  LLVM_ABI FPClassTest getNoFPClass() const;
+  LLVM_ABI std::string getAsString(bool InAttrGrp = false) const;
 
   /// Return true if this attribute set belongs to the LLVMContext.
-  bool hasParentContext(LLVMContext &C) const;
+  LLVM_ABI bool hasParentContext(LLVMContext &C) const;
 
   using iterator = const Attribute *;
 
-  iterator begin() const;
-  iterator end() const;
+  LLVM_ABI iterator begin() const;
+  LLVM_ABI iterator end() const;
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const;
 #endif
@@ -513,16 +527,16 @@ class AttributeList {
 
 public:
   /// Create an AttributeList with the specified parameters in it.
-  static AttributeList get(LLVMContext &C,
-                           ArrayRef<std::pair<unsigned, Attribute>> Attrs);
-  static AttributeList get(LLVMContext &C,
-                           ArrayRef<std::pair<unsigned, AttributeSet>> Attrs);
+  LLVM_ABI static AttributeList
+  get(LLVMContext &C, ArrayRef<std::pair<unsigned, Attribute>> Attrs);
+  LLVM_ABI static AttributeList
+  get(LLVMContext &C, ArrayRef<std::pair<unsigned, AttributeSet>> Attrs);
 
   /// Create an AttributeList from attribute sets for a function, its
   /// return value, and all of its arguments.
-  static AttributeList get(LLVMContext &C, AttributeSet FnAttrs,
-                           AttributeSet RetAttrs,
-                           ArrayRef<AttributeSet> ArgAttrs);
+  LLVM_ABI static AttributeList get(LLVMContext &C, AttributeSet FnAttrs,
+                                    AttributeSet RetAttrs,
+                                    ArrayRef<AttributeSet> ArgAttrs);
 
 private:
   explicit AttributeList(AttributeListImpl *LI) : pImpl(LI) {}
@@ -540,42 +554,42 @@ class AttributeList {
   //===--------------------------------------------------------------------===//
 
   /// Return an AttributeList with the specified parameters in it.
-  static AttributeList get(LLVMContext &C, ArrayRef<AttributeList> Attrs);
-  static AttributeList get(LLVMContext &C, unsigned Index,
-                           ArrayRef<Attribute::AttrKind> Kinds);
-  static AttributeList get(LLVMContext &C, unsigned Index,
-                           ArrayRef<Attribute::AttrKind> Kinds,
-                           ArrayRef<uint64_t> Values);
-  static AttributeList get(LLVMContext &C, unsigned Index,
-                           ArrayRef<StringRef> Kind);
-  static AttributeList get(LLVMContext &C, unsigned Index,
-                           AttributeSet Attrs);
-  static AttributeList get(LLVMContext &C, unsigned Index,
-                           const AttrBuilder &B);
+  LLVM_ABI static AttributeList get(LLVMContext &C,
+                                    ArrayRef<AttributeList> Attrs);
+  LLVM_ABI static AttributeList get(LLVMContext &C, unsigned Index,
+                                    ArrayRef<Attribute::AttrKind> Kinds);
+  LLVM_ABI static AttributeList get(LLVMContext &C, unsigned Index,
+                                    ArrayRef<Attribute::AttrKind> Kinds,
+                                    ArrayRef<uint64_t> Values);
+  LLVM_ABI static AttributeList get(LLVMContext &C, unsigned Index,
+                                    ArrayRef<StringRef> Kind);
+  LLVM_ABI static AttributeList get(LLVMContext &C, unsigned Index,
+                                    AttributeSet Attrs);
+  LLVM_ABI static AttributeList get(LLVMContext &C, unsigned Index,
+                                    const AttrBuilder &B);
 
   // TODO: remove non-AtIndex versions of these methods.
   /// Add an attribute to the attribute set at the given index.
   /// Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList
-  addAttributeAtIndex(LLVMContext &C, unsigned Index,
-                      Attribute::AttrKind Kind) const;
+  [[nodiscard]] LLVM_ABI AttributeList addAttributeAtIndex(
+      LLVMContext &C, unsigned Index, Attribute::AttrKind Kind) const;
 
   /// Add an attribute to the attribute set at the given index.
   /// Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList
+  [[nodiscard]] LLVM_ABI AttributeList
   addAttributeAtIndex(LLVMContext &C, unsigned Index, StringRef Kind,
                       StringRef Value = StringRef()) const;
 
   /// Add an attribute to the attribute set at the given index.
   /// Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList
-  addAttributeAtIndex(LLVMContext &C, unsigned Index, Attribute A) const;
+  [[nodiscard]] LLVM_ABI AttributeList addAttributeAtIndex(LLVMContext &C,
+                                                           unsigned Index,
+                                                           Attribute A) const;
 
   /// Add attributes to the attribute set at the given index.
   /// Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList addAttributesAtIndex(LLVMContext &C,
-                                                   unsigned Index,
-                                                   const AttrBuilder &B) const;
+  [[nodiscard]] LLVM_ABI AttributeList addAttributesAtIndex(
+      LLVMContext &C, unsigned Index, const AttrBuilder &B) const;
 
   /// Add a function attribute to the list. Returns a new list because
   /// attribute lists are immutable.
@@ -645,9 +659,8 @@ class AttributeList {
 
   /// Add an attribute to the attribute list at the given arg indices. Returns a
   /// new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList addParamAttribute(LLVMContext &C,
-                                                ArrayRef<unsigned> ArgNos,
-                                                Attribute A) const;
+  [[nodiscard]] LLVM_ABI AttributeList addParamAttribute(
+      LLVMContext &C, ArrayRef<unsigned> ArgNos, Attribute A) const;
 
   /// Add an argument attribute to the list. Returns a new list because
   /// attribute lists are immutable.
@@ -658,13 +671,12 @@ class AttributeList {
 
   /// Remove the specified attribute at the specified index from this
   /// attribute list. Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList
-  removeAttributeAtIndex(LLVMContext &C, unsigned Index,
-                         Attribute::AttrKind Kind) const;
+  [[nodiscard]] LLVM_ABI AttributeList removeAttributeAtIndex(
+      LLVMContext &C, unsigned Index, Attribute::AttrKind Kind) const;
 
   /// Remove the specified attribute at the specified index from this
   /// attribute list. Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList
+  [[nodiscard]] LLVM_ABI AttributeList
   removeAttributeAtIndex(LLVMContext &C, unsigned Index, StringRef Kind) const;
   [[nodiscard]] AttributeList removeAttribute(LLVMContext &C, unsigned Index,
                                               StringRef Kind) const {
@@ -673,14 +685,13 @@ class AttributeList {
 
   /// Remove the specified attributes at the specified index from this
   /// attribute list. Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList
-  removeAttributesAtIndex(LLVMContext &C, unsigned Index,
-                          const AttributeMask &AttrsToRemove) const;
+  [[nodiscard]] LLVM_ABI AttributeList removeAttributesAtIndex(
+      LLVMContext &C, unsigned Index, const AttributeMask &AttrsToRemove) const;
 
   /// Remove all attributes at the specified index from this
   /// attribute list. Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList removeAttributesAtIndex(LLVMContext &C,
-                                                      unsigned Index) const;
+  [[nodiscard]] LLVM_ABI AttributeList
+  removeAttributesAtIndex(LLVMContext &C, unsigned Index) const;
 
   /// Remove the specified attribute at the function index from this
   /// attribute list. Returns a new list because attribute lists are immutable.
@@ -775,37 +786,35 @@ class AttributeList {
 
   /// \brief Add the dereferenceable attribute to the attribute set at the given
   /// index. Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList addDereferenceableRetAttr(LLVMContext &C,
-                                                        uint64_t Bytes) const;
+  [[nodiscard]] LLVM_ABI AttributeList
+  addDereferenceableRetAttr(LLVMContext &C, uint64_t Bytes) const;
 
   /// \brief Add the dereferenceable attribute to the attribute set at the given
   /// arg index. Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList addDereferenceableParamAttr(LLVMContext &C,
-                                                          unsigned ArgNo,
-                                                          uint64_t Bytes) const;
+  [[nodiscard]] LLVM_ABI AttributeList addDereferenceableParamAttr(
+      LLVMContext &C, unsigned ArgNo, uint64_t Bytes) const;
 
   /// Add the dereferenceable_or_null attribute to the attribute set at
   /// the given arg index. Returns a new list because attribute lists are
   /// immutable.
-  [[nodiscard]] AttributeList
-  addDereferenceableOrNullParamAttr(LLVMContext &C, unsigned ArgNo,
-                                    uint64_t Bytes) const;
+  [[nodiscard]] LLVM_ABI AttributeList addDereferenceableOrNullParamAttr(
+      LLVMContext &C, unsigned ArgNo, uint64_t Bytes) const;
 
   /// Add the range attribute to the attribute set at the return value index.
   /// Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList addRangeRetAttr(LLVMContext &C,
-                                              const ConstantRange &CR) const;
+  [[nodiscard]] LLVM_ABI AttributeList
+  addRangeRetAttr(LLVMContext &C, const ConstantRange &CR) const;
 
   /// Add the allocsize attribute to the attribute set at the given arg index.
   /// Returns a new list because attribute lists are immutable.
-  [[nodiscard]] AttributeList
+  [[nodiscard]] LLVM_ABI AttributeList
   addAllocSizeParamAttr(LLVMContext &C, unsigned ArgNo, unsigned ElemSizeArg,
                         const std::optional<unsigned> &NumElemsArg) const;
 
   /// Try to intersect this AttributeList with Other. Returns std::nullopt if
   /// the two lists are inherently incompatible (imply different behavior, not
   /// just analysis).
-  [[nodiscard]] std::optional<AttributeList>
+  [[nodiscard]] LLVM_ABI std::optional<AttributeList>
   intersectWith(LLVMContext &C, AttributeList Other) const;
 
   //===--------------------------------------------------------------------===//
@@ -813,26 +822,27 @@ class AttributeList {
   //===--------------------------------------------------------------------===//
 
   /// The attributes for the specified index are returned.
-  AttributeSet getAttributes(unsigned Index) const;
+  LLVM_ABI AttributeSet getAttributes(unsigned Index) const;
 
   /// The attributes for the argument or parameter at the given index are
   /// returned.
-  AttributeSet getParamAttrs(unsigned ArgNo) const;
+  LLVM_ABI AttributeSet getParamAttrs(unsigned ArgNo) const;
 
   /// The attributes for the ret value are returned.
-  AttributeSet getRetAttrs() const;
+  LLVM_ABI AttributeSet getRetAttrs() const;
 
   /// The function attributes are returned.
-  AttributeSet getFnAttrs() const;
+  LLVM_ABI AttributeSet getFnAttrs() const;
 
   /// Return true if the attribute exists at the given index.
-  bool hasAttributeAtIndex(unsigned Index, Attribute::AttrKind Kind) const;
+  LLVM_ABI bool hasAttributeAtIndex(unsigned Index,
+                                    Attribute::AttrKind Kind) const;
 
   /// Return true if the attribute exists at the given index.
-  bool hasAttributeAtIndex(unsigned Index, StringRef Kind) const;
+  LLVM_ABI bool hasAttributeAtIndex(unsigned Index, StringRef Kind) const;
 
   /// Return true if attribute exists at the given index.
-  bool hasAttributesAtIndex(unsigned Index) const;
+  LLVM_ABI bool hasAttributesAtIndex(unsigned Index) const;
 
   /// Return true if the attribute exists for the given argument
   bool hasParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
@@ -863,10 +873,10 @@ class AttributeList {
   bool hasRetAttrs() const { return hasAttributesAtIndex(ReturnIndex); }
 
   /// Return true if the attribute exists for the function.
-  bool hasFnAttr(Attribute::AttrKind Kind) const;
+  LLVM_ABI bool hasFnAttr(Attribute::AttrKind Kind) const;
 
   /// Return true if the attribute exists for the function.
-  bool hasFnAttr(StringRef Kind) const;
+  LLVM_ABI bool hasFnAttr(StringRef Kind) const;
 
   /// Return true the attributes exist for the function.
   bool hasFnAttrs() const { return hasAttributesAtIndex(FunctionIndex); }
@@ -874,14 +884,15 @@ class AttributeList {
   /// Return true if the specified attribute is set for at least one
   /// parameter or for the return value. If Index is not nullptr, the index
   /// of a parameter with the specified attribute is provided.
-  bool hasAttrSomewhere(Attribute::AttrKind Kind,
-                        unsigned *Index = nullptr) const;
+  LLVM_ABI bool hasAttrSomewhere(Attribute::AttrKind Kind,
+                                 unsigned *Index = nullptr) const;
 
   /// Return the attribute object that exists at the given index.
-  Attribute getAttributeAtIndex(unsigned Index, Attribute::AttrKind Kind) const;
+  LLVM_ABI Attribute getAttributeAtIndex(unsigned Index,
+                                         Attribute::AttrKind Kind) const;
 
   /// Return the attribute object that exists at the given index.
-  Attribute getAttributeAtIndex(unsigned Index, StringRef Kind) const;
+  LLVM_ABI Attribute getAttributeAtIndex(unsigned Index, StringRef Kind) const;
 
   /// Return the attribute object that exists at the arg index.
   Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
@@ -909,75 +920,76 @@ class AttributeList {
   }
 
   /// Return the alignment of the return value.
-  MaybeAlign getRetAlignment() const;
+  LLVM_ABI MaybeAlign getRetAlignment() const;
 
   /// Return the alignment for the specified function parameter.
-  MaybeAlign getParamAlignment(unsigned ArgNo) const;
+  LLVM_ABI MaybeAlign getParamAlignment(unsigned ArgNo) const;
 
   /// Return the stack alignment for the specified function parameter.
-  MaybeAlign getParamStackAlignment(unsigned ArgNo) const;
+  LLVM_ABI MaybeAlign getParamStackAlignment(unsigned ArgNo) const;
 
   /// Return the byval type for the specified function parameter.
-  Type *getParamByValType(unsigned ArgNo) const;
+  LLVM_ABI Type *getParamByValType(unsigned ArgNo) const;
 
   /// Return the sret type for the specified function parameter.
-  Type *getParamStructRetType(unsigned ArgNo) const;
+  LLVM_ABI Type *getParamStructRetType(unsigned ArgNo) const;
 
   /// Return the byref type for the specified function parameter.
-  Type *getParamByRefType(unsigned ArgNo) const;
+  LLVM_ABI Type *getParamByRefType(unsigned ArgNo) const;
 
   /// Return the preallocated type for the specified function parameter.
-  Type *getParamPreallocatedType(unsigned ArgNo) const;
+  LLVM_ABI Type *getParamPreallocatedType(unsigned ArgNo) const;
 
   /// Return the inalloca type for the specified function parameter.
-  Type *getParamInAllocaType(unsigned ArgNo) const;
+  LLVM_ABI Type *getParamInAllocaType(unsigned ArgNo) const;
 
   /// Return the elementtype type for the specified function parameter.
-  Type *getParamElementType(unsigned ArgNo) const;
+  LLVM_ABI Type *getParamElementType(unsigned ArgNo) const;
 
   /// Get the stack alignment of the function.
-  MaybeAlign getFnStackAlignment() const;
+  LLVM_ABI MaybeAlign getFnStackAlignment() const;
 
   /// Get the stack alignment of the return value.
-  MaybeAlign getRetStackAlignment() const;
+  LLVM_ABI MaybeAlign getRetStackAlignment() const;
 
   /// Get the number of dereferenceable bytes (or zero if unknown) of the return
   /// value.
-  uint64_t getRetDereferenceableBytes() const;
+  LLVM_ABI uint64_t getRetDereferenceableBytes() const;
 
   /// Get the number of dereferenceable bytes (or zero if unknown) of an arg.
-  uint64_t getParamDereferenceableBytes(unsigned Index) const;
+  LLVM_ABI uint64_t getParamDereferenceableBytes(unsigned Index) const;
 
   /// Get the number of dereferenceable_or_null bytes (or zero if unknown) of
   /// the return value.
-  uint64_t getRetDereferenceableOrNullBytes() const;
+  LLVM_ABI uint64_t getRetDereferenceableOrNullBytes() const;
 
   /// Get the number of dereferenceable_or_null bytes (or zero if unknown) of an
   /// arg.
-  uint64_t getParamDereferenceableOrNullBytes(unsigned ArgNo) const;
+  LLVM_ABI uint64_t getParamDereferenceableOrNullBytes(unsigned ArgNo) const;
 
   /// Get range (or std::nullopt if unknown) of an arg.
-  std::optional<ConstantRange> getParamRange(unsigned ArgNo) const;
+  LLVM_ABI std::optional<ConstantRange> getParamRange(unsigned ArgNo) const;
 
   /// Get the disallowed floating-point classes of the return value.
-  FPClassTest getRetNoFPClass() const;
+  LLVM_ABI FPClassTest getRetNoFPClass() const;
 
   /// Get the disallowed floating-point classes of the argument value.
-  FPClassTest getParamNoFPClass(unsigned ArgNo) const;
+  LLVM_ABI FPClassTest getParamNoFPClass(unsigned ArgNo) const;
 
   /// Get the unwind table kind requested for the function.
-  UWTableKind getUWTableKind() const;
+  LLVM_ABI UWTableKind getUWTableKind() const;
 
-  AllocFnKind getAllocKind() const;
+  LLVM_ABI AllocFnKind getAllocKind() const;
 
   /// Returns memory effects of the function.
-  MemoryEffects getMemoryEffects() const;
+  LLVM_ABI MemoryEffects getMemoryEffects() const;
 
   /// Return the attributes at the index as a string.
-  std::string getAsString(unsigned Index, bool InAttrGrp = false) const;
+  LLVM_ABI std::string getAsString(unsigned Index,
+                                   bool InAttrGrp = false) const;
 
   /// Return true if this attribute list belongs to the LLVMContext.
-  bool hasParentContext(LLVMContext &C) const;
+  LLVM_ABI bool hasParentContext(LLVMContext &C) const;
 
   //===--------------------------------------------------------------------===//
   // AttributeList Introspection
@@ -985,10 +997,10 @@ class AttributeList {
 
   using iterator = const AttributeSet *;
 
-  iterator begin() const;
-  iterator end() const;
+  LLVM_ABI iterator begin() const;
+  LLVM_ABI iterator end() const;
 
-  unsigned getNumAttrSets() const;
+  LLVM_ABI unsigned getNumAttrSets() const;
 
   // Implementation of indexes(). Produces iterators that wrap an index. Mostly
   // to hide the awkwardness of unsigned wrapping when iterating over valid
@@ -1029,9 +1041,9 @@ class AttributeList {
   /// Return true if there are no attributes.
   bool isEmpty() const { return pImpl == nullptr; }
 
-  void print(raw_ostream &O) const;
+  LLVM_ABI void print(raw_ostream &O) const;
 
-  void dump() const;
+  LLVM_ABI void dump() const;
 };
 
 //===----------------------------------------------------------------------===//
@@ -1079,24 +1091,24 @@ class AttrBuilder {
     addAttribute(A);
   }
 
-  AttrBuilder(LLVMContext &Ctx, AttributeSet AS);
+  LLVM_ABI AttrBuilder(LLVMContext &Ctx, AttributeSet AS);
 
-  void clear();
+  LLVM_ABI void clear();
 
   /// Add an attribute to the builder.
-  AttrBuilder &addAttribute(Attribute::AttrKind Val);
+  LLVM_ABI AttrBuilder &addAttribute(Attribute::AttrKind Val);
 
   /// Add the Attribute object to the builder.
-  AttrBuilder &addAttribute(Attribute A);
+  LLVM_ABI AttrBuilder &addAttribute(Attribute A);
 
   /// Add the target-dependent attribute to the builder.
-  AttrBuilder &addAttribute(StringRef A, StringRef V = StringRef());
+  LLVM_ABI AttrBuilder &addAttribute(StringRef A, StringRef V = StringRef());
 
   /// Remove an attribute from the builder.
-  AttrBuilder &removeAttribute(Attribute::AttrKind Val);
+  LLVM_ABI AttrBuilder &removeAttribute(Attribute::AttrKind Val);
 
   /// Remove the target-dependent attribute from the builder.
-  AttrBuilder &removeAttribute(StringRef A);
+  LLVM_ABI AttrBuilder &removeAttribute(StringRef A);
 
   /// Remove the target-dependent attribute from the builder.
   AttrBuilder &removeAttribute(Attribute A) {
@@ -1108,40 +1120,41 @@ class AttrBuilder {
 
   /// Add the attributes from the builder. Attributes in the passed builder
   /// overwrite attributes in this builder if they have the same key.
-  AttrBuilder &merge(const AttrBuilder &B);
+  LLVM_ABI AttrBuilder &merge(const AttrBuilder &B);
 
   /// Remove the attributes from the builder.
-  AttrBuilder &remove(const AttributeMask &AM);
+  LLVM_ABI AttrBuilder &remove(const AttributeMask &AM);
 
   /// Return true if the builder has any attribute that's in the
   /// specified builder.
-  bool overlaps(const AttributeMask &AM) const;
+  LLVM_ABI bool overlaps(const AttributeMask &AM) const;
 
   /// Return true if the builder has the specified attribute.
-  bool contains(Attribute::AttrKind A) const;
+  LLVM_ABI bool contains(Attribute::AttrKind A) const;
 
   /// Return true if the builder has the specified target-dependent
   /// attribute.
-  bool contains(StringRef A) const;
+  LLVM_ABI bool contains(StringRef A) const;
 
   /// Return true if the builder has IR-level attributes.
   bool hasAttributes() const { return !Attrs.empty(); }
 
   /// Return Attribute with the given Kind. The returned attribute will be
   /// invalid if the Kind is not present in the builder.
-  Attribute getAttribute(Attribute::AttrKind Kind) const;
+  LLVM_ABI Attribute getAttribute(Attribute::AttrKind Kind) const;
 
   /// Return Attribute with the given Kind. The returned attribute will be
   /// invalid if the Kind is not present in the builder.
-  Attribute getAttribute(StringRef Kind) const;
+  LLVM_ABI Attribute getAttribute(StringRef Kind) const;
 
   /// Retrieve the range if the attribute exists (std::nullopt is returned
   /// otherwise).
-  std::optional<ConstantRange> getRange() const;
+  LLVM_ABI std::optional<ConstantRange> getRange() const;
 
   /// Return raw (possibly packed/encoded) value of integer attribute or
   /// std::nullopt if not set.
-  std::optional<uint64_t> getRawIntAttr(Attribute::AttrKind Kind) const;
+  LLVM_ABI std::optional<uint64_t>
+  getRawIntAttr(Attribute::AttrKind Kind) const;
 
   /// Retrieve the alignment attribute, if it exists.
   MaybeAlign getAlignment() const {
@@ -1173,7 +1186,7 @@ class AttrBuilder {
   }
 
   /// Retrieve type for the given type attribute.
-  Type *getTypeAttr(Attribute::AttrKind Kind) const;
+  LLVM_ABI Type *getTypeAttr(Attribute::AttrKind Kind) const;
 
   /// Retrieve the byval type.
   Type *getByValType() const { return getTypeAttr(Attribute::ByVal); }
@@ -1194,15 +1207,15 @@ class AttrBuilder {
 
   /// Retrieve the allocsize args, or std::nullopt if the attribute does not
   /// exist.
-  std::optional<std::pair<unsigned, std::optional<unsigned>>> getAllocSizeArgs()
-      const;
+  LLVM_ABI std::optional<std::pair<unsigned, std::optional<unsigned>>>
+  getAllocSizeArgs() const;
 
   /// Add integer attribute with raw value (packed/encoded if necessary).
-  AttrBuilder &addRawIntAttr(Attribute::AttrKind Kind, uint64_t Value);
+  LLVM_ABI AttrBuilder &addRawIntAttr(Attribute::AttrKind Kind, uint64_t Value);
 
   /// This turns an alignment into the form used internally in Attribute.
   /// This call has no effect if Align is not set.
-  AttrBuilder &addAlignmentAttr(MaybeAlign Align);
+  LLVM_ABI AttrBuilder &addAlignmentAttr(MaybeAlign Align);
 
   /// This turns an int alignment (which must be a power of 2) into the
   /// form used internally in Attribute.
@@ -1214,7 +1227,7 @@ class AttrBuilder {
 
   /// This turns a stack alignment into the form used internally in Attribute.
   /// This call has no effect if Align is not set.
-  AttrBuilder &addStackAlignmentAttr(MaybeAlign Align);
+  LLVM_ABI AttrBuilder &addStackAlignmentAttr(MaybeAlign Align);
 
   /// This turns an int stack alignment (which must be a power of 2) into
   /// the form used internally in Attribute.
@@ -1226,84 +1239,86 @@ class AttrBuilder {
 
   /// This turns the number of dereferenceable bytes into the form used
   /// internally in Attribute.
-  AttrBuilder &addDereferenceableAttr(uint64_t Bytes);
+  LLVM_ABI AttrBuilder &addDereferenceableAttr(uint64_t Bytes);
 
   /// This turns the number of dereferenceable_or_null bytes into the
   /// form used internally in Attribute.
-  AttrBuilder &addDereferenceableOrNullAttr(uint64_t Bytes);
+  LLVM_ABI AttrBuilder &addDereferenceableOrNullAttr(uint64_t Bytes);
 
   /// This turns one (or two) ints into the form used internally in Attribute.
-  AttrBuilder &addAllocSizeAttr(unsigned ElemSizeArg,
-                                const std::optional<unsigned> &NumElemsArg);
+  LLVM_ABI AttrBuilder &
+  addAllocSizeAttr(unsigned ElemSizeArg,
+                   const std::optional<unsigned> &NumElemsArg);
 
   /// This turns two ints into the form used internally in Attribute.
-  AttrBuilder &addVScaleRangeAttr(unsigned MinValue,
-                                  std::optional<unsigned> MaxValue);
+  LLVM_ABI AttrBuilder &addVScaleRangeAttr(unsigned MinValue,
+                                           std::optional<unsigned> MaxValue);
 
   /// Add a type attribute with the given type.
-  AttrBuilder &addTypeAttr(Attribute::AttrKind Kind, Type *Ty);
+  LLVM_ABI AttrBuilder &addTypeAttr(Attribute::AttrKind Kind, Type *Ty);
 
   /// This turns a byval type into the form used internally in Attribute.
-  AttrBuilder &addByValAttr(Type *Ty);
+  LLVM_ABI AttrBuilder &addByValAttr(Type *Ty);
 
   /// This turns a sret type into the form used internally in Attribute.
-  AttrBuilder &addStructRetAttr(Type *Ty);
+  LLVM_ABI AttrBuilder &addStructRetAttr(Type *Ty);
 
   /// This turns a byref type into the form used internally in Attribute.
-  AttrBuilder &addByRefAttr(Type *Ty);
+  LLVM_ABI AttrBuilder &addByRefAttr(Type *Ty);
 
   /// This turns a preallocated type into the form used internally in Attribute.
-  AttrBuilder &addPreallocatedAttr(Type *Ty);
+  LLVM_ABI AttrBuilder &addPreallocatedAttr(Type *Ty);
 
   /// This turns an inalloca type into the form used internally in Attribute.
-  AttrBuilder &addInAllocaAttr(Type *Ty);
+  LLVM_ABI AttrBuilder &addInAllocaAttr(Type *Ty);
 
   /// Add an allocsize attribute, using the representation returned by
   /// Attribute.getIntValue().
-  AttrBuilder &addAllocSizeAttrFromRawRepr(uint64_t RawAllocSizeRepr);
+  LLVM_ABI AttrBuilder &addAllocSizeAttrFromRawRepr(uint64_t RawAllocSizeRepr);
 
   /// Add a vscale_range attribute, using the representation returned by
   /// Attribute.getIntValue().
-  AttrBuilder &addVScaleRangeAttrFromRawRepr(uint64_t RawVScaleRangeRepr);
+  LLVM_ABI AttrBuilder &
+  addVScaleRangeAttrFromRawRepr(uint64_t RawVScaleRangeRepr);
 
   /// This turns the unwind table kind into the form used internally in
   /// Attribute.
-  AttrBuilder &addUWTableAttr(UWTableKind Kind);
+  LLVM_ABI AttrBuilder &addUWTableAttr(UWTableKind Kind);
 
   // This turns the allocator kind into the form used internally in Attribute.
-  AttrBuilder &addAllocKindAttr(AllocFnKind Kind);
+  LLVM_ABI AttrBuilder &addAllocKindAttr(AllocFnKind Kind);
 
   /// Add memory effect attribute.
-  AttrBuilder &addMemoryAttr(MemoryEffects ME);
+  LLVM_ABI AttrBuilder &addMemoryAttr(MemoryEffects ME);
 
   /// Add captures attribute.
-  AttrBuilder &addCapturesAttr(CaptureInfo CI);
+  LLVM_ABI AttrBuilder &addCapturesAttr(CaptureInfo CI);
 
   // Add nofpclass attribute
-  AttrBuilder &addNoFPClassAttr(FPClassTest NoFPClassMask);
+  LLVM_ABI AttrBuilder &addNoFPClassAttr(FPClassTest NoFPClassMask);
 
   /// Add a ConstantRange attribute with the given range.
-  AttrBuilder &addConstantRangeAttr(Attribute::AttrKind Kind,
-                                    const ConstantRange &CR);
+  LLVM_ABI AttrBuilder &addConstantRangeAttr(Attribute::AttrKind Kind,
+                                             const ConstantRange &CR);
 
   /// Add range attribute.
-  AttrBuilder &addRangeAttr(const ConstantRange &CR);
+  LLVM_ABI AttrBuilder &addRangeAttr(const ConstantRange &CR);
 
   /// Add a ConstantRangeList attribute with the given ranges.
-  AttrBuilder &addConstantRangeListAttr(Attribute::AttrKind Kind,
-                                        ArrayRef<ConstantRange> Val);
+  LLVM_ABI AttrBuilder &addConstantRangeListAttr(Attribute::AttrKind Kind,
+                                                 ArrayRef<ConstantRange> Val);
 
   /// Add initializes attribute.
-  AttrBuilder &addInitializesAttr(const ConstantRangeList &CRL);
+  LLVM_ABI AttrBuilder &addInitializesAttr(const ConstantRangeList &CRL);
 
   /// Add 0 or more parameter attributes which are equivalent to metadata
   /// attached to \p I. e.g. !align -> align. This assumes the argument type is
   /// the same as the original instruction and the attribute is compatible.
-  AttrBuilder &addFromEquivalentMetadata(const Instruction &I);
+  LLVM_ABI AttrBuilder &addFromEquivalentMetadata(const Instruction &I);
 
   ArrayRef<Attribute> attrs() const { return Attrs; }
 
-  bool operator==(const AttrBuilder &B) const;
+  LLVM_ABI bool operator==(const AttrBuilder &B) const;
   bool operator!=(const AttrBuilder &B) const { return !(*this == B); }
 };
 
@@ -1317,7 +1332,7 @@ enum AttributeSafetyKind : uint8_t {
 
 /// Returns true if this is a type legal for the 'nofpclass' attribute. This
 /// follows the same type rules as FPMathOperator.
-bool isNoFPClassCompatibleType(Type *Ty);
+LLVM_ABI bool isNoFPClassCompatibleType(Type *Ty);
 
 /// Which attributes cannot be applied to a type. The argument \p AS
 /// is used as a hint for the attributes whose compatibility is being
@@ -1328,20 +1343,20 @@ bool isNoFPClassCompatibleType(Type *Ty);
 /// attributes that are known to be safely droppable are contained in
 /// the mask; only attributes that might be unsafe to drop (e.g.,
 /// ABI-related attributes) are in the mask; or both.
-AttributeMask typeIncompatible(Type *Ty, AttributeSet AS,
-                               AttributeSafetyKind ASK = ASK_ALL);
+LLVM_ABI AttributeMask typeIncompatible(Type *Ty, AttributeSet AS,
+                                        AttributeSafetyKind ASK = ASK_ALL);
 
 /// Get param/return attributes which imply immediate undefined behavior if an
 /// invalid value is passed. For example, this includes noundef (where undef
 /// implies UB), but not nonnull (where null implies poison). It also does not
 /// include attributes like nocapture, which constrain the function
 /// implementation rather than the passed value.
-AttributeMask getUBImplyingAttributes();
+LLVM_ABI AttributeMask getUBImplyingAttributes();
 
 /// \returns Return true if the two functions have compatible target-independent
 /// attributes for inlining purposes.
-bool areInlineCompatible(const Function &Caller, const Function &Callee);
-
+LLVM_ABI bool areInlineCompatible(const Function &Caller,
+                                  const Function &Callee);
 
 /// Checks  if there are any incompatible function attributes between
 /// \p A and \p B.
@@ -1349,19 +1364,21 @@ bool areInlineCompatible(const Function &Caller, const Function &Callee);
 /// \param [in] A - The first function to be compared with.
 /// \param [in] B - The second function to be compared with.
 /// \returns true if the functions have compatible attributes.
-bool areOutlineCompatible(const Function &A, const Function &B);
+LLVM_ABI bool areOutlineCompatible(const Function &A, const Function &B);
 
 /// Merge caller's and callee's attributes.
-void mergeAttributesForInlining(Function &Caller, const Function &Callee);
+LLVM_ABI void mergeAttributesForInlining(Function &Caller,
+                                         const Function &Callee);
 
 /// Merges the functions attributes from \p ToMerge into function \p Base.
 ///
 /// \param [in,out] Base - The function being merged into.
 /// \param [in] ToMerge - The function to merge attributes from.
-void mergeAttributesForOutlining(Function &Base, const Function &ToMerge);
+LLVM_ABI void mergeAttributesForOutlining(Function &Base,
+                                          const Function &ToMerge);
 
 /// Update min-legal-vector-width if it is in Attribute and less than Width.
-void updateMinLegalVectorWidthAttr(Function &Fn, uint64_t Width);
+LLVM_ABI void updateMinLegalVectorWidthAttr(Function &Fn, uint64_t Width);
 
 } // end namespace AttributeFuncs
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/AutoUpgrade.h b/external/llvm-project/llvm/include/llvm/IR/AutoUpgrade.h
index 8c093568a1e0..31096e86c518 100644
--- a/external/llvm-project/llvm/include/llvm/IR/AutoUpgrade.h
+++ b/external/llvm-project/llvm/include/llvm/IR/AutoUpgrade.h
@@ -14,6 +14,7 @@
 #define LLVM_IR_AUTOUPGRADE_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <vector>
 
 namespace llvm {
@@ -36,62 +37,64 @@ namespace llvm {
   /// for upgrading, and returns true if it requires upgrading. It may return
   /// null in NewFn if the all calls to the original intrinsic function
   /// should be transformed to non-function-call instructions.
-  bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
-                                bool CanUpgradeDebugIntrinsicsToRecords = true);
+  LLVM_ABI bool
+  UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
+                           bool CanUpgradeDebugIntrinsicsToRecords = true);
 
   /// This is the complement to the above, replacing a specific call to an
   /// intrinsic function with a call to the specified new function.
-  void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn);
+  LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn);
 
   // This upgrades the comment for objc retain release markers in inline asm
   // calls
-  void UpgradeInlineAsmString(std::string *AsmStr);
+  LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr);
 
   /// This is an auto-upgrade hook for any old intrinsic function syntaxes
   /// which need to have both the function updated as well as all calls updated
   /// to the new function. This should only be run in a post-processing fashion
   /// so that it can update all calls to the old function.
-  void UpgradeCallsToIntrinsic(Function* F);
+  LLVM_ABI void UpgradeCallsToIntrinsic(Function *F);
 
   /// This checks for global variables which should be upgraded. If it requires
   /// upgrading, returns a pointer to the upgraded variable.
-  GlobalVariable *UpgradeGlobalVariable(GlobalVariable *GV);
+  LLVM_ABI GlobalVariable *UpgradeGlobalVariable(GlobalVariable *GV);
 
   /// This checks for module flags which should be upgraded. It returns true if
   /// module is modified.
-  bool UpgradeModuleFlags(Module &M);
+  LLVM_ABI bool UpgradeModuleFlags(Module &M);
 
   /// Convert legacy nvvm.annotations metadata to appropriate function
   /// attributes.
-  void UpgradeNVVMAnnotations(Module &M);
+  LLVM_ABI void UpgradeNVVMAnnotations(Module &M);
 
   /// Convert calls to ARC runtime functions to intrinsic calls and upgrade the
   /// old retain release marker to new module flag format.
-  void UpgradeARCRuntime(Module &M);
+  LLVM_ABI void UpgradeARCRuntime(Module &M);
 
-  void UpgradeSectionAttributes(Module &M);
+  LLVM_ABI void UpgradeSectionAttributes(Module &M);
 
   /// Correct any IR that is relying on old function attribute behavior.
-  void UpgradeFunctionAttributes(Function &F);
+  LLVM_ABI void UpgradeFunctionAttributes(Function &F);
 
   /// If the given TBAA tag uses the scalar TBAA format, create a new node
   /// corresponding to the upgrade to the struct-path aware TBAA format.
   /// Otherwise return the \p TBAANode itself.
-  MDNode *UpgradeTBAANode(MDNode &TBAANode);
+  LLVM_ABI MDNode *UpgradeTBAANode(MDNode &TBAANode);
 
   /// This is an auto-upgrade for bitcast between pointers with different
   /// address spaces: the instruction is replaced by a pair ptrtoint+inttoptr.
-  Instruction *UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
-                                  Instruction *&Temp);
+  LLVM_ABI Instruction *UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
+                                           Instruction *&Temp);
 
   /// This is an auto-upgrade for bitcast constant expression between pointers
   /// with different address spaces: the instruction is replaced by a pair
   /// ptrtoint+inttoptr.
-  Constant *UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy);
+  LLVM_ABI Constant *UpgradeBitCastExpr(unsigned Opc, Constant *C,
+                                        Type *DestTy);
 
   /// Check the debug info version number, if it is out-dated, drop the debug
   /// info. Return true if module is modified.
-  bool UpgradeDebugInfo(Module &M);
+  LLVM_ABI bool UpgradeDebugInfo(Module &M);
 
   /// Check whether a string looks like an old loop attachment tag.
   inline bool mayBeOldLoopAttachmentTag(StringRef Name) {
@@ -99,17 +102,18 @@ namespace llvm {
   }
 
   /// Upgrade the loop attachment metadata node.
-  MDNode *upgradeInstructionLoopAttachment(MDNode &N);
+  LLVM_ABI MDNode *upgradeInstructionLoopAttachment(MDNode &N);
 
   /// Upgrade the datalayout string by adding a section for address space
   /// pointers.
-  std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple);
+  LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple);
 
   /// Upgrade attributes that changed format or kind.
-  void UpgradeAttributes(AttrBuilder &B);
+  LLVM_ABI void UpgradeAttributes(AttrBuilder &B);
 
   /// Upgrade operand bundles (without knowing about their user instruction).
-  void UpgradeOperandBundles(std::vector<OperandBundleDef> &OperandBundles);
+  LLVM_ABI void
+  UpgradeOperandBundles(std::vector<OperandBundleDef> &OperandBundles);
 
 } // End llvm namespace
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/BasicBlock.h b/external/llvm-project/llvm/include/llvm/IR/BasicBlock.h
index 9ee0bacb5c25..c24f01fe26cc 100644
--- a/external/llvm-project/llvm/include/llvm/IR/BasicBlock.h
+++ b/external/llvm-project/llvm/include/llvm/IR/BasicBlock.h
@@ -24,6 +24,7 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/SymbolTableListTraits.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstddef>
 #include <iterator>
@@ -62,9 +63,6 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
 public:
   using InstListType = SymbolTableList<Instruction, ilist_iterator_bits<true>,
                                        ilist_parent<BasicBlock>>;
-  /// Flag recording whether or not this block stores debug-info in the form
-  /// of intrinsic instructions (false) or non-instruction records (true).
-  bool IsNewDbgInfoFormat;
 
 private:
   // Allow Function to renumber blocks.
@@ -81,25 +79,18 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
 public:
   /// Attach a DbgMarker to the given instruction. Enables the storage of any
   /// debug-info at this position in the program.
-  DbgMarker *createMarker(Instruction *I);
-  DbgMarker *createMarker(InstListType::iterator It);
+  LLVM_ABI DbgMarker *createMarker(Instruction *I);
+  LLVM_ABI DbgMarker *createMarker(InstListType::iterator It);
 
   /// Convert variable location debugging information stored in dbg.value
   /// intrinsics into DbgMarkers / DbgRecords. Deletes all dbg.values in
-  /// the process and sets IsNewDbgInfoFormat = true. Only takes effect if
-  /// the UseNewDbgInfoFormat LLVM command line option is given.
-  void convertToNewDbgValues();
+  /// the process and sets IsNewDbgInfoFormat = true.
+  LLVM_ABI void convertToNewDbgValues();
 
   /// Convert variable location debugging information stored in DbgMarkers and
   /// DbgRecords into the dbg.value intrinsic representation. Sets
   /// IsNewDbgInfoFormat = false.
-  void convertFromNewDbgValues();
-
-  /// Ensure the block is in "old" dbg.value format (\p NewFlag == false) or
-  /// in the new format (\p NewFlag == true), converting to the desired format
-  /// if necessary.
-  void setIsNewDbgInfoFormat(bool NewFlag);
-  void setNewDbgInfoFormatFlag(bool NewFlag);
+  LLVM_ABI void convertFromNewDbgValues();
 
   unsigned getNumber() const {
     assert(getParent() && "only basic blocks in functions have valid numbers");
@@ -110,48 +101,50 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// instruction of this block. These are equivalent to dbg.value intrinsics
   /// that exist at the end of a basic block with no terminator (a transient
   /// state that occurs regularly).
-  void setTrailingDbgRecords(DbgMarker *M);
+  LLVM_ABI void setTrailingDbgRecords(DbgMarker *M);
 
   /// Fetch the collection of DbgRecords that "trail" after the last instruction
   /// of this block, see \ref setTrailingDbgRecords. If there are none, returns
   /// nullptr.
-  DbgMarker *getTrailingDbgRecords();
+  LLVM_ABI DbgMarker *getTrailingDbgRecords();
 
   /// Delete any trailing DbgRecords at the end of this block, see
   /// \ref setTrailingDbgRecords.
-  void deleteTrailingDbgRecords();
+  LLVM_ABI void deleteTrailingDbgRecords();
 
-  void dumpDbgValues() const;
+  LLVM_ABI void dumpDbgValues() const;
 
   /// Return the DbgMarker for the position given by \p It, so that DbgRecords
   /// can be inserted there. This will either be nullptr if not present, a
   /// DbgMarker, or TrailingDbgRecords if It is end().
-  DbgMarker *getMarker(InstListType::iterator It);
+  LLVM_ABI DbgMarker *getMarker(InstListType::iterator It);
 
   /// Return the DbgMarker for the position that comes after \p I. \see
   /// BasicBlock::getMarker, this can be nullptr, a DbgMarker, or
   /// TrailingDbgRecords if there is no next instruction.
-  DbgMarker *getNextMarker(Instruction *I);
+  LLVM_ABI DbgMarker *getNextMarker(Instruction *I);
 
   /// Insert a DbgRecord into a block at the position given by \p I.
-  void insertDbgRecordAfter(DbgRecord *DR, Instruction *I);
+  LLVM_ABI void insertDbgRecordAfter(DbgRecord *DR, Instruction *I);
 
   /// Insert a DbgRecord into a block at the position given by \p Here.
-  void insertDbgRecordBefore(DbgRecord *DR, InstListType::iterator Here);
+  LLVM_ABI void insertDbgRecordBefore(DbgRecord *DR,
+                                      InstListType::iterator Here);
 
   /// Eject any debug-info trailing at the end of a block. DbgRecords can
   /// transiently be located "off the end" of a block if the blocks terminator
   /// is temporarily removed. Once a terminator is re-inserted this method will
   /// move such DbgRecords back to the right place (ahead of the terminator).
-  void flushTerminatorDbgRecords();
+  LLVM_ABI void flushTerminatorDbgRecords();
 
   /// In rare circumstances instructions can be speculatively removed from
   /// blocks, and then be re-inserted back into that position later. When this
   /// happens in RemoveDIs debug-info mode, some special patching-up needs to
   /// occur: inserting into the middle of a sequence of dbg.value intrinsics
   /// does not have an equivalent with DbgRecords.
-  void reinsertInstInDbgRecords(Instruction *I,
-                                std::optional<DbgRecord::self_iterator> Pos);
+  LLVM_ABI void
+  reinsertInstInDbgRecords(Instruction *I,
+                           std::optional<DbgRecord::self_iterator> Pos);
 
 private:
   void setParent(Function *parent);
@@ -161,17 +154,17 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// If the function parameter is specified, the basic block is automatically
   /// inserted at either the end of the function (if InsertBefore is null), or
   /// before the specified basic block.
-  explicit BasicBlock(LLVMContext &C, const Twine &Name = "",
-                      Function *Parent = nullptr,
-                      BasicBlock *InsertBefore = nullptr);
+  LLVM_ABI explicit BasicBlock(LLVMContext &C, const Twine &Name = "",
+                               Function *Parent = nullptr,
+                               BasicBlock *InsertBefore = nullptr);
 
 public:
   BasicBlock(const BasicBlock &) = delete;
   BasicBlock &operator=(const BasicBlock &) = delete;
-  ~BasicBlock();
+  LLVM_ABI ~BasicBlock();
 
   /// Get the context in which this basic block lives.
-  LLVMContext &getContext() const;
+  LLVM_ABI LLVMContext &getContext() const;
 
   /// Instruction iterators...
   using iterator = InstListType::iterator;
@@ -218,25 +211,25 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
 
   /// Return the enclosing method, or null if none.
   const Function *getParent() const { return Parent; }
-        Function *getParent()       { return Parent; }
+  Function *getParent() { return Parent; }
 
   /// Return the module owning the function this basic block belongs to, or
   /// nullptr if the function does not have a module.
   ///
   /// Note: this is undefined behavior if the block does not have a parent.
-  const Module *getModule() const;
+  LLVM_ABI const Module *getModule() const;
   Module *getModule() {
     return const_cast<Module *>(
-                            static_cast<const BasicBlock *>(this)->getModule());
+        static_cast<const BasicBlock *>(this)->getModule());
   }
 
   /// Get the data layout of the module this basic block belongs to.
   ///
   /// Requires the basic block to have a parent module.
-  const DataLayout &getDataLayout() const;
+  LLVM_ABI const DataLayout &getDataLayout() const;
 
-  /// Returns the terminator instruction if the block is well formed or null
-  /// if the block is not well formed.
+  /// Returns the terminator instruction if the block is well formed or
+  /// null if the block is not well formed.
   const Instruction *getTerminator() const LLVM_READONLY {
     if (InstList.empty() || !InstList.back().isTerminator())
       return nullptr;
@@ -250,28 +243,29 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Returns the call instruction calling \@llvm.experimental.deoptimize
   /// prior to the terminating return instruction of this basic block, if such
   /// a call is present.  Otherwise, returns null.
-  const CallInst *getTerminatingDeoptimizeCall() const;
+  LLVM_ABI const CallInst *getTerminatingDeoptimizeCall() const;
   CallInst *getTerminatingDeoptimizeCall() {
     return const_cast<CallInst *>(
-         static_cast<const BasicBlock *>(this)->getTerminatingDeoptimizeCall());
+        static_cast<const BasicBlock *>(this)->getTerminatingDeoptimizeCall());
   }
 
   /// Returns the call instruction calling \@llvm.experimental.deoptimize
   /// that is present either in current basic block or in block that is a unique
-  /// successor to current block, if such call is present. Otherwise, returns null.
-  const CallInst *getPostdominatingDeoptimizeCall() const;
+  /// successor to current block, if such call is present. Otherwise, returns
+  /// null.
+  LLVM_ABI const CallInst *getPostdominatingDeoptimizeCall() const;
   CallInst *getPostdominatingDeoptimizeCall() {
-    return const_cast<CallInst *>(
-         static_cast<const BasicBlock *>(this)->getPostdominatingDeoptimizeCall());
+    return const_cast<CallInst *>(static_cast<const BasicBlock *>(this)
+                                      ->getPostdominatingDeoptimizeCall());
   }
 
   /// Returns the call instruction marked 'musttail' prior to the terminating
   /// return instruction of this basic block, if such a call is present.
   /// Otherwise, returns null.
-  const CallInst *getTerminatingMustTailCall() const;
+  LLVM_ABI const CallInst *getTerminatingMustTailCall() const;
   CallInst *getTerminatingMustTailCall() {
     return const_cast<CallInst *>(
-           static_cast<const BasicBlock *>(this)->getTerminatingMustTailCall());
+        static_cast<const BasicBlock *>(this)->getTerminatingMustTailCall());
   }
 
   /// Returns a pointer to the first instruction in this block that is not a
@@ -283,11 +277,11 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   ///
   /// Deprecated in favour of getFirstNonPHIIt, which returns an iterator that
   /// preserves some debugging information.
-  LLVM_DEPRECATED("Use iterators as instruction positions", "getFirstNonPHIIt")
-  const Instruction *getFirstNonPHI() const;
-  LLVM_DEPRECATED("Use iterators as instruction positions instead",
-                  "getFirstNonPHIIt")
-  Instruction *getFirstNonPHI();
+  LLVM_ABI LLVM_DEPRECATED("Use iterators as instruction positions",
+                           "getFirstNonPHIIt") const
+      Instruction *getFirstNonPHI() const;
+  LLVM_ABI LLVM_DEPRECATED("Use iterators as instruction positions instead",
+                           "getFirstNonPHIIt") Instruction *getFirstNonPHI();
 
   /// Returns an iterator to the first instruction in this block that is not a
   /// PHINode instruction.
@@ -298,7 +292,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   ///
   /// Avoid unwrapping the iterator to an Instruction* before inserting here,
   /// as important debug-info is preserved in the iterator.
-  InstListType::const_iterator getFirstNonPHIIt() const;
+  LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const;
   InstListType::iterator getFirstNonPHIIt() {
     BasicBlock::iterator It =
         static_cast<const BasicBlock *>(this)->getFirstNonPHIIt().getNonConst();
@@ -309,7 +303,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Returns a pointer to the first instruction in this block that is not a
   /// PHINode or a debug intrinsic, or any pseudo operation if \c SkipPseudoOp
   /// is true.
-  InstListType::const_iterator
+  LLVM_ABI InstListType::const_iterator
   getFirstNonPHIOrDbg(bool SkipPseudoOp = true) const;
   InstListType::iterator getFirstNonPHIOrDbg(bool SkipPseudoOp = true) {
     return static_cast<const BasicBlock *>(this)
@@ -320,7 +314,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Returns a pointer to the first instruction in this block that is not a
   /// PHINode, a debug intrinsic, or a lifetime intrinsic, or any pseudo
   /// operation if \c SkipPseudoOp is true.
-  InstListType::const_iterator
+  LLVM_ABI InstListType::const_iterator
   getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp = true) const;
   InstListType::iterator
   getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp = true) {
@@ -333,15 +327,16 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// suitable for inserting a non-PHI instruction.
   ///
   /// In particular, it skips all PHIs and LandingPad instructions.
-  const_iterator getFirstInsertionPt() const;
+  LLVM_ABI const_iterator getFirstInsertionPt() const;
   iterator getFirstInsertionPt() {
     return static_cast<const BasicBlock *>(this)
-                                          ->getFirstInsertionPt().getNonConst();
+        ->getFirstInsertionPt()
+        .getNonConst();
   }
 
   /// Returns an iterator to the first instruction in this block that is
   /// not a PHINode, a debug intrinsic, a static alloca or any pseudo operation.
-  const_iterator getFirstNonPHIOrDbgOrAlloca() const;
+  LLVM_ABI const_iterator getFirstNonPHIOrDbgOrAlloca() const;
   iterator getFirstNonPHIOrDbgOrAlloca() {
     return static_cast<const BasicBlock *>(this)
         ->getFirstNonPHIOrDbgOrAlloca()
@@ -351,15 +346,16 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Returns the first potential AsynchEH faulty instruction
   /// currently it checks for loads/stores (which may dereference a null
   /// pointer) and calls/invokes (which may propagate exceptions)
-  const Instruction* getFirstMayFaultInst() const;
-  Instruction* getFirstMayFaultInst() {
-      return const_cast<Instruction*>(
-          static_cast<const BasicBlock*>(this)->getFirstMayFaultInst());
+  LLVM_ABI const Instruction *getFirstMayFaultInst() const;
+  Instruction *getFirstMayFaultInst() {
+    return const_cast<Instruction *>(
+        static_cast<const BasicBlock *>(this)->getFirstMayFaultInst());
   }
 
   /// Return a const iterator range over the instructions in the block, skipping
   /// any debug instructions. Skip any pseudo operations as well if \c
   /// SkipPseudoOp is true.
+  LLVM_ABI
   iterator_range<filter_iterator<BasicBlock::const_iterator,
                                  std::function<bool(const Instruction &)>>>
   instructionsWithoutDebug(bool SkipPseudoOp = true) const;
@@ -367,33 +363,34 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Return an iterator range over the instructions in the block, skipping any
   /// debug instructions. Skip and any pseudo operations as well if \c
   /// SkipPseudoOp is true.
-  iterator_range<
+  LLVM_ABI iterator_range<
       filter_iterator<BasicBlock::iterator, std::function<bool(Instruction &)>>>
   instructionsWithoutDebug(bool SkipPseudoOp = true);
 
   /// Return the size of the basic block ignoring debug instructions
+  LLVM_ABI
   filter_iterator<BasicBlock::const_iterator,
                   std::function<bool(const Instruction &)>>::difference_type
   sizeWithoutDebug() const;
 
   /// Unlink 'this' from the containing function, but do not delete it.
-  void removeFromParent();
+  LLVM_ABI void removeFromParent();
 
   /// Unlink 'this' from the containing function and delete it.
   ///
   // \returns an iterator pointing to the element after the erased one.
-  SymbolTableList<BasicBlock>::iterator eraseFromParent();
+  LLVM_ABI SymbolTableList<BasicBlock>::iterator eraseFromParent();
 
   /// Unlink this basic block from its current function and insert it into
   /// the function that \p MovePos lives in, right before \p MovePos.
   inline void moveBefore(BasicBlock *MovePos) {
     moveBefore(MovePos->getIterator());
   }
-  void moveBefore(SymbolTableList<BasicBlock>::iterator MovePos);
+  LLVM_ABI void moveBefore(SymbolTableList<BasicBlock>::iterator MovePos);
 
   /// Unlink this basic block from its current function and insert it
   /// right after \p MovePos in the function \p MovePos lives in.
-  void moveAfter(BasicBlock *MovePos);
+  LLVM_ABI void moveAfter(BasicBlock *MovePos);
 
   /// Insert unlinked basic block into a function.
   ///
@@ -401,14 +398,15 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// provided, inserts before that basic block, otherwise inserts at the end.
   ///
   /// \pre \a getParent() is \c nullptr.
-  void insertInto(Function *Parent, BasicBlock *InsertBefore = nullptr);
+  LLVM_ABI void insertInto(Function *Parent,
+                           BasicBlock *InsertBefore = nullptr);
 
   /// Return the predecessor of this block if it has a single predecessor
   /// block. Otherwise return a null pointer.
-  const BasicBlock *getSinglePredecessor() const;
+  LLVM_ABI const BasicBlock *getSinglePredecessor() const;
   BasicBlock *getSinglePredecessor() {
     return const_cast<BasicBlock *>(
-                 static_cast<const BasicBlock *>(this)->getSinglePredecessor());
+        static_cast<const BasicBlock *>(this)->getSinglePredecessor());
   }
 
   /// Return the predecessor of this block if it has a unique predecessor
@@ -417,43 +415,43 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Note that unique predecessor doesn't mean single edge, there can be
   /// multiple edges from the unique predecessor to this block (for example a
   /// switch statement with multiple cases having the same destination).
-  const BasicBlock *getUniquePredecessor() const;
+  LLVM_ABI const BasicBlock *getUniquePredecessor() const;
   BasicBlock *getUniquePredecessor() {
     return const_cast<BasicBlock *>(
-                 static_cast<const BasicBlock *>(this)->getUniquePredecessor());
+        static_cast<const BasicBlock *>(this)->getUniquePredecessor());
   }
 
   /// Return true if this block has exactly N predecessors.
-  bool hasNPredecessors(unsigned N) const;
+  LLVM_ABI bool hasNPredecessors(unsigned N) const;
 
   /// Return true if this block has N predecessors or more.
-  bool hasNPredecessorsOrMore(unsigned N) const;
+  LLVM_ABI bool hasNPredecessorsOrMore(unsigned N) const;
 
   /// Return the successor of this block if it has a single successor.
   /// Otherwise return a null pointer.
   ///
   /// This method is analogous to getSinglePredecessor above.
-  const BasicBlock *getSingleSuccessor() const;
+  LLVM_ABI const BasicBlock *getSingleSuccessor() const;
   BasicBlock *getSingleSuccessor() {
     return const_cast<BasicBlock *>(
-                   static_cast<const BasicBlock *>(this)->getSingleSuccessor());
+        static_cast<const BasicBlock *>(this)->getSingleSuccessor());
   }
 
   /// Return the successor of this block if it has a unique successor.
   /// Otherwise return a null pointer.
   ///
   /// This method is analogous to getUniquePredecessor above.
-  const BasicBlock *getUniqueSuccessor() const;
+  LLVM_ABI const BasicBlock *getUniqueSuccessor() const;
   BasicBlock *getUniqueSuccessor() {
     return const_cast<BasicBlock *>(
-                   static_cast<const BasicBlock *>(this)->getUniqueSuccessor());
+        static_cast<const BasicBlock *>(this)->getUniqueSuccessor());
   }
 
   /// Print the basic block to an output stream with an optional
   /// AssemblyAnnotationWriter.
-  void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW = nullptr,
-             bool ShouldPreserveUseListOrder = false,
-             bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW = nullptr,
+                      bool ShouldPreserveUseListOrder = false,
+                      bool IsForDebug = false) const;
 
   //===--------------------------------------------------------------------===//
   /// Instruction iterator methods
@@ -471,20 +469,20 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
     It.setHeadBit(true);
     return It;
   }
-  inline iterator                end  ()       { return InstList.end();   }
-  inline const_iterator          end  () const { return InstList.end();   }
+  inline iterator end() { return InstList.end(); }
+  inline const_iterator end() const { return InstList.end(); }
 
-  inline reverse_iterator        rbegin()       { return InstList.rbegin(); }
-  inline const_reverse_iterator  rbegin() const { return InstList.rbegin(); }
-  inline reverse_iterator        rend  ()       { return InstList.rend();   }
-  inline const_reverse_iterator  rend  () const { return InstList.rend();   }
+  inline reverse_iterator rbegin() { return InstList.rbegin(); }
+  inline const_reverse_iterator rbegin() const { return InstList.rbegin(); }
+  inline reverse_iterator rend() { return InstList.rend(); }
+  inline const_reverse_iterator rend() const { return InstList.rend(); }
 
-  inline size_t                   size() const { return InstList.size();  }
-  inline bool                    empty() const { return InstList.empty(); }
-  inline const Instruction      &front() const { return InstList.front(); }
-  inline       Instruction      &front()       { return InstList.front(); }
-  inline const Instruction       &back() const { return InstList.back();  }
-  inline       Instruction       &back()       { return InstList.back();  }
+  inline size_t size() const { return InstList.size(); }
+  inline bool empty() const { return InstList.empty(); }
+  inline const Instruction &front() const { return InstList.front(); }
+  inline Instruction &front() { return InstList.front(); }
+  inline const Instruction &back() const { return InstList.back(); }
+  inline Instruction &back() { return InstList.back(); }
 
   /// Iterator to walk just the phi nodes in the basic block.
   template <typename PHINodeT = PHINode, typename BBIteratorT = iterator>
@@ -530,7 +528,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   iterator_range<const_phi_iterator> phis() const {
     return const_cast<BasicBlock *>(this)->phis();
   }
-  iterator_range<phi_iterator> phis();
+  LLVM_ABI iterator_range<phi_iterator> phis();
 
 private:
   /// Return the underlying instruction list container.
@@ -584,7 +582,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
 
 public:
   /// Returns a pointer to the symbol table if one exists.
-  ValueSymbolTable *getValueSymbolTable();
+  LLVM_ABI ValueSymbolTable *getValueSymbolTable();
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast.
   static bool classof(const Value *V) {
@@ -599,7 +597,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// counts go to zero.  Then everything is delete'd for real.  Note that no
   /// operations are valid on an object that has "dropped all references",
   /// except operator delete.
-  void dropAllReferences();
+  LLVM_ABI void dropAllReferences();
 
   /// Update PHI nodes in this BasicBlock before removal of predecessor \p Pred.
   /// Note that this function does not actually remove the predecessor.
@@ -607,9 +605,10 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// If \p KeepOneInputPHIs is true then don't remove PHIs that are left with
   /// zero or one incoming values, and don't simplify PHIs with all incoming
   /// values the same.
-  void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs = false);
+  LLVM_ABI void removePredecessor(BasicBlock *Pred,
+                                  bool KeepOneInputPHIs = false);
 
-  bool canSplitPredecessors() const;
+  LLVM_ABI bool canSplitPredecessors() const;
 
   /// Split the basic block into two basic blocks at the specified instruction.
   ///
@@ -629,8 +628,8 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   ///
   /// Also note that this doesn't preserve any passes. To split blocks while
   /// keeping loop information consistent, use the SplitBlock utility function.
-  BasicBlock *splitBasicBlock(iterator I, const Twine &BBName = "",
-                              bool Before = false);
+  LLVM_ABI BasicBlock *splitBasicBlock(iterator I, const Twine &BBName = "",
+                                       bool Before = false);
   BasicBlock *splitBasicBlock(Instruction *I, const Twine &BBName = "",
                               bool Before = false) {
     return splitBasicBlock(I->getIterator(), BBName, Before);
@@ -653,7 +652,8 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   /// Also note that this doesn't preserve any passes. To split blocks while
   /// keeping loop information consistent, use the SplitBlockBefore utility
   /// function.
-  BasicBlock *splitBasicBlockBefore(iterator I, const Twine &BBName = "");
+  LLVM_ABI BasicBlock *splitBasicBlockBefore(iterator I,
+                                             const Twine &BBName = "");
   BasicBlock *splitBasicBlockBefore(Instruction *I, const Twine &BBName = "") {
     return splitBasicBlockBefore(I->getIterator(), BBName);
   }
@@ -676,13 +676,14 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
 
   /// Transfer a range of instructions that belong to \p FromBB from \p
   /// FromBeginIt to \p FromEndIt, to this basic block at \p ToIt.
-  void splice(BasicBlock::iterator ToIt, BasicBlock *FromBB,
-              BasicBlock::iterator FromBeginIt,
-              BasicBlock::iterator FromEndIt);
+  LLVM_ABI void splice(BasicBlock::iterator ToIt, BasicBlock *FromBB,
+                       BasicBlock::iterator FromBeginIt,
+                       BasicBlock::iterator FromEndIt);
 
   /// Erases a range of instructions from \p FromIt to (not including) \p ToIt.
   /// \Returns \p ToIt.
-  BasicBlock::iterator erase(BasicBlock::iterator FromIt, BasicBlock::iterator ToIt);
+  LLVM_ABI BasicBlock::iterator erase(BasicBlock::iterator FromIt,
+                                      BasicBlock::iterator ToIt);
 
   /// Returns true if there are any uses of this basic block other than
   /// direct branches, switches, etc. to it.
@@ -692,15 +693,15 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
 
   /// Update all phi nodes in this basic block to refer to basic block \p New
   /// instead of basic block \p Old.
-  void replacePhiUsesWith(BasicBlock *Old, BasicBlock *New);
+  LLVM_ABI void replacePhiUsesWith(BasicBlock *Old, BasicBlock *New);
 
   /// Update all phi nodes in this basic block's successors to refer to basic
   /// block \p New instead of basic block \p Old.
-  void replaceSuccessorsPhiUsesWith(BasicBlock *Old, BasicBlock *New);
+  LLVM_ABI void replaceSuccessorsPhiUsesWith(BasicBlock *Old, BasicBlock *New);
 
   /// Update all phi nodes in this basic block's successors to refer to basic
   /// block \p New instead of to it.
-  void replaceSuccessorsPhiUsesWith(BasicBlock *New);
+  LLVM_ABI void replaceSuccessorsPhiUsesWith(BasicBlock *New);
 
   /// Return true if this basic block is an exception handling block.
   bool isEHPad() const { return getFirstNonPHIIt()->isEHPad(); }
@@ -709,23 +710,23 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   ///
   /// Being a ``landing pad'' means that the basic block is the destination of
   /// the 'unwind' edge of an invoke instruction.
-  bool isLandingPad() const;
+  LLVM_ABI bool isLandingPad() const;
 
   /// Return the landingpad instruction associated with the landing pad.
-  const LandingPadInst *getLandingPadInst() const;
+  LLVM_ABI const LandingPadInst *getLandingPadInst() const;
   LandingPadInst *getLandingPadInst() {
     return const_cast<LandingPadInst *>(
-                    static_cast<const BasicBlock *>(this)->getLandingPadInst());
+        static_cast<const BasicBlock *>(this)->getLandingPadInst());
   }
 
   /// Return true if it is legal to hoist instructions into this block.
-  bool isLegalToHoistInto() const;
+  LLVM_ABI bool isLegalToHoistInto() const;
 
   /// Return true if this is the entry block of the containing function.
   /// This method can only be used on blocks that have a parent function.
-  bool isEntryBlock() const;
+  LLVM_ABI bool isEntryBlock() const;
 
-  std::optional<uint64_t> getIrrLoopHeaderWeight() const;
+  LLVM_ABI std::optional<uint64_t> getIrrLoopHeaderWeight() const;
 
   /// Returns true if the Order field of child Instructions is valid.
   bool isInstrOrderValid() const {
@@ -739,7 +740,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   }
 
   /// Renumber instructions and mark the ordering as valid.
-  void renumberInstructions();
+  LLVM_ABI void renumberInstructions();
 
   /// Asserts that instruction order numbers are marked invalid, or that they
   /// are in ascending order. This is constant time if the ordering is invalid,
@@ -757,7 +758,7 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(BasicBlock, LLVMBasicBlockRef)
 
 /// Advance \p It while it points to a debug instruction and return the result.
 /// This assumes that \p It is not at the end of a block.
-BasicBlock::iterator skipDebugIntrinsics(BasicBlock::iterator It);
+LLVM_ABI BasicBlock::iterator skipDebugIntrinsics(BasicBlock::iterator It);
 
 #ifdef NDEBUG
 /// In release builds, this is a no-op. For !NDEBUG builds, the checks are
diff --git a/external/llvm-project/llvm/include/llvm/IR/BuiltinGCs.h b/external/llvm-project/llvm/include/llvm/IR/BuiltinGCs.h
index 16aff01dbcf3..aeb16efda96e 100644
--- a/external/llvm-project/llvm/include/llvm/IR/BuiltinGCs.h
+++ b/external/llvm-project/llvm/include/llvm/IR/BuiltinGCs.h
@@ -14,19 +14,21 @@
 #ifndef LLVM_IR_BUILTINGCS_H
 #define LLVM_IR_BUILTINGCS_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 /// FIXME: Collector instances are not useful on their own. These no longer
 ///        serve any purpose except to link in the plugins.
 
 /// Ensure the definition of the builtin GCs gets linked in
-void linkAllBuiltinGCs();
+LLVM_ABI void linkAllBuiltinGCs();
 
 /// Creates an ocaml-compatible metadata printer.
-void linkOcamlGCPrinter();
+LLVM_ABI void linkOcamlGCPrinter();
 
 /// Creates an erlang-compatible metadata printer.
-void linkErlangGCPrinter();
+LLVM_ABI void linkErlangGCPrinter();
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/CmpPredicate.h b/external/llvm-project/llvm/include/llvm/IR/CmpPredicate.h
index fca27a76e2eb..96b85426a4ad 100644
--- a/external/llvm-project/llvm/include/llvm/IR/CmpPredicate.h
+++ b/external/llvm-project/llvm/include/llvm/IR/CmpPredicate.h
@@ -14,6 +14,7 @@
 #define LLVM_IR_CMPPREDICATE_H
 
 #include "llvm/IR/InstrTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 /// An abstraction over a floating-point predicate, and a pack of an integer
@@ -54,14 +55,14 @@ class CmpPredicate {
   ///   samesign ult + slt -> slt
   ///   ult + ult -> ult
   ///   ult + slt -> std::nullopt
-  static std::optional<CmpPredicate> getMatching(CmpPredicate A,
-                                                 CmpPredicate B);
+  LLVM_ABI static std::optional<CmpPredicate> getMatching(CmpPredicate A,
+                                                          CmpPredicate B);
 
   /// Attempts to return a signed CmpInst::Predicate from the CmpPredicate. If
   /// the CmpPredicate has samesign, return ICmpInst::getSignedPredicate,
   /// dropping samesign information. Otherwise, return the predicate, dropping
   /// samesign information.
-  CmpInst::Predicate getPreferredSignedPredicate() const;
+  LLVM_ABI CmpInst::Predicate getPreferredSignedPredicate() const;
 
   /// An operator== on the underlying Predicate.
   bool operator==(CmpInst::Predicate P) const { return Pred == P; }
@@ -74,13 +75,13 @@ class CmpPredicate {
 
   /// Do a ICmpInst::getCmpPredicate() or CmpInst::getPredicate(), as
   /// appropriate.
-  static CmpPredicate get(const CmpInst *Cmp);
+  LLVM_ABI static CmpPredicate get(const CmpInst *Cmp);
 
   /// Get the swapped predicate of a CmpPredicate.
-  static CmpPredicate getSwapped(CmpPredicate P);
+  LLVM_ABI static CmpPredicate getSwapped(CmpPredicate P);
 
   /// Get the swapped predicate of a CmpInst.
-  static CmpPredicate getSwapped(const CmpInst *Cmp);
+  LLVM_ABI static CmpPredicate getSwapped(const CmpInst *Cmp);
 };
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/Comdat.h b/external/llvm-project/llvm/include/llvm/IR/Comdat.h
index 1701802e6977..3865d0cc82b0 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Comdat.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Comdat.h
@@ -18,6 +18,7 @@
 #include "llvm-c/Types.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -41,13 +42,13 @@ class Comdat {
   };
 
   Comdat(const Comdat &) = delete;
-  Comdat(Comdat &&C);
+  LLVM_ABI Comdat(Comdat &&C);
 
   SelectionKind getSelectionKind() const { return SK; }
   void setSelectionKind(SelectionKind Val) { SK = Val; }
-  StringRef getName() const;
-  void print(raw_ostream &OS, bool IsForDebug = false) const;
-  void dump() const;
+  LLVM_ABI StringRef getName() const;
+  LLVM_ABI void print(raw_ostream &OS, bool IsForDebug = false) const;
+  LLVM_ABI void dump() const;
   const SmallPtrSetImpl<GlobalObject *> &getUsers() const { return Users; }
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/IR/Constant.h b/external/llvm-project/llvm/include/llvm/IR/Constant.h
index 0aefb5ecf6b7..0be1fc172ebd 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Constant.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Constant.h
@@ -16,6 +16,7 @@
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -51,84 +52,84 @@ class Constant : public User {
   Constant(const Constant &) = delete;
 
   /// Return true if this is the value that would be returned by getNullValue.
-  bool isNullValue() const;
+  LLVM_ABI bool isNullValue() const;
 
   /// Returns true if the value is one.
-  bool isOneValue() const;
+  LLVM_ABI bool isOneValue() const;
 
   /// Return true if the value is not the one value, or,
   /// for vectors, does not contain one value elements.
-  bool isNotOneValue() const;
+  LLVM_ABI bool isNotOneValue() const;
 
   /// Return true if this is the value that would be returned by
   /// getAllOnesValue.
-  bool isAllOnesValue() const;
+  LLVM_ABI bool isAllOnesValue() const;
 
   /// Return true if the value is what would be returned by
   /// getZeroValueForNegation.
-  bool isNegativeZeroValue() const;
+  LLVM_ABI bool isNegativeZeroValue() const;
 
   /// Return true if the value is negative zero or null value.
-  bool isZeroValue() const;
+  LLVM_ABI bool isZeroValue() const;
 
   /// Return true if the value is not the smallest signed value, or,
   /// for vectors, does not contain smallest signed value elements.
-  bool isNotMinSignedValue() const;
+  LLVM_ABI bool isNotMinSignedValue() const;
 
   /// Return true if the value is the smallest signed value.
-  bool isMinSignedValue() const;
+  LLVM_ABI bool isMinSignedValue() const;
 
   /// Return true if this is a finite and non-zero floating-point scalar
   /// constant or a fixed width vector constant with all finite and non-zero
   /// elements.
-  bool isFiniteNonZeroFP() const;
+  LLVM_ABI bool isFiniteNonZeroFP() const;
 
   /// Return true if this is a normal (as opposed to denormal, infinity, nan,
   /// or zero) floating-point scalar constant or a vector constant with all
   /// normal elements. See APFloat::isNormal.
-  bool isNormalFP() const;
+  LLVM_ABI bool isNormalFP() const;
 
   /// Return true if this scalar has an exact multiplicative inverse or this
   /// vector has an exact multiplicative inverse for each element in the vector.
-  bool hasExactInverseFP() const;
+  LLVM_ABI bool hasExactInverseFP() const;
 
   /// Return true if this is a floating-point NaN constant or a vector
   /// floating-point constant with all NaN elements.
-  bool isNaN() const;
+  LLVM_ABI bool isNaN() const;
 
   /// Return true if this constant and a constant 'Y' are element-wise equal.
   /// This is identical to just comparing the pointers, with the exception that
   /// for vectors, if only one of the constants has an `undef` element in some
   /// lane, the constants still match.
-  bool isElementWiseEqual(Value *Y) const;
+  LLVM_ABI bool isElementWiseEqual(Value *Y) const;
 
   /// Return true if this is a vector constant that includes any undef or
   /// poison elements. Since it is impossible to inspect a scalable vector
   /// element- wise at compile time, this function returns true only if the
   /// entire vector is undef or poison.
-  bool containsUndefOrPoisonElement() const;
+  LLVM_ABI bool containsUndefOrPoisonElement() const;
 
   /// Return true if this is a vector constant that includes any poison
   /// elements.
-  bool containsPoisonElement() const;
+  LLVM_ABI bool containsPoisonElement() const;
 
   /// Return true if this is a vector constant that includes any strictly undef
   /// (not poison) elements.
-  bool containsUndefElement() const;
+  LLVM_ABI bool containsUndefElement() const;
 
   /// Return true if this is a fixed width vector constant that includes
   /// any constant expressions.
-  bool containsConstantExpression() const;
+  LLVM_ABI bool containsConstantExpression() const;
 
   /// Return true if the value can vary between threads.
-  bool isThreadDependent() const;
+  LLVM_ABI bool isThreadDependent() const;
 
   /// Return true if the value is dependent on a dllimport variable.
-  bool isDLLImportDependent() const;
+  LLVM_ABI bool isDLLImportDependent() const;
 
   /// Return true if the constant has users other than constant expressions and
   /// other dangling things.
-  bool isConstantUsed() const;
+  LLVM_ABI bool isConstantUsed() const;
 
   /// This method classifies the entry according to whether or not it may
   /// generate a relocation entry (either static or dynamic). This must be
@@ -136,28 +137,28 @@ class Constant : public User {
   /// so.
   ///
   /// FIXME: This really should not be in IR.
-  bool needsRelocation() const;
-  bool needsDynamicRelocation() const;
+  LLVM_ABI bool needsRelocation() const;
+  LLVM_ABI bool needsDynamicRelocation() const;
 
   /// For aggregates (struct/array/vector) return the constant that corresponds
   /// to the specified element if possible, or null if not. This can return null
   /// if the element index is a ConstantExpr, if 'this' is a constant expr or
   /// if the constant does not fit into an uint64_t.
-  Constant *getAggregateElement(unsigned Elt) const;
-  Constant *getAggregateElement(Constant *Elt) const;
+  LLVM_ABI Constant *getAggregateElement(unsigned Elt) const;
+  LLVM_ABI Constant *getAggregateElement(Constant *Elt) const;
 
   /// If all elements of the vector constant have the same value, return that
   /// value. Otherwise, return nullptr. Ignore poison elements by setting
   /// AllowPoison to true.
-  Constant *getSplatValue(bool AllowPoison = false) const;
+  LLVM_ABI Constant *getSplatValue(bool AllowPoison = false) const;
 
   /// If C is a constant integer then return its value, otherwise C must be a
   /// vector of constant integers, all equal, and the common value is returned.
-  const APInt &getUniqueInteger() const;
+  LLVM_ABI const APInt &getUniqueInteger() const;
 
   /// Convert constant to an approximate constant range. For vectors, the
   /// range is the union over the element ranges. Poison elements are ignored.
-  ConstantRange toConstantRange() const;
+  LLVM_ABI ConstantRange toConstantRange() const;
 
   /// Called if some element of this constant is no longer valid.
   /// At this point only other constants may be on the use_list for this
@@ -166,7 +167,7 @@ class Constant : public User {
   /// available cached constants.  Implementations should implement
   /// destroyConstantImpl to remove constants from any pools/maps they are
   /// contained it.
-  void destroyConstant();
+  LLVM_ABI void destroyConstant();
 
   //// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
@@ -184,36 +185,36 @@ class Constant : public User {
   /// use Value::replaceAllUsesWith, which automatically dispatches to this
   /// method as needed.
   ///
-  void handleOperandChange(Value *, Value *);
+  LLVM_ABI void handleOperandChange(Value *, Value *);
 
-  static Constant *getNullValue(Type* Ty);
+  LLVM_ABI static Constant *getNullValue(Type *Ty);
 
   /// @returns the value for an integer or vector of integer constant of the
   /// given type that has all its bits set to true.
   /// Get the all ones value
-  static Constant *getAllOnesValue(Type* Ty);
+  LLVM_ABI static Constant *getAllOnesValue(Type *Ty);
 
   /// Return the value for an integer or pointer constant, or a vector thereof,
   /// with the given scalar value.
-  static Constant *getIntegerValue(Type *Ty, const APInt &V);
+  LLVM_ABI static Constant *getIntegerValue(Type *Ty, const APInt &V);
 
   /// If there are any dead constant users dangling off of this constant, remove
   /// them. This method is useful for clients that want to check to see if a
   /// global is unused, but don't want to deal with potentially dead constants
   /// hanging off of the globals.
-  void removeDeadConstantUsers() const;
+  LLVM_ABI void removeDeadConstantUsers() const;
 
   /// Return true if the constant has exactly one live use.
   ///
   /// This returns the same result as calling Value::hasOneUse after
   /// Constant::removeDeadConstantUsers, but doesn't remove dead constants.
-  bool hasOneLiveUse() const;
+  LLVM_ABI bool hasOneLiveUse() const;
 
   /// Return true if the constant has no live uses.
   ///
   /// This returns the same result as calling Value::use_empty after
   /// Constant::removeDeadConstantUsers, but doesn't remove dead constants.
-  bool hasZeroLiveUses() const;
+  LLVM_ABI bool hasZeroLiveUses() const;
 
   const Constant *stripPointerCasts() const {
     return cast<Constant>(Value::stripPointerCasts());
@@ -226,17 +227,18 @@ class Constant : public User {
 
   /// Try to replace undefined constant C or undefined elements in C with
   /// Replacement. If no changes are made, the constant C is returned.
-  static Constant *replaceUndefsWith(Constant *C, Constant *Replacement);
+  LLVM_ABI static Constant *replaceUndefsWith(Constant *C,
+                                              Constant *Replacement);
 
   /// Merges undefs of a Constant with another Constant, along with the
   /// undefs already present. Other doesn't have to be the same type as C, but
   /// both must either be scalars or vectors with the same element count. If no
   /// changes are made, the constant C is returned.
-  static Constant *mergeUndefsWith(Constant *C, Constant *Other);
+  LLVM_ABI static Constant *mergeUndefsWith(Constant *C, Constant *Other);
 
   /// Return true if a constant is ConstantData or a ConstantAggregate or
   /// ConstantExpr that contain only ConstantData.
-  bool isManifestConstant() const;
+  LLVM_ABI bool isManifestConstant() const;
 
 private:
   enum PossibleRelocationsTy {
diff --git a/external/llvm-project/llvm/include/llvm/IR/ConstantFPRange.h b/external/llvm-project/llvm/include/llvm/IR/ConstantFPRange.h
index e24067129647..930c6f98c033 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ConstantFPRange.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ConstantFPRange.h
@@ -25,6 +25,7 @@
 
 #include "llvm/ADT/APFloat.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -52,17 +53,17 @@ class [[nodiscard]] ConstantFPRange {
   void makeFull();
 
   /// Initialize a full or empty set for the specified semantics.
-  explicit ConstantFPRange(const fltSemantics &Sem, bool IsFullSet);
+  LLVM_ABI explicit ConstantFPRange(const fltSemantics &Sem, bool IsFullSet);
 
 public:
   /// Initialize a range to hold the single specified value.
-  explicit ConstantFPRange(const APFloat &Value);
+  LLVM_ABI explicit ConstantFPRange(const APFloat &Value);
 
   /// Initialize a range of values explicitly.
   /// Note: If \p LowerVal is greater than \p UpperVal, please use the canonical
   /// form [Inf, -Inf].
-  ConstantFPRange(APFloat LowerVal, APFloat UpperVal, bool MayBeQNaN,
-                  bool MayBeSNaN);
+  LLVM_ABI ConstantFPRange(APFloat LowerVal, APFloat UpperVal, bool MayBeQNaN,
+                           bool MayBeSNaN);
 
   /// Create empty constant range with the given semantics.
   static ConstantFPRange getEmpty(const fltSemantics &Sem) {
@@ -75,10 +76,10 @@ class [[nodiscard]] ConstantFPRange {
   }
 
   /// Helper for (-inf, inf) to represent all finite values.
-  static ConstantFPRange getFinite(const fltSemantics &Sem);
+  LLVM_ABI static ConstantFPRange getFinite(const fltSemantics &Sem);
 
   /// Helper for [-inf, inf] to represent all non-NaN values.
-  static ConstantFPRange getNonNaN(const fltSemantics &Sem);
+  LLVM_ABI static ConstantFPRange getNonNaN(const fltSemantics &Sem);
 
   /// Create a range which doesn't contain NaNs.
   static ConstantFPRange getNonNaN(APFloat LowerVal, APFloat UpperVal) {
@@ -93,8 +94,8 @@ class [[nodiscard]] ConstantFPRange {
   }
 
   /// Create a range which only contains NaNs.
-  static ConstantFPRange getNaNOnly(const fltSemantics &Sem, bool MayBeQNaN,
-                                    bool MayBeSNaN);
+  LLVM_ABI static ConstantFPRange getNaNOnly(const fltSemantics &Sem,
+                                             bool MayBeQNaN, bool MayBeSNaN);
 
   /// Produce the smallest range such that all values that may satisfy the given
   /// predicate with any value contained within Other is contained in the
@@ -104,8 +105,8 @@ class [[nodiscard]] ConstantFPRange {
   /// a proper superset of the above.
   ///
   /// Example: Pred = ole and Other = float [2, 5] returns Result = [-inf, 5]
-  static ConstantFPRange makeAllowedFCmpRegion(FCmpInst::Predicate Pred,
-                                               const ConstantFPRange &Other);
+  LLVM_ABI static ConstantFPRange
+  makeAllowedFCmpRegion(FCmpInst::Predicate Pred, const ConstantFPRange &Other);
 
   /// Produce the largest range such that all values in the returned range
   /// satisfy the given predicate with all values contained within Other.
@@ -115,8 +116,9 @@ class [[nodiscard]] ConstantFPRange {
   /// will be a proper subset of the above.
   ///
   /// Example: Pred = ole and Other = float [2, 5] returns [-inf, 2]
-  static ConstantFPRange makeSatisfyingFCmpRegion(FCmpInst::Predicate Pred,
-                                                  const ConstantFPRange &Other);
+  LLVM_ABI static ConstantFPRange
+  makeSatisfyingFCmpRegion(FCmpInst::Predicate Pred,
+                           const ConstantFPRange &Other);
 
   /// Produce the exact range such that all values in the returned range satisfy
   /// the given predicate with any value contained within Other. Formally, this
@@ -125,12 +127,13 @@ class [[nodiscard]] ConstantFPRange {
   /// Example: Pred = olt and Other = float 3 returns [-inf, 3)
   /// If the exact answer is not representable as a ConstantFPRange, returns
   /// std::nullopt.
-  static std::optional<ConstantFPRange>
+  LLVM_ABI static std::optional<ConstantFPRange>
   makeExactFCmpRegion(FCmpInst::Predicate Pred, const APFloat &Other);
 
   /// Does the predicate \p Pred hold between ranges this and \p Other?
   /// NOTE: false does not mean that inverse predicate holds!
-  bool fcmp(FCmpInst::Predicate Pred, const ConstantFPRange &Other) const;
+  LLVM_ABI bool fcmp(FCmpInst::Predicate Pred,
+                     const ConstantFPRange &Other) const;
 
   /// Return the lower value for this range.
   const APFloat &getLower() const { return Lower; }
@@ -141,27 +144,27 @@ class [[nodiscard]] ConstantFPRange {
   bool containsNaN() const { return MayBeQNaN || MayBeSNaN; }
   bool containsQNaN() const { return MayBeQNaN; }
   bool containsSNaN() const { return MayBeSNaN; }
-  bool isNaNOnly() const;
+  LLVM_ABI bool isNaNOnly() const;
 
   /// Get the semantics of this ConstantFPRange.
   const fltSemantics &getSemantics() const { return Lower.getSemantics(); }
 
   /// Return true if this set contains all of the elements possible
   /// for this data-type.
-  bool isFullSet() const;
+  LLVM_ABI bool isFullSet() const;
 
   /// Return true if this set contains no members.
-  bool isEmptySet() const;
+  LLVM_ABI bool isEmptySet() const;
 
   /// Return true if the specified value is in the set.
-  bool contains(const APFloat &Val) const;
+  LLVM_ABI bool contains(const APFloat &Val) const;
 
   /// Return true if the other range is a subset of this one.
-  bool contains(const ConstantFPRange &CR) const;
+  LLVM_ABI bool contains(const ConstantFPRange &CR) const;
 
   /// If this set contains a single element, return it, otherwise return null.
   /// If \p ExcludesNaN is true, return the non-NaN single element.
-  const APFloat *getSingleElement(bool ExcludesNaN = false) const;
+  LLVM_ABI const APFloat *getSingleElement(bool ExcludesNaN = false) const;
 
   /// Return true if this set contains exactly one member.
   /// If \p ExcludesNaN is true, return true if this set contains exactly one
@@ -173,30 +176,30 @@ class [[nodiscard]] ConstantFPRange {
   /// Return true if the sign bit of all values in this range is 1.
   /// Return false if the sign bit of all values in this range is 0.
   /// Otherwise, return std::nullopt.
-  std::optional<bool> getSignBit() const;
+  LLVM_ABI std::optional<bool> getSignBit() const;
 
   /// Return true if this range is equal to another range.
-  bool operator==(const ConstantFPRange &CR) const;
+  LLVM_ABI bool operator==(const ConstantFPRange &CR) const;
   /// Return true if this range is not equal to another range.
   bool operator!=(const ConstantFPRange &CR) const { return !operator==(CR); }
 
   /// Return the FPClassTest which will return true for the value.
-  FPClassTest classify() const;
+  LLVM_ABI FPClassTest classify() const;
 
   /// Print out the bounds to a stream.
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
   /// Allow printing from a debugger easily.
-  void dump() const;
+  LLVM_ABI void dump() const;
 
   /// Return the range that results from the intersection of this range with
   /// another range.
-  ConstantFPRange intersectWith(const ConstantFPRange &CR) const;
+  LLVM_ABI ConstantFPRange intersectWith(const ConstantFPRange &CR) const;
 
   /// Return the smallest range that results from the union of this range
   /// with another range.  The resultant range is guaranteed to include the
   /// elements of both sets, but may contain more.
-  ConstantFPRange unionWith(const ConstantFPRange &CR) const;
+  LLVM_ABI ConstantFPRange unionWith(const ConstantFPRange &CR) const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const ConstantFPRange &CR) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/ConstantFold.h b/external/llvm-project/llvm/include/llvm/IR/ConstantFold.h
index 42043d365b2d..f9f2b3516a4c 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ConstantFold.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ConstantFold.h
@@ -22,6 +22,7 @@
 #define LLVM_IR_CONSTANTFOLD_H
 
 #include "llvm/IR/InstrTypes.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -31,30 +32,36 @@ namespace llvm {
   class Type;
 
   // Constant fold various types of instruction...
-  Constant *ConstantFoldCastInstruction(
-    unsigned opcode,     ///< The opcode of the cast
-    Constant *V,         ///< The source constant
-    Type *DestTy   ///< The destination type
+  LLVM_ABI Constant *
+  ConstantFoldCastInstruction(unsigned opcode, ///< The opcode of the cast
+                              Constant *V,     ///< The source constant
+                              Type *DestTy     ///< The destination type
   );
-  Constant *ConstantFoldSelectInstruction(Constant *Cond,
-                                          Constant *V1, Constant *V2);
-  Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
-  Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt,
-                                                 Constant *Idx);
-  Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
-                                                 ArrayRef<int> Mask);
-  Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
-                                                ArrayRef<unsigned> Idxs);
-  Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
-                                               ArrayRef<unsigned> Idxs);
-  Constant *ConstantFoldUnaryInstruction(unsigned Opcode, Constant *V);
-  Constant *ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1,
-                                          Constant *V2);
-  Constant *ConstantFoldCompareInstruction(CmpInst::Predicate Predicate,
-                                           Constant *C1, Constant *C2);
-  Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C,
-                                      std::optional<ConstantRange> InRange,
-                                      ArrayRef<Value *> Idxs);
+  LLVM_ABI Constant *ConstantFoldSelectInstruction(Constant *Cond, Constant *V1,
+                                                   Constant *V2);
+  LLVM_ABI Constant *ConstantFoldExtractElementInstruction(Constant *Val,
+                                                           Constant *Idx);
+  LLVM_ABI Constant *ConstantFoldInsertElementInstruction(Constant *Val,
+                                                          Constant *Elt,
+                                                          Constant *Idx);
+  LLVM_ABI Constant *ConstantFoldShuffleVectorInstruction(Constant *V1,
+                                                          Constant *V2,
+                                                          ArrayRef<int> Mask);
+  LLVM_ABI Constant *
+  ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef<unsigned> Idxs);
+  LLVM_ABI Constant *
+  ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
+                                     ArrayRef<unsigned> Idxs);
+  LLVM_ABI Constant *ConstantFoldUnaryInstruction(unsigned Opcode, Constant *V);
+  LLVM_ABI Constant *ConstantFoldBinaryInstruction(unsigned Opcode,
+                                                   Constant *V1, Constant *V2);
+  LLVM_ABI Constant *
+  ConstantFoldCompareInstruction(CmpInst::Predicate Predicate, Constant *C1,
+                                 Constant *C2);
+  LLVM_ABI Constant *
+  ConstantFoldGetElementPtr(Type *Ty, Constant *C,
+                            std::optional<ConstantRange> InRange,
+                            ArrayRef<Value *> Idxs);
 } // End llvm namespace
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/IR/ConstantFolder.h b/external/llvm-project/llvm/include/llvm/IR/ConstantFolder.h
index a1d9a8faf69e..26b7242abc4d 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ConstantFolder.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ConstantFolder.h
@@ -23,11 +23,12 @@
 #include "llvm/IR/IRBuilderFolder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// ConstantFolder - Create constants with minimum, target independent, folding.
-class ConstantFolder final : public IRBuilderFolder {
+class LLVM_ABI ConstantFolder final : public IRBuilderFolder {
   LLVM_DECLARE_VIRTUAL_ANCHOR_FUNCTION();
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/IR/ConstantRange.h b/external/llvm-project/llvm/include/llvm/IR/ConstantRange.h
index 3561513212ce..9a6a9db65688 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ConstantRange.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ConstantRange.h
@@ -59,15 +59,15 @@ class [[nodiscard]] ConstantRange {
 
 public:
   /// Initialize a full or empty set for the specified bit width.
-  explicit ConstantRange(uint32_t BitWidth, bool isFullSet);
+  LLVM_ABI explicit ConstantRange(uint32_t BitWidth, bool isFullSet);
 
   /// Initialize a range to hold the single specified value.
-  ConstantRange(APInt Value);
+  LLVM_ABI ConstantRange(APInt Value);
 
   /// Initialize a range of values explicitly. This will assert out if
   /// Lower==Upper and Lower != Min or Max value for its type. It will also
   /// assert out if the two APInt's are not the same bit width.
-  ConstantRange(APInt Lower, APInt Upper);
+  LLVM_ABI ConstantRange(APInt Lower, APInt Upper);
 
   /// Create empty constant range with the given bit width.
   static ConstantRange getEmpty(uint32_t BitWidth) {
@@ -90,11 +90,12 @@ class [[nodiscard]] ConstantRange {
   /// Initialize a range based on a known bits constraint. The IsSigned flag
   /// indicates whether the constant range should not wrap in the signed or
   /// unsigned domain.
-  static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned);
+  LLVM_ABI static ConstantRange fromKnownBits(const KnownBits &Known,
+                                              bool IsSigned);
 
   /// Split the ConstantRange into positive and negative components, ignoring
   /// zero values.
-  std::pair<ConstantRange, ConstantRange> splitPosNeg() const;
+  LLVM_ABI std::pair<ConstantRange, ConstantRange> splitPosNeg() const;
 
   /// Produce the smallest range such that all values that may satisfy the given
   /// predicate with any value contained within Other is contained in the
@@ -104,8 +105,8 @@ class [[nodiscard]] ConstantRange {
   /// proper superset of the above.
   ///
   /// Example: Pred = ult and Other = i8 [2, 5) returns Result = [0, 4)
-  static ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred,
-                                             const ConstantRange &Other);
+  LLVM_ABI static ConstantRange
+  makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other);
 
   /// Produce the largest range such that all values in the returned range
   /// satisfy the given predicate with all values contained within Other.
@@ -115,8 +116,8 @@ class [[nodiscard]] ConstantRange {
   /// will be a proper subset of the above.
   ///
   /// Example: Pred = ult and Other = i8 [2, 5) returns [0, 2)
-  static ConstantRange makeSatisfyingICmpRegion(CmpInst::Predicate Pred,
-                                                const ConstantRange &Other);
+  LLVM_ABI static ConstantRange
+  makeSatisfyingICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other);
 
   /// Produce the exact range such that all values in the returned range satisfy
   /// the given predicate with any value contained within Other. Formally, this
@@ -125,22 +126,22 @@ class [[nodiscard]] ConstantRange {
   /// { x : icmp op x y is true}'.
   ///
   /// Example: Pred = ult and Other = i8 3 returns [0, 3)
-  static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred,
-                                           const APInt &Other);
+  LLVM_ABI static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred,
+                                                    const APInt &Other);
 
   /// Does the predicate \p Pred hold between ranges this and \p Other?
   /// NOTE: false does not mean that inverse predicate holds!
-  bool icmp(CmpInst::Predicate Pred, const ConstantRange &Other) const;
+  LLVM_ABI bool icmp(CmpInst::Predicate Pred, const ConstantRange &Other) const;
 
   /// Return true iff CR1 ult CR2 is equivalent to CR1 slt CR2.
   /// Does not depend on strictness/direction of the predicate.
-  static bool
+  LLVM_ABI static bool
   areInsensitiveToSignednessOfICmpPredicate(const ConstantRange &CR1,
                                             const ConstantRange &CR2);
 
   /// Return true iff CR1 ult CR2 is equivalent to CR1 sge CR2.
   /// Does not depend on strictness/direction of the predicate.
-  static bool
+  LLVM_ABI static bool
   areInsensitiveToSignednessOfInvertedICmpPredicate(const ConstantRange &CR1,
                                                     const ConstantRange &CR2);
 
@@ -149,7 +150,7 @@ class [[nodiscard]] ConstantRange {
   /// return a predicate equivalent to \p Pred, with flipped signedness
   /// (i.e. unsigned instead of signed or vice versa), and maybe inverted,
   /// otherwise returns CmpInst::Predicate::BAD_ICMP_PREDICATE.
-  static CmpInst::Predicate
+  LLVM_ABI static CmpInst::Predicate
   getEquivalentPredWithFlippedSignedness(CmpInst::Predicate Pred,
                                          const ConstantRange &CR1,
                                          const ConstantRange &CR2);
@@ -170,38 +171,39 @@ class [[nodiscard]] ConstantRange {
   ///  MGNR(Add, [i8 -1, 6), OBO::NoSignedWrap) == [INT_MIN+1, INT_MAX-4)
   ///  MGNR(Sub, [i8 1, 2), OBO::NoSignedWrap) == [-127, 128)
   ///  MGNR(Sub, [i8 1, 2), OBO::NoUnsignedWrap) == [1, 0)
-  static ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp,
-                                                  const ConstantRange &Other,
-                                                  unsigned NoWrapKind);
+  LLVM_ABI static ConstantRange
+  makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp,
+                             const ConstantRange &Other, unsigned NoWrapKind);
 
   /// Produce the range that contains X if and only if "X BinOp Other" does
   /// not wrap.
-  static ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp,
-                                             const APInt &Other,
-                                             unsigned NoWrapKind);
+  LLVM_ABI static ConstantRange
+  makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other,
+                        unsigned NoWrapKind);
 
   /// Initialize a range containing all values X that satisfy `(X & Mask)
   /// != C`. Note that the range returned may contain values where `(X & Mask)
   /// == C` holds, making it less precise, but still conservative.
-  static ConstantRange makeMaskNotEqualRange(const APInt &Mask, const APInt &C);
+  LLVM_ABI static ConstantRange makeMaskNotEqualRange(const APInt &Mask,
+                                                      const APInt &C);
 
   /// Returns true if ConstantRange calculations are supported for intrinsic
   /// with \p IntrinsicID.
-  static bool isIntrinsicSupported(Intrinsic::ID IntrinsicID);
+  LLVM_ABI static bool isIntrinsicSupported(Intrinsic::ID IntrinsicID);
 
   /// Compute range of intrinsic result for the given operand ranges.
-  static ConstantRange intrinsic(Intrinsic::ID IntrinsicID,
-                                 ArrayRef<ConstantRange> Ops);
+  LLVM_ABI static ConstantRange intrinsic(Intrinsic::ID IntrinsicID,
+                                          ArrayRef<ConstantRange> Ops);
 
   /// Set up \p Pred and \p RHS such that
   /// ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.  Return true if
   /// successful.
-  bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const;
+  LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const;
 
   /// Set up \p Pred, \p RHS and \p Offset such that (V + Offset) Pred RHS
   /// is true iff V is in the range. Prefers using Offset == 0 if possible.
-  void
-  getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS, APInt &Offset) const;
+  LLVM_ABI void getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS,
+                                  APInt &Offset) const;
 
   /// Return the lower value for this range.
   const APInt &getLower() const { return Lower; }
@@ -214,42 +216,42 @@ class [[nodiscard]] ConstantRange {
 
   /// Return true if this set contains all of the elements possible
   /// for this data-type.
-  bool isFullSet() const;
+  LLVM_ABI bool isFullSet() const;
 
   /// Return true if this set contains no members.
-  bool isEmptySet() const;
+  LLVM_ABI bool isEmptySet() const;
 
   /// Return true if this set wraps around the unsigned domain. Special cases:
   ///  * Empty set: Not wrapped.
   ///  * Full set: Not wrapped.
   ///  * [X, 0) == [X, Max]: Not wrapped.
-  bool isWrappedSet() const;
+  LLVM_ABI bool isWrappedSet() const;
 
   /// Return true if the exclusive upper bound wraps around the unsigned
   /// domain. Special cases:
   ///  * Empty set: Not wrapped.
   ///  * Full set: Not wrapped.
   ///  * [X, 0): Wrapped.
-  bool isUpperWrapped() const;
+  LLVM_ABI bool isUpperWrapped() const;
 
   /// Return true if this set wraps around the signed domain. Special cases:
   ///  * Empty set: Not wrapped.
   ///  * Full set: Not wrapped.
   ///  * [X, SignedMin) == [X, SignedMax]: Not wrapped.
-  bool isSignWrappedSet() const;
+  LLVM_ABI bool isSignWrappedSet() const;
 
   /// Return true if the (exclusive) upper bound wraps around the signed
   /// domain. Special cases:
   ///  * Empty set: Not wrapped.
   ///  * Full set: Not wrapped.
   ///  * [X, SignedMin): Wrapped.
-  bool isUpperSignWrapped() const;
+  LLVM_ABI bool isUpperSignWrapped() const;
 
   /// Return true if the specified value is in the set.
-  bool contains(const APInt &Val) const;
+  LLVM_ABI bool contains(const APInt &Val) const;
 
   /// Return true if the other range is a subset of this one.
-  bool contains(const ConstantRange &CR) const;
+  LLVM_ABI bool contains(const ConstantRange &CR) const;
 
   /// If this set contains a single element, return it, otherwise return null.
   const APInt *getSingleElement() const {
@@ -270,31 +272,31 @@ class [[nodiscard]] ConstantRange {
   bool isSingleElement() const { return getSingleElement() != nullptr; }
 
   /// Compare set size of this range with the range CR.
-  bool isSizeStrictlySmallerThan(const ConstantRange &CR) const;
+  LLVM_ABI bool isSizeStrictlySmallerThan(const ConstantRange &CR) const;
 
   /// Compare set size of this range with Value.
-  bool isSizeLargerThan(uint64_t MaxSize) const;
+  LLVM_ABI bool isSizeLargerThan(uint64_t MaxSize) const;
 
   /// Return true if all values in this range are negative.
-  bool isAllNegative() const;
+  LLVM_ABI bool isAllNegative() const;
 
   /// Return true if all values in this range are non-negative.
-  bool isAllNonNegative() const;
+  LLVM_ABI bool isAllNonNegative() const;
 
   /// Return true if all values in this range are positive.
-  bool isAllPositive() const;
+  LLVM_ABI bool isAllPositive() const;
 
   /// Return the largest unsigned value contained in the ConstantRange.
-  APInt getUnsignedMax() const;
+  LLVM_ABI APInt getUnsignedMax() const;
 
   /// Return the smallest unsigned value contained in the ConstantRange.
-  APInt getUnsignedMin() const;
+  LLVM_ABI APInt getUnsignedMin() const;
 
   /// Return the largest signed value contained in the ConstantRange.
-  APInt getSignedMax() const;
+  LLVM_ABI APInt getSignedMax() const;
 
   /// Return the smallest signed value contained in the ConstantRange.
-  APInt getSignedMin() const;
+  LLVM_ABI APInt getSignedMin() const;
 
   /// Return true if this range is equal to another range.
   bool operator==(const ConstantRange &CR) const {
@@ -306,18 +308,18 @@ class [[nodiscard]] ConstantRange {
 
   /// Compute the maximal number of active bits needed to represent every value
   /// in this range.
-  unsigned getActiveBits() const;
+  LLVM_ABI unsigned getActiveBits() const;
 
   /// Compute the maximal number of bits needed to represent every value
   /// in this signed range.
-  unsigned getMinSignedBits() const;
+  LLVM_ABI unsigned getMinSignedBits() const;
 
   /// Subtract the specified constant from the endpoints of this constant range.
-  ConstantRange subtract(const APInt &CI) const;
+  LLVM_ABI ConstantRange subtract(const APInt &CI) const;
 
   /// Subtract the specified range from this range (aka relative complement of
   /// the sets).
-  ConstantRange difference(const ConstantRange &CR) const;
+  LLVM_ABI ConstantRange difference(const ConstantRange &CR) const;
 
   /// If represented precisely, the result of some range operations may consist
   /// of multiple disjoint ranges. As only a single range may be returned, any
@@ -333,25 +335,26 @@ class [[nodiscard]] ConstantRange {
   /// Return the range that results from the intersection of this range with
   /// another range. If the intersection is disjoint, such that two results
   /// are possible, the preferred range is determined by the PreferredRangeType.
-  ConstantRange intersectWith(const ConstantRange &CR,
-                              PreferredRangeType Type = Smallest) const;
+  LLVM_ABI ConstantRange intersectWith(
+      const ConstantRange &CR, PreferredRangeType Type = Smallest) const;
 
   /// Return the range that results from the union of this range
   /// with another range.  The resultant range is guaranteed to include the
   /// elements of both sets, but may contain more.  For example, [3, 9) union
   /// [12,15) is [3, 15), which includes 9, 10, and 11, which were not included
   /// in either set before.
-  ConstantRange unionWith(const ConstantRange &CR,
-                          PreferredRangeType Type = Smallest) const;
+  LLVM_ABI ConstantRange unionWith(const ConstantRange &CR,
+                                   PreferredRangeType Type = Smallest) const;
 
   /// Intersect the two ranges and return the result if it can be represented
   /// exactly, otherwise return std::nullopt.
-  std::optional<ConstantRange>
+  LLVM_ABI std::optional<ConstantRange>
   exactIntersectWith(const ConstantRange &CR) const;
 
   /// Union the two ranges and return the result if it can be represented
   /// exactly, otherwise return std::nullopt.
-  std::optional<ConstantRange> exactUnionWith(const ConstantRange &CR) const;
+  LLVM_ABI std::optional<ConstantRange>
+  exactUnionWith(const ConstantRange &CR) const;
 
   /// Return a new range representing the possible values resulting
   /// from an application of the specified cast operator to this range. \p
@@ -359,212 +362,215 @@ class [[nodiscard]] ConstantRange {
   /// change bitwidth, it must be the same as the source bitwidth.  For casts
   /// which do change bitwidth, the bitwidth must be consistent with the
   /// requested cast and source bitwidth.
-  ConstantRange castOp(Instruction::CastOps CastOp,
-                       uint32_t BitWidth) const;
+  LLVM_ABI ConstantRange castOp(Instruction::CastOps CastOp,
+                                uint32_t BitWidth) const;
 
   /// Return a new range in the specified integer type, which must
   /// be strictly larger than the current type.  The returned range will
   /// correspond to the possible range of values if the source range had been
   /// zero extended to BitWidth.
-  ConstantRange zeroExtend(uint32_t BitWidth) const;
+  LLVM_ABI ConstantRange zeroExtend(uint32_t BitWidth) const;
 
   /// Return a new range in the specified integer type, which must
   /// be strictly larger than the current type.  The returned range will
   /// correspond to the possible range of values if the source range had been
   /// sign extended to BitWidth.
-  ConstantRange signExtend(uint32_t BitWidth) const;
+  LLVM_ABI ConstantRange signExtend(uint32_t BitWidth) const;
 
   /// Return a new range in the specified integer type, which must be
   /// strictly smaller than the current type.  The returned range will
   /// correspond to the possible range of values if the source range had been
   /// truncated to the specified type.
-  ConstantRange truncate(uint32_t BitWidth) const;
+  LLVM_ABI ConstantRange truncate(uint32_t BitWidth) const;
 
   /// Make this range have the bit width given by \p BitWidth. The
   /// value is zero extended, truncated, or left alone to make it that width.
-  ConstantRange zextOrTrunc(uint32_t BitWidth) const;
+  LLVM_ABI ConstantRange zextOrTrunc(uint32_t BitWidth) const;
 
   /// Make this range have the bit width given by \p BitWidth. The
   /// value is sign extended, truncated, or left alone to make it that width.
-  ConstantRange sextOrTrunc(uint32_t BitWidth) const;
+  LLVM_ABI ConstantRange sextOrTrunc(uint32_t BitWidth) const;
 
   /// Return a new range representing the possible values resulting
   /// from an application of the specified binary operator to an left hand side
   /// of this range and a right hand side of \p Other.
-  ConstantRange binaryOp(Instruction::BinaryOps BinOp,
-                         const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange binaryOp(Instruction::BinaryOps BinOp,
+                                  const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from an application of the specified overflowing binary operator to a
   /// left hand side of this range and a right hand side of \p Other given
   /// the provided knowledge about lack of wrapping \p NoWrapKind.
-  ConstantRange overflowingBinaryOp(Instruction::BinaryOps BinOp,
-                                    const ConstantRange &Other,
-                                    unsigned NoWrapKind) const;
+  LLVM_ABI ConstantRange overflowingBinaryOp(Instruction::BinaryOps BinOp,
+                                             const ConstantRange &Other,
+                                             unsigned NoWrapKind) const;
 
   /// Return a new range representing the possible values resulting
   /// from an addition of a value in this range and a value in \p Other.
-  ConstantRange add(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange add(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from an addition with wrap type \p NoWrapKind of a value in this
   /// range and a value in \p Other.
   /// If the result range is disjoint, the preferred range is determined by the
   /// \p PreferredRangeType.
-  ConstantRange addWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
-                              PreferredRangeType RangeType = Smallest) const;
+  LLVM_ABI ConstantRange
+  addWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
+                PreferredRangeType RangeType = Smallest) const;
 
   /// Return a new range representing the possible values resulting
   /// from a subtraction of a value in this range and a value in \p Other.
-  ConstantRange sub(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange sub(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from an subtraction with wrap type \p NoWrapKind of a value in this
   /// range and a value in \p Other.
   /// If the result range is disjoint, the preferred range is determined by the
   /// \p PreferredRangeType.
-  ConstantRange subWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
-                              PreferredRangeType RangeType = Smallest) const;
+  LLVM_ABI ConstantRange
+  subWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
+                PreferredRangeType RangeType = Smallest) const;
 
   /// Return a new range representing the possible values resulting
   /// from a multiplication of a value in this range and a value in \p Other,
   /// treating both this and \p Other as unsigned ranges.
-  ConstantRange multiply(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange multiply(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a multiplication with wrap type \p NoWrapKind of a value in this
   /// range and a value in \p Other.
   /// If the result range is disjoint, the preferred range is determined by the
   /// \p PreferredRangeType.
-  ConstantRange
+  LLVM_ABI ConstantRange
   multiplyWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
                      PreferredRangeType RangeType = Smallest) const;
 
   /// Return range of possible values for a signed multiplication of this and
   /// \p Other. However, if overflow is possible always return a full range
   /// rather than trying to determine a more precise result.
-  ConstantRange smul_fast(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange smul_fast(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a signed maximum of a value in this range and a value in \p Other.
-  ConstantRange smax(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange smax(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from an unsigned maximum of a value in this range and a value in \p Other.
-  ConstantRange umax(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange umax(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a signed minimum of a value in this range and a value in \p Other.
-  ConstantRange smin(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange smin(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from an unsigned minimum of a value in this range and a value in \p Other.
-  ConstantRange umin(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange umin(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from an unsigned division of a value in this range and a value in
   /// \p Other.
-  ConstantRange udiv(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange udiv(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a signed division of a value in this range and a value in
   /// \p Other. Division by zero and division of SignedMin by -1 are considered
   /// undefined behavior, in line with IR, and do not contribute towards the
   /// result.
-  ConstantRange sdiv(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange sdiv(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from an unsigned remainder operation of a value in this range and a
   /// value in \p Other.
-  ConstantRange urem(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange urem(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a signed remainder operation of a value in this range and a
   /// value in \p Other.
-  ConstantRange srem(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange srem(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting from
   /// a binary-xor of a value in this range by an all-one value,
   /// aka bitwise complement operation.
-  ConstantRange binaryNot() const;
+  LLVM_ABI ConstantRange binaryNot() const;
 
   /// Return a new range representing the possible values resulting
   /// from a binary-and of a value in this range by a value in \p Other.
-  ConstantRange binaryAnd(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange binaryAnd(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a binary-or of a value in this range by a value in \p Other.
-  ConstantRange binaryOr(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange binaryOr(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a binary-xor of a value in this range by a value in \p Other.
-  ConstantRange binaryXor(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange binaryXor(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a left shift of a value in this range by a value in \p Other.
   /// TODO: This isn't fully implemented yet.
-  ConstantRange shl(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange shl(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting
   /// from a left shift with wrap type \p NoWrapKind of a value in this
   /// range and a value in \p Other.
   /// If the result range is disjoint, the preferred range is determined by the
   /// \p PreferredRangeType.
-  ConstantRange shlWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
-                              PreferredRangeType RangeType = Smallest) const;
+  LLVM_ABI ConstantRange
+  shlWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
+                PreferredRangeType RangeType = Smallest) const;
 
   /// Return a new range representing the possible values resulting from a
   /// logical right shift of a value in this range and a value in \p Other.
-  ConstantRange lshr(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange lshr(const ConstantRange &Other) const;
 
   /// Return a new range representing the possible values resulting from a
   /// arithmetic right shift of a value in this range and a value in \p Other.
-  ConstantRange ashr(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange ashr(const ConstantRange &Other) const;
 
   /// Perform an unsigned saturating addition of two constant ranges.
-  ConstantRange uadd_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange uadd_sat(const ConstantRange &Other) const;
 
   /// Perform a signed saturating addition of two constant ranges.
-  ConstantRange sadd_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange sadd_sat(const ConstantRange &Other) const;
 
   /// Perform an unsigned saturating subtraction of two constant ranges.
-  ConstantRange usub_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange usub_sat(const ConstantRange &Other) const;
 
   /// Perform a signed saturating subtraction of two constant ranges.
-  ConstantRange ssub_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange ssub_sat(const ConstantRange &Other) const;
 
   /// Perform an unsigned saturating multiplication of two constant ranges.
-  ConstantRange umul_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange umul_sat(const ConstantRange &Other) const;
 
   /// Perform a signed saturating multiplication of two constant ranges.
-  ConstantRange smul_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange smul_sat(const ConstantRange &Other) const;
 
   /// Perform an unsigned saturating left shift of this constant range by a
   /// value in \p Other.
-  ConstantRange ushl_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange ushl_sat(const ConstantRange &Other) const;
 
   /// Perform a signed saturating left shift of this constant range by a
   /// value in \p Other.
-  ConstantRange sshl_sat(const ConstantRange &Other) const;
+  LLVM_ABI ConstantRange sshl_sat(const ConstantRange &Other) const;
 
   /// Return a new range that is the logical not of the current set.
-  ConstantRange inverse() const;
+  LLVM_ABI ConstantRange inverse() const;
 
   /// Calculate absolute value range. If the original range contains signed
   /// min, then the resulting range will contain signed min if and only if
   /// \p IntMinIsPoison is false.
-  ConstantRange abs(bool IntMinIsPoison = false) const;
+  LLVM_ABI ConstantRange abs(bool IntMinIsPoison = false) const;
 
   /// Calculate ctlz range. If \p ZeroIsPoison is set, the range is computed
   /// ignoring a possible zero value contained in the input range.
-  ConstantRange ctlz(bool ZeroIsPoison = false) const;
+  LLVM_ABI ConstantRange ctlz(bool ZeroIsPoison = false) const;
 
   /// Calculate cttz range. If \p ZeroIsPoison is set, the range is computed
   /// ignoring a possible zero value contained in the input range.
-  ConstantRange cttz(bool ZeroIsPoison = false) const;
+  LLVM_ABI ConstantRange cttz(bool ZeroIsPoison = false) const;
 
   /// Calculate ctpop range.
-  ConstantRange ctpop() const;
+  LLVM_ABI ConstantRange ctpop() const;
 
   /// Represents whether an operation on the given constant range is known to
   /// always or never overflow.
@@ -580,28 +586,33 @@ class [[nodiscard]] ConstantRange {
   };
 
   /// Return whether unsigned add of the two ranges always/never overflows.
-  OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const;
+  LLVM_ABI OverflowResult
+  unsignedAddMayOverflow(const ConstantRange &Other) const;
 
   /// Return whether signed add of the two ranges always/never overflows.
-  OverflowResult signedAddMayOverflow(const ConstantRange &Other) const;
+  LLVM_ABI OverflowResult
+  signedAddMayOverflow(const ConstantRange &Other) const;
 
   /// Return whether unsigned sub of the two ranges always/never overflows.
-  OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const;
+  LLVM_ABI OverflowResult
+  unsignedSubMayOverflow(const ConstantRange &Other) const;
 
   /// Return whether signed sub of the two ranges always/never overflows.
-  OverflowResult signedSubMayOverflow(const ConstantRange &Other) const;
+  LLVM_ABI OverflowResult
+  signedSubMayOverflow(const ConstantRange &Other) const;
 
   /// Return whether unsigned mul of the two ranges always/never overflows.
-  OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const;
+  LLVM_ABI OverflowResult
+  unsignedMulMayOverflow(const ConstantRange &Other) const;
 
   /// Return known bits for values in this range.
-  KnownBits toKnownBits() const;
+  LLVM_ABI KnownBits toKnownBits() const;
 
   /// Print out the bounds to a stream.
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
   /// Allow printing from a debugger easily.
-  void dump() const;
+  LLVM_ABI void dump() const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const ConstantRange &CR) {
@@ -612,7 +623,7 @@ inline raw_ostream &operator<<(raw_ostream &OS, const ConstantRange &CR) {
 /// Parse out a conservative ConstantRange from !range metadata.
 ///
 /// E.g. if RangeMD is !{i32 0, i32 10, i32 15, i32 20} then return [0, 20).
-ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD);
+LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/ConstantRangeList.h b/external/llvm-project/llvm/include/llvm/IR/ConstantRangeList.h
index b12c913103df..104018bc7315 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ConstantRangeList.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ConstantRangeList.h
@@ -18,6 +18,7 @@
 
 #include "llvm/ADT/APInt.h"
 #include "llvm/IR/ConstantRange.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include <cstddef>
 #include <cstdint>
@@ -41,8 +42,8 @@ class [[nodiscard]] ConstantRangeList {
   }
 
   // Return true if the ranges are non-overlapping and increasing.
-  static bool isOrderedRanges(ArrayRef<ConstantRange> RangesRef);
-  static std::optional<ConstantRangeList>
+  LLVM_ABI static bool isOrderedRanges(ArrayRef<ConstantRange> RangesRef);
+  LLVM_ABI static std::optional<ConstantRangeList>
   getConstantRangeList(ArrayRef<ConstantRange> RangesRef);
 
   ArrayRef<ConstantRange> rangesRef() const { return Ranges; }
@@ -67,21 +68,21 @@ class [[nodiscard]] ConstantRangeList {
   size_t size() const { return Ranges.size(); }
 
   /// Insert a new range to Ranges and keep the list ordered.
-  void insert(const ConstantRange &NewRange);
+  LLVM_ABI void insert(const ConstantRange &NewRange);
   void insert(int64_t Lower, int64_t Upper) {
     insert(ConstantRange(APInt(64, Lower, /*isSigned=*/true),
                          APInt(64, Upper, /*isSigned=*/true)));
   }
 
-  void subtract(const ConstantRange &SubRange);
+  LLVM_ABI void subtract(const ConstantRange &SubRange);
 
   /// Return the range list that results from the union of this
   /// ConstantRangeList with another ConstantRangeList, "CRL".
-  ConstantRangeList unionWith(const ConstantRangeList &CRL) const;
+  LLVM_ABI ConstantRangeList unionWith(const ConstantRangeList &CRL) const;
 
   /// Return the range list that results from the intersection of this
   /// ConstantRangeList with another ConstantRangeList, "CRL".
-  ConstantRangeList intersectWith(const ConstantRangeList &CRL) const;
+  LLVM_ABI ConstantRangeList intersectWith(const ConstantRangeList &CRL) const;
 
   /// Return true if this range list is equal to another range list.
   bool operator==(const ConstantRangeList &CRL) const {
@@ -92,7 +93,7 @@ class [[nodiscard]] ConstantRangeList {
   }
 
   /// Print out the ranges to a stream.
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const;
diff --git a/external/llvm-project/llvm/include/llvm/IR/Constants.h b/external/llvm-project/llvm/include/llvm/IR/Constants.h
index 6a01e8209071..9c9fc8892bdb 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Constants.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Constants.h
@@ -103,16 +103,16 @@ class ConstantInt final : public ConstantData {
 public:
   ConstantInt(const ConstantInt &) = delete;
 
-  static ConstantInt *getTrue(LLVMContext &Context);
-  static ConstantInt *getFalse(LLVMContext &Context);
-  static ConstantInt *getBool(LLVMContext &Context, bool V);
-  static Constant *getTrue(Type *Ty);
-  static Constant *getFalse(Type *Ty);
-  static Constant *getBool(Type *Ty, bool V);
+  LLVM_ABI static ConstantInt *getTrue(LLVMContext &Context);
+  LLVM_ABI static ConstantInt *getFalse(LLVMContext &Context);
+  LLVM_ABI static ConstantInt *getBool(LLVMContext &Context, bool V);
+  LLVM_ABI static Constant *getTrue(Type *Ty);
+  LLVM_ABI static Constant *getFalse(Type *Ty);
+  LLVM_ABI static Constant *getBool(Type *Ty, bool V);
 
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantInt for the given value.
-  static Constant *get(Type *Ty, uint64_t V, bool IsSigned = false);
+  LLVM_ABI static Constant *get(Type *Ty, uint64_t V, bool IsSigned = false);
 
   /// Return a ConstantInt with the specified integer value for the specified
   /// type. If the type is wider than 64 bits, the value will be zero-extended
@@ -120,7 +120,8 @@ class ConstantInt final : public ConstantData {
   /// be interpreted as a 64-bit signed integer and sign-extended to fit
   /// the type.
   /// Get a ConstantInt for a specific value.
-  static ConstantInt *get(IntegerType *Ty, uint64_t V, bool IsSigned = false);
+  LLVM_ABI static ConstantInt *get(IntegerType *Ty, uint64_t V,
+                                   bool IsSigned = false);
 
   /// Return a ConstantInt with the specified value for the specified type. The
   /// value V will be canonicalized to a an unsigned APInt. Accessing it with
@@ -136,15 +137,16 @@ class ConstantInt final : public ConstantData {
 
   /// Return a ConstantInt with the specified value and an implied Type. The
   /// type is the integer type that corresponds to the bit width of the value.
-  static ConstantInt *get(LLVMContext &Context, const APInt &V);
+  LLVM_ABI static ConstantInt *get(LLVMContext &Context, const APInt &V);
 
   /// Return a ConstantInt constructed from the string strStart with the given
   /// radix.
-  static ConstantInt *get(IntegerType *Ty, StringRef Str, uint8_t Radix);
+  LLVM_ABI static ConstantInt *get(IntegerType *Ty, StringRef Str,
+                                   uint8_t Radix);
 
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantInt for the given value.
-  static Constant *get(Type *Ty, const APInt &V);
+  LLVM_ABI static Constant *get(Type *Ty, const APInt &V);
 
   /// Return the constant as an APInt value reference. This allows clients to
   /// obtain a full-precision copy of the value.
@@ -201,8 +203,8 @@ class ConstantInt final : public ConstantData {
   /// to the appropriate unsigned type before calling the method.
   /// @returns true if V is a valid value for type Ty
   /// Determine if the value is in range for the given type.
-  static bool isValueValidForType(Type *Ty, uint64_t V);
-  static bool isValueValidForType(Type *Ty, int64_t V);
+  LLVM_ABI static bool isValueValidForType(Type *Ty, uint64_t V);
+  LLVM_ABI static bool isValueValidForType(Type *Ty, int64_t V);
 
   bool isNegative() const { return Val.isNegative(); }
 
@@ -295,26 +297,26 @@ class ConstantFP final : public ConstantData {
   /// for the specified value in the specified type. This should only be used
   /// for simple constant values like 2.0/1.0 etc, that are known-valid both as
   /// host double and as the target format.
-  static Constant *get(Type *Ty, double V);
+  LLVM_ABI static Constant *get(Type *Ty, double V);
 
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantFP for the given value.
-  static Constant *get(Type *Ty, const APFloat &V);
-
-  static Constant *get(Type *Ty, StringRef Str);
-  static ConstantFP *get(LLVMContext &Context, const APFloat &V);
-  static Constant *getNaN(Type *Ty, bool Negative = false,
-                          uint64_t Payload = 0);
-  static Constant *getQNaN(Type *Ty, bool Negative = false,
-                           APInt *Payload = nullptr);
-  static Constant *getSNaN(Type *Ty, bool Negative = false,
-                           APInt *Payload = nullptr);
-  static Constant *getZero(Type *Ty, bool Negative = false);
+  LLVM_ABI static Constant *get(Type *Ty, const APFloat &V);
+
+  LLVM_ABI static Constant *get(Type *Ty, StringRef Str);
+  LLVM_ABI static ConstantFP *get(LLVMContext &Context, const APFloat &V);
+  LLVM_ABI static Constant *getNaN(Type *Ty, bool Negative = false,
+                                   uint64_t Payload = 0);
+  LLVM_ABI static Constant *getQNaN(Type *Ty, bool Negative = false,
+                                    APInt *Payload = nullptr);
+  LLVM_ABI static Constant *getSNaN(Type *Ty, bool Negative = false,
+                                    APInt *Payload = nullptr);
+  LLVM_ABI static Constant *getZero(Type *Ty, bool Negative = false);
   static Constant *getNegativeZero(Type *Ty) { return getZero(Ty, true); }
-  static Constant *getInfinity(Type *Ty, bool Negative = false);
+  LLVM_ABI static Constant *getInfinity(Type *Ty, bool Negative = false);
 
   /// Return true if Ty is big enough to represent V.
-  static bool isValueValidForType(Type *Ty, const APFloat &V);
+  LLVM_ABI static bool isValueValidForType(Type *Ty, const APFloat &V);
   inline const APFloat &getValueAPF() const { return Val; }
   inline const APFloat &getValue() const { return Val; }
 
@@ -336,7 +338,7 @@ class ConstantFP final : public ConstantData {
   /// two floating point values.  The version with a double operand is retained
   /// because it's so convenient to write isExactlyValue(2.0), but please use
   /// it only for simple constants.
-  bool isExactlyValue(const APFloat &V) const;
+  LLVM_ABI bool isExactlyValue(const APFloat &V) const;
 
   bool isExactlyValue(double V) const {
     bool ignored;
@@ -365,25 +367,25 @@ class ConstantAggregateZero final : public ConstantData {
 public:
   ConstantAggregateZero(const ConstantAggregateZero &) = delete;
 
-  static ConstantAggregateZero *get(Type *Ty);
+  LLVM_ABI static ConstantAggregateZero *get(Type *Ty);
 
   /// If this CAZ has array or vector type, return a zero with the right element
   /// type.
-  Constant *getSequentialElement() const;
+  LLVM_ABI Constant *getSequentialElement() const;
 
   /// If this CAZ has struct type, return a zero with the right element type for
   /// the specified element.
-  Constant *getStructElement(unsigned Elt) const;
+  LLVM_ABI Constant *getStructElement(unsigned Elt) const;
 
   /// Return a zero of the right value for the specified GEP index if we can,
   /// otherwise return null (e.g. if C is a ConstantExpr).
-  Constant *getElementValue(Constant *C) const;
+  LLVM_ABI Constant *getElementValue(Constant *C) const;
 
   /// Return a zero of the right value for the specified GEP index.
-  Constant *getElementValue(unsigned Idx) const;
+  LLVM_ABI Constant *getElementValue(unsigned Idx) const;
 
   /// Return the number of elements in the array, vector, or struct.
-  ElementCount getElementCount() const;
+  LLVM_ABI ElementCount getElementCount() const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   ///
@@ -405,8 +407,8 @@ class ConstantAggregateZero final : public ConstantData {
 /// use operands.
 class ConstantAggregate : public Constant {
 protected:
-  ConstantAggregate(Type *T, ValueTy VT, ArrayRef<Constant *> V,
-                    AllocInfo AllocInfo);
+  LLVM_ABI ConstantAggregate(Type *T, ValueTy VT, ArrayRef<Constant *> V,
+                             AllocInfo AllocInfo);
 
 public:
   /// Transparently provide more efficient getOperand methods.
@@ -439,7 +441,7 @@ class ConstantArray final : public ConstantAggregate {
 
 public:
   // ConstantArray accessors
-  static Constant *get(ArrayType *T, ArrayRef<Constant *> V);
+  LLVM_ABI static Constant *get(ArrayType *T, ArrayRef<Constant *> V);
 
 private:
   static Constant *getImpl(ArrayType *T, ArrayRef<Constant *> V);
@@ -471,7 +473,7 @@ class ConstantStruct final : public ConstantAggregate {
 
 public:
   // ConstantStruct accessors
-  static Constant *get(StructType *T, ArrayRef<Constant *> V);
+  LLVM_ABI static Constant *get(StructType *T, ArrayRef<Constant *> V);
 
   template <typename... Csts>
   static std::enable_if_t<are_base_of<Constant, Csts...>::value, Constant *>
@@ -491,12 +493,12 @@ class ConstantStruct final : public ConstantAggregate {
 
   /// Return an anonymous struct type to use for a constant with the specified
   /// set of elements. The list must not be empty.
-  static StructType *getTypeForElements(ArrayRef<Constant *> V,
-                                        bool Packed = false);
+  LLVM_ABI static StructType *getTypeForElements(ArrayRef<Constant *> V,
+                                                 bool Packed = false);
   /// This version of the method allows an empty list.
-  static StructType *getTypeForElements(LLVMContext &Ctx,
-                                        ArrayRef<Constant *> V,
-                                        bool Packed = false);
+  LLVM_ABI static StructType *getTypeForElements(LLVMContext &Ctx,
+                                                 ArrayRef<Constant *> V,
+                                                 bool Packed = false);
 
   /// Specialization - reduce amount of casting.
   inline StructType *getType() const {
@@ -523,7 +525,7 @@ class ConstantVector final : public ConstantAggregate {
 
 public:
   // ConstantVector accessors
-  static Constant *get(ArrayRef<Constant *> V);
+  LLVM_ABI static Constant *get(ArrayRef<Constant *> V);
 
 private:
   static Constant *getImpl(ArrayRef<Constant *> V);
@@ -531,7 +533,7 @@ class ConstantVector final : public ConstantAggregate {
 public:
   /// Return a ConstantVector with the specified constant in each element.
   /// Note that this might not return an instance of ConstantVector
-  static Constant *getSplat(ElementCount EC, Constant *Elt);
+  LLVM_ABI static Constant *getSplat(ElementCount EC, Constant *Elt);
 
   /// Specialize the getType() method to always return a FixedVectorType,
   /// which reduces the amount of casting needed in parts of the compiler.
@@ -542,7 +544,7 @@ class ConstantVector final : public ConstantAggregate {
   /// If all elements of the vector constant have the same value, return that
   /// value. Otherwise, return nullptr. Ignore poison elements by setting
   /// AllowPoison to true.
-  Constant *getSplatValue(bool AllowPoison = false) const;
+  LLVM_ABI Constant *getSplatValue(bool AllowPoison = false) const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
@@ -565,7 +567,7 @@ class ConstantPointerNull final : public ConstantData {
   ConstantPointerNull(const ConstantPointerNull &) = delete;
 
   /// Static factory methods - Return objects of the specified value
-  static ConstantPointerNull *get(PointerType *T);
+  LLVM_ABI static ConstantPointerNull *get(PointerType *T);
 
   /// Specialize the getType() method to always return an PointerType,
   /// which reduces the amount of casting needed in parts of the compiler.
@@ -608,7 +610,7 @@ class ConstantDataSequential : public ConstantData {
   explicit ConstantDataSequential(Type *ty, ValueTy VT, const char *Data)
       : ConstantData(ty, VT), DataElements(Data) {}
 
-  static Constant *getImpl(StringRef Bytes, Type *Ty);
+  LLVM_ABI static Constant *getImpl(StringRef Bytes, Type *Ty);
 
 public:
   ConstantDataSequential(const ConstantDataSequential &) = delete;
@@ -617,49 +619,49 @@ class ConstantDataSequential : public ConstantData {
   /// array of the specified element type.
   /// ConstantDataArray only works with normal float and int types that are
   /// stored densely in memory, not with things like i42 or x86_f80.
-  static bool isElementTypeCompatible(Type *Ty);
+  LLVM_ABI static bool isElementTypeCompatible(Type *Ty);
 
   /// If this is a sequential container of integers (of any size), return the
   /// specified element in the low bits of a uint64_t.
-  uint64_t getElementAsInteger(uint64_t i) const;
+  LLVM_ABI uint64_t getElementAsInteger(uint64_t i) const;
 
   /// If this is a sequential container of integers (of any size), return the
   /// specified element as an APInt.
-  APInt getElementAsAPInt(uint64_t i) const;
+  LLVM_ABI APInt getElementAsAPInt(uint64_t i) const;
 
   /// If this is a sequential container of floating point type, return the
   /// specified element as an APFloat.
-  APFloat getElementAsAPFloat(uint64_t i) const;
+  LLVM_ABI APFloat getElementAsAPFloat(uint64_t i) const;
 
   /// If this is an sequential container of floats, return the specified element
   /// as a float.
-  float getElementAsFloat(uint64_t i) const;
+  LLVM_ABI float getElementAsFloat(uint64_t i) const;
 
   /// If this is an sequential container of doubles, return the specified
   /// element as a double.
-  double getElementAsDouble(uint64_t i) const;
+  LLVM_ABI double getElementAsDouble(uint64_t i) const;
 
   /// Return a Constant for a specified index's element.
   /// Note that this has to compute a new constant to return, so it isn't as
   /// efficient as getElementAsInteger/Float/Double.
-  Constant *getElementAsConstant(uint64_t i) const;
+  LLVM_ABI Constant *getElementAsConstant(uint64_t i) const;
 
   /// Return the element type of the array/vector.
-  Type *getElementType() const;
+  LLVM_ABI Type *getElementType() const;
 
   /// Return the number of elements in the array or vector.
-  uint64_t getNumElements() const;
+  LLVM_ABI uint64_t getNumElements() const;
 
   /// Return the size (in bytes) of each element in the array/vector.
   /// The size of the elements is known to be a multiple of one byte.
-  uint64_t getElementByteSize() const;
+  LLVM_ABI uint64_t getElementByteSize() const;
 
   /// This method returns true if this is an array of \p CharSize integers.
-  bool isString(unsigned CharSize = 8) const;
+  LLVM_ABI bool isString(unsigned CharSize = 8) const;
 
   /// This method returns true if the array "isString", ends with a null byte,
   /// and does not contains any other null bytes.
-  bool isCString() const;
+  LLVM_ABI bool isCString() const;
 
   /// If this array is isString(), then this method returns the array as a
   /// StringRef. Otherwise, it asserts out.
@@ -679,7 +681,7 @@ class ConstantDataSequential : public ConstantData {
   /// Return the raw, underlying, bytes of this data. Note that this is an
   /// extremely tricky thing to work with, as it exposes the host endianness of
   /// the data elements.
-  StringRef getRawDataValues() const;
+  LLVM_ABI StringRef getRawDataValues() const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
@@ -741,17 +743,17 @@ class ConstantDataArray final : public ConstantDataSequential {
   /// number of bits of the type contained in the passed in ArrayRef.
   /// (i.e. half or bfloat for 16bits, float for 32bits, double for 64bits) Note
   /// that this can return a ConstantAggregateZero object.
-  static Constant *getFP(Type *ElementType, ArrayRef<uint16_t> Elts);
-  static Constant *getFP(Type *ElementType, ArrayRef<uint32_t> Elts);
-  static Constant *getFP(Type *ElementType, ArrayRef<uint64_t> Elts);
+  LLVM_ABI static Constant *getFP(Type *ElementType, ArrayRef<uint16_t> Elts);
+  LLVM_ABI static Constant *getFP(Type *ElementType, ArrayRef<uint32_t> Elts);
+  LLVM_ABI static Constant *getFP(Type *ElementType, ArrayRef<uint64_t> Elts);
 
   /// This method constructs a CDS and initializes it with a text string.
   /// The default behavior (AddNull==true) causes a null terminator to
   /// be placed at the end of the array (increasing the length of the string by
   /// one more than the StringRef would normally indicate.  Pass AddNull=false
   /// to disable this behavior.
-  static Constant *getString(LLVMContext &Context, StringRef Initializer,
-                             bool AddNull = true);
+  LLVM_ABI static Constant *
+  getString(LLVMContext &Context, StringRef Initializer, bool AddNull = true);
 
   /// Specialize the getType() method to always return an ArrayType,
   /// which reduces the amount of casting needed in parts of the compiler.
@@ -788,12 +790,12 @@ class ConstantDataVector final : public ConstantDataSequential {
   /// get() constructors - Return a constant with vector type with an element
   /// count and element type matching the ArrayRef passed in.  Note that this
   /// can return a ConstantAggregateZero object.
-  static Constant *get(LLVMContext &Context, ArrayRef<uint8_t> Elts);
-  static Constant *get(LLVMContext &Context, ArrayRef<uint16_t> Elts);
-  static Constant *get(LLVMContext &Context, ArrayRef<uint32_t> Elts);
-  static Constant *get(LLVMContext &Context, ArrayRef<uint64_t> Elts);
-  static Constant *get(LLVMContext &Context, ArrayRef<float> Elts);
-  static Constant *get(LLVMContext &Context, ArrayRef<double> Elts);
+  LLVM_ABI static Constant *get(LLVMContext &Context, ArrayRef<uint8_t> Elts);
+  LLVM_ABI static Constant *get(LLVMContext &Context, ArrayRef<uint16_t> Elts);
+  LLVM_ABI static Constant *get(LLVMContext &Context, ArrayRef<uint32_t> Elts);
+  LLVM_ABI static Constant *get(LLVMContext &Context, ArrayRef<uint64_t> Elts);
+  LLVM_ABI static Constant *get(LLVMContext &Context, ArrayRef<float> Elts);
+  LLVM_ABI static Constant *get(LLVMContext &Context, ArrayRef<double> Elts);
 
   /// getRaw() constructor - Return a constant with vector type with an element
   /// count and element type matching the NumElements and ElementTy parameters
@@ -813,22 +815,22 @@ class ConstantDataVector final : public ConstantDataSequential {
   /// number of bits of the type contained in the passed in ArrayRef.
   /// (i.e. half or bfloat for 16bits, float for 32bits, double for 64bits) Note
   /// that this can return a ConstantAggregateZero object.
-  static Constant *getFP(Type *ElementType, ArrayRef<uint16_t> Elts);
-  static Constant *getFP(Type *ElementType, ArrayRef<uint32_t> Elts);
-  static Constant *getFP(Type *ElementType, ArrayRef<uint64_t> Elts);
+  LLVM_ABI static Constant *getFP(Type *ElementType, ArrayRef<uint16_t> Elts);
+  LLVM_ABI static Constant *getFP(Type *ElementType, ArrayRef<uint32_t> Elts);
+  LLVM_ABI static Constant *getFP(Type *ElementType, ArrayRef<uint64_t> Elts);
 
   /// Return a ConstantVector with the specified constant in each element.
   /// The specified constant has to be a of a compatible type (i8/i16/
   /// i32/i64/half/bfloat/float/double) and must be a ConstantFP or ConstantInt.
-  static Constant *getSplat(unsigned NumElts, Constant *Elt);
+  LLVM_ABI static Constant *getSplat(unsigned NumElts, Constant *Elt);
 
   /// Returns true if this is a splat constant, meaning that all elements have
   /// the same value.
-  bool isSplat() const;
+  LLVM_ABI bool isSplat() const;
 
   /// If this is a splat constant, meaning that all of the elements have the
   /// same value, return that value. Otherwise return NULL.
-  Constant *getSplatValue() const;
+  LLVM_ABI Constant *getSplatValue() const;
 
   /// Specialize the getType() method to always return a FixedVectorType,
   /// which reduces the amount of casting needed in parts of the compiler.
@@ -857,7 +859,7 @@ class ConstantTokenNone final : public ConstantData {
   ConstantTokenNone(const ConstantTokenNone &) = delete;
 
   /// Return the ConstantTokenNone.
-  static ConstantTokenNone *get(LLVMContext &Context);
+  LLVM_ABI static ConstantTokenNone *get(LLVMContext &Context);
 
   /// Methods to support type inquiry through isa, cast, and dyn_cast.
   static bool classof(const Value *V) {
@@ -878,7 +880,7 @@ class ConstantTargetNone final : public ConstantData {
   ConstantTargetNone(const ConstantTargetNone &) = delete;
 
   /// Static factory methods - Return objects of the specified value.
-  static ConstantTargetNone *get(TargetExtType *T);
+  LLVM_ABI static ConstantTargetNone *get(TargetExtType *T);
 
   /// Specialize the getType() method to always return an TargetExtType,
   /// which reduces the amount of casting needed in parts of the compiler.
@@ -910,21 +912,21 @@ class BlockAddress final : public Constant {
   void operator delete(void *Ptr) { User::operator delete(Ptr); }
 
   /// Return a BlockAddress for the specified function and basic block.
-  static BlockAddress *get(Function *F, BasicBlock *BB);
+  LLVM_ABI static BlockAddress *get(Function *F, BasicBlock *BB);
 
   /// Return a BlockAddress for the specified basic block.  The basic
   /// block must be embedded into a function.
-  static BlockAddress *get(BasicBlock *BB);
+  LLVM_ABI static BlockAddress *get(BasicBlock *BB);
 
   /// Return a BlockAddress for the specified basic block, which may not be
   /// part of a function. The specified type must match the type of the function
   /// the block will be inserted into.
-  static BlockAddress *get(Type *Ty, BasicBlock *BB);
+  LLVM_ABI static BlockAddress *get(Type *Ty, BasicBlock *BB);
 
   /// Lookup an existing \c BlockAddress constant for the given BasicBlock.
   ///
   /// \returns 0 if \c !BB->hasAddressTaken(), otherwise the \c BlockAddress.
-  static BlockAddress *lookup(const BasicBlock *BB);
+  LLVM_ABI static BlockAddress *lookup(const BasicBlock *BB);
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -963,7 +965,7 @@ class DSOLocalEquivalent final : public Constant {
   void operator delete(void *Ptr) { User::operator delete(Ptr); }
 
   /// Return a DSOLocalEquivalent for the specified global value.
-  static DSOLocalEquivalent *get(GlobalValue *GV);
+  LLVM_ABI static DSOLocalEquivalent *get(GlobalValue *GV);
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -1000,7 +1002,7 @@ class NoCFIValue final : public Constant {
 
 public:
   /// Return a NoCFIValue for the specified function.
-  static NoCFIValue *get(GlobalValue *GV);
+  LLVM_ABI static NoCFIValue *get(GlobalValue *GV);
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -1043,12 +1045,12 @@ class ConstantPtrAuth final : public Constant {
 
 public:
   /// Return a pointer signed with the specified parameters.
-  static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key,
-                              ConstantInt *Disc, Constant *AddrDisc);
+  LLVM_ABI static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key,
+                                       ConstantInt *Disc, Constant *AddrDisc);
 
   /// Produce a new ptrauth expression signing the given value using
   /// the same schema as is stored in one.
-  ConstantPtrAuth *getWithSameSchema(Constant *Pointer) const;
+  LLVM_ABI ConstantPtrAuth *getWithSameSchema(Constant *Pointer) const;
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant);
@@ -1087,13 +1089,14 @@ class ConstantPtrAuth final : public Constant {
   /// These discriminators can't be used in real pointer-auth values; they
   /// can only be used in "prototype" values that indicate how some real
   /// schema is supposed to be produced.
-  bool hasSpecialAddressDiscriminator(uint64_t Value) const;
+  LLVM_ABI bool hasSpecialAddressDiscriminator(uint64_t Value) const;
 
   /// Check whether an authentication operation with key \p Key and (possibly
   /// blended) discriminator \p Discriminator is known to be compatible with
   /// this ptrauth signed pointer.
-  bool isKnownCompatibleWith(const Value *Key, const Value *Discriminator,
-                             const DataLayout &DL) const;
+  LLVM_ABI bool isKnownCompatibleWith(const Value *Key,
+                                      const Value *Discriminator,
+                                      const DataLayout &DL) const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
@@ -1138,30 +1141,31 @@ class ConstantExpr : public Constant {
 
   /// getAlignOf constant expr - computes the alignment of a type in a target
   /// independent way (Note: the return type is an i64).
-  static Constant *getAlignOf(Type *Ty);
+  LLVM_ABI static Constant *getAlignOf(Type *Ty);
 
   /// getSizeOf constant expr - computes the (alloc) size of a type (in
   /// address-units, not bits) in a target independent way (Note: the return
   /// type is an i64).
   ///
-  static Constant *getSizeOf(Type *Ty);
-
-  static Constant *getNeg(Constant *C, bool HasNSW = false);
-  static Constant *getNot(Constant *C);
-  static Constant *getAdd(Constant *C1, Constant *C2, bool HasNUW = false,
-                          bool HasNSW = false);
-  static Constant *getSub(Constant *C1, Constant *C2, bool HasNUW = false,
-                          bool HasNSW = false);
-  static Constant *getXor(Constant *C1, Constant *C2);
-  static Constant *getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced = false);
-  static Constant *getPtrToInt(Constant *C, Type *Ty,
-                               bool OnlyIfReduced = false);
-  static Constant *getIntToPtr(Constant *C, Type *Ty,
-                               bool OnlyIfReduced = false);
-  static Constant *getBitCast(Constant *C, Type *Ty,
-                              bool OnlyIfReduced = false);
-  static Constant *getAddrSpaceCast(Constant *C, Type *Ty,
-                                    bool OnlyIfReduced = false);
+  LLVM_ABI static Constant *getSizeOf(Type *Ty);
+
+  LLVM_ABI static Constant *getNeg(Constant *C, bool HasNSW = false);
+  LLVM_ABI static Constant *getNot(Constant *C);
+  LLVM_ABI static Constant *getAdd(Constant *C1, Constant *C2,
+                                   bool HasNUW = false, bool HasNSW = false);
+  LLVM_ABI static Constant *getSub(Constant *C1, Constant *C2,
+                                   bool HasNUW = false, bool HasNSW = false);
+  LLVM_ABI static Constant *getXor(Constant *C1, Constant *C2);
+  LLVM_ABI static Constant *getTrunc(Constant *C, Type *Ty,
+                                     bool OnlyIfReduced = false);
+  LLVM_ABI static Constant *getPtrToInt(Constant *C, Type *Ty,
+                                        bool OnlyIfReduced = false);
+  LLVM_ABI static Constant *getIntToPtr(Constant *C, Type *Ty,
+                                        bool OnlyIfReduced = false);
+  LLVM_ABI static Constant *getBitCast(Constant *C, Type *Ty,
+                                       bool OnlyIfReduced = false);
+  LLVM_ABI static Constant *getAddrSpaceCast(Constant *C, Type *Ty,
+                                             bool OnlyIfReduced = false);
 
   static Constant *getNSWNeg(Constant *C) { return getNeg(C, /*HasNSW=*/true); }
 
@@ -1185,7 +1189,7 @@ class ConstantExpr : public Constant {
   /// function returns a new scalar/fixed width vector obtained from logBase2
   /// of C. Undef vector elements are set to zero.
   /// Return a null pointer otherwise.
-  static Constant *getExactLogBase2(Constant *C);
+  LLVM_ABI static Constant *getExactLogBase2(Constant *C);
 
   /// Return the identity constant for a binary opcode.
   /// If the binop is not commutative, callers can acquire the operand 1
@@ -1194,17 +1198,18 @@ class ConstantExpr : public Constant {
   /// is a fadd/fsub operation and we don't care about signed zeros, then
   /// setting NSZ to true returns the identity +0.0 instead of -0.0. Return
   /// nullptr if the operator does not have an identity constant.
-  static Constant *getBinOpIdentity(unsigned Opcode, Type *Ty,
-                                    bool AllowRHSConstant = false,
-                                    bool NSZ = false);
+  LLVM_ABI static Constant *getBinOpIdentity(unsigned Opcode, Type *Ty,
+                                             bool AllowRHSConstant = false,
+                                             bool NSZ = false);
 
-  static Constant *getIntrinsicIdentity(Intrinsic::ID, Type *Ty);
+  LLVM_ABI static Constant *getIntrinsicIdentity(Intrinsic::ID, Type *Ty);
 
   /// Return the identity constant for a binary or intrinsic Instruction.
   /// The identity constant C is defined as X op C = X and C op X = X where C
   /// and X are the first two operands, and the operation is commutative.
-  static Constant *getIdentity(Instruction *I, Type *Ty,
-                               bool AllowRHSConstant = false, bool NSZ = false);
+  LLVM_ABI static Constant *getIdentity(Instruction *I, Type *Ty,
+                                        bool AllowRHSConstant = false,
+                                        bool NSZ = false);
 
   /// Return the absorbing element for the given binary
   /// operation, i.e. a constant C such that X op C = C and C op X = C for
@@ -1212,8 +1217,8 @@ class ConstantExpr : public Constant {
   /// If AllowLHSConstant is true, the LHS operand is a constant C that must be
   /// defined as C op X = C. It returns null if the operator doesn't have
   /// an absorbing element.
-  static Constant *getBinOpAbsorber(unsigned Opcode, Type *Ty,
-                                    bool AllowLHSConstant = false);
+  LLVM_ABI static Constant *getBinOpAbsorber(unsigned Opcode, Type *Ty,
+                                             bool AllowLHSConstant = false);
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant);
@@ -1224,38 +1229,39 @@ class ConstantExpr : public Constant {
   /// \param C  The constant to be converted
   /// \param Ty The type to which the constant is converted
   /// \param OnlyIfReduced see \a getWithOperands() docs.
-  static Constant *getCast(unsigned ops, Constant *C, Type *Ty,
-                           bool OnlyIfReduced = false);
+  LLVM_ABI static Constant *getCast(unsigned ops, Constant *C, Type *Ty,
+                                    bool OnlyIfReduced = false);
 
   // Create a Trunc or BitCast cast constant expression
-  static Constant *
+  LLVM_ABI static Constant *
   getTruncOrBitCast(Constant *C, ///< The constant to trunc or bitcast
                     Type *Ty     ///< The type to trunc or bitcast C to
   );
 
   /// Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant
   /// expression.
-  static Constant *
+  LLVM_ABI static Constant *
   getPointerCast(Constant *C, ///< The pointer value to be casted (operand 0)
                  Type *Ty     ///< The type to which cast should be made
   );
 
   /// Create a BitCast or AddrSpaceCast for a pointer type depending on
   /// the address space.
-  static Constant *getPointerBitCastOrAddrSpaceCast(
+  LLVM_ABI static Constant *getPointerBitCastOrAddrSpaceCast(
       Constant *C, ///< The constant to addrspacecast or bitcast
       Type *Ty     ///< The type to bitcast or addrspacecast C to
   );
 
   /// Return true if this is a convert constant expression
-  bool isCast() const;
+  LLVM_ABI bool isCast() const;
 
   /// get - Return a binary or shift operator constant expression,
   /// folding if possible.
   ///
   /// \param OnlyIfReducedTy see \a getWithOperands() docs.
-  static Constant *get(unsigned Opcode, Constant *C1, Constant *C2,
-                       unsigned Flags = 0, Type *OnlyIfReducedTy = nullptr);
+  LLVM_ABI static Constant *get(unsigned Opcode, Constant *C1, Constant *C2,
+                                unsigned Flags = 0,
+                                Type *OnlyIfReducedTy = nullptr);
 
   /// Getelementptr form.  Value* is only accepted for convenience;
   /// all elements must be Constants.
@@ -1282,7 +1288,7 @@ class ConstantExpr : public Constant {
     return getGetElementPtr(Ty, C, cast<Value>(Idx), NW, InRange,
                             OnlyIfReducedTy);
   }
-  static Constant *
+  LLVM_ABI static Constant *
   getGetElementPtr(Type *Ty, Constant *C, ArrayRef<Value *> IdxList,
                    GEPNoWrapFlags NW = GEPNoWrapFlags::none(),
                    std::optional<ConstantRange> InRange = std::nullopt,
@@ -1306,29 +1312,30 @@ class ConstantExpr : public Constant {
     return getGetElementPtr(Ty, C, IdxList, GEPNoWrapFlags::inBounds());
   }
 
-  static Constant *getExtractElement(Constant *Vec, Constant *Idx,
-                                     Type *OnlyIfReducedTy = nullptr);
-  static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx,
-                                    Type *OnlyIfReducedTy = nullptr);
-  static Constant *getShuffleVector(Constant *V1, Constant *V2,
-                                    ArrayRef<int> Mask,
-                                    Type *OnlyIfReducedTy = nullptr);
+  LLVM_ABI static Constant *getExtractElement(Constant *Vec, Constant *Idx,
+                                              Type *OnlyIfReducedTy = nullptr);
+  LLVM_ABI static Constant *getInsertElement(Constant *Vec, Constant *Elt,
+                                             Constant *Idx,
+                                             Type *OnlyIfReducedTy = nullptr);
+  LLVM_ABI static Constant *getShuffleVector(Constant *V1, Constant *V2,
+                                             ArrayRef<int> Mask,
+                                             Type *OnlyIfReducedTy = nullptr);
 
   /// Return the opcode at the root of this constant expression
   unsigned getOpcode() const { return getSubclassDataFromValue(); }
 
   /// Assert that this is a shufflevector and return the mask. See class
   /// ShuffleVectorInst for a description of the mask representation.
-  ArrayRef<int> getShuffleMask() const;
+  LLVM_ABI ArrayRef<int> getShuffleMask() const;
 
   /// Assert that this is a shufflevector and return the mask.
   ///
   /// TODO: This is a temporary hack until we update the bitcode format for
   /// shufflevector.
-  Constant *getShuffleMaskForBitcode() const;
+  LLVM_ABI Constant *getShuffleMaskForBitcode() const;
 
   /// Return a string representation for an opcode.
-  const char *getOpcodeName() const;
+  LLVM_ABI const char *getOpcodeName() const;
 
   /// This returns the current constant expression with the operands replaced
   /// with the specified values. The specified array must have the same number
@@ -1346,9 +1353,9 @@ class ConstantExpr : public Constant {
   /// If \c OnlyIfReduced is \c true, nullptr will be returned unless something
   /// gets constant-folded, the type changes, or the expression is otherwise
   /// canonicalized.  This parameter should almost always be \c false.
-  Constant *getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
-                            bool OnlyIfReduced = false,
-                            Type *SrcTy = nullptr) const;
+  LLVM_ABI Constant *getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
+                                     bool OnlyIfReduced = false,
+                                     Type *SrcTy = nullptr) const;
 
   /// Returns an Instruction which implements the same operation as this
   /// ConstantExpr. It is not inserted into any basic block.
@@ -1357,21 +1364,21 @@ class ConstantExpr : public Constant {
   /// which would take a ConstantExpr parameter, but that would have spread
   /// implementation details of ConstantExpr outside of Constants.cpp, which
   /// would make it harder to remove ConstantExprs altogether.
-  Instruction *getAsInstruction() const;
+  LLVM_ABI Instruction *getAsInstruction() const;
 
   /// Whether creating a constant expression for this binary operator is
   /// desirable.
-  static bool isDesirableBinOp(unsigned Opcode);
+  LLVM_ABI static bool isDesirableBinOp(unsigned Opcode);
 
   /// Whether creating a constant expression for this binary operator is
   /// supported.
-  static bool isSupportedBinOp(unsigned Opcode);
+  LLVM_ABI static bool isSupportedBinOp(unsigned Opcode);
 
   /// Whether creating a constant expression for this cast is desirable.
-  static bool isDesirableCastOp(unsigned Opcode);
+  LLVM_ABI static bool isDesirableCastOp(unsigned Opcode);
 
   /// Whether creating a constant expression for this cast is supported.
-  static bool isSupportedCastOp(unsigned Opcode);
+  LLVM_ABI static bool isSupportedCastOp(unsigned Opcode);
 
   /// Whether creating a constant expression for this getelementptr type is
   /// supported.
@@ -1422,25 +1429,25 @@ class UndefValue : public ConstantData {
   UndefValue(const UndefValue &) = delete;
 
   /// Static factory methods - Return an 'undef' object of the specified type.
-  static UndefValue *get(Type *T);
+  LLVM_ABI static UndefValue *get(Type *T);
 
   /// If this Undef has array or vector type, return a undef with the right
   /// element type.
-  UndefValue *getSequentialElement() const;
+  LLVM_ABI UndefValue *getSequentialElement() const;
 
   /// If this undef has struct type, return a undef with the right element type
   /// for the specified element.
-  UndefValue *getStructElement(unsigned Elt) const;
+  LLVM_ABI UndefValue *getStructElement(unsigned Elt) const;
 
   /// Return an undef of the right value for the specified GEP index if we can,
   /// otherwise return null (e.g. if C is a ConstantExpr).
-  UndefValue *getElementValue(Constant *C) const;
+  LLVM_ABI UndefValue *getElementValue(Constant *C) const;
 
   /// Return an undef of the right value for the specified GEP index.
-  UndefValue *getElementValue(unsigned Idx) const;
+  LLVM_ABI UndefValue *getElementValue(unsigned Idx) const;
 
   /// Return the number of elements in the array, vector, or struct.
-  unsigned getNumElements() const;
+  LLVM_ABI unsigned getNumElements() const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
@@ -1467,22 +1474,22 @@ class PoisonValue final : public UndefValue {
   PoisonValue(const PoisonValue &) = delete;
 
   /// Static factory methods - Return an 'poison' object of the specified type.
-  static PoisonValue *get(Type *T);
+  LLVM_ABI static PoisonValue *get(Type *T);
 
   /// If this poison has array or vector type, return a poison with the right
   /// element type.
-  PoisonValue *getSequentialElement() const;
+  LLVM_ABI PoisonValue *getSequentialElement() const;
 
   /// If this poison has struct type, return a poison with the right element
   /// type for the specified element.
-  PoisonValue *getStructElement(unsigned Elt) const;
+  LLVM_ABI PoisonValue *getStructElement(unsigned Elt) const;
 
   /// Return an poison of the right value for the specified GEP index if we can,
   /// otherwise return null (e.g. if C is a ConstantExpr).
-  PoisonValue *getElementValue(Constant *C) const;
+  LLVM_ABI PoisonValue *getElementValue(Constant *C) const;
 
   /// Return an poison of the right value for the specified GEP index.
-  PoisonValue *getElementValue(unsigned Idx) const;
+  LLVM_ABI PoisonValue *getElementValue(unsigned Idx) const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/DIBuilder.h b/external/llvm-project/llvm/include/llvm/IR/DIBuilder.h
index de987472bf6a..ca818c8d502d 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DIBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DIBuilder.h
@@ -24,6 +24,7 @@
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/TrackingMDRef.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include <algorithm>
 #include <cstdint>
 #include <optional>
@@ -46,11 +47,7 @@ namespace llvm {
     Module &M;
     LLVMContext &VMContext;
 
-    DICompileUnit *CUNode;   ///< The one compile unit created by this DIBuiler.
-    Function *DeclareFn;     ///< llvm.dbg.declare
-    Function *ValueFn;       ///< llvm.dbg.value
-    Function *LabelFn;       ///< llvm.dbg.label
-    Function *AssignFn;      ///< llvm.dbg.assign
+    DICompileUnit *CUNode; ///< The one compile unit created by this DIBuiler.
 
     SmallVector<TrackingMDNodeRef, 4> AllEnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
@@ -115,17 +112,17 @@ namespace llvm {
     /// in order to resolve cycles during \a finalize().
     ///
     /// If \p CU is given a value other than nullptr, then set \p CUNode to CU.
-    explicit DIBuilder(Module &M, bool AllowUnresolved = true,
-                       DICompileUnit *CU = nullptr);
+    LLVM_ABI explicit DIBuilder(Module &M, bool AllowUnresolved = true,
+                                DICompileUnit *CU = nullptr);
     DIBuilder(const DIBuilder &) = delete;
     DIBuilder &operator=(const DIBuilder &) = delete;
 
     /// Construct any deferred debug info descriptors.
-    void finalize();
+    LLVM_ABI void finalize();
 
     /// Finalize a specific subprogram - no new variables may be added to this
     /// subprogram afterwards.
-    void finalizeSubprogram(DISubprogram *SP);
+    LLVM_ABI void finalizeSubprogram(DISubprogram *SP);
 
     /// A CompileUnit provides an anchor for all debugging
     /// information generated during this instance of compilation.
@@ -154,7 +151,7 @@ namespace llvm {
     /// \param SysRoot       The clang system root (value of -isysroot).
     /// \param SDK           The SDK name. On Darwin, this is the last component
     ///                      of the sysroot.
-    DICompileUnit *
+    LLVM_ABI DICompileUnit *
     createCompileUnit(unsigned Lang, DIFile *File, StringRef Producer,
                       bool isOptimized, StringRef Flags, unsigned RV,
                       StringRef SplitName = StringRef(),
@@ -173,7 +170,7 @@ namespace llvm {
     /// \param Checksum  Optional checksum kind (e.g. CSK_MD5, CSK_SHA1, etc.)
     ///                  and value.
     /// \param Source    Optional source text.
-    DIFile *createFile(
+    LLVM_ABI DIFile *createFile(
         StringRef Filename, StringRef Directory,
         std::optional<DIFile::ChecksumInfo<StringRef>> Checksum = std::nullopt,
         std::optional<StringRef> Source = std::nullopt);
@@ -184,8 +181,9 @@ namespace llvm {
     /// \param MacroType  DW_MACINFO_define or DW_MACINFO_undef.
     /// \param Name       Macro name.
     /// \param Value      Macro value.
-    DIMacro *createMacro(DIMacroFile *Parent, unsigned Line, unsigned MacroType,
-                         StringRef Name, StringRef Value = StringRef());
+    LLVM_ABI DIMacro *createMacro(DIMacroFile *Parent, unsigned Line,
+                                  unsigned MacroType, StringRef Name,
+                                  StringRef Value = StringRef());
 
     /// Create debugging information temporary entry for a macro file.
     /// List of macro node direct children will be calculated by DIBuilder,
@@ -193,19 +191,20 @@ namespace llvm {
     /// \param Parent     Macro file parent (could be nullptr).
     /// \param Line       Source line number where the macro file is included.
     /// \param File       File descriptor containing the name of the macro file.
-    DIMacroFile *createTempMacroFile(DIMacroFile *Parent, unsigned Line,
-                                     DIFile *File);
+    LLVM_ABI DIMacroFile *createTempMacroFile(DIMacroFile *Parent,
+                                              unsigned Line, DIFile *File);
 
     /// Create a single enumerator value.
-    DIEnumerator *createEnumerator(StringRef Name, const APSInt &Value);
-    DIEnumerator *createEnumerator(StringRef Name, uint64_t Val,
-                                   bool IsUnsigned = false);
+    LLVM_ABI DIEnumerator *createEnumerator(StringRef Name,
+                                            const APSInt &Value);
+    LLVM_ABI DIEnumerator *createEnumerator(StringRef Name, uint64_t Val,
+                                            bool IsUnsigned = false);
 
     /// Create a DWARF unspecified type.
-    DIBasicType *createUnspecifiedType(StringRef Name);
+    LLVM_ABI DIBasicType *createUnspecifiedType(StringRef Name);
 
     /// Create C++11 nullptr type.
-    DIBasicType *createNullPtrType();
+    LLVM_ABI DIBasicType *createNullPtrType();
 
     /// Create debugging information entry for a basic
     /// type.
@@ -216,10 +215,10 @@ namespace llvm {
     /// \param NumExtraInhabitants The number of extra inhabitants of the type.
     /// An extra inhabitant is a bit pattern that does not represent a valid
     /// value for instances of a given type. This is used by the Swift language.
-    DIBasicType *createBasicType(StringRef Name, uint64_t SizeInBits,
-                                 unsigned Encoding,
-                                 DINode::DIFlags Flags = DINode::FlagZero,
-                                 uint32_t NumExtraInhabitants = 0);
+    LLVM_ABI DIBasicType *
+    createBasicType(StringRef Name, uint64_t SizeInBits, unsigned Encoding,
+                    DINode::DIFlags Flags = DINode::FlagZero,
+                    uint32_t NumExtraInhabitants = 0);
 
     /// Create debugging information entry for a binary fixed-point type.
     /// \param Name        Type name.
@@ -227,7 +226,7 @@ namespace llvm {
     ///                    dwarf::DW_ATE_signed_fixed or DW_ATE_unsigned_fixed.
     /// \param Flags       Optional DWARF attributes, e.g., DW_AT_endianity.
     /// \param Factor      Binary scale factor.
-    DIFixedPointType *
+    LLVM_ABI DIFixedPointType *
     createBinaryFixedPointType(StringRef Name, uint64_t SizeInBits,
                                uint32_t AlignInBits, unsigned Encoding,
                                DINode::DIFlags Flags, int Factor);
@@ -238,7 +237,7 @@ namespace llvm {
     ///                    dwarf::DW_ATE_signed_fixed or DW_ATE_unsigned_fixed.
     /// \param Flags       Optional DWARF attributes, e.g., DW_AT_endianity.
     /// \param Factor      Decimal scale factor.
-    DIFixedPointType *
+    LLVM_ABI DIFixedPointType *
     createDecimalFixedPointType(StringRef Name, uint64_t SizeInBits,
                                 uint32_t AlignInBits, unsigned Encoding,
                                 DINode::DIFlags Flags, int Factor);
@@ -251,7 +250,7 @@ namespace llvm {
     /// \param Flags       Optional DWARF attributes, e.g., DW_AT_endianity.
     /// \param Numerator   Numerator of scale factor.
     /// \param Denominator Denominator of scale factor.
-    DIFixedPointType *
+    LLVM_ABI DIFixedPointType *
     createRationalFixedPointType(StringRef Name, uint64_t SizeInBits,
                                  uint32_t AlignInBits, unsigned Encoding,
                                  DINode::DIFlags Flags, APInt Numerator,
@@ -261,30 +260,32 @@ namespace llvm {
     /// type.
     /// \param Name        Type name.
     /// \param SizeInBits  Size of the type.
-    DIStringType *createStringType(StringRef Name, uint64_t SizeInBits);
+    LLVM_ABI DIStringType *createStringType(StringRef Name,
+                                            uint64_t SizeInBits);
 
     /// Create debugging information entry for Fortran
     /// assumed length string type.
     /// \param Name            Type name.
     /// \param StringLength    String length expressed as DIVariable *.
     /// \param StrLocationExp  Optional memory location of the string.
-    DIStringType *createStringType(StringRef Name, DIVariable *StringLength,
-                                   DIExpression *StrLocationExp = nullptr);
+    LLVM_ABI DIStringType *
+    createStringType(StringRef Name, DIVariable *StringLength,
+                     DIExpression *StrLocationExp = nullptr);
 
     /// Create debugging information entry for Fortran
     /// assumed length string type.
     /// \param Name             Type name.
     /// \param StringLengthExp  String length expressed in DIExpression form.
     /// \param StrLocationExp   Optional memory location of the string.
-    DIStringType *createStringType(StringRef Name,
-                                   DIExpression *StringLengthExp,
-                                   DIExpression *StrLocationExp = nullptr);
+    LLVM_ABI DIStringType *
+    createStringType(StringRef Name, DIExpression *StringLengthExp,
+                     DIExpression *StrLocationExp = nullptr);
 
     /// Create debugging information entry for a qualified
     /// type, e.g. 'const int'.
     /// \param Tag         Tag identifing type, e.g. dwarf::TAG_volatile_type
     /// \param FromTy      Base Type.
-    DIDerivedType *createQualifiedType(unsigned Tag, DIType *FromTy);
+    LLVM_ABI DIDerivedType *createQualifiedType(unsigned Tag, DIType *FromTy);
 
     /// Create debugging information entry for a pointer.
     /// \param PointeeTy         Type pointed by this pointer.
@@ -294,32 +295,33 @@ namespace llvm {
     /// \param DWARFMemorySpace  DWARF memory space. (optional)
     /// \param Name              Pointer type name. (optional)
     /// \param Annotations       Member annotations.
-    DIDerivedType *createPointerType(
-        DIType *PointeeTy, uint64_t SizeInBits, uint32_t AlignInBits = 0,
-        std::optional<unsigned> DWARFAddressSpace = std::nullopt,
-        dwarf::MemorySpace DWARFMemorySpace = dwarf::DW_MSPACE_LLVM_none,
-        StringRef Name = "", DINodeArray Annotations = nullptr);
+    LLVM_ABI DIDerivedType *
+    createPointerType(DIType *PointeeTy, uint64_t SizeInBits,
+                      uint32_t AlignInBits = 0,
+                      std::optional<unsigned> DWARFAddressSpace = std::nullopt,
+                      dwarf::MemorySpace DWARFMemorySpace = dwarf::DW_MSPACE_LLVM_none,
+                      StringRef Name = "", DINodeArray Annotations = nullptr);
 
     /// Create a __ptrauth qualifier.
-    DIDerivedType *createPtrAuthQualifiedType(DIType *FromTy, unsigned Key,
-                                              bool IsAddressDiscriminated,
-                                              unsigned ExtraDiscriminator,
-                                              bool IsaPointer,
-                                              bool authenticatesNullValues);
+    LLVM_ABI DIDerivedType *
+    createPtrAuthQualifiedType(DIType *FromTy, unsigned Key,
+                               bool IsAddressDiscriminated,
+                               unsigned ExtraDiscriminator, bool IsaPointer,
+                               bool authenticatesNullValues);
 
     /// Create debugging information entry for a pointer to member.
     /// \param PointeeTy Type pointed to by this pointer.
     /// \param SizeInBits  Size.
     /// \param AlignInBits Alignment. (optional)
     /// \param Class Type for which this pointer points to members of.
-    DIDerivedType *
+    LLVM_ABI DIDerivedType *
     createMemberPointerType(DIType *PointeeTy, DIType *Class,
                             uint64_t SizeInBits, uint32_t AlignInBits = 0,
                             DINode::DIFlags Flags = DINode::FlagZero);
 
     /// Create debugging information entry for a c++
     /// style reference or rvalue reference type.
-    DIDerivedType *createReferenceType(
+    LLVM_ABI DIDerivedType *createReferenceType(
         unsigned Tag, DIType *RTy, uint64_t SizeInBits = 0,
         uint32_t AlignInBits = 0,
         std::optional<unsigned> DWARFAddressSpace = std::nullopt,
@@ -334,11 +336,11 @@ namespace llvm {
     /// \param AlignInBits Alignment. (optional)
     /// \param Flags       Flags to describe inheritance attribute, e.g. private
     /// \param Annotations Annotations. (optional)
-    DIDerivedType *createTypedef(DIType *Ty, StringRef Name, DIFile *File,
-                                 unsigned LineNo, DIScope *Context,
-                                 uint32_t AlignInBits = 0,
-                                 DINode::DIFlags Flags = DINode::FlagZero,
-                                 DINodeArray Annotations = nullptr);
+    LLVM_ABI DIDerivedType *
+    createTypedef(DIType *Ty, StringRef Name, DIFile *File, unsigned LineNo,
+                  DIScope *Context, uint32_t AlignInBits = 0,
+                  DINode::DIFlags Flags = DINode::FlagZero,
+                  DINodeArray Annotations = nullptr);
 
     /// Create debugging information entry for a template alias.
     /// \param Ty          Original type.
@@ -351,15 +353,15 @@ namespace llvm {
     /// \param Flags       Flags to describe inheritance attribute (optional),
     ///                    e.g. private.
     /// \param Annotations Annotations. (optional)
-    DIDerivedType *createTemplateAlias(DIType *Ty, StringRef Name, DIFile *File,
-                                       unsigned LineNo, DIScope *Context,
-                                       DINodeArray TParams,
-                                       uint32_t AlignInBits = 0,
-                                       DINode::DIFlags Flags = DINode::FlagZero,
-                                       DINodeArray Annotations = nullptr);
+    LLVM_ABI DIDerivedType *
+    createTemplateAlias(DIType *Ty, StringRef Name, DIFile *File,
+                        unsigned LineNo, DIScope *Context, DINodeArray TParams,
+                        uint32_t AlignInBits = 0,
+                        DINode::DIFlags Flags = DINode::FlagZero,
+                        DINodeArray Annotations = nullptr);
 
     /// Create debugging information entry for a 'friend'.
-    DIDerivedType *createFriend(DIType *Ty, DIType *FriendTy);
+    LLVM_ABI DIDerivedType *createFriend(DIType *Ty, DIType *FriendTy);
 
     /// Create debugging information entry to establish
     /// inheritance relationship between two types.
@@ -369,9 +371,10 @@ namespace llvm {
     /// \param VBPtrOffset  Virtual base pointer offset.
     /// \param Flags        Flags to describe inheritance attribute,
     ///                     e.g. private
-    DIDerivedType *createInheritance(DIType *Ty, DIType *BaseTy,
-                                     uint64_t BaseOffset, uint32_t VBPtrOffset,
-                                     DINode::DIFlags Flags);
+    LLVM_ABI DIDerivedType *createInheritance(DIType *Ty, DIType *BaseTy,
+                                              uint64_t BaseOffset,
+                                              uint32_t VBPtrOffset,
+                                              DINode::DIFlags Flags);
 
     /// Create debugging information entry for a member.
     /// \param Scope        Member scope.
@@ -384,12 +387,11 @@ namespace llvm {
     /// \param Flags        Flags to encode member attribute, e.g. private
     /// \param Ty           Parent type.
     /// \param Annotations  Member annotations.
-    DIDerivedType *createMemberType(DIScope *Scope, StringRef Name,
-                                    DIFile *File, unsigned LineNo,
-                                    uint64_t SizeInBits, uint32_t AlignInBits,
-                                    uint64_t OffsetInBits,
-                                    DINode::DIFlags Flags, DIType *Ty,
-                                    DINodeArray Annotations = nullptr);
+    LLVM_ABI DIDerivedType *
+    createMemberType(DIScope *Scope, StringRef Name, DIFile *File,
+                     unsigned LineNo, uint64_t SizeInBits, uint32_t AlignInBits,
+                     uint64_t OffsetInBits, DINode::DIFlags Flags, DIType *Ty,
+                     DINodeArray Annotations = nullptr);
 
     /// Create debugging information entry for a variant.  A variant
     /// normally should be a member of a variant part.
@@ -406,13 +408,10 @@ namespace llvm {
     ///                     ConstantDataArray if the variant applies
     ///                     for multiple discriminants.
     /// \param Ty           Parent type.
-    DIDerivedType *createVariantMemberType(DIScope *Scope, StringRef Name,
-					   DIFile *File, unsigned LineNo,
-					   uint64_t SizeInBits,
-					   uint32_t AlignInBits,
-					   uint64_t OffsetInBits,
-					   Constant *Discriminant,
-					   DINode::DIFlags Flags, DIType *Ty);
+    LLVM_ABI DIDerivedType *createVariantMemberType(
+        DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo,
+        uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+        Constant *Discriminant, DINode::DIFlags Flags, DIType *Ty);
 
     /// Create debugging information entry for a variant.  A variant
     /// created this way "inlines" multiple members into the enclosing
@@ -424,8 +423,10 @@ namespace llvm {
     ///                     ConstantDataArray if the variant applies
     ///                     for multiple discriminants.
     /// \param Ty           Parent type.
-    DIDerivedType *createVariantMemberType(DIScope *Scope, DINodeArray Elements,
-                                           Constant *Discriminant, DIType *Ty);
+    LLVM_ABI DIDerivedType *createVariantMemberType(DIScope *Scope,
+                                                    DINodeArray Elements,
+                                                    Constant *Discriminant,
+                                                    DIType *Ty);
 
     /// Create debugging information entry for a bit field member.
     /// \param Scope               Member scope.
@@ -438,13 +439,11 @@ namespace llvm {
     /// \param Flags               Flags to encode member attribute.
     /// \param Ty                  Parent type.
     /// \param Annotations         Member annotations.
-    DIDerivedType *createBitFieldMemberType(DIScope *Scope, StringRef Name,
-                                            DIFile *File, unsigned LineNo,
-                                            uint64_t SizeInBits,
-                                            uint64_t OffsetInBits,
-                                            uint64_t StorageOffsetInBits,
-                                            DINode::DIFlags Flags, DIType *Ty,
-                                            DINodeArray Annotations = nullptr);
+    LLVM_ABI DIDerivedType *createBitFieldMemberType(
+        DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo,
+        uint64_t SizeInBits, uint64_t OffsetInBits,
+        uint64_t StorageOffsetInBits, DINode::DIFlags Flags, DIType *Ty,
+        DINodeArray Annotations = nullptr);
 
     /// Create debugging information entry for a
     /// C++ static data member.
@@ -457,11 +456,12 @@ namespace llvm {
     /// \param Val        Const initializer of the member.
     /// \param Tag        DWARF tag of the static member.
     /// \param AlignInBits  Member alignment.
-    DIDerivedType *createStaticMemberType(DIScope *Scope, StringRef Name,
-                                          DIFile *File, unsigned LineNo,
-                                          DIType *Ty, DINode::DIFlags Flags,
-                                          Constant *Val, unsigned Tag,
-                                          uint32_t AlignInBits = 0);
+    LLVM_ABI DIDerivedType *createStaticMemberType(DIScope *Scope,
+                                                   StringRef Name, DIFile *File,
+                                                   unsigned LineNo, DIType *Ty,
+                                                   DINode::DIFlags Flags,
+                                                   Constant *Val, unsigned Tag,
+                                                   uint32_t AlignInBits = 0);
 
     /// Create debugging information entry for Objective-C
     /// instance variable.
@@ -474,10 +474,12 @@ namespace llvm {
     /// \param Flags        Flags to encode member attribute, e.g. private
     /// \param Ty           Parent type.
     /// \param PropertyNode Property associated with this ivar.
-    DIDerivedType *createObjCIVar(StringRef Name, DIFile *File, unsigned LineNo,
-                                  uint64_t SizeInBits, uint32_t AlignInBits,
-                                  uint64_t OffsetInBits, DINode::DIFlags Flags,
-                                  DIType *Ty, MDNode *PropertyNode);
+    LLVM_ABI DIDerivedType *createObjCIVar(StringRef Name, DIFile *File,
+                                           unsigned LineNo, uint64_t SizeInBits,
+                                           uint32_t AlignInBits,
+                                           uint64_t OffsetInBits,
+                                           DINode::DIFlags Flags, DIType *Ty,
+                                           MDNode *PropertyNode);
 
     /// Create debugging information entry for Objective-C
     /// property.
@@ -488,11 +490,10 @@ namespace llvm {
     /// \param SetterName   Name of the Objective C property setter selector.
     /// \param PropertyAttributes Objective C property attributes.
     /// \param Ty           Type.
-    DIObjCProperty *createObjCProperty(StringRef Name, DIFile *File,
-                                       unsigned LineNumber,
-                                       StringRef GetterName,
-                                       StringRef SetterName,
-                                       unsigned PropertyAttributes, DIType *Ty);
+    LLVM_ABI DIObjCProperty *
+    createObjCProperty(StringRef Name, DIFile *File, unsigned LineNumber,
+                       StringRef GetterName, StringRef SetterName,
+                       unsigned PropertyAttributes, DIType *Ty);
 
     /// Create debugging information entry for a class.
     /// \param Scope        Scope in which this class is defined.
@@ -511,7 +512,7 @@ namespace llvm {
     ///                     for more info.
     /// \param TemplateParms Template type parameters.
     /// \param UniqueIdentifier A unique identifier for the class.
-    DICompositeType *createClassType(
+    LLVM_ABI DICompositeType *createClassType(
         DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
         uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
         DINode::DIFlags Flags, DIType *DerivedFrom, DINodeArray Elements,
@@ -534,7 +535,7 @@ namespace llvm {
     /// \param NumExtraInhabitants The number of extra inhabitants of the type.
     /// An extra inhabitant is a bit pattern that does not represent a valid
     /// value for instances of a given type. This is used by the Swift language.
-    DICompositeType *createStructType(
+    LLVM_ABI DICompositeType *createStructType(
         DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
         uint64_t SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags,
         DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang = 0,
@@ -552,13 +553,12 @@ namespace llvm {
     /// \param Elements     Union elements.
     /// \param RunTimeLang  Optional parameter, Objective-C runtime version.
     /// \param UniqueIdentifier A unique identifier for the union.
-    DICompositeType *createUnionType(DIScope *Scope, StringRef Name,
-                                     DIFile *File, unsigned LineNumber,
-                                     uint64_t SizeInBits, uint32_t AlignInBits,
-                                     DINode::DIFlags Flags,
-                                     DINodeArray Elements,
-                                     unsigned RunTimeLang = 0,
-                                     StringRef UniqueIdentifier = "");
+    LLVM_ABI DICompositeType *
+    createUnionType(DIScope *Scope, StringRef Name, DIFile *File,
+                    unsigned LineNumber, uint64_t SizeInBits,
+                    uint32_t AlignInBits, DINode::DIFlags Flags,
+                    DINodeArray Elements, unsigned RunTimeLang = 0,
+                    StringRef UniqueIdentifier = "");
 
     /// Create debugging information entry for a variant part.  A
     /// variant part normally has a discriminator (though this is not
@@ -573,13 +573,12 @@ namespace llvm {
     /// \param Discriminator Discriminant member
     /// \param Elements     Variant elements.
     /// \param UniqueIdentifier A unique identifier for the union.
-    DICompositeType *createVariantPart(DIScope *Scope, StringRef Name,
-				       DIFile *File, unsigned LineNumber,
-				       uint64_t SizeInBits, uint32_t AlignInBits,
-				       DINode::DIFlags Flags,
-				       DIDerivedType *Discriminator,
-				       DINodeArray Elements,
-				       StringRef UniqueIdentifier = "");
+    LLVM_ABI DICompositeType *
+    createVariantPart(DIScope *Scope, StringRef Name, DIFile *File,
+                      unsigned LineNumber, uint64_t SizeInBits,
+                      uint32_t AlignInBits, DINode::DIFlags Flags,
+                      DIDerivedType *Discriminator, DINodeArray Elements,
+                      StringRef UniqueIdentifier = "");
 
     /// Create debugging information for template
     /// type parameter.
@@ -587,10 +586,9 @@ namespace llvm {
     /// \param Name         Type parameter name.
     /// \param Ty           Parameter type.
     /// \param IsDefault    Parameter is default or not
-    DITemplateTypeParameter *createTemplateTypeParameter(DIScope *Scope,
-                                                         StringRef Name,
-                                                         DIType *Ty,
-                                                         bool IsDefault);
+    LLVM_ABI DITemplateTypeParameter *
+    createTemplateTypeParameter(DIScope *Scope, StringRef Name, DIType *Ty,
+                                bool IsDefault);
 
     /// Create debugging information for template
     /// value parameter.
@@ -599,7 +597,7 @@ namespace llvm {
     /// \param Ty           Parameter type.
     /// \param IsDefault    Parameter is default or not
     /// \param Val          Constant parameter value.
-    DITemplateValueParameter *
+    LLVM_ABI DITemplateValueParameter *
     createTemplateValueParameter(DIScope *Scope, StringRef Name, DIType *Ty,
                                  bool IsDefault, Constant *Val);
 
@@ -609,7 +607,7 @@ namespace llvm {
     /// \param Ty           Parameter type.
     /// \param Val          The fully qualified name of the template.
     /// \param IsDefault    Parameter is default or not.
-    DITemplateValueParameter *
+    LLVM_ABI DITemplateValueParameter *
     createTemplateTemplateParameter(DIScope *Scope, StringRef Name, DIType *Ty,
                                     StringRef Val, bool IsDefault = false);
 
@@ -618,10 +616,9 @@ namespace llvm {
     /// \param Name         Value parameter name.
     /// \param Ty           Parameter type.
     /// \param Val          An array of types in the pack.
-    DITemplateValueParameter *createTemplateParameterPack(DIScope *Scope,
-                                                          StringRef Name,
-                                                          DIType *Ty,
-                                                          DINodeArray Val);
+    LLVM_ABI DITemplateValueParameter *
+    createTemplateParameterPack(DIScope *Scope, StringRef Name, DIType *Ty,
+                                DINodeArray Val);
 
     /// Create debugging information entry for an array.
     /// \param Size         Array size.
@@ -640,7 +637,7 @@ namespace llvm {
     /// \param Rank         The rank attribute of a descriptor-based
     ///                     Fortran array, either a DIExpression* or
     ///                     a DIVariable*.
-    DICompositeType *createArrayType(
+    LLVM_ABI DICompositeType *createArrayType(
         uint64_t Size, uint32_t AlignInBits, DIType *Ty, DINodeArray Subscripts,
         PointerUnion<DIExpression *, DIVariable *> DataLocation = nullptr,
         PointerUnion<DIExpression *, DIVariable *> Associated = nullptr,
@@ -669,7 +666,7 @@ namespace llvm {
     ///                       Fortran array, either a DIExpression* or
     ///                       a DIVariable*.
     /// \param BitStride      The bit size of an element of the array.
-    DICompositeType *createArrayType(
+    LLVM_ABI DICompositeType *createArrayType(
         DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
         uint64_t Size, uint32_t AlignInBits, DIType *Ty, DINodeArray Subscripts,
         PointerUnion<DIExpression *, DIVariable *> DataLocation = nullptr,
@@ -683,8 +680,9 @@ namespace llvm {
     /// \param AlignInBits  Alignment.
     /// \param Ty           Element type.
     /// \param Subscripts   Subscripts.
-    DICompositeType *createVectorType(uint64_t Size, uint32_t AlignInBits,
-                                      DIType *Ty, DINodeArray Subscripts);
+    LLVM_ABI DICompositeType *createVectorType(uint64_t Size,
+                                               uint32_t AlignInBits, DIType *Ty,
+                                               DINodeArray Subscripts);
 
     /// Create debugging information entry for an
     /// enumeration.
@@ -700,7 +698,7 @@ namespace llvm {
     /// \param UniqueIdentifier A unique identifier for the enum.
     /// \param IsScoped Boolean flag indicate if this is C++11/ObjC 'enum
     /// class'.
-    DICompositeType *createEnumerationType(
+    LLVM_ABI DICompositeType *createEnumerationType(
         DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
         uint64_t SizeInBits, uint32_t AlignInBits, DINodeArray Elements,
         DIType *UnderlyingType, unsigned RunTimeLang = 0,
@@ -714,9 +712,10 @@ namespace llvm {
     /// \param SizeInBits     Set size.
     /// \param AlignInBits    Set alignment.
     /// \param Ty             Base type of the set.
-    DIDerivedType *createSetType(DIScope *Scope, StringRef Name, DIFile *File,
-                                 unsigned LineNo, uint64_t SizeInBits,
-                                 uint32_t AlignInBits, DIType *Ty);
+    LLVM_ABI DIDerivedType *createSetType(DIScope *Scope, StringRef Name,
+                                          DIFile *File, unsigned LineNo,
+                                          uint64_t SizeInBits,
+                                          uint32_t AlignInBits, DIType *Ty);
 
     /// Create subroutine type.
     /// \param ParameterTypes  An array of subroutine parameter types. This
@@ -724,20 +723,20 @@ namespace llvm {
     /// \param Flags           E.g.: LValueReference.
     ///                        These flags are used to emit dwarf attributes.
     /// \param CC              Calling convention, e.g. dwarf::DW_CC_normal
-    DISubroutineType *
+    LLVM_ABI DISubroutineType *
     createSubroutineType(DITypeRefArray ParameterTypes,
                          DINode::DIFlags Flags = DINode::FlagZero,
                          unsigned CC = 0);
 
     /// Create a distinct clone of \p SP with FlagArtificial set.
-    static DISubprogram *createArtificialSubprogram(DISubprogram *SP);
+    LLVM_ABI static DISubprogram *createArtificialSubprogram(DISubprogram *SP);
 
     /// Create a uniqued clone of \p Ty with FlagArtificial set.
-    static DIType *createArtificialType(DIType *Ty);
+    LLVM_ABI static DIType *createArtificialType(DIType *Ty);
 
     /// Create a uniqued clone of \p Ty with FlagObjectPointer set.
     /// If \p Implicit is true, also set FlagArtificial.
-    static DIType *createObjectPointerType(DIType *Ty, bool Implicit);
+    LLVM_ABI static DIType *createObjectPointerType(DIType *Ty, bool Implicit);
 
     /// Create a type describing a subrange of another type.
     /// \param Scope          Scope in which this set is defined.
@@ -752,7 +751,7 @@ namespace llvm {
     /// \param UpperBound     Upper bound.
     /// \param Stride         Stride, if any.
     /// \param Bias           Bias, if any.
-    DISubrangeType *
+    LLVM_ABI DISubrangeType *
     createSubrangeType(StringRef Name, DIFile *File, unsigned LineNo,
                        DIScope *Scope, uint64_t SizeInBits,
                        uint32_t AlignInBits, DINode::DIFlags Flags, DIType *Ty,
@@ -760,7 +759,7 @@ namespace llvm {
                        Metadata *Stride, Metadata *Bias);
 
     /// Create a permanent forward-declared type.
-    DICompositeType *
+    LLVM_ABI DICompositeType *
     createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F,
                       unsigned Line, unsigned RuntimeLang = 0,
                       uint64_t SizeInBits = 0, uint32_t AlignInBits = 0,
@@ -768,7 +767,7 @@ namespace llvm {
                       std::optional<uint32_t> EnumKind = std::nullopt);
 
     /// Create a temporary forward-declared type.
-    DICompositeType *createReplaceableCompositeType(
+    LLVM_ABI DICompositeType *createReplaceableCompositeType(
         unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line,
         unsigned RuntimeLang = 0, uint64_t SizeInBits = 0,
         uint32_t AlignInBits = 0, DINode::DIFlags Flags = DINode::FlagFwdDecl,
@@ -777,29 +776,32 @@ namespace llvm {
 
     /// Retain DIScope* in a module even if it is not referenced
     /// through debug info anchors.
-    void retainType(DIScope *T);
+    LLVM_ABI void retainType(DIScope *T);
 
     /// Create unspecified parameter type
     /// for a subroutine type.
-    DIBasicType *createUnspecifiedParameter();
+    LLVM_ABI DIBasicType *createUnspecifiedParameter();
 
     /// Get a DINodeArray, create one if required.
-    DINodeArray getOrCreateArray(ArrayRef<Metadata *> Elements);
+    LLVM_ABI DINodeArray getOrCreateArray(ArrayRef<Metadata *> Elements);
 
     /// Get a DIMacroNodeArray, create one if required.
-    DIMacroNodeArray getOrCreateMacroArray(ArrayRef<Metadata *> Elements);
+    LLVM_ABI DIMacroNodeArray
+    getOrCreateMacroArray(ArrayRef<Metadata *> Elements);
 
     /// Get a DITypeRefArray, create one if required.
-    DITypeRefArray getOrCreateTypeArray(ArrayRef<Metadata *> Elements);
+    LLVM_ABI DITypeRefArray getOrCreateTypeArray(ArrayRef<Metadata *> Elements);
 
     /// Create a descriptor for a value range.  This
     /// implicitly uniques the values returned.
-    DISubrange *getOrCreateSubrange(int64_t Lo, int64_t Count);
-    DISubrange *getOrCreateSubrange(int64_t Lo, Metadata *CountNode);
-    DISubrange *getOrCreateSubrange(Metadata *Count, Metadata *LowerBound,
-                                    Metadata *UpperBound, Metadata *Stride);
-
-    DIGenericSubrange *
+    LLVM_ABI DISubrange *getOrCreateSubrange(int64_t Lo, int64_t Count);
+    LLVM_ABI DISubrange *getOrCreateSubrange(int64_t Lo, Metadata *CountNode);
+    LLVM_ABI DISubrange *getOrCreateSubrange(Metadata *Count,
+                                             Metadata *LowerBound,
+                                             Metadata *UpperBound,
+                                             Metadata *Stride);
+
+    LLVM_ABI DIGenericSubrange *
     getOrCreateGenericSubrange(DIGenericSubrange::BoundType Count,
                                DIGenericSubrange::BoundType LowerBound,
                                DIGenericSubrange::BoundType UpperBound,
@@ -840,7 +842,7 @@ namespace llvm {
     /// \param MS          DWARF memory space.
     /// \param AlignInBits Variable alignment(or 0 if no alignment attr was
     ///                    specified)
-    DIGlobalVariableExpression *createGlobalVariableExpression(
+    LLVM_ABI DIGlobalVariableExpression *createGlobalVariableExpression(
         DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File,
         unsigned LineNo, DIType *Ty, bool IsLocalToUnit, bool isDefined = true,
         DIExpression *Expr = nullptr, MDNode *Decl = nullptr,
@@ -850,7 +852,7 @@ namespace llvm {
 
     /// Identical to createGlobalVariable
     /// except that the resulting DbgNode is temporary and meant to be RAUWed.
-    DIGlobalVariable *createTempGlobalVariableFwdDecl(
+    LLVM_ABI DIGlobalVariable *createTempGlobalVariableFwdDecl(
         DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File,
         unsigned LineNo, DIType *Ty, bool IsLocalToUnit, MDNode *Decl = nullptr,
         MDTuple *TemplateParams = nullptr,
@@ -865,7 +867,7 @@ namespace llvm {
     ///
     /// If \c AlwaysPreserve, this variable will be referenced from its
     /// containing subprogram, and will survive some optimizations.
-    DILocalVariable *
+    LLVM_ABI DILocalVariable *
     createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File,
                        unsigned LineNo, DIType *Ty, bool AlwaysPreserve = false,
                        DINode::DIFlags Flags = DINode::FlagZero,
@@ -876,9 +878,8 @@ namespace llvm {
     ///
     /// \c Scope must be a \a DILocalScope, and thus its scope chain eventually
     /// leads to a \a DISubprogram.
-    DILabel *
-    createLabel(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo,
-                bool AlwaysPreserve = false);
+    LLVM_ABI DILabel *createLabel(DIScope *Scope, StringRef Name, DIFile *File,
+                                  unsigned LineNo, bool AlwaysPreserve = false);
 
     /// Create a new descriptor for a parameter variable.
     ///
@@ -891,7 +892,7 @@ namespace llvm {
     ///
     /// If \c AlwaysPreserve, this variable will be referenced from its
     /// containing subprogram, and will survive some optimizations.
-    DILocalVariable *
+    LLVM_ABI DILocalVariable *
     createParameterVariable(DIScope *Scope, StringRef Name, unsigned ArgNo,
                             DIFile *File, unsigned LineNo, DIType *Ty,
                             bool AlwaysPreserve = false,
@@ -902,7 +903,7 @@ namespace llvm {
     /// Create a new descriptor for the specified
     /// variable which has a complex address expression for its address.
     /// \param Addr        An array of complex address operations.
-    DIExpression *createExpression(ArrayRef<uint64_t> Addr = {});
+    LLVM_ABI DIExpression *createExpression(ArrayRef<uint64_t> Addr = {});
 
     /// Create an expression for a variable that does not have an address, but
     /// does have a constant value.
@@ -928,20 +929,18 @@ namespace llvm {
     /// \param Annotations   Attribute Annotations.
     /// \param TargetFuncName The name of the target function if this is
     ///                       a trampoline.
-    DISubprogram *
-    createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName,
-                   DIFile *File, unsigned LineNo, DISubroutineType *Ty,
-                   unsigned ScopeLine, DINode::DIFlags Flags = DINode::FlagZero,
-                   DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
-                   DITemplateParameterArray TParams = nullptr,
-                   DISubprogram *Decl = nullptr,
-                   DITypeArray ThrownTypes = nullptr,
-                   DINodeArray Annotations = nullptr,
-                   StringRef TargetFuncName = "");
+    LLVM_ABI DISubprogram *createFunction(
+        DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File,
+        unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine,
+        DINode::DIFlags Flags = DINode::FlagZero,
+        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
+        DITemplateParameterArray TParams = nullptr,
+        DISubprogram *Decl = nullptr, DITypeArray ThrownTypes = nullptr,
+        DINodeArray Annotations = nullptr, StringRef TargetFuncName = "");
 
     /// Identical to createFunction,
     /// except that the resulting DbgNode is meant to be RAUWed.
-    DISubprogram *createTempFunctionFwdDecl(
+    LLVM_ABI DISubprogram *createTempFunctionFwdDecl(
         DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File,
         unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine,
         DINode::DIFlags Flags = DINode::FlagZero,
@@ -968,7 +967,7 @@ namespace llvm {
     /// \param SPFlags       Additional flags specific to subprograms.
     /// \param TParams       Function template parameters.
     /// \param ThrownTypes   Exception types this function may throw.
-    DISubprogram *
+    LLVM_ABI DISubprogram *
     createMethod(DIScope *Scope, StringRef Name, StringRef LinkageName,
                  DIFile *File, unsigned LineNo, DISubroutineType *Ty,
                  unsigned VTableIndex = 0, int ThisAdjustment = 0,
@@ -984,17 +983,18 @@ namespace llvm {
     /// \param Name        The name of this common block.
     /// \param File        The file this common block is defined.
     /// \param LineNo      Line number.
-    DICommonBlock *createCommonBlock(DIScope *Scope, DIGlobalVariable *decl,
-                                     StringRef Name, DIFile *File,
-                                     unsigned LineNo);
+    LLVM_ABI DICommonBlock *createCommonBlock(DIScope *Scope,
+                                              DIGlobalVariable *decl,
+                                              StringRef Name, DIFile *File,
+                                              unsigned LineNo);
 
     /// This creates new descriptor for a namespace with the specified
     /// parent scope.
     /// \param Scope       Namespace scope
     /// \param Name        Name of this namespace
     /// \param ExportSymbols True for C++ inline namespaces.
-    DINamespace *createNameSpace(DIScope *Scope, StringRef Name,
-                                 bool ExportSymbols);
+    LLVM_ABI DINamespace *createNameSpace(DIScope *Scope, StringRef Name,
+                                          bool ExportSymbols);
 
     /// This creates new descriptor for a module with the specified
     /// parent scope.
@@ -1013,10 +1013,12 @@ namespace llvm {
     ///                    when set to true, only Scope and Name are required
     ///                    as this entry is just a hint for the debugger to find
     ///                    the corresponding definition in the global scope.
-    DIModule *createModule(DIScope *Scope, StringRef Name,
-                           StringRef ConfigurationMacros, StringRef IncludePath,
-                           StringRef APINotesFile = {}, DIFile *File = nullptr,
-                           unsigned LineNo = 0, bool IsDecl = false);
+    LLVM_ABI DIModule *createModule(DIScope *Scope, StringRef Name,
+                                    StringRef ConfigurationMacros,
+                                    StringRef IncludePath,
+                                    StringRef APINotesFile = {},
+                                    DIFile *File = nullptr, unsigned LineNo = 0,
+                                    bool IsDecl = false);
 
     /// This creates a descriptor for a lexical block with a new file
     /// attached. This merely extends the existing
@@ -1024,8 +1026,9 @@ namespace llvm {
     /// \param Scope       Lexical block.
     /// \param File        Source file.
     /// \param Discriminator DWARF path discriminator value.
-    DILexicalBlockFile *createLexicalBlockFile(DIScope *Scope, DIFile *File,
-                                               unsigned Discriminator = 0);
+    LLVM_ABI DILexicalBlockFile *
+    createLexicalBlockFile(DIScope *Scope, DIFile *File,
+                           unsigned Discriminator = 0);
 
     /// This creates a descriptor for a lexical block with the
     /// specified parent context.
@@ -1033,8 +1036,8 @@ namespace llvm {
     /// \param File          Source file.
     /// \param Line          Line number.
     /// \param Col           Column number.
-    DILexicalBlock *createLexicalBlock(DIScope *Scope, DIFile *File,
-                                       unsigned Line, unsigned Col);
+    LLVM_ABI DILexicalBlock *createLexicalBlock(DIScope *Scope, DIFile *File,
+                                                unsigned Line, unsigned Col);
 
     /// Create a descriptor for an imported module.
     /// \param Context        The scope this module is imported into
@@ -1042,9 +1045,9 @@ namespace llvm {
     /// \param File           File where the declaration is located.
     /// \param Line           Line number of the declaration.
     /// \param Elements       Renamed elements.
-    DIImportedEntity *createImportedModule(DIScope *Context, DINamespace *NS,
-                                           DIFile *File, unsigned Line,
-                                           DINodeArray Elements = nullptr);
+    LLVM_ABI DIImportedEntity *
+    createImportedModule(DIScope *Context, DINamespace *NS, DIFile *File,
+                         unsigned Line, DINodeArray Elements = nullptr);
 
     /// Create a descriptor for an imported module.
     /// \param Context The scope this module is imported into.
@@ -1052,10 +1055,9 @@ namespace llvm {
     /// \param File    File where the declaration is located.
     /// \param Line    Line number of the declaration.
     /// \param Elements       Renamed elements.
-    DIImportedEntity *createImportedModule(DIScope *Context,
-                                           DIImportedEntity *NS, DIFile *File,
-                                           unsigned Line,
-                                           DINodeArray Elements = nullptr);
+    LLVM_ABI DIImportedEntity *
+    createImportedModule(DIScope *Context, DIImportedEntity *NS, DIFile *File,
+                         unsigned Line, DINodeArray Elements = nullptr);
 
     /// Create a descriptor for an imported module.
     /// \param Context        The scope this module is imported into.
@@ -1063,9 +1065,9 @@ namespace llvm {
     /// \param File           File where the declaration is located.
     /// \param Line           Line number of the declaration.
     /// \param Elements       Renamed elements.
-    DIImportedEntity *createImportedModule(DIScope *Context, DIModule *M,
-                                           DIFile *File, unsigned Line,
-                                           DINodeArray Elements = nullptr);
+    LLVM_ABI DIImportedEntity *
+    createImportedModule(DIScope *Context, DIModule *M, DIFile *File,
+                         unsigned Line, DINodeArray Elements = nullptr);
 
     /// Create a descriptor for an imported function.
     /// \param Context The scope this module is imported into.
@@ -1074,10 +1076,10 @@ namespace llvm {
     /// \param File    File where the declaration is located.
     /// \param Line    Line number of the declaration.
     /// \param Elements       Renamed elements.
-    DIImportedEntity *createImportedDeclaration(DIScope *Context, DINode *Decl,
-                                                DIFile *File, unsigned Line,
-                                                StringRef Name = "",
-                                                DINodeArray Elements = nullptr);
+    LLVM_ABI DIImportedEntity *
+    createImportedDeclaration(DIScope *Context, DINode *Decl, DIFile *File,
+                              unsigned Line, StringRef Name = "",
+                              DINodeArray Elements = nullptr);
 
     /// Insert a new llvm.dbg.declare intrinsic call.
     /// \param Storage     llvm::Value of the variable
@@ -1085,9 +1087,10 @@ namespace llvm {
     /// \param Expr        A complex location expression.
     /// \param DL          Debug info location.
     /// \param InsertAtEnd Location for the new intrinsic.
-    DbgInstPtr insertDeclare(llvm::Value *Storage, DILocalVariable *VarInfo,
-                             DIExpression *Expr, const DILocation *DL,
-                             BasicBlock *InsertAtEnd);
+    LLVM_ABI DbgInstPtr insertDeclare(llvm::Value *Storage,
+                                      DILocalVariable *VarInfo,
+                                      DIExpression *Expr, const DILocation *DL,
+                                      BasicBlock *InsertAtEnd);
 
     /// Insert a new llvm.dbg.assign intrinsic call.
     /// \param LinkedInstr   Instruction with a DIAssignID to link with the new
@@ -1103,10 +1106,11 @@ namespace llvm {
     /// \param DL            Debug info location, usually: (line: 0,
     ///                      column: 0, scope: var-decl-scope). See
     ///                      getDebugValueLoc.
-    DbgInstPtr insertDbgAssign(Instruction *LinkedInstr, Value *Val,
-                               DILocalVariable *SrcVar, DIExpression *ValExpr,
-                               Value *Addr, DIExpression *AddrExpr,
-                               const DILocation *DL);
+    LLVM_ABI DbgInstPtr insertDbgAssign(Instruction *LinkedInstr, Value *Val,
+                                        DILocalVariable *SrcVar,
+                                        DIExpression *ValExpr, Value *Addr,
+                                        DIExpression *AddrExpr,
+                                        const DILocation *DL);
 
     /// Insert a new llvm.dbg.declare intrinsic call.
     /// \param Storage      llvm::Value of the variable
@@ -1114,16 +1118,17 @@ namespace llvm {
     /// \param Expr         A complex location expression.
     /// \param DL           Debug info location.
     /// \param InsertPt     Location for the new intrinsic.
-    DbgInstPtr insertDeclare(llvm::Value *Storage, DILocalVariable *VarInfo,
-                             DIExpression *Expr, const DILocation *DL,
-                             InsertPosition InsertPt);
+    LLVM_ABI DbgInstPtr insertDeclare(llvm::Value *Storage,
+                                      DILocalVariable *VarInfo,
+                                      DIExpression *Expr, const DILocation *DL,
+                                      InsertPosition InsertPt);
 
     /// Insert a new llvm.dbg.label intrinsic call.
     /// \param LabelInfo    Label's debug info descriptor.
     /// \param DL           Debug info location.
     /// \param InsertBefore Location for the new intrinsic.
-    DbgInstPtr insertLabel(DILabel *LabelInfo, const DILocation *DL,
-                           InsertPosition InsertPt);
+    LLVM_ABI DbgInstPtr insertLabel(DILabel *LabelInfo, const DILocation *DL,
+                                    InsertPosition InsertPt);
 
     /// Insert a new llvm.dbg.value intrinsic call.
     /// \param Val          llvm::Value of the variable
@@ -1131,25 +1136,26 @@ namespace llvm {
     /// \param Expr         A complex location expression.
     /// \param DL           Debug info location.
     /// \param InsertPt     Location for the new intrinsic.
-    DbgInstPtr insertDbgValueIntrinsic(llvm::Value *Val,
-                                       DILocalVariable *VarInfo,
-                                       DIExpression *Expr, const DILocation *DL,
-                                       InsertPosition InsertPt);
+    LLVM_ABI DbgInstPtr insertDbgValueIntrinsic(llvm::Value *Val,
+                                                DILocalVariable *VarInfo,
+                                                DIExpression *Expr,
+                                                const DILocation *DL,
+                                                InsertPosition InsertPt);
 
     /// Replace the vtable holder in the given type.
     ///
     /// If this creates a self reference, it may orphan some unresolved cycles
     /// in the operands of \c T, so \a DIBuilder needs to track that.
-    void replaceVTableHolder(DICompositeType *&T,
-                             DIType *VTableHolder);
+    LLVM_ABI void replaceVTableHolder(DICompositeType *&T,
+                                      DIType *VTableHolder);
 
     /// Replace arrays on a composite type.
     ///
     /// If \c T is resolved, but the arrays aren't -- which can happen if \c T
     /// has a self-reference -- \a DIBuilder needs to track the array to
     /// resolve cycles.
-    void replaceArrays(DICompositeType *&T, DINodeArray Elements,
-                       DINodeArray TParams = DINodeArray());
+    LLVM_ABI void replaceArrays(DICompositeType *&T, DINodeArray Elements,
+                                DINodeArray TParams = DINodeArray());
 
     /// Replace a temporary node.
     ///
diff --git a/external/llvm-project/llvm/include/llvm/IR/DataLayout.h b/external/llvm-project/llvm/include/llvm/IR/DataLayout.h
index e399a2f29e54..2992484c47d0 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DataLayout.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DataLayout.h
@@ -68,7 +68,7 @@ class DataLayout {
     Align ABIAlign;
     Align PrefAlign;
 
-    bool operator==(const PrimitiveSpec &Other) const;
+    LLVM_ABI bool operator==(const PrimitiveSpec &Other) const;
   };
 
   /// Pointer type specification.
@@ -83,7 +83,7 @@ class DataLayout {
     /// Additionally, they may also be non-integral (i.e. containing additional
     /// metadata such as bounds information/permissions).
     bool IsNonIntegral;
-    bool operator==(const PointerSpec &Other) const;
+    LLVM_ABI bool operator==(const PointerSpec &Other) const;
   };
 
   enum class FunctionPtrAlignType {
@@ -144,7 +144,7 @@ class DataLayout {
 
   /// Searches for a pointer specification that matches the given address space.
   /// Returns the default address space specification if not found.
-  const PointerSpec &getPointerSpec(uint32_t AddrSpace) const;
+  LLVM_ABI const PointerSpec &getPointerSpec(uint32_t AddrSpace) const;
 
   /// Sets or updates the specification for pointer in the given address space.
   void setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth, Align ABIAlign,
@@ -152,7 +152,7 @@ class DataLayout {
                       bool IsNonIntegral);
 
   /// Internal helper to get alignment for integer of given bitwidth.
-  Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;
+  LLVM_ABI Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;
 
   /// Internal helper method that returns requested alignment for type.
   Align getAlignment(Type *Ty, bool abi_or_pref) const;
@@ -175,24 +175,24 @@ class DataLayout {
 
 public:
   /// Constructs a DataLayout with default values.
-  DataLayout();
+  LLVM_ABI DataLayout();
 
   /// Constructs a DataLayout from a specification string.
   /// WARNING: Aborts execution if the string is malformed. Use parse() instead.
-  explicit DataLayout(StringRef LayoutString);
+  LLVM_ABI explicit DataLayout(StringRef LayoutString);
 
   DataLayout(const DataLayout &DL) { *this = DL; }
 
-  ~DataLayout(); // Not virtual, do not subclass this class
+  LLVM_ABI ~DataLayout(); // Not virtual, do not subclass this class
 
-  DataLayout &operator=(const DataLayout &Other);
+  LLVM_ABI DataLayout &operator=(const DataLayout &Other);
 
-  bool operator==(const DataLayout &Other) const;
+  LLVM_ABI bool operator==(const DataLayout &Other) const;
   bool operator!=(const DataLayout &Other) const { return !(*this == Other); }
 
   /// Parse a data layout string and return the layout. Return an error
   /// description on failure.
-  static Expected<DataLayout> parse(StringRef LayoutString);
+  LLVM_ABI static Expected<DataLayout> parse(StringRef LayoutString);
 
   /// Layout endianness...
   bool isLittleEndian() const { return !BigEndian; }
@@ -303,7 +303,7 @@ class DataLayout {
     llvm_unreachable("invalid mangling mode");
   }
 
-  static const char *getManglingComponent(const Triple &T);
+  LLVM_ABI static const char *getManglingComponent(const Triple &T);
 
   /// Returns true if the specified type fits in a native integer type
   /// supported by the CPU.
@@ -318,12 +318,12 @@ class DataLayout {
   }
 
   /// Layout pointer alignment
-  Align getPointerABIAlignment(unsigned AS) const;
+  LLVM_ABI Align getPointerABIAlignment(unsigned AS) const;
 
   /// Return target's alignment for stack-based pointers
   /// FIXME: The defaults need to be removed once all of
   /// the backends/clients are updated.
-  Align getPointerPrefAlignment(unsigned AS = 0) const;
+  LLVM_ABI Align getPointerPrefAlignment(unsigned AS = 0) const;
 
   /// The pointer representation size in bytes, rounded up to a whole number of
   /// bytes. The difference between this function and getAddressSize() is that
@@ -333,7 +333,7 @@ class DataLayout {
   /// \sa DataLayout::getAddressSizeInBits
   /// FIXME: The defaults need to be removed once all of
   /// the backends/clients are updated.
-  unsigned getPointerSize(unsigned AS = 0) const;
+  LLVM_ABI unsigned getPointerSize(unsigned AS = 0) const;
 
   /// The index size in bytes used for address calculation, rounded up to a
   /// whole number of bytes. This not only defines the size used in
@@ -344,7 +344,7 @@ class DataLayout {
   /// getPointerSize() returns 16, but getIndexSize() returns 8.
   /// To help with code understanding, the alias getAddressSize() can be used
   /// instead of getIndexSize() to clarify that an address width is needed.
-  unsigned getIndexSize(unsigned AS) const;
+  LLVM_ABI unsigned getIndexSize(unsigned AS) const;
 
   /// The integral size of a pointer in a given address space in bytes, which
   /// is defined to be the same as getIndexSize(). This exists as a separate
@@ -413,13 +413,13 @@ class DataLayout {
   /// If this function is called with a vector of pointers, then the type size
   /// of the pointer is returned.  This should only be called with a pointer or
   /// vector of pointers.
-  unsigned getPointerTypeSizeInBits(Type *) const;
+  LLVM_ABI unsigned getPointerTypeSizeInBits(Type *) const;
 
   /// The size in bits of the index used in GEP calculation for this type.
   /// The function should be called with pointer or vector of pointers type.
   /// This is defined to return the same value as getAddressSizeInBits(),
   /// but separate functions exist for code clarity.
-  unsigned getIndexTypeSizeInBits(Type *Ty) const;
+  LLVM_ABI unsigned getIndexTypeSizeInBits(Type *Ty) const;
 
   /// The size in bits of an address for this type.
   /// This is defined to return the same value as getIndexTypeSizeInBits(),
@@ -519,7 +519,7 @@ class DataLayout {
   }
 
   /// Returns the minimum ABI-required alignment for the specified type.
-  Align getABITypeAlign(Type *Ty) const;
+  LLVM_ABI Align getABITypeAlign(Type *Ty) const;
 
   /// Helper function to return `Alignment` if it's set or the result of
   /// `getABITypeAlign(Ty)`, in any case the result is a valid alignment.
@@ -538,19 +538,21 @@ class DataLayout {
   /// type.
   ///
   /// This is always at least as good as the ABI alignment.
-  Align getPrefTypeAlign(Type *Ty) const;
+  LLVM_ABI Align getPrefTypeAlign(Type *Ty) const;
 
   /// Returns an integer type with size at least as big as that of a
   /// pointer in the given address space.
-  IntegerType *getIntPtrType(LLVMContext &C, unsigned AddressSpace = 0) const;
+  LLVM_ABI IntegerType *getIntPtrType(LLVMContext &C,
+                                      unsigned AddressSpace = 0) const;
 
   /// Returns an integer (vector of integer) type with size at least as
   /// big as that of a pointer of the given pointer (vector of pointer) type.
-  Type *getIntPtrType(Type *) const;
+  LLVM_ABI Type *getIntPtrType(Type *) const;
 
   /// Returns the smallest integer type with size at least as big as
   /// Width bits.
-  Type *getSmallestLegalIntType(LLVMContext &C, unsigned Width = 0) const;
+  LLVM_ABI Type *getSmallestLegalIntType(LLVMContext &C,
+                                         unsigned Width = 0) const;
 
   /// Returns the largest legal integer type, or null if none are set.
   Type *getLargestLegalIntType(LLVMContext &C) const {
@@ -560,12 +562,13 @@ class DataLayout {
 
   /// Returns the size of largest legal integer type size, or 0 if none
   /// are set.
-  unsigned getLargestLegalIntTypeSizeInBits() const;
+  LLVM_ABI unsigned getLargestLegalIntTypeSizeInBits() const;
 
   /// Returns the type of a GEP index in \p AddressSpace.
   /// If it was not specified explicitly, it will be the integer type of the
   /// pointer width - IntPtrType.
-  IntegerType *getIndexType(LLVMContext &C, unsigned AddressSpace) const;
+  LLVM_ABI IntegerType *getIndexType(LLVMContext &C,
+                                     unsigned AddressSpace) const;
   /// Returns the type of an address in \p AddressSpace
   IntegerType *getAddressType(LLVMContext &C, unsigned AddressSpace) const {
     return getIndexType(C, AddressSpace);
@@ -574,7 +577,7 @@ class DataLayout {
   /// Returns the type of a GEP index.
   /// If it was not specified explicitly, it will be the integer type of the
   /// pointer width - IntPtrType.
-  Type *getIndexType(Type *PtrTy) const;
+  LLVM_ABI Type *getIndexType(Type *PtrTy) const;
   /// Returns the type of an address in \p AddressSpace
   Type *getAddressType(Type *PtrTy) const { return getIndexType(PtrTy); }
 
@@ -583,28 +586,31 @@ class DataLayout {
   ///
   /// Note that this takes the element type, not the pointer type.
   /// This is used to implement getelementptr.
-  int64_t getIndexedOffsetInType(Type *ElemTy, ArrayRef<Value *> Indices) const;
+  LLVM_ABI int64_t getIndexedOffsetInType(Type *ElemTy,
+                                          ArrayRef<Value *> Indices) const;
 
   /// Get GEP indices to access Offset inside ElemTy. ElemTy is updated to be
   /// the result element type and Offset to be the residual offset.
-  SmallVector<APInt> getGEPIndicesForOffset(Type *&ElemTy, APInt &Offset) const;
+  LLVM_ABI SmallVector<APInt> getGEPIndicesForOffset(Type *&ElemTy,
+                                                     APInt &Offset) const;
 
   /// Get single GEP index to access Offset inside ElemTy. Returns std::nullopt
   /// if index cannot be computed, e.g. because the type is not an aggregate.
   /// ElemTy is updated to be the result element type and Offset to be the
   /// residual offset.
-  std::optional<APInt> getGEPIndexForOffset(Type *&ElemTy, APInt &Offset) const;
+  LLVM_ABI std::optional<APInt> getGEPIndexForOffset(Type *&ElemTy,
+                                                     APInt &Offset) const;
 
   /// Returns a StructLayout object, indicating the alignment of the
   /// struct, its size, and the offsets of its fields.
   ///
   /// Note that this information is lazily cached.
-  const StructLayout *getStructLayout(StructType *Ty) const;
+  LLVM_ABI const StructLayout *getStructLayout(StructType *Ty) const;
 
   /// Returns the preferred alignment of the specified global.
   ///
   /// This includes an explicitly requested alignment (if the global has one).
-  Align getPreferredAlign(const GlobalVariable *GV) const;
+  LLVM_ABI Align getPreferredAlign(const GlobalVariable *GV) const;
 };
 
 inline DataLayout *unwrap(LLVMTargetDataRef P) {
@@ -638,7 +644,7 @@ class StructLayout final : private TrailingObjects<StructLayout, TypeSize> {
 
   /// Given a valid byte offset into the structure, returns the structure
   /// index that contains it.
-  unsigned getElementContainingOffset(uint64_t FixedOffset) const;
+  LLVM_ABI unsigned getElementContainingOffset(uint64_t FixedOffset) const;
 
   MutableArrayRef<TypeSize> getMemberOffsets() {
     return getTrailingObjects(NumElements);
diff --git a/external/llvm-project/llvm/include/llvm/IR/DebugInfo.h b/external/llvm-project/llvm/include/llvm/IR/DebugInfo.h
index c47738a6442e..77cee875f16e 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DebugInfo.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DebugInfo.h
@@ -27,6 +27,7 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -40,37 +41,37 @@ class Module;
 
 /// Finds dbg.declare intrinsics declaring local variables as living in the
 /// memory that 'V' points to.
-TinyPtrVector<DbgDeclareInst *> findDbgDeclares(Value *V);
+LLVM_ABI TinyPtrVector<DbgDeclareInst *> findDbgDeclares(Value *V);
 /// As above, for DVRDeclares.
-TinyPtrVector<DbgVariableRecord *> findDVRDeclares(Value *V);
+LLVM_ABI TinyPtrVector<DbgVariableRecord *> findDVRDeclares(Value *V);
 /// As above, for DVRValues.
-TinyPtrVector<DbgVariableRecord *> findDVRValues(Value *V);
+LLVM_ABI TinyPtrVector<DbgVariableRecord *> findDVRValues(Value *V);
 
 /// Finds the llvm.dbg.value intrinsics describing a value.
-void findDbgValues(
+LLVM_ABI void findDbgValues(
     SmallVectorImpl<DbgValueInst *> &DbgValues, Value *V,
     SmallVectorImpl<DbgVariableRecord *> *DbgVariableRecords = nullptr);
 
 /// Finds the debug info intrinsics describing a value.
-void findDbgUsers(
+LLVM_ABI void findDbgUsers(
     SmallVectorImpl<DbgVariableIntrinsic *> &DbgInsts, Value *V,
     SmallVectorImpl<DbgVariableRecord *> *DbgVariableRecords = nullptr);
 
 /// Find subprogram that is enclosing this scope.
-DISubprogram *getDISubprogram(const MDNode *Scope);
+LLVM_ABI DISubprogram *getDISubprogram(const MDNode *Scope);
 
 /// Produce a DebugLoc to use for each dbg.declare that is promoted to a
 /// dbg.value.
-DebugLoc getDebugValueLoc(DbgVariableIntrinsic *DII);
-DebugLoc getDebugValueLoc(DbgVariableRecord *DVR);
+LLVM_ABI DebugLoc getDebugValueLoc(DbgVariableIntrinsic *DII);
+LLVM_ABI DebugLoc getDebugValueLoc(DbgVariableRecord *DVR);
 
 /// Strip debug info in the module if it exists.
 ///
 /// To do this, we remove all calls to the debugger intrinsics and any named
 /// metadata for debugging. We also remove debug locations for instructions.
 /// Return true if module is modified.
-bool StripDebugInfo(Module &M);
-bool stripDebugInfo(Function &F);
+LLVM_ABI bool StripDebugInfo(Module &M);
+LLVM_ABI bool stripDebugInfo(Function &F);
 
 /// Downgrade the debug info in a module to contain only line table information.
 ///
@@ -82,17 +83,18 @@ bool stripDebugInfo(Function &F);
 ///   4) Create a new CU debug info, and similarly for every metadata node
 ///      that's reachable from the CU debug info.
 ///   All debug type metadata nodes are unreachable and garbage collected.
-bool stripNonLineTableDebugInfo(Module &M);
+LLVM_ABI bool stripNonLineTableDebugInfo(Module &M);
 
 /// Update the debug locations contained within the MD_loop metadata attached
 /// to the instruction \p I, if one exists. \p Updater is applied to Metadata
 /// operand in the MD_loop metadata: the returned value is included in the
 /// updated loop metadata node if it is non-null.
-void updateLoopMetadataDebugLocations(
-    Instruction &I, function_ref<Metadata *(Metadata *)> Updater);
+LLVM_ABI void
+updateLoopMetadataDebugLocations(Instruction &I,
+                                 function_ref<Metadata *(Metadata *)> Updater);
 
 /// Return Debug Info Metadata Version by checking module flags.
-unsigned getDebugMetadataVersionFromModule(const Module &M);
+LLVM_ABI unsigned getDebugMetadataVersionFromModule(const Module &M);
 
 /// Utility to find all debug info in a module.
 ///
@@ -105,23 +107,23 @@ unsigned getDebugMetadataVersionFromModule(const Module &M);
 class DebugInfoFinder {
 public:
   /// Process entire module and collect debug info anchors.
-  void processModule(const Module &M);
+  LLVM_ABI void processModule(const Module &M);
   /// Process a single instruction and collect debug info anchors.
-  void processInstruction(const Module &M, const Instruction &I);
+  LLVM_ABI void processInstruction(const Module &M, const Instruction &I);
 
   /// Process a DILocalVariable.
-  void processVariable(DILocalVariable *DVI);
+  LLVM_ABI void processVariable(DILocalVariable *DVI);
   /// Process debug info location.
-  void processLocation(const Module &M, const DILocation *Loc);
+  LLVM_ABI void processLocation(const Module &M, const DILocation *Loc);
   /// Process a DbgRecord (e.g, treat a DbgVariableRecord like a
   /// DbgVariableIntrinsic).
-  void processDbgRecord(const Module &M, const DbgRecord &DR);
+  LLVM_ABI void processDbgRecord(const Module &M, const DbgRecord &DR);
 
   /// Process subprogram.
-  void processSubprogram(DISubprogram *SP);
+  LLVM_ABI void processSubprogram(DISubprogram *SP);
 
   /// Clear all lists.
-  void reset();
+  LLVM_ABI void reset();
 
 private:
   void processCompileUnit(DICompileUnit *CU);
@@ -190,7 +192,7 @@ using AssignmentInstRange =
 /// as an attachment.
 /// Iterators invalidated by adding or removing DIAssignID metadata to/from any
 /// instruction (including by deleting or cloning instructions).
-AssignmentInstRange getAssignmentInsts(DIAssignID *ID);
+LLVM_ABI AssignmentInstRange getAssignmentInsts(DIAssignID *ID);
 /// Return a range of instructions (typically just one) that perform the
 /// assignment that \p DAI encodes.
 /// Iterators invalidated by adding or removing DIAssignID metadata to/from any
@@ -226,7 +228,7 @@ class DbgAssignIt
 using AssignmentMarkerRange = iterator_range<DbgAssignIt>;
 /// Return a range of dbg.assign intrinsics which use \ID as an operand.
 /// Iterators invalidated by deleting an intrinsic contained in this range.
-AssignmentMarkerRange getAssignmentMarkers(DIAssignID *ID);
+LLVM_ABI AssignmentMarkerRange getAssignmentMarkers(DIAssignID *ID);
 /// Return a range of dbg.assign intrinsics for which \p Inst performs the
 /// assignment they encode.
 /// Iterators invalidated by deleting an intrinsic contained in this range.
@@ -245,13 +247,13 @@ getDVRAssignmentMarkers(const Instruction *Inst) {
 }
 
 /// Delete the llvm.dbg.assign intrinsics linked to \p Inst.
-void deleteAssignmentMarkers(const Instruction *Inst);
+LLVM_ABI void deleteAssignmentMarkers(const Instruction *Inst);
 
 /// Replace all uses (and attachments) of \p Old with \p New.
-void RAUW(DIAssignID *Old, DIAssignID *New);
+LLVM_ABI void RAUW(DIAssignID *Old, DIAssignID *New);
 
 /// Remove all Assignment Tracking related intrinsics and metadata from \p F.
-void deleteAll(Function *F);
+LLVM_ABI void deleteAll(Function *F);
 
 /// Calculate the fragment of the variable in \p DAI covered
 /// from (Dest + SliceOffsetInBits) to
@@ -262,18 +264,21 @@ void deleteAll(Function *F);
 /// variable size) in DAI.
 ///
 /// Result contains a zero-sized fragment if there's no intersect.
-bool calculateFragmentIntersect(
-    const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
-    uint64_t SliceSizeInBits, const DbgAssignIntrinsic *DbgAssign,
-    std::optional<DIExpression::FragmentInfo> &Result);
-bool calculateFragmentIntersect(
-    const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
-    uint64_t SliceSizeInBits, const DbgVariableRecord *DVRAssign,
-    std::optional<DIExpression::FragmentInfo> &Result);
+LLVM_ABI bool
+calculateFragmentIntersect(const DataLayout &DL, const Value *Dest,
+                           uint64_t SliceOffsetInBits, uint64_t SliceSizeInBits,
+                           const DbgAssignIntrinsic *DbgAssign,
+                           std::optional<DIExpression::FragmentInfo> &Result);
+LLVM_ABI bool
+calculateFragmentIntersect(const DataLayout &DL, const Value *Dest,
+                           uint64_t SliceOffsetInBits, uint64_t SliceSizeInBits,
+                           const DbgVariableRecord *DVRAssign,
+                           std::optional<DIExpression::FragmentInfo> &Result);
 
 /// Replace DIAssignID uses and attachments with IDs from \p Map.
 /// If an ID is unmapped a new ID is generated and added to \p Map.
-void remapAssignID(DenseMap<DIAssignID *, DIAssignID *> &Map, Instruction &I);
+LLVM_ABI void remapAssignID(DenseMap<DIAssignID *, DIAssignID *> &Map,
+                            Instruction &I);
 
 /// Helper struct for trackAssignments, below. We don't use the similar
 /// DebugVariable class because trackAssignments doesn't (yet?) understand
@@ -330,9 +335,9 @@ using StorageToVarsMap =
 
 /// Track assignments to \p Vars between \p Start and \p End.
 
-void trackAssignments(Function::iterator Start, Function::iterator End,
-                      const StorageToVarsMap &Vars, const DataLayout &DL,
-                      bool DebugPrints = false);
+LLVM_ABI void trackAssignments(Function::iterator Start, Function::iterator End,
+                               const StorageToVarsMap &Vars,
+                               const DataLayout &DL, bool DebugPrints = false);
 
 /// Describes properties of a store that has a static size and offset into a
 /// some base storage. Used by the getAssignmentInfo functions.
@@ -350,12 +355,12 @@ struct AssignmentInfo {
             SizeInBits == DL.getTypeSizeInBits(Base->getAllocatedType())) {}
 };
 
-std::optional<AssignmentInfo> getAssignmentInfo(const DataLayout &DL,
-                                                const MemIntrinsic *I);
-std::optional<AssignmentInfo> getAssignmentInfo(const DataLayout &DL,
-                                                const StoreInst *SI);
-std::optional<AssignmentInfo> getAssignmentInfo(const DataLayout &DL,
-                                                const AllocaInst *AI);
+LLVM_ABI std::optional<AssignmentInfo> getAssignmentInfo(const DataLayout &DL,
+                                                         const MemIntrinsic *I);
+LLVM_ABI std::optional<AssignmentInfo> getAssignmentInfo(const DataLayout &DL,
+                                                         const StoreInst *SI);
+LLVM_ABI std::optional<AssignmentInfo> getAssignmentInfo(const DataLayout &DL,
+                                                         const AllocaInst *AI);
 
 } // end namespace at
 
@@ -371,12 +376,12 @@ class AssignmentTrackingPass : public PassInfoMixin<AssignmentTrackingPass> {
   bool runOnFunction(Function &F);
 
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 /// Return true if assignment tracking is enabled for module \p M.
-bool isAssignmentTrackingEnabled(const Module &M);
+LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/DebugInfoMetadata.h b/external/llvm-project/llvm/include/llvm/IR/DebugInfoMetadata.h
index fb625d8226ce..b86e5aa6b541 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -90,7 +90,7 @@ enum Tag : uint16_t;
 class DbgVariableIntrinsic;
 class DbgVariableRecord;
 
-extern cl::opt<bool> EnableFSDiscriminator;
+LLVM_ABI extern cl::opt<bool> EnableFSDiscriminator;
 
 class DITypeRefArray {
   const MDTuple *N = nullptr;
@@ -187,7 +187,7 @@ class DINode : public MDNode {
   void setTag(unsigned Tag) { SubclassData16 = Tag; }
 
 public:
-  dwarf::Tag getTag() const;
+  LLVM_ABI dwarf::Tag getTag() const;
 
   /// Debug info flags.
   ///
@@ -203,15 +203,15 @@ class DINode : public MDNode {
     LLVM_MARK_AS_BITMASK_ENUM(FlagLargest)
   };
 
-  static DIFlags getFlag(StringRef Flag);
-  static StringRef getFlagString(DIFlags Flag);
+  LLVM_ABI static DIFlags getFlag(StringRef Flag);
+  LLVM_ABI static StringRef getFlagString(DIFlags Flag);
 
   /// Split up a flags bitfield.
   ///
   /// Split \c Flags into \c SplitFlags, a vector of its components.  Returns
   /// any remaining (unrecognized) bits.
-  static DIFlags splitFlags(DIFlags Flags,
-                            SmallVectorImpl<DIFlags> &SplitFlags);
+  LLVM_ABI static DIFlags splitFlags(DIFlags Flags,
+                                     SmallVectorImpl<DIFlags> &SplitFlags);
 
   static bool classof(const Metadata *MD) {
     switch (MD->getMetadataID()) {
@@ -280,9 +280,11 @@ class GenericDINode : public DINode {
                    DwarfOps, Storage, ShouldCreate);
   }
 
-  static GenericDINode *getImpl(LLVMContext &Context, unsigned Tag,
-                                MDString *Header, ArrayRef<Metadata *> DwarfOps,
-                                StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static GenericDINode *getImpl(LLVMContext &Context, unsigned Tag,
+                                         MDString *Header,
+                                         ArrayRef<Metadata *> DwarfOps,
+                                         StorageType Storage,
+                                         bool ShouldCreate = true);
 
   TempGenericDINode cloneImpl() const {
     return getTemporary(getContext(), getTag(), getHeader(),
@@ -304,7 +306,7 @@ class GenericDINode : public DINode {
   /// Return a (temporary) clone of this.
   TempGenericDINode clone() const { return cloneImpl(); }
 
-  dwarf::Tag getTag() const;
+  LLVM_ABI dwarf::Tag getTag() const;
   StringRef getHeader() const { return getStringOperand(0); }
   MDString *getRawHeader() const { return getOperandAs<MDString>(0); }
 
@@ -340,8 +342,8 @@ class DIAssignID : public MDNode {
 
   ~DIAssignID() { dropAllReferences(); }
 
-  static DIAssignID *getImpl(LLVMContext &Context, StorageType Storage,
-                             bool ShouldCreate = true);
+  LLVM_ABI static DIAssignID *getImpl(LLVMContext &Context, StorageType Storage,
+                                      bool ShouldCreate = true);
 
   TempDIAssignID cloneImpl() const { return getTemporary(getContext()); }
 
@@ -375,18 +377,19 @@ class DISubrange : public DINode {
 
   ~DISubrange() = default;
 
-  static DISubrange *getImpl(LLVMContext &Context, int64_t Count,
-                             int64_t LowerBound, StorageType Storage,
-                             bool ShouldCreate = true);
+  LLVM_ABI static DISubrange *getImpl(LLVMContext &Context, int64_t Count,
+                                      int64_t LowerBound, StorageType Storage,
+                                      bool ShouldCreate = true);
 
-  static DISubrange *getImpl(LLVMContext &Context, Metadata *CountNode,
-                             int64_t LowerBound, StorageType Storage,
-                             bool ShouldCreate = true);
+  LLVM_ABI static DISubrange *getImpl(LLVMContext &Context, Metadata *CountNode,
+                                      int64_t LowerBound, StorageType Storage,
+                                      bool ShouldCreate = true);
 
-  static DISubrange *getImpl(LLVMContext &Context, Metadata *CountNode,
-                             Metadata *LowerBound, Metadata *UpperBound,
-                             Metadata *Stride, StorageType Storage,
-                             bool ShouldCreate = true);
+  LLVM_ABI static DISubrange *getImpl(LLVMContext &Context, Metadata *CountNode,
+                                      Metadata *LowerBound,
+                                      Metadata *UpperBound, Metadata *Stride,
+                                      StorageType Storage,
+                                      bool ShouldCreate = true);
 
   TempDISubrange cloneImpl() const {
     return getTemporary(getContext(), getRawCountNode(), getRawLowerBound(),
@@ -417,13 +420,13 @@ class DISubrange : public DINode {
 
   typedef PointerUnion<ConstantInt *, DIVariable *, DIExpression *> BoundType;
 
-  BoundType getCount() const;
+  LLVM_ABI BoundType getCount() const;
 
-  BoundType getLowerBound() const;
+  LLVM_ABI BoundType getLowerBound() const;
 
-  BoundType getUpperBound() const;
+  LLVM_ABI BoundType getUpperBound() const;
 
-  BoundType getStride() const;
+  LLVM_ABI BoundType getStride() const;
 
   static bool classof(const Metadata *MD) {
     return MD->getMetadataID() == DISubrangeKind;
@@ -439,10 +442,10 @@ class DIGenericSubrange : public DINode {
 
   ~DIGenericSubrange() = default;
 
-  static DIGenericSubrange *getImpl(LLVMContext &Context, Metadata *CountNode,
-                                    Metadata *LowerBound, Metadata *UpperBound,
-                                    Metadata *Stride, StorageType Storage,
-                                    bool ShouldCreate = true);
+  LLVM_ABI static DIGenericSubrange *
+  getImpl(LLVMContext &Context, Metadata *CountNode, Metadata *LowerBound,
+          Metadata *UpperBound, Metadata *Stride, StorageType Storage,
+          bool ShouldCreate = true);
 
   TempDIGenericSubrange cloneImpl() const {
     return getTemporary(getContext(), getRawCountNode(), getRawLowerBound(),
@@ -464,10 +467,10 @@ class DIGenericSubrange : public DINode {
 
   using BoundType = PointerUnion<DIVariable *, DIExpression *>;
 
-  BoundType getCount() const;
-  BoundType getLowerBound() const;
-  BoundType getUpperBound() const;
-  BoundType getStride() const;
+  LLVM_ABI BoundType getCount() const;
+  LLVM_ABI BoundType getLowerBound() const;
+  LLVM_ABI BoundType getUpperBound() const;
+  LLVM_ABI BoundType getStride() const;
 
   static bool classof(const Metadata *MD) {
     return MD->getMetadataID() == DIGenericSubrangeKind;
@@ -483,8 +486,8 @@ class DIEnumerator : public DINode {
   friend class MDNode;
 
   APInt Value;
-  DIEnumerator(LLVMContext &C, StorageType Storage, const APInt &Value,
-               bool IsUnsigned, ArrayRef<Metadata *> Ops);
+  LLVM_ABI DIEnumerator(LLVMContext &C, StorageType Storage, const APInt &Value,
+                        bool IsUnsigned, ArrayRef<Metadata *> Ops);
   DIEnumerator(LLVMContext &C, StorageType Storage, int64_t Value,
                bool IsUnsigned, ArrayRef<Metadata *> Ops)
       : DIEnumerator(C, Storage, APInt(64, Value, !IsUnsigned), IsUnsigned,
@@ -497,9 +500,10 @@ class DIEnumerator : public DINode {
     return getImpl(Context, Value, IsUnsigned,
                    getCanonicalMDString(Context, Name), Storage, ShouldCreate);
   }
-  static DIEnumerator *getImpl(LLVMContext &Context, const APInt &Value,
-                               bool IsUnsigned, MDString *Name,
-                               StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DIEnumerator *getImpl(LLVMContext &Context,
+                                        const APInt &Value, bool IsUnsigned,
+                                        MDString *Name, StorageType Storage,
+                                        bool ShouldCreate = true);
 
   TempDIEnumerator cloneImpl() const {
     return getTemporary(getContext(), getValue(), isUnsigned(), getName());
@@ -552,8 +556,8 @@ class DIScope : public DINode {
   inline StringRef getDirectory() const;
   inline std::optional<StringRef> getSource() const;
 
-  StringRef getName() const;
-  DIScope *getScope() const;
+  LLVM_ABI StringRef getName() const;
+  LLVM_ABI DIScope *getScope() const;
 
   /// Return the raw underlying file.
   ///
@@ -653,11 +657,11 @@ class DIFile : public DIScope {
                    Source ? MDString::get(Context, *Source) : nullptr, Storage,
                    ShouldCreate);
   }
-  static DIFile *getImpl(LLVMContext &Context, MDString *Filename,
-                         MDString *Directory,
-                         std::optional<ChecksumInfo<MDString *>> CS,
-                         MDString *Source, StorageType Storage,
-                         bool ShouldCreate = true);
+  LLVM_ABI static DIFile *getImpl(LLVMContext &Context, MDString *Filename,
+                                  MDString *Directory,
+                                  std::optional<ChecksumInfo<MDString *>> CS,
+                                  MDString *Source, StorageType Storage,
+                                  bool ShouldCreate = true);
 
   TempDIFile cloneImpl() const {
     return getTemporary(getContext(), getFilename(), getDirectory(),
@@ -698,8 +702,9 @@ class DIFile : public DIScope {
   }
   MDString *getRawSource() const { return Source; }
 
-  static StringRef getChecksumKindAsString(ChecksumKind CSKind);
-  static std::optional<ChecksumKind> getChecksumKind(StringRef CSKindStr);
+  LLVM_ABI static StringRef getChecksumKindAsString(ChecksumKind CSKind);
+  LLVM_ABI static std::optional<ChecksumKind>
+  getChecksumKind(StringRef CSKindStr);
 
   static bool classof(const Metadata *MD) {
     return MD->getMetadataID() == DIFileKind;
@@ -777,7 +782,7 @@ class DIType : public DIScope {
 
   unsigned getLine() const { return Line; }
   uint64_t getSizeInBits() const { return SizeInBits; }
-  uint32_t getAlignInBits() const;
+  LLVM_ABI uint32_t getAlignInBits() const;
   uint32_t getAlignInBytes() const { return getAlignInBits() / CHAR_BIT; }
   uint64_t getOffsetInBits() const { return OffsetInBits; }
   uint32_t getNumExtraInhabitants() const { return NumExtraInhabitants; }
@@ -878,11 +883,12 @@ class DIBasicType : public DIType {
                    SizeInBits, AlignInBits, Encoding, NumExtraInhabitants,
                    Flags, Storage, ShouldCreate);
   }
-  static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
-                              MDString *Name, uint64_t SizeInBits,
-                              uint32_t AlignInBits, unsigned Encoding,
-                              uint32_t NumExtraInhabitants, DIFlags Flags,
-                              StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
+                                       MDString *Name, uint64_t SizeInBits,
+                                       uint32_t AlignInBits, unsigned Encoding,
+                                       uint32_t NumExtraInhabitants,
+                                       DIFlags Flags, StorageType Storage,
+                                       bool ShouldCreate = true);
 
   TempDIBasicType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getName(), getSizeInBits(),
@@ -928,7 +934,7 @@ class DIBasicType : public DIType {
 
   /// Return the signedness of this type, or std::nullopt if this type is
   /// neither signed nor unsigned.
-  std::optional<Signedness> getSignedness() const;
+  LLVM_ABI std::optional<Signedness> getSignedness() const;
 
   static bool classof(const Metadata *MD) {
     return MD->getMetadataID() == DIBasicTypeKind ||
@@ -986,7 +992,7 @@ class DIFixedPointType : public DIBasicType {
                    SizeInBits, AlignInBits, Encoding, Flags, Kind, Factor,
                    Numerator, Denominator, Storage, ShouldCreate);
   }
-  static DIFixedPointType *
+  LLVM_ABI static DIFixedPointType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
           uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
           DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
@@ -1009,8 +1015,9 @@ class DIFixedPointType : public DIBasicType {
     LastFixedPointKind = FixedPointRational,
   };
 
-  static std::optional<FixedPointKind> getFixedPointKind(StringRef Str);
-  static const char *fixedPointKindString(FixedPointKind);
+  LLVM_ABI static std::optional<FixedPointKind>
+  getFixedPointKind(StringRef Str);
+  LLVM_ABI static const char *fixedPointKindString(FixedPointKind);
 
   DEFINE_MDNODE_GET(DIFixedPointType,
                     (unsigned Tag, MDString *Name, uint64_t SizeInBits,
@@ -1033,7 +1040,7 @@ class DIFixedPointType : public DIBasicType {
   bool isDecimal() const { return Kind == FixedPointDecimal; }
   bool isRational() const { return Kind == FixedPointRational; }
 
-  bool isSigned() const;
+  LLVM_ABI bool isSigned() const;
 
   FixedPointKind getKind() const { return static_cast<FixedPointKind>(Kind); }
 
@@ -1085,12 +1092,11 @@ class DIStringType : public DIType {
                    StringLength, StrLenExp, StrLocationExp, SizeInBits,
                    AlignInBits, Encoding, Storage, ShouldCreate);
   }
-  static DIStringType *getImpl(LLVMContext &Context, unsigned Tag,
-                               MDString *Name, Metadata *StringLength,
-                               Metadata *StrLenExp, Metadata *StrLocationExp,
-                               uint64_t SizeInBits, uint32_t AlignInBits,
-                               unsigned Encoding, StorageType Storage,
-                               bool ShouldCreate = true);
+  LLVM_ABI static DIStringType *
+  getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
+          Metadata *StringLength, Metadata *StrLenExp, Metadata *StrLocationExp,
+          uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDIStringType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getRawName(),
@@ -1215,7 +1221,7 @@ class DIDerivedType : public DIType {
                    DWARFAddressSpace, MS, PtrAuthData, Flags, ExtraData, Annotations.get(),
                    Storage, ShouldCreate);
   }
-  static DIDerivedType *
+  LLVM_ABI static DIDerivedType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
           unsigned Line, Metadata *Scope, Metadata *BaseType,
           uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
@@ -1256,6 +1262,12 @@ class DIDerivedType : public DIType {
 
   TempDIDerivedType clone() const { return cloneImpl(); }
 
+  TempDIDerivedType cloneWithAddressSpace(unsigned DWARFAddrSpace) const {
+    auto Tmp = clone();
+    Tmp->DWARFAddressSpace = DWARFAddrSpace;
+    return Tmp;
+  }
+
   /// Get the base type this is derived from.
   DIType *getBaseType() const { return cast_or_null<DIType>(getRawBaseType()); }
   Metadata *getRawBaseType() const { return getOperand(3); }
@@ -1270,7 +1282,7 @@ class DIDerivedType : public DIType {
   /// a pointer or reference type respectively.
   dwarf::MemorySpace getDWARFMemorySpace() const { return DWARFMemorySpace; }
 
-  std::optional<PtrAuthData> getPtrAuthData() const;
+  LLVM_ABI std::optional<PtrAuthData> getPtrAuthData() const;
 
   /// Get extra data associated with this derived type.
   /// Class type for pointer-to-members, objective-c property node for ivars,
@@ -1295,19 +1307,19 @@ class DIDerivedType : public DIType {
 
   /// Get casted version of extra data.
   /// @{
-  DIType *getClassType() const;
+  LLVM_ABI DIType *getClassType() const;
 
   DIObjCProperty *getObjCProperty() const {
     return dyn_cast_or_null<DIObjCProperty>(getExtraData());
   }
 
-  uint32_t getVBPtrOffset() const;
+  LLVM_ABI uint32_t getVBPtrOffset() const;
 
-  Constant *getStorageOffsetInBits() const;
+  LLVM_ABI Constant *getStorageOffsetInBits() const;
 
-  Constant *getConstant() const;
+  LLVM_ABI Constant *getConstant() const;
 
-  Constant *getDiscriminantValue() const;
+  LLVM_ABI Constant *getDiscriminantValue() const;
   /// @}
 
   static bool classof(const Metadata *MD) {
@@ -1352,13 +1364,12 @@ class DISubrangeType : public DIType {
                    UpperBound, Stride, Bias, Storage, ShouldCreate);
   }
 
-  static DISubrangeType *getImpl(LLVMContext &Context, MDString *Name,
-                                 Metadata *File, unsigned Line, Metadata *Scope,
-                                 uint64_t SizeInBits, uint32_t AlignInBits,
-                                 DIFlags Flags, Metadata *BaseType,
-                                 Metadata *LowerBound, Metadata *UpperBound,
-                                 Metadata *Stride, Metadata *Bias,
-                                 StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DISubrangeType *
+  getImpl(LLVMContext &Context, MDString *Name, Metadata *File, unsigned Line,
+          Metadata *Scope, uint64_t SizeInBits, uint32_t AlignInBits,
+          DIFlags Flags, Metadata *BaseType, Metadata *LowerBound,
+          Metadata *UpperBound, Metadata *Stride, Metadata *Bias,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDISubrangeType cloneImpl() const {
     return getTemporary(getContext(), getName(), getFile(), getLine(),
@@ -1367,7 +1378,7 @@ class DISubrangeType : public DIType {
                         getRawUpperBound(), getRawStride(), getRawBias());
   }
 
-  BoundType convertRawToBound(Metadata *IN) const;
+  LLVM_ABI BoundType convertRawToBound(Metadata *IN) const;
 
 public:
   DEFINE_MDNODE_GET(DISubrangeType,
@@ -1470,7 +1481,7 @@ class DICompositeType : public DIType {
         Associated, Allocated, Rank, Annotations.get(), Specification,
         NumExtraInhabitants, BitStride, Storage, ShouldCreate);
   }
-  static DICompositeType *
+  LLVM_ABI static DICompositeType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
           unsigned Line, Metadata *Scope, Metadata *BaseType,
           uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
@@ -1540,7 +1551,7 @@ class DICompositeType : public DIType {
   /// a new node.
   ///
   /// Else, returns \c nullptr.
-  static DICompositeType *
+  LLVM_ABI static DICompositeType *
   getODRType(LLVMContext &Context, MDString &Identifier, unsigned Tag,
              MDString *Name, Metadata *File, unsigned Line, Metadata *Scope,
              Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits,
@@ -1551,8 +1562,8 @@ class DICompositeType : public DIType {
              Metadata *Discriminator, Metadata *DataLocation,
              Metadata *Associated, Metadata *Allocated, Metadata *Rank,
              Metadata *Annotations, Metadata *BitStride);
-  static DICompositeType *getODRTypeIfExists(LLVMContext &Context,
-                                             MDString &Identifier);
+  LLVM_ABI static DICompositeType *getODRTypeIfExists(LLVMContext &Context,
+                                                      MDString &Identifier);
 
   /// Build a DICompositeType with the given ODR identifier.
   ///
@@ -1563,7 +1574,7 @@ class DICompositeType : public DIType {
   ///
   /// If not \a LLVMContext::isODRUniquingDebugTypes(), this function returns
   /// nullptr.
-  static DICompositeType *
+  LLVM_ABI static DICompositeType *
   buildODRType(LLVMContext &Context, MDString &Identifier, unsigned Tag,
                MDString *Name, Metadata *File, unsigned Line, Metadata *Scope,
                Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits,
@@ -1696,10 +1707,10 @@ class DISubroutineType : public DIType {
                                    bool ShouldCreate = true) {
     return getImpl(Context, Flags, CC, TypeArray.get(), Storage, ShouldCreate);
   }
-  static DISubroutineType *getImpl(LLVMContext &Context, DIFlags Flags,
-                                   uint8_t CC, Metadata *TypeArray,
-                                   StorageType Storage,
-                                   bool ShouldCreate = true);
+  LLVM_ABI static DISubroutineType *getImpl(LLVMContext &Context, DIFlags Flags,
+                                            uint8_t CC, Metadata *TypeArray,
+                                            StorageType Storage,
+                                            bool ShouldCreate = true);
 
   TempDISubroutineType cloneImpl() const {
     return getTemporary(getContext(), getFlags(), getCC(), getTypeArray());
@@ -1756,10 +1767,12 @@ class DICompileUnit : public DIScope {
     LastDebugNameTableKind = Apple
   };
 
-  static std::optional<DebugEmissionKind> getEmissionKind(StringRef Str);
-  static const char *emissionKindString(DebugEmissionKind EK);
-  static std::optional<DebugNameTableKind> getNameTableKind(StringRef Str);
-  static const char *nameTableKindString(DebugNameTableKind PK);
+  LLVM_ABI static std::optional<DebugEmissionKind>
+  getEmissionKind(StringRef Str);
+  LLVM_ABI static const char *emissionKindString(DebugEmissionKind EK);
+  LLVM_ABI static std::optional<DebugNameTableKind>
+  getNameTableKind(StringRef Str);
+  LLVM_ABI static const char *nameTableKindString(DebugNameTableKind PK);
 
 private:
   unsigned SourceLanguage;
@@ -1800,7 +1813,7 @@ class DICompileUnit : public DIScope {
         getCanonicalMDString(Context, SysRoot),
         getCanonicalMDString(Context, SDK), Storage, ShouldCreate);
   }
-  static DICompileUnit *
+  LLVM_ABI static DICompileUnit *
   getImpl(LLVMContext &Context, unsigned SourceLanguage, Metadata *File,
           MDString *Producer, bool IsOptimized, MDString *Flags,
           unsigned RuntimeVersion, MDString *SplitDebugFilename,
@@ -1953,11 +1966,11 @@ class DILocalScope : public DIScope {
   ///
   /// Return this if it's an \a DISubprogram; otherwise, look up the scope
   /// chain.
-  DISubprogram *getSubprogram() const;
+  LLVM_ABI DISubprogram *getSubprogram() const;
 
   /// Traverses the scope chain rooted at RootScope until it hits a Subprogram,
   /// recreating the chain with "NewSP" instead.
-  static DILocalScope *
+  LLVM_ABI static DILocalScope *
   cloneScopeForSubprogram(DILocalScope &RootScope, DISubprogram &NewSP,
                           LLVMContext &Ctx,
                           DenseMap<const MDNode *, MDNode *> &Cache);
@@ -1966,7 +1979,7 @@ class DILocalScope : public DIScope {
   ///
   /// Return this if it's not a \a DILexicalBlockFIle; otherwise, look up the
   /// scope chain.
-  DILocalScope *getNonLexicalBlockFileScope() const;
+  LLVM_ABI DILocalScope *getNonLexicalBlockFileScope() const;
 
   static bool classof(const Metadata *MD) {
     return MD->getMetadataID() == DISubprogramKind ||
@@ -2000,21 +2013,21 @@ class DISubprogram : public DILocalScope {
     LLVM_MARK_AS_BITMASK_ENUM(SPFlagLargest)
   };
 
-  static DISPFlags getFlag(StringRef Flag);
-  static StringRef getFlagString(DISPFlags Flag);
+  LLVM_ABI static DISPFlags getFlag(StringRef Flag);
+  LLVM_ABI static StringRef getFlagString(DISPFlags Flag);
 
   /// Split up a flags bitfield for easier printing.
   ///
   /// Split \c Flags into \c SplitFlags, a vector of its components.  Returns
   /// any remaining (unrecognized) bits.
-  static DISPFlags splitFlags(DISPFlags Flags,
-                              SmallVectorImpl<DISPFlags> &SplitFlags);
+  LLVM_ABI static DISPFlags splitFlags(DISPFlags Flags,
+                                       SmallVectorImpl<DISPFlags> &SplitFlags);
 
   // Helper for converting old bitfields to new flags word.
-  static DISPFlags toSPFlags(bool IsLocalToUnit, bool IsDefinition,
-                             bool IsOptimized,
-                             unsigned Virtuality = SPFlagNonvirtual,
-                             bool IsMainSubprogram = false);
+  LLVM_ABI static DISPFlags toSPFlags(bool IsLocalToUnit, bool IsDefinition,
+                                      bool IsOptimized,
+                                      unsigned Virtuality = SPFlagNonvirtual,
+                                      bool IsMainSubprogram = false);
 
 private:
   DIFlags Flags;
@@ -2043,7 +2056,7 @@ class DISubprogram : public DILocalScope {
                    getCanonicalMDString(Context, TargetFuncName),
                    Storage, ShouldCreate);
   }
-  static DISubprogram *
+  LLVM_ABI static DISubprogram *
   getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
           MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type,
           unsigned ScopeLine, Metadata *ContainingType, unsigned VirtualIndex,
@@ -2240,7 +2253,7 @@ class DISubprogram : public DILocalScope {
   /// Check if this subprogram describes the given function.
   ///
   /// FIXME: Should this be looking through bitcasts?
-  bool describes(const Function *F) const;
+  LLVM_ABI bool describes(const Function *F) const;
 
   static bool classof(const Metadata *MD) {
     return MD->getMetadataID() == DISubprogramKind;
@@ -2267,11 +2280,10 @@ class DILocation : public MDNode {
              ArrayRef<Metadata *> MDs, bool ImplicitCode);
   ~DILocation() { dropAllReferences(); }
 
-  static DILocation *getImpl(LLVMContext &Context, unsigned Line,
-                             unsigned Column, Metadata *Scope,
-                             Metadata *InlinedAt, bool ImplicitCode,
-                             uint64_t AtomGroup, uint8_t AtomRank,
-                             StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DILocation *
+  getImpl(LLVMContext &Context, unsigned Line, unsigned Column, Metadata *Scope,
+          Metadata *InlinedAt, bool ImplicitCode, uint64_t AtomGroup,
+          uint8_t AtomRank, StorageType Storage, bool ShouldCreate = true);
   static DILocation *getImpl(LLVMContext &Context, unsigned Line,
                              unsigned Column, DILocalScope *Scope,
                              DILocation *InlinedAt, bool ImplicitCode,
@@ -2451,27 +2463,24 @@ class DILocation : public MDNode {
   inline std::optional<const DILocation *>
   cloneByMultiplyingDuplicationFactor(unsigned DF) const;
 
-  /// When two instructions are combined into a single instruction we also
-  /// need to combine the original locations into a single location.
-  /// When the locations are the same we can use either location.
-  /// When they differ, we need a third location which is distinct from either.
-  /// If they share a common scope, use this scope and compare the line/column
-  /// pair of the locations with the common scope:
-  /// * if both match, keep the line and column;
-  /// * if only the line number matches, keep the line and set the column as 0;
-  /// * otherwise set line and column as 0.
-  /// If they do not share a common scope the location is ambiguous and can't be
-  /// represented in a line entry. In this case, set line and column as 0 and
-  /// use the scope of any location.
-  ///
-  /// \p LocA \p LocB: The locations to be merged.
-  static DILocation *getMergedLocation(DILocation *LocA, DILocation *LocB);
+  /// Attempts to merge \p LocA and \p LocB into a single location; see
+  /// DebugLoc::getMergedLocation for more details.
+  /// NB: When merging the locations of instructions, prefer to use
+  /// DebugLoc::getMergedLocation(), as an instruction's DebugLoc may contain
+  /// additional metadata that will not be preserved when merging the unwrapped
+  /// DILocations.
+  LLVM_ABI static DILocation *getMergedLocation(DILocation *LocA,
+                                                DILocation *LocB);
 
   /// Try to combine the vector of locations passed as input in a single one.
   /// This function applies getMergedLocation() repeatedly left-to-right.
+  /// NB: When merging the locations of instructions, prefer to use
+  /// DebugLoc::getMergedLocations(), as an instruction's DebugLoc may contain
+  /// additional metadata that will not be preserved when merging the unwrapped
+  /// DILocations.
   ///
   /// \p Locs: The locations to be merged.
-  static DILocation *getMergedLocations(ArrayRef<DILocation *> Locs);
+  LLVM_ABI static DILocation *getMergedLocations(ArrayRef<DILocation *> Locs);
 
   /// Return the masked discriminator value for an input discrimnator value D
   /// (i.e. zero out the (B+1)-th and above bits for D (B is 0-base).
@@ -2520,12 +2529,12 @@ class DILocation : public MDNode {
   /// The return is std::nullopt if the values cannot be encoded in 32 bits -
   /// for example, values for BD or DF larger than 12 bits. Otherwise, the
   /// return is the encoded value.
-  static std::optional<unsigned> encodeDiscriminator(unsigned BD, unsigned DF,
-                                                     unsigned CI);
+  LLVM_ABI static std::optional<unsigned>
+  encodeDiscriminator(unsigned BD, unsigned DF, unsigned CI);
 
   /// Raw decoder for values in an encoded discriminator D.
-  static void decodeDiscriminator(unsigned D, unsigned &BD, unsigned &DF,
-                                  unsigned &CI);
+  LLVM_ABI static void decodeDiscriminator(unsigned D, unsigned &BD,
+                                           unsigned &DF, unsigned &CI);
 
   /// Returns the duplication factor for a given encoded discriminator \p D, or
   /// 1 if no value or 0 is encoded.
@@ -2559,8 +2568,8 @@ class DILocation : public MDNode {
 
 class DILexicalBlockBase : public DILocalScope {
 protected:
-  DILexicalBlockBase(LLVMContext &C, unsigned ID, StorageType Storage,
-                     ArrayRef<Metadata *> Ops);
+  LLVM_ABI DILexicalBlockBase(LLVMContext &C, unsigned ID, StorageType Storage,
+                              ArrayRef<Metadata *> Ops);
   ~DILexicalBlockBase() = default;
 
 public:
@@ -2606,9 +2615,10 @@ class DILexicalBlock : public DILexicalBlockBase {
                    ShouldCreate);
   }
 
-  static DILexicalBlock *getImpl(LLVMContext &Context, Metadata *Scope,
-                                 Metadata *File, unsigned Line, unsigned Column,
-                                 StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DILexicalBlock *getImpl(LLVMContext &Context, Metadata *Scope,
+                                          Metadata *File, unsigned Line,
+                                          unsigned Column, StorageType Storage,
+                                          bool ShouldCreate = true);
 
   TempDILexicalBlock cloneImpl() const {
     return getTemporary(getContext(), getScope(), getFile(), getLine(),
@@ -2655,10 +2665,11 @@ class DILexicalBlockFile : public DILexicalBlockBase {
                    ShouldCreate);
   }
 
-  static DILexicalBlockFile *getImpl(LLVMContext &Context, Metadata *Scope,
-                                     Metadata *File, unsigned Discriminator,
-                                     StorageType Storage,
-                                     bool ShouldCreate = true);
+  LLVM_ABI static DILexicalBlockFile *getImpl(LLVMContext &Context,
+                                              Metadata *Scope, Metadata *File,
+                                              unsigned Discriminator,
+                                              StorageType Storage,
+                                              bool ShouldCreate = true);
 
   TempDILexicalBlockFile cloneImpl() const {
     return getTemporary(getContext(), getScope(), getFile(),
@@ -2776,9 +2787,10 @@ class DINamespace : public DIScope {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name),
                    ExportSymbols, Storage, ShouldCreate);
   }
-  static DINamespace *getImpl(LLVMContext &Context, Metadata *Scope,
-                              MDString *Name, bool ExportSymbols,
-                              StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DINamespace *getImpl(LLVMContext &Context, Metadata *Scope,
+                                       MDString *Name, bool ExportSymbols,
+                                       StorageType Storage,
+                                       bool ShouldCreate = true);
 
   TempDINamespace cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(),
@@ -2830,11 +2842,11 @@ class DIModule : public DIScope {
                    getCanonicalMDString(Context, APINotesFile), LineNo, IsDecl,
                    Storage, ShouldCreate);
   }
-  static DIModule *getImpl(LLVMContext &Context, Metadata *File,
-                           Metadata *Scope, MDString *Name,
-                           MDString *ConfigurationMacros, MDString *IncludePath,
-                           MDString *APINotesFile, unsigned LineNo, bool IsDecl,
-                           StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DIModule *
+  getImpl(LLVMContext &Context, Metadata *File, Metadata *Scope, MDString *Name,
+          MDString *ConfigurationMacros, MDString *IncludePath,
+          MDString *APINotesFile, unsigned LineNo, bool IsDecl,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDIModule cloneImpl() const {
     return getTemporary(getContext(), getFile(), getScope(), getName(),
@@ -2922,10 +2934,9 @@ class DITemplateTypeParameter : public DITemplateParameter {
     return getImpl(Context, getCanonicalMDString(Context, Name), Type,
                    IsDefault, Storage, ShouldCreate);
   }
-  static DITemplateTypeParameter *getImpl(LLVMContext &Context, MDString *Name,
-                                          Metadata *Type, bool IsDefault,
-                                          StorageType Storage,
-                                          bool ShouldCreate = true);
+  LLVM_ABI static DITemplateTypeParameter *
+  getImpl(LLVMContext &Context, MDString *Name, Metadata *Type, bool IsDefault,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDITemplateTypeParameter cloneImpl() const {
     return getTemporary(getContext(), getName(), getType(), isDefault());
@@ -2965,11 +2976,10 @@ class DITemplateValueParameter : public DITemplateParameter {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name), Type,
                    IsDefault, Value, Storage, ShouldCreate);
   }
-  static DITemplateValueParameter *getImpl(LLVMContext &Context, unsigned Tag,
-                                           MDString *Name, Metadata *Type,
-                                           bool IsDefault, Metadata *Value,
-                                           StorageType Storage,
-                                           bool ShouldCreate = true);
+  LLVM_ABI static DITemplateValueParameter *
+  getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *Type,
+          bool IsDefault, Metadata *Value, StorageType Storage,
+          bool ShouldCreate = true);
 
   TempDITemplateValueParameter cloneImpl() const {
     return getTemporary(getContext(), getTag(), getName(), getType(),
@@ -3053,9 +3063,10 @@ class DIVariable : public DIObject {
   dwarf::MemorySpace MemorySpace;
 
 protected:
-  DIVariable(LLVMContext &C, unsigned ID, StorageType Storage, signed Line,
-             ArrayRef<Metadata *> Ops, dwarf::MemorySpace MS,
-             uint32_t AlignInBits = 0);
+  LLVM_ABI DIVariable(LLVMContext &C, unsigned ID, StorageType Storage,
+                      signed Line, ArrayRef<Metadata *> Ops,
+                      dwarf::MemorySpace MS,
+                      uint32_t AlignInBits = 0);
   ~DIVariable() = default;
 
 public:
@@ -3067,7 +3078,7 @@ class DIVariable : public DIObject {
   uint32_t getAlignInBits() const { return SubclassData32; }
   uint32_t getAlignInBytes() const { return getAlignInBits() / CHAR_BIT; }
   /// Determines the size of the variable's type.
-  std::optional<uint64_t> getSizeInBits() const;
+  LLVM_ABI std::optional<uint64_t> getSizeInBits() const;
 
   /// Return the signedness of this variable's type, or std::nullopt if this
   /// type is neither signed nor unsigned.
@@ -3601,12 +3612,15 @@ class DIExpression : public MDNode {
   ~DIExpression() = default;
 
   // FIXME: workaround to avoid updating callsites for now
+  LLVM_ABI
   static DIExpression *getImpl(LLVMContext &Context, std::nullopt_t Elements,
                                StorageType Storage, bool ShouldCreate = true);
 
+  LLVM_ABI
   static DIExpression *getImpl(LLVMContext &Context, OldElementsRef Elements,
                                StorageType Storage, bool ShouldCreate = true);
 
+  LLVM_ABI
   static DIExpression *getImpl(LLVMContext &Context, bool /*ignored*/,
                                NewElementsRef Elements, StorageType Storage,
                                bool ShouldCreate = true);
@@ -3672,7 +3686,7 @@ class DIExpression : public MDNode {
   enum SignedOrUnsignedConstant { SignedConstant, UnsignedConstant };
   /// Determine whether this represents a constant value, if so
   // return it's sign information.
-  std::optional<SignedOrUnsignedConstant> isConstant() const;
+  LLVM_ABI std::optional<SignedOrUnsignedConstant> isConstant() const;
 
   /// Return the number of unique location operands referred to (via
   /// DW_OP_LLVM_arg) in this expression; this is not necessarily the number of
@@ -3682,7 +3696,7 @@ class DIExpression : public MDNode {
   ///    DW_OP_LLVM_arg 0, DW_OP_mul)
   /// This function would return 2, as there are two unique location operands
   /// (0 and 1).
-  uint64_t getNumLocationOperands() const;
+  LLVM_ABI uint64_t getNumLocationOperands() const;
 
   /// Return the number of unique location operands referred to (via DIOpArg) in
   /// this expression. Like getNumLocationOperands, but for DIOp-DIExpressions.
@@ -3693,6 +3707,19 @@ class DIExpression : public MDNode {
   element_iterator elements_begin() const { return getElements().begin(); }
   element_iterator elements_end() const { return getElements().end(); }
 
+  /// Returns the pointer address space this DIOp-based DIExpression produces.
+  /// Note that this may diverge from the the pointer address space of the
+  /// result type. When there is a divergent address space, the DIExpression
+  /// must produce a generic pointer whose value can be proven to belong to a
+  /// more specific address space. For instance in this expression, this
+  /// function returns 4:
+  ///
+  ///   !DIExpression(DIOpArg(0, ptr addrspace(4)), DIOpConvert(ptr))
+  ///
+  /// A divergent address space can be created by a DIOpConvert, and is
+  /// preserved across DIOpReinterpret.
+  std::optional<unsigned> getNewDivergentAddrSpace() const;
+
   /// A lightweight wrapper around an expression operand.
   ///
   /// TODO: Store arguments directly and change \a DIExpression to store a
@@ -3719,7 +3746,7 @@ class DIExpression : public MDNode {
     /// Return the size of the operand.
     ///
     /// Return the number of elements in the operand (1 + args).
-    unsigned getSize() const;
+    LLVM_ABI unsigned getSize() const;
 
     /// Append the elements of this operand to \p V.
     void appendToVector(SmallVectorImpl<uint64_t> &V) const {
@@ -3792,7 +3819,7 @@ class DIExpression : public MDNode {
   }
   /// @}
 
-  bool isValid(std::optional<DIExpressionEnv> Env = std::nullopt,
+  LLVM_ABI bool isValid(std::optional<DIExpressionEnv> Env = std::nullopt,
                std::optional<std::reference_wrapper<llvm::raw_ostream>> ErrS =
                    std::nullopt) const;
 
@@ -3801,10 +3828,10 @@ class DIExpression : public MDNode {
   }
 
   /// Return whether the first element a DW_OP_deref.
-  bool startsWithDeref() const;
+  LLVM_ABI bool startsWithDeref() const;
 
   /// Return whether there is exactly one operator and it is a DW_OP_deref;
-  bool isDeref() const;
+  LLVM_ABI bool isDeref() const;
 
   using FragmentInfo = DbgVariableFragmentInfo;
 
@@ -3812,11 +3839,11 @@ class DIExpression : public MDNode {
   /// aren't known to be zero/sign (depending on the type of Var) and which
   /// are within the size of this fragment (if it is one). If we can't deduce
   /// anything from the expression this will return the size of Var.
-  std::optional<uint64_t> getActiveBits(DIVariable *Var);
+  LLVM_ABI std::optional<uint64_t> getActiveBits(DIVariable *Var);
 
   /// Retrieve the details of this fragment expression.
-  static std::optional<FragmentInfo> getFragmentInfo(expr_op_iterator Start,
-                                                     expr_op_iterator End);
+  LLVM_ABI static std::optional<FragmentInfo>
+  getFragmentInfo(expr_op_iterator Start, expr_op_iterator End);
 
   static std::optional<FragmentInfo> getFragmentInfo(NewElementsRef E);
 
@@ -3831,23 +3858,24 @@ class DIExpression : public MDNode {
   bool isFragment() const { return getFragmentInfo().has_value(); }
 
   /// Return whether this is an implicit location description.
-  bool isImplicit() const;
+  LLVM_ABI bool isImplicit() const;
 
   /// Return whether the location is computed on the expression stack, meaning
   /// it cannot be a simple register location.
-  bool isComplex() const;
+  LLVM_ABI bool isComplex() const;
 
   /// Return whether the evaluated expression makes use of a single location at
   /// the start of the expression, i.e. if it contains only a single
   /// DW_OP_LLVM_arg op as its first operand, or if it contains none.
-  bool isSingleLocationExpression() const;
+  LLVM_ABI bool isSingleLocationExpression() const;
 
   /// Returns a reference to the elements contained in this expression, skipping
   /// past the leading `DW_OP_LLVM_arg, 0` if one is present.
   /// Similar to `convertToNonVariadicExpression`, but faster and cheaper - it
   /// does not check whether the expression is a single-location expression, and
   /// it returns elements rather than creating a new DIExpression.
-  std::optional<ArrayRef<uint64_t>> getSingleLocationExpressionElements() const;
+  LLVM_ABI std::optional<ArrayRef<uint64_t>>
+  getSingleLocationExpressionElements() const;
 
   /// Removes all elements from \p Expr that do not apply to an undef debug
   /// value, which includes every operator that computes the value/location on
@@ -3855,12 +3883,13 @@ class DIExpression : public MDNode {
   /// of this function always a single-location expression) while leaving
   /// everything that defines what the computed value applies to, i.e. the
   /// fragment information.
-  static const DIExpression *convertToUndefExpression(const DIExpression *Expr);
+  LLVM_ABI static const DIExpression *
+  convertToUndefExpression(const DIExpression *Expr);
 
   /// If \p Expr is a non-variadic expression (i.e. one that does not contain
   /// DW_OP_LLVM_arg), returns \p Expr converted to variadic form by adding a
   /// leading [DW_OP_LLVM_arg, 0] to the expression; otherwise returns \p Expr.
-  static const DIExpression *
+  LLVM_ABI static const DIExpression *
   convertToVariadicExpression(const DIExpression *Expr);
 
   /// If \p Expr is a valid single-location expression, i.e. it refers to only a
@@ -3870,7 +3899,7 @@ class DIExpression : public MDNode {
   /// See also `getSingleLocationExpressionElements` above, which skips
   /// checking `isSingleLocationExpression` and returns a list of elements
   /// rather than a DIExpression.
-  static std::optional<const DIExpression *>
+  LLVM_ABI static std::optional<const DIExpression *>
   convertToNonVariadicExpression(const DIExpression *Expr);
 
   /// Inserts the elements of \p Expr into \p Ops modified to a canonical form,
@@ -3878,9 +3907,9 @@ class DIExpression : public MDNode {
   /// implied derefence from the \p IsIndirect flag into the expression. This
   /// allows us to check equivalence between expressions with differing
   /// directness or variadicness.
-  static void canonicalizeExpressionOps(SmallVectorImpl<uint64_t> &Ops,
-                                        const DIExpression *Expr,
-                                        bool IsIndirect);
+  LLVM_ABI static void canonicalizeExpressionOps(SmallVectorImpl<uint64_t> &Ops,
+                                                 const DIExpression *Expr,
+                                                 bool IsIndirect);
 
   /// Determines whether two debug values should produce equivalent DWARF
   /// expressions, using their DIExpressions and directness, ignoring the
@@ -3893,17 +3922,18 @@ class DIExpression : public MDNode {
   /// whose second operand is an immediate value.
   /// \p SecondExpr and \p SecondIndirect have the same meaning as the prior
   /// arguments, but apply to the second debug value.
-  static bool isEqualExpression(const DIExpression *FirstExpr,
-                                bool FirstIndirect,
-                                const DIExpression *SecondExpr,
-                                bool SecondIndirect);
+  LLVM_ABI static bool isEqualExpression(const DIExpression *FirstExpr,
+                                         bool FirstIndirect,
+                                         const DIExpression *SecondExpr,
+                                         bool SecondIndirect);
 
   /// Append \p Ops with operations to apply the \p Offset.
-  static void appendOffset(SmallVectorImpl<uint64_t> &Ops, int64_t Offset);
+  LLVM_ABI static void appendOffset(SmallVectorImpl<uint64_t> &Ops,
+                                    int64_t Offset);
 
   /// If this is a constant offset, extract it. If there is no expression,
   /// return true with an offset of zero.
-  bool extractIfOffset(int64_t &Offset) const;
+  LLVM_ABI bool extractIfOffset(int64_t &Offset) const;
 
   /// Assuming that the expression operates on an address, extract a constant
   /// offset and the successive ops. Return false if the expression contains
@@ -3912,18 +3942,19 @@ class DIExpression : public MDNode {
   ///
   /// We don't try very hard to interpret the expression because we assume that
   /// foldConstantMath has canonicalized the expression.
-  bool extractLeadingOffset(int64_t &OffsetInBytes,
-                            SmallVectorImpl<uint64_t> &RemainingOps) const;
+  LLVM_ABI bool
+  extractLeadingOffset(int64_t &OffsetInBytes,
+                       SmallVectorImpl<uint64_t> &RemainingOps) const;
 
   /// Returns true iff this DIExpression contains at least one instance of
   /// `DW_OP_LLVM_arg, n` for all n in [0, N).
-  bool hasAllLocationOps(unsigned N) const;
+  LLVM_ABI bool hasAllLocationOps(unsigned N) const;
 
   /// Checks if the last 4 elements of the expression are DW_OP_constu <DWARF
   /// Address Space> DW_OP_swap DW_OP_xderef and extracts the <DWARF Address
   /// Space>.
-  static const DIExpression *extractAddressClass(const DIExpression *Expr,
-                                                 unsigned &AddrClass);
+  LLVM_ABI static const DIExpression *
+  extractAddressClass(const DIExpression *Expr, unsigned &AddrClass);
 
   /// Used for DIExpression::prepend.
   enum PrependOps : uint8_t {
@@ -3936,36 +3967,38 @@ class DIExpression : public MDNode {
 
   /// Prepend \p DIExpr with a deref and offset operation and optionally turn it
   /// into a stack value or/and an entry value.
-  static DIExpression *prepend(const DIExpression *Expr, uint8_t Flags,
-                               int64_t Offset = 0);
+  LLVM_ABI static DIExpression *prepend(const DIExpression *Expr, uint8_t Flags,
+                                        int64_t Offset = 0);
 
   /// Prepend \p DIExpr with the given opcodes and optionally turn it into a
   /// stack value.
-  static DIExpression *prependOpcodes(const DIExpression *Expr,
-                                      SmallVectorImpl<uint64_t> &Ops,
-                                      bool StackValue = false,
-                                      bool EntryValue = false);
+  LLVM_ABI static DIExpression *prependOpcodes(const DIExpression *Expr,
+                                               SmallVectorImpl<uint64_t> &Ops,
+                                               bool StackValue = false,
+                                               bool EntryValue = false);
 
   /// Append the opcodes \p Ops to \p DIExpr. Unlike \ref appendToStack, the
   /// returned expression is a stack value only if \p DIExpr is a stack value.
   /// If \p DIExpr describes a fragment, the returned expression will describe
   /// the same fragment.
-  static DIExpression *append(const DIExpression *Expr, ArrayRef<uint64_t> Ops);
+  LLVM_ABI static DIExpression *append(const DIExpression *Expr,
+                                       ArrayRef<uint64_t> Ops);
 
   /// Convert \p DIExpr into a stack value if it isn't one already by appending
   /// DW_OP_deref if needed, and appending \p Ops to the resulting expression.
   /// If \p DIExpr describes a fragment, the returned expression will describe
   /// the same fragment.
-  static DIExpression *appendToStack(const DIExpression *Expr,
-                                     ArrayRef<uint64_t> Ops);
+  LLVM_ABI static DIExpression *appendToStack(const DIExpression *Expr,
+                                              ArrayRef<uint64_t> Ops);
 
   /// Create a copy of \p Expr by appending the given list of \p Ops to each
   /// instance of the operand `DW_OP_LLVM_arg, \p ArgNo`. This is used to
   /// modify a specific location used by \p Expr, such as when salvaging that
   /// location.
-  static DIExpression *appendOpsToArg(const DIExpression *Expr,
-                                      ArrayRef<uint64_t> Ops, unsigned ArgNo,
-                                      bool StackValue = false);
+  LLVM_ABI static DIExpression *appendOpsToArg(const DIExpression *Expr,
+                                               ArrayRef<uint64_t> Ops,
+                                               unsigned ArgNo,
+                                               bool StackValue = false);
 
   /// Create a copy of \p Expr by appending the given list of \p Ops to each
   /// instance of the operand `DIOpArg(ArgNo, OldArgType)`, updating OldArgType
@@ -3982,8 +4015,8 @@ class DIExpression : public MDNode {
   /// for all Arg > \p OldArg.
   /// This is used when replacing one of the operands of a debug value list
   /// with another operand in the same list and deleting the old operand.
-  static DIExpression *replaceArg(const DIExpression *Expr, uint64_t OldArg,
-                                  uint64_t NewArg);
+  LLVM_ABI static DIExpression *replaceArg(const DIExpression *Expr,
+                                           uint64_t OldArg, uint64_t NewArg);
 
   /// Create a DIExpression to describe one part of an aggregate variable that
   /// is fragmented across multiple Values. The DW_OP_LLVM_fragment operation
@@ -3996,7 +4029,7 @@ class DIExpression : public MDNode {
   /// \return             Creating a fragment expression may fail if \c Expr
   ///                     contains arithmetic operations that would be
   ///                     truncated.
-  static std::optional<DIExpression *>
+  LLVM_ABI static std::optional<DIExpression *>
   createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits,
                            unsigned SizeInBits);
 
@@ -4046,7 +4079,7 @@ class DIExpression : public MDNode {
   /// and offsets because different debug records store the information in
   /// different places (dbg_assign has two DIExpressions - one contains the
   /// fragment info for the entire intrinsic).
-  static bool calculateFragmentIntersect(
+  LLVM_ABI static bool calculateFragmentIntersect(
       const DataLayout &DL, const Value *SliceStart, uint64_t SliceOffsetInBits,
       uint64_t SliceSizeInBits, const Value *DbgPtr, int64_t DbgPtrOffsetInBits,
       int64_t DbgExtractOffsetInBits, DIExpression::FragmentInfo VarFrag,
@@ -4056,12 +4089,14 @@ class DIExpression : public MDNode {
   using ExtOps = std::array<uint64_t, 6>;
 
   /// Returns the ops for a zero- or sign-extension in a DIExpression.
-  static ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed);
+  LLVM_ABI static ExtOps getExtOps(unsigned FromSize, unsigned ToSize,
+                                   bool Signed);
 
   /// Append a zero- or sign-extension to \p Expr. Converts the expression to a
   /// stack value if it isn't one already.
-  static DIExpression *appendExt(const DIExpression *Expr, unsigned FromSize,
-                                 unsigned ToSize, bool Signed);
+  LLVM_ABI static DIExpression *appendExt(const DIExpression *Expr,
+                                          unsigned FromSize, unsigned ToSize,
+                                          bool Signed);
 
   /// Check if fragments overlap between a pair of FragmentInfos.
   static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B) {
@@ -4085,18 +4120,18 @@ class DIExpression : public MDNode {
 
   /// Check if the expression consists of exactly one entry value operand.
   /// (This is the only configuration of entry values that is supported.)
-  bool isEntryValue() const;
+  LLVM_ABI bool isEntryValue() const;
 
   /// Try to shorten an expression with an initial constant operand.
   /// Returns a new expression and constant on success, or the original
   /// expression and constant on failure.
-  std::pair<DIExpression *, const ConstantInt *>
+  LLVM_ABI std::pair<DIExpression *, const ConstantInt *>
   constantFold(const ConstantInt *CI);
 
   /// Try to shorten an expression with constant math operations that can be
   /// evaluated at compile time. Returns a new expression on success, or the old
   /// expression if there is nothing to be reduced.
-  DIExpression *foldConstantMath();
+  LLVM_ABI DIExpression *foldConstantMath();
 };
 
 inline bool operator==(const DIExpression::FragmentInfo &A,
@@ -4415,7 +4450,7 @@ class DIGlobalVariable : public DIVariable {
                    cast_or_null<Metadata>(TemplateParams), MS, AlignInBits,
                    Annotations.get(), Storage, ShouldCreate);
   }
-  static DIGlobalVariable *
+  LLVM_ABI static DIGlobalVariable *
   getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
           MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type,
           bool IsLocalToUnit, bool IsDefinition,
@@ -4494,10 +4529,11 @@ class DICommonBlock : public DIScope {
     return getImpl(Context, Scope, Decl, getCanonicalMDString(Context, Name),
                    File, LineNo, Storage, ShouldCreate);
   }
-  static DICommonBlock *getImpl(LLVMContext &Context, Metadata *Scope,
-                                Metadata *Decl, MDString *Name, Metadata *File,
-                                unsigned LineNo, StorageType Storage,
-                                bool ShouldCreate = true);
+  LLVM_ABI static DICommonBlock *getImpl(LLVMContext &Context, Metadata *Scope,
+                                         Metadata *Decl, MDString *Name,
+                                         Metadata *File, unsigned LineNo,
+                                         StorageType Storage,
+                                         bool ShouldCreate = true);
 
   TempDICommonBlock cloneImpl() const {
     return getTemporary(getContext(), getScope(), getDecl(), getName(),
@@ -4563,12 +4599,11 @@ class DILocalVariable : public DIVariable {
                    Line, Type, Arg, Flags, MS, AlignInBits, Annotations.get(),
                    Storage, ShouldCreate);
   }
-  static DILocalVariable *getImpl(LLVMContext &Context, Metadata *Scope,
-                                  MDString *Name, Metadata *File, unsigned Line,
-                                  Metadata *Type, unsigned Arg, DIFlags Flags,
-                                  dwarf::MemorySpace MS, uint32_t AlignInBits,
-                                  Metadata *Annotations, StorageType Storage,
-                                  bool ShouldCreate = true);
+  LLVM_ABI static DILocalVariable *
+  getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, Metadata *File,
+          unsigned Line, Metadata *Type, unsigned Arg, DIFlags Flags,
+          dwarf::MemorySpace MS, uint32_t AlignInBits, Metadata *Annotations, StorageType Storage,
+          bool ShouldCreate = true);
 
   TempDILocalVariable cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getFile(),
@@ -4645,9 +4680,10 @@ class DILabel : public DINode {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name), File,
                    Line, Storage, ShouldCreate);
   }
-  static DILabel *getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
-                          Metadata *File, unsigned Line, StorageType Storage,
-                          bool ShouldCreate = true);
+  LLVM_ABI static DILabel *getImpl(LLVMContext &Context, Metadata *Scope,
+                                   MDString *Name, Metadata *File,
+                                   unsigned Line, StorageType Storage,
+                                   bool ShouldCreate = true);
 
   TempDILabel cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getFile(),
@@ -4714,11 +4750,10 @@ class DIObjCProperty : public DINode {
                    getCanonicalMDString(Context, SetterName), Attributes, Type,
                    Storage, ShouldCreate);
   }
-  static DIObjCProperty *getImpl(LLVMContext &Context, MDString *Name,
-                                 Metadata *File, unsigned Line,
-                                 MDString *GetterName, MDString *SetterName,
-                                 unsigned Attributes, Metadata *Type,
-                                 StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DIObjCProperty *
+  getImpl(LLVMContext &Context, MDString *Name, Metadata *File, unsigned Line,
+          MDString *GetterName, MDString *SetterName, unsigned Attributes,
+          Metadata *Type, StorageType Storage, bool ShouldCreate = true);
 
   TempDIObjCProperty cloneImpl() const {
     return getTemporary(getContext(), getName(), getFile(), getLine(),
@@ -4796,7 +4831,7 @@ class DIImportedEntity : public DINode {
                    getCanonicalMDString(Context, Name), Elements.get(), Storage,
                    ShouldCreate);
   }
-  static DIImportedEntity *
+  LLVM_ABI static DIImportedEntity *
   getImpl(LLVMContext &Context, unsigned Tag, Metadata *Scope, Metadata *Entity,
           Metadata *File, unsigned Line, MDString *Name, Metadata *Elements,
           StorageType Storage, bool ShouldCreate = true);
@@ -4850,7 +4885,7 @@ class DIGlobalVariableExpression : public MDNode {
       : MDNode(C, DIGlobalVariableExpressionKind, Storage, Ops) {}
   ~DIGlobalVariableExpression() = default;
 
-  static DIGlobalVariableExpression *
+  LLVM_ABI static DIGlobalVariableExpression *
   getImpl(LLVMContext &Context, Metadata *Variable, Metadata *Expression,
           StorageType Storage, bool ShouldCreate = true);
 
@@ -4953,9 +4988,10 @@ class DIMacro : public DIMacroNode {
     return getImpl(Context, MIType, Line, getCanonicalMDString(Context, Name),
                    getCanonicalMDString(Context, Value), Storage, ShouldCreate);
   }
-  static DIMacro *getImpl(LLVMContext &Context, unsigned MIType, unsigned Line,
-                          MDString *Name, MDString *Value, StorageType Storage,
-                          bool ShouldCreate = true);
+  LLVM_ABI static DIMacro *getImpl(LLVMContext &Context, unsigned MIType,
+                                   unsigned Line, MDString *Name,
+                                   MDString *Value, StorageType Storage,
+                                   bool ShouldCreate = true);
 
   TempDIMacro cloneImpl() const {
     return getTemporary(getContext(), getMacinfoType(), getLine(), getName(),
@@ -5009,9 +5045,10 @@ class DIMacroFile : public DIMacroNode {
                    Elements.get(), Storage, ShouldCreate);
   }
 
-  static DIMacroFile *getImpl(LLVMContext &Context, unsigned MIType,
-                              unsigned Line, Metadata *File, Metadata *Elements,
-                              StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DIMacroFile *getImpl(LLVMContext &Context, unsigned MIType,
+                                       unsigned Line, Metadata *File,
+                                       Metadata *Elements, StorageType Storage,
+                                       bool ShouldCreate = true);
 
   TempDIMacroFile cloneImpl() const {
     return getTemporary(getContext(), getMacinfoType(), getLine(), getFile(),
@@ -5070,12 +5107,13 @@ class DIArgList : public Metadata, ReplaceableMetadataImpl {
   }
   ~DIArgList() { untrack(); }
 
-  void track();
-  void untrack();
+  LLVM_ABI void track();
+  LLVM_ABI void untrack();
   void dropAllReferences(bool Untrack);
 
 public:
-  static DIArgList *get(LLVMContext &Context, ArrayRef<ValueAsMetadata *> Args);
+  LLVM_ABI static DIArgList *get(LLVMContext &Context,
+                                 ArrayRef<ValueAsMetadata *> Args);
 
   ArrayRef<ValueAsMetadata *> getArgs() const { return Args; }
 
@@ -5090,7 +5128,7 @@ class DIArgList : public Metadata, ReplaceableMetadataImpl {
     return ReplaceableMetadataImpl::getAllDbgVariableRecordUsers();
   }
 
-  void handleChangedOperand(void *Ref, Metadata *New);
+  LLVM_ABI void handleChangedOperand(void *Ref, Metadata *New);
 };
 
 /// Represents one lifetime segment of a DIObject.
@@ -5202,11 +5240,11 @@ class DebugVariable {
 
   /// Fragment that will overlap all other fragments. Used as default when
   /// caller demands a fragment.
-  static const FragmentInfo DefaultFragment;
+  LLVM_ABI static const FragmentInfo DefaultFragment;
 
 public:
-  DebugVariable(const DbgVariableIntrinsic *DII);
-  DebugVariable(const DbgVariableRecord *DVR);
+  LLVM_ABI DebugVariable(const DbgVariableIntrinsic *DII);
+  LLVM_ABI DebugVariable(const DbgVariableRecord *DVR);
 
   DebugVariable(const DILocalVariable *Var,
                 std::optional<FragmentInfo> FragmentInfo,
@@ -5273,7 +5311,7 @@ template <> struct DenseMapInfo<DebugVariable> {
 /// information).
 class DebugVariableAggregate : public DebugVariable {
 public:
-  DebugVariableAggregate(const DbgVariableIntrinsic *DVI);
+  LLVM_ABI DebugVariableAggregate(const DbgVariableIntrinsic *DVI);
   DebugVariableAggregate(const DebugVariable &V)
       : DebugVariable(V.getVariable(), std::nullopt, V.getInlinedAt()) {}
 };
diff --git a/external/llvm-project/llvm/include/llvm/IR/DebugLoc.h b/external/llvm-project/llvm/include/llvm/IR/DebugLoc.h
index ffb8ed458e36..999e03b6374a 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DebugLoc.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DebugLoc.h
@@ -16,6 +16,7 @@
 
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/TrackingMDRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
 
 namespace llvm {
@@ -25,7 +26,7 @@ namespace llvm {
   class DILocation;
   class Function;
 
-#if LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
   // Used to represent different "kinds" of DebugLoc, expressing that the
   // instruction it is part of is either normal and should contain a valid
   // DILocation, or otherwise describing the reason why the instruction does
@@ -66,7 +67,7 @@ namespace llvm {
     DILocAndCoverageTracking(const MDNode *Loc)
         : TrackingMDNodeRef(const_cast<MDNode *>(Loc)),
           Kind(DebugLocKind::Normal) {}
-    DILocAndCoverageTracking(const DILocation *Loc);
+    LLVM_ABI DILocAndCoverageTracking(const DILocation *Loc);
     // Explicit DebugLocKind, which always means a nullptr MDNode*.
     DILocAndCoverageTracking(DebugLocKind Kind)
         : TrackingMDNodeRef(nullptr), Kind(Kind) {}
@@ -89,7 +90,7 @@ namespace llvm {
   using DebugLocTrackingRef = DILocAndCoverageTracking;
 #else
   using DebugLocTrackingRef = TrackingMDNodeRef;
-#endif // LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
 
   /// A debug info location.
   ///
@@ -106,7 +107,7 @@ namespace llvm {
     DebugLoc() = default;
 
     /// Construct from an \a DILocation.
-    DebugLoc(const DILocation *L);
+    LLVM_ABI DebugLoc(const DILocation *L);
 
     /// Construct from an \a MDNode.
     ///
@@ -114,14 +115,14 @@ namespace llvm {
     /// accessors will crash.  However, construction from other nodes is
     /// supported in order to handle forward references when reading textual
     /// IR.
-    explicit DebugLoc(const MDNode *N);
+    LLVM_ABI explicit DebugLoc(const MDNode *N);
 
-#if LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
     DebugLoc(DebugLocKind Kind) : Loc(Kind) {}
     DebugLocKind getKind() const { return Loc.Kind; }
 #endif
 
-#if LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
     static inline DebugLoc getTemporary() {
       return DebugLoc(DebugLocKind::Temporary);
     }
@@ -139,13 +140,58 @@ namespace llvm {
     static inline DebugLoc getUnknown() { return DebugLoc(); }
     static inline DebugLoc getCompilerGenerated() { return DebugLoc(); }
     static inline DebugLoc getDropped() { return DebugLoc(); }
-#endif // LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
+
+    /// When two instructions are combined into a single instruction we also
+    /// need to combine the original locations into a single location.
+    /// When the locations are the same we can use either location.
+    /// When they differ, we need a third location which is distinct from
+    /// either. If they share a common scope, use this scope and compare the
+    /// line/column pair of the locations with the common scope:
+    /// * if both match, keep the line and column;
+    /// * if only the line number matches, keep the line and set the column as
+    /// 0;
+    /// * otherwise set line and column as 0.
+    /// If they do not share a common scope the location is ambiguous and can't
+    /// be represented in a line entry. In this case, set line and column as 0
+    /// and use the scope of any location.
+    ///
+    /// \p LocA \p LocB: The locations to be merged.
+    LLVM_ABI static DebugLoc getMergedLocation(DebugLoc LocA, DebugLoc LocB);
+
+    /// Try to combine the vector of locations passed as input in a single one.
+    /// This function applies getMergedLocation() repeatedly left-to-right.
+    ///
+    /// \p Locs: The locations to be merged.
+    LLVM_ABI static DebugLoc getMergedLocations(ArrayRef<DebugLoc> Locs);
+
+    /// If this DebugLoc is non-empty, returns this DebugLoc; otherwise, selects
+    /// \p Other.
+    /// In coverage-tracking builds, this also accounts for whether this or
+    /// \p Other have an annotative DebugLocKind applied, such that if both are
+    /// empty but exactly one has an annotation, we prefer that annotated
+    /// location.
+    DebugLoc orElse(DebugLoc Other) const {
+      if (*this)
+        return *this;
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
+      if (Other)
+        return Other;
+      if (getKind() != DebugLocKind::Normal)
+        return *this;
+      if (Other.getKind() != DebugLocKind::Normal)
+        return Other;
+      return *this;
+#else
+      return Other;
+#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
+    }
 
     /// Get the underlying \a DILocation.
     ///
     /// \pre !*this or \c isa<DILocation>(getAsMDNode()).
     /// @{
-    DILocation *get() const;
+    LLVM_ABI DILocation *get() const;
     operator DILocation *() const { return get(); }
     DILocation *operator->() const { return get(); }
     DILocation &operator*() const { return *get(); }
@@ -166,9 +212,9 @@ namespace llvm {
     /// Rebuild the entire inlined-at chain for this instruction so that the top of
     /// the chain now is inlined-at the new call site.
     /// \param   InlinedAt    The new outermost inlined-at in the chain.
-    static DebugLoc appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt,
-                                    LLVMContext &Ctx,
-                                    DenseMap<const MDNode *, MDNode *> &Cache);
+    LLVM_ABI static DebugLoc
+    appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt, LLVMContext &Ctx,
+                    DenseMap<const MDNode *, MDNode *> &Cache);
 
     /// Return true if the source locations match, ignoring isImplicitCode and
     /// source atom info.
@@ -180,19 +226,19 @@ namespace llvm {
              getInlinedAt() == Other.getInlinedAt();
     }
 
-    unsigned getLine() const;
-    unsigned getCol() const;
-    MDNode *getScope() const;
-    DILocation *getInlinedAt() const;
+    LLVM_ABI unsigned getLine() const;
+    LLVM_ABI unsigned getCol() const;
+    LLVM_ABI MDNode *getScope() const;
+    LLVM_ABI DILocation *getInlinedAt() const;
 
     /// Get the fully inlined-at scope for a DebugLoc.
     ///
     /// Gets the inlined-at scope for a DebugLoc.
-    MDNode *getInlinedAtScope() const;
+    LLVM_ABI MDNode *getInlinedAtScope() const;
 
     /// Rebuild the entire inline-at chain by replacing the subprogram at the
     /// end of the chain with NewSP.
-    static DebugLoc
+    LLVM_ABI static DebugLoc
     replaceInlinedAtSubprogram(const DebugLoc &DL, DISubprogram &NewSP,
                                LLVMContext &Ctx,
                                DenseMap<const MDNode *, MDNode *> &Cache);
@@ -204,22 +250,22 @@ namespace llvm {
     ///
     /// FIXME: Remove this.  Users should use DILocation/DILocalScope API to
     /// find the subprogram, and then DILocation::get().
-    DebugLoc getFnDebugLoc() const;
+    LLVM_ABI DebugLoc getFnDebugLoc() const;
 
     /// Return \c this as a bar \a MDNode.
     MDNode *getAsMDNode() const { return Loc; }
 
     /// Check if the DebugLoc corresponds to an implicit code.
-    bool isImplicitCode() const;
-    void setImplicitCode(bool ImplicitCode);
+    LLVM_ABI bool isImplicitCode() const;
+    LLVM_ABI void setImplicitCode(bool ImplicitCode);
 
     bool operator==(const DebugLoc &DL) const { return Loc == DL.Loc; }
     bool operator!=(const DebugLoc &DL) const { return Loc != DL.Loc; }
 
-    void dump() const;
+    LLVM_ABI void dump() const;
 
     /// prints source location /path/to/file.exe:line:col @[inlined at]
-    void print(raw_ostream &OS) const;
+    LLVM_ABI void print(raw_ostream &OS) const;
   };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/IR/DebugProgramInstruction.h b/external/llvm-project/llvm/include/llvm/IR/DebugProgramInstruction.h
index 37db7894d173..e0292c2b8d2d 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DebugProgramInstruction.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DebugProgramInstruction.h
@@ -55,6 +55,7 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/SymbolTableListTraits.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -117,6 +118,10 @@ template <typename T> class DbgRecordParamRef {
   }
 };
 
+extern template class LLVM_TEMPLATE_ABI DbgRecordParamRef<DIExpression>;
+extern template class LLVM_TEMPLATE_ABI DbgRecordParamRef<DILabel>;
+extern template class LLVM_TEMPLATE_ABI DbgRecordParamRef<DILocalVariable>;
+
 /// Base class for non-instruction debug metadata records that have positions
 /// within IR. Features various methods copied across from the Instruction
 /// class to aid ease-of-use. DbgRecords should always be linked into a
@@ -149,20 +154,21 @@ class DbgRecord : public ilist_node<DbgRecord> {
   /// Methods that dispatch to subclass implementations. These need to be
   /// manually updated when a new subclass is added.
   ///@{
-  void deleteRecord();
-  DbgRecord *clone() const;
-  void print(raw_ostream &O, bool IsForDebug = false) const;
-  void print(raw_ostream &O, ModuleSlotTracker &MST, bool IsForDebug) const;
-  bool isIdenticalToWhenDefined(const DbgRecord &R) const;
+  LLVM_ABI void deleteRecord();
+  LLVM_ABI DbgRecord *clone() const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &O, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+  LLVM_ABI bool isIdenticalToWhenDefined(const DbgRecord &R) const;
   /// Convert this DbgRecord back into an appropriate llvm.dbg.* intrinsic.
   /// \p InsertBefore Optional position to insert this intrinsic.
   /// \returns A new llvm.dbg.* intrinsic representiung this DbgRecord.
-  DbgInfoIntrinsic *createDebugIntrinsic(Module *M,
-                                         Instruction *InsertBefore) const;
+  LLVM_ABI DbgInfoIntrinsic *
+  createDebugIntrinsic(Module *M, Instruction *InsertBefore) const;
   ///@}
 
   /// Same as isIdenticalToWhenDefined but checks DebugLoc too.
-  bool isEquivalentTo(const DbgRecord &R) const;
+  LLVM_ABI bool isEquivalentTo(const DbgRecord &R) const;
 
   Kind getRecordKind() const { return RecordKind; }
 
@@ -171,44 +177,44 @@ class DbgRecord : public ilist_node<DbgRecord> {
   DbgMarker *getMarker() { return Marker; }
   const DbgMarker *getMarker() const { return Marker; }
 
-  BasicBlock *getBlock();
-  const BasicBlock *getBlock() const;
+  LLVM_ABI BasicBlock *getBlock();
+  LLVM_ABI const BasicBlock *getBlock() const;
 
-  Function *getFunction();
-  const Function *getFunction() const;
+  LLVM_ABI Function *getFunction();
+  LLVM_ABI const Function *getFunction() const;
 
-  Module *getModule();
-  const Module *getModule() const;
+  LLVM_ABI Module *getModule();
+  LLVM_ABI const Module *getModule() const;
 
-  LLVMContext &getContext();
-  const LLVMContext &getContext() const;
+  LLVM_ABI LLVMContext &getContext();
+  LLVM_ABI const LLVMContext &getContext() const;
 
-  const Instruction *getInstruction() const;
-  const BasicBlock *getParent() const;
-  BasicBlock *getParent();
+  LLVM_ABI const Instruction *getInstruction() const;
+  LLVM_ABI const BasicBlock *getParent() const;
+  LLVM_ABI BasicBlock *getParent();
 
-  void removeFromParent();
-  void eraseFromParent();
+  LLVM_ABI void removeFromParent();
+  LLVM_ABI void eraseFromParent();
 
   DbgRecord *getNextNode() { return &*std::next(getIterator()); }
   DbgRecord *getPrevNode() { return &*std::prev(getIterator()); }
 
   // Some generic lambdas supporting intrinsic-based debug-info mean we need
   // to support both iterator and instruction position based insertion.
-  void insertBefore(DbgRecord *InsertBefore);
-  void insertAfter(DbgRecord *InsertAfter);
-  void moveBefore(DbgRecord *MoveBefore);
-  void moveAfter(DbgRecord *MoveAfter);
+  LLVM_ABI void insertBefore(DbgRecord *InsertBefore);
+  LLVM_ABI void insertAfter(DbgRecord *InsertAfter);
+  LLVM_ABI void moveBefore(DbgRecord *MoveBefore);
+  LLVM_ABI void moveAfter(DbgRecord *MoveAfter);
 
-  void insertBefore(self_iterator InsertBefore);
-  void insertAfter(self_iterator InsertAfter);
-  void moveBefore(self_iterator MoveBefore);
-  void moveAfter(self_iterator MoveAfter);
+  LLVM_ABI void insertBefore(self_iterator InsertBefore);
+  LLVM_ABI void insertAfter(self_iterator InsertAfter);
+  LLVM_ABI void moveBefore(self_iterator MoveBefore);
+  LLVM_ABI void moveAfter(self_iterator MoveAfter);
 
   DebugLoc getDebugLoc() const { return DbgLoc; }
   void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); }
 
-  void dump() const;
+  LLVM_ABI void dump() const;
 
   using self_iterator = simple_ilist<DbgRecord>::iterator;
   using const_self_iterator = simple_ilist<DbgRecord>::const_iterator;
@@ -237,20 +243,21 @@ class DbgLabelRecord : public DbgRecord {
   DbgLabelRecord(MDNode *Label, MDNode *DL);
 
 public:
-  DbgLabelRecord(DILabel *Label, DebugLoc DL);
+  LLVM_ABI DbgLabelRecord(DILabel *Label, DebugLoc DL);
 
   /// For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved
   /// MDNodes. Trying to access the resulting DbgLabelRecord's fields before
   /// they are resolved, or if they resolve to the wrong type, will result in a
   /// crash.
-  static DbgLabelRecord *createUnresolvedDbgLabelRecord(MDNode *Label,
-                                                        MDNode *DL);
+  LLVM_ABI static DbgLabelRecord *createUnresolvedDbgLabelRecord(MDNode *Label,
+                                                                 MDNode *DL);
 
-  DbgLabelRecord *clone() const;
-  void print(raw_ostream &O, bool IsForDebug = false) const;
-  void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const;
-  DbgLabelInst *createDebugIntrinsic(Module *M,
-                                     Instruction *InsertBefore) const;
+  LLVM_ABI DbgLabelRecord *clone() const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+  LLVM_ABI DbgLabelInst *createDebugIntrinsic(Module *M,
+                                              Instruction *InsertBefore) const;
 
   void setLabel(DILabel *NewLabel) { Label = NewLabel; }
   DILabel *getLabel() const { return Label.get(); }
@@ -296,17 +303,17 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
 public:
   /// Create a new DbgVariableRecord representing the intrinsic \p DVI, for
   /// example the assignment represented by a dbg.value.
-  DbgVariableRecord(const DbgVariableIntrinsic *DVI);
-  DbgVariableRecord(const DbgVariableRecord &DVR);
+  LLVM_ABI DbgVariableRecord(const DbgVariableIntrinsic *DVI);
+  LLVM_ABI DbgVariableRecord(const DbgVariableRecord &DVR);
   /// Directly construct a new DbgVariableRecord representing a dbg.value
   /// intrinsic assigning \p Location to the DV / Expr / DI variable.
-  DbgVariableRecord(Metadata *Location, DILocalVariable *DV, DIExpression *Expr,
-                    const DILocation *DI,
-                    LocationType Type = LocationType::Value);
-  DbgVariableRecord(Metadata *Value, DILocalVariable *Variable,
-                    DIExpression *Expression, DIAssignID *AssignID,
-                    Metadata *Address, DIExpression *AddressExpression,
-                    const DILocation *DI);
+  LLVM_ABI DbgVariableRecord(Metadata *Location, DILocalVariable *DV,
+                             DIExpression *Expr, const DILocation *DI,
+                             LocationType Type = LocationType::Value);
+  LLVM_ABI DbgVariableRecord(Metadata *Value, DILocalVariable *Variable,
+                             DIExpression *Expression, DIAssignID *AssignID,
+                             Metadata *Address, DIExpression *AddressExpression,
+                             const DILocation *DI);
 
 private:
   /// Private constructor for creating new instances during parsing only. Only
@@ -325,36 +332,35 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
   /// for all types of DbgVariableRecords for simplicity while parsing, but
   /// asserts if any necessary fields are empty or unused fields are not empty,
   /// i.e. if the #dbg_assign fields are used for a non-dbg-assign type.
-  static DbgVariableRecord *
+  LLVM_ABI static DbgVariableRecord *
   createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val,
                                     MDNode *Variable, MDNode *Expression,
                                     MDNode *AssignID, Metadata *Address,
                                     MDNode *AddressExpression, MDNode *DI);
 
-  static DbgVariableRecord *
+  LLVM_ABI static DbgVariableRecord *
   createDVRAssign(Value *Val, DILocalVariable *Variable,
                   DIExpression *Expression, DIAssignID *AssignID,
                   Value *Address, DIExpression *AddressExpression,
                   const DILocation *DI);
-  static DbgVariableRecord *
+  LLVM_ABI static DbgVariableRecord *
   createLinkedDVRAssign(Instruction *LinkedInstr, Value *Val,
                         DILocalVariable *Variable, DIExpression *Expression,
                         Value *Address, DIExpression *AddressExpression,
                         const DILocation *DI);
 
-  static DbgVariableRecord *createDbgVariableRecord(Value *Location,
-                                                    DILocalVariable *DV,
-                                                    DIExpression *Expr,
-                                                    const DILocation *DI);
-  static DbgVariableRecord *
+  LLVM_ABI static DbgVariableRecord *
+  createDbgVariableRecord(Value *Location, DILocalVariable *DV,
+                          DIExpression *Expr, const DILocation *DI);
+  LLVM_ABI static DbgVariableRecord *
   createDbgVariableRecord(Value *Location, DILocalVariable *DV,
                           DIExpression *Expr, const DILocation *DI,
                           DbgVariableRecord &InsertBefore);
-  static DbgVariableRecord *createDVRDeclare(Value *Address,
-                                             DILocalVariable *DV,
-                                             DIExpression *Expr,
-                                             const DILocation *DI);
-  static DbgVariableRecord *
+  LLVM_ABI static DbgVariableRecord *createDVRDeclare(Value *Address,
+                                                      DILocalVariable *DV,
+                                                      DIExpression *Expr,
+                                                      const DILocation *DI);
+  LLVM_ABI static DbgVariableRecord *
   createDVRDeclare(Value *Address, DILocalVariable *DV, DIExpression *Expr,
                    const DILocation *DI, DbgVariableRecord &InsertBefore);
 
@@ -412,20 +418,20 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
   /// Get the locations corresponding to the variable referenced by the debug
   /// info intrinsic.  Depending on the intrinsic, this could be the
   /// variable's value or its address.
-  iterator_range<location_op_iterator> location_ops() const;
+  LLVM_ABI iterator_range<location_op_iterator> location_ops() const;
 
-  Value *getVariableLocationOp(unsigned OpIdx) const;
+  LLVM_ABI Value *getVariableLocationOp(unsigned OpIdx) const;
 
-  void replaceVariableLocationOp(Value *OldValue, Value *NewValue,
-                                 bool AllowEmpty = false);
-  void replaceVariableLocationOp(unsigned OpIdx, Value *NewValue);
+  LLVM_ABI void replaceVariableLocationOp(Value *OldValue, Value *NewValue,
+                                          bool AllowEmpty = false);
+  LLVM_ABI void replaceVariableLocationOp(unsigned OpIdx, Value *NewValue);
   /// Adding a new location operand will always result in this intrinsic using
   /// an ArgList, and must always be accompanied by a new expression that uses
   /// the new operand.
-  void addVariableLocationOps(ArrayRef<Value *> NewValues,
-                              DIExpression *NewExpr);
+  LLVM_ABI void addVariableLocationOps(ArrayRef<Value *> NewValues,
+                                       DIExpression *NewExpr);
 
-  unsigned getNumVariableLocationOps() const;
+  LLVM_ABI unsigned getNumVariableLocationOps() const;
 
   bool hasArgList() const { return isa<DIArgList>(getRawLocation()); }
   /// Returns true if this DbgVariableRecord has no empty MDNodes in its
@@ -442,8 +448,8 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
 
   LocationType getType() const { return Type; }
 
-  void setKillLocation();
-  bool isKillLocation() const;
+  LLVM_ABI void setKillLocation();
+  LLVM_ABI bool isKillLocation() const;
 
   void setVariable(DILocalVariable *NewVar) { Variable = NewVar; }
   DILocalVariable *getVariable() const { return Variable.get(); };
@@ -474,7 +480,7 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
     resetDebugValue(0, NewLocation);
   }
 
-  std::optional<DbgVariableFragmentInfo> getFragment() const;
+  LLVM_ABI std::optional<DbgVariableFragmentInfo> getFragment() const;
   /// Get the FragmentInfo for the variable if it exists, otherwise return a
   /// FragmentInfo that covers the entire variable if the variable size is
   /// known, otherwise return a zero-sized fragment.
@@ -487,7 +493,7 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
   }
   /// Get the size (in bits) of the variable, or fragment of the variable that
   /// is described.
-  std::optional<uint64_t> getFragmentSizeInBits() const;
+  LLVM_ABI std::optional<uint64_t> getFragmentSizeInBits() const;
 
   bool isEquivalentTo(const DbgVariableRecord &Other) const {
     return DbgLoc == Other.DbgLoc && isIdenticalToWhenDefined(Other);
@@ -505,12 +511,12 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
   /// @{
   bool isDbgAssign() const { return getType() == LocationType::Assign; }
 
-  Value *getAddress() const;
+  LLVM_ABI Value *getAddress() const;
   Metadata *getRawAddress() const {
     return isDbgAssign() ? DebugValues[1] : DebugValues[0];
   }
   Metadata *getRawAssignID() const { return DebugValues[2]; }
-  DIAssignID *getAssignID() const;
+  LLVM_ABI DIAssignID *getAssignID() const;
   DIExpression *getAddressExpression() const { return AddressExpression.get(); }
   MDNode *getRawAddressExpression() const {
     return AddressExpression.getAsMDNode();
@@ -518,31 +524,32 @@ class DbgVariableRecord : public DbgRecord, protected DebugValueUser {
   void setAddressExpression(DIExpression *NewExpr) {
     AddressExpression = NewExpr;
   }
-  void setAssignId(DIAssignID *New);
+  LLVM_ABI void setAssignId(DIAssignID *New);
   void setAddress(Value *V) { resetDebugValue(1, ValueAsMetadata::get(V)); }
   /// Kill the address component.
-  void setKillAddress();
+  LLVM_ABI void setKillAddress();
   /// Check whether this kills the address component. This doesn't take into
   /// account the position of the intrinsic, therefore a returned value of false
   /// does not guarentee the address is a valid location for the variable at the
   /// intrinsic's position in IR.
-  bool isKillAddress() const;
+  LLVM_ABI bool isKillAddress() const;
 
   /// @}
 
-  DbgVariableRecord *clone() const;
+  LLVM_ABI DbgVariableRecord *clone() const;
   /// Convert this DbgVariableRecord back into a dbg.value intrinsic.
   /// \p InsertBefore Optional position to insert this intrinsic.
   /// \returns A new dbg.value intrinsic representiung this DbgVariableRecord.
-  DbgVariableIntrinsic *createDebugIntrinsic(Module *M,
-                                             Instruction *InsertBefore) const;
+  LLVM_ABI DbgVariableIntrinsic *
+  createDebugIntrinsic(Module *M, Instruction *InsertBefore) const;
 
   /// Handle changes to the location of the Value(s) that we refer to happening
   /// "under our feet".
-  void handleChangedLocation(Metadata *NewLocation);
+  LLVM_ABI void handleChangedLocation(Metadata *NewLocation);
 
-  void print(raw_ostream &O, bool IsForDebug = false) const;
-  void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
 
   /// Support type inquiry through isa, cast, and dyn_cast.
   static bool classof(const DbgRecord *E) {
@@ -594,42 +601,45 @@ class DbgMarker {
   simple_ilist<DbgRecord> StoredDbgRecords;
   bool empty() const { return StoredDbgRecords.empty(); }
 
-  const BasicBlock *getParent() const;
-  BasicBlock *getParent();
+  LLVM_ABI const BasicBlock *getParent() const;
+  LLVM_ABI BasicBlock *getParent();
 
   /// Handle the removal of a marker: the position of debug-info has gone away,
   /// but the stored debug records should not. Drop them onto the next
   /// instruction, or otherwise work out what to do with them.
-  void removeMarker();
-  void dump() const;
+  LLVM_ABI void removeMarker();
+  LLVM_ABI void dump() const;
 
-  void removeFromParent();
-  void eraseFromParent();
+  LLVM_ABI void removeFromParent();
+  LLVM_ABI void eraseFromParent();
 
   /// Implement operator<< on DbgMarker.
-  void print(raw_ostream &O, bool IsForDebug = false) const;
-  void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
 
   /// Produce a range over all the DbgRecords in this Marker.
-  iterator_range<simple_ilist<DbgRecord>::iterator> getDbgRecordRange();
-  iterator_range<simple_ilist<DbgRecord>::const_iterator>
+  LLVM_ABI iterator_range<simple_ilist<DbgRecord>::iterator>
+  getDbgRecordRange();
+  LLVM_ABI iterator_range<simple_ilist<DbgRecord>::const_iterator>
   getDbgRecordRange() const;
   /// Transfer any DbgRecords from \p Src into this DbgMarker. If \p
   /// InsertAtHead is true, place them before existing DbgRecords, otherwise
   /// afterwards.
-  void absorbDebugValues(DbgMarker &Src, bool InsertAtHead);
+  LLVM_ABI void absorbDebugValues(DbgMarker &Src, bool InsertAtHead);
   /// Transfer the DbgRecords in \p Range from \p Src into this DbgMarker. If
   /// \p InsertAtHead is true, place them before existing DbgRecords, otherwise
   // afterwards.
-  void absorbDebugValues(iterator_range<DbgRecord::self_iterator> Range,
-                         DbgMarker &Src, bool InsertAtHead);
+  LLVM_ABI void
+  absorbDebugValues(iterator_range<DbgRecord::self_iterator> Range,
+                    DbgMarker &Src, bool InsertAtHead);
   /// Insert a DbgRecord into this DbgMarker, at the end of the list. If
   /// \p InsertAtHead is true, at the start.
-  void insertDbgRecord(DbgRecord *New, bool InsertAtHead);
+  LLVM_ABI void insertDbgRecord(DbgRecord *New, bool InsertAtHead);
   /// Insert a DbgRecord prior to a DbgRecord contained within this marker.
-  void insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore);
+  LLVM_ABI void insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore);
   /// Insert a DbgRecord after a DbgRecord contained within this marker.
-  void insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter);
+  LLVM_ABI void insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter);
   /// Clone all DbgMarkers from \p From into this marker. There are numerous
   /// options to customise the source/destination, due to gnarliness, see class
   /// comment.
@@ -638,16 +648,16 @@ class DbgMarker {
   /// \p InsertAtHead Place the cloned DbgRecords at the start of
   /// StoredDbgRecords
   /// \returns Range over all the newly cloned DbgRecords
-  iterator_range<simple_ilist<DbgRecord>::iterator>
+  LLVM_ABI iterator_range<simple_ilist<DbgRecord>::iterator>
   cloneDebugInfoFrom(DbgMarker *From,
                      std::optional<simple_ilist<DbgRecord>::iterator> FromHere,
                      bool InsertAtHead = false);
   /// Erase all DbgRecords in this DbgMarker.
-  void dropDbgRecords();
+  LLVM_ABI void dropDbgRecords();
   /// Erase a single DbgRecord from this marker. In an ideal future, we would
   /// never erase an assignment in this way, but it's the equivalent to
   /// erasing a debug intrinsic from a block.
-  void dropOneDbgRecord(DbgRecord *DR);
+  LLVM_ABI void dropOneDbgRecord(DbgRecord *DR);
 
   /// We generally act like all llvm Instructions have a range of DbgRecords
   /// attached to them, but in reality sometimes we don't allocate the DbgMarker
@@ -656,7 +666,7 @@ class DbgMarker {
   /// static markers range instead. This will bite us if someone tries to insert
   /// a DbgRecord in that range, but they should be using the Official (TM) API
   /// for that.
-  static DbgMarker EmptyDbgMarker;
+  LLVM_ABI static DbgMarker EmptyDbgMarker;
   static iterator_range<simple_ilist<DbgRecord>::iterator>
   getEmptyDbgRecordRange() {
     return make_range(EmptyDbgMarker.StoredDbgRecords.end(),
@@ -682,25 +692,6 @@ getDbgRecordRange(DbgMarker *DebugMarker) {
 
 DEFINE_ISA_CONVERSION_FUNCTIONS(DbgRecord, LLVMDbgRecordRef)
 
-/// Used to temporarily set the debug info format of a function, module, or
-/// basic block for the duration of this object's lifetime, after which the
-/// prior state will be restored.
-template <typename T> class ScopedDbgInfoFormatSetter {
-  T &Obj;
-  bool OldState;
-
-public:
-  ScopedDbgInfoFormatSetter(T &Obj, bool NewState)
-      : Obj(Obj), OldState(Obj.IsNewDbgInfoFormat) {
-    Obj.setIsNewDbgInfoFormat(NewState);
-  }
-  ~ScopedDbgInfoFormatSetter() { Obj.setIsNewDbgInfoFormat(OldState); }
-};
-
-template <typename T>
-ScopedDbgInfoFormatSetter(T &Obj,
-                          bool NewState) -> ScopedDbgInfoFormatSetter<T>;
-
 } // namespace llvm
 
 #endif // LLVM_IR_DEBUGPROGRAMINSTRUCTION_H
diff --git a/external/llvm-project/llvm/include/llvm/IR/DerivedTypes.h b/external/llvm-project/llvm/include/llvm/IR/DerivedTypes.h
index fba6f45d37d1..fa62bc09b61a 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DerivedTypes.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DerivedTypes.h
@@ -63,7 +63,7 @@ class IntegerType : public Type {
   /// that instance will be returned. Otherwise a new one will be created. Only
   /// one instance with a given NumBits value is ever created.
   /// Get or create an IntegerType instance.
-  static IntegerType *get(LLVMContext &C, unsigned NumBits);
+  LLVM_ABI static IntegerType *get(LLVMContext &C, unsigned NumBits);
 
   /// Returns type twice as wide the input type.
   IntegerType *getExtendedType() const {
@@ -88,7 +88,7 @@ class IntegerType : public Type {
   /// For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
   /// @returns a bit mask with ones set for all the bits of this type.
   /// Get a bit mask for this type.
-  APInt getMask() const;
+  LLVM_ABI APInt getMask() const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast.
   static bool classof(const Type *T) {
@@ -110,17 +110,17 @@ class FunctionType : public Type {
   FunctionType &operator=(const FunctionType &) = delete;
 
   /// This static method is the primary way of constructing a FunctionType.
-  static FunctionType *get(Type *Result,
-                           ArrayRef<Type*> Params, bool isVarArg);
+  LLVM_ABI static FunctionType *get(Type *Result, ArrayRef<Type *> Params,
+                                    bool isVarArg);
 
   /// Create a FunctionType taking no parameters.
-  static FunctionType *get(Type *Result, bool isVarArg);
+  LLVM_ABI static FunctionType *get(Type *Result, bool isVarArg);
 
   /// Return true if the specified type is valid as a return type.
-  static bool isValidReturnType(Type *RetTy);
+  LLVM_ABI static bool isValidReturnType(Type *RetTy);
 
   /// Return true if the specified type is valid as an argument type.
-  static bool isValidArgumentType(Type *ArgTy);
+  LLVM_ABI static bool isValidArgumentType(Type *ArgTy);
 
   bool isVarArg() const { return getSubclassData()!=0; }
   Type *getReturnType() const { return ContainedTys[0]; }
@@ -243,15 +243,17 @@ class StructType : public Type {
   StructType &operator=(const StructType &) = delete;
 
   /// This creates an identified struct.
-  static StructType *create(LLVMContext &Context, StringRef Name);
-  static StructType *create(LLVMContext &Context);
-
-  static StructType *create(ArrayRef<Type *> Elements, StringRef Name,
-                            bool isPacked = false);
-  static StructType *create(ArrayRef<Type *> Elements);
-  static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements,
-                            StringRef Name, bool isPacked = false);
-  static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements);
+  LLVM_ABI static StructType *create(LLVMContext &Context, StringRef Name);
+  LLVM_ABI static StructType *create(LLVMContext &Context);
+
+  LLVM_ABI static StructType *create(ArrayRef<Type *> Elements, StringRef Name,
+                                     bool isPacked = false);
+  LLVM_ABI static StructType *create(ArrayRef<Type *> Elements);
+  LLVM_ABI static StructType *create(LLVMContext &Context,
+                                     ArrayRef<Type *> Elements, StringRef Name,
+                                     bool isPacked = false);
+  LLVM_ABI static StructType *create(LLVMContext &Context,
+                                     ArrayRef<Type *> Elements);
   template <class... Tys>
   static std::enable_if_t<are_base_of<Type, Tys...>::value, StructType *>
   create(StringRef Name, Type *elt1, Tys *... elts) {
@@ -260,11 +262,11 @@ class StructType : public Type {
   }
 
   /// This static method is the primary way to create a literal StructType.
-  static StructType *get(LLVMContext &Context, ArrayRef<Type*> Elements,
-                         bool isPacked = false);
+  LLVM_ABI static StructType *
+  get(LLVMContext &Context, ArrayRef<Type *> Elements, bool isPacked = false);
 
   /// Create an empty structure type.
-  static StructType *get(LLVMContext &Context, bool isPacked = false);
+  LLVM_ABI static StructType *get(LLVMContext &Context, bool isPacked = false);
 
   /// This static method is a convenience method for creating structure types by
   /// specifying the elements as arguments. Note that this method always returns
@@ -279,7 +281,7 @@ class StructType : public Type {
 
   /// Return the type with the specified name, or null if there is none by that
   /// name.
-  static StructType *getTypeByName(LLVMContext &C, StringRef Name);
+  LLVM_ABI static StructType *getTypeByName(LLVMContext &C, StringRef Name);
 
   bool isPacked() const { return (getSubclassData() & SCDB_Packed) != 0; }
 
@@ -292,21 +294,21 @@ class StructType : public Type {
   bool isOpaque() const { return (getSubclassData() & SCDB_HasBody) == 0; }
 
   /// isSized - Return true if this is a sized type.
-  bool isSized(SmallPtrSetImpl<Type *> *Visited = nullptr) const;
+  LLVM_ABI bool isSized(SmallPtrSetImpl<Type *> *Visited = nullptr) const;
 
   /// Returns true if this struct contains a scalable vector.
-  bool isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const;
+  LLVM_ABI bool isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const;
   using Type::isScalableTy;
 
   /// Return true if this type is or contains a target extension type that
   /// disallows being used as a global.
-  bool
+  LLVM_ABI bool
   containsNonGlobalTargetExtType(SmallPtrSetImpl<const Type *> &Visited) const;
   using Type::containsNonGlobalTargetExtType;
 
   /// Return true if this type is or contains a target extension type that
   /// disallows being used as a local.
-  bool
+  LLVM_ABI bool
   containsNonLocalTargetExtType(SmallPtrSetImpl<const Type *> &Visited) const;
   using Type::containsNonLocalTargetExtType;
 
@@ -316,11 +318,11 @@ class StructType : public Type {
   /// when calling this function.
   /// {{<vscale x 2 x i32>, <vscale x 4 x i64>},
   ///  {<vscale x 2 x i32>, <vscale x 4 x i64>}}
-  bool containsHomogeneousScalableVectorTypes() const;
+  LLVM_ABI bool containsHomogeneousScalableVectorTypes() const;
 
   /// Return true if this struct is non-empty and all element types are the
   /// same.
-  bool containsHomogeneousTypes() const;
+  LLVM_ABI bool containsHomogeneousTypes() const;
 
   /// Return true if this is a named struct that has a non-empty name.
   bool hasName() const { return SymbolTableEntry != nullptr; }
@@ -328,26 +330,27 @@ class StructType : public Type {
   /// Return the name for this struct type if it has an identity.
   /// This may return an empty string for an unnamed struct type.  Do not call
   /// this on an literal type.
-  StringRef getName() const;
+  LLVM_ABI StringRef getName() const;
 
   /// Change the name of this type to the specified name, or to a name with a
   /// suffix if there is a collision. Do not call this on an literal type.
-  void setName(StringRef Name);
+  LLVM_ABI void setName(StringRef Name);
 
   /// Specify a body for an opaque identified type, which must not make the type
   /// recursive.
-  void setBody(ArrayRef<Type*> Elements, bool isPacked = false);
+  LLVM_ABI void setBody(ArrayRef<Type *> Elements, bool isPacked = false);
 
   /// Specify a body for an opaque identified type or return an error if it
   /// would make the type recursive.
-  Error setBodyOrError(ArrayRef<Type *> Elements, bool isPacked = false);
+  LLVM_ABI Error setBodyOrError(ArrayRef<Type *> Elements,
+                                bool isPacked = false);
 
   /// Return an error if the body for an opaque identified type would make it
   /// recursive.
-  Error checkBody(ArrayRef<Type *> Elements);
+  LLVM_ABI Error checkBody(ArrayRef<Type *> Elements);
 
   /// Return true if the specified type is valid as a element type.
-  static bool isValidElementType(Type *ElemTy);
+  LLVM_ABI static bool isValidElementType(Type *ElemTy);
 
   // Iterator access to the elements.
   using element_iterator = Type::subtype_iterator;
@@ -359,7 +362,7 @@ class StructType : public Type {
   }
 
   /// Return true if this is layout identical to the specified struct.
-  bool isLayoutIdentical(StructType *Other) const;
+  LLVM_ABI bool isLayoutIdentical(StructType *Other) const;
 
   /// Random access to the elements
   unsigned getNumElements() const { return NumContainedTys; }
@@ -368,9 +371,9 @@ class StructType : public Type {
     return ContainedTys[N];
   }
   /// Given an index value into the type, return the type of the element.
-  Type *getTypeAtIndex(const Value *V) const;
+  LLVM_ABI Type *getTypeAtIndex(const Value *V) const;
   Type *getTypeAtIndex(unsigned N) const { return getElementType(N); }
-  bool indexValid(const Value *V) const;
+  LLVM_ABI bool indexValid(const Value *V) const;
   bool indexValid(unsigned Idx) const { return Idx < getNumElements(); }
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast.
@@ -408,10 +411,10 @@ class ArrayType : public Type {
   Type *getElementType() const { return ContainedType; }
 
   /// This static method is the primary way to construct an ArrayType
-  static ArrayType *get(Type *ElementType, uint64_t NumElements);
+  LLVM_ABI static ArrayType *get(Type *ElementType, uint64_t NumElements);
 
   /// Return true if the specified type is valid as a element type.
-  static bool isValidElementType(Type *ElemTy);
+  LLVM_ABI static bool isValidElementType(Type *ElemTy);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast.
   static bool classof(const Type *T) {
@@ -451,7 +454,7 @@ class VectorType : public Type {
   ///   vscale is a runtime-constant integer greater than 0.
   const unsigned ElementQuantity;
 
-  VectorType(Type *ElType, unsigned EQ, Type::TypeID TID);
+  LLVM_ABI VectorType(Type *ElType, unsigned EQ, Type::TypeID TID);
 
 public:
   VectorType(const VectorType &) = delete;
@@ -460,7 +463,7 @@ class VectorType : public Type {
   Type *getElementType() const { return ContainedType; }
 
   /// This static method is the primary way to construct an VectorType.
-  static VectorType *get(Type *ElementType, ElementCount EC);
+  LLVM_ABI static VectorType *get(Type *ElementType, ElementCount EC);
 
   static VectorType *get(Type *ElementType, unsigned NumElements,
                          bool Scalable) {
@@ -572,7 +575,7 @@ class VectorType : public Type {
   }
 
   /// Return true if the specified type is valid as a element type.
-  static bool isValidElementType(Type *ElemTy);
+  LLVM_ABI static bool isValidElementType(Type *ElemTy);
 
   /// Return an ElementCount instance to represent the (possibly scalable)
   /// number of elements in the vector.
@@ -592,7 +595,7 @@ class FixedVectorType : public VectorType {
       : VectorType(ElTy, NumElts, FixedVectorTyID) {}
 
 public:
-  static FixedVectorType *get(Type *ElementType, unsigned NumElts);
+  LLVM_ABI static FixedVectorType *get(Type *ElementType, unsigned NumElts);
 
   static FixedVectorType *get(Type *ElementType, const FixedVectorType *FVTy) {
     return get(ElementType, FVTy->getNumElements());
@@ -639,7 +642,8 @@ class ScalableVectorType : public VectorType {
       : VectorType(ElTy, MinNumElts, ScalableVectorTyID) {}
 
 public:
-  static ScalableVectorType *get(Type *ElementType, unsigned MinNumElts);
+  LLVM_ABI static ScalableVectorType *get(Type *ElementType,
+                                          unsigned MinNumElts);
 
   static ScalableVectorType *get(Type *ElementType,
                                  const ScalableVectorType *SVTy) {
@@ -704,10 +708,10 @@ class PointerType : public Type {
   /// address space.
   [[deprecated("PointerType::get with pointee type is pending removal. Use "
                "Context overload.")]]
-  static PointerType *get(Type *ElementType, unsigned AddressSpace);
+  LLVM_ABI static PointerType *get(Type *ElementType, unsigned AddressSpace);
   /// This constructs an opaque pointer to an object in a numbered address
   /// space.
-  static PointerType *get(LLVMContext &C, unsigned AddressSpace);
+  LLVM_ABI static PointerType *get(LLVMContext &C, unsigned AddressSpace);
 
   /// This constructs a pointer to an object of the specified type in the
   /// default address space (address space zero).
@@ -727,10 +731,10 @@ class PointerType : public Type {
   }
 
   /// Return true if the specified type is valid as a element type.
-  static bool isValidElementType(Type *ElemTy);
+  LLVM_ABI static bool isValidElementType(Type *ElemTy);
 
   /// Return true if we can load or store from a pointer to this type.
-  static bool isLoadableOrStorableType(Type *ElemTy);
+  LLVM_ABI static bool isLoadableOrStorableType(Type *ElemTy);
 
   /// Return the address space of the Pointer type.
   inline unsigned getAddressSpace() const { return getSubclassData(); }
@@ -788,22 +792,21 @@ class TargetExtType : public Type {
 
   /// Return a target extension type having the specified name and optional
   /// type and integer parameters.
-  static TargetExtType *get(LLVMContext &Context, StringRef Name,
-                            ArrayRef<Type *> Types = {},
-                            ArrayRef<unsigned> Ints = {});
+  LLVM_ABI static TargetExtType *get(LLVMContext &Context, StringRef Name,
+                                     ArrayRef<Type *> Types = {},
+                                     ArrayRef<unsigned> Ints = {});
 
   /// Return a target extension type having the specified name and optional
   /// type and integer parameters, or an appropriate Error if it fails the
   /// parameters check.
-  static Expected<TargetExtType *> getOrError(LLVMContext &Context,
-                                              StringRef Name,
-                                              ArrayRef<Type *> Types = {},
-                                              ArrayRef<unsigned> Ints = {});
+  LLVM_ABI static Expected<TargetExtType *>
+  getOrError(LLVMContext &Context, StringRef Name, ArrayRef<Type *> Types = {},
+             ArrayRef<unsigned> Ints = {});
 
   /// Check that a newly created target extension type has the expected number
   /// of type parameters and integer parameters, returning the type itself if OK
   /// or an appropriate Error if not.
-  static Expected<TargetExtType *> checkParams(TargetExtType *TTy);
+  LLVM_ABI static Expected<TargetExtType *> checkParams(TargetExtType *TTy);
 
   /// Return the name for this target extension type. Two distinct target
   /// extension types may have the same name if their type or integer parameters
@@ -842,16 +845,18 @@ class TargetExtType : public Type {
     /// This type may be allocated on the stack, either as the allocated type
     /// of an alloca instruction or as a byval function parameter.
     CanBeLocal = 1U << 2,
+    // This type may be used as an element in a vector.
+    CanBeVectorElement = 1U << 3,
   };
 
   /// Returns true if the target extension type contains the given property.
-  bool hasProperty(Property Prop) const;
+  LLVM_ABI bool hasProperty(Property Prop) const;
 
   /// Returns an underlying layout type for the target extension type. This
   /// type can be used to query size and alignment information, if it is
   /// appropriate (although note that the layout type may also be void). It is
   /// not legal to bitcast between this type and the layout type, however.
-  Type *getLayoutType() const;
+  LLVM_ABI Type *getLayoutType() const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast.
   static bool classof(const Type *T) { return T->getTypeID() == TargetExtTyID; }
diff --git a/external/llvm-project/llvm/include/llvm/IR/DiagnosticHandler.h b/external/llvm-project/llvm/include/llvm/IR/DiagnosticHandler.h
index 1f0544b9fa9f..54880d8e6273 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DiagnosticHandler.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DiagnosticHandler.h
@@ -13,6 +13,7 @@
 #define LLVM_IR_DIAGNOSTICHANDLER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class DiagnosticInfo;
@@ -21,7 +22,7 @@ class DiagnosticInfo;
 /// The handleDiagnostics method must be overriden by the subclasses to handle
 /// diagnostic. The *RemarkEnabled methods can be overriden to control
 /// which remarks are enabled.
-struct DiagnosticHandler {
+struct LLVM_ABI DiagnosticHandler {
   void *DiagnosticContext = nullptr;
   bool HasErrors = false;
   DiagnosticHandler(void *DiagContext = nullptr)
diff --git a/external/llvm-project/llvm/include/llvm/IR/DiagnosticInfo.h b/external/llvm-project/llvm/include/llvm/IR/DiagnosticInfo.h
index 4ffd63f14d2f..862be04e0fc0 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -21,6 +21,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/TypeSize.h"
@@ -103,14 +104,14 @@ enum DiagnosticKind {
 /// The returned ID will be greater than or equal to DK_FirstPluginKind.
 /// Thus, the plugin identifiers will not conflict with the
 /// DiagnosticKind values.
-int getNextAvailablePluginDiagnosticKind();
+LLVM_ABI int getNextAvailablePluginDiagnosticKind();
 
 /// This is the base abstract class for diagnostic reporting in
 /// the backend.
 /// The print method must be overloaded by the subclasses to print a
 /// user-friendly message in the client of the backend (let us call it a
 /// frontend).
-class DiagnosticInfo {
+class LLVM_ABI DiagnosticInfo {
 private:
   /// Kind defines the kind of report this is about.
   const /* DiagnosticKind */ int Kind;
@@ -138,7 +139,7 @@ class DiagnosticInfo {
 
 using DiagnosticHandlerFunction = std::function<void(const DiagnosticInfo &)>;
 
-class DiagnosticInfoGeneric : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoGeneric : public DiagnosticInfo {
   const Twine &MsgStr;
   const Instruction *Inst = nullptr;
 
@@ -168,7 +169,7 @@ class DiagnosticInfoGeneric : public DiagnosticInfo {
 
 /// Diagnostic information for inline asm reporting.
 /// This is basically a message and an optional location.
-class DiagnosticInfoInlineAsm : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoInlineAsm : public DiagnosticInfo {
 private:
   /// Optional line information. 0 if not set.
   uint64_t LocCookie = 0;
@@ -209,7 +210,7 @@ class DiagnosticInfoInlineAsm : public DiagnosticInfo {
 
 /// Diagnostic information for debug metadata version reporting.
 /// This is basically a module and a version.
-class DiagnosticInfoDebugMetadataVersion : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoDebugMetadataVersion : public DiagnosticInfo {
 private:
   /// The module that is concerned by this debug metadata version diagnostic.
   const Module &M;
@@ -236,7 +237,8 @@ class DiagnosticInfoDebugMetadataVersion : public DiagnosticInfo {
 };
 
 /// Diagnostic information for stripping invalid debug metadata.
-class DiagnosticInfoIgnoringInvalidDebugMetadata : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoIgnoringInvalidDebugMetadata
+    : public DiagnosticInfo {
 private:
   /// The module that is concerned by this debug metadata version diagnostic.
   const Module &M;
@@ -258,7 +260,7 @@ class DiagnosticInfoIgnoringInvalidDebugMetadata : public DiagnosticInfo {
 };
 
 /// Diagnostic information for the sample profiler.
-class DiagnosticInfoSampleProfile : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoSampleProfile : public DiagnosticInfo {
 public:
   DiagnosticInfoSampleProfile(StringRef FileName, unsigned LineNum,
                               const Twine &Msg LLVM_LIFETIME_BOUND,
@@ -298,7 +300,7 @@ class DiagnosticInfoSampleProfile : public DiagnosticInfo {
 };
 
 /// Diagnostic information for the PGO profiler.
-class DiagnosticInfoPGOProfile : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoPGOProfile : public DiagnosticInfo {
 public:
   DiagnosticInfoPGOProfile(const char *FileName,
                            const Twine &Msg LLVM_LIFETIME_BOUND,
@@ -330,20 +332,20 @@ class DiagnosticLocation {
 
 public:
   DiagnosticLocation() = default;
-  DiagnosticLocation(const DebugLoc &DL);
-  DiagnosticLocation(const DISubprogram *SP);
+  LLVM_ABI DiagnosticLocation(const DebugLoc &DL);
+  LLVM_ABI DiagnosticLocation(const DISubprogram *SP);
 
   bool isValid() const { return File; }
   /// Return the full path to the file.
-  std::string getAbsolutePath() const;
+  LLVM_ABI std::string getAbsolutePath() const;
   /// Return the file name relative to the compilation directory.
-  StringRef getRelativePath() const;
+  LLVM_ABI StringRef getRelativePath() const;
   unsigned getLine() const { return Line; }
   unsigned getColumn() const { return Column; }
 };
 
 /// Common features for diagnostics with an associated location.
-class DiagnosticInfoWithLocationBase : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoWithLocationBase : public DiagnosticInfo {
   void anchor() override;
 public:
   /// \p Fn is the function where the diagnostic is being emitted. \p Loc is
@@ -381,7 +383,8 @@ class DiagnosticInfoWithLocationBase : public DiagnosticInfo {
   DiagnosticLocation Loc;
 };
 
-class DiagnosticInfoGenericWithLoc : public DiagnosticInfoWithLocationBase {
+class LLVM_ABI DiagnosticInfoGenericWithLoc
+    : public DiagnosticInfoWithLocationBase {
 private:
   /// Message to be reported.
   const Twine &MsgStr;
@@ -406,7 +409,8 @@ class DiagnosticInfoGenericWithLoc : public DiagnosticInfoWithLocationBase {
   }
 };
 
-class DiagnosticInfoRegAllocFailure : public DiagnosticInfoWithLocationBase {
+class LLVM_ABI DiagnosticInfoRegAllocFailure
+    : public DiagnosticInfoWithLocationBase {
 private:
   /// Message to be reported.
   const Twine &MsgStr;
@@ -434,7 +438,8 @@ class DiagnosticInfoRegAllocFailure : public DiagnosticInfoWithLocationBase {
 
 /// Diagnostic information for stack size etc. reporting.
 /// This is basically a function and a size.
-class DiagnosticInfoResourceLimit : public DiagnosticInfoWithLocationBase {
+class LLVM_ABI DiagnosticInfoResourceLimit
+    : public DiagnosticInfoWithLocationBase {
 private:
   /// The function that is concerned by this resource limit diagnostic.
   const Function &Fn;
@@ -469,7 +474,7 @@ class DiagnosticInfoResourceLimit : public DiagnosticInfoWithLocationBase {
   }
 };
 
-class DiagnosticInfoStackSize : public DiagnosticInfoResourceLimit {
+class LLVM_ABI DiagnosticInfoStackSize : public DiagnosticInfoResourceLimit {
   void anchor() override;
 
 public:
@@ -489,7 +494,8 @@ class DiagnosticInfoStackSize : public DiagnosticInfoResourceLimit {
 
 /// Common features for diagnostics dealing with optimization remarks
 /// that are used by both IR and MIR passes.
-class DiagnosticInfoOptimizationBase : public DiagnosticInfoWithLocationBase {
+class LLVM_ABI DiagnosticInfoOptimizationBase
+    : public DiagnosticInfoWithLocationBase {
 public:
   /// Used to set IsVerbose via the stream interface.
   struct setIsVerbose {};
@@ -509,21 +515,21 @@ class DiagnosticInfoOptimizationBase : public DiagnosticInfoWithLocationBase {
     DiagnosticLocation Loc;
 
     explicit Argument(StringRef Str = "") : Key("String"), Val(Str) {}
-    Argument(StringRef Key, const Value *V);
-    Argument(StringRef Key, const Type *T);
-    Argument(StringRef Key, StringRef S);
+    LLVM_ABI Argument(StringRef Key, const Value *V);
+    LLVM_ABI Argument(StringRef Key, const Type *T);
+    LLVM_ABI Argument(StringRef Key, StringRef S);
     Argument(StringRef Key, const char *S) : Argument(Key, StringRef(S)) {};
-    Argument(StringRef Key, int N);
-    Argument(StringRef Key, float N);
-    Argument(StringRef Key, long N);
-    Argument(StringRef Key, long long N);
-    Argument(StringRef Key, unsigned N);
-    Argument(StringRef Key, unsigned long N);
-    Argument(StringRef Key, unsigned long long N);
-    Argument(StringRef Key, ElementCount EC);
+    LLVM_ABI Argument(StringRef Key, int N);
+    LLVM_ABI Argument(StringRef Key, float N);
+    LLVM_ABI Argument(StringRef Key, long N);
+    LLVM_ABI Argument(StringRef Key, long long N);
+    LLVM_ABI Argument(StringRef Key, unsigned N);
+    LLVM_ABI Argument(StringRef Key, unsigned long N);
+    LLVM_ABI Argument(StringRef Key, unsigned long long N);
+    LLVM_ABI Argument(StringRef Key, ElementCount EC);
     Argument(StringRef Key, bool B) : Key(Key), Val(B ? "true" : "false") {}
-    Argument(StringRef Key, DebugLoc dl);
-    Argument(StringRef Key, InstructionCost C);
+    LLVM_ABI Argument(StringRef Key, DebugLoc dl);
+    LLVM_ABI Argument(StringRef Key, InstructionCost C);
   };
 
   /// \p PassName is the name of the pass emitting this diagnostic. \p
@@ -663,7 +669,8 @@ operator<<(RemarkT &&R,
 
 /// Common features for diagnostics dealing with optimization remarks
 /// that are used by IR passes.
-class DiagnosticInfoIROptimization : public DiagnosticInfoOptimizationBase {
+class LLVM_ABI DiagnosticInfoIROptimization
+    : public DiagnosticInfoOptimizationBase {
   void anchor() override;
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. \p
@@ -729,7 +736,7 @@ class DiagnosticInfoIROptimization : public DiagnosticInfoOptimizationBase {
 };
 
 /// Diagnostic information for applied optimization remarks.
-class OptimizationRemark : public DiagnosticInfoIROptimization {
+class LLVM_ABI OptimizationRemark : public DiagnosticInfoIROptimization {
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. If this name
   /// matches the regular expression given in -Rpass=, then the diagnostic will
@@ -774,7 +781,7 @@ class OptimizationRemark : public DiagnosticInfoIROptimization {
 };
 
 /// Diagnostic information for missed-optimization remarks.
-class OptimizationRemarkMissed : public DiagnosticInfoIROptimization {
+class LLVM_ABI OptimizationRemarkMissed : public DiagnosticInfoIROptimization {
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. If this name
   /// matches the regular expression given in -Rpass-missed=, then the
@@ -819,7 +826,8 @@ class OptimizationRemarkMissed : public DiagnosticInfoIROptimization {
 };
 
 /// Diagnostic information for optimization analysis remarks.
-class OptimizationRemarkAnalysis : public DiagnosticInfoIROptimization {
+class LLVM_ABI OptimizationRemarkAnalysis
+    : public DiagnosticInfoIROptimization {
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. If this name
   /// matches the regular expression given in -Rpass-analysis=, then the
@@ -891,7 +899,8 @@ class OptimizationRemarkAnalysis : public DiagnosticInfoIROptimization {
 
 /// Diagnostic information for optimization analysis remarks related to
 /// floating-point non-commutativity.
-class OptimizationRemarkAnalysisFPCommute : public OptimizationRemarkAnalysis {
+class LLVM_ABI OptimizationRemarkAnalysisFPCommute
+    : public OptimizationRemarkAnalysis {
   void anchor() override;
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. If this name
@@ -933,7 +942,8 @@ class OptimizationRemarkAnalysisFPCommute : public OptimizationRemarkAnalysis {
 
 /// Diagnostic information for optimization analysis remarks related to
 /// pointer aliasing.
-class OptimizationRemarkAnalysisAliasing : public OptimizationRemarkAnalysis {
+class LLVM_ABI OptimizationRemarkAnalysisAliasing
+    : public OptimizationRemarkAnalysis {
   void anchor() override;
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. If this name
@@ -974,7 +984,7 @@ class OptimizationRemarkAnalysisAliasing : public OptimizationRemarkAnalysis {
 
 /// Diagnostic information for machine IR parser.
 // FIXME: Remove this, use DiagnosticInfoSrcMgr instead.
-class DiagnosticInfoMIRParser : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoMIRParser : public DiagnosticInfo {
   const SMDiagnostic &Diagnostic;
 
 public:
@@ -992,7 +1002,7 @@ class DiagnosticInfoMIRParser : public DiagnosticInfo {
 };
 
 /// Diagnostic information for IR instrumentation reporting.
-class DiagnosticInfoInstrumentation : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoInstrumentation : public DiagnosticInfo {
   const Twine &Msg;
 
 public:
@@ -1008,7 +1018,7 @@ class DiagnosticInfoInstrumentation : public DiagnosticInfo {
 };
 
 /// Diagnostic information for ISel fallback path.
-class DiagnosticInfoISelFallback : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoISelFallback : public DiagnosticInfo {
   /// The function that is concerned by this diagnostic.
   const Function &Fn;
 
@@ -1030,7 +1040,8 @@ class DiagnosticInfoISelFallback : public DiagnosticInfo {
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DiagnosticInfo, LLVMDiagnosticInfoRef)
 
 /// Diagnostic information for optimization failures.
-class DiagnosticInfoOptimizationFailure : public DiagnosticInfoIROptimization {
+class LLVM_ABI DiagnosticInfoOptimizationFailure
+    : public DiagnosticInfoIROptimization {
 public:
   /// \p Fn is the function where the diagnostic is being emitted. \p Loc is
   /// the location information to use in the diagnostic. If line table
@@ -1061,7 +1072,8 @@ class DiagnosticInfoOptimizationFailure : public DiagnosticInfoIROptimization {
 };
 
 /// Diagnostic information for unsupported feature in backend.
-class DiagnosticInfoUnsupported : public DiagnosticInfoWithLocationBase {
+class LLVM_ABI DiagnosticInfoUnsupported
+    : public DiagnosticInfoWithLocationBase {
 private:
   const Twine &Msg;
 
@@ -1089,7 +1101,7 @@ class DiagnosticInfoUnsupported : public DiagnosticInfoWithLocationBase {
 };
 
 /// Diagnostic information for MisExpect analysis.
-class DiagnosticInfoMisExpect : public DiagnosticInfoWithLocationBase {
+class LLVM_ABI DiagnosticInfoMisExpect : public DiagnosticInfoWithLocationBase {
 public:
   DiagnosticInfoMisExpect(const Instruction *Inst,
                           const Twine &Msg LLVM_LIFETIME_BOUND);
@@ -1127,7 +1139,7 @@ static DiagnosticSeverity getDiagnosticSeverity(SourceMgr::DiagKind DK) {
 }
 
 /// Diagnostic information for SMDiagnostic reporting.
-class DiagnosticInfoSrcMgr : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoSrcMgr : public DiagnosticInfo {
   const SMDiagnostic &Diagnostic;
   StringRef ModName;
 
@@ -1153,9 +1165,9 @@ class DiagnosticInfoSrcMgr : public DiagnosticInfo {
   }
 };
 
-void diagnoseDontCall(const CallInst &CI);
+LLVM_ABI void diagnoseDontCall(const CallInst &CI);
 
-class DiagnosticInfoDontCall : public DiagnosticInfo {
+class LLVM_ABI DiagnosticInfoDontCall : public DiagnosticInfo {
   StringRef CalleeName;
   StringRef Note;
   uint64_t LocCookie;
diff --git a/external/llvm-project/llvm/include/llvm/IR/DiagnosticPrinter.h b/external/llvm-project/llvm/include/llvm/IR/DiagnosticPrinter.h
index 2df6fc3dfe73..331fc94aeb31 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DiagnosticPrinter.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DiagnosticPrinter.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_IR_DIAGNOSTICPRINTER_H
 #define LLVM_IR_DIAGNOSTICPRINTER_H
 
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -58,7 +59,7 @@ class DiagnosticPrinter {
 };
 
 /// Basic diagnostic printer that uses an underlying raw_ostream.
-class DiagnosticPrinterRawOStream : public DiagnosticPrinter {
+class LLVM_ABI DiagnosticPrinterRawOStream : public DiagnosticPrinter {
 protected:
   raw_ostream &Stream;
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/Dominators.h b/external/llvm-project/llvm/include/llvm/IR/Dominators.h
index 287f419f893d..bf128a3936cb 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Dominators.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Dominators.h
@@ -30,6 +30,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/CFGDiff.h"
 #include "llvm/Support/CFGUpdate.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/GenericDomTree.h"
 #include <algorithm>
 #include <utility>
@@ -43,9 +44,11 @@ class Value;
 class raw_ostream;
 template <class GraphType> struct GraphTraits;
 
-extern template class DomTreeNodeBase<BasicBlock>;
-extern template class DominatorTreeBase<BasicBlock, false>; // DomTree
-extern template class DominatorTreeBase<BasicBlock, true>; // PostDomTree
+extern template class LLVM_TEMPLATE_ABI DomTreeNodeBase<BasicBlock>;
+extern template class LLVM_TEMPLATE_ABI
+    DominatorTreeBase<BasicBlock, false>; // DomTree
+extern template class LLVM_TEMPLATE_ABI
+    DominatorTreeBase<BasicBlock, true>; // PostDomTree
 
 extern template class cfg::Update<BasicBlock *>;
 
@@ -58,35 +61,35 @@ using BBUpdates = ArrayRef<llvm::cfg::Update<BasicBlock *>>;
 using BBDomTreeGraphDiff = GraphDiff<BasicBlock *, false>;
 using BBPostDomTreeGraphDiff = GraphDiff<BasicBlock *, true>;
 
-extern template void Calculate<BBDomTree>(BBDomTree &DT);
-extern template void CalculateWithUpdates<BBDomTree>(BBDomTree &DT,
-                                                     BBUpdates U);
-
-extern template void Calculate<BBPostDomTree>(BBPostDomTree &DT);
-
-extern template void InsertEdge<BBDomTree>(BBDomTree &DT, BasicBlock *From,
-                                           BasicBlock *To);
-extern template void InsertEdge<BBPostDomTree>(BBPostDomTree &DT,
-                                               BasicBlock *From,
-                                               BasicBlock *To);
-
-extern template void DeleteEdge<BBDomTree>(BBDomTree &DT, BasicBlock *From,
-                                           BasicBlock *To);
-extern template void DeleteEdge<BBPostDomTree>(BBPostDomTree &DT,
-                                               BasicBlock *From,
-                                               BasicBlock *To);
-
-extern template void ApplyUpdates<BBDomTree>(BBDomTree &DT,
-                                             BBDomTreeGraphDiff &,
-                                             BBDomTreeGraphDiff *);
-extern template void ApplyUpdates<BBPostDomTree>(BBPostDomTree &DT,
-                                                 BBPostDomTreeGraphDiff &,
-                                                 BBPostDomTreeGraphDiff *);
-
-extern template bool Verify<BBDomTree>(const BBDomTree &DT,
-                                       BBDomTree::VerificationLevel VL);
-extern template bool Verify<BBPostDomTree>(const BBPostDomTree &DT,
-                                           BBPostDomTree::VerificationLevel VL);
+extern template LLVM_TEMPLATE_ABI void Calculate<BBDomTree>(BBDomTree &DT);
+extern template LLVM_TEMPLATE_ABI void
+CalculateWithUpdates<BBDomTree>(BBDomTree &DT, BBUpdates U);
+
+extern template LLVM_TEMPLATE_ABI void
+Calculate<BBPostDomTree>(BBPostDomTree &DT);
+
+extern template LLVM_TEMPLATE_ABI void
+InsertEdge<BBDomTree>(BBDomTree &DT, BasicBlock *From, BasicBlock *To);
+extern template LLVM_TEMPLATE_ABI void
+InsertEdge<BBPostDomTree>(BBPostDomTree &DT, BasicBlock *From, BasicBlock *To);
+
+extern template LLVM_TEMPLATE_ABI void
+DeleteEdge<BBDomTree>(BBDomTree &DT, BasicBlock *From, BasicBlock *To);
+extern template LLVM_TEMPLATE_ABI void
+DeleteEdge<BBPostDomTree>(BBPostDomTree &DT, BasicBlock *From, BasicBlock *To);
+
+extern template LLVM_TEMPLATE_ABI void
+ApplyUpdates<BBDomTree>(BBDomTree &DT, BBDomTreeGraphDiff &,
+                        BBDomTreeGraphDiff *);
+extern template LLVM_TEMPLATE_ABI void
+ApplyUpdates<BBPostDomTree>(BBPostDomTree &DT, BBPostDomTreeGraphDiff &,
+                            BBPostDomTreeGraphDiff *);
+
+extern template LLVM_TEMPLATE_ABI bool
+Verify<BBDomTree>(const BBDomTree &DT, BBDomTree::VerificationLevel VL);
+extern template LLVM_TEMPLATE_ABI bool
+Verify<BBPostDomTree>(const BBPostDomTree &DT,
+                      BBPostDomTree::VerificationLevel VL);
 }  // namespace DomTreeBuilder
 
 using DomTreeNode = DomTreeNodeBase<BasicBlock>;
@@ -114,13 +117,13 @@ class BasicBlockEdge {
   }
 
   /// Check if this is the only edge between Start and End.
-  bool isSingleEdge() const;
+  LLVM_ABI bool isSingleEdge() const;
 };
 
 template <> struct DenseMapInfo<BasicBlockEdge> {
   using BBInfo = DenseMapInfo<const BasicBlock *>;
 
-  static unsigned getHashValue(const BasicBlockEdge *V);
+  LLVM_ABI static unsigned getHashValue(const BasicBlockEdge *V);
 
   static inline BasicBlockEdge getEmptyKey() {
     return BasicBlockEdge(BBInfo::getEmptyKey(), BBInfo::getEmptyKey());
@@ -170,14 +173,14 @@ class DominatorTree : public DominatorTreeBase<BasicBlock, false> {
   }
 
   /// Handle invalidation explicitly.
-  bool invalidate(Function &F, const PreservedAnalyses &PA,
-                  FunctionAnalysisManager::Invalidator &);
+  LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA,
+                           FunctionAnalysisManager::Invalidator &);
 
   // Ensure base-class overloads are visible.
   using Base::dominates;
 
   /// Return true if the (end of the) basic block BB dominates the use U.
-  bool dominates(const BasicBlock *BB, const Use &U) const;
+  LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const;
 
   /// Return true if value Def dominates use U, in the sense that Def is
   /// available at U, and could be substituted as the used value without
@@ -188,11 +191,11 @@ class DominatorTree : public DominatorTreeBase<BasicBlock, false> {
   ///  * Def does not dominate a use in Def itself (outside of degenerate cases
   ///    like unreachable code or trivial phi cycles).
   ///  * Invoke Defs only dominate uses in their default destination.
-  bool dominates(const Value *Def, const Use &U) const;
+  LLVM_ABI bool dominates(const Value *Def, const Use &U) const;
 
   /// Return true if value Def dominates all possible uses inside instruction
   /// User. Same comments as for the Use-based API apply.
-  bool dominates(const Value *Def, const Instruction *User) const;
+  LLVM_ABI bool dominates(const Value *Def, const Instruction *User) const;
   bool dominates(const Value *Def, BasicBlock::iterator User) const {
     return dominates(Def, &*User);
   }
@@ -202,34 +205,36 @@ class DominatorTree : public DominatorTreeBase<BasicBlock, false> {
   ///
   /// Does not accept Value to avoid ambiguity with dominance checks between
   /// two basic blocks.
-  bool dominates(const Instruction *Def, const BasicBlock *BB) const;
+  LLVM_ABI bool dominates(const Instruction *Def, const BasicBlock *BB) const;
 
   /// Return true if an edge dominates a use.
   ///
   /// If BBE is not a unique edge between start and end of the edge, it can
   /// never dominate the use.
-  bool dominates(const BasicBlockEdge &BBE, const Use &U) const;
-  bool dominates(const BasicBlockEdge &BBE, const BasicBlock *BB) const;
+  LLVM_ABI bool dominates(const BasicBlockEdge &BBE, const Use &U) const;
+  LLVM_ABI bool dominates(const BasicBlockEdge &BBE,
+                          const BasicBlock *BB) const;
   /// Returns true if edge \p BBE1 dominates edge \p BBE2.
-  bool dominates(const BasicBlockEdge &BBE1, const BasicBlockEdge &BBE2) const;
+  LLVM_ABI bool dominates(const BasicBlockEdge &BBE1,
+                          const BasicBlockEdge &BBE2) const;
 
   // Ensure base class overloads are visible.
   using Base::isReachableFromEntry;
 
   /// Provide an overload for a Use.
-  bool isReachableFromEntry(const Use &U) const;
+  LLVM_ABI bool isReachableFromEntry(const Use &U) const;
 
   // Ensure base class overloads are visible.
   using Base::findNearestCommonDominator;
 
   /// Find the nearest instruction I that dominates both I1 and I2, in the sense
   /// that a result produced before I will be available at both I1 and I2.
-  Instruction *findNearestCommonDominator(Instruction *I1,
-                                          Instruction *I2) const;
+  LLVM_ABI Instruction *findNearestCommonDominator(Instruction *I1,
+                                                   Instruction *I2) const;
 
   // Pop up a GraphViz/gv window with the Dominator Tree rendered using `dot`.
-  void viewGraph(const Twine &Name, const Twine &Title);
-  void viewGraph();
+  LLVM_ABI void viewGraph(const Twine &Name, const Twine &Title);
+  LLVM_ABI void viewGraph();
 };
 
 //===-------------------------------------
@@ -278,14 +283,14 @@ template <> struct GraphTraits<DominatorTree*>
 /// Analysis pass which computes a \c DominatorTree.
 class DominatorTreeAnalysis : public AnalysisInfoMixin<DominatorTreeAnalysis> {
   friend AnalysisInfoMixin<DominatorTreeAnalysis>;
-  static AnalysisKey Key;
+  LLVM_ABI static AnalysisKey Key;
 
 public:
   /// Provide the result typedef for this analysis pass.
   using Result = DominatorTree;
 
   /// Run the analysis pass over a function and produce a dominator tree.
-  DominatorTree run(Function &F, FunctionAnalysisManager &);
+  LLVM_ABI DominatorTree run(Function &F, FunctionAnalysisManager &);
 };
 
 /// Printer pass for the \c DominatorTree.
@@ -294,16 +299,16 @@ class DominatorTreePrinterPass
   raw_ostream &OS;
 
 public:
-  explicit DominatorTreePrinterPass(raw_ostream &OS);
+  LLVM_ABI explicit DominatorTreePrinterPass(raw_ostream &OS);
 
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   static bool isRequired() { return true; }
 };
 
 /// Verifier pass for the \c DominatorTree.
 struct DominatorTreeVerifierPass : PassInfoMixin<DominatorTreeVerifierPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
@@ -311,10 +316,10 @@ struct DominatorTreeVerifierPass : PassInfoMixin<DominatorTreeVerifierPass> {
 ///
 /// This check is expensive and is disabled by default.  `-verify-dom-info`
 /// allows selectively enabling the check without needing to recompile.
-extern bool VerifyDomInfo;
+LLVM_ABI extern bool VerifyDomInfo;
 
 /// Legacy analysis pass which computes a \c DominatorTree.
-class DominatorTreeWrapperPass : public FunctionPass {
+class LLVM_ABI DominatorTreeWrapperPass : public FunctionPass {
   DominatorTree DT;
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStats.h b/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStats.h
index ebd74a69a8b9..42e86dd96675 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStats.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStats.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
 #include <tuple>
 
 namespace llvm {
@@ -39,7 +40,7 @@ using VarID =
 /// statistics.
 class DroppedVariableStats {
 public:
-  DroppedVariableStats(bool DroppedVarStatsEnabled);
+  LLVM_ABI DroppedVariableStats(bool DroppedVarStatsEnabled);
 
   virtual ~DroppedVariableStats() {}
 
@@ -50,9 +51,9 @@ class DroppedVariableStats {
   bool getPassDroppedVariables() { return PassDroppedVariables; }
 
 protected:
-  void setup();
+  LLVM_ABI void setup();
 
-  void cleanup();
+  LLVM_ABI void cleanup();
 
   bool DroppedVariableStatsEnabled = false;
   struct DebugVariables {
@@ -75,26 +76,26 @@ class DroppedVariableStats {
   SmallVector<DenseMap<StringRef, DenseMap<VarID, DILocation *>>> InlinedAts;
   /// Calculate the number of dropped variables in an llvm::Function or
   /// llvm::MachineFunction and print the relevant information to stdout.
-  void calculateDroppedStatsAndPrint(DebugVariables &DbgVariables,
-                                     StringRef FuncName, StringRef PassID,
-                                     StringRef FuncOrModName,
-                                     StringRef PassLevel, const Function *Func);
+  LLVM_ABI void calculateDroppedStatsAndPrint(
+      DebugVariables &DbgVariables, StringRef FuncName, StringRef PassID,
+      StringRef FuncOrModName, StringRef PassLevel, const Function *Func);
 
   /// Check if a \p Var has been dropped or is a false positive. Also update the
   /// \p DroppedCount if a debug variable is dropped.
-  bool updateDroppedCount(DILocation *DbgLoc, const DIScope *Scope,
-                          const DIScope *DbgValScope,
-                          DenseMap<VarID, DILocation *> &InlinedAtsMap,
-                          VarID Var, unsigned &DroppedCount);
+  LLVM_ABI bool updateDroppedCount(DILocation *DbgLoc, const DIScope *Scope,
+                                   const DIScope *DbgValScope,
+                                   DenseMap<VarID, DILocation *> &InlinedAtsMap,
+                                   VarID Var, unsigned &DroppedCount);
 
   /// Run code to populate relevant data structures over an llvm::Function or
   /// llvm::MachineFunction.
-  void run(DebugVariables &DbgVariables, StringRef FuncName, bool Before);
+  LLVM_ABI void run(DebugVariables &DbgVariables, StringRef FuncName,
+                    bool Before);
 
   /// Populate the VarIDSet and InlinedAtMap with the relevant information
   /// needed for before and after pass analysis to determine dropped variable
   /// status.
-  void populateVarIDSetAndInlinedMap(
+  LLVM_ABI void populateVarIDSetAndInlinedMap(
       const DILocalVariable *DbgVar, DebugLoc DbgLoc, DenseSet<VarID> &VarIDSet,
       DenseMap<StringRef, DenseMap<VarID, DILocation *>> &InlinedAtsMap,
       StringRef FuncName, bool Before);
diff --git a/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStatsIR.h b/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStatsIR.h
index 72b91dbc7ed5..3e182566f8cc 100644
--- a/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStatsIR.h
+++ b/external/llvm-project/llvm/include/llvm/IR/DroppedVariableStatsIR.h
@@ -15,6 +15,7 @@
 #define LLVM_CODEGEN_DROPPEDVARIABLESTATSIR_H
 
 #include "llvm/IR/DroppedVariableStats.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -28,7 +29,7 @@ class DILocation;
 /// A class to collect and print dropped debug information due to LLVM IR
 /// optimization passes. After every LLVM IR pass is run, it will print how many
 /// #dbg_values were dropped due to that pass.
-class DroppedVariableStatsIR : public DroppedVariableStats {
+class LLVM_ABI DroppedVariableStatsIR : public DroppedVariableStats {
 public:
   DroppedVariableStatsIR(bool DroppedVarStatsEnabled)
       : llvm::DroppedVariableStats(DroppedVarStatsEnabled) {}
diff --git a/external/llvm-project/llvm/include/llvm/IR/EHPersonalities.h b/external/llvm-project/llvm/include/llvm/IR/EHPersonalities.h
index c70f832de40b..39f6817b3dc3 100644
--- a/external/llvm-project/llvm/include/llvm/IR/EHPersonalities.h
+++ b/external/llvm-project/llvm/include/llvm/IR/EHPersonalities.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class BasicBlock;
@@ -39,11 +40,11 @@ enum class EHPersonality {
 /// See if the given exception handling personality function is one
 /// that we understand.  If so, return a description of it; otherwise return
 /// Unknown.
-EHPersonality classifyEHPersonality(const Value *Pers);
+LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers);
 
-StringRef getEHPersonalityName(EHPersonality Pers);
+LLVM_ABI StringRef getEHPersonalityName(EHPersonality Pers);
 
-EHPersonality getDefaultEHPersonality(const Triple &T);
+LLVM_ABI EHPersonality getDefaultEHPersonality(const Triple &T);
 
 /// Returns true if this personality function catches asynchronous
 /// exceptions.
@@ -104,7 +105,7 @@ inline bool isNoOpWithoutInvoke(EHPersonality Pers) {
   llvm_unreachable("invalid enum");
 }
 
-bool canSimplifyInvokeNoUnwind(const Function *F);
+LLVM_ABI bool canSimplifyInvokeNoUnwind(const Function *F);
 
 typedef TinyPtrVector<BasicBlock *> ColorVector;
 
@@ -112,7 +113,7 @@ typedef TinyPtrVector<BasicBlock *> ColorVector;
 /// this will recompute which blocks are in which funclet. It is possible that
 /// some blocks are in multiple funclets. Consider this analysis to be
 /// expensive.
-DenseMap<BasicBlock *, ColorVector> colorEHFunclets(Function &F);
+LLVM_ABI DenseMap<BasicBlock *, ColorVector> colorEHFunclets(Function &F);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/FMF.h b/external/llvm-project/llvm/include/llvm/IR/FMF.h
index da274a1d6ec4..8bd591250a38 100644
--- a/external/llvm-project/llvm/include/llvm/IR/FMF.h
+++ b/external/llvm-project/llvm/include/llvm/IR/FMF.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_IR_FMF_H
 #define LLVM_IR_FMF_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 class raw_ostream;
 
@@ -104,7 +106,7 @@ class FastMathFlags {
   }
 
   /// Print fast-math flags to \p O.
-  void print(raw_ostream &O) const;
+  LLVM_ABI void print(raw_ostream &O) const;
 
   /// Intersect rewrite-based flags
   static inline FastMathFlags intersectRewrite(FastMathFlags LHS,
diff --git a/external/llvm-project/llvm/include/llvm/IR/FPEnv.h b/external/llvm-project/llvm/include/llvm/IR/FPEnv.h
index a0197377759d..38395b15c8c0 100644
--- a/external/llvm-project/llvm/include/llvm/IR/FPEnv.h
+++ b/external/llvm-project/llvm/include/llvm/IR/FPEnv.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/IR/FMF.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -46,19 +47,21 @@ enum ExceptionBehavior : uint8_t {
 /// Returns a valid RoundingMode enumerator when given a string
 /// that is valid as input in constrained intrinsic rounding mode
 /// metadata.
-std::optional<RoundingMode> convertStrToRoundingMode(StringRef);
+LLVM_ABI std::optional<RoundingMode> convertStrToRoundingMode(StringRef);
 
 /// For any RoundingMode enumerator, returns a string valid as input in
 /// constrained intrinsic rounding mode metadata.
-std::optional<StringRef> convertRoundingModeToStr(RoundingMode);
+LLVM_ABI std::optional<StringRef> convertRoundingModeToStr(RoundingMode);
 
 /// Returns a valid ExceptionBehavior enumerator when given a string
 /// valid as input in constrained intrinsic exception behavior metadata.
-std::optional<fp::ExceptionBehavior> convertStrToExceptionBehavior(StringRef);
+LLVM_ABI std::optional<fp::ExceptionBehavior>
+    convertStrToExceptionBehavior(StringRef);
 
 /// For any ExceptionBehavior enumerator, returns a string valid as
 /// input in constrained intrinsic exception behavior metadata.
-std::optional<StringRef> convertExceptionBehaviorToStr(fp::ExceptionBehavior);
+LLVM_ABI std::optional<StringRef>
+    convertExceptionBehaviorToStr(fp::ExceptionBehavior);
 
 /// Returns true if the exception handling behavior and rounding mode
 /// match what is used in the default floating point environment.
@@ -70,7 +73,7 @@ inline bool isDefaultFPEnvironment(fp::ExceptionBehavior EB, RoundingMode RM) {
 /// strictfp function. If the instruction is already a constrained intrinsic or
 /// does not have a constrained intrinsic counterpart, the function returns
 /// zero.
-Intrinsic::ID getConstrainedIntrinsicID(const Instruction &Instr);
+LLVM_ABI Intrinsic::ID getConstrainedIntrinsicID(const Instruction &Instr);
 
 /// Returns true if the rounding mode RM may be QRM at compile time or
 /// at run time.
diff --git a/external/llvm-project/llvm/include/llvm/IR/Function.h b/external/llvm-project/llvm/include/llvm/IR/Function.h
index 6d4a53da7ff2..c361be3e752a 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Function.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Function.h
@@ -32,6 +32,7 @@
 #include "llvm/IR/OperandTraits.h"
 #include "llvm/IR/SymbolTableListTraits.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -110,11 +111,6 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
   friend class SymbolTableListTraits<Function>;
 
 public:
-  /// Is this function using intrinsics to record the position of debugging
-  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
-  /// \ref BasicBlock.
-  bool IsNewDbgInfoFormat;
-
   /// hasLazyArguments/CheckLazyArguments - The argument list of a function is
   /// built on demand, so that the list isn't allocated until the first client
   /// needs it.  The hasLazyArguments predicate returns true if the arg list
@@ -129,9 +125,6 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
   /// \see BasicBlock::convertFromNewDbgValues.
   void convertFromNewDbgValues();
 
-  void setIsNewDbgInfoFormat(bool NewVal);
-  void setNewDbgInfoFormatFlag(bool NewVal);
-
 private:
   friend class TargetLibraryInfoImpl;
 
@@ -759,7 +752,6 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
   /// to the newly inserted BB.
   Function::iterator insert(Function::iterator Position, BasicBlock *BB) {
     Function::iterator FIt = BasicBlocks.insert(Position, BB);
-    BB->setIsNewDbgInfoFormat(IsNewDbgInfoFormat);
     return FIt;
   }
 
@@ -791,8 +783,9 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
 
 private:
   // These need access to the underlying BB list.
-  friend void BasicBlock::removeFromParent();
-  friend iplist<BasicBlock>::iterator BasicBlock::eraseFromParent();
+  LLVM_ABI_FRIEND friend void BasicBlock::removeFromParent();
+  LLVM_ABI_FRIEND friend iplist<BasicBlock>::iterator
+  BasicBlock::eraseFromParent();
   template <class BB_t, class BB_i_t, class BI_t, class II_t>
   friend class InstIterator;
   friend class llvm::SymbolTableListTraits<llvm::BasicBlock>;
@@ -1036,6 +1029,25 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
   /// Return value: true =>  null pointer dereference is not undefined.
   bool nullPointerIsDefined() const;
 
+  /// Returns the alignment of the given function.
+  ///
+  /// Note that this is the alignment of the code, not the alignment of a
+  /// function pointer.
+  MaybeAlign getAlign() const { return GlobalObject::getAlign(); }
+
+  /// Sets the alignment attribute of the Function.
+  void setAlignment(Align Align) { GlobalObject::setAlignment(Align); }
+
+  /// Sets the alignment attribute of the Function.
+  ///
+  /// This method will be deprecated as the alignment property should always be
+  /// defined.
+  void setAlignment(MaybeAlign Align) { GlobalObject::setAlignment(Align); }
+
+  /// Return the value for vscale based on the vscale_range attribute or 0 when
+  /// unknown.
+  unsigned getVScaleValue() const;
+
 private:
   void allocHungoffUselist();
   template<int Idx> void setHungoffOperand(Constant *C);
@@ -1052,8 +1064,7 @@ namespace CallingConv {
 
 // TODO: Need similar function for support of argument in position. General
 // version on FunctionType + Attributes + CallingConv::ID?
-LLVM_READNONE
-bool supportsNonVoidReturnType(CallingConv::ID CC);
+LLVM_ABI LLVM_READNONE bool supportsNonVoidReturnType(CallingConv::ID CC);
 } // namespace CallingConv
 
 /// Check whether null pointer dereferencing is considered undefined behavior
@@ -1061,7 +1072,7 @@ bool supportsNonVoidReturnType(CallingConv::ID CC);
 /// Null pointer access in non-zero address space is not considered undefined.
 /// Return value: false => null pointer dereference is undefined.
 /// Return value: true =>  null pointer dereference is not undefined.
-bool NullPointerIsDefined(const Function *F, unsigned AS = 0);
+LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS = 0);
 
 template <> struct OperandTraits<Function> : public HungoffOperandTraits {};
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h b/external/llvm-project/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h
index 49c5fe0aed6e..8aac9d5b49db 100644
--- a/external/llvm-project/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h
+++ b/external/llvm-project/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -29,13 +30,14 @@ template <typename ContextT> class GenericFloatingPointPredicateUtils {
   constexpr static ValueRefT Invalid = {};
 
 private:
-  static DenormalMode queryDenormalMode(const FunctionT &F, ValueRefT Val);
+  LLVM_ABI static DenormalMode queryDenormalMode(const FunctionT &F,
+                                                 ValueRefT Val);
 
-  static bool lookThroughFAbs(const FunctionT &F, ValueRefT LHS,
-                              ValueRefT &Src);
+  LLVM_ABI static bool lookThroughFAbs(const FunctionT &F, ValueRefT LHS,
+                                       ValueRefT &Src);
 
-  static std::optional<APFloat> matchConstantFloat(const FunctionT &F,
-                                                   ValueRefT Val);
+  LLVM_ABI static std::optional<APFloat> matchConstantFloat(const FunctionT &F,
+                                                            ValueRefT Val);
 
   /// Return the return value for fcmpImpliesClass for a compare that produces
   /// an exact class test.
diff --git a/external/llvm-project/llvm/include/llvm/IR/GlobalAlias.h b/external/llvm-project/llvm/include/llvm/IR/GlobalAlias.h
index 3db6984c4a30..12b9f053abb1 100644
--- a/external/llvm-project/llvm/include/llvm/IR/GlobalAlias.h
+++ b/external/llvm-project/llvm/include/llvm/IR/GlobalAlias.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/OperandTraits.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -39,26 +40,26 @@ class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> {
 
   /// If a parent module is specified, the alias is automatically inserted into
   /// the end of the specified module's alias list.
-  static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
-                             LinkageTypes Linkage, const Twine &Name,
-                             Constant *Aliasee, Module *Parent);
+  LLVM_ABI static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
+                                      LinkageTypes Linkage, const Twine &Name,
+                                      Constant *Aliasee, Module *Parent);
 
   // Without the Aliasee.
-  static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
-                             LinkageTypes Linkage, const Twine &Name,
-                             Module *Parent);
+  LLVM_ABI static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
+                                      LinkageTypes Linkage, const Twine &Name,
+                                      Module *Parent);
 
   // The module is taken from the Aliasee.
-  static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
-                             LinkageTypes Linkage, const Twine &Name,
-                             GlobalValue *Aliasee);
+  LLVM_ABI static GlobalAlias *create(Type *Ty, unsigned AddressSpace,
+                                      LinkageTypes Linkage, const Twine &Name,
+                                      GlobalValue *Aliasee);
 
   // Type, Parent and AddressSpace taken from the Aliasee.
-  static GlobalAlias *create(LinkageTypes Linkage, const Twine &Name,
-                             GlobalValue *Aliasee);
+  LLVM_ABI static GlobalAlias *create(LinkageTypes Linkage, const Twine &Name,
+                                      GlobalValue *Aliasee);
 
   // Linkage, Type, Parent and AddressSpace taken from the Aliasee.
-  static GlobalAlias *create(const Twine &Name, GlobalValue *Aliasee);
+  LLVM_ABI static GlobalAlias *create(const Twine &Name, GlobalValue *Aliasee);
 
   // allocate space for exactly one operand
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -74,21 +75,21 @@ class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> {
   /// removeFromParent - This method unlinks 'this' from the containing module,
   /// but does not delete it.
   ///
-  void removeFromParent();
+  LLVM_ABI void removeFromParent();
 
   /// eraseFromParent - This method unlinks 'this' from the containing module
   /// and deletes it.
   ///
-  void eraseFromParent();
+  LLVM_ABI void eraseFromParent();
 
   /// These methods retrieve and set alias target.
-  void setAliasee(Constant *Aliasee);
+  LLVM_ABI void setAliasee(Constant *Aliasee);
   const Constant *getAliasee() const {
     return static_cast<Constant *>(Op<0>().get());
   }
   Constant *getAliasee() { return static_cast<Constant *>(Op<0>().get()); }
 
-  const GlobalObject *getAliaseeObject() const;
+  LLVM_ABI const GlobalObject *getAliaseeObject() const;
   GlobalObject *getAliaseeObject() {
     return const_cast<GlobalObject *>(
         static_cast<const GlobalAlias *>(this)->getAliaseeObject());
diff --git a/external/llvm-project/llvm/include/llvm/IR/GlobalIFunc.h b/external/llvm-project/llvm/include/llvm/IR/GlobalIFunc.h
index 0d2f152cef40..d93146aa016b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/GlobalIFunc.h
+++ b/external/llvm-project/llvm/include/llvm/IR/GlobalIFunc.h
@@ -22,6 +22,7 @@
 #include "llvm/IR/GlobalObject.h"
 #include "llvm/IR/OperandTraits.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -45,9 +46,9 @@ class GlobalIFunc final : public GlobalObject, public ilist_node<GlobalIFunc> {
 
   /// If a parent module is specified, the ifunc is automatically inserted into
   /// the end of the specified module's ifunc list.
-  static GlobalIFunc *create(Type *Ty, unsigned AddressSpace,
-                             LinkageTypes Linkage, const Twine &Name,
-                             Constant *Resolver, Module *Parent);
+  LLVM_ABI static GlobalIFunc *create(Type *Ty, unsigned AddressSpace,
+                                      LinkageTypes Linkage, const Twine &Name,
+                                      Constant *Resolver, Module *Parent);
 
   // allocate space for exactly one operand
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -62,10 +63,10 @@ class GlobalIFunc final : public GlobalObject, public ilist_node<GlobalIFunc> {
 
   /// This method unlinks 'this' from the containing module, but does not
   /// delete it.
-  void removeFromParent();
+  LLVM_ABI void removeFromParent();
 
   /// This method unlinks 'this' from the containing module and deletes it.
-  void eraseFromParent();
+  LLVM_ABI void eraseFromParent();
 
   /// These methods retrieve and set ifunc resolver function.
   void setResolver(Constant *Resolver) { Op<0>().set(Resolver); }
@@ -76,7 +77,7 @@ class GlobalIFunc final : public GlobalObject, public ilist_node<GlobalIFunc> {
 
   // Return the resolver function after peeling off potential ConstantExpr
   // indirection.
-  const Function *getResolverFunction() const;
+  LLVM_ABI const Function *getResolverFunction() const;
   Function *getResolverFunction() {
     return const_cast<Function *>(
         static_cast<const GlobalIFunc *>(this)->getResolverFunction());
@@ -96,7 +97,8 @@ class GlobalIFunc final : public GlobalObject, public ilist_node<GlobalIFunc> {
   // is already a global object, then apply the operation to it directly. If
   // target is a GlobalExpr or a GlobalAlias, evaluate it to its base object and
   // apply the operation for the base object and all aliases along the path.
-  void applyAlongResolverPath(function_ref<void(const GlobalValue &)> Op) const;
+  LLVM_ABI void
+  applyAlongResolverPath(function_ref<void(const GlobalValue &)> Op) const;
 };
 
 template <>
diff --git a/external/llvm-project/llvm/include/llvm/IR/GlobalObject.h b/external/llvm-project/llvm/include/llvm/IR/GlobalObject.h
index 73c1b87d6771..08a02b42bdc1 100644
--- a/external/llvm-project/llvm/include/llvm/IR/GlobalObject.h
+++ b/external/llvm-project/llvm/include/llvm/IR/GlobalObject.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -45,7 +46,7 @@ class GlobalObject : public GlobalValue {
       : GlobalValue(Ty, VTy, AllocInfo, Linkage, Name, AddressSpace) {
     setGlobalValueSubClassData(0);
   }
-  ~GlobalObject();
+  LLVM_ABI ~GlobalObject();
 
   Comdat *ObjComdat = nullptr;
   enum {
@@ -66,12 +67,7 @@ class GlobalObject : public GlobalValue {
 public:
   GlobalObject(const GlobalObject &) = delete;
 
-  /// FIXME: Remove this function once transition to Align is over.
-  uint64_t getAlignment() const {
-    MaybeAlign Align = getAlign();
-    return Align ? Align->value() : 0;
-  }
-
+protected:
   /// Returns the alignment of the given variable or function.
   ///
   /// Note that for functions this is the alignment of the code, not the
@@ -83,12 +79,12 @@ class GlobalObject : public GlobalValue {
   }
 
   /// Sets the alignment attribute of the GlobalObject.
-  void setAlignment(Align Align);
+  LLVM_ABI void setAlignment(Align Align);
 
   /// Sets the alignment attribute of the GlobalObject.
   /// This method will be deprecated as the alignment property should always be
   /// defined.
-  void setAlignment(MaybeAlign Align);
+  LLVM_ABI void setAlignment(MaybeAlign Align);
 
   unsigned getGlobalObjectSubClassData() const {
     unsigned ValueData = getGlobalValueSubClassData();
@@ -102,6 +98,7 @@ class GlobalObject : public GlobalValue {
     assert(getGlobalObjectSubClassData() == Val && "representation error");
   }
 
+public:
   /// Check if this global has a custom object file section.
   ///
   /// This is more efficient than calling getSection() and checking for an empty
@@ -122,18 +119,18 @@ class GlobalObject : public GlobalValue {
   ///
   /// Setting the section to the empty string tells LLVM to choose an
   /// appropriate default object file section.
-  void setSection(StringRef S);
+  LLVM_ABI void setSection(StringRef S);
 
   /// Set the section prefix for this global object.
-  void setSectionPrefix(StringRef Prefix);
+  LLVM_ABI void setSectionPrefix(StringRef Prefix);
 
   /// Get the section prefix for this global object.
-  std::optional<StringRef> getSectionPrefix() const;
+  LLVM_ABI std::optional<StringRef> getSectionPrefix() const;
 
   bool hasComdat() const { return getComdat() != nullptr; }
   const Comdat *getComdat() const { return ObjComdat; }
   Comdat *getComdat() { return ObjComdat; }
-  void setComdat(Comdat *C);
+  LLVM_ABI void setComdat(Comdat *C);
 
   using Value::addMetadata;
   using Value::clearMetadata;
@@ -145,21 +142,21 @@ class GlobalObject : public GlobalValue {
   using Value::setMetadata;
 
   /// Copy metadata from Src, adjusting offsets by Offset.
-  void copyMetadata(const GlobalObject *Src, unsigned Offset);
+  LLVM_ABI void copyMetadata(const GlobalObject *Src, unsigned Offset);
 
-  void addTypeMetadata(unsigned Offset, Metadata *TypeID);
-  void setVCallVisibilityMetadata(VCallVisibility Visibility);
-  VCallVisibility getVCallVisibility() const;
+  LLVM_ABI void addTypeMetadata(unsigned Offset, Metadata *TypeID);
+  LLVM_ABI void setVCallVisibilityMetadata(VCallVisibility Visibility);
+  LLVM_ABI VCallVisibility getVCallVisibility() const;
 
   /// Returns true if the alignment of the value can be unilaterally
   /// increased.
   ///
   /// Note that for functions this is the alignment of the code, not the
   /// alignment of a function pointer.
-  bool canIncreaseAlignment() const;
+  LLVM_ABI bool canIncreaseAlignment() const;
 
 protected:
-  void copyAttributesFrom(const GlobalObject *Src);
+  LLVM_ABI void copyAttributesFrom(const GlobalObject *Src);
 
 public:
   // Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -176,7 +173,7 @@ class GlobalObject : public GlobalValue {
                                (Val ? Mask : 0u));
   }
 
-  StringRef getSectionImpl() const;
+  LLVM_ABI StringRef getSectionImpl() const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/IR/GlobalValue.h b/external/llvm-project/llvm/include/llvm/IR/GlobalValue.h
index d9ecbb9f8f93..83e695cdd27d 100644
--- a/external/llvm-project/llvm/include/llvm/IR/GlobalValue.h
+++ b/external/llvm-project/llvm/include/llvm/IR/GlobalValue.h
@@ -23,6 +23,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 #include <cstdint>
@@ -162,7 +163,7 @@ class GlobalValue : public Constant {
 
   /// Returns true if the global is a function definition with the nobuiltin
   /// attribute.
-  bool isNobuiltinFnDef() const;
+  LLVM_ABI bool isNobuiltinFnDef() const;
 
 protected:
   /// The intrinsic ID for this subclass (which must be a Function).
@@ -240,7 +241,7 @@ class GlobalValue : public Constant {
   }
 
   bool hasComdat() const { return getComdat() != nullptr; }
-  const Comdat *getComdat() const;
+  LLVM_ABI const Comdat *getComdat() const;
   Comdat *getComdat() {
     return const_cast<Comdat *>(
                            static_cast<const GlobalValue *>(this)->getComdat());
@@ -289,7 +290,7 @@ class GlobalValue : public Constant {
   }
 
   bool hasSection() const { return !getSection().empty(); }
-  StringRef getSection() const;
+  LLVM_ABI StringRef getSection() const;
 
   /// Global values are always pointers.
   PointerType *getType() const { return cast<PointerType>(User::getType()); }
@@ -310,8 +311,8 @@ class GlobalValue : public Constant {
   bool hasPartition() const {
     return HasPartition;
   }
-  StringRef getPartition() const;
-  void setPartition(StringRef Part);
+  LLVM_ABI StringRef getPartition() const;
+  LLVM_ABI void setPartition(StringRef Part);
 
   // ASan, HWASan and Memtag sanitizers have some instrumentation that applies
   // specifically to global variables.
@@ -354,14 +355,14 @@ class GlobalValue : public Constant {
   };
 
   bool hasSanitizerMetadata() const { return HasSanitizerMetadata; }
-  const SanitizerMetadata &getSanitizerMetadata() const;
+  LLVM_ABI const SanitizerMetadata &getSanitizerMetadata() const;
   // Note: Not byref as it's a POD and otherwise it's too easy to call
   // G.setSanitizerMetadata(G2.getSanitizerMetadata()), and the argument becomes
   // dangling when the backing storage allocates the metadata for `G`, as the
   // storage is shared between `G1` and `G2`.
-  void setSanitizerMetadata(SanitizerMetadata Meta);
-  void removeSanitizerMetadata();
-  void setNoSanitizeMetadata();
+  LLVM_ABI void setSanitizerMetadata(SanitizerMetadata Meta);
+  LLVM_ABI void removeSanitizerMetadata();
+  LLVM_ABI void setNoSanitizeMetadata();
 
   bool isTagged() const {
     return hasSanitizerMetadata() && getSanitizerMetadata().Memtag;
@@ -506,8 +507,8 @@ class GlobalValue : public Constant {
   /// *arbitrary* definition at link time or load time. We cannot do any IPO or
   /// inlining across interposable call edges, since the callee can be
   /// replaced with something arbitrary.
-  bool isInterposable() const;
-  bool canBenefitFromLocalAlias() const;
+  LLVM_ABI bool isInterposable() const;
+  LLVM_ABI bool canBenefitFromLocalAlias() const;
 
   bool hasExternalLinkage() const { return isExternalLinkage(getLinkage()); }
   bool hasAvailableExternallyLinkage() const {
@@ -555,7 +556,7 @@ class GlobalValue : public Constant {
 protected:
   /// Copy all additional attributes (those not needed to create a GlobalValue)
   /// from the GlobalValue Src to this one.
-  void copyAttributesFrom(const GlobalValue *Src);
+  LLVM_ABI void copyAttributesFrom(const GlobalValue *Src);
 
 public:
   /// If the given string begins with the GlobalValue name mangling escape
@@ -579,20 +580,20 @@ class GlobalValue : public Constant {
   /// used as the key for a global lookup (e.g. profile or ThinLTO).
   /// The value's original name is \c Name and has linkage of type
   /// \c Linkage. The value is defined in module \c FileName.
-  static std::string getGlobalIdentifier(StringRef Name,
-                                         GlobalValue::LinkageTypes Linkage,
-                                         StringRef FileName);
+  LLVM_ABI static std::string
+  getGlobalIdentifier(StringRef Name, GlobalValue::LinkageTypes Linkage,
+                      StringRef FileName);
 
 private:
   /// Return the modified name for this global value suitable to be
   /// used as the key for a global lookup (e.g. profile or ThinLTO).
-  std::string getGlobalIdentifier() const;
+  LLVM_ABI std::string getGlobalIdentifier() const;
 
 public:
   /// Return a 64-bit global unique ID constructed from the name of a global
   /// symbol. Since this call doesn't supply the linkage or defining filename,
   /// the GUID computation will assume that the global has external linkage.
-  static GUID getGUIDAssumingExternalLinkage(StringRef GlobalName);
+  LLVM_ABI static GUID getGUIDAssumingExternalLinkage(StringRef GlobalName);
 
   /// Return a 64-bit global unique ID constructed from global value name
   /// (i.e. returned by getGlobalIdentifier()).
@@ -610,16 +611,16 @@ class GlobalValue : public Constant {
   /// If this function's Module is being lazily streamed in functions from disk
   /// or some other source, this method can be used to check to see if the
   /// function has been read in yet or not.
-  bool isMaterializable() const;
+  LLVM_ABI bool isMaterializable() const;
 
   /// Make sure this GlobalValue is fully read.
-  Error materialize();
+  LLVM_ABI Error materialize();
 
-/// @}
+  /// @}
 
   /// Return true if the primary definition of this global value is outside of
   /// the current translation unit.
-  bool isDeclaration() const;
+  LLVM_ABI bool isDeclaration() const;
 
   bool isDeclarationForLinker() const {
     if (hasAvailableExternallyLinkage())
@@ -638,25 +639,25 @@ class GlobalValue : public Constant {
     return !(isDeclarationForLinker() || isWeakForLinker());
   }
 
-  const GlobalObject *getAliaseeObject() const;
+  LLVM_ABI const GlobalObject *getAliaseeObject() const;
   GlobalObject *getAliaseeObject() {
     return const_cast<GlobalObject *>(
         static_cast<const GlobalValue *>(this)->getAliaseeObject());
   }
 
   /// Returns whether this is a reference to an absolute symbol.
-  bool isAbsoluteSymbolRef() const;
+  LLVM_ABI bool isAbsoluteSymbolRef() const;
 
   /// If this is an absolute symbol reference, returns the range of the symbol,
   /// otherwise returns std::nullopt.
-  std::optional<ConstantRange> getAbsoluteSymbolRange() const;
+  LLVM_ABI std::optional<ConstantRange> getAbsoluteSymbolRange() const;
 
   /// This method unlinks 'this' from the containing module, but does not delete
   /// it.
-  void removeFromParent();
+  LLVM_ABI void removeFromParent();
 
   /// This method unlinks 'this' from the containing module and deletes it.
-  void eraseFromParent();
+  LLVM_ABI void eraseFromParent();
 
   /// Get the module that this global value is contained inside of...
   Module *getParent() { return Parent; }
@@ -665,7 +666,7 @@ class GlobalValue : public Constant {
   /// Get the data layout of the module this global belongs to.
   ///
   /// Requires the global to have a parent module.
-  const DataLayout &getDataLayout() const;
+  LLVM_ABI const DataLayout &getDataLayout() const;
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
@@ -680,7 +681,7 @@ class GlobalValue : public Constant {
   /// is not normally profitable to omit them from the .o symbol table. Using
   /// this analysis makes sense when the information can be passed down to the
   /// linker or we are in LTO.
-  bool canBeOmittedFromSymbolTable() const;
+  LLVM_ABI bool canBeOmittedFromSymbolTable() const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/IR/GlobalVariable.h b/external/llvm-project/llvm/include/llvm/IR/GlobalVariable.h
index 781ac50ca7f3..13626a8f3003 100644
--- a/external/llvm-project/llvm/include/llvm/IR/GlobalVariable.h
+++ b/external/llvm-project/llvm/include/llvm/IR/GlobalVariable.h
@@ -25,6 +25,7 @@
 #include "llvm/IR/GlobalObject.h"
 #include "llvm/IR/OperandTraits.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstddef>
 
@@ -60,18 +61,21 @@ class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
 public:
   /// GlobalVariable ctor - If a parent module is specified, the global is
   /// automatically inserted into the end of the specified modules global list.
-  GlobalVariable(Type *Ty, bool isConstant, LinkageTypes Linkage,
-                 Constant *Initializer = nullptr, const Twine &Name = "",
-                 ThreadLocalMode = NotThreadLocal, unsigned AddressSpace = 0,
-                 bool isExternallyInitialized = false);
+  LLVM_ABI GlobalVariable(Type *Ty, bool isConstant, LinkageTypes Linkage,
+                          Constant *Initializer = nullptr,
+                          const Twine &Name = "",
+                          ThreadLocalMode = NotThreadLocal,
+                          unsigned AddressSpace = 0,
+                          bool isExternallyInitialized = false);
   /// GlobalVariable ctor - This creates a global and inserts it before the
   /// specified other global.
-  GlobalVariable(Module &M, Type *Ty, bool isConstant, LinkageTypes Linkage,
-                 Constant *Initializer, const Twine &Name = "",
-                 GlobalVariable *InsertBefore = nullptr,
-                 ThreadLocalMode = NotThreadLocal,
-                 std::optional<unsigned> AddressSpace = std::nullopt,
-                 bool isExternallyInitialized = false);
+  LLVM_ABI GlobalVariable(Module &M, Type *Ty, bool isConstant,
+                          LinkageTypes Linkage, Constant *Initializer,
+                          const Twine &Name = "",
+                          GlobalVariable *InsertBefore = nullptr,
+                          ThreadLocalMode = NotThreadLocal,
+                          std::optional<unsigned> AddressSpace = std::nullopt,
+                          bool isExternallyInitialized = false);
   GlobalVariable(const GlobalVariable &) = delete;
   GlobalVariable &operator=(const GlobalVariable &) = delete;
 
@@ -161,13 +165,13 @@ class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
   /// setInitializer - Sets the initializer for this global variable, removing
   /// any existing initializer if InitVal==NULL. The initializer must have the
   /// type getValueType().
-  void setInitializer(Constant *InitVal);
+  LLVM_ABI void setInitializer(Constant *InitVal);
 
   /// replaceInitializer - Sets the initializer for this global variable, and
   /// sets the value type of the global to the type of the initializer. The
   /// initializer must not be null.  This may affect the global's alignment if
   /// it isn't explicitly set.
-  void replaceInitializer(Constant *InitVal);
+  LLVM_ABI void replaceInitializer(Constant *InitVal);
 
   /// If the value is a global constant, its value is immutable throughout the
   /// runtime execution of the program.  Assigning a value into the constant
@@ -185,27 +189,28 @@ class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
 
   /// copyAttributesFrom - copy all additional attributes (those not needed to
   /// create a GlobalVariable) from the GlobalVariable Src to this one.
-  void copyAttributesFrom(const GlobalVariable *Src);
+  LLVM_ABI void copyAttributesFrom(const GlobalVariable *Src);
 
   /// removeFromParent - This method unlinks 'this' from the containing module,
   /// but does not delete it.
   ///
-  void removeFromParent();
+  LLVM_ABI void removeFromParent();
 
   /// eraseFromParent - This method unlinks 'this' from the containing module
   /// and deletes it.
   ///
-  void eraseFromParent();
+  LLVM_ABI void eraseFromParent();
 
   /// Drop all references in preparation to destroy the GlobalVariable. This
   /// drops not only the reference to the initializer but also to any metadata.
-  void dropAllReferences();
+  LLVM_ABI void dropAllReferences();
 
   /// Attach a DIGlobalVariableExpression.
-  void addDebugInfo(DIGlobalVariableExpression *GV);
+  LLVM_ABI void addDebugInfo(DIGlobalVariableExpression *GV);
 
   /// Fill the vector with all debug info attachements.
-  void getDebugInfo(SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const;
+  LLVM_ABI void
+  getDebugInfo(SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const;
 
   /// Attach a DIGlobalVariable.
   void addDebugInfo(DIGlobalVariable *GV);
@@ -302,11 +307,28 @@ class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
 
   /// Change the code model for this global.
   ///
-  void setCodeModel(CodeModel::Model CM);
+  LLVM_ABI void setCodeModel(CodeModel::Model CM);
 
   /// Remove the code model for this global.
   ///
-  void clearCodeModel();
+  LLVM_ABI void clearCodeModel();
+
+  /// FIXME: Remove this function once transition to Align is over.
+  uint64_t getAlignment() const {
+    MaybeAlign Align = getAlign();
+    return Align ? Align->value() : 0;
+  }
+
+  /// Returns the alignment of the given variable.
+  MaybeAlign getAlign() const { return GlobalObject::getAlign(); }
+
+  /// Sets the alignment attribute of the GlobalVariable.
+  void setAlignment(Align Align) { GlobalObject::setAlignment(Align); }
+
+  /// Sets the alignment attribute of the GlobalVariable.
+  /// This method will be deprecated as the alignment property should always be
+  /// defined.
+  void setAlignment(MaybeAlign Align) { GlobalObject::setAlignment(Align); }
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/IRBuilder.h b/external/llvm-project/llvm/include/llvm/IR/IRBuilder.h
index 7e0521e72ceb..59295089d6e9 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IRBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/IR/IRBuilder.h
@@ -41,6 +41,7 @@
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 #include <functional>
@@ -57,7 +58,7 @@ class Use;
 /// IRBuilder and needs to be inserted.
 ///
 /// By default, this inserts the instruction at the insertion point.
-class IRBuilderDefaultInserter {
+class LLVM_ABI IRBuilderDefaultInserter {
 public:
   virtual ~IRBuilderDefaultInserter();
 
@@ -71,7 +72,7 @@ class IRBuilderDefaultInserter {
 
 /// Provides an 'InsertHelper' that calls a user-provided callback after
 /// performing the default insertion.
-class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
+class LLVM_ABI IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
   std::function<void(Instruction *)> Callback;
 
 public:
@@ -112,12 +113,19 @@ class FMFSource {
 /// Common base class shared among various IRBuilders.
 class IRBuilderBase {
   /// Pairs of (metadata kind, MDNode *) that should be added to all newly
-  /// created instructions, like !dbg metadata.
+  /// created instructions, excluding !dbg metadata, which is stored in the
+  /// StoredDL field.
   SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;
+  /// The DebugLoc that will be applied to instructions inserted by this
+  /// builder.
+  DebugLoc StoredDL;
 
   /// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
   /// null. If \p MD is null, remove the entry with \p Kind.
   void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
+    assert(Kind != LLVMContext::MD_dbg &&
+           "MD_dbg metadata must be stored in StoredDL");
+
     if (!MD) {
       erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
         return KV.first == Kind;
@@ -237,7 +245,9 @@ class IRBuilderBase {
 
   /// Set location information used by debugging information.
   void SetCurrentDebugLocation(DebugLoc L) {
-    AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
+    // For !dbg metadata attachments, we use DebugLoc instead of the raw MDNode
+    // to include optional introspection data for use in Debugify.
+    StoredDL = std::move(L);
   }
 
   /// Set nosanitize metadata.
@@ -251,26 +261,31 @@ class IRBuilderBase {
   /// not on \p Src will be dropped from MetadataToCopy.
   void CollectMetadataToCopy(Instruction *Src,
                              ArrayRef<unsigned> MetadataKinds) {
-    for (unsigned K : MetadataKinds)
-      AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
+    for (unsigned K : MetadataKinds) {
+      if (K == LLVMContext::MD_dbg)
+        SetCurrentDebugLocation(Src->getDebugLoc());
+      else
+        AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
+    }
   }
 
   /// Get location information used by debugging information.
-  DebugLoc getCurrentDebugLocation() const;
+  LLVM_ABI DebugLoc getCurrentDebugLocation() const;
 
   /// If this builder has a current debug location, set it on the
   /// specified instruction.
-  void SetInstDebugLocation(Instruction *I) const;
+  LLVM_ABI void SetInstDebugLocation(Instruction *I) const;
 
   /// Add all entries in MetadataToCopy to \p I.
   void AddMetadataToInst(Instruction *I) const {
     for (const auto &KV : MetadataToCopy)
       I->setMetadata(KV.first, KV.second);
+    SetInstDebugLocation(I);
   }
 
   /// Get the return type of the current function that we're emitting
   /// into.
-  Type *getCurrentFunctionReturnType() const;
+  LLVM_ABI Type *getCurrentFunctionReturnType() const;
 
   /// InsertPoint - A saved insertion point.
   class InsertPoint {
@@ -472,9 +487,11 @@ class IRBuilderBase {
   ///
   /// If no module is given via \p M, it is take from the insertion point basic
   /// block.
-  GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
-                                     unsigned AddressSpace = 0,
-                                     Module *M = nullptr, bool AddNull = true);
+  LLVM_ABI GlobalVariable *CreateGlobalString(StringRef Str,
+                                              const Twine &Name = "",
+                                              unsigned AddressSpace = 0,
+                                              Module *M = nullptr,
+                                              bool AddNull = true);
 
   /// Get a constant value representing either true or false.
   ConstantInt *getInt1(bool V) {
@@ -616,13 +633,14 @@ class IRBuilderBase {
     return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile, AAInfo);
   }
 
-  CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
-                         bool isVolatile = false,
-                         const AAMDNodes &AAInfo = AAMDNodes());
+  LLVM_ABI CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size,
+                                  MaybeAlign Align, bool isVolatile = false,
+                                  const AAMDNodes &AAInfo = AAMDNodes());
 
-  CallInst *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val,
-                               Value *Size, bool IsVolatile = false,
-                               const AAMDNodes &AAInfo = AAMDNodes());
+  LLVM_ABI CallInst *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
+                                        Value *Val, Value *Size,
+                                        bool IsVolatile = false,
+                                        const AAMDNodes &AAInfo = AAMDNodes());
 
   /// Create and insert an element unordered-atomic memset of the region of
   /// memory starting at the given pointer to the given value.
@@ -637,22 +655,26 @@ class IRBuilderBase {
         Ptr, Val, getInt64(Size), Align(Alignment), ElementSize, AAInfo);
   }
 
-  CallInst *CreateMalloc(Type *IntPtrTy, Type *AllocTy, Value *AllocSize,
-                         Value *ArraySize, ArrayRef<OperandBundleDef> OpB,
-                         Function *MallocF = nullptr, const Twine &Name = "");
+  LLVM_ABI CallInst *CreateMalloc(Type *IntPtrTy, Type *AllocTy,
+                                  Value *AllocSize, Value *ArraySize,
+                                  ArrayRef<OperandBundleDef> OpB,
+                                  Function *MallocF = nullptr,
+                                  const Twine &Name = "");
 
   /// CreateMalloc - Generate the IR for a call to malloc:
   /// 1. Compute the malloc call's argument as the specified type's size,
   ///    possibly multiplied by the array size if the array size is not
   ///    constant 1.
   /// 2. Call malloc with that argument.
-  CallInst *CreateMalloc(Type *IntPtrTy, Type *AllocTy, Value *AllocSize,
-                         Value *ArraySize, Function *MallocF = nullptr,
-                         const Twine &Name = "");
+  LLVM_ABI CallInst *CreateMalloc(Type *IntPtrTy, Type *AllocTy,
+                                  Value *AllocSize, Value *ArraySize,
+                                  Function *MallocF = nullptr,
+                                  const Twine &Name = "");
   /// Generate the IR for a call to the builtin free function.
-  CallInst *CreateFree(Value *Source, ArrayRef<OperandBundleDef> Bundles = {});
+  LLVM_ABI CallInst *CreateFree(Value *Source,
+                                ArrayRef<OperandBundleDef> Bundles = {});
 
-  CallInst *
+  LLVM_ABI CallInst *
   CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, Value *Size,
                                      Align Alignment, uint32_t ElementSize,
                                      const AAMDNodes &AAInfo = AAMDNodes());
@@ -670,11 +692,11 @@ class IRBuilderBase {
                         isVolatile, AAInfo);
   }
 
-  CallInst *CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
-                                  MaybeAlign DstAlign, Value *Src,
-                                  MaybeAlign SrcAlign, Value *Size,
-                                  bool isVolatile = false,
-                                  const AAMDNodes &AAInfo = AAMDNodes());
+  LLVM_ABI CallInst *
+  CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign,
+                        Value *Src, MaybeAlign SrcAlign, Value *Size,
+                        bool isVolatile = false,
+                        const AAMDNodes &AAInfo = AAMDNodes());
 
   CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
                          MaybeAlign SrcAlign, Value *Size,
@@ -700,7 +722,7 @@ class IRBuilderBase {
   ///
   /// If the pointers aren't i8*, they will be converted.  If alias metadata is
   /// specified, it will be added to the instruction.
-  CallInst *CreateElementUnorderedAtomicMemCpy(
+  LLVM_ABI CallInst *CreateElementUnorderedAtomicMemCpy(
       Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
       uint32_t ElementSize, const AAMDNodes &AAInfo = AAMDNodes());
 
@@ -728,7 +750,7 @@ class IRBuilderBase {
   ///
   /// If the pointers aren't i8*, they will be converted.  If alias metadata is
   /// specified, it will be added to the instruction.
-  CallInst *CreateElementUnorderedAtomicMemMove(
+  LLVM_ABI CallInst *CreateElementUnorderedAtomicMemMove(
       Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
       uint32_t ElementSize, const AAMDNodes &AAInfo = AAMDNodes());
 
@@ -740,99 +762,106 @@ class IRBuilderBase {
   /// The first parameter is a scalar accumulator value. An unordered reduction
   /// can be created by adding the reassoc fast-math flag to the resulting
   /// sequential reduction.
-  CallInst *CreateFAddReduce(Value *Acc, Value *Src);
+  LLVM_ABI CallInst *CreateFAddReduce(Value *Acc, Value *Src);
 
   /// Create a sequential vector fmul reduction intrinsic of the source vector.
   /// The first parameter is a scalar accumulator value. An unordered reduction
   /// can be created by adding the reassoc fast-math flag to the resulting
   /// sequential reduction.
-  CallInst *CreateFMulReduce(Value *Acc, Value *Src);
+  LLVM_ABI CallInst *CreateFMulReduce(Value *Acc, Value *Src);
 
   /// Create a vector int add reduction intrinsic of the source vector.
-  CallInst *CreateAddReduce(Value *Src);
+  LLVM_ABI CallInst *CreateAddReduce(Value *Src);
 
   /// Create a vector int mul reduction intrinsic of the source vector.
-  CallInst *CreateMulReduce(Value *Src);
+  LLVM_ABI CallInst *CreateMulReduce(Value *Src);
 
   /// Create a vector int AND reduction intrinsic of the source vector.
-  CallInst *CreateAndReduce(Value *Src);
+  LLVM_ABI CallInst *CreateAndReduce(Value *Src);
 
   /// Create a vector int OR reduction intrinsic of the source vector.
-  CallInst *CreateOrReduce(Value *Src);
+  LLVM_ABI CallInst *CreateOrReduce(Value *Src);
 
   /// Create a vector int XOR reduction intrinsic of the source vector.
-  CallInst *CreateXorReduce(Value *Src);
+  LLVM_ABI CallInst *CreateXorReduce(Value *Src);
 
   /// Create a vector integer max reduction intrinsic of the source
   /// vector.
-  CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
+  LLVM_ABI CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
 
   /// Create a vector integer min reduction intrinsic of the source
   /// vector.
-  CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
+  LLVM_ABI CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
 
   /// Create a vector float max reduction intrinsic of the source
   /// vector.
-  CallInst *CreateFPMaxReduce(Value *Src);
+  LLVM_ABI CallInst *CreateFPMaxReduce(Value *Src);
 
   /// Create a vector float min reduction intrinsic of the source
   /// vector.
-  CallInst *CreateFPMinReduce(Value *Src);
+  LLVM_ABI CallInst *CreateFPMinReduce(Value *Src);
 
   /// Create a vector float maximum reduction intrinsic of the source
   /// vector. This variant follows the NaN and signed zero semantic of
   /// llvm.maximum intrinsic.
-  CallInst *CreateFPMaximumReduce(Value *Src);
+  LLVM_ABI CallInst *CreateFPMaximumReduce(Value *Src);
 
   /// Create a vector float minimum reduction intrinsic of the source
   /// vector. This variant follows the NaN and signed zero semantic of
   /// llvm.minimum intrinsic.
-  CallInst *CreateFPMinimumReduce(Value *Src);
+  LLVM_ABI CallInst *CreateFPMinimumReduce(Value *Src);
 
   /// Create a lifetime.start intrinsic.
   ///
   /// If the pointer isn't i8* it will be converted.
-  CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);
+  LLVM_ABI CallInst *CreateLifetimeStart(Value *Ptr,
+                                         ConstantInt *Size = nullptr);
 
   /// Create a lifetime.end intrinsic.
   ///
   /// If the pointer isn't i8* it will be converted.
-  CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
+  LLVM_ABI CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
 
   /// Create a call to invariant.start intrinsic.
   ///
   /// If the pointer isn't i8* it will be converted.
-  CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
+  LLVM_ABI CallInst *CreateInvariantStart(Value *Ptr,
+                                          ConstantInt *Size = nullptr);
 
   /// Create a call to llvm.threadlocal.address intrinsic.
-  CallInst *CreateThreadLocalAddress(Value *Ptr);
+  LLVM_ABI CallInst *CreateThreadLocalAddress(Value *Ptr);
 
   /// Create a call to Masked Load intrinsic
-  CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask,
-                             Value *PassThru = nullptr, const Twine &Name = "");
+  LLVM_ABI CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
+                                      Value *Mask, Value *PassThru = nullptr,
+                                      const Twine &Name = "");
 
   /// Create a call to Masked Store intrinsic
-  CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
-                              Value *Mask);
+  LLVM_ABI CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
+                                       Value *Mask);
 
   /// Create a call to Masked Gather intrinsic
-  CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
-                               Value *Mask = nullptr, Value *PassThru = nullptr,
-                               const Twine &Name = "");
+  LLVM_ABI CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
+                                        Value *Mask = nullptr,
+                                        Value *PassThru = nullptr,
+                                        const Twine &Name = "");
 
   /// Create a call to Masked Scatter intrinsic
-  CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
-                                Value *Mask = nullptr);
+  LLVM_ABI CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs,
+                                         Align Alignment,
+                                         Value *Mask = nullptr);
 
   /// Create a call to Masked Expand Load intrinsic
-  CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
-                                   Value *Mask = nullptr,
-                                   Value *PassThru = nullptr,
-                                   const Twine &Name = "");
+  LLVM_ABI CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
+                                            MaybeAlign Align,
+                                            Value *Mask = nullptr,
+                                            Value *PassThru = nullptr,
+                                            const Twine &Name = "");
 
   /// Create a call to Masked Compress Store intrinsic
-  CallInst *CreateMaskedCompressStore(Value *Val, Value *Ptr, MaybeAlign Align,
-                                      Value *Mask = nullptr);
+  LLVM_ABI CallInst *CreateMaskedCompressStore(Value *Val, Value *Ptr,
+                                               MaybeAlign Align,
+                                               Value *Mask = nullptr);
 
   /// Return an all true boolean vector (mask) with \p NumElts lanes.
   Value *getAllOnesMask(ElementCount NumElts) {
@@ -845,11 +874,11 @@ class IRBuilderBase {
   ///
   /// The optional argument \p OpBundles specifies operand bundles that are
   /// added to the call instruction.
-  CallInst *CreateAssumption(Value *Cond,
-                             ArrayRef<OperandBundleDef> OpBundles = {});
+  LLVM_ABI CallInst *
+  CreateAssumption(Value *Cond, ArrayRef<OperandBundleDef> OpBundles = {});
 
   /// Create a llvm.experimental.noalias.scope.decl intrinsic call.
-  Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
+  LLVM_ABI Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
   Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
     return CreateNoAliasScopeDeclaration(
         MetadataAsValue::get(Context, ScopeTag));
@@ -857,36 +886,33 @@ class IRBuilderBase {
 
   /// Create a call to the experimental.gc.statepoint intrinsic to
   /// start a new statepoint sequence.
-  CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
-                                   FunctionCallee ActualCallee,
-                                   ArrayRef<Value *> CallArgs,
-                                   std::optional<ArrayRef<Value *>> DeoptArgs,
-                                   ArrayRef<Value *> GCArgs,
-                                   const Twine &Name = "");
+  LLVM_ABI CallInst *CreateGCStatepointCall(
+      uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
+      ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
+      ArrayRef<Value *> GCArgs, const Twine &Name = "");
 
   /// Create a call to the experimental.gc.statepoint intrinsic to
   /// start a new statepoint sequence.
-  CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
-                                   FunctionCallee ActualCallee, uint32_t Flags,
-                                   ArrayRef<Value *> CallArgs,
-                                   std::optional<ArrayRef<Use>> TransitionArgs,
-                                   std::optional<ArrayRef<Use>> DeoptArgs,
-                                   ArrayRef<Value *> GCArgs,
-                                   const Twine &Name = "");
+  LLVM_ABI CallInst *
+  CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
+                         FunctionCallee ActualCallee, uint32_t Flags,
+                         ArrayRef<Value *> CallArgs,
+                         std::optional<ArrayRef<Use>> TransitionArgs,
+                         std::optional<ArrayRef<Use>> DeoptArgs,
+                         ArrayRef<Value *> GCArgs, const Twine &Name = "");
 
   /// Conveninence function for the common case when CallArgs are filled
   /// in using ArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
   /// .get()'ed to get the Value pointer.
-  CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
-                                   FunctionCallee ActualCallee,
-                                   ArrayRef<Use> CallArgs,
-                                   std::optional<ArrayRef<Value *>> DeoptArgs,
-                                   ArrayRef<Value *> GCArgs,
-                                   const Twine &Name = "");
+  LLVM_ABI CallInst *
+  CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
+                         FunctionCallee ActualCallee, ArrayRef<Use> CallArgs,
+                         std::optional<ArrayRef<Value *>> DeoptArgs,
+                         ArrayRef<Value *> GCArgs, const Twine &Name = "");
 
   /// Create an invoke to the experimental.gc.statepoint intrinsic to
   /// start a new statepoint sequence.
-  InvokeInst *
+  LLVM_ABI InvokeInst *
   CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
                            FunctionCallee ActualInvokee, BasicBlock *NormalDest,
                            BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
@@ -895,7 +921,7 @@ class IRBuilderBase {
 
   /// Create an invoke to the experimental.gc.statepoint intrinsic to
   /// start a new statepoint sequence.
-  InvokeInst *CreateGCStatepointInvoke(
+  LLVM_ABI InvokeInst *CreateGCStatepointInvoke(
       uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
       BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
       ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
@@ -905,7 +931,7 @@ class IRBuilderBase {
   // Convenience function for the common case when CallArgs are filled in using
   // ArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
   // get the Value *.
-  InvokeInst *
+  LLVM_ABI InvokeInst *
   CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
                            FunctionCallee ActualInvokee, BasicBlock *NormalDest,
                            BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
@@ -914,66 +940,68 @@ class IRBuilderBase {
 
   /// Create a call to the experimental.gc.result intrinsic to extract
   /// the result from a call wrapped in a statepoint.
-  CallInst *CreateGCResult(Instruction *Statepoint,
-                           Type *ResultType,
-                           const Twine &Name = "");
+  LLVM_ABI CallInst *CreateGCResult(Instruction *Statepoint, Type *ResultType,
+                                    const Twine &Name = "");
 
   /// Create a call to the experimental.gc.relocate intrinsics to
   /// project the relocated value of one pointer from the statepoint.
-  CallInst *CreateGCRelocate(Instruction *Statepoint,
-                             int BaseOffset,
-                             int DerivedOffset,
-                             Type *ResultType,
-                             const Twine &Name = "");
+  LLVM_ABI CallInst *CreateGCRelocate(Instruction *Statepoint, int BaseOffset,
+                                      int DerivedOffset, Type *ResultType,
+                                      const Twine &Name = "");
 
   /// Create a call to the experimental.gc.pointer.base intrinsic to get the
   /// base pointer for the specified derived pointer.
-  CallInst *CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name = "");
+  LLVM_ABI CallInst *CreateGCGetPointerBase(Value *DerivedPtr,
+                                            const Twine &Name = "");
 
   /// Create a call to the experimental.gc.get.pointer.offset intrinsic to get
   /// the offset of the specified derived pointer from its base.
-  CallInst *CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name = "");
+  LLVM_ABI CallInst *CreateGCGetPointerOffset(Value *DerivedPtr,
+                                              const Twine &Name = "");
 
-  /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
-  /// will be the same type as that of \p Scaling.
-  Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
+  /// Create a call to llvm.vscale.<Ty>().
+  LLVM_ABI Value *CreateVScale(Type *Ty, const Twine &Name = "") {
+    return CreateIntrinsic(Intrinsic::vscale, {Ty}, {}, {}, Name);
+  }
 
   /// Create an expression which evaluates to the number of elements in \p EC
   /// at runtime.
-  Value *CreateElementCount(Type *DstType, ElementCount EC);
+  LLVM_ABI Value *CreateElementCount(Type *Ty, ElementCount EC);
 
   /// Create an expression which evaluates to the number of units in \p Size
   /// at runtime.  This works for both units of bits and bytes.
-  Value *CreateTypeSize(Type *DstType, TypeSize Size);
+  LLVM_ABI Value *CreateTypeSize(Type *Ty, TypeSize Size);
 
   /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
-  Value *CreateStepVector(Type *DstType, const Twine &Name = "");
+  LLVM_ABI Value *CreateStepVector(Type *DstType, const Twine &Name = "");
 
   /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
   /// type.
-  CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
-                                 FMFSource FMFSource = {},
-                                 const Twine &Name = "");
+  LLVM_ABI CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
+                                          FMFSource FMFSource = {},
+                                          const Twine &Name = "");
 
   /// Create a call to intrinsic \p ID with 2 operands which is mangled on the
   /// first type.
-  Value *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
-                               FMFSource FMFSource = {},
-                               const Twine &Name = "");
+  LLVM_ABI Value *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
+                                        Value *RHS, FMFSource FMFSource = {},
+                                        const Twine &Name = "");
 
   /// Create a call to intrinsic \p ID with \p Args, mangled using \p Types. If
   /// \p FMFSource is provided, copy fast-math-flags from that instruction to
   /// the intrinsic.
-  CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
-                            ArrayRef<Value *> Args, FMFSource FMFSource = {},
-                            const Twine &Name = "");
+  LLVM_ABI CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
+                                     ArrayRef<Value *> Args,
+                                     FMFSource FMFSource = {},
+                                     const Twine &Name = "");
 
   /// Create a call to intrinsic \p ID with \p RetTy and \p Args. If
   /// \p FMFSource is provided, copy fast-math-flags from that instruction to
   /// the intrinsic.
-  CallInst *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
-                            ArrayRef<Value *> Args, FMFSource FMFSource = {},
-                            const Twine &Name = "");
+  LLVM_ABI CallInst *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
+                                     ArrayRef<Value *> Args,
+                                     FMFSource FMFSource = {},
+                                     const Twine &Name = "");
 
   /// Create a call to non-overloaded intrinsic \p ID with \p Args. If
   /// \p FMFSource is provided, copy fast-math-flags from that instruction to
@@ -1732,19 +1760,19 @@ class IRBuilderBase {
   /// the created intrinsic call according to \p Rounding and \p
   /// Except and it sets \p FPMathTag as the 'fpmath' metadata, using
   /// defaults if a value equals nullopt/null.
-  CallInst *CreateConstrainedFPIntrinsic(
+  LLVM_ABI CallInst *CreateConstrainedFPIntrinsic(
       Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
       FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag = nullptr,
       std::optional<RoundingMode> Rounding = std::nullopt,
       std::optional<fp::ExceptionBehavior> Except = std::nullopt);
 
-  CallInst *CreateConstrainedFPBinOp(
+  LLVM_ABI CallInst *CreateConstrainedFPBinOp(
       Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource = {},
       const Twine &Name = "", MDNode *FPMathTag = nullptr,
       std::optional<RoundingMode> Rounding = std::nullopt,
       std::optional<fp::ExceptionBehavior> Except = std::nullopt);
 
-  CallInst *CreateConstrainedFPUnroundedBinOp(
+  LLVM_ABI CallInst *CreateConstrainedFPUnroundedBinOp(
       Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource = {},
       const Twine &Name = "", MDNode *FPMathTag = nullptr,
       std::optional<fp::ExceptionBehavior> Except = std::nullopt);
@@ -1790,8 +1818,9 @@ class IRBuilderBase {
 
   /// Create either a UnaryOperator or BinaryOperator depending on \p Opc.
   /// Correct number of operands must be passed accordingly.
-  Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
-                      const Twine &Name = "", MDNode *FPMathTag = nullptr);
+  LLVM_ABI Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
+                               const Twine &Name = "",
+                               MDNode *FPMathTag = nullptr);
 
   //===--------------------------------------------------------------------===//
   // Instruction creation methods: Memory Instructions
@@ -2271,7 +2300,7 @@ class IRBuilderBase {
     return CreateCast(CastOp, V, DestTy, Name, FPMathTag);
   }
 
-  CallInst *CreateConstrainedFPCast(
+  LLVM_ABI CallInst *CreateConstrainedFPCast(
       Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource = {},
       const Twine &Name = "", MDNode *FPMathTag = nullptr,
       std::optional<RoundingMode> Rounding = std::nullopt,
@@ -2287,7 +2316,7 @@ class IRBuilderBase {
   /// casted, and then reinserted into a value of type DestTy. The leaf types
   /// must be castable using a bitcast or ptrcast, because signedness is
   /// not specified.
-  Value *CreateAggregateCast(Value *V, Type *DestTy);
+  LLVM_ABI Value *CreateAggregateCast(Value *V, Type *DestTy);
 
   //===--------------------------------------------------------------------===//
   // Instruction creation methods: Compare Instructions
@@ -2444,12 +2473,12 @@ class IRBuilderBase {
 
 private:
   // Helper routine to create either a signaling or a quiet FP comparison.
-  Value *CreateFCmpHelper(CmpInst::Predicate P, Value *LHS, Value *RHS,
-                          const Twine &Name, MDNode *FPMathTag,
-                          FMFSource FMFSource, bool IsSignaling);
+  LLVM_ABI Value *CreateFCmpHelper(CmpInst::Predicate P, Value *LHS, Value *RHS,
+                                   const Twine &Name, MDNode *FPMathTag,
+                                   FMFSource FMFSource, bool IsSignaling);
 
 public:
-  CallInst *CreateConstrainedFPCmp(
+  LLVM_ABI CallInst *CreateConstrainedFPCmp(
       Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
       const Twine &Name = "",
       std::optional<fp::ExceptionBehavior> Except = std::nullopt);
@@ -2507,16 +2536,17 @@ class IRBuilderBase {
                       OpBundles, Name, FPMathTag);
   }
 
-  CallInst *CreateConstrainedFPCall(
+  LLVM_ABI CallInst *CreateConstrainedFPCall(
       Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "",
       std::optional<RoundingMode> Rounding = std::nullopt,
       std::optional<fp::ExceptionBehavior> Except = std::nullopt);
 
-  Value *CreateSelect(Value *C, Value *True, Value *False,
-                      const Twine &Name = "", Instruction *MDFrom = nullptr);
-  Value *CreateSelectFMF(Value *C, Value *True, Value *False,
-                         FMFSource FMFSource, const Twine &Name = "",
-                         Instruction *MDFrom = nullptr);
+  LLVM_ABI Value *CreateSelect(Value *C, Value *True, Value *False,
+                               const Twine &Name = "",
+                               Instruction *MDFrom = nullptr);
+  LLVM_ABI Value *CreateSelectFMF(Value *C, Value *True, Value *False,
+                                  FMFSource FMFSource, const Twine &Name = "",
+                                  Instruction *MDFrom = nullptr);
 
   VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") {
     return Insert(new VAArgInst(List, Ty), Name);
@@ -2632,21 +2662,21 @@ class IRBuilderBase {
   /// This is intended to implement C-style pointer subtraction. As such, the
   /// pointers must be appropriately aligned for their element types and
   /// pointing into the same object.
-  Value *CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
-                       const Twine &Name = "");
+  LLVM_ABI Value *CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
+                                const Twine &Name = "");
 
   /// Create a launder.invariant.group intrinsic call. If Ptr type is
   /// different from pointer to i8, it's casted to pointer to i8 in the same
   /// address space before call and casted back to Ptr type after call.
-  Value *CreateLaunderInvariantGroup(Value *Ptr);
+  LLVM_ABI Value *CreateLaunderInvariantGroup(Value *Ptr);
 
   /// \brief Create a strip.invariant.group intrinsic call. If Ptr type is
   /// different from pointer to i8, it's casted to pointer to i8 in the same
   /// address space before call and casted back to Ptr type after call.
-  Value *CreateStripInvariantGroup(Value *Ptr);
+  LLVM_ABI Value *CreateStripInvariantGroup(Value *Ptr);
 
   /// Return a vector value that contains the vector V reversed
-  Value *CreateVectorReverse(Value *V, const Twine &Name = "");
+  LLVM_ABI Value *CreateVectorReverse(Value *V, const Twine &Name = "");
 
   /// Return a vector splice intrinsic if using scalable vectors, otherwise
   /// return a shufflevector. If the immediate is positive, a vector is
@@ -2655,29 +2685,34 @@ class IRBuilderBase {
   /// elements from V2. Imm is a signed integer in the range
   /// -VL <= Imm < VL (where VL is the runtime vector length of the
   /// source/result vector)
-  Value *CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
-                            const Twine &Name = "");
+  LLVM_ABI Value *CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
+                                     const Twine &Name = "");
 
   /// Return a vector value that contains \arg V broadcasted to \p
   /// NumElts elements.
-  Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");
+  LLVM_ABI Value *CreateVectorSplat(unsigned NumElts, Value *V,
+                                    const Twine &Name = "");
 
   /// Return a vector value that contains \arg V broadcasted to \p
   /// EC elements.
-  Value *CreateVectorSplat(ElementCount EC, Value *V, const Twine &Name = "");
+  LLVM_ABI Value *CreateVectorSplat(ElementCount EC, Value *V,
+                                    const Twine &Name = "");
 
-  Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
-                                        unsigned Dimension, unsigned LastIndex,
-                                        MDNode *DbgInfo);
+  LLVM_ABI Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
+                                                 unsigned Dimension,
+                                                 unsigned LastIndex,
+                                                 MDNode *DbgInfo);
 
-  Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex,
-                                        MDNode *DbgInfo);
+  LLVM_ABI Value *CreatePreserveUnionAccessIndex(Value *Base,
+                                                 unsigned FieldIndex,
+                                                 MDNode *DbgInfo);
 
-  Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base,
-                                         unsigned Index, unsigned FieldIndex,
-                                         MDNode *DbgInfo);
+  LLVM_ABI Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base,
+                                                  unsigned Index,
+                                                  unsigned FieldIndex,
+                                                  MDNode *DbgInfo);
 
-  Value *createIsFPClass(Value *FPNum, unsigned Test);
+  LLVM_ABI Value *createIsFPClass(Value *FPNum, unsigned Test);
 
 private:
   /// Helper function that creates an assume intrinsic call that
@@ -2694,9 +2729,10 @@ class IRBuilderBase {
   /// An optional offset can be provided, and if it is provided, the offset
   /// must be subtracted from the provided pointer to get the pointer with the
   /// specified alignment.
-  CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
-                                      unsigned Alignment,
-                                      Value *OffsetValue = nullptr);
+  LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
+                                               Value *PtrValue,
+                                               unsigned Alignment,
+                                               Value *OffsetValue = nullptr);
 
   /// Create an assume intrinsic call that represents an alignment
   /// assumption on the provided pointer.
@@ -2707,13 +2743,15 @@ class IRBuilderBase {
   ///
   /// This overload handles the condition where the Alignment is dependent
   /// on an existing value rather than a static value.
-  CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
-                                      Value *Alignment,
-                                      Value *OffsetValue = nullptr);
+  LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
+                                               Value *PtrValue,
+                                               Value *Alignment,
+                                               Value *OffsetValue = nullptr);
 
   /// Create an assume intrinsic call that represents an dereferencable
   /// assumption on the provided pointer.
-  CallInst *CreateDereferenceableAssumption(Value *PtrValue, Value *SizeValue);
+  LLVM_ABI CallInst *CreateDereferenceableAssumption(Value *PtrValue,
+                                                     Value *SizeValue);
 };
 
 /// This provides a uniform API for creating instructions and inserting
diff --git a/external/llvm-project/llvm/include/llvm/IR/IRBuilderFolder.h b/external/llvm-project/llvm/include/llvm/IR/IRBuilderFolder.h
index 921001c8a5d5..db4ab5af2433 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/external/llvm-project/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -18,11 +18,12 @@
 #include "llvm/IR/GEPNoWrapFlags.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// IRBuilderFolder - Interface for constant folding in IRBuilder.
-class IRBuilderFolder {
+class LLVM_ABI IRBuilderFolder {
 public:
   virtual ~IRBuilderFolder();
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/IRPrintingPasses.h b/external/llvm-project/llvm/include/llvm/IR/IRPrintingPasses.h
index 8928ecfce313..1b2d38d6190e 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IRPrintingPasses.h
+++ b/external/llvm-project/llvm/include/llvm/IR/IRPrintingPasses.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_IR_IRPRINTINGPASSES_H
 #define LLVM_IR_IRPRINTINGPASSES_H
 
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -26,23 +27,23 @@ class Pass;
 
 /// Create and return a pass that writes the module to the specified
 /// \c raw_ostream.
-ModulePass *createPrintModulePass(raw_ostream &OS,
-                                  const std::string &Banner = "",
-                                  bool ShouldPreserveUseListOrder = false);
+LLVM_ABI ModulePass *
+createPrintModulePass(raw_ostream &OS, const std::string &Banner = "",
+                      bool ShouldPreserveUseListOrder = false);
 
 /// Create and return a pass that prints functions to the specified
 /// \c raw_ostream as they are processed.
-FunctionPass *createPrintFunctionPass(raw_ostream &OS,
-                                      const std::string &Banner = "");
+LLVM_ABI FunctionPass *createPrintFunctionPass(raw_ostream &OS,
+                                               const std::string &Banner = "");
 
 /// Print out a name of an LLVM value without any prefixes.
 ///
 /// The name is surrounded with ""'s and escaped if it has any special or
 /// non-printable characters in it.
-void printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name);
+LLVM_ABI void printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name);
 
 /// Return true if a pass is for IR printing.
-bool isIRPrintingPass(Pass *P);
+LLVM_ABI bool isIRPrintingPass(Pass *P);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/InlineAsm.h b/external/llvm-project/llvm/include/llvm/IR/InlineAsm.h
index 3501c877a5c2..96887d129a69 100644
--- a/external/llvm-project/llvm/include/llvm/IR/InlineAsm.h
+++ b/external/llvm-project/llvm/include/llvm/IR/InlineAsm.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 #include <string>
@@ -63,10 +64,11 @@ class InlineAsm final : public Value {
 
   /// InlineAsm::get - Return the specified uniqued inline asm string.
   ///
-  static InlineAsm *get(FunctionType *Ty, StringRef AsmString,
-                        StringRef Constraints, bool hasSideEffects,
-                        bool isAlignStack = false,
-                        AsmDialect asmDialect = AD_ATT, bool canThrow = false);
+  LLVM_ABI static InlineAsm *get(FunctionType *Ty, StringRef AsmString,
+                                 StringRef Constraints, bool hasSideEffects,
+                                 bool isAlignStack = false,
+                                 AsmDialect asmDialect = AD_ATT,
+                                 bool canThrow = false);
 
   bool hasSideEffects() const { return HasSideEffects; }
   bool isAlignStack() const { return IsAlignStack; }
@@ -81,15 +83,15 @@ class InlineAsm final : public Value {
 
   /// getFunctionType - InlineAsm's are always pointers to functions.
   ///
-  FunctionType *getFunctionType() const;
+  LLVM_ABI FunctionType *getFunctionType() const;
 
   StringRef getAsmString() const { return AsmString; }
   StringRef getConstraintString() const { return Constraints; }
-  void collectAsmStrs(SmallVectorImpl<StringRef> &AsmStrs) const;
+  LLVM_ABI void collectAsmStrs(SmallVectorImpl<StringRef> &AsmStrs) const;
 
   /// This static method can be used by the parser to check to see if the
   /// specified constraint string is legal for the type.
-  static Error verify(FunctionType *Ty, StringRef Constraints);
+  LLVM_ABI static Error verify(FunctionType *Ty, StringRef Constraints);
 
   // Constraint String Parsing
   enum ConstraintPrefix {
@@ -169,11 +171,11 @@ class InlineAsm final : public Value {
     /// Parse - Analyze the specified string (e.g. "=*&{eax}") and fill in the
     /// fields in this structure.  If the constraint string is not understood,
     /// return true, otherwise return false.
-    bool Parse(StringRef Str, ConstraintInfoVector &ConstraintsSoFar);
+    LLVM_ABI bool Parse(StringRef Str, ConstraintInfoVector &ConstraintsSoFar);
 
     /// selectAlternative - Point this constraint to the alternative constraint
     /// indicated by the index.
-    void selectAlternative(unsigned index);
+    LLVM_ABI void selectAlternative(unsigned index);
 
     /// Whether this constraint corresponds to an argument.
     bool hasArg() const {
@@ -184,7 +186,8 @@ class InlineAsm final : public Value {
   /// ParseConstraints - Split up the constraint string into the specific
   /// constraints and their prefixes.  If this returns an empty vector, and if
   /// the constraint string itself isn't empty, there was an error parsing.
-  static ConstraintInfoVector ParseConstraints(StringRef ConstraintString);
+  LLVM_ABI static ConstraintInfoVector
+  ParseConstraints(StringRef ConstraintString);
 
   /// ParseConstraints - Parse the constraints of this inlineasm object,
   /// returning them the same way that ParseConstraints(str) does.
diff --git a/external/llvm-project/llvm/include/llvm/IR/InstrTypes.h b/external/llvm-project/llvm/include/llvm/IR/InstrTypes.h
index 8e47e3c7b3a7..14685abef01e 100644
--- a/external/llvm-project/llvm/include/llvm/IR/InstrTypes.h
+++ b/external/llvm-project/llvm/include/llvm/IR/InstrTypes.h
@@ -30,6 +30,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/OperandTraits.h"
 #include "llvm/IR/User.h"
+#include "llvm/Support/Compiler.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
@@ -101,13 +102,13 @@ class UnaryOperator : public UnaryInstruction {
   void AssertOK();
 
 protected:
-  UnaryOperator(UnaryOps iType, Value *S, Type *Ty, const Twine &Name,
-                InsertPosition InsertBefore);
+  LLVM_ABI UnaryOperator(UnaryOps iType, Value *S, Type *Ty, const Twine &Name,
+                         InsertPosition InsertBefore);
 
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  UnaryOperator *cloneImpl() const;
+  LLVM_ABI UnaryOperator *cloneImpl() const;
 
 public:
   /// Construct a unary instruction, given the opcode and an operand.
@@ -115,9 +116,9 @@ class UnaryOperator : public UnaryInstruction {
   /// into a BasicBlock right before the specified instruction.  The specified
   /// Instruction is allowed to be a dereferenced end iterator.
   ///
-  static UnaryOperator *Create(UnaryOps Op, Value *S,
-                               const Twine &Name = Twine(),
-                               InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static UnaryOperator *Create(UnaryOps Op, Value *S,
+                                        const Twine &Name = Twine(),
+                                        InsertPosition InsertBefore = nullptr);
 
   /// These methods just forward to Create, and are useful when you
   /// statically know what type of instruction you're going to create.  These
@@ -173,13 +174,13 @@ class BinaryOperator : public Instruction {
   void AssertOK();
 
 protected:
-  BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
-                 const Twine &Name, InsertPosition InsertBefore);
+  LLVM_ABI BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,
+                          const Twine &Name, InsertPosition InsertBefore);
 
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  BinaryOperator *cloneImpl() const;
+  LLVM_ABI BinaryOperator *cloneImpl() const;
 
 public:
   // allocate space for exactly two operands
@@ -194,9 +195,9 @@ class BinaryOperator : public Instruction {
   /// into a BasicBlock right before the specified instruction.  The specified
   /// Instruction is allowed to be a dereferenced end iterator.
   ///
-  static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
-                                const Twine &Name = Twine(),
-                                InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static BinaryOperator *Create(BinaryOps Op, Value *S1, Value *S2,
+                                         const Twine &Name = Twine(),
+                                         InsertPosition InsertBefore = nullptr);
 
   /// These methods just forward to Create, and are useful when you
   /// statically know what type of instruction you're going to create.  These
@@ -360,12 +361,15 @@ class BinaryOperator : public Instruction {
   ///
   /// Create the NEG and NOT instructions out of SUB and XOR instructions.
   ///
-  static BinaryOperator *CreateNeg(Value *Op, const Twine &Name = "",
-                                   InsertPosition InsertBefore = nullptr);
-  static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name = "",
-                                      InsertPosition InsertBefore = nullptr);
-  static BinaryOperator *CreateNot(Value *Op, const Twine &Name = "",
-                                   InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static BinaryOperator *
+  CreateNeg(Value *Op, const Twine &Name = "",
+            InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static BinaryOperator *
+  CreateNSWNeg(Value *Op, const Twine &Name = "",
+               InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static BinaryOperator *
+  CreateNot(Value *Op, const Twine &Name = "",
+            InsertPosition InsertBefore = nullptr);
 
   BinaryOps getOpcode() const {
     return static_cast<BinaryOps>(Instruction::getOpcode());
@@ -376,7 +380,7 @@ class BinaryOperator : public Instruction {
   /// does not modify the semantics of the instruction.  If the instruction
   /// cannot be reversed (ie, it's a Div), then return true.
   ///
-  bool swapOperands();
+  LLVM_ABI bool swapOperands();
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
@@ -457,7 +461,7 @@ class CastInst : public UnaryInstruction {
   /// constructor has insert-before-instruction semantics to automatically
   /// insert the new CastInst before InsertBefore (if it is non-null).
   /// Construct any of the CastInst subclasses
-  static CastInst *Create(
+  LLVM_ABI static CastInst *Create(
       Instruction::CastOps,   ///< The opcode of the cast instruction
       Value *S,               ///< The value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
@@ -466,7 +470,7 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Create a ZExt or BitCast cast instruction
-  static CastInst *CreateZExtOrBitCast(
+  LLVM_ABI static CastInst *CreateZExtOrBitCast(
       Value *S,               ///< The value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
       const Twine &Name = "", ///< Name for the instruction
@@ -474,7 +478,7 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Create a SExt or BitCast cast instruction
-  static CastInst *CreateSExtOrBitCast(
+  LLVM_ABI static CastInst *CreateSExtOrBitCast(
       Value *S,               ///< The value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
       const Twine &Name = "", ///< Name for the instruction
@@ -482,7 +486,7 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Create a BitCast, AddrSpaceCast or a PtrToInt cast instruction.
-  static CastInst *CreatePointerCast(
+  LLVM_ABI static CastInst *CreatePointerCast(
       Value *S,               ///< The pointer value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
       const Twine &Name = "", ///< Name for the instruction
@@ -490,7 +494,7 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Create a BitCast or an AddrSpaceCast cast instruction.
-  static CastInst *CreatePointerBitCastOrAddrSpaceCast(
+  LLVM_ABI static CastInst *CreatePointerBitCastOrAddrSpaceCast(
       Value *S,               ///< The pointer value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
       const Twine &Name = "", ///< Name for the instruction
@@ -503,7 +507,7 @@ class CastInst : public UnaryInstruction {
   /// creates a PtrToInt cast. If the value is an integer type and the
   /// destination a pointer type, creates an IntToPtr cast. Otherwise, creates
   /// a bitcast.
-  static CastInst *CreateBitOrPointerCast(
+  LLVM_ABI static CastInst *CreateBitOrPointerCast(
       Value *S,               ///< The pointer value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
       const Twine &Name = "", ///< Name for the instruction
@@ -511,7 +515,7 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Create a ZExt, BitCast, or Trunc for int -> int casts.
-  static CastInst *CreateIntegerCast(
+  LLVM_ABI static CastInst *CreateIntegerCast(
       Value *S,               ///< The pointer value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
       bool isSigned,          ///< Whether to regard S as signed or not
@@ -520,7 +524,7 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Create an FPExt, BitCast, or FPTrunc for fp -> fp casts
-  static CastInst *CreateFPCast(
+  LLVM_ABI static CastInst *CreateFPCast(
       Value *S,               ///< The floating point value to be casted
       Type *Ty,               ///< The floating point type to cast to
       const Twine &Name = "", ///< Name for the instruction
@@ -528,7 +532,7 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Create a Trunc or BitCast cast instruction
-  static CastInst *CreateTruncOrBitCast(
+  LLVM_ABI static CastInst *CreateTruncOrBitCast(
       Value *S,               ///< The value to be casted (operand 0)
       Type *Ty,               ///< The type to which cast should be made
       const Twine &Name = "", ///< Name for the instruction
@@ -536,9 +540,9 @@ class CastInst : public UnaryInstruction {
   );
 
   /// Check whether a bitcast between these types is valid
-  static bool isBitCastable(
-    Type *SrcTy, ///< The Type from which the value should be cast.
-    Type *DestTy ///< The Type to which the value should be cast.
+  LLVM_ABI static bool
+  isBitCastable(Type *SrcTy, ///< The Type from which the value should be cast.
+                Type *DestTy ///< The Type to which the value should be cast.
   );
 
   /// Check whether a bitcast, inttoptr, or ptrtoint cast between these
@@ -546,7 +550,7 @@ class CastInst : public UnaryInstruction {
   ///
   /// This ensures that any pointer<->integer cast has enough bits in the
   /// integer and any other cast is a bitcast.
-  static bool isBitOrNoopPointerCastable(
+  LLVM_ABI static bool isBitOrNoopPointerCastable(
       Type *SrcTy,  ///< The Type from which the value should be cast.
       Type *DestTy, ///< The Type to which the value should be cast.
       const DataLayout &DL);
@@ -554,11 +558,11 @@ class CastInst : public UnaryInstruction {
   /// Returns the opcode necessary to cast Val into Ty using usual casting
   /// rules.
   /// Infer the opcode for cast operand and type
-  static Instruction::CastOps getCastOpcode(
-    const Value *Val, ///< The value to cast
-    bool SrcIsSigned, ///< Whether to treat the source as signed
-    Type *Ty,   ///< The Type to which the value should be casted
-    bool DstIsSigned  ///< Whether to treate the dest. as signed
+  LLVM_ABI static Instruction::CastOps
+  getCastOpcode(const Value *Val, ///< The value to cast
+                bool SrcIsSigned, ///< Whether to treat the source as signed
+                Type *Ty, ///< The Type to which the value should be casted
+                bool DstIsSigned ///< Whether to treate the dest. as signed
   );
 
   /// There are several places where we need to know if a cast instruction
@@ -566,7 +570,7 @@ class CastInst : public UnaryInstruction {
   /// logic, this method is provided.
   /// @returns true iff the cast has only integral typed operand and dest type.
   /// Determine if this is an integer-only cast.
-  bool isIntegerCast() const;
+  LLVM_ABI bool isIntegerCast() const;
 
   /// A no-op cast is one that can be effected without changing any bits.
   /// It implies that the source and destination types are the same size. The
@@ -575,17 +579,17 @@ class CastInst : public UnaryInstruction {
   /// is the same size as the pointer. However, pointer size varies with
   /// platform.  Note that a precondition of this method is that the cast is
   /// legal - i.e. the instruction formed with these operands would verify.
-  static bool isNoopCast(
-    Instruction::CastOps Opcode, ///< Opcode of cast
-    Type *SrcTy,         ///< SrcTy of cast
-    Type *DstTy,         ///< DstTy of cast
-    const DataLayout &DL ///< DataLayout to get the Int Ptr type from.
+  LLVM_ABI static bool
+  isNoopCast(Instruction::CastOps Opcode, ///< Opcode of cast
+             Type *SrcTy,                 ///< SrcTy of cast
+             Type *DstTy,                 ///< DstTy of cast
+             const DataLayout &DL ///< DataLayout to get the Int Ptr type from.
   );
 
   /// Determine if this cast is a no-op cast.
   ///
   /// \param DL is the DataLayout to determine pointer size.
-  bool isNoopCast(const DataLayout &DL) const;
+  LLVM_ABI bool isNoopCast(const DataLayout &DL) const;
 
   /// Determine how a pair of casts can be eliminated, if they can be at all.
   /// This is a helper function for both CastInst and ConstantExpr.
@@ -593,15 +597,15 @@ class CastInst : public UnaryInstruction {
   /// returns Instruction::CastOps value for a cast that can replace
   /// the pair, casting SrcTy to DstTy.
   /// Determine if a cast pair is eliminable
-  static unsigned isEliminableCastPair(
-    Instruction::CastOps firstOpcode,  ///< Opcode of first cast
-    Instruction::CastOps secondOpcode, ///< Opcode of second cast
-    Type *SrcTy, ///< SrcTy of 1st cast
-    Type *MidTy, ///< DstTy of 1st cast & SrcTy of 2nd cast
-    Type *DstTy, ///< DstTy of 2nd cast
-    Type *SrcIntPtrTy, ///< Integer type corresponding to Ptr SrcTy, or null
-    Type *MidIntPtrTy, ///< Integer type corresponding to Ptr MidTy, or null
-    Type *DstIntPtrTy  ///< Integer type corresponding to Ptr DstTy, or null
+  LLVM_ABI static unsigned isEliminableCastPair(
+      Instruction::CastOps firstOpcode,  ///< Opcode of first cast
+      Instruction::CastOps secondOpcode, ///< Opcode of second cast
+      Type *SrcTy,                       ///< SrcTy of 1st cast
+      Type *MidTy,       ///< DstTy of 1st cast & SrcTy of 2nd cast
+      Type *DstTy,       ///< DstTy of 2nd cast
+      Type *SrcIntPtrTy, ///< Integer type corresponding to Ptr SrcTy, or null
+      Type *MidIntPtrTy, ///< Integer type corresponding to Ptr MidTy, or null
+      Type *DstIntPtrTy  ///< Integer type corresponding to Ptr DstTy, or null
   );
 
   /// Return the opcode of this CastInst
@@ -618,7 +622,8 @@ class CastInst : public UnaryInstruction {
   /// Opcode op is valid or not.
   /// @returns true iff the proposed cast is valid.
   /// Determine if a cast is valid without creating one.
-  static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy);
+  LLVM_ABI static bool castIsValid(Instruction::CastOps op, Type *SrcTy,
+                                   Type *DstTy);
   static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
     return castIsValid(op, S->getType(), DstTy);
   }
@@ -723,10 +728,10 @@ class CmpInst : public Instruction {
   }
 
 protected:
-  CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred, Value *LHS,
-          Value *RHS, const Twine &Name = "",
-          InsertPosition InsertBefore = nullptr,
-          Instruction *FlagsSource = nullptr);
+  LLVM_ABI CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred,
+                   Value *LHS, Value *RHS, const Twine &Name = "",
+                   InsertPosition InsertBefore = nullptr,
+                   Instruction *FlagsSource = nullptr);
 
 public:
   // allocate space for exactly two operands
@@ -738,9 +743,9 @@ class CmpInst : public Instruction {
   /// instruction into a BasicBlock right before the specified instruction.
   /// The specified Instruction is allowed to be a dereferenced end iterator.
   /// Create a CmpInst
-  static CmpInst *Create(OtherOps Op, Predicate Pred, Value *S1, Value *S2,
-                         const Twine &Name = "",
-                         InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static CmpInst *Create(OtherOps Op, Predicate Pred, Value *S1,
+                                  Value *S2, const Twine &Name = "",
+                                  InsertPosition InsertBefore = nullptr);
 
   /// Construct a compare instruction, given the opcode, the predicate,
   /// the two operands and the instruction to copy the flags from. Optionally
@@ -748,11 +753,10 @@ class CmpInst : public Instruction {
   /// right before the specified instruction. The specified Instruction is
   /// allowed to be a dereferenced end iterator.
   /// Create a CmpInst
-  static CmpInst *CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1,
-                                        Value *S2,
-                                        const Instruction *FlagsSource,
-                                        const Twine &Name = "",
-                                        InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static CmpInst *
+  CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, Value *S2,
+                        const Instruction *FlagsSource, const Twine &Name = "",
+                        InsertPosition InsertBefore = nullptr);
 
   /// Get the opcode casted to the right type
   OtherOps getOpcode() const {
@@ -775,7 +779,7 @@ class CmpInst : public Instruction {
     return P >= FIRST_ICMP_PREDICATE && P <= LAST_ICMP_PREDICATE;
   }
 
-  static StringRef getPredicateName(Predicate P);
+  LLVM_ABI static StringRef getPredicateName(Predicate P);
 
   bool isFPPredicate() const { return isFPPredicate(getPredicate()); }
   bool isIntPredicate() const { return isIntPredicate(getPredicate()); }
@@ -814,7 +818,7 @@ class CmpInst : public Instruction {
   ///              OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
   /// @returns the inverse predicate for predicate provided in \p pred.
   /// Return the inverse of a given predicate
-  static Predicate getInversePredicate(Predicate pred);
+  LLVM_ABI static Predicate getInversePredicate(Predicate pred);
 
   /// For example, EQ->EQ, SLE->SGE, ULT->UGT,
   ///              OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
@@ -829,12 +833,12 @@ class CmpInst : public Instruction {
   /// This is a static version that you can use without an instruction
   /// available.
   /// Return the predicate as if the operands were swapped.
-  static Predicate getSwappedPredicate(Predicate pred);
+  LLVM_ABI static Predicate getSwappedPredicate(Predicate pred);
 
   /// This is a static version that you can use without an instruction
   /// available.
   /// @returns true if the comparison predicate is strict, false otherwise.
-  static bool isStrictPredicate(Predicate predicate);
+  LLVM_ABI static bool isStrictPredicate(Predicate predicate);
 
   /// @returns true if the comparison predicate is strict, false otherwise.
   /// Determine if this instruction is using an strict comparison predicate.
@@ -843,7 +847,7 @@ class CmpInst : public Instruction {
   /// This is a static version that you can use without an instruction
   /// available.
   /// @returns true if the comparison predicate is non-strict, false otherwise.
-  static bool isNonStrictPredicate(Predicate predicate);
+  LLVM_ABI static bool isNonStrictPredicate(Predicate predicate);
 
   /// @returns true if the comparison predicate is non-strict, false otherwise.
   /// Determine if this instruction is using an non-strict comparison predicate.
@@ -862,7 +866,7 @@ class CmpInst : public Instruction {
   /// @returns the strict version of comparison provided in \p pred.
   /// If \p pred is not a strict comparison predicate, returns \p pred.
   /// Returns the strict version of non-strict comparisons.
-  static Predicate getStrictPredicate(Predicate pred);
+  LLVM_ABI static Predicate getStrictPredicate(Predicate pred);
 
   /// For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
   /// Returns the non-strict version of strict comparisons.
@@ -875,12 +879,12 @@ class CmpInst : public Instruction {
   /// @returns the non-strict version of comparison provided in \p pred.
   /// If \p pred is not a strict comparison predicate, returns \p pred.
   /// Returns the non-strict version of strict comparisons.
-  static Predicate getNonStrictPredicate(Predicate pred);
+  LLVM_ABI static Predicate getNonStrictPredicate(Predicate pred);
 
   /// This is a static version that you can use without an instruction
   /// available.
   /// Return the flipped strictness of predicate
-  static Predicate getFlippedStrictnessPredicate(Predicate pred);
+  LLVM_ABI static Predicate getFlippedStrictnessPredicate(Predicate pred);
 
   /// For predicate of kind "is X or equal to 0" returns the predicate "is X".
   /// For predicate of kind "is X" returns the predicate "is X or equal to 0".
@@ -898,16 +902,16 @@ class CmpInst : public Instruction {
   /// This is just a convenience that dispatches to the subclasses.
   /// Swap the operands and adjust predicate accordingly to retain
   /// the same comparison.
-  void swapOperands();
+  LLVM_ABI void swapOperands();
 
   /// This is just a convenience that dispatches to the subclasses.
   /// Determine if this CmpInst is commutative.
-  bool isCommutative() const;
+  LLVM_ABI bool isCommutative() const;
 
   /// Determine if this is an equals/not equals predicate.
   /// This is a static version that you can use without an instruction
   /// available.
-  static bool isEquality(Predicate pred);
+  LLVM_ABI static bool isEquality(Predicate pred);
 
   /// Determine if this is an equals/not equals predicate.
   bool isEquality() const { return isEquality(getPredicate()); }
@@ -915,7 +919,7 @@ class CmpInst : public Instruction {
   /// Determine if one operand of this compare can always be replaced by the
   /// other operand, ignoring provenance considerations. If \p Invert, check for
   /// equivalence with the inverse predicate.
-  bool isEquivalence(bool Invert = false) const;
+  LLVM_ABI bool isEquivalence(bool Invert = false) const;
 
   /// Return true if the predicate is relational (not EQ or NE).
   static bool isRelational(Predicate P) { return !isEquality(P); }
@@ -949,23 +953,23 @@ class CmpInst : public Instruction {
 
   /// @returns true if the predicate is unsigned, false otherwise.
   /// Determine if the predicate is an unsigned operation.
-  static bool isUnsigned(Predicate predicate);
+  LLVM_ABI static bool isUnsigned(Predicate predicate);
 
   /// @returns true if the predicate is signed, false otherwise.
   /// Determine if the predicate is an signed operation.
-  static bool isSigned(Predicate predicate);
+  LLVM_ABI static bool isSigned(Predicate predicate);
 
   /// Determine if the predicate is an ordered operation.
-  static bool isOrdered(Predicate predicate);
+  LLVM_ABI static bool isOrdered(Predicate predicate);
 
   /// Determine if the predicate is an unordered operation.
-  static bool isUnordered(Predicate predicate);
+  LLVM_ABI static bool isUnordered(Predicate predicate);
 
   /// Determine if the predicate is true when comparing a value with itself.
-  static bool isTrueWhenEqual(Predicate predicate);
+  LLVM_ABI static bool isTrueWhenEqual(Predicate predicate);
 
   /// Determine if the predicate is false when comparing a value with itself.
-  static bool isFalseWhenEqual(Predicate predicate);
+  LLVM_ABI static bool isFalseWhenEqual(Predicate predicate);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
@@ -1000,7 +1004,7 @@ struct OperandTraits<CmpInst> : public FixedNumOperandTraits<CmpInst, 2> {
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CmpInst, Value)
 
-raw_ostream &operator<<(raw_ostream &OS, CmpInst::Predicate Pred);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, CmpInst::Predicate Pred);
 
 /// A lightweight accessor for an operand bundle meant to be passed
 /// around by value.
@@ -1148,7 +1152,7 @@ class CallBase : public Instruction {
 
   /// Get the number of extra operands for instructions that don't have a fixed
   /// number of extra operands.
-  unsigned getNumSubclassExtraOperandsDynamic() const;
+  LLVM_ABI unsigned getNumSubclassExtraOperandsDynamic() const;
 
 public:
   using Instruction::getContext;
@@ -1159,25 +1163,27 @@ class CallBase : public Instruction {
   /// The returned call instruction is identical \p CB in every way except that
   /// the operand bundles for the new instruction are set to the operand bundles
   /// in \p Bundles.
-  static CallBase *Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,
-                          InsertPosition InsertPt = nullptr);
+  LLVM_ABI static CallBase *Create(CallBase *CB,
+                                   ArrayRef<OperandBundleDef> Bundles,
+                                   InsertPosition InsertPt = nullptr);
 
   /// Create a clone of \p CB with the operand bundle with the tag matching
   /// \p Bundle's tag replaced with Bundle, and insert it before \p InsertPt.
   ///
   /// The returned call instruction is identical \p CI in every way except that
   /// the specified operand bundle has been replaced.
-  static CallBase *Create(CallBase *CB, OperandBundleDef Bundle,
-                          InsertPosition InsertPt = nullptr);
+  LLVM_ABI static CallBase *Create(CallBase *CB, OperandBundleDef Bundle,
+                                   InsertPosition InsertPt = nullptr);
 
   /// Create a clone of \p CB with operand bundle \p OB added.
-  static CallBase *addOperandBundle(CallBase *CB, uint32_t ID,
-                                    OperandBundleDef OB,
-                                    InsertPosition InsertPt = nullptr);
+  LLVM_ABI static CallBase *addOperandBundle(CallBase *CB, uint32_t ID,
+                                             OperandBundleDef OB,
+                                             InsertPosition InsertPt = nullptr);
 
   /// Create a clone of \p CB with operand bundle \p ID removed.
-  static CallBase *removeOperandBundle(CallBase *CB, uint32_t ID,
-                                       InsertPosition InsertPt = nullptr);
+  LLVM_ABI static CallBase *
+  removeOperandBundle(CallBase *CB, uint32_t ID,
+                      InsertPosition InsertPt = nullptr);
 
   /// Return the convergence control token for this call, if it exists.
   Value *getConvergenceControlToken() const {
@@ -1337,7 +1343,8 @@ class CallBase : public Instruction {
   Use &getCalledOperandUse() { return Op<CalledOperandOpEndIdx>(); }
 
   /// Returns the function called, or null if this is an indirect function
-  /// invocation or the function signature does not match the call signature.
+  /// invocation or the function signature does not match the call signature, or
+  /// the call target is an alias.
   Function *getCalledFunction() const {
     if (auto *F = dyn_cast_or_null<Function>(getCalledOperand()))
       if (F->getValueType() == getFunctionType())
@@ -1346,7 +1353,7 @@ class CallBase : public Instruction {
   }
 
   /// Return true if the callsite is an indirect call.
-  bool isIndirectCall() const;
+  LLVM_ABI bool isIndirectCall() const;
 
   /// Determine whether the passed iterator points to the callee operand's Use.
   bool isCallee(Value::const_user_iterator UI) const {
@@ -1357,22 +1364,22 @@ class CallBase : public Instruction {
   bool isCallee(const Use *U) const { return &getCalledOperandUse() == U; }
 
   /// Helper to get the caller (the parent function).
-  Function *getCaller();
+  LLVM_ABI Function *getCaller();
   const Function *getCaller() const {
     return const_cast<CallBase *>(this)->getCaller();
   }
 
   /// Tests if this call site must be tail call optimized. Only a CallInst can
   /// be tail call optimized.
-  bool isMustTailCall() const;
+  LLVM_ABI bool isMustTailCall() const;
 
   /// Tests if this call site is marked as a tail call.
-  bool isTailCall() const;
+  LLVM_ABI bool isTailCall() const;
 
   /// Returns the intrinsic ID of the intrinsic called or
   /// Intrinsic::not_intrinsic if the called function is not an intrinsic, or if
   /// this is an indirect call.
-  Intrinsic::ID getIntrinsicID() const;
+  LLVM_ABI Intrinsic::ID getIntrinsicID() const;
 
   void setCalledOperand(Value *V) { Op<CalledOperandOpEndIdx>() = V; }
 
@@ -1600,7 +1607,7 @@ class CallBase : public Instruction {
   }
 
   /// Determine whether the argument or parameter has the given attribute.
-  bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;
+  LLVM_ABI bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;
 
   /// Return true if this argument has the nonnull attribute on either the
   /// CallBase instruction or the called function. Also returns true if at least
@@ -1608,7 +1615,8 @@ class CallBase : public Instruction {
   /// addrspace(0). If \p AllowUndefOrPoison is true, respect the semantics of
   /// nonnull attribute and return true even if the argument can be undef or
   /// poison.
-  bool paramHasNonNullAttr(unsigned ArgNo, bool AllowUndefOrPoison) const;
+  LLVM_ABI bool paramHasNonNullAttr(unsigned ArgNo,
+                                    bool AllowUndefOrPoison) const;
 
   /// Get the attribute of a given kind at a position.
   Attribute getAttributeAtIndex(unsigned i, Attribute::AttrKind Kind) const {
@@ -1683,7 +1691,7 @@ class CallBase : public Instruction {
   }
 
   /// Return which pointer components this operand may capture.
-  CaptureInfo getCaptureInfo(unsigned OpNo) const;
+  LLVM_ABI CaptureInfo getCaptureInfo(unsigned OpNo) const;
 
   /// Determine whether this data operand is not captured.
   // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
@@ -1695,7 +1703,7 @@ class CallBase : public Instruction {
   /// Returns whether the call has an argument that has an attribute like
   /// captures(ret: address, provenance), where the return capture components
   /// are not a subset of the other capture components.
-  bool hasArgumentWithAdditionalReturnCaptureComponents() const;
+  LLVM_ABI bool hasArgumentWithAdditionalReturnCaptureComponents() const;
 
   /// Determine whether this argument is passed by value.
   bool isByValArgument(unsigned ArgNo) const {
@@ -1862,20 +1870,20 @@ class CallBase : public Instruction {
 
   /// Extract a test mask for disallowed floating-point value classes for the
   /// return value.
-  FPClassTest getRetNoFPClass() const;
+  LLVM_ABI FPClassTest getRetNoFPClass() const;
 
   /// Extract a test mask for disallowed floating-point value classes for the
   /// parameter.
-  FPClassTest getParamNoFPClass(unsigned i) const;
+  LLVM_ABI FPClassTest getParamNoFPClass(unsigned i) const;
 
   /// If this return value has a range attribute, return the value range of the
   /// argument. Otherwise, std::nullopt is returned.
-  std::optional<ConstantRange> getRange() const;
+  LLVM_ABI std::optional<ConstantRange> getRange() const;
 
   /// Return true if the return value is known to be not null.
   /// This may be because it has the nonnull attribute, or because at least
   /// one byte is dereferenceable and the pointer is in addrspace(0).
-  bool isReturnNonNull() const;
+  LLVM_ABI bool isReturnNonNull() const;
 
   /// Determine if the return value is marked with NoAlias attribute.
   bool returnDoesNotAlias() const {
@@ -1890,7 +1898,7 @@ class CallBase : public Instruction {
 
   /// If one of the arguments has the specified attribute, returns its
   /// operand value. Otherwise, return nullptr.
-  Value *getArgOperandWithAttribute(Attribute::AttrKind Kind) const;
+  LLVM_ABI Value *getArgOperandWithAttribute(Attribute::AttrKind Kind) const;
 
   /// Return true if the call should not be treated as a call to a
   /// builtin.
@@ -1906,35 +1914,35 @@ class CallBase : public Instruction {
   bool isNoInline() const { return hasFnAttr(Attribute::NoInline); }
   void setIsNoInline() { addFnAttr(Attribute::NoInline); }
 
-  MemoryEffects getMemoryEffects() const;
-  void setMemoryEffects(MemoryEffects ME);
+  LLVM_ABI MemoryEffects getMemoryEffects() const;
+  LLVM_ABI void setMemoryEffects(MemoryEffects ME);
 
   /// Determine if the call does not access memory.
-  bool doesNotAccessMemory() const;
-  void setDoesNotAccessMemory();
+  LLVM_ABI bool doesNotAccessMemory() const;
+  LLVM_ABI void setDoesNotAccessMemory();
 
   /// Determine if the call does not access or only reads memory.
-  bool onlyReadsMemory() const;
-  void setOnlyReadsMemory();
+  LLVM_ABI bool onlyReadsMemory() const;
+  LLVM_ABI void setOnlyReadsMemory();
 
   /// Determine if the call does not access or only writes memory.
-  bool onlyWritesMemory() const;
-  void setOnlyWritesMemory();
+  LLVM_ABI bool onlyWritesMemory() const;
+  LLVM_ABI void setOnlyWritesMemory();
 
   /// Determine if the call can access memmory only using pointers based
   /// on its arguments.
-  bool onlyAccessesArgMemory() const;
-  void setOnlyAccessesArgMemory();
+  LLVM_ABI bool onlyAccessesArgMemory() const;
+  LLVM_ABI void setOnlyAccessesArgMemory();
 
   /// Determine if the function may only access memory that is
   /// inaccessible from the IR.
-  bool onlyAccessesInaccessibleMemory() const;
-  void setOnlyAccessesInaccessibleMemory();
+  LLVM_ABI bool onlyAccessesInaccessibleMemory() const;
+  LLVM_ABI void setOnlyAccessesInaccessibleMemory();
 
   /// Determine if the function may only access memory that is
   /// either inaccessible from the IR or pointed to by its arguments.
-  bool onlyAccessesInaccessibleMemOrArgMem() const;
-  void setOnlyAccessesInaccessibleMemOrArgMem();
+  LLVM_ABI bool onlyAccessesInaccessibleMemOrArgMem() const;
+  LLVM_ABI void setOnlyAccessesInaccessibleMemOrArgMem();
 
   /// Determine if the call cannot return.
   bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
@@ -2107,7 +2115,8 @@ class CallBase : public Instruction {
   /// OperandBundleUser to a vector of OperandBundleDefs.  Note:
   /// OperandBundeUses and OperandBundleDefs are non-trivially *different*
   /// representations of operand bundles (see documentation above).
-  void getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const;
+  LLVM_ABI void
+  getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const;
 
   /// Return the operand bundle for the operand at index OpIdx.
   ///
@@ -2119,11 +2128,11 @@ class CallBase : public Instruction {
 
   /// Return true if this operand bundle user has operand bundles that
   /// may read from the heap.
-  bool hasReadingOperandBundles() const;
+  LLVM_ABI bool hasReadingOperandBundles() const;
 
   /// Return true if this operand bundle user has operand bundles that
   /// may write to the heap.
-  bool hasClobberingOperandBundles() const;
+  LLVM_ABI bool hasClobberingOperandBundles() const;
 
   /// Return true if the bundle operand at index \p OpIdx has the
   /// attribute \p A.
@@ -2281,8 +2290,8 @@ class CallBase : public Instruction {
   ///
   /// Each \p OperandBundleDef instance is tracked by a OperandBundleInfo
   /// instance allocated in this User's descriptor.
-  op_iterator populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles,
-                                         const unsigned BeginIndex);
+  LLVM_ABI op_iterator populateBundleOperandInfos(
+      ArrayRef<OperandBundleDef> Bundles, const unsigned BeginIndex);
 
   /// Return true if the call has deopt state bundle.
   bool hasDeoptState() const {
@@ -2294,7 +2303,7 @@ class CallBase : public Instruction {
   ///
   /// It is an error to call this with an OpIdx that does not correspond to an
   /// bundle operand.
-  BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx);
+  LLVM_ABI BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx);
   const BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx) const {
     return const_cast<CallBase *>(this)->getBundleOpInfoForOperand(OpIdx);
   }
@@ -2312,8 +2321,8 @@ class CallBase : public Instruction {
   // End of operand bundle API.
 
 private:
-  bool hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
-  bool hasFnAttrOnCalledFunction(StringRef Kind) const;
+  LLVM_ABI bool hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
+  LLVM_ABI bool hasFnAttrOnCalledFunction(StringRef Kind) const;
 
   template <typename AttrKind> bool hasFnAttrImpl(AttrKind Kind) const {
     if (Attrs.hasFnAttr(Kind))
@@ -2350,9 +2359,10 @@ class FuncletPadInst : public Instruction {
 private:
   FuncletPadInst(const FuncletPadInst &CPI, AllocInfo AllocInfo);
 
-  explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
-                          ArrayRef<Value *> Args, AllocInfo AllocInfo,
-                          const Twine &NameStr, InsertPosition InsertBefore);
+  LLVM_ABI explicit FuncletPadInst(Instruction::FuncletPadOps Op,
+                                   Value *ParentPad, ArrayRef<Value *> Args,
+                                   AllocInfo AllocInfo, const Twine &NameStr,
+                                   InsertPosition InsertBefore);
 
   void init(Value *ParentPad, ArrayRef<Value *> Args, const Twine &NameStr);
 
@@ -2362,7 +2372,7 @@ class FuncletPadInst : public Instruction {
   friend class CatchPadInst;
   friend class CleanupPadInst;
 
-  FuncletPadInst *cloneImpl() const;
+  LLVM_ABI FuncletPadInst *cloneImpl() const;
 
 public:
   /// Provide fast operand accessors
diff --git a/external/llvm-project/llvm/include/llvm/IR/Instruction.h b/external/llvm-project/llvm/include/llvm/IR/Instruction.h
index d8069b2fb02a..8e1ef2422678 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Instruction.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Instruction.h
@@ -23,6 +23,7 @@
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <utility>
 
@@ -42,7 +43,7 @@ template <> struct ilist_alloc_traits<Instruction> {
   static inline void deleteNode(Instruction *V);
 };
 
-iterator_range<simple_ilist<DbgRecord>::iterator>
+LLVM_ABI iterator_range<simple_ilist<DbgRecord>::iterator>
 getDbgRecordRange(DbgMarker *);
 
 class InsertPosition {
@@ -52,10 +53,10 @@ class InsertPosition {
 
 public:
   InsertPosition(std::nullptr_t) : InsertAt() {}
-  LLVM_DEPRECATED("Use BasicBlock::iterators for insertion instead",
-                  "BasicBlock::iterator")
-  InsertPosition(Instruction *InsertBefore);
-  InsertPosition(BasicBlock *InsertAtEnd);
+  LLVM_ABI LLVM_DEPRECATED("Use BasicBlock::iterators for insertion instead",
+                           "BasicBlock::iterator")
+      InsertPosition(Instruction *InsertBefore);
+  LLVM_ABI InsertPosition(BasicBlock *InsertAtEnd);
   InsertPosition(InstListType::iterator InsertAt) : InsertAt(InsertAt) {}
   operator InstListType::iterator() const { return InsertAt; }
   bool isValid() const { return InsertAt.isValid(); }
@@ -95,7 +96,7 @@ class Instruction : public User,
   /// \p InsertAtHead Whether the cloned DbgRecords should be placed at the end
   ///    or the beginning of existing DbgRecords attached to this.
   /// \returns A range over the newly cloned DbgRecords.
-  iterator_range<simple_ilist<DbgRecord>::iterator> cloneDebugInfoFrom(
+  LLVM_ABI iterator_range<simple_ilist<DbgRecord>::iterator> cloneDebugInfoFrom(
       const Instruction *From,
       std::optional<simple_ilist<DbgRecord>::iterator> FromHere = std::nullopt,
       bool InsertAtHead = false);
@@ -108,26 +109,27 @@ class Instruction : public User,
   /// Return an iterator to the position of the "Next" DbgRecord after this
   /// instruction, or std::nullopt. This is the position to pass to
   /// BasicBlock::reinsertInstInDbgRecords when re-inserting an instruction.
-  std::optional<simple_ilist<DbgRecord>::iterator> getDbgReinsertionPosition();
+  LLVM_ABI std::optional<simple_ilist<DbgRecord>::iterator>
+  getDbgReinsertionPosition();
 
   /// Returns true if any DbgRecords are attached to this instruction.
-  bool hasDbgRecords() const;
+  LLVM_ABI bool hasDbgRecords() const;
 
   /// Transfer any DbgRecords on the position \p It onto this instruction,
   /// by simply adopting the sequence of DbgRecords (which is efficient) if
   /// possible, by merging two sequences otherwise.
-  void adoptDbgRecords(BasicBlock *BB, InstListType::iterator It,
-                       bool InsertAtHead);
+  LLVM_ABI void adoptDbgRecords(BasicBlock *BB, InstListType::iterator It,
+                                bool InsertAtHead);
 
   /// Erase any DbgRecords attached to this instruction.
-  void dropDbgRecords();
+  LLVM_ABI void dropDbgRecords();
 
   /// Erase a single DbgRecord \p I that is attached to this instruction.
-  void dropOneDbgRecord(DbgRecord *I);
+  LLVM_ABI void dropOneDbgRecord(DbgRecord *I);
 
   /// Handle the debug-info implications of this instruction being removed. Any
   /// attached DbgRecords need to "fall" down onto the next instruction.
-  void handleMarkerRemoval();
+  LLVM_ABI void handleMarkerRemoval();
 
 protected:
   // The 15 first bits of `Value::SubclassData` are available for subclasses of
@@ -158,7 +160,7 @@ class Instruction : public User,
   using HasMetadataField = Bitfield::Element<bool, 15, 1>;
 
 protected:
-  ~Instruction(); // Use deleteValue() to delete a generic Instruction.
+  LLVM_ABI ~Instruction(); // Use deleteValue() to delete a generic Instruction.
 
 public:
   Instruction(const Instruction &) = delete;
@@ -174,7 +176,7 @@ class Instruction : public User,
   ///
   /// Note: this is undefined behavior if the instruction does not have a
   /// parent, or the parent basic block does not have a parent function.
-  const Module *getModule() const;
+  LLVM_ABI const Module *getModule() const;
   Module *getModule() {
     return const_cast<Module *>(
                            static_cast<const Instruction *>(this)->getModule());
@@ -184,7 +186,7 @@ class Instruction : public User,
   ///
   /// Note: it is undefined behavior to call this on an instruction not
   /// currently inserted into a function.
-  const Function *getFunction() const;
+  LLVM_ABI const Function *getFunction() const;
   Function *getFunction() {
     return const_cast<Function *>(
                          static_cast<const Instruction *>(this)->getFunction());
@@ -193,16 +195,16 @@ class Instruction : public User,
   /// Get the data layout of the module this instruction belongs to.
   ///
   /// Requires the instruction to have a parent module.
-  const DataLayout &getDataLayout() const;
+  LLVM_ABI const DataLayout &getDataLayout() const;
 
   /// This method unlinks 'this' from the containing basic block, but does not
   /// delete it.
-  void removeFromParent();
+  LLVM_ABI void removeFromParent();
 
   /// This method unlinks 'this' from the containing basic block and deletes it.
   ///
   /// \returns an iterator pointing to the element after the erased one
-  InstListType::iterator eraseFromParent();
+  LLVM_ABI InstListType::iterator eraseFromParent();
 
   /// Insert an unlinked instruction into a basic block immediately before
   /// the specified instruction.
@@ -211,27 +213,27 @@ class Instruction : public User,
   /// start of a block such as BasicBlock::getFirstNonPHIIt must be passed into
   /// insertBefore without unwrapping/rewrapping. For all other positions, call
   /// getIterator to fetch the instruction iterator.
-  LLVM_DEPRECATED("Use iterators as instruction positions", "")
-  void insertBefore(Instruction *InsertPos);
+  LLVM_ABI LLVM_DEPRECATED("Use iterators as instruction positions",
+                           "") void insertBefore(Instruction *InsertPos);
 
   /// Insert an unlinked instruction into a basic block immediately before
   /// the specified position.
-  void insertBefore(InstListType::iterator InsertPos);
+  LLVM_ABI void insertBefore(InstListType::iterator InsertPos);
 
   /// Insert an unlinked instruction into a basic block immediately after the
   /// specified instruction.
-  void insertAfter(Instruction *InsertPos);
+  LLVM_ABI void insertAfter(Instruction *InsertPos);
 
   /// Insert an unlinked instruction into a basic block immediately after the
   /// specified position.
-  void insertAfter(InstListType::iterator InsertPos);
+  LLVM_ABI void insertAfter(InstListType::iterator InsertPos);
 
   /// Inserts an unlinked instruction into \p ParentBB at position \p It and
   /// returns the iterator of the inserted instruction.
-  InstListType::iterator insertInto(BasicBlock *ParentBB,
-                                    InstListType::iterator It);
+  LLVM_ABI InstListType::iterator insertInto(BasicBlock *ParentBB,
+                                             InstListType::iterator It);
 
-  void insertBefore(BasicBlock &BB, InstListType::iterator InsertPos);
+  LLVM_ABI void insertBefore(BasicBlock &BB, InstListType::iterator InsertPos);
 
   /// Unlink this instruction from its current basic block and insert it into
   /// the basic block that MovePos lives in, right before MovePos.
@@ -240,22 +242,22 @@ class Instruction : public User,
   /// start of a block such as BasicBlock::getFirstNonPHIIt must be passed into
   /// moveBefore without unwrapping/rewrapping. For all other positions, call
   /// getIterator to fetch the instruction iterator.
-  LLVM_DEPRECATED("Use iterators as instruction positions", "")
-  void moveBefore(Instruction *MovePos);
+  LLVM_ABI LLVM_DEPRECATED("Use iterators as instruction positions",
+                           "") void moveBefore(Instruction *MovePos);
 
   /// Unlink this instruction from its current basic block and insert it into
   /// the basic block that MovePos lives in, right before MovePos.
-  void moveBefore(InstListType::iterator InsertPos);
+  LLVM_ABI void moveBefore(InstListType::iterator InsertPos);
 
   /// Perform a \ref moveBefore operation, while signalling that the caller
   /// intends to preserve the original ordering of instructions. This implicitly
   /// means that any adjacent debug-info should move with this instruction.
-  void moveBeforePreserving(InstListType::iterator MovePos);
+  LLVM_ABI void moveBeforePreserving(InstListType::iterator MovePos);
 
   /// Perform a \ref moveBefore operation, while signalling that the caller
   /// intends to preserve the original ordering of instructions. This implicitly
   /// means that any adjacent debug-info should move with this instruction.
-  void moveBeforePreserving(BasicBlock &BB, InstListType::iterator I);
+  LLVM_ABI void moveBeforePreserving(BasicBlock &BB, InstListType::iterator I);
 
   /// Perform a \ref moveBefore operation, while signalling that the caller
   /// intends to preserve the original ordering of instructions. This implicitly
@@ -263,8 +265,8 @@ class Instruction : public User,
   ///
   /// Deprecated in favour of the iterator-accepting flavour of
   /// moveBeforePreserving, as all insertions should be at iterator positions.
-  LLVM_DEPRECATED("Use iterators as instruction positions", "")
-  void moveBeforePreserving(Instruction *MovePos);
+  LLVM_ABI LLVM_DEPRECATED("Use iterators as instruction positions",
+                           "") void moveBeforePreserving(Instruction *MovePos);
 
 private:
   /// RemoveDIs project: all other moves implemented with this method,
@@ -275,32 +277,32 @@ class Instruction : public User,
   /// Unlink this instruction and insert into BB before I.
   ///
   /// \pre I is a valid iterator into BB.
-  void moveBefore(BasicBlock &BB, InstListType::iterator I);
+  LLVM_ABI void moveBefore(BasicBlock &BB, InstListType::iterator I);
 
   /// Unlink this instruction from its current basic block and insert it into
   /// the basic block that MovePos lives in, right after MovePos.
-  void moveAfter(Instruction *MovePos);
+  LLVM_ABI void moveAfter(Instruction *MovePos);
 
   /// Unlink this instruction from its current basic block and insert it into
   /// the basic block that MovePos lives in, right after MovePos.
-  void moveAfter(InstListType::iterator MovePos);
+  LLVM_ABI void moveAfter(InstListType::iterator MovePos);
 
   /// See \ref moveBeforePreserving .
-  void moveAfterPreserving(Instruction *MovePos);
+  LLVM_ABI void moveAfterPreserving(Instruction *MovePos);
 
   /// Given an instruction Other in the same basic block as this instruction,
   /// return true if this instruction comes before Other. In this worst case,
   /// this takes linear time in the number of instructions in the block. The
   /// results are cached, so in common cases when the block remains unmodified,
   /// it takes constant time.
-  bool comesBefore(const Instruction *Other) const;
+  LLVM_ABI bool comesBefore(const Instruction *Other) const;
 
   /// Get the first insertion point at which the result of this instruction
   /// is defined. This is *not* the directly following instruction in a number
   /// of cases, e.g. phi nodes or terminators that return values. This function
   /// may return null if the insertion after the definition is not possible,
   /// e.g. due to a catchswitch terminator.
-  std::optional<InstListType::iterator> getInsertionPointAfterDef();
+  LLVM_ABI std::optional<InstListType::iterator> getInsertionPointAfterDef();
 
   //===--------------------------------------------------------------------===//
   // Subclass classification.
@@ -322,9 +324,9 @@ class Instruction : public User,
 
   /// It checks if this instruction is the only user of at least one of
   /// its operands.
-  bool isOnlyUserOfAnyOperand();
+  LLVM_ABI bool isOnlyUserOfAnyOperand();
 
-  static const char *getOpcodeName(unsigned Opcode);
+  LLVM_ABI static const char *getOpcodeName(unsigned Opcode);
 
   static inline bool isTerminator(unsigned Opcode) {
     return Opcode >= TermOpsBegin && Opcode < TermOpsEnd;
@@ -405,7 +407,7 @@ class Instruction : public User,
 
   // Return true if this instruction contains loop metadata other than
   // a debug location
-  bool hasNonDebugLocLoopMetadata() const;
+  LLVM_ABI bool hasNonDebugLocLoopMetadata() const;
 
   /// Return true if this instruction has metadata attached to it other than a
   /// debug location.
@@ -455,22 +457,22 @@ class Instruction : public User,
 
   /// Set the metadata of the specified kind to the specified node. This updates
   /// or replaces metadata if already present, or removes it if Node is null.
-  void setMetadata(unsigned KindID, MDNode *Node);
-  void setMetadata(StringRef Kind, MDNode *Node);
+  LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node);
+  LLVM_ABI void setMetadata(StringRef Kind, MDNode *Node);
 
   /// Copy metadata from \p SrcInst to this instruction. \p WL, if not empty,
   /// specifies the list of meta data that needs to be copied. If \p WL is
   /// empty, all meta data will be copied.
-  void copyMetadata(const Instruction &SrcInst,
-                    ArrayRef<unsigned> WL = ArrayRef<unsigned>());
+  LLVM_ABI void copyMetadata(const Instruction &SrcInst,
+                             ArrayRef<unsigned> WL = ArrayRef<unsigned>());
 
   /// Erase all metadata that matches the predicate.
-  void eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred);
+  LLVM_ABI void eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred);
 
   /// If the instruction has "branch_weights" MD_prof metadata and the MDNode
   /// has three operands (including name string), swap the order of the
   /// metadata.
-  void swapProfMetadata();
+  LLVM_ABI void swapProfMetadata();
 
   /// Drop all unknown metadata except for debug locations.
   /// @{
@@ -478,31 +480,31 @@ class Instruction : public User,
   /// convenience method for passes to do so.
   /// dropUBImplyingAttrsAndUnknownMetadata should be used instead of
   /// this API if the Instruction being modified is a call.
-  void dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs = {});
+  LLVM_ABI void dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs = {});
   /// @}
 
   /// Adds an !annotation metadata node with \p Annotation to this instruction.
   /// If this instruction already has !annotation metadata, append \p Annotation
   /// to the existing node.
-  void addAnnotationMetadata(StringRef Annotation);
+  LLVM_ABI void addAnnotationMetadata(StringRef Annotation);
   /// Adds an !annotation metadata node with an array of \p Annotations
   /// as a tuple to this instruction. If this instruction already has
   /// !annotation metadata, append the tuple to
   /// the existing node.
-  void addAnnotationMetadata(SmallVector<StringRef> Annotations);
+  LLVM_ABI void addAnnotationMetadata(SmallVector<StringRef> Annotations);
   /// Returns the AA metadata for this instruction.
-  AAMDNodes getAAMetadata() const;
+  LLVM_ABI AAMDNodes getAAMetadata() const;
 
   /// Sets the AA metadata on this instruction from the AAMDNodes structure.
-  void setAAMetadata(const AAMDNodes &N);
+  LLVM_ABI void setAAMetadata(const AAMDNodes &N);
 
   /// Sets the nosanitize metadata on this instruction.
-  void setNoSanitizeMetadata();
+  LLVM_ABI void setNoSanitizeMetadata();
 
   /// Retrieve total raw weight values of a branch.
   /// Returns true on success with profile total weights filled in.
   /// Returns false if no metadata was found.
-  bool extractProfTotalWeight(uint64_t &TotalVal) const;
+  LLVM_ABI bool extractProfTotalWeight(uint64_t &TotalVal) const;
 
   /// Set the debug location information for this instruction.
   void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); }
@@ -512,52 +514,52 @@ class Instruction : public User,
 
   /// Fetch the debug location for this node, unless this is a debug intrinsic,
   /// in which case fetch the debug location of the next non-debug node.
-  const DebugLoc &getStableDebugLoc() const;
+  LLVM_ABI const DebugLoc &getStableDebugLoc() const;
 
   /// Set or clear the nuw flag on this instruction, which must be an operator
   /// which supports this flag. See LangRef.html for the meaning of this flag.
-  void setHasNoUnsignedWrap(bool b = true);
+  LLVM_ABI void setHasNoUnsignedWrap(bool b = true);
 
   /// Set or clear the nsw flag on this instruction, which must be an operator
   /// which supports this flag. See LangRef.html for the meaning of this flag.
-  void setHasNoSignedWrap(bool b = true);
+  LLVM_ABI void setHasNoSignedWrap(bool b = true);
 
   /// Set or clear the exact flag on this instruction, which must be an operator
   /// which supports this flag. See LangRef.html for the meaning of this flag.
-  void setIsExact(bool b = true);
+  LLVM_ABI void setIsExact(bool b = true);
 
   /// Set or clear the nneg flag on this instruction, which must be a zext
   /// instruction.
-  void setNonNeg(bool b = true);
+  LLVM_ABI void setNonNeg(bool b = true);
 
   /// Determine whether the no unsigned wrap flag is set.
-  bool hasNoUnsignedWrap() const LLVM_READONLY;
+  LLVM_ABI bool hasNoUnsignedWrap() const LLVM_READONLY;
 
   /// Determine whether the no signed wrap flag is set.
-  bool hasNoSignedWrap() const LLVM_READONLY;
+  LLVM_ABI bool hasNoSignedWrap() const LLVM_READONLY;
 
   /// Determine whether the the nneg flag is set.
-  bool hasNonNeg() const LLVM_READONLY;
+  LLVM_ABI bool hasNonNeg() const LLVM_READONLY;
 
   /// Return true if this operator has flags which may cause this instruction
   /// to evaluate to poison despite having non-poison inputs.
-  bool hasPoisonGeneratingFlags() const LLVM_READONLY;
+  LLVM_ABI bool hasPoisonGeneratingFlags() const LLVM_READONLY;
 
   /// Drops flags that may cause this instruction to evaluate to poison despite
   /// having non-poison inputs.
-  void dropPoisonGeneratingFlags();
+  LLVM_ABI void dropPoisonGeneratingFlags();
 
   /// Return true if this instruction has poison-generating metadata.
-  bool hasPoisonGeneratingMetadata() const LLVM_READONLY;
+  LLVM_ABI bool hasPoisonGeneratingMetadata() const LLVM_READONLY;
 
   /// Drops metadata that may generate poison.
-  void dropPoisonGeneratingMetadata();
+  LLVM_ABI void dropPoisonGeneratingMetadata();
 
   /// Return true if this instruction has poison-generating attribute.
-  bool hasPoisonGeneratingReturnAttributes() const LLVM_READONLY;
+  LLVM_ABI bool hasPoisonGeneratingReturnAttributes() const LLVM_READONLY;
 
   /// Drops return attributes that may generate poison.
-  void dropPoisonGeneratingReturnAttributes();
+  LLVM_ABI void dropPoisonGeneratingReturnAttributes();
 
   /// Return true if this instruction has poison-generating flags,
   /// return attributes or metadata.
@@ -578,109 +580,110 @@ class Instruction : public User,
   /// dropUnknownNonDebugMetadata). For calls, it also drops parameter and
   /// return attributes that can cause undefined behaviour. Both of these should
   /// be done by passes which move instructions in IR.
-  void dropUBImplyingAttrsAndUnknownMetadata(ArrayRef<unsigned> KnownIDs = {});
+  LLVM_ABI void
+  dropUBImplyingAttrsAndUnknownMetadata(ArrayRef<unsigned> KnownIDs = {});
 
   /// Drop any attributes or metadata that can cause immediate undefined
   /// behavior. Retain other attributes/metadata on a best-effort basis.
   /// This should be used when speculating instructions.
-  void dropUBImplyingAttrsAndMetadata();
+  LLVM_ABI void dropUBImplyingAttrsAndMetadata();
 
   /// Return true if this instruction has UB-implying attributes
   /// that can cause immediate undefined behavior.
-  bool hasUBImplyingAttrs() const LLVM_READONLY;
+  LLVM_ABI bool hasUBImplyingAttrs() const LLVM_READONLY;
 
   /// Determine whether the exact flag is set.
-  bool isExact() const LLVM_READONLY;
+  LLVM_ABI bool isExact() const LLVM_READONLY;
 
   /// Set or clear all fast-math-flags on this instruction, which must be an
   /// operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setFast(bool B);
+  LLVM_ABI void setFast(bool B);
 
   /// Set or clear the reassociation flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasAllowReassoc(bool B);
+  LLVM_ABI void setHasAllowReassoc(bool B);
 
   /// Set or clear the no-nans flag on this instruction, which must be an
   /// operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasNoNaNs(bool B);
+  LLVM_ABI void setHasNoNaNs(bool B);
 
   /// Set or clear the no-infs flag on this instruction, which must be an
   /// operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasNoInfs(bool B);
+  LLVM_ABI void setHasNoInfs(bool B);
 
   /// Set or clear the no-signed-zeros flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasNoSignedZeros(bool B);
+  LLVM_ABI void setHasNoSignedZeros(bool B);
 
   /// Set or clear the allow-reciprocal flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasAllowReciprocal(bool B);
+  LLVM_ABI void setHasAllowReciprocal(bool B);
 
   /// Set or clear the allow-contract flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasAllowContract(bool B);
+  LLVM_ABI void setHasAllowContract(bool B);
 
   /// Set or clear the approximate-math-functions flag on this instruction,
   /// which must be an operator which supports this flag. See LangRef.html for
   /// the meaning of this flag.
-  void setHasApproxFunc(bool B);
+  LLVM_ABI void setHasApproxFunc(bool B);
 
   /// Convenience function for setting multiple fast-math flags on this
   /// instruction, which must be an operator which supports these flags. See
   /// LangRef.html for the meaning of these flags.
-  void setFastMathFlags(FastMathFlags FMF);
+  LLVM_ABI void setFastMathFlags(FastMathFlags FMF);
 
   /// Convenience function for transferring all fast-math flag values to this
   /// instruction, which must be an operator which supports these flags. See
   /// LangRef.html for the meaning of these flags.
-  void copyFastMathFlags(FastMathFlags FMF);
+  LLVM_ABI void copyFastMathFlags(FastMathFlags FMF);
 
   /// Determine whether all fast-math-flags are set.
-  bool isFast() const LLVM_READONLY;
+  LLVM_ABI bool isFast() const LLVM_READONLY;
 
   /// Determine whether the allow-reassociation flag is set.
-  bool hasAllowReassoc() const LLVM_READONLY;
+  LLVM_ABI bool hasAllowReassoc() const LLVM_READONLY;
 
   /// Determine whether the no-NaNs flag is set.
-  bool hasNoNaNs() const LLVM_READONLY;
+  LLVM_ABI bool hasNoNaNs() const LLVM_READONLY;
 
   /// Determine whether the no-infs flag is set.
-  bool hasNoInfs() const LLVM_READONLY;
+  LLVM_ABI bool hasNoInfs() const LLVM_READONLY;
 
   /// Determine whether the no-signed-zeros flag is set.
-  bool hasNoSignedZeros() const LLVM_READONLY;
+  LLVM_ABI bool hasNoSignedZeros() const LLVM_READONLY;
 
   /// Determine whether the allow-reciprocal flag is set.
-  bool hasAllowReciprocal() const LLVM_READONLY;
+  LLVM_ABI bool hasAllowReciprocal() const LLVM_READONLY;
 
   /// Determine whether the allow-contract flag is set.
-  bool hasAllowContract() const LLVM_READONLY;
+  LLVM_ABI bool hasAllowContract() const LLVM_READONLY;
 
   /// Determine whether the approximate-math-functions flag is set.
-  bool hasApproxFunc() const LLVM_READONLY;
+  LLVM_ABI bool hasApproxFunc() const LLVM_READONLY;
 
   /// Convenience function for getting all the fast-math flags, which must be an
   /// operator which supports these flags. See LangRef.html for the meaning of
   /// these flags.
-  FastMathFlags getFastMathFlags() const LLVM_READONLY;
+  LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY;
 
   /// Copy I's fast-math flags
-  void copyFastMathFlags(const Instruction *I);
+  LLVM_ABI void copyFastMathFlags(const Instruction *I);
 
   /// Convenience method to copy supported exact, fast-math, and (optionally)
   /// wrapping flags from V to this instruction.
-  void copyIRFlags(const Value *V, bool IncludeWrapFlags = true);
+  LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags = true);
 
   /// Logical 'and' of any supported wrapping, exact, and fast-math flags of
   /// V and this instruction.
-  void andIRFlags(const Value *V);
+  LLVM_ABI void andIRFlags(const Value *V);
 
   /// Merge 2 debug locations and apply it to the Instruction. If the
   /// instruction is a CallIns, we need to traverse the inline chain to find
@@ -695,13 +698,13 @@ class Instruction : public User,
   ///     applications, thus the N-way merging should be in code path.
   /// The DebugLoc attached to this instruction will be overwritten by the
   /// merged DebugLoc.
-  void applyMergedLocation(DILocation *LocA, DILocation *LocB);
+  LLVM_ABI void applyMergedLocation(DebugLoc LocA, DebugLoc LocB);
 
   /// Updates the debug location given that the instruction has been hoisted
   /// from a block to a predecessor of that block.
   /// Note: it is undefined behavior to call this on an instruction not
   /// currently inserted into a function.
-  void updateLocationAfterHoist();
+  LLVM_ABI void updateLocationAfterHoist();
 
   /// Drop the instruction's debug location. This does not guarantee removal
   /// of the !dbg source location attachment, as it must set a line 0 location
@@ -709,7 +712,7 @@ class Instruction : public User,
   /// removal of the !dbg attachment, use the \ref setDebugLoc() API.
   /// Note: it is undefined behavior to call this on an instruction not
   /// currently inserted into a function.
-  void dropLocation();
+  LLVM_ABI void dropLocation();
 
   /// Merge the DIAssignID metadata from this instruction and those attached to
   /// instructions in \p SourceInstructions. This process performs a RAUW on
@@ -720,12 +723,13 @@ class Instruction : public User,
   /// SourceInstructions does then the merged one will be attached to
   /// it. However, instructions without attachments in \p SourceInstructions
   /// are not modified.
-  void mergeDIAssignID(ArrayRef<const Instruction *> SourceInstructions);
+  LLVM_ABI void
+  mergeDIAssignID(ArrayRef<const Instruction *> SourceInstructions);
 
 private:
   // These are all implemented in Metadata.cpp.
-  MDNode *getMetadataImpl(StringRef Kind) const;
-  void
+  LLVM_ABI MDNode *getMetadataImpl(StringRef Kind) const;
+  LLVM_ABI void
   getAllMetadataImpl(SmallVectorImpl<std::pair<unsigned, MDNode *>> &) const;
 
   /// Update the LLVMContext ID-to-Instruction(s) mapping. If \p ID is nullptr
@@ -743,7 +747,7 @@ class Instruction : public User,
   ///
   /// In LLVM, the Add, Mul, And, Or, and Xor operators are associative.
   ///
-  bool isAssociative() const LLVM_READONLY;
+  LLVM_ABI bool isAssociative() const LLVM_READONLY;
   static bool isAssociative(unsigned Opcode) {
     return Opcode == And || Opcode == Or || Opcode == Xor ||
            Opcode == Add || Opcode == Mul;
@@ -756,7 +760,7 @@ class Instruction : public User,
   /// In LLVM, these are the commutative operators, plus SetEQ and SetNE, when
   /// applied to any type.
   ///
-  bool isCommutative() const LLVM_READONLY;
+  LLVM_ABI bool isCommutative() const LLVM_READONLY;
   static bool isCommutative(unsigned Opcode) {
     switch (Opcode) {
     case Add: case FAdd:
@@ -794,10 +798,10 @@ class Instruction : public User,
   }
 
   /// Return true if this instruction may modify memory.
-  bool mayWriteToMemory() const LLVM_READONLY;
+  LLVM_ABI bool mayWriteToMemory() const LLVM_READONLY;
 
   /// Return true if this instruction may read memory.
-  bool mayReadFromMemory() const LLVM_READONLY;
+  LLVM_ABI bool mayReadFromMemory() const LLVM_READONLY;
 
   /// Return true if this instruction may read or write memory.
   bool mayReadOrWriteMemory() const {
@@ -806,26 +810,27 @@ class Instruction : public User,
 
   /// Return true if this instruction has an AtomicOrdering of unordered or
   /// higher.
-  bool isAtomic() const LLVM_READONLY;
+  LLVM_ABI bool isAtomic() const LLVM_READONLY;
 
   /// Return true if this atomic instruction loads from memory.
-  bool hasAtomicLoad() const LLVM_READONLY;
+  LLVM_ABI bool hasAtomicLoad() const LLVM_READONLY;
 
   /// Return true if this atomic instruction stores to memory.
-  bool hasAtomicStore() const LLVM_READONLY;
+  LLVM_ABI bool hasAtomicStore() const LLVM_READONLY;
 
   /// Return true if this instruction has a volatile memory access.
-  bool isVolatile() const LLVM_READONLY;
+  LLVM_ABI bool isVolatile() const LLVM_READONLY;
 
   /// Return the type this instruction accesses in memory, if any.
-  Type *getAccessType() const LLVM_READONLY;
+  LLVM_ABI Type *getAccessType() const LLVM_READONLY;
 
   /// Return true if this instruction may throw an exception.
   ///
   /// If IncludePhaseOneUnwind is set, this will also include cases where
   /// phase one unwinding may unwind past this frame due to skipping of
   /// cleanup landingpads.
-  bool mayThrow(bool IncludePhaseOneUnwind = false) const LLVM_READONLY;
+  LLVM_ABI bool
+  mayThrow(bool IncludePhaseOneUnwind = false) const LLVM_READONLY;
 
   /// Return true if this instruction behaves like a memory fence: it can load
   /// or store to memory location without being given a memory location.
@@ -855,7 +860,7 @@ class Instruction : public User,
   /// effects because the newly allocated memory is completely invisible to
   /// instructions which don't use the returned value.  For cases where this
   /// matters, isSafeToSpeculativelyExecute may be more appropriate.
-  bool mayHaveSideEffects() const LLVM_READONLY;
+  LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY;
 
   /// Return true if the instruction can be removed if the result is unused.
   ///
@@ -863,11 +868,11 @@ class Instruction : public User,
   /// results are unused. Specifically terminator instructions and calls that
   /// may have side effects cannot be removed without semantically changing the
   /// generated program.
-  bool isSafeToRemove() const LLVM_READONLY;
+  LLVM_ABI bool isSafeToRemove() const LLVM_READONLY;
 
   /// Return true if the instruction will return (unwinding is considered as
   /// a form of returning control flow here).
-  bool willReturn() const LLVM_READONLY;
+  LLVM_ABI bool willReturn() const LLVM_READONLY;
 
   /// Return true if the instruction is a variety of EH-block.
   bool isEHPad() const {
@@ -884,19 +889,19 @@ class Instruction : public User,
 
   /// Return true if the instruction is a llvm.lifetime.start or
   /// llvm.lifetime.end marker.
-  bool isLifetimeStartOrEnd() const LLVM_READONLY;
+  LLVM_ABI bool isLifetimeStartOrEnd() const LLVM_READONLY;
 
   /// Return true if the instruction is a llvm.launder.invariant.group or
   /// llvm.strip.invariant.group.
-  bool isLaunderOrStripInvariantGroup() const LLVM_READONLY;
+  LLVM_ABI bool isLaunderOrStripInvariantGroup() const LLVM_READONLY;
 
   /// Return true if the instruction is a DbgInfoIntrinsic or PseudoProbeInst.
-  bool isDebugOrPseudoInst() const LLVM_READONLY;
+  LLVM_ABI bool isDebugOrPseudoInst() const LLVM_READONLY;
 
   /// Return a pointer to the next non-debug instruction in the same basic
   /// block as 'this', or nullptr if no such instruction exists. Skip any pseudo
   /// operations if \c SkipPseudoOp is true.
-  const Instruction *
+  LLVM_ABI const Instruction *
   getNextNonDebugInstruction(bool SkipPseudoOp = false) const;
   Instruction *getNextNonDebugInstruction(bool SkipPseudoOp = false) {
     return const_cast<Instruction *>(
@@ -907,7 +912,7 @@ class Instruction : public User,
   /// Return a pointer to the previous non-debug instruction in the same basic
   /// block as 'this', or nullptr if no such instruction exists. Skip any pseudo
   /// operations if \c SkipPseudoOp is true.
-  const Instruction *
+  LLVM_ABI const Instruction *
   getPrevNonDebugInstruction(bool SkipPseudoOp = false) const;
   Instruction *getPrevNonDebugInstruction(bool SkipPseudoOp = false) {
     return const_cast<Instruction *>(
@@ -920,17 +925,17 @@ class Instruction : public User,
   ///   * The instruction has no parent
   ///   * The instruction has no name
   ///
-  Instruction *clone() const;
+  LLVM_ABI Instruction *clone() const;
 
   /// Return true if the specified instruction is exactly identical to the
   /// current one. This means that all operands match and any extra information
   /// (e.g. load is volatile) agree.
-  bool isIdenticalTo(const Instruction *I) const LLVM_READONLY;
+  LLVM_ABI bool isIdenticalTo(const Instruction *I) const LLVM_READONLY;
 
   /// This is like isIdenticalTo, except that it ignores the
   /// SubclassOptionalData flags, which may specify conditions under which the
   /// instruction's result is undefined.
-  bool
+  LLVM_ABI bool
   isIdenticalToWhenDefined(const Instruction *I,
                            bool IntersectAttrs = false) const LLVM_READONLY;
 
@@ -954,7 +959,8 @@ class Instruction : public User,
   /// @returns true if the specified instruction is the same operation as
   /// the current one.
   /// Determine if one instruction is the same operation as another.
-  bool isSameOperationAs(const Instruction *I, unsigned flags = 0) const LLVM_READONLY;
+  LLVM_ABI bool isSameOperationAs(const Instruction *I,
+                                  unsigned flags = 0) const LLVM_READONLY;
 
   /// This function determines if the speficied instruction has the same
   /// "special" characteristics as the current one. This means that opcode
@@ -964,28 +970,29 @@ class Instruction : public User,
   /// @returns true if the specific instruction has the same opcde specific
   /// characteristics as the current one. Determine if one instruction has the
   /// same state as another.
-  bool hasSameSpecialState(const Instruction *I2, bool IgnoreAlignment = false,
-                           bool IntersectAttrs = false) const LLVM_READONLY;
+  LLVM_ABI bool
+  hasSameSpecialState(const Instruction *I2, bool IgnoreAlignment = false,
+                      bool IntersectAttrs = false) const LLVM_READONLY;
 
   /// Return true if there are any uses of this instruction in blocks other than
   /// the specified block. Note that PHI nodes are considered to evaluate their
   /// operands in the corresponding predecessor block.
-  bool isUsedOutsideOfBlock(const BasicBlock *BB) const LLVM_READONLY;
+  LLVM_ABI bool isUsedOutsideOfBlock(const BasicBlock *BB) const LLVM_READONLY;
 
   /// Return the number of successors that this instruction has. The instruction
   /// must be a terminator.
-  unsigned getNumSuccessors() const LLVM_READONLY;
+  LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY;
 
   /// Return the specified successor. This instruction must be a terminator.
-  BasicBlock *getSuccessor(unsigned Idx) const LLVM_READONLY;
+  LLVM_ABI BasicBlock *getSuccessor(unsigned Idx) const LLVM_READONLY;
 
   /// Update the specified successor to point at the provided block. This
   /// instruction must be a terminator.
-  void setSuccessor(unsigned Idx, BasicBlock *BB);
+  LLVM_ABI void setSuccessor(unsigned Idx, BasicBlock *BB);
 
   /// Replace specified successor OldBB to point at the provided block.
   /// This instruction must be a terminator.
-  void replaceSuccessorWith(BasicBlock *OldBB, BasicBlock *NewBB);
+  LLVM_ABI void replaceSuccessorWith(BasicBlock *OldBB, BasicBlock *NewBB);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
@@ -1083,8 +1090,8 @@ class Instruction : public User,
     setValueSubclassData(Storage);
   }
 
-  Instruction(Type *Ty, unsigned iType, AllocInfo AllocInfo,
-              InsertPosition InsertBefore = nullptr);
+  LLVM_ABI Instruction(Type *Ty, unsigned iType, AllocInfo AllocInfo,
+                       InsertPosition InsertBefore = nullptr);
 
 private:
   /// Create a copy of this instruction.
diff --git a/external/llvm-project/llvm/include/llvm/IR/Instructions.h b/external/llvm-project/llvm/include/llvm/IR/Instructions.h
index c164f76eb335..6f69b68f628f 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Instructions.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Instructions.h
@@ -35,6 +35,7 @@
 #include "llvm/IR/Use.h"
 #include "llvm/IR/User.h"
 #include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 #include <cstddef>
@@ -74,21 +75,22 @@ class AllocaInst : public UnaryInstruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  AllocaInst *cloneImpl() const;
+  LLVM_ABI AllocaInst *cloneImpl() const;
 
 public:
-  explicit AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
-                      const Twine &Name, InsertPosition InsertBefore);
+  LLVM_ABI explicit AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
+                               const Twine &Name, InsertPosition InsertBefore);
 
-  AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
-             InsertPosition InsertBefore);
+  LLVM_ABI AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
+                      InsertPosition InsertBefore);
 
-  AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
-             const Twine &Name = "", InsertPosition InsertBefore = nullptr);
+  LLVM_ABI AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
+                      Align Align, const Twine &Name = "",
+                      InsertPosition InsertBefore = nullptr);
 
   /// Return true if there is an allocation size parameter to the allocation
   /// instruction that is not 1.
-  bool isArrayAllocation() const;
+  LLVM_ABI bool isArrayAllocation() const;
 
   /// Get the number of elements allocated. For a simple allocation of a single
   /// element, this will return a constant 1 value.
@@ -107,11 +109,13 @@ class AllocaInst : public UnaryInstruction {
 
   /// Get allocation size in bytes. Returns std::nullopt if size can't be
   /// determined, e.g. in case of a VLA.
-  std::optional<TypeSize> getAllocationSize(const DataLayout &DL) const;
+  LLVM_ABI std::optional<TypeSize>
+  getAllocationSize(const DataLayout &DL) const;
 
   /// Get allocation size in bits. Returns std::nullopt if size can't be
   /// determined, e.g. in case of a VLA.
-  std::optional<TypeSize> getAllocationSizeInBits(const DataLayout &DL) const;
+  LLVM_ABI std::optional<TypeSize>
+  getAllocationSizeInBits(const DataLayout &DL) const;
 
   /// Return the type that is being allocated by the instruction.
   Type *getAllocatedType() const { return AllocatedType; }
@@ -132,7 +136,7 @@ class AllocaInst : public UnaryInstruction {
   /// Return true if this alloca is in the entry block of the function and is a
   /// constant size. If so, the code generator will fold it into the
   /// prolog/epilog code, so it is basically free.
-  bool isStaticAlloca() const;
+  LLVM_ABI bool isStaticAlloca() const;
 
   /// Return true if this alloca is used as an inalloca argument to a call. Such
   /// allocas are never considered static even if they are in the entry block.
@@ -187,19 +191,19 @@ class LoadInst : public UnaryInstruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  LoadInst *cloneImpl() const;
+  LLVM_ABI LoadInst *cloneImpl() const;
 
 public:
-  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
-           InsertPosition InsertBefore);
-  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
-           InsertPosition InsertBefore);
-  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
-           Align Align, InsertPosition InsertBefore = nullptr);
-  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
-           Align Align, AtomicOrdering Order,
-           SyncScope::ID SSID = SyncScope::System,
-           InsertPosition InsertBefore = nullptr);
+  LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
+                    InsertPosition InsertBefore);
+  LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+                    InsertPosition InsertBefore);
+  LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+                    Align Align, InsertPosition InsertBefore = nullptr);
+  LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+                    Align Align, AtomicOrdering Order,
+                    SyncScope::ID SSID = SyncScope::System,
+                    InsertPosition InsertBefore = nullptr);
 
   /// Return true if this is a load from a volatile memory location.
   bool isVolatile() const { return getSubclassData<VolatileField>(); }
@@ -305,17 +309,18 @@ class StoreInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  StoreInst *cloneImpl() const;
+  LLVM_ABI StoreInst *cloneImpl() const;
 
 public:
-  StoreInst(Value *Val, Value *Ptr, InsertPosition InsertBefore);
-  StoreInst(Value *Val, Value *Ptr, bool isVolatile,
-            InsertPosition InsertBefore);
-  StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
-            InsertPosition InsertBefore = nullptr);
-  StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
-            AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
-            InsertPosition InsertBefore = nullptr);
+  LLVM_ABI StoreInst(Value *Val, Value *Ptr, InsertPosition InsertBefore);
+  LLVM_ABI StoreInst(Value *Val, Value *Ptr, bool isVolatile,
+                     InsertPosition InsertBefore);
+  LLVM_ABI StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
+                     InsertPosition InsertBefore = nullptr);
+  LLVM_ABI StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
+                     AtomicOrdering Order,
+                     SyncScope::ID SSID = SyncScope::System,
+                     InsertPosition InsertBefore = nullptr);
 
   // allocate space for exactly two operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -432,14 +437,14 @@ class FenceInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  FenceInst *cloneImpl() const;
+  LLVM_ABI FenceInst *cloneImpl() const;
 
 public:
   // Ordering may only be Acquire, Release, AcquireRelease, or
   // SequentiallyConsistent.
-  FenceInst(LLVMContext &C, AtomicOrdering Ordering,
-            SyncScope::ID SSID = SyncScope::System,
-            InsertPosition InsertBefore = nullptr);
+  LLVM_ABI FenceInst(LLVMContext &C, AtomicOrdering Ordering,
+                     SyncScope::ID SSID = SyncScope::System,
+                     InsertPosition InsertBefore = nullptr);
 
   // allocate space for exactly zero operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -514,13 +519,13 @@ class AtomicCmpXchgInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  AtomicCmpXchgInst *cloneImpl() const;
+  LLVM_ABI AtomicCmpXchgInst *cloneImpl() const;
 
 public:
-  AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
-                    AtomicOrdering SuccessOrdering,
-                    AtomicOrdering FailureOrdering, SyncScope::ID SSID,
-                    InsertPosition InsertBefore = nullptr);
+  LLVM_ABI AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
+                             Align Alignment, AtomicOrdering SuccessOrdering,
+                             AtomicOrdering FailureOrdering, SyncScope::ID SSID,
+                             InsertPosition InsertBefore = nullptr);
 
   // allocate space for exactly three operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -706,7 +711,7 @@ class AtomicRMWInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  AtomicRMWInst *cloneImpl() const;
+  LLVM_ABI AtomicRMWInst *cloneImpl() const;
 
 public:
   /// This enumeration lists the possible modifications atomicrmw can make.  In
@@ -793,9 +798,10 @@ class AtomicRMWInst : public Instruction {
   constexpr static IntrusiveOperandsAllocMarker AllocMarker{2};
 
 public:
-  AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
-                AtomicOrdering Ordering, SyncScope::ID SSID,
-                InsertPosition InsertBefore = nullptr);
+  LLVM_ABI AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+                         Align Alignment, AtomicOrdering Ordering,
+                         SyncScope::ID SSID,
+                         InsertPosition InsertBefore = nullptr);
 
   // allocate space for exactly two operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -812,7 +818,7 @@ class AtomicRMWInst : public Instruction {
 
   BinOp getOperation() const { return getSubclassData<OperationField>(); }
 
-  static StringRef getOperationName(BinOp Op);
+  LLVM_ABI static StringRef getOperationName(BinOp Op);
 
   static bool isFPOperation(BinOp Op) {
     switch (Op) {
@@ -954,13 +960,14 @@ class GetElementPtrInst : public Instruction {
                            ArrayRef<Value *> IdxList, AllocInfo AllocInfo,
                            const Twine &NameStr, InsertPosition InsertBefore);
 
-  void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
+  LLVM_ABI void init(Value *Ptr, ArrayRef<Value *> IdxList,
+                     const Twine &NameStr);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  GetElementPtrInst *cloneImpl() const;
+  LLVM_ABI GetElementPtrInst *cloneImpl() const;
 
 public:
   static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
@@ -1018,17 +1025,17 @@ class GetElementPtrInst : public Instruction {
   ///
   /// Null is returned if the indices are invalid for the specified
   /// source element type.
-  static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
-  static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
-  static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
+  LLVM_ABI static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
+  LLVM_ABI static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
+  LLVM_ABI static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
 
   /// Return the type of the element at the given index of an indexable
   /// type.  This is equivalent to "getIndexedType(Agg, {Zero, Idx})".
   ///
   /// Returns null if the type can't be indexed, or the given index is not
   /// legal for the given type.
-  static Type *getTypeAtIndex(Type *Ty, Value *Idx);
-  static Type *getTypeAtIndex(Type *Ty, uint64_t Idx);
+  LLVM_ABI static Type *getTypeAtIndex(Type *Ty, Value *Idx);
+  LLVM_ABI static Type *getTypeAtIndex(Type *Ty, uint64_t Idx);
 
   inline op_iterator       idx_begin()       { return op_begin()+1; }
   inline const_op_iterator idx_begin() const { return op_begin()+1; }
@@ -1092,32 +1099,32 @@ class GetElementPtrInst : public Instruction {
   /// Return true if all of the indices of this GEP are
   /// zeros.  If so, the result pointer and the first operand have the same
   /// value, just potentially different types.
-  bool hasAllZeroIndices() const;
+  LLVM_ABI bool hasAllZeroIndices() const;
 
   /// Return true if all of the indices of this GEP are
   /// constant integers.  If so, the result pointer and the first operand have
   /// a constant offset between them.
-  bool hasAllConstantIndices() const;
+  LLVM_ABI bool hasAllConstantIndices() const;
 
   /// Set nowrap flags for GEP instruction.
-  void setNoWrapFlags(GEPNoWrapFlags NW);
+  LLVM_ABI void setNoWrapFlags(GEPNoWrapFlags NW);
 
   /// Set or clear the inbounds flag on this GEP instruction.
   /// See LangRef.html for the meaning of inbounds on a getelementptr.
   /// TODO: Remove this method in favor of setNoWrapFlags().
-  void setIsInBounds(bool b = true);
+  LLVM_ABI void setIsInBounds(bool b = true);
 
   /// Get the nowrap flags for the GEP instruction.
-  GEPNoWrapFlags getNoWrapFlags() const;
+  LLVM_ABI GEPNoWrapFlags getNoWrapFlags() const;
 
   /// Determine whether the GEP has the inbounds flag.
-  bool isInBounds() const;
+  LLVM_ABI bool isInBounds() const;
 
   /// Determine whether the GEP has the nusw flag.
-  bool hasNoUnsignedSignedWrap() const;
+  LLVM_ABI bool hasNoUnsignedSignedWrap() const;
 
   /// Determine whether the GEP has the nuw flag.
-  bool hasNoUnsignedWrap() const;
+  LLVM_ABI bool hasNoUnsignedWrap() const;
 
   /// Accumulate the constant address offset of this GEP if possible.
   ///
@@ -1127,10 +1134,12 @@ class GetElementPtrInst : public Instruction {
   /// undefined (it is *not* preserved!). The APInt passed into this routine
   /// must be at least as wide as the IntPtr type for the address space of
   /// the base GEP pointer.
-  bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
-  bool collectOffset(const DataLayout &DL, unsigned BitWidth,
-                     SmallMapVector<Value *, APInt, 4> &VariableOffsets,
-                     APInt &ConstantOffset) const;
+  LLVM_ABI bool accumulateConstantOffset(const DataLayout &DL,
+                                         APInt &Offset) const;
+  LLVM_ABI bool
+  collectOffset(const DataLayout &DL, unsigned BitWidth,
+                SmallMapVector<Value *, APInt, 4> &VariableOffsets,
+                APInt &ConstantOffset) const;
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
     return (I->getOpcode() == Instruction::GetElementPtr);
@@ -1184,7 +1193,7 @@ class ICmpInst: public CmpInst {
   friend class Instruction;
 
   /// Clone an identical ICmpInst
-  ICmpInst *cloneImpl() const;
+  LLVM_ABI ICmpInst *cloneImpl() const;
 
 public:
   /// Constructor with insertion semantics.
@@ -1262,7 +1271,7 @@ class ICmpInst: public CmpInst {
   }
 
   /// Return the signed version of the predicate: static variant.
-  static Predicate getSignedPredicate(Predicate Pred);
+  LLVM_ABI static Predicate getSignedPredicate(Predicate Pred);
 
   /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
   /// @returns the predicate that would be the result if the operand were
@@ -1273,13 +1282,13 @@ class ICmpInst: public CmpInst {
   }
 
   /// Return the unsigned version of the predicate: static variant.
-  static Predicate getUnsignedPredicate(Predicate Pred);
+  LLVM_ABI static Predicate getUnsignedPredicate(Predicate Pred);
 
   /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ
   /// @returns the unsigned version of the signed predicate pred or
   ///          the signed version of the signed predicate pred.
   /// Static variant.
-  static Predicate getFlippedSignednessPredicate(Predicate Pred);
+  LLVM_ABI static Predicate getFlippedSignednessPredicate(Predicate Pred);
 
   /// For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ
   /// @returns the unsigned version of the signed predicate pred or
@@ -1290,8 +1299,8 @@ class ICmpInst: public CmpInst {
 
   /// Determine if Pred1 implies Pred2 is true, false, or if nothing can be
   /// inferred about the implication, when two compares have matching operands.
-  static std::optional<bool> isImpliedByMatchingCmp(CmpPredicate Pred1,
-                                                    CmpPredicate Pred2);
+  LLVM_ABI static std::optional<bool>
+  isImpliedByMatchingCmp(CmpPredicate Pred1, CmpPredicate Pred2);
 
   void setSameSign(bool B = true) {
     SubclassOptionalData = (SubclassOptionalData & ~SameSign) | (B * SameSign);
@@ -1373,13 +1382,13 @@ class ICmpInst: public CmpInst {
   }
 
   /// Return result of `LHS Pred RHS` comparison.
-  static bool compare(const APInt &LHS, const APInt &RHS,
-                      ICmpInst::Predicate Pred);
+  LLVM_ABI static bool compare(const APInt &LHS, const APInt &RHS,
+                               ICmpInst::Predicate Pred);
 
   /// Return result of `LHS Pred RHS`, if it can be determined from the
   /// KnownBits. Otherwise return nullopt.
-  static std::optional<bool> compare(const KnownBits &LHS, const KnownBits &RHS,
-                                     ICmpInst::Predicate Pred);
+  LLVM_ABI static std::optional<bool>
+  compare(const KnownBits &LHS, const KnownBits &RHS, ICmpInst::Predicate Pred);
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
@@ -1413,7 +1422,7 @@ class FCmpInst: public CmpInst {
   friend class Instruction;
 
   /// Clone an identical FCmpInst
-  FCmpInst *cloneImpl() const;
+  LLVM_ABI FCmpInst *cloneImpl() const;
 
 public:
   /// Constructor with insertion semantics.
@@ -1480,8 +1489,8 @@ class FCmpInst: public CmpInst {
   static auto predicates() { return FCmpPredicates(); }
 
   /// Return result of `LHS Pred RHS` comparison.
-  static bool compare(const APFloat &LHS, const APFloat &RHS,
-                      FCmpInst::Predicate Pred);
+  LLVM_ABI static bool compare(const APFloat &LHS, const APFloat &RHS,
+                               FCmpInst::Predicate Pred);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
@@ -1511,11 +1520,11 @@ class CallInst : public CallBase {
                   InsertPosition InsertBefore)
       : CallInst(Ty, Func, Args, {}, NameStr, AllocInfo, InsertBefore) {}
 
-  explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
-                    AllocInfo AllocInfo, InsertPosition InsertBefore);
+  LLVM_ABI explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
+                             AllocInfo AllocInfo, InsertPosition InsertBefore);
 
-  void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
-            ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
+  LLVM_ABI void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
+                     ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
   void init(FunctionType *FTy, Value *Func, const Twine &NameStr);
 
   /// Compute the number of operands to allocate.
@@ -1530,7 +1539,7 @@ class CallInst : public CallBase {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  CallInst *cloneImpl() const;
+  LLVM_ABI CallInst *cloneImpl() const;
 
 public:
   static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr = "",
@@ -1587,8 +1596,9 @@ class CallInst : public CallBase {
   /// The returned call instruction is identical \p CI in every way except that
   /// the operand bundles for the new instruction are set to the operand bundles
   /// in \p Bundles.
-  static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles,
-                          InsertPosition InsertPt = nullptr);
+  LLVM_ABI static CallInst *Create(CallInst *CI,
+                                   ArrayRef<OperandBundleDef> Bundles,
+                                   InsertPosition InsertPt = nullptr);
 
   // Note that 'musttail' implies 'tail'.
   enum TailCallKind : unsigned {
@@ -1649,7 +1659,7 @@ class CallInst : public CallBase {
   }
 
   /// Updates profile metadata by scaling it by \p S / \p T.
-  void updateProfWeight(uint64_t S, uint64_t T);
+  LLVM_ABI void updateProfWeight(uint64_t S, uint64_t T);
 
 private:
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
@@ -1698,7 +1708,7 @@ class SelectInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  SelectInst *cloneImpl() const;
+  LLVM_ABI SelectInst *cloneImpl() const;
 
 public:
   static SelectInst *Create(Value *C, Value *S1, Value *S2,
@@ -1729,7 +1739,8 @@ class SelectInst : public Instruction {
 
   /// Return a string if the specified operands are invalid
   /// for a select operation, otherwise return null.
-  static const char *areInvalidOperands(Value *Cond, Value *True, Value *False);
+  LLVM_ABI static const char *areInvalidOperands(Value *Cond, Value *True,
+                                                 Value *False);
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -1765,7 +1776,7 @@ class VAArgInst : public UnaryInstruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  VAArgInst *cloneImpl() const;
+  LLVM_ABI VAArgInst *cloneImpl() const;
 
 public:
   VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
@@ -1797,14 +1808,14 @@ class VAArgInst : public UnaryInstruction {
 class ExtractElementInst : public Instruction {
   constexpr static IntrusiveOperandsAllocMarker AllocMarker{2};
 
-  ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
-                     InsertPosition InsertBefore = nullptr);
+  LLVM_ABI ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
+                              InsertPosition InsertBefore = nullptr);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  ExtractElementInst *cloneImpl() const;
+  LLVM_ABI ExtractElementInst *cloneImpl() const;
 
 public:
   static ExtractElementInst *Create(Value *Vec, Value *Idx,
@@ -1816,7 +1827,7 @@ class ExtractElementInst : public Instruction {
 
   /// Return true if an extractelement instruction can be
   /// formed with the specified operands.
-  static bool isValidOperands(const Value *Vec, const Value *Idx);
+  LLVM_ABI static bool isValidOperands(const Value *Vec, const Value *Idx);
 
   Value *getVectorOperand() { return Op<0>(); }
   Value *getIndexOperand() { return Op<1>(); }
@@ -1856,15 +1867,15 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementInst, Value)
 class InsertElementInst : public Instruction {
   constexpr static IntrusiveOperandsAllocMarker AllocMarker{3};
 
-  InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
-                    const Twine &NameStr = "",
-                    InsertPosition InsertBefore = nullptr);
+  LLVM_ABI InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
+                             const Twine &NameStr = "",
+                             InsertPosition InsertBefore = nullptr);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  InsertElementInst *cloneImpl() const;
+  LLVM_ABI InsertElementInst *cloneImpl() const;
 
 public:
   static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
@@ -1876,8 +1887,8 @@ class InsertElementInst : public Instruction {
 
   /// Return true if an insertelement instruction can be
   /// formed with the specified operands.
-  static bool isValidOperands(const Value *Vec, const Value *NewElt,
-                              const Value *Idx);
+  LLVM_ABI static bool isValidOperands(const Value *Vec, const Value *NewElt,
+                                       const Value *Idx);
 
   /// Overload to return most specific vector type.
   ///
@@ -1930,33 +1941,34 @@ class ShuffleVectorInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  ShuffleVectorInst *cloneImpl() const;
+  LLVM_ABI ShuffleVectorInst *cloneImpl() const;
 
 public:
-  ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr = "",
-                    InsertPosition InsertBefore = nullptr);
-  ShuffleVectorInst(Value *V1, ArrayRef<int> Mask, const Twine &NameStr = "",
-                    InsertPosition InsertBefore = nullptr);
-  ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
-                    const Twine &NameStr = "",
-                    InsertPosition InsertBefore = nullptr);
-  ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
-                    const Twine &NameStr = "",
-                    InsertPosition InsertBefore = nullptr);
+  LLVM_ABI ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr = "",
+                             InsertPosition InsertBefore = nullptr);
+  LLVM_ABI ShuffleVectorInst(Value *V1, ArrayRef<int> Mask,
+                             const Twine &NameStr = "",
+                             InsertPosition InsertBefore = nullptr);
+  LLVM_ABI ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
+                             const Twine &NameStr = "",
+                             InsertPosition InsertBefore = nullptr);
+  LLVM_ABI ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+                             const Twine &NameStr = "",
+                             InsertPosition InsertBefore = nullptr);
 
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
   void operator delete(void *Ptr) { return User::operator delete(Ptr); }
 
   /// Swap the operands and adjust the mask to preserve the semantics
   /// of the instruction.
-  void commute();
+  LLVM_ABI void commute();
 
   /// Return true if a shufflevector instruction can be
   /// formed with the specified operands.
-  static bool isValidOperands(const Value *V1, const Value *V2,
-                              const Value *Mask);
-  static bool isValidOperands(const Value *V1, const Value *V2,
-                              ArrayRef<int> Mask);
+  LLVM_ABI static bool isValidOperands(const Value *V1, const Value *V2,
+                                       const Value *Mask);
+  LLVM_ABI static bool isValidOperands(const Value *V1, const Value *V2,
+                                       ArrayRef<int> Mask);
 
   /// Overload to return most specific vector type.
   ///
@@ -1973,8 +1985,8 @@ class ShuffleVectorInst : public Instruction {
 
   /// Convert the input shuffle mask operand to a vector of integers. Undefined
   /// elements of the mask are returned as PoisonMaskElem.
-  static void getShuffleMask(const Constant *Mask,
-                             SmallVectorImpl<int> &Result);
+  LLVM_ABI static void getShuffleMask(const Constant *Mask,
+                                      SmallVectorImpl<int> &Result);
 
   /// Return the mask for this instruction as a vector of integers. Undefined
   /// elements of the mask are returned as PoisonMaskElem.
@@ -1988,10 +2000,10 @@ class ShuffleVectorInst : public Instruction {
   /// shufflevector.
   Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
 
-  static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
-                                                Type *ResultTy);
+  LLVM_ABI static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+                                                         Type *ResultTy);
 
-  void setShuffleMask(ArrayRef<int> Mask);
+  LLVM_ABI void setShuffleMask(ArrayRef<int> Mask);
 
   ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
 
@@ -2023,7 +2035,7 @@ class ShuffleVectorInst : public Instruction {
   /// Example: <7,5,undef,7>
   /// This assumes that vector operands (of length \p NumSrcElts) are the same
   /// length as the mask.
-  static bool isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts);
+  LLVM_ABI static bool isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts);
   static bool isSingleSourceMask(const Constant *Mask, int NumSrcElts) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
     SmallVector<int, 16> MaskAsInts;
@@ -2045,7 +2057,7 @@ class ShuffleVectorInst : public Instruction {
   /// necessarily a no-op because it may change the number of elements from its
   /// input vectors or it may provide demanded bits knowledge via undef lanes.
   /// Example: <undef,undef,2,3>
-  static bool isIdentityMask(ArrayRef<int> Mask, int NumSrcElts);
+  LLVM_ABI static bool isIdentityMask(ArrayRef<int> Mask, int NumSrcElts);
   static bool isIdentityMask(const Constant *Mask, int NumSrcElts) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
 
@@ -2074,16 +2086,16 @@ class ShuffleVectorInst : public Instruction {
 
   /// Return true if this shuffle lengthens exactly one source vector with
   /// undefs in the high elements.
-  bool isIdentityWithPadding() const;
+  LLVM_ABI bool isIdentityWithPadding() const;
 
   /// Return true if this shuffle extracts the first N elements of exactly one
   /// source vector.
-  bool isIdentityWithExtract() const;
+  LLVM_ABI bool isIdentityWithExtract() const;
 
   /// Return true if this shuffle concatenates its 2 source vectors. This
   /// returns false if either input is undefined. In that case, the shuffle is
   /// is better classified as an identity with padding operation.
-  bool isConcat() const;
+  LLVM_ABI bool isConcat() const;
 
   /// Return true if this shuffle mask chooses elements from its source vectors
   /// without lane crossings. A shuffle using this mask would be
@@ -2093,7 +2105,7 @@ class ShuffleVectorInst : public Instruction {
   /// In that case, the shuffle is better classified as an identity shuffle.
   /// This assumes that vector operands are the same length as the mask
   /// (a length-changing shuffle can never be equivalent to a vector select).
-  static bool isSelectMask(ArrayRef<int> Mask, int NumSrcElts);
+  LLVM_ABI static bool isSelectMask(ArrayRef<int> Mask, int NumSrcElts);
   static bool isSelectMask(const Constant *Mask, int NumSrcElts) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
     SmallVector<int, 16> MaskAsInts;
@@ -2118,7 +2130,7 @@ class ShuffleVectorInst : public Instruction {
   /// Example: <7,6,undef,4>
   /// This assumes that vector operands (of length \p NumSrcElts) are the same
   /// length as the mask.
-  static bool isReverseMask(ArrayRef<int> Mask, int NumSrcElts);
+  LLVM_ABI static bool isReverseMask(ArrayRef<int> Mask, int NumSrcElts);
   static bool isReverseMask(const Constant *Mask, int NumSrcElts) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
     SmallVector<int, 16> MaskAsInts;
@@ -2139,7 +2151,7 @@ class ShuffleVectorInst : public Instruction {
   /// Example: <4,undef,undef,4>
   /// This assumes that vector operands (of length \p NumSrcElts) are the same
   /// length as the mask.
-  static bool isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts);
+  LLVM_ABI static bool isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts);
   static bool isZeroEltSplatMask(const Constant *Mask, int NumSrcElts) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
     SmallVector<int, 16> MaskAsInts;
@@ -2190,7 +2202,7 @@ class ShuffleVectorInst : public Instruction {
   ///   ; Transposed matrix
   ///   t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
   ///   t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
-  static bool isTransposeMask(ArrayRef<int> Mask, int NumSrcElts);
+  LLVM_ABI static bool isTransposeMask(ArrayRef<int> Mask, int NumSrcElts);
   static bool isTransposeMask(const Constant *Mask, int NumSrcElts) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
     SmallVector<int, 16> MaskAsInts;
@@ -2213,7 +2225,8 @@ class ShuffleVectorInst : public Instruction {
   /// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
   /// This assumes that vector operands (of length \p NumSrcElts) are the same
   /// length as the mask.
-  static bool isSpliceMask(ArrayRef<int> Mask, int NumSrcElts, int &Index);
+  LLVM_ABI static bool isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,
+                                    int &Index);
   static bool isSpliceMask(const Constant *Mask, int NumSrcElts, int &Index) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
     SmallVector<int, 16> MaskAsInts;
@@ -2233,8 +2246,8 @@ class ShuffleVectorInst : public Instruction {
   /// Return true if this shuffle mask is an extract subvector mask.
   /// A valid extract subvector mask returns a smaller vector from a single
   /// source operand. The base extraction index is returned as well.
-  static bool isExtractSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
-                                     int &Index);
+  LLVM_ABI static bool isExtractSubvectorMask(ArrayRef<int> Mask,
+                                              int NumSrcElts, int &Index);
   static bool isExtractSubvectorMask(const Constant *Mask, int NumSrcElts,
                                      int &Index) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
@@ -2263,8 +2276,8 @@ class ShuffleVectorInst : public Instruction {
   /// A valid insert subvector mask inserts the lowest elements of a second
   /// source operand into an in-place first source operand.
   /// Both the sub vector width and the insertion index is returned.
-  static bool isInsertSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
-                                    int &NumSubElts, int &Index);
+  LLVM_ABI static bool isInsertSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
+                                             int &NumSubElts, int &Index);
   static bool isInsertSubvectorMask(const Constant *Mask, int NumSrcElts,
                                     int &NumSubElts, int &Index) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
@@ -2293,8 +2306,8 @@ class ShuffleVectorInst : public Instruction {
   /// in a vector \p ReplicationFactor times.
   /// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is:
   ///   <0,0,0,1,1,1,2,2,2,3,3,3>
-  static bool isReplicationMask(ArrayRef<int> Mask, int &ReplicationFactor,
-                                int &VF);
+  LLVM_ABI static bool isReplicationMask(ArrayRef<int> Mask,
+                                         int &ReplicationFactor, int &VF);
   static bool isReplicationMask(const Constant *Mask, int &ReplicationFactor,
                                 int &VF) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
@@ -2308,7 +2321,7 @@ class ShuffleVectorInst : public Instruction {
   }
 
   /// Return true if this shuffle mask is a replication mask.
-  bool isReplicationMask(int &ReplicationFactor, int &VF) const;
+  LLVM_ABI bool isReplicationMask(int &ReplicationFactor, int &VF) const;
 
   /// Return true if this shuffle mask represents "clustered" mask of size VF,
   /// i.e. each index between [0..VF) is used exactly once in each submask of
@@ -2319,11 +2332,11 @@ class ShuffleVectorInst : public Instruction {
   /// 0, 1, 2, 3, 3, 3, 1, 0 - not "clustered", because
   ///                          element 3 is used twice in the second submask
   ///                          (3,3,1,0) and index 2 is not used at all.
-  static bool isOneUseSingleSourceMask(ArrayRef<int> Mask, int VF);
+  LLVM_ABI static bool isOneUseSingleSourceMask(ArrayRef<int> Mask, int VF);
 
   /// Return true if this shuffle mask is a one-use-single-source("clustered")
   /// mask.
-  bool isOneUseSingleSourceMask(int VF) const;
+  LLVM_ABI bool isOneUseSingleSourceMask(int VF) const;
 
   /// Change values in a shuffle permute mask assuming the two vector operands
   /// of length InVecNumElts have swapped position.
@@ -2339,7 +2352,7 @@ class ShuffleVectorInst : public Instruction {
   }
 
   /// Return if this shuffle interleaves its two input vectors together.
-  bool isInterleave(unsigned Factor);
+  LLVM_ABI bool isInterleave(unsigned Factor);
 
   /// Return true if the mask interleaves one or more input vectors together.
   ///
@@ -2360,9 +2373,9 @@ class ShuffleVectorInst : public Instruction {
   /// Note that this does not check if the input vectors are consecutive:
   /// It will return true for masks such as
   /// <0, 4, 6, 1, 5, 7> (Factor=3, LaneLen=2)
-  static bool isInterleaveMask(ArrayRef<int> Mask, unsigned Factor,
-                               unsigned NumInputElts,
-                               SmallVectorImpl<unsigned> &StartIndexes);
+  LLVM_ABI static bool
+  isInterleaveMask(ArrayRef<int> Mask, unsigned Factor, unsigned NumInputElts,
+                   SmallVectorImpl<unsigned> &StartIndexes);
   static bool isInterleaveMask(ArrayRef<int> Mask, unsigned Factor,
                                unsigned NumInputElts) {
     SmallVector<unsigned, 8> StartIndexes;
@@ -2372,8 +2385,9 @@ class ShuffleVectorInst : public Instruction {
   /// Check if the mask is a DE-interleave mask of the given factor
   /// \p Factor like:
   ///     <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>
-  static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask, unsigned Factor,
-                                         unsigned &Index);
+  LLVM_ABI static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask,
+                                                  unsigned Factor,
+                                                  unsigned &Index);
   static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask, unsigned Factor) {
     unsigned Unused;
     return isDeInterleaveMaskOfFactor(Mask, Factor, Unused);
@@ -2390,9 +2404,11 @@ class ShuffleVectorInst : public Instruction {
   ///
   /// If it can be expressed as a rotation, returns the number of subelements to
   /// group by in NumSubElts and the number of bits to rotate left in RotateAmt.
-  static bool isBitRotateMask(ArrayRef<int> Mask, unsigned EltSizeInBits,
-                              unsigned MinSubElts, unsigned MaxSubElts,
-                              unsigned &NumSubElts, unsigned &RotateAmt);
+  LLVM_ABI static bool isBitRotateMask(ArrayRef<int> Mask,
+                                       unsigned EltSizeInBits,
+                                       unsigned MinSubElts, unsigned MaxSubElts,
+                                       unsigned &NumSubElts,
+                                       unsigned &RotateAmt);
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
@@ -2428,13 +2444,13 @@ class ExtractValueInst : public UnaryInstruction {
   inline ExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,
                           const Twine &NameStr, InsertPosition InsertBefore);
 
-  void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
+  LLVM_ABI void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  ExtractValueInst *cloneImpl() const;
+  LLVM_ABI ExtractValueInst *cloneImpl() const;
 
 public:
   static ExtractValueInst *Create(Value *Agg, ArrayRef<unsigned> Idxs,
@@ -2448,7 +2464,7 @@ class ExtractValueInst : public UnaryInstruction {
   /// with an extractvalue instruction with the specified parameters.
   ///
   /// Null is returned if the indices are invalid for the specified type.
-  static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
+  LLVM_ABI static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
 
   using idx_iterator = const unsigned*;
 
@@ -2524,14 +2540,14 @@ class InsertValueInst : public Instruction {
                   const Twine &NameStr = "",
                   InsertPosition InsertBefore = nullptr);
 
-  void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
-            const Twine &NameStr);
+  LLVM_ABI void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
+                     const Twine &NameStr);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  InsertValueInst *cloneImpl() const;
+  LLVM_ABI InsertValueInst *cloneImpl() const;
 
 public:
   // allocate space for exactly two operands
@@ -2642,7 +2658,7 @@ class PHINode : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  PHINode *cloneImpl() const;
+  LLVM_ABI PHINode *cloneImpl() const;
 
   // allocHungoffUses - this is more complicated than the generic
   // User::allocHungoffUses, because we have to allocate Uses for the incoming
@@ -2771,7 +2787,8 @@ class PHINode : public Instruction {
   /// dummy values.  The only time there should be zero incoming values to a PHI
   /// node is when the block is dead, so this strategy is sound.
   ///
-  Value *removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty = true);
+  LLVM_ABI Value *removeIncomingValue(unsigned Idx,
+                                      bool DeletePHIIfEmpty = true);
 
   Value *removeIncomingValue(const BasicBlock *BB, bool DeletePHIIfEmpty=true) {
     int Idx = getBasicBlockIndex(BB);
@@ -2781,8 +2798,8 @@ class PHINode : public Instruction {
 
   /// Remove all incoming values for which the predicate returns true.
   /// The predicate accepts the incoming value index.
-  void removeIncomingValueIf(function_ref<bool(unsigned)> Predicate,
-                             bool DeletePHIIfEmpty = true);
+  LLVM_ABI void removeIncomingValueIf(function_ref<bool(unsigned)> Predicate,
+                                      bool DeletePHIIfEmpty = true);
 
   /// Return the first index of the specified basic
   /// block in the value list for this PHI.  Returns -1 if no instance.
@@ -2815,12 +2832,12 @@ class PHINode : public Instruction {
 
   /// If the specified PHI node always merges together the
   /// same value, return the value, otherwise return null.
-  Value *hasConstantValue() const;
+  LLVM_ABI Value *hasConstantValue() const;
 
   /// Whether the specified PHI node always merges
   /// together the same value, assuming undefs are equal to a unique
   /// non-undef value.
-  bool hasConstantOrUndefValue() const;
+  LLVM_ABI bool hasConstantOrUndefValue() const;
 
   /// If the PHI node is complete which means all of its parent's predecessors
   /// have incoming value in this PHI, return true, otherwise return false.
@@ -2840,7 +2857,7 @@ class PHINode : public Instruction {
   }
 
 private:
-  void growOperands();
+  LLVM_ABI void growOperands();
 };
 
 template <> struct OperandTraits<PHINode> : public HungoffOperandTraits {};
@@ -2880,23 +2897,24 @@ class LandingPadInst : public Instruction {
   // Allocate space for exactly zero operands.
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
 
-  void growOperands(unsigned Size);
+  LLVM_ABI void growOperands(unsigned Size);
   void init(unsigned NumReservedValues, const Twine &NameStr);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  LandingPadInst *cloneImpl() const;
+  LLVM_ABI LandingPadInst *cloneImpl() const;
 
 public:
   void operator delete(void *Ptr) { User::operator delete(Ptr); }
 
   /// Constructors - NumReservedClauses is a hint for the number of incoming
   /// clauses that this landingpad will have (use 0 if you really have no idea).
-  static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
-                                const Twine &NameStr = "",
-                                InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static LandingPadInst *Create(Type *RetTy,
+                                         unsigned NumReservedClauses,
+                                         const Twine &NameStr = "",
+                                         InsertPosition InsertBefore = nullptr);
 
   /// Provide fast operand accessors
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -2910,7 +2928,7 @@ class LandingPadInst : public Instruction {
   void setCleanup(bool V) { setSubclassData<CleanupField>(V); }
 
   /// Add a catch or filter clause to the landing pad.
-  void addClause(Constant *ClauseVal);
+  LLVM_ABI void addClause(Constant *ClauseVal);
 
   /// Get the value of the clause at index Idx. Use isCatch/isFilter to
   /// determine what type of clause this is.
@@ -2974,14 +2992,15 @@ class ReturnInst : public Instruction {
   //
   // NOTE: If the Value* passed is of type void then the constructor behaves as
   // if it was passed NULL.
-  explicit ReturnInst(LLVMContext &C, Value *retVal, AllocInfo AllocInfo,
-                      InsertPosition InsertBefore);
+  LLVM_ABI explicit ReturnInst(LLVMContext &C, Value *retVal,
+                               AllocInfo AllocInfo,
+                               InsertPosition InsertBefore);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  ReturnInst *cloneImpl() const;
+  LLVM_ABI ReturnInst *cloneImpl() const;
 
 public:
   static ReturnInst *Create(LLVMContext &C, Value *retVal = nullptr,
@@ -3050,10 +3069,10 @@ class BranchInst : public Instruction {
   // BranchInst(BB* T, BB *F, Value *C, Inst *I) - 'br C, T, F', insert before I
   // BranchInst(BB* B, BB *I)                    - 'br B'        insert at end
   // BranchInst(BB* T, BB *F, Value *C, BB *I)   - 'br C, T, F', insert at end
-  explicit BranchInst(BasicBlock *IfTrue, AllocInfo AllocInfo,
-                      InsertPosition InsertBefore);
-  BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
-             AllocInfo AllocInfo, InsertPosition InsertBefore);
+  LLVM_ABI explicit BranchInst(BasicBlock *IfTrue, AllocInfo AllocInfo,
+                               InsertPosition InsertBefore);
+  LLVM_ABI BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
+                      AllocInfo AllocInfo, InsertPosition InsertBefore);
 
   void AssertOK();
 
@@ -3061,7 +3080,7 @@ class BranchInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  BranchInst *cloneImpl() const;
+  LLVM_ABI BranchInst *cloneImpl() const;
 
 public:
   /// Iterator type that casts an operand to a basic block.
@@ -3138,7 +3157,7 @@ class BranchInst : public Instruction {
   /// Swaps the successors of the branch instruction. This also swaps any
   /// branch weight metadata associated with the instruction so that it
   /// continues to map correctly to each operand.
-  void swapSuccessors();
+  LLVM_ABI void swapSuccessors();
 
   iterator_range<succ_op_iterator> successors() {
     return make_range(
@@ -3188,8 +3207,8 @@ class SwitchInst : public Instruction {
   /// default destination. The number of additional cases can be specified here
   /// to make memory allocation more efficient. This constructor can also
   /// auto-insert before another instruction.
-  SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
-             InsertPosition InsertBefore);
+  LLVM_ABI SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
+                      InsertPosition InsertBefore);
 
   // allocate space for exactly zero operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -3201,7 +3220,7 @@ class SwitchInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  SwitchInst *cloneImpl() const;
+  LLVM_ABI SwitchInst *cloneImpl() const;
 
 public:
   void operator delete(void *Ptr) { User::operator delete(Ptr); }
@@ -3479,7 +3498,7 @@ class SwitchInst : public Instruction {
   /// Note:
   /// This action invalidates case_end(). Old case_end() iterator will
   /// point to the added case.
-  void addCase(ConstantInt *OnVal, BasicBlock *Dest);
+  LLVM_ABI void addCase(ConstantInt *OnVal, BasicBlock *Dest);
 
   /// This method removes the specified case and its successor from the switch
   /// instruction. Note that this operation may reorder the remaining cases at
@@ -3488,7 +3507,7 @@ class SwitchInst : public Instruction {
   /// This action invalidates iterators for all cases following the one removed,
   /// including the case_end() iterator. It returns an iterator for the next
   /// case.
-  CaseIt removeCase(CaseIt I);
+  LLVM_ABI CaseIt removeCase(CaseIt I);
 
   unsigned getNumSuccessors() const { return getNumOperands()/2; }
   BasicBlock *getSuccessor(unsigned idx) const {
@@ -3517,9 +3536,9 @@ class SwitchInstProfUpdateWrapper {
   bool Changed = false;
 
 protected:
-  MDNode *buildProfBranchWeightsMD();
+  LLVM_ABI MDNode *buildProfBranchWeightsMD();
 
-  void init();
+  LLVM_ABI void init();
 
 public:
   using CaseWeightOpt = std::optional<uint32_t>;
@@ -3536,20 +3555,21 @@ class SwitchInstProfUpdateWrapper {
 
   /// Delegate the call to the underlying SwitchInst::removeCase() and remove
   /// correspondent branch weight.
-  SwitchInst::CaseIt removeCase(SwitchInst::CaseIt I);
+  LLVM_ABI SwitchInst::CaseIt removeCase(SwitchInst::CaseIt I);
 
   /// Delegate the call to the underlying SwitchInst::addCase() and set the
   /// specified branch weight for the added case.
-  void addCase(ConstantInt *OnVal, BasicBlock *Dest, CaseWeightOpt W);
+  LLVM_ABI void addCase(ConstantInt *OnVal, BasicBlock *Dest, CaseWeightOpt W);
 
   /// Delegate the call to the underlying SwitchInst::eraseFromParent() and mark
   /// this object to not touch the underlying SwitchInst in destructor.
-  Instruction::InstListType::iterator eraseFromParent();
+  LLVM_ABI Instruction::InstListType::iterator eraseFromParent();
 
-  void setSuccessorWeight(unsigned idx, CaseWeightOpt W);
-  CaseWeightOpt getSuccessorWeight(unsigned idx);
+  LLVM_ABI void setSuccessorWeight(unsigned idx, CaseWeightOpt W);
+  LLVM_ABI CaseWeightOpt getSuccessorWeight(unsigned idx);
 
-  static CaseWeightOpt getSuccessorWeight(const SwitchInst &SI, unsigned idx);
+  LLVM_ABI static CaseWeightOpt getSuccessorWeight(const SwitchInst &SI,
+                                                   unsigned idx);
 };
 
 template <> struct OperandTraits<SwitchInst> : public HungoffOperandTraits {};
@@ -3576,8 +3596,8 @@ class IndirectBrInst : public Instruction {
   /// Address to jump to.  The number of expected destinations can be specified
   /// here to make memory allocation more efficient.  This constructor can also
   /// autoinsert before another instruction.
-  IndirectBrInst(Value *Address, unsigned NumDests,
-                 InsertPosition InsertBefore);
+  LLVM_ABI IndirectBrInst(Value *Address, unsigned NumDests,
+                          InsertPosition InsertBefore);
 
   // allocate space for exactly zero operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -3589,7 +3609,7 @@ class IndirectBrInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  IndirectBrInst *cloneImpl() const;
+  LLVM_ABI IndirectBrInst *cloneImpl() const;
 
 public:
   void operator delete(void *Ptr) { User::operator delete(Ptr); }
@@ -3644,11 +3664,11 @@ class IndirectBrInst : public Instruction {
 
   /// Add a destination.
   ///
-  void addDestination(BasicBlock *Dest);
+  LLVM_ABI void addDestination(BasicBlock *Dest);
 
   /// This method removes the specified successor from the
   /// indirectbr instruction.
-  void removeDestination(unsigned i);
+  LLVM_ABI void removeDestination(unsigned i);
 
   unsigned getNumSuccessors() const { return getNumOperands()-1; }
   BasicBlock *getSuccessor(unsigned i) const {
@@ -3710,9 +3730,9 @@ class InvokeInst : public CallBase {
                     ArrayRef<OperandBundleDef> Bundles, AllocInfo AllocInfo,
                     const Twine &NameStr, InsertPosition InsertBefore);
 
-  void init(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
-            BasicBlock *IfException, ArrayRef<Value *> Args,
-            ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
+  LLVM_ABI void init(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
+                     BasicBlock *IfException, ArrayRef<Value *> Args,
+                     ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
 
   /// Compute the number of operands to allocate.
   static unsigned ComputeNumOperands(unsigned NumArgs,
@@ -3726,7 +3746,7 @@ class InvokeInst : public CallBase {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  InvokeInst *cloneImpl() const;
+  LLVM_ABI InvokeInst *cloneImpl() const;
 
 public:
   static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
@@ -3776,8 +3796,9 @@ class InvokeInst : public CallBase {
   /// The returned invoke instruction is identical to \p II in every way except
   /// that the operand bundles for the new instruction are set to the operand
   /// bundles in \p Bundles.
-  static InvokeInst *Create(InvokeInst *II, ArrayRef<OperandBundleDef> Bundles,
-                            InsertPosition InsertPt = nullptr);
+  LLVM_ABI static InvokeInst *Create(InvokeInst *II,
+                                     ArrayRef<OperandBundleDef> Bundles,
+                                     InsertPosition InsertPt = nullptr);
 
   // get*Dest - Return the destination basic blocks...
   BasicBlock *getNormalDest() const {
@@ -3795,7 +3816,7 @@ class InvokeInst : public CallBase {
 
   /// Get the landingpad instruction from the landing pad
   /// block (the unwind destination).
-  LandingPadInst *getLandingPadInst() const;
+  LLVM_ABI LandingPadInst *getLandingPadInst() const;
 
   BasicBlock *getSuccessor(unsigned i) const {
     assert(i < 2 && "Successor # out of range for invoke!");
@@ -3813,7 +3834,7 @@ class InvokeInst : public CallBase {
   unsigned getNumSuccessors() const { return 2; }
 
   /// Updates profile metadata by scaling it by \p S / \p T.
-  void updateProfWeight(uint64_t S, uint64_t T);
+  LLVM_ABI void updateProfWeight(uint64_t S, uint64_t T);
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
@@ -3864,9 +3885,10 @@ class CallBrInst : public CallBase {
                     AllocInfo AllocInfo, const Twine &NameStr,
                     InsertPosition InsertBefore);
 
-  void init(FunctionType *FTy, Value *Func, BasicBlock *DefaultDest,
-            ArrayRef<BasicBlock *> IndirectDests, ArrayRef<Value *> Args,
-            ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
+  LLVM_ABI void init(FunctionType *FTy, Value *Func, BasicBlock *DefaultDest,
+                     ArrayRef<BasicBlock *> IndirectDests,
+                     ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles,
+                     const Twine &NameStr);
 
   /// Compute the number of operands to allocate.
   static unsigned ComputeNumOperands(int NumArgs, int NumIndirectDests,
@@ -3880,7 +3902,7 @@ class CallBrInst : public CallBase {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  CallBrInst *cloneImpl() const;
+  LLVM_ABI CallBrInst *cloneImpl() const;
 
 public:
   static CallBrInst *Create(FunctionType *Ty, Value *Func,
@@ -3934,8 +3956,9 @@ class CallBrInst : public CallBase {
   /// The returned callbr instruction is identical to \p CBI in every way
   /// except that the operand bundles for the new instruction are set to the
   /// operand bundles in \p Bundles.
-  static CallBrInst *Create(CallBrInst *CBI, ArrayRef<OperandBundleDef> Bundles,
-                            InsertPosition InsertBefore = nullptr);
+  LLVM_ABI static CallBrInst *Create(CallBrInst *CBI,
+                                     ArrayRef<OperandBundleDef> Bundles,
+                                     InsertPosition InsertBefore = nullptr);
 
   /// Return the number of callbr indirect dest labels.
   ///
@@ -4026,13 +4049,14 @@ class ResumeInst : public Instruction {
 
   ResumeInst(const ResumeInst &RI);
 
-  explicit ResumeInst(Value *Exn, InsertPosition InsertBefore = nullptr);
+  LLVM_ABI explicit ResumeInst(Value *Exn,
+                               InsertPosition InsertBefore = nullptr);
 
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  ResumeInst *cloneImpl() const;
+  LLVM_ABI ResumeInst *cloneImpl() const;
 
 public:
   static ResumeInst *Create(Value *Exn, InsertPosition InsertBefore = nullptr) {
@@ -4093,9 +4117,9 @@ class CatchSwitchInst : public Instruction {
   /// default destination.  The number of additional handlers can be specified
   /// here to make memory allocation more efficient.
   /// This constructor can also autoinsert before another instruction.
-  CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
-                  unsigned NumHandlers, const Twine &NameStr,
-                  InsertPosition InsertBefore);
+  LLVM_ABI CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
+                           unsigned NumHandlers, const Twine &NameStr,
+                           InsertPosition InsertBefore);
 
   // allocate space for exactly zero operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -4107,7 +4131,7 @@ class CatchSwitchInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  CatchSwitchInst *cloneImpl() const;
+  LLVM_ABI CatchSwitchInst *cloneImpl() const;
 
 public:
   void operator delete(void *Ptr) { return User::operator delete(Ptr); }
@@ -4207,9 +4231,9 @@ class CatchSwitchInst : public Instruction {
   /// Note:
   /// This action invalidates handler_end(). Old handler_end() iterator will
   /// point to the added handler.
-  void addHandler(BasicBlock *Dest);
+  LLVM_ABI void addHandler(BasicBlock *Dest);
 
-  void removeHandler(handler_iterator HI);
+  LLVM_ABI void removeHandler(handler_iterator HI);
 
   unsigned getNumSuccessors() const { return getNumOperands() - 1; }
   BasicBlock *getSuccessor(unsigned Idx) const {
@@ -4312,7 +4336,8 @@ class CatchReturnInst : public Instruction {
   constexpr static IntrusiveOperandsAllocMarker AllocMarker{2};
 
   CatchReturnInst(const CatchReturnInst &RI);
-  CatchReturnInst(Value *CatchPad, BasicBlock *BB, InsertPosition InsertBefore);
+  LLVM_ABI CatchReturnInst(Value *CatchPad, BasicBlock *BB,
+                           InsertPosition InsertBefore);
 
   void init(Value *CatchPad, BasicBlock *BB);
 
@@ -4320,7 +4345,7 @@ class CatchReturnInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  CatchReturnInst *cloneImpl() const;
+  LLVM_ABI CatchReturnInst *cloneImpl() const;
 
 public:
   static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB,
@@ -4388,8 +4413,9 @@ class CleanupReturnInst : public Instruction {
 
 private:
   CleanupReturnInst(const CleanupReturnInst &RI, AllocInfo AllocInfo);
-  CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB,
-                    AllocInfo AllocInfo, InsertPosition InsertBefore = nullptr);
+  LLVM_ABI CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB,
+                             AllocInfo AllocInfo,
+                             InsertPosition InsertBefore = nullptr);
 
   void init(Value *CleanupPad, BasicBlock *UnwindBB);
 
@@ -4397,7 +4423,7 @@ class CleanupReturnInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  CleanupReturnInst *cloneImpl() const;
+  LLVM_ABI CleanupReturnInst *cloneImpl() const;
 
 public:
   static CleanupReturnInst *Create(Value *CleanupPad,
@@ -4487,11 +4513,11 @@ class UnreachableInst : public Instruction {
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
 
-  UnreachableInst *cloneImpl() const;
+  LLVM_ABI UnreachableInst *cloneImpl() const;
 
 public:
-  explicit UnreachableInst(LLVMContext &C,
-                           InsertPosition InsertBefore = nullptr);
+  LLVM_ABI explicit UnreachableInst(LLVMContext &C,
+                                    InsertPosition InsertBefore = nullptr);
 
   // allocate space for exactly zero operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
@@ -4508,7 +4534,8 @@ class UnreachableInst : public Instruction {
   }
 
   // Whether to do target lowering in SelectionDAG.
-  bool shouldLowerToTrap(bool TrapUnreachable, bool NoTrapAfterNoreturn) const;
+  LLVM_ABI bool shouldLowerToTrap(bool TrapUnreachable,
+                                  bool NoTrapAfterNoreturn) const;
 
 private:
   BasicBlock *getSuccessor(unsigned idx) const {
@@ -4531,12 +4558,13 @@ class TruncInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical TruncInst
-  TruncInst *cloneImpl() const;
+  LLVM_ABI TruncInst *cloneImpl() const;
 
 public:
   enum { AnyWrap = 0, NoUnsignedWrap = (1 << 0), NoSignedWrap = (1 << 1) };
 
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   TruncInst(Value *S,                  ///< The value to be truncated
             Type *Ty,                  ///< The (smaller) type to truncate to
             const Twine &NameStr = "", ///< A name for the new instruction
@@ -4597,10 +4625,11 @@ class ZExtInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical ZExtInst
-  ZExtInst *cloneImpl() const;
+  LLVM_ABI ZExtInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   ZExtInst(Value *S,                  ///< The value to be zero extended
            Type *Ty,                  ///< The type to zero extend to
            const Twine &NameStr = "", ///< A name for the new instruction
@@ -4628,10 +4657,11 @@ class SExtInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical SExtInst
-  SExtInst *cloneImpl() const;
+  LLVM_ABI SExtInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   SExtInst(Value *S,                  ///< The value to be sign extended
            Type *Ty,                  ///< The type to sign extend to
            const Twine &NameStr = "", ///< A name for the new instruction
@@ -4659,11 +4689,12 @@ class FPTruncInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical FPTruncInst
-  FPTruncInst *cloneImpl() const;
+  LLVM_ABI FPTruncInst *cloneImpl() const;
 
 public:                 /// Constructor with insert-before-instruction semantics
-  FPTruncInst(Value *S, ///< The value to be truncated
-              Type *Ty, ///< The type to truncate to
+  LLVM_ABI
+  FPTruncInst(Value *S,                  ///< The value to be truncated
+              Type *Ty,                  ///< The type to truncate to
               const Twine &NameStr = "", ///< A name for the new instruction
               InsertPosition InsertBefore =
                   nullptr ///< Where to insert the new instruction
@@ -4689,10 +4720,11 @@ class FPExtInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical FPExtInst
-  FPExtInst *cloneImpl() const;
+  LLVM_ABI FPExtInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   FPExtInst(Value *S,                  ///< The value to be extended
             Type *Ty,                  ///< The type to extend to
             const Twine &NameStr = "", ///< A name for the new instruction
@@ -4720,10 +4752,11 @@ class UIToFPInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical UIToFPInst
-  UIToFPInst *cloneImpl() const;
+  LLVM_ABI UIToFPInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   UIToFPInst(Value *S,                  ///< The value to be converted
              Type *Ty,                  ///< The type to convert to
              const Twine &NameStr = "", ///< A name for the new instruction
@@ -4751,10 +4784,11 @@ class SIToFPInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical SIToFPInst
-  SIToFPInst *cloneImpl() const;
+  LLVM_ABI SIToFPInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   SIToFPInst(Value *S,                  ///< The value to be converted
              Type *Ty,                  ///< The type to convert to
              const Twine &NameStr = "", ///< A name for the new instruction
@@ -4782,10 +4816,11 @@ class FPToUIInst  : public CastInst {
   friend class Instruction;
 
   /// Clone an identical FPToUIInst
-  FPToUIInst *cloneImpl() const;
+  LLVM_ABI FPToUIInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   FPToUIInst(Value *S,                  ///< The value to be converted
              Type *Ty,                  ///< The type to convert to
              const Twine &NameStr = "", ///< A name for the new instruction
@@ -4813,10 +4848,11 @@ class FPToSIInst  : public CastInst {
   friend class Instruction;
 
   /// Clone an identical FPToSIInst
-  FPToSIInst *cloneImpl() const;
+  LLVM_ABI FPToSIInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   FPToSIInst(Value *S,                  ///< The value to be converted
              Type *Ty,                  ///< The type to convert to
              const Twine &NameStr = "", ///< A name for the new instruction
@@ -4844,6 +4880,7 @@ class IntToPtrInst : public CastInst {
   friend class Instruction;
 
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   IntToPtrInst(Value *S,                  ///< The value to be converted
                Type *Ty,                  ///< The type to convert to
                const Twine &NameStr = "", ///< A name for the new instruction
@@ -4852,7 +4889,7 @@ class IntToPtrInst : public CastInst {
   );
 
   /// Clone an identical IntToPtrInst.
-  IntToPtrInst *cloneImpl() const;
+  LLVM_ABI IntToPtrInst *cloneImpl() const;
 
   /// Returns the address space of this instruction's pointer type.
   unsigned getAddressSpace() const {
@@ -4879,10 +4916,11 @@ class PtrToIntInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical PtrToIntInst.
-  PtrToIntInst *cloneImpl() const;
+  LLVM_ABI PtrToIntInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   PtrToIntInst(Value *S,                  ///< The value to be converted
                Type *Ty,                  ///< The type to convert to
                const Twine &NameStr = "", ///< A name for the new instruction
@@ -4922,10 +4960,11 @@ class BitCastInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical BitCastInst.
-  BitCastInst *cloneImpl() const;
+  LLVM_ABI BitCastInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
+  LLVM_ABI
   BitCastInst(Value *S,                  ///< The value to be casted
               Type *Ty,                  ///< The type to casted to
               const Twine &NameStr = "", ///< A name for the new instruction
@@ -4954,11 +4993,11 @@ class AddrSpaceCastInst : public CastInst {
   friend class Instruction;
 
   /// Clone an identical AddrSpaceCastInst.
-  AddrSpaceCastInst *cloneImpl() const;
+  LLVM_ABI AddrSpaceCastInst *cloneImpl() const;
 
 public:
   /// Constructor with insert-before-instruction semantics
-  AddrSpaceCastInst(
+  LLVM_ABI AddrSpaceCastInst(
       Value *S,                  ///< The value to be casted
       Type *Ty,                  ///< The type to casted to
       const Twine &NameStr = "", ///< A name for the new instruction
@@ -5116,11 +5155,11 @@ class FreezeInst : public UnaryInstruction {
   friend class Instruction;
 
   /// Clone an identical FreezeInst
-  FreezeInst *cloneImpl() const;
+  LLVM_ABI FreezeInst *cloneImpl() const;
 
 public:
-  explicit FreezeInst(Value *S, const Twine &NameStr = "",
-                      InsertPosition InsertBefore = nullptr);
+  LLVM_ABI explicit FreezeInst(Value *S, const Twine &NameStr = "",
+                               InsertPosition InsertBefore = nullptr);
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static inline bool classof(const Instruction *I) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h b/external/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h
index c4cb44256c8c..08046f00d174 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/external/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h
@@ -33,6 +33,7 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
 #include <cstdint>
@@ -128,7 +129,7 @@ class IntrinsicInst : public CallInst {
 
   /// Check if the intrinsic might lower into a regular function call in the
   /// course of IR transformations
-  static bool mayLowerToFunctionCall(Intrinsic::ID IID);
+  LLVM_ABI static bool mayLowerToFunctionCall(Intrinsic::ID IID);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const CallInst *I) {
@@ -261,8 +262,8 @@ class RawLocationWrapper {
   /// Get the locations corresponding to the variable referenced by the debug
   /// info intrinsic.  Depending on the intrinsic, this could be the
   /// variable's value or its address.
-  iterator_range<location_op_iterator> location_ops() const;
-  Value *getVariableLocationOp(unsigned OpIdx) const;
+  LLVM_ABI iterator_range<location_op_iterator> location_ops() const;
+  LLVM_ABI Value *getVariableLocationOp(unsigned OpIdx) const;
   unsigned getNumVariableLocationOps() const {
     if (hasArgList())
       return cast<DIArgList>(getRawLocation())->getArgs().size();
@@ -314,18 +315,18 @@ class DbgVariableIntrinsic : public DbgInfoIntrinsic {
   /// Get the locations corresponding to the variable referenced by the debug
   /// info intrinsic.  Depending on the intrinsic, this could be the
   /// variable's value or its address.
-  iterator_range<location_op_iterator> location_ops() const;
+  LLVM_ABI iterator_range<location_op_iterator> location_ops() const;
 
-  Value *getVariableLocationOp(unsigned OpIdx) const;
+  LLVM_ABI Value *getVariableLocationOp(unsigned OpIdx) const;
 
-  void replaceVariableLocationOp(Value *OldValue, Value *NewValue,
-                                 bool AllowEmpty = false);
-  void replaceVariableLocationOp(unsigned OpIdx, Value *NewValue);
+  LLVM_ABI void replaceVariableLocationOp(Value *OldValue, Value *NewValue,
+                                          bool AllowEmpty = false);
+  LLVM_ABI void replaceVariableLocationOp(unsigned OpIdx, Value *NewValue);
   /// Adding a new location operand will always result in this intrinsic using
   /// an ArgList, and must always be accompanied by a new expression that uses
   /// the new operand.
-  void addVariableLocationOps(ArrayRef<Value *> NewValues,
-                              DIExpression *NewExpr);
+  LLVM_ABI void addVariableLocationOps(ArrayRef<Value *> NewValues,
+                                       DIExpression *NewExpr);
 
   void setVariable(DILocalVariable *NewVar) {
     setArgOperand(1, MetadataAsValue::get(NewVar->getContext(), NewVar));
@@ -405,7 +406,7 @@ class DbgVariableIntrinsic : public DbgInfoIntrinsic {
 
   /// Get the size (in bits) of the variable, or fragment of the variable that
   /// is described.
-  std::optional<uint64_t> getFragmentSizeInBits() const;
+  LLVM_ABI std::optional<uint64_t> getFragmentSizeInBits() const;
 
   /// Get the FragmentInfo for the variable.
   std::optional<DIExpression::FragmentInfo> getFragment() const {
@@ -504,7 +505,7 @@ class DbgAssignIntrinsic : public DbgValueInst {
   };
 
 public:
-  Value *getAddress() const;
+  LLVM_ABI Value *getAddress() const;
   Metadata *getRawAddress() const {
     return cast<MetadataAsValue>(getArgOperand(OpAddress))->getMetadata();
   }
@@ -522,16 +523,16 @@ class DbgAssignIntrinsic : public DbgValueInst {
     setArgOperand(OpAddressExpr,
                   MetadataAsValue::get(NewExpr->getContext(), NewExpr));
   }
-  void setAssignId(DIAssignID *New);
-  void setAddress(Value *V);
+  LLVM_ABI void setAssignId(DIAssignID *New);
+  LLVM_ABI void setAddress(Value *V);
   /// Kill the address component.
-  void setKillAddress();
+  LLVM_ABI void setKillAddress();
   /// Check whether this kills the address component. This doesn't take into
   /// account the position of the intrinsic, therefore a returned value of false
   /// does not guarentee the address is a valid location for the variable at the
   /// intrinsic's position in IR.
-  bool isKillAddress() const;
-  void setValue(Value *V);
+  LLVM_ABI bool isKillAddress() const;
+  LLVM_ABI void setValue(Value *V);
   /// \name Casting methods
   /// @{
   static bool classof(const IntrinsicInst *I) {
@@ -630,51 +631,53 @@ class VPIntrinsic : public IntrinsicInst {
   /// \brief Declares a llvm.vp.* intrinsic in \p M that matches the parameters
   /// \p Params. Additionally, the load and gather intrinsics require
   /// \p ReturnType to be specified.
-  static Function *getOrInsertDeclarationForParams(Module *M, Intrinsic::ID,
-                                                   Type *ReturnType,
-                                                   ArrayRef<Value *> Params);
+  LLVM_ABI static Function *
+  getOrInsertDeclarationForParams(Module *M, Intrinsic::ID, Type *ReturnType,
+                                  ArrayRef<Value *> Params);
 
-  static std::optional<unsigned> getMaskParamPos(Intrinsic::ID IntrinsicID);
-  static std::optional<unsigned> getVectorLengthParamPos(
-      Intrinsic::ID IntrinsicID);
+  LLVM_ABI static std::optional<unsigned>
+  getMaskParamPos(Intrinsic::ID IntrinsicID);
+  LLVM_ABI static std::optional<unsigned>
+  getVectorLengthParamPos(Intrinsic::ID IntrinsicID);
 
   /// The llvm.vp.* intrinsics for this instruction Opcode
-  static Intrinsic::ID getForOpcode(unsigned OC);
+  LLVM_ABI static Intrinsic::ID getForOpcode(unsigned OC);
 
   /// The llvm.vp.* intrinsics for this intrinsic ID \p Id. Return \p Id if it
   /// is already a VP intrinsic.
-  static Intrinsic::ID getForIntrinsic(Intrinsic::ID Id);
+  LLVM_ABI static Intrinsic::ID getForIntrinsic(Intrinsic::ID Id);
 
   // Whether \p ID is a VP intrinsic ID.
-  static bool isVPIntrinsic(Intrinsic::ID);
+  LLVM_ABI static bool isVPIntrinsic(Intrinsic::ID);
 
   /// \return The mask parameter or nullptr.
-  Value *getMaskParam() const;
-  void setMaskParam(Value *);
+  LLVM_ABI Value *getMaskParam() const;
+  LLVM_ABI void setMaskParam(Value *);
 
   /// \return The vector length parameter or nullptr.
-  Value *getVectorLengthParam() const;
-  void setVectorLengthParam(Value *);
+  LLVM_ABI Value *getVectorLengthParam() const;
+  LLVM_ABI void setVectorLengthParam(Value *);
 
   /// \return Whether the vector length param can be ignored.
-  bool canIgnoreVectorLengthParam() const;
+  LLVM_ABI bool canIgnoreVectorLengthParam() const;
 
   /// \return The static element count (vector number of elements) the vector
   /// length parameter applies to.
-  ElementCount getStaticVectorLength() const;
+  LLVM_ABI ElementCount getStaticVectorLength() const;
 
   /// \return The alignment of the pointer used by this load/store/gather or
   /// scatter.
-  MaybeAlign getPointerAlignment() const;
+  LLVM_ABI MaybeAlign getPointerAlignment() const;
   // MaybeAlign setPointerAlignment(Align NewAlign); // TODO
 
   /// \return The pointer operand of this load,store, gather or scatter.
-  Value *getMemoryPointerParam() const;
-  static std::optional<unsigned> getMemoryPointerParamPos(Intrinsic::ID);
+  LLVM_ABI Value *getMemoryPointerParam() const;
+  LLVM_ABI static std::optional<unsigned>
+      getMemoryPointerParamPos(Intrinsic::ID);
 
   /// \return The data (payload) operand of this store or scatter.
-  Value *getMemoryDataParam() const;
-  static std::optional<unsigned> getMemoryDataParamPos(Intrinsic::ID);
+  LLVM_ABI Value *getMemoryDataParam() const;
+  LLVM_ABI static std::optional<unsigned> getMemoryDataParamPos(Intrinsic::ID);
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const IntrinsicInst *I) {
@@ -700,27 +703,28 @@ class VPIntrinsic : public IntrinsicInst {
   }
 
   // Equivalent non-predicated opcode
-  static std::optional<unsigned> getFunctionalOpcodeForVP(Intrinsic::ID ID);
+  LLVM_ABI static std::optional<unsigned>
+  getFunctionalOpcodeForVP(Intrinsic::ID ID);
 
   // Equivalent non-predicated intrinsic ID
-  static std::optional<Intrinsic::ID>
+  LLVM_ABI static std::optional<Intrinsic::ID>
   getFunctionalIntrinsicIDForVP(Intrinsic::ID ID);
 
   // Equivalent non-predicated constrained ID
-  static std::optional<Intrinsic::ID>
+  LLVM_ABI static std::optional<Intrinsic::ID>
   getConstrainedIntrinsicIDForVP(Intrinsic::ID ID);
 };
 
 /// This represents vector predication reduction intrinsics.
 class VPReductionIntrinsic : public VPIntrinsic {
 public:
-  static bool isVPReduction(Intrinsic::ID ID);
+  LLVM_ABI static bool isVPReduction(Intrinsic::ID ID);
 
-  unsigned getStartParamPos() const;
-  unsigned getVectorParamPos() const;
+  LLVM_ABI unsigned getStartParamPos() const;
+  LLVM_ABI unsigned getVectorParamPos() const;
 
-  static std::optional<unsigned> getStartParamPos(Intrinsic::ID ID);
-  static std::optional<unsigned> getVectorParamPos(Intrinsic::ID ID);
+  LLVM_ABI static std::optional<unsigned> getStartParamPos(Intrinsic::ID ID);
+  LLVM_ABI static std::optional<unsigned> getVectorParamPos(Intrinsic::ID ID);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   /// @{
@@ -735,7 +739,7 @@ class VPReductionIntrinsic : public VPIntrinsic {
 
 class VPCastIntrinsic : public VPIntrinsic {
 public:
-  static bool isVPCast(Intrinsic::ID ID);
+  LLVM_ABI static bool isVPCast(Intrinsic::ID ID);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   /// @{
@@ -750,9 +754,9 @@ class VPCastIntrinsic : public VPIntrinsic {
 
 class VPCmpIntrinsic : public VPIntrinsic {
 public:
-  static bool isVPCmp(Intrinsic::ID ID);
+  LLVM_ABI static bool isVPCmp(Intrinsic::ID ID);
 
-  CmpInst::Predicate getPredicate() const;
+  LLVM_ABI CmpInst::Predicate getPredicate() const;
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   /// @{
@@ -767,7 +771,7 @@ class VPCmpIntrinsic : public VPIntrinsic {
 
 class VPBinOpIntrinsic : public VPIntrinsic {
 public:
-  static bool isVPBinOp(Intrinsic::ID ID);
+  LLVM_ABI static bool isVPBinOp(Intrinsic::ID ID);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   /// @{
@@ -784,13 +788,13 @@ class VPBinOpIntrinsic : public VPIntrinsic {
 /// This is the common base class for constrained floating point intrinsics.
 class ConstrainedFPIntrinsic : public IntrinsicInst {
 public:
-  unsigned getNonMetadataArgCount() const;
-  std::optional<RoundingMode> getRoundingMode() const;
-  std::optional<fp::ExceptionBehavior> getExceptionBehavior() const;
-  bool isDefaultFPEnvironment() const;
+  LLVM_ABI unsigned getNonMetadataArgCount() const;
+  LLVM_ABI std::optional<RoundingMode> getRoundingMode() const;
+  LLVM_ABI std::optional<fp::ExceptionBehavior> getExceptionBehavior() const;
+  LLVM_ABI bool isDefaultFPEnvironment() const;
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
-  static bool classof(const IntrinsicInst *I);
+  LLVM_ABI static bool classof(const IntrinsicInst *I);
   static bool classof(const Value *V) {
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
@@ -799,7 +803,7 @@ class ConstrainedFPIntrinsic : public IntrinsicInst {
 /// Constrained floating point compare intrinsics.
 class ConstrainedFPCmpIntrinsic : public ConstrainedFPIntrinsic {
 public:
-  FCmpInst::Predicate getPredicate() const;
+  LLVM_ABI FCmpInst::Predicate getPredicate() const;
   bool isSignaling() const {
     return getIntrinsicID() == Intrinsic::experimental_constrained_fcmps;
   }
@@ -976,13 +980,13 @@ class BinaryOpIntrinsic : public IntrinsicInst {
   Value *getRHS() const { return const_cast<Value *>(getArgOperand(1)); }
 
   /// Returns the binary operation underlying the intrinsic.
-  Instruction::BinaryOps getBinaryOp() const;
+  LLVM_ABI Instruction::BinaryOps getBinaryOp() const;
 
   /// Whether the intrinsic is signed or unsigned.
-  bool isSigned() const;
+  LLVM_ABI bool isSigned() const;
 
   /// Returns one of OBO::NoSignedWrap or OBO::NoUnsignedWrap.
-  unsigned getNoWrapKind() const;
+  LLVM_ABI unsigned getNoWrapKind() const;
 };
 
 /// Represents an op.with.overflow intrinsic.
@@ -1537,10 +1541,10 @@ class InstrProfCntrInstBase : public InstrProfInstBase {
   }
 
   // The number of counters for the instrumented function.
-  ConstantInt *getNumCounters() const;
+  LLVM_ABI ConstantInt *getNumCounters() const;
   // The index of the counter that this instruction acts on.
-  ConstantInt *getIndex() const;
-  void setIndex(uint32_t Idx);
+  LLVM_ABI ConstantInt *getIndex() const;
+  LLVM_ABI void setIndex(uint32_t Idx);
 };
 
 /// This represents the llvm.instrprof.cover intrinsic.
@@ -1564,7 +1568,7 @@ class InstrProfIncrementInst : public InstrProfCntrInstBase {
   static bool classof(const Value *V) {
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
-  Value *getStep() const;
+  LLVM_ABI Value *getStep() const;
 };
 
 /// This represents the llvm.instrprof.increment.step intrinsic.
@@ -1594,10 +1598,10 @@ class InstrProfCallsite : public InstrProfCntrInstBase {
   static bool canInstrumentCallsite(const CallBase &CB) {
     return !CB.isInlineAsm() &&
            (CB.isIndirectCall() ||
-            (CB.getCalledFunction() && !CB.getCalledFunction()->isIntrinsic()));
+            (CB.getIntrinsicID() == Intrinsic::not_intrinsic));
   }
-  Value *getCallee() const;
-  void setCallee(Value *Callee);
+  LLVM_ABI Value *getCallee() const;
+  LLVM_ABI void setCallee(Value *Callee);
 };
 
 /// This represents the llvm.instrprof.timestamp intrinsic.
@@ -1763,7 +1767,7 @@ class GCProjectionInst : public IntrinsicInst {
   }
 
   /// The statepoint with which this gc.relocate is associated.
-  const Value *getStatepoint() const;
+  LLVM_ABI const Value *getStatepoint() const;
 };
 
 /// Represents calls to the gc.relocate intrinsic.
@@ -1790,8 +1794,8 @@ class GCRelocateInst : public GCProjectionInst {
     return cast<ConstantInt>(getArgOperand(2))->getZExtValue();
   }
 
-  Value *getBasePtr() const;
-  Value *getDerivedPtr() const;
+  LLVM_ABI Value *getBasePtr() const;
+  LLVM_ABI Value *getDerivedPtr() const;
 };
 
 /// Represents calls to the gc.result intrinsic.
@@ -1851,10 +1855,10 @@ class ConvergenceControlInst : public IntrinsicInst {
     return getIntrinsicID() == Intrinsic::experimental_convergence_loop;
   }
 
-  static ConvergenceControlInst *CreateAnchor(BasicBlock &BB);
-  static ConvergenceControlInst *CreateEntry(BasicBlock &BB);
-  static ConvergenceControlInst *CreateLoop(BasicBlock &BB,
-                                            ConvergenceControlInst *Parent);
+  LLVM_ABI static ConvergenceControlInst *CreateAnchor(BasicBlock &BB);
+  LLVM_ABI static ConvergenceControlInst *CreateEntry(BasicBlock &BB);
+  LLVM_ABI static ConvergenceControlInst *
+  CreateLoop(BasicBlock &BB, ConvergenceControlInst *Parent);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/IR/Intrinsics.h b/external/llvm-project/llvm/include/llvm/IR/Intrinsics.h
index 7aa6cbe93c13..156805293367 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Intrinsics.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Intrinsics.h
@@ -16,6 +16,7 @@
 #define LLVM_IR_INTRINSICS_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/TypeSize.h"
 #include <optional>
 #include <string>
@@ -53,11 +54,11 @@ namespace Intrinsic {
   /// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
   /// Note, this version is for intrinsics with no overloads.  Use the other
   /// version of getName if overloads are required.
-  StringRef getName(ID id);
+  LLVM_ABI StringRef getName(ID id);
 
   /// Return the LLVM name for an intrinsic, without encoded types for
   /// overloading, such as "llvm.ssa.copy".
-  StringRef getBaseName(ID id);
+  LLVM_ABI StringRef getBaseName(ID id);
 
   /// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx" or
   /// "llvm.ssa.copy.p0s_s.1". Note, this version of getName supports overloads.
@@ -65,31 +66,32 @@ namespace Intrinsic {
   /// overloads are required, it is safe to use this version, but better to use
   /// the StringRef version. If one of the types is based on an unnamed type, a
   /// function type will be computed. Providing FT will avoid this computation.
-  std::string getName(ID Id, ArrayRef<Type *> Tys, Module *M,
-                      FunctionType *FT = nullptr);
+  LLVM_ABI std::string getName(ID Id, ArrayRef<Type *> Tys, Module *M,
+                               FunctionType *FT = nullptr);
 
   /// Return the LLVM name for an intrinsic. This is a special version only to
   /// be used by LLVMIntrinsicCopyOverloadedName. It only supports overloads
   /// based on named types.
-  std::string getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys);
+  LLVM_ABI std::string getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys);
 
   /// Return the function type for an intrinsic.
-  FunctionType *getType(LLVMContext &Context, ID id, ArrayRef<Type *> Tys = {});
+  LLVM_ABI FunctionType *getType(LLVMContext &Context, ID id,
+                                 ArrayRef<Type *> Tys = {});
 
   /// Returns true if the intrinsic can be overloaded.
-  bool isOverloaded(ID id);
+  LLVM_ABI bool isOverloaded(ID id);
 
   /// isTargetIntrinsic - Returns true if IID is an intrinsic specific to a
   /// certain target. If it is a generic intrinsic false is returned.
-  bool isTargetIntrinsic(ID IID);
+  LLVM_ABI bool isTargetIntrinsic(ID IID);
 
-  ID lookupIntrinsicID(StringRef Name);
+  LLVM_ABI ID lookupIntrinsicID(StringRef Name);
 
   /// Return the attributes for an intrinsic.
-  AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT);
+  LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT);
 
   /// Return the function attributes for an intrinsic.
-  AttributeSet getFnAttributes(LLVMContext &C, ID id);
+  LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id);
 
   /// Look up the Function declaration of the intrinsic \p id in the Module
   /// \p M. If it does not exist, add a declaration and return it. Otherwise,
@@ -99,7 +101,8 @@ namespace Intrinsic {
   /// using iAny, fAny, vAny, or pAny).  For a declaration of an overloaded
   /// intrinsic, Tys must provide exactly one type for each overloaded type in
   /// the intrinsic.
-  Function *getOrInsertDeclaration(Module *M, ID id, ArrayRef<Type *> Tys = {});
+  LLVM_ABI Function *getOrInsertDeclaration(Module *M, ID id,
+                                            ArrayRef<Type *> Tys = {});
 
   LLVM_DEPRECATED("Use getOrInsertDeclaration instead",
                   "getOrInsertDeclaration")
@@ -110,25 +113,28 @@ namespace Intrinsic {
   /// Look up the Function declaration of the intrinsic \p id in the Module
   /// \p M and return it if it exists. Otherwise, return nullptr. This version
   /// supports non-overloaded intrinsics.
-  Function *getDeclarationIfExists(const Module *M, ID id);
+  LLVM_ABI Function *getDeclarationIfExists(const Module *M, ID id);
 
   /// This version supports overloaded intrinsics.
-  Function *getDeclarationIfExists(Module *M, ID id, ArrayRef<Type *> Tys,
-                                   FunctionType *FT = nullptr);
+  LLVM_ABI Function *getDeclarationIfExists(Module *M, ID id,
+                                            ArrayRef<Type *> Tys,
+                                            FunctionType *FT = nullptr);
 
   /// Map a Clang builtin name to an intrinsic ID.
-  ID getIntrinsicForClangBuiltin(StringRef TargetPrefix, StringRef BuiltinName);
+  LLVM_ABI ID getIntrinsicForClangBuiltin(StringRef TargetPrefix,
+                                          StringRef BuiltinName);
 
   /// Map a MS builtin name to an intrinsic ID.
-  ID getIntrinsicForMSBuiltin(StringRef TargetPrefix, StringRef BuiltinName);
+  LLVM_ABI ID getIntrinsicForMSBuiltin(StringRef TargetPrefix,
+                                       StringRef BuiltinName);
 
   /// Returns true if the intrinsic ID is for one of the "Constrained
   /// Floating-Point Intrinsics".
-  bool isConstrainedFPIntrinsic(ID QID);
+  LLVM_ABI bool isConstrainedFPIntrinsic(ID QID);
 
   /// Returns true if the intrinsic ID is for one of the "Constrained
   /// Floating-Point Intrinsics" that take rounding mode metadata.
-  bool hasConstrainedFPRoundingModeOperand(ID QID);
+  LLVM_ABI bool hasConstrainedFPRoundingModeOperand(ID QID);
 
   /// This is a type descriptor which explains the type requirements of an
   /// intrinsic. This is returned by getIntrinsicInfoTableEntries.
@@ -232,7 +238,8 @@ namespace Intrinsic {
 
   /// Return the IIT table descriptor for the specified intrinsic into an array
   /// of IITDescriptors.
-  void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl<IITDescriptor> &T);
+  LLVM_ABI void getIntrinsicInfoTableEntries(ID id,
+                                             SmallVectorImpl<IITDescriptor> &T);
 
   enum MatchIntrinsicTypesResult {
     MatchIntrinsicTypes_Match = 0,
@@ -246,7 +253,7 @@ namespace Intrinsic {
   ///
   /// Returns false if the given type matches with the constraints, true
   /// otherwise.
-  MatchIntrinsicTypesResult
+  LLVM_ABI MatchIntrinsicTypesResult
   matchIntrinsicSignature(FunctionType *FTy, ArrayRef<IITDescriptor> &Infos,
                           SmallVectorImpl<Type *> &ArgTys);
 
@@ -254,7 +261,8 @@ namespace Intrinsic {
   /// be called after all the fixed arguments have been matched first.
   ///
   /// This method returns true on error.
-  bool matchIntrinsicVarArg(bool isVarArg, ArrayRef<IITDescriptor> &Infos);
+  LLVM_ABI bool matchIntrinsicVarArg(bool isVarArg,
+                                     ArrayRef<IITDescriptor> &Infos);
 
   /// Gets the type arguments of an intrinsic call by matching type contraints
   /// specified by the .td file. The overloaded types are pushed into the
@@ -262,17 +270,18 @@ namespace Intrinsic {
   ///
   /// Returns false if the given ID and function type combination is not a
   /// valid intrinsic call.
-  bool getIntrinsicSignature(Intrinsic::ID, FunctionType *FT,
-                             SmallVectorImpl<Type *> &ArgTys);
+  LLVM_ABI bool getIntrinsicSignature(Intrinsic::ID, FunctionType *FT,
+                                      SmallVectorImpl<Type *> &ArgTys);
 
   /// Same as previous, but accepts a Function instead of ID and FunctionType.
-  bool getIntrinsicSignature(Function *F, SmallVectorImpl<Type *> &ArgTys);
+  LLVM_ABI bool getIntrinsicSignature(Function *F,
+                                      SmallVectorImpl<Type *> &ArgTys);
 
   // Checks if the intrinsic name matches with its signature and if not
   // returns the declaration with the same signature and remangled name.
   // An existing GlobalValue with the wanted name but with a wrong prototype
   // or of the wrong kind will be renamed by adding ".renamed" to the name.
-  std::optional<Function *> remangleIntrinsicFunction(Function *F);
+  LLVM_ABI std::optional<Function *> remangleIntrinsicFunction(Function *F);
 
 } // End Intrinsic namespace
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/Intrinsics.td b/external/llvm-project/llvm/include/llvm/IR/Intrinsics.td
index bc028c72b106..d00c173e21d6 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Intrinsics.td
+++ b/external/llvm-project/llvm/include/llvm/IR/Intrinsics.td
@@ -335,8 +335,8 @@ def IIT_I4 : IIT_Int<4, 58>;
 def IIT_AARCH64_SVCOUNT : IIT_VT<aarch64svcount, 59>;
 def IIT_V6 : IIT_Vec<6, 60>;
 def IIT_V10 : IIT_Vec<10, 61>;
-def IIT_V2048: IIT_Vec<2048, 62>;
-def IIT_V4096: IIT_Vec<4096, 63>;
+def IIT_V2048 : IIT_Vec<2048, 62>;
+def IIT_V4096 : IIT_Vec<4096, 63>;
 }
 
 defvar IIT_all_FixedTypes = !filter(iit, IIT_all,
@@ -575,6 +575,7 @@ def llvm_v64i32_ty     : LLVMType<v64i32>;    //   64 x i32
 def llvm_v128i32_ty    : LLVMType<v128i32>;   //  128 x i32
 def llvm_v256i32_ty    : LLVMType<v256i32>;   //  256 x i32
 def llvm_v2048i32_ty   : LLVMType<v2048i32>;  // 2048 x i32
+def llvm_v4096i32_ty   : LLVMType<v4096i32>;  // 4096 x i32
 
 def llvm_v1i64_ty      : LLVMType<v1i64>;    //  1 x i64
 def llvm_v2i64_ty      : LLVMType<v2i64>;    //  2 x i64
@@ -605,7 +606,7 @@ def llvm_v8f32_ty      : LLVMType<v8f32>;       //    8 x float
 def llvm_v10f32_ty     : LLVMType<v10f32>;      //   10 x float
 def llvm_v16f32_ty     : LLVMType<v16f32>;      //   16 x float
 def llvm_v32f32_ty     : LLVMType<v32f32>;      //   32 x float
-def llvm_v2048f32_ty     : LLVMType<v2048f32>;  // 2048 x float
+def llvm_v2048f32_ty   : LLVMType<v2048f32>;    // 2048 x float
 def llvm_v1f64_ty      : LLVMType<v1f64>;       //    1 x double
 def llvm_v2f64_ty      : LLVMType<v2f64>;       //    2 x double
 def llvm_v4f64_ty      : LLVMType<v4f64>;       //    4 x double
@@ -1973,6 +1974,24 @@ def int_experimental_vector_histogram_add : DefaultAttrsIntrinsic<[],
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
                              [ IntrArgMemOnly ]>;
 
+def int_experimental_vector_histogram_uadd_sat : DefaultAttrsIntrinsic<[],
+                             [ llvm_anyvector_ty, // Vector of pointers
+                               llvm_anyint_ty,    // Increment
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
+                             [ IntrArgMemOnly ]>;
+
+def int_experimental_vector_histogram_umin : DefaultAttrsIntrinsic<[],
+                             [ llvm_anyvector_ty, // Vector of pointers
+                               llvm_anyint_ty,    // Update value
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
+                             [ IntrArgMemOnly ]>;
+
+def int_experimental_vector_histogram_umax : DefaultAttrsIntrinsic<[],
+                             [ llvm_anyvector_ty, // Vector of pointers
+                               llvm_anyint_ty,    // Update value
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
+                             [ IntrArgMemOnly ]>;
+
 // Experimental match
 def int_experimental_vector_match : DefaultAttrsIntrinsic<
                              [ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty> ],
diff --git a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsARM.td b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsARM.td
index 9b7dd8099368..3ee69b72cc5c 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsARM.td
+++ b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsARM.td
@@ -1306,8 +1306,6 @@ foreach suffix = ["a","n","p","m"] in {
     [llvm_anyvector_ty /* input */], LLVMMatchType<0>, llvm_anyvector_ty>;
 }
 
-def int_arm_mve_vrintn: DefaultAttrsIntrinsic<
-  [llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 def int_arm_mve_vcls: DefaultAttrsIntrinsic<
   [llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsDirectX.td b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 0f73084f88c5..68b31a899003 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -148,6 +148,8 @@ def int_dx_wave_reduce_sum : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType
 def int_dx_wave_reduce_usum : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
 def int_dx_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
 def int_dx_wave_readlane : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
+def int_dx_wave_get_lane_count
+    : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrConvergent]>;
 def int_dx_sign : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_any_ty], [IntrNoMem]>;
 def int_dx_step : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty, LLVMMatchType<0>], [IntrNoMem]>;
 def int_dx_splitdouble : DefaultAttrsIntrinsic<[llvm_anyint_ty, LLVMMatchType<0>],
diff --git a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsLoongArch.td b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsLoongArch.td
index 4621f1689b46..f5e3d412666a 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsLoongArch.td
+++ b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsLoongArch.td
@@ -25,12 +25,13 @@ class MaskedAtomicRMW<LLVMType itype>
 multiclass MaskedAtomicRMWIntrinsics {
   // i32 @llvm.<name>.i32.<p>(any*, i32, i32, i32 imm);
   def _i32 : MaskedAtomicRMW<llvm_i32_ty>;
-  // i64 @llvm.<name>.i32.<p>(any*, i64, i64, i64 imm);
+  // i64 @llvm.<name>.i64.<p>(any*, i64, i64, i64 imm);
   def _i64 : MaskedAtomicRMW<llvm_i64_ty>;
 }
 
 multiclass MaskedAtomicRMWFiveOpIntrinsics {
-  // TODO: Support cmpxchg on LA32.
+  // i32 @llvm.<name>.i32.<p>(any*, i32, i32, i32, i32 imm);
+  def _i32 : MaskedAtomicRMWFiveArg<llvm_i32_ty>;
   // i64 @llvm.<name>.i64.<p>(any*, i64, i64, i64, i64 imm);
   def _i64 : MaskedAtomicRMWFiveArg<llvm_i64_ty>;
 }
@@ -44,7 +45,7 @@ defm int_loongarch_masked_atomicrmw_umin : MaskedAtomicRMWIntrinsics;
 defm int_loongarch_masked_atomicrmw_max : MaskedAtomicRMWFiveOpIntrinsics;
 defm int_loongarch_masked_atomicrmw_min : MaskedAtomicRMWFiveOpIntrinsics;
 
-// @llvm.loongarch.masked.cmpxchg.i64.<p>(
+// @llvm.loongarch.masked.cmpxchg.<i32,i64>.<p>(
 //   ptr addr, grlen cmpval, grlen newval, grlen mask, grlenimm ordering)
 defm int_loongarch_masked_cmpxchg : MaskedAtomicRMWFiveOpIntrinsics;
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsNVVM.td b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 91e7d188c853..4efdff71c016 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -768,6 +768,11 @@ let TargetPrefix = "nvvm" in {
       DefaultAttrsIntrinsic<[], [llvm_i32_ty],
                             [IntrConvergent, IntrNoMem, IntrHasSideEffects]>;
 
+  // Performance Monitor Events (pm events) intrinsics
+  def int_nvvm_pm_event_mask : NVVMBuiltin,
+      DefaultAttrsIntrinsic<[], [llvm_i16_ty],
+                [IntrConvergent, IntrNoMem, IntrHasSideEffects,
+                 ImmArg<ArgIndex<0>>]>;
 //
 // Min Max
 //
@@ -2015,20 +2020,26 @@ foreach dim = 1...5 in {
     defvar num_im2col_offsets = !if(is_im2col, !add(dim, -2), 0);
     defvar im2col_offsets_args = !listsplat(llvm_i16_ty, num_im2col_offsets);
 
+    defvar g2s_params = !listconcat(
+                          [llvm_shared_cluster_ptr_ty, // dst_ptr
+                           llvm_shared_ptr_ty,  // mbarrier_ptr
+                           llvm_ptr_ty],        // tensormap_ptr
+                          tensor_dim_args,      // actual tensor dims
+                          im2col_offsets_args,  // im2col offsets
+                          [llvm_i16_ty,         // cta_mask
+                           llvm_i64_ty]);       // cache_hint
+    defvar g2s_flags = [llvm_i1_ty,             // Flag for cta_mask
+                        llvm_i1_ty,             // Flag for cache_hint
+                        llvm_i32_ty];           // Flag for cta_group
+    defvar cta_group_idx = !add(
+                             !size(g2s_params),
+                             !sub(!size(g2s_flags), 1));
+    defvar g2s_props = [IntrConvergent,
+                        WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
+                        // Allowed values for cta_group are {0,1,2} i.e [0, 3).
+                        Range<ArgIndex<cta_group_idx>, 0, 3>];
     def int_nvvm_cp_async_bulk_tensor_g2s_ # mode # _ # dim # d :
-      DefaultAttrsIntrinsicFlags<[],
-          !listconcat([llvm_shared_cluster_ptr_ty,  // dst_shared_cluster_ptr
-                       llvm_shared_ptr_ty,          // mbarrier_smem_ptr
-                       llvm_ptr_ty],                // tensormap_ptr
-                      tensor_dim_args,              // actual tensor dims
-                      im2col_offsets_args,          // im2col offsets
-                      [llvm_i16_ty,                 // cta_mask
-                       llvm_i64_ty]),               // cache_hint
-          [llvm_i1_ty,                              // Flag for cta_mask
-           llvm_i1_ty],                             // Flag for cache_hint
-          [IntrConvergent,
-           WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
-           NoCapture<ArgIndex<0>>, NoCapture<ArgIndex<1>>, NoCapture<ArgIndex<2>>]>;
+      DefaultAttrsIntrinsicFlags<[], g2s_params, g2s_flags, g2s_props>;
 
     def int_nvvm_cp_async_bulk_tensor_s2g_ # mode # _ # dim # d :
       DefaultAttrsIntrinsicFlags<[],
diff --git a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 751628cee58c..84c26599b5b7 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -1835,6 +1835,19 @@ let TargetPrefix = "ppc" in {
                       Intrinsic<[],[],[]>;
   def int_ppc_iospace_eieio : ClangBuiltin<"__builtin_ppc_iospace_eieio">,
                               Intrinsic<[],[],[]>;
+  def int_ppc_lbarx :
+    Intrinsic<[llvm_i32_ty], [llvm_ptr_ty],
+              [IntrReadMem, IntrArgMemOnly]>;
+  def int_ppc_lharx :
+    Intrinsic<[llvm_i32_ty],[llvm_ptr_ty],
+              [IntrReadMem, IntrArgMemOnly]>;
+  def int_ppc_lwarx :
+    Intrinsic<[llvm_i32_ty], [llvm_ptr_ty],
+              [IntrReadMem, IntrArgMemOnly]>;
+  def int_ppc_ldarx :
+    Intrinsic<[llvm_i64_ty],[llvm_ptr_ty],
+              [IntrReadMem, IntrArgMemOnly]>;
+
   def int_ppc_stdcx :
     ClangBuiltin<"__builtin_ppc_stdcx">,
     Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i64_ty],
@@ -1844,7 +1857,7 @@ let TargetPrefix = "ppc" in {
     Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty],
               [IntrWriteMem, IntrArgMemOnly]>;
   def int_ppc_sthcx :
-    Intrinsic<[llvm_i32_ty], [ llvm_ptr_ty, llvm_i32_ty ],
+    Intrinsic<[llvm_i32_ty], [ llvm_ptr_ty, llvm_i32_ty],
               [IntrWriteMem, IntrArgMemOnly, IntrNoDuplicate]>;
   def int_ppc_stbcx :
     ClangBuiltin<"__builtin_ppc_stbcx">,
diff --git a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 8d984d6ce58d..43335f81ed87 100644
--- a/external/llvm-project/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/external/llvm-project/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -102,6 +102,8 @@ let TargetPrefix = "spv" in {
   def int_spv_wave_reduce_sum : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>;
   def int_spv_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
   def int_spv_wave_readlane : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
+  def int_spv_wave_get_lane_count
+      : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrConvergent]>;
   def int_spv_sign : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_any_ty], [IntrNoMem]>;
   def int_spv_radians : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
   def int_spv_group_memory_barrier_with_group_sync : DefaultAttrsIntrinsic<[], [], []>;
@@ -115,15 +117,15 @@ let TargetPrefix = "spv" in {
   // array size of the binding, as well as an index and an indicator
   // whether that index may be non-uniform.
   def int_spv_resource_handlefrombinding
-      : DefaultAttrsIntrinsic<
-            [llvm_any_ty],
-            [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty],
-            [IntrNoMem]>;
+      : DefaultAttrsIntrinsic<[llvm_any_ty],
+                              [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty,
+                               llvm_i32_ty, llvm_i1_ty, llvm_ptr_ty],
+                              [IntrNoMem]>;
   def int_spv_resource_handlefromimplicitbinding
-      : DefaultAttrsIntrinsic<
-            [llvm_any_ty],
-            [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty],
-            [IntrNoMem]>;
+      : DefaultAttrsIntrinsic<[llvm_any_ty],
+                              [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty,
+                               llvm_i32_ty, llvm_i1_ty, llvm_ptr_ty],
+                              [IntrNoMem]>;
 
   def int_spv_firstbituhigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>;
   def int_spv_firstbitshigh : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_anyint_ty], [IntrNoMem]>;
diff --git a/external/llvm-project/llvm/include/llvm/IR/LLVMContext.h b/external/llvm-project/llvm/include/llvm/IR/LLVMContext.h
index 6dde7a968b57..852a3a4e2f63 100644
--- a/external/llvm-project/llvm/include/llvm/IR/LLVMContext.h
+++ b/external/llvm-project/llvm/include/llvm/IR/LLVMContext.h
@@ -17,6 +17,7 @@
 #include "llvm-c/Types.h"
 #include "llvm/IR/DiagnosticHandler.h"
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <memory>
 #include <optional>
@@ -67,10 +68,10 @@ enum {
 class LLVMContext {
 public:
   LLVMContextImpl *const pImpl;
-  LLVMContext();
+  LLVM_ABI LLVMContext();
   LLVMContext(const LLVMContext &) = delete;
   LLVMContext &operator=(const LLVMContext &) = delete;
-  ~LLVMContext();
+  LLVM_ABI ~LLVMContext();
 
   // Pinned metadata names, which always have the same value.  This is a
   // compile-time performance optimization, not a correctness optimization.
@@ -100,68 +101,69 @@ class LLVMContext {
 
   /// getMDKindID - Return a unique non-zero ID for the specified metadata kind.
   /// This ID is uniqued across modules in the current LLVMContext.
-  unsigned getMDKindID(StringRef Name) const;
+  LLVM_ABI unsigned getMDKindID(StringRef Name) const;
 
   /// getMDKindNames - Populate client supplied SmallVector with the name for
   /// custom metadata IDs registered in this LLVMContext.
-  void getMDKindNames(SmallVectorImpl<StringRef> &Result) const;
+  LLVM_ABI void getMDKindNames(SmallVectorImpl<StringRef> &Result) const;
 
   /// getOperandBundleTags - Populate client supplied SmallVector with the
   /// bundle tags registered in this LLVMContext.  The bundle tags are ordered
   /// by increasing bundle IDs.
   /// \see LLVMContext::getOperandBundleTagID
-  void getOperandBundleTags(SmallVectorImpl<StringRef> &Result) const;
+  LLVM_ABI void getOperandBundleTags(SmallVectorImpl<StringRef> &Result) const;
 
   /// getOrInsertBundleTag - Returns the Tag to use for an operand bundle of
   /// name TagName.
-  StringMapEntry<uint32_t> *getOrInsertBundleTag(StringRef TagName) const;
+  LLVM_ABI StringMapEntry<uint32_t> *
+  getOrInsertBundleTag(StringRef TagName) const;
 
   /// getOperandBundleTagID - Maps a bundle tag to an integer ID.  Every bundle
   /// tag registered with an LLVMContext has an unique ID.
-  uint32_t getOperandBundleTagID(StringRef Tag) const;
+  LLVM_ABI uint32_t getOperandBundleTagID(StringRef Tag) const;
 
   /// getOrInsertSyncScopeID - Maps synchronization scope name to
   /// synchronization scope ID.  Every synchronization scope registered with
   /// LLVMContext has unique ID except pre-defined ones.
-  SyncScope::ID getOrInsertSyncScopeID(StringRef SSN);
+  LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN);
 
   /// getSyncScopeNames - Populates client supplied SmallVector with
   /// synchronization scope names registered with LLVMContext.  Synchronization
   /// scope names are ordered by increasing synchronization scope IDs.
-  void getSyncScopeNames(SmallVectorImpl<StringRef> &SSNs) const;
+  LLVM_ABI void getSyncScopeNames(SmallVectorImpl<StringRef> &SSNs) const;
 
   /// getSyncScopeName - Returns the name of a SyncScope::ID
   /// registered with LLVMContext, if any.
-  std::optional<StringRef> getSyncScopeName(SyncScope::ID Id) const;
+  LLVM_ABI std::optional<StringRef> getSyncScopeName(SyncScope::ID Id) const;
 
   /// Define the GC for a function
-  void setGC(const Function &Fn, std::string GCName);
+  LLVM_ABI void setGC(const Function &Fn, std::string GCName);
 
   /// Return the GC for a function
-  const std::string &getGC(const Function &Fn);
+  LLVM_ABI const std::string &getGC(const Function &Fn);
 
   /// Remove the GC for a function
-  void deleteGC(const Function &Fn);
+  LLVM_ABI void deleteGC(const Function &Fn);
 
   /// Return true if the Context runtime configuration is set to discard all
   /// value names. When true, only GlobalValue names will be available in the
   /// IR.
-  bool shouldDiscardValueNames() const;
+  LLVM_ABI bool shouldDiscardValueNames() const;
 
   /// Set the Context runtime configuration to discard all value name (but
   /// GlobalValue). Clients can use this flag to save memory and runtime,
   /// especially in release mode.
-  void setDiscardValueNames(bool Discard);
+  LLVM_ABI void setDiscardValueNames(bool Discard);
 
   /// Whether there is a string map for uniquing debug info
   /// identifiers across the context.  Off by default.
-  bool isODRUniquingDebugTypes() const;
-  void enableDebugTypeODRUniquing();
-  void disableDebugTypeODRUniquing();
+  LLVM_ABI bool isODRUniquingDebugTypes() const;
+  LLVM_ABI void enableDebugTypeODRUniquing();
+  LLVM_ABI void disableDebugTypeODRUniquing();
 
   /// generateMachineFunctionNum - Get a unique number for MachineFunction
   /// that associated with the given Function.
-  unsigned generateMachineFunctionNum(Function &);
+  LLVM_ABI unsigned generateMachineFunctionNum(Function &);
 
   /// Defines the type of a yield callback.
   /// \see LLVMContext::setYieldCallback.
@@ -175,7 +177,7 @@ class LLVMContext {
   ///
   /// LLVMContext doesn't take ownership or interpret either of these
   /// pointers.
-  void setDiagnosticHandlerCallBack(
+  LLVM_ABI void setDiagnosticHandlerCallBack(
       DiagnosticHandler::DiagnosticHandlerTy DiagHandler,
       void *DiagContext = nullptr, bool RespectFilters = false);
 
@@ -186,36 +188,38 @@ class LLVMContext {
   /// expects enabled diagnostics.
   ///
   /// Ownership of this pointer is moved to LLVMContextImpl.
-  void setDiagnosticHandler(std::unique_ptr<DiagnosticHandler> &&DH,
-                            bool RespectFilters = false);
+  LLVM_ABI void setDiagnosticHandler(std::unique_ptr<DiagnosticHandler> &&DH,
+                                     bool RespectFilters = false);
 
   /// getDiagnosticHandlerCallBack - Return the diagnostic handler call back set by
   /// setDiagnosticHandlerCallBack.
-  DiagnosticHandler::DiagnosticHandlerTy getDiagnosticHandlerCallBack() const;
+  LLVM_ABI DiagnosticHandler::DiagnosticHandlerTy
+  getDiagnosticHandlerCallBack() const;
 
   /// getDiagnosticContext - Return the diagnostic context set by
   /// setDiagnosticContext.
-  void *getDiagnosticContext() const;
+  LLVM_ABI void *getDiagnosticContext() const;
 
   /// getDiagHandlerPtr - Returns const raw pointer of DiagnosticHandler set by
   /// setDiagnosticHandler.
-  const DiagnosticHandler *getDiagHandlerPtr() const;
+  LLVM_ABI const DiagnosticHandler *getDiagHandlerPtr() const;
 
   /// getDiagnosticHandler - transfers ownership of DiagnosticHandler unique_ptr
   /// to caller.
-  std::unique_ptr<DiagnosticHandler> getDiagnosticHandler();
+  LLVM_ABI std::unique_ptr<DiagnosticHandler> getDiagnosticHandler();
 
   /// Return if a code hotness metric should be included in optimization
   /// diagnostics.
-  bool getDiagnosticsHotnessRequested() const;
+  LLVM_ABI bool getDiagnosticsHotnessRequested() const;
   /// Set if a code hotness metric should be included in optimization
   /// diagnostics.
-  void setDiagnosticsHotnessRequested(bool Requested);
+  LLVM_ABI void setDiagnosticsHotnessRequested(bool Requested);
 
-  bool getMisExpectWarningRequested() const;
-  void setMisExpectWarningRequested(bool Requested);
-  void setDiagnosticsMisExpectTolerance(std::optional<uint32_t> Tolerance);
-  uint32_t getDiagnosticsMisExpectTolerance() const;
+  LLVM_ABI bool getMisExpectWarningRequested() const;
+  LLVM_ABI void setMisExpectWarningRequested(bool Requested);
+  LLVM_ABI void
+  setDiagnosticsMisExpectTolerance(std::optional<uint32_t> Tolerance);
+  LLVM_ABI uint32_t getDiagnosticsMisExpectTolerance() const;
 
   /// Return the minimum hotness value a diagnostic would need in order
   /// to be included in optimization diagnostics.
@@ -227,14 +231,15 @@ class LLVMContext {
   ///                profile summary. Note that in case of missing profile
   ///                summary, threshold will be kept at "MAX", effectively
   ///                suppresses all remarks output.
-  uint64_t getDiagnosticsHotnessThreshold() const;
+  LLVM_ABI uint64_t getDiagnosticsHotnessThreshold() const;
 
   /// Set the minimum hotness value a diagnostic needs in order to be
   /// included in optimization diagnostics.
-  void setDiagnosticsHotnessThreshold(std::optional<uint64_t> Threshold);
+  LLVM_ABI void
+  setDiagnosticsHotnessThreshold(std::optional<uint64_t> Threshold);
 
   /// Return if hotness threshold is requested from PSI.
-  bool isDiagnosticsHotnessThresholdSetFromPSI() const;
+  LLVM_ABI bool isDiagnosticsHotnessThresholdSetFromPSI() const;
 
   /// The "main remark streamer" used by all the specialized remark streamers.
   /// This streamer keeps generic remark metadata in memory throughout the life
@@ -243,9 +248,9 @@ class LLVMContext {
   ///
   /// All specialized remark streamers should convert remarks to
   /// llvm::remarks::Remark and emit them through this streamer.
-  remarks::RemarkStreamer *getMainRemarkStreamer();
-  const remarks::RemarkStreamer *getMainRemarkStreamer() const;
-  void setMainRemarkStreamer(
+  LLVM_ABI remarks::RemarkStreamer *getMainRemarkStreamer();
+  LLVM_ABI const remarks::RemarkStreamer *getMainRemarkStreamer() const;
+  LLVM_ABI void setMainRemarkStreamer(
       std::unique_ptr<remarks::RemarkStreamer> MainRemarkStreamer);
 
   /// The "LLVM remark streamer" used by LLVM to serialize remark diagnostics
@@ -253,14 +258,15 @@ class LLVMContext {
   ///
   /// If it does not exist, diagnostics are not saved in a file but only emitted
   /// via the diagnostic handler.
-  LLVMRemarkStreamer *getLLVMRemarkStreamer();
-  const LLVMRemarkStreamer *getLLVMRemarkStreamer() const;
-  void
+  LLVM_ABI LLVMRemarkStreamer *getLLVMRemarkStreamer();
+  LLVM_ABI const LLVMRemarkStreamer *getLLVMRemarkStreamer() const;
+  LLVM_ABI void
   setLLVMRemarkStreamer(std::unique_ptr<LLVMRemarkStreamer> RemarkStreamer);
 
   /// Get the prefix that should be printed in front of a diagnostic of
   ///        the given \p Severity
-  static const char *getDiagnosticMessagePrefix(DiagnosticSeverity Severity);
+  LLVM_ABI static const char *
+  getDiagnosticMessagePrefix(DiagnosticSeverity Severity);
 
   /// Report a message to the currently installed diagnostic handler.
   ///
@@ -272,7 +278,7 @@ class LLVMContext {
   /// The diagnostic message will be implicitly prefixed with a severity keyword
   /// according to \p DI.getSeverity(), i.e., "error: " for \a DS_Error,
   /// "warning: " for \a DS_Warning, and "note: " for \a DS_Note.
-  void diagnose(const DiagnosticInfo &DI);
+  LLVM_ABI void diagnose(const DiagnosticInfo &DI);
 
   /// Registers a yield callback with the given context.
   ///
@@ -291,33 +297,33 @@ class LLVMContext {
   /// yield callback are allowed to be used. Any other API calls into the
   /// context are not supported until the yield callback function returns
   /// control to LLVM. Other LLVM contexts are unaffected by this restriction.
-  void setYieldCallback(YieldCallbackTy Callback, void *OpaqueHandle);
+  LLVM_ABI void setYieldCallback(YieldCallbackTy Callback, void *OpaqueHandle);
 
   /// Calls the yield callback (if applicable).
   ///
   /// This transfers control of the current thread back to the client, which may
   /// suspend the current thread. Only call this method when LLVM doesn't hold
   /// any global mutex or cannot block the execution in another LLVM context.
-  void yield();
+  LLVM_ABI void yield();
 
   /// emitError - Emit an error message to the currently installed error handler
   /// with optional location information.  This function returns, so code should
   /// be prepared to drop the erroneous construct on the floor and "not crash".
   /// The generated code need not be correct.  The error message will be
   /// implicitly prefixed with "error: " and should not end with a ".".
-  void emitError(const Instruction *I, const Twine &ErrorStr);
-  void emitError(const Twine &ErrorStr);
+  LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr);
+  LLVM_ABI void emitError(const Twine &ErrorStr);
 
   /// Access the object which can disable optional passes and individual
   /// optimizations at compile time.
-  OptPassGate &getOptPassGate() const;
+  LLVM_ABI OptPassGate &getOptPassGate() const;
 
   /// Set the object which can disable optional passes and individual
   /// optimizations at compile time.
   ///
   /// The lifetime of the object must be guaranteed to extend as long as the
   /// LLVMContext is used by compilation.
-  void setOptPassGate(OptPassGate&);
+  LLVM_ABI void setOptPassGate(OptPassGate &);
 
   /// Get or set the current "default" target CPU (target-cpu function
   /// attribute). The intent is that compiler frontends will set this to a value
@@ -328,20 +334,20 @@ class LLVMContext {
   /// Function::createWithDefaultAttr() will create functions with this
   /// attribute. This function should only be called by passes that run at
   /// compile time and not by the backend or LTO passes.
-  StringRef getDefaultTargetCPU();
-  void setDefaultTargetCPU(StringRef CPU);
+  LLVM_ABI StringRef getDefaultTargetCPU();
+  LLVM_ABI void setDefaultTargetCPU(StringRef CPU);
 
   /// Similar to {get,set}DefaultTargetCPU() but for default target-features.
-  StringRef getDefaultTargetFeatures();
-  void setDefaultTargetFeatures(StringRef Features);
+  LLVM_ABI StringRef getDefaultTargetFeatures();
+  LLVM_ABI void setDefaultTargetFeatures(StringRef Features);
 
   /// Key Instructions: update the highest number atom group emitted for any
   /// function.
-  void updateDILocationAtomGroupWaterline(uint64_t G);
+  LLVM_ABI void updateDILocationAtomGroupWaterline(uint64_t G);
 
   /// Key Instructions: get the next free atom group number and increment
   /// the global tracker.
-  uint64_t incNextDILocationAtomGroup();
+  LLVM_ABI uint64_t incNextDILocationAtomGroup();
 
 private:
   // Module needs access to the add/removeModule methods.
diff --git a/external/llvm-project/llvm/include/llvm/IR/LLVMRemarkStreamer.h b/external/llvm-project/llvm/include/llvm/IR/LLVMRemarkStreamer.h
index 7e78c4f806b4..376acdec49fb 100644
--- a/external/llvm-project/llvm/include/llvm/IR/LLVMRemarkStreamer.h
+++ b/external/llvm-project/llvm/include/llvm/IR/LLVMRemarkStreamer.h
@@ -15,6 +15,7 @@
 #define LLVM_IR_LLVMREMARKSTREAMER_H
 
 #include "llvm/Remarks/Remark.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <memory>
 #include <optional>
@@ -41,7 +42,7 @@ class LLVMRemarkStreamer {
 public:
   LLVMRemarkStreamer(remarks::RemarkStreamer &RS) : RS(RS) {}
   /// Emit a diagnostic through the streamer.
-  void emit(const DiagnosticInfoOptimizationBase &Diag);
+  LLVM_ABI void emit(const DiagnosticInfoOptimizationBase &Diag);
 };
 
 template <typename ThisError>
@@ -62,36 +63,35 @@ struct LLVMRemarkSetupErrorInfo : public ErrorInfo<ThisError> {
 
 struct LLVMRemarkSetupFileError
     : LLVMRemarkSetupErrorInfo<LLVMRemarkSetupFileError> {
-  static char ID;
+  LLVM_ABI static char ID;
   using LLVMRemarkSetupErrorInfo<
       LLVMRemarkSetupFileError>::LLVMRemarkSetupErrorInfo;
 };
 
 struct LLVMRemarkSetupPatternError
     : LLVMRemarkSetupErrorInfo<LLVMRemarkSetupPatternError> {
-  static char ID;
+  LLVM_ABI static char ID;
   using LLVMRemarkSetupErrorInfo<
       LLVMRemarkSetupPatternError>::LLVMRemarkSetupErrorInfo;
 };
 
 struct LLVMRemarkSetupFormatError
     : LLVMRemarkSetupErrorInfo<LLVMRemarkSetupFormatError> {
-  static char ID;
+  LLVM_ABI static char ID;
   using LLVMRemarkSetupErrorInfo<
       LLVMRemarkSetupFormatError>::LLVMRemarkSetupErrorInfo;
 };
 
 /// Setup optimization remarks that output to a file.
-Expected<std::unique_ptr<ToolOutputFile>>
-setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename,
-                             StringRef RemarksPasses, StringRef RemarksFormat,
-                             bool RemarksWithHotness,
-                             std::optional<uint64_t> RemarksHotnessThreshold = 0);
+LLVM_ABI Expected<std::unique_ptr<ToolOutputFile>> setupLLVMOptimizationRemarks(
+    LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
+    StringRef RemarksFormat, bool RemarksWithHotness,
+    std::optional<uint64_t> RemarksHotnessThreshold = 0);
 
 /// Setup optimization remarks that output directly to a raw_ostream.
 /// \p OS is managed by the caller and should be open for writing as long as \p
 /// Context is streaming remarks to it.
-Error setupLLVMOptimizationRemarks(
+LLVM_ABI Error setupLLVMOptimizationRemarks(
     LLVMContext &Context, raw_ostream &OS, StringRef RemarksPasses,
     StringRef RemarksFormat, bool RemarksWithHotness,
     std::optional<uint64_t> RemarksHotnessThreshold = 0);
diff --git a/external/llvm-project/llvm/include/llvm/IR/LegacyPassManager.h b/external/llvm-project/llvm/include/llvm/IR/LegacyPassManager.h
index b3a4820ba0e4..770b3d8f6eaa 100644
--- a/external/llvm-project/llvm/include/llvm/IR/LegacyPassManager.h
+++ b/external/llvm-project/llvm/include/llvm/IR/LegacyPassManager.h
@@ -17,6 +17,7 @@
 #define LLVM_IR_LEGACYPASSMANAGER_H
 
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -28,7 +29,7 @@ namespace legacy {
 
 // Whether or not -debug-pass has been specified. For use to check if it's
 // specified alongside the new PM.
-bool debugPassSpecified();
+LLVM_ABI bool debugPassSpecified();
 
 class PassManagerImpl;
 class FunctionPassManagerImpl;
@@ -36,7 +37,7 @@ class FunctionPassManagerImpl;
 /// PassManagerBase - An abstract interface to allow code to add passes to
 /// a pass manager without having to hard-code what kind of pass manager
 /// it is.
-class PassManagerBase {
+class LLVM_ABI PassManagerBase {
 public:
   virtual ~PassManagerBase();
 
@@ -49,7 +50,7 @@ class PassManagerBase {
 };
 
 /// PassManager manages ModulePassManagers
-class PassManager : public PassManagerBase {
+class LLVM_ABI PassManager : public PassManagerBase {
 public:
 
   PassManager();
@@ -68,7 +69,7 @@ class PassManager : public PassManagerBase {
 };
 
 /// FunctionPassManager manages FunctionPasses.
-class FunctionPassManager : public PassManagerBase {
+class LLVM_ABI FunctionPassManager : public PassManagerBase {
 public:
   /// FunctionPassManager ctor - This initializes the pass manager.  It needs,
   /// but does not take ownership of, the specified Module.
diff --git a/external/llvm-project/llvm/include/llvm/IR/LegacyPassManagers.h b/external/llvm-project/llvm/include/llvm/IR/LegacyPassManagers.h
index 6c490791fda4..1e5b92636d0a 100644
--- a/external/llvm-project/llvm/include/llvm/IR/LegacyPassManagers.h
+++ b/external/llvm-project/llvm/include/llvm/IR/LegacyPassManagers.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 #include <vector>
 
 //===----------------------------------------------------------------------===//
@@ -106,7 +107,7 @@ enum PassDebuggingString {
 
 /// PassManagerPrettyStackEntry - This is used to print informative information
 /// about what pass is running when/if a stack trace is generated.
-class PassManagerPrettyStackEntry : public PrettyStackTraceEntry {
+class LLVM_ABI PassManagerPrettyStackEntry : public PrettyStackTraceEntry {
   Pass *P;
   Value *V;
   Module *M;
@@ -139,12 +140,12 @@ class PMStack {
   iterator begin() const { return S.rbegin(); }
   iterator end() const { return S.rend(); }
 
-  void pop();
+  LLVM_ABI void pop();
   PMDataManager *top() const { return S.back(); }
-  void push(PMDataManager *PM);
+  LLVM_ABI void push(PMDataManager *PM);
   bool empty() const { return S.empty(); }
 
-  void dump() const;
+  LLVM_ABI void dump() const;
 
 private:
   std::vector<PMDataManager *> S;
@@ -155,7 +156,7 @@ class PMStack {
 //
 /// PMTopLevelManager manages LastUser info and collects common APIs used by
 /// top level pass managers.
-class PMTopLevelManager {
+class LLVM_ABI PMTopLevelManager {
 protected:
   explicit PMTopLevelManager(PMDataManager *PMDM);
 
@@ -292,7 +293,7 @@ class PMTopLevelManager {
 
 /// PMDataManager provides the common place to manage the analysis data
 /// used by pass managers.
-class PMDataManager {
+class LLVM_ABI PMDataManager {
 public:
   explicit PMDataManager() { initializeAnalysisInfo(); }
 
@@ -454,7 +455,7 @@ class PMDataManager {
 /// It batches all function passes and basic block pass managers together and
 /// sequence them to process one function at a time before processing next
 /// function.
-class FPPassManager : public ModulePass, public PMDataManager {
+class LLVM_ABI FPPassManager : public ModulePass, public PMDataManager {
 public:
   static char ID;
   explicit FPPassManager() : ModulePass(ID) {}
@@ -508,7 +509,6 @@ class FPPassManager : public ModulePass, public PMDataManager {
     return PMT_FunctionPassManager;
   }
 };
-
 }
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/IR/LegacyPassNameParser.h b/external/llvm-project/llvm/include/llvm/IR/LegacyPassNameParser.h
index c33b9fc40472..3460de0d9022 100644
--- a/external/llvm-project/llvm/include/llvm/IR/LegacyPassNameParser.h
+++ b/external/llvm-project/llvm/include/llvm/IR/LegacyPassNameParser.h
@@ -28,6 +28,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstring>
@@ -38,8 +39,8 @@ namespace llvm {
 // PassNameParser class - Make use of the pass registration mechanism to
 // automatically add a command line argument to opt for each pass.
 //
-class PassNameParser : public PassRegistrationListener,
-                       public cl::parser<const PassInfo*> {
+class LLVM_ABI PassNameParser : public PassRegistrationListener,
+                                public cl::parser<const PassInfo *> {
 public:
   PassNameParser(cl::Option &O);
   ~PassNameParser() override;
diff --git a/external/llvm-project/llvm/include/llvm/IR/MDBuilder.h b/external/llvm-project/llvm/include/llvm/IR/MDBuilder.h
index ce4e1da65604..df1c66cf24fc 100644
--- a/external/llvm-project/llvm/include/llvm/IR/MDBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/IR/MDBuilder.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/GlobalValue.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
 #include <utility>
 
@@ -40,10 +41,10 @@ class MDBuilder {
   MDBuilder(LLVMContext &context) : Context(context) {}
 
   /// Return the given string as metadata.
-  MDString *createString(StringRef Str);
+  LLVM_ABI MDString *createString(StringRef Str);
 
   /// Return the given constant as metadata.
-  ConstantAsMetadata *createConstant(Constant *C);
+  LLVM_ABI ConstantAsMetadata *createConstant(Constant *C);
 
   //===------------------------------------------------------------------===//
   // FPMath metadata.
@@ -52,7 +53,7 @@ class MDBuilder {
   /// Return metadata with the given settings.  The special value 0.0
   /// for the Accuracy parameter indicates the default (maximal precision)
   /// setting.
-  MDNode *createFPMath(float Accuracy);
+  LLVM_ABI MDNode *createFPMath(float Accuracy);
 
   //===------------------------------------------------------------------===//
   // Prof metadata.
@@ -62,41 +63,44 @@ class MDBuilder {
   /// @param TrueWeight the weight of the true branch
   /// @param FalseWeight the weight of the false branch
   /// @param Do these weights come from __builtin_expect*
-  MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight,
-                              bool IsExpected = false);
+  LLVM_ABI MDNode *createBranchWeights(uint32_t TrueWeight,
+                                       uint32_t FalseWeight,
+                                       bool IsExpected = false);
 
   /// Return metadata containing two branch weights, with significant bias
   /// towards `true` destination.
-  MDNode *createLikelyBranchWeights();
+  LLVM_ABI MDNode *createLikelyBranchWeights();
 
   /// Return metadata containing two branch weights, with significant bias
   /// towards `false` destination.
-  MDNode *createUnlikelyBranchWeights();
+  LLVM_ABI MDNode *createUnlikelyBranchWeights();
 
   /// Return metadata containing a number of branch weights.
   /// @param Weights the weights of all the branches
   /// @param Do these weights come from __builtin_expect*
-  MDNode *createBranchWeights(ArrayRef<uint32_t> Weights,
-                              bool IsExpected = false);
+  LLVM_ABI MDNode *createBranchWeights(ArrayRef<uint32_t> Weights,
+                                       bool IsExpected = false);
 
   /// Return metadata specifying that a branch or switch is unpredictable.
-  MDNode *createUnpredictable();
+  LLVM_ABI MDNode *createUnpredictable();
 
   /// Return metadata containing the entry \p Count for a function, a boolean
   /// \Synthetic indicating whether the counts were synthetized, and the
   /// GUIDs stored in \p Imports that need to be imported for sample PGO, to
   /// enable the same inlines as the profiled optimized binary
-  MDNode *createFunctionEntryCount(uint64_t Count, bool Synthetic,
-                                   const DenseSet<GlobalValue::GUID> *Imports);
+  LLVM_ABI MDNode *
+  createFunctionEntryCount(uint64_t Count, bool Synthetic,
+                           const DenseSet<GlobalValue::GUID> *Imports);
 
   /// Return metadata containing the section prefix for a global object.
-  MDNode *createGlobalObjectSectionPrefix(StringRef Prefix);
+  LLVM_ABI MDNode *createGlobalObjectSectionPrefix(StringRef Prefix);
 
   /// Return metadata containing the pseudo probe descriptor for a function.
-  MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, StringRef FName);
+  LLVM_ABI MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash,
+                                         StringRef FName);
 
   /// Return metadata containing llvm statistics.
-  MDNode *
+  LLVM_ABI MDNode *
   createLLVMStats(ArrayRef<std::pair<StringRef, uint64_t>> LLVMStatsVec);
 
   //===------------------------------------------------------------------===//
@@ -104,10 +108,10 @@ class MDBuilder {
   //===------------------------------------------------------------------===//
 
   /// Return metadata describing the range [Lo, Hi).
-  MDNode *createRange(const APInt &Lo, const APInt &Hi);
+  LLVM_ABI MDNode *createRange(const APInt &Lo, const APInt &Hi);
 
   /// Return metadata describing the range [Lo, Hi).
-  MDNode *createRange(Constant *Lo, Constant *Hi);
+  LLVM_ABI MDNode *createRange(Constant *Lo, Constant *Hi);
 
   //===------------------------------------------------------------------===//
   // Callees metadata.
@@ -115,22 +119,25 @@ class MDBuilder {
 
   /// Return metadata indicating the possible callees of indirect
   /// calls.
-  MDNode *createCallees(ArrayRef<Function *> Callees);
+  LLVM_ABI MDNode *createCallees(ArrayRef<Function *> Callees);
 
   //===------------------------------------------------------------------===//
   // Callback metadata.
   //===------------------------------------------------------------------===//
 
   /// Return metadata describing a callback (see llvm::AbstractCallSite).
-  MDNode *createCallbackEncoding(unsigned CalleeArgNo, ArrayRef<int> Arguments,
-                                 bool VarArgsArePassed);
+  LLVM_ABI MDNode *createCallbackEncoding(unsigned CalleeArgNo,
+                                          ArrayRef<int> Arguments,
+                                          bool VarArgsArePassed);
 
   /// Merge the new callback encoding \p NewCB into \p ExistingCallbacks.
-  MDNode *mergeCallbackEncodings(MDNode *ExistingCallbacks, MDNode *NewCB);
+  LLVM_ABI MDNode *mergeCallbackEncodings(MDNode *ExistingCallbacks,
+                                          MDNode *NewCB);
 
   /// Return metadata feeding to the CodeGen about how to generate a function
   /// prologue for the "function" santizier.
-  MDNode *createRTTIPointerPrologue(Constant *PrologueSig, Constant *RTTI);
+  LLVM_ABI MDNode *createRTTIPointerPrologue(Constant *PrologueSig,
+                                             Constant *RTTI);
 
   //===------------------------------------------------------------------===//
   // PC sections metadata.
@@ -140,7 +147,7 @@ class MDBuilder {
   using PCSection = std::pair<StringRef, SmallVector<Constant *>>;
 
   /// Return metadata for PC sections.
-  MDNode *createPCSections(ArrayRef<PCSection> Sections);
+  LLVM_ABI MDNode *createPCSections(ArrayRef<PCSection> Sections);
 
   //===------------------------------------------------------------------===//
   // AA metadata.
@@ -150,8 +157,8 @@ class MDBuilder {
   /// Return metadata appropriate for a AA root node (scope or TBAA).
   /// Each returned node is distinct from all other metadata and will never
   /// be identified (uniqued) with anything else.
-  MDNode *createAnonymousAARoot(StringRef Name = StringRef(),
-                                MDNode *Extra = nullptr);
+  LLVM_ABI MDNode *createAnonymousAARoot(StringRef Name = StringRef(),
+                                         MDNode *Extra = nullptr);
 
 public:
   /// Return metadata appropriate for a TBAA root node. Each returned
@@ -179,22 +186,22 @@ class MDBuilder {
   /// Return metadata appropriate for a TBAA root node with the given
   /// name.  This may be identified (uniqued) with other roots with the same
   /// name.
-  MDNode *createTBAARoot(StringRef Name);
+  LLVM_ABI MDNode *createTBAARoot(StringRef Name);
 
   /// Return metadata appropriate for an alias scope domain node with
   /// the given name. This may be identified (uniqued) with other roots with
   /// the same name.
-  MDNode *createAliasScopeDomain(StringRef Name);
+  LLVM_ABI MDNode *createAliasScopeDomain(StringRef Name);
 
   /// Return metadata appropriate for an alias scope node with
   /// the given name. This may be identified (uniqued) with other scopes with
   /// the same name and domain.
-  MDNode *createAliasScope(StringRef Name, MDNode *Domain);
+  LLVM_ABI MDNode *createAliasScope(StringRef Name, MDNode *Domain);
 
   /// Return metadata for a non-root TBAA node with the given name,
   /// parent in the TBAA tree, and value for 'pointsToConstantMemory'.
-  MDNode *createTBAANode(StringRef Name, MDNode *Parent,
-                         bool isConstant = false);
+  LLVM_ABI MDNode *createTBAANode(StringRef Name, MDNode *Parent,
+                                  bool isConstant = false);
 
   struct TBAAStructField {
     uint64_t Offset;
@@ -206,44 +213,45 @@ class MDBuilder {
 
   /// Return metadata for a tbaa.struct node with the given
   /// struct field descriptions.
-  MDNode *createTBAAStructNode(ArrayRef<TBAAStructField> Fields);
+  LLVM_ABI MDNode *createTBAAStructNode(ArrayRef<TBAAStructField> Fields);
 
   /// Return metadata for a TBAA struct node in the type DAG
   /// with the given name, a list of pairs (offset, field type in the type DAG).
-  MDNode *
+  LLVM_ABI MDNode *
   createTBAAStructTypeNode(StringRef Name,
                            ArrayRef<std::pair<MDNode *, uint64_t>> Fields);
 
   /// Return metadata for a TBAA scalar type node with the
   /// given name, an offset and a parent in the TBAA type DAG.
-  MDNode *createTBAAScalarTypeNode(StringRef Name, MDNode *Parent,
-                                   uint64_t Offset = 0);
+  LLVM_ABI MDNode *createTBAAScalarTypeNode(StringRef Name, MDNode *Parent,
+                                            uint64_t Offset = 0);
 
   /// Return metadata for a TBAA tag node with the given
   /// base type, access type and offset relative to the base type.
-  MDNode *createTBAAStructTagNode(MDNode *BaseType, MDNode *AccessType,
-                                  uint64_t Offset, bool IsConstant = false);
+  LLVM_ABI MDNode *createTBAAStructTagNode(MDNode *BaseType, MDNode *AccessType,
+                                           uint64_t Offset,
+                                           bool IsConstant = false);
 
   /// Return metadata for a TBAA type node in the TBAA type DAG with the
   /// given parent type, size in bytes, type identifier and a list of fields.
-  MDNode *createTBAATypeNode(MDNode *Parent, uint64_t Size, Metadata *Id,
-                             ArrayRef<TBAAStructField> Fields =
-                                 ArrayRef<TBAAStructField>());
+  LLVM_ABI MDNode *createTBAATypeNode(
+      MDNode *Parent, uint64_t Size, Metadata *Id,
+      ArrayRef<TBAAStructField> Fields = ArrayRef<TBAAStructField>());
 
   /// Return metadata for a TBAA access tag with the given base type,
   /// final access type, offset of the access relative to the base type, size of
   /// the access and flag indicating whether the accessed object can be
   /// considered immutable for the purposes of the TBAA analysis.
-  MDNode *createTBAAAccessTag(MDNode *BaseType, MDNode *AccessType,
-                              uint64_t Offset, uint64_t Size,
-                              bool IsImmutable = false);
+  LLVM_ABI MDNode *createTBAAAccessTag(MDNode *BaseType, MDNode *AccessType,
+                                       uint64_t Offset, uint64_t Size,
+                                       bool IsImmutable = false);
 
   /// Return mutable version of the given mutable or immutable TBAA
   /// access tag.
-  MDNode *createMutableTBAAAccessTag(MDNode *Tag);
+  LLVM_ABI MDNode *createMutableTBAAAccessTag(MDNode *Tag);
 
   /// Return metadata containing an irreducible loop header weight.
-  MDNode *createIrrLoopHeaderWeight(uint64_t Weight);
+  LLVM_ABI MDNode *createIrrLoopHeaderWeight(uint64_t Weight);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/IR/Mangler.h b/external/llvm-project/llvm/include/llvm/IR/Mangler.h
index edbd0a5efb5d..232101a8926b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Mangler.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Mangler.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -25,7 +26,16 @@ class Triple;
 class Twine;
 class raw_ostream;
 
-constexpr std::string_view HybridPatchableTargetSuffix = "$hp_target";
+// TODO: The weird assignment of HybridPatchableTargetSuffix below is a
+// temporary workaround for a linker failure that is only hit when compiling
+// llvm for arm64ec on windows. The description and context of the issue is at
+// https://github.com/llvm/llvm-project/issues/143575.
+// An upstream MSVC bug is filed at
+// https://developercommunity.visualstudio.com/t/MSVC-Linker-Issue-When-Cross-
+// Compiling-L/10920141.
+constexpr char HybridPatchableTargetSuffixArr[] = "$hp_target";
+constexpr std::string_view HybridPatchableTargetSuffix =
+    HybridPatchableTargetSuffixArr;
 
 class Mangler {
   /// We need to give global values the same name every time they are mangled.
@@ -36,32 +46,37 @@ class Mangler {
   /// Print the appropriate prefix and the specified global variable's name.
   /// If the global variable doesn't have a name, this fills in a unique name
   /// for the global.
-  void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV,
-                         bool CannotUsePrivateLabel) const;
-  void getNameWithPrefix(SmallVectorImpl<char> &OutName, const GlobalValue *GV,
-                         bool CannotUsePrivateLabel) const;
+  LLVM_ABI void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV,
+                                  bool CannotUsePrivateLabel) const;
+  LLVM_ABI void getNameWithPrefix(SmallVectorImpl<char> &OutName,
+                                  const GlobalValue *GV,
+                                  bool CannotUsePrivateLabel) const;
 
   /// Print the appropriate prefix and the specified name as the global variable
   /// name. GVName must not be empty.
-  static void getNameWithPrefix(raw_ostream &OS, const Twine &GVName,
-                                const DataLayout &DL);
-  static void getNameWithPrefix(SmallVectorImpl<char> &OutName,
-                                const Twine &GVName, const DataLayout &DL);
+  LLVM_ABI static void getNameWithPrefix(raw_ostream &OS, const Twine &GVName,
+                                         const DataLayout &DL);
+  LLVM_ABI static void getNameWithPrefix(SmallVectorImpl<char> &OutName,
+                                         const Twine &GVName,
+                                         const DataLayout &DL);
 };
 
-void emitLinkerFlagsForGlobalCOFF(raw_ostream &OS, const GlobalValue *GV,
-                                  const Triple &TT, Mangler &Mangler);
+LLVM_ABI void emitLinkerFlagsForGlobalCOFF(raw_ostream &OS,
+                                           const GlobalValue *GV,
+                                           const Triple &TT, Mangler &Mangler);
 
-void emitLinkerFlagsForUsedCOFF(raw_ostream &OS, const GlobalValue *GV,
-                                const Triple &T, Mangler &M);
+LLVM_ABI void emitLinkerFlagsForUsedCOFF(raw_ostream &OS, const GlobalValue *GV,
+                                         const Triple &T, Mangler &M);
 
 /// Returns the ARM64EC mangled function name unless the input is already
 /// mangled.
-std::optional<std::string> getArm64ECMangledFunctionName(StringRef Name);
+LLVM_ABI std::optional<std::string>
+getArm64ECMangledFunctionName(StringRef Name);
 
 /// Returns the ARM64EC demangled function name, unless the input is not
 /// mangled.
-std::optional<std::string> getArm64ECDemangledFunctionName(StringRef Name);
+LLVM_ABI std::optional<std::string>
+getArm64ECDemangledFunctionName(StringRef Name);
 
 /// Check if an ARM64EC function name is mangled.
 bool inline isArm64ECMangledFunctionName(StringRef Name) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/MemoryModelRelaxationAnnotations.h b/external/llvm-project/llvm/include/llvm/IR/MemoryModelRelaxationAnnotations.h
index a9ded6034d0b..535635a9ad9b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/MemoryModelRelaxationAnnotations.h
+++ b/external/llvm-project/llvm/include/llvm/IR/MemoryModelRelaxationAnnotations.h
@@ -20,6 +20,7 @@
 
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <tuple> // for std::pair
 
 namespace llvm {
@@ -52,8 +53,8 @@ class MMRAMetadata {
   /// \name Constructors
   /// @{
   MMRAMetadata() = default;
-  MMRAMetadata(const Instruction &I);
-  MMRAMetadata(MDNode *MD);
+  LLVM_ABI MMRAMetadata(const Instruction &I);
+  LLVM_ABI MMRAMetadata(MDNode *MD);
   /// @}
 
   /// \name Metadata Helpers & Builders
@@ -61,25 +62,25 @@ class MMRAMetadata {
 
   /// Combines \p A and \p B according to MMRA semantics.
   /// \returns !mmra metadata for the combined MMRAs.
-  static MDNode *combine(LLVMContext &Ctx, const MMRAMetadata &A,
-                         const MMRAMetadata &B);
+  LLVM_ABI static MDNode *combine(LLVMContext &Ctx, const MMRAMetadata &A,
+                                  const MMRAMetadata &B);
 
   /// Creates !mmra metadata for a single tag.
   ///
   /// !mmra metadata can either be a single tag, or a MDTuple containing
   /// multiple tags.
-  static MDTuple *getTagMD(LLVMContext &Ctx, StringRef Prefix,
-                           StringRef Suffix);
+  LLVM_ABI static MDTuple *getTagMD(LLVMContext &Ctx, StringRef Prefix,
+                                    StringRef Suffix);
   static MDTuple *getTagMD(LLVMContext &Ctx, const TagT &T) {
     return getTagMD(Ctx, T.first, T.second);
   }
 
   /// Creates !mmra metadata from \p Tags.
   /// \returns nullptr or a MDTuple* from \p Tags.
-  static MDTuple *getMD(LLVMContext &Ctx, ArrayRef<TagT> Tags);
+  LLVM_ABI static MDTuple *getMD(LLVMContext &Ctx, ArrayRef<TagT> Tags);
 
   /// \returns true if \p MD is a well-formed MMRA tag.
-  static bool isTagMD(const Metadata *MD);
+  LLVM_ABI static bool isTagMD(const Metadata *MD);
 
   /// @}
 
@@ -92,25 +93,25 @@ class MMRAMetadata {
   }
 
   /// \returns whether this set of tags is compatible with \p Other.
-  bool isCompatibleWith(const MMRAMetadata &Other) const;
+  LLVM_ABI bool isCompatibleWith(const MMRAMetadata &Other) const;
 
   /// @}
 
   /// \name Content Queries
   /// @{
 
-  bool hasTag(StringRef Prefix, StringRef Suffix) const;
-  bool hasTagWithPrefix(StringRef Prefix) const;
+  LLVM_ABI bool hasTag(StringRef Prefix, StringRef Suffix) const;
+  LLVM_ABI bool hasTagWithPrefix(StringRef Prefix) const;
 
-  const_iterator begin() const;
-  const_iterator end() const;
-  bool empty() const;
-  unsigned size() const;
+  LLVM_ABI const_iterator begin() const;
+  LLVM_ABI const_iterator end() const;
+  LLVM_ABI bool empty() const;
+  LLVM_ABI unsigned size() const;
 
   /// @}
 
-  void print(raw_ostream &OS) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &OS) const;
+  LLVM_ABI void dump() const;
 
   operator bool() const { return !Tags.empty(); }
   bool operator==(const MMRAMetadata &Other) const {
@@ -125,7 +126,7 @@ class MMRAMetadata {
 };
 
 /// \returns true if \p I can have !mmra metadata.
-bool canInstructionHaveMMRAs(const Instruction &I);
+LLVM_ABI bool canInstructionHaveMMRAs(const Instruction &I);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/Metadata.h b/external/llvm-project/llvm/include/llvm/IR/Metadata.h
index 17eda71cd4b6..7fc2d5d7f2be 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Metadata.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Metadata.h
@@ -28,6 +28,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 #include <cstddef>
@@ -114,8 +115,8 @@ class Metadata {
   /// the nullptr version is easy to call from a debugger.
   ///
   /// @{
-  void dump() const;
-  void dump(const Module *M) const;
+  LLVM_ABI void dump() const;
+  LLVM_ABI void dump(const Module *M) const;
   /// @}
 
   /// Print.
@@ -125,10 +126,10 @@ class Metadata {
   /// If \c M is provided, metadata nodes will be numbered canonically;
   /// otherwise, pointer addresses are substituted.
   /// @{
-  void print(raw_ostream &OS, const Module *M = nullptr,
-             bool IsForDebug = false) const;
-  void print(raw_ostream &OS, ModuleSlotTracker &MST, const Module *M = nullptr,
-             bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &OS, const Module *M = nullptr,
+                      bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &OS, ModuleSlotTracker &MST,
+                      const Module *M = nullptr, bool IsForDebug = false) const;
   /// @}
 
   /// Print as operand.
@@ -138,9 +139,10 @@ class Metadata {
   /// If \c M is provided, metadata nodes will be numbered canonically;
   /// otherwise, pointer addresses are substituted.
   /// @{
-  void printAsOperand(raw_ostream &OS, const Module *M = nullptr) const;
-  void printAsOperand(raw_ostream &OS, ModuleSlotTracker &MST,
-                      const Module *M = nullptr) const;
+  LLVM_ABI void printAsOperand(raw_ostream &OS,
+                               const Module *M = nullptr) const;
+  LLVM_ABI void printAsOperand(raw_ostream &OS, ModuleSlotTracker &MST,
+                               const Module *M = nullptr) const;
   /// @}
 
   /// Metadata IDs that may generate poison.
@@ -193,10 +195,11 @@ class MetadataAsValue : public Value {
   void dropUse() { MD = nullptr; }
 
 public:
-  ~MetadataAsValue();
+  LLVM_ABI ~MetadataAsValue();
 
-  static MetadataAsValue *get(LLVMContext &Context, Metadata *MD);
-  static MetadataAsValue *getIfExists(LLVMContext &Context, Metadata *MD);
+  LLVM_ABI static MetadataAsValue *get(LLVMContext &Context, Metadata *MD);
+  LLVM_ABI static MetadataAsValue *getIfExists(LLVMContext &Context,
+                                               Metadata *MD);
 
   Metadata *getMetadata() const { return MD; }
 
@@ -228,15 +231,15 @@ class DebugValueUser {
   ArrayRef<Metadata *> getDebugValues() const { return DebugValues; }
 
 public:
-  DbgVariableRecord *getUser();
-  const DbgVariableRecord *getUser() const;
+  LLVM_ABI DbgVariableRecord *getUser();
+  LLVM_ABI const DbgVariableRecord *getUser() const;
   /// To be called by ReplaceableMetadataImpl::replaceAllUsesWith, where `Old`
   /// is a pointer to one of the pointers in `DebugValues` (so should be type
   /// Metadata**), and `NewDebugValue` is the new Metadata* that is replacing
   /// *Old.
   /// For manually replacing elements of DebugValues,
   /// `resetDebugValue(Idx, NewDebugValue)` should be used instead.
-  void handleChangedValue(void *Old, Metadata *NewDebugValue);
+  LLVM_ABI void handleChangedValue(void *Old, Metadata *NewDebugValue);
   DebugValueUser() = default;
   explicit DebugValueUser(std::array<Metadata *, 3> DebugValues)
       : DebugValues(DebugValues) {
@@ -293,13 +296,13 @@ class DebugValueUser {
   }
 
 private:
-  void trackDebugValue(size_t Idx);
-  void trackDebugValues();
+  LLVM_ABI void trackDebugValue(size_t Idx);
+  LLVM_ABI void trackDebugValues();
 
-  void untrackDebugValue(size_t Idx);
-  void untrackDebugValues();
+  LLVM_ABI void untrackDebugValue(size_t Idx);
+  LLVM_ABI void untrackDebugValues();
 
-  void retrackDebugValues(DebugValueUser &X);
+  LLVM_ABI void retrackDebugValues(DebugValueUser &X);
 };
 
 /// API for tracking metadata references through RAUW and deletion.
@@ -355,7 +358,7 @@ class MetadataTracking {
   ///
   /// Stops \c *MD from tracking \c MD.
   static void untrack(Metadata *&MD) { untrack(&MD, *MD); }
-  static void untrack(void *Ref, Metadata &MD);
+  LLVM_ABI static void untrack(void *Ref, Metadata &MD);
 
   /// Move tracking from one reference to another.
   ///
@@ -368,10 +371,10 @@ class MetadataTracking {
   static bool retrack(Metadata *&MD, Metadata *&New) {
     return retrack(&MD, *MD, &New);
   }
-  static bool retrack(void *Ref, Metadata &MD, void *New);
+  LLVM_ABI static bool retrack(void *Ref, Metadata &MD, void *New);
 
   /// Check whether metadata is replaceable.
-  static bool isReplaceable(const Metadata &MD);
+  LLVM_ABI static bool isReplaceable(const Metadata &MD);
 
   using OwnerTy = PointerUnion<MetadataAsValue *, Metadata *, DebugValueUser *>;
 
@@ -379,7 +382,7 @@ class MetadataTracking {
   /// Track a reference to metadata for an owner.
   ///
   /// Generalized version of tracking.
-  static bool track(void *Ref, Metadata &MD, OwnerTy Owner);
+  LLVM_ABI static bool track(void *Ref, Metadata &MD, OwnerTy Owner);
 };
 
 /// Shared implementation of use-lists for replaceable metadata.
@@ -410,20 +413,20 @@ class ReplaceableMetadataImpl {
   /// Replace all uses of this with MD.
   ///
   /// Replace all uses of this with \c MD, which is allowed to be null.
-  void replaceAllUsesWith(Metadata *MD);
-   /// Replace all uses of the constant with Undef in debug info metadata
-  static void SalvageDebugInfo(const Constant &C); 
+  LLVM_ABI void replaceAllUsesWith(Metadata *MD);
+  /// Replace all uses of the constant with Undef in debug info metadata
+  LLVM_ABI static void SalvageDebugInfo(const Constant &C);
   /// Returns the list of all DIArgList users of this.
-  SmallVector<Metadata *> getAllArgListUsers();
+  LLVM_ABI SmallVector<Metadata *> getAllArgListUsers();
   /// Returns the list of all DbgVariableRecord users of this.
-  SmallVector<DbgVariableRecord *> getAllDbgVariableRecordUsers();
+  LLVM_ABI SmallVector<DbgVariableRecord *> getAllDbgVariableRecordUsers();
 
   /// Resolve all uses of this.
   ///
   /// Resolve all uses of this, turning off RAUW permanently.  If \c
   /// ResolveUsers, call \a MDNode::resolve() on any users whose last operand
   /// is resolved.
-  void resolveAllUses(bool ResolveUsers = true);
+  LLVM_ABI void resolveAllUses(bool ResolveUsers = true);
 
   unsigned getNumUses() const { return UseMap.size(); }
 
@@ -475,7 +478,7 @@ class ValueAsMetadata : public Metadata, ReplaceableMetadataImpl {
   ~ValueAsMetadata() = default;
 
 public:
-  static ValueAsMetadata *get(Value *V);
+  LLVM_ABI static ValueAsMetadata *get(Value *V);
 
   static ConstantAsMetadata *getConstant(Value *C) {
     return cast<ConstantAsMetadata>(get(C));
@@ -485,7 +488,7 @@ class ValueAsMetadata : public Metadata, ReplaceableMetadataImpl {
     return cast<LocalAsMetadata>(get(Local));
   }
 
-  static ValueAsMetadata *getIfExists(Value *V);
+  LLVM_ABI static ValueAsMetadata *getIfExists(Value *V);
 
   static ConstantAsMetadata *getConstantIfExists(Value *C) {
     return cast_or_null<ConstantAsMetadata>(getIfExists(C));
@@ -506,8 +509,8 @@ class ValueAsMetadata : public Metadata, ReplaceableMetadataImpl {
     return ReplaceableMetadataImpl::getAllDbgVariableRecordUsers();
   }
 
-  static void handleDeletion(Value *V);
-  static void handleRAUW(Value *From, Value *To);
+  LLVM_ABI static void handleDeletion(Value *V);
+  LLVM_ABI static void handleRAUW(Value *From, Value *To);
 
 protected:
   /// Handle collisions after \a Value::replaceAllUsesWith().
@@ -730,12 +733,12 @@ class MDString : public Metadata {
   MDString &operator=(MDString &&) = delete;
   MDString &operator=(const MDString &) = delete;
 
-  static MDString *get(LLVMContext &Context, StringRef Str);
+  LLVM_ABI static MDString *get(LLVMContext &Context, StringRef Str);
   static MDString *get(LLVMContext &Context, const char *Str) {
     return get(Context, Str ? StringRef(Str) : StringRef());
   }
 
-  StringRef getString() const;
+  LLVM_ABI StringRef getString() const;
 
   unsigned getLength() const { return (unsigned)getString().size(); }
 
@@ -787,14 +790,14 @@ struct AAMDNodes {
   MDNode *NoAlias = nullptr;
 
   // Shift tbaa Metadata node to start off bytes later
-  static MDNode *shiftTBAA(MDNode *M, size_t off);
+  LLVM_ABI static MDNode *shiftTBAA(MDNode *M, size_t off);
 
   // Shift tbaa.struct Metadata node to start off bytes later
-  static MDNode *shiftTBAAStruct(MDNode *M, size_t off);
+  LLVM_ABI static MDNode *shiftTBAAStruct(MDNode *M, size_t off);
 
   // Extend tbaa Metadata node to apply to a series of bytes of length len.
   // A size of -1 denotes an unknown size.
-  static MDNode *extendToTBAA(MDNode *TBAA, ssize_t len);
+  LLVM_ABI static MDNode *extendToTBAA(MDNode *TBAA, ssize_t len);
 
   /// Given two sets of AAMDNodes that apply to the same pointer,
   /// give the best AAMDNodes that are compatible with both (i.e. a set of
@@ -839,22 +842,22 @@ struct AAMDNodes {
 
   /// Given two sets of AAMDNodes applying to potentially different locations,
   /// determine the best AAMDNodes that apply to both.
-  AAMDNodes merge(const AAMDNodes &Other) const;
+  LLVM_ABI AAMDNodes merge(const AAMDNodes &Other) const;
 
   /// Determine the best AAMDNodes after concatenating two different locations
   /// together. Different from `merge`, where different locations should
   /// overlap each other, `concat` puts non-overlapping locations together.
-  AAMDNodes concat(const AAMDNodes &Other) const;
+  LLVM_ABI AAMDNodes concat(const AAMDNodes &Other) const;
 
   /// Create a new AAMDNode for accessing \p AccessSize bytes of this AAMDNode.
   /// If this AAMDNode has !tbaa.struct and \p AccessSize matches the size of
   /// the field at offset 0, get the TBAA tag describing the accessed field.
   /// If such an AAMDNode already embeds !tbaa, the existing one is retrieved.
   /// Finally, !tbaa.struct is zeroed out.
-  AAMDNodes adjustForAccess(unsigned AccessSize);
-  AAMDNodes adjustForAccess(size_t Offset, Type *AccessTy,
-                            const DataLayout &DL);
-  AAMDNodes adjustForAccess(size_t Offset, unsigned AccessSize);
+  LLVM_ABI AAMDNodes adjustForAccess(unsigned AccessSize);
+  LLVM_ABI AAMDNodes adjustForAccess(size_t Offset, Type *AccessTy,
+                                     const DataLayout &DL);
+  LLVM_ABI AAMDNodes adjustForAccess(size_t Offset, unsigned AccessSize);
 };
 
 // Specialize DenseMapInfo for AAMDNodes.
@@ -1132,7 +1135,7 @@ class MDNode : public Metadata {
              sizeof(LargeStorageVector);
     }
 
-    void *getSmallPtr();
+    LLVM_ABI void *getSmallPtr();
 
     LargeStorageVector &getLarge() {
       assert(IsLarge);
@@ -1144,12 +1147,12 @@ class MDNode : public Metadata {
       return *reinterpret_cast<const LargeStorageVector *>(getLargePtr());
     }
 
-    void resizeSmall(size_t NumOps);
-    void resizeSmallToLarge(size_t NumOps);
-    void resize(size_t NumOps);
+    LLVM_ABI void resizeSmall(size_t NumOps);
+    LLVM_ABI void resizeSmallToLarge(size_t NumOps);
+    LLVM_ABI void resize(size_t NumOps);
 
-    explicit Header(size_t NumOps, StorageType Storage);
-    ~Header();
+    LLVM_ABI explicit Header(size_t NumOps, StorageType Storage);
+    LLVM_ABI ~Header();
 
     MutableArrayRef<MDOperand> operands() {
       if (IsLarge)
@@ -1181,12 +1184,12 @@ class MDNode : public Metadata {
   ContextAndReplaceableUses Context;
 
 protected:
-  MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
-         ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2 = {});
+  LLVM_ABI MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
+                  ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2 = {});
   ~MDNode() = default;
 
-  void *operator new(size_t Size, size_t NumOps, StorageType Storage);
-  void operator delete(void *Mem);
+  LLVM_ABI void *operator new(size_t Size, size_t NumOps, StorageType Storage);
+  LLVM_ABI void operator delete(void *Mem);
 
   /// Required by std, but never called.
   void operator delete(void *, unsigned) {
@@ -1198,7 +1201,7 @@ class MDNode : public Metadata {
     llvm_unreachable("Constructor throws?");
   }
 
-  void dropAllReferences();
+  LLVM_ABI void dropAllReferences();
 
   MDOperand *mutable_begin() { return getHeader().operands().begin(); }
   MDOperand *mutable_end() { return getHeader().operands().end(); }
@@ -1223,18 +1226,18 @@ class MDNode : public Metadata {
                                          ArrayRef<Metadata *> MDs);
 
   /// Create a (temporary) clone of this.
-  TempMDNode clone() const;
+  LLVM_ABI TempMDNode clone() const;
 
   /// Deallocate a node created by getTemporary.
   ///
   /// Calls \c replaceAllUsesWith(nullptr) before deleting, so any remaining
   /// references will be reset.
-  static void deleteTemporary(MDNode *N);
+  LLVM_ABI static void deleteTemporary(MDNode *N);
 
   LLVMContext &getContext() const { return Context.getContext(); }
 
   /// Replace a specific operand.
-  void replaceOperandWith(unsigned I, Metadata *New);
+  LLVM_ABI void replaceOperandWith(unsigned I, Metadata *New);
 
   /// Check if node is fully resolved.
   ///
@@ -1276,10 +1279,10 @@ class MDNode : public Metadata {
   /// resolved.
   ///
   /// \pre No operands (or operands' operands, etc.) have \a isTemporary().
-  void resolveCycles();
+  LLVM_ABI void resolveCycles();
 
   /// Resolve a unique, unresolved node.
-  void resolve();
+  LLVM_ABI void resolve();
 
   /// Replace a temporary node with a permanent one.
   ///
@@ -1320,9 +1323,9 @@ class MDNode : public Metadata {
   /// If \c M is provided, metadata nodes will be numbered canonically;
   /// otherwise, pointer addresses are substituted.
   /// @{
-  void printTree(raw_ostream &OS, const Module *M = nullptr) const;
-  void printTree(raw_ostream &OS, ModuleSlotTracker &MST,
-                 const Module *M = nullptr) const;
+  LLVM_ABI void printTree(raw_ostream &OS, const Module *M = nullptr) const;
+  LLVM_ABI void printTree(raw_ostream &OS, ModuleSlotTracker &MST,
+                          const Module *M = nullptr) const;
   /// @}
 
   /// User-friendly dump in tree shape.
@@ -1334,25 +1337,25 @@ class MDNode : public Metadata {
   /// the nullptr version is easy to call from a debugger.
   ///
   /// @{
-  void dumpTree() const;
-  void dumpTree(const Module *M) const;
+  LLVM_ABI void dumpTree() const;
+  LLVM_ABI void dumpTree(const Module *M) const;
   /// @}
 
 private:
-  MDNode *replaceWithPermanentImpl();
-  MDNode *replaceWithUniquedImpl();
-  MDNode *replaceWithDistinctImpl();
+  LLVM_ABI MDNode *replaceWithPermanentImpl();
+  LLVM_ABI MDNode *replaceWithUniquedImpl();
+  LLVM_ABI MDNode *replaceWithDistinctImpl();
 
 protected:
   /// Set an operand.
   ///
   /// Sets the operand directly, without worrying about uniquing.
-  void setOperand(unsigned I, Metadata *New);
+  LLVM_ABI void setOperand(unsigned I, Metadata *New);
 
   unsigned getNumUnresolved() const { return getHeader().NumUnresolved; }
 
   void setNumUnresolved(unsigned N) { getHeader().NumUnresolved = N; }
-  void storeDistinctInContext();
+  LLVM_ABI void storeDistinctInContext();
   template <class T, class StoreT>
   static T *storeImpl(T *N, StorageType Storage, StoreT &Store);
   template <class T> static T *storeImpl(T *N, StorageType Storage);
@@ -1449,24 +1452,25 @@ class MDNode : public Metadata {
   }
 
   /// Check whether MDNode is a vtable access.
-  bool isTBAAVtableAccess() const;
+  LLVM_ABI bool isTBAAVtableAccess() const;
 
   /// Methods for metadata merging.
-  static MDNode *concatenate(MDNode *A, MDNode *B);
-  static MDNode *intersect(MDNode *A, MDNode *B);
-  static MDNode *getMostGenericTBAA(MDNode *A, MDNode *B);
-  static MDNode *getMostGenericFPMath(MDNode *A, MDNode *B);
-  static MDNode *getMostGenericRange(MDNode *A, MDNode *B);
-  static MDNode *getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B);
-  static MDNode *getMostGenericAliasScope(MDNode *A, MDNode *B);
-  static MDNode *getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *concatenate(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *intersect(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMostGenericTBAA(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMostGenericFPMath(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMostGenericRange(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMostGenericAliasScope(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMostGenericAlignmentOrDereferenceable(MDNode *A,
+                                                                   MDNode *B);
   /// Merge !prof metadata from two instructions.
   /// Currently only implemented with direct callsites with branch weights.
-  static MDNode *getMergedProfMetadata(MDNode *A, MDNode *B,
-                                       const Instruction *AInstr,
-                                       const Instruction *BInstr);
-  static MDNode *getMergedMemProfMetadata(MDNode *A, MDNode *B);
-  static MDNode *getMergedCallsiteMetadata(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMergedProfMetadata(MDNode *A, MDNode *B,
+                                                const Instruction *AInstr,
+                                                const Instruction *BInstr);
+  LLVM_ABI static MDNode *getMergedMemProfMetadata(MDNode *A, MDNode *B);
+  LLVM_ABI static MDNode *getMergedCallsiteMetadata(MDNode *A, MDNode *B);
 };
 
 /// Tuple of metadata.
@@ -1488,8 +1492,10 @@ class MDTuple : public MDNode {
   void setHash(unsigned Hash) { SubclassData32 = Hash; }
   void recalculateHash();
 
-  static MDTuple *getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs,
-                          StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static MDTuple *getImpl(LLVMContext &Context,
+                                   ArrayRef<Metadata *> MDs,
+                                   StorageType Storage,
+                                   bool ShouldCreate = true);
 
   TempMDTuple cloneImpl() const {
     ArrayRef<MDOperand> Operands = operands();
@@ -1790,29 +1796,29 @@ class NamedMDNode : public ilist_node<NamedMDNode> {
 
 public:
   NamedMDNode(const NamedMDNode &) = delete;
-  ~NamedMDNode();
+  LLVM_ABI ~NamedMDNode();
 
   /// Drop all references and remove the node from parent module.
-  void eraseFromParent();
+  LLVM_ABI void eraseFromParent();
 
   /// Remove all uses and clear node vector.
   void dropAllReferences() { clearOperands(); }
   /// Drop all references to this node's operands.
-  void clearOperands();
+  LLVM_ABI void clearOperands();
 
   /// Get the module that holds this named metadata collection.
   inline Module *getParent() { return Parent; }
   inline const Module *getParent() const { return Parent; }
 
-  MDNode *getOperand(unsigned i) const;
-  unsigned getNumOperands() const;
-  void addOperand(MDNode *M);
-  void setOperand(unsigned I, MDNode *New);
-  StringRef getName() const;
-  void print(raw_ostream &ROS, bool IsForDebug = false) const;
-  void print(raw_ostream &ROS, ModuleSlotTracker &MST,
-             bool IsForDebug = false) const;
-  void dump() const;
+  LLVM_ABI MDNode *getOperand(unsigned i) const;
+  LLVM_ABI unsigned getNumOperands() const;
+  LLVM_ABI void addOperand(MDNode *M);
+  LLVM_ABI void setOperand(unsigned I, MDNode *New);
+  LLVM_ABI StringRef getName() const;
+  LLVM_ABI void print(raw_ostream &ROS, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug = false) const;
+  LLVM_ABI void dump() const;
 
   // ---------------------------------------------------------------------------
   // Operand Iterator interface...
diff --git a/external/llvm-project/llvm/include/llvm/IR/Module.h b/external/llvm-project/llvm/include/llvm/IR/Module.h
index 0dc8164b9c95..f4420f460741 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Module.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Module.h
@@ -31,6 +31,7 @@
 #include "llvm/IR/SymbolTableListTraits.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cstddef>
 #include <cstdint>
@@ -214,11 +215,6 @@ class LLVM_ABI Module {
 /// @name Constructors
 /// @{
 public:
-  /// Is this Module using intrinsics to record the position of debugging
-  /// information, or non-intrinsic records? See IsNewDbgInfoFormat in
-  /// \ref BasicBlock.
-  bool IsNewDbgInfoFormat;
-
   /// Used when printing this module in the new debug info format; removes all
   /// declarations of debug intrinsics that are replaced by non-intrinsic
   /// records in the new format.
@@ -229,7 +225,6 @@ class LLVM_ABI Module {
     for (auto &F : *this) {
       F.convertToNewDbgValues();
     }
-    IsNewDbgInfoFormat = true;
   }
 
   /// \see BasicBlock::convertFromNewDbgValues.
@@ -237,20 +232,6 @@ class LLVM_ABI Module {
     for (auto &F : *this) {
       F.convertFromNewDbgValues();
     }
-    IsNewDbgInfoFormat = false;
-  }
-
-  void setIsNewDbgInfoFormat(bool UseNewFormat) {
-    if (UseNewFormat && !IsNewDbgInfoFormat)
-      convertToNewDbgValues();
-    else if (!UseNewFormat && IsNewDbgInfoFormat)
-      convertFromNewDbgValues();
-  }
-  void setNewDbgInfoFormatFlag(bool NewFlag) {
-    for (auto &F : *this) {
-      F.setNewDbgInfoFormatFlag(NewFlag);
-    }
-    IsNewDbgInfoFormat = NewFlag;
   }
 
   /// The Module constructor. Note that there is no default constructor. You
@@ -817,7 +798,7 @@ class LLVM_ABI Module {
     NamedMDNode *CUs;
     unsigned Idx;
 
-    void SkipNoDebugCUs();
+    LLVM_ABI void SkipNoDebugCUs();
 
   public:
     using iterator_category = std::input_iterator_tag;
@@ -851,8 +832,8 @@ class LLVM_ABI Module {
       return Idx != I.Idx;
     }
 
-    DICompileUnit *operator*() const;
-    DICompileUnit *operator->() const;
+    LLVM_ABI DICompileUnit *operator*() const;
+    LLVM_ABI DICompileUnit *operator->() const;
   };
 
   debug_compile_units_iterator debug_compile_units_begin() const {
@@ -1065,9 +1046,9 @@ class LLVM_ABI Module {
 /// Given "llvm.used" or "llvm.compiler.used" as a global name, collect the
 /// initializer elements of that global in a SmallVector and return the global
 /// itself.
-GlobalVariable *collectUsedGlobalVariables(const Module &M,
-                                           SmallVectorImpl<GlobalValue *> &Vec,
-                                           bool CompilerUsed);
+LLVM_ABI GlobalVariable *
+collectUsedGlobalVariables(const Module &M, SmallVectorImpl<GlobalValue *> &Vec,
+                           bool CompilerUsed);
 
 /// An raw_ostream inserter for modules.
 inline raw_ostream &operator<<(raw_ostream &O, const Module &M) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/ModuleSlotTracker.h b/external/llvm-project/llvm/include/llvm/IR/ModuleSlotTracker.h
index 37cfc0f07280..a3882a81e117 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ModuleSlotTracker.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ModuleSlotTracker.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_IR_MODULESLOTTRACKER_H
 #define LLVM_IR_MODULESLOTTRACKER_H
 
+#include "llvm/Support/Compiler.h"
 #include <functional>
 #include <memory>
 #include <utility>
@@ -23,7 +24,7 @@ class Value;
 class MDNode;
 
 /// Abstract interface of slot tracker storage.
-class AbstractSlotTrackerStorage {
+class LLVM_ABI AbstractSlotTrackerStorage {
 public:
   virtual ~AbstractSlotTrackerStorage();
 
@@ -41,7 +42,7 @@ class AbstractSlotTrackerStorage {
 ///
 /// If the IR changes from underneath \a ModuleSlotTracker, strings like
 /// "<badref>" will be printed, or, worse, the wrong slots entirely.
-class ModuleSlotTracker {
+class LLVM_ABI ModuleSlotTracker {
   /// Storage for a slot tracker.
   std::unique_ptr<SlotTracker> MachineStorage;
   bool ShouldCreateStorage = false;
diff --git a/external/llvm-project/llvm/include/llvm/IR/ModuleSummaryIndex.h b/external/llvm-project/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 65e428a3adea..ac79d91d417c 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -249,14 +249,14 @@ struct ValueInfo {
   /// Returns the most constraining visibility among summaries. The
   /// visibilities, ordered from least to most constraining, are: default,
   /// protected and hidden.
-  GlobalValue::VisibilityTypes getELFVisibility() const;
+  LLVM_ABI GlobalValue::VisibilityTypes getELFVisibility() const;
 
   /// Checks if all summaries are DSO local (have the flag set). When DSOLocal
   /// propagation has been done, set the parameter to enable fast check.
-  bool isDSOLocal(bool WithDSOLocalPropagation = false) const;
+  LLVM_ABI bool isDSOLocal(bool WithDSOLocalPropagation = false) const;
 
   /// Checks if all copies are eligible for auto-hiding (have flag set).
-  bool canAutoHide() const;
+  LLVM_ABI bool canAutoHide() const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const ValueInfo &VI) {
@@ -873,7 +873,7 @@ class FunctionSummary : public GlobalValueSummary {
   }
 
   /// A dummy node to reference external functions that aren't in the index
-  static FunctionSummary ExternalNode;
+  LLVM_ABI static FunctionSummary ExternalNode;
 
 private:
   /// Number of instructions (ignoring debug instructions, e.g.) computed
@@ -943,7 +943,7 @@ class FunctionSummary : public GlobalValueSummary {
       Allocs = std::make_unique<AllocsTy>(std::move(AllocList));
   }
   // Gets the number of readonly and writeonly refs in RefEdgeList
-  std::pair<unsigned, unsigned> specialRefCounts() const;
+  LLVM_ABI std::pair<unsigned, unsigned> specialRefCounts() const;
 
   /// Check if this is a function summary.
   static bool classof(const GlobalValueSummary *GVS) {
@@ -1527,8 +1527,8 @@ class ModuleSummaryIndex {
 
   bool haveGVs() const { return HaveGVs; }
 
-  uint64_t getFlags() const;
-  void setFlags(uint64_t Flags);
+  LLVM_ABI uint64_t getFlags() const;
+  LLVM_ABI void setFlags(uint64_t Flags);
 
   uint64_t getBlockCount() const { return BlockCount; }
   void addBlockCount(uint64_t C) { BlockCount += C; }
@@ -1672,7 +1672,7 @@ class ModuleSummaryIndex {
   bool isGlobalValueLive(const GlobalValueSummary *GVS) const {
     return !WithGlobalValueDeadStripping || GVS->isLive();
   }
-  bool isGUIDLive(GlobalValue::GUID GUID) const;
+  LLVM_ABI bool isGUIDLive(GlobalValue::GUID GUID) const;
 
   /// Return a ValueInfo for the index value_type (convenient when iterating
   /// index).
@@ -1796,8 +1796,9 @@ class ModuleSummaryIndex {
   /// Returns the first GlobalValueSummary for \p ValueGUID, asserting that
   /// there
   /// is only one if \p PerModuleIndex.
-  GlobalValueSummary *getGlobalValueSummary(GlobalValue::GUID ValueGUID,
-                                            bool PerModuleIndex = true) const;
+  LLVM_ABI GlobalValueSummary *
+  getGlobalValueSummary(GlobalValue::GUID ValueGUID,
+                        bool PerModuleIndex = true) const;
 
   /// Table of modules, containing module hash and id.
   const StringMap<ModuleHash> &modulePaths() const {
@@ -1925,8 +1926,9 @@ class ModuleSummaryIndex {
 
   /// Collect for the given module the list of functions it defines
   /// (GUID -> Summary).
-  void collectDefinedFunctionsForModule(StringRef ModulePath,
-                                        GVSummaryMapTy &GVSummaryMap) const;
+  LLVM_ABI void
+  collectDefinedFunctionsForModule(StringRef ModulePath,
+                                   GVSummaryMapTy &GVSummaryMap) const;
 
   /// Collect for each module the list of Summaries it defines (GUID ->
   /// Summary).
@@ -1942,29 +1944,31 @@ class ModuleSummaryIndex {
   }
 
   /// Print to an output stream.
-  void print(raw_ostream &OS, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &OS, bool IsForDebug = false) const;
 
   /// Dump to stderr (for debugging).
-  void dump() const;
+  LLVM_ABI void dump() const;
 
   /// Export summary to dot file for GraphViz.
-  void
+  LLVM_ABI void
   exportToDot(raw_ostream &OS,
               const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) const;
 
   /// Print out strongly connected components for debugging.
-  void dumpSCCs(raw_ostream &OS);
+  LLVM_ABI void dumpSCCs(raw_ostream &OS);
 
   /// Do the access attribute and DSOLocal propagation in combined index.
-  void propagateAttributes(const DenseSet<GlobalValue::GUID> &PreservedSymbols);
+  LLVM_ABI void
+  propagateAttributes(const DenseSet<GlobalValue::GUID> &PreservedSymbols);
 
   /// Checks if we can import global variable from another module.
-  bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs) const;
+  LLVM_ABI bool canImportGlobalVar(const GlobalValueSummary *S,
+                                   bool AnalyzeRefs) const;
 
   /// Same as above but checks whether the global var is importable as a
   /// declaration.
-  bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs,
-                          bool &CanImportDecl) const;
+  LLVM_ABI bool canImportGlobalVar(const GlobalValueSummary *S,
+                                   bool AnalyzeRefs, bool &CanImportDecl) const;
 };
 
 /// GraphTraits definition to build SCC for the index
diff --git a/external/llvm-project/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/external/llvm-project/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index ce794e257363..737610b73b08 100644
--- a/external/llvm-project/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/external/llvm-project/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -38,6 +38,15 @@ enum class TMAReductionOp : uint8_t {
   XOR = 7,
 };
 
+// Enum to represent the cta_group::1 and
+// cta_group::2 variants in TMA/TCGEN05 family of
+// PTX instructions.
+enum class CTAGroupKind : uint8_t {
+  CG_NONE = 0, // default with no cta_group modifier
+  CG_1 = 1,    // cta_group::1 modifier
+  CG_2 = 2,    // cta_group::2 modifier
+};
+
 inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
   switch (IntrinsicID) {
   case Intrinsic::nvvm_f2i_rm_ftz:
diff --git a/external/llvm-project/llvm/include/llvm/IR/NoFolder.h b/external/llvm-project/llvm/include/llvm/IR/NoFolder.h
index 2a2318dfd786..9f16c6983313 100644
--- a/external/llvm-project/llvm/include/llvm/IR/NoFolder.h
+++ b/external/llvm-project/llvm/include/llvm/IR/NoFolder.h
@@ -28,11 +28,12 @@
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// NoFolder - Create "constants" (actually, instructions) with no folding.
-class NoFolder final : public IRBuilderFolder {
+class LLVM_ABI NoFolder final : public IRBuilderFolder {
   LLVM_DECLARE_VIRTUAL_ANCHOR_FUNCTION();
 
 public:
diff --git a/external/llvm-project/llvm/include/llvm/IR/Operator.h b/external/llvm-project/llvm/include/llvm/IR/Operator.h
index 38f84647ec74..8344eaec807b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Operator.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Operator.h
@@ -22,6 +22,7 @@
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include <cstddef>
 #include <optional>
 
@@ -63,12 +64,12 @@ class Operator : public User {
 
   /// Return true if this operator has flags which may cause this operator
   /// to evaluate to poison despite having non-poison inputs.
-  bool hasPoisonGeneratingFlags() const;
+  LLVM_ABI bool hasPoisonGeneratingFlags() const;
 
   /// Return true if this operator has poison-generating flags,
   /// return attributes or metadata. The latter two is only possible for
   /// instructions.
-  bool hasPoisonGeneratingAnnotations() const;
+  LLVM_ABI bool hasPoisonGeneratingAnnotations() const;
 };
 
 /// Utility class for integer operators which may exhibit overflow - Add, Sub,
@@ -336,7 +337,7 @@ class FPMathOperator : public Operator {
   /// Get the maximum error permitted by this operation in ULPs. An accuracy of
   /// 0.0 means that the operation should be performed with the default
   /// precision.
-  float getFPAccuracy() const;
+  LLVM_ABI float getFPAccuracy() const;
 
   /// Returns true if `Ty` is a supported floating-point type for phi, select,
   /// or call FPMathOperators.
@@ -438,7 +439,7 @@ class GEPOperator
 
   /// Returns the offset of the index with an inrange attachment, or
   /// std::nullopt if none.
-  std::optional<ConstantRange> getInRange() const;
+  LLVM_ABI std::optional<ConstantRange> getInRange() const;
 
   inline op_iterator       idx_begin()       { return op_begin()+1; }
   inline const_op_iterator idx_begin() const { return op_begin()+1; }
@@ -468,8 +469,8 @@ class GEPOperator
     return getPointerOperand()->getType();
   }
 
-  Type *getSourceElementType() const;
-  Type *getResultElementType() const;
+  LLVM_ABI Type *getSourceElementType() const;
+  LLVM_ABI Type *getResultElementType() const;
 
   /// Method to return the address space of the pointer operand.
   unsigned getPointerAddressSpace() const {
@@ -515,7 +516,7 @@ class GEPOperator
   }
 
   /// Compute the maximum alignment that this GEP is garranteed to preserve.
-  Align getMaxPreservedAlignment(const DataLayout &DL) const;
+  LLVM_ABI Align getMaxPreservedAlignment(const DataLayout &DL) const;
 
   /// Accumulate the constant address offset of this GEP if possible.
   ///
@@ -534,20 +535,21 @@ class GEPOperator
   ///
   /// The APInt passed into this routine must be at exactly as wide as the
   /// IntPtr type for the address space of the base GEP pointer.
-  bool accumulateConstantOffset(
+  LLVM_ABI bool accumulateConstantOffset(
       const DataLayout &DL, APInt &Offset,
       function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr) const;
 
-  static bool accumulateConstantOffset(
+  LLVM_ABI static bool accumulateConstantOffset(
       Type *SourceType, ArrayRef<const Value *> Index, const DataLayout &DL,
       APInt &Offset,
       function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr);
 
   /// Collect the offset of this GEP as a map of Values to their associated
   /// APInt multipliers, as well as a total Constant Offset.
-  bool collectOffset(const DataLayout &DL, unsigned BitWidth,
-                     SmallMapVector<Value *, APInt, 4> &VariableOffsets,
-                     APInt &ConstantOffset) const;
+  LLVM_ABI bool
+  collectOffset(const DataLayout &DL, unsigned BitWidth,
+                SmallMapVector<Value *, APInt, 4> &VariableOffsets,
+                APInt &ConstantOffset) const;
 };
 
 template <>
diff --git a/external/llvm-project/llvm/include/llvm/IR/OptBisect.h b/external/llvm-project/llvm/include/llvm/IR/OptBisect.h
index 507d415d5e11..be6aef3298b2 100644
--- a/external/llvm-project/llvm/include/llvm/IR/OptBisect.h
+++ b/external/llvm-project/llvm/include/llvm/IR/OptBisect.h
@@ -15,6 +15,7 @@
 #define LLVM_IR_OPTBISECT_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <limits>
 
 namespace llvm {
@@ -40,7 +41,7 @@ class OptPassGate {
 /// optimizations at compile time based on a command line option
 /// (-opt-bisect-limit) in order to perform a bisecting search for
 /// optimization-related problems.
-class OptBisect : public OptPassGate {
+class LLVM_ABI OptBisect : public OptPassGate {
 public:
   /// Default constructor. Initializes the state to "disabled". The bisection
   /// will be enabled by the cl::opt call-back when the command line option
@@ -83,7 +84,7 @@ class OptBisect : public OptPassGate {
 
 /// Singleton instance of the OptBisect class, so multiple pass managers don't
 /// need to coordinate their uses of OptBisect.
-OptPassGate &getGlobalPassGate();
+LLVM_ABI OptPassGate &getGlobalPassGate();
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/PassInstrumentation.h b/external/llvm-project/llvm/include/llvm/IR/PassInstrumentation.h
index 3172cefe7e8c..031571599f9a 100644
--- a/external/llvm-project/llvm/include/llvm/IR/PassInstrumentation.h
+++ b/external/llvm-project/llvm/include/llvm/IR/PassInstrumentation.h
@@ -163,9 +163,9 @@ class PassInstrumentationCallbacks {
   }
 
   /// Add a class name to pass name mapping for use by pass instrumentation.
-  void addClassToPassName(StringRef ClassName, StringRef PassName);
+  LLVM_ABI void addClassToPassName(StringRef ClassName, StringRef PassName);
   /// Get the pass name for a given pass class name.
-  StringRef getPassNameForClassName(StringRef ClassName);
+  LLVM_ABI StringRef getPassNameForClassName(StringRef ClassName);
 
 private:
   friend class PassInstrumentation;
@@ -344,14 +344,15 @@ class PassInstrumentation {
   }
 };
 
-bool isSpecialPass(StringRef PassID, const std::vector<StringRef> &Specials);
+LLVM_ABI bool isSpecialPass(StringRef PassID,
+                            const std::vector<StringRef> &Specials);
 
 /// Pseudo-analysis pass that exposes the \c PassInstrumentation to pass
 /// managers.
 class PassInstrumentationAnalysis
     : public AnalysisInfoMixin<PassInstrumentationAnalysis> {
   friend AnalysisInfoMixin<PassInstrumentationAnalysis>;
-  static AnalysisKey Key;
+  LLVM_ABI static AnalysisKey Key;
 
   PassInstrumentationCallbacks *Callbacks;
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/PassManager.h b/external/llvm-project/llvm/include/llvm/IR/PassManager.h
index b5230047b0e1..4f44ae56eb3c 100644
--- a/external/llvm-project/llvm/include/llvm/IR/PassManager.h
+++ b/external/llvm-project/llvm/include/llvm/IR/PassManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/IR/Analysis.h"
 #include "llvm/IR/PassManagerInternal.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/TypeName.h"
 #include <cassert>
 #include <cstring>
@@ -229,18 +230,19 @@ template <typename IRUnitT>
 void printIRUnitNameForStackTrace(raw_ostream &OS, const IRUnitT &IR);
 
 template <>
-void printIRUnitNameForStackTrace<Module>(raw_ostream &OS, const Module &IR);
+LLVM_ABI void printIRUnitNameForStackTrace<Module>(raw_ostream &OS,
+                                                   const Module &IR);
 
-extern template class PassManager<Module>;
+extern template class LLVM_TEMPLATE_ABI PassManager<Module>;
 
 /// Convenience typedef for a pass manager over modules.
 using ModulePassManager = PassManager<Module>;
 
 template <>
-void printIRUnitNameForStackTrace<Function>(raw_ostream &OS,
-                                            const Function &IR);
+LLVM_ABI void printIRUnitNameForStackTrace<Function>(raw_ostream &OS,
+                                                     const Function &IR);
 
-extern template class PassManager<Function>;
+extern template class LLVM_TEMPLATE_ABI PassManager<Function>;
 
 /// Convenience typedef for a pass manager over functions.
 using FunctionPassManager = PassManager<Function>;
@@ -535,12 +537,12 @@ template <typename IRUnitT, typename... ExtraArgTs> class AnalysisManager {
   AnalysisResultMapT AnalysisResults;
 };
 
-extern template class AnalysisManager<Module>;
+extern template class LLVM_TEMPLATE_ABI AnalysisManager<Module>;
 
 /// Convenience typedef for the Module analysis manager.
 using ModuleAnalysisManager = AnalysisManager<Module>;
 
-extern template class AnalysisManager<Function>;
+extern template class LLVM_TEMPLATE_ABI AnalysisManager<Function>;
 
 /// Convenience typedef for the Function analysis manager.
 using FunctionAnalysisManager = AnalysisManager<Function>;
@@ -562,7 +564,7 @@ using FunctionAnalysisManager = AnalysisManager<Function>;
 /// Note that the proxy's result is a move-only RAII object.  The validity of
 /// the analyses in the inner analysis manager is tied to its lifetime.
 template <typename AnalysisManagerT, typename IRUnitT, typename... ExtraArgTs>
-class InnerAnalysisManagerProxy
+class LLVM_TEMPLATE_ABI InnerAnalysisManagerProxy
     : public AnalysisInfoMixin<
           InnerAnalysisManagerProxy<AnalysisManagerT, IRUnitT>> {
 public:
@@ -650,7 +652,7 @@ using FunctionAnalysisManagerModuleProxy =
 /// Specialization of the invalidate method for the \c
 /// FunctionAnalysisManagerModuleProxy's result.
 template <>
-bool FunctionAnalysisManagerModuleProxy::Result::invalidate(
+LLVM_ABI bool FunctionAnalysisManagerModuleProxy::Result::invalidate(
     Module &M, const PreservedAnalyses &PA,
     ModuleAnalysisManager::Invalidator &Inv);
 
@@ -795,8 +797,8 @@ template <typename AnalysisManagerT, typename IRUnitT, typename... ExtraArgTs>
 AnalysisKey
     OuterAnalysisManagerProxy<AnalysisManagerT, IRUnitT, ExtraArgTs...>::Key;
 
-extern template class OuterAnalysisManagerProxy<ModuleAnalysisManager,
-                                                Function>;
+extern template class LLVM_TEMPLATE_ABI
+    OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>;
 /// Provide the \c ModuleAnalysisManager to \c Function proxy.
 using ModuleAnalysisManagerFunctionProxy =
     OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>;
@@ -834,9 +836,10 @@ class ModuleToFunctionPassAdaptor
       : Pass(std::move(Pass)), EagerlyInvalidate(EagerlyInvalidate) {}
 
   /// Runs the function pass across every function in the module.
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
   static bool isRequired() { return true; }
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h b/external/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h
index 55f504c92185..ade13f10c54e 100644
--- a/external/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h
+++ b/external/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h
@@ -19,10 +19,9 @@
 #include "llvm/IR/PassInstrumentation.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/PrettyStackTrace.h"
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace llvm {
 
 template <typename IRUnitT, typename AnalysisManagerT, typename... ExtraArgTs>
@@ -64,10 +63,6 @@ PreservedAnalyses PassManager<IRUnitT, AnalysisManagerT, ExtraArgTs...>::run(
       detail::getAnalysisResult<PassInstrumentationAnalysis>(
           AM, IR, std::tuple<ExtraArgTs...>(ExtraArgs...));
 
-  // RemoveDIs: if requested, convert debug-info to DbgRecord representation
-  // for duration of these passes.
-  ScopedDbgInfoFormatSetter FormatSetter(IR, UseNewDbgInfoFormat);
-
   StackTraceEntry Entry(PI, IR);
   for (auto &Pass : Passes) {
     Entry.setPass(&*Pass);
diff --git a/external/llvm-project/llvm/include/llvm/IR/PassTimingInfo.h b/external/llvm-project/llvm/include/llvm/IR/PassTimingInfo.h
index b47ba7f16ef3..03e481a50342 100644
--- a/external/llvm-project/llvm/include/llvm/IR/PassTimingInfo.h
+++ b/external/llvm-project/llvm/include/llvm/IR/PassTimingInfo.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Timer.h"
 #include <memory>
 #include <utility>
@@ -31,10 +32,10 @@ class raw_ostream;
 /// If -time-passes has been specified, report the timings immediately and then
 /// reset the timers to zero. By default it uses the stream created by
 /// CreateInfoOutputFile().
-void reportAndResetTimings(raw_ostream *OutStream = nullptr);
+LLVM_ABI void reportAndResetTimings(raw_ostream *OutStream = nullptr);
 
 /// Request the timer for this legacy-pass-manager's pass instance.
-Timer *getPassTimer(Pass *);
+LLVM_ABI Timer *getPassTimer(Pass *);
 
 /// This class implements -time-passes functionality for new pass manager.
 /// It provides the pass-instrumentation callbacks that measure the pass
@@ -78,20 +79,20 @@ class TimePassesHandler {
   static constexpr StringRef AnalysisGroupDesc =
       "Analysis execution timing report";
 
-  TimePassesHandler();
-  TimePassesHandler(bool Enabled, bool PerRun = false);
+  LLVM_ABI TimePassesHandler();
+  LLVM_ABI TimePassesHandler(bool Enabled, bool PerRun = false);
 
   /// Prints out timing information and then resets the timers.
-  void print();
+  LLVM_ABI void print();
 
   // We intend this to be unique per-compilation, thus no copies.
   TimePassesHandler(const TimePassesHandler &) = delete;
   void operator=(const TimePassesHandler &) = delete;
 
-  void registerCallbacks(PassInstrumentationCallbacks &PIC);
+  LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC);
 
   /// Set a custom output stream for subsequent reporting.
-  void setOutStream(raw_ostream &OutStream);
+  LLVM_ABI void setOutStream(raw_ostream &OutStream);
 
 private:
   /// Dumps information for running/triggered timers, useful for debugging
diff --git a/external/llvm-project/llvm/include/llvm/IR/PatternMatch.h b/external/llvm-project/llvm/include/llvm/IR/PatternMatch.h
index 6242a686e7bc..2eaa7d0faabc 100644
--- a/external/llvm-project/llvm/include/llvm/IR/PatternMatch.h
+++ b/external/llvm-project/llvm/include/llvm/IR/PatternMatch.h
@@ -3046,35 +3046,8 @@ inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val,
   return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt);
 }
 
-/// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or
-/// the constant expression
-///  `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>`
-/// under the right conditions determined by DataLayout.
-struct VScaleVal_match {
-  template <typename ITy> bool match(ITy *V) const {
-    if (m_Intrinsic<Intrinsic::vscale>().match(V))
-      return true;
-
-    Value *Ptr;
-    if (m_PtrToInt(m_Value(Ptr)).match(V)) {
-      if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
-        auto *DerefTy =
-            dyn_cast<ScalableVectorType>(GEP->getSourceElementType());
-        if (GEP->getNumIndices() == 1 && DerefTy &&
-            DerefTy->getElementType()->isIntegerTy(8) &&
-            m_Zero().match(GEP->getPointerOperand()) &&
-            m_SpecificInt(1).match(GEP->idx_begin()->get()))
-          return true;
-      }
-    }
-
-    return false;
-  }
-};
-
-inline VScaleVal_match m_VScale() {
-  return VScaleVal_match();
-}
+/// Matches a call to `llvm.vscale()`.
+inline IntrinsicID_match m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
 
 template <typename Opnd0, typename Opnd1>
 inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty
diff --git a/external/llvm-project/llvm/include/llvm/IR/ProfDataUtils.h b/external/llvm-project/llvm/include/llvm/IR/ProfDataUtils.h
index 0bea517df832..8e8d069b836f 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ProfDataUtils.h
@@ -18,55 +18,56 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/Metadata.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// Checks if an Instruction has MD_prof Metadata
-bool hasProfMD(const Instruction &I);
+LLVM_ABI bool hasProfMD(const Instruction &I);
 
 /// Checks if an MDNode contains Branch Weight Metadata
-bool isBranchWeightMD(const MDNode *ProfileData);
+LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData);
 
 /// Checks if an instructions has Branch Weight Metadata
 ///
 /// \param I The instruction to check
 /// \returns True if I has an MD_prof node containing Branch Weights. False
 /// otherwise.
-bool hasBranchWeightMD(const Instruction &I);
+LLVM_ABI bool hasBranchWeightMD(const Instruction &I);
 
 /// Checks if an instructions has valid Branch Weight Metadata
 ///
 /// \param I The instruction to check
 /// \returns True if I has an MD_prof node containing valid Branch Weights,
 /// i.e., one weight for each successor. False otherwise.
-bool hasValidBranchWeightMD(const Instruction &I);
+LLVM_ABI bool hasValidBranchWeightMD(const Instruction &I);
 
 /// Get the branch weights metadata node
 ///
 /// \param I The Instruction to get the weights from.
 /// \returns A pointer to I's branch weights metadata node, if it exists.
 /// Nullptr otherwise.
-MDNode *getBranchWeightMDNode(const Instruction &I);
+LLVM_ABI MDNode *getBranchWeightMDNode(const Instruction &I);
 
 /// Get the valid branch weights metadata node
 ///
 /// \param I The Instruction to get the weights from.
 /// \returns A pointer to I's valid branch weights metadata node, if it exists.
 /// Nullptr otherwise.
-MDNode *getValidBranchWeightMDNode(const Instruction &I);
+LLVM_ABI MDNode *getValidBranchWeightMDNode(const Instruction &I);
 
 /// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
 /// intrinsic
-bool hasBranchWeightOrigin(const Instruction &I);
+LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I);
 
 /// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
 /// intrinsic
-bool hasBranchWeightOrigin(const MDNode *ProfileData);
+LLVM_ABI bool hasBranchWeightOrigin(const MDNode *ProfileData);
 
 /// Return the offset to the first branch weight data
-unsigned getBranchWeightOffset(const MDNode *ProfileData);
+LLVM_ABI unsigned getBranchWeightOffset(const MDNode *ProfileData);
 
-unsigned getNumBranchWeights(const MDNode &ProfileData);
+LLVM_ABI unsigned getNumBranchWeights(const MDNode &ProfileData);
 
 /// Extract branch weights from MD_prof metadata
 ///
@@ -74,18 +75,18 @@ unsigned getNumBranchWeights(const MDNode &ProfileData);
 /// \param [out] Weights An output vector to fill with branch weights
 /// \returns True if weights were extracted, False otherwise. When false Weights
 /// will be cleared.
-bool extractBranchWeights(const MDNode *ProfileData,
-                          SmallVectorImpl<uint32_t> &Weights);
+LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData,
+                                   SmallVectorImpl<uint32_t> &Weights);
 
 /// Faster version of extractBranchWeights() that skips checks and must only
 /// be called with "branch_weights" metadata nodes. Supports uint32_t.
-void extractFromBranchWeightMD32(const MDNode *ProfileData,
-                                 SmallVectorImpl<uint32_t> &Weights);
+LLVM_ABI void extractFromBranchWeightMD32(const MDNode *ProfileData,
+                                          SmallVectorImpl<uint32_t> &Weights);
 
 /// Faster version of extractBranchWeights() that skips checks and must only
 /// be called with "branch_weights" metadata nodes. Supports uint64_t.
-void extractFromBranchWeightMD64(const MDNode *ProfileData,
-                                 SmallVectorImpl<uint64_t> &Weights);
+LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData,
+                                          SmallVectorImpl<uint64_t> &Weights);
 
 /// Extract branch weights attatched to an Instruction
 ///
@@ -93,8 +94,8 @@ void extractFromBranchWeightMD64(const MDNode *ProfileData,
 /// \param [out] Weights An output vector to fill with branch weights
 /// \returns True if weights were extracted, False otherwise. When false Weights
 /// will be cleared.
-bool extractBranchWeights(const Instruction &I,
-                          SmallVectorImpl<uint32_t> &Weights);
+LLVM_ABI bool extractBranchWeights(const Instruction &I,
+                                   SmallVectorImpl<uint32_t> &Weights);
 
 /// Extract branch weights from a conditional branch or select Instruction.
 ///
@@ -103,8 +104,8 @@ bool extractBranchWeights(const Instruction &I,
 /// \param [out] FalseVal will contain the branch weight for the False branch
 /// \returns True on success with profile weights filled in. False if no
 /// metadata or invalid metadata was found.
-bool extractBranchWeights(const Instruction &I, uint64_t &TrueVal,
-                          uint64_t &FalseVal);
+LLVM_ABI bool extractBranchWeights(const Instruction &I, uint64_t &TrueVal,
+                                   uint64_t &FalseVal);
 
 /// Retrieve the total of all weights from MD_prof data.
 ///
@@ -112,7 +113,8 @@ bool extractBranchWeights(const Instruction &I, uint64_t &TrueVal,
 /// \param [out] TotalWeights input variable to fill with total weights
 /// \returns True on success with profile total weights filled in. False if no
 /// metadata was found.
-bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights);
+LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData,
+                                     uint64_t &TotalWeights);
 
 /// Retrieve the total of all weights from an instruction.
 ///
@@ -120,18 +122,19 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights);
 /// \param [out] TotalWeights input variable to fill with total weights
 /// \returns True on success with profile total weights filled in. False if no
 /// metadata was found.
-bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights);
+LLVM_ABI bool extractProfTotalWeight(const Instruction &I,
+                                     uint64_t &TotalWeights);
 
 /// Create a new `branch_weights` metadata node and add or overwrite
 /// a `prof` metadata reference to instruction `I`.
 /// \param I the Instruction to set branch weights on.
 /// \param Weights an array of weights to set on instruction I.
 /// \param IsExpected were these weights added from an llvm.expect* intrinsic.
-void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
-                      bool IsExpected);
+LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
+                               bool IsExpected);
 
 /// Scaling the profile data attached to 'I' using the ratio of S/T.
-void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
+LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
 
 } // namespace llvm
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/IR/ProfileSummary.h b/external/llvm-project/llvm/include/llvm/IR/ProfileSummary.h
index b23d5a30846a..6c087ea02b3c 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ProfileSummary.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ProfileSummary.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_IR_PROFILESUMMARY_H
 #define LLVM_IR_PROFILESUMMARY_H
 
+#include "llvm/Support/Compiler.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -81,10 +82,10 @@ class ProfileSummary {
 
   Kind getKind() const { return PSK; }
   /// Return summary information as metadata.
-  Metadata *getMD(LLVMContext &Context, bool AddPartialField = true,
-                  bool AddPartialProfileRatioField = true);
+  LLVM_ABI Metadata *getMD(LLVMContext &Context, bool AddPartialField = true,
+                           bool AddPartialProfileRatioField = true);
   /// Construct profile summary from metdata.
-  static ProfileSummary *getFromMD(Metadata *MD);
+  LLVM_ABI static ProfileSummary *getFromMD(Metadata *MD);
   const SummaryEntryVector &getDetailedSummary() { return DetailedSummary; }
   uint32_t getNumFunctions() const { return NumFunctions; }
   uint64_t getMaxFunctionCount() const { return MaxFunctionCount; }
@@ -99,8 +100,8 @@ class ProfileSummary {
     assert(isPartialProfile() && "Unexpected when not partial profile");
     PartialProfileRatio = R;
   }
-  void printSummary(raw_ostream &OS) const;
-  void printDetailedSummary(raw_ostream &OS) const;
+  LLVM_ABI void printSummary(raw_ostream &OS) const;
+  LLVM_ABI void printDetailedSummary(raw_ostream &OS) const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/IR/PseudoProbe.h b/external/llvm-project/llvm/include/llvm/IR/PseudoProbe.h
index 7308904da3fd..5c652b14650d 100644
--- a/external/llvm-project/llvm/include/llvm/IR/PseudoProbe.h
+++ b/external/llvm-project/llvm/include/llvm/IR/PseudoProbe.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_IR_PSEUDOPROBE_H
 #define LLVM_IR_PSEUDOPROBE_H
 
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 #include <limits>
@@ -133,9 +134,9 @@ static inline bool hasDiscriminator(uint32_t Flags) {
   return Flags & (uint32_t)PseudoProbeAttributes::HasDiscriminator;
 }
 
-std::optional<PseudoProbe> extractProbe(const Instruction &Inst);
+LLVM_ABI std::optional<PseudoProbe> extractProbe(const Instruction &Inst);
 
-void setProbeDistributionFactor(Instruction &Inst, float Factor);
+LLVM_ABI void setProbeDistributionFactor(Instruction &Inst, float Factor);
 } // end namespace llvm
 
 #endif // LLVM_IR_PSEUDOPROBE_H
diff --git a/external/llvm-project/llvm/include/llvm/IR/ReplaceConstant.h b/external/llvm-project/llvm/include/llvm/IR/ReplaceConstant.h
index 8c497eb4022c..cbea22115310 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ReplaceConstant.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ReplaceConstant.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_IR_REPLACECONSTANT_H
 #define LLVM_IR_REPLACECONSTANT_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 template <typename T> class ArrayRef;
@@ -33,10 +35,9 @@ class Function;
 ///
 /// If \p IncludeSelf is enabled, also convert the passed constants themselves
 /// to instructions, rather than only their users.
-bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts,
-                                           Function *RestrictToFunc = nullptr,
-                                           bool RemoveDeadConstants = true,
-                                           bool IncludeSelf = false);
+LLVM_ABI bool convertUsersOfConstantsToInstructions(
+    ArrayRef<Constant *> Consts, Function *RestrictToFunc = nullptr,
+    bool RemoveDeadConstants = true, bool IncludeSelf = false);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.def b/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.def
index cd8e9b598044..247643525ff4 100644
--- a/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -70,16 +70,16 @@ HANDLE_LIBCALL(UREM_I16, "__umodhi3")
 HANDLE_LIBCALL(UREM_I32, "__umodsi3")
 HANDLE_LIBCALL(UREM_I64, "__umoddi3")
 HANDLE_LIBCALL(UREM_I128, "__umodti3")
-HANDLE_LIBCALL(SDIVREM_I8, nullptr)
-HANDLE_LIBCALL(SDIVREM_I16, nullptr)
-HANDLE_LIBCALL(SDIVREM_I32, nullptr)
-HANDLE_LIBCALL(SDIVREM_I64, nullptr)
-HANDLE_LIBCALL(SDIVREM_I128, nullptr)
-HANDLE_LIBCALL(UDIVREM_I8, nullptr)
-HANDLE_LIBCALL(UDIVREM_I16, nullptr)
-HANDLE_LIBCALL(UDIVREM_I32, nullptr)
-HANDLE_LIBCALL(UDIVREM_I64, nullptr)
-HANDLE_LIBCALL(UDIVREM_I128, nullptr)
+HANDLE_LIBCALL(SDIVREM_I8, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SDIVREM_I16, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SDIVREM_I32, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SDIVREM_I64, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SDIVREM_I128, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(UDIVREM_I8, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(UDIVREM_I16, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(UDIVREM_I32, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(UDIVREM_I64, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(UDIVREM_I128, LIBCALL_NO_NAME)
 HANDLE_LIBCALL(NEG_I32, "__negsi2")
 HANDLE_LIBCALL(NEG_I64, "__negdi2")
 HANDLE_LIBCALL(CTLZ_I32, "__clzsi2")
@@ -240,13 +240,13 @@ HANDLE_LIBCALL(ATAN2_F64, "atan2")
 HANDLE_LIBCALL(ATAN2_F80, "atan2l")
 HANDLE_LIBCALL(ATAN2_F128,"atan2l")
 HANDLE_LIBCALL(ATAN2_PPCF128, "atan2l")
-HANDLE_LIBCALL(SINCOS_F32, nullptr)
-HANDLE_LIBCALL(SINCOS_F64, nullptr)
-HANDLE_LIBCALL(SINCOS_F80, nullptr)
-HANDLE_LIBCALL(SINCOS_F128, nullptr)
-HANDLE_LIBCALL(SINCOS_PPCF128, nullptr)
-HANDLE_LIBCALL(SINCOS_STRET_F32, nullptr)
-HANDLE_LIBCALL(SINCOS_STRET_F64, nullptr)
+HANDLE_LIBCALL(SINCOS_F32, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SINCOS_F64, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SINCOS_F80, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SINCOS_F128, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SINCOS_PPCF128, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SINCOS_STRET_F32, LIBCALL_NO_NAME)
+HANDLE_LIBCALL(SINCOS_STRET_F64, LIBCALL_NO_NAME)
 HANDLE_LIBCALL(POW_F32, "powf")
 HANDLE_LIBCALL(POW_F64, "pow")
 HANDLE_LIBCALL(POW_F80, "powl")
@@ -317,16 +317,16 @@ HANDLE_LIBCALL(FMAXIMUM_F64, "fmaximum")
 HANDLE_LIBCALL(FMAXIMUM_F80, "fmaximuml")
 HANDLE_LIBCALL(FMAXIMUM_F128, "fmaximuml")
 HANDLE_LIBCALL(FMAXIMUM_PPCF128, "fmaximum_numl")
-HANDLE_LIBCALL(FMINIMUMNUM_F32, "fminimum_numf")
-HANDLE_LIBCALL(FMINIMUMNUM_F64, "fminimum_num")
-HANDLE_LIBCALL(FMINIMUMNUM_F80, "fminimum_numl")
-HANDLE_LIBCALL(FMINIMUMNUM_F128, "fminimum_numl")
-HANDLE_LIBCALL(FMINIMUMNUM_PPCF128, "fminimum_numl")
-HANDLE_LIBCALL(FMAXIMUMNUM_F32, "fmaximum_numf")
-HANDLE_LIBCALL(FMAXIMUMNUM_F64, "fmaximum_num")
-HANDLE_LIBCALL(FMAXIMUMNUM_F80, "fmaximum_numl")
-HANDLE_LIBCALL(FMAXIMUMNUM_F128, "fmaximum_numl")
-HANDLE_LIBCALL(FMAXIMUMNUM_PPCF128, "fmaximum_numl")
+HANDLE_LIBCALL(FMINIMUM_NUM_F32, "fminimum_numf")
+HANDLE_LIBCALL(FMINIMUM_NUM_F64, "fminimum_num")
+HANDLE_LIBCALL(FMINIMUM_NUM_F80, "fminimum_numl")
+HANDLE_LIBCALL(FMINIMUM_NUM_F128, "fminimum_numl")
+HANDLE_LIBCALL(FMINIMUM_NUM_PPCF128, "fminimum_numl")
+HANDLE_LIBCALL(FMAXIMUM_NUM_F32, "fmaximum_numf")
+HANDLE_LIBCALL(FMAXIMUM_NUM_F64, "fmaximum_num")
+HANDLE_LIBCALL(FMAXIMUM_NUM_F80, "fmaximum_numl")
+HANDLE_LIBCALL(FMAXIMUM_NUM_F128, "fmaximum_numl")
+HANDLE_LIBCALL(FMAXIMUM_NUM_PPCF128, "fmaximum_numl")
 HANDLE_LIBCALL(LROUND_F32, "lroundf")
 HANDLE_LIBCALL(LROUND_F64, "lround")
 HANDLE_LIBCALL(LROUND_F80, "lroundl")
@@ -518,7 +518,7 @@ HANDLE_LIBCALL(MEMMOVE, "memmove")
 HANDLE_LIBCALL(MEMSET, "memset")
 // DSEPass can emit calloc if it finds a pair of malloc/memset
 HANDLE_LIBCALL(CALLOC, "calloc")
-HANDLE_LIBCALL(BZERO, nullptr)
+HANDLE_LIBCALL(BZERO, LIBCALL_NO_NAME)
 
 // Element-wise unordered-atomic memory of different sizes
 HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_1, "__llvm_memcpy_element_unordered_atomic_1")
@@ -669,10 +669,10 @@ HANDLE_LIBCALL(ATOMIC_FETCH_NAND_16, "__atomic_fetch_nand_16")
 
 // Out-of-line atomics libcalls
 #define HLCALLS(A, N)                                                          \
-  HANDLE_LIBCALL(A##N##_RELAX, nullptr)                                        \
-  HANDLE_LIBCALL(A##N##_ACQ, nullptr)                                          \
-  HANDLE_LIBCALL(A##N##_REL, nullptr)                                          \
-  HANDLE_LIBCALL(A##N##_ACQ_REL, nullptr)
+  HANDLE_LIBCALL(A##N##_RELAX, LIBCALL_NO_NAME)                                \
+  HANDLE_LIBCALL(A##N##_ACQ, LIBCALL_NO_NAME)                                  \
+  HANDLE_LIBCALL(A##N##_REL, LIBCALL_NO_NAME)                                  \
+  HANDLE_LIBCALL(A##N##_ACQ_REL, LIBCALL_NO_NAME)
 #define HLCALL5(A)                                                             \
   HLCALLS(A, 1) HLCALLS(A, 2) HLCALLS(A, 4) HLCALLS(A, 8) HLCALLS(A, 16)
 HLCALL5(OUTLINE_ATOMIC_CAS)
@@ -691,11 +691,11 @@ HANDLE_LIBCALL(STACKPROTECTOR_CHECK_FAIL, "__stack_chk_fail")
 HANDLE_LIBCALL(DEOPTIMIZE, "__llvm_deoptimize")
 
 // Return address
-HANDLE_LIBCALL(RETURN_ADDRESS, nullptr)
+HANDLE_LIBCALL(RETURN_ADDRESS, LIBCALL_NO_NAME)
 
 // Clear cache
 HANDLE_LIBCALL(CLEAR_CACHE, "__clear_cache")
 HANDLE_LIBCALL(RISCV_FLUSH_ICACHE, "__riscv_flush_icache")
 
-HANDLE_LIBCALL(UNKNOWN_LIBCALL, nullptr)
+HANDLE_LIBCALL(UNKNOWN_LIBCALL, LIBCALL_NO_NAME)
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.h b/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.h
index b3648f5a31e2..051bcc147cb7 100644
--- a/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ b/external/llvm-project/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -15,8 +15,10 @@
 #define LLVM_IR_RUNTIME_LIBCALLS_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Sequence.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
@@ -35,6 +37,18 @@ enum Libcall {
 #include "llvm/IR/RuntimeLibcalls.def"
 #undef HANDLE_LIBCALL
 };
+} // namespace RTLIB
+
+template <> struct enum_iteration_traits<RTLIB::Libcall> {
+  static constexpr bool is_iterable = true;
+};
+
+namespace RTLIB {
+
+// Return an iterator over all Libcall values.
+static inline auto libcalls() {
+  return enum_seq(static_cast<RTLIB::Libcall>(0), RTLIB::UNKNOWN_LIBCALL);
+}
 
 /// A simple container for information about the supported runtime calls.
 struct RuntimeLibcallsInfo {
@@ -67,9 +81,9 @@ struct RuntimeLibcallsInfo {
     return LibcallCallingConvs[Call];
   }
 
-  iterator_range<const char **> getLibcallNames() {
-    return llvm::make_range(LibcallRoutineNames,
-                            LibcallRoutineNames + RTLIB::UNKNOWN_LIBCALL);
+  ArrayRef<const char *> getLibcallNames() const {
+    // Trim UNKNOWN_LIBCALL from the end
+    return ArrayRef(LibcallRoutineNames).drop_back();
   }
 
 private:
@@ -96,7 +110,7 @@ struct RuntimeLibcallsInfo {
 
   /// Set default libcall names. If a target wants to opt-out of a libcall it
   /// should be placed here.
-  void initLibcalls(const Triple &TT);
+  LLVM_ABI void initLibcalls(const Triple &TT);
 };
 
 } // namespace RTLIB
diff --git a/external/llvm-project/llvm/include/llvm/IR/StructuralHash.h b/external/llvm-project/llvm/include/llvm/IR/StructuralHash.h
index 514dd6f174b9..fc4b97ee2d41 100644
--- a/external/llvm-project/llvm/include/llvm/IR/StructuralHash.h
+++ b/external/llvm-project/llvm/include/llvm/IR/StructuralHash.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StableHashing.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -29,16 +30,17 @@ class Module;
 /// \param DetailedHash Whether or not to encode additional information in the
 /// hash. The additional information added into the hash when this flag is set
 /// to true includes instruction and operand type information.
-stable_hash StructuralHash(const Function &F, bool DetailedHash = false);
+LLVM_ABI stable_hash StructuralHash(const Function &F,
+                                    bool DetailedHash = false);
 
 /// Returns a hash of the global variable \p G.
-stable_hash StructuralHash(const GlobalVariable &G);
+LLVM_ABI stable_hash StructuralHash(const GlobalVariable &G);
 
 /// Returns a hash of the module \p M by hashing all functions and global
 /// variables contained within. \param M The module to hash. \param DetailedHash
 /// Whether or not to encode additional information in the function hashes that
 /// composed the module hash.
-stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
+LLVM_ABI stable_hash StructuralHash(const Module &M, bool DetailedHash = false);
 
 /// The pair of an instruction index and a operand index.
 using IndexPair = std::pair<unsigned, unsigned>;
@@ -80,8 +82,8 @@ struct FunctionHashInfo {
 /// \param IgnoreOp A callable that takes an instruction and an operand index,
 /// and returns true if the operand should be ignored in the hash computation.
 /// \return A FunctionHashInfo structure
-FunctionHashInfo StructuralHashWithDifferences(const Function &F,
-                                               IgnoreOperandFunc IgnoreOp);
+LLVM_ABI FunctionHashInfo
+StructuralHashWithDifferences(const Function &F, IgnoreOperandFunc IgnoreOp);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/SymbolTableListTraits.h b/external/llvm-project/llvm/include/llvm/IR/SymbolTableListTraits.h
index fcf6f0fb1528..e8fcab89005b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/SymbolTableListTraits.h
+++ b/external/llvm-project/llvm/include/llvm/IR/SymbolTableListTraits.h
@@ -26,6 +26,7 @@
 
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/simple_ilist.h"
+#include "llvm/Support/Compiler.h"
 #include <cstddef>
 
 namespace llvm {
@@ -109,11 +110,11 @@ class SymbolTableListTraits : public ilist_alloc_traits<ValueSubClass> {
 // The SymbolTableListTraits template is explicitly instantiated for the
 // following data types, so add extern template statements to prevent implicit
 // instantiation.
-extern template class SymbolTableListTraits<BasicBlock>;
-extern template class SymbolTableListTraits<Function>;
-extern template class SymbolTableListTraits<GlobalAlias>;
-extern template class SymbolTableListTraits<GlobalIFunc>;
-extern template class SymbolTableListTraits<GlobalVariable>;
+extern template class LLVM_TEMPLATE_ABI SymbolTableListTraits<BasicBlock>;
+extern template class LLVM_TEMPLATE_ABI SymbolTableListTraits<Function>;
+extern template class LLVM_TEMPLATE_ABI SymbolTableListTraits<GlobalAlias>;
+extern template class LLVM_TEMPLATE_ABI SymbolTableListTraits<GlobalIFunc>;
+extern template class LLVM_TEMPLATE_ABI SymbolTableListTraits<GlobalVariable>;
 
 /// List that automatically updates parent links and symbol tables.
 ///
diff --git a/external/llvm-project/llvm/include/llvm/IR/Type.h b/external/llvm-project/llvm/include/llvm/IR/Type.h
index 37f6b2ed8ee6..fd987179645d 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Type.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Type.h
@@ -119,10 +119,10 @@ class Type {
   /// When \p NoDetails is true, we only print %st.
   /// Put differently, \p NoDetails prints the type as if
   /// inlined with the operands when printing an instruction.
-  void print(raw_ostream &O, bool IsForDebug = false,
-             bool NoDetails = false) const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false,
+                      bool NoDetails = false) const;
 
-  void dump() const;
+  LLVM_ABI void dump() const;
 
   /// Return the LLVMContext in which this type was uniqued.
   LLVMContext &getContext() const { return Context; }
@@ -194,7 +194,7 @@ class Type {
     return getTypeID() == PPC_FP128TyID;
   }
 
-  const fltSemantics &getFltSemantics() const;
+  LLVM_ABI const fltSemantics &getFltSemantics() const;
 
   /// Return true if this is X86 AMX.
   bool isX86_AMXTy() const { return getTypeID() == X86_AMXTyID; }
@@ -203,23 +203,23 @@ class Type {
   bool isTargetExtTy() const { return getTypeID() == TargetExtTyID; }
 
   /// Return true if this is a target extension type with a scalable layout.
-  bool isScalableTargetExtTy() const;
+  LLVM_ABI bool isScalableTargetExtTy() const;
 
   /// Return true if this is a type whose size is a known multiple of vscale.
-  bool isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const;
-  bool isScalableTy() const;
+  LLVM_ABI bool isScalableTy(SmallPtrSetImpl<const Type *> &Visited) const;
+  LLVM_ABI bool isScalableTy() const;
 
   /// Return true if this type is or contains a target extension type that
   /// disallows being used as a global.
-  bool
+  LLVM_ABI bool
   containsNonGlobalTargetExtType(SmallPtrSetImpl<const Type *> &Visited) const;
-  bool containsNonGlobalTargetExtType() const;
+  LLVM_ABI bool containsNonGlobalTargetExtType() const;
 
   /// Return true if this type is or contains a target extension type that
   /// disallows being used as a local.
-  bool
+  LLVM_ABI bool
   containsNonLocalTargetExtType(SmallPtrSetImpl<const Type *> &Visited) const;
-  bool containsNonLocalTargetExtType() const;
+  LLVM_ABI bool containsNonLocalTargetExtType() const;
 
   /// Return true if this is a FP type or a vector of FP.
   bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); }
@@ -237,7 +237,7 @@ class Type {
   bool isIntegerTy() const { return getTypeID() == IntegerTyID; }
 
   /// Return true if this is an IntegerType of the given width.
-  bool isIntegerTy(unsigned Bitwidth) const;
+  LLVM_ABI bool isIntegerTy(unsigned Bitwidth) const;
 
   /// Return true if this is an integer type or a vector of integer types.
   bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); }
@@ -272,21 +272,21 @@ class Type {
   }
 
   // True if this is an instance of TargetExtType of RISC-V vector tuple.
-  bool isRISCVVectorTupleTy() const;
+  LLVM_ABI bool isRISCVVectorTupleTy() const;
 
   /// Return true if this type could be converted with a lossless BitCast to
   /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the
   /// same size only where no re-interpretation of the bits is done.
   /// Determine if this type could be losslessly bitcast to Ty
-  bool canLosslesslyBitCastTo(Type *Ty) const;
+  LLVM_ABI bool canLosslesslyBitCastTo(Type *Ty) const;
 
   /// Return true if this type is empty, that is, it has no elements or all of
   /// its elements are empty.
-  bool isEmptyTy() const;
+  LLVM_ABI bool isEmptyTy() const;
 
   /// Return true if the type is "first class", meaning it is a valid type for a
   /// Value.
-  bool isFirstClassType() const;
+  LLVM_ABI bool isFirstClassType() const;
 
   /// Return true if the type is a valid type for a register in codegen. This
   /// includes all first-class types except struct and array types.
@@ -332,17 +332,17 @@ class Type {
   /// instance of the type is stored to memory. The DataLayout class provides
   /// additional query functions to provide this information.
   ///
-  TypeSize getPrimitiveSizeInBits() const LLVM_READONLY;
+  LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY;
 
   /// If this is a vector type, return the getPrimitiveSizeInBits value for the
   /// element type. Otherwise return the getPrimitiveSizeInBits value for this
   /// type.
-  unsigned getScalarSizeInBits() const LLVM_READONLY;
+  LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY;
 
   /// Return the width of the mantissa of this type. This is only valid on
   /// floating-point types. If the FP type does not have a stable mantissa (e.g.
   /// ppc long double), this method returns -1.
-  int getFPMantissaWidth() const;
+  LLVM_ABI int getFPMantissaWidth() const;
 
   /// If this is a vector type, return the element type, otherwise return
   /// 'this'.
@@ -390,40 +390,40 @@ class Type {
   // only intended to cover the core methods that are frequently used, helper
   // methods should not be added here.
 
-  inline unsigned getIntegerBitWidth() const;
+  LLVM_ABI inline unsigned getIntegerBitWidth() const;
 
-  inline Type *getFunctionParamType(unsigned i) const;
-  inline unsigned getFunctionNumParams() const;
-  inline bool isFunctionVarArg() const;
+  LLVM_ABI inline Type *getFunctionParamType(unsigned i) const;
+  LLVM_ABI inline unsigned getFunctionNumParams() const;
+  LLVM_ABI inline bool isFunctionVarArg() const;
 
-  inline StringRef getStructName() const;
-  inline unsigned getStructNumElements() const;
-  inline Type *getStructElementType(unsigned N) const;
+  LLVM_ABI inline StringRef getStructName() const;
+  LLVM_ABI inline unsigned getStructNumElements() const;
+  LLVM_ABI inline Type *getStructElementType(unsigned N) const;
 
-  inline uint64_t getArrayNumElements() const;
+  LLVM_ABI inline uint64_t getArrayNumElements() const;
 
   Type *getArrayElementType() const {
     assert(getTypeID() == ArrayTyID);
     return ContainedTys[0];
   }
 
-  inline StringRef getTargetExtName() const;
+  LLVM_ABI inline StringRef getTargetExtName() const;
 
   /// Given vector type, change the element type,
   /// whilst keeping the old number of elements.
   /// For non-vectors simply returns \p EltTy.
-  inline Type *getWithNewType(Type *EltTy) const;
+  LLVM_ABI inline Type *getWithNewType(Type *EltTy) const;
 
   /// Given an integer or vector type, change the lane bitwidth to NewBitwidth,
   /// whilst keeping the old number of lanes.
-  inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;
+  LLVM_ABI inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;
 
   /// Given scalar/vector integer type, returns a type with elements twice as
   /// wide as in the original type. For vectors, preserves element count.
-  inline Type *getExtendedType() const;
+  LLVM_ABI inline Type *getExtendedType() const;
 
   /// Get the address space of this pointer or pointer vector type.
-  inline unsigned getPointerAddressSpace() const;
+  LLVM_ABI inline unsigned getPointerAddressSpace() const;
 
   //===--------------------------------------------------------------------===//
   // Static members exported by the Type class itself.  Useful for getting
@@ -431,30 +431,30 @@ class Type {
   //
 
   /// Return a type based on an identifier.
-  static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber);
+  LLVM_ABI static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber);
 
   //===--------------------------------------------------------------------===//
   // These are the builtin types that are always available.
   //
-  static Type *getVoidTy(LLVMContext &C);
-  static Type *getLabelTy(LLVMContext &C);
-  static Type *getHalfTy(LLVMContext &C);
-  static Type *getBFloatTy(LLVMContext &C);
-  static Type *getFloatTy(LLVMContext &C);
-  static Type *getDoubleTy(LLVMContext &C);
-  static Type *getMetadataTy(LLVMContext &C);
-  static Type *getX86_FP80Ty(LLVMContext &C);
-  static Type *getFP128Ty(LLVMContext &C);
-  static Type *getPPC_FP128Ty(LLVMContext &C);
-  static Type *getX86_AMXTy(LLVMContext &C);
-  static Type *getTokenTy(LLVMContext &C);
-  static IntegerType *getIntNTy(LLVMContext &C, unsigned N);
-  static IntegerType *getInt1Ty(LLVMContext &C);
-  static IntegerType *getInt8Ty(LLVMContext &C);
-  static IntegerType *getInt16Ty(LLVMContext &C);
-  static IntegerType *getInt32Ty(LLVMContext &C);
-  static IntegerType *getInt64Ty(LLVMContext &C);
-  static IntegerType *getInt128Ty(LLVMContext &C);
+  LLVM_ABI static Type *getVoidTy(LLVMContext &C);
+  LLVM_ABI static Type *getLabelTy(LLVMContext &C);
+  LLVM_ABI static Type *getHalfTy(LLVMContext &C);
+  LLVM_ABI static Type *getBFloatTy(LLVMContext &C);
+  LLVM_ABI static Type *getFloatTy(LLVMContext &C);
+  LLVM_ABI static Type *getDoubleTy(LLVMContext &C);
+  LLVM_ABI static Type *getMetadataTy(LLVMContext &C);
+  LLVM_ABI static Type *getX86_FP80Ty(LLVMContext &C);
+  LLVM_ABI static Type *getFP128Ty(LLVMContext &C);
+  LLVM_ABI static Type *getPPC_FP128Ty(LLVMContext &C);
+  LLVM_ABI static Type *getX86_AMXTy(LLVMContext &C);
+  LLVM_ABI static Type *getTokenTy(LLVMContext &C);
+  LLVM_ABI static IntegerType *getIntNTy(LLVMContext &C, unsigned N);
+  LLVM_ABI static IntegerType *getInt1Ty(LLVMContext &C);
+  LLVM_ABI static IntegerType *getInt8Ty(LLVMContext &C);
+  LLVM_ABI static IntegerType *getInt16Ty(LLVMContext &C);
+  LLVM_ABI static IntegerType *getInt32Ty(LLVMContext &C);
+  LLVM_ABI static IntegerType *getInt64Ty(LLVMContext &C);
+  LLVM_ABI static IntegerType *getInt128Ty(LLVMContext &C);
   template <typename ScalarTy> static Type *getScalarTy(LLVMContext &C) {
     int noOfBits = sizeof(ScalarTy) * CHAR_BIT;
     if (std::is_integral<ScalarTy>::value) {
@@ -469,25 +469,27 @@ class Type {
     }
     llvm_unreachable("Unsupported type in Type::getScalarTy");
   }
-  static Type *getFloatingPointTy(LLVMContext &C, const fltSemantics &S);
+  LLVM_ABI static Type *getFloatingPointTy(LLVMContext &C,
+                                           const fltSemantics &S);
 
   //===--------------------------------------------------------------------===//
   // Convenience methods for getting pointer types.
   //
-  static Type *getWasm_ExternrefTy(LLVMContext &C);
-  static Type *getWasm_FuncrefTy(LLVMContext &C);
+  LLVM_ABI static Type *getWasm_ExternrefTy(LLVMContext &C);
+  LLVM_ABI static Type *getWasm_FuncrefTy(LLVMContext &C);
 
   /// Return a pointer to the current type. This is equivalent to
   /// PointerType::get(Ctx, AddrSpace).
   /// TODO: Remove this after opaque pointer transition is complete.
-  LLVM_DEPRECATED("Use PointerType::get instead", "PointerType::get")
-  PointerType *getPointerTo(unsigned AddrSpace = 0) const;
+  LLVM_ABI LLVM_DEPRECATED("Use PointerType::get instead", "PointerType::get")
+      PointerType *getPointerTo(unsigned AddrSpace = 0) const;
 
 private:
   /// Derived types like structures and arrays are sized iff all of the members
   /// of the type are sized as well. Since asking for their size is relatively
   /// uncommon, move this operation out-of-line.
-  bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const;
+  LLVM_ABI bool
+  isSizedDerivedType(SmallPtrSetImpl<Type *> *Visited = nullptr) const;
 };
 
 // Printing of types.
diff --git a/external/llvm-project/llvm/include/llvm/IR/TypedPointerType.h b/external/llvm-project/llvm/include/llvm/IR/TypedPointerType.h
index 1bea715a64b6..6d1486bb319b 100644
--- a/external/llvm-project/llvm/include/llvm/IR/TypedPointerType.h
+++ b/external/llvm-project/llvm/include/llvm/IR/TypedPointerType.h
@@ -15,6 +15,7 @@
 #define LLVM_IR_TYPEDPOINTERTYPE_H
 
 #include "llvm/IR/Type.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -33,10 +34,11 @@ class TypedPointerType : public Type {
 
   /// This constructs a pointer to an object of the specified type in a numbered
   /// address space.
-  static TypedPointerType *get(Type *ElementType, unsigned AddressSpace);
+  LLVM_ABI static TypedPointerType *get(Type *ElementType,
+                                        unsigned AddressSpace);
 
   /// Return true if the specified type is valid as a element type.
-  static bool isValidElementType(Type *ElemTy);
+  LLVM_ABI static bool isValidElementType(Type *ElemTy);
 
   /// Return the address space of the Pointer type.
   unsigned getAddressSpace() const { return getSubclassData(); }
diff --git a/external/llvm-project/llvm/include/llvm/IR/Use.h b/external/llvm-project/llvm/include/llvm/IR/Use.h
index 0d5d878e4689..0894d945391a 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Use.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Use.h
@@ -38,7 +38,7 @@ class Use {
 
   /// Provide a fast substitute to std::swap<Use>
   /// that also works with less standard-compliant compilers
-  void swap(Use &RHS);
+  LLVM_ABI void swap(Use &RHS);
 
 private:
   /// Destructor - Only for zap()
@@ -60,10 +60,10 @@ class Use {
   /// instruction.
   User *getUser() const { return Parent; };
 
-  inline void set(Value *Val);
+  LLVM_ABI inline void set(Value *Val);
 
-  inline Value *operator=(Value *RHS);
-  inline const Use &operator=(const Use &RHS);
+  LLVM_ABI inline Value *operator=(Value *RHS);
+  LLVM_ABI inline const Use &operator=(const Use &RHS);
 
   Value *operator->() { return Val; }
   const Value *operator->() const { return Val; }
@@ -71,11 +71,11 @@ class Use {
   Use *getNext() const { return Next; }
 
   /// Return the operand # of this use in its User.
-  unsigned getOperandNo() const;
+  LLVM_ABI unsigned getOperandNo() const;
 
   /// Destroys Use operands when the number of operands of
   /// a User changes.
-  static void zap(Use *Start, const Use *Stop, bool del = false);
+  LLVM_ABI static void zap(Use *Start, const Use *Stop, bool del = false);
 
 private:
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/User.h b/external/llvm-project/llvm/include/llvm/IR/User.h
index 25ca8d744a59..cbb4379b68c4 100644
--- a/external/llvm-project/llvm/include/llvm/IR/User.h
+++ b/external/llvm-project/llvm/include/llvm/IR/User.h
@@ -101,18 +101,20 @@ class User : public Value {
   ///
   /// This is used for subclasses which need to allocate a variable number
   /// of operands, ie, 'hung off uses'.
-  void *operator new(size_t Size, HungOffOperandsAllocMarker);
+  LLVM_ABI void *operator new(size_t Size, HungOffOperandsAllocMarker);
 
   /// Allocate a User with the operands co-allocated.
   ///
   /// This is used for subclasses which have a fixed number of operands.
-  void *operator new(size_t Size, IntrusiveOperandsAllocMarker allocTrait);
+  LLVM_ABI void *operator new(size_t Size,
+                              IntrusiveOperandsAllocMarker allocTrait);
 
   /// Allocate a User with the operands co-allocated.  If DescBytes is non-zero
   /// then allocate an additional DescBytes bytes before the operands. These
   /// bytes can be accessed by calling getDescriptor.
-  void *operator new(size_t Size,
-                     IntrusiveOperandsAndDescriptorAllocMarker allocTrait);
+  LLVM_ABI void *
+  operator new(size_t Size,
+               IntrusiveOperandsAndDescriptorAllocMarker allocTrait);
 
   User(Type *ty, unsigned vty, AllocInfo AllocInfo) : Value(ty, vty) {
     assert(AllocInfo.NumOps < (1u << NumUserOperandsBits) &&
@@ -132,11 +134,11 @@ class User : public Value {
   /// (with bottom bit set) to the User.
   /// \param IsPhi identifies callers which are phi nodes and which need
   /// N BasicBlock* allocated along with N
-  void allocHungoffUses(unsigned N, bool IsPhi = false);
+  LLVM_ABI void allocHungoffUses(unsigned N, bool IsPhi = false);
 
   /// Grow the number of hung off uses.  Note that allocHungoffUses
   /// should be called if there are no uses.
-  void growHungoffUses(unsigned N, bool IsPhi = false);
+  LLVM_ABI void growHungoffUses(unsigned N, bool IsPhi = false);
 
 protected:
   ~User() = default; // Use deleteValue() to delete a generic Instruction.
@@ -145,7 +147,7 @@ class User : public Value {
   User(const User &) = delete;
 
   /// Free memory allocated for User and Use objects.
-  void operator delete(void *Usr);
+  LLVM_ABI void operator delete(void *Usr);
   /// Placement delete - required by std, called if the ctor throws.
   void operator delete(void *Usr, HungOffOperandsAllocMarker) {
     // Note: If a subclass manipulates the information which is required to
@@ -252,10 +254,10 @@ class User : public Value {
   unsigned getNumOperands() const { return NumUserOperands; }
 
   /// Returns the descriptor co-allocated with this User instance.
-  ArrayRef<const uint8_t> getDescriptor() const;
+  LLVM_ABI ArrayRef<const uint8_t> getDescriptor() const;
 
   /// Returns the descriptor co-allocated with this User instance.
-  MutableArrayRef<uint8_t> getDescriptor();
+  LLVM_ABI MutableArrayRef<uint8_t> getDescriptor();
 
   /// Subclasses with hung off uses need to manage the operand count
   /// themselves.  In these instances, the operand count isn't used to find the
@@ -269,7 +271,7 @@ class User : public Value {
   /// A droppable user is a user for which uses can be dropped without affecting
   /// correctness and should be dropped rather than preventing a transformation
   /// from happening.
-  bool isDroppable() const;
+  LLVM_ABI bool isDroppable() const;
 
   // ---------------------------------------------------------------------------
   // Operand Iterator interface...
@@ -353,7 +355,7 @@ class User : public Value {
   ///
   /// Replaces all references to the "From" definition with references to the
   /// "To" definition. Returns whether any uses were replaced.
-  bool replaceUsesOfWith(Value *From, Value *To);
+  LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To);
 
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
diff --git a/external/llvm-project/llvm/include/llvm/IR/VFABIDemangler.h b/external/llvm-project/llvm/include/llvm/IR/VFABIDemangler.h
index de731cd7051c..0b287987665e 100644
--- a/external/llvm-project/llvm/include/llvm/IR/VFABIDemangler.h
+++ b/external/llvm-project/llvm/include/llvm/IR/VFABIDemangler.h
@@ -17,6 +17,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/TypeSize.h"
 
 namespace llvm {
@@ -117,7 +118,7 @@ struct VFShape {
     return {EC, Parameters};
   }
   /// Validation check on the Parameters in the VFShape.
-  bool hasValidParameterList() const;
+  LLVM_ABI bool hasValidParameterList() const;
 };
 
 /// Holds the VFShape for a specific scalar to vector function mapping.
@@ -180,11 +181,11 @@ static constexpr char const *_LLVM_Scalarize_ = "_LLVM_Scalarize_";
 /// factor for scalable vectors, since the mangled name doesn't encode that;
 /// it needs to be derived from the widest element types of vector arguments
 /// or return values.
-std::optional<VFInfo> tryDemangleForVFABI(StringRef MangledName,
-                                          const FunctionType *FTy);
+LLVM_ABI std::optional<VFInfo> tryDemangleForVFABI(StringRef MangledName,
+                                                   const FunctionType *FTy);
 
 /// Retrieve the `VFParamKind` from a string token.
-VFParamKind getVFParamKindFromString(const StringRef Token);
+LLVM_ABI VFParamKind getVFParamKindFromString(const StringRef Token);
 
 // Name of the attribute where the variant mappings are stored.
 static constexpr char const *MappingsAttrName = "vector-function-abi-variant";
@@ -194,8 +195,9 @@ static constexpr char const *MappingsAttrName = "vector-function-abi-variant";
 /// vector-function-abi-variant attribute, we return without populating
 /// VariantMappings, i.e. callers of getVectorVariantNames need not check for
 /// the presence of the attribute (see InjectTLIMappings).
-void getVectorVariantNames(const CallInst &CI,
-                           SmallVectorImpl<std::string> &VariantMappings);
+LLVM_ABI void
+getVectorVariantNames(const CallInst &CI,
+                      SmallVectorImpl<std::string> &VariantMappings);
 
 /// Constructs a FunctionType by applying vector function information to the
 /// type of a matching scalar function.
@@ -204,12 +206,13 @@ void getVectorVariantNames(const CallInst &CI,
 /// \param ScalarFTy gets the Type information of parameters, as it is not
 /// stored in \p Info.
 /// \returns a pointer to a newly created vector FunctionType
-FunctionType *createFunctionType(const VFInfo &Info,
-                                 const FunctionType *ScalarFTy);
+LLVM_ABI FunctionType *createFunctionType(const VFInfo &Info,
+                                          const FunctionType *ScalarFTy);
 
 /// Overwrite the Vector Function ABI variants attribute with the names provide
 /// in \p VariantMappings.
-void setVectorVariantNames(CallInst *CI, ArrayRef<std::string> VariantMappings);
+LLVM_ABI void setVectorVariantNames(CallInst *CI,
+                                    ArrayRef<std::string> VariantMappings);
 
 } // end namespace VFABI
 
diff --git a/external/llvm-project/llvm/include/llvm/IR/Value.h b/external/llvm-project/llvm/include/llvm/IR/Value.h
index 999eeb3dad87..04d0391c0409 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Value.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Value.h
@@ -21,6 +21,7 @@
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <iterator>
 #include <memory>
@@ -212,30 +213,30 @@ class Value {
   };
 
 protected:
-  Value(Type *Ty, unsigned scid);
+  LLVM_ABI Value(Type *Ty, unsigned scid);
 
   /// Value's destructor should be virtual by design, but that would require
   /// that Value and all of its subclasses have a vtable that effectively
   /// duplicates the information in the value ID. As a size optimization, the
   /// destructor has been protected, and the caller should manually call
   /// deleteValue.
-  ~Value(); // Use deleteValue() to delete a generic Value.
+  LLVM_ABI ~Value(); // Use deleteValue() to delete a generic Value.
 
 public:
   Value(const Value &) = delete;
   Value &operator=(const Value &) = delete;
 
   /// Delete a pointer to a generic Value.
-  void deleteValue();
+  LLVM_ABI void deleteValue();
 
   /// Support for debugging, callable in GDB: V->dump()
-  void dump() const;
+  LLVM_ABI void dump() const;
 
   /// Implement operator<< on Value.
   /// @{
-  void print(raw_ostream &O, bool IsForDebug = false) const;
-  void print(raw_ostream &O, ModuleSlotTracker &MST,
-             bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &O, ModuleSlotTracker &MST,
+                      bool IsForDebug = false) const;
   /// @}
 
   /// Print the name of this Value out to the specified raw_ostream.
@@ -245,22 +246,22 @@ class Value {
   /// even constants get pretty-printed; for example, the type of a null
   /// pointer is printed symbolically.
   /// @{
-  void printAsOperand(raw_ostream &O, bool PrintType = true,
-                      const Module *M = nullptr) const;
-  void printAsOperand(raw_ostream &O, bool PrintType,
-                      ModuleSlotTracker &MST) const;
+  LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType = true,
+                               const Module *M = nullptr) const;
+  LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType,
+                               ModuleSlotTracker &MST) const;
   /// @}
 
   /// All values are typed, get the type of this value.
   Type *getType() const { return VTy; }
 
   /// All values hold a context through their type.
-  LLVMContext &getContext() const;
+  LLVM_ABI LLVMContext &getContext() const;
 
   // All values can potentially be named.
   bool hasName() const { return HasName; }
-  ValueName *getValueName() const;
-  void setValueName(ValueName *VN);
+  LLVM_ABI ValueName *getValueName() const;
+  LLVM_ABI void setValueName(ValueName *VN);
 
 private:
   void destroyValueName();
@@ -274,50 +275,50 @@ class Value {
   /// This guaranteed to return the same reference as long as the value is not
   /// modified.  If the value has a name, this does a hashtable lookup, so it's
   /// not free.
-  StringRef getName() const;
+  LLVM_ABI StringRef getName() const;
 
   /// Change the name of the value.
   ///
   /// Choose a new unique name if the provided name is taken.
   ///
   /// \param Name The new name; or "" if the value's name should be removed.
-  void setName(const Twine &Name);
+  LLVM_ABI void setName(const Twine &Name);
 
   /// Transfer the name from V to this value.
   ///
   /// After taking V's name, sets V's name to empty.
   ///
   /// \note It is an error to call V->takeName(V).
-  void takeName(Value *V);
+  LLVM_ABI void takeName(Value *V);
 
-  std::string getNameOrAsOperand() const;
+  LLVM_ABI std::string getNameOrAsOperand() const;
 
   /// Change all uses of this to point to a new Value.
   ///
   /// Go through the uses list for this definition and make each use point to
   /// "V" instead of "this".  After this completes, 'this's use list is
   /// guaranteed to be empty.
-  void replaceAllUsesWith(Value *V);
+  LLVM_ABI void replaceAllUsesWith(Value *V);
 
   /// Change non-metadata uses of this to point to a new Value.
   ///
   /// Go through the uses list for this definition and make each use point to
   /// "V" instead of "this". This function skips metadata entries in the list.
-  void replaceNonMetadataUsesWith(Value *V);
+  LLVM_ABI void replaceNonMetadataUsesWith(Value *V);
 
   /// Go through the uses list for this definition and make each use point
   /// to "V" if the callback ShouldReplace returns true for the given Use.
   /// Unlike replaceAllUsesWith() this function does not support basic block
   /// values.
-  void replaceUsesWithIf(Value *New,
-                         llvm::function_ref<bool(Use &U)> ShouldReplace);
+  LLVM_ABI void
+  replaceUsesWithIf(Value *New, llvm::function_ref<bool(Use &U)> ShouldReplace);
 
   /// replaceUsesOutsideBlock - Go through the uses list for this definition and
   /// make each use point to "V" instead of "this" when the use is outside the
   /// block. 'This's use list is expected to have at least one element.
   /// Unlike replaceAllUsesWith() this function does not support basic block
   /// values.
-  void replaceUsesOutsideBlock(Value *V, BasicBlock *BB);
+  LLVM_ABI void replaceUsesOutsideBlock(Value *V, BasicBlock *BB);
 
   //----------------------------------------------------------------------
   // Methods for handling the chain of uses of this Value.
@@ -329,7 +330,7 @@ class Value {
   // when using them since you might not get all uses.
   // The methods that don't start with materialized_ assert that modules is
   // fully materialized.
-  void assertModuleIsMaterializedImpl() const;
+  LLVM_ABI void assertModuleIsMaterializedImpl() const;
   // This indirection exists so we can keep assertModuleIsMaterializedImpl()
   // around in release builds of Value.cpp to be linked with other code built
   // in debug mode. But this avoids calling it in any of the release built code.
@@ -438,12 +439,12 @@ class Value {
   bool hasOneUse() const { return UseList && hasSingleElement(uses()); }
 
   /// Return true if this Value has exactly N uses.
-  bool hasNUses(unsigned N) const;
+  LLVM_ABI bool hasNUses(unsigned N) const;
 
   /// Return true if this value has N uses or more.
   ///
   /// This is logically equivalent to getNumUses() >= N.
-  bool hasNUsesOrMore(unsigned N) const;
+  LLVM_ABI bool hasNUsesOrMore(unsigned N) const;
 
   /// Return true if there is exactly one user of this value.
   ///
@@ -453,18 +454,18 @@ class Value {
   ///
   /// This check is potentially costly, since it requires traversing,
   /// in the worst case, the whole use list of a value.
-  bool hasOneUser() const;
+  LLVM_ABI bool hasOneUser() const;
 
   /// Return true if there is exactly one use of this value that cannot be
   /// dropped.
-  Use *getSingleUndroppableUse();
+  LLVM_ABI Use *getSingleUndroppableUse();
   const Use *getSingleUndroppableUse() const {
     return const_cast<Value *>(this)->getSingleUndroppableUse();
   }
 
   /// Return true if there is exactly one unique user of this value that cannot be
   /// dropped (that user can have multiple uses of this value).
-  User *getUniqueUndroppableUser();
+  LLVM_ABI User *getUniqueUndroppableUser();
   const User *getUniqueUndroppableUser() const {
     return const_cast<Value *>(this)->getUniqueUndroppableUser();
   }
@@ -473,12 +474,12 @@ class Value {
   ///
   /// This is specialized because it is a common request and does not require
   /// traversing the whole use list.
-  bool hasNUndroppableUses(unsigned N) const;
+  LLVM_ABI bool hasNUndroppableUses(unsigned N) const;
 
   /// Return true if this value has N uses or more.
   ///
   /// This is logically equivalent to getNumUses() >= N.
-  bool hasNUndroppableUsesOrMore(unsigned N) const;
+  LLVM_ABI bool hasNUndroppableUsesOrMore(unsigned N) const;
 
   /// Remove every uses that can safely be removed.
   ///
@@ -487,25 +488,26 @@ class Value {
   /// some Droppable uses pervent it.
   /// This function optionally takes a filter to only remove some droppable
   /// uses.
-  void dropDroppableUses(llvm::function_ref<bool(const Use *)> ShouldDrop =
-                             [](const Use *) { return true; });
+  LLVM_ABI void
+  dropDroppableUses(llvm::function_ref<bool(const Use *)> ShouldDrop =
+                        [](const Use *) { return true; });
 
   /// Remove every use of this value in \p User that can safely be removed.
-  void dropDroppableUsesIn(User &Usr);
+  LLVM_ABI void dropDroppableUsesIn(User &Usr);
 
   /// Remove the droppable use \p U.
-  static void dropDroppableUse(Use &U);
+  LLVM_ABI static void dropDroppableUse(Use &U);
 
   /// Check if this value is used in the specified basic block.
   ///
   /// Not supported for ConstantData.
-  bool isUsedInBasicBlock(const BasicBlock *BB) const;
+  LLVM_ABI bool isUsedInBasicBlock(const BasicBlock *BB) const;
 
   /// This method computes the number of uses of this Value.
   ///
   /// This is a linear time operation.  Use hasOneUse, hasNUses, or
   /// hasNUsesOrMore to check for specific values.
-  unsigned getNumUses() const;
+  LLVM_ABI unsigned getNumUses() const;
 
   /// This method should only be used by the Use class.
   void addUse(Use &U) {
@@ -576,22 +578,24 @@ class Value {
       return nullptr;
     return getMetadataImpl(KindID);
   }
-  MDNode *getMetadata(StringRef Kind) const;
+  LLVM_ABI MDNode *getMetadata(StringRef Kind) const;
   /// @}
 
   /// Appends all attachments with the given ID to \c MDs in insertion order.
   /// If the Value has no attachments with the given ID, or if ID is invalid,
   /// leaves MDs unchanged.
   /// @{
-  void getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const;
-  void getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const;
+  LLVM_ABI void getMetadata(unsigned KindID,
+                            SmallVectorImpl<MDNode *> &MDs) const;
+  LLVM_ABI void getMetadata(StringRef Kind,
+                            SmallVectorImpl<MDNode *> &MDs) const;
   /// @}
 
   /// Appends all metadata attached to this value to \c MDs, sorting by
   /// KindID. The first element of each pair returned is the KindID, the second
   /// element is the metadata value. Attachments with the same ID appear in
   /// insertion order.
-  void
+  LLVM_ABI void
   getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const;
 
   /// Return true if this value has any metadata attached to it.
@@ -612,44 +616,44 @@ class Value {
   /// Sets the given attachment to \c MD, erasing it if \c MD is \c nullptr or
   /// replacing it if it already exists.
   /// @{
-  void setMetadata(unsigned KindID, MDNode *Node);
-  void setMetadata(StringRef Kind, MDNode *Node);
+  LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node);
+  LLVM_ABI void setMetadata(StringRef Kind, MDNode *Node);
   /// @}
 
   /// Add a metadata attachment.
   /// @{
-  void addMetadata(unsigned KindID, MDNode &MD);
-  void addMetadata(StringRef Kind, MDNode &MD);
+  LLVM_ABI void addMetadata(unsigned KindID, MDNode &MD);
+  LLVM_ABI void addMetadata(StringRef Kind, MDNode &MD);
   /// @}
 
   /// Erase all metadata attachments with the given kind.
   ///
   /// \returns true if any metadata was removed.
-  bool eraseMetadata(unsigned KindID);
+  LLVM_ABI bool eraseMetadata(unsigned KindID);
 
   /// Erase all metadata attachments matching the given predicate.
-  void eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred);
+  LLVM_ABI void eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred);
 
   /// Erase all metadata attached to this Value.
-  void clearMetadata();
+  LLVM_ABI void clearMetadata();
 
   /// Get metadata for the given kind, if any.
   /// This is an internal function that must only be called after
   /// checking that `hasMetadata()` returns true.
-  MDNode *getMetadataImpl(unsigned KindID) const;
+  LLVM_ABI MDNode *getMetadataImpl(unsigned KindID) const;
 
 public:
   /// Return true if this value is a swifterror value.
   ///
   /// swifterror values can be either a function argument or an alloca with a
   /// swifterror attribute.
-  bool isSwiftError() const;
+  LLVM_ABI bool isSwiftError() const;
 
   /// Strip off pointer casts, all-zero GEPs and address space casts.
   ///
   /// Returns the original uncasted value.  If this is called on a non-pointer
   /// value, it returns 'this'.
-  const Value *stripPointerCasts() const;
+  LLVM_ABI const Value *stripPointerCasts() const;
   Value *stripPointerCasts() {
     return const_cast<Value *>(
         static_cast<const Value *>(this)->stripPointerCasts());
@@ -659,7 +663,7 @@ class Value {
   ///
   /// Returns the original uncasted value.  If this is called on a non-pointer
   /// value, it returns 'this'.
-  const Value *stripPointerCastsAndAliases() const;
+  LLVM_ABI const Value *stripPointerCastsAndAliases() const;
   Value *stripPointerCastsAndAliases() {
     return const_cast<Value *>(
         static_cast<const Value *>(this)->stripPointerCastsAndAliases());
@@ -670,7 +674,7 @@ class Value {
   ///
   /// Returns the original uncasted value with the same representation. If this
   /// is called on a non-pointer value, it returns 'this'.
-  const Value *stripPointerCastsSameRepresentation() const;
+  LLVM_ABI const Value *stripPointerCastsSameRepresentation() const;
   Value *stripPointerCastsSameRepresentation() {
     return const_cast<Value *>(static_cast<const Value *>(this)
                                    ->stripPointerCastsSameRepresentation());
@@ -682,7 +686,7 @@ class Value {
   /// Returns the original uncasted value.  If this is called on a non-pointer
   /// value, it returns 'this'. This function should be used only in
   /// Alias analysis.
-  const Value *stripPointerCastsForAliasAnalysis() const;
+  LLVM_ABI const Value *stripPointerCastsForAliasAnalysis() const;
   Value *stripPointerCastsForAliasAnalysis() {
     return const_cast<Value *>(static_cast<const Value *>(this)
                                    ->stripPointerCastsForAliasAnalysis());
@@ -692,7 +696,7 @@ class Value {
   ///
   /// Returns the original pointer value.  If this is called on a non-pointer
   /// value, it returns 'this'.
-  const Value *stripInBoundsConstantOffsets() const;
+  LLVM_ABI const Value *stripInBoundsConstantOffsets() const;
   Value *stripInBoundsConstantOffsets() {
     return const_cast<Value *>(
               static_cast<const Value *>(this)->stripInBoundsConstantOffsets());
@@ -731,7 +735,7 @@ class Value {
   /// between the underlying value and the returned one. Thus, if no constant
   /// offset was found, the returned value is the underlying one and \p Offset
   /// is unchanged.
-  const Value *stripAndAccumulateConstantOffsets(
+  LLVM_ABI const Value *stripAndAccumulateConstantOffsets(
       const DataLayout &DL, APInt &Offset, bool AllowNonInbounds,
       bool AllowInvariantGroup = false,
       function_ref<bool(Value &Value, APInt &Offset)> ExternalAnalysis =
@@ -767,8 +771,8 @@ class Value {
   ///
   /// Returns the original pointer value.  If this is called on a non-pointer
   /// value, it returns 'this'.
-  const Value *stripInBoundsOffsets(function_ref<void(const Value *)> Func =
-                                        [](const Value *) {}) const;
+  LLVM_ABI const Value *stripInBoundsOffsets(
+      function_ref<void(const Value *)> Func = [](const Value *) {}) const;
   inline Value *stripInBoundsOffsets(function_ref<void(const Value *)> Func =
                                   [](const Value *) {}) {
     return const_cast<Value *>(
@@ -777,14 +781,14 @@ class Value {
 
   /// If this ptr is provably equal to \p Other plus a constant offset, return
   /// that offset in bytes. Essentially `ptr this` subtract `ptr Other`.
-  std::optional<int64_t> getPointerOffsetFrom(const Value *Other,
-                                              const DataLayout &DL) const;
+  LLVM_ABI std::optional<int64_t>
+  getPointerOffsetFrom(const Value *Other, const DataLayout &DL) const;
 
   /// Return true if the memory object referred to by V can by freed in the
   /// scope for which the SSA value defining the allocation is statically
   /// defined.  E.g.  deallocation after the static scope of a value does not
   /// count, but a deallocation before that does.
-  bool canBeFreed() const;
+  LLVM_ABI bool canBeFreed() const;
 
   /// Returns the number of bytes known to be dereferenceable for the
   /// pointer value.
@@ -795,15 +799,15 @@ class Value {
   /// IF CanBeFreed is true, the pointer is known to be dereferenceable at
   /// point of definition only.  Caller must prove that allocation is not
   /// deallocated between point of definition and use.
-  uint64_t getPointerDereferenceableBytes(const DataLayout &DL,
-                                          bool &CanBeNull,
-                                          bool &CanBeFreed) const;
+  LLVM_ABI uint64_t getPointerDereferenceableBytes(const DataLayout &DL,
+                                                   bool &CanBeNull,
+                                                   bool &CanBeFreed) const;
 
   /// Returns an alignment of the pointer value.
   ///
   /// Returns an alignment which is either specified explicitly, e.g. via
   /// align attribute of a function argument, or guaranteed by DataLayout.
-  Align getPointerAlignment(const DataLayout &DL) const;
+  LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const;
 
   /// Translate PHI node to its predecessor from the given basic block.
   ///
@@ -811,8 +815,8 @@ class Value {
   /// the PHI node corresponding to PredBB.  If not, return ourself.  This is
   /// useful if you want to know the value something has in a predecessor
   /// block.
-  const Value *DoPHITranslation(const BasicBlock *CurBB,
-                                const BasicBlock *PredBB) const;
+  LLVM_ABI const Value *DoPHITranslation(const BasicBlock *CurBB,
+                                         const BasicBlock *PredBB) const;
   Value *DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) {
     return const_cast<Value *>(
              static_cast<const Value *>(this)->DoPHITranslation(CurBB, PredBB));
@@ -842,7 +846,7 @@ class Value {
   template <class Compare> void sortUseList(Compare Cmp);
 
   /// Reverse the use-list.
-  void reverseUseList();
+  LLVM_ABI void reverseUseList();
 
 private:
   /// Merge two lists together.
diff --git a/external/llvm-project/llvm/include/llvm/IR/ValueHandle.h b/external/llvm-project/llvm/include/llvm/IR/ValueHandle.h
index 29560815ea55..05555bd5efb1 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ValueHandle.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ValueHandle.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 
 namespace llvm {
@@ -105,7 +106,7 @@ class ValueHandleBase {
   }
 
   /// Remove this ValueHandle from its current use list.
-  void RemoveFromUseList();
+  LLVM_ABI void RemoveFromUseList();
 
   /// Clear the underlying pointer without clearing the use list.
   ///
@@ -115,8 +116,8 @@ class ValueHandleBase {
 
 public:
   // Callbacks made from Value.
-  static void ValueIsDeleted(Value *V);
-  static void ValueIsRAUWd(Value *Old, Value *New);
+  LLVM_ABI static void ValueIsDeleted(Value *V);
+  LLVM_ABI static void ValueIsRAUWd(Value *Old, Value *New);
 
 private:
   // Internal implementation details.
@@ -128,13 +129,13 @@ class ValueHandleBase {
   ///
   /// List is the address of either the head of the list or a Next node within
   /// the existing use list.
-  void AddToExistingUseList(ValueHandleBase **List);
+  LLVM_ABI void AddToExistingUseList(ValueHandleBase **List);
 
   /// Add this ValueHandle to the use list after Node.
   void AddToExistingUseListAfter(ValueHandleBase *Node);
 
   /// Add this ValueHandle to the use list for V.
-  void AddToUseList();
+  LLVM_ABI void AddToUseList();
 };
 
 /// A nullable Value handle that is nullable.
@@ -380,7 +381,7 @@ template <typename ValueTy> class TrackingVH {
 /// class can be used as the key of a map, as long as the user takes it out of
 /// the map before calling setValPtr() (since the map has to rearrange itself
 /// when the pointer changes).  Unlike ValueHandleBase, this class has a vtable.
-class CallbackVH : public ValueHandleBase {
+class LLVM_ABI CallbackVH : public ValueHandleBase {
   virtual void anchor();
 protected:
   ~CallbackVH() = default;
diff --git a/external/llvm-project/llvm/include/llvm/IR/ValueSymbolTable.h b/external/llvm-project/llvm/include/llvm/IR/ValueSymbolTable.h
index cd1dbbe1688a..b792ee64a02e 100644
--- a/external/llvm-project/llvm/include/llvm/IR/ValueSymbolTable.h
+++ b/external/llvm-project/llvm/include/llvm/IR/ValueSymbolTable.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -64,7 +65,7 @@ class ValueSymbolTable {
 /// @{
 
   ValueSymbolTable(int MaxNameSize = -1) : vmap(0), MaxNameSize(MaxNameSize) {}
-  ~ValueSymbolTable();
+  LLVM_ABI ~ValueSymbolTable();
 
   /// @}
   /// @name Accessors
@@ -91,11 +92,11 @@ class ValueSymbolTable {
   /// This function can be used from the debugger to display the
   /// content of the symbol table while debugging.
   /// Print out symbol table on stderr
-  void dump() const;
+  LLVM_ABI void dump() const;
 
-/// @}
-/// @name Iteration
-/// @{
+  /// @}
+  /// @name Iteration
+  /// @{
 
   /// Get an iterator that from the beginning of the symbol table.
   inline iterator begin() { return vmap.begin(); }
diff --git a/external/llvm-project/llvm/include/llvm/IR/VectorBuilder.h b/external/llvm-project/llvm/include/llvm/IR/VectorBuilder.h
deleted file mode 100644
index d526d52b433b..000000000000
--- a/external/llvm-project/llvm/include/llvm/IR/VectorBuilder.h
+++ /dev/null
@@ -1,119 +0,0 @@
-//===- llvm/VectorBuilder.h - Builder for VP Intrinsics ---------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the VectorBuilder class, which is used as a convenient way
-// to create VP intrinsics as if they were LLVM instructions with a consistent
-// and simplified interface.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_IR_VECTORBUILDER_H
-#define LLVM_IR_VECTORBUILDER_H
-
-#include <llvm/IR/IRBuilder.h>
-#include <llvm/IR/InstrTypes.h>
-#include <llvm/IR/Instruction.h>
-#include <llvm/IR/Value.h>
-
-namespace llvm {
-
-class VectorBuilder {
-public:
-  enum class Behavior {
-    // Abort if the requested VP intrinsic could not be created.
-    // This is useful for strict consistency.
-    ReportAndAbort = 0,
-
-    // Return a default-initialized value if the requested VP intrinsic could
-    // not be created.
-    // This is useful for a defensive fallback to non-VP code.
-    SilentlyReturnNone = 1,
-  };
-
-private:
-  IRBuilderBase &Builder;
-  Behavior ErrorHandling;
-
-  // Explicit mask parameter.
-  Value *Mask;
-  // Explicit vector length parameter.
-  Value *ExplicitVectorLength;
-  // Compile-time vector length.
-  ElementCount StaticVectorLength;
-
-  // Get mask/evl value handles for the current configuration.
-  Value &requestMask();
-  Value &requestEVL();
-
-  void handleError(const char *ErrorMsg) const;
-  template <typename RetType>
-  RetType returnWithError(const char *ErrorMsg) const {
-    handleError(ErrorMsg);
-    return RetType();
-  }
-
-  /// Helper function for creating VP intrinsic call.
-  Value *createVectorInstructionImpl(Intrinsic::ID VPID, Type *ReturnTy,
-                                     ArrayRef<Value *> VecOpArray,
-                                     const Twine &Name = Twine());
-
-public:
-  VectorBuilder(IRBuilderBase &Builder,
-                Behavior ErrorHandling = Behavior::ReportAndAbort)
-      : Builder(Builder), ErrorHandling(ErrorHandling), Mask(nullptr),
-        ExplicitVectorLength(nullptr),
-        StaticVectorLength(ElementCount::getFixed(0)) {}
-
-  Module &getModule() const;
-  LLVMContext &getContext() const { return Builder.getContext(); }
-
-  // All-true mask for the currently configured explicit vector length.
-  Value *getAllTrueMask();
-
-  VectorBuilder &setMask(Value *NewMask) {
-    Mask = NewMask;
-    return *this;
-  }
-  VectorBuilder &setEVL(Value *NewExplicitVectorLength) {
-    ExplicitVectorLength = NewExplicitVectorLength;
-    return *this;
-  }
-  VectorBuilder &setStaticVL(unsigned NewFixedVL) {
-    StaticVectorLength = ElementCount::getFixed(NewFixedVL);
-    return *this;
-  }
-
-  /// Get the flags to be applied to created floating point ops.
-  const FastMathFlags &getFastMathFlags() const {
-    return Builder.getFastMathFlags();
-  }
-
-  // TODO: setStaticVL(ElementCount) for scalable types.
-
-  // Emit a VP intrinsic call that mimics a regular instruction.
-  // This operation behaves according to the VectorBuilderBehavior.
-  // \p Opcode      The functional instruction opcode of the emitted intrinsic.
-  // \p ReturnTy    The return type of the operation.
-  // \p VecOpArray  The operand list.
-  Value *createVectorInstruction(unsigned Opcode, Type *ReturnTy,
-                                 ArrayRef<Value *> VecOpArray,
-                                 const Twine &Name = Twine());
-
-  /// Emit a VP reduction intrinsic call for recurrence kind.
-  /// \param RdxID       The intrinsic ID of llvm.vector.reduce.*
-  /// \param ValTy       The type of operand which the reduction operation is
-  ///                    performed.
-  /// \param VecOpArray  The operand list.
-  Value *createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy,
-                               ArrayRef<Value *> VecOpArray,
-                               const Twine &Name = Twine());
-};
-
-} // namespace llvm
-
-#endif // LLVM_IR_VECTORBUILDER_H
diff --git a/external/llvm-project/llvm/include/llvm/IR/VectorTypeUtils.h b/external/llvm-project/llvm/include/llvm/IR/VectorTypeUtils.h
index dd9b18268200..e3d7fadad608 100644
--- a/external/llvm-project/llvm/include/llvm/IR/VectorTypeUtils.h
+++ b/external/llvm-project/llvm/include/llvm/IR/VectorTypeUtils.h
@@ -10,6 +10,7 @@
 #define LLVM_IR_VECTORTYPEUTILS_H
 
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -30,19 +31,19 @@ inline Type *toVectorTy(Type *Scalar, unsigned VF) {
 /// Note:
 ///   - If \p EC is scalar, \p StructTy is returned unchanged
 ///   - Only unpacked literal struct types are supported
-Type *toVectorizedStructTy(StructType *StructTy, ElementCount EC);
+LLVM_ABI Type *toVectorizedStructTy(StructType *StructTy, ElementCount EC);
 
 /// A helper for converting structs of vector types to structs of scalar types.
 /// Note: Only unpacked literal struct types are supported.
-Type *toScalarizedStructTy(StructType *StructTy);
+LLVM_ABI Type *toScalarizedStructTy(StructType *StructTy);
 
 /// Returns true if `StructTy` is an unpacked literal struct where all elements
 /// are vectors of matching element count. This does not include empty structs.
-bool isVectorizedStructTy(StructType *StructTy);
+LLVM_ABI bool isVectorizedStructTy(StructType *StructTy);
 
 /// Returns true if `StructTy` is an unpacked literal struct where all elements
 /// are scalars that can be used as vector element types.
-bool canVectorizeStructTy(StructType *StructTy);
+LLVM_ABI bool canVectorizeStructTy(StructType *StructTy);
 
 /// A helper for converting to vectorized types. For scalar types, this is
 /// equivalent to calling `toVectorTy`. For struct types, this returns a new
diff --git a/external/llvm-project/llvm/include/llvm/IR/Verifier.h b/external/llvm-project/llvm/include/llvm/IR/Verifier.h
index b25f8eb77ee3..8dbb9c8a41d7 100644
--- a/external/llvm-project/llvm/include/llvm/IR/Verifier.h
+++ b/external/llvm-project/llvm/include/llvm/IR/Verifier.h
@@ -22,6 +22,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <utility>
 
 namespace llvm {
@@ -76,7 +77,7 @@ class TBAAVerifier {
       : Diagnostic(Diagnostic) {}
   /// Visit an instruction and return true if it is valid, return false if an
   /// invalid TBAA is attached.
-  bool visitTBAAMetadata(Instruction &I, const MDNode *MD);
+  LLVM_ABI bool visitTBAAMetadata(Instruction &I, const MDNode *MD);
 };
 
 /// Check a function for errors, useful for use when debugging a
@@ -85,7 +86,7 @@ class TBAAVerifier {
 /// If there are no errors, the function returns false. If an error is found,
 /// a message describing the error is written to OS (if non-null) and true is
 /// returned.
-bool verifyFunction(const Function &F, raw_ostream *OS = nullptr);
+LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS = nullptr);
 
 /// Check a module for errors.
 ///
@@ -97,25 +98,25 @@ bool verifyFunction(const Function &F, raw_ostream *OS = nullptr);
 /// supplied, DebugInfo verification failures won't be considered as
 /// error and instead *BrokenDebugInfo will be set to true. Debug
 /// info errors can be "recovered" from by stripping the debug info.
-bool verifyModule(const Module &M, raw_ostream *OS = nullptr,
-                  bool *BrokenDebugInfo = nullptr);
+LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS = nullptr,
+                           bool *BrokenDebugInfo = nullptr);
 
-FunctionPass *createVerifierPass(bool FatalErrors = true);
+LLVM_ABI FunctionPass *createVerifierPass(bool FatalErrors = true);
 
 /// Check a module for errors, and report separate error states for IR
 /// and debug info errors.
 class VerifierAnalysis : public AnalysisInfoMixin<VerifierAnalysis> {
   friend AnalysisInfoMixin<VerifierAnalysis>;
 
-  static AnalysisKey Key;
+  LLVM_ABI static AnalysisKey Key;
 
 public:
   struct Result {
     bool IRBroken, DebugInfoBroken;
   };
 
-  Result run(Module &M, ModuleAnalysisManager &);
-  Result run(Function &F, FunctionAnalysisManager &);
+  LLVM_ABI Result run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI Result run(Function &F, FunctionAnalysisManager &);
   static bool isRequired() { return true; }
 };
 
@@ -135,8 +136,8 @@ class VerifierPass : public PassInfoMixin<VerifierPass> {
 public:
   explicit VerifierPass(bool FatalErrors = true) : FatalErrors(FatalErrors) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/IRPrinter/IRPrintingPasses.h b/external/llvm-project/llvm/include/llvm/IRPrinter/IRPrintingPasses.h
index 4be1f3c967e2..0d5dd1124733 100644
--- a/external/llvm-project/llvm/include/llvm/IRPrinter/IRPrintingPasses.h
+++ b/external/llvm-project/llvm/include/llvm/IRPrinter/IRPrintingPasses.h
@@ -19,6 +19,7 @@
 #define LLVM_IRPRINTER_IRPRINTINGPASSES_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -36,12 +37,12 @@ class PrintModulePass : public PassInfoMixin<PrintModulePass> {
   bool EmitSummaryIndex;
 
 public:
-  PrintModulePass();
-  PrintModulePass(raw_ostream &OS, const std::string &Banner = "",
-                  bool ShouldPreserveUseListOrder = false,
-                  bool EmitSummaryIndex = false);
+  LLVM_ABI PrintModulePass();
+  LLVM_ABI PrintModulePass(raw_ostream &OS, const std::string &Banner = "",
+                           bool ShouldPreserveUseListOrder = false,
+                           bool EmitSummaryIndex = false);
 
-  PreservedAnalyses run(Module &M, AnalysisManager<Module> &);
+  LLVM_ABI PreservedAnalyses run(Module &M, AnalysisManager<Module> &);
   static bool isRequired() { return true; }
 };
 
@@ -52,10 +53,10 @@ class PrintFunctionPass : public PassInfoMixin<PrintFunctionPass> {
   std::string Banner;
 
 public:
-  PrintFunctionPass();
-  PrintFunctionPass(raw_ostream &OS, const std::string &Banner = "");
+  LLVM_ABI PrintFunctionPass();
+  LLVM_ABI PrintFunctionPass(raw_ostream &OS, const std::string &Banner = "");
 
-  PreservedAnalyses run(Function &F, AnalysisManager<Function> &);
+  LLVM_ABI PreservedAnalyses run(Function &F, AnalysisManager<Function> &);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/IRReader/IRReader.h b/external/llvm-project/llvm/include/llvm/IRReader/IRReader.h
index 099b958efb0c..790140f19934 100644
--- a/external/llvm-project/llvm/include/llvm/IRReader/IRReader.h
+++ b/external/llvm-project/llvm/include/llvm/IRReader/IRReader.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -32,16 +33,16 @@ class LLVMContext;
 /// Module. The ShouldLazyLoadMetadata flag is passed down to the bitcode
 /// reader to optionally enable lazy metadata loading. This takes ownership
 /// of \p Buffer.
-std::unique_ptr<Module> getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer,
-                                        SMDiagnostic &Err, LLVMContext &Context,
-                                        bool ShouldLazyLoadMetadata = false);
+LLVM_ABI std::unique_ptr<Module>
+getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer, SMDiagnostic &Err,
+                LLVMContext &Context, bool ShouldLazyLoadMetadata = false);
 
 /// If the given file holds a bitcode image, return a Module
 /// for it which does lazy deserialization of function bodies.  Otherwise,
 /// attempt to parse it as LLVM Assembly and return a fully populated
 /// Module. The ShouldLazyLoadMetadata flag is passed down to the bitcode
 /// reader to optionally enable lazy metadata loading.
-std::unique_ptr<Module>
+LLVM_ABI std::unique_ptr<Module>
 getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context,
                     bool ShouldLazyLoadMetadata = false);
 
@@ -49,17 +50,19 @@ getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context,
 /// for it.  Otherwise, attempt to parse it as LLVM Assembly and return
 /// a Module for it.
 /// \param DataLayoutCallback Override datalayout in the llvm assembly.
-std::unique_ptr<Module> parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err,
-                                LLVMContext &Context,
-                                ParserCallbacks Callbacks = {});
+LLVM_ABI std::unique_ptr<Module> parseIR(MemoryBufferRef Buffer,
+                                         SMDiagnostic &Err,
+                                         LLVMContext &Context,
+                                         ParserCallbacks Callbacks = {});
 
 /// If the given file holds a bitcode image, return a Module for it.
 /// Otherwise, attempt to parse it as LLVM Assembly and return a Module
 /// for it.
 /// \param DataLayoutCallback Override datalayout in the llvm assembly.
-std::unique_ptr<Module> parseIRFile(StringRef Filename, SMDiagnostic &Err,
-                                    LLVMContext &Context,
-                                    ParserCallbacks Callbacks = {});
+LLVM_ABI std::unique_ptr<Module> parseIRFile(StringRef Filename,
+                                             SMDiagnostic &Err,
+                                             LLVMContext &Context,
+                                             ParserCallbacks Callbacks = {});
 }
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/InterfaceStub/ELFObjHandler.h b/external/llvm-project/llvm/include/llvm/InterfaceStub/ELFObjHandler.h
index c15838c4ae0a..0de0b3280480 100644
--- a/external/llvm-project/llvm/include/llvm/InterfaceStub/ELFObjHandler.h
+++ b/external/llvm-project/llvm/include/llvm/InterfaceStub/ELFObjHandler.h
@@ -14,6 +14,7 @@
 #define LLVM_INTERFACESTUB_ELFOBJHANDLER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include <memory>
@@ -24,7 +25,7 @@ namespace ifs {
 struct IFSStub;
 
 /// Attempt to read a binary ELF file from a MemoryBuffer.
-Expected<std::unique_ptr<IFSStub>> readELFFile(MemoryBufferRef Buf);
+LLVM_ABI Expected<std::unique_ptr<IFSStub>> readELFFile(MemoryBufferRef Buf);
 
 /// Attempt to write a binary ELF stub.
 /// This function determines appropriate ELFType using the passed ELFTarget and
@@ -34,8 +35,8 @@ Expected<std::unique_ptr<IFSStub>> readELFFile(MemoryBufferRef Buf);
 /// @param Stub Source ELFStub to generate a binary ELF stub from.
 /// @param WriteIfChanged Whether or not to preserve timestamp if
 ///        the output stays the same.
-Error writeBinaryStub(StringRef FilePath, const IFSStub &Stub,
-                      bool WriteIfChanged = false);
+LLVM_ABI Error writeBinaryStub(StringRef FilePath, const IFSStub &Stub,
+                               bool WriteIfChanged = false);
 
 } // end namespace ifs
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSHandler.h b/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSHandler.h
index 09687bfb9e44..d0f6eb52c576 100644
--- a/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSHandler.h
+++ b/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSHandler.h
@@ -16,6 +16,7 @@
 #define LLVM_INTERFACESTUB_IFSHANDLER_H
 
 #include "IFSStub.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/VersionTuple.h"
 #include <memory>
@@ -36,29 +37,30 @@ struct IFSStub;
 const VersionTuple IFSVersionCurrent(3, 0);
 
 /// Attempts to read an IFS interface file from a StringRef buffer.
-Expected<std::unique_ptr<IFSStub>> readIFSFromBuffer(StringRef Buf);
+LLVM_ABI Expected<std::unique_ptr<IFSStub>> readIFSFromBuffer(StringRef Buf);
 
 /// Attempts to write an IFS interface file to a raw_ostream.
-Error writeIFSToOutputStream(raw_ostream &OS, const IFSStub &Stub);
+LLVM_ABI Error writeIFSToOutputStream(raw_ostream &OS, const IFSStub &Stub);
 
 /// Override the target platform inforation in the text stub.
-Error overrideIFSTarget(IFSStub &Stub, std::optional<IFSArch> OverrideArch,
-                        std::optional<IFSEndiannessType> OverrideEndianness,
-                        std::optional<IFSBitWidthType> OverrideBitWidth,
-                        std::optional<std::string> OverrideTriple);
+LLVM_ABI Error
+overrideIFSTarget(IFSStub &Stub, std::optional<IFSArch> OverrideArch,
+                  std::optional<IFSEndiannessType> OverrideEndianness,
+                  std::optional<IFSBitWidthType> OverrideBitWidth,
+                  std::optional<std::string> OverrideTriple);
 
 /// Validate the target platform inforation in the text stub.
-Error validateIFSTarget(IFSStub &Stub, bool ParseTriple);
+LLVM_ABI Error validateIFSTarget(IFSStub &Stub, bool ParseTriple);
 
 /// Strips target platform information from the text stub.
-void stripIFSTarget(IFSStub &Stub, bool StripTriple, bool StripArch,
-                    bool StripEndianness, bool StripBitWidth);
+LLVM_ABI void stripIFSTarget(IFSStub &Stub, bool StripTriple, bool StripArch,
+                             bool StripEndianness, bool StripBitWidth);
 
-Error filterIFSSyms(IFSStub &Stub, bool StripUndefined,
-                    const std::vector<std::string> &Exclude = {});
+LLVM_ABI Error filterIFSSyms(IFSStub &Stub, bool StripUndefined,
+                             const std::vector<std::string> &Exclude = {});
 
 /// Parse llvm triple string into a IFSTarget struct.
-IFSTarget parseTriple(StringRef TripleStr);
+LLVM_ABI IFSTarget parseTriple(StringRef TripleStr);
 
 } // end namespace ifs
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSStub.h b/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSStub.h
index 09f96f72950c..ddde889d4689 100644
--- a/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSStub.h
+++ b/external/llvm-project/llvm/include/llvm/InterfaceStub/IFSStub.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_INTERFACESTUB_IFSSTUB_H
 #define LLVM_INTERFACESTUB_IFSSTUB_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
 #include <optional>
 #include <vector>
@@ -69,7 +70,7 @@ struct IFSTarget {
   std::optional<IFSEndiannessType> Endianness;
   std::optional<IFSBitWidthType> BitWidth;
 
-  bool empty();
+  LLVM_ABI bool empty();
 };
 
 inline bool operator==(const IFSTarget &Lhs, const IFSTarget &Rhs) {
@@ -95,8 +96,8 @@ struct IFSStub {
   std::vector<IFSSymbol> Symbols;
 
   IFSStub() = default;
-  IFSStub(const IFSStub &Stub);
-  IFSStub(IFSStub &&Stub);
+  LLVM_ABI IFSStub(const IFSStub &Stub);
+  LLVM_ABI IFSStub(IFSStub &&Stub);
   virtual ~IFSStub() = default;
 };
 
@@ -107,42 +108,42 @@ struct IFSStub {
 // structure can be used for 2 different yaml schema.
 struct IFSStubTriple : IFSStub {
   IFSStubTriple() = default;
-  IFSStubTriple(const IFSStub &Stub);
-  IFSStubTriple(const IFSStubTriple &Stub);
-  IFSStubTriple(IFSStubTriple &&Stub);
+  LLVM_ABI IFSStubTriple(const IFSStub &Stub);
+  LLVM_ABI IFSStubTriple(const IFSStubTriple &Stub);
+  LLVM_ABI IFSStubTriple(IFSStubTriple &&Stub);
 };
 
 /// This function convert bit width type from IFS enum to ELF format
 /// Currently, ELFCLASS32 and ELFCLASS64 are supported.
 ///
 /// @param BitWidth IFS bit width type.
-uint8_t convertIFSBitWidthToELF(IFSBitWidthType BitWidth);
+LLVM_ABI uint8_t convertIFSBitWidthToELF(IFSBitWidthType BitWidth);
 
 /// This function convert endianness type from IFS enum to ELF format
 /// Currently, ELFDATA2LSB and ELFDATA2MSB are supported.
 ///
 /// @param Endianness IFS endianness type.
-uint8_t convertIFSEndiannessToELF(IFSEndiannessType Endianness);
+LLVM_ABI uint8_t convertIFSEndiannessToELF(IFSEndiannessType Endianness);
 
 /// This function convert symbol type from IFS enum to ELF format
 /// Currently, STT_NOTYPE, STT_OBJECT, STT_FUNC, and STT_TLS are supported.
 ///
 /// @param SymbolType IFS symbol type.
-uint8_t convertIFSSymbolTypeToELF(IFSSymbolType SymbolType);
+LLVM_ABI uint8_t convertIFSSymbolTypeToELF(IFSSymbolType SymbolType);
 
 /// This function extracts ELF bit width from e_ident[EI_CLASS] of an ELF file
 /// Currently, ELFCLASS32 and ELFCLASS64 are supported.
 /// Other endianness types are mapped to IFSBitWidthType::Unknown.
 ///
 /// @param BitWidth e_ident[EI_CLASS] value to extract bit width from.
-IFSBitWidthType convertELFBitWidthToIFS(uint8_t BitWidth);
+LLVM_ABI IFSBitWidthType convertELFBitWidthToIFS(uint8_t BitWidth);
 
 /// This function extracts ELF endianness from e_ident[EI_DATA] of an ELF file
 /// Currently, ELFDATA2LSB and ELFDATA2MSB are supported.
 /// Other endianness types are mapped to IFSEndiannessType::Unknown.
 ///
 /// @param Endianness e_ident[EI_DATA] value to extract endianness type from.
-IFSEndiannessType convertELFEndiannessToIFS(uint8_t Endianness);
+LLVM_ABI IFSEndiannessType convertELFEndiannessToIFS(uint8_t Endianness);
 
 /// This function extracts symbol type from a symbol's st_info member and
 /// maps it to an IFSSymbolType enum.
@@ -150,7 +151,7 @@ IFSEndiannessType convertELFEndiannessToIFS(uint8_t Endianness);
 /// Other symbol types are mapped to IFSSymbolType::Unknown.
 ///
 /// @param SymbolType Binary symbol st_info to extract symbol type from.
-IFSSymbolType convertELFSymbolTypeToIFS(uint8_t SymbolType);
+LLVM_ABI IFSSymbolType convertELFSymbolTypeToIFS(uint8_t SymbolType);
 } // namespace ifs
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/LTO/Config.h b/external/llvm-project/llvm/include/llvm/LTO/Config.h
index f1d5436dbe22..4e08e7854570 100644
--- a/external/llvm-project/llvm/include/llvm/LTO/Config.h
+++ b/external/llvm-project/llvm/include/llvm/LTO/Config.h
@@ -22,6 +22,7 @@
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Target/TargetOptions.h"
 
 #include <functional>
@@ -279,9 +280,9 @@ struct Config {
   ///
   /// SaveTempsArgs can be specified to select which temps to save.
   /// If SaveTempsArgs is not provided, all temps are saved.
-  Error addSaveTemps(std::string OutputFileName,
-                     bool UseInputModulePath = false,
-                     const DenseSet<StringRef> &SaveTempsArgs = {});
+  LLVM_ABI Error addSaveTemps(std::string OutputFileName,
+                              bool UseInputModulePath = false,
+                              const DenseSet<StringRef> &SaveTempsArgs = {});
 };
 
 struct LTOLLVMDiagnosticHandler : public DiagnosticHandler {
diff --git a/external/llvm-project/llvm/include/llvm/LTO/LTO.h b/external/llvm-project/llvm/include/llvm/LTO/LTO.h
index ea045dcc5d7c..d8e632b5a49d 100644
--- a/external/llvm-project/llvm/include/llvm/LTO/LTO.h
+++ b/external/llvm-project/llvm/include/llvm/LTO/LTO.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_LTO_LTO_H
 #define LLVM_LTO_LTO_H
 
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 #include "llvm/ADT/DenseMap.h"
@@ -47,7 +48,7 @@ class ToolOutputFile;
 ///
 /// This is done for correctness (if value exported, ensure we always
 /// emit a copy), and compile-time optimization (allow drop of duplicates).
-void thinLTOResolvePrevailingInIndex(
+LLVM_ABI void thinLTOResolvePrevailingInIndex(
     const lto::Config &C, ModuleSummaryIndex &Index,
     function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
         isPrevailing,
@@ -58,7 +59,7 @@ void thinLTOResolvePrevailingInIndex(
 /// Update the linkages in the given \p Index to mark exported values
 /// as external and non-exported values as internal. The ThinLTO backends
 /// must apply the changes to the Module via thinLTOInternalizeModule.
-void thinLTOInternalizeAndPromoteInIndex(
+LLVM_ABI void thinLTOInternalizeAndPromoteInIndex(
     ModuleSummaryIndex &Index,
     function_ref<bool(StringRef, ValueInfo)> isExported,
     function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
@@ -66,7 +67,7 @@ void thinLTOInternalizeAndPromoteInIndex(
 
 /// Computes a unique hash for the Module considering the current list of
 /// export/import and other global analysis results.
-std::string computeLTOCacheKey(
+LLVM_ABI std::string computeLTOCacheKey(
     const lto::Config &Conf, const ModuleSummaryIndex &Index,
     StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
     const FunctionImporter::ExportSetTy &ExportList,
@@ -76,36 +77,38 @@ std::string computeLTOCacheKey(
     const DenseSet<GlobalValue::GUID> &CfiFunctionDecls = {});
 
 /// Recomputes the LTO cache key for a given key with an extra identifier.
-std::string recomputeLTOCacheKey(const std::string &Key, StringRef ExtraID);
+LLVM_ABI std::string recomputeLTOCacheKey(const std::string &Key,
+                                          StringRef ExtraID);
 
 namespace lto {
 
-StringLiteral getThinLTODefaultCPU(const Triple &TheTriple);
+LLVM_ABI StringLiteral getThinLTODefaultCPU(const Triple &TheTriple);
 
 /// Given the original \p Path to an output file, replace any path
 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
 /// resulting directory if it does not yet exist.
-std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
-                                 StringRef NewPrefix);
+LLVM_ABI std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
+                                          StringRef NewPrefix);
 
 /// Setup optimization remarks.
-Expected<std::unique_ptr<ToolOutputFile>> setupLLVMOptimizationRemarks(
+LLVM_ABI Expected<std::unique_ptr<ToolOutputFile>> setupLLVMOptimizationRemarks(
     LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
     StringRef RemarksFormat, bool RemarksWithHotness,
     std::optional<uint64_t> RemarksHotnessThreshold = 0, int Count = -1);
 
 /// Setups the output file for saving statistics.
-Expected<std::unique_ptr<ToolOutputFile>>
+LLVM_ABI Expected<std::unique_ptr<ToolOutputFile>>
 setupStatsFile(StringRef StatsFilename);
 
 /// Produces a container ordering for optimal multi-threaded processing. Returns
 /// ordered indices to elements in the input array.
-std::vector<int> generateModulesOrdering(ArrayRef<BitcodeModule *> R);
+LLVM_ABI std::vector<int> generateModulesOrdering(ArrayRef<BitcodeModule *> R);
 
 /// Updates MemProf attributes (and metadata) based on whether the index
 /// has recorded that we are linking with allocation libraries containing
 /// the necessary APIs for downstream transformations.
-void updateMemProfAttributes(Module &Mod, const ModuleSummaryIndex &Index);
+LLVM_ABI void updateMemProfAttributes(Module &Mod,
+                                      const ModuleSummaryIndex &Index);
 
 class LTO;
 struct SymbolResolution;
@@ -133,10 +136,11 @@ class InputFile {
   std::vector<std::pair<StringRef, Comdat::SelectionKind>> ComdatTable;
 
 public:
-  ~InputFile();
+  LLVM_ABI ~InputFile();
 
   /// Create an InputFile.
-  static Expected<std::unique_ptr<InputFile>> create(MemoryBufferRef Object);
+  LLVM_ABI static Expected<std::unique_ptr<InputFile>>
+  create(MemoryBufferRef Object);
 
   /// The purpose of this struct is to only expose the symbol information that
   /// an LTO client should need in order to do symbol resolution.
@@ -174,7 +178,7 @@ class InputFile {
   ArrayRef<StringRef> getDependentLibraries() const { return DependentLibraries; }
 
   /// Returns the path to the InputFile.
-  StringRef getName() const;
+  LLVM_ABI StringRef getName() const;
 
   /// Returns the input file's target triple.
   StringRef getTargetTriple() const { return TargetTriple; }
@@ -188,7 +192,7 @@ class InputFile {
   }
 
   // Returns the only BitcodeModule from InputFile.
-  BitcodeModule &getSingleBitcodeModule();
+  LLVM_ABI BitcodeModule &getSingleBitcodeModule();
 
 private:
   ArrayRef<Symbol> module_symbols(unsigned I) const {
@@ -243,16 +247,17 @@ class ThinBackendProc {
   virtual bool isSensitiveToInputOrder() { return false; }
 
   // Write sharded indices and (optionally) imports to disk
-  Error emitFiles(const FunctionImporter::ImportMapTy &ImportList,
-                  StringRef ModulePath, const std::string &NewModulePath) const;
+  LLVM_ABI Error emitFiles(const FunctionImporter::ImportMapTy &ImportList,
+                           StringRef ModulePath,
+                           const std::string &NewModulePath) const;
 
   // Write sharded indices to SummaryPath, (optionally) imports to disk, and
   // (optionally) record imports in ImportsFiles.
-  Error emitFiles(const FunctionImporter::ImportMapTy &ImportList,
-                  StringRef ModulePath, const std::string &NewModulePath,
-                  StringRef SummaryPath,
-                  std::optional<std::reference_wrapper<ImportsFilesContainer>>
-                      ImportsFiles) const;
+  LLVM_ABI Error emitFiles(
+      const FunctionImporter::ImportMapTy &ImportList, StringRef ModulePath,
+      const std::string &NewModulePath, StringRef SummaryPath,
+      std::optional<std::reference_wrapper<ImportsFilesContainer>> ImportsFiles)
+      const;
 };
 
 /// This callable defines the behavior of a ThinLTO backend after the thin-link
@@ -300,10 +305,9 @@ struct ThinBackend {
 /// to the same path as the input module, with suffix ".thinlto.bc"
 /// ShouldEmitImportsFiles is true it also writes a list of imported files to a
 /// similar path with ".imports" appended instead.
-ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism,
-                                       IndexWriteCallback OnWrite = nullptr,
-                                       bool ShouldEmitIndexFiles = false,
-                                       bool ShouldEmitImportsFiles = false);
+LLVM_ABI ThinBackend createInProcessThinBackend(
+    ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite = nullptr,
+    bool ShouldEmitIndexFiles = false, bool ShouldEmitImportsFiles = false);
 
 /// This ThinBackend generates the index shards and then runs the individual
 /// backend jobs via an external process. It takes the same parameters as the
@@ -322,7 +326,7 @@ ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism,
 /// backend compilations.
 /// SaveTemps is a debugging tool that prevents temporary files created by this
 /// backend from being cleaned up.
-ThinBackend createOutOfProcessThinBackend(
+LLVM_ABI ThinBackend createOutOfProcessThinBackend(
     ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite,
     bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
     StringRef LinkerOutputFile, StringRef Distributor,
@@ -344,13 +348,11 @@ ThinBackend createOutOfProcessThinBackend(
 /// the objects with NativeObjectPrefix instead of NewPrefix. OnWrite is
 /// callback which receives module identifier and notifies LTO user that index
 /// file for the module (and optionally imports file) was created.
-ThinBackend createWriteIndexesThinBackend(ThreadPoolStrategy Parallelism,
-                                          std::string OldPrefix,
-                                          std::string NewPrefix,
-                                          std::string NativeObjectPrefix,
-                                          bool ShouldEmitImportsFiles,
-                                          raw_fd_ostream *LinkedObjectsFile,
-                                          IndexWriteCallback OnWrite);
+LLVM_ABI ThinBackend createWriteIndexesThinBackend(
+    ThreadPoolStrategy Parallelism, std::string OldPrefix,
+    std::string NewPrefix, std::string NativeObjectPrefix,
+    bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile,
+    IndexWriteCallback OnWrite);
 
 /// This class implements a resolution-based interface to LLVM's LTO
 /// functionality. It supports regular LTO, parallel LTO code generation and
@@ -389,20 +391,21 @@ class LTO {
   /// this constructor.
   /// FIXME: We do currently require the DiagHandler field to be set in Conf.
   /// Until that is fixed, a Config argument is required.
-  LTO(Config Conf, ThinBackend Backend = {},
-      unsigned ParallelCodeGenParallelismLevel = 1,
-      LTOKind LTOMode = LTOK_Default);
-  ~LTO();
+  LLVM_ABI LTO(Config Conf, ThinBackend Backend = {},
+               unsigned ParallelCodeGenParallelismLevel = 1,
+               LTOKind LTOMode = LTOK_Default);
+  LLVM_ABI ~LTO();
 
   /// Add an input file to the LTO link, using the provided symbol resolutions.
   /// The symbol resolutions must appear in the enumeration order given by
   /// InputFile::symbols().
-  Error add(std::unique_ptr<InputFile> Obj, ArrayRef<SymbolResolution> Res);
+  LLVM_ABI Error add(std::unique_ptr<InputFile> Obj,
+                     ArrayRef<SymbolResolution> Res);
 
   /// Returns an upper bound on the number of tasks that the client may expect.
   /// This may only be called after all IR object files have been added. For a
   /// full description of tasks see LTOBackend.h.
-  unsigned getMaxTasks() const;
+  LLVM_ABI unsigned getMaxTasks() const;
 
   /// Runs the LTO pipeline. This function calls the supplied AddStream
   /// function to add native object files to the link.
@@ -412,18 +415,19 @@ class LTO {
   ///
   /// The client will receive at most one callback (via either AddStream or
   /// Cache) for each task identifier.
-  Error run(AddStreamFn AddStream, FileCache Cache = {});
+  LLVM_ABI Error run(AddStreamFn AddStream, FileCache Cache = {});
 
   /// Static method that returns a list of libcall symbols that can be generated
   /// by LTO but might not be visible from bitcode symbol table.
-  static SmallVector<const char *> getRuntimeLibcallSymbols(const Triple &TT);
+  LLVM_ABI static SmallVector<const char *>
+  getRuntimeLibcallSymbols(const Triple &TT);
 
 private:
   Config Conf;
 
   struct RegularLTOState {
-    RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
-                    const Config &Conf);
+    LLVM_ABI RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
+                             const Config &Conf);
     struct CommonResolution {
       uint64_t Size = 0;
       Align Alignment;
@@ -452,7 +456,7 @@ class LTO {
   using ModuleMapType = MapVector<StringRef, BitcodeModule>;
 
   struct ThinLTOState {
-    ThinLTOState(ThinBackend Backend);
+    LLVM_ABI ThinLTOState(ThinBackend Backend);
 
     ThinBackend Backend;
     ModuleSummaryIndex CombinedIndex;
diff --git a/external/llvm-project/llvm/include/llvm/LTO/LTOBackend.h b/external/llvm-project/llvm/include/llvm/LTO/LTOBackend.h
index 2769e58f2490..86b488c764e0 100644
--- a/external/llvm-project/llvm/include/llvm/LTO/LTOBackend.h
+++ b/external/llvm-project/llvm/include/llvm/LTO/LTOBackend.h
@@ -20,6 +20,7 @@
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/LTO/LTO.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/IPO/FunctionImport.h"
@@ -34,16 +35,17 @@ class Target;
 namespace lto {
 
 /// Runs middle-end LTO optimizations on \p Mod.
-bool opt(const Config &Conf, TargetMachine *TM, unsigned Task, Module &Mod,
-         bool IsThinLTO, ModuleSummaryIndex *ExportSummary,
-         const ModuleSummaryIndex *ImportSummary,
-         const std::vector<uint8_t> &CmdArgs);
+LLVM_ABI bool opt(const Config &Conf, TargetMachine *TM, unsigned Task,
+                  Module &Mod, bool IsThinLTO,
+                  ModuleSummaryIndex *ExportSummary,
+                  const ModuleSummaryIndex *ImportSummary,
+                  const std::vector<uint8_t> &CmdArgs);
 
 /// Runs a regular LTO backend. The regular LTO backend can also act as the
 /// regular LTO phase of ThinLTO, which may need to access the combined index.
-Error backend(const Config &C, AddStreamFn AddStream,
-              unsigned ParallelCodeGenParallelismLevel, Module &M,
-              ModuleSummaryIndex &CombinedIndex);
+LLVM_ABI Error backend(const Config &C, AddStreamFn AddStream,
+                       unsigned ParallelCodeGenParallelismLevel, Module &M,
+                       ModuleSummaryIndex &CombinedIndex);
 
 /// Runs a ThinLTO backend.
 /// If \p ModuleMap is not nullptr, all the module files to be imported have
@@ -54,28 +56,30 @@ Error backend(const Config &C, AddStreamFn AddStream,
 /// the backend will skip optimization and only perform code generation. If
 /// \p IRAddStream is not nullptr, it will be called just before code generation
 /// to serialize the optimized IR.
-Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream,
-                  Module &M, const ModuleSummaryIndex &CombinedIndex,
-                  const FunctionImporter::ImportMapTy &ImportList,
-                  const GVSummaryMapTy &DefinedGlobals,
-                  MapVector<StringRef, BitcodeModule> *ModuleMap,
-                  bool CodeGenOnly, AddStreamFn IRAddStream = nullptr,
-                  const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>());
+LLVM_ABI Error
+thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M,
+            const ModuleSummaryIndex &CombinedIndex,
+            const FunctionImporter::ImportMapTy &ImportList,
+            const GVSummaryMapTy &DefinedGlobals,
+            MapVector<StringRef, BitcodeModule> *ModuleMap, bool CodeGenOnly,
+            AddStreamFn IRAddStream = nullptr,
+            const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>());
 
-Error finalizeOptimizationRemarks(
-    std::unique_ptr<ToolOutputFile> DiagOutputFile);
+LLVM_ABI Error
+finalizeOptimizationRemarks(std::unique_ptr<ToolOutputFile> DiagOutputFile);
 
 /// Returns the BitcodeModule that is ThinLTO.
-BitcodeModule *findThinLTOModule(MutableArrayRef<BitcodeModule> BMs);
+LLVM_ABI BitcodeModule *findThinLTOModule(MutableArrayRef<BitcodeModule> BMs);
 
 /// Variant of the above.
-Expected<BitcodeModule> findThinLTOModule(MemoryBufferRef MBRef);
+LLVM_ABI Expected<BitcodeModule> findThinLTOModule(MemoryBufferRef MBRef);
 
 /// Distributed ThinLTO: collect the referenced modules based on
 /// module summary and initialize ImportList. Returns false if the
 /// operation failed.
-bool initImportList(const Module &M, const ModuleSummaryIndex &CombinedIndex,
-                    FunctionImporter::ImportMapTy &ImportList);
+LLVM_ABI bool initImportList(const Module &M,
+                             const ModuleSummaryIndex &CombinedIndex,
+                             FunctionImporter::ImportMapTy &ImportList);
 }
 }
 
diff --git a/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
index d8e41fe92258..391589acba40 100644
--- a/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
+++ b/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
@@ -44,6 +44,7 @@
 #include "llvm/LTO/Config.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Target/TargetMachine.h"
@@ -53,41 +54,41 @@
 
 namespace llvm {
 template <typename T> class ArrayRef;
-  class LLVMContext;
-  class DiagnosticInfo;
-  class Linker;
-  class Mangler;
-  class MemoryBuffer;
-  class TargetLibraryInfo;
-  class TargetMachine;
-  class raw_ostream;
-  class raw_pwrite_stream;
+class LLVMContext;
+class DiagnosticInfo;
+class Linker;
+class Mangler;
+class MemoryBuffer;
+class TargetLibraryInfo;
+class TargetMachine;
+class raw_ostream;
+class raw_pwrite_stream;
 
 /// Enable global value internalization in LTO.
-extern cl::opt<bool> EnableLTOInternalization;
+LLVM_ABI extern cl::opt<bool> EnableLTOInternalization;
 
 //===----------------------------------------------------------------------===//
 /// C++ class which implements the opaque lto_code_gen_t type.
 ///
 struct LTOCodeGenerator {
-  static const char *getVersionString();
+  LLVM_ABI static const char *getVersionString();
 
-  LTOCodeGenerator(LLVMContext &Context);
-  ~LTOCodeGenerator();
+  LLVM_ABI LTOCodeGenerator(LLVMContext &Context);
+  LLVM_ABI ~LTOCodeGenerator();
 
   /// Merge given module.  Return true on success.
   ///
   /// Resets \a HasVerifiedInput.
-  bool addModule(struct LTOModule *);
+  LLVM_ABI bool addModule(struct LTOModule *);
 
   /// Set the destination module.
   ///
   /// Resets \a HasVerifiedInput.
-  void setModule(std::unique_ptr<LTOModule> M);
+  LLVM_ABI void setModule(std::unique_ptr<LTOModule> M);
 
-  void setAsmUndefinedRefs(struct LTOModule *);
-  void setTargetOptions(const TargetOptions &Options);
-  void setDebugInfo(lto_debug_model);
+  LLVM_ABI void setAsmUndefinedRefs(struct LTOModule *);
+  LLVM_ABI void setTargetOptions(const TargetOptions &Options);
+  LLVM_ABI void setDebugInfo(lto_debug_model);
   void setCodePICModel(std::optional<Reloc::Model> Model) {
     Config.RelocModel = Model;
   }
@@ -100,7 +101,7 @@ struct LTOCodeGenerator {
   void setAttrs(std::vector<std::string> MAttrs) {
     Config.MAttrs = std::move(MAttrs);
   }
-  void setOptLevel(unsigned OptLevel);
+  LLVM_ABI void setOptLevel(unsigned OptLevel);
 
   void setShouldInternalize(bool Value) { ShouldInternalize = Value; }
   void setShouldEmbedUselists(bool Value) { ShouldEmbedUselists = Value; }
@@ -132,20 +133,20 @@ struct LTOCodeGenerator {
   /// name is misleading).  This function should be called before
   /// LTOCodeGenerator::compilexxx(), and
   /// LTOCodeGenerator::writeMergedModules().
-  void setCodeGenDebugOptions(ArrayRef<StringRef> Opts);
+  LLVM_ABI void setCodeGenDebugOptions(ArrayRef<StringRef> Opts);
 
   /// Parse the options set in setCodeGenDebugOptions.
   ///
   /// Like \a setCodeGenDebugOptions(), this must be called before
   /// LTOCodeGenerator::compilexxx() and
   /// LTOCodeGenerator::writeMergedModules().
-  void parseCodeGenDebugOptions();
+  LLVM_ABI void parseCodeGenDebugOptions();
 
   /// Write the merged module to the file specified by the given path.  Return
   /// true on success.
   ///
   /// Calls \a verifyMergedModuleOnce().
-  bool writeMergedModules(StringRef Path);
+  LLVM_ABI bool writeMergedModules(StringRef Path);
 
   /// Compile the merged module into a *single* output file; the path to output
   /// file is returned to the caller via argument "name". Return true on
@@ -154,7 +155,7 @@ struct LTOCodeGenerator {
   /// \note It is up to the linker to remove the intermediate output file.  Do
   /// not try to remove the object file in LTOCodeGenerator's destructor as we
   /// don't who (LTOCodeGenerator or the output file) will last longer.
-  bool compile_to_file(const char **Name);
+  LLVM_ABI bool compile_to_file(const char **Name);
 
   /// As with compile_to_file(), this function compiles the merged module into
   /// single output file. Instead of returning the output file path to the
@@ -162,17 +163,17 @@ struct LTOCodeGenerator {
   /// to the caller. This function should delete the intermediate file once
   /// its content is brought to memory. Return NULL if the compilation was not
   /// successful.
-  std::unique_ptr<MemoryBuffer> compile();
+  LLVM_ABI std::unique_ptr<MemoryBuffer> compile();
 
   /// Optimizes the merged module.  Returns true on success.
   ///
   /// Calls \a verifyMergedModuleOnce().
-  bool optimize();
+  LLVM_ABI bool optimize();
 
   /// Compiles the merged optimized module into a single output file. It brings
   /// the output to a buffer, and returns the buffer to the caller. Return NULL
   /// if the compilation was not successful.
-  std::unique_ptr<MemoryBuffer> compileOptimized();
+  LLVM_ABI std::unique_ptr<MemoryBuffer> compileOptimized();
 
   /// Compile the merged optimized module \p ParallelismLevel output files each
   /// representing a linkable partition of the module. If out contains more
@@ -181,7 +182,8 @@ struct LTOCodeGenerator {
   /// created using the \p AddStream callback. Returns true on success.
   ///
   /// Calls \a verifyMergedModuleOnce().
-  bool compileOptimized(AddStreamFn AddStream, unsigned ParallelismLevel);
+  LLVM_ABI bool compileOptimized(AddStreamFn AddStream,
+                                 unsigned ParallelismLevel);
 
   /// Enable the Freestanding mode: indicate that the optimizer should not
   /// assume builtins are present on the target.
@@ -191,12 +193,12 @@ struct LTOCodeGenerator {
 
   void setDebugPassManager(bool Enabled) { Config.DebugPassManager = Enabled; }
 
-  void setDiagnosticHandler(lto_diagnostic_handler_t, void *);
+  LLVM_ABI void setDiagnosticHandler(lto_diagnostic_handler_t, void *);
 
   LLVMContext &getContext() { return Context; }
 
   void resetMergedModule() { MergedModule.reset(); }
-  void DiagnosticHandler(const DiagnosticInfo &DI);
+  LLVM_ABI void DiagnosticHandler(const DiagnosticInfo &DI);
 
 private:
   /// Verify the merged module on first call.
@@ -252,6 +254,6 @@ struct LTOCodeGenerator {
 
 /// A convenience function that calls cl::ParseCommandLineOptions on the given
 /// set of options.
-void parseCommandLineOptions(std::vector<std::string> &Options);
-}
+LLVM_ABI void parseCommandLineOptions(std::vector<std::string> &Options);
+} // namespace llvm
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOModule.h b/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOModule.h
index 91a38b7c0da8..2b6a8734e78f 100644
--- a/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOModule.h
+++ b/external/llvm-project/llvm/include/llvm/LTO/legacy/LTOModule.h
@@ -20,6 +20,7 @@
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Object/ModuleSymbolTable.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Target/TargetMachine.h"
 #include <string>
 #include <vector>
@@ -63,27 +64,27 @@ struct LTOModule {
             TargetMachine *TM);
 
 public:
-  ~LTOModule();
+  LLVM_ABI ~LTOModule();
 
   /// Returns 'true' if the file or memory contents is LLVM bitcode.
-  static bool isBitcodeFile(const void *mem, size_t length);
-  static bool isBitcodeFile(StringRef path);
+  LLVM_ABI static bool isBitcodeFile(const void *mem, size_t length);
+  LLVM_ABI static bool isBitcodeFile(StringRef path);
 
   /// Returns 'true' if the Module is produced for ThinLTO.
-  bool isThinLTO();
+  LLVM_ABI bool isThinLTO();
 
   /// Returns 'true' if the memory buffer is LLVM bitcode for the specified
   /// triple.
-  static bool isBitcodeForTarget(MemoryBuffer *memBuffer,
-                                 StringRef triplePrefix);
+  LLVM_ABI static bool isBitcodeForTarget(MemoryBuffer *memBuffer,
+                                          StringRef triplePrefix);
 
   /// Returns a string representing the producer identification stored in the
   /// bitcode, or "" if the bitcode does not contains any.
   ///
-  static std::string getProducerString(MemoryBuffer *Buffer);
+  LLVM_ABI static std::string getProducerString(MemoryBuffer *Buffer);
 
   /// Create a MemoryBuffer from a memory range with an optional name.
-  static std::unique_ptr<MemoryBuffer>
+  LLVM_ABI static std::unique_ptr<MemoryBuffer>
   makeBuffer(const void *mem, size_t length, StringRef name = "");
 
   /// Create an LTOModule. N.B. These methods take ownership of the buffer. The
@@ -94,20 +95,20 @@ struct LTOModule {
   /// InitializeAllTargetMCs();
   /// InitializeAllAsmPrinters();
   /// InitializeAllAsmParsers();
-  static ErrorOr<std::unique_ptr<LTOModule>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<LTOModule>>
   createFromFile(LLVMContext &Context, StringRef path,
                  const TargetOptions &options);
-  static ErrorOr<std::unique_ptr<LTOModule>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<LTOModule>>
   createFromOpenFile(LLVMContext &Context, int fd, StringRef path, size_t size,
                      const TargetOptions &options);
-  static ErrorOr<std::unique_ptr<LTOModule>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<LTOModule>>
   createFromOpenFileSlice(LLVMContext &Context, int fd, StringRef path,
                           size_t map_size, off_t offset,
                           const TargetOptions &options);
-  static ErrorOr<std::unique_ptr<LTOModule>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<LTOModule>>
   createFromBuffer(LLVMContext &Context, const void *mem, size_t length,
                    const TargetOptions &options, StringRef path = "");
-  static ErrorOr<std::unique_ptr<LTOModule>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<LTOModule>>
   createInLocalContext(std::unique_ptr<LLVMContext> Context, const void *mem,
                        size_t length, const TargetOptions &options,
                        StringRef path);
@@ -152,20 +153,23 @@ struct LTOModule {
 
   const std::vector<StringRef> &getAsmUndefinedRefs() { return _asm_undefines; }
 
-  static lto::InputFile *createInputFile(const void *buffer, size_t buffer_size,
-                                         const char *path, std::string &out_error);
+  LLVM_ABI static lto::InputFile *createInputFile(const void *buffer,
+                                                  size_t buffer_size,
+                                                  const char *path,
+                                                  std::string &out_error);
 
-  static size_t getDependentLibraryCount(lto::InputFile *input);
+  LLVM_ABI static size_t getDependentLibraryCount(lto::InputFile *input);
 
-  static const char *getDependentLibrary(lto::InputFile *input, size_t index, size_t *size);
+  LLVM_ABI static const char *getDependentLibrary(lto::InputFile *input,
+                                                  size_t index, size_t *size);
 
-  Expected<uint32_t> getMachOCPUType() const;
+  LLVM_ABI Expected<uint32_t> getMachOCPUType() const;
 
-  Expected<uint32_t> getMachOCPUSubType() const;
+  LLVM_ABI Expected<uint32_t> getMachOCPUSubType() const;
 
   /// Returns true if the module has either the @llvm.global_ctors or the
   /// @llvm.global_dtors symbol. Otherwise returns false.
-  bool hasCtorDtor() const;
+  LLVM_ABI bool hasCtorDtor() const;
 
 private:
   /// Parse metadata from the module
diff --git a/external/llvm-project/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h b/external/llvm-project/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
index 676d4e179bed..d7e8356fa5f5 100644
--- a/external/llvm-project/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
+++ b/external/llvm-project/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
@@ -21,6 +21,7 @@
 #include "llvm/LTO/LTO.h"
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/TargetParser/Triple.h"
@@ -47,7 +48,7 @@ struct ThinLTOCodeGeneratorImpl::TargetMachineBuilder {
   std::optional<Reloc::Model> RelocModel;
   CodeGenOptLevel CGOptLevel = CodeGenOptLevel::Aggressive;
 
-  std::unique_ptr<TargetMachine> create() const;
+  LLVM_ABI std::unique_ptr<TargetMachine> create() const;
 };
 
 /// This class define an interface similar to the LTOCodeGenerator, but adapted
@@ -59,14 +60,14 @@ struct ThinLTOCodeGeneratorImpl::TargetMachineBuilder {
 class ThinLTOCodeGenerator {
 public:
   /// Add given module to the code generator.
-  void addModule(StringRef Identifier, StringRef Data);
+  LLVM_ABI void addModule(StringRef Identifier, StringRef Data);
 
   /**
    * Adds to a list of all global symbols that must exist in the final generated
    * code. If a symbol is not listed there, it will be optimized away if it is
    * inlined into every usage.
    */
-  void preserveSymbol(StringRef Name);
+  LLVM_ABI void preserveSymbol(StringRef Name);
 
   /**
    * Adds to a list of all global symbols that are cross-referenced between
@@ -74,7 +75,7 @@ class ThinLTOCodeGenerator {
    * references from a ThinLTO module to this symbol is optimized away, then
    * the symbol can be discarded.
    */
-  void crossReferenceSymbol(StringRef Name);
+  LLVM_ABI void crossReferenceSymbol(StringRef Name);
 
   /**
    * Process all the modules that were added to the code generator in parallel.
@@ -83,7 +84,7 @@ class ThinLTOCodeGenerator {
    * unless setGeneratedObjectsDirectory() has been called, in which case
    * results are available through getProducedBinaryFiles().
    */
-  void run();
+  LLVM_ABI void run();
 
   /**
    * Return the "in memory" binaries produced by the code generator. This is
@@ -253,35 +254,35 @@ class ThinLTOCodeGenerator {
    * Produce the combined summary index from all the bitcode files:
    * "thin-link".
    */
-  std::unique_ptr<ModuleSummaryIndex> linkCombinedIndex();
+  LLVM_ABI std::unique_ptr<ModuleSummaryIndex> linkCombinedIndex();
 
   /**
    * Perform promotion and renaming of exported internal functions,
    * and additionally resolve weak and linkonce symbols.
    * Index is updated to reflect linkage changes from weak resolution.
    */
-  void promote(Module &Module, ModuleSummaryIndex &Index,
-               const lto::InputFile &File);
+  LLVM_ABI void promote(Module &Module, ModuleSummaryIndex &Index,
+                        const lto::InputFile &File);
 
   /**
    * Compute and emit the imported files for module at \p ModulePath.
    */
-  void emitImports(Module &Module, StringRef OutputName,
-                   ModuleSummaryIndex &Index,
-                   const lto::InputFile &File);
+  LLVM_ABI void emitImports(Module &Module, StringRef OutputName,
+                            ModuleSummaryIndex &Index,
+                            const lto::InputFile &File);
 
   /**
    * Perform cross-module importing for the module identified by
    * ModuleIdentifier.
    */
-  void crossModuleImport(Module &Module, ModuleSummaryIndex &Index,
-                         const lto::InputFile &File);
+  LLVM_ABI void crossModuleImport(Module &Module, ModuleSummaryIndex &Index,
+                                  const lto::InputFile &File);
 
   /**
    * Compute the list of summaries and the subset of declaration summaries
    * needed for importing into module.
    */
-  void gatherImportedSummariesForModule(
+  LLVM_ABI void gatherImportedSummariesForModule(
       Module &Module, ModuleSummaryIndex &Index,
       ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
       GVSummaryPtrSet &DecSummaries, const lto::InputFile &File);
@@ -289,21 +290,21 @@ class ThinLTOCodeGenerator {
   /**
    * Perform internalization. Index is updated to reflect linkage changes.
    */
-  void internalize(Module &Module, ModuleSummaryIndex &Index,
-                   const lto::InputFile &File);
+  LLVM_ABI void internalize(Module &Module, ModuleSummaryIndex &Index,
+                            const lto::InputFile &File);
 
   /**
    * Perform post-importing ThinLTO optimizations.
    */
-  void optimize(Module &Module);
+  LLVM_ABI void optimize(Module &Module);
 
   /**
    * Write temporary object file to SavedObjectDirectoryPath, write symlink
    * to Cache directory if needed. Returns the path to the generated file in
    * SavedObjectsDirectoryPath.
    */
-  std::string writeGeneratedObject(int count, StringRef CacheEntryPath,
-                                   const MemoryBuffer &OutputBuffer);
+  LLVM_ABI std::string writeGeneratedObject(int count, StringRef CacheEntryPath,
+                                            const MemoryBuffer &OutputBuffer);
   /**@}*/
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/LineEditor/LineEditor.h b/external/llvm-project/llvm/include/llvm/LineEditor/LineEditor.h
index ca0312bf6297..5934caad2e40 100644
--- a/external/llvm-project/llvm/include/llvm/LineEditor/LineEditor.h
+++ b/external/llvm-project/llvm/include/llvm/LineEditor/LineEditor.h
@@ -10,6 +10,7 @@
 #define LLVM_LINEEDITOR_LINEEDITOR_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdio>
 #include <memory>
 #include <optional>
@@ -30,20 +31,20 @@ class LineEditor {
   /// \param In The input stream used by the editor.
   /// \param Out The output stream used by the editor.
   /// \param Err The error stream used by the editor.
-  LineEditor(StringRef ProgName, StringRef HistoryPath = "", FILE *In = stdin,
-             FILE *Out = stdout, FILE *Err = stderr);
-  ~LineEditor();
+  LLVM_ABI LineEditor(StringRef ProgName, StringRef HistoryPath = "",
+                      FILE *In = stdin, FILE *Out = stdout, FILE *Err = stderr);
+  LLVM_ABI ~LineEditor();
 
   /// Reads a line.
   ///
   /// \return The line, or std::optional<std::string>() on EOF.
-  std::optional<std::string> readLine() const;
+  LLVM_ABI std::optional<std::string> readLine() const;
 
-  void saveHistory();
-  void loadHistory();
-  void setHistorySize(int size);
+  LLVM_ABI void saveHistory();
+  LLVM_ABI void loadHistory();
+  LLVM_ABI void setHistorySize(int size);
 
-  static std::string getDefaultHistoryPath(StringRef ProgName);
+  LLVM_ABI static std::string getDefaultHistoryPath(StringRef ProgName);
 
   /// The action to perform upon a completion request.
   struct CompletionAction {
@@ -101,7 +102,8 @@ class LineEditor {
   ///
   /// \param Buffer The string to complete
   /// \param Pos The zero-based cursor position in the StringRef
-  CompletionAction getCompletionAction(StringRef Buffer, size_t Pos) const;
+  LLVM_ABI CompletionAction getCompletionAction(StringRef Buffer,
+                                                size_t Pos) const;
 
   const std::string &getPrompt() const { return Prompt; }
   void setPrompt(const std::string &P) { Prompt = P; }
@@ -114,12 +116,12 @@ class LineEditor {
   std::string HistoryPath;
   std::unique_ptr<InternalData> Data;
 
-  struct CompleterConcept {
+  struct LLVM_ABI CompleterConcept {
     virtual ~CompleterConcept();
     virtual CompletionAction complete(StringRef Buffer, size_t Pos) const = 0;
   };
 
-  struct ListCompleterConcept : CompleterConcept {
+  struct LLVM_ABI ListCompleterConcept : CompleterConcept {
     ~ListCompleterConcept() override;
     CompletionAction complete(StringRef Buffer, size_t Pos) const override;
     static std::string getCommonPrefix(const std::vector<Completion> &Comps);
diff --git a/external/llvm-project/llvm/include/llvm/Linker/IRMover.h b/external/llvm-project/llvm/include/llvm/Linker/IRMover.h
index 1e3c5394ffa2..39929311a322 100644
--- a/external/llvm-project/llvm/include/llvm/Linker/IRMover.h
+++ b/external/llvm-project/llvm/include/llvm/Linker/IRMover.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/FunctionExtras.h"
+#include "llvm/Support/Compiler.h"
 #include <functional>
 
 namespace llvm {
@@ -28,17 +29,17 @@ class IRMover {
     struct KeyTy {
       ArrayRef<Type *> ETypes;
       bool IsPacked;
-      KeyTy(ArrayRef<Type *> E, bool P);
-      KeyTy(const StructType *ST);
-      bool operator==(const KeyTy &that) const;
-      bool operator!=(const KeyTy &that) const;
+      LLVM_ABI KeyTy(ArrayRef<Type *> E, bool P);
+      LLVM_ABI KeyTy(const StructType *ST);
+      LLVM_ABI bool operator==(const KeyTy &that) const;
+      LLVM_ABI bool operator!=(const KeyTy &that) const;
     };
-    static StructType *getEmptyKey();
-    static StructType *getTombstoneKey();
-    static unsigned getHashValue(const KeyTy &Key);
-    static unsigned getHashValue(const StructType *ST);
-    static bool isEqual(const KeyTy &LHS, const StructType *RHS);
-    static bool isEqual(const StructType *LHS, const StructType *RHS);
+    LLVM_ABI static StructType *getEmptyKey();
+    LLVM_ABI static StructType *getTombstoneKey();
+    LLVM_ABI static unsigned getHashValue(const KeyTy &Key);
+    LLVM_ABI static unsigned getHashValue(const StructType *ST);
+    LLVM_ABI static bool isEqual(const KeyTy &LHS, const StructType *RHS);
+    LLVM_ABI static bool isEqual(const StructType *LHS, const StructType *RHS);
   };
 
   /// Type of the Metadata map in \a ValueToValueMapTy.
@@ -53,14 +54,14 @@ class IRMover {
     DenseSet<StructType *, StructTypeKeyInfo> NonOpaqueStructTypes;
 
   public:
-    void addNonOpaque(StructType *Ty);
-    void switchToNonOpaque(StructType *Ty);
-    void addOpaque(StructType *Ty);
-    StructType *findNonOpaque(ArrayRef<Type *> ETypes, bool IsPacked);
-    bool hasType(StructType *Ty);
+    LLVM_ABI void addNonOpaque(StructType *Ty);
+    LLVM_ABI void switchToNonOpaque(StructType *Ty);
+    LLVM_ABI void addOpaque(StructType *Ty);
+    LLVM_ABI StructType *findNonOpaque(ArrayRef<Type *> ETypes, bool IsPacked);
+    LLVM_ABI bool hasType(StructType *Ty);
   };
 
-  IRMover(Module &M);
+  LLVM_ABI IRMover(Module &M);
 
   typedef std::function<void(GlobalValue &)> ValueAdder;
   using LazyCallback =
@@ -76,8 +77,9 @@ class IRMover {
   ///   Pass nullptr if there's no work to be done in such cases.
   /// - \p IsPerformingImport is true when this IR link is to perform ThinLTO
   ///   function importing from Src.
-  Error move(std::unique_ptr<Module> Src, ArrayRef<GlobalValue *> ValuesToLink,
-             LazyCallback AddLazyFor, bool IsPerformingImport);
+  LLVM_ABI Error move(std::unique_ptr<Module> Src,
+                      ArrayRef<GlobalValue *> ValuesToLink,
+                      LazyCallback AddLazyFor, bool IsPerformingImport);
   Module &getModule() { return Composite; }
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/Linker/Linker.h b/external/llvm-project/llvm/include/llvm/Linker/Linker.h
index f8bc2837b413..2cbf20034a6e 100644
--- a/external/llvm-project/llvm/include/llvm/Linker/Linker.h
+++ b/external/llvm-project/llvm/include/llvm/Linker/Linker.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Linker/IRMover.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Module;
@@ -30,7 +31,7 @@ class Linker {
     LinkOnlyNeeded = (1 << 1),
   };
 
-  Linker(Module &M);
+  LLVM_ABI Linker(Module &M);
 
   /// Link \p Src into the composite.
   ///
@@ -39,14 +40,15 @@ class Linker {
   /// callback.
   ///
   /// Returns true on error.
-  bool linkInModule(std::unique_ptr<Module> Src, unsigned Flags = Flags::None,
-                    std::function<void(Module &, const StringSet<> &)>
-                        InternalizeCallback = {});
-
-  static bool linkModules(Module &Dest, std::unique_ptr<Module> Src,
-                          unsigned Flags = Flags::None,
-                          std::function<void(Module &, const StringSet<> &)>
-                              InternalizeCallback = {});
+  LLVM_ABI bool linkInModule(std::unique_ptr<Module> Src,
+                             unsigned Flags = Flags::None,
+                             std::function<void(Module &, const StringSet<> &)>
+                                 InternalizeCallback = {});
+
+  LLVM_ABI static bool linkModules(
+      Module &Dest, std::unique_ptr<Module> Src, unsigned Flags = Flags::None,
+      std::function<void(Module &, const StringSet<> &)> InternalizeCallback =
+          {});
 };
 
 } // End llvm namespace
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCAsmBackend.h b/external/llvm-project/llvm/include/llvm/MC/MCAsmBackend.h
index 3b095772bd14..e49e786a10f5 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCAsmBackend.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCFixup.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include <cstdint>
 
@@ -39,7 +40,7 @@ class StringRef;
 class raw_ostream;
 
 /// Generic interface to target specific assembler backends.
-class MCAsmBackend {
+class LLVM_ABI MCAsmBackend {
 protected: // Can only create subclasses.
   MCAsmBackend(llvm::endianness Endian) : Endian(Endian) {}
 
@@ -198,8 +199,9 @@ class MCAsmBackend {
   virtual bool writeNopData(raw_ostream &OS, uint64_t Count,
                             const MCSubtargetInfo *STI) const = 0;
 
-  /// Give backend an opportunity to finish layout after relaxation
-  virtual void finishLayout(MCAssembler const &Asm) const {}
+  // Return true if fragment offsets have been adjusted and an extra layout
+  // iteration is needed.
+  virtual bool finishLayout(const MCAssembler &Asm) const { return false; }
 
   /// Generate the compact unwind encoding for the CFI instructions.
   virtual uint64_t generateCompactUnwindEncoding(const MCDwarfFrameInfo *FI,
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCAsmInfo.h b/external/llvm-project/llvm/include/llvm/MC/MCAsmInfo.h
index fdd62c676973..337ac44f6eba 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCAsmInfo.h
@@ -20,17 +20,22 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCTargetOptions.h"
+#include "llvm/Support/Compiler.h"
 #include <vector>
 
 namespace llvm {
 
+class MCAssembler;
 class MCContext;
 class MCCFIInstruction;
 class MCExpr;
+class MCSpecifierExpr;
 class MCSection;
 class MCStreamer;
 class MCSubtargetInfo;
 class MCSymbol;
+class MCValue;
+class raw_ostream;
 
 namespace WinEH {
 
@@ -55,7 +60,7 @@ enum LCOMMType { NoAlignment, ByteAlignment, Log2Alignment };
 
 /// This class is intended to be used as a base class for asm
 /// properties and features specific to the target.
-class MCAsmInfo {
+class LLVM_ABI MCAsmInfo {
 public:
   /// Assembly character literal syntax types.
   enum AsmCharLiteralSyntax {
@@ -140,6 +145,9 @@ class MCAsmInfo {
   /// This is appended to emitted labels.  Defaults to ":"
   const char *LabelSuffix;
 
+  /// Use .set instead of = to equate a symbol to an expression.
+  bool UsesSetToEquateSymbol = false;
+
   // Print the EH begin symbol with an assignment. Defaults to false.
   bool UseAssignmentForEHBegin = false;
 
@@ -432,6 +440,10 @@ class MCAsmInfo {
   explicit MCAsmInfo();
   virtual ~MCAsmInfo();
 
+  // Explicitly non-copyable.
+  MCAsmInfo(MCAsmInfo const &) = delete;
+  MCAsmInfo &operator=(MCAsmInfo const &) = delete;
+
   /// Get the code pointer size in bytes.
   unsigned getCodePointerSize() const { return CodePointerSize; }
 
@@ -525,6 +537,7 @@ class MCAsmInfo {
   bool shouldAllowAdditionalComments() const { return AllowAdditionalComments; }
   const char *getLabelSuffix() const { return LabelSuffix; }
 
+  bool usesSetToEquateSymbol() const { return UsesSetToEquateSymbol; }
   bool useAssignmentForEHBegin() const { return UseAssignmentForEHBegin; }
   bool needsLocalForSize() const { return NeedsLocalForSize; }
   StringRef getPrivateGlobalPrefix() const { return PrivateGlobalPrefix; }
@@ -712,6 +725,11 @@ class MCAsmInfo {
 
   StringRef getSpecifierName(uint32_t S) const;
   std::optional<uint32_t> getSpecifierForName(StringRef Name) const;
+
+  void printExpr(raw_ostream &, const MCExpr &) const;
+  virtual void printSpecifierExpr(raw_ostream &, const MCSpecifierExpr &) const;
+  virtual bool evaluateAsRelocatableImpl(const MCSpecifierExpr &, MCValue &Res,
+                                         const MCAssembler *Asm) const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCAsmMacro.h b/external/llvm-project/llvm/include/llvm/MC/MCAsmMacro.h
index c0c895fa94e6..ce50fb641884 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCAsmMacro.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCAsmMacro.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/SMLoc.h"
 #include <vector>
@@ -74,9 +75,9 @@ class AsmToken {
   bool is(TokenKind K) const { return Kind == K; }
   bool isNot(TokenKind K) const { return Kind != K; }
 
-  SMLoc getLoc() const;
-  SMLoc getEndLoc() const;
-  SMRange getLocRange() const;
+  LLVM_ABI SMLoc getLoc() const;
+  LLVM_ABI SMLoc getEndLoc() const;
+  LLVM_ABI SMRange getLocRange() const;
 
   /// Get the contents of a string token (without quotes).
   StringRef getStringContents() const {
@@ -115,7 +116,7 @@ class AsmToken {
     return IntVal;
   }
 
-  void dump(raw_ostream &OS) const;
+  LLVM_ABI void dump(raw_ostream &OS) const;
 };
 
 struct MCAsmMacroParameter {
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCAssembler.h b/external/llvm-project/llvm/include/llvm/MC/MCAssembler.h
index 3e141734ceac..5e009090311c 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCAssembler.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCAssembler.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include <algorithm>
 #include <cassert>
@@ -111,12 +112,12 @@ class MCAssembler {
   /// Check whether the given fragment needs relaxation.
   bool fragmentNeedsRelaxation(const MCRelaxableFragment *IF) const;
 
+  void layoutSection(MCSection &Sec);
   /// Perform one layout iteration and return true if any offsets
   /// were adjusted.
-  bool layoutOnce();
+  bool relaxOnce();
 
-  /// Perform relaxation on a single fragment - returns true if the fragment
-  /// changes as a result of relaxation.
+  /// Perform relaxation on a single fragment.
   bool relaxFragment(MCFragment &F);
   bool relaxInstruction(MCRelaxableFragment &IF);
   bool relaxLEB(MCLEBFragment &IF);
@@ -125,6 +126,7 @@ class MCAssembler {
   bool relaxDwarfCallFrameFragment(MCDwarfCallFrameFragment &DF);
   bool relaxCVInlineLineTable(MCCVInlineLineTableFragment &DF);
   bool relaxCVDefRange(MCCVDefRangeFragment &DF);
+  bool relaxFill(MCFillFragment &F);
   bool relaxPseudoProbeAddr(MCPseudoProbeAddrFragment &DF);
 
 public:
@@ -134,47 +136,48 @@ class MCAssembler {
   // concrete and require clients to pass in a target like object. The other
   // option is to make this abstract, and have targets provide concrete
   // implementations as we do with AsmParser.
-  MCAssembler(MCContext &Context, std::unique_ptr<MCAsmBackend> Backend,
-              std::unique_ptr<MCCodeEmitter> Emitter,
-              std::unique_ptr<MCObjectWriter> Writer);
+  LLVM_ABI MCAssembler(MCContext &Context,
+                       std::unique_ptr<MCAsmBackend> Backend,
+                       std::unique_ptr<MCCodeEmitter> Emitter,
+                       std::unique_ptr<MCObjectWriter> Writer);
   MCAssembler(const MCAssembler &) = delete;
   MCAssembler &operator=(const MCAssembler &) = delete;
 
   /// Compute the effective fragment size.
-  uint64_t computeFragmentSize(const MCFragment &F) const;
+  LLVM_ABI uint64_t computeFragmentSize(const MCFragment &F) const;
 
-  void layoutBundle(MCFragment *Prev, MCFragment *F) const;
-  void ensureValid(MCSection &Sec) const;
+  LLVM_ABI void layoutBundle(MCFragment *Prev, MCFragment *F) const;
 
   // Get the offset of the given fragment inside its containing section.
-  uint64_t getFragmentOffset(const MCFragment &F) const;
+  uint64_t getFragmentOffset(const MCFragment &F) const { return F.Offset; }
 
-  uint64_t getSectionAddressSize(const MCSection &Sec) const;
-  uint64_t getSectionFileSize(const MCSection &Sec) const;
+  LLVM_ABI uint64_t getSectionAddressSize(const MCSection &Sec) const;
+  LLVM_ABI uint64_t getSectionFileSize(const MCSection &Sec) const;
 
   // Get the offset of the given symbol, as computed in the current
   // layout.
   // \return True on success.
-  bool getSymbolOffset(const MCSymbol &S, uint64_t &Val) const;
+  LLVM_ABI bool getSymbolOffset(const MCSymbol &S, uint64_t &Val) const;
 
   // Variant that reports a fatal error if the offset is not computable.
-  uint64_t getSymbolOffset(const MCSymbol &S) const;
+  LLVM_ABI uint64_t getSymbolOffset(const MCSymbol &S) const;
 
   // If this symbol is equivalent to A + Constant, return A.
-  const MCSymbol *getBaseSymbol(const MCSymbol &Symbol) const;
+  LLVM_ABI const MCSymbol *getBaseSymbol(const MCSymbol &Symbol) const;
 
   /// Emit the section contents to \p OS.
-  void writeSectionData(raw_ostream &OS, const MCSection *Section) const;
+  LLVM_ABI void writeSectionData(raw_ostream &OS,
+                                 const MCSection *Section) const;
 
   /// Check whether a given symbol has been flagged with .thumb_func.
-  bool isThumbFunc(const MCSymbol *Func) const;
+  LLVM_ABI bool isThumbFunc(const MCSymbol *Func) const;
 
   /// Flag a function symbol as the target of a .thumb_func directive.
   void setIsThumbFunc(const MCSymbol *Func) { ThumbFuncs.insert(Func); }
 
   /// Reuse an assembler instance
   ///
-  void reset();
+  LLVM_ABI void reset();
 
   MCContext &getContext() const { return Context; }
 
@@ -193,10 +196,10 @@ class MCAssembler {
   /// Finish - Do final processing and write the object to the output stream.
   /// \p Writer is used for custom object writer (as the MCJIT does),
   /// if not specified it is automatically created from backend.
-  void Finish();
+  LLVM_ABI void Finish();
 
   // Layout all section and prepare them for emission.
-  void layout();
+  LLVM_ABI void layout();
 
   bool hasLayout() const { return HasLayout; }
   bool hasFinalLayout() const { return HasFinalLayout; }
@@ -223,21 +226,22 @@ class MCAssembler {
     return make_pointee_range(Symbols);
   }
 
-  bool registerSection(MCSection &Section);
-  bool registerSymbol(const MCSymbol &Symbol);
+  LLVM_ABI bool registerSection(MCSection &Section);
+  LLVM_ABI bool registerSymbol(const MCSymbol &Symbol);
 
   /// Write the necessary bundle padding to \p OS.
   /// Expects a fragment \p F containing instructions and its size \p FSize.
-  void writeFragmentPadding(raw_ostream &OS, const MCEncodedFragment &F,
-                            uint64_t FSize) const;
+  LLVM_ABI void writeFragmentPadding(raw_ostream &OS,
+                                     const MCEncodedFragment &F,
+                                     uint64_t FSize) const;
 
-  void reportError(SMLoc L, const Twine &Msg) const;
+  LLVM_ABI void reportError(SMLoc L, const Twine &Msg) const;
   // Record pending errors during layout iteration, as they may go away once the
   // layout is finalized.
-  void recordError(SMLoc L, const Twine &Msg) const;
-  void flushPendingErrors() const;
+  LLVM_ABI void recordError(SMLoc L, const Twine &Msg) const;
+  LLVM_ABI void flushPendingErrors() const;
 
-  void dump() const;
+  LLVM_ABI void dump() const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCCodeEmitter.h b/external/llvm-project/llvm/include/llvm/MC/MCCodeEmitter.h
index ceb16c9b4da7..1c454c3795c2 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCCodeEmitter.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCCodeEmitter.h
@@ -9,6 +9,8 @@
 #ifndef LLVM_MC_MCCODEEMITTER_H
 #define LLVM_MC_MCCODEEMITTER_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 class MCFixup;
@@ -18,7 +20,7 @@ class raw_ostream;
 template<typename T> class SmallVectorImpl;
 
 /// MCCodeEmitter - Generic instruction encoding interface.
-class MCCodeEmitter {
+class LLVM_ABI MCCodeEmitter {
 protected: // Can only create subclasses.
   MCCodeEmitter();
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCContext.h b/external/llvm-project/llvm/include/llvm/MC/MCContext.h
index 7327e25d99d0..3f8e2d9ae4d6 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCContext.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCContext.h
@@ -378,15 +378,16 @@ class MCContext {
   DenseSet<StringRef> ELFSeenGenericMergeableSections;
 
 public:
-  explicit MCContext(const Triple &TheTriple, const MCAsmInfo *MAI,
-                     const MCRegisterInfo *MRI, const MCSubtargetInfo *MSTI,
-                     const SourceMgr *Mgr = nullptr,
-                     MCTargetOptions const *TargetOpts = nullptr,
-                     bool DoAutoReset = true,
-                     StringRef Swift5ReflSegmentName = {});
+  LLVM_ABI explicit MCContext(const Triple &TheTriple, const MCAsmInfo *MAI,
+                              const MCRegisterInfo *MRI,
+                              const MCSubtargetInfo *MSTI,
+                              const SourceMgr *Mgr = nullptr,
+                              MCTargetOptions const *TargetOpts = nullptr,
+                              bool DoAutoReset = true,
+                              StringRef Swift5ReflSegmentName = {});
   MCContext(const MCContext &) = delete;
   MCContext &operator=(const MCContext &) = delete;
-  ~MCContext();
+  LLVM_ABI ~MCContext();
 
   Environment getObjectFileType() const { return Env; }
 
@@ -396,7 +397,7 @@ class MCContext {
   const Triple &getTargetTriple() const { return TT; }
   const SourceMgr *getSourceManager() const { return SrcMgr; }
 
-  void initInlineSourceManager();
+  LLVM_ABI void initInlineSourceManager();
   SourceMgr *getInlineSourceManager() { return InlineSrcMgr.get(); }
   std::vector<const MDNode *> &getLocInfos() { return LocInfos; }
   void setDiagnosticHandler(DiagHandlerTy DiagHandler) {
@@ -415,7 +416,7 @@ class MCContext {
 
   const MCTargetOptions *getTargetOptions() const { return TargetOptions; }
 
-  CodeViewContext &getCVContext();
+  LLVM_ABI CodeViewContext &getCVContext();
 
   void setUseNamesOnTempLabels(bool Value) { UseNamesOnTempLabels = Value; }
 
@@ -424,14 +425,14 @@ class MCContext {
 
   /// reset - return object to right after construction state to prepare
   /// to process a new module
-  void reset();
+  LLVM_ABI void reset();
 
   /// @}
 
   /// \name McInst Management
 
   /// Create and return a new MC instruction.
-  MCInst *createMCInst();
+  LLVM_ABI MCInst *createMCInst();
 
   template <typename F, typename... Args> F *allocFragment(Args &&...args) {
     return new (FragmentAllocator.Allocate(sizeof(F), alignof(F)))
@@ -444,64 +445,69 @@ class MCContext {
   /// Create a new linker temporary symbol with the specified prefix (Name) or
   /// "tmp". This creates a "l"-prefixed symbol for Mach-O and is identical to
   /// createNamedTempSymbol for other object file formats.
-  MCSymbol *createLinkerPrivateTempSymbol();
-  MCSymbol *createLinkerPrivateSymbol(const Twine &Name);
+  LLVM_ABI MCSymbol *createLinkerPrivateTempSymbol();
+  LLVM_ABI MCSymbol *createLinkerPrivateSymbol(const Twine &Name);
 
   /// Create a temporary symbol with a unique name. The name will be omitted
   /// in the symbol table if UseNamesOnTempLabels is false (default except
   /// MCAsmStreamer). The overload without Name uses an unspecified name.
-  MCSymbol *createTempSymbol();
-  MCSymbol *createTempSymbol(const Twine &Name, bool AlwaysAddSuffix = true);
+  LLVM_ABI MCSymbol *createTempSymbol();
+  LLVM_ABI MCSymbol *createTempSymbol(const Twine &Name,
+                                      bool AlwaysAddSuffix = true);
 
   /// Create a temporary symbol with a unique name whose name cannot be
   /// omitted in the symbol table. This is rarely used.
-  MCSymbol *createNamedTempSymbol();
-  MCSymbol *createNamedTempSymbol(const Twine &Name);
+  LLVM_ABI MCSymbol *createNamedTempSymbol();
+  LLVM_ABI MCSymbol *createNamedTempSymbol(const Twine &Name);
 
   /// Get or create a symbol for a basic block. For non-always-emit symbols,
   /// this behaves like createTempSymbol, except that it uses the
   /// PrivateLabelPrefix instead of the PrivateGlobalPrefix. When AlwaysEmit is
   /// true, behaves like getOrCreateSymbol, prefixed with PrivateLabelPrefix.
-  MCSymbol *createBlockSymbol(const Twine &Name, bool AlwaysEmit = false);
+  LLVM_ABI MCSymbol *createBlockSymbol(const Twine &Name,
+                                       bool AlwaysEmit = false);
 
   /// Create a local, non-temporary symbol like an ELF mapping symbol. Calling
   /// the function with the same name will generate new, unique instances.
-  MCSymbol *createLocalSymbol(StringRef Name);
+  LLVM_ABI MCSymbol *createLocalSymbol(StringRef Name);
 
   /// Create the definition of a directional local symbol for numbered label
   /// (used for "1:" definitions).
-  MCSymbol *createDirectionalLocalSymbol(unsigned LocalLabelVal);
+  LLVM_ABI MCSymbol *createDirectionalLocalSymbol(unsigned LocalLabelVal);
 
   /// Create and return a directional local symbol for numbered label (used
   /// for "1b" or 1f" references).
-  MCSymbol *getDirectionalLocalSymbol(unsigned LocalLabelVal, bool Before);
+  LLVM_ABI MCSymbol *getDirectionalLocalSymbol(unsigned LocalLabelVal,
+                                               bool Before);
 
   /// Lookup the symbol inside with the specified \p Name.  If it exists,
   /// return it.  If not, create a forward reference and return it.
   ///
   /// \param Name - The symbol name, which must be unique across all symbols.
-  MCSymbol *getOrCreateSymbol(const Twine &Name);
+  LLVM_ABI MCSymbol *getOrCreateSymbol(const Twine &Name);
 
   /// Gets a symbol that will be defined to the final stack offset of a local
   /// variable after codegen.
   ///
   /// \param Idx - The index of a local variable passed to \@llvm.localescape.
-  MCSymbol *getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx);
+  LLVM_ABI MCSymbol *getOrCreateFrameAllocSymbol(const Twine &FuncName,
+                                                 unsigned Idx);
 
-  MCSymbol *getOrCreateParentFrameOffsetSymbol(const Twine &FuncName);
+  LLVM_ABI MCSymbol *getOrCreateParentFrameOffsetSymbol(const Twine &FuncName);
 
-  MCSymbol *getOrCreateLSDASymbol(const Twine &FuncName);
+  LLVM_ABI MCSymbol *getOrCreateLSDASymbol(const Twine &FuncName);
 
   /// Get the symbol for \p Name, or null.
-  MCSymbol *lookupSymbol(const Twine &Name) const;
+  LLVM_ABI MCSymbol *lookupSymbol(const Twine &Name) const;
 
   /// Clone a symbol for the .set directive, replacing it in the symbol table.
   /// Existing references to the original symbol remain unchanged, and the
   /// original symbol is not emitted to the symbol table.
-  MCSymbol *cloneSymbol(MCSymbol &Sym);
+  LLVM_ABI MCSymbol *cloneSymbol(MCSymbol &Sym);
 
   /// Set value for a symbol.
-  void setSymbolValue(MCStreamer &Streamer, const Twine &Sym, uint64_t Val);
+  LLVM_ABI void setSymbolValue(MCStreamer &Streamer, const Twine &Sym,
+                               uint64_t Val);
 
   /// getSymbols - Get a reference for the symbol table for clients that
   /// want to, for example, iterate over all symbols. 'const' because we
@@ -517,11 +523,11 @@ class MCContext {
 
   /// registerInlineAsmLabel - Records that the name is a label referenced in
   /// inline assembly.
-  void registerInlineAsmLabel(MCSymbol *Sym);
+  LLVM_ABI void registerInlineAsmLabel(MCSymbol *Sym);
 
   /// Allocates and returns a new `WasmSignature` instance (with empty parameter
   /// and return type lists).
-  wasm::WasmSignature *createWasmSignature();
+  LLVM_ABI wasm::WasmSignature *createWasmSignature();
 
   /// @}
 
@@ -530,10 +536,10 @@ class MCContext {
 
   /// Return the MCSection for the specified mach-o section.  This requires
   /// the operands to be valid.
-  MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section,
-                                  unsigned TypeAndAttributes,
-                                  unsigned Reserved2, SectionKind K,
-                                  const char *BeginSymName = nullptr);
+  LLVM_ABI MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section,
+                                           unsigned TypeAndAttributes,
+                                           unsigned Reserved2, SectionKind K,
+                                           const char *BeginSymName = nullptr);
 
   MCSectionMachO *getMachOSection(StringRef Segment, StringRef Section,
                                   unsigned TypeAndAttributes, SectionKind K,
@@ -560,85 +566,93 @@ class MCContext {
                          MCSection::NonUniqueID, nullptr);
   }
 
-  MCSectionELF *getELFSection(const Twine &Section, unsigned Type,
-                              unsigned Flags, unsigned EntrySize,
-                              const Twine &Group, bool IsComdat,
-                              unsigned UniqueID,
-                              const MCSymbolELF *LinkedToSym);
+  LLVM_ABI MCSectionELF *getELFSection(const Twine &Section, unsigned Type,
+                                       unsigned Flags, unsigned EntrySize,
+                                       const Twine &Group, bool IsComdat,
+                                       unsigned UniqueID,
+                                       const MCSymbolELF *LinkedToSym);
 
-  MCSectionELF *getELFSection(const Twine &Section, unsigned Type,
-                              unsigned Flags, unsigned EntrySize,
-                              const MCSymbolELF *Group, bool IsComdat,
-                              unsigned UniqueID,
-                              const MCSymbolELF *LinkedToSym);
+  LLVM_ABI MCSectionELF *getELFSection(const Twine &Section, unsigned Type,
+                                       unsigned Flags, unsigned EntrySize,
+                                       const MCSymbolELF *Group, bool IsComdat,
+                                       unsigned UniqueID,
+                                       const MCSymbolELF *LinkedToSym);
 
   /// Get a section with the provided group identifier. This section is
   /// named by concatenating \p Prefix with '.' then \p Suffix. The \p Type
   /// describes the type of the section and \p Flags are used to further
   /// configure this named section.
-  MCSectionELF *getELFNamedSection(const Twine &Prefix, const Twine &Suffix,
-                                   unsigned Type, unsigned Flags,
-                                   unsigned EntrySize = 0);
+  LLVM_ABI MCSectionELF *getELFNamedSection(const Twine &Prefix,
+                                            const Twine &Suffix, unsigned Type,
+                                            unsigned Flags,
+                                            unsigned EntrySize = 0);
 
-  MCSectionELF *createELFRelSection(const Twine &Name, unsigned Type,
-                                    unsigned Flags, unsigned EntrySize,
-                                    const MCSymbolELF *Group,
-                                    const MCSectionELF *RelInfoSection);
+  LLVM_ABI MCSectionELF *
+  createELFRelSection(const Twine &Name, unsigned Type, unsigned Flags,
+                      unsigned EntrySize, const MCSymbolELF *Group,
+                      const MCSectionELF *RelInfoSection);
 
-  MCSectionELF *createELFGroupSection(const MCSymbolELF *Group, bool IsComdat);
+  LLVM_ABI MCSectionELF *createELFGroupSection(const MCSymbolELF *Group,
+                                               bool IsComdat);
 
-  void recordELFMergeableSectionInfo(StringRef SectionName, unsigned Flags,
-                                     unsigned UniqueID, unsigned EntrySize);
+  LLVM_ABI void recordELFMergeableSectionInfo(StringRef SectionName,
+                                              unsigned Flags, unsigned UniqueID,
+                                              unsigned EntrySize);
 
-  bool isELFImplicitMergeableSectionNamePrefix(StringRef Name);
+  LLVM_ABI bool isELFImplicitMergeableSectionNamePrefix(StringRef Name);
 
-  bool isELFGenericMergeableSection(StringRef Name);
+  LLVM_ABI bool isELFGenericMergeableSection(StringRef Name);
 
   /// Return the unique ID of the section with the given name, flags and entry
   /// size, if it exists.
-  std::optional<unsigned> getELFUniqueIDForEntsize(StringRef SectionName,
-                                                   unsigned Flags,
-                                                   unsigned EntrySize);
+  LLVM_ABI std::optional<unsigned>
+  getELFUniqueIDForEntsize(StringRef SectionName, unsigned Flags,
+                           unsigned EntrySize);
 
-  MCSectionGOFF *getGOFFSection(StringRef Section, SectionKind Kind,
-                                MCSection *Parent, uint32_t Subsection = 0);
+  LLVM_ABI MCSectionGOFF *getGOFFSection(StringRef Section, SectionKind Kind,
+                                         MCSection *Parent,
+                                         uint32_t Subsection = 0);
 
-  MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics,
-                                StringRef COMDATSymName, int Selection,
-                                unsigned UniqueID = MCSection::NonUniqueID);
+  LLVM_ABI MCSectionCOFF *
+  getCOFFSection(StringRef Section, unsigned Characteristics,
+                 StringRef COMDATSymName, int Selection,
+                 unsigned UniqueID = MCSection::NonUniqueID);
 
-  MCSectionCOFF *getCOFFSection(StringRef Section, unsigned Characteristics);
+  LLVM_ABI MCSectionCOFF *getCOFFSection(StringRef Section,
+                                         unsigned Characteristics);
 
   /// Gets or creates a section equivalent to Sec that is associated with the
   /// section containing KeySym. For example, to create a debug info section
   /// associated with an inline function, pass the normal debug info section
   /// as Sec and the function symbol as KeySym.
-  MCSectionCOFF *
+  LLVM_ABI MCSectionCOFF *
   getAssociativeCOFFSection(MCSectionCOFF *Sec, const MCSymbol *KeySym,
                             unsigned UniqueID = MCSection::NonUniqueID);
 
-  MCSectionSPIRV *getSPIRVSection();
+  LLVM_ABI MCSectionSPIRV *getSPIRVSection();
 
   MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K,
                                 unsigned Flags = 0) {
     return getWasmSection(Section, K, Flags, "", ~0);
   }
 
-  MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K,
-                                unsigned Flags, const Twine &Group,
-                                unsigned UniqueID);
+  LLVM_ABI MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K,
+                                         unsigned Flags, const Twine &Group,
+                                         unsigned UniqueID);
 
-  MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K,
-                                unsigned Flags, const MCSymbolWasm *Group,
-                                unsigned UniqueID);
+  LLVM_ABI MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K,
+                                         unsigned Flags,
+                                         const MCSymbolWasm *Group,
+                                         unsigned UniqueID);
 
   /// Get the section for the provided Section name
-  MCSectionDXContainer *getDXContainerSection(StringRef Section, SectionKind K);
+  LLVM_ABI MCSectionDXContainer *getDXContainerSection(StringRef Section,
+                                                       SectionKind K);
 
-  bool hasXCOFFSection(StringRef Section,
-                       XCOFF::CsectProperties CsectProp) const;
+  LLVM_ABI bool hasXCOFFSection(StringRef Section,
+                                XCOFF::CsectProperties CsectProp) const;
 
-  MCSectionXCOFF *getXCOFFSection(
+  LLVM_ABI MCSectionXCOFF *getXCOFFSection(
       StringRef Section, SectionKind K,
       std::optional<XCOFF::CsectProperties> CsectProp = std::nullopt,
       bool MultiSymbolsAllowed = false,
@@ -646,7 +660,7 @@ class MCContext {
           std::nullopt);
 
   // Create and save a copy of STI and return a reference to the copy.
-  MCSubtargetInfo &getSubtargetCopy(const MCSubtargetInfo &STI);
+  LLVM_ABI MCSubtargetInfo &getSubtargetCopy(const MCSubtargetInfo &STI);
 
   uint8_t getBBAddrMapVersion() const { return BBAddrMapVersion; }
 
@@ -664,13 +678,14 @@ class MCContext {
   void setCompilationDir(StringRef S) { CompilationDir = S.str(); }
 
   /// Add an entry to the debug prefix map.
-  void addDebugPrefixMapEntry(const std::string &From, const std::string &To);
+  LLVM_ABI void addDebugPrefixMapEntry(const std::string &From,
+                                       const std::string &To);
 
   /// Remap one path in-place as per the debug prefix map.
-  void remapDebugPath(SmallVectorImpl<char> &Path);
+  LLVM_ABI void remapDebugPath(SmallVectorImpl<char> &Path);
 
   // Remaps all debug directory paths in-place as per the debug prefix map.
-  void RemapDebugPaths();
+  LLVM_ABI void RemapDebugPaths();
 
   /// Get the main file name for use in error messages and debug
   /// info. This can be set to ensure we've got the correct file name
@@ -681,13 +696,12 @@ class MCContext {
   void setMainFileName(StringRef S) { MainFileName = std::string(S); }
 
   /// Creates an entry in the dwarf file and directory tables.
-  Expected<unsigned> getDwarfFile(StringRef Directory, StringRef FileName,
-                                  unsigned FileNumber,
-                                  std::optional<MD5::MD5Result> Checksum,
-                                  std::optional<StringRef> Source,
-                                  unsigned CUID);
+  LLVM_ABI Expected<unsigned>
+  getDwarfFile(StringRef Directory, StringRef FileName, unsigned FileNumber,
+               std::optional<MD5::MD5Result> Checksum,
+               std::optional<StringRef> Source, unsigned CUID);
 
-  bool isValidDwarfFileNumber(unsigned FileNumber, unsigned CUID = 0);
+  LLVM_ABI bool isValidDwarfFileNumber(unsigned FileNumber, unsigned CUID = 0);
 
   const std::map<unsigned, MCDwarfLineTable> &getMCDwarfLineTables() const {
     return MCDwarfLineTablesCUMap;
@@ -754,8 +768,8 @@ class MCContext {
   bool getGenDwarfForAssembly() { return GenDwarfForAssembly; }
   void setGenDwarfForAssembly(bool Value) { GenDwarfForAssembly = Value; }
   unsigned getGenDwarfFileNumber() { return GenDwarfFileNumber; }
-  EmitDwarfUnwindType emitDwarfUnwindInfo() const;
-  bool emitCompactUnwindNonCanonical() const;
+  LLVM_ABI EmitDwarfUnwindType emitDwarfUnwindInfo() const;
+  LLVM_ABI bool emitCompactUnwindNonCanonical() const;
 
   void setGenDwarfFileNumber(unsigned FileNumber) {
     GenDwarfFileNumber = FileNumber;
@@ -763,7 +777,7 @@ class MCContext {
 
   /// Specifies information about the "root file" for assembler clients
   /// (e.g., llvm-mc). Assumes compilation dir etc. have been set up.
-  void setGenDwarfRootFile(StringRef FileName, StringRef Buffer);
+  LLVM_ABI void setGenDwarfRootFile(StringRef FileName, StringRef Buffer);
 
   const SetVector<MCSection *> &getGenDwarfSectionSyms() {
     return SectionsForRanges;
@@ -773,7 +787,7 @@ class MCContext {
     return SectionsForRanges.insert(Sec);
   }
 
-  void finalizeDwarfSections(MCStreamer &MCOS);
+  LLVM_ABI void finalizeDwarfSections(MCStreamer &MCOS);
 
   const std::vector<MCGenDwarfLabelEntry> &getMCGenDwarfLabelEntries() const {
     return MCGenDwarfLabelEntries;
@@ -820,9 +834,9 @@ class MCContext {
   }
 
   bool hadError() { return HadError; }
-  void diagnose(const SMDiagnostic &SMD);
-  void reportError(SMLoc L, const Twine &Msg);
-  void reportWarning(SMLoc L, const Twine &Msg);
+  LLVM_ABI void diagnose(const SMDiagnostic &SMD);
+  LLVM_ABI void reportError(SMLoc L, const Twine &Msg);
+  LLVM_ABI void reportWarning(SMLoc L, const Twine &Msg);
 
   MCAsmMacro *lookupMacro(StringRef Name) {
     StringMap<MCAsmMacro>::iterator I = MacroMap.find(Name);
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCDecoderOps.h b/external/llvm-project/llvm/include/llvm/MC/MCDecoderOps.h
index 3c0b68101e34..50fcf2a40e26 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCDecoderOps.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCDecoderOps.h
@@ -10,24 +10,29 @@
 #ifndef LLVM_MC_MCDECODEROPS_H
 #define LLVM_MC_MCDECODEROPS_H
 
-namespace llvm {
+namespace llvm::MCD {
 
-namespace MCD {
 // Disassembler state machine opcodes.
+// nts_t is either uint16_t or uint24_t based on whether large decoder table is
+// enabled.
 enum DecoderOps {
-  OPC_ExtractField = 1, // OPC_ExtractField(uleb128 Start, uint8_t Len)
-  OPC_FilterValue,      // OPC_FilterValue(uleb128 Val, uint16_t NumToSkip)
-  OPC_CheckField,       // OPC_CheckField(uleb128 Start, uint8_t Len,
-                        //                uleb128 Val, uint16_t NumToSkip)
-  OPC_CheckPredicate,   // OPC_CheckPredicate(uleb128 PIdx, uint16_t NumToSkip)
-  OPC_Decode,           // OPC_Decode(uleb128 Opcode, uleb128 DIdx)
-  OPC_TryDecode,        // OPC_TryDecode(uleb128 Opcode, uleb128 DIdx,
-                        //               uint16_t NumToSkip)
-  OPC_SoftFail,         // OPC_SoftFail(uleb128 PMask, uleb128 NMask)
-  OPC_Fail              // OPC_Fail()
+  OPC_ExtractField = 1,     // OPC_ExtractField(uleb128 Start, uint8_t Len)
+  OPC_FilterValue,          // OPC_FilterValue(uleb128 Val, nts_t NumToSkip)
+  OPC_FilterValueOrFail,    // OPC_FilterValueOrFail(uleb128 Val)
+  OPC_CheckField,           // OPC_CheckField(uleb128 Start, uint8_t Len,
+                            //                uleb128 Val, nts_t NumToSkip)
+  OPC_CheckFieldOrFail,     // OPC_CheckFieldOrFail(uleb128 Start, uint8_t Len,
+                            //                uleb128 Val)
+  OPC_CheckPredicate,       // OPC_CheckPredicate(uleb128 PIdx, nts_t NumToSkip)
+  OPC_CheckPredicateOrFail, // OPC_CheckPredicateOrFail(uleb128 PIdx)
+  OPC_Decode,               // OPC_Decode(uleb128 Opcode, uleb128 DIdx)
+  OPC_TryDecode,            // OPC_TryDecode(uleb128 Opcode, uleb128 DIdx,
+                            //               nts_t NumToSkip)
+  OPC_TryDecodeOrFail,      // OPC_TryDecodeOrFail(uleb128 Opcode, uleb128 DIdx)
+  OPC_SoftFail,             // OPC_SoftFail(uleb128 PMask, uleb128 NMask)
+  OPC_Fail                  // OPC_Fail()
 };
 
-} // namespace MCD
-} // namespace llvm
+} // namespace llvm::MCD
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h b/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h
index 901bfcf5fa54..3a7ca1a69ab8 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/MC/MCDisassembler/MCSymbolizer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <memory>
@@ -23,7 +24,7 @@ struct XCOFFSymbolInfoTy {
   std::optional<XCOFF::StorageMappingClass> StorageMappingClass;
   std::optional<uint32_t> Index;
   bool IsLabel = false;
-  bool operator<(const XCOFFSymbolInfoTy &SymInfo) const;
+  LLVM_ABI bool operator<(const XCOFFSymbolInfoTy &SymInfo) const;
 };
 
 struct SymbolInfoTy {
@@ -81,7 +82,7 @@ class raw_ostream;
 
 /// Superclass for all disassemblers. Consumes a memory region and provides an
 /// array of assembly instructions.
-class MCDisassembler {
+class LLVM_ABI MCDisassembler {
 public:
   /// Ternary decode status. Most backends will just use Fail and
   /// Success, however some have a concept of an instruction with
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCRelocationInfo.h b/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCRelocationInfo.h
index efc59da19335..2cf4af02bef6 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCRelocationInfo.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCRelocationInfo.h
@@ -15,13 +15,15 @@
 #ifndef LLVM_MC_MCDISASSEMBLER_MCRELOCATIONINFO_H
 #define LLVM_MC_MCDISASSEMBLER_MCRELOCATIONINFO_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 class MCContext;
 class MCExpr;
 
 /// Create MCExprs from relocations found in an object file.
-class MCRelocationInfo {
+class LLVM_ABI MCRelocationInfo {
 protected:
   MCContext &Ctx;
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h b/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h
index 1efb63f1a142..1e4c9f20481d 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCDisassembler/MCSymbolizer.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MC/MCDisassembler/MCRelocationInfo.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <memory>
 #include <utility>
@@ -36,7 +37,7 @@ class raw_ostream;
 /// inside each disassembler, but would use the instr info to determine what
 /// operands are actually symbolizable, and in what way. I don't think this
 /// information exists right now.
-class MCSymbolizer {
+class LLVM_ABI MCSymbolizer {
 protected:
   MCContext &Ctx;
   std::unique_ptr<MCRelocationInfo> RelInfo;
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCDwarf.h b/external/llvm-project/llvm/include/llvm/MC/MCDwarf.h
index 36b4326de2c7..c457a2be63dd 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCDwarf.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCDwarf.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/StringTableBuilder.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/SMLoc.h"
@@ -44,7 +45,7 @@ class SourceMgr;
 
 namespace mcdwarf {
 // Emit the common part of the DWARF 5 range/locations list tables header.
-MCSymbol *emitListsTableHeaderStart(MCStreamer &S);
+LLVM_ABI MCSymbol *emitListsTableHeaderStart(MCStreamer &S);
 } // namespace mcdwarf
 
 /// Manage the .debug_line_str section contents, if we use it.
@@ -58,22 +59,22 @@ class MCDwarfLineStr {
 public:
   /// Construct an instance that can emit .debug_line_str (for use in a normal
   /// v5 line table).
-  explicit MCDwarfLineStr(MCContext &Ctx);
+  LLVM_ABI explicit MCDwarfLineStr(MCContext &Ctx);
 
   StringSaver &getSaver() { return Saver; }
 
   /// Emit a reference to the string.
-  void emitRef(MCStreamer *MCOS, StringRef Path);
+  LLVM_ABI void emitRef(MCStreamer *MCOS, StringRef Path);
 
   /// Emit the .debug_line_str section if appropriate.
-  void emitSection(MCStreamer *MCOS);
+  LLVM_ABI void emitSection(MCStreamer *MCOS);
 
   /// Returns finalized section.
-  SmallString<0> getFinalizedData();
+  LLVM_ABI SmallString<0> getFinalizedData();
 
   /// Adds path \p Path to the line string. Returns offset in the
   /// .debug_line_str section.
-  size_t addString(StringRef Path);
+  LLVM_ABI size_t addString(StringRef Path);
 };
 
 /// Instances of this class represent the name of the dwarf .file directive and
@@ -216,6 +217,9 @@ class MCDwarfLineEntry : public MCDwarfLoc {
 
   // Override the label with the given EndLabel.
   void setEndLabel(MCSymbol *EndLabel) {
+    // If we're setting this to be an end entry, make sure we don't have
+    // LineStreamLabel set.
+    assert(LineStreamLabel == nullptr);
     Label = EndLabel;
     IsEndEntry = true;
   }
@@ -223,7 +227,7 @@ class MCDwarfLineEntry : public MCDwarfLoc {
   // This is called when an instruction is assembled into the specified
   // section and if there is information from the last .loc directive that
   // has yet to have a line entry made for it is made.
-  static void make(MCStreamer *MCOS, MCSection *Section);
+  LLVM_ABI static void make(MCStreamer *MCOS, MCSection *Section);
 };
 
 /// Instances of this class represent the line information for a compile
@@ -239,7 +243,7 @@ class MCLineSection {
 
   // Add an end entry by cloning the last entry, if exists, for the section
   // the given EndLabel belongs to. The label is replaced by the given EndLabel.
-  void addEndEntry(MCSymbol *EndLabel);
+  LLVM_ABI void addEndEntry(MCSymbol *EndLabel);
 
   using MCDwarfLineEntryCollection = std::vector<MCDwarfLineEntry>;
   using iterator = MCDwarfLineEntryCollection::iterator;
@@ -286,14 +290,16 @@ struct MCDwarfLineTableHeader {
 public:
   MCDwarfLineTableHeader() = default;
 
-  Expected<unsigned> tryGetFile(StringRef &Directory, StringRef &FileName,
-                                std::optional<MD5::MD5Result> Checksum,
-                                std::optional<StringRef> Source,
-                                uint16_t DwarfVersion, unsigned FileNumber = 0);
-  std::pair<MCSymbol *, MCSymbol *>
+  LLVM_ABI Expected<unsigned> tryGetFile(StringRef &Directory,
+                                         StringRef &FileName,
+                                         std::optional<MD5::MD5Result> Checksum,
+                                         std::optional<StringRef> Source,
+                                         uint16_t DwarfVersion,
+                                         unsigned FileNumber = 0);
+  LLVM_ABI std::pair<MCSymbol *, MCSymbol *>
   Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params,
        std::optional<MCDwarfLineStr> &LineStr) const;
-  std::pair<MCSymbol *, MCSymbol *>
+  LLVM_ABI std::pair<MCSymbol *, MCSymbol *>
   Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params,
        ArrayRef<char> SpecialOpcodeLengths,
        std::optional<MCDwarfLineStr> &LineStr) const;
@@ -356,8 +362,8 @@ class MCDwarfDwoLineTable {
                                       DwarfVersion));
   }
 
-  void Emit(MCStreamer &MCOS, MCDwarfLineTableParams Params,
-            MCSection *Section) const;
+  LLVM_ABI void Emit(MCStreamer &MCOS, MCDwarfLineTableParams Params,
+                     MCSection *Section) const;
 };
 
 class MCDwarfLineTable {
@@ -366,24 +372,27 @@ class MCDwarfLineTable {
 
 public:
   // This emits the Dwarf file and the line tables for all Compile Units.
-  static void emit(MCStreamer *MCOS, MCDwarfLineTableParams Params);
+  LLVM_ABI static void emit(MCStreamer *MCOS, MCDwarfLineTableParams Params);
 
   // This emits the Dwarf file and the line tables for a given Compile Unit.
-  void emitCU(MCStreamer *MCOS, MCDwarfLineTableParams Params,
-              std::optional<MCDwarfLineStr> &LineStr) const;
+  LLVM_ABI void emitCU(MCStreamer *MCOS, MCDwarfLineTableParams Params,
+                       std::optional<MCDwarfLineStr> &LineStr) const;
 
   // This emits a single line table associated with a given Section.
-  static void
+  LLVM_ABI static void
   emitOne(MCStreamer *MCOS, MCSection *Section,
           const MCLineSection::MCDwarfLineEntryCollection &LineEntries);
 
-  void endCurrentSeqAndEmitLineStreamLabel(MCStreamer *MCOS, SMLoc DefLoc,
-                                           StringRef Name);
+  LLVM_ABI void endCurrentSeqAndEmitLineStreamLabel(MCStreamer *MCOS,
+                                                    SMLoc DefLoc,
+                                                    StringRef Name);
 
-  Expected<unsigned> tryGetFile(StringRef &Directory, StringRef &FileName,
-                                std::optional<MD5::MD5Result> Checksum,
-                                std::optional<StringRef> Source,
-                                uint16_t DwarfVersion, unsigned FileNumber = 0);
+  LLVM_ABI Expected<unsigned> tryGetFile(StringRef &Directory,
+                                         StringRef &FileName,
+                                         std::optional<MD5::MD5Result> Checksum,
+                                         std::optional<StringRef> Source,
+                                         uint16_t DwarfVersion,
+                                         unsigned FileNumber = 0);
   unsigned getFile(StringRef &Directory, StringRef &FileName,
                    std::optional<MD5::MD5Result> Checksum,
                    std::optional<StringRef> Source, uint16_t DwarfVersion,
@@ -449,12 +458,13 @@ class MCDwarfLineTable {
 class MCDwarfLineAddr {
 public:
   /// Utility function to encode a Dwarf pair of LineDelta and AddrDeltas.
-  static void encode(MCContext &Context, MCDwarfLineTableParams Params,
-                     int64_t LineDelta, uint64_t AddrDelta, SmallVectorImpl<char> &OS);
+  LLVM_ABI static void encode(MCContext &Context, MCDwarfLineTableParams Params,
+                              int64_t LineDelta, uint64_t AddrDelta,
+                              SmallVectorImpl<char> &OS);
 
   /// Utility function to emit the encoding to a streamer.
-  static void Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params,
-                   int64_t LineDelta, uint64_t AddrDelta);
+  LLVM_ABI static void Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params,
+                            int64_t LineDelta, uint64_t AddrDelta);
 };
 
 class MCGenDwarfInfo {
@@ -463,7 +473,7 @@ class MCGenDwarfInfo {
   // When generating dwarf for assembly source files this emits the Dwarf
   // sections.
   //
-  static void Emit(MCStreamer *MCOS);
+  LLVM_ABI static void Emit(MCStreamer *MCOS);
 };
 
 // When generating dwarf for assembly source files this is the info that is
@@ -492,8 +502,8 @@ class MCGenDwarfLabelEntry {
 
   // This is called when label is created when we are generating dwarf for
   // assembly source files.
-  static void Make(MCSymbol *Symbol, MCStreamer *MCOS, SourceMgr &SrcMgr,
-                   SMLoc &Loc);
+  LLVM_ABI static void Make(MCSymbol *Symbol, MCStreamer *MCOS,
+                            SourceMgr &SrcMgr, SMLoc &Loc);
 };
 
 class MCCFIInstruction {
@@ -895,9 +905,10 @@ class MCDwarfFrameEmitter {
   //
   // This emits the frame info section.
   //
-  static void Emit(MCObjectStreamer &streamer, MCAsmBackend *MAB, bool isEH);
-  static void encodeAdvanceLoc(MCContext &Context, uint64_t AddrDelta,
-                               SmallVectorImpl<char> &OS);
+  LLVM_ABI static void Emit(MCObjectStreamer &streamer, MCAsmBackend *MAB,
+                            bool isEH);
+  LLVM_ABI static void encodeAdvanceLoc(MCContext &Context, uint64_t AddrDelta,
+                                        SmallVectorImpl<char> &OS);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCExpr.h b/external/llvm-project/llvm/include/llvm/MC/MCExpr.h
index d63fd1354255..cd57fafc50b5 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCExpr.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCExpr.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCEXPR_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include <cstdint>
 
@@ -37,6 +38,7 @@ class MCExpr {
     Constant,  ///< Constant expressions.
     SymbolRef, ///< References to labels and assigned expressions.
     Unary,     ///< Unary expressions.
+    Specifier, ///< Expression with a relocation specifier.
     Target     ///< Target specific expression.
   };
 
@@ -61,8 +63,8 @@ class MCExpr {
            "Subclass data too large");
   }
 
-  bool evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
-                                 bool InSet) const;
+  LLVM_ABI bool evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
+                                          bool InSet) const;
 
   unsigned getSubclassData() const { return SubclassData; }
 
@@ -80,13 +82,10 @@ class MCExpr {
   /// \name Utility Methods
   /// @{
 
-  void print(raw_ostream &OS, const MCAsmInfo *MAI,
-             int SurroundingPrec = 0) const;
-  void dump() const;
-
-  /// Returns whether the given symbol is used anywhere in the expression or
-  /// subexpressions.
-  bool isSymbolUsedInExpression(const MCSymbol *Sym) const;
+  // TODO: Make this private. Users should call MCAsmInfo::printExpr instead.
+  LLVM_ABI void print(raw_ostream &OS, const MCAsmInfo *MAI,
+                      int SurroundingPrec = 0) const;
+  LLVM_ABI void dump() const;
 
   /// @}
   /// \name Expression Evaluation
@@ -96,14 +95,15 @@ class MCExpr {
   ///
   /// \param Res - The absolute value, if evaluation succeeds.
   /// \return - True on success.
-  bool evaluateAsAbsolute(int64_t &Res) const;
-  bool evaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const;
-  bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm) const;
+  LLVM_ABI bool evaluateAsAbsolute(int64_t &Res) const;
+  LLVM_ABI bool evaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const;
+  LLVM_ABI bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm) const;
 
   /// Aggressive variant of evaluateAsRelocatable when relocations are
   /// unavailable (e.g. .fill). Expects callers to handle errors when true is
   /// returned.
-  bool evaluateKnownAbsolute(int64_t &Res, const MCAssembler &Asm) const;
+  LLVM_ABI bool evaluateKnownAbsolute(int64_t &Res,
+                                      const MCAssembler &Asm) const;
 
   /// Try to evaluate the expression to a relocatable value, i.e. an
   /// expression of the fixed form (a - b + constant).
@@ -111,25 +111,27 @@ class MCExpr {
   /// \param Res - The relocatable value, if evaluation succeeds.
   /// \param Asm - The assembler object to use for evaluating values.
   /// \return - True on success.
-  bool evaluateAsRelocatable(MCValue &Res, const MCAssembler *Asm) const;
+  LLVM_ABI bool evaluateAsRelocatable(MCValue &Res,
+                                      const MCAssembler *Asm) const;
 
   /// Try to evaluate the expression to the form (a - b + constant) where
   /// neither a nor b are variables.
   ///
   /// This is a more aggressive variant of evaluateAsRelocatable. The intended
   /// use is for when relocations are not available, like the .size directive.
-  bool evaluateAsValue(MCValue &Res, const MCAssembler &Asm) const;
+  LLVM_ABI bool evaluateAsValue(MCValue &Res, const MCAssembler &Asm) const;
 
   /// Find the "associated section" for this expression, which is
   /// currently defined as the absolute section for constants, or
   /// otherwise the section associated with the first defined symbol in the
   /// expression.
-  MCFragment *findAssociatedFragment() const;
+  LLVM_ABI MCFragment *findAssociatedFragment() const;
 
   /// @}
 
-  static bool evaluateSymbolicAdd(const MCAssembler *, bool, const MCValue &,
-                                  const MCValue &, MCValue &);
+  LLVM_ABI static bool evaluateSymbolicAdd(const MCAssembler *, bool,
+                                           const MCValue &, const MCValue &,
+                                           MCValue &);
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const MCExpr &E) {
@@ -159,9 +161,9 @@ class MCConstantExpr : public MCExpr {
   /// \name Construction
   /// @{
 
-  static const MCConstantExpr *create(int64_t Value, MCContext &Ctx,
-                                      bool PrintInHex = false,
-                                      unsigned SizeInBytes = 0);
+  LLVM_ABI static const MCConstantExpr *create(int64_t Value, MCContext &Ctx,
+                                               bool PrintInHex = false,
+                                               unsigned SizeInBytes = 0);
 
   /// @}
   /// \name Accessors
@@ -220,8 +222,10 @@ class MCSymbolRefExpr : public MCExpr {
     return MCSymbolRefExpr::create(Symbol, VK_None, Ctx, Loc);
   }
 
-  static const MCSymbolRefExpr *create(const MCSymbol *Symbol, VariantKind Kind,
-                                       MCContext &Ctx, SMLoc Loc = SMLoc());
+  LLVM_ABI static const MCSymbolRefExpr *create(const MCSymbol *Symbol,
+                                                VariantKind Kind,
+                                                MCContext &Ctx,
+                                                SMLoc Loc = SMLoc());
   static const MCSymbolRefExpr *create(const MCSymbol *Symbol, uint16_t Kind,
                                        MCContext &Ctx, SMLoc Loc = SMLoc()) {
     return MCSymbolRefExpr::create(Symbol, VariantKind(Kind), Ctx, Loc);
@@ -266,8 +270,8 @@ class MCUnaryExpr : public MCExpr {
   /// \name Construction
   /// @{
 
-  static const MCUnaryExpr *create(Opcode Op, const MCExpr *Expr,
-                                   MCContext &Ctx, SMLoc Loc = SMLoc());
+  LLVM_ABI static const MCUnaryExpr *
+  create(Opcode Op, const MCExpr *Expr, MCContext &Ctx, SMLoc Loc = SMLoc());
 
   static const MCUnaryExpr *createLNot(const MCExpr *Expr, MCContext &Ctx, SMLoc Loc = SMLoc()) {
     return create(LNot, Expr, Ctx, Loc);
@@ -343,9 +347,9 @@ class MCBinaryExpr : public MCExpr {
   /// \name Construction
   /// @{
 
-  static const MCBinaryExpr *create(Opcode Op, const MCExpr *LHS,
-                                    const MCExpr *RHS, MCContext &Ctx,
-                                    SMLoc Loc = SMLoc());
+  LLVM_ABI static const MCBinaryExpr *create(Opcode Op, const MCExpr *LHS,
+                                             const MCExpr *RHS, MCContext &Ctx,
+                                             SMLoc Loc = SMLoc());
 
   static const MCBinaryExpr *createAdd(const MCExpr *LHS, const MCExpr *RHS,
                                        MCContext &Ctx) {
@@ -469,7 +473,7 @@ class MCBinaryExpr : public MCExpr {
 ///
 /// NOTE: All subclasses are required to have trivial destructors because
 /// MCExprs are bump pointer allocated and not destructed.
-class MCTargetExpr : public MCExpr {
+class LLVM_ABI MCTargetExpr : public MCExpr {
   virtual void anchor();
 
 protected:
@@ -482,9 +486,6 @@ class MCTargetExpr : public MCExpr {
                                          const MCAssembler *Asm) const = 0;
   // allow Target Expressions to be checked for equality
   virtual bool isEqualTo(const MCExpr *x) const { return false; }
-  virtual bool isSymbolUsedInExpression(const MCSymbol *Sym) const {
-    return false;
-  }
   // This should be set when assigned expressions are not valid ".set"
   // expressions, e.g. registers, and must be inlined.
   virtual bool inlineAssignedExpr() const { return false; }
@@ -496,6 +497,43 @@ class MCTargetExpr : public MCExpr {
   }
 };
 
+/// Extension point for target-specific MCExpr subclasses with a relocation
+/// specifier, serving as a replacement for MCSymbolRefExpr::VariantKind.
+/// Limit this to top-level use, avoiding its inclusion as a subexpression.
+///
+/// NOTE: All subclasses are required to have trivial destructors because
+/// MCExprs are bump pointer allocated and not destructed.
+class LLVM_ABI MCSpecifierExpr : public MCExpr {
+protected:
+  using Spec = uint16_t;
+  const MCExpr *Expr;
+  // Target-specific relocation specifier code
+  const Spec specifier;
+
+  explicit MCSpecifierExpr(const MCExpr *Expr, Spec S, SMLoc Loc = SMLoc())
+      : MCExpr(Specifier, Loc), Expr(Expr), specifier(S) {}
+  virtual ~MCSpecifierExpr() = default;
+
+public:
+  LLVM_ABI static const MCSpecifierExpr *
+  create(const MCExpr *Expr, Spec S, MCContext &Ctx, SMLoc Loc = SMLoc());
+  LLVM_ABI static const MCSpecifierExpr *
+  create(const MCSymbol *Sym, Spec S, MCContext &Ctx, SMLoc Loc = SMLoc());
+
+  Spec getSpecifier() const { return specifier; }
+  const MCExpr *getSubExpr() const { return Expr; }
+
+  virtual void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
+    llvm_unreachable("Replace MCExpr::print calls with MCAsmInfo::printExpr");
+  }
+  virtual bool evaluateAsRelocatableImpl(MCValue &Res,
+                                         const MCAssembler *Asm) const;
+
+  static bool classof(const MCExpr *E) {
+    return E->getKind() == MCExpr::Specifier;
+  }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_MC_MCEXPR_H
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCFragment.h b/external/llvm-project/llvm/include/llvm/MC/MCFragment.h
index fdbfef28088d..473dec6477df 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCFragment.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCFragment.h
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include <cstdint>
 #include <utility>
@@ -78,7 +79,7 @@ class MCFragment {
   /// MCRelaxableFragment: x86-specific
   bool AllowAutoPadding : 1;
 
-  MCFragment(FragmentType Kind, bool HasInstructions);
+  LLVM_ABI MCFragment(FragmentType Kind, bool HasInstructions);
 
 public:
   MCFragment() = delete;
@@ -89,7 +90,7 @@ class MCFragment {
   ///
   /// This must be used instead of delete as MCFragment is non-virtual.
   /// This method will dispatch to the appropriate subclass.
-  void destroy();
+  LLVM_ABI void destroy();
 
   MCFragment *getNext() const { return Next; }
 
@@ -98,7 +99,7 @@ class MCFragment {
   MCSection *getParent() const { return Parent; }
   void setParent(MCSection *Value) { Parent = Value; }
 
-  const MCSymbol *getAtom() const;
+  LLVM_ABI const MCSymbol *getAtom() const;
 
   unsigned getLayoutOrder() const { return LayoutOrder; }
   void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
@@ -107,7 +108,7 @@ class MCFragment {
   /// this is false, but specific fragment types may set it to true.
   bool hasInstructions() const { return HasInstructions; }
 
-  void dump() const;
+  LLVM_ABI void dump() const;
 };
 
 /// Interface implemented by fragments that contain encoded instructions and/or
@@ -293,6 +294,7 @@ class MCFillFragment : public MCFragment {
   uint64_t Value;
   /// The number of bytes to insert.
   const MCExpr &NumValues;
+  uint64_t Size = 0;
 
   /// Source location of the directive that this fragment was created for.
   SMLoc Loc;
@@ -306,6 +308,8 @@ class MCFillFragment : public MCFragment {
   uint64_t getValue() const { return Value; }
   uint8_t getValueSize() const { return ValueSize; }
   const MCExpr &getNumValues() const { return NumValues; }
+  uint64_t getSize() const { return Size; }
+  void setSize(uint64_t Value) { Size = Value; }
 
   SMLoc getLoc() const { return Loc; }
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCInst.h b/external/llvm-project/llvm/include/llvm/MC/MCInst.h
index b3d615b4392f..581dc609b913 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCInst.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCInst.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/bit.h"
 #include "llvm/MC/MCRegister.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include <cassert>
 #include <cstddef>
@@ -174,10 +175,11 @@ class MCOperand {
     return Op;
   }
 
-  void print(raw_ostream &OS, const MCRegisterInfo *RegInfo = nullptr) const;
-  void dump() const;
-  bool isBareSymbolRef() const;
-  bool evaluateAsConstantImm(int64_t &Imm) const;
+  LLVM_ABI void print(raw_ostream &OS,
+                      const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void dump() const;
+  LLVM_ABI bool isBareSymbolRef() const;
+  LLVM_ABI bool evaluateAsConstantImm(int64_t &Imm) const;
 };
 
 /// Instances of this class represent a single low-level machine
@@ -226,17 +228,20 @@ class MCInst {
     return Operands.insert(I, Op);
   }
 
-  void print(raw_ostream &OS, const MCRegisterInfo *RegInfo = nullptr) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &OS,
+                      const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void dump() const;
 
   /// Dump the MCInst as prettily as possible using the additional MC
   /// structures, if given. Operators are separated by the \p Separator
   /// string.
-  void dump_pretty(raw_ostream &OS, const MCInstPrinter *Printer = nullptr,
-                   StringRef Separator = " ",
-                   const MCRegisterInfo *RegInfo = nullptr) const;
-  void dump_pretty(raw_ostream &OS, StringRef Name, StringRef Separator = " ",
-                   const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void dump_pretty(raw_ostream &OS,
+                            const MCInstPrinter *Printer = nullptr,
+                            StringRef Separator = " ",
+                            const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void dump_pretty(raw_ostream &OS, StringRef Name,
+                            StringRef Separator = " ",
+                            const MCRegisterInfo *RegInfo = nullptr) const;
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const MCOperand &MO) {
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCInstPrinter.h b/external/llvm-project/llvm/include/llvm/MC/MCInstPrinter.h
index ab1361313be0..434c2e762356 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCInstPrinter.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCInstPrinter.h
@@ -28,7 +28,7 @@ class MCSubtargetInfo;
 class StringRef;
 
 /// Convert `Bytes' to a hex string and output to `OS'
-void dumpBytes(ArrayRef<uint8_t> Bytes, raw_ostream &OS);
+LLVM_ABI void dumpBytes(ArrayRef<uint8_t> Bytes, raw_ostream &OS);
 
 namespace HexStyle {
 
@@ -43,7 +43,7 @@ struct AliasMatchingData;
 
 /// This is an instance of a target assembly language printer that
 /// converts an MCInst to valid target assembly syntax.
-class MCInstPrinter {
+class LLVM_ABI MCInstPrinter {
 protected:
   /// A stream that comments can be emitted to if desired.  Each comment
   /// must end with a newline.  This will be null if verbose assembly emission
@@ -101,9 +101,10 @@ class MCInstPrinter {
 
   class WithMarkup {
   public:
-    LLVM_CTOR_NODISCARD WithMarkup(MCInstPrinter &IP, raw_ostream &OS, Markup M,
-                                   bool EnableMarkup, bool EnableColor);
-    ~WithMarkup();
+    LLVM_CTOR_NODISCARD LLVM_ABI WithMarkup(MCInstPrinter &IP, raw_ostream &OS,
+                                            Markup M, bool EnableMarkup,
+                                            bool EnableColor);
+    LLVM_ABI ~WithMarkup();
 
     template <typename T> WithMarkup &operator<<(T &O) {
       OS << O;
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCInstrAnalysis.h b/external/llvm-project/llvm/include/llvm/MC/MCInstrAnalysis.h
index 7ad4c5f49623..63a4e02a9236 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCInstrAnalysis.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCInstrAnalysis.h
@@ -19,6 +19,7 @@
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <vector>
 
@@ -27,7 +28,7 @@ namespace llvm {
 class MCRegisterInfo;
 class Triple;
 
-class MCInstrAnalysis {
+class LLVM_ABI MCInstrAnalysis {
 protected:
   friend class Target;
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCInstrDesc.h b/external/llvm-project/llvm/include/llvm/MC/MCInstrDesc.h
index 5bb7f18bf258..8c70925d4780 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCInstrDesc.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCInstrDesc.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/MC/MCRegister.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class MCRegisterInfo;
@@ -329,7 +330,8 @@ class MCInstrDesc {
   /// Return true if this is a branch or an instruction which directly
   /// writes to the program counter. Considered 'may' affect rather than
   /// 'does' affect as things like predication are not taken into account.
-  bool mayAffectControlFlow(const MCInst &MI, const MCRegisterInfo &RI) const;
+  LLVM_ABI bool mayAffectControlFlow(const MCInst &MI,
+                                     const MCRegisterInfo &RI) const;
 
   /// Return true if this instruction has a predicate operand
   /// that controls execution. It may be set to 'always', or may be set to other
@@ -590,8 +592,9 @@ class MCInstrDesc {
 
   /// Return true if this instruction implicitly
   /// defines the specified physical register.
-  bool hasImplicitDefOfPhysReg(MCRegister Reg,
-                               const MCRegisterInfo *MRI = nullptr) const;
+  LLVM_ABI bool
+  hasImplicitDefOfPhysReg(MCRegister Reg,
+                          const MCRegisterInfo *MRI = nullptr) const;
 
   /// Return the scheduling class for this instruction.  The
   /// scheduling class is an index into the InstrItineraryData table.  This
@@ -617,8 +620,8 @@ class MCInstrDesc {
 
   /// Return true if this instruction defines the specified physical
   /// register, either explicitly or implicitly.
-  bool hasDefOfPhysReg(const MCInst &MI, MCRegister Reg,
-                       const MCRegisterInfo &RI) const;
+  LLVM_ABI bool hasDefOfPhysReg(const MCInst &MI, MCRegister Reg,
+                                const MCRegisterInfo &RI) const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCInstrInfo.h b/external/llvm-project/llvm/include/llvm/MC/MCInstrInfo.h
index 35ec1b797051..77ead222b954 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCInstrInfo.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCInstrInfo.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCInstrDesc.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 
 namespace llvm {
@@ -74,8 +75,8 @@ class MCInstrInfo {
 
   /// Returns true if a certain instruction is deprecated and if so
   /// returns the reason in \p Info.
-  bool getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
-                         std::string &Info) const;
+  LLVM_ABI bool getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
+                                  std::string &Info) const;
 };
 
 } // End llvm namespace
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCLinkerOptimizationHint.h b/external/llvm-project/llvm/include/llvm/MC/MCLinkerOptimizationHint.h
index dc086f94e5c5..d9dad791d6a2 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCLinkerOptimizationHint.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCLinkerOptimizationHint.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 
@@ -125,12 +126,12 @@ class MCLOHDirective {
 
   /// Emit this directive as:
   /// <kind, numArgs, addr1, ..., addrN>
-  void emit(const MCAssembler &Asm, MachObjectWriter &ObjWriter) const;
+  LLVM_ABI void emit(const MCAssembler &Asm, MachObjectWriter &ObjWriter) const;
 
   /// Get the size in bytes of this directive if emitted in \p ObjWriter with
   /// the given \p Layout.
-  uint64_t getEmitSize(const MCAssembler &Asm,
-                       const MachObjectWriter &ObjWriter) const;
+  LLVM_ABI uint64_t getEmitSize(const MCAssembler &Asm,
+                                const MachObjectWriter &ObjWriter) const;
 };
 
 class MCLOHContainer {
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCMachObjectWriter.h b/external/llvm-project/llvm/include/llvm/MC/MCMachObjectWriter.h
index 5d8e69b32ba6..a6ccdf06eb91 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -18,6 +18,7 @@
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/StringTableBuilder.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/VersionTuple.h"
 #include <cstdint>
@@ -29,7 +30,7 @@ namespace llvm {
 
 class MachObjectWriter;
 
-class MCMachObjectTargetWriter : public MCObjectTargetWriter {
+class LLVM_ABI MCMachObjectTargetWriter : public MCObjectTargetWriter {
   const unsigned Is64Bit : 1;
   const uint32_t CPUType;
 protected:
@@ -83,7 +84,7 @@ class MCMachObjectTargetWriter : public MCObjectTargetWriter {
   /// @}
 };
 
-class MachObjectWriter final : public MCObjectWriter {
+class LLVM_ABI MachObjectWriter final : public MCObjectWriter {
 public:
   struct DataRegionData {
     MachO::DataRegionType Kind;
@@ -114,7 +115,7 @@ class MachObjectWriter final : public MCObjectWriter {
     uint8_t SectionIndex;
 
     // Support lexicographic sorting.
-    bool operator<(const MachSymbolData &RHS) const;
+    LLVM_ABI bool operator<(const MachSymbolData &RHS) const;
   };
 
   struct IndirectSymbolData {
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCObjectFileInfo.h b/external/llvm-project/llvm/include/llvm/MC/MCObjectFileInfo.h
index fb575fe72101..20c819bdfd96 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCObjectFileInfo.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -15,6 +15,7 @@
 
 #include "llvm/BinaryFormat/Swift.h"
 #include "llvm/MC/MCSection.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -25,7 +26,7 @@ namespace llvm {
 class MCContext;
 class MCSection;
 
-class MCObjectFileInfo {
+class LLVM_ABI MCObjectFileInfo {
 protected:
   /// True if target object file supports a weak_definition of constant 0 for an
   /// omitted EH frame.
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCObjectWriter.h b/external/llvm-project/llvm/include/llvm/MC/MCObjectWriter.h
index 19530d2f651a..54a8f8236f6e 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCObjectWriter.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCObjectWriter.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCOBJECTWRITER_H
 
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cstdint>
 
@@ -30,7 +31,7 @@ class MCValue;
 /// points. Once assembly is complete, the object writer is given the
 /// MCAssembler instance, which contains all the symbol and section data which
 /// should be emitted as part of writeObject().
-class MCObjectWriter {
+class LLVM_ABI MCObjectWriter {
 protected:
   MCAssembler *Asm = nullptr;
   /// List of declared file names
@@ -139,8 +140,8 @@ class MCObjectTargetWriter {
   virtual Triple::ObjectFormatType getFormat() const = 0;
 
 protected:
-  MCContext &getContext() const;
-  void reportError(SMLoc L, const Twine &Msg) const;
+  LLVM_ABI MCContext &getContext() const;
+  LLVM_ABI void reportError(SMLoc L, const Twine &Msg) const;
 
   MCAssembler *Asm = nullptr;
 };
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCParser/AsmLexer.h b/external/llvm-project/llvm/include/llvm/MC/MCParser/AsmLexer.h
index 2e027f92adaf..11d32fbb6470 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCParser/AsmLexer.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCParser/AsmLexer.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCAsmMacro.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstddef>
 #include <string>
@@ -72,7 +73,7 @@ class AsmLexer {
   bool LexHLASMStrings = false;
   AsmCommentConsumer *CommentConsumer = nullptr;
 
-  AsmToken LexToken();
+  LLVM_ABI AsmToken LexToken();
 
   void SetError(SMLoc errLoc, const std::string &err) {
     ErrLoc = errLoc;
@@ -80,7 +81,7 @@ class AsmLexer {
   }
 
 public:
-  AsmLexer(const MCAsmInfo &MAI);
+  LLVM_ABI AsmLexer(const MCAsmInfo &MAI);
   AsmLexer(const AsmLexer &) = delete;
   AsmLexer &operator=(const AsmLexer &) = delete;
 
@@ -108,7 +109,7 @@ class AsmLexer {
 
   bool justConsumedEOL() { return JustConsumedEOL; }
 
-  StringRef LexUntilEndOfStatement();
+  LLVM_ABI StringRef LexUntilEndOfStatement();
 
   /// Get the current source location.
   SMLoc getLoc() const { return SMLoc::getFromPointer(TokStart); }
@@ -130,7 +131,8 @@ class AsmLexer {
   }
 
   /// Look ahead an arbitrary number of tokens.
-  size_t peekTokens(MutableArrayRef<AsmToken> Buf, bool ShouldSkipSpace = true);
+  LLVM_ABI size_t peekTokens(MutableArrayRef<AsmToken> Buf,
+                             bool ShouldSkipSpace = true);
 
   /// Get the current error location
   SMLoc getErrLoc() { return ErrLoc; }
@@ -189,8 +191,8 @@ class AsmLexer {
   /// literals.
   void setLexHLASMStrings(bool V) { LexHLASMStrings = V; }
 
-  void setBuffer(StringRef Buf, const char *ptr = nullptr,
-                 bool EndStatementAtEOF = true);
+  LLVM_ABI void setBuffer(StringRef Buf, const char *ptr = nullptr,
+                          bool EndStatementAtEOF = true);
 
   const MCAsmInfo &getMAI() const { return MAI; }
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParser.h
index e567a9ffbc0c..cb9bd5c600d5 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParser.h
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCAsmMacro.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include <cstdint>
 #include <string>
@@ -105,7 +106,7 @@ struct AsmFieldInfo {
 };
 
 /// Generic Sema callback for assembly parser.
-class MCAsmParserSemaCallback {
+class LLVM_ABI MCAsmParserSemaCallback {
 public:
   virtual ~MCAsmParserSemaCallback();
 
@@ -120,7 +121,7 @@ class MCAsmParserSemaCallback {
 
 /// Generic assembler parser interface, for use by target specific
 /// assembly parsers.
-class MCAsmParser {
+class LLVM_ABI MCAsmParser {
 public:
   using DirectiveHandler = bool (*)(MCAsmParserExtension*, StringRef, SMLoc);
   using ExtensionDirectiveHandler =
@@ -338,12 +339,13 @@ class MCAsmParser {
 };
 
 /// Create an MCAsmParser instance for parsing assembly similar to gas syntax
-MCAsmParser *createMCAsmParser(SourceMgr &, MCContext &, MCStreamer &,
-                               const MCAsmInfo &, unsigned CB = 0);
+LLVM_ABI MCAsmParser *createMCAsmParser(SourceMgr &, MCContext &, MCStreamer &,
+                                        const MCAsmInfo &, unsigned CB = 0);
 
 /// Create an MCAsmParser instance for parsing Microsoft MASM-style assembly
-MCAsmParser *createMCMasmParser(SourceMgr &, MCContext &, MCStreamer &,
-                                const MCAsmInfo &, struct tm, unsigned CB = 0);
+LLVM_ABI MCAsmParser *createMCMasmParser(SourceMgr &, MCContext &, MCStreamer &,
+                                         const MCAsmInfo &, struct tm,
+                                         unsigned CB = 0);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
index 5081b7cbcf11..f8387d888fc2 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCAsmParserExtension.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 
 namespace llvm {
@@ -21,7 +22,7 @@ class Twine;
 /// Generic interface for extending the MCAsmParser,
 /// which is implemented by target and object file assembly parser
 /// implementations.
-class MCAsmParserExtension {
+class LLVM_ABI MCAsmParserExtension {
   MCAsmParser *Parser = nullptr;
 
 protected:
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
index 27ecb7b85d22..fd52fbac5199 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCPARSER_MCPARSEDASMOPERAND_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include <string>
 
@@ -22,7 +23,7 @@ class raw_ostream;
 /// instruction operand.  It should be subclassed by target-specific code.  This
 /// base class is used by target-independent clients and is the interface
 /// between parsing an asm instruction and recognizing it.
-class MCParsedAsmOperand {
+class LLVM_ABI MCParsedAsmOperand {
   /// MCOperandNum - The corresponding MCInst operand number.  Only valid when
   /// parsing MS-style inline assembly.
   unsigned MCOperandNum = ~0u;
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
index c94ae9442f02..7628987b9587 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
@@ -15,6 +15,7 @@
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCRegister.h"
 #include "llvm/MC/MCTargetOptions.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
 #include <cstdint>
@@ -331,7 +332,7 @@ class NearMissInfo {
 };
 
 /// MCTargetAsmParser - Generic interface to target specific assembly parsers.
-class MCTargetAsmParser : public MCAsmParserExtension {
+class LLVM_ABI MCTargetAsmParser : public MCAsmParserExtension {
 public:
   enum MatchResultTy {
     Match_InvalidOperand,
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCPseudoProbe.h b/external/llvm-project/llvm/include/llvm/MC/MCPseudoProbe.h
index dc14038a03ac..ac28e45891df 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCPseudoProbe.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCPseudoProbe.h
@@ -62,6 +62,7 @@
 #include "llvm/ADT/iterator.h"
 #include "llvm/IR/PseudoProbe.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorOr.h"
 #include <functional>
 #include <memory>
@@ -92,7 +93,7 @@ struct MCPseudoProbeFuncDesc {
   MCPseudoProbeFuncDesc(uint64_t GUID, uint64_t Hash, StringRef Name)
       : FuncGUID(GUID), FuncHash(Hash), FuncName(Name){};
 
-  void print(raw_ostream &OS);
+  LLVM_ABI void print(raw_ostream &OS);
 };
 
 class MCDecodedPseudoProbe;
@@ -178,7 +179,8 @@ class MCPseudoProbe : public MCPseudoProbeBase {
 
   uint64_t getGuid() const { return Guid; };
   MCSymbol *getLabel() const { return Label; }
-  void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *LastProbe) const;
+  LLVM_ABI void emit(MCObjectStreamer *MCOS,
+                     const MCPseudoProbe *LastProbe) const;
 };
 
 // Represents a callsite with caller function name and probe id
@@ -193,7 +195,7 @@ class MCDecodedPseudoProbe : public MCPseudoProbeBase {
                        uint32_t D, MCDecodedPseudoProbeInlineTree *Tree)
       : MCPseudoProbeBase(I, At, static_cast<uint8_t>(K), D), Address(Ad),
         InlineTree(Tree){};
-  uint64_t getGuid() const;
+  LLVM_ABI uint64_t getGuid() const;
 
   uint64_t getAddress() const { return Address; }
 
@@ -206,17 +208,17 @@ class MCDecodedPseudoProbe : public MCPseudoProbeBase {
   // Get the inlined context by traversing current inline tree backwards,
   // each tree node has its InlineSite which is taken as the context.
   // \p ContextStack is populated in root to leaf order
-  void
+  LLVM_ABI void
   getInlineContext(SmallVectorImpl<MCPseudoProbeFrameLocation> &ContextStack,
                    const GUIDProbeFunctionMap &GUID2FuncMAP) const;
 
   // Helper function to get the string from context stack
-  std::string
+  LLVM_ABI std::string
   getInlineContextStr(const GUIDProbeFunctionMap &GUID2FuncMAP) const;
 
   // Print pseudo probe while disassembling
-  void print(raw_ostream &OS, const GUIDProbeFunctionMap &GUID2FuncMAP,
-             bool ShowName) const;
+  LLVM_ABI void print(raw_ostream &OS, const GUIDProbeFunctionMap &GUID2FuncMAP,
+                      bool ShowName) const;
 };
 
 // Address to pseudo probes map.
@@ -302,9 +304,9 @@ class MCPseudoProbeInlineTree
   }
 
   // MCPseudoProbeInlineTree method based on Inlinees
-  void addPseudoProbe(const MCPseudoProbe &Probe,
-                      const MCPseudoProbeInlineStack &InlineStack);
-  void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *&LastProbe);
+  LLVM_ABI void addPseudoProbe(const MCPseudoProbe &Probe,
+                               const MCPseudoProbeInlineStack &InlineStack);
+  LLVM_ABI void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *&LastProbe);
 };
 
 // inline tree node for the decoded pseudo probe
@@ -360,7 +362,7 @@ class MCPseudoProbeSections {
 
   bool empty() const { return MCProbeDivisions.empty(); }
 
-  void emit(MCObjectStreamer *MCOS);
+  LLVM_ABI void emit(MCObjectStreamer *MCOS);
 };
 
 class MCPseudoProbeTable {
@@ -373,7 +375,7 @@ class MCPseudoProbeTable {
   MCPseudoProbeSections MCProbeSections;
 
 public:
-  static void emit(MCObjectStreamer *MCOS);
+  LLVM_ABI static void emit(MCObjectStreamer *MCOS);
 
   MCPseudoProbeSections &getProbeSections() { return MCProbeSections; }
 
@@ -433,8 +435,8 @@ class MCPseudoProbeDecoder {
   // Decode pseudo_probe_desc section to build GUID to PseudoProbeFuncDesc map.
   // If pseudo_probe_desc section is mapped to memory and \p IsMMapped is true,
   // uses StringRefs pointing to the section.
-  bool buildGUID2FuncDescMap(const uint8_t *Start, std::size_t Size,
-                             bool IsMMapped = false);
+  LLVM_ABI bool buildGUID2FuncDescMap(const uint8_t *Start, std::size_t Size,
+                                      bool IsMMapped = false);
 
   // Decode pseudo_probe section to count the number of probes and inlined
   // function records for each function record.
@@ -444,23 +446,24 @@ class MCPseudoProbeDecoder {
 
   // Decode pseudo_probe section to build address to probes map for specifed
   // functions only.
-  bool buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size,
-                             const Uint64Set &GuildFilter,
-                             const Uint64Map &FuncStartAddrs);
+  LLVM_ABI bool buildAddress2ProbeMap(const uint8_t *Start, std::size_t Size,
+                                      const Uint64Set &GuildFilter,
+                                      const Uint64Map &FuncStartAddrs);
 
   // Print pseudo_probe_desc section info
-  void printGUID2FuncDescMap(raw_ostream &OS);
+  LLVM_ABI void printGUID2FuncDescMap(raw_ostream &OS);
 
   // Print pseudo_probe section info, used along with show-disassembly
-  void printProbeForAddress(raw_ostream &OS, uint64_t Address);
+  LLVM_ABI void printProbeForAddress(raw_ostream &OS, uint64_t Address);
 
   // do printProbeForAddress for all addresses
-  void printProbesForAllAddresses(raw_ostream &OS);
+  LLVM_ABI void printProbesForAllAddresses(raw_ostream &OS);
 
   // Look up the probe of a call for the input address
-  const MCDecodedPseudoProbe *getCallProbeForAddr(uint64_t Address) const;
+  LLVM_ABI const MCDecodedPseudoProbe *
+  getCallProbeForAddr(uint64_t Address) const;
 
-  const MCPseudoProbeFuncDesc *getFuncDescForGUID(uint64_t GUID) const;
+  LLVM_ABI const MCPseudoProbeFuncDesc *getFuncDescForGUID(uint64_t GUID) const;
 
   // Helper function to populate one probe's inline stack into
   // \p InlineContextStack.
@@ -469,7 +472,7 @@ class MCPseudoProbeDecoder {
   //  Current probe(bar:3) inlined at foo:2 then inlined at main:1
   //  IncludeLeaf = true,  Output: [main:1, foo:2, bar:3]
   //  IncludeLeaf = false, Output: [main:1, foo:2]
-  void getInlineContextForProbe(
+  LLVM_ABI void getInlineContextForProbe(
       const MCDecodedPseudoProbe *Probe,
       SmallVectorImpl<MCPseudoProbeFrameLocation> &InlineContextStack,
       bool IncludeLeaf) const;
@@ -484,7 +487,7 @@ class MCPseudoProbeDecoder {
     return GUID2FuncDescMap;
   }
 
-  const MCPseudoProbeFuncDesc *
+  LLVM_ABI const MCPseudoProbeFuncDesc *
   getInlinerDescForProbe(const MCDecodedPseudoProbe *Probe) const;
 
   const MCDecodedPseudoProbeInlineTree &getDummyInlineRoot() const {
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCRegisterInfo.h b/external/llvm-project/llvm/include/llvm/MC/MCRegisterInfo.h
index f780bdb97d4f..aad3792f8878 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCRegisterInfo.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCRegisterInfo.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/MC/LaneBitmask.h"
 #include "llvm/MC/MCRegister.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -146,7 +147,7 @@ struct MCRegisterDesc {
 /// TableGen generated physical register data. It must not be extended with
 /// virtual methods.
 ///
-class MCRegisterInfo {
+class LLVM_ABI MCRegisterInfo {
 public:
   using regclass_iterator = const MCRegisterClass *;
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCSchedule.h b/external/llvm-project/llvm/include/llvm/MC/MCSchedule.h
index 57c8ebeee02a..eb71f4581bd6 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCSchedule.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCSchedule.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCInstrDesc.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 #include <optional>
@@ -368,13 +369,15 @@ struct MCSchedModel {
   }
 
   /// Returns the latency value for the scheduling class.
-  static int computeInstrLatency(const MCSubtargetInfo &STI,
-                                 const MCSchedClassDesc &SCDesc);
+  LLVM_ABI static int computeInstrLatency(const MCSubtargetInfo &STI,
+                                          const MCSchedClassDesc &SCDesc);
 
-  int computeInstrLatency(const MCSubtargetInfo &STI, unsigned SClass) const;
+  LLVM_ABI int computeInstrLatency(const MCSubtargetInfo &STI,
+                                   unsigned SClass) const;
 
-  int computeInstrLatency(const MCSubtargetInfo &STI, const MCInstrInfo &MCII,
-                          const MCInst &Inst) const;
+  LLVM_ABI int computeInstrLatency(const MCSubtargetInfo &STI,
+                                   const MCInstrInfo &MCII,
+                                   const MCInst &Inst) const;
 
   template <typename MCSubtargetInfo, typename MCInstrInfo,
             typename InstrItineraryData, typename MCInstOrMachineInstr>
@@ -386,28 +389,29 @@ struct MCSchedModel {
               [](const MCSchedClassDesc *SCDesc) { return SCDesc; }) const;
 
   // Returns the reciprocal throughput information from a MCSchedClassDesc.
-  static double
+  LLVM_ABI static double
   getReciprocalThroughput(const MCSubtargetInfo &STI,
                           const MCSchedClassDesc &SCDesc);
 
-  static double
-  getReciprocalThroughput(unsigned SchedClass, const InstrItineraryData &IID);
+  LLVM_ABI static double getReciprocalThroughput(unsigned SchedClass,
+                                                 const InstrItineraryData &IID);
 
-  double
-  getReciprocalThroughput(const MCSubtargetInfo &STI, const MCInstrInfo &MCII,
-                          const MCInst &Inst) const;
+  LLVM_ABI double getReciprocalThroughput(const MCSubtargetInfo &STI,
+                                          const MCInstrInfo &MCII,
+                                          const MCInst &Inst) const;
 
   /// Returns the maximum forwarding delay for register reads dependent on
   /// writes of scheduling class WriteResourceIdx.
-  static unsigned getForwardingDelayCycles(ArrayRef<MCReadAdvanceEntry> Entries,
-                                           unsigned WriteResourceIdx = 0);
+  LLVM_ABI static unsigned
+  getForwardingDelayCycles(ArrayRef<MCReadAdvanceEntry> Entries,
+                           unsigned WriteResourceIdx = 0);
 
   /// Returns the bypass delay cycle for the maximum latency write cycle
-  static unsigned getBypassDelayCycles(const MCSubtargetInfo &STI,
-                                       const MCSchedClassDesc &SCDesc);
+  LLVM_ABI static unsigned getBypassDelayCycles(const MCSubtargetInfo &STI,
+                                                const MCSchedClassDesc &SCDesc);
 
   /// Returns the default initialized model.
-  static const MCSchedModel Default;
+  LLVM_ABI static const MCSchedModel Default;
 };
 
 // The first three are only template'd arguments so we can get away with leaving
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCSection.h b/external/llvm-project/llvm/include/llvm/MC/MCSection.h
index 942d09bad594..7a94a7d65613 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCSection.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCSection.h
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCFragment.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <utility>
 
@@ -33,7 +34,7 @@ class Triple;
 
 /// Instances of this class represent a uniqued identifier for a section in the
 /// current translation unit.  The MCContext class uniques and creates these.
-class MCSection {
+class LLVM_ABI MCSection {
 public:
   friend MCAssembler;
   friend MCObjectStreamer;
@@ -99,8 +100,6 @@ class MCSection {
   /// Whether this section has had instructions emitted into it.
   bool HasInstructions : 1;
 
-  bool HasLayout : 1;
-
   bool IsRegistered : 1;
 
   bool IsText : 1;
@@ -171,9 +170,6 @@ class MCSection {
   bool hasInstructions() const { return HasInstructions; }
   void setHasInstructions(bool Value) { HasInstructions = Value; }
 
-  bool hasLayout() const { return HasLayout; }
-  void setHasLayout(bool Value) { HasLayout = Value; }
-
   bool isRegistered() const { return IsRegistered; }
   void setIsRegistered(bool Value) { IsRegistered = Value; }
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCSectionMachO.h b/external/llvm-project/llvm/include/llvm/MC/MCSectionMachO.h
index 1f38d24a20d2..4312175fd16d 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCSectionMachO.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCSectionMachO.h
@@ -16,12 +16,13 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/MC/MCSection.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// This represents a section on a Mach-O system (used by Mac OS X).  On a Mac
 /// system, these are also described in /usr/include/mach-o/loader.h.
-class MCSectionMachO final : public MCSection {
+class LLVM_ABI MCSectionMachO final : public MCSection {
   char SegmentName[16];  // Not necessarily null terminated!
 
   /// This is the SECTION_TYPE and SECTION_ATTRIBUTES field of a section, drawn
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCStreamer.h b/external/llvm-project/llvm/include/llvm/MC/MCStreamer.h
index 17af845a9e87..455796aeff9f 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCStreamer.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCStreamer.h
@@ -23,6 +23,7 @@
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCWinEH.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/SMLoc.h"
@@ -91,7 +92,7 @@ using MCSectionSubPair = std::pair<MCSection *, uint32_t>;
 /// The base classes FooTargetAsmStreamer and FooTargetELFStreamer should
 /// *never* be treated differently. Callers should always talk to a
 /// FooTargetStreamer.
-class MCTargetStreamer {
+class LLVM_ABI MCTargetStreamer {
 protected:
   MCStreamer &Streamer;
 
@@ -134,7 +135,7 @@ class MCTargetStreamer {
 
 // FIXME: declared here because it is used from
 // lib/CodeGen/AsmPrinter/ARMException.cpp.
-class ARMTargetStreamer : public MCTargetStreamer {
+class LLVM_ABI ARMTargetStreamer : public MCTargetStreamer {
 public:
   ARMTargetStreamer(MCStreamer &S);
   ~ARMTargetStreamer() override;
@@ -218,7 +219,7 @@ class ARMTargetStreamer : public MCTargetStreamer {
 /// There are multiple implementations of this interface: one for writing out
 /// a .s file, and implementations that write out .o files of various formats.
 ///
-class MCStreamer {
+class LLVM_ABI MCStreamer {
   MCContext &Context;
   std::unique_ptr<MCTargetStreamer> TargetStreamer;
 
@@ -1147,7 +1148,7 @@ inline MCContext &MCTargetStreamer::getContext() {
 
 /// Create a dummy machine code streamer, which does nothing. This is useful for
 /// timing the assembler front end.
-MCStreamer *createNullStreamer(MCContext &Ctx);
+LLVM_ABI MCStreamer *createNullStreamer(MCContext &Ctx);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCSubtargetInfo.h b/external/llvm-project/llvm/include/llvm/MC/MCSubtargetInfo.h
index 535bcfe2fb6d..44b4cb8fa17b 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/MC/MCSchedule.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cassert>
@@ -73,7 +74,7 @@ struct SubtargetSubTypeKV {
 ///
 /// Generic base class for all target subtargets.
 ///
-class MCSubtargetInfo {
+class LLVM_ABI MCSubtargetInfo {
   Triple TargetTriple;
   std::string CPU; // CPU being targeted.
   std::string TuneCPU; // CPU being tuned for.
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCSymbol.h b/external/llvm-project/llvm/include/llvm/MC/MCSymbol.h
index a990f360b0f7..b02bc111955a 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCSymbol.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCSymbol.h
@@ -18,6 +18,7 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCSymbolTableEntry.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
@@ -63,7 +64,7 @@ class MCSymbol {
   };
 
   // Special sentinel value for the absolute pseudo fragment.
-  static MCFragment *AbsolutePseudoFragment;
+  LLVM_ABI static MCFragment *AbsolutePseudoFragment;
 
   /// If a symbol has a Fragment, the section is implied, so we only need
   /// one pointer.
@@ -177,7 +178,8 @@ class MCSymbol {
 
   // Provide custom new/delete as we will only allocate space for a name
   // if we need one.
-  void *operator new(size_t s, const MCSymbolTableEntry *Name, MCContext &Ctx);
+  LLVM_ABI void *operator new(size_t s, const MCSymbolTableEntry *Name,
+                              MCContext &Ctx);
 
 private:
   void operator delete(void *);
@@ -306,7 +308,7 @@ class MCSymbol {
     return Value;
   }
 
-  void setVariableValue(const MCExpr *Value);
+  LLVM_ABI void setVariableValue(const MCExpr *Value);
 
   /// @}
 
@@ -409,10 +411,10 @@ class MCSymbol {
   bool isWeakExternal() const { return IsWeakExternal; }
 
   /// print - Print the value to the stream \p OS.
-  void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
+  LLVM_ABI void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
 
   /// dump - Print the value to stderr.
-  void dump() const;
+  LLVM_ABI void dump() const;
 
 protected:
   /// Get the (implementation defined) symbol flags.
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCTargetOptions.h b/external/llvm-project/llvm/include/llvm/MC/MCTargetOptions.h
index fd554bc0ba7c..3ee21d9cda4b 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCTargetOptions.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCTargetOptions.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCTARGETOPTIONS_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include <string>
 #include <vector>
@@ -115,17 +116,17 @@ class MCTargetOptions {
   // Whether or not to use full register names on PowerPC.
   bool PPCUseFullRegisterNames : 1;
 
-  MCTargetOptions();
+  LLVM_ABI MCTargetOptions();
 
   /// getABIName - If this returns a non-empty string this represents the
   /// textual name of the ABI that we want the backend to use, e.g. o32, or
   /// aapcs-linux.
-  StringRef getABIName() const;
+  LLVM_ABI StringRef getABIName() const;
 
   /// getAssemblyLanguage - If this returns a non-empty string this represents
   /// the textual name of the assembly language that we will use for this
   /// target, e.g. masm.
-  StringRef getAssemblyLanguage() const;
+  LLVM_ABI StringRef getAssemblyLanguage() const;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h b/external/llvm-project/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h
index 5e82bc53f3b5..b057effd8812 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_MC_MCTARGETOPTIONSCOMMANDFLAGS_H
 #define LLVM_MC_MCTARGETOPTIONSCOMMANDFLAGS_H
 
+#include "llvm/Support/Compiler.h"
 #include <optional>
 #include <string>
 
@@ -24,52 +25,52 @@ enum class EmitDwarfUnwindType;
 
 namespace mc {
 
-bool getRelaxAll();
-std::optional<bool> getExplicitRelaxAll();
+LLVM_ABI bool getRelaxAll();
+LLVM_ABI std::optional<bool> getExplicitRelaxAll();
 
-bool getIncrementalLinkerCompatible();
+LLVM_ABI bool getIncrementalLinkerCompatible();
 
-bool getFDPIC();
+LLVM_ABI bool getFDPIC();
 
-int getDwarfVersion();
+LLVM_ABI int getDwarfVersion();
 
-bool getDwarf64();
+LLVM_ABI bool getDwarf64();
 
-EmitDwarfUnwindType getEmitDwarfUnwind();
+LLVM_ABI EmitDwarfUnwindType getEmitDwarfUnwind();
 
-bool getEmitCompactUnwindNonCanonical();
+LLVM_ABI bool getEmitCompactUnwindNonCanonical();
 
-bool getShowMCInst();
+LLVM_ABI bool getShowMCInst();
 
-bool getFatalWarnings();
+LLVM_ABI bool getFatalWarnings();
 
-bool getNoWarn();
+LLVM_ABI bool getNoWarn();
 
-bool getNoDeprecatedWarn();
+LLVM_ABI bool getNoDeprecatedWarn();
 
-bool getNoTypeCheck();
+LLVM_ABI bool getNoTypeCheck();
 
-bool getSaveTempLabels();
+LLVM_ABI bool getSaveTempLabels();
 
-bool getCrel();
+LLVM_ABI bool getCrel();
 
-bool getImplicitMapSyms();
+LLVM_ABI bool getImplicitMapSyms();
 
-bool getX86RelaxRelocations();
+LLVM_ABI bool getX86RelaxRelocations();
 
-bool getX86Sse2Avx();
+LLVM_ABI bool getX86Sse2Avx();
 
-std::string getABIName();
+LLVM_ABI std::string getABIName();
 
-std::string getAsSecureLogFile();
+LLVM_ABI std::string getAsSecureLogFile();
 
 /// Create this object with static storage to register mc-related command
 /// line options.
 struct RegisterMCTargetOptionsFlags {
-  RegisterMCTargetOptionsFlags();
+  LLVM_ABI RegisterMCTargetOptionsFlags();
 };
 
-MCTargetOptions InitMCTargetOptionsFromFlags();
+LLVM_ABI MCTargetOptions InitMCTargetOptionsFromFlags();
 
 } // namespace mc
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/MCWinEH.h b/external/llvm-project/llvm/include/llvm/MC/MCWinEH.h
index d889ec71e7d8..1bbfb9f59bc4 100644
--- a/external/llvm-project/llvm/include/llvm/MC/MCWinEH.h
+++ b/external/llvm-project/llvm/include/llvm/MC/MCWinEH.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCWINEH_H
 
 #include "llvm/ADT/MapVector.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SMLoc.h"
 #include <vector>
 
@@ -103,7 +104,7 @@ struct FrameInfo {
   }
 };
 
-class UnwindEmitter {
+class LLVM_ABI UnwindEmitter {
 public:
   virtual ~UnwindEmitter();
 
diff --git a/external/llvm-project/llvm/include/llvm/MC/StringTableBuilder.h b/external/llvm-project/llvm/include/llvm/MC/StringTableBuilder.h
index a738683548cf..83d55ef8512f 100644
--- a/external/llvm-project/llvm/include/llvm/MC/StringTableBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/MC/StringTableBuilder.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Compiler.h"
 #include <cstddef>
 #include <cstdint>
 
@@ -47,26 +48,26 @@ class StringTableBuilder {
   void initSize();
 
 public:
-  StringTableBuilder(Kind K, Align Alignment = Align(1));
-  ~StringTableBuilder();
+  LLVM_ABI StringTableBuilder(Kind K, Align Alignment = Align(1));
+  LLVM_ABI ~StringTableBuilder();
 
   /// Add a string to the builder. Returns the position of S in the
   /// table. The position will be changed if finalize is used.
   /// Can only be used before the table is finalized.
-  size_t add(CachedHashStringRef S);
+  LLVM_ABI size_t add(CachedHashStringRef S);
   size_t add(StringRef S) { return add(CachedHashStringRef(S)); }
 
   /// Analyze the strings and build the final table. No more strings can
   /// be added after this point.
-  void finalize();
+  LLVM_ABI void finalize();
 
   /// Finalize the string table without reording it. In this mode, offsets
   /// returned by add will still be valid.
-  void finalizeInOrder();
+  LLVM_ABI void finalizeInOrder();
 
   /// Get the offest of a string in the string table. Can only be used
   /// after the table is finalized.
-  size_t getOffset(CachedHashStringRef S) const;
+  LLVM_ABI size_t getOffset(CachedHashStringRef S) const;
   size_t getOffset(StringRef S) const {
     return getOffset(CachedHashStringRef(S));
   }
@@ -78,10 +79,10 @@ class StringTableBuilder {
   bool contains(CachedHashStringRef S) const { return StringIndexMap.count(S); }
 
   size_t getSize() const { return Size; }
-  void clear();
+  LLVM_ABI void clear();
 
-  void write(raw_ostream &OS) const;
-  void write(uint8_t *Buf) const;
+  LLVM_ABI void write(raw_ostream &OS) const;
+  LLVM_ABI void write(uint8_t *Buf) const;
 
   bool isFinalized() const { return Finalized; }
 };
diff --git a/external/llvm-project/llvm/include/llvm/MC/TargetRegistry.h b/external/llvm-project/llvm/include/llvm/MC/TargetRegistry.h
index f54f7ce75653..efff08d7f8ea 100644
--- a/external/llvm-project/llvm/include/llvm/MC/TargetRegistry.h
+++ b/external/llvm-project/llvm/include/llvm/MC/TargetRegistry.h
@@ -65,7 +65,7 @@ class InstrumentManager;
 struct SourceMgr;
 } // namespace mca
 
-MCStreamer *createNullStreamer(MCContext &Ctx);
+LLVM_ABI MCStreamer *createNullStreamer(MCContext &Ctx);
 // Takes ownership of \p TAB and \p CE.
 
 /// Create a machine code streamer which will print out assembly for the native
@@ -84,55 +84,56 @@ MCStreamer *createNullStreamer(MCContext &Ctx);
 ///
 /// \param ShowInst - Whether to show the MCInst representation inline with
 /// the assembly.
-MCStreamer *createAsmStreamer(MCContext &Ctx,
-                              std::unique_ptr<formatted_raw_ostream> OS,
-                              std::unique_ptr<MCInstPrinter> InstPrint,
-                              std::unique_ptr<MCCodeEmitter> CE,
-                              std::unique_ptr<MCAsmBackend> TAB);
-
-MCStreamer *createELFStreamer(MCContext &Ctx,
-                              std::unique_ptr<MCAsmBackend> &&TAB,
-                              std::unique_ptr<MCObjectWriter> &&OW,
-                              std::unique_ptr<MCCodeEmitter> &&CE);
-MCStreamer *createGOFFStreamer(MCContext &Ctx,
-                               std::unique_ptr<MCAsmBackend> &&TAB,
-                               std::unique_ptr<MCObjectWriter> &&OW,
-                               std::unique_ptr<MCCodeEmitter> &&CE);
-MCStreamer *createMachOStreamer(MCContext &Ctx,
-                                std::unique_ptr<MCAsmBackend> &&TAB,
-                                std::unique_ptr<MCObjectWriter> &&OW,
-                                std::unique_ptr<MCCodeEmitter> &&CE,
-                                bool DWARFMustBeAtTheEnd,
-                                bool LabelSections = false);
-MCStreamer *createWasmStreamer(MCContext &Ctx,
-                               std::unique_ptr<MCAsmBackend> &&TAB,
-                               std::unique_ptr<MCObjectWriter> &&OW,
-                               std::unique_ptr<MCCodeEmitter> &&CE);
-MCStreamer *createSPIRVStreamer(MCContext &Ctx,
-                                std::unique_ptr<MCAsmBackend> &&TAB,
-                                std::unique_ptr<MCObjectWriter> &&OW,
-                                std::unique_ptr<MCCodeEmitter> &&CE);
-MCStreamer *createDXContainerStreamer(MCContext &Ctx,
-                                      std::unique_ptr<MCAsmBackend> &&TAB,
-                                      std::unique_ptr<MCObjectWriter> &&OW,
-                                      std::unique_ptr<MCCodeEmitter> &&CE);
-
-MCRelocationInfo *createMCRelocationInfo(const Triple &TT, MCContext &Ctx);
-
-MCSymbolizer *createMCSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo,
-                                 LLVMSymbolLookupCallback SymbolLookUp,
-                                 void *DisInfo, MCContext *Ctx,
-                                 std::unique_ptr<MCRelocationInfo> &&RelInfo);
-
-mca::CustomBehaviour *createCustomBehaviour(const MCSubtargetInfo &STI,
-                                            const mca::SourceMgr &SrcMgr,
-                                            const MCInstrInfo &MCII);
-
-mca::InstrPostProcess *createInstrPostProcess(const MCSubtargetInfo &STI,
-                                              const MCInstrInfo &MCII);
-
-mca::InstrumentManager *createInstrumentManager(const MCSubtargetInfo &STI,
-                                                const MCInstrInfo &MCII);
+LLVM_ABI MCStreamer *
+createAsmStreamer(MCContext &Ctx, std::unique_ptr<formatted_raw_ostream> OS,
+                  std::unique_ptr<MCInstPrinter> InstPrint,
+                  std::unique_ptr<MCCodeEmitter> CE,
+                  std::unique_ptr<MCAsmBackend> TAB);
+
+LLVM_ABI MCStreamer *createELFStreamer(MCContext &Ctx,
+                                       std::unique_ptr<MCAsmBackend> &&TAB,
+                                       std::unique_ptr<MCObjectWriter> &&OW,
+                                       std::unique_ptr<MCCodeEmitter> &&CE);
+LLVM_ABI MCStreamer *createGOFFStreamer(MCContext &Ctx,
+                                        std::unique_ptr<MCAsmBackend> &&TAB,
+                                        std::unique_ptr<MCObjectWriter> &&OW,
+                                        std::unique_ptr<MCCodeEmitter> &&CE);
+LLVM_ABI MCStreamer *createMachOStreamer(MCContext &Ctx,
+                                         std::unique_ptr<MCAsmBackend> &&TAB,
+                                         std::unique_ptr<MCObjectWriter> &&OW,
+                                         std::unique_ptr<MCCodeEmitter> &&CE,
+                                         bool DWARFMustBeAtTheEnd,
+                                         bool LabelSections = false);
+LLVM_ABI MCStreamer *createWasmStreamer(MCContext &Ctx,
+                                        std::unique_ptr<MCAsmBackend> &&TAB,
+                                        std::unique_ptr<MCObjectWriter> &&OW,
+                                        std::unique_ptr<MCCodeEmitter> &&CE);
+LLVM_ABI MCStreamer *createSPIRVStreamer(MCContext &Ctx,
+                                         std::unique_ptr<MCAsmBackend> &&TAB,
+                                         std::unique_ptr<MCObjectWriter> &&OW,
+                                         std::unique_ptr<MCCodeEmitter> &&CE);
+LLVM_ABI MCStreamer *
+createDXContainerStreamer(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB,
+                          std::unique_ptr<MCObjectWriter> &&OW,
+                          std::unique_ptr<MCCodeEmitter> &&CE);
+
+LLVM_ABI MCRelocationInfo *createMCRelocationInfo(const Triple &TT,
+                                                  MCContext &Ctx);
+
+LLVM_ABI MCSymbolizer *
+createMCSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo,
+                   LLVMSymbolLookupCallback SymbolLookUp, void *DisInfo,
+                   MCContext *Ctx, std::unique_ptr<MCRelocationInfo> &&RelInfo);
+
+LLVM_ABI mca::CustomBehaviour *
+createCustomBehaviour(const MCSubtargetInfo &STI, const mca::SourceMgr &SrcMgr,
+                      const MCInstrInfo &MCII);
+
+LLVM_ABI mca::InstrPostProcess *
+createInstrPostProcess(const MCSubtargetInfo &STI, const MCInstrInfo &MCII);
+
+LLVM_ABI mca::InstrumentManager *
+createInstrumentManager(const MCSubtargetInfo &STI, const MCInstrInfo &MCII);
 
 /// Target - Wrapper for Target specific information.
 ///
@@ -545,17 +546,16 @@ class Target {
   /// \param TAB The target assembler backend object. Takes ownership.
   /// \param OW The stream object.
   /// \param Emitter The target independent assembler object.Takes ownership.
-  MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx,
-                                     std::unique_ptr<MCAsmBackend> TAB,
-                                     std::unique_ptr<MCObjectWriter> OW,
-                                     std::unique_ptr<MCCodeEmitter> Emitter,
-                                     const MCSubtargetInfo &STI) const;
-
-  MCStreamer *createAsmStreamer(MCContext &Ctx,
-                                std::unique_ptr<formatted_raw_ostream> OS,
-                                std::unique_ptr<MCInstPrinter> IP,
-                                std::unique_ptr<MCCodeEmitter> CE,
-                                std::unique_ptr<MCAsmBackend> TAB) const;
+  LLVM_ABI MCStreamer *createMCObjectStreamer(
+      const Triple &T, MCContext &Ctx, std::unique_ptr<MCAsmBackend> TAB,
+      std::unique_ptr<MCObjectWriter> OW,
+      std::unique_ptr<MCCodeEmitter> Emitter, const MCSubtargetInfo &STI) const;
+
+  LLVM_ABI MCStreamer *
+  createAsmStreamer(MCContext &Ctx, std::unique_ptr<formatted_raw_ostream> OS,
+                    std::unique_ptr<MCInstPrinter> IP,
+                    std::unique_ptr<MCCodeEmitter> CE,
+                    std::unique_ptr<MCAsmBackend> TAB) const;
 
   MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
                                             formatted_raw_ostream &OS,
@@ -693,12 +693,12 @@ struct TargetRegistry {
 
   /// printRegisteredTargetsForVersion - Print the registered targets
   /// appropriately for inclusion in a tool's version output.
-  static void printRegisteredTargetsForVersion(raw_ostream &OS);
+  LLVM_ABI static void printRegisteredTargetsForVersion(raw_ostream &OS);
 
   /// @name Registry Access
   /// @{
 
-  static iterator_range<iterator> targets();
+  LLVM_ABI static iterator_range<iterator> targets();
 
   /// lookupTarget - Lookup a target based on a target triple.
   ///
@@ -715,8 +715,8 @@ struct TargetRegistry {
   /// \param Triple - The triple to use for finding a target.
   /// \param Error - On failure, an error string describing why no target was
   /// found.
-  static const Target *lookupTarget(const Triple &TheTriple,
-                                    std::string &Error);
+  LLVM_ABI static const Target *lookupTarget(const Triple &TheTriple,
+                                             std::string &Error);
 
   /// lookupTarget - Lookup a target based on an architecture name
   /// and a target triple.  If the architecture name is non-empty,
@@ -729,8 +729,8 @@ struct TargetRegistry {
   /// by architecture is done.
   /// \param Error - On failure, an error string describing why no target was
   /// found.
-  static const Target *lookupTarget(StringRef ArchName, Triple &TheTriple,
-                                    std::string &Error);
+  LLVM_ABI static const Target *
+  lookupTarget(StringRef ArchName, Triple &TheTriple, std::string &Error);
 
   /// @}
   /// @name Target Registration
@@ -754,10 +754,11 @@ struct TargetRegistry {
   /// @param ArchMatchFn - The arch match checking function for this target.
   /// @param HasJIT - Whether the target supports JIT code
   /// generation.
-  static void RegisterTarget(Target &T, const char *Name, const char *ShortDesc,
-                             const char *BackendName,
-                             Target::ArchMatchFnTy ArchMatchFn,
-                             bool HasJIT = false);
+  LLVM_ABI static void RegisterTarget(Target &T, const char *Name,
+                                      const char *ShortDesc,
+                                      const char *BackendName,
+                                      Target::ArchMatchFnTy ArchMatchFn,
+                                      bool HasJIT = false);
 
   /// RegisterMCAsmInfo - Register a MCAsmInfo implementation for the
   /// given target.
diff --git a/external/llvm-project/llvm/include/llvm/MCA/CodeEmitter.h b/external/llvm-project/llvm/include/llvm/MCA/CodeEmitter.h
index 431ceea27064..cc978c78244f 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/CodeEmitter.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/CodeEmitter.h
@@ -23,6 +23,7 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
@@ -46,7 +47,7 @@ class CodeEmitter {
   // A cache of encodings.
   SmallVector<EncodingInfo, 16> Encodings;
 
-  EncodingInfo getOrCreateEncodingInfo(unsigned MCID);
+  LLVM_ABI EncodingInfo getOrCreateEncodingInfo(unsigned MCID);
 
 public:
   CodeEmitter(const MCSubtargetInfo &ST, const MCAsmBackend &AB,
diff --git a/external/llvm-project/llvm/include/llvm/MCA/Context.h b/external/llvm-project/llvm/include/llvm/MCA/Context.h
index 0abcfd7ce9f0..b364cee5c169 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/Context.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/Context.h
@@ -23,6 +23,7 @@
 #include "llvm/MCA/HardwareUnits/HardwareUnit.h"
 #include "llvm/MCA/Pipeline.h"
 #include "llvm/MCA/SourceMgr.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -67,15 +68,15 @@ class Context {
 
   /// Construct a basic pipeline for simulating an out-of-order pipeline.
   /// This pipeline consists of Fetch, Dispatch, Execute, and Retire stages.
-  std::unique_ptr<Pipeline> createDefaultPipeline(const PipelineOptions &Opts,
-                                                  SourceMgr &SrcMgr,
-                                                  CustomBehaviour &CB);
+  LLVM_ABI std::unique_ptr<Pipeline>
+  createDefaultPipeline(const PipelineOptions &Opts, SourceMgr &SrcMgr,
+                        CustomBehaviour &CB);
 
   /// Construct a basic pipeline for simulating an in-order pipeline.
   /// This pipeline consists of Fetch, InOrderIssue, and Retire stages.
-  std::unique_ptr<Pipeline> createInOrderPipeline(const PipelineOptions &Opts,
-                                                  SourceMgr &SrcMgr,
-                                                  CustomBehaviour &CB);
+  LLVM_ABI std::unique_ptr<Pipeline>
+  createInOrderPipeline(const PipelineOptions &Opts, SourceMgr &SrcMgr,
+                        CustomBehaviour &CB);
 };
 
 } // namespace mca
diff --git a/external/llvm-project/llvm/include/llvm/MCA/CustomBehaviour.h b/external/llvm-project/llvm/include/llvm/MCA/CustomBehaviour.h
index b3774894517a..0bf86ce688f4 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/CustomBehaviour.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/CustomBehaviour.h
@@ -24,6 +24,7 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MCA/SourceMgr.h"
 #include "llvm/MCA/View.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
@@ -63,7 +64,7 @@ class InstrPostProcess {
 /// them properly.
 /// If you implement this class for your target, make sure to also implement
 /// a target specific InstrPostProcess class as well.
-class CustomBehaviour {
+class LLVM_ABI CustomBehaviour {
 protected:
   const MCSubtargetInfo &STI;
   const mca::SourceMgr &SrcMgr;
@@ -138,7 +139,7 @@ using UniqueInstrument = std::unique_ptr<Instrument>;
 /// This class allows targets to optionally customize the logic that resolves
 /// scheduling class IDs. Targets can use information encoded in Instrument
 /// objects to make more informed scheduling decisions.
-class InstrumentManager {
+class LLVM_ABI InstrumentManager {
 protected:
   const MCSubtargetInfo &STI;
   const MCInstrInfo &MCII;
diff --git a/external/llvm-project/llvm/include/llvm/MCA/HWEventListener.h b/external/llvm-project/llvm/include/llvm/MCA/HWEventListener.h
index a27b1f12e6a6..e217c4893149 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/HWEventListener.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/HWEventListener.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MCA/Instruction.h"
 #include "llvm/MCA/Support.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
@@ -160,7 +161,7 @@ class HWPressureEvent {
   const uint64_t ResourceMask;
 };
 
-class HWEventListener {
+class LLVM_ABI HWEventListener {
 public:
   // Generic events generated by the pipeline.
   virtual void onCycleBegin() {}
diff --git a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/HardwareUnit.h b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/HardwareUnit.h
index 85350e4eb7f0..9d0e3749d5cf 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/HardwareUnit.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/HardwareUnit.h
@@ -15,10 +15,12 @@
 #ifndef LLVM_MCA_HARDWAREUNITS_HARDWAREUNIT_H
 #define LLVM_MCA_HARDWAREUNITS_HARDWAREUNIT_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 namespace mca {
 
-class HardwareUnit {
+class LLVM_ABI HardwareUnit {
   HardwareUnit(const HardwareUnit &H) = delete;
   HardwareUnit &operator=(const HardwareUnit &H) = delete;
 
diff --git a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
index 42bab5f93fe6..3700901d723c 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
@@ -20,12 +20,13 @@
 #include "llvm/MC/MCSchedule.h"
 #include "llvm/MCA/HardwareUnits/HardwareUnit.h"
 #include "llvm/MCA/Instruction.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
 
 /// Abstract base interface for LS (load/store) units in llvm-mca.
-class LSUnitBase : public HardwareUnit {
+class LLVM_ABI LSUnitBase : public HardwareUnit {
   /// Load queue size.
   ///
   /// A value of zero for this field means that the load queue is unbounded.
@@ -192,7 +193,7 @@ class LSUnitBase : public HardwareUnit {
 /// A load/store barrier is "executed" when it becomes the oldest entry in
 /// the load/store queue(s). That also means, all the older loads/stores have
 /// already been executed.
-class LSUnit : public LSUnitBase {
+class LLVM_ABI LSUnit : public LSUnitBase {
 
   // This class doesn't know about the latency of a load instruction. So, it
   // conservatively/pessimistically assumes that the latency of a load opcode
diff --git a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h
index f71928f835c0..d88ee7cbb958 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h
@@ -20,6 +20,7 @@
 #include "llvm/MC/MCSchedule.h"
 #include "llvm/MCA/Instruction.h"
 #include "llvm/MCA/Support.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
@@ -44,7 +45,7 @@ enum ResourceStateEvent {
 };
 
 /// Resource allocation strategy used by hardware scheduler resources.
-class ResourceStrategy {
+class LLVM_ABI ResourceStrategy {
   ResourceStrategy(const ResourceStrategy &) = delete;
   ResourceStrategy &operator=(const ResourceStrategy &) = delete;
 
@@ -64,7 +65,7 @@ class ResourceStrategy {
 
 /// Default resource allocation strategy used by processor resource groups and
 /// processor resources with multiple units.
-class DefaultResourceStrategy final : public ResourceStrategy {
+class LLVM_ABI DefaultResourceStrategy final : public ResourceStrategy {
   /// A Mask of resource unit identifiers.
   ///
   /// There is one bit set for every available resource unit.
@@ -211,7 +212,8 @@ class ResourceState {
   }
 
 public:
-  ResourceState(const MCProcResourceDesc &Desc, unsigned Index, uint64_t Mask);
+  LLVM_ABI ResourceState(const MCProcResourceDesc &Desc, unsigned Index,
+                         uint64_t Mask);
 
   unsigned getProcResourceID() const { return ProcResourceDescIndex; }
   uint64_t getResourceMask() const { return ResourceMask; }
@@ -230,7 +232,7 @@ class ResourceState {
 
   /// Returs true if this resource is not reserved, and if there are at least
   /// `NumUnits` available units.
-  bool isReady(unsigned NumUnits = 1) const;
+  LLVM_ABI bool isReady(unsigned NumUnits = 1) const;
 
   uint64_t getNumReadyUnits() const { return llvm::popcount(ReadyMask); }
 
@@ -261,7 +263,7 @@ class ResourceState {
   /// is reserved.
   ///
   /// Returns RS_BUFFER_UNAVAILABLE if there are no available slots.
-  ResourceStateEvent isBufferAvailable() const;
+  LLVM_ABI ResourceStateEvent isBufferAvailable() const;
 
   /// Reserve a buffer slot.
   ///
@@ -381,11 +383,11 @@ class ResourceManager {
 
   // Overrides the selection strategy for the processor resource with the given
   // mask.
-  void setCustomStrategyImpl(std::unique_ptr<ResourceStrategy> S,
-                             uint64_t ResourceMask);
+  LLVM_ABI void setCustomStrategyImpl(std::unique_ptr<ResourceStrategy> S,
+                                      uint64_t ResourceMask);
 
 public:
-  ResourceManager(const MCSchedModel &SM);
+  LLVM_ABI ResourceManager(const MCSchedModel &SM);
   virtual ~ResourceManager() = default;
 
   // Overrides the selection strategy for the resource at index ResourceID in
@@ -399,33 +401,33 @@ class ResourceManager {
 
   // Returns RS_BUFFER_AVAILABLE if buffered resources are not reserved, and if
   // there are enough available slots in the buffers.
-  ResourceStateEvent canBeDispatched(uint64_t ConsumedBuffers) const;
+  LLVM_ABI ResourceStateEvent canBeDispatched(uint64_t ConsumedBuffers) const;
 
   // Return the processor resource identifier associated to this Mask.
-  unsigned resolveResourceMask(uint64_t Mask) const;
+  LLVM_ABI unsigned resolveResourceMask(uint64_t Mask) const;
 
   // Acquires a slot from every buffered resource in mask `ConsumedBuffers`.
   // Units that are dispatch hazards (i.e. BufferSize=0) are marked as reserved.
-  void reserveBuffers(uint64_t ConsumedBuffers);
+  LLVM_ABI void reserveBuffers(uint64_t ConsumedBuffers);
 
   // Releases a slot from every buffered resource in mask `ConsumedBuffers`.
   // ConsumedBuffers is a bitmask of previously acquired buffers (using method
   // `reserveBuffers`). Units that are dispatch hazards (i.e. BufferSize=0) are
   // not automatically unreserved by this method.
-  void releaseBuffers(uint64_t ConsumedBuffers);
+  LLVM_ABI void releaseBuffers(uint64_t ConsumedBuffers);
 
   // Reserve a processor resource. A reserved resource is not available for
   // instruction issue until it is released.
-  void reserveResource(uint64_t ResourceID);
+  LLVM_ABI void reserveResource(uint64_t ResourceID);
 
   // Release a previously reserved processor resource.
-  void releaseResource(uint64_t ResourceID);
+  LLVM_ABI void releaseResource(uint64_t ResourceID);
 
   // Returns a zero mask if resources requested by Desc are all available during
   // this cycle. It returns a non-zero mask value only if there are unavailable
   // processor resources; each bit set in the mask represents a busy processor
   // resource unit or a reserved processor resource group.
-  uint64_t checkAvailability(const InstrDesc &Desc) const;
+  LLVM_ABI uint64_t checkAvailability(const InstrDesc &Desc) const;
 
   uint64_t getProcResUnitMask() const { return ProcResUnitMask; }
   uint64_t getAvailableProcResUnits() const { return AvailableProcResUnits; }
@@ -446,18 +448,20 @@ class ResourceManager {
   // schedule, no matter in which order individual uses are processed. For that
   // reason, the vector of resource uses is simply (and quickly) processed in
   // sequence. The resulting schedule is eventually stored into vector `Pipes`.
-  void fastIssueInstruction(const InstrDesc &Desc,
-                            SmallVectorImpl<ResourceWithCycles> &Pipes);
+  LLVM_ABI void
+  fastIssueInstruction(const InstrDesc &Desc,
+                       SmallVectorImpl<ResourceWithCycles> &Pipes);
 
   // Selects pipeline resources consumed by an instruction.
   // This method works under the assumption that used resource groups may
   // partially overlap. This complicates the selection process, because the
   // order in which uses are processed matters. The logic internally prioritizes
   // groups which are more constrained than others.
-  void issueInstructionImpl(const InstrDesc &Desc,
-                            SmallVectorImpl<ResourceWithCycles> &Pipes);
+  LLVM_ABI void
+  issueInstructionImpl(const InstrDesc &Desc,
+                       SmallVectorImpl<ResourceWithCycles> &Pipes);
 
-  void cycleEvent(SmallVectorImpl<ResourceRef> &ResourcesFreed);
+  LLVM_ABI void cycleEvent(SmallVectorImpl<ResourceRef> &ResourcesFreed);
 
 #ifndef NDEBUG
   void dump() const {
diff --git a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h
index 272f6b197868..0372600e9963 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h
@@ -20,11 +20,12 @@
 #include "llvm/MCA/HardwareUnits/LSUnit.h"
 #include "llvm/MCA/HardwareUnits/ResourceManager.h"
 #include "llvm/MCA/Support.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
 
-class SchedulerStrategy {
+class LLVM_ABI SchedulerStrategy {
 public:
   SchedulerStrategy() = default;
   virtual ~SchedulerStrategy();
@@ -37,7 +38,7 @@ class SchedulerStrategy {
 };
 
 /// Default instruction selection strategy used by class Scheduler.
-class DefaultSchedulerStrategy : public SchedulerStrategy {
+class LLVM_ABI DefaultSchedulerStrategy : public SchedulerStrategy {
   /// This method ranks instructions based on their age, and the number of known
   /// users. The lower the rank value, the better.
   int computeRank(const InstRef &Lhs) const {
@@ -131,7 +132,7 @@ class Scheduler : public HardwareUnit {
   /// Verify the given selection strategy and set the Strategy member
   /// accordingly.  If no strategy is provided, the DefaultSchedulerStrategy is
   /// used.
-  void initializeStrategy(std::unique_ptr<SchedulerStrategy> S);
+  LLVM_ABI void initializeStrategy(std::unique_ptr<SchedulerStrategy> S);
 
   /// Issue an instruction without updating the ready queue.
   void issueInstructionImpl(
@@ -183,7 +184,7 @@ class Scheduler : public HardwareUnit {
   ///
   /// This method is also responsible for setting field HadTokenStall if
   /// IR cannot be dispatched to the Scheduler due to unavailable resources.
-  Status isAvailable(const InstRef &IR);
+  LLVM_ABI Status isAvailable(const InstRef &IR);
 
   /// Reserves buffer and LSUnit queue resources that are necessary to issue
   /// this instruction.
@@ -195,20 +196,19 @@ class Scheduler : public HardwareUnit {
   /// If IR is a memory operation, then the Scheduler queries the LS unit to
   /// obtain a LS token. An LS token is used internally to track memory
   /// dependencies.
-  bool dispatch(InstRef &IR);
+  LLVM_ABI bool dispatch(InstRef &IR);
 
   /// Issue an instruction and populates a vector of used pipeline resources,
   /// and a vector of instructions that transitioned to the ready state as a
   /// result of this event.
-  void issueInstruction(
+  LLVM_ABI void issueInstruction(
       InstRef &IR,
       SmallVectorImpl<std::pair<ResourceRef, ReleaseAtCycles>> &Used,
-      SmallVectorImpl<InstRef> &Pending,
-      SmallVectorImpl<InstRef> &Ready);
+      SmallVectorImpl<InstRef> &Pending, SmallVectorImpl<InstRef> &Ready);
 
   /// Returns true if IR has to be issued immediately, or if IR is a zero
   /// latency instruction.
-  bool mustIssueImmediately(const InstRef &IR) const;
+  LLVM_ABI bool mustIssueImmediately(const InstRef &IR) const;
 
   /// This routine notifies the Scheduler that a new cycle just started.
   ///
@@ -222,10 +222,10 @@ class Scheduler : public HardwareUnit {
   /// disjoint. An instruction is allowed to transition from the WAIT state to
   /// the READY state (going through the PENDING state) within a single cycle.
   /// That means, instructions may appear in both the Pending and Ready set.
-  void cycleEvent(SmallVectorImpl<ResourceRef> &Freed,
-                  SmallVectorImpl<InstRef> &Executed,
-                  SmallVectorImpl<InstRef> &Pending,
-                  SmallVectorImpl<InstRef> &Ready);
+  LLVM_ABI void cycleEvent(SmallVectorImpl<ResourceRef> &Freed,
+                           SmallVectorImpl<InstRef> &Executed,
+                           SmallVectorImpl<InstRef> &Pending,
+                           SmallVectorImpl<InstRef> &Ready);
 
   /// Convert a resource mask into a valid llvm processor resource identifier.
   ///
@@ -238,7 +238,7 @@ class Scheduler : public HardwareUnit {
   /// Select the next instruction to issue from the ReadySet. Returns an invalid
   /// instruction reference if there are no ready instructions, or if processor
   /// resources are not available.
-  InstRef select();
+  LLVM_ABI InstRef select();
 
   bool isReadySetEmpty() const { return ReadySet.empty(); }
   bool isWaitSetEmpty() const { return WaitSet.empty(); }
@@ -248,13 +248,13 @@ class Scheduler : public HardwareUnit {
   /// populated by instructions that were not issued because of unsolved
   /// register dependencies.  Vector MemDeps is populated by instructions that
   /// were not issued because of unsolved memory dependencies.
-  void analyzeDataDependencies(SmallVectorImpl<InstRef> &RegDeps,
-                               SmallVectorImpl<InstRef> &MemDeps);
+  LLVM_ABI void analyzeDataDependencies(SmallVectorImpl<InstRef> &RegDeps,
+                                        SmallVectorImpl<InstRef> &MemDeps);
 
   /// Returns a mask of busy resources, and populates vector Insts with
   /// instructions that could not be issued to the underlying pipelines because
   /// not all pipeline resources were available.
-  uint64_t analyzeResourcePressure(SmallVectorImpl<InstRef> &Insts);
+  LLVM_ABI uint64_t analyzeResourcePressure(SmallVectorImpl<InstRef> &Insts);
 
   // Returns true if the dispatch logic couldn't dispatch a full group due to
   // unavailable scheduler and/or LS resources.
diff --git a/external/llvm-project/llvm/include/llvm/MCA/IncrementalSourceMgr.h b/external/llvm-project/llvm/include/llvm/MCA/IncrementalSourceMgr.h
index 81f9b51cf42f..a28663de5384 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/IncrementalSourceMgr.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/IncrementalSourceMgr.h
@@ -15,6 +15,7 @@
 #define LLVM_MCA_INCREMENTALSOURCEMGR_H
 
 #include "llvm/MCA/SourceMgr.h"
+#include "llvm/Support/Compiler.h"
 #include <deque>
 
 namespace llvm {
@@ -23,7 +24,7 @@ namespace mca {
 /// An implementation of \a SourceMgr that allows users to add new instructions
 /// incrementally / dynamically.
 /// Note that this SourceMgr takes ownership of all \a mca::Instruction.
-class IncrementalSourceMgr : public SourceMgr {
+class LLVM_ABI IncrementalSourceMgr : public SourceMgr {
   /// Owner of all mca::Instruction instances. Note that we use std::deque here
   /// to have a better throughput, in comparison to std::vector or
   /// llvm::SmallVector, as they usually pay a higher re-allocation cost when
@@ -47,6 +48,10 @@ class IncrementalSourceMgr : public SourceMgr {
 public:
   IncrementalSourceMgr() = default;
 
+  // Explicitly non-copyable.
+  IncrementalSourceMgr &operator=(const IncrementalSourceMgr &) = delete;
+  IncrementalSourceMgr(const IncrementalSourceMgr &) = delete;
+
   void clear();
 
   /// Set a callback that is invoked when a mca::Instruction is
diff --git a/external/llvm-project/llvm/include/llvm/MCA/InstrBuilder.h b/external/llvm-project/llvm/include/llvm/MCA/InstrBuilder.h
index 00b19a65dc16..e0949d975fa9 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/InstrBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/InstrBuilder.h
@@ -23,6 +23,7 @@
 #include "llvm/MCA/CustomBehaviour.h"
 #include "llvm/MCA/Instruction.h"
 #include "llvm/MCA/Support.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -32,7 +33,7 @@ class RecycledInstErr : public ErrorInfo<RecycledInstErr> {
   Instruction *RecycledInst;
 
 public:
-  static char ID;
+  LLVM_ABI static char ID;
 
   explicit RecycledInstErr(Instruction *Inst) : RecycledInst(Inst) {}
   // Always need to carry an Instruction
@@ -99,9 +100,9 @@ class InstrBuilder {
   Error verifyInstrDesc(const InstrDesc &ID, const MCInst &MCI) const;
 
 public:
-  InstrBuilder(const MCSubtargetInfo &STI, const MCInstrInfo &MCII,
-               const MCRegisterInfo &RI, const MCInstrAnalysis *IA,
-               const InstrumentManager &IM, unsigned CallLatency);
+  LLVM_ABI InstrBuilder(const MCSubtargetInfo &STI, const MCInstrInfo &MCII,
+                        const MCRegisterInfo &RI, const MCInstrAnalysis *IA,
+                        const InstrumentManager &IM, unsigned CallLatency);
 
   void clear() {
     Descriptors.clear();
@@ -114,7 +115,7 @@ class InstrBuilder {
   /// or null if there isn't any.
   void setInstRecycleCallback(InstRecycleCallback CB) { InstRecycleCB = CB; }
 
-  Expected<std::unique_ptr<Instruction>>
+  LLVM_ABI Expected<std::unique_ptr<Instruction>>
   createInstruction(const MCInst &MCI, const SmallVector<Instrument *> &IVec);
 };
 } // namespace mca
diff --git a/external/llvm-project/llvm/include/llvm/MCA/Instruction.h b/external/llvm-project/llvm/include/llvm/MCA/Instruction.h
index e48a70164bec..4c683aa3421d 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/Instruction.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/Instruction.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCRegister.h" // definition of MCPhysReg.
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 
 #ifndef NDEBUG
@@ -271,11 +272,11 @@ class WriteState {
   // instruction identifier associated with this write. ReadAdvance is the
   // number of cycles to subtract from the latency of this data dependency.
   // Use is in a RAW dependency with this write.
-  void addUser(unsigned IID, ReadState *Use, int ReadAdvance);
+  LLVM_ABI void addUser(unsigned IID, ReadState *Use, int ReadAdvance);
 
   // Use is a younger register write that is in a false dependency with this
   // write. IID is the instruction identifier associated with this write.
-  void addUser(unsigned IID, WriteState *Use);
+  LLVM_ABI void addUser(unsigned IID, WriteState *Use);
 
   unsigned getNumUsers() const {
     unsigned NumUsers = Users.size();
@@ -300,7 +301,7 @@ class WriteState {
   }
 
   void setDependentWrite(const WriteState *Other) { DependentWrite = Other; }
-  void writeStartEvent(unsigned IID, MCPhysReg RegID, unsigned Cycles);
+  LLVM_ABI void writeStartEvent(unsigned IID, MCPhysReg RegID, unsigned Cycles);
   void setWriteZero() { WritesZero = true; }
   void setEliminated() {
     assert(Users.empty() && "Write is in an inconsistent state.");
@@ -311,8 +312,8 @@ class WriteState {
   void setPRF(unsigned PRF) { PRFID = PRF; }
 
   // On every cycle, update CyclesLeft and notify dependent users.
-  void cycleEvent();
-  void onInstructionIssued(unsigned IID);
+  LLVM_ABI void cycleEvent();
+  LLVM_ABI void onInstructionIssued(unsigned IID);
 
 #ifndef NDEBUG
   void dump() const;
@@ -371,8 +372,8 @@ class ReadState {
   bool isIndependentFromDef() const { return IndependentFromDef; }
   void setIndependentFromDef() { IndependentFromDef = true; }
 
-  void cycleEvent();
-  void writeStartEvent(unsigned IID, MCPhysReg RegID, unsigned Cycles);
+  LLVM_ABI void cycleEvent();
+  LLVM_ABI void writeStartEvent(unsigned IID, MCPhysReg RegID, unsigned Cycles);
   void setDependentWrites(unsigned Writes) {
     DependentWrites = Writes;
     IsReady = !Writes;
@@ -650,7 +651,7 @@ class Instruction : public InstructionBase {
         UsedBuffers(D.UsedBuffers), CriticalRegDep(), CriticalMemDep(),
         CriticalResourceMask(0), IsEliminated(false) {}
 
-  void reset();
+  LLVM_ABI void reset();
 
   unsigned getRCUTokenID() const { return RCUTokenID; }
   unsigned getLSUTokenID() const { return LSUTokenID; }
@@ -665,11 +666,11 @@ class Instruction : public InstructionBase {
   // Transition to the dispatch stage, and assign a RCUToken to this
   // instruction. The RCUToken is used to track the completion of every
   // register write performed by this instruction.
-  void dispatch(unsigned RCUTokenID);
+  LLVM_ABI void dispatch(unsigned RCUTokenID);
 
   // Instruction issued. Transition to the IS_EXECUTING state, and update
   // all the register definitions.
-  void execute(unsigned IID);
+  LLVM_ABI void execute(unsigned IID);
 
   // Force a transition from the IS_DISPATCHED state to the IS_READY or
   // IS_PENDING state. State transitions normally occur either at the beginning
@@ -677,9 +678,9 @@ class Instruction : public InstructionBase {
   // event. This method is called every time the instruction might have changed
   // in state. It internally delegates to method updateDispatched() and
   // updateWaiting().
-  void update();
-  bool updateDispatched();
-  bool updatePending();
+  LLVM_ABI void update();
+  LLVM_ABI bool updateDispatched();
+  LLVM_ABI bool updatePending();
 
   bool isInvalid() const { return Stage == IS_INVALID; }
   bool isDispatched() const { return Stage == IS_DISPATCHED; }
@@ -691,7 +692,7 @@ class Instruction : public InstructionBase {
   bool isEliminated() const { return IsEliminated; }
 
   // Forces a transition from state IS_DISPATCHED to state IS_EXECUTED.
-  void forceExecuted();
+  LLVM_ABI void forceExecuted();
   void setEliminated() { IsEliminated = true; }
 
   void retire() {
@@ -701,7 +702,7 @@ class Instruction : public InstructionBase {
 
   const CriticalDependency &getCriticalRegDep() const { return CriticalRegDep; }
   const CriticalDependency &getCriticalMemDep() const { return CriticalMemDep; }
-  const CriticalDependency &computeCriticalRegDep();
+  LLVM_ABI const CriticalDependency &computeCriticalRegDep();
   void setCriticalMemDep(const CriticalDependency &MemDep) {
     CriticalMemDep = MemDep;
   }
@@ -711,7 +712,7 @@ class Instruction : public InstructionBase {
     CriticalResourceMask = ResourceMask;
   }
 
-  void cycleEvent();
+  LLVM_ABI void cycleEvent();
 };
 
 /// An InstRef contains both a SourceMgr index and Instruction pair.  The index
diff --git a/external/llvm-project/llvm/include/llvm/MCA/Pipeline.h b/external/llvm-project/llvm/include/llvm/MCA/Pipeline.h
index 18032fdfe012..1fbc56d86a98 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/Pipeline.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/Pipeline.h
@@ -16,6 +16,7 @@
 #define LLVM_MCA_PIPELINE_H
 
 #include "llvm/MCA/Stages/Stage.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -70,12 +71,12 @@ class Pipeline {
 
 public:
   Pipeline() = default;
-  void appendStage(std::unique_ptr<Stage> S);
+  LLVM_ABI void appendStage(std::unique_ptr<Stage> S);
 
   /// Returns the total number of simulated cycles.
-  Expected<unsigned> run();
+  LLVM_ABI Expected<unsigned> run();
 
-  void addEventListener(HWEventListener *Listener);
+  LLVM_ABI void addEventListener(HWEventListener *Listener);
 
   /// Returns whether the pipeline is currently paused.
   bool isPaused() const { return CurrentState == State::Paused; }
diff --git a/external/llvm-project/llvm/include/llvm/MCA/Stages/EntryStage.h b/external/llvm-project/llvm/include/llvm/MCA/Stages/EntryStage.h
index fb1244aa1933..262863b6996c 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/Stages/EntryStage.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/Stages/EntryStage.h
@@ -19,11 +19,12 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MCA/SourceMgr.h"
 #include "llvm/MCA/Stages/Stage.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
 
-class EntryStage final : public Stage {
+class LLVM_ABI EntryStage final : public Stage {
   InstRef CurrentInstruction;
   SmallVector<std::unique_ptr<Instruction>, 16> Instructions;
   SourceMgr &SM;
diff --git a/external/llvm-project/llvm/include/llvm/MCA/Stages/InstructionTables.h b/external/llvm-project/llvm/include/llvm/MCA/Stages/InstructionTables.h
index 9617fd49db6e..7a96e82dd995 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/Stages/InstructionTables.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/Stages/InstructionTables.h
@@ -21,11 +21,12 @@
 #include "llvm/MCA/HardwareUnits/Scheduler.h"
 #include "llvm/MCA/Stages/Stage.h"
 #include "llvm/MCA/Support.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace mca {
 
-class InstructionTables final : public Stage {
+class LLVM_ABI InstructionTables final : public Stage {
   const MCSchedModel &SM;
   SmallVector<ResourceUse, 4> UsedResources;
   SmallVector<uint64_t, 8> Masks;
diff --git a/external/llvm-project/llvm/include/llvm/MCA/Stages/Stage.h b/external/llvm-project/llvm/include/llvm/MCA/Stages/Stage.h
index a1fc7a8af7ad..69a6b382032b 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/Stages/Stage.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/Stages/Stage.h
@@ -16,6 +16,7 @@
 #define LLVM_MCA_STAGES_STAGE_H
 
 #include "llvm/MCA/HWEventListener.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <set>
 
@@ -24,7 +25,7 @@ namespace mca {
 
 class InstRef;
 
-class Stage {
+class LLVM_ABI Stage {
   Stage *NextInSequence = nullptr;
   std::set<HWEventListener *> Listeners;
 
@@ -88,7 +89,7 @@ class Stage {
 /// This is actually not an error but a marker to indicate that
 /// the instruction stream is paused.
 struct InstStreamPause : public ErrorInfo<InstStreamPause> {
-  static char ID;
+  LLVM_ABI static char ID;
 
   std::error_code convertToErrorCode() const override {
     return llvm::inconvertibleErrorCode();
diff --git a/external/llvm-project/llvm/include/llvm/MCA/Support.h b/external/llvm-project/llvm/include/llvm/MCA/Support.h
index e3c155435e23..ce2ac9b4b6cd 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/Support.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/Support.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCSchedule.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MathExtras.h"
 
@@ -67,7 +68,7 @@ class ReleaseAtCycles {
   // Add the components of RHS to this instance.  Instead of calculating
   // the final value here, we keep track of the numerator and denominator
   // separately, to reduce floating point error.
-  ReleaseAtCycles &operator+=(const ReleaseAtCycles &RHS);
+  LLVM_ABI ReleaseAtCycles &operator+=(const ReleaseAtCycles &RHS);
 };
 
 /// Populates vector Masks with processor resource masks.
@@ -92,8 +93,8 @@ class ReleaseAtCycles {
 ///
 /// Resource masks are used by the ResourceManager to solve set membership
 /// problems with simple bit manipulation operations.
-void computeProcResourceMasks(const MCSchedModel &SM,
-                              MutableArrayRef<uint64_t> Masks);
+LLVM_ABI void computeProcResourceMasks(const MCSchedModel &SM,
+                                       MutableArrayRef<uint64_t> Masks);
 
 // Returns the index of the highest bit set. For resource masks, the position of
 // the highest bit set can be used to construct a resource mask identifier.
@@ -106,9 +107,10 @@ inline unsigned getResourceStateIndex(uint64_t Mask) {
 /// cycles. The reciprocal block throughput is computed as the MAX between:
 ///  - NumMicroOps / DispatchWidth
 ///  - ProcReleaseAtCycles / #ProcResourceUnits  (for every consumed resource).
-double computeBlockRThroughput(const MCSchedModel &SM, unsigned DispatchWidth,
-                               unsigned NumMicroOps,
-                               ArrayRef<unsigned> ProcResourceUsage);
+LLVM_ABI double computeBlockRThroughput(const MCSchedModel &SM,
+                                        unsigned DispatchWidth,
+                                        unsigned NumMicroOps,
+                                        ArrayRef<unsigned> ProcResourceUsage);
 } // namespace mca
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/MCA/View.h b/external/llvm-project/llvm/include/llvm/MCA/View.h
index ff8fc1ceb3f1..4d6f93066320 100644
--- a/external/llvm-project/llvm/include/llvm/MCA/View.h
+++ b/external/llvm-project/llvm/include/llvm/MCA/View.h
@@ -17,13 +17,14 @@
 
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MCA/HWEventListener.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 namespace mca {
 
-class View : public HWEventListener {
+class LLVM_ABI View : public HWEventListener {
 public:
   virtual ~View() = default;
 
diff --git a/external/llvm-project/llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h b/external/llvm-project/llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h
index d9043d6c5d01..2ec1ca961411 100644
--- a/external/llvm-project/llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h
+++ b/external/llvm-project/llvm/include/llvm/ObjCopy/COFF/COFFObjcopy.h
@@ -9,6 +9,8 @@
 #ifndef LLVM_OBJCOPY_COFF_COFFOBJCOPY_H
 #define LLVM_OBJCOPY_COFF_COFFOBJCOPY_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 class Error;
 class raw_ostream;
@@ -26,8 +28,10 @@ namespace coff {
 /// Apply the transformations described by \p Config and \p COFFConfig
 /// to \p In and writes the result into \p Out.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnBinary(const CommonConfig &Config, const COFFConfig &,
-                             object::COFFObjectFile &In, raw_ostream &Out);
+LLVM_ABI Error executeObjcopyOnBinary(const CommonConfig &Config,
+                                      const COFFConfig &,
+                                      object::COFFObjectFile &In,
+                                      raw_ostream &Out);
 
 } // end namespace coff
 } // end namespace objcopy
diff --git a/external/llvm-project/llvm/include/llvm/ObjCopy/CommonConfig.h b/external/llvm-project/llvm/include/llvm/ObjCopy/CommonConfig.h
index aea9cd6f9a9c..faa7b0db757a 100644
--- a/external/llvm-project/llvm/include/llvm/ObjCopy/CommonConfig.h
+++ b/external/llvm-project/llvm/include/llvm/ObjCopy/CommonConfig.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/ELFTypes.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/GlobPattern.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -104,7 +105,7 @@ class NameOrPattern {
 public:
   // ErrorCallback is used to handle recoverable errors. An Error returned
   // by the callback aborts the parsing and is then returned by this function.
-  static Expected<NameOrPattern>
+  LLVM_ABI static Expected<NameOrPattern>
   create(StringRef Pattern, MatchStyle MS,
          llvm::function_ref<Error(Error)> ErrorCallback);
 
diff --git a/external/llvm-project/llvm/include/llvm/ObjCopy/ConfigManager.h b/external/llvm-project/llvm/include/llvm/ObjCopy/ConfigManager.h
index 2962cf99b270..e7b3775f0e9f 100644
--- a/external/llvm-project/llvm/include/llvm/ObjCopy/ConfigManager.h
+++ b/external/llvm-project/llvm/include/llvm/ObjCopy/ConfigManager.h
@@ -14,13 +14,14 @@
 #include "llvm/ObjCopy/ELF/ELFConfig.h"
 #include "llvm/ObjCopy/MachO/MachOConfig.h"
 #include "llvm/ObjCopy/MultiFormatConfig.h"
-#include "llvm/ObjCopy/wasm/WasmConfig.h"
 #include "llvm/ObjCopy/XCOFF/XCOFFConfig.h"
+#include "llvm/ObjCopy/wasm/WasmConfig.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace objcopy {
 
-struct ConfigManager : public MultiFormatConfig {
+struct LLVM_ABI ConfigManager : public MultiFormatConfig {
   virtual ~ConfigManager() {}
 
   const CommonConfig &getCommonConfig() const override { return Common; }
diff --git a/external/llvm-project/llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h b/external/llvm-project/llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h
index 552b6fb655f1..05fe2828a86f 100644
--- a/external/llvm-project/llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h
+++ b/external/llvm-project/llvm/include/llvm/ObjCopy/ELF/ELFObjcopy.h
@@ -9,6 +9,8 @@
 #ifndef LLVM_OBJCOPY_ELF_ELFOBJCOPY_H
 #define LLVM_OBJCOPY_ELF_ELFOBJCOPY_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 class Error;
 class MemoryBuffer;
@@ -27,24 +29,25 @@ namespace elf {
 /// \p In, which must represent an IHex file, and writes the result
 /// into \p Out.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnIHex(const CommonConfig &Config,
-                           const ELFConfig &ELFConfig, MemoryBuffer &In,
-                           raw_ostream &Out);
+LLVM_ABI Error executeObjcopyOnIHex(const CommonConfig &Config,
+                                    const ELFConfig &ELFConfig,
+                                    MemoryBuffer &In, raw_ostream &Out);
 
 /// Apply the transformations described by \p Config and \p ELFConfig to
 /// \p In, which is treated as a raw binary input, and writes the result
 /// into \p Out.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnRawBinary(const CommonConfig &Config,
-                                const ELFConfig &ELFConfig, MemoryBuffer &In,
-                                raw_ostream &Out);
+LLVM_ABI Error executeObjcopyOnRawBinary(const CommonConfig &Config,
+                                         const ELFConfig &ELFConfig,
+                                         MemoryBuffer &In, raw_ostream &Out);
 
 /// Apply the transformations described by \p Config and \p ELFConfig to
 /// \p In and writes the result into \p Out.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnBinary(const CommonConfig &Config,
-                             const ELFConfig &ELFConfig,
-                             object::ELFObjectFileBase &In, raw_ostream &Out);
+LLVM_ABI Error executeObjcopyOnBinary(const CommonConfig &Config,
+                                      const ELFConfig &ELFConfig,
+                                      object::ELFObjectFileBase &In,
+                                      raw_ostream &Out);
 
 } // end namespace elf
 } // end namespace objcopy
diff --git a/external/llvm-project/llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h b/external/llvm-project/llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h
index 73690d7ace8a..1efe61a819fd 100644
--- a/external/llvm-project/llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h
+++ b/external/llvm-project/llvm/include/llvm/ObjCopy/MachO/MachOObjcopy.h
@@ -9,6 +9,8 @@
 #ifndef LLVM_OBJCOPY_MACHO_MACHOOBJCOPY_H
 #define LLVM_OBJCOPY_MACHO_MACHOOBJCOPY_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 class Error;
 class raw_ostream;
@@ -27,14 +29,15 @@ namespace macho {
 /// Apply the transformations described by \p Config and \p MachOConfig to
 /// \p In and writes the result into \p Out.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnBinary(const CommonConfig &Config,
-                             const MachOConfig &MachOConfig,
-                             object::MachOObjectFile &In, raw_ostream &Out);
+LLVM_ABI Error executeObjcopyOnBinary(const CommonConfig &Config,
+                                      const MachOConfig &MachOConfig,
+                                      object::MachOObjectFile &In,
+                                      raw_ostream &Out);
 
 /// Apply the transformations described by \p Config and \p MachOConfig to
 /// \p In and writes the result into \p Out.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnMachOUniversalBinary(
+LLVM_ABI Error executeObjcopyOnMachOUniversalBinary(
     const MultiFormatConfig &Config, const object::MachOUniversalBinary &In,
     raw_ostream &Out);
 
diff --git a/external/llvm-project/llvm/include/llvm/ObjCopy/ObjCopy.h b/external/llvm-project/llvm/include/llvm/ObjCopy/ObjCopy.h
index 023814002c72..9554930e2097 100644
--- a/external/llvm-project/llvm/include/llvm/ObjCopy/ObjCopy.h
+++ b/external/llvm-project/llvm/include/llvm/ObjCopy/ObjCopy.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_OBJCOPY_OBJCOPY_H
 #define LLVM_OBJCOPY_OBJCOPY_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -26,15 +27,15 @@ class MultiFormatConfig;
 /// each member in archive \p Ar.
 /// Writes a result in a file specified by \p Config.OutputFilename.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnArchive(const MultiFormatConfig &Config,
-                              const object::Archive &Ar);
+LLVM_ABI Error executeObjcopyOnArchive(const MultiFormatConfig &Config,
+                                       const object::Archive &Ar);
 
 /// Applies the transformations described by \p Config to \p In and writes
 /// the result into \p Out. This function does the dispatch based on the
 /// format of the input binary (COFF, ELF, MachO or wasm).
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnBinary(const MultiFormatConfig &Config,
-                             object::Binary &In, raw_ostream &Out);
+LLVM_ABI Error executeObjcopyOnBinary(const MultiFormatConfig &Config,
+                                      object::Binary &In, raw_ostream &Out);
 
 } // end namespace objcopy
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h b/external/llvm-project/llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h
index 5b4181c22b97..da8ce374dc40 100644
--- a/external/llvm-project/llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h
+++ b/external/llvm-project/llvm/include/llvm/ObjCopy/wasm/WasmObjcopy.h
@@ -9,6 +9,8 @@
 #ifndef LLVM_OBJCOPY_WASM_WASMOBJCOPY_H
 #define LLVM_OBJCOPY_WASM_WASMOBJCOPY_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 class Error;
 class raw_ostream;
@@ -25,8 +27,10 @@ namespace wasm {
 /// Apply the transformations described by \p Config and \p WasmConfig
 /// to \p In and writes the result into \p Out.
 /// \returns any Error encountered whilst performing the operation.
-Error executeObjcopyOnBinary(const CommonConfig &Config, const WasmConfig &,
-                             object::WasmObjectFile &In, raw_ostream &Out);
+LLVM_ABI Error executeObjcopyOnBinary(const CommonConfig &Config,
+                                      const WasmConfig &,
+                                      object::WasmObjectFile &In,
+                                      raw_ostream &Out);
 
 } // end namespace wasm
 } // end namespace objcopy
diff --git a/external/llvm-project/llvm/include/llvm/Object/Archive.h b/external/llvm-project/llvm/include/llvm/Object/Archive.h
index b2dd970b00c0..9a4311993acd 100644
--- a/external/llvm-project/llvm/include/llvm/Object/Archive.h
+++ b/external/llvm-project/llvm/include/llvm/Object/Archive.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Support/Chrono.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -61,10 +62,11 @@ class AbstractArchiveMemberHeader {
   virtual Expected<const char *> getNextChildLoc() const = 0;
   virtual Expected<bool> isThin() const = 0;
 
-  Expected<sys::fs::perms> getAccessMode() const;
-  Expected<sys::TimePoint<std::chrono::seconds>> getLastModified() const;
-  Expected<unsigned> getUID() const;
-  Expected<unsigned> getGID() const;
+  LLVM_ABI Expected<sys::fs::perms> getAccessMode() const;
+  LLVM_ABI Expected<sys::TimePoint<std::chrono::seconds>>
+  getLastModified() const;
+  LLVM_ABI Expected<unsigned> getUID() const;
+  LLVM_ABI Expected<unsigned> getGID() const;
 
   /// Returns the size in bytes of the format-defined member header of the
   /// concrete archive type.
@@ -74,7 +76,7 @@ class AbstractArchiveMemberHeader {
 };
 
 template <typename T>
-class CommonArchiveMemberHeader : public AbstractArchiveMemberHeader {
+class LLVM_ABI CommonArchiveMemberHeader : public AbstractArchiveMemberHeader {
 public:
   CommonArchiveMemberHeader(const Archive *Parent, const T *RawHeaderPtr)
       : AbstractArchiveMemberHeader(Parent), ArMemHdr(RawHeaderPtr){};
@@ -99,7 +101,8 @@ struct UnixArMemHdrType {
   char Terminator[2];
 };
 
-class ArchiveMemberHeader : public CommonArchiveMemberHeader<UnixArMemHdrType> {
+class LLVM_ABI ArchiveMemberHeader
+    : public CommonArchiveMemberHeader<UnixArMemHdrType> {
 public:
   ArchiveMemberHeader(const Archive *Parent, const char *RawHeaderPtr,
                       uint64_t Size, Error *Err);
@@ -133,7 +136,7 @@ struct BigArMemHdrType {
 };
 
 // Define file member header of AIX big archive.
-class BigArchiveMemberHeader
+class LLVM_ABI BigArchiveMemberHeader
     : public CommonArchiveMemberHeader<BigArMemHdrType> {
 
 public:
@@ -153,7 +156,7 @@ class BigArchiveMemberHeader
   Expected<bool> isThin() const override { return false; }
 };
 
-class Archive : public Binary {
+class LLVM_ABI Archive : public Binary {
   virtual void anchor();
 
 public:
@@ -171,8 +174,8 @@ class Archive : public Binary {
     Expected<bool> isThinMember() const;
 
   public:
-    Child(const Archive *Parent, const char *Start, Error *Err);
-    Child(const Archive *Parent, StringRef Data, uint16_t StartOfFile);
+    LLVM_ABI Child(const Archive *Parent, const char *Start, Error *Err);
+    LLVM_ABI Child(const Archive *Parent, StringRef Data, uint16_t StartOfFile);
 
     Child(const Child &C)
         : Parent(C.Parent), Data(C.Data), StartOfFile(C.StartOfFile) {
@@ -218,10 +221,10 @@ class Archive : public Binary {
     }
 
     const Archive *getParent() const { return Parent; }
-    Expected<Child> getNext() const;
+    LLVM_ABI Expected<Child> getNext() const;
 
-    Expected<StringRef> getName() const;
-    Expected<std::string> getFullName() const;
+    LLVM_ABI Expected<StringRef> getName() const;
+    LLVM_ABI Expected<std::string> getFullName() const;
     Expected<StringRef> getRawName() const { return Header->getRawName(); }
 
     Expected<sys::TimePoint<std::chrono::seconds>> getLastModified() const {
@@ -240,17 +243,17 @@ class Archive : public Binary {
     }
 
     /// \return the size of the archive member without the header or padding.
-    Expected<uint64_t> getSize() const;
+    LLVM_ABI Expected<uint64_t> getSize() const;
     /// \return the size in the archive header for this member.
-    Expected<uint64_t> getRawSize() const;
+    LLVM_ABI Expected<uint64_t> getRawSize() const;
 
-    Expected<StringRef> getBuffer() const;
-    uint64_t getChildOffset() const;
+    LLVM_ABI Expected<StringRef> getBuffer() const;
+    LLVM_ABI uint64_t getChildOffset() const;
     uint64_t getDataOffset() const { return getChildOffset() + StartOfFile; }
 
-    Expected<MemoryBufferRef> getMemoryBufferRef() const;
+    LLVM_ABI Expected<MemoryBufferRef> getMemoryBufferRef() const;
 
-    Expected<std::unique_ptr<Binary>>
+    LLVM_ABI Expected<std::unique_ptr<Binary>>
     getAsBinary(LLVMContext *Context = nullptr) const;
   };
 
@@ -299,10 +302,10 @@ class Archive : public Binary {
       return (Parent == other.Parent) && (SymbolIndex == other.SymbolIndex);
     }
 
-    StringRef getName() const;
-    Expected<Child> getMember() const;
-    Symbol getNext() const;
-    bool isECSymbol() const;
+    LLVM_ABI StringRef getName() const;
+    LLVM_ABI Expected<Child> getMember() const;
+    LLVM_ABI Symbol getNext() const;
+    LLVM_ABI bool isECSymbol() const;
   };
 
   class symbol_iterator {
@@ -331,6 +334,10 @@ class Archive : public Binary {
   Archive(MemoryBufferRef Source, Error &Err);
   static Expected<std::unique_ptr<Archive>> create(MemoryBufferRef Source);
 
+  // Explicitly non-copyable.
+  Archive(Archive const &) = delete;
+  Archive &operator=(Archive const &) = delete;
+
   /// Size field is 10 decimal digits long
   static const uint64_t MaxMemberSize = 9999999999;
 
@@ -416,7 +423,7 @@ class BigArchive : public Archive {
   bool Has64BitGlobalSymtab = false;
 
 public:
-  BigArchive(MemoryBufferRef Source, Error &Err);
+  LLVM_ABI BigArchive(MemoryBufferRef Source, Error &Err);
   uint64_t getFirstChildOffset() const override { return FirstChildOffset; }
   uint64_t getLastChildOffset() const { return LastChildOffset; }
   bool isEmpty() const override { return getFirstChildOffset() == 0; }
diff --git a/external/llvm-project/llvm/include/llvm/Object/ArchiveWriter.h b/external/llvm-project/llvm/include/llvm/Object/ArchiveWriter.h
index 95b81f5f64c8..dad150ce1d84 100644
--- a/external/llvm-project/llvm/include/llvm/Object/ArchiveWriter.h
+++ b/external/llvm-project/llvm/include/llvm/Object/ArchiveWriter.h
@@ -14,6 +14,7 @@
 #define LLVM_OBJECT_ARCHIVEWRITER_H
 
 #include "llvm/Object/Archive.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -24,21 +25,22 @@ struct NewArchiveMember {
   unsigned UID = 0, GID = 0, Perms = 0644;
 
   NewArchiveMember() = default;
-  NewArchiveMember(MemoryBufferRef BufRef);
+  LLVM_ABI NewArchiveMember(MemoryBufferRef BufRef);
 
   // Detect the archive format from the object or bitcode file. This helps
   // assume the archive format when creating or editing archives in the case
   // one isn't explicitly set.
-  object::Archive::Kind detectKindFromObject() const;
+  LLVM_ABI object::Archive::Kind detectKindFromObject() const;
 
-  static Expected<NewArchiveMember>
+  LLVM_ABI static Expected<NewArchiveMember>
   getOldMember(const object::Archive::Child &OldMember, bool Deterministic);
 
-  static Expected<NewArchiveMember> getFile(StringRef FileName,
-                                            bool Deterministic);
+  LLVM_ABI static Expected<NewArchiveMember> getFile(StringRef FileName,
+                                                     bool Deterministic);
 };
 
-Expected<std::string> computeArchiveRelativePath(StringRef From, StringRef To);
+LLVM_ABI Expected<std::string> computeArchiveRelativePath(StringRef From,
+                                                          StringRef To);
 
 enum class SymtabWritingMode {
   NoSymtab,     // Do not write symbol table.
@@ -48,26 +50,26 @@ enum class SymtabWritingMode {
   BigArchive64  // Only write the 64-bit symbol table.
 };
 
-void warnToStderr(Error Err);
+LLVM_ABI void warnToStderr(Error Err);
 
 // Write an archive directly to an output stream.
-Error writeArchiveToStream(raw_ostream &Out,
-                           ArrayRef<NewArchiveMember> NewMembers,
-                           SymtabWritingMode WriteSymtab,
-                           object::Archive::Kind Kind, bool Deterministic,
-                           bool Thin, std::optional<bool> IsEC = std::nullopt,
-                           function_ref<void(Error)> Warn = warnToStderr);
+LLVM_ABI Error writeArchiveToStream(
+    raw_ostream &Out, ArrayRef<NewArchiveMember> NewMembers,
+    SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
+    bool Deterministic, bool Thin, std::optional<bool> IsEC = std::nullopt,
+    function_ref<void(Error)> Warn = warnToStderr);
 
-Error writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers,
-                   SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
-                   bool Deterministic, bool Thin,
-                   std::unique_ptr<MemoryBuffer> OldArchiveBuf = nullptr,
-                   std::optional<bool> IsEC = std::nullopt,
-                   function_ref<void(Error)> Warn = warnToStderr);
+LLVM_ABI Error
+writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers,
+             SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
+             bool Deterministic, bool Thin,
+             std::unique_ptr<MemoryBuffer> OldArchiveBuf = nullptr,
+             std::optional<bool> IsEC = std::nullopt,
+             function_ref<void(Error)> Warn = warnToStderr);
 
 // writeArchiveToBuffer is similar to writeArchive but returns the Archive in a
 // buffer instead of writing it out to a file.
-Expected<std::unique_ptr<MemoryBuffer>>
+LLVM_ABI Expected<std::unique_ptr<MemoryBuffer>>
 writeArchiveToBuffer(ArrayRef<NewArchiveMember> NewMembers,
                      SymtabWritingMode WriteSymtab, object::Archive::Kind Kind,
                      bool Deterministic, bool Thin,
diff --git a/external/llvm-project/llvm/include/llvm/Object/Binary.h b/external/llvm-project/llvm/include/llvm/Object/Binary.h
index ce870e25acaf..bd98f40dbc70 100644
--- a/external/llvm-project/llvm/include/llvm/Object/Binary.h
+++ b/external/llvm-project/llvm/include/llvm/Object/Binary.h
@@ -16,6 +16,7 @@
 #include "llvm-c/Types.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/TargetParser/Triple.h"
@@ -29,7 +30,7 @@ class StringRef;
 
 namespace object {
 
-class Binary {
+class LLVM_ABI Binary {
 private:
   unsigned int TypeID;
 
@@ -189,9 +190,9 @@ DEFINE_ISA_CONVERSION_FUNCTIONS(Binary, LLVMBinaryRef)
 /// Create a Binary from Source, autodetecting the file type.
 ///
 /// @param Source The data to create the Binary from.
-Expected<std::unique_ptr<Binary>> createBinary(MemoryBufferRef Source,
-                                               LLVMContext *Context = nullptr,
-                                               bool InitContent = true);
+LLVM_ABI Expected<std::unique_ptr<Binary>>
+createBinary(MemoryBufferRef Source, LLVMContext *Context = nullptr,
+             bool InitContent = true);
 
 template <typename T> class OwningBinary {
   std::unique_ptr<T> Bin;
@@ -241,9 +242,9 @@ template <typename T> const T* OwningBinary<T>::getBinary() const {
   return Bin.get();
 }
 
-Expected<OwningBinary<Binary>> createBinary(StringRef Path,
-                                            LLVMContext *Context = nullptr,
-                                            bool InitContent = true);
+LLVM_ABI Expected<OwningBinary<Binary>>
+createBinary(StringRef Path, LLVMContext *Context = nullptr,
+             bool InitContent = true);
 
 } // end namespace object
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/BuildID.h b/external/llvm-project/llvm/include/llvm/Object/BuildID.h
index b20f32b4d133..65ba00f75dd6 100644
--- a/external/llvm-project/llvm/include/llvm/Object/BuildID.h
+++ b/external/llvm-project/llvm/include/llvm/Object/BuildID.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace object {
@@ -30,13 +31,13 @@ typedef ArrayRef<uint8_t> BuildIDRef;
 class ObjectFile;
 
 /// Parses a build ID from a hex string.
-BuildID parseBuildID(StringRef Str);
+LLVM_ABI BuildID parseBuildID(StringRef Str);
 
 /// Returns the build ID, if any, contained in the given object file.
-BuildIDRef getBuildID(const ObjectFile *Obj);
+LLVM_ABI BuildIDRef getBuildID(const ObjectFile *Obj);
 
 /// BuildIDFetcher searches local cache directories for debug info.
-class BuildIDFetcher {
+class LLVM_ABI BuildIDFetcher {
 public:
   BuildIDFetcher(std::vector<std::string> DebugFileDirectories)
       : DebugFileDirectories(std::move(DebugFileDirectories)) {}
diff --git a/external/llvm-project/llvm/include/llvm/Object/COFF.h b/external/llvm-project/llvm/include/llvm/Object/COFF.h
index 3d0738c40904..ed2addbc6948 100644
--- a/external/llvm-project/llvm/include/llvm/Object/COFF.h
+++ b/external/llvm-project/llvm/include/llvm/Object/COFF.h
@@ -20,6 +20,7 @@
 #include "llvm/Object/Error.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/BinaryByteStream.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -876,7 +877,7 @@ struct coff_dynamic_relocation64_v2 {
   support::ulittle32_t Flags;
 };
 
-class COFFObjectFile : public ObjectFile {
+class LLVM_ABI COFFObjectFile : public ObjectFile {
 private:
   COFFObjectFile(MemoryBufferRef Object);
 
@@ -939,6 +940,10 @@ class COFFObjectFile : public ObjectFile {
     return uintptr_t(0);
   }
 
+  StringRef getStringTable() const {
+    return StringRef(StringTable, StringTableSize);
+  }
+
   uint16_t getMachine() const {
     if (COFFHeader) {
       if (CHPEMetadata) {
@@ -1230,22 +1235,23 @@ class ImportDirectoryEntryRef {
                           uint32_t I, const COFFObjectFile *Owner)
       : ImportTable(Table), Index(I), OwningObject(Owner) {}
 
-  bool operator==(const ImportDirectoryEntryRef &Other) const;
-  void moveNext();
+  LLVM_ABI bool operator==(const ImportDirectoryEntryRef &Other) const;
+  LLVM_ABI void moveNext();
 
-  imported_symbol_iterator imported_symbol_begin() const;
-  imported_symbol_iterator imported_symbol_end() const;
-  iterator_range<imported_symbol_iterator> imported_symbols() const;
+  LLVM_ABI imported_symbol_iterator imported_symbol_begin() const;
+  LLVM_ABI imported_symbol_iterator imported_symbol_end() const;
+  LLVM_ABI iterator_range<imported_symbol_iterator> imported_symbols() const;
 
-  imported_symbol_iterator lookup_table_begin() const;
-  imported_symbol_iterator lookup_table_end() const;
-  iterator_range<imported_symbol_iterator> lookup_table_symbols() const;
+  LLVM_ABI imported_symbol_iterator lookup_table_begin() const;
+  LLVM_ABI imported_symbol_iterator lookup_table_end() const;
+  LLVM_ABI iterator_range<imported_symbol_iterator>
+  lookup_table_symbols() const;
 
-  Error getName(StringRef &Result) const;
-  Error getImportLookupTableRVA(uint32_t &Result) const;
-  Error getImportAddressTableRVA(uint32_t &Result) const;
+  LLVM_ABI Error getName(StringRef &Result) const;
+  LLVM_ABI Error getImportLookupTableRVA(uint32_t &Result) const;
+  LLVM_ABI Error getImportAddressTableRVA(uint32_t &Result) const;
 
-  Error
+  LLVM_ABI Error
   getImportTableEntry(const coff_import_directory_table_entry *&Result) const;
 
 private:
@@ -1261,17 +1267,17 @@ class DelayImportDirectoryEntryRef {
                                uint32_t I, const COFFObjectFile *Owner)
       : Table(T), Index(I), OwningObject(Owner) {}
 
-  bool operator==(const DelayImportDirectoryEntryRef &Other) const;
-  void moveNext();
+  LLVM_ABI bool operator==(const DelayImportDirectoryEntryRef &Other) const;
+  LLVM_ABI void moveNext();
 
-  imported_symbol_iterator imported_symbol_begin() const;
-  imported_symbol_iterator imported_symbol_end() const;
-  iterator_range<imported_symbol_iterator> imported_symbols() const;
+  LLVM_ABI imported_symbol_iterator imported_symbol_begin() const;
+  LLVM_ABI imported_symbol_iterator imported_symbol_end() const;
+  LLVM_ABI iterator_range<imported_symbol_iterator> imported_symbols() const;
 
-  Error getName(StringRef &Result) const;
-  Error getDelayImportTable(
-      const delay_import_directory_table_entry *&Result) const;
-  Error getImportAddress(int AddrIndex, uint64_t &Result) const;
+  LLVM_ABI Error getName(StringRef &Result) const;
+  LLVM_ABI Error
+  getDelayImportTable(const delay_import_directory_table_entry *&Result) const;
+  LLVM_ABI Error getImportAddress(int AddrIndex, uint64_t &Result) const;
 
 private:
   const delay_import_directory_table_entry *Table;
@@ -1287,17 +1293,17 @@ class ExportDirectoryEntryRef {
                           const COFFObjectFile *Owner)
       : ExportTable(Table), Index(I), OwningObject(Owner) {}
 
-  bool operator==(const ExportDirectoryEntryRef &Other) const;
-  void moveNext();
+  LLVM_ABI bool operator==(const ExportDirectoryEntryRef &Other) const;
+  LLVM_ABI void moveNext();
 
-  Error getDllName(StringRef &Result) const;
-  Error getOrdinalBase(uint32_t &Result) const;
-  Error getOrdinal(uint32_t &Result) const;
-  Error getExportRVA(uint32_t &Result) const;
-  Error getSymbolName(StringRef &Result) const;
+  LLVM_ABI Error getDllName(StringRef &Result) const;
+  LLVM_ABI Error getOrdinalBase(uint32_t &Result) const;
+  LLVM_ABI Error getOrdinal(uint32_t &Result) const;
+  LLVM_ABI Error getExportRVA(uint32_t &Result) const;
+  LLVM_ABI Error getSymbolName(StringRef &Result) const;
 
-  Error isForwarder(bool &Result) const;
-  Error getForwardTo(StringRef &Result) const;
+  LLVM_ABI Error isForwarder(bool &Result) const;
+  LLVM_ABI Error getForwardTo(StringRef &Result) const;
 
 private:
   const export_directory_table_entry *ExportTable;
@@ -1315,13 +1321,13 @@ class ImportedSymbolRef {
                     const COFFObjectFile *Owner)
       : Entry32(nullptr), Entry64(Entry), Index(I), OwningObject(Owner) {}
 
-  bool operator==(const ImportedSymbolRef &Other) const;
-  void moveNext();
+  LLVM_ABI bool operator==(const ImportedSymbolRef &Other) const;
+  LLVM_ABI void moveNext();
 
-  Error getSymbolName(StringRef &Result) const;
-  Error isOrdinal(bool &Result) const;
-  Error getOrdinal(uint16_t &Result) const;
-  Error getHintNameRVA(uint32_t &Result) const;
+  LLVM_ABI Error getSymbolName(StringRef &Result) const;
+  LLVM_ABI Error isOrdinal(bool &Result) const;
+  LLVM_ABI Error getOrdinal(uint16_t &Result) const;
+  LLVM_ABI Error getHintNameRVA(uint32_t &Result) const;
 
 private:
   const import_lookup_table_entry32 *Entry32;
@@ -1337,11 +1343,11 @@ class BaseRelocRef {
                const COFFObjectFile *Owner)
       : Header(Header), Index(0) {}
 
-  bool operator==(const BaseRelocRef &Other) const;
-  void moveNext();
+  LLVM_ABI bool operator==(const BaseRelocRef &Other) const;
+  LLVM_ABI void moveNext();
 
-  Error getType(uint8_t &Type) const;
-  Error getRVA(uint32_t &Result) const;
+  LLVM_ABI Error getType(uint8_t &Type) const;
+  LLVM_ABI Error getRVA(uint32_t &Result) const;
 
 private:
   const coff_base_reloc_block_header *Header;
@@ -1354,14 +1360,14 @@ class DynamicRelocRef {
   DynamicRelocRef(const void *Header, const COFFObjectFile *Owner)
       : Obj(Owner), Header(reinterpret_cast<const uint8_t *>(Header)) {}
 
-  bool operator==(const DynamicRelocRef &Other) const;
-  void moveNext();
-  uint32_t getType() const;
-  void getContents(ArrayRef<uint8_t> &Ref) const;
+  LLVM_ABI bool operator==(const DynamicRelocRef &Other) const;
+  LLVM_ABI void moveNext();
+  LLVM_ABI uint32_t getType() const;
+  LLVM_ABI void getContents(ArrayRef<uint8_t> &Ref) const;
 
-  arm64x_reloc_iterator arm64x_reloc_begin() const;
-  arm64x_reloc_iterator arm64x_reloc_end() const;
-  iterator_range<arm64x_reloc_iterator> arm64x_relocs() const;
+  LLVM_ABI arm64x_reloc_iterator arm64x_reloc_begin() const;
+  LLVM_ABI arm64x_reloc_iterator arm64x_reloc_end() const;
+  LLVM_ABI iterator_range<arm64x_reloc_iterator> arm64x_relocs() const;
 
 private:
   Error validate() const;
@@ -1378,15 +1384,15 @@ class Arm64XRelocRef {
   Arm64XRelocRef(const coff_base_reloc_block_header *Header, uint32_t Index = 0)
       : Header(Header), Index(Index) {}
 
-  bool operator==(const Arm64XRelocRef &Other) const;
-  void moveNext();
+  LLVM_ABI bool operator==(const Arm64XRelocRef &Other) const;
+  LLVM_ABI void moveNext();
 
   COFF::Arm64XFixupType getType() const {
     return COFF::Arm64XFixupType((getReloc() >> 12) & 3);
   }
   uint32_t getRVA() const { return Header->PageRVA + (getReloc() & 0xfff); }
-  uint8_t getSize() const;
-  uint64_t getValue() const;
+  LLVM_ABI uint8_t getSize() const;
+  LLVM_ABI uint64_t getValue() const;
 
 private:
   const support::ulittle16_t &getReloc(uint32_t Offset = 0) const {
@@ -1410,20 +1416,21 @@ class ResourceSectionRef {
   explicit ResourceSectionRef(StringRef Ref)
       : BBS(Ref, llvm::endianness::little) {}
 
-  Error load(const COFFObjectFile *O);
-  Error load(const COFFObjectFile *O, const SectionRef &S);
+  LLVM_ABI Error load(const COFFObjectFile *O);
+  LLVM_ABI Error load(const COFFObjectFile *O, const SectionRef &S);
 
-  Expected<ArrayRef<UTF16>>
+  LLVM_ABI Expected<ArrayRef<UTF16>>
   getEntryNameString(const coff_resource_dir_entry &Entry);
-  Expected<const coff_resource_dir_table &>
+  LLVM_ABI Expected<const coff_resource_dir_table &>
   getEntrySubDir(const coff_resource_dir_entry &Entry);
-  Expected<const coff_resource_data_entry &>
+  LLVM_ABI Expected<const coff_resource_data_entry &>
   getEntryData(const coff_resource_dir_entry &Entry);
-  Expected<const coff_resource_dir_table &> getBaseTable();
-  Expected<const coff_resource_dir_entry &>
+  LLVM_ABI Expected<const coff_resource_dir_table &> getBaseTable();
+  LLVM_ABI Expected<const coff_resource_dir_entry &>
   getTableEntry(const coff_resource_dir_table &Table, uint32_t Index);
 
-  Expected<StringRef> getContents(const coff_resource_data_entry &Entry);
+  LLVM_ABI Expected<StringRef>
+  getContents(const coff_resource_data_entry &Entry);
 
 private:
   BinaryByteStream BBS;
diff --git a/external/llvm-project/llvm/include/llvm/Object/COFFImportFile.h b/external/llvm-project/llvm/include/llvm/Object/COFFImportFile.h
index e24eb4c380bf..f5b516f406e7 100644
--- a/external/llvm-project/llvm/include/llvm/Object/COFFImportFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/COFFImportFile.h
@@ -21,6 +21,7 @@
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -33,7 +34,7 @@ constexpr std::string_view NullImportDescriptorSymbolName =
 constexpr std::string_view NullThunkDataPrefix = "\x7f";
 constexpr std::string_view NullThunkDataSuffix = "_NULL_THUNK_DATA";
 
-class COFFImportFile : public SymbolicFile {
+class LLVM_ABI COFFImportFile : public SymbolicFile {
 private:
   enum SymbolIndex { ImpSymbol, ThunkSymbol, ECAuxSymbol, ECThunkSymbol };
 
@@ -135,10 +136,10 @@ struct COFFShortExport {
 /// linking both ARM64EC and pure ARM64 objects, and the linker will pick only
 /// the exports relevant to the target platform. For non-hybrid targets,
 /// the NativeExports parameter should not be used.
-Error writeImportLibrary(StringRef ImportName, StringRef Path,
-                         ArrayRef<COFFShortExport> Exports,
-                         COFF::MachineTypes Machine, bool MinGW,
-                         ArrayRef<COFFShortExport> NativeExports = {});
+LLVM_ABI Error writeImportLibrary(StringRef ImportName, StringRef Path,
+                                  ArrayRef<COFFShortExport> Exports,
+                                  COFF::MachineTypes Machine, bool MinGW,
+                                  ArrayRef<COFFShortExport> NativeExports = {});
 
 } // namespace object
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Object/COFFModuleDefinition.h b/external/llvm-project/llvm/include/llvm/Object/COFFModuleDefinition.h
index a4ed9978dcc0..66480277882b 100644
--- a/external/llvm-project/llvm/include/llvm/Object/COFFModuleDefinition.h
+++ b/external/llvm-project/llvm/include/llvm/Object/COFFModuleDefinition.h
@@ -20,6 +20,7 @@
 
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/Object/COFFImportFile.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace object {
@@ -39,7 +40,7 @@ struct COFFModuleDefinition {
   uint32_t MinorOSVersion = 0;
 };
 
-Expected<COFFModuleDefinition>
+LLVM_ABI Expected<COFFModuleDefinition>
 parseCOFFModuleDefinition(MemoryBufferRef MB, COFF::MachineTypes Machine,
                           bool MingwDef = false, bool AddUnderscores = true);
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/DXContainer.h b/external/llvm-project/llvm/include/llvm/Object/DXContainer.h
index 20524bf593d2..4417061da936 100644
--- a/external/llvm-project/llvm/include/llvm/Object/DXContainer.h
+++ b/external/llvm-project/llvm/include/llvm/Object/DXContainer.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/Object/Error.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
@@ -245,7 +246,7 @@ class RootSignature {
 public:
   RootSignature(StringRef PD) : PartData(PD) {}
 
-  Error parse();
+  LLVM_ABI Error parse();
   uint32_t getVersion() const { return Version; }
   uint32_t getNumParameters() const { return NumParameters; }
   uint32_t getRootParametersOffset() const { return RootParametersOffset; }
@@ -337,7 +338,7 @@ class PSVRuntimeInfo {
   PSVRuntimeInfo(StringRef D) : Data(D), Size(0) {}
 
   // Parsing depends on the shader kind
-  Error parse(uint16_t ShaderKind);
+  LLVM_ABI Error parse(uint16_t ShaderKind);
 
   uint32_t getSize() const { return Size; }
   uint32_t getResourceCount() const { return Resources.size(); }
@@ -381,9 +382,9 @@ class PSVRuntimeInfo {
     return SemanticIndexTable;
   }
 
-  uint8_t getSigInputCount() const;
-  uint8_t getSigOutputCount() const;
-  uint8_t getSigPatchOrPrimCount() const;
+  LLVM_ABI uint8_t getSigInputCount() const;
+  LLVM_ABI uint8_t getSigOutputCount() const;
+  LLVM_ABI uint8_t getSigPatchOrPrimCount() const;
 
   SigElementArray getSigInputElements() const { return SigInputElements; }
   SigElementArray getSigOutputElements() const { return SigOutputElements; }
@@ -460,7 +461,7 @@ class Signature {
 
   bool isEmpty() const { return Parameters.isEmpty(); }
 
-  Error initialize(StringRef Part);
+  LLVM_ABI Error initialize(StringRef Part);
 };
 
 } // namespace DirectX
@@ -528,7 +529,7 @@ class DXContainer {
 
     // Implementation for updating the iterator state based on a specified
     // offest.
-    void updateIteratorImpl(const uint32_t Offset);
+    LLVM_ABI void updateIteratorImpl(const uint32_t Offset);
 
   public:
     PartIterator &operator++() {
@@ -564,7 +565,7 @@ class DXContainer {
   PartIterator end() const { return PartIterator(*this, PartOffsets.end()); }
 
   StringRef getData() const { return Data.getBuffer(); }
-  static Expected<DXContainer> create(MemoryBufferRef Object);
+  LLVM_ABI static Expected<DXContainer> create(MemoryBufferRef Object);
 
   const dxbc::Header &getHeader() const { return Header; }
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/Decompressor.h b/external/llvm-project/llvm/include/llvm/Object/Decompressor.h
index d361b3fba81d..852b71ac86a5 100644
--- a/external/llvm-project/llvm/include/llvm/Object/Decompressor.h
+++ b/external/llvm-project/llvm/include/llvm/Object/Decompressor.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Error.h"
 
@@ -25,8 +26,8 @@ class Decompressor {
   /// @param Data        Section content.
   /// @param IsLE        Flag determines if Data is in little endian form.
   /// @param Is64Bit     Flag determines if object is 64 bit.
-  static Expected<Decompressor> create(StringRef Name, StringRef Data,
-                                       bool IsLE, bool Is64Bit);
+  LLVM_ABI static Expected<Decompressor> create(StringRef Name, StringRef Data,
+                                                bool IsLE, bool Is64Bit);
 
   /// Resize the buffer and uncompress section data into it.
   /// @param Out         Destination buffer.
@@ -36,7 +37,7 @@ class Decompressor {
   }
 
   /// Uncompress section data to raw buffer provided.
-  Error decompress(MutableArrayRef<uint8_t> Output);
+  LLVM_ABI Error decompress(MutableArrayRef<uint8_t> Output);
 
   /// Return memory buffer size required for decompression.
   uint64_t getDecompressedSize() { return DecompressedSize; }
diff --git a/external/llvm-project/llvm/include/llvm/Object/ELF.h b/external/llvm-project/llvm/include/llvm/Object/ELF.h
index 57a6db6c4e5a..a0dc522e13ca 100644
--- a/external/llvm-project/llvm/include/llvm/Object/ELF.h
+++ b/external/llvm-project/llvm/include/llvm/Object/ELF.h
@@ -71,9 +71,9 @@ struct VersionEntry {
   bool IsVerDef;
 };
 
-StringRef getELFRelocationTypeName(uint32_t Machine, uint32_t Type);
-uint32_t getELFRelativeRelocationType(uint32_t Machine);
-StringRef getELFSectionTypeName(uint32_t Machine, uint32_t Type);
+LLVM_ABI StringRef getELFRelocationTypeName(uint32_t Machine, uint32_t Type);
+LLVM_ABI uint32_t getELFRelativeRelocationType(uint32_t Machine);
+LLVM_ABI StringRef getELFSectionTypeName(uint32_t Machine, uint32_t Type);
 
 // Subclasses of ELFFile may need this for template instantiation
 inline std::pair<unsigned char, unsigned char>
@@ -256,6 +256,9 @@ class ELFFile {
 public:
   LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
 
+  // Default ctor required to instantiate the template for DLL export.
+  ELFFile(const ELFFile &) = default;
+
   // This is a callback that can be passed to a number of functions.
   // It can be used to ignore non-critical errors (warnings), which is
   // useful for dumpers, like llvm-readobj.
diff --git a/external/llvm-project/llvm/include/llvm/Object/ELFObjectFile.h b/external/llvm-project/llvm/include/llvm/Object/ELFObjectFile.h
index 35f1fc3d5814..103686884e70 100644
--- a/external/llvm-project/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/ELFObjectFile.h
@@ -26,6 +26,7 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/SymbolicFile.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ELFAttributeParser.h"
 #include "llvm/Support/ELFAttributes.h"
 #include "llvm/Support/Error.h"
@@ -45,7 +46,8 @@ template <typename T> class SmallVectorImpl;
 namespace object {
 
 constexpr int NumElfSymbolTypes = 16;
-extern const llvm::EnumEntry<unsigned> ElfSymbolTypes[NumElfSymbolTypes];
+LLVM_ABI extern const llvm::EnumEntry<unsigned>
+    ElfSymbolTypes[NumElfSymbolTypes];
 
 class elf_symbol_iterator;
 
@@ -55,7 +57,7 @@ struct ELFPltEntry {
   uint64_t Address;
 };
 
-class ELFObjectFileBase : public ObjectFile {
+class LLVM_ABI ELFObjectFileBase : public ObjectFile {
   friend class ELFRelocationRef;
   friend class ELFSectionRef;
   friend class ELFSymbolRef;
diff --git a/external/llvm-project/llvm/include/llvm/Object/Error.h b/external/llvm-project/llvm/include/llvm/Object/Error.h
index 8875fb6e1a20..9c522f56b02b 100644
--- a/external/llvm-project/llvm/include/llvm/Object/Error.h
+++ b/external/llvm-project/llvm/include/llvm/Object/Error.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_OBJECT_ERROR_H
 #define LLVM_OBJECT_ERROR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <system_error>
 
@@ -22,7 +23,7 @@ class Twine;
 
 namespace object {
 
-const std::error_category &object_category();
+LLVM_ABI const std::error_category &object_category();
 
 enum class object_error {
   // Error code 0 is absent. Use std::error_code() instead.
@@ -49,7 +50,7 @@ inline std::error_code make_error_code(object_error e) {
 ///
 /// Currently inherits from ECError for easy interoperability with
 /// std::error_code, but this will be removed in the future.
-class BinaryError : public ErrorInfo<BinaryError, ECError> {
+class LLVM_ABI BinaryError : public ErrorInfo<BinaryError, ECError> {
   void anchor() override;
 public:
   static char ID;
@@ -63,7 +64,8 @@ class BinaryError : public ErrorInfo<BinaryError, ECError> {
 ///
 /// For errors that don't require their own specific sub-error (most errors)
 /// this class can be used to describe the error via a string message.
-class GenericBinaryError : public ErrorInfo<GenericBinaryError, BinaryError> {
+class LLVM_ABI GenericBinaryError
+    : public ErrorInfo<GenericBinaryError, BinaryError> {
 public:
   static char ID;
   GenericBinaryError(const Twine &Msg);
@@ -79,7 +81,7 @@ class GenericBinaryError : public ErrorInfo<GenericBinaryError, BinaryError> {
 /// llvm::Error.  In the cases we want to loop through the children and ignore the
 /// non-objects in the archive this is used to test the error to see if an
 /// error() function needs to called on the llvm::Error.
-Error isNotObjectErrorInvalidFileType(llvm::Error Err);
+LLVM_ABI Error isNotObjectErrorInvalidFileType(llvm::Error Err);
 
 inline Error createError(const Twine &Err) {
   return make_error<StringError>(Err, object_error::parse_failed);
diff --git a/external/llvm-project/llvm/include/llvm/Object/FaultMapParser.h b/external/llvm-project/llvm/include/llvm/Object/FaultMapParser.h
index 028d3900d945..ec4281278ce5 100644
--- a/external/llvm-project/llvm/include/llvm/Object/FaultMapParser.h
+++ b/external/llvm-project/llvm/include/llvm/Object/FaultMapParser.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_OBJECT_FAULTMAPPARSER_H
 #define LLVM_OBJECT_FAULTMAPPARSER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include <cassert>
 #include <cstdint>
@@ -154,13 +155,13 @@ class FaultMapParser {
   }
 };
 
-raw_ostream &operator<<(raw_ostream &OS,
-                        const FaultMapParser::FunctionFaultInfoAccessor &);
+LLVM_ABI raw_ostream &
+operator<<(raw_ostream &OS, const FaultMapParser::FunctionFaultInfoAccessor &);
 
-raw_ostream &operator<<(raw_ostream &OS,
-                        const FaultMapParser::FunctionInfoAccessor &);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const FaultMapParser::FunctionInfoAccessor &);
 
-raw_ostream &operator<<(raw_ostream &OS, const FaultMapParser &);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const FaultMapParser &);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/GOFFObjectFile.h b/external/llvm-project/llvm/include/llvm/Object/GOFFObjectFile.h
index 6871641e97ec..b6b22ee7226f 100644
--- a/external/llvm-project/llvm/include/llvm/Object/GOFFObjectFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/GOFFObjectFile.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/BinaryFormat/GOFF.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ConvertEBCDIC.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -28,7 +29,7 @@ namespace llvm {
 
 namespace object {
 
-class GOFFObjectFile : public ObjectFile {
+class LLVM_ABI GOFFObjectFile : public ObjectFile {
   friend class GOFFSymbolRef;
 
   IndexedMap<const uint8_t *> EsdPtrs; // Indexed by EsdId.
diff --git a/external/llvm-project/llvm/include/llvm/Object/IRObjectFile.h b/external/llvm-project/llvm/include/llvm/Object/IRObjectFile.h
index 55d910fe970e..6e61b0d5f108 100644
--- a/external/llvm-project/llvm/include/llvm/Object/IRObjectFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/IRObjectFile.h
@@ -17,6 +17,7 @@
 #include "llvm/Object/IRSymtab.h"
 #include "llvm/Object/ModuleSymbolTable.h"
 #include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Module;
@@ -24,7 +25,7 @@ class Module;
 namespace object {
 class ObjectFile;
 
-class IRObjectFile : public SymbolicFile {
+class LLVM_ABI IRObjectFile : public SymbolicFile {
   std::vector<std::unique_ptr<Module>> Mods;
   ModuleSymbolTable SymTab;
   IRObjectFile(MemoryBufferRef Object,
@@ -80,8 +81,7 @@ struct IRSymtabFile {
 };
 
 /// Reads a bitcode file, creating its irsymtab if necessary.
-Expected<IRSymtabFile> readIRSymtab(MemoryBufferRef MBRef);
-
+LLVM_ABI Expected<IRSymtabFile> readIRSymtab(MemoryBufferRef MBRef);
 }
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Object/IRSymtab.h b/external/llvm-project/llvm/include/llvm/Object/IRSymtab.h
index 4e0013ea767e..0b7d6e0734e8 100644
--- a/external/llvm-project/llvm/include/llvm/Object/IRSymtab.h
+++ b/external/llvm-project/llvm/include/llvm/Object/IRSymtab.h
@@ -30,6 +30,7 @@
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/Object/SymbolicFile.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cassert>
@@ -162,8 +163,9 @@ struct Header {
 
 /// Fills in Symtab and StrtabBuilder with a valid symbol and string table for
 /// Mods.
-Error build(ArrayRef<Module *> Mods, SmallVector<char, 0> &Symtab,
-            StringTableBuilder &StrtabBuilder, BumpPtrAllocator &Alloc);
+LLVM_ABI Error build(ArrayRef<Module *> Mods, SmallVector<char, 0> &Symtab,
+                     StringTableBuilder &StrtabBuilder,
+                     BumpPtrAllocator &Alloc);
 
 /// This represents a symbol that has been read from a storage::Symbol and
 /// possibly a storage::Uncommon.
@@ -373,7 +375,7 @@ struct FileContents {
 };
 
 /// Reads the contents of a bitcode file, creating its irsymtab if necessary.
-Expected<FileContents> readBitcode(const BitcodeFileContents &BFC);
+LLVM_ABI Expected<FileContents> readBitcode(const BitcodeFileContents &BFC);
 
 } // end namespace irsymtab
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Object/MachO.h b/external/llvm-project/llvm/include/llvm/Object/MachO.h
index 61d05352ec75..3f4a21d2d1ba 100644
--- a/external/llvm-project/llvm/include/llvm/Object/MachO.h
+++ b/external/llvm-project/llvm/include/llvm/Object/MachO.h
@@ -25,6 +25,7 @@
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -72,18 +73,19 @@ using dice_iterator = content_iterator<DiceRef>;
 ///      if (Err) { report error ...
 class ExportEntry {
 public:
-  ExportEntry(Error *Err, const MachOObjectFile *O, ArrayRef<uint8_t> Trie);
+  LLVM_ABI ExportEntry(Error *Err, const MachOObjectFile *O,
+                       ArrayRef<uint8_t> Trie);
 
-  StringRef name() const;
-  uint64_t flags() const;
-  uint64_t address() const;
-  uint64_t other() const;
-  StringRef otherName() const;
-  uint32_t nodeOffset() const;
+  LLVM_ABI StringRef name() const;
+  LLVM_ABI uint64_t flags() const;
+  LLVM_ABI uint64_t address() const;
+  LLVM_ABI uint64_t other() const;
+  LLVM_ABI StringRef otherName() const;
+  LLVM_ABI uint32_t nodeOffset() const;
 
-  bool operator==(const ExportEntry &) const;
+  LLVM_ABI bool operator==(const ExportEntry &) const;
 
-  void moveNext();
+  LLVM_ABI void moveNext();
 
 private:
   friend class MachOObjectFile;
@@ -96,7 +98,7 @@ class ExportEntry {
 
   // Represents a node in the mach-o exports trie.
   struct NodeState {
-    NodeState(const uint8_t *Ptr);
+    LLVM_ABI NodeState(const uint8_t *Ptr);
 
     const uint8_t *Start;
     const uint8_t *Current;
@@ -131,16 +133,17 @@ using export_iterator = content_iterator<ExportEntry>;
 // address() methods below.
 class BindRebaseSegInfo {
 public:
-  BindRebaseSegInfo(const MachOObjectFile *Obj);
+  LLVM_ABI BindRebaseSegInfo(const MachOObjectFile *Obj);
 
   // Used to check a Mach-O Bind or Rebase entry for errors when iterating.
-  const char *checkSegAndOffsets(int32_t SegIndex, uint64_t SegOffset,
-                                 uint8_t PointerSize, uint64_t Count = 1,
-                                 uint64_t Skip = 0);
+  LLVM_ABI const char *checkSegAndOffsets(int32_t SegIndex, uint64_t SegOffset,
+                                          uint8_t PointerSize,
+                                          uint64_t Count = 1,
+                                          uint64_t Skip = 0);
   // Used with valid SegIndex/SegOffset values from checked entries.
-  StringRef segmentName(int32_t SegIndex);
-  StringRef sectionName(int32_t SegIndex, uint64_t SegOffset);
-  uint64_t address(uint32_t SegIndex, uint64_t SegOffset);
+  LLVM_ABI StringRef segmentName(int32_t SegIndex);
+  LLVM_ABI StringRef sectionName(int32_t SegIndex, uint64_t SegOffset);
+  LLVM_ABI uint64_t address(uint32_t SegIndex, uint64_t SegOffset);
 
 private:
   struct SectionInfo {
@@ -167,19 +170,19 @@ class BindRebaseSegInfo {
 ///    if (Err) { report error ...
 class MachORebaseEntry {
 public:
-  MachORebaseEntry(Error *Err, const MachOObjectFile *O,
-                   ArrayRef<uint8_t> opcodes, bool is64Bit);
+  LLVM_ABI MachORebaseEntry(Error *Err, const MachOObjectFile *O,
+                            ArrayRef<uint8_t> opcodes, bool is64Bit);
 
-  int32_t segmentIndex() const;
-  uint64_t segmentOffset() const;
-  StringRef typeName() const;
-  StringRef segmentName() const;
-  StringRef sectionName() const;
-  uint64_t address() const;
+  LLVM_ABI int32_t segmentIndex() const;
+  LLVM_ABI uint64_t segmentOffset() const;
+  LLVM_ABI StringRef typeName() const;
+  LLVM_ABI StringRef segmentName() const;
+  LLVM_ABI StringRef sectionName() const;
+  LLVM_ABI uint64_t address() const;
 
-  bool operator==(const MachORebaseEntry &) const;
+  LLVM_ABI bool operator==(const MachORebaseEntry &) const;
 
-  void moveNext();
+  LLVM_ABI void moveNext();
 
 private:
   friend class MachOObjectFile;
@@ -213,24 +216,25 @@ class MachOBindEntry {
 public:
   enum class Kind { Regular, Lazy, Weak };
 
-  MachOBindEntry(Error *Err, const MachOObjectFile *O,
-                 ArrayRef<uint8_t> Opcodes, bool is64Bit, MachOBindEntry::Kind);
+  LLVM_ABI MachOBindEntry(Error *Err, const MachOObjectFile *O,
+                          ArrayRef<uint8_t> Opcodes, bool is64Bit,
+                          MachOBindEntry::Kind);
 
-  int32_t segmentIndex() const;
-  uint64_t segmentOffset() const;
-  StringRef typeName() const;
-  StringRef symbolName() const;
-  uint32_t flags() const;
-  int64_t addend() const;
-  int ordinal() const;
+  LLVM_ABI int32_t segmentIndex() const;
+  LLVM_ABI uint64_t segmentOffset() const;
+  LLVM_ABI StringRef typeName() const;
+  LLVM_ABI StringRef symbolName() const;
+  LLVM_ABI uint32_t flags() const;
+  LLVM_ABI int64_t addend() const;
+  LLVM_ABI int ordinal() const;
 
-  StringRef segmentName() const;
-  StringRef sectionName() const;
-  uint64_t address() const;
+  LLVM_ABI StringRef segmentName() const;
+  LLVM_ABI StringRef sectionName() const;
+  LLVM_ABI uint64_t address() const;
 
-  bool operator==(const MachOBindEntry &) const;
+  LLVM_ABI bool operator==(const MachOBindEntry &) const;
 
-  void moveNext();
+  LLVM_ABI void moveNext();
 
 private:
   friend class MachOObjectFile;
@@ -321,22 +325,22 @@ struct ChainedFixupsSegment {
 ///   MachOChainedFixupEntry - for pointer chains embedded in data pages.
 class MachOAbstractFixupEntry {
 public:
-  MachOAbstractFixupEntry(Error *Err, const MachOObjectFile *O);
-
-  int32_t segmentIndex() const;
-  uint64_t segmentOffset() const;
-  uint64_t segmentAddress() const;
-  StringRef segmentName() const;
-  StringRef sectionName() const;
-  StringRef typeName() const;
-  StringRef symbolName() const;
-  uint32_t flags() const;
-  int64_t addend() const;
-  int ordinal() const;
+  LLVM_ABI MachOAbstractFixupEntry(Error *Err, const MachOObjectFile *O);
+
+  LLVM_ABI int32_t segmentIndex() const;
+  LLVM_ABI uint64_t segmentOffset() const;
+  LLVM_ABI uint64_t segmentAddress() const;
+  LLVM_ABI StringRef segmentName() const;
+  LLVM_ABI StringRef sectionName() const;
+  LLVM_ABI StringRef typeName() const;
+  LLVM_ABI StringRef symbolName() const;
+  LLVM_ABI uint32_t flags() const;
+  LLVM_ABI int64_t addend() const;
+  LLVM_ABI int ordinal() const;
 
   /// \return the location of this fixup as a VM Address. For the VM
   /// Address this fixup is pointing to, use pointerValue().
-  uint64_t address() const;
+  LLVM_ABI uint64_t address() const;
 
   /// \return the VM Address pointed to by this fixup. Use
   /// pointerValue() to compare against other VM Addresses, such as
@@ -350,7 +354,7 @@ class MachOAbstractFixupEntry {
   /// raw bits.
   uint64_t rawValue() const { return RawValue; }
 
-  void moveNext();
+  LLVM_ABI void moveNext();
 
 protected:
   Error *E;
@@ -365,8 +369,8 @@ class MachOAbstractFixupEntry {
   uint64_t RawValue = 0;
   bool Done = false;
 
-  void moveToFirst();
-  void moveToEnd();
+  LLVM_ABI void moveToFirst();
+  LLVM_ABI void moveToEnd();
 
   /// \return the vm address of the start of __TEXT segment.
   uint64_t textAddress() const { return TextAddress; }
@@ -379,16 +383,17 @@ class MachOChainedFixupEntry : public MachOAbstractFixupEntry {
 public:
   enum class FixupKind { Bind, Rebase };
 
-  MachOChainedFixupEntry(Error *Err, const MachOObjectFile *O, bool Parse);
+  LLVM_ABI MachOChainedFixupEntry(Error *Err, const MachOObjectFile *O,
+                                  bool Parse);
 
-  bool operator==(const MachOChainedFixupEntry &) const;
+  LLVM_ABI bool operator==(const MachOChainedFixupEntry &) const;
 
   bool isBind() const { return Kind == FixupKind::Bind; }
   bool isRebase() const { return Kind == FixupKind::Rebase; }
 
-  void moveNext();
-  void moveToFirst();
-  void moveToEnd();
+  LLVM_ABI void moveNext();
+  LLVM_ABI void moveToFirst();
+  LLVM_ABI void moveToEnd();
 
 private:
   void findNextPageWithFixups();
@@ -403,7 +408,7 @@ class MachOChainedFixupEntry : public MachOAbstractFixupEntry {
 };
 using fixup_iterator = content_iterator<MachOChainedFixupEntry>;
 
-class MachOObjectFile : public ObjectFile {
+class LLVM_ABI MachOObjectFile : public ObjectFile {
 public:
   struct LoadCommandInfo {
     const char *Ptr;      // Where in memory the load command is.
diff --git a/external/llvm-project/llvm/include/llvm/Object/MachOUniversal.h b/external/llvm-project/llvm/include/llvm/Object/MachOUniversal.h
index 5e3a63e05dbf..edc518845e8e 100644
--- a/external/llvm-project/llvm/include/llvm/Object/MachOUniversal.h
+++ b/external/llvm-project/llvm/include/llvm/Object/MachOUniversal.h
@@ -17,6 +17,7 @@
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/MachO.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
@@ -27,7 +28,7 @@ namespace object {
 class Archive;
 class IRObjectFile;
 
-class MachOUniversalBinary : public Binary {
+class LLVM_ABI MachOUniversalBinary : public Binary {
   virtual void anchor();
 
   uint32_t Magic;
@@ -44,7 +45,7 @@ class MachOUniversalBinary : public Binary {
     MachO::fat_arch_64 Header64;
 
   public:
-    ObjectForArch(const MachOUniversalBinary *Parent, uint32_t Index);
+    LLVM_ABI ObjectForArch(const MachOUniversalBinary *Parent, uint32_t Index);
 
     void clear() {
       Parent = nullptr;
@@ -102,11 +103,11 @@ class MachOUniversalBinary : public Binary {
       return ArchFlag ? ArchFlag : std::string();
     }
 
-    Expected<std::unique_ptr<MachOObjectFile>> getAsObjectFile() const;
-    Expected<std::unique_ptr<IRObjectFile>>
+    LLVM_ABI Expected<std::unique_ptr<MachOObjectFile>> getAsObjectFile() const;
+    LLVM_ABI Expected<std::unique_ptr<IRObjectFile>>
     getAsIRObject(LLVMContext &Ctx) const;
 
-    Expected<std::unique_ptr<Archive>> getAsArchive() const;
+    LLVM_ABI Expected<std::unique_ptr<Archive>> getAsArchive() const;
   };
 
   class object_iterator {
@@ -164,7 +165,6 @@ class MachOUniversalBinary : public Binary {
   Expected<std::unique_ptr<Archive>>
   getArchiveForArch(StringRef ArchName) const;
 };
-
 }
 }
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/MachOUniversalWriter.h b/external/llvm-project/llvm/include/llvm/Object/MachOUniversalWriter.h
index f910ca0cf968..094c82d1fcb8 100644
--- a/external/llvm-project/llvm/include/llvm/Object/MachOUniversalWriter.h
+++ b/external/llvm-project/llvm/include/llvm/Object/MachOUniversalWriter.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <string>
@@ -46,20 +47,21 @@ class Slice {
         std::string ArchName, uint32_t Align);
 
 public:
-  explicit Slice(const MachOObjectFile &O);
+  LLVM_ABI explicit Slice(const MachOObjectFile &O);
 
-  Slice(const MachOObjectFile &O, uint32_t Align);
+  LLVM_ABI Slice(const MachOObjectFile &O, uint32_t Align);
 
   /// This constructor takes pre-specified \param CPUType , \param CPUSubType ,
   /// \param ArchName , \param Align instead of inferring them from the archive
   /// members.
-  Slice(const Archive &A, uint32_t CPUType, uint32_t CPUSubType,
-        std::string ArchName, uint32_t Align);
+  LLVM_ABI Slice(const Archive &A, uint32_t CPUType, uint32_t CPUSubType,
+                 std::string ArchName, uint32_t Align);
 
-  static Expected<Slice> create(const Archive &A,
-                                LLVMContext *LLVMCtx = nullptr);
+  LLVM_ABI static Expected<Slice> create(const Archive &A,
+                                         LLVMContext *LLVMCtx = nullptr);
 
-  static Expected<Slice> create(const IRObjectFile &IRO, uint32_t Align);
+  LLVM_ABI static Expected<Slice> create(const IRObjectFile &IRO,
+                                         uint32_t Align);
 
   void setP2Alignment(uint32_t Align) { P2Alignment = Align; }
 
@@ -99,10 +101,11 @@ class Slice {
 
 enum class FatHeaderType { FatHeader, Fat64Header };
 
-Error writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName,
-                           FatHeaderType FatHeader = FatHeaderType::FatHeader);
+LLVM_ABI Error
+writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName,
+                     FatHeaderType FatHeader = FatHeaderType::FatHeader);
 
-Error writeUniversalBinaryToStream(
+LLVM_ABI Error writeUniversalBinaryToStream(
     ArrayRef<Slice> Slices, raw_ostream &Out,
     FatHeaderType FatHeader = FatHeaderType::FatHeader);
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/Minidump.h b/external/llvm-project/llvm/include/llvm/Object/Minidump.h
index 831be1c51d57..b68996eafda4 100644
--- a/external/llvm-project/llvm/include/llvm/Object/Minidump.h
+++ b/external/llvm-project/llvm/include/llvm/Object/Minidump.h
@@ -15,6 +15,7 @@
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/Minidump.h"
 #include "llvm/Object/Binary.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -26,7 +27,8 @@ class MinidumpFile : public Binary {
   /// Construct a new MinidumpFile object from the given memory buffer. Returns
   /// an error if this file cannot be identified as a minidump file, or if its
   /// contents are badly corrupted (i.e. we cannot read the stream directory).
-  static Expected<std::unique_ptr<MinidumpFile>> create(MemoryBufferRef Source);
+  LLVM_ABI static Expected<std::unique_ptr<MinidumpFile>>
+  create(MemoryBufferRef Source);
 
   static bool classof(const Binary *B) { return B->isMinidump(); }
 
@@ -43,7 +45,7 @@ class MinidumpFile : public Binary {
 
   /// Returns the raw contents of the stream of the given type, or std::nullopt
   /// if the file does not contain a stream of this type.
-  std::optional<ArrayRef<uint8_t>>
+  LLVM_ABI std::optional<ArrayRef<uint8_t>>
   getRawStream(minidump::StreamType Type) const;
 
   /// Returns the raw contents of an object given by the LocationDescriptor. An
@@ -55,7 +57,7 @@ class MinidumpFile : public Binary {
 
   /// Returns the minidump string at the given offset. An error is returned if
   /// we fail to parse the string, or the string is invalid UTF16.
-  Expected<std::string> getString(size_t Offset) const;
+  LLVM_ABI Expected<std::string> getString(size_t Offset) const;
 
   /// Returns the contents of the SystemInfo stream, cast to the appropriate
   /// type. An error is returned if the file does not contain this stream, or
@@ -265,13 +267,14 @@ class MinidumpFile : public Binary {
   /// contents of the exception strema are not validated before being read, an
   /// error will be returned if the stream is not large enough to contain an
   /// exception stream, or if the stream points beyond the end of the file.
-  iterator_range<ExceptionStreamsIterator> getExceptionStreams() const;
+  LLVM_ABI iterator_range<ExceptionStreamsIterator> getExceptionStreams() const;
 
   /// Returns an iterator that pairs each descriptor with it's respective
   /// content from the Memory64List stream. An error is returned if the file
   /// does not contain a Memory64List stream, or if the descriptor data is
   /// unreadable.
-  iterator_range<FallibleMemory64Iterator> getMemory64List(Error &Err) const;
+  LLVM_ABI iterator_range<FallibleMemory64Iterator>
+  getMemory64List(Error &Err) const;
 
   /// Returns the list of descriptors embedded in the MemoryInfoList stream. The
   /// descriptors provide properties (e.g. permissions) of interesting regions
@@ -280,7 +283,8 @@ class MinidumpFile : public Binary {
   /// contain the number of memory descriptors declared in the stream header.
   /// The consistency of the MemoryInfoList entries themselves is not checked
   /// in any way.
-  Expected<iterator_range<MemoryInfoIterator>> getMemoryInfoList() const;
+  LLVM_ABI Expected<iterator_range<MemoryInfoIterator>>
+  getMemoryInfoList() const;
 
 private:
   static Error createError(StringRef Str) {
@@ -293,7 +297,7 @@ class MinidumpFile : public Binary {
   }
 
   /// Return a slice of the given data array, with bounds checking.
-  static Expected<ArrayRef<uint8_t>>
+  LLVM_ABI static Expected<ArrayRef<uint8_t>>
   getDataSlice(ArrayRef<uint8_t> Data, uint64_t Offset, uint64_t Size);
 
   /// Return the slice of the given data array as an array of objects of the
diff --git a/external/llvm-project/llvm/include/llvm/Object/ModuleSymbolTable.h b/external/llvm-project/llvm/include/llvm/Object/ModuleSymbolTable.h
index 1134b98c2247..564ce76b3feb 100644
--- a/external/llvm-project/llvm/include/llvm/Object/ModuleSymbolTable.h
+++ b/external/llvm-project/llvm/include/llvm/Object/ModuleSymbolTable.h
@@ -20,6 +20,7 @@
 #include "llvm/IR/Mangler.h"
 #include "llvm/Object/SymbolicFile.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <string>
 #include <utility>
@@ -44,17 +45,17 @@ class ModuleSymbolTable {
 
 public:
   ArrayRef<Symbol> symbols() const { return SymTab; }
-  void addModule(Module *M);
+  LLVM_ABI void addModule(Module *M);
 
-  void printSymbolName(raw_ostream &OS, Symbol S) const;
-  uint32_t getSymbolFlags(Symbol S) const;
+  LLVM_ABI void printSymbolName(raw_ostream &OS, Symbol S) const;
+  LLVM_ABI uint32_t getSymbolFlags(Symbol S) const;
 
   /// Parse inline ASM and collect the symbols that are defined or referenced in
   /// the current module.
   ///
   /// For each found symbol, call \p AsmSymbol with the name of the symbol found
   /// and the associated flags.
-  static void CollectAsmSymbols(
+  LLVM_ABI static void CollectAsmSymbols(
       const Module &M,
       function_ref<void(StringRef, object::BasicSymbolRef::Flags)> AsmSymbol);
 
@@ -63,7 +64,7 @@ class ModuleSymbolTable {
   ///
   /// For each found symbol, call \p AsmSymver with the name of the symbol and
   /// its alias.
-  static void
+  LLVM_ABI static void
   CollectAsmSymvers(const Module &M,
                     function_ref<void(StringRef, StringRef)> AsmSymver);
 };
diff --git a/external/llvm-project/llvm/include/llvm/Object/ObjectFile.h b/external/llvm-project/llvm/include/llvm/Object/ObjectFile.h
index 20c0ef5ccfce..613c36a6a56d 100644
--- a/external/llvm-project/llvm/include/llvm/Object/ObjectFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/ObjectFile.h
@@ -23,6 +23,7 @@
 #include "llvm/Object/Error.h"
 #include "llvm/Object/SymbolicFile.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/TargetParser/Triple.h"
@@ -126,7 +127,7 @@ class SectionRef {
   /// Whether this section is a debug section.
   bool isDebugSection() const;
 
-  bool containsSymbol(SymbolRef S) const;
+  LLVM_ABI bool containsSymbol(SymbolRef S) const;
 
   relocation_iterator relocation_begin() const;
   relocation_iterator relocation_end() const;
@@ -161,7 +162,7 @@ inline bool operator==(const SectionedAddress &LHS,
          std::tie(RHS.SectionIndex, RHS.Address);
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const SectionedAddress &Addr);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SectionedAddress &Addr);
 
 /// This is a value type class that represents a single symbol in the list of
 /// symbols in the object file.
@@ -226,7 +227,7 @@ class symbol_iterator : public basic_symbol_iterator {
 /// This class is the base class for all object file types. Concrete instances
 /// of this object are created by createObjectFile, which figures out which type
 /// to create.
-class ObjectFile : public SymbolicFile {
+class LLVM_ABI ObjectFile : public SymbolicFile {
   virtual void anchor();
 
 protected:
diff --git a/external/llvm-project/llvm/include/llvm/Object/OffloadBinary.h b/external/llvm-project/llvm/include/llvm/Object/OffloadBinary.h
index aba89de825b7..d50ad40869e1 100644
--- a/external/llvm-project/llvm/include/llvm/Object/OffloadBinary.h
+++ b/external/llvm-project/llvm/include/llvm/Object/OffloadBinary.h
@@ -23,6 +23,7 @@
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <memory>
@@ -79,10 +80,11 @@ class OffloadBinary : public Binary {
   };
 
   /// Attempt to parse the offloading binary stored in \p Data.
-  static Expected<std::unique_ptr<OffloadBinary>> create(MemoryBufferRef);
+  LLVM_ABI static Expected<std::unique_ptr<OffloadBinary>>
+      create(MemoryBufferRef);
 
   /// Serialize the contents of \p File to a binary buffer to be read later.
-  static SmallString<0> write(const OffloadingImage &);
+  LLVM_ABI static SmallString<0> write(const OffloadingImage &);
 
   static uint64_t getAlignment() { return 8; }
 
@@ -188,20 +190,20 @@ class OffloadFile : public OwningBinary<OffloadBinary> {
 
 /// Extracts embedded device offloading code from a memory \p Buffer to a list
 /// of \p Binaries.
-Error extractOffloadBinaries(MemoryBufferRef Buffer,
-                             SmallVectorImpl<OffloadFile> &Binaries);
+LLVM_ABI Error extractOffloadBinaries(MemoryBufferRef Buffer,
+                                      SmallVectorImpl<OffloadFile> &Binaries);
 
 /// Convert a string \p Name to an image kind.
-ImageKind getImageKind(StringRef Name);
+LLVM_ABI ImageKind getImageKind(StringRef Name);
 
 /// Convert an image kind to its string representation.
-StringRef getImageKindName(ImageKind Name);
+LLVM_ABI StringRef getImageKindName(ImageKind Name);
 
 /// Convert a string \p Name to an offload kind.
-OffloadKind getOffloadKind(StringRef Name);
+LLVM_ABI OffloadKind getOffloadKind(StringRef Name);
 
 /// Convert an offload kind to its string representation.
-StringRef getOffloadKindName(OffloadKind Name);
+LLVM_ABI StringRef getOffloadKindName(OffloadKind Name);
 
 /// If the target is AMD we check the target IDs for mutual compatibility. A
 /// target id is a string conforming to the folowing BNF syntax:
@@ -212,8 +214,8 @@ StringRef getOffloadKindName(OffloadKind Name);
 /// the state of on, off, and any when unspecified. A target marked as any can
 /// bind with either on or off. This is used to link mutually compatible
 /// architectures together. Returns false in the case of an exact match.
-bool areTargetsCompatible(const OffloadFile::TargetID &LHS,
-                          const OffloadFile::TargetID &RHS);
+LLVM_ABI bool areTargetsCompatible(const OffloadFile::TargetID &LHS,
+                                   const OffloadFile::TargetID &RHS);
 
 } // namespace object
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/OffloadBundle.h b/external/llvm-project/llvm/include/llvm/Object/OffloadBundle.h
index 207b4b9d713c..f4d5a1d878b8 100644
--- a/external/llvm-project/llvm/include/llvm/Object/OffloadBundle.h
+++ b/external/llvm-project/llvm/include/llvm/Object/OffloadBundle.h
@@ -22,6 +22,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -49,10 +50,10 @@ class CompressedOffloadBundle {
   static inline const uint16_t Version = 2;
 
 public:
-  static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
+  LLVM_ABI static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
   compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input,
            bool Verbose = false);
-  static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
+  LLVM_ABI static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
   decompress(llvm::MemoryBufferRef &Input, bool Verbose = false);
 };
 
@@ -88,13 +89,13 @@ class OffloadBundleFatBin {
   StringRef getFileName() const { return FileName; }
   uint64_t getNumEntries() const { return NumberOfEntries; }
 
-  static Expected<std::unique_ptr<OffloadBundleFatBin>>
+  LLVM_ABI static Expected<std::unique_ptr<OffloadBundleFatBin>>
   create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName);
-  Error extractBundle(const ObjectFile &Source);
+  LLVM_ABI Error extractBundle(const ObjectFile &Source);
 
-  Error dumpEntryToCodeObject();
+  LLVM_ABI Error dumpEntryToCodeObject();
 
-  Error readEntries(StringRef Section, uint64_t SectionOffset);
+  LLVM_ABI Error readEntries(StringRef Section, uint64_t SectionOffset);
   void dumpEntries() {
     for (OffloadBundleEntry &Entry : Entries)
       Entry.dumpInfo(outs());
@@ -182,16 +183,16 @@ struct OffloadBundleURI {
 
 /// Extracts fat binary in binary clang-offload-bundler format from object \p
 /// Obj and return it in \p Bundles
-Error extractOffloadBundleFatBinary(
+LLVM_ABI Error extractOffloadBundleFatBinary(
     const ObjectFile &Obj, SmallVectorImpl<OffloadBundleFatBin> &Bundles);
 
 /// Extract code object memory from the given \p Source object file at \p Offset
 /// and of \p Size, and copy into \p OutputFileName.
-Error extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_t Size,
-                        StringRef OutputFileName);
+LLVM_ABI Error extractCodeObject(const ObjectFile &Source, int64_t Offset,
+                                 int64_t Size, StringRef OutputFileName);
 
 /// Extracts an Offload Bundle Entry given by URI
-Error extractOffloadBundleByURI(StringRef URIstr);
+LLVM_ABI Error extractOffloadBundleByURI(StringRef URIstr);
 
 } // namespace object
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/RelocationResolver.h b/external/llvm-project/llvm/include/llvm/Object/RelocationResolver.h
index 2acdf5ed2fe1..da98d387ce38 100644
--- a/external/llvm-project/llvm/include/llvm/Object/RelocationResolver.h
+++ b/external/llvm-project/llvm/include/llvm/Object/RelocationResolver.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_OBJECT_RELOCATIONRESOLVER_H
 #define LLVM_OBJECT_RELOCATIONRESOLVER_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <utility>
 
@@ -29,11 +30,12 @@ using RelocationResolver = uint64_t (*)(uint64_t Type, uint64_t Offset,
                                         uint64_t S, uint64_t LocData,
                                         int64_t Addend);
 
-std::pair<SupportsRelocation, RelocationResolver>
+LLVM_ABI std::pair<SupportsRelocation, RelocationResolver>
 getRelocationResolver(const ObjectFile &Obj);
 
-uint64_t resolveRelocation(RelocationResolver Resolver, const RelocationRef &R,
-                           uint64_t S, uint64_t LocData);
+LLVM_ABI uint64_t resolveRelocation(RelocationResolver Resolver,
+                                    const RelocationRef &R, uint64_t S,
+                                    uint64_t LocData);
 
 } // end namespace object
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Object/SymbolSize.h b/external/llvm-project/llvm/include/llvm/Object/SymbolSize.h
index 085623e35907..33ad3306f764 100644
--- a/external/llvm-project/llvm/include/llvm/Object/SymbolSize.h
+++ b/external/llvm-project/llvm/include/llvm/Object/SymbolSize.h
@@ -11,6 +11,7 @@
 #define LLVM_OBJECT_SYMBOLSIZE_H
 
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace object {
@@ -22,11 +23,10 @@ struct SymEntry {
   unsigned SectionID;
 };
 
-int compareAddress(const SymEntry *A, const SymEntry *B);
+LLVM_ABI int compareAddress(const SymEntry *A, const SymEntry *B);
 
-std::vector<std::pair<SymbolRef, uint64_t>>
+LLVM_ABI std::vector<std::pair<SymbolRef, uint64_t>>
 computeSymbolSizes(const ObjectFile &O);
-
 }
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/SymbolicFile.h b/external/llvm-project/llvm/include/llvm/Object/SymbolicFile.h
index 2c857e72c3e5..da59eec7700b 100644
--- a/external/llvm-project/llvm/include/llvm/Object/SymbolicFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/SymbolicFile.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Object/Binary.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MemoryBufferRef.h"
@@ -142,7 +143,7 @@ class BasicSymbolRef {
 
 using basic_symbol_iterator = content_iterator<BasicSymbolRef>;
 
-class SymbolicFile : public Binary {
+class LLVM_ABI SymbolicFile : public Binary {
 public:
   SymbolicFile(unsigned int Type, MemoryBufferRef Source);
   ~SymbolicFile() override;
diff --git a/external/llvm-project/llvm/include/llvm/Object/TapiFile.h b/external/llvm-project/llvm/include/llvm/Object/TapiFile.h
index c1de6608bb62..dc34ae84f08b 100644
--- a/external/llvm-project/llvm/include/llvm/Object/TapiFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/TapiFile.h
@@ -17,6 +17,7 @@
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/SymbolicFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/TextAPI/Architecture.h"
@@ -28,7 +29,7 @@ class raw_ostream;
 
 namespace object {
 
-class TapiFile : public SymbolicFile {
+class LLVM_ABI TapiFile : public SymbolicFile {
 public:
   TapiFile(MemoryBufferRef Source, const MachO::InterfaceFile &Interface,
            MachO::Architecture Arch);
diff --git a/external/llvm-project/llvm/include/llvm/Object/TapiUniversal.h b/external/llvm-project/llvm/include/llvm/Object/TapiUniversal.h
index 558916f7af79..3cc2060dc46e 100644
--- a/external/llvm-project/llvm/include/llvm/Object/TapiUniversal.h
+++ b/external/llvm-project/llvm/include/llvm/Object/TapiUniversal.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/Binary.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/TextAPI/Architecture.h"
@@ -25,7 +26,7 @@ namespace object {
 
 class TapiFile;
 
-class TapiUniversal : public Binary {
+class LLVM_ABI TapiUniversal : public Binary {
 public:
   class ObjectForArch {
     const TapiUniversal *Parent;
@@ -65,7 +66,7 @@ class TapiUniversal : public Binary {
       return Parent->ParsedFile->getInstallName() == getInstallName();
     }
 
-    Expected<std::unique_ptr<TapiFile>> getAsObjectFile() const;
+    LLVM_ABI Expected<std::unique_ptr<TapiFile>> getAsObjectFile() const;
   };
 
   class object_iterator {
diff --git a/external/llvm-project/llvm/include/llvm/Object/Wasm.h b/external/llvm-project/llvm/include/llvm/Object/Wasm.h
index cda896b2b780..b4ba50778c15 100644
--- a/external/llvm-project/llvm/include/llvm/Object/Wasm.h
+++ b/external/llvm-project/llvm/include/llvm/Object/Wasm.h
@@ -23,6 +23,7 @@
 #include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cstddef>
@@ -98,7 +99,7 @@ class WasmSymbol {
     return Info.Flags & wasm::WASM_SYMBOL_VISIBILITY_MASK;
   }
 
-  void print(raw_ostream &Out) const;
+  LLVM_ABI void print(raw_ostream &Out) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   LLVM_DUMP_METHOD void dump() const;
@@ -123,7 +124,7 @@ struct WasmSegment {
   wasm::WasmDataSegment Data;
 };
 
-class WasmObjectFile : public ObjectFile {
+class LLVM_ABI WasmObjectFile : public ObjectFile {
 
 public:
   WasmObjectFile(MemoryBufferRef Object, Error &Err);
@@ -353,9 +354,11 @@ class WasmSectionOrderChecker {
   };
 
   // Sections that may or may not be present, but cannot be predecessors
-  static int DisallowedPredecessors[WASM_NUM_SEC_ORDERS][WASM_NUM_SEC_ORDERS];
+  LLVM_ABI static int DisallowedPredecessors[WASM_NUM_SEC_ORDERS]
+                                            [WASM_NUM_SEC_ORDERS];
 
-  bool isValidSectionOrder(unsigned ID, StringRef CustomSectionName = "");
+  LLVM_ABI bool isValidSectionOrder(unsigned ID,
+                                    StringRef CustomSectionName = "");
 
 private:
   bool Seen[WASM_NUM_SEC_ORDERS] = {}; // Sections that have been seen already
diff --git a/external/llvm-project/llvm/include/llvm/Object/WindowsMachineFlag.h b/external/llvm-project/llvm/include/llvm/Object/WindowsMachineFlag.h
index ce5b356f8bfe..7655819ef50f 100644
--- a/external/llvm-project/llvm/include/llvm/Object/WindowsMachineFlag.h
+++ b/external/llvm-project/llvm/include/llvm/Object/WindowsMachineFlag.h
@@ -14,6 +14,7 @@
 #define LLVM_OBJECT_WINDOWSMACHINEFLAG_H
 
 #include "llvm/BinaryFormat/COFF.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
@@ -25,11 +26,11 @@ enum MachineTypes : unsigned;
 
 // Returns a user-readable string for ARMNT, ARM64, AMD64, I386.
 // Other MachineTypes values must not be passed in.
-StringRef machineToStr(COFF::MachineTypes MT);
+LLVM_ABI StringRef machineToStr(COFF::MachineTypes MT);
 
 // Maps /machine: arguments to a MachineTypes value.
 // Only returns ARMNT, ARM64, AMD64, I386, or IMAGE_FILE_MACHINE_UNKNOWN.
-COFF::MachineTypes getMachineType(StringRef S);
+LLVM_ABI COFF::MachineTypes getMachineType(StringRef S);
 
 template <typename T> Triple::ArchType getMachineArchType(T machine) {
   switch (machine) {
diff --git a/external/llvm-project/llvm/include/llvm/Object/WindowsResource.h b/external/llvm-project/llvm/include/llvm/Object/WindowsResource.h
index ec390a4814cc..230de3a06f77 100644
--- a/external/llvm-project/llvm/include/llvm/Object/WindowsResource.h
+++ b/external/llvm-project/llvm/include/llvm/Object/WindowsResource.h
@@ -34,6 +34,7 @@
 #include "llvm/Object/Error.h"
 #include "llvm/Support/BinaryByteStream.h"
 #include "llvm/Support/BinaryStreamReader.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
@@ -97,7 +98,7 @@ class EmptyResError : public GenericBinaryError {
 
 class ResourceEntryRef {
 public:
-  Error moveNext(bool &End);
+  LLVM_ABI Error moveNext(bool &End);
   bool checkTypeString() const { return IsStringType; }
   ArrayRef<UTF16> getTypeString() const { return Type; }
   uint16_t getTypeID() const { return TypeID; }
@@ -135,11 +136,11 @@ class ResourceEntryRef {
 
 class WindowsResource : public Binary {
 public:
-  Expected<ResourceEntryRef> getHeadEntry();
+  LLVM_ABI Expected<ResourceEntryRef> getHeadEntry();
 
   static bool classof(const Binary *V) { return V->isWinRes(); }
 
-  static Expected<std::unique_ptr<WindowsResource>>
+  LLVM_ABI static Expected<std::unique_ptr<WindowsResource>>
   createWindowsResource(MemoryBufferRef Source);
 
 private:
@@ -153,12 +154,13 @@ class WindowsResource : public Binary {
 class WindowsResourceParser {
 public:
   class TreeNode;
-  WindowsResourceParser(bool MinGW = false);
-  Error parse(WindowsResource *WR, std::vector<std::string> &Duplicates);
-  Error parse(ResourceSectionRef &RSR, StringRef Filename,
-              std::vector<std::string> &Duplicates);
-  void cleanUpManifests(std::vector<std::string> &Duplicates);
-  void printTree(raw_ostream &OS) const;
+  LLVM_ABI WindowsResourceParser(bool MinGW = false);
+  LLVM_ABI Error parse(WindowsResource *WR,
+                       std::vector<std::string> &Duplicates);
+  LLVM_ABI Error parse(ResourceSectionRef &RSR, StringRef Filename,
+                       std::vector<std::string> &Duplicates);
+  LLVM_ABI void cleanUpManifests(std::vector<std::string> &Duplicates);
+  LLVM_ABI void printTree(raw_ostream &OS) const;
   const TreeNode &getTree() const { return Root; }
   ArrayRef<std::vector<uint8_t>> getData() const { return Data; }
   ArrayRef<std::vector<UTF16>> getStringTable() const { return StringTable; }
@@ -168,8 +170,8 @@ class WindowsResourceParser {
     template <typename T>
     using Children = std::map<T, std::unique_ptr<TreeNode>>;
 
-    void print(ScopedPrinter &Writer, StringRef Name) const;
-    uint32_t getTreeSize() const;
+    LLVM_ABI void print(ScopedPrinter &Writer, StringRef Name) const;
+    LLVM_ABI uint32_t getTreeSize() const;
     uint32_t getStringIndex() const { return StringIndex; }
     uint32_t getDataIndex() const { return DataIndex; }
     uint16_t getMajorVersion() const { return MajorVersion; }
@@ -257,12 +259,12 @@ class WindowsResourceParser {
   bool MinGW;
 };
 
-Expected<std::unique_ptr<MemoryBuffer>>
+LLVM_ABI Expected<std::unique_ptr<MemoryBuffer>>
 writeWindowsResourceCOFF(llvm::COFF::MachineTypes MachineType,
                          const WindowsResourceParser &Parser,
                          uint32_t TimeDateStamp);
 
-void printResourceTypeName(uint16_t TypeID, raw_ostream &OS);
+LLVM_ABI void printResourceTypeName(uint16_t TypeID, raw_ostream &OS);
 } // namespace object
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Object/XCOFFObjectFile.h b/external/llvm-project/llvm/include/llvm/Object/XCOFFObjectFile.h
index ec48843cd532..da3538ecb302 100644
--- a/external/llvm-project/llvm/include/llvm/Object/XCOFFObjectFile.h
+++ b/external/llvm-project/llvm/include/llvm/Object/XCOFFObjectFile.h
@@ -176,8 +176,10 @@ template <typename T> struct XCOFFSectionHeader {
 // Explicit extern template declarations.
 struct XCOFFSectionHeader32;
 struct XCOFFSectionHeader64;
-extern template struct XCOFFSectionHeader<XCOFFSectionHeader32>;
-extern template struct XCOFFSectionHeader<XCOFFSectionHeader64>;
+extern template struct LLVM_TEMPLATE_ABI
+    XCOFFSectionHeader<XCOFFSectionHeader32>;
+extern template struct LLVM_TEMPLATE_ABI
+    XCOFFSectionHeader<XCOFFSectionHeader64>;
 
 struct XCOFFSectionHeader32 : XCOFFSectionHeader<XCOFFSectionHeader32> {
   char Name[XCOFF::NameSize];
@@ -222,7 +224,7 @@ struct LoaderSectionSymbolEntry32 {
   support::ubig32_t ImportFileID;
   support::ubig32_t ParameterTypeCheck;
 
-  Expected<StringRef>
+  LLVM_ABI Expected<StringRef>
   getSymbolName(const LoaderSectionHeader32 *LoaderSecHeader) const;
 };
 
@@ -235,7 +237,7 @@ struct LoaderSectionSymbolEntry64 {
   support::ubig32_t ImportFileID;
   support::ubig32_t ParameterTypeCheck;
 
-  Expected<StringRef>
+  LLVM_ABI Expected<StringRef>
   getSymbolName(const LoaderSectionHeader64 *LoaderSecHeader) const;
 };
 
@@ -318,8 +320,10 @@ typedef ExceptionSectionEntry<support::ubig32_t> ExceptionSectionEntry32;
 typedef ExceptionSectionEntry<support::ubig64_t> ExceptionSectionEntry64;
 
 // Explicit extern template declarations.
-extern template struct ExceptionSectionEntry<support::ubig32_t>;
-extern template struct ExceptionSectionEntry<support::ubig64_t>;
+extern template struct LLVM_TEMPLATE_ABI
+    ExceptionSectionEntry<support::ubig32_t>;
+extern template struct LLVM_TEMPLATE_ABI
+    ExceptionSectionEntry<support::ubig64_t>;
 
 struct XCOFFStringTable {
   uint32_t Size;
@@ -521,15 +525,17 @@ template <typename AddressType> struct XCOFFRelocation {
   uint8_t getRelocatedLength() const;
 };
 
-extern template struct XCOFFRelocation<llvm::support::ubig32_t>;
-extern template struct XCOFFRelocation<llvm::support::ubig64_t>;
+extern template struct LLVM_TEMPLATE_ABI
+    XCOFFRelocation<llvm::support::ubig32_t>;
+extern template struct LLVM_TEMPLATE_ABI
+    XCOFFRelocation<llvm::support::ubig64_t>;
 
 struct XCOFFRelocation32 : XCOFFRelocation<llvm::support::ubig32_t> {};
 struct XCOFFRelocation64 : XCOFFRelocation<llvm::support::ubig64_t> {};
 
 class XCOFFSymbolRef;
 
-class XCOFFObjectFile : public ObjectFile {
+class LLVM_ABI XCOFFObjectFile : public ObjectFile {
 private:
   const void *FileHeader = nullptr;
   const void *AuxiliaryHeader = nullptr;
@@ -852,10 +858,10 @@ class XCOFFSymbolRef : public SymbolRef {
     return getRawDataRefImpl().p;
   }
 
-  Expected<StringRef> getName() const;
-  Expected<bool> isFunction() const;
-  bool isCsectSymbol() const;
-  Expected<XCOFFCsectAuxRef> getXCOFFCsectAuxRef() const;
+  LLVM_ABI Expected<StringRef> getName() const;
+  LLVM_ABI Expected<bool> isFunction() const;
+  LLVM_ABI bool isCsectSymbol() const;
+  LLVM_ABI Expected<XCOFFCsectAuxRef> getXCOFFCsectAuxRef() const;
 
 private:
   const XCOFFObjectFile *getObject() const {
@@ -887,12 +893,12 @@ class TBVectorExt {
   TBVectorExt(StringRef TBvectorStrRef, Error &Err);
 
 public:
-  static Expected<TBVectorExt> create(StringRef TBvectorStrRef);
-  uint8_t getNumberOfVRSaved() const;
-  bool isVRSavedOnStack() const;
-  bool hasVarArgs() const;
-  uint8_t getNumberOfVectorParms() const;
-  bool hasVMXInstruction() const;
+  LLVM_ABI static Expected<TBVectorExt> create(StringRef TBvectorStrRef);
+  LLVM_ABI uint8_t getNumberOfVRSaved() const;
+  LLVM_ABI bool isVRSavedOnStack() const;
+  LLVM_ABI bool hasVarArgs() const;
+  LLVM_ABI uint8_t getNumberOfVectorParms() const;
+  LLVM_ABI bool hasVMXInstruction() const;
   SmallString<32> getVectorParmsInfo() const { return VecParmsInfo; };
 };
 
@@ -930,39 +936,39 @@ class XCOFFTracebackTable {
   ///    If the XCOFF Traceback Table is not parsed successfully or there are
   ///    extra bytes that are not recognized, \a Size will be updated to be the
   ///    size up to the end of the last successfully parsed field of the table.
-  static Expected<XCOFFTracebackTable>
+  LLVM_ABI static Expected<XCOFFTracebackTable>
   create(const uint8_t *Ptr, uint64_t &Size, bool Is64Bits = false);
-  uint8_t getVersion() const;
-  uint8_t getLanguageID() const;
-
-  bool isGlobalLinkage() const;
-  bool isOutOfLineEpilogOrPrologue() const;
-  bool hasTraceBackTableOffset() const;
-  bool isInternalProcedure() const;
-  bool hasControlledStorage() const;
-  bool isTOCless() const;
-  bool isFloatingPointPresent() const;
-  bool isFloatingPointOperationLogOrAbortEnabled() const;
-
-  bool isInterruptHandler() const;
-  bool isFuncNamePresent() const;
-  bool isAllocaUsed() const;
-  uint8_t getOnConditionDirective() const;
-  bool isCRSaved() const;
-  bool isLRSaved() const;
-
-  bool isBackChainStored() const;
-  bool isFixup() const;
-  uint8_t getNumOfFPRsSaved() const;
-
-  bool hasVectorInfo() const;
-  bool hasExtensionTable() const;
-  uint8_t getNumOfGPRsSaved() const;
-
-  uint8_t getNumberOfFixedParms() const;
-
-  uint8_t getNumberOfFPParms() const;
-  bool hasParmsOnStack() const;
+  LLVM_ABI uint8_t getVersion() const;
+  LLVM_ABI uint8_t getLanguageID() const;
+
+  LLVM_ABI bool isGlobalLinkage() const;
+  LLVM_ABI bool isOutOfLineEpilogOrPrologue() const;
+  LLVM_ABI bool hasTraceBackTableOffset() const;
+  LLVM_ABI bool isInternalProcedure() const;
+  LLVM_ABI bool hasControlledStorage() const;
+  LLVM_ABI bool isTOCless() const;
+  LLVM_ABI bool isFloatingPointPresent() const;
+  LLVM_ABI bool isFloatingPointOperationLogOrAbortEnabled() const;
+
+  LLVM_ABI bool isInterruptHandler() const;
+  LLVM_ABI bool isFuncNamePresent() const;
+  LLVM_ABI bool isAllocaUsed() const;
+  LLVM_ABI uint8_t getOnConditionDirective() const;
+  LLVM_ABI bool isCRSaved() const;
+  LLVM_ABI bool isLRSaved() const;
+
+  LLVM_ABI bool isBackChainStored() const;
+  LLVM_ABI bool isFixup() const;
+  LLVM_ABI uint8_t getNumOfFPRsSaved() const;
+
+  LLVM_ABI bool hasVectorInfo() const;
+  LLVM_ABI bool hasExtensionTable() const;
+  LLVM_ABI uint8_t getNumOfGPRsSaved() const;
+
+  LLVM_ABI uint8_t getNumberOfFixedParms() const;
+
+  LLVM_ABI uint8_t getNumberOfFPParms() const;
+  LLVM_ABI bool hasParmsOnStack() const;
 
   const std::optional<SmallString<32>> &getParmsType() const {
     return ParmsType;
@@ -991,7 +997,7 @@ class XCOFFTracebackTable {
   const std::optional<uint64_t> &getEhInfoDisp() const { return EhInfoDisp; }
 };
 
-bool doesXCOFFTracebackTableBegin(ArrayRef<uint8_t> Bytes);
+LLVM_ABI bool doesXCOFFTracebackTableBegin(ArrayRef<uint8_t> Bytes);
 } // namespace object
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Option/Arg.h b/external/llvm-project/llvm/include/llvm/Option/Arg.h
index 5a718438bf4a..b1e56b58da68 100644
--- a/external/llvm-project/llvm/include/llvm/Option/Arg.h
+++ b/external/llvm-project/llvm/include/llvm/Option/Arg.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -70,15 +71,16 @@ class Arg {
   std::unique_ptr<Arg> Alias;
 
 public:
-  Arg(const Option Opt, StringRef Spelling, unsigned Index,
-      const Arg *BaseArg = nullptr);
-  Arg(const Option Opt, StringRef Spelling, unsigned Index,
-      const char *Value0, const Arg *BaseArg = nullptr);
-  Arg(const Option Opt, StringRef Spelling, unsigned Index,
-      const char *Value0, const char *Value1, const Arg *BaseArg = nullptr);
+  LLVM_ABI Arg(const Option Opt, StringRef Spelling, unsigned Index,
+               const Arg *BaseArg = nullptr);
+  LLVM_ABI Arg(const Option Opt, StringRef Spelling, unsigned Index,
+               const char *Value0, const Arg *BaseArg = nullptr);
+  LLVM_ABI Arg(const Option Opt, StringRef Spelling, unsigned Index,
+               const char *Value0, const char *Value1,
+               const Arg *BaseArg = nullptr);
   Arg(const Arg &) = delete;
   Arg &operator=(const Arg &) = delete;
-  ~Arg();
+  LLVM_ABI ~Arg();
 
   const Option &getOption() const { return Opt; }
 
@@ -134,23 +136,23 @@ class Arg {
   }
 
   /// Append the argument onto the given array as strings.
-  void render(const ArgList &Args, ArgStringList &Output) const;
+  LLVM_ABI void render(const ArgList &Args, ArgStringList &Output) const;
 
   /// Append the argument, render as an input, onto the given
   /// array as strings.
   ///
   /// The distinction is that some options only render their values
   /// when rendered as a input (e.g., Xlinker).
-  void renderAsInput(const ArgList &Args, ArgStringList &Output) const;
+  LLVM_ABI void renderAsInput(const ArgList &Args, ArgStringList &Output) const;
 
-  void print(raw_ostream &O) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &O) const;
+  LLVM_ABI void dump() const;
 
   /// Return a formatted version of the argument and its values, for
   /// diagnostics. Since this is for diagnostics, if this Arg was produced
   /// through an alias, this returns the string representation of the alias
   /// that the user wrote.
-  std::string getAsString(const ArgList &Args) const;
+  LLVM_ABI std::string getAsString(const ArgList &Args) const;
 };
 
 } // end namespace opt
diff --git a/external/llvm-project/llvm/include/llvm/Option/ArgList.h b/external/llvm-project/llvm/include/llvm/Option/ArgList.h
index 09812f976d01..313164bc2968 100644
--- a/external/llvm-project/llvm/include/llvm/Option/ArgList.h
+++ b/external/llvm-project/llvm/include/llvm/Option/ArgList.h
@@ -11,14 +11,15 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/OptSpecifier.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Support/Compiler.h"
 #include <algorithm>
 #include <cstddef>
 #include <initializer_list>
@@ -139,7 +140,7 @@ class ArgList {
 
   /// Get the range of indexes in which options with the specified IDs might
   /// reside, or (0, 0) if there are no such options.
-  OptRange getRange(std::initializer_list<OptSpecifier> Ids) const;
+  LLVM_ABI OptRange getRange(std::initializer_list<OptSpecifier> Ids) const;
 
 protected:
   // Make the default special members protected so they won't be used to slice
@@ -178,7 +179,7 @@ class ArgList {
   /// @{
 
   /// append - Append \p A to the arg list.
-  void append(Arg *A);
+  LLVM_ABI void append(Arg *A);
 
   const arglist_type &getArgs() const { return Args; }
 
@@ -227,7 +228,7 @@ class ArgList {
   /// @{
 
   /// eraseArg - Remove any option matching \p Id.
-  void eraseArg(OptSpecifier Id);
+  LLVM_ABI void eraseArg(OptSpecifier Id);
 
   /// @}
   /// @name Arg Access
@@ -284,11 +285,12 @@ class ArgList {
   /// @{
 
   /// getLastArgValue - Return the value of the last argument, or a default.
-  StringRef getLastArgValue(OptSpecifier Id, StringRef Default = "") const;
+  LLVM_ABI StringRef getLastArgValue(OptSpecifier Id,
+                                     StringRef Default = "") const;
 
   /// getAllArgValues - Get the values of all instances of the given argument
   /// as strings.
-  std::vector<std::string> getAllArgValues(OptSpecifier Id) const;
+  LLVM_ABI std::vector<std::string> getAllArgValues(OptSpecifier Id) const;
 
   /// @}
   /// @name Translation Utilities
@@ -298,20 +300,21 @@ class ArgList {
   /// true if the option is present, false if the negation is present, and
   /// \p Default if neither option is given. If both the option and its
   /// negation are present, the last one wins.
-  bool hasFlag(OptSpecifier Pos, OptSpecifier Neg, bool Default) const;
-  bool hasFlagNoClaim(OptSpecifier Pos, OptSpecifier Neg, bool Default) const;
+  LLVM_ABI bool hasFlag(OptSpecifier Pos, OptSpecifier Neg, bool Default) const;
+  LLVM_ABI bool hasFlagNoClaim(OptSpecifier Pos, OptSpecifier Neg,
+                               bool Default) const;
 
   /// hasFlag - Given an option \p Pos, an alias \p PosAlias and its negative
   /// form \p Neg, return true if the option or its alias is present, false if
   /// the negation is present, and \p Default if none of the options are
   /// given. If multiple options are present, the last one wins.
-  bool hasFlag(OptSpecifier Pos, OptSpecifier PosAlias, OptSpecifier Neg,
-               bool Default) const;
+  LLVM_ABI bool hasFlag(OptSpecifier Pos, OptSpecifier PosAlias,
+                        OptSpecifier Neg, bool Default) const;
 
   /// Given an option Pos and its negative form Neg, render the option if Pos is
   /// present.
-  void addOptInFlag(ArgStringList &Output, OptSpecifier Pos,
-                    OptSpecifier Neg) const;
+  LLVM_ABI void addOptInFlag(ArgStringList &Output, OptSpecifier Pos,
+                             OptSpecifier Neg) const;
   /// Render the option if Neg is present.
   void addOptOutFlag(ArgStringList &Output, OptSpecifier Pos,
                      OptSpecifier Neg) const {
@@ -331,18 +334,21 @@ class ArgList {
 
   /// AddAllArgsExcept - Render all arguments matching any of the given ids
   /// and not matching any of the excluded ids.
-  void AddAllArgsExcept(ArgStringList &Output, ArrayRef<OptSpecifier> Ids,
-                        ArrayRef<OptSpecifier> ExcludeIds) const;
+  LLVM_ABI void AddAllArgsExcept(ArgStringList &Output,
+                                 ArrayRef<OptSpecifier> Ids,
+                                 ArrayRef<OptSpecifier> ExcludeIds) const;
   /// Render all arguments matching any of the given ids.
-  void addAllArgs(ArgStringList &Output, ArrayRef<OptSpecifier> Ids) const;
+  LLVM_ABI void addAllArgs(ArgStringList &Output,
+                           ArrayRef<OptSpecifier> Ids) const;
 
   /// AddAllArgs - Render all arguments matching the given ids.
-  void AddAllArgs(ArgStringList &Output, OptSpecifier Id0) const;
+  LLVM_ABI void AddAllArgs(ArgStringList &Output, OptSpecifier Id0) const;
 
   /// AddAllArgValues - Render the argument values of all arguments
   /// matching the given ids.
-  void AddAllArgValues(ArgStringList &Output, OptSpecifier Id0,
-                       OptSpecifier Id1 = 0U, OptSpecifier Id2 = 0U) const;
+  LLVM_ABI void AddAllArgValues(ArgStringList &Output, OptSpecifier Id0,
+                                OptSpecifier Id1 = 0U,
+                                OptSpecifier Id2 = 0U) const;
 
   /// AddAllArgsTranslated - Render all the arguments matching the
   /// given ids, but forced to separate args and using the provided
@@ -350,13 +356,13 @@ class ArgList {
   ///
   /// \param Joined - If true, render the argument as joined with
   /// the option specifier.
-  void AddAllArgsTranslated(ArgStringList &Output, OptSpecifier Id0,
-                            const char *Translation,
-                            bool Joined = false) const;
+  LLVM_ABI void AddAllArgsTranslated(ArgStringList &Output, OptSpecifier Id0,
+                                     const char *Translation,
+                                     bool Joined = false) const;
 
   /// ClaimAllArgs - Claim all arguments which match the given
   /// option id.
-  void ClaimAllArgs(OptSpecifier Id0) const;
+  LLVM_ABI void ClaimAllArgs(OptSpecifier Id0) const;
 
   template <typename... OptSpecifiers>
   void claimAllArgs(OptSpecifiers... Ids) const {
@@ -366,7 +372,7 @@ class ArgList {
 
   /// ClaimAllArgs - Claim all arguments.
   ///
-  void ClaimAllArgs() const;
+  LLVM_ABI void ClaimAllArgs() const;
   /// @}
   /// @name Arg Synthesis
   /// @{
@@ -381,16 +387,16 @@ class ArgList {
 
   /// Create an arg string for (\p LHS + \p RHS), reusing the
   /// string at \p Index if possible.
-  const char *GetOrMakeJoinedArgString(unsigned Index, StringRef LHS,
-                                        StringRef RHS) const;
+  LLVM_ABI const char *GetOrMakeJoinedArgString(unsigned Index, StringRef LHS,
+                                                StringRef RHS) const;
 
-  void print(raw_ostream &O) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &O) const;
+  LLVM_ABI void dump() const;
 
   /// @}
 };
 
-class InputArgList final : public ArgList {
+class LLVM_ABI InputArgList final : public ArgList {
 private:
   /// List of argument strings used by the contained Args.
   ///
@@ -463,7 +469,7 @@ class InputArgList final : public ArgList {
 
 /// DerivedArgList - An ordered collection of driver arguments,
 /// whose storage may be in another argument list.
-class DerivedArgList final : public ArgList {
+class LLVM_ABI DerivedArgList final : public ArgList {
   const InputArgList &BaseArgs;
 
   /// The list of arguments we synthesized.
diff --git a/external/llvm-project/llvm/include/llvm/Option/OptSpecifier.h b/external/llvm-project/llvm/include/llvm/Option/OptSpecifier.h
index 7a5fcfb18b38..dc6acae7fc00 100644
--- a/external/llvm-project/llvm/include/llvm/Option/OptSpecifier.h
+++ b/external/llvm-project/llvm/include/llvm/Option/OptSpecifier.h
@@ -22,7 +22,7 @@ class OptSpecifier {
   OptSpecifier() = default;
   explicit OptSpecifier(bool) = delete;
   /*implicit*/ OptSpecifier(unsigned ID) : ID(ID) {}
-  /*implicit*/ OptSpecifier(const Option *Opt);
+  /*implicit*/ LLVM_ABI OptSpecifier(const Option *Opt);
 
   bool isValid() const { return ID != 0; }
 
diff --git a/external/llvm-project/llvm/include/llvm/Option/OptTable.h b/external/llvm-project/llvm/include/llvm/Option/OptTable.h
index 61a58aa304ec..df42ee341ee5 100644
--- a/external/llvm-project/llvm/include/llvm/Option/OptTable.h
+++ b/external/llvm-project/llvm/include/llvm/Option/OptTable.h
@@ -14,6 +14,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringTable.h"
 #include "llvm/Option/OptSpecifier.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/StringSaver.h"
 #include <cassert>
 #include <string>
@@ -50,7 +51,7 @@ class Visibility {
 /// be needed at runtime; the OptTable class maintains enough information to
 /// parse command lines without instantiating Options, while letting other
 /// parts of the driver still use Option instances where convenient.
-class OptTable {
+class LLVM_ABI OptTable {
 public:
   /// Entry for a single option instance in the option data table.
   struct Info {
@@ -425,9 +426,9 @@ class OptTable {
 /// Specialization of OptTable
 class GenericOptTable : public OptTable {
 protected:
-  GenericOptTable(const StringTable &StrTable,
-                  ArrayRef<StringTable::Offset> PrefixesTable,
-                  ArrayRef<Info> OptionInfos, bool IgnoreCase = false);
+  LLVM_ABI GenericOptTable(const StringTable &StrTable,
+                           ArrayRef<StringTable::Offset> PrefixesTable,
+                           ArrayRef<Info> OptionInfos, bool IgnoreCase = false);
 };
 
 class PrecomputedOptTable : public OptTable {
diff --git a/external/llvm-project/llvm/include/llvm/Option/Option.h b/external/llvm-project/llvm/include/llvm/Option/Option.h
index a0563da15c8e..51c330a90813 100644
--- a/external/llvm-project/llvm/include/llvm/Option/Option.h
+++ b/external/llvm-project/llvm/include/llvm/Option/Option.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Option/OptSpecifier.h"
 #include "llvm/Option/OptTable.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
 
@@ -81,7 +82,7 @@ class Option {
   const OptTable *Owner;
 
 public:
-  Option(const OptTable::Info *Info, const OptTable *Owner);
+  LLVM_ABI Option(const OptTable::Info *Info, const OptTable *Owner);
 
   bool isValid() const {
     return Info != nullptr;
@@ -213,7 +214,7 @@ class Option {
   /// Note that matches against options which are an alias should never be
   /// done -- aliases do not participate in matching and so such a query will
   /// always be false.
-  bool matches(OptSpecifier ID) const;
+  LLVM_ABI bool matches(OptSpecifier ID) const;
 
   /// Potentially accept the current argument, returning a new Arg instance,
   /// or 0 if the option does not accept this argument (or the argument is
@@ -227,16 +228,17 @@ class Option {
   /// underlying storage to represent a Joined argument.
   /// \p GroupedShortOption If true, we are handling the fallback case of
   /// parsing a prefix of the current argument as a short option.
-  std::unique_ptr<Arg> accept(const ArgList &Args, StringRef CurArg,
-                              bool GroupedShortOption, unsigned &Index) const;
+  LLVM_ABI std::unique_ptr<Arg> accept(const ArgList &Args, StringRef CurArg,
+                                       bool GroupedShortOption,
+                                       unsigned &Index) const;
 
 private:
   std::unique_ptr<Arg> acceptInternal(const ArgList &Args, StringRef CurArg,
                                       unsigned &Index) const;
 
 public:
-  void print(raw_ostream &O, bool AddNewLine = true) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &O, bool AddNewLine = true) const;
+  LLVM_ABI void dump() const;
 };
 
 } // end namespace opt
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index 4fc013344319..d1230b0ba7c5 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -71,13 +71,13 @@ enum class coveragemap_error {
   invalid_or_missing_arch_specifier
 };
 
-const std::error_category &coveragemap_category();
+LLVM_ABI const std::error_category &coveragemap_category();
 
 inline std::error_code make_error_code(coveragemap_error E) {
   return std::error_code(static_cast<int>(E), coveragemap_category());
 }
 
-class CoverageMapError : public ErrorInfo<CoverageMapError> {
+class LLVM_ABI CoverageMapError : public ErrorInfo<CoverageMapError> {
 public:
   CoverageMapError(coveragemap_error Err, const Twine &ErrStr = Twine())
       : Err(Err), Msg(ErrStr.str()) {
@@ -210,11 +210,11 @@ class CounterExpressionBuilder {
   ArrayRef<CounterExpression> getExpressions() const { return Expressions; }
 
   /// Return a counter that represents the expression that adds LHS and RHS.
-  Counter add(Counter LHS, Counter RHS, bool Simplify = true);
+  LLVM_ABI Counter add(Counter LHS, Counter RHS, bool Simplify = true);
 
   /// Return a counter that represents the expression that subtracts RHS from
   /// LHS.
-  Counter subtract(Counter LHS, Counter RHS, bool Simplify = true);
+  LLVM_ABI Counter subtract(Counter LHS, Counter RHS, bool Simplify = true);
 
   /// K to V map. K will be Counter in most cases. V may be Counter or
   /// Expression.
@@ -222,7 +222,7 @@ class CounterExpressionBuilder {
 
   /// \return A counter equivalent to \C, with each term in its
   /// expression replaced with term from \p Map.
-  Counter subst(Counter C, const SubstMap &Map);
+  LLVM_ABI Counter subst(Counter C, const SubstMap &Map);
 };
 
 using LineColPair = std::pair<unsigned, unsigned>;
@@ -473,7 +473,7 @@ struct MCDCRecord {
 
   // Compare executed test vectors against each other to find an independence
   // pairs for each condition.  This processing takes the most time.
-  void findIndependencePairs();
+  LLVM_ABI void findIndependencePairs();
 
   const CounterMappingRegion &getDecisionRegion() const { return Region; }
   unsigned getNumConditions() const {
@@ -665,7 +665,8 @@ class TVIdxBuilder {
   /// \param NextIDs The list of {FalseID, TrueID} indexed by ID
   ///        The first element [0] should be the root node.
   /// \param Offset Offset of index to final decisions.
-  TVIdxBuilder(const SmallVectorImpl<ConditionIDs> &NextIDs, int Offset = 0);
+  LLVM_ABI TVIdxBuilder(const SmallVectorImpl<ConditionIDs> &NextIDs,
+                        int Offset = 0);
 };
 } // namespace mcdc
 
@@ -684,21 +685,21 @@ class CounterMappingContext {
   void setCounts(ArrayRef<uint64_t> Counts) { CounterValues = Counts; }
   void setBitmap(BitVector &&Bitmap_) { Bitmap = std::move(Bitmap_); }
 
-  void dump(const Counter &C, raw_ostream &OS) const;
+  LLVM_ABI void dump(const Counter &C, raw_ostream &OS) const;
   void dump(const Counter &C) const { dump(C, dbgs()); }
 
   /// Return the number of times that a region of code associated with this
   /// counter was executed.
-  Expected<int64_t> evaluate(const Counter &C) const;
+  LLVM_ABI Expected<int64_t> evaluate(const Counter &C) const;
 
   /// Return an MCDC record that indicates executed test vectors and condition
   /// pairs.
-  Expected<MCDCRecord>
+  LLVM_ABI Expected<MCDCRecord>
   evaluateMCDCRegion(const CounterMappingRegion &Region,
                      ArrayRef<const CounterMappingRegion *> Branches,
                      bool IsVersion11);
 
-  unsigned getMaxCounterID(const Counter &C) const;
+  LLVM_ABI unsigned getMaxCounterID(const Counter &C) const;
 };
 
 /// Code coverage information for a single function.
@@ -761,7 +762,7 @@ class FunctionRecordIterator
   StringRef Filename;
 
   /// Skip records whose primary file is not \c Filename.
-  void skipOtherFiles();
+  LLVM_ABI void skipOtherFiles();
 
 public:
   FunctionRecordIterator(ArrayRef<FunctionRecord> Records_,
@@ -990,24 +991,29 @@ class CoverageMapping {
   // Load coverage records from readers.
   static Error loadFromReaders(
       ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
-      IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage);
+      std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+          &ProfileReader,
+      CoverageMapping &Coverage);
 
   // Load coverage records from file.
   static Error
   loadFromFile(StringRef Filename, StringRef Arch, StringRef CompilationDir,
-               IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage,
-               bool &DataFound,
+               std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+                   &ProfileReader,
+               CoverageMapping &Coverage, bool &DataFound,
                SmallVectorImpl<object::BuildID> *FoundBinaryIDs = nullptr);
 
   /// Add a function record corresponding to \p Record.
-  Error loadFunctionRecord(const CoverageMappingRecord &Record,
-                           IndexedInstrProfReader &ProfileReader);
+  Error loadFunctionRecord(
+      const CoverageMappingRecord &Record,
+      const std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+          &ProfileReader);
 
   /// Look up the indices for function records which are at least partially
   /// defined in the specified file. This is guaranteed to return a superset of
   /// such records: extra records not in the file may be included if there is
   /// a hash collision on the filename. Clients must be robust to collisions.
-  ArrayRef<unsigned>
+  LLVM_ABI ArrayRef<unsigned>
   getImpreciseRecordIndicesForFilename(StringRef Filename) const;
 
 public:
@@ -1015,17 +1021,18 @@ class CoverageMapping {
   CoverageMapping &operator=(const CoverageMapping &) = delete;
 
   /// Load the coverage mapping using the given readers.
-  static Expected<std::unique_ptr<CoverageMapping>>
+  LLVM_ABI static Expected<std::unique_ptr<CoverageMapping>>
   load(ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
-       IndexedInstrProfReader &ProfileReader);
+       std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+           &ProfileReader);
 
   /// Load the coverage mapping from the given object files and profile. If
   /// \p Arches is non-empty, it must specify an architecture for each object.
   /// Ignores non-instrumented object files unless all are not instrumented.
-  static Expected<std::unique_ptr<CoverageMapping>>
-  load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename,
-       vfs::FileSystem &FS, ArrayRef<StringRef> Arches = {},
-       StringRef CompilationDir = "",
+  LLVM_ABI static Expected<std::unique_ptr<CoverageMapping>>
+  load(ArrayRef<StringRef> ObjectFilenames,
+       std::optional<StringRef> ProfileFilename, vfs::FileSystem &FS,
+       ArrayRef<StringRef> Arches = {}, StringRef CompilationDir = "",
        const object::BuildIDFetcher *BIDFetcher = nullptr,
        bool CheckBinaryIDs = false);
 
@@ -1045,20 +1052,22 @@ class CoverageMapping {
 
   /// Returns a lexicographically sorted, unique list of files that are
   /// covered.
-  std::vector<StringRef> getUniqueSourceFiles() const;
+  LLVM_ABI std::vector<StringRef> getUniqueSourceFiles() const;
 
   /// Get the coverage for a particular file.
   ///
   /// The given filename must be the name as recorded in the coverage
   /// information. That is, only names returned from getUniqueSourceFiles will
   /// yield a result.
-  CoverageData getCoverageForFile(StringRef Filename) const;
+  LLVM_ABI CoverageData getCoverageForFile(StringRef Filename) const;
 
   /// Get the coverage for a particular function.
-  CoverageData getCoverageForFunction(const FunctionRecord &Function) const;
+  LLVM_ABI CoverageData
+  getCoverageForFunction(const FunctionRecord &Function) const;
 
   /// Get the coverage for an expansion within a coverage set.
-  CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion) const;
+  LLVM_ABI CoverageData
+  getCoverageForExpansion(const ExpansionRecord &Expansion) const;
 
   /// Gets all of the functions covered by this profile.
   iterator_range<FunctionRecordIterator> getCoveredFunctions() const {
@@ -1079,7 +1088,7 @@ class CoverageMapping {
   ///
   /// Every instantiation group in a program is attributed to exactly one file:
   /// the file in which the definition for the common function begins.
-  std::vector<InstantiationGroup>
+  LLVM_ABI std::vector<InstantiationGroup>
   getInstantiationGroups(StringRef Filename) const;
 };
 
@@ -1096,8 +1105,9 @@ class LineCoverageStats {
   LineCoverageStats() = default;
 
 public:
-  LineCoverageStats(ArrayRef<const CoverageSegment *> LineSegments,
-                    const CoverageSegment *WrappedSegment, unsigned Line);
+  LLVM_ABI LineCoverageStats(ArrayRef<const CoverageSegment *> LineSegments,
+                             const CoverageSegment *WrappedSegment,
+                             unsigned Line);
 
   uint64_t getExecutionCount() const { return ExecutionCount; }
 
@@ -1136,7 +1146,7 @@ class LineCoverageIterator
 
   const LineCoverageStats &operator*() const { return Stats; }
 
-  LineCoverageIterator &operator++();
+  LLVM_ABI LineCoverageIterator &operator++();
 
   LineCoverageIterator getEnd() const {
     auto EndIt = *this;
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
index 886b4d3d6894..e91ba9147a74 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ProfileData/Coverage/CoverageMapping.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cstddef>
@@ -46,7 +47,7 @@ class CoverageMappingIterator {
   CoverageMappingRecord Record;
   coveragemap_error ReadErr;
 
-  void increment();
+  LLVM_ABI void increment();
 
 public:
   using iterator_category = std::input_iterator_tag;
@@ -112,10 +113,10 @@ class RawCoverageReader {
 
   RawCoverageReader(StringRef Data) : Data(Data) {}
 
-  Error readULEB128(uint64_t &Result);
-  Error readIntMax(uint64_t &Result, uint64_t MaxPlus1);
-  Error readSize(uint64_t &Result);
-  Error readString(StringRef &Result);
+  LLVM_ABI Error readULEB128(uint64_t &Result);
+  LLVM_ABI Error readIntMax(uint64_t &Result, uint64_t MaxPlus1);
+  LLVM_ABI Error readSize(uint64_t &Result);
+  LLVM_ABI Error readString(StringRef &Result);
 };
 
 /// Checks if the given coverage mapping data is exported for
@@ -125,7 +126,7 @@ class RawCoverageMappingDummyChecker : public RawCoverageReader {
   RawCoverageMappingDummyChecker(StringRef MappingData)
       : RawCoverageReader(MappingData) {}
 
-  Expected<bool> isDummy();
+  LLVM_ABI Expected<bool> isDummy();
 };
 
 /// Reader for the raw coverage mapping data.
@@ -149,7 +150,7 @@ class RawCoverageMappingReader : public RawCoverageReader {
   RawCoverageMappingReader &
   operator=(const RawCoverageMappingReader &) = delete;
 
-  Error read();
+  LLVM_ABI Error read();
 
 private:
   Error decodeCounter(unsigned Value, Counter &C);
@@ -161,7 +162,7 @@ class RawCoverageMappingReader : public RawCoverageReader {
 
 /// Reader for the coverage mapping data that is emitted by the
 /// frontend and stored in an object file.
-class BinaryCoverageReader : public CoverageMappingReader {
+class LLVM_ABI BinaryCoverageReader : public CoverageMappingReader {
 public:
   struct ProfileMappingRecord {
     CovMapVersion Version;
@@ -245,7 +246,7 @@ class RawCoverageFilenamesReader : public RawCoverageReader {
   RawCoverageFilenamesReader &
   operator=(const RawCoverageFilenamesReader &) = delete;
 
-  Error read(CovMapVersion Version);
+  LLVM_ABI Error read(CovMapVersion Version);
 };
 
 } // end namespace coverage
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
index 02848deaba9d..5341b9879a9f 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ProfileData/Coverage/CoverageMapping.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -30,11 +31,11 @@ class CoverageFilenamesSectionWriter {
   ArrayRef<std::string> Filenames;
 
 public:
-  CoverageFilenamesSectionWriter(ArrayRef<std::string> Filenames);
+  LLVM_ABI CoverageFilenamesSectionWriter(ArrayRef<std::string> Filenames);
 
   /// Write encoded filenames to the given output stream. If \p Compress is
   /// true, attempt to compress the filenames.
-  void write(raw_ostream &OS, bool Compress = true);
+  LLVM_ABI void write(raw_ostream &OS, bool Compress = true);
 };
 
 /// Writer for instrumentation based coverage mapping data.
@@ -51,7 +52,7 @@ class CoverageMappingWriter {
         MappingRegions(MappingRegions) {}
 
   /// Write encoded coverage mapping data to the given output stream.
-  void write(raw_ostream &OS);
+  LLVM_ABI void write(raw_ostream &OS);
 };
 
 /// Writer for the coverage mapping testing format.
@@ -70,7 +71,7 @@ class TestingFormatWriter {
         CoverageRecordsData(CoverageRecordsData) {}
 
   /// Encode to the given output stream.
-  void
+  LLVM_ABI void
   write(raw_ostream &OS,
         TestingFormatVersion Version = TestingFormatVersion::CurrentVersion);
 };
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/DataAccessProf.h b/external/llvm-project/llvm/include/llvm/ProfileData/DataAccessProf.h
index c0f0c6d9c9fc..f410096d83f5 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/DataAccessProf.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/DataAccessProf.h
@@ -24,6 +24,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/StringSaver.h"
 
@@ -138,33 +139,35 @@ class DataAccessProfData {
   /// - Serialized strings.
   /// - The encoded hashes.
   /// - Records.
-  Error serialize(ProfOStream &OS) const;
+  LLVM_ABI Error serialize(ProfOStream &OS) const;
 
   /// Deserialize this class from the given buffer.
-  Error deserialize(const unsigned char *&Ptr);
+  LLVM_ABI Error deserialize(const unsigned char *&Ptr);
 
   /// Returns a profile record for \p SymbolID, or std::nullopt if there
   /// isn't a record. Internally, this function will canonicalize the symbol
   /// name before the lookup.
-  std::optional<DataAccessProfRecord>
+  LLVM_ABI std::optional<DataAccessProfRecord>
   getProfileRecord(const SymbolHandleRef SymID) const;
 
   /// Returns true if \p SymID is seen in profiled binaries and cold.
-  bool isKnownColdSymbol(const SymbolHandleRef SymID) const;
+  LLVM_ABI bool isKnownColdSymbol(const SymbolHandleRef SymID) const;
 
   /// Methods to set symbolized data access profile. Returns error if
   /// duplicated symbol names or content hashes are seen. The user of this
   /// class should aggregate counters that correspond to the same symbol name
   /// or with the same string literal hash before calling 'set*' methods.
-  Error setDataAccessProfile(SymbolHandleRef SymbolID, uint64_t AccessCount);
+  LLVM_ABI Error setDataAccessProfile(SymbolHandleRef SymbolID,
+                                      uint64_t AccessCount);
   /// Similar to the method above, for records with \p Locations representing
   /// the `filename:line` where this symbol shows up. Note because of linker's
   /// merge of identical symbols (e.g., unnamed_addr string literals), one
   /// symbol is likely to have multiple locations.
-  Error setDataAccessProfile(SymbolHandleRef SymbolID, uint64_t AccessCount,
-                             ArrayRef<SourceLocation> Locations);
+  LLVM_ABI Error setDataAccessProfile(SymbolHandleRef SymbolID,
+                                      uint64_t AccessCount,
+                                      ArrayRef<SourceLocation> Locations);
   /// Add a symbol that's seen in the profiled binary without samples.
-  Error addKnownSymbolWithoutSamples(SymbolHandleRef SymbolID);
+  LLVM_ABI Error addKnownSymbolWithoutSamples(SymbolHandleRef SymbolID);
 
   /// The following methods return array reference for various internal data
   /// structures.
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/GCOV.h b/external/llvm-project/llvm/include/llvm/ProfileData/GCOV.h
index c8af71dbf61e..0dc33d062e4f 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/GCOV.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/GCOV.h
@@ -21,6 +21,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
@@ -192,11 +193,11 @@ class GCOVFile {
 public:
   GCOVFile() = default;
 
-  bool readGCNO(GCOVBuffer &Buffer);
-  bool readGCDA(GCOVBuffer &Buffer);
+  LLVM_ABI bool readGCNO(GCOVBuffer &Buffer);
+  LLVM_ABI bool readGCDA(GCOVBuffer &Buffer);
   GCOV::GCOVVersion getVersion() const { return version; }
-  void print(raw_ostream &OS) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &OS) const;
+  LLVM_ABI void dump() const;
 
   std::vector<std::string> filenames;
   StringMap<unsigned> filenameToIdx;
@@ -223,7 +224,7 @@ class GCOVFile {
 struct GCOVArc {
   GCOVArc(GCOVBlock &src, GCOVBlock &dst, uint32_t flags)
       : src(src), dst(dst), flags(flags) {}
-  bool onTree() const;
+  LLVM_ABI bool onTree() const;
 
   GCOVBlock &src;
   GCOVBlock &dst;
@@ -240,18 +241,18 @@ class GCOVFunction {
 
   GCOVFunction(GCOVFile &file) : file(file) {}
 
-  StringRef getName(bool demangle) const;
-  StringRef getFilename() const;
-  uint64_t getEntryCount() const;
-  GCOVBlock &getExitBlock() const;
+  LLVM_ABI StringRef getName(bool demangle) const;
+  LLVM_ABI StringRef getFilename() const;
+  LLVM_ABI uint64_t getEntryCount() const;
+  LLVM_ABI GCOVBlock &getExitBlock() const;
 
   iterator_range<BlockIterator> blocksRange() const {
     return make_range(blocks.begin(), blocks.end());
   }
 
-  void propagateCounts(const GCOVBlock &v, GCOVArc *pred);
-  void print(raw_ostream &OS) const;
-  void dump() const;
+  LLVM_ABI void propagateCounts(const GCOVBlock &v, GCOVArc *pred);
+  LLVM_ABI void print(raw_ostream &OS) const;
+  LLVM_ABI void dump() const;
 
   GCOVFile &file;
   uint32_t ident = 0;
@@ -296,14 +297,14 @@ class GCOVBlock {
     return make_range(succ.begin(), succ.end());
   }
 
-  void print(raw_ostream &OS) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &OS) const;
+  LLVM_ABI void dump() const;
 
-  static uint64_t
+  LLVM_ABI static uint64_t
   augmentOneCycle(GCOVBlock *src,
                   std::vector<std::pair<GCOVBlock *, size_t>> &stack);
-  static uint64_t getCyclesCount(const BlockVector &blocks);
-  static uint64_t getLineCount(const BlockVector &Blocks);
+  LLVM_ABI static uint64_t getCyclesCount(const BlockVector &blocks);
+  LLVM_ABI static uint64_t getLineCount(const BlockVector &Blocks);
 
 public:
   uint32_t number;
@@ -315,8 +316,8 @@ class GCOVBlock {
   GCOVArc *incoming = nullptr;
 };
 
-void gcovOneInput(const GCOV::Options &options, StringRef filename,
-                  StringRef gcno, StringRef gcda, GCOVFile &file);
+LLVM_ABI void gcovOneInput(const GCOV::Options &options, StringRef filename,
+                           StringRef gcno, StringRef gcda, GCOVFile &file);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/IndexedMemProfData.h b/external/llvm-project/llvm/include/llvm/ProfileData/IndexedMemProfData.h
index 9af8755281be..dae0928d50ca 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/IndexedMemProfData.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/IndexedMemProfData.h
@@ -18,6 +18,7 @@
 #include "llvm/ProfileData/DataAccessProf.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/MemProf.h"
+#include "llvm/Support/Compiler.h"
 
 #include <functional>
 #include <optional>
@@ -87,7 +88,7 @@ struct IndexedMemProfData {
 } // namespace memprof
 
 // Write the MemProf data to OS.
-Error writeMemProf(
+LLVM_ABI Error writeMemProf(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
     std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData,
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProf.h b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProf.h
index 544a59df43ed..f53602424b58 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProf.h
@@ -69,18 +69,18 @@ struct PatchItem {
 // back patching.
 class ProfOStream {
 public:
-  ProfOStream(raw_fd_ostream &FD);
-  ProfOStream(raw_string_ostream &STR);
+  LLVM_ABI ProfOStream(raw_fd_ostream &FD);
+  LLVM_ABI ProfOStream(raw_string_ostream &STR);
 
-  [[nodiscard]] uint64_t tell() const;
-  void write(uint64_t V);
-  void write32(uint32_t V);
-  void writeByte(uint8_t V);
+  [[nodiscard]] LLVM_ABI uint64_t tell() const;
+  LLVM_ABI void write(uint64_t V);
+  LLVM_ABI void write32(uint32_t V);
+  LLVM_ABI void writeByte(uint8_t V);
 
   // \c patch can only be called when all data is written and flushed.
   // For raw_string_ostream, the patch is done on the target string
   // directly and it won't be reflected in the stream's internal buffer.
-  void patch(ArrayRef<PatchItem> P);
+  LLVM_ABI void patch(ArrayRef<PatchItem> P);
 
   // If \c OS is an instance of \c raw_fd_ostream, this field will be
   // true. Otherwise, \c OS will be an raw_string_ostream.
@@ -104,9 +104,9 @@ inline uint64_t getInstrMaxCountValue() {
 /// The name of the section depends on the object format type \p OF. If
 /// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
 /// be added to the section name (this is the default).
-std::string getInstrProfSectionName(InstrProfSectKind IPSK,
-                                    Triple::ObjectFormatType OF,
-                                    bool AddSegmentInfo = true);
+LLVM_ABI std::string getInstrProfSectionName(InstrProfSectKind IPSK,
+                                             Triple::ObjectFormatType OF,
+                                             bool AddSegmentInfo = true);
 
 /// Return the name profile runtime entry point to do value profiling
 /// for a given site.
@@ -215,62 +215,64 @@ inline StringRef getInstrProfNameSeparator() { return "\01"; }
 
 /// Determines whether module targets a GPU eligable for PGO
 /// instrumentation
-bool isGPUProfTarget(const Module &M);
+LLVM_ABI bool isGPUProfTarget(const Module &M);
 
 /// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
 /// for front-end (Clang, etc) instrumentation.
 /// Return the modified name for function \c F suitable to be
 /// used the key for profile lookup. Variable \c InLTO indicates if this
 /// is called in LTO optimization passes.
-std::string getPGOFuncName(const Function &F, bool InLTO = false,
-                           uint64_t Version = INSTR_PROF_INDEX_VERSION);
+LLVM_ABI std::string
+getPGOFuncName(const Function &F, bool InLTO = false,
+               uint64_t Version = INSTR_PROF_INDEX_VERSION);
 
 /// Return the modified name for a function suitable to be
 /// used the key for profile lookup. The function's original
 /// name is \c RawFuncName and has linkage of type \c Linkage.
 /// The function is defined in module \c FileName.
-std::string getPGOFuncName(StringRef RawFuncName,
-                           GlobalValue::LinkageTypes Linkage,
-                           StringRef FileName,
-                           uint64_t Version = INSTR_PROF_INDEX_VERSION);
+LLVM_ABI std::string
+getPGOFuncName(StringRef RawFuncName, GlobalValue::LinkageTypes Linkage,
+               StringRef FileName, uint64_t Version = INSTR_PROF_INDEX_VERSION);
 
 /// \return the modified name for function \c F suitable to be
 /// used as the key for IRPGO profile lookup. \c InLTO indicates if this is
 /// called from LTO optimization passes.
-std::string getIRPGOFuncName(const Function &F, bool InLTO = false);
+LLVM_ABI std::string getIRPGOFuncName(const Function &F, bool InLTO = false);
 
 /// \return the filename and the function name parsed from the output of
 /// \c getIRPGOFuncName()
-std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName);
+LLVM_ABI std::pair<StringRef, StringRef>
+getParsedIRPGOName(StringRef IRPGOName);
 
 /// Return the name of the global variable used to store a function
 /// name in PGO instrumentation. \c FuncName is the IRPGO function name
 /// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO
 /// function name (returned by \c getPGOFuncName) for front-end instrumentation.
-std::string getPGOFuncNameVarName(StringRef FuncName,
-                                  GlobalValue::LinkageTypes Linkage);
+LLVM_ABI std::string getPGOFuncNameVarName(StringRef FuncName,
+                                           GlobalValue::LinkageTypes Linkage);
 
 /// Create and return the global variable for function name used in PGO
 /// instrumentation. \c FuncName is the IRPGO function name (returned by
 /// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
 /// (returned by \c getPGOFuncName) for front-end instrumentation.
-GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
+LLVM_ABI GlobalVariable *createPGOFuncNameVar(Function &F,
+                                              StringRef PGOFuncName);
 
 /// Create and return the global variable for function name used in PGO
 /// instrumentation. \c FuncName is the IRPGO function name (returned by
 /// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
 /// (returned by \c getPGOFuncName) for front-end instrumentation.
-GlobalVariable *createPGOFuncNameVar(Module &M,
-                                     GlobalValue::LinkageTypes Linkage,
-                                     StringRef PGOFuncName);
+LLVM_ABI GlobalVariable *createPGOFuncNameVar(Module &M,
+                                              GlobalValue::LinkageTypes Linkage,
+                                              StringRef PGOFuncName);
 
 /// Return the initializer in string of the PGO name var \c NameVar.
-StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
+LLVM_ABI StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
 
 /// Given a PGO function name, remove the filename prefix and return
 /// the original (static) function name.
-StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
-                                   StringRef FileName = "<unknown>");
+LLVM_ABI StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
+                                            StringRef FileName = "<unknown>");
 
 /// Given a vector of strings (names of global objects like functions or,
 /// virtual tables) \c NameStrs, the method generates a combined string \c
@@ -281,28 +283,31 @@ StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
 ///  third field is the uncompressed strings; otherwise it is the
 /// compressed string. When the string compression is off, the
 /// second field will have value zero.
-Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
-                                     bool doCompression, std::string &Result);
+LLVM_ABI Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
+                                              bool doCompression,
+                                              std::string &Result);
 
 /// Produce \c Result string with the same format described above. The input
 /// is vector of PGO function name variables that are referenced.
 /// The global variable element in 'NameVars' is a string containing the pgo
 /// name of a function. See `createPGOFuncNameVar` that creates these global
 /// variables.
-Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
-                                std::string &Result, bool doCompression = true);
+LLVM_ABI Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
+                                         std::string &Result,
+                                         bool doCompression = true);
 
-Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
-                           std::string &Result, bool doCompression);
+LLVM_ABI Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
+                                    std::string &Result, bool doCompression);
 
 /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
 /// set in IR PGO compilation.
-bool isIRPGOFlagSet(const Module *M);
+LLVM_ABI bool isIRPGOFlagSet(const Module *M);
 
 /// Check if we can safely rename this Comdat function. Instances of the same
 /// comdat function may have different control flows thus can not share the
 /// same counter variable.
-bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
+LLVM_ABI bool canRenameComdatFunc(const Function &F,
+                                  bool CheckAddressTaken = false);
 
 enum InstrProfValueKind : uint32_t {
 #define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
@@ -312,23 +317,24 @@ enum InstrProfValueKind : uint32_t {
 /// Get the value profile data for value site \p SiteIdx from \p InstrProfR
 /// and annotate the instruction \p Inst with the value profile meta data.
 /// Annotate up to \p MaxMDCount (default 3) number of records per value site.
-void annotateValueSite(Module &M, Instruction &Inst,
-                       const InstrProfRecord &InstrProfR,
-                       InstrProfValueKind ValueKind, uint32_t SiteIndx,
-                       uint32_t MaxMDCount = 3);
+LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst,
+                                const InstrProfRecord &InstrProfR,
+                                InstrProfValueKind ValueKind, uint32_t SiteIndx,
+                                uint32_t MaxMDCount = 3);
 
 /// Same as the above interface but using an ArrayRef, as well as \p Sum.
 /// This function will not annotate !prof metadata on the instruction if the
 /// referenced array is empty.
-void annotateValueSite(Module &M, Instruction &Inst,
-                       ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
-                       InstrProfValueKind ValueKind, uint32_t MaxMDCount);
+LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst,
+                                ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
+                                InstrProfValueKind ValueKind,
+                                uint32_t MaxMDCount);
 
 // TODO: Unify metadata name 'PGOFuncName' and 'PGOName', by supporting read
 // of this metadata for backward compatibility and generating 'PGOName' only.
 /// Extract the value profile data from \p Inst and returns them if \p Inst is
 /// annotated with value profile data. Returns an empty vector otherwise.
-SmallVector<InstrProfValueData, 4>
+LLVM_ABI SmallVector<InstrProfValueData, 4>
 getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
                          uint32_t MaxNumValueData, uint64_t &TotalC,
                          bool GetNoICPValue = false);
@@ -338,31 +344,31 @@ inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
 inline StringRef getPGONameMetadataName() { return "PGOName"; }
 
 /// Return the PGOFuncName meta data associated with a function.
-MDNode *getPGOFuncNameMetadata(const Function &F);
+LLVM_ABI MDNode *getPGOFuncNameMetadata(const Function &F);
 
-std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
+LLVM_ABI std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
 
 /// Create the PGOFuncName meta data if PGOFuncName is different from
 /// function's raw name. This should only apply to internal linkage functions
 /// declared by users only.
 /// TODO: Update all callers to 'createPGONameMetadata' and deprecate this
 /// function.
-void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
+LLVM_ABI void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
 
 /// Create the PGOName metadata if a global object's PGO name is different from
 /// its mangled name. This should apply to local-linkage global objects only.
-void createPGONameMetadata(GlobalObject &GO, StringRef PGOName);
+LLVM_ABI void createPGONameMetadata(GlobalObject &GO, StringRef PGOName);
 
 /// Check if we can use Comdat for profile variables. This will eliminate
 /// the duplicated profile variables for Comdat functions.
-bool needsComdatForCounter(const GlobalObject &GV, const Module &M);
+LLVM_ABI bool needsComdatForCounter(const GlobalObject &GV, const Module &M);
 
 /// \c NameStrings is a string composed of one or more possibly encoded
 /// sub-strings. The substrings are separated by `\01` (returned by
 /// InstrProf.h:getInstrProfNameSeparator). This method decodes the string and
 /// calls `NameCallback` for each substring.
-Error readAndDecodeStrings(StringRef NameStrings,
-                           std::function<Error(StringRef)> NameCallback);
+LLVM_ABI Error readAndDecodeStrings(
+    StringRef NameStrings, std::function<Error(StringRef)> NameCallback);
 
 /// An enum describing the attributes of an instrumented profile.
 enum class InstrProfKind {
@@ -388,7 +394,7 @@ enum class InstrProfKind {
   LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/LoopEntriesInstrumentation)
 };
 
-const std::error_category &instrprof_category();
+LLVM_ABI const std::error_category &instrprof_category();
 
 enum class instrprof_error {
   success = 0,
@@ -431,16 +437,17 @@ struct TemporalProfTraceTy {
   /// Use a set of temporal profile traces to create a list of balanced
   /// partitioning function nodes used by BalancedPartitioning to generate a
   /// function order that reduces page faults during startup
-  static void createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces,
-                                    std::vector<BPFunctionNode> &Nodes,
-                                    bool RemoveOutlierUNs = true);
+  LLVM_ABI static void
+  createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces,
+                        std::vector<BPFunctionNode> &Nodes,
+                        bool RemoveOutlierUNs = true);
 };
 
 inline std::error_code make_error_code(instrprof_error E) {
   return std::error_code(static_cast<int>(E), instrprof_category());
 }
 
-class InstrProfError : public ErrorInfo<InstrProfError> {
+class LLVM_ABI InstrProfError : public ErrorInfo<InstrProfError> {
 public:
   InstrProfError(instrprof_error Err, const Twine &ErrStr = Twine())
       : Err(Err), Msg(ErrStr.str()) {
@@ -503,7 +510,7 @@ class InstrProfSymtab {
   // Returns the canonical name of the given PGOName. In a canonical name, all
   // suffixes that begins with "." except ".__uniq." are stripped.
   // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`.
-  static StringRef getCanonicalName(StringRef PGOName);
+  LLVM_ABI static StringRef getCanonicalName(StringRef PGOName);
 
 private:
   using AddrIntervalMap =
@@ -577,22 +584,23 @@ class InstrProfSymtab {
   /// only initialize the symtab with reference to the data and
   /// the section base address. The decompression will be delayed
   /// until before it is used. See also \c create(StringRef) method.
-  Error create(object::SectionRef &Section);
+  LLVM_ABI Error create(object::SectionRef &Section);
 
   /// \c NameStrings is a string composed of one of more sub-strings
   ///  encoded in the format described in \c collectPGOFuncNameStrings.
   /// This method is a wrapper to \c readAndDecodeStrings method.
-  Error create(StringRef NameStrings);
+  LLVM_ABI Error create(StringRef NameStrings);
 
   /// Initialize symtab states with function names and vtable names. \c
   /// FuncNameStrings is a string composed of one or more encoded function name
   /// strings, and \c VTableNameStrings composes of one or more encoded vtable
   /// names. This interface is solely used by raw profile reader.
-  Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
+  LLVM_ABI Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
 
   /// Initialize 'this' with the set of vtable names encoded in
   /// \c CompressedVTableNames.
-  Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
+  LLVM_ABI Error
+  initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
 
   /// This interface is used by reader of CoverageMapping test
   /// format.
@@ -604,7 +612,8 @@ class InstrProfSymtab {
   /// indicates if this is called from LTO optimization passes.
   /// A canonical name, removing non-__uniq suffixes, is added if
   /// \c AddCanonical is true.
-  Error create(Module &M, bool InLTO = false, bool AddCanonical = true);
+  LLVM_ABI Error create(Module &M, bool InLTO = false,
+                        bool AddCanonical = true);
 
   /// Create InstrProfSymtab from a set of names iteratable from
   /// \p IterRange. This interface is used by IndexedProfReader.
@@ -667,15 +676,15 @@ class InstrProfSymtab {
   }
 
   /// Return a function's hash, or 0, if the function isn't in this SymTab.
-  uint64_t getFunctionHashFromAddress(uint64_t Address);
+  LLVM_ABI uint64_t getFunctionHashFromAddress(uint64_t Address);
 
   /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
-  uint64_t getVTableHashFromAddress(uint64_t Address);
+  LLVM_ABI uint64_t getVTableHashFromAddress(uint64_t Address);
 
   /// Return function's PGO name from the function name's symbol
   /// address in the object file. If an error occurs, return
   /// an empty string.
-  StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
+  LLVM_ABI StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
 
   /// Return name of functions or global variables from the name's md5 hash
   /// value. If not found, return an empty string.
@@ -702,7 +711,7 @@ class InstrProfSymtab {
   inline StringRef getNameData() const { return Data; }
 
   /// Dump the symbols in this table.
-  void dumpNames(raw_ostream &OS) const;
+  LLVM_ABI void dumpNames(raw_ostream &OS) const;
 };
 
 Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
@@ -814,17 +823,17 @@ struct OverlapStats {
 
   OverlapStats(OverlapStatsLevel L = ProgramLevel) : Level(L) {}
 
-  void dump(raw_fd_ostream &OS) const;
+  LLVM_ABI void dump(raw_fd_ostream &OS) const;
 
   void setFuncInfo(StringRef Name, uint64_t Hash) {
     FuncName = Name;
     FuncHash = Hash;
   }
 
-  Error accumulateCounts(const std::string &BaseFilename,
-                         const std::string &TestFilename, bool IsCS);
-  void addOneMismatch(const CountSumOrPercent &MismatchFunc);
-  void addOneUnique(const CountSumOrPercent &UniqueFunc);
+  LLVM_ABI Error accumulateCounts(const std::string &BaseFilename,
+                                  const std::string &TestFilename, bool IsCS);
+  LLVM_ABI void addOneMismatch(const CountSumOrPercent &MismatchFunc);
+  LLVM_ABI void addOneUnique(const CountSumOrPercent &UniqueFunc);
 
   static inline double score(uint64_t Val1, uint64_t Val2, double Sum1,
                              double Sum2) {
@@ -861,14 +870,15 @@ struct InstrProfValueSiteRecord {
 
   /// Merge data from another InstrProfValueSiteRecord
   /// Optionally scale merged counts by \p Weight.
-  void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
-             function_ref<void(instrprof_error)> Warn);
+  LLVM_ABI void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
+                      function_ref<void(instrprof_error)> Warn);
   /// Scale up value profile data counts by N (Numerator) / D (Denominator).
-  void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
+  LLVM_ABI void scale(uint64_t N, uint64_t D,
+                      function_ref<void(instrprof_error)> Warn);
 
   /// Compute the overlap b/w this record and Input record.
-  void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
-               OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);
+  LLVM_ABI void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
+                        OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);
 };
 
 /// Profiling information for a single function.
@@ -920,18 +930,19 @@ struct InstrProfRecord {
 
   /// Add ValueData for ValueKind at value Site.  We do not support adding sites
   /// out of order.  Site must go up from 0 one by one.
-  void addValueData(uint32_t ValueKind, uint32_t Site,
-                    ArrayRef<InstrProfValueData> VData,
-                    InstrProfSymtab *SymTab);
+  LLVM_ABI void addValueData(uint32_t ValueKind, uint32_t Site,
+                             ArrayRef<InstrProfValueData> VData,
+                             InstrProfSymtab *SymTab);
 
   /// Merge the counts in \p Other into this one.
   /// Optionally scale merged counts by \p Weight.
-  void merge(InstrProfRecord &Other, uint64_t Weight,
-             function_ref<void(instrprof_error)> Warn);
+  LLVM_ABI void merge(InstrProfRecord &Other, uint64_t Weight,
+                      function_ref<void(instrprof_error)> Warn);
 
   /// Scale up profile counts (including value profile data) by
   /// a factor of (N / D).
-  void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
+  LLVM_ABI void scale(uint64_t N, uint64_t D,
+                      function_ref<void(instrprof_error)> Warn);
 
   /// Sort value profile data (per site) by count.
   void sortValueData() {
@@ -950,16 +961,16 @@ struct InstrProfRecord {
   void clearValueData() { ValueData = nullptr; }
 
   /// Compute the sums of all counts and store in Sum.
-  void accumulateCounts(CountSumOrPercent &Sum) const;
+  LLVM_ABI void accumulateCounts(CountSumOrPercent &Sum) const;
 
   /// Compute the overlap b/w this IntrprofRecord and Other.
-  void overlap(InstrProfRecord &Other, OverlapStats &Overlap,
-               OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);
+  LLVM_ABI void overlap(InstrProfRecord &Other, OverlapStats &Overlap,
+                        OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);
 
   /// Compute the overlap of value profile counts.
-  void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
-                            OverlapStats &Overlap,
-                            OverlapStats &FuncLevelOverlap);
+  LLVM_ABI void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
+                                     OverlapStats &Overlap,
+                                     OverlapStats &FuncLevelOverlap);
 
   enum CountPseudoKind {
     NotPseudo = 0,
@@ -1187,15 +1198,15 @@ struct Header {
 
   // Reads a header struct from the buffer. Header fields are in machine native
   // endianness.
-  static Expected<Header> readFromBuffer(const unsigned char *Buffer);
+  LLVM_ABI static Expected<Header> readFromBuffer(const unsigned char *Buffer);
 
   // Returns the size of the header in bytes for all valid fields based on the
   // version. I.e a older version header will return a smaller size.
-  size_t size() const;
+  LLVM_ABI size_t size() const;
 
   // Return the indexed profile version, i.e., the least significant 32 bits
   // in Header.Version.
-  uint64_t getIndexedProfileVersion() const;
+  LLVM_ABI uint64_t getIndexedProfileVersion() const;
 };
 
 // Profile summary data recorded in the profile data file in indexed
@@ -1340,11 +1351,11 @@ struct Header {
 } // end namespace RawInstrProf
 
 // Create the variable for the profile file name.
-void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
+LLVM_ABI void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
 
 // Whether to compress function names in profile records, and filenames in
 // code coverage mappings. Used by the Instrumentation library and unit tests.
-extern cl::opt<bool> DoInstrProfNameCompression;
+LLVM_ABI extern cl::opt<bool> DoInstrProfNameCompression;
 
 } // end namespace llvm
 #endif // LLVM_PROFILEDATA_INSTRPROF_H
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
index ed8642495cd7..d460eb1cdf52 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
@@ -16,6 +16,7 @@
 #include "llvm/Debuginfod/BuildIDFetcher.h"
 #include "llvm/Object/BuildID.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/YAMLTraits.h"
@@ -37,7 +38,7 @@ class InstrProfCorrelator {
   /// correlate.
   enum ProfCorrelatorKind { NONE, DEBUG_INFO, BINARY };
 
-  static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
+  LLVM_ABI static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
   get(StringRef Filename, ProfCorrelatorKind FileKind,
       const object::BuildIDFetcher *BIDFetcher = nullptr,
       const ArrayRef<llvm::object::BuildID> BIs = {});
@@ -52,7 +53,7 @@ class InstrProfCorrelator {
   virtual Error dumpYaml(int MaxWarnings, raw_ostream &OS) = 0;
 
   /// Return the number of ProfileData elements.
-  std::optional<size_t> getDataSize() const;
+  LLVM_ABI std::optional<size_t> getDataSize() const;
 
   /// Return a pointer to the names string that this class constructs.
   const char *getNamesPointer() const { return Names.c_str(); }
@@ -65,9 +66,9 @@ class InstrProfCorrelator {
     return Ctx->CountersSectionEnd - Ctx->CountersSectionStart;
   }
 
-  static const char *FunctionNameAttributeName;
-  static const char *CFGHashAttributeName;
-  static const char *NumCountersAttributeName;
+  LLVM_ABI static const char *FunctionNameAttributeName;
+  LLVM_ABI static const char *CFGHashAttributeName;
+  LLVM_ABI static const char *NumCountersAttributeName;
 
   enum InstrProfCorrelatorKind { CK_32Bit, CK_64Bit };
   InstrProfCorrelatorKind getKind() const { return Kind; }
@@ -75,7 +76,7 @@ class InstrProfCorrelator {
 
 protected:
   struct Context {
-    static llvm::Expected<std::unique_ptr<Context>>
+    LLVM_ABI static llvm::Expected<std::unique_ptr<Context>>
     get(std::unique_ptr<MemoryBuffer> Buffer, const object::ObjectFile &Obj,
         ProfCorrelatorKind FileKind);
     std::unique_ptr<MemoryBuffer> Buffer;
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfData.inc b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfData.inc
index 6d18680f8916..0496f240dc82 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -405,14 +405,14 @@ typedef struct ValueProfRecord {
   /*!
    * Read data from this record and save it to Record.
    */
-  void deserializeTo(InstrProfRecord &Record,
+  LLVM_ABI void deserializeTo(InstrProfRecord &Record,
                      InstrProfSymtab *SymTab);
   /*
    * In-place byte swap:
    * Do byte swap for this instance. \c Old is the original order before
    * the swap, and \c New is the New byte order.
    */
-  void swapBytes(llvm::endianness Old, llvm::endianness New);
+  LLVM_ABI void swapBytes(llvm::endianness Old, llvm::endianness New);
 #endif
 } ValueProfRecord;
 
@@ -449,41 +449,41 @@ typedef struct ValueProfData {
    * Return the total size in bytes of the on-disk value profile data
    * given the data stored in Record.
    */
-  static uint32_t getSize(const InstrProfRecord &Record);
+  LLVM_ABI static uint32_t getSize(const InstrProfRecord &Record);
   /*!
    * Return a pointer to \c ValueProfData instance ready to be streamed.
    */
-  static std::unique_ptr<ValueProfData>
+  LLVM_ABI static std::unique_ptr<ValueProfData>
   serializeFrom(const InstrProfRecord &Record);
   /*!
    * Check the integrity of the record.
    */
-  Error checkIntegrity();
+  LLVM_ABI Error checkIntegrity();
   /*!
    * Return a pointer to \c ValueProfileData instance ready to be read.
    * All data in the instance are properly byte swapped. The input
    * data is assumed to be in little endian order.
    */
-  static Expected<std::unique_ptr<ValueProfData>>
+  LLVM_ABI static Expected<std::unique_ptr<ValueProfData>>
   getValueProfData(const unsigned char *SrcBuffer,
                    const unsigned char *const SrcBufferEnd,
                    llvm::endianness SrcDataEndianness);
   /*!
    * Swap byte order from \c Endianness order to host byte order.
    */
-  void swapBytesToHost(llvm::endianness Endianness);
+  LLVM_ABI void swapBytesToHost(llvm::endianness Endianness);
   /*!
    * Swap byte order from host byte order to \c Endianness order.
    */
-  void swapBytesFromHost(llvm::endianness Endianness);
+  LLVM_ABI void swapBytesFromHost(llvm::endianness Endianness);
   /*!
    * Return the total size of \c ValueProfileData.
    */
-  uint32_t getSize() const { return TotalSize; }
+  LLVM_ABI uint32_t getSize() const { return TotalSize; }
   /*!
    * Read data from this data and save it to \c Record.
    */
-  void deserializeTo(InstrProfRecord &Record,
+  LLVM_ABI void deserializeTo(InstrProfRecord &Record,
                      InstrProfSymtab *SymTab);
   void operator delete(void *ptr) { ::operator delete(ptr); }
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfReader.h b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfReader.h
index 99ea3c1808f5..3b12505d3326 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -24,6 +24,7 @@
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/ProfileData/MemProfSummary.h"
 #include "llvm/ProfileData/MemProfYAML.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/LineIterator.h"
@@ -158,7 +159,7 @@ class InstrProfReader {
   virtual InstrProfSymtab &getSymtab() = 0;
 
   /// Compute the sum of counts and return in Sum.
-  void accumulateCounts(CountSumOrPercent &Sum, bool IsCS);
+  LLVM_ABI void accumulateCounts(CountSumOrPercent &Sum, bool IsCS);
 
 protected:
   std::unique_ptr<InstrProfSymtab> Symtab;
@@ -203,7 +204,7 @@ class InstrProfReader {
 
   /// Factory method to create an appropriately typed reader for the given
   /// instrprof file.
-  static Expected<std::unique_ptr<InstrProfReader>> create(
+  LLVM_ABI static Expected<std::unique_ptr<InstrProfReader>> create(
       const Twine &Path, vfs::FileSystem &FS,
       const InstrProfCorrelator *Correlator = nullptr,
       const object::BuildIDFetcher *BIDFetcher = nullptr,
@@ -211,7 +212,7 @@ class InstrProfReader {
           InstrProfCorrelator::ProfCorrelatorKind::NONE,
       std::function<void(Error)> Warn = nullptr);
 
-  static Expected<std::unique_ptr<InstrProfReader>> create(
+  LLVM_ABI static Expected<std::unique_ptr<InstrProfReader>> create(
       std::unique_ptr<MemoryBuffer> Buffer,
       const InstrProfCorrelator *Correlator = nullptr,
       const object::BuildIDFetcher *BIDFetcher = nullptr,
@@ -242,7 +243,7 @@ class InstrProfReader {
 ///
 /// Each record consists of a function name, a function hash, a number of
 /// counters, and then each counter value, in that order.
-class TextInstrProfReader : public InstrProfReader {
+class LLVM_ABI TextInstrProfReader : public InstrProfReader {
 private:
   /// The profile data file contents.
   std::unique_ptr<MemoryBuffer> DataBuffer;
@@ -535,7 +536,7 @@ class InstrProfLookupTrait {
   static StringRef GetInternalKey(StringRef K) { return K; }
   static StringRef GetExternalKey(StringRef K) { return K; }
 
-  hash_value_type ComputeHash(StringRef K);
+  LLVM_ABI hash_value_type ComputeHash(StringRef K);
 
   static std::pair<offset_type, offset_type>
   ReadKeyDataLength(const unsigned char *&D) {
@@ -552,9 +553,10 @@ class InstrProfLookupTrait {
     return StringRef((const char *)D, N);
   }
 
-  bool readValueProfilingData(const unsigned char *&D,
-                              const unsigned char *const End);
-  data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
+  LLVM_ABI bool readValueProfilingData(const unsigned char *&D,
+                                       const unsigned char *const End);
+  LLVM_ABI data_type ReadData(StringRef K, const unsigned char *D,
+                              offset_type N);
 
   // Used for testing purpose only.
   void setValueProfDataEndianness(llvm::endianness Endianness) {
@@ -718,22 +720,23 @@ class IndexedMemProfReader {
 public:
   IndexedMemProfReader() = default;
 
-  Error deserialize(const unsigned char *Start, uint64_t MemProfOffset);
+  LLVM_ABI Error deserialize(const unsigned char *Start,
+                             uint64_t MemProfOffset);
 
-  Expected<memprof::MemProfRecord>
+  LLVM_ABI Expected<memprof::MemProfRecord>
   getMemProfRecord(const uint64_t FuncNameHash) const;
 
-  DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>>
+  LLVM_ABI DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>>
   getMemProfCallerCalleePairs() const;
 
   // Return the entire MemProf profile.
-  memprof::AllMemProfData getAllMemProfData() const;
+  LLVM_ABI memprof::AllMemProfData getAllMemProfData() const;
 
   memprof::MemProfSummary *getSummary() const { return MemProfSum.get(); }
 };
 
 /// Reader for the indexed binary instrprof format.
-class IndexedInstrProfReader : public InstrProfReader {
+class LLVM_ABI IndexedInstrProfReader : public InstrProfReader {
 private:
   /// The profile data file contents.
   std::unique_ptr<MemoryBuffer> DataBuffer;
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfWriter.h b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfWriter.h
index ee541c465bd1..f339fe2c2a9e 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -23,6 +23,7 @@
 #include "llvm/ProfileData/IndexedMemProfData.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/MemProfSummaryBuilder.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <memory>
@@ -94,25 +95,25 @@ class InstrProfWriter {
   // MemprofGenerateRandomHotness is enabled. The random seed can be either
   // provided by MemprofGenerateRandomHotnessSeed, or if that is 0, one will be
   // generated in the writer using the current time.
-  InstrProfWriter(bool Sparse = false,
-                  uint64_t TemporalProfTraceReservoirSize = 0,
-                  uint64_t MaxTemporalProfTraceLength = 0,
-                  bool WritePrevVersion = false,
-                  memprof::IndexedVersion MemProfVersionRequested =
-                      static_cast<memprof::IndexedVersion>(
-                          memprof::MinimumSupportedVersion),
-                  bool MemProfFullSchema = false,
-                  bool MemprofGenerateRandomHotness = false,
-                  unsigned MemprofGenerateRandomHotnessSeed = 0);
-  ~InstrProfWriter();
+  LLVM_ABI InstrProfWriter(bool Sparse = false,
+                           uint64_t TemporalProfTraceReservoirSize = 0,
+                           uint64_t MaxTemporalProfTraceLength = 0,
+                           bool WritePrevVersion = false,
+                           memprof::IndexedVersion MemProfVersionRequested =
+                               static_cast<memprof::IndexedVersion>(
+                                   memprof::MinimumSupportedVersion),
+                           bool MemProfFullSchema = false,
+                           bool MemprofGenerateRandomHotness = false,
+                           unsigned MemprofGenerateRandomHotnessSeed = 0);
+  LLVM_ABI ~InstrProfWriter();
 
   StringMap<ProfilingData> &getProfileData() { return FunctionData; }
 
   /// Add function counts for the given function. If there are already counts
   /// for this function and the hash and number of counts match, each counter is
   /// summed. Optionally scale counts by \p Weight.
-  void addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
-                 function_ref<void(Error)> Warn);
+  LLVM_ABI void addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
+                          function_ref<void(Error)> Warn);
   void addRecord(NamedInstrProfRecord &&I, function_ref<void(Error)> Warn) {
     addRecord(std::move(I), 1, Warn);
   }
@@ -120,45 +121,47 @@ class InstrProfWriter {
 
   /// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the
   /// total number of temporal profiling traces the source has seen.
-  void addTemporalProfileTraces(SmallVectorImpl<TemporalProfTraceTy> &SrcTraces,
-                                uint64_t SrcStreamSize);
+  LLVM_ABI void
+  addTemporalProfileTraces(SmallVectorImpl<TemporalProfTraceTy> &SrcTraces,
+                           uint64_t SrcStreamSize);
 
   /// Add the entire MemProfData \p Incoming to the writer context.
-  bool addMemProfData(memprof::IndexedMemProfData Incoming,
-                      function_ref<void(Error)> Warn);
+  LLVM_ABI bool addMemProfData(memprof::IndexedMemProfData Incoming,
+                               function_ref<void(Error)> Warn);
 
   // Add a binary id to the binary ids list.
-  void addBinaryIds(ArrayRef<llvm::object::BuildID> BIs);
+  LLVM_ABI void addBinaryIds(ArrayRef<llvm::object::BuildID> BIs);
 
-  void addDataAccessProfData(
+  LLVM_ABI void addDataAccessProfData(
       std::unique_ptr<memprof::DataAccessProfData> DataAccessProfile);
 
   /// Merge existing function counts from the given writer.
-  void mergeRecordsFromWriter(InstrProfWriter &&IPW,
-                              function_ref<void(Error)> Warn);
+  LLVM_ABI void mergeRecordsFromWriter(InstrProfWriter &&IPW,
+                                       function_ref<void(Error)> Warn);
 
   /// Write the profile to \c OS
-  Error write(raw_fd_ostream &OS);
+  LLVM_ABI Error write(raw_fd_ostream &OS);
 
   /// Write the profile to a string output stream \c OS
-  Error write(raw_string_ostream &OS);
+  LLVM_ABI Error write(raw_string_ostream &OS);
 
   /// Write the profile in text format to \c OS
-  Error writeText(raw_fd_ostream &OS);
+  LLVM_ABI Error writeText(raw_fd_ostream &OS);
 
   /// Write temporal profile trace data to the header in text format to \c OS
-  void writeTextTemporalProfTraceData(raw_fd_ostream &OS,
-                                      InstrProfSymtab &Symtab);
+  LLVM_ABI void writeTextTemporalProfTraceData(raw_fd_ostream &OS,
+                                               InstrProfSymtab &Symtab);
 
-  Error validateRecord(const InstrProfRecord &Func);
+  LLVM_ABI Error validateRecord(const InstrProfRecord &Func);
 
   /// Write \c Record in text format to \c OS
-  static void writeRecordInText(StringRef Name, uint64_t Hash,
-                                const InstrProfRecord &Counters,
-                                InstrProfSymtab &Symtab, raw_fd_ostream &OS);
+  LLVM_ABI static void writeRecordInText(StringRef Name, uint64_t Hash,
+                                         const InstrProfRecord &Counters,
+                                         InstrProfSymtab &Symtab,
+                                         raw_fd_ostream &OS);
 
   /// Write the profile, returning the raw data. For testing.
-  std::unique_ptr<MemoryBuffer> writeBuffer();
+  LLVM_ABI std::unique_ptr<MemoryBuffer> writeBuffer();
 
   /// Update the attributes of the current profile from the attributes
   /// specified. An error is returned if IR and FE profiles are mixed.
@@ -206,17 +209,18 @@ class InstrProfWriter {
   }
 
   // Internal interfaces for testing purpose only.
-  void setValueProfDataEndianness(llvm::endianness Endianness);
-  void setOutputSparse(bool Sparse);
+  LLVM_ABI void setValueProfDataEndianness(llvm::endianness Endianness);
+  LLVM_ABI void setOutputSparse(bool Sparse);
   void setMemProfVersionRequested(memprof::IndexedVersion Version) {
     MemProfVersionRequested = Version;
   }
   void setMemProfFullSchema(bool Full) { MemProfFullSchema = Full; }
   // Compute the overlap b/w this object and Other. Program level result is
   // stored in Overlap and function level result is stored in FuncLevelOverlap.
-  void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap,
-                     OverlapStats &FuncLevelOverlap,
-                     const OverlapFuncFilters &FuncFilter);
+  LLVM_ABI void overlapRecord(NamedInstrProfRecord &&Other,
+                              OverlapStats &Overlap,
+                              OverlapStats &FuncLevelOverlap,
+                              const OverlapFuncFilters &FuncFilter);
 
 private:
   void addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I,
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/ItaniumManglingCanonicalizer.h b/external/llvm-project/llvm/include/llvm/ProfileData/ItaniumManglingCanonicalizer.h
index e634f9c223e1..bb99710c092c 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/ItaniumManglingCanonicalizer.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/ItaniumManglingCanonicalizer.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_PROFILEDATA_ITANIUMMANGLINGCANONICALIZER_H
 #define LLVM_PROFILEDATA_ITANIUMMANGLINGCANONICALIZER_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -34,10 +35,10 @@ class StringRef;
 /// different manglings.
 class ItaniumManglingCanonicalizer {
 public:
-  ItaniumManglingCanonicalizer();
+  LLVM_ABI ItaniumManglingCanonicalizer();
   ItaniumManglingCanonicalizer(const ItaniumManglingCanonicalizer &) = delete;
   void operator=(const ItaniumManglingCanonicalizer &) = delete;
-  ~ItaniumManglingCanonicalizer();
+  LLVM_ABI ~ItaniumManglingCanonicalizer();
 
   enum class EquivalenceError {
     Success,
@@ -65,8 +66,8 @@ class ItaniumManglingCanonicalizer {
 
   /// Add an equivalence between \p First and \p Second. Both manglings must
   /// live at least as long as the canonicalizer.
-  EquivalenceError addEquivalence(FragmentKind Kind, StringRef First,
-                                  StringRef Second);
+  LLVM_ABI EquivalenceError addEquivalence(FragmentKind Kind, StringRef First,
+                                           StringRef Second);
 
   using Key = uintptr_t;
 
@@ -78,11 +79,11 @@ class ItaniumManglingCanonicalizer {
   /// ABI mangling.
   ///
   /// The string denoted by Mangling must live as long as the canonicalizer.
-  Key canonicalize(StringRef Mangling);
+  LLVM_ABI Key canonicalize(StringRef Mangling);
 
   /// Find a canonical key for the specified mangling, if one has already been
   /// formed. Otherwise returns Key().
-  Key lookup(StringRef Mangling);
+  LLVM_ABI Key lookup(StringRef Mangling);
 
 private:
   struct Impl;
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/MemProf.h b/external/llvm-project/llvm/include/llvm/ProfileData/MemProf.h
index 683193aa4274..9df4efd8e085 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/MemProf.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/MemProf.h
@@ -23,6 +23,7 @@
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/ProfileData/MemProfData.inc"
 #include "llvm/Support/BLAKE3.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/HashBuilder.h"
@@ -79,10 +80,10 @@ enum class Meta : uint64_t {
 using MemProfSchema = llvm::SmallVector<Meta, static_cast<int>(Meta::Size)>;
 
 // Returns the full schema currently in use.
-MemProfSchema getFullSchema();
+LLVM_ABI MemProfSchema getFullSchema();
 
 // Returns the schema consisting of the fields used for hot cold memory hinting.
-MemProfSchema getHotColdSchema();
+LLVM_ABI MemProfSchema getHotColdSchema();
 
 // Holds the actual MemInfoBlock data with all fields. Contents may be read or
 // written partially by providing an appropriate schema to the serialize and
@@ -396,8 +397,8 @@ struct IndexedAllocationInfo {
       : CSId(CSId), Info(MB) {}
 
   // Returns the size in bytes when this allocation info struct is serialized.
-  size_t serializedSize(const MemProfSchema &Schema,
-                        IndexedVersion Version) const;
+  LLVM_ABI size_t serializedSize(const MemProfSchema &Schema,
+                                 IndexedVersion Version) const;
 
   bool operator==(const IndexedAllocationInfo &Other) const {
     if (Other.Info != Info)
@@ -457,8 +458,8 @@ struct IndexedMemProfRecord {
     AllocSites.append(Other.AllocSites);
   }
 
-  size_t serializedSize(const MemProfSchema &Schema,
-                        IndexedVersion Version) const;
+  LLVM_ABI size_t serializedSize(const MemProfSchema &Schema,
+                                 IndexedVersion Version) const;
 
   bool operator==(const IndexedMemProfRecord &Other) const {
     if (Other.AllocSites != AllocSites)
@@ -471,26 +472,26 @@ struct IndexedMemProfRecord {
 
   // Serializes the memprof records in \p Records to the ostream \p OS based
   // on the schema provided in \p Schema.
-  void serialize(const MemProfSchema &Schema, raw_ostream &OS,
-                 IndexedVersion Version,
-                 llvm::DenseMap<CallStackId, LinearCallStackId>
-                     *MemProfCallStackIndexes = nullptr) const;
+  LLVM_ABI void serialize(const MemProfSchema &Schema, raw_ostream &OS,
+                          IndexedVersion Version,
+                          llvm::DenseMap<CallStackId, LinearCallStackId>
+                              *MemProfCallStackIndexes = nullptr) const;
 
   // Deserializes memprof records from the Buffer.
-  static IndexedMemProfRecord deserialize(const MemProfSchema &Schema,
-                                          const unsigned char *Buffer,
-                                          IndexedVersion Version);
+  LLVM_ABI static IndexedMemProfRecord deserialize(const MemProfSchema &Schema,
+                                                   const unsigned char *Buffer,
+                                                   IndexedVersion Version);
 
   // Convert IndexedMemProfRecord to MemProfRecord.  Callback is used to
   // translate CallStackId to call stacks with frames inline.
-  MemProfRecord toMemProfRecord(
+  LLVM_ABI MemProfRecord toMemProfRecord(
       llvm::function_ref<std::vector<Frame>(const CallStackId)> Callback) const;
 };
 
 // Returns the GUID for the function name after canonicalization. For
 // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are
 // mapped to functions using this GUID.
-GlobalValue::GUID getGUID(const StringRef FunctionName);
+LLVM_ABI GlobalValue::GUID getGUID(const StringRef FunctionName);
 
 // Holds call site information with frame contents inline.
 struct CallSiteInfo {
@@ -551,7 +552,8 @@ struct MemProfRecord {
 // ids in the schema. Subsequent entries are integers which map to memprof::Meta
 // enum class entries. After successfully reading the schema, the pointer is one
 // byte past the schema contents.
-Expected<MemProfSchema> readMemProfSchema(const unsigned char *&Buffer);
+LLVM_ABI Expected<MemProfSchema>
+readMemProfSchema(const unsigned char *&Buffer);
 
 // Trait for reading IndexedMemProfRecord data from the on-disk hash table.
 class RecordLookupTrait {
@@ -835,8 +837,7 @@ struct LineLocation {
   LineLocation(uint32_t L, uint32_t D) : LineOffset(L), Column(D) {}
 
   bool operator<(const LineLocation &O) const {
-    return LineOffset < O.LineOffset ||
-           (LineOffset == O.LineOffset && Column < O.Column);
+    return std::tie(LineOffset, Column) < std::tie(O.LineOffset, O.Column);
   }
 
   bool operator==(const LineLocation &O) const {
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfCommon.h b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfCommon.h
index ff2aeee83436..53be4984caa3 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfCommon.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfCommon.h
@@ -14,6 +14,7 @@
 #define LLVM_PROFILEDATA_MEMPROFCOMMON_H
 
 #include "llvm/IR/ModuleSummaryIndex.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace memprof {
@@ -21,13 +22,14 @@ namespace memprof {
 struct Frame;
 
 /// Return the allocation type for a given set of memory profile values.
-AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity,
-                            uint64_t AllocCount, uint64_t TotalLifetime);
+LLVM_ABI AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity,
+                                     uint64_t AllocCount,
+                                     uint64_t TotalLifetime);
 
 /// Helper to generate a single hash id for a given callstack, used for emitting
 /// matching statistics and useful for uniquing such statistics across modules.
 /// Also used to dedup contexts when computing the summary.
-uint64_t computeFullStackId(ArrayRef<Frame> CallStack);
+LLVM_ABI uint64_t computeFullStackId(ArrayRef<Frame> CallStack);
 
 } // namespace memprof
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfRadixTree.h b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfRadixTree.h
index fd6dce4536c6..35df7a8f16a6 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfRadixTree.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfRadixTree.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ProfileData/IndexedMemProfData.h"
 #include "llvm/ProfileData/MemProf.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace memprof {
@@ -353,6 +354,12 @@ template <typename FrameIdTy> class CallStackRadixTreeBuilder {
     return std::move(CallStackPos);
   }
 };
+
+// Defined in MemProfRadixTree.cpp
+extern template class LLVM_TEMPLATE_ABI CallStackRadixTreeBuilder<FrameId>;
+extern template class LLVM_TEMPLATE_ABI
+    CallStackRadixTreeBuilder<LinearFrameId>;
+
 } // namespace memprof
 } // namespace llvm
 #endif // LLVM_PROFILEDATA_MEMPROFRADIXTREE_H
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfReader.h b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfReader.h
index 3bfcdf0f42cd..4d41d05b1457 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfReader.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfReader.h
@@ -25,6 +25,7 @@
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/ProfileData/MemProfData.inc"
 #include "llvm/ProfileData/MemProfRadixTree.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 
@@ -106,7 +107,7 @@ using CallStackMap = llvm::DenseMap<uint64_t, llvm::SmallVector<uint64_t>>;
 
 // Specializes the MemProfReader class to populate the contents from raw binary
 // memprof profiles from instrumentation based profiling.
-class RawMemProfReader final : public MemProfReader {
+class LLVM_ABI RawMemProfReader final : public MemProfReader {
 public:
   RawMemProfReader(const RawMemProfReader &) = delete;
   RawMemProfReader &operator=(const RawMemProfReader &) = delete;
@@ -217,18 +218,19 @@ class YAMLMemProfReader final : public MemProfReader {
 
   // Return true if the \p DataBuffer starts with "---" indicating it is a YAML
   // file.
-  static bool hasFormat(const MemoryBuffer &DataBuffer);
+  LLVM_ABI static bool hasFormat(const MemoryBuffer &DataBuffer);
   // Wrapper around hasFormat above, reading the file instead of the memory
   // buffer.
-  static bool hasFormat(const StringRef Path);
+  LLVM_ABI static bool hasFormat(const StringRef Path);
 
   // Create a YAMLMemProfReader after sanity checking the contents of the file
   // at \p Path or the \p Buffer.
-  static Expected<std::unique_ptr<YAMLMemProfReader>> create(const Twine &Path);
-  static Expected<std::unique_ptr<YAMLMemProfReader>>
+  LLVM_ABI static Expected<std::unique_ptr<YAMLMemProfReader>>
+  create(const Twine &Path);
+  LLVM_ABI static Expected<std::unique_ptr<YAMLMemProfReader>>
   create(std::unique_ptr<MemoryBuffer> Buffer);
 
-  void parse(StringRef YAMLData);
+  LLVM_ABI void parse(StringRef YAMLData);
 
   std::unique_ptr<memprof::DataAccessProfData> takeDataAccessProfData() {
     return std::move(DataAccessProfileData);
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummary.h b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummary.h
index f1aa818d9ed7..8bf6155bd43e 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummary.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummary.h
@@ -14,6 +14,7 @@
 #define LLVM_PROFILEDATA_MEMPROFSUMMARY_H
 
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace memprof {
@@ -46,11 +47,12 @@ class MemProfSummary {
   uint64_t getMaxColdTotalSize() const { return MaxColdTotalSize; }
   uint64_t getMaxWarmTotalSize() const { return MaxWarmTotalSize; }
   uint64_t getMaxHotTotalSize() const { return MaxHotTotalSize; }
-  void printSummaryYaml(raw_ostream &OS) const;
+  LLVM_ABI void printSummaryYaml(raw_ostream &OS) const;
   /// Write to indexed MemProf profile.
-  void write(ProfOStream &OS) const;
+  LLVM_ABI void write(ProfOStream &OS) const;
   /// Read from indexed MemProf profile.
-  static std::unique_ptr<MemProfSummary> deserialize(const unsigned char *&);
+  LLVM_ABI static std::unique_ptr<MemProfSummary>
+  deserialize(const unsigned char *&);
 };
 
 } // namespace memprof
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h
index c994f823fb94..1ce046549095 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/ProfileData/MemProfSummary.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace memprof {
@@ -40,9 +41,9 @@ class MemProfSummaryBuilder {
   MemProfSummaryBuilder() = default;
   ~MemProfSummaryBuilder() = default;
 
-  void addRecord(const IndexedMemProfRecord &);
-  void addRecord(const MemProfRecord &);
-  std::unique_ptr<MemProfSummary> getSummary();
+  LLVM_ABI void addRecord(const IndexedMemProfRecord &);
+  LLVM_ABI void addRecord(const MemProfRecord &);
+  LLVM_ABI std::unique_ptr<MemProfSummary> getSummary();
 };
 
 } // namespace memprof
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
index 5b59a0995045..cc55886aa601 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
@@ -18,6 +18,7 @@
 #include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <map>
 
@@ -219,9 +220,10 @@ class PGOCtxProfileReader final {
       : Magic(Buffer.substr(0, PGOCtxProfileWriter::ContainerMagic.size())),
         Cursor(Buffer.substr(PGOCtxProfileWriter::ContainerMagic.size())) {}
 
-  Expected<PGOCtxProfile> loadProfiles();
+  LLVM_ABI Expected<PGOCtxProfile> loadProfiles();
 };
 
-void convertCtxProfToYaml(raw_ostream &OS, const PGOCtxProfile &Profile);
+LLVM_ABI void convertCtxProfToYaml(raw_ostream &OS,
+                                   const PGOCtxProfile &Profile);
 } // namespace llvm
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
index 36ad5622a854..7031728c1e34 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
@@ -17,6 +17,7 @@
 #include "llvm/Bitstream/BitCodeEnums.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/ProfileData/CtxInstrContextNode.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 enum PGOCtxProfileRecords {
@@ -74,7 +75,7 @@ enum PGOCtxProfileBlockIDs {
 /// like value profiling - which would appear as additional records. For
 /// example, value profiling would produce a new record with a new record ID,
 /// containing the profiled values (much like the counters)
-class PGOCtxProfileWriter final : public ctx_profile::ProfileWriter {
+class LLVM_ABI PGOCtxProfileWriter final : public ctx_profile::ProfileWriter {
   enum class EmptyContextCriteria { None, EntryIsZero, AllAreZero };
 
   BitstreamWriter Writer;
@@ -111,6 +112,6 @@ class PGOCtxProfileWriter final : public ctx_profile::ProfileWriter {
   static constexpr StringRef ContainerMagic = "CTXP";
 };
 
-Error createCtxProfFromYAML(StringRef Profile, raw_ostream &Out);
+LLVM_ABI Error createCtxProfFromYAML(StringRef Profile, raw_ostream &Out);
 } // namespace llvm
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h b/external/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h
index edd8e1f644ad..6f71097a94a4 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <algorithm>
 #include <cstdint>
@@ -28,13 +29,13 @@
 
 namespace llvm {
 
-extern cl::opt<bool> UseContextLessSummary;
-extern cl::opt<int> ProfileSummaryCutoffHot;
-extern cl::opt<int> ProfileSummaryCutoffCold;
-extern cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold;
-extern cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold;
-extern cl::opt<uint64_t> ProfileSummaryHotCount;
-extern cl::opt<uint64_t> ProfileSummaryColdCount;
+LLVM_ABI extern cl::opt<bool> UseContextLessSummary;
+LLVM_ABI extern cl::opt<int> ProfileSummaryCutoffHot;
+LLVM_ABI extern cl::opt<int> ProfileSummaryCutoffCold;
+LLVM_ABI extern cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold;
+LLVM_ABI extern cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold;
+LLVM_ABI extern cl::opt<uint64_t> ProfileSummaryHotCount;
+LLVM_ABI extern cl::opt<uint64_t> ProfileSummaryColdCount;
 
 namespace sampleprof {
 
@@ -63,17 +64,17 @@ class ProfileSummaryBuilder {
   ~ProfileSummaryBuilder() = default;
 
   inline void addCount(uint64_t Count);
-  void computeDetailedSummary();
+  LLVM_ABI void computeDetailedSummary();
 
 public:
   /// A vector of useful cutoff values for detailed summary.
-  static const ArrayRef<uint32_t> DefaultCutoffs;
+  LLVM_ABI static const ArrayRef<uint32_t> DefaultCutoffs;
 
   /// Find the summary entry for a desired percentile of counts.
-  static const ProfileSummaryEntry &
+  LLVM_ABI static const ProfileSummaryEntry &
   getEntryForPercentile(const SummaryEntryVector &DS, uint64_t Percentile);
-  static uint64_t getHotCountThreshold(const SummaryEntryVector &DS);
-  static uint64_t getColdCountThreshold(const SummaryEntryVector &DS);
+  LLVM_ABI static uint64_t getHotCountThreshold(const SummaryEntryVector &DS);
+  LLVM_ABI static uint64_t getColdCountThreshold(const SummaryEntryVector &DS);
 };
 
 class InstrProfSummaryBuilder final : public ProfileSummaryBuilder {
@@ -83,11 +84,11 @@ class InstrProfSummaryBuilder final : public ProfileSummaryBuilder {
   InstrProfSummaryBuilder(std::vector<uint32_t> Cutoffs)
       : ProfileSummaryBuilder(std::move(Cutoffs)) {}
 
-  void addEntryCount(uint64_t Count);
-  void addInternalCount(uint64_t Count);
+  LLVM_ABI void addEntryCount(uint64_t Count);
+  LLVM_ABI void addInternalCount(uint64_t Count);
 
-  void addRecord(const InstrProfRecord &);
-  std::unique_ptr<ProfileSummary> getSummary();
+  LLVM_ABI void addRecord(const InstrProfRecord &);
+  LLVM_ABI std::unique_ptr<ProfileSummary> getSummary();
 };
 
 class SampleProfileSummaryBuilder final : public ProfileSummaryBuilder {
@@ -95,11 +96,11 @@ class SampleProfileSummaryBuilder final : public ProfileSummaryBuilder {
   SampleProfileSummaryBuilder(std::vector<uint32_t> Cutoffs)
       : ProfileSummaryBuilder(std::move(Cutoffs)) {}
 
-  void addRecord(const sampleprof::FunctionSamples &FS,
-                 bool isCallsiteSample = false);
-  std::unique_ptr<ProfileSummary>
+  LLVM_ABI void addRecord(const sampleprof::FunctionSamples &FS,
+                          bool isCallsiteSample = false);
+  LLVM_ABI std::unique_ptr<ProfileSummary>
   computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles);
-  std::unique_ptr<ProfileSummary> getSummary();
+  LLVM_ABI std::unique_ptr<ProfileSummary> getSummary();
 };
 
 /// This is called when a count is seen in the profile.
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h b/external/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h
index 3704de6e49f4..09dc7a294415 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h
@@ -24,6 +24,7 @@
 #include "llvm/ProfileData/FunctionId.h"
 #include "llvm/ProfileData/HashKeyMap.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/MathExtras.h"
@@ -43,7 +44,7 @@ namespace llvm {
 class DILocation;
 class raw_ostream;
 
-const std::error_category &sampleprof_category();
+LLVM_ABI const std::error_category &sampleprof_category();
 
 enum class sampleprof_error {
   success = 0,
@@ -281,11 +282,11 @@ static inline bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag) {
 struct LineLocation {
   LineLocation(uint32_t L, uint32_t D) : LineOffset(L), Discriminator(D) {}
 
-  void print(raw_ostream &OS) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &OS) const;
+  LLVM_ABI void dump() const;
 
   // Serialize the line location to the output stream using ULEB128 encoding.
-  void serialize(raw_ostream &OS);
+  LLVM_ABI void serialize(raw_ostream &OS);
 
   bool operator<(const LineLocation &O) const {
     return LineOffset < O.LineOffset ||
@@ -314,7 +315,7 @@ struct LineLocationHash {
   }
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc);
 
 /// Representation of a single sample record.
 ///
@@ -428,12 +429,13 @@ class SampleRecord {
 
   /// Merge the samples in \p Other into this record.
   /// Optionally scale sample counts by \p Weight.
-  sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1);
-  void print(raw_ostream &OS, unsigned Indent) const;
-  void dump() const;
+  LLVM_ABI sampleprof_error merge(const SampleRecord &Other,
+                                  uint64_t Weight = 1);
+  LLVM_ABI void print(raw_ostream &OS, unsigned Indent) const;
+  LLVM_ABI void dump() const;
   /// Serialize the sample record to the output stream using ULEB128 encoding.
   /// The \p NameTable is used to map function names to their IDs.
-  std::error_code
+  LLVM_ABI std::error_code
   serialize(raw_ostream &OS,
             const MapVector<FunctionId, uint32_t> &NameTable) const;
 
@@ -450,7 +452,7 @@ class SampleRecord {
   CallTargetMap CallTargets;
 };
 
-raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);
 
 // State of context associated with FunctionSamples
 enum ContextStateMask {
@@ -755,8 +757,8 @@ class FunctionSamples {
 public:
   FunctionSamples() = default;
 
-  void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const;
-  void dump() const;
+  LLVM_ABI void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const;
+  LLVM_ABI void dump() const;
 
   sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) {
     bool Overflowed;
@@ -931,7 +933,7 @@ class FunctionSamples {
   /// \p Loc with the maximum total sample count. If \p Remapper or \p
   /// FuncNameToProfNameMap is not nullptr, use them to find FunctionSamples
   /// with equivalent name as \p CalleeName.
-  const FunctionSamples *findFunctionSamplesAt(
+  LLVM_ABI const FunctionSamples *findFunctionSamplesAt(
       const LineLocation &Loc, StringRef CalleeName,
       SampleProfileReaderItaniumRemapper *Remapper,
       const HashKeyMap<std::unordered_map, FunctionId, FunctionId>
@@ -1156,14 +1158,14 @@ class FunctionSamples {
 
   /// Returns the line offset to the start line of the subprogram.
   /// We assume that a single function will not exceed 65535 LOC.
-  static unsigned getOffset(const DILocation *DIL);
+  LLVM_ABI static unsigned getOffset(const DILocation *DIL);
 
   /// Returns a unique call site identifier for a given debug location of a call
   /// instruction. This is wrapper of two scenarios, the probe-based profile and
   /// regular profile, to hide implementation details from the sample loader and
   /// the context tracker.
-  static LineLocation getCallSiteIdentifier(const DILocation *DIL,
-                                            bool ProfileIsFS = false);
+  LLVM_ABI static LineLocation getCallSiteIdentifier(const DILocation *DIL,
+                                                     bool ProfileIsFS = false);
 
   /// Returns a unique hash code for a combination of a callsite location and
   /// the callee function name.
@@ -1186,30 +1188,30 @@ class FunctionSamples {
   /// If \p Remapper or \p FuncNameToProfNameMap is not nullptr, it will be used
   /// to find matching FunctionSamples with not exactly the same but equivalent
   /// name.
-  const FunctionSamples *findFunctionSamples(
+  LLVM_ABI const FunctionSamples *findFunctionSamples(
       const DILocation *DIL,
       SampleProfileReaderItaniumRemapper *Remapper = nullptr,
       const HashKeyMap<std::unordered_map, FunctionId, FunctionId>
           *FuncNameToProfNameMap = nullptr) const;
 
-  static bool ProfileIsProbeBased;
+  LLVM_ABI static bool ProfileIsProbeBased;
 
-  static bool ProfileIsCS;
+  LLVM_ABI static bool ProfileIsCS;
 
-  static bool ProfileIsPreInlined;
+  LLVM_ABI static bool ProfileIsPreInlined;
 
   SampleContext &getContext() const { return Context; }
 
   void setContext(const SampleContext &FContext) { Context = FContext; }
 
   /// Whether the profile uses MD5 to represent string.
-  static bool UseMD5;
+  LLVM_ABI static bool UseMD5;
 
   /// Whether the profile contains any ".__uniq." suffix in a name.
-  static bool HasUniqSuffix;
+  LLVM_ABI static bool HasUniqSuffix;
 
   /// If this profile uses flow sensitive discriminators.
-  static bool ProfileIsFS;
+  LLVM_ABI static bool ProfileIsFS;
 
   /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
   /// all the function symbols defined or declared in current module.
@@ -1223,7 +1225,7 @@ class FunctionSamples {
 
   // Find all the names in the current FunctionSamples including names in
   // all the inline instances and names of call targets.
-  void findAllNames(DenseSet<FunctionId> &NameSet) const;
+  LLVM_ABI void findAllNames(DenseSet<FunctionId> &NameSet) const;
 
   bool operator==(const FunctionSamples &Other) const {
     return (GUIDToFuncNameMap == Other.GUIDToFuncNameMap ||
@@ -1311,7 +1313,7 @@ static inline FunctionId getRepInFormat(StringRef Name) {
   return FunctionId(Function::getGUIDAssumingExternalLinkage(Name));
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS);
 
 /// This class provides operator overloads to the map container using MD5 as the
 /// key type, so that existing code can still work in most cases using
@@ -1352,8 +1354,9 @@ class SampleProfileMap
 
 using NameFunctionSamples = std::pair<hash_code, const FunctionSamples *>;
 
-void sortFuncProfiles(const SampleProfileMap &ProfileMap,
-                      std::vector<NameFunctionSamples> &SortedProfiles);
+LLVM_ABI void
+sortFuncProfiles(const SampleProfileMap &ProfileMap,
+                 std::vector<NameFunctionSamples> &SortedProfiles);
 
 /// Sort a LocationT->SampleT map by LocationT.
 ///
@@ -1391,11 +1394,11 @@ class SampleContextTrimmer {
   // mainly to honor the preinliner decsion. Note that when MergeColdContext is
   // true, preinliner decsion is not honored anyway so TrimBaseProfileOnly will
   // be ignored.
-  void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold,
-                                       bool TrimColdContext,
-                                       bool MergeColdContext,
-                                       uint32_t ColdContextFrameLength,
-                                       bool TrimBaseProfileOnly);
+  LLVM_ABI void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold,
+                                                bool TrimColdContext,
+                                                bool MergeColdContext,
+                                                uint32_t ColdContextFrameLength,
+                                                bool TrimBaseProfileOnly);
 
 private:
   SampleProfileMap &ProfileMap;
@@ -1407,10 +1410,10 @@ class SampleContextTrimmer {
 /// nested profile to flatten profile conversion, etc.
 class ProfileConverter {
 public:
-  ProfileConverter(SampleProfileMap &Profiles);
+  LLVM_ABI ProfileConverter(SampleProfileMap &Profiles);
   // Convert a full context-sensitive flat sample profile into a nested sample
   // profile.
-  void convertCSProfiles();
+  LLVM_ABI void convertCSProfiles();
   struct FrameNode {
     FrameNode(FunctionId FName = FunctionId(),
               FunctionSamples *FSamples = nullptr,
@@ -1426,8 +1429,8 @@ class ProfileConverter {
     // Callsite location in parent context
     LineLocation CallSiteLoc;
 
-    FrameNode *getOrCreateChildFrame(const LineLocation &CallSite,
-                                     FunctionId CalleeName);
+    LLVM_ABI FrameNode *getOrCreateChildFrame(const LineLocation &CallSite,
+                                              FunctionId CalleeName);
   };
 
   static void flattenProfile(SampleProfileMap &ProfileMap,
@@ -1543,9 +1546,9 @@ class ProfileSymbolList {
   void setToCompress(bool TC) { ToCompress = TC; }
   bool toCompress() { return ToCompress; }
 
-  std::error_code read(const uint8_t *Data, uint64_t ListSize);
-  std::error_code write(raw_ostream &OS);
-  void dump(raw_ostream &OS = dbgs()) const;
+  LLVM_ABI std::error_code read(const uint8_t *Data, uint64_t ListSize);
+  LLVM_ABI std::error_code write(raw_ostream &OS);
+  LLVM_ABI void dump(raw_ostream &OS = dbgs()) const;
 
 private:
   // Determine whether or not to compress the symbol list when
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h b/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h
index 76c7cecded62..bfe079fbe536 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -233,6 +233,7 @@
 #include "llvm/ProfileData/GCOV.h"
 #include "llvm/ProfileData/SampleProf.h"
 #include "llvm/ProfileData/SymbolRemappingReader.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Discriminator.h"
 #include "llvm/Support/ErrorOr.h"
@@ -273,18 +274,18 @@ class SampleProfileReaderItaniumRemapper {
 
   /// Create a remapper from the given remapping file. The remapper will
   /// be used for profile read in by Reader.
-  static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
   create(StringRef Filename, vfs::FileSystem &FS, SampleProfileReader &Reader,
          LLVMContext &C);
 
   /// Create a remapper from the given Buffer. The remapper will
   /// be used for profile read in by Reader.
-  static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
   create(std::unique_ptr<MemoryBuffer> &B, SampleProfileReader &Reader,
          LLVMContext &C);
 
   /// Apply remappings to the profile read by Reader.
-  void applyRemapping(LLVMContext &Ctx);
+  LLVM_ABI void applyRemapping(LLVMContext &Ctx);
 
   bool hasApplied() { return RemappingApplied; }
 
@@ -299,7 +300,7 @@ class SampleProfileReaderItaniumRemapper {
 
   /// Return the equivalent name in the profile for \p FunctionName if
   /// it exists.
-  std::optional<StringRef> lookUpNameInProfile(StringRef FunctionName);
+  LLVM_ABI std::optional<StringRef> lookUpNameInProfile(StringRef FunctionName);
 
 private:
   // The buffer holding the content read from remapping file.
@@ -395,7 +396,8 @@ class SampleProfileReader {
   virtual std::error_code readImpl() = 0;
 
   /// Print the profile for \p FunctionSamples on stream \p OS.
-  void dumpFunctionProfile(const FunctionSamples &FS, raw_ostream &OS = dbgs());
+  LLVM_ABI void dumpFunctionProfile(const FunctionSamples &FS,
+                                    raw_ostream &OS = dbgs());
 
   /// Collect functions with definitions in Module M. For reader which
   /// support loading function profiles on demand, return true when the
@@ -404,10 +406,10 @@ class SampleProfileReader {
   virtual bool collectFuncsFromModule() { return false; }
 
   /// Print all the profiles on stream \p OS.
-  void dump(raw_ostream &OS = dbgs());
+  LLVM_ABI void dump(raw_ostream &OS = dbgs());
 
   /// Print all the profiles on stream \p OS in the JSON format.
-  void dumpJson(raw_ostream &OS = dbgs());
+  LLVM_ABI void dumpJson(raw_ostream &OS = dbgs());
 
   /// Return the samples collected for function \p F.
   FunctionSamples *getSamplesFor(const Function &F) {
@@ -456,7 +458,7 @@ class SampleProfileReader {
   /// Create a sample profile reader appropriate to the file format.
   /// Create a remapper underlying if RemapFilename is not empty.
   /// Parameter P specifies the FSDiscriminatorPass.
-  static ErrorOr<std::unique_ptr<SampleProfileReader>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<SampleProfileReader>>
   create(StringRef Filename, LLVMContext &C, vfs::FileSystem &FS,
          FSDiscriminatorPass P = FSDiscriminatorPass::Base,
          StringRef RemapFilename = "");
@@ -464,7 +466,7 @@ class SampleProfileReader {
   /// Create a sample profile reader from the supplied memory buffer.
   /// Create a remapper underlying if RemapFilename is not empty.
   /// Parameter P specifies the FSDiscriminatorPass.
-  static ErrorOr<std::unique_ptr<SampleProfileReader>>
+  LLVM_ABI static ErrorOr<std::unique_ptr<SampleProfileReader>>
   create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, vfs::FileSystem &FS,
          FSDiscriminatorPass P = FSDiscriminatorPass::Base,
          StringRef RemapFilename = "");
@@ -544,7 +546,7 @@ class SampleProfileReader {
   }
 
   /// Compute summary for this profile.
-  void computeSummary();
+  LLVM_ABI void computeSummary();
 
   /// Read sample profiles for the given functions and write them to the given
   /// profile map. Currently it's only used for extended binary format to load
@@ -608,7 +610,7 @@ class SampleProfileReader {
   bool SkipFlatProf = false;
 };
 
-class SampleProfileReaderText : public SampleProfileReader {
+class LLVM_ABI SampleProfileReaderText : public SampleProfileReader {
 public:
   SampleProfileReaderText(std::unique_ptr<MemoryBuffer> B, LLVMContext &C)
       : SampleProfileReader(std::move(B), C, SPF_Text) {}
@@ -631,7 +633,7 @@ class SampleProfileReaderText : public SampleProfileReader {
   std::list<SampleContextFrameVector> CSNameTable;
 };
 
-class SampleProfileReaderBinary : public SampleProfileReader {
+class LLVM_ABI SampleProfileReaderBinary : public SampleProfileReader {
 public:
   SampleProfileReaderBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C,
                             SampleProfileFormat Format = SPF_None)
@@ -730,7 +732,7 @@ class SampleProfileReaderBinary : public SampleProfileReader {
   virtual std::error_code verifySPMagic(uint64_t Magic) = 0;
 };
 
-class SampleProfileReaderRawBinary : public SampleProfileReaderBinary {
+class LLVM_ABI SampleProfileReaderRawBinary : public SampleProfileReaderBinary {
 private:
   std::error_code verifySPMagic(uint64_t Magic) override;
 
@@ -762,7 +764,8 @@ class SampleProfileReaderRawBinary : public SampleProfileReaderBinary {
 /// commonly used sections of a profile in extensible binary format. It is
 /// possible to define other types of profile inherited from
 /// SampleProfileReaderExtBinaryBase/SampleProfileWriterExtBinaryBase.
-class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
+class LLVM_ABI SampleProfileReaderExtBinaryBase
+    : public SampleProfileReaderBinary {
 private:
   std::error_code decompressSection(const uint8_t *SecStart,
                                     const uint64_t SecSize,
@@ -845,7 +848,8 @@ class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
                        SampleProfileMap &Profiles) override;
 };
 
-class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase {
+class LLVM_ABI SampleProfileReaderExtBinary
+    : public SampleProfileReaderExtBinaryBase {
 private:
   std::error_code verifySPMagic(uint64_t Magic) override;
   std::error_code readCustomSection(const SecHdrTableEntry &Entry) override {
@@ -878,7 +882,7 @@ enum HistType {
   HIST_TYPE_INDIR_CALL_TOPN
 };
 
-class SampleProfileReaderGCC : public SampleProfileReader {
+class LLVM_ABI SampleProfileReaderGCC : public SampleProfileReader {
 public:
   SampleProfileReaderGCC(std::unique_ptr<MemoryBuffer> B, LLVMContext &C)
       : SampleProfileReader(std::move(B), C, SPF_GCC),
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfWriter.h b/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 4b659eaf950b..146d051f770a 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
@@ -65,7 +66,7 @@ class FunctionPruningStrategy {
   virtual void Erase(size_t CurrentOutputSize) = 0;
 };
 
-class DefaultFunctionPruningStrategy : public FunctionPruningStrategy {
+class LLVM_ABI DefaultFunctionPruningStrategy : public FunctionPruningStrategy {
   std::vector<NameFunctionSamples> SortedFunctions;
 
 public:
@@ -86,7 +87,7 @@ class DefaultFunctionPruningStrategy : public FunctionPruningStrategy {
 };
 
 /// Sample-based profile writer. Base class.
-class SampleProfileWriter {
+class LLVM_ABI SampleProfileWriter {
 public:
   virtual ~SampleProfileWriter() = default;
 
@@ -163,7 +164,7 @@ class SampleProfileWriter {
 };
 
 /// Sample-based profile writer (text format).
-class SampleProfileWriterText : public SampleProfileWriter {
+class LLVM_ABI SampleProfileWriterText : public SampleProfileWriter {
 public:
   std::error_code writeSample(const FunctionSamples &S) override;
 
@@ -192,13 +193,13 @@ class SampleProfileWriterText : public SampleProfileWriter {
   /// cannot be skipped.
   bool MarkFlatProfiles = false;
 
-  friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
+  LLVM_ABI_FRIEND friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
   SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
                               SampleProfileFormat Format);
 };
 
 /// Sample-based profile writer (binary format).
-class SampleProfileWriterBinary : public SampleProfileWriter {
+class LLVM_ABI SampleProfileWriterBinary : public SampleProfileWriter {
 public:
   SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS)
       : SampleProfileWriter(OS) {}
@@ -224,7 +225,7 @@ class SampleProfileWriterBinary : public SampleProfileWriter {
   void addNames(const FunctionSamples &S);
 
 private:
-  friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
+  LLVM_ABI_FRIEND friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
   SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
                               SampleProfileFormat Format);
 };
@@ -264,7 +265,8 @@ const std::array<SmallVector<SecHdrTableEntry, 8>, NumOfLayout>
                                           {SecFuncMetadata, 0, 0, 0, 0}}),
 };
 
-class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary {
+class LLVM_ABI SampleProfileWriterExtBinaryBase
+    : public SampleProfileWriterBinary {
   using SampleProfileWriterBinary::SampleProfileWriterBinary;
 public:
   std::error_code write(const SampleProfileMap &ProfileMap) override;
@@ -407,7 +409,8 @@ class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary {
   ProfileSymbolList *ProfSymList = nullptr;
 };
 
-class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
+class LLVM_ABI SampleProfileWriterExtBinary
+    : public SampleProfileWriterExtBinaryBase {
 public:
   SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS)
       : SampleProfileWriterExtBinaryBase(OS) {}
diff --git a/external/llvm-project/llvm/include/llvm/ProfileData/SymbolRemappingReader.h b/external/llvm-project/llvm/include/llvm/ProfileData/SymbolRemappingReader.h
index 61d32134e981..d81b64347745 100644
--- a/external/llvm-project/llvm/include/llvm/ProfileData/SymbolRemappingReader.h
+++ b/external/llvm-project/llvm/include/llvm/ProfileData/SymbolRemappingReader.h
@@ -61,6 +61,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ProfileData/ItaniumManglingCanonicalizer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -83,7 +84,7 @@ class SymbolRemappingParseError : public ErrorInfo<SymbolRemappingParseError> {
   int64_t getLineNum() const { return Line; }
   StringRef getMessage() const { return Message; }
 
-  static char ID;
+  LLVM_ABI static char ID;
 
 private:
   std::string File;
@@ -99,7 +100,7 @@ class SymbolRemappingReader {
 public:
   /// Read remappings from the given buffer, which must live as long as
   /// the remapper.
-  Error read(MemoryBuffer &B);
+  LLVM_ABI Error read(MemoryBuffer &B);
 
   /// A Key represents an equivalence class of symbol names.
   using Key = uintptr_t;
diff --git a/external/llvm-project/llvm/include/llvm/Remarks/Remark.h b/external/llvm-project/llvm/include/llvm/Remarks/Remark.h
index de81c5a99280..8c8ca769c7d1 100644
--- a/external/llvm-project/llvm/include/llvm/Remarks/Remark.h
+++ b/external/llvm-project/llvm/include/llvm/Remarks/Remark.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 #include <string>
@@ -35,7 +36,7 @@ struct RemarkLocation {
   unsigned SourceColumn = 0;
 
   /// Implement operator<< on RemarkLocation.
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 };
 
 // Create wrappers for C Binding types (see CBindingWrapping.h).
@@ -51,11 +52,11 @@ struct Argument {
   std::optional<RemarkLocation> Loc;
 
   /// Implement operator<< on Argument.
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
   /// Return the value of argument as int.
-  std::optional<int> getValAsInt() const;
+  LLVM_ABI std::optional<int> getValAsInt() const;
   /// Check if the argument value can be parsed as int.
-  bool isValInt() const;
+  LLVM_ABI bool isValInt() const;
 };
 
 // Create wrappers for C Binding types (see CBindingWrapping.h).
@@ -124,13 +125,13 @@ struct Remark {
   Remark &operator=(Remark &&) = default;
 
   /// Return a message composed from the arguments as a string.
-  std::string getArgsAsMsg() const;
+  LLVM_ABI std::string getArgsAsMsg() const;
 
   /// Clone this remark to explicitly ask for a copy.
   Remark clone() const { return *this; }
 
   /// Implement operator<< on Remark.
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
 private:
   /// In order to avoid unwanted copies, "delete" the copy constructor.
diff --git a/external/llvm-project/llvm/include/llvm/Remarks/RemarkFormat.h b/external/llvm-project/llvm/include/llvm/Remarks/RemarkFormat.h
index 9c589eed44f3..64d08bcc9b8a 100644
--- a/external/llvm-project/llvm/include/llvm/Remarks/RemarkFormat.h
+++ b/external/llvm-project/llvm/include/llvm/Remarks/RemarkFormat.h
@@ -14,6 +14,7 @@
 #define LLVM_REMARKS_REMARKFORMAT_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -25,10 +26,10 @@ constexpr StringLiteral Magic("REMARKS");
 enum class Format { Unknown, YAML, YAMLStrTab, Bitstream };
 
 /// Parse and validate a string for the remark format.
-Expected<Format> parseFormat(StringRef FormatStr);
+LLVM_ABI Expected<Format> parseFormat(StringRef FormatStr);
 
 /// Parse and validate a magic number to a remark format.
-Expected<Format> magicToFormat(StringRef Magic);
+LLVM_ABI Expected<Format> magicToFormat(StringRef Magic);
 
 } // end namespace remarks
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Remarks/RemarkLinker.h b/external/llvm-project/llvm/include/llvm/Remarks/RemarkLinker.h
index f538718941c5..5343c6214470 100644
--- a/external/llvm-project/llvm/include/llvm/Remarks/RemarkLinker.h
+++ b/external/llvm-project/llvm/include/llvm/Remarks/RemarkLinker.h
@@ -16,6 +16,7 @@
 #include "llvm/Remarks/Remark.h"
 #include "llvm/Remarks/RemarkFormat.h"
 #include "llvm/Remarks/RemarkStringTable.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <memory>
 #include <optional>
@@ -69,7 +70,7 @@ struct RemarkLinker {
 
 public:
   /// Set a path to prepend to the external file path.
-  void setExternalFilePrependPath(StringRef PrependPath);
+  LLVM_ABI void setExternalFilePrependPath(StringRef PrependPath);
 
   /// Set KeepAllRemarks to \p B.
   void setKeepAllRemarks(bool B) { KeepAllRemarks = B; }
@@ -79,19 +80,19 @@ struct RemarkLinker {
   /// \p Buffer.
   /// \p Buffer can be either a standalone remark container or just
   /// metadata. This takes care of uniquing and merging the remarks.
-  Error link(StringRef Buffer,
-             std::optional<Format> RemarkFormat = std::nullopt);
+  LLVM_ABI Error link(StringRef Buffer,
+                      std::optional<Format> RemarkFormat = std::nullopt);
 
   /// Link the remarks found in \p Obj by looking for the right section and
   /// calling the method above.
-  Error link(const object::ObjectFile &Obj,
-             std::optional<Format> RemarkFormat = std::nullopt);
+  LLVM_ABI Error link(const object::ObjectFile &Obj,
+                      std::optional<Format> RemarkFormat = std::nullopt);
 
   /// Serialize the linked remarks to the stream \p OS, using the format \p
   /// RemarkFormat.
   /// This clears internal state such as the string table.
   /// Note: this implies that the serialization mode is standalone.
-  Error serialize(raw_ostream &OS, Format RemarksFormat) const;
+  LLVM_ABI Error serialize(raw_ostream &OS, Format RemarksFormat) const;
 
   /// Check whether there are any remarks linked.
   bool empty() const { return Remarks.empty(); }
@@ -109,7 +110,7 @@ struct RemarkLinker {
 /// Returns a buffer with the contents of the remarks section depending on the
 /// format of the file. If the section doesn't exist, this returns an empty
 /// optional.
-Expected<std::optional<StringRef>>
+LLVM_ABI Expected<std::optional<StringRef>>
 getRemarksSectionContents(const object::ObjectFile &Obj);
 
 } // end namespace remarks
diff --git a/external/llvm-project/llvm/include/llvm/Remarks/RemarkParser.h b/external/llvm-project/llvm/include/llvm/Remarks/RemarkParser.h
index 1333c582eba4..abb1fb86a87e 100644
--- a/external/llvm-project/llvm/include/llvm/Remarks/RemarkParser.h
+++ b/external/llvm-project/llvm/include/llvm/Remarks/RemarkParser.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Remarks/RemarkFormat.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <memory>
 #include <optional>
@@ -26,7 +27,7 @@ struct Remark;
 
 class EndOfFileError : public ErrorInfo<EndOfFileError> {
 public:
-  static char ID;
+  LLVM_ABI static char ID;
 
   EndOfFileError() = default;
 
@@ -64,7 +65,7 @@ struct ParsedStringTable {
   /// SmallVector for once.
   std::vector<size_t> Offsets;
 
-  ParsedStringTable(StringRef Buffer);
+  LLVM_ABI ParsedStringTable(StringRef Buffer);
   /// Disable copy.
   ParsedStringTable(const ParsedStringTable &) = delete;
   ParsedStringTable &operator=(const ParsedStringTable &) = delete;
@@ -73,17 +74,17 @@ struct ParsedStringTable {
   ParsedStringTable &operator=(ParsedStringTable &&) = default;
 
   size_t size() const { return Offsets.size(); }
-  Expected<StringRef> operator[](size_t Index) const;
+  LLVM_ABI Expected<StringRef> operator[](size_t Index) const;
 };
 
-Expected<std::unique_ptr<RemarkParser>> createRemarkParser(Format ParserFormat,
-                                                           StringRef Buf);
+LLVM_ABI Expected<std::unique_ptr<RemarkParser>>
+createRemarkParser(Format ParserFormat, StringRef Buf);
 
-Expected<std::unique_ptr<RemarkParser>>
+LLVM_ABI Expected<std::unique_ptr<RemarkParser>>
 createRemarkParser(Format ParserFormat, StringRef Buf,
                    ParsedStringTable StrTab);
 
-Expected<std::unique_ptr<RemarkParser>> createRemarkParserFromMeta(
+LLVM_ABI Expected<std::unique_ptr<RemarkParser>> createRemarkParserFromMeta(
     Format ParserFormat, StringRef Buf,
     std::optional<ParsedStringTable> StrTab = std::nullopt,
     std::optional<StringRef> ExternalFilePrependPath = std::nullopt);
diff --git a/external/llvm-project/llvm/include/llvm/Remarks/RemarkSerializer.h b/external/llvm-project/llvm/include/llvm/Remarks/RemarkSerializer.h
index f73135f4fbd3..05ef14ae5566 100644
--- a/external/llvm-project/llvm/include/llvm/Remarks/RemarkSerializer.h
+++ b/external/llvm-project/llvm/include/llvm/Remarks/RemarkSerializer.h
@@ -15,6 +15,7 @@
 
 #include "llvm/Remarks/RemarkFormat.h"
 #include "llvm/Remarks/RemarkStringTable.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -77,12 +78,12 @@ struct MetaSerializer {
 };
 
 /// Create a remark serializer.
-Expected<std::unique_ptr<RemarkSerializer>>
+LLVM_ABI Expected<std::unique_ptr<RemarkSerializer>>
 createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
                        raw_ostream &OS);
 
 /// Create a remark serializer that uses a pre-filled string table.
-Expected<std::unique_ptr<RemarkSerializer>>
+LLVM_ABI Expected<std::unique_ptr<RemarkSerializer>>
 createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
                        raw_ostream &OS, remarks::StringTable StrTab);
 
diff --git a/external/llvm-project/llvm/include/llvm/Remarks/RemarkStringTable.h b/external/llvm-project/llvm/include/llvm/Remarks/RemarkStringTable.h
index fe302c64d17f..fd88361e3d58 100644
--- a/external/llvm-project/llvm/include/llvm/Remarks/RemarkStringTable.h
+++ b/external/llvm-project/llvm/include/llvm/Remarks/RemarkStringTable.h
@@ -18,6 +18,7 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include <vector>
 
 namespace llvm {
@@ -50,22 +51,22 @@ struct StringTable {
   StringTable &operator=(StringTable &&) = default;
 
   /// Construct a string table from a ParsedStringTable.
-  StringTable(const ParsedStringTable &Other);
+  LLVM_ABI StringTable(const ParsedStringTable &Other);
 
   /// Add a string to the table. It returns an unique ID of the string.
-  std::pair<unsigned, StringRef> add(StringRef Str);
+  LLVM_ABI std::pair<unsigned, StringRef> add(StringRef Str);
   /// Modify \p R to use strings from this string table. If the string table
   /// does not contain the strings, it adds them.
-  void internalize(Remark &R);
+  LLVM_ABI void internalize(Remark &R);
   /// Serialize the string table to a stream. It is serialized as a little
   /// endian uint64 (the size of the table in bytes) followed by a sequence of
   /// NULL-terminated strings, where the N-th string is the string with the ID N
   /// in the StrTab map.
-  void serialize(raw_ostream &OS) const;
+  LLVM_ABI void serialize(raw_ostream &OS) const;
   /// Serialize the string table to a vector. This allows users to do the actual
   /// writing to file/memory/other.
   /// The string with the ID == N should be the N-th element in the vector.
-  std::vector<StringRef> serialize() const;
+  LLVM_ABI std::vector<StringRef> serialize() const;
 };
 
 } // end namespace remarks
diff --git a/external/llvm-project/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h b/external/llvm-project/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
index d2a1db5791ae..a2214c349e1c 100644
--- a/external/llvm-project/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
+++ b/external/llvm-project/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
@@ -14,6 +14,7 @@
 #define LLVM_REMARKS_YAMLREMARKSERIALIZER_H
 
 #include "llvm/Remarks/RemarkSerializer.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/YAMLTraits.h"
 #include <optional>
 
@@ -31,7 +32,7 @@ namespace remarks {
 ///   - <KEY>: <VALUE>
 ///     DebugLoc:        { File: <FILE>, Line: <LINE>, Column: <COL> }
 /// ...
-struct YAMLRemarkSerializer : public RemarkSerializer {
+struct LLVM_ABI YAMLRemarkSerializer : public RemarkSerializer {
   /// The YAML streamer.
   yaml::Output YAMLOutput;
 
@@ -53,7 +54,7 @@ struct YAMLRemarkSerializer : public RemarkSerializer {
                        std::optional<StringTable> StrTab = std::nullopt);
 };
 
-struct YAMLMetaSerializer : public MetaSerializer {
+struct LLVM_ABI YAMLMetaSerializer : public MetaSerializer {
   std::optional<StringRef> ExternalFilename;
 
   YAMLMetaSerializer(raw_ostream &OS, std::optional<StringRef> ExternalFilename)
@@ -65,7 +66,7 @@ struct YAMLMetaSerializer : public MetaSerializer {
 /// Serialize the remarks to YAML using a string table. An remark entry looks
 /// like the regular YAML remark but instead of string entries it's using
 /// numbers that map to an index in the string table.
-struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
+struct LLVM_ABI YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
   /// Wether we already emitted the metadata in standalone mode.
   /// This should be set to true after the first invocation of `emit`.
   bool DidEmitMeta = false;
@@ -91,7 +92,7 @@ struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
   }
 };
 
-struct YAMLStrTabMetaSerializer : public YAMLMetaSerializer {
+struct LLVM_ABI YAMLStrTabMetaSerializer : public YAMLMetaSerializer {
   /// The string table is part of the metadata.
   const StringTable &StrTab;
 
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/BasicBlock.h b/external/llvm-project/llvm/include/llvm/SandboxIR/BasicBlock.h
index 93e79e2a421f..25bbb6c058fa 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/BasicBlock.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/BasicBlock.h
@@ -11,6 +11,7 @@
 
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/SandboxIR/Value.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::sandboxir {
 
@@ -32,20 +33,20 @@ class BBIterator {
   llvm::BasicBlock *BB;
   llvm::BasicBlock::iterator It;
   Context *Ctx;
-  pointer getInstr(llvm::BasicBlock::iterator It) const;
+  LLVM_ABI pointer getInstr(llvm::BasicBlock::iterator It) const;
 
 public:
   BBIterator() : BB(nullptr), Ctx(nullptr) {}
   BBIterator(llvm::BasicBlock *BB, llvm::BasicBlock::iterator It, Context *Ctx)
       : BB(BB), It(It), Ctx(Ctx) {}
   reference operator*() const { return *getInstr(It); }
-  BBIterator &operator++();
+  LLVM_ABI BBIterator &operator++();
   BBIterator operator++(int) {
     auto Copy = *this;
     ++*this;
     return Copy;
   }
-  BBIterator &operator--();
+  LLVM_ABI BBIterator &operator--();
   BBIterator operator--(int) {
     auto Copy = *this;
     --*this;
@@ -60,14 +61,14 @@ class BBIterator {
   /// the instruction is not found in the IR-to-SandboxIR tables.
   pointer get() const { return getInstr(It); }
   /// \Returns the parent BB.
-  BasicBlock *getNodeParent() const;
+  LLVM_ABI BasicBlock *getNodeParent() const;
 };
 
 /// Contains a list of sandboxir::Instruction's.
 class BasicBlock : public Value {
   /// Builds a graph that contains all values in \p BB in their original form
   /// i.e., no vectorization is taking place here.
-  void buildBasicBlockFromLLVMIR(llvm::BasicBlock *LLVMBB);
+  LLVM_ABI void buildBasicBlockFromLLVMIR(llvm::BasicBlock *LLVMBB);
   friend class Context;     // For `buildBasicBlockFromIR`
   friend class Instruction; // For LLVM Val.
 
@@ -82,9 +83,9 @@ class BasicBlock : public Value {
   static bool classof(const Value *From) {
     return From->getSubclassID() == Value::ClassID::Block;
   }
-  Function *getParent() const;
+  LLVM_ABI Function *getParent() const;
   using iterator = BBIterator;
-  iterator begin() const;
+  LLVM_ABI iterator begin() const;
   iterator end() const {
     auto *BB = cast<llvm::BasicBlock>(Val);
     return iterator(BB, BB->end(), &Ctx);
@@ -96,10 +97,10 @@ class BasicBlock : public Value {
     return std::make_reverse_iterator(begin());
   }
   Context &getContext() const { return Ctx; }
-  Instruction *getTerminator() const;
+  LLVM_ABI Instruction *getTerminator() const;
   bool empty() const { return begin() == end(); }
-  Instruction &front() const;
-  Instruction &back() const;
+  LLVM_ABI Instruction &front() const;
+  LLVM_ABI Instruction &back() const;
 
 #ifndef NDEBUG
   void verify() const final;
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Constant.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Constant.h
index 2012cf8a8ed3..6f682a7059d1 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Constant.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Constant.h
@@ -76,16 +76,16 @@ class ConstantInt : public Constant {
   }
 
 public:
-  static ConstantInt *getTrue(Context &Ctx);
-  static ConstantInt *getFalse(Context &Ctx);
-  static ConstantInt *getBool(Context &Ctx, bool V);
-  static Constant *getTrue(Type *Ty);
-  static Constant *getFalse(Type *Ty);
-  static Constant *getBool(Type *Ty, bool V);
+  LLVM_ABI static ConstantInt *getTrue(Context &Ctx);
+  LLVM_ABI static ConstantInt *getFalse(Context &Ctx);
+  LLVM_ABI static ConstantInt *getBool(Context &Ctx, bool V);
+  LLVM_ABI static Constant *getTrue(Type *Ty);
+  LLVM_ABI static Constant *getFalse(Type *Ty);
+  LLVM_ABI static Constant *getBool(Type *Ty, bool V);
 
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantInt for the given value.
-  static ConstantInt *get(Type *Ty, uint64_t V, bool IsSigned = false);
+  LLVM_ABI static ConstantInt *get(Type *Ty, uint64_t V, bool IsSigned = false);
 
   /// Return a ConstantInt with the specified integer value for the specified
   /// type. If the type is wider than 64 bits, the value will be zero-extended
@@ -93,27 +93,29 @@ class ConstantInt : public Constant {
   /// be interpreted as a 64-bit signed integer and sign-extended to fit
   /// the type.
   /// Get a ConstantInt for a specific value.
-  static ConstantInt *get(IntegerType *Ty, uint64_t V, bool IsSigned = false);
+  LLVM_ABI static ConstantInt *get(IntegerType *Ty, uint64_t V,
+                                   bool IsSigned = false);
 
   /// Return a ConstantInt with the specified value for the specified type. The
   /// value V will be canonicalized to a an unsigned APInt. Accessing it with
   /// either getSExtValue() or getZExtValue() will yield a correctly sized and
   /// signed value for the type Ty.
   /// Get a ConstantInt for a specific signed value.
-  static ConstantInt *getSigned(IntegerType *Ty, int64_t V);
-  static Constant *getSigned(Type *Ty, int64_t V);
+  LLVM_ABI static ConstantInt *getSigned(IntegerType *Ty, int64_t V);
+  LLVM_ABI static Constant *getSigned(Type *Ty, int64_t V);
 
   /// Return a ConstantInt with the specified value and an implied Type. The
   /// type is the integer type that corresponds to the bit width of the value.
-  static ConstantInt *get(Context &Ctx, const APInt &V);
+  LLVM_ABI static ConstantInt *get(Context &Ctx, const APInt &V);
 
   /// Return a ConstantInt constructed from the string strStart with the given
   /// radix.
-  static ConstantInt *get(IntegerType *Ty, StringRef Str, uint8_t Radix);
+  LLVM_ABI static ConstantInt *get(IntegerType *Ty, StringRef Str,
+                                   uint8_t Radix);
 
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantInt for the given value.
-  static Constant *get(Type *Ty, const APInt &V);
+  LLVM_ABI static Constant *get(Type *Ty, const APInt &V);
 
   /// Return the constant as an APInt value reference. This allows clients to
   /// obtain a full-precision copy of the value.
@@ -166,7 +168,7 @@ class ConstantInt : public Constant {
 
   /// Variant of the getType() method to always return an IntegerType, which
   /// reduces the amount of casting needed in parts of the compiler.
-  IntegerType *getIntegerType() const;
+  LLVM_ABI IntegerType *getIntegerType() const;
 
   /// This static method returns true if the type Ty is big enough to
   /// represent the value V. This can be used to avoid having the get method
@@ -177,8 +179,8 @@ class ConstantInt : public Constant {
   /// to the appropriate unsigned type before calling the method.
   /// @returns true if V is a valid value for type Ty
   /// Determine if the value is in range for the given type.
-  static bool isValueValidForType(Type *Ty, uint64_t V);
-  static bool isValueValidForType(Type *Ty, int64_t V);
+  LLVM_ABI static bool isValueValidForType(Type *Ty, uint64_t V);
+  LLVM_ABI static bool isValueValidForType(Type *Ty, int64_t V);
 
   bool isNegative() const { return cast<llvm::ConstantInt>(Val)->isNegative(); }
 
@@ -264,29 +266,29 @@ class ConstantFP final : public Constant {
   /// for the specified value in the specified type. This should only be used
   /// for simple constant values like 2.0/1.0 etc, that are known-valid both as
   /// host double and as the target format.
-  static Constant *get(Type *Ty, double V);
+  LLVM_ABI static Constant *get(Type *Ty, double V);
 
   /// If Ty is a vector type, return a Constant with a splat of the given
   /// value. Otherwise return a ConstantFP for the given value.
-  static Constant *get(Type *Ty, const APFloat &V);
+  LLVM_ABI static Constant *get(Type *Ty, const APFloat &V);
 
-  static Constant *get(Type *Ty, StringRef Str);
+  LLVM_ABI static Constant *get(Type *Ty, StringRef Str);
 
-  static ConstantFP *get(const APFloat &V, Context &Ctx);
+  LLVM_ABI static ConstantFP *get(const APFloat &V, Context &Ctx);
 
-  static Constant *getNaN(Type *Ty, bool Negative = false,
-                          uint64_t Payload = 0);
-  static Constant *getQNaN(Type *Ty, bool Negative = false,
-                           APInt *Payload = nullptr);
-  static Constant *getSNaN(Type *Ty, bool Negative = false,
-                           APInt *Payload = nullptr);
-  static Constant *getZero(Type *Ty, bool Negative = false);
+  LLVM_ABI static Constant *getNaN(Type *Ty, bool Negative = false,
+                                   uint64_t Payload = 0);
+  LLVM_ABI static Constant *getQNaN(Type *Ty, bool Negative = false,
+                                    APInt *Payload = nullptr);
+  LLVM_ABI static Constant *getSNaN(Type *Ty, bool Negative = false,
+                                    APInt *Payload = nullptr);
+  LLVM_ABI static Constant *getZero(Type *Ty, bool Negative = false);
 
-  static Constant *getNegativeZero(Type *Ty);
-  static Constant *getInfinity(Type *Ty, bool Negative = false);
+  LLVM_ABI static Constant *getNegativeZero(Type *Ty);
+  LLVM_ABI static Constant *getInfinity(Type *Ty, bool Negative = false);
 
   /// Return true if Ty is big enough to represent V.
-  static bool isValueValidForType(Type *Ty, const APFloat &V);
+  LLVM_ABI static bool isValueValidForType(Type *Ty, const APFloat &V);
 
   inline const APFloat &getValueAPF() const {
     return cast<llvm::ConstantFP>(Val)->getValueAPF();
@@ -362,8 +364,8 @@ class ConstantArray final : public ConstantAggregate {
   friend class Context; // For constructor.
 
 public:
-  static Constant *get(ArrayType *T, ArrayRef<Constant *> V);
-  ArrayType *getType() const;
+  LLVM_ABI static Constant *get(ArrayType *T, ArrayRef<Constant *> V);
+  LLVM_ABI ArrayType *getType() const;
 
   // TODO: Missing functions: getType(), getTypeForElements(), getAnon(), get().
 
@@ -379,7 +381,7 @@ class ConstantStruct final : public ConstantAggregate {
   friend class Context; // For constructor.
 
 public:
-  static Constant *get(StructType *T, ArrayRef<Constant *> V);
+  LLVM_ABI static Constant *get(StructType *T, ArrayRef<Constant *> V);
 
   template <typename... Csts>
   static std::enable_if_t<are_base_of<Constant, Csts...>::value, Constant *>
@@ -396,8 +398,8 @@ class ConstantStruct final : public ConstantAggregate {
     return get(getTypeForElements(Ctx, V, Packed), V);
   }
   /// This version of the method allows an empty list.
-  static StructType *getTypeForElements(Context &Ctx, ArrayRef<Constant *> V,
-                                        bool Packed = false);
+  LLVM_ABI static StructType *
+  getTypeForElements(Context &Ctx, ArrayRef<Constant *> V, bool Packed = false);
   /// Return an anonymous struct type to use for a constant with the specified
   /// set of elements. The list must not be empty.
   static StructType *getTypeForElements(ArrayRef<Constant *> V,
@@ -424,10 +426,10 @@ class ConstantVector final : public ConstantAggregate {
   friend class Context; // For constructor.
 
 public:
-  static Constant *get(ArrayRef<Constant *> V);
+  LLVM_ABI static Constant *get(ArrayRef<Constant *> V);
   /// Return a ConstantVector with the specified constant in each element.
   /// Note that this might not return an instance of ConstantVector
-  static Constant *getSplat(ElementCount EC, Constant *Elt);
+  LLVM_ABI static Constant *getSplat(ElementCount EC, Constant *Elt);
   /// Specialize the getType() method to always return a FixedVectorType,
   /// which reduces the amount of casting needed in parts of the compiler.
   inline FixedVectorType *getType() const {
@@ -436,7 +438,7 @@ class ConstantVector final : public ConstantAggregate {
   /// If all elements of the vector constant have the same value, return that
   /// value. Otherwise, return nullptr. Ignore poison elements by setting
   /// AllowPoison to true.
-  Constant *getSplatValue(bool AllowPoison = false) const;
+  LLVM_ABI Constant *getSplatValue(bool AllowPoison = false) const;
 
   /// For isa/dyn_cast.
   static bool classof(const Value *From) {
@@ -451,18 +453,18 @@ class ConstantAggregateZero final : public Constant {
   friend class Context; // For constructor.
 
 public:
-  static ConstantAggregateZero *get(Type *Ty);
+  LLVM_ABI static ConstantAggregateZero *get(Type *Ty);
   /// If this CAZ has array or vector type, return a zero with the right element
   /// type.
-  Constant *getSequentialElement() const;
+  LLVM_ABI Constant *getSequentialElement() const;
   /// If this CAZ has struct type, return a zero with the right element type for
   /// the specified element.
-  Constant *getStructElement(unsigned Elt) const;
+  LLVM_ABI Constant *getStructElement(unsigned Elt) const;
   /// Return a zero of the right value for the specified GEP index if we can,
   /// otherwise return null (e.g. if C is a ConstantExpr).
-  Constant *getElementValue(Constant *C) const;
+  LLVM_ABI Constant *getElementValue(Constant *C) const;
   /// Return a zero of the right value for the specified GEP index.
-  Constant *getElementValue(unsigned Idx) const;
+  LLVM_ABI Constant *getElementValue(unsigned Idx) const;
   /// Return the number of elements in the array, vector, or struct.
   ElementCount getElementCount() const {
     return cast<llvm::ConstantAggregateZero>(Val)->getElementCount();
@@ -769,9 +771,9 @@ class ConstantPointerNull final : public Constant {
   friend class Context; // For constructor.
 
 public:
-  static ConstantPointerNull *get(PointerType *Ty);
+  LLVM_ABI static ConstantPointerNull *get(PointerType *Ty);
 
-  PointerType *getType() const;
+  LLVM_ABI PointerType *getType() const;
 
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
@@ -802,22 +804,22 @@ class UndefValue : public Constant {
 
 public:
   /// Static factory methods - Return an 'undef' object of the specified type.
-  static UndefValue *get(Type *T);
+  LLVM_ABI static UndefValue *get(Type *T);
 
   /// If this Undef has array or vector type, return a undef with the right
   /// element type.
-  UndefValue *getSequentialElement() const;
+  LLVM_ABI UndefValue *getSequentialElement() const;
 
   /// If this undef has struct type, return a undef with the right element type
   /// for the specified element.
-  UndefValue *getStructElement(unsigned Elt) const;
+  LLVM_ABI UndefValue *getStructElement(unsigned Elt) const;
 
   /// Return an undef of the right value for the specified GEP index if we can,
   /// otherwise return null (e.g. if C is a ConstantExpr).
-  UndefValue *getElementValue(Constant *C) const;
+  LLVM_ABI UndefValue *getElementValue(Constant *C) const;
 
   /// Return an undef of the right value for the specified GEP index.
-  UndefValue *getElementValue(unsigned Idx) const;
+  LLVM_ABI UndefValue *getElementValue(unsigned Idx) const;
 
   /// Return the number of elements in the array, vector, or struct.
   unsigned getNumElements() const {
@@ -850,22 +852,22 @@ class PoisonValue final : public UndefValue {
 
 public:
   /// Static factory methods - Return an 'poison' object of the specified type.
-  static PoisonValue *get(Type *T);
+  LLVM_ABI static PoisonValue *get(Type *T);
 
   /// If this poison has array or vector type, return a poison with the right
   /// element type.
-  PoisonValue *getSequentialElement() const;
+  LLVM_ABI PoisonValue *getSequentialElement() const;
 
   /// If this poison has struct type, return a poison with the right element
   /// type for the specified element.
-  PoisonValue *getStructElement(unsigned Elt) const;
+  LLVM_ABI PoisonValue *getStructElement(unsigned Elt) const;
 
   /// Return an poison of the right value for the specified GEP index if we can,
   /// otherwise return null (e.g. if C is a ConstantExpr).
-  PoisonValue *getElementValue(Constant *C) const;
+  LLVM_ABI PoisonValue *getElementValue(Constant *C) const;
 
   /// Return an poison of the right value for the specified GEP index.
-  PoisonValue *getElementValue(unsigned Idx) const;
+  LLVM_ABI PoisonValue *getElementValue(unsigned Idx) const;
 
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
@@ -924,7 +926,7 @@ class GlobalValue : public Constant {
   UnnamedAddr getUnnamedAddr() const {
     return cast<llvm::GlobalValue>(Val)->getUnnamedAddr();
   }
-  void setUnnamedAddr(UnnamedAddr V);
+  LLVM_ABI void setUnnamedAddr(UnnamedAddr V);
 
   static UnnamedAddr getMinUnnamedAddr(UnnamedAddr A, UnnamedAddr B) {
     return llvm::GlobalValue::getMinUnnamedAddr(A, B);
@@ -946,7 +948,7 @@ class GlobalValue : public Constant {
   bool hasProtectedVisibility() const {
     return cast<llvm::GlobalValue>(Val)->hasProtectedVisibility();
   }
-  void setVisibility(VisibilityTypes V);
+  LLVM_ABI void setVisibility(VisibilityTypes V);
 
   // TODO: Add missing functions.
 };
@@ -976,32 +978,6 @@ class GlobalObject : public GlobalValue {
     }
   }
 
-  /// FIXME: Remove this function once transition to Align is over.
-  uint64_t getAlignment() const {
-    return cast<llvm::GlobalObject>(Val)->getAlignment();
-  }
-
-  /// Returns the alignment of the given variable or function.
-  ///
-  /// Note that for functions this is the alignment of the code, not the
-  /// alignment of a function pointer.
-  MaybeAlign getAlign() const {
-    return cast<llvm::GlobalObject>(Val)->getAlign();
-  }
-
-  // TODO: Add missing: setAlignment(Align)
-
-  /// Sets the alignment attribute of the GlobalObject.
-  /// This method will be deprecated as the alignment property should always be
-  /// defined.
-  void setAlignment(MaybeAlign Align);
-
-  unsigned getGlobalObjectSubClassData() const {
-    return cast<llvm::GlobalObject>(Val)->getGlobalObjectSubClassData();
-  }
-
-  void setGlobalObjectSubClassData(unsigned V);
-
   /// Check if this global has a custom object file section.
   ///
   /// This is more efficient than calling getSection() and checking for an empty
@@ -1022,7 +998,7 @@ class GlobalObject : public GlobalValue {
   ///
   /// Setting the section to the empty string tells LLVM to choose an
   /// appropriate default object file section.
-  void setSection(StringRef S);
+  LLVM_ABI void setSection(StringRef S);
 
   bool hasComdat() const { return cast<llvm::GlobalObject>(Val)->hasComdat(); }
 
@@ -1057,7 +1033,7 @@ class GlobalWithNodeAPI : public ParentT {
   struct LLVMGVToGV {
     Context &Ctx;
     LLVMGVToGV(Context &Ctx) : Ctx(Ctx) {}
-    GlobalT &operator()(LLVMGlobalT &LLVMGV) const;
+    LLVM_ABI GlobalT &operator()(LLVMGlobalT &LLVMGV) const;
   };
 
 public:
@@ -1086,24 +1062,15 @@ class GlobalWithNodeAPI : public ParentT {
   }
 };
 
-// These are needed for SandboxIRTest when building with LLVM_BUILD_LLVM_DYLIB
-extern template LLVM_TEMPLATE_ABI GlobalIFunc &
-GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
-                  llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalIFunc
-                                                                  &LLVMGV)
-    const;
-extern template LLVM_TEMPLATE_ABI Function &
-GlobalWithNodeAPI<Function, llvm::Function, GlobalObject, llvm::GlobalObject>::
-    LLVMGVToGV::operator()(llvm::Function &LLVMGV) const;
-
-extern template LLVM_TEMPLATE_ABI GlobalVariable &GlobalWithNodeAPI<
-    GlobalVariable, llvm::GlobalVariable, GlobalObject,
-    llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalVariable &LLVMGV)
-    const;
-extern template LLVM_TEMPLATE_ABI GlobalAlias &
-GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
-                  llvm::GlobalValue>::LLVMGVToGV::operator()(llvm::GlobalAlias
-                                                                 &LLVMGV) const;
+// Explicit instantiations.
+extern template class LLVM_TEMPLATE_ABI GlobalWithNodeAPI<
+    GlobalIFunc, llvm::GlobalIFunc, GlobalObject, llvm::GlobalObject>;
+extern template class LLVM_TEMPLATE_ABI GlobalWithNodeAPI<
+    Function, llvm::Function, GlobalObject, llvm::GlobalObject>;
+extern template class LLVM_TEMPLATE_ABI GlobalWithNodeAPI<
+    GlobalVariable, llvm::GlobalVariable, GlobalObject, llvm::GlobalObject>;
+extern template class LLVM_TEMPLATE_ABI GlobalWithNodeAPI<
+    GlobalAlias, llvm::GlobalAlias, GlobalValue, llvm::GlobalValue>;
 
 class GlobalIFunc final
     : public GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
@@ -1123,13 +1090,13 @@ class GlobalIFunc final
   // TODO: Missing functions: copyAttributesFrom(), removeFromParent(),
   // eraseFromParent()
 
-  void setResolver(Constant *Resolver);
+  LLVM_ABI void setResolver(Constant *Resolver);
 
-  Constant *getResolver() const;
+  LLVM_ABI Constant *getResolver() const;
 
   // Return the resolver function after peeling off potential ConstantExpr
   // indirection.
-  Function *getResolverFunction();
+  LLVM_ABI Function *getResolverFunction();
   const Function *getResolverFunction() const {
     return const_cast<GlobalIFunc *>(this)->getResolverFunction();
   }
@@ -1162,7 +1129,7 @@ class GlobalVariable final
   struct LLVMGVToGV {
     Context &Ctx;
     LLVMGVToGV(Context &Ctx) : Ctx(Ctx) {}
-    GlobalVariable &operator()(llvm::GlobalVariable &LLVMGV) const;
+    LLVM_ABI GlobalVariable &operator()(llvm::GlobalVariable &LLVMGV) const;
   };
 
 public:
@@ -1207,11 +1174,11 @@ class GlobalVariable final
   /// illegal to call this method if the global is external, because we cannot
   /// tell what the value is initialized to!
   ///
-  Constant *getInitializer() const;
+  LLVM_ABI Constant *getInitializer() const;
   /// setInitializer - Sets the initializer for this global variable, removing
   /// any existing initializer if InitVal==NULL. The initializer must have the
   /// type getValueType().
-  void setInitializer(Constant *InitVal);
+  LLVM_ABI void setInitializer(Constant *InitVal);
 
   // TODO: Add missing replaceInitializer(). Requires special tracker
 
@@ -1222,12 +1189,12 @@ class GlobalVariable final
   bool isConstant() const {
     return cast<llvm::GlobalVariable>(Val)->isConstant();
   }
-  void setConstant(bool V);
+  LLVM_ABI void setConstant(bool V);
 
   bool isExternallyInitialized() const {
     return cast<llvm::GlobalVariable>(Val)->isExternallyInitialized();
   }
-  void setExternallyInitialized(bool Val);
+  LLVM_ABI void setExternallyInitialized(bool Val);
 
   // TODO: Missing copyAttributesFrom()
 
@@ -1294,6 +1261,18 @@ class GlobalVariable final
     return cast<llvm::GlobalVariable>(Val)->getCodeModel();
   }
 
+  /// Returns the alignment of the given variable.
+  MaybeAlign getAlign() const {
+    return cast<llvm::GlobalVariable>(Val)->getAlign();
+  }
+
+  // TODO: Add missing: setAligment(Align)
+
+  /// Sets the alignment attribute of the GlobalVariable.
+  /// This method will be deprecated as the alignment property should always be
+  /// defined.
+  LLVM_ABI void setAlignment(MaybeAlign Align);
+
   // TODO: Missing setCodeModel(). Requires custom tracker.
 
 #ifndef NDEBUG
@@ -1325,10 +1304,10 @@ class GlobalAlias final
   // TODO: Missing copyAttributresFrom().
   // TODO: Missing removeFromParent(), eraseFromParent().
 
-  void setAliasee(Constant *Aliasee);
-  Constant *getAliasee() const;
+  LLVM_ABI void setAliasee(Constant *Aliasee);
+  LLVM_ABI Constant *getAliasee() const;
 
-  const GlobalObject *getAliaseeObject() const;
+  LLVM_ABI const GlobalObject *getAliaseeObject() const;
   GlobalObject *getAliaseeObject() {
     return const_cast<GlobalObject *>(
         static_cast<const GlobalAlias *>(this)->getAliaseeObject());
@@ -1350,12 +1329,12 @@ class NoCFIValue final : public Constant {
 
 public:
   /// Return a NoCFIValue for the specified function.
-  static NoCFIValue *get(GlobalValue *GV);
+  LLVM_ABI static NoCFIValue *get(GlobalValue *GV);
 
-  GlobalValue *getGlobalValue() const;
+  LLVM_ABI GlobalValue *getGlobalValue() const;
 
   /// NoCFIValue is always a pointer.
-  PointerType *getType() const;
+  LLVM_ABI PointerType *getType() const;
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
     return From->getSubclassID() == ClassID::NoCFIValue;
@@ -1383,21 +1362,21 @@ class ConstantPtrAuth final : public Constant {
 
 public:
   /// Return a pointer signed with the specified parameters.
-  static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key,
-                              ConstantInt *Disc, Constant *AddrDisc);
+  LLVM_ABI static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key,
+                                       ConstantInt *Disc, Constant *AddrDisc);
   /// The pointer that is signed in this ptrauth signed pointer.
-  Constant *getPointer() const;
+  LLVM_ABI Constant *getPointer() const;
 
   /// The Key ID, an i32 constant.
-  ConstantInt *getKey() const;
+  LLVM_ABI ConstantInt *getKey() const;
 
   /// The integer discriminator, an i64 constant, or 0.
-  ConstantInt *getDiscriminator() const;
+  LLVM_ABI ConstantInt *getDiscriminator() const;
 
   /// The address discriminator if any, or the null constant.
   /// If present, this must be a value equivalent to the storage location of
   /// the only global-initializer user of the ptrauth signed pointer.
-  Constant *getAddrDiscriminator() const;
+  LLVM_ABI Constant *getAddrDiscriminator() const;
 
   /// Whether there is any non-null address discriminator.
   bool hasAddressDiscriminator() const {
@@ -1424,7 +1403,7 @@ class ConstantPtrAuth final : public Constant {
 
   /// Produce a new ptrauth expression signing the given value using
   /// the same schema as is stored in one.
-  ConstantPtrAuth *getWithSameSchema(Constant *Pointer) const;
+  LLVM_ABI ConstantPtrAuth *getWithSameSchema(Constant *Pointer) const;
 
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
@@ -1452,19 +1431,19 @@ class BlockAddress final : public Constant {
 
 public:
   /// Return a BlockAddress for the specified function and basic block.
-  static BlockAddress *get(Function *F, BasicBlock *BB);
+  LLVM_ABI static BlockAddress *get(Function *F, BasicBlock *BB);
 
   /// Return a BlockAddress for the specified basic block.  The basic
   /// block must be embedded into a function.
-  static BlockAddress *get(BasicBlock *BB);
+  LLVM_ABI static BlockAddress *get(BasicBlock *BB);
 
   /// Lookup an existing \c BlockAddress constant for the given BasicBlock.
   ///
   /// \returns 0 if \c !BB->hasAddressTaken(), otherwise the \c BlockAddress.
-  static BlockAddress *lookup(const BasicBlock *BB);
+  LLVM_ABI static BlockAddress *lookup(const BasicBlock *BB);
 
-  Function *getFunction() const;
-  BasicBlock *getBasicBlock() const;
+  LLVM_ABI Function *getFunction() const;
+  LLVM_ABI BasicBlock *getBasicBlock() const;
 
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
@@ -1479,9 +1458,9 @@ class DSOLocalEquivalent final : public Constant {
 
 public:
   /// Return a DSOLocalEquivalent for the specified global value.
-  static DSOLocalEquivalent *get(GlobalValue *GV);
+  LLVM_ABI static DSOLocalEquivalent *get(GlobalValue *GV);
 
-  GlobalValue *getGlobalValue() const;
+  LLVM_ABI GlobalValue *getGlobalValue() const;
 
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
@@ -1512,7 +1491,7 @@ class ConstantTokenNone final : public Constant {
 
 public:
   /// Return the ConstantTokenNone.
-  static ConstantTokenNone *get(Context &Ctx);
+  LLVM_ABI static ConstantTokenNone *get(Context &Ctx);
 
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Context.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Context.h
index a8a21b0db855..7d8b2c86e94a 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Context.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Context.h
@@ -15,6 +15,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/SandboxIR/Tracker.h"
 #include "llvm/SandboxIR/Type.h"
+#include "llvm/Support/Compiler.h"
 
 #include <cstdint>
 
@@ -112,32 +113,33 @@ class Context {
   CallbackID::ValTy NextCallbackID = 1;
 
   /// Remove \p V from the maps and returns the unique_ptr.
-  std::unique_ptr<Value> detachLLVMValue(llvm::Value *V);
+  LLVM_ABI std::unique_ptr<Value> detachLLVMValue(llvm::Value *V);
   /// Remove \p SBV from all SandboxIR maps and stop owning it. This effectively
   /// detaches \p V from the underlying IR.
-  std::unique_ptr<Value> detach(Value *V);
+  LLVM_ABI std::unique_ptr<Value> detach(Value *V);
   friend class Instruction; // For detach().
   /// Take ownership of VPtr and store it in `LLVMValueToValueMap`.
-  Value *registerValue(std::unique_ptr<Value> &&VPtr);
+  LLVM_ABI Value *registerValue(std::unique_ptr<Value> &&VPtr);
   friend class EraseFromParent; // For registerValue().
   /// This is the actual function that creates sandboxir values for \p V,
   /// and among others handles all instruction types.
-  Value *getOrCreateValueInternal(llvm::Value *V, llvm::User *U = nullptr);
+  LLVM_ABI Value *getOrCreateValueInternal(llvm::Value *V,
+                                           llvm::User *U = nullptr);
   /// Get or create a sandboxir::Argument for an existing LLVM IR \p LLVMArg.
-  Argument *getOrCreateArgument(llvm::Argument *LLVMArg);
+  LLVM_ABI Argument *getOrCreateArgument(llvm::Argument *LLVMArg);
   /// Get or create a sandboxir::Value for an existing LLVM IR \p LLVMV.
   Value *getOrCreateValue(llvm::Value *LLVMV) {
     return getOrCreateValueInternal(LLVMV, 0);
   }
   /// Get or create a sandboxir::Constant from an existing LLVM IR \p LLVMC.
-  Constant *getOrCreateConstant(llvm::Constant *LLVMC);
+  LLVM_ABI Constant *getOrCreateConstant(llvm::Constant *LLVMC);
   friend class ConstantDataSequential; // For getOrCreateConstant().
   friend class Utils; // For getMemoryBase
 
-  void runEraseInstrCallbacks(Instruction *I);
-  void runCreateInstrCallbacks(Instruction *I);
-  void runMoveInstrCallbacks(Instruction *I, const BBIterator &Where);
-  void runSetUseCallbacks(const Use &U, Value *NewSrc);
+  LLVM_ABI void runEraseInstrCallbacks(Instruction *I);
+  LLVM_ABI void runCreateInstrCallbacks(Instruction *I);
+  LLVM_ABI void runMoveInstrCallbacks(Instruction *I, const BBIterator &Where);
+  LLVM_ABI void runSetUseCallbacks(const Use &U, Value *NewSrc);
 
   friend class User;  // For runSetUseCallbacks().
   friend class Value; // For runSetUseCallbacks().
@@ -148,90 +150,97 @@ class Context {
 
   /// Create a sandboxir::BasicBlock for an existing LLVM IR \p BB. This will
   /// also create all contents of the block.
-  BasicBlock *createBasicBlock(llvm::BasicBlock *BB);
+  LLVM_ABI BasicBlock *createBasicBlock(llvm::BasicBlock *BB);
   friend class BasicBlock; // For getOrCreateValue().
 
   IRBuilder<ConstantFolder> LLVMIRBuilder;
   auto &getLLVMIRBuilder() { return LLVMIRBuilder; }
 
-  VAArgInst *createVAArgInst(llvm::VAArgInst *SI);
+  LLVM_ABI VAArgInst *createVAArgInst(llvm::VAArgInst *SI);
   friend VAArgInst; // For createVAArgInst()
-  FreezeInst *createFreezeInst(llvm::FreezeInst *SI);
+  LLVM_ABI FreezeInst *createFreezeInst(llvm::FreezeInst *SI);
   friend FreezeInst; // For createFreezeInst()
-  FenceInst *createFenceInst(llvm::FenceInst *SI);
+  LLVM_ABI FenceInst *createFenceInst(llvm::FenceInst *SI);
   friend FenceInst; // For createFenceInst()
-  SelectInst *createSelectInst(llvm::SelectInst *SI);
+  LLVM_ABI SelectInst *createSelectInst(llvm::SelectInst *SI);
   friend SelectInst; // For createSelectInst()
-  InsertElementInst *createInsertElementInst(llvm::InsertElementInst *IEI);
+  LLVM_ABI InsertElementInst *
+  createInsertElementInst(llvm::InsertElementInst *IEI);
   friend InsertElementInst; // For createInsertElementInst()
-  ExtractElementInst *createExtractElementInst(llvm::ExtractElementInst *EEI);
+  LLVM_ABI ExtractElementInst *
+  createExtractElementInst(llvm::ExtractElementInst *EEI);
   friend ExtractElementInst; // For createExtractElementInst()
-  ShuffleVectorInst *createShuffleVectorInst(llvm::ShuffleVectorInst *SVI);
+  LLVM_ABI ShuffleVectorInst *
+  createShuffleVectorInst(llvm::ShuffleVectorInst *SVI);
   friend ShuffleVectorInst; // For createShuffleVectorInst()
-  ExtractValueInst *createExtractValueInst(llvm::ExtractValueInst *IVI);
+  LLVM_ABI ExtractValueInst *
+  createExtractValueInst(llvm::ExtractValueInst *IVI);
   friend ExtractValueInst; // For createExtractValueInst()
-  InsertValueInst *createInsertValueInst(llvm::InsertValueInst *IVI);
+  LLVM_ABI InsertValueInst *createInsertValueInst(llvm::InsertValueInst *IVI);
   friend InsertValueInst; // For createInsertValueInst()
-  BranchInst *createBranchInst(llvm::BranchInst *I);
+  LLVM_ABI BranchInst *createBranchInst(llvm::BranchInst *I);
   friend BranchInst; // For createBranchInst()
-  LoadInst *createLoadInst(llvm::LoadInst *LI);
+  LLVM_ABI LoadInst *createLoadInst(llvm::LoadInst *LI);
   friend LoadInst; // For createLoadInst()
-  StoreInst *createStoreInst(llvm::StoreInst *SI);
+  LLVM_ABI StoreInst *createStoreInst(llvm::StoreInst *SI);
   friend StoreInst; // For createStoreInst()
-  ReturnInst *createReturnInst(llvm::ReturnInst *I);
+  LLVM_ABI ReturnInst *createReturnInst(llvm::ReturnInst *I);
   friend ReturnInst; // For createReturnInst()
-  CallInst *createCallInst(llvm::CallInst *I);
+  LLVM_ABI CallInst *createCallInst(llvm::CallInst *I);
   friend CallInst; // For createCallInst()
-  InvokeInst *createInvokeInst(llvm::InvokeInst *I);
+  LLVM_ABI InvokeInst *createInvokeInst(llvm::InvokeInst *I);
   friend InvokeInst; // For createInvokeInst()
-  CallBrInst *createCallBrInst(llvm::CallBrInst *I);
+  LLVM_ABI CallBrInst *createCallBrInst(llvm::CallBrInst *I);
   friend CallBrInst; // For createCallBrInst()
-  LandingPadInst *createLandingPadInst(llvm::LandingPadInst *I);
+  LLVM_ABI LandingPadInst *createLandingPadInst(llvm::LandingPadInst *I);
   friend LandingPadInst; // For createLandingPadInst()
-  CatchPadInst *createCatchPadInst(llvm::CatchPadInst *I);
+  LLVM_ABI CatchPadInst *createCatchPadInst(llvm::CatchPadInst *I);
   friend CatchPadInst; // For createCatchPadInst()
-  CleanupPadInst *createCleanupPadInst(llvm::CleanupPadInst *I);
+  LLVM_ABI CleanupPadInst *createCleanupPadInst(llvm::CleanupPadInst *I);
   friend CleanupPadInst; // For createCleanupPadInst()
-  CatchReturnInst *createCatchReturnInst(llvm::CatchReturnInst *I);
+  LLVM_ABI CatchReturnInst *createCatchReturnInst(llvm::CatchReturnInst *I);
   friend CatchReturnInst; // For createCatchReturnInst()
-  CleanupReturnInst *createCleanupReturnInst(llvm::CleanupReturnInst *I);
+  LLVM_ABI CleanupReturnInst *
+  createCleanupReturnInst(llvm::CleanupReturnInst *I);
   friend CleanupReturnInst; // For createCleanupReturnInst()
-  GetElementPtrInst *createGetElementPtrInst(llvm::GetElementPtrInst *I);
+  LLVM_ABI GetElementPtrInst *
+  createGetElementPtrInst(llvm::GetElementPtrInst *I);
   friend GetElementPtrInst; // For createGetElementPtrInst()
-  CatchSwitchInst *createCatchSwitchInst(llvm::CatchSwitchInst *I);
+  LLVM_ABI CatchSwitchInst *createCatchSwitchInst(llvm::CatchSwitchInst *I);
   friend CatchSwitchInst; // For createCatchSwitchInst()
-  ResumeInst *createResumeInst(llvm::ResumeInst *I);
+  LLVM_ABI ResumeInst *createResumeInst(llvm::ResumeInst *I);
   friend ResumeInst; // For createResumeInst()
-  SwitchInst *createSwitchInst(llvm::SwitchInst *I);
+  LLVM_ABI SwitchInst *createSwitchInst(llvm::SwitchInst *I);
   friend SwitchInst; // For createSwitchInst()
-  UnaryOperator *createUnaryOperator(llvm::UnaryOperator *I);
+  LLVM_ABI UnaryOperator *createUnaryOperator(llvm::UnaryOperator *I);
   friend UnaryOperator; // For createUnaryOperator()
-  BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
+  LLVM_ABI BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
   friend BinaryOperator; // For createBinaryOperator()
-  AtomicRMWInst *createAtomicRMWInst(llvm::AtomicRMWInst *I);
+  LLVM_ABI AtomicRMWInst *createAtomicRMWInst(llvm::AtomicRMWInst *I);
   friend AtomicRMWInst; // For createAtomicRMWInst()
-  AtomicCmpXchgInst *createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
+  LLVM_ABI AtomicCmpXchgInst *
+  createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
   friend AtomicCmpXchgInst; // For createAtomicCmpXchgInst()
-  AllocaInst *createAllocaInst(llvm::AllocaInst *I);
+  LLVM_ABI AllocaInst *createAllocaInst(llvm::AllocaInst *I);
   friend AllocaInst; // For createAllocaInst()
-  CastInst *createCastInst(llvm::CastInst *I);
+  LLVM_ABI CastInst *createCastInst(llvm::CastInst *I);
   friend CastInst; // For createCastInst()
-  PHINode *createPHINode(llvm::PHINode *I);
+  LLVM_ABI PHINode *createPHINode(llvm::PHINode *I);
   friend PHINode; // For createPHINode()
-  UnreachableInst *createUnreachableInst(llvm::UnreachableInst *UI);
+  LLVM_ABI UnreachableInst *createUnreachableInst(llvm::UnreachableInst *UI);
   friend UnreachableInst; // For createUnreachableInst()
-  CmpInst *createCmpInst(llvm::CmpInst *I);
+  LLVM_ABI CmpInst *createCmpInst(llvm::CmpInst *I);
   friend CmpInst; // For createCmpInst()
-  ICmpInst *createICmpInst(llvm::ICmpInst *I);
+  LLVM_ABI ICmpInst *createICmpInst(llvm::ICmpInst *I);
   friend ICmpInst; // For createICmpInst()
-  FCmpInst *createFCmpInst(llvm::FCmpInst *I);
+  LLVM_ABI FCmpInst *createFCmpInst(llvm::FCmpInst *I);
   friend FCmpInst; // For createFCmpInst()
 
 public:
-  Context(LLVMContext &LLVMCtx);
-  ~Context();
+  LLVM_ABI Context(LLVMContext &LLVMCtx);
+  LLVM_ABI ~Context();
   /// Clears function-level state.
-  void clear();
+  LLVM_ABI void clear();
 
   Tracker &getTracker() { return IRTracker; }
   /// Convenience function for `getTracker().save()`
@@ -241,14 +250,14 @@ class Context {
   /// Convenience function for `getTracker().accept()`
   void accept() { IRTracker.accept(); }
 
-  sandboxir::Value *getValue(llvm::Value *V) const;
+  LLVM_ABI sandboxir::Value *getValue(llvm::Value *V) const;
   const sandboxir::Value *getValue(const llvm::Value *V) const {
     return getValue(const_cast<llvm::Value *>(V));
   }
 
-  Module *getModule(llvm::Module *LLVMM) const;
+  LLVM_ABI Module *getModule(llvm::Module *LLVMM) const;
 
-  Module *getOrCreateModule(llvm::Module *LLVMM);
+  LLVM_ABI Module *getOrCreateModule(llvm::Module *LLVMM);
 
   Type *getType(llvm::Type *LLVMTy) {
     if (LLVMTy == nullptr)
@@ -265,10 +274,10 @@ class Context {
   /// This is the main API function for creating Sandbox IR.
   /// Note: this will not fully populate its parent module. The only globals
   /// that will be available are those used within the function.
-  Function *createFunction(llvm::Function *F);
+  LLVM_ABI Function *createFunction(llvm::Function *F);
 
   /// Create a sandboxir::Module corresponding to \p LLVMM.
-  Module *createModule(llvm::Module *LLVMM);
+  LLVM_ABI Module *createModule(llvm::Module *LLVMM);
 
   /// \Returns the number of values registered with Context.
   size_t getNumValues() const { return LLVMValueToValueMap.size(); }
@@ -277,26 +286,26 @@ class Context {
   /// to be removed from its parent. Note that this will also be called when
   /// reverting the creation of an instruction.
   /// \Returns a callback ID for later deregistration.
-  CallbackID registerEraseInstrCallback(EraseInstrCallback CB);
-  void unregisterEraseInstrCallback(CallbackID ID);
+  LLVM_ABI CallbackID registerEraseInstrCallback(EraseInstrCallback CB);
+  LLVM_ABI void unregisterEraseInstrCallback(CallbackID ID);
 
   /// Register a callback that gets called right after a SandboxIR instruction
   /// is created. Note that this will also be called when reverting the removal
   /// of an instruction.
   /// \Returns a callback ID for later deregistration.
-  CallbackID registerCreateInstrCallback(CreateInstrCallback CB);
-  void unregisterCreateInstrCallback(CallbackID ID);
+  LLVM_ABI CallbackID registerCreateInstrCallback(CreateInstrCallback CB);
+  LLVM_ABI void unregisterCreateInstrCallback(CallbackID ID);
 
   /// Register a callback that gets called when a SandboxIR instruction is about
   /// to be moved. Note that this will also be called when reverting a move.
   /// \Returns a callback ID for later deregistration.
-  CallbackID registerMoveInstrCallback(MoveInstrCallback CB);
-  void unregisterMoveInstrCallback(CallbackID ID);
+  LLVM_ABI CallbackID registerMoveInstrCallback(MoveInstrCallback CB);
+  LLVM_ABI void unregisterMoveInstrCallback(CallbackID ID);
 
   /// Register a callback that gets called when a Use gets set.
   /// \Returns a callback ID for later deregistration.
-  CallbackID registerSetUseCallback(SetUseCallback CB);
-  void unregisterSetUseCallback(CallbackID ID);
+  LLVM_ABI CallbackID registerSetUseCallback(SetUseCallback CB);
+  LLVM_ABI void unregisterSetUseCallback(CallbackID ID);
 };
 
 } // namespace sandboxir
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Function.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Function.h
index a810533f769f..28c69112b2b7 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Function.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Function.h
@@ -11,6 +11,7 @@
 
 #include "llvm/IR/Function.h"
 #include "llvm/SandboxIR/Constant.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::sandboxir {
 
@@ -56,7 +57,17 @@ class Function : public GlobalWithNodeAPI<Function, llvm::Function,
     LLVMBBToBB BBGetter(Ctx);
     return iterator(cast<llvm::Function>(Val)->end(), BBGetter);
   }
-  FunctionType *getFunctionType() const;
+  LLVM_ABI FunctionType *getFunctionType() const;
+
+  /// Returns the alignment of the given function.
+  MaybeAlign getAlign() const { return cast<llvm::Function>(Val)->getAlign(); }
+
+  // TODO: Add missing: setAligment(Align)
+
+  /// Sets the alignment attribute of the Function.
+  /// This method will be deprecated as the alignment property should always be
+  /// defined.
+  LLVM_ABI void setAlignment(MaybeAlign Align);
 
 #ifndef NDEBUG
   void verify() const final {
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Instruction.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Instruction.h
index ce5a2cbec85b..4e3ff19d4778 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Instruction.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Instruction.h
@@ -16,6 +16,7 @@
 #include "llvm/SandboxIR/BasicBlock.h"
 #include "llvm/SandboxIR/Constant.h"
 #include "llvm/SandboxIR/User.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::sandboxir {
 
@@ -57,7 +58,7 @@ class Instruction : public User {
 
   /// A SandboxIR Instruction may map to multiple LLVM IR Instruction. This
   /// returns its topmost LLVM IR instruction.
-  llvm::Instruction *getTopmostLLVMInstruction() const;
+  LLVM_ABI llvm::Instruction *getTopmostLLVMInstruction() const;
   friend class VAArgInst;          // For getTopmostLLVMInstruction().
   friend class FreezeInst;         // For getTopmostLLVMInstruction().
   friend class FenceInst;          // For getTopmostLLVMInstruction().
@@ -113,17 +114,17 @@ class Instruction : public User {
   }
 
 public:
-  static const char *getOpcodeName(Opcode Opc);
+  LLVM_ABI static const char *getOpcodeName(Opcode Opc);
   /// This is used by BasicBlock::iterator.
   virtual unsigned getNumOfIRInstrs() const = 0;
   /// \Returns a BasicBlock::iterator for this Instruction.
-  BBIterator getIterator() const;
+  LLVM_ABI BBIterator getIterator() const;
   /// \Returns the next sandboxir::Instruction in the block, or nullptr if at
   /// the end of the block.
-  Instruction *getNextNode() const;
+  LLVM_ABI Instruction *getNextNode() const;
   /// \Returns the previous sandboxir::Instruction in the block, or nullptr if
   /// at the beginning of the block.
-  Instruction *getPrevNode() const;
+  LLVM_ABI Instruction *getPrevNode() const;
   /// \Returns this Instruction's opcode. Note that SandboxIR has its own opcode
   /// state to allow for new SandboxIR-specific instructions.
   Opcode getOpcode() const { return Opc; }
@@ -188,17 +189,17 @@ class Instruction : public User {
   // TODO: More missing functions
 
   /// Detach this from its parent BasicBlock without deleting it.
-  void removeFromParent();
+  LLVM_ABI void removeFromParent();
   /// Detach this Value from its parent and delete it.
-  void eraseFromParent();
+  LLVM_ABI void eraseFromParent();
   /// Insert this detached instruction before \p BeforeI.
-  void insertBefore(Instruction *BeforeI);
+  LLVM_ABI void insertBefore(Instruction *BeforeI);
   /// Insert this detached instruction after \p AfterI.
-  void insertAfter(Instruction *AfterI);
+  LLVM_ABI void insertAfter(Instruction *AfterI);
   /// Insert this detached instruction into \p BB at \p WhereIt.
-  void insertInto(BasicBlock *BB, const BBIterator &WhereIt);
+  LLVM_ABI void insertInto(BasicBlock *BB, const BBIterator &WhereIt);
   /// Move this instruction to \p WhereIt.
-  void moveBefore(BasicBlock &BB, const BBIterator &WhereIt);
+  LLVM_ABI void moveBefore(BasicBlock &BB, const BBIterator &WhereIt);
   /// Move this instruction before \p Before.
   void moveBefore(Instruction *Before) {
     moveBefore(*Before->getParent(), Before->getIterator());
@@ -217,9 +218,9 @@ class Instruction : public User {
   }
   /// \Returns the BasicBlock containing this Instruction, or null if it is
   /// detached.
-  BasicBlock *getParent() const;
+  LLVM_ABI BasicBlock *getParent() const;
   /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From);
+  LLVM_ABI static bool classof(const sandboxir::Value *From);
 
   /// Determine whether the no signed wrap flag is set.
   bool hasNoUnsignedWrap() const {
@@ -227,20 +228,20 @@ class Instruction : public User {
   }
   /// Set or clear the nuw flag on this instruction, which must be an operator
   /// which supports this flag. See LangRef.html for the meaning of this flag.
-  void setHasNoUnsignedWrap(bool B = true);
+  LLVM_ABI void setHasNoUnsignedWrap(bool B = true);
   /// Determine whether the no signed wrap flag is set.
   bool hasNoSignedWrap() const {
     return cast<llvm::Instruction>(Val)->hasNoSignedWrap();
   }
   /// Set or clear the nsw flag on this instruction, which must be an operator
   /// which supports this flag. See LangRef.html for the meaning of this flag.
-  void setHasNoSignedWrap(bool B = true);
+  LLVM_ABI void setHasNoSignedWrap(bool B = true);
   /// Determine whether all fast-math-flags are set.
   bool isFast() const { return cast<llvm::Instruction>(Val)->isFast(); }
   /// Set or clear all fast-math-flags on this instruction, which must be an
   /// operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setFast(bool B);
+  LLVM_ABI void setFast(bool B);
   /// Determine whether the allow-reassociation flag is set.
   bool hasAllowReassoc() const {
     return cast<llvm::Instruction>(Val)->hasAllowReassoc();
@@ -248,24 +249,24 @@ class Instruction : public User {
   /// Set or clear the reassociation flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasAllowReassoc(bool B);
+  LLVM_ABI void setHasAllowReassoc(bool B);
   /// Determine whether the exact flag is set.
   bool isExact() const { return cast<llvm::Instruction>(Val)->isExact(); }
   /// Set or clear the exact flag on this instruction, which must be an operator
   /// which supports this flag. See LangRef.html for the meaning of this flag.
-  void setIsExact(bool B = true);
+  LLVM_ABI void setIsExact(bool B = true);
   /// Determine whether the no-NaNs flag is set.
   bool hasNoNaNs() const { return cast<llvm::Instruction>(Val)->hasNoNaNs(); }
   /// Set or clear the no-nans flag on this instruction, which must be an
   /// operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasNoNaNs(bool B);
+  LLVM_ABI void setHasNoNaNs(bool B);
   /// Determine whether the no-infs flag is set.
   bool hasNoInfs() const { return cast<llvm::Instruction>(Val)->hasNoInfs(); }
   /// Set or clear the no-infs flag on this instruction, which must be an
   /// operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasNoInfs(bool B);
+  LLVM_ABI void setHasNoInfs(bool B);
   /// Determine whether the no-signed-zeros flag is set.
   bool hasNoSignedZeros() const {
     return cast<llvm::Instruction>(Val)->hasNoSignedZeros();
@@ -273,7 +274,7 @@ class Instruction : public User {
   /// Set or clear the no-signed-zeros flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasNoSignedZeros(bool B);
+  LLVM_ABI void setHasNoSignedZeros(bool B);
   /// Determine whether the allow-reciprocal flag is set.
   bool hasAllowReciprocal() const {
     return cast<llvm::Instruction>(Val)->hasAllowReciprocal();
@@ -281,7 +282,7 @@ class Instruction : public User {
   /// Set or clear the allow-reciprocal flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasAllowReciprocal(bool B);
+  LLVM_ABI void setHasAllowReciprocal(bool B);
   /// Determine whether the allow-contract flag is set.
   bool hasAllowContract() const {
     return cast<llvm::Instruction>(Val)->hasAllowContract();
@@ -289,7 +290,7 @@ class Instruction : public User {
   /// Set or clear the allow-contract flag on this instruction, which must be
   /// an operator which supports this flag. See LangRef.html for the meaning of
   /// this flag.
-  void setHasAllowContract(bool B);
+  LLVM_ABI void setHasAllowContract(bool B);
   /// Determine whether the approximate-math-functions flag is set.
   bool hasApproxFunc() const {
     return cast<llvm::Instruction>(Val)->hasApproxFunc();
@@ -297,7 +298,7 @@ class Instruction : public User {
   /// Set or clear the approximate-math-functions flag on this instruction,
   /// which must be an operator which supports this flag. See LangRef.html for
   /// the meaning of this flag.
-  void setHasApproxFunc(bool B);
+  LLVM_ABI void setHasApproxFunc(bool B);
   /// Convenience function for getting all the fast-math flags, which must be an
   /// operator which supports these flags. See LangRef.html for the meaning of
   /// these flags.
@@ -307,11 +308,11 @@ class Instruction : public User {
   /// Convenience function for setting multiple fast-math flags on this
   /// instruction, which must be an operator which supports these flags. See
   /// LangRef.html for the meaning of these flags.
-  void setFastMathFlags(FastMathFlags FMF);
+  LLVM_ABI void setFastMathFlags(FastMathFlags FMF);
   /// Convenience function for transferring all fast-math flag values to this
   /// instruction, which must be an operator which supports these flags. See
   /// LangRef.html for the meaning of these flags.
-  void copyFastMathFlags(FastMathFlags FMF);
+  LLVM_ABI void copyFastMathFlags(FastMathFlags FMF);
 
   bool isAssociative() const {
     return cast<llvm::Instruction>(Val)->isAssociative();
@@ -352,7 +353,7 @@ class Instruction : public User {
 
   bool isVolatile() const { return cast<llvm::Instruction>(Val)->isVolatile(); }
 
-  Type *getAccessType() const;
+  LLVM_ABI Type *getAccessType() const;
 
   bool mayThrow(bool IncludePhaseOneUnwind = false) const {
     return cast<llvm::Instruction>(Val)->mayThrow(IncludePhaseOneUnwind);
@@ -414,22 +415,22 @@ class FenceInst : public SingleLLVMInstructionImpl<llvm::FenceInst> {
   friend Context; // For constructor;
 
 public:
-  static FenceInst *create(AtomicOrdering Ordering, InsertPosition Pos,
-                           Context &Ctx,
-                           SyncScope::ID SSID = SyncScope::System);
+  LLVM_ABI static FenceInst *create(AtomicOrdering Ordering, InsertPosition Pos,
+                                    Context &Ctx,
+                                    SyncScope::ID SSID = SyncScope::System);
   /// Returns the ordering constraint of this fence instruction.
   AtomicOrdering getOrdering() const {
     return cast<llvm::FenceInst>(Val)->getOrdering();
   }
   /// Sets the ordering constraint of this fence instruction.  May only be
   /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
-  void setOrdering(AtomicOrdering Ordering);
+  LLVM_ABI void setOrdering(AtomicOrdering Ordering);
   /// Returns the synchronization scope ID of this fence instruction.
   SyncScope::ID getSyncScopeID() const {
     return cast<llvm::FenceInst>(Val)->getSyncScopeID();
   }
   /// Sets the synchronization scope ID of this fence instruction.
-  void setSyncScopeID(SyncScope::ID SSID);
+  LLVM_ABI void setSyncScopeID(SyncScope::ID SSID);
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Fence;
   }
@@ -443,9 +444,9 @@ class SelectInst : public SingleLLVMInstructionImpl<llvm::SelectInst> {
   friend Context; // for SelectInst()
 
 public:
-  static Value *create(Value *Cond, Value *True, Value *False,
-                       InsertPosition Pos, Context &Ctx,
-                       const Twine &Name = "");
+  LLVM_ABI static Value *create(Value *Cond, Value *True, Value *False,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
 
   const Value *getCondition() const { return getOperand(0); }
   const Value *getTrueValue() const { return getOperand(1); }
@@ -457,7 +458,7 @@ class SelectInst : public SingleLLVMInstructionImpl<llvm::SelectInst> {
   void setCondition(Value *New) { setOperand(0, New); }
   void setTrueValue(Value *New) { setOperand(1, New); }
   void setFalseValue(Value *New) { setOperand(2, New); }
-  void swapValues();
+  LLVM_ABI void swapValues();
 
   /// Return a string if the specified operands are invalid for a select
   /// operation, otherwise return null.
@@ -468,7 +469,7 @@ class SelectInst : public SingleLLVMInstructionImpl<llvm::SelectInst> {
   }
 
   /// For isa/dyn_cast.
-  static bool classof(const Value *From);
+  LLVM_ABI static bool classof(const Value *From);
 };
 
 class InsertElementInst final
@@ -480,9 +481,9 @@ class InsertElementInst final
   friend class Context; // For accessing the constructor in create*()
 
 public:
-  static Value *create(Value *Vec, Value *NewElt, Value *Idx,
-                       InsertPosition Pos, Context &Ctx,
-                       const Twine &Name = "");
+  LLVM_ABI static Value *create(Value *Vec, Value *NewElt, Value *Idx,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::InsertElement;
   }
@@ -503,8 +504,8 @@ class ExtractElementInst final
                         // create*()
 
 public:
-  static Value *create(Value *Vec, Value *Idx, InsertPosition Pos, Context &Ctx,
-                       const Twine &Name = "");
+  LLVM_ABI static Value *create(Value *Vec, Value *Idx, InsertPosition Pos,
+                                Context &Ctx, const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::ExtractElement;
   }
@@ -516,7 +517,7 @@ class ExtractElementInst final
   Value *getIndexOperand() { return getOperand(1); }
   const Value *getVectorOperand() const { return getOperand(0); }
   const Value *getIndexOperand() const { return getOperand(1); }
-  VectorType *getVectorOperandType() const;
+  LLVM_ABI VectorType *getVectorOperandType() const;
 };
 
 class ShuffleVectorInst final
@@ -528,18 +529,19 @@ class ShuffleVectorInst final
   friend class Context; // For accessing the constructor in create*()
 
 public:
-  static Value *create(Value *V1, Value *V2, Value *Mask, InsertPosition Pos,
-                       Context &Ctx, const Twine &Name = "");
-  static Value *create(Value *V1, Value *V2, ArrayRef<int> Mask,
-                       InsertPosition Pos, Context &Ctx,
-                       const Twine &Name = "");
+  LLVM_ABI static Value *create(Value *V1, Value *V2, Value *Mask,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
+  LLVM_ABI static Value *create(Value *V1, Value *V2, ArrayRef<int> Mask,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::ShuffleVector;
   }
 
   /// Swap the operands and adjust the mask to preserve the semantics of the
   /// instruction.
-  void commute();
+  LLVM_ABI void commute();
 
   /// Return true if a shufflevector instruction can be formed with the
   /// specified operands.
@@ -554,7 +556,7 @@ class ShuffleVectorInst final
   }
 
   /// Overload to return most specific vector type.
-  VectorType *getType() const;
+  LLVM_ABI VectorType *getType() const;
 
   /// Return the shuffle mask value of this instruction for the given element
   /// index. Return PoisonMaskElem if the element is undef.
@@ -577,12 +579,12 @@ class ShuffleVectorInst final
   }
 
   /// Return the mask for this instruction, for use in bitcode.
-  Constant *getShuffleMaskForBitcode() const;
+  LLVM_ABI Constant *getShuffleMaskForBitcode() const;
 
-  static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
-                                                Type *ResultTy);
+  LLVM_ABI static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+                                                         Type *ResultTy);
 
-  void setShuffleMask(ArrayRef<int> Mask);
+  LLVM_ABI void setShuffleMask(ArrayRef<int> Mask);
 
   ArrayRef<int> getShuffleMask() const {
     return cast<llvm::ShuffleVectorInst>(Val)->getShuffleMask();
@@ -965,9 +967,9 @@ class InsertValueInst
   friend Context; // for InsertValueInst()
 
 public:
-  static Value *create(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
-                       InsertPosition Pos, Context &Ctx,
-                       const Twine &Name = "");
+  LLVM_ABI static Value *create(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
 
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::InsertValue;
@@ -1024,36 +1026,37 @@ class BranchInst : public SingleLLVMInstructionImpl<llvm::BranchInst> {
   friend Context; // for BranchInst()
 
 public:
-  static BranchInst *create(BasicBlock *IfTrue, InsertPosition Pos,
-                            Context &Ctx);
-  static BranchInst *create(BasicBlock *IfTrue, BasicBlock *IfFalse,
-                            Value *Cond, InsertPosition Pos, Context &Ctx);
+  LLVM_ABI static BranchInst *create(BasicBlock *IfTrue, InsertPosition Pos,
+                                     Context &Ctx);
+  LLVM_ABI static BranchInst *create(BasicBlock *IfTrue, BasicBlock *IfFalse,
+                                     Value *Cond, InsertPosition Pos,
+                                     Context &Ctx);
   /// For isa/dyn_cast.
-  static bool classof(const Value *From);
+  LLVM_ABI static bool classof(const Value *From);
   bool isUnconditional() const {
     return cast<llvm::BranchInst>(Val)->isUnconditional();
   }
   bool isConditional() const {
     return cast<llvm::BranchInst>(Val)->isConditional();
   }
-  Value *getCondition() const;
+  LLVM_ABI Value *getCondition() const;
   void setCondition(Value *V) { setOperand(0, V); }
   unsigned getNumSuccessors() const { return 1 + isConditional(); }
-  BasicBlock *getSuccessor(unsigned SuccIdx) const;
-  void setSuccessor(unsigned Idx, BasicBlock *NewSucc);
+  LLVM_ABI BasicBlock *getSuccessor(unsigned SuccIdx) const;
+  LLVM_ABI void setSuccessor(unsigned Idx, BasicBlock *NewSucc);
   void swapSuccessors() { swapOperandsInternal(1, 2); }
 
 private:
   struct LLVMBBToSBBB {
     Context &Ctx;
     LLVMBBToSBBB(Context &Ctx) : Ctx(Ctx) {}
-    BasicBlock *operator()(llvm::BasicBlock *BB) const;
+    LLVM_ABI BasicBlock *operator()(llvm::BasicBlock *BB) const;
   };
 
   struct ConstLLVMBBToSBBB {
     Context &Ctx;
     ConstLLVMBBToSBBB(Context &Ctx) : Ctx(Ctx) {}
-    const BasicBlock *operator()(const llvm::BasicBlock *BB) const;
+    LLVM_ABI const BasicBlock *operator()(const llvm::BasicBlock *BB) const;
   };
 
 public:
@@ -1109,8 +1112,9 @@ class ExtractValueInst : public UnaryInstruction {
   friend Context; // for ExtractValueInst()
 
 public:
-  static Value *create(Value *Agg, ArrayRef<unsigned> Idxs, InsertPosition Pos,
-                       Context &Ctx, const Twine &Name = "");
+  LLVM_ABI static Value *create(Value *Agg, ArrayRef<unsigned> Idxs,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
 
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::ExtractValue;
@@ -1120,7 +1124,7 @@ class ExtractValueInst : public UnaryInstruction {
   /// with an extractvalue instruction with the specified parameters.
   ///
   /// Null is returned if the indices are invalid for the specified type.
-  static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
+  LLVM_ABI static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
 
   using idx_iterator = llvm::ExtractValueInst::idx_iterator;
 
@@ -1163,9 +1167,9 @@ class VAArgInst : public UnaryInstruction {
   friend Context; // For constructor;
 
 public:
-  static VAArgInst *create(Value *List, Type *Ty, InsertPosition Pos,
-                           Context &Ctx, const Twine &Name = "");
-  Value *getPointerOperand();
+  LLVM_ABI static VAArgInst *create(Value *List, Type *Ty, InsertPosition Pos,
+                                    Context &Ctx, const Twine &Name = "");
+  LLVM_ABI Value *getPointerOperand();
   const Value *getPointerOperand() const {
     return const_cast<VAArgInst *>(this)->getPointerOperand();
   }
@@ -1183,8 +1187,8 @@ class FreezeInst : public UnaryInstruction {
   friend Context; // For constructor;
 
 public:
-  static FreezeInst *create(Value *V, InsertPosition Pos, Context &Ctx,
-                            const Twine &Name = "");
+  LLVM_ABI static FreezeInst *create(Value *V, InsertPosition Pos, Context &Ctx,
+                                     const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Freeze;
   }
@@ -1200,11 +1204,11 @@ class LoadInst final : public UnaryInstruction {
   /// Return true if this is a load from a volatile memory location.
   bool isVolatile() const { return cast<llvm::LoadInst>(Val)->isVolatile(); }
   /// Specify whether this is a volatile load or not.
-  void setVolatile(bool V);
+  LLVM_ABI void setVolatile(bool V);
 
-  static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                          InsertPosition Pos, bool IsVolatile, Context &Ctx,
-                          const Twine &Name = "");
+  LLVM_ABI static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                                   InsertPosition Pos, bool IsVolatile,
+                                   Context &Ctx, const Twine &Name = "");
   static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
                           InsertPosition Pos, Context &Ctx,
                           const Twine &Name = "") {
@@ -1212,8 +1216,8 @@ class LoadInst final : public UnaryInstruction {
   }
 
   /// For isa/dyn_cast.
-  static bool classof(const Value *From);
-  Value *getPointerOperand() const;
+  LLVM_ABI static bool classof(const Value *From);
+  LLVM_ABI Value *getPointerOperand() const;
   Align getAlign() const { return cast<llvm::LoadInst>(Val)->getAlign(); }
   bool isUnordered() const { return cast<llvm::LoadInst>(Val)->isUnordered(); }
   bool isSimple() const { return cast<llvm::LoadInst>(Val)->isSimple(); }
@@ -1229,19 +1233,20 @@ class StoreInst final : public SingleLLVMInstructionImpl<llvm::StoreInst> {
   /// Return true if this is a store from a volatile memory location.
   bool isVolatile() const { return cast<llvm::StoreInst>(Val)->isVolatile(); }
   /// Specify whether this is a volatile store or not.
-  void setVolatile(bool V);
+  LLVM_ABI void setVolatile(bool V);
 
-  static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
-                           InsertPosition Pos, bool IsVolatile, Context &Ctx);
+  LLVM_ABI static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
+                                    InsertPosition Pos, bool IsVolatile,
+                                    Context &Ctx);
   static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
                            InsertPosition Pos, Context &Ctx) {
     return create(V, Ptr, Align, Pos, /*IsVolatile=*/false, Ctx);
   }
 
   /// For isa/dyn_cast.
-  static bool classof(const Value *From);
-  Value *getValueOperand() const;
-  Value *getPointerOperand() const;
+  LLVM_ABI static bool classof(const Value *From);
+  LLVM_ABI Value *getValueOperand() const;
+  LLVM_ABI Value *getPointerOperand() const;
   Align getAlign() const { return cast<llvm::StoreInst>(Val)->getAlign(); }
   bool isSimple() const { return cast<llvm::StoreInst>(Val)->isSimple(); }
   bool isUnordered() const { return cast<llvm::StoreInst>(Val)->isUnordered(); }
@@ -1260,8 +1265,8 @@ class UnreachableInst final : public Instruction {
   }
 
 public:
-  static UnreachableInst *create(InsertPosition Pos, Context &Ctx);
-  static bool classof(const Value *From);
+  LLVM_ABI static UnreachableInst *create(InsertPosition Pos, Context &Ctx);
+  LLVM_ABI static bool classof(const Value *From);
   unsigned getNumSuccessors() const { return 0; }
   unsigned getUseOperandNo(const Use &Use) const final {
     llvm_unreachable("UnreachableInst has no operands!");
@@ -1280,12 +1285,13 @@ class ReturnInst final : public SingleLLVMInstructionImpl<llvm::ReturnInst> {
                                   Context &Ctx);
 
 public:
-  static ReturnInst *create(Value *RetVal, InsertPosition Pos, Context &Ctx);
+  LLVM_ABI static ReturnInst *create(Value *RetVal, InsertPosition Pos,
+                                     Context &Ctx);
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Ret;
   }
   /// \Returns null if there is no return value.
-  Value *getReturnValue() const;
+  LLVM_ABI Value *getReturnValue() const;
 };
 
 class CallBase : public SingleLLVMInstructionImpl<llvm::CallBase> {
@@ -1303,7 +1309,7 @@ class CallBase : public SingleLLVMInstructionImpl<llvm::CallBase> {
            Opc == Instruction::ClassID::CallBr;
   }
 
-  FunctionType *getFunctionType() const;
+  LLVM_ABI FunctionType *getFunctionType() const;
 
   op_iterator data_operands_begin() { return op_begin(); }
   const_op_iterator data_operands_begin() const {
@@ -1390,17 +1396,17 @@ class CallBase : public SingleLLVMInstructionImpl<llvm::CallBase> {
   }
   bool hasArgument(const Value *V) const { return is_contained(args(), V); }
 
-  Value *getCalledOperand() const;
-  Use getCalledOperandUse() const;
+  LLVM_ABI Value *getCalledOperand() const;
+  LLVM_ABI Use getCalledOperandUse() const;
 
-  Function *getCalledFunction() const;
+  LLVM_ABI Function *getCalledFunction() const;
   bool isIndirectCall() const {
     return cast<llvm::CallBase>(Val)->isIndirectCall();
   }
   bool isCallee(Use U) const {
     return cast<llvm::CallBase>(Val)->isCallee(U.LLVMUse);
   }
-  Function *getCaller();
+  LLVM_ABI Function *getCaller();
   const Function *getCaller() const {
     return const_cast<CallBase *>(this)->getCaller();
   }
@@ -1412,7 +1418,7 @@ class CallBase : public SingleLLVMInstructionImpl<llvm::CallBase> {
     return cast<llvm::CallBase>(Val)->getIntrinsicID();
   }
   void setCalledOperand(Value *V) { getCalledOperandUse().set(V); }
-  void setCalledFunction(Function *F);
+  LLVM_ABI void setCalledFunction(Function *F);
   CallingConv::ID getCallingConv() const {
     return cast<llvm::CallBase>(Val)->getCallingConv();
   }
@@ -1428,9 +1434,9 @@ class CallInst : public CallBase {
   friend class IntrinsicInst; // For constructor
 
 public:
-  static CallInst *create(FunctionType *FTy, Value *Func,
-                          ArrayRef<Value *> Args, InsertPosition Pos,
-                          Context &Ctx, const Twine &NameStr = "");
+  LLVM_ABI static CallInst *create(FunctionType *FTy, Value *Func,
+                                   ArrayRef<Value *> Args, InsertPosition Pos,
+                                   Context &Ctx, const Twine &NameStr = "");
 
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Call;
@@ -1446,20 +1452,21 @@ class InvokeInst final : public CallBase {
                         // create*()
 
 public:
-  static InvokeInst *create(FunctionType *FTy, Value *Func,
-                            BasicBlock *IfNormal, BasicBlock *IfException,
-                            ArrayRef<Value *> Args, InsertPosition Pos,
-                            Context &Ctx, const Twine &NameStr = "");
+  LLVM_ABI static InvokeInst *create(FunctionType *FTy, Value *Func,
+                                     BasicBlock *IfNormal,
+                                     BasicBlock *IfException,
+                                     ArrayRef<Value *> Args, InsertPosition Pos,
+                                     Context &Ctx, const Twine &NameStr = "");
 
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Invoke;
   }
-  BasicBlock *getNormalDest() const;
-  BasicBlock *getUnwindDest() const;
-  void setNormalDest(BasicBlock *BB);
-  void setUnwindDest(BasicBlock *BB);
-  LandingPadInst *getLandingPadInst() const;
-  BasicBlock *getSuccessor(unsigned SuccIdx) const;
+  LLVM_ABI BasicBlock *getNormalDest() const;
+  LLVM_ABI BasicBlock *getUnwindDest() const;
+  LLVM_ABI void setNormalDest(BasicBlock *BB);
+  LLVM_ABI void setUnwindDest(BasicBlock *BB);
+  LLVM_ABI LandingPadInst *getLandingPadInst() const;
+  LLVM_ABI BasicBlock *getSuccessor(unsigned SuccIdx) const;
   void setSuccessor(unsigned SuccIdx, BasicBlock *NewSucc) {
     assert(SuccIdx < 2 && "Successor # out of range for invoke!");
     if (SuccIdx == 0)
@@ -1481,25 +1488,25 @@ class CallBrInst final : public CallBase {
                         // create*()
 
 public:
-  static CallBrInst *create(FunctionType *FTy, Value *Func,
-                            BasicBlock *DefaultDest,
-                            ArrayRef<BasicBlock *> IndirectDests,
-                            ArrayRef<Value *> Args, InsertPosition Pos,
-                            Context &Ctx, const Twine &NameStr = "");
+  LLVM_ABI static CallBrInst *create(FunctionType *FTy, Value *Func,
+                                     BasicBlock *DefaultDest,
+                                     ArrayRef<BasicBlock *> IndirectDests,
+                                     ArrayRef<Value *> Args, InsertPosition Pos,
+                                     Context &Ctx, const Twine &NameStr = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CallBr;
   }
   unsigned getNumIndirectDests() const {
     return cast<llvm::CallBrInst>(Val)->getNumIndirectDests();
   }
-  Value *getIndirectDestLabel(unsigned Idx) const;
-  Value *getIndirectDestLabelUse(unsigned Idx) const;
-  BasicBlock *getDefaultDest() const;
-  BasicBlock *getIndirectDest(unsigned Idx) const;
-  SmallVector<BasicBlock *, 16> getIndirectDests() const;
-  void setDefaultDest(BasicBlock *BB);
-  void setIndirectDest(unsigned Idx, BasicBlock *BB);
-  BasicBlock *getSuccessor(unsigned Idx) const;
+  LLVM_ABI Value *getIndirectDestLabel(unsigned Idx) const;
+  LLVM_ABI Value *getIndirectDestLabelUse(unsigned Idx) const;
+  LLVM_ABI BasicBlock *getDefaultDest() const;
+  LLVM_ABI BasicBlock *getIndirectDest(unsigned Idx) const;
+  LLVM_ABI SmallVector<BasicBlock *, 16> getIndirectDests() const;
+  LLVM_ABI void setDefaultDest(BasicBlock *BB);
+  LLVM_ABI void setIndirectDest(unsigned Idx, BasicBlock *BB);
+  LLVM_ABI BasicBlock *getSuccessor(unsigned Idx) const;
   unsigned getNumSuccessors() const {
     return cast<llvm::CallBrInst>(Val)->getNumSuccessors();
   }
@@ -1512,9 +1519,10 @@ class LandingPadInst : public SingleLLVMInstructionImpl<llvm::LandingPadInst> {
   friend class Context; // For constructor.
 
 public:
-  static LandingPadInst *create(Type *RetTy, unsigned NumReservedClauses,
-                                InsertPosition Pos, Context &Ctx,
-                                const Twine &Name = "");
+  LLVM_ABI static LandingPadInst *create(Type *RetTy,
+                                         unsigned NumReservedClauses,
+                                         InsertPosition Pos, Context &Ctx,
+                                         const Twine &Name = "");
   /// Return 'true' if this landingpad instruction is a
   /// cleanup. I.e., it should be run when unwinding even if its landing pad
   /// doesn't catch the exception.
@@ -1522,14 +1530,14 @@ class LandingPadInst : public SingleLLVMInstructionImpl<llvm::LandingPadInst> {
     return cast<llvm::LandingPadInst>(Val)->isCleanup();
   }
   /// Indicate that this landingpad instruction is a cleanup.
-  void setCleanup(bool V);
+  LLVM_ABI void setCleanup(bool V);
 
   // TODO: We are not implementing addClause() because we have no way to revert
   // it for now.
 
   /// Get the value of the clause at index Idx. Use isCatch/isFilter to
   /// determine what type of clause this is.
-  Constant *getClause(unsigned Idx) const;
+  LLVM_ABI Constant *getClause(unsigned Idx) const;
 
   /// Return 'true' if the clause and index Idx is a catch clause.
   bool isCatch(unsigned Idx) const {
@@ -1565,12 +1573,12 @@ class FuncletPadInst : public SingleLLVMInstructionImpl<llvm::FuncletPadInst> {
   ///
   /// Note: This returns the associated CatchSwitchInst if this FuncletPadInst
   /// is a CatchPadInst.
-  Value *getParentPad() const;
-  void setParentPad(Value *ParentPad);
+  LLVM_ABI Value *getParentPad() const;
+  LLVM_ABI void setParentPad(Value *ParentPad);
   /// Return the Idx-th funcletpad argument.
-  Value *getArgOperand(unsigned Idx) const;
+  LLVM_ABI Value *getArgOperand(unsigned Idx) const;
   /// Set the Idx-th funcletpad argument.
-  void setArgOperand(unsigned Idx, Value *V);
+  LLVM_ABI void setArgOperand(unsigned Idx, Value *V);
 
   // TODO: Implement missing functions: arg_operands().
   static bool classof(const Value *From) {
@@ -1585,13 +1593,13 @@ class CatchPadInst : public FuncletPadInst {
   friend class Context; // For constructor.
 
 public:
-  CatchSwitchInst *getCatchSwitch() const;
+  LLVM_ABI CatchSwitchInst *getCatchSwitch() const;
   // TODO: We have not implemented setCatchSwitch() because we can't revert it
   // for now, as there is no CatchPadInst member function that can undo it.
 
-  static CatchPadInst *create(Value *ParentPad, ArrayRef<Value *> Args,
-                              InsertPosition Pos, Context &Ctx,
-                              const Twine &Name = "");
+  LLVM_ABI static CatchPadInst *create(Value *ParentPad, ArrayRef<Value *> Args,
+                                       InsertPosition Pos, Context &Ctx,
+                                       const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CatchPad;
   }
@@ -1603,9 +1611,10 @@ class CleanupPadInst : public FuncletPadInst {
   friend class Context; // For constructor.
 
 public:
-  static CleanupPadInst *create(Value *ParentPad, ArrayRef<Value *> Args,
-                                InsertPosition Pos, Context &Ctx,
-                                const Twine &Name = "");
+  LLVM_ABI static CleanupPadInst *create(Value *ParentPad,
+                                         ArrayRef<Value *> Args,
+                                         InsertPosition Pos, Context &Ctx,
+                                         const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CleanupPad;
   }
@@ -1619,16 +1628,17 @@ class CatchReturnInst
   friend class Context; // For constructor.
 
 public:
-  static CatchReturnInst *create(CatchPadInst *CatchPad, BasicBlock *BB,
-                                 InsertPosition Pos, Context &Ctx);
-  CatchPadInst *getCatchPad() const;
-  void setCatchPad(CatchPadInst *CatchPad);
-  BasicBlock *getSuccessor() const;
-  void setSuccessor(BasicBlock *NewSucc);
+  LLVM_ABI static CatchReturnInst *create(CatchPadInst *CatchPad,
+                                          BasicBlock *BB, InsertPosition Pos,
+                                          Context &Ctx);
+  LLVM_ABI CatchPadInst *getCatchPad() const;
+  LLVM_ABI void setCatchPad(CatchPadInst *CatchPad);
+  LLVM_ABI BasicBlock *getSuccessor() const;
+  LLVM_ABI void setSuccessor(BasicBlock *NewSucc);
   unsigned getNumSuccessors() {
     return cast<llvm::CatchReturnInst>(Val)->getNumSuccessors();
   }
-  Value *getCatchSwitchParentPad() const;
+  LLVM_ABI Value *getCatchSwitchParentPad() const;
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CatchRet;
   }
@@ -1642,22 +1652,22 @@ class CleanupReturnInst
   friend class Context; // For constructor.
 
 public:
-  static CleanupReturnInst *create(CleanupPadInst *CleanupPad,
-                                   BasicBlock *UnwindBB, InsertPosition Pos,
-                                   Context &Ctx);
+  LLVM_ABI static CleanupReturnInst *create(CleanupPadInst *CleanupPad,
+                                            BasicBlock *UnwindBB,
+                                            InsertPosition Pos, Context &Ctx);
   bool hasUnwindDest() const {
     return cast<llvm::CleanupReturnInst>(Val)->hasUnwindDest();
   }
   bool unwindsToCaller() const {
     return cast<llvm::CleanupReturnInst>(Val)->unwindsToCaller();
   }
-  CleanupPadInst *getCleanupPad() const;
-  void setCleanupPad(CleanupPadInst *CleanupPad);
+  LLVM_ABI CleanupPadInst *getCleanupPad() const;
+  LLVM_ABI void setCleanupPad(CleanupPadInst *CleanupPad);
   unsigned getNumSuccessors() const {
     return cast<llvm::CleanupReturnInst>(Val)->getNumSuccessors();
   }
-  BasicBlock *getUnwindDest() const;
-  void setUnwindDest(BasicBlock *NewDest);
+  LLVM_ABI BasicBlock *getUnwindDest() const;
+  LLVM_ABI void setUnwindDest(BasicBlock *NewDest);
 
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CleanupRet;
@@ -1677,16 +1687,16 @@ class GetElementPtrInst final
                         // create*()
 
 public:
-  static Value *create(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
-                       InsertPosition Pos, Context &Ctx,
-                       const Twine &NameStr = "");
+  LLVM_ABI static Value *create(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &NameStr = "");
 
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::GetElementPtr;
   }
 
-  Type *getSourceElementType() const;
-  Type *getResultElementType() const;
+  LLVM_ABI Type *getSourceElementType() const;
+  LLVM_ABI Type *getResultElementType() const;
   unsigned getAddressSpace() const {
     return cast<llvm::GetElementPtrInst>(Val)->getAddressSpace();
   }
@@ -1706,11 +1716,11 @@ class GetElementPtrInst final
     return const_cast<GetElementPtrInst *>(this)->indices();
   }
 
-  Value *getPointerOperand() const;
+  LLVM_ABI Value *getPointerOperand() const;
   static unsigned getPointerOperandIndex() {
     return llvm::GetElementPtrInst::getPointerOperandIndex();
   }
-  Type *getPointerOperandType() const;
+  LLVM_ABI Type *getPointerOperandType() const;
   unsigned getPointerAddressSpace() const {
     return cast<llvm::GetElementPtrInst>(Val)->getPointerAddressSpace();
   }
@@ -1750,12 +1760,12 @@ class CatchSwitchInst
   friend class Context; // For accessing the constructor in create*()
 
 public:
-  static CatchSwitchInst *create(Value *ParentPad, BasicBlock *UnwindBB,
-                                 unsigned NumHandlers, InsertPosition Pos,
-                                 Context &Ctx, const Twine &Name = "");
+  LLVM_ABI static CatchSwitchInst *
+  create(Value *ParentPad, BasicBlock *UnwindBB, unsigned NumHandlers,
+         InsertPosition Pos, Context &Ctx, const Twine &Name = "");
 
-  Value *getParentPad() const;
-  void setParentPad(Value *ParentPad);
+  LLVM_ABI Value *getParentPad() const;
+  LLVM_ABI void setParentPad(Value *ParentPad);
 
   bool hasUnwindDest() const {
     return cast<llvm::CatchSwitchInst>(Val)->hasUnwindDest();
@@ -1763,8 +1773,8 @@ class CatchSwitchInst
   bool unwindsToCaller() const {
     return cast<llvm::CatchSwitchInst>(Val)->unwindsToCaller();
   }
-  BasicBlock *getUnwindDest() const;
-  void setUnwindDest(BasicBlock *UnwindDest);
+  LLVM_ABI BasicBlock *getUnwindDest() const;
+  LLVM_ABI void setUnwindDest(BasicBlock *UnwindDest);
 
   unsigned getNumHandlers() const {
     return cast<llvm::CatchSwitchInst>(Val)->getNumHandlers();
@@ -1810,7 +1820,7 @@ class CatchSwitchInst
     return make_range(handler_begin(), handler_end());
   }
 
-  void addHandler(BasicBlock *Dest);
+  LLVM_ABI void addHandler(BasicBlock *Dest);
 
   // TODO: removeHandler() cannot be reverted because there is no equivalent
   // addHandler() with a handler_iterator to specify the position. So we can't
@@ -1839,8 +1849,9 @@ class ResumeInst : public SingleLLVMInstructionImpl<llvm::ResumeInst> {
   friend class Context; // For accessing the constructor in create*()
 
 public:
-  static ResumeInst *create(Value *Exn, InsertPosition Pos, Context &Ctx);
-  Value *getValue() const;
+  LLVM_ABI static ResumeInst *create(Value *Exn, InsertPosition Pos,
+                                     Context &Ctx);
+  LLVM_ABI Value *getValue() const;
   unsigned getNumSuccessors() const {
     return cast<llvm::ResumeInst>(Val)->getNumSuccessors();
   }
@@ -1858,17 +1869,17 @@ class SwitchInst : public SingleLLVMInstructionImpl<llvm::SwitchInst> {
   static constexpr const unsigned DefaultPseudoIndex =
       llvm::SwitchInst::DefaultPseudoIndex;
 
-  static SwitchInst *create(Value *V, BasicBlock *Dest, unsigned NumCases,
-                            InsertPosition Pos, Context &Ctx,
-                            const Twine &Name = "");
+  LLVM_ABI static SwitchInst *create(Value *V, BasicBlock *Dest,
+                                     unsigned NumCases, InsertPosition Pos,
+                                     Context &Ctx, const Twine &Name = "");
 
-  Value *getCondition() const;
-  void setCondition(Value *V);
-  BasicBlock *getDefaultDest() const;
+  LLVM_ABI Value *getCondition() const;
+  LLVM_ABI void setCondition(Value *V);
+  LLVM_ABI BasicBlock *getDefaultDest() const;
   bool defaultDestUnreachable() const {
     return cast<llvm::SwitchInst>(Val)->defaultDestUnreachable();
   }
-  void setDefaultDest(BasicBlock *DefaultCase);
+  LLVM_ABI void setDefaultDest(BasicBlock *DefaultCase);
   unsigned getNumCases() const {
     return cast<llvm::SwitchInst>(Val)->getNumCases();
   }
@@ -1913,9 +1924,9 @@ class SwitchInst : public SingleLLVMInstructionImpl<llvm::SwitchInst> {
       return I;
     return case_default();
   }
-  ConstantInt *findCaseDest(BasicBlock *BB);
+  LLVM_ABI ConstantInt *findCaseDest(BasicBlock *BB);
 
-  void addCase(ConstantInt *OnVal, BasicBlock *Dest);
+  LLVM_ABI void addCase(ConstantInt *OnVal, BasicBlock *Dest);
   /// This method removes the specified case and its successor from the switch
   /// instruction. Note that this operation may reorder the remaining cases at
   /// index idx and above.
@@ -1923,13 +1934,13 @@ class SwitchInst : public SingleLLVMInstructionImpl<llvm::SwitchInst> {
   /// This action invalidates iterators for all cases following the one removed,
   /// including the case_end() iterator. It returns an iterator for the next
   /// case.
-  CaseIt removeCase(CaseIt It);
+  LLVM_ABI CaseIt removeCase(CaseIt It);
 
   unsigned getNumSuccessors() const {
     return cast<llvm::SwitchInst>(Val)->getNumSuccessors();
   }
-  BasicBlock *getSuccessor(unsigned Idx) const;
-  void setSuccessor(unsigned Idx, BasicBlock *NewSucc);
+  LLVM_ABI BasicBlock *getSuccessor(unsigned Idx) const;
+  LLVM_ABI void setSuccessor(unsigned Idx, BasicBlock *NewSucc);
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Switch;
   }
@@ -1950,11 +1961,13 @@ class UnaryOperator : public UnaryInstruction {
                          Ctx) {}
   friend Context; // for constructor.
 public:
-  static Value *create(Instruction::Opcode Op, Value *OpV, InsertPosition Pos,
-                       Context &Ctx, const Twine &Name = "");
-  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
-                                      Value *CopyFrom, InsertPosition Pos,
-                                      Context &Ctx, const Twine &Name = "");
+  LLVM_ABI static Value *create(Instruction::Opcode Op, Value *OpV,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
+  LLVM_ABI static Value *createWithCopiedFlags(Instruction::Opcode Op,
+                                               Value *OpV, Value *CopyFrom,
+                                               InsertPosition Pos, Context &Ctx,
+                                               const Twine &Name = "");
   /// For isa/dyn_cast.
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::UnOp;
@@ -2013,14 +2026,15 @@ class BinaryOperator : public SingleLLVMInstructionImpl<llvm::BinaryOperator> {
   friend class Context; // For constructor.
 
 public:
-  static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
-                       InsertPosition Pos, Context &Ctx,
-                       const Twine &Name = "");
-
-  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
-                                      Value *RHS, Value *CopyFrom,
-                                      InsertPosition Pos, Context &Ctx,
-                                      const Twine &Name = "");
+  LLVM_ABI static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
+
+  LLVM_ABI static Value *createWithCopiedFlags(Instruction::Opcode Op,
+                                               Value *LHS, Value *RHS,
+                                               Value *CopyFrom,
+                                               InsertPosition Pos, Context &Ctx,
+                                               const Twine &Name = "");
   /// For isa/dyn_cast.
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::BinaryOperator;
@@ -2033,7 +2047,7 @@ class BinaryOperator : public SingleLLVMInstructionImpl<llvm::BinaryOperator> {
 /// can also be treated as an add.
 class PossiblyDisjointInst : public BinaryOperator {
 public:
-  void setIsDisjoint(bool B);
+  LLVM_ABI void setIsDisjoint(bool B);
   bool isDisjoint() const {
     return cast<llvm::PossiblyDisjointInst>(Val)->isDisjoint();
   }
@@ -2066,24 +2080,24 @@ class AtomicRMWInst : public SingleLLVMInstructionImpl<llvm::AtomicRMWInst> {
     cast<llvm::AtomicRMWInst>(Val)->setOperation(Op);
   }
   Align getAlign() const { return cast<llvm::AtomicRMWInst>(Val)->getAlign(); }
-  void setAlignment(Align Align);
+  LLVM_ABI void setAlignment(Align Align);
   bool isVolatile() const {
     return cast<llvm::AtomicRMWInst>(Val)->isVolatile();
   }
-  void setVolatile(bool V);
+  LLVM_ABI void setVolatile(bool V);
   AtomicOrdering getOrdering() const {
     return cast<llvm::AtomicRMWInst>(Val)->getOrdering();
   }
-  void setOrdering(AtomicOrdering Ordering);
+  LLVM_ABI void setOrdering(AtomicOrdering Ordering);
   SyncScope::ID getSyncScopeID() const {
     return cast<llvm::AtomicRMWInst>(Val)->getSyncScopeID();
   }
-  void setSyncScopeID(SyncScope::ID SSID);
-  Value *getPointerOperand();
+  LLVM_ABI void setSyncScopeID(SyncScope::ID SSID);
+  LLVM_ABI Value *getPointerOperand();
   const Value *getPointerOperand() const {
     return const_cast<AtomicRMWInst *>(this)->getPointerOperand();
   }
-  Value *getValOperand();
+  LLVM_ABI Value *getValOperand();
   const Value *getValOperand() const {
     return const_cast<AtomicRMWInst *>(this)->getValOperand();
   }
@@ -2097,11 +2111,10 @@ class AtomicRMWInst : public SingleLLVMInstructionImpl<llvm::AtomicRMWInst> {
     return From->getSubclassID() == ClassID::AtomicRMW;
   }
 
-  static AtomicRMWInst *create(BinOp Op, Value *Ptr, Value *Val,
-                               MaybeAlign Align, AtomicOrdering Ordering,
-                               InsertPosition Pos, Context &Ctx,
-                               SyncScope::ID SSID = SyncScope::System,
-                               const Twine &Name = "");
+  LLVM_ABI static AtomicRMWInst *
+  create(BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align,
+         AtomicOrdering Ordering, InsertPosition Pos, Context &Ctx,
+         SyncScope::ID SSID = SyncScope::System, const Twine &Name = "");
 };
 
 class AtomicCmpXchgInst
@@ -2119,17 +2132,17 @@ class AtomicCmpXchgInst
     return cast<llvm::AtomicCmpXchgInst>(Val)->getAlign();
   }
 
-  void setAlignment(Align Align);
+  LLVM_ABI void setAlignment(Align Align);
   /// Return true if this is a cmpxchg from a volatile memory
   /// location.
   bool isVolatile() const {
     return cast<llvm::AtomicCmpXchgInst>(Val)->isVolatile();
   }
   /// Specify whether this is a volatile cmpxchg.
-  void setVolatile(bool V);
+  LLVM_ABI void setVolatile(bool V);
   /// Return true if this cmpxchg may spuriously fail.
   bool isWeak() const { return cast<llvm::AtomicCmpXchgInst>(Val)->isWeak(); }
-  void setWeak(bool IsWeak);
+  LLVM_ABI void setWeak(bool IsWeak);
   static bool isValidSuccessOrdering(AtomicOrdering Ordering) {
     return llvm::AtomicCmpXchgInst::isValidSuccessOrdering(Ordering);
   }
@@ -2139,30 +2152,30 @@ class AtomicCmpXchgInst
   AtomicOrdering getSuccessOrdering() const {
     return cast<llvm::AtomicCmpXchgInst>(Val)->getSuccessOrdering();
   }
-  void setSuccessOrdering(AtomicOrdering Ordering);
+  LLVM_ABI void setSuccessOrdering(AtomicOrdering Ordering);
 
   AtomicOrdering getFailureOrdering() const {
     return cast<llvm::AtomicCmpXchgInst>(Val)->getFailureOrdering();
   }
-  void setFailureOrdering(AtomicOrdering Ordering);
+  LLVM_ABI void setFailureOrdering(AtomicOrdering Ordering);
   AtomicOrdering getMergedOrdering() const {
     return cast<llvm::AtomicCmpXchgInst>(Val)->getMergedOrdering();
   }
   SyncScope::ID getSyncScopeID() const {
     return cast<llvm::AtomicCmpXchgInst>(Val)->getSyncScopeID();
   }
-  void setSyncScopeID(SyncScope::ID SSID);
-  Value *getPointerOperand();
+  LLVM_ABI void setSyncScopeID(SyncScope::ID SSID);
+  LLVM_ABI Value *getPointerOperand();
   const Value *getPointerOperand() const {
     return const_cast<AtomicCmpXchgInst *>(this)->getPointerOperand();
   }
 
-  Value *getCompareOperand();
+  LLVM_ABI Value *getCompareOperand();
   const Value *getCompareOperand() const {
     return const_cast<AtomicCmpXchgInst *>(this)->getCompareOperand();
   }
 
-  Value *getNewValOperand();
+  LLVM_ABI Value *getNewValOperand();
   const Value *getNewValOperand() const {
     return const_cast<AtomicCmpXchgInst *>(this)->getNewValOperand();
   }
@@ -2172,7 +2185,7 @@ class AtomicCmpXchgInst
     return cast<llvm::AtomicCmpXchgInst>(Val)->getPointerAddressSpace();
   }
 
-  static AtomicCmpXchgInst *
+  LLVM_ABI static AtomicCmpXchgInst *
   create(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
          AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
          InsertPosition Pos, Context &Ctx,
@@ -2190,9 +2203,10 @@ class AllocaInst final : public UnaryInstruction {
   friend class Context; // For constructor.
 
 public:
-  static AllocaInst *create(Type *Ty, unsigned AddrSpace, InsertPosition Pos,
-                            Context &Ctx, Value *ArraySize = nullptr,
-                            const Twine &Name = "");
+  LLVM_ABI static AllocaInst *create(Type *Ty, unsigned AddrSpace,
+                                     InsertPosition Pos, Context &Ctx,
+                                     Value *ArraySize = nullptr,
+                                     const Twine &Name = "");
 
   /// Return true if there is an allocation size parameter to the allocation
   /// instruction that is not 1.
@@ -2201,12 +2215,12 @@ class AllocaInst final : public UnaryInstruction {
   }
   /// Get the number of elements allocated. For a simple allocation of a single
   /// element, this will return a constant 1 value.
-  Value *getArraySize();
+  LLVM_ABI Value *getArraySize();
   const Value *getArraySize() const {
     return const_cast<AllocaInst *>(this)->getArraySize();
   }
   /// Overload to return most specific pointer type.
-  PointerType *getType() const;
+  LLVM_ABI PointerType *getType() const;
   /// Return the address space for the allocation.
   unsigned getAddressSpace() const {
     return cast<llvm::AllocaInst>(Val)->getAddressSpace();
@@ -2222,14 +2236,14 @@ class AllocaInst final : public UnaryInstruction {
     return cast<llvm::AllocaInst>(Val)->getAllocationSizeInBits(DL);
   }
   /// Return the type that is being allocated by the instruction.
-  Type *getAllocatedType() const;
+  LLVM_ABI Type *getAllocatedType() const;
   /// for use only in special circumstances that need to generically
   /// transform a whole instruction (eg: IR linking and vectorization).
-  void setAllocatedType(Type *Ty);
+  LLVM_ABI void setAllocatedType(Type *Ty);
   /// Return the alignment of the memory that is being allocated by the
   /// instruction.
   Align getAlign() const { return cast<llvm::AllocaInst>(Val)->getAlign(); }
-  void setAlignment(Align Align);
+  LLVM_ABI void setAlignment(Align Align);
   /// Return true if this alloca is in the entry block of the function and is a
   /// constant size. If so, the code generator will fold it into the
   /// prolog/epilog code, so it is basically free.
@@ -2242,7 +2256,7 @@ class AllocaInst final : public UnaryInstruction {
     return cast<llvm::AllocaInst>(Val)->isUsedWithInAlloca();
   }
   /// Specify whether this alloca is used to represent the arguments to a call.
-  void setUsedWithInAlloca(bool V);
+  LLVM_ABI void setUsedWithInAlloca(bool V);
 
   static bool classof(const Value *From) {
     if (auto *I = dyn_cast<Instruction>(From))
@@ -2293,13 +2307,13 @@ class CastInst : public UnaryInstruction {
   friend Context; // for SBCastInstruction()
 
 public:
-  static Value *create(Type *DestTy, Opcode Op, Value *Operand,
-                       InsertPosition Pos, Context &Ctx,
-                       const Twine &Name = "");
+  LLVM_ABI static Value *create(Type *DestTy, Opcode Op, Value *Operand,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
   /// For isa/dyn_cast.
-  static bool classof(const Value *From);
-  Type *getSrcTy() const;
-  Type *getDestTy() const;
+  LLVM_ABI static bool classof(const Value *From);
+  LLVM_ABI Type *getSrcTy() const;
+  LLVM_ABI Type *getDestTy() const;
 };
 
 /// Instruction that can have a nneg flag (zext/uitofp).
@@ -2308,7 +2322,7 @@ class PossiblyNonNegInst : public CastInst {
   bool hasNonNeg() const {
     return cast<llvm::PossiblyNonNegInst>(Val)->hasNonNeg();
   }
-  void setNonNeg(bool B);
+  LLVM_ABI void setNonNeg(bool B);
   /// For isa/dyn_cast.
   static bool classof(const Value *From) {
     if (auto *I = dyn_cast<Instruction>(From)) {
@@ -2383,15 +2397,15 @@ class PHINode final : public SingleLLVMInstructionImpl<llvm::PHINode> {
   struct LLVMBBToBB {
     Context &Ctx;
     LLVMBBToBB(Context &Ctx) : Ctx(Ctx) {}
-    BasicBlock *operator()(llvm::BasicBlock *LLVMBB) const;
+    LLVM_ABI BasicBlock *operator()(llvm::BasicBlock *LLVMBB) const;
   };
 
 public:
-  static PHINode *create(Type *Ty, unsigned NumReservedValues,
-                         InsertPosition Pos, Context &Ctx,
-                         const Twine &Name = "");
+  LLVM_ABI static PHINode *create(Type *Ty, unsigned NumReservedValues,
+                                  InsertPosition Pos, Context &Ctx,
+                                  const Twine &Name = "");
   /// For isa/dyn_cast.
-  static bool classof(const Value *From);
+  LLVM_ABI static bool classof(const Value *From);
 
   using const_block_iterator =
       mapped_iterator<llvm::PHINode::const_block_iterator, LLVMBBToBB>;
@@ -2417,35 +2431,36 @@ class PHINode final : public SingleLLVMInstructionImpl<llvm::PHINode> {
   unsigned getNumIncomingValues() const {
     return cast<llvm::PHINode>(Val)->getNumIncomingValues();
   }
-  Value *getIncomingValue(unsigned Idx) const;
-  void setIncomingValue(unsigned Idx, Value *V);
+  LLVM_ABI Value *getIncomingValue(unsigned Idx) const;
+  LLVM_ABI void setIncomingValue(unsigned Idx, Value *V);
   static unsigned getOperandNumForIncomingValue(unsigned Idx) {
     return llvm::PHINode::getOperandNumForIncomingValue(Idx);
   }
   static unsigned getIncomingValueNumForOperand(unsigned Idx) {
     return llvm::PHINode::getIncomingValueNumForOperand(Idx);
   }
-  BasicBlock *getIncomingBlock(unsigned Idx) const;
-  BasicBlock *getIncomingBlock(const Use &U) const;
+  LLVM_ABI BasicBlock *getIncomingBlock(unsigned Idx) const;
+  LLVM_ABI BasicBlock *getIncomingBlock(const Use &U) const;
 
-  void setIncomingBlock(unsigned Idx, BasicBlock *BB);
+  LLVM_ABI void setIncomingBlock(unsigned Idx, BasicBlock *BB);
 
-  void addIncoming(Value *V, BasicBlock *BB);
+  LLVM_ABI void addIncoming(Value *V, BasicBlock *BB);
 
-  Value *removeIncomingValue(unsigned Idx);
-  Value *removeIncomingValue(BasicBlock *BB);
+  LLVM_ABI Value *removeIncomingValue(unsigned Idx);
+  LLVM_ABI Value *removeIncomingValue(BasicBlock *BB);
 
-  int getBasicBlockIndex(const BasicBlock *BB) const;
-  Value *getIncomingValueForBlock(const BasicBlock *BB) const;
+  LLVM_ABI int getBasicBlockIndex(const BasicBlock *BB) const;
+  LLVM_ABI Value *getIncomingValueForBlock(const BasicBlock *BB) const;
 
-  Value *hasConstantValue() const;
+  LLVM_ABI Value *hasConstantValue() const;
 
   bool hasConstantOrUndefValue() const {
     return cast<llvm::PHINode>(Val)->hasConstantOrUndefValue();
   }
   bool isComplete() const { return cast<llvm::PHINode>(Val)->isComplete(); }
-  void replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New);
-  void removeIncomingValueIf(function_ref<bool(unsigned)> Predicate);
+  LLVM_ABI void replaceIncomingBlockWith(const BasicBlock *Old,
+                                         BasicBlock *New);
+  LLVM_ABI void removeIncomingValueIf(function_ref<bool(unsigned)> Predicate);
   // TODO: Implement
   // void copyIncomingBlocks(iterator_range<const_block_iterator> BBRange,
   //                         uint32_t ToIdx = 0)
@@ -2471,21 +2486,23 @@ class CmpInst : public SingleLLVMInstructionImpl<llvm::CmpInst> {
   CmpInst(llvm::CmpInst *CI, Context &Ctx, ClassID Id, Opcode Opc)
       : SingleLLVMInstructionImpl(Id, Opc, CI, Ctx) {}
   friend Context; // for CmpInst()
-  static Value *createCommon(Value *Cond, Value *True, Value *False,
-                             const Twine &Name, IRBuilder<> &Builder,
-                             Context &Ctx);
+  LLVM_ABI static Value *createCommon(Value *Cond, Value *True, Value *False,
+                                      const Twine &Name, IRBuilder<> &Builder,
+                                      Context &Ctx);
 
 public:
   using Predicate = llvm::CmpInst::Predicate;
 
-  static Value *create(Predicate Pred, Value *S1, Value *S2, InsertPosition Pos,
-                       Context &Ctx, const Twine &Name = "");
-  static Value *createWithCopiedFlags(Predicate Pred, Value *S1, Value *S2,
-                                      const Instruction *FlagsSource,
-                                      InsertPosition Pos, Context &Ctx,
-                                      const Twine &Name = "");
-  void setPredicate(Predicate P);
-  void swapOperands();
+  LLVM_ABI static Value *create(Predicate Pred, Value *S1, Value *S2,
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
+  LLVM_ABI static Value *createWithCopiedFlags(Predicate Pred, Value *S1,
+                                               Value *S2,
+                                               const Instruction *FlagsSource,
+                                               InsertPosition Pos, Context &Ctx,
+                                               const Twine &Name = "");
+  LLVM_ABI void setPredicate(Predicate P);
+  LLVM_ABI void swapOperands();
 
   WRAP_MEMBER(getPredicate);
   WRAP_BOTH(isFPPredicate);
@@ -2517,7 +2534,7 @@ class CmpInst : public SingleLLVMInstructionImpl<llvm::CmpInst> {
   }
 
   /// Create a result type for fcmp/icmp
-  static Type *makeCmpResultType(Type *OpndType);
+  LLVM_ABI static Type *makeCmpResultType(Type *OpndType);
 
 #ifndef NDEBUG
   void dumpOS(raw_ostream &OS) const override;
@@ -2533,7 +2550,7 @@ class ICmpInst : public CmpInst {
   using LLVMValType = llvm::ICmpInst;
 
 public:
-  void swapOperands();
+  LLVM_ABI void swapOperands();
 
   WRAP_BOTH(getSignedPredicate);
   WRAP_BOTH(getUnsignedPredicate);
@@ -2570,7 +2587,7 @@ class FCmpInst : public CmpInst {
   using LLVMValType = llvm::FCmpInst;
 
 public:
-  void swapOperands();
+  LLVM_ABI void swapOperands();
 
   WRAP_BOTH(isEquality);
   WRAP_MEMBER(isCommutative);
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Module.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Module.h
index 429bb04539bc..275960392211 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Module.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Module.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -38,7 +39,7 @@ class Module {
 public:
   Context &getContext() const { return Ctx; }
 
-  Function *getFunction(StringRef Name) const;
+  LLVM_ABI Function *getFunction(StringRef Name) const;
 
   const DataLayout &getDataLayout() const { return LLVMM.getDataLayout(); }
 
@@ -50,7 +51,8 @@ class Module {
   /// does not exist, return null. If AllowInternal is set to true, this
   /// function will return types that have InternalLinkage. By default, these
   /// types are not returned.
-  GlobalVariable *getGlobalVariable(StringRef Name, bool AllowInternal) const;
+  LLVM_ABI GlobalVariable *getGlobalVariable(StringRef Name,
+                                             bool AllowInternal) const;
   GlobalVariable *getGlobalVariable(StringRef Name) const {
     return getGlobalVariable(Name, /*AllowInternal=*/false);
   }
@@ -66,12 +68,12 @@ class Module {
   /// Return the global alias in the module with the specified name, of
   /// arbitrary type. This method returns null if a global with the specified
   /// name is not found.
-  GlobalAlias *getNamedAlias(StringRef Name) const;
+  LLVM_ABI GlobalAlias *getNamedAlias(StringRef Name) const;
 
   /// Return the global ifunc in the module with the specified name, of
   /// arbitrary type. This method returns null if a global with the specified
   /// name is not found.
-  GlobalIFunc *getNamedIFunc(StringRef Name) const;
+  LLVM_ABI GlobalIFunc *getNamedIFunc(StringRef Name) const;
 
   // TODO: Missing removeGlobalVariable() eraseGlobalVariable(),
   // insertGlobalVariable()
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/PassManager.h b/external/llvm-project/llvm/include/llvm/SandboxIR/PassManager.h
index 55a0301f4756..6fccaf04b270 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/PassManager.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/PassManager.h
@@ -18,6 +18,7 @@
 #ifndef LLVM_SANDBOXIR_PASSMANAGER_H
 #define LLVM_SANDBOXIR_PASSMANAGER_H
 
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 #include "llvm/ADT/DenseMap.h"
@@ -201,7 +202,7 @@ class PassManager : public ParentPass {
   }
 };
 
-class FunctionPassManager final
+class LLVM_ABI FunctionPassManager final
     : public PassManager<FunctionPass, FunctionPass> {
 public:
   FunctionPassManager(StringRef Name) : PassManager(Name) {}
@@ -211,7 +212,8 @@ class FunctionPassManager final
   bool runOnFunction(Function &F, const Analyses &A) final;
 };
 
-class RegionPassManager final : public PassManager<RegionPass, RegionPass> {
+class LLVM_ABI RegionPassManager final
+    : public PassManager<RegionPass, RegionPass> {
 public:
   RegionPassManager(StringRef Name) : PassManager(Name) {}
   RegionPassManager(StringRef Name, StringRef Pipeline,
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Region.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Region.h
index f86199ab6c22..d70f21277fb1 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Region.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Region.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_SANDBOXIR_REGION_H
 #define LLVM_SANDBOXIR_REGION_H
 
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 #include "llvm/ADT/SetVector.h"
@@ -30,7 +31,7 @@ class ScoreBoard {
   /// The cost of all instructions that got removed and replaced by new ones.
   InstructionCost BeforeCost = 0;
   /// Helper for both add() and remove(). \Returns the TTI cost of \p I.
-  InstructionCost getCost(Instruction *I) const;
+  LLVM_ABI InstructionCost getCost(Instruction *I) const;
   /// No need to allow copies.
   ScoreBoard(const ScoreBoard &) = delete;
   const ScoreBoard &operator=(const ScoreBoard &) = delete;
@@ -40,7 +41,7 @@ class ScoreBoard {
   /// Mark \p I as a newly added instruction to the region.
   void add(Instruction *I) { AfterCost += getCost(I); }
   /// Mark \p I as a deleted instruction from the region.
-  void remove(Instruction *I);
+  LLVM_ABI void remove(Instruction *I);
   /// \Returns the cost of the newly added instructions.
   InstructionCost getAfterCost() const { return AfterCost; }
   /// \Returns the cost of the Removed instructions.
@@ -122,12 +123,12 @@ class Region {
   /// add an instruction to the auxiliary vector it does get tagged as being a
   /// member of the region (for ownership reasons), but its cost does not get
   /// counted because the instruction hasn't been added in the "normal" way.
-  void addImpl(Instruction *I, bool IgnoreCost);
+  LLVM_ABI void addImpl(Instruction *I, bool IgnoreCost);
   /// Adds I to the set. This is the main API for adding an instruction to the
   /// region.
   void add(Instruction *I) { addImpl(I, /*IgnoreCost=*/false); }
   /// Removes I from the set.
-  void remove(Instruction *I);
+  LLVM_ABI void remove(Instruction *I);
   friend class Context; // The callbacks need to call add() and remove().
   friend class RegionInternalsAttorney; // For unit tests.
   friend class RegionsFromBBs;          // For add().
@@ -141,8 +142,8 @@ class Region {
   void removeFromAux(Instruction *I);
 
 public:
-  Region(Context &Ctx, TargetTransformInfo &TTI);
-  ~Region();
+  LLVM_ABI Region(Context &Ctx, TargetTransformInfo &TTI);
+  LLVM_ABI ~Region();
 
   Context &getContext() const { return Ctx; }
   /// Returns true if I is in the Region.
@@ -150,18 +151,18 @@ class Region {
   /// Returns true if the Region has no instructions.
   bool empty() const { return Insts.empty(); }
   /// Set the auxiliary vector.
-  void setAux(ArrayRef<Instruction *> Aux);
+  LLVM_ABI void setAux(ArrayRef<Instruction *> Aux);
   /// \Returns the auxiliary vector.
   const SmallVector<Instruction *> &getAux() const { return Aux; }
   /// Clears all auxiliary data.
-  void clearAux();
+  LLVM_ABI void clearAux();
 
   using iterator = decltype(Insts.begin());
   iterator begin() { return Insts.begin(); }
   iterator end() { return Insts.end(); }
   iterator_range<iterator> insts() { return make_range(begin(), end()); }
 
-  static SmallVector<std::unique_ptr<Region>>
+  LLVM_ABI static SmallVector<std::unique_ptr<Region>>
   createRegionsFromMD(Function &F, TargetTransformInfo &TTI);
   /// \Returns the ScoreBoard data structure that keeps track of instr costs.
   const ScoreBoard &getScoreboard() const { return Scoreboard; }
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Tracker.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Tracker.h
index f7b469965eae..9a2c9dd51648 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Tracker.h
@@ -46,6 +46,8 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/SandboxIR/Use.h"
+#include "llvm/SandboxIR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include <memory>
 
@@ -149,7 +151,7 @@ class UseSet : public IRChangeBase {
 #endif
 };
 
-class PHIRemoveIncoming : public IRChangeBase {
+class LLVM_ABI PHIRemoveIncoming : public IRChangeBase {
   PHINode *PHI;
   unsigned RemovedIdx;
   Value *RemovedV;
@@ -165,7 +167,7 @@ class PHIRemoveIncoming : public IRChangeBase {
 #endif
 };
 
-class PHIAddIncoming : public IRChangeBase {
+class LLVM_ABI PHIAddIncoming : public IRChangeBase {
   PHINode *PHI;
   unsigned Idx;
 
@@ -179,7 +181,7 @@ class PHIAddIncoming : public IRChangeBase {
 #endif
 };
 
-class CmpSwapOperands : public IRChangeBase {
+class LLVM_ABI CmpSwapOperands : public IRChangeBase {
   CmpInst *Cmp;
 
 public:
@@ -210,7 +212,7 @@ class UseSwap : public IRChangeBase {
 #endif
 };
 
-class EraseFromParent : public IRChangeBase {
+class LLVM_ABI EraseFromParent : public IRChangeBase {
   /// Contains all the data we need to restore an "erased" (i.e., detached)
   /// instruction: the instruction itself and its operands in order.
   struct InstrAndOperands {
@@ -242,7 +244,7 @@ class EraseFromParent : public IRChangeBase {
 #endif
 };
 
-class RemoveFromParent : public IRChangeBase {
+class LLVM_ABI RemoveFromParent : public IRChangeBase {
   /// The instruction that is about to get removed.
   Instruction *RemovedI = nullptr;
   /// This is either the next instr, or the parent BB if at the end of the BB.
@@ -327,7 +329,7 @@ class GenericSetterWithIdx final : public IRChangeBase {
 #endif
 };
 
-class CatchSwitchAddHandler : public IRChangeBase {
+class LLVM_ABI CatchSwitchAddHandler : public IRChangeBase {
   CatchSwitchInst *CSI;
   unsigned HandlerIdx;
 
@@ -344,7 +346,7 @@ class CatchSwitchAddHandler : public IRChangeBase {
 #endif // NDEBUG
 };
 
-class SwitchAddCase : public IRChangeBase {
+class LLVM_ABI SwitchAddCase : public IRChangeBase {
   SwitchInst *Switch;
   ConstantInt *Val;
 
@@ -359,7 +361,7 @@ class SwitchAddCase : public IRChangeBase {
 #endif // NDEBUG
 };
 
-class SwitchRemoveCase : public IRChangeBase {
+class LLVM_ABI SwitchRemoveCase : public IRChangeBase {
   SwitchInst *Switch;
   struct Case {
     ConstantInt *Val;
@@ -378,7 +380,7 @@ class SwitchRemoveCase : public IRChangeBase {
 #endif // NDEBUG
 };
 
-class MoveInstr : public IRChangeBase {
+class LLVM_ABI MoveInstr : public IRChangeBase {
   /// The instruction that moved.
   Instruction *MovedI;
   /// This is either the next instruction in the block, or the parent BB if at
@@ -395,7 +397,7 @@ class MoveInstr : public IRChangeBase {
 #endif // NDEBUG
 };
 
-class InsertIntoBB final : public IRChangeBase {
+class LLVM_ABI InsertIntoBB final : public IRChangeBase {
   Instruction *InsertedI = nullptr;
 
 public:
@@ -408,7 +410,7 @@ class InsertIntoBB final : public IRChangeBase {
 #endif // NDEBUG
 };
 
-class CreateAndInsertInst final : public IRChangeBase {
+class LLVM_ABI CreateAndInsertInst final : public IRChangeBase {
   Instruction *NewI = nullptr;
 
 public:
@@ -421,7 +423,7 @@ class CreateAndInsertInst final : public IRChangeBase {
 #endif
 };
 
-class ShuffleVectorSetMask final : public IRChangeBase {
+class LLVM_ABI ShuffleVectorSetMask final : public IRChangeBase {
   ShuffleVectorInst *SVI;
   SmallVector<int, 8> PrevMask;
 
@@ -472,7 +474,7 @@ class Tracker {
   {
   }
 
-  ~Tracker();
+  LLVM_ABI ~Tracker();
   Context &getContext() const { return Ctx; }
   /// \Returns true if there are no changes tracked.
   bool empty() const { return Changes.empty(); }
@@ -506,11 +508,11 @@ class Tracker {
   /// \Returns the current state of the tracker.
   TrackerState getState() const { return State; }
   /// Turns on IR tracking.
-  void save();
+  LLVM_ABI void save();
   /// Stops tracking and accept changes.
-  void accept();
+  LLVM_ABI void accept();
   /// Stops tracking and reverts to saved state.
-  void revert();
+  LLVM_ABI void revert();
 
 #ifndef NDEBUG
   void dump(raw_ostream &OS) const;
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Type.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Type.h
index f90ae096443b..d9c5e6c098da 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Type.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Type.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -264,18 +265,18 @@ class Type {
 
   /// If this is a vector type, return the element type, otherwise return
   /// 'this'.
-  Type *getScalarType() const;
+  LLVM_ABI Type *getScalarType() const;
 
   // TODO: ADD MISSING
 
-  static Type *getInt64Ty(Context &Ctx);
-  static Type *getInt32Ty(Context &Ctx);
-  static Type *getInt16Ty(Context &Ctx);
-  static Type *getInt8Ty(Context &Ctx);
-  static Type *getInt1Ty(Context &Ctx);
-  static Type *getDoubleTy(Context &Ctx);
-  static Type *getFloatTy(Context &Ctx);
-  static Type *getHalfTy(Context &Ctx);
+  LLVM_ABI static Type *getInt64Ty(Context &Ctx);
+  LLVM_ABI static Type *getInt32Ty(Context &Ctx);
+  LLVM_ABI static Type *getInt16Ty(Context &Ctx);
+  LLVM_ABI static Type *getInt8Ty(Context &Ctx);
+  LLVM_ABI static Type *getInt1Ty(Context &Ctx);
+  LLVM_ABI static Type *getDoubleTy(Context &Ctx);
+  LLVM_ABI static Type *getFloatTy(Context &Ctx);
+  LLVM_ABI static Type *getHalfTy(Context &Ctx);
   // TODO: missing get*
 
   /// Get the address space of this pointer or pointer vector type.
@@ -293,7 +294,7 @@ class PointerType : public Type {
 public:
   // TODO: add missing functions
 
-  static PointerType *get(Context &Ctx, unsigned AddressSpace);
+  LLVM_ABI static PointerType *get(Context &Ctx, unsigned AddressSpace);
 
   static bool classof(const Type *From) {
     return isa<llvm::PointerType>(From->LLVMTy);
@@ -302,7 +303,7 @@ class PointerType : public Type {
 
 class ArrayType : public Type {
 public:
-  static ArrayType *get(Type *ElementType, uint64_t NumElements);
+  LLVM_ABI static ArrayType *get(Type *ElementType, uint64_t NumElements);
   // TODO: add missing functions
   static bool classof(const Type *From) {
     return isa<llvm::ArrayType>(From->LLVMTy);
@@ -312,8 +313,8 @@ class ArrayType : public Type {
 class StructType : public Type {
 public:
   /// This static method is the primary way to create a literal StructType.
-  static StructType *get(Context &Ctx, ArrayRef<Type *> Elements,
-                         bool IsPacked = false);
+  LLVM_ABI static StructType *get(Context &Ctx, ArrayRef<Type *> Elements,
+                                  bool IsPacked = false);
 
   bool isPacked() const { return cast<llvm::StructType>(LLVMTy)->isPacked(); }
 
@@ -325,13 +326,13 @@ class StructType : public Type {
 
 class VectorType : public Type {
 public:
-  static VectorType *get(Type *ElementType, ElementCount EC);
+  LLVM_ABI static VectorType *get(Type *ElementType, ElementCount EC);
   static VectorType *get(Type *ElementType, unsigned NumElements,
                          bool Scalable) {
     return VectorType::get(ElementType,
                            ElementCount::get(NumElements, Scalable));
   }
-  Type *getElementType() const;
+  LLVM_ABI Type *getElementType() const;
 
   static VectorType *get(Type *ElementType, const VectorType *Other) {
     return VectorType::get(ElementType, Other->getElementCount());
@@ -340,13 +341,14 @@ class VectorType : public Type {
   inline ElementCount getElementCount() const {
     return cast<llvm::VectorType>(LLVMTy)->getElementCount();
   }
-  static VectorType *getInteger(VectorType *VTy);
-  static VectorType *getExtendedElementVectorType(VectorType *VTy);
-  static VectorType *getTruncatedElementVectorType(VectorType *VTy);
-  static VectorType *getSubdividedVectorType(VectorType *VTy, int NumSubdivs);
-  static VectorType *getHalfElementsVectorType(VectorType *VTy);
-  static VectorType *getDoubleElementsVectorType(VectorType *VTy);
-  static bool isValidElementType(Type *ElemTy);
+  LLVM_ABI static VectorType *getInteger(VectorType *VTy);
+  LLVM_ABI static VectorType *getExtendedElementVectorType(VectorType *VTy);
+  LLVM_ABI static VectorType *getTruncatedElementVectorType(VectorType *VTy);
+  LLVM_ABI static VectorType *getSubdividedVectorType(VectorType *VTy,
+                                                      int NumSubdivs);
+  LLVM_ABI static VectorType *getHalfElementsVectorType(VectorType *VTy);
+  LLVM_ABI static VectorType *getDoubleElementsVectorType(VectorType *VTy);
+  LLVM_ABI static bool isValidElementType(Type *ElemTy);
 
   static bool classof(const Type *From) {
     return isa<llvm::VectorType>(From->LLVMTy);
@@ -355,7 +357,7 @@ class VectorType : public Type {
 
 class FixedVectorType : public VectorType {
 public:
-  static FixedVectorType *get(Type *ElementType, unsigned NumElts);
+  LLVM_ABI static FixedVectorType *get(Type *ElementType, unsigned NumElts);
 
   static FixedVectorType *get(Type *ElementType, const FixedVectorType *FVTy) {
     return get(ElementType, FVTy->getNumElements());
@@ -399,7 +401,8 @@ class FixedVectorType : public VectorType {
 
 class ScalableVectorType : public VectorType {
 public:
-  static ScalableVectorType *get(Type *ElementType, unsigned MinNumElts);
+  LLVM_ABI static ScalableVectorType *get(Type *ElementType,
+                                          unsigned MinNumElts);
 
   static ScalableVectorType *get(Type *ElementType,
                                  const ScalableVectorType *SVTy) {
@@ -462,7 +465,7 @@ class FunctionType : public Type {
 /// Integer representation type
 class IntegerType : public Type {
 public:
-  static IntegerType *get(Context &C, unsigned NumBits);
+  LLVM_ABI static IntegerType *get(Context &C, unsigned NumBits);
   // TODO: add missing functions
   static bool classof(const Type *From) {
     return isa<llvm::IntegerType>(From->LLVMTy);
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Use.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Use.h
index c4a774aa3a89..5c02c4f2b349 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Use.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Use.h
@@ -14,6 +14,7 @@
 #define LLVM_SANDBOXIR_USE_H
 
 #include "llvm/IR/Use.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm::sandboxir {
@@ -49,11 +50,11 @@ class Use {
 
 public:
   operator Value *() const { return get(); }
-  Value *get() const;
-  void set(Value *V);
+  LLVM_ABI Value *get() const;
+  LLVM_ABI void set(Value *V);
   class User *getUser() const { return Usr; }
-  unsigned getOperandNo() const;
-  void swap(Use &OtherUse);
+  LLVM_ABI unsigned getOperandNo() const;
+  LLVM_ABI void swap(Use &OtherUse);
   Context *getContext() const { return Ctx; }
   bool operator==(const Use &Other) const {
     assert(Ctx == Other.Ctx && "Contexts differ!");
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/User.h b/external/llvm-project/llvm/include/llvm/SandboxIR/User.h
index 80e672de3490..c552e2e3378b 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/User.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/User.h
@@ -13,6 +13,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/SandboxIR/Use.h"
 #include "llvm/SandboxIR/Value.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::sandboxir {
 
@@ -36,8 +37,8 @@ class OperandUseIterator {
   using iterator_category = std::input_iterator_tag;
 
   OperandUseIterator() = default;
-  value_type operator*() const;
-  OperandUseIterator &operator++();
+  LLVM_ABI value_type operator*() const;
+  LLVM_ABI OperandUseIterator &operator++();
   OperandUseIterator operator++(int) {
     auto Copy = *this;
     this->operator++();
@@ -49,13 +50,13 @@ class OperandUseIterator {
   bool operator!=(const OperandUseIterator &Other) const {
     return !(*this == Other);
   }
-  OperandUseIterator operator+(unsigned Num) const;
-  OperandUseIterator operator-(unsigned Num) const;
-  int operator-(const OperandUseIterator &Other) const;
+  LLVM_ABI OperandUseIterator operator+(unsigned Num) const;
+  LLVM_ABI OperandUseIterator operator-(unsigned Num) const;
+  LLVM_ABI int operator-(const OperandUseIterator &Other) const;
 };
 
 /// A sandboxir::User has operands.
-class User : public Value {
+class LLVM_ABI User : public Value {
 protected:
   User(ClassID ID, llvm::Value *V, Context &Ctx) : Value(ID, V, Ctx) {}
 
diff --git a/external/llvm-project/llvm/include/llvm/SandboxIR/Value.h b/external/llvm-project/llvm/include/llvm/SandboxIR/Value.h
index dbd0208b4f3f..dd0bc76db3e3 100644
--- a/external/llvm-project/llvm/include/llvm/SandboxIR/Value.h
+++ b/external/llvm-project/llvm/include/llvm/SandboxIR/Value.h
@@ -12,6 +12,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Value.h"
 #include "llvm/SandboxIR/Use.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm::sandboxir {
 
@@ -50,7 +51,7 @@ class UserUseIterator {
 
   UserUseIterator() = default;
   value_type operator*() const { return Use; }
-  UserUseIterator &operator++();
+  LLVM_ABI UserUseIterator &operator++();
   bool operator==(const UserUseIterator &Other) const {
     return Use == Other.Use;
   }
@@ -179,7 +180,7 @@ class Value {
   void clearValue() { Val = nullptr; }
   template <typename ItTy, typename SBTy> friend class LLVMOpUserItToSBTy;
 
-  Value(ClassID SubclassID, llvm::Value *Val, Context &Ctx);
+  LLVM_ABI Value(ClassID SubclassID, llvm::Value *Val, Context &Ctx);
   /// Disable copies.
   Value(const Value &) = delete;
   Value &operator=(const Value &) = delete;
@@ -191,7 +192,7 @@ class Value {
   using use_iterator = UserUseIterator;
   using const_use_iterator = UserUseIterator;
 
-  use_iterator use_begin();
+  LLVM_ABI use_iterator use_begin();
   const_use_iterator use_begin() const {
     return const_cast<Value *>(this)->use_begin();
   }
@@ -215,7 +216,7 @@ class Value {
   using user_iterator = mapped_iterator<sandboxir::UserUseIterator, UseToUser>;
   using const_user_iterator = user_iterator;
 
-  user_iterator user_begin();
+  LLVM_ABI user_iterator user_begin();
   user_iterator user_end() {
     return user_iterator(Use(nullptr, nullptr, Ctx), UseToUser());
   }
@@ -234,7 +235,7 @@ class Value {
   }
   /// \Returns the number of user edges (not necessarily to unique users).
   /// WARNING: This is a linear-time operation.
-  unsigned getNumUses() const;
+  LLVM_ABI unsigned getNumUses() const;
   /// Return true if this value has N uses or more.
   /// This is logically equivalent to getNumUses() >= N.
   /// WARNING: This can be expensive, as it is linear to the number of users.
@@ -256,13 +257,14 @@ class Value {
     return Cnt == Num;
   }
 
-  Type *getType() const;
+  LLVM_ABI Type *getType() const;
 
   Context &getContext() const { return Ctx; }
 
-  void replaceUsesWithIf(Value *OtherV,
-                         llvm::function_ref<bool(const Use &)> ShouldReplace);
-  void replaceAllUsesWith(Value *Other);
+  LLVM_ABI void
+  replaceUsesWithIf(Value *OtherV,
+                    llvm::function_ref<bool(const Use &)> ShouldReplace);
+  LLVM_ABI void replaceAllUsesWith(Value *Other);
 
   /// \Returns the LLVM IR name of the bottom-most LLVM value.
   StringRef getName() const { return Val->getName(); }
diff --git a/external/llvm-project/llvm/include/llvm/Support/AutoConvert.h b/external/llvm-project/llvm/include/llvm/Support/AutoConvert.h
index 352493e9be25..1e6792636e16 100644
--- a/external/llvm-project/llvm/include/llvm/Support/AutoConvert.h
+++ b/external/llvm-project/llvm/include/llvm/Support/AutoConvert.h
@@ -16,8 +16,9 @@
 
 #ifdef __MVS__
 #include <_Ccsid.h>
+#endif
 #ifdef __cplusplus
-#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/Error.h"
 #include <system_error>
 #endif /* __cplusplus */
 
@@ -28,9 +29,11 @@
 #ifdef __cplusplus
 extern "C" {
 #endif /* __cplusplus */
+
 int enablezOSAutoConversion(int FD);
 int disablezOSAutoConversion(int FD);
 int restorezOSStdHandleAutoConversion(int FD);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */
@@ -38,19 +41,7 @@ int restorezOSStdHandleAutoConversion(int FD);
 #ifdef __cplusplus
 namespace llvm {
 
-/** \brief Disable the z/OS enhanced ASCII auto-conversion for the file
- * descriptor.
- */
-std::error_code disablezOSAutoConversion(int FD);
-
-/** \brief Query the z/OS enhanced ASCII auto-conversion status of a file
- * descriptor and force the conversion if the file is not tagged with a
- * codepage.
- */
-std::error_code enablezOSAutoConversion(int FD);
-
-/** Restore the z/OS enhanced ASCII auto-conversion for the std handle. */
-std::error_code restorezOSStdHandleAutoConversion(int FD);
+#ifdef __MVS__
 
 /** \brief Set the tag information for a file descriptor. */
 std::error_code setzOSFileTag(int FD, int CCSID, bool Text);
@@ -63,9 +54,47 @@ ErrorOr<__ccsid_t> getzOSFileTag(const char *FileName, const int FD = -1);
  */
 ErrorOr<bool> needzOSConversion(const char *FileName, const int FD = -1);
 
+#endif /* __MVS__*/
+
+inline std::error_code disableAutoConversion(int FD) {
+#ifdef __MVS__
+  if (::disablezOSAutoConversion(FD) == -1)
+    return errnoAsErrorCode();
+#endif
+  return std::error_code();
+}
+
+inline std::error_code enableAutoConversion(int FD) {
+#ifdef __MVS__
+  if (::enablezOSAutoConversion(FD) == -1)
+    return errnoAsErrorCode();
+#endif
+  return std::error_code();
+}
+
+inline std::error_code restoreStdHandleAutoConversion(int FD) {
+#ifdef __MVS__
+  if (::restorezOSStdHandleAutoConversion(FD) == -1)
+    return errnoAsErrorCode();
+#endif
+  return std::error_code();
+}
+
+inline std::error_code setFileTag(int FD, int CCSID, bool Text) {
+#ifdef __MVS__
+  return setzOSFileTag(FD, CCSID, Text);
+#endif
+  return std::error_code();
+}
+
+inline ErrorOr<bool> needConversion(const char *FileName, const int FD = -1) {
+#ifdef __MVS__
+  return needzOSConversion(FileName, FD);
+#endif
+  return false;
+}
+
 } /* namespace llvm */
 #endif /* __cplusplus */
 
-#endif /* __MVS__ */
-
 #endif /* LLVM_SUPPORT_AUTOCONVERT_H */
diff --git a/external/llvm-project/llvm/include/llvm/Support/Compiler.h b/external/llvm-project/llvm/include/llvm/Support/Compiler.h
index b4382057654b..0de789ec68c4 100644
--- a/external/llvm-project/llvm/include/llvm/Support/Compiler.h
+++ b/external/llvm-project/llvm/include/llvm/Support/Compiler.h
@@ -209,7 +209,7 @@
 #define LLVM_ABI_FRIEND LLVM_ABI
 #define LLVM_ABI_EXPORT __declspec(dllexport)
 #elif defined(__ELF__) || defined(__MINGW32__) || defined(_AIX) ||             \
-    defined(__MVS__)
+    defined(__MVS__) || defined(__CYGWIN__)
 #define LLVM_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
 #define LLVM_ABI_FRIEND
 #define LLVM_TEMPLATE_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
@@ -238,6 +238,12 @@
 #define LLVM_PREFETCH(addr, rw, locality)
 #endif
 
+#if __has_attribute(uninitialized)
+#define LLVM_ATTRIBUTE_UNINITIALIZED __attribute__((uninitialized))
+#else
+#define LLVM_ATTRIBUTE_UNINITIALIZED
+#endif
+
 #if __has_attribute(used)
 #define LLVM_ATTRIBUTE_USED __attribute__((__used__))
 #else
diff --git a/external/llvm-project/llvm/include/llvm/Support/DebugCounter.h b/external/llvm-project/llvm/include/llvm/Support/DebugCounter.h
index 9611586a92c3..89349d1ebffe 100644
--- a/external/llvm-project/llvm/include/llvm/Support/DebugCounter.h
+++ b/external/llvm-project/llvm/include/llvm/Support/DebugCounter.h
@@ -60,7 +60,7 @@ class DebugCounter {
     int64_t Begin;
     int64_t End;
     LLVM_ABI void print(llvm::raw_ostream &OS);
-    bool contains(int64_t Idx) { return Idx >= Begin && Idx <= End; }
+    bool contains(int64_t Idx) const { return Idx >= Begin && Idx <= End; }
   };
 
   LLVM_ABI static void printChunks(raw_ostream &OS, ArrayRef<Chunk>);
diff --git a/external/llvm-project/llvm/include/llvm/Support/MathExtras.h b/external/llvm-project/llvm/include/llvm/Support/MathExtras.h
index 246080aa45ce..ae3150e5602e 100644
--- a/external/llvm-project/llvm/include/llvm/Support/MathExtras.h
+++ b/external/llvm-project/llvm/include/llvm/Support/MathExtras.h
@@ -328,8 +328,7 @@ inline bool isShiftedMask_64(uint64_t Value, unsigned &MaskIdx,
 /// Compile time Log2.
 /// Valid only for positive powers of two.
 template <size_t kValue> constexpr size_t CTLog2() {
-  static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue),
-                "Value is not a valid power of 2");
+  static_assert(llvm::isPowerOf2_64(kValue), "Value is not a valid power of 2");
   return 1 + CTLog2<kValue / 2>();
 }
 
diff --git a/external/llvm-project/llvm/include/llvm/TableGen/DirectiveEmitter.h b/external/llvm-project/llvm/include/llvm/TableGen/DirectiveEmitter.h
index 6defc8722d81..ce3e87e470b9 100644
--- a/external/llvm-project/llvm/include/llvm/TableGen/DirectiveEmitter.h
+++ b/external/llvm-project/llvm/include/llvm/TableGen/DirectiveEmitter.h
@@ -17,6 +17,8 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/Directive/Spelling.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/TableGen/Record.h"
 #include <algorithm>
 #include <string>
@@ -94,6 +96,42 @@ class DirectiveLanguage {
   }
 };
 
+class Versioned {
+public:
+  int getMinVersion(const Record *R) const {
+    int64_t Min = R->getValueAsInt("minVersion");
+    assert(llvm::isInt<IntWidth>(Min) && "Value out of range of 'int'");
+    return Min;
+  }
+
+  int getMaxVersion(const Record *R) const {
+    int64_t Max = R->getValueAsInt("maxVersion");
+    assert(llvm::isInt<IntWidth>(Max) && "Value out of range of 'int'");
+    return Max;
+  }
+
+private:
+  constexpr static int IntWidth = 8 * sizeof(int);
+};
+
+class Spelling : public Versioned {
+public:
+  using Value = directive::Spelling;
+
+  Spelling(const Record *Def) : Def(Def) {}
+
+  StringRef getText() const { return Def->getValueAsString("spelling"); }
+  llvm::directive::VersionRange getVersions() const {
+    return llvm::directive::VersionRange{getMinVersion(Def),
+                                         getMaxVersion(Def)};
+  }
+
+  Value get() const { return Value{getText(), getVersions()}; }
+
+private:
+  const Record *Def;
+};
+
 // Note: In all the classes below, allow implicit construction from Record *,
 // to allow writing code like:
 //  for (const Directive D : getDirectives()) {
@@ -109,18 +147,73 @@ class BaseRecord {
 public:
   BaseRecord(const Record *Def) : Def(Def) {}
 
-  StringRef getName() const { return Def->getValueAsString("name"); }
+  std::vector<Spelling::Value> getSpellings() const {
+    std::vector<Spelling::Value> List;
+    llvm::transform(Def->getValueAsListOfDefs("spellings"),
+                    std::back_inserter(List),
+                    [](const Record *R) { return Spelling(R).get(); });
+    return List;
+  }
+
+  StringRef getSpellingForIdentifier() const {
+    // From all spellings, pick the first one with the minimum version
+    // (i.e. pick the first from all the oldest ones). This guarantees
+    // that given several equivalent (in terms of versions) names, the
+    // first one is used, e.g. given
+    //   Clause<[Spelling<"foo">, Spelling<"bar">]> ...
+    // "foo" will be the selected spelling.
+    //
+    // This is a suitable spelling for generating an identifier name,
+    // since it will remain unchanged when any potential new spellings
+    // are added.
+    Spelling::Value Oldest{"not found", {/*Min=*/INT_MAX, 0}};
+    for (auto V : getSpellings())
+      if (V.Versions.Min < Oldest.Versions.Min)
+        Oldest = V;
+    return Oldest.Name;
+  }
 
   // Returns the name of the directive formatted for output. Whitespace are
   // replaced with underscores.
-  static std::string formatName(StringRef Name) {
+  static std::string getSnakeName(StringRef Name) {
     std::string N = Name.str();
     llvm::replace(N, ' ', '_');
     return N;
   }
 
+  // Take a string Name with sub-words separated with characters from Sep,
+  // and return a string with each of the sub-words capitalized, and the
+  // separators removed, e.g.
+  //   Name = "some_directive^name", Sep = "_^"  ->  "SomeDirectiveName".
+  static std::string getUpperCamelName(StringRef Name, StringRef Sep) {
+    std::string Camel = Name.str();
+    // Convert to uppercase
+    bool Cap = true;
+    llvm::transform(Camel, Camel.begin(), [&](unsigned char C) {
+      if (Sep.contains(C)) {
+        assert(!Cap && "No initial or repeated separators");
+        Cap = true;
+      } else if (Cap) {
+        C = llvm::toUpper(C);
+        Cap = false;
+      }
+      return C;
+    });
+    size_t Out = 0;
+    // Remove separators
+    for (size_t In = 0, End = Camel.size(); In != End; ++In) {
+      unsigned char C = Camel[In];
+      if (!Sep.contains(C))
+        Camel[Out++] = C;
+    }
+    Camel.resize(Out);
+    return Camel;
+  }
+
   std::string getFormattedName() const {
-    return formatName(Def->getValueAsString("name"));
+    if (auto maybeName = Def->getValueAsOptionalString("name"))
+      return getSnakeName(*maybeName);
+    return getSnakeName(getSpellingForIdentifier());
   }
 
   bool isDefault() const { return Def->getValueAsBit("isDefault"); }
@@ -172,26 +265,13 @@ class Directive : public BaseRecord {
 
   // Clang uses a different format for names of its directives enum.
   std::string getClangAccSpelling() const {
-    std::string Name = Def->getValueAsString("name").str();
+    StringRef Name = getSpellingForIdentifier();
 
     // Clang calls the 'unknown' value 'invalid'.
     if (Name == "unknown")
       return "Invalid";
 
-    // Clang entries all start with a capital letter, so apply that.
-    Name[0] = std::toupper(Name[0]);
-    // Additionally, spaces/underscores are handled by capitalizing the next
-    // letter of the name and removing the space/underscore.
-    for (unsigned I = 0; I < Name.size(); ++I) {
-      if (Name[I] == ' ' || Name[I] == '_') {
-        Name.erase(I, 1);
-        assert(Name[I] != ' ' && Name[I] != '_' &&
-               "No double spaces/underscores");
-        Name[I] = std::toupper(Name[I]);
-      }
-    }
-
-    return Name;
+    return BaseRecord::getUpperCamelName(Name, " _");
   }
 };
 
@@ -217,20 +297,8 @@ class Clause : public BaseRecord {
   // ex: async -> Async
   //     num_threads -> NumThreads
   std::string getFormattedParserClassName() const {
-    StringRef Name = Def->getValueAsString("name");
-    std::string N = Name.str();
-    bool Cap = true;
-    llvm::transform(N, N.begin(), [&Cap](unsigned char C) {
-      if (Cap == true) {
-        C = toUpper(C);
-        Cap = false;
-      } else if (C == '_') {
-        Cap = true;
-      }
-      return C;
-    });
-    erase(N, '_');
-    return N;
+    StringRef Name = getSpellingForIdentifier();
+    return BaseRecord::getUpperCamelName(Name, "_");
   }
 
   // Clang uses a different format for names of its clause enum, which can be
@@ -241,20 +309,8 @@ class Clause : public BaseRecord {
         !ClangSpelling.empty())
       return ClangSpelling.str();
 
-    std::string Name = Def->getValueAsString("name").str();
-    // Clang entries all start with a capital letter, so apply that.
-    Name[0] = std::toupper(Name[0]);
-    // Additionally, underscores are handled by capitalizing the next letter of
-    // the name and removing the underscore.
-    for (unsigned I = 0; I < Name.size(); ++I) {
-      if (Name[I] == '_') {
-        Name.erase(I, 1);
-        assert(Name[I] != '_' && "No double underscores");
-        Name[I] = std::toupper(Name[I]);
-      }
-    }
-
-    return Name;
+    StringRef Name = getSpellingForIdentifier();
+    return BaseRecord::getUpperCamelName(Name, "_");
   }
 
   // Optional field.
@@ -308,9 +364,9 @@ class VersionedClause {
   const Record *Def;
 };
 
-class ClauseVal : public BaseRecord {
+class EnumVal : public BaseRecord {
 public:
-  ClauseVal(const Record *Def) : BaseRecord(Def) {}
+  EnumVal(const Record *Def) : BaseRecord(Def) {}
 
   int getValue() const { return Def->getValueAsInt("value"); }
 
diff --git a/external/llvm-project/llvm/include/llvm/Target/Target.td b/external/llvm-project/llvm/include/llvm/Target/Target.td
index e8b460aaf803..ce9a2b275196 100644
--- a/external/llvm-project/llvm/include/llvm/Target/Target.td
+++ b/external/llvm-project/llvm/include/llvm/Target/Target.td
@@ -314,7 +314,7 @@ class RegisterClass<string namespace, list<ValueType> regTypes, int alignment,
   // to use in a given machine function. The code will be inserted in a
   // function like this:
   //
-  //   static inline unsigned f(const MachineFunction &MF) { ... }
+  //   static inline unsigned f(const MachineFunction &MF, bool Rev) { ... }
   //
   // The function should return 0 to select the default order defined by
   // MemberList, 1 to select the first AltOrders entry and so on.
diff --git a/external/llvm-project/llvm/include/llvm/Target/TargetOptions.h b/external/llvm-project/llvm/include/llvm/Target/TargetOptions.h
index fd8dad4f6f79..10638a0ec902 100644
--- a/external/llvm-project/llvm/include/llvm/Target/TargetOptions.h
+++ b/external/llvm-project/llvm/include/llvm/Target/TargetOptions.h
@@ -20,458 +20,450 @@
 #include <memory>
 
 namespace llvm {
-  struct fltSemantics;
-  class MachineFunction;
-  class MemoryBuffer;
-
-  namespace FloatABI {
-    enum ABIType {
-      Default, // Target-specific (either soft or hard depending on triple, etc).
-      Soft,    // Soft float.
-      Hard     // Hard float.
-    };
-  }
-
-  namespace FPOpFusion {
-    enum FPOpFusionMode {
-      Fast,     // Enable fusion of FP ops wherever it's profitable.
-      Standard, // Only allow fusion of 'blessed' ops (currently just fmuladd).
-      Strict    // Never fuse FP-ops.
-    };
-  }
-
-  namespace JumpTable {
-    enum JumpTableType {
-      Single,          // Use a single table for all indirect jumptable calls.
-      Arity,           // Use one table per number of function parameters.
-      Simplified,      // Use one table per function type, with types projected
-                       // into 4 types: pointer to non-function, struct,
-                       // primitive, and function pointer.
-      Full             // Use one table per unique function type
-    };
-  }
-
-  namespace ThreadModel {
-    enum Model {
-      POSIX,  // POSIX Threads
-      Single  // Single Threaded Environment
-    };
-  }
-
-  enum class BasicBlockSection {
-    All,    // Use Basic Block Sections for all basic blocks.  A section
-            // for every basic block can significantly bloat object file sizes.
-    List,   // Get list of functions & BBs from a file. Selectively enables
-            // basic block sections for a subset of basic blocks which can be
-            // used to control object size bloats from creating sections.
-    Preset, // Similar to list but the blocks are identified by passes which
-            // seek to use Basic Block Sections, e.g. MachineFunctionSplitter.
-            // This option cannot be set via the command line.
-    None    // Do not use Basic Block Sections.
-  };
-
-  enum class EABI {
-    Unknown,
-    Default, // Default means not specified
-    EABI4,   // Target-specific (either 4, 5 or gnu depending on triple).
-    EABI5,
-    GNU
-  };
-
-  /// Identify a debugger for "tuning" the debug info.
+struct fltSemantics;
+class MachineFunction;
+class MemoryBuffer;
+
+namespace FloatABI {
+enum ABIType {
+  Default, // Target-specific (either soft or hard depending on triple, etc).
+  Soft,    // Soft float.
+  Hard     // Hard float.
+};
+}
+
+namespace FPOpFusion {
+enum FPOpFusionMode {
+  Fast,     // Enable fusion of FP ops wherever it's profitable.
+  Standard, // Only allow fusion of 'blessed' ops (currently just fmuladd).
+  Strict    // Never fuse FP-ops.
+};
+}
+
+namespace JumpTable {
+enum JumpTableType {
+  Single,     // Use a single table for all indirect jumptable calls.
+  Arity,      // Use one table per number of function parameters.
+  Simplified, // Use one table per function type, with types projected
+              // into 4 types: pointer to non-function, struct,
+              // primitive, and function pointer.
+  Full        // Use one table per unique function type
+};
+}
+
+namespace ThreadModel {
+enum Model {
+  POSIX, // POSIX Threads
+  Single // Single Threaded Environment
+};
+}
+
+enum class BasicBlockSection {
+  All,    // Use Basic Block Sections for all basic blocks.  A section
+          // for every basic block can significantly bloat object file sizes.
+  List,   // Get list of functions & BBs from a file. Selectively enables
+          // basic block sections for a subset of basic blocks which can be
+          // used to control object size bloats from creating sections.
+  Preset, // Similar to list but the blocks are identified by passes which
+          // seek to use Basic Block Sections, e.g. MachineFunctionSplitter.
+          // This option cannot be set via the command line.
+  None    // Do not use Basic Block Sections.
+};
+
+enum class EABI {
+  Unknown,
+  Default, // Default means not specified
+  EABI4,   // Target-specific (either 4, 5 or gnu depending on triple).
+  EABI5,
+  GNU
+};
+
+/// Identify a debugger for "tuning" the debug info.
+///
+/// The "debugger tuning" concept allows us to present a more intuitive
+/// interface that unpacks into different sets of defaults for the various
+/// individual feature-flag settings, that suit the preferences of the
+/// various debuggers.  However, it's worth remembering that debuggers are
+/// not the only consumers of debug info, and some variations in DWARF might
+/// better be treated as target/platform issues. Fundamentally,
+/// o if the feature is useful (or not) to a particular debugger, regardless
+///   of the target, that's a tuning decision;
+/// o if the feature is useful (or not) on a particular platform, regardless
+///   of the debugger, that's a target decision.
+/// It's not impossible to see both factors in some specific case.
+enum class DebuggerKind {
+  Default, ///< No specific tuning requested.
+  GDB,     ///< Tune debug info for gdb.
+  LLDB,    ///< Tune debug info for lldb.
+  SCE,     ///< Tune debug info for SCE targets (e.g. PS4).
+  DBX      ///< Tune debug info for dbx.
+};
+
+/// Enable abort calls when global instruction selection fails to lower/select
+/// an instruction.
+enum class GlobalISelAbortMode {
+  Disable,        // Disable the abort.
+  Enable,         // Enable the abort.
+  DisableWithDiag // Disable the abort but emit a diagnostic on failure.
+};
+
+/// Indicates when and how the Swift async frame pointer bit should be set.
+enum class SwiftAsyncFramePointerMode {
+  /// Determine whether to set the bit statically or dynamically based
+  /// on the deployment target.
+  DeploymentBased,
+  /// Always set the bit.
+  Always,
+  /// Never set the bit.
+  Never,
+};
+
+/// \brief Enumeration value for AMDGPU code object version, which is the
+/// code object version times 100.
+enum CodeObjectVersionKind {
+  COV_None,
+  COV_2 = 200, // Unsupported.
+  COV_3 = 300, // Unsupported.
+  COV_4 = 400,
+  COV_5 = 500,
+  COV_6 = 600,
+};
+
+class TargetOptions {
+public:
+  TargetOptions()
+      : UnsafeFPMath(false), NoInfsFPMath(false), NoNaNsFPMath(false),
+        NoTrappingFPMath(true), NoSignedZerosFPMath(false),
+        ApproxFuncFPMath(false), EnableAIXExtendedAltivecABI(false),
+        HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false),
+        GuaranteedTailCallOpt(false), StackSymbolOrdering(true),
+        EnableFastISel(false), EnableGlobalISel(false), UseInitArray(false),
+        DisableIntegratedAS(false), FunctionSections(false),
+        DataSections(false), IgnoreXCOFFVisibility(false),
+        XCOFFTracebackTable(true), UniqueSectionNames(true),
+        UniqueBasicBlockSectionNames(false), SeparateNamedSections(false),
+        TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0),
+        EmulatedTLS(false), EnableTLSDESC(false), EnableIPRA(false),
+        EmitStackSizeSection(false), EnableMachineOutliner(false),
+        EnableMachineFunctionSplitter(false),
+        EnableStaticDataPartitioning(false), SupportsDefaultOutlining(false),
+        EmitAddrsig(false), BBAddrMap(false), EmitCallSiteInfo(false),
+        SupportsDebugEntryValues(false), EnableDebugEntryValues(false),
+        ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false),
+        XRayFunctionIndex(true), DebugStrictDwarf(false), Hotpatch(false),
+        PPCGenScalarMASSEntries(false), JMCInstrument(false),
+        EnableCFIFixup(false), MisExpect(false), XCOFFReadOnlyPointers(false),
+        VerifyArgABICompliance(true),
+        FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {}
+
+  /// DisableFramePointerElim - This returns true if frame pointer elimination
+  /// optimization should be disabled for the given machine function.
+  bool DisableFramePointerElim(const MachineFunction &MF) const;
+
+  /// FramePointerIsReserved - This returns true if the frame pointer must
+  /// always either point to a new frame record or be un-modified in the given
+  /// function.
+  bool FramePointerIsReserved(const MachineFunction &MF) const;
+
+  /// If greater than 0, override the default value of
+  /// MCAsmInfo::BinutilsVersion.
+  std::pair<int, int> BinutilsVersion{0, 0};
+
+  /// UnsafeFPMath - This flag is enabled when the
+  /// -enable-unsafe-fp-math flag is specified on the command line.  When
+  /// this flag is off (the default), the code generator is not allowed to
+  /// produce results that are "less precise" than IEEE allows.  This includes
+  /// use of X86 instructions like FSIN and FCOS instead of libcalls.
+  unsigned UnsafeFPMath : 1;
+
+  /// NoInfsFPMath - This flag is enabled when the
+  /// -enable-no-infs-fp-math flag is specified on the command line. When
+  /// this flag is off (the default), the code generator is not allowed to
+  /// assume the FP arithmetic arguments and results are never +-Infs.
+  unsigned NoInfsFPMath : 1;
+
+  /// NoNaNsFPMath - This flag is enabled when the
+  /// -enable-no-nans-fp-math flag is specified on the command line. When
+  /// this flag is off (the default), the code generator is not allowed to
+  /// assume the FP arithmetic arguments and results are never NaNs.
+  unsigned NoNaNsFPMath : 1;
+
+  /// NoTrappingFPMath - This flag is enabled when the
+  /// -enable-no-trapping-fp-math is specified on the command line. This
+  /// specifies that there are no trap handlers to handle exceptions.
+  unsigned NoTrappingFPMath : 1;
+
+  /// NoSignedZerosFPMath - This flag is enabled when the
+  /// -enable-no-signed-zeros-fp-math is specified on the command line. This
+  /// specifies that optimizations are allowed to treat the sign of a zero
+  /// argument or result as insignificant.
+  unsigned NoSignedZerosFPMath : 1;
+
+  /// ApproxFuncFPMath - This flag is enabled when the
+  /// -enable-approx-func-fp-math is specified on the command line. This
+  /// specifies that optimizations are allowed to substitute math functions
+  /// with approximate calculations
+  unsigned ApproxFuncFPMath : 1;
+
+  /// EnableAIXExtendedAltivecABI - This flag returns true when -vec-extabi is
+  /// specified. The code generator is then able to use both volatile and
+  /// nonvolitle vector registers. When false, the code generator only uses
+  /// volatile vector registers which is the default setting on AIX.
+  unsigned EnableAIXExtendedAltivecABI : 1;
+
+  /// HonorSignDependentRoundingFPMath - This returns true when the
+  /// -enable-sign-dependent-rounding-fp-math is specified.  If this returns
+  /// false (the default), the code generator is allowed to assume that the
+  /// rounding behavior is the default (round-to-zero for all floating point
+  /// to integer conversions, and round-to-nearest for all other arithmetic
+  /// truncations).  If this is enabled (set to true), the code generator must
+  /// assume that the rounding mode may dynamically change.
+  unsigned HonorSignDependentRoundingFPMathOption : 1;
+  bool HonorSignDependentRoundingFPMath() const;
+
+  /// NoZerosInBSS - By default some codegens place zero-initialized data to
+  /// .bss section. This flag disables such behaviour (necessary, e.g. for
+  /// crt*.o compiling).
+  unsigned NoZerosInBSS : 1;
+
+  /// GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is
+  /// specified on the commandline. When the flag is on, participating targets
+  /// will perform tail call optimization on all calls which use the fastcc
+  /// calling convention and which satisfy certain target-independent
+  /// criteria (being at the end of a function, having the same return type
+  /// as their parent function, etc.), using an alternate ABI if necessary.
+  unsigned GuaranteedTailCallOpt : 1;
+
+  /// StackSymbolOrdering - When true, this will allow CodeGen to order
+  /// the local stack symbols (for code size, code locality, or any other
+  /// heuristics). When false, the local symbols are left in whatever order
+  /// they were generated. Default is true.
+  unsigned StackSymbolOrdering : 1;
+
+  /// EnableFastISel - This flag enables fast-path instruction selection
+  /// which trades away generated code quality in favor of reducing
+  /// compile time.
+  unsigned EnableFastISel : 1;
+
+  /// EnableGlobalISel - This flag enables global instruction selection.
+  unsigned EnableGlobalISel : 1;
+
+  /// EnableGlobalISelAbort - Control abort behaviour when global instruction
+  /// selection fails to lower/select an instruction.
+  GlobalISelAbortMode GlobalISelAbort = GlobalISelAbortMode::Enable;
+
+  /// Control when and how the Swift async frame pointer bit should
+  /// be set.
+  SwiftAsyncFramePointerMode SwiftAsyncFramePointer =
+      SwiftAsyncFramePointerMode::Always;
+
+  /// UseInitArray - Use .init_array instead of .ctors for static
+  /// constructors.
+  unsigned UseInitArray : 1;
+
+  /// Disable the integrated assembler.
+  unsigned DisableIntegratedAS : 1;
+
+  /// Emit functions into separate sections.
+  unsigned FunctionSections : 1;
+
+  /// Emit data into separate sections.
+  unsigned DataSections : 1;
+
+  /// Do not emit visibility attribute for xcoff.
+  unsigned IgnoreXCOFFVisibility : 1;
+
+  /// Emit XCOFF traceback table.
+  unsigned XCOFFTracebackTable : 1;
+
+  unsigned UniqueSectionNames : 1;
+
+  /// Use unique names for basic block sections.
+  unsigned UniqueBasicBlockSectionNames : 1;
+
+  /// Emit named sections with the same name into different sections.
+  unsigned SeparateNamedSections : 1;
+
+  /// Emit target-specific trap instruction for 'unreachable' IR instructions.
+  unsigned TrapUnreachable : 1;
+
+  /// Do not emit a trap instruction for 'unreachable' IR instructions behind
+  /// noreturn calls, even if TrapUnreachable is true.
+  unsigned NoTrapAfterNoreturn : 1;
+
+  /// Bit size of immediate TLS offsets (0 == use the default).
+  unsigned TLSSize : 8;
+
+  /// EmulatedTLS - This flag enables emulated TLS model, using emutls
+  /// function in the runtime library..
+  unsigned EmulatedTLS : 1;
+
+  /// EnableTLSDESC - This flag enables TLS Descriptors.
+  unsigned EnableTLSDESC : 1;
+
+  /// This flag enables InterProcedural Register Allocation (IPRA).
+  unsigned EnableIPRA : 1;
+
+  /// Emit section containing metadata on function stack sizes.
+  unsigned EmitStackSizeSection : 1;
+
+  /// Enables the MachineOutliner pass.
+  unsigned EnableMachineOutliner : 1;
+
+  /// Enables the MachineFunctionSplitter pass.
+  unsigned EnableMachineFunctionSplitter : 1;
+
+  /// Enables the StaticDataSplitter pass.
+  unsigned EnableStaticDataPartitioning : 1;
+
+  /// Set if the target supports default outlining behaviour.
+  unsigned SupportsDefaultOutlining : 1;
+
+  /// Emit address-significance table.
+  unsigned EmitAddrsig : 1;
+
+  // Emit the SHT_LLVM_BB_ADDR_MAP section containing basic block address
+  // which can be used to map virtual addresses to machine basic blocks.
+  unsigned BBAddrMap : 1;
+
+  /// Emit basic blocks into separate sections.
+  BasicBlockSection BBSections = BasicBlockSection::None;
+
+  /// Memory Buffer that contains information on sampled basic blocks and used
+  /// to selectively generate basic block sections.
+  std::shared_ptr<MemoryBuffer> BBSectionsFuncListBuf;
+
+  /// The flag enables call site info production. It is used only for debug
+  /// info, and it is restricted only to optimized code. This can be used for
+  /// something else, so that should be controlled in the frontend.
+  unsigned EmitCallSiteInfo : 1;
+  /// Set if the target supports the debug entry values by default.
+  unsigned SupportsDebugEntryValues : 1;
+  /// When set to true, the EnableDebugEntryValues option forces production
+  /// of debug entry values even if the target does not officially support
+  /// it. Useful for testing purposes only. This flag should never be checked
+  /// directly, always use \ref ShouldEmitDebugEntryValues instead.
+  unsigned EnableDebugEntryValues : 1;
+  /// NOTE: There are targets that still do not support the debug entry values
+  /// production.
+  bool ShouldEmitDebugEntryValues() const;
+
+  // When set to true, use experimental new debug variable location tracking,
+  // which seeks to follow the values of variables rather than their location,
+  // post isel.
+  unsigned ValueTrackingVariableLocations : 1;
+
+  /// Emit DWARF debug frame section.
+  unsigned ForceDwarfFrameSection : 1;
+
+  /// Emit XRay Function Index section
+  unsigned XRayFunctionIndex : 1;
+
+  /// When set to true, don't use DWARF extensions in later DWARF versions.
+  /// By default, it is set to false.
+  unsigned DebugStrictDwarf : 1;
+
+  /// Emit the hotpatch flag in CodeView debug.
+  unsigned Hotpatch : 1;
+
+  /// Enables scalar MASS conversions
+  unsigned PPCGenScalarMASSEntries : 1;
+
+  /// Enable JustMyCode instrumentation.
+  unsigned JMCInstrument : 1;
+
+  /// Enable the CFIFixup pass.
+  unsigned EnableCFIFixup : 1;
+
+  /// When set to true, enable MisExpect Diagnostics
+  /// By default, it is set to false
+  unsigned MisExpect : 1;
+
+  /// When set to true, const objects with relocatable address values are put
+  /// into the RO data section.
+  unsigned XCOFFReadOnlyPointers : 1;
+
+  /// When set to true, call/return argument extensions of narrow integers
+  /// are verified in the target backend if it cares about them. This is
+  /// not done with internal tools like llc that run many tests that ignore
+  /// (lack) these extensions.
+  unsigned VerifyArgABICompliance : 1;
+
+  /// Name of the stack usage file (i.e., .su file) if user passes
+  /// -fstack-usage. If empty, it can be implied that -fstack-usage is not
+  /// passed on the command line.
+  std::string StackUsageOutput;
+
+  /// If greater than 0, override TargetLoweringBase::PrefLoopAlignment.
+  unsigned LoopAlignment = 0;
+
+  /// FloatABIType - This setting is set by -float-abi=xxx option is specfied
+  /// on the command line. This setting may either be Default, Soft, or Hard.
+  /// Default selects the target's default behavior. Soft selects the ABI for
+  /// software floating point, but does not indicate that FP hardware may not
+  /// be used. Such a combination is unfortunately popular (e.g.
+  /// arm-apple-darwin). Hard presumes that the normal FP ABI is used.
+  FloatABI::ABIType FloatABIType = FloatABI::Default;
+
+  /// AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
+  /// This controls the creation of fused FP ops that store intermediate
+  /// results in higher precision than IEEE allows (E.g. FMAs).
   ///
-  /// The "debugger tuning" concept allows us to present a more intuitive
-  /// interface that unpacks into different sets of defaults for the various
-  /// individual feature-flag settings, that suit the preferences of the
-  /// various debuggers.  However, it's worth remembering that debuggers are
-  /// not the only consumers of debug info, and some variations in DWARF might
-  /// better be treated as target/platform issues. Fundamentally,
-  /// o if the feature is useful (or not) to a particular debugger, regardless
-  ///   of the target, that's a tuning decision;
-  /// o if the feature is useful (or not) on a particular platform, regardless
-  ///   of the debugger, that's a target decision.
-  /// It's not impossible to see both factors in some specific case.
-  enum class DebuggerKind {
-    Default, ///< No specific tuning requested.
-    GDB,     ///< Tune debug info for gdb.
-    LLDB,    ///< Tune debug info for lldb.
-    SCE,     ///< Tune debug info for SCE targets (e.g. PS4).
-    DBX      ///< Tune debug info for dbx.
-  };
-
-  /// Enable abort calls when global instruction selection fails to lower/select
-  /// an instruction.
-  enum class GlobalISelAbortMode {
-    Disable,        // Disable the abort.
-    Enable,         // Enable the abort.
-    DisableWithDiag // Disable the abort but emit a diagnostic on failure.
-  };
-
-  /// Indicates when and how the Swift async frame pointer bit should be set.
-  enum class SwiftAsyncFramePointerMode {
-    /// Determine whether to set the bit statically or dynamically based
-    /// on the deployment target.
-    DeploymentBased,
-    /// Always set the bit.
-    Always,
-    /// Never set the bit.
-    Never,
-  };
-
-  /// \brief Enumeration value for AMDGPU code object version, which is the
-  /// code object version times 100.
-  enum CodeObjectVersionKind {
-    COV_None,
-    COV_2 = 200, // Unsupported.
-    COV_3 = 300, // Unsupported.
-    COV_4 = 400,
-    COV_5 = 500,
-    COV_6 = 600,
-  };
-
-  class TargetOptions {
-  public:
-    TargetOptions()
-        : UnsafeFPMath(false), NoInfsFPMath(false), NoNaNsFPMath(false),
-          NoTrappingFPMath(true), NoSignedZerosFPMath(false),
-          ApproxFuncFPMath(false), EnableAIXExtendedAltivecABI(false),
-          HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false),
-          GuaranteedTailCallOpt(false), StackSymbolOrdering(true),
-          EnableFastISel(false), EnableGlobalISel(false), UseInitArray(false),
-          DisableIntegratedAS(false), FunctionSections(false),
-          DataSections(false), IgnoreXCOFFVisibility(false),
-          XCOFFTracebackTable(true), UniqueSectionNames(true),
-          UniqueBasicBlockSectionNames(false), SeparateNamedSections(false),
-          TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0),
-          EmulatedTLS(false), EnableTLSDESC(false), EnableIPRA(false),
-          EmitStackSizeSection(false), EnableMachineOutliner(false),
-          EnableMachineFunctionSplitter(false),
-          EnableStaticDataPartitioning(false), SupportsDefaultOutlining(false),
-          EmitAddrsig(false), BBAddrMap(false), EmitCallSiteInfo(false),
-          SupportsDebugEntryValues(false), EnableDebugEntryValues(false),
-          ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false),
-          XRayFunctionIndex(true), DebugStrictDwarf(false), Hotpatch(false),
-          PPCGenScalarMASSEntries(false), JMCInstrument(false),
-          EnableCFIFixup(false), MisExpect(false), XCOFFReadOnlyPointers(false),
-          VerifyArgABICompliance(true),
-          FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {}
-
-    /// DisableFramePointerElim - This returns true if frame pointer elimination
-    /// optimization should be disabled for the given machine function.
-    bool DisableFramePointerElim(const MachineFunction &MF) const;
-
-    /// FramePointerIsReserved - This returns true if the frame pointer must
-    /// always either point to a new frame record or be un-modified in the given
-    /// function.
-    bool FramePointerIsReserved(const MachineFunction &MF) const;
-
-    /// If greater than 0, override the default value of
-    /// MCAsmInfo::BinutilsVersion.
-    std::pair<int, int> BinutilsVersion{0, 0};
-
-    /// UnsafeFPMath - This flag is enabled when the
-    /// -enable-unsafe-fp-math flag is specified on the command line.  When
-    /// this flag is off (the default), the code generator is not allowed to
-    /// produce results that are "less precise" than IEEE allows.  This includes
-    /// use of X86 instructions like FSIN and FCOS instead of libcalls.
-    unsigned UnsafeFPMath : 1;
-
-    /// NoInfsFPMath - This flag is enabled when the
-    /// -enable-no-infs-fp-math flag is specified on the command line. When
-    /// this flag is off (the default), the code generator is not allowed to
-    /// assume the FP arithmetic arguments and results are never +-Infs.
-    unsigned NoInfsFPMath : 1;
-
-    /// NoNaNsFPMath - This flag is enabled when the
-    /// -enable-no-nans-fp-math flag is specified on the command line. When
-    /// this flag is off (the default), the code generator is not allowed to
-    /// assume the FP arithmetic arguments and results are never NaNs.
-    unsigned NoNaNsFPMath : 1;
-
-    /// NoTrappingFPMath - This flag is enabled when the
-    /// -enable-no-trapping-fp-math is specified on the command line. This
-    /// specifies that there are no trap handlers to handle exceptions.
-    unsigned NoTrappingFPMath : 1;
-
-    /// NoSignedZerosFPMath - This flag is enabled when the
-    /// -enable-no-signed-zeros-fp-math is specified on the command line. This
-    /// specifies that optimizations are allowed to treat the sign of a zero
-    /// argument or result as insignificant.
-    unsigned NoSignedZerosFPMath : 1;
-
-    /// ApproxFuncFPMath - This flag is enabled when the
-    /// -enable-approx-func-fp-math is specified on the command line. This
-    /// specifies that optimizations are allowed to substitute math functions
-    /// with approximate calculations
-    unsigned ApproxFuncFPMath : 1;
-
-    /// EnableAIXExtendedAltivecABI - This flag returns true when -vec-extabi is
-    /// specified. The code generator is then able to use both volatile and
-    /// nonvolitle vector registers. When false, the code generator only uses
-    /// volatile vector registers which is the default setting on AIX.
-    unsigned EnableAIXExtendedAltivecABI : 1;
-
-    /// HonorSignDependentRoundingFPMath - This returns true when the
-    /// -enable-sign-dependent-rounding-fp-math is specified.  If this returns
-    /// false (the default), the code generator is allowed to assume that the
-    /// rounding behavior is the default (round-to-zero for all floating point
-    /// to integer conversions, and round-to-nearest for all other arithmetic
-    /// truncations).  If this is enabled (set to true), the code generator must
-    /// assume that the rounding mode may dynamically change.
-    unsigned HonorSignDependentRoundingFPMathOption : 1;
-    bool HonorSignDependentRoundingFPMath() const;
-
-    /// NoZerosInBSS - By default some codegens place zero-initialized data to
-    /// .bss section. This flag disables such behaviour (necessary, e.g. for
-    /// crt*.o compiling).
-    unsigned NoZerosInBSS : 1;
-
-    /// GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is
-    /// specified on the commandline. When the flag is on, participating targets
-    /// will perform tail call optimization on all calls which use the fastcc
-    /// calling convention and which satisfy certain target-independent
-    /// criteria (being at the end of a function, having the same return type
-    /// as their parent function, etc.), using an alternate ABI if necessary.
-    unsigned GuaranteedTailCallOpt : 1;
-
-    /// StackSymbolOrdering - When true, this will allow CodeGen to order
-    /// the local stack symbols (for code size, code locality, or any other
-    /// heuristics). When false, the local symbols are left in whatever order
-    /// they were generated. Default is true.
-    unsigned StackSymbolOrdering : 1;
-
-    /// EnableFastISel - This flag enables fast-path instruction selection
-    /// which trades away generated code quality in favor of reducing
-    /// compile time.
-    unsigned EnableFastISel : 1;
-
-    /// EnableGlobalISel - This flag enables global instruction selection.
-    unsigned EnableGlobalISel : 1;
-
-    /// EnableGlobalISelAbort - Control abort behaviour when global instruction
-    /// selection fails to lower/select an instruction.
-    GlobalISelAbortMode GlobalISelAbort = GlobalISelAbortMode::Enable;
-
-    /// Control when and how the Swift async frame pointer bit should
-    /// be set.
-    SwiftAsyncFramePointerMode SwiftAsyncFramePointer =
-        SwiftAsyncFramePointerMode::Always;
-
-    /// UseInitArray - Use .init_array instead of .ctors for static
-    /// constructors.
-    unsigned UseInitArray : 1;
-
-    /// Disable the integrated assembler.
-    unsigned DisableIntegratedAS : 1;
-
-    /// Emit functions into separate sections.
-    unsigned FunctionSections : 1;
-
-    /// Emit data into separate sections.
-    unsigned DataSections : 1;
-
-    /// Do not emit visibility attribute for xcoff.
-    unsigned IgnoreXCOFFVisibility : 1;
-
-    /// Emit XCOFF traceback table.
-    unsigned XCOFFTracebackTable : 1;
-
-    unsigned UniqueSectionNames : 1;
-
-    /// Use unique names for basic block sections.
-    unsigned UniqueBasicBlockSectionNames : 1;
-
-    /// Emit named sections with the same name into different sections.
-    unsigned SeparateNamedSections : 1;
-
-    /// Emit target-specific trap instruction for 'unreachable' IR instructions.
-    unsigned TrapUnreachable : 1;
-
-    /// Do not emit a trap instruction for 'unreachable' IR instructions behind
-    /// noreturn calls, even if TrapUnreachable is true.
-    unsigned NoTrapAfterNoreturn : 1;
-
-    /// Bit size of immediate TLS offsets (0 == use the default).
-    unsigned TLSSize : 8;
-
-    /// EmulatedTLS - This flag enables emulated TLS model, using emutls
-    /// function in the runtime library..
-    unsigned EmulatedTLS : 1;
-
-    /// EnableTLSDESC - This flag enables TLS Descriptors.
-    unsigned EnableTLSDESC : 1;
-
-    /// This flag enables InterProcedural Register Allocation (IPRA).
-    unsigned EnableIPRA : 1;
-
-    /// Emit section containing metadata on function stack sizes.
-    unsigned EmitStackSizeSection : 1;
-
-    /// Enables the MachineOutliner pass.
-    unsigned EnableMachineOutliner : 1;
-
-    /// Enables the MachineFunctionSplitter pass.
-    unsigned EnableMachineFunctionSplitter : 1;
-
-    /// Enables the StaticDataSplitter pass.
-    unsigned EnableStaticDataPartitioning : 1;
-
-    /// Set if the target supports default outlining behaviour.
-    unsigned SupportsDefaultOutlining : 1;
-
-    /// Emit address-significance table.
-    unsigned EmitAddrsig : 1;
-
-    // Emit the SHT_LLVM_BB_ADDR_MAP section containing basic block address
-    // which can be used to map virtual addresses to machine basic blocks.
-    unsigned BBAddrMap : 1;
-
-    /// Emit basic blocks into separate sections.
-    BasicBlockSection BBSections = BasicBlockSection::None;
-
-    /// Memory Buffer that contains information on sampled basic blocks and used
-    /// to selectively generate basic block sections.
-    std::shared_ptr<MemoryBuffer> BBSectionsFuncListBuf;
-
-    /// The flag enables call site info production. It is used only for debug
-    /// info, and it is restricted only to optimized code. This can be used for
-    /// something else, so that should be controlled in the frontend.
-    unsigned EmitCallSiteInfo : 1;
-    /// Set if the target supports the debug entry values by default.
-    unsigned SupportsDebugEntryValues : 1;
-    /// When set to true, the EnableDebugEntryValues option forces production
-    /// of debug entry values even if the target does not officially support
-    /// it. Useful for testing purposes only. This flag should never be checked
-    /// directly, always use \ref ShouldEmitDebugEntryValues instead.
-     unsigned EnableDebugEntryValues : 1;
-    /// NOTE: There are targets that still do not support the debug entry values
-    /// production.
-    bool ShouldEmitDebugEntryValues() const;
-
-    // When set to true, use experimental new debug variable location tracking,
-    // which seeks to follow the values of variables rather than their location,
-    // post isel.
-    unsigned ValueTrackingVariableLocations : 1;
-
-    /// Emit DWARF debug frame section.
-    unsigned ForceDwarfFrameSection : 1;
-
-    /// Emit XRay Function Index section
-    unsigned XRayFunctionIndex : 1;
-
-    /// When set to true, don't use DWARF extensions in later DWARF versions.
-    /// By default, it is set to false.
-    unsigned DebugStrictDwarf : 1;
+  /// Fast mode - allows formation of fused FP ops whenever they're
+  /// profitable.
+  /// Standard mode - allow fusion only for 'blessed' FP ops. At present the
+  /// only blessed op is the fmuladd intrinsic. In the future more blessed ops
+  /// may be added.
+  /// Strict mode - allow fusion only if/when it can be proven that the excess
+  /// precision won't effect the result.
+  ///
+  /// Note: This option only controls formation of fused ops by the
+  /// optimizers.  Fused operations that are explicitly specified (e.g. FMA
+  /// via the llvm.fma.* intrinsic) will always be honored, regardless of
+  /// the value of this option.
+  FPOpFusion::FPOpFusionMode AllowFPOpFusion = FPOpFusion::Standard;
+
+  /// ThreadModel - This flag specifies the type of threading model to assume
+  /// for things like atomics
+  ThreadModel::Model ThreadModel = ThreadModel::POSIX;
+
+  /// EABIVersion - This flag specifies the EABI version
+  EABI EABIVersion = EABI::Default;
+
+  /// Which debugger to tune for.
+  DebuggerKind DebuggerTuning = DebuggerKind::Default;
+
+private:
+  /// Flushing mode to assume in default FP environment.
+  DenormalMode FPDenormalMode;
+
+  /// Flushing mode to assume in default FP environment, for float/vector of
+  /// float.
+  DenormalMode FP32DenormalMode;
+
+public:
+  void setFPDenormalMode(DenormalMode Mode) { FPDenormalMode = Mode; }
+
+  void setFP32DenormalMode(DenormalMode Mode) { FP32DenormalMode = Mode; }
+
+  DenormalMode getRawFPDenormalMode() const { return FPDenormalMode; }
+
+  DenormalMode getRawFP32DenormalMode() const { return FP32DenormalMode; }
+
+  DenormalMode getDenormalMode(const fltSemantics &FPType) const;
+
+  /// What exception model to use
+  ExceptionHandling ExceptionModel = ExceptionHandling::None;
+
+  /// Machine level options.
+  MCTargetOptions MCOptions;
 
-    /// Emit the hotpatch flag in CodeView debug.
-    unsigned Hotpatch : 1;
+  /// Stores the filename/path of the final .o/.obj file, to be written in the
+  /// debug information. This is used for emitting the CodeView S_OBJNAME
+  /// record.
+  std::string ObjectFilenameForDebug;
+};
 
-    /// Enables scalar MASS conversions
-    unsigned PPCGenScalarMASSEntries : 1;
-
-    /// Enable JustMyCode instrumentation.
-    unsigned JMCInstrument : 1;
-
-    /// Enable the CFIFixup pass.
-    unsigned EnableCFIFixup : 1;
-
-    /// When set to true, enable MisExpect Diagnostics
-    /// By default, it is set to false
-    unsigned MisExpect : 1;
-
-    /// When set to true, const objects with relocatable address values are put
-    /// into the RO data section.
-    unsigned XCOFFReadOnlyPointers : 1;
-
-    /// When set to true, call/return argument extensions of narrow integers
-    /// are verified in the target backend if it cares about them. This is
-    /// not done with internal tools like llc that run many tests that ignore
-    /// (lack) these extensions.
-    unsigned VerifyArgABICompliance : 1;
-
-    /// Name of the stack usage file (i.e., .su file) if user passes
-    /// -fstack-usage. If empty, it can be implied that -fstack-usage is not
-    /// passed on the command line.
-    std::string StackUsageOutput;
-
-    /// If greater than 0, override TargetLoweringBase::PrefLoopAlignment.
-    unsigned LoopAlignment = 0;
-
-    /// FloatABIType - This setting is set by -float-abi=xxx option is specfied
-    /// on the command line. This setting may either be Default, Soft, or Hard.
-    /// Default selects the target's default behavior. Soft selects the ABI for
-    /// software floating point, but does not indicate that FP hardware may not
-    /// be used. Such a combination is unfortunately popular (e.g.
-    /// arm-apple-darwin). Hard presumes that the normal FP ABI is used.
-    FloatABI::ABIType FloatABIType = FloatABI::Default;
-
-    /// AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
-    /// This controls the creation of fused FP ops that store intermediate
-    /// results in higher precision than IEEE allows (E.g. FMAs).
-    ///
-    /// Fast mode - allows formation of fused FP ops whenever they're
-    /// profitable.
-    /// Standard mode - allow fusion only for 'blessed' FP ops. At present the
-    /// only blessed op is the fmuladd intrinsic. In the future more blessed ops
-    /// may be added.
-    /// Strict mode - allow fusion only if/when it can be proven that the excess
-    /// precision won't effect the result.
-    ///
-    /// Note: This option only controls formation of fused ops by the
-    /// optimizers.  Fused operations that are explicitly specified (e.g. FMA
-    /// via the llvm.fma.* intrinsic) will always be honored, regardless of
-    /// the value of this option.
-    FPOpFusion::FPOpFusionMode AllowFPOpFusion = FPOpFusion::Standard;
-
-    /// ThreadModel - This flag specifies the type of threading model to assume
-    /// for things like atomics
-    ThreadModel::Model ThreadModel = ThreadModel::POSIX;
-
-    /// EABIVersion - This flag specifies the EABI version
-    EABI EABIVersion = EABI::Default;
-
-    /// Which debugger to tune for.
-    DebuggerKind DebuggerTuning = DebuggerKind::Default;
-
-  private:
-    /// Flushing mode to assume in default FP environment.
-    DenormalMode FPDenormalMode;
-
-    /// Flushing mode to assume in default FP environment, for float/vector of
-    /// float.
-    DenormalMode FP32DenormalMode;
-
-  public:
-    void setFPDenormalMode(DenormalMode Mode) {
-      FPDenormalMode = Mode;
-    }
-
-    void setFP32DenormalMode(DenormalMode Mode) {
-      FP32DenormalMode = Mode;
-    }
-
-    DenormalMode getRawFPDenormalMode() const {
-      return FPDenormalMode;
-    }
-
-    DenormalMode getRawFP32DenormalMode() const {
-      return FP32DenormalMode;
-    }
-
-    DenormalMode getDenormalMode(const fltSemantics &FPType) const;
-
-    /// What exception model to use
-    ExceptionHandling ExceptionModel = ExceptionHandling::None;
-
-    /// Machine level options.
-    MCTargetOptions MCOptions;
-
-    /// Stores the filename/path of the final .o/.obj file, to be written in the
-    /// debug information. This is used for emitting the CodeView S_OBJNAME
-    /// record.
-    std::string ObjectFilenameForDebug;
-  };
-
-} // End llvm namespace
+} // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/Target/TargetSelectionDAG.td b/external/llvm-project/llvm/include/llvm/Target/TargetSelectionDAG.td
index d56216e8b637..9ac228110eb9 100644
--- a/external/llvm-project/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/external/llvm-project/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -521,6 +521,8 @@ def partial_reduce_umla : SDNode<"ISD::PARTIAL_REDUCE_UMLA",
                                  SDTPartialReduceMLA>;
 def partial_reduce_smla : SDNode<"ISD::PARTIAL_REDUCE_SMLA",
                                  SDTPartialReduceMLA>;
+def partial_reduce_sumla : SDNode<"ISD::PARTIAL_REDUCE_SUMLA",
+                                 SDTPartialReduceMLA>;
 
 def fadd       : SDNode<"ISD::FADD"       , SDTFPBinOp, [SDNPCommutative]>;
 def fsub       : SDNode<"ISD::FSUB"       , SDTFPBinOp>;
@@ -1061,7 +1063,9 @@ class OutPatFrag<dag ops, dag frag>
 // PatLeaf's are pattern fragments that have no operands.  This is just a helper
 // to define immediates and other common things concisely.
 class PatLeaf<dag frag, code pred = [{}], SDNodeXForm xform = NOOP_SDNodeXForm>
- : PatFrag<(ops), frag, pred, xform>;
+ : PatFrag<(ops), frag, pred, xform> {
+  code GISelLeafPredicateCode = ?;
+}
 
 
 // ImmLeaf is a pattern fragment with a constraint on the immediate.  The
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 0338770593bc..59e8117ccb73 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
@@ -79,7 +80,7 @@ struct FMVInfo {
       : Name(Name), FeatureBit(FeatureBit), PriorityBit(PriorityBit), ID(ID) {};
 };
 
-const std::vector<FMVInfo> &getFMVInfo();
+LLVM_ABI const std::vector<FMVInfo> &getFMVInfo();
 
 // Represents a dependency between two architecture extensions. Later is the
 // feature which was added to the architecture after Earlier, and expands the
@@ -146,7 +147,7 @@ struct ArchInfo {
   StringRef getSubArch() const { return ArchFeature.substr(1); }
 
   // Search for ArchInfo by SubArch name
-  static std::optional<ArchInfo> findBySubArch(StringRef SubArch);
+  LLVM_ABI static std::optional<ArchInfo> findBySubArch(StringRef SubArch);
 };
 
 #define EMIT_ARCHITECTURES
@@ -182,34 +183,36 @@ struct ExtensionSet {
   // Enable the given architecture extension, and any other extensions it
   // depends on. Does not change the base architecture, or follow dependencies
   // between features which are only related by required arcitecture versions.
-  void enable(ArchExtKind E);
+  LLVM_ABI void enable(ArchExtKind E);
 
   // Disable the given architecture extension, and any other extensions which
   // depend on it. Does not change the base architecture, or follow
   // dependencies between features which are only related by required
   // arcitecture versions.
-  void disable(ArchExtKind E);
+  LLVM_ABI void disable(ArchExtKind E);
 
   // Add default extensions for the given CPU. Records the base architecture,
   // to later resolve dependencies which depend on it.
-  void addCPUDefaults(const CpuInfo &CPU);
+  LLVM_ABI void addCPUDefaults(const CpuInfo &CPU);
 
   // Add default extensions for the given architecture version. Records the
   // base architecture, to later resolve dependencies which depend on it.
-  void addArchDefaults(const ArchInfo &Arch);
+  LLVM_ABI void addArchDefaults(const ArchInfo &Arch);
 
   // Add or remove a feature based on a modifier string. The string must be of
   // the form "<name>" to enable a feature or "no<name>" to disable it. This
   // will also enable or disable any features as required by the dependencies
   // between them.
-  bool parseModifier(StringRef Modifier, const bool AllowNoDashForm = false);
+  LLVM_ABI bool parseModifier(StringRef Modifier,
+                              const bool AllowNoDashForm = false);
 
   // Constructs a new ExtensionSet by toggling the corresponding bits for every
   // feature in the \p Features list without expanding their dependencies. Used
   // for reconstructing an ExtensionSet from the output of toLLVMFeatures().
   // Features that are not recognized are pushed back to \p NonExtensions.
-  void reconstructFromParsedFeatures(const std::vector<std::string> &Features,
-                                     std::vector<std::string> &NonExtensions);
+  LLVM_ABI void
+  reconstructFromParsedFeatures(const std::vector<std::string> &Features,
+                                std::vector<std::string> &NonExtensions);
 
   // Convert the set of enabled extension to an LLVM feature list, appending
   // them to Features.
@@ -227,7 +230,7 @@ struct ExtensionSet {
     }
   }
 
-  void dump() const;
+  LLVM_ABI void dump() const;
 };
 
 // Name alias.
@@ -239,52 +242,53 @@ struct Alias {
 #define EMIT_CPU_ALIAS
 #include "llvm/TargetParser/AArch64TargetParserDef.inc"
 
-const ExtensionInfo &getExtensionByID(ArchExtKind(ExtID));
+LLVM_ABI const ExtensionInfo &getExtensionByID(ArchExtKind(ExtID));
 
-bool getExtensionFeatures(
-    const AArch64::ExtensionBitset &Extensions,
-    std::vector<StringRef> &Features);
+LLVM_ABI bool getExtensionFeatures(const AArch64::ExtensionBitset &Extensions,
+                                   std::vector<StringRef> &Features);
 
-StringRef getArchExtFeature(StringRef ArchExt);
-StringRef resolveCPUAlias(StringRef CPU);
+LLVM_ABI StringRef getArchExtFeature(StringRef ArchExt);
+LLVM_ABI StringRef resolveCPUAlias(StringRef CPU);
 
 // Information by Name
-const ArchInfo *getArchForCpu(StringRef CPU);
+LLVM_ABI const ArchInfo *getArchForCpu(StringRef CPU);
 
 // Parser
-const ArchInfo *parseArch(StringRef Arch);
+LLVM_ABI const ArchInfo *parseArch(StringRef Arch);
 
 // Return the extension which has the given -target-feature name.
-std::optional<ExtensionInfo> targetFeatureToExtension(StringRef TargetFeature);
+LLVM_ABI std::optional<ExtensionInfo>
+targetFeatureToExtension(StringRef TargetFeature);
 
 // Parse a name as defined by the Extension class in tablegen.
-std::optional<ExtensionInfo> parseArchExtension(StringRef Extension);
+LLVM_ABI std::optional<ExtensionInfo> parseArchExtension(StringRef Extension);
 
 // Parse a name as defined by the FMVInfo class in tablegen.
-std::optional<FMVInfo> parseFMVExtension(StringRef Extension);
+LLVM_ABI std::optional<FMVInfo> parseFMVExtension(StringRef Extension);
 
 // Given the name of a CPU or alias, return the correponding CpuInfo.
-std::optional<CpuInfo> parseCpu(StringRef Name);
+LLVM_ABI std::optional<CpuInfo> parseCpu(StringRef Name);
 // Used by target parser tests
-void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
 
-bool isX18ReservedByDefault(const Triple &TT);
+LLVM_ABI bool isX18ReservedByDefault(const Triple &TT);
 
 // For a given set of feature names, which can be either target-features, or
 // fmv-features metadata, expand their dependencies and then return a bitmask
 // corresponding to the entries of AArch64::FeatPriorities.
-uint64_t getFMVPriority(ArrayRef<StringRef> Features);
+LLVM_ABI uint64_t getFMVPriority(ArrayRef<StringRef> Features);
 
 // For a given set of FMV feature names, expand their dependencies and then
 // return a bitmask corresponding to the entries of AArch64::CPUFeatures.
 // The values in CPUFeatures are not bitmasks themselves, they are sequential
 // (0, 1, 2, 3, ...). The resulting bitmask is used at runtime to test whether
 // a certain FMV feature is available on the host.
-uint64_t getCpuSupportsMask(ArrayRef<StringRef> Features);
+LLVM_ABI uint64_t getCpuSupportsMask(ArrayRef<StringRef> Features);
 
-void PrintSupportedExtensions();
+LLVM_ABI void PrintSupportedExtensions();
 
-void printEnabledExtensions(const std::set<StringRef> &EnabledFeatureNames);
+LLVM_ABI void
+printEnabledExtensions(const std::set<StringRef> &EnabledFeatureNames);
 
 } // namespace AArch64
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParser.h
index b2403f42f1b7..798c578ced93 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParser.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ARMBuildAttributes.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/ARMTargetParserCommon.h"
 #include <vector>
 
@@ -223,53 +224,55 @@ inline ArchKind &operator--(ArchKind &Kind) {
 }
 
 // Information by ID
-StringRef getFPUName(FPUKind FPUKind);
-FPUVersion getFPUVersion(FPUKind FPUKind);
-NeonSupportLevel getFPUNeonSupportLevel(FPUKind FPUKind);
-FPURestriction getFPURestriction(FPUKind FPUKind);
-
-bool getFPUFeatures(FPUKind FPUKind, std::vector<StringRef> &Features);
-bool getHWDivFeatures(uint64_t HWDivKind, std::vector<StringRef> &Features);
-bool getExtensionFeatures(uint64_t Extensions,
-                          std::vector<StringRef> &Features);
-
-StringRef getArchName(ArchKind AK);
-unsigned getArchAttr(ArchKind AK);
-StringRef getCPUAttr(ArchKind AK);
-StringRef getSubArch(ArchKind AK);
-StringRef getArchExtName(uint64_t ArchExtKind);
-StringRef getArchExtFeature(StringRef ArchExt);
-bool appendArchExtFeatures(StringRef CPU, ARM::ArchKind AK, StringRef ArchExt,
-                           std::vector<StringRef> &Features,
-                           FPUKind &ArgFPUKind);
-ArchKind convertV9toV8(ArchKind AK);
+LLVM_ABI StringRef getFPUName(FPUKind FPUKind);
+LLVM_ABI FPUVersion getFPUVersion(FPUKind FPUKind);
+LLVM_ABI NeonSupportLevel getFPUNeonSupportLevel(FPUKind FPUKind);
+LLVM_ABI FPURestriction getFPURestriction(FPUKind FPUKind);
+
+LLVM_ABI bool getFPUFeatures(FPUKind FPUKind, std::vector<StringRef> &Features);
+LLVM_ABI bool getHWDivFeatures(uint64_t HWDivKind,
+                               std::vector<StringRef> &Features);
+LLVM_ABI bool getExtensionFeatures(uint64_t Extensions,
+                                   std::vector<StringRef> &Features);
+
+LLVM_ABI StringRef getArchName(ArchKind AK);
+LLVM_ABI unsigned getArchAttr(ArchKind AK);
+LLVM_ABI StringRef getCPUAttr(ArchKind AK);
+LLVM_ABI StringRef getSubArch(ArchKind AK);
+LLVM_ABI StringRef getArchExtName(uint64_t ArchExtKind);
+LLVM_ABI StringRef getArchExtFeature(StringRef ArchExt);
+LLVM_ABI bool appendArchExtFeatures(StringRef CPU, ARM::ArchKind AK,
+                                    StringRef ArchExt,
+                                    std::vector<StringRef> &Features,
+                                    FPUKind &ArgFPUKind);
+LLVM_ABI ArchKind convertV9toV8(ArchKind AK);
 
 // Information by Name
-FPUKind getDefaultFPU(StringRef CPU, ArchKind AK);
-uint64_t getDefaultExtensions(StringRef CPU, ArchKind AK);
-StringRef getDefaultCPU(StringRef Arch);
-StringRef getCanonicalArchName(StringRef Arch);
-StringRef getFPUSynonym(StringRef FPU);
+LLVM_ABI FPUKind getDefaultFPU(StringRef CPU, ArchKind AK);
+LLVM_ABI uint64_t getDefaultExtensions(StringRef CPU, ArchKind AK);
+LLVM_ABI StringRef getDefaultCPU(StringRef Arch);
+LLVM_ABI StringRef getFPUSynonym(StringRef FPU);
 
 // Parser
-uint64_t parseHWDiv(StringRef HWDiv);
-FPUKind parseFPU(StringRef FPU);
-ArchKind parseArch(StringRef Arch);
-uint64_t parseArchExt(StringRef ArchExt);
-ArchKind parseCPUArch(StringRef CPU);
-ProfileKind parseArchProfile(StringRef Arch);
-unsigned parseArchVersion(StringRef Arch);
+LLVM_ABI uint64_t parseHWDiv(StringRef HWDiv);
+LLVM_ABI FPUKind parseFPU(StringRef FPU);
+LLVM_ABI ArchKind parseArch(StringRef Arch);
+LLVM_ABI uint64_t parseArchExt(StringRef ArchExt);
+LLVM_ABI ArchKind parseCPUArch(StringRef CPU);
+LLVM_ABI ProfileKind parseArchProfile(StringRef Arch);
+LLVM_ABI unsigned parseArchVersion(StringRef Arch);
 
-void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
-StringRef computeDefaultTargetABI(const Triple &TT, StringRef CPU);
+LLVM_ABI void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI StringRef computeDefaultTargetABI(const Triple &TT, StringRef CPU);
 
 /// Get the (LLVM) name of the minimum ARM CPU for the arch we are targeting.
 ///
 /// \param Arch the architecture name (e.g., "armv7s"). If it is an empty
 /// string then the triple's arch name is used.
-StringRef getARMCPUForArch(const llvm::Triple &Triple, StringRef MArch = {});
+LLVM_ABI StringRef getARMCPUForArch(const llvm::Triple &Triple,
+                                    StringRef MArch = {});
 
-void PrintSupportedExtensions(StringMap<StringRef> DescMap);
+LLVM_ABI void PrintSupportedExtensions(StringMap<StringRef> DescMap);
 
 } // namespace ARM
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h b/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h
index f6115718e9f5..7c8030dd5576 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h
@@ -14,6 +14,7 @@
 #define LLVM_TARGETPARSER_ARMTARGETPARSERCOMMON_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace ARM {
@@ -23,19 +24,19 @@ enum class ISAKind { INVALID = 0, ARM, THUMB, AARCH64 };
 enum class EndianKind { INVALID = 0, LITTLE, BIG };
 
 /// Converts e.g. "armv8" -> "armv8-a"
-StringRef getArchSynonym(StringRef Arch);
+LLVM_ABI StringRef getArchSynonym(StringRef Arch);
 
 /// MArch is expected to be of the form (arm|thumb)?(eb)?(v.+)?(eb)?, but
 /// (iwmmxt|xscale)(eb)? is also permitted. If the former, return
 /// "v.+", if the latter, return unmodified string, minus 'eb'.
 /// If invalid, return empty string.
-StringRef getCanonicalArchName(StringRef Arch);
+LLVM_ABI StringRef getCanonicalArchName(StringRef Arch);
 
 // ARM, Thumb, AArch64
-ISAKind parseArchISA(StringRef Arch);
+LLVM_ABI ISAKind parseArchISA(StringRef Arch);
 
 // Little/Big endian
-EndianKind parseArchEndian(StringRef Arch);
+LLVM_ABI EndianKind parseArchEndian(StringRef Arch);
 
 struct ParsedBranchProtection {
   StringRef Scope;
@@ -45,8 +46,8 @@ struct ParsedBranchProtection {
   bool GuardedControlStack;
 };
 
-bool parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP,
-                           StringRef &Err, bool EnablePAuthLR = false);
+LLVM_ABI bool parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP,
+                                    StringRef &Err, bool EnablePAuthLR = false);
 
 } // namespace ARM
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/CSKYTargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/CSKYTargetParser.h
index 4c4ec06f758a..8eab03ca0149 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/CSKYTargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/CSKYTargetParser.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_TARGETPARSER_CSKYTARGETPARSER_H
 #define LLVM_TARGETPARSER_CSKYTARGETPARSER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 #include <vector>
 
@@ -176,25 +177,26 @@ const ArchNames<CSKY::ArchKind> ARCHNames[] = {
 #include "llvm/TargetParser/CSKYTargetParser.def"
 };
 
-StringRef getArchName(ArchKind AK);
-StringRef getDefaultCPU(StringRef Arch);
-StringRef getArchExtName(uint64_t ArchExtKind);
-StringRef getArchExtFeature(StringRef ArchExt);
-uint64_t getDefaultExtensions(StringRef CPU);
-bool getExtensionFeatures(uint64_t Extensions,
-                          std::vector<StringRef> &Features);
+LLVM_ABI StringRef getArchName(ArchKind AK);
+LLVM_ABI StringRef getDefaultCPU(StringRef Arch);
+LLVM_ABI StringRef getArchExtName(uint64_t ArchExtKind);
+LLVM_ABI StringRef getArchExtFeature(StringRef ArchExt);
+LLVM_ABI uint64_t getDefaultExtensions(StringRef CPU);
+LLVM_ABI bool getExtensionFeatures(uint64_t Extensions,
+                                   std::vector<StringRef> &Features);
 
 // Information by ID
-StringRef getFPUName(unsigned FPUKind);
-FPUVersion getFPUVersion(unsigned FPUKind);
+LLVM_ABI StringRef getFPUName(unsigned FPUKind);
+LLVM_ABI FPUVersion getFPUVersion(unsigned FPUKind);
 
-bool getFPUFeatures(CSKYFPUKind Kind, std::vector<StringRef> &Features);
+LLVM_ABI bool getFPUFeatures(CSKYFPUKind Kind,
+                             std::vector<StringRef> &Features);
 
 // Parser
-ArchKind parseArch(StringRef Arch);
-ArchKind parseCPUArch(StringRef CPU);
-uint64_t parseArchExt(StringRef ArchExt);
-void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI ArchKind parseArch(StringRef Arch);
+LLVM_ABI ArchKind parseCPUArch(StringRef CPU);
+LLVM_ABI uint64_t parseArchExt(StringRef ArchExt);
+LLVM_ABI void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
 
 } // namespace CSKY
 
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/Host.h b/external/llvm-project/llvm/include/llvm/TargetParser/Host.h
index a639b093f21b..be3d41e022ad 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/Host.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/Host.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_TARGETPARSER_HOST_H
 #define LLVM_TARGETPARSER_HOST_H
 
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
@@ -23,65 +24,65 @@ class raw_ostream;
 
 namespace sys {
 
-  /// getDefaultTargetTriple() - Return the default target triple the compiler
-  /// has been configured to produce code for.
-  ///
-  /// The target triple is a string in the format of:
-  ///   CPU_TYPE-VENDOR-OPERATING_SYSTEM
-  /// or
-  ///   CPU_TYPE-VENDOR-KERNEL-OPERATING_SYSTEM
-  std::string getDefaultTargetTriple();
+/// getDefaultTargetTriple() - Return the default target triple the compiler
+/// has been configured to produce code for.
+///
+/// The target triple is a string in the format of:
+///   CPU_TYPE-VENDOR-OPERATING_SYSTEM
+/// or
+///   CPU_TYPE-VENDOR-KERNEL-OPERATING_SYSTEM
+LLVM_ABI std::string getDefaultTargetTriple();
 
-  /// getProcessTriple() - Return an appropriate target triple for generating
-  /// code to be loaded into the current process, e.g. when using the JIT.
-  std::string getProcessTriple();
+/// getProcessTriple() - Return an appropriate target triple for generating
+/// code to be loaded into the current process, e.g. when using the JIT.
+LLVM_ABI std::string getProcessTriple();
 
-  /// getHostCPUName - Get the LLVM name for the host CPU. The particular format
-  /// of the name is target dependent, and suitable for passing as -mcpu to the
-  /// target which matches the host.
-  ///
-  /// \return - The host CPU name, or empty if the CPU could not be determined.
-  StringRef getHostCPUName();
+/// getHostCPUName - Get the LLVM name for the host CPU. The particular format
+/// of the name is target dependent, and suitable for passing as -mcpu to the
+/// target which matches the host.
+///
+/// \return - The host CPU name, or empty if the CPU could not be determined.
+LLVM_ABI StringRef getHostCPUName();
 
-  /// getHostCPUFeatures - Get the LLVM names for the host CPU features.
-  /// The particular format of the names are target dependent, and suitable for
-  /// passing as -mattr to the target which matches the host.
-  ///
-  /// \return - A string map mapping feature names to either true (if enabled)
-  /// or false (if disabled). This routine makes no guarantees about exactly
-  /// which features may appear in this map, except that they are all valid LLVM
-  /// feature names. The map can be empty, for example if feature detection
-  /// fails.
-  const StringMap<bool, MallocAllocator> getHostCPUFeatures();
+/// getHostCPUFeatures - Get the LLVM names for the host CPU features.
+/// The particular format of the names are target dependent, and suitable for
+/// passing as -mattr to the target which matches the host.
+///
+/// \return - A string map mapping feature names to either true (if enabled)
+/// or false (if disabled). This routine makes no guarantees about exactly
+/// which features may appear in this map, except that they are all valid LLVM
+/// feature names. The map can be empty, for example if feature detection
+/// fails.
+LLVM_ABI const StringMap<bool, MallocAllocator> getHostCPUFeatures();
 
-  /// This is a function compatible with cl::AddExtraVersionPrinter, which adds
-  /// info about the current target triple and detected CPU.
-  void printDefaultTargetAndDetectedCPU(raw_ostream &OS);
+/// This is a function compatible with cl::AddExtraVersionPrinter, which adds
+/// info about the current target triple and detected CPU.
+LLVM_ABI void printDefaultTargetAndDetectedCPU(raw_ostream &OS);
 
-  namespace detail {
-  /// Helper functions to extract HostCPUName from /proc/cpuinfo on linux.
-  StringRef getHostCPUNameForPowerPC(StringRef ProcCpuinfoContent);
-  StringRef getHostCPUNameForARM(StringRef ProcCpuinfoContent);
-  StringRef getHostCPUNameForS390x(StringRef ProcCpuinfoContent);
-  StringRef getHostCPUNameForRISCV(StringRef ProcCpuinfoContent);
-  StringRef getHostCPUNameForSPARC(StringRef ProcCpuinfoContent);
-  StringRef getHostCPUNameForBPF();
+namespace detail {
+/// Helper functions to extract HostCPUName from /proc/cpuinfo on linux.
+LLVM_ABI StringRef getHostCPUNameForPowerPC(StringRef ProcCpuinfoContent);
+LLVM_ABI StringRef getHostCPUNameForARM(StringRef ProcCpuinfoContent);
+LLVM_ABI StringRef getHostCPUNameForS390x(StringRef ProcCpuinfoContent);
+LLVM_ABI StringRef getHostCPUNameForRISCV(StringRef ProcCpuinfoContent);
+LLVM_ABI StringRef getHostCPUNameForSPARC(StringRef ProcCpuinfoContent);
+LLVM_ABI StringRef getHostCPUNameForBPF();
 
-  /// Helper functions to extract CPU details from CPUID on x86.
-  namespace x86 {
-  enum class VendorSignatures {
-    UNKNOWN,
-    GENUINE_INTEL,
-    AUTHENTIC_AMD,
-  };
+/// Helper functions to extract CPU details from CPUID on x86.
+namespace x86 {
+enum class VendorSignatures {
+  UNKNOWN,
+  GENUINE_INTEL,
+  AUTHENTIC_AMD,
+};
 
-  /// Returns the host CPU's vendor.
-  /// MaxLeaf: if a non-nullptr pointer is specified, the EAX value will be
-  /// assigned to its pointee.
-  VendorSignatures getVendorSignature(unsigned *MaxLeaf = nullptr);
-  } // namespace x86
-  }
-}
-}
+/// Returns the host CPU's vendor.
+/// MaxLeaf: if a non-nullptr pointer is specified, the EAX value will be
+/// assigned to its pointee.
+LLVM_ABI VendorSignatures getVendorSignature(unsigned *MaxLeaf = nullptr);
+} // namespace x86
+} // namespace detail
+} // namespace sys
+} // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/LoongArchTargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/LoongArchTargetParser.h
index e08e7bc182e1..1357d7474459 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/LoongArchTargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/LoongArchTargetParser.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TARGETPARSER_LOONGARCHTARGETPARSER_H
 #define LLVM_TARGETPARSER_LOONGARCHTARGETPARSER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 #include <vector>
 
@@ -84,11 +85,12 @@ struct ArchInfo {
   uint32_t Features;
 };
 
-bool isValidArchName(StringRef Arch);
-bool getArchFeatures(StringRef Arch, std::vector<StringRef> &Features);
-bool isValidCPUName(StringRef TuneCPU);
-void fillValidCPUList(SmallVectorImpl<StringRef> &Values);
-StringRef getDefaultArch(bool Is64Bit);
+LLVM_ABI bool isValidArchName(StringRef Arch);
+LLVM_ABI bool isValidFeatureName(StringRef Feature);
+LLVM_ABI bool getArchFeatures(StringRef Arch, std::vector<StringRef> &Features);
+LLVM_ABI bool isValidCPUName(StringRef TuneCPU);
+LLVM_ABI void fillValidCPUList(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI StringRef getDefaultArch(bool Is64Bit);
 
 } // namespace LoongArch
 
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/PPCTargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/PPCTargetParser.h
index 5f9fe543aff0..59d9f867005a 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/PPCTargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/PPCTargetParser.h
@@ -15,25 +15,28 @@
 #define LLVM_TARGETPARSER_PPCTARGETPARSER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
 namespace PPC {
-bool isValidCPU(StringRef CPU);
-void fillValidCPUList(SmallVectorImpl<StringRef> &Values);
-void fillValidTuneCPUList(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI bool isValidCPU(StringRef CPU);
+LLVM_ABI void fillValidCPUList(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI void fillValidTuneCPUList(SmallVectorImpl<StringRef> &Values);
 
 // Get target CPU name.
 // If CPUName is empty or generic, return the default CPU name.
 // If CPUName is not empty or generic, return the normalized CPU name.
-StringRef getNormalizedPPCTargetCPU(const Triple &T, StringRef CPUName = "");
+LLVM_ABI StringRef getNormalizedPPCTargetCPU(const Triple &T,
+                                             StringRef CPUName = "");
 
 // Get the tune CPU name.
-StringRef getNormalizedPPCTuneCPU(const Triple &T, StringRef CPUName = "");
+LLVM_ABI StringRef getNormalizedPPCTuneCPU(const Triple &T,
+                                           StringRef CPUName = "");
 
 // For PPC, there are some cpu names for same CPU, like pwr10 and power10,
 // normalize them.
-StringRef normalizeCPUName(StringRef CPUName);
+LLVM_ABI StringRef normalizeCPUName(StringRef CPUName);
 } // namespace PPC
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/RISCVISAInfo.h b/external/llvm-project/llvm/include/llvm/TargetParser/RISCVISAInfo.h
index 5b2b6f29fd3d..0c308cadba79 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/RISCVISAInfo.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/RISCVISAInfo.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/RISCVISAUtils.h"
 
@@ -31,27 +32,27 @@ class RISCVISAInfo {
   /// extensions with unrecognised versions will be silently dropped, except
   /// for the special case of the base 'i' and 'e' extensions, where the
   /// default version will be used (as ignoring the base is not possible).
-  static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
+  LLVM_ABI static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
   parseArchString(StringRef Arch, bool EnableExperimentalExtension,
                   bool ExperimentalExtensionVersionCheck = true);
 
   /// Parse RISC-V ISA info from an arch string that is already in normalized
   /// form (as defined in the psABI). Unlike parseArchString, this function
   /// will not error for unrecognized extension names or extension versions.
-  static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
+  LLVM_ABI static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
   parseNormalizedArchString(StringRef Arch);
 
   /// Parse RISC-V ISA info from feature vector.
-  static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
+  LLVM_ABI static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
   parseFeatures(unsigned XLen, const std::vector<std::string> &Features);
 
-  static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
+  LLVM_ABI static llvm::Expected<std::unique_ptr<RISCVISAInfo>>
   createFromExtMap(unsigned XLen,
                    const RISCVISAUtils::OrderedExtensionMap &Exts);
 
   /// Convert RISC-V ISA info to a feature vector.
-  std::vector<std::string> toFeatures(bool AddAllExtensions = false,
-                                      bool IgnoreUnknown = true) const;
+  LLVM_ABI std::vector<std::string> toFeatures(bool AddAllExtensions = false,
+                                               bool IgnoreUnknown = true) const;
 
   const RISCVISAUtils::OrderedExtensionMap &getExtensions() const {
     return Exts;
@@ -64,25 +65,26 @@ class RISCVISAInfo {
   unsigned getMaxELen() const { return MaxELen; }
   unsigned getMaxELenFp() const { return MaxELenFp; }
 
-  bool hasExtension(StringRef Ext) const;
-  std::string toString() const;
-  StringRef computeDefaultABI() const;
+  LLVM_ABI bool hasExtension(StringRef Ext) const;
+  LLVM_ABI std::string toString() const;
+  LLVM_ABI StringRef computeDefaultABI() const;
 
-  static bool isSupportedExtensionFeature(StringRef Ext);
-  static bool isSupportedExtension(StringRef Ext);
-  static bool isSupportedExtensionWithVersion(StringRef Ext);
-  static bool isSupportedExtension(StringRef Ext, unsigned MajorVersion,
-                                   unsigned MinorVersion);
-  static std::string getTargetFeatureForExtension(StringRef Ext);
+  LLVM_ABI static bool isSupportedExtensionFeature(StringRef Ext);
+  LLVM_ABI static bool isSupportedExtension(StringRef Ext);
+  LLVM_ABI static bool isSupportedExtensionWithVersion(StringRef Ext);
+  LLVM_ABI static bool isSupportedExtension(StringRef Ext,
+                                            unsigned MajorVersion,
+                                            unsigned MinorVersion);
+  LLVM_ABI static std::string getTargetFeatureForExtension(StringRef Ext);
 
-  static void printSupportedExtensions(StringMap<StringRef> &DescMap);
-  static void printEnabledExtensions(bool IsRV64,
-                                     std::set<StringRef> &EnabledFeatureNames,
-                                     StringMap<StringRef> &DescMap);
+  LLVM_ABI static void printSupportedExtensions(StringMap<StringRef> &DescMap);
+  LLVM_ABI static void
+  printEnabledExtensions(bool IsRV64, std::set<StringRef> &EnabledFeatureNames,
+                         StringMap<StringRef> &DescMap);
 
   /// Return the group id and bit position of __riscv_feature_bits.  Returns
   /// <-1, -1> if not supported.
-  static std::pair<int, int> getRISCVFeaturesBitsInfo(StringRef Ext);
+  LLVM_ABI static std::pair<int, int> getRISCVFeaturesBitsInfo(StringRef Ext);
 
   // The maximum value of the group ID obtained from getRISCVFeaturesBitsInfo.
   static constexpr unsigned FeatureBitSize = 2;
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/RISCVTargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/RISCVTargetParser.h
index a529479b546d..19a8af0cb956 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/RISCVTargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/RISCVTargetParser.h
@@ -15,6 +15,7 @@
 #define LLVM_TARGETPARSER_RISCVTARGETPARSER_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -28,6 +29,13 @@ struct CPUModel {
   uint32_t MVendorID;
   uint64_t MArchID;
   uint64_t MImpID;
+
+  bool isValid() const { return MVendorID != 0 && MArchID != 0 && MImpID != 0; }
+
+  bool operator==(const CPUModel &Other) const {
+    return MVendorID == Other.MVendorID && MArchID == Other.MArchID &&
+           MImpID == Other.MImpID;
+  }
 };
 
 struct CPUInfo {
@@ -43,18 +51,21 @@ struct CPUInfo {
 static constexpr unsigned RVVBitsPerBlock = 64;
 static constexpr unsigned RVVBytesPerBlock = RVVBitsPerBlock / 8;
 
-void getFeaturesForCPU(StringRef CPU,
-                       SmallVectorImpl<std::string> &EnabledFeatures,
-                       bool NeedPlus = false);
-bool parseCPU(StringRef CPU, bool IsRV64);
-bool parseTuneCPU(StringRef CPU, bool IsRV64);
-StringRef getMArchFromMcpu(StringRef CPU);
-void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64);
-void fillValidTuneCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64);
-bool hasFastScalarUnalignedAccess(StringRef CPU);
-bool hasFastVectorUnalignedAccess(StringRef CPU);
-bool hasValidCPUModel(StringRef CPU);
-CPUModel getCPUModel(StringRef CPU);
+LLVM_ABI void getFeaturesForCPU(StringRef CPU,
+                                SmallVectorImpl<std::string> &EnabledFeatures,
+                                bool NeedPlus = false);
+LLVM_ABI bool parseCPU(StringRef CPU, bool IsRV64);
+LLVM_ABI bool parseTuneCPU(StringRef CPU, bool IsRV64);
+LLVM_ABI StringRef getMArchFromMcpu(StringRef CPU);
+LLVM_ABI void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values,
+                                   bool IsRV64);
+LLVM_ABI void fillValidTuneCPUArchList(SmallVectorImpl<StringRef> &Values,
+                                       bool IsRV64);
+LLVM_ABI bool hasFastScalarUnalignedAccess(StringRef CPU);
+LLVM_ABI bool hasFastVectorUnalignedAccess(StringRef CPU);
+LLVM_ABI bool hasValidCPUModel(StringRef CPU);
+LLVM_ABI CPUModel getCPUModel(StringRef CPU);
+LLVM_ABI StringRef getCPUNameFromCPUModel(const CPUModel &Model);
 
 } // namespace RISCV
 
@@ -86,10 +97,10 @@ inline static bool isValidLMUL(unsigned LMUL, bool Fractional) {
   return isPowerOf2_32(LMUL) && LMUL <= 8 && (!Fractional || LMUL != 1);
 }
 
-unsigned encodeVTYPE(VLMUL VLMUL, unsigned SEW, bool TailAgnostic,
-                     bool MaskAgnostic);
+LLVM_ABI unsigned encodeVTYPE(VLMUL VLMUL, unsigned SEW, bool TailAgnostic,
+                              bool MaskAgnostic);
 
-unsigned encodeXSfmmVType(unsigned SEW, unsigned Widen, bool AltFmt);
+LLVM_ABI unsigned encodeXSfmmVType(unsigned SEW, unsigned Widen, bool AltFmt);
 
 inline static VLMUL getVLMUL(unsigned VType) {
   unsigned VLMul = VType & 0x7;
@@ -97,7 +108,7 @@ inline static VLMUL getVLMUL(unsigned VType) {
 }
 
 // Decode VLMUL into 1,2,4,8 and fractional indicator.
-std::pair<unsigned, bool> decodeVLMUL(VLMUL VLMul);
+LLVM_ABI std::pair<unsigned, bool> decodeVLMUL(VLMUL VLMul);
 
 inline static VLMUL encodeLMUL(unsigned LMUL, bool Fractional) {
   assert(isValidLMUL(LMUL, Fractional) && "Unsupported LMUL");
@@ -148,11 +159,12 @@ inline static bool isMaskAgnostic(unsigned VType) { return VType & 0x80; }
 
 inline static bool isAltFmt(unsigned VType) { return VType & 0x100; }
 
-void printVType(unsigned VType, raw_ostream &OS);
+LLVM_ABI void printVType(unsigned VType, raw_ostream &OS);
 
-unsigned getSEWLMULRatio(unsigned SEW, VLMUL VLMul);
+LLVM_ABI unsigned getSEWLMULRatio(unsigned SEW, VLMUL VLMul);
 
-std::optional<VLMUL> getSameRatioLMUL(unsigned SEW, VLMUL VLMUL, unsigned EEW);
+LLVM_ABI std::optional<VLMUL> getSameRatioLMUL(unsigned SEW, VLMUL VLMUL,
+                                               unsigned EEW);
 } // namespace RISCVVType
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/SubtargetFeature.h b/external/llvm-project/llvm/include/llvm/TargetParser/SubtargetFeature.h
index 2e1f00dad2df..6f1723dec5d0 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/SubtargetFeature.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/SubtargetFeature.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include <array>
 #include <initializer_list>
@@ -175,27 +176,27 @@ class SubtargetFeatures {
   std::vector<std::string> Features;    ///< Subtarget features as a vector
 
 public:
-  explicit SubtargetFeatures(StringRef Initial = "");
+  LLVM_ABI explicit SubtargetFeatures(StringRef Initial = "");
 
   /// Returns features as a string.
-  std::string getString() const;
+  LLVM_ABI std::string getString() const;
 
   /// Adds Features.
-  void AddFeature(StringRef String, bool Enable = true);
+  LLVM_ABI void AddFeature(StringRef String, bool Enable = true);
 
-  void addFeaturesVector(const ArrayRef<std::string> OtherFeatures);
+  LLVM_ABI void addFeaturesVector(const ArrayRef<std::string> OtherFeatures);
 
   /// Returns the vector of individual subtarget features.
   const std::vector<std::string> &getFeatures() const { return Features; }
 
   /// Prints feature string.
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
   // Dumps feature info.
-  void dump() const;
+  LLVM_ABI void dump() const;
 
   /// Adds the default features for the specified target triple.
-  void getDefaultSubtargetFeatures(const Triple& Triple);
+  LLVM_ABI void getDefaultSubtargetFeatures(const Triple &Triple);
 
   /// Determine if a feature has a flag; '+' or '-'
   static bool hasFlag(StringRef Feature) {
@@ -221,7 +222,7 @@ class SubtargetFeatures {
   }
 
   /// Splits a string of comma separated items in to a vector of strings.
-  static void Split(std::vector<std::string> &V, StringRef S);
+  LLVM_ABI static void Split(std::vector<std::string> &V, StringRef S);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/TargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/TargetParser.h
index f776b41f3d7c..176205e17ae0 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/TargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/TargetParser.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -164,27 +165,27 @@ enum FeatureError : uint32_t {
   UNSUPPORTED_TARGET_FEATURE
 };
 
-StringRef getArchFamilyNameAMDGCN(GPUKind AK);
+LLVM_ABI StringRef getArchFamilyNameAMDGCN(GPUKind AK);
 
-StringRef getArchNameAMDGCN(GPUKind AK);
-StringRef getArchNameR600(GPUKind AK);
-StringRef getCanonicalArchName(const Triple &T, StringRef Arch);
-GPUKind parseArchAMDGCN(StringRef CPU);
-GPUKind parseArchR600(StringRef CPU);
-unsigned getArchAttrAMDGCN(GPUKind AK);
-unsigned getArchAttrR600(GPUKind AK);
+LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK);
+LLVM_ABI StringRef getArchNameR600(GPUKind AK);
+LLVM_ABI StringRef getCanonicalArchName(const Triple &T, StringRef Arch);
+LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU);
+LLVM_ABI GPUKind parseArchR600(StringRef CPU);
+LLVM_ABI unsigned getArchAttrAMDGCN(GPUKind AK);
+LLVM_ABI unsigned getArchAttrR600(GPUKind AK);
 
-void fillValidArchListAMDGCN(SmallVectorImpl<StringRef> &Values);
-void fillValidArchListR600(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI void fillValidArchListAMDGCN(SmallVectorImpl<StringRef> &Values);
+LLVM_ABI void fillValidArchListR600(SmallVectorImpl<StringRef> &Values);
 
-IsaVersion getIsaVersion(StringRef GPU);
+LLVM_ABI IsaVersion getIsaVersion(StringRef GPU);
 
 /// Fills Features map with default values for given target GPU
-void fillAMDGPUFeatureMap(StringRef GPU, const Triple &T,
-                          StringMap<bool> &Features);
+LLVM_ABI void fillAMDGPUFeatureMap(StringRef GPU, const Triple &T,
+                                   StringMap<bool> &Features);
 
 /// Inserts wave size feature for given GPU into features map
-std::pair<FeatureError, StringRef>
+LLVM_ABI std::pair<FeatureError, StringRef>
 insertWaveSizeFeature(StringRef GPU, const Triple &T,
                       StringMap<bool> &Features);
 
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/Triple.h b/external/llvm-project/llvm/include/llvm/TargetParser/Triple.h
index 7fd5278f1ed5..b6f15ef13191 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/Triple.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/Triple.h
@@ -10,6 +10,7 @@
 #define LLVM_TARGETPARSER_TRIPLE_H
 
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
 
 // Some system headers or GCC predefined macros conflict with identifiers in
@@ -348,10 +349,11 @@ class Triple {
   /// triple fields unknown.
   Triple() = default;
 
-  explicit Triple(const Twine &Str);
-  Triple(const Twine &ArchStr, const Twine &VendorStr, const Twine &OSStr);
-  Triple(const Twine &ArchStr, const Twine &VendorStr, const Twine &OSStr,
-         const Twine &EnvironmentStr);
+  LLVM_ABI explicit Triple(const Twine &Str);
+  LLVM_ABI Triple(const Twine &ArchStr, const Twine &VendorStr,
+                  const Twine &OSStr);
+  LLVM_ABI Triple(const Twine &ArchStr, const Twine &VendorStr,
+                  const Twine &OSStr, const Twine &EnvironmentStr);
 
   bool operator==(const Triple &Other) const {
     return Arch == Other.Arch && SubArch == Other.SubArch &&
@@ -381,8 +383,8 @@ class Triple {
   /// reasonably be done).  In particular, it handles the common case in which
   /// otherwise valid components are in the wrong order. \p Form is used to
   /// specify the output canonical form.
-  static std::string normalize(StringRef Str,
-                               CanonicalForm Form = CanonicalForm::ANY);
+  LLVM_ABI static std::string
+  normalize(StringRef Str, CanonicalForm Form = CanonicalForm::ANY);
 
   /// Return the normalized form of this triple's string.
   std::string normalize(CanonicalForm Form = CanonicalForm::ANY) const {
@@ -417,7 +419,7 @@ class Triple {
   /// triple, if present.
   ///
   /// For example, "fooos1.2.3" would return (1, 2, 3).
-  VersionTuple getEnvironmentVersion() const;
+  LLVM_ABI VersionTuple getEnvironmentVersion() const;
 
   /// Get the object format for this triple.
   ObjectFormatType getObjectFormat() const { return ObjectFormat; }
@@ -426,7 +428,7 @@ class Triple {
   /// present.
   ///
   /// For example, "fooos1.2.3" would return (1, 2, 3).
-  VersionTuple getOSVersion() const;
+  LLVM_ABI VersionTuple getOSVersion() const;
 
   /// Return just the major version number, this is specialized because it is a
   /// common query.
@@ -436,26 +438,26 @@ class Triple {
   /// "darwin" versions to the corresponding OS X versions.  This may also be
   /// called with IOS triples but the OS X version number is just set to a
   /// constant 10.4.0 in that case.  Returns true if successful.
-  bool getMacOSXVersion(VersionTuple &Version) const;
+  LLVM_ABI bool getMacOSXVersion(VersionTuple &Version) const;
 
   /// Parse the version number as with getOSVersion.  This should only be called
   /// with IOS or generic triples.
-  VersionTuple getiOSVersion() const;
+  LLVM_ABI VersionTuple getiOSVersion() const;
 
   /// Parse the version number as with getOSVersion.  This should only be called
   /// with WatchOS or generic triples.
-  VersionTuple getWatchOSVersion() const;
+  LLVM_ABI VersionTuple getWatchOSVersion() const;
 
   /// Parse the version number as with getOSVersion.
-  VersionTuple getDriverKitVersion() const;
+  LLVM_ABI VersionTuple getDriverKitVersion() const;
 
   /// Parse the Vulkan version number from the OSVersion and SPIR-V version
   /// (SubArch).  This should only be called with Vulkan SPIR-V triples.
-  VersionTuple getVulkanVersion() const;
+  LLVM_ABI VersionTuple getVulkanVersion() const;
 
   /// Parse the DXIL version number from the OSVersion and DXIL version
   /// (SubArch).  This should only be called with DXIL triples.
-  VersionTuple getDXILVersion() const;
+  LLVM_ABI VersionTuple getDXILVersion() const;
 
   /// @}
   /// @name Direct Component Access
@@ -469,34 +471,34 @@ class Triple {
   bool empty() const { return Data.empty(); }
 
   /// Get the architecture (first) component of the triple.
-  StringRef getArchName() const;
+  LLVM_ABI StringRef getArchName() const;
 
   /// Get the vendor (second) component of the triple.
-  StringRef getVendorName() const;
+  LLVM_ABI StringRef getVendorName() const;
 
   /// Get the operating system (third) component of the triple.
-  StringRef getOSName() const;
+  LLVM_ABI StringRef getOSName() const;
 
   /// Get the optional environment (fourth) component of the triple, or "" if
   /// empty.
-  StringRef getEnvironmentName() const;
+  LLVM_ABI StringRef getEnvironmentName() const;
 
   /// Get the operating system and optional environment components as a single
   /// string (separated by a '-' if the environment component is present).
-  StringRef getOSAndEnvironmentName() const;
+  LLVM_ABI StringRef getOSAndEnvironmentName() const;
 
   /// Get the version component of the environment component as a single
   /// string (the version after the environment).
   ///
   /// For example, "fooos1.2.3" would return "1.2.3".
-  StringRef getEnvironmentVersionString() const;
+  LLVM_ABI StringRef getEnvironmentVersionString() const;
 
   /// @}
   /// @name Convenience Predicates
   /// @{
 
   /// Returns the pointer width of this architecture.
-  static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch);
+  LLVM_ABI static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch);
 
   /// Returns the pointer width of this architecture.
   unsigned getArchPointerBitWidth() const {
@@ -504,7 +506,7 @@ class Triple {
   }
 
   /// Returns the trampoline size in bytes for this configuration.
-  unsigned getTrampolineSize() const;
+  LLVM_ABI unsigned getTrampolineSize() const;
 
   /// Test whether the architecture is 64-bit
   ///
@@ -513,17 +515,17 @@ class Triple {
   /// 16-bit. The inner details of pointer width for particular architectures
   /// is not summed up in the triple, and so only a coarse grained predicate
   /// system is provided.
-  bool isArch64Bit() const;
+  LLVM_ABI bool isArch64Bit() const;
 
   /// Test whether the architecture is 32-bit
   ///
   /// Note that this tests for 32-bit pointer width, and nothing else.
-  bool isArch32Bit() const;
+  LLVM_ABI bool isArch32Bit() const;
 
   /// Test whether the architecture is 16-bit
   ///
   /// Note that this tests for 16-bit pointer width, and nothing else.
-  bool isArch16Bit() const;
+  LLVM_ABI bool isArch16Bit() const;
 
   /// Helper function for doing comparisons against version numbers included in
   /// the target triple.
@@ -544,8 +546,8 @@ class Triple {
 
   /// Comparison function for checking OS X version compatibility, which handles
   /// supporting skewed version numbering schemes used by the "darwin" triples.
-  bool isMacOSXVersionLT(unsigned Major, unsigned Minor = 0,
-                         unsigned Micro = 0) const;
+  LLVM_ABI bool isMacOSXVersionLT(unsigned Major, unsigned Minor = 0,
+                                  unsigned Micro = 0) const;
 
   /// Is this a Mac OS X triple. For legacy reasons, we support both "darwin"
   /// and "osx" as OS X triples.
@@ -917,6 +919,34 @@ class Triple {
            isOSBinFormatELF();
   }
 
+  // ARM EABI is the bare-metal EABI described in ARM ABI documents and
+  // can be accessed via -target arm-none-eabi. This is NOT GNUEABI.
+  // FIXME: Add a flag for bare-metal for that target and set Triple::EABI
+  // even for GNUEABI, so we can make a distinction here and still conform to
+  // the EABI on GNU (and Android) mode. This requires change in Clang, too.
+  // FIXME: The Darwin exception is temporary, while we move users to
+  // "*-*-*-macho" triples as quickly as possible.
+  bool isTargetAEABI() const {
+    return (getEnvironment() == Triple::EABI ||
+            getEnvironment() == Triple::EABIHF) &&
+           !isOSDarwin() && !isOSWindows();
+  }
+
+  bool isTargetGNUAEABI() const {
+    return (getEnvironment() == Triple::GNUEABI ||
+            getEnvironment() == Triple::GNUEABIT64 ||
+            getEnvironment() == Triple::GNUEABIHF ||
+            getEnvironment() == Triple::GNUEABIHFT64) &&
+           !isOSDarwin() && !isOSWindows();
+  }
+
+  bool isTargetMuslAEABI() const {
+    return (getEnvironment() == Triple::MuslEABI ||
+            getEnvironment() == Triple::MuslEABIHF ||
+            getEnvironment() == Triple::OpenHOS) &&
+           !isOSDarwin() && !isOSWindows();
+  }
+
   /// Tests whether the target is T32.
   bool isArmT32() const {
     switch (getSubArch()) {
@@ -1143,38 +1173,38 @@ class Triple {
   /// @{
 
   /// Set the architecture (first) component of the triple to a known type.
-  void setArch(ArchType Kind, SubArchType SubArch = NoSubArch);
+  LLVM_ABI void setArch(ArchType Kind, SubArchType SubArch = NoSubArch);
 
   /// Set the vendor (second) component of the triple to a known type.
-  void setVendor(VendorType Kind);
+  LLVM_ABI void setVendor(VendorType Kind);
 
   /// Set the operating system (third) component of the triple to a known type.
-  void setOS(OSType Kind);
+  LLVM_ABI void setOS(OSType Kind);
 
   /// Set the environment (fourth) component of the triple to a known type.
-  void setEnvironment(EnvironmentType Kind);
+  LLVM_ABI void setEnvironment(EnvironmentType Kind);
 
   /// Set the object file format.
-  void setObjectFormat(ObjectFormatType Kind);
+  LLVM_ABI void setObjectFormat(ObjectFormatType Kind);
 
   /// Set all components to the new triple \p Str.
-  void setTriple(const Twine &Str);
+  LLVM_ABI void setTriple(const Twine &Str);
 
   /// Set the architecture (first) component of the triple by name.
-  void setArchName(StringRef Str);
+  LLVM_ABI void setArchName(StringRef Str);
 
   /// Set the vendor (second) component of the triple by name.
-  void setVendorName(StringRef Str);
+  LLVM_ABI void setVendorName(StringRef Str);
 
   /// Set the operating system (third) component of the triple by name.
-  void setOSName(StringRef Str);
+  LLVM_ABI void setOSName(StringRef Str);
 
   /// Set the optional environment (fourth) component of the triple by name.
-  void setEnvironmentName(StringRef Str);
+  LLVM_ABI void setEnvironmentName(StringRef Str);
 
   /// Set the operating system and optional environment components with a single
   /// string.
-  void setOSAndEnvironmentName(StringRef Str);
+  LLVM_ABI void setOSAndEnvironmentName(StringRef Str);
 
   /// @}
   /// @name Helpers to build variants of a particular triple.
@@ -1186,7 +1216,7 @@ class Triple {
   ///
   /// \returns A new triple with a 32-bit architecture or an unknown
   ///          architecture if no such variant can be found.
-  llvm::Triple get32BitArchVariant() const;
+  LLVM_ABI llvm::Triple get32BitArchVariant() const;
 
   /// Form a triple with a 64-bit variant of the current architecture.
   ///
@@ -1194,7 +1224,7 @@ class Triple {
   ///
   /// \returns A new triple with a 64-bit architecture or an unknown
   ///          architecture if no such variant can be found.
-  llvm::Triple get64BitArchVariant() const;
+  LLVM_ABI llvm::Triple get64BitArchVariant() const;
 
   /// Form a triple with a big endian variant of the current architecture.
   ///
@@ -1202,7 +1232,7 @@ class Triple {
   ///
   /// \returns A new triple with a big endian architecture or an unknown
   ///          architecture if no such variant can be found.
-  llvm::Triple getBigEndianArchVariant() const;
+  LLVM_ABI llvm::Triple getBigEndianArchVariant() const;
 
   /// Form a triple with a little endian variant of the current architecture.
   ///
@@ -1210,69 +1240,76 @@ class Triple {
   ///
   /// \returns A new triple with a little endian architecture or an unknown
   ///          architecture if no such variant can be found.
-  llvm::Triple getLittleEndianArchVariant() const;
+  LLVM_ABI llvm::Triple getLittleEndianArchVariant() const;
 
   /// Tests whether the target triple is little endian.
   ///
   /// \returns true if the triple is little endian, false otherwise.
-  bool isLittleEndian() const;
+  LLVM_ABI bool isLittleEndian() const;
 
   /// Test whether target triples are compatible.
-  bool isCompatibleWith(const Triple &Other) const;
+  LLVM_ABI bool isCompatibleWith(const Triple &Other) const;
 
   /// Test whether the target triple is for a GPU.
   bool isGPU() const { return isSPIRV() || isNVPTX() || isAMDGPU(); }
 
   /// Merge target triples.
-  std::string merge(const Triple &Other) const;
+  LLVM_ABI std::string merge(const Triple &Other) const;
 
   /// Some platforms have different minimum supported OS versions that
   /// varies by the architecture specified in the triple. This function
   /// returns the minimum supported OS version for this triple if one an exists,
   /// or an invalid version tuple if this triple doesn't have one.
-  VersionTuple getMinimumSupportedOSVersion() const;
+  LLVM_ABI VersionTuple getMinimumSupportedOSVersion() const;
 
   /// @}
   /// @name Static helpers for IDs.
   /// @{
 
   /// Get the canonical name for the \p Kind architecture.
-  static StringRef getArchTypeName(ArchType Kind);
+  LLVM_ABI static StringRef getArchTypeName(ArchType Kind);
 
   /// Get the architecture name based on \p Kind and \p SubArch.
-  static StringRef getArchName(ArchType Kind, SubArchType SubArch = NoSubArch);
+  LLVM_ABI static StringRef getArchName(ArchType Kind,
+                                        SubArchType SubArch = NoSubArch);
 
   /// Get the "prefix" canonical name for the \p Kind architecture. This is the
   /// prefix used by the architecture specific builtins, and is suitable for
   /// passing to \see Intrinsic::getIntrinsicForClangBuiltin().
   ///
   /// \return - The architecture prefix, or 0 if none is defined.
-  static StringRef getArchTypePrefix(ArchType Kind);
+  LLVM_ABI static StringRef getArchTypePrefix(ArchType Kind);
 
   /// Get the canonical name for the \p Kind vendor.
-  static StringRef getVendorTypeName(VendorType Kind);
+  LLVM_ABI static StringRef getVendorTypeName(VendorType Kind);
 
   /// Get the canonical name for the \p Kind operating system.
-  static StringRef getOSTypeName(OSType Kind);
+  LLVM_ABI static StringRef getOSTypeName(OSType Kind);
 
   /// Get the canonical name for the \p Kind environment.
-  static StringRef getEnvironmentTypeName(EnvironmentType Kind);
+  LLVM_ABI static StringRef getEnvironmentTypeName(EnvironmentType Kind);
 
   /// Get the name for the \p Object format.
-  static StringRef getObjectFormatTypeName(ObjectFormatType ObjectFormat);
+  LLVM_ABI static StringRef
+  getObjectFormatTypeName(ObjectFormatType ObjectFormat);
 
   /// @}
   /// @name Static helpers for converting alternate architecture names.
   /// @{
 
   /// The canonical type for the given LLVM architecture name (e.g., "x86").
-  static ArchType getArchTypeForLLVMName(StringRef Str);
+  LLVM_ABI static ArchType getArchTypeForLLVMName(StringRef Str);
 
   /// @}
 
   /// Returns a canonicalized OS version number for the specified OS.
-  static VersionTuple getCanonicalVersionForOS(OSType OSKind,
-                                               const VersionTuple &Version);
+  LLVM_ABI static VersionTuple
+  getCanonicalVersionForOS(OSType OSKind, const VersionTuple &Version,
+                           bool IsInValidRange);
+
+  /// Returns whether an OS version is invalid and would not map to an Apple OS.
+  LLVM_ABI static bool isValidVersionForOS(OSType OSKind,
+                                           const VersionTuple &Version);
 };
 
 } // End llvm namespace
diff --git a/external/llvm-project/llvm/include/llvm/TargetParser/X86TargetParser.h b/external/llvm-project/llvm/include/llvm/TargetParser/X86TargetParser.h
index 8447aca7bb92..f6aeaada346e 100644
--- a/external/llvm-project/llvm/include/llvm/TargetParser/X86TargetParser.h
+++ b/external/llvm-project/llvm/include/llvm/TargetParser/X86TargetParser.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Compiler.h"
 #include <array>
 
 namespace llvm {
@@ -153,34 +154,36 @@ enum CPUKind {
 
 /// Parse \p CPU string into a CPUKind. Will only accept 64-bit capable CPUs if
 /// \p Only64Bit is true.
-CPUKind parseArchX86(StringRef CPU, bool Only64Bit = false);
-CPUKind parseTuneCPU(StringRef CPU, bool Only64Bit = false);
+LLVM_ABI CPUKind parseArchX86(StringRef CPU, bool Only64Bit = false);
+LLVM_ABI CPUKind parseTuneCPU(StringRef CPU, bool Only64Bit = false);
 
 /// Provide a list of valid CPU names. If \p Only64Bit is true, the list will
 /// only contain 64-bit capable CPUs.
-void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values,
-                          bool Only64Bit = false);
+LLVM_ABI void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values,
+                                   bool Only64Bit = false);
 /// Provide a list of valid -mtune names.
-void fillValidTuneCPUList(SmallVectorImpl<StringRef> &Values,
-                          bool Only64Bit = false);
+LLVM_ABI void fillValidTuneCPUList(SmallVectorImpl<StringRef> &Values,
+                                   bool Only64Bit = false);
 
 /// Get the key feature prioritizing target multiversioning.
-ProcessorFeatures getKeyFeature(CPUKind Kind);
+LLVM_ABI ProcessorFeatures getKeyFeature(CPUKind Kind);
 
 /// Fill in the features that \p CPU supports into \p Features.
 /// "+" will be append in front of each feature if NeedPlus is true.
-void getFeaturesForCPU(StringRef CPU, SmallVectorImpl<StringRef> &Features,
-                       bool NeedPlus = false);
+LLVM_ABI void getFeaturesForCPU(StringRef CPU,
+                                SmallVectorImpl<StringRef> &Features,
+                                bool NeedPlus = false);
 
 /// Set or clear entries in \p Features that are implied to be enabled/disabled
 /// by the provided \p Feature.
-void updateImpliedFeatures(StringRef Feature, bool Enabled,
-                           StringMap<bool> &Features);
-
-char getCPUDispatchMangling(StringRef Name);
-bool validateCPUSpecificCPUDispatch(StringRef Name);
-std::array<uint32_t, 4> getCpuSupportsMask(ArrayRef<StringRef> FeatureStrs);
-unsigned getFeaturePriority(ProcessorFeatures Feat);
+LLVM_ABI void updateImpliedFeatures(StringRef Feature, bool Enabled,
+                                    StringMap<bool> &Features);
+
+LLVM_ABI char getCPUDispatchMangling(StringRef Name);
+LLVM_ABI bool validateCPUSpecificCPUDispatch(StringRef Name);
+LLVM_ABI std::array<uint32_t, 4>
+getCpuSupportsMask(ArrayRef<StringRef> FeatureStrs);
+LLVM_ABI unsigned getFeaturePriority(ProcessorFeatures Feat);
 
 } // namespace X86
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/Architecture.h b/external/llvm-project/llvm/include/llvm/TextAPI/Architecture.h
index 978359995074..7a7f5416fe7c 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/Architecture.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/Architecture.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_TEXTAPI_ARCHITECTURE_H
 #define LLVM_TEXTAPI_ARCHITECTURE_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <utility>
 
@@ -32,24 +33,26 @@ enum Architecture : uint8_t {
 };
 
 /// Convert a CPU Type and Subtype pair to an architecture slice.
-Architecture getArchitectureFromCpuType(uint32_t CPUType, uint32_t CPUSubType);
+LLVM_ABI Architecture getArchitectureFromCpuType(uint32_t CPUType,
+                                                 uint32_t CPUSubType);
 
 /// Convert a name to an architecture slice.
-Architecture getArchitectureFromName(StringRef Name);
+LLVM_ABI Architecture getArchitectureFromName(StringRef Name);
 
 /// Convert an architecture slice to a string.
-StringRef getArchitectureName(Architecture Arch);
+LLVM_ABI StringRef getArchitectureName(Architecture Arch);
 
 /// Convert an architecture slice to a CPU Type and Subtype pair.
-std::pair<uint32_t, uint32_t> getCPUTypeFromArchitecture(Architecture Arch);
+LLVM_ABI std::pair<uint32_t, uint32_t>
+getCPUTypeFromArchitecture(Architecture Arch);
 
 /// Convert a target to an architecture slice.
-Architecture mapToArchitecture(const llvm::Triple &Target);
+LLVM_ABI Architecture mapToArchitecture(const llvm::Triple &Target);
 
 /// Check if architecture is 64 bit.
-bool is64Bit(Architecture);
+LLVM_ABI bool is64Bit(Architecture);
 
-raw_ostream &operator<<(raw_ostream &OS, Architecture Arch);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, Architecture Arch);
 
 } // end namespace MachO.
 } // end namespace llvm.
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/ArchitectureSet.h b/external/llvm-project/llvm/include/llvm/TextAPI/ArchitectureSet.h
index 2cce9dbf0d80..a7d3394c9982 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/ArchitectureSet.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/ArchitectureSet.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_TEXTAPI_ARCHITECTURESET_H
 #define LLVM_TEXTAPI_ARCHITECTURESET_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/TextAPI/Architecture.h"
 #include <cstddef>
 #include <iterator>
@@ -38,7 +39,7 @@ class ArchitectureSet {
   constexpr ArchitectureSet() = default;
   constexpr ArchitectureSet(ArchSetType Raw) : ArchSet(Raw) {}
   ArchitectureSet(Architecture Arch) : ArchitectureSet() { set(Arch); }
-  ArchitectureSet(const std::vector<Architecture> &Archs);
+  LLVM_ABI ArchitectureSet(const std::vector<Architecture> &Archs);
 
   static ArchitectureSet All() { return ArchitectureSet(EndIndexVal); }
 
@@ -61,7 +62,7 @@ class ArchitectureSet {
     return (ArchSet & Archs.ArchSet) == Archs.ArchSet;
   }
 
-  size_t count() const;
+  LLVM_ABI size_t count() const;
 
   bool empty() const { return ArchSet == 0; }
 
@@ -158,9 +159,9 @@ class ArchitectureSet {
   const_iterator begin() const { return {&ArchSet}; }
   const_iterator end() const { return {&ArchSet, EndIndexVal}; }
 
-  operator std::string() const;
-  operator std::vector<Architecture>() const;
-  void print(raw_ostream &OS) const;
+  LLVM_ABI operator std::string() const;
+  LLVM_ABI operator std::vector<Architecture>() const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 };
 
 inline ArchitectureSet operator|(const Architecture &lhs,
@@ -168,7 +169,7 @@ inline ArchitectureSet operator|(const Architecture &lhs,
   return ArchitectureSet(lhs) | ArchitectureSet(rhs);
 }
 
-raw_ostream &operator<<(raw_ostream &OS, ArchitectureSet Set);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, ArchitectureSet Set);
 
 } // end namespace MachO.
 } // end namespace llvm.
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/DylibReader.h b/external/llvm-project/llvm/include/llvm/TextAPI/DylibReader.h
index 6861d3cb1591..f3a806d78df7 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/DylibReader.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/DylibReader.h
@@ -14,6 +14,7 @@
 #define LLVM_TEXTAPI_DYLIBREADER_H
 
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/TextAPI/ArchitectureSet.h"
@@ -37,20 +38,21 @@ struct ParseOption {
 /// \param Buffer Data that points to dylib.
 /// \param Options Determines which attributes to extract.
 /// \return List of record slices.
-Expected<Records> readFile(MemoryBufferRef Buffer, const ParseOption &Opt);
+LLVM_ABI Expected<Records> readFile(MemoryBufferRef Buffer,
+                                    const ParseOption &Opt);
 
 /// Get TAPI file representation of binary dylib.
 ///
 /// \param Buffer Data that points to dylib.
-Expected<std::unique_ptr<InterfaceFile>> get(MemoryBufferRef Buffer);
+LLVM_ABI Expected<std::unique_ptr<InterfaceFile>> get(MemoryBufferRef Buffer);
 
 using SymbolToSourceLocMap = llvm::StringMap<RecordLoc>;
 /// Get the source location for each symbol from dylib.
 ///
 /// \param DSYM Path to DSYM file.
 /// \param T Requested target slice for dylib.
-SymbolToSourceLocMap accumulateSourceLocFromDSYM(const StringRef DSYM,
-                                                 const Target &T);
+LLVM_ABI SymbolToSourceLocMap accumulateSourceLocFromDSYM(const StringRef DSYM,
+                                                          const Target &T);
 
 } // namespace llvm::MachO::DylibReader
 
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/InterfaceFile.h b/external/llvm-project/llvm/include/llvm/TextAPI/InterfaceFile.h
index 23c27cb0f474..747c8d0a208c 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/InterfaceFile.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/InterfaceFile.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TextAPI/ArchitectureSet.h"
 #include "llvm/TextAPI/FileTypes.h"
 #include "llvm/TextAPI/PackedVersion.h"
@@ -60,7 +61,7 @@ class InterfaceFileRef {
 
   StringRef getInstallName() const { return InstallName; };
 
-  void addTarget(const Target &Target);
+  LLVM_ABI void addTarget(const Target &Target);
   template <typename RangeT> void addTargets(RangeT &&Targets) {
     for (const auto &Target : Targets)
       addTarget(Target(Target));
@@ -146,7 +147,7 @@ class InterfaceFile {
   /// Set and add target.
   ///
   /// \param Target the target to add into.
-  void addTarget(const Target &Target);
+  LLVM_ABI void addTarget(const Target &Target);
 
   /// Determine if target triple slice exists in file.
   ///
@@ -174,7 +175,7 @@ class InterfaceFile {
                             std::function<bool(const Target &)>>;
   using const_filtered_target_range =
       llvm::iterator_range<const_filtered_target_iterator>;
-  const_filtered_target_range targets(ArchitectureSet Archs) const;
+  LLVM_ABI const_filtered_target_range targets(ArchitectureSet Archs) const;
 
   /// Set the install name of the library.
   void setInstallName(StringRef InstallName_) {
@@ -241,7 +242,7 @@ class InterfaceFile {
   /// Set the parent umbrella frameworks.
   /// \param Target_ The target applicable to Parent
   /// \param Parent  The name of Parent
-  void addParentUmbrella(const Target &Target_, StringRef Parent);
+  LLVM_ABI void addParentUmbrella(const Target &Target_, StringRef Parent);
 
   /// Get the list of Parent Umbrella frameworks.
   ///
@@ -261,7 +262,7 @@ class InterfaceFile {
   /// \param InstallName The name of the client that is allowed to link this
   /// library.
   /// \param Target The target triple for which this applies.
-  void addAllowableClient(StringRef InstallName, const Target &Target);
+  LLVM_ABI void addAllowableClient(StringRef InstallName, const Target &Target);
 
   /// Get the list of allowable clients.
   ///
@@ -274,7 +275,8 @@ class InterfaceFile {
   ///
   /// \param InstallName The name of the library to re-export.
   /// \param Target The target triple for which this applies.
-  void addReexportedLibrary(StringRef InstallName, const Target &Target);
+  LLVM_ABI void addReexportedLibrary(StringRef InstallName,
+                                     const Target &Target);
 
   /// Get the list of re-exported libraries.
   ///
@@ -286,7 +288,7 @@ class InterfaceFile {
   /// Add a library for inlining to top level library.
   ///
   ///\param Document The library to inline with top level library.
-  void addDocument(std::shared_ptr<InterfaceFile> &&Document);
+  LLVM_ABI void addDocument(std::shared_ptr<InterfaceFile> &&Document);
 
   /// Returns the pointer to parent document if exists or nullptr otherwise.
   InterfaceFile *getParent() const { return Parent; }
@@ -301,7 +303,7 @@ class InterfaceFile {
   /// Set the runpath search paths.
   /// \param RPath The name of runpath.
   /// \param InputTarget The target applicable to runpath search path.
-  void addRPath(StringRef RPath, const Target &InputTarget);
+  LLVM_ABI void addRPath(StringRef RPath, const Target &InputTarget);
 
   /// Get the list of runpath search paths.
   ///
@@ -373,14 +375,14 @@ class InterfaceFile {
   ///
   /// \param Arch architecture to extract from.
   /// \return New InterfaceFile with extracted architecture slice.
-  llvm::Expected<std::unique_ptr<InterfaceFile>>
+  LLVM_ABI llvm::Expected<std::unique_ptr<InterfaceFile>>
   extract(Architecture Arch) const;
 
   /// Remove architecture slice from Interface.
   ///
   /// \param Arch architecture to remove.
   /// \return New Interface File with removed architecture slice.
-  llvm::Expected<std::unique_ptr<InterfaceFile>>
+  LLVM_ABI llvm::Expected<std::unique_ptr<InterfaceFile>>
   remove(Architecture Arch) const;
 
   /// Merge Interfaces for the same library. The following library attributes
@@ -390,29 +392,29 @@ class InterfaceFile {
   ///
   /// \param O The Interface to merge.
   /// \return New Interface File that was merged.
-  llvm::Expected<std::unique_ptr<InterfaceFile>>
+  LLVM_ABI llvm::Expected<std::unique_ptr<InterfaceFile>>
   merge(const InterfaceFile *O) const;
 
   /// Inline reexported library into Interface.
   ///
   /// \param Library Interface of reexported library.
   /// \param Overwrite Whether to overwrite preexisting inlined library.
-  void inlineLibrary(std::shared_ptr<InterfaceFile> Library,
-                     bool Overwrite = false);
+  LLVM_ABI void inlineLibrary(std::shared_ptr<InterfaceFile> Library,
+                              bool Overwrite = false);
 
   /// Set InterfaceFile properties from pre-gathered binary attributes,
   /// if they are not set already.
   ///
   /// \param BA Attributes typically represented in load commands.
   /// \param Targ MachO Target slice to add attributes to.
-  void setFromBinaryAttrs(const RecordsSlice::BinaryAttrs &BA,
-                          const Target &Targ);
+  LLVM_ABI void setFromBinaryAttrs(const RecordsSlice::BinaryAttrs &BA,
+                                   const Target &Targ);
 
   /// The equality is determined by attributes that impact linking
   /// compatibilities. Path, & FileKind are irrelevant since these by
   /// itself should not impact linking.
   /// This is an expensive operation.
-  bool operator==(const InterfaceFile &O) const;
+  LLVM_ABI bool operator==(const InterfaceFile &O) const;
 
   bool operator!=(const InterfaceFile &O) const { return !(*this == O); }
 
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/PackedVersion.h b/external/llvm-project/llvm/include/llvm/TextAPI/PackedVersion.h
index e680d40c7104..cabe365e6d97 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/PackedVersion.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/PackedVersion.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_TEXTAPI_PACKEDVERSION_H
 #define LLVM_TEXTAPI_PACKEDVERSION_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
 #include <cstdint>
 #include <string>
@@ -53,8 +54,8 @@ class PackedVersion {
   /// Retrieve the subminor version number, if provided.
   unsigned getSubminor() const { return Version & 0xff; }
 
-  bool parse32(StringRef Str);
-  std::pair<bool, bool> parse64(StringRef Str);
+  LLVM_ABI bool parse32(StringRef Str);
+  LLVM_ABI std::pair<bool, bool> parse64(StringRef Str);
 
   bool operator<(const PackedVersion &O) const { return Version < O.Version; }
 
@@ -64,9 +65,9 @@ class PackedVersion {
 
   uint32_t rawValue() const { return Version; }
 
-  operator std::string() const;
+  LLVM_ABI operator std::string() const;
 
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const PackedVersion &Version) {
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/Platform.h b/external/llvm-project/llvm/include/llvm/TextAPI/Platform.h
index d828d9ac49f6..8ea187acc02f 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/Platform.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/Platform.h
@@ -14,6 +14,7 @@
 
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
 
 namespace llvm {
@@ -22,14 +23,14 @@ namespace MachO {
 using PlatformSet = SmallSet<PlatformType, 3>;
 using PlatformVersionSet = SmallSet<std::pair<PlatformType, VersionTuple>, 3>;
 
-PlatformType mapToPlatformType(PlatformType Platform, bool WantSim);
-PlatformType mapToPlatformType(const Triple &Target);
-PlatformSet mapToPlatformSet(ArrayRef<Triple> Targets);
-StringRef getPlatformName(PlatformType Platform);
-PlatformType getPlatformFromName(StringRef Name);
-std::string getOSAndEnvironmentName(PlatformType Platform,
-                                    std::string Version = "");
-VersionTuple mapToSupportedOSVersion(const Triple &Triple);
+LLVM_ABI PlatformType mapToPlatformType(PlatformType Platform, bool WantSim);
+LLVM_ABI PlatformType mapToPlatformType(const Triple &Target);
+LLVM_ABI PlatformSet mapToPlatformSet(ArrayRef<Triple> Targets);
+LLVM_ABI StringRef getPlatformName(PlatformType Platform);
+LLVM_ABI PlatformType getPlatformFromName(StringRef Name);
+LLVM_ABI std::string getOSAndEnvironmentName(PlatformType Platform,
+                                             std::string Version = "");
+LLVM_ABI VersionTuple mapToSupportedOSVersion(const Triple &Triple);
 
 } // end namespace MachO.
 } // end namespace llvm.
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/Record.h b/external/llvm-project/llvm/include/llvm/TextAPI/Record.h
index 7d721988ec3d..6e470d97325f 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/Record.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/Record.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TextAPI/Symbol.h"
 #include <string>
 
@@ -104,7 +105,7 @@ class Record {
   SymbolFlags getFlags() const { return Flags; }
 
 private:
-  SymbolFlags mergeFlags(SymbolFlags Flags, RecordLinkage Linkage);
+  LLVM_ABI SymbolFlags mergeFlags(SymbolFlags Flags, RecordLinkage Linkage);
 
 protected:
   StringRef Name;
@@ -164,9 +165,9 @@ class ObjCContainerRecord : public Record {
   ObjCContainerRecord(StringRef Name, RecordLinkage Linkage)
       : Record({Name, Linkage, SymbolFlags::Data}) {}
 
-  ObjCIVarRecord *addObjCIVar(StringRef IVar, RecordLinkage Linkage);
-  ObjCIVarRecord *findObjCIVar(StringRef IVar) const;
-  std::vector<ObjCIVarRecord *> getObjCIVars() const;
+  LLVM_ABI ObjCIVarRecord *addObjCIVar(StringRef IVar, RecordLinkage Linkage);
+  LLVM_ABI ObjCIVarRecord *findObjCIVar(StringRef IVar) const;
+  LLVM_ABI std::vector<ObjCIVarRecord *> getObjCIVars() const;
   RecordLinkage getLinkage() const { return Linkage; }
 
 private:
@@ -207,11 +208,12 @@ class ObjCInterfaceRecord : public ObjCContainerRecord {
     return getLinkageForSymbol(CurrType) >= RecordLinkage::Rexported;
   }
 
-  RecordLinkage getLinkageForSymbol(ObjCIFSymbolKind CurrType) const;
-  void updateLinkageForSymbols(ObjCIFSymbolKind SymType, RecordLinkage Link);
+  LLVM_ABI RecordLinkage getLinkageForSymbol(ObjCIFSymbolKind CurrType) const;
+  LLVM_ABI void updateLinkageForSymbols(ObjCIFSymbolKind SymType,
+                                        RecordLinkage Link);
 
-  bool addObjCCategory(ObjCCategoryRecord *Record);
-  std::vector<ObjCCategoryRecord *> getObjCCategories() const;
+  LLVM_ABI bool addObjCCategory(ObjCCategoryRecord *Record);
+  LLVM_ABI std::vector<ObjCCategoryRecord *> getObjCCategories() const;
 
 private:
   /// Linkage level for each symbol represented in ObjCInterfaceRecord.
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/RecordVisitor.h b/external/llvm-project/llvm/include/llvm/TextAPI/RecordVisitor.h
index 34e43f5b0027..65bc96df244d 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/RecordVisitor.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/RecordVisitor.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_TEXTAPI_RECORDVISITOR_H
 #define LLVM_TEXTAPI_RECORDVISITOR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/TextAPI/Record.h"
 #include "llvm/TextAPI/SymbolSet.h"
 
@@ -20,7 +21,7 @@ namespace llvm {
 namespace MachO {
 
 /// Base class for any usage of traversing over collected Records.
-class RecordVisitor {
+class LLVM_ABI RecordVisitor {
 public:
   virtual ~RecordVisitor();
 
@@ -32,7 +33,7 @@ class RecordVisitor {
 /// Specialized RecordVisitor for collecting exported symbols
 /// and undefined symbols if RecordSlice being visited represents a
 /// flat-namespaced library.
-class SymbolConverter : public RecordVisitor {
+class LLVM_ABI SymbolConverter : public RecordVisitor {
 public:
   SymbolConverter(SymbolSet *Symbols, const Target &T,
                   const bool RecordUndefs = false)
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/RecordsSlice.h b/external/llvm-project/llvm/include/llvm/TextAPI/RecordsSlice.h
index f934cf7607f1..6ecb79a115ae 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/RecordsSlice.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/RecordsSlice.h
@@ -15,6 +15,7 @@
 #define LLVM_TEXTAPI_RECORDSLICE_H
 
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TextAPI/FileTypes.h"
 #include "llvm/TextAPI/PackedVersion.h"
 #include "llvm/TextAPI/Record.h"
@@ -43,9 +44,10 @@ class RecordsSlice {
   /// symbol.
   /// \param Linkage The linkage of symbol.
   /// \return The non-owning pointer to added record in slice.
-  Record *addRecord(StringRef Name, SymbolFlags Flags,
-                    GlobalRecord::Kind GV = GlobalRecord::Kind::Unknown,
-                    RecordLinkage Linkage = RecordLinkage::Unknown);
+  LLVM_ABI Record *
+  addRecord(StringRef Name, SymbolFlags Flags,
+            GlobalRecord::Kind GV = GlobalRecord::Kind::Unknown,
+            RecordLinkage Linkage = RecordLinkage::Unknown);
 
   /// Add non-ObjC global record.
   ///
@@ -56,10 +58,10 @@ class RecordsSlice {
   /// \param Inlined Whether declaration is inlined, only applicable to
   /// functions.
   /// \return The non-owning pointer to added record in slice.
-  GlobalRecord *addGlobal(StringRef Name, RecordLinkage Linkage,
-                          GlobalRecord::Kind GV,
-                          SymbolFlags Flags = SymbolFlags::None,
-                          bool Inlined = false);
+  LLVM_ABI GlobalRecord *addGlobal(StringRef Name, RecordLinkage Linkage,
+                                   GlobalRecord::Kind GV,
+                                   SymbolFlags Flags = SymbolFlags::None,
+                                   bool Inlined = false);
 
   /// Add ObjC Class record.
   ///
@@ -67,8 +69,9 @@ class RecordsSlice {
   /// \param Linkage The linkage of symbol.
   /// \param SymType The symbols this class represents.
   /// \return The non-owning pointer to added record in slice.
-  ObjCInterfaceRecord *addObjCInterface(StringRef Name, RecordLinkage Linkage,
-                                        ObjCIFSymbolKind SymType);
+  LLVM_ABI ObjCInterfaceRecord *addObjCInterface(StringRef Name,
+                                                 RecordLinkage Linkage,
+                                                 ObjCIFSymbolKind SymType);
 
   /// Add ObjC IVar record.
   ///
@@ -76,8 +79,8 @@ class RecordsSlice {
   /// \param Name The name of ivar, not symbol.
   /// \param Linkage The linkage of symbol.
   /// \return The non-owning pointer to added record in slice.
-  ObjCIVarRecord *addObjCIVar(ObjCContainerRecord *Container, StringRef Name,
-                              RecordLinkage Linkage);
+  LLVM_ABI ObjCIVarRecord *addObjCIVar(ObjCContainerRecord *Container,
+                                       StringRef Name, RecordLinkage Linkage);
 
   /// Add ObjC Category record.
   ///
@@ -85,22 +88,22 @@ class RecordsSlice {
   /// category, not symbol.
   /// \param Category The name of category.
   /// \return The non-owning pointer to added record in slice.
-  ObjCCategoryRecord *addObjCCategory(StringRef ClassToExtend,
-                                      StringRef Category);
+  LLVM_ABI ObjCCategoryRecord *addObjCCategory(StringRef ClassToExtend,
+                                               StringRef Category);
 
   /// Find ObjC Class.
   ///
   /// \param Name name of class, not full symbol name.
   /// \return The non-owning pointer to record in slice.
-  ObjCInterfaceRecord *findObjCInterface(StringRef Name) const;
+  LLVM_ABI ObjCInterfaceRecord *findObjCInterface(StringRef Name) const;
 
   /// Find ObjC Category.
   ///
   /// \param ClassToExtend The name of class, not full symbol name.
   /// \param Category The name of category.
   /// \return The non-owning pointer to record in slice.
-  ObjCCategoryRecord *findObjCCategory(StringRef ClassToExtend,
-                                       StringRef Category) const;
+  LLVM_ABI ObjCCategoryRecord *findObjCCategory(StringRef ClassToExtend,
+                                                StringRef Category) const;
 
   /// Find ObjC Container. This is commonly used for assigning for looking up
   /// instance variables that are assigned to either a category or class.
@@ -110,21 +113,23 @@ class RecordsSlice {
   /// \param Name Either the name of ivar or name of container.
   /// \return The non-owning pointer to record in
   /// slice.
-  ObjCContainerRecord *findContainer(bool IsIVar, StringRef Name) const;
+  LLVM_ABI ObjCContainerRecord *findContainer(bool IsIVar,
+                                              StringRef Name) const;
 
   /// Find ObjC instance variable.
   ///
   /// \param IsScopedName This is used to determine how to parse the name.
   /// \param Name Either the full name of the symbol or just the ivar.
   /// \return The non-owning pointer to record in slice.
-  ObjCIVarRecord *findObjCIVar(bool IsScopedName, StringRef Name) const;
+  LLVM_ABI ObjCIVarRecord *findObjCIVar(bool IsScopedName,
+                                        StringRef Name) const;
 
   /// Find non-objc global.
   ///
   /// \param Name The name of symbol.
   /// \param GV The Kind of global to find.
   /// \return The non-owning pointer to record in slice.
-  GlobalRecord *
+  LLVM_ABI GlobalRecord *
   findGlobal(StringRef Name,
              GlobalRecord::Kind GV = GlobalRecord::Kind::Unknown) const;
 
@@ -138,7 +143,7 @@ class RecordsSlice {
   }
 
   // Visit all records known to RecordsSlice.
-  void visit(RecordVisitor &V) const;
+  LLVM_ABI void visit(RecordVisitor &V) const;
 
   struct BinaryAttrs {
     std::vector<StringRef> AllowableClients;
@@ -158,11 +163,11 @@ class RecordsSlice {
   };
 
   /// Return reference to BinaryAttrs.
-  BinaryAttrs &getBinaryAttrs();
+  LLVM_ABI BinaryAttrs &getBinaryAttrs();
 
   /// Store any strings owned by RecordSlice into allocator and return back
   /// reference to that.
-  StringRef copyString(StringRef String);
+  LLVM_ABI StringRef copyString(StringRef String);
 
 private:
   const llvm::Triple TargetTriple;
@@ -196,7 +201,8 @@ class RecordsSlice {
 
 using Records = llvm::SmallVector<std::shared_ptr<RecordsSlice>, 4>;
 class InterfaceFile;
-std::unique_ptr<InterfaceFile> convertToInterfaceFile(const Records &Slices);
+LLVM_ABI std::unique_ptr<InterfaceFile>
+convertToInterfaceFile(const Records &Slices);
 
 } // namespace MachO
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/Symbol.h b/external/llvm-project/llvm/include/llvm/TextAPI/Symbol.h
index 5a5eb0eb4832..92ff0746f799 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/Symbol.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/Symbol.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TextAPI/ArchitectureSet.h"
 #include "llvm/TextAPI/Target.h"
@@ -152,14 +153,15 @@ class Symbol {
                             std::function<bool(const Target &)>>;
   using const_filtered_target_range =
       llvm::iterator_range<const_filtered_target_iterator>;
-  const_filtered_target_range targets(ArchitectureSet architectures) const;
+  LLVM_ABI const_filtered_target_range
+  targets(ArchitectureSet architectures) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump(raw_ostream &OS) const;
   void dump() const { dump(llvm::errs()); }
 #endif
 
-  bool operator==(const Symbol &O) const;
+  LLVM_ABI bool operator==(const Symbol &O) const;
 
   bool operator!=(const Symbol &O) const { return !(*this == O); }
 
@@ -189,7 +191,7 @@ struct SimpleSymbol {
 /// Get symbol classification by parsing the name of a symbol.
 ///
 /// \param SymName The name of symbol.
-SimpleSymbol parseSymbol(StringRef SymName);
+LLVM_ABI SimpleSymbol parseSymbol(StringRef SymName);
 
 } // end namespace MachO.
 } // end namespace llvm.
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/SymbolSet.h b/external/llvm-project/llvm/include/llvm/TextAPI/SymbolSet.h
index 6ccabb907720..cd3066317f3a 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/SymbolSet.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/SymbolSet.h
@@ -15,6 +15,7 @@
 #include "llvm/ADT/iterator.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TextAPI/Architecture.h"
 #include "llvm/TextAPI/ArchitectureSet.h"
 #include "llvm/TextAPI/Symbol.h"
@@ -87,12 +88,12 @@ class SymbolSet {
   using SymbolsMapType = llvm::DenseMap<SymbolsMapKey, Symbol *>;
   SymbolsMapType Symbols;
 
-  Symbol *addGlobalImpl(EncodeKind, StringRef Name, SymbolFlags Flags);
+  LLVM_ABI Symbol *addGlobalImpl(EncodeKind, StringRef Name, SymbolFlags Flags);
 
 public:
   SymbolSet() = default;
-  Symbol *addGlobal(EncodeKind Kind, StringRef Name, SymbolFlags Flags,
-                    const Target &Targ);
+  LLVM_ABI Symbol *addGlobal(EncodeKind Kind, StringRef Name, SymbolFlags Flags,
+                             const Target &Targ);
   size_t size() const { return Symbols.size(); }
 
   template <typename RangeT, typename ElT = std::remove_reference_t<
@@ -107,7 +108,7 @@ class SymbolSet {
     return Global;
   }
 
-  const Symbol *
+  LLVM_ABI const Symbol *
   findSymbol(EncodeKind Kind, StringRef Name,
              ObjCIFSymbolKind ObjCIF = ObjCIFSymbolKind::None) const;
 
@@ -169,7 +170,7 @@ class SymbolSet {
         fn);
   }
 
-  bool operator==(const SymbolSet &O) const;
+  LLVM_ABI bool operator==(const SymbolSet &O) const;
 
   bool operator!=(const SymbolSet &O) const { return !(Symbols == O.Symbols); }
 
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/Target.h b/external/llvm-project/llvm/include/llvm/TextAPI/Target.h
index edcc0708d147..5cc507fe21f7 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/Target.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/Target.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_TEXTAPI_TARGET_H
 #define LLVM_TEXTAPI_TARGET_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TargetParser/Triple.h"
@@ -35,9 +36,9 @@ class Target {
       : Arch(mapToArchitecture(Triple)), Platform(mapToPlatformType(Triple)),
         MinDeployment(mapToSupportedOSVersion(Triple)) {}
 
-  static llvm::Expected<Target> create(StringRef Target);
+  LLVM_ABI static llvm::Expected<Target> create(StringRef Target);
 
-  operator std::string() const;
+  LLVM_ABI operator std::string() const;
 
   Architecture Arch;
   PlatformType Platform;
@@ -66,13 +67,13 @@ inline bool operator!=(const Target &LHS, const Architecture &RHS) {
   return LHS.Arch != RHS;
 }
 
-PlatformVersionSet mapToPlatformVersionSet(ArrayRef<Target> Targets);
-PlatformSet mapToPlatformSet(ArrayRef<Target> Targets);
-ArchitectureSet mapToArchitectureSet(ArrayRef<Target> Targets);
+LLVM_ABI PlatformVersionSet mapToPlatformVersionSet(ArrayRef<Target> Targets);
+LLVM_ABI PlatformSet mapToPlatformSet(ArrayRef<Target> Targets);
+LLVM_ABI ArchitectureSet mapToArchitectureSet(ArrayRef<Target> Targets);
 
-std::string getTargetTripleName(const Target &Targ);
+LLVM_ABI std::string getTargetTripleName(const Target &Targ);
 
-raw_ostream &operator<<(raw_ostream &OS, const Target &Target);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const Target &Target);
 
 } // namespace MachO
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIError.h b/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIError.h
index f0578654697b..7b2182edd621 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIError.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIError.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TEXTAPI_TEXTAPIERROR_H
 #define LLVM_TEXTAPI_TEXTAPIERROR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm::MachO {
@@ -25,7 +26,7 @@ enum class TextAPIErrorCode {
   UnsupportedTarget
 };
 
-class TextAPIError : public llvm::ErrorInfo<TextAPIError> {
+class LLVM_ABI TextAPIError : public llvm::ErrorInfo<TextAPIError> {
 public:
   static char ID;
   TextAPIErrorCode EC;
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIReader.h b/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIReader.h
index 32af0e3601f1..603b24b47283 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIReader.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIReader.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_TEXTAPI_TEXTAPIREADER_H
 #define LLVM_TEXTAPI_TEXTAPIREADER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -29,13 +30,13 @@ class TextAPIReader {
   ///
   /// \param InputBuffer Buffer holding contents of TAPI text file.
   /// \return The file format version of TAPI text file.
-  static Expected<FileType> canRead(MemoryBufferRef InputBuffer);
+  LLVM_ABI static Expected<FileType> canRead(MemoryBufferRef InputBuffer);
 
   /// Parse and get an InterfaceFile that represents the full
   /// library.
   ///
   /// \param InputBuffer Buffer holding contents of TAPI text file.
-  static Expected<std::unique_ptr<InterfaceFile>>
+  LLVM_ABI static Expected<std::unique_ptr<InterfaceFile>>
   get(MemoryBufferRef InputBuffer);
 
   TextAPIReader() = delete;
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIWriter.h b/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIWriter.h
index 7fd32c6fe2a9..5f06c372fe85 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIWriter.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/TextAPIWriter.h
@@ -10,6 +10,7 @@
 #define LLVM_TEXTAPI_TEXTAPIWRITER_H
 
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TextAPI/InterfaceFile.h"
 
 namespace llvm {
@@ -30,9 +31,10 @@ class TextAPIWriter {
   /// \param FileKind File format to write text file as. If not specified, it
   /// will read from File.
   /// \param Compact Whether to limit whitespace in text file.
-  static Error writeToStream(raw_ostream &OS, const InterfaceFile &File,
-                             const FileType FileKind = FileType::Invalid,
-                             bool Compact = false);
+  LLVM_ABI static Error
+  writeToStream(raw_ostream &OS, const InterfaceFile &File,
+                const FileType FileKind = FileType::Invalid,
+                bool Compact = false);
 
   /// Get TAPI FileType from the input string.
   ///
diff --git a/external/llvm-project/llvm/include/llvm/TextAPI/Utils.h b/external/llvm-project/llvm/include/llvm/TextAPI/Utils.h
index 00dfd63e14f9..27db717f5a63 100644
--- a/external/llvm-project/llvm/include/llvm/TextAPI/Utils.h
+++ b/external/llvm-project/llvm/include/llvm/TextAPI/Utils.h
@@ -14,6 +14,7 @@
 #define LLVM_TEXTAPI_UTILS_H
 
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -51,34 +52,35 @@ struct SymLink {
 ///
 /// \param Path Location of file.
 /// \param Extension File extension to update with.
-void replace_extension(SmallVectorImpl<char> &Path, const Twine &Extension);
+LLVM_ABI void replace_extension(SmallVectorImpl<char> &Path,
+                                const Twine &Extension);
 
 /// Determine whether to skip over symlink due to either too many symlink levels
 /// or is cyclic.
 ///
 /// \param Path Location to symlink.
 /// \param Result Holds whether to skip over Path.
-std::error_code shouldSkipSymLink(const Twine &Path, bool &Result);
+LLVM_ABI std::error_code shouldSkipSymLink(const Twine &Path, bool &Result);
 
 /// Turn absolute symlink into relative.
 ///
 /// \param From The symlink.
 /// \param To What the symlink points to.
 /// \param RelativePath Path location to update what the symlink points to.
-std::error_code make_relative(StringRef From, StringRef To,
-                              SmallVectorImpl<char> &RelativePath);
+LLVM_ABI std::error_code make_relative(StringRef From, StringRef To,
+                                       SmallVectorImpl<char> &RelativePath);
 
 /// Determine if library is private by parsing file path.
 /// It does not touch the file system.
 ///
 /// \param Path File path for library.
 /// \param IsSymLink Whether path points to a symlink.
-bool isPrivateLibrary(StringRef Path, bool IsSymLink = false);
+LLVM_ABI bool isPrivateLibrary(StringRef Path, bool IsSymLink = false);
 
 /// Create a regex rule from provided glob string.
 /// \param Glob String that represents glob input.
 /// \return The equivalent regex rule.
-llvm::Expected<llvm::Regex> createRegexFromGlob(llvm::StringRef Glob);
+LLVM_ABI llvm::Expected<llvm::Regex> createRegexFromGlob(llvm::StringRef Glob);
 
 using AliasEntry = std::pair<std::string, EncodeKind>;
 using AliasMap = std::map<AliasEntry, AliasEntry>;
@@ -87,14 +89,15 @@ using AliasMap = std::map<AliasEntry, AliasEntry>;
 ///
 /// \param Buffer Data contents of file for the alias list.
 /// \return Lookup table of alias to their base symbol.
-Expected<AliasMap> parseAliasList(std::unique_ptr<llvm::MemoryBuffer> &Buffer);
+LLVM_ABI Expected<AliasMap>
+parseAliasList(std::unique_ptr<llvm::MemoryBuffer> &Buffer);
 
 /// Pickup active paths for a given platform.
 ///
 /// \param Paths File or search paths to pick up.
 /// \param Platform Platform to collect paths for.
-PathSeq getPathsForPlatform(const PathToPlatformSeq &Paths,
-                            PlatformType Platform);
+LLVM_ABI PathSeq getPathsForPlatform(const PathToPlatformSeq &Paths,
+                                     PlatformType Platform);
 
 } // namespace llvm::MachO
 #endif // LLVM_TEXTAPI_UTILS_H
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/ABI.h b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/ABI.h
index 0b2d405f3cae..30d0e214d619 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/ABI.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/ABI.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_COROUTINES_ABI_H
 
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Coroutines/CoroShape.h"
 #include "llvm/Transforms/Coroutines/MaterializationUtils.h"
 #include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h"
@@ -37,7 +38,7 @@ namespace coro {
 // index of an ABI generator for the custom ABI object in a SmallVector passed
 // to CoroSplitPass ctor.
 
-class BaseABI {
+class LLVM_ABI BaseABI {
 public:
   BaseABI(Function &F, coro::Shape &S,
           std::function<bool(Instruction &)> IsMaterializable)
@@ -63,7 +64,7 @@ class BaseABI {
   std::function<bool(Instruction &I)> IsMaterializable;
 };
 
-class SwitchABI : public BaseABI {
+class LLVM_ABI SwitchABI : public BaseABI {
 public:
   SwitchABI(Function &F, coro::Shape &S,
             std::function<bool(Instruction &)> IsMaterializable)
@@ -76,7 +77,7 @@ class SwitchABI : public BaseABI {
                       TargetTransformInfo &TTI) override;
 };
 
-class AsyncABI : public BaseABI {
+class LLVM_ABI AsyncABI : public BaseABI {
 public:
   AsyncABI(Function &F, coro::Shape &S,
            std::function<bool(Instruction &)> IsMaterializable)
@@ -89,7 +90,7 @@ class AsyncABI : public BaseABI {
                       TargetTransformInfo &TTI) override;
 };
 
-class AnyRetconABI : public BaseABI {
+class LLVM_ABI AnyRetconABI : public BaseABI {
 public:
   AnyRetconABI(Function &F, coro::Shape &S,
                std::function<bool(Instruction &)> IsMaterializable)
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
index b8c846596c99..0688068167ae 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h
@@ -27,6 +27,7 @@
 
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
@@ -237,7 +238,7 @@ class AnyCoroIdRetconInst : public AnyCoroIdInst {
   enum { SizeArg, AlignArg, StorageArg, PrototypeArg, AllocArg, DeallocArg };
 
 public:
-  void checkWellFormed() const;
+  LLVM_ABI void checkWellFormed() const;
 
   uint64_t getStorageSize() const {
     return cast<ConstantInt>(getArgOperand(SizeArg))->getZExtValue();
@@ -306,7 +307,7 @@ class CoroIdAsyncInst : public AnyCoroIdInst {
   enum { SizeArg, AlignArg, StorageArg, AsyncFuncPtrArg };
 
 public:
-  void checkWellFormed() const;
+  LLVM_ABI void checkWellFormed() const;
 
   /// The initial async function context size. The fields of which are reserved
   /// for use by the frontend. The frame will be allocated as a tail of this
@@ -568,7 +569,7 @@ class CoroSuspendAsyncInst : public AnyCoroSuspendInst {
     MustTailCallFuncArg
   };
 
-  void checkWellFormed() const;
+  LLVM_ABI void checkWellFormed() const;
 
   unsigned getStorageArgumentIndex() const {
     auto *Arg = cast<ConstantInt>(getArgOperand(StorageArgNoArg));
@@ -722,7 +723,7 @@ class CoroAsyncEndInst : public AnyCoroEndInst {
   enum { FrameArg, UnwindArg, MustTailCallFuncArg };
 
 public:
-  void checkWellFormed() const;
+  LLVM_ABI void checkWellFormed() const;
 
   Function *getMustTailCallFunction() const {
     if (arg_size() < 3)
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroShape.h b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroShape.h
index 891774b44657..c54081de2d9d 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroShape.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroShape.h
@@ -14,6 +14,7 @@
 
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Coroutines/CoroInstr.h"
 
 namespace llvm {
@@ -78,18 +79,20 @@ struct Shape {
   }
 
   // Scan the function and collect the above intrinsics for later processing
-  void analyze(Function &F, SmallVectorImpl<CoroFrameInst *> &CoroFrames,
-               SmallVectorImpl<CoroSaveInst *> &UnusedCoroSaves,
-               CoroPromiseInst *&CoroPromise);
+  LLVM_ABI void analyze(Function &F,
+                        SmallVectorImpl<CoroFrameInst *> &CoroFrames,
+                        SmallVectorImpl<CoroSaveInst *> &UnusedCoroSaves,
+                        CoroPromiseInst *&CoroPromise);
   // If for some reason, we were not able to find coro.begin, bailout.
-  void invalidateCoroutine(Function &F,
-                           SmallVectorImpl<CoroFrameInst *> &CoroFrames);
+  LLVM_ABI void
+  invalidateCoroutine(Function &F,
+                      SmallVectorImpl<CoroFrameInst *> &CoroFrames);
   // Perform ABI related initial transformation
-  void initABI();
+  LLVM_ABI void initABI();
   // Remove orphaned and unnecessary intrinsics
-  void cleanCoroutine(SmallVectorImpl<CoroFrameInst *> &CoroFrames,
-                      SmallVectorImpl<CoroSaveInst *> &UnusedCoroSaves,
-                      CoroPromiseInst *CoroPromise);
+  LLVM_ABI void cleanCoroutine(SmallVectorImpl<CoroFrameInst *> &CoroFrames,
+                               SmallVectorImpl<CoroSaveInst *> &UnusedCoroSaves,
+                               CoroPromiseInst *CoroPromise);
 
   // Field indexes for special fields in the switch lowering.
   struct SwitchFieldIndex {
@@ -256,12 +259,14 @@ struct Shape {
   /// Allocate memory according to the rules of the active lowering.
   ///
   /// \param CG - if non-null, will be updated for the new call
-  Value *emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const;
+  LLVM_ABI Value *emitAlloc(IRBuilder<> &Builder, Value *Size,
+                            CallGraph *CG) const;
 
   /// Deallocate memory according to the rules of the active lowering.
   ///
   /// \param CG - if non-null, will be updated for the new call
-  void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const;
+  LLVM_ABI void emitDealloc(IRBuilder<> &Builder, Value *Ptr,
+                            CallGraph *CG) const;
 
   Shape() = default;
   explicit Shape(Function &F) {
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
index 6c6a982e8280..f7cc6f29879f 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Coroutines/ABI.h"
 
 namespace llvm {
@@ -31,20 +32,23 @@ struct CoroSplitPass : PassInfoMixin<CoroSplitPass> {
   using BaseABITy =
       std::function<std::unique_ptr<coro::BaseABI>(Function &, coro::Shape &)>;
 
-  CoroSplitPass(bool OptimizeFrame = false);
+  LLVM_ABI CoroSplitPass(bool OptimizeFrame = false);
 
-  CoroSplitPass(SmallVector<BaseABITy> GenCustomABIs,
-                bool OptimizeFrame = false);
+  LLVM_ABI CoroSplitPass(SmallVector<BaseABITy> GenCustomABIs,
+                         bool OptimizeFrame = false);
 
+  LLVM_ABI
   CoroSplitPass(std::function<bool(Instruction &)> MaterializableCallback,
                 bool OptimizeFrame = false);
 
+  LLVM_ABI
   CoroSplitPass(std::function<bool(Instruction &)> MaterializableCallback,
                 SmallVector<BaseABITy> GenCustomABIs,
                 bool OptimizeFrame = false);
 
-  PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
-                        LazyCallGraph &CG, CGSCCUpdateResult &UR);
+  LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C,
+                                 CGSCCAnalysisManager &AM, LazyCallGraph &CG,
+                                 CGSCCUpdateResult &UR);
 
   static bool isRequired() { return true; }
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/MaterializationUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/MaterializationUtils.h
index d8fc0c86a6fb..558984fed030 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/MaterializationUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/MaterializationUtils.h
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h"
 
 #ifndef LLVM_TRANSFORMS_COROUTINES_MATERIALIZATIONUTILS_H
@@ -16,11 +17,12 @@ namespace llvm {
 namespace coro {
 
 // True if I is trivially rematerialzable, e.g. InsertElementInst
-bool isTriviallyMaterializable(Instruction &I);
+LLVM_ABI bool isTriviallyMaterializable(Instruction &I);
 
 // Performs rematerialization, invoked from buildCoroutineFrame.
-void doRematerializations(Function &F, SuspendCrossingInfo &Checker,
-                          std::function<bool(Instruction &)> IsMaterializable);
+LLVM_ABI void
+doRematerializations(Function &F, SuspendCrossingInfo &Checker,
+                     std::function<bool(Instruction &)> IsMaterializable);
 
 } // namespace coro
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/SuspendCrossingInfo.h b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/SuspendCrossingInfo.h
index 88cbf88acc4c..1e04f09ddc27 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/SuspendCrossingInfo.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Coroutines/SuspendCrossingInfo.h
@@ -21,6 +21,7 @@
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Coroutines/CoroInstr.h"
 
 namespace llvm {
@@ -102,19 +103,21 @@ class SuspendCrossingInfo {
             ModuleSlotTracker &MST) const;
 #endif
 
+  LLVM_ABI
   SuspendCrossingInfo(Function &F,
                       const SmallVectorImpl<AnyCoroSuspendInst *> &CoroSuspends,
                       const SmallVectorImpl<AnyCoroEndInst *> &CoroEnds);
 
   /// Returns true if there is a path from \p From to \p To crossing a suspend
   /// point without crossing \p From a 2nd time.
-  bool hasPathCrossingSuspendPoint(BasicBlock *From, BasicBlock *To) const;
+  LLVM_ABI bool hasPathCrossingSuspendPoint(BasicBlock *From,
+                                            BasicBlock *To) const;
 
   /// Returns true if there is a path from \p From to \p To crossing a suspend
   /// point without crossing \p From a 2nd time. If \p From is the same as \p To
   /// this will also check if there is a looping path crossing a suspend point.
-  bool hasPathOrLoopCrossingSuspendPoint(BasicBlock *From,
-                                         BasicBlock *To) const;
+  LLVM_ABI bool hasPathOrLoopCrossingSuspendPoint(BasicBlock *From,
+                                                  BasicBlock *To) const;
 
   bool isDefinitionAcrossSuspend(BasicBlock *DefBB, User *U) const {
     auto *I = cast<Instruction>(U);
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h b/external/llvm-project/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
index 5ff38bdf0481..20850ba4c02d 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
@@ -19,6 +19,7 @@
 #define LLVM_TRANSFORMS_HIPSTDPAR_HIPSTDPAR_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -27,7 +28,7 @@ class Module;
 class HipStdParAcceleratorCodeSelectionPass
   : public PassInfoMixin<HipStdParAcceleratorCodeSelectionPass> {
 public:
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
   static bool isRequired() { return true; }
 };
@@ -35,7 +36,7 @@ class HipStdParAcceleratorCodeSelectionPass
 class HipStdParAllocationInterpositionPass
   : public PassInfoMixin<HipStdParAllocationInterpositionPass> {
 public:
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
   static bool isRequired() { return true; }
 };
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO.h
index ee0e35aa6183..56b30968ffd7 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO.h
@@ -24,29 +24,29 @@ class raw_ostream;
 /// createDeadArgEliminationPass - This pass removes arguments from functions
 /// which are not used by the body of the function.
 ///
-ModulePass *createDeadArgEliminationPass();
+LLVM_ABI ModulePass *createDeadArgEliminationPass();
 
 /// DeadArgHacking pass - Same as DAE, but delete arguments of external
 /// functions as well.  This is definitely not safe, and should only be used by
 /// bugpoint.
-ModulePass *createDeadArgHackingPass();
+LLVM_ABI ModulePass *createDeadArgHackingPass();
 
 //===----------------------------------------------------------------------===//
 //
 /// createLoopExtractorPass - This pass extracts all natural loops from the
 /// program into a function if it can.
 ///
-Pass *createLoopExtractorPass();
+LLVM_ABI Pass *createLoopExtractorPass();
 
 /// createSingleLoopExtractorPass - This pass extracts one natural loop from the
 /// program into a function if it can.  This is used by bugpoint.
 ///
-Pass *createSingleLoopExtractorPass();
+LLVM_ABI Pass *createSingleLoopExtractorPass();
 
 //===----------------------------------------------------------------------===//
 /// createBarrierNoopPass - This pass is purely a module pass barrier in a pass
 /// manager.
-ModulePass *createBarrierNoopPass();
+LLVM_ABI ModulePass *createBarrierNoopPass();
 
 /// What to do with the summary when running passes that operate on it.
 enum class PassSummaryAction {
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h
index 252cfd4dc5f3..679126b011d8 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/AlwaysInliner.h
@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -36,14 +37,13 @@ class AlwaysInlinerPass : public PassInfoMixin<AlwaysInlinerPass> {
   AlwaysInlinerPass(bool InsertLifetime = true)
       : InsertLifetime(InsertLifetime) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
   static bool isRequired() { return true; }
 };
 
 /// Create a legacy pass manager instance of a pass to inline and remove
 /// functions marked as "always_inline".
-Pass *createAlwaysInlinerLegacyPass(bool InsertLifetime = true);
-
+LLVM_ABI Pass *createAlwaysInlinerLegacyPass(bool InsertLifetime = true);
 }
 
 #endif // LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/Attributor.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/Attributor.h
index c628bbb00723..e6eb756df987 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -129,6 +129,7 @@
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DOTGraphTraits.h"
 #include "llvm/Support/DebugCounter.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -172,7 +173,7 @@ enum class GPUAddressSpace : unsigned {
 };
 
 /// Return true iff \p M target a GPU (and we can use GPU AS reasoning).
-bool isGPU(const Module &M);
+LLVM_ABI bool isGPU(const Module &M);
 
 /// Flags to distinguish intra-procedural queries from *potentially*
 /// inter-procedural queries. Not that information can be valid for both and
@@ -195,30 +196,32 @@ struct ValueAndContext : public std::pair<Value *, const Instruction *> {
 
 /// Return true if \p I is a `nosync` instruction. Use generic reasoning and
 /// potentially the corresponding AANoSync.
-bool isNoSyncInst(Attributor &A, const Instruction &I,
-                  const AbstractAttribute &QueryingAA);
+LLVM_ABI bool isNoSyncInst(Attributor &A, const Instruction &I,
+                           const AbstractAttribute &QueryingAA);
 
 /// Return true if \p V is dynamically unique, that is, there are no two
 /// "instances" of \p V at runtime with different values.
 /// Note: If \p ForAnalysisOnly is set we only check that the Attributor will
 /// never use \p V to represent two "instances" not that \p V could not
 /// technically represent them.
-bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA,
-                         const Value &V, bool ForAnalysisOnly = true);
+LLVM_ABI bool isDynamicallyUnique(Attributor &A,
+                                  const AbstractAttribute &QueryingAA,
+                                  const Value &V, bool ForAnalysisOnly = true);
 
 /// Return true if \p V is a valid value in \p Scope, that is a constant or an
 /// instruction/argument of \p Scope.
-bool isValidInScope(const Value &V, const Function *Scope);
+LLVM_ABI bool isValidInScope(const Value &V, const Function *Scope);
 
 /// Return true if the value of \p VAC is a valid at the position of \p VAC,
 /// that is a constant, an argument of the same function, or an instruction in
 /// that function that dominates the position.
-bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache);
+LLVM_ABI bool isValidAtPosition(const ValueAndContext &VAC,
+                                InformationCache &InfoCache);
 
 /// Try to convert \p V to type \p Ty without introducing new instructions. If
 /// this is not possible return `nullptr`. Note: this function basically knows
 /// how to cast various constants.
-Value *getWithType(Value &V, Type &Ty);
+LLVM_ABI Value *getWithType(Value &V, Type &Ty);
 
 /// Return the combination of \p A and \p B such that the result is a possible
 /// value of both. \p B is potentially casted to match the type \p Ty or the
@@ -228,7 +231,7 @@ Value *getWithType(Value &V, Type &Ty);
 ///        X + none  => X
 /// not_none + undef => not_none
 ///          V1 + V2 => nullptr
-std::optional<Value *>
+LLVM_ABI std::optional<Value *>
 combineOptionalValuesInAAValueLatice(const std::optional<Value *> &A,
                                      const std::optional<Value *> &B, Type *Ty);
 
@@ -328,11 +331,10 @@ inline bool operator==(const RangeTy &A, const RangeTy &B) {
 inline bool operator!=(const RangeTy &A, const RangeTy &B) { return !(A == B); }
 
 /// Return the initial value of \p Obj with type \p Ty if that is a constant.
-Constant *getInitialValueForObj(Attributor &A,
-                                const AbstractAttribute &QueryingAA, Value &Obj,
-                                Type &Ty, const TargetLibraryInfo *TLI,
-                                const DataLayout &DL,
-                                RangeTy *RangePtr = nullptr);
+LLVM_ABI Constant *
+getInitialValueForObj(Attributor &A, const AbstractAttribute &QueryingAA,
+                      Value &Obj, Type &Ty, const TargetLibraryInfo *TLI,
+                      const DataLayout &DL, RangeTy *RangePtr = nullptr);
 
 /// Collect all potential values \p LI could read into \p PotentialValues. That
 /// is, the only values read by \p LI are assumed to be known and all are in
@@ -345,7 +347,7 @@ Constant *getInitialValueForObj(Attributor &A,
 /// \returns True if the assumed potential copies are all in \p PotentialValues,
 ///          false if something went wrong and the copies could not be
 ///          determined.
-bool getPotentiallyLoadedValues(
+LLVM_ABI bool getPotentiallyLoadedValues(
     Attributor &A, LoadInst &LI, SmallSetVector<Value *, 4> &PotentialValues,
     SmallSetVector<Instruction *, 4> &PotentialValueOrigins,
     const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation,
@@ -360,20 +362,22 @@ bool getPotentiallyLoadedValues(
 /// \returns True if the assumed potential copies are all in \p PotentialCopies,
 ///          false if something went wrong and the copies could not be
 ///          determined.
-bool getPotentialCopiesOfStoredValue(
+LLVM_ABI bool getPotentialCopiesOfStoredValue(
     Attributor &A, StoreInst &SI, SmallSetVector<Value *, 4> &PotentialCopies,
     const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation,
     bool OnlyExact = false);
 
 /// Return true if \p IRP is readonly. This will query respective AAs that
 /// deduce the information and introduce dependences for \p QueryingAA.
-bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP,
-                       const AbstractAttribute &QueryingAA, bool &IsKnown);
+LLVM_ABI bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP,
+                                const AbstractAttribute &QueryingAA,
+                                bool &IsKnown);
 
 /// Return true if \p IRP is readnone. This will query respective AAs that
 /// deduce the information and introduce dependences for \p QueryingAA.
-bool isAssumedReadNone(Attributor &A, const IRPosition &IRP,
-                       const AbstractAttribute &QueryingAA, bool &IsKnown);
+LLVM_ABI bool isAssumedReadNone(Attributor &A, const IRPosition &IRP,
+                                const AbstractAttribute &QueryingAA,
+                                bool &IsKnown);
 
 /// Return true if \p ToI is potentially reachable from \p FromI without running
 /// into any instruction in \p ExclusionSet The two instructions do not need to
@@ -383,29 +387,31 @@ bool isAssumedReadNone(Attributor &A, const IRPosition &IRP,
 /// reached. If the query is not interested in callers beyond a certain point,
 /// e.g., a GPU kernel entry or the function containing an alloca, the
 /// \p GoBackwardsCB should return false.
-bool isPotentiallyReachable(
+LLVM_ABI bool isPotentiallyReachable(
     Attributor &A, const Instruction &FromI, const Instruction &ToI,
     const AbstractAttribute &QueryingAA,
     const AA::InstExclusionSetTy *ExclusionSet = nullptr,
     std::function<bool(const Function &F)> GoBackwardsCB = nullptr);
 
 /// Same as above but it is sufficient to reach any instruction in \p ToFn.
-bool isPotentiallyReachable(
+LLVM_ABI bool isPotentiallyReachable(
     Attributor &A, const Instruction &FromI, const Function &ToFn,
     const AbstractAttribute &QueryingAA,
     const AA::InstExclusionSetTy *ExclusionSet = nullptr,
     std::function<bool(const Function &F)> GoBackwardsCB = nullptr);
 
 /// Return true if \p Obj is assumed to be a thread local object.
-bool isAssumedThreadLocalObject(Attributor &A, Value &Obj,
-                                const AbstractAttribute &QueryingAA);
+LLVM_ABI bool isAssumedThreadLocalObject(Attributor &A, Value &Obj,
+                                         const AbstractAttribute &QueryingAA);
 
 /// Return true if \p I is potentially affected by a barrier.
-bool isPotentiallyAffectedByBarrier(Attributor &A, const Instruction &I,
-                                    const AbstractAttribute &QueryingAA);
-bool isPotentiallyAffectedByBarrier(Attributor &A, ArrayRef<const Value *> Ptrs,
-                                    const AbstractAttribute &QueryingAA,
-                                    const Instruction *CtxI);
+LLVM_ABI bool
+isPotentiallyAffectedByBarrier(Attributor &A, const Instruction &I,
+                               const AbstractAttribute &QueryingAA);
+LLVM_ABI bool
+isPotentiallyAffectedByBarrier(Attributor &A, ArrayRef<const Value *> Ptrs,
+                               const AbstractAttribute &QueryingAA,
+                               const Instruction *CtxI);
 } // namespace AA
 
 template <>
@@ -483,7 +489,7 @@ struct DenseMapInfo<const AA::InstExclusionSetTy *>
 
 /// The value passed to the line option that defines the maximal initialization
 /// chain length.
-extern unsigned MaxInitializationChainLength;
+LLVM_ABI extern unsigned MaxInitializationChainLength;
 
 ///{
 enum class ChangeStatus {
@@ -491,10 +497,10 @@ enum class ChangeStatus {
   UNCHANGED,
 };
 
-ChangeStatus operator|(ChangeStatus l, ChangeStatus r);
-ChangeStatus &operator|=(ChangeStatus &l, ChangeStatus r);
-ChangeStatus operator&(ChangeStatus l, ChangeStatus r);
-ChangeStatus &operator&=(ChangeStatus &l, ChangeStatus r);
+LLVM_ABI ChangeStatus operator|(ChangeStatus l, ChangeStatus r);
+LLVM_ABI ChangeStatus &operator|=(ChangeStatus &l, ChangeStatus r);
+LLVM_ABI ChangeStatus operator&(ChangeStatus l, ChangeStatus r);
+LLVM_ABI ChangeStatus &operator&=(ChangeStatus &l, ChangeStatus r);
 
 enum class DepClassTy {
   REQUIRED, ///< The target cannot be valid if the source is not.
@@ -565,13 +571,13 @@ struct AADepGraph {
   iterator begin() { return SyntheticRoot.child_begin(); }
   iterator end() { return SyntheticRoot.child_end(); }
 
-  void viewGraph();
+  LLVM_ABI void viewGraph();
 
   /// Dump graph to file
-  void dumpGraph();
+  LLVM_ABI void dumpGraph();
 
   /// Print dependency graph
-  void print();
+  LLVM_ABI void print();
 };
 
 /// Helper to describe and deal with positions in the LLVM-IR.
@@ -728,7 +734,7 @@ struct IRPosition {
   }
 
   /// Return the associated argument, if any.
-  Argument *getAssociatedArgument() const;
+  LLVM_ABI Argument *getAssociatedArgument() const;
 
   /// Return true if the position refers to a function interface, that is the
   /// function scope, the function return, or an argument.
@@ -937,8 +943,8 @@ struct IRPosition {
   /// Special DenseMap key values.
   ///
   ///{
-  static const IRPosition EmptyKey;
-  static const IRPosition TombstoneKey;
+  LLVM_ABI static const IRPosition EmptyKey;
+  LLVM_ABI static const IRPosition TombstoneKey;
   ///}
 
   /// Conversion into a void * to allow reuse of pointer hashing.
@@ -1015,7 +1021,7 @@ struct IRPosition {
   }
 
   /// Verify internal invariants.
-  void verify();
+  LLVM_ABI void verify();
 
   /// Return the underlying pointer as Value *, valid for all positions but
   /// IRP_CALL_SITE_ARGUMENT.
@@ -1118,7 +1124,7 @@ class SubsumingPositionIterator {
   using iterator = decltype(IRPositions)::iterator;
 
 public:
-  SubsumingPositionIterator(const IRPosition &IRP);
+  LLVM_ABI SubsumingPositionIterator(const IRPosition &IRP);
   iterator begin() { return IRPositions.begin(); }
   iterator end() { return IRPositions.end(); }
 };
@@ -1343,14 +1349,15 @@ struct InformationCache {
 
   /// Return all functions that might be called indirectly, only valid for
   /// closed world modules (see isClosedWorldModule).
-  ArrayRef<Function *> getIndirectlyCallableFunctions(Attributor &A) const;
+  LLVM_ABI ArrayRef<Function *>
+  getIndirectlyCallableFunctions(Attributor &A) const;
 
   /// Return the flat address space if the associated target has.
-  std::optional<unsigned> getFlatAddressSpace() const;
+  LLVM_ABI std::optional<unsigned> getFlatAddressSpace() const;
 
 private:
   struct FunctionInfo {
-    ~FunctionInfo();
+    LLVM_ABI ~FunctionInfo();
 
     /// A nested map that remembers all instructions in a function with a
     /// certain instruction opcode (Instruction::getOpcode()).
@@ -1391,7 +1398,7 @@ struct InformationCache {
   ///
   /// This method needs to be called for all function that might be looked at
   /// through the information cache interface *prior* to looking at them.
-  void initializeInformationCache(const Function &F, FunctionInfo &FI);
+  LLVM_ABI void initializeInformationCache(const Function &F, FunctionInfo &FI);
 
   /// The datalayout used in the module.
   const DataLayout &DL;
@@ -1529,10 +1536,11 @@ struct Attributor {
   ///                  the abstract attributes.
   /// \param Configuration The Attributor configuration which determines what
   ///                      generic features to use.
-  Attributor(SetVector<Function *> &Functions, InformationCache &InfoCache,
-             AttributorConfig Configuration);
+  LLVM_ABI Attributor(SetVector<Function *> &Functions,
+                      InformationCache &InfoCache,
+                      AttributorConfig Configuration);
 
-  ~Attributor();
+  LLVM_ABI ~Attributor();
 
   /// Run the analyses until a fixpoint is reached or enforced (timeout).
   ///
@@ -1540,7 +1548,7 @@ struct Attributor {
   /// as the Attributor is not destroyed (it owns the attributes now).
   ///
   /// \Returns CHANGED if the IR was changed, otherwise UNCHANGED.
-  ChangeStatus run();
+  LLVM_ABI ChangeStatus run();
 
   /// Lookup an abstract attribute of type \p AAType at position \p IRP. While
   /// no abstract attribute is found equivalent positions are checked, see
@@ -1681,7 +1689,7 @@ struct Attributor {
   }
 
   /// Allows a query AA to request an update if a new query was received.
-  void registerForUpdate(AbstractAttribute &AA);
+  LLVM_ABI void registerForUpdate(AbstractAttribute &AA);
 
   /// Explicitly record a dependence from \p FromAA to \p ToAA, that is if
   /// \p FromAA changes \p ToAA should be updated as well.
@@ -1694,8 +1702,9 @@ struct Attributor {
   /// That means for required dependences, if \p FromAA changes to an invalid
   /// state, \p ToAA can be moved to a pessimistic fixpoint because it required
   /// information from \p FromAA but none are available anymore.
-  void recordDependence(const AbstractAttribute &FromAA,
-                        const AbstractAttribute &ToAA, DepClassTy DepClass);
+  LLVM_ABI void recordDependence(const AbstractAttribute &FromAA,
+                                 const AbstractAttribute &ToAA,
+                                 DepClassTy DepClass);
 
   /// Introduce a new abstract attribute into the fixpoint analysis.
   ///
@@ -1743,7 +1752,7 @@ struct Attributor {
 
   /// Return true if the module contains the whole world, thus, no outside
   /// functions exist.
-  bool isClosedWorldModule() const;
+  LLVM_ABI bool isClosedWorldModule() const;
 
   /// Return true if we derive attributes for \p Fn
   bool isRunOn(Function &Fn) const { return isRunOn(&Fn); }
@@ -1819,7 +1828,7 @@ struct Attributor {
   /// reason for this is the single interface, the one of the abstract attribute
   /// instance, which can be queried without the need to look at the IR in
   /// various places.
-  void identifyDefaultAbstractAttributes(Function &F);
+  LLVM_ABI void identifyDefaultAbstractAttributes(Function &F);
 
   /// Determine whether the function \p F is IPO amendable
   ///
@@ -1914,17 +1923,19 @@ struct Attributor {
 
   /// Return the attributes of kind \p AK existing in the IR as operand bundles
   /// of an llvm.assume.
-  bool getAttrsFromAssumes(const IRPosition &IRP, Attribute::AttrKind AK,
-                           SmallVectorImpl<Attribute> &Attrs);
+  LLVM_ABI bool getAttrsFromAssumes(const IRPosition &IRP,
+                                    Attribute::AttrKind AK,
+                                    SmallVectorImpl<Attribute> &Attrs);
 
   /// Return true if any kind in \p AKs existing in the IR at a position that
   /// will affect this one. See also getAttrs(...).
   /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions,
   ///                                 e.g., the function position if this is an
   ///                                 argument position, should be ignored.
-  bool hasAttr(const IRPosition &IRP, ArrayRef<Attribute::AttrKind> AKs,
-               bool IgnoreSubsumingPositions = false,
-               Attribute::AttrKind ImpliedAttributeKind = Attribute::None);
+  LLVM_ABI bool
+  hasAttr(const IRPosition &IRP, ArrayRef<Attribute::AttrKind> AKs,
+          bool IgnoreSubsumingPositions = false,
+          Attribute::AttrKind ImpliedAttributeKind = Attribute::None);
 
   /// Return the attributes of any kind in \p AKs existing in the IR at a
   /// position that will affect this one. While each position can only have a
@@ -1934,20 +1945,22 @@ struct Attributor {
   /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions,
   ///                                 e.g., the function position if this is an
   ///                                 argument position, should be ignored.
-  void getAttrs(const IRPosition &IRP, ArrayRef<Attribute::AttrKind> AKs,
-                SmallVectorImpl<Attribute> &Attrs,
-                bool IgnoreSubsumingPositions = false);
+  LLVM_ABI void getAttrs(const IRPosition &IRP,
+                         ArrayRef<Attribute::AttrKind> AKs,
+                         SmallVectorImpl<Attribute> &Attrs,
+                         bool IgnoreSubsumingPositions = false);
 
   /// Remove all \p AttrKinds attached to \p IRP.
-  ChangeStatus removeAttrs(const IRPosition &IRP,
-                           ArrayRef<Attribute::AttrKind> AttrKinds);
-  ChangeStatus removeAttrs(const IRPosition &IRP, ArrayRef<StringRef> Attrs);
+  LLVM_ABI ChangeStatus removeAttrs(const IRPosition &IRP,
+                                    ArrayRef<Attribute::AttrKind> AttrKinds);
+  LLVM_ABI ChangeStatus removeAttrs(const IRPosition &IRP,
+                                    ArrayRef<StringRef> Attrs);
 
   /// Attach \p DeducedAttrs to \p IRP, if \p ForceReplace is set we do this
   /// even if the same attribute kind was already present.
-  ChangeStatus manifestAttrs(const IRPosition &IRP,
-                             ArrayRef<Attribute> DeducedAttrs,
-                             bool ForceReplace = false);
+  LLVM_ABI ChangeStatus manifestAttrs(const IRPosition &IRP,
+                                      ArrayRef<Attribute> DeducedAttrs,
+                                      bool ForceReplace = false);
 
 private:
   /// Helper to check \p Attrs for \p AK, if not found, check if \p
@@ -1972,9 +1985,9 @@ struct Attributor {
 public:
   /// If \p IRP is assumed to be a constant, return it, if it is unclear yet,
   /// return std::nullopt, otherwise return `nullptr`.
-  std::optional<Constant *> getAssumedConstant(const IRPosition &IRP,
-                                               const AbstractAttribute &AA,
-                                               bool &UsedAssumedInformation);
+  LLVM_ABI std::optional<Constant *>
+  getAssumedConstant(const IRPosition &IRP, const AbstractAttribute &AA,
+                     bool &UsedAssumedInformation);
   std::optional<Constant *> getAssumedConstant(const Value &V,
                                                const AbstractAttribute &AA,
                                                bool &UsedAssumedInformation) {
@@ -2001,10 +2014,9 @@ struct Attributor {
   /// return std::nullopt, otherwise return `nullptr`. Same as the public
   /// version except that it can be used without recording dependences on any \p
   /// AA.
-  std::optional<Value *> getAssumedSimplified(const IRPosition &V,
-                                              const AbstractAttribute *AA,
-                                              bool &UsedAssumedInformation,
-                                              AA::ValueScope S);
+  LLVM_ABI std::optional<Value *>
+  getAssumedSimplified(const IRPosition &V, const AbstractAttribute *AA,
+                       bool &UsedAssumedInformation, AA::ValueScope S);
 
   /// Try to simplify \p IRP and in the scope \p S. If successful, true is
   /// returned and all potential values \p IRP can take are put into \p Values.
@@ -2012,12 +2024,11 @@ struct Attributor {
   /// those could not be simplified to a single value. Recursive calls with
   /// these instructions will yield their respective potential values. If false
   /// is returned no other information is valid.
-  bool getAssumedSimplifiedValues(const IRPosition &IRP,
-                                  const AbstractAttribute *AA,
-                                  SmallVectorImpl<AA::ValueAndContext> &Values,
-                                  AA::ValueScope S,
-                                  bool &UsedAssumedInformation,
-                                  bool RecurseForSelectAndPHI = true);
+  LLVM_ABI bool
+  getAssumedSimplifiedValues(const IRPosition &IRP, const AbstractAttribute *AA,
+                             SmallVectorImpl<AA::ValueAndContext> &Values,
+                             AA::ValueScope S, bool &UsedAssumedInformation,
+                             bool RecurseForSelectAndPHI = true);
 
   /// Register \p CB as a simplification callback.
   /// `Attributor::getAssumedSimplified` will use these callbacks before
@@ -2093,7 +2104,7 @@ struct Attributor {
 
 public:
   /// Translate \p V from the callee context into the call site context.
-  std::optional<Value *>
+  LLVM_ABI std::optional<Value *>
   translateArgumentToCallSiteContent(std::optional<Value *> V, CallBase &CB,
                                      const AbstractAttribute &AA,
                                      bool &UsedAssumedInformation);
@@ -2101,49 +2112,56 @@ struct Attributor {
   /// Return true if \p AA (or its context instruction) is assumed dead.
   ///
   /// If \p LivenessAA is not provided it is queried.
-  bool isAssumedDead(const AbstractAttribute &AA, const AAIsDead *LivenessAA,
-                     bool &UsedAssumedInformation,
-                     bool CheckBBLivenessOnly = false,
-                     DepClassTy DepClass = DepClassTy::OPTIONAL);
+  LLVM_ABI bool isAssumedDead(const AbstractAttribute &AA,
+                              const AAIsDead *LivenessAA,
+                              bool &UsedAssumedInformation,
+                              bool CheckBBLivenessOnly = false,
+                              DepClassTy DepClass = DepClassTy::OPTIONAL);
 
   /// Return true if \p I is assumed dead.
   ///
   /// If \p LivenessAA is not provided it is queried.
-  bool isAssumedDead(const Instruction &I, const AbstractAttribute *QueryingAA,
-                     const AAIsDead *LivenessAA, bool &UsedAssumedInformation,
-                     bool CheckBBLivenessOnly = false,
-                     DepClassTy DepClass = DepClassTy::OPTIONAL,
-                     bool CheckForDeadStore = false);
+  LLVM_ABI bool isAssumedDead(const Instruction &I,
+                              const AbstractAttribute *QueryingAA,
+                              const AAIsDead *LivenessAA,
+                              bool &UsedAssumedInformation,
+                              bool CheckBBLivenessOnly = false,
+                              DepClassTy DepClass = DepClassTy::OPTIONAL,
+                              bool CheckForDeadStore = false);
 
   /// Return true if \p U is assumed dead.
   ///
   /// If \p FnLivenessAA is not provided it is queried.
-  bool isAssumedDead(const Use &U, const AbstractAttribute *QueryingAA,
-                     const AAIsDead *FnLivenessAA, bool &UsedAssumedInformation,
-                     bool CheckBBLivenessOnly = false,
-                     DepClassTy DepClass = DepClassTy::OPTIONAL);
+  LLVM_ABI bool isAssumedDead(const Use &U, const AbstractAttribute *QueryingAA,
+                              const AAIsDead *FnLivenessAA,
+                              bool &UsedAssumedInformation,
+                              bool CheckBBLivenessOnly = false,
+                              DepClassTy DepClass = DepClassTy::OPTIONAL);
 
   /// Return true if \p IRP is assumed dead.
   ///
   /// If \p FnLivenessAA is not provided it is queried.
-  bool isAssumedDead(const IRPosition &IRP, const AbstractAttribute *QueryingAA,
-                     const AAIsDead *FnLivenessAA, bool &UsedAssumedInformation,
-                     bool CheckBBLivenessOnly = false,
-                     DepClassTy DepClass = DepClassTy::OPTIONAL);
+  LLVM_ABI bool isAssumedDead(const IRPosition &IRP,
+                              const AbstractAttribute *QueryingAA,
+                              const AAIsDead *FnLivenessAA,
+                              bool &UsedAssumedInformation,
+                              bool CheckBBLivenessOnly = false,
+                              DepClassTy DepClass = DepClassTy::OPTIONAL);
 
   /// Return true if \p BB is assumed dead.
   ///
   /// If \p LivenessAA is not provided it is queried.
-  bool isAssumedDead(const BasicBlock &BB, const AbstractAttribute *QueryingAA,
-                     const AAIsDead *FnLivenessAA,
-                     DepClassTy DepClass = DepClassTy::OPTIONAL);
+  LLVM_ABI bool isAssumedDead(const BasicBlock &BB,
+                              const AbstractAttribute *QueryingAA,
+                              const AAIsDead *FnLivenessAA,
+                              DepClassTy DepClass = DepClassTy::OPTIONAL);
 
   /// Check \p Pred on all potential Callees of \p CB.
   ///
   /// This method will evaluate \p Pred with all potential callees of \p CB as
   /// input and return true if \p Pred does. If some callees might be unknown
   /// this function will return false.
-  bool checkForAllCallees(
+  LLVM_ABI bool checkForAllCallees(
       function_ref<bool(ArrayRef<const Function *> Callees)> Pred,
       const AbstractAttribute &QueryingAA, const CallBase &CB);
 
@@ -2156,13 +2174,14 @@ struct Attributor {
   /// what original used was replaced by a new one (or new ones). The visit is
   /// cut short if \p EquivalentUseCB returns false and the function will return
   /// false as well.
-  bool checkForAllUses(function_ref<bool(const Use &, bool &)> Pred,
-                       const AbstractAttribute &QueryingAA, const Value &V,
-                       bool CheckBBLivenessOnly = false,
-                       DepClassTy LivenessDepClass = DepClassTy::OPTIONAL,
-                       bool IgnoreDroppableUses = true,
-                       function_ref<bool(const Use &OldU, const Use &NewU)>
-                           EquivalentUseCB = nullptr);
+  LLVM_ABI bool checkForAllUses(
+      function_ref<bool(const Use &, bool &)> Pred,
+      const AbstractAttribute &QueryingAA, const Value &V,
+      bool CheckBBLivenessOnly = false,
+      DepClassTy LivenessDepClass = DepClassTy::OPTIONAL,
+      bool IgnoreDroppableUses = true,
+      function_ref<bool(const Use &OldU, const Use &NewU)> EquivalentUseCB =
+          nullptr);
 
   /// Emit a remark generically.
   ///
@@ -2300,8 +2319,9 @@ struct Attributor {
   ///
   /// \returns True, if the replacement can be registered, via
   /// registerFunctionSignatureRewrite, false otherwise.
-  bool isValidFunctionSignatureRewrite(Argument &Arg,
-                                       ArrayRef<Type *> ReplacementTypes);
+  LLVM_ABI bool
+  isValidFunctionSignatureRewrite(Argument &Arg,
+                                  ArrayRef<Type *> ReplacementTypes);
 
   /// Register a rewrite for a function signature.
   ///
@@ -2311,7 +2331,7 @@ struct Attributor {
   /// \p CalleeRepairCB.
   ///
   /// \returns True, if the replacement was registered, false otherwise.
-  bool registerFunctionSignatureRewrite(
+  LLVM_ABI bool registerFunctionSignatureRewrite(
       Argument &Arg, ArrayRef<Type *> ReplacementTypes,
       ArgumentReplacementInfo::CalleeRepairCBTy &&CalleeRepairCB,
       ArgumentReplacementInfo::ACSRepairCBTy &&ACSRepairCB);
@@ -2323,10 +2343,10 @@ struct Attributor {
   /// all call sites are known, hence the function has internal linkage.
   /// If true is returned, \p UsedAssumedInformation is set if assumed
   /// information was used to skip or simplify potential call sites.
-  bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
-                            const AbstractAttribute &QueryingAA,
-                            bool RequireAllCallSites,
-                            bool &UsedAssumedInformation);
+  LLVM_ABI bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
+                                     const AbstractAttribute &QueryingAA,
+                                     bool RequireAllCallSites,
+                                     bool &UsedAssumedInformation);
 
   /// Check \p Pred on all call sites of \p Fn.
   ///
@@ -2335,17 +2355,18 @@ struct Attributor {
   /// all call sites are known, hence the function has internal linkage.
   /// If true is returned, \p UsedAssumedInformation is set if assumed
   /// information was used to skip or simplify potential call sites.
-  bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
-                            const Function &Fn, bool RequireAllCallSites,
-                            const AbstractAttribute *QueryingAA,
-                            bool &UsedAssumedInformation,
-                            bool CheckPotentiallyDead = false);
+  LLVM_ABI bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
+                                     const Function &Fn,
+                                     bool RequireAllCallSites,
+                                     const AbstractAttribute *QueryingAA,
+                                     bool &UsedAssumedInformation,
+                                     bool CheckPotentiallyDead = false);
 
   /// Check \p Pred on all values potentially returned by the function
   /// associated with \p QueryingAA.
   ///
   /// This is the context insensitive version of the method above.
-  bool
+  LLVM_ABI bool
   checkForAllReturnedValues(function_ref<bool(Value &)> Pred,
                             const AbstractAttribute &QueryingAA,
                             AA::ValueScope S = AA::ValueScope::Intraprocedural,
@@ -2356,24 +2377,24 @@ struct Attributor {
   ///
   /// This method will evaluate \p Pred on all instructions with an opcode
   /// present in \p Opcode and return true if \p Pred holds on all of them.
-  bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
-                               const Function *Fn,
-                               const AbstractAttribute *QueryingAA,
-                               ArrayRef<unsigned> Opcodes,
-                               bool &UsedAssumedInformation,
-                               bool CheckBBLivenessOnly = false,
-                               bool CheckPotentiallyDead = false);
+  LLVM_ABI bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
+                                        const Function *Fn,
+                                        const AbstractAttribute *QueryingAA,
+                                        ArrayRef<unsigned> Opcodes,
+                                        bool &UsedAssumedInformation,
+                                        bool CheckBBLivenessOnly = false,
+                                        bool CheckPotentiallyDead = false);
 
   /// Check \p Pred on all instructions with an opcode present in \p Opcodes.
   ///
   /// This method will evaluate \p Pred on all instructions with an opcode
   /// present in \p Opcode and return true if \p Pred holds on all of them.
-  bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
-                               const AbstractAttribute &QueryingAA,
-                               ArrayRef<unsigned> Opcodes,
-                               bool &UsedAssumedInformation,
-                               bool CheckBBLivenessOnly = false,
-                               bool CheckPotentiallyDead = false);
+  LLVM_ABI bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
+                                        const AbstractAttribute &QueryingAA,
+                                        ArrayRef<unsigned> Opcodes,
+                                        bool &UsedAssumedInformation,
+                                        bool CheckBBLivenessOnly = false,
+                                        bool CheckPotentiallyDead = false);
 
   /// Check \p Pred on all call-like instructions (=CallBased derived).
   ///
@@ -2395,9 +2416,10 @@ struct Attributor {
   /// This method will evaluate \p Pred on all instructions that read or write
   /// to memory present in the information cache and return true if \p Pred
   /// holds on all of them.
-  bool checkForAllReadWriteInstructions(function_ref<bool(Instruction &)> Pred,
-                                        AbstractAttribute &QueryingAA,
-                                        bool &UsedAssumedInformation);
+  LLVM_ABI bool
+  checkForAllReadWriteInstructions(function_ref<bool(Instruction &)> Pred,
+                                   AbstractAttribute &QueryingAA,
+                                   bool &UsedAssumedInformation);
 
   /// Create a shallow wrapper for \p F such that \p F has internal linkage
   /// afterwards. It also sets the original \p F 's name to anonymous
@@ -2413,11 +2435,11 @@ struct Attributor {
   ///     return F(arg0, ..., argN);
   ///   }
   ///
-  static void createShallowWrapper(Function &F);
+  LLVM_ABI static void createShallowWrapper(Function &F);
 
   /// Returns true if the function \p F can be internalized. i.e. it has a
   /// compatible linkage.
-  static bool isInternalizable(Function &F);
+  LLVM_ABI static bool isInternalizable(Function &F);
 
   /// Make another copy of the function \p F such that the copied version has
   /// internal linkage afterwards and can be analysed. Then we replace all uses
@@ -2432,7 +2454,8 @@ struct Attributor {
   ///
   /// If the function \p F failed to be internalized the return value will be a
   /// null pointer.
-  static Function *internalizeFunction(Function &F, bool Force = false);
+  LLVM_ABI static Function *internalizeFunction(Function &F,
+                                                bool Force = false);
 
   /// Make copies of each function in the set \p FnSet such that the copied
   /// version has internal linkage afterwards and can be analysed. Then we
@@ -2447,8 +2470,9 @@ struct Attributor {
   /// once and then replace the uses. This prevents internalized functions being
   /// called by external functions when there is an internalized version in the
   /// module.
-  static bool internalizeFunctions(SmallPtrSetImpl<Function *> &FnSet,
-                                   DenseMap<Function *, Function *> &FnMap);
+  LLVM_ABI static bool
+  internalizeFunctions(SmallPtrSetImpl<Function *> &FnSet,
+                       DenseMap<Function *, Function *> &FnMap);
 
   /// Return the data layout associated with the anchor scope.
   const DataLayout &getDataLayout() const { return InfoCache.DL; }
@@ -2483,14 +2507,14 @@ struct Attributor {
 
   /// Run `::update` on \p AA and track the dependences queried while doing so.
   /// Also adjust the state if we know further updates are not necessary.
-  ChangeStatus updateAA(AbstractAttribute &AA);
+  LLVM_ABI ChangeStatus updateAA(AbstractAttribute &AA);
 
   /// Remember the dependences on the top of the dependence stack such that they
   /// may trigger further updates. (\see DependenceStack)
   void rememberDependences();
 
   /// Determine if CallBase context in \p IRP should be propagated.
-  bool shouldPropagateCallBaseContext(const IRPosition &IRP);
+  LLVM_ABI bool shouldPropagateCallBaseContext(const IRPosition &IRP);
 
   /// Apply all requested function signature rewrites
   /// (\see registerFunctionSignatureRewrite) and return Changed if the module
@@ -2500,7 +2524,7 @@ struct Attributor {
 
   /// Check if the Attribute \p AA should be seeded.
   /// See getOrCreateAAFor.
-  bool shouldSeedAttribute(AbstractAttribute &AA);
+  LLVM_ABI bool shouldSeedAttribute(AbstractAttribute &AA);
 
   /// A nested map to lookup abstract attributes based on the argument position
   /// on the outer level, and the addresses of the static member (AAType::ID) on
@@ -3292,7 +3316,7 @@ struct IRAttribute : public BaseType {
 ///       both directions will be added in the future.
 /// NOTE: The mechanics of adding a new "concrete" abstract attribute are
 ///       described in the file comment.
-struct AbstractAttribute : public IRPosition, public AADepGraphNode {
+struct LLVM_ABI AbstractAttribute : public IRPosition, public AADepGraphNode {
   using StateType = AbstractState;
 
   AbstractAttribute(const IRPosition &IRP) : IRPosition(IRP) {}
@@ -3431,11 +3455,11 @@ struct AbstractAttribute : public IRPosition, public AADepGraphNode {
 /// Forward declarations of output streams for debug purposes.
 ///
 ///{
-raw_ostream &operator<<(raw_ostream &OS, const AbstractAttribute &AA);
-raw_ostream &operator<<(raw_ostream &OS, ChangeStatus S);
-raw_ostream &operator<<(raw_ostream &OS, IRPosition::Kind);
-raw_ostream &operator<<(raw_ostream &OS, const IRPosition &);
-raw_ostream &operator<<(raw_ostream &OS, const AbstractState &State);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const AbstractAttribute &AA);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, ChangeStatus S);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, IRPosition::Kind);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const IRPosition &);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const AbstractState &State);
 template <typename base_ty, base_ty BestState, base_ty WorstState>
 raw_ostream &
 operator<<(raw_ostream &OS,
@@ -3443,29 +3467,32 @@ operator<<(raw_ostream &OS,
   return OS << "(" << S.getKnown() << "-" << S.getAssumed() << ")"
             << static_cast<const AbstractState &>(S);
 }
-raw_ostream &operator<<(raw_ostream &OS, const IntegerRangeState &State);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const IntegerRangeState &State);
 ///}
 
 struct AttributorPass : public PassInfoMixin<AttributorPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 struct AttributorCGSCCPass : public PassInfoMixin<AttributorCGSCCPass> {
-  PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
-                        LazyCallGraph &CG, CGSCCUpdateResult &UR);
+  LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C,
+                                 CGSCCAnalysisManager &AM, LazyCallGraph &CG,
+                                 CGSCCUpdateResult &UR);
 };
 
 /// A more lightweight version of the Attributor which only runs attribute
 /// inference but no simplifications.
 struct AttributorLightPass : public PassInfoMixin<AttributorLightPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 /// A more lightweight version of the Attributor which only runs attribute
 /// inference but no simplifications.
 struct AttributorLightCGSCCPass
     : public PassInfoMixin<AttributorLightCGSCCPass> {
-  PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
-                        LazyCallGraph &CG, CGSCCUpdateResult &UR);
+  LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C,
+                                 CGSCCAnalysisManager &AM, LazyCallGraph &CG,
+                                 CGSCCUpdateResult &UR);
 };
 
 /// Helper function to clamp a state \p S of type \p StateType with the
@@ -3496,7 +3523,8 @@ struct AANoUnwind
   bool isKnownNoUnwind() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoUnwind &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoUnwind &createForPosition(const IRPosition &IRP,
+                                                Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoUnwind"; }
@@ -3510,7 +3538,7 @@ struct AANoUnwind
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 struct AANoSync
@@ -3566,20 +3594,22 @@ struct AANoSync
   /// Helper function used to determine whether an instruction is non-relaxed
   /// atomic. In other words, if an atomic instruction does not have unordered
   /// or monotonic ordering
-  static bool isNonRelaxedAtomic(const Instruction *I);
+  LLVM_ABI static bool isNonRelaxedAtomic(const Instruction *I);
 
   /// Helper function specific for intrinsics which are potentially volatile.
-  static bool isNoSyncIntrinsic(const Instruction *I);
+  LLVM_ABI static bool isNoSyncIntrinsic(const Instruction *I);
 
   /// Helper function to determine if \p CB is an aligned (GPU) barrier. Aligned
   /// barriers have to be executed by all threads. The flag \p ExecutedAligned
   /// indicates if the call is executed by all threads in a (thread) block in an
   /// aligned way. If that is the case, non-aligned barriers are effectively
   /// aligned barriers.
-  static bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned);
+  LLVM_ABI static bool isAlignedBarrier(const CallBase &CB,
+                                        bool ExecutedAligned);
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoSync &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoSync &createForPosition(const IRPosition &IRP,
+                                              Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoSync"; }
@@ -3593,7 +3623,7 @@ struct AANoSync
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for all nonnull attributes.
@@ -3619,8 +3649,8 @@ struct AAMustProgress
   bool isKnownMustProgress() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAMustProgress &createForPosition(const IRPosition &IRP,
-                                           Attributor &A);
+  LLVM_ABI static AAMustProgress &createForPosition(const IRPosition &IRP,
+                                                    Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAMustProgress"; }
@@ -3635,7 +3665,7 @@ struct AAMustProgress
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for all nonnull attributes.
@@ -3661,9 +3691,9 @@ struct AANonNull
   }
 
   /// See AbstractAttribute::isImpliedByIR(...).
-  static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
-                            Attribute::AttrKind ImpliedAttributeKind,
-                            bool IgnoreSubsumingPositions = false);
+  LLVM_ABI static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+                                     Attribute::AttrKind ImpliedAttributeKind,
+                                     bool IgnoreSubsumingPositions = false);
 
   /// Return true if we assume that the underlying value is nonnull.
   bool isAssumedNonNull() const { return getAssumed(); }
@@ -3672,7 +3702,8 @@ struct AANonNull
   bool isKnownNonNull() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANonNull &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANonNull &createForPosition(const IRPosition &IRP,
+                                               Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANonNull"; }
@@ -3686,7 +3717,7 @@ struct AANonNull
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract attribute for norecurse.
@@ -3703,7 +3734,8 @@ struct AANoRecurse
   bool isKnownNoRecurse() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoRecurse &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoRecurse &createForPosition(const IRPosition &IRP,
+                                                 Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoRecurse"; }
@@ -3717,7 +3749,7 @@ struct AANoRecurse
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract attribute for willreturn.
@@ -3767,7 +3799,8 @@ struct AAWillReturn
   bool isKnownWillReturn() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAWillReturn &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AAWillReturn &createForPosition(const IRPosition &IRP,
+                                                  Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAWillReturn"; }
@@ -3781,7 +3814,7 @@ struct AAWillReturn
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract attribute for undefined behavior.
@@ -3803,8 +3836,8 @@ struct AAUndefinedBehavior
   virtual bool isKnownToCauseUB(Instruction *I) const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAUndefinedBehavior &createForPosition(const IRPosition &IRP,
-                                                Attributor &A);
+  LLVM_ABI static AAUndefinedBehavior &createForPosition(const IRPosition &IRP,
+                                                         Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAUndefinedBehavior"; }
@@ -3819,7 +3852,7 @@ struct AAUndefinedBehavior
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface to determine reachability of point A to B.
@@ -3836,8 +3869,8 @@ struct AAIntraFnReachability
       const AA::InstExclusionSetTy *ExclusionSet = nullptr) const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAIntraFnReachability &createForPosition(const IRPosition &IRP,
-                                                  Attributor &A);
+  LLVM_ABI static AAIntraFnReachability &
+  createForPosition(const IRPosition &IRP, Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAIntraFnReachability"; }
@@ -3852,7 +3885,7 @@ struct AAIntraFnReachability
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for all noalias attributes.
@@ -3870,9 +3903,9 @@ struct AANoAlias
   }
 
   /// See IRAttribute::isImpliedByIR
-  static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
-                            Attribute::AttrKind ImpliedAttributeKind,
-                            bool IgnoreSubsumingPositions = false);
+  LLVM_ABI static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+                                     Attribute::AttrKind ImpliedAttributeKind,
+                                     bool IgnoreSubsumingPositions = false);
 
   /// See AbstractAttribute::requiresCallersForArgOrFunction
   static bool requiresCallersForArgOrFunction() { return true; }
@@ -3884,7 +3917,8 @@ struct AANoAlias
   bool isKnownNoAlias() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoAlias &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoAlias &createForPosition(const IRPosition &IRP,
+                                               Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoAlias"; }
@@ -3898,7 +3932,7 @@ struct AANoAlias
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An AbstractAttribute for nofree.
@@ -3934,7 +3968,8 @@ struct AANoFree
   bool isKnownNoFree() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoFree &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoFree &createForPosition(const IRPosition &IRP,
+                                              Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoFree"; }
@@ -3948,7 +3983,7 @@ struct AANoFree
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An AbstractAttribute for noreturn.
@@ -3965,7 +4000,8 @@ struct AANoReturn
   bool isKnownNoReturn() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoReturn &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoReturn &createForPosition(const IRPosition &IRP,
+                                                Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoReturn"; }
@@ -3979,7 +4015,7 @@ struct AANoReturn
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for liveness abstract attribute.
@@ -4046,7 +4082,8 @@ struct AAIsDead
 
 public:
   /// Create an abstract attribute view for the position \p IRP.
-  static AAIsDead &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AAIsDead &createForPosition(const IRPosition &IRP,
+                                              Attributor &A);
 
   /// Determine if \p F might catch asynchronous exceptions.
   static bool mayCatchAsynchronousExceptions(const Function &F) {
@@ -4074,7 +4111,7 @@ struct AAIsDead
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 
   friend struct Attributor;
 };
@@ -4255,8 +4292,8 @@ struct AADereferenceable
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AADereferenceable &createForPosition(const IRPosition &IRP,
-                                              Attributor &A);
+  LLVM_ABI static AADereferenceable &createForPosition(const IRPosition &IRP,
+                                                       Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AADereferenceable"; }
@@ -4271,7 +4308,7 @@ struct AADereferenceable
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 using AAAlignmentStateType =
@@ -4308,10 +4345,11 @@ struct AAAlign
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAAlign &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AAAlign &createForPosition(const IRPosition &IRP,
+                                             Attributor &A);
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface to track if a value leaves it's defining function
@@ -4335,8 +4373,8 @@ struct AAInstanceInfo : public StateWrapper<BooleanState, AbstractAttribute> {
   bool isAssumedUniqueForAnalysis() const { return isAssumed(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAInstanceInfo &createForPosition(const IRPosition &IRP,
-                                           Attributor &A);
+  LLVM_ABI static AAInstanceInfo &createForPosition(const IRPosition &IRP,
+                                                    Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAInstanceInfo"; }
@@ -4351,7 +4389,7 @@ struct AAInstanceInfo : public StateWrapper<BooleanState, AbstractAttribute> {
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for all nocapture attributes.
@@ -4363,15 +4401,15 @@ struct AANoCapture
   AANoCapture(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
 
   /// See IRAttribute::isImpliedByIR
-  static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
-                            Attribute::AttrKind ImpliedAttributeKind,
-                            bool IgnoreSubsumingPositions = false);
+  LLVM_ABI static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+                                     Attribute::AttrKind ImpliedAttributeKind,
+                                     bool IgnoreSubsumingPositions = false);
 
   /// Update \p State according to the capture capabilities of \p F for position
   /// \p IRP.
-  static void determineFunctionCaptureCapabilities(const IRPosition &IRP,
-                                                   const Function &F,
-                                                   BitIntegerState &State);
+  LLVM_ABI static void
+  determineFunctionCaptureCapabilities(const IRPosition &IRP, const Function &F,
+                                       BitIntegerState &State);
 
   /// See AbstractAttribute::isValidIRPositionForInit
   static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
@@ -4418,7 +4456,8 @@ struct AANoCapture
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoCapture &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoCapture &createForPosition(const IRPosition &IRP,
+                                                 Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoCapture"; }
@@ -4432,7 +4471,7 @@ struct AANoCapture
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 struct ValueSimplifyStateType : public AbstractState {
@@ -4497,7 +4536,7 @@ struct ValueSimplifyStateType : public AbstractState {
   Type *Ty;
 
   /// Merge \p Other into the currently assumed simplified value
-  bool unionAssumed(std::optional<Value *> Other);
+  LLVM_ABI bool unionAssumed(std::optional<Value *> Other);
 
   /// Helper to track validity and fixpoint
   BooleanState BS;
@@ -4517,8 +4556,8 @@ struct AAValueSimplify
       : Base(IRP, IRP.getAssociatedType()) {}
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAValueSimplify &createForPosition(const IRPosition &IRP,
-                                            Attributor &A);
+  LLVM_ABI static AAValueSimplify &createForPosition(const IRPosition &IRP,
+                                                     Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAValueSimplify"; }
@@ -4533,7 +4572,7 @@ struct AAValueSimplify
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 
 private:
   /// Return an assumed simplified value if a single candidate is found. If
@@ -4559,7 +4598,8 @@ struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute> {
   virtual bool isAssumedHeapToStackRemovedFree(CallBase &CB) const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAHeapToStack &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AAHeapToStack &createForPosition(const IRPosition &IRP,
+                                                   Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAHeapToStack"; }
@@ -4573,7 +4613,7 @@ struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute> {
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for privatizability.
@@ -4613,8 +4653,8 @@ struct AAPrivatizablePtr
   virtual std::optional<Type *> getPrivatizableType() const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAPrivatizablePtr &createForPosition(const IRPosition &IRP,
-                                              Attributor &A);
+  LLVM_ABI static AAPrivatizablePtr &createForPosition(const IRPosition &IRP,
+                                                       Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAPrivatizablePtr"; }
@@ -4629,7 +4669,7 @@ struct AAPrivatizablePtr
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for memory access kind related attributes
@@ -4687,8 +4727,8 @@ struct AAMemoryBehavior
   bool isAssumedWriteOnly() const { return isAssumed(NO_READS); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAMemoryBehavior &createForPosition(const IRPosition &IRP,
-                                             Attributor &A);
+  LLVM_ABI static AAMemoryBehavior &createForPosition(const IRPosition &IRP,
+                                                      Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAMemoryBehavior"; }
@@ -4703,7 +4743,7 @@ struct AAMemoryBehavior
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for all memory location attributes
@@ -4845,7 +4885,7 @@ struct AAMemoryLocation
   };
 
   /// Return the locations encoded by \p MLK as a readable string.
-  static std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK);
+  LLVM_ABI static std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK);
 
   /// Simple enum to distinguish read/write/read-write accesses.
   enum AccessKind {
@@ -4867,8 +4907,8 @@ struct AAMemoryLocation
       MemoryLocationsKind MLK) const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAMemoryLocation &createForPosition(const IRPosition &IRP,
-                                             Attributor &A);
+  LLVM_ABI static AAMemoryLocation &createForPosition(const IRPosition &IRP,
+                                                      Attributor &A);
 
   /// See AbstractState::getAsStr(Attributor).
   const std::string getAsStr(Attributor *A) const override {
@@ -4888,7 +4928,7 @@ struct AAMemoryLocation
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for range value analysis.
@@ -4913,8 +4953,8 @@ struct AAValueConstantRange
   const IntegerRangeState &getState() const override { return *this; }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAValueConstantRange &createForPosition(const IRPosition &IRP,
-                                                 Attributor &A);
+  LLVM_ABI static AAValueConstantRange &createForPosition(const IRPosition &IRP,
+                                                          Attributor &A);
 
   /// Return an assumed range for the associated value a program point \p CtxI.
   /// If \p I is nullptr, simply return an assumed range.
@@ -4956,7 +4996,7 @@ struct AAValueConstantRange
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// A class for a set state.
@@ -5231,9 +5271,10 @@ using PotentialConstantIntValuesState = PotentialValuesState<APInt>;
 using PotentialLLVMValuesState =
     PotentialValuesState<std::pair<AA::ValueAndContext, AA::ValueScope>>;
 
-raw_ostream &operator<<(raw_ostream &OS,
-                        const PotentialConstantIntValuesState &R);
-raw_ostream &operator<<(raw_ostream &OS, const PotentialLLVMValuesState &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const PotentialConstantIntValuesState &R);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const PotentialLLVMValuesState &R);
 
 /// An abstract interface for potential values analysis.
 ///
@@ -5272,8 +5313,8 @@ struct AAPotentialConstantValues
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAPotentialConstantValues &createForPosition(const IRPosition &IRP,
-                                                      Attributor &A);
+  LLVM_ABI static AAPotentialConstantValues &
+  createForPosition(const IRPosition &IRP, Attributor &A);
 
   /// Return assumed constant for the associated value
   std::optional<Constant *>
@@ -5308,7 +5349,7 @@ struct AAPotentialConstantValues
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 struct AAPotentialValues
@@ -5324,13 +5365,14 @@ struct AAPotentialValues
   const PotentialLLVMValuesState &getState() const override { return *this; }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAPotentialValues &createForPosition(const IRPosition &IRP,
-                                              Attributor &A);
+  LLVM_ABI static AAPotentialValues &createForPosition(const IRPosition &IRP,
+                                                       Attributor &A);
 
   /// Extract the single value in \p Values if any.
-  static Value *getSingleValue(Attributor &A, const AbstractAttribute &AA,
-                               const IRPosition &IRP,
-                               SmallVectorImpl<AA::ValueAndContext> &Values);
+  LLVM_ABI static Value *
+  getSingleValue(Attributor &A, const AbstractAttribute &AA,
+                 const IRPosition &IRP,
+                 SmallVectorImpl<AA::ValueAndContext> &Values);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAPotentialValues"; }
@@ -5345,7 +5387,7 @@ struct AAPotentialValues
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 
 private:
   virtual bool getAssumedSimplifiedValues(
@@ -5369,9 +5411,9 @@ struct AANoUndef
   static bool isImpliedByPoison() { return false; }
 
   /// See IRAttribute::isImpliedByIR
-  static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
-                            Attribute::AttrKind ImpliedAttributeKind,
-                            bool IgnoreSubsumingPositions = false);
+  LLVM_ABI static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+                                     Attribute::AttrKind ImpliedAttributeKind,
+                                     bool IgnoreSubsumingPositions = false);
 
   /// Return true if we assume that the underlying value is noundef.
   bool isAssumedNoUndef() const { return getAssumed(); }
@@ -5380,7 +5422,8 @@ struct AANoUndef
   bool isKnownNoUndef() const { return getKnown(); }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoUndef &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoUndef &createForPosition(const IRPosition &IRP,
+                                               Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoUndef"; }
@@ -5394,7 +5437,7 @@ struct AANoUndef
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 struct AANoFPClass
@@ -5431,7 +5474,8 @@ struct AANoFPClass
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANoFPClass &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AANoFPClass &createForPosition(const IRPosition &IRP,
+                                                 Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AANoFPClass"; }
@@ -5445,7 +5489,7 @@ struct AANoFPClass
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 struct AACallGraphNode;
@@ -5461,7 +5505,7 @@ class AACallEdgeIterator
       : iterator_adaptor_base(Begin), A(A) {}
 
 public:
-  AACallGraphNode *operator*() const;
+  LLVM_ABI AACallGraphNode *operator*() const;
 
 private:
   Attributor &A;
@@ -5520,7 +5564,8 @@ struct AACallEdges : public StateWrapper<BooleanState, AbstractAttribute>,
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AACallEdges &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AACallEdges &createForPosition(const IRPosition &IRP,
+                                                 Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AACallEdges"; }
@@ -5534,7 +5579,7 @@ struct AACallEdges : public StateWrapper<BooleanState, AbstractAttribute>,
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 // Synthetic root node for the Attributor's internal call graph.
@@ -5558,7 +5603,7 @@ struct AttributorCallGraph : public AACallGraphNode {
     }
   }
 
-  void print();
+  LLVM_ABI void print();
 };
 
 template <> struct GraphTraits<AACallGraphNode *> {
@@ -5641,8 +5686,8 @@ struct AAExecutionDomain
   };
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAExecutionDomain &createForPosition(const IRPosition &IRP,
-                                              Attributor &A);
+  LLVM_ABI static AAExecutionDomain &createForPosition(const IRPosition &IRP,
+                                                       Attributor &A);
 
   /// See AbstractAttribute::getName().
   StringRef getName() const override { return "AAExecutionDomain"; }
@@ -5682,7 +5727,7 @@ struct AAExecutionDomain
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract Attribute for computing reachability between functions.
@@ -5707,8 +5752,8 @@ struct AAInterFnReachability
       const AA::InstExclusionSetTy *ExclusionSet = nullptr) const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAInterFnReachability &createForPosition(const IRPosition &IRP,
-                                                  Attributor &A);
+  LLVM_ABI static AAInterFnReachability &
+  createForPosition(const IRPosition &IRP, Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAInterFnReachability"; }
@@ -5722,7 +5767,7 @@ struct AAInterFnReachability
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract Attribute for determining the necessity of the convergent
@@ -5733,8 +5778,8 @@ struct AANonConvergent : public StateWrapper<BooleanState, AbstractAttribute> {
   AANonConvergent(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AANonConvergent &createForPosition(const IRPosition &IRP,
-                                            Attributor &A);
+  LLVM_ABI static AANonConvergent &createForPosition(const IRPosition &IRP,
+                                                     Attributor &A);
 
   /// Return true if "non-convergent" is assumed.
   bool isAssumedNotConvergent() const { return getAssumed(); }
@@ -5755,7 +5800,7 @@ struct AANonConvergent : public StateWrapper<BooleanState, AbstractAttribute> {
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for struct information.
@@ -6163,7 +6208,8 @@ struct AAPointerInfo : public AbstractAttribute {
   };
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAPointerInfo &createForPosition(const IRPosition &IRP, Attributor &A);
+  LLVM_ABI static AAPointerInfo &createForPosition(const IRPosition &IRP,
+                                                   Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAPointerInfo"; }
@@ -6208,10 +6254,10 @@ struct AAPointerInfo : public AbstractAttribute {
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
-raw_ostream &operator<<(raw_ostream &, const AAPointerInfo::Access &);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &, const AAPointerInfo::Access &);
 
 /// An abstract attribute for getting assumption information.
 struct AAAssumptionInfo
@@ -6228,8 +6274,8 @@ struct AAAssumptionInfo
   virtual bool hasAssumption(const StringRef Assumption) const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAAssumptionInfo &createForPosition(const IRPosition &IRP,
-                                             Attributor &A);
+  LLVM_ABI static AAAssumptionInfo &createForPosition(const IRPosition &IRP,
+                                                      Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAAssumptionInfo"; }
@@ -6244,7 +6290,7 @@ struct AAAssumptionInfo
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract attribute for getting all assumption underlying objects.
@@ -6262,8 +6308,8 @@ struct AAUnderlyingObjects : AbstractAttribute {
   static bool requiresCallersForArgOrFunction() { return true; }
 
   /// Create an abstract attribute biew for the position \p IRP.
-  static AAUnderlyingObjects &createForPosition(const IRPosition &IRP,
-                                                Attributor &A);
+  LLVM_ABI static AAUnderlyingObjects &createForPosition(const IRPosition &IRP,
+                                                         Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAUnderlyingObjects"; }
@@ -6278,7 +6324,7 @@ struct AAUnderlyingObjects : AbstractAttribute {
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 
   /// Check \p Pred on all underlying objects in \p Scope collected so far.
   ///
@@ -6310,8 +6356,8 @@ struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
   virtual uint32_t getAddressSpace() const = 0;
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAAddressSpace &createForPosition(const IRPosition &IRP,
-                                           Attributor &A);
+  LLVM_ABI static AAAddressSpace &createForPosition(const IRPosition &IRP,
+                                                    Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AAAddressSpace"; }
@@ -6326,7 +6372,7 @@ struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 
 protected:
   // Invalid address space which indicates the associated value is dead.
@@ -6345,8 +6391,8 @@ struct AAAllocationInfo : public StateWrapper<BooleanState, AbstractAttribute> {
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAAllocationInfo &createForPosition(const IRPosition &IRP,
-                                             Attributor &A);
+  LLVM_ABI static AAAllocationInfo &createForPosition(const IRPosition &IRP,
+                                                      Attributor &A);
 
   virtual std::optional<TypeSize> getAllocatedSize() const = 0;
 
@@ -6365,7 +6411,7 @@ struct AAAllocationInfo : public StateWrapper<BooleanState, AbstractAttribute> {
   constexpr static const std::optional<TypeSize> HasNoAllocationSize =
       std::optional<TypeSize>(TypeSize(-1, true));
 
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for llvm::GlobalValue information interference.
@@ -6385,8 +6431,8 @@ struct AAGlobalValueInfo
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAGlobalValueInfo &createForPosition(const IRPosition &IRP,
-                                              Attributor &A);
+  LLVM_ABI static AAGlobalValueInfo &createForPosition(const IRPosition &IRP,
+                                                       Attributor &A);
 
   /// Return true iff \p U is a potential use of the associated global value.
   virtual bool isPotentialUse(const Use &U) const = 0;
@@ -6404,7 +6450,7 @@ struct AAGlobalValueInfo
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract interface for indirect call information interference.
@@ -6423,8 +6469,8 @@ struct AAIndirectCallInfo
   }
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AAIndirectCallInfo &createForPosition(const IRPosition &IRP,
-                                               Attributor &A);
+  LLVM_ABI static AAIndirectCallInfo &createForPosition(const IRPosition &IRP,
+                                                        Attributor &A);
 
   /// Call \CB on each potential callee value and return true if all were known
   /// and \p CB returned true on all of them. Otherwise, return false.
@@ -6445,7 +6491,7 @@ struct AAIndirectCallInfo
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
 /// An abstract Attribute for specializing "dynamic" components of
@@ -6457,8 +6503,8 @@ struct AADenormalFPMath
   AADenormalFPMath(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
 
   /// Create an abstract attribute view for the position \p IRP.
-  static AADenormalFPMath &createForPosition(const IRPosition &IRP,
-                                             Attributor &A);
+  LLVM_ABI static AADenormalFPMath &createForPosition(const IRPosition &IRP,
+                                                      Attributor &A);
 
   /// See AbstractAttribute::getName()
   StringRef getName() const override { return "AADenormalFPMath"; }
@@ -6473,11 +6519,9 @@ struct AADenormalFPMath
   }
 
   /// Unique ID (due to the unique address)
-  static const char ID;
+  LLVM_ABI static const char ID;
 };
 
-raw_ostream &operator<<(raw_ostream &, const AAPointerInfo::Access &);
-
 /// Run options, used by the pass manager.
 enum AttributorRunOption {
   NONE = 0,
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/BlockExtractor.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/BlockExtractor.h
index cf6b1666b4fc..2f8db6f32818 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/BlockExtractor.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/BlockExtractor.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TRANSFORMS_IPO_BLOCKEXTRACTOR_H
 #define LLVM_TRANSFORMS_IPO_BLOCKEXTRACTOR_H
 
+#include "llvm/Support/Compiler.h"
 #include <vector>
 
 #include "llvm/IR/PassManager.h"
@@ -22,9 +23,10 @@ namespace llvm {
 class BasicBlock;
 
 struct BlockExtractorPass : PassInfoMixin<BlockExtractorPass> {
+  LLVM_ABI
   BlockExtractorPass(std::vector<std::vector<BasicBlock *>> &&GroupsOfBlocks,
                      bool EraseFunctions);
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
 private:
   std::vector<std::vector<BasicBlock *>> GroupsOfBlocks;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h
index 12bf0dd6581c..dcd6d5f446c0 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h
@@ -19,6 +19,7 @@
 #define LLVM_TRANSFORMS_IPO_EMBEDBITCODEPASS_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Module;
@@ -44,7 +45,7 @@ class EmbedBitcodePass : public PassInfoMixin<EmbedBitcodePass> {
   EmbedBitcodePass(bool IsThinLTO, bool EmitLTOSummary)
       : IsThinLTO(IsThinLTO), EmitLTOSummary(EmitLTOSummary) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
 
   static bool isRequired() { return true; }
 };
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/ExtractGV.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/ExtractGV.h
index 1c2e7d9465fd..eed7dfb8b6a1 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/ExtractGV.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/ExtractGV.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/SetVector.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class GlobalValue;
@@ -22,9 +23,9 @@ class ExtractGVPass : public PassInfoMixin<ExtractGVPass> {
   bool keepConstInit;
 
 public:
-  ExtractGVPass(std::vector<GlobalValue *> &GVs, bool deleteS = true,
-                bool keepConstInit = false);
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI ExtractGVPass(std::vector<GlobalValue *> &GVs, bool deleteS = true,
+                         bool keepConstInit = false);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
 };
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h
index 6a21ff616d50..754714dceb7a 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h
@@ -19,6 +19,7 @@
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -28,11 +29,12 @@ class Function;
 class Module;
 
 /// Returns the memory access properties of this copy of the function.
-MemoryEffects computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR);
+LLVM_ABI MemoryEffects computeFunctionBodyMemoryAccess(Function &F,
+                                                       AAResults &AAR);
 
 /// Propagate function attributes for function summaries along the index's
 /// callgraph during thinlink
-bool thinLTOPropagateFunctionAttrs(
+LLVM_ABI bool thinLTOPropagateFunctionAttrs(
     ModuleSummaryIndex &Index,
     function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
         isPrevailing);
@@ -49,11 +51,13 @@ bool thinLTOPropagateFunctionAttrs(
 struct PostOrderFunctionAttrsPass : PassInfoMixin<PostOrderFunctionAttrsPass> {
   PostOrderFunctionAttrsPass(bool SkipNonRecursive = false)
       : SkipNonRecursive(SkipNonRecursive) {}
-  PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
-                        LazyCallGraph &CG, CGSCCUpdateResult &UR);
+  LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C,
+                                 CGSCCAnalysisManager &AM, LazyCallGraph &CG,
+                                 CGSCCUpdateResult &UR);
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   bool SkipNonRecursive;
@@ -72,7 +76,7 @@ struct PostOrderFunctionAttrsPass : PassInfoMixin<PostOrderFunctionAttrsPass> {
 class ReversePostOrderFunctionAttrsPass
     : public PassInfoMixin<ReversePostOrderFunctionAttrsPass> {
 public:
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionImport.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionImport.h
index 5e4116834b7f..e6ae9ee831d5 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionImport.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionImport.h
@@ -15,6 +15,7 @@
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <functional>
 #include <memory>
@@ -199,13 +200,14 @@ class FunctionImporter {
 
     // Add the given GUID to ImportList as a definition.  If the same GUID has
     // been added as a declaration previously, that entry is overridden.
-    AddDefinitionStatus addDefinition(StringRef FromModule,
-                                      GlobalValue::GUID GUID);
+    LLVM_ABI AddDefinitionStatus addDefinition(StringRef FromModule,
+                                               GlobalValue::GUID GUID);
 
     // Add the given GUID to ImportList as a declaration.  If the same GUID has
     // been added as a definition previously, that entry takes precedence, and
     // no change is made.
-    void maybeAddDeclaration(StringRef FromModule, GlobalValue::GUID GUID);
+    LLVM_ABI void maybeAddDeclaration(StringRef FromModule,
+                                      GlobalValue::GUID GUID);
 
     void addGUID(StringRef FromModule, GlobalValue::GUID GUID,
                  GlobalValueSummary::ImportKind ImportKind) {
@@ -217,9 +219,9 @@ class FunctionImporter {
 
     // Return the list of source modules sorted in the ascending alphabetical
     // order.
-    SmallVector<StringRef, 0> getSourceModules() const;
+    LLVM_ABI SmallVector<StringRef, 0> getSourceModules() const;
 
-    std::optional<GlobalValueSummary::ImportKind>
+    LLVM_ABI std::optional<GlobalValueSummary::ImportKind>
     getImportType(StringRef FromModule, GlobalValue::GUID GUID) const;
 
     // Iterate over the import list.  The caller gets tuples of FromModule,
@@ -270,8 +272,9 @@ class FunctionImporter {
   // A map from destination modules to lists of imports.
   class ImportListsTy {
   public:
-    ImportListsTy() : EmptyList(ImportIDs) {}
-    ImportListsTy(size_t Size) : EmptyList(ImportIDs), ListsImpl(Size) {}
+    ImportListsTy() : ImportIDs(), EmptyList(ImportIDs) {}
+    ImportListsTy(size_t Size)
+        : ImportIDs(), EmptyList(ImportIDs), ListsImpl(Size) {}
 
     ImportMapTy &operator[](StringRef DestMod) {
       return ListsImpl.try_emplace(DestMod, ImportIDs).first->second;
@@ -291,9 +294,9 @@ class FunctionImporter {
     const_iterator end() const { return ListsImpl.end(); }
 
   private:
+    ImportIDTable ImportIDs;
     ImportMapTy EmptyList;
     DenseMap<StringRef, ImportMapTy> ListsImpl;
-    ImportIDTable ImportIDs;
   };
 
   /// The set contains an entry for every global value that the module exports.
@@ -312,7 +315,8 @@ class FunctionImporter {
         ClearDSOLocalOnDeclarations(ClearDSOLocalOnDeclarations) {}
 
   /// Import functions in Module \p M based on the supplied import list.
-  Expected<bool> importFunctions(Module &M, const ImportMapTy &ImportList);
+  LLVM_ABI Expected<bool> importFunctions(Module &M,
+                                          const ImportMapTy &ImportList);
 
 private:
   /// The summaries index used to trigger importing.
@@ -329,7 +333,7 @@ class FunctionImporter {
 /// The function importing pass
 class FunctionImportPass : public PassInfoMixin<FunctionImportPass> {
 public:
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 /// Compute all the imports and exports for every module in the Index.
@@ -353,7 +357,7 @@ class FunctionImportPass : public PassInfoMixin<FunctionImportPass> {
 /// are owned by the in-memory ModuleSummaryIndex the importing decisions
 /// are made from (the module path for each summary is owned by the index's
 /// module path string table).
-void ComputeCrossModuleImport(
+LLVM_ABI void ComputeCrossModuleImport(
     const ModuleSummaryIndex &Index,
     const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
     function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
@@ -370,7 +374,7 @@ enum class PrevailingType { Yes, No, Unknown };
 /// SamplePGO when needed. Normally this is done during
 /// computeDeadSymbolsAndUpdateIndirectCalls, but can be called standalone
 /// when that is not called (e.g. during testing).
-void updateIndirectCalls(ModuleSummaryIndex &Index);
+LLVM_ABI void updateIndirectCalls(ModuleSummaryIndex &Index);
 
 /// Compute all the symbols that are "dead": i.e these that can't be reached
 /// in the graph from any of the given symbols listed in
@@ -379,14 +383,14 @@ void updateIndirectCalls(ModuleSummaryIndex &Index);
 /// predicate returns status of symbol.
 /// Also update call edges for indirect calls to local functions added from
 /// SamplePGO when needed.
-void computeDeadSymbolsAndUpdateIndirectCalls(
+LLVM_ABI void computeDeadSymbolsAndUpdateIndirectCalls(
     ModuleSummaryIndex &Index,
     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
     function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing);
 
 /// Compute dead symbols and run constant propagation in combined index
 /// after that.
-void computeDeadSymbolsWithConstProp(
+LLVM_ABI void computeDeadSymbolsWithConstProp(
     ModuleSummaryIndex &Index,
     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
     function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
@@ -394,7 +398,7 @@ void computeDeadSymbolsWithConstProp(
 
 /// Converts value \p GV to declaration, or replaces with a declaration if
 /// it is an alias. Returns true if converted, false if replaced.
-bool convertToDeclaration(GlobalValue &GV);
+LLVM_ABI bool convertToDeclaration(GlobalValue &GV);
 
 /// Compute the set of summaries needed for a ThinLTO backend compilation of
 /// \p ModulePath.
@@ -409,7 +413,7 @@ bool convertToDeclaration(GlobalValue &GV);
 ///
 /// \p DecSummaries will be popluated with the subset of of summary pointers
 /// that have 'declaration' import type among all summaries the module need.
-void gatherImportedSummariesForModule(
+LLVM_ABI void gatherImportedSummariesForModule(
     StringRef ModulePath,
     const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
     const FunctionImporter::ImportMapTy &ImportList,
@@ -417,12 +421,12 @@ void gatherImportedSummariesForModule(
     GVSummaryPtrSet &DecSummaries);
 
 /// Emit into \p OutputFilename the files module \p ModulePath will import from.
-Error EmitImportsFiles(
-    StringRef ModulePath, StringRef OutputFilename,
-    const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex);
+LLVM_ABI Error
+EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename,
+                 const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex);
 
 /// Call \p F passing each of the files module \p ModulePath will import from.
-void processImportsFiles(
+LLVM_ABI void processImportsFiles(
     StringRef ModulePath,
     const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex,
     function_ref<void(const std::string &)> F);
@@ -433,14 +437,14 @@ void processImportsFiles(
 ///    and consider visibility from other definitions for ELF) in \p TheModule
 /// 2. (optional) Apply propagated function attributes to \p TheModule if
 ///    PropagateAttrs is true
-void thinLTOFinalizeInModule(Module &TheModule,
-                             const GVSummaryMapTy &DefinedGlobals,
-                             bool PropagateAttrs);
+LLVM_ABI void thinLTOFinalizeInModule(Module &TheModule,
+                                      const GVSummaryMapTy &DefinedGlobals,
+                                      bool PropagateAttrs);
 
 /// Internalize \p TheModule based on the information recorded in the summaries
 /// during global summary-based analysis.
-void thinLTOInternalizeModule(Module &TheModule,
-                              const GVSummaryMapTy &DefinedGlobals);
+LLVM_ABI void thinLTOInternalizeModule(Module &TheModule,
+                                       const GVSummaryMapTy &DefinedGlobals);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
index d15a3d777ab0..1575afa50e19 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
@@ -86,6 +86,7 @@
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/InstVisitor.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Scalar/SCCP.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/SCCPSolver.h"
@@ -179,11 +180,11 @@ class InstCostVisitor : public InstVisitor<InstCostVisitor, Constant *> {
     return Solver.isBlockExecutable(BB) && !DeadBlocks.contains(BB);
   }
 
-  Cost getCodeSizeSavingsForArg(Argument *A, Constant *C);
+  LLVM_ABI Cost getCodeSizeSavingsForArg(Argument *A, Constant *C);
 
-  Cost getCodeSizeSavingsFromPendingPHIs();
+  LLVM_ABI Cost getCodeSizeSavingsFromPendingPHIs();
 
-  Cost getLatencySavingsForKnownConstants();
+  LLVM_ABI Cost getLatencySavingsForKnownConstants();
 
 private:
   friend class InstVisitor<InstCostVisitor, Constant *>;
@@ -260,9 +261,9 @@ class FunctionSpecializer {
       : Solver(Solver), M(M), FAM(FAM), GetBFI(GetBFI), GetTLI(GetTLI),
         GetTTI(GetTTI), GetAC(GetAC) {}
 
-  ~FunctionSpecializer();
+  LLVM_ABI ~FunctionSpecializer();
 
-  bool run();
+  LLVM_ABI bool run();
 
   InstCostVisitor getInstCostVisitorFor(Function *F) {
     auto &TTI = GetTTI(*F);
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/GlobalDCE.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/GlobalDCE.h
index 92c30d4b54a2..f8f775b8ea96 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/GlobalDCE.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/GlobalDCE.h
@@ -21,6 +21,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <unordered_map>
 
 namespace llvm {
@@ -37,10 +38,11 @@ class GlobalDCEPass : public PassInfoMixin<GlobalDCEPass> {
 public:
   GlobalDCEPass(bool InLTOPostLink = false) : InLTOPostLink(InLTOPostLink) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   bool InLTOPostLink = false;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/Inliner.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/Inliner.h
index 401aa2d3a0cc..dd877b5cdc72 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/Inliner.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/Inliner.h
@@ -15,6 +15,7 @@
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -39,11 +40,13 @@ class InlinerPass : public PassInfoMixin<InlinerPass> {
       : OnlyMandatory(OnlyMandatory), LTOPhase(LTOPhase) {}
   InlinerPass(InlinerPass &&Arg) = default;
 
-  PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
-                        LazyCallGraph &CG, CGSCCUpdateResult &UR);
+  LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C,
+                                 CGSCCAnalysisManager &AM, LazyCallGraph &CG,
+                                 CGSCCUpdateResult &UR);
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
@@ -61,14 +64,14 @@ class InlinerPass : public PassInfoMixin<InlinerPass> {
 class ModuleInlinerWrapperPass
     : public PassInfoMixin<ModuleInlinerWrapperPass> {
 public:
-  ModuleInlinerWrapperPass(
+  LLVM_ABI ModuleInlinerWrapperPass(
       InlineParams Params = getInlineParams(), bool MandatoryFirst = true,
       InlineContext IC = {},
       InliningAdvisorMode Mode = InliningAdvisorMode::Default,
       unsigned MaxDevirtIterations = 0);
   ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default;
 
-  PreservedAnalyses run(Module &, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &, ModuleAnalysisManager &);
 
   /// Allow adding more CGSCC passes, besides inlining. This should be called
   /// before run is called, as part of pass pipeline building.
@@ -84,8 +87,9 @@ class ModuleInlinerWrapperPass
     AfterCGMPM.addPass(std::move(Pass));
   }
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   const InlineParams Params;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/Internalize.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/Internalize.h
index c9b2d3ba2d71..9d13f09d7d81 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/Internalize.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/Internalize.h
@@ -24,6 +24,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <functional>
 
 namespace llvm {
@@ -61,15 +62,15 @@ class InternalizePass : public PassInfoMixin<InternalizePass> {
                    DenseMap<const Comdat *, ComdatInfo> &ComdatMap);
 
 public:
-  InternalizePass();
+  LLVM_ABI InternalizePass();
   InternalizePass(std::function<bool(const GlobalValue &)> MustPreserveGV)
       : MustPreserveGV(std::move(MustPreserveGV)) {}
 
   /// Run the internalizer on \p TheModule, returns true if any changes was
   /// made.
-  bool internalizeModule(Module &TheModule);
+  LLVM_ABI bool internalizeModule(Module &TheModule);
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 /// Helper function to internalize functions and variables in a Module.
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
index 02adcd8bfd45..253d13f81857 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <cstring>
 #include <limits>
@@ -53,9 +54,9 @@ struct BitSetInfo {
     return Bits.size() == BitSize;
   }
 
-  bool containsGlobalOffset(uint64_t Offset) const;
+  LLVM_ABI bool containsGlobalOffset(uint64_t Offset) const;
 
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 };
 
 struct BitSetBuilder {
@@ -74,7 +75,7 @@ struct BitSetBuilder {
     Offsets.push_back(Offset);
   }
 
-  BitSetInfo build();
+  LLVM_ABI BitSetInfo build();
 };
 
 /// This class implements a layout algorithm for globals referenced by bit sets
@@ -137,7 +138,7 @@ struct GlobalLayoutBuilder {
   /// Add F to the layout while trying to keep its indices contiguous.
   /// If a previously seen fragment uses any of F's indices, that
   /// fragment will be laid out inside F.
-  void addFragment(const std::set<uint64_t> &F);
+  LLVM_ABI void addFragment(const std::set<uint64_t> &F);
 };
 
 /// This class is used to build a byte array containing overlapping bit sets. By
@@ -189,11 +190,11 @@ struct ByteArrayBuilder {
   /// AllocMask is set to the bitmask for those bits. This uses the LPT (Longest
   /// Processing Time) multiprocessor scheduling algorithm to lay out the bits
   /// efficiently; the pass allocates bit sets in decreasing size order.
-  void allocate(const std::set<uint64_t> &Bits, uint64_t BitSize,
-                uint64_t &AllocByteOffset, uint8_t &AllocMask);
+  LLVM_ABI void allocate(const std::set<uint64_t> &Bits, uint64_t BitSize,
+                         uint64_t &AllocByteOffset, uint8_t &AllocMask);
 };
 
-bool isJumpTableCanonical(Function *F);
+LLVM_ABI bool isJumpTableCanonical(Function *F);
 
 /// Specifies how to drop type tests.
 enum class DropTestKind {
@@ -220,7 +221,12 @@ class LowerTypeTestsPass : public PassInfoMixin<LowerTypeTestsPass> {
                          lowertypetests::DropTestKind::None)
       : ExportSummary(ExportSummary), ImportSummary(ImportSummary),
         DropTypeTests(DropTypeTests) {}
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+class SimplifyTypeTestsPass : public PassInfoMixin<SimplifyTypeTestsPass> {
+public:
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/MergeFunctions.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/MergeFunctions.h
index 71f175c6472b..559d5a77618a 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/MergeFunctions.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/MergeFunctions.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_IPO_MERGEFUNCTIONS_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -25,10 +26,10 @@ class Function;
 /// Merge identical functions.
 class MergeFunctionsPass : public PassInfoMixin<MergeFunctionsPass> {
 public:
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
-  static bool runOnModule(Module &M);
-  static DenseMap<Function *, Function *>
+  LLVM_ABI static bool runOnModule(Module &M);
+  LLVM_ABI static DenseMap<Function *, Function *>
   runOnFunctions(ArrayRef<Function *> F);
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/ModuleInliner.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/ModuleInliner.h
index 24cfff6083ff..6b09d18ee81c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/ModuleInliner.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/ModuleInliner.h
@@ -12,6 +12,7 @@
 #include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -32,7 +33,7 @@ class ModuleInlinerPass : public PassInfoMixin<ModuleInlinerPass> {
       : Params(Params), Mode(Mode), LTOPhase(LTOPhase){};
   ModuleInlinerPass(ModuleInlinerPass &&Arg) = default;
 
-  PreservedAnalyses run(Module &, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &, ModuleAnalysisManager &);
 
 private:
   InlineAdvisor &getAdvisor(const ModuleAnalysisManager &MAM,
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 626a7a09084e..1fececbef2ad 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/Compiler.h"
 #include <map>
 #include <queue>
 #include <vector>
@@ -39,25 +40,27 @@ class ContextTrieNode {
                   LineLocation CallLoc = {0, 0})
       : ParentContext(Parent), FuncName(FName), FuncSamples(FSamples),
         CallSiteLoc(CallLoc){};
-  ContextTrieNode *getChildContext(const LineLocation &CallSite,
+  LLVM_ABI ContextTrieNode *getChildContext(const LineLocation &CallSite,
+                                            FunctionId ChildName);
+  LLVM_ABI ContextTrieNode *
+  getHottestChildContext(const LineLocation &CallSite);
+  LLVM_ABI ContextTrieNode *
+  getOrCreateChildContext(const LineLocation &CallSite, FunctionId ChildName,
+                          bool AllowCreate = true);
+  LLVM_ABI void removeChildContext(const LineLocation &CallSite,
                                    FunctionId ChildName);
-  ContextTrieNode *getHottestChildContext(const LineLocation &CallSite);
-  ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite,
-                                           FunctionId ChildName,
-                                           bool AllowCreate = true);
-  void removeChildContext(const LineLocation &CallSite, FunctionId ChildName);
-  std::map<uint64_t, ContextTrieNode> &getAllChildContext();
-  FunctionId getFuncName() const;
-  FunctionSamples *getFunctionSamples() const;
-  void setFunctionSamples(FunctionSamples *FSamples);
-  std::optional<uint32_t> getFunctionSize() const;
-  void addFunctionSize(uint32_t FSize);
-  LineLocation getCallSiteLoc() const;
-  ContextTrieNode *getParentContext() const;
-  void setParentContext(ContextTrieNode *Parent);
-  void setCallSiteLoc(const LineLocation &Loc);
-  void dumpNode();
-  void dumpTree();
+  LLVM_ABI std::map<uint64_t, ContextTrieNode> &getAllChildContext();
+  LLVM_ABI FunctionId getFuncName() const;
+  LLVM_ABI FunctionSamples *getFunctionSamples() const;
+  LLVM_ABI void setFunctionSamples(FunctionSamples *FSamples);
+  LLVM_ABI std::optional<uint32_t> getFunctionSize() const;
+  LLVM_ABI void addFunctionSize(uint32_t FSize);
+  LLVM_ABI LineLocation getCallSiteLoc() const;
+  LLVM_ABI ContextTrieNode *getParentContext() const;
+  LLVM_ABI void setParentContext(ContextTrieNode *Parent);
+  LLVM_ABI void setCallSiteLoc(const LineLocation &Loc);
+  LLVM_ABI void dumpNode();
+  LLVM_ABI void dumpTree();
 
 private:
   // Map line+discriminator location to child context
@@ -91,48 +94,51 @@ class SampleContextTracker {
   using ContextSamplesTy = std::vector<FunctionSamples *>;
 
   SampleContextTracker() = default;
+  LLVM_ABI
   SampleContextTracker(SampleProfileMap &Profiles,
                        const DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap);
   // Populate the FuncToCtxtProfiles map after the trie is built.
-  void populateFuncToCtxtMap();
+  LLVM_ABI void populateFuncToCtxtMap();
   // Query context profile for a specific callee with given name at a given
   // call-site. The full context is identified by location of call instruction.
-  FunctionSamples *getCalleeContextSamplesFor(const CallBase &Inst,
-                                              StringRef CalleeName);
+  LLVM_ABI FunctionSamples *getCalleeContextSamplesFor(const CallBase &Inst,
+                                                       StringRef CalleeName);
   // Get samples for indirect call targets for call site at given location.
-  std::vector<const FunctionSamples *>
+  LLVM_ABI std::vector<const FunctionSamples *>
   getIndirectCalleeContextSamplesFor(const DILocation *DIL);
   // Query context profile for a given location. The full context
   // is identified by input DILocation.
-  FunctionSamples *getContextSamplesFor(const DILocation *DIL);
+  LLVM_ABI FunctionSamples *getContextSamplesFor(const DILocation *DIL);
   // Query context profile for a given sample contxt of a function.
-  FunctionSamples *getContextSamplesFor(const SampleContext &Context);
+  LLVM_ABI FunctionSamples *getContextSamplesFor(const SampleContext &Context);
   // Get all context profile for given function.
-  ContextSamplesTy &getAllContextSamplesFor(const Function &Func);
-  ContextSamplesTy &getAllContextSamplesFor(StringRef Name);
-  ContextTrieNode *getOrCreateContextPath(const SampleContext &Context,
-                                          bool AllowCreate);
+  LLVM_ABI ContextSamplesTy &getAllContextSamplesFor(const Function &Func);
+  LLVM_ABI ContextSamplesTy &getAllContextSamplesFor(StringRef Name);
+  LLVM_ABI ContextTrieNode *getOrCreateContextPath(const SampleContext &Context,
+                                                   bool AllowCreate);
   // Query base profile for a given function. A base profile is a merged view
   // of all context profiles for contexts that are not inlined.
-  FunctionSamples *getBaseSamplesFor(const Function &Func,
-                                     bool MergeContext = true);
+  LLVM_ABI FunctionSamples *getBaseSamplesFor(const Function &Func,
+                                              bool MergeContext = true);
   // Query base profile for a given function by name.
-  FunctionSamples *getBaseSamplesFor(FunctionId Name,
-                                     bool MergeContext = true);
+  LLVM_ABI FunctionSamples *getBaseSamplesFor(FunctionId Name,
+                                              bool MergeContext = true);
   // Retrieve the context trie node for given profile context
-  ContextTrieNode *getContextFor(const SampleContext &Context);
+  LLVM_ABI ContextTrieNode *getContextFor(const SampleContext &Context);
   // Get real function name for a given trie node.
-  StringRef getFuncNameFor(ContextTrieNode *Node) const;
+  LLVM_ABI StringRef getFuncNameFor(ContextTrieNode *Node) const;
   // Mark a context profile as inlined when function is inlined.
   // This makes sure that inlined context profile will be excluded in
   // function's base profile.
-  void markContextSamplesInlined(const FunctionSamples *InlinedSamples);
-  ContextTrieNode &getRootContext();
-  void promoteMergeContextSamplesTree(const Instruction &Inst,
-                                      FunctionId CalleeName);
+  LLVM_ABI void
+  markContextSamplesInlined(const FunctionSamples *InlinedSamples);
+  LLVM_ABI ContextTrieNode &getRootContext();
+  LLVM_ABI void promoteMergeContextSamplesTree(const Instruction &Inst,
+                                               FunctionId CalleeName);
 
   // Create a merged conext-less profile map.
-  void createContextLessProfileMap(SampleProfileMap &ContextLessProfiles);
+  LLVM_ABI void
+  createContextLessProfileMap(SampleProfileMap &ContextLessProfiles);
   ContextTrieNode *
   getContextNodeForProfile(const FunctionSamples *FSamples) const {
     auto I = ProfileToNodeMap.find(FSamples);
@@ -185,7 +191,7 @@ class SampleContextTracker {
   std::string getContextString(ContextTrieNode *Node) const;
 #endif
   // Dump the internal context profile trie.
-  void dump();
+  LLVM_ABI void dump();
 
 private:
   ContextTrieNode *getContextFor(const DILocation *DIL);
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfile.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfile.h
index 6f4f9701bae9..5cb57f5b109d 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfile.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfile.h
@@ -18,18 +18,19 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 
 namespace llvm {
 
 class Module;
 
-extern cl::opt<int> SampleHotCallSiteThreshold;
-extern cl::opt<int> SampleColdCallSiteThreshold;
-extern cl::opt<int> ProfileInlineGrowthLimit;
-extern cl::opt<int> ProfileInlineLimitMin;
-extern cl::opt<int> ProfileInlineLimitMax;
-extern cl::opt<bool> SortProfiledSCC;
+LLVM_ABI extern cl::opt<int> SampleHotCallSiteThreshold;
+LLVM_ABI extern cl::opt<int> SampleColdCallSiteThreshold;
+LLVM_ABI extern cl::opt<int> ProfileInlineGrowthLimit;
+LLVM_ABI extern cl::opt<int> ProfileInlineLimitMin;
+LLVM_ABI extern cl::opt<int> ProfileInlineLimitMax;
+LLVM_ABI extern cl::opt<bool> SortProfiledSCC;
 
 namespace vfs {
 class FileSystem;
@@ -38,14 +39,14 @@ class FileSystem;
 /// The sample profiler data loader pass.
 class SampleProfileLoaderPass : public PassInfoMixin<SampleProfileLoaderPass> {
 public:
-  SampleProfileLoaderPass(
+  LLVM_ABI SampleProfileLoaderPass(
       std::string File = "", std::string RemappingFile = "",
       ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None,
       IntrusiveRefCntPtr<vfs::FileSystem> FS = nullptr,
       bool DisableSampleProfileInlining = false,
       bool UseFlattenedProfile = false);
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
 private:
   std::string ProfileFileName;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h
index 2a47581f1ad9..f05d0c532aa7 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h
@@ -16,9 +16,10 @@
 #define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
 
 #include "llvm/Analysis/LazyCallGraph.h"
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/Compiler.h"
 #include <unordered_map>
 
 namespace llvm {
@@ -47,10 +48,10 @@ using FuncProbeFactorMap = StringMap<ProbeFactorMap>;
 // function pass, the factor sum for a probe would be typically 100%.
 class PseudoProbeVerifier {
 public:
-  void registerCallbacks(PassInstrumentationCallbacks &PIC);
+  LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC);
 
   // Implementation of pass instrumentation callbacks for new pass manager.
-  void runAfterPass(StringRef PassID, Any IR);
+  LLVM_ABI void runAfterPass(StringRef PassID, Any IR);
 
 private:
   // Allow a little bias due the rounding to integral factors.
@@ -74,8 +75,8 @@ class PseudoProbeVerifier {
 class SampleProfileProber {
 public:
   // Give an empty module id when the prober is not used for instrumentation.
-  SampleProfileProber(Function &F);
-  void instrumentOneFunc(Function &F, TargetMachine *TM);
+  LLVM_ABI SampleProfileProber(Function &F);
+  LLVM_ABI void instrumentOneFunc(Function &F, TargetMachine *TM);
 
 private:
   Function *getFunction() const { return F; }
@@ -117,7 +118,7 @@ class SampleProfileProbePass : public PassInfoMixin<SampleProfileProbePass> {
 
 public:
   SampleProfileProbePass(TargetMachine *TM) : TM(TM) {}
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 // Pseudo probe distribution factor updater.
@@ -137,7 +138,7 @@ class PseudoProbeUpdatePass : public PassInfoMixin<PseudoProbeUpdatePass> {
 
 public:
   PseudoProbeUpdatePass() = default;
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h
index 4a2eaad63113..f307da31bbb2 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripDeadPrototypes.h
@@ -17,6 +17,7 @@
 #define LLVM_TRANSFORMS_IPO_STRIPDEADPROTOTYPES_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -24,7 +25,7 @@ class Module;
 
 /// Pass to remove unused function declarations.
 struct StripDeadPrototypesPass : PassInfoMixin<StripDeadPrototypesPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
 };
 
 }
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripSymbols.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripSymbols.h
index bd5cdde290dd..7843c2ebc38f 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripSymbols.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/StripSymbols.h
@@ -23,27 +23,28 @@
 #define LLVM_TRANSFORMS_IPO_STRIPSYMBOLS_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 struct StripSymbolsPass : PassInfoMixin<StripSymbolsPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 struct StripNonDebugSymbolsPass : PassInfoMixin<StripNonDebugSymbolsPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 struct StripDebugDeclarePass : PassInfoMixin<StripDebugDeclarePass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 struct StripDeadDebugInfoPass : PassInfoMixin<StripDeadDebugInfoPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 struct StripDeadCGProfilePass : PassInfoMixin<StripDeadCGProfilePass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h
index b8fed10404df..a21db26f1edb 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h
@@ -16,6 +16,7 @@
 #ifndef LLVM_TRANSFORMS_IPO_THINLTOBITCODEWRITER_H
 #define LLVM_TRANSFORMS_IPO_THINLTOBITCODEWRITER_H
 
+#include "llvm/Support/Compiler.h"
 #include <llvm/IR/PassManager.h>
 
 namespace llvm {
@@ -36,7 +37,7 @@ class ThinLTOBitcodeWriterPass
       : OS(OS), ThinLinkOS(ThinLinkOS),
         ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
   static bool isRequired() { return true; }
 };
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/external/llvm-project/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
index 415bb65352da..7a03405b4f46 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 #include <map>
@@ -112,13 +113,13 @@ struct TypeMemberInfo {
   uint64_t Offset;
 
   bool operator<(const TypeMemberInfo &other) const {
-    return Bits < other.Bits || (Bits == other.Bits && Offset < other.Offset);
+    return std::tie(Bits, Offset) < std::tie(other.Bits, other.Offset);
   }
 };
 
 // A virtual call target, i.e. an entry in a particular vtable.
 struct VirtualCallTarget {
-  VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM);
+  LLVM_ABI VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM);
 
   // For testing only.
   VirtualCallTarget(const TypeMemberInfo *TM, bool IsBigEndian)
@@ -202,22 +203,22 @@ struct VirtualCallTarget {
 // Find the minimum offset that we may store a value of size Size bits at. If
 // IsAfter is set, look for an offset before the object, otherwise look for an
 // offset after the object.
-uint64_t findLowestOffset(ArrayRef<VirtualCallTarget> Targets, bool IsAfter,
-                          uint64_t Size);
+LLVM_ABI uint64_t findLowestOffset(ArrayRef<VirtualCallTarget> Targets,
+                                   bool IsAfter, uint64_t Size);
 
 // Set the stored value in each of Targets to VirtualCallTarget::RetVal at the
 // given allocation offset before the vtable address. Stores the computed
 // byte/bit offset to OffsetByte/OffsetBit.
-void setBeforeReturnValues(MutableArrayRef<VirtualCallTarget> Targets,
-                           uint64_t AllocBefore, unsigned BitWidth,
-                           int64_t &OffsetByte, uint64_t &OffsetBit);
+LLVM_ABI void setBeforeReturnValues(MutableArrayRef<VirtualCallTarget> Targets,
+                                    uint64_t AllocBefore, unsigned BitWidth,
+                                    int64_t &OffsetByte, uint64_t &OffsetBit);
 
 // Set the stored value in each of Targets to VirtualCallTarget::RetVal at the
 // given allocation offset after the vtable address. Stores the computed
 // byte/bit offset to OffsetByte/OffsetBit.
-void setAfterReturnValues(MutableArrayRef<VirtualCallTarget> Targets,
-                          uint64_t AllocAfter, unsigned BitWidth,
-                          int64_t &OffsetByte, uint64_t &OffsetBit);
+LLVM_ABI void setAfterReturnValues(MutableArrayRef<VirtualCallTarget> Targets,
+                                   uint64_t AllocAfter, unsigned BitWidth,
+                                   int64_t &OffsetByte, uint64_t &OffsetBit);
 
 } // end namespace wholeprogramdevirt
 
@@ -232,27 +233,28 @@ struct WholeProgramDevirtPass : public PassInfoMixin<WholeProgramDevirtPass> {
       : ExportSummary(ExportSummary), ImportSummary(ImportSummary) {
     assert(!(ExportSummary && ImportSummary));
   }
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
 };
 
 struct VTableSlotSummary {
   StringRef TypeID;
   uint64_t ByteOffset;
 };
-bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO);
-void updatePublicTypeTestCalls(Module &M,
-                               bool WholeProgramVisibilityEnabledInLTO);
-void updateVCallVisibilityInModule(
+LLVM_ABI bool
+hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO);
+LLVM_ABI void
+updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO);
+LLVM_ABI void updateVCallVisibilityInModule(
     Module &M, bool WholeProgramVisibilityEnabledInLTO,
     const DenseSet<GlobalValue::GUID> &DynamicExportSymbols,
     bool ValidateAllVtablesHaveTypeInfos,
     function_ref<bool(StringRef)> IsVisibleToRegularObj);
-void updateVCallVisibilityInIndex(
+LLVM_ABI void updateVCallVisibilityInIndex(
     ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO,
     const DenseSet<GlobalValue::GUID> &DynamicExportSymbols,
     const DenseSet<GlobalValue::GUID> &VisibleToRegularObjSymbols);
 
-void getVisibleToRegularObjVtableGUIDs(
+LLVM_ABI void getVisibleToRegularObjVtableGUIDs(
     ModuleSummaryIndex &Index,
     DenseSet<GlobalValue::GUID> &VisibleToRegularObjSymbols,
     function_ref<bool(StringRef)> IsVisibleToRegularObj);
@@ -264,13 +266,13 @@ void getVisibleToRegularObjVtableGUIDs(
 /// locating the corresponding WPD resolution is recorded for the ValueInfo
 /// in case it is exported by cross module importing (in which case the
 /// devirtualized target name will need adjustment).
-void runWholeProgramDevirtOnIndex(
+LLVM_ABI void runWholeProgramDevirtOnIndex(
     ModuleSummaryIndex &Summary, std::set<GlobalValue::GUID> &ExportedGUIDs,
     std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap);
 
 /// Call after cross-module importing to update the recorded single impl
 /// devirt target names for any locals that were exported.
-void updateIndexWPDForExports(
+LLVM_ABI void updateIndexWPDForExports(
     ModuleSummaryIndex &Summary,
     function_ref<bool(StringRef, ValueInfo)> isExported,
     std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap);
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombine.h b/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
index c12d749709cd..1f758b8015ef 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
@@ -19,6 +19,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 
 #define DEBUG_TYPE "instcombine"
 #include "llvm/Transforms/Utils/InstructionWorklist.h"
@@ -52,18 +53,19 @@ class InstCombinePass : public PassInfoMixin<InstCombinePass> {
   static char ID;
 
 public:
-  explicit InstCombinePass(InstCombineOptions Opts = {});
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI explicit InstCombinePass(InstCombineOptions Opts = {});
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 /// The legacy pass manager's instcombine pass.
 ///
 /// This is a basic whole-function wrapper around the instcombine utility. It
 /// will try to combine all instructions in the function.
-class InstructionCombiningPass : public FunctionPass {
+class LLVM_ABI InstructionCombiningPass : public FunctionPass {
   InstructionWorklist Worklist;
 
 public:
@@ -87,7 +89,7 @@ class InstructionCombiningPass : public FunctionPass {
 // into:
 //    %Z = add int 2, %X
 //
-FunctionPass *createInstructionCombiningPass();
+LLVM_ABI FunctionPass *createInstructionCombiningPass();
 }
 
 #undef DEBUG_TYPE
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
index fa6b60cba15a..fa313f529077 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
@@ -430,36 +430,39 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
   /// methods should return the value returned by this function.
   virtual Instruction *eraseInstFromFunction(Instruction &I) = 0;
 
-  void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth,
-                        const Instruction *CxtI) const {
-    llvm::computeKnownBits(V, Known, Depth, SQ.getWithInstruction(CxtI));
+  void computeKnownBits(const Value *V, KnownBits &Known,
+                        const Instruction *CxtI, unsigned Depth = 0) const {
+    llvm::computeKnownBits(V, Known, SQ.getWithInstruction(CxtI), Depth);
   }
 
-  KnownBits computeKnownBits(const Value *V, unsigned Depth,
-                             const Instruction *CxtI) const {
-    return llvm::computeKnownBits(V, Depth, SQ.getWithInstruction(CxtI));
+  KnownBits computeKnownBits(const Value *V, const Instruction *CxtI,
+                             unsigned Depth = 0) const {
+    return llvm::computeKnownBits(V, SQ.getWithInstruction(CxtI), Depth);
   }
 
   bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero = false,
-                              unsigned Depth = 0,
-                              const Instruction *CxtI = nullptr) {
-    return llvm::isKnownToBeAPowerOfTwo(V, OrZero, Depth,
-                                        SQ.getWithInstruction(CxtI));
+                              const Instruction *CxtI = nullptr,
+                              unsigned Depth = 0) {
+    return llvm::isKnownToBeAPowerOfTwo(V, OrZero, SQ.getWithInstruction(CxtI),
+                                        Depth);
   }
 
-  bool MaskedValueIsZero(const Value *V, const APInt &Mask, unsigned Depth = 0,
-                         const Instruction *CxtI = nullptr) const {
+  bool MaskedValueIsZero(const Value *V, const APInt &Mask,
+                         const Instruction *CxtI = nullptr,
+                         unsigned Depth = 0) const {
     return llvm::MaskedValueIsZero(V, Mask, SQ.getWithInstruction(CxtI), Depth);
   }
 
-  unsigned ComputeNumSignBits(const Value *Op, unsigned Depth = 0,
-                              const Instruction *CxtI = nullptr) const {
-    return llvm::ComputeNumSignBits(Op, DL, Depth, &AC, CxtI, &DT);
+  unsigned ComputeNumSignBits(const Value *Op,
+                              const Instruction *CxtI = nullptr,
+                              unsigned Depth = 0) const {
+    return llvm::ComputeNumSignBits(Op, DL, &AC, CxtI, &DT, Depth);
   }
 
-  unsigned ComputeMaxSignificantBits(const Value *Op, unsigned Depth = 0,
-                                     const Instruction *CxtI = nullptr) const {
-    return llvm::ComputeMaxSignificantBits(Op, DL, Depth, &AC, CxtI, &DT);
+  unsigned ComputeMaxSignificantBits(const Value *Op,
+                                     const Instruction *CxtI = nullptr,
+                                     unsigned Depth = 0) const {
+    return llvm::ComputeMaxSignificantBits(Op, DL, &AC, CxtI, &DT, Depth);
   }
 
   OverflowResult computeOverflowForUnsignedMul(const Value *LHS,
@@ -507,12 +510,13 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
 
   virtual bool SimplifyDemandedBits(Instruction *I, unsigned OpNo,
                                     const APInt &DemandedMask, KnownBits &Known,
-                                    unsigned Depth, const SimplifyQuery &Q) = 0;
+                                    const SimplifyQuery &Q,
+                                    unsigned Depth = 0) = 0;
 
   bool SimplifyDemandedBits(Instruction *I, unsigned OpNo,
                             const APInt &DemandedMask, KnownBits &Known) {
     return SimplifyDemandedBits(I, OpNo, DemandedMask, Known,
-                                /*Depth=*/0, SQ.getWithInstruction(I));
+                                SQ.getWithInstruction(I));
   }
 
   virtual Value *
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h
index 1b85766f6cb7..2a9fe91b32f3 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h
@@ -14,6 +14,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_ADDRESSSANITIZER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Instrumentation/AddressSanitizerOptions.h"
 
 namespace llvm {
@@ -38,13 +39,15 @@ struct AddressSanitizerOptions {
 /// run intependently of the function address sanitizer.
 class AddressSanitizerPass : public PassInfoMixin<AddressSanitizerPass> {
 public:
+  LLVM_ABI
   AddressSanitizerPass(const AddressSanitizerOptions &Options,
                        bool UseGlobalGC = true, bool UseOdrIndicator = true,
                        AsanDtorKind DestructorKind = AsanDtorKind::Global,
                        AsanCtorKind ConstructorKind = AsanCtorKind::Global);
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
   static bool isRequired() { return true; }
 
 private:
@@ -61,8 +64,9 @@ struct ASanAccessInfo {
   const bool IsWrite;
   const bool CompileKernel;
 
-  explicit ASanAccessInfo(int32_t Packed);
-  ASanAccessInfo(bool IsWrite, bool CompileKernel, uint8_t AccessSizeIndex);
+  LLVM_ABI explicit ASanAccessInfo(int32_t Packed);
+  LLVM_ABI ASanAccessInfo(bool IsWrite, bool CompileKernel,
+                          uint8_t AccessSizeIndex);
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h
index ab2dcee06551..8e7df5e6b10f 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h
@@ -10,6 +10,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_BOUNDSCHECKING_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -33,10 +34,11 @@ class BoundsCheckingPass : public PassInfoMixin<BoundsCheckingPass> {
   };
 
   BoundsCheckingPass(Options Opts) : Opts(Opts) {}
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
   static bool isRequired() { return true; }
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   Options Opts;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h
index 3256dddd12b3..af3662e4a656 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h
@@ -9,6 +9,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_DATAFLOWSANITIZER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <string>
 #include <vector>
 
@@ -23,7 +24,7 @@ class DataFlowSanitizerPass : public PassInfoMixin<DataFlowSanitizerPass> {
   DataFlowSanitizerPass(
       const std::vector<std::string> &ABIListFiles = std::vector<std::string>())
       : ABIListFiles(ABIListFiles) {}
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h
index f933332cee0c..4d3ead29af0d 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/GCOVProfiler.h
@@ -13,6 +13,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_GCOVPROFILER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/Instrumentation.h"
 
 namespace llvm {
@@ -20,7 +21,7 @@ namespace llvm {
 class GCOVProfilerPass : public PassInfoMixin<GCOVProfilerPass> {
 public:
   GCOVProfilerPass(const GCOVOptions &Options = GCOVOptions::getDefault()) : GCOVOpts(Options) { }
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
 private:
   GCOVOptions GCOVOpts;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h
index 11ea66780d8c..d9292ea7d161 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Module;
@@ -40,10 +41,11 @@ class HWAddressSanitizerPass : public PassInfoMixin<HWAddressSanitizerPass> {
 public:
   explicit HWAddressSanitizerPass(HWAddressSanitizerOptions Options)
       : Options(Options){};
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
   static bool isRequired() { return true; }
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   HWAddressSanitizerOptions Options;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index 2042700ad6b6..469c12ee6965 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -14,6 +14,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_INSTRPROFILING_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/Instrumentation.h"
 
 namespace llvm {
@@ -33,7 +34,7 @@ class InstrProfilingLoweringPass
   InstrProfilingLoweringPass(const InstrProfOptions &Options, bool IsCS = false)
       : Options(Options), IsCS(IsCS) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/KCFI.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/KCFI.h
index 9caa644f4747..2711930a20b5 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/KCFI.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/KCFI.h
@@ -15,12 +15,13 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_KCFI_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class KCFIPass : public PassInfoMixin<KCFIPass> {
 public:
   static bool isRequired() { return true; }
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 } // namespace llvm
 #endif // LLVM_TRANSFORMS_INSTRUMENTATION_KCFI_H
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
index 2c6e60138f2a..7b095ea24b4b 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
@@ -17,6 +17,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -30,11 +31,12 @@ class LowerAllowCheckPass : public PassInfoMixin<LowerAllowCheckPass> {
 
   explicit LowerAllowCheckPass(LowerAllowCheckPass::Options Opts)
       : Opts(std::move(Opts)) {};
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
-  static bool IsRequested();
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI static bool IsRequested();
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
 private:
   LowerAllowCheckPass::Options Opts;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfInstrumentation.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfInstrumentation.h
new file mode 100644
index 000000000000..521491b87a1e
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfInstrumentation.h
@@ -0,0 +1,48 @@
+//===--- MemProfInstrumentation.h - Memory profiler instrumentation ----*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the MemProf instrumentation pass classes.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROF_INSTRUMENTATION_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROF_INSTRUMENTATION_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+class Function;
+class Module;
+
+/// Public interface to the memory profiler pass for instrumenting code to
+/// profile memory accesses.
+///
+/// The profiler itself is a function pass that works by inserting various
+/// calls to the MemProfiler runtime library functions. The runtime library
+/// essentially replaces malloc() and free() with custom implementations that
+/// record data about the allocations.
+class MemProfilerPass : public PassInfoMixin<MemProfilerPass> {
+public:
+  LLVM_ABI explicit MemProfilerPass();
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  static bool isRequired() { return true; }
+};
+
+/// Public interface to the memory profiler module pass for instrumenting code
+/// to profile memory allocations and accesses.
+class ModuleMemProfilerPass : public PassInfoMixin<ModuleMemProfilerPass> {
+public:
+  LLVM_ABI explicit ModuleMemProfilerPass();
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
+};
+
+} // namespace llvm
+
+#endif
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfUse.h
similarity index 52%
rename from external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h
rename to external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfUse.h
index 169f757e580d..6170bf48e469 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemProfUse.h
@@ -1,4 +1,4 @@
-//===--------- Definition of the MemProfiler class --------------*- C++ -*-===//
+//===--------- MemProfUse.h - Memory profiler use pass ----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,20 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file declares the MemProfiler class.
+// This file declares the MemProfUsePass class and related utilities.
 //
 //===----------------------------------------------------------------------===//
-#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFILER_H
-#define LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFILER_H
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFUSE_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_MEMPROFUSE_H
 
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/ProfileData/MemProf.h"
+#include "llvm/Support/Compiler.h"
 
 #include <unordered_map>
 
 namespace llvm {
-class Function;
 class IndexedInstrProfReader;
 class Module;
 class TargetLibraryInfo;
@@ -28,34 +28,12 @@ namespace vfs {
 class FileSystem;
 } // namespace vfs
 
-/// Public interface to the memory profiler pass for instrumenting code to
-/// profile memory accesses.
-///
-/// The profiler itself is a function pass that works by inserting various
-/// calls to the MemProfiler runtime library functions. The runtime library
-/// essentially replaces malloc() and free() with custom implementations that
-/// record data about the allocations.
-class MemProfilerPass : public PassInfoMixin<MemProfilerPass> {
-public:
-  explicit MemProfilerPass();
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-  static bool isRequired() { return true; }
-};
-
-/// Public interface to the memory profiler module pass for instrumenting code
-/// to profile memory allocations and accesses.
-class ModuleMemProfilerPass : public PassInfoMixin<ModuleMemProfilerPass> {
-public:
-  explicit ModuleMemProfilerPass();
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-  static bool isRequired() { return true; }
-};
-
 class MemProfUsePass : public PassInfoMixin<MemProfUsePass> {
 public:
-  explicit MemProfUsePass(std::string MemoryProfileFile,
-                          IntrusiveRefCntPtr<vfs::FileSystem> FS = nullptr);
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI explicit MemProfUsePass(
+      std::string MemoryProfileFile,
+      IntrusiveRefCntPtr<vfs::FileSystem> FS = nullptr);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
 private:
   std::string MemoryProfileFileName;
@@ -66,7 +44,7 @@ namespace memprof {
 
 // Extract all calls from the IR.  Arrange them in a map from caller GUIDs to a
 // list of call sites, each of the form {LineLocation, CalleeGUID}.
-DenseMap<uint64_t, SmallVector<CallEdgeTy, 0>> extractCallsFromIR(
+LLVM_ABI DenseMap<uint64_t, SmallVector<CallEdgeTy, 0>> extractCallsFromIR(
     Module &M, const TargetLibraryInfo &TLI,
     function_ref<bool(uint64_t)> IsPresentInProfile = [](uint64_t) {
       return true;
@@ -83,7 +61,7 @@ using LocToLocMap =
 
 // Compute an undrifting map.  The result is a map from caller GUIDs to an inner
 // map that maps source locations in the profile to those in the current IR.
-DenseMap<uint64_t, LocToLocMap>
+LLVM_ABI DenseMap<uint64_t, LocToLocMap>
 computeUndriftMap(Module &M, IndexedInstrProfReader *MemProfReader,
                   const TargetLibraryInfo &TLI);
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h
index f88d83235111..1095e9d76d04 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Module;
@@ -25,8 +26,8 @@ struct MemorySanitizerOptions {
   MemorySanitizerOptions() : MemorySanitizerOptions(0, false, false, false){};
   MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel)
       : MemorySanitizerOptions(TrackOrigins, Recover, Kernel, false) {}
-  MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel,
-                         bool EagerChecks);
+  LLVM_ABI MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel,
+                                  bool EagerChecks);
   bool Kernel;
   int TrackOrigins;
   bool Recover;
@@ -42,9 +43,10 @@ struct MemorySanitizerOptions {
 struct MemorySanitizerPass : public PassInfoMixin<MemorySanitizerPass> {
   MemorySanitizerPass(MemorySanitizerOptions Options) : Options(Options) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
   static bool isRequired() { return true; }
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h
index 9aafe6cc48f8..7ba938e01d81 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h
@@ -16,6 +16,7 @@
 
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -27,7 +28,7 @@ namespace llvm {
 /// functions aren't declared yet, the pass inserts the declarations.
 struct NumericalStabilitySanitizerPass
     : public PassInfoMixin<NumericalStabilitySanitizerPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
index 1d0af87e965a..5084e53b2439 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
@@ -19,21 +19,19 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <cstdint>
 #include <string>
 
 namespace llvm {
 
-extern cl::opt<bool> DebugInfoCorrelate;
+LLVM_ABI extern cl::opt<bool> DebugInfoCorrelate;
 
 class Function;
 class Instruction;
 class Module;
 
-namespace vfs {
-class FileSystem;
-} // namespace vfs
-
 /// The instrumentation (profile-instr-gen) pass for IR based PGO.
 // We use this pass to create COMDAT profile variables for context
 // sensitive PGO (CSPGO). The reason to have a pass for this is CSPGO
@@ -46,7 +44,7 @@ class PGOInstrumentationGenCreateVar
   PGOInstrumentationGenCreateVar(std::string CSInstrName = "",
                                  bool Sampling = false)
       : CSInstrName(CSInstrName), ProfileSampling(Sampling) {}
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
 private:
   std::string CSInstrName;
@@ -60,7 +58,7 @@ class PGOInstrumentationGen : public PassInfoMixin<PGOInstrumentationGen> {
   PGOInstrumentationGen(
       PGOInstrumentationType InstrumentationType = PGOInstrumentationType ::FDO)
       : InstrumentationType(InstrumentationType) {}
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
 private:
   // If this is a context sensitive instrumentation.
@@ -70,11 +68,12 @@ class PGOInstrumentationGen : public PassInfoMixin<PGOInstrumentationGen> {
 /// The profile annotation (profile-instr-use) pass for IR based PGO.
 class PGOInstrumentationUse : public PassInfoMixin<PGOInstrumentationUse> {
 public:
+  LLVM_ABI
   PGOInstrumentationUse(std::string Filename = "",
                         std::string RemappingFilename = "", bool IsCS = false,
                         IntrusiveRefCntPtr<vfs::FileSystem> FS = nullptr);
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
 private:
   std::string ProfileFileName;
@@ -90,7 +89,7 @@ class PGOIndirectCallPromotion : public PassInfoMixin<PGOIndirectCallPromotion>
   PGOIndirectCallPromotion(bool IsInLTO = false, bool SamplePGO = false)
       : InLTO(IsInLTO), SamplePGO(SamplePGO) {}
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
 private:
   bool InLTO;
@@ -102,13 +101,14 @@ class PGOMemOPSizeOpt : public PassInfoMixin<PGOMemOPSizeOpt> {
 public:
   PGOMemOPSizeOpt() = default;
 
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &MAM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &MAM);
 };
 
-void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
-                     uint64_t MaxCount);
+LLVM_ABI void setProfMetadata(Module *M, Instruction *TI,
+                              ArrayRef<uint64_t> EdgeCounts, uint64_t MaxCount);
 
-void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count);
+LLVM_ABI void setIrrLoopHeaderMetadata(Module *M, Instruction *TI,
+                                       uint64_t Count);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h
index 5ee1cb044393..67697c695f1a 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/RealtimeSanitizer.h
@@ -20,12 +20,13 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_REALTIMESANITIZER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// Create ctor and init functions.
 struct RealtimeSanitizerPass : public PassInfoMixin<RealtimeSanitizerPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h
index 9efa23108d6a..054016622a57 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h
@@ -16,6 +16,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/Instrumentation.h"
 
 namespace llvm {
@@ -50,10 +51,10 @@ inline constexpr char kSanitizerBinaryMetadataAtomicsSection[] =
 class SanitizerBinaryMetadataPass
     : public PassInfoMixin<SanitizerBinaryMetadataPass> {
 public:
-  explicit SanitizerBinaryMetadataPass(
+  LLVM_ABI explicit SanitizerBinaryMetadataPass(
       SanitizerBinaryMetadataOptions Opts = {},
       ArrayRef<std::string> IgnorelistFiles = {});
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
   static bool isRequired() { return true; }
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h
index a49a8cb28239..f14f5b90a5cc 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/SanitizerCoverage.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_SANITIZERCOVERAGE_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SpecialCaseList.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Transforms/Utils/Instrumentation.h"
@@ -43,7 +44,7 @@ class SanitizerCoveragePass : public PassInfoMixin<SanitizerCoveragePass> {
       Blocklist = SpecialCaseList::createOrDie(BlocklistFiles,
                                                *vfs::getRealFileSystem());
   }
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
   static bool isRequired() { return true; }
 
 private:
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h
index 346951febf73..dea3b6b754b6 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h
@@ -14,6 +14,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Function;
@@ -25,7 +26,7 @@ class Module;
 /// inserts calls to runtime library functions. If the functions aren't declared
 /// yet, the pass inserts the declarations. Otherwise the existing globals are
 struct ThreadSanitizerPass : public PassInfoMixin<ThreadSanitizerPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
   static bool isRequired() { return true; }
 };
 
@@ -34,7 +35,7 @@ struct ThreadSanitizerPass : public PassInfoMixin<ThreadSanitizerPass> {
 /// Create ctor and init functions.
 struct ModuleThreadSanitizerPass
   : public PassInfoMixin<ModuleThreadSanitizerPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/TypeSanitizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/TypeSanitizer.h
index 20f08b679085..61a1c3e01c2c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/TypeSanitizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Instrumentation/TypeSanitizer.h
@@ -14,6 +14,7 @@
 #define LLVM_TRANSFORMS_INSTRUMENTATION_TYPESANITIZER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Function;
@@ -21,7 +22,7 @@ class FunctionPass;
 class Module;
 
 struct TypeSanitizerPass : public PassInfoMixin<TypeSanitizerPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/ObjCARC.h b/external/llvm-project/llvm/include/llvm/Transforms/ObjCARC.h
index bd17c58e842e..c927513469a3 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/ObjCARC.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/ObjCARC.h
@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_OBJCARC_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -24,26 +25,26 @@ class Pass;
 //
 // ObjCARCContract - Late ObjC ARC cleanups.
 //
-Pass *createObjCARCContractPass();
+LLVM_ABI Pass *createObjCARCContractPass();
 
 struct ObjCARCOptPass : public PassInfoMixin<ObjCARCOptPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 struct ObjCARCContractPass : public PassInfoMixin<ObjCARCContractPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 struct ObjCARCAPElimPass : public PassInfoMixin<ObjCARCAPElimPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 struct ObjCARCExpandPass : public PassInfoMixin<ObjCARCExpandPass> {
-  PreservedAnalyses run(Function &M, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &M, FunctionAnalysisManager &AM);
 };
 
 struct PAEvalPass : public PassInfoMixin<PAEvalPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // End llvm namespace
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar.h
index fc772a7639c4..1398f171b0f7 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TRANSFORMS_SCALAR_H
 #define LLVM_TRANSFORMS_SCALAR_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
 #include <functional>
 
@@ -30,26 +31,26 @@ class Pass;
 // their other instructions become dead, to eliminate chains of dead
 // computations.
 //
-FunctionPass *createDeadCodeEliminationPass();
+LLVM_ABI FunctionPass *createDeadCodeEliminationPass();
 
 //===----------------------------------------------------------------------===//
 //
 // SROA - Replace aggregates or pieces of aggregates with scalar SSA values.
 //
-FunctionPass *createSROAPass(bool PreserveCFG = true);
+LLVM_ABI FunctionPass *createSROAPass(bool PreserveCFG = true);
 
 //===----------------------------------------------------------------------===//
 //
 // LICM - This pass is a loop invariant code motion and memory promotion pass.
 //
-Pass *createLICMPass();
+LLVM_ABI Pass *createLICMPass();
 
 //===----------------------------------------------------------------------===//
 //
 // LoopStrengthReduce - This pass is strength reduces GEP instructions that use
 // a loop's canonical induction variable as one of their indices.
 //
-Pass *createLoopStrengthReducePass();
+LLVM_ABI Pass *createLoopStrengthReducePass();
 
 //===----------------------------------------------------------------------===//
 //
@@ -57,17 +58,18 @@ Pass *createLoopStrengthReducePass();
 // a loop terminator instruction by rewriting it in terms of another IV.
 // Expected to be run immediately after LSR.
 //
-Pass *createLoopTermFoldPass();
+LLVM_ABI Pass *createLoopTermFoldPass();
 
 //===----------------------------------------------------------------------===//
 //
 // LoopUnroll - This pass is a simple loop unrolling pass.
 //
-Pass *createLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false,
-                           bool ForgetAllSCEV = false, int Threshold = -1,
-                           int Count = -1, int AllowPartial = -1,
-                           int Runtime = -1, int UpperBound = -1,
-                           int AllowPeeling = -1);
+LLVM_ABI Pass *createLoopUnrollPass(int OptLevel = 2,
+                                    bool OnlyWhenForced = false,
+                                    bool ForgetAllSCEV = false,
+                                    int Threshold = -1, int Count = -1,
+                                    int AllowPartial = -1, int Runtime = -1,
+                                    int UpperBound = -1, int AllowPeeling = -1);
 
 //===----------------------------------------------------------------------===//
 //
@@ -76,14 +78,14 @@ Pass *createLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false,
 //
 // For example:  4 + (x + 5)  ->  x + (4 + 5)
 //
-FunctionPass *createReassociatePass();
+LLVM_ABI FunctionPass *createReassociatePass();
 
 //===----------------------------------------------------------------------===//
 //
 // CFGSimplification - Merge basic blocks, eliminate unreachable blocks,
 // simplify terminator instructions, convert switches to lookup tables, etc.
 //
-FunctionPass *createCFGSimplificationPass(
+LLVM_ABI FunctionPass *createCFGSimplificationPass(
     SimplifyCFGOptions Options = SimplifyCFGOptions(),
     std::function<bool(const Function &)> Ftor = nullptr);
 
@@ -92,7 +94,7 @@ FunctionPass *createCFGSimplificationPass(
 // FlattenCFG - flatten CFG, reduce number of conditional branches by using
 // parallel-and and parallel-or mode, etc...
 //
-FunctionPass *createFlattenCFGPass();
+LLVM_ABI FunctionPass *createFlattenCFGPass();
 
 //===----------------------------------------------------------------------===//
 //
@@ -101,45 +103,45 @@ FunctionPass *createFlattenCFGPass();
 ///
 /// When \p SkipUniformRegions is true the structizer will not structurize
 /// regions that only contain uniform branches.
-Pass *createStructurizeCFGPass(bool SkipUniformRegions = false);
+LLVM_ABI Pass *createStructurizeCFGPass(bool SkipUniformRegions = false);
 
 //===----------------------------------------------------------------------===//
 //
 // TailCallElimination - This pass eliminates call instructions to the current
 // function which occur immediately before return instructions.
 //
-FunctionPass *createTailCallEliminationPass();
+LLVM_ABI FunctionPass *createTailCallEliminationPass();
 
 //===----------------------------------------------------------------------===//
 //
 // EarlyCSE - This pass performs a simple and fast CSE pass over the dominator
 // tree.
 //
-FunctionPass *createEarlyCSEPass(bool UseMemorySSA = false);
+LLVM_ABI FunctionPass *createEarlyCSEPass(bool UseMemorySSA = false);
 
 //===----------------------------------------------------------------------===//
 //
 // ConstantHoisting - This pass prepares a function for expensive constants.
 //
-FunctionPass *createConstantHoistingPass();
+LLVM_ABI FunctionPass *createConstantHoistingPass();
 
 //===----------------------------------------------------------------------===//
 //
 // Sink - Code Sinking
 //
-FunctionPass *createSinkingPass();
+LLVM_ABI FunctionPass *createSinkingPass();
 
 //===----------------------------------------------------------------------===//
 //
 // LowerAtomic - Lower atomic intrinsics to non-atomic form
 //
-Pass *createLowerAtomicPass();
+LLVM_ABI Pass *createLowerAtomicPass();
 
 //===----------------------------------------------------------------------===//
 //
 // MergeICmps - Merge integer comparison chains into a memcmp
 //
-Pass *createMergeICmpsLegacyPass();
+LLVM_ABI Pass *createMergeICmpsLegacyPass();
 
 //===----------------------------------------------------------------------===//
 //
@@ -148,66 +150,67 @@ Pass *createMergeICmpsLegacyPass();
 // on the target. If AddressSpace is left to its default value, it will be
 // obtained from the TargetTransformInfo.
 //
-FunctionPass *createInferAddressSpacesPass(unsigned AddressSpace = ~0u);
-extern char &InferAddressSpacesID;
+LLVM_ABI FunctionPass *
+createInferAddressSpacesPass(unsigned AddressSpace = ~0u);
+LLVM_ABI extern char &InferAddressSpacesID;
 
 //===----------------------------------------------------------------------===//
 //
 // PartiallyInlineLibCalls - Tries to inline the fast path of library
 // calls such as sqrt.
 //
-FunctionPass *createPartiallyInlineLibCallsPass();
+LLVM_ABI FunctionPass *createPartiallyInlineLibCallsPass();
 
 //===----------------------------------------------------------------------===//
 //
 // SeparateConstOffsetFromGEP - Split GEPs for better CSE
 //
-FunctionPass *createSeparateConstOffsetFromGEPPass(bool LowerGEP = false);
+LLVM_ABI FunctionPass *
+createSeparateConstOffsetFromGEPPass(bool LowerGEP = false);
 
 //===----------------------------------------------------------------------===//
 //
 // SpeculativeExecution - Aggressively hoist instructions to enable
 // speculative execution on targets where branches are expensive.
 //
-FunctionPass *createSpeculativeExecutionPass();
+LLVM_ABI FunctionPass *createSpeculativeExecutionPass();
 
 // Same as createSpeculativeExecutionPass, but does nothing unless
 // TargetTransformInfo::hasBranchDivergence() is true.
-FunctionPass *createSpeculativeExecutionIfHasBranchDivergencePass();
+LLVM_ABI FunctionPass *createSpeculativeExecutionIfHasBranchDivergencePass();
 
 //===----------------------------------------------------------------------===//
 //
 // StraightLineStrengthReduce - This pass strength-reduces some certain
 // instruction patterns in straight-line code.
 //
-FunctionPass *createStraightLineStrengthReducePass();
+LLVM_ABI FunctionPass *createStraightLineStrengthReducePass();
 
 //===----------------------------------------------------------------------===//
 //
 // NaryReassociate - Simplify n-ary operations by reassociation.
 //
-FunctionPass *createNaryReassociatePass();
+LLVM_ABI FunctionPass *createNaryReassociatePass();
 
 //===----------------------------------------------------------------------===//
 //
 // LoopDataPrefetch - Perform data prefetching in loops.
 //
-FunctionPass *createLoopDataPrefetchPass();
+LLVM_ABI FunctionPass *createLoopDataPrefetchPass();
 
 //===----------------------------------------------------------------------===//
 //
 // This pass does instruction simplification on each
 // instruction in a function.
 //
-FunctionPass *createInstSimplifyLegacyPass();
-
+LLVM_ABI FunctionPass *createInstSimplifyLegacyPass();
 
 //===----------------------------------------------------------------------===//
 //
 // createScalarizeMaskedMemIntrinPass - Replace masked load, store, gather
 // and scatter intrinsics with scalar code when target doesn't support them.
 //
-FunctionPass *createScalarizeMaskedMemIntrinLegacyPass();
+LLVM_ABI FunctionPass *createScalarizeMaskedMemIntrinLegacyPass();
 } // End llvm namespace
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h
index 877d8145e746..cbde7ff18ff7 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h
@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_SCALAR_EARLYCSE_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -31,9 +32,10 @@ struct EarlyCSEPass : PassInfoMixin<EarlyCSEPass> {
   EarlyCSEPass(bool UseMemorySSA = false) : UseMemorySSA(UseMemorySSA) {}
 
   /// Run the pass over the function.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
   bool UseMemorySSA;
 };
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/GVN.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/GVN.h
index 25b042eebf66..e156ec469a14 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -130,25 +130,26 @@ class GVNPass : public PassInfoMixin<GVNPass> {
   GVNPass(GVNOptions Options = {}) : Options(Options) {}
 
   /// Run the pass over the function.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
   /// This removes the specified instruction from
   /// our various maps and marks it for deletion.
-  void salvageAndRemoveInstruction(Instruction *I);
+  LLVM_ABI void salvageAndRemoveInstruction(Instruction *I);
 
   DominatorTree &getDominatorTree() const { return *DT; }
   AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
   MemoryDependenceResults &getMemDep() const { return *MD; }
 
-  bool isPREEnabled() const;
-  bool isLoadPREEnabled() const;
-  bool isLoadInLoopPREEnabled() const;
-  bool isLoadPRESplitBackedgeEnabled() const;
-  bool isMemDepEnabled() const;
-  bool isMemorySSAEnabled() const;
+  LLVM_ABI bool isPREEnabled() const;
+  LLVM_ABI bool isLoadPREEnabled() const;
+  LLVM_ABI bool isLoadInLoopPREEnabled() const;
+  LLVM_ABI bool isLoadPRESplitBackedgeEnabled() const;
+  LLVM_ABI bool isMemDepEnabled() const;
+  LLVM_ABI bool isMemorySSAEnabled() const;
 
   /// This class holds the mapping between values and value numbers.  It is used
   /// as an efficient mechanism to determine the expression-wise equivalence of
@@ -194,29 +195,31 @@ class GVNPass : public PassInfoMixin<GVNPass> {
     bool areAllValsInBB(uint32_t Num, const BasicBlock *BB, GVNPass &GVN);
 
   public:
-    ValueTable();
-    ValueTable(const ValueTable &Arg);
-    ValueTable(ValueTable &&Arg);
-    ~ValueTable();
-    ValueTable &operator=(const ValueTable &Arg);
-
-    uint32_t lookupOrAdd(Value *V);
-    uint32_t lookup(Value *V, bool Verify = true) const;
-    uint32_t lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Pred,
-                            Value *LHS, Value *RHS);
-    uint32_t phiTranslate(const BasicBlock *BB, const BasicBlock *PhiBlock,
-                          uint32_t Num, GVNPass &GVN);
-    void eraseTranslateCacheEntry(uint32_t Num, const BasicBlock &CurrBlock);
-    bool exists(Value *V) const;
-    void add(Value *V, uint32_t Num);
-    void clear();
-    void erase(Value *V);
+    LLVM_ABI ValueTable();
+    LLVM_ABI ValueTable(const ValueTable &Arg);
+    LLVM_ABI ValueTable(ValueTable &&Arg);
+    LLVM_ABI ~ValueTable();
+    LLVM_ABI ValueTable &operator=(const ValueTable &Arg);
+
+    LLVM_ABI uint32_t lookupOrAdd(Value *V);
+    LLVM_ABI uint32_t lookup(Value *V, bool Verify = true) const;
+    LLVM_ABI uint32_t lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Pred,
+                                     Value *LHS, Value *RHS);
+    LLVM_ABI uint32_t phiTranslate(const BasicBlock *BB,
+                                   const BasicBlock *PhiBlock, uint32_t Num,
+                                   GVNPass &GVN);
+    LLVM_ABI void eraseTranslateCacheEntry(uint32_t Num,
+                                           const BasicBlock &CurrBlock);
+    LLVM_ABI bool exists(Value *V) const;
+    LLVM_ABI void add(Value *V, uint32_t Num);
+    LLVM_ABI void clear();
+    LLVM_ABI void erase(Value *V);
     void setAliasAnalysis(AAResults *A) { AA = A; }
     AAResults *getAliasAnalysis() const { return AA; }
     void setMemDep(MemoryDependenceResults *M) { MD = M; }
     void setDomTree(DominatorTree *D) { DT = D; }
     uint32_t getNextUnusedValueNumber() { return NextValueNumber; }
-    void verifyRemoved(const Value *) const;
+    LLVM_ABI void verifyRemoved(const Value *) const;
   };
 
 private:
@@ -289,9 +292,9 @@ class GVNPass : public PassInfoMixin<GVNPass> {
                             leader_iterator(nullptr));
     }
 
-    void insert(uint32_t N, Value *V, const BasicBlock *BB);
-    void erase(uint32_t N, Instruction *I, const BasicBlock *BB);
-    void verifyRemoved(const Value *Inst) const;
+    LLVM_ABI void insert(uint32_t N, Value *V, const BasicBlock *BB);
+    LLVM_ABI void erase(uint32_t N, Instruction *I, const BasicBlock *BB);
+    LLVM_ABI void verifyRemoved(const Value *Inst) const;
     void clear() {
       NumToLeaders.clear();
       TableAllocator.Reset();
@@ -388,20 +391,20 @@ class GVNPass : public PassInfoMixin<GVNPass> {
 };
 
 /// Create a legacy GVN pass.
-FunctionPass *createGVNPass();
+LLVM_ABI FunctionPass *createGVNPass();
 
 /// A simple and fast domtree-based GVN pass to hoist common expressions
 /// from sibling branches.
 struct GVNHoistPass : PassInfoMixin<GVNHoistPass> {
   /// Run the pass over the function.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 /// Uses an "inverted" value numbering to decide the similarity of
 /// expressions and sinks similar expressions into successors.
 struct GVNSinkPass : PassInfoMixin<GVNSinkPass> {
   /// Run the pass over the function.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
index 182cab02e640..75b5cf2371fd 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
@@ -22,6 +22,7 @@
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <optional>
 #include <utility>
@@ -102,36 +103,38 @@ class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> {
   unsigned DefaultBBDupThreshold;
 
 public:
-  JumpThreadingPass(int T = -1);
+  LLVM_ABI JumpThreadingPass(int T = -1);
 
   // Glue for old PM.
-  bool runImpl(Function &F, FunctionAnalysisManager *FAM,
-               TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
-               LazyValueInfo *LVI, AAResults *AA,
-               std::unique_ptr<DomTreeUpdater> DTU,
-               std::optional<BlockFrequencyInfo *> BFI,
-               std::optional<BranchProbabilityInfo *> BPI);
+  LLVM_ABI bool runImpl(Function &F, FunctionAnalysisManager *FAM,
+                        TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
+                        LazyValueInfo *LVI, AAResults *AA,
+                        std::unique_ptr<DomTreeUpdater> DTU,
+                        std::optional<BlockFrequencyInfo *> BFI,
+                        std::optional<BranchProbabilityInfo *> BPI);
 
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   DomTreeUpdater *getDomTreeUpdater() const { return DTU.get(); }
-  void findLoopHeaders(Function &F);
-  bool processBlock(BasicBlock *BB);
-  bool maybeMergeBasicBlockIntoOnlyPred(BasicBlock *BB);
-  void updateSSA(BasicBlock *BB, BasicBlock *NewBB,
-                 ValueToValueMapTy &ValueMapping);
-  void cloneInstructions(ValueToValueMapTy &ValueMapping,
-                         BasicBlock::iterator BI, BasicBlock::iterator BE,
-                         BasicBlock *NewBB, BasicBlock *PredBB);
-  bool tryThreadEdge(BasicBlock *BB,
-                     const SmallVectorImpl<BasicBlock *> &PredBBs,
-                     BasicBlock *SuccBB);
-  void threadEdge(BasicBlock *BB, const SmallVectorImpl<BasicBlock *> &PredBBs,
-                  BasicBlock *SuccBB);
-  bool duplicateCondBranchOnPHIIntoPred(
+  LLVM_ABI void findLoopHeaders(Function &F);
+  LLVM_ABI bool processBlock(BasicBlock *BB);
+  LLVM_ABI bool maybeMergeBasicBlockIntoOnlyPred(BasicBlock *BB);
+  LLVM_ABI void updateSSA(BasicBlock *BB, BasicBlock *NewBB,
+                          ValueToValueMapTy &ValueMapping);
+  LLVM_ABI void cloneInstructions(ValueToValueMapTy &ValueMapping,
+                                  BasicBlock::iterator BI,
+                                  BasicBlock::iterator BE, BasicBlock *NewBB,
+                                  BasicBlock *PredBB);
+  LLVM_ABI bool tryThreadEdge(BasicBlock *BB,
+                              const SmallVectorImpl<BasicBlock *> &PredBBs,
+                              BasicBlock *SuccBB);
+  LLVM_ABI void threadEdge(BasicBlock *BB,
+                           const SmallVectorImpl<BasicBlock *> &PredBBs,
+                           BasicBlock *SuccBB);
+  LLVM_ABI bool duplicateCondBranchOnPHIIntoPred(
       BasicBlock *BB, const SmallVectorImpl<BasicBlock *> &PredBBs);
 
-  bool computeValueKnownInPredecessorsImpl(
+  LLVM_ABI bool computeValueKnownInPredecessorsImpl(
       Value *V, BasicBlock *BB, jumpthreading::PredValueInfo &Result,
       jumpthreading::ConstantPreference Preference,
       SmallPtrSet<Value *, 4> &RecursionSet, Instruction *CxtI = nullptr);
@@ -145,29 +148,34 @@ class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> {
                                                RecursionSet, CxtI);
   }
 
-  Constant *evaluateOnPredecessorEdge(BasicBlock *BB, BasicBlock *PredPredBB,
-                                      Value *cond, const DataLayout &DL);
-  bool maybethreadThroughTwoBasicBlocks(BasicBlock *BB, Value *Cond);
-  void threadThroughTwoBasicBlocks(BasicBlock *PredPredBB, BasicBlock *PredBB,
-                                   BasicBlock *BB, BasicBlock *SuccBB);
-  bool processThreadableEdges(Value *Cond, BasicBlock *BB,
-                              jumpthreading::ConstantPreference Preference,
-                              Instruction *CxtI = nullptr);
-
-  bool processBranchOnPHI(PHINode *PN);
-  bool processBranchOnXOR(BinaryOperator *BO);
-  bool processImpliedCondition(BasicBlock *BB);
-
-  bool simplifyPartiallyRedundantLoad(LoadInst *LI);
-  void unfoldSelectInstr(BasicBlock *Pred, BasicBlock *BB, SelectInst *SI,
-                         PHINode *SIUse, unsigned Idx);
-
-  bool tryToUnfoldSelect(CmpInst *CondCmp, BasicBlock *BB);
-  bool tryToUnfoldSelect(SwitchInst *SI, BasicBlock *BB);
-  bool tryToUnfoldSelectInCurrBB(BasicBlock *BB);
-
-  bool processGuards(BasicBlock *BB);
-  bool threadGuard(BasicBlock *BB, IntrinsicInst *Guard, BranchInst *BI);
+  LLVM_ABI Constant *evaluateOnPredecessorEdge(BasicBlock *BB,
+                                               BasicBlock *PredPredBB,
+                                               Value *cond,
+                                               const DataLayout &DL);
+  LLVM_ABI bool maybethreadThroughTwoBasicBlocks(BasicBlock *BB, Value *Cond);
+  LLVM_ABI void threadThroughTwoBasicBlocks(BasicBlock *PredPredBB,
+                                            BasicBlock *PredBB, BasicBlock *BB,
+                                            BasicBlock *SuccBB);
+  LLVM_ABI bool
+  processThreadableEdges(Value *Cond, BasicBlock *BB,
+                         jumpthreading::ConstantPreference Preference,
+                         Instruction *CxtI = nullptr);
+
+  LLVM_ABI bool processBranchOnPHI(PHINode *PN);
+  LLVM_ABI bool processBranchOnXOR(BinaryOperator *BO);
+  LLVM_ABI bool processImpliedCondition(BasicBlock *BB);
+
+  LLVM_ABI bool simplifyPartiallyRedundantLoad(LoadInst *LI);
+  LLVM_ABI void unfoldSelectInstr(BasicBlock *Pred, BasicBlock *BB,
+                                  SelectInst *SI, PHINode *SIUse, unsigned Idx);
+
+  LLVM_ABI bool tryToUnfoldSelect(CmpInst *CondCmp, BasicBlock *BB);
+  LLVM_ABI bool tryToUnfoldSelect(SwitchInst *SI, BasicBlock *BB);
+  LLVM_ABI bool tryToUnfoldSelectInCurrBB(BasicBlock *BB);
+
+  LLVM_ABI bool processGuards(BasicBlock *BB);
+  LLVM_ABI bool threadGuard(BasicBlock *BB, IntrinsicInst *Guard,
+                            BranchInst *BI);
 
 private:
   BasicBlock *splitBlockPreds(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
index f445e0696b5f..44ba3c24f6af 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
@@ -20,9 +20,11 @@ class raw_ostream;
 class LoopAccessInfoPrinterPass
     : public PassInfoMixin<LoopAccessInfoPrinterPass> {
   raw_ostream &OS;
+  bool AllowPartial;
 
 public:
-  explicit LoopAccessInfoPrinterPass(raw_ostream &OS) : OS(OS) {}
+  explicit LoopAccessInfoPrinterPass(raw_ostream &OS, bool AllowPartial)
+      : OS(OS), AllowPartial(AllowPartial) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index e1e1cbfdbf7a..79ccd63fd834 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -40,8 +40,9 @@
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopNestAnalysis.h"
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/LCSSA.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
@@ -91,11 +92,13 @@ class PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
     return *this;
   }
 
-  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
-                        LoopStandardAnalysisResults &AR, LPMUpdater &U);
+  LLVM_ABI PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
+                                 LoopStandardAnalysisResults &AR,
+                                 LPMUpdater &U);
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
   /// Add either a loop pass or a loop-nest pass to the pass manager. Append \p
   /// Pass to the list of loop passes if it has a dedicated \fn run() method for
   /// loops and to the list of loop-nest passes if the \fn run() method is for
@@ -154,12 +157,12 @@ class PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
                 LoopStandardAnalysisResults &AR, LPMUpdater &U,
                 PassInstrumentation &PI);
 
-  PreservedAnalyses runWithLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
-                                          LoopStandardAnalysisResults &AR,
-                                          LPMUpdater &U);
-  PreservedAnalyses runWithoutLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
-                                             LoopStandardAnalysisResults &AR,
-                                             LPMUpdater &U);
+  LLVM_ABI PreservedAnalyses
+  runWithLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
+                        LoopStandardAnalysisResults &AR, LPMUpdater &U);
+  LLVM_ABI PreservedAnalyses
+  runWithoutLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
+                           LoopStandardAnalysisResults &AR, LPMUpdater &U);
 
 private:
   static const Loop &getLoopFromIR(Loop &L) { return L; }
@@ -413,9 +416,10 @@ class FunctionToLoopPassAdaptor
   }
 
   /// Runs the loop passes across every loop in the function.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
   static bool isRequired() { return true; }
 
@@ -497,11 +501,11 @@ class PrintLoopPass : public PassInfoMixin<PrintLoopPass> {
   std::string Banner;
 
 public:
-  PrintLoopPass();
-  PrintLoopPass(raw_ostream &OS, const std::string &Banner = "");
+  LLVM_ABI PrintLoopPass();
+  LLVM_ABI PrintLoopPass(raw_ostream &OS, const std::string &Banner = "");
 
-  PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
-                        LoopStandardAnalysisResults &, LPMUpdater &);
+  LLVM_ABI PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
+                                 LoopStandardAnalysisResults &, LPMUpdater &);
 };
 }
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
index 95ef0f73e8af..7df903f15e4e 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_SCALAR_LOWEREXPECTINTRINSIC_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -29,7 +30,7 @@ struct LowerExpectIntrinsicPass : PassInfoMixin<LowerExpectIntrinsicPass> {
   /// of the probabilities and frequencies of the CFG. After running this pass,
   /// no more expect intrinsics remain, allowing the rest of the optimizer to
   /// ignore them.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
 };
 
 }
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SCCP.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SCCP.h
index 7803f008c618..d45dca97d12d 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SCCP.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SCCP.h
@@ -21,6 +21,7 @@
 #define LLVM_TRANSFORMS_SCALAR_SCCP_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Function;
@@ -28,7 +29,7 @@ class Function;
 /// This pass performs function-level constant propagation and merging.
 class SCCPPass : public PassInfoMixin<SCCPPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/Scalarizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/Scalarizer.h
index 401ca4d99534..12513c2a704f 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/Scalarizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/Scalarizer.h
@@ -19,6 +19,7 @@
 #define LLVM_TRANSFORMS_SCALAR_SCALARIZER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -55,7 +56,7 @@ class ScalarizerPass : public PassInfoMixin<ScalarizerPass> {
   ScalarizerPass() = default;
   ScalarizerPass(const ScalarizerPassOptions &Options) : Options(Options) {}
 
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   void setScalarizeVariableInsertExtract(bool Value) {
     Options.ScalarizeVariableInsertExtract = Value;
@@ -65,7 +66,7 @@ class ScalarizerPass : public PassInfoMixin<ScalarizerPass> {
 };
 
 /// Create a legacy pass manager instance of the Scalarizer pass
-FunctionPass *createScalarizerPass(
+LLVM_ABI FunctionPass *createScalarizerPass(
     const ScalarizerPassOptions &Options = ScalarizerPassOptions());
 }
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h
index 67db5031a443..b2b7f4619da8 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h
@@ -16,6 +16,7 @@
 
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
 
 namespace llvm {
@@ -34,16 +35,17 @@ class SimplifyCFGPass : public PassInfoMixin<SimplifyCFGPass> {
   /// rather than optimal IR. That is, by default we bypass transformations that
   /// are likely to improve performance but make analysis for other passes more
   /// difficult.
-  SimplifyCFGPass();
+  LLVM_ABI SimplifyCFGPass();
 
   /// Construct a pass with optional optimizations.
-  SimplifyCFGPass(const SimplifyCFGOptions &PassOptions);
+  LLVM_ABI SimplifyCFGPass(const SimplifyCFGOptions &PassOptions);
 
   /// Run the pass over the function.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 };
 
 }
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils.h
index ff306dbe3580..840e085379a1 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_TRANSFORMS_UTILS_H
 #define LLVM_TRANSFORMS_UTILS_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 class ModulePass;
@@ -25,16 +27,16 @@ class Pass;
 // LowerInvoke - This pass removes invoke instructions, converting them to call
 // instructions.
 //
-FunctionPass *createLowerInvokePass();
-extern char &LowerInvokePassID;
+LLVM_ABI FunctionPass *createLowerInvokePass();
+LLVM_ABI extern char &LowerInvokePassID;
 
 //===----------------------------------------------------------------------===//
 //
 // LowerSwitch - This pass converts SwitchInst instructions into a sequence of
 // chained binary branch instructions.
 //
-FunctionPass *createLowerSwitchPass();
-extern char &LowerSwitchID;
+LLVM_ABI FunctionPass *createLowerSwitchPass();
+LLVM_ABI extern char &LowerSwitchID;
 
 //===----------------------------------------------------------------------===//
 //
@@ -43,7 +45,7 @@ extern char &LowerSwitchID;
 // variants, intended to run pre- and post-inlining, respectively. Only the
 // post-inlining variant is used with the legacy pass manager.
 //
-FunctionPass *createPostInlineEntryExitInstrumenterPass();
+LLVM_ABI FunctionPass *createPostInlineEntryExitInstrumenterPass();
 
 //===----------------------------------------------------------------------===//
 //
@@ -56,16 +58,16 @@ FunctionPass *createPostInlineEntryExitInstrumenterPass();
 // This pass obviously invalidates the CFG, but can update forward dominator
 // (set, immediate dominators, tree, and frontier) information.
 //
-FunctionPass *createBreakCriticalEdgesPass();
-extern char &BreakCriticalEdgesID;
+LLVM_ABI FunctionPass *createBreakCriticalEdgesPass();
+LLVM_ABI extern char &BreakCriticalEdgesID;
 
 //===----------------------------------------------------------------------===//
 //
 // LCSSA - This pass inserts phi nodes at loop boundaries to simplify other loop
 // optimizations.
 //
-Pass *createLCSSAPass();
-extern char &LCSSAID;
+LLVM_ABI Pass *createLCSSAPass();
+LLVM_ABI extern char &LCSSAID;
 
 //===----------------------------------------------------------------------===//
 //
@@ -79,7 +81,7 @@ extern char &LCSSAID;
 //   %Y = load i32* %X
 //   ret i32 %Y
 //
-FunctionPass *createPromoteMemoryToRegisterPass();
+LLVM_ABI FunctionPass *createPromoteMemoryToRegisterPass();
 
 //===----------------------------------------------------------------------===//
 //
@@ -87,7 +89,7 @@ FunctionPass *createPromoteMemoryToRegisterPass();
 // references. In basically undoes the PromoteMemoryToRegister pass to make cfg
 // hacking easier.
 //
-FunctionPass *createRegToMemWrapperPass();
+LLVM_ABI FunctionPass *createRegToMemWrapperPass();
 
 //===----------------------------------------------------------------------===//
 //
@@ -97,8 +99,8 @@ FunctionPass *createRegToMemWrapperPass();
 //
 //   AU.addRequiredID(LoopSimplifyID);
 //
-Pass *createLoopSimplifyPass();
-extern char &LoopSimplifyID;
+LLVM_ABI Pass *createLoopSimplifyPass();
+LLVM_ABI extern char &LoopSimplifyID;
 
 //===----------------------------------------------------------------------===//
 //
@@ -106,27 +108,27 @@ extern char &LoopSimplifyID;
 // blocks branch to N, and then N distributes control flow to all the original
 // exit blocks.
 //
-FunctionPass *createUnifyLoopExitsPass();
+LLVM_ABI FunctionPass *createUnifyLoopExitsPass();
 
 //===----------------------------------------------------------------------===//
 //
 // FixIrreducible - Convert each SCC with irreducible control-flow
 // into a natural loop.
 //
-FunctionPass *createFixIrreduciblePass();
+LLVM_ABI FunctionPass *createFixIrreduciblePass();
 
 //===----------------------------------------------------------------------===//
 //
 // CanonicalizeFreezeInLoops - Canonicalize freeze instructions in loops so they
 // don't block SCEV.
 //
-Pass *createCanonicalizeFreezeInLoopsPass();
+LLVM_ABI Pass *createCanonicalizeFreezeInLoopsPass();
 
 //===----------------------------------------------------------------------===//
 // LowerGlobalDtorsLegacy - Lower @llvm.global_dtors by creating wrapper
 // functions that are registered in @llvm.global_ctors and which contain a call
 // to `__cxa_atexit` to register their destructor functions.
-ModulePass *createLowerGlobalDtorsLegacyPass();
+LLVM_ABI ModulePass *createLowerGlobalDtorsLegacyPass();
 } // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/AMDGPUEmitPrintf.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/AMDGPUEmitPrintf.h
index 55e92c37a167..e7adf91b3f56 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/AMDGPUEmitPrintf.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/AMDGPUEmitPrintf.h
@@ -15,11 +15,12 @@
 #define LLVM_TRANSFORMS_UTILS_AMDGPUEMITPRINTF_H
 
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
-Value *emitAMDGPUPrintfCall(IRBuilder<> &Builder, ArrayRef<Value *> Args,
-                            bool isBuffered);
+LLVM_ABI Value *emitAMDGPUPrintfCall(IRBuilder<> &Builder,
+                                     ArrayRef<Value *> Args, bool isBuffered);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ASanStackFrameLayout.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
index f87588db4ee2..1360879b8b6c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
@@ -13,6 +13,7 @@
 #define LLVM_TRANSFORMS_UTILS_ASANSTACKFRAMELAYOUT_H
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -47,7 +48,7 @@ struct ASanStackFrameLayout {
   uint64_t FrameSize;       // Size of the frame in bytes.
 };
 
-ASanStackFrameLayout ComputeASanStackFrameLayout(
+LLVM_ABI ASanStackFrameLayout ComputeASanStackFrameLayout(
     // The array of stack variables. The elements may get reordered and changed.
     SmallVectorImpl<ASanStackVariableDescription> &Vars,
     // AddressSanitizer's shadow granularity. Usually 8, may also be 16, 32, 64.
@@ -58,19 +59,19 @@ ASanStackFrameLayout ComputeASanStackFrameLayout(
     uint64_t MinHeaderSize);
 
 // Compute frame description, see DescribeAddressIfStack in ASan runtime.
-SmallString<64> ComputeASanStackFrameDescription(
+LLVM_ABI SmallString<64> ComputeASanStackFrameDescription(
     const SmallVectorImpl<ASanStackVariableDescription> &Vars);
 
 // Returns shadow bytes with marked red zones. This shadow represents the state
 // if the stack frame when all local variables are inside of the own scope.
-SmallVector<uint8_t, 64>
+LLVM_ABI SmallVector<uint8_t, 64>
 GetShadowBytes(const SmallVectorImpl<ASanStackVariableDescription> &Vars,
                const ASanStackFrameLayout &Layout);
 
 // Returns shadow bytes with marked red zones and after scope. This shadow
 // represents the state if the stack frame when all local variables are outside
 // of the own scope.
-SmallVector<uint8_t, 64> GetShadowBytesAfterScope(
+LLVM_ABI SmallVector<uint8_t, 64> GetShadowBytesAfterScope(
     // The array of stack variables. The elements may get reordered and changed.
     const SmallVectorImpl<ASanStackVariableDescription> &Vars,
     const ASanStackFrameLayout &Layout);
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h
index 85c49d39098b..eb91367dfdce 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h
@@ -19,6 +19,7 @@
 #include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class AssumeInst;
@@ -27,13 +28,13 @@ class Instruction;
 class AssumptionCache;
 class DominatorTree;
 
-extern cl::opt<bool> EnableKnowledgeRetention;
+LLVM_ABI extern cl::opt<bool> EnableKnowledgeRetention;
 
 /// Build a call to llvm.assume to preserve informations that can be derived
 /// from the given instruction.
 /// If no information derived from \p I, this call returns null.
 /// The returned instruction is not inserted anywhere.
-AssumeInst *buildAssumeFromInst(Instruction *I);
+LLVM_ABI AssumeInst *buildAssumeFromInst(Instruction *I);
 
 /// Calls BuildAssumeFromInst and if the resulting llvm.assume is valid insert
 /// if before I. This is usually what need to be done to salvage the knowledge
@@ -43,35 +44,35 @@ AssumeInst *buildAssumeFromInst(Instruction *I);
 /// The DominatorTree can optionally be provided to enable cross-block
 /// reasoning.
 /// This returns if a change was made.
-bool salvageKnowledge(Instruction *I, AssumptionCache *AC = nullptr,
-                      DominatorTree *DT = nullptr);
+LLVM_ABI bool salvageKnowledge(Instruction *I, AssumptionCache *AC = nullptr,
+                               DominatorTree *DT = nullptr);
 
 /// Build and return a new assume created from the provided knowledge
 /// if the knowledge in the assume is fully redundant this will return nullptr
-AssumeInst *buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
-                                     Instruction *CtxI,
-                                     AssumptionCache *AC = nullptr,
-                                     DominatorTree *DT = nullptr);
+LLVM_ABI AssumeInst *
+buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
+                         Instruction *CtxI, AssumptionCache *AC = nullptr,
+                         DominatorTree *DT = nullptr);
 
 /// This pass attempts to minimize the number of assume without loosing any
 /// information.
 struct AssumeSimplifyPass : public PassInfoMixin<AssumeSimplifyPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 /// This pass will try to build an llvm.assume for every instruction in the
 /// function. Its main purpose is testing.
 struct AssumeBuilderPass : public PassInfoMixin<AssumeBuilderPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 /// canonicalize the RetainedKnowledge RK. it is assumed that RK is part of
 /// Assume. This will return an empty RetainedKnowledge if the knowledge is
 /// useless.
-RetainedKnowledge simplifyRetainedKnowledge(AssumeInst *Assume,
-                                            RetainedKnowledge RK,
-                                            AssumptionCache *AC,
-                                            DominatorTree *DT);
+LLVM_ABI RetainedKnowledge simplifyRetainedKnowledge(AssumeInst *Assume,
+                                                     RetainedKnowledge RK,
+                                                     AssumptionCache *AC,
+                                                     DominatorTree *DT);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index adc1851c2ec2..979f3b3eb72f 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 
 namespace llvm {
@@ -46,13 +47,14 @@ class Value;
 /// instruction. If \p Updates is specified, collect all necessary DT updates
 /// into this vector. If \p KeepOneInputPHIs is true, one-input Phis in
 /// successors of blocks being deleted will be preserved.
-void detachDeadBlocks(ArrayRef <BasicBlock *> BBs,
-                      SmallVectorImpl<DominatorTree::UpdateType> *Updates,
-                      bool KeepOneInputPHIs = false);
+LLVM_ABI void
+detachDeadBlocks(ArrayRef<BasicBlock *> BBs,
+                 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
+                 bool KeepOneInputPHIs = false);
 
 /// Delete the specified block, which must have no predecessors.
-void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU = nullptr,
-                     bool KeepOneInputPHIs = false);
+LLVM_ABI void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU = nullptr,
+                              bool KeepOneInputPHIs = false);
 
 /// Delete the specified blocks from \p BB. The set of deleted blocks must have
 /// no predecessors that are not being deleted themselves. \p BBs must have no
@@ -60,29 +62,32 @@ void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU = nullptr,
 /// relevant loop info updates should be done before this function is called.
 /// If \p KeepOneInputPHIs is true, one-input Phis in successors of blocks
 /// being deleted will be preserved.
-void DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs,
-                      DomTreeUpdater *DTU = nullptr,
-                      bool KeepOneInputPHIs = false);
+LLVM_ABI void DeleteDeadBlocks(ArrayRef<BasicBlock *> BBs,
+                               DomTreeUpdater *DTU = nullptr,
+                               bool KeepOneInputPHIs = false);
 
 /// Delete all basic blocks from \p F that are not reachable from its entry
 /// node. If \p KeepOneInputPHIs is true, one-input Phis in successors of
 /// blocks being deleted will be preserved.
-bool EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU = nullptr,
-                                bool KeepOneInputPHIs = false);
+LLVM_ABI bool EliminateUnreachableBlocks(Function &F,
+                                         DomTreeUpdater *DTU = nullptr,
+                                         bool KeepOneInputPHIs = false);
 
 /// We know that BB has one predecessor. If there are any single-entry PHI nodes
 /// in it, fold them away. This handles the case when all entries to the PHI
 /// nodes in a block are guaranteed equal, such as when the block has exactly
 /// one predecessor.
-bool FoldSingleEntryPHINodes(BasicBlock *BB,
-                             MemoryDependenceResults *MemDep = nullptr);
+LLVM_ABI bool
+FoldSingleEntryPHINodes(BasicBlock *BB,
+                        MemoryDependenceResults *MemDep = nullptr);
 
 /// Examine each PHI in the given block and delete it if it is dead. Also
 /// recursively delete any operands that become dead as a result. This includes
 /// tracing the def-use list from the PHI to see if it is ultimately unused or
 /// if it reaches an unused cycle. Return true if any PHIs were deleted.
-bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI = nullptr,
-                    MemorySSAUpdater *MSSAU = nullptr);
+LLVM_ABI bool DeleteDeadPHIs(BasicBlock *BB,
+                             const TargetLibraryInfo *TLI = nullptr,
+                             MemorySSAUpdater *MSSAU = nullptr);
 
 /// Attempts to merge a block into its predecessor, if possible. The return
 /// value indicates success or failure.
@@ -93,12 +98,11 @@ bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI = nullptr,
 /// BB, and BB will still be merged into its predecessor and removed.
 /// If \p DT is not nullptr, update it directly; in that case, DTU must be
 /// nullptr.
-bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU = nullptr,
-                               LoopInfo *LI = nullptr,
-                               MemorySSAUpdater *MSSAU = nullptr,
-                               MemoryDependenceResults *MemDep = nullptr,
-                               bool PredecessorWithTwoSuccessors = false,
-                               DominatorTree *DT = nullptr);
+LLVM_ABI bool MergeBlockIntoPredecessor(
+    BasicBlock *BB, DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
+    MemorySSAUpdater *MSSAU = nullptr,
+    MemoryDependenceResults *MemDep = nullptr,
+    bool PredecessorWithTwoSuccessors = false, DominatorTree *DT = nullptr);
 
 /// Merge block(s) sucessors, if possible. Return true if at least two
 /// of the blocks were merged together.
@@ -107,36 +111,36 @@ bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU = nullptr,
 /// must be in L. In addition, This utility calls on another utility:
 /// MergeBlockIntoPredecessor. Blocks are successfully merged when the call to
 /// MergeBlockIntoPredecessor returns true.
-bool MergeBlockSuccessorsIntoGivenBlocks(
+LLVM_ABI bool MergeBlockSuccessorsIntoGivenBlocks(
     SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L = nullptr,
     DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr);
 
 /// Try to remove redundant dbg.value instructions from given basic block.
 /// Returns true if at least one instruction was removed. Remove redundant
 /// pseudo ops when RemovePseudoOp is true.
-bool RemoveRedundantDbgInstrs(BasicBlock *BB);
+LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB);
 
 /// Replace all uses of an instruction (specified by BI) with a value, then
 /// remove and delete the original instruction.
-void ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V);
+LLVM_ABI void ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V);
 
 /// Replace the instruction specified by BI with the instruction specified by I.
 /// Copies DebugLoc from BI to I, if I doesn't already have a DebugLoc. The
 /// original instruction is deleted and BI is updated to point to the new
 /// instruction.
-void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
-                         Instruction *I);
+LLVM_ABI void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
+                                  Instruction *I);
 
 /// Replace the instruction specified by From with the instruction specified by
 /// To. Copies DebugLoc from BI to I, if I doesn't already have a DebugLoc.
-void ReplaceInstWithInst(Instruction *From, Instruction *To);
+LLVM_ABI void ReplaceInstWithInst(Instruction *From, Instruction *To);
 
 /// Check if we can prove that all paths starting from this block converge
 /// to a block that either has a @llvm.experimental.deoptimize call
 /// prior to its terminating return instruction or is terminated by unreachable.
 /// All blocks in the traversed sequence must have an unique successor, maybe
 /// except for the last one.
-bool IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB);
+LLVM_ABI bool IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB);
 
 /// Option class for critical edge splitting.
 ///
@@ -192,8 +196,9 @@ struct CriticalEdgeSplittingOptions {
 /// exit block. This function inserts the new PHIs, as needed. Preds is a list
 /// of preds inside the loop, SplitBB is the new loop exit block, and DestBB is
 /// the old loop exit, now the successor of SplitBB.
-void createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
-                                BasicBlock *SplitBB, BasicBlock *DestBB);
+LLVM_ABI void createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
+                                         BasicBlock *SplitBB,
+                                         BasicBlock *DestBB);
 
 /// If this edge is a critical edge, insert a new node to split the critical
 /// edge. This will update the analyses passed in through the option struct.
@@ -211,17 +216,19 @@ void createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
 /// IndirectBrInst.  Splitting these edges will almost always create an invalid
 /// program because the address of the new block won't be the one that is jumped
 /// to.
-BasicBlock *SplitCriticalEdge(Instruction *TI, unsigned SuccNum,
-                              const CriticalEdgeSplittingOptions &Options =
-                                  CriticalEdgeSplittingOptions(),
-                              const Twine &BBName = "");
+LLVM_ABI BasicBlock *
+SplitCriticalEdge(Instruction *TI, unsigned SuccNum,
+                  const CriticalEdgeSplittingOptions &Options =
+                      CriticalEdgeSplittingOptions(),
+                  const Twine &BBName = "");
 
 /// If it is known that an edge is critical, SplitKnownCriticalEdge can be
 /// called directly, rather than calling SplitCriticalEdge first.
-BasicBlock *SplitKnownCriticalEdge(Instruction *TI, unsigned SuccNum,
-                                   const CriticalEdgeSplittingOptions &Options =
-                                       CriticalEdgeSplittingOptions(),
-                                   const Twine &BBName = "");
+LLVM_ABI BasicBlock *
+SplitKnownCriticalEdge(Instruction *TI, unsigned SuccNum,
+                       const CriticalEdgeSplittingOptions &Options =
+                           CriticalEdgeSplittingOptions(),
+                       const Twine &BBName = "");
 
 /// If an edge from Src to Dst is critical, split the edge and return true,
 /// otherwise return false. This method requires that there be an edge between
@@ -242,33 +249,35 @@ SplitCriticalEdge(BasicBlock *Src, BasicBlock *Dst,
 
 /// Loop over all of the edges in the CFG, breaking critical edges as they are
 /// found. Returns the number of broken edges.
-unsigned SplitAllCriticalEdges(Function &F,
-                               const CriticalEdgeSplittingOptions &Options =
-                                   CriticalEdgeSplittingOptions());
+LLVM_ABI unsigned
+SplitAllCriticalEdges(Function &F, const CriticalEdgeSplittingOptions &Options =
+                                       CriticalEdgeSplittingOptions());
 
 /// Split the edge connecting the specified blocks, and return the newly created
 /// basic block between \p From and \p To.
-BasicBlock *SplitEdge(BasicBlock *From, BasicBlock *To,
-                      DominatorTree *DT = nullptr, LoopInfo *LI = nullptr,
-                      MemorySSAUpdater *MSSAU = nullptr,
-                      const Twine &BBName = "");
+LLVM_ABI BasicBlock *SplitEdge(BasicBlock *From, BasicBlock *To,
+                               DominatorTree *DT = nullptr,
+                               LoopInfo *LI = nullptr,
+                               MemorySSAUpdater *MSSAU = nullptr,
+                               const Twine &BBName = "");
 
 /// Sets the unwind edge of an instruction to a particular successor.
-void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ);
+LLVM_ABI void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ);
 
 /// Replaces all uses of OldPred with the NewPred block in all PHINodes in a
 /// block.
-void updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
-                    BasicBlock *NewPred, PHINode *Until = nullptr);
+LLVM_ABI void updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
+                             BasicBlock *NewPred, PHINode *Until = nullptr);
 
 /// Split the edge connect the specficed blocks in the case that \p Succ is an
 /// Exception Handling Block
-BasicBlock *ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
-                             LandingPadInst *OriginalPad = nullptr,
-                             PHINode *LandingPadReplacement = nullptr,
-                             const CriticalEdgeSplittingOptions &Options =
-                                 CriticalEdgeSplittingOptions(),
-                             const Twine &BBName = "");
+LLVM_ABI BasicBlock *
+ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
+                 LandingPadInst *OriginalPad = nullptr,
+                 PHINode *LandingPadReplacement = nullptr,
+                 const CriticalEdgeSplittingOptions &Options =
+                     CriticalEdgeSplittingOptions(),
+                 const Twine &BBName = "");
 
 /// Split the specified block at the specified instruction.
 ///
@@ -280,10 +289,10 @@ BasicBlock *ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
 /// branch. The new block with name \p BBName is returned.
 ///
 /// FIXME: deprecated, switch to the DomTreeUpdater-based one.
-BasicBlock *SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT,
-                       LoopInfo *LI = nullptr,
-                       MemorySSAUpdater *MSSAU = nullptr,
-                       const Twine &BBName = "", bool Before = false);
+LLVM_ABI BasicBlock *SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
+                                DominatorTree *DT, LoopInfo *LI = nullptr,
+                                MemorySSAUpdater *MSSAU = nullptr,
+                                const Twine &BBName = "", bool Before = false);
 inline BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT,
                        LoopInfo *LI = nullptr,
                        MemorySSAUpdater *MSSAU = nullptr,
@@ -299,10 +308,11 @@ inline BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTr
 /// Everything before \p SplitPt stays in \p Old and everything starting with \p
 /// SplitPt moves to a new block. The two blocks are joined by an unconditional
 /// branch. The new block with name \p BBName is returned.
-BasicBlock *SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
-                       DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
-                       MemorySSAUpdater *MSSAU = nullptr,
-                       const Twine &BBName = "", bool Before = false);
+LLVM_ABI BasicBlock *SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
+                                DomTreeUpdater *DTU = nullptr,
+                                LoopInfo *LI = nullptr,
+                                MemorySSAUpdater *MSSAU = nullptr,
+                                const Twine &BBName = "", bool Before = false);
 inline BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt,
                        DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
                        MemorySSAUpdater *MSSAU = nullptr,
@@ -315,9 +325,11 @@ inline BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt,
 /// instructions after \p SplitPt stay in the old block. The new block and the
 /// old block are joined by inserting an unconditional branch to the end of the
 /// new block. The new block with name \p BBName is returned.
-BasicBlock *splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt,
-                             DomTreeUpdater *DTU, LoopInfo *LI,
-                             MemorySSAUpdater *MSSAU, const Twine &BBName = "");
+LLVM_ABI BasicBlock *splitBlockBefore(BasicBlock *Old,
+                                      BasicBlock::iterator SplitPt,
+                                      DomTreeUpdater *DTU, LoopInfo *LI,
+                                      MemorySSAUpdater *MSSAU,
+                                      const Twine &BBName = "");
 inline BasicBlock *splitBlockBefore(BasicBlock *Old, Instruction *SplitPt,
                              DomTreeUpdater *DTU, LoopInfo *LI,
                              MemorySSAUpdater *MSSAU, const Twine &BBName = "") {
@@ -340,11 +352,10 @@ inline BasicBlock *splitBlockBefore(BasicBlock *Old, Instruction *SplitPt,
 /// split is an exit of a loop with other exits).
 ///
 /// FIXME: deprecated, switch to the DomTreeUpdater-based one.
-BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
-                                   const char *Suffix, DominatorTree *DT,
-                                   LoopInfo *LI = nullptr,
-                                   MemorySSAUpdater *MSSAU = nullptr,
-                                   bool PreserveLCSSA = false);
+LLVM_ABI BasicBlock *SplitBlockPredecessors(
+    BasicBlock *BB, ArrayRef<BasicBlock *> Preds, const char *Suffix,
+    DominatorTree *DT, LoopInfo *LI = nullptr,
+    MemorySSAUpdater *MSSAU = nullptr, bool PreserveLCSSA = false);
 
 /// This method introduces at least one new basic block into the function and
 /// moves some of the predecessors of BB to be predecessors of the new block.
@@ -360,12 +371,10 @@ BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
 /// no other analyses. In particular, it does not preserve LoopSimplify
 /// (because it's complicated to handle the case where one of the edges being
 /// split is an exit of a loop with other exits).
-BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
-                                   const char *Suffix,
-                                   DomTreeUpdater *DTU = nullptr,
-                                   LoopInfo *LI = nullptr,
-                                   MemorySSAUpdater *MSSAU = nullptr,
-                                   bool PreserveLCSSA = false);
+LLVM_ABI BasicBlock *SplitBlockPredecessors(
+    BasicBlock *BB, ArrayRef<BasicBlock *> Preds, const char *Suffix,
+    DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
+    MemorySSAUpdater *MSSAU = nullptr, bool PreserveLCSSA = false);
 
 /// This method transforms the landing pad, OrigBB, by introducing two new basic
 /// blocks into the function. One of those new basic blocks gets the
@@ -378,7 +387,7 @@ BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
 /// no other analyses. In particular, it does not preserve LoopSimplify
 /// (because it's complicated to handle the case where one of the edges being
 /// split is an exit of a loop with other exits).
-void SplitLandingPadPredecessors(
+LLVM_ABI void SplitLandingPadPredecessors(
     BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix,
     const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
     DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
@@ -388,9 +397,9 @@ void SplitLandingPadPredecessors(
 /// which ends in an unconditional branch. If the return instruction returns a
 /// value defined by a PHI, propagate the right value into the return. It
 /// returns the new return instruction in the predecessor.
-ReturnInst *FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
-                                       BasicBlock *Pred,
-                                       DomTreeUpdater *DTU = nullptr);
+LLVM_ABI ReturnInst *FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
+                                                BasicBlock *Pred,
+                                                DomTreeUpdater *DTU = nullptr);
 
 /// Split the containing block at the specified instruction - everything before
 /// SplitBefore stays in the old basic block, and the rest of the instructions
@@ -413,12 +422,11 @@ ReturnInst *FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
 /// Returns the NewBasicBlock's terminator.
 ///
 /// Updates DTU and LI if given.
-Instruction *SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore,
-                                       bool Unreachable,
-                                       MDNode *BranchWeights = nullptr,
-                                       DomTreeUpdater *DTU = nullptr,
-                                       LoopInfo *LI = nullptr,
-                                       BasicBlock *ThenBlock = nullptr);
+LLVM_ABI Instruction *
+SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore,
+                          bool Unreachable, MDNode *BranchWeights = nullptr,
+                          DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
+                          BasicBlock *ThenBlock = nullptr);
 
 inline Instruction *SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore,
                                        bool Unreachable,
@@ -433,12 +441,11 @@ inline Instruction *SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBef
 
 /// Similar to SplitBlockAndInsertIfThen, but the inserted block is on the false
 /// path of the branch.
-Instruction *SplitBlockAndInsertIfElse(Value *Cond, BasicBlock::iterator SplitBefore,
-                                       bool Unreachable,
-                                       MDNode *BranchWeights = nullptr,
-                                       DomTreeUpdater *DTU = nullptr,
-                                       LoopInfo *LI = nullptr,
-                                       BasicBlock *ElseBlock = nullptr);
+LLVM_ABI Instruction *
+SplitBlockAndInsertIfElse(Value *Cond, BasicBlock::iterator SplitBefore,
+                          bool Unreachable, MDNode *BranchWeights = nullptr,
+                          DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
+                          BasicBlock *ElseBlock = nullptr);
 
 inline Instruction *SplitBlockAndInsertIfElse(Value *Cond, Instruction *SplitBefore,
                                        bool Unreachable,
@@ -467,13 +474,10 @@ inline Instruction *SplitBlockAndInsertIfElse(Value *Cond, Instruction *SplitBef
 ///   Tail
 ///
 /// Updates DT if given.
-void SplitBlockAndInsertIfThenElse(Value *Cond,
-                                   BasicBlock::iterator SplitBefore,
-                                   Instruction **ThenTerm,
-                                   Instruction **ElseTerm,
-                                   MDNode *BranchWeights = nullptr,
-                                   DomTreeUpdater *DTU = nullptr,
-                                   LoopInfo *LI = nullptr);
+LLVM_ABI void SplitBlockAndInsertIfThenElse(
+    Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm,
+    Instruction **ElseTerm, MDNode *BranchWeights = nullptr,
+    DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr);
 
 inline void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
                                    Instruction **ThenTerm,
@@ -513,15 +517,11 @@ inline void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
 /// caller must ensure that Tail is reachable from Head.
 /// Returns the newly created blocks in \p ThenBlock and \p ElseBlock.
 /// Updates DTU and LI if given.
-void SplitBlockAndInsertIfThenElse(Value *Cond,
-                                   BasicBlock::iterator SplitBefore,
-                                   BasicBlock **ThenBlock,
-                                   BasicBlock **ElseBlock,
-                                   bool UnreachableThen = false,
-                                   bool UnreachableElse = false,
-                                   MDNode *BranchWeights = nullptr,
-                                   DomTreeUpdater *DTU = nullptr,
-                                   LoopInfo *LI = nullptr);
+LLVM_ABI void SplitBlockAndInsertIfThenElse(
+    Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
+    BasicBlock **ElseBlock, bool UnreachableThen = false,
+    bool UnreachableElse = false, MDNode *BranchWeights = nullptr,
+    DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr);
 
 inline void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
                                    BasicBlock **ThenBlock,
@@ -539,7 +539,7 @@ inline void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore,
 /// that \p End is assumed > 0, and thus not checked on entry) at \p
 /// SplitBefore.  Returns the first insert point in the loop body, and the
 /// PHINode for the induction variable (i.e. "i" above).
-std::pair<Instruction*, Value*>
+LLVM_ABI std::pair<Instruction *, Value *>
 SplitBlockAndInsertSimpleForLoop(Value *End, BasicBlock::iterator SplitBefore);
 
 /// Utility function for performing a given action on each lane of a vector
@@ -550,7 +550,7 @@ SplitBlockAndInsertSimpleForLoop(Value *End, BasicBlock::iterator SplitBefore);
 /// IRBuilder whose insert point is correctly set for instantiating the
 /// given index, and a value which is (at runtime) the index to access.
 /// This index *may* be a constant.
-void SplitBlockAndInsertForEachLane(
+LLVM_ABI void SplitBlockAndInsertForEachLane(
     ElementCount EC, Type *IndexTy, BasicBlock::iterator InsertBefore,
     std::function<void(IRBuilderBase &, Value *)> Func);
 
@@ -562,7 +562,7 @@ void SplitBlockAndInsertForEachLane(
 /// arguments an IRBuilder whose insert point is correctly set for instantiating
 /// the given index, and a value which is (at runtime) the index to access. This
 /// index *may* be a constant.
-void SplitBlockAndInsertForEachLane(
+LLVM_ABI void SplitBlockAndInsertForEachLane(
     Value *End, BasicBlock::iterator InsertBefore,
     std::function<void(IRBuilderBase &, Value *)> Func);
 
@@ -574,8 +574,8 @@ void SplitBlockAndInsertForEachLane(
 ///
 /// This does no checking to see if the true/false blocks have large or unsavory
 /// instructions in them.
-BranchInst *GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
-                           BasicBlock *&IfFalse);
+LLVM_ABI BranchInst *GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
+                                    BasicBlock *&IfFalse);
 
 // Split critical edges where the source of the edge is an indirectbr
 // instruction. This isn't always possible, but we can handle some easy cases.
@@ -598,17 +598,18 @@ BranchInst *GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
 // If BPI and BFI aren't non-null, BPI/BFI will be updated accordingly.
 // When `IgnoreBlocksWithoutPHI` is set to `true` critical edges leading to a
 // block without phi-instructions will not be split.
-bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI,
-                                  BranchProbabilityInfo *BPI = nullptr,
-                                  BlockFrequencyInfo *BFI = nullptr);
+LLVM_ABI bool SplitIndirectBrCriticalEdges(Function &F,
+                                           bool IgnoreBlocksWithoutPHI,
+                                           BranchProbabilityInfo *BPI = nullptr,
+                                           BlockFrequencyInfo *BFI = nullptr);
 
 // Utility function for inverting branch condition and for swapping its
 // successors
-void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder);
+LLVM_ABI void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder);
 
 // Check whether the function only has simple terminator:
 // br/brcond/unreachable/ret
-bool hasOnlySimpleTerminator(const Function &F);
+LLVM_ABI bool hasOnlySimpleTerminator(const Function &F);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
index 50f695dbe6c0..f53e7d056ca9 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_UTILS_BUILDLIBCALLS_H
 
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
   class Value;
@@ -28,17 +29,22 @@ namespace llvm {
   /// If the library function is unavailable, this doesn't modify it.
   ///
   /// Returns true if any attributes were set and false otherwise.
-  bool inferNonMandatoryLibFuncAttrs(Module *M, StringRef Name,
-                                     const TargetLibraryInfo &TLI);
-  bool inferNonMandatoryLibFuncAttrs(Function &F, const TargetLibraryInfo &TLI);
+  LLVM_ABI bool inferNonMandatoryLibFuncAttrs(Module *M, StringRef Name,
+                                              const TargetLibraryInfo &TLI);
+  LLVM_ABI bool inferNonMandatoryLibFuncAttrs(Function &F,
+                                              const TargetLibraryInfo &TLI);
 
   /// Calls getOrInsertFunction() and then makes sure to add mandatory
   /// argument attributes.
-  FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI,
-                                    LibFunc TheLibFunc, FunctionType *T,
-                                    AttributeList AttributeList);
-  FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI,
-                                    LibFunc TheLibFunc, FunctionType *T);
+  LLVM_ABI FunctionCallee getOrInsertLibFunc(Module *M,
+                                             const TargetLibraryInfo &TLI,
+                                             LibFunc TheLibFunc,
+                                             FunctionType *T,
+                                             AttributeList AttributeList);
+  LLVM_ABI FunctionCallee getOrInsertLibFunc(Module *M,
+                                             const TargetLibraryInfo &TLI,
+                                             LibFunc TheLibFunc,
+                                             FunctionType *T);
   template <typename... ArgsTy>
   FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI,
                                LibFunc TheLibFunc, AttributeList AttributeList,
@@ -67,219 +73,238 @@ namespace llvm {
   // function signatures; it does not apply other relevant attributes for
   // function signatures, including sign/zero-extension for arguments and return
   // values.
-  void markRegisterParameterAttributes(Function *F);
+  LLVM_ABI void markRegisterParameterAttributes(Function *F);
 
   /// Check whether the library function is available on target and also that
   /// it in the current Module is a Function with the right type.
-  bool isLibFuncEmittable(const Module *M, const TargetLibraryInfo *TLI,
-                          LibFunc TheLibFunc);
-  bool isLibFuncEmittable(const Module *M, const TargetLibraryInfo *TLI,
-                          StringRef Name);
+  LLVM_ABI bool isLibFuncEmittable(const Module *M,
+                                   const TargetLibraryInfo *TLI,
+                                   LibFunc TheLibFunc);
+  LLVM_ABI bool isLibFuncEmittable(const Module *M,
+                                   const TargetLibraryInfo *TLI,
+                                   StringRef Name);
 
   /// Check whether the overloaded floating point function
   /// corresponding to \a Ty is available.
-  bool hasFloatFn(const Module *M, const TargetLibraryInfo *TLI, Type *Ty,
-                  LibFunc DoubleFn, LibFunc FloatFn, LibFunc LongDoubleFn);
+  LLVM_ABI bool hasFloatFn(const Module *M, const TargetLibraryInfo *TLI,
+                           Type *Ty, LibFunc DoubleFn, LibFunc FloatFn,
+                           LibFunc LongDoubleFn);
 
   /// Get the name of the overloaded floating point function
   /// corresponding to \a Ty. Return the LibFunc in \a TheLibFunc.
-  StringRef getFloatFn(const Module *M, const TargetLibraryInfo *TLI, Type *Ty,
-                       LibFunc DoubleFn, LibFunc FloatFn, LibFunc LongDoubleFn,
-                       LibFunc &TheLibFunc);
+  LLVM_ABI StringRef getFloatFn(const Module *M, const TargetLibraryInfo *TLI,
+                                Type *Ty, LibFunc DoubleFn, LibFunc FloatFn,
+                                LibFunc LongDoubleFn, LibFunc &TheLibFunc);
 
   /// Emit a call to the strlen function to the builder, for the specified
   /// pointer. Ptr is required to be some pointer type, and the return value has
   /// 'size_t' type.
-  Value *emitStrLen(Value *Ptr, IRBuilderBase &B, const DataLayout &DL,
-                    const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrLen(Value *Ptr, IRBuilderBase &B, const DataLayout &DL,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the wcslen function to the builder, for the specified
   /// pointer. Ptr is required to be some pointer type, and the return value has
   /// 'size_t' type.
-  Value *emitWcsLen(Value *Ptr, IRBuilderBase &B, const DataLayout &DL,
-                    const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitWcsLen(Value *Ptr, IRBuilderBase &B, const DataLayout &DL,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strdup function to the builder, for the specified
   /// pointer. Ptr is required to be some pointer type, and the return value has
   /// 'i8*' type.
-  Value *emitStrDup(Value *Ptr, IRBuilderBase &B, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrDup(Value *Ptr, IRBuilderBase &B,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strchr function to the builder, for the specified
   /// pointer and character. Ptr is required to be some pointer type, and the
   /// return value has 'i8*' type.
-  Value *emitStrChr(Value *Ptr, char C, IRBuilderBase &B,
-                    const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrChr(Value *Ptr, char C, IRBuilderBase &B,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strncmp function to the builder.
-  Value *emitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
-                     const DataLayout &DL, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len,
+                              IRBuilderBase &B, const DataLayout &DL,
+                              const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strcpy function to the builder, for the specified
   /// pointer arguments.
-  Value *emitStrCpy(Value *Dst, Value *Src, IRBuilderBase &B,
-                    const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrCpy(Value *Dst, Value *Src, IRBuilderBase &B,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the stpcpy function to the builder, for the specified
   /// pointer arguments.
-  Value *emitStpCpy(Value *Dst, Value *Src, IRBuilderBase &B,
-                    const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStpCpy(Value *Dst, Value *Src, IRBuilderBase &B,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strncpy function to the builder, for the specified
   /// pointer arguments and length.
-  Value *emitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
-                     const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrNCpy(Value *Dst, Value *Src, Value *Len,
+                              IRBuilderBase &B, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the stpncpy function to the builder, for the specified
   /// pointer arguments and length.
-  Value *emitStpNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
-                     const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStpNCpy(Value *Dst, Value *Src, Value *Len,
+                              IRBuilderBase &B, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the __memcpy_chk function to the builder. This expects that
   /// the Len and ObjSize have type 'size_t' and Dst/Src are pointers.
-  Value *emitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize,
-                       IRBuilderBase &B, const DataLayout &DL,
-                       const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitMemCpyChk(Value *Dst, Value *Src, Value *Len,
+                                Value *ObjSize, IRBuilderBase &B,
+                                const DataLayout &DL,
+                                const TargetLibraryInfo *TLI);
 
   /// Emit a call to the mempcpy function.
-  Value *emitMemPCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B,
-                     const DataLayout &DL, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitMemPCpy(Value *Dst, Value *Src, Value *Len,
+                              IRBuilderBase &B, const DataLayout &DL,
+                              const TargetLibraryInfo *TLI);
 
   /// Emit a call to the memchr function. This assumes that Ptr is a pointer,
   /// Val is an 'int' value, and Len is an 'size_t' value.
-  Value *emitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B,
-                    const DataLayout &DL, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitMemChr(Value *Ptr, Value *Val, Value *Len,
+                             IRBuilderBase &B, const DataLayout &DL,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the memrchr function, analogously to emitMemChr.
-  Value *emitMemRChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B,
-                    const DataLayout &DL, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitMemRChr(Value *Ptr, Value *Val, Value *Len,
+                              IRBuilderBase &B, const DataLayout &DL,
+                              const TargetLibraryInfo *TLI);
 
   /// Emit a call to the memcmp function.
-  Value *emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
-                    const DataLayout &DL, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len,
+                             IRBuilderBase &B, const DataLayout &DL,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the bcmp function.
-  Value *emitBCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B,
-                  const DataLayout &DL, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitBCmp(Value *Ptr1, Value *Ptr2, Value *Len,
+                           IRBuilderBase &B, const DataLayout &DL,
+                           const TargetLibraryInfo *TLI);
 
   /// Emit a call to the memccpy function.
-  Value *emitMemCCpy(Value *Ptr1, Value *Ptr2, Value *Val, Value *Len,
-                     IRBuilderBase &B, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitMemCCpy(Value *Ptr1, Value *Ptr2, Value *Val, Value *Len,
+                              IRBuilderBase &B, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the snprintf function.
-  Value *emitSNPrintf(Value *Dest, Value *Size, Value *Fmt,
-                      ArrayRef<Value *> Args, IRBuilderBase &B,
-                      const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitSNPrintf(Value *Dest, Value *Size, Value *Fmt,
+                               ArrayRef<Value *> Args, IRBuilderBase &B,
+                               const TargetLibraryInfo *TLI);
 
   /// Emit a call to the sprintf function.
-  Value *emitSPrintf(Value *Dest, Value *Fmt, ArrayRef<Value *> VariadicArgs,
-                     IRBuilderBase &B, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitSPrintf(Value *Dest, Value *Fmt,
+                              ArrayRef<Value *> VariadicArgs, IRBuilderBase &B,
+                              const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strcat function.
-  Value *emitStrCat(Value *Dest, Value *Src, IRBuilderBase &B,
-                    const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrCat(Value *Dest, Value *Src, IRBuilderBase &B,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strlcpy function.
-  Value *emitStrLCpy(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
-                     const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrLCpy(Value *Dest, Value *Src, Value *Size,
+                              IRBuilderBase &B, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strlcat function.
-  Value *emitStrLCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
-                     const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrLCat(Value *Dest, Value *Src, Value *Size,
+                              IRBuilderBase &B, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the strncat function.
-  Value *emitStrNCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B,
-                     const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitStrNCat(Value *Dest, Value *Src, Value *Size,
+                              IRBuilderBase &B, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the vsnprintf function.
-  Value *emitVSNPrintf(Value *Dest, Value *Size, Value *Fmt, Value *VAList,
-                       IRBuilderBase &B, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitVSNPrintf(Value *Dest, Value *Size, Value *Fmt,
+                                Value *VAList, IRBuilderBase &B,
+                                const TargetLibraryInfo *TLI);
 
   /// Emit a call to the vsprintf function.
-  Value *emitVSPrintf(Value *Dest, Value *Fmt, Value *VAList, IRBuilderBase &B,
-                      const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitVSPrintf(Value *Dest, Value *Fmt, Value *VAList,
+                               IRBuilderBase &B, const TargetLibraryInfo *TLI);
 
   /// Emit a call to the unary function named 'Name' (e.g.  'floor'). This
   /// function is known to take a single of type matching 'Op' and returns one
   /// value with the same type. If 'Op' is a long double, 'l' is added as the
   /// suffix of name, if 'Op' is a float, we add a 'f' suffix.
-  Value *emitUnaryFloatFnCall(Value *Op, const TargetLibraryInfo *TLI,
-                              StringRef Name, IRBuilderBase &B,
-                              const AttributeList &Attrs);
+  LLVM_ABI Value *emitUnaryFloatFnCall(Value *Op, const TargetLibraryInfo *TLI,
+                                       StringRef Name, IRBuilderBase &B,
+                                       const AttributeList &Attrs);
 
   /// Emit a call to the unary function DoubleFn, FloatFn or LongDoubleFn,
   /// depending of the type of Op.
-  Value *emitUnaryFloatFnCall(Value *Op, const TargetLibraryInfo *TLI,
-                              LibFunc DoubleFn, LibFunc FloatFn,
-                              LibFunc LongDoubleFn, IRBuilderBase &B,
-                              const AttributeList &Attrs);
+  LLVM_ABI Value *emitUnaryFloatFnCall(Value *Op, const TargetLibraryInfo *TLI,
+                                       LibFunc DoubleFn, LibFunc FloatFn,
+                                       LibFunc LongDoubleFn, IRBuilderBase &B,
+                                       const AttributeList &Attrs);
 
   /// Emit a call to the binary function named 'Name' (e.g. 'fmin'). This
   /// function is known to take type matching 'Op1' and 'Op2' and return one
   /// value with the same type. If 'Op1/Op2' are long double, 'l' is added as
   /// the suffix of name, if 'Op1/Op2' are float, we add a 'f' suffix.
-  Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2,
-                               const TargetLibraryInfo *TLI,
-                               StringRef Name, IRBuilderBase &B,
-                               const AttributeList &Attrs);
+  LLVM_ABI Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2,
+                                        const TargetLibraryInfo *TLI,
+                                        StringRef Name, IRBuilderBase &B,
+                                        const AttributeList &Attrs);
 
   /// Emit a call to the binary function DoubleFn, FloatFn or LongDoubleFn,
   /// depending of the type of Op1.
-  Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2,
-                               const TargetLibraryInfo *TLI, LibFunc DoubleFn,
-                               LibFunc FloatFn, LibFunc LongDoubleFn,
-                               IRBuilderBase &B, const AttributeList &Attrs);
+  LLVM_ABI Value *emitBinaryFloatFnCall(Value *Op1, Value *Op2,
+                                        const TargetLibraryInfo *TLI,
+                                        LibFunc DoubleFn, LibFunc FloatFn,
+                                        LibFunc LongDoubleFn, IRBuilderBase &B,
+                                        const AttributeList &Attrs);
 
   /// Emit a call to the putchar function. This assumes that Char is an 'int'.
-  Value *emitPutChar(Value *Char, IRBuilderBase &B,
-                     const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitPutChar(Value *Char, IRBuilderBase &B,
+                              const TargetLibraryInfo *TLI);
 
   /// Emit a call to the puts function. This assumes that Str is some pointer.
-  Value *emitPutS(Value *Str, IRBuilderBase &B, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitPutS(Value *Str, IRBuilderBase &B,
+                           const TargetLibraryInfo *TLI);
 
   /// Emit a call to the fputc function. This assumes that Char is an 'int', and
   /// File is a pointer to FILE.
-  Value *emitFPutC(Value *Char, Value *File, IRBuilderBase &B,
-                   const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitFPutC(Value *Char, Value *File, IRBuilderBase &B,
+                            const TargetLibraryInfo *TLI);
 
   /// Emit a call to the fputs function. Str is required to be a pointer and
   /// File is a pointer to FILE.
-  Value *emitFPutS(Value *Str, Value *File, IRBuilderBase &B,
-                   const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitFPutS(Value *Str, Value *File, IRBuilderBase &B,
+                            const TargetLibraryInfo *TLI);
 
   /// Emit a call to the fwrite function. This assumes that Ptr is a pointer,
   /// Size is an 'size_t', and File is a pointer to FILE.
-  Value *emitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilderBase &B,
-                    const DataLayout &DL, const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitFWrite(Value *Ptr, Value *Size, Value *File,
+                             IRBuilderBase &B, const DataLayout &DL,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the malloc function.
-  Value *emitMalloc(Value *Num, IRBuilderBase &B, const DataLayout &DL,
-                    const TargetLibraryInfo *TLI);
+  LLVM_ABI Value *emitMalloc(Value *Num, IRBuilderBase &B, const DataLayout &DL,
+                             const TargetLibraryInfo *TLI);
 
   /// Emit a call to the calloc function.
-  Value *emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
-                    const TargetLibraryInfo &TLI, unsigned AddrSpace);
+  LLVM_ABI Value *emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
+                             const TargetLibraryInfo &TLI, unsigned AddrSpace);
 
   /// Emit a call to the hot/cold operator new function.
-  Value *emitHotColdNew(Value *Num, IRBuilderBase &B,
-                        const TargetLibraryInfo *TLI, LibFunc NewFunc,
-                        uint8_t HotCold);
-  Value *emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
-                               const TargetLibraryInfo *TLI, LibFunc NewFunc,
-                               uint8_t HotCold);
-  Value *emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
-                               const TargetLibraryInfo *TLI, LibFunc NewFunc,
-                               uint8_t HotCold);
-  Value *emitHotColdNewAlignedNoThrow(Value *Num, Value *Align, Value *NoThrow,
-                                      IRBuilderBase &B,
-                                      const TargetLibraryInfo *TLI,
-                                      LibFunc NewFunc, uint8_t HotCold);
-  Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+  LLVM_ABI Value *emitHotColdNew(Value *Num, IRBuilderBase &B,
+                                 const TargetLibraryInfo *TLI, LibFunc NewFunc,
+                                 uint8_t HotCold);
+  LLVM_ABI Value *emitHotColdNewNoThrow(Value *Num, Value *NoThrow,
+                                        IRBuilderBase &B,
+                                        const TargetLibraryInfo *TLI,
+                                        LibFunc NewFunc, uint8_t HotCold);
+  LLVM_ABI Value *emitHotColdNewAligned(Value *Num, Value *Align,
+                                        IRBuilderBase &B,
+                                        const TargetLibraryInfo *TLI,
+                                        LibFunc NewFunc, uint8_t HotCold);
+  LLVM_ABI Value *emitHotColdNewAlignedNoThrow(Value *Num, Value *Align,
+                                               Value *NoThrow, IRBuilderBase &B,
+                                               const TargetLibraryInfo *TLI,
+                                               LibFunc NewFunc,
+                                               uint8_t HotCold);
+  LLVM_ABI Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+                                              const TargetLibraryInfo *TLI,
+                                              LibFunc NewFunc, uint8_t HotCold);
+  LLVM_ABI Value *
+  emitHotColdSizeReturningNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
                                      const TargetLibraryInfo *TLI,
                                      LibFunc NewFunc, uint8_t HotCold);
-  Value *emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
-                                            IRBuilderBase &B,
-                                            const TargetLibraryInfo *TLI,
-                                            LibFunc NewFunc, uint8_t HotCold);
 }
 
 #endif
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h
index 8dc9648059fe..11122b12d0c6 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallGraphUpdater.h
@@ -17,6 +17,7 @@
 
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -66,25 +67,25 @@ class CallGraphUpdater {
   ///}
 
   /// Finalizer that will trigger actions like function removal from the CG.
-  bool finalize();
+  LLVM_ABI bool finalize();
 
   /// Remove \p Fn from the call graph.
-  void removeFunction(Function &Fn);
+  LLVM_ABI void removeFunction(Function &Fn);
 
   /// After an CGSCC pass changes a function in ways that affect the call
   /// graph, this method can be called to update it.
-  void reanalyzeFunction(Function &Fn);
+  LLVM_ABI void reanalyzeFunction(Function &Fn);
 
   /// If a new function was created by outlining, this method can be called
   /// to update the call graph for the new function. Note that the old one
   /// still needs to be re-analyzed or manually updated.
-  void registerOutlinedFunction(Function &OriginalFn, Function &NewFn);
+  LLVM_ABI void registerOutlinedFunction(Function &OriginalFn, Function &NewFn);
 
   /// Replace \p OldFn in the call graph (and SCC) with \p NewFn. The uses
   /// outside the call graph and the function \p OldFn are not modified.
   /// Note that \p OldFn is also removed from the call graph
   /// (\see removeFunction).
-  void replaceFunctionWith(Function &OldFn, Function &NewFn);
+  LLVM_ABI void replaceFunctionWith(Function &OldFn, Function &NewFn);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h
index 58af26f31417..e9660ac25bc8 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h
@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_UTILS_CALLPROMOTIONUTILS_H
 
 #include "llvm/Analysis/CtxProfAnalysis.h"
+#include "llvm/Support/Compiler.h"
 namespace llvm {
 template <typename T> class ArrayRef;
 class Constant;
@@ -32,8 +33,8 @@ class Value;
 /// match exactly, they must at least be bitcast compatible. If \p FailureReason
 /// is non-null and the indirect call cannot be promoted, the failure reason
 /// will be stored in it.
-bool isLegalToPromote(const CallBase &CB, Function *Callee,
-                      const char **FailureReason = nullptr);
+LLVM_ABI bool isLegalToPromote(const CallBase &CB, Function *Callee,
+                               const char **FailureReason = nullptr);
 
 /// Promote the given indirect call site to unconditionally call \p Callee.
 ///
@@ -42,8 +43,8 @@ bool isLegalToPromote(const CallBase &CB, Function *Callee,
 /// of the callee, bitcast instructions are inserted where appropriate. If \p
 /// RetBitCast is non-null, it will be used to store the return value bitcast,
 /// if created.
-CallBase &promoteCall(CallBase &CB, Function *Callee,
-                      CastInst **RetBitCast = nullptr);
+LLVM_ABI CallBase &promoteCall(CallBase &CB, Function *Callee,
+                               CastInst **RetBitCast = nullptr);
 
 /// Promote the given indirect call site to conditionally call \p Callee. The
 /// promoted direct call instruction is predicated on `CB.getCalledOperand() ==
@@ -54,11 +55,11 @@ CallBase &promoteCall(CallBase &CB, Function *Callee,
 /// indirect call site is promoted, placed in the "then" block, and returned. If
 /// \p BranchWeights is non-null, it will be used to set !prof metadata on the
 /// new conditional branch.
-CallBase &promoteCallWithIfThenElse(CallBase &CB, Function *Callee,
-                                    MDNode *BranchWeights = nullptr);
+LLVM_ABI CallBase &promoteCallWithIfThenElse(CallBase &CB, Function *Callee,
+                                             MDNode *BranchWeights = nullptr);
 
-CallBase *promoteCallWithIfThenElse(CallBase &CB, Function &Callee,
-                                    PGOContextualProfile &CtxProf);
+LLVM_ABI CallBase *promoteCallWithIfThenElse(CallBase &CB, Function &Callee,
+                                             PGOContextualProfile &CtxProf);
 
 /// This is similar to `promoteCallWithIfThenElse` except that the condition to
 /// promote a virtual call is that \p VPtr is the same as any of \p
@@ -71,10 +72,10 @@ CallBase *promoteCallWithIfThenElse(CallBase &CB, Function &Callee,
 ///
 /// TODO: sink the address-calculation instructions of indirect callee to the
 /// indirect call fallback after transformation.
-CallBase &promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr,
-                                   Function *Callee,
-                                   ArrayRef<Constant *> AddressPoints,
-                                   MDNode *BranchWeights);
+LLVM_ABI CallBase &promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr,
+                                            Function *Callee,
+                                            ArrayRef<Constant *> AddressPoints,
+                                            MDNode *BranchWeights);
 
 /// Try to promote (devirtualize) a virtual call on an Alloca. Return true on
 /// success.
@@ -97,7 +98,7 @@ CallBase &promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr,
 ///     [i8* null, i8* bitcast ({ i8*, i8*, i8* }* @_ZTI4Impl to i8*),
 ///     i8* bitcast (void (%class.Impl*)* @_ZN4Impl3RunEv to i8*)] }
 ///
-bool tryPromoteCall(CallBase &CB);
+LLVM_ABI bool tryPromoteCall(CallBase &CB);
 
 /// Predicate and clone the given call site.
 ///
@@ -106,7 +107,8 @@ bool tryPromoteCall(CallBase &CB);
 /// the given callee. The original call site is moved into the "else" block,
 /// and a clone of the call site is placed in the "then" block. The cloned
 /// instruction is returned.
-CallBase &versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights);
+LLVM_ABI CallBase &versionCallSite(CallBase &CB, Value *Callee,
+                                   MDNode *BranchWeights);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h
index 58a5faf57a36..6b56230a6e1d 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h
@@ -24,6 +24,7 @@
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <functional>
 #include <memory>
@@ -49,14 +50,15 @@ class ReturnInst;
 class DomTreeUpdater;
 
 /// Return an exact copy of the specified module
-std::unique_ptr<Module> CloneModule(const Module &M);
-std::unique_ptr<Module> CloneModule(const Module &M, ValueToValueMapTy &VMap);
+LLVM_ABI std::unique_ptr<Module> CloneModule(const Module &M);
+LLVM_ABI std::unique_ptr<Module> CloneModule(const Module &M,
+                                             ValueToValueMapTy &VMap);
 
 /// Return a copy of the specified module. The ShouldCloneDefinition function
 /// controls whether a specific GlobalValue's definition is cloned. If the
 /// function returns false, the module copy will contain an external reference
 /// in place of the global definition.
-std::unique_ptr<Module>
+LLVM_ABI std::unique_ptr<Module>
 CloneModule(const Module &M, ValueToValueMapTy &VMap,
             function_ref<bool(const GlobalValue *)> ShouldCloneDefinition);
 
@@ -127,14 +129,14 @@ struct ClonedCodeInfo {
 /// Setting true (default) is always safe (won't produce incorrect debug info)
 /// but is sometimes unnecessary, causing extra work that could be avoided by
 /// setting the parameter to false.
-BasicBlock *CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
-                            const Twine &NameSuffix = "", Function *F = nullptr,
-                            ClonedCodeInfo *CodeInfo = nullptr,
-                            bool MapAtoms = true);
+LLVM_ABI BasicBlock *
+CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
+                const Twine &NameSuffix = "", Function *F = nullptr,
+                ClonedCodeInfo *CodeInfo = nullptr, bool MapAtoms = true);
 
 /// Mark a cloned instruction as a new instance so that its source loc can
 /// be updated when remapped.
-void mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap);
+LLVM_ABI void mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap);
 
 /// Return a copy of the specified function and add it to that
 /// function's module.  Also, any references specified in the VMap are changed
@@ -147,8 +149,8 @@ void mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap);
 ///
 /// \pre VMap contains no non-identity GlobalValue mappings.
 ///
-Function *CloneFunction(Function *F, ValueToValueMapTy &VMap,
-                        ClonedCodeInfo *CodeInfo = nullptr);
+LLVM_ABI Function *CloneFunction(Function *F, ValueToValueMapTy &VMap,
+                                 ClonedCodeInfo *CodeInfo = nullptr);
 
 enum class CloneFunctionChangeType {
   LocalChangesOnly,
@@ -181,21 +183,22 @@ enum class CloneFunctionChangeType {
 /// FIXME: Consider simplifying this function by splitting out \a
 /// CloneFunctionMetadataInto() and expecting / updating callers to call it
 /// first when / how it's needed.
-void CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
-                       ValueToValueMapTy &VMap, CloneFunctionChangeType Changes,
-                       SmallVectorImpl<ReturnInst *> &Returns,
-                       const char *NameSuffix = "",
-                       ClonedCodeInfo *CodeInfo = nullptr,
-                       ValueMapTypeRemapper *TypeMapper = nullptr,
-                       ValueMaterializer *Materializer = nullptr);
+LLVM_ABI void CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
+                                ValueToValueMapTy &VMap,
+                                CloneFunctionChangeType Changes,
+                                SmallVectorImpl<ReturnInst *> &Returns,
+                                const char *NameSuffix = "",
+                                ClonedCodeInfo *CodeInfo = nullptr,
+                                ValueMapTypeRemapper *TypeMapper = nullptr,
+                                ValueMaterializer *Materializer = nullptr);
 
 /// Clone OldFunc's attributes into NewFunc, transforming values based on the
 /// mappings in VMap.
-void CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc,
-                                 ValueToValueMapTy &VMap,
-                                 bool ModuleLevelChanges,
-                                 ValueMapTypeRemapper *TypeMapper = nullptr,
-                                 ValueMaterializer *Materializer = nullptr);
+LLVM_ABI void
+CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc,
+                            ValueToValueMapTy &VMap, bool ModuleLevelChanges,
+                            ValueMapTypeRemapper *TypeMapper = nullptr,
+                            ValueMaterializer *Materializer = nullptr);
 
 /// Clone OldFunc's metadata into NewFunc.
 ///
@@ -205,28 +208,27 @@ void CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc,
 ///
 /// NOTE: This function doesn't clone !llvm.dbg.cu when cloning into a different
 /// module. Use CloneFunctionInto for that behavior.
-void CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc,
-                               ValueToValueMapTy &VMap, RemapFlags RemapFlag,
-                               ValueMapTypeRemapper *TypeMapper = nullptr,
-                               ValueMaterializer *Materializer = nullptr,
-                               const MetadataPredicate *IdentityMD = nullptr);
+LLVM_ABI void
+CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc,
+                          ValueToValueMapTy &VMap, RemapFlags RemapFlag,
+                          ValueMapTypeRemapper *TypeMapper = nullptr,
+                          ValueMaterializer *Materializer = nullptr,
+                          const MetadataPredicate *IdentityMD = nullptr);
 
 /// Clone OldFunc's body into NewFunc.
-void CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc,
-                           ValueToValueMapTy &VMap, RemapFlags RemapFlag,
-                           SmallVectorImpl<ReturnInst *> &Returns,
-                           const char *NameSuffix = "",
-                           ClonedCodeInfo *CodeInfo = nullptr,
-                           ValueMapTypeRemapper *TypeMapper = nullptr,
-                           ValueMaterializer *Materializer = nullptr,
-                           const MetadataPredicate *IdentityMD = nullptr);
-
-void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
-                               const Instruction *StartingInst,
-                               ValueToValueMapTy &VMap, bool ModuleLevelChanges,
-                               SmallVectorImpl<ReturnInst *> &Returns,
-                               const char *NameSuffix = "",
-                               ClonedCodeInfo *CodeInfo = nullptr);
+LLVM_ABI void CloneFunctionBodyInto(
+    Function &NewFunc, const Function &OldFunc, ValueToValueMapTy &VMap,
+    RemapFlags RemapFlag, SmallVectorImpl<ReturnInst *> &Returns,
+    const char *NameSuffix = "", ClonedCodeInfo *CodeInfo = nullptr,
+    ValueMapTypeRemapper *TypeMapper = nullptr,
+    ValueMaterializer *Materializer = nullptr,
+    const MetadataPredicate *IdentityMD = nullptr);
+
+LLVM_ABI void CloneAndPruneIntoFromInst(
+    Function *NewFunc, const Function *OldFunc, const Instruction *StartingInst,
+    ValueToValueMapTy &VMap, bool ModuleLevelChanges,
+    SmallVectorImpl<ReturnInst *> &Returns, const char *NameSuffix = "",
+    ClonedCodeInfo *CodeInfo = nullptr);
 
 /// This works exactly like CloneFunctionInto,
 /// except that it does some simple constant prop and DCE on the fly.  The
@@ -239,11 +241,10 @@ void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
 /// If ModuleLevelChanges is false, VMap contains no non-identity GlobalValue
 /// mappings.
 ///
-void CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
-                               ValueToValueMapTy &VMap, bool ModuleLevelChanges,
-                               SmallVectorImpl<ReturnInst*> &Returns,
-                               const char *NameSuffix = "",
-                               ClonedCodeInfo *CodeInfo = nullptr);
+LLVM_ABI void CloneAndPruneFunctionInto(
+    Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap,
+    bool ModuleLevelChanges, SmallVectorImpl<ReturnInst *> &Returns,
+    const char *NameSuffix = "", ClonedCodeInfo *CodeInfo = nullptr);
 
 /// This class captures the data input to the InlineFunction call, and records
 /// the auxiliary results produced by it.
@@ -311,23 +312,23 @@ class InlineFunctionInfo {
 ///
 /// The callee's function attributes are merged into the callers' if
 /// MergeAttributes is set to true.
-InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
-                            bool MergeAttributes = false,
-                            AAResults *CalleeAAR = nullptr,
-                            bool InsertLifetime = true,
-                            Function *ForwardVarArgsTo = nullptr,
-                            OptimizationRemarkEmitter *ORE = nullptr);
+LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
+                                     bool MergeAttributes = false,
+                                     AAResults *CalleeAAR = nullptr,
+                                     bool InsertLifetime = true,
+                                     Function *ForwardVarArgsTo = nullptr,
+                                     OptimizationRemarkEmitter *ORE = nullptr);
 
 /// Same as above, but it will update the contextual profile. If the contextual
 /// profile is invalid (i.e. not loaded because it is not present), it defaults
 /// to the behavior of the non-contextual profile updating variant above. This
 /// makes it easy to drop-in replace uses of the non-contextual overload.
-InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
-                            PGOContextualProfile &CtxProf,
-                            bool MergeAttributes = false,
-                            AAResults *CalleeAAR = nullptr,
-                            bool InsertLifetime = true,
-                            Function *ForwardVarArgsTo = nullptr);
+LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
+                                     PGOContextualProfile &CtxProf,
+                                     bool MergeAttributes = false,
+                                     AAResults *CalleeAAR = nullptr,
+                                     bool InsertLifetime = true,
+                                     Function *ForwardVarArgsTo = nullptr);
 
 /// Clones a loop \p OrigLoop.  Returns the loop and the blocks in \p
 /// Blocks.
@@ -335,15 +336,15 @@ InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
 /// Updates LoopInfo and DominatorTree assuming the loop is dominated by block
 /// \p LoopDomBB.  Insert the new blocks before block specified in \p Before.
 /// Note: Only innermost loops are supported.
-Loop *cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
-                             Loop *OrigLoop, ValueToValueMapTy &VMap,
-                             const Twine &NameSuffix, LoopInfo *LI,
-                             DominatorTree *DT,
-                             SmallVectorImpl<BasicBlock *> &Blocks);
+LLVM_ABI Loop *cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
+                                      Loop *OrigLoop, ValueToValueMapTy &VMap,
+                                      const Twine &NameSuffix, LoopInfo *LI,
+                                      DominatorTree *DT,
+                                      SmallVectorImpl<BasicBlock *> &Blocks);
 
 /// Remaps instructions in \p Blocks using the mapping in \p VMap.
-void remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,
-                               ValueToValueMapTy &VMap);
+LLVM_ABI void remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,
+                                        ValueToValueMapTy &VMap);
 
 /// Split edge between BB and PredBB and duplicate all non-Phi instructions
 /// from BB between its beginning and the StopAt instruction into the split
@@ -351,62 +352,62 @@ void remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,
 /// we replace them with the uses of corresponding Phi inputs. ValueMapping
 /// is used to map the original instructions from BB to their newly-created
 /// copies. Returns the split block.
-BasicBlock *DuplicateInstructionsInSplitBetween(BasicBlock *BB,
-                                                BasicBlock *PredBB,
-                                                Instruction *StopAt,
-                                                ValueToValueMapTy &ValueMapping,
-                                                DomTreeUpdater &DTU);
+LLVM_ABI BasicBlock *DuplicateInstructionsInSplitBetween(
+    BasicBlock *BB, BasicBlock *PredBB, Instruction *StopAt,
+    ValueToValueMapTy &ValueMapping, DomTreeUpdater &DTU);
 
 /// Updates profile information by adjusting the entry count by adding
 /// EntryDelta then scaling callsite information by the new count divided by the
 /// old count. VMap is used during inlinng to also update the new clone
-void updateProfileCallee(
+LLVM_ABI void updateProfileCallee(
     Function *Callee, int64_t EntryDelta,
     const ValueMap<const Value *, WeakTrackingVH> *VMap = nullptr);
 
 /// Find the 'llvm.experimental.noalias.scope.decl' intrinsics in the specified
 /// basic blocks and extract their scope. These are candidates for duplication
 /// when cloning.
-void identifyNoAliasScopesToClone(
-    ArrayRef<BasicBlock *> BBs, SmallVectorImpl<MDNode *> &NoAliasDeclScopes);
+LLVM_ABI void
+identifyNoAliasScopesToClone(ArrayRef<BasicBlock *> BBs,
+                             SmallVectorImpl<MDNode *> &NoAliasDeclScopes);
 
 /// Find the 'llvm.experimental.noalias.scope.decl' intrinsics in the specified
 /// instruction range and extract their scope. These are candidates for
 /// duplication when cloning.
-void identifyNoAliasScopesToClone(
-    BasicBlock::iterator Start, BasicBlock::iterator End,
-    SmallVectorImpl<MDNode *> &NoAliasDeclScopes);
+LLVM_ABI void
+identifyNoAliasScopesToClone(BasicBlock::iterator Start,
+                             BasicBlock::iterator End,
+                             SmallVectorImpl<MDNode *> &NoAliasDeclScopes);
 
 /// Duplicate the specified list of noalias decl scopes.
 /// The 'Ext' string is added as an extension to the name.
 /// Afterwards, the ClonedScopes contains the mapping of the original scope
 /// MDNode onto the cloned scope.
 /// Be aware that the cloned scopes are still part of the original scope domain.
-void cloneNoAliasScopes(
-    ArrayRef<MDNode *> NoAliasDeclScopes,
-    DenseMap<MDNode *, MDNode *> &ClonedScopes,
-    StringRef Ext, LLVMContext &Context);
+LLVM_ABI void cloneNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
+                                 DenseMap<MDNode *, MDNode *> &ClonedScopes,
+                                 StringRef Ext, LLVMContext &Context);
 
 /// Adapt the metadata for the specified instruction according to the
 /// provided mapping. This is normally used after cloning an instruction, when
 /// some noalias scopes needed to be cloned.
-void adaptNoAliasScopes(
-    llvm::Instruction *I, const DenseMap<MDNode *, MDNode *> &ClonedScopes,
-    LLVMContext &Context);
+LLVM_ABI void
+adaptNoAliasScopes(llvm::Instruction *I,
+                   const DenseMap<MDNode *, MDNode *> &ClonedScopes,
+                   LLVMContext &Context);
 
 /// Clone the specified noalias decl scopes. Then adapt all instructions in the
 /// NewBlocks basicblocks to the cloned versions.
 /// 'Ext' will be added to the duplicate scope names.
-void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
-                                ArrayRef<BasicBlock *> NewBlocks,
-                                LLVMContext &Context, StringRef Ext);
+LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
+                                         ArrayRef<BasicBlock *> NewBlocks,
+                                         LLVMContext &Context, StringRef Ext);
 
 /// Clone the specified noalias decl scopes. Then adapt all instructions in the
 /// [IStart, IEnd] (IEnd included !) range to the cloned versions. 'Ext' will be
 /// added to the duplicate scope names.
-void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
-                                Instruction *IStart, Instruction *IEnd,
-                                LLVMContext &Context, StringRef Ext);
+LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
+                                         Instruction *IStart, Instruction *IEnd,
+                                         LLVMContext &Context, StringRef Ext);
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_UTILS_CLONING_H
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
index 08c2fe655ac7..407eb50d2c7a 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/Support/Compiler.h"
 #include <limits>
 
 namespace llvm {
@@ -57,7 +58,7 @@ class CodeExtractorAnalysisCache {
   void findSideEffectInfoForBlock(BasicBlock &BB);
 
 public:
-  CodeExtractorAnalysisCache(Function &F);
+  LLVM_ABI CodeExtractorAnalysisCache(Function &F);
 
   /// Get the allocas in the function at the time the analysis was created.
   /// Note that some of these allocas may no longer be present in the function,
@@ -66,7 +67,8 @@ class CodeExtractorAnalysisCache {
 
   /// Check whether \p BB contains an instruction thought to load from, store
   /// to, or otherwise clobber the alloca \p Addr.
-  bool doesBlockContainClobberOfAddr(BasicBlock &BB, AllocaInst *Addr) const;
+  LLVM_ABI bool doesBlockContainClobberOfAddr(BasicBlock &BB,
+                                              AllocaInst *Addr) const;
 };
 
   /// Utility class for extracting code into a new function.
@@ -148,6 +150,7 @@ class CodeExtractorAnalysisCache {
     /// If ArgsInZeroAddressSpace param is set to true, then the aggregate
     /// param pointer of the outlined function is declared in zero address
     /// space.
+    LLVM_ABI
     CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT = nullptr,
                   bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr,
                   BranchProbabilityInfo *BPI = nullptr,
@@ -160,7 +163,8 @@ class CodeExtractorAnalysisCache {
     ///
     /// Returns zero when called on a CodeExtractor instance where isEligible
     /// returns false.
-    Function *extractCodeRegion(const CodeExtractorAnalysisCache &CEAC);
+    LLVM_ABI Function *
+    extractCodeRegion(const CodeExtractorAnalysisCache &CEAC);
 
     /// Perform the extraction, returning the new function and providing an
     /// interface to see what was categorized as inputs and outputs.
@@ -173,15 +177,15 @@ class CodeExtractorAnalysisCache {
     /// newly outlined function.
     /// \returns zero when called on a CodeExtractor instance where isEligible
     /// returns false.
-    Function *extractCodeRegion(const CodeExtractorAnalysisCache &CEAC,
-                                ValueSet &Inputs, ValueSet &Outputs);
+    LLVM_ABI Function *extractCodeRegion(const CodeExtractorAnalysisCache &CEAC,
+                                         ValueSet &Inputs, ValueSet &Outputs);
 
     /// Verify that assumption cache isn't stale after a region is extracted.
     /// Returns true when verifier finds errors. AssumptionCache is passed as
     /// parameter to make this function stateless.
-    static bool verifyAssumptionCache(const Function &OldFunc,
-                                      const Function &NewFunc,
-                                      AssumptionCache *AC);
+    LLVM_ABI static bool verifyAssumptionCache(const Function &OldFunc,
+                                               const Function &NewFunc,
+                                               AssumptionCache *AC);
 
     /// Test whether this code extractor is eligible.
     ///
@@ -190,7 +194,7 @@ class CodeExtractorAnalysisCache {
     ///
     /// Checks that varargs handling (with vastart and vaend) is only done in
     /// the outlined blocks.
-    bool isEligible() const;
+    LLVM_ABI bool isEligible() const;
 
     /// Compute the set of input values and output values for the code.
     ///
@@ -200,15 +204,15 @@ class CodeExtractorAnalysisCache {
     /// a code sequence, that sequence is modified, including changing these
     /// sets, before extraction occurs. These modifications won't have any
     /// significant impact on the cost however.
-    void findInputsOutputs(ValueSet &Inputs, ValueSet &Outputs,
-                           const ValueSet &Allocas,
-                           bool CollectGlobalInputs = false) const;
+    LLVM_ABI void findInputsOutputs(ValueSet &Inputs, ValueSet &Outputs,
+                                    const ValueSet &Allocas,
+                                    bool CollectGlobalInputs = false) const;
 
     /// Check if life time marker nodes can be hoisted/sunk into the outline
     /// region.
     ///
     /// Returns true if it is safe to do the code motion.
-    bool
+    LLVM_ABI bool
     isLegalToShrinkwrapLifetimeMarkers(const CodeExtractorAnalysisCache &CEAC,
                                        Instruction *AllocaAddr) const;
 
@@ -220,9 +224,9 @@ class CodeExtractorAnalysisCache {
     /// are used by the lifetime markers are also candidates for shrink-
     /// wrapping. The instructions that need to be sunk are collected in
     /// 'Allocas'.
-    void findAllocas(const CodeExtractorAnalysisCache &CEAC,
-                     ValueSet &SinkCands, ValueSet &HoistCands,
-                     BasicBlock *&ExitBlock) const;
+    LLVM_ABI void findAllocas(const CodeExtractorAnalysisCache &CEAC,
+                              ValueSet &SinkCands, ValueSet &HoistCands,
+                              BasicBlock *&ExitBlock) const;
 
     /// Find or create a block within the outline region for placing hoisted
     /// code.
@@ -232,11 +236,12 @@ class CodeExtractorAnalysisCache {
     /// inside the region that is the predecessor of CommonExitBlock, that block
     /// will be returned. Otherwise CommonExitBlock will be split and the
     /// original block will be added to the outline region.
-    BasicBlock *findOrCreateBlockForHoisting(BasicBlock *CommonExitBlock);
+    LLVM_ABI BasicBlock *
+    findOrCreateBlockForHoisting(BasicBlock *CommonExitBlock);
 
     /// Exclude a value from aggregate argument passing when extracting a code
     /// region, passing it instead as a scalar.
-    void excludeArgFromAggregate(Value *Arg);
+    LLVM_ABI void excludeArgFromAggregate(Value *Arg);
 
   private:
     struct LifetimeMarkerInfo {
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeLayout.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeLayout.h
index c737643ee101..c4b12a5e0e70 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeLayout.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeLayout.h
@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_UTILS_CODELAYOUT_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Compiler.h"
 
 #include <utility>
 #include <vector>
@@ -40,20 +41,21 @@ struct EdgeCount {
 /// \p EdgeCounts: The execution counts of every edge (jump) in the profile. The
 ///    map also defines the edges in CFG and should include 0-count edges.
 /// \returns The best block order found.
-std::vector<uint64_t> computeExtTspLayout(ArrayRef<uint64_t> NodeSizes,
-                                          ArrayRef<uint64_t> NodeCounts,
-                                          ArrayRef<EdgeCount> EdgeCounts);
+LLVM_ABI std::vector<uint64_t>
+computeExtTspLayout(ArrayRef<uint64_t> NodeSizes, ArrayRef<uint64_t> NodeCounts,
+                    ArrayRef<EdgeCount> EdgeCounts);
 
 /// Estimate the "quality" of a given node order in CFG. The higher the score,
 /// the better the order is. The score is designed to reflect the locality of
 /// the given order, which is anti-correlated with the number of I-cache misses
 /// in a typical execution of the function.
-double calcExtTspScore(ArrayRef<uint64_t> Order, ArrayRef<uint64_t> NodeSizes,
-                       ArrayRef<EdgeCount> EdgeCounts);
+LLVM_ABI double calcExtTspScore(ArrayRef<uint64_t> Order,
+                                ArrayRef<uint64_t> NodeSizes,
+                                ArrayRef<EdgeCount> EdgeCounts);
 
 /// Estimate the "quality" of the current node order in CFG.
-double calcExtTspScore(ArrayRef<uint64_t> NodeSizes,
-                       ArrayRef<EdgeCount> EdgeCounts);
+LLVM_ABI double calcExtTspScore(ArrayRef<uint64_t> NodeSizes,
+                                ArrayRef<EdgeCount> EdgeCounts);
 
 /// Algorithm-specific params for Cache-Directed Sort. The values are tuned for
 /// the best performance of large-scale front-end bound binaries.
@@ -79,12 +81,12 @@ struct CDSortConfig {
 ///    map also defines the edges in CFG and should include 0-count edges.
 /// \p CallOffsets: The offsets of the calls from their source nodes.
 /// \returns The best function order found.
-std::vector<uint64_t> computeCacheDirectedLayout(
+LLVM_ABI std::vector<uint64_t> computeCacheDirectedLayout(
     ArrayRef<uint64_t> FuncSizes, ArrayRef<uint64_t> FuncCounts,
     ArrayRef<EdgeCount> CallCounts, ArrayRef<uint64_t> CallOffsets);
 
 /// Apply a Cache-Directed Sort with a custom config.
-std::vector<uint64_t> computeCacheDirectedLayout(
+LLVM_ABI std::vector<uint64_t> computeCacheDirectedLayout(
     const CDSortConfig &Config, ArrayRef<uint64_t> FuncSizes,
     ArrayRef<uint64_t> FuncCounts, ArrayRef<EdgeCount> CallCounts,
     ArrayRef<uint64_t> CallOffsets);
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h
index 4fd147a1f7b3..877872485ab5 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h
@@ -17,6 +17,8 @@
 #ifndef LLVM_TRANSFORMS_UTILS_CODEMOVERUTILS_H
 #define LLVM_TRANSFORMS_UTILS_CODEMOVERUTILS_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 class BasicBlock;
@@ -28,56 +30,61 @@ class PostDominatorTree;
 /// Return true if \p I0 and \p I1 are control flow equivalent.
 /// Two instructions are control flow equivalent if their basic blocks are
 /// control flow equivalent.
-bool isControlFlowEquivalent(const Instruction &I0, const Instruction &I1,
-                             const DominatorTree &DT,
-                             const PostDominatorTree &PDT);
+LLVM_ABI bool isControlFlowEquivalent(const Instruction &I0,
+                                      const Instruction &I1,
+                                      const DominatorTree &DT,
+                                      const PostDominatorTree &PDT);
 
 /// Return true if \p BB0 and \p BB1 are control flow equivalent.
 /// Two basic blocks are control flow equivalent if when one executes, the other
 /// is guaranteed to execute.
-bool isControlFlowEquivalent(const BasicBlock &BB0, const BasicBlock &BB1,
-                             const DominatorTree &DT,
-                             const PostDominatorTree &PDT);
+LLVM_ABI bool isControlFlowEquivalent(const BasicBlock &BB0,
+                                      const BasicBlock &BB1,
+                                      const DominatorTree &DT,
+                                      const PostDominatorTree &PDT);
 
 /// Return true if \p I can be safely moved before \p InsertPoint.
-bool isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
-                        DominatorTree &DT,
-                        const PostDominatorTree *PDT = nullptr,
-                        DependenceInfo *DI = nullptr,
-                        bool CheckForEntireBlock = false);
+LLVM_ABI bool isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint,
+                                 DominatorTree &DT,
+                                 const PostDominatorTree *PDT = nullptr,
+                                 DependenceInfo *DI = nullptr,
+                                 bool CheckForEntireBlock = false);
 
 /// Return true if all instructions (except the terminator) in \p BB can be
 /// safely moved before \p InsertPoint.
-bool isSafeToMoveBefore(BasicBlock &BB, Instruction &InsertPoint,
-                        DominatorTree &DT,
-                        const PostDominatorTree *PDT = nullptr,
-                        DependenceInfo *DI = nullptr);
+LLVM_ABI bool isSafeToMoveBefore(BasicBlock &BB, Instruction &InsertPoint,
+                                 DominatorTree &DT,
+                                 const PostDominatorTree *PDT = nullptr,
+                                 DependenceInfo *DI = nullptr);
 
 /// Move instructions, in an order-preserving manner, from \p FromBB to the
 /// beginning of \p ToBB when proven safe.
-void moveInstructionsToTheBeginning(BasicBlock &FromBB, BasicBlock &ToBB,
-                                    DominatorTree &DT,
-                                    const PostDominatorTree &PDT,
-                                    DependenceInfo &DI);
+LLVM_ABI void moveInstructionsToTheBeginning(BasicBlock &FromBB,
+                                             BasicBlock &ToBB,
+                                             DominatorTree &DT,
+                                             const PostDominatorTree &PDT,
+                                             DependenceInfo &DI);
 
 /// Move instructions, in an order-preserving manner, from \p FromBB to the end
 /// of \p ToBB when proven safe.
-void moveInstructionsToTheEnd(BasicBlock &FromBB, BasicBlock &ToBB,
-                              DominatorTree &DT, const PostDominatorTree &PDT,
-                              DependenceInfo &DI);
+LLVM_ABI void moveInstructionsToTheEnd(BasicBlock &FromBB, BasicBlock &ToBB,
+                                       DominatorTree &DT,
+                                       const PostDominatorTree &PDT,
+                                       DependenceInfo &DI);
 
 /// In case that two BBs \p ThisBlock and \p OtherBlock are control flow
 /// equivalent but they do not strictly dominate and post-dominate each
 /// other, we determine if \p ThisBlock is reached after \p OtherBlock
 /// in the control flow.
-bool nonStrictlyPostDominate(const BasicBlock *ThisBlock,
-                             const BasicBlock *OtherBlock,
-                             const DominatorTree *DT,
-                             const PostDominatorTree *PDT);
+LLVM_ABI bool nonStrictlyPostDominate(const BasicBlock *ThisBlock,
+                                      const BasicBlock *OtherBlock,
+                                      const DominatorTree *DT,
+                                      const PostDominatorTree *PDT);
 
 // Check if I0 is reached before I1 in the control flow.
-bool isReachedBefore(const Instruction *I0, const Instruction *I1,
-                     const DominatorTree *DT, const PostDominatorTree *PDT);
+LLVM_ABI bool isReachedBefore(const Instruction *I0, const Instruction *I1,
+                              const DominatorTree *DT,
+                              const PostDominatorTree *PDT);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Debugify.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Debugify.h
index 10c8c5897b94..64404efabed1 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Debugify.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Debugify.h
@@ -24,6 +24,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 
 using DebugFnMap =
     llvm::MapVector<const llvm::Function *, const llvm::DISubprogram *>;
@@ -56,14 +57,15 @@ class DIBuilder;
 /// \param ApplyToMF A call back that will add debug information to the
 ///                  MachineFunction for a Function. If nullptr, then the
 ///                  MachineFunction (if any) will not be modified.
-bool applyDebugifyMetadata(
-    Module &M, iterator_range<Module::iterator> Functions, StringRef Banner,
-    std::function<bool(DIBuilder &, Function &)> ApplyToMF);
+LLVM_ABI bool
+applyDebugifyMetadata(Module &M, iterator_range<Module::iterator> Functions,
+                      StringRef Banner,
+                      std::function<bool(DIBuilder &, Function &)> ApplyToMF);
 
 /// Strip out all of the metadata and debug info inserted by debugify. If no
 /// llvm.debugify module-level named metadata is present, this is a no-op.
 /// Returns true if any change was made.
-bool stripDebugifyMetadata(Module &M);
+LLVM_ABI bool stripDebugifyMetadata(Module &M);
 
 /// Collect original debug information before a pass.
 ///
@@ -72,10 +74,10 @@ bool stripDebugifyMetadata(Module &M);
 /// \param DebugInfoBeforePass DI metadata before a pass.
 /// \param Banner A prefix string to add to debug/error messages.
 /// \param NameOfWrappedPass A name of a pass to add to debug/error messages.
-bool collectDebugInfoMetadata(Module &M,
-                              iterator_range<Module::iterator> Functions,
-                              DebugInfoPerPass &DebugInfoBeforePass,
-                              StringRef Banner, StringRef NameOfWrappedPass);
+LLVM_ABI bool
+collectDebugInfoMetadata(Module &M, iterator_range<Module::iterator> Functions,
+                         DebugInfoPerPass &DebugInfoBeforePass,
+                         StringRef Banner, StringRef NameOfWrappedPass);
 
 /// Check original debug information after a pass.
 ///
@@ -84,21 +86,22 @@ bool collectDebugInfoMetadata(Module &M,
 /// \param DebugInfoBeforePass DI metadata before a pass.
 /// \param Banner A prefix string to add to debug/error messages.
 /// \param NameOfWrappedPass A name of a pass to add to debug/error messages.
-bool checkDebugInfoMetadata(Module &M,
-                            iterator_range<Module::iterator> Functions,
-                            DebugInfoPerPass &DebugInfoBeforePass,
-                            StringRef Banner, StringRef NameOfWrappedPass,
-                            StringRef OrigDIVerifyBugsReportFilePath);
+LLVM_ABI bool checkDebugInfoMetadata(Module &M,
+                                     iterator_range<Module::iterator> Functions,
+                                     DebugInfoPerPass &DebugInfoBeforePass,
+                                     StringRef Banner,
+                                     StringRef NameOfWrappedPass,
+                                     StringRef OrigDIVerifyBugsReportFilePath);
 } // namespace llvm
 
 /// Used to check whether we track synthetic or original debug info.
 enum class DebugifyMode { NoDebugify, SyntheticDebugInfo, OriginalDebugInfo };
 
-llvm::ModulePass *createDebugifyModulePass(
+LLVM_ABI llvm::ModulePass *createDebugifyModulePass(
     enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo,
     llvm::StringRef NameOfWrappedPass = "",
     DebugInfoPerPass *DebugInfoBeforePass = nullptr);
-llvm::FunctionPass *createDebugifyFunctionPass(
+LLVM_ABI llvm::FunctionPass *createDebugifyFunctionPass(
     enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo,
     llvm::StringRef NameOfWrappedPass = "",
     DebugInfoPerPass *DebugInfoBeforePass = nullptr);
@@ -115,7 +118,8 @@ class NewPMDebugifyPass : public llvm::PassInfoMixin<NewPMDebugifyPass> {
       : NameOfWrappedPass(NameOfWrappedPass),
         DebugInfoBeforePass(DebugInfoBeforePass), Mode(Mode) {}
 
-  llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &AM);
+  LLVM_ABI llvm::PreservedAnalyses run(llvm::Module &M,
+                                       llvm::ModuleAnalysisManager &AM);
 };
 
 /// Track how much `debugify` information (in the `synthetic` mode only)
@@ -147,14 +151,14 @@ struct DebugifyStatistics {
 /// Map pass names to a per-pass DebugifyStatistics instance.
 using DebugifyStatsMap = llvm::MapVector<llvm::StringRef, DebugifyStatistics>;
 
-llvm::ModulePass *createCheckDebugifyModulePass(
+LLVM_ABI llvm::ModulePass *createCheckDebugifyModulePass(
     bool Strip = false, llvm::StringRef NameOfWrappedPass = "",
     DebugifyStatsMap *StatsMap = nullptr,
     enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo,
     DebugInfoPerPass *DebugInfoBeforePass = nullptr,
     llvm::StringRef OrigDIVerifyBugsReportFilePath = "");
 
-llvm::FunctionPass *createCheckDebugifyFunctionPass(
+LLVM_ABI llvm::FunctionPass *createCheckDebugifyFunctionPass(
     bool Strip = false, llvm::StringRef NameOfWrappedPass = "",
     DebugifyStatsMap *StatsMap = nullptr,
     enum DebugifyMode Mode = DebugifyMode::SyntheticDebugInfo,
@@ -181,11 +185,12 @@ class NewPMCheckDebugifyPass
         StatsMap(StatsMap), DebugInfoBeforePass(DebugInfoBeforePass), Mode(Mode),
         Strip(Strip) {}
 
-  llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &AM);
+  LLVM_ABI llvm::PreservedAnalyses run(llvm::Module &M,
+                                       llvm::ModuleAnalysisManager &AM);
 };
 
 namespace llvm {
-void exportDebugifyStats(StringRef Path, const DebugifyStatsMap &Map);
+LLVM_ABI void exportDebugifyStats(StringRef Path, const DebugifyStatsMap &Map);
 
 class DebugifyEachInstrumentation {
   llvm::StringRef OrigDIVerifyBugsReportFilePath = "";
@@ -194,8 +199,8 @@ class DebugifyEachInstrumentation {
   DebugifyStatsMap *DIStatsMap = nullptr;
 
 public:
-  void registerCallbacks(PassInstrumentationCallbacks &PIC,
-                         ModuleAnalysisManager &MAM);
+  LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC,
+                                  ModuleAnalysisManager &MAM);
   // Used within DebugifyMode::SyntheticDebugInfo mode.
   void setDIStatsMap(DebugifyStatsMap &StatMap) { DIStatsMap = &StatMap; }
   const DebugifyStatsMap &getDebugifyStatsMap() const { return *DIStatsMap; }
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionComparator.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
index 19c5f7449f23..f3a83c2a04b5 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
@@ -21,6 +21,7 @@
 #include "llvm/IR/ValueMap.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <tuple>
 
@@ -97,7 +98,7 @@ class FunctionComparator {
       : FnL(F1), FnR(F2), GlobalNumbers(GN) {}
 
   /// Test whether the two functions have equivalent behaviour.
-  int compare();
+  LLVM_ABI int compare();
 
 protected:
   /// Start the comparison.
@@ -107,10 +108,11 @@ class FunctionComparator {
   }
 
   /// Compares the signature and other general attributes of the two functions.
-  int compareSignature() const;
+  LLVM_ABI int compareSignature() const;
 
   /// Test whether two basic blocks have equivalent behaviour.
-  int cmpBasicBlocks(const BasicBlock *BBL, const BasicBlock *BBR) const;
+  LLVM_ABI int cmpBasicBlocks(const BasicBlock *BBL,
+                              const BasicBlock *BBR) const;
 
   /// Constants comparison.
   /// Its analog to lexicographical comparison between hypothetical numbers
@@ -214,11 +216,11 @@ class FunctionComparator {
   /// look at their particular properties (bit-width for vectors, and
   /// address space for pointers).
   /// If these properties are equal - compare their contents.
-  int cmpConstants(const Constant *L, const Constant *R) const;
+  LLVM_ABI int cmpConstants(const Constant *L, const Constant *R) const;
 
   /// Compares two global values by number. Uses the GlobalNumbersState to
   /// identify the same gobals across function calls.
-  int cmpGlobalValues(GlobalValue *L, GlobalValue *R) const;
+  LLVM_ABI int cmpGlobalValues(GlobalValue *L, GlobalValue *R) const;
 
   /// Assign or look up previously assigned numbers for the two values, and
   /// return whether the numbers are equal. Numbers are assigned in the order
@@ -238,7 +240,7 @@ class FunctionComparator {
   ///          then left value is greater.
   ///          In another words, we compare serial numbers, for more details
   ///          see comments for sn_mapL and sn_mapR.
-  int cmpValues(const Value *L, const Value *R) const;
+  LLVM_ABI int cmpValues(const Value *L, const Value *R) const;
 
   /// Compare two Instructions for equivalence, similar to
   /// Instruction::isSameOperationAs.
@@ -269,8 +271,8 @@ class FunctionComparator {
   /// Sets \p needToCmpOperands to true if the operands of the instructions
   /// still must be compared afterwards. In this case it's already guaranteed
   /// that both instructions have the same number of operands.
-  int cmpOperations(const Instruction *L, const Instruction *R,
-                    bool &needToCmpOperands) const;
+  LLVM_ABI int cmpOperations(const Instruction *L, const Instruction *R,
+                             bool &needToCmpOperands) const;
 
   /// cmpType - compares two types,
   /// defines total ordering among the types set.
@@ -312,14 +314,15 @@ class FunctionComparator {
   /// be checked with the same way. If we get Res != 0 on some stage, return it.
   /// Otherwise return 0.
   /// 6. For all other cases put llvm_unreachable.
-  int cmpTypes(Type *TyL, Type *TyR) const;
+  LLVM_ABI int cmpTypes(Type *TyL, Type *TyR) const;
 
-  int cmpNumbers(uint64_t L, uint64_t R) const;
-  int cmpAligns(Align L, Align R) const;
-  int cmpAPInts(const APInt &L, const APInt &R) const;
-  int cmpConstantRanges(const ConstantRange &L, const ConstantRange &R) const;
-  int cmpAPFloats(const APFloat &L, const APFloat &R) const;
-  int cmpMem(StringRef L, StringRef R) const;
+  LLVM_ABI int cmpNumbers(uint64_t L, uint64_t R) const;
+  LLVM_ABI int cmpAligns(Align L, Align R) const;
+  LLVM_ABI int cmpAPInts(const APInt &L, const APInt &R) const;
+  LLVM_ABI int cmpConstantRanges(const ConstantRange &L,
+                                 const ConstantRange &R) const;
+  LLVM_ABI int cmpAPFloats(const APFloat &L, const APFloat &R) const;
+  LLVM_ABI int cmpMem(StringRef L, StringRef R) const;
 
   // The two functions undergoing comparison.
   const Function *FnL, *FnR;
@@ -341,7 +344,7 @@ class FunctionComparator {
   /// 3. Pointer operand type (using cmpType method).
   /// 4. Number of operands.
   /// 5. Compare operands, using cmpValues method.
-  int cmpGEPs(const GEPOperator *GEPL, const GEPOperator *GEPR) const;
+  LLVM_ABI int cmpGEPs(const GEPOperator *GEPL, const GEPOperator *GEPR) const;
   int cmpGEPs(const GetElementPtrInst *GEPL,
               const GetElementPtrInst *GEPR) const {
     return cmpGEPs(cast<GEPOperator>(GEPL), cast<GEPOperator>(GEPR));
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h
index 28ba20bc18cf..c4c7b403bed8 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/SetVector.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Module;
@@ -102,18 +103,19 @@ class FunctionImportGlobalProcessing {
   DenseSet<GlobalValue::GUID> SymbolsToMove;
 
 public:
+  LLVM_ABI
   FunctionImportGlobalProcessing(Module &M, const ModuleSummaryIndex &Index,
                                  SetVector<GlobalValue *> *GlobalsToImport,
                                  bool ClearDSOLocalOnDeclarations);
-  void run();
+  LLVM_ABI void run();
 };
 
 /// Perform in-place global value handling on the given Module for
 /// exported local functions renamed and promoted for ThinLTO.
-void renameModuleForThinLTO(
-    Module &M, const ModuleSummaryIndex &Index,
-    bool ClearDSOLocalOnDeclarations,
-    SetVector<GlobalValue *> *GlobalsToImport = nullptr);
+LLVM_ABI void
+renameModuleForThinLTO(Module &M, const ModuleSummaryIndex &Index,
+                       bool ClearDSOLocalOnDeclarations,
+                       SetVector<GlobalValue *> *GlobalsToImport = nullptr);
 
 } // End llvm namespace
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Instrumentation.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Instrumentation.h
index e57083b2dc1b..962d9e734a40 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Instrumentation.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Instrumentation.h
@@ -19,6 +19,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstdint>
 #include <limits>
@@ -33,35 +34,35 @@ class CallBase;
 class Module;
 
 /// Check if module has flag attached, if not add the flag.
-bool checkIfAlreadyInstrumented(Module &M, StringRef Flag);
+LLVM_ABI bool checkIfAlreadyInstrumented(Module &M, StringRef Flag);
 
 /// Instrumentation passes often insert conditional checks into entry blocks.
 /// Call this function before splitting the entry block to move instructions
 /// that must remain in the entry block up before the split point. Static
 /// allocas and llvm.localescape calls, for example, must remain in the entry
 /// block.
-BasicBlock::iterator PrepareToSplitEntryBlock(BasicBlock &BB,
-                                              BasicBlock::iterator IP);
+LLVM_ABI BasicBlock::iterator PrepareToSplitEntryBlock(BasicBlock &BB,
+                                                       BasicBlock::iterator IP);
 
 // Create a constant for Str so that we can pass it to the run-time lib.
-GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str,
-                                             bool AllowMerging,
-                                             Twine NamePrefix = "");
+LLVM_ABI GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str,
+                                                      bool AllowMerging,
+                                                      Twine NamePrefix = "");
 
 // Returns F.getComdat() if it exists.
 // Otherwise creates a new comdat, sets F's comdat, and returns it.
 // Returns nullptr on failure.
-Comdat *getOrCreateFunctionComdat(Function &F, Triple &T);
+LLVM_ABI Comdat *getOrCreateFunctionComdat(Function &F, Triple &T);
 
 // Place global in a large section for x86-64 ELF binaries to mitigate
 // relocation overflow pressure. This can be be used for metadata globals that
 // aren't directly accessed by code, which has no performance impact.
-void setGlobalVariableLargeSection(const Triple &TargetTriple,
-                                   GlobalVariable &GV);
+LLVM_ABI void setGlobalVariableLargeSection(const Triple &TargetTriple,
+                                            GlobalVariable &GV);
 
 // Insert GCOV profiling instrumentation
 struct GCOVOptions {
-  static GCOVOptions getDefault();
+  LLVM_ABI static GCOVOptions getDefault();
 
   // Specify whether to emit .gcno files.
   bool EmitNotes;
@@ -106,9 +107,10 @@ namespace pgo {
 // If \p AttachProfToDirectCall is true, a prof metadata is attached to the
 // new direct call to contain \p Count.
 // Returns the promoted direct call instruction.
-CallBase &promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count,
-                              uint64_t TotalCount, bool AttachProfToDirectCall,
-                              OptimizationRemarkEmitter *ORE);
+LLVM_ABI CallBase &promoteIndirectCall(CallBase &CB, Function *F,
+                                       uint64_t Count, uint64_t TotalCount,
+                                       bool AttachProfToDirectCall,
+                                       OptimizationRemarkEmitter *ORE);
 } // namespace pgo
 
 /// Options for the frontend instrumentation based profiling pass.
@@ -135,7 +137,7 @@ struct InstrProfOptions {
 };
 
 // Create the variable for profile sampling.
-void createProfileSamplingVar(Module &M);
+LLVM_ABI void createProfileSamplingVar(Module &M);
 
 // Options for sanitizer coverage instrumentation.
 struct SanitizerCoverageOptions {
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/IntegerDivision.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/IntegerDivision.h
index 35cae9aa2269..020db95bd1e6 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/IntegerDivision.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/IntegerDivision.h
@@ -16,6 +16,8 @@
 #ifndef LLVM_TRANSFORMS_UTILS_INTEGERDIVISION_H
 #define LLVM_TRANSFORMS_UTILS_INTEGERDIVISION_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
   class BinaryOperator;
 }
@@ -29,43 +31,43 @@ namespace llvm {
   /// 32bit and 64bit scalar division.
   ///
   /// Replace Rem with generated code.
-  bool expandRemainder(BinaryOperator *Rem);
+LLVM_ABI bool expandRemainder(BinaryOperator *Rem);
 
-  /// Generate code to divide two integers, replacing Div with the generated
-  /// code. This currently generates code similarly to compiler-rt's
-  /// implementations, but future work includes generating more specialized code
-  /// when more information about the operands are known. Implements both
-  /// 32bit and 64bit scalar division.
-  ///
-  /// Replace Div with generated code.
-  bool expandDivision(BinaryOperator* Div);
+/// Generate code to divide two integers, replacing Div with the generated
+/// code. This currently generates code similarly to compiler-rt's
+/// implementations, but future work includes generating more specialized code
+/// when more information about the operands are known. Implements both
+/// 32bit and 64bit scalar division.
+///
+/// Replace Div with generated code.
+LLVM_ABI bool expandDivision(BinaryOperator *Div);
 
-  /// Generate code to calculate the remainder of two integers, replacing Rem
-  /// with the generated code. Uses ExpandReminder with a 32bit Rem which
-  /// makes it useful for targets with little or no support for less than
-  /// 32 bit arithmetic.
-  ///
-  /// Replace Rem with generated code.
-  bool expandRemainderUpTo32Bits(BinaryOperator *Rem);
+/// Generate code to calculate the remainder of two integers, replacing Rem
+/// with the generated code. Uses ExpandReminder with a 32bit Rem which
+/// makes it useful for targets with little or no support for less than
+/// 32 bit arithmetic.
+///
+/// Replace Rem with generated code.
+LLVM_ABI bool expandRemainderUpTo32Bits(BinaryOperator *Rem);
 
-  /// Generate code to calculate the remainder of two integers, replacing Rem
-  /// with the generated code. Uses ExpandReminder with a 64bit Rem.
-  ///
-  /// Replace Rem with generated code.
-  bool expandRemainderUpTo64Bits(BinaryOperator *Rem);
+/// Generate code to calculate the remainder of two integers, replacing Rem
+/// with the generated code. Uses ExpandReminder with a 64bit Rem.
+///
+/// Replace Rem with generated code.
+LLVM_ABI bool expandRemainderUpTo64Bits(BinaryOperator *Rem);
 
-  /// Generate code to divide two integers, replacing Div with the generated
-  /// code. Uses ExpandDivision with a 32bit Div which makes it useful for
-  /// targets with little or no support for less than 32 bit arithmetic.
-  ///
-  /// Replace Rem with generated code.
-  bool expandDivisionUpTo32Bits(BinaryOperator *Div);
+/// Generate code to divide two integers, replacing Div with the generated
+/// code. Uses ExpandDivision with a 32bit Div which makes it useful for
+/// targets with little or no support for less than 32 bit arithmetic.
+///
+/// Replace Rem with generated code.
+LLVM_ABI bool expandDivisionUpTo32Bits(BinaryOperator *Div);
 
-  /// Generate code to divide two integers, replacing Div with the generated
-  /// code. Uses ExpandDivision with a 64bit Div.
-  ///
-  /// Replace Rem with generated code.
-  bool expandDivisionUpTo64Bits(BinaryOperator *Div);
+/// Generate code to divide two integers, replacing Div with the generated
+/// code. Uses ExpandDivision with a 64bit Div.
+///
+/// Replace Rem with generated code.
+LLVM_ABI bool expandDivisionUpTo64Bits(BinaryOperator *Div);
 
 } // End llvm namespace
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LCSSA.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LCSSA.h
index b01c8022a65b..2ea4b193f33c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LCSSA.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LCSSA.h
@@ -30,13 +30,14 @@
 #define LLVM_TRANSFORMS_UTILS_LCSSA_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 /// Converts loops into loop-closed SSA form.
 class LCSSAPass : public PassInfoMixin<LCSSAPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Local.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Local.h
index 616255749640..ad7a83e4471c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/Local.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <cstdint>
@@ -60,9 +61,10 @@ class TargetTransformInfo;
 /// Also calls RecursivelyDeleteTriviallyDeadInstructions() on any branch/switch
 /// conditions and indirectbr addresses this might make dead if
 /// DeleteDeadConditions is true.
-bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions = false,
-                            const TargetLibraryInfo *TLI = nullptr,
-                            DomTreeUpdater *DTU = nullptr);
+LLVM_ABI bool ConstantFoldTerminator(BasicBlock *BB,
+                                     bool DeleteDeadConditions = false,
+                                     const TargetLibraryInfo *TLI = nullptr,
+                                     DomTreeUpdater *DTU = nullptr);
 
 //===----------------------------------------------------------------------===//
 //  Local dead code elimination.
@@ -71,27 +73,29 @@ bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions = false,
 /// Return true if the result produced by the instruction is not used, and the
 /// instruction will return. Certain side-effecting instructions are also
 /// considered dead if there are no uses of the instruction.
-bool isInstructionTriviallyDead(Instruction *I,
-                                const TargetLibraryInfo *TLI = nullptr);
+LLVM_ABI bool
+isInstructionTriviallyDead(Instruction *I,
+                           const TargetLibraryInfo *TLI = nullptr);
 
 /// Return true if the result produced by the instruction would have no side
 /// effects if it was not used. This is equivalent to checking whether
 /// isInstructionTriviallyDead would be true if the use count was 0.
-bool wouldInstructionBeTriviallyDead(const Instruction *I,
-                                     const TargetLibraryInfo *TLI = nullptr);
+LLVM_ABI bool
+wouldInstructionBeTriviallyDead(const Instruction *I,
+                                const TargetLibraryInfo *TLI = nullptr);
 
 /// Return true if the result produced by the instruction has no side effects on
 /// any paths other than where it is used. This is less conservative than
 /// wouldInstructionBeTriviallyDead which is based on the assumption
 /// that the use count will be 0. An example usage of this API is for
 /// identifying instructions that can be sunk down to use(s).
-bool wouldInstructionBeTriviallyDeadOnUnusedPaths(
+LLVM_ABI bool wouldInstructionBeTriviallyDeadOnUnusedPaths(
     Instruction *I, const TargetLibraryInfo *TLI = nullptr);
 
 /// If the specified value is a trivially dead instruction, delete it.
 /// If that makes any of its operands trivially dead, delete them too,
 /// recursively. Return true if any instructions were deleted.
-bool RecursivelyDeleteTriviallyDeadInstructions(
+LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructions(
     Value *V, const TargetLibraryInfo *TLI = nullptr,
     MemorySSAUpdater *MSSAU = nullptr,
     std::function<void(Value *)> AboutToDeleteCallback =
@@ -105,7 +109,7 @@ bool RecursivelyDeleteTriviallyDeadInstructions(
 ///
 /// `DeadInsts` will be used as scratch storage for this routine and will be
 /// empty afterward.
-void RecursivelyDeleteTriviallyDeadInstructions(
+LLVM_ABI void RecursivelyDeleteTriviallyDeadInstructions(
     SmallVectorImpl<WeakTrackingVH> &DeadInsts,
     const TargetLibraryInfo *TLI = nullptr, MemorySSAUpdater *MSSAU = nullptr,
     std::function<void(Value *)> AboutToDeleteCallback =
@@ -115,7 +119,7 @@ void RecursivelyDeleteTriviallyDeadInstructions(
 /// instructions that are not trivially dead. These will be ignored.
 /// Returns true if any changes were made, i.e. any instructions trivially dead
 /// were found and deleted.
-bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(
+LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(
     SmallVectorImpl<WeakTrackingVH> &DeadInsts,
     const TargetLibraryInfo *TLI = nullptr, MemorySSAUpdater *MSSAU = nullptr,
     std::function<void(Value *)> AboutToDeleteCallback =
@@ -126,23 +130,25 @@ bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(
 /// by a trivially dead instruction, delete it. If that makes any of its
 /// operands trivially dead, delete them too, recursively. Return true if a
 /// change was made.
-bool RecursivelyDeleteDeadPHINode(PHINode *PN,
-                                  const TargetLibraryInfo *TLI = nullptr,
-                                  MemorySSAUpdater *MSSAU = nullptr);
+LLVM_ABI bool
+RecursivelyDeleteDeadPHINode(PHINode *PN,
+                             const TargetLibraryInfo *TLI = nullptr,
+                             MemorySSAUpdater *MSSAU = nullptr);
 
 /// Scan the specified basic block and try to simplify any instructions in it
 /// and recursively delete dead instructions.
 ///
 /// This returns true if it changed the code, note that it can delete
 /// instructions in other blocks as well in this block.
-bool SimplifyInstructionsInBlock(BasicBlock *BB,
-                                 const TargetLibraryInfo *TLI = nullptr);
+LLVM_ABI bool
+SimplifyInstructionsInBlock(BasicBlock *BB,
+                            const TargetLibraryInfo *TLI = nullptr);
 
 /// Replace all the uses of an SSA value in @llvm.dbg intrinsics with
 /// undef. This is useful for signaling that a variable, e.g. has been
 /// found dead and hence it's unavailable at a given program point.
 /// Returns true if the dbg values have been changed.
-bool replaceDbgUsesWithUndef(Instruction *I);
+LLVM_ABI bool replaceDbgUsesWithUndef(Instruction *I);
 
 //===----------------------------------------------------------------------===//
 //  Control Flow Graph Restructuring.
@@ -151,29 +157,31 @@ bool replaceDbgUsesWithUndef(Instruction *I);
 /// BB is a block with one predecessor and its predecessor is known to have one
 /// successor (BB!). Eliminate the edge between them, moving the instructions in
 /// the predecessor into BB. This deletes the predecessor block.
-void MergeBasicBlockIntoOnlyPred(BasicBlock *BB, DomTreeUpdater *DTU = nullptr);
+LLVM_ABI void MergeBasicBlockIntoOnlyPred(BasicBlock *BB,
+                                          DomTreeUpdater *DTU = nullptr);
 
 /// BB is known to contain an unconditional branch, and contains no instructions
 /// other than PHI nodes, potential debug intrinsics and the branch. If
 /// possible, eliminate BB by rewriting all the predecessors to branch to the
 /// successor block and return true. If we can't transform, return false.
-bool TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
-                                             DomTreeUpdater *DTU = nullptr);
+LLVM_ABI bool
+TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
+                                        DomTreeUpdater *DTU = nullptr);
 
 /// Check for and eliminate duplicate PHI nodes in this block. This doesn't try
 /// to be clever about PHI nodes which differ only in the order of the incoming
 /// values, but instcombine orders them so it usually won't matter.
 ///
 /// This overload removes the duplicate PHI nodes directly.
-bool EliminateDuplicatePHINodes(BasicBlock *BB);
+LLVM_ABI bool EliminateDuplicatePHINodes(BasicBlock *BB);
 
 /// Check for and eliminate duplicate PHI nodes in this block. This doesn't try
 /// to be clever about PHI nodes which differ only in the order of the incoming
 /// values, but instcombine orders them so it usually won't matter.
 ///
 /// This overload collects the PHI nodes to be removed into the ToRemove set.
-bool EliminateDuplicatePHINodes(BasicBlock *BB,
-                                SmallPtrSetImpl<PHINode *> &ToRemove);
+LLVM_ABI bool EliminateDuplicatePHINodes(BasicBlock *BB,
+                                         SmallPtrSetImpl<PHINode *> &ToRemove);
 
 /// This function is used to do simplification of a CFG.  For example, it
 /// adjusts branches to branches to eliminate the extra hop, it eliminates
@@ -181,38 +189,40 @@ bool EliminateDuplicatePHINodes(BasicBlock *BB,
 /// It returns true if a modification was made, possibly deleting the basic
 /// block that was pointed to. LoopHeaders is an optional input parameter
 /// providing the set of loop headers that SimplifyCFG should not eliminate.
-extern cl::opt<bool> RequireAndPreserveDomTree;
-bool simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
-                 DomTreeUpdater *DTU = nullptr,
-                 const SimplifyCFGOptions &Options = {},
-                 ArrayRef<WeakVH> LoopHeaders = {});
+LLVM_ABI extern cl::opt<bool> RequireAndPreserveDomTree;
+LLVM_ABI bool simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
+                          DomTreeUpdater *DTU = nullptr,
+                          const SimplifyCFGOptions &Options = {},
+                          ArrayRef<WeakVH> LoopHeaders = {});
 
 /// This function is used to flatten a CFG. For example, it uses parallel-and
 /// and parallel-or mode to collapse if-conditions and merge if-regions with
 /// identical statements.
-bool FlattenCFG(BasicBlock *BB, AAResults *AA = nullptr);
+LLVM_ABI bool FlattenCFG(BasicBlock *BB, AAResults *AA = nullptr);
 
 /// If this basic block is ONLY a setcc and a branch, and if a predecessor
 /// branches to us and one of our successors, fold the setcc into the
 /// predecessor and use logical operations to pick the right destination.
-bool foldBranchToCommonDest(BranchInst *BI, llvm::DomTreeUpdater *DTU = nullptr,
-                            MemorySSAUpdater *MSSAU = nullptr,
-                            const TargetTransformInfo *TTI = nullptr,
-                            unsigned BonusInstThreshold = 1);
+LLVM_ABI bool foldBranchToCommonDest(BranchInst *BI,
+                                     llvm::DomTreeUpdater *DTU = nullptr,
+                                     MemorySSAUpdater *MSSAU = nullptr,
+                                     const TargetTransformInfo *TTI = nullptr,
+                                     unsigned BonusInstThreshold = 1);
 
 /// This function takes a virtual register computed by an Instruction and
 /// replaces it with a slot in the stack frame, allocated via alloca.
 /// This allows the CFG to be changed around without fear of invalidating the
 /// SSA information for the value. It returns the pointer to the alloca inserted
 /// to create a stack slot for X.
-AllocaInst *DemoteRegToStack(Instruction &X,
-                             bool VolatileLoads = false,
-                             std::optional<BasicBlock::iterator> AllocaPoint = std::nullopt);
+LLVM_ABI AllocaInst *DemoteRegToStack(
+    Instruction &X, bool VolatileLoads = false,
+    std::optional<BasicBlock::iterator> AllocaPoint = std::nullopt);
 
 /// This function takes a virtual register computed by a phi node and replaces
 /// it with a slot in the stack frame, allocated via alloca. The phi node is
 /// deleted and it returns the pointer to the alloca inserted.
-AllocaInst *DemotePHIToStack(PHINode *P, std::optional<BasicBlock::iterator> AllocaPoint = std::nullopt);
+LLVM_ABI AllocaInst *DemotePHIToStack(
+    PHINode *P, std::optional<BasicBlock::iterator> AllocaPoint = std::nullopt);
 
 /// If the specified pointer points to an object that we control, try to modify
 /// the object's alignment to PrefAlign. Returns a minimum known alignment of
@@ -221,7 +231,8 @@ AllocaInst *DemotePHIToStack(PHINode *P, std::optional<BasicBlock::iterator> All
 /// Increating value alignment isn't often possible though. If alignment is
 /// important, a more reliable approach is to simply align all global variables
 /// and allocation instructions to their preferred alignment from the beginning.
-Align tryEnforceAlignment(Value *V, Align PrefAlign, const DataLayout &DL);
+LLVM_ABI Align tryEnforceAlignment(Value *V, Align PrefAlign,
+                                   const DataLayout &DL);
 
 /// Try to ensure that the alignment of \p V is at least \p PrefAlign bytes. If
 /// the owning object can be modified and has an alignment less than \p
@@ -232,11 +243,11 @@ Align tryEnforceAlignment(Value *V, Align PrefAlign, const DataLayout &DL);
 /// so if alignment is important, a more reliable approach is to simply align
 /// all global variables and allocation instructions to their preferred
 /// alignment from the beginning.
-Align getOrEnforceKnownAlignment(Value *V, MaybeAlign PrefAlign,
-                                 const DataLayout &DL,
-                                 const Instruction *CxtI = nullptr,
-                                 AssumptionCache *AC = nullptr,
-                                 const DominatorTree *DT = nullptr);
+LLVM_ABI Align getOrEnforceKnownAlignment(Value *V, MaybeAlign PrefAlign,
+                                          const DataLayout &DL,
+                                          const Instruction *CxtI = nullptr,
+                                          AssumptionCache *AC = nullptr,
+                                          const DominatorTree *DT = nullptr);
 
 /// Try to infer an alignment for the specified pointer.
 inline Align getKnownAlignment(Value *V, const DataLayout &DL,
@@ -250,10 +261,10 @@ inline Align getKnownAlignment(Value *V, const DataLayout &DL,
 /// attributes, debug information, etc. The call is not placed in a block and it
 /// will not have a name. The invoke instruction is not removed, nor are the
 /// uses replaced by the new call.
-CallInst *createCallMatchingInvoke(InvokeInst *II);
+LLVM_ABI CallInst *createCallMatchingInvoke(InvokeInst *II);
 
 /// This function converts the specified invoke into a normal call.
-CallInst *changeToCall(InvokeInst *II, DomTreeUpdater *DTU = nullptr);
+LLVM_ABI CallInst *changeToCall(InvokeInst *II, DomTreeUpdater *DTU = nullptr);
 
 ///===---------------------------------------------------------------------===//
 ///  Dbg Intrinsic utilities
@@ -261,70 +272,75 @@ CallInst *changeToCall(InvokeInst *II, DomTreeUpdater *DTU = nullptr);
 
 /// Creates and inserts a dbg_value record intrinsic before a store
 /// that has an associated llvm.dbg.value intrinsic.
-void InsertDebugValueAtStoreLoc(DbgVariableRecord *DVR, StoreInst *SI,
-                                DIBuilder &Builder);
+LLVM_ABI void InsertDebugValueAtStoreLoc(DbgVariableRecord *DVR, StoreInst *SI,
+                                         DIBuilder &Builder);
 
 /// Creates and inserts an llvm.dbg.value intrinsic before a store
 /// that has an associated llvm.dbg.value intrinsic.
-void InsertDebugValueAtStoreLoc(DbgVariableIntrinsic *DII, StoreInst *SI,
-                                DIBuilder &Builder);
+LLVM_ABI void InsertDebugValueAtStoreLoc(DbgVariableIntrinsic *DII,
+                                         StoreInst *SI, DIBuilder &Builder);
 
 /// Inserts a llvm.dbg.value intrinsic before a store to an alloca'd value
 /// that has an associated llvm.dbg.declare intrinsic.
-void ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
-                                     StoreInst *SI, DIBuilder &Builder);
-void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR, StoreInst *SI,
-                                     DIBuilder &Builder);
+LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
+                                              StoreInst *SI,
+                                              DIBuilder &Builder);
+LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR,
+                                              StoreInst *SI,
+                                              DIBuilder &Builder);
 
 /// Inserts a llvm.dbg.value intrinsic before a load of an alloca'd value
 /// that has an associated llvm.dbg.declare intrinsic.
-void ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
-                                     LoadInst *LI, DIBuilder &Builder);
-void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR, LoadInst *LI,
-                                     DIBuilder &Builder);
+LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
+                                              LoadInst *LI, DIBuilder &Builder);
+LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR,
+                                              LoadInst *LI, DIBuilder &Builder);
 
 /// Inserts a llvm.dbg.value intrinsic after a phi that has an associated
 /// llvm.dbg.declare intrinsic.
-void ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
-                                     PHINode *LI, DIBuilder &Builder);
-void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR, PHINode *LI,
-                                     DIBuilder &Builder);
+LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
+                                              PHINode *LI, DIBuilder &Builder);
+LLVM_ABI void ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR,
+                                              PHINode *LI, DIBuilder &Builder);
 
 /// Lowers llvm.dbg.declare intrinsics into appropriate set of
 /// llvm.dbg.value intrinsics.
-bool LowerDbgDeclare(Function &F);
+LLVM_ABI bool LowerDbgDeclare(Function &F);
 
 /// Propagate dbg.value intrinsics through the newly inserted PHIs.
-void insertDebugValuesForPHIs(BasicBlock *BB,
-                              SmallVectorImpl<PHINode *> &InsertedPHIs);
+LLVM_ABI void
+insertDebugValuesForPHIs(BasicBlock *BB,
+                         SmallVectorImpl<PHINode *> &InsertedPHIs);
 
 /// Replaces llvm.dbg.declare instruction when the address it
 /// describes is replaced with a new value. If Deref is true, an
 /// additional DW_OP_deref is prepended to the expression. If Offset
 /// is non-zero, a constant displacement is added to the expression
 /// (between the optional Deref operations). Offset can be negative.
-bool replaceDbgDeclare(Value *Address, Value *NewAddress, DIBuilder &Builder,
-                       uint8_t DIExprFlags, int Offset);
+LLVM_ABI bool replaceDbgDeclare(Value *Address, Value *NewAddress,
+                                DIBuilder &Builder, uint8_t DIExprFlags,
+                                int Offset);
 
 /// Replaces multiple llvm.dbg.value instructions when the alloca it describes
 /// is replaced with a new value. If Offset is non-zero, a constant displacement
 /// is added to the expression (after the mandatory Deref). Offset can be
 /// negative. New llvm.dbg.value instructions are inserted at the locations of
 /// the instructions they replace.
-void replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
-                              DIBuilder &Builder, int Offset = 0);
+LLVM_ABI void replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
+                                       DIBuilder &Builder, int Offset = 0);
 
 /// Assuming the instruction \p I is going to be deleted, attempt to salvage
 /// debug users of \p I by writing the effect of \p I in a DIExpression. If it
 /// cannot be salvaged changes its debug uses to undef.
-void salvageDebugInfo(Instruction &I);
+LLVM_ABI void salvageDebugInfo(Instruction &I);
 
 /// Implementation of salvageDebugInfo, applying only to instructions in
 /// \p Insns, rather than all debug users from findDbgUsers( \p I).
 /// Mark undef if salvaging cannot be completed.
-void salvageDebugInfoForDbgValues(Instruction &I,
-                                  ArrayRef<DbgVariableIntrinsic *> Insns,
-                                  ArrayRef<DbgVariableRecord *> DPInsns);
+LLVM_ABI void
+salvageDebugInfoForDbgValues(Instruction &I,
+                             ArrayRef<DbgVariableIntrinsic *> Insns,
+                             ArrayRef<DbgVariableRecord *> DPInsns);
 
 /// Given an instruction \p I and DIExpression \p DIExpr operating on
 /// it, append the effects of \p I to the DIExpression operand list
@@ -347,9 +363,10 @@ void salvageDebugInfoForDbgValues(Instruction &I,
 ///   Return = %a
 ///   Ops = llvm::dwarf::DW_OP_LLVM_arg0 llvm::dwarf::DW_OP_add
 ///   AdditionalValues = %b
-Value *salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps,
-                            SmallVectorImpl<uint64_t> &Ops,
-                            SmallVectorImpl<Value *> &AdditionalValues);
+LLVM_ABI Value *
+salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps,
+                     SmallVectorImpl<uint64_t> &Ops,
+                     SmallVectorImpl<Value *> &AdditionalValues);
 
 /// Point debug users of \p From to \p To or salvage them. Use this function
 /// only when replacing all uses of \p From with \p To, with a guarantee that
@@ -365,37 +382,39 @@ Value *salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps,
 /// If a debug user cannot be preserved without reordering variable updates or
 /// introducing a use-before-def, it is either salvaged (\ref salvageDebugInfo)
 /// or deleted. Returns true if any debug users were updated.
-bool replaceAllDbgUsesWith(Instruction &From, Value &To, Instruction &DomPoint,
-                           DominatorTree &DT);
+LLVM_ABI bool replaceAllDbgUsesWith(Instruction &From, Value &To,
+                                    Instruction &DomPoint, const DominatorTree &DT);
 
 /// If a terminator in an unreachable basic block has an operand of type
 /// Instruction, transform it into poison. Return true if any operands
 /// are changed to poison. Original Values prior to being changed to poison
 /// are returned in \p PoisonedValues.
-bool handleUnreachableTerminator(Instruction *I,
-                                 SmallVectorImpl<Value *> &PoisonedValues);
+LLVM_ABI bool
+handleUnreachableTerminator(Instruction *I,
+                            SmallVectorImpl<Value *> &PoisonedValues);
 
 /// Remove all instructions from a basic block other than its terminator
 /// and any present EH pad instructions. Returns a pair where the first element
 /// is the number of instructions (excluding debug info intrinsics) that have
 /// been removed, and the second element is the number of debug info intrinsics
 /// that have been removed.
-std::pair<unsigned, unsigned>
+LLVM_ABI std::pair<unsigned, unsigned>
 removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB);
 
 /// Insert an unreachable instruction before the specified
 /// instruction, making it and the rest of the code in the block dead.
-unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA = false,
-                             DomTreeUpdater *DTU = nullptr,
-                             MemorySSAUpdater *MSSAU = nullptr);
+LLVM_ABI unsigned changeToUnreachable(Instruction *I,
+                                      bool PreserveLCSSA = false,
+                                      DomTreeUpdater *DTU = nullptr,
+                                      MemorySSAUpdater *MSSAU = nullptr);
 
 /// Convert the CallInst to InvokeInst with the specified unwind edge basic
 /// block.  This also splits the basic block where CI is located, because
 /// InvokeInst is a terminator instruction.  Returns the newly split basic
 /// block.
-BasicBlock *changeToInvokeAndSplitBasicBlock(CallInst *CI,
-                                             BasicBlock *UnwindEdge,
-                                             DomTreeUpdater *DTU = nullptr);
+LLVM_ABI BasicBlock *
+changeToInvokeAndSplitBasicBlock(CallInst *CI, BasicBlock *UnwindEdge,
+                                 DomTreeUpdater *DTU = nullptr);
 
 /// Replace 'BB's terminator with one that does not have an unwind successor
 /// block. Rewrites `invoke` to `call`, etc. Updates any PHIs in unwind
@@ -404,13 +423,15 @@ BasicBlock *changeToInvokeAndSplitBasicBlock(CallInst *CI,
 ///
 /// \param BB  Block whose terminator will be replaced.  Its terminator must
 ///            have an unwind successor.
-Instruction *removeUnwindEdge(BasicBlock *BB, DomTreeUpdater *DTU = nullptr);
+LLVM_ABI Instruction *removeUnwindEdge(BasicBlock *BB,
+                                       DomTreeUpdater *DTU = nullptr);
 
 /// Remove all blocks that can not be reached from the function's entry.
 ///
 /// Returns true if any basic block was removed.
-bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU = nullptr,
-                             MemorySSAUpdater *MSSAU = nullptr);
+LLVM_ABI bool removeUnreachableBlocks(Function &F,
+                                      DomTreeUpdater *DTU = nullptr,
+                                      MemorySSAUpdater *MSSAU = nullptr);
 
 /// Combine the metadata of two instructions so that K can replace J. This
 /// specifically handles the case of CSE-like transformations. Some
@@ -418,45 +439,47 @@ bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU = nullptr,
 /// K cannot be hoisted.
 ///
 /// Unknown metadata is removed.
-void combineMetadataForCSE(Instruction *K, const Instruction *J,
-                           bool DoesKMove);
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+                                    bool DoesKMove);
 
 /// Combine metadata of two instructions, where instruction J is a memory
 /// access that has been merged into K. This will intersect alias-analysis
 /// metadata, while preserving other known metadata.
-void combineAAMetadata(Instruction *K, const Instruction *J);
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
 
 /// Copy the metadata from the source instruction to the destination (the
 /// replacement for the source instruction).
-void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
+LLVM_ABI void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
 
 /// Patch the replacement so that it is not more restrictive than the value
 /// being replaced. It assumes that the replacement does not get moved from
 /// its original position.
-void patchReplacementInstruction(Instruction *I, Value *Repl);
+LLVM_ABI void patchReplacementInstruction(Instruction *I, Value *Repl);
 
 // Replace each use of 'From' with 'To', if that use does not belong to basic
 // block where 'From' is defined. Returns the number of replacements made.
-unsigned replaceNonLocalUsesWith(Instruction *From, Value *To);
+LLVM_ABI unsigned replaceNonLocalUsesWith(Instruction *From, Value *To);
 
 /// Replace each use of 'From' with 'To' if that use is dominated by
 /// the given edge.  Returns the number of replacements made.
-unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
-                                  const BasicBlockEdge &Edge);
+LLVM_ABI unsigned replaceDominatedUsesWith(Value *From, Value *To,
+                                           DominatorTree &DT,
+                                           const BasicBlockEdge &Edge);
 /// Replace each use of 'From' with 'To' if that use is dominated by
 /// the end of the given BasicBlock. Returns the number of replacements made.
-unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
-                                  const BasicBlock *BB);
+LLVM_ABI unsigned replaceDominatedUsesWith(Value *From, Value *To,
+                                           DominatorTree &DT,
+                                           const BasicBlock *BB);
 /// Replace each use of 'From' with 'To' if that use is dominated by
 /// the given edge and the callback ShouldReplace returns true. Returns the
 /// number of replacements made.
-unsigned replaceDominatedUsesWithIf(
+LLVM_ABI unsigned replaceDominatedUsesWithIf(
     Value *From, Value *To, DominatorTree &DT, const BasicBlockEdge &Edge,
     function_ref<bool(const Use &U, const Value *To)> ShouldReplace);
 /// Replace each use of 'From' with 'To' if that use is dominated by
 /// the end of the given BasicBlock and the callback ShouldReplace returns true.
 /// Returns the number of replacements made.
-unsigned replaceDominatedUsesWithIf(
+LLVM_ABI unsigned replaceDominatedUsesWithIf(
     Value *From, Value *To, DominatorTree &DT, const BasicBlock *BB,
     function_ref<bool(const Use &U, const Value *To)> ShouldReplace);
 
@@ -468,39 +491,41 @@ unsigned replaceDominatedUsesWithIf(
 ///
 /// Most passes can and should ignore this information, and it is only used
 /// during lowering by the GC infrastructure.
-bool callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI);
+LLVM_ABI bool callsGCLeafFunction(const CallBase *Call,
+                                  const TargetLibraryInfo &TLI);
 
 /// Copy a nonnull metadata node to a new load instruction.
 ///
 /// This handles mapping it to range metadata if the new load is an integer
 /// load instead of a pointer load.
-void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N, LoadInst &NewLI);
+LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
+                                  LoadInst &NewLI);
 
 /// Copy a range metadata node to a new load instruction.
 ///
 /// This handles mapping it to nonnull metadata if the new load is a pointer
 /// load instead of an integer load and the range doesn't cover null.
-void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N,
-                       LoadInst &NewLI);
+LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
+                                MDNode *N, LoadInst &NewLI);
 
 /// Remove the debug intrinsic instructions for the given instruction.
-void dropDebugUsers(Instruction &I);
+LLVM_ABI void dropDebugUsers(Instruction &I);
 
 /// Hoist all of the instructions in the \p IfBlock to the dominant block
 /// \p DomBlock, by moving its instructions to the insertion point \p InsertPt.
 ///
 /// The moved instructions receive the insertion point debug location values
 /// (DILocations) and their debug intrinsic instructions are removed.
-void hoistAllInstructionsInto(BasicBlock *DomBlock, Instruction *InsertPt,
-                              BasicBlock *BB);
+LLVM_ABI void hoistAllInstructionsInto(BasicBlock *DomBlock,
+                                       Instruction *InsertPt, BasicBlock *BB);
 
 /// Given a constant, create a debug information expression.
-DIExpression *getExpressionForConstant(DIBuilder &DIB, const Constant &C,
-                                       Type &Ty);
+LLVM_ABI DIExpression *getExpressionForConstant(DIBuilder &DIB,
+                                                const Constant &C, Type &Ty);
 
 /// Remap the operands of the debug records attached to \p Inst, and the
 /// operands of \p Inst itself if it's a debug intrinsic.
-void remapDebugVariable(ValueToValueMapTy &Mapping, Instruction *Inst);
+LLVM_ABI void remapDebugVariable(ValueToValueMapTy &Mapping, Instruction *Inst);
 
 //===----------------------------------------------------------------------===//
 //  Intrinsic pattern matching
@@ -517,9 +542,10 @@ void remapDebugVariable(ValueToValueMapTy &Mapping, Instruction *Inst);
 /// to BW / 4 nodes to be searched, so is significantly faster.
 ///
 /// This function returns true on a successful match or false otherwise.
-bool recognizeBSwapOrBitReverseIdiom(
-    Instruction *I, bool MatchBSwaps, bool MatchBitReversals,
-    SmallVectorImpl<Instruction *> &InsertedInsts);
+LLVM_ABI bool
+recognizeBSwapOrBitReverseIdiom(Instruction *I, bool MatchBSwaps,
+                                bool MatchBitReversals,
+                                SmallVectorImpl<Instruction *> &InsertedInsts);
 
 //===----------------------------------------------------------------------===//
 //  Sanitizer utilities
@@ -529,8 +555,9 @@ bool recognizeBSwapOrBitReverseIdiom(
 /// and mark it with NoBuiltin if so.  To be used by sanitizers that intend
 /// to intercept string functions and want to avoid converting them to target
 /// specific instructions.
-void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI,
-                                            const TargetLibraryInfo *TLI);
+LLVM_ABI void
+maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI,
+                                       const TargetLibraryInfo *TLI);
 
 //===----------------------------------------------------------------------===//
 //  Transform predicates
@@ -538,15 +565,15 @@ void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI,
 
 /// Given an instruction, is it legal to set operand OpIdx to a non-constant
 /// value?
-bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx);
+LLVM_ABI bool canReplaceOperandWithVariable(const Instruction *I,
+                                            unsigned OpIdx);
 
 //===----------------------------------------------------------------------===//
 //  Value helper functions
 //
 
 /// Invert the given true/false value, possibly reusing an existing copy.
-Value *invertCondition(Value *Condition);
-
+LLVM_ABI Value *invertCondition(Value *Condition);
 
 //===----------------------------------------------------------------------===//
 //  Assorted
@@ -554,7 +581,7 @@ Value *invertCondition(Value *Condition);
 
 /// If we can infer one attribute from another on the declaration of a
 /// function, explicitly materialize the maximal set in the IR.
-bool inferAttributesFromOthers(Function &F);
+LLVM_ABI bool inferAttributesFromOthers(Function &F);
 
 //===----------------------------------------------------------------------===//
 //  Helpers to track and update flags on instructions.
@@ -581,10 +608,10 @@ struct OverflowTracking {
   OverflowTracking() = default;
 
   /// Merge in the no-wrap flags from \p I.
-  void mergeFlags(Instruction &I);
+  LLVM_ABI void mergeFlags(Instruction &I);
 
   /// Apply the no-wrap flags to \p I if applicable.
-  void applyFlags(Instruction &I);
+  LLVM_ABI void applyFlags(Instruction &I);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopRotationUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopRotationUtils.h
index 61bf93b74a15..c3643e0f27f9 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopRotationUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopRotationUtils.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_TRANSFORMS_UTILS_LOOPROTATIONUTILS_H
 #define LLVM_TRANSFORMS_UTILS_LOOPROTATIONUTILS_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 
 class AssumptionCache;
@@ -30,11 +32,12 @@ class TargetTransformInfo;
 /// header. If the loop header's size exceeds the threshold, the loop rotation
 /// will give up. The flag IsUtilMode controls the heuristic used in the
 /// LoopRotation. If it is true, the profitability heuristic will be ignored.
-bool LoopRotation(Loop *L, LoopInfo *LI, const TargetTransformInfo *TTI,
-                  AssumptionCache *AC, DominatorTree *DT, ScalarEvolution *SE,
-                  MemorySSAUpdater *MSSAU, const SimplifyQuery &SQ,
-                  bool RotationOnly, unsigned Threshold, bool IsUtilMode,
-                  bool PrepareForLTO = false);
+LLVM_ABI bool LoopRotation(Loop *L, LoopInfo *LI,
+                           const TargetTransformInfo *TTI, AssumptionCache *AC,
+                           DominatorTree *DT, ScalarEvolution *SE,
+                           MemorySSAUpdater *MSSAU, const SimplifyQuery &SQ,
+                           bool RotationOnly, unsigned Threshold,
+                           bool IsUtilMode, bool PrepareForLTO = false);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopSimplify.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
index d017fd12026d..8f3fa1f2b18e 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
@@ -39,6 +39,7 @@
 #define LLVM_TRANSFORMS_UTILS_LOOPSIMPLIFY_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -52,7 +53,7 @@ class ScalarEvolution;
 /// This pass is responsible for loop canonicalization.
 class LoopSimplifyPass : public PassInfoMixin<LoopSimplifyPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 /// Simplify each loop in a loop nest recursively.
@@ -61,9 +62,9 @@ class LoopSimplifyPass : public PassInfoMixin<LoopSimplifyPass> {
 /// it into a simplified loop nest with preheaders and single backedges. It will
 /// update \c DominatorTree, \c LoopInfo, \c ScalarEvolution and \c MemorySSA
 /// analyses if they're non-null, and LCSSA if \c PreserveLCSSA is true.
-bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE,
-                  AssumptionCache *AC, MemorySSAUpdater *MSSAU,
-                  bool PreserveLCSSA);
+LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
+                           ScalarEvolution *SE, AssumptionCache *AC,
+                           MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 3ad1113ecacf..12be3bad04d3 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -16,7 +16,7 @@
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/IR/VectorBuilder.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 
 namespace llvm {
@@ -52,16 +52,19 @@ typedef std::pair<const RuntimeCheckingPtrGroup *,
 template <typename T, unsigned N> class SmallSetVector;
 template <typename T, unsigned N> class SmallPriorityWorklist;
 
-BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
-                                   MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
+LLVM_ABI BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT,
+                                            LoopInfo *LI,
+                                            MemorySSAUpdater *MSSAU,
+                                            bool PreserveLCSSA);
 
 /// Ensure that all exit blocks of the loop are dedicated exits.
 ///
 /// For any loop exit block with non-loop predecessors, we split the loop
 /// predecessors to use a dedicated loop exit block. We update the dominator
 /// tree and loop info if provided, and will preserve LCSSA if requested.
-bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
-                             MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
+LLVM_ABI bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
+                                      MemorySSAUpdater *MSSAU,
+                                      bool PreserveLCSSA);
 
 /// Ensures LCSSA form for every instruction from the Worklist in the scope of
 /// innermost containing loop.
@@ -81,11 +84,12 @@ bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
 ///
 /// If \p InsertedPHIs is not nullptr, inserted phis will be added to this
 /// vector.
-bool formLCSSAForInstructions(
-    SmallVectorImpl<Instruction *> &Worklist, const DominatorTree &DT,
-    const LoopInfo &LI, ScalarEvolution *SE,
-    SmallVectorImpl<PHINode *> *PHIsToRemove = nullptr,
-    SmallVectorImpl<PHINode *> *InsertedPHIs = nullptr);
+LLVM_ABI bool
+formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
+                         const DominatorTree &DT, const LoopInfo &LI,
+                         ScalarEvolution *SE,
+                         SmallVectorImpl<PHINode *> *PHIsToRemove = nullptr,
+                         SmallVectorImpl<PHINode *> *InsertedPHIs = nullptr);
 
 /// Put loop into LCSSA form.
 ///
@@ -99,8 +103,8 @@ bool formLCSSAForInstructions(
 /// If ScalarEvolution is passed in, it will be preserved.
 ///
 /// Returns true if any modifications are made to the loop.
-bool formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
-               ScalarEvolution *SE);
+LLVM_ABI bool formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
+                        ScalarEvolution *SE);
 
 /// Put a loop nest into LCSSA form.
 ///
@@ -111,8 +115,8 @@ bool formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
 /// If ScalarEvolution is passed in, it will be preserved.
 ///
 /// Returns true if any modifications are made to the loop.
-bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
-                          ScalarEvolution *SE);
+LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT,
+                                   const LoopInfo *LI, ScalarEvolution *SE);
 
 /// Flags controlling how much is checked when sinking or hoisting
 /// instructions.  The number of memory access in the loop (and whether there
@@ -120,11 +124,11 @@ bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
 class SinkAndHoistLICMFlags {
 public:
   // Explicitly set limits.
-  SinkAndHoistLICMFlags(unsigned LicmMssaOptCap,
-                        unsigned LicmMssaNoAccForPromotionCap, bool IsSink,
-                        Loop &L, MemorySSA &MSSA);
+  LLVM_ABI SinkAndHoistLICMFlags(unsigned LicmMssaOptCap,
+                                 unsigned LicmMssaNoAccForPromotionCap,
+                                 bool IsSink, Loop &L, MemorySSA &MSSA);
   // Use default limits.
-  SinkAndHoistLICMFlags(bool IsSink, Loop &L, MemorySSA &MSSA);
+  LLVM_ABI SinkAndHoistLICMFlags(bool IsSink, Loop &L, MemorySSA &MSSA);
 
   void setIsSink(bool B) { IsSink = B; }
   bool getIsSink() { return IsSink; }
@@ -150,19 +154,21 @@ class SinkAndHoistLICMFlags {
 /// arguments. Diagnostics is emitted via \p ORE. It returns changed status.
 /// \p CurLoop is a loop to do sinking on. \p OutermostLoop is used only when
 /// this function is called by \p sinkRegionForLoopNest.
-bool sinkRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *,
-                TargetLibraryInfo *, TargetTransformInfo *, Loop *CurLoop,
-                MemorySSAUpdater &, ICFLoopSafetyInfo *,
-                SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *,
-                Loop *OutermostLoop = nullptr);
+LLVM_ABI bool sinkRegion(DomTreeNode *, AAResults *, LoopInfo *,
+                         DominatorTree *, TargetLibraryInfo *,
+                         TargetTransformInfo *, Loop *CurLoop,
+                         MemorySSAUpdater &, ICFLoopSafetyInfo *,
+                         SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *,
+                         Loop *OutermostLoop = nullptr);
 
 /// Call sinkRegion on loops contained within the specified loop
 /// in order from innermost to outermost.
-bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *,
-                           DominatorTree *, TargetLibraryInfo *,
-                           TargetTransformInfo *, Loop *, MemorySSAUpdater &,
-                           ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &,
-                           OptimizationRemarkEmitter *);
+LLVM_ABI bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *,
+                                    DominatorTree *, TargetLibraryInfo *,
+                                    TargetTransformInfo *, Loop *,
+                                    MemorySSAUpdater &, ICFLoopSafetyInfo *,
+                                    SinkAndHoistLICMFlags &,
+                                    OptimizationRemarkEmitter *);
 
 /// Walk the specified region of the CFG (defined by all blocks
 /// dominated by the specified block, and that are in the current loop) in depth
@@ -174,16 +180,17 @@ bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *,
 /// Diagnostics is emitted via \p ORE. It returns changed status.
 /// \p AllowSpeculation is whether values should be hoisted even if they are not
 /// guaranteed to execute in the loop, but are safe to speculatively execute.
-bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *,
-                 AssumptionCache *, TargetLibraryInfo *, Loop *,
-                 MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *,
-                 SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool,
-                 bool AllowSpeculation);
+LLVM_ABI bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *,
+                          DominatorTree *, AssumptionCache *,
+                          TargetLibraryInfo *, Loop *, MemorySSAUpdater &,
+                          ScalarEvolution *, ICFLoopSafetyInfo *,
+                          SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *,
+                          bool, bool AllowSpeculation);
 
 /// Return true if the induction variable \p IV in a Loop whose latch is
 /// \p LatchBlock would become dead if the exit test \p Cond were removed.
 /// Conservatively returns false if analysis is insufficient.
-bool isAlmostDeadIV(PHINode *IV, BasicBlock *LatchBlock, Value *Cond);
+LLVM_ABI bool isAlmostDeadIV(PHINode *IV, BasicBlock *LatchBlock, Value *Cond);
 
 /// This function deletes dead loops. The caller of this function needs to
 /// guarantee that the loop is infact dead.
@@ -196,14 +203,14 @@ bool isAlmostDeadIV(PHINode *IV, BasicBlock *LatchBlock, Value *Cond);
 /// and \p MSSA if pointers to those are provided.
 /// It also updates the loop PM if an updater struct is provided.
 
-void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE,
-                    LoopInfo *LI, MemorySSA *MSSA = nullptr);
+LLVM_ABI void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE,
+                             LoopInfo *LI, MemorySSA *MSSA = nullptr);
 
 /// Remove the backedge of the specified loop.  Handles loop nests and general
 /// loop structures subject to the precondition that the loop has no parent
 /// loop and has a single latch block.  Preserves all listed analyses.
-void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE,
-                       LoopInfo &LI, MemorySSA *MSSA);
+LLVM_ABI void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE,
+                                LoopInfo &LI, MemorySSA *MSSA);
 
 /// Try to promote memory values to scalars by sinking stores out of
 /// the loop and moving loads to before the loop.  We do this by looping over
@@ -215,7 +222,7 @@ void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE,
 /// Diagnostics is emitted via \p ORE. It returns changed status.
 /// \p AllowSpeculation is whether values should be hoisted even if they are not
 /// guaranteed to execute in the loop, but are safe to speculatively execute.
-bool promoteLoopAccessesToScalars(
+LLVM_ABI bool promoteLoopAccessesToScalars(
     const SmallSetVector<Value *, 8> &, SmallVectorImpl<BasicBlock *> &,
     SmallVectorImpl<BasicBlock::iterator> &, SmallVectorImpl<MemoryAccess *> &,
     PredIteratorCache &, LoopInfo *, DominatorTree *, AssumptionCache *AC,
@@ -225,17 +232,17 @@ bool promoteLoopAccessesToScalars(
 
 /// Does a BFS from a given node to all of its children inside a given loop.
 /// The returned vector of basic blocks includes the starting point.
-SmallVector<BasicBlock *, 16>
+LLVM_ABI SmallVector<BasicBlock *, 16>
 collectChildrenInLoop(DominatorTree *DT, DomTreeNode *N, const Loop *CurLoop);
 
 /// Returns the instructions that use values defined in the loop.
-SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L);
+LLVM_ABI SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L);
 
 /// Find a combination of metadata ("llvm.loop.vectorize.width" and
 /// "llvm.loop.vectorize.scalable.enable") for a loop and use it to construct a
 /// ElementCount. If the metadata "llvm.loop.vectorize.width" cannot be found
 /// then std::nullopt is returned.
-std::optional<ElementCount>
+LLVM_ABI std::optional<ElementCount>
 getOptionalElementCountLoopAttribute(const Loop *TheLoop);
 
 /// Create a new loop identifier for a loop created from a loop transformation.
@@ -262,16 +269,16 @@ getOptionalElementCountLoopAttribute(const Loop *TheLoop);
 ///         @p OrigLoopID: The original identifier can be reused.
 ///         nullptr      : The new loop has no attributes.
 ///         MDNode*      : A new unique loop identifier.
-std::optional<MDNode *>
+LLVM_ABI std::optional<MDNode *>
 makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef<StringRef> FollowupAttrs,
                    const char *InheritOptionsAttrsPrefix = "",
                    bool AlwaysNew = false);
 
 /// Look for the loop attribute that disables all transformation heuristic.
-bool hasDisableAllTransformsHint(const Loop *L);
+LLVM_ABI bool hasDisableAllTransformsHint(const Loop *L);
 
 /// Look for the loop attribute that disables the LICM transformation heuristics.
-bool hasDisableLICMTransformsHint(const Loop *L);
+LLVM_ABI bool hasDisableLICMTransformsHint(const Loop *L);
 
 /// The mode sets how eager a transformation should be applied.
 enum TransformationMode {
@@ -302,25 +309,25 @@ enum TransformationMode {
 
 /// @{
 /// Get the mode for LLVM's supported loop transformations.
-TransformationMode hasUnrollTransformation(const Loop *L);
-TransformationMode hasUnrollAndJamTransformation(const Loop *L);
-TransformationMode hasVectorizeTransformation(const Loop *L);
-TransformationMode hasDistributeTransformation(const Loop *L);
-TransformationMode hasLICMVersioningTransformation(const Loop *L);
+LLVM_ABI TransformationMode hasUnrollTransformation(const Loop *L);
+LLVM_ABI TransformationMode hasUnrollAndJamTransformation(const Loop *L);
+LLVM_ABI TransformationMode hasVectorizeTransformation(const Loop *L);
+LLVM_ABI TransformationMode hasDistributeTransformation(const Loop *L);
+LLVM_ABI TransformationMode hasLICMVersioningTransformation(const Loop *L);
 /// @}
 
 /// Set input string into loop metadata by keeping other values intact.
 /// If the string is already in loop metadata update value if it is
 /// different.
-void addStringMetadataToLoop(Loop *TheLoop, const char *MDString,
-                             unsigned V = 0);
+LLVM_ABI void addStringMetadataToLoop(Loop *TheLoop, const char *MDString,
+                                      unsigned V = 0);
 
 /// Returns a loop's estimated trip count based on branch weight metadata.
 /// In addition if \p EstimatedLoopInvocationWeight is not null it is
 /// initialized with weight of loop's latch leading to the exit.
 /// Returns a valid positive trip count, saturated at UINT_MAX, or std::nullopt
 /// when a meaningful estimate cannot be made.
-std::optional<unsigned>
+LLVM_ABI std::optional<unsigned>
 getLoopEstimatedTripCount(Loop *L,
                           unsigned *EstimatedLoopInvocationWeight = nullptr);
 
@@ -329,20 +336,20 @@ getLoopEstimatedTripCount(Loop *L,
 /// through latch. Returns true if metadata is successfully updated, false
 /// otherwise. Note that loop must have a latch block which controls loop exit
 /// in order to succeed.
-bool setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount,
-                               unsigned EstimatedLoopInvocationWeight);
+LLVM_ABI bool setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount,
+                                        unsigned EstimatedLoopInvocationWeight);
 
 /// Check inner loop (L) backedge count is known to be invariant on all
 /// iterations of its outer loop. If the loop has no parent, this is trivially
 /// true.
-bool hasIterationCountInvariantInParent(Loop *L, ScalarEvolution &SE);
+LLVM_ABI bool hasIterationCountInvariantInParent(Loop *L, ScalarEvolution &SE);
 
 /// Helper to consistently add the set of standard passes to a loop pass's \c
 /// AnalysisUsage.
 ///
 /// All loop passes should call this as part of implementing their \c
 /// getAnalysisUsage.
-void getLoopAnalysisUsage(AnalysisUsage &AU);
+LLVM_ABI void getLoopAnalysisUsage(AnalysisUsage &AU);
 
 /// Returns true if is legal to hoist or sink this instruction disregarding the
 /// possible introduction of faults.  Reasoning about potential faulting
@@ -353,84 +360,91 @@ void getLoopAnalysisUsage(AnalysisUsage &AU);
 /// to assess the legality of duplicating atomic loads.  Generally, this is
 /// true when moving out of loop and not true when moving into loops.
 /// If \p ORE is set use it to emit optimization remarks.
-bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
-                        Loop *CurLoop, MemorySSAUpdater &MSSAU,
-                        bool TargetExecutesOncePerLoop,
-                        SinkAndHoistLICMFlags &LICMFlags,
-                        OptimizationRemarkEmitter *ORE = nullptr);
+LLVM_ABI bool canSinkOrHoistInst(Instruction &I, AAResults *AA,
+                                 DominatorTree *DT, Loop *CurLoop,
+                                 MemorySSAUpdater &MSSAU,
+                                 bool TargetExecutesOncePerLoop,
+                                 SinkAndHoistLICMFlags &LICMFlags,
+                                 OptimizationRemarkEmitter *ORE = nullptr);
 
 /// Returns the llvm.vector.reduce intrinsic that corresponds to the recurrence
 /// kind.
-constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK);
+LLVM_ABI constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK);
 
 /// Returns the arithmetic instruction opcode used when expanding a reduction.
-unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID);
+LLVM_ABI unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID);
 /// Returns the reduction intrinsic id corresponding to the binary operation.
-Intrinsic::ID getReductionForBinop(Instruction::BinaryOps Opc);
+LLVM_ABI Intrinsic::ID getReductionForBinop(Instruction::BinaryOps Opc);
 
 /// Returns the min/max intrinsic used when expanding a min/max reduction.
-Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID);
+LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID);
 
 /// Returns the min/max intrinsic used when expanding a min/max reduction.
-Intrinsic::ID getMinMaxReductionIntrinsicOp(RecurKind RK);
+LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(RecurKind RK);
 
 /// Returns the recurence kind used when expanding a min/max reduction.
-RecurKind getMinMaxReductionRecurKind(Intrinsic::ID RdxID);
+LLVM_ABI RecurKind getMinMaxReductionRecurKind(Intrinsic::ID RdxID);
 
 /// Returns the comparison predicate used when expanding a min/max reduction.
-CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK);
+LLVM_ABI CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK);
 
 /// Given information about an @llvm.vector.reduce.* intrinsic, return
 /// the identity value for the reduction.
-Value *getReductionIdentity(Intrinsic::ID RdxID, Type *Ty, FastMathFlags FMF);
+LLVM_ABI Value *getReductionIdentity(Intrinsic::ID RdxID, Type *Ty,
+                                     FastMathFlags FMF);
 
 /// Given information about an recurrence kind, return the identity
 /// for the @llvm.vector.reduce.* used to generate it.
-Value *getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF);
+LLVM_ABI Value *getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF);
 
 /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
 /// The Builder's fast-math-flags must be set to propagate the expected values.
-Value *createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
-                      Value *Right);
+LLVM_ABI Value *createMinMaxOp(IRBuilderBase &Builder, RecurKind RK,
+                               Value *Left, Value *Right);
 
 /// Generates an ordered vector reduction using extracts to reduce the value.
-Value *getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
-                           unsigned Op, RecurKind MinMaxKind = RecurKind::None);
+LLVM_ABI Value *getOrderedReduction(IRBuilderBase &Builder, Value *Acc,
+                                    Value *Src, unsigned Op,
+                                    RecurKind MinMaxKind = RecurKind::None);
 
 /// Generates a vector reduction using shufflevectors to reduce the value.
 /// Fast-math-flags are propagated using the IRBuilder's setting.
-Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
-                           TargetTransformInfo::ReductionShuffle RS,
-                           RecurKind MinMaxKind = RecurKind::None);
+LLVM_ABI Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src,
+                                    unsigned Op,
+                                    TargetTransformInfo::ReductionShuffle RS,
+                                    RecurKind MinMaxKind = RecurKind::None);
 
 /// Create a reduction of the given vector. The reduction operation
 /// is described by the \p Opcode parameter. min/max reductions require
 /// additional information supplied in \p RdxKind.
 /// Fast-math-flags are propagated using the IRBuilder's setting.
-Value *createSimpleReduction(IRBuilderBase &B, Value *Src,
-                             RecurKind RdxKind);
+LLVM_ABI Value *createSimpleReduction(IRBuilderBase &B, Value *Src,
+                                      RecurKind RdxKind);
 /// Overloaded function to generate vector-predication intrinsics for
 /// reduction.
-Value *createSimpleReduction(VectorBuilder &VB, Value *Src, RecurKind RdxKind);
+LLVM_ABI Value *createSimpleReduction(IRBuilderBase &B, Value *Src,
+                                      RecurKind RdxKind, Value *Mask,
+                                      Value *EVL);
 
 /// Create a reduction of the given vector \p Src for a reduction of kind
 /// RecurKind::AnyOf. The start value of the reduction is \p InitVal.
-Value *createAnyOfReduction(IRBuilderBase &B, Value *Src, Value *InitVal,
-                            PHINode *OrigPhi);
+LLVM_ABI Value *createAnyOfReduction(IRBuilderBase &B, Value *Src,
+                                     Value *InitVal, PHINode *OrigPhi);
 
 /// Create a reduction of the given vector \p Src for a reduction of the
 /// kind RecurKind::FindLastIV.
-Value *createFindLastIVReduction(IRBuilderBase &B, Value *Src, Value *Start,
-                                 Value *Sentinel);
+LLVM_ABI Value *createFindLastIVReduction(IRBuilderBase &B, Value *Src,
+                                          Value *Start, Value *Sentinel);
 
 /// Create an ordered reduction intrinsic using the given recurrence
 /// kind \p RdxKind.
-Value *createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind, Value *Src,
-                              Value *Start);
+LLVM_ABI Value *createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind,
+                                       Value *Src, Value *Start);
 /// Overloaded function to generate vector-predication intrinsics for ordered
 /// reduction.
-Value *createOrderedReduction(VectorBuilder &VB, RecurKind RdxKind, Value *Src,
-                              Value *Start);
+LLVM_ABI Value *createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind,
+                                       Value *Src, Value *Start, Value *Mask,
+                                       Value *EVL);
 
 /// Get the intersection (logical and) of all of the potential IR flags
 /// of each scalar operation (VL) that will be converted into a vector (I).
@@ -438,33 +452,36 @@ Value *createOrderedReduction(VectorBuilder &VB, RecurKind RdxKind, Value *Src,
 /// when intersecting.
 /// Flag set: NSW, NUW (if IncludeWrapFlags is true), exact, and all of
 /// fast-math.
-void propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue = nullptr,
-                      bool IncludeWrapFlags = true);
+LLVM_ABI void propagateIRFlags(Value *I, ArrayRef<Value *> VL,
+                               Value *OpValue = nullptr,
+                               bool IncludeWrapFlags = true);
 
 /// Returns true if we can prove that \p S is defined and always negative in
 /// loop \p L.
-bool isKnownNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE);
+LLVM_ABI bool isKnownNegativeInLoop(const SCEV *S, const Loop *L,
+                                    ScalarEvolution &SE);
 
 /// Returns true if we can prove that \p S is defined and always non-negative in
 /// loop \p L.
-bool isKnownNonNegativeInLoop(const SCEV *S, const Loop *L,
-                              ScalarEvolution &SE);
+LLVM_ABI bool isKnownNonNegativeInLoop(const SCEV *S, const Loop *L,
+                                       ScalarEvolution &SE);
 /// Returns true if we can prove that \p S is defined and always positive in
 /// loop \p L.
-bool isKnownPositiveInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE);
+LLVM_ABI bool isKnownPositiveInLoop(const SCEV *S, const Loop *L,
+                                    ScalarEvolution &SE);
 
 /// Returns true if we can prove that \p S is defined and always non-positive in
 /// loop \p L.
-bool isKnownNonPositiveInLoop(const SCEV *S, const Loop *L,
-                              ScalarEvolution &SE);
+LLVM_ABI bool isKnownNonPositiveInLoop(const SCEV *S, const Loop *L,
+                                       ScalarEvolution &SE);
 
 /// Returns true if \p S is defined and never is equal to signed/unsigned max.
-bool cannotBeMaxInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
-                       bool Signed);
+LLVM_ABI bool cannotBeMaxInLoop(const SCEV *S, const Loop *L,
+                                ScalarEvolution &SE, bool Signed);
 
 /// Returns true if \p S is defined and never is equal to signed/unsigned min.
-bool cannotBeMinInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
-                       bool Signed);
+LLVM_ABI bool cannotBeMinInLoop(const SCEV *S, const Loop *L,
+                                ScalarEvolution &SE, bool Signed);
 
 enum ReplaceExitVal {
   NeverRepl,
@@ -479,11 +496,12 @@ enum ReplaceExitVal {
 /// outside of the loop that use the final values of the current expressions.
 /// Return the number of loop exit values that have been replaced, and the
 /// corresponding phi node will be added to DeadInsts.
-int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI,
-                          ScalarEvolution *SE, const TargetTransformInfo *TTI,
-                          SCEVExpander &Rewriter, DominatorTree *DT,
-                          ReplaceExitVal ReplaceExitValue,
-                          SmallVector<WeakTrackingVH, 16> &DeadInsts);
+LLVM_ABI int rewriteLoopExitValues(Loop *L, LoopInfo *LI,
+                                   TargetLibraryInfo *TLI, ScalarEvolution *SE,
+                                   const TargetTransformInfo *TTI,
+                                   SCEVExpander &Rewriter, DominatorTree *DT,
+                                   ReplaceExitVal ReplaceExitValue,
+                                   SmallVector<WeakTrackingVH, 16> &DeadInsts);
 
 /// Set weights for \p UnrolledLoop and \p RemainderLoop based on weights for
 /// \p OrigLoop and the following distribution of \p OrigLoop iteration among \p
@@ -499,8 +517,8 @@ int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI,
 ///
 /// This utility may be useful for such optimizations as unroller and
 /// vectorizer as it's typical transformation for them.
-void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop,
-                                  Loop *RemainderLoop, uint64_t UF);
+LLVM_ABI void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop,
+                                           Loop *RemainderLoop, uint64_t UF);
 
 /// Utility that implements appending of loops onto a worklist given a range.
 /// We want to process loops in postorder, but the worklist is a LIFO data
@@ -508,7 +526,8 @@ void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop,
 /// For trees, a preorder traversal is a viable reverse postorder, so we
 /// actually append using a preorder walk algorithm.
 template <typename RangeT>
-void appendLoopsToWorklist(RangeT &&, SmallPriorityWorklist<Loop *, 4> &);
+LLVM_TEMPLATE_ABI void
+appendLoopsToWorklist(RangeT &&, SmallPriorityWorklist<Loop *, 4> &);
 /// Utility that implements appending of loops onto a worklist given a range.
 /// It has the same behavior as appendLoopsToWorklist, but assumes the range of
 /// loops has already been reversed, so it processes loops in the given order.
@@ -516,6 +535,14 @@ template <typename RangeT>
 void appendReversedLoopsToWorklist(RangeT &&,
                                    SmallPriorityWorklist<Loop *, 4> &);
 
+extern template LLVM_TEMPLATE_ABI void
+appendLoopsToWorklist<ArrayRef<Loop *> &>(
+    ArrayRef<Loop *> &Loops, SmallPriorityWorklist<Loop *, 4> &Worklist);
+
+extern template LLVM_TEMPLATE_ABI void
+appendLoopsToWorklist<Loop &>(Loop &L,
+                              SmallPriorityWorklist<Loop *, 4> &Worklist);
+
 /// Utility that implements appending of loops onto a worklist given LoopInfo.
 /// Calls the templated utility taking a Range of loops, handing it the Loops
 /// in LoopInfo, iterated in reverse. This is because the loops are stored in
@@ -525,21 +552,22 @@ void appendReversedLoopsToWorklist(RangeT &&,
 /// loop nest into the next. Calls appendReversedLoopsToWorklist with the
 /// already reversed loops in LI.
 /// FIXME: Consider changing the order in LoopInfo.
-void appendLoopsToWorklist(LoopInfo &, SmallPriorityWorklist<Loop *, 4> &);
+LLVM_ABI void appendLoopsToWorklist(LoopInfo &,
+                                    SmallPriorityWorklist<Loop *, 4> &);
 
 /// Recursively clone the specified loop and all of its children,
 /// mapping the blocks with the specified map.
-Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM,
-                LoopInfo *LI, LPPassManager *LPM);
+LLVM_ABI Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, LoopInfo *LI,
+                         LPPassManager *LPM);
 
 /// Add code that checks at runtime if the accessed arrays in \p PointerChecks
 /// overlap. Returns the final comparator value or NULL if no check is needed.
-Value *
+LLVM_ABI Value *
 addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
                  const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
                  SCEVExpander &Expander, bool HoistRuntimeChecks = false);
 
-Value *addDiffRuntimeChecks(
+LLVM_ABI Value *addDiffRuntimeChecks(
     Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
     function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC);
 
@@ -572,10 +600,9 @@ struct IVConditionInfo {
 /// If the branch condition of the header is partially invariant, return a pair
 /// containing the instructions to duplicate and a boolean Constant to update
 /// the condition in the loops created for the true or false successors.
-std::optional<IVConditionInfo> hasPartialIVCondition(const Loop &L,
-                                                     unsigned MSSAThreshold,
-                                                     const MemorySSA &MSSA,
-                                                     AAResults &AA);
+LLVM_ABI std::optional<IVConditionInfo>
+hasPartialIVCondition(const Loop &L, unsigned MSSAThreshold,
+                      const MemorySSA &MSSA, AAResults &AA);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
index 6a2c44a5ceb6..fe2625cb993a 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_TRANSFORMS_UTILS_LOWERMEMINTRINSICS_H
 #define LLVM_TRANSFORMS_UTILS_LOWERMEMINTRINSICS_H
 
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <optional>
 
@@ -33,7 +34,7 @@ struct Align;
 
 /// Emit a loop implementing the semantics of llvm.memcpy where the size is not
 /// a compile-time constant. Loop will be insterted at \p InsertBefore.
-void createMemCpyLoopUnknownSize(
+LLVM_ABI void createMemCpyLoopUnknownSize(
     Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, Value *CopyLen,
     Align SrcAlign, Align DestAlign, bool SrcIsVolatile, bool DstIsVolatile,
     bool CanOverlap, const TargetTransformInfo &TTI,
@@ -41,30 +42,32 @@ void createMemCpyLoopUnknownSize(
 
 /// Emit a loop implementing the semantics of an llvm.memcpy whose size is a
 /// compile time constant. Loop is inserted at \p InsertBefore.
-void createMemCpyLoopKnownSize(
+LLVM_ABI void createMemCpyLoopKnownSize(
     Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr,
     ConstantInt *CopyLen, Align SrcAlign, Align DestAlign, bool SrcIsVolatile,
     bool DstIsVolatile, bool CanOverlap, const TargetTransformInfo &TTI,
     std::optional<uint32_t> AtomicCpySize = std::nullopt);
 
 /// Expand \p MemCpy as a loop. \p MemCpy is not deleted.
-void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI,
-                        ScalarEvolution *SE = nullptr);
+LLVM_ABI void expandMemCpyAsLoop(MemCpyInst *MemCpy,
+                                 const TargetTransformInfo &TTI,
+                                 ScalarEvolution *SE = nullptr);
 
 /// Expand \p MemMove as a loop. \p MemMove is not deleted. Returns true if the
 /// memmove was lowered.
-bool expandMemMoveAsLoop(MemMoveInst *MemMove, const TargetTransformInfo &TTI);
+LLVM_ABI bool expandMemMoveAsLoop(MemMoveInst *MemMove,
+                                  const TargetTransformInfo &TTI);
 
 /// Expand \p MemSet as a loop. \p MemSet is not deleted.
-void expandMemSetAsLoop(MemSetInst *MemSet);
+LLVM_ABI void expandMemSetAsLoop(MemSetInst *MemSet);
 
 /// Expand \p MemSetPattern as a loop. \p MemSet is not deleted.
-void expandMemSetPatternAsLoop(MemSetPatternInst *MemSet);
+LLVM_ABI void expandMemSetPatternAsLoop(MemSetPatternInst *MemSet);
 
 /// Expand \p AtomicMemCpy as a loop. \p AtomicMemCpy is not deleted.
-void expandAtomicMemCpyAsLoop(AnyMemCpyInst *AtomicMemCpy,
-                              const TargetTransformInfo &TTI,
-                              ScalarEvolution *SE);
+LLVM_ABI void expandAtomicMemCpyAsLoop(AnyMemCpyInst *AtomicMemCpy,
+                                       const TargetTransformInfo &TTI,
+                                       ScalarEvolution *SE);
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ModuleUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ModuleUtils.h
index 2a52d3ed1431..4036c4e947c7 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ModuleUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ModuleUtils.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/GlobalIFunc.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include <utility> // for std::pair
 
@@ -38,41 +39,44 @@ class Type;
 /// This wraps the function in the appropriate structure and stores it along
 /// side other global constructors. For details see
 /// https://llvm.org/docs/LangRef.html#the-llvm-global-ctors-global-variable
-void appendToGlobalCtors(Module &M, Function *F, int Priority,
-                         Constant *Data = nullptr);
+LLVM_ABI void appendToGlobalCtors(Module &M, Function *F, int Priority,
+                                  Constant *Data = nullptr);
 
 /// Same as appendToGlobalCtors(), but for global dtors.
-void appendToGlobalDtors(Module &M, Function *F, int Priority,
-                         Constant *Data = nullptr);
+LLVM_ABI void appendToGlobalDtors(Module &M, Function *F, int Priority,
+                                  Constant *Data = nullptr);
 
 /// Apply 'Fn' to the list of global ctors of module M and replace contructor
 /// record with the one returned by `Fn`. If `nullptr` was returned, the
 /// corresponding constructor will be removed from the array. For details see
 /// https://llvm.org/docs/LangRef.html#the-llvm-global-ctors-global-variable
 using GlobalCtorTransformFn = llvm::function_ref<Constant *(Constant *)>;
-void transformGlobalCtors(Module &M, const GlobalCtorTransformFn &Fn);
-void transformGlobalDtors(Module &M, const GlobalCtorTransformFn &Fn);
+LLVM_ABI void transformGlobalCtors(Module &M, const GlobalCtorTransformFn &Fn);
+LLVM_ABI void transformGlobalDtors(Module &M, const GlobalCtorTransformFn &Fn);
 
 /// Sets the KCFI type for the function. Used for compiler-generated functions
 /// that are indirectly called in instrumented code.
-void setKCFIType(Module &M, Function &F, StringRef MangledType);
+LLVM_ABI void setKCFIType(Module &M, Function &F, StringRef MangledType);
 
-FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName,
-                                            ArrayRef<Type *> InitArgTypes,
-                                            bool Weak = false);
+LLVM_ABI FunctionCallee
+declareSanitizerInitFunction(Module &M, StringRef InitName,
+                             ArrayRef<Type *> InitArgTypes, bool Weak = false);
 
 /// Creates sanitizer constructor function.
 /// \return Returns pointer to constructor.
-Function *createSanitizerCtor(Module &M, StringRef CtorName);
+LLVM_ABI Function *createSanitizerCtor(Module &M, StringRef CtorName);
 
 /// Creates sanitizer constructor function, and calls sanitizer's init
 /// function from it.
 /// \return Returns pair of pointers to constructor, and init functions
 /// respectively.
-std::pair<Function *, FunctionCallee> createSanitizerCtorAndInitFunctions(
-    Module &M, StringRef CtorName, StringRef InitName,
-    ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
-    StringRef VersionCheckName = StringRef(), bool Weak = false);
+LLVM_ABI std::pair<Function *, FunctionCallee>
+createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName,
+                                    StringRef InitName,
+                                    ArrayRef<Type *> InitArgTypes,
+                                    ArrayRef<Value *> InitArgs,
+                                    StringRef VersionCheckName = StringRef(),
+                                    bool Weak = false);
 
 /// Creates sanitizer constructor function lazily. If a constructor and init
 /// function already exist, this function returns it. Otherwise it calls \c
@@ -81,7 +85,8 @@ std::pair<Function *, FunctionCallee> createSanitizerCtorAndInitFunctions(
 ///
 /// \return Returns pair of pointers to constructor, and init functions
 /// respectively.
-std::pair<Function *, FunctionCallee> getOrCreateSanitizerCtorAndInitFunctions(
+LLVM_ABI std::pair<Function *, FunctionCallee>
+getOrCreateSanitizerCtorAndInitFunctions(
     Module &M, StringRef CtorName, StringRef InitName,
     ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
     function_ref<void(Function *, FunctionCallee)> FunctionsCreatedCallback,
@@ -89,19 +94,19 @@ std::pair<Function *, FunctionCallee> getOrCreateSanitizerCtorAndInitFunctions(
 
 /// Rename all the anon globals in the module using a hash computed from
 /// the list of public globals in the module.
-bool nameUnamedGlobals(Module &M);
+LLVM_ABI bool nameUnamedGlobals(Module &M);
 
 /// Adds global values to the llvm.used list.
-void appendToUsed(Module &M, ArrayRef<GlobalValue *> Values);
+LLVM_ABI void appendToUsed(Module &M, ArrayRef<GlobalValue *> Values);
 
 /// Adds global values to the llvm.compiler.used list.
-void appendToCompilerUsed(Module &M, ArrayRef<GlobalValue *> Values);
+LLVM_ABI void appendToCompilerUsed(Module &M, ArrayRef<GlobalValue *> Values);
 
 /// Removes global values from the llvm.used and llvm.compiler.used arrays. \p
 /// ShouldRemove should return true for any initializer field that should not be
 /// included in the replacement global.
-void removeFromUsedLists(Module &M,
-                         function_ref<bool(Constant *)> ShouldRemove);
+LLVM_ABI void removeFromUsedLists(Module &M,
+                                  function_ref<bool(Constant *)> ShouldRemove);
 
 /// Filter out potentially dead comdat functions where other entries keep the
 /// entire comdat group alive.
@@ -116,8 +121,8 @@ void removeFromUsedLists(Module &M,
 /// After this routine finishes, the only remaining `Function`s in \p
 /// DeadComdatFunctions are those where every member of the comdat is listed
 /// and thus removing them is safe (provided *all* are removed).
-void filterDeadComdatFunctions(
-    SmallVectorImpl<Function *> &DeadComdatFunctions);
+LLVM_ABI void
+filterDeadComdatFunctions(SmallVectorImpl<Function *> &DeadComdatFunctions);
 
 /// Produce a unique identifier for this module by taking the MD5 sum of
 /// the names of the module's strong external symbols that are not comdat
@@ -129,13 +134,14 @@ void filterDeadComdatFunctions(
 /// If the module has no strong external symbols (such a module may still have a
 /// semantic effect if it performs global initialization), we cannot produce a
 /// unique identifier for this module, so we return the empty string.
-std::string getUniqueModuleId(Module *M);
+LLVM_ABI std::string getUniqueModuleId(Module *M);
 
 /// Embed the memory buffer \p Buf into the module \p M as a global using the
 /// specified section name. Also provide a metadata entry to identify it in the
 /// module using the same section name.
-void embedBufferInModule(Module &M, MemoryBufferRef Buf, StringRef SectionName,
-                         Align Alignment = Align(1));
+LLVM_ABI void embedBufferInModule(Module &M, MemoryBufferRef Buf,
+                                  StringRef SectionName,
+                                  Align Alignment = Align(1));
 
 /// Lower all calls to ifuncs by replacing uses with indirect calls loaded out
 /// of a global table initialized in a global constructor. This will introduce
@@ -149,8 +155,9 @@ void embedBufferInModule(Module &M, MemoryBufferRef Buf, StringRef SectionName,
 ///
 /// The processed ifuncs without remaining users will be removed from the
 /// module.
-bool lowerGlobalIFuncUsersAsGlobalCtor(
-    Module &M, ArrayRef<GlobalIFunc *> IFuncsToLower = {});
+LLVM_ABI bool
+lowerGlobalIFuncUsersAsGlobalCtor(Module &M,
+                                  ArrayRef<GlobalIFunc *> IFuncsToLower = {});
 
 } // End llvm namespace
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
index 365b215c051f..670bfaa5ad6f 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
@@ -57,6 +57,7 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -102,7 +103,7 @@ class PredicateBase : public ilist_node<PredicateBase> {
   }
 
   /// Fetch condition in the form of PredicateConstraint, if possible.
-  std::optional<PredicateConstraint> getConstraint() const;
+  LLVM_ABI std::optional<PredicateConstraint> getConstraint() const;
 
 protected:
   PredicateBase(PredicateType PT, Value *Op, Value *Condition)
@@ -176,13 +177,13 @@ class PredicateSwitch : public PredicateWithEdge {
 /// accesses.
 class PredicateInfo {
 public:
-  PredicateInfo(Function &, DominatorTree &, AssumptionCache &);
-  ~PredicateInfo();
+  LLVM_ABI PredicateInfo(Function &, DominatorTree &, AssumptionCache &);
+  LLVM_ABI ~PredicateInfo();
 
-  void verifyPredicateInfo() const;
+  LLVM_ABI void verifyPredicateInfo() const;
 
-  void dump() const;
-  void print(raw_ostream &) const;
+  LLVM_ABI void dump() const;
+  LLVM_ABI void print(raw_ostream &) const;
 
   const PredicateBase *getPredicateInfoFor(const Value *V) const {
     return PredicateMap.lookup(V);
@@ -214,13 +215,13 @@ class PredicateInfoPrinterPass
 
 public:
   explicit PredicateInfoPrinterPass(raw_ostream &OS) : OS(OS) {}
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
 /// Verifier pass for \c PredicateInfo.
 struct PredicateInfoVerifierPass : PassInfoMixin<PredicateInfoVerifierPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
   static bool isRequired() { return true; }
 };
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/PromoteMemToReg.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/PromoteMemToReg.h
index f827ffd3e676..ab0bd3a5a996 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/PromoteMemToReg.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/PromoteMemToReg.h
@@ -27,7 +27,7 @@ class AssumptionCache;
 /// (transitively) using this alloca. This also enforces that there is only
 /// ever one layer of bitcasts or GEPs between the alloca and the lifetime
 /// markers.
-bool isAllocaPromotable(const AllocaInst *AI);
+LLVM_ABI bool isAllocaPromotable(const AllocaInst *AI);
 
 /// Promote the specified list of alloca instructions into scalar
 /// registers, inserting PHI nodes as appropriate.
@@ -36,8 +36,8 @@ bool isAllocaPromotable(const AllocaInst *AI);
 /// does not modify the CFG of the function at all.  All allocas must be from
 /// the same function.
 ///
-void PromoteMemToReg(ArrayRef<AllocaInst *> Allocas, DominatorTree &DT,
-                     AssumptionCache *AC = nullptr);
+LLVM_ABI void PromoteMemToReg(ArrayRef<AllocaInst *> Allocas, DominatorTree &DT,
+                              AssumptionCache *AC = nullptr);
 
 } // End llvm namespace
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SCCPSolver.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
index a5b56dcdc629..c8fe4301b2b4 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/PredicateInfo.h"
 #include <vector>
 
@@ -66,149 +67,155 @@ class SCCPSolver {
   std::unique_ptr<SCCPInstVisitor> Visitor;
 
 public:
+  LLVM_ABI
   SCCPSolver(const DataLayout &DL,
              std::function<const TargetLibraryInfo &(Function &)> GetTLI,
              LLVMContext &Ctx);
 
-  ~SCCPSolver();
+  LLVM_ABI ~SCCPSolver();
 
-  void addPredicateInfo(Function &F, DominatorTree &DT, AssumptionCache &AC);
+  LLVM_ABI void addPredicateInfo(Function &F, DominatorTree &DT,
+                                 AssumptionCache &AC);
 
   /// markBlockExecutable - This method can be used by clients to mark all of
   /// the blocks that are known to be intrinsically live in the processed unit.
   /// This returns true if the block was not considered live before.
-  bool markBlockExecutable(BasicBlock *BB);
+  LLVM_ABI bool markBlockExecutable(BasicBlock *BB);
 
-  const PredicateBase *getPredicateInfoFor(Instruction *I);
+  LLVM_ABI const PredicateBase *getPredicateInfoFor(Instruction *I);
 
   /// trackValueOfGlobalVariable - Clients can use this method to
   /// inform the SCCPSolver that it should track loads and stores to the
   /// specified global variable if it can.  This is only legal to call if
   /// performing Interprocedural SCCP.
-  void trackValueOfGlobalVariable(GlobalVariable *GV);
+  LLVM_ABI void trackValueOfGlobalVariable(GlobalVariable *GV);
 
   /// addTrackedFunction - If the SCCP solver is supposed to track calls into
   /// and out of the specified function (which cannot have its address taken),
   /// this method must be called.
-  void addTrackedFunction(Function *F);
+  LLVM_ABI void addTrackedFunction(Function *F);
 
   /// Add function to the list of functions whose return cannot be modified.
-  void addToMustPreserveReturnsInFunctions(Function *F);
+  LLVM_ABI void addToMustPreserveReturnsInFunctions(Function *F);
 
   /// Returns true if the return of the given function cannot be modified.
-  bool mustPreserveReturn(Function *F);
+  LLVM_ABI bool mustPreserveReturn(Function *F);
 
-  void addArgumentTrackedFunction(Function *F);
+  LLVM_ABI void addArgumentTrackedFunction(Function *F);
 
   /// Returns true if the given function is in the solver's set of
   /// argument-tracked functions.
-  bool isArgumentTrackedFunction(Function *F);
+  LLVM_ABI bool isArgumentTrackedFunction(Function *F);
 
-  const SmallPtrSetImpl<Function *> &getArgumentTrackedFunctions() const;
+  LLVM_ABI const SmallPtrSetImpl<Function *> &
+  getArgumentTrackedFunctions() const;
 
   /// Solve - Solve for constants and executable blocks.
-  void solve();
+  LLVM_ABI void solve();
 
   /// resolvedUndefsIn - While solving the dataflow for a function, we assume
   /// that branches on undef values cannot reach any of their successors.
   /// However, this is not a safe assumption.  After we solve dataflow, this
   /// method should be use to handle this.  If this returns true, the solver
   /// should be rerun.
-  bool resolvedUndefsIn(Function &F);
+  LLVM_ABI bool resolvedUndefsIn(Function &F);
 
-  void solveWhileResolvedUndefsIn(Module &M);
+  LLVM_ABI void solveWhileResolvedUndefsIn(Module &M);
 
-  void solveWhileResolvedUndefsIn(SmallVectorImpl<Function *> &WorkList);
+  LLVM_ABI void
+  solveWhileResolvedUndefsIn(SmallVectorImpl<Function *> &WorkList);
 
-  void solveWhileResolvedUndefs();
+  LLVM_ABI void solveWhileResolvedUndefs();
 
-  bool isBlockExecutable(BasicBlock *BB) const;
+  LLVM_ABI bool isBlockExecutable(BasicBlock *BB) const;
 
   // isEdgeFeasible - Return true if the control flow edge from the 'From' basic
   // block to the 'To' basic block is currently feasible.
-  bool isEdgeFeasible(BasicBlock *From, BasicBlock *To) const;
+  LLVM_ABI bool isEdgeFeasible(BasicBlock *From, BasicBlock *To) const;
 
-  std::vector<ValueLatticeElement> getStructLatticeValueFor(Value *V) const;
+  LLVM_ABI std::vector<ValueLatticeElement>
+  getStructLatticeValueFor(Value *V) const;
 
-  void removeLatticeValueFor(Value *V);
+  LLVM_ABI void removeLatticeValueFor(Value *V);
 
   /// Invalidate the Lattice Value of \p Call and its users after specializing
   /// the call. Then recompute it.
-  void resetLatticeValueFor(CallBase *Call);
+  LLVM_ABI void resetLatticeValueFor(CallBase *Call);
 
-  const ValueLatticeElement &getLatticeValueFor(Value *V) const;
+  LLVM_ABI const ValueLatticeElement &getLatticeValueFor(Value *V) const;
 
   /// getTrackedRetVals - Get the inferred return value map.
-  const MapVector<Function *, ValueLatticeElement> &getTrackedRetVals() const;
+  LLVM_ABI const MapVector<Function *, ValueLatticeElement> &
+  getTrackedRetVals() const;
 
   /// getTrackedGlobals - Get and return the set of inferred initializers for
   /// global variables.
-  const DenseMap<GlobalVariable *, ValueLatticeElement> &
+  LLVM_ABI const DenseMap<GlobalVariable *, ValueLatticeElement> &
   getTrackedGlobals() const;
 
   /// getMRVFunctionsTracked - Get the set of functions which return multiple
   /// values tracked by the pass.
-  const SmallPtrSet<Function *, 16> &getMRVFunctionsTracked() const;
+  LLVM_ABI const SmallPtrSet<Function *, 16> &getMRVFunctionsTracked() const;
 
   /// markOverdefined - Mark the specified value overdefined.  This
   /// works with both scalars and structs.
-  void markOverdefined(Value *V);
+  LLVM_ABI void markOverdefined(Value *V);
 
   /// trackValueOfArgument - Mark the specified argument overdefined unless it
   /// have range attribute.  This works with both scalars and structs.
-  void trackValueOfArgument(Argument *V);
+  LLVM_ABI void trackValueOfArgument(Argument *V);
 
   // isStructLatticeConstant - Return true if all the lattice values
   // corresponding to elements of the structure are constants,
   // false otherwise.
-  bool isStructLatticeConstant(Function *F, StructType *STy);
+  LLVM_ABI bool isStructLatticeConstant(Function *F, StructType *STy);
 
   /// Helper to return a Constant if \p LV is either a constant or a constant
   /// range with a single element.
-  Constant *getConstant(const ValueLatticeElement &LV, Type *Ty) const;
+  LLVM_ABI Constant *getConstant(const ValueLatticeElement &LV, Type *Ty) const;
 
   /// Return either a Constant or nullptr for a given Value.
-  Constant *getConstantOrNull(Value *V) const;
+  LLVM_ABI Constant *getConstantOrNull(Value *V) const;
 
   /// Set the Lattice Value for the arguments of a specialization \p F.
   /// If an argument is Constant then its lattice value is marked with the
   /// corresponding actual argument in \p Args. Otherwise, its lattice value
   /// is inherited (copied) from the corresponding formal argument in \p Args.
-  void setLatticeValueForSpecializationArguments(Function *F,
-                                       const SmallVectorImpl<ArgInfo> &Args);
+  LLVM_ABI void setLatticeValueForSpecializationArguments(
+      Function *F, const SmallVectorImpl<ArgInfo> &Args);
 
   /// Mark all of the blocks in function \p F non-executable. Clients can used
   /// this method to erase a function from the module (e.g., if it has been
   /// completely specialized and is no longer needed).
-  void markFunctionUnreachable(Function *F);
+  LLVM_ABI void markFunctionUnreachable(Function *F);
 
-  void visit(Instruction *I);
-  void visitCall(CallInst &I);
+  LLVM_ABI void visit(Instruction *I);
+  LLVM_ABI void visitCall(CallInst &I);
 
-  bool simplifyInstsInBlock(BasicBlock &BB,
-                            SmallPtrSetImpl<Value *> &InsertedValues,
-                            Statistic &InstRemovedStat,
-                            Statistic &InstReplacedStat);
+  LLVM_ABI bool simplifyInstsInBlock(BasicBlock &BB,
+                                     SmallPtrSetImpl<Value *> &InsertedValues,
+                                     Statistic &InstRemovedStat,
+                                     Statistic &InstReplacedStat);
 
-  bool removeNonFeasibleEdges(BasicBlock *BB, DomTreeUpdater &DTU,
-                              BasicBlock *&NewUnreachableBB) const;
+  LLVM_ABI bool removeNonFeasibleEdges(BasicBlock *BB, DomTreeUpdater &DTU,
+                                       BasicBlock *&NewUnreachableBB) const;
 
-  void inferReturnAttributes() const;
-  void inferArgAttributes() const;
+  LLVM_ABI void inferReturnAttributes() const;
+  LLVM_ABI void inferArgAttributes() const;
 
-  bool tryToReplaceWithConstant(Value *V);
+  LLVM_ABI bool tryToReplaceWithConstant(Value *V);
 
   // Helper to check if \p LV is either a constant or a constant
   // range with a single element. This should cover exactly the same cases as
   // the old ValueLatticeElement::isConstant() and is intended to be used in the
   // transition to ValueLatticeElement.
-  static bool isConstant(const ValueLatticeElement &LV);
+  LLVM_ABI static bool isConstant(const ValueLatticeElement &LV);
 
   // Helper to check if \p LV is either overdefined or a constant range with
   // more than a single element. This should cover exactly the same cases as the
   // old ValueLatticeElement::isOverdefined() and is intended to be used in the
   // transition to ValueLatticeElement.
-  static bool isOverdefined(const ValueLatticeElement &LV);
+  LLVM_ABI static bool isOverdefined(const ValueLatticeElement &LV);
 };
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SSAUpdaterBulk.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SSAUpdaterBulk.h
index b2cf29608f58..48e8c86f7d87 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SSAUpdaterBulk.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SSAUpdaterBulk.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/PredIteratorCache.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -58,15 +59,15 @@ class SSAUpdaterBulk {
   /// Add a new variable to the SSA rewriter. This needs to be called before
   /// AddAvailableValue or AddUse calls. The return value is the variable ID,
   /// which needs to be passed to AddAvailableValue and AddUse.
-  unsigned AddVariable(StringRef Name, Type *Ty);
+  LLVM_ABI unsigned AddVariable(StringRef Name, Type *Ty);
 
   /// Indicate that a rewritten value is available in the specified block with
   /// the specified value.
-  void AddAvailableValue(unsigned Var, BasicBlock *BB, Value *V);
+  LLVM_ABI void AddAvailableValue(unsigned Var, BasicBlock *BB, Value *V);
 
   /// Record a use of the symbolic value. This use will be updated with a
   /// rewritten value when RewriteAllUses is called.
-  void AddUse(unsigned Var, Use *U);
+  LLVM_ABI void AddUse(unsigned Var, Use *U);
 
   /// Perform all the necessary updates, including new PHI-nodes insertion and
   /// the requested uses update.
@@ -75,8 +76,9 @@ class SSAUpdaterBulk {
   /// locations for new phi-nodes insertions. If a nonnull pointer to a vector
   /// InsertedPHIs is passed, all the new phi-nodes will be added to this
   /// vector.
-  void RewriteAllUses(DominatorTree *DT,
-                      SmallVectorImpl<PHINode *> *InsertedPHIs = nullptr);
+  LLVM_ABI void
+  RewriteAllUses(DominatorTree *DT,
+                 SmallVectorImpl<PHINode *> *InsertedPHIs = nullptr);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SanitizerStats.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SanitizerStats.h
index 14e8ae045cdd..05539c724d3e 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SanitizerStats.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SanitizerStats.h
@@ -14,6 +14,7 @@
 #define LLVM_TRANSFORMS_UTILS_SANITIZERSTATS_H
 
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -30,14 +31,14 @@ enum SanitizerStatKind {
 };
 
 struct SanitizerStatReport {
-  SanitizerStatReport(Module *M);
+  LLVM_ABI SanitizerStatReport(Module *M);
 
   /// Generates code into B that increments a location-specific counter tagged
   /// with the given sanitizer kind SK.
-  void create(IRBuilder<> &B, SanitizerStatKind SK);
+  LLVM_ABI void create(IRBuilder<> &B, SanitizerStatKind SK);
 
   /// Finalize module stats array and add global constructor to register it.
-  void finish();
+  LLVM_ABI void finish();
 
 private:
   Module *M;
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 7dd754a2bc0d..a101151eed7c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -23,10 +23,11 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/InstructionCost.h"
 
 namespace llvm {
-extern cl::opt<unsigned> SCEVCheapExpansionBudget;
+LLVM_ABI extern cl::opt<unsigned> SCEVCheapExpansionBudget;
 
 /// struct for holding enough information to help calculate the cost of the
 /// given SCEV when expanded into IR.
@@ -50,8 +51,8 @@ struct PoisonFlags {
   unsigned SameSign : 1;
   GEPNoWrapFlags GEPNW;
 
-  PoisonFlags(const Instruction *I);
-  void apply(Instruction *I);
+  LLVM_ABI PoisonFlags(const Instruction *I);
+  LLVM_ABI void apply(Instruction *I);
 };
 
 /// This class uses information about analyze scalars to rewrite expressions
@@ -271,8 +272,8 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   }
 
   /// Return the induction variable increment's IV operand.
-  Instruction *getIVIncOperand(Instruction *IncV, Instruction *InsertPos,
-                               bool allowScale);
+  LLVM_ABI Instruction *
+  getIVIncOperand(Instruction *IncV, Instruction *InsertPos, bool allowScale);
 
   /// Utility for hoisting \p IncV (with all subexpressions requried for its
   /// computation) before \p InsertPos. If \p RecomputePoisonFlags is set, drops
@@ -280,36 +281,40 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   /// re-infer them in the new location. It should be used when we are going to
   /// introduce a new use in the new position that didn't exist before, and may
   /// trigger new UB in case of poison.
-  bool hoistIVInc(Instruction *IncV, Instruction *InsertPos,
-                  bool RecomputePoisonFlags = false);
+  LLVM_ABI bool hoistIVInc(Instruction *IncV, Instruction *InsertPos,
+                           bool RecomputePoisonFlags = false);
 
   /// Return true if both increments directly increment the corresponding IV PHI
   /// nodes and have the same opcode. It is not safe to re-use the flags from
   /// the original increment, if it is more complex and SCEV expansion may have
   /// yielded a more simplified wider increment.
-  static bool canReuseFlagsFromOriginalIVInc(PHINode *OrigPhi, PHINode *WidePhi,
-                                             Instruction *OrigInc,
-                                             Instruction *WideInc);
+  LLVM_ABI static bool canReuseFlagsFromOriginalIVInc(PHINode *OrigPhi,
+                                                      PHINode *WidePhi,
+                                                      Instruction *OrigInc,
+                                                      Instruction *WideInc);
 
   /// replace congruent phis with their most canonical representative. Return
   /// the number of phis eliminated.
-  unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT,
-                               SmallVectorImpl<WeakTrackingVH> &DeadInsts,
-                               const TargetTransformInfo *TTI = nullptr);
+  LLVM_ABI unsigned
+  replaceCongruentIVs(Loop *L, const DominatorTree *DT,
+                      SmallVectorImpl<WeakTrackingVH> &DeadInsts,
+                      const TargetTransformInfo *TTI = nullptr);
 
   /// Return true if the given expression is safe to expand in the sense that
   /// all materialized values are safe to speculate anywhere their operands are
   /// defined, and the expander is capable of expanding the expression.
-  bool isSafeToExpand(const SCEV *S) const;
+  LLVM_ABI bool isSafeToExpand(const SCEV *S) const;
 
   /// Return true if the given expression is safe to expand in the sense that
   /// all materialized values are defined and safe to speculate at the specified
   /// location and their operands are defined at this location.
-  bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint) const;
+  LLVM_ABI bool isSafeToExpandAt(const SCEV *S,
+                                 const Instruction *InsertionPoint) const;
 
   /// Insert code to directly compute the specified SCEV expression into the
   /// program.  The code is inserted into the specified block.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty, BasicBlock::iterator I);
+  LLVM_ABI Value *expandCodeFor(const SCEV *SH, Type *Ty,
+                                BasicBlock::iterator I);
   Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I) {
     return expandCodeFor(SH, Ty, I->getIterator());
   }
@@ -318,29 +323,32 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   /// program.  The code is inserted into the SCEVExpander's current
   /// insertion point. If a type is specified, the result will be expanded to
   /// have that type, with a cast if necessary.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr);
+  LLVM_ABI Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr);
 
   /// Generates a code sequence that evaluates this predicate.  The inserted
   /// instructions will be at position \p Loc.  The result will be of type i1
   /// and will have a value of 0 when the predicate is false and 1 otherwise.
-  Value *expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc);
+  LLVM_ABI Value *expandCodeForPredicate(const SCEVPredicate *Pred,
+                                         Instruction *Loc);
 
   /// A specialized variant of expandCodeForPredicate, handling the case when
   /// we are expanding code for a SCEVComparePredicate.
-  Value *expandComparePredicate(const SCEVComparePredicate *Pred,
-                                Instruction *Loc);
+  LLVM_ABI Value *expandComparePredicate(const SCEVComparePredicate *Pred,
+                                         Instruction *Loc);
 
   /// Generates code that evaluates if the \p AR expression will overflow.
-  Value *generateOverflowCheck(const SCEVAddRecExpr *AR, Instruction *Loc,
-                               bool Signed);
+  LLVM_ABI Value *generateOverflowCheck(const SCEVAddRecExpr *AR,
+                                        Instruction *Loc, bool Signed);
 
   /// A specialized variant of expandCodeForPredicate, handling the case when
   /// we are expanding code for a SCEVWrapPredicate.
-  Value *expandWrapPredicate(const SCEVWrapPredicate *P, Instruction *Loc);
+  LLVM_ABI Value *expandWrapPredicate(const SCEVWrapPredicate *P,
+                                      Instruction *Loc);
 
   /// A specialized variant of expandCodeForPredicate, handling the case when
   /// we are expanding code for a SCEVUnionPredicate.
-  Value *expandUnionPredicate(const SCEVUnionPredicate *Pred, Instruction *Loc);
+  LLVM_ABI Value *expandUnionPredicate(const SCEVUnionPredicate *Pred,
+                                       Instruction *Loc);
 
   /// Set the current IV increment loop and position.
   void setIVIncInsertPos(const Loop *L, Instruction *Pos) {
@@ -417,24 +425,24 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   ///
   /// Note that this function does not perform an exhaustive search. I.e if it
   /// didn't find any value it does not mean that there is no such value.
-  bool hasRelatedExistingExpansion(const SCEV *S, const Instruction *At,
-                                   Loop *L);
+  LLVM_ABI bool hasRelatedExistingExpansion(const SCEV *S,
+                                            const Instruction *At, Loop *L);
 
   /// Returns a suitable insert point after \p I, that dominates \p
   /// MustDominate. Skips instructions inserted by the expander.
-  BasicBlock::iterator findInsertPointAfter(Instruction *I,
-                                            Instruction *MustDominate) const;
+  LLVM_ABI BasicBlock::iterator
+  findInsertPointAfter(Instruction *I, Instruction *MustDominate) const;
 
 private:
   LLVMContext &getContext() const { return SE.getContext(); }
 
   /// Recursive helper function for isHighCostExpansion.
-  bool isHighCostExpansionHelper(const SCEVOperand &WorkItem, Loop *L,
-                                 const Instruction &At, InstructionCost &Cost,
-                                 unsigned Budget,
-                                 const TargetTransformInfo &TTI,
-                                 SmallPtrSetImpl<const SCEV *> &Processed,
-                                 SmallVectorImpl<SCEVOperand> &Worklist);
+  LLVM_ABI bool
+  isHighCostExpansionHelper(const SCEVOperand &WorkItem, Loop *L,
+                            const Instruction &At, InstructionCost &Cost,
+                            unsigned Budget, const TargetTransformInfo &TTI,
+                            SmallPtrSetImpl<const SCEV *> &Processed,
+                            SmallVectorImpl<SCEVOperand> &Worklist);
 
   /// Insert the specified binary operator, doing a small amount of work to
   /// avoid inserting an obviously redundant operation, and hoisting to an
@@ -466,7 +474,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
       const SCEV *S, const Instruction *InsertPt,
       SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts);
 
-  Value *expand(const SCEV *S);
+  LLVM_ABI Value *expand(const SCEV *S);
   Value *expand(const SCEV *S, BasicBlock::iterator I) {
     setInsertPoint(I);
     return expand(S);
@@ -514,7 +522,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 
   Value *visitUnknown(const SCEVUnknown *S) { return S->getValue(); }
 
-  void rememberInstruction(Value *I);
+  LLVM_ABI void rememberInstruction(Value *I);
 
   void rememberFlags(Instruction *I);
 
@@ -561,7 +569,7 @@ class SCEVExpanderCleaner {
   /// Indicate that the result of the expansion is used.
   void markResultUsed() { ResultUsed = true; }
 
-  void cleanup();
+  LLVM_ABI void cleanup();
 };
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SizeOpts.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SizeOpts.h
index a9e72768f81e..595ee91e5ff0 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SizeOpts.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SizeOpts.h
@@ -15,17 +15,18 @@
 
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
-extern cl::opt<bool> EnablePGSO;
-extern cl::opt<bool> PGSOLargeWorkingSetSizeOnly;
-extern cl::opt<bool> PGSOColdCodeOnly;
-extern cl::opt<bool> PGSOColdCodeOnlyForInstrPGO;
-extern cl::opt<bool> PGSOColdCodeOnlyForSamplePGO;
-extern cl::opt<bool> PGSOColdCodeOnlyForPartialSamplePGO;
-extern cl::opt<bool> ForcePGSO;
-extern cl::opt<int> PgsoCutoffInstrProf;
-extern cl::opt<int> PgsoCutoffSampleProf;
+LLVM_ABI extern cl::opt<bool> EnablePGSO;
+LLVM_ABI extern cl::opt<bool> PGSOLargeWorkingSetSizeOnly;
+LLVM_ABI extern cl::opt<bool> PGSOColdCodeOnly;
+LLVM_ABI extern cl::opt<bool> PGSOColdCodeOnlyForInstrPGO;
+LLVM_ABI extern cl::opt<bool> PGSOColdCodeOnlyForSamplePGO;
+LLVM_ABI extern cl::opt<bool> PGSOColdCodeOnlyForPartialSamplePGO;
+LLVM_ABI extern cl::opt<bool> ForcePGSO;
+LLVM_ABI extern cl::opt<int> PgsoCutoffInstrProf;
+LLVM_ABI extern cl::opt<int> PgsoCutoffSampleProf;
 
 class BasicBlock;
 class BlockFrequencyInfo;
@@ -90,15 +91,17 @@ bool shouldOptimizeForSizeImpl(BlockTOrBlockFreq BBOrBlockFreq,
 
 /// Returns true if function \p F is suggested to be size-optimized based on the
 /// profile.
-bool shouldOptimizeForSize(const Function *F, ProfileSummaryInfo *PSI,
-                           BlockFrequencyInfo *BFI,
-                           PGSOQueryType QueryType = PGSOQueryType::Other);
+LLVM_ABI bool
+shouldOptimizeForSize(const Function *F, ProfileSummaryInfo *PSI,
+                      BlockFrequencyInfo *BFI,
+                      PGSOQueryType QueryType = PGSOQueryType::Other);
 
 /// Returns true if basic block \p BB is suggested to be size-optimized based on
 /// the profile.
-bool shouldOptimizeForSize(const BasicBlock *BB, ProfileSummaryInfo *PSI,
-                           BlockFrequencyInfo *BFI,
-                           PGSOQueryType QueryType = PGSOQueryType::Other);
+LLVM_ABI bool
+shouldOptimizeForSize(const BasicBlock *BB, ProfileSummaryInfo *PSI,
+                      BlockFrequencyInfo *BFI,
+                      PGSOQueryType QueryType = PGSOQueryType::Other);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SplitModule.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SplitModule.h
index e7e8ee6279ac..74e8bf21294a 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SplitModule.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SplitModule.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_UTILS_SPLITMODULE_H
 
 #include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 namespace llvm {
@@ -35,10 +36,10 @@ class Module;
 ///   module.
 /// - Internal symbols defined in module-level inline asm should be visible to
 ///   each partition.
-void SplitModule(
-    Module &M, unsigned N,
-    function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback,
-    bool PreserveLocals = false, bool RoundRobin = false);
+LLVM_ABI void
+SplitModule(Module &M, unsigned N,
+            function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback,
+            bool PreserveLocals = false, bool RoundRobin = false);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h
index 6f05e95b2f5b..10c5c7c49771 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h
@@ -33,6 +33,7 @@
 #define LLVM_TRANSFORMS_UTILS_SYMBOLREWRITER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 #include <list>
 #include <memory>
 #include <string>
@@ -92,7 +93,8 @@ using RewriteDescriptorList = std::list<std::unique_ptr<RewriteDescriptor>>;
 
 class RewriteMapParser {
 public:
-  bool parse(const std::string &MapFile, RewriteDescriptorList *Descriptors);
+  LLVM_ABI bool parse(const std::string &MapFile,
+                      RewriteDescriptorList *Descriptors);
 
 private:
   bool parse(std::unique_ptr<MemoryBuffer> &MapFile, RewriteDescriptorList *DL);
@@ -121,13 +123,13 @@ class RewriteSymbolPass : public PassInfoMixin<RewriteSymbolPass> {
     Descriptors.splice(Descriptors.begin(), DL);
   }
 
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
   // Glue for old PM
-  bool runImpl(Module &M);
+  LLVM_ABI bool runImpl(Module &M);
 
 private:
-  void loadAndParseMapFiles();
+  LLVM_ABI void loadAndParseMapFiles();
 
   SymbolRewriter::RewriteDescriptorList Descriptors;
 };
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 6759afd8077e..765c613b04a4 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Analysis/CodeMetrics.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/InstructionCost.h"
 
 namespace llvm {
@@ -48,9 +49,10 @@ const char *const LLVMLoopUnrollFollowupRemainder =
     "llvm.loop.unroll.followup_remainder";
 /// @}
 
-const Loop* addClonedBlockToLoopInfo(BasicBlock *OriginalBB,
-                                     BasicBlock *ClonedBB, LoopInfo *LI,
-                                     NewLoopsMap &NewLoops);
+LLVM_ABI const Loop *addClonedBlockToLoopInfo(BasicBlock *OriginalBB,
+                                              BasicBlock *ClonedBB,
+                                              LoopInfo *LI,
+                                              NewLoopsMap &NewLoops);
 
 /// Represents the result of a \c UnrollLoop invocation.
 enum class LoopUnrollResult {
@@ -79,15 +81,16 @@ struct UnrollLoopOptions {
   bool RuntimeUnrollMultiExit = false;
 };
 
-LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
-                            ScalarEvolution *SE, DominatorTree *DT,
-                            AssumptionCache *AC,
-                            const llvm::TargetTransformInfo *TTI,
-                            OptimizationRemarkEmitter *ORE, bool PreserveLCSSA,
-                            Loop **RemainderLoop = nullptr,
-                            AAResults *AA = nullptr);
+LLVM_ABI LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO,
+                                     LoopInfo *LI, ScalarEvolution *SE,
+                                     DominatorTree *DT, AssumptionCache *AC,
+                                     const llvm::TargetTransformInfo *TTI,
+                                     OptimizationRemarkEmitter *ORE,
+                                     bool PreserveLCSSA,
+                                     Loop **RemainderLoop = nullptr,
+                                     AAResults *AA = nullptr);
 
-bool UnrollRuntimeLoopRemainder(
+LLVM_ABI bool UnrollRuntimeLoopRemainder(
     Loop *L, unsigned Count, bool AllowExpensiveTripCount,
     bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
     LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
@@ -95,26 +98,25 @@ bool UnrollRuntimeLoopRemainder(
     unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit,
     Loop **ResultLoop = nullptr);
 
-LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
-                                  unsigned TripMultiple, bool UnrollRemainder,
-                                  LoopInfo *LI, ScalarEvolution *SE,
-                                  DominatorTree *DT, AssumptionCache *AC,
-                                  const TargetTransformInfo *TTI,
-                                  OptimizationRemarkEmitter *ORE,
-                                  Loop **EpilogueLoop = nullptr);
+LLVM_ABI LoopUnrollResult UnrollAndJamLoop(
+    Loop *L, unsigned Count, unsigned TripCount, unsigned TripMultiple,
+    bool UnrollRemainder, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
+    AssumptionCache *AC, const TargetTransformInfo *TTI,
+    OptimizationRemarkEmitter *ORE, Loop **EpilogueLoop = nullptr);
 
-bool isSafeToUnrollAndJam(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
-                          DependenceInfo &DI, LoopInfo &LI);
+LLVM_ABI bool isSafeToUnrollAndJam(Loop *L, ScalarEvolution &SE,
+                                   DominatorTree &DT, DependenceInfo &DI,
+                                   LoopInfo &LI);
 
-void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
-                             ScalarEvolution *SE, DominatorTree *DT,
-                             AssumptionCache *AC,
-                             const TargetTransformInfo *TTI,
-                             AAResults *AA = nullptr);
+LLVM_ABI void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
+                                      ScalarEvolution *SE, DominatorTree *DT,
+                                      AssumptionCache *AC,
+                                      const TargetTransformInfo *TTI,
+                                      AAResults *AA = nullptr);
 
-MDNode *GetUnrollMetadata(MDNode *LoopID, StringRef Name);
+LLVM_ABI MDNode *GetUnrollMetadata(MDNode *LoopID, StringRef Name);
 
-TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
+LLVM_ABI TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
     Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI,
     BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
     llvm::OptimizationRemarkEmitter &ORE, int OptLevel,
@@ -136,32 +138,30 @@ class UnrollCostEstimator {
   ConvergenceKind Convergence;
   bool ConvergenceAllowsRuntime;
 
-  UnrollCostEstimator(const Loop *L, const TargetTransformInfo &TTI,
-                      const SmallPtrSetImpl<const Value *> &EphValues,
-                      unsigned BEInsns);
+  LLVM_ABI UnrollCostEstimator(const Loop *L, const TargetTransformInfo &TTI,
+                               const SmallPtrSetImpl<const Value *> &EphValues,
+                               unsigned BEInsns);
 
   /// Whether it is legal to unroll this loop.
-  bool canUnroll() const;
+  LLVM_ABI bool canUnroll() const;
 
   uint64_t getRolledLoopSize() const { return LoopSize.getValue(); }
 
   /// Returns loop size estimation for unrolled loop, given the unrolling
   /// configuration specified by UP.
-  uint64_t
+  LLVM_ABI uint64_t
   getUnrolledLoopSize(const TargetTransformInfo::UnrollingPreferences &UP,
                       unsigned CountOverwrite = 0) const;
 };
 
-bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
-                        DominatorTree &DT, LoopInfo *LI, AssumptionCache *AC,
-                        ScalarEvolution &SE,
-                        const SmallPtrSetImpl<const Value *> &EphValues,
-                        OptimizationRemarkEmitter *ORE, unsigned TripCount,
-                        unsigned MaxTripCount, bool MaxOrZero,
-                        unsigned TripMultiple, const UnrollCostEstimator &UCE,
-                        TargetTransformInfo::UnrollingPreferences &UP,
-                        TargetTransformInfo::PeelingPreferences &PP,
-                        bool &UseUpperBound);
+LLVM_ABI bool computeUnrollCount(
+    Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI,
+    AssumptionCache *AC, ScalarEvolution &SE,
+    const SmallPtrSetImpl<const Value *> &EphValues,
+    OptimizationRemarkEmitter *ORE, unsigned TripCount, unsigned MaxTripCount,
+    bool MaxOrZero, unsigned TripMultiple, const UnrollCostEstimator &UCE,
+    TargetTransformInfo::UnrollingPreferences &UP,
+    TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound);
 
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ValueMapper.h b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ValueMapper.h
index ac7199687227..17b5d4b89123 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Utils/ValueMapper.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Utils/ValueMapper.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/simple_ilist.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/IR/ValueMap.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -41,7 +42,7 @@ using MetadataPredicate = std::function<bool(const Metadata *)>;
 
 /// This is a class that can be implemented by clients to remap types when
 /// cloning constants and instructions.
-class ValueMapTypeRemapper {
+class LLVM_ABI ValueMapTypeRemapper {
   virtual void anchor(); // Out of line method.
 
 public:
@@ -54,7 +55,7 @@ class ValueMapTypeRemapper {
 
 /// This is a class that can be implemented by clients to materialize Values on
 /// demand.
-class ValueMaterializer {
+class LLVM_ABI ValueMaterializer {
   virtual void anchor(); // Out of line method.
 
 protected:
@@ -163,21 +164,21 @@ class ValueMapper {
   void *pImpl;
 
 public:
-  ValueMapper(ValueToValueMapTy &VM, RemapFlags Flags = RF_None,
-              ValueMapTypeRemapper *TypeMapper = nullptr,
-              ValueMaterializer *Materializer = nullptr,
-              const MetadataPredicate *IdentityMD = nullptr);
+  LLVM_ABI ValueMapper(ValueToValueMapTy &VM, RemapFlags Flags = RF_None,
+                       ValueMapTypeRemapper *TypeMapper = nullptr,
+                       ValueMaterializer *Materializer = nullptr,
+                       const MetadataPredicate *IdentityMD = nullptr);
   ValueMapper(ValueMapper &&) = delete;
   ValueMapper(const ValueMapper &) = delete;
   ValueMapper &operator=(ValueMapper &&) = delete;
   ValueMapper &operator=(const ValueMapper &) = delete;
-  ~ValueMapper();
+  LLVM_ABI ~ValueMapper();
 
   /// Register an alternate mapping context.
   ///
   /// Returns a MappingContextID that can be used with the various schedule*()
   /// API to switch in a different value map on-the-fly.
-  unsigned
+  LLVM_ABI unsigned
   registerAlternateMappingContext(ValueToValueMapTy &VM,
                                   ValueMaterializer *Materializer = nullptr);
 
@@ -185,31 +186,34 @@ class ValueMapper {
   ///
   /// \note Like the top-level mapping functions, \a addFlags() must be called
   /// at the top level, not during a callback in a \a ValueMaterializer.
-  void addFlags(RemapFlags Flags);
-
-  Metadata *mapMetadata(const Metadata &MD);
-  MDNode *mapMDNode(const MDNode &N);
-
-  Value *mapValue(const Value &V);
-  Constant *mapConstant(const Constant &C);
-
-  void remapInstruction(Instruction &I);
-  void remapDbgRecord(Module *M, DbgRecord &V);
-  void remapDbgRecordRange(Module *M, iterator_range<DbgRecordIterator> Range);
-  void remapFunction(Function &F);
-  void remapGlobalObjectMetadata(GlobalObject &GO);
-
-  void scheduleMapGlobalInitializer(GlobalVariable &GV, Constant &Init,
-                                    unsigned MappingContextID = 0);
-  void scheduleMapAppendingVariable(GlobalVariable &GV, Constant *InitPrefix,
-                                    bool IsOldCtorDtor,
-                                    ArrayRef<Constant *> NewMembers,
-                                    unsigned MappingContextID = 0);
-  void scheduleMapGlobalAlias(GlobalAlias &GA, Constant &Aliasee,
-                              unsigned MappingContextID = 0);
-  void scheduleMapGlobalIFunc(GlobalIFunc &GI, Constant &Resolver,
-                              unsigned MappingContextID = 0);
-  void scheduleRemapFunction(Function &F, unsigned MappingContextID = 0);
+  LLVM_ABI void addFlags(RemapFlags Flags);
+
+  LLVM_ABI Metadata *mapMetadata(const Metadata &MD);
+  LLVM_ABI MDNode *mapMDNode(const MDNode &N);
+
+  LLVM_ABI Value *mapValue(const Value &V);
+  LLVM_ABI Constant *mapConstant(const Constant &C);
+
+  LLVM_ABI void remapInstruction(Instruction &I);
+  LLVM_ABI void remapDbgRecord(Module *M, DbgRecord &V);
+  LLVM_ABI void remapDbgRecordRange(Module *M,
+                                    iterator_range<DbgRecordIterator> Range);
+  LLVM_ABI void remapFunction(Function &F);
+  LLVM_ABI void remapGlobalObjectMetadata(GlobalObject &GO);
+
+  LLVM_ABI void scheduleMapGlobalInitializer(GlobalVariable &GV, Constant &Init,
+                                             unsigned MappingContextID = 0);
+  LLVM_ABI void scheduleMapAppendingVariable(GlobalVariable &GV,
+                                             Constant *InitPrefix,
+                                             bool IsOldCtorDtor,
+                                             ArrayRef<Constant *> NewMembers,
+                                             unsigned MappingContextID = 0);
+  LLVM_ABI void scheduleMapGlobalAlias(GlobalAlias &GA, Constant &Aliasee,
+                                       unsigned MappingContextID = 0);
+  LLVM_ABI void scheduleMapGlobalIFunc(GlobalIFunc &GI, Constant &Resolver,
+                                       unsigned MappingContextID = 0);
+  LLVM_ABI void scheduleRemapFunction(Function &F,
+                                      unsigned MappingContextID = 0);
 };
 
 /// Look up or compute a value in the value map.
@@ -295,7 +299,7 @@ inline void RemapInstruction(Instruction *I, ValueToValueMapTy &VM,
 /// instruction's atom group number if it has been mapped (e.g. with
 /// llvm::mapAtomInstance), which is necessary to distinguish source code
 /// atoms on duplicated code paths.
-void RemapSourceAtom(Instruction *I, ValueToValueMapTy &VM);
+LLVM_ABI void RemapSourceAtom(Instruction *I, ValueToValueMapTy &VM);
 
 /// Remap the Values used in the DbgRecord \a DR using the value map \a
 /// VM.
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h
index 15a46baa190d..d8f52b3707d4 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h
@@ -10,6 +10,7 @@
 #define LLVM_TRANSFORMS_VECTORIZE_LOADSTOREVECTORIZER_H
 
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 class Pass;
@@ -17,12 +18,11 @@ class Function;
 
 class LoadStoreVectorizerPass : public PassInfoMixin<LoadStoreVectorizerPass> {
 public:
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 /// Create a legacy pass manager instance of the LoadStoreVectorizer pass
-Pass *createLoadStoreVectorizerPass();
-
+LLVM_ABI Pass *createLoadStoreVectorizerPass();
 }
 
 #endif /* LLVM_TRANSFORMS_VECTORIZE_LOADSTOREVECTORIZER_H */
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index b4e5735eb143..db1971aca4bf 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -58,6 +58,7 @@
 
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Utils/ExtraPassManager.h"
 #include <functional>
 
@@ -78,8 +79,8 @@ class ScalarEvolution;
 class TargetLibraryInfo;
 class TargetTransformInfo;
 
-extern cl::opt<bool> EnableLoopInterleaving;
-extern cl::opt<bool> EnableLoopVectorization;
+LLVM_ABI extern cl::opt<bool> EnableLoopInterleaving;
+LLVM_ABI extern cl::opt<bool> EnableLoopVectorization;
 
 struct LoopVectorizeOptions {
   /// If false, consider all loops for interleaving.
@@ -138,7 +139,7 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> {
   bool VectorizeOnlyWhenForced;
 
 public:
-  LoopVectorizePass(LoopVectorizeOptions Opts = {});
+  LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts = {});
 
   ScalarEvolution *SE;
   LoopInfo *LI;
@@ -152,22 +153,23 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> {
   OptimizationRemarkEmitter *ORE;
   ProfileSummaryInfo *PSI;
 
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI void
+  printPipeline(raw_ostream &OS,
+                function_ref<StringRef(StringRef)> MapClassName2PassName);
 
   // Shim for old PM.
-  LoopVectorizeResult runImpl(Function &F);
+  LLVM_ABI LoopVectorizeResult runImpl(Function &F);
 
-  bool processLoop(Loop *L);
+  LLVM_ABI bool processLoop(Loop *L);
 };
 
 /// Reports a vectorization failure: print \p DebugMsg for debugging
 /// purposes along with the corresponding optimization remark \p RemarkName.
 /// If \p I is passed, it is an instruction that prevents vectorization.
 /// Otherwise, the loop \p TheLoop is used for the location of the remark.
-void reportVectorizationFailure(const StringRef DebugMsg,
-    const StringRef OREMsg, const StringRef ORETag,
+LLVM_ABI void reportVectorizationFailure(
+    const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag,
     OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I = nullptr);
 
 /// Same as above, but the debug message and optimization remark are identical
@@ -184,7 +186,7 @@ inline void reportVectorizationFailure(const StringRef DebugMsg,
 struct ShouldRunExtraVectorPasses
     : public ShouldRunExtraPasses<ShouldRunExtraVectorPasses>,
       public AnalysisInfoMixin<ShouldRunExtraVectorPasses> {
-  static AnalysisKey Key;
+  LLVM_ABI static AnalysisKey Key;
 };
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
index aff2068b2833..7d02d9a68310 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
@@ -27,6 +27,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/SandboxIR/Instruction.h"
 #include "llvm/SandboxIR/IntrinsicInst.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h"
 
 namespace llvm::sandboxir {
@@ -45,6 +46,9 @@ class DGNode;
 class MemDGNode;
 class DependencyGraph;
 
+// Defined in Transforms/Vectorize/SandboxVectorizer/Interval.cpp
+extern template class LLVM_TEMPLATE_ABI Interval<MemDGNode>;
+
 /// Iterate over both def-use and mem dependencies.
 class PredIterator {
   User::op_iterator OpIt;
@@ -65,9 +69,9 @@ class PredIterator {
 
   /// Skip iterators that don't point instructions or are outside \p DAG,
   /// starting from \p OpIt and ending before \p OpItE.n
-  static User::op_iterator skipBadIt(User::op_iterator OpIt,
-                                     User::op_iterator OpItE,
-                                     const DependencyGraph &DAG);
+  LLVM_ABI static User::op_iterator skipBadIt(User::op_iterator OpIt,
+                                              User::op_iterator OpItE,
+                                              const DependencyGraph &DAG);
 
 public:
   using difference_type = std::ptrdiff_t;
@@ -75,20 +79,20 @@ class PredIterator {
   using pointer = value_type *;
   using reference = value_type &;
   using iterator_category = std::input_iterator_tag;
-  value_type operator*();
-  PredIterator &operator++();
+  LLVM_ABI value_type operator*();
+  LLVM_ABI PredIterator &operator++();
   PredIterator operator++(int) {
     auto Copy = *this;
     ++(*this);
     return Copy;
   }
-  bool operator==(const PredIterator &Other) const;
+  LLVM_ABI bool operator==(const PredIterator &Other) const;
   bool operator!=(const PredIterator &Other) const { return !(*this == Other); }
 };
 
 /// A DependencyGraph Node that points to an Instruction and contains memory
 /// dependency edges.
-class DGNode {
+class LLVM_ABI DGNode {
 protected:
   Instruction *I;
   // TODO: Use a PointerIntPair for SubclassID and I.
@@ -310,17 +314,17 @@ class MemDGNodeIntervalBuilder {
 public:
   /// Scans the instruction chain in \p Intvl top-down, returning the top-most
   /// MemDGNode, or nullptr.
-  static MemDGNode *getTopMemDGNode(const Interval<Instruction> &Intvl,
-                                    const DependencyGraph &DAG);
+  LLVM_ABI static MemDGNode *getTopMemDGNode(const Interval<Instruction> &Intvl,
+                                             const DependencyGraph &DAG);
   /// Scans the instruction chain in \p Intvl bottom-up, returning the
   /// bottom-most MemDGNode, or nullptr.
-  static MemDGNode *getBotMemDGNode(const Interval<Instruction> &Intvl,
-                                    const DependencyGraph &DAG);
+  LLVM_ABI static MemDGNode *getBotMemDGNode(const Interval<Instruction> &Intvl,
+                                             const DependencyGraph &DAG);
   /// Given \p Instrs it finds their closest mem nodes in the interval and
   /// returns the corresponding mem range. Note: BotN (or its neighboring mem
   /// node) is included in the range.
-  static Interval<MemDGNode> make(const Interval<Instruction> &Instrs,
-                                  DependencyGraph &DAG);
+  LLVM_ABI static Interval<MemDGNode> make(const Interval<Instruction> &Instrs,
+                                           DependencyGraph &DAG);
   static Interval<MemDGNode> makeEmpty() { return {}; }
 };
 
@@ -383,15 +387,15 @@ class DependencyGraph {
                                MemDGNode *SkipN = nullptr) const;
 
   /// Called by the callbacks when a new instruction \p I has been created.
-  void notifyCreateInstr(Instruction *I);
+  LLVM_ABI void notifyCreateInstr(Instruction *I);
   /// Called by the callbacks when instruction \p I is about to get
   /// deleted.
-  void notifyEraseInstr(Instruction *I);
+  LLVM_ABI void notifyEraseInstr(Instruction *I);
   /// Called by the callbacks when instruction \p I is about to be moved to
   /// \p To.
-  void notifyMoveInstr(Instruction *I, const BBIterator &To);
+  LLVM_ABI void notifyMoveInstr(Instruction *I, const BBIterator &To);
   /// Called by the callbacks when \p U's source is about to be set to \p NewSrc
-  void notifySetUse(const Use &U, Value *NewSrc);
+  LLVM_ABI void notifySetUse(const Use &U, Value *NewSrc);
 
 public:
   /// This constructor also registers callbacks.
@@ -441,7 +445,7 @@ class DependencyGraph {
   }
   /// Build/extend the dependency graph such that it includes \p Instrs. Returns
   /// the range of instructions added to the DAG.
-  Interval<Instruction> extend(ArrayRef<Instruction *> Instrs);
+  LLVM_ABI Interval<Instruction> extend(ArrayRef<Instruction *> Instrs);
   /// \Returns the range of instructions included in the DAG.
   Interval<Instruction> getInterval() const { return DAGInterval; }
   void clear() {
@@ -460,7 +464,6 @@ class DependencyGraph {
   LLVM_DUMP_METHOD void dump() const;
 #endif // NDEBUG
 };
-
 } // namespace llvm::sandboxir
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_DEPENDENCYGRAPH_H
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
index f6c5a2046733..2524ed6d98e7 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
@@ -22,6 +22,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/SandboxIR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <iterator>
 #include <type_traits>
@@ -235,6 +236,9 @@ template <typename T> class Interval {
 #endif
 };
 
+// Defined in Transforms/Vectorize/SandboxVectorizer/Interval.cpp
+extern template class LLVM_TEMPLATE_ABI Interval<Instruction>;
+
 } // namespace llvm::sandboxir
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_INSTRINTERVAL_H
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index f74dcecb7e3e..96a234840393 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -16,6 +16,7 @@
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h"
@@ -347,13 +348,13 @@ class LegalityAnalysis {
   /// \Returns a LegalityResult object owned by LegalityAnalysis.
   /// \p SkipScheduling skips the scheduler check and is only meant for testing.
   // TODO: Try to remove the SkipScheduling argument by refactoring the tests.
-  const LegalityResult &canVectorize(ArrayRef<Value *> Bndl,
-                                     bool SkipScheduling = false);
+  LLVM_ABI const LegalityResult &canVectorize(ArrayRef<Value *> Bndl,
+                                              bool SkipScheduling = false);
   /// \Returns a Pack with reason 'ForcePackForDebugging'.
   const LegalityResult &getForcedPackForDebugging() {
     return createLegalityResult<Pack>(ResultReason::ForcePackForDebugging);
   }
-  void clear();
+  LLVM_ABI void clear();
 };
 
 } // namespace llvm::sandboxir
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index af0e07de4f51..c65e2cd45ab4 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -34,7 +34,6 @@ namespace llvm::sandboxir {
 /// transaction, depending on the cost.
 class BottomUpVec final : public RegionPass {
   bool Change = false;
-  std::unique_ptr<LegalityAnalysis> Legality;
   /// The original instructions that are potentially dead after vectorization.
   DenseSet<Instruction *> DeadInstrCandidates;
   /// Maps scalars to vectors.
@@ -88,12 +87,12 @@ class BottomUpVec final : public RegionPass {
   /// Recursively try to vectorize \p Bndl and its operands. This populates the
   /// `Actions` vector.
   Action *vectorizeRec(ArrayRef<Value *> Bndl, ArrayRef<Value *> UserBndl,
-                       unsigned Depth);
+                       unsigned Depth, LegalityAnalysis &Legality);
   /// Generate vector instructions based on `Actions` and return the last vector
   /// created.
   Value *emitVectors();
   /// Entry point for vectorization starting from \p Seeds.
-  bool tryVectorize(ArrayRef<Value *> Seeds);
+  bool tryVectorize(ArrayRef<Value *> Seeds, LegalityAnalysis &Legality);
 
 public:
   BottomUpVec() : RegionPass("bottom-up-vec") {}
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.h
new file mode 100644
index 000000000000..bd0bf244f6b1
--- /dev/null
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.h
@@ -0,0 +1,36 @@
+//===- PackReuse.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A pack de-duplication pass.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_PACKREUSE_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_PACKREUSE_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/Region.h"
+
+namespace llvm::sandboxir {
+
+/// This pass aims at de-duplicating packs, i.e., try to reuse already existing
+/// pack patterns instead of keeping both.
+/// This is useful because even though the duplicates will most probably be
+/// optimized away by future passes, their added cost can make vectorization
+/// more conservative than it should be.
+class PackReuse final : public RegionPass {
+  bool Change = false;
+
+public:
+  PackReuse() : RegionPass("pack-reuse") {}
+  bool runOnRegion(Region &Rgn, const Analyses &A) final;
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_PACKREUSE_H
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
index fea53329719b..fc8765217c0c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h
@@ -8,6 +8,7 @@
 #ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_SANDBOXVECTORIZER_H
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_SANDBOXVECTORIZER_H
 
+#include "llvm/Support/Compiler.h"
 #include <memory>
 
 #include "llvm/Analysis/AliasAnalysis.h"
@@ -49,11 +50,11 @@ class SandboxVectorizerPass : public PassInfoMixin<SandboxVectorizerPass> {
   // Vectorizer component can't find the vtable for classes like
   // sandboxir::Pass. This way we don't have to make LLVMPasses add a direct
   // dependency on SandboxIR.
-  SandboxVectorizerPass();
-  SandboxVectorizerPass(SandboxVectorizerPass &&);
-  ~SandboxVectorizerPass();
+  LLVM_ABI SandboxVectorizerPass();
+  LLVM_ABI SandboxVectorizerPass(SandboxVectorizerPass &&);
+  LLVM_ABI ~SandboxVectorizerPass();
 
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h
index 681cf881bce0..66ba91c58430 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Scheduler.h
@@ -22,6 +22,7 @@
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_SCHEDULER_H
 
 #include "llvm/SandboxIR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h"
 #include <queue>
 
@@ -143,11 +144,11 @@ class SchedBundle {
   const_iterator begin() const { return Nodes.begin(); }
   const_iterator end() const { return Nodes.end(); }
   /// \Returns the bundle node that comes before the others in program order.
-  DGNode *getTop() const;
+  LLVM_ABI DGNode *getTop() const;
   /// \Returns the bundle node that comes after the others in program order.
-  DGNode *getBot() const;
+  LLVM_ABI DGNode *getBot() const;
   /// Move all bundle instructions to \p Where back-to-back.
-  void cluster(BasicBlock::iterator Where);
+  LLVM_ABI void cluster(BasicBlock::iterator Where);
   /// \Returns true if all nodes in the bundle are ready.
   bool ready() const {
     return all_of(Nodes, [](const auto *N) { return N->ready(); });
@@ -182,7 +183,7 @@ class Scheduler {
   /// Called by Sandbox IR's callback system, after \p I has been created.
   /// NOTE: This should run after DAG's callback has run.
   // TODO: Perhaps call DAG's notify function from within this one?
-  void notifyCreateInstr(Instruction *I);
+  LLVM_ABI void notifyCreateInstr(Instruction *I);
 
   /// \Returns a scheduling bundle containing \p Instrs.
   SchedBundle *createBundle(ArrayRef<Instruction *> Instrs);
@@ -206,7 +207,8 @@ class Scheduler {
                     /// were in the same SchedBundle.
   };
   /// \Returns whether none/some/all of \p Instrs have been scheduled.
-  BndlSchedState getBndlSchedState(ArrayRef<Instruction *> Instrs) const;
+  LLVM_ABI BndlSchedState
+  getBndlSchedState(ArrayRef<Instruction *> Instrs) const;
   /// Destroy the top-most part of the schedule that includes \p Instrs.
   void trimSchedule(ArrayRef<Instruction *> Instrs);
   /// Disable copies.
@@ -229,7 +231,7 @@ class Scheduler {
   /// dependencies among \p Instrs. This function may involve scheduling
   /// intermediate instructions or canceling and re-scheduling if needed.
   /// \Returns true on success, false otherwise.
-  bool trySchedule(ArrayRef<Instruction *> Instrs);
+  LLVM_ABI bool trySchedule(ArrayRef<Instruction *> Instrs);
   /// Clear the scheduler's state, including the DAG.
   void clear() {
     Bndls.clear();
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
index 9c1374719298..6d2144b14bb0 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
@@ -18,6 +18,7 @@
 #include "llvm/SandboxIR/Instruction.h"
 #include "llvm/SandboxIR/Utils.h"
 #include "llvm/SandboxIR/Value.h"
+#include "llvm/Support/Compiler.h"
 #include <iterator>
 #include <memory>
 
@@ -67,7 +68,7 @@ class SeedBundle {
   /// the seeds in a bundle. This allows constant time evaluation
   /// and "removal" from the list.
   void setUsed(Instruction *I) {
-    auto It = std::find(begin(), end(), I);
+    auto It = llvm::find(*this, I);
     assert(It != end() && "Instruction not in the bundle!");
     auto Idx = It - begin();
     setUsed(Idx, 1, /*VerifyUnused=*/false);
@@ -95,8 +96,8 @@ class SeedBundle {
   /// with a total size <= \p MaxVecRegBits, or an empty slice if the
   /// requirements cannot be met . If \p ForcePowOf2 is true, then the returned
   /// slice will have a total number of bits that is a power of 2.
-  ArrayRef<Instruction *> getSlice(unsigned StartIdx, unsigned MaxVecRegBits,
-                                   bool ForcePowOf2);
+  LLVM_ABI ArrayRef<Instruction *>
+  getSlice(unsigned StartIdx, unsigned MaxVecRegBits, bool ForcePowOf2);
 
   /// \Returns the number of seed elements in the bundle.
   std::size_t size() const { return Seeds.size(); }
@@ -269,7 +270,7 @@ class SeedContainer {
   template <typename LoadOrStoreT> void insert(LoadOrStoreT *LSI);
   // To support constant-time erase, these just mark the element used, rather
   // than actually removing them from the bundle.
-  bool erase(Instruction *I);
+  LLVM_ABI bool erase(Instruction *I);
   bool erase(const KeyT &Key) { return Bundles.erase(Key); }
   iterator begin() {
     if (Bundles.empty())
@@ -288,6 +289,12 @@ class SeedContainer {
 #endif // NDEBUG
 };
 
+// Explicit instantiations
+extern template LLVM_TEMPLATE_ABI void
+SeedContainer::insert<LoadInst>(LoadInst *);
+extern template LLVM_TEMPLATE_ABI void
+SeedContainer::insert<StoreInst>(StoreInst *);
+
 class SeedCollector {
   SeedContainer StoreSeeds;
   SeedContainer LoadSeeds;
@@ -300,8 +307,8 @@ class SeedCollector {
   }
 
 public:
-  SeedCollector(BasicBlock *BB, ScalarEvolution &SE);
-  ~SeedCollector();
+  LLVM_ABI SeedCollector(BasicBlock *BB, ScalarEvolution &SE);
+  LLVM_ABI ~SeedCollector();
 
   iterator_range<SeedContainer::iterator> getStoreSeeds() {
     return {StoreSeeds.begin(), StoreSeeds.end()};
diff --git a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
index bec1cecf241f..d32bfbaf7a4c 100644
--- a/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
+++ b/external/llvm-project/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h
@@ -16,8 +16,27 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/SandboxIR/Type.h"
 #include "llvm/SandboxIR/Utils.h"
+#include "llvm/Support/Compiler.h"
 
-namespace llvm::sandboxir {
+namespace llvm {
+/// Traits for DenseMap.
+template <> struct DenseMapInfo<SmallVector<sandboxir::Value *>> {
+  static inline SmallVector<sandboxir::Value *> getEmptyKey() {
+    return SmallVector<sandboxir::Value *>({(sandboxir::Value *)-1});
+  }
+  static inline SmallVector<sandboxir::Value *> getTombstoneKey() {
+    return SmallVector<sandboxir::Value *>({(sandboxir::Value *)-2});
+  }
+  static unsigned getHashValue(const SmallVector<sandboxir::Value *> &Vec) {
+    return hash_combine_range(Vec);
+  }
+  static bool isEqual(const SmallVector<sandboxir::Value *> &Vec1,
+                      const SmallVector<sandboxir::Value *> &Vec2) {
+    return Vec1 == Vec2;
+  }
+};
+
+namespace sandboxir {
 
 class VecUtils {
 public:
@@ -177,7 +196,71 @@ class VecUtils {
     return ScalarTy;
   }
   /// \Returns the first integer power of 2 that is <= Num.
-  static unsigned getFloorPowerOf2(unsigned Num);
+  LLVM_ABI static unsigned getFloorPowerOf2(unsigned Num);
+
+  /// Helper struct for `matchPack()`. Describes the instructions and operands
+  /// of a pack pattern.
+  struct PackPattern {
+    /// The insertelement instructions that form the pack pattern in bottom-up
+    /// order, i.e., the first instruction in `Instrs` is the bottom-most
+    /// InsertElement instruction of the pack pattern.
+    /// For example in this simple pack pattern:
+    ///  %Pack0 = insertelement <2 x i8> poison, i8 %v0, i64 0
+    ///  %Pack1 = insertelement <2 x i8> %Pack0, i8 %v1, i64 1
+    /// this is [ %Pack1, %Pack0 ].
+    SmallVector<Instruction *> Instrs;
+    /// The "external" operands of the pack pattern, i.e., the values that get
+    /// packed into a vector, skipping the ones in `Instrs`. The operands are in
+    /// bottom-up order, starting from the operands of the bottom-most insert.
+    /// So in our example this would be [ %v1, %v0 ].
+    SmallVector<Value *> Operands;
+  };
+
+  /// If \p I is the last instruction of a pack pattern (i.e., an InsertElement
+  /// into a vector), then this function returns the instructions in the pack
+  /// and the operands in the pack, else returns nullopt.
+  /// Here is an example of a matched pattern:
+  ///  %PackA0 = insertelement <2 x i8> poison, i8 %v0, i64 0
+  ///  %PackA1 = insertelement <2 x i8> %PackA0, i8 %v1, i64 1
+  /// TODO: this currently detects only simple canonicalized patterns.
+  static std::optional<PackPattern> matchPack(Instruction *I) {
+    // TODO: Support vector pack patterns.
+    // TODO: Support out-of-order inserts.
+
+    // Early return if `I` is not an Insert.
+    if (!isa<InsertElementInst>(I))
+      return std::nullopt;
+    auto *BB0 = I->getParent();
+    // The pack contains as many instrs as the lanes of the bottom-most Insert
+    unsigned ExpectedNumInserts = VecUtils::getNumLanes(I);
+    assert(ExpectedNumInserts >= 2 && "Expected at least 2 inserts!");
+    PackPattern Pack;
+    Pack.Operands.resize(ExpectedNumInserts);
+    // Collect the inserts by walking up the use-def chain.
+    Instruction *InsertI = I;
+    for (auto ExpectedLane : reverse(seq<unsigned>(ExpectedNumInserts))) {
+      if (InsertI == nullptr)
+        return std::nullopt;
+      if (InsertI->getParent() != BB0)
+        return std::nullopt;
+      // Check the lane.
+      auto *LaneC = dyn_cast<ConstantInt>(InsertI->getOperand(2));
+      if (LaneC == nullptr || LaneC->getSExtValue() != ExpectedLane)
+        return std::nullopt;
+      Pack.Instrs.push_back(InsertI);
+      Pack.Operands[ExpectedLane] = InsertI->getOperand(1);
+
+      Value *Op = InsertI->getOperand(0);
+      if (ExpectedLane == 0) {
+        // Check the topmost insert. The operand should be a Poison.
+        if (!isa<PoisonValue>(Op))
+          return std::nullopt;
+      } else {
+        InsertI = dyn_cast<InsertElementInst>(Op);
+      }
+    }
+    return Pack;
+  }
 
 #ifndef NDEBUG
   /// Helper dump function for debugging.
@@ -186,6 +269,8 @@ class VecUtils {
 #endif // NDEBUG
 };
 
-} // namespace llvm::sandboxir
+} // namespace sandboxir
+
+} // namespace llvm
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
diff --git a/external/llvm-project/llvm/include/module.modulemap b/external/llvm-project/llvm/include/module.modulemap
index b37802345324..9d7dbb8e4b07 100644
--- a/external/llvm-project/llvm/include/module.modulemap
+++ b/external/llvm-project/llvm/include/module.modulemap
@@ -178,9 +178,10 @@ module LLVM_DebugInfo_CodeView {
   module * { export * }
 
   // These are intended for (repeated) textual inclusion.
+  textual header "llvm/DebugInfo/CodeView/CodeViewLanguages.def"
   textual header "llvm/DebugInfo/CodeView/CodeViewRegisters.def"
-  textual header "llvm/DebugInfo/CodeView/CodeViewTypes.def"
   textual header "llvm/DebugInfo/CodeView/CodeViewSymbols.def"
+  textual header "llvm/DebugInfo/CodeView/CodeViewTypes.def"
 }
 
 module LLVM_DWARFLinker {
diff --git a/external/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp b/external/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp
index 2afabb75c7cc..3ec009ca4add 100644
--- a/external/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -857,13 +857,13 @@ bool llvm::isEscapeSource(const Value *V) {
     return !CB->hasArgumentWithAdditionalReturnCaptureComponents();
   }
 
-  // The load case works because isNonEscapingLocalObject considers all
+  // The load case works because isNotCapturedBefore considers all
   // stores to be escapes (it passes true for the StoreCaptures argument
   // to PointerMayBeCaptured).
   if (isa<LoadInst>(V))
     return true;
 
-  // The inttoptr case works because isNonEscapingLocalObject considers all
+  // The inttoptr case works because isNotCapturedBefore considers all
   // means of converting or equating a pointer to an int (ptrtoint, ptr store
   // which could be followed by an integer load, ptr<->int compare) as
   // escaping, and objects located at well-known addresses via platform-specific
diff --git a/external/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/external/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index bcc9a71917aa..31611dfe4fd2 100644
--- a/external/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -79,10 +79,6 @@ STATISTIC(SearchLimitReached, "Number of times the limit to "
                               "decompose GEPs is reached");
 STATISTIC(SearchTimes, "Number of times a GEP is decomposed");
 
-// The max limit of the search depth in DecomposeGEPExpression() and
-// getUnderlyingObject().
-static const unsigned MaxLookupSearchDepth = 6;
-
 bool BasicAAResult::invalidate(Function &Fn, const PreservedAnalyses &PA,
                                FunctionAnalysisManager::Invalidator &Inv) {
   // We don't care if this analysis itself is preserved, it has no state. But
@@ -196,10 +192,22 @@ static bool areBothVScale(const Value *V1, const Value *V2) {
 
 CaptureAnalysis::~CaptureAnalysis() = default;
 
-bool SimpleCaptureAnalysis::isNotCapturedBefore(const Value *Object,
-                                                const Instruction *I,
-                                                bool OrAt) {
-  return isNonEscapingLocalObject(Object, &IsCapturedCache);
+CaptureComponents SimpleCaptureAnalysis::getCapturesBefore(const Value *Object,
+                                                           const Instruction *I,
+                                                           bool OrAt) {
+  if (!isIdentifiedFunctionLocal(Object))
+    return CaptureComponents::Provenance;
+
+  auto [CacheIt, Inserted] =
+      IsCapturedCache.insert({Object, CaptureComponents::Provenance});
+  if (!Inserted)
+    return CacheIt->second;
+
+  CaptureComponents Ret = PointerMayBeCaptured(
+      Object, /*ReturnCaptures=*/false, CaptureComponents::Provenance,
+      [](CaptureComponents CC) { return capturesFullProvenance(CC); });
+  CacheIt->second = Ret;
+  return Ret;
 }
 
 static bool isNotInCycle(const Instruction *I, const DominatorTree *DT,
@@ -210,37 +218,44 @@ static bool isNotInCycle(const Instruction *I, const DominatorTree *DT,
          !isPotentiallyReachableFromMany(Succs, BB, nullptr, DT, LI);
 }
 
-bool EarliestEscapeAnalysis::isNotCapturedBefore(const Value *Object,
-                                                 const Instruction *I,
-                                                 bool OrAt) {
+CaptureComponents
+EarliestEscapeAnalysis::getCapturesBefore(const Value *Object,
+                                          const Instruction *I, bool OrAt) {
   if (!isIdentifiedFunctionLocal(Object))
-    return false;
+    return CaptureComponents::Provenance;
 
   auto Iter = EarliestEscapes.try_emplace(Object);
   if (Iter.second) {
-    Instruction *EarliestCapture = FindEarliestCapture(
-        Object, *const_cast<Function *>(DT.getRoot()->getParent()),
-        /*ReturnCaptures=*/false, DT, CaptureComponents::Provenance);
-    if (EarliestCapture)
-      Inst2Obj[EarliestCapture].push_back(Object);
+    std::pair<Instruction *, CaptureComponents> EarliestCapture =
+        FindEarliestCapture(
+            Object, *const_cast<Function *>(DT.getRoot()->getParent()),
+            /*ReturnCaptures=*/false, DT, CaptureComponents::Provenance);
+    if (EarliestCapture.first)
+      Inst2Obj[EarliestCapture.first].push_back(Object);
     Iter.first->second = EarliestCapture;
   }
 
-  // No capturing instruction.
-  if (!Iter.first->second)
-    return true;
-
-  // No context instruction means any use is capturing.
-  if (!I)
-    return false;
+  auto IsNotCapturedBefore = [&]() {
+    // No capturing instruction.
+    Instruction *CaptureInst = Iter.first->second.first;
+    if (!CaptureInst)
+      return true;
 
-  if (I == Iter.first->second) {
-    if (OrAt)
+    // No context instruction means any use is capturing.
+    if (!I)
       return false;
-    return isNotInCycle(I, &DT, LI);
-  }
 
-  return !isPotentiallyReachable(Iter.first->second, I, nullptr, &DT, LI);
+    if (I == CaptureInst) {
+      if (OrAt)
+        return false;
+      return isNotInCycle(I, &DT, LI);
+    }
+
+    return !isPotentiallyReachable(CaptureInst, I, nullptr, &DT, LI);
+  };
+  if (IsNotCapturedBefore())
+    return CaptureComponents::None;
+  return Iter.first->second.second;
 }
 
 void EarliestEscapeAnalysis::removeInstruction(Instruction *I) {
@@ -940,9 +955,14 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
   // As an exception, ignore allocas, as setjmp is not required to preserve
   // non-volatile stores for them.
   if (isModOrRefSet(OtherMR) && !isa<Constant>(Object) && Call != Object &&
-      AAQI.CA->isNotCapturedBefore(Object, Call, /*OrAt=*/false) &&
-      (isa<AllocaInst>(Object) || !Call->hasFnAttr(Attribute::ReturnsTwice)))
-    OtherMR = ModRefInfo::NoModRef;
+      (isa<AllocaInst>(Object) || !Call->hasFnAttr(Attribute::ReturnsTwice))) {
+    CaptureComponents CC =
+        AAQI.CA->getCapturesBefore(Object, Call, /*OrAt=*/false);
+    if (capturesNothing(CC))
+      OtherMR = ModRefInfo::NoModRef;
+    else if (capturesReadProvenanceOnly(CC))
+      OtherMR = ModRefInfo::Ref;
+  }
 
   // Refine the modref info for argument memory. We only bother to do this
   // if ArgMR is not a subset of OtherMR, otherwise this won't have an impact
@@ -1261,8 +1281,7 @@ AliasResult BasicAAResult::aliasGEP(
 
     ConstantRange CR = computeConstantRange(Index.Val.V, /* ForSigned */ false,
                                             true, &AC, Index.CxtI);
-    KnownBits Known =
-        computeKnownBits(Index.Val.V, DL, 0, &AC, Index.CxtI, DT);
+    KnownBits Known = computeKnownBits(Index.Val.V, DL, &AC, Index.CxtI, DT);
     CR = CR.intersectWith(
         ConstantRange::fromKnownBits(Known, /* Signed */ true),
         ConstantRange::Signed);
@@ -1609,11 +1628,13 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size,
     // temporary store the nocapture argument's value in a temporary memory
     // location if that memory location doesn't escape. Or it may pass a
     // nocapture value to other functions as long as they don't capture it.
-    if (isEscapeSource(O1) && AAQI.CA->isNotCapturedBefore(
-                                  O2, dyn_cast<Instruction>(O1), /*OrAt*/ true))
+    if (isEscapeSource(O1) &&
+        capturesNothing(AAQI.CA->getCapturesBefore(
+            O2, dyn_cast<Instruction>(O1), /*OrAt*/ true)))
       return AliasResult::NoAlias;
-    if (isEscapeSource(O2) && AAQI.CA->isNotCapturedBefore(
-                                  O1, dyn_cast<Instruction>(O2), /*OrAt*/ true))
+    if (isEscapeSource(O2) &&
+        capturesNothing(AAQI.CA->getCapturesBefore(
+            O1, dyn_cast<Instruction>(O2), /*OrAt*/ true)))
       return AliasResult::NoAlias;
   }
 
diff --git a/external/llvm-project/llvm/lib/Analysis/CFG.cpp b/external/llvm-project/llvm/lib/Analysis/CFG.cpp
index 0d32e101ee0b..1676d6b6f592 100644
--- a/external/llvm-project/llvm/lib/Analysis/CFG.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/CFG.cpp
@@ -368,4 +368,4 @@ bool llvm::isPresplitCoroSuspendExitEdge(const BasicBlock &Src,
       return Intr->getIntrinsicID() == Intrinsic::coro_suspend &&
              SW->getDefaultDest() == &Dest;
   return false;
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/llvm/lib/Analysis/CMakeLists.txt b/external/llvm-project/llvm/lib/Analysis/CMakeLists.txt
index 8fe03d46946a..cfde787b1779 100644
--- a/external/llvm-project/llvm/lib/Analysis/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Analysis/CMakeLists.txt
@@ -78,6 +78,7 @@ add_llvm_component_library(LLVMAnalysis
   FunctionPropertiesAnalysis.cpp
   GlobalsModRef.cpp
   GuardUtils.cpp
+  HashRecognize.cpp
   HeatUtils.cpp
   IR2Vec.cpp
   IRSimilarityIdentifier.cpp
diff --git a/external/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp b/external/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp
index 2550b7a4732c..076f4176c021 100644
--- a/external/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp
@@ -249,11 +249,10 @@ bool llvm::PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures,
       capturesAnything, LI, MaxUsesToExplore));
 }
 
-Instruction *llvm::FindEarliestCapture(const Value *V, Function &F,
-                                       bool ReturnCaptures,
-                                       const DominatorTree &DT,
-                                       CaptureComponents Mask,
-                                       unsigned MaxUsesToExplore) {
+std::pair<Instruction *, CaptureComponents>
+llvm::FindEarliestCapture(const Value *V, Function &F, bool ReturnCaptures,
+                          const DominatorTree &DT, CaptureComponents Mask,
+                          unsigned MaxUsesToExplore) {
   assert(!isa<GlobalValue>(V) &&
          "It doesn't make sense to ask whether a global is captured.");
 
@@ -263,7 +262,7 @@ Instruction *llvm::FindEarliestCapture(const Value *V, Function &F,
     ++NumCapturedBefore;
   else
     ++NumNotCapturedBefore;
-  return CB.EarliestCapture;
+  return {CB.EarliestCapture, CB.CC};
 }
 
 UseCaptureInfo llvm::DetermineUseCaptureKind(const Use &U, const Value *Base) {
@@ -447,27 +446,3 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
 
   // All uses examined.
 }
-
-bool llvm::isNonEscapingLocalObject(
-    const Value *V, SmallDenseMap<const Value *, bool, 8> *IsCapturedCache) {
-  SmallDenseMap<const Value *, bool, 8>::iterator CacheIt;
-  if (IsCapturedCache) {
-    bool Inserted;
-    std::tie(CacheIt, Inserted) = IsCapturedCache->insert({V, false});
-    if (!Inserted)
-      // Found cached result, return it!
-      return CacheIt->second;
-  }
-
-  // If this is an identified function-local object, check to see if it escapes.
-  // We only care about provenance here, not address capture.
-  if (isIdentifiedFunctionLocal(V)) {
-    bool Ret = !capturesAnything(PointerMayBeCaptured(
-        V, /*ReturnCaptures=*/false, CaptureComponents::Provenance));
-    if (IsCapturedCache)
-      CacheIt->second = Ret;
-    return Ret;
-  }
-
-  return false;
-}
diff --git a/external/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp b/external/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp
index 7dd7f413783c..2b7a438a9ef0 100644
--- a/external/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1635,6 +1635,10 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::vector_reduce_smax:
   case Intrinsic::vector_reduce_umin:
   case Intrinsic::vector_reduce_umax:
+  case Intrinsic::vector_extract:
+  case Intrinsic::vector_insert:
+  case Intrinsic::vector_interleave2:
+  case Intrinsic::vector_deinterleave2:
   // Target intrinsics
   case Intrinsic::amdgcn_perm:
   case Intrinsic::amdgcn_wave_reduce_umin:
@@ -1672,6 +1676,7 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::sincos:
   case Intrinsic::sinh:
   case Intrinsic::cosh:
+  case Intrinsic::atan:
   case Intrinsic::pow:
   case Intrinsic::powi:
   case Intrinsic::ldexp:
@@ -2127,7 +2132,7 @@ static bool mayFoldConstrained(ConstrainedFPIntrinsic *CI,
 
   // If evaluation raised FP exception, the result can depend on rounding
   // mode. If the latter is unknown, folding is not possible.
-  if (ORM && *ORM == RoundingMode::Dynamic)
+  if (ORM == RoundingMode::Dynamic)
     return false;
 
   // If FP exceptions are ignored, fold the call, even if such exception is
@@ -2222,8 +2227,13 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
 
   if (isa<PoisonValue>(Operands[0])) {
     // TODO: All of these operations should probably propagate poison.
-    if (IntrinsicID == Intrinsic::canonicalize)
+    switch (IntrinsicID) {
+    case Intrinsic::canonicalize:
+    case Intrinsic::sqrt:
       return PoisonValue::get(Ty);
+    default:
+      break;
+    }
   }
 
   if (isa<UndefValue>(Operands[0])) {
@@ -2408,7 +2418,7 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
         if (IntrinsicID == Intrinsic::experimental_constrained_rint &&
             St == APFloat::opInexact) {
           std::optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
-          if (EB && *EB == fp::ebStrict)
+          if (EB == fp::ebStrict)
             return nullptr;
         }
       } else if (U.isSignaling()) {
@@ -2538,6 +2548,8 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
         return ConstantFoldFP(sinh, APF, Ty);
       case Intrinsic::cosh:
         return ConstantFoldFP(cosh, APF, Ty);
+      case Intrinsic::atan:
+        return ConstantFoldFP(atan, APF, Ty);
       case Intrinsic::sqrt:
         return ConstantFoldFP(sqrt, APF, Ty);
       case Intrinsic::amdgcn_cos:
@@ -3750,6 +3762,72 @@ static Constant *ConstantFoldFixedVectorCall(
     }
     return nullptr;
   }
+  case Intrinsic::vector_extract: {
+    auto *Idx = dyn_cast<ConstantInt>(Operands[1]);
+    Constant *Vec = Operands[0];
+    if (!Idx || !isa<FixedVectorType>(Vec->getType()))
+      return nullptr;
+
+    unsigned NumElements = FVTy->getNumElements();
+    unsigned VecNumElements =
+        cast<FixedVectorType>(Vec->getType())->getNumElements();
+    unsigned StartingIndex = Idx->getZExtValue();
+
+    // Extracting entire vector is nop
+    if (NumElements == VecNumElements && StartingIndex == 0)
+      return Vec;
+
+    for (unsigned I = StartingIndex, E = StartingIndex + NumElements; I < E;
+         ++I) {
+      Constant *Elt = Vec->getAggregateElement(I);
+      if (!Elt)
+        return nullptr;
+      Result[I - StartingIndex] = Elt;
+    }
+
+    return ConstantVector::get(Result);
+  }
+  case Intrinsic::vector_insert: {
+    Constant *Vec = Operands[0];
+    Constant *SubVec = Operands[1];
+    auto *Idx = dyn_cast<ConstantInt>(Operands[2]);
+    if (!Idx || !isa<FixedVectorType>(Vec->getType()))
+      return nullptr;
+
+    unsigned SubVecNumElements =
+        cast<FixedVectorType>(SubVec->getType())->getNumElements();
+    unsigned VecNumElements =
+        cast<FixedVectorType>(Vec->getType())->getNumElements();
+    unsigned IdxN = Idx->getZExtValue();
+    // Replacing entire vector with a subvec is nop
+    if (SubVecNumElements == VecNumElements && IdxN == 0)
+      return SubVec;
+
+    for (unsigned I = 0; I < VecNumElements; ++I) {
+      Constant *Elt;
+      if (I < IdxN + SubVecNumElements)
+        Elt = SubVec->getAggregateElement(I - IdxN);
+      else
+        Elt = Vec->getAggregateElement(I);
+      if (!Elt)
+        return nullptr;
+      Result[I] = Elt;
+    }
+    return ConstantVector::get(Result);
+  }
+  case Intrinsic::vector_interleave2: {
+    unsigned NumElements =
+        cast<FixedVectorType>(Operands[0]->getType())->getNumElements();
+    for (unsigned I = 0; I < NumElements; ++I) {
+      Constant *Elt0 = Operands[0]->getAggregateElement(I);
+      Constant *Elt1 = Operands[1]->getAggregateElement(I);
+      if (!Elt0 || !Elt1)
+        return nullptr;
+      Result[2 * I] = Elt0;
+      Result[2 * I + 1] = Elt1;
+    }
+    return ConstantVector::get(Result);
+  }
   default:
     break;
   }
@@ -3911,6 +3989,32 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
       return nullptr;
     return ConstantStruct::get(StTy, SinResult, CosResult);
   }
+  case Intrinsic::vector_deinterleave2: {
+    auto *Vec = Operands[0];
+    auto *VecTy = cast<VectorType>(Vec->getType());
+
+    if (auto *EltC = Vec->getSplatValue()) {
+      ElementCount HalfEC = VecTy->getElementCount().divideCoefficientBy(2);
+      auto *HalfVec = ConstantVector::getSplat(HalfEC, EltC);
+      return ConstantStruct::get(StTy, HalfVec, HalfVec);
+    }
+
+    if (!isa<FixedVectorType>(Vec->getType()))
+      return nullptr;
+
+    unsigned NumElements = VecTy->getElementCount().getFixedValue() / 2;
+    SmallVector<Constant *, 4> Res0(NumElements), Res1(NumElements);
+    for (unsigned I = 0; I < NumElements; ++I) {
+      Constant *Elt0 = Vec->getAggregateElement(2 * I);
+      Constant *Elt1 = Vec->getAggregateElement(2 * I + 1);
+      if (!Elt0 || !Elt1)
+        return nullptr;
+      Res0[I] = Elt0;
+      Res1[I] = Elt1;
+    }
+    return ConstantStruct::get(StTy, ConstantVector::get(Res0),
+                               ConstantVector::get(Res1));
+  }
   default:
     // TODO: Constant folding of vector intrinsics that fall through here does
     // not work (e.g. overflow intrinsics)
diff --git a/external/llvm-project/llvm/lib/Analysis/DemandedBits.cpp b/external/llvm-project/llvm/lib/Analysis/DemandedBits.cpp
index b538e16f2585..d7e2a3fa4fc5 100644
--- a/external/llvm-project/llvm/lib/Analysis/DemandedBits.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/DemandedBits.cpp
@@ -70,11 +70,11 @@ void DemandedBits::determineLiveOperandBits(
 
         const DataLayout &DL = UserI->getDataLayout();
         Known = KnownBits(BitWidth);
-        computeKnownBits(V1, Known, DL, 0, &AC, UserI, &DT);
+        computeKnownBits(V1, Known, DL, &AC, UserI, &DT);
 
         if (V2) {
           Known2 = KnownBits(BitWidth);
-          computeKnownBits(V2, Known2, DL, 0, &AC, UserI, &DT);
+          computeKnownBits(V2, Known2, DL, &AC, UserI, &DT);
         }
       };
 
diff --git a/external/llvm-project/llvm/lib/Analysis/HashRecognize.cpp b/external/llvm-project/llvm/lib/Analysis/HashRecognize.cpp
new file mode 100644
index 000000000000..1edb8b3bdc9a
--- /dev/null
+++ b/external/llvm-project/llvm/lib/Analysis/HashRecognize.cpp
@@ -0,0 +1,705 @@
+//===- HashRecognize.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The HashRecognize analysis recognizes unoptimized polynomial hash functions
+// with operations over a Galois field of characteristic 2, also called binary
+// fields, or GF(2^n): this class of hash functions can be optimized using a
+// lookup-table-driven implementation, or with target-specific instructions.
+// Examples:
+//
+//  1. Cyclic redundancy check (CRC), which is a polynomial division in GF(2).
+//  2. Rabin fingerprint, a component of the Rabin-Karp algorithm, which is a
+//     rolling hash polynomial division in GF(2).
+//  3. Rijndael MixColumns, a step in AES computation, which is a polynomial
+//     multiplication in GF(2^3).
+//  4. GHASH, the authentication mechanism in AES Galois/Counter Mode (GCM),
+//     which is a polynomial evaluation in GF(2^128).
+//
+// All of them use an irreducible generating polynomial of degree m,
+//
+//    c_m * x^m + c_(m-1) * x^(m-1) + ... + c_0 * x^0
+//
+// where each coefficient c is can take values in GF(2^n), where 2^n is termed
+// the order of the Galois field. For GF(2), each coefficient can take values
+// either 0 or 1, and the polynomial is simply represented by m+1 bits,
+// corresponding to the coefficients. The different variants of CRC are named by
+// degree of generating polynomial used: so CRC-32 would use a polynomial of
+// degree 32.
+//
+// The reason algorithms on GF(2^n) can be optimized with a lookup-table is the
+// following: in such fields, polynomial addition and subtraction are identical
+// and equivalent to XOR, polynomial multiplication is an AND, and polynomial
+// division is identity: the XOR and AND operations in unoptimized
+// implementations are performed bit-wise, and can be optimized to be performed
+// chunk-wise, by interleaving copies of the generating polynomial, and storing
+// the pre-computed values in a table.
+//
+// A generating polynomial of m bits always has the MSB set, so we usually
+// omit it. An example of a 16-bit polynomial is the CRC-16-CCITT polynomial:
+//
+//   (x^16) + x^12 + x^5 + 1 = (1) 0001 0000 0010 0001 = 0x1021
+//
+// Transmissions are either in big-endian or little-endian form, and hash
+// algorithms are written according to this. For example, IEEE 802 and RS-232
+// specify little-endian transmission.
+//
+//===----------------------------------------------------------------------===//
+//
+// At the moment, we only recognize the CRC algorithm.
+// Documentation on CRC32 from the kernel:
+// https://www.kernel.org/doc/Documentation/crc32.txt
+//
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/HashRecognize.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/Support/KnownBits.h"
+
+using namespace llvm;
+using namespace PatternMatch;
+using namespace SCEVPatternMatch;
+
+#define DEBUG_TYPE "hash-recognize"
+
+// KnownBits for a PHI node. There are at most two PHI nodes, corresponding to
+// the Simple Recurrence and Conditional Recurrence. The IndVar PHI is not
+// relevant.
+using KnownPhiMap = SmallDenseMap<const PHINode *, KnownBits, 2>;
+
+// A pair of a PHI node along with its incoming value from within a loop.
+using PhiStepPair = std::pair<const PHINode *, const Instruction *>;
+
+/// A much simpler version of ValueTracking, in that it computes KnownBits of
+/// values, except that it computes the evolution of KnownBits in a loop with a
+/// given trip count, and predication is specialized for a significant-bit
+/// check.
+class ValueEvolution {
+  const unsigned TripCount;
+  const bool ByteOrderSwapped;
+  APInt GenPoly;
+  StringRef ErrStr;
+
+  // Compute the KnownBits of a BinaryOperator.
+  KnownBits computeBinOp(const BinaryOperator *I);
+
+  // Compute the KnownBits of an Instruction.
+  KnownBits computeInstr(const Instruction *I);
+
+  // Compute the KnownBits of a Value.
+  KnownBits compute(const Value *V);
+
+public:
+  // ValueEvolution is meant to be constructed with the TripCount of the loop,
+  // and whether the polynomial algorithm is big-endian, for the significant-bit
+  // check.
+  ValueEvolution(unsigned TripCount, bool ByteOrderSwapped);
+
+  // Given a list of PHI nodes along with their incoming value from within the
+  // loop, computeEvolutions computes the KnownBits of each of the PHI nodes on
+  // the final iteration. Returns true on success and false on error.
+  bool computeEvolutions(ArrayRef<PhiStepPair> PhiEvolutions);
+
+  // In case ValueEvolution encounters an error, this is meant to be used for a
+  // precise error message.
+  StringRef getError() const { return ErrStr; }
+
+  // The computed KnownBits for each PHI node, which is populated after
+  // computeEvolutions is called.
+  KnownPhiMap KnownPhis;
+};
+
+ValueEvolution::ValueEvolution(unsigned TripCount, bool ByteOrderSwapped)
+    : TripCount(TripCount), ByteOrderSwapped(ByteOrderSwapped) {}
+
+KnownBits ValueEvolution::computeBinOp(const BinaryOperator *I) {
+  KnownBits KnownL(compute(I->getOperand(0)));
+  KnownBits KnownR(compute(I->getOperand(1)));
+
+  switch (I->getOpcode()) {
+  case Instruction::BinaryOps::And:
+    return KnownL & KnownR;
+  case Instruction::BinaryOps::Or:
+    return KnownL | KnownR;
+  case Instruction::BinaryOps::Xor:
+    return KnownL ^ KnownR;
+  case Instruction::BinaryOps::Shl: {
+    auto *OBO = cast<OverflowingBinaryOperator>(I);
+    return KnownBits::shl(KnownL, KnownR, OBO->hasNoUnsignedWrap(),
+                          OBO->hasNoSignedWrap());
+  }
+  case Instruction::BinaryOps::LShr:
+    return KnownBits::lshr(KnownL, KnownR);
+  case Instruction::BinaryOps::AShr:
+    return KnownBits::ashr(KnownL, KnownR);
+  case Instruction::BinaryOps::Add: {
+    auto *OBO = cast<OverflowingBinaryOperator>(I);
+    return KnownBits::add(KnownL, KnownR, OBO->hasNoUnsignedWrap(),
+                          OBO->hasNoSignedWrap());
+  }
+  case Instruction::BinaryOps::Sub: {
+    auto *OBO = cast<OverflowingBinaryOperator>(I);
+    return KnownBits::sub(KnownL, KnownR, OBO->hasNoUnsignedWrap(),
+                          OBO->hasNoSignedWrap());
+  }
+  case Instruction::BinaryOps::Mul: {
+    Value *Op0 = I->getOperand(0);
+    Value *Op1 = I->getOperand(1);
+    bool SelfMultiply = Op0 == Op1 && isGuaranteedNotToBeUndef(Op0);
+    return KnownBits::mul(KnownL, KnownR, SelfMultiply);
+  }
+  case Instruction::BinaryOps::UDiv:
+    return KnownBits::udiv(KnownL, KnownR);
+  case Instruction::BinaryOps::SDiv:
+    return KnownBits::sdiv(KnownL, KnownR);
+  case Instruction::BinaryOps::URem:
+    return KnownBits::urem(KnownL, KnownR);
+  case Instruction::BinaryOps::SRem:
+    return KnownBits::srem(KnownL, KnownR);
+  default:
+    ErrStr = "Unknown BinaryOperator";
+    unsigned BitWidth = I->getType()->getScalarSizeInBits();
+    return {BitWidth};
+  }
+}
+
+KnownBits ValueEvolution::computeInstr(const Instruction *I) {
+  unsigned BitWidth = I->getType()->getScalarSizeInBits();
+
+  // We look up in the map that contains the KnownBits of the PHI from the
+  // previous iteration.
+  if (const PHINode *P = dyn_cast<PHINode>(I))
+    return KnownPhis.lookup_or(P, BitWidth);
+
+  // Compute the KnownBits for a Select(Cmp()), forcing it to take the branch
+  // that is predicated on the (least|most)-significant-bit check.
+  CmpPredicate Pred;
+  Value *L, *R, *TV, *FV;
+  if (match(I, m_Select(m_ICmp(Pred, m_Value(L), m_Value(R)), m_Value(TV),
+                        m_Value(FV)))) {
+    // We need to check LCR against [0, 2) in the little-endian case, because
+    // the RCR check is insufficient: it is simply [0, 1).
+    if (!ByteOrderSwapped) {
+      KnownBits KnownL = compute(L);
+      unsigned ICmpBW = KnownL.getBitWidth();
+      auto LCR = ConstantRange::fromKnownBits(KnownL, false);
+      auto CheckLCR = ConstantRange(APInt::getZero(ICmpBW), APInt(ICmpBW, 2));
+      if (LCR != CheckLCR) {
+        ErrStr = "Bad LHS of significant-bit-check";
+        return {BitWidth};
+      }
+    }
+
+    // Check that the predication is on (most|least) significant bit.
+    KnownBits KnownR = compute(R);
+    unsigned ICmpBW = KnownR.getBitWidth();
+    auto RCR = ConstantRange::fromKnownBits(KnownR, false);
+    auto AllowedR = ConstantRange::makeAllowedICmpRegion(Pred, RCR);
+    ConstantRange CheckRCR(APInt::getZero(ICmpBW),
+                           ByteOrderSwapped ? APInt::getSignedMinValue(ICmpBW)
+                                            : APInt(ICmpBW, 1));
+    if (AllowedR == CheckRCR)
+      return compute(TV);
+    if (AllowedR.inverse() == CheckRCR)
+      return compute(FV);
+
+    ErrStr = "Bad RHS of significant-bit-check";
+    return {BitWidth};
+  }
+
+  if (auto *BO = dyn_cast<BinaryOperator>(I))
+    return computeBinOp(BO);
+
+  switch (I->getOpcode()) {
+  case Instruction::CastOps::Trunc:
+    return compute(I->getOperand(0)).trunc(BitWidth);
+  case Instruction::CastOps::ZExt:
+    return compute(I->getOperand(0)).zext(BitWidth);
+  case Instruction::CastOps::SExt:
+    return compute(I->getOperand(0)).sext(BitWidth);
+  default:
+    ErrStr = "Unknown Instruction";
+    return {BitWidth};
+  }
+}
+
+KnownBits ValueEvolution::compute(const Value *V) {
+  if (auto *CI = dyn_cast<ConstantInt>(V))
+    return KnownBits::makeConstant(CI->getValue());
+
+  if (auto *I = dyn_cast<Instruction>(V))
+    return computeInstr(I);
+
+  ErrStr = "Unknown Value";
+  unsigned BitWidth = V->getType()->getScalarSizeInBits();
+  return {BitWidth};
+}
+
+bool ValueEvolution::computeEvolutions(ArrayRef<PhiStepPair> PhiEvolutions) {
+  for (unsigned I = 0; I < TripCount; ++I) {
+    for (auto [Phi, Step] : PhiEvolutions) {
+      KnownBits KnownAtIter = computeInstr(Step);
+      if (KnownAtIter.getBitWidth() < I + 1) {
+        ErrStr = "Loop iterations exceed bitwidth of result";
+        return false;
+      }
+      KnownPhis.emplace_or_assign(Phi, KnownAtIter);
+    }
+  }
+  return ErrStr.empty();
+}
+
+/// A structure that can hold either a Simple Recurrence or a Conditional
+/// Recurrence. Note that in the case of a Simple Recurrence, Step is an operand
+/// of the BO, while in a Conditional Recurrence, it is a SelectInst.
+struct RecurrenceInfo {
+  const Loop &L;
+  const PHINode *Phi = nullptr;
+  BinaryOperator *BO = nullptr;
+  Value *Start = nullptr;
+  Value *Step = nullptr;
+  std::optional<APInt> ExtraConst;
+
+  RecurrenceInfo(const Loop &L) : L(L) {}
+  operator bool() const { return BO; }
+
+  void print(raw_ostream &OS, unsigned Indent = 0) const {
+    OS.indent(Indent) << "Phi: ";
+    Phi->print(OS);
+    OS << "\n";
+    OS.indent(Indent) << "BinaryOperator: ";
+    BO->print(OS);
+    OS << "\n";
+    OS.indent(Indent) << "Start: ";
+    Start->print(OS);
+    OS << "\n";
+    OS.indent(Indent) << "Step: ";
+    Step->print(OS);
+    OS << "\n";
+    if (ExtraConst) {
+      OS.indent(Indent) << "ExtraConst: ";
+      ExtraConst->print(OS, false);
+      OS << "\n";
+    }
+  }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
+#endif
+
+  bool matchSimpleRecurrence(const PHINode *P);
+  bool matchConditionalRecurrence(
+      const PHINode *P,
+      Instruction::BinaryOps BOWithConstOpToMatch = Instruction::BinaryOpsEnd);
+
+private:
+  BinaryOperator *digRecurrence(
+      Instruction *V,
+      Instruction::BinaryOps BOWithConstOpToMatch = Instruction::BinaryOpsEnd);
+};
+
+/// Wraps llvm::matchSimpleRecurrence. Match a simple first order recurrence
+/// cycle of the form:
+///
+/// loop:
+///    %rec = phi [%start, %entry], [%BO, %loop]
+///     ...
+///     %BO = binop %rec, %step
+///
+/// or
+///
+/// loop:
+///    %rec = phi [%start, %entry], [%BO, %loop]
+///    ...
+///    %BO = binop %step, %rec
+///
+bool RecurrenceInfo::matchSimpleRecurrence(const PHINode *P) {
+  Phi = P;
+  return llvm::matchSimpleRecurrence(Phi, BO, Start, Step);
+}
+
+/// Digs for a recurrence starting with \p V hitting the PHI node in a use-def
+/// chain. Used by matchConditionalRecurrence.
+BinaryOperator *
+RecurrenceInfo::digRecurrence(Instruction *V,
+                              Instruction::BinaryOps BOWithConstOpToMatch) {
+  SmallVector<Instruction *> Worklist;
+  Worklist.push_back(V);
+  while (!Worklist.empty()) {
+    Instruction *I = Worklist.pop_back_val();
+
+    // Don't add a PHI's operands to the Worklist.
+    if (isa<PHINode>(I))
+      continue;
+
+    // Find a recurrence over a BinOp, by matching either of its operands
+    // with with the PHINode.
+    if (match(I, m_c_BinOp(m_Value(), m_Specific(Phi))))
+      return cast<BinaryOperator>(I);
+
+    // Bind to ExtraConst, if we match exactly one.
+    if (I->getOpcode() == BOWithConstOpToMatch) {
+      if (ExtraConst)
+        return nullptr;
+      const APInt *C = nullptr;
+      if (match(I, m_c_BinOp(m_APInt(C), m_Value())))
+        ExtraConst = *C;
+    }
+
+    // Continue along the use-def chain.
+    for (Use &U : I->operands())
+      if (auto *UI = dyn_cast<Instruction>(U))
+        if (L.contains(UI))
+          Worklist.push_back(UI);
+  }
+  return nullptr;
+}
+
+/// A Conditional Recurrence is a recurrence of the form:
+///
+/// loop:
+///    %rec = [%start, %entry], [%step, %loop]
+///    ...
+///    %step = select _, %tv, %fv
+///
+/// where %tv and %fv ultimately end up using %rec via the same %BO instruction,
+/// after digging through the use-def chain.
+///
+/// ExtraConst is relevant if \p BOWithConstOpToMatch is supplied: when digging
+/// the use-def chain, a BinOp with opcode \p BOWithConstOpToMatch is matched,
+/// and ExtraConst is a constant operand of that BinOp. This peculiarity exists,
+/// because in a CRC algorithm, the \p BOWithConstOpToMatch is an XOR, and the
+/// ExtraConst ends up being the generating polynomial.
+bool RecurrenceInfo::matchConditionalRecurrence(
+    const PHINode *P, Instruction::BinaryOps BOWithConstOpToMatch) {
+  Phi = P;
+  if (Phi->getNumIncomingValues() != 2)
+    return false;
+
+  for (unsigned Idx = 0; Idx != 2; ++Idx) {
+    Value *FoundStep = Phi->getIncomingValue(Idx);
+    Value *FoundStart = Phi->getIncomingValue(!Idx);
+
+    Instruction *TV, *FV;
+    if (!match(FoundStep,
+               m_Select(m_Cmp(), m_Instruction(TV), m_Instruction(FV))))
+      continue;
+
+    // For a conditional recurrence, both the true and false values of the
+    // select must ultimately end up in the same recurrent BinOp.
+    BinaryOperator *FoundBO = digRecurrence(TV, BOWithConstOpToMatch);
+    BinaryOperator *AltBO = digRecurrence(FV, BOWithConstOpToMatch);
+    if (!FoundBO || FoundBO != AltBO)
+      return false;
+
+    if (BOWithConstOpToMatch != Instruction::BinaryOpsEnd && !ExtraConst) {
+      LLVM_DEBUG(dbgs() << "HashRecognize: Unable to match single BinaryOp "
+                           "with constant in conditional recurrence\n");
+      return false;
+    }
+
+    BO = FoundBO;
+    Start = FoundStart;
+    Step = FoundStep;
+    return true;
+  }
+  return false;
+}
+
+/// Iterates over all the phis in \p LoopLatch, and attempts to extract a
+/// Conditional Recurrence and an optional Simple Recurrence.
+static std::optional<std::pair<RecurrenceInfo, RecurrenceInfo>>
+getRecurrences(BasicBlock *LoopLatch, const PHINode *IndVar, const Loop &L) {
+  auto Phis = LoopLatch->phis();
+  unsigned NumPhis = std::distance(Phis.begin(), Phis.end());
+  if (NumPhis != 2 && NumPhis != 3)
+    return {};
+
+  RecurrenceInfo SimpleRecurrence(L);
+  RecurrenceInfo ConditionalRecurrence(L);
+  for (PHINode &P : Phis) {
+    if (&P == IndVar)
+      continue;
+    if (!SimpleRecurrence)
+      SimpleRecurrence.matchSimpleRecurrence(&P);
+    if (!ConditionalRecurrence)
+      ConditionalRecurrence.matchConditionalRecurrence(
+          &P, Instruction::BinaryOps::Xor);
+  }
+  if (NumPhis == 3 && (!SimpleRecurrence || !ConditionalRecurrence))
+    return {};
+  return std::make_pair(SimpleRecurrence, ConditionalRecurrence);
+}
+
+PolynomialInfo::PolynomialInfo(unsigned TripCount, const Value *LHS,
+                               const APInt &RHS, const Value *ComputedValue,
+                               bool ByteOrderSwapped, const Value *LHSAux)
+    : TripCount(TripCount), LHS(LHS), RHS(RHS), ComputedValue(ComputedValue),
+      ByteOrderSwapped(ByteOrderSwapped), LHSAux(LHSAux) {}
+
+/// In the big-endian case, checks the bottom N bits against CheckFn, and that
+/// the rest are unknown. In the little-endian case, checks the top N bits
+/// against CheckFn, and that the rest are unknown. Callers usually call this
+/// function with N = TripCount, and CheckFn checking that the remainder bits of
+/// the CRC polynomial division are zero.
+static bool checkExtractBits(const KnownBits &Known, unsigned N,
+                             function_ref<bool(const KnownBits &)> CheckFn,
+                             bool ByteOrderSwapped) {
+  // Check that the entire thing is a constant.
+  if (N == Known.getBitWidth())
+    return CheckFn(Known.extractBits(N, 0));
+
+  // Check that the {top, bottom} N bits are not unknown and that the {bottom,
+  // top} N bits are known.
+  unsigned BitPos = ByteOrderSwapped ? 0 : Known.getBitWidth() - N;
+  unsigned SwappedBitPos = ByteOrderSwapped ? N : 0;
+  return CheckFn(Known.extractBits(N, BitPos)) &&
+         Known.extractBits(Known.getBitWidth() - N, SwappedBitPos).isUnknown();
+}
+
+/// Generate a lookup table of 256 entries by interleaving the generating
+/// polynomial. The optimization technique of table-lookup for CRC is also
+/// called the Sarwate algorithm.
+CRCTable HashRecognize::genSarwateTable(const APInt &GenPoly,
+                                        bool ByteOrderSwapped) const {
+  unsigned BW = GenPoly.getBitWidth();
+  CRCTable Table;
+  Table[0] = APInt::getZero(BW);
+
+  if (ByteOrderSwapped) {
+    APInt CRCInit(BW, 128);
+    for (unsigned I = 1; I < 256; I <<= 1) {
+      CRCInit = CRCInit.shl(1) ^
+                (CRCInit.isSignBitSet() ? GenPoly : APInt::getZero(BW));
+      for (unsigned J = 0; J < I; ++J)
+        Table[I + J] = CRCInit ^ Table[J];
+    }
+    return Table;
+  }
+
+  APInt CRCInit(BW, 1);
+  for (unsigned I = 128; I; I >>= 1) {
+    CRCInit = CRCInit.lshr(1) ^ (CRCInit[0] ? GenPoly : APInt::getZero(BW));
+    for (unsigned J = 0; J < 256; J += (I << 1))
+      Table[I + J] = CRCInit ^ Table[J];
+  }
+  return Table;
+}
+
+/// Checks if \p Reference is reachable from \p Needle on the use-def chain, and
+/// that there are no stray PHI nodes while digging the use-def chain. \p
+/// BOToMatch is a CRC peculiarity: at least one of the Users of Needle needs to
+/// match this OpCode, which is XOR for CRC.
+static bool arePHIsIntertwined(
+    const PHINode *Needle, const PHINode *Reference, const Loop &L,
+    Instruction::BinaryOps BOToMatch = Instruction::BinaryOpsEnd) {
+  // Initialize the worklist with Users of the Needle.
+  SmallVector<const Instruction *> Worklist;
+  for (const User *U : Needle->users()) {
+    if (auto *UI = dyn_cast<Instruction>(U))
+      if (L.contains(UI))
+        Worklist.push_back(UI);
+  }
+
+  // BOToMatch is usually XOR for CRC.
+  if (BOToMatch != Instruction::BinaryOpsEnd) {
+    if (count_if(Worklist, [BOToMatch](const Instruction *I) {
+          return I->getOpcode() == BOToMatch;
+        }) != 1)
+      return false;
+  }
+
+  while (!Worklist.empty()) {
+    const Instruction *I = Worklist.pop_back_val();
+
+    // Since Needle is never pushed onto the Worklist, I must either be the
+    // Reference PHI node (in which case we're done), or a stray PHI node (in
+    // which case we abort).
+    if (isa<PHINode>(I))
+      return I == Reference;
+
+    for (const Use &U : I->operands())
+      if (auto *UI = dyn_cast<Instruction>(U))
+        // Don't push Needle back onto the Worklist.
+        if (UI != Needle && L.contains(UI))
+          Worklist.push_back(UI);
+  }
+  return false;
+}
+
+// Recognizes a multiplication or division by the constant two, using SCEV. By
+// doing this, we're immune to whether the IR expression is mul/udiv or
+// equivalently shl/lshr. Return false when it is a UDiv, true when it is a Mul,
+// and std::nullopt otherwise.
+static std::optional<bool> isBigEndianBitShift(Value *V, ScalarEvolution &SE) {
+  if (!V->getType()->isIntegerTy())
+    return {};
+
+  const SCEV *E = SE.getSCEV(V);
+  if (match(E, m_scev_UDiv(m_SCEV(), m_scev_SpecificInt(2))))
+    return false;
+  if (match(E, m_scev_Mul(m_scev_SpecificInt(2), m_SCEV())))
+    return true;
+  return {};
+}
+
+/// The main entry point for analyzing a loop and recognizing the CRC algorithm.
+/// Returns a PolynomialInfo on success, and either an ErrBits or a StringRef on
+/// failure.
+std::variant<PolynomialInfo, ErrBits, StringRef>
+HashRecognize::recognizeCRC() const {
+  if (!L.isInnermost())
+    return "Loop is not innermost";
+  unsigned TC = SE.getSmallConstantMaxTripCount(&L);
+  if (!TC || TC > 256)
+    return "Unable to find a small constant trip count";
+  BasicBlock *Latch = L.getLoopLatch();
+  BasicBlock *Exit = L.getExitBlock();
+  const PHINode *IndVar = L.getCanonicalInductionVariable();
+  if (!Latch || !Exit || !IndVar)
+    return "Loop not in canonical form";
+
+  auto R = getRecurrences(Latch, IndVar, L);
+  if (!R)
+    return "Found stray PHI";
+  auto [SimpleRecurrence, ConditionalRecurrence] = *R;
+  if (!ConditionalRecurrence)
+    return "Unable to find conditional recurrence";
+
+  // Make sure that all recurrences are either all SCEVMul with two or SCEVDiv
+  // with two, or in other words, that they're single bit-shifts.
+  std::optional<bool> ByteOrderSwapped =
+      isBigEndianBitShift(ConditionalRecurrence.BO, SE);
+  if (!ByteOrderSwapped)
+    return "Loop with non-unit bitshifts";
+  if (SimpleRecurrence) {
+    if (isBigEndianBitShift(SimpleRecurrence.BO, SE) != ByteOrderSwapped)
+      return "Loop with non-unit bitshifts";
+    if (!arePHIsIntertwined(SimpleRecurrence.Phi, ConditionalRecurrence.Phi, L,
+                            Instruction::BinaryOps::Xor))
+      return "Simple recurrence doesn't use conditional recurrence with XOR";
+  }
+
+  // Make sure that the computed value is used in the exit block: this should be
+  // true even if it is only really used in an outer loop's exit block, since
+  // the loop is in LCSSA form.
+  auto *ComputedValue = cast<SelectInst>(ConditionalRecurrence.Step);
+  if (none_of(ComputedValue->users(), [Exit](User *U) {
+        auto *UI = dyn_cast<Instruction>(U);
+        return UI && UI->getParent() == Exit;
+      }))
+    return "Unable to find use of computed value in loop exit block";
+
+  assert(ConditionalRecurrence.ExtraConst &&
+         "Expected ExtraConst in conditional recurrence");
+  const APInt &GenPoly = *ConditionalRecurrence.ExtraConst;
+
+  // PhiEvolutions are pairs of PHINodes along with their incoming value from
+  // within the loop, which we term as their step. Note that in the case of a
+  // Simple Recurrence, Step is an operand of the BO, while in a Conditional
+  // Recurrence, it is a SelectInst.
+  SmallVector<PhiStepPair, 2> PhiEvolutions;
+  PhiEvolutions.emplace_back(ConditionalRecurrence.Phi, ComputedValue);
+  if (SimpleRecurrence)
+    PhiEvolutions.emplace_back(SimpleRecurrence.Phi, SimpleRecurrence.BO);
+
+  ValueEvolution VE(TC, *ByteOrderSwapped);
+  if (!VE.computeEvolutions(PhiEvolutions))
+    return VE.getError();
+  KnownBits ResultBits = VE.KnownPhis.at(ConditionalRecurrence.Phi);
+
+  auto IsZero = [](const KnownBits &K) { return K.isZero(); };
+  if (!checkExtractBits(ResultBits, TC, IsZero, *ByteOrderSwapped))
+    return ErrBits(ResultBits, TC, *ByteOrderSwapped);
+
+  const Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start : nullptr;
+  return PolynomialInfo(TC, ConditionalRecurrence.Start, GenPoly, ComputedValue,
+                        *ByteOrderSwapped, LHSAux);
+}
+
+void CRCTable::print(raw_ostream &OS) const {
+  for (unsigned I = 0; I < 256; I++) {
+    (*this)[I].print(OS, false);
+    OS << (I % 16 == 15 ? '\n' : ' ');
+  }
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void CRCTable::dump() const { print(dbgs()); }
+#endif
+
+void HashRecognize::print(raw_ostream &OS) const {
+  if (!L.isInnermost())
+    return;
+  OS << "HashRecognize: Checking a loop in '"
+     << L.getHeader()->getParent()->getName() << "' from " << L.getLocStr()
+     << "\n";
+  auto Ret = recognizeCRC();
+  if (!std::holds_alternative<PolynomialInfo>(Ret)) {
+    OS << "Did not find a hash algorithm\n";
+    if (std::holds_alternative<StringRef>(Ret))
+      OS << "Reason: " << std::get<StringRef>(Ret) << "\n";
+    if (std::holds_alternative<ErrBits>(Ret)) {
+      auto [Actual, Iter, ByteOrderSwapped] = std::get<ErrBits>(Ret);
+      OS << "Reason: Expected " << (ByteOrderSwapped ? "bottom " : "top ")
+         << Iter << " bits zero (";
+      Actual.print(OS);
+      OS << ")\n";
+    }
+    return;
+  }
+
+  auto Info = std::get<PolynomialInfo>(Ret);
+  OS << "Found" << (Info.ByteOrderSwapped ? " big-endian " : " little-endian ")
+     << "CRC-" << Info.RHS.getBitWidth() << " loop with trip count "
+     << Info.TripCount << "\n";
+  OS.indent(2) << "Initial CRC: ";
+  Info.LHS->print(OS);
+  OS << "\n";
+  OS.indent(2) << "Generating polynomial: ";
+  Info.RHS.print(OS, false);
+  OS << "\n";
+  OS.indent(2) << "Computed CRC: ";
+  Info.ComputedValue->print(OS);
+  OS << "\n";
+  if (Info.LHSAux) {
+    OS.indent(2) << "Auxiliary data: ";
+    Info.LHSAux->print(OS);
+    OS << "\n";
+  }
+  OS.indent(2) << "Computed CRC lookup table:\n";
+  genSarwateTable(Info.RHS, Info.ByteOrderSwapped).print(OS);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void HashRecognize::dump() const { print(dbgs()); }
+#endif
+
+HashRecognize::HashRecognize(const Loop &L, ScalarEvolution &SE)
+    : L(L), SE(SE) {}
+
+PreservedAnalyses HashRecognizePrinterPass::run(Loop &L,
+                                                LoopAnalysisManager &AM,
+                                                LoopStandardAnalysisResults &AR,
+                                                LPMUpdater &) {
+  AM.getResult<HashRecognizeAnalysis>(L, AR).print(OS);
+  return PreservedAnalyses::all();
+}
+
+HashRecognize HashRecognizeAnalysis::run(Loop &L, LoopAnalysisManager &AM,
+                                         LoopStandardAnalysisResults &AR) {
+  return {L, AR.SE};
+}
+
+AnalysisKey HashRecognizeAnalysis::Key;
diff --git a/external/llvm-project/llvm/lib/Analysis/IR2Vec.cpp b/external/llvm-project/llvm/lib/Analysis/IR2Vec.cpp
index 67af44dcac42..0f7303c1b091 100644
--- a/external/llvm-project/llvm/lib/Analysis/IR2Vec.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/IR2Vec.cpp
@@ -16,13 +16,11 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
-#include "llvm/Support/JSON.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 using namespace llvm;
@@ -33,6 +31,8 @@ using namespace ir2vec;
 STATISTIC(VocabMissCounter,
           "Number of lookups to entites not present in the vocabulary");
 
+namespace llvm {
+namespace ir2vec {
 static cl::OptionCategory IR2VecCategory("IR2Vec Options");
 
 // FIXME: Use a default vocab when not specified
@@ -40,54 +40,81 @@ static cl::opt<std::string>
     VocabFile("ir2vec-vocab-path", cl::Optional,
               cl::desc("Path to the vocabulary file for IR2Vec"), cl::init(""),
               cl::cat(IR2VecCategory));
-static cl::opt<float> OpcWeight("ir2vec-opc-weight", cl::Optional,
-                                cl::init(1.0),
-                                cl::desc("Weight for opcode embeddings"),
-                                cl::cat(IR2VecCategory));
-static cl::opt<float> TypeWeight("ir2vec-type-weight", cl::Optional,
-                                 cl::init(0.5),
-                                 cl::desc("Weight for type embeddings"),
-                                 cl::cat(IR2VecCategory));
-static cl::opt<float> ArgWeight("ir2vec-arg-weight", cl::Optional,
-                                cl::init(0.2),
-                                cl::desc("Weight for argument embeddings"),
-                                cl::cat(IR2VecCategory));
+cl::opt<float> OpcWeight("ir2vec-opc-weight", cl::Optional, cl::init(1.0),
+                         cl::desc("Weight for opcode embeddings"),
+                         cl::cat(IR2VecCategory));
+cl::opt<float> TypeWeight("ir2vec-type-weight", cl::Optional, cl::init(0.5),
+                          cl::desc("Weight for type embeddings"),
+                          cl::cat(IR2VecCategory));
+cl::opt<float> ArgWeight("ir2vec-arg-weight", cl::Optional, cl::init(0.2),
+                         cl::desc("Weight for argument embeddings"),
+                         cl::cat(IR2VecCategory));
+} // namespace ir2vec
+} // namespace llvm
 
 AnalysisKey IR2VecVocabAnalysis::Key;
 
+namespace llvm::json {
+inline bool fromJSON(const llvm::json::Value &E, Embedding &Out,
+                     llvm::json::Path P) {
+  std::vector<double> TempOut;
+  if (!llvm::json::fromJSON(E, TempOut, P))
+    return false;
+  Out = Embedding(std::move(TempOut));
+  return true;
+}
+} // namespace llvm::json
+
 // ==----------------------------------------------------------------------===//
-// Embedder and its subclasses
+// Embedding
 //===----------------------------------------------------------------------===//
 
-Embedder::Embedder(const Function &F, const Vocab &Vocabulary,
-                   unsigned Dimension)
-    : F(F), Vocabulary(Vocabulary), Dimension(Dimension),
-      OpcWeight(::OpcWeight), TypeWeight(::TypeWeight), ArgWeight(::ArgWeight) {
+Embedding &Embedding::operator+=(const Embedding &RHS) {
+  assert(this->size() == RHS.size() && "Vectors must have the same dimension");
+  std::transform(this->begin(), this->end(), RHS.begin(), this->begin(),
+                 std::plus<double>());
+  return *this;
 }
 
-Expected<std::unique_ptr<Embedder>> Embedder::create(IR2VecKind Mode,
-                                                     const Function &F,
-                                                     const Vocab &Vocabulary,
-                                                     unsigned Dimension) {
-  switch (Mode) {
-  case IR2VecKind::Symbolic:
-    return std::make_unique<SymbolicEmbedder>(F, Vocabulary, Dimension);
-  }
-  return make_error<StringError>("Unknown IR2VecKind", errc::invalid_argument);
+Embedding &Embedding::operator-=(const Embedding &RHS) {
+  assert(this->size() == RHS.size() && "Vectors must have the same dimension");
+  std::transform(this->begin(), this->end(), RHS.begin(), this->begin(),
+                 std::minus<double>());
+  return *this;
 }
 
-void Embedder::addVectors(Embedding &Dst, const Embedding &Src) {
-  assert(Dst.size() == Src.size() && "Vectors must have the same dimension");
-  std::transform(Dst.begin(), Dst.end(), Src.begin(), Dst.begin(),
-                 std::plus<double>());
+Embedding &Embedding::scaleAndAdd(const Embedding &Src, float Factor) {
+  assert(this->size() == Src.size() && "Vectors must have the same dimension");
+  for (size_t Itr = 0; Itr < this->size(); ++Itr)
+    (*this)[Itr] += Src[Itr] * Factor;
+  return *this;
+}
+
+bool Embedding::approximatelyEquals(const Embedding &RHS,
+                                    double Tolerance) const {
+  assert(this->size() == RHS.size() && "Vectors must have the same dimension");
+  for (size_t Itr = 0; Itr < this->size(); ++Itr)
+    if (std::abs((*this)[Itr] - RHS[Itr]) > Tolerance)
+      return false;
+  return true;
 }
 
-void Embedder::addScaledVector(Embedding &Dst, const Embedding &Src,
-                               float Factor) {
-  assert(Dst.size() == Src.size() && "Vectors must have the same dimension");
-  for (size_t i = 0; i < Dst.size(); ++i) {
-    Dst[i] += Src[i] * Factor;
+// ==----------------------------------------------------------------------===//
+// Embedder and its subclasses
+//===----------------------------------------------------------------------===//
+
+Embedder::Embedder(const Function &F, const Vocab &Vocabulary)
+    : F(F), Vocabulary(Vocabulary),
+      Dimension(Vocabulary.begin()->second.size()), OpcWeight(::OpcWeight),
+      TypeWeight(::TypeWeight), ArgWeight(::ArgWeight) {}
+
+Expected<std::unique_ptr<Embedder>>
+Embedder::create(IR2VecKind Mode, const Function &F, const Vocab &Vocabulary) {
+  switch (Mode) {
+  case IR2VecKind::Symbolic:
+    return std::make_unique<SymbolicEmbedder>(F, Vocabulary);
   }
+  return make_error<StringError>("Unknown IR2VecKind", errc::invalid_argument);
 }
 
 // FIXME: Currently lookups are string based. Use numeric Keys
@@ -167,20 +194,20 @@ void SymbolicEmbedder::computeEmbeddings(const BasicBlock &BB) const {
     Embedding InstVector(Dimension, 0);
 
     const auto OpcVec = lookupVocab(I.getOpcodeName());
-    addScaledVector(InstVector, OpcVec, OpcWeight);
+    InstVector.scaleAndAdd(OpcVec, OpcWeight);
 
     // FIXME: Currently lookups are string based. Use numeric Keys
     // for efficiency.
     const auto Type = I.getType();
     const auto TypeVec = getTypeEmbedding(Type);
-    addScaledVector(InstVector, TypeVec, TypeWeight);
+    InstVector.scaleAndAdd(TypeVec, TypeWeight);
 
     for (const auto &Op : I.operands()) {
       const auto OperandVec = getOperandEmbedding(Op.get());
-      addScaledVector(InstVector, OperandVec, ArgWeight);
+      InstVector.scaleAndAdd(OperandVec, ArgWeight);
     }
     InstVecMap[&I] = InstVector;
-    addVectors(BBVector, InstVector);
+    BBVector += InstVector;
   }
   BBVecMap[&BB] = BBVector;
 }
@@ -190,7 +217,7 @@ void SymbolicEmbedder::computeEmbeddings() const {
     return;
   for (const auto &BB : F) {
     computeEmbeddings(BB);
-    addVectors(FuncVector, BBVecMap[&BB]);
+    FuncVector += BBVecMap[&BB];
   }
 }
 
@@ -223,9 +250,9 @@ bool IR2VecVocabResult::invalidate(
 // by auto-generating a default vocabulary during the build time.
 Error IR2VecVocabAnalysis::readVocabulary() {
   auto BufOrError = MemoryBuffer::getFileOrSTDIN(VocabFile, /*IsText=*/true);
-  if (!BufOrError) {
+  if (!BufOrError)
     return createFileError(VocabFile, BufOrError.getError());
-  }
+
   auto Content = BufOrError.get()->getBuffer();
   json::Path::Root Path("");
   Expected<json::Value> ParsedVocabValue = json::parse(Content);
@@ -233,39 +260,60 @@ Error IR2VecVocabAnalysis::readVocabulary() {
     return ParsedVocabValue.takeError();
 
   bool Res = json::fromJSON(*ParsedVocabValue, Vocabulary, Path);
-  if (!Res) {
+  if (!Res)
     return createStringError(errc::illegal_byte_sequence,
                              "Unable to parse the vocabulary");
-  }
-  assert(Vocabulary.size() > 0 && "Vocabulary is empty");
+
+  if (Vocabulary.empty())
+    return createStringError(errc::illegal_byte_sequence,
+                             "Vocabulary is empty");
 
   unsigned Dim = Vocabulary.begin()->second.size();
-  assert(Dim > 0 && "Dimension of vocabulary is zero");
-  (void)Dim;
-  assert(std::all_of(Vocabulary.begin(), Vocabulary.end(),
-                     [Dim](const std::pair<StringRef, Embedding> &Entry) {
-                       return Entry.second.size() == Dim;
-                     }) &&
-         "All vectors in the vocabulary are not of the same dimension");
+  if (Dim == 0)
+    return createStringError(errc::illegal_byte_sequence,
+                             "Dimension of vocabulary is zero");
+
+  if (!std::all_of(Vocabulary.begin(), Vocabulary.end(),
+                   [Dim](const std::pair<StringRef, Embedding> &Entry) {
+                     return Entry.second.size() == Dim;
+                   }))
+    return createStringError(
+        errc::illegal_byte_sequence,
+        "All vectors in the vocabulary are not of the same dimension");
+
   return Error::success();
 }
 
+IR2VecVocabAnalysis::IR2VecVocabAnalysis(const Vocab &Vocabulary)
+    : Vocabulary(Vocabulary) {}
+
+IR2VecVocabAnalysis::IR2VecVocabAnalysis(Vocab &&Vocabulary)
+    : Vocabulary(std::move(Vocabulary)) {}
+
+void IR2VecVocabAnalysis::emitError(Error Err, LLVMContext &Ctx) {
+  handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EI) {
+    Ctx.emitError("Error reading vocabulary: " + EI.message());
+  });
+}
+
 IR2VecVocabAnalysis::Result
 IR2VecVocabAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
   auto Ctx = &M.getContext();
+  // FIXME: Scale the vocabulary once. This would avoid scaling per use later.
+  // If vocabulary is already populated by the constructor, use it.
+  if (!Vocabulary.empty())
+    return IR2VecVocabResult(std::move(Vocabulary));
+
+  // Otherwise, try to read from the vocabulary file.
   if (VocabFile.empty()) {
     // FIXME: Use default vocabulary
     Ctx->emitError("IR2Vec vocabulary file path not specified");
     return IR2VecVocabResult(); // Return invalid result
   }
   if (auto Err = readVocabulary()) {
-    handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EI) {
-      Ctx->emitError("Error reading vocabulary: " + EI.message());
-    });
+    emitError(std::move(Err), *Ctx);
     return IR2VecVocabResult();
   }
-  // FIXME: Scale the vocabulary here once. This would avoid scaling per use
-  // later.
   return IR2VecVocabResult(std::move(Vocabulary));
 }
 
@@ -286,10 +334,9 @@ PreservedAnalyses IR2VecPrinterPass::run(Module &M,
   assert(IR2VecVocabResult.isValid() && "IR2Vec Vocabulary is invalid");
 
   auto Vocab = IR2VecVocabResult.getVocabulary();
-  auto Dim = IR2VecVocabResult.getDimension();
   for (Function &F : M) {
     Expected<std::unique_ptr<Embedder>> EmbOrErr =
-        Embedder::create(IR2VecKind::Symbolic, F, Vocab, Dim);
+        Embedder::create(IR2VecKind::Symbolic, F, Vocab);
     if (auto Err = EmbOrErr.takeError()) {
       handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EI) {
         OS << "Error creating IR2Vec embeddings: " << EI.message() << "\n";
diff --git a/external/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp b/external/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp
index b7c7bcab168c..7232283b9101 100644
--- a/external/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp
@@ -111,7 +111,7 @@ static std::pair<Type *, bool> computeRecurrenceType(Instruction *Exit,
     // If demanded bits wasn't able to limit the bit width, we can try to use
     // value tracking instead. This can be the case, for example, if the value
     // may be negative.
-    auto NumSignBits = ComputeNumSignBits(Exit, DL, 0, AC, nullptr, DT);
+    auto NumSignBits = ComputeNumSignBits(Exit, DL, AC, nullptr, DT);
     auto NumTypeBits = DL.getTypeSizeInBits(Exit->getType());
     MaxBitWidth = NumTypeBits - NumSignBits;
     KnownBits Bits = computeKnownBits(Exit, DL);
@@ -483,7 +483,7 @@ bool RecurrenceDescriptor::AddReductionVar(
       } else if (!isa<PHINode>(UI) &&
                  ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) &&
                    !isa<SelectInst>(UI)) ||
-                  (!isConditionalRdxPattern(Kind, UI).isRecurrence() &&
+                  (!isConditionalRdxPattern(UI).isRecurrence() &&
                    !isAnyOfPattern(TheLoop, Phi, UI, IgnoredVal)
                         .isRecurrence() &&
                    !isMinMaxPattern(UI, Kind, IgnoredVal).isRecurrence())))
@@ -802,7 +802,7 @@ RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
 /// %add = fadd %0, %sum.1
 /// %sum.2 = select %cmp, %add, %sum.1
 RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isConditionalRdxPattern(RecurKind Kind, Instruction *I) {
+RecurrenceDescriptor::isConditionalRdxPattern(Instruction *I) {
   SelectInst *SI = dyn_cast<SelectInst>(I);
   if (!SI)
     return InstDesc(false, I);
@@ -874,7 +874,7 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
   case Instruction::Select:
     if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
         Kind == RecurKind::Add || Kind == RecurKind::Mul)
-      return isConditionalRdxPattern(Kind, I);
+      return isConditionalRdxPattern(I);
     if (isFindLastIVRecurrenceKind(Kind) && SE)
       return isFindLastIVPattern(L, OrigPhi, I, *SE);
     [[fallthrough]];
diff --git a/external/llvm-project/llvm/lib/Analysis/InlineCost.cpp b/external/llvm-project/llvm/lib/Analysis/InlineCost.cpp
index 8ddfa1e4eb6f..7bd1f1800458 100644
--- a/external/llvm-project/llvm/lib/Analysis/InlineCost.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/InlineCost.cpp
@@ -263,8 +263,6 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   // Cache the DataLayout since we use it a lot.
   const DataLayout &DL;
 
-  DominatorTree DT;
-
   /// The OptimizationRemarkEmitter available for this compilation.
   OptimizationRemarkEmitter *ORE;
 
@@ -1688,66 +1686,51 @@ bool CallAnalyzer::simplifyCmpInstForRecCall(CmpInst &Cmp) {
   if (!isa<Argument>(Cmp.getOperand(0)) || !isa<Constant>(Cmp.getOperand(1)))
     return false;
   auto *CmpOp = Cmp.getOperand(0);
-  Function *F = Cmp.getFunction();
-  // Iterate over the users of the function to check if it's a recursive
-  // function:
-  for (auto *U : F->users()) {
-    CallInst *Call = dyn_cast<CallInst>(U);
-    if (!Call || Call->getFunction() != F || Call->getCalledFunction() != F)
-      continue;
-    auto *CallBB = Call->getParent();
-    auto *Predecessor = CallBB->getSinglePredecessor();
-    // Only handle the case when the callsite has a single predecessor:
-    if (!Predecessor)
-      continue;
+  // Make sure that the callsite is recursive:
+  if (CandidateCall.getCaller() != &F)
+    return false;
+  // Only handle the case when the callsite has a single predecessor:
+  auto *CallBB = CandidateCall.getParent();
+  auto *Predecessor = CallBB->getSinglePredecessor();
+  if (!Predecessor)
+    return false;
+  // Check if the callsite is guarded by the same Cmp instruction:
+  auto *Br = dyn_cast<BranchInst>(Predecessor->getTerminator());
+  if (!Br || Br->isUnconditional() || Br->getCondition() != &Cmp)
+    return false;
 
-    auto *Br = dyn_cast<BranchInst>(Predecessor->getTerminator());
-    if (!Br || Br->isUnconditional())
-      continue;
-    // Check if the Br condition is the same Cmp instr we are investigating:
-    if (Br->getCondition() != &Cmp)
-      continue;
-    // Check if there are any arg of the recursive callsite is affecting the cmp
-    // instr:
-    bool ArgFound = false;
-    Value *FuncArg = nullptr, *CallArg = nullptr;
-    for (unsigned ArgNum = 0;
-         ArgNum < F->arg_size() && ArgNum < Call->arg_size(); ArgNum++) {
-      FuncArg = F->getArg(ArgNum);
-      CallArg = Call->getArgOperand(ArgNum);
-      if (FuncArg == CmpOp && CallArg != CmpOp) {
-        ArgFound = true;
-        break;
-      }
-    }
-    if (!ArgFound)
-      continue;
-    // Now we have a recursive call that is guarded by a cmp instruction.
-    // Check if this cmp can be simplified:
-    SimplifyQuery SQ(DL, dyn_cast<Instruction>(CallArg));
-    DomConditionCache DC;
-    DC.registerBranch(Br);
-    SQ.DC = &DC;
-    if (DT.root_size() == 0) {
-      // Dominator tree was never constructed for any function yet.
-      DT.recalculate(*F);
-    } else if (DT.getRoot()->getParent() != F) {
-      // Dominator tree was constructed for a different function, recalculate
-      // it for the current function.
-      DT.recalculate(*F);
+  // Check if there is any arg of the recursive callsite is affecting the cmp
+  // instr:
+  bool ArgFound = false;
+  Value *FuncArg = nullptr, *CallArg = nullptr;
+  for (unsigned ArgNum = 0;
+       ArgNum < F.arg_size() && ArgNum < CandidateCall.arg_size(); ArgNum++) {
+    FuncArg = F.getArg(ArgNum);
+    CallArg = CandidateCall.getArgOperand(ArgNum);
+    if (FuncArg == CmpOp && CallArg != CmpOp) {
+      ArgFound = true;
+      break;
     }
-    SQ.DT = &DT;
-    Value *SimplifiedInstruction = llvm::simplifyInstructionWithOperands(
-        cast<CmpInst>(&Cmp), {CallArg, Cmp.getOperand(1)}, SQ);
-    if (auto *ConstVal = dyn_cast_or_null<ConstantInt>(SimplifiedInstruction)) {
-      bool IsTrueSuccessor = CallBB == Br->getSuccessor(0);
-      // Make sure that the BB of the recursive call is NOT the next successor
-      // of the icmp. In other words, make sure that the recursion depth is 1.
-      if ((ConstVal->isOne() && !IsTrueSuccessor) ||
-          (ConstVal->isZero() && IsTrueSuccessor)) {
-        SimplifiedValues[&Cmp] = ConstVal;
-        return true;
-      }
+  }
+  if (!ArgFound)
+    return false;
+
+  // Now we have a recursive call that is guarded by a cmp instruction.
+  // Check if this cmp can be simplified:
+  SimplifyQuery SQ(DL, dyn_cast<Instruction>(CallArg));
+  CondContext CC(&Cmp);
+  CC.Invert = (CallBB != Br->getSuccessor(0));
+  SQ.CC = &CC;
+  CC.AffectedValues.insert(FuncArg);
+  Value *SimplifiedInstruction = llvm::simplifyInstructionWithOperands(
+      cast<CmpInst>(&Cmp), {CallArg, Cmp.getOperand(1)}, SQ);
+  if (auto *ConstVal = dyn_cast_or_null<ConstantInt>(SimplifiedInstruction)) {
+    // Make sure that the BB of the recursive call is NOT the true successor
+    // of the icmp. In other words, make sure that the recursion depth is 1.
+    if ((ConstVal->isOne() && CC.Invert) ||
+        (ConstVal->isZero() && !CC.Invert)) {
+      SimplifiedValues[&Cmp] = ConstVal;
+      return true;
     }
   }
   return false;
diff --git a/external/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp b/external/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp
index 12c84c718d4f..e397a228afee 100644
--- a/external/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -770,7 +770,7 @@ static Value *simplifySubInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,
     if (IsNUW)
       return Constant::getNullValue(Op0->getType());
 
-    KnownBits Known = computeKnownBits(Op1, /* Depth */ 0, Q);
+    KnownBits Known = computeKnownBits(Op1, Q);
     if (Known.Zero.isMaxSignedValue()) {
       // Op1 is either 0 or the minimum signed value. If the sub is NSW, then
       // Op1 must be 0 because negating the minimum signed value is undefined.
@@ -1029,8 +1029,7 @@ static bool isDivZero(Value *X, Value *Y, const SimplifyQuery &Q,
   // TODO: Convert this (and above) to range analysis
   //      ("computeConstantRangeIncludingKnownBits")?
   const APInt *C;
-  if (match(Y, m_APInt(C)) &&
-      computeKnownBits(X, /* Depth */ 0, Q).getMaxValue().ult(*C))
+  if (match(Y, m_APInt(C)) && computeKnownBits(X, Q).getMaxValue().ult(*C))
     return true;
 
   // Try again for any divisor:
@@ -1079,7 +1078,7 @@ static Value *simplifyDivRem(Instruction::BinaryOps Opcode, Value *Op0,
   if (Op0 == Op1)
     return IsDiv ? ConstantInt::get(Ty, 1) : Constant::getNullValue(Ty);
 
-  KnownBits Known = computeKnownBits(Op1, /* Depth */ 0, Q);
+  KnownBits Known = computeKnownBits(Op1, Q);
   // X / 0 -> poison
   // X % 0 -> poison
   // If the divisor is known to be zero, just return poison. This can happen in
@@ -1149,7 +1148,7 @@ static Value *simplifyDiv(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1,
     // have at least as many trailing zeros as the divisor to divide evenly. If
     // it has less trailing zeros, then the result must be poison.
     if (DivC->countr_zero()) {
-      KnownBits KnownOp0 = computeKnownBits(Op0, /* Depth */ 0, Q);
+      KnownBits KnownOp0 = computeKnownBits(Op0, Q);
       if (KnownOp0.countMaxTrailingZeros() < DivC->countr_zero())
         return PoisonValue::get(Op0->getType());
     }
@@ -1336,7 +1335,7 @@ static Value *simplifyShift(Instruction::BinaryOps Opcode, Value *Op0,
 
   // If any bits in the shift amount make that value greater than or equal to
   // the number of bits in the type, the shift is undefined.
-  KnownBits KnownAmt = computeKnownBits(Op1, /* Depth */ 0, Q);
+  KnownBits KnownAmt = computeKnownBits(Op1, Q);
   if (KnownAmt.getMinValue().uge(KnownAmt.getBitWidth()))
     return PoisonValue::get(Op0->getType());
 
@@ -1349,7 +1348,7 @@ static Value *simplifyShift(Instruction::BinaryOps Opcode, Value *Op0,
   // Check for nsw shl leading to a poison value.
   if (IsNSW) {
     assert(Opcode == Instruction::Shl && "Expected shl for nsw instruction");
-    KnownBits KnownVal = computeKnownBits(Op0, /* Depth */ 0, Q);
+    KnownBits KnownVal = computeKnownBits(Op0, Q);
     KnownBits KnownShl = KnownBits::shl(KnownVal, KnownAmt);
 
     if (KnownVal.Zero.isSignBitSet())
@@ -1385,7 +1384,7 @@ static Value *simplifyRightShift(Instruction::BinaryOps Opcode, Value *Op0,
   // The low bit cannot be shifted out of an exact shift if it is set.
   // TODO: Generalize by counting trailing zeros (see fold for exact division).
   if (IsExact) {
-    KnownBits Op0Known = computeKnownBits(Op0, /* Depth */ 0, Q);
+    KnownBits Op0Known = computeKnownBits(Op0, Q);
     if (Op0Known.One[0])
       return Op0;
   }
@@ -1457,7 +1456,7 @@ static Value *simplifyLShrInst(Value *Op0, Value *Op1, bool IsExact,
   if (Q.IIQ.UseInstrInfo && match(Op1, m_APInt(ShRAmt)) &&
       match(Op0, m_c_Or(m_NUWShl(m_Value(X), m_APInt(ShLAmt)), m_Value(Y))) &&
       *ShRAmt == *ShLAmt) {
-    const KnownBits YKnown = computeKnownBits(Y, /* Depth */ 0, Q);
+    const KnownBits YKnown = computeKnownBits(Y, Q);
     const unsigned EffWidthY = YKnown.countMaxActiveBits();
     if (ShRAmt->uge(EffWidthY))
       return X;
@@ -1492,7 +1491,7 @@ static Value *simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact,
     return X;
 
   // Arithmetic shifting an all-sign-bit value is a no-op.
-  unsigned NumSignBits = ComputeNumSignBits(Op0, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
+  unsigned NumSignBits = ComputeNumSignBits(Op0, Q.DL, Q.AC, Q.CxtI, Q.DT);
   if (NumSignBits == Op0->getType()->getScalarSizeInBits())
     return Op0;
 
@@ -1996,13 +1995,13 @@ static Value *simplifyAndCommutative(Value *Op0, Value *Op1,
 
   // -A & A = A if A is a power of two or zero.
   if (match(Op0, m_Neg(m_Specific(Op1))) &&
-      isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT))
+      isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, Q.AC, Q.CxtI, Q.DT))
     return Op1;
 
   // This is a similar pattern used for checking if a value is a power-of-2:
   // (A - 1) & A --> 0 (if A is a power-of-2 or 0)
   if (match(Op0, m_Add(m_Specific(Op1), m_AllOnes())) &&
-      isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT))
+      isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, Q.AC, Q.CxtI, Q.DT))
     return Constant::getNullValue(Op1->getType());
 
   // (x << N) & ((x << M) - 1) --> 0, where x is known to be a power of 2 and
@@ -2010,8 +2009,7 @@ static Value *simplifyAndCommutative(Value *Op0, Value *Op1,
   const APInt *Shift1, *Shift2;
   if (match(Op0, m_Shl(m_Value(X), m_APInt(Shift1))) &&
       match(Op1, m_Add(m_Shl(m_Specific(X), m_APInt(Shift2)), m_AllOnes())) &&
-      isKnownToBeAPowerOfTwo(X, Q.DL, /*OrZero*/ true, /*Depth*/ 0, Q.AC,
-                             Q.CxtI) &&
+      isKnownToBeAPowerOfTwo(X, Q.DL, /*OrZero*/ true, Q.AC, Q.CxtI) &&
       Shift1->uge(*Shift2))
     return Constant::getNullValue(Op0->getType());
 
@@ -2080,9 +2078,9 @@ static Value *simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
   Value *Shift;
   if (match(Op1, m_Power2(PowerC)) &&
       match(Op0, m_Add(m_Value(Shift), m_AllOnes())) &&
-      isKnownToBeAPowerOfTwo(Shift, Q.DL, /*OrZero*/ false, 0, Q.AC, Q.CxtI,
+      isKnownToBeAPowerOfTwo(Shift, Q.DL, /*OrZero*/ false, Q.AC, Q.CxtI,
                              Q.DT)) {
-    KnownBits Known = computeKnownBits(Shift, /* Depth */ 0, Q);
+    KnownBits Known = computeKnownBits(Shift, Q);
     // Use getActiveBits() to make use of the additional power of two knowledge
     if (PowerC->getActiveBits() >= Known.getMaxValue().getActiveBits())
       return ConstantInt::getNullValue(Op1->getType());
@@ -2146,10 +2144,10 @@ static Value *simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
                         m_Value(Y)))) {
     const unsigned Width = Op0->getType()->getScalarSizeInBits();
     const unsigned ShftCnt = ShAmt->getLimitedValue(Width);
-    const KnownBits YKnown = computeKnownBits(Y, /* Depth */ 0, Q);
+    const KnownBits YKnown = computeKnownBits(Y, Q);
     const unsigned EffWidthY = YKnown.countMaxActiveBits();
     if (EffWidthY <= ShftCnt) {
-      const KnownBits XKnown = computeKnownBits(X, /* Depth */ 0, Q);
+      const KnownBits XKnown = computeKnownBits(X, Q);
       const unsigned EffWidthX = XKnown.countMaxActiveBits();
       const APInt EffBitsY = APInt::getLowBitsSet(Width, EffWidthY);
       const APInt EffBitsX = APInt::getLowBitsSet(Width, EffWidthX) << ShftCnt;
@@ -2946,7 +2944,7 @@ static Value *simplifyICmpWithZero(CmpPredicate Pred, Value *LHS, Value *RHS,
       return getTrue(ITy);
     break;
   case ICmpInst::ICMP_SLT: {
-    KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
+    KnownBits LHSKnown = computeKnownBits(LHS, Q);
     if (LHSKnown.isNegative())
       return getTrue(ITy);
     if (LHSKnown.isNonNegative())
@@ -2954,7 +2952,7 @@ static Value *simplifyICmpWithZero(CmpPredicate Pred, Value *LHS, Value *RHS,
     break;
   }
   case ICmpInst::ICMP_SLE: {
-    KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
+    KnownBits LHSKnown = computeKnownBits(LHS, Q);
     if (LHSKnown.isNegative())
       return getTrue(ITy);
     if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, Q))
@@ -2962,7 +2960,7 @@ static Value *simplifyICmpWithZero(CmpPredicate Pred, Value *LHS, Value *RHS,
     break;
   }
   case ICmpInst::ICMP_SGE: {
-    KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
+    KnownBits LHSKnown = computeKnownBits(LHS, Q);
     if (LHSKnown.isNegative())
       return getFalse(ITy);
     if (LHSKnown.isNonNegative())
@@ -2970,7 +2968,7 @@ static Value *simplifyICmpWithZero(CmpPredicate Pred, Value *LHS, Value *RHS,
     break;
   }
   case ICmpInst::ICMP_SGT: {
-    KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
+    KnownBits LHSKnown = computeKnownBits(LHS, Q);
     if (LHSKnown.isNegative())
       return getFalse(ITy);
     if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, Q))
@@ -3112,8 +3110,8 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpPredicate Pred, BinaryOperator *LBO,
   // icmp pred (or X, Y), X
   if (match(LBO, m_c_Or(m_Value(Y), m_Specific(RHS)))) {
     if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
-      KnownBits RHSKnown = computeKnownBits(RHS, /* Depth */ 0, Q);
-      KnownBits YKnown = computeKnownBits(Y, /* Depth */ 0, Q);
+      KnownBits RHSKnown = computeKnownBits(RHS, Q);
+      KnownBits YKnown = computeKnownBits(Y, Q);
       if (RHSKnown.isNonNegative() && YKnown.isNegative())
         return Pred == ICmpInst::ICMP_SLT ? getTrue(ITy) : getFalse(ITy);
       if (RHSKnown.isNegative() || YKnown.isNonNegative())
@@ -3128,7 +3126,7 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpPredicate Pred, BinaryOperator *LBO,
       break;
     case ICmpInst::ICMP_SGT:
     case ICmpInst::ICMP_SGE: {
-      KnownBits Known = computeKnownBits(RHS, /* Depth */ 0, Q);
+      KnownBits Known = computeKnownBits(RHS, Q);
       if (!Known.isNonNegative())
         break;
       [[fallthrough]];
@@ -3139,7 +3137,7 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpPredicate Pred, BinaryOperator *LBO,
       return getFalse(ITy);
     case ICmpInst::ICMP_SLT:
     case ICmpInst::ICMP_SLE: {
-      KnownBits Known = computeKnownBits(RHS, /* Depth */ 0, Q);
+      KnownBits Known = computeKnownBits(RHS, Q);
       if (!Known.isNonNegative())
         break;
       [[fallthrough]];
@@ -4108,10 +4106,8 @@ static Value *simplifyFCmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,
   // This catches the 2 variable input case, constants are handled below as a
   // class-like compare.
   if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO) {
-    KnownFPClass RHSClass =
-        computeKnownFPClass(RHS, fcAllFlags, /*Depth=*/0, Q);
-    KnownFPClass LHSClass =
-        computeKnownFPClass(LHS, fcAllFlags, /*Depth=*/0, Q);
+    KnownFPClass RHSClass = computeKnownFPClass(RHS, fcAllFlags, Q);
+    KnownFPClass LHSClass = computeKnownFPClass(LHS, fcAllFlags, Q);
 
     if (FMF.noNaNs() ||
         (RHSClass.isKnownNeverNaN() && LHSClass.isKnownNeverNaN()))
@@ -4131,7 +4127,7 @@ static Value *simplifyFCmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,
                                                      fcAllFlags) {
     if (FullKnownClassLHS)
       return *FullKnownClassLHS;
-    return computeKnownFPClass(LHS, FMF, InterestedFlags, 0, Q);
+    return computeKnownFPClass(LHS, FMF, InterestedFlags, Q);
   };
 
   if (C && Q.CxtI) {
@@ -5711,7 +5707,7 @@ simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   // fadd X, 0 ==> X, when we know X is not -0
   if (canIgnoreSNaN(ExBehavior, FMF))
     if (match(Op1, m_PosZeroFP()) &&
-        (FMF.noSignedZeros() || cannotBeNegativeZero(Op0, /*Depth=*/0, Q)))
+        (FMF.noSignedZeros() || cannotBeNegativeZero(Op0, Q)))
       return Op0;
 
   if (!isDefaultFPEnvironment(ExBehavior, Rounding))
@@ -5773,7 +5769,7 @@ simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   // fsub X, -0 ==> X, when we know X is not -0
   if (canIgnoreSNaN(ExBehavior, FMF))
     if (match(Op1, m_NegZeroFP()) &&
-        (FMF.noSignedZeros() || cannotBeNegativeZero(Op0, /*Depth=*/0, Q)))
+        (FMF.noSignedZeros() || cannotBeNegativeZero(Op0, Q)))
       return Op0;
 
   // fsub -0.0, (fsub -0.0, X) ==> X
@@ -5841,8 +5837,7 @@ static Value *simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF,
     if (FMF.noNaNs() && FMF.noSignedZeros())
       return ConstantFP::getZero(Op0->getType());
 
-    KnownFPClass Known =
-        computeKnownFPClass(Op0, FMF, fcInf | fcNan, /*Depth=*/0, Q);
+    KnownFPClass Known = computeKnownFPClass(Op0, FMF, fcInf | fcNan, Q);
     if (Known.isKnownNever(fcInf | fcNan)) {
       // if nsz is set, return 0.0
       if (FMF.noSignedZeros())
@@ -6306,7 +6301,7 @@ static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0,
   Value *X;
   switch (IID) {
   case Intrinsic::fabs:
-    if (computeKnownFPSignBit(Op0, /*Depth=*/0, Q) == false)
+    if (computeKnownFPSignBit(Op0, Q) == false)
       return Op0;
     break;
   case Intrinsic::bswap:
@@ -6321,8 +6316,7 @@ static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0,
     break;
   case Intrinsic::ctpop: {
     // ctpop(X) -> 1 iff X is non-zero power of 2.
-    if (isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ false, 0, Q.AC, Q.CxtI,
-                               Q.DT))
+    if (isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ false, Q.AC, Q.CxtI, Q.DT))
       return ConstantInt::get(Op0->getType(), 1);
     // If everything but the lowest bit is zero, that bit is the pop-count. Ex:
     // ctpop(and X, 1) --> and X, 1
@@ -6511,7 +6505,7 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
 
     Constant *C;
     if (match(Op1, m_ImmConstant(C))) {
-      KnownBits PtrKnown = computeKnownBits(Op0, /*Depth=*/0, Q);
+      KnownBits PtrKnown = computeKnownBits(Op0, Q);
       // See if we only masking off bits we know are already zero due to
       // alignment.
       APInt IrrelevantPtrBits =
diff --git a/external/llvm-project/llvm/lib/Analysis/Lint.cpp b/external/llvm-project/llvm/lib/Analysis/Lint.cpp
index 7e540ea90789..1168005f48c0 100644
--- a/external/llvm-project/llvm/lib/Analysis/Lint.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/Lint.cpp
@@ -551,8 +551,7 @@ static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT,
 
   VectorType *VecTy = dyn_cast<VectorType>(V->getType());
   if (!VecTy) {
-    KnownBits Known =
-        computeKnownBits(V, DL, 0, AC, dyn_cast<Instruction>(V), DT);
+    KnownBits Known = computeKnownBits(V, DL, AC, dyn_cast<Instruction>(V), DT);
     return Known.isZero();
   }
 
diff --git a/external/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/external/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 26e11b6bc00a..00cdb66d8b77 100644
--- a/external/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -530,8 +530,10 @@ void RuntimePointerChecking::groupChecks(
     // equivalence class, the iteration order is deterministic.
     for (auto M : DepCands.members(Access)) {
       auto PointerI = PositionMap.find(M.getPointer());
-      assert(PointerI != PositionMap.end() &&
-             "pointer in equivalence class not found in PositionMap");
+      // If we can't find the pointer in PositionMap that means we can't
+      // generate a memcheck for it.
+      if (PointerI == PositionMap.end())
+        continue;
       for (unsigned Pointer : PointerI->second) {
         bool Merged = false;
         // Mark this pointer as seen.
@@ -693,10 +695,13 @@ class AccessAnalysis {
   /// non-intersection.
   ///
   /// Returns true if we need no check or if we do and we can generate them
-  /// (i.e. the pointers have computable bounds).
+  /// (i.e. the pointers have computable bounds). A return value of false means
+  /// we couldn't analyze and generate runtime checks for all pointers in the
+  /// loop, but if \p AllowPartial is set then we will have checks for those
+  /// pointers we could analyze.
   bool canCheckPtrAtRT(RuntimePointerChecking &RtCheck, Loop *TheLoop,
                        const DenseMap<Value *, const SCEV *> &Strides,
-                       Value *&UncomputablePtr);
+                       Value *&UncomputablePtr, bool AllowPartial);
 
   /// Goes over all memory accesses, checks whether a RT check is needed
   /// and builds sets of dependent accesses.
@@ -1001,24 +1006,22 @@ static void findForkedSCEVs(
       break;
     }
 
-    // Find the pointer type we need to extend to.
-    Type *IntPtrTy = SE->getEffectiveSCEVType(
-        SE->getSCEV(GEP->getPointerOperand())->getType());
+    Type *IntPtrTy = SE->getEffectiveSCEVType(GEP->getPointerOperandType());
 
     // Find the size of the type being pointed to. We only have a single
     // index term (guarded above) so we don't need to index into arrays or
     // structures, just get the size of the scalar value.
     const SCEV *Size = SE->getSizeOfExpr(IntPtrTy, SourceTy);
 
-    // Scale up the offsets by the size of the type, then add to the bases.
-    const SCEV *Scaled1 = SE->getMulExpr(
-        Size, SE->getTruncateOrSignExtend(get<0>(OffsetScevs[0]), IntPtrTy));
-    const SCEV *Scaled2 = SE->getMulExpr(
-        Size, SE->getTruncateOrSignExtend(get<0>(OffsetScevs[1]), IntPtrTy));
-    ScevList.emplace_back(SE->getAddExpr(get<0>(BaseScevs[0]), Scaled1),
-                          NeedsFreeze);
-    ScevList.emplace_back(SE->getAddExpr(get<0>(BaseScevs[1]), Scaled2),
-                          NeedsFreeze);
+    for (auto [B, O] : zip(BaseScevs, OffsetScevs)) {
+      const SCEV *Base = get<0>(B);
+      const SCEV *Offset = get<0>(O);
+
+      // Scale up the offsets by the size of the type, then add to the bases.
+      const SCEV *Scaled =
+          SE->getMulExpr(Size, SE->getTruncateOrSignExtend(Offset, IntPtrTy));
+      ScevList.emplace_back(SE->getAddExpr(Base, Scaled), NeedsFreeze);
+    }
     break;
   }
   case Instruction::Select: {
@@ -1028,10 +1031,9 @@ static void findForkedSCEVs(
     // then we just bail out and return the generic SCEV.
     findForkedSCEVs(SE, L, I->getOperand(1), ChildScevs, Depth);
     findForkedSCEVs(SE, L, I->getOperand(2), ChildScevs, Depth);
-    if (ChildScevs.size() == 2) {
-      ScevList.push_back(ChildScevs[0]);
-      ScevList.push_back(ChildScevs[1]);
-    } else
+    if (ChildScevs.size() == 2)
+      append_range(ScevList, ChildScevs);
+    else
       ScevList.emplace_back(Scev, !isGuaranteedNotToBeUndefOrPoison(Ptr));
     break;
   }
@@ -1044,10 +1046,9 @@ static void findForkedSCEVs(
       findForkedSCEVs(SE, L, I->getOperand(0), ChildScevs, Depth);
       findForkedSCEVs(SE, L, I->getOperand(1), ChildScevs, Depth);
     }
-    if (ChildScevs.size() == 2) {
-      ScevList.push_back(ChildScevs[0]);
-      ScevList.push_back(ChildScevs[1]);
-    } else
+    if (ChildScevs.size() == 2)
+      append_range(ScevList, ChildScevs);
+    else
       ScevList.emplace_back(Scev, !isGuaranteedNotToBeUndefOrPoison(Ptr));
     break;
   }
@@ -1074,12 +1075,9 @@ static void findForkedSCEVs(
       break;
     }
 
-    ScevList.emplace_back(
-        GetBinOpExpr(Opcode, get<0>(LScevs[0]), get<0>(RScevs[0])),
-        NeedsFreeze);
-    ScevList.emplace_back(
-        GetBinOpExpr(Opcode, get<0>(LScevs[1]), get<0>(RScevs[1])),
-        NeedsFreeze);
+    for (auto [L, R] : zip(LScevs, RScevs))
+      ScevList.emplace_back(GetBinOpExpr(Opcode, get<0>(L), get<0>(R)),
+                            NeedsFreeze);
     break;
   }
   default:
@@ -1181,8 +1179,8 @@ bool AccessAnalysis::createCheckForAccess(
 
 bool AccessAnalysis::canCheckPtrAtRT(
     RuntimePointerChecking &RtCheck, Loop *TheLoop,
-    const DenseMap<Value *, const SCEV *> &StridesMap,
-    Value *&UncomputablePtr) {
+    const DenseMap<Value *, const SCEV *> &StridesMap, Value *&UncomputablePtr,
+    bool AllowPartial) {
   // Find pointers with computable bounds. We are going to use this information
   // to place a runtime bound check.
   bool CanDoRT = true;
@@ -1275,7 +1273,8 @@ bool AccessAnalysis::canCheckPtrAtRT(
                                   /*Assume=*/true)) {
           CanDoAliasSetRT = false;
           UncomputablePtr = Access.getPointer();
-          break;
+          if (!AllowPartial)
+            break;
         }
       }
     }
@@ -1315,7 +1314,7 @@ bool AccessAnalysis::canCheckPtrAtRT(
     }
   }
 
-  if (MayNeedRTCheck && CanDoRT)
+  if (MayNeedRTCheck && (CanDoRT || AllowPartial))
     RtCheck.generateChecks(DepCands, IsDepCheckNeeded);
 
   LLVM_DEBUG(dbgs() << "LAA: We need to do " << RtCheck.getNumberOfChecks()
@@ -1329,7 +1328,7 @@ bool AccessAnalysis::canCheckPtrAtRT(
   bool CanDoRTIfNeeded = !RtCheck.Need || CanDoRT;
   assert(CanDoRTIfNeeded == (CanDoRT || !MayNeedRTCheck) &&
          "CanDoRTIfNeeded depends on RtCheck.Need");
-  if (!CanDoRTIfNeeded)
+  if (!CanDoRTIfNeeded && !AllowPartial)
     RtCheck.reset();
   return CanDoRTIfNeeded;
 }
@@ -1623,7 +1622,7 @@ bool llvm::isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL,
   std::optional<int64_t> Diff =
       getPointersDiff(ElemTyA, PtrA, ElemTyB, PtrB, DL, SE,
                       /*StrictCheck=*/true, CheckType);
-  return Diff && *Diff == 1;
+  return Diff == 1;
 }
 
 void MemoryDepChecker::addAccess(StoreInst *SI) {
@@ -2599,9 +2598,9 @@ bool LoopAccessInfo::analyzeLoop(AAResults *AA, const LoopInfo *LI,
   // Find pointers with computable bounds. We are going to use this information
   // to place a runtime bound check.
   Value *UncomputablePtr = nullptr;
-  bool CanDoRTIfNeeded = Accesses.canCheckPtrAtRT(
-      *PtrRtChecking, TheLoop, SymbolicStrides, UncomputablePtr);
-  if (!CanDoRTIfNeeded) {
+  HasCompletePtrRtChecking = Accesses.canCheckPtrAtRT(
+      *PtrRtChecking, TheLoop, SymbolicStrides, UncomputablePtr, AllowPartial);
+  if (!HasCompletePtrRtChecking) {
     const auto *I = dyn_cast_or_null<Instruction>(UncomputablePtr);
     recordAnalysis("CantIdentifyArrayBounds", I)
         << "cannot identify array bounds";
@@ -2629,11 +2628,12 @@ bool LoopAccessInfo::analyzeLoop(AAResults *AA, const LoopInfo *LI,
       PtrRtChecking->Need = true;
 
       UncomputablePtr = nullptr;
-      CanDoRTIfNeeded = Accesses.canCheckPtrAtRT(
-          *PtrRtChecking, TheLoop, SymbolicStrides, UncomputablePtr);
+      HasCompletePtrRtChecking =
+          Accesses.canCheckPtrAtRT(*PtrRtChecking, TheLoop, SymbolicStrides,
+                                   UncomputablePtr, AllowPartial);
 
       // Check that we found the bounds for the pointer.
-      if (!CanDoRTIfNeeded) {
+      if (!HasCompletePtrRtChecking) {
         auto *I = dyn_cast_or_null<Instruction>(UncomputablePtr);
         recordAnalysis("CantCheckMemDepsAtRunTime", I)
             << "cannot check memory dependencies at runtime";
@@ -2908,9 +2908,10 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
 LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
                                const TargetTransformInfo *TTI,
                                const TargetLibraryInfo *TLI, AAResults *AA,
-                               DominatorTree *DT, LoopInfo *LI)
+                               DominatorTree *DT, LoopInfo *LI,
+                               bool AllowPartial)
     : PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)),
-      PtrRtChecking(nullptr), TheLoop(L) {
+      PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial) {
   unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
   if (TTI && !TTI->enableScalableVectorization())
     // Scale the vector width by 2 as rough estimate to also consider
@@ -2959,6 +2960,8 @@ void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
 
   // List the pair of accesses need run-time checks to prove independence.
   PtrRtChecking->print(OS, Depth);
+  if (PtrRtChecking->Need && !HasCompletePtrRtChecking)
+    OS.indent(Depth) << "Generated run-time checks are incomplete\n";
   OS << "\n";
 
   OS.indent(Depth)
@@ -2978,12 +2981,15 @@ void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
   PSE->print(OS, Depth);
 }
 
-const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L) {
+const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
+                                                     bool AllowPartial) {
   const auto &[It, Inserted] = LoopAccessInfoMap.try_emplace(&L);
 
-  if (Inserted)
-    It->second =
-        std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT, &LI);
+  // We need to create the LoopAccessInfo if either we don't already have one,
+  // or if it was created with a different value of AllowPartial.
+  if (Inserted || It->second->hasAllowPartial() != AllowPartial)
+    It->second = std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT,
+                                                  &LI, AllowPartial);
 
   return *It->second;
 }
diff --git a/external/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp b/external/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp
index 6b7a3e1ffe34..e0b7f65d18a3 100644
--- a/external/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -428,6 +428,9 @@ llvm::getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI,
 Constant *llvm::getInitialValueOfAllocation(const Value *V,
                                             const TargetLibraryInfo *TLI,
                                             Type *Ty) {
+  if (isa<AllocaInst>(V))
+    return UndefValue::get(Ty);
+
   auto *Alloc = dyn_cast<CallBase>(V);
   if (!Alloc)
     return nullptr;
diff --git a/external/llvm-project/llvm/lib/Analysis/MemoryProfileInfo.cpp b/external/llvm-project/llvm/lib/Analysis/MemoryProfileInfo.cpp
index 347377522101..c08024a38ffc 100644
--- a/external/llvm-project/llvm/lib/Analysis/MemoryProfileInfo.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -46,6 +46,25 @@ cl::opt<unsigned> MinCallsiteColdBytePercent(
     cl::desc("Min percent of cold bytes at a callsite to discard non-cold "
              "contexts"));
 
+// Enable saving context size information for largest cold contexts, which can
+// be used to flag contexts for more aggressive cloning and reporting.
+cl::opt<unsigned> MinPercentMaxColdSize(
+    "memprof-min-percent-max-cold-size", cl::init(100), cl::Hidden,
+    cl::desc("Min percent of max cold bytes for critical cold context"));
+
+bool llvm::memprof::metadataIncludesAllContextSizeInfo() {
+  return MemProfReportHintedSizes || MinClonedColdBytePercent < 100;
+}
+
+bool llvm::memprof::metadataMayIncludeContextSizeInfo() {
+  return metadataIncludesAllContextSizeInfo() || MinPercentMaxColdSize < 100;
+}
+
+bool llvm::memprof::recordContextSizeInfoForAnalysis() {
+  return metadataMayIncludeContextSizeInfo() ||
+         MinCallsiteColdBytePercent < 100;
+}
+
 MDNode *llvm::memprof::buildCallstackMetadata(ArrayRef<uint64_t> CallStack,
                                               LLVMContext &Ctx) {
   SmallVector<Metadata *, 8> StackVals;
@@ -168,7 +187,8 @@ void CallStackTrie::addCallStack(MDNode *MIB) {
 static MDNode *createMIBNode(LLVMContext &Ctx, ArrayRef<uint64_t> MIBCallStack,
                              AllocationType AllocType,
                              ArrayRef<ContextTotalSize> ContextSizeInfo,
-                             uint64_t &TotalBytes, uint64_t &ColdBytes) {
+                             const uint64_t MaxColdSize, uint64_t &TotalBytes,
+                             uint64_t &ColdBytes) {
   SmallVector<Metadata *> MIBPayload(
       {buildCallstackMetadata(MIBCallStack, Ctx)});
   MIBPayload.push_back(
@@ -184,12 +204,21 @@ static MDNode *createMIBNode(LLVMContext &Ctx, ArrayRef<uint64_t> MIBCallStack,
 
   for (const auto &[FullStackId, TotalSize] : ContextSizeInfo) {
     TotalBytes += TotalSize;
-    if (AllocType == AllocationType::Cold)
+    bool LargeColdContext = false;
+    if (AllocType == AllocationType::Cold) {
       ColdBytes += TotalSize;
+      // If we have the max cold context size from summary information and have
+      // requested identification of contexts above a percentage of the max, see
+      // if this context qualifies.
+      if (MaxColdSize > 0 && MinPercentMaxColdSize < 100 &&
+          TotalSize * 100 >= MaxColdSize * MinPercentMaxColdSize)
+        LargeColdContext = true;
+    }
     // Only add the context size info as metadata if we need it in the thin
-    // link (currently if reporting of hinted sizes is enabled or we have
-    // specified a threshold for marking allocations cold after cloning).
-    if (MemProfReportHintedSizes || MinClonedColdBytePercent < 100) {
+    // link (currently if reporting of hinted sizes is enabled, we have
+    // specified a threshold for marking allocations cold after cloning, or we
+    // have identified this as a large cold context of interest above).
+    if (metadataIncludesAllContextSizeInfo() || LargeColdContext) {
       auto *FullStackIdMD = ValueAsMetadata::get(
           ConstantInt::get(Type::getInt64Ty(Ctx), FullStackId));
       auto *TotalSizeMD = ValueAsMetadata::get(
@@ -357,9 +386,9 @@ bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,
   if (hasSingleAllocType(Node->AllocTypes)) {
     std::vector<ContextTotalSize> ContextSizeInfo;
     collectContextSizeInfo(Node, ContextSizeInfo);
-    MIBNodes.push_back(createMIBNode(Ctx, MIBCallStack,
-                                     (AllocationType)Node->AllocTypes,
-                                     ContextSizeInfo, TotalBytes, ColdBytes));
+    MIBNodes.push_back(
+        createMIBNode(Ctx, MIBCallStack, (AllocationType)Node->AllocTypes,
+                      ContextSizeInfo, MaxColdSize, TotalBytes, ColdBytes));
     return true;
   }
 
@@ -413,7 +442,8 @@ bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,
   std::vector<ContextTotalSize> ContextSizeInfo;
   collectContextSizeInfo(Node, ContextSizeInfo);
   MIBNodes.push_back(createMIBNode(Ctx, MIBCallStack, AllocationType::NotCold,
-                                   ContextSizeInfo, TotalBytes, ColdBytes));
+                                   ContextSizeInfo, MaxColdSize, TotalBytes,
+                                   ColdBytes));
   return true;
 }
 
diff --git a/external/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/external/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 59fa1a4b03c3..a317ac471a23 100644
--- a/external/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -525,6 +525,7 @@ static void computeFunctionSummary(
       if (MemProfMD) {
         std::vector<MIBInfo> MIBs;
         std::vector<std::vector<ContextTotalSize>> ContextSizeInfos;
+        bool HasNonZeroContextSizeInfos = false;
         for (auto &MDOp : MemProfMD->operands()) {
           auto *MIBMD = cast<const MDNode>(MDOp);
           MDNode *StackNode = getMIBStackNode(MIBMD);
@@ -544,7 +545,8 @@ static void computeFunctionSummary(
           }
           // If we have context size information, collect it for inclusion in
           // the summary.
-          assert(MIBMD->getNumOperands() > 2 || !MemProfReportHintedSizes);
+          assert(MIBMD->getNumOperands() > 2 ||
+                 !metadataIncludesAllContextSizeInfo());
           if (MIBMD->getNumOperands() > 2) {
             std::vector<ContextTotalSize> ContextSizes;
             for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
@@ -558,14 +560,31 @@ static void computeFunctionSummary(
                                 ->getZExtValue();
               ContextSizes.push_back({FullStackId, TS});
             }
+            // Flag that we need to keep the ContextSizeInfos array for this
+            // alloc as it now contains non-zero context info sizes.
+            HasNonZeroContextSizeInfos = true;
             ContextSizeInfos.push_back(std::move(ContextSizes));
+          } else {
+            // The ContextSizeInfos must be in the same relative position as the
+            // associated MIB. In some cases we only include a ContextSizeInfo
+            // for a subset of MIBs in an allocation. To handle that, eagerly
+            // fill any MIB entries that don't have context size info metadata
+            // with a pair of 0s. Later on we will only use this array if it
+            // ends up containing any non-zero entries (see where we set
+            // HasNonZeroContextSizeInfos above).
+            ContextSizeInfos.push_back({{0, 0}});
           }
           MIBs.push_back(
               MIBInfo(getMIBAllocType(MIBMD), std::move(StackIdIndices)));
         }
         Allocs.push_back(AllocInfo(std::move(MIBs)));
-        assert(!ContextSizeInfos.empty() || !MemProfReportHintedSizes);
-        if (!ContextSizeInfos.empty()) {
+        assert(HasNonZeroContextSizeInfos ||
+               !metadataIncludesAllContextSizeInfo());
+        // We eagerly build the ContextSizeInfos array, but it will be filled
+        // with sub arrays of pairs of 0s if no MIBs on this alloc actually
+        // contained context size info metadata. Only save it if any MIBs had
+        // any such metadata.
+        if (HasNonZeroContextSizeInfos) {
           assert(Allocs.back().MIBs.size() == ContextSizeInfos.size());
           Allocs.back().ContextSizeInfos = std::move(ContextSizeInfos);
         }
diff --git a/external/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp b/external/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp
index 56cdfabccb66..2dfe625eb0dc 100644
--- a/external/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -6367,7 +6367,7 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) {
     // ask ValueTracking for known bits
     const SCEVUnknown *U = cast<SCEVUnknown>(S);
     unsigned Known =
-        computeKnownBits(U->getValue(), getDataLayout(), 0, &AC, nullptr, &DT)
+        computeKnownBits(U->getValue(), getDataLayout(), &AC, nullptr, &DT)
             .countMinTrailingZeros();
     return GetShiftedByZeros(Known);
   }
@@ -6485,8 +6485,8 @@ getRangeForUnknownRecurrence(const SCEVUnknown *U) {
   if (!TC || TC >= BitWidth)
     return FullSet;
 
-  auto KnownStart = computeKnownBits(Start, DL, 0, &AC, nullptr, &DT);
-  auto KnownStep = computeKnownBits(Step, DL, 0, &AC, nullptr, &DT);
+  auto KnownStart = computeKnownBits(Start, DL, &AC, nullptr, &DT);
+  auto KnownStep = computeKnownBits(Step, DL, &AC, nullptr, &DT);
   assert(KnownStart.getBitWidth() == BitWidth &&
          KnownStep.getBitWidth() == BitWidth);
 
@@ -6863,13 +6863,13 @@ const ConstantRange &ScalarEvolution::getRangeRef(
 
     // See if ValueTracking can give us a useful range.
     const DataLayout &DL = getDataLayout();
-    KnownBits Known = computeKnownBits(V, DL, 0, &AC, nullptr, &DT);
+    KnownBits Known = computeKnownBits(V, DL, &AC, nullptr, &DT);
     if (Known.getBitWidth() != BitWidth)
       Known = Known.zextOrTrunc(BitWidth);
 
     // ValueTracking may be able to compute a tighter result for the number of
     // sign bits than for the value of those sign bits.
-    unsigned NS = ComputeNumSignBits(V, DL, 0, &AC, nullptr, &DT);
+    unsigned NS = ComputeNumSignBits(V, DL, &AC, nullptr, &DT);
     if (U->getType()->isPointerTy()) {
       // If the pointer size is larger than the index size type, this can cause
       // NS to be larger than BitWidth. So compensate for this.
@@ -7818,8 +7818,7 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
         unsigned TZ = A.countr_zero();
         unsigned BitWidth = A.getBitWidth();
         KnownBits Known(BitWidth);
-        computeKnownBits(BO->LHS, Known, getDataLayout(),
-                         0, &AC, nullptr, &DT);
+        computeKnownBits(BO->LHS, Known, getDataLayout(), &AC, nullptr, &DT);
 
         APInt EffectiveMask =
             APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
@@ -9485,7 +9484,7 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
     // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
     // bitwidth(K) iterations.
     Value *FirstValue = PN->getIncomingValueForBlock(Predecessor);
-    KnownBits Known = computeKnownBits(FirstValue, DL, 0, &AC,
+    KnownBits Known = computeKnownBits(FirstValue, DL, &AC,
                                        Predecessor->getTerminator(), &DT);
     auto *Ty = cast<IntegerType>(RHS->getType());
     if (Known.isNonNegative())
diff --git a/external/llvm-project/llvm/lib/Analysis/ValueTracking.cpp b/external/llvm-project/llvm/lib/Analysis/ValueTracking.cpp
index fc19b2ccf796..9df667926faf 100644
--- a/external/llvm-project/llvm/lib/Analysis/ValueTracking.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/ValueTracking.cpp
@@ -132,44 +132,45 @@ static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
 }
 
 static void computeKnownBits(const Value *V, const APInt &DemandedElts,
-                             KnownBits &Known, unsigned Depth,
-                             const SimplifyQuery &Q);
+                             KnownBits &Known, const SimplifyQuery &Q,
+                             unsigned Depth);
 
-void llvm::computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth,
-                            const SimplifyQuery &Q) {
+void llvm::computeKnownBits(const Value *V, KnownBits &Known,
+                            const SimplifyQuery &Q, unsigned Depth) {
   // Since the number of lanes in a scalable vector is unknown at compile time,
   // we track one bit which is implicitly broadcast to all lanes.  This means
   // that all lanes in a scalable vector are considered demanded.
   auto *FVTy = dyn_cast<FixedVectorType>(V->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  ::computeKnownBits(V, DemandedElts, Known, Depth, Q);
+  ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
 }
 
 void llvm::computeKnownBits(const Value *V, KnownBits &Known,
-                            const DataLayout &DL, unsigned Depth,
-                            AssumptionCache *AC, const Instruction *CxtI,
-                            const DominatorTree *DT, bool UseInstrInfo) {
-  computeKnownBits(
-      V, Known, Depth,
-      SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
+                            const DataLayout &DL, AssumptionCache *AC,
+                            const Instruction *CxtI, const DominatorTree *DT,
+                            bool UseInstrInfo, unsigned Depth) {
+  computeKnownBits(V, Known,
+                   SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
+                   Depth);
 }
 
 KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
-                                 unsigned Depth, AssumptionCache *AC,
-                                 const Instruction *CxtI,
-                                 const DominatorTree *DT, bool UseInstrInfo) {
+                                 AssumptionCache *AC, const Instruction *CxtI,
+                                 const DominatorTree *DT, bool UseInstrInfo,
+                                 unsigned Depth) {
   return computeKnownBits(
-      V, Depth, SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
+      V, SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
 }
 
 KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
-                                 const DataLayout &DL, unsigned Depth,
-                                 AssumptionCache *AC, const Instruction *CxtI,
-                                 const DominatorTree *DT, bool UseInstrInfo) {
+                                 const DataLayout &DL, AssumptionCache *AC,
+                                 const Instruction *CxtI,
+                                 const DominatorTree *DT, bool UseInstrInfo,
+                                 unsigned Depth) {
   return computeKnownBits(
-      V, DemandedElts, Depth,
-      SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
+      V, DemandedElts,
+      SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
 }
 
 static bool haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
@@ -263,12 +264,13 @@ bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
 }
 
 bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
-                                  bool OrZero, unsigned Depth,
-                                  AssumptionCache *AC, const Instruction *CxtI,
-                                  const DominatorTree *DT, bool UseInstrInfo) {
+                                  bool OrZero, AssumptionCache *AC,
+                                  const Instruction *CxtI,
+                                  const DominatorTree *DT, bool UseInstrInfo,
+                                  unsigned Depth) {
   return ::isKnownToBeAPowerOfTwo(
-      V, OrZero, Depth,
-      SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
+      V, OrZero, SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
+      Depth);
 }
 
 static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
@@ -276,7 +278,7 @@ static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
 
 bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
                               unsigned Depth) {
-  return computeKnownBits(V, Depth, SQ).isNonNegative();
+  return computeKnownBits(V, SQ, Depth).isNonNegative();
 }
 
 bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
@@ -286,19 +288,19 @@ bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
 
   // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
   // this updated.
-  KnownBits Known = computeKnownBits(V, Depth, SQ);
+  KnownBits Known = computeKnownBits(V, SQ, Depth);
   return Known.isNonNegative() &&
          (Known.isNonZero() || isKnownNonZero(V, SQ, Depth));
 }
 
 bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
                            unsigned Depth) {
-  return computeKnownBits(V, Depth, SQ).isNegative();
+  return computeKnownBits(V, SQ, Depth).isNegative();
 }
 
 static bool isKnownNonEqual(const Value *V1, const Value *V2,
-                            const APInt &DemandedElts, unsigned Depth,
-                            const SimplifyQuery &Q);
+                            const APInt &DemandedElts, const SimplifyQuery &Q,
+                            unsigned Depth);
 
 bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
                            const SimplifyQuery &Q, unsigned Depth) {
@@ -308,40 +310,41 @@ bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
   auto *FVTy = dyn_cast<FixedVectorType>(V1->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  return ::isKnownNonEqual(V1, V2, DemandedElts, Depth, Q);
+  return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
 }
 
 bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
                              const SimplifyQuery &SQ, unsigned Depth) {
   KnownBits Known(Mask.getBitWidth());
-  computeKnownBits(V, Known, Depth, SQ);
+  computeKnownBits(V, Known, SQ, Depth);
   return Mask.isSubsetOf(Known.Zero);
 }
 
 static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
-                                   unsigned Depth, const SimplifyQuery &Q);
+                                   const SimplifyQuery &Q, unsigned Depth);
 
-static unsigned ComputeNumSignBits(const Value *V, unsigned Depth,
-                                   const SimplifyQuery &Q) {
+static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
+                                   unsigned Depth = 0) {
   auto *FVTy = dyn_cast<FixedVectorType>(V->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  return ComputeNumSignBits(V, DemandedElts, Depth, Q);
+  return ComputeNumSignBits(V, DemandedElts, Q, Depth);
 }
 
 unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
-                                  unsigned Depth, AssumptionCache *AC,
-                                  const Instruction *CxtI,
-                                  const DominatorTree *DT, bool UseInstrInfo) {
+                                  AssumptionCache *AC, const Instruction *CxtI,
+                                  const DominatorTree *DT, bool UseInstrInfo,
+                                  unsigned Depth) {
   return ::ComputeNumSignBits(
-      V, Depth, SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
+      V, SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
 }
 
 unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
-                                         unsigned Depth, AssumptionCache *AC,
+                                         AssumptionCache *AC,
                                          const Instruction *CxtI,
-                                         const DominatorTree *DT) {
-  unsigned SignBits = ComputeNumSignBits(V, DL, Depth, AC, CxtI, DT);
+                                         const DominatorTree *DT,
+                                         unsigned Depth) {
+  unsigned SignBits = ComputeNumSignBits(V, DL, AC, CxtI, DT, Depth);
   return V->getType()->getScalarSizeInBits() - SignBits + 1;
 }
 
@@ -349,24 +352,24 @@ static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
                                    bool NSW, bool NUW,
                                    const APInt &DemandedElts,
                                    KnownBits &KnownOut, KnownBits &Known2,
-                                   unsigned Depth, const SimplifyQuery &Q) {
-  computeKnownBits(Op1, DemandedElts, KnownOut, Depth + 1, Q);
+                                   const SimplifyQuery &Q, unsigned Depth) {
+  computeKnownBits(Op1, DemandedElts, KnownOut, Q, Depth + 1);
 
   // If one operand is unknown and we have no nowrap information,
   // the result will be unknown independently of the second operand.
   if (KnownOut.isUnknown() && !NSW && !NUW)
     return;
 
-  computeKnownBits(Op0, DemandedElts, Known2, Depth + 1, Q);
+  computeKnownBits(Op0, DemandedElts, Known2, Q, Depth + 1);
   KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, Known2, KnownOut);
 }
 
 static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
                                 bool NUW, const APInt &DemandedElts,
                                 KnownBits &Known, KnownBits &Known2,
-                                unsigned Depth, const SimplifyQuery &Q) {
-  computeKnownBits(Op1, DemandedElts, Known, Depth + 1, Q);
-  computeKnownBits(Op0, DemandedElts, Known2, Depth + 1, Q);
+                                const SimplifyQuery &Q, unsigned Depth) {
+  computeKnownBits(Op1, DemandedElts, Known, Q, Depth + 1);
+  computeKnownBits(Op0, DemandedElts, Known2, Q, Depth + 1);
 
   bool isKnownNegative = false;
   bool isKnownNonNegative = false;
@@ -792,15 +795,15 @@ static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
 }
 
 static void computeKnownBitsFromCond(const Value *V, Value *Cond,
-                                     KnownBits &Known, unsigned Depth,
-                                     const SimplifyQuery &SQ, bool Invert) {
+                                     KnownBits &Known, const SimplifyQuery &SQ,
+                                     bool Invert, unsigned Depth) {
   Value *A, *B;
   if (Depth < MaxAnalysisRecursionDepth &&
       match(Cond, m_LogicalOp(m_Value(A), m_Value(B)))) {
     KnownBits Known2(Known.getBitWidth());
     KnownBits Known3(Known.getBitWidth());
-    computeKnownBitsFromCond(V, A, Known2, Depth + 1, SQ, Invert);
-    computeKnownBitsFromCond(V, B, Known3, Depth + 1, SQ, Invert);
+    computeKnownBitsFromCond(V, A, Known2, SQ, Invert, Depth + 1);
+    computeKnownBitsFromCond(V, B, Known3, SQ, Invert, Depth + 1);
     if (Invert ? match(Cond, m_LogicalOr(m_Value(), m_Value()))
                : match(Cond, m_LogicalAnd(m_Value(), m_Value())))
       Known2 = Known2.unionWith(Known3);
@@ -831,14 +834,14 @@ static void computeKnownBitsFromCond(const Value *V, Value *Cond,
   }
 
   if (Depth < MaxAnalysisRecursionDepth && match(Cond, m_Not(m_Value(A))))
-    computeKnownBitsFromCond(V, A, Known, Depth + 1, SQ, !Invert);
+    computeKnownBitsFromCond(V, A, Known, SQ, !Invert, Depth + 1);
 }
 
 void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
-                                       unsigned Depth, const SimplifyQuery &Q) {
+                                       const SimplifyQuery &Q, unsigned Depth) {
   // Handle injected condition.
   if (Q.CC && Q.CC->AffectedValues.contains(V))
-    computeKnownBitsFromCond(V, Q.CC->Cond, Known, Depth, Q, Q.CC->Invert);
+    computeKnownBitsFromCond(V, Q.CC->Cond, Known, Q, Q.CC->Invert, Depth);
 
   if (!Q.CxtI)
     return;
@@ -848,13 +851,13 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
     for (BranchInst *BI : Q.DC->conditionsFor(V)) {
       BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
       if (Q.DT->dominates(Edge0, Q.CxtI->getParent()))
-        computeKnownBitsFromCond(V, BI->getCondition(), Known, Depth, Q,
-                                 /*Invert*/ false);
+        computeKnownBitsFromCond(V, BI->getCondition(), Known, Q,
+                                 /*Invert*/ false, Depth);
 
       BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
       if (Q.DT->dominates(Edge1, Q.CxtI->getParent()))
-        computeKnownBitsFromCond(V, BI->getCondition(), Known, Depth, Q,
-                                 /*Invert*/ true);
+        computeKnownBitsFromCond(V, BI->getCondition(), Known, Q,
+                                 /*Invert*/ true, Depth);
     }
 
     if (Known.hasConflict())
@@ -953,10 +956,10 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
 /// combined for all permitted shift amounts.
 static void computeKnownBitsFromShiftOperator(
     const Operator *I, const APInt &DemandedElts, KnownBits &Known,
-    KnownBits &Known2, unsigned Depth, const SimplifyQuery &Q,
+    KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
     function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
-  computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
-  computeKnownBits(I->getOperand(1), DemandedElts, Known, Depth + 1, Q);
+  computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
+  computeKnownBits(I->getOperand(1), DemandedElts, Known, Q, Depth + 1);
   // To limit compile-time impact, only query isKnownNonZero() if we know at
   // least something about the shift amount.
   bool ShAmtNonZero =
@@ -969,7 +972,7 @@ static void computeKnownBitsFromShiftOperator(
 static KnownBits
 getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
                          const KnownBits &KnownLHS, const KnownBits &KnownRHS,
-                         unsigned Depth, const SimplifyQuery &Q) {
+                         const SimplifyQuery &Q, unsigned Depth) {
   unsigned BitWidth = KnownLHS.getBitWidth();
   KnownBits KnownOut(BitWidth);
   bool IsAnd = false;
@@ -1026,7 +1029,7 @@ getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
        match(I, m_c_BinOp(m_Value(X), m_Sub(m_Deferred(X), m_Value(Y)))) ||
        match(I, m_c_BinOp(m_Value(X), m_Sub(m_Value(Y), m_Deferred(X)))))) {
     KnownBits KnownY(BitWidth);
-    computeKnownBits(Y, DemandedElts, KnownY, Depth + 1, Q);
+    computeKnownBits(Y, DemandedElts, KnownY, Q, Depth + 1);
     if (KnownY.countMinTrailingOnes() > 0) {
       if (IsAnd)
         KnownOut.Zero.setBit(0);
@@ -1038,8 +1041,8 @@ getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
 }
 
 static KnownBits computeKnownBitsForHorizontalOperation(
-    const Operator *I, const APInt &DemandedElts, unsigned Depth,
-    const SimplifyQuery &Q,
+    const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
+    unsigned Depth,
     const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
         KnownBitsFunc) {
   APInt DemandedEltsLHS, DemandedEltsRHS;
@@ -1050,8 +1053,8 @@ static KnownBits computeKnownBitsForHorizontalOperation(
   const auto ComputeForSingleOpFunc =
       [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
         return KnownBitsFunc(
-            computeKnownBits(Op, DemandedEltsOp, Depth + 1, Q),
-            computeKnownBits(Op, DemandedEltsOp << 1, Depth + 1, Q));
+            computeKnownBits(Op, DemandedEltsOp, Q, Depth + 1),
+            computeKnownBits(Op, DemandedEltsOp << 1, Q, Depth + 1));
       };
 
   if (DemandedEltsRHS.isZero())
@@ -1067,14 +1070,14 @@ static KnownBits computeKnownBitsForHorizontalOperation(
 KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
                                              const KnownBits &KnownLHS,
                                              const KnownBits &KnownRHS,
-                                             unsigned Depth,
-                                             const SimplifyQuery &SQ) {
+                                             const SimplifyQuery &SQ,
+                                             unsigned Depth) {
   auto *FVTy = dyn_cast<FixedVectorType>(I->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
 
-  return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Depth,
-                                  SQ);
+  return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, SQ,
+                                  Depth);
 }
 
 ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
@@ -1097,15 +1100,15 @@ ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
 }
 
 void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
-                                       Value *Arm, bool Invert, unsigned Depth,
-                                       const SimplifyQuery &Q) {
+                                       Value *Arm, bool Invert,
+                                       const SimplifyQuery &Q, unsigned Depth) {
   // If we have a constant arm, we are done.
   if (Known.isConstant())
     return;
 
   // See what condition implies about the bits of the select arm.
   KnownBits CondRes(Known.getBitWidth());
-  computeKnownBitsFromCond(Arm, Cond, CondRes, Depth + 1, Q, Invert);
+  computeKnownBitsFromCond(Arm, Cond, CondRes, Q, Invert, Depth + 1);
   // If we don't get any information from the condition, no reason to
   // proceed.
   if (CondRes.isUnknown())
@@ -1190,8 +1193,9 @@ static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
 
 static void computeKnownBitsFromOperator(const Operator *I,
                                          const APInt &DemandedElts,
-                                         KnownBits &Known, unsigned Depth,
-                                         const SimplifyQuery &Q) {
+                                         KnownBits &Known,
+                                         const SimplifyQuery &Q,
+                                         unsigned Depth) {
   unsigned BitWidth = Known.getBitWidth();
 
   KnownBits Known2(BitWidth);
@@ -1203,40 +1207,40 @@ static void computeKnownBitsFromOperator(const Operator *I,
       computeKnownBitsFromRangeMetadata(*MD, Known);
     break;
   case Instruction::And:
-    computeKnownBits(I->getOperand(1), DemandedElts, Known, Depth + 1, Q);
-    computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+    computeKnownBits(I->getOperand(1), DemandedElts, Known, Q, Depth + 1);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
 
-    Known = getKnownBitsFromAndXorOr(I, DemandedElts, Known2, Known, Depth, Q);
+    Known = getKnownBitsFromAndXorOr(I, DemandedElts, Known2, Known, Q, Depth);
     break;
   case Instruction::Or:
-    computeKnownBits(I->getOperand(1), DemandedElts, Known, Depth + 1, Q);
-    computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+    computeKnownBits(I->getOperand(1), DemandedElts, Known, Q, Depth + 1);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
 
-    Known = getKnownBitsFromAndXorOr(I, DemandedElts, Known2, Known, Depth, Q);
+    Known = getKnownBitsFromAndXorOr(I, DemandedElts, Known2, Known, Q, Depth);
     break;
   case Instruction::Xor:
-    computeKnownBits(I->getOperand(1), DemandedElts, Known, Depth + 1, Q);
-    computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+    computeKnownBits(I->getOperand(1), DemandedElts, Known, Q, Depth + 1);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
 
-    Known = getKnownBitsFromAndXorOr(I, DemandedElts, Known2, Known, Depth, Q);
+    Known = getKnownBitsFromAndXorOr(I, DemandedElts, Known2, Known, Q, Depth);
     break;
   case Instruction::Mul: {
     bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
     bool NUW = Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(I));
     computeKnownBitsMul(I->getOperand(0), I->getOperand(1), NSW, NUW,
-                        DemandedElts, Known, Known2, Depth, Q);
+                        DemandedElts, Known, Known2, Q, Depth);
     break;
   }
   case Instruction::UDiv: {
-    computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
     Known =
         KnownBits::udiv(Known, Known2, Q.IIQ.isExact(cast<BinaryOperator>(I)));
     break;
   }
   case Instruction::SDiv: {
-    computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
     Known =
         KnownBits::sdiv(Known, Known2, Q.IIQ.isExact(cast<BinaryOperator>(I)));
     break;
@@ -1244,8 +1248,8 @@ static void computeKnownBitsFromOperator(const Operator *I,
   case Instruction::Select: {
     auto ComputeForArm = [&](Value *Arm, bool Invert) {
       KnownBits Res(Known.getBitWidth());
-      computeKnownBits(Arm, DemandedElts, Res, Depth + 1, Q);
-      adjustKnownBitsForSelectArm(Res, I->getOperand(0), Arm, Invert, Depth, Q);
+      computeKnownBits(Arm, DemandedElts, Res, Q, Depth + 1);
+      adjustKnownBitsForSelectArm(Res, I->getOperand(0), Arm, Invert, Q, Depth);
       return Res;
     };
     // Only known if known in both the LHS and RHS.
@@ -1279,7 +1283,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
 
     assert(SrcBitWidth && "SrcBitWidth can't be zero");
     Known = Known.anyextOrTrunc(SrcBitWidth);
-    computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
     if (auto *Inst = dyn_cast<PossiblyNonNegInst>(I);
         Inst && Inst->hasNonNeg() && !Known.isNegative())
       Known.makeNonNegative();
@@ -1292,7 +1296,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
         // TODO: For now, not handling conversions like:
         // (bitcast i64 %x to <2 x i32>)
         !I->getType()->isVectorTy()) {
-      computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
+      computeKnownBits(I->getOperand(0), Known, Q, Depth + 1);
       break;
     }
 
@@ -1302,7 +1306,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
         V->getType()->isFPOrFPVectorTy()) {
       Type *FPType = V->getType()->getScalarType();
       KnownFPClass Result =
-          computeKnownFPClass(V, DemandedElts, fcAllFlags, Depth + 1, Q);
+          computeKnownFPClass(V, DemandedElts, fcAllFlags, Q, Depth + 1);
       FPClassTest FPClasses = Result.KnownFPClasses;
 
       // TODO: Treat it as zero/poison if the use of I is unreachable.
@@ -1370,8 +1374,8 @@ static void computeKnownBitsFromOperator(const Operator *I,
 
       KnownBits KnownSrc(SubBitWidth);
       for (unsigned i = 0; i != SubScale; ++i) {
-        computeKnownBits(I->getOperand(0), SubDemandedElts.shl(i), KnownSrc,
-                         Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), SubDemandedElts.shl(i), KnownSrc, Q,
+                         Depth + 1);
         unsigned ShiftElt = Q.DL.isLittleEndian() ? i : SubScale - 1 - i;
         Known.insertBits(KnownSrc, ShiftElt * SubBitWidth);
       }
@@ -1383,7 +1387,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
     unsigned SrcBitWidth = I->getOperand(0)->getType()->getScalarSizeInBits();
 
     Known = Known.trunc(SrcBitWidth);
-    computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
     // If the sign bit of the input is known set or clear, then we know the
     // top bits of the result.
     Known = Known.sext(BitWidth);
@@ -1396,7 +1400,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
                          bool ShAmtNonZero) {
       return KnownBits::shl(KnownVal, KnownAmt, NUW, NSW, ShAmtNonZero);
     };
-    computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Depth, Q,
+    computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
                                       KF);
     // Trailing zeros of a right-shifted constant never decrease.
     const APInt *C;
@@ -1410,7 +1414,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
                       bool ShAmtNonZero) {
       return KnownBits::lshr(KnownVal, KnownAmt, ShAmtNonZero, Exact);
     };
-    computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Depth, Q,
+    computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
                                       KF);
     // Leading zeros of a left-shifted constant never decrease.
     const APInt *C;
@@ -1424,7 +1428,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
                       bool ShAmtNonZero) {
       return KnownBits::ashr(KnownVal, KnownAmt, ShAmtNonZero, Exact);
     };
-    computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Depth, Q,
+    computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
                                       KF);
     break;
   }
@@ -1432,25 +1436,25 @@ static void computeKnownBitsFromOperator(const Operator *I,
     bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
     bool NUW = Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(I));
     computeKnownBitsAddSub(false, I->getOperand(0), I->getOperand(1), NSW, NUW,
-                           DemandedElts, Known, Known2, Depth, Q);
+                           DemandedElts, Known, Known2, Q, Depth);
     break;
   }
   case Instruction::Add: {
     bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
     bool NUW = Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(I));
     computeKnownBitsAddSub(true, I->getOperand(0), I->getOperand(1), NSW, NUW,
-                           DemandedElts, Known, Known2, Depth, Q);
+                           DemandedElts, Known, Known2, Q, Depth);
     break;
   }
   case Instruction::SRem:
-    computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
     Known = KnownBits::srem(Known, Known2);
     break;
 
   case Instruction::URem:
-    computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+    computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
     Known = KnownBits::urem(Known, Known2);
     break;
   case Instruction::Alloca:
@@ -1459,7 +1463,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
   case Instruction::GetElementPtr: {
     // Analyze all of the subscripts of this getelementptr instruction
     // to determine if we can prove known low zero bits.
-    computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
+    computeKnownBits(I->getOperand(0), Known, Q, Depth + 1);
     // Accumulate the constant indices in a separate variable
     // to minimize the number of calls to computeForAddSub.
     unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(I->getType());
@@ -1527,7 +1531,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
       }
 
       KnownBits IndexBits =
-          computeKnownBits(Index, Depth + 1, Q).sextOrTrunc(IndexWidth);
+          computeKnownBits(Index, Q, Depth + 1).sextOrTrunc(IndexWidth);
       KnownBits ScalingFactor(IndexWidth);
       // Multiply by current sizeof type.
       // &A[i] == A + i * sizeof(*A[i]).
@@ -1582,7 +1586,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
         // add sufficient tests to cover.
         SimplifyQuery RecQ = Q.getWithoutCondContext();
         RecQ.CxtI = P;
-        computeKnownBits(R, DemandedElts, Known2, Depth + 1, RecQ);
+        computeKnownBits(R, DemandedElts, Known2, RecQ, Depth + 1);
         switch (Opcode) {
         case Instruction::Shl:
           // A shl recurrence will only increase the tailing zeros
@@ -1625,12 +1629,12 @@ static void computeKnownBitsFromOperator(const Operator *I,
         // Ok, we have a PHI of the form L op= R. Check for low
         // zero bits.
         RecQ.CxtI = RInst;
-        computeKnownBits(R, DemandedElts, Known2, Depth + 1, RecQ);
+        computeKnownBits(R, DemandedElts, Known2, RecQ, Depth + 1);
 
         // We need to take the minimum number of known bits
         KnownBits Known3(BitWidth);
         RecQ.CxtI = LInst;
-        computeKnownBits(L, DemandedElts, Known3, Depth + 1, RecQ);
+        computeKnownBits(L, DemandedElts, Known3, RecQ, Depth + 1);
 
         Known.Zero.setLowBits(std::min(Known2.countMinTrailingZeros(),
                                        Known3.countMinTrailingZeros()));
@@ -1720,8 +1724,8 @@ static void computeKnownBitsFromOperator(const Operator *I,
         // want to waste time spinning around in loops.
         // TODO: See if we can base recursion limiter on number of incoming phi
         // edges so we don't overly clamp analysis.
-        computeKnownBits(IncValue, DemandedElts, Known2,
-                         MaxAnalysisRecursionDepth - 1, RecQ);
+        computeKnownBits(IncValue, DemandedElts, Known2, RecQ,
+                         MaxAnalysisRecursionDepth - 1);
 
         // See if we can further use a conditional branch into the phi
         // to help us determine the range of the value.
@@ -1781,7 +1785,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
 
     if (const Value *RV = CB->getReturnedArgOperand()) {
       if (RV->getType() == I->getType()) {
-        computeKnownBits(RV, Known2, Depth + 1, Q);
+        computeKnownBits(RV, Known2, Q, Depth + 1);
         Known = Known.unionWith(Known2);
         // If the function doesn't return properly for all input values
         // (e.g. unreachable exits) then there might be conflicts between the
@@ -1796,23 +1800,23 @@ static void computeKnownBitsFromOperator(const Operator *I,
       default:
         break;
       case Intrinsic::abs: {
-        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
         bool IntMinIsPoison = match(II->getArgOperand(1), m_One());
         Known = Known2.abs(IntMinIsPoison);
         break;
       }
       case Intrinsic::bitreverse:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
         Known.Zero |= Known2.Zero.reverseBits();
         Known.One |= Known2.One.reverseBits();
         break;
       case Intrinsic::bswap:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
         Known.Zero |= Known2.Zero.byteSwap();
         Known.One |= Known2.One.byteSwap();
         break;
       case Intrinsic::ctlz: {
-        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
         // If we have a known 1, its position is our upper bound.
         unsigned PossibleLZ = Known2.countMaxLeadingZeros();
         // If this call is poison for 0 input, the result will be less than 2^n.
@@ -1823,7 +1827,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
         break;
       }
       case Intrinsic::cttz: {
-        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
         // If we have a known 1, its position is our upper bound.
         unsigned PossibleTZ = Known2.countMaxTrailingZeros();
         // If this call is poison for 0 input, the result will be less than 2^n.
@@ -1834,7 +1838,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
         break;
       }
       case Intrinsic::ctpop: {
-        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
         // We can bound the space the count needs.  Also, bits known to be zero
         // can't contribute to the population.
         unsigned BitsPossiblySet = Known2.countMaxPopulation();
@@ -1856,8 +1860,8 @@ static void computeKnownBitsFromOperator(const Operator *I,
           ShiftAmt = BitWidth - ShiftAmt;
 
         KnownBits Known3(BitWidth);
-        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known3, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known2, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known3, Q, Depth + 1);
 
         Known.Zero =
             Known2.Zero.shl(ShiftAmt) | Known3.Zero.lshr(BitWidth - ShiftAmt);
@@ -1866,29 +1870,29 @@ static void computeKnownBitsFromOperator(const Operator *I,
         break;
       }
       case Intrinsic::uadd_sat:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::uadd_sat(Known, Known2);
         break;
       case Intrinsic::usub_sat:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::usub_sat(Known, Known2);
         break;
       case Intrinsic::sadd_sat:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::sadd_sat(Known, Known2);
         break;
       case Intrinsic::ssub_sat:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::ssub_sat(Known, Known2);
         break;
         // Vec reverse preserves bits from input vec.
       case Intrinsic::vector_reverse:
-        computeKnownBits(I->getOperand(0), DemandedElts.reverseBits(), Known,
-                         Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts.reverseBits(), Known, Q,
+                         Depth + 1);
         break;
         // for min/max/and/or reduce, any bit common to each element in the
         // input vec is set in the output.
@@ -1898,10 +1902,10 @@ static void computeKnownBitsFromOperator(const Operator *I,
       case Intrinsic::vector_reduce_umin:
       case Intrinsic::vector_reduce_smax:
       case Intrinsic::vector_reduce_smin:
-        computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), Known, Q, Depth + 1);
         break;
       case Intrinsic::vector_reduce_xor: {
-        computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), Known, Q, Depth + 1);
         // The zeros common to all vecs are zero in the output.
         // If the number of elements is odd, then the common ones remain. If the
         // number of elements is even, then the common ones becomes zeros.
@@ -1916,33 +1920,33 @@ static void computeKnownBitsFromOperator(const Operator *I,
         break;
       }
       case Intrinsic::umin:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::umin(Known, Known2);
         break;
       case Intrinsic::umax:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::umax(Known, Known2);
         break;
       case Intrinsic::smin:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::smin(Known, Known2);
         unionWithMinMaxIntrinsicClamp(II, Known);
         break;
       case Intrinsic::smax:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::smax(Known, Known2);
         unionWithMinMaxIntrinsicClamp(II, Known);
         break;
       case Intrinsic::ptrmask: {
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
 
         const Value *Mask = I->getOperand(1);
         Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
-        computeKnownBits(Mask, DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(Mask, DemandedElts, Known2, Q, Depth + 1);
         // TODO: 1-extend would be more precise.
         Known &= Known2.anyextOrTrunc(BitWidth);
         break;
@@ -1950,15 +1954,15 @@ static void computeKnownBitsFromOperator(const Operator *I,
       case Intrinsic::x86_sse2_pmulh_w:
       case Intrinsic::x86_avx2_pmulh_w:
       case Intrinsic::x86_avx512_pmulh_w_512:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::mulhs(Known, Known2);
         break;
       case Intrinsic::x86_sse2_pmulhu_w:
       case Intrinsic::x86_avx2_pmulhu_w:
       case Intrinsic::x86_avx512_pmulhu_w_512:
-        computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth + 1, Q);
-        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Depth + 1, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+        computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
         Known = KnownBits::mulhu(Known, Known2);
         break;
       case Intrinsic::x86_sse42_crc32_64_64:
@@ -1969,7 +1973,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
       case Intrinsic::x86_avx2_phadd_d:
       case Intrinsic::x86_avx2_phadd_w: {
         Known = computeKnownBitsForHorizontalOperation(
-            I, DemandedElts, Depth, Q,
+            I, DemandedElts, Q, Depth,
             [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
               return KnownBits::add(KnownLHS, KnownRHS);
             });
@@ -1977,8 +1981,8 @@ static void computeKnownBitsFromOperator(const Operator *I,
       }
       case Intrinsic::x86_ssse3_phadd_sw_128:
       case Intrinsic::x86_avx2_phadd_sw: {
-        Known = computeKnownBitsForHorizontalOperation(I, DemandedElts, Depth,
-                                                       Q, KnownBits::sadd_sat);
+        Known = computeKnownBitsForHorizontalOperation(
+            I, DemandedElts, Q, Depth, KnownBits::sadd_sat);
         break;
       }
       case Intrinsic::x86_ssse3_phsub_d_128:
@@ -1986,7 +1990,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
       case Intrinsic::x86_avx2_phsub_d:
       case Intrinsic::x86_avx2_phsub_w: {
         Known = computeKnownBitsForHorizontalOperation(
-            I, DemandedElts, Depth, Q,
+            I, DemandedElts, Q, Depth,
             [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
               return KnownBits::sub(KnownLHS, KnownRHS);
             });
@@ -1994,8 +1998,8 @@ static void computeKnownBitsFromOperator(const Operator *I,
       }
       case Intrinsic::x86_ssse3_phsub_sw_128:
       case Intrinsic::x86_avx2_phsub_sw: {
-        Known = computeKnownBitsForHorizontalOperation(I, DemandedElts, Depth,
-                                                       Q, KnownBits::ssub_sat);
+        Known = computeKnownBitsForHorizontalOperation(
+            I, DemandedElts, Q, Depth, KnownBits::ssub_sat);
         break;
       }
       case Intrinsic::riscv_vsetvli:
@@ -2049,14 +2053,14 @@ static void computeKnownBitsFromOperator(const Operator *I,
     Known.Zero.setAllBits();
     if (!!DemandedLHS) {
       const Value *LHS = Shuf->getOperand(0);
-      computeKnownBits(LHS, DemandedLHS, Known, Depth + 1, Q);
+      computeKnownBits(LHS, DemandedLHS, Known, Q, Depth + 1);
       // If we don't know any bits, early out.
       if (Known.isUnknown())
         break;
     }
     if (!!DemandedRHS) {
       const Value *RHS = Shuf->getOperand(1);
-      computeKnownBits(RHS, DemandedRHS, Known2, Depth + 1, Q);
+      computeKnownBits(RHS, DemandedRHS, Known2, Q, Depth + 1);
       Known = Known.intersectWith(Known2);
     }
     break;
@@ -2081,14 +2085,14 @@ static void computeKnownBitsFromOperator(const Operator *I,
     Known.One.setAllBits();
     Known.Zero.setAllBits();
     if (NeedsElt) {
-      computeKnownBits(Elt, Known, Depth + 1, Q);
+      computeKnownBits(Elt, Known, Q, Depth + 1);
       // If we don't know any bits, early out.
       if (Known.isUnknown())
         break;
     }
 
     if (!DemandedVecElts.isZero()) {
-      computeKnownBits(Vec, DemandedVecElts, Known2, Depth + 1, Q);
+      computeKnownBits(Vec, DemandedVecElts, Known2, Q, Depth + 1);
       Known = Known.intersectWith(Known2);
     }
     break;
@@ -2108,7 +2112,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
     APInt DemandedVecElts = APInt::getAllOnes(NumElts);
     if (CIdx && CIdx->getValue().ult(NumElts))
       DemandedVecElts = APInt::getOneBitSet(NumElts, CIdx->getZExtValue());
-    computeKnownBits(Vec, DemandedVecElts, Known, Depth + 1, Q);
+    computeKnownBits(Vec, DemandedVecElts, Known, Q, Depth + 1);
     break;
   }
   case Instruction::ExtractValue:
@@ -2122,18 +2126,18 @@ static void computeKnownBitsFromOperator(const Operator *I,
         case Intrinsic::sadd_with_overflow:
           computeKnownBitsAddSub(
               true, II->getArgOperand(0), II->getArgOperand(1), /*NSW=*/false,
-              /* NUW=*/false, DemandedElts, Known, Known2, Depth, Q);
+              /* NUW=*/false, DemandedElts, Known, Known2, Q, Depth);
           break;
         case Intrinsic::usub_with_overflow:
         case Intrinsic::ssub_with_overflow:
           computeKnownBitsAddSub(
               false, II->getArgOperand(0), II->getArgOperand(1), /*NSW=*/false,
-              /* NUW=*/false, DemandedElts, Known, Known2, Depth, Q);
+              /* NUW=*/false, DemandedElts, Known, Known2, Q, Depth);
           break;
         case Intrinsic::umul_with_overflow:
         case Intrinsic::smul_with_overflow:
           computeKnownBitsMul(II->getArgOperand(0), II->getArgOperand(1), false,
-                              false, DemandedElts, Known, Known2, Depth, Q);
+                              false, DemandedElts, Known, Known2, Q, Depth);
           break;
         }
       }
@@ -2142,7 +2146,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
   case Instruction::Freeze:
     if (isGuaranteedNotToBePoison(I->getOperand(0), Q.AC, Q.CxtI, Q.DT,
                                   Depth + 1))
-      computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
+      computeKnownBits(I->getOperand(0), Known, Q, Depth + 1);
     break;
   }
 }
@@ -2150,18 +2154,18 @@ static void computeKnownBitsFromOperator(const Operator *I,
 /// Determine which bits of V are known to be either zero or one and return
 /// them.
 KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
-                                 unsigned Depth, const SimplifyQuery &Q) {
+                                 const SimplifyQuery &Q, unsigned Depth) {
   KnownBits Known(getBitWidth(V->getType(), Q.DL));
-  ::computeKnownBits(V, DemandedElts, Known, Depth, Q);
+  ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
   return Known;
 }
 
 /// Determine which bits of V are known to be either zero or one and return
 /// them.
-KnownBits llvm::computeKnownBits(const Value *V, unsigned Depth,
-                                 const SimplifyQuery &Q) {
+KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
+                                 unsigned Depth) {
   KnownBits Known(getBitWidth(V->getType(), Q.DL));
-  computeKnownBits(V, Known, Depth, Q);
+  computeKnownBits(V, Known, Q, Depth);
   return Known;
 }
 
@@ -2181,8 +2185,8 @@ KnownBits llvm::computeKnownBits(const Value *V, unsigned Depth,
 /// same width as the vector element, and the bit is set only if it is true
 /// for all of the demanded elements in the vector specified by DemandedElts.
 void computeKnownBits(const Value *V, const APInt &DemandedElts,
-                      KnownBits &Known, unsigned Depth,
-                      const SimplifyQuery &Q) {
+                      KnownBits &Known, const SimplifyQuery &Q,
+                      unsigned Depth) {
   if (!DemandedElts) {
     // No demanded elts, better to assume we don't know anything.
     Known.resetAll();
@@ -2296,12 +2300,12 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts,
   // the bits of its aliasee.
   if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
     if (!GA->isInterposable())
-      computeKnownBits(GA->getAliasee(), Known, Depth + 1, Q);
+      computeKnownBits(GA->getAliasee(), Known, Q, Depth + 1);
     return;
   }
 
   if (const Operator *I = dyn_cast<Operator>(V))
-    computeKnownBitsFromOperator(I, DemandedElts, Known, Depth, Q);
+    computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
   else if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
     if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
       Known = CR->toKnownBits();
@@ -2317,13 +2321,13 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts,
   // Therefore, we run them after computeKnownBitsFromOperator.
 
   // Check whether we can determine known bits from context such as assumes.
-  computeKnownBitsFromContext(V, Known, Depth, Q);
+  computeKnownBitsFromContext(V, Known, Q, Depth);
 }
 
 /// Try to detect a recurrence that the value of the induction variable is
 /// always a power of two (or zero).
 static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
-                                   unsigned Depth, SimplifyQuery &Q) {
+                                   SimplifyQuery &Q, unsigned Depth) {
   BinaryOperator *BO = nullptr;
   Value *Start = nullptr, *Step = nullptr;
   if (!matchSimpleRecurrence(PN, BO, Start, Step))
@@ -2335,7 +2339,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
       // Initial value comes from a different BB, need to adjust context
       // instruction for analysis.
       Q.CxtI = PN->getIncomingBlock(U)->getTerminator();
-      if (!isKnownToBeAPowerOfTwo(Start, OrZero, Depth, Q))
+      if (!isKnownToBeAPowerOfTwo(Start, OrZero, Q, Depth))
         return false;
     }
   }
@@ -2351,7 +2355,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
     // Power of two is closed under multiplication.
     return (OrZero || Q.IIQ.hasNoUnsignedWrap(BO) ||
             Q.IIQ.hasNoSignedWrap(BO)) &&
-           isKnownToBeAPowerOfTwo(Step, OrZero, Depth, Q);
+           isKnownToBeAPowerOfTwo(Step, OrZero, Q, Depth);
   case Instruction::SDiv:
     // Start value must not be signmask for signed division, so simply being a
     // power of two is not sufficient, and it has to be a constant.
@@ -2363,7 +2367,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
     // If OrZero is false, cannot guarantee induction variable is non-zero after
     // division, same for Shr, unless it is exact division.
     return (OrZero || Q.IIQ.isExact(BO)) &&
-           isKnownToBeAPowerOfTwo(Step, false, Depth, Q);
+           isKnownToBeAPowerOfTwo(Step, false, Q, Depth);
   case Instruction::Shl:
     return OrZero || Q.IIQ.hasNoUnsignedWrap(BO) || Q.IIQ.hasNoSignedWrap(BO);
   case Instruction::AShr:
@@ -2400,8 +2404,8 @@ static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
 /// bit set when defined. For vectors return true if every element is known to
 /// be a power of two when defined. Supports values with integer or pointer
 /// types and vectors of integers.
-bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
-                                  const SimplifyQuery &Q) {
+bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
+                                  const SimplifyQuery &Q, unsigned Depth) {
   assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
 
   if (isa<Constant>(V))
@@ -2469,30 +2473,30 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
 
   switch (I->getOpcode()) {
   case Instruction::ZExt:
-    return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q);
+    return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Q, Depth);
   case Instruction::Trunc:
-    return OrZero && isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q);
+    return OrZero && isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Q, Depth);
   case Instruction::Shl:
     if (OrZero || Q.IIQ.hasNoUnsignedWrap(I) || Q.IIQ.hasNoSignedWrap(I))
-      return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q);
+      return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Q, Depth);
     return false;
   case Instruction::LShr:
     if (OrZero || Q.IIQ.isExact(cast<BinaryOperator>(I)))
-      return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q);
+      return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Q, Depth);
     return false;
   case Instruction::UDiv:
     if (Q.IIQ.isExact(cast<BinaryOperator>(I)))
-      return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q);
+      return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Q, Depth);
     return false;
   case Instruction::Mul:
-    return isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Depth, Q) &&
-           isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q) &&
+    return isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Q, Depth) &&
+           isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Q, Depth) &&
            (OrZero || isKnownNonZero(I, Q, Depth));
   case Instruction::And:
     // A power of two and'd with anything is a power of two or zero.
     if (OrZero &&
-        (isKnownToBeAPowerOfTwo(I->getOperand(1), /*OrZero*/ true, Depth, Q) ||
-         isKnownToBeAPowerOfTwo(I->getOperand(0), /*OrZero*/ true, Depth, Q)))
+        (isKnownToBeAPowerOfTwo(I->getOperand(1), /*OrZero*/ true, Q, Depth) ||
+         isKnownToBeAPowerOfTwo(I->getOperand(0), /*OrZero*/ true, Q, Depth)))
       return true;
     // X & (-X) is always a power of two or zero.
     if (match(I->getOperand(0), m_Neg(m_Specific(I->getOperand(1)))) ||
@@ -2507,19 +2511,19 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
         Q.IIQ.hasNoSignedWrap(VOBO)) {
       if (match(I->getOperand(0),
                 m_c_And(m_Specific(I->getOperand(1)), m_Value())) &&
-          isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Depth, Q))
+          isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Q, Depth))
         return true;
       if (match(I->getOperand(1),
                 m_c_And(m_Specific(I->getOperand(0)), m_Value())) &&
-          isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q))
+          isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Q, Depth))
         return true;
 
       unsigned BitWidth = V->getType()->getScalarSizeInBits();
       KnownBits LHSBits(BitWidth);
-      computeKnownBits(I->getOperand(0), LHSBits, Depth, Q);
+      computeKnownBits(I->getOperand(0), LHSBits, Q, Depth);
 
       KnownBits RHSBits(BitWidth);
-      computeKnownBits(I->getOperand(1), RHSBits, Depth, Q);
+      computeKnownBits(I->getOperand(1), RHSBits, Q, Depth);
       // If i8 V is a power of two or zero:
       //  ZeroBits: 1 1 1 0 1 1 1 1
       // ~ZeroBits: 0 0 0 1 0 0 0 0
@@ -2537,8 +2541,8 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
     return false;
   }
   case Instruction::Select:
-    return isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Depth, Q) &&
-           isKnownToBeAPowerOfTwo(I->getOperand(2), OrZero, Depth, Q);
+    return isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Q, Depth) &&
+           isKnownToBeAPowerOfTwo(I->getOperand(2), OrZero, Q, Depth);
   case Instruction::PHI: {
     // A PHI node is power of two if all incoming values are power of two, or if
     // it is an induction variable where in each step its value is a power of
@@ -2547,7 +2551,7 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
     SimplifyQuery RecQ = Q.getWithoutCondContext();
 
     // Check if it is an induction variable and always power of two.
-    if (isPowerOfTwoRecurrence(PN, OrZero, Depth, RecQ))
+    if (isPowerOfTwoRecurrence(PN, OrZero, RecQ, Depth))
       return true;
 
     // Recursively check all incoming values. Limit recursion to 2 levels, so
@@ -2561,7 +2565,7 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
       // Change the context instruction to the incoming block where it is
       // evaluated.
       RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
-      return isKnownToBeAPowerOfTwo(U.get(), OrZero, NewDepth, RecQ);
+      return isKnownToBeAPowerOfTwo(U.get(), OrZero, RecQ, NewDepth);
     });
   }
   case Instruction::Invoke:
@@ -2572,18 +2576,18 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
       case Intrinsic::smax:
       case Intrinsic::umin:
       case Intrinsic::smin:
-        return isKnownToBeAPowerOfTwo(II->getArgOperand(1), OrZero, Depth, Q) &&
-               isKnownToBeAPowerOfTwo(II->getArgOperand(0), OrZero, Depth, Q);
+        return isKnownToBeAPowerOfTwo(II->getArgOperand(1), OrZero, Q, Depth) &&
+               isKnownToBeAPowerOfTwo(II->getArgOperand(0), OrZero, Q, Depth);
       // bswap/bitreverse just move around bits, but don't change any 1s/0s
       // thus dont change pow2/non-pow2 status.
       case Intrinsic::bitreverse:
       case Intrinsic::bswap:
-        return isKnownToBeAPowerOfTwo(II->getArgOperand(0), OrZero, Depth, Q);
+        return isKnownToBeAPowerOfTwo(II->getArgOperand(0), OrZero, Q, Depth);
       case Intrinsic::fshr:
       case Intrinsic::fshl:
         // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
         if (II->getArgOperand(0) == II->getArgOperand(1))
-          return isKnownToBeAPowerOfTwo(II->getArgOperand(0), OrZero, Depth, Q);
+          return isKnownToBeAPowerOfTwo(II->getArgOperand(0), OrZero, Q, Depth);
         break;
       default:
         break;
@@ -2602,8 +2606,8 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
 /// to be non-null.
 ///
 /// Currently this routine does not support vector GEPs.
-static bool isGEPKnownNonNull(const GEPOperator *GEP, unsigned Depth,
-                              const SimplifyQuery &Q) {
+static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
+                              unsigned Depth) {
   const Function *F = nullptr;
   if (const Instruction *I = dyn_cast<Instruction>(GEP))
     F = I->getFunction();
@@ -2817,9 +2821,9 @@ static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
                                                 m_Specific(Op0), m_Zero())));
 }
 
-static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth,
-                         const SimplifyQuery &Q, unsigned BitWidth, Value *X,
-                         Value *Y, bool NSW, bool NUW) {
+static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
+                         unsigned BitWidth, Value *X, Value *Y, bool NSW,
+                         bool NUW, unsigned Depth) {
   // (X + (X != 0)) is non zero
   if (matchOpWithOpEqZero(X, Y))
     return true;
@@ -2828,8 +2832,8 @@ static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth,
     return isKnownNonZero(Y, DemandedElts, Q, Depth) ||
            isKnownNonZero(X, DemandedElts, Q, Depth);
 
-  KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q);
-  KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q);
+  KnownBits XKnown = computeKnownBits(X, DemandedElts, Q, Depth);
+  KnownBits YKnown = computeKnownBits(Y, DemandedElts, Q, Depth);
 
   // If X and Y are both non-negative (as signed values) then their sum is not
   // zero unless both X and Y are zero.
@@ -2854,18 +2858,18 @@ static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth,
 
   // The sum of a non-negative number and a power of two is not zero.
   if (XKnown.isNonNegative() &&
-      isKnownToBeAPowerOfTwo(Y, /*OrZero*/ false, Depth, Q))
+      isKnownToBeAPowerOfTwo(Y, /*OrZero*/ false, Q, Depth))
     return true;
   if (YKnown.isNonNegative() &&
-      isKnownToBeAPowerOfTwo(X, /*OrZero*/ false, Depth, Q))
+      isKnownToBeAPowerOfTwo(X, /*OrZero*/ false, Q, Depth))
     return true;
 
   return KnownBits::add(XKnown, YKnown, NSW, NUW).isNonZero();
 }
 
-static bool isNonZeroSub(const APInt &DemandedElts, unsigned Depth,
-                         const SimplifyQuery &Q, unsigned BitWidth, Value *X,
-                         Value *Y) {
+static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
+                         unsigned BitWidth, Value *X, Value *Y,
+                         unsigned Depth) {
   // (X - (X != 0)) is non zero
   // ((X != 0) - X) is non zero
   if (matchOpWithOpEqZero(X, Y))
@@ -2876,12 +2880,12 @@ static bool isNonZeroSub(const APInt &DemandedElts, unsigned Depth,
     if (C->isNullValue() && isKnownNonZero(Y, DemandedElts, Q, Depth))
       return true;
 
-  return ::isKnownNonEqual(X, Y, DemandedElts, Depth, Q);
+  return ::isKnownNonEqual(X, Y, DemandedElts, Q, Depth);
 }
 
-static bool isNonZeroMul(const APInt &DemandedElts, unsigned Depth,
-                         const SimplifyQuery &Q, unsigned BitWidth, Value *X,
-                         Value *Y, bool NSW, bool NUW) {
+static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
+                         unsigned BitWidth, Value *X, Value *Y, bool NSW,
+                         bool NUW, unsigned Depth) {
   // If X and Y are non-zero then so is X * Y as long as the multiplication
   // does not overflow.
   if (NSW || NUW)
@@ -2890,11 +2894,11 @@ static bool isNonZeroMul(const APInt &DemandedElts, unsigned Depth,
 
   // If either X or Y is odd, then if the other is non-zero the result can't
   // be zero.
-  KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q);
+  KnownBits XKnown = computeKnownBits(X, DemandedElts, Q, Depth);
   if (XKnown.One[0])
     return isKnownNonZero(Y, DemandedElts, Q, Depth);
 
-  KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q);
+  KnownBits YKnown = computeKnownBits(Y, DemandedElts, Q, Depth);
   if (YKnown.One[0])
     return XKnown.isNonZero() || isKnownNonZero(X, DemandedElts, Q, Depth);
 
@@ -2908,8 +2912,8 @@ static bool isNonZeroMul(const APInt &DemandedElts, unsigned Depth,
 }
 
 static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
-                           unsigned Depth, const SimplifyQuery &Q,
-                           const KnownBits &KnownVal) {
+                           const SimplifyQuery &Q, const KnownBits &KnownVal,
+                           unsigned Depth) {
   auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
     switch (I->getOpcode()) {
     case Instruction::Shl:
@@ -2939,7 +2943,7 @@ static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
     return false;
 
   KnownBits KnownCnt =
-      computeKnownBits(I->getOperand(1), DemandedElts, Depth, Q);
+      computeKnownBits(I->getOperand(1), DemandedElts, Q, Depth);
   APInt MaxShift = KnownCnt.getMaxValue();
   unsigned NumBits = KnownVal.getBitWidth();
   if (MaxShift.uge(NumBits))
@@ -2960,7 +2964,7 @@ static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
 
 static bool isKnownNonZeroFromOperator(const Operator *I,
                                        const APInt &DemandedElts,
-                                       unsigned Depth, const SimplifyQuery &Q) {
+                                       const SimplifyQuery &Q, unsigned Depth) {
   unsigned BitWidth = getBitWidth(I->getType()->getScalarType(), Q.DL);
   switch (I->getOpcode()) {
   case Instruction::Alloca:
@@ -2968,7 +2972,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     return I->getType()->getPointerAddressSpace() == 0;
   case Instruction::GetElementPtr:
     if (I->getType()->isPointerTy())
-      return isGEPKnownNonNull(cast<GEPOperator>(I), Depth, Q);
+      return isGEPKnownNonNull(cast<GEPOperator>(I), Q, Depth);
     break;
   case Instruction::BitCast: {
     // We need to be a bit careful here. We can only peek through the bitcast
@@ -3028,8 +3032,8 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     break;
 
   case Instruction::Sub:
-    return isNonZeroSub(DemandedElts, Depth, Q, BitWidth, I->getOperand(0),
-                        I->getOperand(1));
+    return isNonZeroSub(DemandedElts, Q, BitWidth, I->getOperand(0),
+                        I->getOperand(1), Depth);
   case Instruction::Xor:
     // (X ^ (X != 0)) is non zero
     if (matchOpWithOpEqZero(I->getOperand(0), I->getOperand(1)))
@@ -3039,6 +3043,10 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     // (X | (X != 0)) is non zero
     if (matchOpWithOpEqZero(I->getOperand(0), I->getOperand(1)))
       return true;
+    // X | Y != 0 if X != Y.
+    if (isKnownNonEqual(I->getOperand(0), I->getOperand(1), DemandedElts, Q,
+                        Depth))
+      return true;
     // X | Y != 0 if X != 0 or Y != 0.
     return isKnownNonZero(I->getOperand(1), DemandedElts, Q, Depth) ||
            isKnownNonZero(I->getOperand(0), DemandedElts, Q, Depth);
@@ -3056,11 +3064,11 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     // shl X, Y != 0 if X is odd.  Note that the value of the shift is undefined
     // if the lowest bit is shifted off the end.
     KnownBits Known(BitWidth);
-    computeKnownBits(I->getOperand(0), DemandedElts, Known, Depth, Q);
+    computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth);
     if (Known.One[0])
       return true;
 
-    return isNonZeroShift(I, DemandedElts, Depth, Q, Known);
+    return isNonZeroShift(I, DemandedElts, Q, Known, Depth);
   }
   case Instruction::LShr:
   case Instruction::AShr: {
@@ -3072,11 +3080,11 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     // shr X, Y != 0 if X is negative.  Note that the value of the shift is not
     // defined if the sign bit is shifted off the end.
     KnownBits Known =
-        computeKnownBits(I->getOperand(0), DemandedElts, Depth, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Q, Depth);
     if (Known.isNegative())
       return true;
 
-    return isNonZeroShift(I, DemandedElts, Depth, Q, Known);
+    return isNonZeroShift(I, DemandedElts, Q, Known, Depth);
   }
   case Instruction::UDiv:
   case Instruction::SDiv: {
@@ -3086,14 +3094,14 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       return isKnownNonZero(I->getOperand(0), DemandedElts, Q, Depth);
 
     KnownBits XKnown =
-        computeKnownBits(I->getOperand(0), DemandedElts, Depth, Q);
+        computeKnownBits(I->getOperand(0), DemandedElts, Q, Depth);
     // If X is fully unknown we won't be able to figure anything out so don't
     // both computing knownbits for Y.
     if (XKnown.isUnknown())
       return false;
 
     KnownBits YKnown =
-        computeKnownBits(I->getOperand(1), DemandedElts, Depth, Q);
+        computeKnownBits(I->getOperand(1), DemandedElts, Q, Depth);
     if (I->getOpcode() == Instruction::SDiv) {
       // For signed division need to compare abs value of the operands.
       XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
@@ -3111,15 +3119,15 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
     // non-zero.
     auto *BO = cast<OverflowingBinaryOperator>(I);
-    return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth, I->getOperand(0),
+    return isNonZeroAdd(DemandedElts, Q, BitWidth, I->getOperand(0),
                         I->getOperand(1), Q.IIQ.hasNoSignedWrap(BO),
-                        Q.IIQ.hasNoUnsignedWrap(BO));
+                        Q.IIQ.hasNoUnsignedWrap(BO), Depth);
   }
   case Instruction::Mul: {
     const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(I);
-    return isNonZeroMul(DemandedElts, Depth, Q, BitWidth, I->getOperand(0),
+    return isNonZeroMul(DemandedElts, Q, BitWidth, I->getOperand(0),
                         I->getOperand(1), Q.IIQ.hasNoSignedWrap(BO),
-                        Q.IIQ.hasNoUnsignedWrap(BO));
+                        Q.IIQ.hasNoUnsignedWrap(BO), Depth);
   }
   case Instruction::Select: {
     // (C ? X : Y) != 0 if X != 0 and Y != 0.
@@ -3265,17 +3273,17 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       default:
         break;
       case Instruction::Add:
-        return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth,
-                            WO->getArgOperand(0), WO->getArgOperand(1),
+        return isNonZeroAdd(DemandedElts, Q, BitWidth, WO->getArgOperand(0),
+                            WO->getArgOperand(1),
                             /*NSW=*/false,
-                            /*NUW=*/false);
+                            /*NUW=*/false, Depth);
       case Instruction::Sub:
-        return isNonZeroSub(DemandedElts, Depth, Q, BitWidth,
-                            WO->getArgOperand(0), WO->getArgOperand(1));
+        return isNonZeroSub(DemandedElts, Q, BitWidth, WO->getArgOperand(0),
+                            WO->getArgOperand(1), Depth);
       case Instruction::Mul:
-        return isNonZeroMul(DemandedElts, Depth, Q, BitWidth,
-                            WO->getArgOperand(0), WO->getArgOperand(1),
-                            /*NSW=*/false, /*NUW=*/false);
+        return isNonZeroMul(DemandedElts, Q, BitWidth, WO->getArgOperand(0),
+                            WO->getArgOperand(1),
+                            /*NSW=*/false, /*NUW=*/false, Depth);
         break;
       }
     }
@@ -3314,12 +3322,12 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         // NB: We don't do usub_sat here as in any case we can prove its
         // non-zero, we will fold it to `sub nuw` in InstCombine.
       case Intrinsic::ssub_sat:
-        return isNonZeroSub(DemandedElts, Depth, Q, BitWidth,
-                            II->getArgOperand(0), II->getArgOperand(1));
+        return isNonZeroSub(DemandedElts, Q, BitWidth, II->getArgOperand(0),
+                            II->getArgOperand(1), Depth);
       case Intrinsic::sadd_sat:
-        return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth,
-                            II->getArgOperand(0), II->getArgOperand(1),
-                            /*NSW=*/true, /* NUW=*/false);
+        return isNonZeroAdd(DemandedElts, Q, BitWidth, II->getArgOperand(0),
+                            II->getArgOperand(1),
+                            /*NSW=*/true, /* NUW=*/false, Depth);
         // Vec reverse preserves zero/non-zero status from input vec.
       case Intrinsic::vector_reverse:
         return isKnownNonZero(II->getArgOperand(0), DemandedElts.reverseBits(),
@@ -3353,12 +3361,12 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         // Avoid re-computing isKnownNonZero.
         std::optional<bool> Op0NonZero, Op1NonZero;
         KnownBits Op1Known =
-            computeKnownBits(II->getArgOperand(1), DemandedElts, Depth, Q);
+            computeKnownBits(II->getArgOperand(1), DemandedElts, Q, Depth);
         if (Op1Known.isNonNegative() &&
             IsNonZero(II->getArgOperand(1), Op1NonZero, Op1Known))
           return true;
         KnownBits Op0Known =
-            computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q);
+            computeKnownBits(II->getArgOperand(0), DemandedElts, Q, Depth);
         if (Op0Known.isNonNegative() &&
             IsNonZero(II->getArgOperand(0), Op0NonZero, Op0Known))
           return true;
@@ -3369,11 +3377,11 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         // If either arg is negative the result is non-zero. Otherwise
         // the result is non-zero if both ops are non-zero.
         KnownBits Op1Known =
-            computeKnownBits(II->getArgOperand(1), DemandedElts, Depth, Q);
+            computeKnownBits(II->getArgOperand(1), DemandedElts, Q, Depth);
         if (Op1Known.isNegative())
           return true;
         KnownBits Op0Known =
-            computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q);
+            computeKnownBits(II->getArgOperand(0), DemandedElts, Q, Depth);
         if (Op0Known.isNegative())
           return true;
 
@@ -3385,10 +3393,10 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         return isKnownNonZero(II->getArgOperand(0), DemandedElts, Q, Depth) &&
                isKnownNonZero(II->getArgOperand(1), DemandedElts, Q, Depth);
       case Intrinsic::cttz:
-        return computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q)
+        return computeKnownBits(II->getArgOperand(0), DemandedElts, Q, Depth)
             .Zero[0];
       case Intrinsic::ctlz:
-        return computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q)
+        return computeKnownBits(II->getArgOperand(0), DemandedElts, Q, Depth)
             .isNonNegative();
       case Intrinsic::fshr:
       case Intrinsic::fshl:
@@ -3411,7 +3419,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
   }
 
   KnownBits Known(BitWidth);
-  computeKnownBits(I, DemandedElts, Known, Depth, Q);
+  computeKnownBits(I, DemandedElts, Known, Q, Depth);
   return Known.One != 0;
 }
 
@@ -3506,7 +3514,7 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
   }
 
   if (const auto *I = dyn_cast<Operator>(V))
-    if (isKnownNonZeroFromOperator(I, DemandedElts, Depth, Q))
+    if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
       return true;
 
   if (!isa<Constant>(V) &&
@@ -3648,8 +3656,8 @@ getInvertibleOperands(const Operator *Op1,
 /// Only handle a small subset of binops where (binop V2, X) with non-zero X
 /// implies V2 != V1.
 static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
-                                      const APInt &DemandedElts, unsigned Depth,
-                                      const SimplifyQuery &Q) {
+                                      const APInt &DemandedElts,
+                                      const SimplifyQuery &Q, unsigned Depth) {
   const BinaryOperator *BO = dyn_cast<BinaryOperator>(V1);
   if (!BO)
     return false;
@@ -3677,8 +3685,8 @@ static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
 /// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
 /// the multiplication is nuw or nsw.
 static bool isNonEqualMul(const Value *V1, const Value *V2,
-                          const APInt &DemandedElts, unsigned Depth,
-                          const SimplifyQuery &Q) {
+                          const APInt &DemandedElts, const SimplifyQuery &Q,
+                          unsigned Depth) {
   if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) {
     const APInt *C;
     return match(OBO, m_Mul(m_Specific(V1), m_APInt(C))) &&
@@ -3692,8 +3700,8 @@ static bool isNonEqualMul(const Value *V1, const Value *V2,
 /// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
 /// the shift is nuw or nsw.
 static bool isNonEqualShl(const Value *V1, const Value *V2,
-                          const APInt &DemandedElts, unsigned Depth,
-                          const SimplifyQuery &Q) {
+                          const APInt &DemandedElts, const SimplifyQuery &Q,
+                          unsigned Depth) {
   if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) {
     const APInt *C;
     return match(OBO, m_Shl(m_Specific(V1), m_APInt(C))) &&
@@ -3704,8 +3712,8 @@ static bool isNonEqualShl(const Value *V1, const Value *V2,
 }
 
 static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
-                           const APInt &DemandedElts, unsigned Depth,
-                           const SimplifyQuery &Q) {
+                           const APInt &DemandedElts, const SimplifyQuery &Q,
+                           unsigned Depth) {
   // Check two PHIs are in same block.
   if (PN1->getParent() != PN2->getParent())
     return false;
@@ -3727,7 +3735,7 @@ static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
 
     SimplifyQuery RecQ = Q.getWithoutCondContext();
     RecQ.CxtI = IncomBB->getTerminator();
-    if (!isKnownNonEqual(IV1, IV2, DemandedElts, Depth + 1, RecQ))
+    if (!isKnownNonEqual(IV1, IV2, DemandedElts, RecQ, Depth + 1))
       return false;
     UsedFullRecursion = true;
   }
@@ -3735,8 +3743,8 @@ static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
 }
 
 static bool isNonEqualSelect(const Value *V1, const Value *V2,
-                             const APInt &DemandedElts, unsigned Depth,
-                             const SimplifyQuery &Q) {
+                             const APInt &DemandedElts, const SimplifyQuery &Q,
+                             unsigned Depth) {
   const SelectInst *SI1 = dyn_cast<SelectInst>(V1);
   if (!SI1)
     return false;
@@ -3746,12 +3754,12 @@ static bool isNonEqualSelect(const Value *V1, const Value *V2,
     const Value *Cond2 = SI2->getCondition();
     if (Cond1 == Cond2)
       return isKnownNonEqual(SI1->getTrueValue(), SI2->getTrueValue(),
-                             DemandedElts, Depth + 1, Q) &&
+                             DemandedElts, Q, Depth + 1) &&
              isKnownNonEqual(SI1->getFalseValue(), SI2->getFalseValue(),
-                             DemandedElts, Depth + 1, Q);
+                             DemandedElts, Q, Depth + 1);
   }
-  return isKnownNonEqual(SI1->getTrueValue(), V2, DemandedElts, Depth + 1, Q) &&
-         isKnownNonEqual(SI1->getFalseValue(), V2, DemandedElts, Depth + 1, Q);
+  return isKnownNonEqual(SI1->getTrueValue(), V2, DemandedElts, Q, Depth + 1) &&
+         isKnownNonEqual(SI1->getFalseValue(), V2, DemandedElts, Q, Depth + 1);
 }
 
 // Check to see if A is both a GEP and is the incoming value for a PHI in the
@@ -3807,7 +3815,7 @@ static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
 }
 
 static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
-                                       unsigned Depth, const SimplifyQuery &Q) {
+                                       const SimplifyQuery &Q, unsigned Depth) {
   if (!Q.CxtI)
     return false;
 
@@ -3865,8 +3873,8 @@ static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
 
 /// Return true if it is known that V1 != V2.
 static bool isKnownNonEqual(const Value *V1, const Value *V2,
-                            const APInt &DemandedElts, unsigned Depth,
-                            const SimplifyQuery &Q) {
+                            const APInt &DemandedElts, const SimplifyQuery &Q,
+                            unsigned Depth) {
   if (V1 == V2)
     return false;
   if (V1->getType() != V2->getType())
@@ -3883,44 +3891,44 @@ static bool isKnownNonEqual(const Value *V1, const Value *V2,
   auto *O2 = dyn_cast<Operator>(V2);
   if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
     if (auto Values = getInvertibleOperands(O1, O2))
-      return isKnownNonEqual(Values->first, Values->second, DemandedElts,
-                             Depth + 1, Q);
+      return isKnownNonEqual(Values->first, Values->second, DemandedElts, Q,
+                             Depth + 1);
 
     if (const PHINode *PN1 = dyn_cast<PHINode>(V1)) {
       const PHINode *PN2 = cast<PHINode>(V2);
       // FIXME: This is missing a generalization to handle the case where one is
       // a PHI and another one isn't.
-      if (isNonEqualPHIs(PN1, PN2, DemandedElts, Depth, Q))
+      if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
         return true;
     };
   }
 
-  if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Depth, Q) ||
-      isModifyingBinopOfNonZero(V2, V1, DemandedElts, Depth, Q))
+  if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
+      isModifyingBinopOfNonZero(V2, V1, DemandedElts, Q, Depth))
     return true;
 
-  if (isNonEqualMul(V1, V2, DemandedElts, Depth, Q) ||
-      isNonEqualMul(V2, V1, DemandedElts, Depth, Q))
+  if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
+      isNonEqualMul(V2, V1, DemandedElts, Q, Depth))
     return true;
 
-  if (isNonEqualShl(V1, V2, DemandedElts, Depth, Q) ||
-      isNonEqualShl(V2, V1, DemandedElts, Depth, Q))
+  if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
+      isNonEqualShl(V2, V1, DemandedElts, Q, Depth))
     return true;
 
   if (V1->getType()->isIntOrIntVectorTy()) {
     // Are any known bits in V1 contradictory to known bits in V2? If V1
     // has a known zero where V2 has a known one, they must not be equal.
-    KnownBits Known1 = computeKnownBits(V1, DemandedElts, Depth, Q);
+    KnownBits Known1 = computeKnownBits(V1, DemandedElts, Q, Depth);
     if (!Known1.isUnknown()) {
-      KnownBits Known2 = computeKnownBits(V2, DemandedElts, Depth, Q);
+      KnownBits Known2 = computeKnownBits(V2, DemandedElts, Q, Depth);
       if (Known1.Zero.intersects(Known2.One) ||
           Known2.Zero.intersects(Known1.One))
         return true;
     }
   }
 
-  if (isNonEqualSelect(V1, V2, DemandedElts, Depth, Q) ||
-      isNonEqualSelect(V2, V1, DemandedElts, Depth, Q))
+  if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
+      isNonEqualSelect(V2, V1, DemandedElts, Q, Depth))
     return true;
 
   if (isNonEqualPointersWithRecursiveGEP(V1, V2, Q) ||
@@ -3932,9 +3940,9 @@ static bool isKnownNonEqual(const Value *V1, const Value *V2,
   // Check PtrToInt type matches the pointer size.
   if (match(V1, m_PtrToIntSameSize(Q.DL, m_Value(A))) &&
       match(V2, m_PtrToIntSameSize(Q.DL, m_Value(B))))
-    return isKnownNonEqual(A, B, DemandedElts, Depth + 1, Q);
+    return isKnownNonEqual(A, B, DemandedElts, Q, Depth + 1);
 
-  if (isKnownNonEqualFromContext(V1, V2, Depth, Q))
+  if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
     return true;
 
   return false;
@@ -3969,11 +3977,11 @@ static unsigned computeNumSignBitsVectorConstant(const Value *V,
 
 static unsigned ComputeNumSignBitsImpl(const Value *V,
                                        const APInt &DemandedElts,
-                                       unsigned Depth, const SimplifyQuery &Q);
+                                       const SimplifyQuery &Q, unsigned Depth);
 
 static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
-                                   unsigned Depth, const SimplifyQuery &Q) {
-  unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Depth, Q);
+                                   const SimplifyQuery &Q, unsigned Depth) {
+  unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
   assert(Result > 0 && "At least one sign bit needs to be present!");
   return Result;
 }
@@ -3987,7 +3995,7 @@ static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
 /// elements in the vector specified by DemandedElts.
 static unsigned ComputeNumSignBitsImpl(const Value *V,
                                        const APInt &DemandedElts,
-                                       unsigned Depth, const SimplifyQuery &Q) {
+                                       const SimplifyQuery &Q, unsigned Depth) {
   Type *Ty = V->getType();
 #ifndef NDEBUG
   assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
@@ -4042,7 +4050,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
       if (isa<FixedVectorType>(Ty)) {
         // Fast case - sign splat can be simply split across the small elements.
         // This works for both vector and scalar sources
-        Tmp = ComputeNumSignBits(Src, Depth + 1, Q);
+        Tmp = ComputeNumSignBits(Src, Q, Depth + 1);
         if (Tmp == SrcBits)
           return TyBits;
       }
@@ -4050,7 +4058,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
     }
     case Instruction::SExt:
       Tmp = TyBits - U->getOperand(0)->getType()->getScalarSizeInBits();
-      return ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q) +
+      return ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1) +
              Tmp;
 
     case Instruction::SDiv: {
@@ -4064,7 +4072,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
 
         // Calculate the incoming numerator bits.
         unsigned NumBits =
-            ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+            ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
 
         // Add floor(log(C)) bits to the numerator bits.
         return std::min(TyBits, NumBits + Denominator->logBase2());
@@ -4073,7 +4081,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
     }
 
     case Instruction::SRem: {
-      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
 
       const APInt *Denominator;
       // srem X, C -> we know that the result is within [-C+1,C) when C is a
@@ -4104,7 +4112,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
     }
 
     case Instruction::AShr: {
-      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
       // ashr X, C   -> adds C sign bits.  Vectors too.
       const APInt *ShAmt;
       if (match(U->getOperand(1), m_APInt(ShAmt))) {
@@ -4127,11 +4135,11 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
         // all extended bits are shifted out.
         if (match(U->getOperand(0), m_ZExt(m_Value(X))) &&
             ShAmt->uge(TyBits - X->getType()->getScalarSizeInBits())) {
-          Tmp = ComputeNumSignBits(X, DemandedElts, Depth + 1, Q);
+          Tmp = ComputeNumSignBits(X, DemandedElts, Q, Depth + 1);
           Tmp += TyBits - X->getType()->getScalarSizeInBits();
         } else
           Tmp =
-              ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+              ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
         if (ShAmt->uge(Tmp))
           break; // Shifted all sign bits out.
         Tmp2 = ShAmt->getZExtValue();
@@ -4143,9 +4151,9 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
     case Instruction::Or:
     case Instruction::Xor: // NOT is handled here.
       // Logical binary ops preserve the number of sign bits at the worst.
-      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
       if (Tmp != 1) {
-        Tmp2 = ComputeNumSignBits(U->getOperand(1), DemandedElts, Depth + 1, Q);
+        Tmp2 = ComputeNumSignBits(U->getOperand(1), DemandedElts, Q, Depth + 1);
         FirstAnswer = std::min(Tmp, Tmp2);
         // We computed what we know about the sign bits as our first
         // answer. Now proceed to the generic code that uses
@@ -4161,24 +4169,24 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
       if (isSignedMinMaxClamp(U, X, CLow, CHigh))
         return std::min(CLow->getNumSignBits(), CHigh->getNumSignBits());
 
-      Tmp = ComputeNumSignBits(U->getOperand(1), DemandedElts, Depth + 1, Q);
+      Tmp = ComputeNumSignBits(U->getOperand(1), DemandedElts, Q, Depth + 1);
       if (Tmp == 1)
         break;
-      Tmp2 = ComputeNumSignBits(U->getOperand(2), DemandedElts, Depth + 1, Q);
+      Tmp2 = ComputeNumSignBits(U->getOperand(2), DemandedElts, Q, Depth + 1);
       return std::min(Tmp, Tmp2);
     }
 
     case Instruction::Add:
       // Add can have at most one carry bit.  Thus we know that the output
       // is, at worst, one more bit than the inputs.
-      Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+      Tmp = ComputeNumSignBits(U->getOperand(0), Q, Depth + 1);
       if (Tmp == 1) break;
 
       // Special case decrementing a value (ADD X, -1):
       if (const auto *CRHS = dyn_cast<Constant>(U->getOperand(1)))
         if (CRHS->isAllOnesValue()) {
           KnownBits Known(TyBits);
-          computeKnownBits(U->getOperand(0), DemandedElts, Known, Depth + 1, Q);
+          computeKnownBits(U->getOperand(0), DemandedElts, Known, Q, Depth + 1);
 
           // If the input is known to be 0 or 1, the output is 0/-1, which is
           // all sign bits set.
@@ -4191,13 +4199,13 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
             return Tmp;
         }
 
-      Tmp2 = ComputeNumSignBits(U->getOperand(1), DemandedElts, Depth + 1, Q);
+      Tmp2 = ComputeNumSignBits(U->getOperand(1), DemandedElts, Q, Depth + 1);
       if (Tmp2 == 1)
         break;
       return std::min(Tmp, Tmp2) - 1;
 
     case Instruction::Sub:
-      Tmp2 = ComputeNumSignBits(U->getOperand(1), DemandedElts, Depth + 1, Q);
+      Tmp2 = ComputeNumSignBits(U->getOperand(1), DemandedElts, Q, Depth + 1);
       if (Tmp2 == 1)
         break;
 
@@ -4205,7 +4213,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
       if (const auto *CLHS = dyn_cast<Constant>(U->getOperand(0)))
         if (CLHS->isNullValue()) {
           KnownBits Known(TyBits);
-          computeKnownBits(U->getOperand(1), DemandedElts, Known, Depth + 1, Q);
+          computeKnownBits(U->getOperand(1), DemandedElts, Known, Q, Depth + 1);
           // If the input is known to be 0 or 1, the output is 0/-1, which is
           // all sign bits set.
           if ((Known.Zero | 1).isAllOnes())
@@ -4222,7 +4230,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
 
       // Sub can have at most one carry bit.  Thus we know that the output
       // is, at worst, one more bit than the inputs.
-      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+      Tmp = ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
       if (Tmp == 1)
         break;
       return std::min(Tmp, Tmp2) - 1;
@@ -4231,11 +4239,11 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
       // The output of the Mul can be at most twice the valid bits in the
       // inputs.
       unsigned SignBitsOp0 =
-          ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+          ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
       if (SignBitsOp0 == 1)
         break;
       unsigned SignBitsOp1 =
-          ComputeNumSignBits(U->getOperand(1), DemandedElts, Depth + 1, Q);
+          ComputeNumSignBits(U->getOperand(1), DemandedElts, Q, Depth + 1);
       if (SignBitsOp1 == 1)
         break;
       unsigned OutValidBits =
@@ -4259,7 +4267,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
         if (Tmp == 1) return Tmp;
         RecQ.CxtI = PN->getIncomingBlock(i)->getTerminator();
         Tmp = std::min(Tmp, ComputeNumSignBits(PN->getIncomingValue(i),
-                                               DemandedElts, Depth + 1, RecQ));
+                                               DemandedElts, RecQ, Depth + 1));
       }
       return Tmp;
     }
@@ -4268,7 +4276,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
       // If the input contained enough sign bits that some remain after the
       // truncation, then we can make use of that. Otherwise we don't know
       // anything.
-      Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+      Tmp = ComputeNumSignBits(U->getOperand(0), Q, Depth + 1);
       unsigned OperandTyBits = U->getOperand(0)->getType()->getScalarSizeInBits();
       if (Tmp > (OperandTyBits - TyBits))
         return Tmp - (OperandTyBits - TyBits);
@@ -4281,7 +4289,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
       // skip tracking the specific element. But at least we might find
       // information valid for all elements of the vector (for example if vector
       // is sign extended, shifted, etc).
-      return ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+      return ComputeNumSignBits(U->getOperand(0), Q, Depth + 1);
 
     case Instruction::ShuffleVector: {
       // Collect the minimum number of sign bits that are shared by every vector
@@ -4299,7 +4307,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
       Tmp = std::numeric_limits<unsigned>::max();
       if (!!DemandedLHS) {
         const Value *LHS = Shuf->getOperand(0);
-        Tmp = ComputeNumSignBits(LHS, DemandedLHS, Depth + 1, Q);
+        Tmp = ComputeNumSignBits(LHS, DemandedLHS, Q, Depth + 1);
       }
       // If we don't know anything, early out and try computeKnownBits
       // fall-back.
@@ -4307,7 +4315,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
         break;
       if (!!DemandedRHS) {
         const Value *RHS = Shuf->getOperand(1);
-        Tmp2 = ComputeNumSignBits(RHS, DemandedRHS, Depth + 1, Q);
+        Tmp2 = ComputeNumSignBits(RHS, DemandedRHS, Q, Depth + 1);
         Tmp = std::min(Tmp, Tmp2);
       }
       // If we don't know anything, early out and try computeKnownBits
@@ -4324,7 +4332,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
           break;
         case Intrinsic::abs:
           Tmp =
-              ComputeNumSignBits(U->getOperand(0), DemandedElts, Depth + 1, Q);
+              ComputeNumSignBits(U->getOperand(0), DemandedElts, Q, Depth + 1);
           if (Tmp == 1)
             break;
 
@@ -4352,7 +4360,7 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
     return VecSignBits;
 
   KnownBits Known(TyBits);
-  computeKnownBits(V, DemandedElts, Known, Depth, Q);
+  computeKnownBits(V, DemandedElts, Known, Q, Depth);
 
   // If we know that the sign bit is either zero or one, determine the number of
   // identical bits in the top of the input value.
@@ -4546,22 +4554,23 @@ bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
 }
 
 static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
-                                        unsigned Depth, bool CondIsTrue,
+                                        bool CondIsTrue,
                                         const Instruction *CxtI,
-                                        KnownFPClass &KnownFromContext) {
+                                        KnownFPClass &KnownFromContext,
+                                        unsigned Depth = 0) {
   Value *A, *B;
   if (Depth < MaxAnalysisRecursionDepth &&
       (CondIsTrue ? match(Cond, m_LogicalAnd(m_Value(A), m_Value(B)))
                   : match(Cond, m_LogicalOr(m_Value(A), m_Value(B))))) {
-    computeKnownFPClassFromCond(V, A, Depth + 1, CondIsTrue, CxtI,
-                                KnownFromContext);
-    computeKnownFPClassFromCond(V, B, Depth + 1, CondIsTrue, CxtI,
-                                KnownFromContext);
+    computeKnownFPClassFromCond(V, A, CondIsTrue, CxtI, KnownFromContext,
+                                Depth + 1);
+    computeKnownFPClassFromCond(V, B, CondIsTrue, CxtI, KnownFromContext,
+                                Depth + 1);
     return;
   }
   if (Depth < MaxAnalysisRecursionDepth && match(Cond, m_Not(m_Value(A)))) {
-    computeKnownFPClassFromCond(V, A, Depth + 1, !CondIsTrue, CxtI,
-                                KnownFromContext);
+    computeKnownFPClassFromCond(V, A, !CondIsTrue, CxtI, KnownFromContext,
+                                Depth + 1);
     return;
   }
   CmpPredicate Pred;
@@ -4594,6 +4603,10 @@ static KnownFPClass computeKnownFPClassFromContext(const Value *V,
                                                    const SimplifyQuery &Q) {
   KnownFPClass KnownFromContext;
 
+  if (Q.CC && Q.CC->AffectedValues.contains(V))
+    computeKnownFPClassFromCond(V, Q.CC->Cond, !Q.CC->Invert, Q.CxtI,
+                                KnownFromContext);
+
   if (!Q.CxtI)
     return KnownFromContext;
 
@@ -4604,13 +4617,13 @@ static KnownFPClass computeKnownFPClassFromContext(const Value *V,
 
       BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
       if (Q.DT->dominates(Edge0, Q.CxtI->getParent()))
-        computeKnownFPClassFromCond(V, Cond, /*Depth=*/0, /*CondIsTrue=*/true,
-                                    Q.CxtI, KnownFromContext);
+        computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, Q.CxtI,
+                                    KnownFromContext);
 
       BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
       if (Q.DT->dominates(Edge1, Q.CxtI->getParent()))
-        computeKnownFPClassFromCond(V, Cond, /*Depth=*/0, /*CondIsTrue=*/false,
-                                    Q.CxtI, KnownFromContext);
+        computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, Q.CxtI,
+                                    KnownFromContext);
     }
   }
 
@@ -4632,7 +4645,7 @@ static KnownFPClass computeKnownFPClassFromContext(const Value *V,
     if (!isValidAssumeForContext(I, Q.CxtI, Q.DT))
       continue;
 
-    computeKnownFPClassFromCond(V, I->getArgOperand(0), /*Depth=*/0,
+    computeKnownFPClassFromCond(V, I->getArgOperand(0),
                                 /*CondIsTrue=*/true, Q.CxtI, KnownFromContext);
   }
 
@@ -4641,29 +4654,30 @@ static KnownFPClass computeKnownFPClassFromContext(const Value *V,
 
 void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
                          FPClassTest InterestedClasses, KnownFPClass &Known,
-                         unsigned Depth, const SimplifyQuery &Q);
+                         const SimplifyQuery &Q, unsigned Depth);
 
 static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
-                                FPClassTest InterestedClasses, unsigned Depth,
-                                const SimplifyQuery &Q) {
+                                FPClassTest InterestedClasses,
+                                const SimplifyQuery &Q, unsigned Depth) {
   auto *FVTy = dyn_cast<FixedVectorType>(V->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Depth, Q);
+  computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
 }
 
 static void computeKnownFPClassForFPTrunc(const Operator *Op,
                                           const APInt &DemandedElts,
                                           FPClassTest InterestedClasses,
-                                          KnownFPClass &Known, unsigned Depth,
-                                          const SimplifyQuery &Q) {
+                                          KnownFPClass &Known,
+                                          const SimplifyQuery &Q,
+                                          unsigned Depth) {
   if ((InterestedClasses &
        (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
     return;
 
   KnownFPClass KnownSrc;
   computeKnownFPClass(Op->getOperand(0), DemandedElts, InterestedClasses,
-                      KnownSrc, Depth + 1, Q);
+                      KnownSrc, Q, Depth + 1);
 
   // Sign should be preserved
   // TODO: Handle cannot be ordered greater than zero
@@ -4677,7 +4691,7 @@ static void computeKnownFPClassForFPTrunc(const Operator *Op,
 
 void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
                          FPClassTest InterestedClasses, KnownFPClass &Known,
-                         unsigned Depth, const SimplifyQuery &Q) {
+                         const SimplifyQuery &Q, unsigned Depth) {
   assert(Known.isUnknown() && "should not be called with known information");
 
   if (!DemandedElts) {
@@ -4787,7 +4801,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
   switch (Opc) {
   case Instruction::FNeg: {
     computeKnownFPClass(Op->getOperand(0), DemandedElts, InterestedClasses,
-                        Known, Depth + 1, Q);
+                        Known, Q, Depth + 1);
     Known.fneg();
     break;
   }
@@ -4833,11 +4847,11 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
     KnownFPClass Known2;
     computeKnownFPClass(LHS, DemandedElts, InterestedClasses & FilterLHS, Known,
-                        Depth + 1, Q);
+                        Q, Depth + 1);
     Known.KnownFPClasses &= FilterLHS;
 
     computeKnownFPClass(RHS, DemandedElts, InterestedClasses & FilterRHS,
-                        Known2, Depth + 1, Q);
+                        Known2, Q, Depth + 1);
     Known2.KnownFPClasses &= FilterRHS;
 
     Known |= Known2;
@@ -4852,7 +4866,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
         // If we only care about the sign bit we don't need to inspect the
         // operand.
         computeKnownFPClass(II->getArgOperand(0), DemandedElts,
-                            InterestedClasses, Known, Depth + 1, Q);
+                            InterestedClasses, Known, Q, Depth + 1);
       }
 
       Known.fabs();
@@ -4862,9 +4876,9 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       KnownFPClass KnownSign;
 
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
-                          Known, Depth + 1, Q);
+                          Known, Q, Depth + 1);
       computeKnownFPClass(II->getArgOperand(1), DemandedElts, InterestedClasses,
-                          KnownSign, Depth + 1, Q);
+                          KnownSign, Q, Depth + 1);
       Known.copysign(KnownSign);
       break;
     }
@@ -4882,7 +4896,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       // x * x + y is non-negative if y is non-negative.
       KnownFPClass KnownAddend;
       computeKnownFPClass(II->getArgOperand(2), DemandedElts, InterestedClasses,
-                          KnownAddend, Depth + 1, Q);
+                          KnownAddend, Q, Depth + 1);
 
       if (KnownAddend.cannotBeOrderedLessThanZero())
         Known.knownNot(fcNegative);
@@ -4896,7 +4910,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
         InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
 
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedSrcs,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
 
       if (KnownSrc.isKnownNeverPosInfinity())
         Known.knownNot(fcPosInf);
@@ -4928,7 +4942,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       // Return NaN on infinite inputs.
       KnownFPClass KnownSrc;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
       Known.knownNot(fcInf);
       if (KnownSrc.isKnownNeverNaN() && KnownSrc.isKnownNeverInfinity())
         Known.knownNot(fcNan);
@@ -4942,9 +4956,9 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     case Intrinsic::maximumnum: {
       KnownFPClass KnownLHS, KnownRHS;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
-                          KnownLHS, Depth + 1, Q);
+                          KnownLHS, Q, Depth + 1);
       computeKnownFPClass(II->getArgOperand(1), DemandedElts, InterestedClasses,
-                          KnownRHS, Depth + 1, Q);
+                          KnownRHS, Q, Depth + 1);
 
       bool NeverNaN = KnownLHS.isKnownNeverNaN() || KnownRHS.isKnownNeverNaN();
       Known = KnownLHS | KnownRHS;
@@ -5035,7 +5049,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     case Intrinsic::canonicalize: {
       KnownFPClass KnownSrc;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
 
       // This is essentially a stronger form of
       // propagateCanonicalizingSrc. Other "canonicalizing" operations don't
@@ -5086,7 +5100,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       // reduce min/max will choose an element from one of the vector elements,
       // so we can infer and class information that is common to all elements.
       Known = computeKnownFPClass(II->getArgOperand(0), II->getFastMathFlags(),
-                                  InterestedClasses, Depth + 1, Q);
+                                  InterestedClasses, Q, Depth + 1);
       // Can only propagate sign if output is never NaN.
       if (!Known.isKnownNeverNaN())
         Known.SignBit.reset();
@@ -5096,7 +5110,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     case Intrinsic::vector_reverse:
       Known = computeKnownFPClass(
           II->getArgOperand(0), DemandedElts.reverseBits(),
-          II->getFastMathFlags(), InterestedClasses, Depth + 1, Q);
+          II->getFastMathFlags(), InterestedClasses, Q, Depth + 1);
       break;
     case Intrinsic::trunc:
     case Intrinsic::floor:
@@ -5112,7 +5126,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       if (InterestedSrcs & fcNegFinite)
         InterestedSrcs |= fcNegFinite;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedSrcs,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
 
       // Integer results cannot be subnormal.
       Known.knownNot(fcSubnormal);
@@ -5145,7 +5159,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
       KnownFPClass KnownSrc;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
       if (KnownSrc.isKnownNeverNaN()) {
         Known.knownNot(fcNan);
         Known.signBitMustBeZero();
@@ -5155,7 +5169,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     }
     case Intrinsic::fptrunc_round: {
       computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
-                                    Depth, Q);
+                                    Q, Depth);
       break;
     }
     case Intrinsic::log:
@@ -5179,7 +5193,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
       KnownFPClass KnownSrc;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedSrcs,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
 
       if (KnownSrc.isKnownNeverPosInfinity())
         Known.knownNot(fcPosInf);
@@ -5210,7 +5224,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       unsigned BitWidth = ExpTy->getScalarType()->getIntegerBitWidth();
       KnownBits ExponentKnownBits(BitWidth);
       computeKnownBits(Exp, isa<VectorType>(ExpTy) ? DemandedElts : APInt(1, 1),
-                       ExponentKnownBits, Depth + 1, Q);
+                       ExponentKnownBits, Q, Depth + 1);
 
       if (ExponentKnownBits.Zero[0]) { // Is even
         Known.knownNot(fcNegative);
@@ -5227,7 +5241,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       //   pow(-inf, exp) --> -inf if exp is positive odd.
       KnownFPClass KnownSrc;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, fcNegative,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
       if (KnownSrc.isKnownNever(fcNegative))
         Known.knownNot(fcNegative);
       break;
@@ -5235,7 +5249,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     case Intrinsic::ldexp: {
       KnownFPClass KnownSrc;
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
-                          KnownSrc, Depth + 1, Q);
+                          KnownSrc, Q, Depth + 1);
       Known.propagateNaN(KnownSrc, /*PropagateSign=*/true);
 
       // Sign is preserved, but underflows may produce zeroes.
@@ -5298,7 +5312,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     }
     case Intrinsic::arithmetic_fence: {
       computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
-                          Known, Depth + 1, Q);
+                          Known, Q, Depth + 1);
       break;
     }
     case Intrinsic::experimental_constrained_sitofp:
@@ -5341,7 +5355,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     if (InterestedClasses & fcNan)
       InterestedSrcs |= fcInf;
     computeKnownFPClass(Op->getOperand(1), DemandedElts, InterestedSrcs,
-                        KnownRHS, Depth + 1, Q);
+                        KnownRHS, Q, Depth + 1);
 
     if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
         (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
@@ -5350,7 +5364,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       // RHS is canonically cheaper to compute. Skip inspecting the LHS if
       // there's no point.
       computeKnownFPClass(Op->getOperand(0), DemandedElts, InterestedSrcs,
-                          KnownLHS, Depth + 1, Q);
+                          KnownLHS, Q, Depth + 1);
       // Adding positive and negative infinity produces NaN.
       // TODO: Check sign of infinities.
       if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
@@ -5409,12 +5423,12 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
     KnownFPClass KnownLHS, KnownRHS;
     computeKnownFPClass(Op->getOperand(1), DemandedElts, NeedForNan, KnownRHS,
-                        Depth + 1, Q);
+                        Q, Depth + 1);
     if (!KnownRHS.isKnownNeverNaN())
       break;
 
     computeKnownFPClass(Op->getOperand(0), DemandedElts, NeedForNan, KnownLHS,
-                        Depth + 1, Q);
+                        Q, Depth + 1);
     if (!KnownLHS.isKnownNeverNaN())
       break;
 
@@ -5472,8 +5486,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     KnownFPClass KnownLHS, KnownRHS;
 
     computeKnownFPClass(Op->getOperand(1), DemandedElts,
-                        fcNan | fcInf | fcZero | fcNegative, KnownRHS,
-                        Depth + 1, Q);
+                        fcNan | fcInf | fcZero | fcNegative, KnownRHS, Q,
+                        Depth + 1);
 
     bool KnowSomethingUseful =
         KnownRHS.isKnownNeverNaN() || KnownRHS.isKnownNever(fcNegative);
@@ -5484,8 +5498,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
                        : fcNan | fcInf | fcZero | fcSubnormal | fcNegative;
 
       computeKnownFPClass(Op->getOperand(0), DemandedElts,
-                          InterestedClasses & InterestedLHS, KnownLHS,
-                          Depth + 1, Q);
+                          InterestedClasses & InterestedLHS, KnownLHS, Q,
+                          Depth + 1);
     }
 
     const Function *F = cast<Instruction>(Op)->getFunction();
@@ -5534,7 +5548,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
   case Instruction::FPExt: {
     // Infinity, nan and zero propagate from source.
     computeKnownFPClass(Op->getOperand(0), DemandedElts, InterestedClasses,
-                        Known, Depth + 1, Q);
+                        Known, Q, Depth + 1);
 
     const fltSemantics &DstTy =
         Op->getType()->getScalarType()->getFltSemantics();
@@ -5556,8 +5570,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     break;
   }
   case Instruction::FPTrunc: {
-    computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
-                                  Depth, Q);
+    computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
+                                  Depth);
     break;
   }
   case Instruction::SIToFP:
@@ -5594,19 +5608,20 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     // Look through extract element. If the index is non-constant or
     // out-of-range demand all elements, otherwise just the extracted element.
     const Value *Vec = Op->getOperand(0);
-    const Value *Idx = Op->getOperand(1);
-    auto *CIdx = dyn_cast<ConstantInt>(Idx);
 
+    APInt DemandedVecElts;
     if (auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType())) {
       unsigned NumElts = VecTy->getNumElements();
-      APInt DemandedVecElts = APInt::getAllOnes(NumElts);
+      DemandedVecElts = APInt::getAllOnes(NumElts);
+      auto *CIdx = dyn_cast<ConstantInt>(Op->getOperand(1));
       if (CIdx && CIdx->getValue().ult(NumElts))
         DemandedVecElts = APInt::getOneBitSet(NumElts, CIdx->getZExtValue());
-      return computeKnownFPClass(Vec, DemandedVecElts, InterestedClasses, Known,
-                                 Depth + 1, Q);
+    } else {
+      DemandedVecElts = APInt(1, 1);
     }
 
-    break;
+    return computeKnownFPClass(Vec, DemandedVecElts, InterestedClasses, Known,
+                               Q, Depth + 1);
   }
   case Instruction::InsertElement: {
     if (isa<ScalableVectorType>(Op->getType()))
@@ -5626,7 +5641,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
     // Do we demand the inserted element?
     if (NeedsElt) {
-      computeKnownFPClass(Elt, Known, InterestedClasses, Depth + 1, Q);
+      computeKnownFPClass(Elt, Known, InterestedClasses, Q, Depth + 1);
       // If we don't know any bits, early out.
       if (Known.isUnknown())
         break;
@@ -5637,8 +5652,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     // Do we need anymore elements from Vec?
     if (!DemandedVecElts.isZero()) {
       KnownFPClass Known2;
-      computeKnownFPClass(Vec, DemandedVecElts, InterestedClasses, Known2,
-                          Depth + 1, Q);
+      computeKnownFPClass(Vec, DemandedVecElts, InterestedClasses, Known2, Q,
+                          Depth + 1);
       Known |= Known2;
     }
 
@@ -5654,8 +5669,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
     if (!!DemandedLHS) {
       const Value *LHS = Shuf->getOperand(0);
-      computeKnownFPClass(LHS, DemandedLHS, InterestedClasses, Known,
-                          Depth + 1, Q);
+      computeKnownFPClass(LHS, DemandedLHS, InterestedClasses, Known, Q,
+                          Depth + 1);
 
       // If we don't know any bits, early out.
       if (Known.isUnknown())
@@ -5667,8 +5682,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     if (!!DemandedRHS) {
       KnownFPClass Known2;
       const Value *RHS = Shuf->getOperand(1);
-      computeKnownFPClass(RHS, DemandedRHS, InterestedClasses, Known2,
-                          Depth + 1, Q);
+      computeKnownFPClass(RHS, DemandedRHS, InterestedClasses, Known2, Q,
+                          Depth + 1);
       Known |= Known2;
     }
 
@@ -5687,7 +5702,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
           KnownFPClass KnownSrc;
           computeKnownFPClass(II->getArgOperand(0), DemandedElts,
-                              InterestedClasses, KnownSrc, Depth + 1, Q);
+                              InterestedClasses, KnownSrc, Q, Depth + 1);
 
           const Function *F = cast<Instruction>(Op)->getFunction();
           const fltSemantics &FltSem =
@@ -5722,8 +5737,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       }
     }
 
-    computeKnownFPClass(Src, DemandedElts, InterestedClasses, Known, Depth + 1,
-                        Q);
+    computeKnownFPClass(Src, DemandedElts, InterestedClasses, Known, Q,
+                        Depth + 1);
     break;
   }
   case Instruction::PHI: {
@@ -5756,8 +5771,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
         // to waste time spinning around in loops. We need at least depth 2 to
         // detect known sign bits.
         computeKnownFPClass(IncValue, DemandedElts, InterestedClasses, KnownSrc,
-                            PhiRecursionLimit,
-                            Q.getWithoutCondContext().getWithInstruction(CxtI));
+                            Q.getWithoutCondContext().getWithInstruction(CxtI),
+                            PhiRecursionLimit);
 
         if (First) {
           Known = KnownSrc;
@@ -5781,7 +5796,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
     const Type *Ty = Op->getType()->getScalarType();
     KnownBits Bits(Ty->getScalarSizeInBits());
-    computeKnownBits(Src, DemandedElts, Bits, Depth + 1, Q);
+    computeKnownBits(Src, DemandedElts, Bits, Q, Depth + 1);
 
     // Transfer information from the sign bit.
     if (Bits.isNonNegative())
@@ -5837,43 +5852,43 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 KnownFPClass llvm::computeKnownFPClass(const Value *V,
                                        const APInt &DemandedElts,
                                        FPClassTest InterestedClasses,
-                                       unsigned Depth,
-                                       const SimplifyQuery &SQ) {
+                                       const SimplifyQuery &SQ,
+                                       unsigned Depth) {
   KnownFPClass KnownClasses;
-  ::computeKnownFPClass(V, DemandedElts, InterestedClasses, KnownClasses, Depth,
-                        SQ);
+  ::computeKnownFPClass(V, DemandedElts, InterestedClasses, KnownClasses, SQ,
+                        Depth);
   return KnownClasses;
 }
 
 KnownFPClass llvm::computeKnownFPClass(const Value *V,
                                        FPClassTest InterestedClasses,
-                                       unsigned Depth,
-                                       const SimplifyQuery &SQ) {
+                                       const SimplifyQuery &SQ,
+                                       unsigned Depth) {
   KnownFPClass Known;
-  ::computeKnownFPClass(V, Known, InterestedClasses, Depth, SQ);
+  ::computeKnownFPClass(V, Known, InterestedClasses, SQ, Depth);
   return Known;
 }
 
 KnownFPClass llvm::computeKnownFPClass(
     const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
-    unsigned Depth, const TargetLibraryInfo *TLI, AssumptionCache *AC,
-    const Instruction *CxtI, const DominatorTree *DT, bool UseInstrInfo) {
-  return computeKnownFPClass(
-      V, InterestedClasses, Depth,
-      SimplifyQuery(DL, TLI, DT, AC, CxtI, UseInstrInfo));
+    const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CxtI,
+    const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
+  return computeKnownFPClass(V, InterestedClasses,
+                             SimplifyQuery(DL, TLI, DT, AC, CxtI, UseInstrInfo),
+                             Depth);
 }
 
 KnownFPClass
 llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
                           FastMathFlags FMF, FPClassTest InterestedClasses,
-                          unsigned Depth, const SimplifyQuery &SQ) {
+                          const SimplifyQuery &SQ, unsigned Depth) {
   if (FMF.noNaNs())
     InterestedClasses &= ~fcNan;
   if (FMF.noInfs())
     InterestedClasses &= ~fcInf;
 
   KnownFPClass Result =
-      computeKnownFPClass(V, DemandedElts, InterestedClasses, Depth, SQ);
+      computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
 
   if (FMF.noNaNs())
     Result.KnownFPClasses &= ~fcNan;
@@ -5884,56 +5899,57 @@ llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
 KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
                                        FPClassTest InterestedClasses,
-                                       unsigned Depth,
-                                       const SimplifyQuery &SQ) {
+                                       const SimplifyQuery &SQ,
+                                       unsigned Depth) {
   auto *FVTy = dyn_cast<FixedVectorType>(V->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, Depth,
-                             SQ);
+  return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
+                             Depth);
 }
 
-bool llvm::cannotBeNegativeZero(const Value *V, unsigned Depth,
-                                const SimplifyQuery &SQ) {
-  KnownFPClass Known = computeKnownFPClass(V, fcNegZero, Depth, SQ);
+bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
+                                unsigned Depth) {
+  KnownFPClass Known = computeKnownFPClass(V, fcNegZero, SQ, Depth);
   return Known.isKnownNeverNegZero();
 }
 
-bool llvm::cannotBeOrderedLessThanZero(const Value *V, unsigned Depth,
-                                       const SimplifyQuery &SQ) {
+bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
+                                       unsigned Depth) {
   KnownFPClass Known =
-      computeKnownFPClass(V, KnownFPClass::OrderedLessThanZeroMask, Depth, SQ);
+      computeKnownFPClass(V, KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
   return Known.cannotBeOrderedLessThanZero();
 }
 
-bool llvm::isKnownNeverInfinity(const Value *V, unsigned Depth,
-                                const SimplifyQuery &SQ) {
-  KnownFPClass Known = computeKnownFPClass(V, fcInf, Depth, SQ);
+bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
+                                unsigned Depth) {
+  KnownFPClass Known = computeKnownFPClass(V, fcInf, SQ, Depth);
   return Known.isKnownNeverInfinity();
 }
 
 /// Return true if the floating-point value can never contain a NaN or infinity.
-bool llvm::isKnownNeverInfOrNaN(const Value *V, unsigned Depth,
-                                const SimplifyQuery &SQ) {
-  KnownFPClass Known = computeKnownFPClass(V, fcInf | fcNan, Depth, SQ);
+bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
+                                unsigned Depth) {
+  KnownFPClass Known = computeKnownFPClass(V, fcInf | fcNan, SQ, Depth);
   return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
 }
 
 /// Return true if the floating-point scalar value is not a NaN or if the
 /// floating-point vector value has no NaN elements. Return false if a value
 /// could ever be NaN.
-bool llvm::isKnownNeverNaN(const Value *V, unsigned Depth,
-                           const SimplifyQuery &SQ) {
-  KnownFPClass Known = computeKnownFPClass(V, fcNan, Depth, SQ);
+bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
+                           unsigned Depth) {
+  KnownFPClass Known = computeKnownFPClass(V, fcNan, SQ, Depth);
   return Known.isKnownNeverNaN();
 }
 
 /// Return false if we can prove that the specified FP value's sign bit is 0.
 /// Return true if we can prove that the specified FP value's sign bit is 1.
 /// Otherwise return std::nullopt.
-std::optional<bool> llvm::computeKnownFPSignBit(const Value *V, unsigned Depth,
-                                                const SimplifyQuery &SQ) {
-  KnownFPClass Known = computeKnownFPClass(V, fcAllFlags, Depth, SQ);
+std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
+                                                const SimplifyQuery &SQ,
+                                                unsigned Depth) {
+  KnownFPClass Known = computeKnownFPClass(V, fcAllFlags, SQ, Depth);
   return Known.SignBit;
 }
 
@@ -7063,8 +7079,8 @@ OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
                                                    const Value *RHS,
                                                    const SimplifyQuery &SQ,
                                                    bool IsNSW) {
-  KnownBits LHSKnown = computeKnownBits(LHS, /*Depth=*/0, SQ);
-  KnownBits RHSKnown = computeKnownBits(RHS, /*Depth=*/0, SQ);
+  KnownBits LHSKnown = computeKnownBits(LHS, SQ);
+  KnownBits RHSKnown = computeKnownBits(RHS, SQ);
 
   // mul nsw of two non-negative numbers is also nuw.
   if (IsNSW && LHSKnown.isNonNegative() && RHSKnown.isNonNegative())
@@ -7089,7 +7105,7 @@ OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
   // Note that underestimating the number of sign bits gives a more
   // conservative answer.
   unsigned SignBits =
-      ::ComputeNumSignBits(LHS, 0, SQ) + ::ComputeNumSignBits(RHS, 0, SQ);
+      ::ComputeNumSignBits(LHS, SQ) + ::ComputeNumSignBits(RHS, SQ);
 
   // First handle the easy case: if we have enough sign bits there's
   // definitely no overflow.
@@ -7106,8 +7122,8 @@ OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
     // product is exactly the minimum negative number.
     // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
     // For simplicity we just check if at least one side is not negative.
-    KnownBits LHSKnown = computeKnownBits(LHS, /*Depth=*/0, SQ);
-    KnownBits RHSKnown = computeKnownBits(RHS, /*Depth=*/0, SQ);
+    KnownBits LHSKnown = computeKnownBits(LHS, SQ);
+    KnownBits RHSKnown = computeKnownBits(RHS, SQ);
     if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
       return OverflowResult::NeverOverflows;
   }
@@ -7147,8 +7163,7 @@ computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
   //
   // Since the carry into the most significant position is always equal to
   // the carry out of the addition, there is no signed overflow.
-  if (::ComputeNumSignBits(LHS, 0, SQ) > 1 &&
-      ::ComputeNumSignBits(RHS, 0, SQ) > 1)
+  if (::ComputeNumSignBits(LHS, SQ) > 1 && ::ComputeNumSignBits(RHS, SQ) > 1)
     return OverflowResult::NeverOverflows;
 
   ConstantRange LHSRange =
@@ -7175,7 +7190,7 @@ computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
       (LHSRange.isAllNegative() || RHSRange.isAllNegative());
   if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
     KnownBits AddKnown(LHSRange.getBitWidth());
-    computeKnownBitsFromContext(Add, AddKnown, /*Depth=*/0, SQ);
+    computeKnownBitsFromContext(Add, AddKnown, SQ);
     if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
         (AddKnown.isNegative() && LHSOrRHSKnownNegative))
       return OverflowResult::NeverOverflows;
@@ -7235,8 +7250,7 @@ OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
 
   // If LHS and RHS each have at least two sign bits, the subtraction
   // cannot overflow.
-  if (::ComputeNumSignBits(LHS, 0, SQ) > 1 &&
-      ::ComputeNumSignBits(RHS, 0, SQ) > 1)
+  if (::ComputeNumSignBits(LHS, SQ) > 1 && ::ComputeNumSignBits(RHS, SQ) > 1)
     return OverflowResult::NeverOverflows;
 
   ConstantRange LHSRange =
@@ -7462,18 +7476,11 @@ static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
   case Instruction::FNeg:
   case Instruction::PHI:
   case Instruction::Select:
-  case Instruction::URem:
-  case Instruction::SRem:
   case Instruction::ExtractValue:
   case Instruction::InsertValue:
   case Instruction::Freeze:
   case Instruction::ICmp:
   case Instruction::FCmp:
-  case Instruction::FAdd:
-  case Instruction::FSub:
-  case Instruction::FMul:
-  case Instruction::FDiv:
-  case Instruction::FRem:
   case Instruction::GetElementPtr:
     return false;
   default: {
@@ -7577,16 +7584,23 @@ static bool isGuaranteedNotToBeUndefOrPoison(
     if (isa<UndefValue>(C))
       return !includesUndef(Kind);
 
-    if (isa<ConstantInt>(C) || isa<GlobalVariable>(C) || isa<ConstantFP>(V) ||
+    if (isa<ConstantInt>(C) || isa<GlobalVariable>(C) || isa<ConstantFP>(C) ||
         isa<ConstantPointerNull>(C) || isa<Function>(C))
       return true;
 
-    if (C->getType()->isVectorTy() && !isa<ConstantExpr>(C)) {
-      if (includesUndef(Kind) && C->containsUndefElement())
-        return false;
-      if (includesPoison(Kind) && C->containsPoisonElement())
-        return false;
-      return !C->containsConstantExpression();
+    if (C->getType()->isVectorTy()) {
+      if (isa<ConstantExpr>(C)) {
+        // Scalable vectors can use a ConstantExpr to build a splat.
+        if (Constant *SplatC = C->getSplatValue())
+          if (isa<ConstantInt>(SplatC) || isa<ConstantFP>(SplatC))
+            return true;
+      } else {
+        if (includesUndef(Kind) && C->containsUndefElement())
+          return false;
+        if (includesPoison(Kind) && C->containsPoisonElement())
+          return false;
+        return !C->containsConstantExpression();
+      }
     }
   }
 
@@ -8650,6 +8664,7 @@ static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
     if (LHSSafe && RHSSafe) {
       // Both operands are known non-NaN.
       NaNBehavior = SPNB_RETURNS_ANY;
+      Ordered = CmpInst::isOrdered(Pred);
     } else if (CmpInst::isOrdered(Pred)) {
       // An ordered comparison will return false when given a NaN, so it
       // returns the RHS.
@@ -8905,19 +8920,20 @@ SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
   Value *TrueVal = SI->getTrueValue();
   Value *FalseVal = SI->getFalseValue();
 
-  return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS,
-                                            CastOp, Depth);
+  return llvm::matchDecomposedSelectPattern(
+      CmpI, TrueVal, FalseVal, LHS, RHS,
+      isa<FPMathOperator>(SI) ? SI->getFastMathFlags() : FastMathFlags(),
+      CastOp, Depth);
 }
 
 SelectPatternResult llvm::matchDecomposedSelectPattern(
     CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
-    Instruction::CastOps *CastOp, unsigned Depth) {
+    FastMathFlags FMF, Instruction::CastOps *CastOp, unsigned Depth) {
   CmpInst::Predicate Pred = CmpI->getPredicate();
   Value *CmpLHS = CmpI->getOperand(0);
   Value *CmpRHS = CmpI->getOperand(1);
-  FastMathFlags FMF;
-  if (isa<FPMathOperator>(CmpI))
-    FMF = CmpI->getFastMathFlags();
+  if (isa<FPMathOperator>(CmpI) && CmpI->hasNoNaNs())
+    FMF.setNoNaNs();
 
   // Bail out early.
   if (CmpI->isEquality())
@@ -9055,6 +9071,7 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
   // Handle the case of a simple two-predecessor recurrence PHI.
   // There's a lot more that could theoretically be done here, but
   // this is sufficient to catch some interesting cases.
+  // TODO: Expand list -- gep, uadd.sat etc.
   if (P->getNumIncomingValues() != 2)
     return false;
 
@@ -9064,36 +9081,16 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
     auto *LU = dyn_cast<BinaryOperator>(L);
     if (!LU)
       continue;
-    unsigned Opcode = LU->getOpcode();
-
-    switch (Opcode) {
-    default:
-      continue;
-    // TODO: Expand list -- xor, gep, uadd.sat etc.
-    case Instruction::LShr:
-    case Instruction::AShr:
-    case Instruction::Shl:
-    case Instruction::Add:
-    case Instruction::Sub:
-    case Instruction::UDiv:
-    case Instruction::URem:
-    case Instruction::And:
-    case Instruction::Or:
-    case Instruction::Mul:
-    case Instruction::FMul: {
-      Value *LL = LU->getOperand(0);
-      Value *LR = LU->getOperand(1);
-      // Find a recurrence.
-      if (LL == P)
-        L = LR;
-      else if (LR == P)
-        L = LL;
-      else
-        continue; // Check for recurrence with L and R flipped.
-
-      break; // Match!
-    }
-    };
+    Value *LL = LU->getOperand(0);
+    Value *LR = LU->getOperand(1);
+
+    // Find a recurrence.
+    if (LL == P)
+      L = LR;
+    else if (LR == P)
+      L = LL;
+    else
+      continue; // Check for recurrence with L and R flipped.
 
     // We have matched a recurrence of the form:
     //   %iv = [R, %entry], [%iv.next, %backedge]
@@ -9564,15 +9561,45 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
   unsigned Width = Lower.getBitWidth();
   const APInt *C;
   switch (BO.getOpcode()) {
-  case Instruction::Add:
-    if (match(BO.getOperand(1), m_APInt(C)) && !C->isZero()) {
+  case Instruction::Sub:
+    if (match(BO.getOperand(0), m_APInt(C))) {
       bool HasNSW = IIQ.hasNoSignedWrap(&BO);
       bool HasNUW = IIQ.hasNoUnsignedWrap(&BO);
 
       // If the caller expects a signed compare, then try to use a signed range.
       // Otherwise if both no-wraps are set, use the unsigned range because it
       // is never larger than the signed range. Example:
-      // "add nuw nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
+      // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
+      // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
+      if (PreferSignedRange && HasNSW && HasNUW)
+        HasNUW = false;
+
+      if (HasNUW) {
+        // 'sub nuw c, x' produces [0, C].
+        Upper = *C + 1;
+      } else if (HasNSW) {
+        if (C->isNegative()) {
+          // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
+          Lower = APInt::getSignedMinValue(Width);
+          Upper = *C - APInt::getSignedMaxValue(Width);
+        } else {
+          // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
+          // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
+          Lower = *C - APInt::getSignedMaxValue(Width);
+          Upper = APInt::getSignedMinValue(Width);
+        }
+      }
+    }
+    break;
+  case Instruction::Add:
+    if (match(BO.getOperand(1), m_APInt(C)) && !C->isZero()) {
+      bool HasNSW = IIQ.hasNoSignedWrap(&BO);
+      bool HasNUW = IIQ.hasNoUnsignedWrap(&BO);
+
+      // If the caller expects a signed compare, then try to use a signed
+      // range. Otherwise if both no-wraps are set, use the unsigned range
+      // because it is never larger than the signed range. Example: "add nuw
+      // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
       if (PreferSignedRange && HasNSW && HasNUW)
         HasNUW = false;
 
diff --git a/external/llvm-project/llvm/lib/Analysis/VectorUtils.cpp b/external/llvm-project/llvm/lib/Analysis/VectorUtils.cpp
index 968fd2f8c5d7..63fccee63c0a 100644
--- a/external/llvm-project/llvm/lib/Analysis/VectorUtils.cpp
+++ b/external/llvm-project/llvm/lib/Analysis/VectorUtils.cpp
@@ -240,6 +240,30 @@ Intrinsic::ID llvm::getVectorIntrinsicIDForCall(const CallInst *CI,
   return Intrinsic::not_intrinsic;
 }
 
+struct InterleaveIntrinsic {
+  Intrinsic::ID Interleave, Deinterleave;
+};
+
+static InterleaveIntrinsic InterleaveIntrinsics[] = {
+    {Intrinsic::vector_interleave2, Intrinsic::vector_deinterleave2},
+    {Intrinsic::vector_interleave3, Intrinsic::vector_deinterleave3},
+    {Intrinsic::vector_interleave4, Intrinsic::vector_deinterleave4},
+    {Intrinsic::vector_interleave5, Intrinsic::vector_deinterleave5},
+    {Intrinsic::vector_interleave6, Intrinsic::vector_deinterleave6},
+    {Intrinsic::vector_interleave7, Intrinsic::vector_deinterleave7},
+    {Intrinsic::vector_interleave8, Intrinsic::vector_deinterleave8},
+};
+
+Intrinsic::ID llvm::getInterleaveIntrinsicID(unsigned Factor) {
+  assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
+  return InterleaveIntrinsics[Factor - 2].Interleave;
+}
+
+Intrinsic::ID llvm::getDeinterleaveIntrinsicID(unsigned Factor) {
+  assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
+  return InterleaveIntrinsics[Factor - 2].Deinterleave;
+}
+
 /// Given a vector and an element number, see if the scalar value is
 /// already around as a register, for example if it were inserted then extracted
 /// from the vector.
diff --git a/external/llvm-project/llvm/lib/AsmParser/LLParser.cpp b/external/llvm-project/llvm/lib/AsmParser/LLParser.cpp
index bfafa613e21a..ecc772023e42 100644
--- a/external/llvm-project/llvm/lib/AsmParser/LLParser.cpp
+++ b/external/llvm-project/llvm/lib/AsmParser/LLParser.cpp
@@ -44,6 +44,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ModRef.h"
@@ -63,8 +64,6 @@ static cl::opt<bool> AllowIncompleteIR(
         "Allow incomplete IR on a best effort basis (references to unknown "
         "metadata will be dropped)"));
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-
 static std::string getTypeString(Type *T) {
   std::string Result;
   raw_string_ostream Tmp(Result);
@@ -442,8 +441,6 @@ bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
   UpgradeNVVMAnnotations(*M);
   UpgradeSectionAttributes(*M);
 
-  M->setIsNewDbgInfoFormat(UseNewDbgInfoFormat);
-
   if (!Slots)
     return false;
   // Initialize the slot mapping.
@@ -7148,8 +7145,6 @@ bool LLParser::parseBasicBlock(PerFunctionState &PFS) {
       if (SeenOldDbgInfoFormat)
         return error(Lex.getLoc(), "debug record should not appear in a module "
                                    "containing debug info intrinsics");
-      if (!SeenNewDbgInfoFormat)
-        M->setNewDbgInfoFormatFlag(true);
       SeenNewDbgInfoFormat = true;
       Lex.Lex();
 
@@ -8578,8 +8573,6 @@ bool LLParser::parseCall(Instruction *&Inst, PerFunctionState &PFS,
       return error(CallLoc, "llvm.dbg intrinsic should not appear in a module "
                             "using non-intrinsic debug info");
     }
-    if (!SeenOldDbgInfoFormat)
-      M->setNewDbgInfoFormatFlag(false);
     SeenOldDbgInfoFormat = true;
   }
   CI->setAttributes(PAL);
diff --git a/external/llvm-project/llvm/lib/BinaryFormat/MsgPackReader.cpp b/external/llvm-project/llvm/lib/BinaryFormat/MsgPackReader.cpp
index 2599d75efa08..99841c5b3bd6 100644
--- a/external/llvm-project/llvm/lib/BinaryFormat/MsgPackReader.cpp
+++ b/external/llvm-project/llvm/lib/BinaryFormat/MsgPackReader.cpp
@@ -14,7 +14,6 @@
 #include "llvm/BinaryFormat/MsgPackReader.h"
 #include "llvm/BinaryFormat/MsgPack.h"
 #include "llvm/Support/Endian.h"
-#include "llvm/Support/MathExtras.h"
 
 using namespace llvm;
 using namespace llvm::support;
diff --git a/external/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/external/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index fa73e6a20ac6..dd44bfd08eff 100644
--- a/external/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/external/llvm-project/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -101,8 +101,6 @@ static cl::opt<bool> ExpandConstantExprs(
     cl::desc(
         "Expand constant expressions to instructions for testing purposes"));
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace {
 
 enum {
@@ -4483,10 +4481,6 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord(
 Error BitcodeReader::parseModule(uint64_t ResumeBit,
                                  bool ShouldLazyLoadMetadata,
                                  ParserCallbacks Callbacks) {
-  // In preparation for the deletion of debug-intrinsics, don't allow module
-  // loading to escape intrinsics being autoupgraded to debug records.
-  TheModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat;
-
   this->ValueTypeCallback = std::move(Callbacks.ValueType);
   if (ResumeBit) {
     if (Error JumpFailed = Stream.JumpToBit(ResumeBit))
@@ -6997,10 +6991,6 @@ Error BitcodeReader::materialize(GlobalValue *GV) {
   if (Error JumpFailed = Stream.JumpToBit(DFII->second))
     return JumpFailed;
 
-  // Regardless of the debug info format we want to end up in, we need
-  // IsNewDbgInfoFormat=true to construct any debug records seen in the bitcode.
-  F->IsNewDbgInfoFormat = true;
-
   if (Error Err = parseFunctionBody(F))
     return Err;
   F->setIsMaterializable(false);
@@ -8165,6 +8155,14 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
           ContextSizes.reserve(NumContextSizeInfoEntries);
           for (unsigned J = 0; J < NumContextSizeInfoEntries; J++) {
             assert(ContextIdIndex < PendingContextIds.size());
+            // Skip any 0 entries for MIBs without the context size info.
+            if (PendingContextIds[ContextIdIndex] == 0) {
+              // The size should also be 0 if the context was 0.
+              assert(!Record[I]);
+              ContextIdIndex++;
+              I++;
+              continue;
+            }
             // PendingContextIds read from the preceding FS_ALLOC_CONTEXT_IDS
             // should be in the same order as the total sizes.
             ContextSizes.push_back(
diff --git a/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index e7551c4f193b..0a3268a5f1ad 100644
--- a/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -23,6 +23,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/MemoryProfileInfo.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Bitcode/BitcodeCommon.h"
 #include "llvm/Bitcode/BitcodeReader.h"
@@ -66,6 +67,7 @@
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -120,8 +122,6 @@ namespace llvm {
 extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
 }
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace {
 
 /// These are manifest constants used by the bitcode writer. They do not need to
@@ -4698,14 +4698,23 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
     Stream.EmitRecord(bitc::FS_STACK_IDS, Vals, StackIdAbbvId);
   }
 
-  // n x context id
-  auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
-  ContextIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
-  ContextIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
-  // The context ids are hashes that are close to 64 bits in size, so emitting
-  // as a pair of 32-bit fixed-width values is more efficient than a VBR.
-  ContextIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
-  unsigned ContextIdAbbvId = Stream.EmitAbbrev(std::move(ContextIdAbbv));
+  unsigned ContextIdAbbvId = 0;
+  if (metadataMayIncludeContextSizeInfo()) {
+    // n x context id
+    auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
+    ContextIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
+    ContextIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+    // The context ids are hashes that are close to 64 bits in size, so emitting
+    // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
+    // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
+    // values that are expected to more frequently occur in an alloc's memprof
+    // summary.
+    if (metadataIncludesAllContextSizeInfo())
+      ContextIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+    else
+      ContextIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+    ContextIdAbbvId = Stream.EmitAbbrev(std::move(ContextIdAbbv));
+  }
 
   // Abbrev for FS_PERMODULE_PROFILE.
   Abbv = std::make_shared<BitCodeAbbrev>();
diff --git a/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp
index fb393d33df3b..e48f735ded83 100644
--- a/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp
+++ b/external/llvm-project/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp
@@ -19,9 +19,7 @@
 using namespace llvm;
 
 PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) {
-  ScopedDbgInfoFormatSetter FormatSetter(M, M.IsNewDbgInfoFormat);
-  if (M.IsNewDbgInfoFormat)
-    M.removeDebugIntrinsicDeclarations();
+  M.removeDebugIntrinsicDeclarations();
 
   const ModuleSummaryIndex *Index =
       EmitSummaryIndex ? &(AM.getResult<ModuleSummaryIndexAnalysis>(M))
@@ -51,9 +49,7 @@ namespace {
     StringRef getPassName() const override { return "Bitcode Writer"; }
 
     bool runOnModule(Module &M) override {
-      ScopedDbgInfoFormatSetter FormatSetter(M, M.IsNewDbgInfoFormat);
-      if (M.IsNewDbgInfoFormat)
-        M.removeDebugIntrinsicDeclarations();
+      M.removeDebugIntrinsicDeclarations();
 
       WriteBitcodeToFile(M, OS, ShouldPreserveUseListOrder, /*Index=*/nullptr,
                          /*EmitModuleHash=*/false);
diff --git a/external/llvm-project/llvm/lib/CGData/CodeGenData.cpp b/external/llvm-project/llvm/lib/CGData/CodeGenData.cpp
index 7b9c584d6486..cd012342e195 100644
--- a/external/llvm-project/llvm/lib/CGData/CodeGenData.cpp
+++ b/external/llvm-project/llvm/lib/CGData/CodeGenData.cpp
@@ -18,8 +18,6 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Path.h"
 #include "llvm/Support/WithColor.h"
 
 #define DEBUG_TYPE "cg-data"
@@ -188,7 +186,7 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Curr) {
     return make_error<CGDataError>(cgdata_error::unsupported_version);
   H.DataKind = endian::readNext<uint32_t, endianness::little, unaligned>(Curr);
 
-  static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version2,
+  static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version3,
                 "Please update the offset computation below if a new field has "
                 "been added to the header.");
   H.OutlinedHashTreeOffset =
diff --git a/external/llvm-project/llvm/lib/CGData/CodeGenDataReader.cpp b/external/llvm-project/llvm/lib/CGData/CodeGenDataReader.cpp
index 39513d422c2c..0ab35499c898 100644
--- a/external/llvm-project/llvm/lib/CGData/CodeGenDataReader.cpp
+++ b/external/llvm-project/llvm/lib/CGData/CodeGenDataReader.cpp
@@ -13,12 +13,19 @@
 #include "llvm/CGData/CodeGenDataReader.h"
 #include "llvm/CGData/OutlinedHashTreeRecord.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 #define DEBUG_TYPE "cg-data-reader"
 
 using namespace llvm;
 
+static cl::opt<bool> IndexedCodeGenDataReadFunctionMapNames(
+    "indexed-codegen-data-read-function-map-names", cl::init(true), cl::Hidden,
+    cl::desc("Read function map names in indexed CodeGenData. Can be "
+             "disabled to save memory and time for final consumption of the "
+             "indexed CodeGenData in production."));
+
 namespace llvm {
 
 static Expected<std::unique_ptr<MemoryBuffer>>
@@ -106,7 +113,7 @@ Error IndexedCodeGenDataReader::read() {
     const unsigned char *Ptr = Start + Header.StableFunctionMapOffset;
     if (Ptr >= End)
       return error(cgdata_error::eof);
-    FunctionMapRecord.deserialize(Ptr);
+    FunctionMapRecord.deserialize(Ptr, IndexedCodeGenDataReadFunctionMapNames);
   }
 
   return success();
diff --git a/external/llvm-project/llvm/lib/CGData/CodeGenDataWriter.cpp b/external/llvm-project/llvm/lib/CGData/CodeGenDataWriter.cpp
index 3a392036198a..14a8558ba63b 100644
--- a/external/llvm-project/llvm/lib/CGData/CodeGenDataWriter.cpp
+++ b/external/llvm-project/llvm/lib/CGData/CodeGenDataWriter.cpp
@@ -19,29 +19,46 @@ using namespace llvm;
 void CGDataOStream::patch(ArrayRef<CGDataPatchItem> P) {
   using namespace support;
 
-  if (IsFDOStream) {
+  switch (Kind) {
+  case OStreamKind::fd: {
     raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
     const uint64_t LastPos = FDOStream.tell();
     for (const auto &K : P) {
       FDOStream.seek(K.Pos);
-      for (int I = 0; I < K.N; I++)
+      for (size_t I = 0; I < K.D.size(); ++I)
         write(K.D[I]);
     }
     // Reset the stream to the last position after patching so that users
     // don't accidentally overwrite data. This makes it consistent with
     // the string stream below which replaces the data directly.
     FDOStream.seek(LastPos);
-  } else {
+    break;
+  }
+  case OStreamKind::string: {
     raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
     std::string &Data = SOStream.str(); // with flush
     for (const auto &K : P) {
-      for (int I = 0; I < K.N; I++) {
+      for (size_t I = 0; I < K.D.size(); ++I) {
         uint64_t Bytes =
             endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[I]);
         Data.replace(K.Pos + I * sizeof(uint64_t), sizeof(uint64_t),
                      reinterpret_cast<const char *>(&Bytes), sizeof(uint64_t));
       }
     }
+    break;
+  }
+  case OStreamKind::svector: {
+    raw_svector_ostream &VOStream = static_cast<raw_svector_ostream &>(OS);
+    for (const auto &K : P) {
+      for (size_t I = 0; I < K.D.size(); ++I) {
+        uint64_t Bytes =
+            endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[I]);
+        VOStream.pwrite(reinterpret_cast<const char *>(&Bytes),
+                        sizeof(uint64_t), K.Pos + I * sizeof(uint64_t));
+      }
+    }
+    break;
+  }
   }
 }
 
@@ -106,17 +123,20 @@ Error CodeGenDataWriter::writeImpl(CGDataOStream &COS) {
   if (Error E = writeHeader(COS))
     return E;
 
+  std::vector<CGDataPatchItem> PatchItems;
+
   uint64_t OutlinedHashTreeFieldStart = COS.tell();
   if (hasOutlinedHashTree())
     HashTreeRecord.serialize(COS.OS);
   uint64_t StableFunctionMapFieldStart = COS.tell();
   if (hasStableFunctionMap())
-    FunctionMapRecord.serialize(COS.OS);
+    FunctionMapRecord.serialize(COS.OS, PatchItems);
 
   // Back patch the offsets.
-  CGDataPatchItem PatchItems[] = {
-      {OutlinedHashTreeOffset, &OutlinedHashTreeFieldStart, 1},
-      {StableFunctionMapOffset, &StableFunctionMapFieldStart, 1}};
+  PatchItems.emplace_back(OutlinedHashTreeOffset, &OutlinedHashTreeFieldStart,
+                          1);
+  PatchItems.emplace_back(StableFunctionMapOffset, &StableFunctionMapFieldStart,
+                          1);
   COS.patch(PatchItems);
 
   return Error::success();
diff --git a/external/llvm-project/llvm/lib/CGData/StableFunctionMapRecord.cpp b/external/llvm-project/llvm/lib/CGData/StableFunctionMapRecord.cpp
index 537793b9e2b4..4e4fceffb52b 100644
--- a/external/llvm-project/llvm/lib/CGData/StableFunctionMapRecord.cpp
+++ b/external/llvm-project/llvm/lib/CGData/StableFunctionMapRecord.cpp
@@ -77,26 +77,32 @@ static IndexOperandHashVecType getStableIndexOperandHashes(
   return IndexOperandHashes;
 }
 
-void StableFunctionMapRecord::serialize(raw_ostream &OS) const {
-  serialize(OS, FunctionMap.get());
+void StableFunctionMapRecord::serialize(
+    raw_ostream &OS, std::vector<CGDataPatchItem> &PatchItems) const {
+  serialize(OS, FunctionMap.get(), PatchItems);
 }
 
-void StableFunctionMapRecord::serialize(raw_ostream &OS,
-                                        const StableFunctionMap *FunctionMap) {
+void StableFunctionMapRecord::serialize(
+    raw_ostream &OS, const StableFunctionMap *FunctionMap,
+    std::vector<CGDataPatchItem> &PatchItems) {
   support::endian::Writer Writer(OS, endianness::little);
 
   // Write Names.
   ArrayRef<std::string> Names = FunctionMap->getNames();
-  uint32_t ByteSize = 4;
   Writer.write<uint32_t>(Names.size());
-  for (auto &Name : Names) {
+  // Remember the position, write back the total size of Names, so we can skip
+  // reading them if needed.
+  const uint64_t NamesByteSizeOffset = Writer.OS.tell();
+  Writer.write<uint64_t>(0);
+  for (auto &Name : Names)
     Writer.OS << Name << '\0';
-    ByteSize += Name.size() + 1;
-  }
-  // Align ByteSize to 4 bytes.
-  uint32_t Padding = offsetToAlignment(ByteSize, Align(4));
+  // Align current position to 4 bytes.
+  uint32_t Padding = offsetToAlignment(Writer.OS.tell(), Align(4));
   for (uint32_t I = 0; I < Padding; ++I)
     Writer.OS << '\0';
+  const auto NamesByteSize =
+      Writer.OS.tell() - NamesByteSizeOffset - sizeof(NamesByteSizeOffset);
+  PatchItems.emplace_back(NamesByteSizeOffset, &NamesByteSize, 1);
 
   // Write StableFunctionEntries whose pointers are sorted.
   auto FuncEntries = getStableFunctionEntries(*FunctionMap);
@@ -120,7 +126,8 @@ void StableFunctionMapRecord::serialize(raw_ostream &OS,
   }
 }
 
-void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr) {
+void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr,
+                                          bool ReadStableFunctionMapNames) {
   // Assert that Ptr is 4-byte aligned
   assert(((uintptr_t)Ptr % 4) == 0);
   // Read Names.
@@ -129,13 +136,23 @@ void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr) {
   // Early exit if there is no name.
   if (NumNames == 0)
     return;
-  for (unsigned I = 0; I < NumNames; ++I) {
-    StringRef Name(reinterpret_cast<const char *>(Ptr));
-    Ptr += Name.size() + 1;
-    FunctionMap->getIdOrCreateForName(Name);
+  const auto NamesByteSize =
+      endian::readNext<uint64_t, endianness::little, unaligned>(Ptr);
+  const auto NamesOffset = reinterpret_cast<uintptr_t>(Ptr);
+  if (ReadStableFunctionMapNames) {
+    for (unsigned I = 0; I < NumNames; ++I) {
+      StringRef Name(reinterpret_cast<const char *>(Ptr));
+      Ptr += Name.size() + 1;
+      FunctionMap->getIdOrCreateForName(Name);
+    }
+    // Align Ptr to 4 bytes.
+    Ptr = reinterpret_cast<const uint8_t *>(alignAddr(Ptr, Align(4)));
+    assert(reinterpret_cast<uintptr_t>(Ptr) - NamesOffset == NamesByteSize &&
+           "NamesByteSize does not match the actual size of names");
+  } else {
+    // skip reading Names by advancing the pointer.
+    Ptr = reinterpret_cast<const uint8_t *>(NamesOffset + NamesByteSize);
   }
-  // Align Ptr to 4 bytes.
-  Ptr = reinterpret_cast<const uint8_t *>(alignAddr(Ptr, Align(4)));
 
   // Read StableFunctionEntries.
   auto NumFuncs =
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index cdb0520d6bee..4211b9e1c2bc 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -369,7 +369,11 @@ Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
     Alignment = InAlign;
 
   // If the GV has a specified alignment, take it into account.
-  const MaybeAlign GVAlign(GV->getAlign());
+  MaybeAlign GVAlign;
+  if (auto *GVar = dyn_cast<GlobalVariable>(GV))
+    GVAlign = GVar->getAlign();
+  else if (auto *F = dyn_cast<Function>(GV))
+    GVAlign = F->getAlign();
   if (!GVAlign)
     return Alignment;
 
@@ -561,8 +565,11 @@ bool AsmPrinter::doInitialization(Module &M) {
 
   if (MAI->doesSupportDebugInformation()) {
     bool EmitCodeView = M.getCodeViewFlag();
-    if (EmitCodeView &&
-        (TM.getTargetTriple().isOSWindows() || TM.getTargetTriple().isUEFI()))
+    // On Windows targets, emit minimal CodeView compiler info even when debug
+    // info is disabled.
+    if ((TM.getTargetTriple().isOSWindows() &&
+         M.getNamedMetadata("llvm.dbg.cu")) ||
+        (TM.getTargetTriple().isUEFI() && EmitCodeView))
       Handlers.push_back(std::make_unique<CodeViewDebug>(this));
     if (!EmitCodeView || M.getDwarfVersion()) {
       if (hasDebugInfo()) {
@@ -4371,6 +4378,8 @@ void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
       OutStreamer->emitLabel(Sym);
   } else if (isVerbose() && MBB.isMachineBlockAddressTaken()) {
     OutStreamer->AddComment("Block address taken");
+  } else if (isVerbose() && MBB.isInlineAsmBrIndirectTarget()) {
+    OutStreamer->AddComment("Inline asm indirect target");
   }
 
   // Print some verbose block comments.
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index fc43bc6f7776..ea57a8fa1f79 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -125,6 +125,8 @@ static CPUType mapArchToCVCPUType(Triple::ArchType Type) {
     return CPUType::ARM64;
   case Triple::ArchType::mipsel:
     return CPUType::MIPS;
+  case Triple::ArchType::UnknownArch:
+    return CPUType::Unknown;
   default:
     report_fatal_error("target architecture doesn't map to a CodeView CPUType");
   }
@@ -611,21 +613,33 @@ static SourceLanguage MapDWLangToCVLang(unsigned DWLang) {
 }
 
 void CodeViewDebug::beginModule(Module *M) {
-  // If module doesn't have named metadata anchors or COFF debug section
-  // is not available, skip any debug info related stuff.
-  if (!Asm->hasDebugInfo() ||
-      !Asm->getObjFileLowering().getCOFFDebugSymbolsSection()) {
+  // If COFF debug section is not available, skip any debug info related stuff.
+  if (!Asm->getObjFileLowering().getCOFFDebugSymbolsSection()) {
     Asm = nullptr;
     return;
   }
 
+  CompilerInfoAsm = Asm;
   TheCPU = mapArchToCVCPUType(M->getTargetTriple().getArch());
 
   // Get the current source language.
-  const MDNode *Node = *M->debug_compile_units_begin();
+  const MDNode *Node;
+  if (Asm->hasDebugInfo()) {
+    Node = *M->debug_compile_units_begin();
+  } else {
+    // When emitting only compiler information, we may have only NoDebug CUs,
+    // which would be skipped by debug_compile_units_begin.
+    NamedMDNode *CUs = MMI->getModule()->getNamedMetadata("llvm.dbg.cu");
+    Node = *CUs->operands().begin();
+  }
   const auto *CU = cast<DICompileUnit>(Node);
 
   CurrentSourceLanguage = MapDWLangToCVLang(CU->getSourceLanguage());
+  if (!M->getCodeViewFlag() ||
+      CU->getEmissionKind() == DICompileUnit::NoDebug) {
+    Asm = nullptr;
+    return;
+  }
 
   collectGlobalVariableInfo();
 
@@ -636,7 +650,7 @@ void CodeViewDebug::beginModule(Module *M) {
 }
 
 void CodeViewDebug::endModule() {
-  if (!Asm || !Asm->hasDebugInfo())
+  if (!CompilerInfoAsm)
     return;
 
   // The COFF .debug$S section consists of several subsections, each starting
@@ -652,6 +666,8 @@ void CodeViewDebug::endModule() {
   emitObjName();
   emitCompilerInformation();
   endCVSubsection(CompilerInfo);
+  if (!Asm)
+    return;
 
   emitInlineeLinesSubsection();
 
@@ -788,7 +804,7 @@ void CodeViewDebug::emitTypeGlobalHashes() {
 void CodeViewDebug::emitObjName() {
   MCSymbol *CompilerEnd = beginSymbolRecord(SymbolKind::S_OBJNAME);
 
-  StringRef PathRef(Asm->TM.Options.ObjectFilenameForDebug);
+  StringRef PathRef(CompilerInfoAsm->TM.Options.ObjectFilenameForDebug);
   llvm::SmallString<256> PathStore(PathRef);
 
   if (PathRef.empty() || PathRef == "-") {
@@ -846,7 +862,7 @@ void CodeViewDebug::emitCompilerInformation() {
   }
   using ArchType = llvm::Triple::ArchType;
   ArchType Arch = MMI->getModule()->getTargetTriple().getArch();
-  if (Asm->TM.Options.Hotpatch || Arch == ArchType::thumb ||
+  if (CompilerInfoAsm->TM.Options.Hotpatch || Arch == ArchType::thumb ||
       Arch == ArchType::aarch64) {
     Flags |= static_cast<uint32_t>(CompileSym3Flags::HotPatch);
   }
@@ -1015,7 +1031,7 @@ void CodeViewDebug::switchToDebugSectionForSymbol(const MCSymbol *GVSym) {
   const MCSymbol *KeySym = GVSec ? GVSec->getCOMDATSymbol() : nullptr;
 
   MCSectionCOFF *DebugSec = cast<MCSectionCOFF>(
-      Asm->getObjFileLowering().getCOFFDebugSymbolsSection());
+      CompilerInfoAsm->getObjFileLowering().getCOFFDebugSymbolsSection());
   DebugSec = OS.getContext().getAssociativeCOFFSection(DebugSec, KeySym);
 
   OS.switchSection(DebugSec);
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
index d13b315135ad..5f4f30271d9c 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
@@ -98,6 +98,10 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
   /// The codeview CPU type used by the translation unit.
   codeview::CPUType TheCPU;
 
+  /// The AsmPrinter used for emitting compiler metadata. When only compiler
+  /// info is being emitted, DebugHandlerBase::Asm may be null.
+  AsmPrinter *CompilerInfoAsm = nullptr;
+
   static LocalVarDef createDefRangeMem(uint16_t CVRegister, int Offset);
 
   /// Similar to DbgVariable in DwarfDebug, but not dwarf-specific.
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.cpp b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
index 700e24a08b5d..3ea472d454fb 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
@@ -40,5 +40,5 @@ void DebugLocStream::finalizeEntry() {
 DebugLocStream::ListBuilder::~ListBuilder() {
   if (!Locs.finalizeList(Asm))
     return;
-  V.emplace<Loc::Multi>(ListIndex, TagOffset);
+  V.emplace<Loc::Multi>(ListIndex, TagOffset, CommonAddrSpace);
 }
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h
index 6f553dc85c64..894e680daa7f 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h
@@ -157,6 +157,7 @@ class DebugLocStream::ListBuilder {
   DbgVariable &V;
   size_t ListIndex;
   std::optional<uint8_t> TagOffset;
+  std::optional<unsigned> CommonAddrSpace;
 
 public:
   ListBuilder(DebugLocStream &Locs, DwarfCompileUnit &CU, AsmPrinter &Asm,
@@ -168,6 +169,11 @@ class DebugLocStream::ListBuilder {
     TagOffset = TO;
   }
 
+  void setCommonDivergentAddrSpace(unsigned AS) { CommonAddrSpace = AS; }
+  bool hasCommonDivergentAddrSpace() const {
+    return CommonAddrSpace != std::nullopt;
+  }
+
   /// Finalize the list.
   ///
   /// If the list is empty, delete it.  Otherwise, finalize it by creating a
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 080f54cab4e7..c512113ae2ac 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -891,6 +891,8 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(
 
   if (Expr) {
     if (auto NewElementsRef = Expr->getNewElementsRef()) {
+      if (DV.isDivergentAddrSpaceCompatible())
+        DwarfExpr.permitDivergentAddrSpace();
       DwarfExpr.addExpression(*NewElementsRef, DVal->getLocEntries(), &TRI);
       addBlock(VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize());
       return;
@@ -973,6 +975,8 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(const Loc::MMI &MMI,
     addBlock(VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize());
     return;
   }
+  if (DV.isDivergentAddrSpaceCompatible())
+    DwarfExpr.permitDivergentAddrSpace();
   for (const auto &Fragment : MMI.getFrameIndexExprs()) {
     Register FrameReg;
     const DIExpression *Expr = Fragment.Expr;
@@ -1758,7 +1762,19 @@ void DwarfCompileUnit::applyCommonDbgVariableAttributes(const DbgVariable &Var,
   }
 
   addSourceLine(VariableDie, DIVar);
-  addType(VariableDie, Var.getType());
+
+  const DIType *VarTy = Var.getType();
+  if (Var.isDivergentAddrSpaceCompatible()) {
+    if (std::optional<unsigned> EntityAS = Var.getCommonDivergentAddrSpace()) {
+      if (auto DwarfAS = getAsmPrinter()->TM.mapToDWARFAddrSpace(*EntityAS)) {
+        TempDIDerivedType Tmp =
+            cast<DIDerivedType>(VarTy)->cloneWithAddressSpace(*DwarfAS);
+        VarTy = MDNode::replaceWithUniqued(std::move(Tmp));
+      }
+    }
+  }
+
+  addType(VariableDie, VarTy);
   if (Var.isArtificial())
     addFlag(VariableDie, dwarf::DW_AT_artificial);
 }
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index f0247df9894e..68ef84fe4ce1 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -233,6 +233,63 @@ void DebugLocDwarfExpression::commitTemporaryBuffer() {
   TmpBuf->Comments.clear();
 }
 
+namespace {
+/// Utility class for finding the common divergent address space of all the
+/// DIExpressions that describe the location of a variable, if such an address
+/// space exists.
+class CommonDivergentAddrSpaceFinder {
+  std::optional<unsigned> CommonAS;
+  bool HasCommonAddrSpace = true;
+
+public:
+  void addSubExpr(const DIExpression *Expr) {
+    if (!Expr || !HasCommonAddrSpace)
+      return;
+    std::optional<unsigned> ExprAS = Expr->getNewDivergentAddrSpace();
+    if (!ExprAS)
+      HasCommonAddrSpace = false;
+    else if (!CommonAS)
+      CommonAS = *ExprAS;
+    else if (*CommonAS != *ExprAS)
+      HasCommonAddrSpace = false;
+  }
+
+  std::optional<unsigned> get() const {
+    return HasCommonAddrSpace ? CommonAS : std::nullopt;
+  }
+};
+} // namespace
+
+std::optional<unsigned> DbgVariable::getCommonDivergentAddrSpace() const {
+  const Loc::Variant *Loc = &asVariant();
+
+  if (auto *LM = std::get_if<Loc::Multi>(Loc))
+    return LM->getCommonDivergentAddrSpace();
+
+  CommonDivergentAddrSpaceFinder Finder;
+  if (auto *LS = std::get_if<Loc::Single>(Loc)) {
+    Finder.addSubExpr(LS->getExpr());
+  } else if (auto *MMI = std::get_if<Loc::MMI>(Loc)) {
+    for (auto &FIE : MMI->getFrameIndexExprs())
+      Finder.addSubExpr(FIE.Expr);
+  } else if (auto *EV = std::get_if<Loc::EntryValue>(Loc)) {
+    for (auto &Val : EV->EntryValues)
+      Finder.addSubExpr(&Val.Expr);
+  }
+
+  return Finder.get();
+}
+
+bool DbgVariable::isDivergentAddrSpaceCompatible() const {
+  if (auto *DT = dyn_cast<DIDerivedType>(getType()))
+    return DT->getTag() == dwarf::DW_TAG_pointer_type ||
+           DT->getTag() == dwarf::DW_TAG_reference_type ||
+           DT->getTag() == dwarf::DW_TAG_rvalue_reference_type;
+  // FIXME: We could support divergent address spaces on pointer/reference
+  // fields of struct types.
+  return false;
+}
+
 const DIType *DbgVariable::getType() const {
   return getVariable()->getType();
 }
@@ -1952,6 +2009,18 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU,
       continue;
     }
 
+    // If all entries in the location list produce a consistent divergent
+    // address space we need to inform the expression emitter that it is
+    // permitted to produce divergent address spaces.
+    if (RegVar->isDivergentAddrSpaceCompatible()) {
+      CommonDivergentAddrSpaceFinder Finder;
+      for (const DebugLocEntry &DLE : Entries)
+        for (const DbgValueLoc &DVL : DLE.getValues())
+          Finder.addSubExpr(DVL.getExpression());
+      if (std::optional<unsigned> AS = Finder.get())
+        List.setCommonDivergentAddrSpace(*AS);
+    }
+
     // If the variable has a DIBasicType, extract it.  Basic types cannot have
     // unique identifiers, so don't bother resolving the type with the
     // identifier map.
@@ -2375,8 +2444,8 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
   // Map {(InlinedAt, Group): (Rank, Instructions)}.
   // NOTE: Anecdotally, for a large C++ blob, 99% of the instruction
   // SmallVectors contain 2 or fewer elements; use 2 inline elements.
-  DenseMap<std::pair<DILocation *, uint32_t>,
-           std::pair<uint16_t, SmallVector<const MachineInstr *, 2>>>
+  DenseMap<std::pair<DILocation *, uint64_t>,
+           std::pair<uint8_t, SmallVector<const MachineInstr *, 2>>>
       GroupCandidates;
 
   // For each instruction:
@@ -3181,6 +3250,8 @@ void DebugLocEntry::finalize(const AsmPrinter &AP,
   DebugLocStream::EntryBuilder Entry(List, Begin, End);
   BufferByteStreamer Streamer = Entry.getStreamer();
   DebugLocDwarfExpression DwarfExpr(AP, Streamer, TheCU);
+  if (List.hasCommonDivergentAddrSpace())
+    DwarfExpr.permitDivergentAddrSpace();
   const DbgValueLoc &Value = Values[0];
   if (Value.isFragment()) {
     // Emit all fragments that belong to the same variable and range.
@@ -3902,7 +3973,7 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU,
   if (!TypeUnitsUnderConstruction.empty() && AddrPool.hasBeenUsed())
     return;
 
-  auto Ins = TypeSignatures.insert(std::make_pair(CTy, 0));
+  auto Ins = TypeSignatures.try_emplace(CTy);
   if (!Ins.second) {
     CU.addDIETypeSignature(RefDie, Ins.first->second);
     return;
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 71a610514a6d..abaa28e44be3 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -145,15 +145,25 @@ class Multi {
   /// DW_OP_LLVM_tag_offset value from DebugLocs.
   std::optional<uint8_t> DebugLocListTagOffset;
 
+  /// In DIOp-DIExpressions, if this variable has pointer type and all entries
+  /// in the loclist produce the same divergent address space, this is set to be
+  /// the that address space.
+  std::optional<unsigned> CommonAddrSpace;
+
 public:
   explicit Multi(unsigned DebugLocListIndex,
-                 std::optional<uint8_t> DebugLocListTagOffset)
+                 std::optional<uint8_t> DebugLocListTagOffset,
+                 std::optional<unsigned> CommonAddrSpace = std::nullopt)
       : DebugLocListIndex(DebugLocListIndex),
-        DebugLocListTagOffset(DebugLocListTagOffset) {}
+        DebugLocListTagOffset(DebugLocListTagOffset),
+        CommonAddrSpace(CommonAddrSpace) {}
   unsigned getDebugLocListIndex() const { return DebugLocListIndex; }
   std::optional<uint8_t> getDebugLocListTagOffset() const {
     return DebugLocListTagOffset;
   }
+  std::optional<unsigned> getCommonDivergentAddrSpace() const {
+    return CommonAddrSpace;
+  }
 };
 /// Single location defined by (potentially multiple) MMI entries.
 struct MMI {
@@ -277,6 +287,9 @@ class DbgVariable : public DbgEntity, public Loc::Variant {
 
   const DIType *getType() const;
 
+  bool isDivergentAddrSpaceCompatible() const;
+  std::optional<unsigned> getCommonDivergentAddrSpace() const;
+
   static bool classof(const DbgEntity *N) {
     return N->getDbgEntityID() == DbgVariableKind;
   }
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 4f2c0cb05fc1..2ce6a1d7322a 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -198,7 +198,7 @@ void DwarfExpression::addStackValue() {
 }
 
 void DwarfExpression::addSignedConstant(int64_t Value) {
-  if (IsPoisonedExpr)
+  if (IsPoisonedExpr || !IsImplemented)
     return;
   assert(isImplicitLocation() || isUnknownLocation());
   LocationKind = Implicit;
@@ -207,7 +207,7 @@ void DwarfExpression::addSignedConstant(int64_t Value) {
 }
 
 void DwarfExpression::addUnsignedConstant(uint64_t Value) {
-  if (IsPoisonedExpr)
+  if (IsPoisonedExpr || !IsImplemented)
     return;
   assert(isImplicitLocation() || isUnknownLocation());
   LocationKind = Implicit;
@@ -215,7 +215,7 @@ void DwarfExpression::addUnsignedConstant(uint64_t Value) {
 }
 
 void DwarfExpression::addUnsignedConstant(const APInt &Value) {
-  if (IsPoisonedExpr)
+  if (IsPoisonedExpr || !IsImplemented)
     return;
   assert(isImplicitLocation() || isUnknownLocation());
   LocationKind = Implicit;
@@ -237,7 +237,7 @@ void DwarfExpression::addUnsignedConstant(const APInt &Value) {
 }
 
 void DwarfExpression::addConstantFP(const APFloat &APF, const AsmPrinter &AP) {
-  if (IsPoisonedExpr)
+  if (IsPoisonedExpr || !IsImplemented)
     return;
   assert(isImplicitLocation() || isUnknownLocation());
   APInt API = APF.bitcastToAPInt();
@@ -269,7 +269,7 @@ bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI,
                                               DIExpressionCursor &ExprCursor,
                                               llvm::Register MachineReg,
                                               unsigned FragmentOffsetInBits) {
-  if (IsPoisonedExpr)
+  if (IsPoisonedExpr || !IsImplemented)
     return true;
   auto Fragment = ExprCursor.getFragmentInfo();
   if (!addMachineReg(TRI, MachineReg, Fragment ? Fragment->SizeInBits : ~1U)) {
@@ -733,6 +733,8 @@ bool DwarfExpression::addExpression(
 void DwarfExpression::addExpression(DIExpression::NewElementsRef Expr,
                                     ArrayRef<DbgValueLocEntry> ArgLocEntries,
                                     const TargetRegisterInfo *TRI) {
+  if (!IsImplemented)
+    return;
   assert(!IsPoisonedExpr && "poisoned exprs should have old elements");
   this->ArgLocEntries = ArgLocEntries;
   this->TRI = TRI;
@@ -745,9 +747,13 @@ void DwarfExpression::addExpression(DIExpression::NewElementsRef Expr,
     }
   }
   buildAST(Expr);
-  traverse(ASTRoot.get(), ValueKind::LocationDesc);
+  traverse(ASTRoot.get(), ValueKind::LocationDesc,
+           /*PermitDivergentAddrSpace=*/
+           PermitDivergentAddrSpaceResult && !IsFragment);
   if (FragOp)
     addOpPiece(FragOp->getBitSize());
+  if (!IsImplemented)
+    emitUserOp(dwarf::DW_OP_LLVM_USER_undefined);
   IsFragment = false;
   ASTRoot.reset();
   this->TRI = nullptr;
@@ -764,9 +770,9 @@ void DwarfExpression::maskSubRegister() {
 }
 
 void DwarfExpression::emitUserOp(uint8_t UserOp, const char *Comment) {
+  auto OptNonUserOp = dwarf::getNonUserOp(UserOp);
+  assert(OptNonUserOp && "non-user op passed to DwarfExpression::emitUserOp");
   if (!EmitHeterogeneousDwarfAsUserOps) {
-    auto OptNonUserOp = dwarf::getNonUserOp(UserOp);
-    assert(OptNonUserOp && "non-user op passed to DwarfExpression::emitUserOp");
     emitOp(*OptNonUserOp, Comment);
     return;
   }
@@ -786,7 +792,7 @@ void DwarfExpression::finalize() {
 }
 
 void DwarfExpression::addFragmentOffset(const DIExpression *Expr) {
-  if (!Expr)
+  if (!Expr || !IsImplemented)
     return;
 
   if (Expr->holdsOldElements() && Expr->isPoisoned())
@@ -842,7 +848,7 @@ void DwarfExpression::emitLegacyZExt(unsigned FromBits) {
 }
 
 void DwarfExpression::addWasmLocation(unsigned Index, uint64_t Offset) {
-  if (IsPoisonedExpr)
+  if (IsPoisonedExpr || !IsImplemented)
     return;
   emitOp(dwarf::DW_OP_WASM_location);
   emitUnsigned(Index == 4/*TI_LOCAL_INDIRECT*/ ? 0/*TI_LOCAL*/ : Index);
@@ -888,7 +894,8 @@ void DwarfExpression::buildAST(DIExpression::NewElementsRef Elements) {
 using NewOpResult = DwarfExpression::OpResult;
 
 std::optional<NewOpResult>
-DwarfExpression::traverse(Node *OpNode, std::optional<ValueKind> ReqVK) {
+DwarfExpression::traverse(Node *OpNode, std::optional<ValueKind> ReqVK,
+                          bool PermitDivergentAddrSpace) {
   std::optional<NewOpResult> Result =
       std::visit([&](auto &&E) { return traverse(E, OpNode->getChildren()); },
                  OpNode->getElement());
@@ -896,6 +903,12 @@ DwarfExpression::traverse(Node *OpNode, std::optional<ValueKind> ReqVK) {
     IsImplemented = false;
     return Result;
   }
+  if (Result->DivergentAddrSpace && !PermitDivergentAddrSpace) {
+    // FIXME: When DWARF supports address space conversions, generate a
+    // DW_OP_convert here to convert to the required address space.
+    IsImplemented = false;
+    return Result;
+  }
   OpNode->setIsLowered();
   OpNode->setResultType(Result->Ty);
   return ReqVK ? convertValueKind(*Result, *ReqVK) : Result;
@@ -905,12 +918,12 @@ NewOpResult DwarfExpression::convertValueKind(const NewOpResult &Res,
                                               ValueKind ReqVK) {
   if (Res.VK == ValueKind::Value && ReqVK == ValueKind::LocationDesc) {
     emitOp(dwarf::DW_OP_stack_value);
-    return {Res.Ty, ValueKind::LocationDesc};
+    return {Res.Ty, ValueKind::LocationDesc, Res.DivergentAddrSpace};
   }
 
   if (Res.VK == ValueKind::LocationDesc && ReqVK == ValueKind::Value) {
     readToValue(Res.Ty);
-    return {Res.Ty, ValueKind::Value};
+    return {Res.Ty, ValueKind::Value, Res.DivergentAddrSpace};
   }
 
   return Res;
@@ -1088,14 +1101,28 @@ std::optional<NewOpResult> DwarfExpression::traverse(DIOp::AddrOf AddrOf,
 
 std::optional<NewOpResult> DwarfExpression::traverse(DIOp::Convert Convert,
                                                      ChildrenT Children) {
-  auto Child = traverse(Children[0].get(), ValueKind::Value);
+  auto Child = traverse(Children[0].get(), /*RequiredVK=*/std::nullopt,
+                        /*PermitDivergentAddrSpace=*/true);
   if (!Child)
     return std::nullopt;
 
   Type *DestTy = Convert.getResultType();
+  if (Child->Ty->isPointerTy() && DestTy->isPointerTy() &&
+      Child->Ty->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
+    unsigned DivAddrSpace = Child->DivergentAddrSpace
+                                ? *Child->DivergentAddrSpace
+                                : Child->Ty->getPointerAddressSpace();
+    return NewOpResult{DestTy, Child->VK, DivAddrSpace};
+  }
+
   if (!Child->Ty->isIntegerTy() || !DestTy->isIntegerTy())
     return std::nullopt;
 
+  // If we're not dealing with the divergent address space case, Convert
+  // requires a value operand.
+  if (Child->VK == ValueKind::LocationDesc)
+    readToValue(Child->Ty);
+
   uint64_t ToBits = DestTy->getPrimitiveSizeInBits().getFixedValue();
   uint64_t FromBits = Child->Ty->getPrimitiveSizeInBits().getFixedValue();
 
@@ -1131,7 +1158,8 @@ std::optional<NewOpResult> DwarfExpression::traverse(DIOp::SExt SExt,
 
 std::optional<NewOpResult> DwarfExpression::traverse(DIOp::Deref Deref,
                                                      ChildrenT Children) {
-  auto Child = traverse(Children[0].get(), ValueKind::LocationDesc);
+  auto Child = traverse(Children[0].get(), ValueKind::LocationDesc,
+                        /*PermitDivergentAddrSpace=*/true);
   if (!Child)
     return std::nullopt;
 
@@ -1142,13 +1170,14 @@ std::optional<NewOpResult> DwarfExpression::traverse(DIOp::Deref Deref,
   PointerType *PointerResultType = dyn_cast<PointerType>(Child->Ty);
   assert(PointerResultType && "Expected PointerType, but got something else");
 
-  uint64_t PointerSizeInBits = AP.getDataLayout().getPointerSizeInBits(
-      PointerResultType->getAddressSpace());
+  unsigned PointerLLVMAddrSpace = Child->DivergentAddrSpace
+                                      ? *Child->DivergentAddrSpace
+                                      : PointerResultType->getAddressSpace();
+  uint64_t PointerSizeInBits =
+      AP.getDataLayout().getPointerSizeInBits(PointerLLVMAddrSpace);
   assert(PointerSizeInBits % 8 == 0 && "Expected multiple of 8");
 
   uint64_t PointerSizeInBytes = PointerSizeInBits / 8;
-
-  unsigned PointerLLVMAddrSpace = PointerResultType->getAddressSpace();
   auto PointerDWARFAddrSpace = AP.TM.mapToDWARFAddrSpace(PointerLLVMAddrSpace);
   if (!PointerDWARFAddrSpace) {
     LLVM_DEBUG(dbgs() << "Failed to lower DIOpDeref of pointer to addrspace("
@@ -1182,10 +1211,12 @@ std::optional<NewOpResult> DwarfExpression::traverse(DIOp::Read Read,
 
 std::optional<NewOpResult>
 DwarfExpression::traverse(DIOp::Reinterpret Reinterpret, ChildrenT Children) {
-  auto Child = traverse(Children[0].get(), ValueKind::LocationDesc);
+  auto Child = traverse(Children[0].get(), ValueKind::LocationDesc,
+                        /*PermitDivergentAddrSpace=*/true);
   if (!Child)
     return Child;
-  return NewOpResult{Reinterpret.getResultType(), Child->VK};
+  return NewOpResult{Reinterpret.getResultType(), Child->VK,
+                     Child->DivergentAddrSpace};
 }
 
 std::optional<NewOpResult> DwarfExpression::traverse(DIOp::Select Select,
@@ -1218,8 +1249,8 @@ std::optional<NewOpResult> DwarfExpression::traverse(DIOp::Composite Composite,
   return NewOpResult{Composite.getResultType(), ValueKind::LocationDesc};
 }
 
-std::optional<NewOpResult> DwarfExpression::traverseMathOp(uint8_t DwarfOp,
-                                                           ChildrenT Children) {
+std::optional<NewOpResult>
+DwarfExpression::traverseMathOp(uint8_t DwarfOp, ChildrenT Children) {
   auto LHS = traverse(Children[0].get(), ValueKind::Value);
   if (!LHS)
     return std::nullopt;
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
index 6da0e26c6ddd..9ed9d2115595 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -368,6 +368,11 @@ class DwarfExpression {
   /// case, since we need to ensure that we don't add any registers or constants
   /// onto the stack. In the non-fragment case it's simply an optimization.
   bool IsPoisonedExpr = false;
+  bool PermitDivergentAddrSpaceResult = false;
+
+  /// Called if we're allowed to produce a stack entry whose address space
+  /// diverges from the IR type the DIExpression produces.
+  void permitDivergentAddrSpace() { PermitDivergentAddrSpaceResult = true; }
 
   void buildAST(DIExpression::NewElementsRef Elements);
 
@@ -384,6 +389,9 @@ class DwarfExpression {
   struct OpResult {
     Type *Ty;
     ValueKind VK;
+    // The real address space of this result, if it diverges from Ty's address
+    // space.
+    std::optional<unsigned> DivergentAddrSpace = std::nullopt;
   };
 
   /// Optionally emit DWARF operations to convert the value at the top of the
@@ -396,9 +404,10 @@ class DwarfExpression {
   using ChildrenT = ArrayRef<std::unique_ptr<Node>>;
 
   /// Dispatch to a specific traverse() function, and convert the result to
-  /// ReqVK if non-nullopt.
-  std::optional<OpResult> traverse(Node *OpNode,
-                                   std::optional<ValueKind> ReqVK);
+  /// ReqVK if non-nullopt. If PermitDivergentAddrSpace, then this function may
+  /// return a pointer in a different address space than the type.
+  std::optional<OpResult> traverse(Node *OpNode, std::optional<ValueKind> ReqVK,
+                                   bool PermitDivergentAddrSpace = false);
 
   std::optional<OpResult> traverse(DIOp::Arg Arg, ChildrenT Children);
   std::optional<OpResult> traverse(DIOp::Constant Constant, ChildrenT Children);
diff --git a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 84af84702adf..ba5af19308ad 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -1965,13 +1965,14 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(const DIDerivedType *DT) {
   if (DIE *StaticMemberDIE = getDIE(DT))
     return StaticMemberDIE;
 
+  DwarfUnit *ContextUnit = static_cast<DwarfUnit *>(ContextDIE->getUnit());
   DIE &StaticMemberDIE = createAndAddDIE(DT->getTag(), *ContextDIE, DT);
 
   const DIType *Ty = DT->getBaseType();
 
   addString(StaticMemberDIE, dwarf::DW_AT_name, DT->getName());
   addType(StaticMemberDIE, Ty);
-  addSourceLine(StaticMemberDIE, DT);
+  ContextUnit->addSourceLine(StaticMemberDIE, DT);
   addFlag(StaticMemberDIE, dwarf::DW_AT_external);
   addFlag(StaticMemberDIE, dwarf::DW_AT_declaration);
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/AtomicExpandPass.cpp b/external/llvm-project/llvm/lib/CodeGen/AtomicExpandPass.cpp
index c376de877ac7..044f0732779f 100644
--- a/external/llvm-project/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -1570,12 +1570,12 @@ bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
 }
 
 bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
+  // TODO: Add floating point support.
   auto C = dyn_cast<ConstantInt>(RMWI->getValOperand());
   if (!C)
     return false;
 
-  AtomicRMWInst::BinOp Op = RMWI->getOperation();
-  switch (Op) {
+  switch (RMWI->getOperation()) {
   case AtomicRMWInst::Add:
   case AtomicRMWInst::Sub:
   case AtomicRMWInst::Or:
@@ -1583,7 +1583,14 @@ bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
     return C->isZero();
   case AtomicRMWInst::And:
     return C->isMinusOne();
-  // FIXME: we could also treat Min/Max/UMin/UMax by the INT_MIN/INT_MAX/...
+  case AtomicRMWInst::Min:
+    return C->isMaxValue(true);
+  case AtomicRMWInst::Max:
+    return C->isMinValue(true);
+  case AtomicRMWInst::UMin:
+    return C->isMaxValue(false);
+  case AtomicRMWInst::UMax:
+    return C->isMinValue(false);
   default:
     return false;
   }
diff --git a/external/llvm-project/llvm/lib/CodeGen/BasicBlockPathCloning.cpp b/external/llvm-project/llvm/lib/CodeGen/BasicBlockPathCloning.cpp
index 19f824850607..b58c60d1db0a 100644
--- a/external/llvm-project/llvm/lib/CodeGen/BasicBlockPathCloning.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/BasicBlockPathCloning.cpp
@@ -121,14 +121,21 @@ bool IsValidCloning(const MachineFunction &MF,
       }
       if (PathBB->isMachineBlockAddressTaken()) {
         // Avoid cloning blocks which have their address taken since we can't
-        // rewire branches to those blocks as easily (e.g., branches within
-        // inline assembly).
+        // rewire branches to those blocks as easily.
         WithColor::warning()
             << "block #" << BBID
             << " has its machine block address taken in function "
             << MF.getName() << "\n";
         return false;
       }
+      if (PathBB->isInlineAsmBrIndirectTarget()) {
+        // Similarly for branches to the block within an asm goto.
+        WithColor::warning()
+            << "block #" << BBID
+            << " is a branch target of an 'asm goto' in function "
+            << MF.getName() << "\n";
+        return false;
+      }
     }
 
     if (I != ClonePath.size() - 1 && !PathBB->empty() &&
diff --git a/external/llvm-project/llvm/lib/CodeGen/BranchFolding.cpp b/external/llvm-project/llvm/lib/CodeGen/BranchFolding.cpp
index e0f7466ceacf..ff9f0ff5d5bc 100644
--- a/external/llvm-project/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/BranchFolding.cpp
@@ -862,7 +862,7 @@ void BranchFolder::mergeCommonTails(unsigned commonTailIndex) {
             "Reached BB end within common tail");
       }
       assert(MI.isIdenticalTo(*Pos) && "Expected matching MIIs!");
-      DL = DILocation::getMergedLocation(DL, Pos->getDebugLoc());
+      DL = DebugLoc::getMergedLocation(DL, Pos->getDebugLoc());
       NextCommonInsts[i] = ++Pos;
     }
     MI.setDebugLoc(DL);
diff --git a/external/llvm-project/llvm/lib/CodeGen/BreakFalseDeps.cpp b/external/llvm-project/llvm/lib/CodeGen/BreakFalseDeps.cpp
index 05eed969a18a..7eef4a9d12b1 100644
--- a/external/llvm-project/llvm/lib/CodeGen/BreakFalseDeps.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/BreakFalseDeps.cpp
@@ -285,7 +285,7 @@ bool BreakFalseDeps::runOnMachineFunction(MachineFunction &mf) {
   TRI = MF->getSubtarget().getRegisterInfo();
   RDA = &getAnalysis<ReachingDefAnalysis>();
 
-  RegClassInfo.runOnMachineFunction(mf);
+  RegClassInfo.runOnMachineFunction(mf, /*Rev=*/true);
 
   LLVM_DEBUG(dbgs() << "********** BREAK FALSE DEPENDENCIES **********\n");
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp b/external/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 52263026d6ce..3792b456c836 100644
--- a/external/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -3335,8 +3335,7 @@ class TypePromotionTransaction {
 
       // Record where we would have to re-insert the instruction in the sequence
       // of DbgRecords, if we ended up reinserting.
-      if (BB->IsNewDbgInfoFormat)
-        BeforeDbgRecord = Inst->getDbgReinsertionPosition();
+      BeforeDbgRecord = Inst->getDbgReinsertionPosition();
 
       if (HasPrevInstruction) {
         Point.PrevInst = std::prev(Inst->getIterator());
@@ -5447,11 +5446,17 @@ bool AddressingModeMatcher::matchAddr(Value *Addr, unsigned Depth) {
       TPT.getRestorationPoint();
   if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {
     if (CI->getValue().isSignedIntN(64)) {
-      // Fold in immediates if legal for the target.
-      AddrMode.BaseOffs += CI->getSExtValue();
-      if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
-        return true;
-      AddrMode.BaseOffs -= CI->getSExtValue();
+      // Check if the addition would result in a signed overflow.
+      int64_t Result;
+      bool Overflow =
+          AddOverflow(AddrMode.BaseOffs, CI->getSExtValue(), Result);
+      if (!Overflow) {
+        // Fold in immediates if legal for the target.
+        AddrMode.BaseOffs = Result;
+        if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
+          return true;
+        AddrMode.BaseOffs -= CI->getSExtValue();
+      }
     }
   } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {
     // If this is a global variable, try to fold it into the addressing mode.
@@ -5939,8 +5944,17 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
   // The current BB may be optimized multiple times, we can't guarantee the
   // reuse of Addr happens later, call findInsertPos to find an appropriate
   // insert position.
-  IRBuilder<> Builder(MemoryInst->getParent(),
-                      findInsertPos(Addr, MemoryInst, SunkAddr));
+  auto InsertPos = findInsertPos(Addr, MemoryInst, SunkAddr);
+
+  // TODO: Adjust insert point considering (Base|Scaled)Reg if possible.
+  if (!SunkAddr) {
+    auto &DT = getDT(*MemoryInst->getFunction());
+    if ((AddrMode.BaseReg && !DT.dominates(AddrMode.BaseReg, &*InsertPos)) ||
+        (AddrMode.ScaledReg && !DT.dominates(AddrMode.ScaledReg, &*InsertPos)))
+      return Modified;
+  }
+
+  IRBuilder<> Builder(MemoryInst->getParent(), InsertPos);
 
   if (SunkAddr) {
     LLVM_DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode
diff --git a/external/llvm-project/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/external/llvm-project/llvm/lib/CodeGen/ExpandVectorPredication.cpp
index 1bb0763fcf57..d8e3f5fbb31d 100644
--- a/external/llvm-project/llvm/lib/CodeGen/ExpandVectorPredication.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/ExpandVectorPredication.cpp
@@ -507,8 +507,7 @@ bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
     // TODO add caching
     IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
     Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
-    Value *VScale = Builder.CreateIntrinsic(Intrinsic::vscale, Int32Ty, {},
-                                            /*FMFSource=*/nullptr, "vscale");
+    Value *VScale = Builder.CreateVScale(Int32Ty, "vscale");
     MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",
                                /*NUW*/ true, /*NSW*/ false);
   } else {
diff --git a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
index 10c72641ce2d..e3e6c72165eb 100644
--- a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -53,8 +53,7 @@ CSEMIRBuilder::getDominatingInstrForID(FoldingSetNodeID &ID,
     } else if (!dominates(MI, CurrPos)) {
       // Update the spliced machineinstr's debug location by merging it with the
       // debug location of the instruction at the insertion point.
-      auto *Loc = DILocation::getMergedLocation(getDebugLoc().get(),
-                                                MI->getDebugLoc().get());
+      auto Loc = DebugLoc::getMergedLocation(getDebugLoc(), MI->getDebugLoc());
       MI->setDebugLoc(Loc);
       CurMBB->splice(CurrPos, CurMBB, MI);
     }
@@ -170,7 +169,7 @@ CSEMIRBuilder::generateCopiesIfRequired(ArrayRef<DstOp> DstOps,
     if (Observer)
       Observer->changingInstr(*MIB);
     MIB->setDebugLoc(
-        DILocation::getMergedLocation(MIB->getDebugLoc(), getDebugLoc()));
+        DebugLoc::getMergedLocation(MIB->getDebugLoc(), getDebugLoc()));
     if (Observer)
       Observer->changedInstr(*MIB);
   }
diff --git a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index 692befde71cb..1286af864fb3 100644
--- a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -222,12 +222,11 @@ void GISelValueTracking::computeKnownBitsImpl(Register R, KnownBits &Known,
     // Collect the known bits that are shared by every demanded vector element.
     Known.Zero.setAllBits();
     Known.One.setAllBits();
-    for (unsigned i = 0, e = MI.getNumOperands() - 1; i < e; ++i) {
-      if (!DemandedElts[i])
+    for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
+      if (!DemandedElts[I])
         continue;
 
-      computeKnownBitsImpl(MI.getOperand(i + 1).getReg(), Known2, APInt(1, 1),
-                           Depth + 1);
+      computeKnownBitsImpl(MO.getReg(), Known2, APInt(1, 1), Depth + 1);
 
       // Known bits are the values that are shared by every demanded element.
       Known = Known.intersectWith(Known2);
@@ -683,14 +682,12 @@ void GISelValueTracking::computeKnownBitsImpl(Register R, KnownBits &Known,
     Known.One.setAllBits();
     unsigned NumSubVectorElts =
         MRI.getType(MI.getOperand(1).getReg()).getNumElements();
-    unsigned NumSubVectors = MI.getNumOperands() - 1;
 
-    for (unsigned i = 0; i != NumSubVectors; ++i) {
+    for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
       APInt DemandedSub =
-          DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
+          DemandedElts.extractBits(NumSubVectorElts, I * NumSubVectorElts);
       if (!!DemandedSub) {
-        computeKnownBitsImpl(MI.getOperand(i + 1).getReg(), Known2, DemandedSub,
-                             Depth + 1);
+        computeKnownBitsImpl(MO.getReg(), Known2, DemandedSub, Depth + 1);
 
         Known = Known.intersectWith(Known2);
       }
@@ -1049,7 +1046,8 @@ void GISelValueTracking::computeKnownFPClass(Register R,
     //
     if ((Known.KnownFPClasses & fcZero) != fcNone &&
         !Known.isKnownNeverSubnormal()) {
-      DenormalMode Mode = MF->getDenormalMode(getFltSemanticForLLT(DstTy));
+      DenormalMode Mode =
+          MF->getDenormalMode(getFltSemanticForLLT(DstTy.getScalarType()));
       if (Mode != DenormalMode::getIEEE())
         Known.KnownFPClasses |= fcZero;
     }
@@ -1111,8 +1109,8 @@ void GISelValueTracking::computeKnownFPClass(Register R,
 
     // If the parent function flushes denormals, the canonical output cannot
     // be a denormal.
-    LLT Ty = MRI.getType(Val);
-    const fltSemantics &FPType = getFltSemanticForLLT(Ty.getScalarType());
+    LLT Ty = MRI.getType(Val).getScalarType();
+    const fltSemantics &FPType = getFltSemanticForLLT(Ty);
     DenormalMode DenormMode = MF->getDenormalMode(FPType);
     if (DenormMode == DenormalMode::getIEEE()) {
       if (KnownSrc.isKnownNever(fcPosZero))
@@ -1222,8 +1220,8 @@ void GISelValueTracking::computeKnownFPClass(Register R,
     if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
       Known.knownNot(fcNan);
 
-    LLT Ty = MRI.getType(Val);
-    const fltSemantics &FltSem = getFltSemanticForLLT(Ty.getScalarType());
+    LLT Ty = MRI.getType(Val).getScalarType();
+    const fltSemantics &FltSem = getFltSemanticForLLT(Ty);
     DenormalMode Mode = MF->getDenormalMode(FltSem);
 
     if (KnownSrc.isKnownNeverLogicalZero(Mode))
@@ -1341,19 +1339,19 @@ void GISelValueTracking::computeKnownFPClass(Register R,
           Known.knownNot(KnownFPClass::OrderedLessThanZeroMask);
 
         // (fadd x, 0.0) is guaranteed to return +0.0, not -0.0.
-        if ((KnownLHS.isKnownNeverLogicalNegZero(
-                 MF->getDenormalMode(getFltSemanticForLLT(DstTy))) ||
-             KnownRHS.isKnownNeverLogicalNegZero(
-                 MF->getDenormalMode(getFltSemanticForLLT(DstTy)))) &&
+        if ((KnownLHS.isKnownNeverLogicalNegZero(MF->getDenormalMode(
+                 getFltSemanticForLLT(DstTy.getScalarType()))) ||
+             KnownRHS.isKnownNeverLogicalNegZero(MF->getDenormalMode(
+                 getFltSemanticForLLT(DstTy.getScalarType())))) &&
             // Make sure output negative denormal can't flush to -0
             outputDenormalIsIEEEOrPosZero(*MF, DstTy))
           Known.knownNot(fcNegZero);
       } else {
         // Only fsub -0, +0 can return -0
-        if ((KnownLHS.isKnownNeverLogicalNegZero(
-                 MF->getDenormalMode(getFltSemanticForLLT(DstTy))) ||
-             KnownRHS.isKnownNeverLogicalPosZero(
-                 MF->getDenormalMode(getFltSemanticForLLT(DstTy)))) &&
+        if ((KnownLHS.isKnownNeverLogicalNegZero(MF->getDenormalMode(
+                 getFltSemanticForLLT(DstTy.getScalarType()))) ||
+             KnownRHS.isKnownNeverLogicalPosZero(MF->getDenormalMode(
+                 getFltSemanticForLLT(DstTy.getScalarType())))) &&
             // Make sure output negative denormal can't flush to -0
             outputDenormalIsIEEEOrPosZero(*MF, DstTy))
           Known.knownNot(fcNegZero);
@@ -1399,11 +1397,11 @@ void GISelValueTracking::computeKnownFPClass(Register R,
     }
 
     if ((KnownRHS.isKnownNeverInfinity() ||
-         KnownLHS.isKnownNeverLogicalZero(
-             MF->getDenormalMode(getFltSemanticForLLT(DstTy)))) &&
+         KnownLHS.isKnownNeverLogicalZero(MF->getDenormalMode(
+             getFltSemanticForLLT(DstTy.getScalarType())))) &&
         (KnownLHS.isKnownNeverInfinity() ||
          KnownRHS.isKnownNeverLogicalZero(
-             MF->getDenormalMode(getFltSemanticForLLT(DstTy)))))
+             MF->getDenormalMode(getFltSemanticForLLT(DstTy.getScalarType())))))
       Known.knownNot(fcNan);
 
     break;
@@ -1455,10 +1453,10 @@ void GISelValueTracking::computeKnownFPClass(Register R,
       if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
           (KnownLHS.isKnownNeverInfinity() ||
            KnownRHS.isKnownNeverInfinity()) &&
-          ((KnownLHS.isKnownNeverLogicalZero(
-               MF->getDenormalMode(getFltSemanticForLLT(DstTy)))) ||
-           (KnownRHS.isKnownNeverLogicalZero(
-               MF->getDenormalMode(getFltSemanticForLLT(DstTy)))))) {
+          ((KnownLHS.isKnownNeverLogicalZero(MF->getDenormalMode(
+               getFltSemanticForLLT(DstTy.getScalarType())))) ||
+           (KnownRHS.isKnownNeverLogicalZero(MF->getDenormalMode(
+               getFltSemanticForLLT(DstTy.getScalarType())))))) {
         Known.knownNot(fcNan);
       }
 
@@ -1471,8 +1469,8 @@ void GISelValueTracking::computeKnownFPClass(Register R,
       // Inf REM x and x REM 0 produce NaN.
       if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
           KnownLHS.isKnownNeverInfinity() &&
-          KnownRHS.isKnownNeverLogicalZero(
-              MF->getDenormalMode(getFltSemanticForLLT(DstTy)))) {
+          KnownRHS.isKnownNeverLogicalZero(MF->getDenormalMode(
+              getFltSemanticForLLT(DstTy.getScalarType())))) {
         Known.knownNot(fcNan);
       }
 
@@ -1497,10 +1495,10 @@ void GISelValueTracking::computeKnownFPClass(Register R,
     // Infinity, nan and zero propagate from source.
     computeKnownFPClass(R, DemandedElts, InterestedClasses, Known, Depth + 1);
 
-    LLT DstTy = MRI.getType(Dst);
-    const fltSemantics &DstSem = getFltSemanticForLLT(DstTy.getScalarType());
-    LLT SrcTy = MRI.getType(Src);
-    const fltSemantics &SrcSem = getFltSemanticForLLT(SrcTy.getScalarType());
+    LLT DstTy = MRI.getType(Dst).getScalarType();
+    const fltSemantics &DstSem = getFltSemanticForLLT(DstTy);
+    LLT SrcTy = MRI.getType(Src).getScalarType();
+    const fltSemantics &SrcSem = getFltSemanticForLLT(SrcTy);
 
     // All subnormal inputs should be in the normal range in the result type.
     if (APFloat::isRepresentableAsNormalIn(SrcSem, DstSem)) {
@@ -1693,6 +1691,10 @@ void GISelValueTracking::computeKnownFPClass(Register R,
   }
   case TargetOpcode::COPY: {
     Register Src = MI.getOperand(1).getReg();
+
+    if (!Src.isVirtual())
+      return;
+
     computeKnownFPClass(Src, DemandedElts, InterestedClasses, Known, Depth + 1);
     break;
   }
@@ -1944,12 +1946,35 @@ unsigned GISelValueTracking::computeNumSignBits(Register R,
     // Collect the known bits that are shared by every demanded vector element.
     FirstAnswer = TyBits;
     APInt SingleDemandedElt(1, 1);
-    for (unsigned i = 0, e = MI.getNumOperands() - 1; i < e; ++i) {
-      if (!DemandedElts[i])
+    for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
+      if (!DemandedElts[I])
+        continue;
+
+      unsigned Tmp2 =
+          computeNumSignBits(MO.getReg(), SingleDemandedElt, Depth + 1);
+      FirstAnswer = std::min(FirstAnswer, Tmp2);
+
+      // If we don't know any bits, early out.
+      if (FirstAnswer == 1)
+        break;
+    }
+    break;
+  }
+  case TargetOpcode::G_CONCAT_VECTORS: {
+    if (MRI.getType(MI.getOperand(0).getReg()).isScalableVector())
+      break;
+    FirstAnswer = TyBits;
+    // Determine the minimum number of sign bits across all demanded
+    // elts of the input vectors. Early out if the result is already 1.
+    unsigned NumSubVectorElts =
+        MRI.getType(MI.getOperand(1).getReg()).getNumElements();
+    for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
+      APInt DemandedSub =
+          DemandedElts.extractBits(NumSubVectorElts, I * NumSubVectorElts);
+      if (!DemandedSub)
         continue;
+      unsigned Tmp2 = computeNumSignBits(MO.getReg(), DemandedSub, Depth + 1);
 
-      unsigned Tmp2 = computeNumSignBits(MI.getOperand(i + 1).getReg(),
-                                         SingleDemandedElt, Depth + 1);
       FirstAnswer = std::min(FirstAnswer, Tmp2);
 
       // If we don't know any bits, early out.
diff --git a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 7b18a98d7f3c..028bffd1bf5a 100644
--- a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -118,7 +118,7 @@ LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
 LegalizerHelper::LegalizeResult
 LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
                                    LostDebugLocObserver &LocObserver) {
-  LLVM_DEBUG(dbgs() << "Legalizing: " << MI);
+  LLVM_DEBUG(dbgs() << "\nLegalizing: " << MI);
 
   MIRBuilder.setInstrAndDebugLoc(MI);
 
@@ -8446,10 +8446,10 @@ LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
     }
   }
 
-  if (DstTy.isScalar())
-    MIRBuilder.buildCopy(DstReg, BuildVec[0]);
-  else
+  if (DstTy.isVector())
     MIRBuilder.buildBuildVector(DstReg, BuildVec);
+  else
+    MIRBuilder.buildCopy(DstReg, BuildVec[0]);
   MI.eraseFromParent();
   return Legalized;
 }
diff --git a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 78cd9bc7891e..f68420ed66e4 100644
--- a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -370,7 +370,7 @@ bool LoadStoreOpt::doSingleStoreMerge(SmallVectorImpl<GStore *> &Stores) {
   // For each store, compute pairwise merged debug locs.
   DebugLoc MergedLoc = Stores.front()->getDebugLoc();
   for (auto *Store : drop_begin(Stores))
-    MergedLoc = DILocation::getMergedLocation(MergedLoc, Store->getDebugLoc());
+    MergedLoc = DebugLoc::getMergedLocation(MergedLoc, Store->getDebugLoc());
 
   Builder.setInstr(*Stores.back());
   Builder.setDebugLoc(MergedLoc);
diff --git a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 64af7a57e8d1..2584175121d6 100644
--- a/external/llvm-project/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2003,6 +2003,17 @@ Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) {
   return IntegerType::get(C, Ty.getSizeInBits());
 }
 
+bool llvm::isAssertMI(const MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  default:
+    return false;
+  case TargetOpcode::G_ASSERT_ALIGN:
+  case TargetOpcode::G_ASSERT_SEXT:
+  case TargetOpcode::G_ASSERT_ZEXT:
+    return true;
+  }
+}
+
 APInt llvm::GIConstant::getScalarValue() const {
   assert(Kind == GIConstantKind::Scalar && "Expected scalar constant");
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/GlobalMergeFunctions.cpp b/external/llvm-project/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
index 9f1eb4b24772..84dc4ab0a552 100644
--- a/external/llvm-project/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
 #include "llvm/CGData/CodeGenData.h"
+#include "llvm/CGData/CodeGenDataWriter.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/InitializePasses.h"
@@ -526,7 +527,10 @@ void GlobalMergeFunc::emitFunctionMap(Module &M) {
   SmallVector<char> Buf;
   raw_svector_ostream OS(Buf);
 
-  StableFunctionMapRecord::serialize(OS, LocalFunctionMap.get());
+  std::vector<CGDataPatchItem> PatchItems;
+  StableFunctionMapRecord::serialize(OS, LocalFunctionMap.get(), PatchItems);
+  CGDataOStream COS(OS);
+  COS.patch(PatchItems);
 
   std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer(
       OS.str(), "in-memory stable function map", false);
diff --git a/external/llvm-project/llvm/lib/CodeGen/IndirectBrExpandPass.cpp b/external/llvm-project/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
index 15f24207458c..84f385888b0c 100644
--- a/external/llvm-project/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
@@ -144,8 +144,8 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU) {
     if (BlockAddressUseIt == BB.use_end())
       continue;
 
-    assert(std::find_if(std::next(BlockAddressUseIt), BB.use_end(),
-                        IsBlockAddressUse) == BB.use_end() &&
+    assert(std::none_of(std::next(BlockAddressUseIt), BB.use_end(),
+                        IsBlockAddressUse) &&
            "There should only ever be a single blockaddress use because it is "
            "a constant and should be uniqued.");
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/LexicalScopes.cpp b/external/llvm-project/llvm/lib/CodeGen/LexicalScopes.cpp
index 6dbd2ca00f31..5916f619537a 100644
--- a/external/llvm-project/llvm/lib/CodeGen/LexicalScopes.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/LexicalScopes.cpp
@@ -287,8 +287,7 @@ void LexicalScopes::getMachineBasicBlocks(
     return;
 
   if (Scope == CurrentFnLexicalScope) {
-    for (const auto &MBB : *MF)
-      MBBs.insert(&MBB);
+    MBBs.insert_range(llvm::make_pointer_range(*MF));
     return;
   }
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/external/llvm-project/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index a57bda54f918..1e9fcf35255b 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -763,22 +763,25 @@ bool MIRParserImpl::setupRegisterInfo(const PerFunctionMIParsingState &PFS,
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
 
-  bool Error = false;
+  SmallVector<std::string> Errors;
+
   // Create VRegs
   auto populateVRegInfo = [&](const VRegInfo &Info, const Twine &Name) {
     Register Reg = Info.VReg;
     switch (Info.Kind) {
     case VRegInfo::UNKNOWN:
-      error(Twine("Cannot determine class/bank of virtual register ") +
-            Name + " in function '" + MF.getName() + "'");
-      Error = true;
+      Errors.push_back(
+          (Twine("Cannot determine class/bank of virtual register ") + Name +
+           " in function '" + MF.getName() + "'")
+              .str());
       break;
     case VRegInfo::NORMAL:
       if (!Info.D.RC->isAllocatable()) {
-        error(Twine("Cannot use non-allocatable class '") +
-              TRI->getRegClassName(Info.D.RC) + "' for virtual register " +
-              Name + " in function '" + MF.getName() + "'");
-        Error = true;
+        Errors.push_back((Twine("Cannot use non-allocatable class '") +
+                          TRI->getRegClassName(Info.D.RC) +
+                          "' for virtual register " + Name + " in function '" +
+                          MF.getName() + "'")
+                             .str());
         break;
       }
 
@@ -820,7 +823,14 @@ bool MIRParserImpl::setupRegisterInfo(const PerFunctionMIParsingState &PFS,
     }
   }
 
-  return Error;
+  if (Errors.empty())
+    return false;
+
+  // Report errors in a deterministic order.
+  sort(Errors);
+  for (auto &E : Errors)
+    error(E);
+  return true;
 }
 
 bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS,
diff --git a/external/llvm-project/llvm/lib/CodeGen/MIRPrinter.cpp b/external/llvm-project/llvm/lib/CodeGen/MIRPrinter.cpp
index b7b8f66736a4..7710b503facc 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -70,8 +70,6 @@ static cl::opt<bool> SimplifyMIR(
 static cl::opt<bool> PrintLocations("mir-debug-loc", cl::Hidden, cl::init(true),
                                     cl::desc("Print MIR debug-locations"));
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace {
 
 /// This structure describes how to print out stack object references.
@@ -967,18 +965,11 @@ void MIRFormatter::printIRValue(raw_ostream &OS, const Value &V,
 }
 
 void llvm::printMIR(raw_ostream &OS, const Module &M) {
-  ScopedDbgInfoFormatSetter FormatSetter(const_cast<Module &>(M),
-                                         UseNewDbgInfoFormat);
-
   yaml::Output Out(OS);
   Out << const_cast<Module &>(M);
 }
 
 void llvm::printMIR(raw_ostream &OS, const MachineModuleInfo &MMI,
                     const MachineFunction &MF) {
-  // RemoveDIs: as there's no textual form for DbgRecords yet, print debug-info
-  // in dbg.value format.
-  ScopedDbgInfoFormatSetter FormatSetter(
-      const_cast<Function &>(MF.getFunction()), UseNewDbgInfoFormat);
   printMF(OS, MMI, MF);
 }
diff --git a/external/llvm-project/llvm/lib/CodeGen/MachineBasicBlock.cpp b/external/llvm-project/llvm/lib/CodeGen/MachineBasicBlock.cpp
index ccc164a0881e..48b406e016c0 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -1574,7 +1574,7 @@ MachineBasicBlock::findBranchDebugLoc() {
     DL = TI->getDebugLoc();
     for (++TI ; TI != end() ; ++TI)
       if (TI->isBranch())
-        DL = DILocation::getMergedLocation(DL, TI->getDebugLoc());
+        DL = DebugLoc::getMergedLocation(DL, TI->getDebugLoc());
   }
   return DL;
 }
diff --git a/external/llvm-project/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/external/llvm-project/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 49d62e975fe1..742de1101faa 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -872,12 +872,6 @@ void MachineCopyPropagation::forwardUses(MachineInstr &MI) {
     ++NumCopyForwards;
     Changed = true;
   }
-  // Attempt to canonicalize/optimize the instruction now its arguments have
-  // been mutated.
-  if (TII->simplifyInstruction(MI)) {
-    Changed = true;
-    LLVM_DEBUG(dbgs() << "MCP: After optimizeInstruction: " << MI);
-  }
 }
 
 void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
@@ -889,10 +883,8 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
     std::optional<DestSourcePair> CopyOperands =
         isCopyInstr(MI, *TII, UseCopyInstr);
     if (CopyOperands) {
-
       Register RegSrc = CopyOperands->Source->getReg();
       Register RegDef = CopyOperands->Destination->getReg();
-
       if (!TRI->regsOverlap(RegDef, RegSrc)) {
         assert(RegDef.isPhysical() && RegSrc.isPhysical() &&
               "MachineCopyPropagation should be run after register allocation!");
@@ -917,51 +909,6 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
         //  %ecx = COPY %eax
         if (eraseIfRedundant(MI, Def, Src) || eraseIfRedundant(MI, Src, Def))
           continue;
-
-        forwardUses(MI);
-
-        // Src may have been changed by forwardUses()
-        CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr);
-        Src = CopyOperands->Source->getReg().asMCReg();
-
-        // If Src is defined by a previous copy, the previous copy cannot be
-        // eliminated.
-        ReadRegister(Src, MI, RegularUse);
-        for (const MachineOperand &MO : MI.implicit_operands()) {
-          if (!MO.isReg() || !MO.readsReg())
-            continue;
-          MCRegister Reg = MO.getReg().asMCReg();
-          if (!Reg)
-            continue;
-          ReadRegister(Reg, MI, RegularUse);
-        }
-
-        LLVM_DEBUG(dbgs() << "MCP: Copy is a deletion candidate: "; MI.dump());
-
-        // Copy is now a candidate for deletion.
-        if (!MRI->isReserved(Def))
-          MaybeDeadCopies.insert(&MI);
-
-        // If 'Def' is previously source of another copy, then this earlier copy's
-        // source is no longer available. e.g.
-        // %xmm9 = copy %xmm2
-        // ...
-        // %xmm2 = copy %xmm0
-        // ...
-        // %xmm2 = copy %xmm9
-        Tracker.clobberRegister(Def, *TRI, *TII, UseCopyInstr);
-        for (const MachineOperand &MO : MI.implicit_operands()) {
-          if (!MO.isReg() || !MO.isDef())
-            continue;
-          MCRegister Reg = MO.getReg().asMCReg();
-          if (!Reg)
-            continue;
-          Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr);
-        }
-
-        Tracker.trackCopy(&MI, *TRI, *TII, UseCopyInstr);
-
-        continue;
       }
     }
 
@@ -979,20 +926,36 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
 
     forwardUses(MI);
 
-    // It's possible that the previous transformation has resulted in a no-op
-    // register move (i.e. one where source and destination registers are the
-    // same and are not referring to a reserved register). If so, delete it.
-    CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr);
-    if (CopyOperands &&
-        CopyOperands->Source->getReg() == CopyOperands->Destination->getReg() &&
-        !MRI->isReserved(CopyOperands->Source->getReg())) {
-      MI.eraseFromParent();
-      NumDeletes++;
+    // Attempt to canonicalize/optimize the instruction now its arguments have
+    // been mutated.  This may convert MI from a non-copy to a copy instruction.
+    if (TII->simplifyInstruction(MI)) {
       Changed = true;
-      continue;
+      LLVM_DEBUG(dbgs() << "MCP: After simplifyInstruction: " << MI);
+    }
+
+    CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr);
+    if (CopyOperands) {
+      Register RegSrc = CopyOperands->Source->getReg();
+      Register RegDef = CopyOperands->Destination->getReg();
+      // It's possible that the previous transformations have resulted in a
+      // no-op register move (i.e. one where source and destination registers
+      // are the same and are not referring to a reserved register). If so,
+      // delete it.
+      if (RegSrc == RegDef && !MRI->isReserved(RegSrc)) {
+        MI.eraseFromParent();
+        NumDeletes++;
+        Changed = true;
+        continue;
+      }
+
+      if (!TRI->regsOverlap(RegDef, RegSrc)) {
+        // Copy is now a candidate for deletion.
+        MCRegister Def = RegDef.asMCReg();
+        if (!MRI->isReserved(Def))
+          MaybeDeadCopies.insert(&MI);
+      }
     }
 
-    // Not a copy.
     SmallVector<Register, 4> Defs;
     const MachineOperand *RegMask = nullptr;
     for (const MachineOperand &MO : MI.operands()) {
@@ -1072,6 +1035,14 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
     // Any previous copy definition or reading the Defs is no longer available.
     for (MCRegister Reg : Defs)
       Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr);
+
+    if (CopyOperands) {
+      Register RegSrc = CopyOperands->Source->getReg();
+      Register RegDef = CopyOperands->Destination->getReg();
+      if (!TRI->regsOverlap(RegDef, RegSrc)) {
+        Tracker.trackCopy(&MI, *TRI, *TII, UseCopyInstr);
+      }
+    }
   }
 
   bool TracksLiveness = MRI->tracksLiveness();
diff --git a/external/llvm-project/llvm/lib/CodeGen/MachinePipeliner.cpp b/external/llvm-project/llvm/lib/CodeGen/MachinePipeliner.cpp
index 5a9b21dc67c6..d2c79f64afe6 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -265,6 +265,82 @@ struct SUnitWithMemInfo {
   bool getUnderlyingObjects();
 };
 
+/// Add loop-carried chain dependencies. This class handles the same type of
+/// dependencies added by `ScheduleDAGInstrs::buildSchedGraph`, but takes into
+/// account dependencies across iterations.
+class LoopCarriedOrderDepsTracker {
+  // Type of instruction that is relevant to order-dependencies
+  enum class InstrTag {
+    Barrier = 0,      ///< A barrier event instruction.
+    LoadOrStore = 1,  ///< An instruction that may load or store memory, but is
+                      ///< not a barrier event.
+    FPExceptions = 2, ///< An instruction that does not match above, but may
+                      ///< raise floatin-point exceptions.
+  };
+
+  struct TaggedSUnit : PointerIntPair<SUnit *, 2> {
+    TaggedSUnit(SUnit *SU, InstrTag Tag)
+        : PointerIntPair<SUnit *, 2>(SU, unsigned(Tag)) {}
+
+    InstrTag getTag() const { return InstrTag(getInt()); }
+  };
+
+  /// Holds loads and stores with memory related information.
+  struct LoadStoreChunk {
+    SmallVector<SUnitWithMemInfo, 4> Loads;
+    SmallVector<SUnitWithMemInfo, 4> Stores;
+
+    void append(SUnit *SU);
+  };
+
+  SwingSchedulerDAG *DAG;
+  BatchAAResults *BAA;
+  std::vector<SUnit> &SUnits;
+
+  /// The size of SUnits, for convenience.
+  const unsigned N;
+
+  /// Loop-carried Edges.
+  std::vector<BitVector> LoopCarried;
+
+  /// Instructions related to chain dependencies. They are one of the
+  /// following:
+  ///
+  ///  1. Barrier event.
+  ///  2. Load, but neither a barrier event, invariant load, nor may load trap
+  ///     value.
+  ///  3. Store, but not a barrier event.
+  ///  4. None of them, but may raise floating-point exceptions.
+  ///
+  /// This is used when analyzing loop-carried dependencies that access global
+  /// barrier instructions.
+  std::vector<TaggedSUnit> TaggedSUnits;
+
+  const TargetInstrInfo *TII = nullptr;
+  const TargetRegisterInfo *TRI = nullptr;
+
+public:
+  LoopCarriedOrderDepsTracker(SwingSchedulerDAG *SSD, BatchAAResults *BAA,
+                              const TargetInstrInfo *TII,
+                              const TargetRegisterInfo *TRI);
+
+  /// The main function to compute loop-carried order-dependencies.
+  void computeDependencies();
+
+  const BitVector &getLoopCarried(unsigned Idx) const {
+    return LoopCarried[Idx];
+  }
+
+private:
+  /// Tags to \p SU if the instruction may affect the order-dependencies.
+  std::optional<InstrTag> getInstrTag(SUnit *SU) const;
+
+  void addLoopCarriedDepenenciesForChunks(const LoadStoreChunk &From,
+                                          const LoadStoreChunk &To);
+
+  void computeDependenciesAux();
+};
+
 } // end anonymous namespace
 
 /// The "main" function for implementing Swing Modulo Scheduling.
@@ -592,13 +668,19 @@ void SwingSchedulerDAG::setMAX_II() {
 /// scheduling part of the Swing Modulo Scheduling algorithm.
 void SwingSchedulerDAG::schedule() {
   buildSchedGraph(AA);
-  addLoopCarriedDependences();
+  const LoopCarriedEdges LCE = addLoopCarriedDependences();
   updatePhiDependences();
   Topo.InitDAGTopologicalSorting();
   changeDependences();
   postProcessDAG();
   DDG = std::make_unique<SwingSchedulerDDG>(SUnits, &EntrySU, &ExitSU);
-  LLVM_DEBUG(dump());
+  LLVM_DEBUG({
+    dump();
+    dbgs() << "===== Loop Carried Edges Begin =====\n";
+    for (SUnit &SU : SUnits)
+      LCE.dump(&SU, TRI, &MRI);
+    dbgs() << "===== Loop Carried Edges End =====\n";
+  });
 
   NodeSetType NodeSets;
   findCircuits(NodeSets);
@@ -831,15 +913,6 @@ static bool isSuccOrder(SUnit *SUa, SUnit *SUb) {
   return false;
 }
 
-/// Return true if the instruction causes a chain between memory
-/// references before and after it.
-static bool isDependenceBarrier(MachineInstr &MI) {
-  return MI.isCall() || MI.mayRaiseFPException() ||
-         MI.hasUnmodeledSideEffects() ||
-         (MI.hasOrderedMemoryRef() &&
-          (!MI.mayLoad() || !MI.isDereferenceableInvariantLoad()));
-}
-
 SUnitWithMemInfo::SUnitWithMemInfo(SUnit *SU) : SU(SU) {
   if (!getUnderlyingObjects())
     return;
@@ -940,28 +1013,111 @@ static bool hasLoopCarriedMemDep(const SUnitWithMemInfo &Src,
   return false;
 }
 
+void LoopCarriedOrderDepsTracker::LoadStoreChunk::append(SUnit *SU) {
+  const MachineInstr *MI = SU->getInstr();
+  if (!MI->mayLoadOrStore())
+    return;
+  (MI->mayStore() ? Stores : Loads).emplace_back(SU);
+}
+
+LoopCarriedOrderDepsTracker::LoopCarriedOrderDepsTracker(
+    SwingSchedulerDAG *SSD, BatchAAResults *BAA, const TargetInstrInfo *TII,
+    const TargetRegisterInfo *TRI)
+    : DAG(SSD), BAA(BAA), SUnits(DAG->SUnits), N(SUnits.size()),
+      LoopCarried(N, BitVector(N)), TII(TII), TRI(TRI) {}
+
+void LoopCarriedOrderDepsTracker::computeDependencies() {
+  // Traverse all instructions and extract only what we are targetting.
+  for (auto &SU : SUnits) {
+    auto Tagged = getInstrTag(&SU);
+
+    // This instruction has no loop-carried order-dependencies.
+    if (!Tagged)
+      continue;
+    TaggedSUnits.emplace_back(&SU, *Tagged);
+  }
+
+  computeDependenciesAux();
+}
+
+std::optional<LoopCarriedOrderDepsTracker::InstrTag>
+LoopCarriedOrderDepsTracker::getInstrTag(SUnit *SU) const {
+  MachineInstr *MI = SU->getInstr();
+  if (TII->isGlobalMemoryObject(MI))
+    return InstrTag::Barrier;
+
+  if (MI->mayStore() ||
+      (MI->mayLoad() && !MI->isDereferenceableInvariantLoad()))
+    return InstrTag::LoadOrStore;
+
+  if (MI->mayRaiseFPException())
+    return InstrTag::FPExceptions;
+
+  return std::nullopt;
+}
+
+void LoopCarriedOrderDepsTracker::addLoopCarriedDepenenciesForChunks(
+    const LoadStoreChunk &From, const LoadStoreChunk &To) {
+  // Add dependencies for load-to-store (WAR) from top to bottom.
+  for (const SUnitWithMemInfo &Src : From.Loads)
+    for (const SUnitWithMemInfo &Dst : To.Stores)
+      if (Src.SU->NodeNum < Dst.SU->NodeNum &&
+          hasLoopCarriedMemDep(Src, Dst, *BAA, TII, TRI))
+        LoopCarried[Src.SU->NodeNum].set(Dst.SU->NodeNum);
+
+  // TODO: The following dependencies are missed.
+  //
+  // - Dependencies for load-to-store from bottom to top.
+  // - Dependencies for store-to-load (RAW).
+  // - Dependencies for store-to-store (WAW).
+}
+
+void LoopCarriedOrderDepsTracker::computeDependenciesAux() {
+  SmallVector<LoadStoreChunk, 2> Chunks(1);
+  for (const auto &TSU : TaggedSUnits) {
+    InstrTag Tag = TSU.getTag();
+    SUnit *SU = TSU.getPointer();
+    switch (Tag) {
+    case InstrTag::Barrier:
+      Chunks.emplace_back();
+      break;
+    case InstrTag::LoadOrStore:
+      Chunks.back().append(SU);
+      break;
+    case InstrTag::FPExceptions:
+      // TODO: Handle this properly.
+      break;
+    }
+  }
+
+  // Add dependencies between memory operations. If there are one or more
+  // barrier events between two memory instructions, we don't add a
+  // loop-carried dependence for them.
+  for (const LoadStoreChunk &Chunk : Chunks)
+    addLoopCarriedDepenenciesForChunks(Chunk, Chunk);
+
+  // TODO: If there are multiple barrier instructions, dependencies from the
+  // last barrier instruction (or load/store below it) to the first barrier
+  // instruction (or load/store above it).
+}
+
 /// Add a chain edge between a load and store if the store can be an
 /// alias of the load on a subsequent iteration, i.e., a loop carried
 /// dependence. This code is very similar to the code in ScheduleDAGInstrs
 /// but that code doesn't create loop carried dependences.
-void SwingSchedulerDAG::addLoopCarriedDependences() {
-  SmallVector<SUnitWithMemInfo, 4> PendingLoads;
-  for (auto &SU : SUnits) {
-    MachineInstr &MI = *SU.getInstr();
-    if (isDependenceBarrier(MI))
-      PendingLoads.clear();
-    else if (MI.mayLoad()) {
-      PendingLoads.emplace_back(&SU);
-    } else if (MI.mayStore()) {
-      SUnitWithMemInfo Store(&SU);
-      for (const SUnitWithMemInfo &Load : PendingLoads)
-        if (hasLoopCarriedMemDep(Load, Store, BAA, TII, TRI)) {
-          SDep Dep(Load.SU, SDep::Barrier);
-          Dep.setLatency(1);
-          SU.addPred(Dep);
-        }
-    }
-  }
+/// TODO: Also compute output-dependencies.
+LoopCarriedEdges SwingSchedulerDAG::addLoopCarriedDependences() {
+  LoopCarriedEdges LCE;
+
+  // Add loop-carried order-dependencies
+  LoopCarriedOrderDepsTracker LCODTracker(this, &BAA, TII, TRI);
+  LCODTracker.computeDependencies();
+  for (unsigned I = 0; I != SUnits.size(); I++)
+    for (const int Succ : LCODTracker.getLoopCarried(I).set_bits())
+      LCE.OrderDeps[&SUnits[I]].insert(&SUnits[Succ]);
+
+  LCE.modifySUnits(SUnits);
+  return LCE;
 }
 
 /// Update the phi dependences to the DAG because ScheduleDAGInstrs no longer
@@ -4001,3 +4157,37 @@ const SwingSchedulerDDG::EdgesType &
 SwingSchedulerDDG::getOutEdges(const SUnit *SU) const {
   return getEdges(SU).Succs;
 }
+
+void LoopCarriedEdges::modifySUnits(std::vector<SUnit> &SUnits) {
+  // Currently this function simply adds all dependencies represented by this
+  // object. After we properly handle missed dependencies, the logic here will
+  // be more complex, as currently missed edges should not be added to the DAG.
+  for (SUnit &SU : SUnits) {
+    SUnit *Src = &SU;
+    if (auto *OrderDep = getOrderDepOrNull(Src)) {
+      SDep Dep(Src, SDep::Barrier);
+      Dep.setLatency(1);
+      for (SUnit *Dst : *OrderDep)
+        Dst->addPred(Dep);
+    }
+  }
+}
+
+void LoopCarriedEdges::dump(SUnit *SU, const TargetRegisterInfo *TRI,
+                            const MachineRegisterInfo *MRI) const {
+  const auto *Order = getOrderDepOrNull(SU);
+
+  if (!Order)
+    return;
+
+  const auto DumpSU = [](const SUnit *SU) {
+    std::ostringstream OSS;
+    OSS << "SU(" << SU->NodeNum << ")";
+    return OSS.str();
+  };
+
+  dbgs() << "  Loop carried edges from " << DumpSU(SU) << "\n"
+         << "    Order\n";
+  for (SUnit *Dst : *Order)
+    dbgs() << "      " << DumpSU(Dst) << "\n";
+}
diff --git a/external/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp b/external/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
index 6e9a4a769b4c..7b4f29e82fe6 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/PriorityQueue.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -547,7 +548,7 @@ ScheduleDAGInstrs *MachineSchedulerImpl::createMachineScheduler() {
     return Scheduler;
 
   // Default to GenericScheduler.
-  return createGenericSchedLive(this);
+  return createSchedLive(this);
 }
 
 bool MachineSchedulerImpl::run(MachineFunction &Func, const TargetMachine &TM,
@@ -595,7 +596,7 @@ ScheduleDAGInstrs *PostMachineSchedulerImpl::createPostMachineScheduler() {
     return Scheduler;
 
   // Default to GenericScheduler.
-  return createGenericSchedPostRA(this);
+  return createSchedPostRA(this);
 }
 
 bool PostMachineSchedulerImpl::run(MachineFunction &Func,
@@ -943,8 +944,6 @@ void ScheduleDAGMI::releaseSucc(SUnit *SU, SDep *SuccEdge) {
 
   if (SuccEdge->isWeak()) {
     --SuccSU->WeakPredsLeft;
-    if (SuccEdge->isCluster())
-      NextClusterSucc = SuccSU;
     return;
   }
 #ifndef NDEBUG
@@ -967,12 +966,6 @@ void ScheduleDAGMI::releaseSucc(SUnit *SU, SDep *SuccEdge) {
 
 /// releaseSuccessors - Call releaseSucc on each of SU's successors.
 void ScheduleDAGMI::releaseSuccessors(SUnit *SU) {
-  // Reset the next successor, For example, we want to cluster A B C.
-  // After A is picked, we will set B as next cluster succ, but if we pick
-  // D instead of B after A, then we need to reset the next cluster succ because
-  // we have decided to not pick the cluster candidate B during pickNode().
-  // Leaving B as the NextClusterSucc just make things messy.
-  NextClusterSucc = nullptr;
   for (SDep &Succ : SU->Succs)
     releaseSucc(SU, &Succ);
 }
@@ -986,8 +979,6 @@ void ScheduleDAGMI::releasePred(SUnit *SU, SDep *PredEdge) {
 
   if (PredEdge->isWeak()) {
     --PredSU->WeakSuccsLeft;
-    if (PredEdge->isCluster())
-      NextClusterPred = PredSU;
     return;
   }
 #ifndef NDEBUG
@@ -1010,7 +1001,6 @@ void ScheduleDAGMI::releasePred(SUnit *SU, SDep *PredEdge) {
 
 /// releasePredecessors - Call releasePred on each of SU's predecessors.
 void ScheduleDAGMI::releasePredecessors(SUnit *SU) {
-  NextClusterPred = nullptr;
   for (SDep &Pred : SU->Preds)
     releasePred(SU, &Pred);
 }
@@ -1184,11 +1174,8 @@ findRootsAndBiasEdges(SmallVectorImpl<SUnit*> &TopRoots,
 }
 
 /// Identify DAG roots and setup scheduler queues.
-void ScheduleDAGMI::initQueues(ArrayRef<SUnit*> TopRoots,
-                               ArrayRef<SUnit*> BotRoots) {
-  NextClusterSucc = nullptr;
-  NextClusterPred = nullptr;
-
+void ScheduleDAGMI::initQueues(ArrayRef<SUnit *> TopRoots,
+                               ArrayRef<SUnit *> BotRoots) {
   // Release all DAG roots for scheduling, not including EntrySU/ExitSU.
   //
   // Nodes with unreleased weak edges can still be roots.
@@ -2116,6 +2103,7 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
     ScheduleDAGInstrs *DAG) {
   // Keep track of the current cluster length and bytes for each SUnit.
   DenseMap<unsigned, std::pair<unsigned, unsigned>> SUnit2ClusterInfo;
+  EquivalenceClasses<SUnit *> Clusters;
 
   // At this point, `MemOpRecords` array must hold atleast two mem ops. Try to
   // cluster mem ops collected within `MemOpRecords` array.
@@ -2155,6 +2143,7 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
 
     SUnit *SUa = MemOpa.SU;
     SUnit *SUb = MemOpb.SU;
+
     if (!ReorderWhileClustering && SUa->NodeNum > SUb->NodeNum)
       std::swap(SUa, SUb);
 
@@ -2162,6 +2151,7 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
     if (!DAG->addEdge(SUb, SDep(SUa, SDep::Cluster)))
       continue;
 
+    Clusters.unionSets(SUa, SUb);
     LLVM_DEBUG(dbgs() << "Cluster ld/st SU(" << SUa->NodeNum << ") - SU("
                       << SUb->NodeNum << ")\n");
     ++NumClustered;
@@ -2201,6 +2191,21 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
                       << ", Curr cluster bytes: " << CurrentClusterBytes
                       << "\n");
   }
+
+  // Add cluster group information.
+  // Iterate over all of the equivalence sets.
+  auto &AllClusters = DAG->getClusters();
+  for (const EquivalenceClasses<SUnit *>::ECValue *I : Clusters) {
+    if (!I->isLeader())
+      continue;
+    ClusterInfo Group;
+    unsigned ClusterIdx = AllClusters.size();
+    for (SUnit *MemberI : Clusters.members(*I)) {
+      MemberI->ParentClusterIdx = ClusterIdx;
+      Group.insert(MemberI);
+    }
+    AllClusters.push_back(Group);
+  }
 }
 
 void BaseMemOpClusterMutation::collectMemOpRecords(
@@ -3688,6 +3693,9 @@ void GenericScheduler::initialize(ScheduleDAGMI *dag) {
   }
   TopCand.SU = nullptr;
   BotCand.SU = nullptr;
+
+  TopCluster = nullptr;
+  BotCluster = nullptr;
 }
 
 /// Initialize the per-region scheduling policy.
@@ -3997,13 +4005,11 @@ bool GenericScheduler::tryCandidate(SchedCandidate &Cand,
   // This is a best effort to set things up for a post-RA pass. Optimizations
   // like generating loads of multiple registers should ideally be done within
   // the scheduler pass by combining the loads during DAG postprocessing.
-  const SUnit *CandNextClusterSU =
-    Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  const SUnit *TryCandNextClusterSU =
-    TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
-                 Cand.SU == CandNextClusterSU,
-                 TryCand, Cand, Cluster))
+  const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
+  const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
+  if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
+                 CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
+                 Cluster))
     return TryCand.Reason != NoCand;
 
   if (SameBoundary) {
@@ -4262,39 +4268,33 @@ void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
 void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
   if (IsTopNode) {
     SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
+    TopCluster = DAG->getCluster(SU->ParentClusterIdx);
+    LLVM_DEBUG(if (TopCluster) {
+      dbgs() << "  Top Cluster: ";
+      for (auto *N : *TopCluster)
+        dbgs() << N->NodeNum << '\t';
+      dbgs() << '\n';
+    });
     Top.bumpNode(SU);
     if (SU->hasPhysRegUses)
       reschedulePhysReg(SU, true);
   } else {
     SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
+    BotCluster = DAG->getCluster(SU->ParentClusterIdx);
+    LLVM_DEBUG(if (BotCluster) {
+      dbgs() << "  Bot Cluster: ";
+      for (auto *N : *BotCluster)
+        dbgs() << N->NodeNum << '\t';
+      dbgs() << '\n';
+    });
     Bot.bumpNode(SU);
     if (SU->hasPhysRegDefs)
       reschedulePhysReg(SU, false);
   }
 }
 
-/// Create the standard converging machine scheduler. This will be used as the
-/// default scheduler if the target does not set a default.
-ScheduleDAGMILive *llvm::createGenericSchedLive(MachineSchedContext *C) {
-  ScheduleDAGMILive *DAG =
-      new ScheduleDAGMILive(C, std::make_unique<GenericScheduler>(C));
-  // Register DAG post-processors.
-  //
-  // FIXME: extend the mutation API to allow earlier mutations to instantiate
-  // data and pass it to later mutations. Have a single mutation that gathers
-  // the interesting nodes in one pass.
-  DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII, DAG->TRI));
-
-  const TargetSubtargetInfo &STI = C->MF->getSubtarget();
-  // Add MacroFusion mutation if fusions are not empty.
-  const auto &MacroFusions = STI.getMacroFusions();
-  if (!MacroFusions.empty())
-    DAG->addMutation(createMacroFusionDAGMutation(MacroFusions));
-  return DAG;
-}
-
 static ScheduleDAGInstrs *createConvergingSched(MachineSchedContext *C) {
-  return createGenericSchedLive(C);
+  return createSchedLive(C);
 }
 
 static MachineSchedRegistry
@@ -4323,6 +4323,8 @@ void PostGenericScheduler::initialize(ScheduleDAGMI *Dag) {
   if (!Bot.HazardRec) {
     Bot.HazardRec = DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG);
   }
+  TopCluster = nullptr;
+  BotCluster = nullptr;
 }
 
 void PostGenericScheduler::initPolicy(MachineBasicBlock::iterator Begin,
@@ -4387,14 +4389,12 @@ bool PostGenericScheduler::tryCandidate(SchedCandidate &Cand,
     return TryCand.Reason != NoCand;
 
   // Keep clustered nodes together.
-  const SUnit *CandNextClusterSU =
-      Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  const SUnit *TryCandNextClusterSU =
-      TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
-                 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
+  const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
+  const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
+  if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
+                 CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
+                 Cluster))
     return TryCand.Reason != NoCand;
-
   // Avoid critical resource consumption and balance the schedule.
   if (tryLess(TryCand.ResDelta.CritResources, Cand.ResDelta.CritResources,
               TryCand, Cand, ResourceReduce))
@@ -4591,25 +4591,15 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
 void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
   if (IsTopNode) {
     SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
+    TopCluster = DAG->getCluster(SU->ParentClusterIdx);
     Top.bumpNode(SU);
   } else {
     SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
+    BotCluster = DAG->getCluster(SU->ParentClusterIdx);
     Bot.bumpNode(SU);
   }
 }
 
-ScheduleDAGMI *llvm::createGenericSchedPostRA(MachineSchedContext *C) {
-  ScheduleDAGMI *DAG =
-      new ScheduleDAGMI(C, std::make_unique<PostGenericScheduler>(C),
-                        /*RemoveKillFlags=*/true);
-  const TargetSubtargetInfo &STI = C->MF->getSubtarget();
-  // Add MacroFusion mutation if fusions are not empty.
-  const auto &MacroFusions = STI.getMacroFusions();
-  if (!MacroFusions.empty())
-    DAG->addMutation(createMacroFusionDAGMutation(MacroFusions));
-  return DAG;
-}
-
 //===----------------------------------------------------------------------===//
 // ILP Scheduler. Currently for experimental analysis of heuristics.
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/llvm/lib/CodeGen/MachineSink.cpp b/external/llvm-project/llvm/lib/CodeGen/MachineSink.cpp
index e3f6eda8ff06..8411d5c4b09c 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MachineSink.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MachineSink.cpp
@@ -1611,8 +1611,8 @@ static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo,
   // location to prevent debug-info driven tools from potentially reporting
   // wrong location information.
   if (!SuccToSinkTo.empty() && InsertPos != SuccToSinkTo.end())
-    MI.setDebugLoc(DILocation::getMergedLocation(MI.getDebugLoc(),
-                                                 InsertPos->getDebugLoc()));
+    MI.setDebugLoc(DebugLoc::getMergedLocation(MI.getDebugLoc(),
+                                               InsertPos->getDebugLoc()));
   else
     MI.setDebugLoc(DebugLoc());
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/MacroFusion.cpp b/external/llvm-project/llvm/lib/CodeGen/MacroFusion.cpp
index 5bd6ca0978a4..975a3fe71aba 100644
--- a/external/llvm-project/llvm/lib/CodeGen/MacroFusion.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/MacroFusion.cpp
@@ -61,6 +61,10 @@ bool llvm::fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU,
   for (SDep &SI : SecondSU.Preds)
     if (SI.isCluster())
       return false;
+
+  assert(FirstSU.ParentClusterIdx == InvalidClusterId &&
+         SecondSU.ParentClusterIdx == InvalidClusterId);
+
   // Though the reachability checks above could be made more generic,
   // perhaps as part of ScheduleDAGInstrs::addEdge(), since such edges are valid,
   // the extra computation cost makes it less interesting in general cases.
@@ -70,6 +74,14 @@ bool llvm::fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU,
   if (!DAG.addEdge(&SecondSU, SDep(&FirstSU, SDep::Cluster)))
     return false;
 
+  auto &Clusters = DAG.getClusters();
+
+  FirstSU.ParentClusterIdx = Clusters.size();
+  SecondSU.ParentClusterIdx = Clusters.size();
+
+  SmallSet<SUnit *, 8> Cluster{{&FirstSU, &SecondSU}};
+  Clusters.push_back(Cluster);
+
   // TODO - If we want to chain more than two instructions, we need to create
   // artifical edges to make dependencies from the FirstSU also dependent
   // on other chained instructions, and other chained instructions also
diff --git a/external/llvm-project/llvm/lib/CodeGen/RegisterClassInfo.cpp b/external/llvm-project/llvm/lib/CodeGen/RegisterClassInfo.cpp
index 40fc35a16335..8ead83302c33 100644
--- a/external/llvm-project/llvm/lib/CodeGen/RegisterClassInfo.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/RegisterClassInfo.cpp
@@ -39,14 +39,16 @@ StressRA("stress-regalloc", cl::Hidden, cl::init(0), cl::value_desc("N"),
 
 RegisterClassInfo::RegisterClassInfo() = default;
 
-void RegisterClassInfo::runOnMachineFunction(const MachineFunction &mf) {
+void RegisterClassInfo::runOnMachineFunction(const MachineFunction &mf,
+                                             bool Rev) {
   bool Update = false;
   MF = &mf;
 
   auto &STI = MF->getSubtarget();
 
   // Allocate new array the first time we see a new target.
-  if (STI.getRegisterInfo() != TRI) {
+  if (STI.getRegisterInfo() != TRI || Reverse != Rev) {
+    Reverse = Rev;
     TRI = STI.getRegisterInfo();
     RegClass.reset(new RCInfo[TRI->getNumRegClasses()]);
     Update = true;
@@ -142,7 +144,12 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const {
 
   // FIXME: Once targets reserve registers instead of removing them from the
   // allocation order, we can simply use begin/end here.
-  ArrayRef<MCPhysReg> RawOrder = RC->getRawAllocationOrder(*MF);
+  ArrayRef<MCPhysReg> RawOrder = RC->getRawAllocationOrder(*MF, Reverse);
+  std::vector<MCPhysReg> ReverseOrder;
+  if (Reverse) {
+    llvm::append_range(ReverseOrder, reverse(RawOrder));
+    RawOrder = ArrayRef<MCPhysReg>(ReverseOrder);
+  }
   for (unsigned PhysReg : RawOrder) {
     // Remove reserved registers from the allocation order.
     if (Reserved.test(PhysReg))
diff --git a/external/llvm-project/llvm/lib/CodeGen/ScheduleDAG.cpp b/external/llvm-project/llvm/lib/CodeGen/ScheduleDAG.cpp
index 26857edd871e..e630b80e33ab 100644
--- a/external/llvm-project/llvm/lib/CodeGen/ScheduleDAG.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/ScheduleDAG.cpp
@@ -365,6 +365,9 @@ LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeName(const SUnit &SU) const {
 LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeAll(const SUnit &SU) const {
   dumpNode(SU);
   SU.dumpAttributes();
+  if (SU.ParentClusterIdx != InvalidClusterId)
+    dbgs() << "  Parent Cluster Index: " << SU.ParentClusterIdx << '\n';
+
   if (SU.Preds.size() > 0) {
     dbgs() << "  Predecessors:\n";
     for (const SDep &Dep : SU.Preds) {
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e52cf368e6e1..d21a8fbd711d 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -396,6 +396,8 @@ namespace {
     bool PromoteLoad(SDValue Op);
 
     SDValue foldShiftToAvg(SDNode *N);
+    // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
+    SDValue foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT);
 
     SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
                                 SDValue RHS, SDValue True, SDValue False,
@@ -1992,6 +1994,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: return visitMHISTOGRAM(N);
   case ISD::PARTIAL_REDUCE_SMLA:
   case ISD::PARTIAL_REDUCE_UMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
                                 return visitPARTIAL_REDUCE_MLA(N);
   case ISD::VECTOR_COMPRESS:    return visitVECTOR_COMPRESS(N);
   case ISD::LIFETIME_END:       return visitLIFETIME_END(N);
@@ -7540,6 +7543,12 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
       return DAG.getNode(ISD::AND, DL, VT, X,
                          DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y, Z), VT));
 
+  // Fold (and X, (add (not Y), Z)) -> (and X, (not (sub Y, Z)))
+  // Fold (and X, (sub (not Y), Z)) -> (and X, (not (add Y, Z)))
+  if (TLI.hasAndNot(SDValue(N, 0)))
+    if (SDValue Folded = foldBitwiseOpWithNeg(N, DL, VT))
+      return Folded;
+
   // Fold (and (srl X, C), 1) -> (srl X, BW-1) for signbit extraction
   // If we are shifting down an extended sign bit, see if we can simplify
   // this to shifting the MSB directly to expose further simplifications.
@@ -8127,6 +8136,32 @@ static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1,
   return SDValue();
 }
 
+/// Fold "masked merge" expressions like `(m & x) | (~m & y)` into the
+/// equivalent `((x ^ y) & m) ^ y)` pattern.
+/// This is typically a better representation for targets without a fused
+/// "and-not" operation.
+static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG,
+                               const TargetLowering &TLI, const SDLoc &DL) {
+  // Note that masked-merge variants using XOR or ADD expressions are
+  // normalized to OR by InstCombine so we only check for OR.
+  assert(Node->getOpcode() == ISD::OR && "Must be called with ISD::OR node");
+
+  // If the target supports and-not, don't fold this.
+  if (TLI.hasAndNot(SDValue(Node, 0)))
+    return SDValue();
+
+  SDValue M, X, Y;
+  if (sd_match(Node,
+               m_Or(m_OneUse(m_And(m_OneUse(m_Not(m_Value(M))), m_Value(Y))),
+                    m_OneUse(m_And(m_Deferred(M), m_Value(X)))))) {
+    EVT VT = M.getValueType();
+    SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, Y);
+    SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor, M);
+    return DAG.getNode(ISD::XOR, DL, VT, And, Y);
+  }
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitOR(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -8305,6 +8340,10 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
     if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
       return R;
 
+  if (VT.isScalarInteger() && VT != MVT::i1)
+    if (SDValue R = foldMaskedMerge(N, DAG, TLI, DL))
+      return R;
+
   return SDValue();
 }
 
@@ -11621,6 +11660,22 @@ SDValue DAGCombiner::foldShiftToAvg(SDNode *N) {
   return DAG.getNode(FloorISD, SDLoc(N), N->getValueType(0), {A, B});
 }
 
+SDValue DAGCombiner::foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT) {
+  unsigned Opc = N->getOpcode();
+  SDValue X, Y, Z;
+  if (sd_match(
+          N, m_BitwiseLogic(m_Value(X), m_Add(m_Not(m_Value(Y)), m_Value(Z)))))
+    return DAG.getNode(Opc, DL, VT, X,
+                       DAG.getNOT(DL, DAG.getNode(ISD::SUB, DL, VT, Y, Z), VT));
+
+  if (sd_match(N, m_BitwiseLogic(m_Value(X), m_Sub(m_OneUse(m_Not(m_Value(Y))),
+                                                   m_Value(Z)))))
+    return DAG.getNode(Opc, DL, VT, X,
+                       DAG.getNOT(DL, DAG.getNode(ISD::ADD, DL, VT, Y, Z), VT));
+
+  return SDValue();
+}
+
 /// Generate Min/Max node
 SDValue DAGCombiner::combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
                                          SDValue RHS, SDValue True,
@@ -12680,26 +12735,27 @@ SDValue DAGCombiner::foldPartialReduceMLAMulOp(SDNode *N) {
   SDValue LHSExtOp = LHS->getOperand(0);
   EVT LHSExtOpVT = LHSExtOp.getValueType();
 
-  bool ExtIsSigned = LHSOpcode == ISD::SIGN_EXTEND;
-  unsigned NewOpcode =
-      ExtIsSigned ? ISD::PARTIAL_REDUCE_SMLA : ISD::PARTIAL_REDUCE_UMLA;
-
-  // Only perform these combines if the target supports folding
-  // the extends into the operation.
-  if (!TLI.isPartialReduceMLALegalOrCustom(
-          NewOpcode, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
-          TLI.getTypeToTransformTo(*Context, LHSExtOpVT)))
-    return SDValue();
-
   // partial_reduce_*mla(acc, mul(ext(x), splat(C)), splat(1))
   // -> partial_reduce_*mla(acc, x, C)
   if (ISD::isConstantSplatVector(RHS.getNode(), C)) {
+    // TODO: Make use of partial_reduce_sumla here
     APInt CTrunc = C.trunc(LHSExtOpVT.getScalarSizeInBits());
     unsigned LHSBits = LHS.getValueType().getScalarSizeInBits();
     if ((LHSOpcode != ISD::ZERO_EXTEND || CTrunc.zext(LHSBits) != C) &&
         (LHSOpcode != ISD::SIGN_EXTEND || CTrunc.sext(LHSBits) != C))
       return SDValue();
 
+    unsigned NewOpcode = LHSOpcode == ISD::SIGN_EXTEND
+                             ? ISD::PARTIAL_REDUCE_SMLA
+                             : ISD::PARTIAL_REDUCE_UMLA;
+
+    // Only perform these combines if the target supports folding
+    // the extends into the operation.
+    if (!TLI.isPartialReduceMLALegalOrCustom(
+            NewOpcode, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
+            TLI.getTypeToTransformTo(*Context, LHSExtOpVT)))
+      return SDValue();
+
     return DAG.getNode(NewOpcode, DL, N->getValueType(0), Acc, LHSExtOp,
                        DAG.getConstant(CTrunc, DL, LHSExtOpVT));
   }
@@ -12709,26 +12765,46 @@ SDValue DAGCombiner::foldPartialReduceMLAMulOp(SDNode *N) {
     return SDValue();
 
   SDValue RHSExtOp = RHS->getOperand(0);
-  if (LHSExtOpVT != RHSExtOp.getValueType() || LHSOpcode != RHSOpcode)
+  if (LHSExtOpVT != RHSExtOp.getValueType())
+    return SDValue();
+
+  unsigned NewOpc;
+  if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::SIGN_EXTEND)
+    NewOpc = ISD::PARTIAL_REDUCE_SMLA;
+  else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
+    NewOpc = ISD::PARTIAL_REDUCE_UMLA;
+  else if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
+    NewOpc = ISD::PARTIAL_REDUCE_SUMLA;
+  else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::SIGN_EXTEND) {
+    NewOpc = ISD::PARTIAL_REDUCE_SUMLA;
+    std::swap(LHSExtOp, RHSExtOp);
+  } else
     return SDValue();
-
-  // For a 2-stage extend the signedness of both of the extends must be the
-  // same. This is so the node can be folded into only a signed or unsigned
-  // node.
-  bool NodeIsSigned = N->getOpcode() == ISD::PARTIAL_REDUCE_SMLA;
+  // For a 2-stage extend the signedness of both of the extends must match
+  // If the mul has the same type, there is no outer extend, and thus we
+  // can simply use the inner extends to pick the result node.
+  // TODO: extend to handle nonneg zext as sext
   EVT AccElemVT = Acc.getValueType().getVectorElementType();
-  if (ExtIsSigned != NodeIsSigned &&
-      Op1.getValueType().getVectorElementType() != AccElemVT)
+  if (Op1.getValueType().getVectorElementType() != AccElemVT &&
+      NewOpc != N->getOpcode())
+    return SDValue();
+
+  // Only perform these combines if the target supports folding
+  // the extends into the operation.
+  if (!TLI.isPartialReduceMLALegalOrCustom(
+          NewOpc, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
+          TLI.getTypeToTransformTo(*Context, LHSExtOpVT)))
     return SDValue();
 
-  return DAG.getNode(NewOpcode, DL, N->getValueType(0), Acc, LHSExtOp,
-                     RHSExtOp);
+  return DAG.getNode(NewOpc, DL, N->getValueType(0), Acc, LHSExtOp, RHSExtOp);
 }
 
 // partial.reduce.umla(acc, zext(op), splat(1))
 // -> partial.reduce.umla(acc, op, splat(trunc(1)))
 // partial.reduce.smla(acc, sext(op), splat(1))
 // -> partial.reduce.smla(acc, op, splat(trunc(1)))
+// partial.reduce.sumla(acc, sext(op), splat(1))
+// -> partial.reduce.smla(acc, op, splat(trunc(1)))
 SDValue DAGCombiner::foldPartialReduceAdd(SDNode *N) {
   SDLoc DL(N);
   SDValue Acc = N->getOperand(0);
@@ -12745,7 +12821,7 @@ SDValue DAGCombiner::foldPartialReduceAdd(SDNode *N) {
     return SDValue();
 
   bool Op1IsSigned = Op1Opcode == ISD::SIGN_EXTEND;
-  bool NodeIsSigned = N->getOpcode() == ISD::PARTIAL_REDUCE_SMLA;
+  bool NodeIsSigned = N->getOpcode() != ISD::PARTIAL_REDUCE_UMLA;
   EVT AccElemVT = Acc.getValueType().getVectorElementType();
   if (Op1IsSigned != NodeIsSigned &&
       Op1.getValueType().getVectorElementType() != AccElemVT)
@@ -14933,6 +15009,25 @@ SDValue DAGCombiner::visitAssertExt(SDNode *N) {
     }
   }
 
+  // If we have (AssertZext (and (AssertSext X, iX), M), iY) and Y is smaller
+  // than X, and the And doesn't change the lower iX bits, we can move the
+  // AssertZext in front of the And and drop the AssertSext.
+  if (Opcode == ISD::AssertZext && N0.getOpcode() == ISD::AND &&
+      N0.hasOneUse() && N0.getOperand(0).getOpcode() == ISD::AssertSext &&
+      isa<ConstantSDNode>(N0.getOperand(1))) {
+    SDValue BigA = N0.getOperand(0);
+    EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
+    const APInt &Mask = N0.getConstantOperandAPInt(1);
+    if (AssertVT.bitsLT(BigA_AssertVT) &&
+        Mask.countr_one() >= BigA_AssertVT.getScalarSizeInBits()) {
+      SDLoc DL(N);
+      SDValue NewAssert =
+          DAG.getNode(Opcode, DL, N->getValueType(0), BigA.getOperand(0), N1);
+      return DAG.getNode(ISD::AND, DL, N->getValueType(0), NewAssert,
+                         N0.getOperand(1));
+    }
+  }
+
   return SDValue();
 }
 
@@ -16394,12 +16489,11 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
     return SDValue();
 
   bool AllowMultipleMaybePoisonOperands =
-      N0.getOpcode() == ISD::SELECT_CC ||
-      N0.getOpcode() == ISD::SETCC ||
+      N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC ||
       N0.getOpcode() == ISD::BUILD_VECTOR ||
       N0.getOpcode() == ISD::BUILD_PAIR ||
       N0.getOpcode() == ISD::VECTOR_SHUFFLE ||
-      N0.getOpcode() == ISD::CONCAT_VECTORS;
+      N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL;
 
   // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all
   // ones" or "constant" into something that depends on FrozenUndef. We can
@@ -16497,7 +16591,17 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
                              SVN->getMask());
   } else {
     // NOTE: this strips poison generating flags.
-    R = DAG.getNode(N0.getOpcode(), SDLoc(N0), N0->getVTList(), Ops);
+    // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
+    // ninf, nsz, or fast.
+    // However, contract, reassoc, afn, and arcp should be preserved,
+    // as these fast-math flags do not introduce poison values.
+    SDNodeFlags SrcFlags = N0->getFlags();
+    SDNodeFlags SafeFlags;
+    SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
+    SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
+    SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
+    SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
+    R = DAG.getNode(N0.getOpcode(), SDLoc(N0), N0->getVTList(), Ops, SafeFlags);
   }
   assert(DAG.isGuaranteedNotToBeUndefOrPoison(R, /*PoisonOnly*/ false) &&
          "Can't create node that may be undef/poison!");
@@ -29121,9 +29225,8 @@ SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
       // The estimate is now completely wrong if the input was exactly 0.0 or
       // possibly a denormal. Force the answer to 0.0 or value provided by
       // target for those cases.
-      Est = DAG.getNode(
-          Test.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT,
-          Test, TLI.getSqrtResultForDenormInput(Op, DAG), Est);
+      Est = DAG.getSelect(DL, VT, Test,
+                          TLI.getSqrtResultForDenormInput(Op, DAG), Est);
     }
     return Est;
   }
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index a0814538e317..f9e9524ba484 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1674,9 +1674,6 @@ void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
                               const DebugLoc &DbgLoc) {
   const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
   bool BlockHasMultipleInstrs = &BB->front() != &BB->back();
-  // Handle legacy case of debug intrinsics
-  if (BlockHasMultipleInstrs && !BB->getModule()->IsNewDbgInfoFormat)
-    BlockHasMultipleInstrs = BB->sizeWithoutDebug() > 1;
   if (BlockHasMultipleInstrs && FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
     // For more accurate line information if this is the only non-debug
     // instruction in the block then emit it, otherwise we have the
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 528c07cc5549..f5f4d71236fe 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3558,6 +3558,57 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     Results.push_back(TLI.expandVectorSplice(Node, DAG));
     break;
   }
+  case ISD::VECTOR_DEINTERLEAVE: {
+    unsigned Factor = Node->getNumOperands();
+    if (Factor <= 2 || !isPowerOf2_32(Factor))
+      break;
+    SmallVector<SDValue, 8> Ops(Node->ops());
+    EVT VecVT = Node->getValueType(0);
+    SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
+    // Deinterleave at Factor/2 so each result contains two factors interleaved:
+    // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
+    SDValue L = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,
+                            ArrayRef(Ops).take_front(Factor / 2));
+    SDValue R = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,
+                            ArrayRef(Ops).take_back(Factor / 2));
+    Results.resize(Factor);
+    // Deinterleave the 2 factors out:
+    // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
+    for (unsigned I = 0; I < Factor / 2; I++) {
+      SDValue Deinterleave =
+          DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, {VecVT, VecVT},
+                      {L.getValue(I), R.getValue(I)});
+      Results[I] = Deinterleave.getValue(0);
+      Results[I + Factor / 2] = Deinterleave.getValue(1);
+    }
+    break;
+  }
+  case ISD::VECTOR_INTERLEAVE: {
+    unsigned Factor = Node->getNumOperands();
+    if (Factor <= 2 || !isPowerOf2_32(Factor))
+      break;
+    EVT VecVT = Node->getValueType(0);
+    SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
+    SmallVector<SDValue, 8> LOps, ROps;
+    // Interleave so we have 2 factors per result:
+    // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
+    for (unsigned I = 0; I < Factor / 2; I++) {
+      SDValue Interleave =
+          DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, {VecVT, VecVT},
+                      {Node->getOperand(I), Node->getOperand(I + Factor / 2)});
+      LOps.push_back(Interleave.getValue(0));
+      ROps.push_back(Interleave.getValue(1));
+    }
+    // Interleave at Factor/2:
+    // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
+    SDValue L = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, LOps);
+    SDValue R = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, ROps);
+    for (unsigned I = 0; I < Factor / 2; I++)
+      Results.push_back(L.getValue(I));
+    for (unsigned I = 0; I < Factor / 2; I++)
+      Results.push_back(R.getValue(I));
+    break;
+  }
   case ISD::EXTRACT_ELEMENT: {
     EVT OpTy = Node->getOperand(0).getValueType();
     if (Node->getConstantOperandVal(1)) {
@@ -4548,14 +4599,14 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
                     RTLIB::FMAX_PPCF128, Results);
     break;
   case ISD::FMINIMUMNUM:
-    ExpandFPLibCall(Node, RTLIB::FMINIMUMNUM_F32, RTLIB::FMINIMUMNUM_F64,
-                    RTLIB::FMINIMUMNUM_F80, RTLIB::FMINIMUMNUM_F128,
-                    RTLIB::FMINIMUMNUM_PPCF128, Results);
+    ExpandFPLibCall(Node, RTLIB::FMINIMUM_NUM_F32, RTLIB::FMINIMUM_NUM_F64,
+                    RTLIB::FMINIMUM_NUM_F80, RTLIB::FMINIMUM_NUM_F128,
+                    RTLIB::FMINIMUM_NUM_PPCF128, Results);
     break;
   case ISD::FMAXIMUMNUM:
-    ExpandFPLibCall(Node, RTLIB::FMAXIMUMNUM_F32, RTLIB::FMAXIMUMNUM_F64,
-                    RTLIB::FMAXIMUMNUM_F80, RTLIB::FMAXIMUMNUM_F128,
-                    RTLIB::FMAXIMUMNUM_PPCF128, Results);
+    ExpandFPLibCall(Node, RTLIB::FMAXIMUM_NUM_F32, RTLIB::FMAXIMUM_NUM_F64,
+                    RTLIB::FMAXIMUM_NUM_F80, RTLIB::FMAXIMUM_NUM_F128,
+                    RTLIB::FMAXIMUM_NUM_PPCF128, Results);
     break;
   case ISD::FSQRT:
   case ISD::STRICT_FSQRT:
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 41e85521b41e..7dfb43164be3 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -335,16 +335,16 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
   return SoftenFloatRes_Binary(
-      N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUMNUM_F32,
-                      RTLIB::FMINIMUMNUM_F64, RTLIB::FMINIMUMNUM_F80,
-                      RTLIB::FMINIMUMNUM_F128, RTLIB::FMINIMUMNUM_PPCF128));
+      N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_NUM_F32,
+                      RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
+                      RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128));
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
   return SoftenFloatRes_Binary(
-      N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUMNUM_F32,
-                      RTLIB::FMAXIMUMNUM_F64, RTLIB::FMAXIMUMNUM_F80,
-                      RTLIB::FMAXIMUMNUM_F128, RTLIB::FMAXIMUMNUM_PPCF128));
+      N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_NUM_F32,
+                      RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
+                      RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128));
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
@@ -1143,6 +1143,9 @@ bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
     Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
     break;
   case ISD::FCOPYSIGN:   Res = SoftenFloatOp_FCOPYSIGN(N); break;
+  case ISD::FAKE_USE:
+    Res = SoftenFloatOp_FAKE_USE(N);
+    break;
   }
 
   // If the result is null, the sub-method took care of registering results etc.
@@ -1477,6 +1480,12 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
                                              RTLIB::LLRINT_PPCF128));
 }
 
+SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
+  SDValue Op1 = BitConvertToInteger(N->getOperand(1));
+  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
+                     N->getOperand(0), Op1);
+}
+
 //===----------------------------------------------------------------------===//
 //  Float Result Expansion
 //===----------------------------------------------------------------------===//
@@ -1712,9 +1721,9 @@ void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
                                                   SDValue &Hi) {
   ExpandFloatRes_Binary(
       N,
-      GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUMNUM_F32,
-                   RTLIB::FMINIMUMNUM_F64, RTLIB::FMINIMUMNUM_F80,
-                   RTLIB::FMINIMUMNUM_F128, RTLIB::FMINIMUMNUM_PPCF128),
+      GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_NUM_F32,
+                   RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
+                   RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128),
       Lo, Hi);
 }
 
@@ -1722,9 +1731,9 @@ void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
                                                   SDValue &Hi) {
   ExpandFloatRes_Binary(
       N,
-      GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUMNUM_F32,
-                   RTLIB::FMAXIMUMNUM_F64, RTLIB::FMAXIMUMNUM_F80,
-                   RTLIB::FMAXIMUMNUM_F128, RTLIB::FMAXIMUMNUM_PPCF128),
+      GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_NUM_F32,
+                   RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
+                   RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128),
       Lo, Hi);
 }
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 2bcca91f6f81..dd6467622205 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -166,6 +166,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
 
   case ISD::PARTIAL_REDUCE_UMLA:
   case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     Res = PromoteIntRes_PARTIAL_REDUCE_MLA(N);
     break;
 
@@ -2093,6 +2094,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
     break;
   case ISD::PARTIAL_REDUCE_UMLA:
   case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     Res = PromoteIntOp_PARTIAL_REDUCE_MLA(N);
     break;
   }
@@ -2886,12 +2888,21 @@ SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) {
 
 SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N) {
   SmallVector<SDValue, 1> NewOps(N->ops());
-  if (N->getOpcode() == ISD::PARTIAL_REDUCE_SMLA) {
+  switch (N->getOpcode()) {
+  case ISD::PARTIAL_REDUCE_SMLA:
     NewOps[1] = SExtPromotedInteger(N->getOperand(1));
     NewOps[2] = SExtPromotedInteger(N->getOperand(2));
-  } else {
+    break;
+  case ISD::PARTIAL_REDUCE_UMLA:
     NewOps[1] = ZExtPromotedInteger(N->getOperand(1));
     NewOps[2] = ZExtPromotedInteger(N->getOperand(2));
+    break;
+  case ISD::PARTIAL_REDUCE_SUMLA:
+    NewOps[1] = SExtPromotedInteger(N->getOperand(1));
+    NewOps[2] = ZExtPromotedInteger(N->getOperand(2));
+    break;
+  default:
+    llvm_unreachable("unexpected opcode");
   }
   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
 }
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index dd9af47da528..a541833684f3 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -651,6 +651,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);
   SDValue SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
   SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N);
+  SDValue SoftenFloatOp_FAKE_USE(SDNode *N);
 
   //===--------------------------------------------------------------------===//
   // Float Expansion Support: LegalizeFloatTypes.cpp
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 910a40e5b514..4a1cd642233e 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -530,6 +530,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   }
   case ISD::PARTIAL_REDUCE_UMLA:
   case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     Action =
         TLI.getPartialReduceMLAAction(Op.getOpcode(), Node->getValueType(0),
                                       Node->getOperand(1).getValueType());
@@ -1211,6 +1212,7 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
     return;
   case ISD::PARTIAL_REDUCE_UMLA:
   case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     Results.push_back(TLI.expandPartialReduceMLA(Node, DAG));
     return;
   case ISD::VECREDUCE_SEQ_FADD:
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 4d844f0036a7..c56cfec81acd 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1387,6 +1387,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
     break;
   case ISD::PARTIAL_REDUCE_UMLA:
   case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
     break;
   case ISD::GET_ACTIVE_LANE_MASK:
@@ -3473,6 +3474,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
     break;
   case ISD::PARTIAL_REDUCE_UMLA:
   case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
     break;
   }
@@ -6147,7 +6149,12 @@ SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
 
   if (ExtType == ISD::NON_EXTLOAD &&
       TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WidenVT) &&
-      TLI.isTypeLegal(WideMaskVT)) {
+      TLI.isTypeLegal(WideMaskVT) &&
+      // If there is a passthru, we shouldn't use vp.load. However,
+      // type legalizer will struggle on masked.load with
+      // scalable vectors, so for scalable vectors, we still use vp.load
+      // but manually merge the load result with the passthru using vp.select.
+      (N->getPassThru()->isUndef() || VT.isScalableVector())) {
     Mask = DAG.getInsertSubvector(dl, DAG.getUNDEF(WideMaskVT), Mask, 0);
     SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
                                       VT.getVectorElementCount());
@@ -6155,12 +6162,20 @@ SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
         DAG.getLoadVP(N->getAddressingMode(), ISD::NON_EXTLOAD, WidenVT, dl,
                       N->getChain(), N->getBasePtr(), N->getOffset(), Mask, EVL,
                       N->getMemoryVT(), N->getMemOperand());
+    SDValue NewVal = NewLoad;
+
+    // Manually merge with vp.select
+    if (!N->getPassThru()->isUndef()) {
+      assert(WidenVT.isScalableVector());
+      NewVal =
+          DAG.getNode(ISD::VP_SELECT, dl, WidenVT, Mask, NewVal, PassThru, EVL);
+    }
 
     // Modified the chain - switch anything that used the old chain to use
     // the new one.
     ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1));
 
-    return NewLoad;
+    return NewVal;
   }
 
   // The mask should be widened as well
@@ -6634,8 +6649,12 @@ SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
     InOp1 = GetWidenedVector(InOp1);
     InOp2 = GetWidenedVector(InOp2);
   } else {
-    InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
-    InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
+    SDValue Poison = DAG.getPOISON(WidenInVT);
+    SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(N));
+    InOp1 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WidenInVT, Poison,
+                        InOp1, ZeroIdx);
+    InOp2 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WidenInVT, Poison,
+                        InOp2, ZeroIdx);
   }
 
   // Assume that the input and output will be widen appropriately.  If not,
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 02523df3b902..fca843dbc134 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5553,6 +5553,7 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::BUILD_VECTOR:
   case ISD::BUILD_PAIR:
   case ISD::SPLAT_VECTOR:
+  case ISD::VSELECT:
     return false;
 
   case ISD::SELECT_CC:
@@ -5579,6 +5580,12 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::ADD:
   case ISD::SUB:
   case ISD::MUL:
+  case ISD::FNEG:
+  case ISD::FADD:
+  case ISD::FSUB:
+  case ISD::FMUL:
+  case ISD::FDIV:
+  case ISD::FREM:
     // No poison except from flags (which is handled above)
     return false;
 
@@ -7890,7 +7897,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     break;
   }
   case ISD::INSERT_VECTOR_ELT: {
-    ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
+    assert(VT.isVector() && VT == N1.getValueType() &&
+           "INSERT_VECTOR_ELT vector type mismatch");
+    assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
+           "INSERT_VECTOR_ELT scalar fp/int mismatch");
+    assert((!VT.isFloatingPoint() ||
+            VT.getVectorElementType() == N2.getValueType()) &&
+           "INSERT_VECTOR_ELT fp scalar type mismatch");
+    assert((!VT.isInteger() ||
+            VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
+           "INSERT_VECTOR_ELT int scalar size mismatch");
+
+    auto *N3C = dyn_cast<ConstantSDNode>(N3);
     // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
     // for scalable vectors where we will generate appropriate code to
     // deal with out-of-bounds cases correctly.
@@ -7975,7 +7993,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     break;
   }
   case ISD::PARTIAL_REDUCE_UMLA:
-  case ISD::PARTIAL_REDUCE_SMLA: {
+  case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA: {
     [[maybe_unused]] EVT AccVT = N1.getValueType();
     [[maybe_unused]] EVT Input1VT = N2.getValueType();
     [[maybe_unused]] EVT Input2VT = N3.getValueType();
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index fb3c09b2ab25..28a30bd4361b 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3037,8 +3037,9 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
 
   // First create the loads to the guard/stack slot for the comparison.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
-  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
+  auto &DL = DAG.getDataLayout();
+  EVT PtrTy = TLI.getFrameIndexTy(DL);
+  EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
 
   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
   int FI = MFI.getStackProtectorIndex();
@@ -3047,8 +3048,8 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
   SDLoc dl = getCurSDLoc();
   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
   const Module &M = *ParentBB->getParent()->getFunction().getParent();
-  Align Align =
-      DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
+  Align Align = DL.getPrefTypeAlign(
+      PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
 
   // Generate code to load the content of the guard slot.
   SDValue GuardVal = DAG.getLoad(
@@ -3059,8 +3060,14 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
   if (TLI.useStackGuardXorFP())
     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
 
-  // Retrieve guard check function, nullptr if instrumentation is inlined.
-  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
+  // If we're using function-based instrumentation, call the guard check
+  // function
+  if (SPD.shouldEmitFunctionBasedCheckStackProtector()) {
+    // Get the guard check function from the target and verify it exists since
+    // we're using function-based instrumentation
+    const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M);
+    assert(GuardCheckFn && "Guard check function is null");
+
     // The target provides a guard check function to validate the guard value.
     // Generate a call to that function with the content of the guard slot as
     // argument.
@@ -3101,10 +3108,9 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
   }
 
   // Perform the comparison via a getsetcc.
-  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
-                                                        *DAG.getContext(),
-                                                        Guard.getValueType()),
-                             Guard, GuardVal, ISD::SETNE);
+  SDValue Cmp = DAG.getSetCC(
+      dl, TLI.getSetCCResultType(DL, *DAG.getContext(), Guard.getValueType()),
+      Guard, GuardVal, ISD::SETNE);
 
   // If the guard/stackslot do not equal, branch to failure MBB.
   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
@@ -3126,14 +3132,69 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
 /// For a high level explanation of how this fits into the stack protector
 /// generation see the comment on the declaration of class
 /// StackProtectorDescriptor.
-void
-SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
+void SelectionDAGBuilder::visitSPDescriptorFailure(
+    StackProtectorDescriptor &SPD) {
+
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setDiscardResult(true);
-  SDValue Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
-                                  MVT::isVoid, {}, CallOptions, getCurSDLoc())
-                      .second;
+  MachineBasicBlock *ParentBB = SPD.getParentMBB();
+  const Module &M = *ParentBB->getParent()->getFunction().getParent();
+  SDValue Chain;
+
+  // For -Oz builds with a guard check function, we use function-based
+  // instrumentation. Otherwise, if we have a guard check function, we call it
+  // in the failure block.
+  auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M);
+  if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
+    // First create the loads to the guard/stack slot for the comparison.
+    auto &DL = DAG.getDataLayout();
+    EVT PtrTy = TLI.getFrameIndexTy(DL);
+    EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
+
+    MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
+    int FI = MFI.getStackProtectorIndex();
+
+    SDLoc dl = getCurSDLoc();
+    SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
+    Align Align = DL.getPrefTypeAlign(
+        PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
+
+    // Generate code to load the content of the guard slot.
+    SDValue GuardVal = DAG.getLoad(
+        PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
+        MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
+        MachineMemOperand::MOVolatile);
+
+    if (TLI.useStackGuardXorFP())
+      GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
+
+    // The target provides a guard check function to validate the guard value.
+    // Generate a call to that function with the content of the guard slot as
+    // argument.
+    FunctionType *FnTy = GuardCheckFn->getFunctionType();
+    assert(FnTy->getNumParams() == 1 && "Invalid function signature");
+
+    TargetLowering::ArgListTy Args;
+    TargetLowering::ArgListEntry Entry;
+    Entry.Node = GuardVal;
+    Entry.Ty = FnTy->getParamType(0);
+    if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
+      Entry.IsInReg = true;
+    Args.push_back(Entry);
+
+    TargetLowering::CallLoweringInfo CLI(DAG);
+    CLI.setDebugLoc(getCurSDLoc())
+        .setChain(DAG.getEntryNode())
+        .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
+                   getValue(GuardCheckFn), std::move(Args));
+
+    Chain = TLI.LowerCallTo(CLI).second;
+  } else {
+    TargetLowering::MakeLibCallOptions CallOptions;
+    CallOptions.setDiscardResult(true);
+    Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
+                            {}, CallOptions, getCurSDLoc())
+                .second;
+  }
 
   // Emit a trap instruction if we are required to do so.
   const TargetOptions &TargetOpts = DAG.getTarget().Options;
@@ -3275,12 +3336,13 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
 
   // Deopt and ptrauth bundles are lowered in helper functions, and we don't
   // have to do anything here to lower funclet bundles.
-  assert(!I.hasOperandBundlesOtherThan(
-             {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
-              LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
-              LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
-              LLVMContext::OB_clang_arc_attachedcall}) &&
-         "Cannot lower invokes with arbitrary operand bundles yet!");
+  if (I.hasOperandBundlesOtherThan(
+          {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
+           LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
+           LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
+           LLVMContext::OB_clang_arc_attachedcall}))
+    reportFatalUsageError(
+        "cannot lower invokes with arbitrary operand bundles!");
 
   const Value *Callee(I.getCalledOperand());
   const Function *Fn = dyn_cast<Function>(Callee);
@@ -3380,9 +3442,10 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
 
   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
   // have to do anything here to lower funclet bundles.
-  assert(!I.hasOperandBundlesOtherThan(
-             {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
-         "Cannot lower callbrs with arbitrary operand bundles yet!");
+  if (I.hasOperandBundlesOtherThan(
+          {LLVMContext::OB_deopt, LLVMContext::OB_funclet}))
+    reportFatalUsageError(
+        "cannot lower callbrs with arbitrary operand bundles!");
 
   assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
   visitInlineAsm(I);
@@ -3399,7 +3462,11 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
     BasicBlock *Dest = I.getIndirectDest(i);
     MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
     Target->setIsInlineAsmBrIndirectTarget();
-    Target->setMachineBlockAddressTaken();
+    // If we introduce a type of asm goto statement that is permitted to use an
+    // indirect call instruction to jump to its labels, then we should add a
+    // call to Target->setMachineBlockAddressTaken() here, to mark the target
+    // block as requiring a BTI.
+
     Target->setLabelMustBeEmitted();
     // Don't add duplicate machine successors.
     if (Dests.insert(Dest).second)
@@ -7383,6 +7450,11 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     setValue(&I, getValue(I.getOperand(0)));
     return;
 
+  case Intrinsic::type_test:
+  case Intrinsic::public_type_test:
+    setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
+    return;
+
   case Intrinsic::assume:
   case Intrinsic::experimental_noalias_scope_decl:
   case Intrinsic::var_annotation:
@@ -9550,12 +9622,12 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
   // have to do anything here to lower funclet bundles.
   // CFGuardTarget bundles are lowered in LowerCallTo.
-  assert(!I.hasOperandBundlesOtherThan(
-             {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
-              LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
-              LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
-              LLVMContext::OB_convergencectrl}) &&
-         "Cannot lower calls with arbitrary operand bundles!");
+  if (I.hasOperandBundlesOtherThan(
+          {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
+           LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
+           LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
+           LLVMContext::OB_convergencectrl}))
+    reportFatalUsageError("cannot lower calls with arbitrary operand bundles!");
 
   SDValue Callee = getValue(I.getCalledOperand());
 
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 539f583ea361..7fc15581c17e 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -585,6 +585,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
     return "partial_reduce_umla";
   case ISD::PARTIAL_REDUCE_SMLA:
     return "partial_reduce_smla";
+  case ISD::PARTIAL_REDUCE_SUMLA:
+    return "partial_reduce_sumla";
 
     // Vector Predication
 #define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...)                    \
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 56ee4e98406b..b02a03c0b0cb 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -386,10 +386,7 @@ SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, CodeGenOptLevel OL)
   initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
 }
 
-SelectionDAGISel::~SelectionDAGISel() {
-  delete CurDAG;
-  delete SwiftError;
-}
+SelectionDAGISel::~SelectionDAGISel() { delete CurDAG; }
 
 void SelectionDAGISelLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   CodeGenOptLevel OptLevel = Selector->OptLevel;
@@ -1880,7 +1877,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
 
     if (SP->shouldEmitSDCheck(*LLVMBB)) {
       bool FunctionBasedInstrumentation =
-          TLI->getSSPStackGuardCheck(*Fn.getParent());
+          TLI->getSSPStackGuardCheck(*Fn.getParent()) && Fn.hasMinSize();
       SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->getMBB(LLVMBB),
                                    FunctionBasedInstrumentation);
     }
@@ -1953,8 +1950,7 @@ SelectionDAGISel::FinishBasicBlock() {
 
     // Add load and check to the basicblock.
     FuncInfo->MBB = ParentMBB;
-    FuncInfo->InsertPt =
-        findSplitPointForStackProtector(ParentMBB, *TII);
+    FuncInfo->InsertPt = findSplitPointForStackProtector(ParentMBB, *TII);
     SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
     CurDAG->setRoot(SDB->getRoot());
     SDB->clear();
@@ -1976,8 +1972,7 @@ SelectionDAGISel::FinishBasicBlock() {
         findSplitPointForStackProtector(ParentMBB, *TII);
 
     // Splice the terminator of ParentMBB into SuccessMBB.
-    SuccessMBB->splice(SuccessMBB->end(), ParentMBB,
-                       SplitPoint,
+    SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
                        ParentMBB->end());
 
     // Add compare/jump on neq/jump to the parent BB.
diff --git a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 22d0bc991458..a0ffb4b6d5a4 100644
--- a/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -429,8 +429,16 @@ void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
     // Update Chain.
     Chain = Call.second;
   } else {
+    assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
+           "unordered call should be simple boolean");
+
     EVT SetCCVT =
         getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
+    if (getBooleanContents(RetVT) == ZeroOrOneBooleanContent) {
+      NewLHS = DAG.getNode(ISD::AssertZext, dl, RetVT, Call.first,
+                           DAG.getValueType(MVT::i1));
+    }
+
     SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
     auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
     CCCode = getCmpLibcallCC(LC2);
@@ -8690,11 +8698,8 @@ SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
 
   SDValue MinMax =
       DAG.getSelectCC(DL, LHS, RHS, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT);
-  // If MinMax is NaN, let's quiet it.
-  if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(LHS) &&
-      !DAG.isKnownNeverNaN(RHS)) {
-    MinMax = DAG.getNode(ISD::FCANONICALIZE, DL, VT, MinMax, Flags);
-  }
+
+  // TODO: We need quiet sNaN if strictfp.
 
   // Fixup signed zero behavior.
   if (Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros() ||
@@ -11894,13 +11899,17 @@ SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
   EVT ExtMulOpVT =
       EVT::getVectorVT(*DAG.getContext(), AccVT.getVectorElementType(),
                        MulOpVT.getVectorElementCount());
-  unsigned ExtOpc = N->getOpcode() == ISD::PARTIAL_REDUCE_SMLA
-                        ? ISD::SIGN_EXTEND
-                        : ISD::ZERO_EXTEND;
+
+  unsigned ExtOpcLHS = N->getOpcode() == ISD::PARTIAL_REDUCE_UMLA
+                      ? ISD::ZERO_EXTEND
+                      : ISD::SIGN_EXTEND;
+  unsigned ExtOpcRHS = N->getOpcode() == ISD::PARTIAL_REDUCE_SMLA
+                      ? ISD::SIGN_EXTEND
+                      : ISD::ZERO_EXTEND;
 
   if (ExtMulOpVT != MulOpVT) {
-    MulLHS = DAG.getNode(ExtOpc, DL, ExtMulOpVT, MulLHS);
-    MulRHS = DAG.getNode(ExtOpc, DL, ExtMulOpVT, MulRHS);
+    MulLHS = DAG.getNode(ExtOpcLHS, DL, ExtMulOpVT, MulLHS);
+    MulRHS = DAG.getNode(ExtOpcRHS, DL, ExtMulOpVT, MulRHS);
   }
   SDValue Input = MulLHS;
   APInt ConstantOne;
diff --git a/external/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/external/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index b8c632d11f2e..c804a179d886 100644
--- a/external/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -1219,7 +1219,7 @@ const MCExpr *TargetLoweringObjectFileELF::lowerSymbolDifference(
     if (Addend)
       Res = MCBinaryExpr::createAdd(Res, MCConstantExpr::create(Addend, Ctx),
                                     Ctx);
-    return createTargetMCExpr(Res, PLTPCRelativeSpecifier);
+    return MCSpecifierExpr::create(Res, PLTPCRelativeSpecifier, getContext());
   }
 
   if (!PLTRelativeSpecifier)
diff --git a/external/llvm-project/llvm/lib/CodeGen/TargetOptionsImpl.cpp b/external/llvm-project/llvm/lib/CodeGen/TargetOptionsImpl.cpp
index 01ffaed585ea..def325efd544 100644
--- a/external/llvm-project/llvm/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/external/llvm-project/llvm/lib/CodeGen/TargetOptionsImpl.cpp
@@ -24,9 +24,10 @@ using namespace llvm;
 bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
   const Function &F = MF.getFunction();
 
-  if (!F.hasFnAttribute("frame-pointer"))
+  Attribute FPAttr = F.getFnAttribute("frame-pointer");
+  if (!FPAttr.isValid())
     return false;
-  StringRef FP = F.getFnAttribute("frame-pointer").getValueAsString();
+  StringRef FP = FPAttr.getValueAsString();
   if (FP == "all")
     return true;
   if (FP == "non-leaf")
diff --git a/external/llvm-project/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp b/external/llvm-project/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
index 8485d3c9128e..4e4d86e5cb8d 100644
--- a/external/llvm-project/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
+++ b/external/llvm-project/llvm/lib/DWARFLinker/Classic/DWARFStreamer.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
 #include "llvm/MC/TargetRegistry.h"
diff --git a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
index 38373b2c5491..d291be6c6439 100644
--- a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
+++ b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
@@ -15,7 +15,6 @@
 #include "llvm/DWARFLinker/Utils.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugMacro.h"
-#include "llvm/Support/DJB.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
diff --git a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp
index 23e618adeaeb..e9ee3ce1b8b2 100644
--- a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp
+++ b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "DWARFLinkerImpl.h"
-#include "DIEGenerator.h"
 #include "DependencyTracker.h"
 #include "llvm/DWARFLinker/Utils.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h"
diff --git a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerTypeUnit.cpp b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerTypeUnit.cpp
index 9b626239e075..07bbde70c77c 100644
--- a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerTypeUnit.cpp
+++ b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/DWARFLinkerTypeUnit.cpp
@@ -8,7 +8,6 @@
 
 #include "DWARFLinkerTypeUnit.h"
 #include "DIEGenerator.h"
-#include "DWARFEmitterImpl.h"
 #include "llvm/Support/LEB128.h"
 
 using namespace llvm;
diff --git a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.cpp b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.cpp
index d03f9b40d490..9bcd2b895be8 100644
--- a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.cpp
+++ b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.cpp
@@ -9,7 +9,6 @@
 #include "OutputSections.h"
 #include "DWARFLinkerCompileUnit.h"
 #include "DWARFLinkerTypeUnit.h"
-#include "llvm/ADT/StringSwitch.h"
 
 using namespace llvm;
 using namespace dwarf_linker;
diff --git a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.h b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.h
index da47f53b6c3d..5043e918013e 100644
--- a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.h
+++ b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/OutputSections.h
@@ -181,6 +181,11 @@ struct SectionDescriptor : SectionDescriptorBase {
   /// to the debug section, corresponding to this object.
   uint64_t StartOffset = 0;
 
+protected:
+  /// Section data bits.
+  OutSectionDataTy Contents;
+
+public:
   /// Stream which stores data to the Contents.
   raw_svector_ostream OS;
 
@@ -287,9 +292,6 @@ struct SectionDescriptor : SectionDescriptorBase {
 
   LinkingGlobalData &GlobalData;
 
-  /// Section data bits.
-  OutSectionDataTy Contents;
-
   /// Some sections are generated using AsmPrinter. The real section data
   /// located inside elf file in that case. Following fields points to the
   /// real section content inside elf file.
diff --git a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/SyntheticTypeNameBuilder.cpp b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/SyntheticTypeNameBuilder.cpp
index 1554946c2c04..34174f98b7e3 100644
--- a/external/llvm-project/llvm/lib/DWARFLinker/Parallel/SyntheticTypeNameBuilder.cpp
+++ b/external/llvm-project/llvm/lib/DWARFLinker/Parallel/SyntheticTypeNameBuilder.cpp
@@ -10,7 +10,6 @@
 #include "DWARFLinkerCompileUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h"
-#include "llvm/Support/ScopedPrinter.h"
 
 using namespace llvm;
 using namespace dwarf_linker;
diff --git a/external/llvm-project/llvm/lib/DWP/DWP.cpp b/external/llvm-project/llvm/lib/DWP/DWP.cpp
index dc364017bf82..b565edbfe96d 100644
--- a/external/llvm-project/llvm/lib/DWP/DWP.cpp
+++ b/external/llvm-project/llvm/lib/DWP/DWP.cpp
@@ -18,8 +18,6 @@
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
 #include "llvm/Object/Decompressor.h"
 #include "llvm/Object/ELFObjectFile.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MemoryBuffer.h"
 #include <limits>
 
 using namespace llvm;
diff --git a/external/llvm-project/llvm/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp b/external/llvm-project/llvm/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp
index 0adbdb596218..3761263b283b 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp
@@ -245,7 +245,8 @@ std::vector<CVType> ContinuationRecordBuilder::end(TypeIndex Index) {
 #define TYPE_RECORD(EnumName, EnumVal, Name)
 #define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
-  template void llvm::codeview::ContinuationRecordBuilder::writeMemberType(    \
+  template LLVM_ABI void                                                       \
+  llvm::codeview::ContinuationRecordBuilder::writeMemberType(                  \
       Name##Record &Record);
 #define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #include "llvm/DebugInfo/CodeView/CodeViewTypes.def"
diff --git a/external/llvm-project/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp b/external/llvm-project/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp
index 25725853fb39..84a8f19fae51 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp
@@ -11,6 +11,7 @@
 #include "llvm/DebugInfo/CodeView/RecordSerialization.h"
 #include "llvm/DebugInfo/CodeView/TypeRecordMapping.h"
 #include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Compiler.h"
 
 using namespace llvm;
 using namespace llvm::codeview;
@@ -60,8 +61,8 @@ ArrayRef<uint8_t> SimpleTypeSerializer::serialize(T &Record) {
 // Explicitly instantiate the member function for each known type so that we can
 // implement this in the cpp file.
 #define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
-  template ArrayRef<uint8_t> llvm::codeview::SimpleTypeSerializer::serialize(  \
-      Name##Record &Record);
+  template LLVM_ABI ArrayRef<uint8_t>                                          \
+  llvm::codeview::SimpleTypeSerializer::serialize(Name##Record &Record);
 #define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #define MEMBER_RECORD(EnumName, EnumVal, Name)
 #define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/CMakeLists.txt b/external/llvm-project/llvm/lib/DebugInfo/DWARF/CMakeLists.txt
index e565821cf294..cc9734f9f22b 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/CMakeLists.txt
@@ -2,9 +2,9 @@ add_llvm_component_library(LLVMDebugInfoDWARF
   DWARFAbbreviationDeclaration.cpp
   DWARFAddressRange.cpp
   DWARFAcceleratorTable.cpp
+  DWARFCFIProgram.cpp
   DWARFCompileUnit.cpp
   DWARFContext.cpp
-  DWARFDataExtractor.cpp
   DWARFDebugAbbrev.cpp
   DWARFDebugAddr.cpp
   DWARFDebugArangeSet.cpp
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFCFIProgram.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFCFIProgram.cpp
new file mode 100644
index 000000000000..8d25599627c4
--- /dev/null
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFCFIProgram.cpp
@@ -0,0 +1,445 @@
+//===- DWARFCFIProgram.cpp - Parsing the cfi-portions of .debug_frame -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <cinttypes>
+#include <cstdint>
+#include <optional>
+
+using namespace llvm;
+using namespace dwarf;
+
+static void printRegister(raw_ostream &OS, DIDumpOptions DumpOpts,
+                          unsigned RegNum) {
+  if (DumpOpts.GetNameForDWARFReg) {
+    auto RegName = DumpOpts.GetNameForDWARFReg(RegNum, DumpOpts.IsEH);
+    if (!RegName.empty()) {
+      OS << RegName;
+      return;
+    }
+  }
+  OS << "reg" << RegNum;
+}
+
+// See DWARF standard v3, section 7.23
+const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;
+const uint8_t DWARF_CFI_PRIMARY_OPERAND_MASK = 0x3f;
+
+Error CFIProgram::parse(DWARFDataExtractor Data, uint64_t *Offset,
+                        uint64_t EndOffset) {
+  DataExtractor::Cursor C(*Offset);
+  while (C && C.tell() < EndOffset) {
+    uint8_t Opcode = Data.getRelocatedValue(C, 1);
+    if (!C)
+      break;
+
+    // Some instructions have a primary opcode encoded in the top bits.
+    if (uint8_t Primary = Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK) {
+      // If it's a primary opcode, the first operand is encoded in the bottom
+      // bits of the opcode itself.
+      uint64_t Op1 = Opcode & DWARF_CFI_PRIMARY_OPERAND_MASK;
+      switch (Primary) {
+      case DW_CFA_advance_loc:
+      case DW_CFA_restore:
+        addInstruction(Primary, Op1);
+        break;
+      case DW_CFA_offset:
+        addInstruction(Primary, Op1, Data.getULEB128(C));
+        break;
+      default:
+        llvm_unreachable("invalid primary CFI opcode");
+      }
+      continue;
+    }
+
+    // Extended opcode - its value is Opcode itself.
+    switch (Opcode) {
+    default:
+      return createStringError(errc::illegal_byte_sequence,
+                               "invalid extended CFI opcode 0x%" PRIx8, Opcode);
+    case DW_CFA_nop:
+    case DW_CFA_remember_state:
+    case DW_CFA_restore_state:
+    case DW_CFA_GNU_window_save:
+    case DW_CFA_AARCH64_negate_ra_state_with_pc:
+      // No operands
+      addInstruction(Opcode);
+      break;
+    case DW_CFA_set_loc:
+      // Operands: Address
+      addInstruction(Opcode, Data.getRelocatedAddress(C));
+      break;
+    case DW_CFA_advance_loc1:
+      // Operands: 1-byte delta
+      addInstruction(Opcode, Data.getRelocatedValue(C, 1));
+      break;
+    case DW_CFA_advance_loc2:
+      // Operands: 2-byte delta
+      addInstruction(Opcode, Data.getRelocatedValue(C, 2));
+      break;
+    case DW_CFA_advance_loc4:
+      // Operands: 4-byte delta
+      addInstruction(Opcode, Data.getRelocatedValue(C, 4));
+      break;
+    case DW_CFA_restore_extended:
+    case DW_CFA_undefined:
+    case DW_CFA_same_value:
+    case DW_CFA_def_cfa_register:
+    case DW_CFA_def_cfa_offset:
+    case DW_CFA_GNU_args_size:
+      // Operands: ULEB128
+      addInstruction(Opcode, Data.getULEB128(C));
+      break;
+    case DW_CFA_def_cfa_offset_sf:
+      // Operands: SLEB128
+      addInstruction(Opcode, Data.getSLEB128(C));
+      break;
+    case DW_CFA_LLVM_def_aspace_cfa:
+    case DW_CFA_LLVM_def_aspace_cfa_sf: {
+      auto RegNum = Data.getULEB128(C);
+      auto CfaOffset = Opcode == DW_CFA_LLVM_def_aspace_cfa
+                           ? Data.getULEB128(C)
+                           : Data.getSLEB128(C);
+      auto AddressSpace = Data.getULEB128(C);
+      addInstruction(Opcode, RegNum, CfaOffset, AddressSpace);
+      break;
+    }
+    case DW_CFA_offset_extended:
+    case DW_CFA_register:
+    case DW_CFA_def_cfa:
+    case DW_CFA_val_offset: {
+      // Operands: ULEB128, ULEB128
+      // Note: We can not embed getULEB128 directly into function
+      // argument list. getULEB128 changes Offset and order of evaluation
+      // for arguments is unspecified.
+      uint64_t op1 = Data.getULEB128(C);
+      uint64_t op2 = Data.getULEB128(C);
+      addInstruction(Opcode, op1, op2);
+      break;
+    }
+    case DW_CFA_offset_extended_sf:
+    case DW_CFA_def_cfa_sf:
+    case DW_CFA_val_offset_sf: {
+      // Operands: ULEB128, SLEB128
+      // Note: see comment for the previous case
+      uint64_t op1 = Data.getULEB128(C);
+      uint64_t op2 = (uint64_t)Data.getSLEB128(C);
+      addInstruction(Opcode, op1, op2);
+      break;
+    }
+    case DW_CFA_def_cfa_expression: {
+      uint64_t ExprLength = Data.getULEB128(C);
+      addInstruction(Opcode, 0);
+      StringRef Expression = Data.getBytes(C, ExprLength);
+
+      DataExtractor Extractor(Expression, Data.isLittleEndian(),
+                              Data.getAddressSize());
+      // Note. We do not pass the DWARF format to DWARFExpression, because
+      // DW_OP_call_ref, the only operation which depends on the format, is
+      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
+      Instructions.back().Expression =
+          DWARFExpression(Extractor, Data.getAddressSize());
+      break;
+    }
+    case DW_CFA_expression:
+    case DW_CFA_val_expression: {
+      uint64_t RegNum = Data.getULEB128(C);
+      addInstruction(Opcode, RegNum, 0);
+
+      uint64_t BlockLength = Data.getULEB128(C);
+      StringRef Expression = Data.getBytes(C, BlockLength);
+      DataExtractor Extractor(Expression, Data.isLittleEndian(),
+                              Data.getAddressSize());
+      // Note. We do not pass the DWARF format to DWARFExpression, because
+      // DW_OP_call_ref, the only operation which depends on the format, is
+      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
+      Instructions.back().Expression =
+          DWARFExpression(Extractor, Data.getAddressSize());
+      break;
+    }
+    }
+  }
+
+  *Offset = C.tell();
+  return C.takeError();
+}
+
+StringRef CFIProgram::callFrameString(unsigned Opcode) const {
+  return dwarf::CallFrameString(Opcode, Arch);
+}
+
+const char *CFIProgram::operandTypeString(CFIProgram::OperandType OT) {
+#define ENUM_TO_CSTR(e)                                                        \
+  case e:                                                                      \
+    return #e;
+  switch (OT) {
+    ENUM_TO_CSTR(OT_Unset);
+    ENUM_TO_CSTR(OT_None);
+    ENUM_TO_CSTR(OT_Address);
+    ENUM_TO_CSTR(OT_Offset);
+    ENUM_TO_CSTR(OT_FactoredCodeOffset);
+    ENUM_TO_CSTR(OT_SignedFactDataOffset);
+    ENUM_TO_CSTR(OT_UnsignedFactDataOffset);
+    ENUM_TO_CSTR(OT_Register);
+    ENUM_TO_CSTR(OT_AddressSpace);
+    ENUM_TO_CSTR(OT_Expression);
+  }
+  return "<unknown CFIProgram::OperandType>";
+}
+
+llvm::Expected<uint64_t>
+CFIProgram::Instruction::getOperandAsUnsigned(const CFIProgram &CFIP,
+                                              uint32_t OperandIdx) const {
+  if (OperandIdx >= MaxOperands)
+    return createStringError(errc::invalid_argument,
+                             "operand index %" PRIu32 " is not valid",
+                             OperandIdx);
+  OperandType Type = CFIP.getOperandTypes()[Opcode][OperandIdx];
+  uint64_t Operand = Ops[OperandIdx];
+  switch (Type) {
+  case OT_Unset:
+  case OT_None:
+  case OT_Expression:
+    return createStringError(errc::invalid_argument,
+                             "op[%" PRIu32 "] has type %s which has no value",
+                             OperandIdx, CFIProgram::operandTypeString(Type));
+
+  case OT_Offset:
+  case OT_SignedFactDataOffset:
+  case OT_UnsignedFactDataOffset:
+    return createStringError(
+        errc::invalid_argument,
+        "op[%" PRIu32 "] has OperandType OT_Offset which produces a signed "
+        "result, call getOperandAsSigned instead",
+        OperandIdx);
+
+  case OT_Address:
+  case OT_Register:
+  case OT_AddressSpace:
+    return Operand;
+
+  case OT_FactoredCodeOffset: {
+    const uint64_t CodeAlignmentFactor = CFIP.codeAlign();
+    if (CodeAlignmentFactor == 0)
+      return createStringError(
+          errc::invalid_argument,
+          "op[%" PRIu32 "] has type OT_FactoredCodeOffset but code alignment "
+          "is zero",
+          OperandIdx);
+    return Operand * CodeAlignmentFactor;
+  }
+  }
+  llvm_unreachable("invalid operand type");
+}
+
+llvm::Expected<int64_t>
+CFIProgram::Instruction::getOperandAsSigned(const CFIProgram &CFIP,
+                                            uint32_t OperandIdx) const {
+  if (OperandIdx >= MaxOperands)
+    return createStringError(errc::invalid_argument,
+                             "operand index %" PRIu32 " is not valid",
+                             OperandIdx);
+  OperandType Type = CFIP.getOperandTypes()[Opcode][OperandIdx];
+  uint64_t Operand = Ops[OperandIdx];
+  switch (Type) {
+  case OT_Unset:
+  case OT_None:
+  case OT_Expression:
+    return createStringError(errc::invalid_argument,
+                             "op[%" PRIu32 "] has type %s which has no value",
+                             OperandIdx, CFIProgram::operandTypeString(Type));
+
+  case OT_Address:
+  case OT_Register:
+  case OT_AddressSpace:
+    return createStringError(
+        errc::invalid_argument,
+        "op[%" PRIu32 "] has OperandType %s which produces an unsigned result, "
+        "call getOperandAsUnsigned instead",
+        OperandIdx, CFIProgram::operandTypeString(Type));
+
+  case OT_Offset:
+    return (int64_t)Operand;
+
+  case OT_FactoredCodeOffset:
+  case OT_SignedFactDataOffset: {
+    const int64_t DataAlignmentFactor = CFIP.dataAlign();
+    if (DataAlignmentFactor == 0)
+      return createStringError(errc::invalid_argument,
+                               "op[%" PRIu32 "] has type %s but data "
+                               "alignment is zero",
+                               OperandIdx, CFIProgram::operandTypeString(Type));
+    return int64_t(Operand) * DataAlignmentFactor;
+  }
+
+  case OT_UnsignedFactDataOffset: {
+    const int64_t DataAlignmentFactor = CFIP.dataAlign();
+    if (DataAlignmentFactor == 0)
+      return createStringError(errc::invalid_argument,
+                               "op[%" PRIu32
+                               "] has type OT_UnsignedFactDataOffset but data "
+                               "alignment is zero",
+                               OperandIdx);
+    return Operand * DataAlignmentFactor;
+  }
+  }
+  llvm_unreachable("invalid operand type");
+}
+
+ArrayRef<CFIProgram::OperandType[CFIProgram::MaxOperands]>
+CFIProgram::getOperandTypes() {
+  static OperandType OpTypes[DW_CFA_restore + 1][MaxOperands];
+  static bool Initialized = false;
+  if (Initialized) {
+    return ArrayRef<OperandType[MaxOperands]>(&OpTypes[0], DW_CFA_restore + 1);
+  }
+  Initialized = true;
+
+#define DECLARE_OP3(OP, OPTYPE0, OPTYPE1, OPTYPE2)                             \
+  do {                                                                         \
+    OpTypes[OP][0] = OPTYPE0;                                                  \
+    OpTypes[OP][1] = OPTYPE1;                                                  \
+    OpTypes[OP][2] = OPTYPE2;                                                  \
+  } while (false)
+#define DECLARE_OP2(OP, OPTYPE0, OPTYPE1)                                      \
+  DECLARE_OP3(OP, OPTYPE0, OPTYPE1, OT_None)
+#define DECLARE_OP1(OP, OPTYPE0) DECLARE_OP2(OP, OPTYPE0, OT_None)
+#define DECLARE_OP0(OP) DECLARE_OP1(OP, OT_None)
+
+  DECLARE_OP1(DW_CFA_set_loc, OT_Address);
+  DECLARE_OP1(DW_CFA_advance_loc, OT_FactoredCodeOffset);
+  DECLARE_OP1(DW_CFA_advance_loc1, OT_FactoredCodeOffset);
+  DECLARE_OP1(DW_CFA_advance_loc2, OT_FactoredCodeOffset);
+  DECLARE_OP1(DW_CFA_advance_loc4, OT_FactoredCodeOffset);
+  DECLARE_OP1(DW_CFA_MIPS_advance_loc8, OT_FactoredCodeOffset);
+  DECLARE_OP2(DW_CFA_def_cfa, OT_Register, OT_Offset);
+  DECLARE_OP2(DW_CFA_def_cfa_sf, OT_Register, OT_SignedFactDataOffset);
+  DECLARE_OP1(DW_CFA_def_cfa_register, OT_Register);
+  DECLARE_OP3(DW_CFA_LLVM_def_aspace_cfa, OT_Register, OT_Offset,
+              OT_AddressSpace);
+  DECLARE_OP3(DW_CFA_LLVM_def_aspace_cfa_sf, OT_Register,
+              OT_SignedFactDataOffset, OT_AddressSpace);
+  DECLARE_OP1(DW_CFA_def_cfa_offset, OT_Offset);
+  DECLARE_OP1(DW_CFA_def_cfa_offset_sf, OT_SignedFactDataOffset);
+  DECLARE_OP1(DW_CFA_def_cfa_expression, OT_Expression);
+  DECLARE_OP1(DW_CFA_undefined, OT_Register);
+  DECLARE_OP1(DW_CFA_same_value, OT_Register);
+  DECLARE_OP2(DW_CFA_offset, OT_Register, OT_UnsignedFactDataOffset);
+  DECLARE_OP2(DW_CFA_offset_extended, OT_Register, OT_UnsignedFactDataOffset);
+  DECLARE_OP2(DW_CFA_offset_extended_sf, OT_Register, OT_SignedFactDataOffset);
+  DECLARE_OP2(DW_CFA_val_offset, OT_Register, OT_UnsignedFactDataOffset);
+  DECLARE_OP2(DW_CFA_val_offset_sf, OT_Register, OT_SignedFactDataOffset);
+  DECLARE_OP2(DW_CFA_register, OT_Register, OT_Register);
+  DECLARE_OP2(DW_CFA_expression, OT_Register, OT_Expression);
+  DECLARE_OP2(DW_CFA_val_expression, OT_Register, OT_Expression);
+  DECLARE_OP1(DW_CFA_restore, OT_Register);
+  DECLARE_OP1(DW_CFA_restore_extended, OT_Register);
+  DECLARE_OP0(DW_CFA_remember_state);
+  DECLARE_OP0(DW_CFA_restore_state);
+  DECLARE_OP0(DW_CFA_GNU_window_save);
+  DECLARE_OP0(DW_CFA_AARCH64_negate_ra_state_with_pc);
+  DECLARE_OP1(DW_CFA_GNU_args_size, OT_Offset);
+  DECLARE_OP0(DW_CFA_nop);
+
+#undef DECLARE_OP0
+#undef DECLARE_OP1
+#undef DECLARE_OP2
+
+  return ArrayRef<OperandType[MaxOperands]>(&OpTypes[0], DW_CFA_restore + 1);
+}
+
+/// Print \p Opcode's operand number \p OperandIdx which has value \p Operand.
+void CFIProgram::printOperand(raw_ostream &OS, DIDumpOptions DumpOpts,
+                              const Instruction &Instr, unsigned OperandIdx,
+                              uint64_t Operand,
+                              std::optional<uint64_t> &Address) const {
+  assert(OperandIdx < MaxOperands);
+  uint8_t Opcode = Instr.Opcode;
+  OperandType Type = getOperandTypes()[Opcode][OperandIdx];
+
+  switch (Type) {
+  case OT_Unset: {
+    OS << " Unsupported " << (OperandIdx ? "second" : "first") << " operand to";
+    auto OpcodeName = callFrameString(Opcode);
+    if (!OpcodeName.empty())
+      OS << " " << OpcodeName;
+    else
+      OS << format(" Opcode %x", Opcode);
+    break;
+  }
+  case OT_None:
+    break;
+  case OT_Address:
+    OS << format(" %" PRIx64, Operand);
+    Address = Operand;
+    break;
+  case OT_Offset:
+    // The offsets are all encoded in a unsigned form, but in practice
+    // consumers use them signed. It's most certainly legacy due to
+    // the lack of signed variants in the first Dwarf standards.
+    OS << format(" %+" PRId64, int64_t(Operand));
+    break;
+  case OT_FactoredCodeOffset: // Always Unsigned
+    if (CodeAlignmentFactor)
+      OS << format(" %" PRId64, Operand * CodeAlignmentFactor);
+    else
+      OS << format(" %" PRId64 "*code_alignment_factor", Operand);
+    if (Address && CodeAlignmentFactor) {
+      *Address += Operand * CodeAlignmentFactor;
+      OS << format(" to 0x%" PRIx64, *Address);
+    }
+    break;
+  case OT_SignedFactDataOffset:
+    if (DataAlignmentFactor)
+      OS << format(" %" PRId64, int64_t(Operand) * DataAlignmentFactor);
+    else
+      OS << format(" %" PRId64 "*data_alignment_factor", int64_t(Operand));
+    break;
+  case OT_UnsignedFactDataOffset:
+    if (DataAlignmentFactor)
+      OS << format(" %" PRId64, Operand * DataAlignmentFactor);
+    else
+      OS << format(" %" PRId64 "*data_alignment_factor", Operand);
+    break;
+  case OT_Register:
+    OS << ' ';
+    printRegister(OS, DumpOpts, Operand);
+    break;
+  case OT_AddressSpace:
+    OS << format(" in addrspace%" PRId64, Operand);
+    break;
+  case OT_Expression:
+    assert(Instr.Expression && "missing DWARFExpression object");
+    OS << " ";
+    DWARFExpressionPrinter::print(&Instr.Expression.value(), OS, DumpOpts,
+                                  nullptr);
+    break;
+  }
+}
+
+void CFIProgram::dump(raw_ostream &OS, DIDumpOptions DumpOpts,
+                      unsigned IndentLevel,
+                      std::optional<uint64_t> Address) const {
+  for (const auto &Instr : Instructions) {
+    uint8_t Opcode = Instr.Opcode;
+    OS.indent(2 * IndentLevel);
+    OS << callFrameString(Opcode) << ":";
+    for (unsigned i = 0; i < Instr.Ops.size(); ++i)
+      printOperand(OS, DumpOpts, Instr, i, Instr.Ops[i], Address);
+    OS << '\n';
+  }
+}
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
index 986f604c7f10..6d0a94d8a336 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -621,10 +621,6 @@ class ThreadUnsafeDWARFContextState : public DWARFContext::DWARFContextState {
     else
       return getNormalTypeUnitMap();
   }
-
-  Error doWorkThreadSafely(function_ref<Error()> Work) override {
-    return Work();
-  }
 };
 
 class ThreadSafeState : public ThreadUnsafeDWARFContextState {
@@ -740,11 +736,6 @@ class ThreadSafeState : public ThreadUnsafeDWARFContextState {
     std::unique_lock<std::recursive_mutex> LockGuard(Mutex);
     return ThreadUnsafeDWARFContextState::getTypeUnitMap(IsDWO);
   }
-
-  Error doWorkThreadSafely(function_ref<Error()> Work) override {
-    std::unique_lock<std::recursive_mutex> LockGuard(Mutex);
-    return ThreadUnsafeDWARFContextState::doWorkThreadSafely(Work);
-  }
 };
 } // namespace
 
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp
deleted file mode 100644
index 97434b3cfab8..000000000000
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp
+++ /dev/null
@@ -1,137 +0,0 @@
-//===- DWARFDataExtractor.cpp ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
-#include "llvm/DebugInfo/DWARF/DWARFObject.h"
-#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h"
-#include "llvm/Support/Errc.h"
-
-using namespace llvm;
-
-std::pair<uint64_t, dwarf::DwarfFormat>
-DWARFDataExtractor::getInitialLength(uint64_t *Off, Error *Err) const {
-  ErrorAsOutParameter ErrAsOut(Err);
-  if (Err && *Err)
-    return {0, dwarf::DWARF32};
-
-  Cursor C(*Off);
-  uint64_t Length = getRelocatedValue(C, 4);
-  dwarf::DwarfFormat Format = dwarf::DWARF32;
-  if (Length == dwarf::DW_LENGTH_DWARF64) {
-    Length = getRelocatedValue(C, 8);
-    Format = dwarf::DWARF64;
-  } else if (Length >= dwarf::DW_LENGTH_lo_reserved) {
-    cantFail(C.takeError());
-    if (Err)
-      *Err = createStringError(
-          errc::invalid_argument,
-          "unsupported reserved unit length of value 0x%8.8" PRIx64, Length);
-    return {0, dwarf::DWARF32};
-  }
-
-  if (C) {
-    *Off = C.tell();
-    return {Length, Format};
-  }
-  if (Err)
-    *Err = C.takeError();
-  else
-    consumeError(C.takeError());
-  return {0, dwarf::DWARF32};
-}
-
-uint64_t DWARFDataExtractor::getRelocatedValue(uint32_t Size, uint64_t *Off,
-                                               uint64_t *SecNdx,
-                                               Error *Err) const {
-  if (SecNdx)
-    *SecNdx = object::SectionedAddress::UndefSection;
-  if (!Section)
-    return getUnsigned(Off, Size, Err);
-
-  ErrorAsOutParameter ErrAsOut(Err);
-  std::optional<RelocAddrEntry> E = Obj->find(*Section, *Off);
-  uint64_t LocData = getUnsigned(Off, Size, Err);
-  if (!E || (Err && *Err))
-    return LocData;
-  if (SecNdx)
-    *SecNdx = E->SectionIndex;
-
-  uint64_t R =
-      object::resolveRelocation(E->Resolver, E->Reloc, E->SymbolValue, LocData);
-  if (E->Reloc2)
-    R = object::resolveRelocation(E->Resolver, *E->Reloc2, E->SymbolValue2, R);
-  return R;
-}
-
-std::optional<uint64_t>
-DWARFDataExtractor::getEncodedPointer(uint64_t *Offset, uint8_t Encoding,
-                                      uint64_t PCRelOffset) const {
-  if (Encoding == dwarf::DW_EH_PE_omit)
-    return std::nullopt;
-
-  uint64_t Result = 0;
-  uint64_t OldOffset = *Offset;
-  // First get value
-  switch (Encoding & 0x0F) {
-  case dwarf::DW_EH_PE_absptr:
-    switch (getAddressSize()) {
-    case 2:
-    case 4:
-    case 8:
-      Result = getUnsigned(Offset, getAddressSize());
-      break;
-    default:
-      return std::nullopt;
-    }
-    break;
-  case dwarf::DW_EH_PE_uleb128:
-    Result = getULEB128(Offset);
-    break;
-  case dwarf::DW_EH_PE_sleb128:
-    Result = getSLEB128(Offset);
-    break;
-  case dwarf::DW_EH_PE_udata2:
-    Result = getUnsigned(Offset, 2);
-    break;
-  case dwarf::DW_EH_PE_udata4:
-    Result = getUnsigned(Offset, 4);
-    break;
-  case dwarf::DW_EH_PE_udata8:
-    Result = getUnsigned(Offset, 8);
-    break;
-  case dwarf::DW_EH_PE_sdata2:
-    Result = getSigned(Offset, 2);
-    break;
-  case dwarf::DW_EH_PE_sdata4:
-    Result = SignExtend64<32>(getRelocatedValue(4, Offset));
-    break;
-  case dwarf::DW_EH_PE_sdata8:
-    Result = getRelocatedValue(8, Offset);
-    break;
-  default:
-    return std::nullopt;
-  }
-  // Then add relative offset, if required
-  switch (Encoding & 0x70) {
-  case dwarf::DW_EH_PE_absptr:
-    // do nothing
-    break;
-  case dwarf::DW_EH_PE_pcrel:
-    Result += PCRelOffset;
-    break;
-  case dwarf::DW_EH_PE_datarel:
-  case dwarf::DW_EH_PE_textrel:
-  case dwarf::DW_EH_PE_funcrel:
-  case dwarf::DW_EH_PE_aligned:
-  default:
-    *Offset = OldOffset;
-    return std::nullopt;
-  }
-
-  return Result;
-}
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
index 5bdc257fd8d8..c46b14b4446f 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
@@ -12,7 +12,9 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Errc.h"
@@ -109,7 +111,8 @@ void UnwindLocation::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
       OS << " in addrspace" << *AddrSpace;
     break;
   case DWARFExpr: {
-    Expr->print(OS, DumpOpts, nullptr);
+    if (Expr)
+      DWARFExpressionPrinter::print(&(*Expr), OS, DumpOpts, nullptr);
     break;
   }
   case Constant:
@@ -247,271 +250,6 @@ Expected<UnwindTable> UnwindTable::create(const CIE *Cie) {
   return UT;
 }
 
-// See DWARF standard v3, section 7.23
-const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;
-const uint8_t DWARF_CFI_PRIMARY_OPERAND_MASK = 0x3f;
-
-Error CFIProgram::parse(DWARFDataExtractor Data, uint64_t *Offset,
-                        uint64_t EndOffset) {
-  DataExtractor::Cursor C(*Offset);
-  while (C && C.tell() < EndOffset) {
-    uint8_t Opcode = Data.getRelocatedValue(C, 1);
-    if (!C)
-      break;
-
-    // Some instructions have a primary opcode encoded in the top bits.
-    if (uint8_t Primary = Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK) {
-      // If it's a primary opcode, the first operand is encoded in the bottom
-      // bits of the opcode itself.
-      uint64_t Op1 = Opcode & DWARF_CFI_PRIMARY_OPERAND_MASK;
-      switch (Primary) {
-      case DW_CFA_advance_loc:
-      case DW_CFA_restore:
-        addInstruction(Primary, Op1);
-        break;
-      case DW_CFA_offset:
-        addInstruction(Primary, Op1, Data.getULEB128(C));
-        break;
-      default:
-        llvm_unreachable("invalid primary CFI opcode");
-      }
-      continue;
-    }
-
-    // Extended opcode - its value is Opcode itself.
-    switch (Opcode) {
-    default:
-      return createStringError(errc::illegal_byte_sequence,
-                               "invalid extended CFI opcode 0x%" PRIx8, Opcode);
-    case DW_CFA_nop:
-    case DW_CFA_remember_state:
-    case DW_CFA_restore_state:
-    case DW_CFA_GNU_window_save:
-    case DW_CFA_AARCH64_negate_ra_state_with_pc:
-      // No operands
-      addInstruction(Opcode);
-      break;
-    case DW_CFA_set_loc:
-      // Operands: Address
-      addInstruction(Opcode, Data.getRelocatedAddress(C));
-      break;
-    case DW_CFA_advance_loc1:
-      // Operands: 1-byte delta
-      addInstruction(Opcode, Data.getRelocatedValue(C, 1));
-      break;
-    case DW_CFA_advance_loc2:
-      // Operands: 2-byte delta
-      addInstruction(Opcode, Data.getRelocatedValue(C, 2));
-      break;
-    case DW_CFA_advance_loc4:
-      // Operands: 4-byte delta
-      addInstruction(Opcode, Data.getRelocatedValue(C, 4));
-      break;
-    case DW_CFA_restore_extended:
-    case DW_CFA_undefined:
-    case DW_CFA_same_value:
-    case DW_CFA_def_cfa_register:
-    case DW_CFA_def_cfa_offset:
-    case DW_CFA_GNU_args_size:
-      // Operands: ULEB128
-      addInstruction(Opcode, Data.getULEB128(C));
-      break;
-    case DW_CFA_def_cfa_offset_sf:
-      // Operands: SLEB128
-      addInstruction(Opcode, Data.getSLEB128(C));
-      break;
-    case DW_CFA_LLVM_def_aspace_cfa:
-    case DW_CFA_LLVM_def_aspace_cfa_sf: {
-      auto RegNum = Data.getULEB128(C);
-      auto CfaOffset = Opcode == DW_CFA_LLVM_def_aspace_cfa
-                           ? Data.getULEB128(C)
-                           : Data.getSLEB128(C);
-      auto AddressSpace = Data.getULEB128(C);
-      addInstruction(Opcode, RegNum, CfaOffset, AddressSpace);
-      break;
-    }
-    case DW_CFA_offset_extended:
-    case DW_CFA_register:
-    case DW_CFA_def_cfa:
-    case DW_CFA_val_offset: {
-      // Operands: ULEB128, ULEB128
-      // Note: We can not embed getULEB128 directly into function
-      // argument list. getULEB128 changes Offset and order of evaluation
-      // for arguments is unspecified.
-      uint64_t op1 = Data.getULEB128(C);
-      uint64_t op2 = Data.getULEB128(C);
-      addInstruction(Opcode, op1, op2);
-      break;
-    }
-    case DW_CFA_offset_extended_sf:
-    case DW_CFA_def_cfa_sf:
-    case DW_CFA_val_offset_sf: {
-      // Operands: ULEB128, SLEB128
-      // Note: see comment for the previous case
-      uint64_t op1 = Data.getULEB128(C);
-      uint64_t op2 = (uint64_t)Data.getSLEB128(C);
-      addInstruction(Opcode, op1, op2);
-      break;
-    }
-    case DW_CFA_def_cfa_expression: {
-      uint64_t ExprLength = Data.getULEB128(C);
-      addInstruction(Opcode, 0);
-      StringRef Expression = Data.getBytes(C, ExprLength);
-
-      DataExtractor Extractor(Expression, Data.isLittleEndian(),
-                              Data.getAddressSize());
-      // Note. We do not pass the DWARF format to DWARFExpression, because
-      // DW_OP_call_ref, the only operation which depends on the format, is
-      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
-      Instructions.back().Expression =
-          DWARFExpression(Extractor, Data.getAddressSize());
-      break;
-    }
-    case DW_CFA_expression:
-    case DW_CFA_val_expression: {
-      uint64_t RegNum = Data.getULEB128(C);
-      addInstruction(Opcode, RegNum, 0);
-
-      uint64_t BlockLength = Data.getULEB128(C);
-      StringRef Expression = Data.getBytes(C, BlockLength);
-      DataExtractor Extractor(Expression, Data.isLittleEndian(),
-                              Data.getAddressSize());
-      // Note. We do not pass the DWARF format to DWARFExpression, because
-      // DW_OP_call_ref, the only operation which depends on the format, is
-      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
-      Instructions.back().Expression =
-          DWARFExpression(Extractor, Data.getAddressSize());
-      break;
-    }
-    }
-  }
-
-  *Offset = C.tell();
-  return C.takeError();
-}
-
-StringRef CFIProgram::callFrameString(unsigned Opcode) const {
-  return dwarf::CallFrameString(Opcode, Arch);
-}
-
-const char *CFIProgram::operandTypeString(CFIProgram::OperandType OT) {
-#define ENUM_TO_CSTR(e)                                                        \
-  case e:                                                                      \
-    return #e;
-  switch (OT) {
-    ENUM_TO_CSTR(OT_Unset);
-    ENUM_TO_CSTR(OT_None);
-    ENUM_TO_CSTR(OT_Address);
-    ENUM_TO_CSTR(OT_Offset);
-    ENUM_TO_CSTR(OT_FactoredCodeOffset);
-    ENUM_TO_CSTR(OT_SignedFactDataOffset);
-    ENUM_TO_CSTR(OT_UnsignedFactDataOffset);
-    ENUM_TO_CSTR(OT_Register);
-    ENUM_TO_CSTR(OT_AddressSpace);
-    ENUM_TO_CSTR(OT_Expression);
-  }
-  return "<unknown CFIProgram::OperandType>";
-}
-
-llvm::Expected<uint64_t>
-CFIProgram::Instruction::getOperandAsUnsigned(const CFIProgram &CFIP,
-                                              uint32_t OperandIdx) const {
-  if (OperandIdx >= MaxOperands)
-    return createStringError(errc::invalid_argument,
-                             "operand index %" PRIu32 " is not valid",
-                             OperandIdx);
-  OperandType Type = CFIP.getOperandTypes()[Opcode][OperandIdx];
-  uint64_t Operand = Ops[OperandIdx];
-  switch (Type) {
-  case OT_Unset:
-  case OT_None:
-  case OT_Expression:
-    return createStringError(errc::invalid_argument,
-                             "op[%" PRIu32 "] has type %s which has no value",
-                             OperandIdx, CFIProgram::operandTypeString(Type));
-
-  case OT_Offset:
-  case OT_SignedFactDataOffset:
-  case OT_UnsignedFactDataOffset:
-    return createStringError(
-        errc::invalid_argument,
-        "op[%" PRIu32 "] has OperandType OT_Offset which produces a signed "
-        "result, call getOperandAsSigned instead",
-        OperandIdx);
-
-  case OT_Address:
-  case OT_Register:
-  case OT_AddressSpace:
-    return Operand;
-
-  case OT_FactoredCodeOffset: {
-    const uint64_t CodeAlignmentFactor = CFIP.codeAlign();
-    if (CodeAlignmentFactor == 0)
-      return createStringError(
-          errc::invalid_argument,
-          "op[%" PRIu32 "] has type OT_FactoredCodeOffset but code alignment "
-          "is zero",
-          OperandIdx);
-    return Operand * CodeAlignmentFactor;
-  }
-  }
-  llvm_unreachable("invalid operand type");
-}
-
-llvm::Expected<int64_t>
-CFIProgram::Instruction::getOperandAsSigned(const CFIProgram &CFIP,
-                                            uint32_t OperandIdx) const {
-  if (OperandIdx >= MaxOperands)
-    return createStringError(errc::invalid_argument,
-                             "operand index %" PRIu32 " is not valid",
-                             OperandIdx);
-  OperandType Type = CFIP.getOperandTypes()[Opcode][OperandIdx];
-  uint64_t Operand = Ops[OperandIdx];
-  switch (Type) {
-  case OT_Unset:
-  case OT_None:
-  case OT_Expression:
-    return createStringError(errc::invalid_argument,
-                             "op[%" PRIu32 "] has type %s which has no value",
-                             OperandIdx, CFIProgram::operandTypeString(Type));
-
-  case OT_Address:
-  case OT_Register:
-  case OT_AddressSpace:
-    return createStringError(
-        errc::invalid_argument,
-        "op[%" PRIu32 "] has OperandType %s which produces an unsigned result, "
-        "call getOperandAsUnsigned instead",
-        OperandIdx, CFIProgram::operandTypeString(Type));
-
-  case OT_Offset:
-    return (int64_t)Operand;
-
-  case OT_FactoredCodeOffset:
-  case OT_SignedFactDataOffset: {
-    const int64_t DataAlignmentFactor = CFIP.dataAlign();
-    if (DataAlignmentFactor == 0)
-      return createStringError(errc::invalid_argument,
-                               "op[%" PRIu32 "] has type %s but data "
-                               "alignment is zero",
-                               OperandIdx, CFIProgram::operandTypeString(Type));
-    return int64_t(Operand) * DataAlignmentFactor;
-  }
-
-  case OT_UnsignedFactDataOffset: {
-    const int64_t DataAlignmentFactor = CFIP.dataAlign();
-    if (DataAlignmentFactor == 0)
-      return createStringError(errc::invalid_argument,
-                               "op[%" PRIu32
-                               "] has type OT_UnsignedFactDataOffset but data "
-                               "alignment is zero",
-                               OperandIdx);
-    return Operand * DataAlignmentFactor;
-  }
-  }
-  llvm_unreachable("invalid operand type");
-}
-
 Error UnwindTable::parseRows(const CFIProgram &CFIP, UnwindRow &Row,
                              const RegisterLocations *InitialLocs) {
   // State consists of CFA value and register locations.
@@ -818,149 +556,6 @@ Error UnwindTable::parseRows(const CFIProgram &CFIP, UnwindRow &Row,
   return Error::success();
 }
 
-ArrayRef<CFIProgram::OperandType[CFIProgram::MaxOperands]>
-CFIProgram::getOperandTypes() {
-  static OperandType OpTypes[DW_CFA_restore + 1][MaxOperands];
-  static bool Initialized = false;
-  if (Initialized) {
-    return ArrayRef<OperandType[MaxOperands]>(&OpTypes[0], DW_CFA_restore + 1);
-  }
-  Initialized = true;
-
-#define DECLARE_OP3(OP, OPTYPE0, OPTYPE1, OPTYPE2)                             \
-  do {                                                                         \
-    OpTypes[OP][0] = OPTYPE0;                                                  \
-    OpTypes[OP][1] = OPTYPE1;                                                  \
-    OpTypes[OP][2] = OPTYPE2;                                                  \
-  } while (false)
-#define DECLARE_OP2(OP, OPTYPE0, OPTYPE1)                                      \
-  DECLARE_OP3(OP, OPTYPE0, OPTYPE1, OT_None)
-#define DECLARE_OP1(OP, OPTYPE0) DECLARE_OP2(OP, OPTYPE0, OT_None)
-#define DECLARE_OP0(OP) DECLARE_OP1(OP, OT_None)
-
-  DECLARE_OP1(DW_CFA_set_loc, OT_Address);
-  DECLARE_OP1(DW_CFA_advance_loc, OT_FactoredCodeOffset);
-  DECLARE_OP1(DW_CFA_advance_loc1, OT_FactoredCodeOffset);
-  DECLARE_OP1(DW_CFA_advance_loc2, OT_FactoredCodeOffset);
-  DECLARE_OP1(DW_CFA_advance_loc4, OT_FactoredCodeOffset);
-  DECLARE_OP1(DW_CFA_MIPS_advance_loc8, OT_FactoredCodeOffset);
-  DECLARE_OP2(DW_CFA_def_cfa, OT_Register, OT_Offset);
-  DECLARE_OP2(DW_CFA_def_cfa_sf, OT_Register, OT_SignedFactDataOffset);
-  DECLARE_OP1(DW_CFA_def_cfa_register, OT_Register);
-  DECLARE_OP3(DW_CFA_LLVM_def_aspace_cfa, OT_Register, OT_Offset,
-              OT_AddressSpace);
-  DECLARE_OP3(DW_CFA_LLVM_def_aspace_cfa_sf, OT_Register,
-              OT_SignedFactDataOffset, OT_AddressSpace);
-  DECLARE_OP1(DW_CFA_def_cfa_offset, OT_Offset);
-  DECLARE_OP1(DW_CFA_def_cfa_offset_sf, OT_SignedFactDataOffset);
-  DECLARE_OP1(DW_CFA_def_cfa_expression, OT_Expression);
-  DECLARE_OP1(DW_CFA_undefined, OT_Register);
-  DECLARE_OP1(DW_CFA_same_value, OT_Register);
-  DECLARE_OP2(DW_CFA_offset, OT_Register, OT_UnsignedFactDataOffset);
-  DECLARE_OP2(DW_CFA_offset_extended, OT_Register, OT_UnsignedFactDataOffset);
-  DECLARE_OP2(DW_CFA_offset_extended_sf, OT_Register, OT_SignedFactDataOffset);
-  DECLARE_OP2(DW_CFA_val_offset, OT_Register, OT_UnsignedFactDataOffset);
-  DECLARE_OP2(DW_CFA_val_offset_sf, OT_Register, OT_SignedFactDataOffset);
-  DECLARE_OP2(DW_CFA_register, OT_Register, OT_Register);
-  DECLARE_OP2(DW_CFA_expression, OT_Register, OT_Expression);
-  DECLARE_OP2(DW_CFA_val_expression, OT_Register, OT_Expression);
-  DECLARE_OP1(DW_CFA_restore, OT_Register);
-  DECLARE_OP1(DW_CFA_restore_extended, OT_Register);
-  DECLARE_OP0(DW_CFA_remember_state);
-  DECLARE_OP0(DW_CFA_restore_state);
-  DECLARE_OP0(DW_CFA_GNU_window_save);
-  DECLARE_OP0(DW_CFA_AARCH64_negate_ra_state_with_pc);
-  DECLARE_OP1(DW_CFA_GNU_args_size, OT_Offset);
-  DECLARE_OP0(DW_CFA_nop);
-
-#undef DECLARE_OP0
-#undef DECLARE_OP1
-#undef DECLARE_OP2
-
-  return ArrayRef<OperandType[MaxOperands]>(&OpTypes[0], DW_CFA_restore + 1);
-}
-
-/// Print \p Opcode's operand number \p OperandIdx which has value \p Operand.
-void CFIProgram::printOperand(raw_ostream &OS, DIDumpOptions DumpOpts,
-                              const Instruction &Instr, unsigned OperandIdx,
-                              uint64_t Operand,
-                              std::optional<uint64_t> &Address) const {
-  assert(OperandIdx < MaxOperands);
-  uint8_t Opcode = Instr.Opcode;
-  OperandType Type = getOperandTypes()[Opcode][OperandIdx];
-
-  switch (Type) {
-  case OT_Unset: {
-    OS << " Unsupported " << (OperandIdx ? "second" : "first") << " operand to";
-    auto OpcodeName = callFrameString(Opcode);
-    if (!OpcodeName.empty())
-      OS << " " << OpcodeName;
-    else
-      OS << format(" Opcode %x",  Opcode);
-    break;
-  }
-  case OT_None:
-    break;
-  case OT_Address:
-    OS << format(" %" PRIx64, Operand);
-    Address = Operand;
-    break;
-  case OT_Offset:
-    // The offsets are all encoded in a unsigned form, but in practice
-    // consumers use them signed. It's most certainly legacy due to
-    // the lack of signed variants in the first Dwarf standards.
-    OS << format(" %+" PRId64, int64_t(Operand));
-    break;
-  case OT_FactoredCodeOffset: // Always Unsigned
-    if (CodeAlignmentFactor)
-      OS << format(" %" PRId64, Operand * CodeAlignmentFactor);
-    else
-      OS << format(" %" PRId64 "*code_alignment_factor", Operand);
-    if (Address && CodeAlignmentFactor) {
-      *Address += Operand * CodeAlignmentFactor;
-      OS << format(" to 0x%" PRIx64, *Address);
-    }
-    break;
-  case OT_SignedFactDataOffset:
-    if (DataAlignmentFactor)
-      OS << format(" %" PRId64, int64_t(Operand) * DataAlignmentFactor);
-    else
-      OS << format(" %" PRId64 "*data_alignment_factor" , int64_t(Operand));
-    break;
-  case OT_UnsignedFactDataOffset:
-    if (DataAlignmentFactor)
-      OS << format(" %" PRId64, Operand * DataAlignmentFactor);
-    else
-      OS << format(" %" PRId64 "*data_alignment_factor" , Operand);
-    break;
-  case OT_Register:
-    OS << ' ';
-    printRegister(OS, DumpOpts, Operand);
-    break;
-  case OT_AddressSpace:
-    OS << format(" in addrspace%" PRId64, Operand);
-    break;
-  case OT_Expression:
-    assert(Instr.Expression && "missing DWARFExpression object");
-    OS << " ";
-    Instr.Expression->print(OS, DumpOpts, nullptr);
-    break;
-  }
-}
-
-void CFIProgram::dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-                      unsigned IndentLevel,
-                      std::optional<uint64_t> Address) const {
-  for (const auto &Instr : Instructions) {
-    uint8_t Opcode = Instr.Opcode;
-    OS.indent(2 * IndentLevel);
-    OS << callFrameString(Opcode) << ":";
-    for (unsigned i = 0; i < Instr.Ops.size(); ++i)
-      printOperand(OS, DumpOpts, Instr, i, Instr.Ops[i], Address);
-    OS << '\n';
-  }
-}
-
 // Returns the CIE identifier to be used by the requested format.
 // CIE ids for .debug_frame sections are defined in Section 7.24 of DWARFv5.
 // For CIE ID in .eh_frame sections see
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
index ec7af792efb0..fc71be32fdd7 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
@@ -116,7 +116,8 @@ static void dumpExpression(raw_ostream &OS, DIDumpOptions DumpOpts,
   std::optional<dwarf::DwarfFormat> Format;
   if (U)
     Format = U->getFormat();
-  DWARFExpression(Extractor, AddressSize, Format).print(OS, DumpOpts, U);
+  DWARFExpression E(Extractor, AddressSize, Format);
+  DWARFExpressionPrinter::print(&E, OS, DumpOpts, U);
 }
 
 bool DWARFLocationTable::dumpLocationList(
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
index 1406909bfae1..80857c0c7473 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
@@ -112,8 +112,8 @@ static void dumpLocationExpr(raw_ostream &OS, const DWARFFormValue &FormValue,
   ArrayRef<uint8_t> Expr = *FormValue.getAsBlock();
   DataExtractor Data(StringRef((const char *)Expr.data(), Expr.size()),
                      Ctx.isLittleEndian(), 0);
-  DWARFExpression(Data, U->getAddressByteSize(), U->getFormParams().Format)
-      .print(OS, DumpOpts, U);
+  DWARFExpression DE(Data, U->getAddressByteSize(), U->getFormParams().Format);
+  DWARFExpressionPrinter::print(&DE, OS, DumpOpts, U);
 }
 
 static DWARFDie resolveReferencedType(DWARFDie D, DWARFFormValue F) {
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index a08914c25f22..8d7bd39993cf 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -123,6 +123,8 @@ static std::vector<Desc> getOpDescriptions() {
   Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB);
   Descriptions[DW_OP_regval_type] =
       Desc(Op::Dwarf5, Op::SizeLEB, Op::BaseTypeRef);
+  Descriptions[DW_OP_GNU_implicit_pointer] =
+      Desc(Op::Dwarf4, Op::SizeRefAddr, Op::SignedSizeLEB);
   // This Description acts as a marker that getSubOpDesc must be called
   // to fetch the final Description for the operation. Each such final
   // Description must share the same first SizeSubOpLEB operand.
@@ -275,10 +277,23 @@ bool DWARFExpression::Operation::extract(DataExtractor Data,
   return true;
 }
 
-static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
-                                   DIDumpOptions DumpOpts,
-                                   ArrayRef<uint64_t> Operands,
-                                   unsigned Operand) {
+std::optional<unsigned> DWARFExpression::Operation::getSubCode() const {
+  if (!Desc.Op.size() || Desc.Op[0] != Operation::SizeSubOpLEB)
+    return std::nullopt;
+  return Operands[0];
+}
+
+bool DWARFExpression::operator==(const DWARFExpression &RHS) const {
+  if (AddressSize != RHS.AddressSize || Format != RHS.Format)
+    return false;
+  return Data.getData() == RHS.Data.getData();
+}
+
+void DWARFExpressionPrinter::prettyPrintBaseTypeRef(DWARFUnit *U,
+                                                    raw_ostream &OS,
+                                                    DIDumpOptions DumpOpts,
+                                                    ArrayRef<uint64_t> Operands,
+                                                    unsigned Operand) {
   assert(Operand < Operands.size() && "operand out of bounds");
   if (!U) {
     OS << format(" <base_type ref: 0x%" PRIx64 ">", Operands[Operand]);
@@ -297,10 +312,9 @@ static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
   }
 }
 
-bool DWARFExpression::prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
-                                            DIDumpOptions DumpOpts,
-                                            uint8_t Opcode,
-                                            ArrayRef<uint64_t> Operands) {
+bool DWARFExpressionPrinter::prettyPrintRegisterOp(
+    DWARFUnit *U, raw_ostream &OS, DIDumpOptions DumpOpts, uint8_t Opcode,
+    ArrayRef<uint64_t> Operands) {
   if (!DumpOpts.GetNameForDWARFReg)
     return false;
 
@@ -341,24 +355,20 @@ bool DWARFExpression::prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
   return false;
 }
 
-std::optional<unsigned> DWARFExpression::Operation::getSubCode() const {
-  if (!Desc.Op.size() || Desc.Op[0] != Operation::SizeSubOpLEB)
-    return std::nullopt;
-  return Operands[0];
-}
-
-bool DWARFExpression::Operation::print(raw_ostream &OS, DIDumpOptions DumpOpts,
-                                       const DWARFExpression *Expr,
-                                       DWARFUnit *U) const {
-  if (Error) {
+bool DWARFExpressionPrinter::printOp(const DWARFExpression::Operation *Op,
+                                     raw_ostream &OS, DIDumpOptions DumpOpts,
+                                     const DWARFExpression *Expr,
+                                     DWARFUnit *U) {
+  if (Op->Error) {
     OS << "<decoding error>";
     return false;
   }
 
-  StringRef Name = OperationEncodingString(Opcode);
+  StringRef Name = OperationEncodingString(Op->Opcode);
   assert(!Name.empty() && "DW_OP has no name!");
   OS << Name;
 
+#if 0//<<<<<<< HEAD
   std::optional<unsigned> SubOpcode = getSubCode();
   if (SubOpcode) {
     StringRef SubName = SubOperationEncodingString(Opcode, *SubOpcode);
@@ -375,63 +385,80 @@ bool DWARFExpression::Operation::print(raw_ostream &OS, DIDumpOptions DumpOpts,
       (SubOpcode && (*SubOpcode == DW_OP_LLVM_USER_call_frame_entry_reg ||
                      *SubOpcode == DW_OP_LLVM_USER_aspace_bregx)))
     if (prettyPrintRegisterOp(U, OS, DumpOpts, Opcode, Operands))
+#else//=======
+  if ((Op->Opcode >= DW_OP_breg0 && Op->Opcode <= DW_OP_breg31) ||
+      (Op->Opcode >= DW_OP_reg0 && Op->Opcode <= DW_OP_reg31) ||
+      Op->Opcode == DW_OP_bregx || Op->Opcode == DW_OP_regx ||
+      Op->Opcode == DW_OP_regval_type)
+    if (prettyPrintRegisterOp(U, OS, DumpOpts, Op->Opcode, Op->Operands))
+#endif//>>>>>>> 68bb005ae021cbfaef8fe9fa8b3efcdf0e13e2b3
       return true;
 
-  for (unsigned Operand = 0; Operand < Desc.Op.size(); ++Operand) {
-    unsigned Size = Desc.Op[Operand];
-    unsigned Signed = Size & Operation::SignBit;
+  for (unsigned Operand = 0; Operand < Op->Desc.Op.size(); ++Operand) {
+    unsigned Size = Op->Desc.Op[Operand];
+    unsigned Signed = Size & DWARFExpression::Operation::SignBit;
 
+#if 0//<<<<<<< HEAD
     if (Size == Operation::SizeSubOpLEB) {
       assert(Operand == 0);
       assert(SubOpcode);
     } else if (Size == Operation::BaseTypeRef && U) {
+#else//=======
+    if (Size == DWARFExpression::Operation::SizeSubOpLEB) {
+      StringRef SubName =
+          SubOperationEncodingString(Op->Opcode, Op->Operands[Operand]);
+      assert(!SubName.empty() && "DW_OP SubOp has no name!");
+      OS << " " << SubName;
+    } else if (Size == DWARFExpression::Operation::BaseTypeRef && U) {
+#endif//>>>>>>> 68bb005ae021cbfaef8fe9fa8b3efcdf0e13e2b3
       // For DW_OP_convert the operand may be 0 to indicate that conversion to
       // the generic type should be done. The same holds for DW_OP_reinterpret,
       // which is currently not supported.
-      if (Opcode == DW_OP_convert && Operands[Operand] == 0)
+      if (Op->Opcode == DW_OP_convert && Op->Operands[Operand] == 0)
         OS << " 0x0";
       else
-        prettyPrintBaseTypeRef(U, OS, DumpOpts, Operands, Operand);
-    } else if (Size == Operation::WasmLocationArg) {
+        prettyPrintBaseTypeRef(U, OS, DumpOpts, Op->Operands, Operand);
+    } else if (Size == DWARFExpression::Operation::WasmLocationArg) {
       assert(Operand == 1);
-      switch (Operands[0]) {
+      switch (Op->Operands[0]) {
       case 0:
       case 1:
       case 2:
       case 3: // global as uint32
       case 4:
-        OS << format(" 0x%" PRIx64, Operands[Operand]);
+        OS << format(" 0x%" PRIx64, Op->Operands[Operand]);
         break;
       default: assert(false);
       }
-    } else if (Size == Operation::SizeBlock) {
-      uint64_t Offset = Operands[Operand];
-      for (unsigned i = 0; i < Operands[Operand - 1]; ++i)
+    } else if (Size == DWARFExpression::Operation::SizeBlock) {
+      uint64_t Offset = Op->Operands[Operand];
+      for (unsigned i = 0; i < Op->Operands[Operand - 1]; ++i)
         OS << format(" 0x%02x", Expr->Data.getU8(&Offset));
     } else {
       if (Signed)
-        OS << format(" %+" PRId64, (int64_t)Operands[Operand]);
-      else if (Opcode != DW_OP_entry_value &&
-               Opcode != DW_OP_GNU_entry_value)
-        OS << format(" 0x%" PRIx64, Operands[Operand]);
+        OS << format(" %+" PRId64, (int64_t)Op->Operands[Operand]);
+      else if (Op->Opcode != DW_OP_entry_value &&
+               Op->Opcode != DW_OP_GNU_entry_value)
+        OS << format(" 0x%" PRIx64, Op->Operands[Operand]);
     }
   }
   return true;
 }
 
-void DWARFExpression::print(raw_ostream &OS, DIDumpOptions DumpOpts,
-                            DWARFUnit *U, bool IsEH) const {
+void DWARFExpressionPrinter::print(const DWARFExpression *E, raw_ostream &OS,
+                                   DIDumpOptions DumpOpts, DWARFUnit *U,
+                                   bool IsEH) {
   uint32_t EntryValExprSize = 0;
   uint64_t EntryValStartOffset = 0;
-  if (Data.getData().empty())
+  if (E->Data.getData().empty())
     OS << "<empty>";
 
-  for (auto &Op : *this) {
+  for (auto &Op : *E) {
     DumpOpts.IsEH = IsEH;
-    if (!Op.print(OS, DumpOpts, this, U)) {
+    if (!printOp(&Op, OS, DumpOpts, E, U)) {
       uint64_t FailOffset = Op.getEndOffset();
-      while (FailOffset < Data.getData().size())
-        OS << format(" %02x", Data.getU8(&FailOffset));
+      while (FailOffset < E->Data.getData().size())
+        OS << format(" %02x", E->Data.getU8(&FailOffset));
       return;
     }
 
@@ -449,39 +476,11 @@ void DWARFExpression::print(raw_ostream &OS, DIDumpOptions DumpOpts,
         OS << ")";
     }
 
-    if (Op.getEndOffset() < Data.getData().size())
+    if (Op.getEndOffset() < E->Data.getData().size())
       OS << ", ";
   }
 }
 
-bool DWARFExpression::Operation::verify(const Operation &Op, DWARFUnit *U) {
-  for (unsigned Operand = 0; Operand < Op.Desc.Op.size(); ++Operand) {
-    unsigned Size = Op.Desc.Op[Operand];
-
-    if (Size == Operation::BaseTypeRef) {
-      // For DW_OP_convert the operand may be 0 to indicate that conversion to
-      // the generic type should be done, so don't look up a base type in that
-      // case. The same holds for DW_OP_reinterpret, which is currently not
-      // supported.
-      if (Op.Opcode == DW_OP_convert && Op.Operands[Operand] == 0)
-        continue;
-      auto Die = U->getDIEForOffset(U->getOffset() + Op.Operands[Operand]);
-      if (!Die || Die.getTag() != dwarf::DW_TAG_base_type)
-        return false;
-    }
-  }
-
-  return true;
-}
-
-bool DWARFExpression::verify(DWARFUnit *U) {
-  for (auto &Op : *this)
-    if (!Operation::verify(Op, U))
-      return false;
-
-  return true;
-}
-
 /// A user-facing string representation of a DWARF expression. This might be an
 /// Address expression, in which case it will be implicitly dereferenced, or a
 /// Value expression.
@@ -604,16 +603,10 @@ static bool printCompactDWARFExpr(
   return true;
 }
 
-bool DWARFExpression::printCompact(
-    raw_ostream &OS,
+bool DWARFExpressionPrinter::printCompact(
+    const DWARFExpression *E, raw_ostream &OS,
     std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg) {
-  return printCompactDWARFExpr(OS, begin(), end(), GetNameForDWARFReg);
-}
-
-bool DWARFExpression::operator==(const DWARFExpression &RHS) const {
-  if (AddressSize != RHS.AddressSize || Format != RHS.Format)
-    return false;
-  return Data.getData() == RHS.Data.getData();
+  return printCompactDWARFExpr(OS, E->begin(), E->end(), GetNameForDWARFReg);
 }
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
index 43e82aee0784..d719a47c8407 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -496,111 +496,107 @@ void DWARFUnit::extractDIEsIfNeeded(bool CUDieOnly) {
 }
 
 Error DWARFUnit::tryExtractDIEsIfNeeded(bool CUDieOnly) {
-  return Context.doWorkThreadSafely([&]() -> Error {
-    if ((CUDieOnly && !DieArray.empty()) || DieArray.size() > 1)
-      return Error::success(); // Already parsed.
-
-    bool HasCUDie = !DieArray.empty();
-    extractDIEsToVector(!HasCUDie, !CUDieOnly, DieArray);
-
-    if (DieArray.empty())
-      return Error::success();
-
-    // If CU DIE was just parsed, copy several attribute values from it.
-    if (HasCUDie)
-      return Error::success();
-
-    DWARFDie UnitDie(this, &DieArray[0]);
-    if (std::optional<uint64_t> DWOId =
-            toUnsigned(UnitDie.find(DW_AT_GNU_dwo_id)))
-      Header.setDWOId(*DWOId);
-    if (!IsDWO) {
-      assert(AddrOffsetSectionBase == std::nullopt);
-      assert(RangeSectionBase == 0);
-      assert(LocSectionBase == 0);
-      AddrOffsetSectionBase = toSectionOffset(UnitDie.find(DW_AT_addr_base));
-      if (!AddrOffsetSectionBase)
-        AddrOffsetSectionBase =
-            toSectionOffset(UnitDie.find(DW_AT_GNU_addr_base));
-      RangeSectionBase = toSectionOffset(UnitDie.find(DW_AT_rnglists_base), 0);
-      LocSectionBase = toSectionOffset(UnitDie.find(DW_AT_loclists_base), 0);
-    }
+  if ((CUDieOnly && !DieArray.empty()) || DieArray.size() > 1)
+    return Error::success(); // Already parsed.
 
-    // In general, in DWARF v5 and beyond we derive the start of the unit's
-    // contribution to the string offsets table from the unit DIE's
-    // DW_AT_str_offsets_base attribute. Split DWARF units do not use this
-    // attribute, so we assume that there is a contribution to the string
-    // offsets table starting at offset 0 of the debug_str_offsets.dwo section.
-    // In both cases we need to determine the format of the contribution,
-    // which may differ from the unit's format.
-    DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection,
-                          IsLittleEndian, 0);
-    if (IsDWO || getVersion() >= 5) {
-      auto StringOffsetOrError =
-          IsDWO ? determineStringOffsetsTableContributionDWO(DA)
-                : determineStringOffsetsTableContribution(DA);
-      if (!StringOffsetOrError) {
-        return createStringError(errc::invalid_argument,
-                                 "invalid reference to or invalid content in "
-                                 ".debug_str_offsets[.dwo]: " +
-                                     toString(StringOffsetOrError.takeError()));
-      }
+  bool HasCUDie = !DieArray.empty();
+  extractDIEsToVector(!HasCUDie, !CUDieOnly, DieArray);
 
-      StringOffsetsTableContribution = *StringOffsetOrError;
-    }
+  if (DieArray.empty())
+    return Error::success();
 
-    // DWARF v5 uses the .debug_rnglists and .debug_rnglists.dwo sections to
-    // describe address ranges.
-    if (getVersion() >= 5) {
-      // In case of DWP, the base offset from the index has to be added.
-      if (IsDWO) {
-        uint64_t ContributionBaseOffset = 0;
-        if (auto *IndexEntry = Header.getIndexEntry())
-          if (auto *Contrib = IndexEntry->getContribution(DW_SECT_RNGLISTS))
-            ContributionBaseOffset = Contrib->getOffset();
-        setRangesSection(
-            &Context.getDWARFObj().getRnglistsDWOSection(),
-            ContributionBaseOffset +
-                DWARFListTableHeader::getHeaderSize(Header.getFormat()));
-      } else
-        setRangesSection(&Context.getDWARFObj().getRnglistsSection(),
-                         toSectionOffset(UnitDie.find(DW_AT_rnglists_base),
-                                         DWARFListTableHeader::getHeaderSize(
-                                             Header.getFormat())));
-    }
+  // If CU DIE was just parsed, copy several attribute values from it.
+  if (HasCUDie)
+    return Error::success();
+
+  DWARFDie UnitDie(this, &DieArray[0]);
+  if (std::optional<uint64_t> DWOId =
+          toUnsigned(UnitDie.find(DW_AT_GNU_dwo_id)))
+    Header.setDWOId(*DWOId);
+  if (!IsDWO) {
+    assert(AddrOffsetSectionBase == std::nullopt);
+    assert(RangeSectionBase == 0);
+    assert(LocSectionBase == 0);
+    AddrOffsetSectionBase = toSectionOffset(UnitDie.find(DW_AT_addr_base));
+    if (!AddrOffsetSectionBase)
+      AddrOffsetSectionBase =
+          toSectionOffset(UnitDie.find(DW_AT_GNU_addr_base));
+    RangeSectionBase = toSectionOffset(UnitDie.find(DW_AT_rnglists_base), 0);
+    LocSectionBase = toSectionOffset(UnitDie.find(DW_AT_loclists_base), 0);
+  }
+
+  // In general, in DWARF v5 and beyond we derive the start of the unit's
+  // contribution to the string offsets table from the unit DIE's
+  // DW_AT_str_offsets_base attribute. Split DWARF units do not use this
+  // attribute, so we assume that there is a contribution to the string
+  // offsets table starting at offset 0 of the debug_str_offsets.dwo section.
+  // In both cases we need to determine the format of the contribution,
+  // which may differ from the unit's format.
+  DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection,
+                        IsLittleEndian, 0);
+  if (IsDWO || getVersion() >= 5) {
+    auto StringOffsetOrError =
+        IsDWO ? determineStringOffsetsTableContributionDWO(DA)
+              : determineStringOffsetsTableContribution(DA);
+    if (!StringOffsetOrError)
+      return createStringError(errc::invalid_argument,
+                               "invalid reference to or invalid content in "
+                               ".debug_str_offsets[.dwo]: " +
+                                   toString(StringOffsetOrError.takeError()));
+
+    StringOffsetsTableContribution = *StringOffsetOrError;
+  }
 
+  // DWARF v5 uses the .debug_rnglists and .debug_rnglists.dwo sections to
+  // describe address ranges.
+  if (getVersion() >= 5) {
+    // In case of DWP, the base offset from the index has to be added.
     if (IsDWO) {
-      // If we are reading a package file, we need to adjust the location list
-      // data based on the index entries.
-      StringRef Data = Header.getVersion() >= 5
-                           ? Context.getDWARFObj().getLoclistsDWOSection().Data
-                           : Context.getDWARFObj().getLocDWOSection().Data;
+      uint64_t ContributionBaseOffset = 0;
       if (auto *IndexEntry = Header.getIndexEntry())
-        if (const auto *C = IndexEntry->getContribution(
-                Header.getVersion() >= 5 ? DW_SECT_LOCLISTS : DW_SECT_EXT_LOC))
-          Data = Data.substr(C->getOffset(), C->getLength());
-
-      DWARFDataExtractor DWARFData(Data, IsLittleEndian, getAddressByteSize());
-      LocTable =
-          std::make_unique<DWARFDebugLoclists>(DWARFData, Header.getVersion());
-      LocSectionBase = DWARFListTableHeader::getHeaderSize(Header.getFormat());
-    } else if (getVersion() >= 5) {
-      LocTable = std::make_unique<DWARFDebugLoclists>(
-          DWARFDataExtractor(Context.getDWARFObj(),
-                             Context.getDWARFObj().getLoclistsSection(),
-                             IsLittleEndian, getAddressByteSize()),
-          getVersion());
-    } else {
-      LocTable = std::make_unique<DWARFDebugLoc>(DWARFDataExtractor(
-          Context.getDWARFObj(), Context.getDWARFObj().getLocSection(),
-          IsLittleEndian, getAddressByteSize()));
-    }
+        if (auto *Contrib = IndexEntry->getContribution(DW_SECT_RNGLISTS))
+          ContributionBaseOffset = Contrib->getOffset();
+      setRangesSection(
+          &Context.getDWARFObj().getRnglistsDWOSection(),
+          ContributionBaseOffset +
+              DWARFListTableHeader::getHeaderSize(Header.getFormat()));
+    } else
+      setRangesSection(&Context.getDWARFObj().getRnglistsSection(),
+                       toSectionOffset(UnitDie.find(DW_AT_rnglists_base),
+                                       DWARFListTableHeader::getHeaderSize(
+                                           Header.getFormat())));
+  }
 
-    // Don't fall back to DW_AT_GNU_ranges_base: it should be ignored for
-    // skeleton CU DIE, so that DWARF users not aware of it are not broken.
+  if (IsDWO) {
+    // If we are reading a package file, we need to adjust the location list
+    // data based on the index entries.
+    StringRef Data = Header.getVersion() >= 5
+                         ? Context.getDWARFObj().getLoclistsDWOSection().Data
+                         : Context.getDWARFObj().getLocDWOSection().Data;
+    if (auto *IndexEntry = Header.getIndexEntry())
+      if (const auto *C = IndexEntry->getContribution(
+              Header.getVersion() >= 5 ? DW_SECT_LOCLISTS : DW_SECT_EXT_LOC))
+        Data = Data.substr(C->getOffset(), C->getLength());
+
+    DWARFDataExtractor DWARFData(Data, IsLittleEndian, getAddressByteSize());
+    LocTable =
+        std::make_unique<DWARFDebugLoclists>(DWARFData, Header.getVersion());
+    LocSectionBase = DWARFListTableHeader::getHeaderSize(Header.getFormat());
+  } else if (getVersion() >= 5) {
+    LocTable = std::make_unique<DWARFDebugLoclists>(
+        DWARFDataExtractor(Context.getDWARFObj(),
+                           Context.getDWARFObj().getLoclistsSection(),
+                           IsLittleEndian, getAddressByteSize()),
+        getVersion());
+  } else {
+    LocTable = std::make_unique<DWARFDebugLoc>(DWARFDataExtractor(
+        Context.getDWARFObj(), Context.getDWARFObj().getLocSection(),
+        IsLittleEndian, getAddressByteSize()));
+  }
 
-    return Error::success();
-  });
+  // Don't fall back to DW_AT_GNU_ranges_base: it should be ignored for
+  // skeleton CU DIE, so that DWARF users not aware of it are not broken.
+  return Error::success();
 }
 
 bool DWARFUnit::parseDWO(StringRef DWOAlternativeLocation) {
@@ -655,26 +651,15 @@ bool DWARFUnit::parseDWO(StringRef DWOAlternativeLocation) {
   return true;
 }
 
-void DWARFUnit::clearDIEs(bool KeepCUDie, bool KeepDWODies) {
-  cantFail(Context.doWorkThreadSafely([&] {
-    if (!KeepDWODies && DWO) {
-      DWO->clearDIEs(KeepCUDie, KeepDWODies);
-    }
-    if (!IsDWO) {
-      RangeSectionBase = 0;
-      LocSectionBase = 0;
-      AddrOffsetSectionBase = std::nullopt;
-    }
-    // Do not use resize() + shrink_to_fit() to free memory occupied by dies.
-    // shrink_to_fit() is a *non-binding* request to reduce capacity() to
-    // size(). It depends on the implementation whether the request is
-    // fulfilled. Create a new vector with a small capacity and assign it to the
-    // DieArray to have previous contents freed.
-    DieArray = (KeepCUDie && !DieArray.empty())
-                   ? std::vector<DWARFDebugInfoEntry>({DieArray[0]})
-                   : std::vector<DWARFDebugInfoEntry>();
-    return Error::success();
-  }));
+void DWARFUnit::clearDIEs(bool KeepCUDie) {
+  // Do not use resize() + shrink_to_fit() to free memory occupied by dies.
+  // shrink_to_fit() is a *non-binding* request to reduce capacity() to size().
+  // It depends on the implementation whether the request is fulfilled.
+  // Create a new vector with a small capacity and assign it to the DieArray to
+  // have previous contents freed.
+  DieArray = (KeepCUDie && !DieArray.empty())
+                 ? std::vector<DWARFDebugInfoEntry>({DieArray[0]})
+                 : std::vector<DWARFDebugInfoEntry>();
 }
 
 Expected<DWARFAddressRangesVector>
diff --git a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
index 43a62bdd8390..c12786cac868 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
@@ -659,6 +659,35 @@ unsigned DWARFVerifier::verifyDieRanges(const DWARFDie &Die,
   return NumErrors;
 }
 
+bool DWARFVerifier::verifyExpressionOp(const DWARFExpression::Operation &Op,
+                                       DWARFUnit *U) {
+  for (unsigned Operand = 0; Operand < Op.Desc.Op.size(); ++Operand) {
+    unsigned Size = Op.Desc.Op[Operand];
+
+    if (Size == DWARFExpression::Operation::BaseTypeRef) {
+      // For DW_OP_convert the operand may be 0 to indicate that conversion to
+      // the generic type should be done, so don't look up a base type in that
+      // case. The same holds for DW_OP_reinterpret, which is currently not
+      // supported.
+      if (Op.Opcode == DW_OP_convert && Op.Operands[Operand] == 0)
+        continue;
+      auto Die = U->getDIEForOffset(U->getOffset() + Op.Operands[Operand]);
+      if (!Die || Die.getTag() != dwarf::DW_TAG_base_type)
+        return false;
+    }
+  }
+
+  return true;
+}
+
+bool DWARFVerifier::verifyExpression(const DWARFExpression &E, DWARFUnit *U) {
+  for (auto &Op : E)
+    if (!verifyExpressionOp(Op, U))
+      return false;
+
+  return true;
+}
+
 unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die,
                                                  DWARFAttribute &AttrValue) {
   unsigned NumErrors = 0;
@@ -727,7 +756,7 @@ unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die,
             any_of(Expression, [](const DWARFExpression::Operation &Op) {
               return Op.isError();
             });
-        if (Error || !Expression.verify(U))
+        if (Error || !verifyExpression(Expression, U))
           ReportError("Invalid DWARF expressions",
                       "DIE contains invalid DWARF expression:");
       }
diff --git a/external/llvm-project/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp b/external/llvm-project/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp
index c112c0bc3ddc..8cd4e22d34fc 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/GSYM/CallSiteInfo.h"
-#include "llvm/ADT/CachedHashString.h"
 #include "llvm/DebugInfo/GSYM/FileWriter.h"
 #include "llvm/DebugInfo/GSYM/FunctionInfo.h"
 #include "llvm/DebugInfo/GSYM/GsymCreator.h"
@@ -16,9 +15,7 @@
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
-#include <fstream>
 #include <string>
-#include <unordered_map>
 #include <vector>
 
 using namespace llvm;
diff --git a/external/llvm-project/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp b/external/llvm-project/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
index 1f70d273a9d9..7a0256f10ea6 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
@@ -656,11 +656,6 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
       DWARFDie Die = getDie(*CU);
       CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
       handleDie(Out, CUI, Die);
-      // Release the line table, once we're done.
-      DICtx.clearLineTableForUnit(CU.get());
-      // Free any DIEs that were allocated by the DWARF parser.
-      // If/when they're needed by other CU's, they'll be recreated.
-      CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
     }
   } else {
     // LLVM Dwarf parser is not thread-safe and we need to parse all DWARF up
@@ -673,7 +668,12 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
     for (const auto &CU : DICtx.compile_units())
       CU->getAbbreviations();
 
+    // Now parse all DIEs in case we have cross compile unit references in a
+    // thread pool.
     DefaultThreadPool pool(hardware_concurrency(NumThreads));
+    for (const auto &CU : DICtx.compile_units())
+      pool.async([&CU]() { CU->getUnitDIE(false /*CUDieOnly*/); });
+    pool.wait();
 
     // Now convert all DWARF to GSYM in a thread pool.
     std::mutex LogMutex;
@@ -681,15 +681,11 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
       DWARFDie Die = getDie(*CU);
       if (Die) {
         CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
-        pool.async([this, CUI, &CU, &LogMutex, &Out, Die]() mutable {
+        pool.async([this, CUI, &LogMutex, &Out, Die]() mutable {
           std::string storage;
           raw_string_ostream StrStream(storage);
           OutputAggregator ThreadOut(Out.GetOS() ? &StrStream : nullptr);
           handleDie(ThreadOut, CUI, Die);
-          DICtx.clearLineTableForUnit(CU.get());
-          // Free any DIEs that were allocated by the DWARF parser.
-          // If/when they're needed by other CU's, they'll be recreated.
-          CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
           // Print ThreadLogStorage lines into an actual stream under a lock
           std::lock_guard<std::mutex> guard(LogMutex);
           if (Out.GetOS()) {
@@ -701,9 +697,6 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
     }
     pool.wait();
   }
-  // Now get rid of all the DIEs that may have been recreated
-  for (const auto &CU : DICtx.compile_units())
-    CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
   size_t FunctionsAddedCount = Gsym.getNumFunctionInfos() - NumBefore;
   Out << "Loaded " << FunctionsAddedCount << " functions from DWARF.\n";
   return Error::success();
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt
index f5adab723927..67ee7698a382 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt
@@ -14,6 +14,7 @@ add_lv_impl_folder(Core
   Core/LVReader.cpp
   Core/LVScope.cpp
   Core/LVSort.cpp
+  Core/LVSourceLanguage.cpp
   Core/LVSupport.cpp
   Core/LVSymbol.cpp
   Core/LVType.cpp
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVObject.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVObject.cpp
index 75acbf3225e0..5ccbcbfa4f0a 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVObject.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVObject.cpp
@@ -21,9 +21,7 @@ using namespace llvm::logicalview;
 
 #define DEBUG_TYPE "Object"
 
-#ifndef NDEBUG
-uint64_t LVObject::GID = 0;
-#endif
+uint32_t LVObject::GID = 0;
 
 StringRef llvm::logicalview::typeNone() { return StringRef(); }
 StringRef llvm::logicalview::typeVoid() { return "void"; }
@@ -137,10 +135,8 @@ void LVObject::printAttributes(raw_ostream &OS, bool Full, StringRef Name,
 }
 
 void LVObject::printAttributes(raw_ostream &OS, bool Full) const {
-#ifndef NDEBUG
   if (options().getInternalID())
     OS << hexSquareString(getID());
-#endif
   if (options().getCompareExecute() &&
       (options().getAttributeAdded() || options().getAttributeMissing()))
     OS << (getIsAdded() ? '+' : getIsMissing() ? '-' : ' ');
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp
index c08e887fde3e..af35e58ac0dd 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp
@@ -39,6 +39,7 @@ void LVOptions::resolveDependencies() {
     setAttributeFilename();
     setAttributeFiles();
     setAttributeFormat();
+    setAttributeLanguage();
     setAttributeLevel();
     setAttributeProducer();
     setAttributePublics();
@@ -258,12 +259,10 @@ void LVOptions::resolveDependencies() {
 }
 
 void LVOptions::calculateIndentationSize() {
-#ifndef NDEBUG
   if (getInternalID()) {
     std::string String = hexSquareString(0);
     IndentationSize += String.length();
   }
-#endif
   if (getCompareExecute() && (getAttributeAdded() || getAttributeMissing()))
     ++IndentationSize;
   if (getAttributeOffset()) {
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVReader.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVReader.cpp
index 328ced9f4eb6..c1017d8a3c22 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVReader.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVReader.cpp
@@ -194,6 +194,253 @@ StringRef LVReader::getFilename(LVObject *Object, size_t Index) const {
   return CompileUnit ? CompileUnit->getFilename(Index) : StringRef();
 }
 
+void LVReader::addSectionRange(LVSectionIndex SectionIndex, LVScope *Scope) {
+  LVRange *ScopesWithRanges = getSectionRanges(SectionIndex);
+  ScopesWithRanges->addEntry(Scope);
+}
+
+void LVReader::addSectionRange(LVSectionIndex SectionIndex, LVScope *Scope,
+                               LVAddress LowerAddress, LVAddress UpperAddress) {
+  LVRange *ScopesWithRanges = getSectionRanges(SectionIndex);
+  ScopesWithRanges->addEntry(Scope, LowerAddress, UpperAddress);
+}
+
+LVRange *LVReader::getSectionRanges(LVSectionIndex SectionIndex) {
+  // Check if we already have a mapping for this section index.
+  LVSectionRanges::iterator IterSection = SectionRanges.find(SectionIndex);
+  if (IterSection == SectionRanges.end())
+    IterSection =
+        SectionRanges.emplace(SectionIndex, std::make_unique<LVRange>()).first;
+  LVRange *Range = IterSection->second.get();
+  assert(Range && "Range is null.");
+  return Range;
+}
+
+LVElement *LVReader::createElement(dwarf::Tag Tag) {
+  CurrentScope = nullptr;
+  CurrentSymbol = nullptr;
+  CurrentType = nullptr;
+  CurrentRanges.clear();
+
+  LLVM_DEBUG(
+      { dbgs() << "\n[createElement] " << dwarf::TagString(Tag) << "\n"; });
+
+  if (!options().getPrintSymbols()) {
+    switch (Tag) {
+    // As the command line options did not specify a request to print
+    // logical symbols (--print=symbols or --print=all or --print=elements),
+    // skip its creation.
+    case dwarf::DW_TAG_formal_parameter:
+    case dwarf::DW_TAG_unspecified_parameters:
+    case dwarf::DW_TAG_member:
+    case dwarf::DW_TAG_variable:
+    case dwarf::DW_TAG_inheritance:
+    case dwarf::DW_TAG_constant:
+    case dwarf::DW_TAG_call_site_parameter:
+    case dwarf::DW_TAG_GNU_call_site_parameter:
+      return nullptr;
+    default:
+      break;
+    }
+  }
+
+  switch (Tag) {
+  // Types.
+  case dwarf::DW_TAG_base_type:
+    CurrentType = createType();
+    CurrentType->setIsBase();
+    if (options().getAttributeBase())
+      CurrentType->setIncludeInPrint();
+    return CurrentType;
+  case dwarf::DW_TAG_const_type:
+    CurrentType = createType();
+    CurrentType->setIsConst();
+    CurrentType->setName("const");
+    return CurrentType;
+  case dwarf::DW_TAG_enumerator:
+    CurrentType = createTypeEnumerator();
+    return CurrentType;
+  case dwarf::DW_TAG_imported_declaration:
+    CurrentType = createTypeImport();
+    CurrentType->setIsImportDeclaration();
+    return CurrentType;
+  case dwarf::DW_TAG_imported_module:
+    CurrentType = createTypeImport();
+    CurrentType->setIsImportModule();
+    return CurrentType;
+  case dwarf::DW_TAG_pointer_type:
+    CurrentType = createType();
+    CurrentType->setIsPointer();
+    CurrentType->setName("*");
+    return CurrentType;
+  case dwarf::DW_TAG_ptr_to_member_type:
+    CurrentType = createType();
+    CurrentType->setIsPointerMember();
+    CurrentType->setName("*");
+    return CurrentType;
+  case dwarf::DW_TAG_reference_type:
+    CurrentType = createType();
+    CurrentType->setIsReference();
+    CurrentType->setName("&");
+    return CurrentType;
+  case dwarf::DW_TAG_restrict_type:
+    CurrentType = createType();
+    CurrentType->setIsRestrict();
+    CurrentType->setName("restrict");
+    return CurrentType;
+  case dwarf::DW_TAG_rvalue_reference_type:
+    CurrentType = createType();
+    CurrentType->setIsRvalueReference();
+    CurrentType->setName("&&");
+    return CurrentType;
+  case dwarf::DW_TAG_subrange_type:
+    CurrentType = createTypeSubrange();
+    return CurrentType;
+  case dwarf::DW_TAG_template_value_parameter:
+    CurrentType = createTypeParam();
+    CurrentType->setIsTemplateValueParam();
+    return CurrentType;
+  case dwarf::DW_TAG_template_type_parameter:
+    CurrentType = createTypeParam();
+    CurrentType->setIsTemplateTypeParam();
+    return CurrentType;
+  case dwarf::DW_TAG_GNU_template_template_param:
+    CurrentType = createTypeParam();
+    CurrentType->setIsTemplateTemplateParam();
+    return CurrentType;
+  case dwarf::DW_TAG_typedef:
+    CurrentType = createTypeDefinition();
+    return CurrentType;
+  case dwarf::DW_TAG_unspecified_type:
+    CurrentType = createType();
+    CurrentType->setIsUnspecified();
+    return CurrentType;
+  case dwarf::DW_TAG_volatile_type:
+    CurrentType = createType();
+    CurrentType->setIsVolatile();
+    CurrentType->setName("volatile");
+    return CurrentType;
+
+  // Symbols.
+  case dwarf::DW_TAG_formal_parameter:
+    CurrentSymbol = createSymbol();
+    CurrentSymbol->setIsParameter();
+    return CurrentSymbol;
+  case dwarf::DW_TAG_unspecified_parameters:
+    CurrentSymbol = createSymbol();
+    CurrentSymbol->setIsUnspecified();
+    CurrentSymbol->setName("...");
+    return CurrentSymbol;
+  case dwarf::DW_TAG_member:
+    CurrentSymbol = createSymbol();
+    CurrentSymbol->setIsMember();
+    return CurrentSymbol;
+  case dwarf::DW_TAG_variable:
+    CurrentSymbol = createSymbol();
+    CurrentSymbol->setIsVariable();
+    return CurrentSymbol;
+  case dwarf::DW_TAG_inheritance:
+    CurrentSymbol = createSymbol();
+    CurrentSymbol->setIsInheritance();
+    return CurrentSymbol;
+  case dwarf::DW_TAG_call_site_parameter:
+  case dwarf::DW_TAG_GNU_call_site_parameter:
+    CurrentSymbol = createSymbol();
+    CurrentSymbol->setIsCallSiteParameter();
+    return CurrentSymbol;
+  case dwarf::DW_TAG_constant:
+    CurrentSymbol = createSymbol();
+    CurrentSymbol->setIsConstant();
+    return CurrentSymbol;
+
+  // Scopes.
+  case dwarf::DW_TAG_catch_block:
+    CurrentScope = createScope();
+    CurrentScope->setIsCatchBlock();
+    return CurrentScope;
+  case dwarf::DW_TAG_lexical_block:
+    CurrentScope = createScope();
+    CurrentScope->setIsLexicalBlock();
+    return CurrentScope;
+  case dwarf::DW_TAG_try_block:
+    CurrentScope = createScope();
+    CurrentScope->setIsTryBlock();
+    return CurrentScope;
+  case dwarf::DW_TAG_compile_unit:
+  case dwarf::DW_TAG_skeleton_unit:
+    CurrentScope = createScopeCompileUnit();
+    CompileUnit = static_cast<LVScopeCompileUnit *>(CurrentScope);
+    return CurrentScope;
+  case dwarf::DW_TAG_inlined_subroutine:
+    CurrentScope = createScopeFunctionInlined();
+    return CurrentScope;
+  case dwarf::DW_TAG_namespace:
+    CurrentScope = createScopeNamespace();
+    return CurrentScope;
+  case dwarf::DW_TAG_template_alias:
+    CurrentScope = createScopeAlias();
+    return CurrentScope;
+  case dwarf::DW_TAG_array_type:
+    CurrentScope = createScopeArray();
+    return CurrentScope;
+  case dwarf::DW_TAG_call_site:
+  case dwarf::DW_TAG_GNU_call_site:
+    CurrentScope = createScopeFunction();
+    CurrentScope->setIsCallSite();
+    return CurrentScope;
+  case dwarf::DW_TAG_entry_point:
+    CurrentScope = createScopeFunction();
+    CurrentScope->setIsEntryPoint();
+    return CurrentScope;
+  case dwarf::DW_TAG_subprogram:
+    CurrentScope = createScopeFunction();
+    CurrentScope->setIsSubprogram();
+    return CurrentScope;
+  case dwarf::DW_TAG_subroutine_type:
+    CurrentScope = createScopeFunctionType();
+    return CurrentScope;
+  case dwarf::DW_TAG_label:
+    CurrentScope = createScopeFunction();
+    CurrentScope->setIsLabel();
+    return CurrentScope;
+  case dwarf::DW_TAG_class_type:
+    CurrentScope = createScopeAggregate();
+    CurrentScope->setIsClass();
+    return CurrentScope;
+  case dwarf::DW_TAG_structure_type:
+    CurrentScope = createScopeAggregate();
+    CurrentScope->setIsStructure();
+    return CurrentScope;
+  case dwarf::DW_TAG_union_type:
+    CurrentScope = createScopeAggregate();
+    CurrentScope->setIsUnion();
+    return CurrentScope;
+  case dwarf::DW_TAG_enumeration_type:
+    CurrentScope = createScopeEnumeration();
+    return CurrentScope;
+  case dwarf::DW_TAG_GNU_formal_parameter_pack:
+    CurrentScope = createScopeFormalPack();
+    return CurrentScope;
+  case dwarf::DW_TAG_GNU_template_parameter_pack:
+    CurrentScope = createScopeTemplatePack();
+    return CurrentScope;
+  case dwarf::DW_TAG_module:
+    CurrentScope = createScopeModule();
+    return CurrentScope;
+  default:
+    // Collect TAGs not implemented.
+    if (options().getInternalTag() && Tag)
+      CompileUnit->addDebugTag(Tag, CurrentOffset);
+    break;
+  }
+
+  LLVM_DEBUG({
+    dbgs() << "DWARF Tag not implemented: " << dwarf::TagString(Tag) << "\n";
+  });
+
+  return nullptr;
+}
+
 // The Reader is the module that creates the logical view using the debug
 // information contained in the binary file specified in the command line.
 // This is the main entry point for the Reader and performs the following
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
index 06a11635a629..55880fab5e88 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
@@ -1717,11 +1717,19 @@ void LVScopeCompileUnit::print(raw_ostream &OS, bool Full) const {
 
 void LVScopeCompileUnit::printExtra(raw_ostream &OS, bool Full) const {
   OS << formattedKind(kind()) << " '" << getName() << "'\n";
-  if (options().getPrintFormatting() && options().getAttributeProducer())
-    printAttributes(OS, Full, "{Producer} ",
-                    const_cast<LVScopeCompileUnit *>(this), getProducer(),
-                    /*UseQuotes=*/true,
-                    /*PrintRef=*/false);
+  if (options().getPrintFormatting()) {
+    if (options().getAttributeProducer())
+      printAttributes(OS, Full, "{Producer} ",
+                      const_cast<LVScopeCompileUnit *>(this), getProducer(),
+                      /*UseQuotes=*/true,
+                      /*PrintRef=*/false);
+    if (options().getAttributeLanguage())
+      if (auto SL = getSourceLanguage(); SL.isValid())
+        printAttributes(OS, Full, "{Language} ",
+                        const_cast<LVScopeCompileUnit *>(this), SL.getName(),
+                        /*UseQuotes=*/true,
+                        /*PrintRef=*/false);
+  }
 
   // Reset file index, to allow its children to print the correct filename.
   options().resetFilenameIndex();
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVSourceLanguage.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVSourceLanguage.cpp
new file mode 100644
index 000000000000..e7b122de1bba
--- /dev/null
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Core/LVSourceLanguage.cpp
@@ -0,0 +1,33 @@
+//===-- LVSourceLanguage.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements LVSourceLanguage.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h"
+#include "llvm/DebugInfo/CodeView/EnumTables.h"
+#include "llvm/Support/ScopedPrinter.h"
+
+using namespace llvm;
+using namespace llvm::logicalview;
+
+StringRef LVSourceLanguage::getName() const {
+  if (!isValid())
+    return {};
+  switch (getTag()) {
+  case LVSourceLanguage::TagDwarf:
+    return llvm::dwarf::LanguageString(getLang());
+  case LVSourceLanguage::TagCodeView: {
+    static auto LangNames = llvm::codeview::getSourceLanguageNames();
+    return LangNames[getLang()].Name;
+  }
+  default:
+    llvm_unreachable("Unsupported language");
+  }
+}
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
index 434709f076dc..80b4185b7c60 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
@@ -278,7 +278,7 @@ Error LVBinaryReader::loadGenericTargetInfo(StringRef TheTriple,
                                             StringRef TheFeatures) {
   std::string TargetLookupError;
   const Target *TheTarget =
-      TargetRegistry::lookupTarget(std::string(TheTriple), TargetLookupError);
+      TargetRegistry::lookupTarget(TheTriple, TargetLookupError);
   if (!TheTarget)
     return createStringError(errc::invalid_argument, TargetLookupError.c_str());
 
@@ -367,30 +367,6 @@ LVBinaryReader::getSection(LVScope *Scope, LVAddress Address,
   return std::make_pair(Iter->first, Iter->second);
 }
 
-void LVBinaryReader::addSectionRange(LVSectionIndex SectionIndex,
-                                     LVScope *Scope) {
-  LVRange *ScopesWithRanges = getSectionRanges(SectionIndex);
-  ScopesWithRanges->addEntry(Scope);
-}
-
-void LVBinaryReader::addSectionRange(LVSectionIndex SectionIndex,
-                                     LVScope *Scope, LVAddress LowerAddress,
-                                     LVAddress UpperAddress) {
-  LVRange *ScopesWithRanges = getSectionRanges(SectionIndex);
-  ScopesWithRanges->addEntry(Scope, LowerAddress, UpperAddress);
-}
-
-LVRange *LVBinaryReader::getSectionRanges(LVSectionIndex SectionIndex) {
-  // Check if we already have a mapping for this section index.
-  LVSectionRanges::iterator IterSection = SectionRanges.find(SectionIndex);
-  if (IterSection == SectionRanges.end())
-    IterSection =
-        SectionRanges.emplace(SectionIndex, std::make_unique<LVRange>()).first;
-  LVRange *Range = IterSection->second.get();
-  assert(Range && "Range is null.");
-  return Range;
-}
-
 Error LVBinaryReader::createInstructions(LVScope *Scope,
                                          LVSectionIndex SectionIndex,
                                          const LVNameInfo &NameInfo) {
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
index 4a1eb888095e..24eaa1234727 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
@@ -949,6 +949,9 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record,
     Scope->setName(CurrentObjectName);
     if (options().getAttributeProducer())
       Scope->setProducer(Compile2.Version);
+    if (options().getAttributeLanguage())
+      Scope->setSourceLanguage(LVSourceLanguage{
+          static_cast<llvm::codeview::SourceLanguage>(Compile2.getLanguage())});
     getReader().isSystemEntry(Scope, CurrentObjectName);
 
     // The line records in CodeView are recorded per Module ID. Update
@@ -994,6 +997,9 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record,
     Scope->setName(CurrentObjectName);
     if (options().getAttributeProducer())
       Scope->setProducer(Compile3.Version);
+    if (options().getAttributeLanguage())
+      Scope->setSourceLanguage(LVSourceLanguage{
+          static_cast<llvm::codeview::SourceLanguage>(Compile3.getLanguage())});
     getReader().isSystemEntry(Scope, CurrentObjectName);
 
     // The line records in CodeView are recorded per Module ID. Update
diff --git a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
index 14538c1aefa1..7ff96ae90a7f 100644
--- a/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
+++ b/external/llvm-project/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
@@ -28,223 +28,6 @@ using namespace llvm::logicalview;
 
 #define DEBUG_TYPE "DWARFReader"
 
-LVElement *LVDWARFReader::createElement(dwarf::Tag Tag) {
-  CurrentScope = nullptr;
-  CurrentSymbol = nullptr;
-  CurrentType = nullptr;
-  CurrentRanges.clear();
-
-  if (!options().getPrintSymbols()) {
-    switch (Tag) {
-    // As the command line options did not specify a request to print
-    // logical symbols (--print=symbols or --print=all or --print=elements),
-    // skip its creation.
-    case dwarf::DW_TAG_formal_parameter:
-    case dwarf::DW_TAG_unspecified_parameters:
-    case dwarf::DW_TAG_member:
-    case dwarf::DW_TAG_variable:
-    case dwarf::DW_TAG_inheritance:
-    case dwarf::DW_TAG_constant:
-    case dwarf::DW_TAG_call_site_parameter:
-    case dwarf::DW_TAG_GNU_call_site_parameter:
-      return nullptr;
-    default:
-      break;
-    }
-  }
-
-  switch (Tag) {
-  // Types.
-  case dwarf::DW_TAG_base_type:
-    CurrentType = createType();
-    CurrentType->setIsBase();
-    if (options().getAttributeBase())
-      CurrentType->setIncludeInPrint();
-    return CurrentType;
-  case dwarf::DW_TAG_const_type:
-    CurrentType = createType();
-    CurrentType->setIsConst();
-    CurrentType->setName("const");
-    return CurrentType;
-  case dwarf::DW_TAG_enumerator:
-    CurrentType = createTypeEnumerator();
-    return CurrentType;
-  case dwarf::DW_TAG_imported_declaration:
-    CurrentType = createTypeImport();
-    CurrentType->setIsImportDeclaration();
-    return CurrentType;
-  case dwarf::DW_TAG_imported_module:
-    CurrentType = createTypeImport();
-    CurrentType->setIsImportModule();
-    return CurrentType;
-  case dwarf::DW_TAG_pointer_type:
-    CurrentType = createType();
-    CurrentType->setIsPointer();
-    CurrentType->setName("*");
-    return CurrentType;
-  case dwarf::DW_TAG_ptr_to_member_type:
-    CurrentType = createType();
-    CurrentType->setIsPointerMember();
-    CurrentType->setName("*");
-    return CurrentType;
-  case dwarf::DW_TAG_reference_type:
-    CurrentType = createType();
-    CurrentType->setIsReference();
-    CurrentType->setName("&");
-    return CurrentType;
-  case dwarf::DW_TAG_restrict_type:
-    CurrentType = createType();
-    CurrentType->setIsRestrict();
-    CurrentType->setName("restrict");
-    return CurrentType;
-  case dwarf::DW_TAG_rvalue_reference_type:
-    CurrentType = createType();
-    CurrentType->setIsRvalueReference();
-    CurrentType->setName("&&");
-    return CurrentType;
-  case dwarf::DW_TAG_subrange_type:
-    CurrentType = createTypeSubrange();
-    return CurrentType;
-  case dwarf::DW_TAG_template_value_parameter:
-    CurrentType = createTypeParam();
-    CurrentType->setIsTemplateValueParam();
-    return CurrentType;
-  case dwarf::DW_TAG_template_type_parameter:
-    CurrentType = createTypeParam();
-    CurrentType->setIsTemplateTypeParam();
-    return CurrentType;
-  case dwarf::DW_TAG_GNU_template_template_param:
-    CurrentType = createTypeParam();
-    CurrentType->setIsTemplateTemplateParam();
-    return CurrentType;
-  case dwarf::DW_TAG_typedef:
-    CurrentType = createTypeDefinition();
-    return CurrentType;
-  case dwarf::DW_TAG_unspecified_type:
-    CurrentType = createType();
-    CurrentType->setIsUnspecified();
-    return CurrentType;
-  case dwarf::DW_TAG_volatile_type:
-    CurrentType = createType();
-    CurrentType->setIsVolatile();
-    CurrentType->setName("volatile");
-    return CurrentType;
-
-  // Symbols.
-  case dwarf::DW_TAG_formal_parameter:
-    CurrentSymbol = createSymbol();
-    CurrentSymbol->setIsParameter();
-    return CurrentSymbol;
-  case dwarf::DW_TAG_unspecified_parameters:
-    CurrentSymbol = createSymbol();
-    CurrentSymbol->setIsUnspecified();
-    CurrentSymbol->setName("...");
-    return CurrentSymbol;
-  case dwarf::DW_TAG_member:
-    CurrentSymbol = createSymbol();
-    CurrentSymbol->setIsMember();
-    return CurrentSymbol;
-  case dwarf::DW_TAG_variable:
-    CurrentSymbol = createSymbol();
-    CurrentSymbol->setIsVariable();
-    return CurrentSymbol;
-  case dwarf::DW_TAG_inheritance:
-    CurrentSymbol = createSymbol();
-    CurrentSymbol->setIsInheritance();
-    return CurrentSymbol;
-  case dwarf::DW_TAG_call_site_parameter:
-  case dwarf::DW_TAG_GNU_call_site_parameter:
-    CurrentSymbol = createSymbol();
-    CurrentSymbol->setIsCallSiteParameter();
-    return CurrentSymbol;
-  case dwarf::DW_TAG_constant:
-    CurrentSymbol = createSymbol();
-    CurrentSymbol->setIsConstant();
-    return CurrentSymbol;
-
-  // Scopes.
-  case dwarf::DW_TAG_catch_block:
-    CurrentScope = createScope();
-    CurrentScope->setIsCatchBlock();
-    return CurrentScope;
-  case dwarf::DW_TAG_lexical_block:
-    CurrentScope = createScope();
-    CurrentScope->setIsLexicalBlock();
-    return CurrentScope;
-  case dwarf::DW_TAG_try_block:
-    CurrentScope = createScope();
-    CurrentScope->setIsTryBlock();
-    return CurrentScope;
-  case dwarf::DW_TAG_compile_unit:
-  case dwarf::DW_TAG_skeleton_unit:
-    CurrentScope = createScopeCompileUnit();
-    CompileUnit = static_cast<LVScopeCompileUnit *>(CurrentScope);
-    return CurrentScope;
-  case dwarf::DW_TAG_inlined_subroutine:
-    CurrentScope = createScopeFunctionInlined();
-    return CurrentScope;
-  case dwarf::DW_TAG_namespace:
-    CurrentScope = createScopeNamespace();
-    return CurrentScope;
-  case dwarf::DW_TAG_template_alias:
-    CurrentScope = createScopeAlias();
-    return CurrentScope;
-  case dwarf::DW_TAG_array_type:
-    CurrentScope = createScopeArray();
-    return CurrentScope;
-  case dwarf::DW_TAG_call_site:
-  case dwarf::DW_TAG_GNU_call_site:
-    CurrentScope = createScopeFunction();
-    CurrentScope->setIsCallSite();
-    return CurrentScope;
-  case dwarf::DW_TAG_entry_point:
-    CurrentScope = createScopeFunction();
-    CurrentScope->setIsEntryPoint();
-    return CurrentScope;
-  case dwarf::DW_TAG_subprogram:
-    CurrentScope = createScopeFunction();
-    CurrentScope->setIsSubprogram();
-    return CurrentScope;
-  case dwarf::DW_TAG_subroutine_type:
-    CurrentScope = createScopeFunctionType();
-    return CurrentScope;
-  case dwarf::DW_TAG_label:
-    CurrentScope = createScopeFunction();
-    CurrentScope->setIsLabel();
-    return CurrentScope;
-  case dwarf::DW_TAG_class_type:
-    CurrentScope = createScopeAggregate();
-    CurrentScope->setIsClass();
-    return CurrentScope;
-  case dwarf::DW_TAG_structure_type:
-    CurrentScope = createScopeAggregate();
-    CurrentScope->setIsStructure();
-    return CurrentScope;
-  case dwarf::DW_TAG_union_type:
-    CurrentScope = createScopeAggregate();
-    CurrentScope->setIsUnion();
-    return CurrentScope;
-  case dwarf::DW_TAG_enumeration_type:
-    CurrentScope = createScopeEnumeration();
-    return CurrentScope;
-  case dwarf::DW_TAG_GNU_formal_parameter_pack:
-    CurrentScope = createScopeFormalPack();
-    return CurrentScope;
-  case dwarf::DW_TAG_GNU_template_parameter_pack:
-    CurrentScope = createScopeTemplatePack();
-    return CurrentScope;
-  case dwarf::DW_TAG_module:
-    CurrentScope = createScopeModule();
-    return CurrentScope;
-  default:
-    // Collect TAGs not implemented.
-    if (options().getInternalTag() && Tag)
-      CompileUnit->addDebugTag(Tag, CurrentOffset);
-    break;
-  }
-  return nullptr;
-}
-
 void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
                                         LVOffset *OffsetPtr,
                                         const AttributeSpec &AttrSpec) {
@@ -389,6 +172,11 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
     if (options().getAttributeProducer())
       CurrentElement->setProducer(dwarf::toStringRef(FormValue));
     break;
+  case dwarf::DW_AT_language:
+    if (options().getAttributeLanguage())
+      CurrentElement->setSourceLanguage(LVSourceLanguage{
+          static_cast<llvm::dwarf::SourceLanguage>(GetAsUnsignedConstant())});
+    break;
   case dwarf::DW_AT_upper_bound:
     CurrentElement->setUpperBound(GetBoundValue(FormValue));
     break;
@@ -804,8 +592,8 @@ std::string LVDWARFReader::getRegisterName(LVSmall Opcode,
     return {};
   };
   DumpOpts.GetNameForDWARFReg = GetRegName;
-  DWARFExpression::prettyPrintRegisterOp(/*U=*/nullptr, Stream, DumpOpts,
-                                         Opcode, Operands);
+  DWARFExpressionPrinter::prettyPrintRegisterOp(/*U=*/nullptr, Stream, DumpOpts,
+                                                Opcode, Operands);
   return Stream.str();
 }
 
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFOptions.td b/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFOptions.td
index 0a0ce2fc76dd..a515cbdcc35f 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFOptions.td
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFOptions.td
@@ -18,4 +18,4 @@ multiclass B_priv<string name> {
 
 def export  : P<"export">;
 def alternatename : P<"alternatename">;
-def incl : Joined<["/", "-", "/?", "-?"], "include:">;
\ No newline at end of file
+def incl : Joined<["/", "-", "/?", "-?"], "include:">;
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp b/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
index 475de4866013..f5ca72796df5 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ExecutionEngine/JITLink/aarch32.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/ARMTargetParser.h"
 
@@ -31,7 +32,7 @@ namespace llvm {
 namespace jitlink {
 
 /// Translate from ELF relocation type to JITLink-internal edge kind.
-Expected<aarch32::EdgeKind_aarch32>
+LLVM_ABI Expected<aarch32::EdgeKind_aarch32>
 getJITLinkEdgeKind(uint32_t ELFType, const aarch32::ArmConfig &ArmCfg) {
   switch (ELFType) {
   case ELF::R_ARM_ABS32:
@@ -75,7 +76,7 @@ getJITLinkEdgeKind(uint32_t ELFType, const aarch32::ArmConfig &ArmCfg) {
 }
 
 /// Translate from JITLink-internal edge kind back to ELF relocation type.
-Expected<uint32_t> getELFRelocationType(Edge::Kind Kind) {
+LLVM_ABI Expected<uint32_t> getELFRelocationType(Edge::Kind Kind) {
   switch (static_cast<aarch32::EdgeKind_aarch32>(Kind)) {
   case aarch32::Data_Delta32:
     return ELF::R_ARM_REL32;
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
index 01bb6e040357..1e325d76bd51 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MathExtras.h"
@@ -60,7 +61,7 @@ int64_t decodeImmBT4BlT1BlxT2(uint32_t Hi, uint32_t Lo) {
 ///
 ///   S:I1:I2:Imm10:Imm11:0 -> [ 00000:S:Imm10, 00:J1:0:J2:Imm11 ]
 ///
-HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {
+LLVM_ABI HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {
   uint32_t S = (Value >> 14) & 0x0400;
   uint32_t J1 = (((~(Value >> 10)) ^ (Value >> 11)) & 0x2000);
   uint32_t J2 = (((~(Value >> 11)) ^ (Value >> 13)) & 0x0800);
@@ -74,7 +75,7 @@ HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {
 ///
 ///   [ 00000:S:Imm10, 00:J1:0:J2:Imm11] -> S:I1:I2:Imm10:Imm11:0
 ///
-int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {
+LLVM_ABI int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {
   uint32_t S = Hi & 0x0400;
   uint32_t I1 = ~((Lo ^ (Hi << 3)) << 10) & 0x00800000;
   uint32_t I2 = ~((Lo ^ (Hi << 1)) << 11) & 0x00400000;
@@ -88,7 +89,7 @@ int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {
 ///
 ///   Imm24:00 ->  00000000:Imm24
 ///
-uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {
+LLVM_ABI uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {
   return (Value >> 2) & 0x00ffffff;
 }
 
@@ -97,7 +98,7 @@ uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {
 ///
 ///   00000000:Imm24 ->  Imm24:00
 ///
-int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {
+LLVM_ABI int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {
   return SignExtend64<26>((Value & 0x00ffffff) << 2);
 }
 
@@ -106,7 +107,7 @@ int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {
 ///
 ///   Imm4:Imm1:Imm3:Imm8 -> [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ]
 ///
-HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {
+LLVM_ABI HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {
   uint32_t Imm4 = (Value >> 12) & 0x0f;
   uint32_t Imm1 = (Value >> 11) & 0x01;
   uint32_t Imm3 = (Value >> 8) & 0x07;
@@ -119,7 +120,7 @@ HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {
 ///
 ///   [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] -> Imm4:Imm1:Imm3:Imm8
 ///
-uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
+LLVM_ABI uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
   uint32_t Imm4 = Hi & 0x0f;
   uint32_t Imm1 = (Hi >> 10) & 0x01;
   uint32_t Imm3 = (Lo >> 12) & 0x07;
@@ -133,7 +134,7 @@ uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
 ///
 ///   Rd4 -> [0000000000000000, 0000:Rd4:00000000]
 ///
-HalfWords encodeRegMovtT1MovwT3(int64_t Value) {
+LLVM_ABI HalfWords encodeRegMovtT1MovwT3(int64_t Value) {
   uint32_t Rd4 = (Value & 0x0f) << 8;
   return HalfWords{0, Rd4};
 }
@@ -142,7 +143,7 @@ HalfWords encodeRegMovtT1MovwT3(int64_t Value) {
 ///
 ///   [0000000000000000, 0000:Rd4:00000000] -> Rd4
 ///
-int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
+LLVM_ABI int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
   uint32_t Rd4 = (Lo >> 8) & 0x0f;
   return Rd4;
 }
@@ -152,7 +153,7 @@ int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
 ///
 ///   Imm4:Imm12 -> 000000000000:Imm4:0000:Imm12
 ///
-uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {
+LLVM_ABI uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {
   uint32_t Imm4 = (Value >> 12) & 0x0f;
   uint32_t Imm12 = Value & 0x0fff;
   return (Imm4 << 16) | Imm12;
@@ -163,7 +164,7 @@ uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {
 ///
 ///   000000000000:Imm4:0000:Imm12 -> Imm4:Imm12
 ///
-uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {
+LLVM_ABI uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {
   uint32_t Imm4 = (Value >> 16) & 0x0f;
   uint32_t Imm12 = Value & 0x0fff;
   return (Imm4 << 12) | Imm12;
@@ -174,7 +175,7 @@ uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {
 ///
 ///   Rd4 -> 0000000000000000:Rd4:000000000000
 ///
-uint32_t encodeRegMovtA1MovwA2(int64_t Value) {
+LLVM_ABI uint32_t encodeRegMovtA1MovwA2(int64_t Value) {
   uint32_t Rd4 = (Value & 0x00000f) << 12;
   return Rd4;
 }
@@ -184,7 +185,7 @@ uint32_t encodeRegMovtA1MovwA2(int64_t Value) {
 ///
 ///   0000000000000000:Rd4:000000000000 -> Rd4
 ///
-int64_t decodeRegMovtA1MovwA2(uint64_t Value) {
+LLVM_ABI int64_t decodeRegMovtA1MovwA2(uint64_t Value) {
   uint32_t Rd4 = (Value >> 12) & 0x00000f;
   return Rd4;
 }
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 21ebe82c8a71..f1a71e4acb46 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -269,9 +269,8 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
   }
 
   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
-    getExecutionSession().runSessionLocked([&]() {
-        InitFunctions[&JD].add(InitName);
-      });
+    getExecutionSession().runSessionLocked(
+        [&]() { InitFunctions[&JD].add(InitName); });
   }
 
   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
@@ -935,8 +934,8 @@ Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
 Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
                                                   SymbolStringPtr Name) {
   if (auto Sym = ES->lookup(
-        makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),
-        Name))
+          makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),
+          Name))
     return Sym->getAddress();
   else
     return Sym.takeError();
@@ -951,7 +950,9 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
 
   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
   // a new SectionMemoryManager for each object.
-  auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
+  auto GetMemMgr = [](const MemoryBuffer &) {
+    return std::make_unique<SectionMemoryManager>();
+  };
   auto Layer =
       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
 
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
index d44199f1698e..9999e1ff3bf0 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
@@ -1021,8 +1021,10 @@ LLVMOrcObjectLayerRef
 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(
     LLVMOrcExecutionSessionRef ES) {
   assert(ES && "ES must not be null");
-  return wrap(new RTDyldObjectLinkingLayer(
-      *unwrap(ES), [] { return std::make_unique<SectionMemoryManager>(); }));
+  return wrap(
+      new RTDyldObjectLinkingLayer(*unwrap(ES), [](const MemoryBuffer &) {
+        return std::make_unique<SectionMemoryManager>();
+      }));
 }
 
 LLVMOrcObjectLayerRef
@@ -1128,9 +1130,10 @@ LLVMOrcCreateRTDyldObjectLinkingLayerWithMCJITMemoryManagerLikeCallbacks(
       CreateContextCtx, CreateContext, NotifyTerminating, AllocateCodeSection,
       AllocateDataSection, FinalizeMemory, Destroy);
 
-  return wrap(new RTDyldObjectLinkingLayer(*unwrap(ES), [CBs = std::move(CBs)] {
-    return std::make_unique<MCJITMemoryManagerLikeCallbacksMemMgr>(CBs);
-  }));
+  return wrap(new RTDyldObjectLinkingLayer(
+      *unwrap(ES), [CBs = std::move(CBs)](const MemoryBuffer &) {
+        return std::make_unique<MCJITMemoryManagerLikeCallbacksMemMgr>(CBs);
+      }));
 
   return nullptr;
 }
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
index 88cceacb7118..4d4a705ced64 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -22,7 +22,8 @@ class JITDylibSearchOrderResolver : public JITSymbolResolver {
                               SymbolDependenceMap &Deps)
       : MR(MR), Deps(Deps) {}
 
-  void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) override {
+  void lookup(const LookupSet &Symbols,
+              OnResolvedFunction OnResolved) override {
     auto &ES = MR.getTargetJITDylib().getExecutionSession();
     SymbolLookupSet InternedSymbols;
 
@@ -181,7 +182,7 @@ void RTDyldObjectLinkingLayer::emit(
     }
   }
 
-  auto MemMgr = GetMemoryManager();
+  auto MemMgr = GetMemoryManager(*O);
   auto &MemMgrRef = *MemMgr;
 
   // Switch to shared ownership of MR so that it can be captured by both
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
index fd626321c973..cb1b3b05cd24 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
@@ -24,14 +24,13 @@ extern "C" {
 // We put information about the JITed function in this global, which the
 // debugger reads.  Make sure to specify the version statically, because the
 // debugger checks the version before we can set it during runtime.
-LLVM_ALWAYS_EXPORT
-struct jit_descriptor __jit_debug_descriptor = {JitDescriptorVersion, 0,
-                                                nullptr, nullptr};
+LLVM_ABI LLVM_ALWAYS_EXPORT struct jit_descriptor __jit_debug_descriptor = {
+    JitDescriptorVersion, 0, nullptr, nullptr};
 
 // Debuggers that implement the GDB JIT interface put a special breakpoint in
 // this function.
-LLVM_ALWAYS_EXPORT
-LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() {
+LLVM_ABI LLVM_ALWAYS_EXPORT LLVM_ATTRIBUTE_NOINLINE void
+__jit_debug_register_code() {
   // The noinline and the asm prevent calls to this function from being
   // optimized out.
 #if !defined(_MSC_VER)
diff --git a/external/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/external/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index 24b03a058981..89b20978c40e 100644
--- a/external/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/external/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -202,15 +202,9 @@ class RelocationValueRef {
            IsStubThumb == Other.IsStubThumb;
   }
   inline bool operator<(const RelocationValueRef &Other) const {
-    if (SectionID != Other.SectionID)
-      return SectionID < Other.SectionID;
-    if (Offset != Other.Offset)
-      return Offset < Other.Offset;
-    if (Addend != Other.Addend)
-      return Addend < Other.Addend;
-    if (IsStubThumb != Other.IsStubThumb)
-      return IsStubThumb < Other.IsStubThumb;
-    return SymbolName < Other.SymbolName;
+    return std::tie(SectionID, Offset, Addend, IsStubThumb, SymbolName) <
+           std::tie(Other.SectionID, Other.Offset, Other.Addend,
+                    Other.IsStubThumb, Other.SymbolName);
   }
 };
 
diff --git a/external/llvm-project/llvm/lib/FileCheck/FileCheck.cpp b/external/llvm-project/llvm/lib/FileCheck/FileCheck.cpp
index 71b47a04fd13..bcca499322ae 100644
--- a/external/llvm-project/llvm/lib/FileCheck/FileCheck.cpp
+++ b/external/llvm-project/llvm/lib/FileCheck/FileCheck.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Support/CheckedArithmetic.h"
 #include "llvm/Support/FormatVariadic.h"
 #include <cstdint>
 #include <list>
diff --git a/external/llvm-project/llvm/lib/Frontend/CMakeLists.txt b/external/llvm-project/llvm/lib/Frontend/CMakeLists.txt
index b305ce7d771c..3b31e6f8dec9 100644
--- a/external/llvm-project/llvm/lib/Frontend/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Frontend/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(Atomic)
+add_subdirectory(Directive)
 add_subdirectory(Driver)
 add_subdirectory(HLSL)
 add_subdirectory(OpenACC)
diff --git a/external/llvm-project/llvm/lib/Frontend/Directive/CMakeLists.txt b/external/llvm-project/llvm/lib/Frontend/Directive/CMakeLists.txt
new file mode 100644
index 000000000000..a567e1affb17
--- /dev/null
+++ b/external/llvm-project/llvm/lib/Frontend/Directive/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_llvm_component_library(LLVMFrontendDirective
+  Spelling.cpp
+
+  LINK_COMPONENTS
+  Support
+)
diff --git a/external/llvm-project/llvm/lib/Frontend/Directive/Spelling.cpp b/external/llvm-project/llvm/lib/Frontend/Directive/Spelling.cpp
new file mode 100644
index 000000000000..4992268f2e67
--- /dev/null
+++ b/external/llvm-project/llvm/lib/Frontend/Directive/Spelling.cpp
@@ -0,0 +1,38 @@
+//===-------------------------------------------------------------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/Directive/Spelling.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MathExtras.h"
+
+#include <cassert>
+
+using namespace llvm;
+
+static bool Contains(directive::VersionRange V, int P) {
+  return V.Min <= P && P <= V.Max;
+}
+
+llvm::StringRef llvm::directive::FindName(
+    llvm::iterator_range<const directive::Spelling *> Range, unsigned Version) {
+  assert(llvm::isInt<8 * sizeof(int)>(Version) && "Version value out of range");
+
+  int V = Version;
+  // Do a linear search to find the first Spelling that contains Version.
+  // The condition "contains(S, Version)" does not partition the list of
+  // spellings, so std::[lower|upper]_bound cannot be used.
+  // In practice the list of spellings is expected to be very short, so
+  // linear search seems appropriate. In general, an interval tree may be
+  // a better choice, but in this case it may be an overkill.
+  for (auto &S : Range) {
+    if (Contains(S.Versions, V))
+      return S.Name;
+  }
+  return StringRef();
+}
diff --git a/external/llvm-project/llvm/lib/Frontend/Driver/CMakeLists.txt b/external/llvm-project/llvm/lib/Frontend/Driver/CMakeLists.txt
index 23de4994a300..9feee6fe6929 100644
--- a/external/llvm-project/llvm/lib/Frontend/Driver/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Frontend/Driver/CMakeLists.txt
@@ -12,4 +12,5 @@ add_llvm_component_library(LLVMFrontendDriver
   Core
   Support
   Analysis
+  Instrumentation
   )
diff --git a/external/llvm-project/llvm/lib/Frontend/Driver/CodeGenOptions.cpp b/external/llvm-project/llvm/lib/Frontend/Driver/CodeGenOptions.cpp
index 52080dea93c9..df884908845d 100644
--- a/external/llvm-project/llvm/lib/Frontend/Driver/CodeGenOptions.cpp
+++ b/external/llvm-project/llvm/lib/Frontend/Driver/CodeGenOptions.cpp
@@ -8,8 +8,15 @@
 
 #include "llvm/Frontend/Driver/CodeGenOptions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/TargetParser/Triple.h"
 
+namespace llvm {
+extern llvm::cl::opt<bool> DebugInfoCorrelate;
+extern llvm::cl::opt<llvm::InstrProfCorrelator::ProfCorrelatorKind>
+    ProfileCorrelate;
+} // namespace llvm
+
 namespace llvm::driver {
 
 TargetLibraryInfoImpl *createTLII(const llvm::Triple &TargetTriple,
@@ -56,4 +63,10 @@ TargetLibraryInfoImpl *createTLII(const llvm::Triple &TargetTriple,
   return TLII;
 }
 
+std::string getDefaultProfileGenName() {
+  return llvm::DebugInfoCorrelate ||
+                 llvm::ProfileCorrelate != InstrProfCorrelator::NONE
+             ? "default_%m.proflite"
+             : "default_%m.profraw";
+}
 } // namespace llvm::driver
diff --git a/external/llvm-project/llvm/lib/Frontend/HLSL/CMakeLists.txt b/external/llvm-project/llvm/lib/Frontend/HLSL/CMakeLists.txt
index dd987544fe80..dfebe354807b 100644
--- a/external/llvm-project/llvm/lib/Frontend/HLSL/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Frontend/HLSL/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_llvm_component_library(LLVMFrontendHLSL
   CBuffer.cpp
   HLSLResource.cpp
-  HLSLRootSignature.cpp
+  HLSLRootSignatureUtils.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend
diff --git a/external/llvm-project/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp b/external/llvm-project/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
similarity index 73%
rename from external/llvm-project/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
rename to external/llvm-project/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
index b8d6e229006d..765a3bcbed7e 100644
--- a/external/llvm-project/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
+++ b/external/llvm-project/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
@@ -1,4 +1,4 @@
-//===- HLSLRootSignature.cpp - HLSL Root Signature helper objects ---------===//
+//===- HLSLRootSignatureUtils.cpp - HLSL Root Signature helpers -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -10,12 +10,11 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/bit.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/Module.h"
 
 namespace llvm {
 namespace hlsl {
@@ -169,25 +168,44 @@ void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements) {
   OS << "}";
 }
 
+namespace {
+
+// We use the OverloadBuild with std::visit to ensure the compiler catches if a
+// new RootElement variant type is added but it's metadata generation isn't
+// handled.
+template <class... Ts> struct OverloadedBuild : Ts... {
+  using Ts::operator()...;
+};
+template <class... Ts> OverloadedBuild(Ts...) -> OverloadedBuild<Ts...>;
+
+} // namespace
+
 MDNode *MetadataBuilder::BuildRootSignature() {
+  const auto Visitor = OverloadedBuild{
+      [this](const RootFlags &Flags) -> MDNode * {
+        return BuildRootFlags(Flags);
+      },
+      [this](const RootConstants &Constants) -> MDNode * {
+        return BuildRootConstants(Constants);
+      },
+      [this](const RootDescriptor &Descriptor) -> MDNode * {
+        return BuildRootDescriptor(Descriptor);
+      },
+      [this](const DescriptorTableClause &Clause) -> MDNode * {
+        return BuildDescriptorTableClause(Clause);
+      },
+      [this](const DescriptorTable &Table) -> MDNode * {
+        return BuildDescriptorTable(Table);
+      },
+      [this](const StaticSampler &Sampler) -> MDNode * {
+        return BuildStaticSampler(Sampler);
+      },
+  };
+
   for (const RootElement &Element : Elements) {
-    MDNode *ElementMD = nullptr;
-    if (const auto &Flags = std::get_if<RootFlags>(&Element))
-      ElementMD = BuildRootFlags(*Flags);
-    else if (const auto &Constants = std::get_if<RootConstants>(&Element))
-      ElementMD = BuildRootConstants(*Constants);
-    else if (const auto &Descriptor = std::get_if<RootDescriptor>(&Element))
-      ElementMD = BuildRootDescriptor(*Descriptor);
-    else if (const auto &Clause = std::get_if<DescriptorTableClause>(&Element))
-      ElementMD = BuildDescriptorTableClause(*Clause);
-    else if (const auto &Table = std::get_if<DescriptorTable>(&Element))
-      ElementMD = BuildDescriptorTable(*Table);
-
-    // FIXME(#126586): remove once all RootElemnt variants are handled in a
-    // visit or otherwise
+    MDNode *ElementMD = std::visit(Visitor, Element);
     assert(ElementMD != nullptr &&
-           "Constructed an unhandled root element type.");
-
+           "Root Element must be initialized and validated");
     GeneratedMetadata.push_back(ElementMD);
   }
 
@@ -274,6 +292,37 @@ MDNode *MetadataBuilder::BuildDescriptorTableClause(
            });
 }
 
+MDNode *MetadataBuilder::BuildStaticSampler(const StaticSampler &Sampler) {
+  IRBuilder<> Builder(Ctx);
+  Metadata *Operands[] = {
+      MDString::get(Ctx, "StaticSampler"),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.Filter))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressU))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressV))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressW))),
+      ConstantAsMetadata::get(llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx),
+                                                    Sampler.MipLODBias)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.MaxAnisotropy)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.CompFunc))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.BorderColor))),
+      ConstantAsMetadata::get(
+          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MinLOD)),
+      ConstantAsMetadata::get(
+          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MaxLOD)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.Space)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.Visibility))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
 } // namespace rootsig
 } // namespace hlsl
 } // namespace llvm
diff --git a/external/llvm-project/llvm/lib/Frontend/OpenACC/CMakeLists.txt b/external/llvm-project/llvm/lib/Frontend/OpenACC/CMakeLists.txt
index f35201497869..4664b71407c4 100644
--- a/external/llvm-project/llvm/lib/Frontend/OpenACC/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Frontend/OpenACC/CMakeLists.txt
@@ -9,5 +9,5 @@ add_llvm_component_library(LLVMFrontendOpenACC
   acc_gen
 )
 
-target_link_libraries(LLVMFrontendOpenACC LLVMSupport)
+target_link_libraries(LLVMFrontendOpenACC LLVMSupport LLVMFrontendDirective)
 
diff --git a/external/llvm-project/llvm/lib/Frontend/OpenMP/CMakeLists.txt b/external/llvm-project/llvm/lib/Frontend/OpenMP/CMakeLists.txt
index 35c607866a94..5bf15ca3a899 100644
--- a/external/llvm-project/llvm/lib/Frontend/OpenMP/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Frontend/OpenMP/CMakeLists.txt
@@ -23,4 +23,5 @@ add_llvm_component_library(LLVMFrontendOpenMP
   BitReader
   FrontendOffloading
   FrontendAtomic
+  FrontendDirective
   )
diff --git a/external/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/external/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 0051c286d50f..1eaea9b22600 100644
--- a/external/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/external/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6986,22 +6986,17 @@ static void FixupDebugInfoForOutlinedFunction(
       // snippet) as location of variable. The AMDGPU backend drops the debug
       // info for variable in such cases. So we change the location to alloca
       // instead.
+      if (DR->getNumVariableLocationOps() != 1u)
+        return;
+      auto Loc = DR->getVariableLocationOp(0u);
       bool PassByRef = false;
-      llvm::Type *locType = nullptr;
-      for (auto Loc : DR->location_ops()) {
-        locType = Loc->getType();
-        if (llvm::LoadInst *Load = dyn_cast<llvm::LoadInst>(Loc)) {
-          DR->replaceVariableLocationOp(Loc, Load->getPointerOperand());
-          PassByRef = true;
-        }
+      if (llvm::LoadInst *Load = dyn_cast<llvm::LoadInst>(Loc)) {
+        Loc = Load->getPointerOperand();
+        PassByRef = true;
       }
       // Add DIOps based expression. Note that we generate an extra indirection
       // if an argument is mapped by reference. The first reads the pointer
       // from alloca and 2nd read the value of the variable from that pointer.
-      llvm::DIExprBuilder ExprBuilder(Builder.getContext());
-      unsigned int allocaAS = M->getDataLayout().getAllocaAddrSpace();
-      unsigned int defaultAS = M->getDataLayout().getProgramAddressSpace();
-      ExprBuilder.append<llvm::DIOp::Arg>(0u, Builder.getPtrTy(allocaAS));
       // We have 2 options for the variables that are mapped byRef.
       // 1. Use a single indirection but change the type to the reference to the
       // original type. It will show up in the debugger as
@@ -7010,10 +7005,15 @@ static void FixupDebugInfoForOutlinedFunction(
       // 2. Use double indirection and keep the original type. It will show up
       // in debugger as "x=5". This approached is used here as it is
       // consistent with the normal fortran parameters display.
-      if (PassByRef)
-        ExprBuilder.append<llvm::DIOp::Deref>(Builder.getPtrTy(defaultAS));
-      ExprBuilder.append<llvm::DIOp::Deref>(locType);
-      DR->setExpression(ExprBuilder.intoExpression());
+      if (auto AI = dyn_cast<llvm::AllocaInst>(Loc->stripPointerCasts())) {
+        DR->replaceVariableLocationOp(0u, AI);
+        llvm::DIExprBuilder ExprBuilder(Builder.getContext());
+        ExprBuilder.append<llvm::DIOp::Arg>(0u, AI->getType());
+        if (PassByRef)
+          ExprBuilder.append<llvm::DIOp::Deref>(AI->getAllocatedType());
+        ExprBuilder.append<llvm::DIOp::Deref>(AI->getAllocatedType());
+        DR->setExpression(ExprBuilder.intoExpression());
+      }
     }
 
     DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
diff --git a/external/llvm-project/llvm/lib/FuzzMutate/IRMutator.cpp b/external/llvm-project/llvm/lib/FuzzMutate/IRMutator.cpp
index 672c666d4e05..0070fc1a6c7e 100644
--- a/external/llvm-project/llvm/lib/FuzzMutate/IRMutator.cpp
+++ b/external/llvm-project/llvm/lib/FuzzMutate/IRMutator.cpp
@@ -356,6 +356,62 @@ static uint64_t getUniqueCaseValue(SmallSet<uint64_t, 4> &CasesTaken,
   return tmp;
 }
 
+/// Determines whether a function is unsupported by the current mutator's
+/// implementation. The function returns true if any of the following criteria
+/// are met:
+///   * The function accepts metadata or token types as arguments.
+///   * The function has ABI attributes that could cause UB.
+///   * The function uses a non-callable CC that may result in UB.
+static bool isUnsupportedFunction(Function *F) {
+  // Some functions accept metadata type or token type as arguments.
+  // We don't call those functions for now.
+  // For example, `@llvm.dbg.declare(metadata, metadata, metadata)`
+  // https://llvm.org/docs/SourceLevelDebugging.html#llvm-dbg-declare
+  auto IsUnsupportedTy = [](Type *T) {
+    return T->isMetadataTy() || T->isTokenTy();
+  };
+
+  if (IsUnsupportedTy(F->getReturnType()) ||
+      any_of(F->getFunctionType()->params(), IsUnsupportedTy)) {
+    return true;
+  }
+
+  // ABI attributes must be specified both at the function
+  // declaration/definition and call-site, otherwise the
+  // behavior may be undefined.
+  // We don't call those functions for now to prevent UB from happening.
+  auto IsABIAttribute = [](AttributeSet A) {
+    static const Attribute::AttrKind ABIAttrs[] = {
+        Attribute::StructRet,      Attribute::ByVal,
+        Attribute::InAlloca,       Attribute::InReg,
+        Attribute::StackAlignment, Attribute::SwiftSelf,
+        Attribute::SwiftAsync,     Attribute::SwiftError,
+        Attribute::Preallocated,   Attribute::ByRef,
+        Attribute::ZExt,           Attribute::SExt};
+
+    return llvm::any_of(ABIAttrs, [&](Attribute::AttrKind kind) {
+      return A.hasAttribute(kind);
+    });
+  };
+
+  auto FuncAttrs = F->getAttributes();
+  if (IsABIAttribute(FuncAttrs.getRetAttrs())) {
+    return true;
+  }
+  for (size_t i = 0; i < F->arg_size(); i++) {
+    if (IsABIAttribute(FuncAttrs.getParamAttrs(i))) {
+      return true;
+    }
+  }
+
+  // If it is not satisfied, the IR will be invalid.
+  if (!isCallableCC(F->getCallingConv())) {
+    return true;
+  }
+
+  return false;
+}
+
 void InsertFunctionStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
   Module *M = BB.getParent()->getParent();
   // If nullptr is selected, we will create a new function declaration.
@@ -366,16 +422,8 @@ void InsertFunctionStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
 
   auto RS = makeSampler(IB.Rand, Functions);
   Function *F = RS.getSelection();
-  // Some functions accept metadata type or token type as arguments.
-  // We don't call those functions for now.
-  // For example, `@llvm.dbg.declare(metadata, metadata, metadata)`
-  // https://llvm.org/docs/SourceLevelDebugging.html#llvm-dbg-declare
-  auto IsUnsupportedTy = [](Type *T) {
-    return T->isMetadataTy() || T->isTokenTy();
-  };
-  if (!F || IsUnsupportedTy(F->getReturnType()) ||
-      any_of(F->getFunctionType()->params(), IsUnsupportedTy) ||
-      !isCallableCC(F->getCallingConv())) {
+
+  if (!F || isUnsupportedFunction(F)) {
     F = IB.createFunctionDeclaration(*M);
   }
 
diff --git a/external/llvm-project/llvm/lib/FuzzMutate/RandomIRBuilder.cpp b/external/llvm-project/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
index 22ce5200e132..5e793997aee6 100644
--- a/external/llvm-project/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
+++ b/external/llvm-project/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
@@ -16,7 +16,6 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 
 using namespace llvm;
diff --git a/external/llvm-project/llvm/lib/IR/AsmWriter.cpp b/external/llvm-project/llvm/lib/IR/AsmWriter.cpp
index 0b1fd24e0765..ee5cd49f113b 100644
--- a/external/llvm-project/llvm/lib/IR/AsmWriter.cpp
+++ b/external/llvm-project/llvm/lib/IR/AsmWriter.cpp
@@ -1204,17 +1204,23 @@ void SlotTracker::processFunctionMetadata(const Function &F) {
 }
 
 void SlotTracker::processDbgRecordMetadata(const DbgRecord &DR) {
+  // Tolerate null metadata pointers: it's a completely illegal debug record,
+  // but we can have faulty metadata from debug-intrinsic days being
+  // autoupgraded into debug records. This gets caught by the verifier, which
+  // then will print the faulty IR, hitting this code path.
   if (const DbgVariableRecord *DVR = dyn_cast<const DbgVariableRecord>(&DR)) {
     // Process metadata used by DbgRecords; we only specifically care about the
     // DILocalVariable, DILocation, and DIAssignID fields, as the Value and
     // Expression fields should only be printed inline and so do not use a slot.
     // Note: The above doesn't apply for empty-metadata operands.
-    if (auto *Empty = dyn_cast<MDNode>(DVR->getRawLocation()))
+    if (auto *Empty = dyn_cast_if_present<MDNode>(DVR->getRawLocation()))
       CreateMetadataSlot(Empty);
-    CreateMetadataSlot(DVR->getRawVariable());
+    if (DVR->getRawVariable())
+      CreateMetadataSlot(DVR->getRawVariable());
     if (DVR->isDbgAssign()) {
-      CreateMetadataSlot(cast<MDNode>(DVR->getRawAssignID()));
-      if (auto *Empty = dyn_cast<MDNode>(DVR->getRawAddress()))
+      if (auto *AssignID = DVR->getRawAssignID())
+        CreateMetadataSlot(cast<MDNode>(AssignID));
+      if (auto *Empty = dyn_cast_if_present<MDNode>(DVR->getRawAddress()))
         CreateMetadataSlot(Empty);
     }
   } else if (const DbgLabelRecord *DLR = dyn_cast<const DbgLabelRecord>(&DR)) {
@@ -1222,7 +1228,8 @@ void SlotTracker::processDbgRecordMetadata(const DbgRecord &DR) {
   } else {
     llvm_unreachable("unsupported DbgRecord kind");
   }
-  CreateMetadataSlot(DR.getDebugLoc().getAsMDNode());
+  if (DR.getDebugLoc())
+    CreateMetadataSlot(DR.getDebugLoc().getAsMDNode());
 }
 
 void SlotTracker::processInstructionMetadata(const Instruction &I) {
@@ -4981,22 +4988,30 @@ void AssemblyWriter::printDbgVariableRecord(const DbgVariableRecord &DVR) {
     llvm_unreachable(
         "Tried to print a DbgVariableRecord with an invalid LocationType!");
   }
+
+  auto PrintOrNull = [&](Metadata *M) {
+    if (!M)
+      Out << "(null)";
+    else
+      WriteAsOperandInternal(Out, M, WriterCtx, true);
+  };
+
   Out << "(";
-  WriteAsOperandInternal(Out, DVR.getRawLocation(), WriterCtx, true);
+  PrintOrNull(DVR.getRawLocation());
   Out << ", ";
-  WriteAsOperandInternal(Out, DVR.getRawVariable(), WriterCtx, true);
+  PrintOrNull(DVR.getRawVariable());
   Out << ", ";
-  WriteAsOperandInternal(Out, DVR.getRawExpression(), WriterCtx, true);
+  PrintOrNull(DVR.getRawExpression());
   Out << ", ";
   if (DVR.isDbgAssign()) {
-    WriteAsOperandInternal(Out, DVR.getRawAssignID(), WriterCtx, true);
+    PrintOrNull(DVR.getRawAssignID());
     Out << ", ";
-    WriteAsOperandInternal(Out, DVR.getRawAddress(), WriterCtx, true);
+    PrintOrNull(DVR.getRawAddress());
     Out << ", ";
-    WriteAsOperandInternal(Out, DVR.getRawAddressExpression(), WriterCtx, true);
+    PrintOrNull(DVR.getRawAddressExpression());
     Out << ", ";
   }
-  WriteAsOperandInternal(Out, DVR.getDebugLoc().getAsMDNode(), WriterCtx, true);
+  PrintOrNull(DVR.getDebugLoc().getAsMDNode());
   Out << ")";
 }
 
diff --git a/external/llvm-project/llvm/lib/IR/Attributes.cpp b/external/llvm-project/llvm/lib/IR/Attributes.cpp
index b34a546a93f5..2189b11c5697 100644
--- a/external/llvm-project/llvm/lib/IR/Attributes.cpp
+++ b/external/llvm-project/llvm/lib/IR/Attributes.cpp
@@ -295,11 +295,15 @@ Attribute Attribute::getWithCaptureInfo(LLVMContext &Context, CaptureInfo CI) {
 Attribute
 Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg,
                                 const std::optional<unsigned> &NumElemsArg) {
-  assert(!(ElemSizeArg == 0 && NumElemsArg && *NumElemsArg == 0) &&
+  assert(!(ElemSizeArg == 0 && NumElemsArg == 0) &&
          "Invalid allocsize arguments -- given allocsize(0, 0)");
   return get(Context, AllocSize, packAllocSizeArgs(ElemSizeArg, NumElemsArg));
 }
 
+Attribute Attribute::getWithAllocKind(LLVMContext &Context, AllocFnKind Kind) {
+  return get(Context, AllocKind, static_cast<uint64_t>(Kind));
+}
+
 Attribute Attribute::getWithVScaleRangeArgs(LLVMContext &Context,
                                             unsigned MinValue,
                                             unsigned MaxValue) {
diff --git a/external/llvm-project/llvm/lib/IR/AutoUpgrade.cpp b/external/llvm-project/llvm/lib/IR/AutoUpgrade.cpp
index 3fbaeca3e611..2b40c13fd53b 100644
--- a/external/llvm-project/llvm/lib/IR/AutoUpgrade.cpp
+++ b/external/llvm-project/llvm/lib/IR/AutoUpgrade.cpp
@@ -767,6 +767,12 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
         return false; // Not 'arm.mve.vctp64'.
       }
 
+      if (Name.starts_with("vrintn.v")) {
+        NewFn = Intrinsic::getOrInsertDeclaration(
+            F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
+        return true;
+      }
+
       // These too are changed to accept a v2i1 instead of the old v4i1.
       if (Name.consume_back(".v4i1")) {
         // 'arm.mve.*.v4i1'.
@@ -939,6 +945,53 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
   return false; // No other 'arm.*', 'aarch64.*'.
 }
 
+static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
+                                                        StringRef Name) {
+  if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {
+    Intrinsic::ID ID =
+        StringSwitch<Intrinsic::ID>(Name)
+            .Case("im2col.3d",
+                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
+            .Case("im2col.4d",
+                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
+            .Case("im2col.5d",
+                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
+            .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
+            .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
+            .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
+            .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
+            .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
+            .Default(Intrinsic::not_intrinsic);
+
+    if (ID == Intrinsic::not_intrinsic)
+      return ID;
+
+    // These intrinsics may need upgrade for two reasons:
+    // (1) When the address-space of the first argument is shared[AS=3]
+    //     (and we upgrade it to use shared_cluster address-space[AS=7])
+    if (F->getArg(0)->getType()->getPointerAddressSpace() ==
+        NVPTXAS::ADDRESS_SPACE_SHARED)
+      return ID;
+
+    // (2) When there are only two boolean flag arguments at the end:
+    //
+    // The last three parameters of the older version of these
+    // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
+    //
+    // The newer version reads as:
+    // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
+    //
+    // So, when the type of the [N-3]rd argument is "not i1", then
+    // it is the older version and we need to upgrade.
+    size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
+    Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);
+    if (!ArgType->isIntegerTy(1))
+      return ID;
+  }
+
+  return Intrinsic::not_intrinsic;
+}
+
 static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
                                                               StringRef Name) {
   if (Name.consume_front("mapa.shared.cluster"))
@@ -953,22 +1006,6 @@ static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
                   Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
             .Case("shared.cta.to.cluster",
                   Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
-            .Case("tensor.g2s.im2col.3d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
-            .Case("tensor.g2s.im2col.4d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
-            .Case("tensor.g2s.im2col.5d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
-            .Case("tensor.g2s.tile.1d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
-            .Case("tensor.g2s.tile.2d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
-            .Case("tensor.g2s.tile.3d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
-            .Case("tensor.g2s.tile.4d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
-            .Case("tensor.g2s.tile.5d",
-                  Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
             .Default(Intrinsic::not_intrinsic);
 
     if (ID != Intrinsic::not_intrinsic)
@@ -1149,8 +1186,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
   case 'd':
     if (Name.consume_front("dbg.")) {
       // Mark debug intrinsics for upgrade to new debug format.
-      if (CanUpgradeDebugIntrinsicsToRecords &&
-          F->getParent()->IsNewDbgInfoFormat) {
+      if (CanUpgradeDebugIntrinsicsToRecords) {
         if (Name == "addr" || Name == "value" || Name == "assign" ||
             Name == "declare" || Name == "label") {
           // There's no function to replace these with.
@@ -1334,6 +1370,14 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         return true;
       }
 
+      // Upgrade TMA copy G2S Intrinsics
+      IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
+      if (IID != Intrinsic::not_intrinsic) {
+        rename(F);
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
+        return true;
+      }
+
       // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
       // not to an intrinsic alone.  We expand them in UpgradeIntrinsicCall.
       //
@@ -4389,39 +4433,66 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
   return Builder.CreateBitCast(RMW, RetTy);
 }
 
-/// Helper to unwrap intrinsic call MetadataAsValue operands.
-template <typename MDType>
-static MDType *unwrapMAVOp(CallBase *CI, unsigned Op) {
-  if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
-    return dyn_cast<MDType>(MAV->getMetadata());
+/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
+/// plain MDNode, as it's the verifier's job to check these are the correct
+/// types later.
+static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
+  if (Op < CI->arg_size()) {
+    if (MetadataAsValue *MAV =
+            dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
+      Metadata *MD = MAV->getMetadata();
+      return dyn_cast_if_present<MDNode>(MD);
+    }
+  }
+  return nullptr;
+}
+
+/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
+static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
+  if (Op < CI->arg_size())
+    if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
+      return MAV->getMetadata();
   return nullptr;
 }
 
+static MDNode *getDebugLocSafe(const Instruction *I) {
+  // The MDNode attached to this instruction might not be the correct type,
+  // as the verifier has not yet be run. Fetch it as a bare MDNode.
+  return I->getDebugLoc().getAsMDNode();
+}
+
 /// Convert debug intrinsic calls to non-instruction debug records.
 /// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
 /// \p CI - The debug intrinsic call.
 static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
   DbgRecord *DR = nullptr;
   if (Name == "label") {
-    DR = new DbgLabelRecord(unwrapMAVOp<DILabel>(CI, 0), CI->getDebugLoc());
+    DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0),
+                                                        CI->getDebugLoc());
   } else if (Name == "assign") {
-    DR = new DbgVariableRecord(
-        unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
-        unwrapMAVOp<DIExpression>(CI, 2), unwrapMAVOp<DIAssignID>(CI, 3),
-        unwrapMAVOp<Metadata>(CI, 4), unwrapMAVOp<DIExpression>(CI, 5),
-        CI->getDebugLoc());
+    DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
+        DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
+        unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
+        unwrapMAVMetadataOp(CI, 4),
+        /*The address is a Value ref, it will be stored as a Metadata */
+        unwrapMAVOp(CI, 5), getDebugLocSafe(CI));
   } else if (Name == "declare") {
-    DR = new DbgVariableRecord(
-        unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
-        unwrapMAVOp<DIExpression>(CI, 2), CI->getDebugLoc(),
-        DbgVariableRecord::LocationType::Declare);
+    DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
+        DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
+        unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr,
+        getDebugLocSafe(CI));
   } else if (Name == "addr") {
     // Upgrade dbg.addr to dbg.value with DW_OP_deref.
-    DIExpression *Expr = unwrapMAVOp<DIExpression>(CI, 2);
-    Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
-    DR = new DbgVariableRecord(unwrapMAVOp<Metadata>(CI, 0),
-                               unwrapMAVOp<DILocalVariable>(CI, 1), Expr,
-                               CI->getDebugLoc());
+    MDNode *ExprNode = unwrapMAVOp(CI, 2);
+    // Don't try to add something to the expression if it's not an expression.
+    // Instead, allow the verifier to fail later.
+    if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
+      ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
+    }
+    DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
+        DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
+        unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr,
+        getDebugLocSafe(CI));
   } else if (Name == "value") {
     // An old version of dbg.value had an extra offset argument.
     unsigned VarOp = 1;
@@ -4434,9 +4505,10 @@ static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
       VarOp = 2;
       ExprOp = 3;
     }
-    DR = new DbgVariableRecord(
-        unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, VarOp),
-        unwrapMAVOp<DIExpression>(CI, ExprOp), CI->getDebugLoc());
+    DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
+        DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
+        unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
+        nullptr, getDebugLocSafe(CI));
   }
   assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
   CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
@@ -4457,7 +4529,6 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
   Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
 
   if (!NewFn) {
-    bool FallthroughToDefaultUpgrade = false;
     // Get the Function's name.
     StringRef Name = F->getName();
 
@@ -4485,29 +4556,15 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
     } else if (IsAMDGCN) {
       Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
     } else if (IsDbg) {
-      // We might have decided we don't want the new format after all between
-      // first requesting the upgrade and now; skip the conversion if that is
-      // the case, and check here to see if the intrinsic needs to be upgraded
-      // normally.
-      if (!CI->getModule()->IsNewDbgInfoFormat) {
-        bool NeedsUpgrade =
-            upgradeIntrinsicFunction1(CI->getCalledFunction(), NewFn, false);
-        if (!NeedsUpgrade)
-          return;
-        FallthroughToDefaultUpgrade = true;
-      } else {
-        upgradeDbgIntrinsicToDbgRecord(Name, CI);
-      }
+      upgradeDbgIntrinsicToDbgRecord(Name, CI);
     } else {
       llvm_unreachable("Unknown function for CallBase upgrade.");
     }
 
-    if (!FallthroughToDefaultUpgrade) {
-      if (Rep)
-        CI->replaceAllUsesWith(Rep);
-      CI->eraseFromParent();
-      return;
-    }
+    if (Rep)
+      CI->replaceAllUsesWith(Rep);
+    CI->eraseFromParent();
+    return;
   }
 
   const auto &DefaultCase = [&]() -> void {
@@ -4813,7 +4870,18 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
     return;
   }
   case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
-  case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster:
+  case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
+    // Create a new call with the correct address space.
+    SmallVector<Value *, 4> Args(CI->args());
+    Args[0] = Builder.CreateAddrSpaceCast(
+        Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
+
+    NewCall = Builder.CreateCall(NewFn, Args);
+    NewCall->takeName(CI);
+    CI->replaceAllUsesWith(NewCall);
+    CI->eraseFromParent();
+    return;
+  }
   case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
   case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
   case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
@@ -4822,10 +4890,22 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
   case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
   case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
   case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
-    // Create a new call with the correct address space.
-    SmallVector<Value *, 4> Args(CI->args());
-    Args[0] = Builder.CreateAddrSpaceCast(
-        Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
+    SmallVector<Value *, 16> Args(CI->args());
+
+    // Create AddrSpaceCast to shared_cluster if needed.
+    // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
+    unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
+    if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
+      Args[0] = Builder.CreateAddrSpaceCast(
+          Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
+
+    // Attach the flag argument for cta_group, with a
+    // default value of 0. This handles case (2) in
+    // shouldUpgradeNVPTXTMAG2SIntrinsics().
+    size_t NumArgs = CI->arg_size();
+    Value *FlagArg = CI->getArgOperand(NumArgs - 3);
+    if (!FlagArg->getType()->isIntegerTy(1))
+      Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));
 
     NewCall = Builder.CreateCall(NewFn, Args);
     NewCall->takeName(CI);
diff --git a/external/llvm-project/llvm/lib/IR/BasicBlock.cpp b/external/llvm-project/llvm/lib/IR/BasicBlock.cpp
index d56f4d399ea5..8b3e91750f86 100644
--- a/external/llvm-project/llvm/lib/IR/BasicBlock.cpp
+++ b/external/llvm-project/llvm/lib/IR/BasicBlock.cpp
@@ -22,6 +22,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 
 #include "LLVMContextImpl.h"
 
@@ -30,33 +31,7 @@ using namespace llvm;
 #define DEBUG_TYPE "ir"
 STATISTIC(NumInstrRenumberings, "Number of renumberings across all blocks");
 
-// This cl-opt exists to control whether variable-location information is
-// produced using intrinsics, or whether DbgRecords are produced. However,
-// it's imminently being phased out, so give it a flag-name that is very
-// unlikely to be used anywhere.
-//
-// If you find yourself needing to use this flag for any period longer than
-// five minutes, please revert the patch making this change, and make contact
-// in this discourse post, where we can discuss any further transition work
-// that might be needed to remove debug intrinsics.
-//
-// https://discourse.llvm.org/t/psa-ir-output-changing-from-debug-intrinsics-to-debug-records/79578
-cl::opt<bool> UseNewDbgInfoFormat(
-    "dont-pass-this-flag-please-experimental-debuginfo", cl::Hidden,
-    cl::init(true));
-
-// This cl-opt collects the --experimental-debuginfo-iterators flag and then
-// does nothing with it (because the it gets stored into an otherwise unused
-// cl-opt), so that we can disable debug-intrinsic production without
-// immediately modifying lots of tests. If your tests break because of this
-// change, please see the next comment up.
-static cl::opt<bool> DeliberatelyUnseenDbgInfoFlag(
-    "experimental-debuginfo-iterators", cl::Hidden,
-    cl::init(true));
-
 DbgMarker *BasicBlock::createMarker(Instruction *I) {
-  assert(IsNewDbgInfoFormat &&
-         "Tried to create a marker in a non new debug-info block!");
   if (I->DebugMarker)
     return I->DebugMarker;
   DbgMarker *Marker = new DbgMarker();
@@ -66,8 +41,6 @@ DbgMarker *BasicBlock::createMarker(Instruction *I) {
 }
 
 DbgMarker *BasicBlock::createMarker(InstListType::iterator It) {
-  assert(IsNewDbgInfoFormat &&
-         "Tried to create a marker in a non new debug-info block!");
   if (It != end())
     return createMarker(&*It);
   DbgMarker *DM = getTrailingDbgRecords();
@@ -79,15 +52,12 @@ DbgMarker *BasicBlock::createMarker(InstListType::iterator It) {
 }
 
 void BasicBlock::convertToNewDbgValues() {
-  IsNewDbgInfoFormat = true;
-
   // Iterate over all instructions in the instruction list, collecting debug
   // info intrinsics and converting them to DbgRecords. Once we find a "real"
   // instruction, attach all those DbgRecords to a DbgMarker in that
   // instruction.
   SmallVector<DbgRecord *, 4> DbgVarRecs;
   for (Instruction &I : make_early_inc_range(InstList)) {
-    assert(!I.DebugMarker && "DebugMarker already set on old-format instrs?");
     if (DbgVariableIntrinsic *DVI = dyn_cast<DbgVariableIntrinsic>(&I)) {
       // Convert this dbg.value to a DbgVariableRecord.
       DbgVariableRecord *Value = new DbgVariableRecord(DVI);
@@ -119,7 +89,6 @@ void BasicBlock::convertToNewDbgValues() {
 
 void BasicBlock::convertFromNewDbgValues() {
   invalidateOrders();
-  IsNewDbgInfoFormat = false;
 
   // Iterate over the block, finding instructions annotated with DbgMarkers.
   // Convert any attached DbgRecords to debug intrinsics and insert ahead of the
@@ -154,16 +123,6 @@ void BasicBlock::dumpDbgValues() const {
 }
 #endif
 
-void BasicBlock::setIsNewDbgInfoFormat(bool NewFlag) {
-  if (NewFlag && !IsNewDbgInfoFormat)
-    convertToNewDbgValues();
-  else if (!NewFlag && IsNewDbgInfoFormat)
-    convertFromNewDbgValues();
-}
-void BasicBlock::setNewDbgInfoFormatFlag(bool NewFlag) {
-  IsNewDbgInfoFormat = NewFlag;
-}
-
 ValueSymbolTable *BasicBlock::getValueSymbolTable() {
   if (Function *F = getParent())
     return F->getValueSymbolTable();
@@ -185,8 +144,7 @@ template class llvm::SymbolTableListTraits<
 
 BasicBlock::BasicBlock(LLVMContext &C, const Twine &Name, Function *NewParent,
                        BasicBlock *InsertBefore)
-    : Value(Type::getLabelTy(C), Value::BasicBlockVal),
-      IsNewDbgInfoFormat(UseNewDbgInfoFormat), Parent(nullptr) {
+    : Value(Type::getLabelTy(C), Value::BasicBlockVal), Parent(nullptr) {
 
   if (NewParent)
     insertInto(NewParent, InsertBefore);
@@ -196,8 +154,6 @@ BasicBlock::BasicBlock(LLVMContext &C, const Twine &Name, Function *NewParent,
 
   end().getNodePtr()->setParent(this);
   setName(Name);
-  if (NewParent)
-    setIsNewDbgInfoFormat(NewParent->IsNewDbgInfoFormat);
 }
 
 void BasicBlock::insertInto(Function *NewParent, BasicBlock *InsertBefore) {
@@ -208,8 +164,6 @@ void BasicBlock::insertInto(Function *NewParent, BasicBlock *InsertBefore) {
     NewParent->insert(InsertBefore->getIterator(), this);
   else
     NewParent->insert(NewParent->end(), this);
-
-  setIsNewDbgInfoFormat(NewParent->IsNewDbgInfoFormat);
 }
 
 BasicBlock::~BasicBlock() {
@@ -753,10 +707,6 @@ void BasicBlock::flushTerminatorDbgRecords() {
   // check whether there's anything trailing at the end and move those
   // DbgRecords in front of the terminator.
 
-  // Do nothing if we're not in new debug-info format.
-  if (!IsNewDbgInfoFormat)
-    return;
-
   // If there's no terminator, there's nothing to do.
   Instruction *Term = getTerminator();
   if (!Term)
@@ -793,10 +743,6 @@ void BasicBlock::spliceDebugInfoEmptyBlock(BasicBlock::iterator Dest,
   // in the iterators whether there was the intention to transfer any debug
   // info.
 
-  // If we're not in "new" debug-info format, do nothing.
-  if (!IsNewDbgInfoFormat)
-    return;
-
   assert(First == Last);
   bool InsertAtHead = Dest.getHeadBit();
   bool ReadFromHead = First.getHeadBit();
@@ -1057,8 +1003,6 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src,
 
 void BasicBlock::splice(iterator Dest, BasicBlock *Src, iterator First,
                         iterator Last) {
-  assert(Src->IsNewDbgInfoFormat == IsNewDbgInfoFormat);
-
 #ifdef EXPENSIVE_CHECKS
   // Check that First is before Last.
   auto FromBBEnd = Src->end();
@@ -1073,9 +1017,7 @@ void BasicBlock::splice(iterator Dest, BasicBlock *Src, iterator First,
     return;
   }
 
-  // Handle non-instr debug-info specific juggling.
-  if (IsNewDbgInfoFormat)
-    spliceDebugInfo(Dest, Src, First, Last);
+  spliceDebugInfo(Dest, Src, First, Last);
 
   // And move the instructions.
   getInstList().splice(Dest, Src->getInstList(), First, Last);
@@ -1084,7 +1026,6 @@ void BasicBlock::splice(iterator Dest, BasicBlock *Src, iterator First,
 }
 
 void BasicBlock::insertDbgRecordAfter(DbgRecord *DR, Instruction *I) {
-  assert(IsNewDbgInfoFormat);
   assert(I->getParent() == this);
 
   iterator NextIt = std::next(I->getIterator());
diff --git a/external/llvm-project/llvm/lib/IR/CMakeLists.txt b/external/llvm-project/llvm/lib/IR/CMakeLists.txt
index eb00829fd8c7..10572ff708bd 100644
--- a/external/llvm-project/llvm/lib/IR/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/IR/CMakeLists.txt
@@ -74,7 +74,6 @@ add_llvm_component_library(LLVMCore
   User.cpp
   Value.cpp
   ValueSymbolTable.cpp
-  VectorBuilder.cpp
   VectorTypeUtils.cpp
   Verifier.cpp
   VFABIDemangler.cpp
diff --git a/external/llvm-project/llvm/lib/IR/ConstantFold.cpp b/external/llvm-project/llvm/lib/IR/ConstantFold.cpp
index 7e5fda229b85..d4ad21e69e84 100644
--- a/external/llvm-project/llvm/lib/IR/ConstantFold.cpp
+++ b/external/llvm-project/llvm/lib/IR/ConstantFold.cpp
@@ -463,10 +463,9 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
     Constant *Elt =
         ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0));
 
-    if (Elt->isNullValue()) {
-      auto *VTy = VectorType::get(EltTy, MaskEltCount);
-      return ConstantAggregateZero::get(VTy);
-    } else if (!MaskEltCount.isScalable())
+    // For scalable vectors, make sure this doesn't fold back into a
+    // shufflevector.
+    if (!MaskEltCount.isScalable() || Elt->isNullValue() || isa<UndefValue>(Elt))
       return ConstantVector::getSplat(MaskEltCount, Elt);
   }
 
@@ -1149,10 +1148,10 @@ Constant *llvm::ConstantFoldCompareInstruction(CmpInst::Predicate Predicate,
   }
 
   // If the comparison is a comparison between two i1's, simplify it.
-  if (C1->getType()->isIntegerTy(1)) {
+  if (C1->getType()->isIntOrIntVectorTy(1)) {
     switch (Predicate) {
     case ICmpInst::ICMP_EQ:
-      if (isa<ConstantInt>(C2))
+      if (isa<ConstantExpr>(C1))
         return ConstantExpr::getXor(C1, ConstantExpr::getNot(C2));
       return ConstantExpr::getXor(ConstantExpr::getNot(C1), C2);
     case ICmpInst::ICMP_NE:
diff --git a/external/llvm-project/llvm/lib/IR/Core.cpp b/external/llvm-project/llvm/lib/IR/Core.cpp
index 1954b44af22a..9810f04cc503 100644
--- a/external/llvm-project/llvm/lib/IR/Core.cpp
+++ b/external/llvm-project/llvm/lib/IR/Core.cpp
@@ -431,12 +431,12 @@ void LLVMAddModuleFlag(LLVMModuleRef M, LLVMModuleFlagBehavior Behavior,
                            {Key, KeyLen}, unwrap(Val));
 }
 
-LLVMBool LLVMIsNewDbgInfoFormat(LLVMModuleRef M) {
-  return unwrap(M)->IsNewDbgInfoFormat;
-}
+LLVMBool LLVMIsNewDbgInfoFormat(LLVMModuleRef M) { return true; }
 
 void LLVMSetIsNewDbgInfoFormat(LLVMModuleRef M, LLVMBool UseNewFormat) {
-  unwrap(M)->setIsNewDbgInfoFormat(UseNewFormat);
+  if (!UseNewFormat)
+    llvm_unreachable("LLVM no longer supports intrinsic based debug-info");
+  (void)M;
 }
 
 /*--.. Printing modules ....................................................--*/
@@ -2108,8 +2108,10 @@ LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef Global) {
 
 unsigned LLVMGetAlignment(LLVMValueRef V) {
   Value *P = unwrap(V);
-  if (GlobalObject *GV = dyn_cast<GlobalObject>(P))
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(P))
     return GV->getAlign() ? GV->getAlign()->value() : 0;
+  if (Function *F = dyn_cast<Function>(P))
+    return F->getAlign() ? F->getAlign()->value() : 0;
   if (AllocaInst *AI = dyn_cast<AllocaInst>(P))
     return AI->getAlign().value();
   if (LoadInst *LI = dyn_cast<LoadInst>(P))
@@ -2128,8 +2130,10 @@ unsigned LLVMGetAlignment(LLVMValueRef V) {
 
 void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes) {
   Value *P = unwrap(V);
-  if (GlobalObject *GV = dyn_cast<GlobalObject>(P))
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(P))
     GV->setAlignment(MaybeAlign(Bytes));
+  else if (Function *F = dyn_cast<Function>(P))
+    F->setAlignment(MaybeAlign(Bytes));
   else if (AllocaInst *AI = dyn_cast<AllocaInst>(P))
     AI->setAlignment(Align(Bytes));
   else if (LoadInst *LI = dyn_cast<LoadInst>(P))
diff --git a/external/llvm-project/llvm/lib/IR/DIBuilder.cpp b/external/llvm-project/llvm/lib/IR/DIBuilder.cpp
index b60916ec3432..17a1f71148e5 100644
--- a/external/llvm-project/llvm/lib/IR/DIBuilder.cpp
+++ b/external/llvm-project/llvm/lib/IR/DIBuilder.cpp
@@ -25,8 +25,7 @@ using namespace llvm;
 using namespace llvm::dwarf;
 
 DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU)
-    : M(m), VMContext(M.getContext()), CUNode(CU), DeclareFn(nullptr),
-      ValueFn(nullptr), LabelFn(nullptr), AssignFn(nullptr),
+    : M(m), VMContext(M.getContext()), CUNode(CU),
       AllowUnresolvedNodes(AllowUnresolvedNodes) {
   if (CUNode) {
     if (const auto &ETs = CUNode->getEnumTypes())
@@ -442,6 +441,9 @@ DIDerivedType *DIBuilder::createVariantMemberType(
     DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
     uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
     Constant *Discriminant, DINode::DIFlags Flags, DIType *Ty) {
+  // "ExtraData" is overloaded for bit fields and for variants, so
+  // make sure to disallow this.
+  assert((Flags & DINode::FlagBitField) == 0);
   return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File,
                             LineNumber, getNonCompileUnitScope(Scope), Ty,
                             SizeInBits, AlignInBits, OffsetInBits, std::nullopt,
@@ -960,8 +962,7 @@ DISubprogram *DIBuilder::createFunction(
       SPFlags, IsDefinition ? CUNode : nullptr, TParams, Decl, nullptr,
       ThrownTypes, Annotations, TargetFuncName);
 
-  if (IsDefinition)
-    AllSubprograms.push_back(Node);
+  AllSubprograms.push_back(Node);
   trackIfUnresolved(Node);
   return Node;
 }
@@ -998,8 +999,7 @@ DISubprogram *DIBuilder::createMethod(
       Flags, SPFlags, IsDefinition ? CUNode : nullptr, TParams, nullptr,
       nullptr, ThrownTypes);
 
-  if (IsDefinition)
-    AllSubprograms.push_back(SP);
+  AllSubprograms.push_back(SP);
   trackIfUnresolved(SP);
   return SP;
 }
@@ -1066,36 +1066,13 @@ DbgInstPtr DIBuilder::insertDbgAssign(Instruction *LinkedInstr, Value *Val,
       LinkedInstr->getMetadata(LLVMContext::MD_DIAssignID));
   assert(Link && "Linked instruction must have DIAssign metadata attached");
 
-  if (M.IsNewDbgInfoFormat) {
-    DbgVariableRecord *DVR = DbgVariableRecord::createDVRAssign(
-        Val, SrcVar, ValExpr, Link, Addr, AddrExpr, DL);
-    // Insert after LinkedInstr.
-    BasicBlock::iterator NextIt = std::next(LinkedInstr->getIterator());
-    NextIt.setHeadBit(true);
-    insertDbgVariableRecord(DVR, NextIt);
-    return DVR;
-  }
-
-  LLVMContext &Ctx = LinkedInstr->getContext();
-  Module *M = LinkedInstr->getModule();
-  if (!AssignFn)
-    AssignFn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_assign);
-
-  std::array<Value *, 6> Args = {
-      MetadataAsValue::get(Ctx, ValueAsMetadata::get(Val)),
-      MetadataAsValue::get(Ctx, SrcVar),
-      MetadataAsValue::get(Ctx, ValExpr),
-      MetadataAsValue::get(Ctx, Link),
-      MetadataAsValue::get(Ctx, ValueAsMetadata::get(Addr)),
-      MetadataAsValue::get(Ctx, AddrExpr),
-  };
-
-  IRBuilder<> B(Ctx);
-  B.SetCurrentDebugLocation(DL);
-
-  auto *DVI = cast<DbgAssignIntrinsic>(B.CreateCall(AssignFn, Args));
-  DVI->insertAfter(LinkedInstr->getIterator());
-  return DVI;
+  DbgVariableRecord *DVR = DbgVariableRecord::createDVRAssign(
+      Val, SrcVar, ValExpr, Link, Addr, AddrExpr, DL);
+  // Insert after LinkedInstr.
+  BasicBlock::iterator NextIt = std::next(LinkedInstr->getIterator());
+  NextIt.setHeadBit(true);
+  insertDbgVariableRecord(DVR, NextIt);
+  return DVR;
 }
 
 /// Initialize IRBuilder for inserting dbg.declare and dbg.value intrinsics.
@@ -1111,27 +1088,15 @@ static Value *getDbgIntrinsicValueImpl(LLVMContext &VMContext, Value *V) {
   return MetadataAsValue::get(VMContext, ValueAsMetadata::get(V));
 }
 
-static Function *getDeclareIntrin(Module &M) {
-  return Intrinsic::getOrInsertDeclaration(&M, Intrinsic::dbg_declare);
-}
-
 DbgInstPtr DIBuilder::insertDbgValueIntrinsic(llvm::Value *Val,
                                               DILocalVariable *VarInfo,
                                               DIExpression *Expr,
                                               const DILocation *DL,
                                               InsertPosition InsertPt) {
-  if (M.IsNewDbgInfoFormat) {
-    DbgVariableRecord *DVR =
-        DbgVariableRecord::createDbgVariableRecord(Val, VarInfo, Expr, DL);
-    insertDbgVariableRecord(DVR, InsertPt);
-    return DVR;
-  }
-
-  if (!ValueFn)
-    ValueFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::dbg_value);
-  auto *DVI = insertDbgIntrinsic(ValueFn, Val, VarInfo, Expr, DL, InsertPt);
-  cast<CallInst>(DVI)->setTailCall();
-  return DVI;
+  DbgVariableRecord *DVR =
+      DbgVariableRecord::createDbgVariableRecord(Val, VarInfo, Expr, DL);
+  insertDbgVariableRecord(DVR, InsertPt);
+  return DVR;
 }
 
 DbgInstPtr DIBuilder::insertDeclare(Value *Storage, DILocalVariable *VarInfo,
@@ -1143,25 +1108,10 @@ DbgInstPtr DIBuilder::insertDeclare(Value *Storage, DILocalVariable *VarInfo,
              VarInfo->getScope()->getSubprogram() &&
          "Expected matching subprograms");
 
-  if (M.IsNewDbgInfoFormat) {
-    DbgVariableRecord *DVR =
-        DbgVariableRecord::createDVRDeclare(Storage, VarInfo, Expr, DL);
-    insertDbgVariableRecord(DVR, InsertPt);
-    return DVR;
-  }
-
-  if (!DeclareFn)
-    DeclareFn = getDeclareIntrin(M);
-
-  trackIfUnresolved(VarInfo);
-  trackIfUnresolved(Expr);
-  Value *Args[] = {getDbgIntrinsicValueImpl(VMContext, Storage),
-                   MetadataAsValue::get(VMContext, VarInfo),
-                   MetadataAsValue::get(VMContext, Expr)};
-
-  IRBuilder<> B(DL->getContext());
-  initIRBuilder(B, DL, InsertPt);
-  return B.CreateCall(DeclareFn, Args);
+  DbgVariableRecord *DVR =
+      DbgVariableRecord::createDVRDeclare(Storage, VarInfo, Expr, DL);
+  insertDbgVariableRecord(DVR, InsertPt);
+  return DVR;
 }
 
 void DIBuilder::insertDbgVariableRecord(DbgVariableRecord *DVR,
@@ -1210,23 +1160,12 @@ DbgInstPtr DIBuilder::insertLabel(DILabel *LabelInfo, const DILocation *DL,
          "Expected matching subprograms");
 
   trackIfUnresolved(LabelInfo);
-  if (M.IsNewDbgInfoFormat) {
-    DbgLabelRecord *DLR = new DbgLabelRecord(LabelInfo, DL);
-    if (InsertPt.isValid()) {
-      auto *BB = InsertPt.getBasicBlock();
-      BB->insertDbgRecordBefore(DLR, InsertPt);
-    }
-    return DLR;
+  DbgLabelRecord *DLR = new DbgLabelRecord(LabelInfo, DL);
+  if (InsertPt.isValid()) {
+    auto *BB = InsertPt.getBasicBlock();
+    BB->insertDbgRecordBefore(DLR, InsertPt);
   }
-
-  if (!LabelFn)
-    LabelFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::dbg_label);
-
-  Value *Args[] = {MetadataAsValue::get(VMContext, LabelInfo)};
-
-  IRBuilder<> B(DL->getContext());
-  initIRBuilder(B, DL, InsertPt);
-  return B.CreateCall(LabelFn, Args);
+  return DLR;
 }
 
 void DIBuilder::replaceVTableHolder(DICompositeType *&T, DIType *VTableHolder) {
diff --git a/external/llvm-project/llvm/lib/IR/DebugInfo.cpp b/external/llvm-project/llvm/lib/IR/DebugInfo.cpp
index 675dc6559e86..7e3813a4ea2b 100644
--- a/external/llvm-project/llvm/lib/IR/DebugInfo.cpp
+++ b/external/llvm-project/llvm/lib/IR/DebugInfo.cpp
@@ -961,8 +961,8 @@ unsigned llvm::getDebugMetadataVersionFromModule(const Module &M) {
   return 0;
 }
 
-void Instruction::applyMergedLocation(DILocation *LocA, DILocation *LocB) {
-  setDebugLoc(DILocation::getMergedLocation(LocA, LocB));
+void Instruction::applyMergedLocation(DebugLoc LocA, DebugLoc LocB) {
+  setDebugLoc(DebugLoc::getMergedLocation(LocA, LocB));
 }
 
 void Instruction::mergeDIAssignID(
@@ -2143,22 +2143,10 @@ static void emitDbgAssign(AssignmentInfo Info, Value *Val, Value *Dest,
     Expr = *R;
   }
   DIExpression *AddrExpr = DIExpression::get(StoreLikeInst.getContext(), {});
-  if (StoreLikeInst.getParent()->IsNewDbgInfoFormat) {
-    auto *Assign = DbgVariableRecord::createLinkedDVRAssign(
-        &StoreLikeInst, Val, VarRec.Var, Expr, Dest, AddrExpr, VarRec.DL);
-    (void)Assign;
-    LLVM_DEBUG(if (Assign) errs() << " > INSERT: " << *Assign << "\n");
-    return;
-  }
-  auto Assign = DIB.insertDbgAssign(&StoreLikeInst, Val, VarRec.Var, Expr, Dest,
-                                    AddrExpr, VarRec.DL);
+  auto *Assign = DbgVariableRecord::createLinkedDVRAssign(
+      &StoreLikeInst, Val, VarRec.Var, Expr, Dest, AddrExpr, VarRec.DL);
   (void)Assign;
-  LLVM_DEBUG(if (!Assign.isNull()) {
-    if (const auto *Record = dyn_cast<DbgRecord *>(Assign))
-      errs() << " > INSERT: " << *Record << "\n";
-    else
-      errs() << " > INSERT: " << *cast<Instruction *>(Assign) << "\n";
-  });
+  LLVM_DEBUG(if (Assign) errs() << " > INSERT: " << *Assign << "\n");
 }
 
 #undef DEBUG_TYPE // Silence redefinition warning (from ConstantsContext.h).
diff --git a/external/llvm-project/llvm/lib/IR/DebugInfoMetadata.cpp b/external/llvm-project/llvm/lib/IR/DebugInfoMetadata.cpp
index 7b8044ff7373..bde074ed8a07 100644
--- a/external/llvm-project/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/external/llvm-project/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -22,6 +22,7 @@
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 
 #include <numeric>
 #include <optional>
@@ -36,7 +37,7 @@ cl::opt<bool> EnableFSDiscriminator(
 
 // When true, preserves line and column number by picking one of the merged
 // location info in a deterministic manner to assist sample based PGO.
-cl::opt<bool> PickMergedSourceLocations(
+LLVM_ABI cl::opt<bool> PickMergedSourceLocations(
     "pick-merged-source-locations", cl::init(false), cl::Hidden,
     cl::desc("Preserve line and column number when merging locations."));
 } // namespace llvm
@@ -2883,6 +2884,84 @@ bool DIExpression::isPoisoned() const {
   });
 }
 
+namespace {
+/// Visitor specialization to find the divergent address spaces a DIOp-based
+/// DIExpression produces, if any. See the header comment on
+/// DIExpression::getNewDivergentAddrSpace() for more information.
+class DIOpDivergentAddrSpaceFinder
+    : public DIExprConstVisitor<DIOpDivergentAddrSpaceFinder> {
+
+  // Stack of dwarf stack entries with divergent address spaces. If a stack
+  // entry doesn't have a divergent address space, this contains std::nullopt
+  // for that stack element. Kept in sync with DIExprConstVisitor::Stack.
+  SmallVector<std::optional<unsigned>, 8> AddrSpaceStack;
+  Type *ResultTy = nullptr;
+
+  DIOpDivergentAddrSpaceFinder(LLVMContext &Ctx, ArrayRef<DIOp::Variant> Ops)
+      : DIExprConstVisitor(Ctx, Ops) {}
+
+public:
+  template <class DIOpTy>
+  bool visit(DIOpTy Op, Type *Ty, ArrayRef<StackEntry> Inputs) {
+    assert(Stack.size() == AddrSpaceStack.size() &&
+           "stacks should never get out of sync!");
+
+    if (isDIOpVariantOneOf<DIOp::Reinterpret>(Op)) {
+      // Nothing to do, Reinterpret operations don't change the divergent
+      // address space on the top of the stack.
+    } else if (isDIOpVariantOneOf<DIOp::Convert>(Op)) {
+      // If this Convert is an address space conversion, push a divergent
+      // address space unless we're already converting from a divergent address
+      // space or the conversion is a no-op.
+      Type *FromTy = Inputs[0].ResultType;
+      assert(Ty && FromTy && "failed to get operation types?");
+      if (FromTy->isPointerTy() && Ty->isPointerTy()) {
+        if (AddrSpaceStack.back() == std::nullopt && FromTy != Ty)
+          AddrSpaceStack.back() = FromTy->getPointerAddressSpace();
+      } else
+        AddrSpaceStack.back() = std::nullopt;
+    } else {
+      // No other operation can produce or maintain a divergent address space.
+      AddrSpaceStack.erase(AddrSpaceStack.end() - getNumInputs(Op),
+                           AddrSpaceStack.end());
+      if (Ty)
+        AddrSpaceStack.push_back(std::nullopt);
+    }
+
+    return DIExprConstVisitor::visit(Op, Ty, Inputs);
+  }
+
+  bool visitResult(StackEntry SE) {
+    ResultTy = SE.ResultType;
+    return true;
+  }
+
+  static std::optional<unsigned> find(LLVMContext &C,
+                                      ArrayRef<DIOp::Variant> Ops) {
+    DIOpDivergentAddrSpaceFinder Finder{C, Ops};
+    if (!Finder.visitInOrder())
+      return std::nullopt;
+    assert(Finder.AddrSpaceStack.size() == 1 &&
+           "expected one element on stack after expression!");
+    if (!Finder.ResultTy || !Finder.ResultTy->isPointerTy())
+      return std::nullopt;
+    // Only return a divergent address space when the expression produces a
+    // generic pointer.
+    unsigned DeclaredAddrSpace = Finder.ResultTy->getPointerAddressSpace();
+    if (Finder.AddrSpaceStack.back() && DeclaredAddrSpace == 0)
+      return Finder.AddrSpaceStack.back();
+    return std::nullopt;
+  }
+};
+} // namespace
+
+std::optional<unsigned> DIExpression::getNewDivergentAddrSpace() const {
+  auto Elems = getNewElementsRef();
+  if (!Elems || Elems->empty())
+    return std::nullopt;
+  return DIOpDivergentAddrSpaceFinder::find(getContext(), *Elems);
+}
+
 std::optional<DIExpression::SignedOrUnsignedConstant>
 DIExpression::isConstant() const {
 
diff --git a/external/llvm-project/llvm/lib/IR/DebugLoc.cpp b/external/llvm-project/llvm/lib/IR/DebugLoc.cpp
index 0e65ddcec893..ffeeeb6f1e4b 100644
--- a/external/llvm-project/llvm/lib/IR/DebugLoc.cpp
+++ b/external/llvm-project/llvm/lib/IR/DebugLoc.cpp
@@ -11,11 +11,11 @@
 #include "llvm/IR/DebugInfo.h"
 using namespace llvm;
 
-#if LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
 DILocAndCoverageTracking::DILocAndCoverageTracking(const DILocation *L)
     : TrackingMDNodeRef(const_cast<DILocation *>(L)),
       Kind(DebugLocKind::Normal) {}
-#endif // LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
 
 //===----------------------------------------------------------------------===//
 // DebugLoc Implementation
@@ -143,6 +143,27 @@ DebugLoc DebugLoc::appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt,
   return Last;
 }
 
+DebugLoc DebugLoc::getMergedLocations(ArrayRef<DebugLoc> Locs) {
+  if (Locs.empty())
+    return DebugLoc();
+  if (Locs.size() == 1)
+    return Locs[0];
+  DebugLoc Merged = Locs[0];
+  for (const DebugLoc &DL : llvm::drop_begin(Locs)) {
+    Merged = getMergedLocation(Merged, DL);
+    if (!Merged)
+      break;
+  }
+  return Merged;
+}
+DebugLoc DebugLoc::getMergedLocation(DebugLoc LocA, DebugLoc LocB) {
+  if (!LocA)
+    return LocA;
+  if (!LocB)
+    return LocB;
+  return DILocation::getMergedLocation(LocA, LocB);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void DebugLoc::dump() const { print(dbgs()); }
 #endif
diff --git a/external/llvm-project/llvm/lib/IR/DebugProgramInstruction.cpp b/external/llvm-project/llvm/lib/IR/DebugProgramInstruction.cpp
index 5d945d174dd4..089448a47a2f 100644
--- a/external/llvm-project/llvm/lib/IR/DebugProgramInstruction.cpp
+++ b/external/llvm-project/llvm/lib/IR/DebugProgramInstruction.cpp
@@ -6,10 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugProgramInstruction.h"
 #include "llvm/IR/DIBuilder.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -24,9 +25,9 @@ template <typename T> T *DbgRecordParamRef<T>::get() const {
   return cast<T>(Ref);
 }
 
-template class DbgRecordParamRef<DIExpression>;
-template class DbgRecordParamRef<DILabel>;
-template class DbgRecordParamRef<DILocalVariable>;
+template class LLVM_EXPORT_TEMPLATE DbgRecordParamRef<DIExpression>;
+template class LLVM_EXPORT_TEMPLATE DbgRecordParamRef<DILabel>;
+template class LLVM_EXPORT_TEMPLATE DbgRecordParamRef<DILocalVariable>;
 
 DbgVariableRecord::DbgVariableRecord(const DbgVariableIntrinsic *DVI)
     : DbgRecord(ValueKind, DVI->getDebugLoc()),
diff --git a/external/llvm-project/llvm/lib/IR/Dominators.cpp b/external/llvm-project/llvm/lib/IR/Dominators.cpp
index cc51b4905a62..e1e88bb2aaff 100644
--- a/external/llvm-project/llvm/lib/IR/Dominators.cpp
+++ b/external/llvm-project/llvm/lib/IR/Dominators.cpp
@@ -25,6 +25,7 @@
 #include "llvm/PassRegistry.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/GenericDomTreeConstruction.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -70,43 +71,55 @@ bool BasicBlockEdge::isSingleEdge() const {
 //
 //===----------------------------------------------------------------------===//
 
-template class llvm::DomTreeNodeBase<BasicBlock>;
-template class llvm::DominatorTreeBase<BasicBlock, false>; // DomTreeBase
-template class llvm::DominatorTreeBase<BasicBlock, true>; // PostDomTreeBase
+template class LLVM_EXPORT_TEMPLATE llvm::DomTreeNodeBase<BasicBlock>;
+template class LLVM_EXPORT_TEMPLATE
+    llvm::DominatorTreeBase<BasicBlock, false>; // DomTreeBase
+template class LLVM_EXPORT_TEMPLATE
+    llvm::DominatorTreeBase<BasicBlock, true>; // PostDomTreeBase
 
 template class llvm::cfg::Update<BasicBlock *>;
 
-template void llvm::DomTreeBuilder::Calculate<DomTreeBuilder::BBDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::Calculate<DomTreeBuilder::BBDomTree>(
     DomTreeBuilder::BBDomTree &DT);
-template void
+template LLVM_EXPORT_TEMPLATE void
 llvm::DomTreeBuilder::CalculateWithUpdates<DomTreeBuilder::BBDomTree>(
     DomTreeBuilder::BBDomTree &DT, BBUpdates U);
 
-template void llvm::DomTreeBuilder::Calculate<DomTreeBuilder::BBPostDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::Calculate<DomTreeBuilder::BBPostDomTree>(
     DomTreeBuilder::BBPostDomTree &DT);
 // No CalculateWithUpdates<PostDomTree> instantiation, unless a usecase arises.
 
-template void llvm::DomTreeBuilder::InsertEdge<DomTreeBuilder::BBDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::InsertEdge<DomTreeBuilder::BBDomTree>(
     DomTreeBuilder::BBDomTree &DT, BasicBlock *From, BasicBlock *To);
-template void llvm::DomTreeBuilder::InsertEdge<DomTreeBuilder::BBPostDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::InsertEdge<DomTreeBuilder::BBPostDomTree>(
     DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To);
 
-template void llvm::DomTreeBuilder::DeleteEdge<DomTreeBuilder::BBDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::DeleteEdge<DomTreeBuilder::BBDomTree>(
     DomTreeBuilder::BBDomTree &DT, BasicBlock *From, BasicBlock *To);
-template void llvm::DomTreeBuilder::DeleteEdge<DomTreeBuilder::BBPostDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::DeleteEdge<DomTreeBuilder::BBPostDomTree>(
     DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To);
 
-template void llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBDomTree>(
     DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBDomTreeGraphDiff &,
     DomTreeBuilder::BBDomTreeGraphDiff *);
-template void llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBPostDomTree>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBPostDomTree>(
     DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBPostDomTreeGraphDiff &,
     DomTreeBuilder::BBPostDomTreeGraphDiff *);
 
-template bool llvm::DomTreeBuilder::Verify<DomTreeBuilder::BBDomTree>(
+template LLVM_EXPORT_TEMPLATE bool
+llvm::DomTreeBuilder::Verify<DomTreeBuilder::BBDomTree>(
     const DomTreeBuilder::BBDomTree &DT,
     DomTreeBuilder::BBDomTree::VerificationLevel VL);
-template bool llvm::DomTreeBuilder::Verify<DomTreeBuilder::BBPostDomTree>(
+template LLVM_EXPORT_TEMPLATE bool
+llvm::DomTreeBuilder::Verify<DomTreeBuilder::BBPostDomTree>(
     const DomTreeBuilder::BBPostDomTree &DT,
     DomTreeBuilder::BBPostDomTree::VerificationLevel VL);
 
diff --git a/external/llvm-project/llvm/lib/IR/Function.cpp b/external/llvm-project/llvm/lib/IR/Function.cpp
index 85c28b020569..28fb81055baf 100644
--- a/external/llvm-project/llvm/lib/IR/Function.cpp
+++ b/external/llvm-project/llvm/lib/IR/Function.cpp
@@ -59,14 +59,12 @@ using ProfileCount = Function::ProfileCount;
 
 // Explicit instantiations of SymbolTableListTraits since some of the methods
 // are not in the public header file...
-template class llvm::SymbolTableListTraits<BasicBlock>;
+template class LLVM_EXPORT_TEMPLATE llvm::SymbolTableListTraits<BasicBlock>;
 
 static cl::opt<int> NonGlobalValueMaxNameSize(
     "non-global-value-max-name-size", cl::Hidden, cl::init(1024),
     cl::desc("Maximum size for the name of non-global values."));
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 void Function::renumberBlocks() {
   validateBlockNumbers();
 
@@ -89,32 +87,17 @@ void Function::validateBlockNumbers() const {
 }
 
 void Function::convertToNewDbgValues() {
-  IsNewDbgInfoFormat = true;
   for (auto &BB : *this) {
     BB.convertToNewDbgValues();
   }
 }
 
 void Function::convertFromNewDbgValues() {
-  IsNewDbgInfoFormat = false;
   for (auto &BB : *this) {
     BB.convertFromNewDbgValues();
   }
 }
 
-void Function::setIsNewDbgInfoFormat(bool NewFlag) {
-  if (NewFlag && !IsNewDbgInfoFormat)
-    convertToNewDbgValues();
-  else if (!NewFlag && IsNewDbgInfoFormat)
-    convertFromNewDbgValues();
-}
-void Function::setNewDbgInfoFormatFlag(bool NewFlag) {
-  for (auto &BB : *this) {
-    BB.setNewDbgInfoFormatFlag(NewFlag);
-  }
-  IsNewDbgInfoFormat = NewFlag;
-}
-
 //===----------------------------------------------------------------------===//
 // Argument Implementation
 //===----------------------------------------------------------------------===//
@@ -492,7 +475,7 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace,
                    const Twine &name, Module *ParentModule)
     : GlobalObject(Ty, Value::FunctionVal, AllocMarker, Linkage, name,
                    computeAddrSpace(AddrSpace, ParentModule)),
-      NumArgs(Ty->getNumParams()), IsNewDbgInfoFormat(UseNewDbgInfoFormat) {
+      NumArgs(Ty->getNumParams()) {
   assert(FunctionType::isValidReturnType(getReturnType()) &&
          "invalid return type");
   setGlobalObjectSubClassData(0);
@@ -507,7 +490,6 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace,
 
   if (ParentModule) {
     ParentModule->getFunctionList().push_back(this);
-    IsNewDbgInfoFormat = ParentModule->IsNewDbgInfoFormat;
   }
 
   HasLLVMReservedName = getName().starts_with("llvm.");
@@ -1167,6 +1149,18 @@ bool Function::nullPointerIsDefined() const {
   return hasFnAttribute(Attribute::NullPointerIsValid);
 }
 
+unsigned Function::getVScaleValue() const {
+  Attribute Attr = getFnAttribute(Attribute::VScaleRange);
+  if (!Attr.isValid())
+    return 0;
+
+  unsigned VScale = Attr.getVScaleRangeMin();
+  if (VScale && VScale == Attr.getVScaleRangeMax())
+    return VScale;
+
+  return 0;
+}
+
 bool llvm::NullPointerIsDefined(const Function *F, unsigned AS) {
   if (F && F->nullPointerIsDefined())
     return true;
diff --git a/external/llvm-project/llvm/lib/IR/IRBuilder.cpp b/external/llvm-project/llvm/lib/IR/IRBuilder.cpp
index 580b0af70933..a33ef9c7d4a1 100644
--- a/external/llvm-project/llvm/lib/IR/IRBuilder.cpp
+++ b/external/llvm-project/llvm/lib/IR/IRBuilder.cpp
@@ -61,19 +61,12 @@ Type *IRBuilderBase::getCurrentFunctionReturnType() const {
   return BB->getParent()->getReturnType();
 }
 
-DebugLoc IRBuilderBase::getCurrentDebugLocation() const {
-  for (auto &KV : MetadataToCopy)
-    if (KV.first == LLVMContext::MD_dbg)
-      return {cast<DILocation>(KV.second)};
-
-  return {};
-}
+DebugLoc IRBuilderBase::getCurrentDebugLocation() const { return StoredDL; }
 void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
-  for (const auto &KV : MetadataToCopy)
-    if (KV.first == LLVMContext::MD_dbg) {
-      I->setDebugLoc(DebugLoc(KV.second));
-      return;
-    }
+  // We prefer to set our current debug location if any has been set, but if
+  // our debug location is empty and I has a valid location, we shouldn't
+  // overwrite it.
+  I->setDebugLoc(StoredDL.orElse(I->getDebugLoc()));
 }
 
 Value *IRBuilderBase::CreateAggregateCast(Value *V, Type *DestTy) {
@@ -120,23 +113,26 @@ IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
   return CI;
 }
 
-Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
-  assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
-  if (cast<ConstantInt>(Scaling)->isZero())
-    return Scaling;
-  CallInst *CI =
-      CreateIntrinsic(Intrinsic::vscale, {Scaling->getType()}, {}, {}, Name);
-  return cast<ConstantInt>(Scaling)->isOne() ? CI : CreateMul(CI, Scaling);
+static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {
+  Value *VScale = B.CreateVScale(Ty);
+  if (Scale == 1)
+    return VScale;
+
+  return B.CreateMul(VScale, ConstantInt::get(Ty, Scale));
 }
 
-Value *IRBuilderBase::CreateElementCount(Type *DstType, ElementCount EC) {
-  Constant *MinEC = ConstantInt::get(DstType, EC.getKnownMinValue());
-  return EC.isScalable() ? CreateVScale(MinEC) : MinEC;
+Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {
+  if (EC.isFixed() || EC.isZero())
+    return ConstantInt::get(Ty, EC.getKnownMinValue());
+
+  return CreateVScaleMultiple(*this, Ty, EC.getKnownMinValue());
 }
 
-Value *IRBuilderBase::CreateTypeSize(Type *DstType, TypeSize Size) {
-  Constant *MinSize = ConstantInt::get(DstType, Size.getKnownMinValue());
-  return Size.isScalable() ? CreateVScale(MinSize) : MinSize;
+Value *IRBuilderBase::CreateTypeSize(Type *Ty, TypeSize Size) {
+  if (Size.isFixed() || Size.isZero())
+    return ConstantInt::get(Ty, Size.getKnownMinValue());
+
+  return CreateVScaleMultiple(*this, Ty, Size.getKnownMinValue());
 }
 
 Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
@@ -457,10 +453,10 @@ CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
 }
 
 static MaybeAlign getAlign(Value *Ptr) {
-  if (auto *O = dyn_cast<GlobalObject>(Ptr))
-    return O->getAlign();
+  if (auto *V = dyn_cast<GlobalVariable>(Ptr))
+    return V->getAlign();
   if (auto *A = dyn_cast<GlobalAlias>(Ptr))
-    return A->getAliaseeObject()->getAlign();
+    return getAlign(A->getAliaseeObject());
   return {};
 }
 
diff --git a/external/llvm-project/llvm/lib/IR/IRPrintingPasses.cpp b/external/llvm-project/llvm/lib/IR/IRPrintingPasses.cpp
index 96396d06ebba..5c062800198f 100644
--- a/external/llvm-project/llvm/lib/IR/IRPrintingPasses.cpp
+++ b/external/llvm-project/llvm/lib/IR/IRPrintingPasses.cpp
@@ -18,13 +18,12 @@
 #include "llvm/IR/PrintPasses.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace {
 
 class PrintModulePassWrapper : public ModulePass {
@@ -41,12 +40,9 @@ class PrintModulePassWrapper : public ModulePass {
         ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {}
 
   bool runOnModule(Module &M) override {
-    ScopedDbgInfoFormatSetter FormatSetter(M, UseNewDbgInfoFormat);
     // Remove intrinsic declarations when printing in the new format.
-    // TODO: Move this into Module::setIsNewDbgInfoFormat when we're ready to
-    // update test output.
-    if (UseNewDbgInfoFormat)
-      M.removeDebugIntrinsicDeclarations();
+    // TODO: consider removing this as debug-intrinsics are gone.
+    M.removeDebugIntrinsicDeclarations();
 
     if (llvm::isFunctionInPrintList("*")) {
       if (!Banner.empty())
@@ -87,8 +83,6 @@ class PrintFunctionPassWrapper : public FunctionPass {
 
   // This pass just prints a banner followed by the function as it's processed.
   bool runOnFunction(Function &F) override {
-    ScopedDbgInfoFormatSetter FormatSetter(F, UseNewDbgInfoFormat);
-
     if (isFunctionInPrintList(F.getName())) {
       if (forcePrintModuleIR())
         OS << Banner << " (function: " << F.getName() << ")\n"
diff --git a/external/llvm-project/llvm/lib/IR/Instruction.cpp b/external/llvm-project/llvm/lib/IR/Instruction.cpp
index 54e5e6d53e79..cbf39b8adf1b 100644
--- a/external/llvm-project/llvm/lib/IR/Instruction.cpp
+++ b/external/llvm-project/llvm/lib/IR/Instruction.cpp
@@ -26,6 +26,7 @@
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/ProfDataUtils.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/Compiler.h"
 using namespace llvm;
 
 InsertPosition::InsertPosition(Instruction *InsertBefore)
@@ -85,7 +86,7 @@ void Instruction::removeFromParent() {
 }
 
 void Instruction::handleMarkerRemoval() {
-  if (!getParent()->IsNewDbgInfoFormat || !DebugMarker)
+  if (!DebugMarker)
     return;
 
   DebugMarker->removeMarker();
@@ -129,17 +130,12 @@ BasicBlock::iterator Instruction::insertInto(BasicBlock *ParentBB,
   return getIterator();
 }
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 void Instruction::insertBefore(BasicBlock &BB,
                                InstListType::iterator InsertPos) {
   assert(!DebugMarker);
 
   BB.getInstList().insert(InsertPos, this);
 
-  if (!BB.IsNewDbgInfoFormat)
-    return;
-
   // We've inserted "this": if InsertAtHead is set then it comes before any
   // DbgVariableRecords attached to InsertPos. But if it's not set, then any
   // DbgRecords should now come before "this".
@@ -190,15 +186,22 @@ void Instruction::moveBeforePreserving(BasicBlock::iterator MovePos) {
 
 void Instruction::moveAfter(Instruction *MovePos) {
   auto NextIt = std::next(MovePos->getIterator());
-  // We want this instruction to be moved to before NextIt in the instruction
+  // We want this instruction to be moved to after NextIt in the instruction
   // list, but before NextIt's debug value range.
   NextIt.setHeadBit(true);
   moveBeforeImpl(*MovePos->getParent(), NextIt, false);
 }
 
+void Instruction::moveAfter(InstListType::iterator MovePos) {
+  // We want this instruction to be moved to after NextIt in the instruction
+  // list, but before NextIt's debug value range.
+  MovePos.setHeadBit(true);
+  moveBeforeImpl(*MovePos->getParent(), MovePos, false);
+}
+
 void Instruction::moveAfterPreserving(Instruction *MovePos) {
   auto NextIt = std::next(MovePos->getIterator());
-  // We want this instruction and its debug range to be moved to before NextIt
+  // We want this instruction and its debug range to be moved to after NextIt
   // in the instruction list, but before NextIt's debug value range.
   NextIt.setHeadBit(true);
   moveBeforeImpl(*MovePos->getParent(), NextIt, true);
@@ -220,7 +223,7 @@ void Instruction::moveBeforeImpl(BasicBlock &BB, InstListType::iterator I,
 
   // If we've been given the "Preserve" flag, then just move the DbgRecords with
   // the instruction, no more special handling needed.
-  if (BB.IsNewDbgInfoFormat && DebugMarker && !Preserve) {
+  if (DebugMarker && !Preserve) {
     if (I != this->getIterator() || InsertAtHead) {
       // "this" is definitely moving in the list, or it's moving ahead of its
       // attached DbgVariableRecords. Detach any existing DbgRecords.
@@ -232,7 +235,7 @@ void Instruction::moveBeforeImpl(BasicBlock &BB, InstListType::iterator I,
   // the block splicer, which will do more debug-info things.
   BB.getInstList().splice(I, getParent()->getInstList(), getIterator());
 
-  if (BB.IsNewDbgInfoFormat && !Preserve) {
+  if (!Preserve) {
     DbgMarker *NextMarker = getParent()->getNextMarker(this);
 
     // If we're inserting at point I, and not in front of the DbgRecords
@@ -252,10 +255,6 @@ iterator_range<DbgRecord::self_iterator> Instruction::cloneDebugInfoFrom(
   if (!From->DebugMarker)
     return DbgMarker::getEmptyDbgRecordRange();
 
-  assert(getParent()->IsNewDbgInfoFormat);
-  assert(getParent()->IsNewDbgInfoFormat ==
-         From->getParent()->IsNewDbgInfoFormat);
-
   if (!DebugMarker)
     getParent()->createMarker(this);
 
@@ -1355,6 +1354,9 @@ void Instruction::swapProfMetadata() {
 
 void Instruction::copyMetadata(const Instruction &SrcInst,
                                ArrayRef<unsigned> WL) {
+  if (WL.empty() || is_contained(WL, LLVMContext::MD_dbg))
+    setDebugLoc(SrcInst.getDebugLoc().orElse(getDebugLoc()));
+
   if (!SrcInst.hasMetadata())
     return;
 
@@ -1368,8 +1370,6 @@ void Instruction::copyMetadata(const Instruction &SrcInst,
     if (WL.empty() || WLS.count(MD.first))
       setMetadata(MD.first, MD.second);
   }
-  if (WL.empty() || WLS.count(LLVMContext::MD_dbg))
-    setDebugLoc(SrcInst.getDebugLoc());
 }
 
 Instruction *Instruction::clone() const {
diff --git a/external/llvm-project/llvm/lib/IR/Instructions.cpp b/external/llvm-project/llvm/lib/IR/Instructions.cpp
index f404e11b9c0f..2d89ec1b0a8d 100644
--- a/external/llvm-project/llvm/lib/IR/Instructions.cpp
+++ b/external/llvm-project/llvm/lib/IR/Instructions.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CheckedArithmetic.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/MathExtras.h"
@@ -448,7 +449,7 @@ bool CallBase::paramHasNonNullAttr(unsigned ArgNo,
       (AllowUndefOrPoison || paramHasAttr(ArgNo, Attribute::NoUndef)))
     return true;
 
-  if (getParamDereferenceableBytes(ArgNo) > 0 &&
+  if (paramHasAttr(ArgNo, Attribute::Dereferenceable) &&
       !NullPointerIsDefined(
           getCaller(),
           getArgOperand(ArgNo)->getType()->getPointerAddressSpace()))
@@ -485,9 +486,10 @@ Attribute CallBase::getFnAttrOnCalledFunction(AK Kind) const {
   return Attribute();
 }
 
-template Attribute
+template LLVM_ABI Attribute
 CallBase::getFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
-template Attribute CallBase::getFnAttrOnCalledFunction(StringRef Kind) const;
+template LLVM_ABI Attribute
+CallBase::getFnAttrOnCalledFunction(StringRef Kind) const;
 
 template <typename AK>
 Attribute CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
@@ -499,11 +501,10 @@ Attribute CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
 
   return Attribute();
 }
-template Attribute
-CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
-                                       Attribute::AttrKind Kind) const;
-template Attribute CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
-                                                          StringRef Kind) const;
+template LLVM_ABI Attribute CallBase::getParamAttrOnCalledFunction(
+    unsigned ArgNo, Attribute::AttrKind Kind) const;
+template LLVM_ABI Attribute
+CallBase::getParamAttrOnCalledFunction(unsigned ArgNo, StringRef Kind) const;
 
 void CallBase::getOperandBundlesAsDefs(
     SmallVectorImpl<OperandBundleDef> &Defs) const {
@@ -1853,23 +1854,18 @@ void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
                                        SmallVectorImpl<int> &Result) {
   ElementCount EC = cast<VectorType>(Mask->getType())->getElementCount();
 
-  if (isa<ConstantAggregateZero>(Mask)) {
-    Result.resize(EC.getKnownMinValue(), 0);
+  if (isa<ConstantAggregateZero>(Mask) || isa<UndefValue>(Mask)) {
+    int MaskVal = isa<UndefValue>(Mask) ? -1 : 0;
+    Result.append(EC.getKnownMinValue(), MaskVal);
     return;
   }
 
-  Result.reserve(EC.getKnownMinValue());
+  assert(!EC.isScalable() &&
+         "Scalable vector shuffle mask must be undef or zeroinitializer");
 
-  if (EC.isScalable()) {
-    assert((isa<ConstantAggregateZero>(Mask) || isa<UndefValue>(Mask)) &&
-           "Scalable vector shuffle mask must be undef or zeroinitializer");
-    int MaskVal = isa<UndefValue>(Mask) ? -1 : 0;
-    for (unsigned I = 0; I < EC.getKnownMinValue(); ++I)
-      Result.emplace_back(MaskVal);
-    return;
-  }
+  unsigned NumElts = EC.getFixedValue();
 
-  unsigned NumElts = EC.getKnownMinValue();
+  Result.reserve(NumElts);
 
   if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) {
     for (unsigned i = 0; i != NumElts; ++i)
diff --git a/external/llvm-project/llvm/lib/IR/LLVMContextImpl.cpp b/external/llvm-project/llvm/lib/IR/LLVMContextImpl.cpp
index ace36ffee26b..1f5e78732761 100644
--- a/external/llvm-project/llvm/lib/IR/LLVMContextImpl.cpp
+++ b/external/llvm-project/llvm/lib/IR/LLVMContextImpl.cpp
@@ -12,7 +12,6 @@
 
 #include "LLVMContextImpl.h"
 #include "AttributeImpl.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringMapEntry.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/IR/DiagnosticHandler.h"
diff --git a/external/llvm-project/llvm/lib/IR/LLVMContextImpl.h b/external/llvm-project/llvm/lib/IR/LLVMContextImpl.h
index 0e7e4f68f3f8..6808fac673f1 100644
--- a/external/llvm-project/llvm/lib/IR/LLVMContextImpl.h
+++ b/external/llvm-project/llvm/lib/IR/LLVMContextImpl.h
@@ -310,36 +310,33 @@ template <> struct MDNodeKeyImpl<MDTuple> : MDNodeOpsKey {
 
 /// DenseMapInfo for DILocation.
 template <> struct MDNodeKeyImpl<DILocation> {
-  unsigned Line;
-  unsigned Column;
   Metadata *Scope;
   Metadata *InlinedAt;
-  bool ImplicitCode;
 #ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
   uint64_t AtomGroup : 61;
   uint64_t AtomRank : 3;
 #endif
+  unsigned Line;
+  uint16_t Column;
+  bool ImplicitCode;
 
-  MDNodeKeyImpl(unsigned Line, unsigned Column, Metadata *Scope,
+  MDNodeKeyImpl(unsigned Line, uint16_t Column, Metadata *Scope,
                 Metadata *InlinedAt, bool ImplicitCode, uint64_t AtomGroup,
                 uint8_t AtomRank)
-      : Line(Line), Column(Column), Scope(Scope), InlinedAt(InlinedAt),
-        ImplicitCode(ImplicitCode)
+      : Scope(Scope), InlinedAt(InlinedAt),
 #ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-        ,
-        AtomGroup(AtomGroup), AtomRank(AtomRank)
+        AtomGroup(AtomGroup), AtomRank(AtomRank),
 #endif
-  {
+        Line(Line), Column(Column), ImplicitCode(ImplicitCode) {
   }
 
   MDNodeKeyImpl(const DILocation *L)
-      : Line(L->getLine()), Column(L->getColumn()), Scope(L->getRawScope()),
-        InlinedAt(L->getRawInlinedAt()), ImplicitCode(L->isImplicitCode())
+      : Scope(L->getRawScope()), InlinedAt(L->getRawInlinedAt()),
 #ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-        ,
-        AtomGroup(L->getAtomGroup()), AtomRank(L->getAtomRank())
+        AtomGroup(L->getAtomGroup()), AtomRank(L->getAtomRank()),
 #endif
-  {
+        Line(L->getLine()), Column(L->getColumn()),
+        ImplicitCode(L->isImplicitCode()) {
   }
 
   bool isKeyOf(const DILocation *RHS) const {
@@ -355,13 +352,19 @@ template <> struct MDNodeKeyImpl<DILocation> {
   }
 
   unsigned getHashValue() const {
-    return hash_combine(Line, Column, Scope, InlinedAt, ImplicitCode
 #ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-                        ,
-                        AtomGroup, (uint8_t)AtomRank);
-#else
-    );
+    // Hashing AtomGroup and AtomRank substantially impacts performance whether
+    // Key Instructions is enabled or not. We can't detect whether it's enabled
+    // here cheaply; avoiding hashing zero values is a good approximation. This
+    // affects Key Instruction builds too, but any potential costs incurred by
+    // messing with the hash distribution* appear to still be massively
+    // outweighed by the overall compile time savings by performing this check.
+    // * (hash_combine(x) != hash_combine(x, 0))
+    if (AtomGroup || AtomRank)
+      return hash_combine(Line, Column, Scope, InlinedAt, ImplicitCode,
+                          AtomGroup, (uint8_t)AtomRank);
 #endif
+    return hash_combine(Line, Column, Scope, InlinedAt, ImplicitCode);
   }
 };
 
diff --git a/external/llvm-project/llvm/lib/IR/LegacyPassManager.cpp b/external/llvm-project/llvm/lib/IR/LegacyPassManager.cpp
index 9ac6e9c1e97e..c8f1606ea06c 100644
--- a/external/llvm-project/llvm/lib/IR/LegacyPassManager.cpp
+++ b/external/llvm-project/llvm/lib/IR/LegacyPassManager.cpp
@@ -21,6 +21,7 @@
 #include "llvm/IR/PrintPasses.h"
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -31,7 +32,6 @@
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
 // See PassManagers.h for Pass Manager infrastructure overview.
 
 //===----------------------------------------------------------------------===//
@@ -526,11 +526,6 @@ bool PassManagerImpl::run(Module &M) {
   dumpArguments();
   dumpPasses();
 
-  // RemoveDIs: if a command line flag is given, convert to the
-  // DbgVariableRecord representation of debug-info for the duration of these
-  // passes.
-  ScopedDbgInfoFormatSetter FormatSetter(M, UseNewDbgInfoFormat);
-
   for (ImmutablePass *ImPass : getImmutablePasses())
     Changed |= ImPass->doInitialization(M);
 
diff --git a/external/llvm-project/llvm/lib/IR/Module.cpp b/external/llvm-project/llvm/lib/IR/Module.cpp
index eb635dc29aee..37f4a72d8c20 100644
--- a/external/llvm-project/llvm/lib/IR/Module.cpp
+++ b/external/llvm-project/llvm/lib/IR/Module.cpp
@@ -39,6 +39,7 @@
 #include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
@@ -53,18 +54,16 @@
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 //===----------------------------------------------------------------------===//
 // Methods to implement the globals and functions lists.
 //
 
 // Explicit instantiations of SymbolTableListTraits since some of the methods
 // are not in the public header file.
-template class llvm::SymbolTableListTraits<Function>;
-template class llvm::SymbolTableListTraits<GlobalVariable>;
-template class llvm::SymbolTableListTraits<GlobalAlias>;
-template class llvm::SymbolTableListTraits<GlobalIFunc>;
+template class LLVM_EXPORT_TEMPLATE llvm::SymbolTableListTraits<Function>;
+template class LLVM_EXPORT_TEMPLATE llvm::SymbolTableListTraits<GlobalVariable>;
+template class LLVM_EXPORT_TEMPLATE llvm::SymbolTableListTraits<GlobalAlias>;
+template class LLVM_EXPORT_TEMPLATE llvm::SymbolTableListTraits<GlobalIFunc>;
 
 //===----------------------------------------------------------------------===//
 // Primitive Module methods.
@@ -72,8 +71,7 @@ template class llvm::SymbolTableListTraits<GlobalIFunc>;
 
 Module::Module(StringRef MID, LLVMContext &C)
     : Context(C), ValSymTab(std::make_unique<ValueSymbolTable>(-1)),
-      ModuleID(std::string(MID)), SourceFileName(std::string(MID)),
-      IsNewDbgInfoFormat(UseNewDbgInfoFormat) {
+      ModuleID(std::string(MID)), SourceFileName(std::string(MID)) {
   Context.addModule(this);
 }
 
@@ -84,7 +82,6 @@ Module &Module::operator=(Module &&Other) {
 
   ModuleID = std::move(Other.ModuleID);
   SourceFileName = std::move(Other.SourceFileName);
-  IsNewDbgInfoFormat = std::move(Other.IsNewDbgInfoFormat);
 
   GlobalList.clear();
   GlobalList.splice(GlobalList.begin(), Other.GlobalList);
diff --git a/external/llvm-project/llvm/lib/IR/PassManager.cpp b/external/llvm-project/llvm/lib/IR/PassManager.cpp
index bf8f7906d336..af22b779ddbb 100644
--- a/external/llvm-project/llvm/lib/IR/PassManager.cpp
+++ b/external/llvm-project/llvm/lib/IR/PassManager.cpp
@@ -9,6 +9,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManagerImpl.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 using namespace llvm;
@@ -16,14 +17,16 @@ using namespace llvm;
 namespace llvm {
 // Explicit template instantiations and specialization defininitions for core
 // template typedefs.
-template class AllAnalysesOn<Module>;
-template class AllAnalysesOn<Function>;
-template class PassManager<Module>;
-template class PassManager<Function>;
-template class AnalysisManager<Module>;
-template class AnalysisManager<Function>;
-template class InnerAnalysisManagerProxy<FunctionAnalysisManager, Module>;
-template class OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>;
+template class LLVM_EXPORT_TEMPLATE AllAnalysesOn<Module>;
+template class LLVM_EXPORT_TEMPLATE AllAnalysesOn<Function>;
+template class LLVM_EXPORT_TEMPLATE PassManager<Module>;
+template class LLVM_EXPORT_TEMPLATE PassManager<Function>;
+template class LLVM_EXPORT_TEMPLATE AnalysisManager<Module>;
+template class LLVM_EXPORT_TEMPLATE AnalysisManager<Function>;
+template class LLVM_EXPORT_TEMPLATE
+    InnerAnalysisManagerProxy<FunctionAnalysisManager, Module>;
+template class LLVM_EXPORT_TEMPLATE
+    OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>;
 
 template <>
 bool FunctionAnalysisManagerModuleProxy::Result::invalidate(
diff --git a/external/llvm-project/llvm/lib/IR/RuntimeLibcalls.cpp b/external/llvm-project/llvm/lib/IR/RuntimeLibcalls.cpp
index 90c3bf0db023..d63d398e243f 100644
--- a/external/llvm-project/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/external/llvm-project/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -7,10 +7,219 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/RuntimeLibcalls.h"
+#include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
 using namespace RTLIB;
 
+static cl::opt<bool>
+    HexagonEnableFastMathRuntimeCalls("hexagon-fast-math", cl::Hidden,
+                                      cl::desc("Enable Fast Math processing"));
+
+static void setAArch64LibcallNames(RuntimeLibcallsInfo &Info,
+                                   const Triple &TT) {
+  if (TT.isWindowsArm64EC()) {
+    // FIXME: are there calls we need to exclude from this?
+#define HANDLE_LIBCALL(code, name)                                             \
+  if (sizeof(name) != 1) {                                                     \
+    const char *libcallName = Info.getLibcallName(RTLIB::code);                \
+    if (libcallName && libcallName[0] != '#') {                                \
+      assert(strcmp(libcallName, name) == 0 && "Unexpected name");             \
+      Info.setLibcallName(RTLIB::code, "#" name);                              \
+    }                                                                          \
+  }
+#define LIBCALL_NO_NAME ""
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+#undef LIBCALL_NO_NAME
+  }
+}
+
+static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT) {
+  // Register based DivRem for AEABI (RTABI 4.2)
+  if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
+      TT.isTargetMuslAEABI() || TT.isOSWindows()) {
+    if (TT.isOSWindows()) {
+      const struct {
+        const RTLIB::Libcall Op;
+        const char *const Name;
+        const CallingConv::ID CC;
+      } LibraryCalls[] = {
+          {RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS},
+          {RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS},
+      };
+
+      for (const auto &LC : LibraryCalls) {
+        Info.setLibcallName(LC.Op, LC.Name);
+        Info.setLibcallCallingConv(LC.Op, LC.CC);
+      }
+    } else {
+      const struct {
+        const RTLIB::Libcall Op;
+        const char *const Name;
+        const CallingConv::ID CC;
+      } LibraryCalls[] = {
+          {RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS},
+          {RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS},
+      };
+
+      for (const auto &LC : LibraryCalls) {
+        Info.setLibcallName(LC.Op, LC.Name);
+        Info.setLibcallCallingConv(LC.Op, LC.CC);
+      }
+    }
+  }
+
+  if (TT.isOSWindows()) {
+    static const struct {
+      const RTLIB::Libcall Op;
+      const char *const Name;
+      const CallingConv::ID CC;
+    } LibraryCalls[] = {
+        {RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP},
+    };
+
+    for (const auto &LC : LibraryCalls) {
+      Info.setLibcallName(LC.Op, LC.Name);
+      Info.setLibcallCallingConv(LC.Op, LC.CC);
+    }
+  }
+
+  // Use divmod compiler-rt calls for iOS 5.0 and later.
+  if (TT.isOSBinFormatMachO() && (!TT.isiOS() || !TT.isOSVersionLT(5, 0))) {
+    Info.setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
+    Info.setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
+  }
+}
+
+static void setMSP430Libcalls(RuntimeLibcallsInfo &Info, const Triple &TT) {
+  // EABI Libcalls - EABI Section 6.2
+  const struct {
+    const RTLIB::Libcall Op;
+    const char *const Name;
+  } LibraryCalls[] = {
+      // Floating point conversions - EABI Table 6
+      {RTLIB::FPROUND_F64_F32, "__mspabi_cvtdf"},
+      {RTLIB::FPEXT_F32_F64, "__mspabi_cvtfd"},
+      // The following is NOT implemented in libgcc
+      //{ RTLIB::FPTOSINT_F64_I16,  "__mspabi_fixdi" },
+      {RTLIB::FPTOSINT_F64_I32, "__mspabi_fixdli"},
+      {RTLIB::FPTOSINT_F64_I64, "__mspabi_fixdlli"},
+      // The following is NOT implemented in libgcc
+      //{ RTLIB::FPTOUINT_F64_I16,  "__mspabi_fixdu" },
+      {RTLIB::FPTOUINT_F64_I32, "__mspabi_fixdul"},
+      {RTLIB::FPTOUINT_F64_I64, "__mspabi_fixdull"},
+      // The following is NOT implemented in libgcc
+      //{ RTLIB::FPTOSINT_F32_I16,  "__mspabi_fixfi" },
+      {RTLIB::FPTOSINT_F32_I32, "__mspabi_fixfli"},
+      {RTLIB::FPTOSINT_F32_I64, "__mspabi_fixflli"},
+      // The following is NOT implemented in libgcc
+      //{ RTLIB::FPTOUINT_F32_I16,  "__mspabi_fixfu" },
+      {RTLIB::FPTOUINT_F32_I32, "__mspabi_fixful"},
+      {RTLIB::FPTOUINT_F32_I64, "__mspabi_fixfull"},
+      // TODO The following IS implemented in libgcc
+      //{ RTLIB::SINTTOFP_I16_F64,  "__mspabi_fltid" },
+      {RTLIB::SINTTOFP_I32_F64, "__mspabi_fltlid"},
+      // TODO The following IS implemented in libgcc but is not in the EABI
+      {RTLIB::SINTTOFP_I64_F64, "__mspabi_fltllid"},
+      // TODO The following IS implemented in libgcc
+      //{ RTLIB::UINTTOFP_I16_F64,  "__mspabi_fltud" },
+      {RTLIB::UINTTOFP_I32_F64, "__mspabi_fltuld"},
+      // The following IS implemented in libgcc but is not in the EABI
+      {RTLIB::UINTTOFP_I64_F64, "__mspabi_fltulld"},
+      // TODO The following IS implemented in libgcc
+      //{ RTLIB::SINTTOFP_I16_F32,  "__mspabi_fltif" },
+      {RTLIB::SINTTOFP_I32_F32, "__mspabi_fltlif"},
+      // TODO The following IS implemented in libgcc but is not in the EABI
+      {RTLIB::SINTTOFP_I64_F32, "__mspabi_fltllif"},
+      // TODO The following IS implemented in libgcc
+      //{ RTLIB::UINTTOFP_I16_F32,  "__mspabi_fltuf" },
+      {RTLIB::UINTTOFP_I32_F32, "__mspabi_fltulf"},
+      // The following IS implemented in libgcc but is not in the EABI
+      {RTLIB::UINTTOFP_I64_F32, "__mspabi_fltullf"},
+
+      // Floating point comparisons - EABI Table 7
+      {RTLIB::OEQ_F64, "__mspabi_cmpd"},
+      {RTLIB::UNE_F64, "__mspabi_cmpd"},
+      {RTLIB::OGE_F64, "__mspabi_cmpd"},
+      {RTLIB::OLT_F64, "__mspabi_cmpd"},
+      {RTLIB::OLE_F64, "__mspabi_cmpd"},
+      {RTLIB::OGT_F64, "__mspabi_cmpd"},
+      {RTLIB::OEQ_F32, "__mspabi_cmpf"},
+      {RTLIB::UNE_F32, "__mspabi_cmpf"},
+      {RTLIB::OGE_F32, "__mspabi_cmpf"},
+      {RTLIB::OLT_F32, "__mspabi_cmpf"},
+      {RTLIB::OLE_F32, "__mspabi_cmpf"},
+      {RTLIB::OGT_F32, "__mspabi_cmpf"},
+
+      // Floating point arithmetic - EABI Table 8
+      {RTLIB::ADD_F64, "__mspabi_addd"},
+      {RTLIB::ADD_F32, "__mspabi_addf"},
+      {RTLIB::DIV_F64, "__mspabi_divd"},
+      {RTLIB::DIV_F32, "__mspabi_divf"},
+      {RTLIB::MUL_F64, "__mspabi_mpyd"},
+      {RTLIB::MUL_F32, "__mspabi_mpyf"},
+      {RTLIB::SUB_F64, "__mspabi_subd"},
+      {RTLIB::SUB_F32, "__mspabi_subf"},
+      // The following are NOT implemented in libgcc
+      // { RTLIB::NEG_F64,  "__mspabi_negd" },
+      // { RTLIB::NEG_F32,  "__mspabi_negf" },
+
+      // Universal Integer Operations - EABI Table 9
+      {RTLIB::SDIV_I16, "__mspabi_divi"},
+      {RTLIB::SDIV_I32, "__mspabi_divli"},
+      {RTLIB::SDIV_I64, "__mspabi_divlli"},
+      {RTLIB::UDIV_I16, "__mspabi_divu"},
+      {RTLIB::UDIV_I32, "__mspabi_divul"},
+      {RTLIB::UDIV_I64, "__mspabi_divull"},
+      {RTLIB::SREM_I16, "__mspabi_remi"},
+      {RTLIB::SREM_I32, "__mspabi_remli"},
+      {RTLIB::SREM_I64, "__mspabi_remlli"},
+      {RTLIB::UREM_I16, "__mspabi_remu"},
+      {RTLIB::UREM_I32, "__mspabi_remul"},
+      {RTLIB::UREM_I64, "__mspabi_remull"},
+
+      // Bitwise Operations - EABI Table 10
+      // TODO: __mspabi_[srli/srai/slli] ARE implemented in libgcc
+      {RTLIB::SRL_I32, "__mspabi_srll"},
+      {RTLIB::SRA_I32, "__mspabi_sral"},
+      {RTLIB::SHL_I32, "__mspabi_slll"},
+      // __mspabi_[srlll/srall/sllll/rlli/rlll] are NOT implemented in libgcc
+  };
+
+  for (const auto &LC : LibraryCalls)
+    Info.setLibcallName(LC.Op, LC.Name);
+
+  // Several of the runtime library functions use a special calling conv
+  Info.setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::ADD_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::SUB_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::MUL_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::DIV_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::OEQ_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::UNE_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::OGE_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::OLT_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::OLE_F64, CallingConv::MSP430_BUILTIN);
+  Info.setLibcallCallingConv(RTLIB::OGT_F64, CallingConv::MSP430_BUILTIN);
+
+  // TODO: __mspabi_srall, __mspabi_srlll, __mspabi_sllll
+}
+
 /// Set default libcall names. If a target wants to opt-out of a libcall it
 /// should be placed here.
 void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
@@ -18,15 +227,17 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
             nullptr);
 
 #define HANDLE_LIBCALL(code, name) setLibcallName(RTLIB::code, name);
+#define LIBCALL_NO_NAME nullptr
 #include "llvm/IR/RuntimeLibcalls.def"
 #undef HANDLE_LIBCALL
+#undef LIBCALL_NO_NAME
 
   // Initialize calling conventions to their default.
   for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
     setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C);
 
-  // Use the f128 variants of math functions on x86_64
-  if (TT.getArch() == Triple::ArchType::x86_64 && TT.isGNUEnvironment()) {
+  // Use the f128 variants of math functions on x86
+  if (TT.isX86() && TT.isGNUEnvironment()) {
     setLibcallName(RTLIB::REM_F128, "fmodf128");
     setLibcallName(RTLIB::FMA_F128, "fmaf128");
     setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
@@ -65,12 +276,17 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
     setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
     setLibcallName(RTLIB::FMIN_F128, "fminf128");
     setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
+    setLibcallName(RTLIB::FMINIMUM_F128, "fminimumf128");
+    setLibcallName(RTLIB::FMAXIMUM_F128, "fmaximumf128");
+    setLibcallName(RTLIB::FMINIMUM_NUM_F128, "fminimum_numf128");
+    setLibcallName(RTLIB::FMAXIMUM_NUM_F128, "fmaximum_numf128");
     setLibcallName(RTLIB::LROUND_F128, "lroundf128");
     setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
     setLibcallName(RTLIB::LRINT_F128, "lrintf128");
     setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
     setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
     setLibcallName(RTLIB::FREXP_F128, "frexpf128");
+    setLibcallName(RTLIB::MODF_F128, "modff128");
   }
 
   // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
@@ -104,6 +320,40 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
     setLibcallName(RTLIB::OLE_F128, "__lekf2");
     setLibcallName(RTLIB::OGT_F128, "__gtkf2");
     setLibcallName(RTLIB::UO_F128, "__unordkf2");
+
+    setLibcallName(RTLIB::LOG_F128, "logf128");
+    setLibcallName(RTLIB::LOG2_F128, "log2f128");
+    setLibcallName(RTLIB::LOG10_F128, "log10f128");
+    setLibcallName(RTLIB::EXP_F128, "expf128");
+    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
+    setLibcallName(RTLIB::SIN_F128, "sinf128");
+    setLibcallName(RTLIB::COS_F128, "cosf128");
+    setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
+    setLibcallName(RTLIB::POW_F128, "powf128");
+    setLibcallName(RTLIB::FMIN_F128, "fminf128");
+    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
+    setLibcallName(RTLIB::REM_F128, "fmodf128");
+    setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
+    setLibcallName(RTLIB::CEIL_F128, "ceilf128");
+    setLibcallName(RTLIB::FLOOR_F128, "floorf128");
+    setLibcallName(RTLIB::TRUNC_F128, "truncf128");
+    setLibcallName(RTLIB::ROUND_F128, "roundf128");
+    setLibcallName(RTLIB::LROUND_F128, "lroundf128");
+    setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
+    setLibcallName(RTLIB::RINT_F128, "rintf128");
+    setLibcallName(RTLIB::LRINT_F128, "lrintf128");
+    setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
+    setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
+    setLibcallName(RTLIB::FMA_F128, "fmaf128");
+    setLibcallName(RTLIB::FREXP_F128, "frexpf128");
+
+    if (TT.isOSAIX()) {
+      bool isPPC64 = TT.isPPC64();
+      setLibcallName(RTLIB::MEMCPY, isPPC64 ? "___memmove64" : "___memmove");
+      setLibcallName(RTLIB::MEMMOVE, isPPC64 ? "___memmove64" : "___memmove");
+      setLibcallName(RTLIB::MEMSET, isPPC64 ? "___memset64" : "___memset");
+      setLibcallName(RTLIB::BZERO, isPPC64 ? "___bzero64" : "___bzero");
+    }
   }
 
   // A few names are different on particular architectures or environments.
@@ -203,28 +453,46 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
     setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
   }
 
-  // Disable most libcalls on AMDGPU.
-  if (TT.isAMDGPU()) {
-    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) {
-      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
-        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
+  // Disable most libcalls on AMDGPU and NVPTX.
+  if (TT.isAMDGPU() || TT.isNVPTX()) {
+    for (RTLIB::Libcall LC : RTLIB::libcalls()) {
+      if (LC < RTLIB::ATOMIC_LOAD || LC > RTLIB::ATOMIC_FETCH_NAND_16)
+        setLibcallName(LC, nullptr);
     }
   }
 
-  // Disable most libcalls on NVPTX.
-  if (TT.isNVPTX()) {
-    for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
-      if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
-        setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
-  }
-
   if (TT.isOSMSVCRT()) {
     // MSVCRT doesn't have powi; fall back to pow
     setLibcallName(RTLIB::POWI_F32, nullptr);
     setLibcallName(RTLIB::POWI_F64, nullptr);
   }
 
-  if (TT.getArch() == Triple::ArchType::avr) {
+  // Setup Windows compiler runtime calls.
+  if (TT.getArch() == Triple::x86 &&
+      (TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment())) {
+    static const struct {
+      const RTLIB::Libcall Op;
+      const char *const Name;
+      const CallingConv::ID CC;
+    } LibraryCalls[] = {
+        {RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall},
+        {RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall},
+        {RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall},
+        {RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall},
+        {RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall},
+    };
+
+    for (const auto &LC : LibraryCalls) {
+      setLibcallName(LC.Op, LC.Name);
+      setLibcallCallingConv(LC.Op, LC.CC);
+    }
+  }
+
+  if (TT.getArch() == Triple::ArchType::aarch64)
+    setAArch64LibcallNames(*this, TT);
+  else if (TT.isARM() || TT.isThumb())
+    setARMLibcallNames(*this, TT);
+  else if (TT.getArch() == Triple::ArchType::avr) {
     // Division rtlib functions (not supported), use divmod functions instead
     setLibcallName(RTLIB::SDIV_I8, nullptr);
     setLibcallName(RTLIB::SDIV_I16, nullptr);
@@ -240,6 +508,24 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
     setLibcallName(RTLIB::UREM_I8, nullptr);
     setLibcallName(RTLIB::UREM_I16, nullptr);
     setLibcallName(RTLIB::UREM_I32, nullptr);
+
+    // Division and modulus rtlib functions
+    setLibcallName(RTLIB::SDIVREM_I8, "__divmodqi4");
+    setLibcallName(RTLIB::SDIVREM_I16, "__divmodhi4");
+    setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
+    setLibcallName(RTLIB::UDIVREM_I8, "__udivmodqi4");
+    setLibcallName(RTLIB::UDIVREM_I16, "__udivmodhi4");
+    setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
+
+    // Several of the runtime library functions use a special calling conv
+    setLibcallCallingConv(RTLIB::SDIVREM_I8, CallingConv::AVR_BUILTIN);
+    setLibcallCallingConv(RTLIB::SDIVREM_I16, CallingConv::AVR_BUILTIN);
+    setLibcallCallingConv(RTLIB::UDIVREM_I8, CallingConv::AVR_BUILTIN);
+    setLibcallCallingConv(RTLIB::UDIVREM_I16, CallingConv::AVR_BUILTIN);
+
+    // Trigonometric rtlib functions
+    setLibcallName(RTLIB::SIN_F32, "sin");
+    setLibcallName(RTLIB::COS_F32, "cos");
   }
 
   if (!TT.isWasm()) {
@@ -252,5 +538,64 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
       setLibcallName(RTLIB::MULO_I64, nullptr);
     }
     setLibcallName(RTLIB::MULO_I128, nullptr);
+  } else {
+    // Define the emscripten name for return address helper.
+    // TODO: when implementing other Wasm backends, make this generic or only do
+    // this on emscripten depending on what they end up doing.
+    setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
   }
+
+  if (TT.isSystemZ() && TT.isOSzOS()) {
+    struct RTLibCallMapping {
+      RTLIB::Libcall Code;
+      const char *Name;
+    };
+    static RTLibCallMapping RTLibCallCommon[] = {
+#define HANDLE_LIBCALL(code, name) {RTLIB::code, name},
+#include "ZOSLibcallNames.def"
+    };
+    for (auto &E : RTLibCallCommon)
+      setLibcallName(E.Code, E.Name);
+  }
+
+  if (TT.getArch() == Triple::ArchType::hexagon) {
+    setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
+    setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
+    setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
+    setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
+    setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
+    setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
+    setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
+    setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
+
+    const bool FastMath = HexagonEnableFastMathRuntimeCalls;
+    // This is the only fast library function for sqrtd.
+    if (FastMath)
+      setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
+
+    // Prefix is: nothing  for "slow-math",
+    //            "fast2_" for V5+ fast-math double-precision
+    // (actually, keep fast-math and fast-math2 separate for now)
+    if (FastMath) {
+      setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
+      setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
+      setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
+      setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
+      setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
+    } else {
+      setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
+      setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
+      setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
+      setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
+      setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
+    }
+
+    if (FastMath)
+      setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
+    else
+      setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
+  }
+
+  if (TT.getArch() == Triple::ArchType::msp430)
+    setMSP430Libcalls(*this, TT);
 }
diff --git a/external/llvm-project/llvm/lib/IR/Type.cpp b/external/llvm-project/llvm/lib/IR/Type.cpp
index f7342976cd22..d10e8b2a0bb6 100644
--- a/external/llvm-project/llvm/lib/IR/Type.cpp
+++ b/external/llvm-project/llvm/lib/IR/Type.cpp
@@ -791,8 +791,12 @@ VectorType *VectorType::get(Type *ElementType, ElementCount EC) {
 }
 
 bool VectorType::isValidElementType(Type *ElemTy) {
-  return ElemTy->isIntegerTy() || ElemTy->isFloatingPointTy() ||
-         ElemTy->isPointerTy() || ElemTy->getTypeID() == TypedPointerTyID;
+  if (ElemTy->isIntegerTy() || ElemTy->isFloatingPointTy() ||
+      ElemTy->isPointerTy() || ElemTy->getTypeID() == TypedPointerTyID)
+    return true;
+  if (auto *TTy = dyn_cast<TargetExtType>(ElemTy))
+    return TTy->hasProperty(TargetExtType::CanBeVectorElement);
+  return false;
 }
 
 //===----------------------------------------------------------------------===//
@@ -802,8 +806,9 @@ bool VectorType::isValidElementType(Type *ElemTy) {
 FixedVectorType *FixedVectorType::get(Type *ElementType, unsigned NumElts) {
   assert(NumElts > 0 && "#Elements of a VectorType must be greater than 0");
   assert(isValidElementType(ElementType) && "Element type of a VectorType must "
-                                            "be an integer, floating point, or "
-                                            "pointer type.");
+                                            "be an integer, floating point, "
+                                            "pointer type, or a valid target "
+                                            "extension type.");
 
   auto EC = ElementCount::getFixed(NumElts);
 
@@ -969,7 +974,11 @@ struct TargetTypeInfo {
 
   template <typename... ArgTys>
   TargetTypeInfo(Type *LayoutType, ArgTys... Properties)
-      : LayoutType(LayoutType), Properties((0 | ... | Properties)) {}
+      : LayoutType(LayoutType), Properties((0 | ... | Properties)) {
+    assert((!(this->Properties & TargetExtType::CanBeVectorElement) ||
+            LayoutType->isSized()) &&
+           "Vector element type must be sized");
+  }
 };
 } // anonymous namespace
 
@@ -1038,6 +1047,13 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
                           TargetExtType::CanBeGlobal);
   }
 
+  // Type used to test vector element target extension property.
+  // Can be removed once a public target extension type uses CanBeVectorElement.
+  if (Name == "llvm.test.vectorelement") {
+    return TargetTypeInfo(Type::getInt32Ty(C), TargetExtType::CanBeLocal,
+                          TargetExtType::CanBeVectorElement);
+  }
+
   return TargetTypeInfo(Type::getVoidTy(C));
 }
 
diff --git a/external/llvm-project/llvm/lib/IR/Value.cpp b/external/llvm-project/llvm/lib/IR/Value.cpp
index d6cb65d94a11..02c16e201abe 100644
--- a/external/llvm-project/llvm/lib/IR/Value.cpp
+++ b/external/llvm-project/llvm/lib/IR/Value.cpp
@@ -957,30 +957,27 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL,
 
 Align Value::getPointerAlignment(const DataLayout &DL) const {
   assert(getType()->isPointerTy() && "must be pointer");
-  if (auto *GO = dyn_cast<GlobalObject>(this)) {
-    if (isa<Function>(GO)) {
-      Align FunctionPtrAlign = DL.getFunctionPtrAlign().valueOrOne();
-      switch (DL.getFunctionPtrAlignType()) {
-      case DataLayout::FunctionPtrAlignType::Independent:
-        return FunctionPtrAlign;
-      case DataLayout::FunctionPtrAlignType::MultipleOfFunctionAlign:
-        return std::max(FunctionPtrAlign, GO->getAlign().valueOrOne());
-      }
-      llvm_unreachable("Unhandled FunctionPtrAlignType");
+  if (const Function *F = dyn_cast<Function>(this)) {
+    Align FunctionPtrAlign = DL.getFunctionPtrAlign().valueOrOne();
+    switch (DL.getFunctionPtrAlignType()) {
+    case DataLayout::FunctionPtrAlignType::Independent:
+      return FunctionPtrAlign;
+    case DataLayout::FunctionPtrAlignType::MultipleOfFunctionAlign:
+      return std::max(FunctionPtrAlign, F->getAlign().valueOrOne());
     }
-    const MaybeAlign Alignment(GO->getAlign());
+    llvm_unreachable("Unhandled FunctionPtrAlignType");
+  } else if (auto *GVar = dyn_cast<GlobalVariable>(this)) {
+    const MaybeAlign Alignment(GVar->getAlign());
     if (!Alignment) {
-      if (auto *GVar = dyn_cast<GlobalVariable>(GO)) {
-        Type *ObjectType = GVar->getValueType();
-        if (ObjectType->isSized()) {
-          // If the object is defined in the current Module, we'll be giving
-          // it the preferred alignment. Otherwise, we have to assume that it
-          // may only have the minimum ABI alignment.
-          if (GVar->isStrongDefinitionForLinker())
-            return DL.getPreferredAlign(GVar);
-          else
-            return DL.getABITypeAlign(ObjectType);
-        }
+      Type *ObjectType = GVar->getValueType();
+      if (ObjectType->isSized()) {
+        // If the object is defined in the current Module, we'll be giving
+        // it the preferred alignment. Otherwise, we have to assume that it
+        // may only have the minimum ABI alignment.
+        if (GVar->isStrongDefinitionForLinker())
+          return DL.getPreferredAlign(GVar);
+        else
+          return DL.getABITypeAlign(ObjectType);
       }
     }
     return Alignment.valueOrOne();
diff --git a/external/llvm-project/llvm/lib/IR/VectorBuilder.cpp b/external/llvm-project/llvm/lib/IR/VectorBuilder.cpp
deleted file mode 100644
index 737f49b1334d..000000000000
--- a/external/llvm-project/llvm/lib/IR/VectorBuilder.cpp
+++ /dev/null
@@ -1,116 +0,0 @@
-//===- VectorBuilder.cpp - Builder for VP Intrinsics ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the VectorBuilder class, which is used as a convenient
-// way to create VP intrinsics as if they were LLVM instructions with a
-// consistent and simplified interface.
-//
-//===----------------------------------------------------------------------===//
-
-#include <llvm/ADT/SmallVector.h>
-#include <llvm/IR/FPEnv.h>
-#include <llvm/IR/Instructions.h>
-#include <llvm/IR/IntrinsicInst.h>
-#include <llvm/IR/Intrinsics.h>
-#include <llvm/IR/VectorBuilder.h>
-
-namespace llvm {
-
-void VectorBuilder::handleError(const char *ErrorMsg) const {
-  if (ErrorHandling == Behavior::SilentlyReturnNone)
-    return;
-  report_fatal_error(ErrorMsg);
-}
-
-Module &VectorBuilder::getModule() const {
-  return *Builder.GetInsertBlock()->getModule();
-}
-
-Value *VectorBuilder::getAllTrueMask() {
-  return Builder.getAllOnesMask(StaticVectorLength);
-}
-
-Value &VectorBuilder::requestMask() {
-  if (Mask)
-    return *Mask;
-
-  return *getAllTrueMask();
-}
-
-Value &VectorBuilder::requestEVL() {
-  if (ExplicitVectorLength)
-    return *ExplicitVectorLength;
-
-  assert(!StaticVectorLength.isScalable() && "TODO vscale lowering");
-  auto *IntTy = Builder.getInt32Ty();
-  return *ConstantInt::get(IntTy, StaticVectorLength.getFixedValue());
-}
-
-Value *VectorBuilder::createVectorInstruction(unsigned Opcode, Type *ReturnTy,
-                                              ArrayRef<Value *> InstOpArray,
-                                              const Twine &Name) {
-  auto VPID = VPIntrinsic::getForOpcode(Opcode);
-  if (VPID == Intrinsic::not_intrinsic)
-    return returnWithError<Value *>("No VPIntrinsic for this opcode");
-  return createVectorInstructionImpl(VPID, ReturnTy, InstOpArray, Name);
-}
-
-Value *VectorBuilder::createSimpleReduction(Intrinsic::ID RdxID,
-                                            Type *ValTy,
-                                            ArrayRef<Value *> InstOpArray,
-                                            const Twine &Name) {
-  auto VPID = VPIntrinsic::getForIntrinsic(RdxID);
-  assert(VPReductionIntrinsic::isVPReduction(VPID) &&
-         "No VPIntrinsic for this reduction");
-  return createVectorInstructionImpl(VPID, ValTy, InstOpArray, Name);
-}
-
-Value *VectorBuilder::createVectorInstructionImpl(Intrinsic::ID VPID,
-                                                  Type *ReturnTy,
-                                                  ArrayRef<Value *> InstOpArray,
-                                                  const Twine &Name) {
-  auto MaskPosOpt = VPIntrinsic::getMaskParamPos(VPID);
-  auto VLenPosOpt = VPIntrinsic::getVectorLengthParamPos(VPID);
-  size_t NumInstParams = InstOpArray.size();
-  size_t NumVPParams =
-      NumInstParams + MaskPosOpt.has_value() + VLenPosOpt.has_value();
-
-  SmallVector<Value *, 6> IntrinParams;
-
-  // Whether the mask and vlen parameter are at the end of the parameter list.
-  bool TrailingMaskAndVLen =
-      std::min<size_t>(MaskPosOpt.value_or(NumInstParams),
-                       VLenPosOpt.value_or(NumInstParams)) >= NumInstParams;
-
-  if (TrailingMaskAndVLen) {
-    // Fast path for trailing mask, vector length.
-    IntrinParams.append(InstOpArray.begin(), InstOpArray.end());
-    IntrinParams.resize(NumVPParams);
-  } else {
-    IntrinParams.resize(NumVPParams);
-    // Insert mask and evl operands in between the instruction operands.
-    for (size_t VPParamIdx = 0, ParamIdx = 0; VPParamIdx < NumVPParams;
-         ++VPParamIdx) {
-      if (MaskPosOpt == VPParamIdx || VLenPosOpt == VPParamIdx)
-        continue;
-      assert(ParamIdx < NumInstParams);
-      IntrinParams[VPParamIdx] = InstOpArray[ParamIdx++];
-    }
-  }
-
-  if (MaskPosOpt)
-    IntrinParams[*MaskPosOpt] = &requestMask();
-  if (VLenPosOpt)
-    IntrinParams[*VLenPosOpt] = &requestEVL();
-
-  auto *VPDecl = VPIntrinsic::getOrInsertDeclarationForParams(
-      &getModule(), VPID, ReturnTy, IntrinParams);
-  return Builder.CreateCall(VPDecl, IntrinParams, Name);
-}
-
-} // namespace llvm
diff --git a/external/llvm-project/llvm/lib/IR/Verifier.cpp b/external/llvm-project/llvm/lib/IR/Verifier.cpp
index ffe5bcf60f8f..67b2882319f1 100644
--- a/external/llvm-project/llvm/lib/IR/Verifier.cpp
+++ b/external/llvm-project/llvm/lib/IR/Verifier.cpp
@@ -735,12 +735,6 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
         "Global is external, but doesn't have external or weak linkage!", &GV);
 
   if (const GlobalObject *GO = dyn_cast<GlobalObject>(&GV)) {
-
-    if (MaybeAlign A = GO->getAlign()) {
-      Check(A->value() <= Value::MaximumAlignment,
-            "huge alignment values are unsupported", GO);
-    }
-
     if (const MDNode *Associated =
             GO->getMetadata(LLVMContext::MD_associated)) {
       Check(Associated->getNumOperands() == 1,
@@ -830,6 +824,11 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
 void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
   Type *GVType = GV.getValueType();
 
+  if (MaybeAlign A = GV.getAlign()) {
+    Check(A->value() <= Value::MaximumAlignment,
+          "huge alignment values are unsupported", &GV);
+  }
+
   if (GV.hasInitializer()) {
     Check(GV.getInitializer()->getType() == GVType,
           "Global variable initializer type does not match global "
@@ -2405,6 +2404,31 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
       CheckFailed("'allockind()' can't be both zeroed and uninitialized");
   }
 
+  if (Attribute A = Attrs.getFnAttr("alloc-variant-zeroed"); A.isValid()) {
+    StringRef S = A.getValueAsString();
+    Check(!S.empty(), "'alloc-variant-zeroed' must not be empty");
+    Function *Variant = M.getFunction(S);
+    if (Variant) {
+      Attribute Family = Attrs.getFnAttr("alloc-family");
+      Attribute VariantFamily = Variant->getFnAttribute("alloc-family");
+      if (Family.isValid())
+        Check(VariantFamily.isValid() &&
+                  VariantFamily.getValueAsString() == Family.getValueAsString(),
+              "'alloc-variant-zeroed' must name a function belonging to the "
+              "same 'alloc-family'");
+
+      Check(Variant->hasFnAttribute(Attribute::AllocKind) &&
+                (Variant->getFnAttribute(Attribute::AllocKind).getAllocKind() &
+                 AllocFnKind::Zeroed) != AllocFnKind::Unknown,
+            "'alloc-variant-zeroed' must name a function with "
+            "'allockind(\"zeroed\")'");
+
+      Check(FT == Variant->getFunctionType(),
+            "'alloc-variant-zeroed' must name a function with the same "
+            "signature");
+    }
+  }
+
   if (Attrs.hasFnAttr(Attribute::VScaleRange)) {
     unsigned VScaleMin = Attrs.getFnAttrs().getVScaleRangeMin();
     if (VScaleMin == 0)
@@ -2867,16 +2891,16 @@ void Verifier::visitFunction(const Function &F) {
   Check(!F.hasStructRetAttr() || F.getReturnType()->isVoidTy(),
         "Invalid struct return type!", &F);
 
+  if (MaybeAlign A = F.getAlign()) {
+    Check(A->value() <= Value::MaximumAlignment,
+          "huge alignment values are unsupported", &F);
+  }
+
   AttributeList Attrs = F.getAttributes();
 
   Check(verifyAttributeCount(Attrs, FT->getNumParams()),
         "Attribute after last parameter!", &F);
 
-  CheckDI(F.IsNewDbgInfoFormat == F.getParent()->IsNewDbgInfoFormat,
-          "Function debug format should match parent module", &F,
-          F.IsNewDbgInfoFormat, F.getParent(),
-          F.getParent()->IsNewDbgInfoFormat);
-
   bool IsIntrinsic = F.isIntrinsic();
 
   // Check function attributes.
@@ -3227,15 +3251,9 @@ void Verifier::visitBasicBlock(BasicBlock &BB) {
     Check(I.getParent() == &BB, "Instruction has bogus parent pointer!");
   }
 
-  CheckDI(BB.IsNewDbgInfoFormat == BB.getParent()->IsNewDbgInfoFormat,
-          "BB debug format should match parent function", &BB,
-          BB.IsNewDbgInfoFormat, BB.getParent(),
-          BB.getParent()->IsNewDbgInfoFormat);
-
   // Confirm that no issues arise from the debug program.
-  if (BB.IsNewDbgInfoFormat)
-    CheckDI(!BB.getTrailingDbgRecords(), "Basic Block has trailing DbgRecords!",
-            &BB);
+  CheckDI(!BB.getTrailingDbgRecords(), "Basic Block has trailing DbgRecords!",
+          &BB);
 }
 
 void Verifier::visitTerminator(Instruction &I) {
@@ -6711,7 +6729,7 @@ void Verifier::visit(DbgVariableRecord &DVR) {
   CheckDI(DVR.getType() == DbgVariableRecord::LocationType::Value ||
               DVR.getType() == DbgVariableRecord::LocationType::Declare ||
               DVR.getType() == DbgVariableRecord::LocationType::Assign,
-          "invalid #dbg record type", &DVR, DVR.getType());
+          "invalid #dbg record type", &DVR, DVR.getType(), BB, F);
 
   // The location for a DbgVariableRecord must be either a ValueAsMetadata,
   // DIArgList, or an empty MDNode (which is a legacy representation for an
@@ -6719,30 +6737,40 @@ void Verifier::visit(DbgVariableRecord &DVR) {
   auto *MD = DVR.getRawLocation();
   CheckDI(MD && (isa<ValueAsMetadata>(MD) || isa<DIArgList>(MD) ||
                  (isa<MDNode>(MD) && !cast<MDNode>(MD)->getNumOperands())),
-          "invalid #dbg record address/value", &DVR, MD);
+          "invalid #dbg record address/value", &DVR, MD, BB, F);
   if (auto *VAM = dyn_cast<ValueAsMetadata>(MD)) {
     visitValueAsMetadata(*VAM, F);
     if (DVR.isDbgDeclare()) {
       // Allow integers here to support inttoptr salvage.
       Type *Ty = VAM->getValue()->getType();
       CheckDI(Ty->isPointerTy() || Ty->isIntegerTy(),
-              "location of #dbg_declare must be a pointer or int", &DVR, MD);
+              "location of #dbg_declare must be a pointer or int", &DVR, MD, BB,
+              F);
     }
   } else if (auto *AL = dyn_cast<DIArgList>(MD)) {
     visitDIArgList(*AL, F);
   }
 
   CheckDI(isa_and_nonnull<DILocalVariable>(DVR.getRawVariable()),
-          "invalid #dbg record variable", &DVR, DVR.getRawVariable());
+          "invalid #dbg record variable", &DVR, DVR.getRawVariable(), BB, F);
   visitMDNode(*DVR.getRawVariable(), AreDebugLocsAllowed::No);
 
   CheckDI(isa_and_nonnull<DIExpression>(DVR.getRawExpression()),
-          "invalid #dbg record expression", &DVR, DVR.getRawExpression());
+          "invalid #dbg record expression", &DVR, DVR.getRawExpression(), BB,
+          F);
   visitMDNode(*DVR.getExpression(), AreDebugLocsAllowed::No);
 
+  // This is redundant with the visitMDNode check above, but here we can include
+  // arguments for DIOp-based expression checking.
+  SmallVector<const Value *> Arguments{DVR.location_ops()};
+  DIExpressionEnv ExprEnv{DVR.getVariable(), Arguments, DL};
+  CheckDI(DVR.getExpression()->isValid(ExprEnv, dbgs()),
+          "invalid #dbg record expression", &DVR, DVR.getRawExpression());
+
   if (DVR.isDbgAssign()) {
     CheckDI(isa_and_nonnull<DIAssignID>(DVR.getRawAssignID()),
-            "invalid #dbg_assign DIAssignID", &DVR, DVR.getRawAssignID());
+            "invalid #dbg_assign DIAssignID", &DVR, DVR.getRawAssignID(), BB,
+            F);
     visitMDNode(*cast<DIAssignID>(DVR.getRawAssignID()),
                 AreDebugLocsAllowed::No);
 
@@ -6753,29 +6781,29 @@ void Verifier::visit(DbgVariableRecord &DVR) {
     CheckDI(
         isa<ValueAsMetadata>(RawAddr) ||
             (isa<MDNode>(RawAddr) && !cast<MDNode>(RawAddr)->getNumOperands()),
-        "invalid #dbg_assign address", &DVR, DVR.getRawAddress());
+        "invalid #dbg_assign address", &DVR, DVR.getRawAddress(), BB, F);
     if (auto *VAM = dyn_cast<ValueAsMetadata>(RawAddr))
       visitValueAsMetadata(*VAM, F);
 
     CheckDI(isa_and_nonnull<DIExpression>(DVR.getRawAddressExpression()),
             "invalid #dbg_assign address expression", &DVR,
-            DVR.getRawAddressExpression());
+            DVR.getRawAddressExpression(), BB, F);
     visitMDNode(*DVR.getAddressExpression(), AreDebugLocsAllowed::No);
 
     // All of the linked instructions should be in the same function as DVR.
     for (Instruction *I : at::getAssignmentInsts(&DVR))
       CheckDI(DVR.getFunction() == I->getFunction(),
-              "inst not in same function as #dbg_assign", I, &DVR);
+              "inst not in same function as #dbg_assign", I, &DVR, BB, F);
   }
 
   // This check is redundant with one in visitLocalVariable().
   DILocalVariable *Var = DVR.getVariable();
-  CheckDI(isType(Var->getRawType()), "invalid type ref", Var,
-          Var->getRawType());
+  CheckDI(isType(Var->getRawType()), "invalid type ref", Var, Var->getRawType(),
+          BB, F);
 
   auto *DLNode = DVR.getDebugLoc().getAsMDNode();
   CheckDI(isa_and_nonnull<DILocation>(DLNode), "invalid #dbg record DILocation",
-          &DVR, DLNode);
+          &DVR, DLNode, BB, F);
   DILocation *Loc = DVR.getDebugLoc();
 
   // The scopes for variables and !dbg attachments must agree.
@@ -6787,7 +6815,7 @@ void Verifier::visit(DbgVariableRecord &DVR) {
   CheckDI(VarSP == LocSP,
           "mismatched subprogram between #dbg record variable and DILocation",
           &DVR, BB, F, Var, Var->getScope()->getSubprogram(), Loc,
-          Loc->getScope()->getSubprogram());
+          Loc->getScope()->getSubprogram(), BB, F);
 
   verifyFnArgs(DVR);
 }
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/ZOSLibcallNames.def b/external/llvm-project/llvm/lib/IR/ZOSLibcallNames.def
similarity index 100%
rename from external/llvm-project/llvm/lib/Target/SystemZ/ZOSLibcallNames.def
rename to external/llvm-project/llvm/lib/IR/ZOSLibcallNames.def
diff --git a/external/llvm-project/llvm/lib/IRPrinter/IRPrintingPasses.cpp b/external/llvm-project/llvm/lib/IRPrinter/IRPrintingPasses.cpp
index b1750a5ca309..81ad284ea164 100644
--- a/external/llvm-project/llvm/lib/IRPrinter/IRPrintingPasses.cpp
+++ b/external/llvm-project/llvm/lib/IRPrinter/IRPrintingPasses.cpp
@@ -17,13 +17,12 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PrintPasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 PrintModulePass::PrintModulePass() : OS(dbgs()) {}
 PrintModulePass::PrintModulePass(raw_ostream &OS, const std::string &Banner,
                                  bool ShouldPreserveUseListOrder,
@@ -33,12 +32,9 @@ PrintModulePass::PrintModulePass(raw_ostream &OS, const std::string &Banner,
       EmitSummaryIndex(EmitSummaryIndex) {}
 
 PreservedAnalyses PrintModulePass::run(Module &M, ModuleAnalysisManager &AM) {
-  ScopedDbgInfoFormatSetter FormatSetter(M, UseNewDbgInfoFormat);
   // Remove intrinsic declarations when printing in the new format.
-  // TODO: Move this into Module::setIsNewDbgInfoFormat when we're ready to
-  // update test output.
-  if (UseNewDbgInfoFormat)
-    M.removeDebugIntrinsicDeclarations();
+  // TODO: consider removing this now that debug intrinsics are gone.
+  M.removeDebugIntrinsicDeclarations();
 
   if (llvm::isFunctionInPrintList("*")) {
     if (!Banner.empty())
@@ -75,8 +71,6 @@ PrintFunctionPass::PrintFunctionPass(raw_ostream &OS, const std::string &Banner)
 
 PreservedAnalyses PrintFunctionPass::run(Function &F,
                                          FunctionAnalysisManager &) {
-  ScopedDbgInfoFormatSetter FormatSetter(F, UseNewDbgInfoFormat);
-
   if (isFunctionInPrintList(F.getName())) {
     if (forcePrintModuleIR())
       OS << Banner << " (function: " << F.getName() << ")\n" << *F.getParent();
diff --git a/external/llvm-project/llvm/lib/LTO/LTO.cpp b/external/llvm-project/llvm/lib/LTO/LTO.cpp
index ea0cadd02d54..adf995cbc9b1 100644
--- a/external/llvm-project/llvm/lib/LTO/LTO.cpp
+++ b/external/llvm-project/llvm/lib/LTO/LTO.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/JSON.h"
@@ -69,8 +70,6 @@ using namespace object;
 
 #define DEBUG_TYPE "lto"
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 static cl::opt<bool>
     DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
                    cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
@@ -600,9 +599,7 @@ LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
                                       const Config &Conf)
     : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
       Ctx(Conf), CombinedModule(std::make_unique<Module>("ld-temp.o", Ctx)),
-      Mover(std::make_unique<IRMover>(*CombinedModule)) {
-  CombinedModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat;
-}
+      Mover(std::make_unique<IRMover>(*CombinedModule)) {}
 
 LTO::ThinLTOState::ThinLTOState(ThinBackend BackendParam)
     : Backend(std::move(BackendParam)), CombinedIndex(/*HaveGVs*/ false) {
diff --git a/external/llvm-project/llvm/lib/LTO/LTOModule.cpp b/external/llvm-project/llvm/lib/LTO/LTOModule.cpp
index 749ae635e610..e0a975806a31 100644
--- a/external/llvm-project/llvm/lib/LTO/LTOModule.cpp
+++ b/external/llvm-project/llvm/lib/LTO/LTOModule.cpp
@@ -414,8 +414,11 @@ void LTOModule::addDefinedFunctionSymbol(StringRef Name, const GlobalValue *F) {
 
 void LTOModule::addDefinedSymbol(StringRef Name, const GlobalValue *def,
                                  bool isFunction) {
-  const GlobalObject *go = dyn_cast<GlobalObject>(def);
-  uint32_t attr = go ? Log2(go->getAlign().valueOrOne()) : 0;
+  uint32_t attr = 0;
+  if (auto *gv = dyn_cast<GlobalVariable>(def))
+    attr = Log2(gv->getAlign().valueOrOne());
+  else if (auto *f = dyn_cast<Function>(def))
+    attr = Log2(f->getAlign().valueOrOne());
 
   // set permissions part
   if (isFunction) {
diff --git a/external/llvm-project/llvm/lib/Linker/IRMover.cpp b/external/llvm-project/llvm/lib/Linker/IRMover.cpp
index 3df3ee84ce21..2a9709050162 100644
--- a/external/llvm-project/llvm/lib/Linker/IRMover.cpp
+++ b/external/llvm-project/llvm/lib/Linker/IRMover.cpp
@@ -8,7 +8,6 @@
 
 #include "llvm/Linker/IRMover.h"
 #include "LinkDiagnosticInfo.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
@@ -27,7 +26,6 @@
 #include "llvm/IR/TypeFinder.h"
 #include "llvm/Object/ModuleSymbolTable.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/Path.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <optional>
@@ -49,21 +47,6 @@ class TypeMapTy : public ValueMapTypeRemapper {
   /// This is a mapping from a source type to a destination type to use.
   DenseMap<Type *, Type *> MappedTypes;
 
-  /// When checking to see if two subgraphs are isomorphic, we speculatively
-  /// add types to MappedTypes, but keep track of them here in case we need to
-  /// roll back.
-  SmallVector<Type *, 16> SpeculativeTypes;
-
-  SmallVector<StructType *, 16> SpeculativeDstOpaqueTypes;
-
-  /// This is a list of non-opaque structs in the source module that are mapped
-  /// to an opaque struct in the destination module.
-  SmallVector<StructType *, 16> SrcDefinitionsToResolve;
-
-  /// This is the set of opaque types in the destination modules who are
-  /// getting a body from the source module.
-  SmallPtrSet<StructType *, 16> DstResolvedOpaqueTypes;
-
 public:
   TypeMapTy(IRMover::IdentifiedStructTypeSet &DstStructTypesSet)
       : DstStructTypesSet(DstStructTypesSet) {}
@@ -73,10 +56,6 @@ class TypeMapTy : public ValueMapTypeRemapper {
   /// equivalent to the specified type in the source module.
   void addTypeMapping(Type *DstTy, Type *SrcTy);
 
-  /// Produce a body for an opaque type in the dest module from a type
-  /// definition in the source module.
-  Error linkDefinedTypeBodies();
-
   /// Return the mapped type to use for the specified input type from the
   /// source module.
   Type *get(Type *SrcTy);
@@ -88,45 +67,19 @@ class TypeMapTy : public ValueMapTypeRemapper {
 private:
   Type *remapType(Type *SrcTy) override { return get(SrcTy); }
 
-  bool areTypesIsomorphic(Type *DstTy, Type *SrcTy);
+  bool recursivelyAddMappingIfTypesAreIsomorphic(Type *DstTy, Type *SrcTy);
 };
 }
 
 void TypeMapTy::addTypeMapping(Type *DstTy, Type *SrcTy) {
-  assert(SpeculativeTypes.empty());
-  assert(SpeculativeDstOpaqueTypes.empty());
-
-  // Check to see if these types are recursively isomorphic and establish a
-  // mapping between them if so.
-  if (!areTypesIsomorphic(DstTy, SrcTy)) {
-    // Oops, they aren't isomorphic.  Just discard this request by rolling out
-    // any speculative mappings we've established.
-    for (Type *Ty : SpeculativeTypes)
-      MappedTypes.erase(Ty);
-
-    SrcDefinitionsToResolve.resize(SrcDefinitionsToResolve.size() -
-                                   SpeculativeDstOpaqueTypes.size());
-    for (StructType *Ty : SpeculativeDstOpaqueTypes)
-      DstResolvedOpaqueTypes.erase(Ty);
-  } else {
-    // SrcTy and DstTy are recursively ismorphic. We clear names of SrcTy
-    // and all its descendants to lower amount of renaming in LLVM context
-    // Renaming occurs because we load all source modules to the same context
-    // and declaration with existing name gets renamed (i.e Foo -> Foo.42).
-    // As a result we may get several different types in the destination
-    // module, which are in fact the same.
-    for (Type *Ty : SpeculativeTypes)
-      if (auto *STy = dyn_cast<StructType>(Ty))
-        if (STy->hasName())
-          STy->setName("");
-  }
-  SpeculativeTypes.clear();
-  SpeculativeDstOpaqueTypes.clear();
+  recursivelyAddMappingIfTypesAreIsomorphic(DstTy, SrcTy);
 }
 
 /// Recursively walk this pair of types, returning true if they are isomorphic,
-/// false if they are not.
-bool TypeMapTy::areTypesIsomorphic(Type *DstTy, Type *SrcTy) {
+/// false if they are not. Types that were determined to be isomorphic are
+/// added to MappedTypes.
+bool TypeMapTy::recursivelyAddMappingIfTypesAreIsomorphic(Type *DstTy,
+                                                          Type *SrcTy) {
   // Two types with differing kinds are clearly not isomorphic.
   if (DstTy->getTypeID() != SrcTy->getTypeID())
     return false;
@@ -145,29 +98,10 @@ bool TypeMapTy::areTypesIsomorphic(Type *DstTy, Type *SrcTy) {
 
   // Okay, we have two types with identical kinds that we haven't seen before.
 
-  // If this is an opaque struct type, special case it.
+  // Always consider opaque struct types non-isomorphic.
   if (StructType *SSTy = dyn_cast<StructType>(SrcTy)) {
-    // Mapping an opaque type to any struct, just keep the dest struct.
-    if (SSTy->isOpaque()) {
-      Entry = DstTy;
-      SpeculativeTypes.push_back(SrcTy);
-      return true;
-    }
-
-    // Mapping a non-opaque source type to an opaque dest.  If this is the first
-    // type that we're mapping onto this destination type then we succeed.  Keep
-    // the dest, but fill it in later. If this is the second (different) type
-    // that we're trying to map onto the same opaque type then we fail.
-    if (cast<StructType>(DstTy)->isOpaque()) {
-      // We can only map one source type onto the opaque destination type.
-      if (!DstResolvedOpaqueTypes.insert(cast<StructType>(DstTy)).second)
-        return false;
-      SrcDefinitionsToResolve.push_back(SSTy);
-      SpeculativeTypes.push_back(SrcTy);
-      SpeculativeDstOpaqueTypes.push_back(cast<StructType>(DstTy));
-      Entry = DstTy;
-      return true;
-    }
+    if (SSTy->isOpaque() || cast<StructType>(DstTy)->isOpaque())
+      return false;
   }
 
   // If the number of subtypes disagree between the two types, then we fail.
@@ -196,38 +130,27 @@ bool TypeMapTy::areTypesIsomorphic(Type *DstTy, Type *SrcTy) {
       return false;
   }
 
-  // Otherwise, we speculate that these two types will line up and recursively
-  // check the subelements.
-  Entry = DstTy;
-  SpeculativeTypes.push_back(SrcTy);
-
+  // Recursively check the subelements.
   for (unsigned I = 0, E = SrcTy->getNumContainedTypes(); I != E; ++I)
-    if (!areTypesIsomorphic(DstTy->getContainedType(I),
-                            SrcTy->getContainedType(I)))
+    if (!recursivelyAddMappingIfTypesAreIsomorphic(DstTy->getContainedType(I),
+                                                   SrcTy->getContainedType(I)))
       return false;
 
   // If everything seems to have lined up, then everything is great.
-  return true;
-}
-
-Error TypeMapTy::linkDefinedTypeBodies() {
-  SmallVector<Type *, 16> Elements;
-  for (StructType *SrcSTy : SrcDefinitionsToResolve) {
-    StructType *DstSTy = cast<StructType>(MappedTypes[SrcSTy]);
-    assert(DstSTy->isOpaque());
+  [[maybe_unused]] auto Res = MappedTypes.insert({SrcTy, DstTy});
+  assert(!Res.second && "Recursive type?");
 
-    // Map the body of the source type over to a new body for the dest type.
-    Elements.resize(SrcSTy->getNumElements());
-    for (unsigned I = 0, E = Elements.size(); I != E; ++I)
-      Elements[I] = get(SrcSTy->getElementType(I));
-
-    if (auto E = DstSTy->setBodyOrError(Elements, SrcSTy->isPacked()))
-      return E;
-    DstStructTypesSet.switchToNonOpaque(DstSTy);
+  if (auto *STy = dyn_cast<StructType>(SrcTy)) {
+    // We clear name of SrcTy to lower amount of renaming in LLVM context.
+    // Renaming occurs because we load all source modules to the same context
+    // and declaration with existing name gets renamed (i.e Foo -> Foo.42).
+    // As a result we may get several different types in the destination
+    // module, which are in fact the same.
+    if (STy->hasName())
+      STy->setName("");
   }
-  SrcDefinitionsToResolve.clear();
-  DstResolvedOpaqueTypes.clear();
-  return Error::success();
+
+  return true;
 }
 
 Type *TypeMapTy::get(Type *Ty) {
@@ -672,7 +595,6 @@ Function *IRLinker::copyFunctionProto(const Function *SF) {
                              SF->getAddressSpace(), SF->getName(), &DstM);
   F->copyAttributesFrom(SF);
   F->setAttributes(mapAttributeTypes(F->getContext(), F->getAttributes()));
-  F->IsNewDbgInfoFormat = SF->IsNewDbgInfoFormat;
   return F;
 }
 
@@ -845,10 +767,6 @@ void IRLinker::computeTypeMapping() {
     if (TypeMap.DstStructTypesSet.hasType(DST))
       TypeMap.addTypeMapping(DST, ST);
   }
-
-  // Now that we have discovered all of the type equivalences, get a body for
-  // any 'opaque' types in the dest module that are now resolved.
-  setError(TypeMap.linkDefinedTypeBodies());
 }
 
 static void getArrayElements(const Constant *C,
@@ -1111,7 +1029,6 @@ Error IRLinker::linkFunctionBody(Function &Dst, Function &Src) {
     Dst.setPrologueData(Src.getPrologueData());
   if (Src.hasPersonalityFn())
     Dst.setPersonalityFn(Src.getPersonalityFn());
-  assert(Src.IsNewDbgInfoFormat == Dst.IsNewDbgInfoFormat);
 
   // Copy over the metadata attachments without remapping.
   Dst.copyMetadata(&Src, 0);
@@ -1525,9 +1442,6 @@ Error IRLinker::run() {
     if (Error Err = SrcM->getMaterializer()->materializeMetadata())
       return Err;
 
-  // Convert source module to match dest for the duration of the link.
-  ScopedDbgInfoFormatSetter FormatSetter(*SrcM, DstM.IsNewDbgInfoFormat);
-
   // Inherit the target data from the source module if the destination
   // module doesn't have one already.
   if (DstM.getDataLayout().isDefault())
diff --git a/external/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp b/external/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
index 933a64d4ddc5..0b67f79c17f7 100644
--- a/external/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1174,7 +1174,9 @@ void ELFObjectWriter::reset() {
   OverrideABIVersion.reset();
   Relocations.clear();
   Renames.clear();
+  Weakrefs.clear();
   Symvers.clear();
+  SeenGnuAbi = false;
   MCObjectWriter::reset();
 }
 
diff --git a/external/llvm-project/llvm/lib/MC/MCAsmInfo.cpp b/external/llvm-project/llvm/lib/MC/MCAsmInfo.cpp
index 86759c32bb75..e8eaf4619df5 100644
--- a/external/llvm-project/llvm/lib/MC/MCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCAsmInfo.cpp
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
@@ -148,3 +149,25 @@ std::optional<uint32_t> MCAsmInfo::getSpecifierForName(StringRef Name) const {
     return It->second;
   return {};
 }
+
+void MCAsmInfo::printExpr(raw_ostream &OS, const MCExpr &Expr) const {
+  if (auto *SE = dyn_cast<MCSpecifierExpr>(&Expr))
+    printSpecifierExpr(OS, *SE);
+  else
+    Expr.print(OS, this);
+}
+
+void MCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                   const MCSpecifierExpr &Expr) const {
+  // TODO: Switch to unreachable after all targets that use MCSpecifierExpr
+  // migrate to MCAsmInfo::printSpecifierExpr.
+  Expr.printImpl(OS, this);
+}
+
+bool MCAsmInfo::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr,
+                                          MCValue &Res,
+                                          const MCAssembler *Asm) const {
+  // TODO: Remove after all targets that use MCSpecifierExpr migrate to
+  // MCAsmInfo::evaluateAsRelocatableImpl.
+  return Expr.evaluateAsRelocatableImpl(Res, Asm);
+}
diff --git a/external/llvm-project/llvm/lib/MC/MCAsmStreamer.cpp b/external/llvm-project/llvm/lib/MC/MCAsmStreamer.cpp
index 691a693e5327..2cf162eda779 100644
--- a/external/llvm-project/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCAsmStreamer.cpp
@@ -710,10 +710,12 @@ void MCAsmStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
     if (E->inlineAssignedExpr())
       EmitSet = false;
   if (EmitSet) {
-    OS << ".set ";
+    bool UseSet = MAI->usesSetToEquateSymbol();
+    if (UseSet)
+      OS << ".set ";
     Symbol->print(OS, MAI);
-    OS << ", ";
-    Value->print(OS, MAI);
+    OS << (UseSet ? ", " : " = ");
+    MAI->printExpr(OS, *Value);
 
     EmitEOL();
   }
@@ -726,7 +728,7 @@ void MCAsmStreamer::emitConditionalAssignment(MCSymbol *Symbol,
   OS << ".lto_set_conditional ";
   Symbol->print(OS, MAI);
   OS << ", ";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   EmitEOL();
 }
 
@@ -1078,7 +1080,7 @@ void MCAsmStreamer::emitELFSize(MCSymbol *Symbol, const MCExpr *Value) {
   OS << "\t.size\t";
   Symbol->print(OS, MAI);
   OS << ", ";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   EmitEOL();
 }
 
@@ -1412,7 +1414,7 @@ void MCAsmStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
   if (MCTargetStreamer *TS = getTargetStreamer()) {
     TS->emitValue(Value);
   } else {
-    Value->print(OS, MAI);
+    MAI->printExpr(OS, *Value);
     EmitEOL();
   }
 }
@@ -1424,7 +1426,7 @@ void MCAsmStreamer::emitULEB128Value(const MCExpr *Value) {
     return;
   }
   OS << "\t.uleb128 ";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   EmitEOL();
 }
 
@@ -1435,7 +1437,7 @@ void MCAsmStreamer::emitSLEB128Value(const MCExpr *Value) {
     return;
   }
   OS << "\t.sleb128 ";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   EmitEOL();
 }
 
@@ -1450,7 +1452,7 @@ void MCAsmStreamer::emitFill(const MCExpr &NumBytes, uint64_t FillValue,
     if (!MAI->isAIX() || FillValue == 0) {
       // FIXME: Emit location directives
       OS << ZeroDirective;
-      NumBytes.print(OS, MAI);
+      MAI->printExpr(OS, NumBytes);
       if (FillValue != 0)
         OS << ',' << (int)FillValue;
       EmitEOL();
@@ -1473,7 +1475,7 @@ void MCAsmStreamer::emitFill(const MCExpr &NumValues, int64_t Size,
                              int64_t Expr, SMLoc Loc) {
   // FIXME: Emit location directives
   OS << "\t.fill\t";
-  NumValues.print(OS, MAI);
+  MAI->printExpr(OS, NumValues);
   OS << ", " << Size << ", 0x";
   OS.write_hex(truncateToSize(Expr, 4));
   EmitEOL();
@@ -1571,7 +1573,7 @@ void MCAsmStreamer::emitValueToOffset(const MCExpr *Offset,
                                       SMLoc Loc) {
   // FIXME: Verify that Offset is associated with the current section.
   OS << ".org ";
-  Offset->print(OS, MAI);
+  MAI->printExpr(OS, *Offset);
   OS << ", " << (unsigned)Value;
   EmitEOL();
 }
@@ -2491,7 +2493,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
     MCFixup &F = Fixups[i];
     OS << "  fixup " << char('A' + i) << " - "
        << "offset: " << F.getOffset() << ", value: ";
-    F.getValue()->print(OS, MAI);
+    MAI->printExpr(OS, *F.getValue());
     auto Kind = F.getKind();
     if (mc::isRelocation(Kind))
       OS << ", relocation type: " << Kind;
@@ -2570,11 +2572,11 @@ MCAsmStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
                                   const MCExpr *Expr, SMLoc,
                                   const MCSubtargetInfo &STI) {
   OS << "\t.reloc ";
-  Offset.print(OS, MAI);
+  MAI->printExpr(OS, Offset);
   OS << ", " << Name;
   if (Expr) {
     OS << ", ";
-    Expr->print(OS, MAI);
+    MAI->printExpr(OS, *Expr);
   }
   EmitEOL();
   return std::nullopt;
diff --git a/external/llvm-project/llvm/lib/MC/MCAssembler.cpp b/external/llvm-project/llvm/lib/MC/MCAssembler.cpp
index bde126c5c109..1866c5b9e0e8 100644
--- a/external/llvm-project/llvm/lib/MC/MCAssembler.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCAssembler.cpp
@@ -90,6 +90,8 @@ MCAssembler::MCAssembler(MCContext &Context,
 }
 
 void MCAssembler::reset() {
+  HasLayout = false;
+  HasFinalLayout = false;
   RelaxAll = false;
   Sections.clear();
   Symbols.clear();
@@ -386,28 +388,6 @@ void MCAssembler::layoutBundle(MCFragment *Prev, MCFragment *F) const {
       DF->Offset = EF->Offset;
 }
 
-void MCAssembler::ensureValid(MCSection &Sec) const {
-  if (Sec.hasLayout())
-    return;
-  Sec.setHasLayout(true);
-  MCFragment *Prev = nullptr;
-  uint64_t Offset = 0;
-  for (MCFragment &F : Sec) {
-    F.Offset = Offset;
-    if (isBundlingEnabled() && F.hasInstructions()) {
-      layoutBundle(Prev, &F);
-      Offset = F.Offset;
-    }
-    Offset += computeFragmentSize(F);
-    Prev = &F;
-  }
-}
-
-uint64_t MCAssembler::getFragmentOffset(const MCFragment &F) const {
-  ensureValid(*F.getParent());
-  return F.Offset;
-}
-
 // Simple getSymbolOffset helper for the non-variable case.
 static bool getLabelOffset(const MCAssembler &Asm, const MCSymbol &S,
                            bool ReportError, uint64_t &Val) {
@@ -862,22 +842,21 @@ void MCAssembler::layout() {
 
   // Layout until everything fits.
   this->HasLayout = true;
-  while (layoutOnce()) {
+  for (MCSection &Sec : *this)
+    layoutSection(Sec);
+  while (relaxOnce())
     if (getContext().hadError())
       return;
-    // Size of fragments in one section can depend on the size of fragments in
-    // another. If any fragment has changed size, we have to re-layout (and
-    // as a result possibly further relax) all.
-    for (MCSection &Sec : *this)
-      Sec.setHasLayout(false);
-  }
 
   DEBUG_WITH_TYPE("mc-dump", {
       errs() << "assembler backend - post-relaxation\n--\n";
       dump(); });
 
-  // Finalize the layout, including fragment lowering.
-  getBackend().finishLayout(*this);
+  // Some targets might want to adjust fragment offsets. If so, perform another
+  // layout iteration.
+  if (getBackend().finishLayout(*this))
+    for (MCSection &Sec : *this)
+      layoutSection(Sec);
 
   flushPendingErrors();
 
@@ -1183,6 +1162,14 @@ bool MCAssembler::relaxCVDefRange(MCCVDefRangeFragment &F) {
   return OldSize != F.getContents().size();
 }
 
+bool MCAssembler::relaxFill(MCFillFragment &F) {
+  uint64_t Size = computeFragmentSize(F);
+  if (F.getSize() == Size)
+    return false;
+  F.setSize(Size);
+  return true;
+}
+
 bool MCAssembler::relaxPseudoProbeAddr(MCPseudoProbeAddrFragment &PF) {
   uint64_t OldSize = PF.getContents().size();
   int64_t AddrDelta;
@@ -1219,21 +1206,54 @@ bool MCAssembler::relaxFragment(MCFragment &F) {
     return relaxCVInlineLineTable(cast<MCCVInlineLineTableFragment>(F));
   case MCFragment::FT_CVDefRange:
     return relaxCVDefRange(cast<MCCVDefRangeFragment>(F));
+  case MCFragment::FT_Fill:
+    return relaxFill(cast<MCFillFragment>(F));
   case MCFragment::FT_PseudoProbe:
     return relaxPseudoProbeAddr(cast<MCPseudoProbeAddrFragment>(F));
   }
 }
 
-bool MCAssembler::layoutOnce() {
+void MCAssembler::layoutSection(MCSection &Sec) {
+  MCFragment *Prev = nullptr;
+  uint64_t Offset = 0;
+  for (MCFragment &F : Sec) {
+    F.Offset = Offset;
+    if (LLVM_UNLIKELY(isBundlingEnabled())) {
+      if (F.hasInstructions()) {
+        layoutBundle(Prev, &F);
+        Offset = F.Offset;
+      }
+      Prev = &F;
+    }
+    Offset += computeFragmentSize(F);
+  }
+}
+
+bool MCAssembler::relaxOnce() {
   ++stats::RelaxationSteps;
   PendingErrors.clear();
 
-  bool Changed = false;
-  for (MCSection &Sec : *this)
-    for (MCFragment &Frag : Sec)
-      if (relaxFragment(Frag))
-        Changed = true;
-  return Changed;
+  // Size of fragments in one section can depend on the size of fragments in
+  // another. If any fragment has changed size, we have to re-layout (and
+  // as a result possibly further relax) all sections.
+  bool ChangedAny = false;
+  for (MCSection &Sec : *this) {
+    // Assume each iteration finalizes at least one extra fragment. If the
+    // layout does not converge after N+1 iterations, bail out.
+    auto MaxIter = Sec.curFragList()->Tail->getLayoutOrder() + 1;
+    for (;;) {
+      bool Changed = false;
+      for (MCFragment &F : Sec)
+        if (relaxFragment(F))
+          Changed = true;
+
+      ChangedAny |= Changed;
+      if (!Changed || --MaxIter == 0)
+        break;
+      layoutSection(Sec);
+    }
+  }
+  return ChangedAny;
 }
 
 void MCAssembler::reportError(SMLoc L, const Twine &Msg) const {
diff --git a/external/llvm-project/llvm/lib/MC/MCContext.cpp b/external/llvm-project/llvm/lib/MC/MCContext.cpp
index bf66284933d5..8f08aca7717b 100644
--- a/external/llvm-project/llvm/lib/MC/MCContext.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCContext.cpp
@@ -185,6 +185,8 @@ void MCContext::reset() {
   XCOFFUniquingMap.clear();
   DXCUniquingMap.clear();
 
+  RelSecNames.clear();
+  MacroMap.clear();
   ELFEntrySizeMap.clear();
   ELFSeenGenericMergeableSections.clear();
 
@@ -632,9 +634,6 @@ MCSectionELF *MCContext::getELFSection(const Twine &Section, unsigned Type,
                                        const MCSymbolELF *GroupSym,
                                        bool IsComdat, unsigned UniqueID,
                                        const MCSymbolELF *LinkedToSym) {
-  StringRef Group = "";
-  if (GroupSym)
-    Group = GroupSym->getName();
   assert(!(LinkedToSym && LinkedToSym->getName().empty()));
 
   // Sections are differentiated by the quadruple (section_name, group_name,
diff --git a/external/llvm-project/llvm/lib/MC/MCDwarf.cpp b/external/llvm-project/llvm/lib/MC/MCDwarf.cpp
index f0fe56b87d63..61f569acbfeb 100644
--- a/external/llvm-project/llvm/lib/MC/MCDwarf.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCDwarf.cpp
@@ -144,19 +144,33 @@ makeStartPlusIntExpr(MCContext &Ctx, const MCSymbol &Start, int IntVal) {
 void MCLineSection::addEndEntry(MCSymbol *EndLabel) {
   auto *Sec = &EndLabel->getSection();
   // The line table may be empty, which we should skip adding an end entry.
-  // There are two cases:
+  // There are three cases:
   // (1) MCAsmStreamer - emitDwarfLocDirective emits a location directive in
   //     place instead of adding a line entry if the target has
   //     usesDwarfFileAndLocDirectives.
   // (2) MCObjectStreamer - if a function has incomplete debug info where
   //     instructions don't have DILocations, the line entries are missing.
+  // (3) It's also possible that there are no prior line entries if the section
+  //     itself is empty before this end label.
   auto I = MCLineDivisions.find(Sec);
-  if (I != MCLineDivisions.end()) {
-    auto &Entries = I->second;
-    auto EndEntry = Entries.back();
-    EndEntry.setEndLabel(EndLabel);
-    Entries.push_back(EndEntry);
-  }
+  if (I == MCLineDivisions.end()) // If section not found, do nothing.
+    return;
+
+  auto &Entries = I->second;
+  // If no entries in this section's list, nothing to base the end entry on.
+  if (Entries.empty())
+    return;
+
+  // Create the end entry based on the last existing entry.
+  MCDwarfLineEntry EndEntry = Entries.back();
+
+  // An end entry is just for marking the end of a sequence of code locations.
+  // It should not carry forward a LineStreamLabel from a previous special entry
+  // if Entries.back() happened to be such an entry. So here we clear
+  // LineStreamLabel.
+  EndEntry.LineStreamLabel = nullptr;
+  EndEntry.setEndLabel(EndLabel);
+  Entries.push_back(EndEntry);
 }
 
 //
@@ -1227,7 +1241,7 @@ void MCGenDwarfInfo::Emit(MCStreamer *MCOS) {
 // a dwarf label.
 //
 void MCGenDwarfLabelEntry::Make(MCSymbol *Symbol, MCStreamer *MCOS,
-                                     SourceMgr &SrcMgr, SMLoc &Loc) {
+                                SourceMgr &SrcMgr, SMLoc &Loc) {
   // We won't create dwarf labels for temporary symbols.
   if (Symbol->isTemporary())
     return;
@@ -1336,7 +1350,7 @@ static unsigned getSizeForEncoding(MCStreamer &streamer,
 }
 
 static void emitFDESymbol(MCObjectStreamer &streamer, const MCSymbol &symbol,
-                       unsigned symbolEncoding, bool isEH) {
+                          unsigned symbolEncoding, bool isEH) {
   MCContext &context = streamer.getContext();
   const MCAsmInfo *asmInfo = context.getAsmInfo();
   const MCExpr *v = asmInfo->getExprForFDESymbol(&symbol,
diff --git a/external/llvm-project/llvm/lib/MC/MCExpr.cpp b/external/llvm-project/llvm/lib/MC/MCExpr.cpp
index f7224d61614e..89191294f3ed 100644
--- a/external/llvm-project/llvm/lib/MC/MCExpr.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCExpr.cpp
@@ -15,6 +15,7 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Casting.h"
@@ -171,6 +172,16 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI,
       OS << ')';
     return;
   }
+
+  case MCExpr::Specifier: {
+    auto &SE = cast<MCSpecifierExpr>(*this);
+    if (MAI)
+      return MAI->printSpecifierExpr(OS, SE);
+    // Used by dump features like -show-inst. Regular MCAsmStreamer output must
+    // set MAI.
+    OS << "specifier(" << SE.getSpecifier() << ',' << *SE.getSubExpr() << ')';
+    return;
+  }
   }
 
   llvm_unreachable("Invalid expression kind!");
@@ -183,35 +194,6 @@ LLVM_DUMP_METHOD void MCExpr::dump() const {
 }
 #endif
 
-bool MCExpr::isSymbolUsedInExpression(const MCSymbol *Sym) const {
-  switch (getKind()) {
-  case MCExpr::Binary: {
-    const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(this);
-    return BE->getLHS()->isSymbolUsedInExpression(Sym) ||
-           BE->getRHS()->isSymbolUsedInExpression(Sym);
-  }
-  case MCExpr::Target: {
-    const MCTargetExpr *TE = static_cast<const MCTargetExpr *>(this);
-    return TE->isSymbolUsedInExpression(Sym);
-  }
-  case MCExpr::Constant:
-    return false;
-  case MCExpr::SymbolRef: {
-    const MCSymbol &S = static_cast<const MCSymbolRefExpr *>(this)->getSymbol();
-    if (S.isVariable() && !S.isWeakExternal())
-      return S.getVariableValue()->isSymbolUsedInExpression(Sym);
-    return &S == Sym;
-  }
-  case MCExpr::Unary: {
-    const MCExpr *SubExpr =
-        static_cast<const MCUnaryExpr *>(this)->getSubExpr();
-    return SubExpr->isSymbolUsedInExpression(Sym);
-  }
-  }
-
-  llvm_unreachable("Unknown expr kind!");
-}
-
 /* *** */
 
 const MCBinaryExpr *MCBinaryExpr::create(Opcode Opc, const MCExpr *LHS,
@@ -702,6 +684,11 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
 
     return true;
   }
+  case Specifier:
+    // Fold the expression during relocation generation. As parse time Asm might
+    // be null, and targets should not rely on the folding.
+    return Asm && Asm->getContext().getAsmInfo()->evaluateAsRelocatableImpl(
+                      cast<MCSpecifierExpr>(*this), Res, Asm);
   }
 
   llvm_unreachable("Invalid assembly expression kind!");
@@ -750,7 +737,29 @@ MCFragment *MCExpr::findAssociatedFragment() const {
     // Otherwise, return the first non-null fragment.
     return LHS_F ? LHS_F : RHS_F;
   }
+
+  case Specifier:
+    return cast<MCSpecifierExpr>(this)->getSubExpr()->findAssociatedFragment();
   }
 
   llvm_unreachable("Invalid assembly expression kind!");
 }
+
+const MCSpecifierExpr *MCSpecifierExpr::create(const MCExpr *Expr, Spec S,
+                                               MCContext &Ctx, SMLoc Loc) {
+  return new (Ctx) MCSpecifierExpr(Expr, S, Loc);
+}
+
+const MCSpecifierExpr *MCSpecifierExpr::create(const MCSymbol *Sym, Spec S,
+                                               MCContext &Ctx, SMLoc Loc) {
+  return new (Ctx) MCSpecifierExpr(MCSymbolRefExpr::create(Sym, Ctx), S, Loc);
+}
+
+bool MCSpecifierExpr::evaluateAsRelocatableImpl(MCValue &Res,
+                                                const MCAssembler *Asm) const {
+  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
+    return false;
+
+  Res.setSpecifier(specifier);
+  return !Res.getSubSym();
+}
diff --git a/external/llvm-project/llvm/lib/MC/MCInst.cpp b/external/llvm-project/llvm/lib/MC/MCInst.cpp
index 639619fe4e99..832d25060f88 100644
--- a/external/llvm-project/llvm/lib/MC/MCInst.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCInst.cpp
@@ -35,7 +35,7 @@ void MCOperand::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
   else if (isDFPImm())
     OS << "DFPImm:" << bit_cast<double>(getDFPImm());
   else if (isExpr()) {
-    OS << "Expr:(" << *getExpr() << ")";
+    OS << "Expr:" << *getExpr();
   } else if (isInst()) {
     OS << "Inst:(";
     if (const auto *Inst = getInst())
diff --git a/external/llvm-project/llvm/lib/MC/MCObjectStreamer.cpp b/external/llvm-project/llvm/lib/MC/MCObjectStreamer.cpp
index e3d5a5a9a132..1bb2143ed6ab 100644
--- a/external/llvm-project/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCObjectStreamer.cpp
@@ -261,7 +261,7 @@ void MCObjectStreamer::emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc,
 
 void MCObjectStreamer::emitULEB128Value(const MCExpr *Value) {
   int64_t IntValue;
-  if (Value->evaluateAsAbsolute(IntValue, getAssemblerPtr())) {
+  if (Value->evaluateAsAbsolute(IntValue, getAssembler())) {
     emitULEB128IntValue(IntValue);
     return;
   }
@@ -270,7 +270,7 @@ void MCObjectStreamer::emitULEB128Value(const MCExpr *Value) {
 
 void MCObjectStreamer::emitSLEB128Value(const MCExpr *Value) {
   int64_t IntValue;
-  if (Value->evaluateAsAbsolute(IntValue, getAssemblerPtr())) {
+  if (Value->evaluateAsAbsolute(IntValue, getAssembler())) {
     emitSLEB128IntValue(IntValue);
     return;
   }
@@ -727,7 +727,7 @@ void MCObjectStreamer::emitFill(const MCExpr &NumValues, int64_t Size,
                                 int64_t Expr, SMLoc Loc) {
   int64_t IntNumValues;
   // Do additional checking now if we can resolve the value.
-  if (NumValues.evaluateAsAbsolute(IntNumValues, getAssemblerPtr())) {
+  if (NumValues.evaluateAsAbsolute(IntNumValues, getAssembler())) {
     if (IntNumValues < 0) {
       getContext().getSourceManager()->PrintMessage(
           Loc, SourceMgr::DK_Warning,
diff --git a/external/llvm-project/llvm/lib/MC/MCParser/AsmParser.cpp b/external/llvm-project/llvm/lib/MC/MCParser/AsmParser.cpp
index d3f534da6194..cb33f5dfbfd3 100644
--- a/external/llvm-project/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -1352,6 +1352,8 @@ const MCExpr *MCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec) {
   // Recurse over the given expression, rebuilding it to apply the given variant
   // if there is exactly one symbol.
   switch (E->getKind()) {
+  case MCExpr::Specifier:
+    llvm_unreachable("cannot apply another specifier to MCSpecifierExpr");
   case MCExpr::Target:
   case MCExpr::Constant:
     return nullptr;
diff --git a/external/llvm-project/llvm/lib/MC/MCSection.cpp b/external/llvm-project/llvm/lib/MC/MCSection.cpp
index 0e64c887e91b..b61738d8837a 100644
--- a/external/llvm-project/llvm/lib/MC/MCSection.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCSection.cpp
@@ -22,8 +22,8 @@ using namespace llvm;
 MCSection::MCSection(SectionVariant V, StringRef Name, bool IsText,
                      bool IsVirtual, MCSymbol *Begin)
     : Begin(Begin), BundleGroupBeforeFirstInst(false), HasInstructions(false),
-      HasLayout(false), IsRegistered(false), IsText(IsText),
-      IsVirtual(IsVirtual), LinkerRelaxable(false), Name(Name), Variant(V) {
+      IsRegistered(false), IsText(IsText), IsVirtual(IsVirtual),
+      LinkerRelaxable(false), Name(Name), Variant(V) {
   // The initial subsection number is 0. Create a fragment list.
   CurFragList = &Subsections.emplace_back(0u, FragList{}).second;
 }
diff --git a/external/llvm-project/llvm/lib/MC/MCStreamer.cpp b/external/llvm-project/llvm/lib/MC/MCStreamer.cpp
index fb1858d25929..cdcf279e6f90 100644
--- a/external/llvm-project/llvm/lib/MC/MCStreamer.cpp
+++ b/external/llvm-project/llvm/lib/MC/MCStreamer.cpp
@@ -72,7 +72,7 @@ void MCTargetStreamer::emitValue(const MCExpr *Value) {
   SmallString<128> Str;
   raw_svector_ostream OS(Str);
 
-  Value->print(OS, Streamer.getContext().getAsmInfo());
+  Streamer.getContext().getAsmInfo()->printExpr(OS, *Value);
   Streamer.emitRawText(OS.str());
 }
 
@@ -1240,6 +1240,10 @@ void MCStreamer::visitUsedExpr(const MCExpr &Expr) {
   case MCExpr::Unary:
     visitUsedExpr(*cast<MCUnaryExpr>(Expr).getSubExpr());
     break;
+
+  case MCExpr::Specifier:
+    visitUsedExpr(*cast<MCSpecifierExpr>(Expr).getSubExpr());
+    break;
   }
 }
 
diff --git a/external/llvm-project/llvm/lib/MC/StringTableBuilder.cpp b/external/llvm-project/llvm/lib/MC/StringTableBuilder.cpp
index df316bae98ce..7accdc2a9e77 100644
--- a/external/llvm-project/llvm/lib/MC/StringTableBuilder.cpp
+++ b/external/llvm-project/llvm/lib/MC/StringTableBuilder.cpp
@@ -204,7 +204,7 @@ size_t StringTableBuilder::add(CachedHashStringRef S) {
     assert(S.size() > COFF::NameSize && "Short string in COFF string table!");
 
   assert(!isFinalized());
-  auto P = StringIndexMap.insert(std::make_pair(S, 0));
+  auto P = StringIndexMap.try_emplace(S);
   if (P.second) {
     size_t Start = alignTo(Size, Alignment);
     P.first->second = Start;
diff --git a/external/llvm-project/llvm/lib/MCA/Stages/DispatchStage.cpp b/external/llvm-project/llvm/lib/MCA/Stages/DispatchStage.cpp
index 10e433bf1689..3e5d35a60516 100644
--- a/external/llvm-project/llvm/lib/MCA/Stages/DispatchStage.cpp
+++ b/external/llvm-project/llvm/lib/MCA/Stages/DispatchStage.cpp
@@ -17,7 +17,6 @@
 
 #include "llvm/MCA/Stages/DispatchStage.h"
 #include "llvm/MCA/HWEventListener.h"
-#include "llvm/MCA/HardwareUnits/Scheduler.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "llvm-mca"
diff --git a/external/llvm-project/llvm/lib/MCA/View.cpp b/external/llvm-project/llvm/lib/MCA/View.cpp
index a56d3a124934..45c7c0935b37 100644
--- a/external/llvm-project/llvm/lib/MCA/View.cpp
+++ b/external/llvm-project/llvm/lib/MCA/View.cpp
@@ -12,8 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MCA/View.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCSubtargetInfo.h"
 
 namespace llvm {
 namespace mca {
diff --git a/external/llvm-project/llvm/lib/ObjCopy/MachO/MachOObject.h b/external/llvm-project/llvm/lib/ObjCopy/MachO/MachOObject.h
index 79eb0133c280..13ac87ed3ed0 100644
--- a/external/llvm-project/llvm/lib/ObjCopy/MachO/MachOObject.h
+++ b/external/llvm-project/llvm/lib/ObjCopy/MachO/MachOObject.h
@@ -71,7 +71,7 @@ struct Section {
   }
 
   bool hasValidOffset() const {
-    return !(isVirtualSection() || (OriginalOffset && *OriginalOffset == 0));
+    return !(isVirtualSection() || OriginalOffset == 0);
   }
 };
 
diff --git a/external/llvm-project/llvm/lib/Object/ELF.cpp b/external/llvm-project/llvm/lib/Object/ELF.cpp
index dfc8acb0e454..e6864ca508a5 100644
--- a/external/llvm-project/llvm/lib/Object/ELF.cpp
+++ b/external/llvm-project/llvm/lib/Object/ELF.cpp
@@ -9,6 +9,7 @@
 #include "llvm/Object/ELF.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/Object/Decompressor.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 
@@ -782,6 +783,27 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
   if (!ContentsOrErr)
     return ContentsOrErr.takeError();
   ArrayRef<uint8_t> Content = *ContentsOrErr;
+
+  // Decompress the section if needed.
+  std::unique_ptr<uint8_t[]> DecompressedContent;
+  if (Sec.sh_flags & llvm::ELF::SHF_COMPRESSED) {
+    Expected<StringRef> SectionNameOrErr = EF.getSectionName(Sec);
+    if (!SectionNameOrErr)
+      return SectionNameOrErr.takeError();
+    auto DecompressorOrErr =
+        Decompressor::create(*SectionNameOrErr, toStringRef(*ContentsOrErr),
+                             EF.isLE(), ELFT::Is64Bits);
+    if (!DecompressorOrErr)
+      return DecompressorOrErr.takeError();
+    size_t DecompressedSize = DecompressorOrErr->getDecompressedSize();
+    DecompressedContent = std::make_unique<uint8_t[]>(DecompressedSize);
+    MutableArrayRef<uint8_t> DecompressedContentRef(DecompressedContent.get(),
+                                                    DecompressedSize);
+    if (Error Err = DecompressorOrErr->decompress(DecompressedContentRef))
+      return std::move(Err);
+    Content = DecompressedContentRef;
+  }
+
   DataExtractor Data(Content, EF.isLE(), ELFT::Is64Bits ? 8 : 4);
   std::vector<BBAddrMap> FunctionEntries;
 
diff --git a/external/llvm-project/llvm/lib/Object/ELFObjectFile.cpp b/external/llvm-project/llvm/lib/Object/ELFObjectFile.cpp
index bf6d51919b5c..dd240550cba0 100644
--- a/external/llvm-project/llvm/lib/Object/ELFObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Object/ELFObjectFile.cpp
@@ -736,8 +736,7 @@ void ELFObjectFileBase::setARMSubArch(Triple &TheTriple) const {
     case ARMBuildAttrs::v7: {
       std::optional<unsigned> ArchProfileAttr =
           Attributes.getAttributeValue(ARMBuildAttrs::CPU_arch_profile);
-      if (ArchProfileAttr &&
-          *ArchProfileAttr == ARMBuildAttrs::MicroControllerProfile)
+      if (ArchProfileAttr == ARMBuildAttrs::MicroControllerProfile)
         Triple += "v7m";
       else
         Triple += "v7";
diff --git a/external/llvm-project/llvm/lib/Object/OffloadBundle.cpp b/external/llvm-project/llvm/lib/Object/OffloadBundle.cpp
index 239a3e2616ba..a4c1e0547a5b 100644
--- a/external/llvm-project/llvm/lib/Object/OffloadBundle.cpp
+++ b/external/llvm-project/llvm/lib/Object/OffloadBundle.cpp
@@ -7,9 +7,7 @@
 //===----------------------------------------------------------------===//
 
 #include "llvm/Object/OffloadBundle.h"
-#include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Magic.h"
-#include "llvm/IR/Constants.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/MC/StringTableBuilder.h"
@@ -20,7 +18,6 @@
 #include "llvm/Object/Error.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Object/ObjectFile.h"
-#include "llvm/Support/Alignment.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/Timer.h"
diff --git a/external/llvm-project/llvm/lib/Object/XCOFFObjectFile.cpp b/external/llvm-project/llvm/lib/Object/XCOFFObjectFile.cpp
index 5a246438e2c0..c5a2ec219e91 100644
--- a/external/llvm-project/llvm/lib/Object/XCOFFObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Object/XCOFFObjectFile.cpp
@@ -93,8 +93,8 @@ uint8_t XCOFFRelocation<AddressType>::getRelocatedLength() const {
   return (Info & XR_BIASED_LENGTH_MASK) + 1;
 }
 
-template struct ExceptionSectionEntry<support::ubig32_t>;
-template struct ExceptionSectionEntry<support::ubig64_t>;
+template struct LLVM_EXPORT_TEMPLATE ExceptionSectionEntry<support::ubig32_t>;
+template struct LLVM_EXPORT_TEMPLATE ExceptionSectionEntry<support::ubig64_t>;
 
 template <typename T>
 Expected<StringRef> getLoaderSecSymNameInStrTbl(const T *LoaderSecHeader,
@@ -1375,11 +1375,11 @@ Expected<StringRef> XCOFFSymbolRef::getName() const {
 }
 
 // Explicitly instantiate template classes.
-template struct XCOFFSectionHeader<XCOFFSectionHeader32>;
-template struct XCOFFSectionHeader<XCOFFSectionHeader64>;
+template struct LLVM_EXPORT_TEMPLATE XCOFFSectionHeader<XCOFFSectionHeader32>;
+template struct LLVM_EXPORT_TEMPLATE XCOFFSectionHeader<XCOFFSectionHeader64>;
 
-template struct XCOFFRelocation<llvm::support::ubig32_t>;
-template struct XCOFFRelocation<llvm::support::ubig64_t>;
+template struct LLVM_EXPORT_TEMPLATE XCOFFRelocation<llvm::support::ubig32_t>;
+template struct LLVM_EXPORT_TEMPLATE XCOFFRelocation<llvm::support::ubig64_t>;
 
 template LLVM_EXPORT_TEMPLATE
     llvm::Expected<llvm::ArrayRef<llvm::object::XCOFFRelocation64>>
diff --git a/external/llvm-project/llvm/lib/ObjectYAML/MachOEmitter.cpp b/external/llvm-project/llvm/lib/ObjectYAML/MachOEmitter.cpp
index 3c62fa137ced..35d442e8e343 100644
--- a/external/llvm-project/llvm/lib/ObjectYAML/MachOEmitter.cpp
+++ b/external/llvm-project/llvm/lib/ObjectYAML/MachOEmitter.cpp
@@ -19,8 +19,8 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/LEB128.h"
-#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
diff --git a/external/llvm-project/llvm/lib/ObjectYAML/MachOYAML.cpp b/external/llvm-project/llvm/lib/ObjectYAML/MachOYAML.cpp
index b7eda97c22ae..133358d39592 100644
--- a/external/llvm-project/llvm/lib/ObjectYAML/MachOYAML.cpp
+++ b/external/llvm-project/llvm/lib/ObjectYAML/MachOYAML.cpp
@@ -13,9 +13,9 @@
 #include "llvm/ObjectYAML/MachOYAML.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/TargetParser/Host.h"
 #include <cstdint>
 #include <cstring>
diff --git a/external/llvm-project/llvm/lib/Option/ArgList.cpp b/external/llvm-project/llvm/lib/Option/ArgList.cpp
index 6e164150d2e5..c4188b3b1211 100644
--- a/external/llvm-project/llvm/lib/Option/ArgList.cpp
+++ b/external/llvm-project/llvm/lib/Option/ArgList.cpp
@@ -6,16 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Option/ArgList.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Option/Arg.h"
-#include "llvm/Option/ArgList.h"
-#include "llvm/Option/Option.h"
 #include "llvm/Option/OptSpecifier.h"
+#include "llvm/Option/Option.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/external/llvm-project/llvm/lib/Passes/PassBuilder.cpp b/external/llvm-project/llvm/lib/Passes/PassBuilder.cpp
index 473858a99ee5..7c0dc3290e10 100644
--- a/external/llvm-project/llvm/lib/Passes/PassBuilder.cpp
+++ b/external/llvm-project/llvm/lib/Passes/PassBuilder.cpp
@@ -22,7 +22,6 @@
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/CFGPrinter.h"
 #include "llvm/Analysis/CFGSCCPrinter.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraph.h"
@@ -42,8 +41,8 @@
 #include "llvm/Analysis/EphemeralValuesCache.h"
 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
 #include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/HashRecognize.h"
 #include "llvm/Analysis/IR2Vec.h"
-#include "llvm/Analysis/IRSimilarityIdentifier.h"
 #include "llvm/Analysis/IVUsers.h"
 #include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
@@ -64,7 +63,6 @@
 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
 #include "llvm/Analysis/MustExecute.h"
 #include "llvm/Analysis/ObjCARCAliasAnalysis.h"
-#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/PhiValues.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
@@ -122,13 +120,10 @@
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
 #include "llvm/CodeGen/MachineCSE.h"
 #include "llvm/CodeGen/MachineCopyPropagation.h"
-#include "llvm/CodeGen/MachineCycleAnalysis.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/CodeGen/MachineLICM.h"
 #include "llvm/CodeGen/MachineLateInstrsCleanup.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
-#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -180,7 +175,6 @@
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/PrintPasses.h"
 #include "llvm/IR/SafepointIRVerifier.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/IRPrinter/IRPrintingPasses.h"
@@ -222,13 +216,11 @@
 #include "llvm/Transforms/IPO/HotColdSplitting.h"
 #include "llvm/Transforms/IPO/IROutliner.h"
 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
-#include "llvm/Transforms/IPO/Inliner.h"
 #include "llvm/Transforms/IPO/Internalize.h"
 #include "llvm/Transforms/IPO/LoopExtractor.h"
 #include "llvm/Transforms/IPO/LowerTypeTests.h"
 #include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
 #include "llvm/Transforms/IPO/MergeFunctions.h"
-#include "llvm/Transforms/IPO/ModuleInliner.h"
 #include "llvm/Transforms/IPO/OpenMPOpt.h"
 #include "llvm/Transforms/IPO/PartialInlining.h"
 #include "llvm/Transforms/IPO/SCCP.h"
@@ -248,7 +240,8 @@
 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
 #include "llvm/Transforms/Instrumentation/KCFI.h"
 #include "llvm/Transforms/Instrumentation/LowerAllowCheckPass.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 #include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
 #include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
@@ -347,13 +340,11 @@
 #include "llvm/Transforms/Utils/DXILUpgrade.h"
 #include "llvm/Transforms/Utils/Debugify.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
-#include "llvm/Transforms/Utils/ExtraPassManager.h"
 #include "llvm/Transforms/Utils/FixIrreducible.h"
 #include "llvm/Transforms/Utils/HelloWorld.h"
 #include "llvm/Transforms/Utils/IRNormalizer.h"
 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
 #include "llvm/Transforms/Utils/InstructionNamer.h"
-#include "llvm/Transforms/Utils/LCSSA.h"
 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopVersioning.h"
diff --git a/external/llvm-project/llvm/lib/Passes/PassBuilderPipelines.cpp b/external/llvm-project/llvm/lib/Passes/PassBuilderPipelines.cpp
index aa1f72b036e8..a08b2c6c5192 100644
--- a/external/llvm-project/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/external/llvm-project/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -24,7 +24,6 @@
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/Analysis/ScopedNoAliasAA.h"
 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
-#include "llvm/CodeGen/GlobalMergeFunctions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Passes/OptimizationLevel.h"
@@ -77,7 +76,8 @@
 #include "llvm/Transforms/Instrumentation/CGProfile.h"
 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
 #include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"
 #include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
 #include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
@@ -142,7 +142,6 @@
 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
 #include "llvm/Transforms/Utils/RelLookupTableConverter.h"
 #include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
-#include "llvm/Transforms/Vectorize/EVLIndVarSimplify.h"
 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
 #include "llvm/Transforms/Vectorize/VectorCombine.h"
@@ -1296,6 +1295,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   // and argument promotion.
   MPM.addPass(DeadArgumentEliminationPass());
 
+  if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)
+    MPM.addPass(SimplifyTypeTestsPass());
+
   if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink)
     MPM.addPass(CoroCleanupPass());
 
diff --git a/external/llvm-project/llvm/lib/Passes/PassRegistry.def b/external/llvm-project/llvm/lib/Passes/PassRegistry.def
index be93a7b1f5ba..f761d0dab09a 100644
--- a/external/llvm-project/llvm/lib/Passes/PassRegistry.def
+++ b/external/llvm-project/llvm/lib/Passes/PassRegistry.def
@@ -101,6 +101,7 @@ MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass())
 MODULE_PASS("lower-emutls", LowerEmuTLSPass())
 MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass())
 MODULE_PASS("lower-ifunc", LowerIFuncPass())
+MODULE_PASS("simplify-type-tests", SimplifyTypeTestsPass())
 MODULE_PASS("lowertypetests", LowerTypeTestsPass())
 MODULE_PASS("fatlto-cleanup", FatLtoCleanup())
 MODULE_PASS("pgo-force-function-attrs", PGOForceFunctionAttrsPass(PGOOpt ? PGOOpt->ColdOptType : PGOOptions::ColdFuncOpt::Default))
@@ -441,7 +442,6 @@ FUNCTION_PASS("print-cfg-sccs", CFGSCCPrinterPass(errs()))
 FUNCTION_PASS("print-memderefs", MemDerefPrinterPass(errs()))
 FUNCTION_PASS("print-mustexecute", MustExecutePrinterPass(errs()))
 FUNCTION_PASS("print-predicateinfo", PredicateInfoPrinterPass(errs()))
-FUNCTION_PASS("print<access-info>", LoopAccessInfoPrinterPass(errs()))
 FUNCTION_PASS("print<assumptions>", AssumptionPrinterPass(errs()))
 FUNCTION_PASS("print<block-freq>", BlockFrequencyPrinterPass(errs()))
 FUNCTION_PASS("print<branch-prob>", BranchProbabilityPrinterPass(errs()))
@@ -583,6 +583,16 @@ FUNCTION_PASS_WITH_PARAMS(
       return MergedLoadStoreMotionPass(Opts);
     },
     parseMergedLoadStoreMotionOptions, "no-split-footer-bb;split-footer-bb")
+FUNCTION_PASS_WITH_PARAMS(
+    "print<access-info>", "LoopAccessInfoPrinterPass",
+    [](bool AllowPartial) {
+      return LoopAccessInfoPrinterPass(errs(), AllowPartial);
+    },
+    [](StringRef Params) {
+      return PassBuilder::parseSinglePassOption(Params, "allow-partial",
+						"LoopAccessInfoPrinterPass");
+    },
+    "allow-partial")
 FUNCTION_PASS_WITH_PARAMS(
     "print<da>", "DependenceAnalysisPrinterPass",
     [](bool NormalizeResults) {
@@ -662,6 +672,7 @@ LOOPNEST_PASS("no-op-loopnest", NoOpLoopNestPass())
 #define LOOP_ANALYSIS(NAME, CREATE_PASS)
 #endif
 LOOP_ANALYSIS("ddg", DDGAnalysis())
+LOOP_ANALYSIS("hash-recognize", HashRecognizeAnalysis())
 LOOP_ANALYSIS("iv-users", IVUsersAnalysis())
 LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis())
 LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
@@ -695,6 +706,7 @@ LOOP_PASS("loop-versioning-licm", LoopVersioningLICMPass())
 LOOP_PASS("no-op-loop", NoOpLoopPass())
 LOOP_PASS("print", PrintLoopPass(errs()))
 LOOP_PASS("print<ddg>", DDGAnalysisPrinterPass(errs()))
+LOOP_PASS("print<hash-recognize>", HashRecognizePrinterPass(errs()))
 LOOP_PASS("print<iv-users>", IVUsersPrinterPass(errs()))
 LOOP_PASS("print<loop-cache-cost>", LoopCachePrinterPass(errs()))
 LOOP_PASS("print<loopnest>", LoopNestPrinterPass(errs()))
diff --git a/external/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp b/external/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp
index dc1dd5d9c7f4..4e8abc43572d 100644
--- a/external/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/external/llvm-project/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -15,7 +15,6 @@
 #include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/ADT/Any.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/CallGraphSCCPass.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/CodeGen/MIRPrinter.h"
@@ -24,8 +23,6 @@
 #include "llvm/CodeGen/MachineVerifier.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/InstIterator.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassInstrumentation.h"
 #include "llvm/IR/PassManager.h"
@@ -33,12 +30,10 @@
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/GraphWriter.h"
-#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Regex.h"
diff --git a/external/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/external/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index dd74eb054a34..429ec5c19f1f 100644
--- a/external/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/external/llvm-project/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -823,7 +823,8 @@ class MCDCDecisionRecorder {
 
 Error CoverageMapping::loadFunctionRecord(
     const CoverageMappingRecord &Record,
-    IndexedInstrProfReader &ProfileReader) {
+    const std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+        &ProfileReader) {
   StringRef OrigFuncName = Record.FunctionName;
   if (OrigFuncName.empty())
     return make_error<CoverageMapError>(coveragemap_error::malformed,
@@ -837,35 +838,44 @@ Error CoverageMapping::loadFunctionRecord(
   CounterMappingContext Ctx(Record.Expressions);
 
   std::vector<uint64_t> Counts;
-  if (Error E = ProfileReader.getFunctionCounts(Record.FunctionName,
-                                                Record.FunctionHash, Counts)) {
-    instrprof_error IPE = std::get<0>(InstrProfError::take(std::move(E)));
-    if (IPE == instrprof_error::hash_mismatch) {
-      FuncHashMismatches.emplace_back(std::string(Record.FunctionName),
-                                      Record.FunctionHash);
-      return Error::success();
+  if (ProfileReader) {
+    if (Error E = ProfileReader.value().get().getFunctionCounts(
+            Record.FunctionName, Record.FunctionHash, Counts)) {
+      instrprof_error IPE = std::get<0>(InstrProfError::take(std::move(E)));
+      if (IPE == instrprof_error::hash_mismatch) {
+        FuncHashMismatches.emplace_back(std::string(Record.FunctionName),
+                                        Record.FunctionHash);
+        return Error::success();
+      }
+      if (IPE != instrprof_error::unknown_function)
+        return make_error<InstrProfError>(IPE);
+      Counts.assign(getMaxCounterID(Ctx, Record) + 1, 0);
     }
-    if (IPE != instrprof_error::unknown_function)
-      return make_error<InstrProfError>(IPE);
+  } else {
     Counts.assign(getMaxCounterID(Ctx, Record) + 1, 0);
   }
   Ctx.setCounts(Counts);
 
   bool IsVersion11 =
-      ProfileReader.getVersion() < IndexedInstrProf::ProfVersion::Version12;
+      ProfileReader && ProfileReader.value().get().getVersion() <
+                           IndexedInstrProf::ProfVersion::Version12;
 
   BitVector Bitmap;
-  if (Error E = ProfileReader.getFunctionBitmap(Record.FunctionName,
-                                                Record.FunctionHash, Bitmap)) {
-    instrprof_error IPE = std::get<0>(InstrProfError::take(std::move(E)));
-    if (IPE == instrprof_error::hash_mismatch) {
-      FuncHashMismatches.emplace_back(std::string(Record.FunctionName),
-                                      Record.FunctionHash);
-      return Error::success();
+  if (ProfileReader) {
+    if (Error E = ProfileReader.value().get().getFunctionBitmap(
+            Record.FunctionName, Record.FunctionHash, Bitmap)) {
+      instrprof_error IPE = std::get<0>(InstrProfError::take(std::move(E)));
+      if (IPE == instrprof_error::hash_mismatch) {
+        FuncHashMismatches.emplace_back(std::string(Record.FunctionName),
+                                        Record.FunctionHash);
+        return Error::success();
+      }
+      if (IPE != instrprof_error::unknown_function)
+        return make_error<InstrProfError>(IPE);
+      Bitmap = BitVector(getMaxBitmapSize(Record, IsVersion11));
     }
-    if (IPE != instrprof_error::unknown_function)
-      return make_error<InstrProfError>(IPE);
-    Bitmap = BitVector(getMaxBitmapSize(Record, IsVersion11));
+  } else {
+    Bitmap = BitVector(getMaxBitmapSize(Record, false));
   }
   Ctx.setBitmap(std::move(Bitmap));
 
@@ -959,10 +969,14 @@ Error CoverageMapping::loadFunctionRecord(
 // of CoverageMappingReader instances.
 Error CoverageMapping::loadFromReaders(
     ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
-    IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage) {
-  assert(!Coverage.SingleByteCoverage ||
-         *Coverage.SingleByteCoverage == ProfileReader.hasSingleByteCoverage());
-  Coverage.SingleByteCoverage = ProfileReader.hasSingleByteCoverage();
+    std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+        &ProfileReader,
+    CoverageMapping &Coverage) {
+  assert(!Coverage.SingleByteCoverage || !ProfileReader ||
+         *Coverage.SingleByteCoverage ==
+             ProfileReader.value().get().hasSingleByteCoverage());
+  Coverage.SingleByteCoverage =
+      !ProfileReader || ProfileReader.value().get().hasSingleByteCoverage();
   for (const auto &CoverageReader : CoverageReaders) {
     for (auto RecordOrErr : *CoverageReader) {
       if (Error E = RecordOrErr.takeError())
@@ -977,7 +991,8 @@ Error CoverageMapping::loadFromReaders(
 
 Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
     ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
-    IndexedInstrProfReader &ProfileReader) {
+    std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+        &ProfileReader) {
   auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping());
   if (Error E = loadFromReaders(CoverageReaders, ProfileReader, *Coverage))
     return std::move(E);
@@ -986,18 +1001,19 @@ Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
 
 // If E is a no_data_found error, returns success. Otherwise returns E.
 static Error handleMaybeNoDataFoundError(Error E) {
-  return handleErrors(
-      std::move(E), [](const CoverageMapError &CME) {
-        if (CME.get() == coveragemap_error::no_data_found)
-          return static_cast<Error>(Error::success());
-        return make_error<CoverageMapError>(CME.get(), CME.getMessage());
-      });
+  return handleErrors(std::move(E), [](const CoverageMapError &CME) {
+    if (CME.get() == coveragemap_error::no_data_found)
+      return static_cast<Error>(Error::success());
+    return make_error<CoverageMapError>(CME.get(), CME.getMessage());
+  });
 }
 
 Error CoverageMapping::loadFromFile(
     StringRef Filename, StringRef Arch, StringRef CompilationDir,
-    IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage,
-    bool &DataFound, SmallVectorImpl<object::BuildID> *FoundBinaryIDs) {
+    std::optional<std::reference_wrapper<IndexedInstrProfReader>>
+        &ProfileReader,
+    CoverageMapping &Coverage, bool &DataFound,
+    SmallVectorImpl<object::BuildID> *FoundBinaryIDs) {
   auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN(
       Filename, /*IsText=*/false, /*RequiresNullTerminator=*/false);
   if (std::error_code EC = CovMappingBufOrErr.getError())
@@ -1033,13 +1049,23 @@ Error CoverageMapping::loadFromFile(
 }
 
 Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
-    ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename,
-    vfs::FileSystem &FS, ArrayRef<StringRef> Arches, StringRef CompilationDir,
+    ArrayRef<StringRef> ObjectFilenames,
+    std::optional<StringRef> ProfileFilename, vfs::FileSystem &FS,
+    ArrayRef<StringRef> Arches, StringRef CompilationDir,
     const object::BuildIDFetcher *BIDFetcher, bool CheckBinaryIDs) {
-  auto ProfileReaderOrErr = IndexedInstrProfReader::create(ProfileFilename, FS);
-  if (Error E = ProfileReaderOrErr.takeError())
-    return createFileError(ProfileFilename, std::move(E));
-  auto ProfileReader = std::move(ProfileReaderOrErr.get());
+  std::unique_ptr<IndexedInstrProfReader> ProfileReader;
+  if (ProfileFilename) {
+    auto ProfileReaderOrErr =
+        IndexedInstrProfReader::create(ProfileFilename.value(), FS);
+    if (Error E = ProfileReaderOrErr.takeError())
+      return createFileError(ProfileFilename.value(), std::move(E));
+    ProfileReader = std::move(ProfileReaderOrErr.get());
+  }
+  auto ProfileReaderRef =
+      ProfileReader
+          ? std::optional<std::reference_wrapper<IndexedInstrProfReader>>(
+                *ProfileReader)
+          : std::nullopt;
   auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping());
   bool DataFound = false;
 
@@ -1053,16 +1079,17 @@ Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
 
   SmallVector<object::BuildID> FoundBinaryIDs;
   for (const auto &File : llvm::enumerate(ObjectFilenames)) {
-    if (Error E =
-            loadFromFile(File.value(), GetArch(File.index()), CompilationDir,
-                         *ProfileReader, *Coverage, DataFound, &FoundBinaryIDs))
+    if (Error E = loadFromFile(File.value(), GetArch(File.index()),
+                               CompilationDir, ProfileReaderRef, *Coverage,
+                               DataFound, &FoundBinaryIDs))
       return std::move(E);
   }
 
   if (BIDFetcher) {
     std::vector<object::BuildID> ProfileBinaryIDs;
-    if (Error E = ProfileReader->readBinaryIds(ProfileBinaryIDs))
-      return createFileError(ProfileFilename, std::move(E));
+    if (ProfileReader)
+      if (Error E = ProfileReader->readBinaryIds(ProfileBinaryIDs))
+        return createFileError(ProfileFilename.value(), std::move(E));
 
     SmallVector<object::BuildIDRef> BinaryIDsToFetch;
     if (!ProfileBinaryIDs.empty()) {
@@ -1082,12 +1109,12 @@ Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
       if (PathOpt) {
         std::string Path = std::move(*PathOpt);
         StringRef Arch = Arches.size() == 1 ? Arches.front() : StringRef();
-        if (Error E = loadFromFile(Path, Arch, CompilationDir, *ProfileReader,
-                                  *Coverage, DataFound))
+        if (Error E = loadFromFile(Path, Arch, CompilationDir, ProfileReaderRef,
+                                   *Coverage, DataFound))
           return std::move(E);
       } else if (CheckBinaryIDs) {
         return createFileError(
-            ProfileFilename,
+            ProfileFilename.value(),
             createStringError(errc::no_such_file_or_directory,
                               "Missing binary ID: " +
                                   llvm::toHex(BinaryID, /*LowerCase=*/true)));
diff --git a/external/llvm-project/llvm/lib/ProfileData/DataAccessProf.cpp b/external/llvm-project/llvm/lib/ProfileData/DataAccessProf.cpp
index 090dcb3dcc1b..a1e686ba0036 100644
--- a/external/llvm-project/llvm/lib/ProfileData/DataAccessProf.cpp
+++ b/external/llvm-project/llvm/lib/ProfileData/DataAccessProf.cpp
@@ -1,5 +1,4 @@
 #include "llvm/ProfileData/DataAccessProf.h"
-#include "llvm/ADT/DenseMapInfoVariant.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Compression.h"
@@ -8,7 +7,6 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/raw_ostream.h"
-#include <sys/types.h>
 
 namespace llvm {
 namespace memprof {
diff --git a/external/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp b/external/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp
index 4d92e9d0ab33..7ca26aa13801 100644
--- a/external/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/external/llvm-project/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -21,7 +21,6 @@
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/ProfileCommon.h"
 #include "llvm/Support/Compression.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
diff --git a/external/llvm-project/llvm/lib/ProfileData/MemProfCommon.cpp b/external/llvm-project/llvm/lib/ProfileData/MemProfCommon.cpp
index 8c0c93123c82..a13a291a4cd2 100644
--- a/external/llvm-project/llvm/lib/ProfileData/MemProfCommon.cpp
+++ b/external/llvm-project/llvm/lib/ProfileData/MemProfCommon.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/Support/BLAKE3.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/HashBuilder.h"
 
 using namespace llvm;
@@ -21,7 +22,7 @@ using namespace llvm::memprof;
 
 // Upper bound on lifetime access density (accesses per byte per lifetime sec)
 // for marking an allocation cold.
-cl::opt<float> MemProfLifetimeAccessDensityColdThreshold(
+LLVM_ABI cl::opt<float> MemProfLifetimeAccessDensityColdThreshold(
     "memprof-lifetime-access-density-cold-threshold", cl::init(0.05),
     cl::Hidden,
     cl::desc("The threshold the lifetime access density (accesses per byte per "
@@ -29,20 +30,20 @@ cl::opt<float> MemProfLifetimeAccessDensityColdThreshold(
 
 // Lower bound on lifetime to mark an allocation cold (in addition to accesses
 // per byte per sec above). This is to avoid pessimizing short lived objects.
-cl::opt<unsigned> MemProfAveLifetimeColdThreshold(
+LLVM_ABI cl::opt<unsigned> MemProfAveLifetimeColdThreshold(
     "memprof-ave-lifetime-cold-threshold", cl::init(200), cl::Hidden,
     cl::desc("The average lifetime (s) for an allocation to be considered "
              "cold"));
 
 // Lower bound on average lifetime accesses density (total life time access
 // density / alloc count) for marking an allocation hot.
-cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold(
+LLVM_ABI cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold(
     "memprof-min-ave-lifetime-access-density-hot-threshold", cl::init(1000),
     cl::Hidden,
     cl::desc("The minimum TotalLifetimeAccessDensity / AllocCount for an "
              "allocation to be considered hot"));
 
-cl::opt<bool>
+LLVM_ABI cl::opt<bool>
     MemProfUseHotHints("memprof-use-hot-hints", cl::init(false), cl::Hidden,
                        cl::desc("Enable use of hot hints (only supported for "
                                 "unambigously hot allocations)"));
diff --git a/external/llvm-project/llvm/lib/ProfileData/MemProfRadixTree.cpp b/external/llvm-project/llvm/lib/ProfileData/MemProfRadixTree.cpp
index ec3ff2e721e2..c0672eb6da28 100644
--- a/external/llvm-project/llvm/lib/ProfileData/MemProfRadixTree.cpp
+++ b/external/llvm-project/llvm/lib/ProfileData/MemProfRadixTree.cpp
@@ -220,8 +220,8 @@ void CallStackRadixTreeBuilder<FrameIdTy>::build(
 }
 
 // Explicitly instantiate class with the utilized FrameIdTy.
-template class CallStackRadixTreeBuilder<FrameId>;
-template class CallStackRadixTreeBuilder<LinearFrameId>;
+template class LLVM_EXPORT_TEMPLATE CallStackRadixTreeBuilder<FrameId>;
+template class LLVM_EXPORT_TEMPLATE CallStackRadixTreeBuilder<LinearFrameId>;
 
 template <typename FrameIdTy>
 llvm::DenseMap<FrameIdTy, FrameStat>
@@ -241,10 +241,11 @@ computeFrameHistogram(llvm::MapVector<CallStackId, llvm::SmallVector<FrameIdTy>>
 }
 
 // Explicitly instantiate function with the utilized FrameIdTy.
-template llvm::DenseMap<FrameId, FrameStat> computeFrameHistogram<FrameId>(
+template LLVM_ABI llvm::DenseMap<FrameId, FrameStat>
+computeFrameHistogram<FrameId>(
     llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
         &MemProfCallStackData);
-template llvm::DenseMap<LinearFrameId, FrameStat>
+template LLVM_ABI llvm::DenseMap<LinearFrameId, FrameStat>
 computeFrameHistogram<LinearFrameId>(
     llvm::MapVector<CallStackId, llvm::SmallVector<LinearFrameId>>
         &MemProfCallStackData);
diff --git a/external/llvm-project/llvm/lib/SandboxIR/Constant.cpp b/external/llvm-project/llvm/lib/SandboxIR/Constant.cpp
index a8ed437a8bae..9de88ef2cf0a 100644
--- a/external/llvm-project/llvm/lib/SandboxIR/Constant.cpp
+++ b/external/llvm-project/llvm/lib/SandboxIR/Constant.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SandboxIR/Constant.h"
-#include "llvm/SandboxIR/Argument.h"
 #include "llvm/SandboxIR/BasicBlock.h"
 #include "llvm/SandboxIR/Context.h"
 #include "llvm/SandboxIR/Function.h"
@@ -283,20 +282,11 @@ PoisonValue *PoisonValue::getElementValue(unsigned Idx) const {
       cast<llvm::PoisonValue>(Val)->getElementValue(Idx)));
 }
 
-void GlobalObject::setAlignment(MaybeAlign Align) {
+void GlobalVariable::setAlignment(MaybeAlign Align) {
   Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalObject::getAlign, &GlobalObject::setAlignment>>(
-          this);
-  cast<llvm::GlobalObject>(Val)->setAlignment(Align);
-}
-
-void GlobalObject::setGlobalObjectSubClassData(unsigned V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalObject::getGlobalObjectSubClassData,
-                        &GlobalObject::setGlobalObjectSubClassData>>(this);
-  cast<llvm::GlobalObject>(Val)->setGlobalObjectSubClassData(V);
+      .emplaceIfTracking<GenericSetter<&GlobalVariable::getAlign,
+                                       &GlobalVariable::setAlignment>>(this);
+  cast<llvm::GlobalVariable>(Val)->setAlignment(Align);
 }
 
 void GlobalObject::setSection(StringRef S) {
@@ -315,35 +305,14 @@ GlobalT &GlobalWithNodeAPI<GlobalT, LLVMGlobalT, ParentT, LLVMParentT>::
 }
 
 // Explicit instantiations.
-template class GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
-                                 llvm::GlobalObject>;
-template class GlobalWithNodeAPI<Function, llvm::Function, GlobalObject,
-                                 llvm::GlobalObject>;
-template class GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
-                                 GlobalObject, llvm::GlobalObject>;
-template class GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
-                                 llvm::GlobalValue>;
-
-#if defined(_MSC_VER) && !defined(__clang__)
-// These are needed for SandboxIRTest when building with LLVM_BUILD_LLVM_DYLIB
-template LLVM_EXPORT_TEMPLATE GlobalIFunc &
-GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
-                  llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalIFunc
-                                                                  &LLVMGV)
-    const;
-template LLVM_EXPORT_TEMPLATE Function &
-GlobalWithNodeAPI<Function, llvm::Function, GlobalObject, llvm::GlobalObject>::
-    LLVMGVToGV::operator()(llvm::Function &LLVMGV) const;
-
-template LLVM_EXPORT_TEMPLATE GlobalVariable &GlobalWithNodeAPI<
-    GlobalVariable, llvm::GlobalVariable, GlobalObject,
-    llvm::GlobalObject>::LLVMGVToGV::operator()(llvm::GlobalVariable &LLVMGV)
-    const;
-template LLVM_EXPORT_TEMPLATE GlobalAlias &
-GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
-                  llvm::GlobalValue>::LLVMGVToGV::operator()(llvm::GlobalAlias
-                                                                 &LLVMGV) const;
-#endif
+template class LLVM_EXPORT_TEMPLATE GlobalWithNodeAPI<
+    GlobalIFunc, llvm::GlobalIFunc, GlobalObject, llvm::GlobalObject>;
+template class LLVM_EXPORT_TEMPLATE GlobalWithNodeAPI<
+    Function, llvm::Function, GlobalObject, llvm::GlobalObject>;
+template class LLVM_EXPORT_TEMPLATE GlobalWithNodeAPI<
+    GlobalVariable, llvm::GlobalVariable, GlobalObject, llvm::GlobalObject>;
+template class LLVM_EXPORT_TEMPLATE GlobalWithNodeAPI<
+    GlobalAlias, llvm::GlobalAlias, GlobalValue, llvm::GlobalValue>;
 
 void GlobalIFunc::setResolver(Constant *Resolver) {
   Ctx.getTracker()
diff --git a/external/llvm-project/llvm/lib/SandboxIR/Function.cpp b/external/llvm-project/llvm/lib/SandboxIR/Function.cpp
index f7a1d35b0046..f400389ed760 100644
--- a/external/llvm-project/llvm/lib/SandboxIR/Function.cpp
+++ b/external/llvm-project/llvm/lib/SandboxIR/Function.cpp
@@ -17,6 +17,13 @@ FunctionType *Function::getFunctionType() const {
       Ctx.getType(cast<llvm::Function>(Val)->getFunctionType()));
 }
 
+void Function::setAlignment(MaybeAlign Align) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&Function::getAlign, &Function::setAlignment>>(this);
+  cast<llvm::Function>(Val)->setAlignment(Align);
+}
+
 #ifndef NDEBUG
 void Function::dumpNameAndArgs(raw_ostream &OS) const {
   auto *F = cast<llvm::Function>(Val);
diff --git a/external/llvm-project/llvm/lib/SandboxIR/Pass.cpp b/external/llvm-project/llvm/lib/SandboxIR/Pass.cpp
index c6ec1aec48b1..64e1b609a9f4 100644
--- a/external/llvm-project/llvm/lib/SandboxIR/Pass.cpp
+++ b/external/llvm-project/llvm/lib/SandboxIR/Pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SandboxIR/Pass.h"
-#include "llvm/SandboxIR/PassManager.h"
 #include "llvm/Support/Debug.h"
 
 using namespace llvm::sandboxir;
diff --git a/external/llvm-project/llvm/lib/SandboxIR/Tracker.cpp b/external/llvm-project/llvm/lib/SandboxIR/Tracker.cpp
index 4fa9e11ae0d4..08dbfaf8398b 100644
--- a/external/llvm-project/llvm/lib/SandboxIR/Tracker.cpp
+++ b/external/llvm-project/llvm/lib/SandboxIR/Tracker.cpp
@@ -10,10 +10,8 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Instruction.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/SandboxIR/Instruction.h"
-#include <sstream>
 
 using namespace llvm::sandboxir;
 
diff --git a/external/llvm-project/llvm/lib/Support/AutoConvert.cpp b/external/llvm-project/llvm/lib/Support/AutoConvert.cpp
index f7918548df1d..0b6928e10ef5 100644
--- a/external/llvm-project/llvm/lib/Support/AutoConvert.cpp
+++ b/external/llvm-project/llvm/lib/Support/AutoConvert.cpp
@@ -14,7 +14,6 @@
 #ifdef __MVS__
 
 #include "llvm/Support/AutoConvert.h"
-#include "llvm/Support/Error.h"
 #include <cassert>
 #include <fcntl.h>
 #include <sys/stat.h>
@@ -83,27 +82,6 @@ int enablezOSAutoConversion(int FD) {
   return fcntl(FD, F_CONTROL_CVT, &Query);
 }
 
-std::error_code llvm::disablezOSAutoConversion(int FD) {
-  if (::disablezOSAutoConversion(FD) == -1)
-    return errnoAsErrorCode();
-
-  return std::error_code();
-}
-
-std::error_code llvm::enablezOSAutoConversion(int FD) {
-  if (::enablezOSAutoConversion(FD) == -1)
-    return errnoAsErrorCode();
-
-  return std::error_code();
-}
-
-std::error_code llvm::restorezOSStdHandleAutoConversion(int FD) {
-  if (::restorezOSStdHandleAutoConversion(FD) == -1)
-    return errnoAsErrorCode();
-
-  return std::error_code();
-}
-
 std::error_code llvm::setzOSFileTag(int FD, int CCSID, bool Text) {
   assert((!Text || (CCSID != FT_UNTAGGED && CCSID != FT_BINARY)) &&
          "FT_UNTAGGED and FT_BINARY are not allowed for text files");
diff --git a/external/llvm-project/llvm/lib/Support/CommandLine.cpp b/external/llvm-project/llvm/lib/Support/CommandLine.cpp
index f56ba28d4a59..1a16fc15620e 100644
--- a/external/llvm-project/llvm/lib/Support/CommandLine.cpp
+++ b/external/llvm-project/llvm/lib/Support/CommandLine.cpp
@@ -2529,7 +2529,7 @@ class VersionPrinter {
 #ifdef PACKAGE_VENDOR
     OS << PACKAGE_VENDOR << " ";
 #else
-    OS << "AOMP-18.0-12 (http://github.com/ROCm-Developer-Tools/aomp):\n Source ID:18.0-12-ce1873ac686bb90ddec72bb99889a4e80e2de382\n  ";
+    OS << "AOMP-18.0-12 (http://github.com/ROCm/aomp):\n Source ID:18.0-12-ce1873ac686bb90ddec72bb99889a4e80e2de382\n  ";
 #endif
     OS << PACKAGE_NAME << " version " << PACKAGE_VERSION << "\n  ";
 #if LLVM_IS_DEBUG_BUILD
diff --git a/external/llvm-project/llvm/lib/Support/CrashRecoveryContext.cpp b/external/llvm-project/llvm/lib/Support/CrashRecoveryContext.cpp
index 88c38d7526e7..1ba0c2383dae 100644
--- a/external/llvm-project/llvm/lib/Support/CrashRecoveryContext.cpp
+++ b/external/llvm-project/llvm/lib/Support/CrashRecoveryContext.cpp
@@ -10,7 +10,6 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ExitCodes.h"
-#include "llvm/Support/ProgramStack.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/thread.h"
 #include <cassert>
diff --git a/external/llvm-project/llvm/lib/Support/DebugCounter.cpp b/external/llvm-project/llvm/lib/Support/DebugCounter.cpp
index 9c4a4429ca0e..6b65720440f3 100644
--- a/external/llvm-project/llvm/lib/Support/DebugCounter.cpp
+++ b/external/llvm-project/llvm/lib/Support/DebugCounter.cpp
@@ -172,6 +172,11 @@ DebugCounter &DebugCounter::instance() {
 void DebugCounter::push_back(const std::string &Val) {
   if (Val.empty())
     return;
+#ifdef NDEBUG
+  // isCountingEnabled is hardcoded to false in NDEBUG.
+  errs() << "Requested --debug-counter in LLVM build without assertions. This "
+            "is a no-op.\n";
+#endif
 
   // The strings should come in as counter=chunk_list
   auto CounterPair = StringRef(Val).split('=');
diff --git a/external/llvm-project/llvm/lib/Support/DynamicLibrary.cpp b/external/llvm-project/llvm/lib/Support/DynamicLibrary.cpp
index e5c61abf5996..3dbde46895c3 100644
--- a/external/llvm-project/llvm/lib/Support/DynamicLibrary.cpp
+++ b/external/llvm-project/llvm/lib/Support/DynamicLibrary.cpp
@@ -26,7 +26,7 @@ using namespace llvm::sys;
 class DynamicLibrary::HandleSet {
   typedef std::vector<void *> HandleList;
   HandleList Handles;
-  void *Process = nullptr;
+  void *Process = &Invalid;
 
 public:
   static void *DLOpen(const char *Filename, std::string *Err);
@@ -59,7 +59,7 @@ class DynamicLibrary::HandleSet {
       Handles.push_back(Handle);
     } else {
 #ifndef _WIN32
-      if (Process) {
+      if (Process != &Invalid) {
         if (CanClose)
           DLClose(Process);
         if (Process == Handle)
@@ -98,11 +98,11 @@ class DynamicLibrary::HandleSet {
     assert(!((Order & SO_LoadedFirst) && (Order & SO_LoadedLast)) &&
            "Invalid Ordering");
 
-    if (!Process || (Order & SO_LoadedFirst)) {
+    if (Process == &Invalid || (Order & SO_LoadedFirst)) {
       if (void *Ptr = LibLookup(Symbol, Order))
         return Ptr;
     }
-    if (Process) {
+    if (Process != &Invalid) {
       // Use OS facilities to search the current binary and all loaded libs.
       if (void *Ptr = DLSym(Process, Symbol))
         return Ptr;
diff --git a/external/llvm-project/llvm/lib/Support/ELFAttrParserExtended.cpp b/external/llvm-project/llvm/lib/Support/ELFAttrParserExtended.cpp
index f98905b6171c..3d9158a8eac6 100644
--- a/external/llvm-project/llvm/lib/Support/ELFAttrParserExtended.cpp
+++ b/external/llvm-project/llvm/lib/Support/ELFAttrParserExtended.cpp
@@ -10,7 +10,6 @@
 #include "llvm/Support/ELFAttrParserExtended.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/AArch64BuildAttributes.h"
 #include "llvm/Support/ELFAttributes.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
diff --git a/external/llvm-project/llvm/lib/Support/InitLLVM.cpp b/external/llvm-project/llvm/lib/Support/InitLLVM.cpp
index 50f7a43cc34a..b8fbfd21c4f2 100644
--- a/external/llvm-project/llvm/lib/Support/InitLLVM.cpp
+++ b/external/llvm-project/llvm/lib/Support/InitLLVM.cpp
@@ -18,18 +18,28 @@
 #include "llvm/Support/Windows/WindowsSupport.h"
 #endif
 
-#ifdef __MVS__
+#if defined(HAVE_UNISTD_H)
 #include <unistd.h>
+#else
+#ifndef STDIN_FILENO
+#define STDIN_FILENO 0
+#endif
+#ifndef STDOUT_FILENO
+#define STDOUT_FILENO 1
+#endif
+#ifndef STDERR_FILENO
+#define STDERR_FILENO 2
+#endif
+#endif
 
 void CleanupStdHandles(void *Cookie) {
   llvm::raw_ostream *Outs = &llvm::outs(), *Errs = &llvm::errs();
   Outs->flush();
   Errs->flush();
-  llvm::restorezOSStdHandleAutoConversion(STDIN_FILENO);
-  llvm::restorezOSStdHandleAutoConversion(STDOUT_FILENO);
-  llvm::restorezOSStdHandleAutoConversion(STDERR_FILENO);
+  llvm::restoreStdHandleAutoConversion(STDIN_FILENO);
+  llvm::restoreStdHandleAutoConversion(STDOUT_FILENO);
+  llvm::restoreStdHandleAutoConversion(STDERR_FILENO);
 }
-#endif
 
 using namespace llvm;
 using namespace llvm::sys;
@@ -41,10 +51,10 @@ InitLLVM::InitLLVM(int &Argc, const char **&Argv,
   assert(!Initialized && "InitLLVM was already initialized!");
   Initialized = true;
 #endif
-#ifdef __MVS__
+
   // Bring stdin/stdout/stderr into a known state.
   sys::AddSignalHandler(CleanupStdHandles, nullptr);
-#endif
+
   if (InstallPipeSignalExitHandler)
     // The pipe signal handler must be installed before any other handlers are
     // registered. This is because the Unix \ref RegisterHandlers function does
@@ -68,8 +78,8 @@ InitLLVM::InitLLVM(int &Argc, const char **&Argv,
 
   // If turning on conversion for stderr fails then the error message
   // may be garbled. There is no solution to this problem.
-  ExitOnErr(errorCodeToError(llvm::enablezOSAutoConversion(STDERR_FILENO)));
-  ExitOnErr(errorCodeToError(llvm::enablezOSAutoConversion(STDOUT_FILENO)));
+  ExitOnErr(errorCodeToError(llvm::enableAutoConversion(STDERR_FILENO)));
+  ExitOnErr(errorCodeToError(llvm::enableAutoConversion(STDOUT_FILENO)));
 #endif
 
 #ifdef _WIN32
@@ -97,8 +107,6 @@ InitLLVM::InitLLVM(int &Argc, const char **&Argv,
 }
 
 InitLLVM::~InitLLVM() {
-#ifdef __MVS__
   CleanupStdHandles(nullptr);
-#endif
   llvm_shutdown();
 }
diff --git a/external/llvm-project/llvm/lib/Support/LockFileManager.cpp b/external/llvm-project/llvm/lib/Support/LockFileManager.cpp
index 633b948e9bdf..cdded51108b5 100644
--- a/external/llvm-project/llvm/lib/Support/LockFileManager.cpp
+++ b/external/llvm-project/llvm/lib/Support/LockFileManager.cpp
@@ -22,8 +22,6 @@
 #include <chrono>
 #include <ctime>
 #include <memory>
-#include <sys/stat.h>
-#include <sys/types.h>
 #include <system_error>
 #include <tuple>
 
diff --git a/external/llvm-project/llvm/lib/Support/MemoryBuffer.cpp b/external/llvm-project/llvm/lib/Support/MemoryBuffer.cpp
index e2044bcc4e4f..601f11f6d23c 100644
--- a/external/llvm-project/llvm/lib/Support/MemoryBuffer.cpp
+++ b/external/llvm-project/llvm/lib/Support/MemoryBuffer.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/AutoConvert.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -34,9 +35,6 @@
 #include <io.h>
 #endif
 
-#ifdef __MVS__
-#include "llvm/Support/AutoConvert.h"
-#endif
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
@@ -508,15 +506,15 @@ getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,
   }
 
 #ifdef __MVS__
-  ErrorOr<bool> NeedConversion = needzOSConversion(Filename.str().c_str(), FD);
-  if (std::error_code EC = NeedConversion.getError())
+  ErrorOr<bool> NeedsConversion = needConversion(Filename.str().c_str(), FD);
+  if (std::error_code EC = NeedsConversion.getError())
     return EC;
   // File size may increase due to EBCDIC -> UTF-8 conversion, therefore we
   // cannot trust the file size and we create the memory buffer by copying
   // off the stream.
   // Note: This only works with the assumption of reading a full file (i.e,
   // Offset == 0 and MapSize == FileSize). Reading a file slice does not work.
-  if (Offset == 0 && MapSize == FileSize && *NeedConversion)
+  if (*NeedsConversion && Offset == 0 && MapSize == FileSize)
     return getMemoryBufferForStream(FD, Filename);
 #endif
 
diff --git a/external/llvm-project/llvm/lib/Support/Mustache.cpp b/external/llvm-project/llvm/lib/Support/Mustache.cpp
index 89900ed4d03a..6c2ed6c84c6c 100644
--- a/external/llvm-project/llvm/lib/Support/Mustache.cpp
+++ b/external/llvm-project/llvm/lib/Support/Mustache.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 #include "llvm/Support/Mustache.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
 #include <sstream>
 
@@ -23,6 +22,13 @@ static bool isFalsey(const json::Value &V) {
          (V.getAsArray() && V.getAsArray()->empty());
 }
 
+static bool isContextFalsey(const json::Value *V) {
+  // A missing context (represented by a nullptr) is defined as falsey.
+  if (!V)
+    return true;
+  return isFalsey(*V);
+}
+
 static Accessor splitMustacheString(StringRef Str) {
   // We split the mustache string into an accessor.
   // For example:
@@ -560,14 +566,15 @@ void toMustacheString(const json::Value &Data, raw_ostream &OS) {
   }
 }
 
-void ASTNode::render(const json::Value &Data, raw_ostream &OS) {
-  ParentContext = &Data;
+void ASTNode::render(const json::Value &CurrentCtx, raw_ostream &OS) {
+  // Set the parent context to the incoming context so that we
+  // can walk up the context tree correctly in findContext().
+  ParentContext = &CurrentCtx;
   const json::Value *ContextPtr = Ty == Root ? ParentContext : findContext();
-  const json::Value &Context = ContextPtr ? *ContextPtr : nullptr;
 
   switch (Ty) {
   case Root:
-    renderChild(Data, OS);
+    renderChild(CurrentCtx, OS);
     return;
   case Text:
     OS << Body;
@@ -575,53 +582,55 @@ void ASTNode::render(const json::Value &Data, raw_ostream &OS) {
   case Partial: {
     auto Partial = Partials.find(AccessorValue[0]);
     if (Partial != Partials.end())
-      renderPartial(Data, OS, Partial->getValue().get());
+      renderPartial(CurrentCtx, OS, Partial->getValue().get());
     return;
   }
   case Variable: {
     auto Lambda = Lambdas.find(AccessorValue[0]);
-    if (Lambda != Lambdas.end())
-      renderLambdas(Data, OS, Lambda->getValue());
-    else {
+    if (Lambda != Lambdas.end()) {
+      renderLambdas(CurrentCtx, OS, Lambda->getValue());
+    } else if (ContextPtr) {
       EscapeStringStream ES(OS, Escapes);
-      toMustacheString(Context, ES);
+      toMustacheString(*ContextPtr, ES);
     }
     return;
   }
   case UnescapeVariable: {
     auto Lambda = Lambdas.find(AccessorValue[0]);
-    if (Lambda != Lambdas.end())
-      renderLambdas(Data, OS, Lambda->getValue());
-    else
-      toMustacheString(Context, OS);
+    if (Lambda != Lambdas.end()) {
+      renderLambdas(CurrentCtx, OS, Lambda->getValue());
+    } else if (ContextPtr) {
+      toMustacheString(*ContextPtr, OS);
+    }
     return;
   }
   case Section: {
-    // Sections are not rendered if the context is falsey.
     auto SectionLambda = SectionLambdas.find(AccessorValue[0]);
     bool IsLambda = SectionLambda != SectionLambdas.end();
-    if (isFalsey(Context) && !IsLambda)
-      return;
 
     if (IsLambda) {
-      renderSectionLambdas(Data, OS, SectionLambda->getValue());
+      renderSectionLambdas(CurrentCtx, OS, SectionLambda->getValue());
       return;
     }
 
-    if (Context.getAsArray()) {
-      const json::Array *Arr = Context.getAsArray();
+    if (isContextFalsey(ContextPtr))
+      return;
+
+    if (const json::Array *Arr = ContextPtr->getAsArray()) {
       for (const json::Value &V : *Arr)
         renderChild(V, OS);
       return;
     }
-    renderChild(Context, OS);
+    renderChild(*ContextPtr, OS);
     return;
   }
   case InvertSection: {
     bool IsLambda = SectionLambdas.contains(AccessorValue[0]);
-    if (!isFalsey(Context) || IsLambda)
-      return;
-    renderChild(Context, OS);
+    if (isContextFalsey(ContextPtr) && !IsLambda) {
+      // The context for the children remains unchanged from the parent's, so
+      // we pass this node's original incoming context.
+      renderChild(CurrentCtx, OS);
+    }
     return;
   }
   }
diff --git a/external/llvm-project/llvm/lib/Support/TextEncoding.cpp b/external/llvm-project/llvm/lib/Support/TextEncoding.cpp
index 969dd419ede7..b675966c8d82 100644
--- a/external/llvm-project/llvm/lib/Support/TextEncoding.cpp
+++ b/external/llvm-project/llvm/lib/Support/TextEncoding.cpp
@@ -17,9 +17,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ConvertEBCDIC.h"
-#include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <limits>
 #include <system_error>
 
 #if HAVE_ICU
diff --git a/external/llvm-project/llvm/lib/Support/Timer.cpp b/external/llvm-project/llvm/lib/Support/Timer.cpp
index 22811d7b4af0..75ec299a9837 100644
--- a/external/llvm-project/llvm/lib/Support/Timer.cpp
+++ b/external/llvm-project/llvm/lib/Support/Timer.cpp
@@ -24,7 +24,6 @@
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Signposts.h"
-#include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 #include <limits>
 #include <optional>
diff --git a/external/llvm-project/llvm/lib/Support/Unix/DynamicLibrary.inc b/external/llvm-project/llvm/lib/Support/Unix/DynamicLibrary.inc
index 7452913049eb..b6f8e38a6612 100644
--- a/external/llvm-project/llvm/lib/Support/Unix/DynamicLibrary.inc
+++ b/external/llvm-project/llvm/lib/Support/Unix/DynamicLibrary.inc
@@ -17,7 +17,7 @@ DynamicLibrary::HandleSet::~HandleSet() {
   // Close the libraries in reverse order.
   for (void *Handle : llvm::reverse(Handles))
     ::dlclose(Handle);
-  if (Process)
+  if (Process != &Invalid)
     ::dlclose(Process);
 
   // llvm_shutdown called, Return to default
diff --git a/external/llvm-project/llvm/lib/Support/Unix/Path.inc b/external/llvm-project/llvm/lib/Support/Unix/Path.inc
index 0728413f4db6..277247e3cc23 100644
--- a/external/llvm-project/llvm/lib/Support/Unix/Path.inc
+++ b/external/llvm-project/llvm/lib/Support/Unix/Path.inc
@@ -1097,14 +1097,14 @@ std::error_code openFile(const Twine &Name, int &ResultFD,
                     !Stat.st_tag.ft_txtflag && !Stat.st_tag.ft_ccsid &&
                     Stat.st_size == 0;
     if (Flags & OF_Text) {
-      if (auto EC = llvm::enablezOSAutoConversion(ResultFD))
+      if (auto EC = llvm::enableAutoConversion(ResultFD))
         return EC;
       if (DoSetTag) {
         if (auto EC = llvm::setzOSFileTag(ResultFD, CCSID_IBM_1047, true))
           return EC;
       }
     } else {
-      if (auto EC = llvm::disablezOSAutoConversion(ResultFD))
+      if (auto EC = llvm::disableAutoConversion(ResultFD))
         return EC;
       if (DoSetTag) {
         if (auto EC = llvm::setzOSFileTag(ResultFD, FT_BINARY, false))
diff --git a/external/llvm-project/llvm/lib/Support/Unix/Program.inc b/external/llvm-project/llvm/lib/Support/Unix/Program.inc
index 6d68369ad191..4f17b2257a75 100644
--- a/external/llvm-project/llvm/lib/Support/Unix/Program.inc
+++ b/external/llvm-project/llvm/lib/Support/Unix/Program.inc
@@ -516,7 +516,7 @@ std::error_code llvm::sys::ChangeStdoutMode(fs::OpenFlags Flags) {
 
 std::error_code llvm::sys::ChangeStdinToBinary() {
 #ifdef __MVS__
-  return disablezOSAutoConversion(STDIN_FILENO);
+  return disableAutoConversion(STDIN_FILENO);
 #else
   // Do nothing, as Unix doesn't differentiate between text and binary.
   return std::error_code();
diff --git a/external/llvm-project/llvm/lib/Support/Unix/Threading.inc b/external/llvm-project/llvm/lib/Support/Unix/Threading.inc
index 15a5b008604c..562971aae32f 100644
--- a/external/llvm-project/llvm/lib/Support/Unix/Threading.inc
+++ b/external/llvm-project/llvm/lib/Support/Unix/Threading.inc
@@ -142,6 +142,8 @@ uint64_t llvm::get_threadid() {
   return uint64_t(gettid());
 #elif defined(__linux__)
   return uint64_t(syscall(__NR_gettid));
+#elif defined(_AIX)
+  return uint64_t(thread_self());
 #else
   return uint64_t(pthread_self());
 #endif
@@ -316,6 +318,7 @@ static int computeHostNumHardwareThreads() {
     return CPU_COUNT(&mask);
 #elif defined(__linux__)
   cpu_set_t Set;
+  CPU_ZERO(&Set);
   if (sched_getaffinity(0, sizeof(Set), &Set) == 0)
     return CPU_COUNT(&Set);
 #endif
diff --git a/external/llvm-project/llvm/lib/Support/Windows/DynamicLibrary.inc b/external/llvm-project/llvm/lib/Support/Windows/DynamicLibrary.inc
index 6bb00c069f81..be3050abd589 100644
--- a/external/llvm-project/llvm/lib/Support/Windows/DynamicLibrary.inc
+++ b/external/llvm-project/llvm/lib/Support/Windows/DynamicLibrary.inc
@@ -26,7 +26,7 @@ DynamicLibrary::HandleSet::~HandleSet() {
     FreeLibrary(HMODULE(Handle));
 
   // 'Process' should not be released on Windows.
-  assert((!Process || Process == this) && "Bad Handle");
+  assert((Process == &Invalid || Process == this) && "Bad Handle");
   // llvm_shutdown called, Return to default
   DynamicLibrary::SearchOrder = DynamicLibrary::SO_Linker;
 }
@@ -62,7 +62,7 @@ static DynamicLibrary::HandleSet *IsOpenedHandlesInstance(void *Handle) {
 
 void DynamicLibrary::HandleSet::DLClose(void *Handle) {
   if (HandleSet *HS = IsOpenedHandlesInstance(Handle))
-    HS->Process = nullptr; // Just drop the *Process* handle.
+    HS->Process = &Invalid; // Just drop the *Process* handle.
   else
     FreeLibrary((HMODULE)Handle);
 }
@@ -91,7 +91,7 @@ void *DynamicLibrary::HandleSet::DLSym(void *Handle, const char *Symbol) {
     return (void *)uintptr_t(GetProcAddress((HMODULE)Handle, Symbol));
 
   // Could have done a dlclose on the *Process* handle
-  if (!HS->Process)
+  if (HS->Process == &Invalid)
     return nullptr;
 
   // Trials indicate EnumProcessModulesEx is consistantly faster than using
diff --git a/external/llvm-project/llvm/lib/Support/Windows/Path.inc b/external/llvm-project/llvm/lib/Support/Windows/Path.inc
index 2ee8c1c6ebe9..821f2d80613b 100644
--- a/external/llvm-project/llvm/lib/Support/Windows/Path.inc
+++ b/external/llvm-project/llvm/lib/Support/Windows/Path.inc
@@ -747,7 +747,7 @@ static std::error_code getStatus(HANDLE FileHandle, file_status &Result) {
     PathHash = (static_cast<uint64_t>(Info.nFileIndexHigh) << 32ULL) |
                static_cast<uint64_t>(Info.nFileIndexLow);
   } else {
-    PathHash = hash_combine_range(ntPath.begin(), ntPath.end());
+    PathHash = hash_combine_range(ntPath);
   }
 
   Result = file_status(
diff --git a/external/llvm-project/llvm/lib/Support/raw_ostream.cpp b/external/llvm-project/llvm/lib/Support/raw_ostream.cpp
index 16631a63d192..07b99896543b 100644
--- a/external/llvm-project/llvm/lib/Support/raw_ostream.cpp
+++ b/external/llvm-project/llvm/lib/Support/raw_ostream.cpp
@@ -894,21 +894,24 @@ void raw_fd_ostream::anchor() {}
 raw_fd_ostream &llvm::outs() {
   // Set buffer settings to model stdout behavior.
   std::error_code EC;
-#ifdef __MVS__
-  EC = enablezOSAutoConversion(STDOUT_FILENO);
-  assert(!EC);
-#endif
+
+  // On z/OS we need to enable auto conversion
+  static std::error_code EC1 = enableAutoConversion(STDOUT_FILENO);
+  assert(!EC1);
+  (void)EC1;
+
   static raw_fd_ostream S("-", EC, sys::fs::OF_None);
   assert(!EC);
   return S;
 }
 
 raw_fd_ostream &llvm::errs() {
-  // Set standard error to be unbuffered.
-#ifdef __MVS__
-  std::error_code EC = enablezOSAutoConversion(STDERR_FILENO);
+  // On z/OS we need to enable auto conversion
+  static std::error_code EC = enableAutoConversion(STDERR_FILENO);
   assert(!EC);
-#endif
+  (void)EC;
+
+  // Set standard error to be unbuffered.
   static raw_fd_ostream S(STDERR_FILENO, false, true);
   return S;
 }
diff --git a/external/llvm-project/llvm/lib/TableGen/JSONBackend.cpp b/external/llvm-project/llvm/lib/TableGen/JSONBackend.cpp
index d7ab043cb8c9..80c1d7fd1dc4 100644
--- a/external/llvm-project/llvm/lib/TableGen/JSONBackend.cpp
+++ b/external/llvm-project/llvm/lib/TableGen/JSONBackend.cpp
@@ -13,7 +13,6 @@
 
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
diff --git a/external/llvm-project/llvm/lib/TableGen/Record.cpp b/external/llvm-project/llvm/lib/TableGen/Record.cpp
index 4c8b41237c60..7f2ed77a7409 100644
--- a/external/llvm-project/llvm/lib/TableGen/Record.cpp
+++ b/external/llvm-project/llvm/lib/TableGen/Record.cpp
@@ -1557,8 +1557,7 @@ const Init *BinOpInit::Fold(const Record *CurRec) const {
 }
 
 const Init *BinOpInit::resolveReferences(Resolver &R) const {
-  const Init *lhs = LHS->resolveReferences(R);
-  const Init *rhs = RHS->resolveReferences(R);
+  const Init *NewLHS = LHS->resolveReferences(R);
 
   unsigned Opc = getOpcode();
   if (Opc == AND || Opc == OR) {
@@ -1570,15 +1569,17 @@ const Init *BinOpInit::resolveReferences(Resolver &R) const {
     // limited version of short-circuit against all ones (`true` is casted
     // to 1 rather than all ones before we evaluate `!or`).
     if (const auto *LHSi = dyn_cast_or_null<IntInit>(
-            lhs->convertInitializerTo(IntRecTy::get(getRecordKeeper())))) {
+            NewLHS->convertInitializerTo(IntRecTy::get(getRecordKeeper())))) {
       if ((Opc == AND && !LHSi->getValue()) ||
           (Opc == OR && LHSi->getValue() == -1))
         return LHSi;
     }
   }
 
-  if (LHS != lhs || RHS != rhs)
-    return (BinOpInit::get(getOpcode(), lhs, rhs, getType()))
+  const Init *NewRHS = RHS->resolveReferences(R);
+
+  if (LHS != NewLHS || RHS != NewRHS)
+    return (BinOpInit::get(getOpcode(), NewLHS, NewRHS, getType()))
         ->Fold(R.getCurrentRecord());
   return this;
 }
diff --git a/external/llvm-project/llvm/lib/TableGen/TGLexer.cpp b/external/llvm-project/llvm/lib/TableGen/TGLexer.cpp
index 4da2505cc065..46487cba9453 100644
--- a/external/llvm-project/llvm/lib/TableGen/TGLexer.cpp
+++ b/external/llvm-project/llvm/lib/TableGen/TGLexer.cpp
@@ -20,9 +20,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/TableGen/Error.h"
-#include <algorithm>
 #include <cerrno>
-#include <cstdint>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64.td
index 29dd6227ba02..f303819f411d 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64.td
@@ -78,7 +78,7 @@ let F = [HasSME2p2, HasSVE2p2_or_SME2p2, HasNonStreamingSVE_or_SME2p2,
 def SME2p2Unsupported : AArch64Unsupported;
 
 def SME2p1Unsupported : AArch64Unsupported {
-  let F = !listconcat([HasSME2p1, HasSVE2p1_or_SME2p1, HasNonStreamingSVE2p1_or_SSVE_AES, 
+  let F = !listconcat([HasSME2p1, HasSVE2p1_or_SME2p1,
                        HasSME_MOP4, HasSME_TMOP, HasNonStreamingSVE_or_SSVE_FEXPA, 
                        HasNonStreamingSVE2_or_SSVE_BitPerm],
                        SME2p2Unsupported.F);
@@ -86,7 +86,8 @@ def SME2p1Unsupported : AArch64Unsupported {
 
 def SME2Unsupported : AArch64Unsupported {
   let F = !listconcat([HasSME2, HasSVE2_or_SME2, HasSVE2p1_or_SME2, HasSSVE_FP8FMA,
-                      HasSMEF8F16, HasSMEF8F32, HasSMEF16F16_or_SMEF8F16, HasSMEB16B16],
+                      HasSMEF8F16, HasSMEF8F32, HasSMEF16F16_or_SMEF8F16, HasSMEB16B16,
+                      HasNonStreamingSVE2_or_SSVE_AES],
                       SME2p1Unsupported.F);
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
index 08a6fa2ea8db..c85adcf85f8d 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
@@ -73,11 +73,11 @@ class AArch64AdvSIMDScalar : public MachineFunctionPass {
   bool isProfitableToTransform(const MachineInstr &MI) const;
 
   // transformInstruction - Perform the transformation of an instruction
-  // to its equivalant AdvSIMD scalar instruction. Update inputs and outputs
+  // to its equivalent AdvSIMD scalar instruction. Update inputs and outputs
   // to be the correct register class, minimizing cross-class copies.
   void transformInstruction(MachineInstr &MI);
 
-  // processMachineBasicBlock - Main optimzation loop.
+  // processMachineBasicBlock - Main optimization loop.
   bool processMachineBasicBlock(MachineBasicBlock *MBB);
 
 public:
@@ -231,7 +231,7 @@ bool AArch64AdvSIMDScalar::isProfitableToTransform(
 
   // If any of the uses of the original instructions is a cross class copy,
   // that's a copy that will be removable if we transform. Likewise, if
-  // any of the uses is a transformable instruction, it's likely the tranforms
+  // any of the uses is a transformable instruction, it's likely the transforms
   // will chain, enabling us to save a copy there, too. This is an aggressive
   // heuristic that approximates the graph based cost analysis described above.
   Register Dst = MI.getOperand(0).getReg();
@@ -280,7 +280,7 @@ static MachineInstr *insertCopy(const TargetInstrInfo *TII, MachineInstr &MI,
 }
 
 // transformInstruction - Perform the transformation of an instruction
-// to its equivalant AdvSIMD scalar instruction. Update inputs and outputs
+// to its equivalent AdvSIMD scalar instruction. Update inputs and outputs
 // to be the correct register class, minimizing cross-class copies.
 void AArch64AdvSIMDScalar::transformInstruction(MachineInstr &MI) {
   LLVM_DEBUG(dbgs() << "Scalar transform: " << MI);
@@ -372,7 +372,7 @@ void AArch64AdvSIMDScalar::transformInstruction(MachineInstr &MI) {
   ++NumScalarInsnsUsed;
 }
 
-// processMachineBasicBlock - Main optimzation loop.
+// processMachineBasicBlock - Main optimization loop.
 bool AArch64AdvSIMDScalar::processMachineBasicBlock(MachineBasicBlock *MBB) {
   bool Changed = false;
   for (MachineInstr &MI : llvm::make_early_inc_range(*MBB)) {
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index e9ccc35f3461..3f92c1dbfbf4 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -467,7 +467,7 @@ void AArch64AsmPrinter::emitAttributes(unsigned Flags,
   PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
 
   if (PAuthABIPlatform || PAuthABIVersion) {
-    TS->emitAtributesSubsection(
+    TS->emitAttributesSubsection(
         AArch64BuildAttributes::getVendorName(
             AArch64BuildAttributes::AEABI_PAUTHABI),
         AArch64BuildAttributes::SubsectionOptional::REQUIRED,
@@ -490,7 +490,7 @@ void AArch64AsmPrinter::emitAttributes(unsigned Flags,
       (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
 
   if (BTIValue || PACValue || GCSValue) {
-    TS->emitAtributesSubsection(
+    TS->emitAttributesSubsection(
         AArch64BuildAttributes::getVendorName(
             AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
         AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
@@ -3273,6 +3273,32 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
                                      -MI->getOperand(2).getImm());
     return;
 
+  case AArch64::SEH_AllocZ:
+    assert(MI->getOperand(0).getImm() >= 0 &&
+           "AllocZ SEH opcode offset must be non-negative");
+    assert(MI->getOperand(0).getImm() <= 255 &&
+           "AllocZ SEH opcode offset must fit into 8 bits");
+    TS->emitARM64WinCFIAllocZ(MI->getOperand(0).getImm());
+    return;
+
+  case AArch64::SEH_SaveZReg:
+    assert(MI->getOperand(1).getImm() >= 0 &&
+           "SaveZReg SEH opcode offset must be non-negative");
+    assert(MI->getOperand(1).getImm() <= 255 &&
+           "SaveZReg SEH opcode offset must fit into 8 bits");
+    TS->emitARM64WinCFISaveZReg(MI->getOperand(0).getImm(),
+                                MI->getOperand(1).getImm());
+    return;
+
+  case AArch64::SEH_SavePReg:
+    assert(MI->getOperand(1).getImm() >= 0 &&
+           "SavePReg SEH opcode offset must be non-negative");
+    assert(MI->getOperand(1).getImm() <= 255 &&
+           "SavePReg SEH opcode offset must fit into 8 bits");
+    TS->emitARM64WinCFISavePReg(MI->getOperand(0).getImm(),
+                                MI->getOperand(1).getImm());
+    return;
+
   case AArch64::BLR:
   case AArch64::BR: {
     recordIfImportCall(MI);
@@ -3505,7 +3531,7 @@ const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
 char AArch64AsmPrinter::ID = 0;
 
 INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
-                "AArch64 Assmebly Printer", false, false)
+                "AArch64 Assembly Printer", false, false)
 
 // Force static initialization.
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp
index 2a96f15c20f6..3436dc9ef452 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp
@@ -99,7 +99,8 @@ bool AArch64BranchTargets::runOnMachineFunction(MachineFunction &MF) {
 
     // If the block itself is address-taken, it could be indirectly branched
     // to, but not called.
-    if (MBB.hasAddressTaken() || JumpTableTargets.count(&MBB))
+    if (MBB.isMachineBlockAddressTaken() || MBB.isIRBlockAddressTaken() ||
+        JumpTableTargets.count(&MBB))
       CouldJump = true;
 
     if (CouldCall || CouldJump) {
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64CallingConvention.td
index 7cca6d9bc6b9..1b5a713bffdc 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@@ -28,6 +28,12 @@ class CCIfSubtarget<string F, CCAction A>
 //===----------------------------------------------------------------------===//
 
 defvar AArch64_Common = [
+  // The 'nest' parameter, if any, is passed in X15.
+  // The previous register used here (X18) is also defined to be unavailable
+  // for this purpose, while all of X9-X15 were defined to be free for LLVM to
+  // use for this, so use X15 (which LLVM often already clobbers anyways).
+  CCIfNest<CCAssignToReg<[X15]>>,
+
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
   CCIfType<[v2f64, v4f32], CCBitConvertToType<v2i64>>,
@@ -117,13 +123,7 @@ defvar AArch64_Common = [
 ];
 
 let Entry = 1 in
-def CC_AArch64_AAPCS : CallingConv<!listconcat(
-  // The 'nest' parameter, if any, is passed in X18.
-  // Darwin and Windows use X18 as the platform register and hence 'nest' isn't
-  // currently supported there.
-  [CCIfNest<CCAssignToReg<[X18]>>],
-  AArch64_Common
-)>;
+def CC_AArch64_AAPCS : CallingConv<AArch64_Common>;
 
 let Entry = 1 in
 def RetCC_AArch64_AAPCS : CallingConv<[
@@ -177,6 +177,8 @@ def CC_AArch64_Win64_VarArg : CallingConv<[
 // a stack layout compatible with the x64 calling convention.
 let Entry = 1 in
 def CC_AArch64_Arm64EC_VarArg : CallingConv<[
+  CCIfNest<CCAssignToReg<[X15]>>,
+
   // Convert small floating-point values to integer.
   CCIfType<[f16, bf16], CCBitConvertToType<i16>>,
   CCIfType<[f32], CCBitConvertToType<i32>>,
@@ -353,6 +355,8 @@ def RetCC_AArch64_Arm64EC_CFGuard_Check : CallingConv<[
 //     + Stack slots are sized as needed rather than being at least 64-bit.
 let Entry = 1 in
 def CC_AArch64_DarwinPCS : CallingConv<[
+  CCIfNest<CCAssignToReg<[X15]>>,
+
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
   CCIfType<[v2f64, v4f32, f128], CCBitConvertToType<v2i64>>,
@@ -427,6 +431,8 @@ def CC_AArch64_DarwinPCS : CallingConv<[
 
 let Entry = 1 in
 def CC_AArch64_DarwinPCS_VarArg : CallingConv<[
+  CCIfNest<CCAssignToReg<[X15]>>,
+
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
   CCIfType<[v2f64, v4f32, f128], CCBitConvertToType<v2i64>>,
@@ -450,6 +456,8 @@ def CC_AArch64_DarwinPCS_VarArg : CallingConv<[
 // same as the normal Darwin VarArgs handling.
 let Entry = 1 in
 def CC_AArch64_DarwinPCS_ILP32_VarArg : CallingConv<[
+  CCIfNest<CCAssignToReg<[X15]>>,
+
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
   CCIfType<[v2f64, v4f32, f128], CCBitConvertToType<v2i64>>,
 
@@ -494,6 +502,8 @@ def CC_AArch64_DarwinPCS_ILP32_VarArg : CallingConv<[
 
 let Entry = 1 in
 def CC_AArch64_GHC : CallingConv<[
+  CCIfNest<CCAssignToReg<[X15]>>,
+
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
 
   // Handle all vector types as either f64 or v2f64.
@@ -522,6 +532,7 @@ def CC_AArch64_Preserve_None : CallingConv<[
 
   // We can pass arguments in all general registers, except:
   // - X8, used for sret
+  // - X15 (on Windows), used as a temporary register in the prologue when allocating call frames
   // - X16/X17, used by the linker as IP0/IP1
   // - X18, the platform register
   // - X19, the base pointer
@@ -606,6 +617,9 @@ def CSR_Win_AArch64_Arm64EC_Thunk : CalleeSavedRegs<(add (sequence "Q%u", 6, 15)
 def CSR_AArch64_AAVPCS : CalleeSavedRegs<(add X19, X20, X21, X22, X23, X24,
                                           X25, X26, X27, X28, LR, FP,
                                           (sequence "Q%u", 8, 23))>;
+def CSR_Win_AArch64_AAVPCS : CalleeSavedRegs<(add X19, X20, X21, X22, X23, X24,
+                                              X25, X26, X27, X28, FP, LR,
+                                              (sequence "Q%u", 8, 23))>;
 
 // Functions taking SVE arguments or returning an SVE type
 // must (additionally) preserve full Z8-Z23 and predicate registers P4-P15
@@ -619,6 +633,11 @@ def CSR_Darwin_AArch64_SVE_AAPCS : CalleeSavedRegs<(add (sequence "Z%u", 8, 23),
                                                         LR, FP, X19, X20, X21, X22,
                                                         X23, X24, X25, X26, X27, X28)>;
 
+def CSR_Win_AArch64_SVE_AAPCS : CalleeSavedRegs<(add (sequence "P%u", 4, 15),
+                                                     (sequence "Z%u", 8, 23),
+                                                     X19, X20, X21, X22, X23, X24,
+                                                     X25, X26, X27, X28, FP, LR)>;
+
 // SME ABI support routines such as __arm_tpidr2_save/restore preserve most registers.
 def CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0
                           : CalleeSavedRegs<(add (sequence "Z%u", 0, 31),
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp
index 64f21c4cb229..064716216d1c 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp
@@ -232,7 +232,7 @@ static bool isCandidateLoad(const MachineInstr &MI) {
   }
 }
 
-/// Check whether the given instruction can load a litteral.
+/// Check whether the given instruction can load a literal.
 static bool supportLoadFromLiteral(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
@@ -247,7 +247,18 @@ static bool supportLoadFromLiteral(const MachineInstr &MI) {
   }
 }
 
-/// Number of GPR registers traked by mapRegToGPRIndex()
+/// Returns \p true if there are no non-debug instructions between \p First and
+/// \p Second
+static bool areInstructionsConsecutive(const MachineInstr *First,
+                                       const MachineInstr *Second) {
+  auto It = First->getIterator();
+  auto EndIt = First->getParent()->instr_end();
+  if (It == EndIt)
+    return false;
+  return next_nodbg(It, EndIt) == Second->getIterator();
+}
+
+/// Number of GPR registers tracked by mapRegToGPRIndex()
 static const unsigned N_GPR_REGS = 31;
 /// Map register number to index from 0-30.
 static int mapRegToGPRIndex(MCRegister Reg) {
@@ -415,7 +426,7 @@ static void handleADRP(const MachineInstr &MI, AArch64FunctionInfo &AFI,
         ++NumADRPToLDR;
       }
       break;
-    case MCLOH_AdrpAddLdr: {
+    case MCLOH_AdrpAddLdr:
       // There is a possibility that the linker may try to rewrite:
       // adrp x0, @sym@PAGE
       // add x1, x0, @sym@PAGEOFF
@@ -432,28 +443,24 @@ static void handleADRP(const MachineInstr &MI, AArch64FunctionInfo &AFI,
       // FIXME: Implement proper liveness tracking for all registers. For now,
       // don't emit the LOH if there are any instructions between the add and
       // the ldr.
-      MachineInstr *AddMI = const_cast<MachineInstr *>(Info.MI1);
-      const MachineInstr *LdrMI = Info.MI0;
-      auto AddIt = MachineBasicBlock::iterator(AddMI);
-      auto EndIt = AddMI->getParent()->end();
-      if (AddMI->getIterator() == EndIt || LdrMI != &*next_nodbg(AddIt, EndIt))
+      if (!areInstructionsConsecutive(Info.MI1, Info.MI0))
         break;
-
       LLVM_DEBUG(dbgs() << "Adding MCLOH_AdrpAddLdr:\n"
                         << '\t' << MI << '\t' << *Info.MI1 << '\t'
                         << *Info.MI0);
       AFI.addLOHDirective(MCLOH_AdrpAddLdr, {&MI, Info.MI1, Info.MI0});
       ++NumADDToLDR;
       break;
-    }
     case MCLOH_AdrpAddStr:
-      if (Info.MI1 != nullptr) {
-        LLVM_DEBUG(dbgs() << "Adding MCLOH_AdrpAddStr:\n"
-                          << '\t' << MI << '\t' << *Info.MI1 << '\t'
-                          << *Info.MI0);
-        AFI.addLOHDirective(MCLOH_AdrpAddStr, {&MI, Info.MI1, Info.MI0});
-        ++NumADDToSTR;
-      }
+      if (!Info.MI1)
+        break;
+      if (!areInstructionsConsecutive(Info.MI1, Info.MI0))
+        break;
+      LLVM_DEBUG(dbgs() << "Adding MCLOH_AdrpAddStr:\n"
+                        << '\t' << MI << '\t' << *Info.MI1 << '\t'
+                        << *Info.MI0);
+      AFI.addLOHDirective(MCLOH_AdrpAddStr, {&MI, Info.MI1, Info.MI0});
+      ++NumADDToSTR;
       break;
     case MCLOH_AdrpLdrGotLdr:
       LLVM_DEBUG(dbgs() << "Adding MCLOH_AdrpLdrGotLdr:\n"
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64Combine.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64Combine.td
index f84e83816bf3..ca09598464d1 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -172,8 +172,23 @@ def form_duplane : GICombineRule <
   (apply [{ applyDupLane(*${root}, MRI, B, ${matchinfo}); }])
 >;
 
-def shuffle_vector_lowering : GICombineGroup<[dup, rev, ext, zip, uzp, trn, fullrev,
-                                              form_duplane, shuf_to_ins]>;
+// Clean up G_UNMERGE(G_DUPLANE16) -> G_DUPLANE16
+class unmerge_duplane<Instruction Op> : GICombineRule <
+  (defs root:$root),
+  (match (Op $a, $src, $c),
+         (G_UNMERGE_VALUES $d1, $d2, $a):$root,
+         [{ return MRI.getType(${d1}.getReg()).getSizeInBits() == 64; }]),
+  (apply (GIReplaceReg $d2, $d1), (Op $d1, $src, $c))
+>;
+def unmerge_duplane8 : unmerge_duplane<G_DUPLANE8>;
+def unmerge_duplane16 : unmerge_duplane<G_DUPLANE16>;
+def unmerge_duplane32 : unmerge_duplane<G_DUPLANE32>;
+// G_DUPLANE64 is not included as the result in scalar.
+def unmerge_duplanes : GICombineGroup<[unmerge_duplane8, unmerge_duplane16,
+                                       unmerge_duplane32]>;
+
+def shuffle_vector_lowering : GICombineGroup<[dup, form_duplane, rev, ext, zip,
+                                              uzp, trn, fullrev, shuf_to_ins]>;
 
 // Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's
 def vector_unmerge_lowering : GICombineRule <
@@ -325,7 +340,8 @@ def AArch64PostLegalizerLowering
                         lower_vector_fcmp, form_truncstore,
                         vector_sext_inreg_to_shift,
                         unmerge_ext_to_unmerge, lower_mulv2s64,
-                        vector_unmerge_lowering, insertelt_nonconst]> {
+                        vector_unmerge_lowering, insertelt_nonconst,
+                        unmerge_duplanes]> {
 }
 
 // Post-legalization combines which are primarily optimizations.
@@ -345,7 +361,7 @@ def AArch64PostLegalizerCombiner
                         ptr_add_immed_chain, overlapping_and,
                         split_store_zero_128, undef_combines,
                         select_to_minmax, or_to_bsp, combine_concat_vector,
-                        commute_constant_to_rhs,
+                        commute_constant_to_rhs, extract_vec_elt_combines,
                         push_freeze_to_prevent_poison_from_propagating,
                         combine_mul_cmlt, combine_use_vector_truncate, extmultomull]> {
 }
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
index 9b59ee6483cd..484bc2a4be8f 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
@@ -573,7 +573,7 @@ void SSACCmpConv::convert(SmallVectorImpl<MachineBasicBlock *> &RemovedBlocks) {
   // Update the CFG first.
   updateTailPHIs();
 
-  // Save successor probabilties before removing CmpBB and Tail from their
+  // Save successor probabilities before removing CmpBB and Tail from their
   // parents.
   BranchProbability Head2CmpBB = MBPI->getEdgeProbability(Head, CmpBB);
   BranchProbability CmpBB2Tail = MBPI->getEdgeProbability(CmpBB, Tail);
@@ -581,7 +581,7 @@ void SSACCmpConv::convert(SmallVectorImpl<MachineBasicBlock *> &RemovedBlocks) {
   Head->removeSuccessor(CmpBB);
   CmpBB->removeSuccessor(Tail);
 
-  // If Head and CmpBB had successor probabilties, udpate the probabilities to
+  // If Head and CmpBB had successor probabilities, update the probabilities to
   // reflect the ccmp-conversion.
   if (Head->hasSuccessorProbabilities() && CmpBB->hasSuccessorProbabilities()) {
 
@@ -596,7 +596,7 @@ void SSACCmpConv::convert(SmallVectorImpl<MachineBasicBlock *> &RemovedBlocks) {
                              Head2Tail + Head2CmpBB * CmpBB2Tail);
 
     // We will transfer successors of CmpBB to Head in a moment without
-    // normalizing the successor probabilities. Set the successor probabilites
+    // normalizing the successor probabilities. Set the successor probabilities
     // before doing so.
     //
     // Pr(I|Head) = Pr(CmpBB|Head) * Pr(I|CmpBB).
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp
index 71284b0574e5..987dfbcdd53e 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp
@@ -64,10 +64,10 @@ static bool usesFrameIndex(const MachineInstr &MI) {
   return false;
 }
 
-// Instructions that lose their 'read' operation for a subesquent fence acquire
+// Instructions that lose their 'read' operation for a subsequent fence acquire
 // (DMB LD) once the zero register is used.
 //
-// WARNING: The aquire variants of the instructions are also affected, but they
+// WARNING: The acquire variants of the instructions are also affected, but they
 // are split out into `atomicBarrierDroppedOnZero()` to support annotations on
 // assembly.
 static bool atomicReadDroppedOnZero(unsigned Opcode) {
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 5ddf83f45ac6..9d74bb5a8661 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -508,7 +508,7 @@ Register AArch64FastISel::materializeGV(const GlobalValue *GV) {
       // also uses BuildMI for making an ADRP (+ MOVK) + ADD, but the operands
       // are not exactly 1:1 with FastISel so we cannot easily abstract this
       // out. At some point, it would be nice to find a way to not have this
-      // duplciate code.
+      // duplicate code.
       Register DstReg = createResultReg(&AArch64::GPR64commonRegClass);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AArch64::MOVKXi),
               DstReg)
@@ -5198,7 +5198,8 @@ bool AArch64FastISel::fastSelectInstruction(const Instruction *I) {
 FastISel *AArch64::createFastISel(FunctionLoweringInfo &FuncInfo,
                                         const TargetLibraryInfo *LibInfo) {
 
-  SMEAttrs CallerAttrs(*FuncInfo.Fn);
+  SMEAttrs CallerAttrs =
+      FuncInfo.MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
   if (CallerAttrs.hasZAState() || CallerAttrs.hasZT0State() ||
       CallerAttrs.hasStreamingInterfaceOrBody() ||
       CallerAttrs.hasStreamingCompatibleInterface() ||
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 0f33e77d4eec..2650c621e19f 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -44,6 +44,10 @@
 // |                                   |
 // |-----------------------------------|
 // |                                   |
+// | (Win64 only) callee-saved SVE reg |
+// |                                   |
+// |-----------------------------------|
+// |                                   |
 // | callee-saved gpr registers        | <--.
 // |                                   |    | On Darwin platforms these
 // |- - - - - - - - - - - - - - - - - -|    | callee saves are swapped,
@@ -327,7 +331,9 @@ static int64_t getArgumentStackToRestore(MachineFunction &MF,
 static bool produceCompactUnwindFrame(MachineFunction &MF);
 static bool needsWinCFI(const MachineFunction &MF);
 static StackOffset getSVEStackSize(const MachineFunction &MF);
-static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB);
+static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB,
+                                                 bool HasCall = false);
+static bool requiresSaveVG(const MachineFunction &MF);
 
 /// Returns true if a homogeneous prolog or epilog code can be emitted
 /// for the size optimization. If possible, a frame helper call is injected.
@@ -395,7 +401,7 @@ static const unsigned DefaultSafeSPDisplacement = 255;
 /// size limit beyond which some of these instructions will require a scratch
 /// register during their expansion later.
 static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
-  // FIXME: For now, just conservatively guestimate based on unscaled indexing
+  // FIXME: For now, just conservatively guesstimate based on unscaled indexing
   // range. We'll end up allocating an unnecessary spill slot a lot, but
   // realistically that's not a big deal at this stage of the game.
   for (MachineBasicBlock &MBB : MF) {
@@ -595,7 +601,7 @@ void AArch64FrameLowering::emitCalleeSavedGPRLocations(
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
-  SMEAttrs Attrs(MF.getFunction());
+  SMEAttrs Attrs = AFI->getSMEFnAttrs();
   bool LocallyStreaming =
       Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
 
@@ -643,7 +649,7 @@ void AArch64FrameLowering::emitCalleeSavedSVELocations(
       continue;
 
     // Not all unwinders may know about SVE registers, so assume the lowest
-    // common demoninator.
+    // common denominator.
     assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
     MCRegister Reg = Info.getReg();
     if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
@@ -797,7 +803,7 @@ void AArch64FrameLowering::allocateStackSpace(
         .addImm(InitialOffset.getFixed())
         .addImm(InitialOffset.getScalable());
     // The fixed allocation may leave unprobed bytes at the top of the
-    // stack. If we have subsequent alocation (e.g. if we have variable-sized
+    // stack. If we have subsequent allocation (e.g. if we have variable-sized
     // objects), we need to issue an extra probe, so these allocations start in
     // a known state.
     if (FollowupAllocs) {
@@ -1002,6 +1008,16 @@ void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
   }
 }
 
+static bool windowsRequiresStackProbe(const MachineFunction &MF,
+                                      uint64_t StackSizeInBytes) {
+  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+  const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>();
+  // TODO: When implementing stack protectors, take that into account
+  // for the probe threshold.
+  return Subtarget.isTargetWindows() && MFI.hasStackProbing() &&
+         StackSizeInBytes >= uint64_t(MFI.getStackProbeSize());
+}
+
 static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
                                    const MachineBasicBlock &MBB) {
   const MachineFunction *MF = MBB.getParent();
@@ -1023,7 +1039,8 @@ static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
 // but we would then have to make sure that we were in fact saving at least one
 // callee-save register in the prologue, which is additional complexity that
 // doesn't seem worth the benefit.
-static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
+static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB,
+                                                 bool HasCall) {
   MachineFunction *MF = MBB->getParent();
 
   // If MBB is an entry block, use X9 as the scratch register
@@ -1037,6 +1054,11 @@ static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
   const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
   LivePhysRegs LiveRegs(TRI);
   getLiveRegsForEntryMBB(LiveRegs, *MBB);
+  if (HasCall) {
+    LiveRegs.addReg(AArch64::X16);
+    LiveRegs.addReg(AArch64::X17);
+    LiveRegs.addReg(AArch64::X18);
+  }
 
   // Prefer X9 since it was historically used for the prologue scratch reg.
   const MachineRegisterInfo &MRI = MF->getRegInfo();
@@ -1077,23 +1099,18 @@ bool AArch64FrameLowering::canUseAsPrologue(
       MBB.isLiveIn(AArch64::NZCV))
     return false;
 
-  // Don't need a scratch register if we're not going to re-align the stack or
-  // emit stack probes.
-  if (!RegInfo->hasStackRealignment(*MF) && !TLI->hasInlineStackProbe(*MF))
-    return true;
-  // Otherwise, we can use any block as long as it has a scratch register
-  // available.
-  return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
-}
+  if (RegInfo->hasStackRealignment(*MF) || TLI->hasInlineStackProbe(*MF))
+    if (findScratchNonCalleeSaveRegister(TmpMBB) == AArch64::NoRegister)
+      return false;
 
-static bool windowsRequiresStackProbe(MachineFunction &MF,
-                                      uint64_t StackSizeInBytes) {
-  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
-  const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>();
-  // TODO: When implementing stack protectors, take that into account
-  // for the probe threshold.
-  return Subtarget.isTargetWindows() && MFI.hasStackProbing() &&
-         StackSizeInBytes >= uint64_t(MFI.getStackProbeSize());
+  // May need a scratch register (for return value) if require making a special
+  // call
+  if (requiresSaveVG(*MF) ||
+      windowsRequiresStackProbe(*MF, std::numeric_limits<uint64_t>::max()))
+    if (findScratchNonCalleeSaveRegister(TmpMBB, true) == AArch64::NoRegister)
+      return false;
+
+  return true;
 }
 
 static bool needsWinCFI(const MachineFunction &MF) {
@@ -1200,7 +1217,25 @@ static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
 
   switch (Opc) {
   default:
-    llvm_unreachable("No SEH Opcode for this instruction");
+    report_fatal_error("No SEH Opcode for this instruction");
+  case AArch64::STR_ZXI:
+  case AArch64::LDR_ZXI: {
+    unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
+    MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveZReg))
+              .addImm(Reg0)
+              .addImm(Imm)
+              .setMIFlag(Flag);
+    break;
+  }
+  case AArch64::STR_PXI:
+  case AArch64::LDR_PXI: {
+    unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
+    MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SavePReg))
+              .addImm(Reg0)
+              .addImm(Imm)
+              .setMIFlag(Flag);
+    break;
+  }
   case AArch64::LDPDpost:
     Imm = -Imm;
     [[fallthrough]];
@@ -1356,8 +1391,8 @@ bool requiresGetVGCall(MachineFunction &MF) {
          !MF.getSubtarget<AArch64Subtarget>().hasSVE();
 }
 
-static bool requiresSaveVG(MachineFunction &MF) {
-  AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+static bool requiresSaveVG(const MachineFunction &MF) {
+  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
   // For Darwin platforms we don't save VG for non-SVE functions, even if SME
   // is enabled with streaming mode changes.
   if (!AFI->hasStreamingModeChanges())
@@ -1592,6 +1627,9 @@ static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
   case AArch64::CMPNE_PPzZI_B:
     return I->getFlag(MachineInstr::FrameSetup) ||
            I->getFlag(MachineInstr::FrameDestroy);
+  case AArch64::SEH_SavePReg:
+  case AArch64::SEH_SaveZReg:
+    return true;
   }
 }
 
@@ -1874,12 +1912,51 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg());
   unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
 
+  // Windows unwind can't represent the required stack adjustments if we have
+  // both SVE callee-saves and dynamic stack allocations, and the frame
+  // pointer is before the SVE spills.  The allocation of the frame pointer
+  // must be the last instruction in the prologue so the unwinder can restore
+  // the stack pointer correctly. (And there isn't any unwind opcode for
+  // `addvl sp, x29, -17`.)
+  //
+  // Because of this, we do spills in the opposite order on Windows: first SVE,
+  // then GPRs. The main side-effect of this is that it makes accessing
+  // parameters passed on the stack more expensive.
+  //
+  // We could consider rearranging the spills for simpler cases.
+  bool FPAfterSVECalleeSaves =
+      Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize();
+
+  if (FPAfterSVECalleeSaves && AFI->hasStackHazardSlotIndex())
+    reportFatalUsageError("SME hazard padding is not supported on Windows");
+
   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
   // All of the remaining stack allocations are for locals.
   AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
   bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
   bool HomPrologEpilog = homogeneousPrologEpilog(MF);
-  if (CombineSPBump) {
+  if (FPAfterSVECalleeSaves) {
+    // If we're doing SVE saves first, we need to immediately allocate space
+    // for fixed objects, then space for the SVE callee saves.
+    //
+    // Windows unwind requires that the scalable size is a multiple of 16;
+    // that's handled when the callee-saved size is computed.
+    auto SaveSize =
+        StackOffset::getScalable(AFI->getSVECalleeSavedStackSize()) +
+        StackOffset::getFixed(FixedObject);
+    allocateStackSpace(MBB, MBBI, 0, SaveSize, NeedsWinCFI, &HasWinCFI,
+                       /*EmitCFI=*/false, StackOffset{},
+                       /*FollowupAllocs=*/true);
+    NumBytes -= FixedObject;
+
+    // Now allocate space for the GPR callee saves.
+    while (MBBI != End && IsSVECalleeSave(MBBI))
+      ++MBBI;
+    MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
+        MBB, MBBI, DL, TII, -AFI->getCalleeSavedStackSize(), NeedsWinCFI,
+        &HasWinCFI, EmitAsyncCFI);
+    NumBytes -= AFI->getCalleeSavedStackSize();
+  } else if (CombineSPBump) {
     assert(!SVEStackSize && "Cannot combine SP bump with SVE");
     emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(-NumBytes), TII,
@@ -1982,12 +2059,38 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
           : 0;
 
   if (windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) {
+    if (AFI->getSVECalleeSavedStackSize())
+      report_fatal_error(
+          "SVE callee saves not yet supported with stack probing");
+
+    // Find an available register to spill the value of X15 to, if X15 is being
+    // used already for nest.
+    unsigned X15Scratch = AArch64::NoRegister;
+    const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
+    if (llvm::any_of(MBB.liveins(),
+                     [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
+                       return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
+                           AArch64::X15, LiveIn.PhysReg);
+                     })) {
+      X15Scratch = findScratchNonCalleeSaveRegister(&MBB, true);
+      assert(X15Scratch != AArch64::NoRegister &&
+             (X15Scratch < AArch64::X15 || X15Scratch > AArch64::X17));
+#ifndef NDEBUG
+      LiveRegs.removeReg(AArch64::X15); // ignore X15 since we restore it
+#endif
+      BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), X15Scratch)
+          .addReg(AArch64::XZR)
+          .addReg(AArch64::X15, RegState::Undef)
+          .addReg(AArch64::X15, RegState::Implicit)
+          .setMIFlag(MachineInstr::FrameSetup);
+    }
+
     uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
     if (NeedsWinCFI) {
       HasWinCFI = true;
       // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
       // exceed this amount.  We need to move at most 2^24 - 1 into x15.
-      // This is at most two instructions, MOVZ follwed by MOVK.
+      // This is at most two instructions, MOVZ followed by MOVK.
       // TODO: Fix to use multiple stack alloc unwind codes for stacks
       // exceeding 256MB in size.
       if (NumBytes >= (1 << 28))
@@ -2104,34 +2207,45 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
       // we've set a frame pointer and already finished the SEH prologue.
       assert(!NeedsWinCFI);
     }
+    if (X15Scratch != AArch64::NoRegister) {
+      BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), AArch64::X15)
+          .addReg(AArch64::XZR)
+          .addReg(X15Scratch, RegState::Undef)
+          .addReg(X15Scratch, RegState::Implicit)
+          .setMIFlag(MachineInstr::FrameSetup);
+    }
   }
 
   StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
-  MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
+  MachineBasicBlock::iterator CalleeSavesEnd = MBBI;
+
+  StackOffset CFAOffset =
+      StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
 
   // Process the SVE callee-saves to determine what space needs to be
   // allocated.
   if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
     LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize
                       << "\n");
-    // Find callee save instructions in frame.
-    CalleeSavesBegin = MBBI;
-    assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
-    while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator())
-      ++MBBI;
-    CalleeSavesEnd = MBBI;
-
     SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize);
     SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
-  }
+    // Find callee save instructions in frame.
+    // Note: With FPAfterSVECalleeSaves the callee saves have already been
+    // allocated.
+    if (!FPAfterSVECalleeSaves) {
+      MachineBasicBlock::iterator CalleeSavesBegin = MBBI;
+      assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
+      while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator())
+        ++MBBI;
+      CalleeSavesEnd = MBBI;
 
-  // Allocate space for the callee saves (if any).
-  StackOffset CFAOffset =
-      StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
-  StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes);
-  allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, false,
-                     nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
-                     MFI.hasVarSizedObjects() || LocalsSize);
+      StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes);
+      // Allocate space for the callee saves (if any).
+      allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, false,
+                         nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
+                         MFI.hasVarSizedObjects() || LocalsSize);
+    }
+  }
   CFAOffset += SVECalleeSavesSize;
 
   if (EmitAsyncCFI)
@@ -2303,10 +2417,16 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
     assert(AfterCSRPopSize == 0);
     return;
   }
+
+  bool FPAfterSVECalleeSaves =
+      Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize();
+
   bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
   // Assume we can't combine the last pop with the sp restore.
   bool CombineAfterCSRBump = false;
-  if (!CombineSPBump && PrologueSaveSize != 0) {
+  if (FPAfterSVECalleeSaves) {
+    AfterCSRPopSize += FixedObject;
+  } else if (!CombineSPBump && PrologueSaveSize != 0) {
     MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
     while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
            AArch64InstrInfo::isSEHInstruction(*Pop))
@@ -2323,7 +2443,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
           MachineInstr::FrameDestroy, PrologueSaveSize);
     } else {
       // If not, make sure to emit an add after the last ldp.
-      // We're doing this by transfering the size to be restored from the
+      // We're doing this by transferring the size to be restored from the
       // adjustment *before* the CSR pops to the adjustment *after* the CSR
       // pops.
       AfterCSRPopSize += PrologueSaveSize;
@@ -2339,7 +2459,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   while (LastPopI != Begin) {
     --LastPopI;
     if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
-        IsSVECalleeSave(LastPopI)) {
+        (!FPAfterSVECalleeSaves && IsSVECalleeSave(LastPopI))) {
       ++LastPopI;
       break;
     } else if (CombineSPBump)
@@ -2415,6 +2535,9 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
   MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
   if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
+    if (FPAfterSVECalleeSaves)
+      RestoreEnd = MBB.getFirstTerminator();
+
     RestoreBegin = std::prev(RestoreEnd);
     while (RestoreBegin != MBB.begin() &&
            IsSVECalleeSave(std::prev(RestoreBegin)))
@@ -2430,7 +2553,31 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   }
 
   // Deallocate the SVE area.
-  if (SVEStackSize) {
+  if (FPAfterSVECalleeSaves) {
+    // If the callee-save area is before FP, restoring the FP implicitly
+    // deallocates non-callee-save SVE allocations.  Otherwise, deallocate
+    // them explicitly.
+    if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) {
+      emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
+                      DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
+                      NeedsWinCFI, &HasWinCFI);
+    }
+
+    // Deallocate callee-save non-SVE registers.
+    emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
+                    StackOffset::getFixed(AFI->getCalleeSavedStackSize()), TII,
+                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
+
+    // Deallocate fixed objects.
+    emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
+                    StackOffset::getFixed(FixedObject), TII,
+                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
+
+    // Deallocate callee-save SVE registers.
+    emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
+                    DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
+                    NeedsWinCFI, &HasWinCFI);
+  } else if (SVEStackSize) {
     // If we have stack realignment or variable sized objects on the stack,
     // restore the stack pointer from the frame pointer prior to SVE CSR
     // restoration.
@@ -2450,20 +2597,20 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
         emitFrameOffset(
             MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
             StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy,
-            false, false, nullptr, EmitCFI && !hasFP(MF),
+            false, NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
             SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
         NumBytes = 0;
       }
 
       emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
                       DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
-                      false, nullptr, EmitCFI && !hasFP(MF),
+                      NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
                       SVEStackSize +
                           StackOffset::getFixed(NumBytes + PrologueSaveSize));
 
       emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
                       DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
-                      false, nullptr, EmitCFI && !hasFP(MF),
+                      NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
                       DeallocateAfter +
                           StackOffset::getFixed(NumBytes + PrologueSaveSize));
     }
@@ -2587,7 +2734,12 @@ AArch64FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
     return StackOffset::getFixed(ObjectOffset - getOffsetOfLocalArea());
 
   const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
+  bool FPAfterSVECalleeSaves =
+      isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
   if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
+    if (FPAfterSVECalleeSaves &&
+        -ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize())
+      return StackOffset::getScalable(ObjectOffset);
     return StackOffset::get(-((int64_t)AFI->getCalleeSavedStackSize()),
                             ObjectOffset);
   }
@@ -2597,8 +2749,12 @@ AArch64FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
       !IsFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
 
   StackOffset ScalableOffset = {};
-  if (!IsFixed && !IsCSR)
+  if (!IsFixed && !IsCSR) {
     ScalableOffset = -SVEStackSize;
+  } else if (FPAfterSVECalleeSaves && IsCSR) {
+    ScalableOffset =
+        -StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
+  }
 
   return StackOffset::getFixed(ObjectOffset) + ScalableOffset;
 }
@@ -2736,13 +2892,21 @@ StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
       "In the presence of dynamic stack pointer realignment, "
       "non-argument/CSR objects cannot be accessed through the frame pointer");
 
+  bool FPAfterSVECalleeSaves =
+      isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
+
   if (isSVE) {
+    assert(-ObjectOffset > (int64_t)AFI->getSVECalleeSavedStackSize() &&
+           "Math isn't correct for CSRs with FPAfterSVECalleeSaves");
     StackOffset FPOffset =
         StackOffset::get(-AFI->getCalleeSaveBaseToFrameRecordOffset(), ObjectOffset);
     StackOffset SPOffset =
         SVEStackSize +
         StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
                          ObjectOffset);
+    if (FPAfterSVECalleeSaves) {
+      FPOffset += StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
+    }
     // Always use the FP for SVE spills if available and beneficial.
     if (hasFP(MF) && (SPOffset.getFixed() ||
                       FPOffset.getScalable() < SPOffset.getScalable() ||
@@ -2757,10 +2921,28 @@ StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
   }
 
   StackOffset ScalableOffset = {};
-  if (UseFP && !(isFixed || isCSR))
-    ScalableOffset = -SVEStackSize;
-  if (!UseFP && (isFixed || isCSR))
-    ScalableOffset = SVEStackSize;
+  if (FPAfterSVECalleeSaves) {
+    // In this stack layout, the FP is in between the callee saves and other
+    // SVE allocations.
+    StackOffset SVECalleeSavedStack =
+        StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
+    if (UseFP) {
+      if (isFixed)
+        ScalableOffset = SVECalleeSavedStack;
+      else if (!isCSR)
+        ScalableOffset = SVECalleeSavedStack - SVEStackSize;
+    } else {
+      if (isFixed)
+        ScalableOffset = SVEStackSize;
+      else if (isCSR)
+        ScalableOffset = SVEStackSize - SVECalleeSavedStack;
+    }
+  } else {
+    if (UseFP && !(isFixed || isCSR))
+      ScalableOffset = -SVEStackSize;
+    if (!UseFP && (isFixed || isCSR))
+      ScalableOffset = SVEStackSize;
+  }
 
   if (UseFP) {
     FrameReg = RegInfo->getFrameRegister(MF);
@@ -2810,7 +2992,7 @@ static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
                                              const TargetRegisterInfo *TRI) {
   // If we are generating register pairs for a Windows function that requires
   // EH support, then pair consecutive registers only.  There are no unwind
-  // opcodes for saves/restores of non-consectuve register pairs.
+  // opcodes for saves/restores of non-consecutive register pairs.
   // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
   // save_lrpair.
   // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
@@ -2887,7 +3069,7 @@ bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget,
   if (DisableMultiVectorSpillFill)
     return false;
 
-  SMEAttrs FuncAttrs(MF.getFunction());
+  SMEAttrs FuncAttrs = MF.getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
   bool IsLocallyStreaming =
       FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
 
@@ -2934,7 +3116,9 @@ static void computeCalleeSaveRegisterPairs(
     RegInc = -1;
     FirstReg = Count - 1;
   }
-  int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
+  bool FPAfterSVECalleeSaves = IsWindows && AFI->getSVECalleeSavedStackSize();
+  int ScalableByteOffset =
+      FPAfterSVECalleeSaves ? 0 : AFI->getSVECalleeSavedStackSize();
   bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
   Register LastReg = 0;
 
@@ -3046,6 +3230,12 @@ static void computeCalleeSaveRegisterPairs(
         RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
       RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
 
+    // Realign the scalable offset if necessary.  This is relevant when
+    // spilling predicates on Windows.
+    if (RPI.isScalable() && ScalableByteOffset % Scale != 0) {
+      ScalableByteOffset = alignTo(ScalableByteOffset, Scale);
+    }
+
     int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
     assert(OffsetPre % Scale == 0);
 
@@ -3208,9 +3398,9 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
     unsigned X0Scratch = AArch64::NoRegister;
     if (Reg1 == AArch64::VG) {
       // Find an available register to store value of VG to.
-      Reg1 = findScratchNonCalleeSaveRegister(&MBB);
+      Reg1 = findScratchNonCalleeSaveRegister(&MBB, true);
       assert(Reg1 != AArch64::NoRegister);
-      SMEAttrs Attrs(MF.getFunction());
+      SMEAttrs Attrs = AFI->getSMEFnAttrs();
 
       if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface() &&
           AFI->getStreamingVGIdx() == std::numeric_limits<int>::max()) {
@@ -3539,12 +3729,13 @@ static std::optional<int> getLdStFrameID(const MachineInstr &MI,
 void AArch64FrameLowering::determineStackHazardSlot(
     MachineFunction &MF, BitVector &SavedRegs) const {
   unsigned StackHazardSize = getStackHazardSize(MF);
+  auto *AFI = MF.getInfo<AArch64FunctionInfo>();
   if (StackHazardSize == 0 || StackHazardSize % 16 != 0 ||
-      MF.getInfo<AArch64FunctionInfo>()->hasStackHazardSlotIndex())
+      AFI->hasStackHazardSlotIndex())
     return;
 
   // Stack hazards are only needed in streaming functions.
-  SMEAttrs Attrs(MF.getFunction());
+  SMEAttrs Attrs = AFI->getSMEFnAttrs();
   if (!StackHazardInNonStreaming && Attrs.hasNonStreamingInterfaceAndBody())
     return;
 
@@ -3581,7 +3772,7 @@ void AArch64FrameLowering::determineStackHazardSlot(
     int ID = MFI.CreateStackObject(StackHazardSize, Align(16), false);
     LLVM_DEBUG(dbgs() << "Created Hazard slot at " << ID << " size "
                       << StackHazardSize << "\n");
-    MF.getInfo<AArch64FunctionInfo>()->setStackHazardSlotIndex(ID);
+    AFI->setStackHazardSlotIndex(ID);
   }
 }
 
@@ -3734,8 +3925,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
   // changes, as we will need to spill the value of the VG register.
   // For locally streaming functions, we spill both the streaming and
   // non-streaming VG value.
-  const Function &F = MF.getFunction();
-  SMEAttrs Attrs(F);
+  SMEAttrs Attrs = AFI->getSMEFnAttrs();
   if (requiresSaveVG(MF)) {
     if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
       CSStackSize += 16;
@@ -3892,7 +4082,7 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
   // Insert VG into the list of CSRs, immediately before LR if saved.
   if (requiresSaveVG(MF)) {
     std::vector<CalleeSavedInfo> VGSaves;
-    SMEAttrs Attrs(MF.getFunction());
+    SMEAttrs Attrs = AFI->getSMEFnAttrs();
 
     auto VGInfo = CalleeSavedInfo(AArch64::VG);
     VGInfo.setRestored(false);
@@ -4875,7 +5065,7 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
   }
 
   // Find contiguous runs of tagged memory and emit shorter instruction
-  // sequencies for them when possible.
+  // sequences for them when possible.
   TagStoreEdit TSE(MBB, FirstZeroData);
   std::optional<int64_t> EndOffset;
   for (auto &Instr : Instrs) {
@@ -4909,10 +5099,10 @@ static void emitVGSaveRestore(MachineBasicBlock::iterator II,
       MI.getOpcode() != AArch64::VGRestorePseudo)
     return;
 
-  SMEAttrs FuncAttrs(MF->getFunction());
+  auto *AFI = MF->getInfo<AArch64FunctionInfo>();
+  SMEAttrs FuncAttrs = AFI->getSMEFnAttrs();
   bool LocallyStreaming =
       FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
-  const AArch64FunctionInfo *AFI = MF->getInfo<AArch64FunctionInfo>();
 
   int64_t VGFrameIdx =
       LocallyStreaming ? AFI->getStreamingVGIdx() : AFI->getVGIdx();
@@ -5402,8 +5592,8 @@ static inline raw_ostream &operator<<(raw_ostream &OS, const StackAccess &SA) {
 void AArch64FrameLowering::emitRemarks(
     const MachineFunction &MF, MachineOptimizationRemarkEmitter *ORE) const {
 
-  SMEAttrs Attrs(MF.getFunction());
-  if (Attrs.hasNonStreamingInterfaceAndBody())
+  auto *AFI = MF.getInfo<AArch64FunctionInfo>();
+  if (AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody())
     return;
 
   unsigned StackHazardSize = getStackHazardSize(MF);
@@ -5444,7 +5634,7 @@ void AArch64FrameLowering::emitRemarks(
           unsigned RegTy = StackAccess::AccessType::GPR;
           if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) {
             // SPILL_PPR_TO_ZPR_SLOT_PSEUDO and FILL_PPR_FROM_ZPR_SLOT_PSEUDO
-            // spill/fill the predicate as a data vector (so are an FPR acess).
+            // spill/fill the predicate as a data vector (so are an FPR access).
             if (MI.getOpcode() != AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO &&
                 MI.getOpcode() != AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO &&
                 AArch64::PPRRegClass.contains(MI.getOperand(0).getReg())) {
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 34f6db9374cb..009d69b2b943 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -991,7 +991,7 @@ bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
 }
 
 /// SelectArithUXTXRegister - Select a "UXTX register" operand. This
-/// operand is refered by the instructions have SP operand
+/// operand is referred by the instructions have SP operand
 bool AArch64DAGToDAGISel::SelectArithUXTXRegister(SDValue N, SDValue &Reg,
                                                   SDValue &Shift) {
   unsigned ShiftVal = 0;
@@ -2841,7 +2841,7 @@ static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
 // After #1, x useful bits are 0x7, then the useful bits of x, live through
 // y.
 // After #2, the useful bits of x are 0x4.
-// However, if x is used on an unpredicatable instruction, then all its bits
+// However, if x is used on an unpredictable instruction, then all its bits
 // are useful.
 // E.g.
 // 1. y = x & 0x7
@@ -3611,7 +3611,7 @@ static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
       DstLSB = 0;
       Width = ImmS - ImmR + 1;
       // FIXME: This constraint is to catch bitfield insertion we may
-      // want to widen the pattern if we want to grab general bitfied
+      // want to widen the pattern if we want to grab general bitfield
       // move case
       if (Width <= 0)
         continue;
@@ -3999,7 +3999,7 @@ static int getIntOperandFromRegisterString(StringRef RegString) {
 
 // Lower the read_register intrinsic to an MRS instruction node if the special
 // register string argument is either of the form detailed in the ALCE (the
-// form described in getIntOperandsFromRegsterString) or is a named register
+// form described in getIntOperandsFromRegisterString) or is a named register
 // known by the MRS SysReg mapper.
 bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
   const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
@@ -4060,7 +4060,7 @@ bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
 
 // Lower the write_register intrinsic to an MSR instruction node if the special
 // register string argument is either of the form detailed in the ALCE (the
-// form described in getIntOperandsFromRegsterString) or is a named register
+// form described in getIntOperandsFromRegisterString) or is a named register
 // known by the MSR SysReg mapper.
 bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
   const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
@@ -4606,7 +4606,33 @@ bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
     return false;
   }
 
-  if (!Subtarget->hasSHA3())
+  // We have Neon SHA3 XAR operation for v2i64 but for types
+  // v4i32, v8i16, v16i8 we can use SVE operations when SVE2-SHA3
+  // is available.
+  EVT SVT;
+  switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::v4i32:
+  case MVT::v2i32:
+    SVT = MVT::nxv4i32;
+    break;
+  case MVT::v8i16:
+  case MVT::v4i16:
+    SVT = MVT::nxv8i16;
+    break;
+  case MVT::v16i8:
+  case MVT::v8i8:
+    SVT = MVT::nxv16i8;
+    break;
+  case MVT::v2i64:
+  case MVT::v1i64:
+    SVT = Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
+    break;
+  default:
+    return false;
+  }
+
+  if ((!SVT.isScalableVector() && !Subtarget->hasSHA3()) ||
+      (SVT.isScalableVector() && !Subtarget->hasSVE2()))
     return false;
 
   if (N0->getOpcode() != AArch64ISD::VSHL ||
@@ -4632,7 +4658,8 @@ bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
   SDValue Imm = CurDAG->getTargetConstant(
       ShAmt, DL, N0.getOperand(1).getValueType(), false);
 
-  if (ShAmt + HsAmt != 64)
+  unsigned VTSizeInBits = VT.getScalarSizeInBits();
+  if (ShAmt + HsAmt != VTSizeInBits)
     return false;
 
   if (!IsXOROperand) {
@@ -4640,33 +4667,76 @@ bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
     SDNode *MOV =
         CurDAG->getMachineNode(AArch64::MOVIv2d_ns, DL, MVT::v2i64, Zero);
     SDValue MOVIV = SDValue(MOV, 0);
+
     R1 = N1->getOperand(0);
     R2 = MOVIV;
   }
 
-  // If the input is a v1i64, widen to a v2i64 to use XAR.
-  assert((VT == MVT::v1i64 || VT == MVT::v2i64) && "Unexpected XAR type!");
-  if (VT == MVT::v1i64) {
-    EVT SVT = MVT::v2i64;
+  if (SVT != VT) {
     SDValue Undef =
-        SDValue(CurDAG->getMachineNode(AArch64::IMPLICIT_DEF, DL, SVT), 0);
-    SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
+        SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, SVT), 0);
+
+    if (SVT.isScalableVector() && VT.is64BitVector()) {
+      EVT QVT = VT.getDoubleNumVectorElementsVT(*CurDAG->getContext());
+
+      SDValue UndefQ = SDValue(
+          CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, QVT), 0);
+      SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
+
+      R1 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, QVT,
+                                          UndefQ, R1, DSub),
+                   0);
+      if (R2.getValueType() == VT)
+        R2 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, QVT,
+                                            UndefQ, R2, DSub),
+                     0);
+    }
+
+    SDValue SubReg = CurDAG->getTargetConstant(
+        (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL, MVT::i32);
+
     R1 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, SVT, Undef,
-                                        R1, DSub),
+                                        R1, SubReg),
                  0);
-    if (R2.getValueType() == MVT::v1i64)
+
+    if (SVT.isScalableVector() || R2.getValueType() != SVT)
       R2 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, SVT,
-                                          Undef, R2, DSub),
+                                          Undef, R2, SubReg),
                    0);
   }
 
   SDValue Ops[] = {R1, R2, Imm};
-  SDNode *XAR = CurDAG->getMachineNode(AArch64::XAR, DL, MVT::v2i64, Ops);
+  SDNode *XAR = nullptr;
+
+  if (SVT.isScalableVector()) {
+    if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
+            SVT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
+                  AArch64::XAR_ZZZI_D}))
+      XAR = CurDAG->getMachineNode(Opc, DL, SVT, Ops);
+  } else {
+    XAR = CurDAG->getMachineNode(AArch64::XAR, DL, SVT, Ops);
+  }
 
-  if (VT == MVT::v1i64) {
-    SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
-    XAR = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG, DL, VT,
-                                 SDValue(XAR, 0), DSub);
+  assert(XAR && "Unexpected NULL value for XAR instruction in DAG");
+
+  if (SVT != VT) {
+    if (VT.is64BitVector() && SVT.isScalableVector()) {
+      EVT QVT = VT.getDoubleNumVectorElementsVT(*CurDAG->getContext());
+
+      SDValue ZSub = CurDAG->getTargetConstant(AArch64::zsub, DL, MVT::i32);
+      SDNode *Q = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG, DL, QVT,
+                                         SDValue(XAR, 0), ZSub);
+
+      SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
+      XAR = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG, DL, VT,
+                                   SDValue(Q, 0), DSub);
+    } else {
+      SDValue SubReg = CurDAG->getTargetConstant(
+          (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL,
+          MVT::i32);
+      XAR = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG, DL, VT,
+                                   SDValue(XAR, 0), SubReg);
+    }
   }
   ReplaceNode(N, XAR);
   return true;
@@ -7278,7 +7348,7 @@ static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
 }
 
 /// Return the EVT of the data associated to a memory operation in \p
-/// Root. If such EVT cannot be retrived, it returns an invalid EVT.
+/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
 static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
   if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Root))
     return MemIntr->getMemoryVT();
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 015d287d216d..dbc20f91e240 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1336,7 +1336,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::MUL, MVT::v1i64, Custom);
 
     // Saturates
-    for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
+    for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v1i64,
                     MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
       setOperationAction(ISD::SADDSAT, VT, Legal);
       setOperationAction(ISD::UADDSAT, VT, Legal);
@@ -1464,6 +1464,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setPartialReduceMLAAction(MLAOps, MVT::v4i32, MVT::v16i8, Legal);
       setPartialReduceMLAAction(MLAOps, MVT::v2i32, MVT::v8i8, Legal);
       setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v16i8, Custom);
+
+      if (Subtarget->hasMatMulInt8()) {
+        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v4i32,
+                                  MVT::v16i8, Legal);
+        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v2i64,
+                                  MVT::v16i8, Custom);
+
+        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v2i32,
+                                  MVT::v8i8, Legal);
+      }
     }
 
   } else /* !isNeonAvailable */ {
@@ -1511,6 +1521,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
     }
 
+    if (Subtarget->hasSVE2p1() ||
+        (Subtarget->hasSME2() && Subtarget->isStreaming()))
+      setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv32i1, Custom);
+
     for (auto VT : {MVT::v16i8, MVT::v8i8, MVT::v4i16, MVT::v2i32})
       setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Custom);
   }
@@ -1891,6 +1905,13 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
     setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom);
 
+    if (Subtarget->hasMatMulInt8()) {
+      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::nxv4i32,
+                                MVT::nxv16i8, Legal);
+      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::nxv2i64,
+                                MVT::nxv16i8, Custom);
+    }
+
     // Wide add types
     if (Subtarget->hasSVE2() || Subtarget->hasSME()) {
       setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv4i32, Legal);
@@ -1982,17 +2003,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
     if (isOperationExpand(Op, MVT::f16))
       setOperationAction(Op, MVT::f16, Promote);
-
-  if (Subtarget->isWindowsArm64EC()) {
-    // FIXME: are there intrinsics we need to exclude from this?
-    for (int i = 0; i < RTLIB::UNKNOWN_LIBCALL; ++i) {
-      auto code = static_cast<RTLIB::Libcall>(i);
-      auto libcallName = getLibcallName(code);
-      if ((libcallName != nullptr) && (libcallName[0] != '#')) {
-        setLibcallName(code, Saver.save(Twine("#") + libcallName).data());
-      }
-    }
-  }
 }
 
 void AArch64TargetLowering::addTypeForNEON(MVT VT) {
@@ -2262,6 +2272,17 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
       setPartialReduceMLAAction(MLAOps, VT,
                                 MVT::getVectorVT(MVT::i8, NumElts * 2), Custom);
     }
+
+    if (Subtarget->hasMatMulInt8()) {
+      if (VT.getVectorElementType() == MVT::i32)
+        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, VT,
+                                  MVT::getVectorVT(MVT::i8, NumElts * 4),
+                                  Custom);
+      else if (VT.getVectorElementType() == MVT::i64)
+        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, VT,
+                                  MVT::getVectorVT(MVT::i8, NumElts * 8),
+                                  Custom);
+    }
   }
 
   // Lower fixed length vector operations to scalable equivalents.
@@ -3382,8 +3403,19 @@ bool isLegalCmpImmed(APInt C) {
   return isLegalArithImmed(C.abs().getZExtValue());
 }
 
-static bool cannotBeIntMin(SDValue CheckedVal, SelectionDAG &DAG) {
-  KnownBits KnownSrc = DAG.computeKnownBits(CheckedVal);
+static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG) {
+  // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
+  if (Op->getFlags().hasNoSignedWrap())
+    return true;
+
+  // We can still figure out if the second operand is safe to use
+  // in a CMN instruction by checking if it is known to be not the minimum
+  // signed value. If it is not, then we can safely use CMN.
+  // Note: We can eventually remove this check and simply rely on
+  // Op->getFlags().hasNoSignedWrap() once SelectionDAG/ISelLowering
+  // consistently sets them appropriately when making said nodes.
+
+  KnownBits KnownSrc = DAG.computeKnownBits(Op.getOperand(1));
   return !KnownSrc.getSignedMinValue().isMinSignedValue();
 }
 
@@ -3392,7 +3424,7 @@ static bool cannotBeIntMin(SDValue CheckedVal, SelectionDAG &DAG) {
 // can be set differently by this operation. It comes down to whether
 // "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
 // everything is fine. If not then the optimization is wrong. Thus general
-// comparisons are only valid if op2 != 0.
+// comparisons are only valid if op2 != 0 and op2 != INT_MIN.
 //
 // So, finally, the only LLVM-native comparisons that don't mention C or V
 // are the ones that aren't unsigned comparisons. They're the only ones we can
@@ -3401,7 +3433,7 @@ static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
   return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
          (isIntEqualitySetCC(CC) ||
           (isUnsignedIntSetCC(CC) && DAG.isKnownNeverZero(Op.getOperand(1))) ||
-          (isSignedIntSetCC(CC) && cannotBeIntMin(Op.getOperand(1), DAG)));
+          (isSignedIntSetCC(CC) && isSafeSignedCMN(Op, DAG)));
 }
 
 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
@@ -5374,7 +5406,7 @@ static unsigned selectUmullSmull(SDValue &N0, SDValue &N1, SelectionDAG &DAG,
       return AArch64ISD::UMULL;
   } else if (VT == MVT::v2i64 && DAG.MaskedValueIsZero(N0, Mask) &&
              DAG.MaskedValueIsZero(N1, Mask)) {
-    // For v2i64 we look more aggresively at both operands being zero, to avoid
+    // For v2i64 we look more aggressively at both operands being zero, to avoid
     // scalarization.
     return AArch64ISD::UMULL;
   }
@@ -5851,7 +5883,7 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
       return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1));
     } else {
-      report_fatal_error("Unexpected type for AArch64 NEON intrinic");
+      report_fatal_error("Unexpected type for AArch64 NEON intrinsic");
     }
   }
   case Intrinsic::aarch64_neon_pmull64: {
@@ -5877,19 +5909,19 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       // If the operand is an higher half itself, rewrite it to
       // extract_high_v2i64; this way aarch64_neon_pmull64 could
       // re-use the dag-combiner function with aarch64_neon_{pmull,smull,umull}.
-      if (NLane && *NLane == 1)
+      if (NLane == 1)
         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i64,
                            N.getOperand(0), DAG.getConstant(1, dl, MVT::i64));
 
       // Operand N is not a higher half but the other operand is.
-      if (OtherLane && *OtherLane == 1) {
+      if (OtherLane == 1) {
         // If this operand is a lower half, rewrite it to
         // extract_high_v2i64(duplane(<2 x Ty>, 0)). This saves a roundtrip to
         // align lanes of two operands. A roundtrip sequence (to move from lane
         // 1 to lane 0) is like this:
         //   mov x8, v0.d[1]
         //   fmov d0, x8
-        if (NLane && *NLane == 0)
+        if (NLane == 0)
           return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i64,
                              DAG.getNode(AArch64ISD::DUPLANE64, dl, MVT::v2i64,
                                          N.getOperand(0),
@@ -7116,59 +7148,81 @@ static SDValue LowerFLDEXP(SDValue Op, SelectionDAG &DAG) {
 
 SDValue AArch64TargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
                                                       SelectionDAG &DAG) const {
-  // Note: x18 cannot be used for the Nest parameter on Windows and macOS.
-  if (Subtarget->isTargetDarwin() || Subtarget->isTargetWindows())
-    report_fatal_error(
-        "ADJUST_TRAMPOLINE operation is only supported on Linux.");
-
   return Op.getOperand(0);
 }
 
 SDValue AArch64TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
                                                     SelectionDAG &DAG) const {
-
-  // Note: x18 cannot be used for the Nest parameter on Windows and macOS.
-  if (Subtarget->isTargetDarwin() || Subtarget->isTargetWindows())
-    report_fatal_error("INIT_TRAMPOLINE operation is only supported on Linux.");
-
   SDValue Chain = Op.getOperand(0);
-  SDValue Trmp = Op.getOperand(1); // trampoline
+  SDValue Trmp = Op.getOperand(1); // trampoline, >=32 bytes
   SDValue FPtr = Op.getOperand(2); // nested function
   SDValue Nest = Op.getOperand(3); // 'nest' parameter value
+
+  const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
+
+  // ldr NestReg, .+16
+  // ldr x17, .+20
+  // br x17
+  // .word 0
+  // .nest: .qword nest
+  // .fptr: .qword fptr
+  SDValue OutChains[5];
+
+  const Function *Func =
+      cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
+  CallingConv::ID CC = Func->getCallingConv();
+  unsigned NestReg;
+
+  switch (CC) {
+  default:
+    NestReg = 0x0f; // X15
+    LLVM_FALLTHROUGH;
+  case CallingConv::ARM64EC_Thunk_Native:
+  case CallingConv::ARM64EC_Thunk_X64:
+    // Must be kept in sync with AArch64CallingConv.td
+    NestReg = 0x04; // X4
+    break;
+  }
+
+  const char FptrReg = 0x11; // X17
+
+  SDValue Addr = Trmp;
+
   SDLoc dl(Op);
+  OutChains[0] = DAG.getStore(
+      Chain, dl, DAG.getConstant(0x58000080u | NestReg, dl, MVT::i32), Addr,
+      MachinePointerInfo(TrmpAddr));
 
-  EVT PtrVT = getPointerTy(DAG.getDataLayout());
-  Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
+  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
+                     DAG.getConstant(4, dl, MVT::i64));
+  OutChains[1] = DAG.getStore(
+      Chain, dl, DAG.getConstant(0x580000b0u | FptrReg, dl, MVT::i32), Addr,
+      MachinePointerInfo(TrmpAddr, 4));
 
-  TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
+  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
+                     DAG.getConstant(8, dl, MVT::i64));
+  OutChains[2] =
+      DAG.getStore(Chain, dl, DAG.getConstant(0xd61f0220u, dl, MVT::i32), Addr,
+                   MachinePointerInfo(TrmpAddr, 8));
 
-  Entry.Ty = IntPtrTy;
-  Entry.Node = Trmp;
-  Args.push_back(Entry);
+  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
+                     DAG.getConstant(16, dl, MVT::i64));
+  OutChains[3] =
+      DAG.getStore(Chain, dl, Nest, Addr, MachinePointerInfo(TrmpAddr, 16));
 
-  if (auto *FI = dyn_cast<FrameIndexSDNode>(Trmp.getNode())) {
-    MachineFunction &MF = DAG.getMachineFunction();
-    MachineFrameInfo &MFI = MF.getFrameInfo();
-    Entry.Node =
-        DAG.getConstant(MFI.getObjectSize(FI->getIndex()), dl, MVT::i64);
-  } else
-    Entry.Node = DAG.getConstant(36, dl, MVT::i64);
+  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
+                     DAG.getConstant(24, dl, MVT::i64));
+  OutChains[4] =
+      DAG.getStore(Chain, dl, FPtr, Addr, MachinePointerInfo(TrmpAddr, 24));
 
-  Args.push_back(Entry);
-  Entry.Node = FPtr;
-  Args.push_back(Entry);
-  Entry.Node = Nest;
-  Args.push_back(Entry);
+  SDValue StoreToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
 
-  // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
-  TargetLowering::CallLoweringInfo CLI(DAG);
-  CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
-      CallingConv::C, Type::getVoidTy(*DAG.getContext()),
-      DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args));
+  SDValue EndOfTrmp = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
+                                  DAG.getConstant(12, dl, MVT::i64));
 
-  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
-  return CallResult.second;
+  // Call clear cache on the trampoline instructions.
+  return DAG.getNode(ISD::CLEAR_CACHE, dl, MVT::Other, StoreToken, Trmp,
+                     EndOfTrmp);
 }
 
 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
@@ -7523,6 +7577,7 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerVECTOR_HISTOGRAM(Op, DAG);
   case ISD::PARTIAL_REDUCE_SMLA:
   case ISD::PARTIAL_REDUCE_UMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     return LowerPARTIAL_REDUCE_MLA(Op, DAG);
   }
 }
@@ -7751,7 +7806,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
     (void)Res;
   }
 
-  SMEAttrs Attrs(MF.getFunction());
+  SMEAttrs Attrs = FuncInfo->getSMEFnAttrs();
   bool IsLocallyStreaming =
       !Attrs.hasStreamingInterface() && Attrs.hasStreamingBody();
   assert(Chain.getOpcode() == ISD::EntryToken && "Unexpected Chain value");
@@ -8105,7 +8160,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
 
   // Create a 16 Byte TPIDR2 object. The dynamic buffer
   // will be expanded and stored in the static object later using a pseudonode.
-  if (SMEAttrs(MF.getFunction()).hasZAState()) {
+  if (Attrs.hasZAState()) {
     TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj();
     TPIDR2.FrameIndex = MFI.CreateStackObject(16, Align(16), false);
     SDValue SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
@@ -8125,7 +8180,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
     Chain = DAG.getNode(
         AArch64ISD::INIT_TPIDR2OBJ, DL, DAG.getVTList(MVT::Other),
         {/*Chain*/ Buffer.getValue(1), /*Buffer ptr*/ Buffer.getValue(0)});
-  } else if (SMEAttrs(MF.getFunction()).hasAgnosticZAInterface()) {
+  } else if (Attrs.hasAgnosticZAInterface()) {
     // Call __arm_sme_state_size().
     SDValue BufferSize =
         DAG.getNode(AArch64ISD::GET_SME_SAVE_SIZE, DL,
@@ -8637,9 +8692,9 @@ static bool checkZExtBool(SDValue Arg, const SelectionDAG &DAG) {
   if (SizeInBits < 8)
     return false;
 
-  APInt RequredZero(SizeInBits, 0xFE);
+  APInt RequiredZero(SizeInBits, 0xFE);
   KnownBits Bits = DAG.computeKnownBits(Arg, 4);
-  bool ZExtBool = (Bits.Zero & RequredZero) == RequredZero;
+  bool ZExtBool = (Bits.Zero & RequiredZero) == RequiredZero;
   return ZExtBool;
 }
 
@@ -8760,6 +8815,45 @@ getSMToggleCondition(const SMECallAttrs &CallAttrs) {
   llvm_unreachable("Unsupported attributes");
 }
 
+/// Check whether a stack argument requires lowering in a tail call.
+static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
+                                        const CCValAssign &VA, SDValue Arg,
+                                        ISD::ArgFlagsTy Flags, int CallOffset) {
+  // FIXME: We should be able to handle this case, but it's not clear how to.
+  if (Flags.isZExt() || Flags.isSExt())
+    return true;
+
+  for (;;) {
+    // Look through nodes that don't alter the bits of the incoming value.
+    unsigned Op = Arg.getOpcode();
+    if (Op == ISD::ZERO_EXTEND || Op == ISD::ANY_EXTEND || Op == ISD::BITCAST ||
+        Arg->isAssert() || Op == AArch64ISD::ASSERT_ZEXT_BOOL) {
+      Arg = Arg.getOperand(0);
+      continue;
+    }
+    break;
+  }
+
+  // If the argument is a load from the same immutable stack slot, we can reuse
+  // it.
+  if (auto *LoadNode = dyn_cast<LoadSDNode>(Arg)) {
+    if (auto *FINode = dyn_cast<FrameIndexSDNode>(LoadNode->getBasePtr())) {
+      const MachineFrameInfo &MFI = MF.getFrameInfo();
+      int FI = FINode->getIndex();
+      if (!MFI.isImmutableObjectIndex(FI))
+        return true;
+      if (CallOffset != MFI.getObjectOffset(FI))
+        return true;
+      uint64_t SizeInBits = LoadNode->getMemoryVT().getFixedSizeInBits();
+      if (SizeInBits / 8 != MFI.getObjectSize(FI))
+        return true;
+      return false;
+    }
+  }
+
+  return true;
+}
+
 /// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
 /// and add input and output parameter nodes.
 SDValue
@@ -9182,10 +9276,13 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
       }
       unsigned LocMemOffset = VA.getLocMemOffset();
       int32_t Offset = LocMemOffset + BEAlign;
-      SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
-      PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
 
       if (IsTailCall) {
+        // When the frame pointer is perfectly aligned for the tail call and the
+        // same stack argument is passed down intact, we can reuse it.
+        if (!FPDiff && !shouldLowerTailCallStackArg(MF, VA, Arg, Flags, Offset))
+          continue;
+
         Offset = Offset + FPDiff;
         int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
 
@@ -9610,7 +9707,7 @@ AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
 
   // Emit SMSTOP before returning from a locally streaming function
-  SMEAttrs FuncAttrs(MF.getFunction());
+  SMEAttrs FuncAttrs = FuncInfo->getSMEFnAttrs();
   if (FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface()) {
     if (FuncAttrs.hasStreamingCompatibleInterface()) {
       Register Reg = FuncInfo->getPStateSMReg();
@@ -11002,10 +11099,125 @@ SDValue AArch64TargetLowering::LowerSETCCCARRY(SDValue Op,
                      Cmp.getValue(1));
 }
 
-SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
-                                              SDValue RHS, SDValue TVal,
-                                              SDValue FVal, const SDLoc &dl,
-                                              SelectionDAG &DAG) const {
+/// Emit vector comparison for floating-point values, producing a mask.
+static SDValue emitVectorComparison(SDValue LHS, SDValue RHS,
+                                    AArch64CC::CondCode CC, bool NoNans, EVT VT,
+                                    const SDLoc &DL, SelectionDAG &DAG) {
+  assert(VT.getSizeInBits() == LHS.getValueType().getSizeInBits() &&
+         "function only supposed to emit natural comparisons");
+
+  switch (CC) {
+  default:
+    return SDValue();
+  case AArch64CC::NE: {
+    SDValue Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, DL, VT, LHS, RHS);
+    // Use vector semantics for the inversion to potentially save a copy between
+    // SIMD and regular registers.
+    if (!LHS.getValueType().isVector()) {
+      EVT VecVT =
+          EVT::getVectorVT(*DAG.getContext(), VT, 128 / VT.getSizeInBits());
+      SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
+      SDValue MaskVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecVT,
+                                    DAG.getUNDEF(VecVT), Fcmeq, Zero);
+      SDValue InvertedMask = DAG.getNOT(DL, MaskVec, VecVT);
+      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, InvertedMask, Zero);
+    }
+    return DAG.getNOT(DL, Fcmeq, VT);
+  }
+  case AArch64CC::EQ:
+    return DAG.getNode(AArch64ISD::FCMEQ, DL, VT, LHS, RHS);
+  case AArch64CC::GE:
+    return DAG.getNode(AArch64ISD::FCMGE, DL, VT, LHS, RHS);
+  case AArch64CC::GT:
+    return DAG.getNode(AArch64ISD::FCMGT, DL, VT, LHS, RHS);
+  case AArch64CC::LE:
+    if (!NoNans)
+      return SDValue();
+    // If we ignore NaNs then we can use to the LS implementation.
+    [[fallthrough]];
+  case AArch64CC::LS:
+    return DAG.getNode(AArch64ISD::FCMGE, DL, VT, RHS, LHS);
+  case AArch64CC::LT:
+    if (!NoNans)
+      return SDValue();
+    // If we ignore NaNs then we can use to the MI implementation.
+    [[fallthrough]];
+  case AArch64CC::MI:
+    return DAG.getNode(AArch64ISD::FCMGT, DL, VT, RHS, LHS);
+  }
+}
+
+/// For SELECT_CC, when the true/false values are (-1, 0) and the compared
+/// values are scalars, try to emit a mask generating vector instruction.
+static SDValue emitFloatCompareMask(SDValue LHS, SDValue RHS, SDValue TVal,
+                                    SDValue FVal, ISD::CondCode CC, bool NoNaNs,
+                                    const SDLoc &DL, SelectionDAG &DAG) {
+  assert(!LHS.getValueType().isVector());
+  assert(!RHS.getValueType().isVector());
+
+  auto *CTVal = dyn_cast<ConstantSDNode>(TVal);
+  auto *CFVal = dyn_cast<ConstantSDNode>(FVal);
+  if (!CTVal || !CFVal)
+    return {};
+  if (!(CTVal->isAllOnes() && CFVal->isZero()) &&
+      !(CTVal->isZero() && CFVal->isAllOnes()))
+    return {};
+
+  if (CTVal->isZero())
+    CC = ISD::getSetCCInverse(CC, LHS.getValueType());
+
+  EVT VT = TVal.getValueType();
+  if (VT.getSizeInBits() != LHS.getValueType().getSizeInBits())
+    return {};
+
+  if (!NoNaNs && (CC == ISD::SETUO || CC == ISD::SETO)) {
+    bool OneNaN = false;
+    if (LHS == RHS) {
+      OneNaN = true;
+    } else if (DAG.isKnownNeverNaN(RHS)) {
+      OneNaN = true;
+      RHS = LHS;
+    } else if (DAG.isKnownNeverNaN(LHS)) {
+      OneNaN = true;
+      LHS = RHS;
+    }
+    if (OneNaN)
+      CC = (CC == ISD::SETUO) ? ISD::SETUNE : ISD::SETOEQ;
+  }
+
+  AArch64CC::CondCode CC1;
+  AArch64CC::CondCode CC2;
+  bool ShouldInvert = false;
+  changeVectorFPCCToAArch64CC(CC, CC1, CC2, ShouldInvert);
+  SDValue Cmp = emitVectorComparison(LHS, RHS, CC1, NoNaNs, VT, DL, DAG);
+  SDValue Cmp2;
+  if (CC2 != AArch64CC::AL) {
+    Cmp2 = emitVectorComparison(LHS, RHS, CC2, NoNaNs, VT, DL, DAG);
+    if (!Cmp2)
+      return {};
+  }
+  if (!Cmp2 && !ShouldInvert)
+    return Cmp;
+
+  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), VT, 128 / VT.getSizeInBits());
+  SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
+  Cmp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecVT, DAG.getUNDEF(VecVT), Cmp,
+                    Zero);
+  if (Cmp2) {
+    Cmp2 = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecVT, DAG.getUNDEF(VecVT),
+                       Cmp2, Zero);
+    Cmp = DAG.getNode(ISD::OR, DL, VecVT, Cmp, Cmp2);
+  }
+  if (ShouldInvert)
+    Cmp = DAG.getNOT(DL, Cmp, VecVT);
+  Cmp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cmp, Zero);
+  return Cmp;
+}
+
+SDValue AArch64TargetLowering::LowerSELECT_CC(
+    ISD::CondCode CC, SDValue LHS, SDValue RHS, SDValue TVal, SDValue FVal,
+    iterator_range<SDNode::user_iterator> Users, bool HasNoNaNs,
+    const SDLoc &dl, SelectionDAG &DAG) const {
   // Handle f128 first, because it will result in a comparison of some RTLIB
   // call result against zero.
   if (LHS.getValueType() == MVT::f128) {
@@ -11188,6 +11400,27 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
          LHS.getValueType() == MVT::f64);
   assert(LHS.getValueType() == RHS.getValueType());
   EVT VT = TVal.getValueType();
+
+  // If the purpose of the comparison is to select between all ones
+  // or all zeros, try to use a vector comparison because the operands are
+  // already stored in SIMD registers.
+  if (Subtarget->isNeonAvailable() && all_of(Users, [](const SDNode *U) {
+        switch (U->getOpcode()) {
+        default:
+          return false;
+        case ISD::INSERT_VECTOR_ELT:
+        case ISD::SCALAR_TO_VECTOR:
+        case AArch64ISD::DUP:
+          return true;
+        }
+      })) {
+    bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath || HasNoNaNs;
+    SDValue VectorCmp =
+        emitFloatCompareMask(LHS, RHS, TVal, FVal, CC, NoNaNs, dl, DAG);
+    if (VectorCmp)
+      return VectorCmp;
+  }
+
   SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
 
   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
@@ -11274,8 +11507,10 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
   SDValue RHS = Op.getOperand(1);
   SDValue TVal = Op.getOperand(2);
   SDValue FVal = Op.getOperand(3);
+  bool HasNoNans = Op->getFlags().hasNoNaNs();
   SDLoc DL(Op);
-  return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
+  return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, Op->users(), HasNoNans, DL,
+                        DAG);
 }
 
 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
@@ -11283,6 +11518,7 @@ SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
   SDValue CCVal = Op->getOperand(0);
   SDValue TVal = Op->getOperand(1);
   SDValue FVal = Op->getOperand(2);
+  bool HasNoNans = Op->getFlags().hasNoNaNs();
   SDLoc DL(Op);
 
   EVT Ty = Op.getValueType();
@@ -11349,7 +11585,8 @@ SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
                                      DAG.getUNDEF(MVT::f32), FVal);
   }
 
-  SDValue Res = LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
+  SDValue Res =
+      LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, Op->users(), HasNoNans, DL, DAG);
 
   if ((Ty == MVT::f16 || Ty == MVT::bf16) && !Subtarget->hasFullFP16()) {
     return DAG.getTargetExtractSubreg(AArch64::hsub, DL, Ty, Res);
@@ -12056,13 +12293,14 @@ enum class PredicateConstraint { Uph, Upl, Upa };
 // not what we want. The code here pre-empts this by matching the register
 // explicitly.
 static std::optional<std::pair<unsigned, const TargetRegisterClass *>>
-parsePredicateRegAsConstraint(StringRef Constraint) {
+parseSVERegAsConstraint(StringRef Constraint) {
   if (!Constraint.starts_with('{') || !Constraint.ends_with('}') ||
-      Constraint[1] != 'p')
+      (Constraint[1] != 'p' && Constraint[1] != 'z'))
     return std::nullopt;
 
+  bool IsPredicate = Constraint[1] == 'p';
   Constraint = Constraint.substr(2, Constraint.size() - 3);
-  bool IsPredicateAsCount = Constraint.starts_with("n");
+  bool IsPredicateAsCount = IsPredicate && Constraint.starts_with("n");
   if (IsPredicateAsCount)
     Constraint = Constraint.drop_front(1);
 
@@ -12072,8 +12310,9 @@ parsePredicateRegAsConstraint(StringRef Constraint) {
 
   if (IsPredicateAsCount)
     return std::make_pair(AArch64::PN0 + V, &AArch64::PNRRegClass);
-  else
+  if (IsPredicate)
     return std::make_pair(AArch64::P0 + V, &AArch64::PPRRegClass);
+  return std::make_pair(AArch64::Z0 + V, &AArch64::ZPRRegClass);
 }
 
 static std::optional<PredicateConstraint>
@@ -12323,8 +12562,16 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
       break;
     }
   } else {
-    if (const auto P = parsePredicateRegAsConstraint(Constraint))
+    if (const auto P = parseSVERegAsConstraint(Constraint)) {
+      // SME functions that are not in streaming mode, should
+      // still observe clobbers of Z-registers by clobbering
+      // the lower 128bits of those registers.
+      if (AArch64::ZPRRegClass.hasSubClassEq(P->second) &&
+          !Subtarget->isSVEorStreamingSVEAvailable())
+        return std::make_pair(TRI->getSubReg(P->first, AArch64::zsub),
+                              &AArch64::FPR128RegClass);
       return *P;
+    }
     if (const auto PC = parsePredicateConstraint(Constraint))
       if (const auto *RegClass = getPredicateRegisterClass(*PC, VT))
         return std::make_pair(0U, RegClass);
@@ -13362,7 +13609,7 @@ static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
         OpLHS = DAG.getBitcast(MVT::v2f32, OpLHS);
       } else {
         assert(VT.getScalarSizeInBits() == 32 &&
-               "Expected 16 or 32 bit shuffle elemements");
+               "Expected 16 or 32 bit shuffle elements");
         Input = DAG.getBitcast(MVT::v2f64, Input);
         OpLHS = DAG.getBitcast(MVT::v2f64, OpLHS);
       }
@@ -13767,7 +14014,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       unsigned NewEltCount = VT.getSizeInBits() / LaneSize;
       MVT NewVecTy = MVT::getVectorVT(NewEltTy, NewEltCount);
       V1 = DAG.getBitcast(NewVecTy, V1);
-      // Constuct the DUP instruction
+      // Construct the DUP instruction
       V1 = constructDup(V1, Lane, dl, NewVecTy, Opcode, DAG);
       // Cast back to the original type
       return DAG.getBitcast(VT, V1);
@@ -15602,47 +15849,6 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
   llvm_unreachable("unexpected shift opcode");
 }
 
-static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
-                                    AArch64CC::CondCode CC, bool NoNans, EVT VT,
-                                    const SDLoc &dl, SelectionDAG &DAG) {
-  EVT SrcVT = LHS.getValueType();
-  assert(VT.getSizeInBits() == SrcVT.getSizeInBits() &&
-         "function only supposed to emit natural comparisons");
-
-  if (SrcVT.getVectorElementType().isFloatingPoint()) {
-    switch (CC) {
-    default:
-      return SDValue();
-    case AArch64CC::NE: {
-      SDValue Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
-      return DAG.getNOT(dl, Fcmeq, VT);
-    }
-    case AArch64CC::EQ:
-      return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
-    case AArch64CC::GE:
-      return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
-    case AArch64CC::GT:
-      return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
-    case AArch64CC::LE:
-      if (!NoNans)
-        return SDValue();
-      // If we ignore NaNs then we can use to the LS implementation.
-      [[fallthrough]];
-    case AArch64CC::LS:
-      return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
-    case AArch64CC::LT:
-      if (!NoNans)
-        return SDValue();
-      // If we ignore NaNs then we can use to the MI implementation.
-      [[fallthrough]];
-    case AArch64CC::MI:
-      return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
-    }
-  }
-
-  return SDValue();
-}
-
 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
                                            SelectionDAG &DAG) const {
   if (Op.getValueType().isScalableVector())
@@ -15691,15 +15897,14 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
   bool ShouldInvert;
   changeVectorFPCCToAArch64CC(CC, CC1, CC2, ShouldInvert);
 
-  bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs();
-  SDValue Cmp =
-      EmitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, dl, DAG);
+  bool NoNaNs =
+      getTargetMachine().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs();
+  SDValue Cmp = emitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, dl, DAG);
   if (!Cmp.getNode())
     return SDValue();
 
   if (CC2 != AArch64CC::AL) {
-    SDValue Cmp2 =
-        EmitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, dl, DAG);
+    SDValue Cmp2 = emitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, dl, DAG);
     if (!Cmp2.getNode())
       return SDValue();
 
@@ -16768,12 +16973,12 @@ bool AArch64TargetLowering::optimizeExtendOrTruncateConversion(
 }
 
 bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType,
-                                          Align &RequiredAligment) const {
+                                          Align &RequiredAlignment) const {
   if (!LoadedType.isSimple() ||
       (!LoadedType.isInteger() && !LoadedType.isFloatingPoint()))
     return false;
   // Cyclone supports unaligned accesses.
-  RequiredAligment = Align(1);
+  RequiredAlignment = Align(1);
   unsigned NumBits = LoadedType.getSizeInBits();
   return NumBits == 32 || NumBits == 64;
 }
@@ -17896,7 +18101,7 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
       EXT1->getOperand(0)->getValueType(0) != MVT::v16i8)
     return SDValue();
 
-  // Pattern is dectected. Let's convert it to sequence of nodes.
+  // Pattern is detected. Let's convert it to sequence of nodes.
   SDLoc DL(N);
 
   // First, create the node pattern of UABD/SABD.
@@ -17939,7 +18144,7 @@ performActiveLaneMaskCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                                 /*IsEqual=*/false))
     return While;
 
-  if (!ST->hasSVE2p1())
+  if (!ST->hasSVE2p1() && !(ST->hasSME2() && ST->isStreaming()))
     return SDValue();
 
   if (!N->hasNUsesOfValue(2, 0))
@@ -18114,10 +18319,10 @@ static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
                                 DAG.getConstant(I * 16, DL, MVT::i64));
   SDValue Dot =
       DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros, Vec8Op0, Vec8Op1);
-  SDValue VecReudceAdd8 =
+  SDValue VecReduceAdd8 =
       DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot);
   return DAG.getNode(ISD::ADD, DL, N->getValueType(0), VecReduceAdd16,
-                     VecReudceAdd8);
+                     VecReduceAdd8);
 }
 
 // Given an (integer) vecreduce, we know the order of the inputs does not
@@ -21342,7 +21547,7 @@ static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
   case Intrinsic::aarch64_neon_ushl:
     // For positive shift amounts we can use SHL, as ushl/sshl perform a regular
     // left shift for positive shift amounts. For negative shifts we can use a
-    // VASHR/VLSHR as appropiate.
+    // VASHR/VLSHR as appropriate.
     if (ShiftAmount < 0) {
       Opcode = IID == Intrinsic::aarch64_neon_sshl ? AArch64ISD::VASHR
                                                    : AArch64ISD::VLSHR;
@@ -22748,7 +22953,7 @@ static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
 }
 
 static SDValue performSpliceCombine(SDNode *N, SelectionDAG &DAG) {
-  assert(N->getOpcode() == AArch64ISD::SPLICE && "Unexepected Opcode!");
+  assert(N->getOpcode() == AArch64ISD::SPLICE && "Unexpected Opcode!");
 
   // splice(pg, op1, undef) -> op1
   if (N->getOperand(2).isUndef())
@@ -23484,10 +23689,10 @@ static SDValue performLOADCombine(SDNode *N,
                   LD->getMemOperand()->getFlags(), LD->getAAInfo());
   SDValue UndefVector = DAG.getUNDEF(NewVT);
   SDValue InsertIdx = DAG.getVectorIdxConstant(0, DL);
-  SDValue ExtendedReminingLoad =
+  SDValue ExtendedRemainingLoad =
       DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT,
                   {UndefVector, RemainingLoad, InsertIdx});
-  LoadOps.push_back(ExtendedReminingLoad);
+  LoadOps.push_back(ExtendedRemainingLoad);
   LoadOpsChain.push_back(SDValue(cast<SDNode>(RemainingLoad), 1));
   EVT ConcatVT =
       EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
@@ -24535,6 +24740,19 @@ static SDValue performBRCONDCombine(SDNode *N,
   if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
     return SDValue();
 
+  // Fold away brcond(NE, cmp(csel(1, 0, CC, Cmp), 1)) -> brcond(~CC, Cmp)
+  if (isCMP(Cmp) && CC == AArch64CC::NE && isOneConstant(Cmp.getOperand(1))) {
+    SDValue CSel = Cmp.getOperand(0);
+    auto CSelCC = getCSETCondCode(CSel);
+    if (CSelCC) {
+      SDLoc DL(N);
+      return DAG.getNode(
+          N->getOpcode(), DL, N->getVTList(), Chain, Dest,
+          DAG.getConstant(getInvertedCondCode(*CSelCC), DL, MVT::i32),
+          CSel.getOperand(3));
+    }
+  }
+
   unsigned CmpOpc = Cmp.getOpcode();
   if (CmpOpc != AArch64ISD::ADDS && CmpOpc != AArch64ISD::SUBS)
     return SDValue();
@@ -25456,6 +25674,28 @@ static SDValue performDUPCombine(SDNode *N,
   }
 
   if (N->getOpcode() == AArch64ISD::DUP) {
+    // If the instruction is known to produce a scalar in SIMD registers, we can
+    // duplicate it across the vector lanes using DUPLANE instead of moving it
+    // to a GPR first. For example, this allows us to handle:
+    //   v4i32 = DUP (i32 (FCMGT (f32, f32)))
+    SDValue Op = N->getOperand(0);
+    // FIXME: Ideally, we should be able to handle all instructions that
+    // produce a scalar value in FPRs.
+    if (Op.getOpcode() == AArch64ISD::FCMEQ ||
+        Op.getOpcode() == AArch64ISD::FCMGE ||
+        Op.getOpcode() == AArch64ISD::FCMGT) {
+      EVT ElemVT = VT.getVectorElementType();
+      EVT ExpandedVT = VT;
+      // Insert into a 128-bit vector to match DUPLANE's pattern.
+      if (VT.getSizeInBits() != 128)
+        ExpandedVT = EVT::getVectorVT(*DCI.DAG.getContext(), ElemVT,
+                                      128 / ElemVT.getSizeInBits());
+      SDValue Zero = DCI.DAG.getConstant(0, DL, MVT::i64);
+      SDValue Vec = DCI.DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ExpandedVT,
+                                    DCI.DAG.getUNDEF(ExpandedVT), Op, Zero);
+      return DCI.DAG.getNode(getDUPLANEOp(ElemVT), DL, VT, Vec, Zero);
+    }
+
     if (DCI.isAfterLegalizeDAG()) {
       // If scalar dup's operand is extract_vector_elt, try to combine them into
       // duplane. For example,
@@ -27096,6 +27336,37 @@ void AArch64TargetLowering::ReplaceExtractSubVectorResults(
   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half));
 }
 
+void AArch64TargetLowering::ReplaceGetActiveLaneMaskResults(
+    SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
+  assert((Subtarget->hasSVE2p1() ||
+          (Subtarget->hasSME2() && Subtarget->isStreaming())) &&
+         "Custom lower of get.active.lane.mask missing required feature.");
+
+  assert(N->getValueType(0) == MVT::nxv32i1 &&
+         "Unexpected result type for get.active.lane.mask");
+
+  SDLoc DL(N);
+  SDValue Idx = N->getOperand(0);
+  SDValue TC = N->getOperand(1);
+
+  assert(Idx.getValueType().getFixedSizeInBits() <= 64 &&
+         "Unexpected operand type for get.active.lane.mask");
+
+  if (Idx.getValueType() != MVT::i64) {
+    Idx = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Idx);
+    TC = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, TC);
+  }
+
+  SDValue ID =
+      DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo_x2, DL, MVT::i64);
+  EVT HalfVT = N->getValueType(0).getHalfNumVectorElementsVT(*DAG.getContext());
+  auto WideMask =
+      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, {HalfVT, HalfVT}, {ID, Idx, TC});
+
+  Results.push_back(DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0),
+                                {WideMask.getValue(0), WideMask.getValue(1)}));
+}
+
 // Create an even/odd pair of X registers holding integer value V.
 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
   SDLoc dl(V.getNode());
@@ -27482,6 +27753,9 @@ void AArch64TargetLowering::ReplaceNodeResults(
     // CONCAT_VECTORS -- but delegate to common code for result type
     // legalisation
     return;
+  case ISD::GET_ACTIVE_LANE_MASK:
+    ReplaceGetActiveLaneMaskResults(N, Results, DAG);
+    return;
   case ISD::INTRINSIC_WO_CHAIN: {
     EVT VT = N->getValueType(0);
 
@@ -29209,6 +29483,34 @@ AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
   EVT OpVT = Op.getValueType();
   assert(OpVT.isScalableVector() &&
          "Expected scalable vector in LowerVECTOR_DEINTERLEAVE.");
+
+  // Are multi-register uzp instructions available?
+  if (Subtarget->hasSME2() && Subtarget->isStreaming() &&
+      OpVT.getVectorElementType() != MVT::i1) {
+    Intrinsic::ID IntID;
+    switch (Op->getNumOperands()) {
+    default:
+      return SDValue();
+    case 2:
+      IntID = Intrinsic::aarch64_sve_uzp_x2;
+      break;
+    case 4:
+      if (Subtarget->getMinSVEVectorSizeInBits() < 256 &&
+          OpVT.getScalarSizeInBits() == 64)
+        return SDValue();
+      IntID = Intrinsic::aarch64_sve_uzp_x4;
+      break;
+    }
+
+    SmallVector<SDValue, 5> Ops;
+    Ops.push_back(DAG.getTargetConstant(IntID, DL, MVT::i64));
+    Ops.append(Op->op_values().begin(), Op->op_values().end());
+    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op->getVTList(), Ops);
+  }
+
+  if (Op->getNumOperands() != 2)
+    return SDValue();
+
   SDValue Even = DAG.getNode(AArch64ISD::UZP1, DL, OpVT, Op.getOperand(0),
                              Op.getOperand(1));
   SDValue Odd = DAG.getNode(AArch64ISD::UZP2, DL, OpVT, Op.getOperand(0),
@@ -29223,6 +29525,33 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
   assert(OpVT.isScalableVector() &&
          "Expected scalable vector in LowerVECTOR_INTERLEAVE.");
 
+  // Are multi-register zip instructions available?
+  if (Subtarget->hasSME2() && Subtarget->isStreaming() &&
+      OpVT.getVectorElementType() != MVT::i1) {
+    Intrinsic::ID IntID;
+    switch (Op->getNumOperands()) {
+    default:
+      return SDValue();
+    case 2:
+      IntID = Intrinsic::aarch64_sve_zip_x2;
+      break;
+    case 4:
+      if (Subtarget->getMinSVEVectorSizeInBits() < 256 &&
+          OpVT.getScalarSizeInBits() == 64)
+        return SDValue();
+      IntID = Intrinsic::aarch64_sve_zip_x4;
+      break;
+    }
+
+    SmallVector<SDValue, 5> Ops;
+    Ops.push_back(DAG.getTargetConstant(IntID, DL, MVT::i64));
+    Ops.append(Op->op_values().begin(), Op->op_values().end());
+    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op->getVTList(), Ops);
+  }
+
+  if (Op->getNumOperands() != 2)
+    return SDValue();
+
   SDValue Lo = DAG.getNode(AArch64ISD::ZIP1, DL, OpVT, Op.getOperand(0),
                            Op.getOperand(1));
   SDValue Hi = DAG.getNode(AArch64ISD::ZIP2, DL, OpVT, Op.getOperand(0),
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 450e2efd7d43..e0b6c1b8c0ba 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -207,7 +207,7 @@ class AArch64TargetLowering : public TargetLowering {
   bool optimizeExtendOrTruncateConversion(
       Instruction *I, Loop *L, const TargetTransformInfo &TTI) const override;
 
-  bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override;
+  bool hasPairedLoad(EVT LoadedType, Align &RequiredAlignment) const override;
 
   unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
 
@@ -449,6 +449,10 @@ class AArch64TargetLowering : public TargetLowering {
   /// Enable aggressive FMA fusion on targets that want it.
   bool enableAggressiveFMAFusion(EVT VT) const override;
 
+  bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override {
+    return true;
+  }
+
   /// Returns the size of the platform's va_list object.
   unsigned getVaListSizeInBits(const DataLayout &DL) const override;
 
@@ -540,9 +544,6 @@ class AArch64TargetLowering : public TargetLowering {
   /// make the right decision when generating code for different targets.
   const AArch64Subtarget *Subtarget;
 
-  llvm::BumpPtrAllocator BumpAlloc;
-  llvm::StringSaver Saver{BumpAlloc};
-
   bool isExtFreeImpl(const Instruction *Ext) const override;
 
   void addTypeForNEON(MVT VT);
@@ -646,7 +647,9 @@ class AArch64TargetLowering : public TargetLowering {
   SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
-                         SDValue TVal, SDValue FVal, const SDLoc &dl,
+                         SDValue TVal, SDValue FVal,
+                         iterator_range<SDNode::user_iterator> Users,
+                         bool HasNoNans, const SDLoc &dl,
                          SelectionDAG &DAG) const;
   SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
@@ -825,6 +828,9 @@ class AArch64TargetLowering : public TargetLowering {
   void ReplaceExtractSubVectorResults(SDNode *N,
                                       SmallVectorImpl<SDValue> &Results,
                                       SelectionDAG &DAG) const;
+  void ReplaceGetActiveLaneMaskResults(SDNode *N,
+                                       SmallVectorImpl<SDValue> &Results,
+                                       SelectionDAG &DAG) const;
 
   bool shouldNormalizeToSelectSequence(LLVMContext &, EVT) const override;
 
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 6adf84879052..9078748c1483 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -282,7 +282,7 @@ def CondCode : AsmOperandClass {
   let DiagnosticType = "InvalidCondCode";
 }
 
-// A 32-bit register pasrsed as 64-bit
+// A 32-bit register parsed as 64-bit
 def GPR32as64Operand : AsmOperandClass {
   let Name = "GPR32as64";
   let ParserMethod =
@@ -292,7 +292,7 @@ def GPR32as64 : RegisterOperand<GPR32> {
   let ParserMatchClass = GPR32as64Operand;
 }
 
-// A 64-bit register pasrsed as 32-bit
+// A 64-bit register parsed as 32-bit
 def GPR64as32Operand : AsmOperandClass {
   let Name = "GPR64as32";
   let ParserMethod =
@@ -580,7 +580,7 @@ def uimm5s8 : Operand<i64>, ImmLeaf<i64,
   let PrintMethod = "printImmScale<8>";
 }
 
-// tuimm5sN predicate - similiar to uimm5sN, but use TImmLeaf (TargetConstant)
+// tuimm5sN predicate - similar to uimm5sN, but use TImmLeaf (TargetConstant)
 // instead of ImmLeaf (Constant)
 def tuimm5s2 : Operand<i64>, TImmLeaf<i64,
                 [{ return Imm >= 0 && Imm < (32*2) && ((Imm % 2) == 0); }],
@@ -1817,7 +1817,6 @@ class APASI : SimpleSystemI<0, (ins GPR64:$Xt), "apas", "\t$Xt">, Sched<[]> {
   bits<5> Xt;
   let Inst{20-5} = 0b0111001110000000;
   let Inst{4-0} = Xt;
-  let DecoderNamespace = "APAS";
 }
 
 // Hint instructions that take both a CRm and a 3-bit immediate.
@@ -3777,7 +3776,7 @@ multiclass StoreUI<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype,
 // Same as StoreUI, but take a RegisterOperand. This is used by GlobalISel to
 // substitute zero-registers automatically.
 //
-// TODO: Roll out zero-register subtitution to GPR32/GPR64 and fold this back
+// TODO: Roll out zero-register substitution to GPR32/GPR64 and fold this back
 //       into StoreUI.
 multiclass StoreUIz<bits<2> sz, bit V, bits<2> opc, RegisterOperand regtype,
              Operand indextype, string asm, list<dag> pattern> {
@@ -7764,9 +7763,9 @@ multiclass SIMDThreeScalarD<bit U, bits<5> opc, string asm,
 }
 
 multiclass SIMDThreeScalarBHSD<bit U, bits<5> opc, string asm,
-                               SDPatternOperator OpNode> {
+                               SDPatternOperator OpNode, SDPatternOperator SatOp> {
   def v1i64  : BaseSIMDThreeScalar<U, 0b111, opc, FPR64, asm,
-    [(set (v1i64 FPR64:$Rd), (OpNode (v1i64 FPR64:$Rn), (v1i64 FPR64:$Rm)))]>;
+    [(set (v1i64 FPR64:$Rd), (SatOp (v1i64 FPR64:$Rn), (v1i64 FPR64:$Rm)))]>;
   def v1i32  : BaseSIMDThreeScalar<U, 0b101, opc, FPR32, asm, []>;
   def v1i16  : BaseSIMDThreeScalar<U, 0b011, opc, FPR16, asm, []>;
   def v1i8   : BaseSIMDThreeScalar<U, 0b001, opc, FPR8 , asm, []>;
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index d1e0d37e33e4..951cb93ea8f8 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -1176,6 +1176,9 @@ bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
     case AArch64::SEH_PACSignLR:
     case AArch64::SEH_SaveAnyRegQP:
     case AArch64::SEH_SaveAnyRegQPX:
+    case AArch64::SEH_AllocZ:
+    case AArch64::SEH_SaveZReg:
+    case AArch64::SEH_SavePReg:
       return true;
   }
 }
@@ -1278,8 +1281,9 @@ bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
                                       int64_t &CmpValue) const {
   // The first operand can be a frame index where we'd normally expect a
   // register.
+  // FIXME: Pass subregisters out of analyzeCompare
   assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
-  if (!MI.getOperand(1).isReg())
+  if (!MI.getOperand(1).isReg() || MI.getOperand(1).getSubReg())
     return false;
 
   switch (MI.getOpcode()) {
@@ -1289,6 +1293,9 @@ bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
   case AArch64::PTEST_PP_ANY:
     SrcReg = MI.getOperand(0).getReg();
     SrcReg2 = MI.getOperand(1).getReg();
+    if (MI.getOperand(2).getSubReg())
+      return false;
+
     // Not sure about the mask and value for now...
     CmpMask = ~0;
     CmpValue = 0;
@@ -1308,6 +1315,11 @@ bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
     // Replace SUBSWrr with SUBWrr if NZCV is not used.
     SrcReg = MI.getOperand(1).getReg();
     SrcReg2 = MI.getOperand(2).getReg();
+
+    // FIXME: Pass subregisters out of analyzeCompare
+    if (MI.getOperand(2).getSubReg())
+      return false;
+
     CmpMask = ~0;
     CmpValue = 0;
     return true;
@@ -3229,6 +3241,8 @@ bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
           ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
     if (MemI.getOperand(2).getImm() != 0)
       return false;
+    if ((unsigned)Scale != Scale)
+      return false;
     if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
       return false;
     AM.BaseReg = AddrI.getOperand(1).getReg();
@@ -5988,10 +6002,16 @@ static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
     }
 
     if (NeedsWinCFI) {
-      assert(Sign == 1 && "SEH directives should always have a positive sign");
       int Imm = (int)(ThisVal << LocalShiftSize);
-      if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
-          (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
+      if (VScale != 1 && DestReg == AArch64::SP) {
+        if (HasWinCFI)
+          *HasWinCFI = true;
+        BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AllocZ))
+            .addImm(ThisVal)
+            .setMIFlag(Flag);
+      } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
+                 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
+        assert(VScale == 1 && "Expected non-scalable operation");
         if (HasWinCFI)
           *HasWinCFI = true;
         if (Imm == 0)
@@ -6003,6 +6023,7 @@ static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
         assert(Offset == 0 && "Expected remaining offset to be zero to "
                               "emit a single SEH directive");
       } else if (DestReg == AArch64::SP) {
+        assert(VScale == 1 && "Expected non-scalable operation");
         if (HasWinCFI)
           *HasWinCFI = true;
         assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
@@ -6057,14 +6078,14 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB,
 
   assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
          "SetNZCV not supported with SVE vectors");
-  assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
-         "WinCFI not supported with SVE vectors");
+  assert(!(NeedsWinCFI && NumPredicateVectors) &&
+         "WinCFI can't allocate fractions of an SVE data vector");
 
   if (NumDataVectors) {
     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumDataVectors,
-                       UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
-                       TII, Flag, NeedsWinCFI, nullptr, EmitCFAOffset,
-                       CFAOffset, FrameReg);
+                       UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
+                       Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
+                       FrameReg);
     CFAOffset += StackOffset::getScalable(-NumDataVectors * 16);
     SrcReg = DestReg;
   }
@@ -6072,9 +6093,9 @@ void llvm::emitFrameOffset(MachineBasicBlock &MBB,
   if (NumPredicateVectors) {
     assert(DestReg != AArch64::SP && "Unaligned access to SP");
     emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumPredicateVectors,
-                       UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
-                       TII, Flag, NeedsWinCFI, nullptr, EmitCFAOffset,
-                       CFAOffset, FrameReg);
+                       UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
+                       Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
+                       FrameReg);
   }
 }
 
@@ -8754,7 +8775,7 @@ bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
     return false;
 
   // Find Definition.
-  assert(MI.getParent() && "Incomplete machine instruciton\n");
+  assert(MI.getParent() && "Incomplete machine instruction\n");
   MachineBasicBlock *MBB = MI.getParent();
   MachineFunction *MF = MBB->getParent();
   MachineRegisterInfo *MRI = &MF->getRegInfo();
@@ -9056,7 +9077,7 @@ AArch64InstrInfo::getOutliningCandidateInfo(
   // address signing attributes, i.e., all share the same value for the
   // attribute "sign-return-address" and all share the same type of key they
   // are signed with.
-  // Additionally we require all functions to simultaniously either support
+  // Additionally we require all functions to simultaneously either support
   // v8.3a features or not. Otherwise an outlined function could get signed
   // using dedicated v8.3 instructions and a call from a function that doesn't
   // support v8.3 instructions would therefore be invalid.
@@ -10298,7 +10319,7 @@ unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
 MachineBasicBlock::iterator
 AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
                                    Register TargetReg, bool FrameSetup) const {
-  assert(TargetReg != AArch64::SP && "New top of stack cannot aleady be in SP");
+  assert(TargetReg != AArch64::SP && "New top of stack cannot already be in SP");
 
   MachineBasicBlock &MBB = *MBBI->getParent();
   MachineFunction &MF = *MBB.getParent();
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 0ffaca9af400..7c255da333e4 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -519,7 +519,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   /// Returns true if the instruction has a shift by immediate that can be
   /// executed in one cycle less.
   static bool isFalkorShiftExtFast(const MachineInstr &MI);
-  /// Return true if the instructions is a SEH instruciton used for unwinding
+  /// Return true if the instructions is a SEH instruction used for unwinding
   /// on Windows.
   static bool isSEHInstruction(const MachineInstr &MI);
 
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 96d0146c1e75..f90f12b5ac3c 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -685,23 +685,35 @@ defm trunc_masked_scatter_i32 : masked_gather_scatter<trunc_masked_scatter_i32>;
 def top16Zero: PatLeaf<(i32 GPR32:$src), [{
   return Op.getValueType() == MVT::i32 &&
          CurDAG->MaskedValueIsZero(Op, APInt::getHighBitsSet(32, 16));
-  }]>;
+  }]> {
+  let GISelLeafPredicateCode = [{
+    return VT && VT->maskedValueIsZero(Reg, APInt::getHighBitsSet(32, 16)); }];
+}
 
 // top32Zero - answer true if the upper 32 bits of $src are 0, false otherwise
 def top32Zero: PatLeaf<(i64 GPR64:$src), [{
   return Op.getValueType() == MVT::i64 &&
          CurDAG->MaskedValueIsZero(Op, APInt::getHighBitsSet(64, 32));
-  }]>;
+  }]> {
+  let GISelLeafPredicateCode = [{ 
+    return VT && VT->maskedValueIsZero(Reg, APInt::getHighBitsSet(64, 32)); }];
+}
 
 // topbitsallzero - Return true if all bits except the lowest bit are known zero
 def topbitsallzero32: PatLeaf<(i32 GPR32:$src), [{
   return Op.getValueType() == MVT::i32 &&
          CurDAG->MaskedValueIsZero(Op, APInt::getHighBitsSet(32, 31));
-  }]>;
+  }]> {
+  let GISelLeafPredicateCode = [{ 
+    return VT && VT->maskedValueIsZero(Reg, APInt::getHighBitsSet(32, 31)); }];
+}
 def topbitsallzero64: PatLeaf<(i64 GPR64:$src), [{
   return Op.getValueType() == MVT::i64 &&
          CurDAG->MaskedValueIsZero(Op, APInt::getHighBitsSet(64, 63));
-  }]>;
+  }]> {
+  let GISelLeafPredicateCode = [{ 
+    return VT && VT->maskedValueIsZero(Reg, APInt::getHighBitsSet(64, 63)); }];
+}
 
 // Node definitions.
 // Compare-and-branch
@@ -853,7 +865,7 @@ def AArch64uzp2      : SDNode<"AArch64ISD::UZP2", SDT_AArch64Zip>;
 def AArch64trn1      : SDNode<"AArch64ISD::TRN1", SDT_AArch64Zip>;
 def AArch64trn2      : SDNode<"AArch64ISD::TRN2", SDT_AArch64Zip>;
 
-// Vector immedate moves
+// Vector immediate moves
 def AArch64movi_edit : SDNode<"AArch64ISD::MOVIedit", SDT_AArch64MOVIedit>;
 def AArch64movi_shift : SDNode<"AArch64ISD::MOVIshift", SDT_AArch64MOVIshift>;
 def AArch64movi_msl : SDNode<"AArch64ISD::MOVImsl", SDT_AArch64MOVIshift>;
@@ -1475,7 +1487,7 @@ let Predicates = [HasPCDPHINT] in {
 }
 
 // In order to be able to write readable assembly, LLVM should accept assembly
-// inputs that use Branch Target Indentification mnemonics, even with BTI disabled.
+// inputs that use Branch Target Identification mnemonics, even with BTI disabled.
 // However, in order to be compatible with other assemblers (e.g. GAS), LLVM
 // should not emit these mnemonics unless BTI is enabled.
 def : InstAlias<"bti",  (HINT 32), 0>;
@@ -1699,6 +1711,11 @@ multiclass SIMDSUDOTIndex {
 
 defm SUDOTlane : SIMDSUDOTIndex;
 
+def : Pat<(v2i32 (partial_reduce_sumla v2i32:$Acc, v8i8:$LHS, v8i8:$RHS)),
+              (USDOTv8i8 $Acc, $RHS, $LHS)>;
+def : Pat<(v4i32 (partial_reduce_sumla v4i32:$Acc, v16i8:$LHS, v16i8:$RHS)),
+              (USDOTv16i8 $Acc, $RHS, $LHS)>;
+
 }
 
 // ARMv8.2-A FP16 Fused Multiply-Add Long
@@ -5572,6 +5589,9 @@ let isPseudo = 1 in {
   def SEH_PACSignLR : Pseudo<(outs), (ins), []>, Sched<[]>;
   def SEH_SaveAnyRegQP : Pseudo<(outs), (ins i32imm:$reg0, i32imm:$reg1, i32imm:$offs), []>, Sched<[]>;
   def SEH_SaveAnyRegQPX : Pseudo<(outs), (ins i32imm:$reg0, i32imm:$reg1, i32imm:$offs), []>, Sched<[]>;
+  def SEH_AllocZ : Pseudo<(outs), (ins i32imm:$offs), []>, Sched<[]>;
+  def SEH_SaveZReg : Pseudo<(outs), (ins i32imm:$reg, i32imm:$offs), []>, Sched<[]>;
+  def SEH_SavePReg : Pseudo<(outs), (ins i32imm:$reg, i32imm:$offs), []>, Sched<[]>;
 }
 
 // Pseudo instructions for Windows EH
@@ -6336,19 +6356,19 @@ defm FCMGT    : SIMDThreeScalarFPCmp<1, 1, 0b100, "fcmgt", AArch64fcmgt>;
 defm FMULX    : SIMDFPThreeScalar<0, 0, 0b011, "fmulx", int_aarch64_neon_fmulx, HasNEONandIsStreamingSafe>;
 defm FRECPS   : SIMDFPThreeScalar<0, 0, 0b111, "frecps", int_aarch64_neon_frecps, HasNEONandIsStreamingSafe>;
 defm FRSQRTS  : SIMDFPThreeScalar<0, 1, 0b111, "frsqrts", int_aarch64_neon_frsqrts, HasNEONandIsStreamingSafe>;
-defm SQADD    : SIMDThreeScalarBHSD<0, 0b00001, "sqadd", int_aarch64_neon_sqadd>;
+defm SQADD    : SIMDThreeScalarBHSD<0, 0b00001, "sqadd", int_aarch64_neon_sqadd, saddsat>;
 defm SQDMULH  : SIMDThreeScalarHS<  0, 0b10110, "sqdmulh", int_aarch64_neon_sqdmulh>;
 defm SQRDMULH : SIMDThreeScalarHS<  1, 0b10110, "sqrdmulh", int_aarch64_neon_sqrdmulh>;
-defm SQRSHL   : SIMDThreeScalarBHSD<0, 0b01011, "sqrshl",int_aarch64_neon_sqrshl>;
-defm SQSHL    : SIMDThreeScalarBHSD<0, 0b01001, "sqshl", int_aarch64_neon_sqshl>;
-defm SQSUB    : SIMDThreeScalarBHSD<0, 0b00101, "sqsub", int_aarch64_neon_sqsub>;
+defm SQRSHL   : SIMDThreeScalarBHSD<0, 0b01011, "sqrshl", int_aarch64_neon_sqrshl, int_aarch64_neon_sqrshl>;
+defm SQSHL    : SIMDThreeScalarBHSD<0, 0b01001, "sqshl", int_aarch64_neon_sqshl, int_aarch64_neon_sqshl>;
+defm SQSUB    : SIMDThreeScalarBHSD<0, 0b00101, "sqsub", int_aarch64_neon_sqsub, ssubsat>;
 defm SRSHL    : SIMDThreeScalarD<   0, 0b01010, "srshl", int_aarch64_neon_srshl>;
 defm SSHL     : SIMDThreeScalarD<   0, 0b01000, "sshl", int_aarch64_neon_sshl>;
 defm SUB      : SIMDThreeScalarD<   1, 0b10000, "sub", sub>;
-defm UQADD    : SIMDThreeScalarBHSD<1, 0b00001, "uqadd", int_aarch64_neon_uqadd>;
-defm UQRSHL   : SIMDThreeScalarBHSD<1, 0b01011, "uqrshl",int_aarch64_neon_uqrshl>;
-defm UQSHL    : SIMDThreeScalarBHSD<1, 0b01001, "uqshl", int_aarch64_neon_uqshl>;
-defm UQSUB    : SIMDThreeScalarBHSD<1, 0b00101, "uqsub", int_aarch64_neon_uqsub>;
+defm UQADD    : SIMDThreeScalarBHSD<1, 0b00001, "uqadd", int_aarch64_neon_uqadd, uaddsat>;
+defm UQRSHL   : SIMDThreeScalarBHSD<1, 0b01011, "uqrshl", int_aarch64_neon_uqrshl, int_aarch64_neon_uqrshl>;
+defm UQSHL    : SIMDThreeScalarBHSD<1, 0b01001, "uqshl", int_aarch64_neon_uqshl, int_aarch64_neon_uqshl>;
+defm UQSUB    : SIMDThreeScalarBHSD<1, 0b00101, "uqsub", int_aarch64_neon_uqsub, usubsat>;
 defm URSHL    : SIMDThreeScalarD<   1, 0b01010, "urshl", int_aarch64_neon_urshl>;
 defm USHL     : SIMDThreeScalarD<   1, 0b01000, "ushl", int_aarch64_neon_ushl>;
 let Predicates = [HasRDM] in {
@@ -7153,6 +7173,16 @@ def : Pat<(v8bf16 (AArch64dup (bf16 FPR16:$Rn))),
             (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)), FPR16:$Rn, hsub),
             (i64 0)))>;
 
+// Patterns for importing fpr i8 G_DUP under GISel.
+def : Pat<(v8i8 (AArch64dup (i8 FPR8:$Rn))),
+          (v8i8 (DUPv8i8lane
+            (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rn, bsub),
+            (i64 0)))>;
+def : Pat<(v16i8 (AArch64dup (i8 FPR8:$Rn))),
+          (v16i8 (DUPv16i8lane
+            (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rn, bsub),
+            (i64 0)))>;
+
 def : Pat<(v4f16 (AArch64duplane16 (v8f16 V128:$Rn), VectorIndexH:$imm)),
           (DUPv4i16lane V128:$Rn, VectorIndexH:$imm)>;
 def : Pat<(v8f16 (AArch64duplane16 (v8f16 V128:$Rn), VectorIndexH:$imm)),
@@ -7266,6 +7296,13 @@ def : Pat<(v4i16 (vec_ins_or_scal_vec GPR32:$Rn)),
           (SUBREG_TO_REG (i32 0),
                          (f32 (COPY_TO_REGCLASS GPR32:$Rn, FPR32)), ssub)>;
 
+def : Pat<(v2i32 (vec_ins_or_scal_vec GPR32:$Rn)),
+          (INSERT_SUBREG (v2i32 (IMPLICIT_DEF)), GPR32:$Rn, ssub)>;
+def : Pat<(v4i32 (vec_ins_or_scal_vec GPR32:$Rn)),
+          (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), GPR32:$Rn, ssub)>;
+def : Pat<(v2i64 (vec_ins_or_scal_vec GPR64:$Rn)),
+          (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), GPR64:$Rn, dsub)>;
+
 def : Pat<(v4f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
           (INSERT_SUBREG (v4f16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
 def : Pat<(v8f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
@@ -7276,16 +7313,6 @@ def : Pat<(v4bf16 (vec_ins_or_scal_vec (bf16 FPR16:$Rn))),
 def : Pat<(v8bf16 (vec_ins_or_scal_vec (bf16 FPR16:$Rn))),
           (INSERT_SUBREG (v8bf16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
 
-def : Pat<(v2i32 (vec_ins_or_scal_vec (i32 FPR32:$Rn))),
-            (v2i32 (INSERT_SUBREG (v2i32 (IMPLICIT_DEF)),
-                                  (i32 FPR32:$Rn), ssub))>;
-def : Pat<(v4i32 (vec_ins_or_scal_vec (i32 FPR32:$Rn))),
-            (v4i32 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)),
-                                  (i32 FPR32:$Rn), ssub))>;
-def : Pat<(v2i64 (vec_ins_or_scal_vec (i64 FPR64:$Rn))),
-            (v2i64 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
-                                  (i64 FPR64:$Rn), dsub))>;
-
 def : Pat<(v4f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
           (INSERT_SUBREG (v4f16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
 def : Pat<(v8f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index b7da07a95c7b..f51f0d11ef9d 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -10,7 +10,7 @@
 // optimizations. This pass should be run after register allocation.
 //
 // The pass runs after the PrologEpilogInserter where we emit the CFI
-// instructions. In order to preserve the correctness of the unwind informaiton,
+// instructions. In order to preserve the correctness of the unwind information,
 // the pass should not change the order of any two instructions, one of which
 // has the FrameSetup/FrameDestroy flag or, alternatively, apply an add-hoc fix
 // to unwind information.
@@ -189,7 +189,7 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   // pre or post indexed addressing with writeback. Scan backwards.
   // `MergeEither` is set to true if the combined instruction may be placed
   // either at the location of the load/store instruction or at the location of
-  // the update intruction.
+  // the update instruction.
   MachineBasicBlock::iterator
   findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit,
                                  bool &MergeEither);
@@ -1281,7 +1281,7 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
     // instruction contains the final value we care about we give it a new
     // debug-instr-number 3. Whereas, $w1 contains the final value that we care
     // about, therefore the LDP instruction is also given a new
-    // debug-instr-number 4. We have to add these subsitutions to the
+    // debug-instr-number 4. We have to add these substitutions to the
     // debugValueSubstitutions table. However, we also have to ensure that the
     // OpIndex that pointed to debug-instr-number 1 gets updated to 1, because
     // $w1 is the second operand of the LDP instruction.
@@ -2602,7 +2602,7 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
   ModifiedRegUnits.clear();
   UsedRegUnits.clear();
   unsigned Count = 0;
-  bool MemAcessBeforeSPPreInc = false;
+  bool MemAccessBeforeSPPreInc = false;
   MergeEither = true;
   do {
     MBBI = prev_nodbg(MBBI, B);
@@ -2617,7 +2617,7 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
     if (isMatchingUpdateInsn(*I, MI, BaseReg, Offset)) {
       // Check that the update value is within our red zone limit (which may be
       // zero).
-      if (MemAcessBeforeSPPreInc && MBBI->getOperand(2).getImm() > RedZoneSize)
+      if (MemAccessBeforeSPPreInc && MBBI->getOperand(2).getImm() > RedZoneSize)
         return E;
       return MBBI;
     }
@@ -2648,7 +2648,7 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
     // case we need to validate later that the update amount respects the red
     // zone.
     if (BaseRegSP && MBBI->mayLoadOrStore())
-      MemAcessBeforeSPPreInc = true;
+      MemAccessBeforeSPPreInc = true;
   } while (MBBI != B && Count < Limit);
   return E;
 }
@@ -2745,7 +2745,7 @@ bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
   if (!TII->isCandidateToMergeOrPair(MI))
     return false;
 
-  // Look ahead up to LdStLimit instructions for a mergable instruction.
+  // Look ahead up to LdStLimit instructions for a mergeable instruction.
   LdStPairFlags Flags;
   MachineBasicBlock::iterator MergeMI =
       findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ true);
@@ -2941,7 +2941,7 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
   AArch64FunctionInfo &AFI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
 
   bool Modified = false;
-  // Four tranformations to do here:
+  // Four transformations to do here:
   // 1) Find loads that directly read from stores and promote them by
   //    replacing with mov instructions. If the store is wider than the load,
   //    the load will be replaced with a bitfield extract.
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
index 66f14b67a31f..d67182d5bb51 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
@@ -402,7 +402,7 @@ static bool shouldUseFrameHelper(MachineBasicBlock &MBB,
     InstCount--;
     break;
   case FrameHelperType::PrologFrame: {
-    // Effecitvely no change in InstCount since FpAdjusment is included.
+    // Effectively no change in InstCount since FpAdjustment is included.
     break;
   }
   case FrameHelperType::Epilog:
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp
index 5bcff61cef4b..4b04b80121ff 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp
@@ -100,6 +100,9 @@ AArch64FunctionInfo::AArch64FunctionInfo(const Function &F,
   BranchTargetEnforcement = F.hasFnAttribute("branch-target-enforcement");
   BranchProtectionPAuthLR = F.hasFnAttribute("branch-protection-pauth-lr");
 
+  // Parse the SME function attributes.
+  SMEFnAttrs = SMEAttrs(F);
+
   // The default stack probe size is 4096 if the function has no
   // stack-probe-size attribute. This is a safe default because it is the
   // smallest possible guard page size.
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index d3026ca45c34..361d5ec3f2b2 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -14,6 +14,7 @@
 #define LLVM_LIB_TARGET_AARCH64_AARCH64MACHINEFUNCTIONINFO_H
 
 #include "AArch64Subtarget.h"
+#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
@@ -245,6 +246,9 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
   int64_t VGIdx = std::numeric_limits<int>::max();
   int64_t StreamingVGIdx = std::numeric_limits<int>::max();
 
+  // Holds the SME function attributes (streaming mode, ZA/ZT0 state).
+  SMEAttrs SMEFnAttrs;
+
 public:
   AArch64FunctionInfo(const Function &F, const AArch64Subtarget *STI);
 
@@ -449,6 +453,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
     StackHazardCSRSlotIndex = Index;
   }
 
+  SMEAttrs getSMEFnAttrs() const { return SMEFnAttrs; }
+
   unsigned getSRetReturnReg() const { return SRetReturnReg; }
   void setSRetReturnReg(unsigned Reg) { SRetReturnReg = Reg; }
 
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64Processors.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64Processors.td
index a5f0f6a2eb15..c7ea6393e2ad 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -302,7 +302,7 @@ def TuneOlympus : SubtargetFeature<"olympus", "ARMProcFamily", "Olympus",
                                    FeatureUseFixedOverScalableIfEqualCost]>;
 
 // Note that cyclone does not fuse AES instructions, but newer apple chips do
-// perform the fusion and cyclone is used by default when targetting apple OSes.
+// perform the fusion and cyclone is used by default when targeting apple OSes.
 def TuneAppleA7  : SubtargetFeature<"apple-a7", "ARMProcFamily", "AppleA7",
                                     "Apple A7 (the CPU formerly known as Cyclone)", [
                                     FeatureAlternateSExtLoadCVTF32Pattern,
@@ -561,7 +561,8 @@ def TuneNeoverseV2 : SubtargetFeature<"neoversev2", "ARMProcFamily", "NeoverseV2
                                       FeatureEnableSelectOptimize,
                                       FeatureUseFixedOverScalableIfEqualCost,
                                       FeatureAvoidLDAPUR,
-                                      FeaturePredictableSelectIsExpensive]>;
+                                      FeaturePredictableSelectIsExpensive,
+                                      FeatureDisableLatencySchedHeuristic]>;
 
 def TuneNeoverseV3 : SubtargetFeature<"neoversev3", "ARMProcFamily", "NeoverseV3",
                                       "Neoverse V3 ARM processors", [
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 8caa49de0af4..fb472ddc719f 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -18,6 +18,7 @@
 #include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64InstPrinter.h"
+#include "Utils/AArch64SMEAttributes.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/LiveRegMatrix.h"
@@ -98,6 +99,13 @@ AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
       return CSR_Win_AArch64_AAPCS_SwiftError_SaveList;
     if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
       return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList;
+    if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
+      return CSR_Win_AArch64_AAVPCS_SaveList;
+    if (MF->getFunction().getCallingConv() ==
+        CallingConv::AArch64_SVE_VectorCall)
+      return CSR_Win_AArch64_SVE_AAPCS_SaveList;
+    if (MF->getInfo<AArch64FunctionInfo>()->isSVECC())
+      return CSR_Win_AArch64_SVE_AAPCS_SaveList;
     return CSR_Win_AArch64_AAPCS_SaveList;
   }
   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
@@ -632,14 +640,26 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
       return true;
 
     auto &ST = MF.getSubtarget<AArch64Subtarget>();
+    const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     if (ST.hasSVE() || ST.isStreaming()) {
-      const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
       // Frames that have variable sized objects and scalable SVE objects,
       // should always use a basepointer.
       if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE())
         return true;
     }
 
+    // Frames with hazard padding can have a large offset between the frame
+    // pointer and GPR locals, which includes the emergency spill slot. If the
+    // emergency spill slot is not within range of the load/store instructions
+    // (which have a signed 9-bit range), we will fail to compile if it is used.
+    // Since hasBasePointer() is called before we know if we have hazard padding
+    // or an emergency spill slot we need to enable the basepointer
+    // conservatively.
+    if (ST.getStreamingHazardSize() &&
+        !AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody()) {
+      return true;
+    }
+
     // Conservatively estimate whether the negative offset from the frame
     // pointer will be sufficient to reach. If a function has a smallish
     // frame, it's less likely to have lots of spills and callee saved
@@ -764,7 +784,8 @@ AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
   assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||
           AFI->hasCalculatedStackSizeSVE()) &&
          "Expected SVE area to be calculated by this point");
-  return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE();
+  return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE() &&
+         !AFI->hasStackHazardSlotIndex();
 }
 
 bool AArch64RegisterInfo::requiresFrameIndexScavenging(
@@ -1197,7 +1218,7 @@ bool AArch64RegisterInfo::getRegAllocationHints(
       //   is valid but { z1, z2, z3, z5 } is not.
       // * One or more of the registers used by FORM_TRANSPOSED_X4 is already
       //   assigned a physical register, which means only checking that a
-      //   consectutive range of free tuple registers exists which includes
+      //   consecutive range of free tuple registers exists which includes
       //   the assigned register.
       //   e.g. in the example above, if { z0, z8 } is already allocated for
       //   %v0, we just need to ensure that { z1, z9 }, { z2, z10 } and
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index d3252ea54321..61bf87fe7144 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -1849,7 +1849,7 @@ def ZTR : RegisterClass<"AArch64", [untyped], 512, (add ZT0)> {
 // * Tile vectors:
 //
 //   Their representation is similar to regular tiles, but they have an extra
-//   'h' or 'v' to tell how the vector at [reg+offset] is layed out in the tile,
+//   'h' or 'v' to tell how the vector at [reg+offset] is laid out in the tile,
 //   horizontally or vertically.
 //
 //   e.g. za1h.h or za15v.q, which corresponds to vectors in registers ZAH1 and
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp
index b3159b444e5b..d695f2678fd4 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp
@@ -147,7 +147,7 @@ struct AArch64SIMDInstrOpt : public MachineFunctionPass {
   };
 
   // A costly instruction is replaced in this work by N efficient instructions
-  // The maximum of N is curently 10 and it is for ST4 case.
+  // The maximum of N is currently 10 and it is for ST4 case.
   static const unsigned MaxNumRepl = 10;
 
   AArch64SIMDInstrOpt() : MachineFunctionPass(ID) {}
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
index 91410a5af3dc..0dc57f735388 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp
@@ -449,7 +449,7 @@ void SLSHardeningInserter::convertBLRToBL(
 
   // Now copy the implicit operands from BLR to BL and copy other necessary
   // info.
-  // However, both BLR and BL instructions implictly use SP and implicitly
+  // However, both BLR and BL instructions implicitly use SP and implicitly
   // define LR. Blindly copying implicit operands would result in SP and LR
   // operands to be present multiple times. While this may not be too much of
   // an issue, let's avoid that for cleanliness, by removing those implicit
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 91db6b6fc798..2360e30de63b 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4064,8 +4064,10 @@ let Predicates = [HasSVE2_or_SME] in {
 
 let Predicates = [HasSVEAES, HasNonStreamingSVE2_or_SSVE_AES] in {
   // SVE2 crypto destructive binary operations
-  defm AESE_ZZZ_B : sve2_crypto_des_bin_op<0b00, "aese", ZPR8, int_aarch64_sve_aese, nxv16i8>;
-  defm AESD_ZZZ_B : sve2_crypto_des_bin_op<0b01, "aesd", ZPR8, int_aarch64_sve_aesd, nxv16i8>;
+  let isCommutable = 1 in {
+  def AESE_ZZZ_B : sve2_crypto_des_bin_op<0b00, "aese", ZPR8, int_aarch64_sve_aese, nxv16i8>;
+  def AESD_ZZZ_B : sve2_crypto_des_bin_op<0b01, "aesd", ZPR8, int_aarch64_sve_aesd, nxv16i8>;
+  }
 
   // SVE2 crypto unary operations
   defm AESMC_ZZ_B  : sve2_crypto_unary_op<0b0, "aesmc",  int_aarch64_sve_aesmc>;
@@ -4082,7 +4084,7 @@ let Predicates = [HasSVE2SM4] in {
   // SVE2 crypto constructive binary operations
   defm SM4EKEY_ZZZ_S : sve2_crypto_cons_bin_op<0b0, "sm4ekey", ZPR32, int_aarch64_sve_sm4ekey, nxv4i32>;
   // SVE2 crypto destructive binary operations
-  defm SM4E_ZZZ_S : sve2_crypto_des_bin_op<0b10, "sm4e", ZPR32, int_aarch64_sve_sm4e, nxv4i32>;
+  def SM4E_ZZZ_S : sve2_crypto_des_bin_op<0b10, "sm4e", ZPR32, int_aarch64_sve_sm4e, nxv4i32>;
 } // End HasSVE2SM4
 
 let Predicates = [HasSVE2SHA3] in {
@@ -4097,7 +4099,7 @@ let Predicates = [HasSVEBitPerm, HasNonStreamingSVE2_or_SSVE_BitPerm] in {
   defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp", int_aarch64_sve_bgrp_x>;
 }
 
-let Predicates = [HasSVEAES2, HasNonStreamingSVE2p1_or_SSVE_AES] in {
+let Predicates = [HasSVEAES2, HasNonStreamingSVE2_or_SSVE_AES] in {
   // SVE_AES2 multi-vector instructions (x2)
   def AESE_2ZZI_B    : sve_crypto_binary_multi2<0b000, "aese">;
   def AESD_2ZZI_B    : sve_crypto_binary_multi2<0b010, "aesd">;
@@ -4114,6 +4116,11 @@ let Predicates = [HasSVEAES2, HasNonStreamingSVE2p1_or_SSVE_AES] in {
   def PMULL_2ZZZ_Q : sve_crypto_pmull_multi<"pmull">;
 }
 
+let Predicates = [HasSVE_or_SME, HasMatMulInt8] in {
+    def : Pat<(nxv4i32 (partial_reduce_sumla nxv4i32:$Acc, nxv16i8:$LHS, nxv16i8:$RHS)),
+              (USDOT_ZZZ $Acc, $RHS, $LHS)>;
+  } // End HasSVE_or_SME, HasMatMulInt8
+
 //===----------------------------------------------------------------------===//
 // SME or SVE2.1 instructions
 //===----------------------------------------------------------------------===//
@@ -4497,7 +4504,7 @@ let Predicates = [HasNonStreamingSVE2p2_or_SME2p2] in {
   // SVE2 EXPAND
   defm EXPAND_ZPZ : sve2_int_perm_expand<"expand">;
   // SVE COMPACT - byte and halfword
-  defm COMPACT_ZPZ : sve_int_perm_compact_bh<"compact">;
+  defm COMPACT_ZPZ : sve_int_perm_compact_bh<"compact", int_aarch64_sve_compact>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedA53.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedA53.td
index c714bad92b7f..66715b9d1db8 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedA53.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedA53.td
@@ -19,7 +19,7 @@ def CortexA53Model : SchedMachineModel {
   let MicroOpBufferSize = 0; // Explicitly set to zero since A53 is in-order.
   let IssueWidth = 2;        // 2 micro-ops are dispatched per cycle.
   let LoadLatency = 3;       // Optimistic load latency assuming bypass.
-                             // This is overriden by OperandCycles if the
+                             // This is overridden by OperandCycles if the
                              // Itineraries are queried instead.
   let MispredictPenalty = 9; // Based on "Cortex-A53 Software Optimisation
                              // Specification - Instruction Timings"
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
index 39f7077ae451..8d3a4553d4b7 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
@@ -14,7 +14,9 @@
 //===----------------------------------------------------------------------===//
 
 def NeoverseV2Model : SchedMachineModel {
-  let IssueWidth            =  16; // Micro-ops dispatched at a time.
+  let IssueWidth            =   6; // This value comes from the decode bandwidth
+                                   // and empirical measurements showed that a
+                                   // lower value is better.
   let MicroOpBufferSize     = 320; // Entries in micro-op re-order buffer.
   let LoadLatency           =   4; // Optimistic load latency.
   let MispredictPenalty     =  10; // Extra cycles for mispredicted branch.  NOTE: Copied from N2.
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedOryon.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedOryon.td
index 09d1af248f0e..5b597b91e745 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedOryon.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SchedOryon.td
@@ -187,7 +187,7 @@ def ORYONFP2 : ProcResGroup<[ORYONP14FP2]> {
   let BufferSize = 48;
 }
 
-// Reciprocal, Squre root on FP0.
+// Reciprocal, Square root on FP0.
 def ORYONFP0 : ProcResGroup<[ORYONP12FP0]> {
   let BufferSize = 48;
 }
@@ -701,7 +701,7 @@ def : InstRW<[ORYONWrite_1Cyc_I0123],
                        "^CSNEG(W|X)r", "^CSINC(W|X)r")>;
 
 //---
-//Compare Instruciton
+//Compare Instruction
 //---
 
 // We have CCMP, CCMN as LLVM DAG node
@@ -1512,7 +1512,7 @@ def : InstRW<[ORYONWrite_10Cyc_FP3_RC], (instregex "^FSQRTv.*32$")>;
 def : InstRW<[ORYONWrite_13Cyc_FP3_RC], (instregex "^FSQRTv.*64$")>;
 
 //==========
-// SIMD binary elememt arithmetic instructions
+// SIMD binary element arithmetic instructions
 //==========
 
 def : InstRW<[ORYONWrite_4Cyc_FP0123], (instregex "^FMLAv", "^FMLSv")>;
@@ -1568,7 +1568,7 @@ def : InstRW<[ORYONWrite_2Cyc_FP0123], (instregex "^ADDPv", "^FADDPv",
                                                    "^(FMAX|FMIN)(NMP|P)v",
                                                    "^(S|U)(MIN|MAX)Pv")>;
 //==========
-// SIMD dot prodcut instructions
+// SIMD dot product instructions
 //==========
 
 def : InstRW<[ORYONWrite_3Cyc_FP0123], (instregex "^(U|S)DOTv")>;
@@ -1581,7 +1581,7 @@ def : InstRW<[ORYONWrite_2Cyc_FP0123], (instrs TBLv8i8One, TBLv16i8One,
                                                 TBXv8i8One, TBXv16i8One,
                                                 TBLv8i8Two, TBLv16i8Two)>;
 
-// TBL 3-reg/4-reg, 3uops, throughtput=4/3=1.33 latency=4
+// TBL 3-reg/4-reg, 3uops, throughput=4/3=1.33 latency=4
 def : InstRW<[ORYONWrite_4Cyc_FP0123_FP0123_FP0123_RC],
             (instrs TBLv8i8Three, TBLv16i8Three,
                     TBLv8i8Four, TBLv16i8Four)>;
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 0d368b7c280c..90f6fc2ea664 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -11,8 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64SelectionDAGInfo.h"
+#include "AArch64MachineFunctionInfo.h"
 #include "AArch64TargetMachine.h"
-#include "Utils/AArch64SMEAttributes.h"
 
 #define GET_SDNODE_DESC
 #include "AArch64GenSDNodeInfo.inc"
@@ -227,7 +227,8 @@ SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(
     return EmitMOPS(AArch64::MOPSMemoryCopyPseudo, DAG, DL, Chain, Dst, Src,
                     Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
 
-  SMEAttrs Attrs(DAG.getMachineFunction().getFunction());
+  auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
+  SMEAttrs Attrs = AFI->getSMEFnAttrs();
   if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody())
     return EmitStreamingCompatibleMemLibCall(DAG, DL, Chain, Dst, Src, Size,
                                              RTLIB::MEMCPY);
@@ -246,7 +247,8 @@ SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
                     Size, Alignment, isVolatile, DstPtrInfo,
                     MachinePointerInfo{});
 
-  SMEAttrs Attrs(DAG.getMachineFunction().getFunction());
+  auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
+  SMEAttrs Attrs = AFI->getSMEFnAttrs();
   if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody())
     return EmitStreamingCompatibleMemLibCall(DAG, dl, Chain, Dst, Src, Size,
                                              RTLIB::MEMSET);
@@ -264,7 +266,8 @@ SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(
     return EmitMOPS(AArch64::MOPSMemoryMovePseudo, DAG, dl, Chain, Dst, Src,
                     Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
 
-  SMEAttrs Attrs(DAG.getMachineFunction().getFunction());
+  auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
+  SMEAttrs Attrs = AFI->getSMEFnAttrs();
   if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody())
     return EmitStreamingCompatibleMemLibCall(DAG, dl, Chain, Dst, Src, Size,
                                              RTLIB::MEMMOVE);
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp
index 96707f20cd75..a591ba9aceb6 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SpeculationHardening.cpp
@@ -13,7 +13,7 @@
 // register. That taint register can then be used to mask off registers with
 // sensitive data when executing under miss-speculation, a.k.a. "transient
 // execution".
-// This pass is aimed at mitigating against SpectreV1-style vulnarabilities.
+// This pass is aimed at mitigating against SpectreV1-style vulnerabilities.
 //
 // It also implements speculative load hardening, i.e. using the taint register
 // to automatically mask off loaded data.
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h b/external/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
index f5ffc72cae53..f95b0fafc607 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/RegisterBankInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/TargetParser/Triple.h"
 
 #define GET_SUBTARGETINFO_HEADER
 #include "AArch64GenSubtargetInfo.inc"
@@ -33,7 +34,6 @@
 namespace llvm {
 class GlobalValue;
 class StringRef;
-class Triple;
 
 class AArch64Subtarget final : public AArch64GenSubtargetInfo {
 public:
@@ -406,7 +406,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   }
 
   // Return the known bit length of SVE data registers. A value of 0 means the
-  // length is unkown beyond what's implied by the architecture.
+  // length is unknown beyond what's implied by the architecture.
   unsigned getSVEVectorSizeInBits() const {
     assert(isSVEorStreamingSVEAvailable() &&
            "Tried to get SVE vector length without SVE support!");
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SystemOperands.td
index 8f6c593d3e68..1f3d619f6dd8 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -941,7 +941,7 @@ defm : TLBI<"RVAE3OS",      0b110, 0b1000, 0b0101, 0b001>;
 defm : TLBI<"RVALE3OS",     0b110, 0b1000, 0b0101, 0b101>;
 } //FeatureTLB_RMI
 
-// Armv9-A Realm Management Extention TLBI Instructions
+// Armv9-A Realm Management Extension TLBI Instructions
 let Requires = ["AArch64::FeatureRME"] in {
 defm : TLBI<"RPAOS",        0b110, 0b1000, 0b0100, 0b011>;
 defm : TLBI<"RPALOS",       0b110, 0b1000, 0b0100, 0b111>;
@@ -1696,7 +1696,7 @@ def : RWSysReg<"PRLAR_EL2",        0b11, 0b100, 0b0110, 0b1000, 0b001>;
 
 foreach n = 1-15 in {
 foreach x = 1-2 in {
-//Direct acces to Protection Region Base Address Register for n th MPU region
+//Direct access to Protection Region Base Address Register for n th MPU region
   def : RWSysReg<!strconcat("PRBAR"#n, "_EL"#x),
     0b11, 0b000, 0b0110, 0b1000, 0b000>{
     let Encoding{5-2} = n;
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 5693a551e679..50f52cca6c8a 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -487,7 +487,7 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
 ScheduleDAGInstrs *
 AArch64TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
   const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
-  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  ScheduleDAGMILive *DAG = createSchedLive(C);
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
   if (ST.hasFusion())
@@ -498,9 +498,7 @@ AArch64TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
 ScheduleDAGInstrs *
 AArch64TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
   const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
-  ScheduleDAGMI *DAG =
-      new ScheduleDAGMI(C, std::make_unique<AArch64PostRASchedStrategy>(C),
-                        /* RemoveKillFlags=*/true);
+  ScheduleDAGMI *DAG = createSchedPostRA<AArch64PostRASchedStrategy>(C);
   if (ST.hasFusion()) {
     // Run the Macro Fusion after RA again since literals are expanded from
     // pseudos then (v. addPreSched2()).
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 68aec80f07e1..0232ac421aed 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1167,7 +1167,7 @@ struct SVEIntrinsicInfo {
   }
 
   // NOTE: Whilst not limited to only inactive lanes, the common use case is:
-  // inactiveLanesAreZerod =
+  // inactiveLanesAreZeroed =
   //     resultIsZeroInitialized() && inactiveLanesAreUnused()
   bool resultIsZeroInitialized() const { return ResultIsZeroInitialized; }
 
@@ -2051,10 +2051,10 @@ instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
   const auto Pattern = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue();
 
   if (Pattern == AArch64SVEPredPattern::all) {
-    Constant *StepVal = ConstantInt::get(II.getType(), NumElts);
-    auto *VScale = IC.Builder.CreateVScale(StepVal);
-    VScale->takeName(&II);
-    return IC.replaceInstUsesWith(II, VScale);
+    Value *Cnt = IC.Builder.CreateElementCount(
+        II.getType(), ElementCount::getScalable(NumElts));
+    Cnt->takeName(&II);
+    return IC.replaceInstUsesWith(II, Cnt);
   }
 
   unsigned MinNumElts = getNumElementsFromSVEPredPattern(Pattern);
@@ -3958,7 +3958,7 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
     scalarization of the division operation.
     2. Constant divisors, either negative in whole or partially, don't result in
     significantly different codegen as compared to positive constant divisors.
-    So, we don't consider negative divisors seperately.
+    So, we don't consider negative divisors separately.
     3. If the codegen is significantly different with SVE, it has been indicated
     using comments at appropriate places.
 
@@ -3980,7 +3980,7 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
 
     other sdiv/srem cases:
     -------------------------------------------------------------------------
-    commom codegen            | + srem     | + sdiv     | pow-of-2  | Type
+    common codegen            | + srem     | + sdiv     | pow-of-2  | Type
     -------------------------------------------------------------------------
     smulh + asr + add + add   | -          | -          | N         | i64
     smull + lsr + add + add   | -          | -          | N         | i32
@@ -4005,7 +4005,8 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
       // have similar cost.
       auto VT = TLI->getValueType(DL, Ty);
       if (VT.isScalarInteger() && VT.getSizeInBits() <= 64) {
-        if (Op2Info.isPowerOf2()) {
+        if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
+          // Neg can be folded into the asr instruction.
           return ISD == ISD::SDIV ? (3 * AddCost + AsrCost)
                                   : (3 * AsrCost + AddCost);
         } else {
@@ -4013,17 +4014,24 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
         }
       } else if (VT.isVector()) {
         InstructionCost UsraCost = 2 * AsrCost;
-        if (Op2Info.isPowerOf2()) {
+        if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
           // Division with scalable types corresponds to native 'asrd'
           // instruction when SVE is available.
           // e.g. %1 = sdiv <vscale x 4 x i32> %a, splat (i32 8)
+
+          // One more for the negation in SDIV
+          InstructionCost Cost =
+              (Op2Info.isNegatedPowerOf2() && ISD == ISD::SDIV) ? AsrCost : 0;
           if (Ty->isScalableTy() && ST->hasSVE())
-            return 2 * AsrCost;
-          return UsraCost +
-                 (ISD == ISD::SDIV
-                      ? (LT.second.getScalarType() == MVT::i64 ? 1 : 2) *
-                            AsrCost
-                      : 2 * AddCost);
+            Cost += 2 * AsrCost;
+          else {
+            Cost +=
+                UsraCost +
+                (ISD == ISD::SDIV
+                     ? (LT.second.getScalarType() == MVT::i64 ? 1 : 2) * AsrCost
+                     : 2 * AddCost);
+          }
+          return Cost;
         } else if (LT.second == MVT::v2i64) {
           return VT.getVectorNumElements() *
                  getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind,
@@ -4575,6 +4583,13 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
   if (VecTy->isScalableTy() && !ST->hasSVE())
     return InstructionCost::getInvalid();
 
+  // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
+  // only have lowering for power-of-2 factors.
+  // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
+  // InterleavedAccessPass for ld3/st3
+  if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
+    return InstructionCost::getInvalid();
+
   // Vectorization for masked interleaved accesses is only enabled for scalable
   // VF.
   if (!VecTy->isScalableTy() && (UseMaskForCond || UseMaskForGaps))
@@ -5921,7 +5936,7 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2,
       !match(Op2, m_Shuffle(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
     return false;
 
-  // If we allow splats, set S1Op1/S2Op1 to nullptr for the relavant arg so that
+  // If we allow splats, set S1Op1/S2Op1 to nullptr for the relevant arg so that
   // it is not checked as an extract below.
   if (AllowSplat && isSplatShuffle(Op1))
     S1Op1 = nullptr;
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 664c360032ea..0184e748b3d8 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -39,7 +39,7 @@ class Type;
 class Value;
 class VectorType;
 
-class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
+class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
   using BaseT = BasicTTIImplBase<AArch64TTIImpl>;
   using TTI = TargetTransformInfo;
 
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/external/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index ce75c052a712..2f67ff55f26b 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -1275,7 +1275,7 @@ class AArch64Operand : public MCParsedAsmOperand {
       RK = RegKind::SVEPredicateAsCounter;
       break;
     default:
-      llvm_unreachable("Unsupport register class");
+      llvm_unreachable("Unsupported register class");
     }
 
     return (Kind == k_Register && Reg.Kind == RK) &&
@@ -1302,7 +1302,7 @@ class AArch64Operand : public MCParsedAsmOperand {
       RK = RegKind::SVEPredicateVector;
       break;
     default:
-      llvm_unreachable("Unsupport register class");
+      llvm_unreachable("Unsupported register class");
     }
 
     return (Kind == k_Register && Reg.Kind == RK) &&
@@ -5405,7 +5405,7 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, SMLoc &IDLoc,
 
   // A prefix only applies to the instruction following it.  Here we extract
   // prefix information for the next instruction before validating the current
-  // one so that in the case of failure we don't erronously continue using the
+  // one so that in the case of failure we don't erroneously continue using the
   // current prefix.
   PrefixInfo Prefix = NextPrefix;
   NextPrefix = PrefixInfo::CreateFromInst(Inst, MCID.TSFlags);
@@ -5417,7 +5417,7 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, SMLoc &IDLoc,
       (Inst.getOpcode() != AArch64::BRK) &&
       (Inst.getOpcode() != AArch64::HLT)) {
 
-    // Prefixed intructions must have a destructive operand.
+    // Prefixed instructions must have a destructive operand.
     if ((MCID.TSFlags & AArch64::DestructiveInstTypeMask) ==
         AArch64::NotDestructive)
       return Error(IDLoc, "instruction is unpredictable when following a"
@@ -6407,7 +6407,7 @@ bool AArch64AsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                                MCStreamer &Out,
                                                uint64_t &ErrorInfo,
                                                bool MatchingInlineAsm) {
-  assert(!Operands.empty() && "Unexpect empty operand list!");
+  assert(!Operands.empty() && "Unexpected empty operand list!");
   AArch64Operand &Op = static_cast<AArch64Operand &>(*Operands[0]);
   assert(Op.isToken() && "Leading operand should always be a mnemonic!");
 
@@ -7942,7 +7942,7 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
   }
 
   std::unique_ptr<MCELFStreamer::AttributeSubSection> SubsectionExists =
-      getTargetStreamer().getAtributesSubsectionByName(SubsectionName);
+      getTargetStreamer().getAttributesSubsectionByName(SubsectionName);
 
   // Consume the first parameter (optionality parameter)
   AArch64BuildAttributes::SubsectionOptional IsOptional;
@@ -8038,7 +8038,7 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
     return true;
   }
 
-  getTargetStreamer().emitAtributesSubsection(SubsectionName, IsOptional, Type);
+  getTargetStreamer().emitAttributesSubsection(SubsectionName, IsOptional, Type);
 
   return false;
 }
@@ -8050,7 +8050,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   MCAsmParser &Parser = getParser();
 
   std::unique_ptr<MCELFStreamer::AttributeSubSection> ActiveSubsection =
-      getTargetStreamer().getActiveAtributesSubsection();
+      getTargetStreamer().getActiveAttributesSubsection();
   if (nullptr == ActiveSubsection) {
     Error(Parser.getTok().getLoc(),
           "no active subsection, build attribute can not be added");
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/CMakeLists.txt b/external/llvm-project/llvm/lib/Target/AArch64/CMakeLists.txt
index 9cf6f8a86b7d..66136a464f05 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -7,8 +7,7 @@ tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
 tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
 tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv)
 tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler
-              --large-decoder-table)
+tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler)
 tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel)
 tablegen(LLVM AArch64GenO0PreLegalizeGICombiner.inc -gen-global-isel-combiner
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp b/external/llvm-project/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
index 3f22292c95b6..f2528bc12104 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
@@ -41,12 +41,12 @@ getMachOSpecifier(uint64_t LLVMDisassembler_VariantKind) {
   }
 }
 
-/// tryAddingSymbolicOperand - tryAddingSymbolicOperand trys to add a symbolic
+/// tryAddingSymbolicOperand - tryAddingSymbolicOperand tries to add a symbolic
 /// operand in place of the immediate Value in the MCInst.  The immediate
 /// Value has not had any PC adjustment made by the caller. If the instruction
 /// is a branch that adds the PC to the immediate Value then isBranch is
 /// Success, else Fail. If GetOpInfo is non-null, then it is called to get any
-/// symbolic information at the Address for this instrution.  If that returns
+/// symbolic information at the Address for this instruction.  If that returns
 /// non-zero then the symbolic information it returns is used to create an
 /// MCExpr and that is added as an operand to the MCInst.  If GetOpInfo()
 /// returns zero and isBranch is Success then a symbol look up for
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 9bef102e8abf..010d0aaa46e7 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -25,6 +25,7 @@
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/LowLevelTypeUtils.h"
@@ -35,6 +36,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -280,7 +282,7 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
 
   /// We need to fixup the reported store size for certain value types because
   /// we invert the interpretation of ValVT and LocVT in certain cases. This is
-  /// for compatability with the DAG call lowering implementation, which we're
+  /// for compatibility with the DAG call lowering implementation, which we're
   /// currently building on top of.
   LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
                              ISD::ArgFlagsTy Flags) const override {
@@ -296,10 +298,57 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
     MIRBuilder.buildCopy(PhysReg, ExtReg);
   }
 
+  /// Check whether a stack argument requires lowering in a tail call.
+  static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
+                                          const CCValAssign &VA,
+                                          Register ValVReg,
+                                          Register StoreAddr) {
+    const MachineRegisterInfo &MRI = MF.getRegInfo();
+    // Print the defining instruction for the value.
+    auto *DefMI = MRI.getVRegDef(ValVReg);
+    assert(DefMI && "No defining instruction");
+    for (;;) {
+      // Look through nodes that don't alter the bits of the incoming value.
+      unsigned Op = DefMI->getOpcode();
+      if (Op == TargetOpcode::G_ZEXT || Op == TargetOpcode::G_ANYEXT ||
+          Op == TargetOpcode::G_BITCAST || isAssertMI(*DefMI)) {
+        DefMI = MRI.getVRegDef(DefMI->getOperand(1).getReg());
+        continue;
+      }
+      break;
+    }
+
+    auto *Load = dyn_cast<GLoad>(DefMI);
+    if (!Load)
+      return true;
+    Register LoadReg = Load->getPointerReg();
+    auto *LoadAddrDef = MRI.getVRegDef(LoadReg);
+    if (LoadAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
+      return true;
+    const MachineFrameInfo &MFI = MF.getFrameInfo();
+    int LoadFI = LoadAddrDef->getOperand(1).getIndex();
+
+    auto *StoreAddrDef = MRI.getVRegDef(StoreAddr);
+    if (StoreAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
+      return true;
+    int StoreFI = StoreAddrDef->getOperand(1).getIndex();
+
+    if (!MFI.isImmutableObjectIndex(LoadFI))
+      return true;
+    if (MFI.getObjectOffset(LoadFI) != MFI.getObjectOffset(StoreFI))
+      return true;
+    if (Load->getMemSize() != MFI.getObjectSize(StoreFI))
+      return true;
+
+    return false;
+  }
+
   void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
                             const MachinePointerInfo &MPO,
                             const CCValAssign &VA) override {
     MachineFunction &MF = MIRBuilder.getMF();
+    if (!FPDiff && !shouldLowerTailCallStackArg(MF, VA, ValVReg, Addr))
+      return;
     auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,
                                        inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildStore(ValVReg, Addr, *MMO);
@@ -539,7 +588,7 @@ bool AArch64CallLowering::fallBackToDAGISel(const MachineFunction &MF) const {
     return true;
   }
 
-  SMEAttrs Attrs(F);
+  SMEAttrs Attrs = MF.getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
   if (Attrs.hasZAState() || Attrs.hasZT0State() ||
       Attrs.hasStreamingInterfaceOrBody() ||
       Attrs.hasStreamingCompatibleInterface())
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index e0c693bff3c0..51b42325ef84 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -22,6 +22,7 @@
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
+#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
@@ -6630,7 +6631,7 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
     Register SizeUse = I.getOperand(4).getReg();
 
     // MOPSMemorySetTaggingPseudo has two defs; the intrinsic call has only one.
-    // Therefore an additional virtual register is requried for the updated size
+    // Therefore an additional virtual register is required for the updated size
     // operand. This value is not accessible via the semantics of the intrinsic.
     Register SizeDef = MRI.createGenericVirtualRegister(LLT::scalar(64));
 
@@ -7418,7 +7419,7 @@ AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
     unsigned Scale = Log2_32(SizeInBytes);
     int64_t ImmOff = ValAndVReg->Value.getSExtValue();
 
-    // Skip immediates that can be selected in the load/store addresing
+    // Skip immediates that can be selected in the load/store addressing
     // mode.
     if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
         ImmOff < (0x1000 << Scale))
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 93f4bc423b63..12843e16d0da 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -370,6 +370,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .clampNumElements(0, v4s16, v8s16)
       .clampNumElements(0, v2s32, v4s32)
       .clampNumElements(0, v2s64, v2s64)
+      .scalarizeIf(scalarOrEltWiderThan(0, 64), 0)
       .moreElementsToNextPow2(0)
       .lower();
 
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index 6bd3fd182485..31954e7954c0 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -476,6 +476,11 @@ static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
   case Intrinsic::aarch64_neon_facgt:
   case Intrinsic::aarch64_neon_fabd:
   case Intrinsic::aarch64_sisd_fabd:
+  case Intrinsic::aarch64_neon_sqrdmlah:
+  case Intrinsic::aarch64_neon_sqrdmlsh:
+  case Intrinsic::aarch64_neon_sqrdmulh:
+  case Intrinsic::aarch64_neon_sqadd:
+  case Intrinsic::aarch64_neon_sqsub:
     return true;
   case Intrinsic::aarch64_neon_saddlv: {
     const LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());
@@ -485,7 +490,7 @@ static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
   }
 }
 
-bool AArch64RegisterBankInfo::isPHIWithFPContraints(
+bool AArch64RegisterBankInfo::isPHIWithFPConstraints(
     const MachineInstr &MI, const MachineRegisterInfo &MRI,
     const TargetRegisterInfo &TRI, const unsigned Depth) const {
   if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
@@ -495,7 +500,7 @@ bool AArch64RegisterBankInfo::isPHIWithFPContraints(
                 [&](const MachineInstr &UseMI) {
                   if (onlyUsesFP(UseMI, MRI, TRI, Depth + 1))
                     return true;
-                  return isPHIWithFPContraints(UseMI, MRI, TRI, Depth + 1);
+                  return isPHIWithFPConstraints(UseMI, MRI, TRI, Depth + 1);
                 });
 }
 
@@ -892,7 +897,7 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
                  // Int->FP conversion operations are also captured in
                  // onlyDefinesFP().
 
-                 if (isPHIWithFPContraints(UseMI, MRI, TRI))
+                 if (isPHIWithFPConstraints(UseMI, MRI, TRI))
                    return true;
 
                  return onlyUsesFP(UseMI, MRI, TRI) ||
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h
index 941499b08d05..3abbc1b68b5b 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h
@@ -122,7 +122,7 @@ class AArch64RegisterBankInfo final : public AArch64GenRegisterBankInfo {
 
   /// \returns true if \p MI is a PHI that its def is used by
   /// any instruction that onlyUsesFP.
-  bool isPHIWithFPContraints(const MachineInstr &MI,
+  bool isPHIWithFPConstraints(const MachineInstr &MI,
                              const MachineRegisterInfo &MRI,
                              const TargetRegisterInfo &TRI,
                              unsigned Depth = 0) const;
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
index 03cbd272757e..f542592d22c5 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
@@ -97,7 +97,7 @@ static inline unsigned getShiftValue(unsigned Imm) {
 ///   {5-0}  = imm
 static inline unsigned getShifterImm(AArch64_AM::ShiftExtendType ST,
                                      unsigned Imm) {
-  assert((Imm & 0x3f) == Imm && "Illegal shifted immedate value!");
+  assert((Imm & 0x3f) == Imm && "Illegal shifted immediate value!");
   unsigned STEnc = 0;
   switch (ST) {
   default:  llvm_unreachable("Invalid shift requested");
@@ -169,7 +169,7 @@ inline unsigned getExtendEncoding(AArch64_AM::ShiftExtendType ET) {
 ///   {2-0}  = imm3
 static inline unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET,
                                          unsigned Imm) {
-  assert((Imm & 0x7) == Imm && "Illegal shifted immedate value!");
+  assert((Imm & 0x7) == Imm && "Illegal shifted immediate value!");
   return (getExtendEncoding(ET) << 3) | (Imm & 0x7);
 }
 
@@ -594,7 +594,7 @@ static inline bool isAdvSIMDModImmType10(uint64_t Imm) {
 #if defined(_MSC_VER) && _MSC_VER == 1937 && !defined(__clang__) &&            \
     defined(_M_ARM64)
   // The MSVC compiler 19.37 for ARM64 has an optimization bug that
-  // causes an incorrect behavior with the orignal version. Work around
+  // causes an incorrect behavior with the original version. Work around
   // by using a slightly different variation.
   // https://developercommunity.visualstudio.com/t/C-ARM64-compiler-optimization-bug/10481261
   constexpr uint64_t Mask = 0xFFULL;
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 6ee12ccf5494..f2144375fd95 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -231,7 +231,7 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
     OS << "\n";
   }
 
-  void emitAtributesSubsection(
+  void emitAttributesSubsection(
       StringRef SubsectionName,
       AArch64BuildAttributes::SubsectionOptional Optional,
       AArch64BuildAttributes::SubsectionType ParameterType) override {
@@ -278,7 +278,7 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
        << ", " << ParameterStr;
     // Keep the data structure consistent with the case of ELF emission
     // (important for llvm-mc asm parsing)
-    AArch64TargetStreamer::emitAtributesSubsection(SubsectionName, Optional,
+    AArch64TargetStreamer::emitAttributesSubsection(SubsectionName, Optional,
                                                    ParameterType);
     OS << "\n";
   }
@@ -433,10 +433,10 @@ AArch64ELFStreamer &AArch64TargetELFStreamer::getStreamer() {
   return static_cast<AArch64ELFStreamer &>(Streamer);
 }
 
-void AArch64TargetELFStreamer::emitAtributesSubsection(
+void AArch64TargetELFStreamer::emitAttributesSubsection(
     StringRef VendorName, AArch64BuildAttributes::SubsectionOptional IsOptional,
     AArch64BuildAttributes::SubsectionType ParameterType) {
-  AArch64TargetStreamer::emitAtributesSubsection(VendorName, IsOptional,
+  AArch64TargetStreamer::emitAttributesSubsection(VendorName, IsOptional,
                                                  ParameterType);
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index 5552cea78694..bbe83821eca8 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -216,7 +216,7 @@ void AArch64InstPrinter::printInst(const MCInst *MI, uint64_t Address,
 
     if ((Op2.getReg() == AArch64::WZR || Op2.getReg() == AArch64::XZR) &&
         (ImmR == 0 || ImmS < ImmR) && STI.hasFeature(AArch64::HasV8_2aOps)) {
-      // BFC takes precedence over its entire range, sligtly differently to BFI.
+      // BFC takes precedence over its entire range, slightly differently to BFI.
       int BitWidth = Opcode == AArch64::BFMXri ? 64 : 32;
       int LSB = (BitWidth - ImmR) % BitWidth;
       int Width = ImmS + 1;
@@ -278,7 +278,7 @@ void AArch64InstPrinter::printInst(const MCInst *MI, uint64_t Address,
     {
       WithMarkup M = markup(O, Markup::Immediate);
       O << "#";
-      MI->getOperand(1).getExpr()->print(O, &MAI);
+      MAI.printExpr(O, *MI->getOperand(1).getExpr());
     }
     return;
   }
@@ -291,7 +291,7 @@ void AArch64InstPrinter::printInst(const MCInst *MI, uint64_t Address,
     {
       WithMarkup M = markup(O, Markup::Immediate);
       O << "#";
-      MI->getOperand(2).getExpr()->print(O, &MAI);
+      MAI.printExpr(O, *MI->getOperand(2).getExpr());
     }
     return;
   }
@@ -1163,7 +1163,7 @@ void AArch64InstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     printImm(MI, OpNo, STI, O);
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
-    Op.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -1240,7 +1240,7 @@ void AArch64InstPrinter::printAddSubImm(const MCInst *MI, unsigned OpNum,
     }
   } else {
     assert(MO.isExpr() && "Unexpected operand type!");
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
     printShifter(MI, OpNum + 1, STI, O);
   }
 }
@@ -1431,7 +1431,7 @@ void AArch64InstPrinter::printUImm12Offset(const MCInst *MI, unsigned OpNum,
     markup(O, Markup::Immediate) << '#' << formatImm(MO.getImm() * Scale);
   } else {
     assert(MO.isExpr() && "Unexpected operand type!");
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
   }
 }
 
@@ -1446,7 +1446,7 @@ void AArch64InstPrinter::printAMIndexedWB(const MCInst *MI, unsigned OpNum,
   } else {
     assert(MO1.isExpr() && "Unexpected operand type!");
     O << ", ";
-    MO1.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO1.getExpr());
   }
   O << ']';
 }
@@ -1805,7 +1805,7 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address,
     markup(O, Markup::Target) << formatHex((uint64_t)TargetAddress);
   } else {
     // Otherwise, just print the expression.
-    MI->getOperand(OpNum).getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MI->getOperand(OpNum).getExpr());
   }
 }
 
@@ -1832,7 +1832,7 @@ void AArch64InstPrinter::printAdrAdrpLabel(const MCInst *MI, uint64_t Address,
   }
 
   // Otherwise, just print the expression.
-  MI->getOperand(OpNum).getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MI->getOperand(OpNum).getExpr());
 }
 
 void AArch64InstPrinter::printBarrierOption(const MCInst *MI, unsigned OpNo,
@@ -2051,7 +2051,7 @@ void AArch64InstPrinter::printImm8OptLsl(const MCInst *MI, unsigned OpNum,
   unsigned UnscaledVal = MI->getOperand(OpNum).getImm();
   unsigned Shift = MI->getOperand(OpNum + 1).getImm();
   assert(AArch64_AM::getShiftType(Shift) == AArch64_AM::LSL &&
-         "Unexepected shift type!");
+         "Unexpected shift type!");
 
   // #0 lsl #8 is never pretty printed
   if ((UnscaledVal == 0) && (AArch64_AM::getShiftValue(Shift) != 0)) {
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
index 88ff61ecedb6..d934af91b9ff 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
@@ -98,14 +98,6 @@ void AArch64MCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   Expr->print(OS, MAI);
 }
 
-void AArch64MCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-MCFragment *AArch64MCExpr::findAssociatedFragment() const {
-  llvm_unreachable("FIXME: what goes here?");
-}
-
 bool AArch64MCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                               const MCAssembler *Asm) const {
   if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
@@ -136,11 +128,3 @@ void AArch64AuthMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
     OS << ",addr";
   OS << ')';
 }
-
-void AArch64AuthMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-MCFragment *AArch64AuthMCExpr::findAssociatedFragment() const {
-  llvm_unreachable("FIXME: what goes here?");
-}
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
index 3f10c69869c2..9c383894c7f5 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
@@ -21,9 +21,10 @@
 
 namespace llvm {
 
-class AArch64MCExpr : public MCTargetExpr {
+class AArch64MCExpr : public MCSpecifierExpr {
 public:
-  enum Specifier : uint16_t {
+  using Specifier = uint16_t;
+  enum {
     // clang-format off
     None          = 0,
     // Symbol locations specifying (roughly speaking) what calculation should be
@@ -139,32 +140,13 @@ class AArch64MCExpr : public MCTargetExpr {
     // clang-format on
   };
 
-private:
-  const MCExpr *Expr;
-  const Specifier specifier;
-
 protected:
   explicit AArch64MCExpr(const MCExpr *Expr, Specifier S)
-      : Expr(Expr), specifier(S) {}
+      : MCSpecifierExpr(Expr, S) {}
 
 public:
-  /// @name Construction
-  /// @{
-
   static const AArch64MCExpr *create(const MCExpr *Expr, Specifier,
                                      MCContext &Ctx);
-
-  /// @}
-  /// @name Accessors
-  /// @{
-
-  /// Get the kind of this expression.
-  Specifier getSpecifier() const { return specifier; }
-
-  /// Get the expression this modifier applies to.
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  /// @}
   /// @name VariantKind information extractors.
   /// @{
 
@@ -185,16 +167,8 @@ class AArch64MCExpr : public MCTargetExpr {
   StringRef getSpecifierName() const;
 
   void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-
-  MCFragment *findAssociatedFragment() const override;
-
   bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAssembler *Asm) const override;
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
 };
 
 class AArch64AuthMCExpr final : public AArch64MCExpr {
@@ -217,10 +191,6 @@ class AArch64AuthMCExpr final : public AArch64MCExpr {
 
   void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
 
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-
-  MCFragment *findAssociatedFragment() const override;
-
   static bool classof(const MCExpr *E) {
     return isa<AArch64MCExpr>(E) && classof(cast<AArch64MCExpr>(E));
   }
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index dd3ecb41494e..b7959e02ec26 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -436,7 +436,7 @@ class AArch64MCInstrAnalysis : public MCInstrAnalysis {
       // architecturally defined to zero extend the upper 32 bits on a write.
       if (GPR32RC.contains(Reg))
         return true;
-      // SIMD&FP instructions operating on scalar data only acccess the lower
+      // SIMD&FP instructions operating on scalar data only access the lower
       // bits of a register, the upper bits are zero extended on a write. For
       // SIMD vector registers smaller than 128-bits, the upper 64-bits of the
       // register are zero extended on a write.
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
index 04defbc97746..d742b282b617 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
@@ -29,7 +29,7 @@ static cl::opt<bool> MarkBTIProperty(
     cl::init(false));
 
 //
-// AArch64TargetStreamer Implemenation
+// AArch64TargetStreamer Implementation
 //
 AArch64TargetStreamer::AArch64TargetStreamer(MCStreamer &S)
     : MCTargetStreamer(S), ConstantPools(new AssemblerConstantPools()) {}
@@ -153,14 +153,14 @@ MCTargetStreamer *llvm::createAArch64NullTargetStreamer(MCStreamer &S) {
   return new AArch64TargetStreamer(S);
 }
 
-void AArch64TargetStreamer::emitAtributesSubsection(
+void AArch64TargetStreamer::emitAttributesSubsection(
     StringRef VendorName, AArch64BuildAttributes::SubsectionOptional IsOptional,
     AArch64BuildAttributes::SubsectionType ParameterType) {
 
   // If exists, return.
   for (MCELFStreamer::AttributeSubSection &SubSection : AttributeSubSections) {
     if (VendorName == SubSection.VendorName) {
-      activateAtributesSubsection(VendorName);
+      activateAttributesSubsection(VendorName);
       return;
     }
   }
@@ -170,11 +170,11 @@ void AArch64TargetStreamer::emitAtributesSubsection(
   AttSubSection.IsOptional = IsOptional;
   AttSubSection.ParameterType = ParameterType;
   AttributeSubSections.push_back(AttSubSection);
-  activateAtributesSubsection(VendorName);
+  activateAttributesSubsection(VendorName);
 }
 
 std::unique_ptr<MCELFStreamer::AttributeSubSection>
-AArch64TargetStreamer::getActiveAtributesSubsection() {
+AArch64TargetStreamer::getActiveAttributesSubsection() {
   for (MCELFStreamer::AttributeSubSection &SubSection : AttributeSubSections) {
     if (SubSection.IsActive) {
       return std::make_unique<MCELFStreamer::AttributeSubSection>(SubSection);
@@ -184,7 +184,7 @@ AArch64TargetStreamer::getActiveAtributesSubsection() {
 }
 
 std::unique_ptr<MCELFStreamer::AttributeSubSection>
-AArch64TargetStreamer::getAtributesSubsectionByName(StringRef Name) {
+AArch64TargetStreamer::getAttributesSubsectionByName(StringRef Name) {
   for (MCELFStreamer::AttributeSubSection &SubSection : AttributeSubSections) {
     if (Name == SubSection.VendorName) {
       return std::make_unique<MCELFStreamer::AttributeSubSection>(SubSection);
@@ -238,7 +238,7 @@ void AArch64TargetStreamer::emitAttribute(StringRef VendorName, unsigned Tag,
               "not exist");
 }
 
-void AArch64TargetStreamer::activateAtributesSubsection(StringRef VendorName) {
+void AArch64TargetStreamer::activateAttributesSubsection(StringRef VendorName) {
   for (MCELFStreamer::AttributeSubSection &SubSection : AttributeSubSections) {
     if (VendorName == SubSection.VendorName) {
       SubSection.IsActive = true;
@@ -246,4 +246,4 @@ void AArch64TargetStreamer::activateAtributesSubsection(StringRef VendorName) {
       SubSection.IsActive = false;
     }
   }
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
index aa26acd85bda..d878f1e044b8 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
+++ b/external/llvm-project/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
@@ -102,16 +102,16 @@ class AArch64TargetStreamer : public MCTargetStreamer {
 
   /// Build attributes implementation
   virtual void
-  emitAtributesSubsection(StringRef VendorName,
+  emitAttributesSubsection(StringRef VendorName,
                           AArch64BuildAttributes::SubsectionOptional IsOptional,
                           AArch64BuildAttributes::SubsectionType ParameterType);
   virtual void emitAttribute(StringRef VendorName, unsigned Tag, unsigned Value,
                              std::string String);
-  void activateAtributesSubsection(StringRef VendorName);
+  void activateAttributesSubsection(StringRef VendorName);
   std::unique_ptr<MCELFStreamer::AttributeSubSection>
-  getActiveAtributesSubsection();
+  getActiveAttributesSubsection();
   std::unique_ptr<MCELFStreamer::AttributeSubSection>
-  getAtributesSubsectionByName(StringRef Name);
+  getAttributesSubsectionByName(StringRef Name);
   void
   insertAttributeInPlace(const MCELFStreamer::AttributeItem &Attr,
                          MCELFStreamer::AttributeSubSection &AttSubSection);
@@ -129,7 +129,7 @@ class AArch64TargetELFStreamer : public AArch64TargetStreamer {
   MCSection *AttributeSection = nullptr;
 
   /// Build attributes implementation
-  void emitAtributesSubsection(
+  void emitAttributesSubsection(
       StringRef VendorName,
       AArch64BuildAttributes::SubsectionOptional IsOptional,
       AArch64BuildAttributes::SubsectionType ParameterType) override;
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp b/external/llvm-project/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
index fc8bef4ad842..bd2871611888 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
+++ b/external/llvm-project/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
@@ -225,7 +225,7 @@ bool SMEPeepholeOpt::optimizeStartStopPairs(
 }
 
 // Using the FORM_TRANSPOSED_REG_TUPLE pseudo can improve register allocation
-// of multi-vector intrinsics. However, the psuedo should only be emitted if
+// of multi-vector intrinsics. However, the pseudo should only be emitted if
 // the input registers of the REG_SEQUENCE are copy nodes where the source
 // register is in a StridedOrContiguous class. For example:
 //
diff --git a/external/llvm-project/llvm/lib/Target/AArch64/SVEInstrFormats.td b/external/llvm-project/llvm/lib/Target/AArch64/SVEInstrFormats.td
index c56713783289..848b1c30bbeb 100644
--- a/external/llvm-project/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/external/llvm-project/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -7884,9 +7884,14 @@ multiclass sve_int_perm_compact_sd<string asm, SDPatternOperator op> {
   def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_int_perm_compact_bh<string asm> {
+multiclass sve_int_perm_compact_bh<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_compact<0b00, asm, ZPR8>;
   def _H : sve_int_perm_compact<0b01, asm, ZPR16>;
+
+  def : SVE_2_Op_Pat<nxv16i8,  op, nxv16i1, nxv16i8,  !cast<Instruction>(NAME # _B)>;
+  def : SVE_2_Op_Pat<nxv8i16,  op, nxv8i1,  nxv8i16,  !cast<Instruction>(NAME # _H)>;
+  def : SVE_2_Op_Pat<nxv8f16,  op, nxv8i1,  nxv8f16,  !cast<Instruction>(NAME # _H)>;
+  def : SVE_2_Op_Pat<nxv8bf16, op, nxv8i1,  nxv8bf16, !cast<Instruction>(NAME # _H)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -9235,11 +9240,12 @@ multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty,
   def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>;
 }
 
-class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty>
+class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty,
+                             SDPatternOperator op, ValueType vt>
 : I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm),
   asm, "\t$Zdn, $_Zdn, $Zm",
   "",
-  []>, Sched<[]> {
+  [(set (vt zprty:$Zdn), (op (vt zprty:$_Zdn), (vt zprty:$Zm)))]>, Sched<[]> {
   bits<5> Zdn;
   bits<5> Zm;
   let Inst{31-17} = 0b010001010010001;
@@ -9253,12 +9259,6 @@ class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty>
   let hasSideEffects = 0;
 }
 
-multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty,
-                                  SDPatternOperator op, ValueType vt> {
-  def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>;
-  def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>;
-}
-
 class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty>
 : I<(outs zprty:$Zdn), (ins zprty:$_Zdn),
   asm, "\t$Zdn, $_Zdn",
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 98a32f9225ba..14101e57f514 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -1090,22 +1090,6 @@ bool AMDGPUCallLowering::areCalleeOutgoingArgsTailCallable(
   return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);
 }
 
-/// Return true if the calling convention is one that we can guarantee TCO for.
-static bool canGuaranteeTCO(CallingConv::ID CC) {
-  return CC == CallingConv::Fast;
-}
-
-/// Return true if we might ever do TCO for calls with this calling convention.
-static bool mayTailCallThisCC(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::C:
-  case CallingConv::AMDGPU_Gfx:
-    return true;
-  default:
-    return canGuaranteeTCO(CC);
-  }
-}
-
 bool AMDGPUCallLowering::isEligibleForTailCallOptimization(
     MachineIRBuilder &B, CallLoweringInfo &Info,
     SmallVectorImpl<ArgInfo> &InArgs, SmallVectorImpl<ArgInfo> &OutArgs) const {
@@ -1130,7 +1114,7 @@ bool AMDGPUCallLowering::isEligibleForTailCallOptimization(
   if (!CallerPreserved)
     return false;
 
-  if (!mayTailCallThisCC(CalleeCC)) {
+  if (!AMDGPU::mayTailCallThisCC(CalleeCC)) {
     LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
     return false;
   }
@@ -1144,8 +1128,10 @@ bool AMDGPUCallLowering::isEligibleForTailCallOptimization(
   }
 
   // If we have -tailcallopt, then we're done.
-  if (MF.getTarget().Options.GuaranteedTailCallOpt)
-    return canGuaranteeTCO(CalleeCC) && CalleeCC == CallerF.getCallingConv();
+  if (MF.getTarget().Options.GuaranteedTailCallOpt) {
+    return AMDGPU::canGuaranteeTCO(CalleeCC) &&
+           CalleeCC == CallerF.getCallingConv();
+  }
 
   // Verify that the incoming and outgoing arguments from the callee are
   // safe to tail call.
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 3cccf40554af..22b921fb2084 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -180,7 +180,7 @@ class AMDGPUCodeGenPrepareImpl
   /// Wrapper to pass all the arguments to computeKnownFPClass
   KnownFPClass computeKnownFPClass(const Value *V, FPClassTest Interested,
                                    const Instruction *CtxI) const {
-    return llvm::computeKnownFPClass(V, DL, Interested, 0, TLI, AC, CtxI, DT);
+    return llvm::computeKnownFPClass(V, DL, Interested, TLI, AC, CtxI, DT);
   }
 
   bool canIgnoreDenormalInput(const Value *V, const Instruction *CtxI) const {
@@ -596,11 +596,11 @@ bool AMDGPUCodeGenPrepareImpl::promoteUniformBitreverseToI32(
 }
 
 unsigned AMDGPUCodeGenPrepareImpl::numBitsUnsigned(Value *Op) const {
-  return computeKnownBits(Op, DL, 0, AC).countMaxActiveBits();
+  return computeKnownBits(Op, DL, AC).countMaxActiveBits();
 }
 
 unsigned AMDGPUCodeGenPrepareImpl::numBitsSigned(Value *Op) const {
-  return ComputeMaxSignificantBits(Op, DL, 0, AC);
+  return ComputeMaxSignificantBits(Op, DL, AC);
 }
 
 static void extractValues(IRBuilder<> &Builder,
@@ -1198,13 +1198,13 @@ unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
          Den->getType()->getScalarSizeInBits());
   unsigned SSBits = Num->getType()->getScalarSizeInBits();
   if (IsSigned) {
-    unsigned RHSSignBits = ComputeNumSignBits(Den, DL, 0, AC, &I);
+    unsigned RHSSignBits = ComputeNumSignBits(Den, DL, AC, &I);
     // A sign bit needs to be reserved for shrinking.
     unsigned DivBits = SSBits - RHSSignBits + 1;
     if (DivBits > MaxDivBits)
       return SSBits;
 
-    unsigned LHSSignBits = ComputeNumSignBits(Num, DL, 0, AC, &I);
+    unsigned LHSSignBits = ComputeNumSignBits(Num, DL, AC, &I);
 
     unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
     DivBits = SSBits - SignBits + 1;
@@ -1213,7 +1213,7 @@ unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
 
   // All bits are used for unsigned division for Num or Den in range
   // (SignedMax, UnsignedMax].
-  KnownBits Known = computeKnownBits(Den, DL, 0, AC, &I);
+  KnownBits Known = computeKnownBits(Den, DL, AC, &I);
   if (Known.isNegative() || !Known.isNonNegative())
     return SSBits;
   unsigned RHSSignBits = Known.countMinLeadingZeros();
@@ -1221,7 +1221,7 @@ unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
   if (DivBits > MaxDivBits)
     return SSBits;
 
-  Known = computeKnownBits(Num, DL, 0, AC, &I);
+  Known = computeKnownBits(Num, DL, AC, &I);
   if (Known.isNegative() || !Known.isNonNegative())
     return SSBits;
   unsigned LHSSignBits = Known.countMinLeadingZeros();
@@ -1358,7 +1358,7 @@ bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
     // If there's no wider mulhi, there's only a better expansion for powers of
     // two.
     // TODO: Should really know for each vector element.
-    if (isKnownToBeAPowerOfTwo(C, DL, true, 0, AC, &I, DT))
+    if (isKnownToBeAPowerOfTwo(C, DL, true, AC, &I, DT))
       return true;
 
     return false;
@@ -1368,8 +1368,7 @@ bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
     // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
     if (BinOpDen->getOpcode() == Instruction::Shl &&
         isa<Constant>(BinOpDen->getOperand(0)) &&
-        isKnownToBeAPowerOfTwo(BinOpDen->getOperand(0), DL, true, 0, AC, &I,
-                               DT)) {
+        isKnownToBeAPowerOfTwo(BinOpDen->getOperand(0), DL, true, AC, &I, DT)) {
       return true;
     }
   }
@@ -2235,8 +2234,8 @@ Value *AMDGPUCodeGenPrepareImpl::matchFractPat(IntrinsicInst &I) {
 
   // The value is only used in contexts where we know the input isn't a nan, so
   // any of the fmin variants are fine.
-  if (IID != Intrinsic::minnum &&
-      IID != Intrinsic::minimumnum & IID != Intrinsic::minimum)
+  if (IID != Intrinsic::minnum && IID != Intrinsic::minimum &&
+      IID != Intrinsic::minimumnum)
     return nullptr;
 
   Type *Ty = I.getType();
@@ -2289,8 +2288,7 @@ bool AMDGPUCodeGenPrepareImpl::visitFMinLike(IntrinsicInst &I) {
 
   // Match pattern for fract intrinsic in contexts where the nan check has been
   // optimized out (and hope the knowledge the source can't be nan wasn't lost).
-  if (!I.hasNoNaNs() &&
-      !isKnownNeverNaN(FractArg, /*Depth=*/0, SimplifyQuery(DL, TLI)))
+  if (!I.hasNoNaNs() && !isKnownNeverNaN(FractArg, SimplifyQuery(DL, TLI)))
     return false;
 
   IRBuilder<> Builder(&I);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 5f41bd7d8a61..c51cc2a2fe52 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -4097,7 +4097,7 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
   if (VT.getScalarType() != MVT::i64)
     return SDValue();
 
-  // i64 (shl x, C) -> (build_pair 0, (shl x, C -32))
+  // i64 (shl x, C) -> (build_pair 0, (shl x, C - 32))
 
   // On some subtargets, 64-bit shift is a quarter rate instruction. In the
   // common case, splitting this into a move and a 32-bit shift is faster and
@@ -4117,12 +4117,12 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
     ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
                                TargetType);
   } else {
-    SDValue truncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
+    SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
     const SDValue ShiftMask =
         DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
     // This AND instruction will clamp out of bounds shift values.
     // It will also be removed during later instruction selection.
-    ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, truncShiftAmt, ShiftMask);
+    ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
   }
 
   SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, TargetType, LHS);
@@ -4181,50 +4181,105 @@ SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
 
 SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
-  auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
-  if (!RHS)
-    return SDValue();
-
+  SDValue RHS = N->getOperand(1);
+  ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS);
   EVT VT = N->getValueType(0);
   SDValue LHS = N->getOperand(0);
-  unsigned ShiftAmt = RHS->getZExtValue();
   SelectionDAG &DAG = DCI.DAG;
   SDLoc SL(N);
+  unsigned RHSVal;
+
+  if (CRHS) {
+    RHSVal = CRHS->getZExtValue();
 
-  // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
-  // this improves the ability to match BFE patterns in isel.
-  if (LHS.getOpcode() == ISD::AND) {
-    if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
-      unsigned MaskIdx, MaskLen;
-      if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
-          MaskIdx == ShiftAmt) {
-        return DAG.getNode(
-            ISD::AND, SL, VT,
-            DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)),
-            DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1)));
+    // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
+    // this improves the ability to match BFE patterns in isel.
+    if (LHS.getOpcode() == ISD::AND) {
+      if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
+        unsigned MaskIdx, MaskLen;
+        if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
+            MaskIdx == RHSVal) {
+          return DAG.getNode(ISD::AND, SL, VT,
+                             DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0),
+                                         N->getOperand(1)),
+                             DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1),
+                                         N->getOperand(1)));
+        }
       }
     }
   }
 
-  if (VT != MVT::i64)
+  if (VT.getScalarType() != MVT::i64)
     return SDValue();
 
-  if (ShiftAmt < 32)
+  // for C >= 32
+  // i64 (srl x, C) -> (build_pair (srl hi_32(x), C -32), 0)
+
+  // On some subtargets, 64-bit shift is a quarter rate instruction. In the
+  // common case, splitting this into a move and a 32-bit shift is faster and
+  // the same code size.
+  KnownBits Known = DAG.computeKnownBits(RHS);
+
+  EVT ElementType = VT.getScalarType();
+  EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
+  EVT TargetType = VT.isVector() ? VT.changeVectorElementType(TargetScalarType)
+                                 : TargetScalarType;
+
+  if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
     return SDValue();
 
-  // srl i64:x, C for C >= 32
-  // =>
-  //   build_pair (srl hi_32(x), C - 32), 0
-  SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
+  SDValue ShiftAmt;
+  if (CRHS) {
+    ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
+                               TargetType);
+  } else {
+    SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
+    const SDValue ShiftMask =
+        DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
+    // This AND instruction will clamp out of bounds shift values.
+    // It will also be removed during later instruction selection.
+    ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
+  }
+
+  const SDValue Zero = DAG.getConstant(0, SL, TargetScalarType);
+  EVT ConcatType;
+  SDValue Hi;
+  SDLoc LHSSL(LHS);
+  // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
+  if (VT.isVector()) {
+    unsigned NElts = TargetType.getVectorNumElements();
+    ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
+    SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+    SmallVector<SDValue, 8> HiOps(NElts);
+    SmallVector<SDValue, 16> HiAndLoOps;
 
-  SDValue Hi = getHiHalf64(LHS, DAG);
+    DAG.ExtractVectorElements(SplitLHS, HiAndLoOps, /*Start=*/0, NElts * 2);
+    for (unsigned I = 0; I != NElts; ++I)
+      HiOps[I] = HiAndLoOps[2 * I + 1];
+    Hi = DAG.getNode(ISD::BUILD_VECTOR, LHSSL, TargetType, HiOps);
+  } else {
+    const SDValue One = DAG.getConstant(1, LHSSL, TargetScalarType);
+    ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
+    SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+    Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
+  }
 
-  SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
-  SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
+  SDValue NewShift = DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt);
 
-  SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
+  SDValue Vec;
+  if (VT.isVector()) {
+    unsigned NElts = TargetType.getVectorNumElements();
+    SmallVector<SDValue, 8> LoOps;
+    SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
 
-  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
+    DAG.ExtractVectorElements(NewShift, LoOps, 0, NElts);
+    for (unsigned I = 0; I != NElts; ++I)
+      HiAndLoOps[2 * I] = LoOps[I];
+    Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
+  } else {
+    Vec = DAG.getBuildVector(ConcatType, SL, {NewShift, Zero});
+  }
+  return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
 }
 
 SDValue AMDGPUTargetLowering::performTruncateCombine(
@@ -5209,21 +5264,18 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
 
     break;
   }
-  case ISD::SHL: {
+  case ISD::SHL:
+  case ISD::SRL: {
     // Range metadata can be invalidated when loads are converted to legal types
     // (e.g. v2i64 -> v4i32).
-    // Try to convert vector shl before type legalization so that range metadata
-    // can be utilized.
+    // Try to convert vector shl/srl before type legalization so that range
+    // metadata can be utilized.
     if (!(N->getValueType(0).isVector() &&
           DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
         DCI.getDAGCombineLevel() < AfterLegalizeDAG)
       break;
-    return performShlCombine(N, DCI);
-  }
-  case ISD::SRL: {
-    if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
-      break;
-
+    if (N->getOpcode() == ISD::SHL)
+      return performShlCombine(N, DCI);
     return performSrlCombine(N, DCI);
   }
   case ISD::SRA: {
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 16f27533a0c1..9be8821d5bf9 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -365,8 +365,7 @@ bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
   }
 
   SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(&I);
-  if (isKnownNeverInfOrNaN(Op0, /*Depth=*/0, SQ) &&
-      isKnownNeverInfOrNaN(Op1, /*Depth=*/0, SQ)) {
+  if (isKnownNeverInfOrNaN(Op0, SQ) && isKnownNeverInfOrNaN(Op1, SQ)) {
     // Neither operand is infinity or NaN.
     return true;
   }
@@ -463,6 +462,8 @@ static bool isTriviallyUniform(const Use &U) {
   Value *V = U.get();
   if (isa<Constant>(V))
     return true;
+  if (const auto *A = dyn_cast<Argument>(V))
+    return AMDGPU::isArgPassedInSGPR(A);
   if (const auto *II = dyn_cast<IntrinsicInst>(V)) {
     if (!AMDGPU::isIntrinsicAlwaysUniform(II->getIntrinsicID()))
       return false;
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index f0d63f523088..df976cf3f7fd 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -61,7 +61,7 @@ class AMDGPULateCodeGenPrepare
 
   // Check if the specified value is at least DWORD aligned.
   bool isDWORDAligned(const Value *V) const {
-    KnownBits Known = computeKnownBits(V, DL, 0, AC);
+    KnownBits Known = computeKnownBits(V, DL, AC);
     return Known.countMinTrailingZeros() >= 2;
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 177750b639c6..e8dff8506438 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1061,10 +1061,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   }
 
   auto &FPTruncActions = getActionDefinitionsBuilder(G_FPTRUNC);
-  if (ST.hasCvtPkF16F32Inst())
-    FPTruncActions.legalFor({{S32, S64}, {S16, S32}, {V2S16, V2S32}});
-  else
+  if (ST.hasCvtPkF16F32Inst()) {
+    FPTruncActions.legalFor({{S32, S64}, {S16, S32}, {V2S16, V2S32}})
+        .clampMaxNumElements(0, S16, 2);
+  } else {
     FPTruncActions.legalFor({{S32, S64}, {S16, S32}});
+  }
   FPTruncActions.scalarize(0).lower();
 
   getActionDefinitionsBuilder(G_FPEXT)
@@ -4261,7 +4263,7 @@ static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI) {
   if (MI.getOpcode() != TargetOpcode::G_XOR)
     return false;
   auto ConstVal = getIConstantVRegSExtVal(MI.getOperand(2).getReg(), MRI);
-  return ConstVal && *ConstVal == -1;
+  return ConstVal == -1;
 }
 
 // Return the use branch instruction, otherwise null if the usage is invalid.
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index 5c3abd9ce1e3..8767208d20ec 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -614,7 +614,7 @@ static bool isKnownIntegral(const Value *V, const DataLayout &DL,
 
     // Need to check int size cannot produce infinity, which computeKnownFPClass
     // knows how to do already.
-    return isKnownNeverInfinity(I, /*Depth=*/0, SimplifyQuery(DL));
+    return isKnownNeverInfinity(I, SimplifyQuery(DL));
   case Instruction::Call: {
     const CallInst *CI = cast<CallInst>(I);
     switch (CI->getIntrinsicID()) {
@@ -626,7 +626,7 @@ static bool isKnownIntegral(const Value *V, const DataLayout &DL,
     case Intrinsic::round:
     case Intrinsic::roundeven:
       return (FMF.noInfs() && FMF.noNaNs()) ||
-             isKnownNeverInfOrNaN(I, /*Depth=*/0, SimplifyQuery(DL));
+             isKnownNeverInfOrNaN(I, SimplifyQuery(DL));
     default:
       break;
     }
@@ -764,7 +764,7 @@ bool AMDGPULibCalls::fold(CallInst *CI) {
       // TODO: Account for flags on current call
       if (PowrFunc &&
           cannotBeOrderedLessThanZero(
-              FPOp->getOperand(0), /*Depth=*/0,
+              FPOp->getOperand(0),
               SimplifyQuery(M->getDataLayout(), TLInfo, DT, AC, Call))) {
         Call->setCalledFunction(PowrFunc);
         return fold_pow(FPOp, B, PowrInfo) || true;
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 0f002b016af0..67db961e60fa 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -2364,7 +2364,6 @@ static Function *moveFunctionAdaptingType(Function *OldF, FunctionType *NewTy,
   bool IsIntrinsic = OldF->isIntrinsic();
   Function *NewF =
       Function::Create(NewTy, OldF->getLinkage(), OldF->getAddressSpace());
-  NewF->IsNewDbgInfoFormat = OldF->IsNewDbgInfoFormat;
   NewF->copyAttributesFrom(OldF);
   NewF->copyMetadata(OldF, 0);
   NewF->takeName(OldF);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
index 7d2596d66618..593b3ab22038 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
@@ -97,6 +97,86 @@ MCSymbol *MCResourceInfo::getMaxSGPRSymbol(MCContext &OutContext) {
   return OutContext.getOrCreateSymbol("amdgpu.max_num_sgpr");
 }
 
+// Tries to flatten recursive call register resource gathering. Simple cycle
+// avoiding dfs to find the constants in the propagated symbols.
+// Assumes:
+// - RecSym has been confirmed to recurse (this means the callee symbols should
+//   all be populated, started at RecSym).
+// - Shape of the resource symbol's MCExpr (`max` args are order agnostic):
+//   RecSym.MCExpr := max(<constant>+, <callee_symbol>*)
+const MCExpr *MCResourceInfo::flattenedCycleMax(MCSymbol *RecSym,
+                                                ResourceInfoKind RIK,
+                                                MCContext &OutContext) {
+  SmallPtrSet<const MCExpr *, 8> Seen;
+  SmallVector<const MCExpr *, 8> WorkList;
+  int64_t Maximum = 0;
+
+  const MCExpr *RecExpr = RecSym->getVariableValue();
+  WorkList.push_back(RecExpr);
+
+  while (!WorkList.empty()) {
+    const MCExpr *CurExpr = WorkList.pop_back_val();
+    switch (CurExpr->getKind()) {
+    default: {
+      // Assuming the recursion is of shape `max(<constant>, <callee_symbol>)`
+      // where <callee_symbol> will eventually recurse. If this condition holds,
+      // the recursion occurs within some other (possibly unresolvable) MCExpr,
+      // thus using the worst case value then.
+      if (!AMDGPUMCExpr::isSymbolUsedInExpression(RecSym, CurExpr)) {
+        LLVM_DEBUG(dbgs() << "MCResUse:   " << RecSym->getName()
+                          << ": Recursion in unexpected sub-expression, using "
+                             "module maximum\n");
+        switch (RIK) {
+        default:
+          break;
+        case RIK_NumVGPR:
+          return MCSymbolRefExpr::create(getMaxVGPRSymbol(OutContext),
+                                         OutContext);
+          break;
+        case RIK_NumSGPR:
+          return MCSymbolRefExpr::create(getMaxSGPRSymbol(OutContext),
+                                         OutContext);
+          break;
+        case RIK_NumAGPR:
+          return MCSymbolRefExpr::create(getMaxAGPRSymbol(OutContext),
+                                         OutContext);
+          break;
+        }
+      }
+      break;
+    }
+    case MCExpr::ExprKind::Constant: {
+      int64_t Val = cast<MCConstantExpr>(CurExpr)->getValue();
+      Maximum = std::max(Maximum, Val);
+      break;
+    }
+    case MCExpr::ExprKind::SymbolRef: {
+      const MCSymbolRefExpr *SymExpr = cast<MCSymbolRefExpr>(CurExpr);
+      const MCSymbol &SymRef = SymExpr->getSymbol();
+      if (SymRef.isVariable()) {
+        const MCExpr *SymVal = SymRef.getVariableValue();
+        if (Seen.insert(SymVal).second)
+          WorkList.push_back(SymVal);
+      }
+      break;
+    }
+    case MCExpr::ExprKind::Target: {
+      const AMDGPUMCExpr *TargetExpr = cast<AMDGPUMCExpr>(CurExpr);
+      if (TargetExpr->getKind() == AMDGPUMCExpr::VariantKind::AGVK_Max) {
+        for (auto &Arg : TargetExpr->getArgs())
+          WorkList.push_back(Arg);
+      }
+      break;
+    }
+    }
+  }
+
+  LLVM_DEBUG(dbgs() << "MCResUse:   " << RecSym->getName()
+                    << ": Using flattened max: << " << Maximum << '\n');
+
+  return MCConstantExpr::create(Maximum, OutContext);
+}
+
 void MCResourceInfo::assignResourceInfoExpr(
     int64_t LocalValue, ResourceInfoKind RIK, AMDGPUMCExpr::VariantKind Kind,
     const MachineFunction &MF, const SmallVectorImpl<const Function *> &Callees,
@@ -127,30 +207,25 @@ void MCResourceInfo::assignResourceInfoExpr(
       // Avoid constructing recursive definitions by detecting whether `Sym` is
       // found transitively within any of its `CalleeValSym`.
       if (!CalleeValSym->isVariable() ||
-          !CalleeValSym->getVariableValue()->isSymbolUsedInExpression(Sym)) {
+          !AMDGPUMCExpr::isSymbolUsedInExpression(
+              Sym, CalleeValSym->getVariableValue())) {
         LLVM_DEBUG(dbgs() << "MCResUse:   " << Sym->getName() << ": Adding "
                           << CalleeValSym->getName() << " as callee\n");
         ArgExprs.push_back(MCSymbolRefExpr::create(CalleeValSym, OutContext));
       } else {
-        LLVM_DEBUG(
-            dbgs() << "MCResUse:   " << Sym->getName()
-                   << ": Recursion found, falling back to module maximum\n");
-        // In case of recursion: make sure to use conservative register counts
-        // (i.e., specifically for VGPR/SGPR/AGPR).
+        LLVM_DEBUG(dbgs() << "MCResUse:   " << Sym->getName()
+                          << ": Recursion found, attempt flattening of cycle "
+                             "for resource usage\n");
+        // In case of recursion for vgpr/sgpr/agpr resource usage: try to
+        // flatten and use the max of the call cycle. May still end up emitting
+        // module max if not fully resolvable.
         switch (RIK) {
         default:
           break;
         case RIK_NumVGPR:
-          ArgExprs.push_back(MCSymbolRefExpr::create(
-              getMaxVGPRSymbol(OutContext), OutContext));
-          break;
         case RIK_NumSGPR:
-          ArgExprs.push_back(MCSymbolRefExpr::create(
-              getMaxSGPRSymbol(OutContext), OutContext));
-          break;
         case RIK_NumAGPR:
-          ArgExprs.push_back(MCSymbolRefExpr::create(
-              getMaxAGPRSymbol(OutContext), OutContext));
+          ArgExprs.push_back(flattenedCycleMax(CalleeValSym, RIK, OutContext));
           break;
         }
       }
@@ -243,7 +318,8 @@ void MCResourceInfo::gatherResourceInfo(
         // Avoid constructing recursive definitions by detecting whether `Sym`
         // is found transitively within any of its `CalleeValSym`.
         if (!CalleeValSym->isVariable() ||
-            !CalleeValSym->getVariableValue()->isSymbolUsedInExpression(Sym)) {
+            !AMDGPUMCExpr::isSymbolUsedInExpression(
+                Sym, CalleeValSym->getVariableValue())) {
           LLVM_DEBUG(dbgs() << "MCResUse:   " << Sym->getName() << ": Adding "
                             << CalleeValSym->getName() << " as callee\n");
           ArgExprs.push_back(MCSymbolRefExpr::create(CalleeValSym, OutContext));
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
index a670878948c3..fa98f82d1102 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
@@ -58,6 +58,12 @@ class MCResourceInfo {
   // Assigns expression for Max S/V/A-GPRs to the referenced symbols.
   void assignMaxRegs(MCContext &OutContext);
 
+  // Take flattened max of cyclic function calls' knowns. For example, for
+  // a cycle A->B->C->D->A, take max(A, B, C, D) for A and have B, C, D have the
+  // propgated value from A.
+  const MCExpr *flattenedCycleMax(MCSymbol *RecSym, ResourceInfoKind RIK,
+                                  MCContext &OutContext);
+
 public:
   MCResourceInfo() = default;
   void addMaxVGPRCandidate(int32_t candidate) {
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index 6568d9031987..241dbd63eb5c 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -125,13 +125,6 @@ void getUsesOfLDSByFunction(const CallGraph &CG, Module &M,
 }
 
 bool isKernelLDS(const Function *F) {
-  // Some weirdness here. AMDGPU::isKernelCC does not call into
-  // AMDGPU::isKernel with the calling conv, it instead calls into
-  // isModuleEntryFunction which returns true for more calling conventions
-  // than AMDGPU::isKernel does. There's a FIXME on AMDGPU::isKernel.
-  // There's also a test that checks that the LDS lowering does not hit on
-  // a graphics shader, denoted amdgpu_ps, so stay with the limited case.
-  // Putting LDS in the name of the function to draw attention to this.
   return AMDGPU::isKernel(F->getCallingConv());
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp
index 5027705ef61d..984c1ee89309 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp
@@ -134,7 +134,6 @@ class PreloadKernelArgInfo {
 
     NF->copyAttributesFrom(&F);
     NF->copyMetadata(&F, 0);
-    NF->setIsNewDbgInfoFormat(F.IsNewDbgInfoFormat);
 
     F.getParent()->getFunctionList().insert(F.getIterator(), NF);
     NF->takeName(&F);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index c4325fa55387..8db7d4c71a54 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -433,9 +433,34 @@ static Value *GEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca,
   unsigned BW = DL.getIndexTypeSizeInBits(GEP->getType());
   SmallMapVector<Value *, APInt, 4> VarOffsets;
   APInt ConstOffset(BW, 0);
-  if (GEP->getPointerOperand()->stripPointerCasts() != Alloca ||
-      !GEP->collectOffset(DL, BW, VarOffsets, ConstOffset))
-    return nullptr;
+
+  // Walk backwards through nested GEPs to collect both constant and variable
+  // offsets, so that nested vector GEP chains can be lowered in one step.
+  //
+  // Given this IR fragment as input:
+  //
+  //   %0 = alloca [10 x <2 x i32>], align 8, addrspace(5)
+  //   %1 = getelementptr [10 x <2 x i32>], ptr addrspace(5) %0, i32 0, i32 %j
+  //   %2 = getelementptr i8, ptr addrspace(5) %1, i32 4
+  //   %3 = load i32, ptr addrspace(5) %2, align 4
+  //
+  // Combine both GEP operations in a single pass, producing:
+  //   BasePtr      = %0
+  //   ConstOffset  = 4
+  //   VarOffsets   = { %j -> element_size(<2 x i32>) }
+  //
+  // That lets us emit a single buffer_load directly into a VGPR, without ever
+  // allocating scratch memory for the intermediate pointer.
+  Value *CurPtr = GEP;
+  while (auto *CurGEP = dyn_cast<GetElementPtrInst>(CurPtr)) {
+    if (!CurGEP->collectOffset(DL, BW, VarOffsets, ConstOffset))
+      return nullptr;
+
+    // Move to the next outer pointer.
+    CurPtr = CurGEP->getPointerOperand();
+  }
+
+  assert(CurPtr == Alloca && "GEP not based on alloca");
 
   unsigned VecElemSize = DL.getTypeAllocSize(VecElemTy);
   if (VarOffsets.size() > 1)
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
index e1008439a33a..4b1f80c77782 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -325,8 +325,6 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
   NewFunc->removeRetAttrs(RetAttrs);
   // TODO: How to preserve metadata?
 
-  NewFunc->setIsNewDbgInfoFormat(F.IsNewDbgInfoFormat);
-
   // Move the body of the function into the new rewritten function, and replace
   // this function with a stub.
   NewFunc->splice(NewFunc->begin(), &F);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 84d39f530303..958de56afeb9 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -752,7 +752,7 @@ StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
 llvm::ScheduleDAGInstrs *
 AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
   const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
-  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  ScheduleDAGMILive *DAG = createSchedLive(C);
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
   if (ST.shouldClusterStores())
     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
@@ -812,6 +812,16 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
 #define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
 #include "llvm/Passes/TargetPassRegistry.inc"
 
+  PB.registerPipelineStartEPCallback(
+      [this](ModulePassManager &PM, OptimizationLevel Level) {
+        PM.addPass(AMDGPUExpandFeaturePredicatesPass(*this));
+      });
+
+  PB.registerFullLinkTimeOptimizationEarlyEPCallback(
+      [this](ModulePassManager &PM, OptimizationLevel Level) {
+        PM.addPass(AMDGPUExpandFeaturePredicatesPass(*this));
+      });
+
   PB.registerScalarOptimizerLateEPCallback(
       [](FunctionPassManager &FPM, OptimizationLevel Level) {
         if (Level == OptimizationLevel::O0)
@@ -1018,7 +1028,9 @@ AMDGPUTargetMachine::mapToDWARFAddrSpace(unsigned LLVMAddrSpace) const {
 
 unsigned AMDGPUTargetMachine::getAssumedAddrSpace(const Value *V) const {
   if (auto *Arg = dyn_cast<Argument>(V);
-      Arg && AMDGPU::isKernelCC(Arg->getParent()) && !Arg->hasByRefAttr())
+      Arg &&
+      AMDGPU::isModuleEntryFunctionCC(Arg->getParent()->getCallingConv()) &&
+      !Arg->hasByRefAttr())
     return AMDGPUAS::GLOBAL_ADDRESS;
 
   const auto *LD = dyn_cast<LoadInst>(V);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index e7f52caf522a..a8ae01874908 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1106,7 +1106,7 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
         return nullptr;
 
       // TODO: Do we need to thread more context in here?
-      KnownBits Known = computeKnownBits(MaskOp, DL, 0, nullptr, II);
+      KnownBits Known = computeKnownBits(MaskOp, DL, nullptr, II);
       if (Known.countMinLeadingOnes() < 32)
         return nullptr;
 
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 95932c493232..0dc1d1377322 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1395,7 +1395,7 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   MCRegister ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
                           unsigned &RegWidth,
                           SmallVectorImpl<AsmToken> &Tokens);
-  bool ParseRegRange(unsigned& Num, unsigned& Width);
+  bool ParseRegRange(unsigned &Num, unsigned &Width, unsigned &SubReg);
   MCRegister getRegularReg(RegisterKind RegKind, unsigned RegNum,
                            unsigned SubReg, unsigned RegWidth, SMLoc Loc);
 
@@ -2857,7 +2857,8 @@ MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind, unsigned RegNum,
   return Reg;
 }
 
-bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth) {
+bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth,
+                                    unsigned &SubReg) {
   int64_t RegLo, RegHi;
   if (!skipToken(AsmToken::LBrac, "missing register index"))
     return false;
@@ -2894,8 +2895,20 @@ bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth) {
     return false;
   }
 
+  if (RegHi == RegLo) {
+    StringRef RegSuffix = getTokenStr();
+    if (RegSuffix == ".l") {
+      SubReg = AMDGPU::lo16;
+      lex();
+    } else if (RegSuffix == ".h") {
+      SubReg = AMDGPU::hi16;
+      lex();
+    }
+  }
+
   Num = static_cast<unsigned>(RegLo);
   RegWidth = 32 * ((RegHi - RegLo) + 1);
+
   return true;
 }
 
@@ -2949,7 +2962,7 @@ MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
     RegWidth = 32;
   } else {
     // Range of registers: v[XX:YY]. ":YY" is optional.
-    if (!ParseRegRange(RegNum, RegWidth))
+    if (!ParseRegRange(RegNum, RegWidth, SubReg))
       return MCRegister();
   }
 
@@ -8826,6 +8839,7 @@ void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
   OptionalImmIndexMap OptionalIdx;
   unsigned Opc = Inst.getOpcode();
   unsigned I = 1;
+  int CbszOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::cbsz);
 
   const MCInstrDesc &Desc = MII.get(Opc);
 
@@ -8834,8 +8848,15 @@ void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
 
   for (unsigned E = Operands.size(); I != E; ++I) {
     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[I]);
-
-    if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
+    int NumOperands = Inst.getNumOperands();
+    // The order of operands in MCInst and parsed operands are different.
+    // Adding dummy cbsz and blgp operands at corresponding MCInst operand
+    // indices for parsing scale values correctly.
+    if (NumOperands == CbszOpIdx) {
+      Inst.addOperand(MCOperand::createImm(0));
+      Inst.addOperand(MCOperand::createImm(0));
+    }
+    if (isRegOrImmWithInputMods(Desc, NumOperands)) {
       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
     } else if (Op.isImmModifier()) {
       OptionalIdx[Op.getImmTy()] = I;
@@ -8845,12 +8866,18 @@ void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
   }
 
   // Insert CBSZ and BLGP operands for F8F6F4 variants
-  int InsertPos = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::cbsz);
-  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCBSZ,
-                        0, InsertPos);
-  InsertPos = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::blgp);
-  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyBLGP,
-                        0, InsertPos);
+  auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
+  if (CbszIdx != OptionalIdx.end()) {
+    int CbszVal = ((AMDGPUOperand &)*Operands[CbszIdx->second]).getImm();
+    Inst.getOperand(CbszOpIdx).setImm(CbszVal);
+  }
+
+  int BlgpOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::blgp);
+  auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
+  if (BlgpIdx != OptionalIdx.end()) {
+    int BlgpVal = ((AMDGPUOperand &)*Operands[BlgpIdx->second]).getImm();
+    Inst.getOperand(BlgpOpIdx).setImm(BlgpVal);
+  }
 
   // Add dummy src_modifiers
   Inst.addOperand(MCOperand::createImm(0));
@@ -8961,10 +8988,14 @@ void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
   // Adding vdst_in operand is already covered for these DPP instructions in
   // cvtVOP3DPP.
   if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vdst_in) &&
-      !(Opc == AMDGPU::V_CVT_PK_BF8_F32_e64_dpp_gfx12 ||
-        Opc == AMDGPU::V_CVT_PK_FP8_F32_e64_dpp_gfx12 ||
-        Opc == AMDGPU::V_CVT_PK_BF8_F32_e64_dpp8_gfx12 ||
-        Opc == AMDGPU::V_CVT_PK_FP8_F32_e64_dpp8_gfx12 ||
+      !(Opc == AMDGPU::V_CVT_PK_BF8_F32_t16_e64_dpp_gfx12 ||
+        Opc == AMDGPU::V_CVT_PK_FP8_F32_t16_e64_dpp_gfx12 ||
+        Opc == AMDGPU::V_CVT_PK_BF8_F32_t16_e64_dpp8_gfx12 ||
+        Opc == AMDGPU::V_CVT_PK_FP8_F32_t16_e64_dpp8_gfx12 ||
+        Opc == AMDGPU::V_CVT_PK_BF8_F32_fake16_e64_dpp_gfx12 ||
+        Opc == AMDGPU::V_CVT_PK_FP8_F32_fake16_e64_dpp_gfx12 ||
+        Opc == AMDGPU::V_CVT_PK_BF8_F32_fake16_e64_dpp8_gfx12 ||
+        Opc == AMDGPU::V_CVT_PK_FP8_F32_fake16_e64_dpp8_gfx12 ||
         Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
         Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
         Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/FLATInstructions.td b/external/llvm-project/llvm/lib/Target/AMDGPU/FLATInstructions.td
index c17fda134611..88b694862d37 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1537,17 +1537,13 @@ multiclass ScratchFLATLoadPats_D16_t16<string inst, SDPatternOperator node, Valu
 let OtherPredicates = [HasFlatAddressSpace] in {
 
 def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i32>;
-def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>;
 def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i32>;
-def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>;
 def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_aext_16_flat, i32>;
-def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>;
 def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i32>;
-def : FlatLoadPat <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>;
 def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>;
@@ -1560,8 +1556,14 @@ let True16Predicate = p in {
   def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>;
   def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>;
   def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>;
+  def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>;
+  def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>;
+  def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>;
+  def : FlatLoadPat <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>;
   def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>;
   def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>;
+  def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
+  def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
 }
 
 let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts in {
@@ -1569,8 +1571,14 @@ let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predi
   def : FlatLoadPat_D16_t16<FLAT_LOAD_UBYTE_D16_t16, zextloadi8_flat, i16>;
   def : FlatLoadPat_D16_t16<FLAT_LOAD_SBYTE_D16_t16, sextloadi8_flat, i16>;
   def : FlatLoadPat_D16_t16<FLAT_LOAD_SHORT_D16_t16, load_flat, i16>;
+  def : FlatLoadPat_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_aext_8_flat, i16>;
+  def : FlatLoadPat_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_zext_8_flat, i16>;
+  def : FlatLoadPat_D16_t16<FLAT_LOAD_SHORT_D16_t16, atomic_load_nonext_16_flat, i16>;
+  def : FlatLoadPat_D16_t16<FLAT_LOAD_SBYTE_D16_t16, atomic_load_sext_8_flat, i16>;
   def : FlatStorePat <FLAT_STORE_BYTE_t16, truncstorei8_flat, i16>;
   def : FlatStorePat <FLAT_STORE_SHORT_t16, store_flat, i16>;
+  def : FlatStorePat <FLAT_STORE_BYTE_t16, atomic_store_8_flat, i16>;
+  def : FlatStorePat <FLAT_STORE_SHORT_t16, atomic_store_16_flat, i16>;
 } // End let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts
 
 def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;
@@ -1599,9 +1607,7 @@ def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>;
 def : FlatStorePat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
 def : FlatStorePat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
 def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
-def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
 def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
-def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
 
 foreach as = [ "flat", "global" ] in {
 defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>;
@@ -1678,15 +1684,10 @@ def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>;
 let OtherPredicates = [HasFlatGlobalInsts] in {
 
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_aext_16_global, i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i32>;
-defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>;
@@ -1702,6 +1703,11 @@ defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, atomic_load_sext_16_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>;
+defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>;
 }
 
 let OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in {
@@ -1709,8 +1715,15 @@ defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", extloadi8_global, i16
 defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", zextloadi8_global, i16>;
 defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SBYTE_D16", sextloadi8_global, i16>;
 defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", load_global, i16>;
+defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_aext_8_global, i16>;
+defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_zext_8_global, i16>;
+defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SBYTE_D16", atomic_load_sext_8_global, i16>;
+defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_nonext_16_global, i16>;
+defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_zext_16_global, i16>;
 defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", truncstorei8_global, i16>;
 defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", store_global, i16>;
+defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", atomic_store_8_global, i16>;
+defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>;
 } // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts
 
 foreach vt = Reg32Types.types in {
@@ -1744,6 +1757,8 @@ foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
 let OtherPredicates = [HasFlatGlobalInsts], True16Predicate =  p in {
 defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;
 }
 
 let OtherPredicates = [HasD16LoadStore] in {
@@ -1769,9 +1784,7 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>
 }
 
 defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
-defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
 defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
 
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 1561efe2cd29..bc95d3f040e1 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -417,7 +417,8 @@ void GCNHazardRecognizer::AdvanceCycle() {
 }
 
 void GCNHazardRecognizer::RecedeCycle() {
-  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
+  assert(!IsHazardRecognizerMode &&
+         "Bottom-up scheduling shouldn't run in hazard recognizer mode");
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index f74d12cfab0c..ce213b91b1f7 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -35,17 +35,9 @@ bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
 ///////////////////////////////////////////////////////////////////////////////
 // GCNRegPressure
 
-unsigned GCNRegPressure::getRegKind(Register Reg,
-                                    const MachineRegisterInfo &MRI) {
-  assert(Reg.isVirtual());
-  const auto *const RC = MRI.getRegClass(Reg);
-  const auto *STI =
-      static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
-  return STI->isSGPRClass(RC)
-             ? (STI->getRegSizeInBits(*RC) == 32 ? SGPR32 : SGPR_TUPLE)
-         : STI->isAGPRClass(RC)
-             ? (STI->getRegSizeInBits(*RC) == 32 ? AGPR32 : AGPR_TUPLE)
-             : (STI->getRegSizeInBits(*RC) == 32 ? VGPR32 : VGPR_TUPLE);
+unsigned GCNRegPressure::getRegKind(const TargetRegisterClass *RC,
+                                    const SIRegisterInfo *STI) {
+  return STI->isSGPRClass(RC) ? SGPR : (STI->isAGPRClass(RC) ? AGPR : VGPR);
 }
 
 void GCNRegPressure::inc(unsigned Reg,
@@ -61,32 +53,22 @@ void GCNRegPressure::inc(unsigned Reg,
     std::swap(NewMask, PrevMask);
     Sign = -1;
   }
+  assert(PrevMask < NewMask && "prev mask should always be lesser than new");
 
-  switch (auto Kind = getRegKind(Reg, MRI)) {
-  case SGPR32:
-  case VGPR32:
-  case AGPR32:
-    Value[Kind] += Sign;
-    break;
-
-  case SGPR_TUPLE:
-  case VGPR_TUPLE:
-  case AGPR_TUPLE:
-    assert(PrevMask < NewMask);
-
-    Value[Kind == SGPR_TUPLE ? SGPR32 : Kind == AGPR_TUPLE ? AGPR32 : VGPR32] +=
-      Sign * SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
-
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+  const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+  const SIRegisterInfo *STI = static_cast<const SIRegisterInfo *>(TRI);
+  unsigned RegKind = getRegKind(RC, STI);
+  if (TRI->getRegSizeInBits(*RC) != 32) {
+    // Reg is from a tuple register class.
     if (PrevMask.none()) {
-      assert(NewMask.any());
-      const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
-      Value[Kind] +=
-          Sign * TRI->getRegClassWeight(MRI.getRegClass(Reg)).RegWeight;
+      unsigned TupleIdx = TOTAL_KINDS + RegKind;
+      Value[TupleIdx] += Sign * TRI->getRegClassWeight(RC).RegWeight;
     }
-    break;
-
-  default: llvm_unreachable("Unknown register kind");
+    // Pressure scales with number of new registers covered by the new mask.
+    Sign *= SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
   }
+  Value[RegKind] += Sign;
 }
 
 bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,
@@ -226,7 +208,7 @@ bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,
 
 Printable llvm::print(const GCNRegPressure &RP, const GCNSubtarget *ST) {
   return Printable([&RP, ST](raw_ostream &OS) {
-    OS << "VGPRs: " << RP.Value[GCNRegPressure::VGPR32] << ' '
+    OS << "VGPRs: " << RP.getArchVGPRNum() << ' '
        << "AGPRs: " << RP.getAGPRNum();
     if (ST)
       OS << "(O"
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 4d1e96ff04e8..f3d7983056cf 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -29,34 +29,26 @@ class raw_ostream;
 class SlotIndex;
 
 struct GCNRegPressure {
-  enum RegKind {
-    SGPR32,
-    SGPR_TUPLE,
-    VGPR32,
-    VGPR_TUPLE,
-    AGPR32,
-    AGPR_TUPLE,
-    TOTAL_KINDS
-  };
+  enum RegKind { SGPR, VGPR, AGPR, TOTAL_KINDS };
 
   GCNRegPressure() {
     clear();
   }
 
-  bool empty() const { return getSGPRNum() == 0 && getVGPRNum(false) == 0; }
+  bool empty() const { return !Value[SGPR] && !Value[VGPR] && !Value[AGPR]; }
 
-  void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); }
+  void clear() { std::fill(&Value[0], &Value[ValueArraySize], 0); }
 
   /// \returns the SGPR32 pressure
-  unsigned getSGPRNum() const { return Value[SGPR32]; }
+  unsigned getSGPRNum() const { return Value[SGPR]; }
   /// \returns the aggregated ArchVGPR32, AccVGPR32 pressure dependent upon \p
   /// UnifiedVGPRFile
   unsigned getVGPRNum(bool UnifiedVGPRFile) const {
     if (UnifiedVGPRFile) {
-      return Value[AGPR32] ? getUnifiedVGPRNum(Value[VGPR32], Value[AGPR32])
-                           : Value[VGPR32] + Value[AGPR32];
+      return Value[AGPR] ? getUnifiedVGPRNum(Value[VGPR], Value[AGPR])
+                         : Value[VGPR];
     }
-    return std::max(Value[VGPR32], Value[AGPR32]);
+    return std::max(Value[VGPR], Value[AGPR]);
   }
 
   /// Returns the aggregated VGPR pressure, assuming \p NumArchVGPRs ArchVGPRs
@@ -68,13 +60,14 @@ struct GCNRegPressure {
   }
 
   /// \returns the ArchVGPR32 pressure
-  unsigned getArchVGPRNum() const { return Value[VGPR32]; }
+  unsigned getArchVGPRNum() const { return Value[VGPR]; }
   /// \returns the AccVGPR32 pressure
-  unsigned getAGPRNum() const { return Value[AGPR32]; }
+  unsigned getAGPRNum() const { return Value[AGPR]; }
 
-  unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE],
-                                                         Value[AGPR_TUPLE]); }
-  unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; }
+  unsigned getVGPRTuplesWeight() const {
+    return std::max(Value[TOTAL_KINDS + VGPR], Value[TOTAL_KINDS + AGPR]);
+  }
+  unsigned getSGPRTuplesWeight() const { return Value[TOTAL_KINDS + SGPR]; }
 
   unsigned getOccupancy(const GCNSubtarget &ST) const {
     return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()),
@@ -106,7 +99,7 @@ struct GCNRegPressure {
             unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
 
   bool operator==(const GCNRegPressure &O) const {
-    return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value);
+    return std::equal(&Value[0], &Value[ValueArraySize], O.Value);
   }
 
   bool operator!=(const GCNRegPressure &O) const {
@@ -114,13 +107,13 @@ struct GCNRegPressure {
   }
 
   GCNRegPressure &operator+=(const GCNRegPressure &RHS) {
-    for (unsigned I = 0; I < TOTAL_KINDS; ++I)
+    for (unsigned I = 0; I < ValueArraySize; ++I)
       Value[I] += RHS.Value[I];
     return *this;
   }
 
   GCNRegPressure &operator-=(const GCNRegPressure &RHS) {
-    for (unsigned I = 0; I < TOTAL_KINDS; ++I)
+    for (unsigned I = 0; I < ValueArraySize; ++I)
       Value[I] -= RHS.Value[I];
     return *this;
   }
@@ -128,9 +121,14 @@ struct GCNRegPressure {
   void dump() const;
 
 private:
-  unsigned Value[TOTAL_KINDS];
+  static constexpr unsigned ValueArraySize = TOTAL_KINDS * 2;
+
+  /// Pressure for all register kinds (first all regular registers kinds, then
+  /// all tuple register kinds).
+  unsigned Value[ValueArraySize];
 
-  static unsigned getRegKind(Register Reg, const MachineRegisterInfo &MRI);
+  static unsigned getRegKind(const TargetRegisterClass *RC,
+                             const SIRegisterInfo *STI);
 
   friend GCNRegPressure max(const GCNRegPressure &P1,
                             const GCNRegPressure &P2);
@@ -140,7 +138,7 @@ struct GCNRegPressure {
 
 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) {
   GCNRegPressure Res;
-  for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I)
+  for (unsigned I = 0; I < GCNRegPressure::ValueArraySize; ++I)
     Res.Value[I] = std::max(P1.Value[I], P2.Value[I]);
   return Res;
 }
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 706ae92c9e47..7165cf89ca45 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -589,12 +589,11 @@ bool GCNMaxILPSchedStrategy::tryCandidate(SchedCandidate &Cand,
   // This is a best effort to set things up for a post-RA pass. Optimizations
   // like generating loads of multiple registers should ideally be done within
   // the scheduler pass by combining the loads during DAG postprocessing.
-  const SUnit *CandNextClusterSU =
-      Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  const SUnit *TryCandNextClusterSU =
-      TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
-                 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
+  const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
+  const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
+  if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
+                 CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
+                 Cluster))
     return TryCand.Reason != NoCand;
 
   // Avoid increasing the max critical pressure in the scheduled region.
@@ -664,12 +663,11 @@ bool GCNMaxMemoryClauseSchedStrategy::tryCandidate(SchedCandidate &Cand,
 
   // MaxMemoryClause-specific: We prioritize clustered instructions as we would
   // get more benefit from clausing these memory instructions.
-  const SUnit *CandNextClusterSU =
-      Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  const SUnit *TryCandNextClusterSU =
-      TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
-                 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
+  const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
+  const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
+  if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
+                 CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
+                 Cluster))
     return TryCand.Reason != NoCand;
 
   // We only compare a subset of features when comparing nodes between
@@ -2248,8 +2246,8 @@ void PreRARematStage::finalizeGCNSchedStage() {
 void GCNScheduleDAGMILive::updateRegionBoundaries(
     RegionBoundaries &RegionBounds, MachineBasicBlock::iterator MI,
     MachineInstr *NewMI) {
-  assert(!NewMI ||
-         NewMI != RegionBounds.second && "cannot remove at region end");
+  assert((!NewMI || NewMI != RegionBounds.second) &&
+         "cannot remove at region end");
 
   if (RegionBounds.first == RegionBounds.second) {
     assert(NewMI && "cannot remove from an empty region");
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index a56bca514aff..a6ce42dca92b 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -12,6 +12,7 @@
 #include "SIDefines.h"
 #include "Utils/AMDGPUAsmUtils.h"
 #include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrDesc.h"
@@ -51,7 +52,7 @@ void AMDGPUInstPrinter::printU16ImmOperand(const MCInst *MI, unsigned OpNo,
                                            raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
   if (Op.isExpr()) {
-    Op.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Op.getExpr());
     return;
   }
 
@@ -787,7 +788,7 @@ void AMDGPUInstPrinter::printRegularOperand(const MCInst *MI, unsigned OpNo,
     }
   } else if (Op.isExpr()) {
     const MCExpr *Exp = Op.getExpr();
-    Exp->print(O, &MAI);
+    MAI.printExpr(O, *Exp);
   } else {
     O << "/*INV_OP*/";
   }
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
index 8ef31b82bf1f..5032b7a9ca87 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
@@ -42,6 +42,7 @@ AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(const Triple &TT,
   CommentString = ";";
   InlineAsmStart = ";#ASMSTART";
   InlineAsmEnd = ";#ASMEND";
+  UsesSetToEquateSymbol = true;
 
   //===--- Data Emission Directives -------------------------------------===//
   UsesELFSectionDirectiveForBSS = true;
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index 34fe093a931d..ab7e51ac2832 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -547,6 +547,7 @@ static bool needsPCRel(const MCExpr *Expr) {
   }
   case MCExpr::Unary:
     return needsPCRel(cast<MCUnaryExpr>(Expr)->getSubExpr());
+  case MCExpr::Specifier:
   case MCExpr::Target:
   case MCExpr::Constant:
     return false;
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
index 8eecfb99c88d..dc1445621c7a 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
@@ -77,7 +77,7 @@ void AMDGPUMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
     break;
   }
   for (const auto *It = Args.begin(); It != Args.end(); ++It) {
-    (*It)->print(OS, MAI);
+    MAI->printExpr(OS, **It);
     if ((It + 1) != Args.end())
       OS << ", ";
   }
@@ -214,6 +214,37 @@ bool AMDGPUMCExpr::evaluateOccupancy(MCValue &Res,
   return true;
 }
 
+bool AMDGPUMCExpr::isSymbolUsedInExpression(const MCSymbol *Sym,
+                                            const MCExpr *E) {
+  switch (E->getKind()) {
+  case MCExpr::Constant:
+    return false;
+  case MCExpr::Unary:
+    return isSymbolUsedInExpression(
+        Sym, static_cast<const MCUnaryExpr *>(E)->getSubExpr());
+  case MCExpr::Binary: {
+    const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(E);
+    return isSymbolUsedInExpression(Sym, BE->getLHS()) ||
+           isSymbolUsedInExpression(Sym, BE->getRHS());
+  }
+  case MCExpr::SymbolRef: {
+    const MCSymbol &S = static_cast<const MCSymbolRefExpr *>(E)->getSymbol();
+    if (S.isVariable())
+      return isSymbolUsedInExpression(Sym, S.getVariableValue());
+    return &S == Sym;
+  }
+  case MCExpr::Specifier:
+  case MCExpr::Target: {
+    auto *TE = static_cast<const AMDGPUMCExpr *>(E);
+    for (const MCExpr *E : TE->getArgs())
+      if (isSymbolUsedInExpression(Sym, E))
+        return true;
+    return false;
+  }
+  }
+  llvm_unreachable("Unknown expr kind!");
+}
+
 bool AMDGPUMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                              const MCAssembler *Asm) const {
   std::optional<int64_t> Total;
@@ -303,14 +334,6 @@ const AMDGPUMCExpr *AMDGPUMCExpr::createOccupancy(unsigned InitOcc,
                 Ctx);
 }
 
-bool AMDGPUMCExpr::isSymbolUsedInExpression(const MCSymbol *Sym) const {
-  for (const MCExpr *E : getArgs()) {
-    if (E->isSymbolUsedInExpression(Sym))
-      return true;
-  }
-  return false;
-}
-
 static KnownBits fromOptionalToKnownBits(std::optional<bool> CompareResult) {
   static constexpr unsigned BitWidth = 64;
   const APInt True(BitWidth, 1);
@@ -557,6 +580,8 @@ static void knownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
   case MCExpr::ExprKind::Target: {
     targetOpKnownBitsMapHelper(Expr, KBM, Depth);
     return;
+  case MCExpr::Specifier:
+    llvm_unreachable("unused by this backend");
   }
   }
 }
@@ -684,5 +709,5 @@ void llvm::AMDGPU::printAMDGPUMCExpr(const MCExpr *Expr, raw_ostream &OS,
     return;
   }
 
-  Expr->print(OS, MAI);
+  MAI->printExpr(OS, *Expr);
 }
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
index f38320ae7985..0da31b344de5 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
@@ -106,13 +106,13 @@ class AMDGPUMCExpr : public MCTargetExpr {
   void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
   bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAssembler *Asm) const override;
-  bool isSymbolUsedInExpression(const MCSymbol *Sym) const override;
   void visitUsedExpr(MCStreamer &Streamer) const override;
   MCFragment *findAssociatedFragment() const override;
 
   static bool classof(const MCExpr *E) {
     return E->getKind() == MCExpr::Target;
   }
+  static bool isSymbolUsedInExpression(const MCSymbol *Sym, const MCExpr *E);
 };
 
 namespace AMDGPU {
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index a6c97a02cb95..6d69bb75f293 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -18,6 +18,7 @@
 #include "Utils/AMDKernelCodeTUtils.h"
 #include "llvm/BinaryFormat/AMDGPUMetadataVerifier.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
@@ -282,7 +283,7 @@ void AMDGPUTargetAsmStreamer::EmitMCResourceInfo(
   OS << "\t.set ";                                                             \
   ARG->print(OS, getContext().getAsmInfo());                                   \
   OS << ", ";                                                                  \
-  ARG->getVariableValue()->print(OS, getContext().getAsmInfo());               \
+  getContext().getAsmInfo()->printExpr(OS, *ARG->getVariableValue());          \
   Streamer.addBlankLine();
 
   PRINT_RES_INFO(NumVGPR);
@@ -304,7 +305,7 @@ void AMDGPUTargetAsmStreamer::EmitMCResourceMaximums(const MCSymbol *MaxVGPR,
   OS << "\t.set ";                                                             \
   ARG->print(OS, getContext().getAsmInfo());                                   \
   OS << ", ";                                                                  \
-  ARG->getVariableValue()->print(OS, getContext().getAsmInfo());               \
+  getContext().getAsmInfo()->printExpr(OS, *ARG->getVariableValue());          \
   Streamer.addBlankLine();
 
   PRINT_RES_INFO(MaxVGPR);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/R600InstPrinter.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/R600InstPrinter.cpp
index 71e06dbbd151..46728e59a644 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/R600InstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/MCTargetDesc/R600InstPrinter.cpp
@@ -10,6 +10,7 @@
 #include "R600InstPrinter.h"
 #include "AMDGPUInstPrinter.h"
 #include "R600MCTargetDesc.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -98,7 +99,8 @@ void R600InstPrinter::printLiteral(const MCInst *MI, unsigned OpNo,
     O << Imm << '(' << llvm::bit_cast<float>(static_cast<uint32_t>(Imm)) << ')';
   }
   if (Op.isExpr()) {
-    Op.getExpr()->print(O << '@', &MAI);
+    O << '@';
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -160,7 +162,7 @@ void R600InstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     }
   } else if (Op.isExpr()) {
     const MCExpr *Exp = Op.getExpr();
-    Exp->print(O, &MAI);
+    MAI.printExpr(O, *Exp);
   } else {
     O << "/*INV_OP*/";
   }
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
index 824bbcfb48cf..7f75f2797124 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
@@ -324,11 +324,8 @@ bool R600VectorRegMerger::runOnMachineFunction(MachineFunction &Fn) {
       if (MI.getOpcode() != R600::REG_SEQUENCE) {
         if (TII->get(MI.getOpcode()).TSFlags & R600_InstFlag::TEX_INST) {
           Register Reg = MI.getOperand(1).getReg();
-          for (MachineRegisterInfo::def_instr_iterator
-               It = MRI->def_instr_begin(Reg), E = MRI->def_instr_end();
-               It != E; ++It) {
-            RemoveMI(&(*It));
-          }
+          for (MachineInstr &DefMI : MRI->def_instructions(Reg))
+            RemoveMI(&DefMI);
         }
         continue;
       }
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index d0af676cd2c4..4715906fc8e0 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -23,6 +23,11 @@
 #define DEBUG_TYPE "si-fold-operands"
 using namespace llvm;
 
+static cl::opt<int> SIFoldOperandsPreheaderThreshold(
+    "amdgpu-si-fold-operands-preheader-threshold", cl::init(1000),
+    cl::desc("Threshold for operand folding hazard check. "
+             "Defaults to 1000 MIs, upper limit 10000."));
+
 namespace {
 
 struct FoldCandidate {
@@ -1253,10 +1258,9 @@ void SIFoldOperandsImpl::foldOperand(
       }
 
       if (OpToFold.isReg() && TRI->isSGPRReg(*MRI, OpToFold.getReg())) {
-        if (execMayBeModifiedBeforeUse(*MRI,
-                                       UseMI->getOperand(UseOpIdx).getReg(),
-                                       *OpToFold.getParent(),
-                                       *UseMI))
+        if (checkIfExecMayBeModifiedBeforeUseAcrossBB(
+                *MRI, UseMI->getOperand(UseOpIdx).getReg(),
+                *OpToFold.getParent(), *UseMI, SIFoldOperandsPreheaderThreshold))
           return;
 
         // %vgpr = COPY %sgpr0
@@ -1674,6 +1678,12 @@ bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
       LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
                         << static_cast<int>(Fold.UseOpNo) << " of "
                         << *Fold.UseMI);
+
+      if (Fold.isImm() && tryConstantFoldOp(Fold.UseMI)) {
+        LLVM_DEBUG(dbgs() << "Constant folded " << *Fold.UseMI);
+        Changed = true;
+      }
+
     } else if (Fold.Commuted) {
       // Restoring instruction's original operand order if fold has failed.
       TII->commuteInstruction(*Fold.UseMI, false);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 21bd49cfe455..1a5e520681d6 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -2225,7 +2225,6 @@ bool SIFrameLowering::spillCalleeSavedRegisters(
   }
 
   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
-  SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
   SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
 
   const TargetRegisterClass *BlockRegClass =
@@ -2252,7 +2251,7 @@ bool SIFrameLowering::spillCalleeSavedRegisters(
             TII->get(AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE))
         .addReg(Reg, getKillRegState(false))
         .addFrameIndex(FrameIndex)
-        .addReg(MFI->getStackPtrOffsetReg())
+        .addReg(FuncInfo->getStackPtrOffsetReg())
         .addImm(0)
         .addImm(Mask)
         .addMemOperand(MMO);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1957e442dbab..30535ae88f7b 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -61,6 +61,14 @@ static cl::opt<bool> UseDivergentRegisterIndexing(
     cl::desc("Use indirect register addressing for divergent indexes"),
     cl::init(false));
 
+// TODO: This option should be removed once we switch to always using PTRADD in
+// the SelectionDAG.
+static cl::opt<bool> UseSelectionDAGPTRADD(
+    "amdgpu-use-sdag-ptradd", cl::Hidden,
+    cl::desc("Generate ISD::PTRADD nodes for 64-bit pointer arithmetic in the "
+             "SelectionDAG ISel"),
+    cl::init(false));
+
 static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
   return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
@@ -751,10 +759,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
                        Subtarget->hasVOP3PInsts() ? Legal : Custom);
 
-    setOperationAction(ISD::FNEG, MVT::v2f16, Legal);
+    setOperationAction(ISD::FNEG, {MVT::v2f16, MVT::v2bf16}, Legal);
     // This isn't really legal, but this avoids the legalizer unrolling it (and
     // allows matching fneg (fabs x) patterns)
-    setOperationAction(ISD::FABS, MVT::v2f16, Legal);
+    setOperationAction(ISD::FABS, {MVT::v2f16, MVT::v2bf16}, Legal);
 
     // Can do this in one BFI plus a constant materialize.
     setOperationAction(ISD::FCOPYSIGN,
@@ -926,8 +934,11 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal);
   }
 
-  if (Subtarget->hasCvtPkF16F32Inst())
-    setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
+  if (Subtarget->hasCvtPkF16F32Inst()) {
+    setOperationAction(ISD::FP_ROUND,
+                       {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
+                       Custom);
+  }
 
   setTargetDAGCombine({ISD::ADD,
                        ISD::UADDO_CARRY,
@@ -3574,21 +3585,6 @@ void SITargetLowering::passSpecialInputs(
   }
 }
 
-static bool canGuaranteeTCO(CallingConv::ID CC) {
-  return CC == CallingConv::Fast;
-}
-
-/// Return true if we might ever do TCO for calls with this calling convention.
-static bool mayTailCallThisCC(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::C:
-  case CallingConv::AMDGPU_Gfx:
-    return true;
-  default:
-    return canGuaranteeTCO(CC);
-  }
-}
-
 bool SITargetLowering::isEligibleForTailCallOptimization(
     SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
     const SmallVectorImpl<ISD::OutputArg> &Outs,
@@ -3597,7 +3593,7 @@ bool SITargetLowering::isEligibleForTailCallOptimization(
   if (AMDGPU::isChainCC(CalleeCC))
     return true;
 
-  if (!mayTailCallThisCC(CalleeCC))
+  if (!AMDGPU::mayTailCallThisCC(CalleeCC))
     return false;
 
   // For a divergent call target, we need to do a waterfall loop over the
@@ -3619,7 +3615,7 @@ bool SITargetLowering::isEligibleForTailCallOptimization(
   bool CCMatch = CallerCC == CalleeCC;
 
   if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
-    if (canGuaranteeTCO(CalleeCC) && CCMatch)
+    if (AMDGPU::canGuaranteeTCO(CalleeCC) && CCMatch)
       return true;
     return false;
   }
@@ -6911,14 +6907,35 @@ SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
                            DAG.getTargetConstant(0, DL, MVT::i32));
 }
 
+SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
+                                                SelectionDAG &DAG) const {
+  EVT DstVT = Op.getValueType();
+  unsigned NumElts = DstVT.getVectorNumElements();
+  assert(NumElts > 2 && isPowerOf2_32(NumElts));
+
+  auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
+
+  SDLoc DL(Op);
+  unsigned Opc = Op.getOpcode();
+  SDValue Flags = Op.getOperand(1);
+  EVT HalfDstVT =
+      EVT::getVectorVT(*DAG.getContext(), DstVT.getScalarType(), NumElts / 2);
+  SDValue OpLo = DAG.getNode(Opc, DL, HalfDstVT, Lo, Flags);
+  SDValue OpHi = DAG.getNode(Opc, DL, HalfDstVT, Hi, Flags);
+
+  return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, OpLo, OpHi);
+}
+
 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(0);
   EVT SrcVT = Src.getValueType();
   EVT DstVT = Op.getValueType();
 
-  if (DstVT == MVT::v2f16) {
+  if (DstVT.isVector() && DstVT.getScalarType() == MVT::f16) {
     assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
-    return SrcVT == MVT::v2f32 ? Op : SDValue();
+    if (SrcVT.getScalarType() != MVT::f32)
+      return SDValue();
+    return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
   }
 
   if (SrcVT.getScalarType() != MVT::f64)
@@ -10448,6 +10465,11 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
   }
 }
 
+bool SITargetLowering::shouldPreservePtrArith(const Function &F,
+                                              EVT PtrVT) const {
+  return UseSelectionDAGPTRADD && PtrVT == MVT::i64;
+}
+
 // The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
 // offset (the offset that is included in bounds checking and swizzling, to be
 // split between the instruction's voffset and immoffset fields) and soffset
@@ -17541,9 +17563,11 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
   // where we only insert a check for private and still use the flat instruction
   // for global and shared.
 
-  bool FullFlatEmulation = RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
-                           Subtarget->hasAtomicFaddInsts() &&
-                           RMW->getType()->isFloatTy();
+  bool FullFlatEmulation =
+      RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
+      ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
+       (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
+        RMW->getType()->isDoubleTy()));
 
   // If the return value isn't used, do not introduce a false use in the phi.
   bool ReturnValueIsUsed = !AI->use_empty();
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.h b/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 283f8136d352..d71a22722129 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -145,6 +145,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   /// Custom lowering for ISD::FP_ROUND for MVT::f16.
   SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+  SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFMINIMUM_FMAXIMUM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const;
@@ -259,6 +260,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   bool shouldExpandVectorDynExt(SDNode *N) const;
 
+  bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const override;
+
 private:
   // Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
   // the three offsets (voffset, soffset and instoffset) into the SDValue[3]
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
index 564d15bc4a00..249274c6df3c 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
@@ -42,6 +42,12 @@ using namespace llvm;
 
 #define DEBUG_TYPE "si-insert-hard-clauses"
 
+static cl::opt<unsigned>
+    HardClauseLengthLimit("amdgpu-hard-clause-length-limit",
+                          cl::desc("Maximum number of memory instructions to "
+                                   "place in the same hard clause"),
+                          cl::Hidden);
+
 namespace {
 
 enum HardClauseType {
@@ -228,11 +234,18 @@ class SIInsertHardClauses {
   }
 
   bool run(MachineFunction &MF) {
-
     ST = &MF.getSubtarget<GCNSubtarget>();
     if (!ST->hasHardClauses())
       return false;
 
+    unsigned MaxClauseLength = MF.getFunction().getFnAttributeAsParsedInteger(
+        "amdgpu-hard-clause-length-limit", 255);
+    if (HardClauseLengthLimit.getNumOccurrences())
+      MaxClauseLength = HardClauseLengthLimit;
+    MaxClauseLength = std::min(MaxClauseLength, ST->maxHardClauseLength());
+    if (MaxClauseLength <= 1)
+      return false;
+
     const SIInstrInfo *SII = ST->getInstrInfo();
     const TargetRegisterInfo *TRI = ST->getRegisterInfo();
 
@@ -255,7 +268,7 @@ class SIInsertHardClauses {
           }
         }
 
-        if (CI.Length == ST->maxHardClauseLength() ||
+        if (CI.Length == MaxClauseLength ||
             (CI.Length && Type != HARDCLAUSE_INTERNAL &&
              Type != HARDCLAUSE_IGNORE &&
              (Type != CI.Type ||
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0706a5bd911d..635edccf42fb 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5588,6 +5588,10 @@ unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
     return AMDGPU::V_ADD_CO_U32_e32;
   case AMDGPU::S_SUB_U32:
     return AMDGPU::V_SUB_CO_U32_e32;
+  case AMDGPU::S_ADD_U64_PSEUDO:
+    return AMDGPU::V_ADD_U64_PSEUDO;
+  case AMDGPU::S_SUB_U64_PSEUDO:
+    return AMDGPU::V_SUB_U64_PSEUDO;
   case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
   case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
   case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
@@ -7287,27 +7291,52 @@ bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
   return DeferredList.contains(MI);
 }
 
-// 16bit SALU use sgpr32. If a 16bit SALU get lowered to VALU in true16 mode,
-// sgpr32 is replaced to vgpr32 which is illegal in t16 inst. Need to add
-// subreg access properly. This can be removed after we have sgpr16 in place
-void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &Inst,
+// Legalize size mismatches between 16bit and 32bit registers in v2s copy
+// lowering (change spgr to vgpr).
+// This is mainly caused by 16bit SALU and 16bit VALU using reg with different
+// size. Need to legalize the size of the operands during the vgpr lowering
+// chain. This can be removed after we have sgpr16 in place
+void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI, unsigned OpIdx,
                                           MachineRegisterInfo &MRI) const {
-  unsigned Opcode = Inst.getOpcode();
-  if (!AMDGPU::isTrue16Inst(Opcode) || !ST.useRealTrue16Insts())
+  if (!ST.useRealTrue16Insts())
     return;
 
-  for (MachineOperand &Op : Inst.explicit_operands()) {
-    unsigned OpIdx = Op.getOperandNo();
-    if (!OpIdx)
-      continue;
-    if (Op.isReg() && RI.isVGPR(MRI, Op.getReg())) {
-      unsigned RCID = get(Opcode).operands()[OpIdx].RegClass;
-      const TargetRegisterClass *RC = RI.getRegClass(RCID);
-      if (RI.getRegSizeInBits(*RC) == 16) {
-        Op.setSubReg(AMDGPU::lo16);
-      }
-    }
-  }
+  unsigned Opcode = MI.getOpcode();
+  MachineBasicBlock *MBB = MI.getParent();
+  // Legalize operands and check for size mismatch
+  if (!OpIdx || OpIdx >= MI.getNumExplicitOperands() ||
+      OpIdx >= get(Opcode).getNumOperands())
+    return;
+
+  MachineOperand &Op = MI.getOperand(OpIdx);
+  if (!Op.isReg() || !Op.getReg().isVirtual())
+    return;
+
+  const TargetRegisterClass *CurrRC = MRI.getRegClass(Op.getReg());
+  if (!RI.isVGPRClass(CurrRC))
+    return;
+
+  unsigned RCID = get(Opcode).operands()[OpIdx].RegClass;
+  const TargetRegisterClass *ExpectedRC = RI.getRegClass(RCID);
+  if (RI.getMatchingSuperRegClass(CurrRC, ExpectedRC, AMDGPU::lo16)) {
+    Op.setSubReg(AMDGPU::lo16);
+  } else if (RI.getMatchingSuperRegClass(ExpectedRC, CurrRC, AMDGPU::lo16)) {
+    const DebugLoc &DL = MI.getDebugLoc();
+    Register NewDstReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+    Register Undef = MRI.createVirtualRegister(&AMDGPU::VGPR_16RegClass);
+    BuildMI(*MBB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);
+    BuildMI(*MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewDstReg)
+        .addReg(Op.getReg())
+        .addImm(AMDGPU::lo16)
+        .addReg(Undef)
+        .addImm(AMDGPU::hi16);
+    Op.setReg(NewDstReg);
+  }
+}
+void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI,
+                                          MachineRegisterInfo &MRI) const {
+  for (unsigned OpIdx = 1; OpIdx < MI.getNumExplicitOperands(); OpIdx++)
+    legalizeOperandsVALUt16(MI, OpIdx, MRI);
 }
 
 void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
@@ -7345,12 +7374,6 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
   switch (Opcode) {
   default:
     break;
-  case AMDGPU::S_ADD_U64_PSEUDO:
-    NewOpcode = AMDGPU::V_ADD_U64_PSEUDO;
-    break;
-  case AMDGPU::S_SUB_U64_PSEUDO:
-    NewOpcode = AMDGPU::V_SUB_U64_PSEUDO;
-    break;
   case AMDGPU::S_ADD_I32:
   case AMDGPU::S_SUB_I32: {
     // FIXME: The u32 versions currently selected use the carry.
@@ -7771,6 +7794,29 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
     Inst.eraseFromParent();
     return;
   }
+  case AMDGPU::V_S_EXP_F16_e64:
+  case AMDGPU::V_S_LOG_F16_e64:
+  case AMDGPU::V_S_RCP_F16_e64:
+  case AMDGPU::V_S_RSQ_F16_e64:
+  case AMDGPU::V_S_SQRT_F16_e64: {
+    const DebugLoc &DL = Inst.getDebugLoc();
+    Register NewDst = MRI.createVirtualRegister(ST.useRealTrue16Insts()
+                                                    ? &AMDGPU::VGPR_16RegClass
+                                                    : &AMDGPU::VGPR_32RegClass);
+    auto NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
+                        .addImm(0) // src0_modifiers
+                        .add(Inst.getOperand(2))
+                        .addImm(0)  // clamp
+                        .addImm(0); // omod
+    if (ST.useRealTrue16Insts())
+      NewInstr.addImm(0); // opsel0
+    MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
+    legalizeOperandsVALUt16(*NewInstr, MRI);
+    legalizeOperands(*NewInstr, MDT);
+    addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
+    Inst.eraseFromParent();
+    return;
+  }
   }
 
   if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
@@ -7829,15 +7875,14 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
       return;
     }
 
-    // If this is a v2s copy src from 16bit to 32bit,
-    // replace vgpr copy to reg_sequence
+    // If this is a v2s copy between 16bit and 32bit reg,
+    // replace vgpr copy to reg_sequence/extract_subreg
     // This can be remove after we have sgpr16 in place
     if (ST.useRealTrue16Insts() && Inst.isCopy() &&
         Inst.getOperand(1).getReg().isVirtual() &&
         RI.isVGPR(MRI, Inst.getOperand(1).getReg())) {
       const TargetRegisterClass *SrcRegRC = getOpRegClass(Inst, 1);
-      if (16 == RI.getRegSizeInBits(*SrcRegRC) &&
-          32 == RI.getRegSizeInBits(*NewDstRC)) {
+      if (RI.getMatchingSuperRegClass(NewDstRC, SrcRegRC, AMDGPU::lo16)) {
         Register NewDstReg = MRI.createVirtualRegister(NewDstRC);
         Register Undef = MRI.createVirtualRegister(&AMDGPU::VGPR_16RegClass);
         BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),
@@ -7849,7 +7894,13 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
             .addReg(Undef)
             .addImm(AMDGPU::hi16);
         Inst.eraseFromParent();
-
+        MRI.replaceRegWith(DstReg, NewDstReg);
+        addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
+        return;
+      } else if (RI.getMatchingSuperRegClass(SrcRegRC, NewDstRC,
+                                             AMDGPU::lo16)) {
+        Inst.getOperand(1).setSubReg(AMDGPU::lo16);
+        Register NewDstReg = MRI.createVirtualRegister(NewDstRC);
         MRI.replaceRegWith(DstReg, NewDstReg);
         addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
         return;
@@ -7945,23 +7996,6 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
     assert(NewDstRC);
     NewDstReg = MRI.createVirtualRegister(NewDstRC);
     MRI.replaceRegWith(DstReg, NewDstReg);
-
-    // Check useMI of NewInstr. If used by a true16 instruction,
-    // add a lo16 subreg access if size mismatched
-    // This can be remove after we have sgpr16 in place
-    if (ST.useRealTrue16Insts() && NewDstRC == &AMDGPU::VGPR_32RegClass) {
-      for (MachineRegisterInfo::use_iterator I = MRI.use_begin(NewDstReg),
-                                             E = MRI.use_end();
-           I != E; ++I) {
-        MachineInstr &UseMI = *I->getParent();
-        unsigned UseMIOpcode = UseMI.getOpcode();
-        if (AMDGPU::isTrue16Inst(UseMIOpcode) &&
-            (16 ==
-             RI.getRegSizeInBits(*getOpRegClass(UseMI, I.getOperandNo())))) {
-          I->setSubReg(AMDGPU::lo16);
-        }
-      }
-    }
   }
   fixImplicitOperands(*NewInstr);
 
@@ -8769,6 +8803,8 @@ void SIInstrInfo::addUsersToMoveToVALUWorklist(
         ++I;
       } while (I != E && I->getParent() == &UseMI);
     } else {
+      legalizeOperandsVALUt16(UseMI, OpNo, MRI);
+
       ++I;
     }
   }
@@ -9799,6 +9835,82 @@ MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
   return nullptr;
 }
 
+// helper function to checkIfExecMayBeModifiedBeforeUseAcrossBB and
+// execMayBeModifiedBeforeUse. This checks possible EXEC register modifications
+// for a straight-line sequence of instructions between BeginIterator and
+// EndIterator (both inclusive) upto a pre-defined limit MaxInstScan
+bool execMayBeModifiedBeforeUseUtil(
+    const TargetRegisterInfo *TRI,
+    const MachineInstrBundleIterator<const MachineInstr> BeginIterator,
+    const MachineInstrBundleIterator<const MachineInstr> EndIterator,
+    const int MaxInstScan) {
+
+  int NumInst = 0;
+  for (auto I = BeginIterator; I != EndIterator; ++I) {
+    if (I->isMetaInstruction())
+      continue;
+
+    if (++NumInst > MaxInstScan)
+      return true;
+
+    if (I->modifiesRegister(AMDGPU::EXEC, TRI))
+      return true;
+  }
+  return false;
+}
+
+// Variant of execMayBeModifiedBeforeUse(), where DefMI and UseMI belong to
+// different basic blocks. Current code is limited to a very simple case: DefMI
+// in the predecessor BB of the single BB loop where UseMI resides.
+bool llvm::checkIfExecMayBeModifiedBeforeUseAcrossBB(
+    const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI,
+    const MachineInstr &UseMI, const int SIFoldOperandsPreheaderThreshold) {
+
+  assert(MRI.isSSA() && "Must be run on SSA");
+  auto *TRI = MRI.getTargetRegisterInfo();
+  auto *DefBB = DefMI.getParent();
+  const int MaxInstScan = (SIFoldOperandsPreheaderThreshold > 10000)
+                              ? 10000
+                              : SIFoldOperandsPreheaderThreshold;
+
+  // Check whether EXEC is modified along all possible control flow between
+  // DefMI and UseMI, which may include loop backedge
+  // 1. UseBB is the only successor of DefBB
+  // 2. UseBB is a single basic block loop (only two predecessor blocks: DefBB
+  // and UseBB)
+  // 3. check if EXEC is modified
+  auto *UseBB = UseMI.getParent();
+  if (UseBB != DefBB) {
+    if (!(DefBB->isSuccessor(UseBB) && (DefBB->succ_size() == 1)))
+      return true;
+
+    if (!((UseBB->pred_size() == 2) && UseBB->isPredecessor(UseBB) &&
+          UseBB->isPredecessor(DefBB)))
+      return true;
+
+    bool canExecBeModifiedBeforeUse = execMayBeModifiedBeforeUseUtil(
+        TRI, UseBB->begin(), UseBB->end(), MaxInstScan);
+    if (canExecBeModifiedBeforeUse)
+      return true;
+
+    // Stop scan at the end of the DEF basic block.
+    // If we are here, we know for sure that the instructions in focus are in
+    // the same basic block. Scan them to be safe.
+    canExecBeModifiedBeforeUse = execMayBeModifiedBeforeUseUtil(
+        TRI, std::next(DefMI.getIterator()), DefBB->end(), MaxInstScan);
+    if (canExecBeModifiedBeforeUse)
+      return true;
+
+  } else {
+      // Stop scan at the use.
+    bool canExecBeModifiedBeforeUse = execMayBeModifiedBeforeUseUtil(
+        TRI, std::next(DefMI.getIterator()), UseMI.getIterator(), MaxInstScan);
+    if (canExecBeModifiedBeforeUse)
+      return true;
+  }
+  return false;
+}
+
 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
                                       Register VReg,
                                       const MachineInstr &DefMI,
@@ -9817,17 +9929,10 @@ bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
   int NumInst = 0;
 
   // Stop scan at the use.
-  auto E = UseMI.getIterator();
-  for (auto I = std::next(DefMI.getIterator()); I != E; ++I) {
-    if (I->isDebugInstr())
-      continue;
-
-    if (++NumInst > MaxInstScan)
-      return true;
-
-    if (I->modifiesRegister(AMDGPU::EXEC, TRI))
-      return true;
-  }
+  bool canExecBeModifiedBeforeUse = execMayBeModifiedBeforeUseUtil(
+      TRI, std::next(DefMI.getIterator()), UseMI.getIterator(), MaxInstScan);
+  if (canExecBeModifiedBeforeUse)
+    return true;
 
   return false;
 }
@@ -9864,7 +9969,7 @@ bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
   for (auto I = std::next(DefMI.getIterator()); ; ++I) {
     assert(I != DefBB->end());
 
-    if (I->isDebugInstr())
+    if (I->isMetaInstruction())
       continue;
 
     if (++NumInst > MaxInstScan)
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index e8a10e448ce3..a1de821bbb65 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1320,6 +1320,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
   void legalizeOperandsVALUt16(MachineInstr &Inst,
                                MachineRegisterInfo &MRI) const;
+  void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
+                               MachineRegisterInfo &MRI) const;
 
   /// Replace the instructions opcode with the equivalent VALU
   /// opcode.  This function will also move the users of MachineInstruntions
@@ -1561,6 +1563,10 @@ bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
                                 const MachineInstr &DefMI,
                                 const MachineInstr &UseMI);
 
+bool checkIfExecMayBeModifiedBeforeUseAcrossBB(
+    const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI,
+    const MachineInstr &UseMI, const int SIFoldOperandsPreheaderThreshold);
+    
 /// \brief Return false if EXEC is not changed between the def of \p VReg at \p
 /// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
 /// track between blocks.
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 84a6aeacc226..e74ccbee975a 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1973,7 +1973,8 @@ class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC,
                 RegisterOperand Src2RC, int NumSrcArgs,
                 bit HasClamp, bit HasModifiers, bit HasSrc2Mods, bit HasOMod,
                 Operand Src0Mod, Operand Src1Mod, Operand Src2Mod,
-                bit HasFP8ByteSel = 0, bit HasFP8DstByteSel = 0> {
+                bit HasFP8ByteSel = 0, bit HasFP8DstByteSel = 0,
+                bit HasBitOp3 = 0> {
   dag src0 = !if(!ge(NumSrcArgs, 1),
                  !if (HasModifiers,
                       (ins Src0Mod:$src0_modifiers, Src0RC:$src0),
@@ -1999,21 +2000,23 @@ class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC,
                     !con(!if(HasFP8DstByteSel, (ins VGPR_32:$vdst_in), (ins)),
                          (ins ByteSel0:$byte_sel)),
                     (ins));
+  dag bitop3 = !if(HasBitOp3, (ins bitop3_0:$bitop3), (ins));
 
-  dag ret = !con(src0, src1, src2, clamp, omod, bytesel);
+  dag ret = !con(src0, src1, src2, clamp, omod, bytesel, bitop3);
 }
 
 class getInsVOP3Base<RegisterOperand Src0RC, RegisterOperand Src1RC,
                 RegisterOperand Src2RC, int NumSrcArgs,
                 bit HasClamp, bit HasModifiers, bit HasSrc2Mods, bit HasOMod,
                 Operand Src0Mod, Operand Src1Mod, Operand Src2Mod, bit HasOpSel,
-                bit HasFP8ByteSel = 0, bit HasFP8DstByteSel = 0> {
+                bit HasFP8ByteSel = 0, bit HasFP8DstByteSel = 0, bit HasBitOp3 = 0> {
   // getInst64 handles clamp and omod. implicit mutex between vop3p and omod
   dag base = getIns64 <Src0RC, Src1RC, Src2RC, NumSrcArgs,
                 HasClamp, HasModifiers, HasSrc2Mods, HasOMod,
                 Src0Mod, Src1Mod, Src2Mod, HasFP8ByteSel, HasFP8DstByteSel>.ret;
   dag opsel = (ins op_sel0:$op_sel);
-  dag ret = !con(base, !if(HasOpSel, opsel, (ins)));
+  dag bitop3 = (ins bitop3_0:$bitop3);
+  dag ret = !con(base, !if(HasBitOp3, bitop3, (ins)), !if(HasOpSel, opsel, (ins)));
 }
 
 class getInsVOP3P <RegisterOperand Src0RC, RegisterOperand Src1RC,
@@ -2034,11 +2037,13 @@ class getInsVOP3P <RegisterOperand Src0RC, RegisterOperand Src1RC,
 class getInsVOP3OpSel <RegisterOperand Src0RC, RegisterOperand Src1RC,
                        RegisterOperand Src2RC, int NumSrcArgs,
                        bit HasClamp, bit HasOMod,
-                       Operand Src0Mod, Operand Src1Mod, Operand Src2Mod> {
+                       Operand Src0Mod, Operand Src1Mod, Operand Src2Mod,
+                       bit HasFP8ByteSel = 0, bit HasFP8DstByteSel = 0, bit HasBitOp3 = 0> {
   dag ret = getInsVOP3Base<Src0RC, Src1RC,
                     Src2RC, NumSrcArgs,
                     HasClamp, 1/*HasModifiers*/, 1/*HasSrc2Mods*/, HasOMod,
-                    Src0Mod, Src1Mod, Src2Mod, /*HasOpSel=*/1>.ret;
+                    Src0Mod, Src1Mod, Src2Mod, /*HasOpSel=*/1,
+                    HasFP8ByteSel, HasFP8DstByteSel, HasBitOp3>.ret;
 }
 
 class getInsDPPBase <RegisterOperand OldRC, RegisterOperand Src0RC, RegisterOperand Src1RC,
@@ -2237,37 +2242,6 @@ class getAsmVOP3P <bit HasDst, int NumSrcArgs, bit HasNeg,
   string ret = dst#src0#src1#src2#opsel#mods#clamp;
 }
 
-// FIXME-TRUE16 AsmVOP3OpSel will be deprecated after all
-// VOP3 16 bit instructions are replaced to true16 format
-class getAsmVOP3OpSel <int NumSrcArgs,
-                       bit HasClamp,
-                       bit HasOMod,
-                       bit Src0HasMods,
-                       bit Src1HasMods,
-                       bit Src2HasMods> {
-  string dst = "$vdst";
-
-  string isrc0 = !if(!eq(NumSrcArgs, 1), "$src0", "$src0,");
-  string isrc1 = !if(!eq(NumSrcArgs, 1), "",
-                     !if(!eq(NumSrcArgs, 2), " $src1",
-                                             " $src1,"));
-  string isrc2 = !if(!eq(NumSrcArgs, 3), " $src2", "");
-
-  string fsrc0 = !if(!eq(NumSrcArgs, 1), "$src0_modifiers", "$src0_modifiers,");
-  string fsrc1 = !if(!eq(NumSrcArgs, 1), "",
-                     !if(!eq(NumSrcArgs, 2), " $src1_modifiers",
-                                             " $src1_modifiers,"));
-  string fsrc2 = !if(!eq(NumSrcArgs, 3), " $src2_modifiers", "");
-
-  string src0 = !if(Src0HasMods, fsrc0, isrc0);
-  string src1 = !if(Src1HasMods, fsrc1, isrc1);
-  string src2 = !if(Src2HasMods, fsrc2, isrc2);
-
-  string clamp = !if(HasClamp, "$clamp", "");
-  string omod = !if(HasOMod, "$omod", "");
-  string ret = dst#", "#src0#src1#src2#"$op_sel"#clamp#omod;
-}
-
 class getAsmDPP <bit HasDst, int NumSrcArgs, bit HasModifiers, ValueType DstVT = i32> {
   string dst = !if(HasDst,
                    !if(!eq(DstVT.Size, 1),
@@ -2297,7 +2271,7 @@ class getAsmVOP3Base <int NumSrcArgs, bit HasDst, bit HasClamp,
                        bit HasOpSel, bit HasOMod, bit IsVOP3P,
                        bit HasNeg, bit Src0HasMods,
                        bit Src1HasMods, bit Src2HasMods, ValueType DstVT = i32,
-                       bit HasByteSel = 0> {
+                       bit HasByteSel = 0, bit HasBitOp3 = 0> {
   string dst = !if(HasDst,
                    !if(!eq(DstVT.Size, 1),
                        "$sdst",
@@ -2320,6 +2294,7 @@ class getAsmVOP3Base <int NumSrcArgs, bit HasDst, bit HasClamp,
   string src2 = !if(Src2HasMods, src2mods, src2nomods);
   string opsel = !if(HasOpSel, "$op_sel", "");
   string bytesel = !if(HasByteSel, "$byte_sel", "");
+  string bitop3 = !if(HasBitOp3, "$bitop3", "");
   string 3PMods = !if(IsVOP3P,
                       !if(HasOpSel, "$op_sel_hi", "")
                         #!if(HasNeg, "$neg_lo$neg_hi", ""),
@@ -2329,7 +2304,7 @@ class getAsmVOP3Base <int NumSrcArgs, bit HasDst, bit HasClamp,
 
   string ret = dst#!if(!eq(NumSrcArgs,0),
                        "",
-                       !if(HasDst,", ", "")#src0#src1#src2#opsel#bytesel#3PMods#clamp#omod);
+                       !if(HasDst,", ", "")#src0#src1#src2#bitop3#opsel#bytesel#3PMods#clamp#omod);
 }
 
 class getAsmVOP3DPP<string base> {
@@ -2554,6 +2529,7 @@ class VOPProfile <list<ValueType> _ArgVT, bit _EnableClamp = 0> {
   field bit HasFP8DstByteSel = 0;
   field bit HasFP4DstByteSel = 0;
   field bit HasFP8ByteSel = !or(HasFP8SrcByteSel, HasFP8DstByteSel);
+  field bit HasBitOp3 = 0;
 
   field bit HasDst = !ne(DstVT.Value, untyped.Value);
   field bit HasDst32 = HasDst;
@@ -2624,13 +2600,14 @@ class VOPProfile <list<ValueType> _ArgVT, bit _EnableClamp = 0> {
   field dag Ins64 = getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,
                              HasClamp, HasModifiers, HasSrc2Mods,
                              HasOMod, Src0Mod, Src1Mod, Src2Mod,
-                             HasFP8ByteSel, HasFP8DstByteSel>.ret;
+                             HasFP8ByteSel, HasFP8DstByteSel, HasBitOp3>.ret;
   field dag InsVOP3P = getInsVOP3P<Src0RC64, Src1RC64, Src2RC64,
                                    NumSrcArgs, HasClamp, HasOpSel, HasNeg,
                                    Src0PackedMod, Src1PackedMod, Src2PackedMod>.ret;
   field dag InsVOP3OpSel = getInsVOP3OpSel<Src0RC64, Src1RC64, Src2RC64,
                                 NumSrcArgs, HasClamp, HasOMod,
-                                Src0Mod, Src1Mod, Src2Mod>.ret;
+                                Src0Mod, Src1Mod, Src2Mod,
+                                HasFP8ByteSel, HasFP8DstByteSel, HasBitOp3>.ret;
   field dag InsDPP = !if(HasExtDPP,
                          getInsDPP<DstRCDPP, Src0DPP, Src1DPP, Src2DPP, NumSrcArgs,
                                    HasModifiers, Src0ModDPP, Src1ModDPP, Src2ModDPP>.ret,
@@ -2643,7 +2620,7 @@ class VOPProfile <list<ValueType> _ArgVT, bit _EnableClamp = 0> {
   defvar InsVOP3DPPBase = getInsVOP3Base<Src0VOP3DPP, Src1VOP3DPP,
                   Src2VOP3DPP, NumSrcArgs, HasClamp, HasModifiers, HasSrc2Mods, HasOMod,
                   Src0ModVOP3DPP, Src1ModVOP3DPP, Src2ModVOP3DPP, HasOpSel,
-                  HasFP8ByteSel, HasFP8DstByteSel>.ret;
+                  HasFP8ByteSel, HasFP8DstByteSel, HasBitOp3>.ret;
   defvar InsVOP3PDPPBase = getInsVOP3P<Src0VOP3DPP, Src1VOP3DPP,
                   Src2VOP3DPP, NumSrcArgs, HasClamp, HasOpSel, HasNeg,
                   Src0ModVOP3DPP, Src1ModVOP3DPP, Src2ModVOP3DPP>.ret;
@@ -2671,16 +2648,11 @@ class VOPProfile <list<ValueType> _ArgVT, bit _EnableClamp = 0> {
   // the asm operand name via this HasModifiers flag
   field string AsmDPP8 = getAsmDPP8<HasDst, NumSrcArgs, 0 /*HasModifiers*/, DstVT>.ret;
   field string AsmVOP3Base = getAsmVOP3Base<NumSrcArgs, HasDst, HasClamp,
-   HasOpSel, HasOMod, IsVOP3P, HasNeg, HasModifiers, HasModifiers,
-   HasModifiers, DstVT, HasFP8ByteSel>.ret;
+   HasOpSel, HasOMod, IsVOP3P, HasNeg, HasSrc0Mods, HasSrc1Mods,
+   HasSrc2Mods, DstVT, HasFP8ByteSel, HasBitOp3>.ret;
   field string Asm64 = AsmVOP3Base;
   field string AsmVOP3P = getAsmVOP3P<HasDst, NumSrcArgs, HasNeg, HasClamp, HasOpSel>.ret;
-  field string AsmVOP3OpSel = getAsmVOP3OpSel<NumSrcArgs,
-                                              HasClamp,
-                                              HasOMod,
-                                              HasSrc0FloatMods,
-                                              HasSrc1FloatMods,
-                                              HasSrc2FloatMods>.ret;
+  field string AsmVOP3OpSel = AsmVOP3Base;
   field string AsmVOP3DPP = getAsmVOP3DPP<AsmVOP3Base>.ret;
   field string AsmVOP3DPP16 = getAsmVOP3DPP16<AsmVOP3Base>.ret;
   field string AsmVOP3DPP8 = getAsmVOP3DPP8<AsmVOP3Base>.ret;
@@ -2822,6 +2794,7 @@ def VOP_F64_F64_I32 : VOPProfile <[f64, f64, i32, untyped]>;
 def VOP_I32_F32_F32 : VOPProfile <[i32, f32, f32, untyped]>;
 def VOP_I32_F32_I32 : VOPProfile <[i32, f32, i32, untyped]>;
 def VOP_I32_I32_I32 : VOPProfile <[i32, i32, i32, untyped]>;
+def VOP_I16_F32_F32 : VOPProfile <[i16, f32, f32, untyped]>;
 def VOP_I32_I32_I32_ARITH : VOPProfile <[i32, i32, i32, untyped], /*EnableClamp=*/1>;
 def VOP_V2F16_F32_F32 : VOPProfile <[v2f16, f32, f32, untyped]>;
 def VOP_F32_F16_F16_F16 : VOPProfile <[f32, f16, f16, f16]>;
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstructions.td b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstructions.td
index 9cecffc86231..c0da836704d2 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1399,6 +1399,20 @@ def : GCNPat <
       (i32 (V_MOV_B32_e32 (i32 0))), sub1)
 >;
 
+//===----------------------------------------------------------------------===//
+// PTRADD Patterns
+//===----------------------------------------------------------------------===//
+
+// GlobalISel shouldn't generate 64-bit addition pseudos.
+let GISelShouldIgnore = 1 in {
+def : GCNPat<
+  (DivergentBinFrag<ptradd> i64:$src0, i64:$src1),
+  (V_ADD_U64_PSEUDO $src0, $src1)>;
+def : GCNPat<
+  (UniformBinFrag<ptradd> i64:$src0, i64:$src1),
+  (S_ADD_U64_PSEUDO $src0, $src1)>;
+}
+
 /********** ============================================ **********/
 /********** Extraction, Insertion, Building and Casting  **********/
 /********** ============================================ **********/
@@ -1853,13 +1867,14 @@ def : GCNPat <
 >;
 } // End foreach fp16vt = ...
 
+foreach v2fp16vt = [v2f16, v2bf16] in {
 def : GCNPat <
-  (UniformUnaryFrag<fneg> (v2f16 SReg_32:$src)),
+  (UniformUnaryFrag<fneg> (v2fp16vt SReg_32:$src)),
   (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000)))
 >;
 
 def : GCNPat <
-  (UniformUnaryFrag<fabs> (v2f16 SReg_32:$src)),
+  (UniformUnaryFrag<fabs> (v2fp16vt SReg_32:$src)),
   (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x7fff7fff)))
 >;
 
@@ -1868,15 +1883,15 @@ def : GCNPat <
 // fabs is not reported as free because there is modifier for it in
 // VOP3P instructions, so it is turned into the bit op.
 def : GCNPat <
-  (UniformUnaryFrag<fneg> (v2f16 (bitconvert (and_oneuse (i32 SReg_32:$src), 0x7fff7fff)))),
+  (UniformUnaryFrag<fneg> (v2fp16vt (bitconvert (and_oneuse (i32 SReg_32:$src), 0x7fff7fff)))),
   (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit
 >;
 
 def : GCNPat <
-  (UniformUnaryFrag<fneg> (v2f16 (fabs SReg_32:$src))),
+  (UniformUnaryFrag<fneg> (v2fp16vt (fabs SReg_32:$src))),
   (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit
 >;
-
+}
 
 // COPY_TO_REGCLASS is needed to avoid using SCC from S_XOR_B32 instead
 // of the real value.
@@ -1997,20 +2012,22 @@ def : GCNPat <
 } // End SubtargetPredicate = UseRealTrue16Insts
 } // End foreach fp16vt = ...
 
+foreach v2fp16vt = [v2f16, v2bf16] in {
 def : GCNPat <
-  (fneg (v2f16 VGPR_32:$src)),
+  (fneg (v2fp16vt VGPR_32:$src)),
   (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src)
 >;
 
 def : GCNPat <
-  (fabs (v2f16 VGPR_32:$src)),
+  (fabs (v2fp16vt VGPR_32:$src)),
   (V_AND_B32_e64 (S_MOV_B32 (i32 0x7fff7fff)), VGPR_32:$src)
 >;
 
 def : GCNPat <
-  (fneg (v2f16 (fabs VGPR_32:$src))),
+  (fneg (v2fp16vt (fabs VGPR_32:$src))),
   (V_OR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src)
 >;
+}
 
 def : GCNPat <
   (fabs (f64 VReg_64:$src)),
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
index 2bbbbf4db02d..fd28abeb887c 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
@@ -287,13 +287,10 @@ void SIScheduleBlock::fastSchedule() {
 static bool isDefBetween(Register Reg, SlotIndex First, SlotIndex Last,
                          const MachineRegisterInfo *MRI,
                          const LiveIntervals *LIS) {
-  for (MachineRegisterInfo::def_instr_iterator
-       UI = MRI->def_instr_begin(Reg),
-       UE = MRI->def_instr_end(); UI != UE; ++UI) {
-    const MachineInstr* MI = &*UI;
-    if (MI->isDebugValue())
+  for (const MachineInstr &MI : MRI->def_instructions(Reg)) {
+    if (MI.isDebugValue())
       continue;
-    SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
+    SlotIndex InstSlot = LIS->getInstructionIndex(MI).getRegSlot();
     if (InstSlot >= First && InstSlot <= Last)
       return true;
   }
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/SOPInstructions.td b/external/llvm-project/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 40b3dfb94ce2..e0a36758534d 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -922,11 +922,11 @@ let SubtargetPredicate = HasSALUFloatInsts, mayRaiseFPException = 1,
 
 // On GFX12 MIN/MAX instructions do not read MODE register.
 let SubtargetPredicate = isGFX12Plus, mayRaiseFPException = 1, isCommutable = 1,
-    isReMaterializable = 1, SchedRW = [WriteSFPU], AddedComplexity = 17 in {
-  def S_MINIMUM_F32 : SOP2_F32_Inst<"s_minimum_f32", fminimum>;
-  def S_MAXIMUM_F32 : SOP2_F32_Inst<"s_maximum_f32", fmaximum>;
-  def S_MINIMUM_F16 : SOP2_F16_Inst<"s_minimum_f16", fminimum>;
-  def S_MAXIMUM_F16 : SOP2_F16_Inst<"s_maximum_f16", fmaximum>;
+    isReMaterializable = 1, SchedRW = [WriteSFPU], AddedComplexity = 25 in {
+  def S_MINIMUM_F32 : SOP2_F32_Inst<"s_minimum_f32", UniformBinFrag<fminimum>>;
+  def S_MAXIMUM_F32 : SOP2_F32_Inst<"s_maximum_f32", UniformBinFrag<fmaximum>>;
+  def S_MINIMUM_F16 : SOP2_F16_Inst<"s_minimum_f16", UniformBinFrag<fminimum>>;
+  def S_MAXIMUM_F16 : SOP2_F16_Inst<"s_maximum_f16", UniformBinFrag<fmaximum>>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 1233973da140..c0cd43a9c35d 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -2124,71 +2124,6 @@ bool getHasDepthExport(const Function &F) {
   return F.getFnAttributeAsParsedInteger("amdgpu-depth-export", 0) != 0;
 }
 
-bool isShader(CallingConv::ID cc) {
-  switch (cc) {
-  case CallingConv::AMDGPU_VS:
-  case CallingConv::AMDGPU_LS:
-  case CallingConv::AMDGPU_HS:
-  case CallingConv::AMDGPU_ES:
-  case CallingConv::AMDGPU_GS:
-  case CallingConv::AMDGPU_PS:
-  case CallingConv::AMDGPU_CS_Chain:
-  case CallingConv::AMDGPU_CS_ChainPreserve:
-  case CallingConv::AMDGPU_CS:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool isGraphics(CallingConv::ID cc) {
-  return isShader(cc) || cc == CallingConv::AMDGPU_Gfx;
-}
-
-bool isCompute(CallingConv::ID cc) {
-  return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS;
-}
-
-bool isEntryFunctionCC(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_KERNEL:
-  case CallingConv::SPIR_KERNEL:
-  case CallingConv::AMDGPU_VS:
-  case CallingConv::AMDGPU_GS:
-  case CallingConv::AMDGPU_PS:
-  case CallingConv::AMDGPU_CS:
-  case CallingConv::AMDGPU_ES:
-  case CallingConv::AMDGPU_HS:
-  case CallingConv::AMDGPU_LS:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool isModuleEntryFunctionCC(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_Gfx:
-    return true;
-  default:
-    return isEntryFunctionCC(CC) || isChainCC(CC);
-  }
-}
-
-bool isChainCC(CallingConv::ID CC) {
-  switch (CC) {
-  case CallingConv::AMDGPU_CS_Chain:
-  case CallingConv::AMDGPU_CS_ChainPreserve:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool isKernelCC(const Function *Func) {
-  return AMDGPU::isModuleEntryFunctionCC(Func->getCallingConv());
-}
-
 bool hasXNACK(const MCSubtargetInfo &STI) {
   return STI.hasFeature(AMDGPU::FeatureXNACK);
 }
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index e0534b2091f5..975a8908059c 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1306,16 +1306,61 @@ bool getHasColorExport(const Function &F);
 bool getHasDepthExport(const Function &F);
 
 LLVM_READNONE
-bool isShader(CallingConv::ID CC);
+constexpr bool isShader(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::AMDGPU_VS:
+  case CallingConv::AMDGPU_LS:
+  case CallingConv::AMDGPU_HS:
+  case CallingConv::AMDGPU_ES:
+  case CallingConv::AMDGPU_GS:
+  case CallingConv::AMDGPU_PS:
+  case CallingConv::AMDGPU_CS_Chain:
+  case CallingConv::AMDGPU_CS_ChainPreserve:
+  case CallingConv::AMDGPU_CS:
+    return true;
+  default:
+    return false;
+  }
+}
 
 LLVM_READNONE
-bool isGraphics(CallingConv::ID CC);
+constexpr bool isGraphics(CallingConv::ID CC) {
+  return isShader(CC) || CC == CallingConv::AMDGPU_Gfx;
+}
 
 LLVM_READNONE
-bool isCompute(CallingConv::ID CC);
+constexpr bool isCompute(CallingConv::ID CC) {
+  return !isGraphics(CC) || CC == CallingConv::AMDGPU_CS;
+}
 
 LLVM_READNONE
-bool isEntryFunctionCC(CallingConv::ID CC);
+constexpr bool isEntryFunctionCC(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::AMDGPU_KERNEL:
+  case CallingConv::SPIR_KERNEL:
+  case CallingConv::AMDGPU_VS:
+  case CallingConv::AMDGPU_GS:
+  case CallingConv::AMDGPU_PS:
+  case CallingConv::AMDGPU_CS:
+  case CallingConv::AMDGPU_ES:
+  case CallingConv::AMDGPU_HS:
+  case CallingConv::AMDGPU_LS:
+    return true;
+  default:
+    return false;
+  }
+}
+
+LLVM_READNONE
+constexpr bool isChainCC(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::AMDGPU_CS_Chain:
+  case CallingConv::AMDGPU_CS_ChainPreserve:
+    return true;
+  default:
+    return false;
+  }
+}
 
 // These functions are considered entrypoints into the current module, i.e. they
 // are allowed to be called from outside the current module. This is different
@@ -1324,16 +1369,17 @@ bool isEntryFunctionCC(CallingConv::ID CC);
 // include functions that can be called from other functions inside or outside
 // the current module. Module entry functions are allowed to allocate LDS.
 LLVM_READNONE
-bool isModuleEntryFunctionCC(CallingConv::ID CC);
-
-LLVM_READNONE
-bool isChainCC(CallingConv::ID CC);
-
-bool isKernelCC(const Function *Func);
+constexpr bool isModuleEntryFunctionCC(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::AMDGPU_Gfx:
+    return true;
+  default:
+    return isEntryFunctionCC(CC) || isChainCC(CC);
+  }
+}
 
-// FIXME: Remove this when calling conventions cleaned up
 LLVM_READNONE
-inline bool isKernel(CallingConv::ID CC) {
+constexpr inline bool isKernel(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::AMDGPU_KERNEL:
   case CallingConv::SPIR_KERNEL:
@@ -1343,6 +1389,23 @@ inline bool isKernel(CallingConv::ID CC) {
   }
 }
 
+LLVM_READNONE
+constexpr bool canGuaranteeTCO(CallingConv::ID CC) {
+  return CC == CallingConv::Fast;
+}
+
+/// Return true if we might ever do TCO for calls with this calling convention.
+LLVM_READNONE
+constexpr bool mayTailCallThisCC(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::C:
+  case CallingConv::AMDGPU_Gfx:
+    return true;
+  default:
+    return canGuaranteeTCO(CC);
+  }
+}
+
 bool hasXNACK(const MCSubtargetInfo &STI);
 bool hasSRAMECC(const MCSubtargetInfo &STI);
 bool hasMIMG_R128(const MCSubtargetInfo &STI);
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 0252c4f1b092..a005e0245b8f 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -168,10 +168,10 @@ defm V_MUL_HI_I32 : VOP3Inst <"v_mul_hi_i32", V_MUL_PROF<VOP_I32_I32_I32>, mulhs
 } // End SchedRW = [WriteIntMul]
 
 let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0, AddedComplexity = 1 in {
-defm V_MINIMUM_F32 : VOP3Inst <"v_minimum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fminimum>>;
-defm V_MAXIMUM_F32 : VOP3Inst <"v_maximum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fmaximum>>;
-defm V_MINIMUM_F16 : VOP3Inst_t16 <"v_minimum_f16", VOP_F16_F16_F16, DivergentBinFrag<fminimum>>;
-defm V_MAXIMUM_F16 : VOP3Inst_t16 <"v_maximum_f16", VOP_F16_F16_F16, DivergentBinFrag<fmaximum>>;
+defm V_MINIMUM_F32 : VOP3Inst <"v_minimum_f32", VOP3_Profile<VOP_F32_F32_F32>, fminimum>;
+defm V_MAXIMUM_F32 : VOP3Inst <"v_maximum_f32", VOP3_Profile<VOP_F32_F32_F32>, fmaximum>;
+defm V_MINIMUM_F16 : VOP3Inst_t16 <"v_minimum_f16", VOP_F16_F16_F16, fminimum>;
+defm V_MAXIMUM_F16 : VOP3Inst_t16 <"v_maximum_f16", VOP_F16_F16_F16, fmaximum>;
 
 let SchedRW = [WriteDoubleAdd] in {
 defm V_MINIMUM_F64 : VOP3Inst <"v_minimum_f64", VOP3_Profile<VOP_F64_F64_F64>, fminimum>;
@@ -570,6 +570,38 @@ def VOP3_CVT_PK_F8_F32_Profile : VOP3_Profile<VOP_I32_F32_F32, VOP3_OPSEL> {
   let HasExtVOP3DPP = 1;
 }
 
+def VOP3_CVT_PK_F8_F32_Profile_fake16 : VOP3_Profile_Fake16<VOP_I16_F32_F32, VOP3_OPSEL> {
+  defvar Tail = (ins VGPR_32:$vdst_in, op_sel0:$op_sel);
+  let InsVOP3OpSel = !con(getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,
+                                   0, HasModifiers, HasSrc2Mods,
+                                   HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret,
+                          Tail);
+  let InsVOP3Base = !con(getInsVOP3Base<Src0VOP3DPP, Src1VOP3DPP,
+                                        Src2VOP3DPP, NumSrcArgs, 0, HasModifiers,
+                                        HasSrc2Mods, HasOMod, Src0ModVOP3DPP, Src1ModVOP3DPP,
+                                        Src2ModVOP3DPP, false>.ret,
+                         Tail);
+  let HasClamp = 0;
+  let HasExtVOP3DPP = 1;
+}
+
+// This t16 profile with vdst_in operand is for backward compatibility and is used
+// for user controlled packing
+def VOP3_CVT_PK_F8_F32_Profile_t16 : VOP3_Profile_True16<VOP_I16_F32_F32, VOP3_OPSEL> {
+  defvar Tail = (ins VGPR_16:$vdst_in, op_sel0:$op_sel);
+  let InsVOP3OpSel = !con(getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,
+                                   0, HasModifiers, HasSrc2Mods,
+                                   HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret,
+                          Tail);
+  let InsVOP3Base = !con(getInsVOP3Base<Src0VOP3DPP, Src1VOP3DPP,
+                                        Src2VOP3DPP, NumSrcArgs, 0, HasModifiers,
+                                        HasSrc2Mods, HasOMod, Src0ModVOP3DPP, Src1ModVOP3DPP,
+                                        Src2ModVOP3DPP, false>.ret,
+                         Tail);
+  let HasClamp = 0;
+  let HasExtVOP3DPP = 1;
+}
+
 def VOP3_CVT_SR_F8_F32_Profile : VOP3_Profile<VOPProfile<[i32, f32, i32, f32]>,
                                               VOP3_OPSEL> {
   let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,
@@ -594,10 +626,6 @@ def VOP3_CVT_SR_F8_F32_Profile : VOP3_Profile<VOPProfile<[i32, f32, i32, f32]>,
   let HasOpSel = 1;
   let HasFP8DstByteSel = 1;
   let HasFP8ByteSel = 0; // It works as a dst-bytesel, but does not have byte_sel operand.
-  let AsmVOP3OpSel = !subst(", $src2_modifiers", "",
-                            getAsmVOP3OpSel<3, HasClamp, HasOMod,
-                                            HasSrc0FloatMods, HasSrc1FloatMods,
-                                            HasSrc2FloatMods>.ret);
   let AsmVOP3Base = !subst(", $src2_modifiers", "",
                     getAsmVOP3Base<NumSrcArgs, HasDst, HasClamp,
                     HasOpSel, HasOMod, IsVOP3P, HasModifiers, HasModifiers, 0/*Src1Mods*/,
@@ -675,8 +703,12 @@ defm V_LSHL_ADD_U64 : VOP3Inst <"v_lshl_add_u64", VOP3_Profile<VOP_I64_I64_I32_I
 let OtherPredicates = [HasFP8ConversionInsts], mayRaiseFPException = 0,
     SchedRW = [WriteFloatCvt] in {
   let Constraints = "$vdst = $vdst_in", DisableEncoding = "$vdst_in" in {
-    defm V_CVT_PK_FP8_F32 : VOP3Inst<"v_cvt_pk_fp8_f32", VOP3_CVT_PK_F8_F32_Profile>;
-    defm V_CVT_PK_BF8_F32 : VOP3Inst<"v_cvt_pk_bf8_f32", VOP3_CVT_PK_F8_F32_Profile>;
+    defm V_CVT_PK_FP8_F32 : VOP3Inst_t16_with_profiles<"v_cvt_pk_fp8_f32", VOP3_CVT_PK_F8_F32_Profile,
+                                                        VOP3_CVT_PK_F8_F32_Profile_t16,
+                                                        VOP3_CVT_PK_F8_F32_Profile_fake16>;
+    defm V_CVT_PK_BF8_F32 : VOP3Inst_t16_with_profiles<"v_cvt_pk_bf8_f32", VOP3_CVT_PK_F8_F32_Profile,
+                                                        VOP3_CVT_PK_F8_F32_Profile_t16,
+                                                        VOP3_CVT_PK_F8_F32_Profile_fake16>;
 
     let SubtargetPredicate = isGFX12Plus in {
       defm V_CVT_SR_FP8_F32_gfx12 : VOP3Inst<"v_cvt_sr_fp8_f32_gfx12", VOP3_CVT_SR_F8_ByteSel_Profile<f32>>;
@@ -698,6 +730,21 @@ class Cvt_PK_F8_F32_Pat<SDPatternOperator node, int index, VOP3_Pseudo inst> : G
     (inst !if(index, SRCMODS.DST_OP_SEL, 0), $src0, 0, $src1, $old, 0)
 >;
 
+multiclass Cvt_PK_F8_F32_t16_Pat<SDPatternOperator node, VOP3_Pseudo inst> {
+def : GCNPat<
+    (i32 (node f32:$src0, f32:$src1, i32:$old, -1)),
+    (REG_SEQUENCE VGPR_32,
+      (i16 (EXTRACT_SUBREG $old, lo16)), lo16,
+      (i16 (inst SRCMODS.DST_OP_SEL, $src0, 0, $src1, (i16 (EXTRACT_SUBREG $old, hi16)), 0)), hi16)
+>;
+def : GCNPat<
+    (i32 (node f32:$src0, f32:$src1, i32:$old, 0)),
+    (REG_SEQUENCE VGPR_32,
+      (i16 (inst 0, $src0, 0, $src1, (i16 (EXTRACT_SUBREG $old, lo16)), 0)), lo16,
+      (i16 (EXTRACT_SUBREG $old, hi16)), hi16)
+>;
+}
+
 class Cvt_SR_F8_F32_Pat<SDPatternOperator node, bits<2> index, VOP3_Pseudo inst> : GCNPat<
     (i32 (node f32:$src0, i32:$src1, i32:$old, index)),
     (inst !if(index{1}, SRCMODS.DST_OP_SEL, 0), $src0, 0, $src1,
@@ -712,9 +759,20 @@ class Cvt_SR_F8_ByteSel_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType
 
 let OtherPredicates = [HasFP8ConversionInsts] in {
 foreach Index = [0, -1] in {
+let True16Predicate = NotHasTrue16BitInsts in {
   def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_fp8_f32, Index, V_CVT_PK_FP8_F32_e64>;
   def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_bf8_f32, Index, V_CVT_PK_BF8_F32_e64>;
 }
+let True16Predicate = UseFakeTrue16Insts in {
+  def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_fp8_f32, Index, V_CVT_PK_FP8_F32_fake16_e64>;
+  def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_bf8_f32, Index, V_CVT_PK_BF8_F32_fake16_e64>;
+}
+}
+
+let True16Predicate = UseRealTrue16Insts in {
+defm : Cvt_PK_F8_F32_t16_Pat<int_amdgcn_cvt_pk_fp8_f32, V_CVT_PK_FP8_F32_t16_e64>;
+defm : Cvt_PK_F8_F32_t16_Pat<int_amdgcn_cvt_pk_bf8_f32, V_CVT_PK_BF8_F32_t16_e64>;
+}
 
 let SubtargetPredicate = isGFX940Plus in {
   foreach Index = [0, 1, 2, 3] in {
@@ -941,18 +999,7 @@ class VOP3_BITOP3_Profile<VOPProfile pfl, VOP3Features f> : VOP3_Profile<pfl, f>
   let HasClamp = 0;
   let HasOMod = 0;
   let HasModifiers = 0;
-
-  let Ins64 = !con(getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,
-                            0 /* HasIntClamp */, HasModifiers, HasSrc2Mods,
-                            HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret,
-                   (ins bitop3_0:$bitop3));
-
-  let InsVOP3OpSel = !con(getInsVOP3Base<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs, 0, 1, 1, 0,
-                                         Src0Mod, Src1Mod, Src2Mod, 0>.ret,
-                          (ins bitop3_0:$bitop3, op_sel0:$op_sel));
-
-  let Asm64 = "$vdst, $src0, $src1, $src2$bitop3";
-  let AsmVOP3OpSel = !subst("$op_sel", "$bitop3$op_sel", getAsmVOP3OpSel<3, 0, 0, 0, 0, 0>.ret);
+  let HasBitOp3 = 1;
 }
 
 class VOP3_CVT_SCALE_F1632_FP8BF8_Profile<ValueType DstTy> : VOP3_Profile<VOPProfile<[DstTy, i32, f32, untyped]>,
@@ -993,6 +1040,7 @@ class VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<VOPProfile P> : VOP3_Profil
 
 class VOP3_CVT_SCALE_FP4_F32_TiedInput_Profile<VOPProfile P> : VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<P> {
   let HasFP8DstByteSel = 1;
+  let HasFP8ByteSel = 0; // It works as a dst-bytesel, but does not have byte_sel operand.
 }
 
 class VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOPProfile P> : VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<P> {
@@ -1001,6 +1049,7 @@ class VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOPProfile P> : VOP3_
                           FP32InputMods:$src2_modifiers, Src2RC64:$src2,
                           VGPR_32:$vdst_in, op_sel0:$op_sel);
   let HasFP8DstByteSel = 1;
+  let HasFP8ByteSel = 0; // It works as a dst-bytesel, but does not have byte_sel operand.
 }
 
 
@@ -1013,12 +1062,10 @@ class VOP3_CVT_SCALE_FP4_F16BF16_TiedInput_Profile<VOPProfile P> : VOP3_Profile<
   let HasSrc2 = 0;
   let HasSrc2Mods = 1;
   let HasOpSel = 1;
-  let AsmVOP3OpSel = !subst(", $src2_modifiers", "",
-                            getAsmVOP3OpSel<3, HasClamp, HasOMod,
-                                            HasSrc0FloatMods, HasSrc1FloatMods,
-                                            HasSrc2FloatMods>.ret);
+  let Asm64 = !subst(", $src2_modifiers", "", AsmVOP3Base);
   let HasExtVOP3DPP = 0;
   let HasFP8DstByteSel = 1;
+  let HasFP8ByteSel = 0;
 }
 
 class VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<ValueType Src0Ty> :
@@ -1271,10 +1318,10 @@ let SubtargetPredicate = isGFX10Plus in {
   } // End isCommutable = 1, isReMaterializable = 1
   def : ThreeOp_i32_Pats<xor, xor, V_XOR3_B32_e64>;
 
-  let Constraints = "$vdst = $vdst_in", DisableEncoding="$vdst_in" in {
+  let Constraints = "$vdst = $vdst_in", DisableEncoding="$vdst_in", isConvergent = 1 in {
     defm V_PERMLANE16_B32 : VOP3Inst<"v_permlane16_b32", VOP3_PERMLANE_Profile>;
     defm V_PERMLANEX16_B32 : VOP3Inst<"v_permlanex16_b32", VOP3_PERMLANE_Profile>;
-  } // End $vdst = $vdst_in, DisableEncoding $vdst_in
+  } // End $vdst = $vdst_in, DisableEncoding $vdst_in, isConvergent = 1
 
   foreach vt = Reg32Types.types in {
     def : PermlanePat<int_amdgcn_permlane16, V_PERMLANE16_B32_e64, vt>;
@@ -1642,8 +1689,8 @@ defm V_MAXIMUM_F16        : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x368, "v_m
 defm V_PERMLANE16_VAR_B32  : VOP3Only_Real_Base_gfx12<0x30f>;
 defm V_PERMLANEX16_VAR_B32 : VOP3Only_Real_Base_gfx12<0x310>;
 
-defm V_CVT_PK_FP8_F32  : VOP3Only_Realtriple_gfx12<0x369>;
-defm V_CVT_PK_BF8_F32  : VOP3Only_Realtriple_gfx12<0x36a>;
+defm V_CVT_PK_FP8_F32  : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x369, "v_cvt_pk_fp8_f32">;
+defm V_CVT_PK_BF8_F32  : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x36a, "v_cvt_pk_bf8_f32">;
 defm V_CVT_SR_FP8_F32_gfx12 : VOP3_Realtriple_with_name_gfx12<0x36b, "V_CVT_SR_FP8_F32_gfx12", "v_cvt_sr_fp8_f32" >;
 defm V_CVT_SR_BF8_F32_gfx12 : VOP3_Realtriple_with_name_gfx12<0x36c, "V_CVT_SR_BF8_F32_gfx12", "v_cvt_sr_bf8_f32">;
 
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 7aa22c40f5ca..07c69d8c48b4 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1856,6 +1856,9 @@ multiclass VOP3P_Realtriple<GFXGen Gen, bits<8> op, string backing_ps_name = NAM
       VOP3P_Real_dpp<Gen, op, backing_ps_name, asmName>,
       VOP3P_Real_dpp8<Gen, op, backing_ps_name, asmName>;
 
+multiclass VOP3P_Realtriple_gfx11_gfx12<bits<8> op>
+  : VOP3P_Realtriple<GFX11Gen, op>, VOP3P_Realtriple<GFX12Gen, op>;
+
 //===----------------------------------------------------------------------===//
 // GFX12
 //===----------------------------------------------------------------------===//
@@ -1887,7 +1890,7 @@ multiclass VOP3P_Real_gfx11_gfx12<bits<8> op> :
 
 defm V_DOT4_I32_IU8  : VOP3P_Real_gfx11_gfx12<0x16>;
 defm V_DOT8_I32_IU4  : VOP3P_Real_gfx11_gfx12<0x18>;
-defm V_DOT2_F32_BF16 : VOP3P_Real_gfx11_gfx12<0x1a>;
+defm V_DOT2_F32_BF16 : VOP3P_Realtriple_gfx11_gfx12<0x1a>;
 
 let AssemblerPredicate = isGFX11Plus in {
   def : AMDGPUMnemonicAlias<"v_dot4_i32_i8", "v_dot4_i32_iu8">;
diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/VOPInstructions.td b/external/llvm-project/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 993c92f6451c..9b85cc3a3a91 100644
--- a/external/llvm-project/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/external/llvm-project/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -112,9 +112,7 @@ class VOP3_Pseudo <string opName, VOPProfile P, list<dag> pattern = [],
   bit HasFP8DstByteSel = P.HasFP8DstByteSel;
   bit HasFP4DstByteSel = P.HasFP4DstByteSel;
 
-  let AsmOperands = !if(!and(!not(P.IsTrue16), isVop3OpSel),
-                        P.AsmVOP3OpSel,
-                        !if(!and(isVOP3P, P.IsPacked), P.AsmVOP3P, P.Asm64));
+  let AsmOperands = !if(!and(isVOP3P, P.IsPacked), P.AsmVOP3P, P.Asm64);
 
   let Size = 8;
   let mayLoad = 0;
@@ -1477,10 +1475,13 @@ class VOP3_Profile_Base<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VO
   let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
   let IsMAI    = !if(Features.IsMAI,    1, P.IsMAI);
   let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
+  let HasFP8SrcByteSel = P.HasFP8SrcByteSel;
+  let HasFP8DstByteSel = P.HasFP8DstByteSel;
+  let HasOMod = P.HasOMod;
 
   let HasModifiers =
       !if (Features.IsMAI, 0,
-           !or(Features.IsPacked, Features.HasOpSel, P.HasModifiers));
+           !or(Features.IsPacked, P.HasModifiers));
 }
 
 class VOP3_Profile<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Profile_Base<P, Features> {
@@ -1493,6 +1494,9 @@ class VOP3_Profile_True16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> :
   let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
   let IsMAI    = !if(Features.IsMAI,    1, P.IsMAI);
   let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
+  let HasFP8SrcByteSel = P.HasFP8SrcByteSel;
+  let HasFP8DstByteSel = P.HasFP8DstByteSel;
+  let HasOMod = P.HasOMod;
 
   let HasModifiers =
       !if (Features.IsMAI, 0,
@@ -1505,6 +1509,9 @@ class VOP3_Profile_Fake16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> :
   let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
   let IsMAI    = !if(Features.IsMAI,    1, P.IsMAI);
   let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
+  let HasFP8SrcByteSel = P.HasFP8SrcByteSel;
+  let HasFP8DstByteSel = P.HasFP8DstByteSel;
+  let HasOMod = P.HasOMod;
 
   let HasModifiers =
       !if (Features.IsMAI, 0,
diff --git a/external/llvm-project/llvm/lib/Target/ARC/ARCTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/ARC/ARCTargetTransformInfo.h
index bb7f2a0a459c..3d5ff6dc256d 100644
--- a/external/llvm-project/llvm/lib/Target/ARC/ARCTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/ARC/ARCTargetTransformInfo.h
@@ -26,7 +26,7 @@ class ARCSubtarget;
 class ARCTargetLowering;
 class ARCTargetMachine;
 
-class ARCTTIImpl : public BasicTTIImplBase<ARCTTIImpl> {
+class ARCTTIImpl final : public BasicTTIImplBase<ARCTTIImpl> {
   using BaseT = BasicTTIImplBase<ARCTTIImpl>;
   friend BaseT;
 
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index 13efd70c0f22..fa1437002551 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -18,7 +18,7 @@
 #include "ARMTargetMachine.h"
 #include "ARMTargetObjectFile.h"
 #include "MCTargetDesc/ARMInstPrinter.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "TargetInfo/ARMTargetInfo.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/BinaryFormat/COFF.h"
@@ -92,8 +92,7 @@ void ARMAsmPrinter::emitXXStructor(const DataLayout &DL, const Constant *CV) {
 
   const MCExpr *E = MCSymbolRefExpr::create(
       GetARMGVSymbol(GV, ARMII::MO_NO_FLAG),
-      (Subtarget->isTargetELF() ? ARMMCExpr::VK_TARGET1 : ARMMCExpr::VK_None),
-      OutContext);
+      (Subtarget->isTargetELF() ? ARM::S_TARGET1 : ARM::S_None), OutContext);
 
   OutStreamer->emitValue(E, Size);
 }
@@ -843,17 +842,17 @@ static MCSymbol *getPICLabel(StringRef Prefix, unsigned FunctionNumber,
 static uint8_t getModifierSpecifier(ARMCP::ARMCPModifier Modifier) {
   switch (Modifier) {
   case ARMCP::no_modifier:
-    return ARMMCExpr::VK_None;
+    return ARM::S_None;
   case ARMCP::TLSGD:
-    return ARMMCExpr::VK_TLSGD;
+    return ARM::S_TLSGD;
   case ARMCP::TPOFF:
-    return ARMMCExpr::VK_TPOFF;
+    return ARM::S_TPOFF;
   case ARMCP::GOTTPOFF:
-    return ARMMCExpr::VK_GOTTPOFF;
+    return ARM::S_GOTTPOFF;
   case ARMCP::SBREL:
-    return ARMMCExpr::VK_SBREL;
+    return ARM::S_SBREL;
   case ARMCP::GOT_PREL:
-    return ARMMCExpr::VK_GOT_PREL;
+    return ARM::S_GOT_PREL;
   case ARMCP::SECREL:
     return MCSymbolRefExpr::VK_SECREL;
   }
@@ -1620,12 +1619,15 @@ void ARMAsmPrinter::emitInstruction(const MachineInstr *MI) {
                     MI->getOperand(2).getImm(), OutContext);
     const MCExpr *LabelSymExpr= MCSymbolRefExpr::create(LabelSym, OutContext);
     unsigned PCAdj = (Opc == ARM::MOVi16_ga_pcrel) ? 8 : 4;
-    const MCExpr *PCRelExpr =
-      ARMMCExpr::createLower16(MCBinaryExpr::createSub(GVSymExpr,
-                                      MCBinaryExpr::createAdd(LabelSymExpr,
-                                      MCConstantExpr::create(PCAdj, OutContext),
-                                      OutContext), OutContext), OutContext);
-      TmpInst.addOperand(MCOperand::createExpr(PCRelExpr));
+    const MCExpr *PCRelExpr = ARM::createLower16(
+        MCBinaryExpr::createSub(
+            GVSymExpr,
+            MCBinaryExpr::createAdd(LabelSymExpr,
+                                    MCConstantExpr::create(PCAdj, OutContext),
+                                    OutContext),
+            OutContext),
+        OutContext);
+    TmpInst.addOperand(MCOperand::createExpr(PCRelExpr));
 
     // Add predicate operands.
     TmpInst.addOperand(MCOperand::createImm(ARMCC::AL));
@@ -1653,12 +1655,15 @@ void ARMAsmPrinter::emitInstruction(const MachineInstr *MI) {
                     MI->getOperand(3).getImm(), OutContext);
     const MCExpr *LabelSymExpr= MCSymbolRefExpr::create(LabelSym, OutContext);
     unsigned PCAdj = (Opc == ARM::MOVTi16_ga_pcrel) ? 8 : 4;
-    const MCExpr *PCRelExpr =
-        ARMMCExpr::createUpper16(MCBinaryExpr::createSub(GVSymExpr,
-                                   MCBinaryExpr::createAdd(LabelSymExpr,
-                                      MCConstantExpr::create(PCAdj, OutContext),
-                                          OutContext), OutContext), OutContext);
-      TmpInst.addOperand(MCOperand::createExpr(PCRelExpr));
+    const MCExpr *PCRelExpr = ARM::createUpper16(
+        MCBinaryExpr::createSub(
+            GVSymExpr,
+            MCBinaryExpr::createAdd(LabelSymExpr,
+                                    MCConstantExpr::create(PCAdj, OutContext),
+                                    OutContext),
+            OutContext),
+        OutContext);
+    TmpInst.addOperand(MCOperand::createExpr(PCRelExpr));
     // Add predicate operands.
     TmpInst.addOperand(MCOperand::createImm(ARMCC::AL));
     TmpInst.addOperand(MCOperand::createReg(0));
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/external/llvm-project/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 4b029fb4be20..8f17c62d4b90 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -55,7 +55,6 @@
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMBranchTargets.cpp b/external/llvm-project/llvm/lib/Target/ARM/ARMBranchTargets.cpp
index 17d0bdd87512..409482b9679d 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMBranchTargets.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMBranchTargets.cpp
@@ -77,7 +77,8 @@ bool ARMBranchTargets::runOnMachineFunction(MachineFunction &MF) {
     // modes. These modes do not support PACBTI. As a result, BTI instructions
     // are not added in the destination blocks.
 
-    if (IsFirstBB || MBB.hasAddressTaken() || MBB.isEHPad()) {
+    if (IsFirstBB || MBB.isMachineBlockAddressTaken() ||
+        MBB.isIRBlockAddressTaken() || MBB.isEHPad()) {
       addBTI(TII, MBB, IsFirstBB);
       MadeChange = true;
     }
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp b/external/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp
index be4876d0667a..5b3664c4e961 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -511,10 +511,11 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() &&
       !Subtarget->isTargetWatchOS() && !Subtarget->isTargetDriverKit()) {
     bool IsHFTarget = TM.Options.FloatABIType == FloatABI::Hard;
-    for (int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
-      setLibcallCallingConv(static_cast<RTLIB::Libcall>(LCID),
-                            IsHFTarget ? CallingConv::ARM_AAPCS_VFP
-                                       : CallingConv::ARM_AAPCS);
+
+    for (RTLIB::Libcall LC : RTLIB::libcalls()) {
+      setLibcallCallingConv(LC, IsHFTarget ? CallingConv::ARM_AAPCS_VFP
+                                           : CallingConv::ARM_AAPCS);
+    }
   }
 
   if (Subtarget->isTargetMachO()) {
@@ -612,7 +613,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
       { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
       { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
       { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
-      { RTLIB::UO_F64,  "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
+      { RTLIB::UO_F64,  "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
 
       // Single-precision floating-point arithmetic helper functions
       // RTABI chapter 4.1.2, Table 4
@@ -629,7 +630,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
       { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
       { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
       { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
-      { RTLIB::UO_F32,  "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
+      { RTLIB::UO_F32,  "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
 
       // Floating-point to integer conversions.
       // RTABI chapter 4.1.2, Table 6
@@ -692,54 +693,21 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
         const RTLIB::Libcall Op;
         const char *const Name;
         const CallingConv::ID CC;
-        const ISD::CondCode Cond;
       } MemOpsLibraryCalls[] = {
-        // Memory operations
-        // RTABI chapter 4.3.4
-        { RTLIB::MEMCPY,  "__aeabi_memcpy",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
-        { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
-        { RTLIB::MEMSET,  "__aeabi_memset",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
+          // Memory operations
+          // RTABI chapter 4.3.4
+          {RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS},
+          {RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS},
+          {RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS},
       };
 
       for (const auto &LC : MemOpsLibraryCalls) {
         setLibcallName(LC.Op, LC.Name);
         setLibcallCallingConv(LC.Op, LC.CC);
-        if (LC.Cond != ISD::SETCC_INVALID)
-          setCmpLibcallCC(LC.Op, LC.Cond);
       }
     }
   }
 
-  if (Subtarget->isTargetWindows()) {
-    static const struct {
-      const RTLIB::Libcall Op;
-      const char * const Name;
-      const CallingConv::ID CC;
-    } LibraryCalls[] = {
-      { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP },
-      { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP },
-      { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP },
-      { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP },
-      { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP },
-      { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP },
-      { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP },
-      { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP },
-    };
-
-    for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
-      setLibcallCallingConv(LC.Op, LC.CC);
-    }
-  }
-
-  // Use divmod compiler-rt calls for iOS 5.0 and later.
-  if (Subtarget->isTargetMachO() &&
-      !(Subtarget->isTargetIOS() &&
-        Subtarget->getTargetTriple().isOSVersionLT(5, 0))) {
-    setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
-    setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
-  }
-
   // The half <-> float conversion functions are always soft-float on
   // non-watchos platforms, but are needed for some targets which use a
   // hard-float calling convention by default.
@@ -1277,50 +1245,6 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::UREM, MVT::i64, Custom);
     HasStandaloneRem = false;
 
-    if (Subtarget->isTargetWindows()) {
-      const struct {
-        const RTLIB::Libcall Op;
-        const char * const Name;
-        const CallingConv::ID CC;
-      } LibraryCalls[] = {
-        { RTLIB::SDIVREM_I8, "__rt_sdiv", CallingConv::ARM_AAPCS },
-        { RTLIB::SDIVREM_I16, "__rt_sdiv", CallingConv::ARM_AAPCS },
-        { RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS },
-        { RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS },
-
-        { RTLIB::UDIVREM_I8, "__rt_udiv", CallingConv::ARM_AAPCS },
-        { RTLIB::UDIVREM_I16, "__rt_udiv", CallingConv::ARM_AAPCS },
-        { RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS },
-        { RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS },
-      };
-
-      for (const auto &LC : LibraryCalls) {
-        setLibcallName(LC.Op, LC.Name);
-        setLibcallCallingConv(LC.Op, LC.CC);
-      }
-    } else {
-      const struct {
-        const RTLIB::Libcall Op;
-        const char * const Name;
-        const CallingConv::ID CC;
-      } LibraryCalls[] = {
-        { RTLIB::SDIVREM_I8, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
-        { RTLIB::SDIVREM_I16, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
-        { RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
-        { RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS },
-
-        { RTLIB::UDIVREM_I8, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
-        { RTLIB::UDIVREM_I16, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
-        { RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
-        { RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS },
-      };
-
-      for (const auto &LC : LibraryCalls) {
-        setLibcallName(LC.Op, LC.Name);
-        setLibcallCallingConv(LC.Op, LC.CC);
-      }
-    }
-
     setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
     setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
     setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
@@ -2764,8 +2688,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         unsigned RegBegin, RegEnd;
         CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
 
-        EVT PtrVT =
-            DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
+        EVT PtrVT = getPointerTy(DAG.getDataLayout());
         unsigned int i, j;
         for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
           SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
@@ -5100,7 +5023,7 @@ ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
 SDValue
 ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
   // Let legalize expand this if it isn't a legal type yet.
-  if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
+  if (!isTypeLegal(Op.getValueType()))
     return SDValue();
 
   SDValue Value, OverflowCmp;
@@ -5146,7 +5069,7 @@ static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
 SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
                                              SelectionDAG &DAG) const {
   // Let legalize expand this if it isn't a legal type yet.
-  if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
+  if (!isTypeLegal(Op.getValueType()))
     return SDValue();
 
   SDValue LHS = Op.getOperand(0);
@@ -5244,7 +5167,7 @@ SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   if (Cond.getResNo() == 1 &&
       (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
        Opc == ISD::USUBO)) {
-    if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
+    if (!isTypeLegal(Cond->getValueType(0)))
       return SDValue();
 
     SDValue Value, OverflowCmp;
@@ -5606,8 +5529,7 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (isUnsupportedFloatingType(LHS.getValueType())) {
-    DAG.getTargetLoweringInfo().softenSetCCOperands(
-        DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
+    softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
 
     // If softenSetCCOperands only returned one value, we should compare it to
     // zero.
@@ -5812,7 +5734,7 @@ SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
       (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
        Opc == ISD::USUBO || OptimizeMul)) {
     // Only lower legal XALUO ops.
-    if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
+    if (!isTypeLegal(Cond->getValueType(0)))
       return SDValue();
 
     // The actual operation with overflow check.
@@ -5842,8 +5764,7 @@ SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
 
   if (isUnsupportedFloatingType(LHS.getValueType())) {
-    DAG.getTargetLoweringInfo().softenSetCCOperands(
-        DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
+    softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
 
     // If softenSetCCOperands only returned one value, we should compare it to
     // zero.
@@ -5863,7 +5784,7 @@ SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
        Opc == ISD::USUBO || OptimizeMul) &&
       (CC == ISD::SETEQ || CC == ISD::SETNE)) {
     // Only lower legal XALUO ops.
-    if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
+    if (!isTypeLegal(LHS->getValueType(0)))
       return SDValue();
 
     // The actual operation with overflow check.
@@ -6331,7 +6252,6 @@ static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
 /// vectors), since the legalizer won't know what to do with that.
 SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
                                          const ARMSubtarget *Subtarget) const {
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDLoc dl(N);
   SDValue Op = N->getOperand(0);
 
@@ -6358,7 +6278,7 @@ SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   // Turn i64->f64 into VMOVDRR.
-  if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) {
+  if (SrcVT == MVT::i64 && isTypeLegal(DstVT)) {
     // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
     // if we can combine the bitcast with its source.
     if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
@@ -6370,7 +6290,7 @@ SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
   }
 
   // Turn f64->i64 into VMOVRRD.
-  if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) {
+  if (DstVT == MVT::i64 && isTypeLegal(SrcVT)) {
     SDValue Cvt;
     if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
         SrcVT.getVectorNumElements() > 1)
@@ -10007,7 +9927,6 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
   auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // Pair of floats / doubles used to pass the result.
   Type *RetTy = StructType::get(ArgTy, ArgTy);
@@ -10021,7 +9940,7 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
     const uint64_t ByteSize = DL.getTypeAllocSize(RetTy);
     const Align StackAlign = DL.getPrefTypeAlign(RetTy);
     int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
-    SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL));
+    SRet = DAG.getFrameIndex(FrameIdx, getPointerTy(DL));
 
     ArgListEntry Entry;
     Entry.Node = SRet;
@@ -10079,15 +9998,14 @@ SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
   SDLoc dl(Op);
 
   const auto &DL = DAG.getDataLayout();
-  const auto &TLI = DAG.getTargetLoweringInfo();
-
-  const char *Name = nullptr;
+  RTLIB::Libcall LC;
   if (Signed)
-    Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64";
+    LC = VT == MVT::i32 ? RTLIB::SDIVREM_I32 : RTLIB::SDIVREM_I64;
   else
-    Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64";
+    LC = VT == MVT::i32 ? RTLIB::UDIVREM_I32 : RTLIB::UDIVREM_I64;
 
-  SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
+  const char *Name = getLibcallName(LC);
+  SDValue ES = DAG.getExternalSymbol(Name, getPointerTy(DL));
 
   ARMTargetLowering::ArgListTy Args;
 
@@ -10177,7 +10095,6 @@ void ARMTargetLowering::ExpandDIV_Windows(
     SDValue Op, SelectionDAG &DAG, bool Signed,
     SmallVectorImpl<SDValue> &Results) const {
   const auto &DL = DAG.getDataLayout();
-  const auto &TLI = DAG.getTargetLoweringInfo();
 
   assert(Op.getValueType() == MVT::i64 &&
          "unexpected type for custom lowering DIV");
@@ -10189,7 +10106,7 @@ void ARMTargetLowering::ExpandDIV_Windows(
 
   SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
   SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
-                              DAG.getConstant(32, dl, TLI.getPointerTy(DL)));
+                              DAG.getConstant(32, dl, getPointerTy(DL)));
   Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper);
 
   Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lower, Upper));
@@ -10601,8 +10518,8 @@ SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
   // If we don't have instructions of this float type then soften to a libcall
   // and use SETCC instead.
   if (isUnsupportedFloatingType(LHS.getValueType())) {
-    DAG.getTargetLoweringInfo().softenSetCCOperands(
-      DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS, Chain, IsSignaling);
+    softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS,
+                        Chain, IsSignaling);
     if (!RHS.getNode()) {
       RHS = DAG.getConstant(0, dl, LHS.getValueType());
       CC = ISD::SETNE;
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMInstrMVE.td b/external/llvm-project/llvm/lib/Target/ARM/ARMInstrMVE.td
index 6dd8a374a92a..9dffd945d5ba 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -3527,7 +3527,7 @@ multiclass MVE_VRINT_m<MVEVectorVTInfo VTI, string suffix, bits<3> opcode,
 }
 
 multiclass MVE_VRINT_ops<MVEVectorVTInfo VTI> {
-  defm N : MVE_VRINT_m<VTI, "n", 0b000, int_arm_mve_vrintn>;
+  defm N : MVE_VRINT_m<VTI, "n", 0b000, froundeven>;
   defm X : MVE_VRINT_m<VTI, "x", 0b001, frint>;
   defm A : MVE_VRINT_m<VTI, "a", 0b010, fround>;
   defm Z : MVE_VRINT_m<VTI, "z", 0b011, ftrunc>;
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMMCInstLower.cpp b/external/llvm-project/llvm/lib/Target/ARM/ARMMCInstLower.cpp
index 6892db6eb52c..f5d6597f214d 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMMCInstLower.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMMCInstLower.cpp
@@ -18,7 +18,7 @@
 #include "ARMSubtarget.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -37,9 +37,9 @@ using namespace llvm;
 
 MCOperand ARMAsmPrinter::GetSymbolRef(const MachineOperand &MO,
                                       const MCSymbol *Symbol) {
-  auto Specifier = ARMMCExpr::VK_None;
+  auto Specifier = ARM::S_None;
   if (MO.getTargetFlags() & ARMII::MO_SBREL)
-    Specifier = ARMMCExpr::VK_SBREL;
+    Specifier = ARM::S_SBREL;
 
   const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, Specifier, OutContext);
   switch (MO.getTargetFlags() & ARMII::MO_OPTION_MASK) {
@@ -49,27 +49,27 @@ MCOperand ARMAsmPrinter::GetSymbolRef(const MachineOperand &MO,
     break;
   case ARMII::MO_LO16:
     Expr = MCSymbolRefExpr::create(Symbol, Specifier, OutContext);
-    Expr = ARMMCExpr::createLower16(Expr, OutContext);
+    Expr = ARM::createLower16(Expr, OutContext);
     break;
   case ARMII::MO_HI16:
     Expr = MCSymbolRefExpr::create(Symbol, Specifier, OutContext);
-    Expr = ARMMCExpr::createUpper16(Expr, OutContext);
+    Expr = ARM::createUpper16(Expr, OutContext);
     break;
   case ARMII::MO_LO_0_7:
     Expr = MCSymbolRefExpr::create(Symbol, Specifier, OutContext);
-    Expr = ARMMCExpr::createLower0_7(Expr, OutContext);
+    Expr = ARM::createLower0_7(Expr, OutContext);
     break;
   case ARMII::MO_LO_8_15:
     Expr = MCSymbolRefExpr::create(Symbol, Specifier, OutContext);
-    Expr = ARMMCExpr::createLower8_15(Expr, OutContext);
+    Expr = ARM::createLower8_15(Expr, OutContext);
     break;
   case ARMII::MO_HI_0_7:
     Expr = MCSymbolRefExpr::create(Symbol, Specifier, OutContext);
-    Expr = ARMMCExpr::createUpper0_7(Expr, OutContext);
+    Expr = ARM::createUpper0_7(Expr, OutContext);
     break;
   case ARMII::MO_HI_8_15:
     Expr = MCSymbolRefExpr::create(Symbol, Specifier, OutContext);
-    Expr = ARMMCExpr::createUpper8_15(Expr, OutContext);
+    Expr = ARM::createUpper8_15(Expr, OutContext);
     break;
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMSubtarget.h b/external/llvm-project/llvm/lib/Target/ARM/ARMSubtarget.h
index 7329d3f2055f..890a22f574a6 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -348,31 +348,11 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
   bool isTargetMachO() const { return TargetTriple.isOSBinFormatMachO(); }
 
-  // ARM EABI is the bare-metal EABI described in ARM ABI documents and
-  // can be accessed via -target arm-none-eabi. This is NOT GNUEABI.
-  // FIXME: Add a flag for bare-metal for that target and set Triple::EABI
-  // even for GNUEABI, so we can make a distinction here and still conform to
-  // the EABI on GNU (and Android) mode. This requires change in Clang, too.
-  // FIXME: The Darwin exception is temporary, while we move users to
-  // "*-*-*-macho" triples as quickly as possible.
-  bool isTargetAEABI() const {
-    return (TargetTriple.getEnvironment() == Triple::EABI ||
-            TargetTriple.getEnvironment() == Triple::EABIHF) &&
-           !isTargetDarwin() && !isTargetWindows();
-  }
-  bool isTargetGNUAEABI() const {
-    return (TargetTriple.getEnvironment() == Triple::GNUEABI ||
-            TargetTriple.getEnvironment() == Triple::GNUEABIT64 ||
-            TargetTriple.getEnvironment() == Triple::GNUEABIHF ||
-            TargetTriple.getEnvironment() == Triple::GNUEABIHFT64) &&
-           !isTargetDarwin() && !isTargetWindows();
-  }
-  bool isTargetMuslAEABI() const {
-    return (TargetTriple.getEnvironment() == Triple::MuslEABI ||
-            TargetTriple.getEnvironment() == Triple::MuslEABIHF ||
-            TargetTriple.getEnvironment() == Triple::OpenHOS) &&
-           !isTargetDarwin() && !isTargetWindows();
-  }
+  bool isTargetAEABI() const { return TargetTriple.isTargetAEABI(); }
+
+  bool isTargetGNUAEABI() const { return TargetTriple.isTargetGNUAEABI(); }
+
+  bool isTargetMuslAEABI() const { return TargetTriple.isTargetMuslAEABI(); }
 
   // ARM Targets that support EHABI exception handling standard
   // Darwin uses SjLj. Other targets might need more checks.
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/external/llvm-project/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 80a1e77a305c..563e69a65ab3 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -328,7 +328,7 @@ ARMBaseTargetMachine::getTargetTransformInfo(const Function &F) const {
 
 ScheduleDAGInstrs *
 ARMBaseTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
-  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  ScheduleDAGMILive *DAG = createSchedLive(C);
   // add DAG Mutations here.
   const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
   if (ST.hasFusion())
@@ -338,7 +338,7 @@ ARMBaseTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
 
 ScheduleDAGInstrs *
 ARMBaseTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
-  ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
+  ScheduleDAGMI *DAG = createSchedPostRA(C);
   // add DAG Mutations here.
   const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
   if (ST.hasFusion())
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMTargetObjectFile.cpp b/external/llvm-project/llvm/lib/Target/ARM/ARMTargetObjectFile.cpp
index 131703ec082b..a0a400f93848 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMTargetObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMTargetObjectFile.cpp
@@ -9,7 +9,7 @@
 #include "ARMTargetObjectFile.h"
 #include "ARMSubtarget.h"
 #include "ARMTargetMachine.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -30,7 +30,7 @@ using namespace dwarf;
 //===----------------------------------------------------------------------===//
 
 ARMElfTargetObjectFile::ARMElfTargetObjectFile() {
-  PLTRelativeSpecifier = ARMMCExpr::VK_PREL31;
+  PLTRelativeSpecifier = ARM::S_PREL31;
   SupportIndirectSymViaGOTPCRel = true;
 }
 
@@ -68,14 +68,14 @@ const MCExpr *ARMElfTargetObjectFile::getIndirectSymViaGOTPCRel(
     int64_t Offset, MachineModuleInfo *MMI, MCStreamer &Streamer) const {
   int64_t FinalOffset = Offset + MV.getConstant();
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, ARMMCExpr::VK_GOT_PREL, getContext());
+      MCSymbolRefExpr::create(Sym, ARM::S_GOT_PREL, getContext());
   const MCExpr *Off = MCConstantExpr::create(FinalOffset, getContext());
   return MCBinaryExpr::createAdd(Res, Off, getContext());
 }
 
 const MCExpr *ARMElfTargetObjectFile::
 getIndirectSymViaRWPI(const MCSymbol *Sym) const {
-  return MCSymbolRefExpr::create(Sym, ARMMCExpr::VK_SBREL, getContext());
+  return MCSymbolRefExpr::create(Sym, ARM::S_SBREL, getContext());
 }
 
 const MCExpr *ARMElfTargetObjectFile::getTTypeGlobalReference(
@@ -87,13 +87,13 @@ const MCExpr *ARMElfTargetObjectFile::getTTypeGlobalReference(
 
   assert(Encoding == DW_EH_PE_absptr && "Can handle absptr encoding only");
 
-  return MCSymbolRefExpr::create(TM.getSymbol(GV), ARMMCExpr::VK_TARGET2,
+  return MCSymbolRefExpr::create(TM.getSymbol(GV), ARM::S_TARGET2,
                                  getContext());
 }
 
 const MCExpr *ARMElfTargetObjectFile::
 getDebugThreadLocalSymbol(const MCSymbol *Sym) const {
-  return MCSymbolRefExpr::create(Sym, ARMMCExpr::VK_TLSLDO, getContext());
+  return MCSymbolRefExpr::create(Sym, ARM::S_TLSLDO, getContext());
 }
 
 static bool isExecuteOnlyFunction(const GlobalObject *GO, SectionKind SK,
diff --git a/external/llvm-project/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 20a2c5951108..c1af4e3dc5da 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -54,7 +54,7 @@ namespace TPLoop {
 enum MemTransfer { ForceDisabled = 0, ForceEnabled, Allow };
 }
 
-class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
+class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
   using BaseT = BasicTTIImplBase<ARMTTIImpl>;
   using TTI = TargetTransformInfo;
 
diff --git a/external/llvm-project/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/external/llvm-project/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 19c417b2c6e9..f3bdcd64805d 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -11,7 +11,7 @@
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
 #include "MCTargetDesc/ARMInstPrinter.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "MCTargetDesc/ARMMCTargetDesc.h"
 #include "TargetInfo/ARMTargetInfo.h"
 #include "Utils/ARMBaseInfo.h"
@@ -454,7 +454,7 @@ class ARMAsmParser : public MCTargetAsmParser {
   bool parseMemory(OperandVector &);
   bool parseOperand(OperandVector &, StringRef Mnemonic);
   bool parseImmExpr(int64_t &Out);
-  bool parsePrefix(ARMMCExpr::Specifier &);
+  bool parsePrefix(ARM::Specifier &);
   bool parseMemRegOffsetShift(ARM_AM::ShiftOpc &ShiftType,
                               unsigned &ShiftAmount);
   bool parseLiteralValues(unsigned Size, SMLoc L);
@@ -1326,9 +1326,9 @@ class ARMOperand : public MCParsedAsmOperand {
     if (isImm() && !isa<MCConstantExpr>(getImm())) {
       // We want to avoid matching :upper16: and :lower16: as we want these
       // expressions to match in isImm0_65535Expr()
-      const ARMMCExpr *ARM16Expr = dyn_cast<ARMMCExpr>(getImm());
-      return (!ARM16Expr || (ARM16Expr->getSpecifier() != ARMMCExpr::VK_HI16 &&
-                             ARM16Expr->getSpecifier() != ARMMCExpr::VK_LO16));
+      auto *ARM16Expr = dyn_cast<MCSpecifierExpr>(getImm());
+      return (!ARM16Expr || (ARM16Expr->getSpecifier() != ARM::S_HI16 &&
+                             ARM16Expr->getSpecifier() != ARM::S_LO16));
     }
     if (!isImm()) return false;
     const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
@@ -6424,7 +6424,7 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
     // ":upper8_15:", expression prefixes
     // FIXME: Check it's an expression prefix,
     // e.g. (FOO - :lower16:BAR) isn't legal.
-    ARMMCExpr::Specifier Spec;
+    ARM::Specifier Spec;
     if (parsePrefix(Spec))
       return true;
 
@@ -6432,7 +6432,8 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
     if (getParser().parseExpression(SubExprVal))
       return true;
 
-    const MCExpr *ExprVal = ARMMCExpr::create(Spec, SubExprVal, getContext());
+    const auto *ExprVal =
+        MCSpecifierExpr::create(SubExprVal, Spec, getContext());
     E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
     Operands.push_back(ARMOperand::CreateImm(ExprVal, S, E, *this));
     return false;
@@ -6471,9 +6472,9 @@ bool ARMAsmParser::parseImmExpr(int64_t &Out) {
 // parsePrefix - Parse ARM 16-bit relocations expression prefixes, i.e.
 // :lower16: and :upper16: and Thumb 8-bit relocation expression prefixes, i.e.
 // :upper8_15:, :upper0_7:, :lower8_15: and :lower0_7:
-bool ARMAsmParser::parsePrefix(ARMMCExpr::Specifier &Spec) {
+bool ARMAsmParser::parsePrefix(ARM::Specifier &Spec) {
   MCAsmParser &Parser = getParser();
-  Spec = ARMMCExpr::VK_None;
+  Spec = ARM::S_None;
 
   // consume an optional '#' (GNU compatibility)
   if (getLexer().is(AsmToken::Hash))
@@ -6495,15 +6496,15 @@ bool ARMAsmParser::parsePrefix(ARMMCExpr::Specifier &Spec) {
   };
   static const struct PrefixEntry {
     const char *Spelling;
-    ARMMCExpr::Specifier Spec;
+    ARM::Specifier Spec;
     uint8_t SupportedFormats;
   } PrefixEntries[] = {
-      {"upper16", ARMMCExpr::VK_HI16, COFF | ELF | MACHO},
-      {"lower16", ARMMCExpr::VK_LO16, COFF | ELF | MACHO},
-      {"upper8_15", ARMMCExpr::VK_HI_8_15, ELF},
-      {"upper0_7", ARMMCExpr::VK_HI_0_7, ELF},
-      {"lower8_15", ARMMCExpr::VK_LO_8_15, ELF},
-      {"lower0_7", ARMMCExpr::VK_LO_0_7, ELF},
+      {"upper16", ARM::S_HI16, COFF | ELF | MACHO},
+      {"lower16", ARM::S_LO16, COFF | ELF | MACHO},
+      {"upper8_15", ARM::S_HI_8_15, ELF},
+      {"upper0_7", ARM::S_HI_0_7, ELF},
+      {"lower8_15", ARM::S_LO_8_15, ELF},
+      {"lower0_7", ARM::S_LO_0_7, ELF},
   };
 
   StringRef IDVal = Parser.getTok().getIdentifier();
@@ -6879,11 +6880,11 @@ static bool isThumbI8Relocation(MCParsedAsmOperand &MCOp) {
   const MCExpr *E = dyn_cast<MCExpr>(Op.getImm());
   if (!E)
     return false;
-  const ARMMCExpr *ARM16Expr = dyn_cast<ARMMCExpr>(E);
-  if (ARM16Expr && (ARM16Expr->getSpecifier() == ARMMCExpr::VK_HI_8_15 ||
-                    ARM16Expr->getSpecifier() == ARMMCExpr::VK_HI_0_7 ||
-                    ARM16Expr->getSpecifier() == ARMMCExpr::VK_LO_8_15 ||
-                    ARM16Expr->getSpecifier() == ARMMCExpr::VK_LO_0_7))
+  auto *ARM16Expr = dyn_cast<MCSpecifierExpr>(E);
+  if (ARM16Expr && (ARM16Expr->getSpecifier() == ARM::S_HI_8_15 ||
+                    ARM16Expr->getSpecifier() == ARM::S_HI_0_7 ||
+                    ARM16Expr->getSpecifier() == ARM::S_LO_8_15 ||
+                    ARM16Expr->getSpecifier() == ARM::S_LO_0_7))
     return true;
   return false;
 }
@@ -8286,9 +8287,9 @@ bool ARMAsmParser::validateInstruction(MCInst &Inst,
     if (CE) break;
     const MCExpr *E = dyn_cast<MCExpr>(Op.getImm());
     if (!E) break;
-    const ARMMCExpr *ARM16Expr = dyn_cast<ARMMCExpr>(E);
-    if (!ARM16Expr || (ARM16Expr->getSpecifier() != ARMMCExpr::VK_HI16 &&
-                       ARM16Expr->getSpecifier() != ARMMCExpr::VK_LO16))
+    auto *ARM16Expr = dyn_cast<MCSpecifierExpr>(E);
+    if (!ARM16Expr || (ARM16Expr->getSpecifier() != ARM::S_HI16 &&
+                       ARM16Expr->getSpecifier() != ARM::S_LO16))
       return Error(
           Op.getStartLoc(),
           "immediate expression for mov requires :lower16: or :upper16");
@@ -12437,7 +12438,7 @@ bool ARMAsmParser::parseDirectiveTLSDescSeq(SMLoc L) {
 
   auto *Sym = getContext().getOrCreateSymbol(Parser.getTok().getIdentifier());
   const auto *SRE =
-      MCSymbolRefExpr::create(Sym, ARMMCExpr::VK_TLSDESCSEQ, getContext());
+      MCSymbolRefExpr::create(Sym, ARM::S_TLSDESCSEQ, getContext());
   Lex();
 
   if (parseEOL())
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 075fe8687b5a..f43fdae554b8 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -12,7 +12,7 @@
 #include "MCTargetDesc/ARMAsmBackendELF.h"
 #include "MCTargetDesc/ARMAsmBackendWinCOFF.h"
 #include "MCTargetDesc/ARMFixupKinds.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "MCTargetDesc/ARMMCTargetDesc.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -619,7 +619,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     // Offset by 8 just as above.
     if (const MCSymbolRefExpr *SRE =
             dyn_cast<MCSymbolRefExpr>(Fixup.getValue()))
-      if (getSpecifier(SRE) == ARMMCExpr::VK_TLSCALL)
+      if (SRE->getSpecifier() == ARM::S_TLSCALL)
         return 0;
     return 0xffffff & (Value >> 2);
   case ARM::fixup_t2_uncondbranch: {
@@ -746,7 +746,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     uint32_t offset = (Value - 4) >> 2;
     if (const MCSymbolRefExpr *SRE =
             dyn_cast<MCSymbolRefExpr>(Fixup.getValue()))
-      if (getSpecifier(SRE) == ARMMCExpr::VK_TLSCALL)
+      if (SRE->getSpecifier() == ARM::S_TLSCALL)
         offset = 0;
     uint32_t signBit = (offset & 0x400000) >> 22;
     uint32_t I1Bit = (offset & 0x200000) >> 21;
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
index f5a6ee5c5a2e..b0ebb74424c7 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/ARMFixupKinds.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "MCTargetDesc/ARMMCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAssembler.h"
@@ -87,16 +87,16 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
   };
 
   switch (Specifier) {
-  case ARMMCExpr::VK_GOTTPOFF:
-  case ARMMCExpr::VK_GOTTPOFF_FDPIC:
-  case ARMMCExpr::VK_TLSCALL:
-  case ARMMCExpr::VK_TLSDESC:
-  case ARMMCExpr::VK_TLSGD:
-  case ARMMCExpr::VK_TLSGD_FDPIC:
-  case ARMMCExpr::VK_TLSLDM:
-  case ARMMCExpr::VK_TLSLDM_FDPIC:
-  case ARMMCExpr::VK_TLSLDO:
-  case ARMMCExpr::VK_TPOFF:
+  case ARM::S_GOTTPOFF:
+  case ARM::S_GOTTPOFF_FDPIC:
+  case ARM::S_TLSCALL:
+  case ARM::S_TLSDESC:
+  case ARM::S_TLSGD:
+  case ARM::S_TLSGD_FDPIC:
+  case ARM::S_TLSLDM:
+  case ARM::S_TLSLDM_FDPIC:
+  case ARM::S_TLSLDO:
+  case ARM::S_TPOFF:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
@@ -115,7 +115,7 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
         reportError(Fixup.getLoc(),
                     "invalid fixup for 4-byte pc-relative data relocation");
         return ELF::R_ARM_NONE;
-      case ARMMCExpr::VK_None: {
+      case ARM::S_None: {
         if (const auto *SA = Target.getAddSym()) {
           // For GNU AS compatibility expressions such as
           // _GLOBAL_OFFSET_TABLE_ - label emit a R_ARM_BASE_PREL relocation.
@@ -124,19 +124,19 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
         }
         return ELF::R_ARM_REL32;
       }
-      case ARMMCExpr::VK_GOTTPOFF:
+      case ARM::S_GOTTPOFF:
         return ELF::R_ARM_TLS_IE32;
-      case ARMMCExpr::VK_GOT_PREL:
+      case ARM::S_GOT_PREL:
         return ELF::R_ARM_GOT_PREL;
-      case ARMMCExpr::VK_PREL31:
+      case ARM::S_PREL31:
         return ELF::R_ARM_PREL31;
       }
     case ARM::fixup_arm_blx:
     case ARM::fixup_arm_uncondbl:
       switch (Specifier) {
-      case ARMMCExpr::VK_PLT:
+      case ARM::S_PLT:
         return ELF::R_ARM_CALL;
-      case ARMMCExpr::VK_TLSCALL:
+      case ARM::S_TLSCALL:
         return ELF::R_ARM_TLS_CALL;
       default:
         return ELF::R_ARM_CALL;
@@ -172,7 +172,7 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     case ARM::fixup_arm_thumb_bl:
     case ARM::fixup_arm_thumb_blx:
       switch (Specifier) {
-      case ARMMCExpr::VK_TLSCALL:
+      case ARM::S_TLSCALL:
         return ELF::R_ARM_THM_TLS_CALL;
       default:
         return ELF::R_ARM_THM_CALL;
@@ -206,7 +206,7 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       reportError(Fixup.getLoc(), "invalid fixup for 1-byte data relocation");
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_None:
+    case ARM::S_None:
       return ELF::R_ARM_ABS8;
     }
   case FK_Data_2:
@@ -214,7 +214,7 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       reportError(Fixup.getLoc(), "invalid fixup for 2-byte data relocation");
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_None:
+    case ARM::S_None:
       return ELF::R_ARM_ABS16;
     }
   case FK_Data_4:
@@ -222,51 +222,51 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       reportError(Fixup.getLoc(), "invalid fixup for 4-byte data relocation");
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_ARM_NONE:
+    case ARM::S_ARM_NONE:
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_GOT:
+    case ARM::S_GOT:
       return ELF::R_ARM_GOT_BREL;
-    case ARMMCExpr::VK_TLSGD:
+    case ARM::S_TLSGD:
       return ELF::R_ARM_TLS_GD32;
-    case ARMMCExpr::VK_TPOFF:
+    case ARM::S_TPOFF:
       return ELF::R_ARM_TLS_LE32;
-    case ARMMCExpr::VK_GOTTPOFF:
+    case ARM::S_GOTTPOFF:
       return ELF::R_ARM_TLS_IE32;
-    case ARMMCExpr::VK_None:
+    case ARM::S_None:
       return ELF::R_ARM_ABS32;
-    case ARMMCExpr::VK_GOTOFF:
+    case ARM::S_GOTOFF:
       return ELF::R_ARM_GOTOFF32;
-    case ARMMCExpr::VK_GOT_PREL:
+    case ARM::S_GOT_PREL:
       return ELF::R_ARM_GOT_PREL;
-    case ARMMCExpr::VK_TARGET1:
+    case ARM::S_TARGET1:
       return ELF::R_ARM_TARGET1;
-    case ARMMCExpr::VK_TARGET2:
+    case ARM::S_TARGET2:
       return ELF::R_ARM_TARGET2;
-    case ARMMCExpr::VK_PREL31:
+    case ARM::S_PREL31:
       return ELF::R_ARM_PREL31;
-    case ARMMCExpr::VK_SBREL:
+    case ARM::S_SBREL:
       return ELF::R_ARM_SBREL32;
-    case ARMMCExpr::VK_TLSLDO:
+    case ARM::S_TLSLDO:
       return ELF::R_ARM_TLS_LDO32;
-    case ARMMCExpr::VK_TLSCALL:
+    case ARM::S_TLSCALL:
       return ELF::R_ARM_TLS_CALL;
-    case ARMMCExpr::VK_TLSDESC:
+    case ARM::S_TLSDESC:
       return ELF::R_ARM_TLS_GOTDESC;
-    case ARMMCExpr::VK_TLSLDM:
+    case ARM::S_TLSLDM:
       return ELF::R_ARM_TLS_LDM32;
-    case ARMMCExpr::VK_TLSDESCSEQ:
+    case ARM::S_TLSDESCSEQ:
       return ELF::R_ARM_TLS_DESCSEQ;
-    case ARMMCExpr::VK_FUNCDESC:
+    case ARM::S_FUNCDESC:
       return CheckFDPIC(ELF::R_ARM_FUNCDESC);
-    case ARMMCExpr::VK_GOTFUNCDESC:
+    case ARM::S_GOTFUNCDESC:
       return CheckFDPIC(ELF::R_ARM_GOTFUNCDESC);
-    case ARMMCExpr::VK_GOTOFFFUNCDESC:
+    case ARM::S_GOTOFFFUNCDESC:
       return CheckFDPIC(ELF::R_ARM_GOTOFFFUNCDESC);
-    case ARMMCExpr::VK_TLSGD_FDPIC:
+    case ARM::S_TLSGD_FDPIC:
       return CheckFDPIC(ELF::R_ARM_TLS_GD32_FDPIC);
-    case ARMMCExpr::VK_TLSLDM_FDPIC:
+    case ARM::S_TLSLDM_FDPIC:
       return CheckFDPIC(ELF::R_ARM_TLS_LDM32_FDPIC);
-    case ARMMCExpr::VK_GOTTPOFF_FDPIC:
+    case ARM::S_GOTTPOFF_FDPIC:
       return CheckFDPIC(ELF::R_ARM_TLS_IE32_FDPIC);
     }
   case ARM::fixup_arm_condbranch:
@@ -277,9 +277,9 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       reportError(Fixup.getLoc(), "invalid fixup for ARM MOVT instruction");
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_None:
+    case ARM::S_None:
       return ELF::R_ARM_MOVT_ABS;
-    case ARMMCExpr::VK_SBREL:
+    case ARM::S_SBREL:
       return ELF::R_ARM_MOVT_BREL;
     }
   case ARM::fixup_arm_movw_lo16:
@@ -287,9 +287,9 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       reportError(Fixup.getLoc(), "invalid fixup for ARM MOVW instruction");
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_None:
+    case ARM::S_None:
       return ELF::R_ARM_MOVW_ABS_NC;
-    case ARMMCExpr::VK_SBREL:
+    case ARM::S_SBREL:
       return ELF::R_ARM_MOVW_BREL_NC;
     }
   case ARM::fixup_t2_movt_hi16:
@@ -297,9 +297,9 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       reportError(Fixup.getLoc(), "invalid fixup for Thumb MOVT instruction");
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_None:
+    case ARM::S_None:
       return ELF::R_ARM_THM_MOVT_ABS;
-    case ARMMCExpr::VK_SBREL:
+    case ARM::S_SBREL:
       return ELF::R_ARM_THM_MOVT_BREL;
     }
   case ARM::fixup_t2_movw_lo16:
@@ -307,9 +307,9 @@ unsigned ARMELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       reportError(Fixup.getLoc(), "invalid fixup for Thumb MOVW instruction");
       return ELF::R_ARM_NONE;
-    case ARMMCExpr::VK_None:
+    case ARM::S_None:
       return ELF::R_ARM_THM_MOVW_ABS_NC;
-    case ARMMCExpr::VK_SBREL:
+    case ARM::S_SBREL:
       return ELF::R_ARM_THM_MOVW_BREL_NC;
     }
 
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index 7884b10a48a9..73ad62ed7953 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -14,7 +14,7 @@
 
 #include "ARMMCTargetDesc.h"
 #include "ARMUnwindOpAsm.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "Utils/ARMBaseInfo.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallString.h"
@@ -288,7 +288,7 @@ void ARMTargetAsmStreamer::emitThumbSet(MCSymbol *Symbol, const MCExpr *Value) {
   OS << "\t.thumb_set\t";
   Symbol->print(OS, MAI);
   OS << ", ";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   OS << '\n';
 }
 
@@ -590,7 +590,7 @@ class ARMELFStreamer : public MCELFStreamer {
   /// necessary.
   void emitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc) override {
     if (const MCSymbolRefExpr *SRE = dyn_cast_or_null<MCSymbolRefExpr>(Value)) {
-      if (getSpecifier(SRE) == ARMMCExpr::VK_SBREL && !(Size == 4)) {
+      if (SRE->getSpecifier() == ARM::S_SBREL && !(Size == 4)) {
         getContext().reportError(Loc, "relocated expression must be 32-bit");
         return;
       }
@@ -1255,7 +1255,7 @@ void ARMELFStreamer::emitFnEnd() {
     EmitPersonalityFixup(GetAEABIUnwindPersonalityName(PersonalityIndex));
 
   const MCSymbolRefExpr *FnStartRef =
-      MCSymbolRefExpr::create(FnStart, ARMMCExpr::VK_PREL31, getContext());
+      MCSymbolRefExpr::create(FnStart, ARM::S_PREL31, getContext());
 
   emitValue(FnStartRef, 4);
 
@@ -1264,7 +1264,7 @@ void ARMELFStreamer::emitFnEnd() {
   } else if (ExTab) {
     // Emit a reference to the unwind opcodes in the ".ARM.extab" section.
     const MCSymbolRefExpr *ExTabEntryRef =
-        MCSymbolRefExpr::create(ExTab, ARMMCExpr::VK_PREL31, getContext());
+        MCSymbolRefExpr::create(ExTab, ARM::S_PREL31, getContext());
     emitValue(ExTabEntryRef, 4);
   } else {
     // For the __aeabi_unwind_cpp_pr0, we have to emit the unwind opcodes in
@@ -1294,8 +1294,8 @@ void ARMELFStreamer::emitCantUnwind() { CantUnwind = true; }
 void ARMELFStreamer::EmitPersonalityFixup(StringRef Name) {
   const MCSymbol *PersonalitySym = getContext().getOrCreateSymbol(Name);
 
-  const MCSymbolRefExpr *PersonalityRef = MCSymbolRefExpr::create(
-      PersonalitySym, ARMMCExpr::VK_ARM_NONE, getContext());
+  const MCSymbolRefExpr *PersonalityRef =
+      MCSymbolRefExpr::create(PersonalitySym, ARM::S_ARM_NONE, getContext());
 
   visitUsedExpr(*PersonalityRef);
   MCDataFragment *DF = getOrCreateDataFragment();
@@ -1341,7 +1341,7 @@ void ARMELFStreamer::FlushUnwindOpcodes(bool NoHandlerData) {
   // Emit personality
   if (Personality) {
     const MCSymbolRefExpr *PersonalityRef = MCSymbolRefExpr::create(
-        Personality, uint16_t(ARMMCExpr::VK_PREL31), getContext());
+        Personality, uint16_t(ARM::S_PREL31), getContext());
 
     emitValue(PersonalityRef, 4);
   }
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
index ac90095a20be..ad00b171aaf6 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
@@ -350,7 +350,7 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     switch (Expr->getKind()) {
     case MCExpr::Binary:
       O << '#';
-      Expr->print(O, &MAI);
+      MAI.printExpr(O, *Expr);
       break;
     case MCExpr::Constant: {
       // If a symbolic branch target was added as a constant expression then
@@ -360,7 +360,7 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
       int64_t TargetAddress;
       if (!Constant->evaluateAsAbsolute(TargetAddress)) {
         O << '#';
-        Expr->print(O, &MAI);
+        MAI.printExpr(O, *Expr);
       } else {
         O << "0x";
         O.write_hex(static_cast<uint32_t>(TargetAddress));
@@ -370,7 +370,7 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     default:
       // FIXME: Should we always treat this as if it is a constant literal and
       // prefix it with '#'?
-      Expr->print(O, &MAI);
+      MAI.printExpr(O, *Expr);
       break;
     }
   }
@@ -395,7 +395,7 @@ void ARMInstPrinter::printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum,
                                                raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(OpNum);
   if (MO1.isExpr()) {
-    MO1.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO1.getExpr());
     return;
   }
 
@@ -1081,7 +1081,7 @@ void ARMInstPrinter::printAdrLabelOperand(const MCInst *MI, unsigned OpNum,
   const MCOperand &MO = MI->getOperand(OpNum);
 
   if (MO.isExpr()) {
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
     return;
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
index 92121dd5704d..a3d86f690e4a 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
@@ -11,37 +11,37 @@
 //===----------------------------------------------------------------------===//
 
 #include "ARMMCAsmInfo.h"
-#include "MCTargetDesc/ARMMCExpr.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
 
 const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
-    {ARMMCExpr::VK_GOT_PREL, "GOT_PREL"},
-    {ARMMCExpr::VK_ARM_NONE, "none"},
-    {ARMMCExpr::VK_PREL31, "prel31"},
-    {ARMMCExpr::VK_SBREL, "sbrel"},
-    {ARMMCExpr::VK_TARGET1, "target1"},
-    {ARMMCExpr::VK_TARGET2, "target2"},
-    {ARMMCExpr::VK_TLSLDO, "TLSLDO"},
+    {ARM::S_GOT_PREL, "GOT_PREL"},
+    {ARM::S_ARM_NONE, "none"},
+    {ARM::S_PREL31, "prel31"},
+    {ARM::S_SBREL, "sbrel"},
+    {ARM::S_TARGET1, "target1"},
+    {ARM::S_TARGET2, "target2"},
+    {ARM::S_TLSLDO, "TLSLDO"},
     {MCSymbolRefExpr::VK_COFF_IMGREL32, "imgrel"},
-    {ARMMCExpr::VK_FUNCDESC, "FUNCDESC"},
-    {ARMMCExpr::VK_GOT, "GOT"},
-    {ARMMCExpr::VK_GOTFUNCDESC, "GOTFUNCDESC"},
-    {ARMMCExpr::VK_GOTOFF, "GOTOFF"},
-    {ARMMCExpr::VK_GOTOFFFUNCDESC, "GOTOFFFUNCDESC"},
-    {ARMMCExpr::VK_GOTTPOFF, "GOTTPOFF"},
-    {ARMMCExpr::VK_GOTTPOFF_FDPIC, "gottpoff_fdpic"},
-    {ARMMCExpr::VK_PLT, "PLT"},
+    {ARM::S_FUNCDESC, "FUNCDESC"},
+    {ARM::S_GOT, "GOT"},
+    {ARM::S_GOTFUNCDESC, "GOTFUNCDESC"},
+    {ARM::S_GOTOFF, "GOTOFF"},
+    {ARM::S_GOTOFFFUNCDESC, "GOTOFFFUNCDESC"},
+    {ARM::S_GOTTPOFF, "GOTTPOFF"},
+    {ARM::S_GOTTPOFF_FDPIC, "gottpoff_fdpic"},
+    {ARM::S_PLT, "PLT"},
     {MCSymbolRefExpr::VK_SECREL, "SECREL32"},
-    {ARMMCExpr::VK_TLSCALL, "tlscall"},
-    {ARMMCExpr::VK_TLSDESC, "tlsdesc"},
-    {ARMMCExpr::VK_TLSGD, "TLSGD"},
-    {ARMMCExpr::VK_TLSGD_FDPIC, "tlsgd_fdpic"},
-    {ARMMCExpr::VK_TLSLDM, "TLSLDM"},
-    {ARMMCExpr::VK_TLSLDM_FDPIC, "tlsldm_fdpic"},
-    {ARMMCExpr::VK_TPOFF, "TPOFF"},
+    {ARM::S_TLSCALL, "tlscall"},
+    {ARM::S_TLSDESC, "tlsdesc"},
+    {ARM::S_TLSGD, "TLSGD"},
+    {ARM::S_TLSGD_FDPIC, "tlsgd_fdpic"},
+    {ARM::S_TLSLDM, "TLSLDM"},
+    {ARM::S_TLSLDM_FDPIC, "tlsldm_fdpic"},
+    {ARM::S_TPOFF, "TPOFF"},
 };
 
 void ARMMCAsmInfoDarwin::anchor() { }
@@ -153,3 +153,62 @@ ARMCOFFMCAsmInfoGNU::ARMCOFFMCAsmInfoGNU() {
 
   initializeVariantKinds(variantKindDescs);
 }
+
+void ARM::printSpecifierExpr(const MCAsmInfo &MAI, raw_ostream &OS,
+                             const MCSpecifierExpr &Expr) {
+  switch (Expr.getSpecifier()) {
+  default:
+    llvm_unreachable("Invalid kind!");
+  case ARM::S_HI16:
+    OS << ":upper16:";
+    break;
+  case ARM::S_LO16:
+    OS << ":lower16:";
+    break;
+  case ARM::S_HI_8_15:
+    OS << ":upper8_15:";
+    break;
+  case ARM::S_HI_0_7:
+    OS << ":upper0_7:";
+    break;
+  case ARM::S_LO_8_15:
+    OS << ":lower8_15:";
+    break;
+  case ARM::S_LO_0_7:
+    OS << ":lower0_7:";
+    break;
+  }
+
+  const MCExpr *Sub = Expr.getSubExpr();
+  if (Sub->getKind() != MCExpr::SymbolRef)
+    OS << '(';
+  MAI.printExpr(OS, *Sub);
+  if (Sub->getKind() != MCExpr::SymbolRef)
+    OS << ')';
+}
+
+const MCSpecifierExpr *ARM::createUpper16(const MCExpr *Expr, MCContext &Ctx) {
+  return MCSpecifierExpr::create(Expr, ARM::S_HI16, Ctx);
+}
+
+const MCSpecifierExpr *ARM::createLower16(const MCExpr *Expr, MCContext &Ctx) {
+  return MCSpecifierExpr::create(Expr, ARM::S_LO16, Ctx);
+}
+
+const MCSpecifierExpr *ARM::createUpper8_15(const MCExpr *Expr,
+                                            MCContext &Ctx) {
+  return MCSpecifierExpr::create(Expr, ARM::S_HI_8_15, Ctx);
+}
+
+const MCSpecifierExpr *ARM::createUpper0_7(const MCExpr *Expr, MCContext &Ctx) {
+  return MCSpecifierExpr::create(Expr, ARM::S_HI_0_7, Ctx);
+}
+
+const MCSpecifierExpr *ARM::createLower8_15(const MCExpr *Expr,
+                                            MCContext &Ctx) {
+  return MCSpecifierExpr::create(Expr, ARM::S_LO_8_15, Ctx);
+}
+
+const MCSpecifierExpr *ARM::createLower0_7(const MCExpr *Expr, MCContext &Ctx) {
+  return MCSpecifierExpr::create(Expr, ARM::S_LO_0_7, Ctx);
+}
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
index 55d7b299674d..f3f075e99d96 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
@@ -16,15 +16,29 @@
 #include "llvm/MC/MCAsmInfoCOFF.h"
 #include "llvm/MC/MCAsmInfoDarwin.h"
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
 
 namespace llvm {
 class Triple;
 
+namespace ARM {
+void printSpecifierExpr(const MCAsmInfo &MAI, raw_ostream &OS,
+                        const MCSpecifierExpr &Expr);
+}
+
 class ARMMCAsmInfoDarwin : public MCAsmInfoDarwin {
   virtual void anchor();
 
 public:
   explicit ARMMCAsmInfoDarwin(const Triple &TheTriple);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override {
+    ARM::printSpecifierExpr(*this, OS, Expr);
+  }
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &, MCValue &,
+                                 const MCAssembler *) const override {
+    return false;
+  }
 };
 
 class ARMELFMCAsmInfo : public MCAsmInfoELF {
@@ -34,6 +48,14 @@ class ARMELFMCAsmInfo : public MCAsmInfoELF {
   explicit ARMELFMCAsmInfo(const Triple &TT);
 
   void setUseIntegratedAssembler(bool Value) override;
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override {
+    ARM::printSpecifierExpr(*this, OS, Expr);
+  }
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &, MCValue &,
+                                 const MCAssembler *) const override {
+    return false;
+  }
 };
 
 class ARMCOFFMCAsmInfoMicrosoft : public MCAsmInfoMicrosoft {
@@ -41,6 +63,14 @@ class ARMCOFFMCAsmInfoMicrosoft : public MCAsmInfoMicrosoft {
 
 public:
   explicit ARMCOFFMCAsmInfoMicrosoft();
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override {
+    ARM::printSpecifierExpr(*this, OS, Expr);
+  }
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &, MCValue &,
+                                 const MCAssembler *) const override {
+    return false;
+  }
 };
 
 class ARMCOFFMCAsmInfoGNU : public MCAsmInfoGNUCOFF {
@@ -48,8 +78,67 @@ class ARMCOFFMCAsmInfoGNU : public MCAsmInfoGNUCOFF {
 
 public:
   explicit ARMCOFFMCAsmInfoGNU();
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override {
+    ARM::printSpecifierExpr(*this, OS, Expr);
+  }
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &, MCValue &,
+                                 const MCAssembler *) const override {
+    return false;
+  }
+};
+
+namespace ARM {
+using Specifier = uint16_t;
+enum {
+  S_None,
+  S_HI16 =
+      MCSymbolRefExpr::FirstTargetSpecifier, // The R_ARM_MOVT_ABS relocation
+                                             // (:upper16: in the .s file)
+  S_LO16, // The R_ARM_MOVW_ABS_NC relocation (:lower16: in the .s file)
+
+  S_HI_8_15, // The R_ARM_THM_ALU_ABS_G3    relocation (:upper8_15: in
+             // the .s file)
+  S_HI_0_7,  // The R_ARM_THM_ALU_ABS_G2_NC relocation (:upper0_8: in the
+             // .s file)
+  S_LO_8_15, // The R_ARM_THM_ALU_ABS_G1_NC relocation (:lower8_15: in
+             // the .s file)
+  S_LO_0_7,  // The R_ARM_THM_ALU_ABS_G0_NC relocation (:lower0_7: in the
+             // .s file)
+
+  S_ARM_NONE,
+  S_FUNCDESC,
+  S_GOT,
+  S_GOTFUNCDESC,
+  S_GOTOFF,
+  S_GOTOFFFUNCDESC,
+  S_GOTTPOFF,
+  S_GOTTPOFF_FDPIC,
+  S_GOT_PREL,
+  S_PLT,
+  S_PREL31,
+  S_SBREL,
+  S_TARGET1,
+  S_TARGET2,
+  S_TLSCALL,
+  S_TLSDESC,
+  S_TLSDESCSEQ,
+  S_TLSGD,
+  S_TLSGD_FDPIC,
+  S_TLSLDM,
+  S_TLSLDM_FDPIC,
+  S_TLSLDO,
+  S_TPOFF,
 };
 
+const MCSpecifierExpr *createUpper16(const MCExpr *Expr, MCContext &Ctx);
+const MCSpecifierExpr *createLower16(const MCExpr *Expr, MCContext &Ctx);
+const MCSpecifierExpr *createUpper8_15(const MCExpr *Expr, MCContext &Ctx);
+const MCSpecifierExpr *createUpper0_7(const MCExpr *Expr, MCContext &Ctx);
+const MCSpecifierExpr *createLower8_15(const MCExpr *Expr, MCContext &Ctx);
+const MCSpecifierExpr *createLower0_7(const MCExpr *Expr, MCContext &Ctx);
+}
+
 } // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index 7fa4baac9970..fba32eae4dfa 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -13,7 +13,7 @@
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
 #include "MCTargetDesc/ARMFixupKinds.h"
-#include "MCTargetDesc/ARMMCExpr.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/SmallVector.h"
@@ -1191,8 +1191,8 @@ uint32_t ARMMCCodeEmitter::getHiLoImmOpValue(const MCInst &MI, unsigned OpIdx,
   // :lower0_7: assembly prefixes.
   const MCExpr *E = MO.getExpr();
   MCFixupKind Kind;
-  if (E->getKind() == MCExpr::Target) {
-    const ARMMCExpr *ARM16Expr = cast<ARMMCExpr>(E);
+  if (E->getKind() == MCExpr::Specifier) {
+    auto *ARM16Expr = cast<MCSpecifierExpr>(E);
     E = ARM16Expr->getSubExpr();
 
     if (const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(E)) {
@@ -1201,18 +1201,18 @@ uint32_t ARMMCCodeEmitter::getHiLoImmOpValue(const MCInst &MI, unsigned OpIdx,
         report_fatal_error("constant value truncated (limited to 32-bit)");
 
       switch (ARM16Expr->getSpecifier()) {
-      case ARMMCExpr::VK_HI16:
+      case ARM::S_HI16:
         return (int32_t(Value) & 0xffff0000) >> 16;
-      case ARMMCExpr::VK_LO16:
+      case ARM::S_LO16:
         return (int32_t(Value) & 0x0000ffff);
 
-      case ARMMCExpr::VK_HI_8_15:
+      case ARM::S_HI_8_15:
         return (int32_t(Value) & 0xff000000) >> 24;
-      case ARMMCExpr::VK_HI_0_7:
+      case ARM::S_HI_0_7:
         return (int32_t(Value) & 0x00ff0000) >> 16;
-      case ARMMCExpr::VK_LO_8_15:
+      case ARM::S_LO_8_15:
         return (int32_t(Value) & 0x0000ff00) >> 8;
-      case ARMMCExpr::VK_LO_0_7:
+      case ARM::S_LO_0_7:
         return (int32_t(Value) & 0x000000ff);
 
       default: llvm_unreachable("Unsupported ARMFixup");
@@ -1221,30 +1221,30 @@ uint32_t ARMMCCodeEmitter::getHiLoImmOpValue(const MCInst &MI, unsigned OpIdx,
 
     switch (ARM16Expr->getSpecifier()) {
     default: llvm_unreachable("Unsupported ARMFixup");
-    case ARMMCExpr::VK_HI16:
+    case ARM::S_HI16:
       Kind = MCFixupKind(isThumb(STI) ? ARM::fixup_t2_movt_hi16
                                       : ARM::fixup_arm_movt_hi16);
       break;
-    case ARMMCExpr::VK_LO16:
+    case ARM::S_LO16:
       Kind = MCFixupKind(isThumb(STI) ? ARM::fixup_t2_movw_lo16
                                       : ARM::fixup_arm_movw_lo16);
       break;
-    case ARMMCExpr::VK_HI_8_15:
+    case ARM::S_HI_8_15:
       if (!isThumb(STI))
         llvm_unreachable(":upper_8_15: not supported in Arm state");
       Kind = MCFixupKind(ARM::fixup_arm_thumb_upper_8_15);
       break;
-    case ARMMCExpr::VK_HI_0_7:
+    case ARM::S_HI_0_7:
       if (!isThumb(STI))
         llvm_unreachable(":upper_0_7: not supported in Arm state");
       Kind = MCFixupKind(ARM::fixup_arm_thumb_upper_0_7);
       break;
-    case ARMMCExpr::VK_LO_8_15:
+    case ARM::S_LO_8_15:
       if (!isThumb(STI))
         llvm_unreachable(":lower_8_15: not supported in Arm state");
       Kind = MCFixupKind(ARM::fixup_arm_thumb_lower_8_15);
       break;
-    case ARMMCExpr::VK_LO_0_7:
+    case ARM::S_LO_0_7:
       if (!isThumb(STI))
         llvm_unreachable(":lower_0_7: not supported in Arm state");
       Kind = MCFixupKind(ARM::fixup_arm_thumb_lower_0_7);
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
deleted file mode 100644
index 6230491a7b01..000000000000
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-//===-- ARMMCExpr.cpp - ARM specific MC expression classes ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "ARMMCExpr.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCStreamer.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "armmcexpr"
-
-const ARMMCExpr *ARMMCExpr::create(Specifier S, const MCExpr *Expr,
-                                   MCContext &Ctx) {
-  return new (Ctx) ARMMCExpr(S, Expr);
-}
-
-void ARMMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  switch (specifier) {
-  default: llvm_unreachable("Invalid kind!");
-  case VK_HI16:
-    OS << ":upper16:";
-    break;
-  case VK_LO16:
-    OS << ":lower16:";
-    break;
-  case VK_HI_8_15:
-    OS << ":upper8_15:";
-    break;
-  case VK_HI_0_7:
-    OS << ":upper0_7:";
-    break;
-  case VK_LO_8_15:
-    OS << ":lower8_15:";
-    break;
-  case VK_LO_0_7:
-    OS << ":lower0_7:";
-    break;
-  }
-
-  const MCExpr *Expr = getSubExpr();
-  if (Expr->getKind() != MCExpr::SymbolRef)
-    OS << '(';
-  Expr->print(OS, MAI);
-  if (Expr->getKind() != MCExpr::SymbolRef)
-    OS << ')';
-}
-
-void ARMMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
deleted file mode 100644
index 9b959ce12f64..000000000000
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
+++ /dev/null
@@ -1,126 +0,0 @@
-//===-- ARMMCExpr.h - ARM specific MC expression classes --------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_ARM_MCTARGETDESC_ARMMCEXPR_H
-#define LLVM_LIB_TARGET_ARM_MCTARGETDESC_ARMMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-
-namespace llvm {
-
-class ARMMCExpr : public MCTargetExpr {
-public:
-  enum Specifier {
-    VK_None,
-    VK_HI16 =
-        MCSymbolRefExpr::FirstTargetSpecifier, // The R_ARM_MOVT_ABS relocation
-                                               // (:upper16: in the .s file)
-    VK_LO16, // The R_ARM_MOVW_ABS_NC relocation (:lower16: in the .s file)
-
-    VK_HI_8_15, // The R_ARM_THM_ALU_ABS_G3    relocation (:upper8_15: in
-                // the .s file)
-    VK_HI_0_7,  // The R_ARM_THM_ALU_ABS_G2_NC relocation (:upper0_8: in the
-                // .s file)
-    VK_LO_8_15, // The R_ARM_THM_ALU_ABS_G1_NC relocation (:lower8_15: in
-                // the .s file)
-    VK_LO_0_7,  // The R_ARM_THM_ALU_ABS_G0_NC relocation (:lower0_7: in the
-                // .s file)
-
-    VK_ARM_NONE,
-    VK_FUNCDESC,
-    VK_GOT,
-    VK_GOTFUNCDESC,
-    VK_GOTOFF,
-    VK_GOTOFFFUNCDESC,
-    VK_GOTTPOFF,
-    VK_GOTTPOFF_FDPIC,
-    VK_GOT_PREL,
-    VK_PLT,
-    VK_PREL31,
-    VK_SBREL,
-    VK_TARGET1,
-    VK_TARGET2,
-    VK_TLSCALL,
-    VK_TLSDESC,
-    VK_TLSDESCSEQ,
-    VK_TLSGD,
-    VK_TLSGD_FDPIC,
-    VK_TLSLDM,
-    VK_TLSLDM_FDPIC,
-    VK_TLSLDO,
-    VK_TPOFF,
-  };
-
-private:
-  const Specifier specifier;
-  const MCExpr *Expr;
-
-  explicit ARMMCExpr(Specifier S, const MCExpr *Expr)
-      : specifier(S), Expr(Expr) {}
-
-public:
-  /// @name Construction
-  /// @{
-
-  static const ARMMCExpr *create(Specifier S, const MCExpr *Expr,
-                                 MCContext &Ctx);
-
-  static const ARMMCExpr *createUpper16(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_HI16, Expr, Ctx);
-  }
-
-  static const ARMMCExpr *createLower16(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_LO16, Expr, Ctx);
-  }
-
-  static const ARMMCExpr *createUpper8_15(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_HI_8_15, Expr, Ctx);
-  }
-
-  static const ARMMCExpr *createUpper0_7(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_HI_0_7, Expr, Ctx);
-  }
-
-  static const ARMMCExpr *createLower8_15(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_LO_8_15, Expr, Ctx);
-  }
-
-  static const ARMMCExpr *createLower0_7(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_LO_0_7, Expr, Ctx);
-  }
-
-  /// @}
-  /// @name Accessors
-  /// @{
-
-  Specifier getSpecifier() const { return specifier; }
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  /// @}
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override {
-    return false;
-  }
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-};
-
-static inline ARMMCExpr::Specifier getSpecifier(const MCSymbolRefExpr *SRE) {
-  return ARMMCExpr::Specifier(SRE->getKind());
-}
-} // end namespace llvm
-
-#endif
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
index 886b7e7bc84e..72d9379f5038 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ARMMCExpr.h"
+#include "ARMMCAsmInfo.h"
 #include "MCTargetDesc/ARMMCTargetDesc.h"
 #include "llvm-c/Disassembler.h"
 #include "llvm/MC/MCDisassembler/MCRelocationInfo.h"
@@ -24,9 +24,9 @@ class ARMMachORelocationInfo : public MCRelocationInfo {
                                              unsigned VariantKind) override {
     switch(VariantKind) {
     case LLVMDisassembler_VariantKind_ARM_HI16:
-      return ARMMCExpr::createUpper16(SubExpr, Ctx);
+      return ARM::createUpper16(SubExpr, Ctx);
     case LLVMDisassembler_VariantKind_ARM_LO16:
-      return ARMMCExpr::createLower16(SubExpr, Ctx);
+      return ARM::createLower16(SubExpr, Ctx);
     default:
       return MCRelocationInfo::createExprForCAPIVariantKind(SubExpr,
                                                             VariantKind);
diff --git a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
index 8b3ef0ee651e..977f8bf5548f 100644
--- a/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
@@ -7,7 +7,6 @@ add_llvm_component_library(LLVMARMDesc
   ARMMachORelocationInfo.cpp
   ARMMCAsmInfo.cpp
   ARMMCCodeEmitter.cpp
-  ARMMCExpr.cpp
   ARMMCTargetDesc.cpp
   ARMTargetStreamer.cpp
   ARMUnwindOpAsm.cpp
diff --git a/external/llvm-project/llvm/lib/Target/AVR/AVRDevices.td b/external/llvm-project/llvm/lib/Target/AVR/AVRDevices.td
index 56147bb473bc..ad760d740357 100644
--- a/external/llvm-project/llvm/lib/Target/AVR/AVRDevices.td
+++ b/external/llvm-project/llvm/lib/Target/AVR/AVRDevices.td
@@ -209,12 +209,27 @@ def FamilyTiny
              [FamilyAVR0, FeatureBREAK, FeatureSRAM, FeatureTinyEncoding,
               FeatureSmallStack]>;
 
+def FamilyXMEGA2 : Family<"xmega2",
+                          [FamilyAVR0, FeatureLPM, FeatureIJMPCALL,
+                           FeatureADDSUBIW, FeatureSRAM, FeatureJMPCALL,
+                           FeatureMultiplication, FeatureMOVW, FeatureLPMX,
+                           FeatureSPM, FeatureSPMX,
+                           FeatureBREAK, FeatureLowByteFirst]>;
+
 def FamilyXMEGA3 : Family<"xmega3",
                           [FamilyAVR0, FeatureLPM, FeatureIJMPCALL,
                            FeatureADDSUBIW, FeatureSRAM, FeatureJMPCALL,
                            FeatureMultiplication, FeatureMOVW, FeatureLPMX,
                            FeatureBREAK, FeatureLowByteFirst]>;
 
+def FamilyXMEGA4 : Family<"xmega4",
+                          [FamilyAVR0, FeatureLPM, FeatureIJMPCALL,
+                           FeatureADDSUBIW, FeatureSRAM, FeatureJMPCALL,
+                           FeatureMultiplication, FeatureMOVW, FeatureLPMX,
+                           FeatureELPM, FeatureELPMX,
+                           FeatureSPM, FeatureSPMX,
+                           FeatureBREAK, FeatureLowByteFirst]>;
+
 def FamilyXMEGA : Family<"xmega",
                          [FamilyAVR0, FeatureLPM, FeatureIJMPCALL,
                           FeatureADDSUBIW, FeatureSRAM, FeatureJMPCALL,
@@ -567,6 +582,9 @@ def : Device<"attiny3217", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"attiny1624", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"attiny1626", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"attiny1627", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"attiny3224", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"attiny3226", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"attiny3227", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"atmega808", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"atmega809", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"atmega1608", FamilyXMEGA3, ELFArchXMEGA3>;
@@ -575,3 +593,69 @@ def : Device<"atmega3208", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"atmega3209", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"atmega4808", FamilyXMEGA3, ELFArchXMEGA3>;
 def : Device<"atmega4809", FamilyXMEGA3, ELFArchXMEGA3>;
+
+// Additions from gcc 14:
+
+def : Device<"avr64da28", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64da32", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64da48", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64da64", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64db28", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64db32", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64db48", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64db64", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64dd14", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64dd20", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64dd28", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64dd32", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64du28", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64du32", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64ea28", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64ea32", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64ea48", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64sd28", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64sd32", FamilyXMEGA2, ELFArchXMEGA2>;
+def : Device<"avr64sd48", FamilyXMEGA2, ELFArchXMEGA2>;
+
+def : Device<"avr16dd20", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16dd28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16dd32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16du14", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16du20", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16du28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16du32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32da28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32da32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32da48", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32db28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32db32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32db48", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32dd14", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32dd20", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32dd28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32dd32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32du14", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32du20", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32du28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32du32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16eb14", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16eb20", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16eb28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16eb32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16ea28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16ea32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr16ea48", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32ea28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32ea32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32ea48", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32sd20", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32sd28", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr32sd32", FamilyXMEGA3, ELFArchXMEGA3>;
+def : Device<"avr128da28", FamilyXMEGA4, ELFArchXMEGA4>;
+def : Device<"avr128da32", FamilyXMEGA4, ELFArchXMEGA4>;
+def : Device<"avr128da48", FamilyXMEGA4, ELFArchXMEGA4>;
+def : Device<"avr128da64", FamilyXMEGA4, ELFArchXMEGA4>;
+def : Device<"avr128db28", FamilyXMEGA4, ELFArchXMEGA4>;
+def : Device<"avr128db32", FamilyXMEGA4, ELFArchXMEGA4>;
+def : Device<"avr128db48", FamilyXMEGA4, ELFArchXMEGA4>;
+def : Device<"avr128db64", FamilyXMEGA4, ELFArchXMEGA4>;
diff --git a/external/llvm-project/llvm/lib/Target/AVR/AVRISelLowering.cpp b/external/llvm-project/llvm/lib/Target/AVR/AVRISelLowering.cpp
index eca682895e24..9747ad0c5cd5 100644
--- a/external/llvm-project/llvm/lib/Target/AVR/AVRISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/AVR/AVRISelLowering.cpp
@@ -198,24 +198,6 @@ AVRTargetLowering::AVRTargetLowering(const AVRTargetMachine &TM,
     // improvements in how we treat 16-bit "registers" to be feasible.
   }
 
-  // Division and modulus rtlib functions
-  setLibcallName(RTLIB::SDIVREM_I8, "__divmodqi4");
-  setLibcallName(RTLIB::SDIVREM_I16, "__divmodhi4");
-  setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
-  setLibcallName(RTLIB::UDIVREM_I8, "__udivmodqi4");
-  setLibcallName(RTLIB::UDIVREM_I16, "__udivmodhi4");
-  setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
-
-  // Several of the runtime library functions use a special calling conv
-  setLibcallCallingConv(RTLIB::SDIVREM_I8, CallingConv::AVR_BUILTIN);
-  setLibcallCallingConv(RTLIB::SDIVREM_I16, CallingConv::AVR_BUILTIN);
-  setLibcallCallingConv(RTLIB::UDIVREM_I8, CallingConv::AVR_BUILTIN);
-  setLibcallCallingConv(RTLIB::UDIVREM_I16, CallingConv::AVR_BUILTIN);
-
-  // Trigonometric rtlib functions
-  setLibcallName(RTLIB::SIN_F32, "sin");
-  setLibcallName(RTLIB::COS_F32, "cos");
-
   setMinFunctionAlignment(Align(2));
   setMinimumJumpTableEntries(UINT_MAX);
 }
diff --git a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
index bf512dd07c0a..1e4b2e27a183 100644
--- a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
@@ -14,6 +14,7 @@
 
 #include "MCTargetDesc/AVRMCTargetDesc.h"
 
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrDesc.h"
@@ -133,7 +134,7 @@ void AVRInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << formatImm(Op.getImm());
   } else {
     assert(Op.isExpr() && "Unknown operand kind in printOperand");
-    O << *Op.getExpr();
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
index 98ddacb31a34..fa01dad5ec12 100644
--- a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
@@ -230,7 +230,7 @@ unsigned AVRMCCodeEmitter::getExprOpValue(const MCExpr *Expr,
     Kind = Expr->getKind();
   }
 
-  if (Kind == MCExpr::Target) {
+  if (Kind == MCExpr::Specifier) {
     AVRMCExpr const *AVRExpr = cast<AVRMCExpr>(Expr);
     int64_t Result;
     if (AVRExpr->evaluateAsConstant(Result)) {
diff --git a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp
index 01bd8973170c..5be799093d2c 100644
--- a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp
@@ -8,6 +8,7 @@
 
 #include "AVRMCExpr.h"
 
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
@@ -44,7 +45,7 @@ void AVRMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   OS << getName() << '(';
   if (isNegated())
     OS << '-' << '(';
-  getSubExpr()->print(OS, MAI);
+  MAI->printExpr(OS, *getSubExpr());
   if (isNegated())
     OS << ')';
   OS << ')';
@@ -69,8 +70,7 @@ bool AVRMCExpr::evaluateAsConstant(int64_t &Result) const {
 bool AVRMCExpr::evaluateAsRelocatableImpl(MCValue &Result,
                                           const MCAssembler *Asm) const {
   MCValue Value;
-  bool isRelocatable = SubExpr->evaluateAsRelocatable(Value, Asm);
-
+  bool isRelocatable = getSubExpr()->evaluateAsRelocatable(Value, Asm);
   if (!isRelocatable)
     return false;
 
@@ -187,10 +187,6 @@ AVR::Fixups AVRMCExpr::getFixupKind() const {
   return Kind;
 }
 
-void AVRMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
 const char *AVRMCExpr::getName() const {
   const auto &Modifier =
       llvm::find_if(ModifierNames, [this](ModifierEntry const &Mod) {
diff --git a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h
index e3c36d20f851..69c60cde1f74 100644
--- a/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h
@@ -16,10 +16,11 @@
 namespace llvm {
 
 /// A expression in AVR machine code.
-class AVRMCExpr : public MCTargetExpr {
+class AVRMCExpr : public MCSpecifierExpr {
 public:
+  using Specifier = Spec;
   /// Specifies the type of an expression.
-  enum Specifier {
+  enum {
     VK_None,
 
     VK_AVR_NONE = MCSymbolRefExpr::FirstTargetSpecifier,
@@ -48,11 +49,8 @@ class AVRMCExpr : public MCTargetExpr {
   static const AVRMCExpr *create(Specifier S, const MCExpr *Expr,
                                  bool isNegated, MCContext &Ctx);
 
-  /// Gets the type of the expression.
-  Specifier getSpecifier() const { return specifier; }
   /// Gets the name of the expression.
   const char *getName() const;
-  const MCExpr *getSubExpr() const { return SubExpr; }
   /// Gets the fixup which corresponds to the expression.
   AVR::Fixups getFixupKind() const;
   /// Evaluates the fixup as a constant value.
@@ -64,15 +62,6 @@ class AVRMCExpr : public MCTargetExpr {
   void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
   bool evaluateAsRelocatableImpl(MCValue &Res,
                                  const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &streamer) const override;
-
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
 
 public:
   static Specifier parseSpecifier(StringRef Name);
@@ -80,14 +69,11 @@ class AVRMCExpr : public MCTargetExpr {
 private:
   int64_t evaluateAsInt64(int64_t Value) const;
 
-  const Specifier specifier;
-  const MCExpr *SubExpr;
   bool Negated;
 
 private:
   explicit AVRMCExpr(Specifier S, const MCExpr *Expr, bool Negated)
-      : specifier(S), SubExpr(Expr), Negated(Negated) {}
-  ~AVRMCExpr() = default;
+      : MCSpecifierExpr(Expr, S), Negated(Negated) {}
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/external/llvm-project/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
index d29ef24174b6..8c7bc2f0f671 100644
--- a/external/llvm-project/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
+++ b/external/llvm-project/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
@@ -414,9 +414,7 @@ static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex,
 
   Constant *Zero =
       ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0);
-  SmallVector<Value *, 4> IdxList;
-  for (unsigned I = 0; I < Dimension; ++I)
-    IdxList.push_back(Zero);
+  SmallVector<Value *, 4> IdxList(Dimension, Zero);
   IdxList.push_back(Call->getArgOperand(GEPIndex));
 
   auto *GEP = GetElementPtrInst::CreateInBounds(getBaseElementType(Call),
diff --git a/external/llvm-project/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp b/external/llvm-project/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
index 77bbeab3c279..222eb19e3eee 100644
--- a/external/llvm-project/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
+++ b/external/llvm-project/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
@@ -150,10 +150,10 @@ static CallInst *isGEPAndStore(Value *I) {
 }
 
 template <class T = Instruction>
-static DILocation *mergeDILocations(SmallVector<T *> &Insns) {
-  DILocation *Merged = (*Insns.begin())->getDebugLoc();
+static DebugLoc mergeDebugLocs(SmallVector<T *> &Insns) {
+  DebugLoc Merged = (*Insns.begin())->getDebugLoc();
   for (T *I : Insns)
-    Merged = DILocation::getMergedLocation(Merged, I->getDebugLoc());
+    Merged = DebugLoc::getMergedLocation(Merged, I->getDebugLoc());
   return Merged;
 }
 
@@ -227,7 +227,7 @@ static Instruction *makeGEPAndLoad(Module *M, GEPChainInfo &GEP,
   CallInst *Call = makeIntrinsicCall(M, Intrinsic::bpf_getelementptr_and_load,
                                      {Load->getType()}, Args);
   setParamElementType(Call, 0, GEP.SourceElementType);
-  Call->applyMergedLocation(mergeDILocations(GEP.Members), Load->getDebugLoc());
+  Call->applyMergedLocation(mergeDebugLocs(GEP.Members), Load->getDebugLoc());
   Call->setName((*GEP.Members.rbegin())->getName());
   if (Load->isUnordered()) {
     Call->setOnlyReadsMemory();
@@ -251,8 +251,7 @@ static Instruction *makeGEPAndStore(Module *M, GEPChainInfo &GEP,
   setParamElementType(Call, 1, GEP.SourceElementType);
   if (Store->getValueOperand()->getType()->isPointerTy())
     setParamReadNone(Call, 0);
-  Call->applyMergedLocation(mergeDILocations(GEP.Members),
-                            Store->getDebugLoc());
+  Call->applyMergedLocation(mergeDebugLocs(GEP.Members), Store->getDebugLoc());
   if (Store->isUnordered()) {
     Call->setOnlyWritesMemory();
     Call->setOnlyAccessesArgMemory();
diff --git a/external/llvm-project/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index e94497896f68..d7b2ceff105c 100644
--- a/external/llvm-project/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -21,7 +21,7 @@
 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 
 namespace llvm {
-class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
+class BPFTTIImpl final : public BasicTTIImplBase<BPFTTIImpl> {
   typedef BasicTTIImplBase<BPFTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
diff --git a/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.cpp b/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.cpp
index 978cd9ab2106..1d7c6fc799ff 100644
--- a/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -1438,7 +1438,7 @@ void BTFDebug::processGlobals(bool ProcessingMapDef) {
     // constant with private linkage and if it won't be in .rodata.str<#>
     // and .rodata.cst<#> sections.
     if (SecName == ".rodata" && Global.hasPrivateLinkage() &&
-        DataSecEntries.find(std::string(SecName)) == DataSecEntries.end()) {
+        DataSecEntries.find(SecName) == DataSecEntries.end()) {
       // skip .rodata.str<#> and .rodata.cst<#> sections
       if (!GVKind->isMergeableCString() && !GVKind->isMergeableConst()) {
         DataSecEntries[std::string(SecName)] =
diff --git a/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.h b/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.h
index b24a79de74ef..75858fcc8bfd 100644
--- a/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.h
+++ b/external/llvm-project/llvm/lib/Target/BPF/BTFDebug.h
@@ -300,7 +300,8 @@ class BTFDebug : public DebugHandlerBase {
   std::map<uint32_t, std::vector<BTFLineInfo>> LineInfoTable;
   std::map<uint32_t, std::vector<BTFFieldReloc>> FieldRelocTable;
   StringMap<std::vector<std::string>> FileContent;
-  std::map<std::string, std::unique_ptr<BTFKindDataSec>> DataSecEntries;
+  std::map<std::string, std::unique_ptr<BTFKindDataSec>, std::less<>>
+      DataSecEntries;
   std::vector<BTFTypeStruct *> StructTypes;
   std::map<const GlobalVariable *, std::pair<int64_t, uint32_t>> PatchImms;
   std::map<const DICompositeType *,
diff --git a/external/llvm-project/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index 10a46f100bbe..bd9d2de58c8b 100644
--- a/external/llvm-project/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -14,6 +14,7 @@
 #include "MCTargetDesc/BPFMCTargetDesc.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
@@ -33,11 +34,12 @@ namespace {
 class BPFMCCodeEmitter : public MCCodeEmitter {
   const MCRegisterInfo &MRI;
   bool IsLittleEndian;
+  MCContext &Ctx;
 
 public:
   BPFMCCodeEmitter(const MCInstrInfo &, const MCRegisterInfo &mri,
-                   bool IsLittleEndian)
-      : MRI(mri), IsLittleEndian(IsLittleEndian) { }
+                   bool IsLittleEndian, MCContext &ctx)
+      : MRI(mri), IsLittleEndian(IsLittleEndian), Ctx(ctx) {}
   BPFMCCodeEmitter(const BPFMCCodeEmitter &) = delete;
   void operator=(const BPFMCCodeEmitter &) = delete;
   ~BPFMCCodeEmitter() override = default;
@@ -67,12 +69,12 @@ class BPFMCCodeEmitter : public MCCodeEmitter {
 
 MCCodeEmitter *llvm::createBPFMCCodeEmitter(const MCInstrInfo &MCII,
                                             MCContext &Ctx) {
-  return new BPFMCCodeEmitter(MCII, *Ctx.getRegisterInfo(), true);
+  return new BPFMCCodeEmitter(MCII, *Ctx.getRegisterInfo(), true, Ctx);
 }
 
 MCCodeEmitter *llvm::createBPFbeMCCodeEmitter(const MCInstrInfo &MCII,
                                               MCContext &Ctx) {
-  return new BPFMCCodeEmitter(MCII, *Ctx.getRegisterInfo(), false);
+  return new BPFMCCodeEmitter(MCII, *Ctx.getRegisterInfo(), false, Ctx);
 }
 
 unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
@@ -81,8 +83,16 @@ unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
                                              const MCSubtargetInfo &STI) const {
   if (MO.isReg())
     return MRI.getEncodingValue(MO.getReg());
-  if (MO.isImm())
-    return static_cast<unsigned>(MO.getImm());
+  if (MO.isImm()) {
+    uint64_t Imm = MO.getImm();
+    uint64_t High32Bits = Imm >> 32, High33Bits = Imm >> 31;
+    if (MI.getOpcode() != BPF::LD_imm64 && High32Bits != 0 &&
+        High33Bits != 0x1FFFFFFFFULL) {
+      Ctx.reportWarning(MI.getLoc(),
+                        "immediate out of range, shall fit in 32 bits");
+    }
+    return static_cast<unsigned>(Imm);
+  }
 
   assert(MO.isExpr());
 
diff --git a/external/llvm-project/llvm/lib/Target/CSKY/CSKYInstrAlias.td b/external/llvm-project/llvm/lib/Target/CSKY/CSKYInstrAlias.td
index e3c0538e752e..bc63ce30d432 100644
--- a/external/llvm-project/llvm/lib/Target/CSKY/CSKYInstrAlias.td
+++ b/external/llvm-project/llvm/lib/Target/CSKY/CSKYInstrAlias.td
@@ -35,4 +35,4 @@ def : InstAlias<"jsri $dst", (PseudoJSRI32 call_symbol:$dst)>;
 def : InstAlias<"jsri $dst", (JSRI32 constpool_symbol:$dst)>;
 
 def : InstAlias<"jmpi $dst", (PseudoJMPI32 br_symbol:$dst)>;
-def : InstAlias<"jmpi $dst", (JMPI32 constpool_symbol:$dst)>;
\ No newline at end of file
+def : InstAlias<"jmpi $dst", (JMPI32 constpool_symbol:$dst)>;
diff --git a/external/llvm-project/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp b/external/llvm-project/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp
index a3163a896964..b1f3f41a28e8 100644
--- a/external/llvm-project/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp
+++ b/external/llvm-project/llvm/lib/Target/DirectX/DXILFlattenArrays.cpp
@@ -42,7 +42,7 @@ class DXILFlattenArraysLegacy : public ModulePass {
 
 struct GEPData {
   ArrayType *ParentArrayType;
-  Value *ParendOperand;
+  Value *ParentOperand;
   SmallVector<Value *> Indices;
   SmallVector<uint64_t> Dims;
   bool AllIndicesAreConstInt;
@@ -211,7 +211,7 @@ bool DXILFlattenArraysVisitor::visitAllocaInst(AllocaInst &AI) {
 
   ArrayType *FattenedArrayType = ArrayType::get(BaseType, TotalElements);
   AllocaInst *FlatAlloca =
-      Builder.CreateAlloca(FattenedArrayType, nullptr, AI.getName() + ".flat");
+      Builder.CreateAlloca(FattenedArrayType, nullptr, AI.getName() + ".1dim");
   FlatAlloca->setAlignment(AI.getAlign());
   AI.replaceAllUsesWith(FlatAlloca);
   AI.eraseFromParent();
@@ -222,6 +222,10 @@ void DXILFlattenArraysVisitor::recursivelyCollectGEPs(
     GetElementPtrInst &CurrGEP, ArrayType *FlattenedArrayType,
     Value *PtrOperand, unsigned &GEPChainUseCount, SmallVector<Value *> Indices,
     SmallVector<uint64_t> Dims, bool AllIndicesAreConstInt) {
+  // Check if this GEP is already in the map to avoid circular references
+  if (GEPChainMap.count(&CurrGEP) > 0)
+    return;
+
   Value *LastIndex = CurrGEP.getOperand(CurrGEP.getNumOperands() - 1);
   AllIndicesAreConstInt &= isa<ConstantInt>(LastIndex);
   Indices.push_back(LastIndex);
@@ -271,9 +275,19 @@ bool DXILFlattenArraysVisitor::visitGetElementPtrInstInGEPChainBase(
         genInstructionFlattenIndices(GEPInfo.Indices, GEPInfo.Dims, Builder);
 
   ArrayType *FlattenedArrayType = GEPInfo.ParentArrayType;
-  Value *FlatGEP =
-      Builder.CreateGEP(FlattenedArrayType, GEPInfo.ParendOperand, FlatIndex,
-                        GEP.getName() + ".flat", GEP.isInBounds());
+
+  // Don't append '.flat' to an empty string. If the SSA name isn't available
+  // it could conflict with the ParentOperand's name.
+  std::string FlatName = GEP.hasName() ? GEP.getName().str() + ".flat" : "";
+
+  Value *FlatGEP = Builder.CreateGEP(FlattenedArrayType, GEPInfo.ParentOperand,
+                                     {Builder.getInt32(0), FlatIndex}, FlatName,
+                                     GEP.getNoWrapFlags());
+
+  // Note: Old gep will become an invalid instruction after replaceAllUsesWith.
+  // Erase the old GEP in the map before to avoid invalid instructions
+  // and circular references.
+  GEPChainMap.erase(&GEP);
 
   GEP.replaceAllUsesWith(FlatGEP);
   GEP.eraseFromParent();
diff --git a/external/llvm-project/llvm/lib/Target/DirectX/DXILLegalizePass.cpp b/external/llvm-project/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
index 23883c936a20..76a46c7a2b76 100644
--- a/external/llvm-project/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
+++ b/external/llvm-project/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
@@ -8,6 +8,8 @@
 
 #include "DXILLegalizePass.h"
 #include "DirectX.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
@@ -87,20 +89,63 @@ static void fixI8UseChain(Instruction &I,
     return;
   }
 
-  if (auto *Load = dyn_cast<LoadInst>(&I)) {
-    if (!I.getType()->isIntegerTy(8))
-      return;
+  if (auto *Load = dyn_cast<LoadInst>(&I);
+      Load && I.getType()->isIntegerTy(8)) {
     SmallVector<Value *> NewOperands;
     ProcessOperands(NewOperands);
     Type *ElementType = NewOperands[0]->getType();
     if (auto *AI = dyn_cast<AllocaInst>(NewOperands[0]))
       ElementType = AI->getAllocatedType();
+    if (auto *GEP = dyn_cast<GetElementPtrInst>(NewOperands[0])) {
+      ElementType = GEP->getSourceElementType();
+      if (ElementType->isArrayTy())
+        ElementType = ElementType->getArrayElementType();
+    }
     LoadInst *NewLoad = Builder.CreateLoad(ElementType, NewOperands[0]);
     ReplacedValues[Load] = NewLoad;
     ToRemove.push_back(Load);
     return;
   }
 
+  if (auto *Load = dyn_cast<LoadInst>(&I);
+      Load && isa<ConstantExpr>(Load->getPointerOperand())) {
+    auto *CE = dyn_cast<ConstantExpr>(Load->getPointerOperand());
+    if (!(CE->getOpcode() == Instruction::GetElementPtr))
+      return;
+    auto *GEP = dyn_cast<GEPOperator>(CE);
+    if (!GEP->getSourceElementType()->isIntegerTy(8))
+      return;
+
+    Type *ElementType = Load->getType();
+    ConstantInt *Offset = dyn_cast<ConstantInt>(GEP->getOperand(1));
+    uint32_t ByteOffset = Offset->getZExtValue();
+    uint32_t ElemSize = Load->getDataLayout().getTypeAllocSize(ElementType);
+    uint32_t Index = ByteOffset / ElemSize;
+
+    Value *PtrOperand = GEP->getPointerOperand();
+    Type *GEPType = GEP->getPointerOperandType();
+
+    if (auto *GV = dyn_cast<GlobalVariable>(PtrOperand))
+      GEPType = GV->getValueType();
+    if (auto *AI = dyn_cast<AllocaInst>(PtrOperand))
+      GEPType = AI->getAllocatedType();
+
+    if (auto *ArrTy = dyn_cast<ArrayType>(GEPType))
+      GEPType = ArrTy;
+    else
+      GEPType = ArrayType::get(ElementType, 1); // its a scalar
+
+    Value *NewGEP = Builder.CreateGEP(
+        GEPType, PtrOperand, {Builder.getInt32(0), Builder.getInt32(Index)},
+        GEP->getName(), GEP->getNoWrapFlags());
+
+    LoadInst *NewLoad = Builder.CreateLoad(ElementType, NewGEP);
+    ReplacedValues[Load] = NewLoad;
+    Load->replaceAllUsesWith(NewLoad);
+    ToRemove.push_back(Load);
+    return;
+  }
+
   if (auto *BO = dyn_cast<BinaryOperator>(&I)) {
     if (!I.getType()->isIntegerTy(8))
       return;
@@ -155,6 +200,7 @@ static void fixI8UseChain(Instruction &I,
       Cast->replaceAllUsesWith(Replacement);
       return;
     }
+
     Value *AdjustedCast = nullptr;
     if (Cast->getOpcode() == Instruction::ZExt)
       AdjustedCast = Builder.CreateZExtOrTrunc(Replacement, Cast->getType());
@@ -164,6 +210,45 @@ static void fixI8UseChain(Instruction &I,
     if (AdjustedCast)
       Cast->replaceAllUsesWith(AdjustedCast);
   }
+  if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
+    if (!GEP->getType()->isPointerTy() ||
+        !GEP->getSourceElementType()->isIntegerTy(8))
+      return;
+
+    Value *BasePtr = GEP->getPointerOperand();
+    if (ReplacedValues.count(BasePtr))
+      BasePtr = ReplacedValues[BasePtr];
+
+    Type *ElementType = BasePtr->getType();
+
+    if (auto *AI = dyn_cast<AllocaInst>(BasePtr))
+      ElementType = AI->getAllocatedType();
+    if (auto *GV = dyn_cast<GlobalVariable>(BasePtr))
+      ElementType = GV->getValueType();
+
+    Type *GEPType = ElementType;
+    if (auto *ArrTy = dyn_cast<ArrayType>(ElementType))
+      ElementType = ArrTy->getArrayElementType();
+    else
+      GEPType = ArrayType::get(ElementType, 1); // its a scalar
+
+    ConstantInt *Offset = dyn_cast<ConstantInt>(GEP->getOperand(1));
+    // Note: i8 to i32 offset conversion without emitting IR requires constant
+    // ints. Since offset conversion is common, we can safely assume Offset is
+    // always a ConstantInt, so no need to have a conditional bail out on
+    // nullptr, instead assert this is the case.
+    assert(Offset && "Offset is expected to be a ConstantInt");
+    uint32_t ByteOffset = Offset->getZExtValue();
+    uint32_t ElemSize = GEP->getDataLayout().getTypeAllocSize(ElementType);
+    assert(ElemSize > 0 && "ElementSize must be set");
+    uint32_t Index = ByteOffset / ElemSize;
+    Value *NewGEP = Builder.CreateGEP(
+        GEPType, BasePtr, {Builder.getInt32(0), Builder.getInt32(Index)},
+        GEP->getName(), GEP->getNoWrapFlags());
+    ReplacedValues[GEP] = NewGEP;
+    GEP->replaceAllUsesWith(NewGEP);
+    ToRemove.push_back(GEP);
+  }
 }
 
 static void upcastI8AllocasAndUses(Instruction &I,
@@ -175,15 +260,12 @@ static void upcastI8AllocasAndUses(Instruction &I,
 
   Type *SmallestType = nullptr;
 
-  for (User *U : AI->users()) {
-    auto *Load = dyn_cast<LoadInst>(U);
-    if (!Load)
-      continue;
+  auto ProcessLoad = [&](LoadInst *Load) {
     for (User *LU : Load->users()) {
       Type *Ty = nullptr;
-      if (auto *Cast = dyn_cast<CastInst>(LU))
+      if (CastInst *Cast = dyn_cast<CastInst>(LU))
         Ty = Cast->getType();
-      if (CallInst *CI = dyn_cast<CallInst>(LU)) {
+      else if (CallInst *CI = dyn_cast<CallInst>(LU)) {
         if (CI->getIntrinsicID() == Intrinsic::memset)
           Ty = Type::getInt32Ty(CI->getContext());
       }
@@ -195,6 +277,17 @@ static void upcastI8AllocasAndUses(Instruction &I,
           Ty->getPrimitiveSizeInBits() < SmallestType->getPrimitiveSizeInBits())
         SmallestType = Ty;
     }
+  };
+
+  for (User *U : AI->users()) {
+    if (auto *Load = dyn_cast<LoadInst>(U))
+      ProcessLoad(Load);
+    else if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {
+      for (User *GU : GEP->users()) {
+        if (auto *Load = dyn_cast<LoadInst>(GU))
+          ProcessLoad(Load);
+      }
+    }
   }
 
   if (!SmallestType)
@@ -377,7 +470,8 @@ static void legalizeMemCpy(Instruction &I,
   Value *Src = CI->getArgOperand(1);
   ConstantInt *Length = dyn_cast<ConstantInt>(CI->getArgOperand(2));
   assert(Length && "Expected Length to be a ConstantInt");
-  ConstantInt *IsVolatile = dyn_cast<ConstantInt>(CI->getArgOperand(3));
+  [[maybe_unused]] ConstantInt *IsVolatile =
+      dyn_cast<ConstantInt>(CI->getArgOperand(3));
   assert(IsVolatile && "Expected IsVolatile to be a ConstantInt");
   assert(IsVolatile->getZExtValue() == 0 && "Expected IsVolatile to be false");
   emitMemcpyExpansion(Builder, Dst, Src, Length);
@@ -419,6 +513,55 @@ static void updateFnegToFsub(Instruction &I,
   ToRemove.push_back(&I);
 }
 
+static void
+legalizeGetHighLowi64Bytes(Instruction &I,
+                           SmallVectorImpl<Instruction *> &ToRemove,
+                           DenseMap<Value *, Value *> &ReplacedValues) {
+  if (auto *BitCast = dyn_cast<BitCastInst>(&I)) {
+    if (BitCast->getDestTy() ==
+            FixedVectorType::get(Type::getInt32Ty(I.getContext()), 2) &&
+        BitCast->getSrcTy()->isIntegerTy(64)) {
+      ToRemove.push_back(BitCast);
+      ReplacedValues[BitCast] = BitCast->getOperand(0);
+      return;
+    }
+  }
+
+  if (auto *Extract = dyn_cast<ExtractElementInst>(&I)) {
+    if (!dyn_cast<BitCastInst>(Extract->getVectorOperand()))
+      return;
+    auto *VecTy = dyn_cast<FixedVectorType>(Extract->getVectorOperandType());
+    if (VecTy && VecTy->getElementType()->isIntegerTy(32) &&
+        VecTy->getNumElements() == 2) {
+      if (auto *Index = dyn_cast<ConstantInt>(Extract->getIndexOperand())) {
+        unsigned Idx = Index->getZExtValue();
+        IRBuilder<> Builder(&I);
+
+        auto *Replacement = ReplacedValues[Extract->getVectorOperand()];
+        assert(Replacement && "The BitCast replacement should have been set "
+                              "before working on ExtractElementInst.");
+        if (Idx == 0) {
+          Value *LowBytes = Builder.CreateTrunc(
+              Replacement, Type::getInt32Ty(I.getContext()));
+          ReplacedValues[Extract] = LowBytes;
+        } else {
+          assert(Idx == 1);
+          Value *LogicalShiftRight = Builder.CreateLShr(
+              Replacement,
+              ConstantInt::get(
+                  Replacement->getType(),
+                  APInt(Replacement->getType()->getIntegerBitWidth(), 32)));
+          Value *HighBytes = Builder.CreateTrunc(
+              LogicalShiftRight, Type::getInt32Ty(I.getContext()));
+          ReplacedValues[Extract] = HighBytes;
+        }
+        ToRemove.push_back(Extract);
+        Extract->replaceAllUsesWith(ReplacedValues[Extract]);
+      }
+    }
+  }
+}
+
 namespace {
 class DXILLegalizationPipeline {
 
@@ -426,33 +569,49 @@ class DXILLegalizationPipeline {
   DXILLegalizationPipeline() { initializeLegalizationPipeline(); }
 
   bool runLegalizationPipeline(Function &F) {
+    bool MadeChange = false;
     SmallVector<Instruction *> ToRemove;
     DenseMap<Value *, Value *> ReplacedValues;
-    for (auto &I : instructions(F)) {
-      for (auto &LegalizationFn : LegalizationPipeline)
-        LegalizationFn(I, ToRemove, ReplacedValues);
-    }
+    for (int Stage = 0; Stage < NumStages; ++Stage) {
+      ToRemove.clear();
+      ReplacedValues.clear();
+      for (auto &I : instructions(F)) {
+        for (auto &LegalizationFn : LegalizationPipeline[Stage])
+          LegalizationFn(I, ToRemove, ReplacedValues);
+      }
 
-    for (auto *Inst : reverse(ToRemove))
-      Inst->eraseFromParent();
+      for (auto *Inst : reverse(ToRemove))
+        Inst->eraseFromParent();
 
-    return !ToRemove.empty();
+      MadeChange |= !ToRemove.empty();
+    }
+    return MadeChange;
   }
 
 private:
-  SmallVector<
+  enum LegalizationStage { Stage1 = 0, Stage2 = 1, NumStages };
+
+  using LegalizationFnTy =
       std::function<void(Instruction &, SmallVectorImpl<Instruction *> &,
-                         DenseMap<Value *, Value *> &)>>
-      LegalizationPipeline;
+                         DenseMap<Value *, Value *> &)>;
+
+  SmallVector<LegalizationFnTy> LegalizationPipeline[NumStages];
 
   void initializeLegalizationPipeline() {
-    LegalizationPipeline.push_back(upcastI8AllocasAndUses);
-    LegalizationPipeline.push_back(fixI8UseChain);
-    LegalizationPipeline.push_back(downcastI64toI32InsertExtractElements);
-    LegalizationPipeline.push_back(legalizeFreeze);
-    LegalizationPipeline.push_back(legalizeMemCpy);
-    LegalizationPipeline.push_back(removeMemSet);
-    LegalizationPipeline.push_back(updateFnegToFsub);
+    LegalizationPipeline[Stage1].push_back(upcastI8AllocasAndUses);
+    LegalizationPipeline[Stage1].push_back(fixI8UseChain);
+    LegalizationPipeline[Stage1].push_back(legalizeGetHighLowi64Bytes);
+    LegalizationPipeline[Stage1].push_back(legalizeFreeze);
+    LegalizationPipeline[Stage1].push_back(legalizeMemCpy);
+    LegalizationPipeline[Stage1].push_back(removeMemSet);
+    LegalizationPipeline[Stage1].push_back(updateFnegToFsub);
+    // Note: legalizeGetHighLowi64Bytes and
+    // downcastI64toI32InsertExtractElements both modify extractelement, so they
+    // must run staggered stages. legalizeGetHighLowi64Bytes runs first b\c it
+    // removes extractelements, reducing the number that
+    // downcastI64toI32InsertExtractElements needs to handle.
+    LegalizationPipeline[Stage2].push_back(
+        downcastI64toI32InsertExtractElements);
   }
 };
 
diff --git a/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.cpp
index 40277899b5c7..3d195acb19e1 100644
--- a/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.cpp
+++ b/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.cpp
@@ -55,6 +55,14 @@ static std::optional<uint32_t> extractMdIntValue(MDNode *Node,
   return std::nullopt;
 }
 
+static std::optional<StringRef> extractMdStringValue(MDNode *Node,
+                                                     unsigned int OpId) {
+  MDString *NodeText = dyn_cast<MDString>(Node->getOperand(OpId));
+  if (NodeText == nullptr)
+    return std::nullopt;
+  return NodeText->getString();
+}
+
 static bool parseRootFlags(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD,
                            MDNode *RootFlagNode) {
 
@@ -107,17 +115,79 @@ static bool parseRootConstants(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD,
   return false;
 }
 
+static bool parseRootDescriptors(LLVMContext *Ctx,
+                                 mcdxbc::RootSignatureDesc &RSD,
+                                 MDNode *RootDescriptorNode,
+                                 RootSignatureElementKind ElementKind) {
+  assert(ElementKind == RootSignatureElementKind::SRV ||
+         ElementKind == RootSignatureElementKind::UAV ||
+         ElementKind == RootSignatureElementKind::CBV &&
+             "parseRootDescriptors should only be called with RootDescriptor "
+             "element kind.");
+  if (RootDescriptorNode->getNumOperands() != 5)
+    return reportError(Ctx, "Invalid format for Root Descriptor Element");
+
+  dxbc::RTS0::v1::RootParameterHeader Header;
+  switch (ElementKind) {
+  case RootSignatureElementKind::SRV:
+    Header.ParameterType = llvm::to_underlying(dxbc::RootParameterType::SRV);
+    break;
+  case RootSignatureElementKind::UAV:
+    Header.ParameterType = llvm::to_underlying(dxbc::RootParameterType::UAV);
+    break;
+  case RootSignatureElementKind::CBV:
+    Header.ParameterType = llvm::to_underlying(dxbc::RootParameterType::CBV);
+    break;
+  default:
+    llvm_unreachable("invalid Root Descriptor kind");
+    break;
+  }
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RootDescriptorNode, 1))
+    Header.ShaderVisibility = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ShaderVisibility");
+
+  dxbc::RTS0::v2::RootDescriptor Descriptor;
+  if (std::optional<uint32_t> Val = extractMdIntValue(RootDescriptorNode, 2))
+    Descriptor.ShaderRegister = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ShaderRegister");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RootDescriptorNode, 3))
+    Descriptor.RegisterSpace = *Val;
+  else
+    return reportError(Ctx, "Invalid value for RegisterSpace");
+
+  if (RSD.Version == 1) {
+    RSD.ParametersContainer.addParameter(Header, Descriptor);
+    return false;
+  }
+  assert(RSD.Version > 1);
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RootDescriptorNode, 4))
+    Descriptor.Flags = *Val;
+  else
+    return reportError(Ctx, "Invalid value for Root Descriptor Flags");
+
+  RSD.ParametersContainer.addParameter(Header, Descriptor);
+  return false;
+}
+
 static bool parseRootSignatureElement(LLVMContext *Ctx,
                                       mcdxbc::RootSignatureDesc &RSD,
                                       MDNode *Element) {
-  MDString *ElementText = cast<MDString>(Element->getOperand(0));
-  if (ElementText == nullptr)
+  std::optional<StringRef> ElementText = extractMdStringValue(Element, 0);
+  if (!ElementText.has_value())
     return reportError(Ctx, "Invalid format for Root Element");
 
   RootSignatureElementKind ElementKind =
-      StringSwitch<RootSignatureElementKind>(ElementText->getString())
+      StringSwitch<RootSignatureElementKind>(*ElementText)
           .Case("RootFlags", RootSignatureElementKind::RootFlags)
           .Case("RootConstants", RootSignatureElementKind::RootConstants)
+          .Case("RootCBV", RootSignatureElementKind::CBV)
+          .Case("RootSRV", RootSignatureElementKind::SRV)
+          .Case("RootUAV", RootSignatureElementKind::UAV)
           .Default(RootSignatureElementKind::Error);
 
   switch (ElementKind) {
@@ -126,10 +196,12 @@ static bool parseRootSignatureElement(LLVMContext *Ctx,
     return parseRootFlags(Ctx, RSD, Element);
   case RootSignatureElementKind::RootConstants:
     return parseRootConstants(Ctx, RSD, Element);
-    break;
+  case RootSignatureElementKind::CBV:
+  case RootSignatureElementKind::SRV:
+  case RootSignatureElementKind::UAV:
+    return parseRootDescriptors(Ctx, RSD, Element, ElementKind);
   case RootSignatureElementKind::Error:
-    return reportError(Ctx, "Invalid Root Signature Element: " +
-                                ElementText->getString());
+    return reportError(Ctx, "Invalid Root Signature Element: " + *ElementText);
   }
 
   llvm_unreachable("Unhandled RootSignatureElementKind enum.");
@@ -157,6 +229,18 @@ static bool verifyVersion(uint32_t Version) {
   return (Version == 1 || Version == 2);
 }
 
+static bool verifyRegisterValue(uint32_t RegisterValue) {
+  return RegisterValue != ~0U;
+}
+
+// This Range is reserverved, therefore invalid, according to the spec
+// https://github.com/llvm/wg-hlsl/blob/main/proposals/0002-root-signature-in-clang.md#all-the-values-should-be-legal
+static bool verifyRegisterSpace(uint32_t RegisterSpace) {
+  return !(RegisterSpace >= 0xFFFFFFF0 && RegisterSpace <= 0xFFFFFFFF);
+}
+
+static bool verifyDescriptorFlag(uint32_t Flags) { return (Flags & ~0xE) == 0; }
+
 static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
 
   if (!verifyVersion(RSD.Version)) {
@@ -174,6 +258,28 @@ static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
 
     assert(dxbc::isValidParameterType(Info.Header.ParameterType) &&
            "Invalid value for ParameterType");
+
+    switch (Info.Header.ParameterType) {
+
+    case llvm::to_underlying(dxbc::RootParameterType::CBV):
+    case llvm::to_underlying(dxbc::RootParameterType::UAV):
+    case llvm::to_underlying(dxbc::RootParameterType::SRV): {
+      const dxbc::RTS0::v2::RootDescriptor &Descriptor =
+          RSD.ParametersContainer.getRootDescriptor(Info.Location);
+      if (!verifyRegisterValue(Descriptor.ShaderRegister))
+        return reportValueError(Ctx, "ShaderRegister",
+                                Descriptor.ShaderRegister);
+
+      if (!verifyRegisterSpace(Descriptor.RegisterSpace))
+        return reportValueError(Ctx, "RegisterSpace", Descriptor.RegisterSpace);
+
+      if (RSD.Version > 1) {
+        if (!verifyDescriptorFlag(Descriptor.Flags))
+          return reportValueError(Ctx, "DescriptorFlag", Descriptor.Flags);
+      }
+      break;
+    }
+    }
   }
 
   return false;
@@ -313,6 +419,20 @@ PreservedAnalyses RootSignatureAnalysisPrinter::run(Module &M,
            << "Shader Register: " << Constants.ShaderRegister << "\n";
         OS << indent(Space + 2)
            << "Num 32 Bit Values: " << Constants.Num32BitValues << "\n";
+        break;
+      }
+      case llvm::to_underlying(dxbc::RootParameterType::CBV):
+      case llvm::to_underlying(dxbc::RootParameterType::UAV):
+      case llvm::to_underlying(dxbc::RootParameterType::SRV): {
+        const dxbc::RTS0::v2::RootDescriptor &Descriptor =
+            RS.ParametersContainer.getRootDescriptor(Loc);
+        OS << indent(Space + 2)
+           << "Register Space: " << Descriptor.RegisterSpace << "\n";
+        OS << indent(Space + 2)
+           << "Shader Register: " << Descriptor.ShaderRegister << "\n";
+        if (RS.Version > 1)
+          OS << indent(Space + 2) << "Flags: " << Descriptor.Flags << "\n";
+        break;
       }
       }
       Space--;
diff --git a/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.h b/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.h
index 93ec614f1ab8..3f25551b2b5e 100644
--- a/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.h
+++ b/external/llvm-project/llvm/lib/Target/DirectX/DXILRootSignature.h
@@ -27,7 +27,10 @@ namespace dxil {
 enum class RootSignatureElementKind {
   Error = 0,
   RootFlags = 1,
-  RootConstants = 2
+  RootConstants = 2,
+  SRV = 3,
+  UAV = 4,
+  CBV = 5,
 };
 class RootSignatureAnalysis : public AnalysisInfoMixin<RootSignatureAnalysis> {
   friend AnalysisInfoMixin<RootSignatureAnalysis>;
diff --git a/external/llvm-project/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
index 9f344d7d52ba..e2dd4354a816 100644
--- a/external/llvm-project/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
@@ -17,7 +17,7 @@
 #include "llvm/IR/Function.h"
 
 namespace llvm {
-class DirectXTTIImpl : public BasicTTIImplBase<DirectXTTIImpl> {
+class DirectXTTIImpl final : public BasicTTIImplBase<DirectXTTIImpl> {
   using BaseT = BasicTTIImplBase<DirectXTTIImpl>;
   using TTI = TargetTransformInfo;
 
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index ae0305d570dc..f9b4bc0d14fd 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/HexagonMCAsmInfo.h"
 #include "MCTargetDesc/HexagonMCChecker.h"
 #include "MCTargetDesc/HexagonMCELFStreamer.h"
 #include "MCTargetDesc/HexagonMCExpr.h"
@@ -457,7 +458,7 @@ struct HexagonOperand : public MCParsedAsmOperand {
 void HexagonOperand::print(raw_ostream &OS) const {
   switch (Kind) {
   case Immediate:
-    getImm()->print(OS, nullptr);
+    HexagonMCAsmInfo(Triple()).printExpr(OS, *getImm());
     break;
   case Register:
     OS << "<register R";
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp
index f38e7b879e5f..5dde47ab3de5 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonConstExtenders.cpp
@@ -253,7 +253,7 @@ namespace {
       bool operator!=(Register R) const { return !operator==(R); }
       bool operator<(Register R) const {
         // For std::map.
-        return Reg < R.Reg || (Reg == R.Reg && Sub < R.Sub);
+        return std::tie(Reg, Sub) < std::tie(R.Reg, R.Sub);
       }
       llvm::Register Reg;
       unsigned Sub = 0;
@@ -298,11 +298,7 @@ namespace {
         return !operator==(Ex);
       }
       bool operator<(const ExtExpr &Ex) const {
-        if (Rs != Ex.Rs)
-          return Rs < Ex.Rs;
-        if (S != Ex.S)
-          return S < Ex.S;
-        return !Neg && Ex.Neg;
+        return std::tie(Rs, S, Neg) < std::tie(Ex.Rs, Ex.S, Ex.Neg);
       }
     };
 
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 01efcedebc80..078eccaa706a 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -73,9 +73,6 @@ static cl::opt<bool>
     EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
                          cl::desc("Enable Hexagon SDNode scheduling"));
 
-static cl::opt<bool> EnableFastMath("ffast-math", cl::Hidden,
-                                    cl::desc("Enable Fast Math processing"));
-
 static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
                                       cl::init(5),
                                       cl::desc("Set minimum jump tables"));
@@ -1850,46 +1847,6 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
     initializeHVXLowering();
 
   computeRegisterProperties(&HRI);
-
-  //
-  // Library calls for unsupported operations
-  //
-  bool FastMath  = EnableFastMath;
-
-  setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
-  setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
-  setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
-  setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
-  setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
-  setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
-  setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
-  setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
-
-  // This is the only fast library function for sqrtd.
-  if (FastMath)
-    setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
-
-  // Prefix is: nothing  for "slow-math",
-  //            "fast2_" for V5+ fast-math double-precision
-  // (actually, keep fast-math and fast-math2 separate for now)
-  if (FastMath) {
-    setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
-    setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
-    setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
-    setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
-    setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
-  } else {
-    setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
-    setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
-    setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
-    setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
-    setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
-  }
-
-  if (FastMath)
-    setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
-  else
-    setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
 }
 
 const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index d7509c3bb1d2..c03cad4713e4 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -30,7 +30,7 @@ class ScalarEvolution;
 class User;
 class Value;
 
-class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
+class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
   using BaseT = BasicTTIImplBase<HexagonTTIImpl>;
   using TTI = TargetTransformInfo;
 
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index d0f14d8ad728..3de6df568c9f 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -2829,13 +2829,13 @@ auto HexagonVectorCombine::calculatePointerDifference(Value *Ptr0,
 auto HexagonVectorCombine::getNumSignificantBits(const Value *V,
                                                  const Instruction *CtxI) const
     -> unsigned {
-  return ComputeMaxSignificantBits(V, DL, /*Depth=*/0, &AC, CtxI, &DT);
+  return ComputeMaxSignificantBits(V, DL, &AC, CtxI, &DT);
 }
 
 auto HexagonVectorCombine::getKnownBits(const Value *V,
                                         const Instruction *CtxI) const
     -> KnownBits {
-  return computeKnownBits(V, DL, /*Depth=*/0, &AC, CtxI, &DT);
+  return computeKnownBits(V, DL, &AC, CtxI, &DT);
 }
 
 auto HexagonVectorCombine::isSafeToClone(const Instruction &In) const -> bool {
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index 624610a66ee9..45bb82c945a7 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -697,7 +697,7 @@ class HexagonAsmBackend : public MCAsmBackend {
     return true;
   }
 
-  void finishLayout(MCAssembler const &Asm) const override {
+  bool finishLayout(const MCAssembler &Asm) const override {
     SmallVector<MCFragment *> Frags;
     for (MCSection &Sec : Asm) {
       Frags.clear();
@@ -760,7 +760,6 @@ class HexagonAsmBackend : public MCAsmBackend {
               //assert(!Error);
               (void)Error;
               ReplaceInstruction(Asm.getEmitter(), RF, Inst);
-              Sec.setHasLayout(false);
               Size = 0; // Only look back one instruction
               break;
             }
@@ -770,6 +769,7 @@ class HexagonAsmBackend : public MCAsmBackend {
         }
       }
     }
+    return true;
   }
 }; // class HexagonAsmBackend
 
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
index 16b8cec54199..9030e43b7149 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
@@ -72,7 +72,7 @@ void HexagonInstPrinter::printOperand(MCInst const *MI, unsigned OpNo,
     if (MO.getExpr()->evaluateAsAbsolute(Value))
       O << formatImm(Value);
     else
-      MO.getExpr()->print(O, &MAI);
+      MAI.printExpr(O, *MO.getExpr());
   } else {
     llvm_unreachable("Unknown operand");
   }
@@ -90,6 +90,6 @@ void HexagonInstPrinter::printBrtarget(MCInst const *MI, unsigned OpNo,
     if (HasExtender || HexagonMCInstrInfo::isConstExtended(MII, *MI))
       if (HexagonMCInstrInfo::getExtendableOp(MII, *MI) == OpNo)
         O << "##";
-    Expr.print(O, &MAI);
+    MAI.printExpr(O, Expr);
   }
 }
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
index 7675b05f106a..ba8faaeb74a0 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
@@ -38,6 +38,7 @@ HexagonMCAsmInfo::HexagonMCAsmInfo(const Triple &TT) {
   LCOMMDirectiveAlignmentType = LCOMM::ByteAlignment;
   InlineAsmStart = "# InlineAsm Start";
   InlineAsmEnd = "# InlineAsm End";
+  UsesSetToEquateSymbol = true;
   ZeroDirective = "\t.space\t";
   AscizDirective = "\t.string\t";
 
diff --git a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp
index 92a8be359d73..d96e9601bf9e 100644
--- a/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "HexagonMCExpr.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCValue.h"
@@ -58,7 +59,7 @@ HexagonMCExpr::HexagonMCExpr(MCExpr const *Expr)
       SignMismatch(false) {}
 
 void HexagonMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  Expr->print(OS, MAI);
+  MAI->printExpr(OS, *Expr);
 }
 
 void HexagonMCExpr::setSignMismatch(bool Val) {
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/external/llvm-project/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
index 78b3d163bfbb..9cb7f71945d1 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
@@ -9,7 +9,7 @@
 #include "LanaiAluCode.h"
 #include "LanaiCondCode.h"
 #include "LanaiInstrInfo.h"
-#include "MCTargetDesc/LanaiMCExpr.h"
+#include "MCTargetDesc/LanaiMCAsmInfo.h"
 #include "TargetInfo/LanaiTargetInfo.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -231,14 +231,14 @@ struct LanaiOperand : public MCParsedAsmOperand {
     }
 
     // Symbolic reference expression
-    if (const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(Imm.Value))
-      return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_HI;
+    if (const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(Imm.Value))
+      return SymbolRefExpr->getSpecifier() == Lanai::S_ABS_HI;
 
     // Binary expression
     if (const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(Imm.Value))
-      if (const LanaiMCExpr *SymbolRefExpr =
-              dyn_cast<LanaiMCExpr>(BinaryExpr->getLHS()))
-        return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_HI;
+      if (const auto *SymbolRefExpr =
+              dyn_cast<MCSpecifierExpr>(BinaryExpr->getLHS()))
+        return SymbolRefExpr->getSpecifier() == Lanai::S_ABS_HI;
 
     return false;
   }
@@ -268,14 +268,14 @@ struct LanaiOperand : public MCParsedAsmOperand {
     }
 
     // Symbolic reference expression
-    if (const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(Imm.Value))
-      return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_LO;
+    if (const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(Imm.Value))
+      return SymbolRefExpr->getSpecifier() == Lanai::S_ABS_LO;
 
     // Binary expression
     if (const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(Imm.Value))
-      if (const LanaiMCExpr *SymbolRefExpr =
-              dyn_cast<LanaiMCExpr>(BinaryExpr->getLHS()))
-        return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_LO;
+      if (const auto *SymbolRefExpr =
+              dyn_cast<MCSpecifierExpr>(BinaryExpr->getLHS()))
+        return SymbolRefExpr->getSpecifier() == Lanai::S_ABS_LO;
 
     return false;
   }
@@ -292,14 +292,14 @@ struct LanaiOperand : public MCParsedAsmOperand {
     }
 
     // Symbolic reference expression
-    if (const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(Imm.Value))
-      return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_LO;
+    if (const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(Imm.Value))
+      return SymbolRefExpr->getSpecifier() == Lanai::S_ABS_LO;
 
     // Binary expression
     if (const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(Imm.Value))
-      if (const LanaiMCExpr *SymbolRefExpr =
-              dyn_cast<LanaiMCExpr>(BinaryExpr->getLHS()))
-        return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_LO;
+      if (const auto *SymbolRefExpr =
+              dyn_cast<MCSpecifierExpr>(BinaryExpr->getLHS()))
+        return SymbolRefExpr->getSpecifier() == Lanai::S_ABS_LO;
 
     return false;
   }
@@ -339,8 +339,8 @@ struct LanaiOperand : public MCParsedAsmOperand {
     }
 
     // Symbolic reference expression
-    if (const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(Imm.Value))
-      return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_None;
+    if (const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(Imm.Value))
+      return SymbolRefExpr->getSpecifier() == Lanai::S_None;
     if (const MCSymbolRefExpr *SymbolRefExpr =
             dyn_cast<MCSymbolRefExpr>(Imm.Value)) {
       return SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None;
@@ -348,9 +348,9 @@ struct LanaiOperand : public MCParsedAsmOperand {
 
     // Binary expression
     if (const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(Imm.Value)) {
-      if (const LanaiMCExpr *SymbolRefExpr =
-              dyn_cast<LanaiMCExpr>(BinaryExpr->getLHS()))
-        return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_None;
+      if (const auto *SymbolRefExpr =
+              dyn_cast<MCSpecifierExpr>(BinaryExpr->getLHS()))
+        return SymbolRefExpr->getSpecifier() == Lanai::S_None;
       if (const MCSymbolRefExpr *SymbolRefExpr =
               dyn_cast<MCSymbolRefExpr>(BinaryExpr->getLHS()))
         return SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None;
@@ -464,19 +464,18 @@ struct LanaiOperand : public MCParsedAsmOperand {
     if (const MCConstantExpr *ConstExpr = dyn_cast<MCConstantExpr>(getImm()))
       Inst.addOperand(
           MCOperand::createImm(static_cast<int32_t>(ConstExpr->getValue())));
-    else if (isa<LanaiMCExpr>(getImm())) {
+    else if (isa<MCSpecifierExpr>(getImm())) {
 #ifndef NDEBUG
-      const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(getImm());
-      assert(SymbolRefExpr &&
-             SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_LO);
+      const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(getImm());
+      assert(SymbolRefExpr && SymbolRefExpr->getSpecifier() == Lanai::S_ABS_LO);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else if (isa<MCBinaryExpr>(getImm())) {
 #ifndef NDEBUG
       const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(getImm());
-      assert(BinaryExpr && isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
-             cast<LanaiMCExpr>(BinaryExpr->getLHS())->getKind() ==
-                 LanaiMCExpr::VK_Lanai_ABS_LO);
+      assert(BinaryExpr && isa<MCSpecifierExpr>(BinaryExpr->getLHS()) &&
+             cast<MCSpecifierExpr>(BinaryExpr->getLHS())->getSpecifier() ==
+                 Lanai::S_ABS_LO);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else
@@ -495,19 +494,18 @@ struct LanaiOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     if (const MCConstantExpr *ConstExpr = dyn_cast<MCConstantExpr>(getImm()))
       Inst.addOperand(MCOperand::createImm(ConstExpr->getValue() >> 16));
-    else if (isa<LanaiMCExpr>(getImm())) {
+    else if (isa<MCSpecifierExpr>(getImm())) {
 #ifndef NDEBUG
-      const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(getImm());
-      assert(SymbolRefExpr &&
-             SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_HI);
+      const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(getImm());
+      assert(SymbolRefExpr && SymbolRefExpr->getSpecifier() == Lanai::S_ABS_HI);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else if (isa<MCBinaryExpr>(getImm())) {
 #ifndef NDEBUG
       const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(getImm());
-      assert(BinaryExpr && isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
-             cast<LanaiMCExpr>(BinaryExpr->getLHS())->getKind() ==
-                 LanaiMCExpr::VK_Lanai_ABS_HI);
+      assert(BinaryExpr && isa<MCSpecifierExpr>(BinaryExpr->getLHS()) &&
+             cast<MCSpecifierExpr>(BinaryExpr->getLHS())->getSpecifier() ==
+                 Lanai::S_ABS_HI);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else
@@ -526,11 +524,10 @@ struct LanaiOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     if (const MCConstantExpr *ConstExpr = dyn_cast<MCConstantExpr>(getImm()))
       Inst.addOperand(MCOperand::createImm(ConstExpr->getValue() & 0x1fffff));
-    else if (isa<LanaiMCExpr>(getImm())) {
+    else if (isa<MCSpecifierExpr>(getImm())) {
 #ifndef NDEBUG
-      const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(getImm());
-      assert(SymbolRefExpr &&
-             SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_None);
+      const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(getImm());
+      assert(SymbolRefExpr && SymbolRefExpr->getSpecifier() == Lanai::S_None);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else if (isa<MCSymbolRefExpr>(getImm())) {
@@ -544,9 +541,9 @@ struct LanaiOperand : public MCParsedAsmOperand {
     } else if (isa<MCBinaryExpr>(getImm())) {
 #ifndef NDEBUG
       const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(getImm());
-      assert(BinaryExpr && isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
-             cast<LanaiMCExpr>(BinaryExpr->getLHS())->getKind() ==
-                 LanaiMCExpr::VK_Lanai_None);
+      assert(BinaryExpr && isa<MCSpecifierExpr>(BinaryExpr->getLHS()) &&
+             cast<MCSpecifierExpr>(BinaryExpr->getLHS())->getSpecifier() ==
+                 Lanai::S_None);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else
@@ -737,7 +734,7 @@ std::unique_ptr<LanaiOperand> LanaiAsmParser::parseIdentifier() {
   SMLoc Start = Parser.getTok().getLoc();
   SMLoc End = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   const MCExpr *Res, *RHS = nullptr;
-  LanaiMCExpr::VariantKind Kind = LanaiMCExpr::VK_Lanai_None;
+  auto Kind = Lanai::S_None;
 
   if (Lexer.getKind() != AsmToken::Identifier)
     return nullptr;
@@ -748,13 +745,13 @@ std::unique_ptr<LanaiOperand> LanaiAsmParser::parseIdentifier() {
 
   // Check if identifier has a modifier
   if (Identifier.equals_insensitive("hi"))
-    Kind = LanaiMCExpr::VK_Lanai_ABS_HI;
+    Kind = Lanai::S_ABS_HI;
   else if (Identifier.equals_insensitive("lo"))
-    Kind = LanaiMCExpr::VK_Lanai_ABS_LO;
+    Kind = Lanai::S_ABS_LO;
 
   // If the identifier corresponds to a variant then extract the real
   // identifier.
-  if (Kind != LanaiMCExpr::VK_Lanai_None) {
+  if (Kind != Lanai::S_None) {
     if (Lexer.getKind() != AsmToken::LParen) {
       Error(Lexer.getLoc(), "Expected '('");
       return nullptr;
@@ -771,7 +768,7 @@ std::unique_ptr<LanaiOperand> LanaiAsmParser::parseIdentifier() {
     return nullptr;
 
   // For variants parse the final ')'
-  if (Kind != LanaiMCExpr::VK_Lanai_None) {
+  if (Kind != Lanai::S_None) {
     if (Lexer.getKind() != AsmToken::RParen) {
       Error(Lexer.getLoc(), "Expected ')'");
       return nullptr;
@@ -781,8 +778,7 @@ std::unique_ptr<LanaiOperand> LanaiAsmParser::parseIdentifier() {
 
   End = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
-  const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
-  Res = LanaiMCExpr::create(Kind, Expr, getContext());
+  Res = MCSpecifierExpr::create(Sym, Kind, getContext());
 
   // Nest if this was an addition
   if (RHS)
@@ -865,16 +861,16 @@ bool shouldBeSls(const LanaiOperand &Op) {
   }
   // The instruction should be encoded as an SLS if the operand is a symbolic
   // reference with no variant.
-  if (const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(Op.getImm()))
-    return SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_None;
+  if (const auto *SymbolRefExpr = dyn_cast<MCSpecifierExpr>(Op.getImm()))
+    return SymbolRefExpr->getSpecifier() == Lanai::S_None;
   // The instruction should be encoded as an SLS if the operand is a binary
   // expression with the left-hand side being a symbolic reference with no
   // variant.
   if (const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(Op.getImm())) {
-    const LanaiMCExpr *LHSSymbolRefExpr =
-        dyn_cast<LanaiMCExpr>(BinaryExpr->getLHS());
+    const auto *LHSSymbolRefExpr =
+        dyn_cast<MCSpecifierExpr>(BinaryExpr->getLHS());
     return (LHSSymbolRefExpr &&
-            LHSSymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_None);
+            LHSSymbolRefExpr->getSpecifier() == Lanai::S_None);
   }
   return false;
 }
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/external/llvm-project/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index 8b50eb2678af..7781817aef71 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -151,9 +151,8 @@ LanaiTargetLowering::LanaiTargetLowering(const TargetMachine &TM,
   setMinimumJumpTableEntries(100);
 
   // Use fast calling convention for library functions.
-  for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) {
-    setLibcallCallingConv(static_cast<RTLIB::Libcall>(I), CallingConv::Fast);
-  }
+  for (RTLIB::Libcall LC : RTLIB::libcalls())
+    setLibcallCallingConv(LC, CallingConv::Fast);
 
   MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
   MaxStoresPerMemsetOptSize = 8;
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/LanaiMCInstLower.cpp b/external/llvm-project/llvm/lib/Target/Lanai/LanaiMCInstLower.cpp
index c2fba5dcd89a..b0db8d088768 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/LanaiMCInstLower.cpp
+++ b/external/llvm-project/llvm/lib/Target/Lanai/LanaiMCInstLower.cpp
@@ -14,7 +14,7 @@
 #include "LanaiMCInstLower.h"
 
 #include "MCTargetDesc/LanaiBaseInfo.h"
-#include "MCTargetDesc/LanaiMCExpr.h"
+#include "MCTargetDesc/LanaiMCAsmInfo.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -64,17 +64,16 @@ LanaiMCInstLower::GetConstantPoolIndexSymbol(const MachineOperand &MO) const {
 
 MCOperand LanaiMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
                                                MCSymbol *Sym) const {
-  LanaiMCExpr::VariantKind Kind;
-
+  Lanai::Specifier Kind;
   switch (MO.getTargetFlags()) {
   case LanaiII::MO_NO_FLAG:
-    Kind = LanaiMCExpr::VK_Lanai_None;
+    Kind = Lanai::S_None;
     break;
   case LanaiII::MO_ABS_HI:
-    Kind = LanaiMCExpr::VK_Lanai_ABS_HI;
+    Kind = Lanai::S_ABS_HI;
     break;
   case LanaiII::MO_ABS_LO:
-    Kind = LanaiMCExpr::VK_Lanai_ABS_LO;
+    Kind = Lanai::S_ABS_LO;
     break;
   default:
     llvm_unreachable("Unknown target flag on GV operand");
@@ -84,7 +83,7 @@ MCOperand LanaiMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   if (!MO.isJTI() && MO.getOffset())
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
-  Expr = LanaiMCExpr::create(Kind, Expr, Ctx);
+  Expr = MCSpecifierExpr::create(Expr, Kind, Ctx);
   return MCOperand::createExpr(Expr);
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
index f17abf4c8af0..0342af65c1ef 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
@@ -25,7 +25,7 @@
 #include "llvm/Support/MathExtras.h"
 
 namespace llvm {
-class LanaiTTIImpl : public BasicTTIImplBase<LanaiTTIImpl> {
+class LanaiTTIImpl final : public BasicTTIImplBase<LanaiTTIImpl> {
   typedef BasicTTIImplBase<LanaiTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/CMakeLists.txt b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/CMakeLists.txt
index 04fca878ca5a..ff3b6abc70ec 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/CMakeLists.txt
@@ -4,7 +4,6 @@ add_llvm_component_library(LLVMLanaiDesc
   LanaiInstPrinter.cpp
   LanaiMCAsmInfo.cpp
   LanaiMCCodeEmitter.cpp
-  LanaiMCExpr.cpp
   LanaiMCTargetDesc.cpp
 
   LINK_COMPONENTS
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
index 837d8fea1c89..add4096ef936 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
@@ -152,7 +152,7 @@ void LanaiInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     OS << formatHex(Op.getImm());
   else {
     assert(Op.isExpr() && "Expected an expression");
-    Op.getExpr()->print(OS, &MAI);
+    MAI.printExpr(OS, *Op.getExpr());
   }
 }
 
@@ -165,7 +165,7 @@ void LanaiInstPrinter::printMemImmOperand(const MCInst *MI, unsigned OpNo,
     // Symbolic operand will be lowered to immediate value by linker
     assert(Op.isExpr() && "Expected an expression");
     OS << '[';
-    Op.getExpr()->print(OS, &MAI);
+    MAI.printExpr(OS, *Op.getExpr());
     OS << ']';
   }
 }
@@ -178,7 +178,7 @@ void LanaiInstPrinter::printHi16ImmOperand(const MCInst *MI, unsigned OpNo,
   } else {
     // Symbolic operand will be lowered to immediate value by linker
     assert(Op.isExpr() && "Expected an expression");
-    Op.getExpr()->print(OS, &MAI);
+    MAI.printExpr(OS, *Op.getExpr());
   }
 }
 
@@ -190,7 +190,7 @@ void LanaiInstPrinter::printHi16AndImmOperand(const MCInst *MI, unsigned OpNo,
   } else {
     // Symbolic operand will be lowered to immediate value by linker
     assert(Op.isExpr() && "Expected an expression");
-    Op.getExpr()->print(OS, &MAI);
+    MAI.printExpr(OS, *Op.getExpr());
   }
 }
 
@@ -202,7 +202,7 @@ void LanaiInstPrinter::printLo16AndImmOperand(const MCInst *MI, unsigned OpNo,
   } else {
     // Symbolic operand will be lowered to immediate value by linker
     assert(Op.isExpr() && "Expected an expression");
-    Op.getExpr()->print(OS, &MAI);
+    MAI.printExpr(OS, *Op.getExpr());
   }
 }
 
@@ -227,7 +227,7 @@ static void printMemoryImmediateOffset(const MCAsmInfo &MAI,
     assert(isInt<SizeInBits>(OffsetOp.getImm()) && "Constant value truncated");
     OS << OffsetOp.getImm();
   } else
-    OffsetOp.getExpr()->print(OS, &MAI);
+    MAI.printExpr(OS, *OffsetOp.getExpr());
 }
 
 void LanaiInstPrinter::printMemRiOperand(const MCInst *MI, int OpNo,
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp
index 7ae693130da5..6ad018c12a28 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp
@@ -11,7 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "LanaiMCAsmInfo.h"
-
+#include "llvm/MC/MCExpr.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
@@ -38,3 +39,26 @@ LanaiMCAsmInfo::LanaiMCAsmInfo(const Triple & /*TheTriple*/,
   // in dwarf generation.
   MinInstAlignment = 4;
 }
+
+void LanaiMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                        const MCSpecifierExpr &Expr) const {
+  if (Expr.getSpecifier() == 0) {
+    printExpr(OS, *Expr.getSubExpr());
+    return;
+  }
+
+  switch (Expr.getSpecifier()) {
+  default:
+    llvm_unreachable("Invalid kind!");
+  case Lanai::S_ABS_HI:
+    OS << "hi";
+    break;
+  case Lanai::S_ABS_LO:
+    OS << "lo";
+    break;
+  }
+
+  OS << '(';
+  printExpr(OS, *Expr.getSubExpr());
+  OS << ')';
+}
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h
index f0352d021291..2696975e71c0 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h
@@ -24,8 +24,15 @@ class LanaiMCAsmInfo : public MCAsmInfoELF {
 public:
   explicit LanaiMCAsmInfo(const Triple &TheTriple,
                           const MCTargetOptions &Options);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
 };
 
+namespace Lanai {
+using Specifier = uint8_t;
+enum { S_None, S_ABS_HI, S_ABS_LO };
+} // namespace Lanai
+
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_LANAI_MCTARGETDESC_LANAIMCASMINFO_H
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
index 7ddfac85760d..d1b2da40446a 100644
--- a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
@@ -13,7 +13,7 @@
 #include "LanaiAluCode.h"
 #include "MCTargetDesc/LanaiBaseInfo.h"
 #include "MCTargetDesc/LanaiFixupKinds.h"
-#include "MCTargetDesc/LanaiMCExpr.h"
+#include "MCTargetDesc/LanaiMCAsmInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/MC/MCCodeEmitter.h"
@@ -89,14 +89,14 @@ class LanaiMCCodeEmitter : public MCCodeEmitter {
 static Lanai::Fixups FixupKind(const MCExpr *Expr) {
   if (isa<MCSymbolRefExpr>(Expr))
     return Lanai::FIXUP_LANAI_21;
-  if (const LanaiMCExpr *McExpr = dyn_cast<LanaiMCExpr>(Expr)) {
-    LanaiMCExpr::VariantKind ExprKind = McExpr->getKind();
+  if (const MCSpecifierExpr *McExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
+    Lanai::Specifier ExprKind = McExpr->getSpecifier();
     switch (ExprKind) {
-    case LanaiMCExpr::VK_Lanai_None:
+    case Lanai::S_None:
       return Lanai::FIXUP_LANAI_21;
-    case LanaiMCExpr::VK_Lanai_ABS_HI:
+    case Lanai::S_ABS_HI:
       return Lanai::FIXUP_LANAI_HI16;
-    case LanaiMCExpr::VK_Lanai_ABS_LO:
+    case Lanai::S_ABS_LO:
       return Lanai::FIXUP_LANAI_LO16;
     }
   }
@@ -123,7 +123,7 @@ unsigned LanaiMCCodeEmitter::getMachineOpValue(
     Expr = BinaryExpr->getLHS();
   }
 
-  assert(isa<LanaiMCExpr>(Expr) || Expr->getKind() == MCExpr::SymbolRef);
+  assert(isa<MCSpecifierExpr>(Expr) || Expr->getKind() == MCExpr::SymbolRef);
   // Push fixup (all info is contained within)
   Fixups.push_back(
       MCFixup::create(0, MCOp.getExpr(), MCFixupKind(FixupKind(Expr))));
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp
deleted file mode 100644
index fced12328628..000000000000
--- a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- LanaiMCExpr.cpp - Lanai specific MC expression classes ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "LanaiMCExpr.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCStreamer.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "lanaimcexpr"
-
-const LanaiMCExpr *LanaiMCExpr::create(VariantKind Kind, const MCExpr *Expr,
-                                       MCContext &Ctx) {
-  return new (Ctx) LanaiMCExpr(Kind, Expr);
-}
-
-void LanaiMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  if (specifier == VK_Lanai_None) {
-    Expr->print(OS, MAI);
-    return;
-  }
-
-  switch (specifier) {
-  default:
-    llvm_unreachable("Invalid kind!");
-  case VK_Lanai_ABS_HI:
-    OS << "hi";
-    break;
-  case VK_Lanai_ABS_LO:
-    OS << "lo";
-    break;
-  }
-
-  OS << '(';
-  const MCExpr *Expr = getSubExpr();
-  Expr->print(OS, MAI);
-  OS << ')';
-}
-
-void LanaiMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-bool LanaiMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                            const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-  return true;
-}
diff --git a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h b/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h
deleted file mode 100644
index 3543fe020f81..000000000000
--- a/external/llvm-project/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- LanaiMCExpr.h - Lanai specific MC expression classes ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_LANAI_MCTARGETDESC_LANAIMCEXPR_H
-#define LLVM_LIB_TARGET_LANAI_MCTARGETDESC_LANAIMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCValue.h"
-
-namespace llvm {
-
-class LanaiMCExpr : public MCTargetExpr {
-public:
-  enum VariantKind { VK_Lanai_None, VK_Lanai_ABS_HI, VK_Lanai_ABS_LO };
-
-private:
-  const VariantKind specifier;
-  const MCExpr *Expr;
-
-  explicit LanaiMCExpr(VariantKind Kind, const MCExpr *Expr)
-      : specifier(Kind), Expr(Expr) {}
-
-public:
-  static const LanaiMCExpr *create(VariantKind Kind, const MCExpr *Expr,
-                                   MCContext &Ctx);
-
-  // Returns the kind of this expression.
-  VariantKind getKind() const { return specifier; }
-
-  // Returns the child of this expression.
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-};
-} // end namespace llvm
-
-#endif
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
index f5c540728852..7d5827008957 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
@@ -8,6 +8,7 @@
 
 #include "MCTargetDesc/LoongArchBaseInfo.h"
 #include "MCTargetDesc/LoongArchInstPrinter.h"
+#include "MCTargetDesc/LoongArchMCAsmInfo.h"
 #include "MCTargetDesc/LoongArchMCExpr.h"
 #include "MCTargetDesc/LoongArchMCTargetDesc.h"
 #include "MCTargetDesc/LoongArchMatInt.h"
@@ -755,7 +756,7 @@ LoongArchAsmParser::parseOperandWithModifier(OperandVector &Operands) {
   if (getLexer().getKind() != AsmToken::Identifier)
     return Error(getLoc(), "expected valid identifier for operand modifier");
   StringRef Identifier = getParser().getTok().getIdentifier();
-  LoongArchMCExpr::Specifier VK = LoongArchMCExpr::parseSpecifier(Identifier);
+  auto VK = LoongArch::parseSpecifier(Identifier);
   if (VK == LoongArchMCExpr::VK_None)
     return Error(getLoc(), "invalid relocation specifier");
 
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArch.h b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArch.h
index b9dc7822893e..e5b308334879 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArch.h
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArch.h
@@ -35,7 +35,8 @@ bool lowerLoongArchMachineOperandToMCOperand(const MachineOperand &MO,
 
 FunctionPass *createLoongArchDeadRegisterDefinitionsPass();
 FunctionPass *createLoongArchExpandAtomicPseudoPass();
-FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM);
+FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM,
+                                     CodeGenOptLevel OptLevel);
 FunctionPass *createLoongArchMergeBaseOffsetOptPass();
 FunctionPass *createLoongArchOptWInstrsPass();
 FunctionPass *createLoongArchPreRAExpandPseudoPass();
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp
index 9181e539f75c..64ac7c03c041 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchAsmPrinter.cpp
@@ -20,6 +20,7 @@
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCSectionELF.h"
@@ -160,9 +161,10 @@ bool LoongArchAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
   else if (OffsetMO.isImm())
     OS << ", " << OffsetMO.getImm();
   else if (OffsetMO.isGlobal() || OffsetMO.isBlockAddress() ||
-           OffsetMO.isMCSymbol())
-    OS << ", " << *MCO.getExpr();
-  else
+           OffsetMO.isMCSymbol()) {
+    OS << ", ";
+    MAI->printExpr(OS, *MCO.getExpr());
+  } else
     return true;
 
   return false;
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp
index 3be012feb238..73874fccc030 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp
@@ -122,6 +122,18 @@ bool LoongArchExpandAtomicPseudo::expandMI(
   case LoongArch::PseudoAtomicLoadXor32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
                              NextMBBI);
+  case LoongArch::PseudoAtomicLoadUMax32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 32,
+                                NextMBBI);
+  case LoongArch::PseudoAtomicLoadUMin32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 32,
+                                NextMBBI);
+  case LoongArch::PseudoAtomicLoadMax32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 32,
+                                NextMBBI);
+  case LoongArch::PseudoAtomicLoadMin32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 32,
+                                NextMBBI);
   case LoongArch::PseudoMaskedAtomicLoadUMax32:
     return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
                                 NextMBBI);
@@ -356,8 +368,6 @@ bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
     MachineBasicBlock::iterator &NextMBBI) {
-  assert(IsMasked == true &&
-         "Should only need to expand masked atomic max/min");
   assert(Width == 32 && "Should never need to expand masked 64-bit operations");
 
   MachineInstr &MI = *MBBI;
@@ -385,79 +395,92 @@ bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
   MBB.addSuccessor(LoopHeadMBB);
 
   Register DestReg = MI.getOperand(0).getReg();
-  Register Scratch1Reg = MI.getOperand(1).getReg();
-  Register Scratch2Reg = MI.getOperand(2).getReg();
-  Register AddrReg = MI.getOperand(3).getReg();
-  Register IncrReg = MI.getOperand(4).getReg();
-  Register MaskReg = MI.getOperand(5).getReg();
+  Register ScratchReg = MI.getOperand(1).getReg();
+  Register AddrReg = MI.getOperand(IsMasked ? 3 : 2).getReg();
+  Register IncrReg = MI.getOperand(IsMasked ? 4 : 3).getReg();
+  Register CmprReg = DestReg;
 
   //
   // .loophead:
   //   ll.w destreg, (alignedaddr)
-  //   and scratch2, destreg, mask
-  //   move scratch1, destreg
   BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_W), DestReg)
       .addReg(AddrReg)
       .addImm(0);
-  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), Scratch2Reg)
-      .addReg(DestReg)
-      .addReg(MaskReg);
-  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::OR), Scratch1Reg)
+  //   and cmpr, destreg, mask
+  if (IsMasked) {
+    Register MaskReg = MI.getOperand(5).getReg();
+    CmprReg = MI.getOperand(2).getReg();
+    BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), CmprReg)
+        .addReg(DestReg)
+        .addReg(MaskReg);
+  }
+  //   move scratch, destreg
+  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::OR), ScratchReg)
       .addReg(DestReg)
       .addReg(LoongArch::R0);
 
   switch (BinOp) {
   default:
     llvm_unreachable("Unexpected AtomicRMW BinOp");
-  // bgeu scratch2, incr, .looptail
+  // bgeu cmpr, incr, .looptail
   case AtomicRMWInst::UMax:
     BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
-        .addReg(Scratch2Reg)
+        .addReg(CmprReg)
         .addReg(IncrReg)
         .addMBB(LoopTailMBB);
     break;
-  // bgeu incr, scratch2, .looptail
+  // bgeu incr, cmpr, .looptail
   case AtomicRMWInst::UMin:
     BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
         .addReg(IncrReg)
-        .addReg(Scratch2Reg)
+        .addReg(CmprReg)
         .addMBB(LoopTailMBB);
     break;
   case AtomicRMWInst::Max:
-    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
-    // bge scratch2, incr, .looptail
+    if (IsMasked)
+      insertSext(TII, DL, LoopHeadMBB, CmprReg, MI.getOperand(6).getReg());
+    // bge cmpr, incr, .looptail
     BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
-        .addReg(Scratch2Reg)
+        .addReg(CmprReg)
         .addReg(IncrReg)
         .addMBB(LoopTailMBB);
     break;
   case AtomicRMWInst::Min:
-    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
-    // bge incr, scratch2, .looptail
+    if (IsMasked)
+      insertSext(TII, DL, LoopHeadMBB, CmprReg, MI.getOperand(6).getReg());
+    // bge incr, cmpr, .looptail
     BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
         .addReg(IncrReg)
-        .addReg(Scratch2Reg)
+        .addReg(CmprReg)
         .addMBB(LoopTailMBB);
     break;
     // TODO: support other AtomicRMWInst.
   }
 
   // .loopifbody:
-  //   xor scratch1, destreg, incr
-  //   and scratch1, scratch1, mask
-  //   xor scratch1, destreg, scratch1
-  insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
-                    MaskReg, Scratch1Reg);
+  if (IsMasked) {
+    Register MaskReg = MI.getOperand(5).getReg();
+    // xor scratch, destreg, incr
+    // and scratch, scratch, mask
+    // xor scratch, destreg, scratch
+    insertMaskedMerge(TII, DL, LoopIfBodyMBB, ScratchReg, DestReg, IncrReg,
+                      MaskReg, ScratchReg);
+  } else {
+    // move scratch, incr
+    BuildMI(LoopIfBodyMBB, DL, TII->get(LoongArch::OR), ScratchReg)
+        .addReg(IncrReg)
+        .addReg(LoongArch::R0);
+  }
 
   // .looptail:
-  //   sc.w scratch1, scratch1, (addr)
-  //   beqz scratch1, loop
-  BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), Scratch1Reg)
-      .addReg(Scratch1Reg)
+  //   sc.w scratch, scratch, (addr)
+  //   beqz scratch, loop
+  BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), ScratchReg)
+      .addReg(ScratchReg)
       .addReg(AddrReg)
       .addImm(0);
   BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQ))
-      .addReg(Scratch1Reg)
+      .addReg(ScratchReg)
       .addReg(LoongArch::R0)
       .addMBB(LoopHeadMBB);
 
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.cpp
index 7c1f55a26c91..71d0263fe376 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.cpp
@@ -25,8 +25,9 @@ using namespace llvm;
 char LoongArchDAGToDAGISelLegacy::ID;
 
 LoongArchDAGToDAGISelLegacy::LoongArchDAGToDAGISelLegacy(
-    LoongArchTargetMachine &TM)
-    : SelectionDAGISelLegacy(ID, std::make_unique<LoongArchDAGToDAGISel>(TM)) {}
+    LoongArchTargetMachine &TM, CodeGenOptLevel OptLevel)
+    : SelectionDAGISelLegacy(
+          ID, std::make_unique<LoongArchDAGToDAGISel>(TM, OptLevel)) {}
 
 INITIALIZE_PASS(LoongArchDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false,
                 false)
@@ -456,6 +457,7 @@ bool LoongArchDAGToDAGISel::selectVSplatUimmPow2(SDValue N,
 
 // This pass converts a legalized DAG into a LoongArch-specific DAG, ready
 // for instruction scheduling.
-FunctionPass *llvm::createLoongArchISelDag(LoongArchTargetMachine &TM) {
-  return new LoongArchDAGToDAGISelLegacy(TM);
+FunctionPass *llvm::createLoongArchISelDag(LoongArchTargetMachine &TM,
+                                           CodeGenOptLevel OptLevel) {
+  return new LoongArchDAGToDAGISelLegacy(TM, OptLevel);
 }
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.h b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.h
index e94f249c14be..1eed877dcef1 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.h
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelDAGToDAG.h
@@ -26,8 +26,9 @@ class LoongArchDAGToDAGISel : public SelectionDAGISel {
 public:
   LoongArchDAGToDAGISel() = delete;
 
-  explicit LoongArchDAGToDAGISel(LoongArchTargetMachine &TM)
-      : SelectionDAGISel(TM) {}
+  explicit LoongArchDAGToDAGISel(LoongArchTargetMachine &TM,
+                                 CodeGenOptLevel OptLevel)
+      : SelectionDAGISel(TM, OptLevel) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override {
     Subtarget = &MF.getSubtarget<LoongArchSubtarget>();
@@ -93,7 +94,8 @@ class LoongArchDAGToDAGISel : public SelectionDAGISel {
 class LoongArchDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
 public:
   static char ID;
-  explicit LoongArchDAGToDAGISelLegacy(LoongArchTargetMachine &TM);
+  explicit LoongArchDAGToDAGISelLegacy(LoongArchTargetMachine &TM,
+                                       CodeGenOptLevel OptLevel);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 7a9ec9f5e96b..99dae6ec3eb0 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -182,6 +182,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   if (Subtarget.hasBasicF()) {
     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
+    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
+    setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
     setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
 
     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
@@ -199,8 +201,13 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
     setOperationAction(ISD::FPOW, MVT::f32, Expand);
     setOperationAction(ISD::FREM, MVT::f32, Expand);
-    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
-    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
+    setOperationAction(ISD::FP16_TO_FP, MVT::f32,
+                       Subtarget.isSoftFPABI() ? LibCall : Custom);
+    setOperationAction(ISD::FP_TO_FP16, MVT::f32,
+                       Subtarget.isSoftFPABI() ? LibCall : Custom);
+    setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
+    setOperationAction(ISD::FP_TO_BF16, MVT::f32,
+                       Subtarget.isSoftFPABI() ? LibCall : Custom);
 
     if (Subtarget.is64Bit())
       setOperationAction(ISD::FRINT, MVT::f32, Legal);
@@ -219,6 +226,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
   if (Subtarget.hasBasicD()) {
     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
+    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
+    setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
     setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
@@ -239,7 +248,11 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FPOW, MVT::f64, Expand);
     setOperationAction(ISD::FREM, MVT::f64, Expand);
     setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
-    setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
+    setOperationAction(ISD::FP_TO_FP16, MVT::f64,
+                       Subtarget.isSoftFPABI() ? LibCall : Custom);
+    setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
+    setOperationAction(ISD::FP_TO_BF16, MVT::f64,
+                       Subtarget.isSoftFPABI() ? LibCall : Custom);
 
     if (Subtarget.is64Bit())
       setOperationAction(ISD::FRINT, MVT::f64, Legal);
@@ -388,8 +401,10 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
 
   // Set DAG combine for 'LSX' feature.
 
-  if (Subtarget.hasExtLSX())
+  if (Subtarget.hasExtLSX()) {
     setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
+    setTargetDAGCombine(ISD::BITCAST);
+  }
 
   // Compute derived properties from the register classes.
   computeRegisterProperties(Subtarget.getRegisterInfo());
@@ -490,6 +505,14 @@ SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
     return lowerPREFETCH(Op, DAG);
   case ISD::SELECT:
     return lowerSELECT(Op, DAG);
+  case ISD::FP_TO_FP16:
+    return lowerFP_TO_FP16(Op, DAG);
+  case ISD::FP16_TO_FP:
+    return lowerFP16_TO_FP(Op, DAG);
+  case ISD::FP_TO_BF16:
+    return lowerFP_TO_BF16(Op, DAG);
+  case ISD::BF16_TO_FP:
+    return lowerBF16_TO_FP(Op, DAG);
   }
   return SDValue();
 }
@@ -2127,6 +2150,51 @@ static void canonicalizeShuffleVectorByLane(const SDLoc &DL,
   }
 }
 
+/// Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
+/// Only for 256-bit vector.
+///
+/// For example:
+/// %2 = shufflevector <4 x i64> %0, <4 x i64> posion,
+///                    <4 x i64> <i32 0, i32 3, i32 2, i32 0>
+/// is lowerded to:
+///     (XVPERMI $xr2, $xr0, 78)
+///     (XVSHUF  $xr1, $xr2, $xr0)
+///     (XVORI   $xr0, $xr1, 0)
+static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL,
+                                                          ArrayRef<int> Mask,
+                                                          MVT VT, SDValue V1,
+                                                          SDValue V2,
+                                                          SelectionDAG &DAG) {
+  assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
+  int Size = Mask.size();
+  int LaneSize = Size / 2;
+
+  bool LaneCrossing[2] = {false, false};
+  for (int i = 0; i < Size; ++i)
+    if (Mask[i] >= 0 && ((Mask[i] % Size) / LaneSize) != (i / LaneSize))
+      LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
+
+  // Ensure that all lanes ared involved.
+  if (!LaneCrossing[0] && !LaneCrossing[1])
+    return SDValue();
+
+  SmallVector<int> InLaneMask;
+  InLaneMask.assign(Mask.begin(), Mask.end());
+  for (int i = 0; i < Size; ++i) {
+    int &M = InLaneMask[i];
+    if (M < 0)
+      continue;
+    if (((M % Size) / LaneSize) != (i / LaneSize))
+      M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
+  }
+
+  SDValue Flipped = DAG.getBitcast(MVT::v4i64, V1);
+  Flipped = DAG.getVectorShuffle(MVT::v4i64, DL, Flipped,
+                                 DAG.getUNDEF(MVT::v4i64), {2, 3, 0, 1});
+  Flipped = DAG.getBitcast(VT, Flipped);
+  return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask);
+}
+
 /// Dispatching routine to lower various 256-bit LoongArch vector shuffles.
 ///
 /// This routine breaks down the specific type of 256-bit shuffle and
@@ -2159,6 +2227,9 @@ static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
       return Result;
     if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, NewMask, VT, V1, V2, DAG)))
       return Result;
+    if ((Result = lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(DL, NewMask, VT,
+                                                             V1, V2, DAG)))
+      return Result;
 
     // TODO: This comment may be enabled in the future to better match the
     // pattern for instruction selection.
@@ -2242,6 +2313,70 @@ SDValue LoongArchTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
   return SDValue();
 }
 
+SDValue LoongArchTargetLowering::lowerFP_TO_FP16(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  // Custom lower to ensure the libcall return is passed in an FPR on hard
+  // float ABIs.
+  SDLoc DL(Op);
+  MakeLibCallOptions CallOptions;
+  SDValue Op0 = Op.getOperand(0);
+  SDValue Chain = SDValue();
+  RTLIB::Libcall LC = RTLIB::getFPROUND(Op0.getValueType(), MVT::f16);
+  SDValue Res;
+  std::tie(Res, Chain) =
+      makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions, DL, Chain);
+  if (Subtarget.is64Bit())
+    return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Res);
+  return DAG.getBitcast(MVT::i32, Res);
+}
+
+SDValue LoongArchTargetLowering::lowerFP16_TO_FP(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  // Custom lower to ensure the libcall argument is passed in an FPR on hard
+  // float ABIs.
+  SDLoc DL(Op);
+  MakeLibCallOptions CallOptions;
+  SDValue Op0 = Op.getOperand(0);
+  SDValue Chain = SDValue();
+  SDValue Arg = Subtarget.is64Bit() ? DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64,
+                                                  DL, MVT::f32, Op0)
+                                    : DAG.getBitcast(MVT::f32, Op0);
+  SDValue Res;
+  std::tie(Res, Chain) = makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
+                                     CallOptions, DL, Chain);
+  return Res;
+}
+
+SDValue LoongArchTargetLowering::lowerFP_TO_BF16(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
+  SDLoc DL(Op);
+  MakeLibCallOptions CallOptions;
+  RTLIB::Libcall LC =
+      RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::bf16);
+  SDValue Res =
+      makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
+  if (Subtarget.is64Bit())
+    return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Res);
+  return DAG.getBitcast(MVT::i32, Res);
+}
+
+SDValue LoongArchTargetLowering::lowerBF16_TO_FP(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
+  MVT VT = Op.getSimpleValueType();
+  SDLoc DL(Op);
+  Op = DAG.getNode(
+      ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0),
+      DAG.getShiftAmountConstant(16, Op.getOperand(0).getValueType(), DL));
+  SDValue Res = Subtarget.is64Bit() ? DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64,
+                                                  DL, MVT::f32, Op)
+                                    : DAG.getBitcast(MVT::f32, Op);
+  if (VT != MVT::f32)
+    return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
+  return Res;
+}
+
 static bool isConstantOrUndef(const SDValue Op) {
   if (Op->isUndef())
     return true;
@@ -3841,6 +3976,8 @@ void LoongArchTargetLowering::ReplaceNodeResults(
     EVT FVT = EVT::getFloatingPointVT(N->getValueSizeInBits(0));
     if (getTypeAction(*DAG.getContext(), Src.getValueType()) !=
         TargetLowering::TypeSoftenFloat) {
+      if (!isTypeLegal(Src.getValueType()))
+        return;
       if (Src.getValueType() == MVT::f16)
         Src = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
       SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, FVT, Src);
@@ -4286,6 +4423,187 @@ static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+// Helper to peek through bitops/trunc/setcc to determine size of source vector.
+// Allows BITCASTCombine to determine what size vector generated a <X x i1>.
+static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
+                                      unsigned Depth) {
+  // Limit recursion.
+  if (Depth >= SelectionDAG::MaxRecursionDepth)
+    return false;
+  switch (Src.getOpcode()) {
+  case ISD::SETCC:
+  case ISD::TRUNCATE:
+    return Src.getOperand(0).getValueSizeInBits() == Size;
+  case ISD::FREEZE:
+    return checkBitcastSrcVectorSize(Src.getOperand(0), Size, Depth + 1);
+  case ISD::AND:
+  case ISD::XOR:
+  case ISD::OR:
+    return checkBitcastSrcVectorSize(Src.getOperand(0), Size, Depth + 1) &&
+           checkBitcastSrcVectorSize(Src.getOperand(1), Size, Depth + 1);
+  case ISD::SELECT:
+  case ISD::VSELECT:
+    return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
+           checkBitcastSrcVectorSize(Src.getOperand(1), Size, Depth + 1) &&
+           checkBitcastSrcVectorSize(Src.getOperand(2), Size, Depth + 1);
+  case ISD::BUILD_VECTOR:
+    return ISD::isBuildVectorAllZeros(Src.getNode()) ||
+           ISD::isBuildVectorAllOnes(Src.getNode());
+  }
+  return false;
+}
+
+// Helper to push sign extension of vXi1 SETCC result through bitops.
+static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
+                                          SDValue Src, const SDLoc &DL) {
+  switch (Src.getOpcode()) {
+  case ISD::SETCC:
+  case ISD::FREEZE:
+  case ISD::TRUNCATE:
+  case ISD::BUILD_VECTOR:
+    return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
+  case ISD::AND:
+  case ISD::XOR:
+  case ISD::OR:
+    return DAG.getNode(
+        Src.getOpcode(), DL, SExtVT,
+        signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
+        signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
+  case ISD::SELECT:
+  case ISD::VSELECT:
+    return DAG.getSelect(
+        DL, SExtVT, Src.getOperand(0),
+        signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL),
+        signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(2), DL));
+  }
+  llvm_unreachable("Unexpected node type for vXi1 sign extension");
+}
+
+static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG,
+                                     TargetLowering::DAGCombinerInfo &DCI,
+                                     const LoongArchSubtarget &Subtarget) {
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+  SDValue Src = N->getOperand(0);
+  EVT SrcVT = Src.getValueType();
+
+  if (!DCI.isBeforeLegalizeOps())
+    return SDValue();
+
+  if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
+    return SDValue();
+
+  unsigned Opc = ISD::DELETED_NODE;
+  // Combine SETCC and BITCAST into [X]VMSK{LT,GE,NE} when possible
+  if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) {
+    bool UseLASX;
+    EVT CmpVT = Src.getOperand(0).getValueType();
+    EVT EltVT = CmpVT.getVectorElementType();
+
+    if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() <= 128)
+      UseLASX = false;
+    else if (Subtarget.has32S() && Subtarget.hasExtLASX() &&
+             CmpVT.getSizeInBits() <= 256)
+      UseLASX = true;
+    else
+      return SDValue();
+
+    SDValue SrcN1 = Src.getOperand(1);
+    switch (cast<CondCodeSDNode>(Src.getOperand(2))->get()) {
+    default:
+      break;
+    case ISD::SETEQ:
+      // x == 0 => not (vmsknez.b x)
+      if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8)
+        Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
+      break;
+    case ISD::SETGT:
+      // x > -1 => vmskgez.b x
+      if (ISD::isBuildVectorAllOnes(SrcN1.getNode()) && EltVT == MVT::i8)
+        Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
+      break;
+    case ISD::SETGE:
+      // x >= 0 => vmskgez.b x
+      if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8)
+        Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
+      break;
+    case ISD::SETLT:
+      // x < 0 => vmskltz.{b,h,w,d} x
+      if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) &&
+          (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
+           EltVT == MVT::i64))
+        Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
+      break;
+    case ISD::SETLE:
+      // x <= -1 => vmskltz.{b,h,w,d} x
+      if (ISD::isBuildVectorAllOnes(SrcN1.getNode()) &&
+          (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
+           EltVT == MVT::i64))
+        Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
+      break;
+    case ISD::SETNE:
+      // x != 0 => vmsknez.b x
+      if (ISD::isBuildVectorAllZeros(SrcN1.getNode()) && EltVT == MVT::i8)
+        Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
+      break;
+    }
+  }
+
+  // Generate vXi1 using [X]VMSKLTZ
+  if (Opc == ISD::DELETED_NODE) {
+    MVT SExtVT;
+    bool UseLASX = false;
+    bool PropagateSExt = false;
+    switch (SrcVT.getSimpleVT().SimpleTy) {
+    default:
+      return SDValue();
+    case MVT::v2i1:
+      SExtVT = MVT::v2i64;
+      break;
+    case MVT::v4i1:
+      SExtVT = MVT::v4i32;
+      if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) {
+        SExtVT = MVT::v4i64;
+        UseLASX = true;
+        PropagateSExt = true;
+      }
+      break;
+    case MVT::v8i1:
+      SExtVT = MVT::v8i16;
+      if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) {
+        SExtVT = MVT::v8i32;
+        UseLASX = true;
+        PropagateSExt = true;
+      }
+      break;
+    case MVT::v16i1:
+      SExtVT = MVT::v16i8;
+      if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) {
+        SExtVT = MVT::v16i16;
+        UseLASX = true;
+        PropagateSExt = true;
+      }
+      break;
+    case MVT::v32i1:
+      SExtVT = MVT::v32i8;
+      UseLASX = true;
+      break;
+    };
+    if (UseLASX && !Subtarget.has32S() && !Subtarget.hasExtLASX())
+      return SDValue();
+    Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
+                        : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
+    Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
+  } else {
+    Src = Src.getOperand(0);
+  }
+
+  SDValue V = DAG.getNode(Opc, DL, MVT::i64, Src);
+  EVT T = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements());
+  V = DAG.getZExtOrTrunc(V, DL, T);
+  return DAG.getBitcast(VT, V);
+}
+
 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
                                 TargetLowering::DAGCombinerInfo &DCI,
                                 const LoongArchSubtarget &Subtarget) {
@@ -5289,6 +5607,48 @@ performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG,
+                                        TargetLowering::DAGCombinerInfo &DCI,
+                                        const LoongArchSubtarget &Subtarget) {
+  // If the input to MOVGR2FR_W_LA64 is just MOVFR2GR_S_LA64 the the
+  // conversion is unnecessary and can be replaced with the
+  // MOVFR2GR_S_LA64 operand.
+  SDValue Op0 = N->getOperand(0);
+  if (Op0.getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
+    return Op0.getOperand(0);
+  return SDValue();
+}
+
+static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG,
+                                        TargetLowering::DAGCombinerInfo &DCI,
+                                        const LoongArchSubtarget &Subtarget) {
+  // If the input to MOVFR2GR_S_LA64 is just MOVGR2FR_W_LA64 then the
+  // conversion is unnecessary and can be replaced with the MOVGR2FR_W_LA64
+  // operand.
+  SDValue Op0 = N->getOperand(0);
+  if (Op0->getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
+    assert(Op0.getOperand(0).getValueType() == N->getSimpleValueType(0) &&
+           "Unexpected value type!");
+    return Op0.getOperand(0);
+  }
+  return SDValue();
+}
+
+static SDValue performVMSKLTZCombine(SDNode *N, SelectionDAG &DAG,
+                                     TargetLowering::DAGCombinerInfo &DCI,
+                                     const LoongArchSubtarget &Subtarget) {
+  MVT VT = N->getSimpleValueType(0);
+  unsigned NumBits = VT.getScalarSizeInBits();
+
+  // Simplify the inputs.
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  APInt DemandedMask(APInt::getAllOnes(NumBits));
+  if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
+    return SDValue(N, 0);
+
+  return SDValue();
+}
+
 SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
                                                    DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -5303,10 +5663,19 @@ SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
     return performSETCCCombine(N, DAG, DCI, Subtarget);
   case ISD::SRL:
     return performSRLCombine(N, DAG, DCI, Subtarget);
+  case ISD::BITCAST:
+    return performBITCASTCombine(N, DAG, DCI, Subtarget);
   case LoongArchISD::BITREV_W:
     return performBITREV_WCombine(N, DAG, DCI, Subtarget);
   case ISD::INTRINSIC_WO_CHAIN:
     return performINTRINSIC_WO_CHAINCombine(N, DAG, DCI, Subtarget);
+  case LoongArchISD::MOVGR2FR_W_LA64:
+    return performMOVGR2FR_WCombine(N, DAG, DCI, Subtarget);
+  case LoongArchISD::MOVFR2GR_S_LA64:
+    return performMOVFR2GR_SCombine(N, DAG, DCI, Subtarget);
+  case LoongArchISD::VMSKLTZ:
+  case LoongArchISD::XVMSKLTZ:
+    return performVMSKLTZCombine(N, DAG, DCI, Subtarget);
   }
   return SDValue();
 }
@@ -5589,6 +5958,120 @@ static MachineBasicBlock *emitPseudoCTPOP(MachineInstr &MI,
   return BB;
 }
 
+static MachineBasicBlock *
+emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB,
+                   const LoongArchSubtarget &Subtarget) {
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
+  const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+  Register Dst = MI.getOperand(0).getReg();
+  Register Src = MI.getOperand(1).getReg();
+  DebugLoc DL = MI.getDebugLoc();
+  unsigned EleBits = 8;
+  unsigned NotOpc = 0;
+  unsigned MskOpc;
+
+  switch (MI.getOpcode()) {
+  default:
+    llvm_unreachable("Unexpected opcode");
+  case LoongArch::PseudoVMSKLTZ_B:
+    MskOpc = LoongArch::VMSKLTZ_B;
+    break;
+  case LoongArch::PseudoVMSKLTZ_H:
+    MskOpc = LoongArch::VMSKLTZ_H;
+    EleBits = 16;
+    break;
+  case LoongArch::PseudoVMSKLTZ_W:
+    MskOpc = LoongArch::VMSKLTZ_W;
+    EleBits = 32;
+    break;
+  case LoongArch::PseudoVMSKLTZ_D:
+    MskOpc = LoongArch::VMSKLTZ_D;
+    EleBits = 64;
+    break;
+  case LoongArch::PseudoVMSKGEZ_B:
+    MskOpc = LoongArch::VMSKGEZ_B;
+    break;
+  case LoongArch::PseudoVMSKEQZ_B:
+    MskOpc = LoongArch::VMSKNZ_B;
+    NotOpc = LoongArch::VNOR_V;
+    break;
+  case LoongArch::PseudoVMSKNEZ_B:
+    MskOpc = LoongArch::VMSKNZ_B;
+    break;
+  case LoongArch::PseudoXVMSKLTZ_B:
+    MskOpc = LoongArch::XVMSKLTZ_B;
+    RC = &LoongArch::LASX256RegClass;
+    break;
+  case LoongArch::PseudoXVMSKLTZ_H:
+    MskOpc = LoongArch::XVMSKLTZ_H;
+    RC = &LoongArch::LASX256RegClass;
+    EleBits = 16;
+    break;
+  case LoongArch::PseudoXVMSKLTZ_W:
+    MskOpc = LoongArch::XVMSKLTZ_W;
+    RC = &LoongArch::LASX256RegClass;
+    EleBits = 32;
+    break;
+  case LoongArch::PseudoXVMSKLTZ_D:
+    MskOpc = LoongArch::XVMSKLTZ_D;
+    RC = &LoongArch::LASX256RegClass;
+    EleBits = 64;
+    break;
+  case LoongArch::PseudoXVMSKGEZ_B:
+    MskOpc = LoongArch::XVMSKGEZ_B;
+    RC = &LoongArch::LASX256RegClass;
+    break;
+  case LoongArch::PseudoXVMSKEQZ_B:
+    MskOpc = LoongArch::XVMSKNZ_B;
+    NotOpc = LoongArch::XVNOR_V;
+    RC = &LoongArch::LASX256RegClass;
+    break;
+  case LoongArch::PseudoXVMSKNEZ_B:
+    MskOpc = LoongArch::XVMSKNZ_B;
+    RC = &LoongArch::LASX256RegClass;
+    break;
+  }
+
+  Register Msk = MRI.createVirtualRegister(RC);
+  if (NotOpc) {
+    Register Tmp = MRI.createVirtualRegister(RC);
+    BuildMI(*BB, MI, DL, TII->get(MskOpc), Tmp).addReg(Src);
+    BuildMI(*BB, MI, DL, TII->get(NotOpc), Msk)
+        .addReg(Tmp, RegState::Kill)
+        .addReg(Tmp, RegState::Kill);
+  } else {
+    BuildMI(*BB, MI, DL, TII->get(MskOpc), Msk).addReg(Src);
+  }
+
+  if (TRI->getRegSizeInBits(*RC) > 128) {
+    Register Lo = MRI.createVirtualRegister(&LoongArch::GPRRegClass);
+    Register Hi = MRI.createVirtualRegister(&LoongArch::GPRRegClass);
+    BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPICKVE2GR_WU), Lo)
+        .addReg(Msk)
+        .addImm(0);
+    BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPICKVE2GR_WU), Hi)
+        .addReg(Msk, RegState::Kill)
+        .addImm(4);
+    BuildMI(*BB, MI, DL,
+            TII->get(Subtarget.is64Bit() ? LoongArch::BSTRINS_D
+                                         : LoongArch::BSTRINS_W),
+            Dst)
+        .addReg(Lo, RegState::Kill)
+        .addReg(Hi, RegState::Kill)
+        .addImm(256 / EleBits - 1)
+        .addImm(128 / EleBits);
+  } else {
+    BuildMI(*BB, MI, DL, TII->get(LoongArch::VPICKVE2GR_HU), Dst)
+        .addReg(Msk, RegState::Kill)
+        .addImm(0);
+  }
+
+  MI.eraseFromParent();
+  return BB;
+}
+
 static bool isSelectPseudo(MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
@@ -5795,6 +6278,21 @@ MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter(
     return emitPseudoXVINSGR2VR(MI, BB, Subtarget);
   case LoongArch::PseudoCTPOP:
     return emitPseudoCTPOP(MI, BB, Subtarget);
+  case LoongArch::PseudoVMSKLTZ_B:
+  case LoongArch::PseudoVMSKLTZ_H:
+  case LoongArch::PseudoVMSKLTZ_W:
+  case LoongArch::PseudoVMSKLTZ_D:
+  case LoongArch::PseudoVMSKGEZ_B:
+  case LoongArch::PseudoVMSKEQZ_B:
+  case LoongArch::PseudoVMSKNEZ_B:
+  case LoongArch::PseudoXVMSKLTZ_B:
+  case LoongArch::PseudoXVMSKLTZ_H:
+  case LoongArch::PseudoXVMSKLTZ_W:
+  case LoongArch::PseudoXVMSKLTZ_D:
+  case LoongArch::PseudoXVMSKGEZ_B:
+  case LoongArch::PseudoXVMSKEQZ_B:
+  case LoongArch::PseudoXVMSKNEZ_B:
+    return emitPseudoVMSKCOND(MI, BB, Subtarget);
   case TargetOpcode::STATEPOINT:
     // STATEPOINT is a pseudo instruction which has no implicit defs/uses
     // while bl call instruction (where statepoint will be lowered at the
@@ -5916,6 +6414,14 @@ const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const {
     NODE_NAME_CASE(VBSLL)
     NODE_NAME_CASE(VBSRL)
     NODE_NAME_CASE(VLDREPL)
+    NODE_NAME_CASE(VMSKLTZ)
+    NODE_NAME_CASE(VMSKGEZ)
+    NODE_NAME_CASE(VMSKEQZ)
+    NODE_NAME_CASE(VMSKNEZ)
+    NODE_NAME_CASE(XVMSKLTZ)
+    NODE_NAME_CASE(XVMSKGEZ)
+    NODE_NAME_CASE(XVMSKEQZ)
+    NODE_NAME_CASE(XVMSKNEZ)
   }
 #undef NODE_NAME_CASE
   return nullptr;
@@ -7070,6 +7576,14 @@ getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen,
       return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
     case AtomicRMWInst::Nand:
       return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
+    case AtomicRMWInst::UMax:
+      return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
+    case AtomicRMWInst::UMin:
+      return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
+    case AtomicRMWInst::Max:
+      return Intrinsic::loongarch_masked_atomicrmw_max_i32;
+    case AtomicRMWInst::Min:
+      return Intrinsic::loongarch_masked_atomicrmw_min_i32;
       // TODO: support other AtomicRMWInst.
     }
   }
@@ -7093,19 +7607,22 @@ LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR(
 Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
+  unsigned GRLen = Subtarget.getGRLen();
   AtomicOrdering FailOrd = CI->getFailureOrdering();
   Value *FailureOrdering =
       Builder.getIntN(Subtarget.getGRLen(), static_cast<uint64_t>(FailOrd));
-
-  // TODO: Support cmpxchg on LA32.
-  Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
-  CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
-  NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
-  Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
+  Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
+  if (GRLen == 64) {
+    CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
+    CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
+    NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
+    Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
+  }
   Type *Tys[] = {AlignedAddr->getType()};
   Value *Result = Builder.CreateIntrinsic(
       CmpXchgIntrID, Tys, {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
-  Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
+  if (GRLen == 64)
+    Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
   return Result;
 }
 
@@ -7633,3 +8150,118 @@ LoongArchTargetLowering::getPreferredVectorAction(MVT VT) const {
 
   return TargetLoweringBase::getPreferredVectorAction(VT);
 }
+
+bool LoongArchTargetLowering::splitValueIntoRegisterParts(
+    SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
+    unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
+  bool IsABIRegCopy = CC.has_value();
+  EVT ValueVT = Val.getValueType();
+
+  if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
+      PartVT == MVT::f32) {
+    // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
+    // nan, and cast to f32.
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
+    Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
+    Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
+                      DAG.getConstant(0xFFFF0000, DL, MVT::i32));
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
+    Parts[0] = Val;
+    return true;
+  }
+
+  return false;
+}
+
+SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
+    SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
+    MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
+  bool IsABIRegCopy = CC.has_value();
+
+  if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
+      PartVT == MVT::f32) {
+    SDValue Val = Parts[0];
+
+    // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
+    Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
+    Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
+    return Val;
+  }
+
+  return SDValue();
+}
+
+MVT LoongArchTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
+                                                           CallingConv::ID CC,
+                                                           EVT VT) const {
+  // Use f32 to pass f16.
+  if (VT == MVT::f16 && Subtarget.hasBasicF())
+    return MVT::f32;
+
+  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
+}
+
+unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
+    LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
+  // Use f32 to pass f16.
+  if (VT == MVT::f16 && Subtarget.hasBasicF())
+    return 1;
+
+  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
+}
+
+bool LoongArchTargetLowering::SimplifyDemandedBitsForTargetNode(
+    SDValue Op, const APInt &OriginalDemandedBits,
+    const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
+    unsigned Depth) const {
+  EVT VT = Op.getValueType();
+  unsigned BitWidth = OriginalDemandedBits.getBitWidth();
+  unsigned Opc = Op.getOpcode();
+  switch (Opc) {
+  default:
+    break;
+  case LoongArchISD::VMSKLTZ:
+  case LoongArchISD::XVMSKLTZ: {
+    SDValue Src = Op.getOperand(0);
+    MVT SrcVT = Src.getSimpleValueType();
+    unsigned SrcBits = SrcVT.getScalarSizeInBits();
+    unsigned NumElts = SrcVT.getVectorNumElements();
+
+    // If we don't need the sign bits at all just return zero.
+    if (OriginalDemandedBits.countr_zero() >= NumElts)
+      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
+
+    // Only demand the vector elements of the sign bits we need.
+    APInt KnownUndef, KnownZero;
+    APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(NumElts);
+    if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero,
+                                   TLO, Depth + 1))
+      return true;
+
+    Known.Zero = KnownZero.zext(BitWidth);
+    Known.Zero.setHighBits(BitWidth - NumElts);
+
+    // [X]VMSKLTZ only uses the MSB from each vector element.
+    KnownBits KnownSrc;
+    APInt DemandedSrcBits = APInt::getSignMask(SrcBits);
+    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, KnownSrc, TLO,
+                             Depth + 1))
+      return true;
+
+    if (KnownSrc.One[SrcBits - 1])
+      Known.One.setLowBits(NumElts);
+    else if (KnownSrc.Zero[SrcBits - 1])
+      Known.Zero.setLowBits(NumElts);
+
+    // Attempt to avoid multi-use ops if we don't need anything from it.
+    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
+            Src, DemandedSrcBits, DemandedElts, TLO.DAG, Depth + 1))
+      return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
+    return false;
+  }
+  }
+
+  return TargetLowering::SimplifyDemandedBitsForTargetNode(
+      Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
+}
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 6bf295984dfc..79aa89726191 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -161,7 +161,17 @@ enum NodeType : unsigned {
   VBSRL,
 
   // Scalar load broadcast to vector
-  VLDREPL
+  VLDREPL,
+
+  // Vector mask set by condition
+  VMSKLTZ,
+  VMSKGEZ,
+  VMSKEQZ,
+  VMSKNEZ,
+  XVMSKLTZ,
+  XVMSKGEZ,
+  XVMSKEQZ,
+  XVMSKNEZ,
 
   // Intrinsic operations end =============================================
 };
@@ -304,6 +314,12 @@ class LoongArchTargetLowering : public TargetLowering {
   bool isFPImmVLDILegal(const APFloat &Imm, EVT VT) const;
   LegalizeTypeAction getPreferredVectorAction(MVT VT) const override;
 
+  bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits,
+                                         const APInt &DemandedElts,
+                                         KnownBits &Known,
+                                         TargetLoweringOpt &TLO,
+                                         unsigned Depth) const override;
+
 private:
   /// Target-specific function used to lower LoongArch calling conventions.
   typedef bool LoongArchCCAssignFn(const DataLayout &DL, LoongArchABI::ABI ABI,
@@ -361,6 +377,10 @@ class LoongArchTargetLowering : public TargetLowering {
   SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerBF16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
 
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
@@ -385,6 +405,28 @@ class LoongArchTargetLowering : public TargetLowering {
       const SmallVectorImpl<CCValAssign> &ArgLocs) const;
 
   bool softPromoteHalfType() const override { return true; }
+
+  bool
+  splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
+                              SDValue *Parts, unsigned NumParts, MVT PartVT,
+                              std::optional<CallingConv::ID> CC) const override;
+
+  SDValue
+  joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL,
+                             const SDValue *Parts, unsigned NumParts,
+                             MVT PartVT, EVT ValueVT,
+                             std::optional<CallingConv::ID> CC) const override;
+
+  /// Return the register type for a given MVT, ensuring vectors are treated
+  /// as a series of gpr sized integers.
+  MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
+                                    EVT VT) const override;
+
+  /// Return the number of registers for a given MVT, ensuring vectors are
+  /// treated as a series of gpr sized integers.
+  unsigned getNumRegistersForCallingConv(LLVMContext &Context,
+                                         CallingConv::ID CC,
+                                         EVT VT) const override;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 449b3363d8a5..b6552ed33f5b 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -2055,6 +2055,10 @@ def PseudoAtomicLoadSub32 : PseudoAM;
 def PseudoAtomicLoadAnd32 : PseudoAM;
 def PseudoAtomicLoadOr32 : PseudoAM;
 def PseudoAtomicLoadXor32 : PseudoAM;
+def PseudoAtomicLoadUMax32 : PseudoAM;
+def PseudoAtomicLoadUMin32 : PseudoAM;
+def PseudoAtomicLoadMax32 : PseudoAM;
+def PseudoAtomicLoadMin32 : PseudoAM;
 
 multiclass PseudoBinPat<string Op, Pseudo BinInst> {
   def : Pat<(!cast<PatFrag>(Op#"_monotonic") GPR:$addr, GPR:$incr),
@@ -2253,6 +2257,22 @@ def : Pat<(atomic_cmp_swap_i64 GPR:$addr, GPR:$cmp, GPR:$new),
           (AMCAS__DB_D GPR:$cmp, GPR:$new, GPR:$addr)>;
 }
 
+// Ordering constants must be kept in sync with the AtomicOrdering enum in
+// AtomicOrdering.h.
+multiclass PseudoCmpXchgPat<string Op, Pseudo CmpXchgInst,
+                            ValueType vt = GRLenVT> {
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_monotonic") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 2)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_acquire") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 4)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_release") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 5)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_acq_rel") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 6)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_seq_cst") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 7)>;
+}
+
 let Predicates = [IsLA64] in {
 defm : binary_atomic_op_wd<"AMSWAP", "atomic_swap">;
 defm : binary_atomic_op_wd<"AMADD", "atomic_load_add">;
@@ -2288,23 +2308,6 @@ def : AtomicPat<int_loongarch_masked_atomicrmw_umax_i64,
 def : AtomicPat<int_loongarch_masked_atomicrmw_umin_i64,
                 PseudoMaskedAtomicLoadUMin32>;
 
-// Ordering constants must be kept in sync with the AtomicOrdering enum in
-// AtomicOrdering.h.
-multiclass PseudoCmpXchgPat<string Op, Pseudo CmpXchgInst,
-                            ValueType vt = GRLenVT> {
-  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_monotonic") GPR:$addr, GPR:$cmp, GPR:$new)),
-            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 2)>;
-  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_acquire") GPR:$addr, GPR:$cmp, GPR:$new)),
-            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 4)>;
-  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_release") GPR:$addr, GPR:$cmp, GPR:$new)),
-            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 5)>;
-  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_acq_rel") GPR:$addr, GPR:$cmp, GPR:$new)),
-            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 6)>;
-  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_seq_cst") GPR:$addr, GPR:$cmp, GPR:$new)),
-            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 7)>;
-}
-
-defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32>;
 defm : PseudoCmpXchgPat<"atomic_cmp_swap_i64", PseudoCmpXchg64, i64>;
 def : Pat<(int_loongarch_masked_cmpxchg_i64
             GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order),
@@ -2317,6 +2320,7 @@ def : PseudoMaskedAMMinMaxPat<int_loongarch_masked_atomicrmw_min_i64,
                               PseudoMaskedAtomicLoadMin32>;
 } // Predicates = [IsLA64]
 
+defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32>;
 defm : PseudoBinPat<"atomic_load_nand_i32", PseudoAtomicLoadNand32>;
 
 let Predicates = [IsLA32] in {
@@ -2329,11 +2333,30 @@ def : AtomicPat<int_loongarch_masked_atomicrmw_sub_i32,
                 PseudoMaskedAtomicLoadSub32>;
 def : AtomicPat<int_loongarch_masked_atomicrmw_nand_i32,
                 PseudoMaskedAtomicLoadNand32>;
+def : AtomicPat<int_loongarch_masked_atomicrmw_umax_i32,
+                PseudoMaskedAtomicLoadUMax32>;
+def : AtomicPat<int_loongarch_masked_atomicrmw_umin_i32,
+                PseudoMaskedAtomicLoadUMin32>;
+
+def : PseudoMaskedAMMinMaxPat<int_loongarch_masked_atomicrmw_max_i32,
+                              PseudoMaskedAtomicLoadMax32>;
+def : PseudoMaskedAMMinMaxPat<int_loongarch_masked_atomicrmw_min_i32,
+                              PseudoMaskedAtomicLoadMin32>;
+
+def : Pat<(int_loongarch_masked_cmpxchg_i32
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order),
+          (PseudoMaskedCmpXchg32
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order)>;
+
 defm : PseudoBinPat<"atomic_load_add_i32", PseudoAtomicLoadAdd32>;
 defm : PseudoBinPat<"atomic_load_sub_i32", PseudoAtomicLoadSub32>;
 defm : PseudoBinPat<"atomic_load_and_i32", PseudoAtomicLoadAnd32>;
 defm : PseudoBinPat<"atomic_load_or_i32", PseudoAtomicLoadOr32>;
 defm : PseudoBinPat<"atomic_load_xor_i32", PseudoAtomicLoadXor32>;
+defm : PseudoBinPat<"atomic_load_umax_i32", PseudoAtomicLoadUMax32>;
+defm : PseudoBinPat<"atomic_load_umin_i32", PseudoAtomicLoadUMin32>;
+defm : PseudoBinPat<"atomic_load_max_i32", PseudoAtomicLoadMax32>;
+defm : PseudoBinPat<"atomic_load_min_i32", PseudoAtomicLoadMin32>;
 } // Predicates = [IsLA32]
 
 /// Intrinsics
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
index 59dec76ef1c2..ff7b0f2ae3f2 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
@@ -12,6 +12,10 @@
 
 // Target nodes.
 def loongarch_xvpermi: SDNode<"LoongArchISD::XVPERMI", SDT_LoongArchV1RUimm>;
+def loongarch_xvmskltz: SDNode<"LoongArchISD::XVMSKLTZ", SDT_LoongArchVMSKCOND>;
+def loongarch_xvmskgez: SDNode<"LoongArchISD::XVMSKGEZ", SDT_LoongArchVMSKCOND>;
+def loongarch_xvmskeqz: SDNode<"LoongArchISD::XVMSKEQZ", SDT_LoongArchVMSKCOND>;
+def loongarch_xvmsknez: SDNode<"LoongArchISD::XVMSKNEZ", SDT_LoongArchVMSKCOND>;
 
 def lasxsplati8
   : PatFrag<(ops node:$e0),
@@ -1086,6 +1090,16 @@ def PseudoXVINSGR2VR_H
   : Pseudo<(outs LASX256:$dst), (ins LASX256:$xd, GPR:$rj, uimm4:$imm)>;
 } //  usesCustomInserter = 1, Constraints = "$xd = $dst"
 
+let usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+def PseudoXVMSKLTZ_B : Pseudo<(outs GPR:$rd), (ins LASX256:$vj)>;
+def PseudoXVMSKLTZ_H : Pseudo<(outs GPR:$rd), (ins LASX256:$vj)>;
+def PseudoXVMSKLTZ_W : Pseudo<(outs GPR:$rd), (ins LASX256:$vj)>;
+def PseudoXVMSKLTZ_D : Pseudo<(outs GPR:$rd), (ins LASX256:$vj)>;
+def PseudoXVMSKGEZ_B : Pseudo<(outs GPR:$rd), (ins LASX256:$vj)>;
+def PseudoXVMSKEQZ_B : Pseudo<(outs GPR:$rd), (ins LASX256:$vj)>;
+def PseudoXVMSKNEZ_B : Pseudo<(outs GPR:$rd), (ins LASX256:$vj)>;
+} // usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0
+
 } // Predicates = [HasExtLASX]
 
 multiclass PatXr<SDPatternOperator OpNode, string Inst> {
@@ -1856,6 +1870,15 @@ def : Pat<(vt (concat_vectors LSX128:$vd, LSX128:$vj)),
 defm : PatXrXr<abds, "XVABSD">;
 defm : PatXrXrU<abdu, "XVABSD">;
 
+// Vector mask set by condition
+def : Pat<(loongarch_xvmskltz (v32i8 LASX256:$vj)), (PseudoXVMSKLTZ_B LASX256:$vj)>;
+def : Pat<(loongarch_xvmskltz (v16i16 LASX256:$vj)), (PseudoXVMSKLTZ_H LASX256:$vj)>;
+def : Pat<(loongarch_xvmskltz (v8i32 LASX256:$vj)), (PseudoXVMSKLTZ_W LASX256:$vj)>;
+def : Pat<(loongarch_xvmskltz (v4i64 LASX256:$vj)), (PseudoXVMSKLTZ_D LASX256:$vj)>;
+def : Pat<(loongarch_xvmskgez (v32i8 LASX256:$vj)), (PseudoXVMSKGEZ_B LASX256:$vj)>;
+def : Pat<(loongarch_xvmskeqz (v32i8 LASX256:$vj)), (PseudoXVMSKEQZ_B LASX256:$vj)>;
+def : Pat<(loongarch_xvmsknez (v32i8 LASX256:$vj)), (PseudoXVMSKNEZ_B LASX256:$vj)>;
+
 } // Predicates = [HasExtLASX]
 
 /// Intrinsic pattern
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
index b2b3b6515526..d73d78083ddc 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
@@ -31,6 +31,7 @@ def SDT_LoongArchVreplgr2vr : SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<0>, S
 def SDT_LoongArchVFRECIPE : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisVec<0>, SDTCisSameAs<0, 1>]>;
 def SDT_LoongArchVFRSQRTE : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisVec<0>, SDTCisSameAs<0, 1>]>;
 def SDT_LoongArchVLDREPL : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisPtrTy<1>]>;
+def SDT_LoongArchVMSKCOND : SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<1>]>;
 
 // Target nodes.
 def loongarch_vreplve : SDNode<"LoongArchISD::VREPLVE", SDT_LoongArchVreplve>;
@@ -74,6 +75,11 @@ def loongarch_vldrepl
     : SDNode<"LoongArchISD::VLDREPL",
              SDT_LoongArchVLDREPL, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
+def loongarch_vmskltz: SDNode<"LoongArchISD::VMSKLTZ", SDT_LoongArchVMSKCOND>;
+def loongarch_vmskgez: SDNode<"LoongArchISD::VMSKGEZ", SDT_LoongArchVMSKCOND>;
+def loongarch_vmskeqz: SDNode<"LoongArchISD::VMSKEQZ", SDT_LoongArchVMSKCOND>;
+def loongarch_vmsknez: SDNode<"LoongArchISD::VMSKNEZ", SDT_LoongArchVMSKCOND>;
+
 def immZExt1 : ImmLeaf<i64, [{return isUInt<1>(Imm);}]>;
 def immZExt2 : ImmLeaf<i64, [{return isUInt<2>(Imm);}]>;
 def immZExt3 : ImmLeaf<i64, [{return isUInt<3>(Imm);}]>;
@@ -1266,6 +1272,16 @@ let usesCustomInserter = 1 in
 def PseudoCTPOP : Pseudo<(outs GPR:$rd), (ins GPR:$rj),
                          [(set GPR:$rd, (ctpop GPR:$rj))]>;
 
+let usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+def PseudoVMSKLTZ_B : Pseudo<(outs GPR:$rd), (ins LSX128:$vj)>;
+def PseudoVMSKLTZ_H : Pseudo<(outs GPR:$rd), (ins LSX128:$vj)>;
+def PseudoVMSKLTZ_W : Pseudo<(outs GPR:$rd), (ins LSX128:$vj)>;
+def PseudoVMSKLTZ_D : Pseudo<(outs GPR:$rd), (ins LSX128:$vj)>;
+def PseudoVMSKGEZ_B : Pseudo<(outs GPR:$rd), (ins LSX128:$vj)>;
+def PseudoVMSKEQZ_B : Pseudo<(outs GPR:$rd), (ins LSX128:$vj)>;
+def PseudoVMSKNEZ_B : Pseudo<(outs GPR:$rd), (ins LSX128:$vj)>;
+} // usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0
+
 } // Predicates = [HasExtLSX]
 
 multiclass PatVr<SDPatternOperator OpNode, string Inst> {
@@ -2050,6 +2066,15 @@ def : Pat<(f64 f64imm_vldi:$in),
 defm : PatVrVr<abds, "VABSD">;
 defm : PatVrVrU<abdu, "VABSD">;
 
+// Vector mask set by condition
+def : Pat<(loongarch_vmskltz (v16i8 LSX128:$vj)), (PseudoVMSKLTZ_B LSX128:$vj)>;
+def : Pat<(loongarch_vmskltz (v8i16 LSX128:$vj)), (PseudoVMSKLTZ_H LSX128:$vj)>;
+def : Pat<(loongarch_vmskltz (v4i32 LSX128:$vj)), (PseudoVMSKLTZ_W LSX128:$vj)>;
+def : Pat<(loongarch_vmskltz (v2i64 LSX128:$vj)), (PseudoVMSKLTZ_D LSX128:$vj)>;
+def : Pat<(loongarch_vmskgez (v16i8 LSX128:$vj)), (PseudoVMSKGEZ_B LSX128:$vj)>;
+def : Pat<(loongarch_vmskeqz (v16i8 LSX128:$vj)), (PseudoVMSKEQZ_B LSX128:$vj)>;
+def : Pat<(loongarch_vmsknez (v16i8 LSX128:$vj)), (PseudoVMSKNEZ_B LSX128:$vj)>;
+
 } // Predicates = [HasExtLSX]
 
 /// Intrinsic pattern
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
index 0c8cb14f1ad3..d63e5a2b50e8 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
@@ -189,7 +189,7 @@ void LoongArchPassConfig::addCodeGenPrepare() {
 }
 
 bool LoongArchPassConfig::addInstSelector() {
-  addPass(createLoongArchISelDag(getLoongArchTargetMachine()));
+  addPass(createLoongArchISelDag(getLoongArchTargetMachine(), getOptLevel()));
 
   return false;
 }
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp
index e59cac7726a6..f912af330e34 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchInstPrinter.cpp
@@ -84,7 +84,7 @@ void LoongArchInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
 }
 
 void LoongArchInstPrinter::printAtomicMemOp(const MCInst *MI, unsigned OpNo,
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp
index 9b7fccd0078e..dc55ceab2dd3 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp
@@ -11,7 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "LoongArchMCAsmInfo.h"
+#include "LoongArchMCExpr.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -32,3 +34,14 @@ LoongArchMCAsmInfo::LoongArchMCAsmInfo(const Triple &TT) {
   DwarfRegNumForCFI = true;
   ExceptionsType = ExceptionHandling::DwarfCFI;
 }
+
+void LoongArchMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                            const MCSpecifierExpr &Expr) const {
+  auto S = Expr.getSpecifier();
+  bool HasSpecifier = S != 0 && S != ELF::R_LARCH_B26;
+  if (HasSpecifier)
+    OS << '%' << LoongArch::getSpecifierName(S) << '(';
+  printExpr(OS, *Expr.getSubExpr());
+  if (HasSpecifier)
+    OS << ')';
+}
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h
index ed1abbf46153..58ffb723d62c 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h
@@ -23,8 +23,15 @@ class LoongArchMCAsmInfo : public MCAsmInfoELF {
 
 public:
   explicit LoongArchMCAsmInfo(const Triple &TargetTriple);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
 };
 
+namespace LoongArch {
+StringRef getSpecifierName(uint16_t S);
+uint16_t parseSpecifier(StringRef name);
+} // namespace LoongArch
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_LOONGARCH_MCTARGETDESC_LOONGARCHMCASMINFO_H
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
index ef0bf683afc7..d4ac1cafd42d 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
@@ -130,7 +130,7 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
   const MCExpr *Expr = MO.getExpr();
   MCExpr::ExprKind Kind = Expr->getKind();
   unsigned FixupKind = LoongArch::fixup_loongarch_invalid;
-  if (Kind == MCExpr::Target) {
+  if (Kind == MCExpr::Specifier) {
     const LoongArchMCExpr *LAExpr = cast<LoongArchMCExpr>(Expr);
     FixupKind = LAExpr->getSpecifier();
     RelaxCandidate = LAExpr->getRelaxHint();
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
index ecf19738bbc5..c763aaa7276f 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LoongArchMCExpr.h"
+#include "LoongArchMCAsmInfo.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
@@ -27,36 +28,7 @@ const LoongArchMCExpr *LoongArchMCExpr::create(const MCExpr *Expr, uint16_t S,
   return new (Ctx) LoongArchMCExpr(Expr, Specifier(S), Hint);
 }
 
-void LoongArchMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  Specifier S = getSpecifier();
-  bool HasVariant = S != VK_None && S != ELF::R_LARCH_B26;
-
-  if (HasVariant)
-    OS << '%' << getSpecifierName(specifier) << '(';
-  Expr->print(OS, MAI);
-  if (HasVariant)
-    OS << ')';
-}
-
-bool LoongArchMCExpr::evaluateAsRelocatableImpl(
-    MCValue &Res,
-    const MCAssembler *Asm) const { // Explicitly drop the layout and assembler
-                                    // to prevent any symbolic folding in
-  // the expression handling.  This is required to preserve symbolic difference
-  // expressions to emit the paired relocations.
-  if (!getSubExpr()->evaluateAsRelocatable(Res, nullptr))
-    return false;
-
-  Res.setSpecifier(specifier);
-  // Custom fixup types are not valid with symbol difference expressions.
-  return !Res.getSubSym();
-}
-
-void LoongArchMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-StringRef LoongArchMCExpr::getSpecifierName(uint16_t S) {
+StringRef LoongArch::getSpecifierName(uint16_t S) {
   switch (S) {
   default:
     llvm_unreachable("Invalid ELF symbol kind");
@@ -167,7 +139,7 @@ StringRef LoongArchMCExpr::getSpecifierName(uint16_t S) {
   }
 }
 
-LoongArchMCExpr::Specifier LoongArchMCExpr::parseSpecifier(StringRef name) {
+LoongArchMCExpr::Specifier LoongArch::parseSpecifier(StringRef name) {
   return StringSwitch<LoongArchMCExpr::Specifier>(name)
       .Case("plt", ELF::R_LARCH_B26)
       .Case("b16", ELF::R_LARCH_B16)
@@ -223,5 +195,5 @@ LoongArchMCExpr::Specifier LoongArchMCExpr::parseSpecifier(StringRef name) {
       .Case("ld_pcrel_20", ELF::R_LARCH_TLS_LD_PCREL20_S2)
       .Case("gd_pcrel_20", ELF::R_LARCH_TLS_GD_PCREL20_S2)
       .Case("desc_pcrel_20", ELF::R_LARCH_TLS_DESC_PCREL20_S2)
-      .Default(VK_None);
+      .Default(0);
 }
diff --git a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
index 06ebbc034042..36563d8a6b60 100644
--- a/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
@@ -20,41 +20,22 @@ namespace llvm {
 
 class StringRef;
 
-class LoongArchMCExpr : public MCTargetExpr {
+class LoongArchMCExpr : public MCSpecifierExpr {
 public:
   using Specifier = uint16_t;
   enum { VK_None };
 
 private:
-  const MCExpr *Expr;
-  const Specifier specifier;
   const bool RelaxHint;
 
   explicit LoongArchMCExpr(const MCExpr *Expr, Specifier S, bool Hint)
-      : Expr(Expr), specifier(S), RelaxHint(Hint) {}
+      : MCSpecifierExpr(Expr, S), RelaxHint(Hint) {}
 
 public:
   static const LoongArchMCExpr *create(const MCExpr *Expr, uint16_t S,
                                        MCContext &Ctx, bool Hint = false);
 
-  Specifier getSpecifier() const { return specifier; }
-  const MCExpr *getSubExpr() const { return Expr; }
   bool getRelaxHint() const { return RelaxHint; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-
-  static StringRef getSpecifierName(uint16_t S);
-  static Specifier parseSpecifier(StringRef name);
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/M68k/M68kISelLowering.cpp b/external/llvm-project/llvm/lib/Target/M68k/M68kISelLowering.cpp
index 9d3ab606ab8c..594ea9f48c20 100644
--- a/external/llvm-project/llvm/lib/Target/M68k/M68kISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/M68k/M68kISelLowering.cpp
@@ -19,7 +19,7 @@
 #include "M68kSubtarget.h"
 #include "M68kTargetMachine.h"
 #include "M68kTargetObjectFile.h"
-#include "MCTargetDesc/M68kMCExpr.h"
+#include "MCTargetDesc/M68kMCAsmInfo.h"
 
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/CallingConvLower.h"
@@ -2833,7 +2833,7 @@ unsigned M68kTargetLowering::getJumpTableEncoding() const {
 const MCExpr *M68kTargetLowering::LowerCustomJumpTableEntry(
     const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
     unsigned uid, MCContext &Ctx) const {
-  return MCSymbolRefExpr::create(MBB->getSymbol(), M68kMCExpr::VK_GOTOFF, Ctx);
+  return MCSymbolRefExpr::create(MBB->getSymbol(), M68k::S_GOTOFF, Ctx);
 }
 
 SDValue M68kTargetLowering::getPICJumpTableRelocBase(SDValue Table,
diff --git a/external/llvm-project/llvm/lib/Target/M68k/M68kMCInstLower.cpp b/external/llvm-project/llvm/lib/Target/M68k/M68kMCInstLower.cpp
index 8698fc0de471..301112c41efb 100644
--- a/external/llvm-project/llvm/lib/Target/M68k/M68kMCInstLower.cpp
+++ b/external/llvm-project/llvm/lib/Target/M68k/M68kMCInstLower.cpp
@@ -18,7 +18,7 @@
 #include "M68kInstrInfo.h"
 
 #include "MCTargetDesc/M68kBaseInfo.h"
-#include "MCTargetDesc/M68kMCExpr.h"
+#include "MCTargetDesc/M68kMCAsmInfo.h"
 
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -76,7 +76,7 @@ MCOperand M68kMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   // FIXME We would like an efficient form for this, so we don't have to do a
   // lot of extra uniquing. This fixme is originally from X86
   const MCExpr *Expr = nullptr;
-  M68kMCExpr::Specifier RefKind = M68kMCExpr::VK_None;
+  M68k::Specifier RefKind = M68k::S_None;
 
   switch (MO.getTargetFlags()) {
   default:
@@ -86,31 +86,31 @@ MCOperand M68kMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   case M68kII::MO_PC_RELATIVE_ADDRESS:
     break;
   case M68kII::MO_GOTPCREL:
-    RefKind = M68kMCExpr::VK_GOTPCREL;
+    RefKind = M68k::S_GOTPCREL;
     break;
   case M68kII::MO_GOT:
-    RefKind = M68kMCExpr::VK_GOT;
+    RefKind = M68k::S_GOT;
     break;
   case M68kII::MO_GOTOFF:
-    RefKind = M68kMCExpr::VK_GOTOFF;
+    RefKind = M68k::S_GOTOFF;
     break;
   case M68kII::MO_PLT:
-    RefKind = M68kMCExpr::VK_PLT;
+    RefKind = M68k::S_PLT;
     break;
   case M68kII::MO_TLSGD:
-    RefKind = M68kMCExpr::VK_TLSGD;
+    RefKind = M68k::S_TLSGD;
     break;
   case M68kII::MO_TLSLD:
-    RefKind = M68kMCExpr::VK_TLSLD;
+    RefKind = M68k::S_TLSLD;
     break;
   case M68kII::MO_TLSLDM:
-    RefKind = M68kMCExpr::VK_TLSLDM;
+    RefKind = M68k::S_TLSLDM;
     break;
   case M68kII::MO_TLSIE:
-    RefKind = M68kMCExpr::VK_GOTTPOFF;
+    RefKind = M68k::S_GOTTPOFF;
     break;
   case M68kII::MO_TLSLE:
-    RefKind = M68kMCExpr::VK_TPOFF;
+    RefKind = M68k::S_TPOFF;
     break;
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/CMakeLists.txt b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/CMakeLists.txt
index 0146e21acf36..1127b3b547f1 100644
--- a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/CMakeLists.txt
@@ -4,7 +4,6 @@ add_llvm_component_library(LLVMM68kDesc
   M68kInstPrinter.cpp
   M68kMCAsmInfo.cpp
   M68kMCCodeEmitter.cpp
-  M68kMCExpr.cpp
   M68kMCTargetDesc.cpp
 
   LINK_COMPONENTS
diff --git a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
index 1a61325008aa..03416df639cf 100644
--- a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/M68kFixupKinds.h"
-#include "MCTargetDesc/M68kMCExpr.h"
+#include "MCTargetDesc/M68kMCAsmInfo.h"
 #include "MCTargetDesc/M68kMCTargetDesc.h"
 
 #include "llvm/BinaryFormat/ELF.h"
@@ -45,7 +45,7 @@ M68kELFObjectWriter::~M68kELFObjectWriter() {}
 
 enum M68kRelType { RT_32, RT_16, RT_8 };
 
-static M68kRelType getType(unsigned Kind, M68kMCExpr::Specifier &Modifier,
+static M68kRelType getType(unsigned Kind, M68k::Specifier &Modifier,
                            bool &IsPCRel) {
   switch (Kind) {
   case FK_Data_4:
@@ -64,15 +64,15 @@ static M68kRelType getType(unsigned Kind, M68kMCExpr::Specifier &Modifier,
 unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
                                            const MCValue &Target,
                                            bool IsPCRel) const {
-  auto Specifier = M68kMCExpr::Specifier(Target.getSpecifier());
+  auto Specifier = M68k::Specifier(Target.getSpecifier());
   unsigned Kind = Fixup.getKind();
   M68kRelType Type = getType(Kind, Specifier, IsPCRel);
   switch (Specifier) {
-  case M68kMCExpr::VK_GOTTPOFF:
-  case M68kMCExpr::VK_TLSGD:
-  case M68kMCExpr::VK_TLSLD:
-  case M68kMCExpr::VK_TLSLDM:
-  case M68kMCExpr::VK_TPOFF:
+  case M68k::S_GOTTPOFF:
+  case M68k::S_TLSGD:
+  case M68k::S_TLSLD:
+  case M68k::S_TLSLDM:
+  case M68k::S_TPOFF:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
@@ -84,7 +84,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
   default:
     llvm_unreachable("Unimplemented");
 
-  case M68kMCExpr::VK_TLSGD:
+  case M68k::S_TLSGD:
     switch (Type) {
     case RT_32:
       return ELF::R_68K_TLS_GD32;
@@ -94,7 +94,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_68K_TLS_GD8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_TLSLDM:
+  case M68k::S_TLSLDM:
     switch (Type) {
     case RT_32:
       return ELF::R_68K_TLS_LDM32;
@@ -104,7 +104,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_68K_TLS_LDM8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_TLSLD:
+  case M68k::S_TLSLD:
     switch (Type) {
     case RT_32:
       return ELF::R_68K_TLS_LDO32;
@@ -114,7 +114,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_68K_TLS_LDO8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_GOTTPOFF:
+  case M68k::S_GOTTPOFF:
     switch (Type) {
     case RT_32:
       return ELF::R_68K_TLS_IE32;
@@ -124,7 +124,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_68K_TLS_IE8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_TPOFF:
+  case M68k::S_TPOFF:
     switch (Type) {
     case RT_32:
       return ELF::R_68K_TLS_LE32;
@@ -134,7 +134,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_68K_TLS_LE8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_None:
+  case M68k::S_None:
     switch (Type) {
     case RT_32:
       return IsPCRel ? ELF::R_68K_PC32 : ELF::R_68K_32;
@@ -144,7 +144,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return IsPCRel ? ELF::R_68K_PC8 : ELF::R_68K_8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_GOTPCREL:
+  case M68k::S_GOTPCREL:
     switch (Type) {
     case RT_32:
       return ELF::R_68K_GOTPCREL32;
@@ -154,7 +154,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_68K_GOTPCREL8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_GOTOFF:
+  case M68k::S_GOTOFF:
     assert(!IsPCRel);
     switch (Type) {
     case RT_32:
@@ -165,7 +165,7 @@ unsigned M68kELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_68K_GOTOFF8;
     }
     llvm_unreachable("Unrecognized size");
-  case M68kMCExpr::VK_PLT:
+  case M68k::S_PLT:
     switch (Type) {
     case RT_32:
       return ELF::R_68K_PLT32;
diff --git a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp
index 68ac15b57508..778d31280adc 100644
--- a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kInstPrinter.cpp
@@ -27,6 +27,7 @@
 #include "M68kBaseInfo.h"
 
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -68,7 +69,7 @@ void M68kInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
 }
 
 void M68kInstPrinter::printImmediate(const MCInst *MI, unsigned opNum,
@@ -78,7 +79,7 @@ void M68kInstPrinter::printImmediate(const MCInst *MI, unsigned opNum,
     O << '#' << MO.getImm();
   else if (MO.isExpr()) {
     O << '#';
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
   } else
     llvm_unreachable("Unknown immediate kind");
 }
@@ -144,7 +145,7 @@ void M68kInstPrinter::printDisp(const MCInst *MI, unsigned opNum,
     return;
   }
   assert(Op.isExpr() && "Unknown operand kind in printOperand");
-  Op.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *Op.getExpr());
 }
 
 // NOTE forcing (W,L) size available since M68020 only
@@ -153,7 +154,7 @@ void M68kInstPrinter::printAbsMem(const MCInst *MI, unsigned opNum,
   const MCOperand &MO = MI->getOperand(opNum);
 
   if (MO.isExpr()) {
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
     return;
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp
index ba1b0dc2bb09..b0a19309b50f 100644
--- a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "M68kMCAsmInfo.h"
-#include "MCTargetDesc/M68kMCExpr.h"
 
 #include "llvm/MC/MCExpr.h"
 #include "llvm/TargetParser/Triple.h"
@@ -20,14 +19,10 @@
 using namespace llvm;
 
 const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
-    {M68kMCExpr::VK_GOTOFF, "GOTOFF"},
-    {M68kMCExpr::VK_GOTPCREL, "GOTPCREL"},
-    {M68kMCExpr::VK_GOTTPOFF, "GOTTPOFF"},
-    {M68kMCExpr::VK_PLT, "PLT"},
-    {M68kMCExpr::VK_TLSGD, "TLSGD"},
-    {M68kMCExpr::VK_TLSLD, "TLSLD"},
-    {M68kMCExpr::VK_TLSLDM, "TLSLDM"},
-    {M68kMCExpr::VK_TPOFF, "TPOFF"},
+    {M68k::S_GOTOFF, "GOTOFF"},     {M68k::S_GOTPCREL, "GOTPCREL"},
+    {M68k::S_GOTTPOFF, "GOTTPOFF"}, {M68k::S_PLT, "PLT"},
+    {M68k::S_TLSGD, "TLSGD"},       {M68k::S_TLSLD, "TLSLD"},
+    {M68k::S_TLSLDM, "TLSLDM"},     {M68k::S_TPOFF, "TPOFF"},
 };
 
 void M68kELFMCAsmInfo::anchor() {}
diff --git a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.h
index 873264d88674..1ab36260cef1 100644
--- a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.h
@@ -26,6 +26,21 @@ class M68kELFMCAsmInfo : public MCAsmInfoELF {
   explicit M68kELFMCAsmInfo(const Triple &Triple);
 };
 
+namespace M68k {
+enum Specifier {
+  S_None,
+  S_GOT,
+  S_GOTOFF,
+  S_GOTPCREL,
+  S_GOTTPOFF,
+  S_PLT,
+  S_TLSGD,
+  S_TLSLD,
+  S_TLSLDM,
+  S_TPOFF,
+};
+}
+
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_M68K_MCTARGETDESC_M68KMCASMINFO_H
diff --git a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCExpr.cpp b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCExpr.cpp
deleted file mode 100644
index 03a174311a6e..000000000000
--- a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCExpr.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-//===- M68k specific MC expression classes ----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "M68kMCExpr.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCValue.h"
-
-using namespace llvm;
-
-const M68kMCExpr *M68kMCExpr::create(const MCExpr *Expr, Specifier S,
-                                     MCContext &Ctx) {
-  return new (Ctx) M68kMCExpr(Expr, S);
-}
-
-void M68kMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {}
-
-bool M68kMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                           const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-
-  Res.setSpecifier(specifier);
-  return !Res.getSubSym();
-}
-
-void M68kMCExpr::visitUsedExpr(MCStreamer &S) const { S.visitUsedExpr(*Expr); }
diff --git a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCExpr.h b/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCExpr.h
deleted file mode 100644
index d9b72da6318f..000000000000
--- a/external/llvm-project/llvm/lib/Target/M68k/MCTargetDesc/M68kMCExpr.h
+++ /dev/null
@@ -1,61 +0,0 @@
-//===- M68k specific MC expression classes ----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// The MCTargetExpr subclass describes a relocatable expression with a
-// M68k-specific relocation specifier.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_M68K_MCTARGETDESC_M68KMCEXPR_H
-#define LLVM_LIB_TARGET_M68K_MCTARGETDESC_M68KMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-
-namespace llvm {
-
-class M68kMCExpr : public MCTargetExpr {
-public:
-  enum Specifier {
-    VK_None,
-
-    VK_GOT = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_GOTOFF,
-    VK_GOTPCREL,
-    VK_GOTTPOFF,
-    VK_PLT,
-    VK_TLSGD,
-    VK_TLSLD,
-    VK_TLSLDM,
-    VK_TPOFF,
-  };
-
-private:
-  const MCExpr *Expr;
-  const Specifier specifier;
-
-protected:
-  explicit M68kMCExpr(const MCExpr *Expr, Specifier S)
-      : Expr(Expr), specifier(S) {}
-
-public:
-  static const M68kMCExpr *create(const MCExpr *, Specifier, MCContext &);
-
-  Specifier getSpecifier() const { return specifier; }
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-};
-} // namespace llvm
-
-#endif
diff --git a/external/llvm-project/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp b/external/llvm-project/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
index f925a1efc88f..80a1e85e4a5d 100644
--- a/external/llvm-project/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
@@ -48,7 +48,7 @@ void MSP430InstPrinter::printPCRelImmOperand(const MCInst *MI, unsigned OpNo,
     O << Imm;
   } else {
     assert(Op.isExpr() && "unknown pcrel immediate operand");
-    Op.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -62,7 +62,7 @@ void MSP430InstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
     O << '#';
-    Op.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -83,7 +83,7 @@ void MSP430InstPrinter::printSrcMemOperand(const MCInst *MI, unsigned OpNo,
     O << '&';
 
   if (Disp.isExpr())
-    Disp.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Disp.getExpr());
   else {
     assert(Disp.isImm() && "Expected immediate in displacement field");
     O << Disp.getImm();
diff --git a/external/llvm-project/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/external/llvm-project/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index 5589cea6e675..c2946de838d7 100644
--- a/external/llvm-project/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -148,108 +148,6 @@ MSP430TargetLowering::MSP430TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::VACOPY,           MVT::Other, Expand);
   setOperationAction(ISD::JumpTable,        MVT::i16,   Custom);
 
-  // EABI Libcalls - EABI Section 6.2
-  const struct {
-    const RTLIB::Libcall Op;
-    const char * const Name;
-    const ISD::CondCode Cond;
-  } LibraryCalls[] = {
-    // Floating point conversions - EABI Table 6
-    { RTLIB::FPROUND_F64_F32,   "__mspabi_cvtdf",   ISD::SETCC_INVALID },
-    { RTLIB::FPEXT_F32_F64,     "__mspabi_cvtfd",   ISD::SETCC_INVALID },
-    // The following is NOT implemented in libgcc
-    //{ RTLIB::FPTOSINT_F64_I16,  "__mspabi_fixdi", ISD::SETCC_INVALID },
-    { RTLIB::FPTOSINT_F64_I32,  "__mspabi_fixdli",  ISD::SETCC_INVALID },
-    { RTLIB::FPTOSINT_F64_I64,  "__mspabi_fixdlli", ISD::SETCC_INVALID },
-    // The following is NOT implemented in libgcc
-    //{ RTLIB::FPTOUINT_F64_I16,  "__mspabi_fixdu", ISD::SETCC_INVALID },
-    { RTLIB::FPTOUINT_F64_I32,  "__mspabi_fixdul",  ISD::SETCC_INVALID },
-    { RTLIB::FPTOUINT_F64_I64,  "__mspabi_fixdull", ISD::SETCC_INVALID },
-    // The following is NOT implemented in libgcc
-    //{ RTLIB::FPTOSINT_F32_I16,  "__mspabi_fixfi", ISD::SETCC_INVALID },
-    { RTLIB::FPTOSINT_F32_I32,  "__mspabi_fixfli",  ISD::SETCC_INVALID },
-    { RTLIB::FPTOSINT_F32_I64,  "__mspabi_fixflli", ISD::SETCC_INVALID },
-    // The following is NOT implemented in libgcc
-    //{ RTLIB::FPTOUINT_F32_I16,  "__mspabi_fixfu", ISD::SETCC_INVALID },
-    { RTLIB::FPTOUINT_F32_I32,  "__mspabi_fixful",  ISD::SETCC_INVALID },
-    { RTLIB::FPTOUINT_F32_I64,  "__mspabi_fixfull", ISD::SETCC_INVALID },
-    // TODO The following IS implemented in libgcc
-    //{ RTLIB::SINTTOFP_I16_F64,  "__mspabi_fltid", ISD::SETCC_INVALID },
-    { RTLIB::SINTTOFP_I32_F64,  "__mspabi_fltlid",  ISD::SETCC_INVALID },
-    // TODO The following IS implemented in libgcc but is not in the EABI
-    { RTLIB::SINTTOFP_I64_F64,  "__mspabi_fltllid", ISD::SETCC_INVALID },
-    // TODO The following IS implemented in libgcc
-    //{ RTLIB::UINTTOFP_I16_F64,  "__mspabi_fltud", ISD::SETCC_INVALID },
-    { RTLIB::UINTTOFP_I32_F64,  "__mspabi_fltuld",  ISD::SETCC_INVALID },
-    // The following IS implemented in libgcc but is not in the EABI
-    { RTLIB::UINTTOFP_I64_F64,  "__mspabi_fltulld", ISD::SETCC_INVALID },
-    // TODO The following IS implemented in libgcc
-    //{ RTLIB::SINTTOFP_I16_F32,  "__mspabi_fltif", ISD::SETCC_INVALID },
-    { RTLIB::SINTTOFP_I32_F32,  "__mspabi_fltlif",  ISD::SETCC_INVALID },
-    // TODO The following IS implemented in libgcc but is not in the EABI
-    { RTLIB::SINTTOFP_I64_F32,  "__mspabi_fltllif", ISD::SETCC_INVALID },
-    // TODO The following IS implemented in libgcc
-    //{ RTLIB::UINTTOFP_I16_F32,  "__mspabi_fltuf", ISD::SETCC_INVALID },
-    { RTLIB::UINTTOFP_I32_F32,  "__mspabi_fltulf",  ISD::SETCC_INVALID },
-    // The following IS implemented in libgcc but is not in the EABI
-    { RTLIB::UINTTOFP_I64_F32,  "__mspabi_fltullf", ISD::SETCC_INVALID },
-
-    // Floating point comparisons - EABI Table 7
-    { RTLIB::OEQ_F64, "__mspabi_cmpd", ISD::SETEQ },
-    { RTLIB::UNE_F64, "__mspabi_cmpd", ISD::SETNE },
-    { RTLIB::OGE_F64, "__mspabi_cmpd", ISD::SETGE },
-    { RTLIB::OLT_F64, "__mspabi_cmpd", ISD::SETLT },
-    { RTLIB::OLE_F64, "__mspabi_cmpd", ISD::SETLE },
-    { RTLIB::OGT_F64, "__mspabi_cmpd", ISD::SETGT },
-    { RTLIB::OEQ_F32, "__mspabi_cmpf", ISD::SETEQ },
-    { RTLIB::UNE_F32, "__mspabi_cmpf", ISD::SETNE },
-    { RTLIB::OGE_F32, "__mspabi_cmpf", ISD::SETGE },
-    { RTLIB::OLT_F32, "__mspabi_cmpf", ISD::SETLT },
-    { RTLIB::OLE_F32, "__mspabi_cmpf", ISD::SETLE },
-    { RTLIB::OGT_F32, "__mspabi_cmpf", ISD::SETGT },
-
-    // Floating point arithmetic - EABI Table 8
-    { RTLIB::ADD_F64,  "__mspabi_addd", ISD::SETCC_INVALID },
-    { RTLIB::ADD_F32,  "__mspabi_addf", ISD::SETCC_INVALID },
-    { RTLIB::DIV_F64,  "__mspabi_divd", ISD::SETCC_INVALID },
-    { RTLIB::DIV_F32,  "__mspabi_divf", ISD::SETCC_INVALID },
-    { RTLIB::MUL_F64,  "__mspabi_mpyd", ISD::SETCC_INVALID },
-    { RTLIB::MUL_F32,  "__mspabi_mpyf", ISD::SETCC_INVALID },
-    { RTLIB::SUB_F64,  "__mspabi_subd", ISD::SETCC_INVALID },
-    { RTLIB::SUB_F32,  "__mspabi_subf", ISD::SETCC_INVALID },
-    // The following are NOT implemented in libgcc
-    // { RTLIB::NEG_F64,  "__mspabi_negd", ISD::SETCC_INVALID },
-    // { RTLIB::NEG_F32,  "__mspabi_negf", ISD::SETCC_INVALID },
-
-    // Universal Integer Operations - EABI Table 9
-    { RTLIB::SDIV_I16,   "__mspabi_divi", ISD::SETCC_INVALID },
-    { RTLIB::SDIV_I32,   "__mspabi_divli", ISD::SETCC_INVALID },
-    { RTLIB::SDIV_I64,   "__mspabi_divlli", ISD::SETCC_INVALID },
-    { RTLIB::UDIV_I16,   "__mspabi_divu", ISD::SETCC_INVALID },
-    { RTLIB::UDIV_I32,   "__mspabi_divul", ISD::SETCC_INVALID },
-    { RTLIB::UDIV_I64,   "__mspabi_divull", ISD::SETCC_INVALID },
-    { RTLIB::SREM_I16,   "__mspabi_remi", ISD::SETCC_INVALID },
-    { RTLIB::SREM_I32,   "__mspabi_remli", ISD::SETCC_INVALID },
-    { RTLIB::SREM_I64,   "__mspabi_remlli", ISD::SETCC_INVALID },
-    { RTLIB::UREM_I16,   "__mspabi_remu", ISD::SETCC_INVALID },
-    { RTLIB::UREM_I32,   "__mspabi_remul", ISD::SETCC_INVALID },
-    { RTLIB::UREM_I64,   "__mspabi_remull", ISD::SETCC_INVALID },
-
-    // Bitwise Operations - EABI Table 10
-    // TODO: __mspabi_[srli/srai/slli] ARE implemented in libgcc
-    { RTLIB::SRL_I32,    "__mspabi_srll", ISD::SETCC_INVALID },
-    { RTLIB::SRA_I32,    "__mspabi_sral", ISD::SETCC_INVALID },
-    { RTLIB::SHL_I32,    "__mspabi_slll", ISD::SETCC_INVALID },
-    // __mspabi_[srlll/srall/sllll/rlli/rlll] are NOT implemented in libgcc
-
-  };
-
-  for (const auto &LC : LibraryCalls) {
-    setLibcallName(LC.Op, LC.Name);
-    if (LC.Cond != ISD::SETCC_INVALID)
-      setCmpLibcallCC(LC.Op, LC.Cond);
-  }
-
   if (STI.hasHWMult16()) {
     const struct {
       const RTLIB::Libcall Op;
@@ -313,23 +211,6 @@ MSP430TargetLowering::MSP430TargetLowering(const TargetMachine &TM,
     setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::MSP430_BUILTIN);
   }
 
-  // Several of the runtime library functions use a special calling conv
-  setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::ADD_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::SUB_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::MUL_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::DIV_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::OEQ_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::UNE_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::OGE_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::OLT_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::OLE_F64, CallingConv::MSP430_BUILTIN);
-  setLibcallCallingConv(RTLIB::OGT_F64, CallingConv::MSP430_BUILTIN);
-  // TODO: __mspabi_srall, __mspabi_srlll, __mspabi_sllll
-
   setMinFunctionAlignment(Align(2));
   setPrefFunctionAlignment(Align(2));
   setMaxAtomicSizeInBitsSupported(0);
diff --git a/external/llvm-project/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/external/llvm-project/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index d1b94410c601..7ea7c58f1a51 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -9,7 +9,7 @@
 #include "MCTargetDesc/MipsABIFlagsSection.h"
 #include "MCTargetDesc/MipsABIInfo.h"
 #include "MCTargetDesc/MipsBaseInfo.h"
-#include "MCTargetDesc/MipsMCExpr.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "MCTargetDesc/MipsTargetStreamer.h"
 #include "TargetInfo/MipsTargetInfo.h"
@@ -1305,7 +1305,7 @@ class MipsOperand : public MCParsedAsmOperand {
       return false;
     if (!getMemBase()->isGPRAsmReg())
       return false;
-    if (isa<MCTargetExpr>(getMemOff()) ||
+    if (isa<MCSpecifierExpr>(getMemOff()) ||
         (isConstantMemOff() &&
          isShiftedInt<Bits, ShiftAmount>(getConstantMemOff())))
       return true;
@@ -1320,7 +1320,7 @@ class MipsOperand : public MCParsedAsmOperand {
     if (!getMemBase()->isGPRAsmReg())
       return false;
     const unsigned PtrBits = AsmParser.getABI().ArePtrs64bit() ? 64 : 32;
-    if (isa<MCTargetExpr>(getMemOff()) ||
+    if (isa<MCSpecifierExpr>(getMemOff()) ||
         (isConstantMemOff() && isIntN(PtrBits, getConstantMemOff())))
       return true;
     MCValue Res;
@@ -1783,8 +1783,10 @@ static bool isEvaluated(const MCExpr *Expr) {
   }
   case MCExpr::Unary:
     return isEvaluated(cast<MCUnaryExpr>(Expr)->getSubExpr());
-  case MCExpr::Target:
+  case MCExpr::Specifier:
     return true;
+  case MCExpr::Target:
+    llvm_unreachable("unused by this backend");
   }
   return false;
 }
@@ -2962,10 +2964,10 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
     if ((DstReg == Mips::T9 || DstReg == Mips::T9_64) && !UseSrcReg &&
         Res.getConstant() == 0 && !IsLocalSym) {
       if (UseXGOT) {
-        const MCExpr *CallHiExpr = MipsMCExpr::create(MipsMCExpr::MEK_CALL_HI16,
-                                                      SymExpr, getContext());
-        const MCExpr *CallLoExpr = MipsMCExpr::create(MipsMCExpr::MEK_CALL_LO16,
-                                                      SymExpr, getContext());
+        const MCExpr *CallHiExpr =
+            MipsMCExpr::create(Mips::S_CALL_HI16, SymExpr, getContext());
+        const MCExpr *CallLoExpr =
+            MipsMCExpr::create(Mips::S_CALL_LO16, SymExpr, getContext());
         TOut.emitRX(Mips::LUi, DstReg, MCOperand::createExpr(CallHiExpr), IDLoc,
                     STI);
         TOut.emitRRR(IsPtr64 ? Mips::DADDu : Mips::ADDu, DstReg, DstReg, GPReg,
@@ -2974,7 +2976,7 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
                      MCOperand::createExpr(CallLoExpr), IDLoc, STI);
       } else {
         const MCExpr *CallExpr =
-            MipsMCExpr::create(MipsMCExpr::MEK_GOT_CALL, SymExpr, getContext());
+            MipsMCExpr::create(Mips::S_GOT_CALL, SymExpr, getContext());
         TOut.emitRRX(IsPtr64 ? Mips::LD : Mips::LW, DstReg, GPReg,
                      MCOperand::createExpr(CallExpr), IDLoc, STI);
       }
@@ -3007,9 +3009,9 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
       // this happens then the last instruction must use $rd as the result
       // register.
       const MCExpr *CallHiExpr =
-          MipsMCExpr::create(MipsMCExpr::MEK_GOT_HI16, SymExpr, getContext());
-      const MCExpr *CallLoExpr = MipsMCExpr::create(
-          Res.getAddSym(), MipsMCExpr::MEK_GOT_LO16, getContext());
+          MipsMCExpr::create(Mips::S_GOT_HI16, SymExpr, getContext());
+      const MCExpr *CallLoExpr =
+          MipsMCExpr::create(Res.getAddSym(), Mips::S_GOT_LO16, getContext());
 
       TOut.emitRX(Mips::LUi, TmpReg, MCOperand::createExpr(CallHiExpr), IDLoc,
                   STI);
@@ -3030,7 +3032,7 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
       return false;
     }
 
-    const MipsMCExpr *GotExpr = nullptr;
+    const MCSpecifierExpr *GotExpr = nullptr;
     const MCExpr *LoExpr = nullptr;
     if (ABI.IsN32() || ABI.IsN64()) {
       // The remaining cases are:
@@ -3040,8 +3042,8 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
       // The daddiu's marked with a '>' may be omitted if they are redundant. If
       // this happens then the last instruction must use $rd as the result
       // register.
-      GotExpr = MipsMCExpr::create(Res.getAddSym(), MipsMCExpr::MEK_GOT_DISP,
-                                   getContext());
+      GotExpr =
+          MipsMCExpr::create(Res.getAddSym(), Mips::S_GOT_DISP, getContext());
       if (Res.getConstant() != 0) {
         // Symbols fully resolve with just the %got_disp(symbol) but we
         // must still account for any offset to the symbol for
@@ -3068,15 +3070,14 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
       // this happens then the last instruction must use $rd as the result
       // register.
       if (IsLocalSym) {
-        GotExpr =
-            MipsMCExpr::create(MipsMCExpr::MEK_GOT, SymExpr, getContext());
-        LoExpr = MipsMCExpr::create(MipsMCExpr::MEK_LO, SymExpr, getContext());
+        GotExpr = MipsMCExpr::create(Mips::S_GOT, SymExpr, getContext());
+        LoExpr = MipsMCExpr::create(Mips::S_LO, SymExpr, getContext());
       } else {
         // External symbols fully resolve the symbol with just the %got(symbol)
         // but we must still account for any offset to the symbol for
         // expressions like symbol+8.
-        GotExpr = MipsMCExpr::create(Res.getAddSym(), MipsMCExpr::MEK_GOT,
-                                     getContext());
+        GotExpr =
+            MipsMCExpr::create(Res.getAddSym(), Mips::S_GOT, getContext());
         if (Res.getConstant() != 0)
           LoExpr = MCConstantExpr::create(Res.getConstant(), getContext());
       }
@@ -3096,10 +3097,8 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
     return false;
   }
 
-  const MipsMCExpr *HiExpr =
-      MipsMCExpr::create(MipsMCExpr::MEK_HI, SymExpr, getContext());
-  const MipsMCExpr *LoExpr =
-      MipsMCExpr::create(MipsMCExpr::MEK_LO, SymExpr, getContext());
+  const auto *HiExpr = MipsMCExpr::create(Mips::S_HI, SymExpr, getContext());
+  const auto *LoExpr = MipsMCExpr::create(Mips::S_LO, SymExpr, getContext());
 
   // This is the 64-bit symbol address expansion.
   if (ABI.ArePtrs64bit() && isGP64bit()) {
@@ -3110,10 +3109,10 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
     // If it is not available we exit if the destination is the same as the
     // source register.
 
-    const MipsMCExpr *HighestExpr =
-        MipsMCExpr::create(MipsMCExpr::MEK_HIGHEST, SymExpr, getContext());
-    const MipsMCExpr *HigherExpr =
-        MipsMCExpr::create(MipsMCExpr::MEK_HIGHER, SymExpr, getContext());
+    const auto *HighestExpr =
+        MipsMCExpr::create(Mips::S_HIGHEST, SymExpr, getContext());
+    const auto *HigherExpr =
+        MipsMCExpr::create(Mips::S_HIGHER, SymExpr, getContext());
 
     bool RdRegIsRsReg =
         UseSrcReg &&
@@ -3311,8 +3310,7 @@ bool MipsAsmParser::emitPartialAddress(MipsTargetStreamer &TOut, SMLoc IDLoc,
 
   if(IsPicEnabled) {
     const MCExpr *GotSym = MCSymbolRefExpr::create(Sym, getContext());
-    const MipsMCExpr *GotExpr =
-        MipsMCExpr::create(MipsMCExpr::MEK_GOT, GotSym, getContext());
+    const auto *GotExpr = MipsMCExpr::create(Mips::S_GOT, GotSym, getContext());
 
     if(isABI_O32() || isABI_N32()) {
       TOut.emitRRX(Mips::LW, ATReg, GPReg, MCOperand::createExpr(GotExpr),
@@ -3323,8 +3321,7 @@ bool MipsAsmParser::emitPartialAddress(MipsTargetStreamer &TOut, SMLoc IDLoc,
     }
   } else { //!IsPicEnabled
     const MCExpr *HiSym = MCSymbolRefExpr::create(Sym, getContext());
-    const MipsMCExpr *HiExpr =
-        MipsMCExpr::create(MipsMCExpr::MEK_HI, HiSym, getContext());
+    const auto *HiExpr = MipsMCExpr::create(Mips::S_HI, HiSym, getContext());
 
     // FIXME: This is technically correct but gives a different result to gas,
     // but gas is incomplete there (it has a fixme noting it doesn't work with
@@ -3336,11 +3333,11 @@ bool MipsAsmParser::emitPartialAddress(MipsTargetStreamer &TOut, SMLoc IDLoc,
       TOut.emitRX(Mips::LUi, ATReg, MCOperand::createExpr(HiExpr), IDLoc, STI);
     } else { //isABI_N64()
       const MCExpr *HighestSym = MCSymbolRefExpr::create(Sym, getContext());
-      const MipsMCExpr *HighestExpr =
-          MipsMCExpr::create(MipsMCExpr::MEK_HIGHEST, HighestSym, getContext());
+      const auto *HighestExpr =
+          MipsMCExpr::create(Mips::S_HIGHEST, HighestSym, getContext());
       const MCExpr *HigherSym = MCSymbolRefExpr::create(Sym, getContext());
-      const MipsMCExpr *HigherExpr =
-          MipsMCExpr::create(MipsMCExpr::MEK_HIGHER, HigherSym, getContext());
+      const auto *HigherExpr =
+          MipsMCExpr::create(Mips::S_HIGHER, HigherSym, getContext());
 
       TOut.emitRX(Mips::LUi, ATReg, MCOperand::createExpr(HighestExpr), IDLoc,
                   STI);
@@ -3427,8 +3424,7 @@ bool MipsAsmParser::expandLoadSingleImmToFPR(MCInst &Inst, SMLoc IDLoc,
 
   MCSymbol *Sym = getContext().createTempSymbol();
   const MCExpr *LoSym = MCSymbolRefExpr::create(Sym, getContext());
-  const MipsMCExpr *LoExpr =
-      MipsMCExpr::create(MipsMCExpr::MEK_LO, LoSym, getContext());
+  const auto *LoExpr = MipsMCExpr::create(Mips::S_LO, LoSym, getContext());
 
   getStreamer().switchSection(ReadOnlySection);
   getStreamer().emitLabel(Sym, IDLoc);
@@ -3478,8 +3474,7 @@ bool MipsAsmParser::expandLoadDoubleImmToGPR(MCInst &Inst, SMLoc IDLoc,
 
   MCSymbol *Sym = getContext().createTempSymbol();
   const MCExpr *LoSym = MCSymbolRefExpr::create(Sym, getContext());
-  const MipsMCExpr *LoExpr =
-      MipsMCExpr::create(MipsMCExpr::MEK_LO, LoSym, getContext());
+  const auto *LoExpr = MipsMCExpr::create(Mips::S_LO, LoSym, getContext());
 
   getStreamer().switchSection(ReadOnlySection);
   getStreamer().emitLabel(Sym, IDLoc);
@@ -3559,8 +3554,7 @@ bool MipsAsmParser::expandLoadDoubleImmToFPR(MCInst &Inst, bool Is64FPU,
 
   MCSymbol *Sym = getContext().createTempSymbol();
   const MCExpr *LoSym = MCSymbolRefExpr::create(Sym, getContext());
-  const MipsMCExpr *LoExpr =
-      MipsMCExpr::create(MipsMCExpr::MEK_LO, LoSym, getContext());
+  const auto *LoExpr = MipsMCExpr::create(Mips::S_LO, LoSym, getContext());
 
   getStreamer().switchSection(ReadOnlySection);
   getStreamer().emitLabel(Sym, IDLoc);
@@ -3783,15 +3777,15 @@ void MipsAsmParser::expandMem16Inst(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out,
       //                  sw  $8,  %lo(sym)($at)
       const MCExpr *OffExpr = OffsetOp.getExpr();
       MCOperand LoOperand = MCOperand::createExpr(
-          MipsMCExpr::create(MipsMCExpr::MEK_LO, OffExpr, getContext()));
+          MipsMCExpr::create(Mips::S_LO, OffExpr, getContext()));
       MCOperand HiOperand = MCOperand::createExpr(
-          MipsMCExpr::create(MipsMCExpr::MEK_HI, OffExpr, getContext()));
+          MipsMCExpr::create(Mips::S_HI, OffExpr, getContext()));
 
       if (ABI.IsN64()) {
         MCOperand HighestOperand = MCOperand::createExpr(
-            MipsMCExpr::create(MipsMCExpr::MEK_HIGHEST, OffExpr, getContext()));
+            MipsMCExpr::create(Mips::S_HIGHEST, OffExpr, getContext()));
         MCOperand HigherOperand = MCOperand::createExpr(
-            MipsMCExpr::create(MipsMCExpr::MEK_HIGHER, OffExpr, getContext()));
+            MipsMCExpr::create(Mips::S_HIGHER, OffExpr, getContext()));
 
         TOut.emitRX(Mips::LUi, TmpReg, HighestOperand, IDLoc, STI);
         TOut.emitRRX(Mips::DADDiu, TmpReg, TmpReg, HigherOperand, IDLoc, STI);
@@ -6352,44 +6346,44 @@ MCRegister MipsAsmParser::getReg(int RC, int RegNo) {
 // e.g. "%lo foo", "(%lo(foo))", "%lo(foo)+1".
 const MCExpr *MipsAsmParser::parseRelocExpr() {
   auto getOp = [](StringRef Op) {
-    return StringSwitch<MipsMCExpr::Specifier>(Op)
-        .Case("call16", MipsMCExpr::MEK_GOT_CALL)
-        .Case("call_hi", MipsMCExpr::MEK_CALL_HI16)
-        .Case("call_lo", MipsMCExpr::MEK_CALL_LO16)
-        .Case("dtprel_hi", MipsMCExpr::MEK_DTPREL_HI)
-        .Case("dtprel_lo", MipsMCExpr::MEK_DTPREL_LO)
-        .Case("got", MipsMCExpr::MEK_GOT)
-        .Case("got_disp", MipsMCExpr::MEK_GOT_DISP)
-        .Case("got_hi", MipsMCExpr::MEK_GOT_HI16)
-        .Case("got_lo", MipsMCExpr::MEK_GOT_LO16)
-        .Case("got_ofst", MipsMCExpr::MEK_GOT_OFST)
-        .Case("got_page", MipsMCExpr::MEK_GOT_PAGE)
-        .Case("gottprel", MipsMCExpr::MEK_GOTTPREL)
-        .Case("gp_rel", MipsMCExpr::MEK_GPREL)
-        .Case("hi", MipsMCExpr::MEK_HI)
-        .Case("higher", MipsMCExpr::MEK_HIGHER)
-        .Case("highest", MipsMCExpr::MEK_HIGHEST)
-        .Case("lo", MipsMCExpr::MEK_LO)
-        .Case("neg", MipsMCExpr::MEK_NEG)
-        .Case("pcrel_hi", MipsMCExpr::MEK_PCREL_HI16)
-        .Case("pcrel_lo", MipsMCExpr::MEK_PCREL_LO16)
-        .Case("tlsgd", MipsMCExpr::MEK_TLSGD)
-        .Case("tlsldm", MipsMCExpr::MEK_TLSLDM)
-        .Case("tprel_hi", MipsMCExpr::MEK_TPREL_HI)
-        .Case("tprel_lo", MipsMCExpr::MEK_TPREL_LO)
-        .Default(MipsMCExpr::MEK_None);
+    return StringSwitch<Mips::Specifier>(Op)
+        .Case("call16", Mips::S_GOT_CALL)
+        .Case("call_hi", Mips::S_CALL_HI16)
+        .Case("call_lo", Mips::S_CALL_LO16)
+        .Case("dtprel_hi", Mips::S_DTPREL_HI)
+        .Case("dtprel_lo", Mips::S_DTPREL_LO)
+        .Case("got", Mips::S_GOT)
+        .Case("got_disp", Mips::S_GOT_DISP)
+        .Case("got_hi", Mips::S_GOT_HI16)
+        .Case("got_lo", Mips::S_GOT_LO16)
+        .Case("got_ofst", Mips::S_GOT_OFST)
+        .Case("got_page", Mips::S_GOT_PAGE)
+        .Case("gottprel", Mips::S_GOTTPREL)
+        .Case("gp_rel", Mips::S_GPREL)
+        .Case("hi", Mips::S_HI)
+        .Case("higher", Mips::S_HIGHER)
+        .Case("highest", Mips::S_HIGHEST)
+        .Case("lo", Mips::S_LO)
+        .Case("neg", Mips::S_NEG)
+        .Case("pcrel_hi", Mips::S_PCREL_HI16)
+        .Case("pcrel_lo", Mips::S_PCREL_LO16)
+        .Case("tlsgd", Mips::S_TLSGD)
+        .Case("tlsldm", Mips::S_TLSLDM)
+        .Case("tprel_hi", Mips::S_TPREL_HI)
+        .Case("tprel_lo", Mips::S_TPREL_LO)
+        .Default(Mips::S_None);
   };
 
   MCAsmParser &Parser = getParser();
   StringRef Name;
   const MCExpr *Res = nullptr;
-  SmallVector<MipsMCExpr::Specifier, 0> Ops;
+  SmallVector<Mips::Specifier, 0> Ops;
   while (parseOptionalToken(AsmToken::Percent)) {
     if (Parser.parseIdentifier(Name) ||
         Parser.parseToken(AsmToken::LParen, "expected '('"))
       return nullptr;
     auto Op = getOp(Name);
-    if (Op == MipsMCExpr::MEK_None) {
+    if (Op == Mips::S_None) {
       Error(Parser.getTok().getLoc(), "invalid relocation operator");
       return nullptr;
     }
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index d5e19ccaa168..58aa374e5302 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/MipsFixupKinds.h"
-#include "MCTargetDesc/MipsMCExpr.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -157,14 +157,14 @@ unsigned MipsELFObjectWriter::getRelocType(const MCFixup &Fixup,
   // Determine the type of the relocation.
   unsigned Kind = Fixup.getTargetKind();
   switch (Target.getSpecifier()) {
-  case MipsMCExpr::MEK_DTPREL:
-  case MipsMCExpr::MEK_DTPREL_HI:
-  case MipsMCExpr::MEK_DTPREL_LO:
-  case MipsMCExpr::MEK_TLSLDM:
-  case MipsMCExpr::MEK_TLSGD:
-  case MipsMCExpr::MEK_GOTTPREL:
-  case MipsMCExpr::MEK_TPREL_HI:
-  case MipsMCExpr::MEK_TPREL_LO:
+  case Mips::S_DTPREL:
+  case Mips::S_DTPREL_HI:
+  case Mips::S_DTPREL_LO:
+  case Mips::S_TLSLDM:
+  case Mips::S_TLSGD:
+  case Mips::S_GOTTPREL:
+  case Mips::S_TPREL_HI:
+  case Mips::S_TPREL_LO:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
index d743f00da273..f67356c105a4 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
@@ -13,6 +13,7 @@
 #include "MipsInstPrinter.h"
 #include "Mips.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -138,7 +139,7 @@ void MipsInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(Op.isExpr() && "unknown operand kind in printOperand");
-  Op.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *Op.getExpr());
 }
 
 void MipsInstPrinter::printJumpOperand(const MCInst *MI, unsigned OpNo,
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
index 11df6fecaf37..b64f86f38297 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
@@ -12,6 +12,7 @@
 
 #include "MipsMCAsmInfo.h"
 #include "MipsABIInfo.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
@@ -57,3 +58,157 @@ MipsCOFFMCAsmInfo::MipsCOFFMCAsmInfo() {
   PrivateLabelPrefix = ".L";
   AllowAtInName = true;
 }
+
+static void printImpl(const MCAsmInfo &MAI, raw_ostream &OS,
+                      const MCSpecifierExpr &Expr) {
+  int64_t AbsVal;
+
+  switch (Expr.getSpecifier()) {
+  case Mips::S_None:
+  case Mips::S_Special:
+    llvm_unreachable("Mips::S_None and MEK_Special are invalid");
+    break;
+  case Mips::S_DTPREL:
+    // Mips::S_DTPREL is used for marking TLS DIEExpr only
+    // and contains a regular sub-expression.
+    MAI.printExpr(OS, *Expr.getSubExpr());
+    return;
+  case Mips::S_CALL_HI16:
+    OS << "%call_hi";
+    break;
+  case Mips::S_CALL_LO16:
+    OS << "%call_lo";
+    break;
+  case Mips::S_DTPREL_HI:
+    OS << "%dtprel_hi";
+    break;
+  case Mips::S_DTPREL_LO:
+    OS << "%dtprel_lo";
+    break;
+  case Mips::S_GOT:
+    OS << "%got";
+    break;
+  case Mips::S_GOTTPREL:
+    OS << "%gottprel";
+    break;
+  case Mips::S_GOT_CALL:
+    OS << "%call16";
+    break;
+  case Mips::S_GOT_DISP:
+    OS << "%got_disp";
+    break;
+  case Mips::S_GOT_HI16:
+    OS << "%got_hi";
+    break;
+  case Mips::S_GOT_LO16:
+    OS << "%got_lo";
+    break;
+  case Mips::S_GOT_PAGE:
+    OS << "%got_page";
+    break;
+  case Mips::S_GOT_OFST:
+    OS << "%got_ofst";
+    break;
+  case Mips::S_GPREL:
+    OS << "%gp_rel";
+    break;
+  case Mips::S_HI:
+    OS << "%hi";
+    break;
+  case Mips::S_HIGHER:
+    OS << "%higher";
+    break;
+  case Mips::S_HIGHEST:
+    OS << "%highest";
+    break;
+  case Mips::S_LO:
+    OS << "%lo";
+    break;
+  case Mips::S_NEG:
+    OS << "%neg";
+    break;
+  case Mips::S_PCREL_HI16:
+    OS << "%pcrel_hi";
+    break;
+  case Mips::S_PCREL_LO16:
+    OS << "%pcrel_lo";
+    break;
+  case Mips::S_TLSGD:
+    OS << "%tlsgd";
+    break;
+  case Mips::S_TLSLDM:
+    OS << "%tlsldm";
+    break;
+  case Mips::S_TPREL_HI:
+    OS << "%tprel_hi";
+    break;
+  case Mips::S_TPREL_LO:
+    OS << "%tprel_lo";
+    break;
+  }
+
+  OS << '(';
+  if (Expr.evaluateAsAbsolute(AbsVal))
+    OS << AbsVal;
+  else
+    MAI.printExpr(OS, *Expr.getSubExpr());
+  OS << ')';
+}
+
+bool Mips::isGpOff(const MCSpecifierExpr &E) {
+  if (E.getSpecifier() == Mips::S_HI || E.getSpecifier() == Mips::S_LO) {
+    if (const auto *S1 = dyn_cast<const MCSpecifierExpr>(E.getSubExpr())) {
+      if (const auto *S2 = dyn_cast<const MCSpecifierExpr>(S1->getSubExpr())) {
+        if (S1->getSpecifier() == Mips::S_NEG &&
+            S2->getSpecifier() == Mips::S_GPREL) {
+          // S = E.getSpecifier();
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+static bool evaluate(const MCSpecifierExpr &Expr, MCValue &Res,
+                     const MCAssembler *Asm) {
+  // Look for the %hi(%neg(%gp_rel(X))) and %lo(%neg(%gp_rel(X)))
+  // special cases.
+  if (Mips::isGpOff(Expr)) {
+    const MCExpr *SubExpr =
+        cast<MCSpecifierExpr>(
+            cast<MCSpecifierExpr>(Expr.getSubExpr())->getSubExpr())
+            ->getSubExpr();
+    if (!SubExpr->evaluateAsRelocatable(Res, Asm))
+      return false;
+
+    Res.setSpecifier(Mips::S_Special);
+    return true;
+  }
+
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(Res, Asm))
+    return false;
+  Res.setSpecifier(Expr.getSpecifier());
+  return !Res.getSubSym();
+}
+
+void MipsELFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                          const MCSpecifierExpr &Expr) const {
+  printImpl(*this, OS, Expr);
+}
+
+bool MipsELFMCAsmInfo::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr,
+                                                 MCValue &Res,
+                                                 const MCAssembler *Asm) const {
+  return evaluate(Expr, Res, Asm);
+}
+
+void MipsCOFFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                           const MCSpecifierExpr &Expr) const {
+  printImpl(*this, OS, Expr);
+}
+
+bool MipsCOFFMCAsmInfo::evaluateAsRelocatableImpl(
+    const MCSpecifierExpr &Expr, MCValue &Res, const MCAssembler *Asm) const {
+  return evaluate(Expr, Res, Asm);
+}
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
index 3a2895a79f9c..0975116328fc 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
@@ -13,8 +13,10 @@
 #ifndef LLVM_LIB_TARGET_MIPS_MCTARGETDESC_MIPSMCASMINFO_H
 #define LLVM_LIB_TARGET_MIPS_MCTARGETDESC_MIPSMCASMINFO_H
 
+#include "MCTargetDesc/MipsMCExpr.h"
 #include "llvm/MC/MCAsmInfoCOFF.h"
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCFixup.h"
 
 namespace llvm {
 class Triple;
@@ -25,6 +27,10 @@ class MipsELFMCAsmInfo : public MCAsmInfoELF {
 public:
   explicit MipsELFMCAsmInfo(const Triple &TheTriple,
                             const MCTargetOptions &Options);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                                 const MCAssembler *Asm) const override;
 };
 
 class MipsCOFFMCAsmInfo : public MCAsmInfoGNUCOFF {
@@ -32,8 +38,47 @@ class MipsCOFFMCAsmInfo : public MCAsmInfoGNUCOFF {
 
 public:
   explicit MipsCOFFMCAsmInfo();
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                                 const MCAssembler *Asm) const override;
 };
 
+namespace Mips {
+using Specifier = uint16_t;
+enum {
+  S_None,
+  S_CALL_HI16 = FirstTargetFixupKind,
+  S_CALL_LO16,
+  S_DTPREL,
+  S_DTPREL_HI,
+  S_DTPREL_LO,
+  S_GOT,
+  S_GOTTPREL,
+  S_GOT_CALL,
+  S_GOT_DISP,
+  S_GOT_HI16,
+  S_GOT_LO16,
+  S_GOT_OFST,
+  S_GOT_PAGE,
+  S_GPREL,
+  S_HI,
+  S_HIGHER,
+  S_HIGHEST,
+  S_LO,
+  S_NEG,
+  S_PCREL_HI16,
+  S_PCREL_LO16,
+  S_TLSGD,
+  S_TLSLDM,
+  S_TPREL_HI,
+  S_TPREL_LO,
+  S_Special,
+};
+
+bool isGpOff(const MCSpecifierExpr &E);
+}
+
 } // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
index b49f86bc5b35..d2981c4ad4d2 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -13,7 +13,7 @@
 #include "MipsMCCodeEmitter.h"
 #include "MCTargetDesc/MipsBaseInfo.h"
 #include "MCTargetDesc/MipsFixupKinds.h"
-#include "MCTargetDesc/MipsMCExpr.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
@@ -580,115 +580,115 @@ unsigned MipsMCCodeEmitter::
 getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups,
                const MCSubtargetInfo &STI) const {
   MCExpr::ExprKind Kind = Expr->getKind();
-  if (Kind == MCExpr::Target) {
+  if (Kind == MCExpr::Specifier) {
     const MipsMCExpr *MipsExpr = cast<MipsMCExpr>(Expr);
 
     Mips::Fixups FixupKind = Mips::Fixups(0);
     switch (MipsExpr->getSpecifier()) {
-    case MipsMCExpr::MEK_None:
-    case MipsMCExpr::MEK_Special:
+    case Mips::S_None:
+    case Mips::S_Special:
       llvm_unreachable("Unhandled fixup kind!");
       break;
-    case MipsMCExpr::MEK_DTPREL:
+    case Mips::S_DTPREL:
       // MEK_DTPREL is used for marking TLS DIEExpr only
       // and contains a regular sub-expression.
       return getExprOpValue(MipsExpr->getSubExpr(), Fixups, STI);
-    case MipsMCExpr::MEK_CALL_HI16:
+    case Mips::S_CALL_HI16:
       FixupKind = Mips::fixup_Mips_CALL_HI16;
       break;
-    case MipsMCExpr::MEK_CALL_LO16:
+    case Mips::S_CALL_LO16:
       FixupKind = Mips::fixup_Mips_CALL_LO16;
       break;
-    case MipsMCExpr::MEK_DTPREL_HI:
+    case Mips::S_DTPREL_HI:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_TLS_DTPREL_HI16
                                    : Mips::fixup_Mips_DTPREL_HI;
       break;
-    case MipsMCExpr::MEK_DTPREL_LO:
+    case Mips::S_DTPREL_LO:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_TLS_DTPREL_LO16
                                    : Mips::fixup_Mips_DTPREL_LO;
       break;
-    case MipsMCExpr::MEK_GOTTPREL:
+    case Mips::S_GOTTPREL:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_GOTTPREL
                                    : Mips::fixup_Mips_GOTTPREL;
       break;
-    case MipsMCExpr::MEK_GOT:
+    case Mips::S_GOT:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_GOT16
                                    : Mips::fixup_Mips_GOT;
       break;
-    case MipsMCExpr::MEK_GOT_CALL:
+    case Mips::S_GOT_CALL:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_CALL16
                                    : Mips::fixup_Mips_CALL16;
       break;
-    case MipsMCExpr::MEK_GOT_DISP:
+    case Mips::S_GOT_DISP:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_GOT_DISP
                                    : Mips::fixup_Mips_GOT_DISP;
       break;
-    case MipsMCExpr::MEK_GOT_HI16:
+    case Mips::S_GOT_HI16:
       FixupKind = Mips::fixup_Mips_GOT_HI16;
       break;
-    case MipsMCExpr::MEK_GOT_LO16:
+    case Mips::S_GOT_LO16:
       FixupKind = Mips::fixup_Mips_GOT_LO16;
       break;
-    case MipsMCExpr::MEK_GOT_PAGE:
+    case Mips::S_GOT_PAGE:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_GOT_PAGE
                                    : Mips::fixup_Mips_GOT_PAGE;
       break;
-    case MipsMCExpr::MEK_GOT_OFST:
+    case Mips::S_GOT_OFST:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_GOT_OFST
                                    : Mips::fixup_Mips_GOT_OFST;
       break;
-    case MipsMCExpr::MEK_GPREL:
+    case Mips::S_GPREL:
       FixupKind = Mips::fixup_Mips_GPREL16;
       break;
-    case MipsMCExpr::MEK_LO:
+    case Mips::S_LO:
       // Check for %lo(%neg(%gp_rel(X)))
-      if (MipsExpr->isGpOff())
+      if (Mips::isGpOff(*MipsExpr))
         FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_GPOFF_LO
                                      : Mips::fixup_Mips_GPOFF_LO;
       else
         FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_LO16
                                      : Mips::fixup_Mips_LO16;
       break;
-    case MipsMCExpr::MEK_HIGHEST:
+    case Mips::S_HIGHEST:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_HIGHEST
                                    : Mips::fixup_Mips_HIGHEST;
       break;
-    case MipsMCExpr::MEK_HIGHER:
+    case Mips::S_HIGHER:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_HIGHER
                                    : Mips::fixup_Mips_HIGHER;
       break;
-    case MipsMCExpr::MEK_HI:
+    case Mips::S_HI:
       // Check for %hi(%neg(%gp_rel(X)))
-      if (MipsExpr->isGpOff())
+      if (Mips::isGpOff(*MipsExpr))
         FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_GPOFF_HI
                                      : Mips::fixup_Mips_GPOFF_HI;
       else
         FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_HI16
                                      : Mips::fixup_Mips_HI16;
       break;
-    case MipsMCExpr::MEK_PCREL_HI16:
+    case Mips::S_PCREL_HI16:
       FixupKind = Mips::fixup_MIPS_PCHI16;
       break;
-    case MipsMCExpr::MEK_PCREL_LO16:
+    case Mips::S_PCREL_LO16:
       FixupKind = Mips::fixup_MIPS_PCLO16;
       break;
-    case MipsMCExpr::MEK_TLSGD:
+    case Mips::S_TLSGD:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_TLS_GD
                                    : Mips::fixup_Mips_TLSGD;
       break;
-    case MipsMCExpr::MEK_TLSLDM:
+    case Mips::S_TLSLDM:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_TLS_LDM
                                    : Mips::fixup_Mips_TLSLDM;
       break;
-    case MipsMCExpr::MEK_TPREL_HI:
+    case Mips::S_TPREL_HI:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_TLS_TPREL_HI16
                                    : Mips::fixup_Mips_TPREL_HI;
       break;
-    case MipsMCExpr::MEK_TPREL_LO:
+    case Mips::S_TPREL_LO:
       FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_TLS_TPREL_LO16
                                    : Mips::fixup_Mips_TPREL_LO;
       break;
-    case MipsMCExpr::MEK_NEG:
+    case Mips::S_NEG:
       FixupKind =
           isMicroMips(STI) ? Mips::fixup_MICROMIPS_SUB : Mips::fixup_Mips_SUB;
       break;
@@ -733,7 +733,7 @@ unsigned MipsMCCodeEmitter::getImmOpValue(const MCInst &MI, const MCOperand &MO,
   if (Expr->evaluateAsAbsolute(Res))
     return Res;
   unsigned MIFrm = MipsII::getFormat(MCII.get(MI.getOpcode()).TSFlags);
-  if (!isa<MCTargetExpr>(Expr) && MIFrm == MipsII::FrmI) {
+  if (!isa<MCSpecifierExpr>(Expr) && MIFrm == MipsII::FrmI) {
     Fixups.push_back(MCFixup::create(
         0, Expr, MCFixupKind(Mips::fixup_Mips_AnyImm16), Expr->getLoc()));
     return 0;
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
index dd800293ae38..821f662f0cbf 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MipsMCExpr.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
@@ -33,139 +34,6 @@ const MipsMCExpr *MipsMCExpr::create(const MCSymbol *Sym, Specifier S,
 
 const MipsMCExpr *MipsMCExpr::createGpOff(MipsMCExpr::Specifier S,
                                           const MCExpr *Expr, MCContext &Ctx) {
-  return create(S, create(MEK_NEG, create(MEK_GPREL, Expr, Ctx), Ctx), Ctx);
-}
-
-void MipsMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  int64_t AbsVal;
-
-  switch (specifier) {
-  case MEK_None:
-  case MEK_Special:
-    llvm_unreachable("MEK_None and MEK_Special are invalid");
-    break;
-  case MEK_DTPREL:
-    // MEK_DTPREL is used for marking TLS DIEExpr only
-    // and contains a regular sub-expression.
-    getSubExpr()->print(OS, MAI);
-    return;
-  case MEK_CALL_HI16:
-    OS << "%call_hi";
-    break;
-  case MEK_CALL_LO16:
-    OS << "%call_lo";
-    break;
-  case MEK_DTPREL_HI:
-    OS << "%dtprel_hi";
-    break;
-  case MEK_DTPREL_LO:
-    OS << "%dtprel_lo";
-    break;
-  case MEK_GOT:
-    OS << "%got";
-    break;
-  case MEK_GOTTPREL:
-    OS << "%gottprel";
-    break;
-  case MEK_GOT_CALL:
-    OS << "%call16";
-    break;
-  case MEK_GOT_DISP:
-    OS << "%got_disp";
-    break;
-  case MEK_GOT_HI16:
-    OS << "%got_hi";
-    break;
-  case MEK_GOT_LO16:
-    OS << "%got_lo";
-    break;
-  case MEK_GOT_PAGE:
-    OS << "%got_page";
-    break;
-  case MEK_GOT_OFST:
-    OS << "%got_ofst";
-    break;
-  case MEK_GPREL:
-    OS << "%gp_rel";
-    break;
-  case MEK_HI:
-    OS << "%hi";
-    break;
-  case MEK_HIGHER:
-    OS << "%higher";
-    break;
-  case MEK_HIGHEST:
-    OS << "%highest";
-    break;
-  case MEK_LO:
-    OS << "%lo";
-    break;
-  case MEK_NEG:
-    OS << "%neg";
-    break;
-  case MEK_PCREL_HI16:
-    OS << "%pcrel_hi";
-    break;
-  case MEK_PCREL_LO16:
-    OS << "%pcrel_lo";
-    break;
-  case MEK_TLSGD:
-    OS << "%tlsgd";
-    break;
-  case MEK_TLSLDM:
-    OS << "%tlsldm";
-    break;
-  case MEK_TPREL_HI:
-    OS << "%tprel_hi";
-    break;
-  case MEK_TPREL_LO:
-    OS << "%tprel_lo";
-    break;
-  }
-
-  OS << '(';
-  if (Expr->evaluateAsAbsolute(AbsVal))
-    OS << AbsVal;
-  else
-    Expr->print(OS, MAI);
-  OS << ')';
-}
-
-bool MipsMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                           const MCAssembler *Asm) const {
-  // Look for the %hi(%neg(%gp_rel(X))) and %lo(%neg(%gp_rel(X)))
-  // special cases.
-  if (isGpOff()) {
-    const MCExpr *SubExpr =
-        cast<MipsMCExpr>(cast<MipsMCExpr>(getSubExpr())->getSubExpr())
-            ->getSubExpr();
-    if (!SubExpr->evaluateAsRelocatable(Res, Asm))
-      return false;
-
-    Res.setSpecifier(MEK_Special);
-    return true;
-  }
-
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-  return !Res.getSubSym();
-}
-
-void MipsMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-bool MipsMCExpr::isGpOff(Specifier &S) const {
-  if (getSpecifier() == MEK_HI || getSpecifier() == MEK_LO) {
-    if (const MipsMCExpr *S1 = dyn_cast<const MipsMCExpr>(getSubExpr())) {
-      if (const MipsMCExpr *S2 = dyn_cast<const MipsMCExpr>(S1->getSubExpr())) {
-        if (S1->getSpecifier() == MEK_NEG && S2->getSpecifier() == MEK_GPREL) {
-          S = getSpecifier();
-          return true;
-        }
-      }
-    }
-  }
-  return false;
+  return create(S, create(Mips::S_NEG, create(Mips::S_GPREL, Expr, Ctx), Ctx),
+                Ctx);
 }
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
index 288ca5c27753..b78aeabb5799 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
@@ -14,44 +14,13 @@
 
 namespace llvm {
 
-class MipsMCExpr : public MCTargetExpr {
+class MipsMCExpr : public MCSpecifierExpr {
 public:
-  enum Specifier {
-    MEK_None,
-    MEK_CALL_HI16,
-    MEK_CALL_LO16,
-    MEK_DTPREL,
-    MEK_DTPREL_HI,
-    MEK_DTPREL_LO,
-    MEK_GOT,
-    MEK_GOTTPREL,
-    MEK_GOT_CALL,
-    MEK_GOT_DISP,
-    MEK_GOT_HI16,
-    MEK_GOT_LO16,
-    MEK_GOT_OFST,
-    MEK_GOT_PAGE,
-    MEK_GPREL,
-    MEK_HI,
-    MEK_HIGHER,
-    MEK_HIGHEST,
-    MEK_LO,
-    MEK_NEG,
-    MEK_PCREL_HI16,
-    MEK_PCREL_LO16,
-    MEK_TLSGD,
-    MEK_TLSLDM,
-    MEK_TPREL_HI,
-    MEK_TPREL_LO,
-    MEK_Special,
-  };
+  using Specifier = Spec;
 
 private:
-  const MCExpr *Expr;
-  const Specifier specifier;
-
   explicit MipsMCExpr(const MCExpr *Expr, Specifier S)
-      : Expr(Expr), specifier(S) {}
+      : MCSpecifierExpr(Expr, S) {}
 
 public:
   static const MipsMCExpr *create(Specifier S, const MCExpr *Expr,
@@ -60,30 +29,6 @@ class MipsMCExpr : public MCTargetExpr {
                                   MCContext &Ctx);
   static const MipsMCExpr *createGpOff(Specifier S, const MCExpr *Expr,
                                        MCContext &Ctx);
-
-  Specifier getSpecifier() const { return specifier; }
-
-  /// Get the child of this expression.
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-
-  bool isGpOff(Specifier &S) const;
-  bool isGpOff() const {
-    Specifier S;
-    return isGpOff(S);
-  }
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index 4919d4f10856..80a854c79901 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -12,6 +12,7 @@
 
 #include "MipsTargetStreamer.h"
 #include "MCTargetDesc/MipsABIInfo.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "MipsBaseInfo.h"
 #include "MipsELFStreamer.h"
 #include "MipsInstPrinter.h"
@@ -398,42 +399,42 @@ MipsTargetAsmStreamer::MipsTargetAsmStreamer(MCStreamer &S,
 void MipsTargetAsmStreamer::emitDTPRel32Value(const MCExpr *Value) {
   auto *MAI = getStreamer().getContext().getAsmInfo();
   OS << "\t.dtprelword\t";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   OS << '\n';
 }
 
 void MipsTargetAsmStreamer::emitDTPRel64Value(const MCExpr *Value) {
   auto *MAI = getStreamer().getContext().getAsmInfo();
   OS << "\t.dtpreldword\t";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   OS << '\n';
 }
 
 void MipsTargetAsmStreamer::emitTPRel32Value(const MCExpr *Value) {
   auto *MAI = getStreamer().getContext().getAsmInfo();
   OS << "\t.tprelword\t";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   OS << '\n';
 }
 
 void MipsTargetAsmStreamer::emitTPRel64Value(const MCExpr *Value) {
   auto *MAI = getStreamer().getContext().getAsmInfo();
   OS << "\t.tpreldword\t";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   OS << '\n';
 }
 
 void MipsTargetAsmStreamer::emitGPRel32Value(const MCExpr *Value) {
   auto *MAI = getStreamer().getContext().getAsmInfo();
   OS << "\t.gpword\t";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   OS << '\n';
 }
 
 void MipsTargetAsmStreamer::emitGPRel64Value(const MCExpr *Value) {
   auto *MAI = getStreamer().getContext().getAsmInfo();
   OS << "\t.gpdword\t";
-  Value->print(OS, MAI);
+  MAI->printExpr(OS, *Value);
   OS << '\n';
 }
 
@@ -1266,7 +1267,7 @@ void MipsTargetELFStreamer::emitDirectiveCpLoad(unsigned RegNo) {
   TmpInst.setOpcode(Mips::LUi);
   TmpInst.addOperand(MCOperand::createReg(GPReg));
   const MCExpr *HiSym = MipsMCExpr::create(
-      MipsMCExpr::MEK_HI, MCSymbolRefExpr::create(GP_Disp, MCA.getContext()),
+      Mips::S_HI, MCSymbolRefExpr::create(GP_Disp, MCA.getContext()),
       MCA.getContext());
   TmpInst.addOperand(MCOperand::createExpr(HiSym));
   getStreamer().emitInstruction(TmpInst, STI);
@@ -1277,7 +1278,7 @@ void MipsTargetELFStreamer::emitDirectiveCpLoad(unsigned RegNo) {
   TmpInst.addOperand(MCOperand::createReg(GPReg));
   TmpInst.addOperand(MCOperand::createReg(GPReg));
   const MCExpr *LoSym = MipsMCExpr::create(
-      MipsMCExpr::MEK_LO, MCSymbolRefExpr::create(GP_Disp, MCA.getContext()),
+      Mips::S_LO, MCSymbolRefExpr::create(GP_Disp, MCA.getContext()),
       MCA.getContext());
   TmpInst.addOperand(MCOperand::createExpr(LoSym));
   getStreamer().emitInstruction(TmpInst, STI);
@@ -1342,10 +1343,10 @@ void MipsTargetELFStreamer::emitDirectiveCpsetup(unsigned RegNo,
   }
 
   const MipsMCExpr *HiExpr = MipsMCExpr::createGpOff(
-      MipsMCExpr::MEK_HI, MCSymbolRefExpr::create(&Sym, MCA.getContext()),
+      Mips::S_HI, MCSymbolRefExpr::create(&Sym, MCA.getContext()),
       MCA.getContext());
   const MipsMCExpr *LoExpr = MipsMCExpr::createGpOff(
-      MipsMCExpr::MEK_LO, MCSymbolRefExpr::create(&Sym, MCA.getContext()),
+      Mips::S_LO, MCSymbolRefExpr::create(&Sym, MCA.getContext()),
       MCA.getContext());
 
   // lui $gp, %hi(%neg(%gp_rel(funcSym)))
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MipsAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
index 1a3e99ec7f68..da3f7cb55b30 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
@@ -15,6 +15,7 @@
 #include "MCTargetDesc/MipsABIInfo.h"
 #include "MCTargetDesc/MipsBaseInfo.h"
 #include "MCTargetDesc/MipsInstPrinter.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "MCTargetDesc/MipsMCNaCl.h"
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "MCTargetDesc/MipsTargetStreamer.h"
@@ -1244,7 +1245,7 @@ void MipsAsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
 // and value for debug thread local expression.
 void MipsAsmPrinter::emitDebugValue(const MCExpr *Value, unsigned Size) const {
   if (auto *MipsExpr = dyn_cast<MipsMCExpr>(Value)) {
-    if (MipsExpr && MipsExpr->getSpecifier() == MipsMCExpr::MEK_DTPREL) {
+    if (MipsExpr && MipsExpr->getSpecifier() == Mips::S_DTPREL) {
       switch (Size) {
       case 4:
         getTargetStreamer().emitDTPRel32Value(MipsExpr->getSubExpr());
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp b/external/llvm-project/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
index 6c006b0acfae..b13394a607f6 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
@@ -692,8 +692,10 @@ bool MipsDelaySlotFiller::searchRange(MachineBasicBlock &MBB, IterTy Begin,
     IterTy CurrI = I;
     ++I;
     LLVM_DEBUG(dbgs() << DEBUG_TYPE ": checking instruction: "; CurrI->dump());
-    // skip debug value
-    if (CurrI->isDebugInstr()) {
+    // Skip debug value.
+    // Instruction TargetOpcode::JUMP_TABLE_DEBUG_INFO is only used to note
+    // jump table debug info.
+    if (CurrI->isDebugInstr() || CurrI->isJumpTableDebugInfo()) {
       LLVM_DEBUG(dbgs() << DEBUG_TYPE ": ignoring debug instruction: ";
                  CurrI->dump());
       continue;
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.cpp b/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.cpp
index d1eef1775aa6..935fcd8fa715 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.cpp
@@ -13,7 +13,7 @@
 
 #include "MipsMCInstLower.h"
 #include "MCTargetDesc/MipsBaseInfo.h"
-#include "MCTargetDesc/MipsMCExpr.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "MipsAsmPrinter.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -35,7 +35,7 @@ void MipsMCInstLower::Initialize(MCContext *C) {
 MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
                                               MachineOperandType MOTy,
                                               int64_t Offset) const {
-  MipsMCExpr::Specifier TargetKind = MipsMCExpr::MEK_None;
+  Mips::Specifier TargetKind = Mips::S_None;
   bool IsGpOff = false;
   const MCSymbol *Symbol;
   SmallString<128> Name;
@@ -53,75 +53,75 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   case MipsII::MO_NO_FLAG:
     break;
   case MipsII::MO_GPREL:
-    TargetKind = MipsMCExpr::MEK_GPREL;
+    TargetKind = Mips::S_GPREL;
     break;
   case MipsII::MO_GOT_CALL:
-    TargetKind = MipsMCExpr::MEK_GOT_CALL;
+    TargetKind = Mips::S_GOT_CALL;
     break;
   case MipsII::MO_GOT:
-    TargetKind = MipsMCExpr::MEK_GOT;
+    TargetKind = Mips::S_GOT;
     break;
   case MipsII::MO_ABS_HI:
-    TargetKind = MipsMCExpr::MEK_HI;
+    TargetKind = Mips::S_HI;
     break;
   case MipsII::MO_ABS_LO:
-    TargetKind = MipsMCExpr::MEK_LO;
+    TargetKind = Mips::S_LO;
     break;
   case MipsII::MO_TLSGD:
-    TargetKind = MipsMCExpr::MEK_TLSGD;
+    TargetKind = Mips::S_TLSGD;
     break;
   case MipsII::MO_TLSLDM:
-    TargetKind = MipsMCExpr::MEK_TLSLDM;
+    TargetKind = Mips::S_TLSLDM;
     break;
   case MipsII::MO_DTPREL_HI:
-    TargetKind = MipsMCExpr::MEK_DTPREL_HI;
+    TargetKind = Mips::S_DTPREL_HI;
     break;
   case MipsII::MO_DTPREL_LO:
-    TargetKind = MipsMCExpr::MEK_DTPREL_LO;
+    TargetKind = Mips::S_DTPREL_LO;
     break;
   case MipsII::MO_GOTTPREL:
-    TargetKind = MipsMCExpr::MEK_GOTTPREL;
+    TargetKind = Mips::S_GOTTPREL;
     break;
   case MipsII::MO_TPREL_HI:
-    TargetKind = MipsMCExpr::MEK_TPREL_HI;
+    TargetKind = Mips::S_TPREL_HI;
     break;
   case MipsII::MO_TPREL_LO:
-    TargetKind = MipsMCExpr::MEK_TPREL_LO;
+    TargetKind = Mips::S_TPREL_LO;
     break;
   case MipsII::MO_GPOFF_HI:
-    TargetKind = MipsMCExpr::MEK_HI;
+    TargetKind = Mips::S_HI;
     IsGpOff = true;
     break;
   case MipsII::MO_GPOFF_LO:
-    TargetKind = MipsMCExpr::MEK_LO;
+    TargetKind = Mips::S_LO;
     IsGpOff = true;
     break;
   case MipsII::MO_GOT_DISP:
-    TargetKind = MipsMCExpr::MEK_GOT_DISP;
+    TargetKind = Mips::S_GOT_DISP;
     break;
   case MipsII::MO_GOT_HI16:
-    TargetKind = MipsMCExpr::MEK_GOT_HI16;
+    TargetKind = Mips::S_GOT_HI16;
     break;
   case MipsII::MO_GOT_LO16:
-    TargetKind = MipsMCExpr::MEK_GOT_LO16;
+    TargetKind = Mips::S_GOT_LO16;
     break;
   case MipsII::MO_GOT_PAGE:
-    TargetKind = MipsMCExpr::MEK_GOT_PAGE;
+    TargetKind = Mips::S_GOT_PAGE;
     break;
   case MipsII::MO_GOT_OFST:
-    TargetKind = MipsMCExpr::MEK_GOT_OFST;
+    TargetKind = Mips::S_GOT_OFST;
     break;
   case MipsII::MO_HIGHER:
-    TargetKind = MipsMCExpr::MEK_HIGHER;
+    TargetKind = Mips::S_HIGHER;
     break;
   case MipsII::MO_HIGHEST:
-    TargetKind = MipsMCExpr::MEK_HIGHEST;
+    TargetKind = Mips::S_HIGHEST;
     break;
   case MipsII::MO_CALL_HI16:
-    TargetKind = MipsMCExpr::MEK_CALL_HI16;
+    TargetKind = Mips::S_CALL_HI16;
     break;
   case MipsII::MO_CALL_LO16:
-    TargetKind = MipsMCExpr::MEK_CALL_LO16;
+    TargetKind = Mips::S_CALL_LO16;
     break;
   case MipsII::MO_JALR:
     return MCOperand();
@@ -176,7 +176,7 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
 
   if (IsGpOff)
     Expr = MipsMCExpr::createGpOff(TargetKind, Expr, *Ctx);
-  else if (TargetKind != MipsMCExpr::MEK_None)
+  else if (TargetKind != Mips::S_None)
     Expr = MipsMCExpr::create(TargetKind, Expr, *Ctx);
 
   return MCOperand::createExpr(Expr);
@@ -211,7 +211,7 @@ MCOperand MipsMCInstLower::LowerOperand(const MachineOperand &MO,
 
 MCOperand MipsMCInstLower::createSub(MachineBasicBlock *BB1,
                                      MachineBasicBlock *BB2,
-                                     MipsMCExpr::Specifier Kind) const {
+                                     Mips::Specifier Kind) const {
   const MCSymbolRefExpr *Sym1 = MCSymbolRefExpr::create(BB1->getSymbol(), *Ctx);
   const MCSymbolRefExpr *Sym2 = MCSymbolRefExpr::create(BB2->getSymbol(), *Ctx);
   const MCBinaryExpr *Sub = MCBinaryExpr::createSub(Sym1, Sym2, *Ctx);
@@ -226,20 +226,20 @@ lowerLongBranchLUi(const MachineInstr *MI, MCInst &OutMI) const {
   // Lower register operand.
   OutMI.addOperand(LowerOperand(MI->getOperand(0)));
 
-  MipsMCExpr::Specifier Spec;
+  Mips::Specifier Spec;
   unsigned TargetFlags = MI->getOperand(1).getTargetFlags();
   switch (TargetFlags) {
   case MipsII::MO_HIGHEST:
-    Spec = MipsMCExpr::MEK_HIGHEST;
+    Spec = Mips::S_HIGHEST;
     break;
   case MipsII::MO_HIGHER:
-    Spec = MipsMCExpr::MEK_HIGHER;
+    Spec = Mips::S_HIGHER;
     break;
   case MipsII::MO_ABS_HI:
-    Spec = MipsMCExpr::MEK_HI;
+    Spec = Mips::S_HI;
     break;
   case MipsII::MO_ABS_LO:
-    Spec = MipsMCExpr::MEK_LO;
+    Spec = Mips::S_LO;
     break;
   default:
     report_fatal_error("Unexpected flags for lowerLongBranchLUi");
@@ -248,7 +248,7 @@ lowerLongBranchLUi(const MachineInstr *MI, MCInst &OutMI) const {
   if (MI->getNumOperands() == 2) {
     const MCExpr *Expr =
         MCSymbolRefExpr::create(MI->getOperand(1).getMBB()->getSymbol(), *Ctx);
-    const MipsMCExpr *MipsExpr = MipsMCExpr::create(Spec, Expr, *Ctx);
+    const auto *MipsExpr = MipsMCExpr::create(Spec, Expr, *Ctx);
     OutMI.addOperand(MCOperand::createExpr(MipsExpr));
   } else if (MI->getNumOperands() == 3) {
     // Create %hi($tgt-$baltgt).
@@ -261,20 +261,20 @@ void MipsMCInstLower::lowerLongBranchADDiu(const MachineInstr *MI,
                                            MCInst &OutMI, int Opcode) const {
   OutMI.setOpcode(Opcode);
 
-  MipsMCExpr::Specifier Spec;
+  Mips::Specifier Spec;
   unsigned TargetFlags = MI->getOperand(2).getTargetFlags();
   switch (TargetFlags) {
   case MipsII::MO_HIGHEST:
-    Spec = MipsMCExpr::MEK_HIGHEST;
+    Spec = Mips::S_HIGHEST;
     break;
   case MipsII::MO_HIGHER:
-    Spec = MipsMCExpr::MEK_HIGHER;
+    Spec = Mips::S_HIGHER;
     break;
   case MipsII::MO_ABS_HI:
-    Spec = MipsMCExpr::MEK_HI;
+    Spec = Mips::S_HI;
     break;
   case MipsII::MO_ABS_LO:
-    Spec = MipsMCExpr::MEK_LO;
+    Spec = Mips::S_LO;
     break;
   default:
     report_fatal_error("Unexpected flags for lowerLongBranchADDiu");
@@ -290,7 +290,7 @@ void MipsMCInstLower::lowerLongBranchADDiu(const MachineInstr *MI,
     // Lower register operand.
     const MCExpr *Expr =
         MCSymbolRefExpr::create(MI->getOperand(2).getMBB()->getSymbol(), *Ctx);
-    const MipsMCExpr *MipsExpr = MipsMCExpr::create(Spec, Expr, *Ctx);
+    const auto *MipsExpr = MipsMCExpr::create(Spec, Expr, *Ctx);
     OutMI.addOperand(MCOperand::createExpr(MipsExpr));
   } else if (MI->getNumOperands() == 4) {
     // Create %lo($tgt-$baltgt) or %hi($tgt-$baltgt).
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.h b/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.h
index b6ddbe98955d..a618c6fb7bfa 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.h
+++ b/external/llvm-project/llvm/lib/Target/Mips/MipsMCInstLower.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIB_TARGET_MIPS_MIPSMCINSTLOWER_H
 #define LLVM_LIB_TARGET_MIPS_MIPSMCINSTLOWER_H
 
-#include "MCTargetDesc/MipsMCExpr.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/Support/Compiler.h"
 
@@ -41,7 +41,7 @@ class LLVM_LIBRARY_VISIBILITY MipsMCInstLower {
   MCOperand LowerSymbolOperand(const MachineOperand &MO,
                                MachineOperandType MOTy, int64_t Offset) const;
   MCOperand createSub(MachineBasicBlock *BB1, MachineBasicBlock *BB2,
-                      MipsMCExpr::Specifier Kind) const;
+                      Mips::Specifier Kind) const;
   void lowerLongBranchLUi(const MachineInstr *MI, MCInst &OutMI) const;
   void lowerLongBranchADDiu(const MachineInstr *MI, MCInst &OutMI,
                             int Opcode) const;
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp b/external/llvm-project/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp
index e74b3ad5ebca..23aa699318a2 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Target/Mips/MipsTargetObjectFile.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MipsTargetObjectFile.h"
-#include "MCTargetDesc/MipsMCExpr.h"
+#include "MCTargetDesc/MipsMCAsmInfo.h"
 #include "MipsSubtarget.h"
 #include "MipsTargetMachine.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -189,5 +189,5 @@ MipsTargetObjectFile::getDebugThreadLocalSymbol(const MCSymbol *Sym) const {
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
   Expr = MCBinaryExpr::createAdd(
       Expr, MCConstantExpr::create(0x8000, getContext()), getContext());
-  return MipsMCExpr::create(MipsMCExpr::MEK_DTPREL, Expr, getContext());
+  return MipsMCExpr::create(Mips::S_DTPREL, Expr, getContext());
 }
diff --git a/external/llvm-project/llvm/lib/Target/Mips/MipsTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/Mips/MipsTargetTransformInfo.h
index 5e3884cd8016..8f8173915b2f 100644
--- a/external/llvm-project/llvm/lib/Target/Mips/MipsTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/Mips/MipsTargetTransformInfo.h
@@ -15,7 +15,7 @@
 
 namespace llvm {
 
-class MipsTTIImpl : public BasicTTIImplBase<MipsTTIImpl> {
+class MipsTTIImpl final : public BasicTTIImplBase<MipsTTIImpl> {
   using BaseT = BasicTTIImplBase<MipsTTIImpl>;
   using TTI = TargetTransformInfo;
 
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index b4616b64bad1..cc79257fb9c8 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -15,6 +15,7 @@
 #include "NVPTXUtilities.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/NVVMIntrinsicUtils.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -90,7 +91,7 @@ void NVPTXInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     markup(O, Markup::Immediate) << formatImm(Op.getImm());
   } else {
     assert(Op.isExpr() && "Unknown operand kind in printOperand");
-    Op.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -437,3 +438,22 @@ void NVPTXInstPrinter::printTmaReductionMode(const MCInst *MI, int OpNum,
   llvm_unreachable(
       "Invalid Reduction Op in printCpAsyncBulkTensorReductionMode");
 }
+
+void NVPTXInstPrinter::printCTAGroup(const MCInst *MI, int OpNum,
+                                     raw_ostream &O) {
+  const MCOperand &MO = MI->getOperand(OpNum);
+  using CGTy = nvvm::CTAGroupKind;
+
+  switch (static_cast<CGTy>(MO.getImm())) {
+  case CGTy::CG_NONE:
+    O << "";
+    return;
+  case CGTy::CG_1:
+    O << ".cta_group::1";
+    return;
+  case CGTy::CG_2:
+    O << ".cta_group::2";
+    return;
+  }
+  llvm_unreachable("Invalid cta_group in printCTAGroup");
+}
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index a2dd772cd86d..f73af7a3f2c6 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -51,6 +51,7 @@ class NVPTXInstPrinter : public MCInstPrinter {
   void printProtoIdent(const MCInst *MI, int OpNum, raw_ostream &O);
   void printPrmtMode(const MCInst *MI, int OpNum, raw_ostream &O);
   void printTmaReductionMode(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printCTAGroup(const MCInst *MI, int OpNum, raw_ostream &O);
 };
 
 }
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index e123849066e1..b4e2c46b9444 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -1849,75 +1849,8 @@ NVPTXAsmPrinter::lowerConstantForGV(const Constant *CV,
   report_fatal_error(Twine(OS.str()));
 }
 
-// Copy of MCExpr::print customized for NVPTX
 void NVPTXAsmPrinter::printMCExpr(const MCExpr &Expr, raw_ostream &OS) const {
-  switch (Expr.getKind()) {
-  case MCExpr::Target:
-    return cast<MCTargetExpr>(&Expr)->printImpl(OS, MAI);
-  case MCExpr::Constant:
-    OS << cast<MCConstantExpr>(Expr).getValue();
-    return;
-
-  case MCExpr::SymbolRef: {
-    const MCSymbolRefExpr &SRE = cast<MCSymbolRefExpr>(Expr);
-    const MCSymbol &Sym = SRE.getSymbol();
-    Sym.print(OS, MAI);
-    return;
-  }
-
-  case MCExpr::Unary: {
-    const MCUnaryExpr &UE = cast<MCUnaryExpr>(Expr);
-    switch (UE.getOpcode()) {
-    case MCUnaryExpr::LNot:  OS << '!'; break;
-    case MCUnaryExpr::Minus: OS << '-'; break;
-    case MCUnaryExpr::Not:   OS << '~'; break;
-    case MCUnaryExpr::Plus:  OS << '+'; break;
-    }
-    printMCExpr(*UE.getSubExpr(), OS);
-    return;
-  }
-
-  case MCExpr::Binary: {
-    const MCBinaryExpr &BE = cast<MCBinaryExpr>(Expr);
-
-    // Only print parens around the LHS if it is non-trivial.
-    if (isa<MCConstantExpr>(BE.getLHS()) || isa<MCSymbolRefExpr>(BE.getLHS()) ||
-        isa<NVPTXGenericMCSymbolRefExpr>(BE.getLHS())) {
-      printMCExpr(*BE.getLHS(), OS);
-    } else {
-      OS << '(';
-      printMCExpr(*BE.getLHS(), OS);
-      OS<< ')';
-    }
-
-    switch (BE.getOpcode()) {
-    case MCBinaryExpr::Add:
-      // Print "X-42" instead of "X+-42".
-      if (const MCConstantExpr *RHSC = dyn_cast<MCConstantExpr>(BE.getRHS())) {
-        if (RHSC->getValue() < 0) {
-          OS << RHSC->getValue();
-          return;
-        }
-      }
-
-      OS <<  '+';
-      break;
-    default: llvm_unreachable("Unhandled binary operator");
-    }
-
-    // Only print parens around the LHS if it is non-trivial.
-    if (isa<MCConstantExpr>(BE.getRHS()) || isa<MCSymbolRefExpr>(BE.getRHS())) {
-      printMCExpr(*BE.getRHS(), OS);
-    } else {
-      OS << '(';
-      printMCExpr(*BE.getRHS(), OS);
-      OS << ')';
-    }
-    return;
-  }
-  }
-
-  llvm_unreachable("Invalid expression kind!");
+  OutContext.getAsmInfo()->printExpr(OS, Expr);
 }
 
 /// PrintAsmOperand - Print out an operand for an inline asm expression.
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 32223bf3d601..79b1bfbc8072 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -71,8 +71,8 @@ NVPTXDAGToDAGISel::getDivF32Level(const SDNode *N) const {
   return Subtarget->getTargetLowering()->getDivF32Level(*MF, *N);
 }
 
-bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
-  return Subtarget->getTargetLowering()->usePrecSqrtF32();
+bool NVPTXDAGToDAGISel::usePrecSqrtF32(const SDNode *N) const {
+  return Subtarget->getTargetLowering()->usePrecSqrtF32(*MF, N);
 }
 
 bool NVPTXDAGToDAGISel::useF32FTZ() const {
@@ -2556,19 +2556,25 @@ void NVPTXDAGToDAGISel::SelectCpAsyncBulkTensorG2SCommon(SDNode *N,
   // We have {Chain, Intrinsic-ID} followed by the actual intrisic args:
   // {dst, mbar, src, dims{d0...dN}, im2col_offsets{dims-2}
   // multicast, cache_hint,
-  // multicast_flag, cache_hint_flag}
+  // multicast_flag, cache_hint_flag, cta_group_flag}
   // NumOperands = {Chain, IID} + {Actual intrinsic args}
-  //             = {2}          + {7 + dims + im2col_offsets}
+  //             = {2}          + {8 + dims + im2col_offsets}
   size_t NumOps = N->getNumOperands();
   size_t NumDims = IsIm2Col ? GetDimsFromIntrinsic(N->getConstantOperandVal(1))
-                            : (NumOps - 9);
+                            : (NumOps - 10);
   // Offsets is always 'NumDims - 2' and only for im2col mode
   size_t NumOffsets = IsIm2Col ? (NumDims - 2) : 0;
-  bool IsCacheHint = N->getConstantOperandVal(NumOps - 1) == 1;
-  bool IsMultiCast = N->getConstantOperandVal(NumOps - 2) == 1;
+  bool IsCacheHint = N->getConstantOperandVal(NumOps - 2) == 1;
+  bool IsMultiCast = N->getConstantOperandVal(NumOps - 3) == 1;
   size_t NumBaseArgs = NumDims + NumOffsets + 3; // for {dst, mbar, src}
   size_t MultiCastIdx = NumBaseArgs + 2;         // for Chain and IID
 
+  unsigned CTAGroupVal = N->getConstantOperandVal(NumOps - 1);
+  if ((CTAGroupVal > 0) && !Subtarget->hasCpAsyncBulkTensorCTAGroupSupport())
+    report_fatal_error(
+        formatv("CpAsyncBulkTensorG2S cta_group::1/2 is not supported on sm_{}",
+                Subtarget->getSmVersion()));
+
   SDLoc DL(N);
   SmallVector<SDValue, 8> Ops(N->ops().slice(2, NumBaseArgs));
 
@@ -2580,6 +2586,9 @@ void NVPTXDAGToDAGISel::SelectCpAsyncBulkTensorG2SCommon(SDNode *N,
   if (IsCacheHint)
     Ops.push_back(N->getOperand(MultiCastIdx + 1));
 
+  // Flag for CTA Group
+  Ops.push_back(getI32Imm(CTAGroupVal, DL));
+
   // Finally, the chain operand
   Ops.push_back(N->getOperand(0));
 
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index 71a5b7ff8cd3..473f4781a6c3 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -44,7 +44,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
   bool doMulWide;
 
   NVPTX::DivPrecisionLevel getDivF32Level(const SDNode *N) const;
-  bool usePrecSqrtF32() const;
+  bool usePrecSqrtF32(const SDNode *N) const;
   bool useF32FTZ() const;
   bool allowFMA() const;
   bool allowUnsafeFPMath() const;
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index d6a134d9abaf..492f4ab76fdb 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -134,14 +134,23 @@ NVPTXTargetLowering::getDivF32Level(const MachineFunction &MF,
   return NVPTX::DivPrecisionLevel::IEEE754;
 }
 
-bool NVPTXTargetLowering::usePrecSqrtF32() const {
-  if (UsePrecSqrtF32.getNumOccurrences() > 0) {
-    // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
+bool NVPTXTargetLowering::usePrecSqrtF32(const MachineFunction &MF,
+                                         const SDNode *N) const {
+  // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
+  if (UsePrecSqrtF32.getNumOccurrences() > 0)
     return UsePrecSqrtF32;
-  } else {
-    // Otherwise, use sqrt.approx if fast math is enabled
-    return !getTargetMachine().Options.UnsafeFPMath;
+
+  // Otherwise, use sqrt.approx if fast math is enabled
+  if (allowUnsafeFPMath(MF))
+    return false;
+
+  if (N) {
+    const SDNodeFlags Flags = N->getFlags();
+    if (Flags.hasApproximateFuncs())
+      return false;
   }
+
+  return true;
 }
 
 bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
@@ -1134,7 +1143,8 @@ SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
                                              bool &UseOneConst,
                                              bool Reciprocal) const {
   if (!(Enabled == ReciprocalEstimate::Enabled ||
-        (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))
+        (Enabled == ReciprocalEstimate::Unspecified &&
+         !usePrecSqrtF32(DAG.getMachineFunction()))))
     return SDValue();
 
   if (ExtraSteps == ReciprocalEstimate::Unspecified)
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 8d71022a1f10..0a54a8fd71f3 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -225,7 +225,8 @@ class NVPTXTargetLowering : public TargetLowering {
 
   // Get whether we should use a precise or approximate 32-bit floating point
   // sqrt instruction.
-  bool usePrecSqrtF32() const;
+  bool usePrecSqrtF32(const MachineFunction &MF,
+                      const SDNode *N = nullptr) const;
 
   // Get whether we should use instructions that flush floating-point denormals
   // to sign-preserving zero.
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index b646d39194c7..4c3501df57f8 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -151,16 +151,12 @@ def doRsqrtOpt : Predicate<"doRsqrtOpt()">;
 
 def doMulWide      : Predicate<"doMulWide">;
 
-def do_SQRTF32_APPROX : Predicate<"!usePrecSqrtF32()">;
-def do_SQRTF32_RN : Predicate<"usePrecSqrtF32()">;
-
 def hasHWROT32 : Predicate<"Subtarget->hasHWROT32()">;
 def noHWROT32 : Predicate<"!Subtarget->hasHWROT32()">;
 def hasDotInstructions : Predicate<"Subtarget->hasDotInstructions()">;
 def hasTcgen05Instructions : Predicate<"Subtarget->hasTcgen05Instructions()">;
 
 def True : Predicate<"true">;
-def False : Predicate<"false">;
 
 class hasPTX<int version>: Predicate<"Subtarget->getPTXVersion() >= " # version>;
 class hasSM<int version>: Predicate<"Subtarget->getSmVersion() >= " # version>;
@@ -257,6 +253,11 @@ def BF16X2RT : RegTyInfo<v2bf16, Int32Regs, ?, ?, supports_imm = 0>;
 //                         "prmt.b32${mode}">;
 //         ---> "prmt.b32${mode} \t$d, $a, $b, $c;"
 //
+//   * BasicFlagsNVPTXInst<(outs Int64Regs:$state),
+//                         (ins ADDR:$addr),
+//                         "mbarrier.arrive.b64">;
+//         ---> "mbarrier.arrive.b64 \t$state, [$addr];"
+//
 class BasicFlagsNVPTXInst<dag outs_dag, dag ins_dag, dag flags_dag, string asmstr,
                           list<dag> pattern = []>
   : NVPTXInst<
@@ -274,7 +275,11 @@ class BasicFlagsNVPTXInst<dag outs_dag, dag ins_dag, dag flags_dag, string asmst
             !if(!or(!empty(ins_dag), !empty(outs_dag)), "", ", "),
             !interleave(
               !foreach(i, !range(!size(ins_dag)),
-                "$" # !getdagname(ins_dag, i)),
+                 !if(!eq(!cast<string>(!getdagarg<DAGOperand>(ins_dag, i)), "ADDR"),
+                    "[$" # !getdagname(ins_dag, i) # "]",
+                    "$" # !getdagname(ins_dag, i)
+                 )
+                ),
               ", "))),
         ";"),
       pattern>;
@@ -956,31 +961,17 @@ def mul_wide_signed : SDNode<"NVPTXISD::MUL_WIDE_SIGNED", SDTMulWide>;
 def mul_wide_unsigned : SDNode<"NVPTXISD::MUL_WIDE_UNSIGNED", SDTMulWide>;
 
 // Matchers for signed, unsigned mul.wide ISD nodes.
-def : Pat<(i32 (mul_wide_signed i16:$a, i16:$b)),
-          (MULWIDES32 $a, $b)>,
-      Requires<[doMulWide]>;
-def : Pat<(i32 (mul_wide_signed i16:$a, imm:$b)),
-          (MULWIDES32Imm $a, imm:$b)>,
-      Requires<[doMulWide]>;
-def : Pat<(i32 (mul_wide_unsigned i16:$a, i16:$b)),
-          (MULWIDEU32 $a, $b)>,
-      Requires<[doMulWide]>;
-def : Pat<(i32 (mul_wide_unsigned i16:$a, imm:$b)),
-          (MULWIDEU32Imm $a, imm:$b)>,
-      Requires<[doMulWide]>;
+let Predicates = [doMulWide] in {
+  def : Pat<(i32 (mul_wide_signed i16:$a, i16:$b)), (MULWIDES32 $a, $b)>;
+  def : Pat<(i32 (mul_wide_signed i16:$a, imm:$b)), (MULWIDES32Imm $a, imm:$b)>;
+  def : Pat<(i32 (mul_wide_unsigned i16:$a, i16:$b)), (MULWIDEU32 $a, $b)>;
+  def : Pat<(i32 (mul_wide_unsigned i16:$a, imm:$b)), (MULWIDEU32Imm $a, imm:$b)>;
 
-def : Pat<(i64 (mul_wide_signed i32:$a, i32:$b)),
-          (MULWIDES64 $a, $b)>,
-      Requires<[doMulWide]>;
-def : Pat<(i64 (mul_wide_signed i32:$a, imm:$b)),
-          (MULWIDES64Imm $a, imm:$b)>,
-      Requires<[doMulWide]>;
-def : Pat<(i64 (mul_wide_unsigned i32:$a, i32:$b)),
-          (MULWIDEU64 $a, $b)>,
-      Requires<[doMulWide]>;
-def : Pat<(i64 (mul_wide_unsigned i32:$a, imm:$b)),
-          (MULWIDEU64Imm $a, imm:$b)>,
-      Requires<[doMulWide]>;
+  def : Pat<(i64 (mul_wide_signed i32:$a, i32:$b)), (MULWIDES64 $a, $b)>;
+  def : Pat<(i64 (mul_wide_signed i32:$a, imm:$b)), (MULWIDES64Imm $a, imm:$b)>;
+  def : Pat<(i64 (mul_wide_unsigned i32:$a, i32:$b)), (MULWIDEU64 $a, $b)>;
+  def : Pat<(i64 (mul_wide_unsigned i32:$a, imm:$b)), (MULWIDEU64Imm $a, imm:$b)>;
+}
 
 // Predicates used for converting some patterns to mul.wide.
 def SInt32Const : PatLeaf<(imm), [{
@@ -1106,18 +1097,12 @@ defm MAD32 : MAD<"mad.lo.s32", i32, Int32Regs, i32imm>;
 defm MAD64 : MAD<"mad.lo.s64", i64, Int64Regs, i64imm>;
 }
 
-def INEG16 :
-  BasicNVPTXInst<(outs Int16Regs:$dst), (ins Int16Regs:$src),
-            "neg.s16",
-            [(set i16:$dst, (ineg i16:$src))]>;
-def INEG32 :
-  BasicNVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src),
-            "neg.s32",
-            [(set i32:$dst, (ineg i32:$src))]>;
-def INEG64 :
-  BasicNVPTXInst<(outs Int64Regs:$dst), (ins Int64Regs:$src),
-            "neg.s64",
-            [(set i64:$dst, (ineg i64:$src))]>;
+foreach t = [I16RT, I32RT, I64RT] in {
+  def NEG_S # t.Size :
+    BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src),
+              "neg.s" # t.Size,
+              [(set t.Ty:$dst, (ineg t.Ty:$src))]>;
+}
 
 //-----------------------------------
 // Floating Point Arithmetic
@@ -1538,7 +1523,7 @@ def bfi : SDNode<"NVPTXISD::BFI", SDTBFI>;
 
 def SDTPRMT :
   SDTypeProfile<1, 4, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
-                       SDTCisVT<2, i32>, SDTCisVT<3, i32>, SDTCisVT<4, i32>,]>;
+                       SDTCisVT<2, i32>, SDTCisVT<3, i32>, SDTCisVT<4, i32>]>;
 def prmt : SDNode<"NVPTXISD::PRMT", SDTPRMT>;
 
 multiclass BFE<string Instr, ValueType T, RegisterClass RC> {
@@ -1636,6 +1621,10 @@ let hasSideEffects = false in {
 
 }
 
+// PRMT folding patterns
+def : Pat<(fshr i32:$hi, i32:$lo, (shl i32:$amt, (i32 3))),
+          (PRMT_B32rrr $lo, $hi, $amt, PrmtF4E)>;
+
 
 // byte extraction + signed/unsigned extension to i32.
 def : Pat<(i32 (sext_inreg (bfe i32:$s, i32:$o, 8), i8)),
@@ -1961,7 +1950,7 @@ multiclass FSET_FORMAT<PatFrag OpNode, PatLeaf Mode, PatLeaf ModeFTZ> {
   // f16 -> pred
   def : Pat<(i1 (OpNode f16:$a, f16:$b)),
             (SETP_f16rr $a, $b, ModeFTZ)>,
-        Requires<[useFP16Math,doF32FTZ]>;
+        Requires<[useFP16Math, doF32FTZ]>;
   def : Pat<(i1 (OpNode f16:$a, f16:$b)),
             (SETP_f16rr $a, $b, Mode)>,
         Requires<[useFP16Math]>;
@@ -1969,7 +1958,7 @@ multiclass FSET_FORMAT<PatFrag OpNode, PatLeaf Mode, PatLeaf ModeFTZ> {
   // bf16 -> pred
   def : Pat<(i1 (OpNode bf16:$a, bf16:$b)),
             (SETP_bf16rr $a, $b, ModeFTZ)>,
-        Requires<[hasBF16Math,doF32FTZ]>;
+        Requires<[hasBF16Math, doF32FTZ]>;
   def : Pat<(i1 (OpNode bf16:$a, bf16:$b)),
             (SETP_bf16rr $a, $b, Mode)>,
         Requires<[hasBF16Math]>;
@@ -2497,24 +2486,20 @@ def : Pat<(f16 (uint_to_fp i32:$a)), (CVT_f16_u32 $a, CvtRN)>;
 def : Pat<(f16 (uint_to_fp i64:$a)), (CVT_f16_u64 $a, CvtRN)>;
 
 // sint -> bf16
-def : Pat<(bf16 (sint_to_fp i1:$a)), (CVT_bf16_s32 (SELP_b32ii 1, 0, $a), CvtRN)>, 
-      Requires<[hasPTX<78>, hasSM<90>]>;
-def : Pat<(bf16 (sint_to_fp i16:$a)), (CVT_bf16_s16 $a, CvtRN)>,
-      Requires<[hasPTX<78>, hasSM<90>]>;
-def : Pat<(bf16 (sint_to_fp i32:$a)), (CVT_bf16_s32 $a, CvtRN)>,
-      Requires<[hasPTX<78>, hasSM<90>]>;
-def : Pat<(bf16 (sint_to_fp i64:$a)), (CVT_bf16_s64 $a, CvtRN)>,
-      Requires<[hasPTX<78>, hasSM<90>]>;
+let Predicates = [hasPTX<78>, hasSM<90>] in {
+  def : Pat<(bf16 (sint_to_fp i1:$a)), (CVT_bf16_s32 (SELP_b32ii 1, 0, $a), CvtRN)>;
+  def : Pat<(bf16 (sint_to_fp i16:$a)), (CVT_bf16_s16 $a, CvtRN)>;
+  def : Pat<(bf16 (sint_to_fp i32:$a)), (CVT_bf16_s32 $a, CvtRN)>;
+  def : Pat<(bf16 (sint_to_fp i64:$a)), (CVT_bf16_s64 $a, CvtRN)>;
+}
 
 // uint -> bf16
-def : Pat<(bf16 (uint_to_fp i1:$a)), (CVT_bf16_u32 (SELP_b32ii 1, 0, $a), CvtRN)>,
-      Requires<[hasPTX<78>, hasSM<90>]>;
-def : Pat<(bf16 (uint_to_fp i16:$a)), (CVT_bf16_u16 $a, CvtRN)>,
-      Requires<[hasPTX<78>, hasSM<90>]>;
-def : Pat<(bf16 (uint_to_fp i32:$a)), (CVT_bf16_u32 $a, CvtRN)>,
-      Requires<[hasPTX<78>, hasSM<90>]>;
-def : Pat<(bf16 (uint_to_fp i64:$a)), (CVT_bf16_u64 $a, CvtRN)>,
-      Requires<[hasPTX<78>, hasSM<90>]>;
+let Predicates = [hasPTX<78>, hasSM<90>] in {
+  def : Pat<(bf16 (uint_to_fp i1:$a)), (CVT_bf16_u32 (SELP_b32ii 1, 0, $a), CvtRN)>;
+  def : Pat<(bf16 (uint_to_fp i16:$a)), (CVT_bf16_u16 $a, CvtRN)>;
+  def : Pat<(bf16 (uint_to_fp i32:$a)), (CVT_bf16_u32 $a, CvtRN)>;
+  def : Pat<(bf16 (uint_to_fp i64:$a)), (CVT_bf16_u64 $a, CvtRN)>;
+}
 
 // sint -> f32
 def : Pat<(f32 (sint_to_fp  i1:$a)), (CVT_f32_s32 (SELP_b32ii -1, 0, $a), CvtRN)>;
@@ -2565,27 +2550,25 @@ def : Pat<(i16 (fp_to_uint bf16:$a)), (CVT_u16_bf16 $a, CvtRZI)>;
 def : Pat<(i32 (fp_to_uint bf16:$a)), (CVT_u32_bf16 $a, CvtRZI)>;
 def : Pat<(i64 (fp_to_uint bf16:$a)), (CVT_u64_bf16 $a, CvtRZI)>;
 // f32 -> sint
-def : Pat<(i1 (fp_to_sint f32:$a)), (SETP_b32ri $a, 0, CmpEQ)>;
-def : Pat<(i16 (fp_to_sint f32:$a)), (CVT_s16_f32 $a, CvtRZI_FTZ)>,
-      Requires<[doF32FTZ]>;
+let Predicates = [doF32FTZ] in {
+  def : Pat<(i16 (fp_to_sint f32:$a)), (CVT_s16_f32 $a, CvtRZI_FTZ)>;
+  def : Pat<(i32 (fp_to_sint f32:$a)), (CVT_s32_f32 $a, CvtRZI_FTZ)>;
+  def : Pat<(i64 (fp_to_sint f32:$a)), (CVT_s64_f32 $a, CvtRZI_FTZ)>;
+}
+def : Pat<(i1  (fp_to_sint f32:$a)), (SETP_b32ri $a, 0, CmpEQ)>;
 def : Pat<(i16 (fp_to_sint f32:$a)), (CVT_s16_f32 $a, CvtRZI)>;
-def : Pat<(i32 (fp_to_sint f32:$a)), (CVT_s32_f32 $a, CvtRZI_FTZ)>,
-      Requires<[doF32FTZ]>;
 def : Pat<(i32 (fp_to_sint f32:$a)), (CVT_s32_f32 $a, CvtRZI)>;
-def : Pat<(i64 (fp_to_sint f32:$a)), (CVT_s64_f32 $a, CvtRZI_FTZ)>,
-      Requires<[doF32FTZ]>;
 def : Pat<(i64 (fp_to_sint f32:$a)), (CVT_s64_f32 $a, CvtRZI)>;
 
 // f32 -> uint
+let Predicates = [doF32FTZ] in {
+  def : Pat<(i16 (fp_to_uint f32:$a)), (CVT_u16_f32 $a, CvtRZI_FTZ)>;
+  def : Pat<(i32 (fp_to_uint f32:$a)), (CVT_u32_f32 $a, CvtRZI_FTZ)>;
+  def : Pat<(i64 (fp_to_uint f32:$a)), (CVT_u64_f32 $a, CvtRZI_FTZ)>;
+}
 def : Pat<(i1  (fp_to_uint f32:$a)), (SETP_b32ri $a, 0, CmpEQ)>;
-def : Pat<(i16 (fp_to_uint f32:$a)), (CVT_u16_f32 $a, CvtRZI_FTZ)>, 
-      Requires<[doF32FTZ]>;
 def : Pat<(i16 (fp_to_uint f32:$a)), (CVT_u16_f32 $a, CvtRZI)>;
-def : Pat<(i32 (fp_to_uint f32:$a)), (CVT_u32_f32 $a, CvtRZI_FTZ)>,
-      Requires<[doF32FTZ]>;
 def : Pat<(i32 (fp_to_uint f32:$a)), (CVT_u32_f32 $a, CvtRZI)>;
-def : Pat<(i64 (fp_to_uint f32:$a)), (CVT_u64_f32 $a, CvtRZI_FTZ)>,
-      Requires<[doF32FTZ]>;
 def : Pat<(i64 (fp_to_uint f32:$a)), (CVT_u64_f32 $a, CvtRZI)>;
 
 // f64 -> sint
@@ -2707,28 +2690,24 @@ let hasSideEffects = false in {
 
   // PTX 7.1 lets you avoid a temp register and just use _ as a "sink" for the
   // unused high/low part.
-  def I32toI16H_Sink  : NVPTXInst<(outs Int16Regs:$high),
-                             (ins Int32Regs:$s),
-                             "mov.b32 \t{{_, $high}}, $s;",
-                             []>, Requires<[hasPTX<71>]>;
-  def I32toI16L_Sink  : NVPTXInst<(outs Int16Regs:$low),
-                             (ins Int32Regs:$s),
-                             "mov.b32 \t{{$low, _}}, $s;",
-                             []>, Requires<[hasPTX<71>]>;
-  def I64toI32H_Sink  : NVPTXInst<(outs Int32Regs:$high),
-                             (ins Int64Regs:$s),
-                             "mov.b64 \t{{_, $high}}, $s;",
-                             []>, Requires<[hasPTX<71>]>;
-  def I64toI32L_Sink  : NVPTXInst<(outs Int32Regs:$low),
-                             (ins Int64Regs:$s),
-                             "mov.b64 \t{{$low, _}}, $s;",
-                             []>, Requires<[hasPTX<71>]>;
+  let Predicates = [hasPTX<71>] in {
+    def I32toI16H_Sink  : NVPTXInst<(outs Int16Regs:$high), (ins Int32Regs:$s),
+                              "mov.b32 \t{{_, $high}}, $s;", []>;
+    def I32toI16L_Sink  : NVPTXInst<(outs Int16Regs:$low), (ins Int32Regs:$s),
+                              "mov.b32 \t{{$low, _}}, $s;", []>;
+    def I64toI32H_Sink  : NVPTXInst<(outs Int32Regs:$high), (ins Int64Regs:$s),
+                              "mov.b64 \t{{_, $high}}, $s;", []>;
+    def I64toI32L_Sink  : NVPTXInst<(outs Int32Regs:$low), (ins Int64Regs:$s),
+                              "mov.b64 \t{{$low, _}}, $s;", []>;
+  }
 }
 
-def : Pat<(i16 (trunc (srl i32:$s, (i32 16)))), (I32toI16H_Sink i32:$s)>, Requires<[hasPTX<71>]>;
-def : Pat<(i16 (trunc (sra i32:$s, (i32 16)))), (I32toI16H_Sink i32:$s)>, Requires<[hasPTX<71>]>;
-def : Pat<(i32 (trunc (srl i64:$s, (i32 32)))), (I64toI32H_Sink i64:$s)>, Requires<[hasPTX<71>]>;
-def : Pat<(i32 (trunc (sra i64:$s, (i32 32)))), (I64toI32H_Sink i64:$s)>, Requires<[hasPTX<71>]>;
+let Predicates = [hasPTX<71>] in {
+  def : Pat<(i16 (trunc (srl i32:$s, (i32 16)))), (I32toI16H_Sink i32:$s)>;
+  def : Pat<(i16 (trunc (sra i32:$s, (i32 16)))), (I32toI16H_Sink i32:$s)>;
+  def : Pat<(i32 (trunc (srl i64:$s, (i32 32)))), (I64toI32H_Sink i64:$s)>;
+  def : Pat<(i32 (trunc (sra i64:$s, (i32 32)))), (I64toI32H_Sink i64:$s)>;
+}
 
 // Fall back to the old way if we don't have PTX 7.1.
 def : Pat<(i16 (trunc (srl i32:$s, (i32 16)))), (I32toI16H $s)>;
@@ -3061,29 +3040,19 @@ def stacksave :
   SDNode<"NVPTXISD::STACKSAVE", SDTIntLeaf,
          [SDNPHasChain, SDNPSideEffect]>;
 
-def STACKRESTORE_32 :
-  BasicNVPTXInst<(outs), (ins Int32Regs:$ptr),
-            "stackrestore.u32",
-            [(stackrestore i32:$ptr)]>,
-            Requires<[hasPTX<73>, hasSM<52>]>;
-
-def STACKSAVE_32 :
-  BasicNVPTXInst<(outs Int32Regs:$dst), (ins),
-            "stacksave.u32",
-            [(set i32:$dst, (i32 stacksave))]>,
-            Requires<[hasPTX<73>, hasSM<52>]>;
-
-def STACKRESTORE_64 :
-  BasicNVPTXInst<(outs), (ins Int64Regs:$ptr),
-            "stackrestore.u64",
-            [(stackrestore i64:$ptr)]>,
-            Requires<[hasPTX<73>, hasSM<52>]>;
-
-def STACKSAVE_64 :
-  BasicNVPTXInst<(outs Int64Regs:$dst), (ins),
-            "stacksave.u64",
-            [(set i64:$dst, (i64 stacksave))]>,
-            Requires<[hasPTX<73>, hasSM<52>]>;
+let Predicates = [hasPTX<73>, hasSM<52>] in {
+  foreach t = [I32RT, I64RT] in {
+    def STACKRESTORE_ # t.Size :
+      BasicNVPTXInst<(outs), (ins t.RC:$ptr),
+                "stackrestore.u" # t.Size,
+              [(stackrestore t.Ty:$ptr)]>;
+
+    def STACKSAVE_ # t.Size :
+      BasicNVPTXInst<(outs t.RC:$dst), (ins),
+                "stacksave.u" # t.Size,
+              [(set t.Ty:$dst, (t.Ty stacksave))]>;
+  }
+}
 
 include "NVPTXIntrinsics.td"
 
@@ -3124,7 +3093,7 @@ def : Pat <
 ////////////////////////////////////////////////////////////////////////////////
 
 class NVPTXFenceInst<string scope, string sem, Predicate ptx>:
-    NVPTXInst<(outs), (ins), "fence."#sem#"."#scope#";", []>,
+    BasicNVPTXInst<(outs), (ins), "fence."#sem#"."#scope>,
     Requires<[ptx, hasSM<70>]>;
 
 foreach scope = ["sys", "gpu", "cluster", "cta"] in {
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index d14c03791feb..b3c1296cf0ca 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -52,7 +52,7 @@ class PTX {
 def ptx : PTX;
 
 // Generates list of n sequential register names.
-// E.g. RegNames<3,"r">.ret -> ["r0", "r1", "r2" ]
+// E.g. RegNames<3, "r">.ret -> ["r0", "r1", "r2" ]
 class RegSeq<int n, string prefix> {
   list<string> ret = !if(n, !listconcat(RegSeq<!sub(n, 1), prefix>.ret,
                                         [prefix # !sub(n, 1)]),
@@ -137,7 +137,7 @@ defm BARRIER_CTA_ARRIVE : BARRIER2<"barrier.arrive", int_nvvm_barrier_cta_arrive
 
 class INT_BARRIER_CLUSTER<string variant, Intrinsic Intr,
                           list<Predicate> Preds = [hasPTX<78>, hasSM<90>]>:
-        NVPTXInst<(outs), (ins), "barrier.cluster."# variant #";", [(Intr)]>,
+        BasicNVPTXInst<(outs), (ins), "barrier.cluster."# variant, [(Intr)]>,
         Requires<Preds>;
 
 def barrier_cluster_arrive:
@@ -400,13 +400,9 @@ def INT_FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE_SYS :
 //-----------------------------------
 
 multiclass CP_ASYNC_MBARRIER_ARRIVE<string NoInc, string AddrSpace, Intrinsic Intrin> {
-  def _32 : NVPTXInst<(outs), (ins Int32Regs:$addr),
-            !strconcat("cp.async.mbarrier.arrive", NoInc, AddrSpace, ".b64 [$addr];"),
-            [(Intrin i32:$addr)]>,
-    Requires<[hasPTX<70>, hasSM<80>]>;
-  def _64 : NVPTXInst<(outs), (ins Int64Regs:$addr),
-            !strconcat("cp.async.mbarrier.arrive", NoInc, AddrSpace, ".b64 [$addr];"),
-            [(Intrin i64:$addr)]>,
+  def "" : BasicNVPTXInst<(outs), (ins ADDR:$addr),
+            "cp.async.mbarrier.arrive" # NoInc # AddrSpace # ".b64",
+            [(Intrin addr:$addr)]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
 
@@ -420,30 +416,19 @@ defm CP_ASYNC_MBARRIER_ARRIVE_NOINC_SHARED :
   CP_ASYNC_MBARRIER_ARRIVE<".noinc", ".shared", int_nvvm_cp_async_mbarrier_arrive_noinc_shared>;
 
 multiclass CP_ASYNC_SHARED_GLOBAL_I<string cc, string cpsize, Intrinsic Intrin, Intrinsic IntrinS> {
-  def _32 : NVPTXInst<(outs), (ins Int32Regs:$dst, Int32Regs:$src),
-            !strconcat("cp.async.", cc, ".shared.global [$dst], [$src], ", cpsize, ";"),
-            [(Intrin i32:$dst, i32:$src)]>,
-    Requires<[hasPTX<70>, hasSM<80>]>;
-  def _64 : NVPTXInst<(outs), (ins Int64Regs:$dst, Int64Regs:$src),
-            !strconcat("cp.async.", cc, ".shared.global [$dst], [$src], ", cpsize, ";"),
-            [(Intrin i64:$dst, i64:$src)]>,
+  def "" : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src),
+            "cp.async." # cc # ".shared.global" # " [$dst], [$src], " # cpsize # ";",
+            [(Intrin addr:$dst, addr:$src)]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
+
   // Variant with src_size parameter
-  def _32s : NVPTXInst<(outs), (ins Int32Regs:$dst, Int32Regs:$src, Int32Regs:$src_size),
-             !strconcat("cp.async.", cc, ".shared.global [$dst], [$src], ", cpsize, ", $src_size;"),
-             [(IntrinS i32:$dst, i32:$src, i32:$src_size)]>,
-    Requires<[hasPTX<70>, hasSM<80>]>;
-  def _32si: NVPTXInst<(outs), (ins Int32Regs:$dst, Int32Regs:$src, i32imm:$src_size),
-             !strconcat("cp.async.", cc, ".shared.global [$dst], [$src], ", cpsize, ", $src_size;"),
-             [(IntrinS i32:$dst, i32:$src, imm:$src_size)]>,
+  def _s : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, Int32Regs:$src_size),
+             "cp.async." # cc # ".shared.global" # " [$dst], [$src], " # cpsize # ", $src_size;",
+             [(IntrinS addr:$dst, addr:$src, i32:$src_size)]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
-  def _64s : NVPTXInst<(outs), (ins Int64Regs:$dst, Int64Regs:$src, Int32Regs:$src_size),
-             !strconcat("cp.async.", cc, ".shared.global [$dst], [$src], ", cpsize, ", $src_size;"),
-             [(IntrinS i64:$dst, i64:$src, i32:$src_size)]>,
-    Requires<[hasPTX<70>, hasSM<80>]>;
-  def _64si: NVPTXInst<(outs), (ins Int64Regs:$dst, Int64Regs:$src, i32imm:$src_size),
-             !strconcat("cp.async.", cc, ".shared.global [$dst], [$src], ", cpsize, ", $src_size;"),
-             [(IntrinS i64:$dst, i64:$src, imm:$src_size)]>,
+  def _si: NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, i32imm:$src_size),
+             "cp.async." # cc # ".shared.global" # " [$dst], [$src], " # cpsize # ", $src_size;",
+             [(IntrinS addr:$dst, addr:$src, imm:$src_size)]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
 
@@ -513,14 +498,14 @@ class CpAsyncBulkStr<bit mc, bit ch, bit mask = 0> {
 }
 
 multiclass CP_ASYNC_BULK_S2G_INTR<bit has_ch> {
-  def NAME : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, Int32Regs:$size, Int64Regs:$ch),
+  def "" : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, Int32Regs:$size, Int64Regs:$ch),
       !if(has_ch,
           CpAsyncBulkStr<0, 1>.S2G # " [$dst], [$src], $size, $ch;",
           CpAsyncBulkStr<0, 0>.S2G # " [$dst], [$src], $size;"),
       [(int_nvvm_cp_async_bulk_shared_cta_to_global addr:$dst, addr:$src, i32:$size, i64:$ch, !if(has_ch, -1, 0))]>,
       Requires<[hasPTX<80>, hasSM<90>]>;
 
-  def NAME # _BM : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, Int32Regs:$size, Int64Regs:$ch, Int16Regs:$mask),
+  def _BM : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, Int32Regs:$size, Int64Regs:$ch, Int16Regs:$mask),
       !if(has_ch,
           CpAsyncBulkStr<0, 1, 1>.S2G # " [$dst], [$src], $size, $ch, $mask;",
           CpAsyncBulkStr<0, 0, 1>.S2G # " [$dst], [$src], $size, $mask;"),
@@ -533,7 +518,7 @@ defm CP_ASYNC_BULK_S2G_CH : CP_ASYNC_BULK_S2G_INTR<has_ch = 1>;
 multiclass CP_ASYNC_BULK_G2S_INTR<bit has_ch> {
   defvar Intr = int_nvvm_cp_async_bulk_global_to_shared_cluster;
 
-  def NAME : NVPTXInst<(outs),
+  def "" : NVPTXInst<(outs),
       (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,
            Int32Regs:$size, Int16Regs:$mask, Int64Regs:$ch),
       !if(has_ch,
@@ -542,7 +527,7 @@ multiclass CP_ASYNC_BULK_G2S_INTR<bit has_ch> {
       [(Intr addr:$dst, addr:$mbar, addr:$src, i32:$size, i16:$mask, i64:$ch, 0, !if(has_ch, -1, 0))]>,
       Requires<[hasPTX<80>, hasSM<90>]>;
 
-  def NAME # _MC : NVPTXInst<(outs),
+  def _MC : NVPTXInst<(outs),
       (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,
            Int32Regs:$size, Int16Regs:$mask, Int64Regs:$ch),
       !if(has_ch,
@@ -561,7 +546,7 @@ def CP_ASYNC_BULK_CTA_TO_CLUSTER : NVPTXInst<(outs),
   Requires<[hasPTX<80>, hasSM<90>]>;
 
 multiclass CP_ASYNC_BULK_PREFETCH_INTR<bit has_ch> {
-  def NAME : NVPTXInst<(outs), (ins ADDR:$src, Int32Regs:$size, Int64Regs:$ch),
+  def "" : NVPTXInst<(outs), (ins ADDR:$src, Int32Regs:$size, Int64Regs:$ch),
       !if(has_ch,
           "cp.async.bulk.prefetch.L2.global.L2::cache_hint" # " [$src], $size, $ch;",
           "cp.async.bulk.prefetch.L2.global" # " [$src], $size;"),
@@ -593,10 +578,14 @@ class G2S_STRINGS<int dim, string mode, bit mc, bit ch, bit is_shared32 = 0> {
                      # !if(!eq(mode, "tile"), "_TILE", "_IM2COL");
 }
 
+def CTAGroupFlags : Operand<i32> {
+  let PrintMethod = "printCTAGroup";
+}
+
 multiclass CP_ASYNC_BULK_TENSOR_G2S_INTR<int dim, bit is_shared32, string mode> {
   defvar dims_dag = !dag(ins, !listsplat(Int32Regs, dim), !foreach(i, !range(dim), "d" # i));
   defvar dims_str = !interleave(!foreach(i, !range(dim), "$d" # i), ", ");
-  defvar asm_str_default = " [$dst], [$tmap, {{" # dims_str # "}}], [$mbar]";
+  defvar asm_str_default = "$cg [$dst], [$tmap, {{" # dims_str # "}}], [$mbar]";
   defvar rc = !if(is_shared32, Int32Regs, Int64Regs);
 
   defvar num_im2col = !if(!ge(dim, 3), !add(dim, -2), 0);
@@ -609,20 +598,23 @@ multiclass CP_ASYNC_BULK_TENSOR_G2S_INTR<int dim, bit is_shared32, string mode>
   defvar asm_str = !if(!eq(mode, "im2col"),
     !strconcat(asm_str_default, im2col_asm_str), asm_str_default);
 
-  def NAME: NVPTXInst<(outs),
-            !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag),
+  def "" : NVPTXInst<(outs),
+            !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins CTAGroupFlags:$cg)),
             !strconcat(G2S_STRINGS<dim, mode, 0, 0>.inst_name, asm_str, ";"), []>,
             Requires<[hasPTX<80>, hasSM<90>]>;
-  def NAME # _MC: NVPTXInst<(outs),
-                  !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int16Regs:$mc)),
+  def _MC : NVPTXInst<(outs),
+                  !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag,
+                       (ins Int16Regs:$mc, CTAGroupFlags:$cg)),
                   !strconcat(G2S_STRINGS<dim, mode, 1, 0>.inst_name, asm_str, ", $mc;"), []>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
-  def NAME # _CH: NVPTXInst<(outs),
-                  !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int64Regs:$ch)),
+  def _CH : NVPTXInst<(outs),
+                  !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag,
+                       (ins Int64Regs:$ch, CTAGroupFlags:$cg)),
                   !strconcat(G2S_STRINGS<dim, mode, 0, 1>.inst_name, asm_str, ", $ch;"), []>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
-  def NAME # _MC_CH: NVPTXInst<(outs),
-                     !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int16Regs:$mc, Int64Regs:$ch)),
+  def _MC_CH : NVPTXInst<(outs),
+                     !con((ins rc:$dst, rc:$mbar, Int64Regs:$tmap), dims_dag, im2col_dag,
+                          (ins Int16Regs:$mc, Int64Regs:$ch, CTAGroupFlags:$cg)),
                      !strconcat(G2S_STRINGS<dim, mode, 1, 1>.inst_name, asm_str, ", $mc, $ch;"), []>,
                      Requires<[hasPTX<80>, hasSM<90>]>;
 }
@@ -661,11 +653,11 @@ multiclass CP_ASYNC_BULK_TENSOR_S2G_INTR<int dim, bit shared32, string mode> {
   defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";
   defvar rc = !if(shared32, Int32Regs, Int64Regs);
 
-  def NAME: NVPTXInst<(outs),
+  def "" : NVPTXInst<(outs),
             !con((ins rc:$src, Int64Regs:$tmap), dims_dag),
             !strconcat(S2G_STRINGS<dim, mode, 0>.inst_name, asm_str, ";"), []>,
             Requires<[hasPTX<80>, hasSM<90>]>;
-  def NAME # _CH: NVPTXInst<(outs),
+  def _CH : NVPTXInst<(outs),
                   !con((ins rc:$src, Int64Regs:$tmap), dims_dag, (ins Int64Regs:$ch)),
                   !strconcat(S2G_STRINGS<dim, mode, 1>.inst_name, asm_str, ", $ch;"), []>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
@@ -685,11 +677,11 @@ multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, bit shared32, string mode>
   defvar prefix = "cp.reduce.async.bulk.tensor" # "." # dim # "d" # ".global.shared::cta";
   defvar suffix = "." # mode # ".bulk_group";
 
-  def NAME: NVPTXInst<(outs),
+  def "" : NVPTXInst<(outs),
             !con((ins rc:$src, Int64Regs:$tmap), dims_dag, (ins TMAReductionFlags:$red_op)),
             !strconcat(prefix, "${red_op}", suffix, asm_str, ";"), []>,
             Requires<[hasPTX<80>, hasSM<90>]>;
-  def NAME # _CH: NVPTXInst<(outs),
+  def _CH : NVPTXInst<(outs),
                   !con((ins rc:$src, Int64Regs:$tmap), dims_dag, (ins Int64Regs:$ch, TMAReductionFlags:$red_op)),
                   !strconcat(prefix, "${red_op}", suffix, ".L2::cache_hint", asm_str, ", $ch;"), []>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
@@ -735,11 +727,11 @@ multiclass CP_ASYNC_BULK_TENSOR_PREFETCH_INTR<int dim, string mode> {
   defvar asm_str = !if(!eq(mode, "im2col"),
     !strconcat(asm_str_default, im2col_asm_str), asm_str_default);
 
-  def NAME: NVPTXInst<(outs),
+  def "" : NVPTXInst<(outs),
             !con((ins Int64Regs:$tmap), dims_dag, im2col_dag),
             !strconcat(PREFETCH_STRINGS<dim, mode, 0>.inst_name, asm_str, ";"), []>,
             Requires<[hasPTX<80>, hasSM<90>]>;
-  def NAME # _CH: NVPTXInst<(outs),
+  def _CH : NVPTXInst<(outs),
                   !con((ins Int64Regs:$tmap), dims_dag, im2col_dag, (ins Int64Regs:$ch)),
                   !strconcat(PREFETCH_STRINGS<dim, mode, 1>.inst_name, asm_str, ", $ch;"), []>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
@@ -755,10 +747,10 @@ foreach dim = [1, 2, 3, 4, 5] in {
 //Prefetch and Prefetchu 
 
 class PREFETCH_INTRS<string InstName> :
-          NVPTXInst<(outs), (ins Int64Regs:$addr),
-          InstName # " [$addr];",
+          BasicNVPTXInst<(outs), (ins ADDR:$addr),
+          InstName,
           [(!cast<Intrinsic>(!strconcat("int_nvvm_",
-          !subst(".", "_", InstName))) i64:$addr)]>,
+          !subst(".", "_", InstName))) addr:$addr)]>,
           Requires<[hasPTX<80>, hasSM<90>]>;
    
 
@@ -769,36 +761,39 @@ def PREFETCH_LOCAL_L1  : PREFETCH_INTRS<"prefetch.local.L1">;
 def PREFETCH_GLOBAL_L2 : PREFETCH_INTRS<"prefetch.global.L2">;
 def PREFETCH_LOCAL_L2 : PREFETCH_INTRS<"prefetch.local.L2">;
 
-def PREFETCH_GLOBAL_L2_EVICT_NORMAL : NVPTXInst<(outs), (ins Int64Regs:$addr),
-                                      "prefetch.global.L2::evict_normal" # " [$addr];",
-                                      [(!cast<Intrinsic>("int_nvvm_prefetch_global_L2_evict_normal") i64:$addr)]>,
+def PREFETCH_GLOBAL_L2_EVICT_NORMAL : BasicNVPTXInst<(outs), (ins ADDR:$addr),
+                                      "prefetch.global.L2::evict_normal",
+                                      [(int_nvvm_prefetch_global_L2_evict_normal addr:$addr)]>,
                                       Requires<[hasPTX<80>, hasSM<90>]>;
 
-def PREFETCH_GLOBAL_L2_EVICT_LAST   : NVPTXInst<(outs), (ins Int64Regs:$addr),
-                                      "prefetch.global.L2::evict_last" # " [$addr];",
-                                      [(!cast<Intrinsic>("int_nvvm_prefetch_global_L2_evict_last") i64:$addr)]>,
+def PREFETCH_GLOBAL_L2_EVICT_LAST   : BasicNVPTXInst<(outs), (ins ADDR:$addr),
+                                      "prefetch.global.L2::evict_last",
+                                      [(int_nvvm_prefetch_global_L2_evict_last addr:$addr)]>,
                                       Requires<[hasPTX<80>, hasSM<90>]>;
 
 
 def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1">;
 
 //Applypriority intrinsics
-class APPLYPRIORITY_L2_INTRS<string addr> :
-          NVPTXInst<(outs), (ins Int64Regs:$addr, Int64Regs:$size),
-          StrJoin<".", ["applypriority", addr , "L2::evict_normal"]>.ret # " [$addr], $size;",
-          [(!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_applypriority", addr , "L2_evict_normal"]>.ret)
-          i64:$addr, i64:$size)]>,
+class APPLYPRIORITY_L2_INTRS<string addrspace> :
+          BasicNVPTXInst<(outs), (ins ADDR:$addr, Int64Regs:$size),
+          StrJoin<".", ["applypriority", addrspace , "L2::evict_normal"]>.ret,
+          [(!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_applypriority", addrspace , "L2_evict_normal"]>.ret)
+          addr:$addr, i64:$size)]>,
           Requires<[hasPTX<74>, hasSM<80>]>;
 
 def APPLYPRIORITY_L2_EVICT_NORMAL        : APPLYPRIORITY_L2_INTRS<"">;
 def APPLYPRIORITY_GLOBAL_L2_EVICT_NORMAL : APPLYPRIORITY_L2_INTRS<"global">;
 
 //Discard Intrinsics
-class DISCARD_L2_INTRS<string Addr> :
-          NVPTXInst<(outs), (ins Int64Regs:$addr),
-          StrJoin<".", ["discard", Addr , "L2"]>.ret # " [$addr], 128;",
-          [(!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_discard", Addr , "L2"]>.ret)
-          i64:$addr, (i64 128))]>,
+
+def discard_size_imm : TImmLeaf<i64, [{ return Imm == 128; }]>;
+
+class DISCARD_L2_INTRS<string addrspace> :
+          BasicNVPTXInst<(outs), (ins ADDR:$addr, i64imm:$size),
+          StrJoin<".", ["discard", addrspace , "L2"]>.ret,
+          [(!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_discard", addrspace , "L2"]>.ret)
+          addr:$addr, discard_size_imm:$size)]>,
           Requires<[hasPTX<74>, hasSM<80>]>;
 
 def DISCARD_L2        : DISCARD_L2_INTRS<"">;
@@ -809,8 +804,8 @@ def DISCARD_GLOBAL_L2 : DISCARD_L2_INTRS<"global">;
 //-----------------------------------
 
 multiclass MBARRIER_INIT<string AddrSpace, Intrinsic Intrin> {
-  def "" : NVPTXInst<(outs), (ins ADDR:$addr, Int32Regs:$count),
-           "mbarrier.init" # AddrSpace # ".b64 [$addr], $count;",
+  def "" : BasicNVPTXInst<(outs), (ins ADDR:$addr, Int32Regs:$count),
+           "mbarrier.init" # AddrSpace # ".b64",
     [(Intrin addr:$addr, i32:$count)]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
@@ -820,8 +815,8 @@ defm MBARRIER_INIT_SHARED : MBARRIER_INIT<".shared",
                                           int_nvvm_mbarrier_init_shared>;
 
 multiclass MBARRIER_INVAL<string AddrSpace, Intrinsic Intrin> {
-  def "" : NVPTXInst<(outs), (ins ADDR:$addr),
-           "mbarrier.inval" # AddrSpace # ".b64 [$addr];",
+  def "" : BasicNVPTXInst<(outs), (ins ADDR:$addr),
+           "mbarrier.inval" # AddrSpace # ".b64",
     [(Intrin addr:$addr)]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
@@ -831,8 +826,8 @@ defm MBARRIER_INVAL_SHARED : MBARRIER_INVAL<".shared",
                                             int_nvvm_mbarrier_inval_shared>;
 
 multiclass MBARRIER_ARRIVE<string AddrSpace, Intrinsic Intrin> {
-  def "" : NVPTXInst<(outs Int64Regs:$state), (ins ADDR:$addr),
-           "mbarrier.arrive" # AddrSpace # ".b64 $state, [$addr];",
+  def "" : BasicNVPTXInst<(outs Int64Regs:$state), (ins ADDR:$addr),
+           "mbarrier.arrive" # AddrSpace # ".b64",
     [(set i64:$state, (Intrin addr:$addr))]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
@@ -842,9 +837,9 @@ defm MBARRIER_ARRIVE_SHARED :
   MBARRIER_ARRIVE<".shared", int_nvvm_mbarrier_arrive_shared>;
 
 multiclass MBARRIER_ARRIVE_NOCOMPLETE<string AddrSpace, Intrinsic Intrin> {
-  def "" : NVPTXInst<(outs Int64Regs:$state),
+  def "" : BasicNVPTXInst<(outs Int64Regs:$state),
            (ins ADDR:$addr, Int32Regs:$count),
-           "mbarrier.arrive.noComplete" # AddrSpace # ".b64 $state, [$addr], $count;",
+           "mbarrier.arrive.noComplete" # AddrSpace # ".b64",
     [(set i64:$state, (Intrin addr:$addr, i32:$count))]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
@@ -855,8 +850,8 @@ defm MBARRIER_ARRIVE_NOCOMPLETE_SHARED :
   MBARRIER_ARRIVE_NOCOMPLETE<".shared", int_nvvm_mbarrier_arrive_noComplete_shared>;
 
 multiclass MBARRIER_ARRIVE_DROP<string AddrSpace, Intrinsic Intrin> {
-  def "" : NVPTXInst<(outs Int64Regs:$state), (ins ADDR:$addr),
-           "mbarrier.arrive_drop" # AddrSpace # ".b64 $state, [$addr];",
+  def "" : BasicNVPTXInst<(outs Int64Regs:$state), (ins ADDR:$addr),
+           "mbarrier.arrive_drop" # AddrSpace # ".b64",
            [(set i64:$state, (Intrin addr:$addr))]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
@@ -867,9 +862,9 @@ defm MBARRIER_ARRIVE_DROP_SHARED :
   MBARRIER_ARRIVE_DROP<".shared", int_nvvm_mbarrier_arrive_drop_shared>;
 
 multiclass MBARRIER_ARRIVE_DROP_NOCOMPLETE<string AddrSpace, Intrinsic Intrin> {
-  def "" : NVPTXInst<(outs Int64Regs:$state),
+  def "" : BasicNVPTXInst<(outs Int64Regs:$state),
            (ins ADDR:$addr, Int32Regs:$count),
-           "mbarrier.arrive_drop.noComplete" # AddrSpace # ".b64 $state, [$addr], $count;",
+           "mbarrier.arrive_drop.noComplete" # AddrSpace # ".b64",
            [(set i64:$state, (Intrin addr:$addr, i32:$count))]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
@@ -881,8 +876,8 @@ defm MBARRIER_ARRIVE_DROP_NOCOMPLETE_SHARED :
                        int_nvvm_mbarrier_arrive_drop_noComplete_shared>;
 
 multiclass MBARRIER_TEST_WAIT<string AddrSpace, Intrinsic Intrin> {
-  def "" : NVPTXInst<(outs Int1Regs:$res), (ins ADDR:$addr, Int64Regs:$state),
-           "mbarrier.test_wait" # AddrSpace # ".b64 $res, [$addr], $state;",
+  def "" : BasicNVPTXInst<(outs Int1Regs:$res), (ins ADDR:$addr, Int64Regs:$state),
+           "mbarrier.test_wait" # AddrSpace # ".b64",
            [(set i1:$res, (Intrin addr:$addr, i64:$state))]>,
     Requires<[hasPTX<70>, hasSM<80>]>;
 }
@@ -997,6 +992,16 @@ def INT_NVVM_NANOSLEEP_I : BasicNVPTXInst<(outs), (ins i32imm:$i), "nanosleep.u3
 def INT_NVVM_NANOSLEEP_R : BasicNVPTXInst<(outs), (ins Int32Regs:$i), "nanosleep.u32",
                              [(int_nvvm_nanosleep i32:$i)]>,
         Requires<[hasPTX<63>, hasSM<70>]>;
+
+let hasSideEffects = 1 in {
+// Performance Monitor events
+def INT_PM_EVENT_MASK : BasicNVPTXInst<(outs),
+                        (ins i16imm:$mask),
+                        "pmevent.mask",
+                        [(int_nvvm_pm_event_mask timm:$mask)]>,
+                        Requires<[hasSM<20>, hasPTX<30>]>;
+} // hasSideEffects
+
 //
 // Min Max
 //
@@ -1515,15 +1520,18 @@ def INT_NVVM_SQRT_RZ_D : F_MATH_1<"sqrt.rz.f64", F64RT, F64RT, int_nvvm_sqrt_rz_
 def INT_NVVM_SQRT_RM_D : F_MATH_1<"sqrt.rm.f64", F64RT, F64RT, int_nvvm_sqrt_rm_d>;
 def INT_NVVM_SQRT_RP_D : F_MATH_1<"sqrt.rp.f64", F64RT, F64RT, int_nvvm_sqrt_rp_d>;
 
+def fsqrt_approx : PatFrags<(ops node:$a),
+                            [(fsqrt node:$a),
+                             (int_nvvm_sqrt_f node:$a)], [{
+  return !usePrecSqrtF32(N);
+}]>;
+
 // nvvm_sqrt intrinsic
-def : Pat<(int_nvvm_sqrt_f f32:$a),
-          (INT_NVVM_SQRT_RN_FTZ_F $a)>, Requires<[doF32FTZ, do_SQRTF32_RN]>;
-def : Pat<(int_nvvm_sqrt_f f32:$a),
-          (INT_NVVM_SQRT_RN_F $a)>, Requires<[do_SQRTF32_RN]>;
-def : Pat<(int_nvvm_sqrt_f f32:$a),
-          (INT_NVVM_SQRT_APPROX_FTZ_F $a)>, Requires<[doF32FTZ]>;
-def : Pat<(int_nvvm_sqrt_f f32:$a),
-          (INT_NVVM_SQRT_APPROX_F $a)>;
+def : Pat<(int_nvvm_sqrt_f f32:$a), (INT_NVVM_SQRT_RN_FTZ_F $a)>, Requires<[doF32FTZ]>;
+def : Pat<(int_nvvm_sqrt_f f32:$a), (INT_NVVM_SQRT_RN_F $a)>;
+
+def : Pat<(fsqrt_approx f32:$a), (INT_NVVM_SQRT_APPROX_FTZ_F $a)>, Requires<[doF32FTZ]>;
+def : Pat<(fsqrt_approx f32:$a), (INT_NVVM_SQRT_APPROX_F $a)>;
 
 //
 // Rsqrt
@@ -1546,20 +1554,14 @@ def: Pat<(fdiv f32imm_1, (int_nvvm_sqrt_approx_f f32:$a)),
 def: Pat<(fdiv f32imm_1, (int_nvvm_sqrt_approx_ftz_f f32:$a)),
          (INT_NVVM_RSQRT_APPROX_FTZ_F $a)>,
          Requires<[doRsqrtOpt]>;
-// same for int_nvvm_sqrt_f when non-precision sqrt is requested
-def: Pat<(fdiv f32imm_1, (int_nvvm_sqrt_f f32:$a)),
-         (INT_NVVM_RSQRT_APPROX_F $a)>,
-         Requires<[doRsqrtOpt, do_SQRTF32_APPROX, doNoF32FTZ]>;
-def: Pat<(fdiv f32imm_1, (int_nvvm_sqrt_f f32:$a)),
-         (INT_NVVM_RSQRT_APPROX_FTZ_F $a)>,
-         Requires<[doRsqrtOpt, do_SQRTF32_APPROX, doF32FTZ]>;
 
-def: Pat<(fdiv f32imm_1, (fsqrt f32:$a)),
+// same for int_nvvm_sqrt_f when non-precision sqrt is requested
+def: Pat<(fdiv f32imm_1, (fsqrt_approx f32:$a)),
          (INT_NVVM_RSQRT_APPROX_F $a)>,
-         Requires<[doRsqrtOpt, do_SQRTF32_APPROX, doNoF32FTZ]>;
-def: Pat<(fdiv f32imm_1, (fsqrt f32:$a)),
+         Requires<[doRsqrtOpt, doNoF32FTZ]>;
+def: Pat<(fdiv f32imm_1, (fsqrt_approx f32:$a)),
          (INT_NVVM_RSQRT_APPROX_FTZ_F $a)>,
-         Requires<[doRsqrtOpt, do_SQRTF32_APPROX, doF32FTZ]>;
+         Requires<[doRsqrtOpt, doF32FTZ]>;
 //
 // Add
 //
@@ -1780,93 +1782,74 @@ def : Pat<(int_nvvm_ff_to_e5m2x2_rn f32:$a, f32:$b),
 def : Pat<(int_nvvm_ff_to_e5m2x2_rn_relu f32:$a, f32:$b),
           (CVT_e5m2x2_f32 $a, $b, CvtRN_RELU)>;
 
-def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn Int32Regs:$a),
+def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn v2f16:$a),
           (CVT_e4m3x2_f16x2 $a, CvtRN)>;
-def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn_relu Int32Regs:$a),
+def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn_relu v2f16:$a),
           (CVT_e4m3x2_f16x2 $a, CvtRN_RELU)>;
-def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn Int32Regs:$a),
+def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn v2f16:$a),
           (CVT_e5m2x2_f16x2 $a, CvtRN)>;
-def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn_relu Int32Regs:$a),
+def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn_relu v2f16:$a),
           (CVT_e5m2x2_f16x2 $a, CvtRN_RELU)>;
 
-def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn Int16Regs:$a),
+def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn i16:$a),
           (CVT_f16x2_e4m3x2 $a, CvtRN)>;
-def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn_relu Int16Regs:$a),
+def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn_relu i16:$a),
           (CVT_f16x2_e4m3x2 $a, CvtRN_RELU)>;
-def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn Int16Regs:$a),
+def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn i16:$a),
           (CVT_f16x2_e5m2x2 $a, CvtRN)>;
-def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn_relu Int16Regs:$a),
+def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn_relu i16:$a),
           (CVT_f16x2_e5m2x2 $a, CvtRN_RELU)>;
 
-def : Pat<(int_nvvm_ff_to_e2m3x2_rn_satfinite f32:$a, f32:$b),
-          (CVT_e2m3x2_f32_sf $a, $b, CvtRN)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_ff_to_e2m3x2_rn_relu_satfinite f32:$a, f32:$b),
-          (CVT_e2m3x2_f32_sf $a, $b, CvtRN_RELU)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_ff_to_e3m2x2_rn_satfinite f32:$a, f32:$b),
-          (CVT_e3m2x2_f32_sf $a, $b, CvtRN)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_ff_to_e3m2x2_rn_relu_satfinite f32:$a, f32:$b),
-          (CVT_e3m2x2_f32_sf $a, $b, CvtRN_RELU)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-
-def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn i16:$a),
-          (CVT_f16x2_e2m3x2 $a, CvtRN)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn_relu i16:$a),
-          (CVT_f16x2_e2m3x2 $a, CvtRN_RELU)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn i16:$a),
-          (CVT_f16x2_e3m2x2 $a, CvtRN)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn_relu i16:$a),
-          (CVT_f16x2_e3m2x2 $a, CvtRN_RELU)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-      
-def : Pat<(int_nvvm_ff_to_e2m1x2_rn_satfinite f32:$a, f32:$b),
-          (CVT_e2m1x2_f32_sf $a, $b, CvtRN)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_ff_to_e2m1x2_rn_relu_satfinite f32:$a, f32:$b),
-          (CVT_e2m1x2_f32_sf $a, $b, CvtRN_RELU)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-      
-def : Pat<(int_nvvm_e2m1x2_to_f16x2_rn Int16Regs:$a),
-          (CVT_f16x2_e2m1x2 $a, CvtRN)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_e2m1x2_to_f16x2_rn_relu Int16Regs:$a),
-          (CVT_f16x2_e2m1x2 $a, CvtRN_RELU)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-
-def : Pat<(int_nvvm_ff_to_ue8m0x2_rz f32:$a, f32:$b),
-          (CVT_ue8m0x2_f32 $a, $b, CvtRZ)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_ff_to_ue8m0x2_rz_satfinite f32:$a, f32:$b),
-          (CVT_ue8m0x2_f32_sf $a, $b, CvtRZ)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_ff_to_ue8m0x2_rp f32:$a, f32:$b),
-          (CVT_ue8m0x2_f32 $a, $b, CvtRP)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_ff_to_ue8m0x2_rp_satfinite f32:$a, f32:$b),
-          (CVT_ue8m0x2_f32_sf $a, $b, CvtRP)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-          
-def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz Int32Regs:$a),
-          (CVT_ue8m0x2_bf16x2 $a, CvtRZ)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz_satfinite Int32Regs:$a),
-          (CVT_ue8m0x2_bf16x2_sf $a, CvtRZ)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp Int32Regs:$a),
-          (CVT_ue8m0x2_bf16x2 $a, CvtRP)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp_satfinite Int32Regs:$a),
-          (CVT_ue8m0x2_bf16x2_sf $a, CvtRP)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
-          
-def : Pat<(int_nvvm_ue8m0x2_to_bf16x2 i16:$a),
-          (CVT_bf16x2_ue8m0x2 $a)>,
-      Requires<[hasPTX<86>, hasSM<100>, hasArchAccelFeatures]>;
+let Predicates = [hasPTX<86>, hasSM<100>, hasArchAccelFeatures] in {
+  def : Pat<(int_nvvm_ff_to_e2m3x2_rn_satfinite f32:$a, f32:$b),
+            (CVT_e2m3x2_f32_sf $a, $b, CvtRN)>;
+  def : Pat<(int_nvvm_ff_to_e2m3x2_rn_relu_satfinite f32:$a, f32:$b),
+            (CVT_e2m3x2_f32_sf $a, $b, CvtRN_RELU)>;
+  def : Pat<(int_nvvm_ff_to_e3m2x2_rn_satfinite f32:$a, f32:$b),
+            (CVT_e3m2x2_f32_sf $a, $b, CvtRN)>;
+  def : Pat<(int_nvvm_ff_to_e3m2x2_rn_relu_satfinite f32:$a, f32:$b),
+            (CVT_e3m2x2_f32_sf $a, $b, CvtRN_RELU)>;
+
+  def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn i16:$a),
+            (CVT_f16x2_e2m3x2 $a, CvtRN)>;
+  def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn_relu i16:$a),
+            (CVT_f16x2_e2m3x2 $a, CvtRN_RELU)>;
+  def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn i16:$a),
+            (CVT_f16x2_e3m2x2 $a, CvtRN)>;
+  def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn_relu i16:$a),
+            (CVT_f16x2_e3m2x2 $a, CvtRN_RELU)>;
+
+  def : Pat<(int_nvvm_ff_to_e2m1x2_rn_satfinite f32:$a, f32:$b),
+            (CVT_e2m1x2_f32_sf $a, $b, CvtRN)>;
+  def : Pat<(int_nvvm_ff_to_e2m1x2_rn_relu_satfinite f32:$a, f32:$b),
+            (CVT_e2m1x2_f32_sf $a, $b, CvtRN_RELU)>;
+
+  def : Pat<(int_nvvm_e2m1x2_to_f16x2_rn i16:$a),
+            (CVT_f16x2_e2m1x2 $a, CvtRN)>;
+  def : Pat<(int_nvvm_e2m1x2_to_f16x2_rn_relu i16:$a),
+            (CVT_f16x2_e2m1x2 $a, CvtRN_RELU)>;
+
+  def : Pat<(int_nvvm_ff_to_ue8m0x2_rz f32:$a, f32:$b),
+            (CVT_ue8m0x2_f32 $a, $b, CvtRZ)>;
+  def : Pat<(int_nvvm_ff_to_ue8m0x2_rz_satfinite f32:$a, f32:$b),
+            (CVT_ue8m0x2_f32_sf $a, $b, CvtRZ)>;
+  def : Pat<(int_nvvm_ff_to_ue8m0x2_rp f32:$a, f32:$b),
+            (CVT_ue8m0x2_f32 $a, $b, CvtRP)>;
+  def : Pat<(int_nvvm_ff_to_ue8m0x2_rp_satfinite f32:$a, f32:$b),
+            (CVT_ue8m0x2_f32_sf $a, $b, CvtRP)>;
+
+  def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz v2bf16:$a),
+            (CVT_ue8m0x2_bf16x2 $a, CvtRZ)>;
+  def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz_satfinite v2bf16:$a),
+            (CVT_ue8m0x2_bf16x2_sf $a, CvtRZ)>;
+  def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp v2bf16:$a),
+            (CVT_ue8m0x2_bf16x2 $a, CvtRP)>;
+  def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp_satfinite v2bf16:$a),
+            (CVT_ue8m0x2_bf16x2_sf $a, CvtRP)>;
+
+  def : Pat<(int_nvvm_ue8m0x2_to_bf16x2 i16:$a),
+            (CVT_bf16x2_ue8m0x2 $a)>;
+}
 
 //
 // FNS
@@ -1910,14 +1893,14 @@ class ATOMIC_GENERIC_CHK <dag frag>
 
 multiclass F_ATOMIC_2<RegTyInfo t, string sem_str, string as_str, string op_str,
                       SDPatternOperator op, list<Predicate> preds> {
-  defvar asm_str = "atom" # sem_str # as_str # "." # op_str # " \t$dst, [$addr], $b;";
+  defvar asm_str = "atom" # sem_str # as_str # "." # op_str;
   let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
-    def r : NVPTXInst<(outs t.RC:$dst), (ins ADDR:$addr, t.RC:$b),
+    def r : BasicNVPTXInst<(outs t.RC:$dst), (ins ADDR:$addr, t.RC:$b),
       asm_str,
       [(set t.Ty:$dst, (op addr:$addr, t.Ty:$b))]>,
     Requires<preds>;
     if t.SupportsImm then
-      def i : NVPTXInst<(outs t.RC:$dst), (ins ADDR:$addr, t.Imm:$b),
+      def i : BasicNVPTXInst<(outs t.RC:$dst), (ins ADDR:$addr, t.Imm:$b),
         asm_str,
         [(set t.Ty:$dst, (op addr:$addr, (t.Ty t.ImmNode:$b)))]>,
       Requires<preds>;
@@ -1927,27 +1910,27 @@ multiclass F_ATOMIC_2<RegTyInfo t, string sem_str, string as_str, string op_str,
 // has 3 operands
 multiclass F_ATOMIC_3<RegTyInfo t, string sem_str, string as_str, string op_str,
                       SDPatternOperator op, list<Predicate> preds> {
-  defvar asm_str = "atom" # sem_str # as_str # "." # op_str # " \t$dst, [$addr], $b, $c;";
+  defvar asm_str = "atom" # sem_str # as_str # "." # op_str;
   let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
-    def rr : NVPTXInst<(outs t.RC:$dst),
+    def rr : BasicNVPTXInst<(outs t.RC:$dst),
       (ins ADDR:$addr, t.RC:$b, t.RC:$c),
       asm_str,
       [(set t.Ty:$dst, (op addr:$addr, t.Ty:$b, t.Ty:$c))]>,
     Requires<preds>;
 
-    def ir : NVPTXInst<(outs t.RC:$dst),
+    def ir : BasicNVPTXInst<(outs t.RC:$dst),
       (ins ADDR:$addr, t.Imm:$b, t.RC:$c),
       asm_str,
       [(set t.Ty:$dst, (op addr:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c))]>,
     Requires<preds>;
 
-    def ri : NVPTXInst<(outs t.RC:$dst),
+    def ri : BasicNVPTXInst<(outs t.RC:$dst),
       (ins ADDR:$addr, t.RC:$b, t.Imm:$c),
       asm_str,
       [(set t.Ty:$dst, (op addr:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c)))]>,
     Requires<preds>;
 
-    def ii : NVPTXInst<(outs t.RC:$dst),
+    def ii : BasicNVPTXInst<(outs t.RC:$dst),
       (ins ADDR:$addr, t.Imm:$b, t.Imm:$c),
       asm_str,
       [(set t.Ty:$dst, (op addr:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c)))]>,
@@ -2090,7 +2073,7 @@ multiclass ATOM3S_impl<string OpStr, string IntTypeStr, string TypeStr,
     // For now we only need variants for generic space pointers.
     foreach space = ["gen"] in {
       defm _#scope#space : ATOM3N_impl<OpStr, IntTypeStr, TypeStr, scope, space,
-                         t, !listconcat(Preds,[hasAtomScope])>;
+                         t, !listconcat(Preds, [hasAtomScope])>;
     }
   }
 }
@@ -4444,1956 +4427,616 @@ defm SULD_3D_V4I32_ZERO : SULD_3D_V4<"suld.b.3d.v4.b32.zero", Int32Regs>;
 //-----------------------------------
 
 let IsSurfTexQuery = true in {
-def TXQ_CHANNEL_ORDER_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.channel_order.b32 \t$d, [$a];",
-              []>;
-def TXQ_CHANNEL_ORDER_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.channel_order.b32 \t$d, [$a];",
-              []>;
-def TXQ_CHANNEL_DATA_TYPE_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.channel_data_type.b32 \t$d, [$a];",
-              []>;
-def TXQ_CHANNEL_DATA_TYPE_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.channel_data_type.b32 \t$d, [$a];",
-              []>;
-def TXQ_WIDTH_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.width.b32 \t$d, [$a];",
-              []>;
-def TXQ_WIDTH_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.width.b32 \t$d, [$a];",
-              []>;
-def TXQ_HEIGHT_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.height.b32 \t$d, [$a];",
-              []>;
-def TXQ_HEIGHT_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.height.b32 \t$d, [$a];",
-              []>;
-def TXQ_DEPTH_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.depth.b32 \t$d, [$a];",
-              []>;
-def TXQ_DEPTH_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.depth.b32 \t$d, [$a];",
-              []>;
-def TXQ_ARRAY_SIZE_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.array_size.b32 \t$d, [$a];",
-              []>;
-def TXQ_ARRAY_SIZE_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.array_size.b32 \t$d, [$a];",
-              []>;
-def TXQ_NUM_SAMPLES_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.num_samples.b32 \t$d, [$a];",
-              []>;
-def TXQ_NUM_SAMPLES_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.num_samples.b32 \t$d, [$a];",
-              []>;
-def TXQ_NUM_MIPMAP_LEVELS_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "txq.num_mipmap_levels.b32 \t$d, [$a];",
-              []>;
-def TXQ_NUM_MIPMAP_LEVELS_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "txq.num_mipmap_levels.b32 \t$d, [$a];",
-              []>;
-}
-
-def : Pat<(int_nvvm_txq_channel_order i64:$a),
-          (TXQ_CHANNEL_ORDER_R $a)>;
-def : Pat<(int_nvvm_txq_channel_data_type i64:$a),
-          (TXQ_CHANNEL_DATA_TYPE_R $a)>;
-def : Pat<(int_nvvm_txq_width i64:$a),
-          (TXQ_WIDTH_R $a)>;
-def : Pat<(int_nvvm_txq_height i64:$a),
-          (TXQ_HEIGHT_R $a)>;
-def : Pat<(int_nvvm_txq_depth i64:$a),
-          (TXQ_DEPTH_R $a)>;
-def : Pat<(int_nvvm_txq_array_size i64:$a),
-          (TXQ_ARRAY_SIZE_R $a)>;
-def : Pat<(int_nvvm_txq_num_samples i64:$a),
-          (TXQ_NUM_SAMPLES_R $a)>;
-def : Pat<(int_nvvm_txq_num_mipmap_levels i64:$a),
-          (TXQ_NUM_MIPMAP_LEVELS_R $a)>;
-
+  foreach query = ["channel_order", "channel_data_type", "width", "height", 
+                   "depth", "array_size", "num_samples", "num_mipmap_levels"] in {
+    def TXQ_ # !toupper(query) # _R
+      : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
+                  "txq." # query # ".b32 \t$d, [$a];",
+                  [(set i32:$d, (!cast<Intrinsic>("int_nvvm_txq_" # query) i64:$a))]>;
+    def TXQ_ # !toupper(query) # _I
+      : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
+                  "txq." # query # ".b32 \t$d, [$a];",
+                  []>;
+  }
+}
 
 //-----------------------------------
 // Surface Query Intrinsics
 //-----------------------------------
 
 let IsSurfTexQuery = true in {
-def SUQ_CHANNEL_ORDER_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "suq.channel_order.b32 \t$d, [$a];",
-              []>;
-def SUQ_CHANNEL_ORDER_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "suq.channel_order.b32 \t$d, [$a];",
-              []>;
-def SUQ_CHANNEL_DATA_TYPE_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "suq.channel_data_type.b32 \t$d, [$a];",
-              []>;
-def SUQ_CHANNEL_DATA_TYPE_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "suq.channel_data_type.b32 \t$d, [$a];",
-              []>;
-def SUQ_WIDTH_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "suq.width.b32 \t$d, [$a];",
-              []>;
-def SUQ_WIDTH_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "suq.width.b32 \t$d, [$a];",
-              []>;
-def SUQ_HEIGHT_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "suq.height.b32 \t$d, [$a];",
-              []>;
-def SUQ_HEIGHT_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "suq.height.b32 \t$d, [$a];",
-              []>;
-def SUQ_DEPTH_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "suq.depth.b32 \t$d, [$a];",
-              []>;
-def SUQ_DEPTH_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "suq.depth.b32 \t$d, [$a];",
-              []>;
-def SUQ_ARRAY_SIZE_R
-  : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
-              "suq.array_size.b32 \t$d, [$a];",
-              []>;
-def SUQ_ARRAY_SIZE_I
-  : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
-              "suq.array_size.b32 \t$d, [$a];",
-              []>;
-}
-
-def : Pat<(int_nvvm_suq_channel_order i64:$a),
-          (SUQ_CHANNEL_ORDER_R $a)>;
-def : Pat<(int_nvvm_suq_channel_data_type i64:$a),
-          (SUQ_CHANNEL_DATA_TYPE_R $a)>;
-def : Pat<(int_nvvm_suq_width i64:$a),
-          (SUQ_WIDTH_R $a)>;
-def : Pat<(int_nvvm_suq_height i64:$a),
-          (SUQ_HEIGHT_R $a)>;
-def : Pat<(int_nvvm_suq_depth i64:$a),
-          (SUQ_DEPTH_R $a)>;
-def : Pat<(int_nvvm_suq_array_size i64:$a),
-          (SUQ_ARRAY_SIZE_R $a)>;
-
+  foreach query = ["channel_order", "channel_data_type", "width", "height", "depth", "array_size"] in {
+    def SUQ_ # !toupper(query) # _R
+      : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
+                  "suq." # query # ".b32 \t$d, [$a];",
+                  [(set i32:$d, (!cast<Intrinsic>("int_nvvm_suq_" # query) i64:$a))]>;
+    def SUQ_ # !toupper(query) # _I
+      : NVPTXInst<(outs Int32Regs:$d), (ins i64imm:$a),
+                  "suq." # query # ".b32 \t$d, [$a];",
+                  []>;
+  }
+}
 
 //===- Handle Query -------------------------------------------------------===//
 
 // TODO: These intrinsics are not yet finalized, pending PTX ISA design work
 def ISTYPEP_SAMPLER
-  : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
-              "istypep.samplerref \t$d, $a;",
+  : BasicNVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
+              "istypep.samplerref",
               [(set i1:$d, (int_nvvm_istypep_sampler i64:$a))]>;
 def ISTYPEP_SURFACE
-  : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
-              "istypep.surfref \t$d, $a;",
+  : BasicNVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
+              "istypep.surfref",
               [(set i1:$d, (int_nvvm_istypep_surface i64:$a))]>;
 def ISTYPEP_TEXTURE
-  : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
-              "istypep.texref \t$d, $a;",
+  : BasicNVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
+              "istypep.texref",
               [(set i1:$d, (int_nvvm_istypep_texture i64:$a))]>;
 
 //===- Surface Stores -----------------------------------------------------===//
 
 let IsSust = true in {
 
-class SUST_1D_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_1D_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, intype:$r)),
-                inst # " \t[$s, \\{$x\\}], \\{$r\\};",
-                []>;
+                inst # " \t[$s, \\{$x\\}], \\{$r\\};", pat>;
 multiclass SUST_1D<string inst, NVPTXRegClass intype> {
-  def _R : SUST_1D_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_1D_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+
+  def _R : SUST_1D_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, intype:$r)]>;
+  def _I : SUST_1D_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_B8_CLAMP : SUST_1D<"sust.b.1d.b8.clamp", Int16Regs>;
-defm SUST_B_1D_B16_CLAMP : SUST_1D<"sust.b.1d.b16.clamp", Int16Regs>;
-defm SUST_B_1D_B32_CLAMP : SUST_1D<"sust.b.1d.b32.clamp", Int32Regs>;
-defm SUST_B_1D_B64_CLAMP : SUST_1D<"sust.b.1d.b64.clamp", Int64Regs>;
+defm SUST_B_1D_I8_CLAMP : SUST_1D<"sust.b.1d.b8.clamp", Int16Regs>;
+defm SUST_B_1D_I16_CLAMP : SUST_1D<"sust.b.1d.b16.clamp", Int16Regs>;
+defm SUST_B_1D_I32_CLAMP : SUST_1D<"sust.b.1d.b32.clamp", Int32Regs>;
+defm SUST_B_1D_I64_CLAMP : SUST_1D<"sust.b.1d.b64.clamp", Int64Regs>;
 
-defm SUST_B_1D_B8_TRAP : SUST_1D<"sust.b.1d.b8.trap", Int16Regs>;
-defm SUST_B_1D_B16_TRAP : SUST_1D<"sust.b.1d.b16.trap", Int16Regs>;
-defm SUST_B_1D_B32_TRAP : SUST_1D<"sust.b.1d.b32.trap", Int32Regs>;
-defm SUST_B_1D_B64_TRAP : SUST_1D<"sust.b.1d.b64.trap", Int64Regs>;
+defm SUST_B_1D_I8_TRAP : SUST_1D<"sust.b.1d.b8.trap", Int16Regs>;
+defm SUST_B_1D_I16_TRAP : SUST_1D<"sust.b.1d.b16.trap", Int16Regs>;
+defm SUST_B_1D_I32_TRAP : SUST_1D<"sust.b.1d.b32.trap", Int32Regs>;
+defm SUST_B_1D_I64_TRAP : SUST_1D<"sust.b.1d.b64.trap", Int64Regs>;
 
-defm SUST_B_1D_B8_ZERO : SUST_1D<"sust.b.1d.b8.zero", Int16Regs>;
-defm SUST_B_1D_B16_ZERO : SUST_1D<"sust.b.1d.b16.zero", Int16Regs>;
-defm SUST_B_1D_B32_ZERO : SUST_1D<"sust.b.1d.b32.zero", Int32Regs>;
-defm SUST_B_1D_B64_ZERO : SUST_1D<"sust.b.1d.b64.zero", Int64Regs>;
+defm SUST_B_1D_I8_ZERO : SUST_1D<"sust.b.1d.b8.zero", Int16Regs>;
+defm SUST_B_1D_I16_ZERO : SUST_1D<"sust.b.1d.b16.zero", Int16Regs>;
+defm SUST_B_1D_I32_ZERO : SUST_1D<"sust.b.1d.b32.zero", Int32Regs>;
+defm SUST_B_1D_I64_ZERO : SUST_1D<"sust.b.1d.b64.zero", Int64Regs>;
 
-defm SUST_P_1D_B8_TRAP : SUST_1D<"sust.p.1d.b8.trap", Int16Regs>;
-defm SUST_P_1D_B16_TRAP : SUST_1D<"sust.p.1d.b16.trap", Int16Regs>;
-defm SUST_P_1D_B32_TRAP : SUST_1D<"sust.p.1d.b32.trap", Int32Regs>;
+defm SUST_P_1D_I8_TRAP : SUST_1D<"sust.p.1d.b8.trap", Int16Regs>;
+defm SUST_P_1D_I16_TRAP : SUST_1D<"sust.p.1d.b16.trap", Int16Regs>;
+defm SUST_P_1D_I32_TRAP : SUST_1D<"sust.p.1d.b32.trap", Int32Regs>;
 
-class SUST_1D_V2_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_1D_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, intype:$r, intype:$g)),
                 inst # " \t[$s, \\{$x\\}], \\{$r, $g\\};",
-                []>;
+                pat>;
 multiclass SUST_1D_V2<string inst, NVPTXRegClass intype> {
-  def _R : SUST_1D_V2_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_1D_V2_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_1D_V2_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, intype:$r, intype:$g)]>;
+  def _I : SUST_1D_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_V2B8_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b8.clamp", Int16Regs>;
-defm SUST_B_1D_V2B16_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b16.clamp", Int16Regs>;
-defm SUST_B_1D_V2B32_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b32.clamp", Int32Regs>;
-defm SUST_B_1D_V2B64_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b64.clamp", Int64Regs>;
+// int_nvvm_sust_b_1d_v2i8_clamp
+
+defm SUST_B_1D_V2I8_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b8.clamp", Int16Regs>;
+defm SUST_B_1D_V2I16_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b16.clamp", Int16Regs>;
+defm SUST_B_1D_V2I32_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b32.clamp", Int32Regs>;
+defm SUST_B_1D_V2I64_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b64.clamp", Int64Regs>;
 
-defm SUST_B_1D_V2B8_TRAP : SUST_1D_V2<"sust.b.1d.v2.b8.trap", Int16Regs>;
-defm SUST_B_1D_V2B16_TRAP : SUST_1D_V2<"sust.b.1d.v2.b16.trap", Int16Regs>;
-defm SUST_B_1D_V2B32_TRAP : SUST_1D_V2<"sust.b.1d.v2.b32.trap", Int32Regs>;
-defm SUST_B_1D_V2B64_TRAP : SUST_1D_V2<"sust.b.1d.v2.b64.trap", Int64Regs>;
+defm SUST_B_1D_V2I8_TRAP : SUST_1D_V2<"sust.b.1d.v2.b8.trap", Int16Regs>;
+defm SUST_B_1D_V2I16_TRAP : SUST_1D_V2<"sust.b.1d.v2.b16.trap", Int16Regs>;
+defm SUST_B_1D_V2I32_TRAP : SUST_1D_V2<"sust.b.1d.v2.b32.trap", Int32Regs>;
+defm SUST_B_1D_V2I64_TRAP : SUST_1D_V2<"sust.b.1d.v2.b64.trap", Int64Regs>;
 
-defm SUST_B_1D_V2B8_ZERO : SUST_1D_V2<"sust.b.1d.v2.b8.zero", Int16Regs>;
-defm SUST_B_1D_V2B16_ZERO : SUST_1D_V2<"sust.b.1d.v2.b16.zero", Int16Regs>;
-defm SUST_B_1D_V2B32_ZERO : SUST_1D_V2<"sust.b.1d.v2.b32.zero", Int32Regs>;
-defm SUST_B_1D_V2B64_ZERO : SUST_1D_V2<"sust.b.1d.v2.b64.zero", Int64Regs>;
+defm SUST_B_1D_V2I8_ZERO : SUST_1D_V2<"sust.b.1d.v2.b8.zero", Int16Regs>;
+defm SUST_B_1D_V2I16_ZERO : SUST_1D_V2<"sust.b.1d.v2.b16.zero", Int16Regs>;
+defm SUST_B_1D_V2I32_ZERO : SUST_1D_V2<"sust.b.1d.v2.b32.zero", Int32Regs>;
+defm SUST_B_1D_V2I64_ZERO : SUST_1D_V2<"sust.b.1d.v2.b64.zero", Int64Regs>;
 
-defm SUST_P_1D_V2B8_TRAP : SUST_1D_V2<"sust.p.1d.v2.b8.trap", Int16Regs>;
-defm SUST_P_1D_V2B16_TRAP : SUST_1D_V2<"sust.p.1d.v2.b16.trap", Int16Regs>;
-defm SUST_P_1D_V2B32_TRAP : SUST_1D_V2<"sust.p.1d.v2.b32.trap", Int32Regs>;
+defm SUST_P_1D_V2I8_TRAP : SUST_1D_V2<"sust.p.1d.v2.b8.trap", Int16Regs>;
+defm SUST_P_1D_V2I16_TRAP : SUST_1D_V2<"sust.p.1d.v2.b16.trap", Int16Regs>;
+defm SUST_P_1D_V2I32_TRAP : SUST_1D_V2<"sust.p.1d.v2.b32.trap", Int32Regs>;
 
-class SUST_1D_V4_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_1D_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, intype:$r, intype:$g,
                                 intype:$b, intype:$a)),
                 inst # " \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
-                []>;
+                pat>;
 multiclass SUST_1D_V4<string inst, NVPTXRegClass intype> {
-  def _R : SUST_1D_V4_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_1D_V4_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_1D_V4_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, intype:$r, intype:$g,
+                intype:$b, intype:$a)]>;
+  def _I : SUST_1D_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_V4B8_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b8.clamp", Int16Regs>;
-defm SUST_B_1D_V4B16_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b16.clamp", Int16Regs>;
-defm SUST_B_1D_V4B32_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b32.clamp", Int32Regs>;
+defm SUST_B_1D_V4I8_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b8.clamp", Int16Regs>;
+defm SUST_B_1D_V4I16_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b16.clamp", Int16Regs>;
+defm SUST_B_1D_V4I32_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b32.clamp", Int32Regs>;
 
-defm SUST_B_1D_V4B8_TRAP : SUST_1D_V4<"sust.b.1d.v4.b8.trap", Int16Regs>;
-defm SUST_B_1D_V4B16_TRAP : SUST_1D_V4<"sust.b.1d.v4.b16.trap", Int16Regs>;
-defm SUST_B_1D_V4B32_TRAP : SUST_1D_V4<"sust.b.1d.v4.b32.trap", Int32Regs>;
+defm SUST_B_1D_V4I8_TRAP : SUST_1D_V4<"sust.b.1d.v4.b8.trap", Int16Regs>;
+defm SUST_B_1D_V4I16_TRAP : SUST_1D_V4<"sust.b.1d.v4.b16.trap", Int16Regs>;
+defm SUST_B_1D_V4I32_TRAP : SUST_1D_V4<"sust.b.1d.v4.b32.trap", Int32Regs>;
 
-defm SUST_B_1D_V4B8_ZERO : SUST_1D_V4<"sust.b.1d.v4.b8.zero", Int16Regs>;
-defm SUST_B_1D_V4B16_ZERO : SUST_1D_V4<"sust.b.1d.v4.b16.zero", Int16Regs>;
-defm SUST_B_1D_V4B32_ZERO : SUST_1D_V4<"sust.b.1d.v4.b32.zero", Int32Regs>;
+defm SUST_B_1D_V4I8_ZERO : SUST_1D_V4<"sust.b.1d.v4.b8.zero", Int16Regs>;
+defm SUST_B_1D_V4I16_ZERO : SUST_1D_V4<"sust.b.1d.v4.b16.zero", Int16Regs>;
+defm SUST_B_1D_V4I32_ZERO : SUST_1D_V4<"sust.b.1d.v4.b32.zero", Int32Regs>;
 
-defm SUST_P_1D_V4B8_TRAP : SUST_1D_V4<"sust.p.1d.v4.b8.trap", Int16Regs>;
-defm SUST_P_1D_V4B16_TRAP : SUST_1D_V4<"sust.p.1d.v4.b16.trap", Int16Regs>;
-defm SUST_P_1D_V4B32_TRAP : SUST_1D_V4<"sust.p.1d.v4.b32.trap", Int32Regs>;
+defm SUST_P_1D_V4I8_TRAP : SUST_1D_V4<"sust.p.1d.v4.b8.trap", Int16Regs>;
+defm SUST_P_1D_V4I16_TRAP : SUST_1D_V4<"sust.p.1d.v4.b16.trap", Int16Regs>;
+defm SUST_P_1D_V4I32_TRAP : SUST_1D_V4<"sust.p.1d.v4.b32.trap", Int32Regs>;
 
-class SUST_1D_ARRAY_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_1D_ARRAY_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$idx, Int32Regs:$x, intype:$r)),
                 inst # " \t[$s, \\{$idx, $x\\}], \\{$r\\};",
-                []>;
+                pat>;
 multiclass SUST_1D_ARRAY<string inst, NVPTXRegClass intype> {
-  def _R : SUST_1D_ARRAY_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_1D_ARRAY_base<inst, intype, (ins i64imm:$s)>;
+    defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_1D_ARRAY_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, intype:$r)]>;
+  def _I : SUST_1D_ARRAY_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_ARRAY_B8_CLAMP
+defm SUST_B_1D_ARRAY_I8_CLAMP
   : SUST_1D_ARRAY<"sust.b.a1d.b8.clamp", Int16Regs>;
-defm SUST_B_1D_ARRAY_B16_CLAMP
+defm SUST_B_1D_ARRAY_I16_CLAMP
   : SUST_1D_ARRAY<"sust.b.a1d.b16.clamp", Int16Regs>;
-defm SUST_B_1D_ARRAY_B32_CLAMP
+defm SUST_B_1D_ARRAY_I32_CLAMP
   : SUST_1D_ARRAY<"sust.b.a1d.b32.clamp", Int32Regs>;
-defm SUST_B_1D_ARRAY_B64_CLAMP
+defm SUST_B_1D_ARRAY_I64_CLAMP
   : SUST_1D_ARRAY<"sust.b.a1d.b64.clamp", Int64Regs>;
 
-defm SUST_B_1D_ARRAY_B8_TRAP
+defm SUST_B_1D_ARRAY_I8_TRAP
   : SUST_1D_ARRAY<"sust.b.a1d.b8.trap", Int16Regs>;
-defm SUST_B_1D_ARRAY_B16_TRAP
+defm SUST_B_1D_ARRAY_I16_TRAP
   : SUST_1D_ARRAY<"sust.b.a1d.b16.trap", Int16Regs>;
-defm SUST_B_1D_ARRAY_B32_TRAP
+defm SUST_B_1D_ARRAY_I32_TRAP
   : SUST_1D_ARRAY<"sust.b.a1d.b32.trap", Int32Regs>;
-defm SUST_B_1D_ARRAY_B64_TRAP
+defm SUST_B_1D_ARRAY_I64_TRAP
   : SUST_1D_ARRAY<"sust.b.a1d.b64.trap", Int64Regs>;
 
-defm SUST_B_1D_ARRAY_B8_ZERO
+defm SUST_B_1D_ARRAY_I8_ZERO
   : SUST_1D_ARRAY<"sust.b.a1d.b8.zero", Int16Regs>;
-defm SUST_B_1D_ARRAY_B16_ZERO
+defm SUST_B_1D_ARRAY_I16_ZERO
   : SUST_1D_ARRAY<"sust.b.a1d.b16.zero", Int16Regs>;
-defm SUST_B_1D_ARRAY_B32_ZERO
+defm SUST_B_1D_ARRAY_I32_ZERO
   : SUST_1D_ARRAY<"sust.b.a1d.b32.zero", Int32Regs>;
-defm SUST_B_1D_ARRAY_B64_ZERO
+defm SUST_B_1D_ARRAY_I64_ZERO
   : SUST_1D_ARRAY<"sust.b.a1d.b64.zero", Int64Regs>;
 
-defm SUST_P_1D_ARRAY_B8_TRAP
+defm SUST_P_1D_ARRAY_I8_TRAP
   : SUST_1D_ARRAY<"sust.p.a1d.b8.trap", Int16Regs>;
-defm SUST_P_1D_ARRAY_B16_TRAP
+defm SUST_P_1D_ARRAY_I16_TRAP
   : SUST_1D_ARRAY<"sust.p.a1d.b16.trap", Int16Regs>;
-defm SUST_P_1D_ARRAY_B32_TRAP
+defm SUST_P_1D_ARRAY_I32_TRAP
   : SUST_1D_ARRAY<"sust.p.a1d.b32.trap", Int32Regs>;
 
-class SUST_1D_ARRAY_V2_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_1D_ARRAY_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$idx, Int32Regs:$x,
                                 intype:$r, intype:$g)),
                 inst # " \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
-                []>;
+                pat>;
 multiclass SUST_1D_ARRAY_V2<string inst, NVPTXRegClass intype> {
-  def _R : SUST_1D_ARRAY_V2_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_1D_ARRAY_V2_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_1D_ARRAY_V2_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x,
+                intype:$r, intype:$g)]>;
+  def _I : SUST_1D_ARRAY_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_ARRAY_V2B8_CLAMP
+defm SUST_B_1D_ARRAY_V2I8_CLAMP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8.clamp", Int16Regs>;
-defm SUST_B_1D_ARRAY_V2B16_CLAMP
+defm SUST_B_1D_ARRAY_V2I16_CLAMP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16.clamp", Int16Regs>;
-defm SUST_B_1D_ARRAY_V2B32_CLAMP
+defm SUST_B_1D_ARRAY_V2I32_CLAMP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32.clamp", Int32Regs>;
-defm SUST_B_1D_ARRAY_V2B64_CLAMP
+defm SUST_B_1D_ARRAY_V2I64_CLAMP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64.clamp", Int64Regs>;
 
-defm SUST_B_1D_ARRAY_V2B8_TRAP
+defm SUST_B_1D_ARRAY_V2I8_TRAP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8.trap", Int16Regs>;
-defm SUST_B_1D_ARRAY_V2B16_TRAP
+defm SUST_B_1D_ARRAY_V2I16_TRAP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16.trap", Int16Regs>;
-defm SUST_B_1D_ARRAY_V2B32_TRAP
+defm SUST_B_1D_ARRAY_V2I32_TRAP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32.trap", Int32Regs>;
-defm SUST_B_1D_ARRAY_V2B64_TRAP
+defm SUST_B_1D_ARRAY_V2I64_TRAP
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64.trap", Int64Regs>;
 
-defm SUST_B_1D_ARRAY_V2B8_ZERO
+defm SUST_B_1D_ARRAY_V2I8_ZERO
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8.zero", Int16Regs>;
-defm SUST_B_1D_ARRAY_V2B16_ZERO
+defm SUST_B_1D_ARRAY_V2I16_ZERO
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16.zero", Int16Regs>;
-defm SUST_B_1D_ARRAY_V2B32_ZERO
+defm SUST_B_1D_ARRAY_V2I32_ZERO
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32.zero", Int32Regs>;
-defm SUST_B_1D_ARRAY_V2B64_ZERO
+defm SUST_B_1D_ARRAY_V2I64_ZERO
   : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64.zero", Int64Regs>;
 
-defm SUST_P_1D_ARRAY_V2B8_TRAP
+defm SUST_P_1D_ARRAY_V2I8_TRAP
   : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b8.trap", Int16Regs>;
-defm SUST_P_1D_ARRAY_V2B16_TRAP
+defm SUST_P_1D_ARRAY_V2I16_TRAP
   : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b16.trap", Int16Regs>;
-defm SUST_P_1D_ARRAY_V2B32_TRAP
+defm SUST_P_1D_ARRAY_V2I32_TRAP
   : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b32.trap", Int32Regs>;
 
-class SUST_1D_ARRAY_V4_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_1D_ARRAY_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$idx, Int32Regs:$x,
                                 intype:$r, intype:$g, intype:$b, intype:$a)),
                 inst # " \t[$s, \\{$idx, $x\\}], \\{$r, $g, $b, $a\\};",
-                []>;
+                pat>;
 multiclass SUST_1D_ARRAY_V4<string inst, NVPTXRegClass intype> {
-  def _R : SUST_1D_ARRAY_V4_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_1D_ARRAY_V4_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_1D_ARRAY_V4_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x,
+                intype:$r, intype:$g, intype:$b, intype:$a)]>;
+  def _I : SUST_1D_ARRAY_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_ARRAY_V4B8_CLAMP
+defm SUST_B_1D_ARRAY_V4I8_CLAMP
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8.clamp", Int16Regs>;
-defm SUST_B_1D_ARRAY_V4B16_CLAMP
+defm SUST_B_1D_ARRAY_V4I16_CLAMP
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16.clamp", Int16Regs>;
-defm SUST_B_1D_ARRAY_V4B32_CLAMP
+defm SUST_B_1D_ARRAY_V4I32_CLAMP
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32.clamp", Int32Regs>;
 
-defm SUST_B_1D_ARRAY_V4B8_TRAP
+defm SUST_B_1D_ARRAY_V4I8_TRAP
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8.trap", Int16Regs>;
-defm SUST_B_1D_ARRAY_V4B16_TRAP
+defm SUST_B_1D_ARRAY_V4I16_TRAP
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16.trap", Int16Regs>;
-defm SUST_B_1D_ARRAY_V4B32_TRAP
+defm SUST_B_1D_ARRAY_V4I32_TRAP
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32.trap", Int32Regs>;
 
-defm SUST_B_1D_ARRAY_V4B8_ZERO
+defm SUST_B_1D_ARRAY_V4I8_ZERO
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8.zero", Int16Regs>;
-defm SUST_B_1D_ARRAY_V4B16_ZERO
+defm SUST_B_1D_ARRAY_V4I16_ZERO
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16.zero", Int16Regs>;
-defm SUST_B_1D_ARRAY_V4B32_ZERO
+defm SUST_B_1D_ARRAY_V4I32_ZERO
   : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32.zero", Int32Regs>;
 
-defm SUST_P_1D_ARRAY_V4B8_TRAP
+defm SUST_P_1D_ARRAY_V4I8_TRAP
   : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b8.trap", Int16Regs>;
-defm SUST_P_1D_ARRAY_V4B16_TRAP
+defm SUST_P_1D_ARRAY_V4I16_TRAP
   : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b16.trap", Int16Regs>;
-defm SUST_P_1D_ARRAY_V4B32_TRAP
+defm SUST_P_1D_ARRAY_V4I32_TRAP
   : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b32.trap", Int32Regs>;
 
-class SUST_2D_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_2D_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, Int32Regs:$y, intype:$r)),
                 inst # " \t[$s, \\{$x, $y\\}], \\{$r\\};",
-                []>;
+                pat>;
 multiclass SUST_2D<string inst, NVPTXRegClass intype> {
-  def _R : SUST_2D_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_2D_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_2D_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, intype:$r)]>;
+  def _I : SUST_2D_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_B8_CLAMP : SUST_2D<"sust.b.2d.b8.clamp", Int16Regs>;
-defm SUST_B_2D_B16_CLAMP : SUST_2D<"sust.b.2d.b16.clamp", Int16Regs>;
-defm SUST_B_2D_B32_CLAMP : SUST_2D<"sust.b.2d.b32.clamp", Int32Regs>;
-defm SUST_B_2D_B64_CLAMP : SUST_2D<"sust.b.2d.b64.clamp", Int64Regs>;
+defm SUST_B_2D_I8_CLAMP : SUST_2D<"sust.b.2d.b8.clamp", Int16Regs>;
+defm SUST_B_2D_I16_CLAMP : SUST_2D<"sust.b.2d.b16.clamp", Int16Regs>;
+defm SUST_B_2D_I32_CLAMP : SUST_2D<"sust.b.2d.b32.clamp", Int32Regs>;
+defm SUST_B_2D_I64_CLAMP : SUST_2D<"sust.b.2d.b64.clamp", Int64Regs>;
 
-defm SUST_B_2D_B8_TRAP : SUST_2D<"sust.b.2d.b8.trap", Int16Regs>;
-defm SUST_B_2D_B16_TRAP : SUST_2D<"sust.b.2d.b16.trap", Int16Regs>;
-defm SUST_B_2D_B32_TRAP : SUST_2D<"sust.b.2d.b32.trap", Int32Regs>;
-defm SUST_B_2D_B64_TRAP : SUST_2D<"sust.b.2d.b64.trap", Int64Regs>;
+defm SUST_B_2D_I8_TRAP : SUST_2D<"sust.b.2d.b8.trap", Int16Regs>;
+defm SUST_B_2D_I16_TRAP : SUST_2D<"sust.b.2d.b16.trap", Int16Regs>;
+defm SUST_B_2D_I32_TRAP : SUST_2D<"sust.b.2d.b32.trap", Int32Regs>;
+defm SUST_B_2D_I64_TRAP : SUST_2D<"sust.b.2d.b64.trap", Int64Regs>;
 
-defm SUST_B_2D_B8_ZERO : SUST_2D<"sust.b.2d.b8.zero", Int16Regs>;
-defm SUST_B_2D_B16_ZERO : SUST_2D<"sust.b.2d.b16.zero", Int16Regs>;
-defm SUST_B_2D_B32_ZERO : SUST_2D<"sust.b.2d.b32.zero", Int32Regs>;
-defm SUST_B_2D_B64_ZERO : SUST_2D<"sust.b.2d.b64.zero", Int64Regs>;
+defm SUST_B_2D_I8_ZERO : SUST_2D<"sust.b.2d.b8.zero", Int16Regs>;
+defm SUST_B_2D_I16_ZERO : SUST_2D<"sust.b.2d.b16.zero", Int16Regs>;
+defm SUST_B_2D_I32_ZERO : SUST_2D<"sust.b.2d.b32.zero", Int32Regs>;
+defm SUST_B_2D_I64_ZERO : SUST_2D<"sust.b.2d.b64.zero", Int64Regs>;
 
-defm SUST_P_2D_B8_TRAP : SUST_2D<"sust.p.2d.b8.trap", Int16Regs>;
-defm SUST_P_2D_B16_TRAP : SUST_2D<"sust.p.2d.b16.trap", Int16Regs>;
-defm SUST_P_2D_B32_TRAP : SUST_2D<"sust.p.2d.b32.trap", Int32Regs>;
+defm SUST_P_2D_I8_TRAP : SUST_2D<"sust.p.2d.b8.trap", Int16Regs>;
+defm SUST_P_2D_I16_TRAP : SUST_2D<"sust.p.2d.b16.trap", Int16Regs>;
+defm SUST_P_2D_I32_TRAP : SUST_2D<"sust.p.2d.b32.trap", Int32Regs>;
 
-class SUST_2D_V2_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_2D_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, Int32Regs:$y,
                                 intype:$r, intype:$g)),
                 inst # " \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
-                []>;
+                pat>;
 multiclass SUST_2D_V2<string inst, NVPTXRegClass intype> {
-  def _R : SUST_2D_V2_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_2D_V2_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_2D_V2_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
+                intype:$r, intype:$g)]>;
+  def _I : SUST_2D_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_V2B8_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b8.clamp", Int16Regs>;
-defm SUST_B_2D_V2B16_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b16.clamp", Int16Regs>;
-defm SUST_B_2D_V2B32_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b32.clamp", Int32Regs>;
-defm SUST_B_2D_V2B64_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b64.clamp", Int64Regs>;
+defm SUST_B_2D_V2I8_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b8.clamp", Int16Regs>;
+defm SUST_B_2D_V2I16_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b16.clamp", Int16Regs>;
+defm SUST_B_2D_V2I32_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b32.clamp", Int32Regs>;
+defm SUST_B_2D_V2I64_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b64.clamp", Int64Regs>;
 
-defm SUST_B_2D_V2B8_TRAP : SUST_2D_V2<"sust.b.2d.v2.b8.trap", Int16Regs>;
-defm SUST_B_2D_V2B16_TRAP : SUST_2D_V2<"sust.b.2d.v2.b16.trap", Int16Regs>;
-defm SUST_B_2D_V2B32_TRAP : SUST_2D_V2<"sust.b.2d.v2.b32.trap", Int32Regs>;
-defm SUST_B_2D_V2B64_TRAP : SUST_2D_V2<"sust.b.2d.v2.b64.trap", Int64Regs>;
+defm SUST_B_2D_V2I8_TRAP : SUST_2D_V2<"sust.b.2d.v2.b8.trap", Int16Regs>;
+defm SUST_B_2D_V2I16_TRAP : SUST_2D_V2<"sust.b.2d.v2.b16.trap", Int16Regs>;
+defm SUST_B_2D_V2I32_TRAP : SUST_2D_V2<"sust.b.2d.v2.b32.trap", Int32Regs>;
+defm SUST_B_2D_V2I64_TRAP : SUST_2D_V2<"sust.b.2d.v2.b64.trap", Int64Regs>;
 
-defm SUST_B_2D_V2B8_ZERO : SUST_2D_V2<"sust.b.2d.v2.b8.zero", Int16Regs>;
-defm SUST_B_2D_V2B16_ZERO : SUST_2D_V2<"sust.b.2d.v2.b16.zero", Int16Regs>;
-defm SUST_B_2D_V2B32_ZERO : SUST_2D_V2<"sust.b.2d.v2.b32.zero", Int32Regs>;
-defm SUST_B_2D_V2B64_ZERO : SUST_2D_V2<"sust.b.2d.v2.b64.zero", Int64Regs>;
+defm SUST_B_2D_V2I8_ZERO : SUST_2D_V2<"sust.b.2d.v2.b8.zero", Int16Regs>;
+defm SUST_B_2D_V2I16_ZERO : SUST_2D_V2<"sust.b.2d.v2.b16.zero", Int16Regs>;
+defm SUST_B_2D_V2I32_ZERO : SUST_2D_V2<"sust.b.2d.v2.b32.zero", Int32Regs>;
+defm SUST_B_2D_V2I64_ZERO : SUST_2D_V2<"sust.b.2d.v2.b64.zero", Int64Regs>;
 
-defm SUST_P_2D_V2B8_TRAP : SUST_2D_V2<"sust.p.2d.v2.b8.trap", Int16Regs>;
-defm SUST_P_2D_V2B16_TRAP : SUST_2D_V2<"sust.p.2d.v2.b16.trap", Int16Regs>;
-defm SUST_P_2D_V2B32_TRAP : SUST_2D_V2<"sust.p.2d.v2.b32.trap", Int32Regs>;
+defm SUST_P_2D_V2I8_TRAP : SUST_2D_V2<"sust.p.2d.v2.b8.trap", Int16Regs>;
+defm SUST_P_2D_V2I16_TRAP : SUST_2D_V2<"sust.p.2d.v2.b16.trap", Int16Regs>;
+defm SUST_P_2D_V2I32_TRAP : SUST_2D_V2<"sust.p.2d.v2.b32.trap", Int32Regs>;
 
-class SUST_2D_V4_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_2D_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, Int32Regs:$y,
                                 intype:$r, intype:$g, intype:$b, intype:$a)),
                 inst # " \t[$s, \\{$x, $y\\}], \\{$r, $g, $b, $a\\};",
-                []>;
+                pat>;
 multiclass SUST_2D_V4<string inst, NVPTXRegClass intype> {
-  def _R : SUST_2D_V4_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_2D_V4_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_2D_V4_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
+                intype:$r, intype:$g, intype:$b, intype:$a)]>;
+  def _I : SUST_2D_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_V4B8_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b8.clamp", Int16Regs>;
-defm SUST_B_2D_V4B16_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b16.clamp", Int16Regs>;
-defm SUST_B_2D_V4B32_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b32.clamp", Int32Regs>;
+defm SUST_B_2D_V4I8_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b8.clamp", Int16Regs>;
+defm SUST_B_2D_V4I16_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b16.clamp", Int16Regs>;
+defm SUST_B_2D_V4I32_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b32.clamp", Int32Regs>;
 
-defm SUST_B_2D_V4B8_TRAP : SUST_2D_V4<"sust.b.2d.v4.b8.trap", Int16Regs>;
-defm SUST_B_2D_V4B16_TRAP : SUST_2D_V4<"sust.b.2d.v4.b16.trap", Int16Regs>;
-defm SUST_B_2D_V4B32_TRAP : SUST_2D_V4<"sust.b.2d.v4.b32.trap", Int32Regs>;
+defm SUST_B_2D_V4I8_TRAP : SUST_2D_V4<"sust.b.2d.v4.b8.trap", Int16Regs>;
+defm SUST_B_2D_V4I16_TRAP : SUST_2D_V4<"sust.b.2d.v4.b16.trap", Int16Regs>;
+defm SUST_B_2D_V4I32_TRAP : SUST_2D_V4<"sust.b.2d.v4.b32.trap", Int32Regs>;
 
-defm SUST_B_2D_V4B8_ZERO : SUST_2D_V4<"sust.b.2d.v4.b8.zero", Int16Regs>;
-defm SUST_B_2D_V4B16_ZERO : SUST_2D_V4<"sust.b.2d.v4.b16.zero", Int16Regs>;
-defm SUST_B_2D_V4B32_ZERO : SUST_2D_V4<"sust.b.2d.v4.b32.zero", Int32Regs>;
+defm SUST_B_2D_V4I8_ZERO : SUST_2D_V4<"sust.b.2d.v4.b8.zero", Int16Regs>;
+defm SUST_B_2D_V4I16_ZERO : SUST_2D_V4<"sust.b.2d.v4.b16.zero", Int16Regs>;
+defm SUST_B_2D_V4I32_ZERO : SUST_2D_V4<"sust.b.2d.v4.b32.zero", Int32Regs>;
 
-defm SUST_P_2D_V4B8_TRAP : SUST_2D_V4<"sust.p.2d.v4.b8.trap", Int16Regs>;
-defm SUST_P_2D_V4B16_TRAP : SUST_2D_V4<"sust.p.2d.v4.b16.trap", Int16Regs>;
-defm SUST_P_2D_V4B32_TRAP : SUST_2D_V4<"sust.p.2d.v4.b32.trap", Int32Regs>;
+defm SUST_P_2D_V4I8_TRAP : SUST_2D_V4<"sust.p.2d.v4.b8.trap", Int16Regs>;
+defm SUST_P_2D_V4I16_TRAP : SUST_2D_V4<"sust.p.2d.v4.b16.trap", Int16Regs>;
+defm SUST_P_2D_V4I32_TRAP : SUST_2D_V4<"sust.p.2d.v4.b32.trap", Int32Regs>;
 
-class SUST_2D_ARRAY_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_2D_ARRAY_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
                                 intype:$r)),
                 inst # " \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
-                []>;
+                pat>;
 multiclass SUST_2D_ARRAY<string inst, NVPTXRegClass intype> {
-  def _R : SUST_2D_ARRAY_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_2D_ARRAY_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_2D_ARRAY_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
+                intype:$r)]>;
+  def _I : SUST_2D_ARRAY_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_ARRAY_B8_CLAMP
+defm SUST_B_2D_ARRAY_I8_CLAMP
   : SUST_2D_ARRAY<"sust.b.a2d.b8.clamp", Int16Regs>;
-defm SUST_B_2D_ARRAY_B16_CLAMP
+defm SUST_B_2D_ARRAY_I16_CLAMP
   : SUST_2D_ARRAY<"sust.b.a2d.b16.clamp", Int16Regs>;
-defm SUST_B_2D_ARRAY_B32_CLAMP
+defm SUST_B_2D_ARRAY_I32_CLAMP
   : SUST_2D_ARRAY<"sust.b.a2d.b32.clamp", Int32Regs>;
-defm SUST_B_2D_ARRAY_B64_CLAMP
+defm SUST_B_2D_ARRAY_I64_CLAMP
   : SUST_2D_ARRAY<"sust.b.a2d.b64.clamp", Int64Regs>;
 
-defm SUST_B_2D_ARRAY_B8_TRAP
+defm SUST_B_2D_ARRAY_I8_TRAP
   : SUST_2D_ARRAY<"sust.b.a2d.b8.trap", Int16Regs>;
-defm SUST_B_2D_ARRAY_B16_TRAP
+defm SUST_B_2D_ARRAY_I16_TRAP
   : SUST_2D_ARRAY<"sust.b.a2d.b16.trap", Int16Regs>;
-defm SUST_B_2D_ARRAY_B32_TRAP
+defm SUST_B_2D_ARRAY_I32_TRAP
   : SUST_2D_ARRAY<"sust.b.a2d.b32.trap", Int32Regs>;
-defm SUST_B_2D_ARRAY_B64_TRAP
+defm SUST_B_2D_ARRAY_I64_TRAP
   : SUST_2D_ARRAY<"sust.b.a2d.b64.trap", Int64Regs>;
 
-defm SUST_B_2D_ARRAY_B8_ZERO
+defm SUST_B_2D_ARRAY_I8_ZERO
   : SUST_2D_ARRAY<"sust.b.a2d.b8.zero", Int16Regs>;
-defm SUST_B_2D_ARRAY_B16_ZERO
+defm SUST_B_2D_ARRAY_I16_ZERO
   : SUST_2D_ARRAY<"sust.b.a2d.b16.zero", Int16Regs>;
-defm SUST_B_2D_ARRAY_B32_ZERO
+defm SUST_B_2D_ARRAY_I32_ZERO
   : SUST_2D_ARRAY<"sust.b.a2d.b32.zero", Int32Regs>;
-defm SUST_B_2D_ARRAY_B64_ZERO
+defm SUST_B_2D_ARRAY_I64_ZERO
   : SUST_2D_ARRAY<"sust.b.a2d.b64.zero", Int64Regs>;
 
-defm SUST_P_2D_ARRAY_B8_TRAP
+defm SUST_P_2D_ARRAY_I8_TRAP
   : SUST_2D_ARRAY<"sust.p.a2d.b8.trap", Int16Regs>;
-defm SUST_P_2D_ARRAY_B16_TRAP
+defm SUST_P_2D_ARRAY_I16_TRAP
   : SUST_2D_ARRAY<"sust.p.a2d.b16.trap", Int16Regs>;
-defm SUST_P_2D_ARRAY_B32_TRAP
+defm SUST_P_2D_ARRAY_I32_TRAP
   : SUST_2D_ARRAY<"sust.p.a2d.b32.trap", Int32Regs>;
 
-class SUST_2D_ARRAY_V2_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_2D_ARRAY_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
                                 intype:$r, intype:$g)),
                 inst # " \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r, $g\\};",
-                []>;
+                pat>;
 multiclass SUST_2D_ARRAY_V2<string inst, NVPTXRegClass intype> {
-  def _R : SUST_2D_ARRAY_V2_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_2D_ARRAY_V2_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_2D_ARRAY_V2_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
+                intype:$r, intype:$g)]>;
+  def _I : SUST_2D_ARRAY_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_ARRAY_V2B8_CLAMP
+defm SUST_B_2D_ARRAY_V2I8_CLAMP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8.clamp", Int16Regs>;
-defm SUST_B_2D_ARRAY_V2B16_CLAMP
+defm SUST_B_2D_ARRAY_V2I16_CLAMP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16.clamp", Int16Regs>;
-defm SUST_B_2D_ARRAY_V2B32_CLAMP
+defm SUST_B_2D_ARRAY_V2I32_CLAMP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32.clamp", Int32Regs>;
-defm SUST_B_2D_ARRAY_V2B64_CLAMP
+defm SUST_B_2D_ARRAY_V2I64_CLAMP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64.clamp", Int64Regs>;
 
-defm SUST_B_2D_ARRAY_V2B8_TRAP
+defm SUST_B_2D_ARRAY_V2I8_TRAP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8.trap", Int16Regs>;
-defm SUST_B_2D_ARRAY_V2B16_TRAP
+defm SUST_B_2D_ARRAY_V2I16_TRAP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16.trap", Int16Regs>;
-defm SUST_B_2D_ARRAY_V2B32_TRAP
+defm SUST_B_2D_ARRAY_V2I32_TRAP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32.trap", Int32Regs>;
-defm SUST_B_2D_ARRAY_V2B64_TRAP
+defm SUST_B_2D_ARRAY_V2I64_TRAP
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64.trap", Int64Regs>;
 
-defm SUST_B_2D_ARRAY_V2B8_ZERO
+defm SUST_B_2D_ARRAY_V2I8_ZERO
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8.zero", Int16Regs>;
-defm SUST_B_2D_ARRAY_V2B16_ZERO
+defm SUST_B_2D_ARRAY_V2I16_ZERO
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16.zero", Int16Regs>;
-defm SUST_B_2D_ARRAY_V2B32_ZERO
+defm SUST_B_2D_ARRAY_V2I32_ZERO
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32.zero", Int32Regs>;
-defm SUST_B_2D_ARRAY_V2B64_ZERO
+defm SUST_B_2D_ARRAY_V2I64_ZERO
   : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64.zero", Int64Regs>;
 
-defm SUST_P_2D_ARRAY_V2B8_TRAP
+defm SUST_P_2D_ARRAY_V2I8_TRAP
   : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b8.trap", Int16Regs>;
-defm SUST_P_2D_ARRAY_V2B16_TRAP
+defm SUST_P_2D_ARRAY_V2I16_TRAP
   : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b16.trap", Int16Regs>;
-defm SUST_P_2D_ARRAY_V2B32_TRAP
+defm SUST_P_2D_ARRAY_V2I32_TRAP
   : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b32.trap", Int32Regs>;
 
-class SUST_2D_ARRAY_V4_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_2D_ARRAY_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
                                 intype:$r, intype:$g, intype:$b, intype:$a)),
                 inst # " \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r, $g, $b, $a\\};",
-                []>;
+                pat>;
 multiclass SUST_2D_ARRAY_V4<string inst, NVPTXRegClass intype> {
-  def _R : SUST_2D_ARRAY_V4_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_2D_ARRAY_V4_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_2D_ARRAY_V4_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
+                intype:$r, intype:$g, intype:$b, intype:$a)]>;
+  def _I : SUST_2D_ARRAY_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_ARRAY_V4B8_CLAMP
+defm SUST_B_2D_ARRAY_V4I8_CLAMP
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8.clamp", Int16Regs>;
-defm SUST_B_2D_ARRAY_V4B16_CLAMP
+defm SUST_B_2D_ARRAY_V4I16_CLAMP
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16.clamp", Int16Regs>;
-defm SUST_B_2D_ARRAY_V4B32_CLAMP
+defm SUST_B_2D_ARRAY_V4I32_CLAMP
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32.clamp", Int32Regs>;
 
-defm SUST_B_2D_ARRAY_V4B8_TRAP
+defm SUST_B_2D_ARRAY_V4I8_TRAP
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8.trap", Int16Regs>;
-defm SUST_B_2D_ARRAY_V4B16_TRAP
+defm SUST_B_2D_ARRAY_V4I16_TRAP
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16.trap", Int16Regs>;
-defm SUST_B_2D_ARRAY_V4B32_TRAP
+defm SUST_B_2D_ARRAY_V4I32_TRAP
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32.trap", Int32Regs>;
 
-defm SUST_B_2D_ARRAY_V4B8_ZERO
+defm SUST_B_2D_ARRAY_V4I8_ZERO
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8.zero", Int16Regs>;
-defm SUST_B_2D_ARRAY_V4B16_ZERO
+defm SUST_B_2D_ARRAY_V4I16_ZERO
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16.zero", Int16Regs>;
-defm SUST_B_2D_ARRAY_V4B32_ZERO
+defm SUST_B_2D_ARRAY_V4I32_ZERO
   : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32.zero", Int32Regs>;
 
-defm SUST_P_2D_ARRAY_V4B8_TRAP
+defm SUST_P_2D_ARRAY_V4I8_TRAP
   : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b8.trap", Int16Regs>;
-defm SUST_P_2D_ARRAY_V4B16_TRAP
+defm SUST_P_2D_ARRAY_V4I16_TRAP
   : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b16.trap", Int16Regs>;
-defm SUST_P_2D_ARRAY_V4B32_TRAP
+defm SUST_P_2D_ARRAY_V4I32_TRAP
   : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b32.trap", Int32Regs>;
 
-class SUST_3D_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_3D_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
                                 intype:$r)),
                 inst # " \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
-                []>;
+                pat>;
 multiclass SUST_3D<string inst, NVPTXRegClass intype> {
-  def _R : SUST_3D_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_3D_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_3D_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
+                intype:$r)]>;
+  def _I : SUST_3D_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_3D_B8_CLAMP : SUST_3D<"sust.b.3d.b8.clamp", Int16Regs>;
-defm SUST_B_3D_B16_CLAMP : SUST_3D<"sust.b.3d.b16.clamp", Int16Regs>;
-defm SUST_B_3D_B32_CLAMP : SUST_3D<"sust.b.3d.b32.clamp", Int32Regs>;
-defm SUST_B_3D_B64_CLAMP : SUST_3D<"sust.b.3d.b64.clamp", Int64Regs>;
+defm SUST_B_3D_I8_CLAMP : SUST_3D<"sust.b.3d.b8.clamp", Int16Regs>;
+defm SUST_B_3D_I16_CLAMP : SUST_3D<"sust.b.3d.b16.clamp", Int16Regs>;
+defm SUST_B_3D_I32_CLAMP : SUST_3D<"sust.b.3d.b32.clamp", Int32Regs>;
+defm SUST_B_3D_I64_CLAMP : SUST_3D<"sust.b.3d.b64.clamp", Int64Regs>;
 
-defm SUST_B_3D_B8_TRAP : SUST_3D<"sust.b.3d.b8.trap", Int16Regs>;
-defm SUST_B_3D_B16_TRAP : SUST_3D<"sust.b.3d.b16.trap", Int16Regs>;
-defm SUST_B_3D_B32_TRAP : SUST_3D<"sust.b.3d.b32.trap", Int32Regs>;
-defm SUST_B_3D_B64_TRAP : SUST_3D<"sust.b.3d.b64.trap", Int64Regs>;
+defm SUST_B_3D_I8_TRAP : SUST_3D<"sust.b.3d.b8.trap", Int16Regs>;
+defm SUST_B_3D_I16_TRAP : SUST_3D<"sust.b.3d.b16.trap", Int16Regs>;
+defm SUST_B_3D_I32_TRAP : SUST_3D<"sust.b.3d.b32.trap", Int32Regs>;
+defm SUST_B_3D_I64_TRAP : SUST_3D<"sust.b.3d.b64.trap", Int64Regs>;
 
-defm SUST_B_3D_B8_ZERO : SUST_3D<"sust.b.3d.b8.zero", Int16Regs>;
-defm SUST_B_3D_B16_ZERO : SUST_3D<"sust.b.3d.b16.zero", Int16Regs>;
-defm SUST_B_3D_B32_ZERO : SUST_3D<"sust.b.3d.b32.zero", Int32Regs>;
-defm SUST_B_3D_B64_ZERO : SUST_3D<"sust.b.3d.b64.zero", Int64Regs>;
+defm SUST_B_3D_I8_ZERO : SUST_3D<"sust.b.3d.b8.zero", Int16Regs>;
+defm SUST_B_3D_I16_ZERO : SUST_3D<"sust.b.3d.b16.zero", Int16Regs>;
+defm SUST_B_3D_I32_ZERO : SUST_3D<"sust.b.3d.b32.zero", Int32Regs>;
+defm SUST_B_3D_I64_ZERO : SUST_3D<"sust.b.3d.b64.zero", Int64Regs>;
 
-defm SUST_P_3D_B8_TRAP : SUST_3D<"sust.p.3d.b8.trap", Int16Regs>;
-defm SUST_P_3D_B16_TRAP : SUST_3D<"sust.p.3d.b16.trap", Int16Regs>;
-defm SUST_P_3D_B32_TRAP : SUST_3D<"sust.p.3d.b32.trap", Int32Regs>;
+defm SUST_P_3D_I8_TRAP : SUST_3D<"sust.p.3d.b8.trap", Int16Regs>;
+defm SUST_P_3D_I16_TRAP : SUST_3D<"sust.p.3d.b16.trap", Int16Regs>;
+defm SUST_P_3D_I32_TRAP : SUST_3D<"sust.p.3d.b32.trap", Int32Regs>;
 
-class SUST_3D_V2_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_3D_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
                                 intype:$r, intype:$g)),
                 inst # " \t[$s, \\{$x, $y, $z, $z\\}], \\{$r, $g\\};",
-                []>;
+                pat>;
 multiclass SUST_3D_V2<string inst, NVPTXRegClass intype> {
-  def _R : SUST_3D_V2_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_3D_V2_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_3D_V2_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
+                intype:$r, intype:$g)]>;
+  def _I : SUST_3D_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_3D_V2B8_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b8.clamp", Int16Regs>;
-defm SUST_B_3D_V2B16_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b16.clamp", Int16Regs>;
-defm SUST_B_3D_V2B32_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b32.clamp", Int32Regs>;
-defm SUST_B_3D_V2B64_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b64.clamp", Int64Regs>;
+defm SUST_B_3D_V2I8_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b8.clamp", Int16Regs>;
+defm SUST_B_3D_V2I16_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b16.clamp", Int16Regs>;
+defm SUST_B_3D_V2I32_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b32.clamp", Int32Regs>;
+defm SUST_B_3D_V2I64_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b64.clamp", Int64Regs>;
 
-defm SUST_B_3D_V2B8_TRAP : SUST_3D_V2<"sust.b.3d.v2.b8.trap", Int16Regs>;
-defm SUST_B_3D_V2B16_TRAP : SUST_3D_V2<"sust.b.3d.v2.b16.trap", Int16Regs>;
-defm SUST_B_3D_V2B32_TRAP : SUST_3D_V2<"sust.b.3d.v2.b32.trap", Int32Regs>;
-defm SUST_B_3D_V2B64_TRAP : SUST_3D_V2<"sust.b.3d.v2.b64.trap", Int64Regs>;
+defm SUST_B_3D_V2I8_TRAP : SUST_3D_V2<"sust.b.3d.v2.b8.trap", Int16Regs>;
+defm SUST_B_3D_V2I16_TRAP : SUST_3D_V2<"sust.b.3d.v2.b16.trap", Int16Regs>;
+defm SUST_B_3D_V2I32_TRAP : SUST_3D_V2<"sust.b.3d.v2.b32.trap", Int32Regs>;
+defm SUST_B_3D_V2I64_TRAP : SUST_3D_V2<"sust.b.3d.v2.b64.trap", Int64Regs>;
 
-defm SUST_B_3D_V2B8_ZERO : SUST_3D_V2<"sust.b.3d.v2.b8.zero", Int16Regs>;
-defm SUST_B_3D_V2B16_ZERO : SUST_3D_V2<"sust.b.3d.v2.b16.zero", Int16Regs>;
-defm SUST_B_3D_V2B32_ZERO : SUST_3D_V2<"sust.b.3d.v2.b32.zero", Int32Regs>;
-defm SUST_B_3D_V2B64_ZERO : SUST_3D_V2<"sust.b.3d.v2.b64.zero", Int64Regs>;
+defm SUST_B_3D_V2I8_ZERO : SUST_3D_V2<"sust.b.3d.v2.b8.zero", Int16Regs>;
+defm SUST_B_3D_V2I16_ZERO : SUST_3D_V2<"sust.b.3d.v2.b16.zero", Int16Regs>;
+defm SUST_B_3D_V2I32_ZERO : SUST_3D_V2<"sust.b.3d.v2.b32.zero", Int32Regs>;
+defm SUST_B_3D_V2I64_ZERO : SUST_3D_V2<"sust.b.3d.v2.b64.zero", Int64Regs>;
 
-defm SUST_P_3D_V2B8_TRAP : SUST_3D_V2<"sust.p.3d.v2.b8.trap", Int16Regs>;
-defm SUST_P_3D_V2B16_TRAP : SUST_3D_V2<"sust.p.3d.v2.b16.trap", Int16Regs>;
-defm SUST_P_3D_V2B32_TRAP : SUST_3D_V2<"sust.p.3d.v2.b32.trap", Int32Regs>;
+defm SUST_P_3D_V2I8_TRAP : SUST_3D_V2<"sust.p.3d.v2.b8.trap", Int16Regs>;
+defm SUST_P_3D_V2I16_TRAP : SUST_3D_V2<"sust.p.3d.v2.b16.trap", Int16Regs>;
+defm SUST_P_3D_V2I32_TRAP : SUST_3D_V2<"sust.p.3d.v2.b32.trap", Int32Regs>;
 
-class SUST_3D_V4_base<string inst, NVPTXRegClass intype, dag surf>
+class SUST_3D_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
                 !con(surf, (ins Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
                                 intype:$r, intype:$g, intype:$b, intype:$a)),
                 inst # " \t[$s, \\{$x, $y, $z, $z\\}], \\{$r, $g, $b, $a\\};",
-                []>;
+                pat>;
 multiclass SUST_3D_V4<string inst, NVPTXRegClass intype> {
-  def _R : SUST_3D_V4_base<inst, intype, (ins Int64Regs:$s)>;
-  def _I : SUST_3D_V4_base<inst, intype, (ins i64imm:$s)>;
+  defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));
+  def _R : SUST_3D_V4_base<inst, intype, (ins Int64Regs:$s),
+              [(intr Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
+                intype:$r, intype:$g, intype:$b, intype:$a)]>;
+  def _I : SUST_3D_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_3D_V4B8_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b8.clamp", Int16Regs>;
-defm SUST_B_3D_V4B16_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b16.clamp", Int16Regs>;
-defm SUST_B_3D_V4B32_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b32.clamp", Int32Regs>;
+defm SUST_B_3D_V4I8_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b8.clamp", Int16Regs>;
+defm SUST_B_3D_V4I16_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b16.clamp", Int16Regs>;
+defm SUST_B_3D_V4I32_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b32.clamp", Int32Regs>;
+
+defm SUST_B_3D_V4I8_TRAP : SUST_3D_V4<"sust.b.3d.v4.b8.trap", Int16Regs>;
+defm SUST_B_3D_V4I16_TRAP : SUST_3D_V4<"sust.b.3d.v4.b16.trap", Int16Regs>;
+defm SUST_B_3D_V4I32_TRAP : SUST_3D_V4<"sust.b.3d.v4.b32.trap", Int32Regs>;
 
-defm SUST_B_3D_V4B8_TRAP : SUST_3D_V4<"sust.b.3d.v4.b8.trap", Int16Regs>;
-defm SUST_B_3D_V4B16_TRAP : SUST_3D_V4<"sust.b.3d.v4.b16.trap", Int16Regs>;
-defm SUST_B_3D_V4B32_TRAP : SUST_3D_V4<"sust.b.3d.v4.b32.trap", Int32Regs>;
-
-defm SUST_B_3D_V4B8_ZERO : SUST_3D_V4<"sust.b.3d.v4.b8.zero", Int16Regs>;
-defm SUST_B_3D_V4B16_ZERO : SUST_3D_V4<"sust.b.3d.v4.b16.zero", Int16Regs>;
-defm SUST_B_3D_V4B32_ZERO : SUST_3D_V4<"sust.b.3d.v4.b32.zero", Int32Regs>;
-
-defm SUST_P_3D_V4B8_TRAP : SUST_3D_V4<"sust.p.3d.v4.b8.trap", Int16Regs>;
-defm SUST_P_3D_V4B16_TRAP : SUST_3D_V4<"sust.p.3d.v4.b16.trap", Int16Regs>;
-defm SUST_P_3D_V4B32_TRAP : SUST_3D_V4<"sust.p.3d.v4.b32.trap", Int32Regs>;
-
-}
-
-// Surface store instruction patterns
-// I'm not sure why we can't just include these in the instruction definitions,
-// but TableGen complains of type errors :(
-
-// .clamp variant
-def : Pat<(int_nvvm_sust_b_1d_i8_clamp
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_B8_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i16_clamp
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_B16_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i32_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
-          (SUST_B_1D_B32_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i64_clamp
-           Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
-          (SUST_B_1D_B64_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
+defm SUST_B_3D_V4I8_ZERO : SUST_3D_V4<"sust.b.3d.v4.b8.zero", Int16Regs>;
+defm SUST_B_3D_V4I16_ZERO : SUST_3D_V4<"sust.b.3d.v4.b16.zero", Int16Regs>;
+defm SUST_B_3D_V4I32_ZERO : SUST_3D_V4<"sust.b.3d.v4.b32.zero", Int32Regs>;
+
+defm SUST_P_3D_V4I8_TRAP : SUST_3D_V4<"sust.p.3d.v4.b8.trap", Int16Regs>;
+defm SUST_P_3D_V4I16_TRAP : SUST_3D_V4<"sust.p.3d.v4.b16.trap", Int16Regs>;
+defm SUST_P_3D_V4I32_TRAP : SUST_3D_V4<"sust.p.3d.v4.b32.trap", Int32Regs>;
+
+}
 
-def : Pat<(int_nvvm_sust_b_1d_v2i8_clamp
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_V2B8_CLAMP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i16_clamp
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_V2B16_CLAMP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i32_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_1D_V2B32_CLAMP_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i64_clamp
-           Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_1D_V2B64_CLAMP_R Int64Regs:$s, Int32Regs:$x,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i8_clamp
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_V4B8_CLAMP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i16_clamp
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_V4B16_CLAMP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i32_clamp
-           Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_1D_V4B32_CLAMP_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_1d_array_i8_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_ARRAY_B8_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i16_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_ARRAY_B16_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i32_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
-          (SUST_B_1D_ARRAY_B32_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i64_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
-          (SUST_B_1D_ARRAY_B64_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i8_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_ARRAY_V2B8_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i16_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_ARRAY_V2B16_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i32_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_1D_ARRAY_V2B32_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i64_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_1D_ARRAY_V2B64_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i8_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_ARRAY_V4B8_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i16_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_ARRAY_V4B16_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i32_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_1D_ARRAY_V4B32_CLAMP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_2d_i8_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_B8_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i16_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_B16_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i32_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_B_2D_B32_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i64_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
-          (SUST_B_2D_B64_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i8_clamp
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_V2B8_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i16_clamp
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_V2B16_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i32_clamp
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_2D_V2B32_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i64_clamp
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_2D_V2B64_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i8_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_V4B8_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i16_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_V4B16_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i32_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_2D_V4B32_CLAMP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_2d_array_i8_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_ARRAY_B8_CLAMP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i16_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_ARRAY_B16_CLAMP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i32_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_B_2D_ARRAY_B32_CLAMP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i64_clamp
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
-          (SUST_B_2D_ARRAY_B64_CLAMP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i8_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_ARRAY_V2B8_CLAMP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i16_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_ARRAY_V2B16_CLAMP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i32_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
-           Int32Regs:$g),
-          (SUST_B_2D_ARRAY_V2B32_CLAMP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i64_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
-           Int64Regs:$g),
-          (SUST_B_2D_ARRAY_V2B64_CLAMP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i8_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_ARRAY_V4B8_CLAMP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i16_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_ARRAY_V4B16_CLAMP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i32_clamp
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_2D_ARRAY_V4B32_CLAMP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_3d_i8_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_B_3D_B8_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i16_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_B_3D_B16_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i32_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r),
-          (SUST_B_3D_B32_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i64_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r),
-          (SUST_B_3D_B64_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i8_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_3D_V2B8_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i16_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_3D_V2B16_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i32_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_3D_V2B32_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i64_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_3D_V2B64_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i8_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_3D_V4B8_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i16_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_3D_V4B16_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i32_clamp
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_3D_V4B32_CLAMP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-// .trap variant
-def : Pat<(int_nvvm_sust_b_1d_i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
-          (SUST_B_1D_B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i64_trap
-           Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
-          (SUST_B_1D_B64_TRAP_R Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_1D_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i64_trap
-           Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_1D_V2B64_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i8_trap
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_V4B8_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i16_trap
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_V4B16_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i32_trap
-           Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_1D_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_1d_array_i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_ARRAY_B8_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_ARRAY_B16_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
-          (SUST_B_1D_ARRAY_B32_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i64_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
-          (SUST_B_1D_ARRAY_B64_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i8_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_ARRAY_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i16_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_ARRAY_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i32_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_1D_ARRAY_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i64_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_1D_ARRAY_V2B64_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_ARRAY_V4B8_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_ARRAY_V4B16_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_1D_ARRAY_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_2d_i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_B_2D_B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i64_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
-          (SUST_B_2D_B64_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i8_trap
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i16_trap
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i32_trap
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_2D_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i64_trap
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_2D_V2B64_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_V4B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_V4B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_2D_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_2d_array_i8_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_ARRAY_B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i16_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_ARRAY_B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i32_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_B_2D_ARRAY_B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i64_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
-          (SUST_B_2D_ARRAY_B64_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_ARRAY_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_ARRAY_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
-           Int32Regs:$g),
-          (SUST_B_2D_ARRAY_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i64_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
-           Int64Regs:$g),
-          (SUST_B_2D_ARRAY_V2B64_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_ARRAY_V4B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_ARRAY_V4B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_2D_ARRAY_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_3d_i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_B_3D_B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_B_3D_B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r),
-          (SUST_B_3D_B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i64_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r),
-          (SUST_B_3D_B64_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_3D_V2B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_3D_V2B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_3D_V2B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i64_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_3D_V2B64_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_3D_V4B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_3D_V4B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_3D_V4B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-// .zero variant
-def : Pat<(int_nvvm_sust_b_1d_i8_zero
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_B8_ZERO_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i16_zero
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_B16_ZERO_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i32_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
-          (SUST_B_1D_B32_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_i64_zero
-           Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
-          (SUST_B_1D_B64_ZERO_R Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i8_zero
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_V2B8_ZERO_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i16_zero
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_V2B16_ZERO_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i32_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_1D_V2B32_ZERO_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v2i64_zero
-           Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_1D_V2B64_ZERO_R Int64Regs:$s, Int32Regs:$x,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i8_zero
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_V4B8_ZERO_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i16_zero
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_V4B16_ZERO_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_v4i32_zero
-           Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_1D_V4B32_ZERO_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_1d_array_i8_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_ARRAY_B8_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i16_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_B_1D_ARRAY_B16_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i32_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
-          (SUST_B_1D_ARRAY_B32_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_i64_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
-          (SUST_B_1D_ARRAY_B64_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i8_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_ARRAY_V2B8_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i16_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_1D_ARRAY_V2B16_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i32_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_1D_ARRAY_V2B32_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v2i64_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_1D_ARRAY_V2B64_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i8_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_ARRAY_V4B8_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i16_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_1D_ARRAY_V4B16_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_1d_array_v4i32_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_1D_ARRAY_V4B32_ZERO_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_2d_i8_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_B8_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i16_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_B16_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i32_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_B_2D_B32_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_i64_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
-          (SUST_B_2D_B64_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i8_zero
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_V2B8_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i16_zero
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_V2B16_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i32_zero
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_2D_V2B32_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v2i64_zero
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_2D_V2B64_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i8_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_V4B8_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i16_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_V4B16_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_v4i32_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_2D_V4B32_ZERO_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_2d_array_i8_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_ARRAY_B8_ZERO_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i16_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_B_2D_ARRAY_B16_ZERO_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i32_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_B_2D_ARRAY_B32_ZERO_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_i64_zero
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
-          (SUST_B_2D_ARRAY_B64_ZERO_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i8_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_ARRAY_V2B8_ZERO_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i16_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_2D_ARRAY_V2B16_ZERO_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i32_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
-           Int32Regs:$g),
-          (SUST_B_2D_ARRAY_V2B32_ZERO_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v2i64_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
-           Int64Regs:$g),
-          (SUST_B_2D_ARRAY_V2B64_ZERO_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i8_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_ARRAY_V4B8_ZERO_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i16_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_2D_ARRAY_V4B16_ZERO_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_2d_array_v4i32_zero
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_2D_ARRAY_V4B32_ZERO_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_b_3d_i8_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_B_3D_B8_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i16_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_B_3D_B16_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i32_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r),
-          (SUST_B_3D_B32_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_i64_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r),
-          (SUST_B_3D_B64_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i8_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_3D_V2B8_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i16_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_B_3D_V2B16_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i32_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g),
-          (SUST_B_3D_V2B32_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v2i64_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r, Int64Regs:$g),
-          (SUST_B_3D_V2B64_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int64Regs:$r, Int64Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i8_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_3D_V4B8_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i16_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_B_3D_V4B16_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_b_3d_v4i32_zero
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_B_3D_V4B32_ZERO_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-
-def : Pat<(int_nvvm_sust_p_1d_i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_P_1D_B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_1d_i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
-          (SUST_P_1D_B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_1d_i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
-          (SUST_P_1D_B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_1d_v2i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_1D_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_1d_v2i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_1D_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_1d_v2i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_P_1D_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_1d_v4i8_trap
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_1D_V4B8_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_1d_v4i16_trap
-           Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_1D_V4B16_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_1d_v4i32_trap
-           Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_P_1D_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_p_1d_array_i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_P_1D_ARRAY_B8_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
-          (SUST_P_1D_ARRAY_B16_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
-          (SUST_P_1D_ARRAY_B32_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_v2i8_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_1D_ARRAY_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_v2i16_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_1D_ARRAY_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_v2i32_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
-          (SUST_P_1D_ARRAY_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_v4i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_1D_ARRAY_V4B8_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_v4i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_1D_ARRAY_V4B16_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_1d_array_v4i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_P_1D_ARRAY_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_p_2d_i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_P_2D_B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_2d_i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_P_2D_B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_2d_i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_P_2D_B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_2d_v2i8_trap
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_2D_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_2d_v2i16_trap
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_2D_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_2d_v2i32_trap
-          Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
-          (SUST_P_2D_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_2d_v4i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_2D_V4B8_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_2d_v4i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_2D_V4B16_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_2d_v4i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_P_2D_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_p_2d_array_i8_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_P_2D_ARRAY_B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_i16_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
-          (SUST_P_2D_ARRAY_B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_i32_trap
-          Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
-          (SUST_P_2D_ARRAY_B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_v2i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_2D_ARRAY_V2B8_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_v2i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_2D_ARRAY_V2B16_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_v2i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
-           Int32Regs:$g),
-          (SUST_P_2D_ARRAY_V2B32_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_v4i8_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_2D_ARRAY_V4B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_v4i16_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_2D_ARRAY_V4B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_2d_array_v4i32_trap
-           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_P_2D_ARRAY_V4B32_TRAP_R Int64Regs:$s, Int32Regs:$l,
-           Int32Regs:$x, Int32Regs:$y,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
-
-
-
-def : Pat<(int_nvvm_sust_p_3d_i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_P_3D_B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_3d_i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r),
-          (SUST_P_3D_B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_3d_i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r),
-          (SUST_P_3D_B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r)>;
-
-def : Pat<(int_nvvm_sust_p_3d_v2i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_3D_V2B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_3d_v2i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g),
-          (SUST_P_3D_V2B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_3d_v2i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g),
-          (SUST_P_3D_V2B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g)>;
-
-def : Pat<(int_nvvm_sust_p_3d_v4i8_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_3D_V4B8_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_3d_v4i16_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
-          (SUST_P_3D_V4B16_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
-
-def : Pat<(int_nvvm_sust_p_3d_v4i32_trap
-           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
-          (SUST_P_3D_V4B32_TRAP_R Int64Regs:$s,
-           Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
-           Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
 
 //-----------------------------------
 // Read Special Registers
@@ -6401,13 +5044,13 @@ def : Pat<(int_nvvm_sust_p_3d_v4i32_trap
 
 class PTX_READ_SREG_R64<string regname, Intrinsic intop, list<Predicate> Preds=[]>
   : NVPTXInst<(outs Int64Regs:$d), (ins),
-              !strconcat("mov.u64 \t$d, %", regname, ";"),
+              "mov.u64 \t$d, %" # regname # ";",
               [(set i64:$d, (intop))]>,
     Requires<Preds>;
 
 class PTX_READ_SREG_R32<string regname, Intrinsic intop, list<Predicate> Preds=[]>
   : NVPTXInst<(outs Int32Regs:$d), (ins),
-              !strconcat("mov.u32 \t$d, %", regname, ";"),
+              "mov.u32 \t$d, %" # regname # ";",
               [(set i32:$d, (intop))]>,
     Requires<Preds>;
 
@@ -6537,7 +5180,7 @@ class WMMA_REGINFO<WMMA_REGS r, string op>
          !or(!eq(ptx_elt_type, "f16"),
              !eq(ptx_elt_type, "f32"))) : [hasSM<70>, hasPTX<60>],
 
-    !and(!eq(geom,"m8n8k4"),
+    !and(!eq(geom, "m8n8k4"),
          !eq(ptx_elt_type, "f64")) : [hasSM<80>, hasPTX<70>],
 
     // fp16 -> fp16/fp32 @ m8n32k16/m32n8k16
@@ -6547,46 +5190,46 @@ class WMMA_REGINFO<WMMA_REGS r, string op>
              !eq(ptx_elt_type, "f32"))) : [hasSM<70>, hasPTX<61>],
 
     // u8/s8 -> s32 @ m16n16k16/m8n32k16/m32n8k16
-    !and(!or(!eq(geom,"m16n16k16"),
-             !eq(geom,"m8n32k16"),
-             !eq(geom,"m32n8k16")),
+    !and(!or(!eq(geom, "m16n16k16"),
+             !eq(geom, "m8n32k16"),
+             !eq(geom, "m32n8k16")),
          !or(!eq(ptx_elt_type, "u8"),
              !eq(ptx_elt_type, "s8"),
              !eq(ptx_elt_type, "s32"))) : [hasSM<72>, hasPTX<63>],
 
-    !and(!or(!eq(geom,"m16n16k16"),
-             !eq(geom,"m8n32k16"),
-             !eq(geom,"m32n8k16")),
+    !and(!or(!eq(geom, "m16n16k16"),
+             !eq(geom, "m8n32k16"),
+             !eq(geom, "m32n8k16")),
          !eq(ptx_elt_type, "bf16")) : [hasSM<80>, hasPTX<70>],
 
-    !and(!eq(geom,"m16n16k8"),
+    !and(!eq(geom, "m16n16k8"),
          !eq(ptx_elt_type, "tf32")) : [hasSM<80>, hasPTX<70>],
 
-    !and(!eq(geom,"m16n16k8"),
+    !and(!eq(geom, "m16n16k8"),
          !eq(ptx_elt_type, "f32")) : [hasSM<80>, hasPTX<70>],
 
     // b1 -> s32 @ m8n8k128(b1)
-    !and(!ne(op,"mma"),
-         !eq(geom,"m8n8k128")) : [hasSM<75>, hasPTX<63>],
+    !and(!ne(op, "mma"),
+         !eq(geom, "m8n8k128")) : [hasSM<75>, hasPTX<63>],
 
     // u4/s4 -> s32 @ m8n8k32 (u4/s4)
-    !and(!ne(op,"mma"),
-         !eq(geom,"m8n8k32")) : [hasSM<75>, hasPTX<63>],
+    !and(!ne(op, "mma"),
+         !eq(geom, "m8n8k32")) : [hasSM<75>, hasPTX<63>],
 
-    !or(!eq(geom,"m16n8k8"),
-        !eq(geom,"m8n8k16")) : [hasSM<75>, hasPTX<65>],
+    !or(!eq(geom, "m16n8k8"),
+        !eq(geom, "m8n8k16")) : [hasSM<75>, hasPTX<65>],
 
-    !and(!ne(ptx_elt_type,"f64"),
+    !and(!ne(ptx_elt_type, "f64"),
          !eq(geom, "m8n8k4")) : [hasSM<70>, hasPTX<64>],
 
     // mma m8n8k32 requires higher PTX version
-    !and(!eq(op,"mma"),
-         !eq(geom,"m8n8k32")) : [hasSM<75>, hasPTX<65>],
+    !and(!eq(op, "mma"),
+         !eq(geom, "m8n8k32")) : [hasSM<75>, hasPTX<65>],
 
-    !and(!eq(ptx_elt_type,"f64"),
+    !and(!eq(ptx_elt_type, "f64"),
          !eq(geom, "m8n8k4")) : [hasSM<80>, hasPTX<70>],
 
-    !and(!eq(op,"mma"),
+    !and(!eq(op, "mma"),
          !or(!eq(geom, "m16n8k16"),
              !eq(geom, "m16n8k4"),
              !eq(geom, "m16n8k32"),
@@ -6595,28 +5238,28 @@ class WMMA_REGINFO<WMMA_REGS r, string op>
              !eq(geom, "m16n8k128"),
              !eq(geom, "m16n8k256"))) : [hasSM<80>, hasPTX<70>],
 
-    !and(!eq(op,"ldmatrix"),
-         !eq(ptx_elt_type,"b16"),
+    !and(!eq(op, "ldmatrix"),
+         !eq(ptx_elt_type, "b16"),
          !eq(geom, "m8n8")) : [hasSM<75>, hasPTX<65>],
 
-    !and(!eq(op,"ldmatrix"),
-         !eq(ptx_elt_type,"b8"),
+    !and(!eq(op, "ldmatrix"),
+         !eq(ptx_elt_type, "b8"),
          !eq(geom, "m16n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],
 
-    !and(!eq(op,"ldmatrix"),
-         !eq(ptx_elt_type,"b8x16.b6x16_p32"),
+    !and(!eq(op, "ldmatrix"),
+         !eq(ptx_elt_type, "b8x16.b6x16_p32"),
          !eq(geom, "m16n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],
 
-    !and(!eq(op,"ldmatrix"),
-         !eq(ptx_elt_type,"b8x16.b4x16_p64"),
+    !and(!eq(op, "ldmatrix"),
+         !eq(ptx_elt_type, "b8x16.b4x16_p64"),
          !eq(geom, "m16n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],
 
-    !and(!eq(op,"ldmatrix"),
-         !eq(ptx_elt_type,"b8x16.b6x16_p32"),
+    !and(!eq(op, "ldmatrix"),
+         !eq(ptx_elt_type, "b8x16.b6x16_p32"),
          !eq(geom, "m8n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],
 
-    !and(!eq(op,"ldmatrix"),
-         !eq(ptx_elt_type,"b8x16.b4x16_p64"),
+    !and(!eq(op, "ldmatrix"),
+         !eq(ptx_elt_type, "b8x16.b4x16_p64"),
          !eq(geom, "m8n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>]);
 
   // template DAGs for instruction inputs/output.
@@ -6645,7 +5288,7 @@ class WMMA_INSTR<string _Intr, list<dag> _Args>
   : NVPTXInst<(outs), (ins), "?", []> {
   Intrinsic Intr = !cast<Intrinsic>(_Intr);
   // Concatenate all arguments into a single dag.
-  dag Args = !foldl((ins), _Args, a, b, !con(a,b));
+  dag Args = !foldl((ins), _Args, a, b, !con(a, b));
   // Pre-build the pattern to match (intrinsic arg0, arg1, ...).
   dag IntrinsicPattern = BuildPatternI<!cast<Intrinsic>(Intr), Args>.ret;
 }
@@ -6751,7 +5394,7 @@ class MMA_OP_PREDICATES<WMMA_REGINFO FragA, string b1op> {
   WMMA_REGINFO Frag = FragA;
   list<Predicate> ret = !listconcat(
     FragA.Predicates,
-    !if(!eq(b1op, ".and.popc"), [hasSM<80>,hasPTX<71>],[])
+    !if(!eq(b1op, ".and.popc"), [hasSM<80>, hasPTX<71>], [])
   );
 }
 // WMMA.MMA
@@ -6998,25 +5641,22 @@ def INT_EXIT : BasicNVPTXInst<(outs), (ins), "exit", [(int_nvvm_exit)]>;
 // Tcgen05 intrinsics
 let isConvergent = true in {
 
-multiclass TCGEN05_ALLOC_INTR<NVPTXRegClass rc, string AS, string num, Intrinsic Intr> {
-  def NAME : NVPTXInst<(outs),
-             (ins rc:$dst, Int32Regs:$ncols),
-             !strconcat("tcgen05.alloc.cta_group::", num, ".sync.aligned", AS, ".b32 [$dst], $ncols;"),
-             [(Intr rc:$dst, Int32Regs:$ncols)]>,
+multiclass TCGEN05_ALLOC_INTR<string AS, string num, Intrinsic Intr> {
+  def "" : BasicNVPTXInst<(outs),
+             (ins ADDR:$dst, Int32Regs:$ncols),
+             "tcgen05.alloc.cta_group::" # num # ".sync.aligned" # AS # ".b32",
+             [(Intr addr:$dst, Int32Regs:$ncols)]>,
              Requires<[hasTcgen05Instructions]>;
 }
 
-defm TCGEN05_ALLOC_CG1 : TCGEN05_ALLOC_INTR<Int64Regs, "", "1", int_nvvm_tcgen05_alloc_cg1>;
-defm TCGEN05_ALLOC_CG2 : TCGEN05_ALLOC_INTR<Int64Regs, "", "2", int_nvvm_tcgen05_alloc_cg2>;
-
-defm TCGEN05_ALLOC_S64_CG1 : TCGEN05_ALLOC_INTR<Int64Regs, ".shared::cta", "1", int_nvvm_tcgen05_alloc_shared_cg1>;
-defm TCGEN05_ALLOC_S64_CG2 : TCGEN05_ALLOC_INTR<Int64Regs, ".shared::cta", "2", int_nvvm_tcgen05_alloc_shared_cg2>;
+defm TCGEN05_ALLOC_CG1 : TCGEN05_ALLOC_INTR<"", "1", int_nvvm_tcgen05_alloc_cg1>;
+defm TCGEN05_ALLOC_CG2 : TCGEN05_ALLOC_INTR<"", "2", int_nvvm_tcgen05_alloc_cg2>;
 
-defm TCGEN05_ALLOC_S32_CG1 : TCGEN05_ALLOC_INTR<Int32Regs, ".shared::cta", "1", int_nvvm_tcgen05_alloc_shared_cg1>;
-defm TCGEN05_ALLOC_S32_CG2 : TCGEN05_ALLOC_INTR<Int32Regs, ".shared::cta", "2", int_nvvm_tcgen05_alloc_shared_cg2>;
+defm TCGEN05_ALLOC_S64_CG1 : TCGEN05_ALLOC_INTR<".shared::cta", "1", int_nvvm_tcgen05_alloc_shared_cg1>;
+defm TCGEN05_ALLOC_S64_CG2 : TCGEN05_ALLOC_INTR<".shared::cta", "2", int_nvvm_tcgen05_alloc_shared_cg2>;
 
 multiclass TCGEN05_DEALLOC_INTR<string num, Intrinsic Intr> {
-  def NAME : BasicNVPTXInst<(outs),
+  def "" : BasicNVPTXInst<(outs),
              (ins Int32Regs:$tmem_addr, Int32Regs:$ncols),
              "tcgen05.dealloc.cta_group::" # num # ".sync.aligned.b32",
              [(Intr Int32Regs:$tmem_addr, Int32Regs:$ncols)]>,
@@ -7026,7 +5666,7 @@ defm TCGEN05_DEALLOC_CG1: TCGEN05_DEALLOC_INTR<"1", int_nvvm_tcgen05_dealloc_cg1
 defm TCGEN05_DEALLOC_CG2: TCGEN05_DEALLOC_INTR<"2", int_nvvm_tcgen05_dealloc_cg2>;
 
 multiclass TCGEN05_RELINQ_PERMIT_INTR<string num, Intrinsic Intr> {
-  def NAME : BasicNVPTXInst<(outs), (ins),
+  def "" : BasicNVPTXInst<(outs), (ins),
              "tcgen05.relinquish_alloc_permit.cta_group::" # num # ".sync.aligned",
              [(Intr)]>,
              Requires<[hasTcgen05Instructions]>;
@@ -7042,36 +5682,33 @@ def tcgen05_wait_st: BasicNVPTXInst<(outs), (ins), "tcgen05.wait::st.sync.aligne
   [(int_nvvm_tcgen05_wait_st)]>,
   Requires<[hasTcgen05Instructions]>;
 
-multiclass TCGEN05_COMMIT_INTR<NVPTXRegClass rc, string AS, string num> {
-  defvar prefix = "tcgen05.commit.cta_group::" # num;
-  defvar suffix = ".mbarrier::arrive::one.shared::cluster";
+multiclass TCGEN05_COMMIT_INTR<string AS, string num> {
+  defvar prefix = "tcgen05.commit.cta_group::" # num #".mbarrier::arrive::one.shared::cluster";
 
   defvar intr_suffix = !if(!eq(AS, "shared"), "_shared", "") # "_cg" # num;
   defvar Intr = !cast<Intrinsic>("int_nvvm_tcgen05_commit" # intr_suffix);
   defvar IntrMC = !cast<Intrinsic>("int_nvvm_tcgen05_commit_mc" # intr_suffix);
 
-  def NAME : NVPTXInst<(outs), (ins rc:$mbar),
-             !strconcat(prefix, suffix, ".b64 [$mbar];"),
-             [(Intr rc:$mbar)]>,
+  def "" : BasicNVPTXInst<(outs), (ins ADDR:$mbar),
+             prefix # ".b64",
+             [(Intr addr:$mbar)]>,
              Requires<[hasTcgen05Instructions]>;
-  def NAME # _MC : NVPTXInst<(outs), (ins rc:$mbar, Int16Regs:$mc),
-                   !strconcat(prefix, suffix, ".multicast::cluster.b64 [$mbar], $mc;"),
-                   [(IntrMC rc:$mbar, Int16Regs:$mc)]>,
+  def _MC : BasicNVPTXInst<(outs), (ins ADDR:$mbar, Int16Regs:$mc),
+                   prefix # ".multicast::cluster.b64",
+                   [(IntrMC addr:$mbar, Int16Regs:$mc)]>,
                    Requires<[hasTcgen05Instructions]>;
 }
 
-defm TCGEN05_COMMIT_CG1 : TCGEN05_COMMIT_INTR<Int64Regs, "", "1">;
-defm TCGEN05_COMMIT_CG2 : TCGEN05_COMMIT_INTR<Int64Regs, "", "2">;
-defm TCGEN05_COMMIT_S64_CG1 : TCGEN05_COMMIT_INTR<Int64Regs, "shared", "1">;
-defm TCGEN05_COMMIT_S64_CG2 : TCGEN05_COMMIT_INTR<Int64Regs, "shared", "2">;
-defm TCGEN05_COMMIT_S32_CG1 : TCGEN05_COMMIT_INTR<Int32Regs, "shared", "1">;
-defm TCGEN05_COMMIT_S32_CG2 : TCGEN05_COMMIT_INTR<Int32Regs, "shared", "2">;
+defm TCGEN05_COMMIT_CG1 : TCGEN05_COMMIT_INTR<"", "1">;
+defm TCGEN05_COMMIT_CG2 : TCGEN05_COMMIT_INTR<"", "2">;
+defm TCGEN05_COMMIT_S64_CG1 : TCGEN05_COMMIT_INTR<"shared", "1">;
+defm TCGEN05_COMMIT_S64_CG2 : TCGEN05_COMMIT_INTR<"shared", "2">;
 
 multiclass TCGEN05_SHIFT_INTR<string num, Intrinsic Intr> {
-  def NAME : NVPTXInst<(outs),
-             (ins Int32Regs:$tmem_addr),
-             !strconcat("tcgen05.shift.cta_group::", num, ".down [$tmem_addr];"),
-             [(Intr Int32Regs:$tmem_addr)]>,
+  def "" : BasicNVPTXInst<(outs),
+             (ins ADDR:$tmem_addr),
+             "tcgen05.shift.cta_group::" # num # ".down",
+             [(Intr addr:$tmem_addr)]>,
              Requires<[hasTcgen05Instructions]>;
 }
 defm TCGEN05_SHIFT_CG1: TCGEN05_SHIFT_INTR<"1", int_nvvm_tcgen05_shift_down_cg1>;
@@ -7089,15 +5726,15 @@ multiclass TCGEN05_CP_INTR<string shape, string src_fmt, string mc = ""> {
   defvar IntrCG1 = !cast<Intrinsic>(intr_prefix # "_cg1");
   defvar IntrCG2 = !cast<Intrinsic>(intr_prefix # "_cg2");
 
-  def NAME # _cg1 : NVPTXInst<(outs),
-                    (ins Int32Regs:$tmem_addr, Int64Regs:$sdesc),
-                    "tcgen05.cp.cta_group::1." # shape_mc_asm # fmt_asm # " [$tmem_addr], $sdesc;",
-                    [(IntrCG1 Int32Regs:$tmem_addr, Int64Regs:$sdesc)]>,
+  def _cg1 : BasicNVPTXInst<(outs),
+                    (ins ADDR:$tmem_addr, Int64Regs:$sdesc),
+                    "tcgen05.cp.cta_group::1." # shape_mc_asm # fmt_asm,
+                    [(IntrCG1 addr:$tmem_addr, Int64Regs:$sdesc)]>,
                     Requires<[hasTcgen05Instructions]>;
-  def NAME # _cg2 : NVPTXInst<(outs),
-                    (ins Int32Regs:$tmem_addr, Int64Regs:$sdesc),
-                    "tcgen05.cp.cta_group::2." # shape_mc_asm # fmt_asm # " [$tmem_addr], $sdesc;",
-                    [(IntrCG2 Int32Regs:$tmem_addr, Int64Regs:$sdesc)]>,
+  def _cg2 : BasicNVPTXInst<(outs),
+                    (ins ADDR:$tmem_addr, Int64Regs:$sdesc),
+                    "tcgen05.cp.cta_group::2." # shape_mc_asm # fmt_asm,
+                    [(IntrCG2 addr:$tmem_addr, Int64Regs:$sdesc)]>,
                     Requires<[hasTcgen05Instructions]>;
 }
 
@@ -7212,17 +5849,18 @@ foreach shape = ["16x64b", "16x128b", "16x256b", "32x32b", "16x32bx2"] in {
 } // isConvergent
 
 // Bulk store instructions
-                            
+def st_bulk_imm : TImmLeaf<i64, [{ return Imm == 0; }]>;
+
 def INT_NVVM_ST_BULK_GENERIC :
-  NVPTXInst<(outs), (ins ADDR:$dest_addr, Int64Regs:$size),
-            "st.bulk [$dest_addr], $size, 0;",
-            [(int_nvvm_st_bulk addr:$dest_addr, i64:$size, (i64 0))]>,
+  BasicNVPTXInst<(outs), (ins ADDR:$dest_addr, Int64Regs:$size, i64imm:$value),
+            "st.bulk",
+            [(int_nvvm_st_bulk addr:$dest_addr, i64:$size, st_bulk_imm:$value)]>,
             Requires<[hasSM<100>, hasPTX<86>]>;
 
 def INT_NVVM_ST_BULK_SHARED_CTA:
-  NVPTXInst<(outs), (ins ADDR:$dest_addr, Int64Regs:$size),
-            "st.bulk.shared::cta [$dest_addr], $size, 0;",
-            [(int_nvvm_st_bulk_shared_cta addr:$dest_addr, i64:$size, (i64 0))]>,
+  BasicNVPTXInst<(outs), (ins ADDR:$dest_addr, Int64Regs:$size, i64imm:$value),
+            "st.bulk.shared::cta",
+            [(int_nvvm_st_bulk_shared_cta addr:$dest_addr, i64:$size, st_bulk_imm:$value)]>,
             Requires<[hasSM<100>, hasPTX<86>]>;
 
 //
@@ -7230,17 +5868,15 @@ def INT_NVVM_ST_BULK_SHARED_CTA:
 //
 
 def CLUSTERLAUNCHCONTRL_TRY_CANCEL:
-      NVPTXInst<(outs), (ins ADDR:$addr, ADDR:$mbar),
-                "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 " #
-                "[$addr], [$mbar];",
+      BasicNVPTXInst<(outs), (ins ADDR:$addr, ADDR:$mbar),
+                "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128",
                 [(int_nvvm_clusterlaunchcontrol_try_cancel_async_shared addr:$addr, addr:$mbar)]>,
       Requires<[hasSM<100>, hasPTX<86>]>;
 
 def CLUSTERLAUNCHCONTRL_TRY_CANCEL_MULTICAST:
-      NVPTXInst<(outs), (ins ADDR:$addr, ADDR:$mbar),
+      BasicNVPTXInst<(outs), (ins ADDR:$addr, ADDR:$mbar),
                 "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes" #
-                ".multicast::cluster::all.b128 " #
-                "[$addr], [$mbar];",
+                ".multicast::cluster::all.b128",
                 [(int_nvvm_clusterlaunchcontrol_try_cancel_async_multicast_shared addr:$addr, addr:$mbar)]>,
       Requires<[hasSM<100>, hasArchAccelFeatures, hasPTX<86>]>;
 
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXMCExpr.cpp b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXMCExpr.cpp
index 95125eb41bc0..8cde0873d4d2 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXMCExpr.cpp
@@ -8,6 +8,7 @@
 
 #include "NVPTXMCExpr.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Support/Format.h"
@@ -64,6 +65,6 @@ NVPTXGenericMCSymbolRefExpr::create(const MCSymbolRefExpr *SymExpr,
 void NVPTXGenericMCSymbolRefExpr::printImpl(raw_ostream &OS,
                                             const MCAsmInfo *MAI) const {
   OS << "generic(";
-  SymExpr->print(OS, MAI);
+  MAI->printExpr(OS, *SymExpr);
   OS << ")";
 }
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
index 9b5fe473521a..320c0fb6950a 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
@@ -408,426 +408,426 @@ static unsigned suldRegisterToIndexOpcode(unsigned RegOC) {
 
 static unsigned sustRegisterToIndexOpcode(unsigned RegOC) {
   switch (RegOC) {
-  case NVPTX::SUST_B_1D_B8_CLAMP_R:
-    return NVPTX::SUST_B_1D_B8_CLAMP_I;
-  case NVPTX::SUST_B_1D_B16_CLAMP_R:
-    return NVPTX::SUST_B_1D_B16_CLAMP_I;
-  case NVPTX::SUST_B_1D_B32_CLAMP_R:
-    return NVPTX::SUST_B_1D_B32_CLAMP_I;
-  case NVPTX::SUST_B_1D_B64_CLAMP_R:
-    return NVPTX::SUST_B_1D_B64_CLAMP_I;
-  case NVPTX::SUST_B_1D_V2B8_CLAMP_R:
-    return NVPTX::SUST_B_1D_V2B8_CLAMP_I;
-  case NVPTX::SUST_B_1D_V2B16_CLAMP_R:
-    return NVPTX::SUST_B_1D_V2B16_CLAMP_I;
-  case NVPTX::SUST_B_1D_V2B32_CLAMP_R:
-    return NVPTX::SUST_B_1D_V2B32_CLAMP_I;
-  case NVPTX::SUST_B_1D_V2B64_CLAMP_R:
-    return NVPTX::SUST_B_1D_V2B64_CLAMP_I;
-  case NVPTX::SUST_B_1D_V4B8_CLAMP_R:
-    return NVPTX::SUST_B_1D_V4B8_CLAMP_I;
-  case NVPTX::SUST_B_1D_V4B16_CLAMP_R:
-    return NVPTX::SUST_B_1D_V4B16_CLAMP_I;
-  case NVPTX::SUST_B_1D_V4B32_CLAMP_R:
-    return NVPTX::SUST_B_1D_V4B32_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B8_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B8_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B16_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B16_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B32_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B32_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B64_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B64_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B8_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B8_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B16_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B16_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B32_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B32_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B64_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B64_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B8_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B8_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B16_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B16_CLAMP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B32_CLAMP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B32_CLAMP_I;
-  case NVPTX::SUST_B_2D_B8_CLAMP_R:
-    return NVPTX::SUST_B_2D_B8_CLAMP_I;
-  case NVPTX::SUST_B_2D_B16_CLAMP_R:
-    return NVPTX::SUST_B_2D_B16_CLAMP_I;
-  case NVPTX::SUST_B_2D_B32_CLAMP_R:
-    return NVPTX::SUST_B_2D_B32_CLAMP_I;
-  case NVPTX::SUST_B_2D_B64_CLAMP_R:
-    return NVPTX::SUST_B_2D_B64_CLAMP_I;
-  case NVPTX::SUST_B_2D_V2B8_CLAMP_R:
-    return NVPTX::SUST_B_2D_V2B8_CLAMP_I;
-  case NVPTX::SUST_B_2D_V2B16_CLAMP_R:
-    return NVPTX::SUST_B_2D_V2B16_CLAMP_I;
-  case NVPTX::SUST_B_2D_V2B32_CLAMP_R:
-    return NVPTX::SUST_B_2D_V2B32_CLAMP_I;
-  case NVPTX::SUST_B_2D_V2B64_CLAMP_R:
-    return NVPTX::SUST_B_2D_V2B64_CLAMP_I;
-  case NVPTX::SUST_B_2D_V4B8_CLAMP_R:
-    return NVPTX::SUST_B_2D_V4B8_CLAMP_I;
-  case NVPTX::SUST_B_2D_V4B16_CLAMP_R:
-    return NVPTX::SUST_B_2D_V4B16_CLAMP_I;
-  case NVPTX::SUST_B_2D_V4B32_CLAMP_R:
-    return NVPTX::SUST_B_2D_V4B32_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B8_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B8_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B16_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B16_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B32_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B32_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B64_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B64_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B8_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B8_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B16_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B16_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B32_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B32_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B64_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B64_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B8_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B8_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B16_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B16_CLAMP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B32_CLAMP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B32_CLAMP_I;
-  case NVPTX::SUST_B_3D_B8_CLAMP_R:
-    return NVPTX::SUST_B_3D_B8_CLAMP_I;
-  case NVPTX::SUST_B_3D_B16_CLAMP_R:
-    return NVPTX::SUST_B_3D_B16_CLAMP_I;
-  case NVPTX::SUST_B_3D_B32_CLAMP_R:
-    return NVPTX::SUST_B_3D_B32_CLAMP_I;
-  case NVPTX::SUST_B_3D_B64_CLAMP_R:
-    return NVPTX::SUST_B_3D_B64_CLAMP_I;
-  case NVPTX::SUST_B_3D_V2B8_CLAMP_R:
-    return NVPTX::SUST_B_3D_V2B8_CLAMP_I;
-  case NVPTX::SUST_B_3D_V2B16_CLAMP_R:
-    return NVPTX::SUST_B_3D_V2B16_CLAMP_I;
-  case NVPTX::SUST_B_3D_V2B32_CLAMP_R:
-    return NVPTX::SUST_B_3D_V2B32_CLAMP_I;
-  case NVPTX::SUST_B_3D_V2B64_CLAMP_R:
-    return NVPTX::SUST_B_3D_V2B64_CLAMP_I;
-  case NVPTX::SUST_B_3D_V4B8_CLAMP_R:
-    return NVPTX::SUST_B_3D_V4B8_CLAMP_I;
-  case NVPTX::SUST_B_3D_V4B16_CLAMP_R:
-    return NVPTX::SUST_B_3D_V4B16_CLAMP_I;
-  case NVPTX::SUST_B_3D_V4B32_CLAMP_R:
-    return NVPTX::SUST_B_3D_V4B32_CLAMP_I;
-  case NVPTX::SUST_B_1D_B8_TRAP_R:
-    return NVPTX::SUST_B_1D_B8_TRAP_I;
-  case NVPTX::SUST_B_1D_B16_TRAP_R:
-    return NVPTX::SUST_B_1D_B16_TRAP_I;
-  case NVPTX::SUST_B_1D_B32_TRAP_R:
-    return NVPTX::SUST_B_1D_B32_TRAP_I;
-  case NVPTX::SUST_B_1D_B64_TRAP_R:
-    return NVPTX::SUST_B_1D_B64_TRAP_I;
-  case NVPTX::SUST_B_1D_V2B8_TRAP_R:
-    return NVPTX::SUST_B_1D_V2B8_TRAP_I;
-  case NVPTX::SUST_B_1D_V2B16_TRAP_R:
-    return NVPTX::SUST_B_1D_V2B16_TRAP_I;
-  case NVPTX::SUST_B_1D_V2B32_TRAP_R:
-    return NVPTX::SUST_B_1D_V2B32_TRAP_I;
-  case NVPTX::SUST_B_1D_V2B64_TRAP_R:
-    return NVPTX::SUST_B_1D_V2B64_TRAP_I;
-  case NVPTX::SUST_B_1D_V4B8_TRAP_R:
-    return NVPTX::SUST_B_1D_V4B8_TRAP_I;
-  case NVPTX::SUST_B_1D_V4B16_TRAP_R:
-    return NVPTX::SUST_B_1D_V4B16_TRAP_I;
-  case NVPTX::SUST_B_1D_V4B32_TRAP_R:
-    return NVPTX::SUST_B_1D_V4B32_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B8_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B8_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B16_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B16_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B32_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B32_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_B64_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_B64_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B8_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B8_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B16_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B16_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B32_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B32_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B64_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B64_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B8_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B8_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B16_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B16_TRAP_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B32_TRAP_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B32_TRAP_I;
-  case NVPTX::SUST_B_2D_B8_TRAP_R:
-    return NVPTX::SUST_B_2D_B8_TRAP_I;
-  case NVPTX::SUST_B_2D_B16_TRAP_R:
-    return NVPTX::SUST_B_2D_B16_TRAP_I;
-  case NVPTX::SUST_B_2D_B32_TRAP_R:
-    return NVPTX::SUST_B_2D_B32_TRAP_I;
-  case NVPTX::SUST_B_2D_B64_TRAP_R:
-    return NVPTX::SUST_B_2D_B64_TRAP_I;
-  case NVPTX::SUST_B_2D_V2B8_TRAP_R:
-    return NVPTX::SUST_B_2D_V2B8_TRAP_I;
-  case NVPTX::SUST_B_2D_V2B16_TRAP_R:
-    return NVPTX::SUST_B_2D_V2B16_TRAP_I;
-  case NVPTX::SUST_B_2D_V2B32_TRAP_R:
-    return NVPTX::SUST_B_2D_V2B32_TRAP_I;
-  case NVPTX::SUST_B_2D_V2B64_TRAP_R:
-    return NVPTX::SUST_B_2D_V2B64_TRAP_I;
-  case NVPTX::SUST_B_2D_V4B8_TRAP_R:
-    return NVPTX::SUST_B_2D_V4B8_TRAP_I;
-  case NVPTX::SUST_B_2D_V4B16_TRAP_R:
-    return NVPTX::SUST_B_2D_V4B16_TRAP_I;
-  case NVPTX::SUST_B_2D_V4B32_TRAP_R:
-    return NVPTX::SUST_B_2D_V4B32_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B8_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B8_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B16_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B16_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B32_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B32_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_B64_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_B64_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B8_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B8_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B16_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B16_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B32_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B32_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B64_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B64_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B8_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B8_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B16_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B16_TRAP_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B32_TRAP_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B32_TRAP_I;
-  case NVPTX::SUST_B_3D_B8_TRAP_R:
-    return NVPTX::SUST_B_3D_B8_TRAP_I;
-  case NVPTX::SUST_B_3D_B16_TRAP_R:
-    return NVPTX::SUST_B_3D_B16_TRAP_I;
-  case NVPTX::SUST_B_3D_B32_TRAP_R:
-    return NVPTX::SUST_B_3D_B32_TRAP_I;
-  case NVPTX::SUST_B_3D_B64_TRAP_R:
-    return NVPTX::SUST_B_3D_B64_TRAP_I;
-  case NVPTX::SUST_B_3D_V2B8_TRAP_R:
-    return NVPTX::SUST_B_3D_V2B8_TRAP_I;
-  case NVPTX::SUST_B_3D_V2B16_TRAP_R:
-    return NVPTX::SUST_B_3D_V2B16_TRAP_I;
-  case NVPTX::SUST_B_3D_V2B32_TRAP_R:
-    return NVPTX::SUST_B_3D_V2B32_TRAP_I;
-  case NVPTX::SUST_B_3D_V2B64_TRAP_R:
-    return NVPTX::SUST_B_3D_V2B64_TRAP_I;
-  case NVPTX::SUST_B_3D_V4B8_TRAP_R:
-    return NVPTX::SUST_B_3D_V4B8_TRAP_I;
-  case NVPTX::SUST_B_3D_V4B16_TRAP_R:
-    return NVPTX::SUST_B_3D_V4B16_TRAP_I;
-  case NVPTX::SUST_B_3D_V4B32_TRAP_R:
-    return NVPTX::SUST_B_3D_V4B32_TRAP_I;
-  case NVPTX::SUST_B_1D_B8_ZERO_R:
-    return NVPTX::SUST_B_1D_B8_ZERO_I;
-  case NVPTX::SUST_B_1D_B16_ZERO_R:
-    return NVPTX::SUST_B_1D_B16_ZERO_I;
-  case NVPTX::SUST_B_1D_B32_ZERO_R:
-    return NVPTX::SUST_B_1D_B32_ZERO_I;
-  case NVPTX::SUST_B_1D_B64_ZERO_R:
-    return NVPTX::SUST_B_1D_B64_ZERO_I;
-  case NVPTX::SUST_B_1D_V2B8_ZERO_R:
-    return NVPTX::SUST_B_1D_V2B8_ZERO_I;
-  case NVPTX::SUST_B_1D_V2B16_ZERO_R:
-    return NVPTX::SUST_B_1D_V2B16_ZERO_I;
-  case NVPTX::SUST_B_1D_V2B32_ZERO_R:
-    return NVPTX::SUST_B_1D_V2B32_ZERO_I;
-  case NVPTX::SUST_B_1D_V2B64_ZERO_R:
-    return NVPTX::SUST_B_1D_V2B64_ZERO_I;
-  case NVPTX::SUST_B_1D_V4B8_ZERO_R:
-    return NVPTX::SUST_B_1D_V4B8_ZERO_I;
-  case NVPTX::SUST_B_1D_V4B16_ZERO_R:
-    return NVPTX::SUST_B_1D_V4B16_ZERO_I;
-  case NVPTX::SUST_B_1D_V4B32_ZERO_R:
-    return NVPTX::SUST_B_1D_V4B32_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_B8_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_B8_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_B16_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_B16_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_B32_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_B32_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_B64_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_B64_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B8_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B8_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B16_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B16_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B32_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B32_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_V2B64_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_V2B64_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B8_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B8_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B16_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B16_ZERO_I;
-  case NVPTX::SUST_B_1D_ARRAY_V4B32_ZERO_R:
-    return NVPTX::SUST_B_1D_ARRAY_V4B32_ZERO_I;
-  case NVPTX::SUST_B_2D_B8_ZERO_R:
-    return NVPTX::SUST_B_2D_B8_ZERO_I;
-  case NVPTX::SUST_B_2D_B16_ZERO_R:
-    return NVPTX::SUST_B_2D_B16_ZERO_I;
-  case NVPTX::SUST_B_2D_B32_ZERO_R:
-    return NVPTX::SUST_B_2D_B32_ZERO_I;
-  case NVPTX::SUST_B_2D_B64_ZERO_R:
-    return NVPTX::SUST_B_2D_B64_ZERO_I;
-  case NVPTX::SUST_B_2D_V2B8_ZERO_R:
-    return NVPTX::SUST_B_2D_V2B8_ZERO_I;
-  case NVPTX::SUST_B_2D_V2B16_ZERO_R:
-    return NVPTX::SUST_B_2D_V2B16_ZERO_I;
-  case NVPTX::SUST_B_2D_V2B32_ZERO_R:
-    return NVPTX::SUST_B_2D_V2B32_ZERO_I;
-  case NVPTX::SUST_B_2D_V2B64_ZERO_R:
-    return NVPTX::SUST_B_2D_V2B64_ZERO_I;
-  case NVPTX::SUST_B_2D_V4B8_ZERO_R:
-    return NVPTX::SUST_B_2D_V4B8_ZERO_I;
-  case NVPTX::SUST_B_2D_V4B16_ZERO_R:
-    return NVPTX::SUST_B_2D_V4B16_ZERO_I;
-  case NVPTX::SUST_B_2D_V4B32_ZERO_R:
-    return NVPTX::SUST_B_2D_V4B32_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_B8_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_B8_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_B16_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_B16_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_B32_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_B32_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_B64_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_B64_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B8_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B8_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B16_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B16_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B32_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B32_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_V2B64_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_V2B64_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B8_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B8_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B16_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B16_ZERO_I;
-  case NVPTX::SUST_B_2D_ARRAY_V4B32_ZERO_R:
-    return NVPTX::SUST_B_2D_ARRAY_V4B32_ZERO_I;
-  case NVPTX::SUST_B_3D_B8_ZERO_R:
-    return NVPTX::SUST_B_3D_B8_ZERO_I;
-  case NVPTX::SUST_B_3D_B16_ZERO_R:
-    return NVPTX::SUST_B_3D_B16_ZERO_I;
-  case NVPTX::SUST_B_3D_B32_ZERO_R:
-    return NVPTX::SUST_B_3D_B32_ZERO_I;
-  case NVPTX::SUST_B_3D_B64_ZERO_R:
-    return NVPTX::SUST_B_3D_B64_ZERO_I;
-  case NVPTX::SUST_B_3D_V2B8_ZERO_R:
-    return NVPTX::SUST_B_3D_V2B8_ZERO_I;
-  case NVPTX::SUST_B_3D_V2B16_ZERO_R:
-    return NVPTX::SUST_B_3D_V2B16_ZERO_I;
-  case NVPTX::SUST_B_3D_V2B32_ZERO_R:
-    return NVPTX::SUST_B_3D_V2B32_ZERO_I;
-  case NVPTX::SUST_B_3D_V2B64_ZERO_R:
-    return NVPTX::SUST_B_3D_V2B64_ZERO_I;
-  case NVPTX::SUST_B_3D_V4B8_ZERO_R:
-    return NVPTX::SUST_B_3D_V4B8_ZERO_I;
-  case NVPTX::SUST_B_3D_V4B16_ZERO_R:
-    return NVPTX::SUST_B_3D_V4B16_ZERO_I;
-  case NVPTX::SUST_B_3D_V4B32_ZERO_R:
-    return NVPTX::SUST_B_3D_V4B32_ZERO_I;
-  case NVPTX::SUST_P_1D_B8_TRAP_R:
-    return NVPTX::SUST_P_1D_B8_TRAP_I;
-  case NVPTX::SUST_P_1D_B16_TRAP_R:
-    return NVPTX::SUST_P_1D_B16_TRAP_I;
-  case NVPTX::SUST_P_1D_B32_TRAP_R:
-    return NVPTX::SUST_P_1D_B32_TRAP_I;
-  case NVPTX::SUST_P_1D_V2B8_TRAP_R:
-    return NVPTX::SUST_P_1D_V2B8_TRAP_I;
-  case NVPTX::SUST_P_1D_V2B16_TRAP_R:
-    return NVPTX::SUST_P_1D_V2B16_TRAP_I;
-  case NVPTX::SUST_P_1D_V2B32_TRAP_R:
-    return NVPTX::SUST_P_1D_V2B32_TRAP_I;
-  case NVPTX::SUST_P_1D_V4B8_TRAP_R:
-    return NVPTX::SUST_P_1D_V4B8_TRAP_I;
-  case NVPTX::SUST_P_1D_V4B16_TRAP_R:
-    return NVPTX::SUST_P_1D_V4B16_TRAP_I;
-  case NVPTX::SUST_P_1D_V4B32_TRAP_R:
-    return NVPTX::SUST_P_1D_V4B32_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_B8_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_B8_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_B16_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_B16_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_B32_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_B32_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_V2B8_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_V2B8_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_V2B16_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_V2B16_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_V2B32_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_V2B32_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_V4B8_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_V4B8_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_V4B16_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_V4B16_TRAP_I;
-  case NVPTX::SUST_P_1D_ARRAY_V4B32_TRAP_R:
-    return NVPTX::SUST_P_1D_ARRAY_V4B32_TRAP_I;
-  case NVPTX::SUST_P_2D_B8_TRAP_R:
-    return NVPTX::SUST_P_2D_B8_TRAP_I;
-  case NVPTX::SUST_P_2D_B16_TRAP_R:
-    return NVPTX::SUST_P_2D_B16_TRAP_I;
-  case NVPTX::SUST_P_2D_B32_TRAP_R:
-    return NVPTX::SUST_P_2D_B32_TRAP_I;
-  case NVPTX::SUST_P_2D_V2B8_TRAP_R:
-    return NVPTX::SUST_P_2D_V2B8_TRAP_I;
-  case NVPTX::SUST_P_2D_V2B16_TRAP_R:
-    return NVPTX::SUST_P_2D_V2B16_TRAP_I;
-  case NVPTX::SUST_P_2D_V2B32_TRAP_R:
-    return NVPTX::SUST_P_2D_V2B32_TRAP_I;
-  case NVPTX::SUST_P_2D_V4B8_TRAP_R:
-    return NVPTX::SUST_P_2D_V4B8_TRAP_I;
-  case NVPTX::SUST_P_2D_V4B16_TRAP_R:
-    return NVPTX::SUST_P_2D_V4B16_TRAP_I;
-  case NVPTX::SUST_P_2D_V4B32_TRAP_R:
-    return NVPTX::SUST_P_2D_V4B32_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_B8_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_B8_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_B16_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_B16_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_B32_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_B32_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_V2B8_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_V2B8_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_V2B16_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_V2B16_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_V2B32_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_V2B32_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_V4B8_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_V4B8_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_V4B16_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_V4B16_TRAP_I;
-  case NVPTX::SUST_P_2D_ARRAY_V4B32_TRAP_R:
-    return NVPTX::SUST_P_2D_ARRAY_V4B32_TRAP_I;
-  case NVPTX::SUST_P_3D_B8_TRAP_R:
-    return NVPTX::SUST_P_3D_B8_TRAP_I;
-  case NVPTX::SUST_P_3D_B16_TRAP_R:
-    return NVPTX::SUST_P_3D_B16_TRAP_I;
-  case NVPTX::SUST_P_3D_B32_TRAP_R:
-    return NVPTX::SUST_P_3D_B32_TRAP_I;
-  case NVPTX::SUST_P_3D_V2B8_TRAP_R:
-    return NVPTX::SUST_P_3D_V2B8_TRAP_I;
-  case NVPTX::SUST_P_3D_V2B16_TRAP_R:
-    return NVPTX::SUST_P_3D_V2B16_TRAP_I;
-  case NVPTX::SUST_P_3D_V2B32_TRAP_R:
-    return NVPTX::SUST_P_3D_V2B32_TRAP_I;
-  case NVPTX::SUST_P_3D_V4B8_TRAP_R:
-    return NVPTX::SUST_P_3D_V4B8_TRAP_I;
-  case NVPTX::SUST_P_3D_V4B16_TRAP_R:
-    return NVPTX::SUST_P_3D_V4B16_TRAP_I;
-  case NVPTX::SUST_P_3D_V4B32_TRAP_R:
-    return NVPTX::SUST_P_3D_V4B32_TRAP_I;
+  case NVPTX::SUST_B_1D_I8_CLAMP_R:
+    return NVPTX::SUST_B_1D_I8_CLAMP_I;
+  case NVPTX::SUST_B_1D_I16_CLAMP_R:
+    return NVPTX::SUST_B_1D_I16_CLAMP_I;
+  case NVPTX::SUST_B_1D_I32_CLAMP_R:
+    return NVPTX::SUST_B_1D_I32_CLAMP_I;
+  case NVPTX::SUST_B_1D_I64_CLAMP_R:
+    return NVPTX::SUST_B_1D_I64_CLAMP_I;
+  case NVPTX::SUST_B_1D_V2I8_CLAMP_R:
+    return NVPTX::SUST_B_1D_V2I8_CLAMP_I;
+  case NVPTX::SUST_B_1D_V2I16_CLAMP_R:
+    return NVPTX::SUST_B_1D_V2I16_CLAMP_I;
+  case NVPTX::SUST_B_1D_V2I32_CLAMP_R:
+    return NVPTX::SUST_B_1D_V2I32_CLAMP_I;
+  case NVPTX::SUST_B_1D_V2I64_CLAMP_R:
+    return NVPTX::SUST_B_1D_V2I64_CLAMP_I;
+  case NVPTX::SUST_B_1D_V4I8_CLAMP_R:
+    return NVPTX::SUST_B_1D_V4I8_CLAMP_I;
+  case NVPTX::SUST_B_1D_V4I16_CLAMP_R:
+    return NVPTX::SUST_B_1D_V4I16_CLAMP_I;
+  case NVPTX::SUST_B_1D_V4I32_CLAMP_R:
+    return NVPTX::SUST_B_1D_V4I32_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I8_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I8_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I16_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I16_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I32_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I32_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I64_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I64_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I8_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I8_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I16_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I16_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I32_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I32_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I64_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I64_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I8_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I8_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I16_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I16_CLAMP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I32_CLAMP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I32_CLAMP_I;
+  case NVPTX::SUST_B_2D_I8_CLAMP_R:
+    return NVPTX::SUST_B_2D_I8_CLAMP_I;
+  case NVPTX::SUST_B_2D_I16_CLAMP_R:
+    return NVPTX::SUST_B_2D_I16_CLAMP_I;
+  case NVPTX::SUST_B_2D_I32_CLAMP_R:
+    return NVPTX::SUST_B_2D_I32_CLAMP_I;
+  case NVPTX::SUST_B_2D_I64_CLAMP_R:
+    return NVPTX::SUST_B_2D_I64_CLAMP_I;
+  case NVPTX::SUST_B_2D_V2I8_CLAMP_R:
+    return NVPTX::SUST_B_2D_V2I8_CLAMP_I;
+  case NVPTX::SUST_B_2D_V2I16_CLAMP_R:
+    return NVPTX::SUST_B_2D_V2I16_CLAMP_I;
+  case NVPTX::SUST_B_2D_V2I32_CLAMP_R:
+    return NVPTX::SUST_B_2D_V2I32_CLAMP_I;
+  case NVPTX::SUST_B_2D_V2I64_CLAMP_R:
+    return NVPTX::SUST_B_2D_V2I64_CLAMP_I;
+  case NVPTX::SUST_B_2D_V4I8_CLAMP_R:
+    return NVPTX::SUST_B_2D_V4I8_CLAMP_I;
+  case NVPTX::SUST_B_2D_V4I16_CLAMP_R:
+    return NVPTX::SUST_B_2D_V4I16_CLAMP_I;
+  case NVPTX::SUST_B_2D_V4I32_CLAMP_R:
+    return NVPTX::SUST_B_2D_V4I32_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I8_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I8_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I16_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I16_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I32_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I32_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I64_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I64_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I8_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I8_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I16_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I16_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I32_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I32_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I64_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I64_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I8_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I8_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I16_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I16_CLAMP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I32_CLAMP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I32_CLAMP_I;
+  case NVPTX::SUST_B_3D_I8_CLAMP_R:
+    return NVPTX::SUST_B_3D_I8_CLAMP_I;
+  case NVPTX::SUST_B_3D_I16_CLAMP_R:
+    return NVPTX::SUST_B_3D_I16_CLAMP_I;
+  case NVPTX::SUST_B_3D_I32_CLAMP_R:
+    return NVPTX::SUST_B_3D_I32_CLAMP_I;
+  case NVPTX::SUST_B_3D_I64_CLAMP_R:
+    return NVPTX::SUST_B_3D_I64_CLAMP_I;
+  case NVPTX::SUST_B_3D_V2I8_CLAMP_R:
+    return NVPTX::SUST_B_3D_V2I8_CLAMP_I;
+  case NVPTX::SUST_B_3D_V2I16_CLAMP_R:
+    return NVPTX::SUST_B_3D_V2I16_CLAMP_I;
+  case NVPTX::SUST_B_3D_V2I32_CLAMP_R:
+    return NVPTX::SUST_B_3D_V2I32_CLAMP_I;
+  case NVPTX::SUST_B_3D_V2I64_CLAMP_R:
+    return NVPTX::SUST_B_3D_V2I64_CLAMP_I;
+  case NVPTX::SUST_B_3D_V4I8_CLAMP_R:
+    return NVPTX::SUST_B_3D_V4I8_CLAMP_I;
+  case NVPTX::SUST_B_3D_V4I16_CLAMP_R:
+    return NVPTX::SUST_B_3D_V4I16_CLAMP_I;
+  case NVPTX::SUST_B_3D_V4I32_CLAMP_R:
+    return NVPTX::SUST_B_3D_V4I32_CLAMP_I;
+  case NVPTX::SUST_B_1D_I8_TRAP_R:
+    return NVPTX::SUST_B_1D_I8_TRAP_I;
+  case NVPTX::SUST_B_1D_I16_TRAP_R:
+    return NVPTX::SUST_B_1D_I16_TRAP_I;
+  case NVPTX::SUST_B_1D_I32_TRAP_R:
+    return NVPTX::SUST_B_1D_I32_TRAP_I;
+  case NVPTX::SUST_B_1D_I64_TRAP_R:
+    return NVPTX::SUST_B_1D_I64_TRAP_I;
+  case NVPTX::SUST_B_1D_V2I8_TRAP_R:
+    return NVPTX::SUST_B_1D_V2I8_TRAP_I;
+  case NVPTX::SUST_B_1D_V2I16_TRAP_R:
+    return NVPTX::SUST_B_1D_V2I16_TRAP_I;
+  case NVPTX::SUST_B_1D_V2I32_TRAP_R:
+    return NVPTX::SUST_B_1D_V2I32_TRAP_I;
+  case NVPTX::SUST_B_1D_V2I64_TRAP_R:
+    return NVPTX::SUST_B_1D_V2I64_TRAP_I;
+  case NVPTX::SUST_B_1D_V4I8_TRAP_R:
+    return NVPTX::SUST_B_1D_V4I8_TRAP_I;
+  case NVPTX::SUST_B_1D_V4I16_TRAP_R:
+    return NVPTX::SUST_B_1D_V4I16_TRAP_I;
+  case NVPTX::SUST_B_1D_V4I32_TRAP_R:
+    return NVPTX::SUST_B_1D_V4I32_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I8_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I8_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I16_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I16_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I32_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I32_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_I64_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_I64_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I8_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I8_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I16_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I16_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I32_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I32_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I64_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I64_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I8_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I8_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I16_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I16_TRAP_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I32_TRAP_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I32_TRAP_I;
+  case NVPTX::SUST_B_2D_I8_TRAP_R:
+    return NVPTX::SUST_B_2D_I8_TRAP_I;
+  case NVPTX::SUST_B_2D_I16_TRAP_R:
+    return NVPTX::SUST_B_2D_I16_TRAP_I;
+  case NVPTX::SUST_B_2D_I32_TRAP_R:
+    return NVPTX::SUST_B_2D_I32_TRAP_I;
+  case NVPTX::SUST_B_2D_I64_TRAP_R:
+    return NVPTX::SUST_B_2D_I64_TRAP_I;
+  case NVPTX::SUST_B_2D_V2I8_TRAP_R:
+    return NVPTX::SUST_B_2D_V2I8_TRAP_I;
+  case NVPTX::SUST_B_2D_V2I16_TRAP_R:
+    return NVPTX::SUST_B_2D_V2I16_TRAP_I;
+  case NVPTX::SUST_B_2D_V2I32_TRAP_R:
+    return NVPTX::SUST_B_2D_V2I32_TRAP_I;
+  case NVPTX::SUST_B_2D_V2I64_TRAP_R:
+    return NVPTX::SUST_B_2D_V2I64_TRAP_I;
+  case NVPTX::SUST_B_2D_V4I8_TRAP_R:
+    return NVPTX::SUST_B_2D_V4I8_TRAP_I;
+  case NVPTX::SUST_B_2D_V4I16_TRAP_R:
+    return NVPTX::SUST_B_2D_V4I16_TRAP_I;
+  case NVPTX::SUST_B_2D_V4I32_TRAP_R:
+    return NVPTX::SUST_B_2D_V4I32_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I8_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I8_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I16_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I16_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I32_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I32_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_I64_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_I64_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I8_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I8_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I16_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I16_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I32_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I32_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I64_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I64_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I8_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I8_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I16_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I16_TRAP_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I32_TRAP_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I32_TRAP_I;
+  case NVPTX::SUST_B_3D_I8_TRAP_R:
+    return NVPTX::SUST_B_3D_I8_TRAP_I;
+  case NVPTX::SUST_B_3D_I16_TRAP_R:
+    return NVPTX::SUST_B_3D_I16_TRAP_I;
+  case NVPTX::SUST_B_3D_I32_TRAP_R:
+    return NVPTX::SUST_B_3D_I32_TRAP_I;
+  case NVPTX::SUST_B_3D_I64_TRAP_R:
+    return NVPTX::SUST_B_3D_I64_TRAP_I;
+  case NVPTX::SUST_B_3D_V2I8_TRAP_R:
+    return NVPTX::SUST_B_3D_V2I8_TRAP_I;
+  case NVPTX::SUST_B_3D_V2I16_TRAP_R:
+    return NVPTX::SUST_B_3D_V2I16_TRAP_I;
+  case NVPTX::SUST_B_3D_V2I32_TRAP_R:
+    return NVPTX::SUST_B_3D_V2I32_TRAP_I;
+  case NVPTX::SUST_B_3D_V2I64_TRAP_R:
+    return NVPTX::SUST_B_3D_V2I64_TRAP_I;
+  case NVPTX::SUST_B_3D_V4I8_TRAP_R:
+    return NVPTX::SUST_B_3D_V4I8_TRAP_I;
+  case NVPTX::SUST_B_3D_V4I16_TRAP_R:
+    return NVPTX::SUST_B_3D_V4I16_TRAP_I;
+  case NVPTX::SUST_B_3D_V4I32_TRAP_R:
+    return NVPTX::SUST_B_3D_V4I32_TRAP_I;
+  case NVPTX::SUST_B_1D_I8_ZERO_R:
+    return NVPTX::SUST_B_1D_I8_ZERO_I;
+  case NVPTX::SUST_B_1D_I16_ZERO_R:
+    return NVPTX::SUST_B_1D_I16_ZERO_I;
+  case NVPTX::SUST_B_1D_I32_ZERO_R:
+    return NVPTX::SUST_B_1D_I32_ZERO_I;
+  case NVPTX::SUST_B_1D_I64_ZERO_R:
+    return NVPTX::SUST_B_1D_I64_ZERO_I;
+  case NVPTX::SUST_B_1D_V2I8_ZERO_R:
+    return NVPTX::SUST_B_1D_V2I8_ZERO_I;
+  case NVPTX::SUST_B_1D_V2I16_ZERO_R:
+    return NVPTX::SUST_B_1D_V2I16_ZERO_I;
+  case NVPTX::SUST_B_1D_V2I32_ZERO_R:
+    return NVPTX::SUST_B_1D_V2I32_ZERO_I;
+  case NVPTX::SUST_B_1D_V2I64_ZERO_R:
+    return NVPTX::SUST_B_1D_V2I64_ZERO_I;
+  case NVPTX::SUST_B_1D_V4I8_ZERO_R:
+    return NVPTX::SUST_B_1D_V4I8_ZERO_I;
+  case NVPTX::SUST_B_1D_V4I16_ZERO_R:
+    return NVPTX::SUST_B_1D_V4I16_ZERO_I;
+  case NVPTX::SUST_B_1D_V4I32_ZERO_R:
+    return NVPTX::SUST_B_1D_V4I32_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_I8_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_I8_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_I16_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_I16_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_I32_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_I32_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_I64_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_I64_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I8_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I8_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I16_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I16_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I32_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I32_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_V2I64_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_V2I64_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I8_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I8_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I16_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I16_ZERO_I;
+  case NVPTX::SUST_B_1D_ARRAY_V4I32_ZERO_R:
+    return NVPTX::SUST_B_1D_ARRAY_V4I32_ZERO_I;
+  case NVPTX::SUST_B_2D_I8_ZERO_R:
+    return NVPTX::SUST_B_2D_I8_ZERO_I;
+  case NVPTX::SUST_B_2D_I16_ZERO_R:
+    return NVPTX::SUST_B_2D_I16_ZERO_I;
+  case NVPTX::SUST_B_2D_I32_ZERO_R:
+    return NVPTX::SUST_B_2D_I32_ZERO_I;
+  case NVPTX::SUST_B_2D_I64_ZERO_R:
+    return NVPTX::SUST_B_2D_I64_ZERO_I;
+  case NVPTX::SUST_B_2D_V2I8_ZERO_R:
+    return NVPTX::SUST_B_2D_V2I8_ZERO_I;
+  case NVPTX::SUST_B_2D_V2I16_ZERO_R:
+    return NVPTX::SUST_B_2D_V2I16_ZERO_I;
+  case NVPTX::SUST_B_2D_V2I32_ZERO_R:
+    return NVPTX::SUST_B_2D_V2I32_ZERO_I;
+  case NVPTX::SUST_B_2D_V2I64_ZERO_R:
+    return NVPTX::SUST_B_2D_V2I64_ZERO_I;
+  case NVPTX::SUST_B_2D_V4I8_ZERO_R:
+    return NVPTX::SUST_B_2D_V4I8_ZERO_I;
+  case NVPTX::SUST_B_2D_V4I16_ZERO_R:
+    return NVPTX::SUST_B_2D_V4I16_ZERO_I;
+  case NVPTX::SUST_B_2D_V4I32_ZERO_R:
+    return NVPTX::SUST_B_2D_V4I32_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_I8_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_I8_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_I16_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_I16_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_I32_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_I32_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_I64_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_I64_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I8_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I8_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I16_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I16_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I32_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I32_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_V2I64_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_V2I64_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I8_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I8_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I16_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I16_ZERO_I;
+  case NVPTX::SUST_B_2D_ARRAY_V4I32_ZERO_R:
+    return NVPTX::SUST_B_2D_ARRAY_V4I32_ZERO_I;
+  case NVPTX::SUST_B_3D_I8_ZERO_R:
+    return NVPTX::SUST_B_3D_I8_ZERO_I;
+  case NVPTX::SUST_B_3D_I16_ZERO_R:
+    return NVPTX::SUST_B_3D_I16_ZERO_I;
+  case NVPTX::SUST_B_3D_I32_ZERO_R:
+    return NVPTX::SUST_B_3D_I32_ZERO_I;
+  case NVPTX::SUST_B_3D_I64_ZERO_R:
+    return NVPTX::SUST_B_3D_I64_ZERO_I;
+  case NVPTX::SUST_B_3D_V2I8_ZERO_R:
+    return NVPTX::SUST_B_3D_V2I8_ZERO_I;
+  case NVPTX::SUST_B_3D_V2I16_ZERO_R:
+    return NVPTX::SUST_B_3D_V2I16_ZERO_I;
+  case NVPTX::SUST_B_3D_V2I32_ZERO_R:
+    return NVPTX::SUST_B_3D_V2I32_ZERO_I;
+  case NVPTX::SUST_B_3D_V2I64_ZERO_R:
+    return NVPTX::SUST_B_3D_V2I64_ZERO_I;
+  case NVPTX::SUST_B_3D_V4I8_ZERO_R:
+    return NVPTX::SUST_B_3D_V4I8_ZERO_I;
+  case NVPTX::SUST_B_3D_V4I16_ZERO_R:
+    return NVPTX::SUST_B_3D_V4I16_ZERO_I;
+  case NVPTX::SUST_B_3D_V4I32_ZERO_R:
+    return NVPTX::SUST_B_3D_V4I32_ZERO_I;
+  case NVPTX::SUST_P_1D_I8_TRAP_R:
+    return NVPTX::SUST_P_1D_I8_TRAP_I;
+  case NVPTX::SUST_P_1D_I16_TRAP_R:
+    return NVPTX::SUST_P_1D_I16_TRAP_I;
+  case NVPTX::SUST_P_1D_I32_TRAP_R:
+    return NVPTX::SUST_P_1D_I32_TRAP_I;
+  case NVPTX::SUST_P_1D_V2I8_TRAP_R:
+    return NVPTX::SUST_P_1D_V2I8_TRAP_I;
+  case NVPTX::SUST_P_1D_V2I16_TRAP_R:
+    return NVPTX::SUST_P_1D_V2I16_TRAP_I;
+  case NVPTX::SUST_P_1D_V2I32_TRAP_R:
+    return NVPTX::SUST_P_1D_V2I32_TRAP_I;
+  case NVPTX::SUST_P_1D_V4I8_TRAP_R:
+    return NVPTX::SUST_P_1D_V4I8_TRAP_I;
+  case NVPTX::SUST_P_1D_V4I16_TRAP_R:
+    return NVPTX::SUST_P_1D_V4I16_TRAP_I;
+  case NVPTX::SUST_P_1D_V4I32_TRAP_R:
+    return NVPTX::SUST_P_1D_V4I32_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_I8_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_I8_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_I16_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_I16_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_I32_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_I32_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_V2I8_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_V2I8_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_V2I16_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_V2I16_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_V2I32_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_V2I32_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_V4I8_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_V4I8_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_V4I16_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_V4I16_TRAP_I;
+  case NVPTX::SUST_P_1D_ARRAY_V4I32_TRAP_R:
+    return NVPTX::SUST_P_1D_ARRAY_V4I32_TRAP_I;
+  case NVPTX::SUST_P_2D_I8_TRAP_R:
+    return NVPTX::SUST_P_2D_I8_TRAP_I;
+  case NVPTX::SUST_P_2D_I16_TRAP_R:
+    return NVPTX::SUST_P_2D_I16_TRAP_I;
+  case NVPTX::SUST_P_2D_I32_TRAP_R:
+    return NVPTX::SUST_P_2D_I32_TRAP_I;
+  case NVPTX::SUST_P_2D_V2I8_TRAP_R:
+    return NVPTX::SUST_P_2D_V2I8_TRAP_I;
+  case NVPTX::SUST_P_2D_V2I16_TRAP_R:
+    return NVPTX::SUST_P_2D_V2I16_TRAP_I;
+  case NVPTX::SUST_P_2D_V2I32_TRAP_R:
+    return NVPTX::SUST_P_2D_V2I32_TRAP_I;
+  case NVPTX::SUST_P_2D_V4I8_TRAP_R:
+    return NVPTX::SUST_P_2D_V4I8_TRAP_I;
+  case NVPTX::SUST_P_2D_V4I16_TRAP_R:
+    return NVPTX::SUST_P_2D_V4I16_TRAP_I;
+  case NVPTX::SUST_P_2D_V4I32_TRAP_R:
+    return NVPTX::SUST_P_2D_V4I32_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_I8_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_I8_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_I16_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_I16_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_I32_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_I32_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_V2I8_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_V2I8_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_V2I16_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_V2I16_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_V2I32_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_V2I32_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_V4I8_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_V4I8_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_V4I16_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_V4I16_TRAP_I;
+  case NVPTX::SUST_P_2D_ARRAY_V4I32_TRAP_R:
+    return NVPTX::SUST_P_2D_ARRAY_V4I32_TRAP_I;
+  case NVPTX::SUST_P_3D_I8_TRAP_R:
+    return NVPTX::SUST_P_3D_I8_TRAP_I;
+  case NVPTX::SUST_P_3D_I16_TRAP_R:
+    return NVPTX::SUST_P_3D_I16_TRAP_I;
+  case NVPTX::SUST_P_3D_I32_TRAP_R:
+    return NVPTX::SUST_P_3D_I32_TRAP_I;
+  case NVPTX::SUST_P_3D_V2I8_TRAP_R:
+    return NVPTX::SUST_P_3D_V2I8_TRAP_I;
+  case NVPTX::SUST_P_3D_V2I16_TRAP_R:
+    return NVPTX::SUST_P_3D_V2I16_TRAP_I;
+  case NVPTX::SUST_P_3D_V2I32_TRAP_R:
+    return NVPTX::SUST_P_3D_V2I32_TRAP_I;
+  case NVPTX::SUST_P_3D_V4I8_TRAP_R:
+    return NVPTX::SUST_P_3D_V4I8_TRAP_I;
+  case NVPTX::SUST_P_3D_V4I16_TRAP_R:
+    return NVPTX::SUST_P_3D_V4I16_TRAP_I;
+  case NVPTX::SUST_P_3D_V4I32_TRAP_R:
+    return NVPTX::SUST_P_3D_V4I32_TRAP_I;
   default:
     llvm_unreachable("Unhandled SUST opcode");
   }
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 5136b1ee2850..d2eae4882682 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -117,6 +117,14 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
     return HasTcgen05 && PTXVersion >= 86;
   }
 
+  // TMA G2S copy with cta_group::1/2 support
+  bool hasCpAsyncBulkTensorCTAGroupSupport() const {
+    // TODO: Update/tidy-up after the family-conditional support arrives
+    return ((FullSmVersion == 1001 || FullSmVersion == 1011) &&
+            PTXVersion >= 86) ||
+           (FullSmVersion == 1031 && PTXVersion >= 88);
+  }
+
   // Prior to CUDA 12.3 ptxas did not recognize that the trap instruction
   // terminates a basic block. Instead, it would assume that control flow
   // continued to the next instruction. The next instruction could be in the
diff --git a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 98aea4e535f0..aa7850acbd64 100644
--- a/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -26,7 +26,7 @@
 
 namespace llvm {
 
-class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
+class NVPTXTTIImpl final : public BasicTTIImplBase<NVPTXTTIImpl> {
   typedef BasicTTIImplBase<NVPTXTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index ab6d25297585..bb4c2fd3e5cf 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -37,9 +37,6 @@ DEFINE_PPC_REGCLASSES
 static int64_t
 EvaluateCRExpr(const MCExpr *E) {
   switch (E->getKind()) {
-  case MCExpr::Target:
-    return -1;
-
   case MCExpr::Constant: {
     int64_t Res = cast<MCConstantExpr>(E)->getValue();
     return Res < 0 ? -1 : Res;
@@ -87,6 +84,10 @@ EvaluateCRExpr(const MCExpr *E) {
 
     return Res < 0 ? -1 : Res;
   }
+  case MCExpr::Specifier:
+    return -1;
+  case MCExpr::Target:
+    llvm_unreachable("unused by this backend");
   }
 
   llvm_unreachable("Invalid expression kind!");
@@ -744,13 +745,13 @@ struct PPCOperand : public MCParsedAsmOperand {
       return CreateImm(CE->getValue(), S, E, IsPPC64);
 
     if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(Val))
-      if (getSpecifier(SRE) == PPCMCExpr::VK_TLS ||
-          getSpecifier(SRE) == PPCMCExpr::VK_TLS_PCREL)
+      if (getSpecifier(SRE) == PPC::S_TLS ||
+          getSpecifier(SRE) == PPC::S_TLS_PCREL)
         return CreateTLSReg(SRE, S, E, IsPPC64);
 
-    if (const PPCMCExpr *TE = dyn_cast<PPCMCExpr>(Val)) {
+    if (const auto *SE = dyn_cast<MCSpecifierExpr>(Val)) {
       int64_t Res;
-      if (TE->evaluateAsConstant(Res))
+      if (PPC::evaluateAsConstant(*SE, Res))
         return CreateContextImm(Res, S, E, IsPPC64);
     }
 
@@ -1372,30 +1373,30 @@ const MCExpr *PPCAsmParser::extractSpecifier(const MCExpr *E,
   switch (E->getKind()) {
   case MCExpr::Constant:
     break;
-  case MCExpr::Target: {
+  case MCExpr::Specifier: {
     // Detect error but do not return a modified expression.
-    auto *TE = cast<PPCMCExpr>(E);
+    auto *TE = cast<MCSpecifierExpr>(E);
     Spec = TE->getSpecifier();
     (void)extractSpecifier(TE->getSubExpr(), Spec);
-    Spec = PPCMCExpr::VK_None;
+    Spec = PPC::S_None;
   } break;
 
   case MCExpr::SymbolRef: {
     const auto *SRE = cast<MCSymbolRefExpr>(E);
     switch (getSpecifier(SRE)) {
-    case PPCMCExpr::VK_None:
+    case PPC::S_None:
     default:
       break;
-    case PPCMCExpr::VK_LO:
-    case PPCMCExpr::VK_HI:
-    case PPCMCExpr::VK_HA:
-    case PPCMCExpr::VK_HIGH:
-    case PPCMCExpr::VK_HIGHA:
-    case PPCMCExpr::VK_HIGHER:
-    case PPCMCExpr::VK_HIGHERA:
-    case PPCMCExpr::VK_HIGHEST:
-    case PPCMCExpr::VK_HIGHESTA:
-      if (Spec == PPCMCExpr::VK_None)
+    case PPC::S_LO:
+    case PPC::S_HI:
+    case PPC::S_HA:
+    case PPC::S_HIGH:
+    case PPC::S_HIGHA:
+    case PPC::S_HIGHER:
+    case PPC::S_HIGHERA:
+    case PPC::S_HIGHEST:
+    case PPC::S_HIGHESTA:
+      if (Spec == PPC::S_None)
         Spec = getSpecifier(SRE);
       else
         Error(E->getLoc(), "cannot contain more than one relocation specifier");
@@ -1407,7 +1408,7 @@ const MCExpr *PPCAsmParser::extractSpecifier(const MCExpr *E,
   case MCExpr::Unary: {
     const MCUnaryExpr *UE = cast<MCUnaryExpr>(E);
     const MCExpr *Sub = extractSpecifier(UE->getSubExpr(), Spec);
-    if (Spec != PPCMCExpr::VK_None)
+    if (Spec != PPC::S_None)
       return MCUnaryExpr::create(UE->getOpcode(), Sub, Context);
     break;
   }
@@ -1416,10 +1417,12 @@ const MCExpr *PPCAsmParser::extractSpecifier(const MCExpr *E,
     const MCBinaryExpr *BE = cast<MCBinaryExpr>(E);
     const MCExpr *LHS = extractSpecifier(BE->getLHS(), Spec);
     const MCExpr *RHS = extractSpecifier(BE->getRHS(), Spec);
-    if (Spec != PPCMCExpr::VK_None)
+    if (Spec != PPC::S_None)
       return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, Context);
     break;
   }
+  case MCExpr::Target:
+    llvm_unreachable("unused by this backend");
   }
 
   return E;
@@ -1433,10 +1436,10 @@ bool PPCAsmParser::parseExpression(const MCExpr *&EVal) {
   if (getParser().parseExpression(EVal))
     return true;
 
-  auto Spec = PPCMCExpr::VK_None;
+  uint16_t Spec = PPC::S_None;
   const MCExpr *E = extractSpecifier(EVal, Spec);
-  if (Spec != PPCMCExpr::VK_None)
-    EVal = PPCMCExpr::create(Spec, E, getParser().getContext());
+  if (Spec != PPC::S_None)
+    EVal = MCSpecifierExpr::create(E, Spec, getParser().getContext());
 
   return false;
 }
@@ -1509,9 +1512,9 @@ bool PPCAsmParser::parseOperand(OperandVector &Operands) {
       if (!(parseOptionalToken(AsmToken::Identifier) &&
             Tok.getString().compare_insensitive("plt") == 0))
         return Error(Tok.getLoc(), "expected 'plt'");
-      EVal = MCSymbolRefExpr::create(
-          getContext().getOrCreateSymbol(TlsGetAddr),
-          MCSymbolRefExpr::VariantKind(PPCMCExpr::VK_PLT), getContext());
+      EVal = MCSymbolRefExpr::create(getContext().getOrCreateSymbol(TlsGetAddr),
+                                     MCSymbolRefExpr::VariantKind(PPC::S_PLT),
+                                     getContext());
       if (parseOptionalToken(AsmToken::Plus)) {
         const MCExpr *Addend = nullptr;
         SMLoc EndLoc;
@@ -1823,20 +1826,20 @@ const MCExpr *PPCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec,
                                            MCContext &Ctx) {
   if (isa<MCConstantExpr>(E)) {
     switch (PPCMCExpr::Specifier(Spec)) {
-    case PPCMCExpr::VK_LO:
-    case PPCMCExpr::VK_HI:
-    case PPCMCExpr::VK_HA:
-    case PPCMCExpr::VK_HIGH:
-    case PPCMCExpr::VK_HIGHA:
-    case PPCMCExpr::VK_HIGHER:
-    case PPCMCExpr::VK_HIGHERA:
-    case PPCMCExpr::VK_HIGHEST:
-    case PPCMCExpr::VK_HIGHESTA:
+    case PPC::S_LO:
+    case PPC::S_HI:
+    case PPC::S_HA:
+    case PPC::S_HIGH:
+    case PPC::S_HIGHA:
+    case PPC::S_HIGHER:
+    case PPC::S_HIGHERA:
+    case PPC::S_HIGHEST:
+    case PPC::S_HIGHESTA:
       break;
     default:
       return nullptr;
     }
   }
 
-  return PPCMCExpr::create(PPCMCExpr::Specifier(Spec), E, Ctx);
+  return MCSpecifierExpr::create(E, Spec, Ctx);
 }
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 84cd12248842..d4b86d5e2811 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -140,7 +140,7 @@ class PPCAsmBackend : public MCAsmBackend {
     // In PPC64 ELFv1, .quad .TOC.@tocbase in the .opd section is expected to
     // reference the null symbol.
     auto Target = TargetVal;
-    if (Target.getSpecifier() == PPCMCExpr::VK_TOCBASE)
+    if (Target.getSpecifier() == PPC::S_TOCBASE)
       Target.setAddSym(nullptr);
     return MCAsmBackend::addReloc(F, Fixup, Target, FixedValue, IsResolved);
   }
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index 096c019f8556..8e885c3d86a0 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -43,49 +43,49 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
   SMLoc Loc = Fixup.getValue()->getLoc();
   auto Spec = static_cast<PPCMCExpr::Specifier>(Target.getSpecifier());
   switch (Spec) {
-  case PPCMCExpr::VK_DTPMOD:
-  case PPCMCExpr::VK_DTPREL:
-  case PPCMCExpr::VK_DTPREL_HA:
-  case PPCMCExpr::VK_DTPREL_HI:
-  case PPCMCExpr::VK_DTPREL_HIGH:
-  case PPCMCExpr::VK_DTPREL_HIGHA:
-  case PPCMCExpr::VK_DTPREL_HIGHER:
-  case PPCMCExpr::VK_DTPREL_HIGHERA:
-  case PPCMCExpr::VK_DTPREL_HIGHEST:
-  case PPCMCExpr::VK_DTPREL_HIGHESTA:
-  case PPCMCExpr::VK_DTPREL_LO:
-  case PPCMCExpr::VK_GOT_DTPREL:
-  case PPCMCExpr::VK_GOT_DTPREL_HA:
-  case PPCMCExpr::VK_GOT_DTPREL_HI:
-  case PPCMCExpr::VK_GOT_DTPREL_LO:
-  case PPCMCExpr::VK_GOT_TLSGD:
-  case PPCMCExpr::VK_GOT_TLSGD_HA:
-  case PPCMCExpr::VK_GOT_TLSGD_HI:
-  case PPCMCExpr::VK_GOT_TLSGD_LO:
-  case PPCMCExpr::VK_GOT_TLSGD_PCREL:
-  case PPCMCExpr::VK_GOT_TLSLD:
-  case PPCMCExpr::VK_GOT_TLSLD_HA:
-  case PPCMCExpr::VK_GOT_TLSLD_HI:
-  case PPCMCExpr::VK_GOT_TLSLD_LO:
-  case PPCMCExpr::VK_GOT_TPREL:
-  case PPCMCExpr::VK_GOT_TPREL_HA:
-  case PPCMCExpr::VK_GOT_TPREL_HI:
-  case PPCMCExpr::VK_GOT_TPREL_LO:
-  case PPCMCExpr::VK_GOT_TPREL_PCREL:
-  case PPCMCExpr::VK_TLS:
-  case PPCMCExpr::VK_TLSGD:
-  case PPCMCExpr::VK_TLSLD:
-  case PPCMCExpr::VK_TLS_PCREL:
-  case PPCMCExpr::VK_TPREL:
-  case PPCMCExpr::VK_TPREL_HA:
-  case PPCMCExpr::VK_TPREL_HI:
-  case PPCMCExpr::VK_TPREL_HIGH:
-  case PPCMCExpr::VK_TPREL_HIGHA:
-  case PPCMCExpr::VK_TPREL_HIGHER:
-  case PPCMCExpr::VK_TPREL_HIGHERA:
-  case PPCMCExpr::VK_TPREL_HIGHEST:
-  case PPCMCExpr::VK_TPREL_HIGHESTA:
-  case PPCMCExpr::VK_TPREL_LO:
+  case PPC::S_DTPMOD:
+  case PPC::S_DTPREL:
+  case PPC::S_DTPREL_HA:
+  case PPC::S_DTPREL_HI:
+  case PPC::S_DTPREL_HIGH:
+  case PPC::S_DTPREL_HIGHA:
+  case PPC::S_DTPREL_HIGHER:
+  case PPC::S_DTPREL_HIGHERA:
+  case PPC::S_DTPREL_HIGHEST:
+  case PPC::S_DTPREL_HIGHESTA:
+  case PPC::S_DTPREL_LO:
+  case PPC::S_GOT_DTPREL:
+  case PPC::S_GOT_DTPREL_HA:
+  case PPC::S_GOT_DTPREL_HI:
+  case PPC::S_GOT_DTPREL_LO:
+  case PPC::S_GOT_TLSGD:
+  case PPC::S_GOT_TLSGD_HA:
+  case PPC::S_GOT_TLSGD_HI:
+  case PPC::S_GOT_TLSGD_LO:
+  case PPC::S_GOT_TLSGD_PCREL:
+  case PPC::S_GOT_TLSLD:
+  case PPC::S_GOT_TLSLD_HA:
+  case PPC::S_GOT_TLSLD_HI:
+  case PPC::S_GOT_TLSLD_LO:
+  case PPC::S_GOT_TPREL:
+  case PPC::S_GOT_TPREL_HA:
+  case PPC::S_GOT_TPREL_HI:
+  case PPC::S_GOT_TPREL_LO:
+  case PPC::S_GOT_TPREL_PCREL:
+  case PPC::S_TLS:
+  case PPC::S_TLSGD:
+  case PPC::S_TLSLD:
+  case PPC::S_TLS_PCREL:
+  case PPC::S_TPREL:
+  case PPC::S_TPREL_HA:
+  case PPC::S_TPREL_HI:
+  case PPC::S_TPREL_HIGH:
+  case PPC::S_TPREL_HIGHA:
+  case PPC::S_TPREL_HIGHER:
+  case PPC::S_TPREL_HIGHERA:
+  case PPC::S_TPREL_HIGHEST:
+  case PPC::S_TPREL_HIGHESTA:
+  case PPC::S_TPREL_LO:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
@@ -106,16 +106,16 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         break;
-      case PPCMCExpr::VK_None:
+      case PPC::S_None:
         Type = ELF::R_PPC_REL24;
         break;
-      case PPCMCExpr::VK_PLT:
+      case PPC::S_PLT:
         Type = ELF::R_PPC_PLTREL24;
         break;
-      case PPCMCExpr::VK_LOCAL:
+      case PPC::S_LOCAL:
         Type = ELF::R_PPC_LOCAL24PC;
         break;
-      case PPCMCExpr::VK_NOTOC:
+      case PPC::S_NOTOC:
         Type = ELF::R_PPC64_REL24_NOTOC;
         break;
       }
@@ -129,13 +129,13 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         return ELF::R_PPC_NONE;
-      case PPCMCExpr::VK_None:
+      case PPC::S_None:
         return ELF::R_PPC_REL16;
-      case PPCMCExpr::VK_LO:
+      case PPC::S_LO:
         return ELF::R_PPC_REL16_LO;
-      case PPCMCExpr::VK_HI:
+      case PPC::S_HI:
         return ELF::R_PPC_REL16_HI;
-      case PPCMCExpr::VK_HA:
+      case PPC::S_HA:
         return ELF::R_PPC_REL16_HA;
       }
       break;
@@ -148,19 +148,19 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         break;
-      case PPCMCExpr::VK_PCREL:
+      case PPC::S_PCREL:
         Type = ELF::R_PPC64_PCREL34;
         break;
-      case PPCMCExpr::VK_GOT_PCREL:
+      case PPC::S_GOT_PCREL:
         Type = ELF::R_PPC64_GOT_PCREL34;
         break;
-      case PPCMCExpr::VK_GOT_TLSGD_PCREL:
+      case PPC::S_GOT_TLSGD_PCREL:
         Type = ELF::R_PPC64_GOT_TLSGD_PCREL34;
         break;
-      case PPCMCExpr::VK_GOT_TLSLD_PCREL:
+      case PPC::S_GOT_TLSLD_PCREL:
         Type = ELF::R_PPC64_GOT_TLSLD_PCREL34;
         break;
-      case PPCMCExpr::VK_GOT_TPREL_PCREL:
+      case PPC::S_GOT_TPREL_PCREL:
         Type = ELF::R_PPC64_GOT_TPREL_PCREL34;
         break;
       }
@@ -186,172 +186,172 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         break;
-      case PPCMCExpr::VK_LO:
+      case PPC::S_LO:
         return ELF::R_PPC_ADDR16_LO;
-      case PPCMCExpr::VK_HI:
+      case PPC::S_HI:
         return ELF::R_PPC_ADDR16_HI;
-      case PPCMCExpr::VK_HA:
+      case PPC::S_HA:
         return ELF::R_PPC_ADDR16_HA;
-      case PPCMCExpr::VK_HIGH:
+      case PPC::S_HIGH:
         return ELF::R_PPC64_ADDR16_HIGH;
-      case PPCMCExpr::VK_HIGHA:
+      case PPC::S_HIGHA:
         return ELF::R_PPC64_ADDR16_HIGHA;
-      case PPCMCExpr::VK_HIGHER:
+      case PPC::S_HIGHER:
         return ELF::R_PPC64_ADDR16_HIGHER;
-      case PPCMCExpr::VK_HIGHERA:
+      case PPC::S_HIGHERA:
         return ELF::R_PPC64_ADDR16_HIGHERA;
-      case PPCMCExpr::VK_HIGHEST:
+      case PPC::S_HIGHEST:
         return ELF::R_PPC64_ADDR16_HIGHEST;
-      case PPCMCExpr::VK_HIGHESTA:
+      case PPC::S_HIGHESTA:
         return ELF::R_PPC64_ADDR16_HIGHESTA;
 
-      case PPCMCExpr::VK_None:
+      case PPC::S_None:
         Type = ELF::R_PPC_ADDR16;
         break;
-      case PPCMCExpr::VK_GOT:
+      case PPC::S_GOT:
         Type = ELF::R_PPC_GOT16;
         break;
-      case PPCMCExpr::VK_GOT_LO:
+      case PPC::S_GOT_LO:
         Type = ELF::R_PPC_GOT16_LO;
         break;
-      case PPCMCExpr::VK_GOT_HI:
+      case PPC::S_GOT_HI:
         Type = ELF::R_PPC_GOT16_HI;
         break;
-      case PPCMCExpr::VK_GOT_HA:
+      case PPC::S_GOT_HA:
         Type = ELF::R_PPC_GOT16_HA;
         break;
-      case PPCMCExpr::VK_TOC:
+      case PPC::S_TOC:
         Type = ELF::R_PPC64_TOC16;
         break;
-      case PPCMCExpr::VK_TOC_LO:
+      case PPC::S_TOC_LO:
         Type = ELF::R_PPC64_TOC16_LO;
         break;
-      case PPCMCExpr::VK_TOC_HI:
+      case PPC::S_TOC_HI:
         Type = ELF::R_PPC64_TOC16_HI;
         break;
-      case PPCMCExpr::VK_TOC_HA:
+      case PPC::S_TOC_HA:
         Type = ELF::R_PPC64_TOC16_HA;
         break;
-      case PPCMCExpr::VK_TPREL:
+      case PPC::S_TPREL:
         Type = ELF::R_PPC_TPREL16;
         break;
-      case PPCMCExpr::VK_TPREL_LO:
+      case PPC::S_TPREL_LO:
         Type = ELF::R_PPC_TPREL16_LO;
         break;
-      case PPCMCExpr::VK_TPREL_HI:
+      case PPC::S_TPREL_HI:
         Type = ELF::R_PPC_TPREL16_HI;
         break;
-      case PPCMCExpr::VK_TPREL_HA:
+      case PPC::S_TPREL_HA:
         Type = ELF::R_PPC_TPREL16_HA;
         break;
-      case PPCMCExpr::VK_TPREL_HIGH:
+      case PPC::S_TPREL_HIGH:
         Type = ELF::R_PPC64_TPREL16_HIGH;
         break;
-      case PPCMCExpr::VK_TPREL_HIGHA:
+      case PPC::S_TPREL_HIGHA:
         Type = ELF::R_PPC64_TPREL16_HIGHA;
         break;
-      case PPCMCExpr::VK_TPREL_HIGHER:
+      case PPC::S_TPREL_HIGHER:
         Type = ELF::R_PPC64_TPREL16_HIGHER;
         break;
-      case PPCMCExpr::VK_TPREL_HIGHERA:
+      case PPC::S_TPREL_HIGHERA:
         Type = ELF::R_PPC64_TPREL16_HIGHERA;
         break;
-      case PPCMCExpr::VK_TPREL_HIGHEST:
+      case PPC::S_TPREL_HIGHEST:
         Type = ELF::R_PPC64_TPREL16_HIGHEST;
         break;
-      case PPCMCExpr::VK_TPREL_HIGHESTA:
+      case PPC::S_TPREL_HIGHESTA:
         Type = ELF::R_PPC64_TPREL16_HIGHESTA;
         break;
-      case PPCMCExpr::VK_DTPREL:
+      case PPC::S_DTPREL:
         Type = ELF::R_PPC64_DTPREL16;
         break;
-      case PPCMCExpr::VK_DTPREL_LO:
+      case PPC::S_DTPREL_LO:
         Type = ELF::R_PPC64_DTPREL16_LO;
         break;
-      case PPCMCExpr::VK_DTPREL_HI:
+      case PPC::S_DTPREL_HI:
         Type = ELF::R_PPC64_DTPREL16_HI;
         break;
-      case PPCMCExpr::VK_DTPREL_HA:
+      case PPC::S_DTPREL_HA:
         Type = ELF::R_PPC64_DTPREL16_HA;
         break;
-      case PPCMCExpr::VK_DTPREL_HIGH:
+      case PPC::S_DTPREL_HIGH:
         Type = ELF::R_PPC64_DTPREL16_HIGH;
         break;
-      case PPCMCExpr::VK_DTPREL_HIGHA:
+      case PPC::S_DTPREL_HIGHA:
         Type = ELF::R_PPC64_DTPREL16_HIGHA;
         break;
-      case PPCMCExpr::VK_DTPREL_HIGHER:
+      case PPC::S_DTPREL_HIGHER:
         Type = ELF::R_PPC64_DTPREL16_HIGHER;
         break;
-      case PPCMCExpr::VK_DTPREL_HIGHERA:
+      case PPC::S_DTPREL_HIGHERA:
         Type = ELF::R_PPC64_DTPREL16_HIGHERA;
         break;
-      case PPCMCExpr::VK_DTPREL_HIGHEST:
+      case PPC::S_DTPREL_HIGHEST:
         Type = ELF::R_PPC64_DTPREL16_HIGHEST;
         break;
-      case PPCMCExpr::VK_DTPREL_HIGHESTA:
+      case PPC::S_DTPREL_HIGHESTA:
         Type = ELF::R_PPC64_DTPREL16_HIGHESTA;
         break;
-      case PPCMCExpr::VK_GOT_TLSGD:
+      case PPC::S_GOT_TLSGD:
         if (is64Bit())
           Type = ELF::R_PPC64_GOT_TLSGD16;
         else
           Type = ELF::R_PPC_GOT_TLSGD16;
         break;
-      case PPCMCExpr::VK_GOT_TLSGD_LO:
+      case PPC::S_GOT_TLSGD_LO:
         Type = ELF::R_PPC64_GOT_TLSGD16_LO;
         break;
-      case PPCMCExpr::VK_GOT_TLSGD_HI:
+      case PPC::S_GOT_TLSGD_HI:
         Type = ELF::R_PPC64_GOT_TLSGD16_HI;
         break;
-      case PPCMCExpr::VK_GOT_TLSGD_HA:
+      case PPC::S_GOT_TLSGD_HA:
         Type = ELF::R_PPC64_GOT_TLSGD16_HA;
         break;
-      case PPCMCExpr::VK_GOT_TLSLD:
+      case PPC::S_GOT_TLSLD:
         if (is64Bit())
           Type = ELF::R_PPC64_GOT_TLSLD16;
         else
           Type = ELF::R_PPC_GOT_TLSLD16;
         break;
-      case PPCMCExpr::VK_GOT_TLSLD_LO:
+      case PPC::S_GOT_TLSLD_LO:
         Type = ELF::R_PPC64_GOT_TLSLD16_LO;
         break;
-      case PPCMCExpr::VK_GOT_TLSLD_HI:
+      case PPC::S_GOT_TLSLD_HI:
         Type = ELF::R_PPC64_GOT_TLSLD16_HI;
         break;
-      case PPCMCExpr::VK_GOT_TLSLD_HA:
+      case PPC::S_GOT_TLSLD_HA:
         Type = ELF::R_PPC64_GOT_TLSLD16_HA;
         break;
-      case PPCMCExpr::VK_GOT_TPREL:
+      case PPC::S_GOT_TPREL:
         /* We don't have R_PPC64_GOT_TPREL16, but since GOT offsets
            are always 4-aligned, we can use R_PPC64_GOT_TPREL16_DS.  */
         Type = ELF::R_PPC64_GOT_TPREL16_DS;
         break;
-      case PPCMCExpr::VK_GOT_TPREL_LO:
+      case PPC::S_GOT_TPREL_LO:
         /* We don't have R_PPC64_GOT_TPREL16_LO, but since GOT offsets
            are always 4-aligned, we can use R_PPC64_GOT_TPREL16_LO_DS.  */
         Type = ELF::R_PPC64_GOT_TPREL16_LO_DS;
         break;
-      case PPCMCExpr::VK_GOT_TPREL_HI:
+      case PPC::S_GOT_TPREL_HI:
         Type = ELF::R_PPC64_GOT_TPREL16_HI;
         break;
-      case PPCMCExpr::VK_GOT_DTPREL:
+      case PPC::S_GOT_DTPREL:
         /* We don't have R_PPC64_GOT_DTPREL16, but since GOT offsets
            are always 4-aligned, we can use R_PPC64_GOT_DTPREL16_DS.  */
         Type = ELF::R_PPC64_GOT_DTPREL16_DS;
         break;
-      case PPCMCExpr::VK_GOT_DTPREL_LO:
+      case PPC::S_GOT_DTPREL_LO:
         /* We don't have R_PPC64_GOT_DTPREL16_LO, but since GOT offsets
            are always 4-aligned, we can use R_PPC64_GOT_DTPREL16_LO_DS.  */
         Type = ELF::R_PPC64_GOT_DTPREL16_LO_DS;
         break;
-      case PPCMCExpr::VK_GOT_TPREL_HA:
+      case PPC::S_GOT_TPREL_HA:
         Type = ELF::R_PPC64_GOT_TPREL16_HA;
         break;
-      case PPCMCExpr::VK_GOT_DTPREL_HI:
+      case PPC::S_GOT_DTPREL_HI:
         Type = ELF::R_PPC64_GOT_DTPREL16_HI;
         break;
-      case PPCMCExpr::VK_GOT_DTPREL_HA:
+      case PPC::S_GOT_DTPREL_HA:
         Type = ELF::R_PPC64_GOT_DTPREL16_HA;
         break;
       }
@@ -362,45 +362,45 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         break;
-      case PPCMCExpr::VK_LO:
+      case PPC::S_LO:
         return ELF::R_PPC64_ADDR16_LO_DS;
-      case PPCMCExpr::VK_None:
+      case PPC::S_None:
         Type = ELF::R_PPC64_ADDR16_DS;
         break;
-      case PPCMCExpr::VK_GOT:
+      case PPC::S_GOT:
         Type = ELF::R_PPC64_GOT16_DS;
         break;
-      case PPCMCExpr::VK_GOT_LO:
+      case PPC::S_GOT_LO:
         Type = ELF::R_PPC64_GOT16_LO_DS;
         break;
-      case PPCMCExpr::VK_TOC:
+      case PPC::S_TOC:
         Type = ELF::R_PPC64_TOC16_DS;
         break;
-      case PPCMCExpr::VK_TOC_LO:
+      case PPC::S_TOC_LO:
         Type = ELF::R_PPC64_TOC16_LO_DS;
         break;
-      case PPCMCExpr::VK_TPREL:
+      case PPC::S_TPREL:
         Type = ELF::R_PPC64_TPREL16_DS;
         break;
-      case PPCMCExpr::VK_TPREL_LO:
+      case PPC::S_TPREL_LO:
         Type = ELF::R_PPC64_TPREL16_LO_DS;
         break;
-      case PPCMCExpr::VK_DTPREL:
+      case PPC::S_DTPREL:
         Type = ELF::R_PPC64_DTPREL16_DS;
         break;
-      case PPCMCExpr::VK_DTPREL_LO:
+      case PPC::S_DTPREL_LO:
         Type = ELF::R_PPC64_DTPREL16_LO_DS;
         break;
-      case PPCMCExpr::VK_GOT_TPREL:
+      case PPC::S_GOT_TPREL:
         Type = ELF::R_PPC64_GOT_TPREL16_DS;
         break;
-      case PPCMCExpr::VK_GOT_TPREL_LO:
+      case PPC::S_GOT_TPREL_LO:
         Type = ELF::R_PPC64_GOT_TPREL16_LO_DS;
         break;
-      case PPCMCExpr::VK_GOT_DTPREL:
+      case PPC::S_GOT_DTPREL:
         Type = ELF::R_PPC64_GOT_DTPREL16_DS;
         break;
-      case PPCMCExpr::VK_GOT_DTPREL_LO:
+      case PPC::S_GOT_DTPREL_LO:
         Type = ELF::R_PPC64_GOT_DTPREL16_LO_DS;
         break;
       }
@@ -410,25 +410,25 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         break;
-      case PPCMCExpr::VK_TLSGD:
+      case PPC::S_TLSGD:
         if (is64Bit())
           Type = ELF::R_PPC64_TLSGD;
         else
           Type = ELF::R_PPC_TLSGD;
         break;
-      case PPCMCExpr::VK_TLSLD:
+      case PPC::S_TLSLD:
         if (is64Bit())
           Type = ELF::R_PPC64_TLSLD;
         else
           Type = ELF::R_PPC_TLSLD;
         break;
-      case PPCMCExpr::VK_TLS:
+      case PPC::S_TLS:
         if (is64Bit())
           Type = ELF::R_PPC64_TLS;
         else
           Type = ELF::R_PPC_TLS;
         break;
-      case PPCMCExpr::VK_TLS_PCREL:
+      case PPC::S_TLS_PCREL:
         Type = ELF::R_PPC64_TLS;
         break;
       }
@@ -438,10 +438,10 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         break;
-      case PPCMCExpr::VK_DTPREL:
+      case PPC::S_DTPREL:
         Type = ELF::R_PPC64_DTPREL34;
         break;
-      case PPCMCExpr::VK_TPREL:
+      case PPC::S_TPREL:
         Type = ELF::R_PPC64_TPREL34;
         break;
       }
@@ -451,26 +451,26 @@ unsigned PPCELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Loc, "unsupported relocation type");
         break;
-      case PPCMCExpr::VK_TOCBASE:
+      case PPC::S_TOCBASE:
         Type = ELF::R_PPC64_TOC;
         break;
-      case PPCMCExpr::VK_None:
+      case PPC::S_None:
         Type = ELF::R_PPC64_ADDR64;
         break;
-      case PPCMCExpr::VK_DTPMOD:
+      case PPC::S_DTPMOD:
         Type = ELF::R_PPC64_DTPMOD64;
         break;
-      case PPCMCExpr::VK_TPREL:
+      case PPC::S_TPREL:
         Type = ELF::R_PPC64_TPREL64;
         break;
-      case PPCMCExpr::VK_DTPREL:
+      case PPC::S_DTPREL:
         Type = ELF::R_PPC64_DTPREL64;
         break;
       }
       break;
     case FK_Data_4:
       switch (Spec) {
-      case PPCMCExpr::VK_DTPREL:
+      case PPC::S_DTPREL:
         Type = ELF::R_PPC_DTPREL32;
         break;
       default:
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
index 501ef460b693..78065541f0d0 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
@@ -139,7 +139,7 @@ void PPCELFStreamer::emitGOTToPCRelReloc(const MCInst &Inst) {
   // Cast the last operand to MCSymbolRefExpr to get the symbol.
   const MCExpr *Expr = Operand.getExpr();
   const MCSymbolRefExpr *SymExpr = static_cast<const MCSymbolRefExpr *>(Expr);
-  assert(getSpecifier(SymExpr) == PPCMCExpr::VK_PCREL_OPT &&
+  assert(getSpecifier(SymExpr) == PPC::S_PCREL_OPT &&
          "Expecting a symbol of type VK_PCREL_OPT");
   MCSymbol *LabelSym =
       getContext().getOrCreateSymbol(SymExpr->getSymbol().getName());
@@ -174,7 +174,7 @@ void PPCELFStreamer::emitGOTToPCRelLabel(const MCInst &Inst) {
   // Cast the last operand to MCSymbolRefExpr to get the symbol.
   const MCExpr *Expr = Operand.getExpr();
   const MCSymbolRefExpr *SymExpr = static_cast<const MCSymbolRefExpr *>(Expr);
-  assert(getSpecifier(SymExpr) == PPCMCExpr::VK_PCREL_OPT &&
+  assert(getSpecifier(SymExpr) == PPC::S_PCREL_OPT &&
          "Expecting a symbol of type VK_PCREL_OPT");
   MCSymbol *LabelSym =
       getContext().getOrCreateSymbol(SymExpr->getSymbol().getName());
@@ -190,7 +190,7 @@ void PPCELFStreamer::emitGOTToPCRelLabel(const MCInst &Inst) {
 // The above is a pair of such instructions and this function will not return
 // std::nullopt for either one of them. In both cases we are looking for the
 // last operand <MCOperand Expr:(.Lpcrel@<<invalid>>)> which needs to be an
-// MCExpr and has the flag PPCMCExpr::VK_PCREL_OPT. After that we just
+// MCExpr and has the flag PPC::S_PCREL_OPT. After that we just
 // look at the opcode and in the case of PLDpc we will return true. For the load
 // (or store) this function will return false indicating it has found the second
 // instruciton in the pair.
@@ -212,7 +212,7 @@ std::optional<bool> llvm::isPartOfGOTToPCRelPair(const MCInst &Inst,
   // Check for the variant kind VK_PCREL_OPT in this expression.
   const MCExpr *Expr = Operand.getExpr();
   const MCSymbolRefExpr *SymExpr = static_cast<const MCSymbolRefExpr *>(Expr);
-  if (!SymExpr || getSpecifier(SymExpr) != PPCMCExpr::VK_PCREL_OPT)
+  if (!SymExpr || getSpecifier(SymExpr) != PPC::S_PCREL_OPT)
     return std::nullopt;
 
   return (Inst.getOpcode() == PPC::PLDpc);
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 727704af5d80..2c232f450e1a 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -92,7 +92,7 @@ void PPCInstPrinter::printInst(const MCInst *MI, uint64_t Address,
       const MCSymbolRefExpr *SymExpr =
           static_cast<const MCSymbolRefExpr *>(Expr);
 
-      if (SymExpr && getSpecifier(SymExpr) == PPCMCExpr::VK_PCREL_OPT) {
+      if (SymExpr && getSpecifier(SymExpr) == PPC::S_PCREL_OPT) {
         const MCSymbol &Symbol = SymExpr->getSymbol();
         if (MI->getOpcode() == PPC::PLDpc) {
           printInstruction(MI, Address, STI, O);
@@ -579,18 +579,18 @@ void PPCInstPrinter::printTLSCall(const MCInst *MI, unsigned OpNo,
   // because we do not want the assembly to print out the @notoc at the
   // end like __tls_get_addr(x@tlsgd)@notoc. Instead we want it to look
   // like __tls_get_addr@notoc(x@tlsgd).
-  if (getSpecifier(RefExp) == PPCMCExpr::VK_NOTOC)
+  if (getSpecifier(RefExp) == PPC::S_NOTOC)
     O << '@' << MAI.getSpecifierName(RefExp->getKind());
   O << '(';
   printOperand(MI, OpNo + 1, STI, O);
   O << ')';
-  if (getSpecifier(RefExp) != PPCMCExpr::VK_None &&
-      getSpecifier(RefExp) != PPCMCExpr::VK_NOTOC)
+  if (getSpecifier(RefExp) != PPC::S_None &&
+      getSpecifier(RefExp) != PPC::S_NOTOC)
     O << '@' << MAI.getSpecifierName(RefExp->getKind());
   if (Rhs) {
     SmallString<0> Buf;
     raw_svector_ostream Tmp(Buf);
-    Rhs->print(Tmp, &MAI);
+    MAI.printExpr(Tmp, *Rhs);
     if (isdigit(Buf[0]))
       O << '+';
     O << Buf;
@@ -671,5 +671,5 @@ void PPCInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(Op.isExpr() && "unknown operand kind in printOperand");
-  Op.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *Op.getExpr());
 }
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
index 160ee07fad5c..971b592643dc 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
@@ -13,6 +13,7 @@
 #include "PPCMCAsmInfo.h"
 #include "PPCMCExpr.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
@@ -20,82 +21,82 @@ using namespace llvm;
 void PPCELFMCAsmInfo::anchor() { }
 
 const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
-    {PPCMCExpr::VK_DTPREL, "DTPREL"},
-    {PPCMCExpr::VK_GOT, "GOT"},
-    {PPCMCExpr::VK_GOT_HA, "got@ha"},
-    {PPCMCExpr::VK_GOT_HI, "got@h"},
-    {PPCMCExpr::VK_GOT_LO, "got@l"},
-    {PPCMCExpr::VK_HA, "ha"},
-    {PPCMCExpr::VK_HI, "h"},
-    {PPCMCExpr::VK_HIGH, "high"},
-    {PPCMCExpr::VK_HIGHA, "higha"},
-    {PPCMCExpr::VK_HIGHER, "higher"},
-    {PPCMCExpr::VK_HIGHERA, "highera"},
-    {PPCMCExpr::VK_HIGHEST, "highest"},
-    {PPCMCExpr::VK_HIGHESTA, "highesta"},
-    {PPCMCExpr::VK_LO, "l"},
-    {PPCMCExpr::VK_L, "l"}, // FIXME: share the name with VK_LO
-    {PPCMCExpr::VK_PCREL, "PCREL"},
-    {PPCMCExpr::VK_PLT, "PLT"},
-    {PPCMCExpr::VK_TLSGD, "tlsgd"},
-    {PPCMCExpr::VK_TLSLD, "tlsld"},
-    {PPCMCExpr::VK_TOC, "toc"},
-    {PPCMCExpr::VK_TOCBASE, "tocbase"},
-    {PPCMCExpr::VK_TOC_HA, "toc@ha"},
-    {PPCMCExpr::VK_TOC_HI, "toc@h"},
-    {PPCMCExpr::VK_TOC_LO, "toc@l"},
-    {PPCMCExpr::VK_TPREL, "TPREL"},
-    {PPCMCExpr::VK_AIX_TLSGD, "gd"},
-    {PPCMCExpr::VK_AIX_TLSGDM, "m"},
-    {PPCMCExpr::VK_AIX_TLSIE, "ie"},
-    {PPCMCExpr::VK_AIX_TLSLD, "ld"},
-    {PPCMCExpr::VK_AIX_TLSLE, "le"},
-    {PPCMCExpr::VK_AIX_TLSML, "ml"},
-    {PPCMCExpr::VK_DTPMOD, "dtpmod"},
-    {PPCMCExpr::VK_DTPREL_HA, "dtprel@ha"},
-    {PPCMCExpr::VK_DTPREL_HI, "dtprel@h"},
-    {PPCMCExpr::VK_DTPREL_HIGH, "dtprel@high"},
-    {PPCMCExpr::VK_DTPREL_HIGHA, "dtprel@higha"},
-    {PPCMCExpr::VK_DTPREL_HIGHER, "dtprel@higher"},
-    {PPCMCExpr::VK_DTPREL_HIGHERA, "dtprel@highera"},
-    {PPCMCExpr::VK_DTPREL_HIGHEST, "dtprel@highest"},
-    {PPCMCExpr::VK_DTPREL_HIGHESTA, "dtprel@highesta"},
-    {PPCMCExpr::VK_DTPREL_LO, "dtprel@l"},
-    {PPCMCExpr::VK_GOT_DTPREL, "got@dtprel"},
-    {PPCMCExpr::VK_GOT_DTPREL_HA, "got@dtprel@ha"},
-    {PPCMCExpr::VK_GOT_DTPREL_HI, "got@dtprel@h"},
-    {PPCMCExpr::VK_GOT_DTPREL_LO, "got@dtprel@l"},
-    {PPCMCExpr::VK_GOT_PCREL, "got@pcrel"},
-    {PPCMCExpr::VK_GOT_TLSGD, "got@tlsgd"},
-    {PPCMCExpr::VK_GOT_TLSGD_HA, "got@tlsgd@ha"},
-    {PPCMCExpr::VK_GOT_TLSGD_HI, "got@tlsgd@h"},
-    {PPCMCExpr::VK_GOT_TLSGD_LO, "got@tlsgd@l"},
-    {PPCMCExpr::VK_GOT_TLSGD_PCREL, "got@tlsgd@pcrel"},
-    {PPCMCExpr::VK_GOT_TLSLD, "got@tlsld"},
-    {PPCMCExpr::VK_GOT_TLSLD_HA, "got@tlsld@ha"},
-    {PPCMCExpr::VK_GOT_TLSLD_HI, "got@tlsld@h"},
-    {PPCMCExpr::VK_GOT_TLSLD_LO, "got@tlsld@l"},
-    {PPCMCExpr::VK_GOT_TLSLD_PCREL, "got@tlsld@pcrel"},
-    {PPCMCExpr::VK_GOT_TPREL, "got@tprel"},
-    {PPCMCExpr::VK_GOT_TPREL_HA, "got@tprel@ha"},
-    {PPCMCExpr::VK_GOT_TPREL_HI, "got@tprel@h"},
-    {PPCMCExpr::VK_GOT_TPREL_LO, "got@tprel@l"},
-    {PPCMCExpr::VK_GOT_TPREL_PCREL, "got@tprel@pcrel"},
-    {PPCMCExpr::VK_LOCAL, "local"},
-    {PPCMCExpr::VK_NOTOC, "notoc"},
-    {PPCMCExpr::VK_PCREL_OPT, "<<invalid>>"},
-    {PPCMCExpr::VK_TLS, "tls"},
-    {PPCMCExpr::VK_TLS_PCREL, "tls@pcrel"},
-    {PPCMCExpr::VK_TPREL_HA, "tprel@ha"},
-    {PPCMCExpr::VK_TPREL_HI, "tprel@h"},
-    {PPCMCExpr::VK_TPREL_HIGH, "tprel@high"},
-    {PPCMCExpr::VK_TPREL_HIGHA, "tprel@higha"},
-    {PPCMCExpr::VK_TPREL_HIGHER, "tprel@higher"},
-    {PPCMCExpr::VK_TPREL_HIGHERA, "tprel@highera"},
-    {PPCMCExpr::VK_TPREL_HIGHEST, "tprel@highest"},
-    {PPCMCExpr::VK_TPREL_HIGHESTA, "tprel@highesta"},
-    {PPCMCExpr::VK_TPREL_LO, "tprel@l"},
-    {PPCMCExpr::VK_U, "u"},
+    {PPC::S_DTPREL, "DTPREL"},
+    {PPC::S_GOT, "GOT"},
+    {PPC::S_GOT_HA, "got@ha"},
+    {PPC::S_GOT_HI, "got@h"},
+    {PPC::S_GOT_LO, "got@l"},
+    {PPC::S_HA, "ha"},
+    {PPC::S_HI, "h"},
+    {PPC::S_HIGH, "high"},
+    {PPC::S_HIGHA, "higha"},
+    {PPC::S_HIGHER, "higher"},
+    {PPC::S_HIGHERA, "highera"},
+    {PPC::S_HIGHEST, "highest"},
+    {PPC::S_HIGHESTA, "highesta"},
+    {PPC::S_LO, "l"},
+    {PPC::S_L, "l"}, // FIXME: share the name with VK_LO
+    {PPC::S_PCREL, "PCREL"},
+    {PPC::S_PLT, "PLT"},
+    {PPC::S_TLSGD, "tlsgd"},
+    {PPC::S_TLSLD, "tlsld"},
+    {PPC::S_TOC, "toc"},
+    {PPC::S_TOCBASE, "tocbase"},
+    {PPC::S_TOC_HA, "toc@ha"},
+    {PPC::S_TOC_HI, "toc@h"},
+    {PPC::S_TOC_LO, "toc@l"},
+    {PPC::S_TPREL, "TPREL"},
+    {PPC::S_AIX_TLSGD, "gd"},
+    {PPC::S_AIX_TLSGDM, "m"},
+    {PPC::S_AIX_TLSIE, "ie"},
+    {PPC::S_AIX_TLSLD, "ld"},
+    {PPC::S_AIX_TLSLE, "le"},
+    {PPC::S_AIX_TLSML, "ml"},
+    {PPC::S_DTPMOD, "dtpmod"},
+    {PPC::S_DTPREL_HA, "dtprel@ha"},
+    {PPC::S_DTPREL_HI, "dtprel@h"},
+    {PPC::S_DTPREL_HIGH, "dtprel@high"},
+    {PPC::S_DTPREL_HIGHA, "dtprel@higha"},
+    {PPC::S_DTPREL_HIGHER, "dtprel@higher"},
+    {PPC::S_DTPREL_HIGHERA, "dtprel@highera"},
+    {PPC::S_DTPREL_HIGHEST, "dtprel@highest"},
+    {PPC::S_DTPREL_HIGHESTA, "dtprel@highesta"},
+    {PPC::S_DTPREL_LO, "dtprel@l"},
+    {PPC::S_GOT_DTPREL, "got@dtprel"},
+    {PPC::S_GOT_DTPREL_HA, "got@dtprel@ha"},
+    {PPC::S_GOT_DTPREL_HI, "got@dtprel@h"},
+    {PPC::S_GOT_DTPREL_LO, "got@dtprel@l"},
+    {PPC::S_GOT_PCREL, "got@pcrel"},
+    {PPC::S_GOT_TLSGD, "got@tlsgd"},
+    {PPC::S_GOT_TLSGD_HA, "got@tlsgd@ha"},
+    {PPC::S_GOT_TLSGD_HI, "got@tlsgd@h"},
+    {PPC::S_GOT_TLSGD_LO, "got@tlsgd@l"},
+    {PPC::S_GOT_TLSGD_PCREL, "got@tlsgd@pcrel"},
+    {PPC::S_GOT_TLSLD, "got@tlsld"},
+    {PPC::S_GOT_TLSLD_HA, "got@tlsld@ha"},
+    {PPC::S_GOT_TLSLD_HI, "got@tlsld@h"},
+    {PPC::S_GOT_TLSLD_LO, "got@tlsld@l"},
+    {PPC::S_GOT_TLSLD_PCREL, "got@tlsld@pcrel"},
+    {PPC::S_GOT_TPREL, "got@tprel"},
+    {PPC::S_GOT_TPREL_HA, "got@tprel@ha"},
+    {PPC::S_GOT_TPREL_HI, "got@tprel@h"},
+    {PPC::S_GOT_TPREL_LO, "got@tprel@l"},
+    {PPC::S_GOT_TPREL_PCREL, "got@tprel@pcrel"},
+    {PPC::S_LOCAL, "local"},
+    {PPC::S_NOTOC, "notoc"},
+    {PPC::S_PCREL_OPT, "<<invalid>>"},
+    {PPC::S_TLS, "tls"},
+    {PPC::S_TLS_PCREL, "tls@pcrel"},
+    {PPC::S_TPREL_HA, "tprel@ha"},
+    {PPC::S_TPREL_HI, "tprel@h"},
+    {PPC::S_TPREL_HIGH, "tprel@high"},
+    {PPC::S_TPREL_HIGHA, "tprel@higha"},
+    {PPC::S_TPREL_HIGHER, "tprel@higher"},
+    {PPC::S_TPREL_HIGHERA, "tprel@highera"},
+    {PPC::S_TPREL_HIGHEST, "tprel@highest"},
+    {PPC::S_TPREL_HIGHESTA, "tprel@highesta"},
+    {PPC::S_TPREL_LO, "tprel@l"},
+    {PPC::S_U, "u"},
 };
 
 PPCELFMCAsmInfo::PPCELFMCAsmInfo(bool is64Bit, const Triple& T) {
@@ -136,6 +137,18 @@ PPCELFMCAsmInfo::PPCELFMCAsmInfo(bool is64Bit, const Triple& T) {
   initializeVariantKinds(variantKindDescs);
 }
 
+void PPCELFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                         const MCSpecifierExpr &Expr) const {
+  printExpr(OS, *Expr.getSubExpr());
+  OS << '@' << getSpecifierName(Expr.getSpecifier());
+}
+
+bool PPCELFMCAsmInfo::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr,
+                                                MCValue &Res,
+                                                const MCAssembler *Asm) const {
+  return PPC::evaluateAsRelocatableImpl(Expr, Res, Asm);
+}
+
 void PPCXCOFFMCAsmInfo::anchor() {}
 
 PPCXCOFFMCAsmInfo::PPCXCOFFMCAsmInfo(bool Is64Bit, const Triple &T) {
@@ -155,5 +168,18 @@ PPCXCOFFMCAsmInfo::PPCXCOFFMCAsmInfo(bool Is64Bit, const Triple &T) {
   // Support $ as PC in inline asm
   DollarIsPC = true;
 
+  UsesSetToEquateSymbol = true;
+
   initializeVariantKinds(variantKindDescs);
 }
+
+void PPCXCOFFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                           const MCSpecifierExpr &Expr) const {
+  printExpr(OS, *Expr.getSubExpr());
+  OS << '@' << getSpecifierName(Expr.getSpecifier());
+}
+
+bool PPCXCOFFMCAsmInfo::evaluateAsRelocatableImpl(
+    const MCSpecifierExpr &Expr, MCValue &Res, const MCAssembler *Asm) const {
+  return PPC::evaluateAsRelocatableImpl(Expr, Res, Asm);
+}
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
index 48806051f581..172fe81c2bce 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
@@ -24,6 +24,10 @@ class PPCELFMCAsmInfo : public MCAsmInfoELF {
 
 public:
   explicit PPCELFMCAsmInfo(bool is64Bit, const Triple &);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                                 const MCAssembler *Asm) const override;
 };
 
 class PPCXCOFFMCAsmInfo : public MCAsmInfoXCOFF {
@@ -31,8 +35,96 @@ class PPCXCOFFMCAsmInfo : public MCAsmInfoXCOFF {
 
 public:
   explicit PPCXCOFFMCAsmInfo(bool is64Bit, const Triple &);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                                 const MCAssembler *Asm) const override;
 };
 
+namespace PPC {
+enum Specifier {
+  S_None,
+
+  S_LO,
+  S_HI,
+  S_HA,
+  S_HIGH,
+  S_HIGHA,
+  S_HIGHER,
+  S_HIGHERA,
+  S_HIGHEST,
+  S_HIGHESTA,
+
+  S_AIX_TLSGD,       // symbol@gd
+  S_AIX_TLSGDM,      // symbol@m
+  S_AIX_TLSIE,       // symbol@ie
+  S_AIX_TLSLD,       // symbol@ld
+  S_AIX_TLSLE,       // symbol@le
+  S_AIX_TLSML,       // symbol@ml
+  S_DTPMOD,          // symbol@dtpmod
+  S_DTPREL,          // symbol@dprel
+  S_DTPREL_HA,       // symbol@dtprel@ha
+  S_DTPREL_HI,       // symbol@dtprel@h
+  S_DTPREL_HIGH,     // symbol@dtprel@high
+  S_DTPREL_HIGHA,    // symbol@dtprel@higha
+  S_DTPREL_HIGHER,   // symbol@dtprel@higher
+  S_DTPREL_HIGHERA,  // symbol@dtprel@highera
+  S_DTPREL_HIGHEST,  // symbol@dtprel@highest
+  S_DTPREL_HIGHESTA, // symbol@dtprel@highesta
+  S_DTPREL_LO,       // symbol@dtprel@l
+  S_GOT,             // symbol@got
+  S_GOT_DTPREL,      // symbol@got@dtprel
+  S_GOT_DTPREL_HA,   // symbol@got@dtprel@ha
+  S_GOT_DTPREL_HI,   // symbol@got@dtprel@h
+  S_GOT_DTPREL_LO,   // symbol@got@dtprel@l
+  S_GOT_HA,          // symbol@got@ha
+  S_GOT_HI,          // symbol@got@h
+  S_GOT_LO,          // symbol@got@l
+  S_GOT_PCREL,       // symbol@got@pcrel
+  S_GOT_TLSGD,       // symbol@got@tlsgd
+  S_GOT_TLSGD_HA,    // symbol@got@tlsgd@ha
+  S_GOT_TLSGD_HI,    // symbol@got@tlsgd@h
+  S_GOT_TLSGD_LO,    // symbol@got@tlsgd@l
+  S_GOT_TLSGD_PCREL, // symbol@got@tlsgd@pcrel
+  S_GOT_TLSLD,       // symbol@got@tlsld
+  S_GOT_TLSLD_HA,    // symbol@got@tlsld@ha
+  S_GOT_TLSLD_HI,    // symbol@got@tlsld@h
+  S_GOT_TLSLD_LO,    // symbol@got@tlsld@l
+  S_GOT_TLSLD_PCREL, // symbol@got@tlsld@pcrel
+  S_GOT_TPREL,       // symbol@got@tprel
+  S_GOT_TPREL_HA,    // symbol@got@tprel@ha
+  S_GOT_TPREL_HI,    // symbol@got@tprel@h
+  S_GOT_TPREL_LO,    // symbol@got@tprel@l
+  S_GOT_TPREL_PCREL, // symbol@got@tprel@pcrel
+  S_L,               // symbol@l
+  S_LOCAL,           // symbol@local
+  S_NOTOC,           // symbol@notoc
+  S_PCREL,
+  S_PCREL_OPT,      // .reloc expr, R_PPC64_PCREL_OPT, expr
+  S_PLT,            // symbol@plt
+  S_TLS,            // symbol@tls
+  S_TLSGD,          // symbol@tlsgd
+  S_TLSLD,          // symbol@tlsld
+  S_TLS_PCREL,      // symbol@tls@pcrel
+  S_TOC,            // symbol@toc
+  S_TOCBASE,        // symbol@tocbase
+  S_TOC_HA,         // symbol@toc@ha
+  S_TOC_HI,         // symbol@toc@h
+  S_TOC_LO,         // symbol@toc@l
+  S_TPREL,          // symbol@tprel
+  S_TPREL_HA,       // symbol@tprel@ha
+  S_TPREL_HI,       // symbol@tprel@h
+  S_TPREL_HIGH,     // symbol@tprel@high
+  S_TPREL_HIGHA,    // symbol@tprel@higha
+  S_TPREL_HIGHER,   // symbol@tprel@higher
+  S_TPREL_HIGHERA,  // symbol@tprel@highera
+  S_TPREL_HIGHEST,  // symbol@tprel@highest
+  S_TPREL_HIGHESTA, // symbol@tprel@highesta
+  S_TPREL_LO,       // symbol@tprel@l
+  S_U,              // symbol@u
+};
+}
+
 } // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index 6a42dfcd3ff3..c395397c88ab 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -329,11 +329,11 @@ PPCMCCodeEmitter::getDispRI34PCRelEncoding(const MCInst &MI, unsigned OpNo,
     const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(Expr);
     (void)SRE;
     // Currently these are the only valid PCRelative Relocations.
-    assert((getSpecifier(SRE) == PPCMCExpr::VK_PCREL ||
-            getSpecifier(SRE) == PPCMCExpr::VK_GOT_PCREL ||
-            getSpecifier(SRE) == PPCMCExpr::VK_GOT_TLSGD_PCREL ||
-            getSpecifier(SRE) == PPCMCExpr::VK_GOT_TLSLD_PCREL ||
-            getSpecifier(SRE) == PPCMCExpr::VK_GOT_TPREL_PCREL) &&
+    assert((getSpecifier(SRE) == PPC::S_PCREL ||
+            getSpecifier(SRE) == PPC::S_GOT_PCREL ||
+            getSpecifier(SRE) == PPC::S_GOT_TLSGD_PCREL ||
+            getSpecifier(SRE) == PPC::S_GOT_TLSLD_PCREL ||
+            getSpecifier(SRE) == PPC::S_GOT_TPREL_PCREL) &&
            "VariantKind must be VK_PCREL or VK_GOT_PCREL or "
            "VK_GOT_TLSGD_PCREL or VK_GOT_TLSLD_PCREL or "
            "VK_GOT_TPREL_PCREL.");
@@ -368,8 +368,8 @@ PPCMCCodeEmitter::getDispRI34PCRelEncoding(const MCInst &MI, unsigned OpNo,
            "Value must fit in 34 bits.");
 
     // Currently these are the only valid PCRelative Relocations.
-    assert((getSpecifier(SRE) == PPCMCExpr::VK_PCREL ||
-            getSpecifier(SRE) == PPCMCExpr::VK_GOT_PCREL) &&
+    assert((getSpecifier(SRE) == PPC::S_PCREL ||
+            getSpecifier(SRE) == PPC::S_GOT_PCREL) &&
            "VariantKind must be VK_PCREL or VK_GOT_PCREL");
     // Generate the fixup for the relocation.
     Fixups.push_back(
@@ -433,7 +433,7 @@ unsigned PPCMCCodeEmitter::getTLSRegEncoding(const MCInst &MI, unsigned OpNo,
   // if using PC relative memops.
   const MCExpr *Expr = MO.getExpr();
   const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(Expr);
-  bool IsPCRel = getSpecifier(SRE) == PPCMCExpr::VK_TLS_PCREL;
+  bool IsPCRel = getSpecifier(SRE) == PPC::S_TLS_PCREL;
   Fixups.push_back(MCFixup::create(IsPCRel ? 1 : 0, Expr,
                                    (MCFixupKind)PPC::fixup_ppc_nofixup));
   const Triple &TT = STI.getTargetTriple();
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
index 376b0e774323..8d9c0892ae16 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
@@ -16,77 +16,62 @@ using namespace llvm;
 
 #define DEBUG_TYPE "ppcmcexpr"
 
-const PPCMCExpr *PPCMCExpr::create(Specifier S, const MCExpr *Expr,
-                                   MCContext &Ctx) {
-  return new (Ctx) PPCMCExpr(S, Expr);
-}
-
-void PPCMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  getSubExpr()->print(OS, MAI);
-  OS << '@' << MAI->getSpecifierName(specifier);
-}
-
-bool
-PPCMCExpr::evaluateAsConstant(int64_t &Res) const {
-  MCValue Value;
-
-  if (!getSubExpr()->evaluateAsRelocatable(Value, nullptr))
-    return false;
-
-  if (!Value.isAbsolute())
-    return false;
-  auto Tmp = evaluateAsInt64(Value.getConstant());
-  if (!Tmp)
-    return false;
-  Res = *Tmp;
-  return true;
-}
-
-std::optional<int64_t> PPCMCExpr::evaluateAsInt64(int64_t Value) const {
+static std::optional<int64_t> evaluateAsInt64(uint16_t specifier,
+                                              int64_t Value) {
   switch (specifier) {
-  case VK_LO:
+  case PPC::S_LO:
     return Value & 0xffff;
-  case VK_HI:
+  case PPC::S_HI:
     return (Value >> 16) & 0xffff;
-  case VK_HA:
+  case PPC::S_HA:
     return ((Value + 0x8000) >> 16) & 0xffff;
-  case VK_HIGH:
+  case PPC::S_HIGH:
     return (Value >> 16) & 0xffff;
-  case VK_HIGHA:
+  case PPC::S_HIGHA:
     return ((Value + 0x8000) >> 16) & 0xffff;
-  case VK_HIGHER:
+  case PPC::S_HIGHER:
     return (Value >> 32) & 0xffff;
-  case VK_HIGHERA:
+  case PPC::S_HIGHERA:
     return ((Value + 0x8000) >> 32) & 0xffff;
-  case VK_HIGHEST:
+  case PPC::S_HIGHEST:
     return (Value >> 48) & 0xffff;
-  case VK_HIGHESTA:
+  case PPC::S_HIGHESTA:
     return ((Value + 0x8000) >> 48) & 0xffff;
   default:
     return {};
   }
 }
 
-bool PPCMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                          const MCAssembler *Asm) const {
-  if (!Asm)
+bool PPC::evaluateAsConstant(const MCSpecifierExpr &Expr, int64_t &Res) {
+  MCValue Value;
+
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(Value, nullptr))
+    return false;
+
+  if (!Value.isAbsolute())
+    return false;
+  auto Tmp = evaluateAsInt64(Expr.getSpecifier(), Value.getConstant());
+  if (!Tmp)
     return false;
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
+  Res = *Tmp;
+  return true;
+}
+
+bool PPC::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                                    const MCAssembler *Asm) {
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(Res, Asm))
     return false;
 
   // The signedness of the result is dependent on the instruction operand. E.g.
   // in addis 3,3,65535@l, 65535@l is signed. In the absence of information at
   // parse time (!Asm), disable the folding.
-  std::optional<int64_t> MaybeInt = evaluateAsInt64(Res.getConstant());
+  std::optional<int64_t> MaybeInt =
+      evaluateAsInt64(Expr.getSpecifier(), Res.getConstant());
   if (Res.isAbsolute() && MaybeInt) {
     Res = MCValue::get(*MaybeInt);
   } else {
-    Res.setSpecifier(specifier);
+    Res.setSpecifier(Expr.getSpecifier());
   }
 
   return true;
 }
-
-void PPCMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
index 19d35649ca36..d97a1204efbc 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
@@ -9,152 +9,27 @@
 #ifndef LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCMCEXPR_H
 #define LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCMCEXPR_H
 
+#include "PPCMCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCValue.h"
 #include <optional>
 
 namespace llvm {
 
-class PPCMCExpr : public MCTargetExpr {
-public:
-  enum Specifier : uint8_t {
-    VK_None,
-
-    VK_LO = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_HI,
-    VK_HA,
-    VK_HIGH,
-    VK_HIGHA,
-    VK_HIGHER,
-    VK_HIGHERA,
-    VK_HIGHEST,
-    VK_HIGHESTA,
-
-    VK_AIX_TLSGD,       // symbol@gd
-    VK_AIX_TLSGDM,      // symbol@m
-    VK_AIX_TLSIE,       // symbol@ie
-    VK_AIX_TLSLD,       // symbol@ld
-    VK_AIX_TLSLE,       // symbol@le
-    VK_AIX_TLSML,       // symbol@ml
-    VK_DTPMOD,          // symbol@dtpmod
-    VK_DTPREL,          // symbol@dprel
-    VK_DTPREL_HA,       // symbol@dtprel@ha
-    VK_DTPREL_HI,       // symbol@dtprel@h
-    VK_DTPREL_HIGH,     // symbol@dtprel@high
-    VK_DTPREL_HIGHA,    // symbol@dtprel@higha
-    VK_DTPREL_HIGHER,   // symbol@dtprel@higher
-    VK_DTPREL_HIGHERA,  // symbol@dtprel@highera
-    VK_DTPREL_HIGHEST,  // symbol@dtprel@highest
-    VK_DTPREL_HIGHESTA, // symbol@dtprel@highesta
-    VK_DTPREL_LO,       // symbol@dtprel@l
-    VK_GOT,             // symbol@got
-    VK_GOT_DTPREL,      // symbol@got@dtprel
-    VK_GOT_DTPREL_HA,   // symbol@got@dtprel@ha
-    VK_GOT_DTPREL_HI,   // symbol@got@dtprel@h
-    VK_GOT_DTPREL_LO,   // symbol@got@dtprel@l
-    VK_GOT_HA,          // symbol@got@ha
-    VK_GOT_HI,          // symbol@got@h
-    VK_GOT_LO,          // symbol@got@l
-    VK_GOT_PCREL,       // symbol@got@pcrel
-    VK_GOT_TLSGD,       // symbol@got@tlsgd
-    VK_GOT_TLSGD_HA,    // symbol@got@tlsgd@ha
-    VK_GOT_TLSGD_HI,    // symbol@got@tlsgd@h
-    VK_GOT_TLSGD_LO,    // symbol@got@tlsgd@l
-    VK_GOT_TLSGD_PCREL, // symbol@got@tlsgd@pcrel
-    VK_GOT_TLSLD,       // symbol@got@tlsld
-    VK_GOT_TLSLD_HA,    // symbol@got@tlsld@ha
-    VK_GOT_TLSLD_HI,    // symbol@got@tlsld@h
-    VK_GOT_TLSLD_LO,    // symbol@got@tlsld@l
-    VK_GOT_TLSLD_PCREL, // symbol@got@tlsld@pcrel
-    VK_GOT_TPREL,       // symbol@got@tprel
-    VK_GOT_TPREL_HA,    // symbol@got@tprel@ha
-    VK_GOT_TPREL_HI,    // symbol@got@tprel@h
-    VK_GOT_TPREL_LO,    // symbol@got@tprel@l
-    VK_GOT_TPREL_PCREL, // symbol@got@tprel@pcrel
-    VK_L,               // symbol@l
-    VK_LOCAL,           // symbol@local
-    VK_NOTOC,           // symbol@notoc
-    VK_PCREL,
-    VK_PCREL_OPT,      // .reloc expr, R_PPC64_PCREL_OPT, expr
-    VK_PLT,            // symbol@plt
-    VK_TLS,            // symbol@tls
-    VK_TLSGD,          // symbol@tlsgd
-    VK_TLSLD,          // symbol@tlsld
-    VK_TLS_PCREL,      // symbol@tls@pcrel
-    VK_TOC,            // symbol@toc
-    VK_TOCBASE,        // symbol@tocbase
-    VK_TOC_HA,         // symbol@toc@ha
-    VK_TOC_HI,         // symbol@toc@h
-    VK_TOC_LO,         // symbol@toc@l
-    VK_TPREL,          // symbol@tprel
-    VK_TPREL_HA,       // symbol@tprel@ha
-    VK_TPREL_HI,       // symbol@tprel@h
-    VK_TPREL_HIGH,     // symbol@tprel@high
-    VK_TPREL_HIGHA,    // symbol@tprel@higha
-    VK_TPREL_HIGHER,   // symbol@tprel@higher
-    VK_TPREL_HIGHERA,  // symbol@tprel@highera
-    VK_TPREL_HIGHEST,  // symbol@tprel@highest
-    VK_TPREL_HIGHESTA, // symbol@tprel@highesta
-    VK_TPREL_LO,       // symbol@tprel@l
-    VK_U,              // symbol@u
-  };
-
-private:
-  const Specifier specifier;
-  const MCExpr *Expr;
-
-  std::optional<int64_t> evaluateAsInt64(int64_t Value) const;
-
-  explicit PPCMCExpr(Specifier S, const MCExpr *Expr)
-      : specifier(S), Expr(Expr) {}
-
-public:
-  /// @name Construction
-  /// @{
-
-  static const PPCMCExpr *create(Specifier S, const MCExpr *Expr,
-                                 MCContext &Ctx);
-
-  static const PPCMCExpr *createLo(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_LO, Expr, Ctx);
-  }
-
-  static const PPCMCExpr *createHi(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_HI, Expr, Ctx);
-  }
-
-  static const PPCMCExpr *createHa(const MCExpr *Expr, MCContext &Ctx) {
-    return create(VK_HA, Expr, Ctx);
-  }
-
-  /// @}
-  /// @name Accessors
-  /// @{
-
-  Specifier getSpecifier() const { return specifier; }
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  /// @}
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  bool evaluateAsConstant(int64_t &Res) const;
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-};
+namespace PPCMCExpr {
+using Specifier = uint16_t;
+}
 
 static inline PPCMCExpr::Specifier getSpecifier(const MCSymbolRefExpr *SRE) {
   return PPCMCExpr::Specifier(SRE->getKind());
 }
 
+namespace PPC {
+bool evaluateAsConstant(const MCSpecifierExpr &Expr, int64_t &Res);
+bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                               const MCAssembler *Asm);
+} // namespace PPC
+
 } // end namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index d7a0bf282036..b0cd108d8334 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -221,9 +221,9 @@ class PPCTargetAsmStreamer : public PPCTargetStreamer {
       // variables. Finally for local-exec and initial-exec, we have a thread
       // pointer, in r13 for 64-bit mode and returned by .__get_tpointer for
       // 32-bit mode.
-      if (Kind == PPCMCExpr::VK_AIX_TLSGD || Kind == PPCMCExpr::VK_AIX_TLSGDM ||
-          Kind == PPCMCExpr::VK_AIX_TLSIE || Kind == PPCMCExpr::VK_AIX_TLSLE ||
-          Kind == PPCMCExpr::VK_AIX_TLSLD || Kind == PPCMCExpr::VK_AIX_TLSML)
+      if (Kind == PPC::S_AIX_TLSGD || Kind == PPC::S_AIX_TLSGDM ||
+          Kind == PPC::S_AIX_TLSIE || Kind == PPC::S_AIX_TLSLE ||
+          Kind == PPC::S_AIX_TLSLD || Kind == PPC::S_AIX_TLSML)
         OS << "\t.tc " << TCSym->getName() << "," << XSym->getName() << "@"
            << getContext().getAsmInfo()->getSpecifierName(Kind) << '\n';
       else
@@ -255,7 +255,7 @@ class PPCTargetAsmStreamer : public PPCTargetStreamer {
     OS << "\t.localentry\t";
     S->print(OS, MAI);
     OS << ", ";
-    LocalOffset->print(OS, MAI);
+    MAI->printExpr(OS, *LocalOffset);
     OS << '\n';
   }
 };
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
index 1b5fe08bea49..8532f537e2d6 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
@@ -61,15 +61,15 @@ std::pair<uint8_t, uint8_t> PPCXCOFFObjectWriter::getRelocTypeAndSignSize(
     switch (Specifier) {
     default:
       report_fatal_error("Unsupported modifier for half16 fixup.");
-    case PPCMCExpr::VK_None:
+    case PPC::S_None:
       return {XCOFF::RelocationType::R_TOC, SignAndSizeForHalf16};
-    case PPCMCExpr::VK_U:
+    case PPC::S_U:
       return {XCOFF::RelocationType::R_TOCU, SignAndSizeForHalf16};
-    case PPCMCExpr::VK_L:
+    case PPC::S_L:
       return {XCOFF::RelocationType::R_TOCL, SignAndSizeForHalf16};
-    case PPCMCExpr::VK_AIX_TLSLE:
+    case PPC::S_AIX_TLSLE:
       return {XCOFF::RelocationType::R_TLS_LE, SignAndSizeForHalf16};
-    case PPCMCExpr::VK_AIX_TLSLD:
+    case PPC::S_AIX_TLSLD:
       return {XCOFF::RelocationType::R_TLS_LD, SignAndSizeForHalf16};
     }
   } break;
@@ -80,13 +80,13 @@ std::pair<uint8_t, uint8_t> PPCXCOFFObjectWriter::getRelocTypeAndSignSize(
     switch (Specifier) {
     default:
       llvm_unreachable("Unsupported Modifier");
-    case PPCMCExpr::VK_None:
+    case PPC::S_None:
       return {XCOFF::RelocationType::R_TOC, 15};
-    case PPCMCExpr::VK_L:
+    case PPC::S_L:
       return {XCOFF::RelocationType::R_TOCL, 15};
-    case PPCMCExpr::VK_AIX_TLSLE:
+    case PPC::S_AIX_TLSLE:
       return {XCOFF::RelocationType::R_TLS_LE, 15};
-    case PPCMCExpr::VK_AIX_TLSLD:
+    case PPC::S_AIX_TLSLD:
       return {XCOFF::RelocationType::R_TLS_LD, 15};
     }
   } break;
@@ -97,7 +97,7 @@ std::pair<uint8_t, uint8_t> PPCXCOFFObjectWriter::getRelocTypeAndSignSize(
   case PPC::fixup_ppc_br24abs:
     return {XCOFF::RelocationType::R_RBA, EncodedSignednessIndicator | 25};
   case PPC::fixup_ppc_nofixup: {
-    if (Specifier == PPCMCExpr::VK_None)
+    if (Specifier == PPC::S_None)
       return {XCOFF::RelocationType::R_REF, 0};
     else
       llvm_unreachable("Unsupported Modifier");
@@ -110,19 +110,19 @@ std::pair<uint8_t, uint8_t> PPCXCOFFObjectWriter::getRelocTypeAndSignSize(
     switch (Specifier) {
     default:
       report_fatal_error("Unsupported modifier");
-    case PPCMCExpr::VK_AIX_TLSGD:
+    case PPC::S_AIX_TLSGD:
       return {XCOFF::RelocationType::R_TLS, SignAndSizeForFKData};
-    case PPCMCExpr::VK_AIX_TLSGDM:
+    case PPC::S_AIX_TLSGDM:
       return {XCOFF::RelocationType::R_TLSM, SignAndSizeForFKData};
-    case PPCMCExpr::VK_AIX_TLSIE:
+    case PPC::S_AIX_TLSIE:
       return {XCOFF::RelocationType::R_TLS_IE, SignAndSizeForFKData};
-    case PPCMCExpr::VK_AIX_TLSLE:
+    case PPC::S_AIX_TLSLE:
       return {XCOFF::RelocationType::R_TLS_LE, SignAndSizeForFKData};
-    case PPCMCExpr::VK_AIX_TLSLD:
+    case PPC::S_AIX_TLSLD:
       return {XCOFF::RelocationType::R_TLS_LD, SignAndSizeForFKData};
-    case PPCMCExpr::VK_AIX_TLSML:
+    case PPC::S_AIX_TLSML:
       return {XCOFF::RelocationType::R_TLSML, SignAndSizeForFKData};
-    case PPCMCExpr::VK_None:
+    case PPC::S_None:
       return {XCOFF::RelocationType::R_POS, SignAndSizeForFKData};
     }
   }
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPC.td b/external/llvm-project/llvm/lib/Target/PowerPC/PPC.td
index 6b058d1a7477..fd850faf7b2f 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPC.td
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPC.td
@@ -482,7 +482,8 @@ def ProcessorFeatures {
   // Future
   // For future CPU we assume that all of the existing features from Power11
   // still exist with the exception of those we know are Power11 specific.
-  list<SubtargetFeature> FutureAdditionalFeatures = [FeatureISAFuture];
+  list<SubtargetFeature> FutureAdditionalFeatures = [DirectivePwrFuture,
+                                                     FeatureISAFuture];
   list<SubtargetFeature> FutureSpecificFeatures = [];
   list<SubtargetFeature> FutureInheritableFeatures =
     !listconcat(P11InheritableFeatures, FutureAdditionalFeatures);
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 1fac1b223f9d..663656ac2d31 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -108,9 +108,9 @@ template <>
 struct DenseMapInfo<std::pair<const MCSymbol *, PPCMCExpr::Specifier>> {
   using TOCKey = std::pair<const MCSymbol *, PPCMCExpr::Specifier>;
 
-  static inline TOCKey getEmptyKey() { return {nullptr, PPCMCExpr::VK_None}; }
+  static inline TOCKey getEmptyKey() { return {nullptr, PPC::S_None}; }
   static inline TOCKey getTombstoneKey() {
-    return {(const MCSymbol *)1, PPCMCExpr::VK_None};
+    return {(const MCSymbol *)1, PPC::S_None};
   }
   static unsigned getHashValue(const TOCKey &PairVal) {
     return detail::combineHashValue(
@@ -174,9 +174,8 @@ class PPCAsmPrinter : public AsmPrinter {
     TOCType_EHBlock
   };
 
-  MCSymbol *
-  lookUpOrCreateTOCEntry(const MCSymbol *Sym, TOCEntryType Type,
-                         PPCMCExpr::Specifier Kind = PPCMCExpr::VK_None);
+  MCSymbol *lookUpOrCreateTOCEntry(const MCSymbol *Sym, TOCEntryType Type,
+                                   PPCMCExpr::Specifier Kind = PPC::S_None);
 
   bool doInitialization(Module &M) override {
     if (!TOC.empty())
@@ -691,13 +690,13 @@ void PPCAsmPrinter::EmitAIXTlsCallHelper(const MachineInstr *MI) {
 /// the current output stream.
 void PPCAsmPrinter::emitTlsCall(const MachineInstr *MI,
                                 PPCMCExpr::Specifier VK) {
-  PPCMCExpr::Specifier Kind = PPCMCExpr::VK_None;
+  PPCMCExpr::Specifier Kind = PPC::S_None;
   unsigned Opcode = PPC::BL8_NOP_TLS;
 
   assert(MI->getNumOperands() >= 3 && "Expecting at least 3 operands from MI");
   if (MI->getOperand(2).getTargetFlags() == PPCII::MO_GOT_TLSGD_PCREL_FLAG ||
       MI->getOperand(2).getTargetFlags() == PPCII::MO_GOT_TLSLD_PCREL_FLAG) {
-    Kind = PPCMCExpr::VK_NOTOC;
+    Kind = PPC::S_NOTOC;
     Opcode = PPC::BL8_NOTOC_TLS;
   }
   const Module *M = MF->getFunction().getParent();
@@ -730,13 +729,13 @@ void PPCAsmPrinter::emitTlsCall(const MachineInstr *MI,
   MCSymbol *TlsGetAddr = OutContext.getOrCreateSymbol("__tls_get_addr");
 
   if (Subtarget->is32BitELFABI() && isPositionIndependent())
-    Kind = PPCMCExpr::VK_PLT;
+    Kind = PPC::S_PLT;
 
   const MCExpr *TlsRef = MCSymbolRefExpr::create(
       TlsGetAddr, MCSymbolRefExpr::VariantKind(Kind), OutContext);
 
   // Add 32768 offset to the symbol so we follow up the latest GOT/PLT ABI.
-  if (Kind == PPCMCExpr::VK_PLT && Subtarget->isSecurePlt() &&
+  if (Kind == PPC::S_PLT && Subtarget->isSecurePlt() &&
       M->getPICLevel() == PICLevel::BigPIC)
     TlsRef = MCBinaryExpr::createAdd(
         TlsRef, MCConstantExpr::create(32768, OutContext), OutContext);
@@ -861,7 +860,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   auto getTOCEntryLoadingExprForXCOFF =
       [IsPPC64, getTOCRelocAdjustedExprForXCOFF,
        this](const MCSymbol *MOSymbol, const MCExpr *Expr,
-             PPCMCExpr::Specifier VK = PPCMCExpr::VK_None) -> const MCExpr * {
+             PPCMCExpr::Specifier VK = PPC::S_None) -> const MCExpr * {
     const unsigned EntryByteSize = IsPPC64 ? 8 : 4;
     const auto TOCEntryIter = TOC.find({MOSymbol, VK});
     assert(TOCEntryIter != TOC.end() &&
@@ -886,9 +885,9 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
       assert(MO.isGlobal() && "Only expecting a global MachineOperand here!\n");
       TLSModel::Model Model = TM.getTLSModel(MO.getGlobal());
       if (Model == TLSModel::LocalExec)
-        return PPCMCExpr::VK_AIX_TLSLE;
+        return PPC::S_AIX_TLSLE;
       if (Model == TLSModel::InitialExec)
-        return PPCMCExpr::VK_AIX_TLSIE;
+        return PPC::S_AIX_TLSIE;
       // On AIX, TLS model opt may have turned local-dynamic accesses into
       // initial-exec accesses.
       PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
@@ -896,7 +895,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
           FuncInfo->isAIXFuncUseTLSIEForLD()) {
         LLVM_DEBUG(
             dbgs() << "Current function uses IE access for default LD vars.\n");
-        return PPCMCExpr::VK_AIX_TLSIE;
+        return PPC::S_AIX_TLSIE;
       }
       llvm_unreachable("Only expecting local-exec or initial-exec accesses!");
     }
@@ -904,17 +903,17 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     // the variable offset and the other for the region handle). They are
     // differentiated by MO_TLSGD_FLAG and MO_TLSGDM_FLAG.
     if (Flag == PPCII::MO_TLSGDM_FLAG)
-      return PPCMCExpr::VK_AIX_TLSGDM;
+      return PPC::S_AIX_TLSGDM;
     if (Flag == PPCII::MO_TLSGD_FLAG || Flag == PPCII::MO_GOT_TLSGD_PCREL_FLAG)
-      return PPCMCExpr::VK_AIX_TLSGD;
+      return PPC::S_AIX_TLSGD;
     // For local-dynamic TLS access on AIX, we have one TOC entry for the symbol
     // (the variable offset) and one shared TOC entry for the module handle.
     // They are differentiated by MO_TLSLD_FLAG and MO_TLSLDM_FLAG.
     if (Flag == PPCII::MO_TLSLD_FLAG && IsAIX)
-      return PPCMCExpr::VK_AIX_TLSLD;
+      return PPC::S_AIX_TLSLD;
     if (Flag == PPCII::MO_TLSLDM_FLAG && IsAIX)
-      return PPCMCExpr::VK_AIX_TLSML;
-    return PPCMCExpr::VK_None;
+      return PPC::S_AIX_TLSML;
+    return PPC::S_None;
   };
 
   // Lower multi-instruction pseudo operations.
@@ -955,8 +954,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
       OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
     const MCExpr *OffsExpr = MCBinaryExpr::createSub(
         MCSymbolRefExpr::create(
-            GOTSymbol, MCSymbolRefExpr::VariantKind(PPCMCExpr::VK_LOCAL),
-            OutContext),
+            GOTSymbol, MCSymbolRefExpr::VariantKind(PPC::S_LOCAL), OutContext),
         MCConstantExpr::create(4, OutContext), OutContext);
 
     // Emit the 'bl'.
@@ -1002,12 +1000,14 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
       const MCExpr *DeltaExpr = MCBinaryExpr::createSub(
           MCSymbolRefExpr::create(BaseSymbol, OutContext), PB, OutContext);
 
-      const MCExpr *DeltaHi = PPCMCExpr::createHa(DeltaExpr, OutContext);
+      const MCExpr *DeltaHi =
+          MCSpecifierExpr::create(DeltaExpr, PPC::S_HA, OutContext);
       EmitToStreamer(
           *OutStreamer,
           MCInstBuilder(PPC::ADDIS).addReg(PICR).addReg(PICR).addExpr(DeltaHi));
 
-      const MCExpr *DeltaLo = PPCMCExpr::createLo(DeltaExpr, OutContext);
+      const MCExpr *DeltaLo =
+          MCSpecifierExpr::create(DeltaExpr, PPC::S_LO, OutContext);
       EmitToStreamer(
           *OutStreamer,
           MCInstBuilder(PPC::ADDI).addReg(PICR).addReg(PICR).addExpr(DeltaLo));
@@ -1055,7 +1055,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     // Create a reference to the GOT entry for the symbol. The GOT entry will be
     // synthesized later.
     if (PL == PICLevel::SmallPIC && !IsAIX) {
-      const MCExpr *Exp = symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_GOT);
+      const MCExpr *Exp = symbolWithSpecifier(MOSymbol, PPC::S_GOT);
       TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
       EmitToStreamer(*OutStreamer, TmpInst);
       return;
@@ -1144,8 +1144,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     MCSymbol *TOCEntry =
         lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK);
 
-    PPCMCExpr::Specifier VKExpr =
-        IsAIX ? PPCMCExpr::VK_None : PPCMCExpr::VK_TOC;
+    PPCMCExpr::Specifier VKExpr = IsAIX ? PPC::S_None : PPC::S_TOC;
     const MCExpr *Exp = symbolWithSpecifier(TOCEntry, VKExpr);
     TmpInst.getOperand(1) = MCOperand::createExpr(
         IsAIX ? getTOCEntryLoadingExprForXCOFF(MOSymbol, Exp, VK) : Exp);
@@ -1195,7 +1194,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
       MOSymbol = lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK);
     }
 
-    const MCExpr *Exp = symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_U);
+    const MCExpr *Exp = symbolWithSpecifier(MOSymbol, PPC::S_U);
     TmpInst.getOperand(2) = MCOperand::createExpr(Exp);
     EmitToStreamer(*OutStreamer, TmpInst);
     return;
@@ -1227,7 +1226,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     // 'MOSymbol'.
     MCSymbol *TOCEntry =
         lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK);
-    const MCExpr *Exp = symbolWithSpecifier(TOCEntry, PPCMCExpr::VK_L);
+    const MCExpr *Exp = symbolWithSpecifier(TOCEntry, PPC::S_L);
     TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
     EmitToStreamer(*OutStreamer, TmpInst);
     return;
@@ -1260,7 +1259,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
         (MO.isCPI() && CM == CodeModel::Large))
       MOSymbol = lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK);
 
-    VK = IsAIX ? PPCMCExpr::VK_U : PPCMCExpr::VK_TOC_HA;
+    VK = IsAIX ? PPC::S_U : PPC::S_TOC_HA;
 
     const MCExpr *Exp = symbolWithSpecifier(MOSymbol, VK);
 
@@ -1302,7 +1301,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     if (!MO.isCPI() || CM == CodeModel::Large)
       MOSymbol = lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK);
 
-    VK = IsAIX ? PPCMCExpr::VK_L : PPCMCExpr::VK_TOC_LO;
+    VK = IsAIX ? PPC::S_L : PPC::S_TOC_LO;
     const MCExpr *Exp = symbolWithSpecifier(MOSymbol, VK);
     TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
     EmitToStreamer(*OutStreamer, TmpInst);
@@ -1332,8 +1331,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
 
     const MCExpr *Exp = MCSymbolRefExpr::create(
         MOSymbol,
-        MCSymbolRefExpr::VariantKind(IsAIX ? PPCMCExpr::VK_L
-                                           : PPCMCExpr::VK_TOC_LO),
+        MCSymbolRefExpr::VariantKind(IsAIX ? PPC::S_L : PPC::S_TOC_LO),
         OutContext);
 
     TmpInst.getOperand(2) = MCOperand::createExpr(Exp);
@@ -1348,7 +1346,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymGotTprel =
-        symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_GOT_TPREL_HA);
+        symbolWithSpecifier(MOSymbol, PPC::S_GOT_TPREL_HA);
     EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8)
                                  .addReg(MI->getOperand(0).getReg())
                                  .addReg(MI->getOperand(1).getReg())
@@ -1365,9 +1363,8 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(1);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *Exp =
-        symbolWithSpecifier(MOSymbol, IsPPC64 ? PPCMCExpr::VK_GOT_TPREL_LO
-                                              : PPCMCExpr::VK_GOT_TPREL);
+    const MCExpr *Exp = symbolWithSpecifier(
+        MOSymbol, IsPPC64 ? PPC::S_GOT_TPREL_LO : PPC::S_GOT_TPREL);
     TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
     EmitToStreamer(*OutStreamer, TmpInst);
     return;
@@ -1404,12 +1401,10 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case PPC::PPC32GOT: {
     MCSymbol *GOTSymbol =
         OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
-    const MCExpr *SymGotTlsL = PPCMCExpr::create(
-        PPCMCExpr::VK_LO, MCSymbolRefExpr::create(GOTSymbol, OutContext),
-        OutContext);
-    const MCExpr *SymGotTlsHA = PPCMCExpr::create(
-        PPCMCExpr::VK_HA, MCSymbolRefExpr::create(GOTSymbol, OutContext),
-        OutContext);
+    const MCExpr *SymGotTlsL =
+        MCSpecifierExpr::create(GOTSymbol, PPC::S_LO, OutContext);
+    const MCExpr *SymGotTlsHA =
+        MCSpecifierExpr::create(GOTSymbol, PPC::S_HA, OutContext);
     EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LI)
                                  .addReg(MI->getOperand(0).getReg())
                                  .addExpr(SymGotTlsL));
@@ -1427,7 +1422,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymGotTlsGD =
-        symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_GOT_TLSGD_HA);
+        symbolWithSpecifier(MOSymbol, PPC::S_GOT_TLSGD_HA);
     EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8)
                                  .addReg(MI->getOperand(0).getReg())
                                  .addReg(MI->getOperand(1).getReg())
@@ -1443,9 +1438,8 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(2);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *SymGotTlsGD =
-        symbolWithSpecifier(MOSymbol, IsPPC64 ? PPCMCExpr::VK_GOT_TLSGD_LO
-                                              : PPCMCExpr::VK_GOT_TLSGD);
+    const MCExpr *SymGotTlsGD = symbolWithSpecifier(
+        MOSymbol, IsPPC64 ? PPC::S_GOT_TLSGD_LO : PPC::S_GOT_TLSGD);
     EmitToStreamer(*OutStreamer,
                    MCInstBuilder(IsPPC64 ? PPC::ADDI8 : PPC::ADDI)
                    .addReg(MI->getOperand(0).getReg())
@@ -1470,7 +1464,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case PPC::GETtlsADDR32: {
     // Transform: %r3 = GETtlsADDR32 %r3, @sym
     // Into: BL_TLS __tls_get_addr(sym at tlsgd)@PLT
-    emitTlsCall(MI, PPCMCExpr::VK_TLSGD);
+    emitTlsCall(MI, PPC::S_TLSGD);
     return;
   }
   case PPC::GETtlsTpointer32AIX: {
@@ -1487,7 +1481,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
     const MCExpr *SymGotTlsLD =
-        symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_GOT_TLSLD_HA);
+        symbolWithSpecifier(MOSymbol, PPC::S_GOT_TLSLD_HA);
     EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8)
                                  .addReg(MI->getOperand(0).getReg())
                                  .addReg(MI->getOperand(1).getReg())
@@ -1503,9 +1497,8 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(2);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *SymGotTlsLD =
-        symbolWithSpecifier(MOSymbol, IsPPC64 ? PPCMCExpr::VK_GOT_TLSLD_LO
-                                              : PPCMCExpr::VK_GOT_TLSLD);
+    const MCExpr *SymGotTlsLD = symbolWithSpecifier(
+        MOSymbol, IsPPC64 ? PPC::S_GOT_TLSLD_LO : PPC::S_GOT_TLSLD);
     EmitToStreamer(*OutStreamer,
                    MCInstBuilder(IsPPC64 ? PPC::ADDI8 : PPC::ADDI)
                        .addReg(MI->getOperand(0).getReg())
@@ -1520,7 +1513,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case PPC::GETtlsldADDR32: {
     // Transform: %r3 = GETtlsldADDR32 %r3, @sym
     // Into: BL_TLS __tls_get_addr(sym at tlsld)@PLT
-    emitTlsCall(MI, PPCMCExpr::VK_TLSLD);
+    emitTlsCall(MI, PPC::S_TLSLD);
     return;
   }
   case PPC::ADDISdtprelHA:
@@ -1532,8 +1525,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(2);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *SymDtprel =
-        symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_DTPREL_HA);
+    const MCExpr *SymDtprel = symbolWithSpecifier(MOSymbol, PPC::S_DTPREL_HA);
     EmitToStreamer(
         *OutStreamer,
         MCInstBuilder(IsPPC64 ? PPC::ADDIS8 : PPC::ADDIS)
@@ -1548,8 +1540,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(2);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *SymDtprel =
-        symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_DTPREL);
+    const MCExpr *SymDtprel = symbolWithSpecifier(MOSymbol, PPC::S_DTPREL);
     EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::PADDI8)
                                      .addReg(MI->getOperand(0).getReg())
                                      .addReg(MI->getOperand(1).getReg())
@@ -1566,8 +1557,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MachineOperand &MO = MI->getOperand(2);
     const GlobalValue *GValue = MO.getGlobal();
     MCSymbol *MOSymbol = getSymbol(GValue);
-    const MCExpr *SymDtprel =
-        symbolWithSpecifier(MOSymbol, PPCMCExpr::VK_DTPREL_LO);
+    const MCExpr *SymDtprel = symbolWithSpecifier(MOSymbol, PPC::S_DTPREL_LO);
     EmitToStreamer(*OutStreamer,
                    MCInstBuilder(IsPPC64 ? PPC::ADDI8 : PPC::ADDI)
                        .addReg(MI->getOperand(0).getReg())
@@ -1737,9 +1727,8 @@ PPCAsmPrinter::getAdjustedFasterLocalExpr(const MachineOperand &MO,
   // assume that the address of extern TLS variables are zero.
   const MCExpr *Expr = MCSymbolRefExpr::create(
       getSymbol(GValue),
-      MCSymbolRefExpr::VariantKind(Model == TLSModel::LocalExec
-                                       ? PPCMCExpr::VK_AIX_TLSLE
-                                       : PPCMCExpr::VK_AIX_TLSLD),
+      MCSymbolRefExpr::VariantKind(
+          Model == TLSModel::LocalExec ? PPC::S_AIX_TLSLE : PPC::S_AIX_TLSLD),
       OutContext);
   Expr = MCBinaryExpr::createAdd(
       Expr, MCConstantExpr::create(Offset, OutContext), OutContext);
@@ -2028,8 +2017,7 @@ void PPCLinuxAsmPrinter::emitFunctionEntryLabel() {
   // Generates a R_PPC64_TOC relocation for TOC base insertion.
   OutStreamer->emitValue(
       MCSymbolRefExpr::create(
-          Symbol2, MCSymbolRefExpr::VariantKind(PPCMCExpr::VK_TOCBASE),
-          OutContext),
+          Symbol2, MCSymbolRefExpr::VariantKind(PPC::S_TOCBASE), OutContext),
       8 /*size*/);
   // Emit a null environment pointer.
   OutStreamer->emitIntValue(0, 8 /* size */);
@@ -2136,13 +2124,15 @@ void PPCLinuxAsmPrinter::emitFunctionBodyStart() {
         MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCSymbol, OutContext),
                                 GlobalEntryLabelExp, OutContext);
 
-      const MCExpr *TOCDeltaHi = PPCMCExpr::createHa(TOCDeltaExpr, OutContext);
+      const MCExpr *TOCDeltaHi =
+          MCSpecifierExpr::create(TOCDeltaExpr, PPC::S_HA, OutContext);
       EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS)
                                    .addReg(PPC::X2)
                                    .addReg(PPC::X12)
                                    .addExpr(TOCDeltaHi));
 
-      const MCExpr *TOCDeltaLo = PPCMCExpr::createLo(TOCDeltaExpr, OutContext);
+      const MCExpr *TOCDeltaLo =
+          MCSpecifierExpr::create(TOCDeltaExpr, PPC::S_LO, OutContext);
       EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDI)
                                    .addReg(PPC::X2)
                                    .addReg(PPC::X2)
@@ -3007,9 +2997,9 @@ void PPCAIXAsmPrinter::emitEndOfAsmFile(Module &M) {
     // new symbol to prefix the name with a dot.
     // If TLS model opt is turned on, create a new symbol to prefix the name
     // with a dot.
-    if (I.first.second == PPCMCExpr::VK_AIX_TLSGDM ||
+    if (I.first.second == PPC::S_AIX_TLSGDM ||
         (Subtarget->hasAIXShLibTLSModelOpt() &&
-         I.first.second == PPCMCExpr::VK_AIX_TLSLD)) {
+         I.first.second == PPC::S_AIX_TLSLD)) {
       SmallString<128> Name;
       StringRef Prefix = ".";
       Name += Prefix;
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 0c2a50600560..0f8e5e57c58b 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -1428,39 +1428,6 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     setTargetDAGCombine({ISD::TRUNCATE, ISD::SETCC, ISD::SELECT_CC});
   }
 
-  setLibcallName(RTLIB::LOG_F128, "logf128");
-  setLibcallName(RTLIB::LOG2_F128, "log2f128");
-  setLibcallName(RTLIB::LOG10_F128, "log10f128");
-  setLibcallName(RTLIB::EXP_F128, "expf128");
-  setLibcallName(RTLIB::EXP2_F128, "exp2f128");
-  setLibcallName(RTLIB::SIN_F128, "sinf128");
-  setLibcallName(RTLIB::COS_F128, "cosf128");
-  setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
-  setLibcallName(RTLIB::POW_F128, "powf128");
-  setLibcallName(RTLIB::FMIN_F128, "fminf128");
-  setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-  setLibcallName(RTLIB::REM_F128, "fmodf128");
-  setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
-  setLibcallName(RTLIB::CEIL_F128, "ceilf128");
-  setLibcallName(RTLIB::FLOOR_F128, "floorf128");
-  setLibcallName(RTLIB::TRUNC_F128, "truncf128");
-  setLibcallName(RTLIB::ROUND_F128, "roundf128");
-  setLibcallName(RTLIB::LROUND_F128, "lroundf128");
-  setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
-  setLibcallName(RTLIB::RINT_F128, "rintf128");
-  setLibcallName(RTLIB::LRINT_F128, "lrintf128");
-  setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
-  setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
-  setLibcallName(RTLIB::FMA_F128, "fmaf128");
-  setLibcallName(RTLIB::FREXP_F128, "frexpf128");
-
-  if (Subtarget.isAIXABI()) {
-    setLibcallName(RTLIB::MEMCPY, isPPC64 ? "___memmove64" : "___memmove");
-    setLibcallName(RTLIB::MEMMOVE, isPPC64 ? "___memmove64" : "___memmove");
-    setLibcallName(RTLIB::MEMSET, isPPC64 ? "___memset64" : "___memset");
-    setLibcallName(RTLIB::BZERO, isPPC64 ? "___bzero64" : "___bzero");
-  }
-
   // With 32 condition bits, we don't need to sink (and duplicate) compares
   // aggressively in CodeGenPrep.
   if (Subtarget.useCRBits()) {
@@ -1475,8 +1442,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   setMinimumJumpTableEntries(PPCMinimumJumpTableEntries);
 
   setMinFunctionAlignment(Align(4));
+  setMinCmpXchgSizeInBits(Subtarget.hasPartwordAtomics() ? 8 : 32);
 
-  switch (Subtarget.getCPUDirective()) {
+  auto CPUDirective = Subtarget.getCPUDirective();
+  switch (CPUDirective) {
   default: break;
   case PPC::DIR_970:
   case PPC::DIR_A2:
@@ -1508,15 +1477,14 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
   // The Freescale cores do better with aggressive inlining of memcpy and
   // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
-  if (Subtarget.getCPUDirective() == PPC::DIR_E500mc ||
-      Subtarget.getCPUDirective() == PPC::DIR_E5500) {
+  if (CPUDirective == PPC::DIR_E500mc || CPUDirective == PPC::DIR_E5500) {
     MaxStoresPerMemset = 32;
     MaxStoresPerMemsetOptSize = 16;
     MaxStoresPerMemcpy = 32;
     MaxStoresPerMemcpyOptSize = 8;
     MaxStoresPerMemmove = 32;
     MaxStoresPerMemmoveOptSize = 8;
-  } else if (Subtarget.getCPUDirective() == PPC::DIR_A2) {
+  } else if (CPUDirective == PPC::DIR_A2) {
     // The A2 also benefits from (very) aggressive inlining of memcpy and
     // friends. The overhead of a the function call, even when warm, can be
     // over one hundred cycles.
@@ -1529,6 +1497,11 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     MaxLoadsPerMemcmpOptSize = 4;
   }
 
+  // Enable generation of STXVP instructions by default for mcpu=future.
+  if (CPUDirective == PPC::DIR_PWR_FUTURE &&
+      DisableAutoPairedVecSt.getNumOccurrences() == 0)
+    DisableAutoPairedVecSt = false;
+
   IsStrictFPEnabled = true;
 
   // Let the subtarget (CPU) decide if a predictable select is more expensive
@@ -9667,7 +9640,24 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     }
   }
 
-  if (!BVNIsConstantSplat || SplatBitSize > 32) {
+  bool IsSplat64 = false;
+  uint64_t SplatBits = 0;
+  int32_t SextVal = 0;
+  if (BVNIsConstantSplat && SplatBitSize <= 64) {
+    SplatBits = APSplatBits.getZExtValue();
+    if (SplatBitSize <= 32) {
+      SextVal = SignExtend32(SplatBits, SplatBitSize);
+    } else if (SplatBitSize == 64 && Subtarget.hasP8Altivec()) {
+      int64_t Splat64Val = static_cast<int64_t>(SplatBits);
+      bool P9Vector = Subtarget.hasP9Vector();
+      int32_t Hi = P9Vector ? 127 : 15;
+      int32_t Lo = P9Vector ? -128 : -16;
+      IsSplat64 = Splat64Val >= Lo && Splat64Val <= Hi;
+      SextVal = static_cast<int32_t>(SplatBits);
+    }
+  }
+
+  if (!BVNIsConstantSplat || (SplatBitSize > 32 && !IsSplat64)) {
     unsigned NewOpcode = PPCISD::LD_SPLAT;
 
     // Handle load-and-splat patterns as we have instructions that will do this
@@ -9753,7 +9743,6 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     return SDValue();
   }
 
-  uint64_t SplatBits = APSplatBits.getZExtValue();
   uint64_t SplatUndef = APSplatUndef.getZExtValue();
   unsigned SplatSize = SplatBitSize / 8;
 
@@ -9788,13 +9777,43 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                   dl);
 
   // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
-  int32_t SextVal = SignExtend32(SplatBits, SplatBitSize);
-  if (SextVal >= -16 && SextVal <= 15)
-    return getCanonicalConstSplat(SextVal, SplatSize, Op.getValueType(), DAG,
-                                  dl);
+  // Use VSPLTIW/VUPKLSW for v2i64 in range [-16,15].
+  if (SextVal >= -16 && SextVal <= 15) {
+    // SplatSize may be 1, 2, 4, or 8. Use size 4 instead of 8 for the splat to
+    // generate a splat word with extend for size 8.
+    unsigned UseSize = SplatSize == 8 ? 4 : SplatSize;
+    SDValue Res =
+        getCanonicalConstSplat(SextVal, UseSize, Op.getValueType(), DAG, dl);
+    if (SplatSize != 8)
+      return Res;
+    return BuildIntrinsicOp(Intrinsic::ppc_altivec_vupklsw, Res, DAG, dl);
+  }
 
   // Two instruction sequences.
 
+  if (Subtarget.hasP9Vector() && SextVal >= -128 && SextVal <= 127) {
+    SDValue C = DAG.getConstant((unsigned char)SextVal, dl, MVT::i32);
+    SmallVector<SDValue, 16> Ops(16, C);
+    SDValue BV = DAG.getBuildVector(MVT::v16i8, dl, Ops);
+    unsigned IID;
+    switch (SplatSize) {
+    default:
+      llvm_unreachable("Unexpected type for vector constant.");
+    case 2:
+      IID = Intrinsic::ppc_altivec_vupklsb;
+      break;
+    case 4:
+      IID = Intrinsic::ppc_altivec_vextsb2w;
+      break;
+    case 8:
+      IID = Intrinsic::ppc_altivec_vextsb2d;
+      break;
+    }
+    SDValue Extend = BuildIntrinsicOp(IID, BV, DAG, dl);
+    return DAG.getBitcast(Op->getValueType(0), Extend);
+  }
+  assert(!IsSplat64 && "Unhandled 64-bit splat pattern");
+
   // If this value is in the range [-32,30] and is even, use:
   //     VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2)
   // If this value is in the range [17,31] and is odd, use:
@@ -12672,6 +12691,76 @@ static Instruction *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
   return Builder.CreateIntrinsic(Id, {});
 }
 
+Value *PPCTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
+                                         Value *Addr,
+                                         AtomicOrdering Ord) const {
+  unsigned SZ = ValueTy->getPrimitiveSizeInBits();
+
+  assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
+         "Only 8/16/32/64-bit atomic loads supported");
+  Intrinsic::ID IntID;
+  switch (SZ) {
+  default:
+    llvm_unreachable("Unexpected PrimitiveSize");
+  case 8:
+    IntID = Intrinsic::ppc_lbarx;
+    assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
+    break;
+  case 16:
+    IntID = Intrinsic::ppc_lharx;
+    assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
+    break;
+  case 32:
+    IntID = Intrinsic::ppc_lwarx;
+    break;
+  case 64:
+    IntID = Intrinsic::ppc_ldarx;
+    break;
+  }
+  Value *Call =
+      Builder.CreateIntrinsic(IntID, Addr, /*FMFSource=*/nullptr, "larx");
+
+  return Builder.CreateTruncOrBitCast(Call, ValueTy);
+}
+
+// Perform a store-conditional operation to Addr. Return the status of the
+// store. This should be 0 if the store succeeded, non-zero otherwise.
+Value *PPCTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
+                                               Value *Val, Value *Addr,
+                                               AtomicOrdering Ord) const {
+  Type *Ty = Val->getType();
+  unsigned SZ = Ty->getPrimitiveSizeInBits();
+
+  assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
+         "Only 8/16/32/64-bit atomic loads supported");
+  Intrinsic::ID IntID;
+  switch (SZ) {
+  default:
+    llvm_unreachable("Unexpected PrimitiveSize");
+  case 8:
+    IntID = Intrinsic::ppc_stbcx;
+    assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
+    break;
+  case 16:
+    IntID = Intrinsic::ppc_sthcx;
+    assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
+    break;
+  case 32:
+    IntID = Intrinsic::ppc_stwcx;
+    break;
+  case 64:
+    IntID = Intrinsic::ppc_stdcx;
+    break;
+  }
+
+  if (SZ == 8 || SZ == 16)
+    Val = Builder.CreateZExt(Val, Builder.getInt32Ty());
+
+  Value *Call = Builder.CreateIntrinsic(IntID, {Addr, Val},
+                                        /*FMFSource=*/nullptr, "stcx");
+  return Builder.CreateXor(Call, Builder.getInt32(1));
+}
+
 // The mappings for emitLeading/TrailingFence is taken from
 // http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
 Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
@@ -19633,7 +19722,7 @@ PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
   unsigned Size = AI->getNewValOperand()->getType()->getPrimitiveSizeInBits();
   if (shouldInlineQuadwordAtomics() && Size == 128)
     return AtomicExpansionKind::MaskedIntrinsic;
-  return TargetLowering::shouldExpandAtomicCmpXchgInIR(AI);
+  return AtomicExpansionKind::LLSC;
 }
 
 static Intrinsic::ID
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h b/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 2c55b5427297..4c88bd372b10 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -927,6 +927,12 @@ namespace llvm {
       return true;
     }
 
+    Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
+                          AtomicOrdering Ord) const override;
+
+    Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,
+                                AtomicOrdering Ord) const override;
+
     Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
                                   AtomicOrdering Ord) const override;
     Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 659c1a9079c3..fd2084398c85 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -2023,6 +2023,8 @@ def SLBSYNC : XForm_0<31, 338, (outs), (ins), "slbsync", IIC_SprSLBSYNC, []>;
 
 } // IsISA3_0
 
+def : Pat<(int_ppc_ldarx ForceXForm:$ptr),
+          (LDARX ForceXForm:$ptr)>;
 def : Pat<(int_ppc_stdcx ForceXForm:$dst, g8rc:$A),
           (RLWINM (STDCX g8rc:$A, ForceXForm:$dst), 31, 31, 31)>;
 def : Pat<(PPCStoreCond ForceXForm:$dst, g8rc:$A, 8),
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td
index 71c6eac0860e..ef8b27f9b8d3 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrFutureMMA.td
@@ -481,7 +481,7 @@ let Predicates = [IsISAFuture] in {
                      RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
 
   def DMSHA3HASH :
-    XForm_ATp2_SR5<31, 15, 177, (outs dmrprc:$ATp), (ins dmrprc:$ATpi , u5imm:$SR),
+    XForm_ATp2_SR5<31, 15, 177, (outs dmrp:$ATp), (ins dmrp:$ATpi , u5imm:$SR),
                    "dmsha3hash $ATp, $SR",
                    [(set v2048i1:$ATp, (int_ppc_mma_dmsha3hash v2048i1:$ATpi, timm:$SR))]>,
                    RegConstraint<"$ATpi = $ATp">, NoEncode<"$ATpi">;
@@ -626,10 +626,10 @@ let Predicates = [IsISAFuture] in {
                   (DMSHA2HASH dmr:$AT, dmr:$AB, 1)>;
 
   def : InstAlias<"dmsha3dw $ATp",
-                  (DMSHA3HASH dmrprc:$ATp, 0)>;
+                  (DMSHA3HASH dmrp:$ATp, 0)>;
 
   def : InstAlias<"dmcryshash $ATp",
-                  (DMSHA3HASH dmrprc:$ATp, 12)>;
+                  (DMSHA3HASH dmrp:$ATp, 12)>;
 
   def : InstAlias<"dmxxsha3512pad $AT, $XB, $E",
                   (DMXXSHAPAD dmr:$AT, vsrc:$XB, 0, u1imm:$E, 0)>;
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 9d4d2d864fc3..57c86d9e5de6 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -1921,6 +1921,14 @@ unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
     OpcodeIndex = SOK_PairedVecSpill;
   } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_PairedG8Spill;
+  } else if (PPC::DMRROWRCRegClass.hasSubClassEq(RC)) {
+    llvm_unreachable("TODO: Implement spill DMRROW regclass!");
+  } else if (PPC::DMRROWpRCRegClass.hasSubClassEq(RC)) {
+    llvm_unreachable("TODO: Implement spill DMRROWp regclass!");
+  } else if (PPC::DMRpRCRegClass.hasSubClassEq(RC)) {
+    llvm_unreachable("TODO: Implement spill DMRp regclass!");
+  } else if (PPC::DMRRCRegClass.hasSubClassEq(RC)) {
+    OpcodeIndex = SOK_DMRSpill;
   } else {
     llvm_unreachable("Unknown regclass!");
   }
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index d0db90b029f6..a27b5718ec89 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -81,6 +81,7 @@ enum SpillOpcodeKey {
   SOK_AccumulatorSpill,
   SOK_UAccumulatorSpill,
   SOK_WAccumulatorSpill,
+  SOK_DMRSpill,
   SOK_SPESpill,
   SOK_PairedG8Spill,
   SOK_LastOpcodeSpill // This must be last on the enum.
@@ -117,6 +118,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::EVLDD,                                                                 \
    PPC::RESTORE_QUADWORD}
 
@@ -137,6 +139,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::RESTORE_QUADWORD}
 
 #define Pwr10LoadOpcodes                                                       \
@@ -156,6 +159,7 @@ enum PPCMachineCombinerPattern : unsigned {
    PPC::RESTORE_UACC,                                                          \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::RESTORE_QUADWORD}
 
 #define FutureLoadOpcodes                                                      \
@@ -174,6 +178,7 @@ enum PPCMachineCombinerPattern : unsigned {
    PPC::RESTORE_ACC,                                                           \
    PPC::RESTORE_UACC,                                                          \
    PPC::RESTORE_WACC,                                                          \
+   PPC::RESTORE_DMR,                                                           \
    NoInstr,                                                                    \
    PPC::RESTORE_QUADWORD}
 
@@ -193,6 +198,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::EVSTDD,                                                                \
    PPC::SPILL_QUADWORD}
 
@@ -213,6 +219,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::SPILL_QUADWORD}
 
 #define Pwr10StoreOpcodes                                                      \
@@ -232,6 +239,7 @@ enum PPCMachineCombinerPattern : unsigned {
    PPC::SPILL_UACC,                                                            \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::SPILL_QUADWORD}
 
 #define FutureStoreOpcodes                                                     \
@@ -250,6 +258,7 @@ enum PPCMachineCombinerPattern : unsigned {
    PPC::SPILL_ACC,                                                             \
    PPC::SPILL_UACC,                                                            \
    PPC::SPILL_WACC,                                                            \
+   PPC::SPILL_DMR,                                                             \
    NoInstr,                                                                    \
    PPC::SPILL_QUADWORD}
 
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index b70290df07b1..99ef89a7fdc0 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -5143,7 +5143,6 @@ def : Pat<(int_ppc_store2r gprc:$a, ForceXForm:$ptr),
 def : Pat<(int_ppc_store4r gprc:$a, ForceXForm:$ptr),
           (STWBRX gprc:$a, ForceXForm:$ptr)>;
 
-
 // Fast 32-bit reverse bits algorithm:
 // Step 1: 1-bit swap (swap odd 1-bit and even 1-bit):
 // n = ((n >> 1) & 0x55555555) | ((n << 1) & 0xAAAAAAAA);
@@ -5324,10 +5323,14 @@ def CFENCE : PPCPostRAExpPseudo<(outs), (ins gprc:$cr), "#CFENCE", []>;
 def : Pat<(i64 (bitreverse i64:$A)),
   (OR8 (RLDICR DWBytes7654.DWord, 32, 31), DWBytes3210.DWord)>;
 
+def : Pat<(int_ppc_lwarx ForceXForm:$ptr),
+          (LWARX ForceXForm:$ptr)>;
 def : Pat<(int_ppc_stwcx ForceXForm:$dst, gprc:$A),
           (RLWINM (STWCX gprc:$A, ForceXForm:$dst), 31, 31, 31)>;
 def : Pat<(PPCStoreCond ForceXForm:$dst, gprc:$A, 4),
           (RLWINM (STWCX gprc:$A, ForceXForm:$dst), 31, 31, 31)>;
+def : Pat<(int_ppc_lbarx ForceXForm:$ptr),
+          (LBARX ForceXForm:$ptr)>;
 def : Pat<(int_ppc_stbcx ForceXForm:$dst, gprc:$A),
           (RLWINM (STBCX gprc:$A, ForceXForm:$dst), 31, 31, 31)>;
 def : Pat<(PPCStoreCond ForceXForm:$dst, gprc:$A, 1),
@@ -5360,6 +5363,9 @@ def : Pat<(int_ppc_mtmsr gprc:$RS),
           (MTMSR $RS, 0)>;
 
 let Predicates = [IsISA2_07] in {
+  def : Pat<(int_ppc_lharx ForceXForm:$ptr),
+          (LHARX ForceXForm:$ptr)>;
+
   def : Pat<(int_ppc_sthcx ForceXForm:$dst, gprc:$A),
             (RLWINM (STHCX gprc:$A, ForceXForm:$dst), 31, 31, 31)>;
   def : Pat<(PPCStoreCond ForceXForm:$dst, gprc:$A, 2),
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrMMA.td b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrMMA.td
index 85a7c8e04c74..82e4a60e0a72 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrMMA.td
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrMMA.td
@@ -565,10 +565,14 @@ let Predicates = [MMA, IsISAFuture], isCodeGenOnly = 1 in {
   let mayStore = 1 in {
     def SPILL_WACC: PPCEmitTimePseudo<(outs), (ins wacc:$AT, memrix16:$dst),
                                       "#SPILL_WACC", []>;
+    def SPILL_DMR: PPCEmitTimePseudo<(outs), (ins dmr:$AT, memrix16:$dst),
+                                      "#SPILL_DMR", []>;
   }
   let mayLoad = 1, hasSideEffects = 0 in {
     def RESTORE_WACC: PPCEmitTimePseudo<(outs wacc:$AT), (ins memrix16:$src),
                                         "#RESTORE_WACC", []>;
+    def RESTORE_DMR: PPCEmitTimePseudo<(outs dmr:$AT), (ins memrix16:$src),
+                                        "#RESTORE_DMR", []>;
   }
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrP10.td b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrP10.td
index a7f758745efe..d295f35fb1dd 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrP10.td
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCInstrP10.td
@@ -2159,7 +2159,7 @@ let AddedComplexity = 400, Predicates = [IsISA3_1, HasVSX] in {
                                (COPY_TO_REGCLASS $VRB, VSRC), 2)))>;
 }
 
-class xxevalPattern <dag pattern, bits<8> imm> :
+class XXEvalPattern <dag pattern, bits<8> imm> :
   Pat<(v4i32 pattern), (XXEVAL $vA, $vB, $vC, imm)> {}
 
 let Predicates = [PrefixInstrs, HasP10Vector] in {
@@ -2192,83 +2192,83 @@ let Predicates = [PrefixInstrs, HasP10Vector] in {
     // Anonymous patterns for XXEVAL
     // AND
     // and(A, B, C)
-    def : xxevalPattern<(and v4i32:$vA, (and v4i32:$vB, v4i32:$vC)), 1>;
+    def : XXEvalPattern<(and v4i32:$vA, (and v4i32:$vB, v4i32:$vC)), 1>;
     // and(A, xor(B, C))
-    def : xxevalPattern<(and v4i32:$vA, (xor v4i32:$vB, v4i32:$vC)), 6>;
+    def : XXEvalPattern<(and v4i32:$vA, (xor v4i32:$vB, v4i32:$vC)), 6>;
     // and(A, or(B, C))
-    def : xxevalPattern<(and v4i32:$vA, (or v4i32:$vB, v4i32:$vC)), 7>;
+    def : XXEvalPattern<(and v4i32:$vA, (or v4i32:$vB, v4i32:$vC)), 7>;
     // and(A, nor(B, C))
-    def : xxevalPattern<(and v4i32:$vA, (vnot (or v4i32:$vB, v4i32:$vC))), 8>;
+    def : XXEvalPattern<(and v4i32:$vA, (vnot (or v4i32:$vB, v4i32:$vC))), 8>;
     // and(A, eqv(B, C))
-    def : xxevalPattern<(and v4i32:$vA, (vnot (xor v4i32:$vB, v4i32:$vC))), 9>;
+    def : XXEvalPattern<(and v4i32:$vA, (vnot (xor v4i32:$vB, v4i32:$vC))), 9>;
     // and(A, nand(B, C))
-    def : xxevalPattern<(and v4i32:$vA, (vnot (and v4i32:$vB, v4i32:$vC))), 14>;
+    def : XXEvalPattern<(and v4i32:$vA, (vnot (and v4i32:$vB, v4i32:$vC))), 14>;
 
     // NAND
     // nand(A, B, C)
-    def : xxevalPattern<(vnot (and v4i32:$vA, (and v4i32:$vB, v4i32:$vC))),
+    def : XXEvalPattern<(vnot (and v4i32:$vA, (and v4i32:$vB, v4i32:$vC))),
                          !sub(255, 1)>;
     // nand(A, xor(B, C))
-    def : xxevalPattern<(vnot (and v4i32:$vA, (xor v4i32:$vB, v4i32:$vC))),
+    def : XXEvalPattern<(vnot (and v4i32:$vA, (xor v4i32:$vB, v4i32:$vC))),
                          !sub(255, 6)>;
     // nand(A, or(B, C))
-    def : xxevalPattern<(vnot (and v4i32:$vA, (or v4i32:$vB, v4i32:$vC))),
+    def : XXEvalPattern<(vnot (and v4i32:$vA, (or v4i32:$vB, v4i32:$vC))),
                          !sub(255, 7)>;
     // nand(A, nor(B, C))
-    def : xxevalPattern<(or (vnot v4i32:$vA), (or v4i32:$vB, v4i32:$vC)),
+    def : XXEvalPattern<(or (vnot v4i32:$vA), (or v4i32:$vB, v4i32:$vC)),
                          !sub(255, 8)>;
     // nand(A, eqv(B, C))
-    def : xxevalPattern<(or (vnot v4i32:$vA), (xor v4i32:$vB, v4i32:$vC)),
+    def : XXEvalPattern<(or (vnot v4i32:$vA), (xor v4i32:$vB, v4i32:$vC)),
                          !sub(255, 9)>;
     // nand(A, nand(B, C))
-    def : xxevalPattern<(or (vnot v4i32:$vA), (and v4i32:$vB, v4i32:$vC)),
+    def : XXEvalPattern<(or (vnot v4i32:$vA), (and v4i32:$vB, v4i32:$vC)),
                          !sub(255, 14)>;
 
     // EQV
     // (eqv A, B, C)
-    def : xxevalPattern<(or (and v4i32:$vA, (and v4i32:$vB, v4i32:$vC)),
+    def : XXEvalPattern<(or (and v4i32:$vA, (and v4i32:$vB, v4i32:$vC)),
                             (vnot (or v4i32:$vA, (or v4i32:$vB, v4i32:$vC)))),
                          150>;
     // (eqv A, (and B, C))
-    def : xxevalPattern<(vnot (xor v4i32:$vA, (and v4i32:$vB, v4i32:$vC))), 225>;
+    def : XXEvalPattern<(vnot (xor v4i32:$vA, (and v4i32:$vB, v4i32:$vC))), 225>;
     // (eqv A, (or B, C))
-    def : xxevalPattern<(vnot (xor v4i32:$vA, (or v4i32:$vB, v4i32:$vC))), 135>;
+    def : XXEvalPattern<(vnot (xor v4i32:$vA, (or v4i32:$vB, v4i32:$vC))), 135>;
 
     // NOR
     // (nor A, B, C)
-    def : xxevalPattern<(vnot (or v4i32:$vA, (or v4i32:$vB, v4i32:$vC))), 128>;
+    def : XXEvalPattern<(vnot (or v4i32:$vA, (or v4i32:$vB, v4i32:$vC))), 128>;
     // (nor A, (and B, C))
-    def : xxevalPattern<(vnot (or v4i32:$vA, (and v4i32:$vB, v4i32:$vC))), 224>;
+    def : XXEvalPattern<(vnot (or v4i32:$vA, (and v4i32:$vB, v4i32:$vC))), 224>;
     // (nor A, (eqv B, C))
-    def : xxevalPattern<(and (vnot v4i32:$vA), (xor v4i32:$vB, v4i32:$vC)), 96>;
+    def : XXEvalPattern<(and (vnot v4i32:$vA), (xor v4i32:$vB, v4i32:$vC)), 96>;
     // (nor A, (nand B, C))
-    def : xxevalPattern<(and (vnot v4i32:$vA), (and v4i32:$vB, v4i32:$vC)), 16>;
+    def : XXEvalPattern<(and (vnot v4i32:$vA), (and v4i32:$vB, v4i32:$vC)), 16>;
     // (nor A, (nor B, C))
-    def : xxevalPattern<(and (vnot v4i32:$vA), (or v4i32:$vB, v4i32:$vC)), 112>;
+    def : XXEvalPattern<(and (vnot v4i32:$vA), (or v4i32:$vB, v4i32:$vC)), 112>;
     // (nor A, (xor B, C))
-    def : xxevalPattern<(vnot (or v4i32:$vA, (xor v4i32:$vB, v4i32:$vC))), 144>;
+    def : XXEvalPattern<(vnot (or v4i32:$vA, (xor v4i32:$vB, v4i32:$vC))), 144>;
 
     // OR
     // (or A, B, C)
-    def : xxevalPattern<(or v4i32:$vA, (or v4i32:$vB, v4i32:$vC)), 127>;
+    def : XXEvalPattern<(or v4i32:$vA, (or v4i32:$vB, v4i32:$vC)), 127>;
     // (or A, (and B, C))
-    def : xxevalPattern<(or v4i32:$vA, (and v4i32:$vB, v4i32:$vC)), 31>;
+    def : XXEvalPattern<(or v4i32:$vA, (and v4i32:$vB, v4i32:$vC)), 31>;
     // (or A, (eqv B, C))
-    def : xxevalPattern<(or v4i32:$vA, (vnot (xor v4i32:$vB, v4i32:$vC))), 159>;
+    def : XXEvalPattern<(or v4i32:$vA, (vnot (xor v4i32:$vB, v4i32:$vC))), 159>;
     // (or A, (nand B, C))
-    def : xxevalPattern<(or v4i32:$vA, (vnot (and v4i32:$vB, v4i32:$vC))), 239>;
+    def : XXEvalPattern<(or v4i32:$vA, (vnot (and v4i32:$vB, v4i32:$vC))), 239>;
     // (or A, (nor B, C))
-    def : xxevalPattern<(or v4i32:$vA, (vnot (or v4i32:$vB, v4i32:$vC))), 143>;
+    def : XXEvalPattern<(or v4i32:$vA, (vnot (or v4i32:$vB, v4i32:$vC))), 143>;
     // (or A, (xor B, C))
-    def : xxevalPattern<(or v4i32:$vA, (xor v4i32:$vB, v4i32:$vC)), 111>;
+    def : XXEvalPattern<(or v4i32:$vA, (xor v4i32:$vB, v4i32:$vC)), 111>;
 
     // XOR
     // (xor A, B, C)
-    def : xxevalPattern<(xor v4i32:$vA, (xor v4i32:$vB, v4i32:$vC)), 105>;
+    def : XXEvalPattern<(xor v4i32:$vA, (xor v4i32:$vB, v4i32:$vC)), 105>;
     // (xor A, (and B, C))
-    def : xxevalPattern<(xor v4i32:$vA, (and v4i32:$vB, v4i32:$vC)), 30>;
+    def : XXEvalPattern<(xor v4i32:$vA, (and v4i32:$vB, v4i32:$vC)), 30>;
     // (xor A, (or B, C))
-    def : xxevalPattern<(xor v4i32:$vA, (or v4i32:$vB, v4i32:$vC)), 120>;
+    def : XXEvalPattern<(xor v4i32:$vA, (or v4i32:$vB, v4i32:$vC)), 120>;
 
     // Anonymous patterns to select prefixed VSX loads and stores.
     // Load / Store f128
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
index 0a04b7fb8d16..cbd53651bbbf 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
@@ -54,31 +54,31 @@ static MCSymbol *GetSymbolFromOperand(const MachineOperand &MO,
 static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
                               AsmPrinter &Printer) {
   MCContext &Ctx = Printer.OutContext;
-  PPCMCExpr::Specifier RefKind = PPCMCExpr::VK_None;
+  PPCMCExpr::Specifier RefKind = PPC::S_None;
 
   unsigned access = MO.getTargetFlags();
 
   switch (access) {
     case PPCII::MO_TPREL_LO:
-      RefKind = PPCMCExpr::VK_TPREL_LO;
+      RefKind = PPC::S_TPREL_LO;
       break;
     case PPCII::MO_TPREL_HA:
-      RefKind = PPCMCExpr::VK_TPREL_HA;
+      RefKind = PPC::S_TPREL_HA;
       break;
     case PPCII::MO_DTPREL_LO:
-      RefKind = PPCMCExpr::VK_DTPREL_LO;
+      RefKind = PPC::S_DTPREL_LO;
       break;
     case PPCII::MO_TLSLD_LO:
-      RefKind = PPCMCExpr::VK_GOT_TLSLD_LO;
+      RefKind = PPC::S_GOT_TLSLD_LO;
       break;
     case PPCII::MO_TOC_LO:
-      RefKind = PPCMCExpr::VK_TOC_LO;
+      RefKind = PPC::S_TOC_LO;
       break;
     case PPCII::MO_TLS:
-      RefKind = PPCMCExpr::VK_TLS;
+      RefKind = PPC::S_TLS;
       break;
     case PPCII::MO_TLS_PCREL_FLAG:
-      RefKind = PPCMCExpr::VK_TLS_PCREL;
+      RefKind = PPC::S_TLS_PCREL;
       break;
   }
 
@@ -87,19 +87,19 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
   const MachineFunction *MF = MI->getMF();
 
   if (MO.getTargetFlags() == PPCII::MO_PLT)
-    RefKind = PPCMCExpr::VK_PLT;
+    RefKind = PPC::S_PLT;
   else if (MO.getTargetFlags() == PPCII::MO_PCREL_FLAG)
-    RefKind = PPCMCExpr::VK_PCREL;
+    RefKind = PPC::S_PCREL;
   else if (MO.getTargetFlags() == PPCII::MO_GOT_PCREL_FLAG)
-    RefKind = PPCMCExpr::VK_GOT_PCREL;
+    RefKind = PPC::S_GOT_PCREL;
   else if (MO.getTargetFlags() == PPCII::MO_TPREL_PCREL_FLAG)
-    RefKind = PPCMCExpr::VK_TPREL;
+    RefKind = PPC::S_TPREL;
   else if (MO.getTargetFlags() == PPCII::MO_GOT_TLSGD_PCREL_FLAG)
-    RefKind = PPCMCExpr::VK_GOT_TLSGD_PCREL;
+    RefKind = PPC::S_GOT_TLSGD_PCREL;
   else if (MO.getTargetFlags() == PPCII::MO_GOT_TLSLD_PCREL_FLAG)
-    RefKind = PPCMCExpr::VK_GOT_TLSLD_PCREL;
+    RefKind = PPC::S_GOT_TLSLD_PCREL;
   else if (MO.getTargetFlags() == PPCII::MO_GOT_TPREL_PCREL_FLAG)
-    RefKind = PPCMCExpr::VK_GOT_TPREL_PCREL;
+    RefKind = PPC::S_GOT_TPREL_PCREL;
   else if (MO.getTargetFlags() == PPCII::MO_TPREL_FLAG ||
            MO.getTargetFlags() == PPCII::MO_TLSLD_FLAG) {
     assert(MO.isGlobal() && "Only expecting a global MachineOperand here!");
@@ -110,14 +110,14 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
     // the relocation type in case the result is used for purposes other than a
     // TOC reference. In TOC reference cases, this result is discarded.
     if (Model == TLSModel::LocalExec)
-      RefKind = PPCMCExpr::VK_AIX_TLSLE;
+      RefKind = PPC::S_AIX_TLSLE;
     else if (Model == TLSModel::LocalDynamic &&
              FuncInfo->isAIXFuncUseTLSIEForLD())
       // On AIX, TLS model opt may have turned local-dynamic accesses into
       // initial-exec accesses.
-      RefKind = PPCMCExpr::VK_AIX_TLSIE;
+      RefKind = PPC::S_AIX_TLSIE;
     else if (Model == TLSModel::LocalDynamic)
-      RefKind = PPCMCExpr::VK_AIX_TLSLD;
+      RefKind = PPC::S_AIX_TLSLD;
   }
 
   const Module *M = MF->getFunction().getParent();
@@ -130,10 +130,10 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
     if (MIOpcode == PPC::TAILB || MIOpcode == PPC::TAILB8 ||
         MIOpcode == PPC::TCRETURNdi || MIOpcode == PPC::TCRETURNdi8 ||
         MIOpcode == PPC::BL8_NOTOC || MIOpcode == PPC::BL8_NOTOC_RM) {
-      RefKind = PPCMCExpr::VK_NOTOC;
+      RefKind = PPC::S_NOTOC;
     }
     if (MO.getTargetFlags() == PPCII::MO_PCREL_OPT_FLAG)
-      RefKind = PPCMCExpr::VK_PCREL_OPT;
+      RefKind = PPC::S_PCREL_OPT;
   }
 
   const MCExpr *Expr = MCSymbolRefExpr::create(
@@ -164,11 +164,11 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
   switch (access) {
     case PPCII::MO_LO:
     case PPCII::MO_PIC_LO_FLAG:
-      Expr = PPCMCExpr::createLo(Expr, Ctx);
+      Expr = MCSpecifierExpr::create(Expr, PPC::S_LO, Ctx);
       break;
     case PPCII::MO_HA:
     case PPCII::MO_PIC_HA_FLAG:
-      Expr = PPCMCExpr::createHa(Expr, Ctx);
+      Expr = MCSpecifierExpr::create(Expr, PPC::S_HA, Ctx);
       break;
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
index 1c53b740b3f8..18350650bfe2 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
@@ -42,6 +42,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/GlobalVariable.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/DebugCounter.h"
 
@@ -997,9 +998,9 @@ bool PPCMIPeephole::simplifyCode() {
           // the transformation.
           bool IsWordAligned = false;
           if (SrcMI->getOperand(1).isGlobal()) {
-            const GlobalObject *GO =
-                dyn_cast<GlobalObject>(SrcMI->getOperand(1).getGlobal());
-            if (GO && GO->getAlign() && *GO->getAlign() >= 4 &&
+            const GlobalVariable *GV =
+                dyn_cast<GlobalVariable>(SrcMI->getOperand(1).getGlobal());
+            if (GV && GV->getAlign() && *GV->getAlign() >= 4 &&
                 (SrcMI->getOperand(1).getOffset() % 4 == 0))
               IsWordAligned = true;
           } else if (SrcMI->getOperand(1).isImm()) {
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCMachineScheduler.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCMachineScheduler.cpp
index 03712879f7c4..5eb1f0128643 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCMachineScheduler.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCMachineScheduler.cpp
@@ -100,12 +100,11 @@ bool PPCPreRASchedStrategy::tryCandidate(SchedCandidate &Cand,
   // This is a best effort to set things up for a post-RA pass. Optimizations
   // like generating loads of multiple registers should ideally be done within
   // the scheduler pass by combining the loads during DAG postprocessing.
-  const SUnit *CandNextClusterSU =
-      Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  const SUnit *TryCandNextClusterSU =
-      TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
-  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
-                 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
+  const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
+  const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
+  if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
+                 CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
+                 Cluster))
     return TryCand.Reason != NoCand;
 
   if (SameBoundary) {
@@ -190,8 +189,11 @@ bool PPCPostRASchedStrategy::tryCandidate(SchedCandidate &Cand,
     return TryCand.Reason != NoCand;
 
   // Keep clustered nodes together.
-  if (tryGreater(TryCand.SU == DAG->getNextClusterSucc(),
-                 Cand.SU == DAG->getNextClusterSucc(), TryCand, Cand, Cluster))
+  const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
+  const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
+  if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
+                 CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
+                 Cluster))
     return TryCand.Reason != NoCand;
 
   // Avoid critical resource consumption and balance the schedule.
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 51902ad218d1..ea34c1aba82e 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -1238,40 +1238,28 @@ static void emitAccSpillRestoreInfo(MachineBasicBlock &MBB, bool IsPrimed,
 #endif
 }
 
-static void spillRegPairs(MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator II, DebugLoc DL,
-                          const TargetInstrInfo &TII, Register SrcReg,
-                          unsigned FrameIndex, bool IsLittleEndian,
-                          bool IsKilled, bool TwoPairs) {
-  unsigned Offset = 0;
-  // The register arithmetic in this function does not support virtual
-  // registers.
-  assert(!SrcReg.isVirtual() &&
+void PPCRegisterInfo::spillRegPair(MachineBasicBlock &MBB,
+                                   MachineBasicBlock::iterator II, DebugLoc DL,
+                                   const TargetInstrInfo &TII,
+                                   unsigned FrameIndex, bool IsLittleEndian,
+                                   bool IsKilled, Register Reg,
+                                   int Offset) const {
+
+  // This function does not support virtual registers.
+  assert(!Reg.isVirtual() &&
          "Spilling register pairs does not support virtual registers.");
 
-  if (TwoPairs)
-    Offset = IsLittleEndian ? 48 : 0;
-  else
-    Offset = IsLittleEndian ? 16 : 0;
-  Register Reg = (SrcReg > PPC::VSRp15) ? PPC::V0 + (SrcReg - PPC::VSRp16) * 2
-                                        : PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXV))
-                        .addReg(Reg, getKillRegState(IsKilled)),
-                    FrameIndex, Offset);
-  Offset += IsLittleEndian ? -16 : 16;
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXV))
-                        .addReg(Reg + 1, getKillRegState(IsKilled)),
-                    FrameIndex, Offset);
-  if (TwoPairs) {
-    Offset += IsLittleEndian ? -16 : 16;
-    addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXV))
-                          .addReg(Reg + 2, getKillRegState(IsKilled)),
-                      FrameIndex, Offset);
-    Offset += IsLittleEndian ? -16 : 16;
-    addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXV))
-                          .addReg(Reg + 3, getKillRegState(IsKilled)),
-                      FrameIndex, Offset);
-  }
+  addFrameReference(
+      BuildMI(MBB, II, DL, TII.get(PPC::STXV))
+          .addReg(TargetRegisterInfo::getSubReg(Reg, PPC::sub_vsx0),
+                  getKillRegState(IsKilled)),
+      FrameIndex, Offset);
+
+  addFrameReference(
+      BuildMI(MBB, II, DL, TII.get(PPC::STXV))
+          .addReg(TargetRegisterInfo::getSubReg(Reg, PPC::sub_vsx1),
+                  getKillRegState(IsKilled)),
+      FrameIndex, IsLittleEndian ? Offset - 16 : Offset + 16);
 }
 
 /// Remove any STXVP[X] instructions and split them out into a pair of
@@ -1290,8 +1278,10 @@ void PPCRegisterInfo::lowerOctWordSpilling(MachineBasicBlock::iterator II,
   Register SrcReg = MI.getOperand(0).getReg();
   bool IsLittleEndian = Subtarget.isLittleEndian();
   bool IsKilled = MI.getOperand(0).isKill();
-  spillRegPairs(MBB, II, DL, TII, SrcReg, FrameIndex, IsLittleEndian, IsKilled,
-                /* TwoPairs */ false);
+
+  spillRegPair(MBB, II, DL, TII, FrameIndex, IsLittleEndian, IsKilled, SrcReg,
+               IsLittleEndian ? 16 : 0);
+
   // Discard the original instruction.
   MBB.erase(II);
 }
@@ -1325,8 +1315,6 @@ void PPCRegisterInfo::lowerACCSpilling(MachineBasicBlock::iterator II,
   bool IsKilled = MI.getOperand(0).isKill();
 
   bool IsPrimed = PPC::ACCRCRegClass.contains(SrcReg);
-  Register Reg =
-      PPC::VSRp0 + (SrcReg - (IsPrimed ? PPC::ACC0 : PPC::UACC0)) * 2;
   bool IsLittleEndian = Subtarget.isLittleEndian();
 
   emitAccSpillRestoreInfo(MBB, IsPrimed, false);
@@ -1337,16 +1325,24 @@ void PPCRegisterInfo::lowerACCSpilling(MachineBasicBlock::iterator II,
   // adjust the offset of the store that is within the 64-byte stack slot.
   if (IsPrimed)
     BuildMI(MBB, II, DL, TII.get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
-  if (DisableAutoPairedVecSt)
-    spillRegPairs(MBB, II, DL, TII, Reg, FrameIndex, IsLittleEndian, IsKilled,
-                  /* TwoPairs */ true);
-  else {
-    addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
-                          .addReg(Reg, getKillRegState(IsKilled)),
-                      FrameIndex, IsLittleEndian ? 32 : 0);
-    addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
-                          .addReg(Reg + 1, getKillRegState(IsKilled)),
-                      FrameIndex, IsLittleEndian ? 0 : 32);
+  if (DisableAutoPairedVecSt) {
+    spillRegPair(MBB, II, DL, TII, FrameIndex, IsLittleEndian, IsKilled,
+                 TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_pair0),
+                 IsLittleEndian ? 48 : 0);
+    spillRegPair(MBB, II, DL, TII, FrameIndex, IsLittleEndian, IsKilled,
+                 TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_pair1),
+                 IsLittleEndian ? 16 : 32);
+  } else {
+    addFrameReference(
+        BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+            .addReg(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_pair0),
+                    getKillRegState(IsKilled)),
+        FrameIndex, IsLittleEndian ? 32 : 0);
+    addFrameReference(
+        BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+            .addReg(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_pair1),
+                    getKillRegState(IsKilled)),
+        FrameIndex, IsLittleEndian ? 0 : 32);
   }
   if (IsPrimed && !IsKilled)
     BuildMI(MBB, II, DL, TII.get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
@@ -1509,6 +1505,95 @@ void PPCRegisterInfo::lowerQuadwordRestore(MachineBasicBlock::iterator II,
   MBB.erase(II);
 }
 
+/// lowerDMRSpilling - Generate the code for spilling the DMR register.
+void PPCRegisterInfo::lowerDMRSpilling(MachineBasicBlock::iterator II,
+                                       unsigned FrameIndex) const {
+  MachineInstr &MI = *II; // SPILL_DMR <SrcReg>, <offset>
+  MachineBasicBlock &MBB = *MI.getParent();
+  MachineFunction &MF = *MBB.getParent();
+  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
+  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
+  DebugLoc DL = MI.getDebugLoc();
+  bool IsLittleEndian = Subtarget.isLittleEndian();
+
+  // DMR is made up of WACC and WACC_HI, so DMXXEXTFDMR512 to spill
+  // the corresponding 512 bits.
+  const TargetRegisterClass *RC = &PPC::VSRpRCRegClass;
+  Register SrcReg = MI.getOperand(0).getReg();
+
+  Register VSRpReg0 = MF.getRegInfo().createVirtualRegister(RC);
+  Register VSRpReg1 = MF.getRegInfo().createVirtualRegister(RC);
+  Register VSRpReg2 = MF.getRegInfo().createVirtualRegister(RC);
+  Register VSRpReg3 = MF.getRegInfo().createVirtualRegister(RC);
+
+  BuildMI(MBB, II, DL, TII.get(PPC::DMXXEXTFDMR512_HI), VSRpReg2)
+      .addDef(VSRpReg3)
+      .addReg(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_wacc_hi));
+
+  BuildMI(MBB, II, DL, TII.get(PPC::DMXXEXTFDMR512), VSRpReg0)
+      .addDef(VSRpReg1)
+      .addReg(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_wacc_lo));
+
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+                        .addReg(VSRpReg0, RegState::Kill),
+                    FrameIndex, IsLittleEndian ? 96 : 0);
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+                        .addReg(VSRpReg1, RegState::Kill),
+                    FrameIndex, IsLittleEndian ? 64 : 32);
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+                        .addReg(VSRpReg2, RegState::Kill),
+                    FrameIndex, IsLittleEndian ? 32 : 64);
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+                        .addReg(VSRpReg3, RegState::Kill),
+                    FrameIndex, IsLittleEndian ? 0 : 96);
+
+  // Discard the pseudo instruction.
+  MBB.erase(II);
+}
+
+/// lowerDMRRestore - Generate the code to restore the DMR register.
+void PPCRegisterInfo::lowerDMRRestore(MachineBasicBlock::iterator II,
+                                      unsigned FrameIndex) const {
+  MachineInstr &MI = *II; // <DestReg> = RESTORE_WACC <offset>
+  MachineBasicBlock &MBB = *MI.getParent();
+  MachineFunction &MF = *MBB.getParent();
+  const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
+  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
+  DebugLoc DL = MI.getDebugLoc();
+  bool IsLittleEndian = Subtarget.isLittleEndian();
+
+  const TargetRegisterClass *RC = &PPC::VSRpRCRegClass;
+  Register DestReg = MI.getOperand(0).getReg();
+
+  Register VSRpReg0 = MF.getRegInfo().createVirtualRegister(RC);
+  Register VSRpReg1 = MF.getRegInfo().createVirtualRegister(RC);
+  Register VSRpReg2 = MF.getRegInfo().createVirtualRegister(RC);
+  Register VSRpReg3 = MF.getRegInfo().createVirtualRegister(RC);
+
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg0),
+                    FrameIndex, IsLittleEndian ? 96 : 0);
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg1),
+                    FrameIndex, IsLittleEndian ? 64 : 32);
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg2),
+                    FrameIndex, IsLittleEndian ? 32 : 64);
+  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg3),
+                    FrameIndex, IsLittleEndian ? 0 : 96);
+
+  // Kill virtual registers (killedRegState::Killed).
+  BuildMI(MBB, II, DL, TII.get(PPC::DMXXINSTDMR512_HI),
+          TargetRegisterInfo::getSubReg(DestReg, PPC::sub_wacc_hi))
+      .addReg(VSRpReg2, RegState::Kill)
+      .addReg(VSRpReg3, RegState::Kill);
+
+  BuildMI(MBB, II, DL, TII.get(PPC::DMXXINSTDMR512),
+          TargetRegisterInfo::getSubReg(DestReg, PPC::sub_wacc_lo))
+      .addReg(VSRpReg0, RegState::Kill)
+      .addReg(VSRpReg1, RegState::Kill);
+
+  // Discard the pseudo instruction.
+  MBB.erase(II);
+}
+
 bool PPCRegisterInfo::hasReservedSpillSlot(const MachineFunction &MF,
                                            Register Reg, int &FrameIdx) const {
   // For the nonvolatile condition registers (CR2, CR3, CR4) return true to
@@ -1671,6 +1756,12 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   case PPC::RESTORE_WACC:
     lowerWACCRestore(II, FrameIndex);
     return true;
+  case PPC::SPILL_DMR:
+    lowerDMRSpilling(II, FrameIndex);
+    return true;
+  case PPC::RESTORE_DMR:
+    lowerDMRRestore(II, FrameIndex);
+    return true;
   case PPC::SPILL_QUADWORD:
     lowerQuadwordSpilling(II, FrameIndex);
     return true;
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index 005d890c57c9..849f856b5419 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -58,6 +58,11 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
   DenseMap<unsigned, unsigned> ImmToIdxMap;
   const PPCTargetMachine &TM;
 
+  void spillRegPair(MachineBasicBlock &MBB, MachineBasicBlock::iterator II,
+                    DebugLoc DL, const TargetInstrInfo &TII,
+                    unsigned FrameIndex, bool IsLittleEndian, bool IsKilled,
+                    Register Reg, int Offset) const;
+
 public:
   PPCRegisterInfo(const PPCTargetMachine &TM);
 
@@ -151,6 +156,11 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
   void lowerQuadwordRestore(MachineBasicBlock::iterator II,
                             unsigned FrameIndex) const;
 
+  void lowerDMRSpilling(MachineBasicBlock::iterator II,
+                        unsigned FrameIndex) const;
+  void lowerDMRRestore(MachineBasicBlock::iterator II,
+                       unsigned FrameIndex) const;
+
   static void emitAccCopyInfo(MachineBasicBlock &MBB, MCRegister DestReg,
                               MCRegister SrcReg);
 
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.td b/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
index 359adc31eb10..8b690b7b833b 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -1140,6 +1140,6 @@ def PPCRegDMRpRCAsmOperand : AsmOperandClass {
   let PredicateMethod = "isDMRpRegNumber";
 }
 
-def dmrprc : RegisterOperand<DMRpRC> {
+def dmrp : RegisterOperand<DMRpRC> {
   let ParserMatchClass = PPCRegDMRpRCAsmOperand;
 }
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index a11e49c486a3..359a43dd001d 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -310,12 +310,10 @@ getEffectivePPCCodeModel(const Triple &TT, std::optional<CodeModel::Model> CM,
 
 static ScheduleDAGInstrs *createPPCMachineScheduler(MachineSchedContext *C) {
   const PPCSubtarget &ST = C->MF->getSubtarget<PPCSubtarget>();
-  ScheduleDAGMILive *DAG =
-    new ScheduleDAGMILive(C, ST.usePPCPreRASchedStrategy() ?
-                          std::make_unique<PPCPreRASchedStrategy>(C) :
-                          std::make_unique<GenericScheduler>(C));
+  ScheduleDAGMILive *DAG = ST.usePPCPreRASchedStrategy()
+                               ? createSchedLive<PPCPreRASchedStrategy>(C)
+                               : createSchedLive<GenericScheduler>(C);
   // add DAG Mutations here.
-  DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII, DAG->TRI));
   if (ST.hasStoreFusion())
     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
   if (ST.hasFusion())
@@ -324,13 +322,12 @@ static ScheduleDAGInstrs *createPPCMachineScheduler(MachineSchedContext *C) {
   return DAG;
 }
 
-static ScheduleDAGInstrs *createPPCPostMachineScheduler(
-  MachineSchedContext *C) {
+static ScheduleDAGInstrs *
+createPPCPostMachineScheduler(MachineSchedContext *C) {
   const PPCSubtarget &ST = C->MF->getSubtarget<PPCSubtarget>();
-  ScheduleDAGMI *DAG =
-    new ScheduleDAGMI(C, ST.usePPCPostRASchedStrategy() ?
-                      std::make_unique<PPCPostRASchedStrategy>(C) :
-                      std::make_unique<PostGenericScheduler>(C), true);
+  ScheduleDAGMI *DAG = ST.usePPCPostRASchedStrategy()
+                           ? createSchedPostRA<PPCPostRASchedStrategy>(C)
+                           : createSchedPostRA<PostGenericScheduler>(C);
   // add DAG Mutations here.
   if (ST.hasStoreFusion())
     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
@@ -562,7 +559,8 @@ void PPCPassConfig::addMachineSSAOptimization() {
 
 void PPCPassConfig::addPreRegAlloc() {
   if (getOptLevel() != CodeGenOptLevel::None) {
-    insertPass(VSXFMAMutateEarly ? &RegisterCoalescerID : &MachineSchedulerID,
+    insertPass(VSXFMAMutateEarly ? &TwoAddressInstructionPassID
+                                 : &MachineSchedulerID,
                &PPCVSXFMAMutateID);
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetObjectFile.cpp b/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetObjectFile.cpp
index 078f4b1effbb..29e4286cf4ad 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetObjectFile.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "PPCTargetObjectFile.h"
-#include "MCTargetDesc/PPCMCExpr.h"
+#include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -49,8 +49,8 @@ MCSection *PPC64LinuxTargetObjectFile::SelectSectionForGlobal(
 
 const MCExpr *PPC64LinuxTargetObjectFile::
 getDebugThreadLocalSymbol(const MCSymbol *Sym) const {
-  const MCExpr *Expr = MCSymbolRefExpr::create(
-      Sym, MCSymbolRefExpr::VariantKind(PPCMCExpr::VK_DTPREL), getContext());
+  const MCExpr *Expr =
+      MCSymbolRefExpr::create(Sym, PPC::S_DTPREL, getContext());
   return MCBinaryExpr::createAdd(Expr,
                                  MCConstantExpr::create(0x8000, getContext()),
                                  getContext());
diff --git a/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 361b2ff223ea..8618f3064c18 100644
--- a/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -24,7 +24,7 @@
 
 namespace llvm {
 
-class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
+class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
   typedef BasicTTIImplBase<PPCTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/external/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 1f434beca538..510ca5f8c0d9 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -9,7 +9,7 @@
 #include "MCTargetDesc/RISCVAsmBackend.h"
 #include "MCTargetDesc/RISCVBaseInfo.h"
 #include "MCTargetDesc/RISCVInstPrinter.h"
-#include "MCTargetDesc/RISCVMCExpr.h"
+#include "MCTargetDesc/RISCVMCAsmInfo.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "MCTargetDesc/RISCVMatInt.h"
 #include "MCTargetDesc/RISCVTargetStreamer.h"
@@ -137,7 +137,7 @@ class RISCVAsmParser : public MCTargetAsmParser {
   // Helper to emit a combination of AUIPC and SecondOpcode. Used to implement
   // helpers such as emitLoadLocalAddress and emitLoadAddress.
   void emitAuipcInstPair(MCRegister DestReg, MCRegister TmpReg,
-                         const MCExpr *Symbol, RISCVMCExpr::Specifier VKHi,
+                         const MCExpr *Symbol, RISCV::Specifier VKHi,
                          unsigned SecondOpcode, SMLoc IDLoc, MCStreamer &Out);
 
   // Helper to emit pseudo instruction "lla" used in PC-rel addressing.
@@ -295,8 +295,7 @@ class RISCVAsmParser : public MCTargetAsmParser {
 #undef GET_OPERAND_DIAGNOSTIC_TYPES
   };
 
-  static bool classifySymbolRef(const MCExpr *Expr,
-                                RISCVMCExpr::Specifier &Kind);
+  static bool classifySymbolRef(const MCExpr *Expr, RISCV::Specifier &Kind);
   static bool isSymbolDiff(const MCExpr *Expr);
 
   RISCVAsmParser(const MCSubtargetInfo &STI, MCAsmParser &Parser,
@@ -544,9 +543,9 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (evaluateConstantImm(getImm(), Imm))
       return isShiftedInt<N - 1, 1>(fixImmediateForRV32(Imm, isRV64Imm()));
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
-           VK == RISCVMCExpr::VK_None;
+           VK == RISCV::S_None;
   }
 
   // True if operand is a symbol with no modifiers, or a constant with no
@@ -559,9 +558,9 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (evaluateConstantImm(getImm(), Imm))
       return isInt<N>(fixImmediateForRV32(Imm, isRV64Imm()));
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
-           VK == RISCVMCExpr::VK_None;
+           VK == RISCV::S_None;
   }
 
   // Predicate methods for AsmOperands defined in RISCVInstrInfo.td
@@ -572,9 +571,9 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (!isImm() || evaluateConstantImm(getImm(), Imm))
       return false;
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
-           VK == RISCVMCExpr::VK_None;
+           VK == RISCV::S_None;
   }
 
   bool isCallSymbol() const {
@@ -583,7 +582,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (!isImm() || evaluateConstantImm(getImm(), Imm))
       return false;
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
            VK == ELF::R_RISCV_CALL_PLT;
   }
@@ -594,7 +593,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (!isImm() || evaluateConstantImm(getImm(), Imm))
       return false;
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
            VK == ELF::R_RISCV_CALL_PLT;
   }
@@ -605,7 +604,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (!isImm() || evaluateConstantImm(getImm(), Imm))
       return false;
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
            VK == ELF::R_RISCV_TPREL_ADD;
   }
@@ -616,7 +615,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (!isImm() || evaluateConstantImm(getImm(), Imm))
       return false;
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
            VK == ELF::R_RISCV_TLSDESC_CALL;
   }
@@ -870,11 +869,10 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (evaluateConstantImm(getImm(), Imm))
       return isInt<12>(fixImmediateForRV32(Imm, isRV64Imm()));
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
-           (VK == RISCVMCExpr::VK_LO || VK == RISCVMCExpr::VK_PCREL_LO ||
-            VK == RISCVMCExpr::VK_TPREL_LO ||
-            VK == ELF::R_RISCV_TLSDESC_LOAD_LO12 ||
+           (VK == RISCV::S_LO || VK == RISCV::S_PCREL_LO ||
+            VK == RISCV::S_TPREL_LO || VK == ELF::R_RISCV_TLSDESC_LOAD_LO12 ||
             VK == ELF::R_RISCV_TLSDESC_ADD_LO12);
   }
 
@@ -903,9 +901,9 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (evaluateConstantImm(getImm(), Imm))
       return isInt<20>(fixImmediateForRV32(Imm, isRV64Imm()));
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
-           VK == RISCVMCExpr::VK_QC_ABS20;
+           VK == RISCV::S_QC_ABS20;
   }
 
   bool isUImm20LUI() const {
@@ -916,7 +914,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (evaluateConstantImm(getImm(), Imm))
       return isUInt<20>(Imm);
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
            (VK == ELF::R_RISCV_HI20 || VK == ELF::R_RISCV_TPREL_HI20);
   }
@@ -929,7 +927,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     if (evaluateConstantImm(getImm(), Imm))
       return isUInt<20>(Imm);
 
-    RISCVMCExpr::Specifier VK = RISCVMCExpr::VK_None;
+    RISCV::Specifier VK = RISCV::S_None;
     return RISCVAsmParser::classifySymbolRef(getImm(), VK) &&
            (VK == ELF::R_RISCV_PCREL_HI20 || VK == ELF::R_RISCV_GOT_HI20 ||
             VK == ELF::R_RISCV_TLS_GOT_HI20 || VK == ELF::R_RISCV_TLS_GD_HI20 ||
@@ -2088,7 +2086,7 @@ bool RISCVAsmParser::parseExprWithSpecifier(const MCExpr *&Res, SMLoc &E) {
   if (getLexer().getKind() != AsmToken::Identifier)
     return Error(getLoc(), "expected '%' relocation specifier");
   StringRef Identifier = getParser().getTok().getIdentifier();
-  auto Spec = RISCVMCExpr::getSpecifierForName(Identifier);
+  auto Spec = RISCV::parseSpecifierName(Identifier);
   if (!Spec)
     return Error(getLoc(), "invalid relocation specifier");
 
@@ -2100,7 +2098,7 @@ bool RISCVAsmParser::parseExprWithSpecifier(const MCExpr *&Res, SMLoc &E) {
   if (getParser().parseParenExpression(SubExpr, E))
     return true;
 
-  Res = RISCVMCExpr::create(SubExpr, *Spec, getContext());
+  Res = MCSpecifierExpr::create(SubExpr, Spec, getContext());
   return false;
 }
 
@@ -2184,11 +2182,11 @@ ParseStatus RISCVAsmParser::parseCallSymbol(OperandVector &Operands) {
   }
 
   SMLoc E = SMLoc::getFromPointer(S.getPointer() + Identifier.size());
-  RISCVMCExpr::Specifier Kind = ELF::R_RISCV_CALL_PLT;
+  RISCV::Specifier Kind = ELF::R_RISCV_CALL_PLT;
 
   MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
   Res = MCSymbolRefExpr::create(Sym, getContext());
-  Res = RISCVMCExpr::create(Res, Kind, getContext());
+  Res = MCSpecifierExpr::create(Res, Kind, getContext());
   Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64()));
   return ParseStatus::Success;
 }
@@ -2204,7 +2202,7 @@ ParseStatus RISCVAsmParser::parsePseudoJumpSymbol(OperandVector &Operands) {
   if (Res->getKind() != MCExpr::ExprKind::SymbolRef)
     return Error(S, "operand must be a valid jump target");
 
-  Res = RISCVMCExpr::create(Res, ELF::R_RISCV_CALL_PLT, getContext());
+  Res = MCSpecifierExpr::create(Res, ELF::R_RISCV_CALL_PLT, getContext());
   Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64()));
   return ParseStatus::Success;
 }
@@ -2919,24 +2917,23 @@ bool RISCVAsmParser::parseInstruction(ParseInstructionInfo &Info,
 }
 
 bool RISCVAsmParser::classifySymbolRef(const MCExpr *Expr,
-                                       RISCVMCExpr::Specifier &Kind) {
-  Kind = RISCVMCExpr::VK_None;
-
-  if (const RISCVMCExpr *RE = dyn_cast<RISCVMCExpr>(Expr)) {
+                                       RISCV::Specifier &Kind) {
+  Kind = RISCV::S_None;
+  if (const auto *RE = dyn_cast<MCSpecifierExpr>(Expr)) {
     Kind = RE->getSpecifier();
     Expr = RE->getSubExpr();
   }
 
   MCValue Res;
   if (Expr->evaluateAsRelocatable(Res, nullptr))
-    return Res.getSpecifier() == RISCVMCExpr::VK_None;
+    return Res.getSpecifier() == RISCV::S_None;
   return false;
 }
 
 bool RISCVAsmParser::isSymbolDiff(const MCExpr *Expr) {
   MCValue Res;
   if (Expr->evaluateAsRelocatable(Res, nullptr)) {
-    return Res.getSpecifier() == RISCVMCExpr::VK_None && Res.getAddSym() &&
+    return Res.getSpecifier() == RISCV::S_None && Res.getAddSym() &&
            Res.getSubSym();
   }
   return false;
@@ -3435,7 +3432,7 @@ void RISCVAsmParser::emitLoadImm(MCRegister DestReg, int64_t Value,
 
 void RISCVAsmParser::emitAuipcInstPair(MCRegister DestReg, MCRegister TmpReg,
                                        const MCExpr *Symbol,
-                                       RISCVMCExpr::Specifier VKHi,
+                                       RISCV::Specifier VKHi,
                                        unsigned SecondOpcode, SMLoc IDLoc,
                                        MCStreamer &Out) {
   // A pair of instructions for PC-relative addressing; expands to
@@ -3446,12 +3443,12 @@ void RISCVAsmParser::emitAuipcInstPair(MCRegister DestReg, MCRegister TmpReg,
   MCSymbol *TmpLabel = Ctx.createNamedTempSymbol("pcrel_hi");
   Out.emitLabel(TmpLabel);
 
-  const RISCVMCExpr *SymbolHi = RISCVMCExpr::create(Symbol, VKHi, Ctx);
+  const auto *SymbolHi = MCSpecifierExpr::create(Symbol, VKHi, Ctx);
   emitToStreamer(Out,
                  MCInstBuilder(RISCV::AUIPC).addReg(TmpReg).addExpr(SymbolHi));
 
-  const MCExpr *RefToLinkTmpLabel = RISCVMCExpr::create(
-      MCSymbolRefExpr::create(TmpLabel, Ctx), RISCVMCExpr::VK_PCREL_LO, Ctx);
+  const MCExpr *RefToLinkTmpLabel = MCSpecifierExpr::create(
+      MCSymbolRefExpr::create(TmpLabel, Ctx), RISCV::S_PCREL_LO, Ctx);
 
   emitToStreamer(Out, MCInstBuilder(SecondOpcode)
                           .addReg(DestReg)
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 963501db118e..2f37c351baf9 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "RISCVAsmBackend.h"
-#include "RISCVMCExpr.h"
+#include "RISCVFixupKinds.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
@@ -26,8 +26,6 @@
 
 using namespace llvm;
 
-static cl::opt<bool> RelaxBranches("riscv-asm-relax-branches", cl::init(true),
-                                   cl::Hidden);
 // Temporary workaround for old linkers that do not support ULEB128 relocations,
 // which are abused by DWARF v5 DW_LLE_offset_pair/DW_RLE_offset_pair
 // implemented in Clang/LLVM.
@@ -109,9 +107,6 @@ bool RISCVAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
                                                    const MCValue &,
                                                    uint64_t Value,
                                                    bool Resolved) const {
-  if (!RelaxBranches)
-    return false;
-
   int64_t Offset = int64_t(Value);
   unsigned Kind = Fixup.getTargetKind();
 
@@ -133,24 +128,44 @@ bool RISCVAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
   case RISCV::fixup_riscv_branch:
   case RISCV::fixup_riscv_qc_e_branch:
     // For conditional branch instructions the immediate must be
-    // in the range [-4096, 4095].
-    return !isInt<13>(Offset);
+    // in the range [-4096, 4094].
+    return Offset > 4094 || Offset < -4096;
+  case RISCV::fixup_riscv_jal:
+    // For jump instructions the immediate must be in the range
+    // [-1048576, 1048574]
+    return Offset > 1048574 || Offset < -1048576;
   }
 }
 
 // Given a compressed control flow instruction this function returns
-// the expanded instruction.
-static unsigned getRelaxedOpcode(unsigned Op) {
-  switch (Op) {
-  default:
-    return Op;
+// the expanded instruction, or the original instruction code if no
+// expansion is available.
+static unsigned getRelaxedOpcode(const MCInst &Inst,
+                                 const MCSubtargetInfo &STI) {
+  switch (Inst.getOpcode()) {
   case RISCV::C_BEQZ:
     return RISCV::BEQ;
   case RISCV::C_BNEZ:
     return RISCV::BNE;
   case RISCV::C_J:
   case RISCV::C_JAL: // fall through.
+    // This only relaxes one "step" - i.e. from C.J to JAL, not from C.J to
+    // QC.E.J, because we can always relax again if needed.
     return RISCV::JAL;
+  case RISCV::JAL: {
+    // We can only relax JAL if we have Xqcilb
+    if (!STI.hasFeature(RISCV::FeatureVendorXqcilb))
+      break;
+
+    // And only if it is using X0 or X1 for rd.
+    MCRegister Reg = Inst.getOperand(0).getReg();
+    if (Reg == RISCV::X0)
+      return RISCV::QC_E_J;
+    if (Reg == RISCV::X1)
+      return RISCV::QC_E_JAL;
+
+    break;
+  }
   case RISCV::BEQ:
     return RISCV::PseudoLongBEQ;
   case RISCV::BNE:
@@ -188,6 +203,9 @@ static unsigned getRelaxedOpcode(unsigned Op) {
   case RISCV::QC_E_BGEUI:
     return RISCV::PseudoLongQC_E_BGEUI;
   }
+
+  // Returning the original opcode means we cannot relax the instruction.
+  return Inst.getOpcode();
 }
 
 void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
@@ -205,6 +223,20 @@ void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
   case RISCV::C_JAL: {
     [[maybe_unused]] bool Success = RISCVRVC::uncompress(Res, Inst, STI);
     assert(Success && "Can't uncompress instruction");
+    assert(Res.getOpcode() == getRelaxedOpcode(Inst, STI) &&
+           "Branch Relaxation Error");
+    break;
+  }
+  case RISCV::JAL: {
+    // This has to be written manually because the QC.E.J -> JAL is
+    // compression-only, so that it is not used when printing disassembly.
+    assert(STI.hasFeature(RISCV::FeatureVendorXqcilb) &&
+           "JAL is only relaxable with Xqcilb");
+    assert((Inst.getOperand(0).getReg() == RISCV::X0 ||
+            Inst.getOperand(0).getReg() == RISCV::X1) &&
+           "JAL only relaxable with rd=x0 or rd=x1");
+    Res.setOpcode(getRelaxedOpcode(Inst, STI));
+    Res.addOperand(Inst.getOperand(1));
     break;
   }
   case RISCV::BEQ:
@@ -225,7 +257,7 @@ void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
   case RISCV::QC_E_BGEI:
   case RISCV::QC_E_BLTUI:
   case RISCV::QC_E_BGEUI:
-    Res.setOpcode(getRelaxedOpcode(Inst.getOpcode()));
+    Res.setOpcode(getRelaxedOpcode(Inst, STI));
     Res.addOperand(Inst.getOperand(0));
     Res.addOperand(Inst.getOperand(1));
     Res.addOperand(Inst.getOperand(2));
@@ -327,14 +359,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
   auto AddFixups = [&Fixups, &AddrDelta](unsigned Offset,
                                          std::pair<unsigned, unsigned> Fixup) {
     const MCBinaryExpr &MBE = cast<MCBinaryExpr>(AddrDelta);
-    Fixups.push_back(
-        MCFixup::create(Offset, MBE.getLHS(),
-                        static_cast<MCFixupKind>(FirstLiteralRelocationKind +
-                                                 std::get<0>(Fixup))));
-    Fixups.push_back(
-        MCFixup::create(Offset, MBE.getRHS(),
-                        static_cast<MCFixupKind>(FirstLiteralRelocationKind +
-                                                 std::get<1>(Fixup))));
+    Fixups.push_back(MCFixup::create(Offset, MBE.getLHS(), std::get<0>(Fixup)));
+    Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(Fixup)));
   };
 
   if (isUIntN(6, Value)) {
@@ -380,7 +406,7 @@ bool RISCVAsmBackend::mayNeedRelaxation(const MCInst &Inst,
   if (STI.hasFeature(RISCV::FeatureExactAssembly))
     return false;
 
-  return getRelaxedOpcode(Inst.getOpcode()) != Inst.getOpcode();
+  return getRelaxedOpcode(Inst, STI) != Inst.getOpcode();
 }
 
 bool RISCVAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
@@ -564,6 +590,57 @@ bool RISCVAsmBackend::isPCRelFixupResolved(const MCSymbol *SymA,
   return !Res.getSubSym();
 }
 
+// Get the corresponding PC-relative HI fixup that a S_PCREL_LO points to, and
+// optionally the fragment containing it.
+//
+// \returns nullptr if this isn't a S_PCREL_LO pointing to a known PC-relative
+// HI fixup.
+static const MCFixup *getPCRelHiFixup(const MCSpecifierExpr &Expr,
+                                      const MCFragment **DFOut) {
+  MCValue AUIPCLoc;
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(AUIPCLoc, nullptr))
+    return nullptr;
+
+  const MCSymbol *AUIPCSymbol = AUIPCLoc.getAddSym();
+  if (!AUIPCSymbol)
+    return nullptr;
+  const auto *DF = dyn_cast_or_null<MCDataFragment>(AUIPCSymbol->getFragment());
+
+  if (!DF)
+    return nullptr;
+
+  uint64_t Offset = AUIPCSymbol->getOffset();
+  if (DF->getContents().size() == Offset) {
+    DF = dyn_cast_or_null<MCDataFragment>(DF->getNext());
+    if (!DF)
+      return nullptr;
+    Offset = 0;
+  }
+
+  for (const MCFixup &F : DF->getFixups()) {
+    if (F.getOffset() != Offset)
+      continue;
+    auto Kind = F.getTargetKind();
+    if (!mc::isRelocation(F.getKind())) {
+      if (Kind == RISCV::fixup_riscv_pcrel_hi20) {
+        *DFOut = DF;
+        return &F;
+      }
+      break;
+    }
+    switch (Kind) {
+    case ELF::R_RISCV_GOT_HI20:
+    case ELF::R_RISCV_TLS_GOT_HI20:
+    case ELF::R_RISCV_TLS_GD_HI20:
+    case ELF::R_RISCV_TLSDESC_HI20:
+      *DFOut = DF;
+      return &F;
+    }
+  }
+
+  return nullptr;
+}
+
 bool RISCVAsmBackend::evaluateTargetFixup(const MCFixup &Fixup,
                                           const MCValue &Target,
                                           uint64_t &Value) {
@@ -575,7 +652,8 @@ bool RISCVAsmBackend::evaluateTargetFixup(const MCFixup &Fixup,
     llvm_unreachable("Unexpected fixup kind!");
   case RISCV::fixup_riscv_pcrel_lo12_i:
   case RISCV::fixup_riscv_pcrel_lo12_s: {
-    AUIPCFixup = cast<RISCVMCExpr>(Fixup.getValue())->getPCRelHiFixup(&AUIPCDF);
+    AUIPCFixup =
+        getPCRelHiFixup(cast<MCSpecifierExpr>(*Fixup.getValue()), &AUIPCDF);
     if (!AUIPCFixup) {
       getContext().reportError(Fixup.getLoc(),
                                "could not find corresponding %pcrel_hi");
@@ -611,6 +689,46 @@ bool RISCVAsmBackend::evaluateTargetFixup(const MCFixup &Fixup,
          isPCRelFixupResolved(AUIPCTarget.getAddSym(), *AUIPCDF);
 }
 
+void RISCVAsmBackend::maybeAddVendorReloc(const MCFragment &F,
+                                          const MCFixup &Fixup) {
+  StringRef VendorIdentifier;
+  switch (Fixup.getTargetKind()) {
+  default:
+    // No Vendor Relocation Required.
+    return;
+  case RISCV::fixup_riscv_qc_e_branch:
+  case RISCV::fixup_riscv_qc_abs20_u:
+  case RISCV::fixup_riscv_qc_e_32:
+  case RISCV::fixup_riscv_qc_e_jump_plt:
+    VendorIdentifier = "QUALCOMM";
+    break;
+  }
+
+  // Create a local symbol for the vendor relocation to reference. It's fine if
+  // the symbol has the same name as an existing symbol.
+  MCContext &Ctx = Asm->getContext();
+  MCSymbol *VendorSymbol = Ctx.createLocalSymbol(VendorIdentifier);
+  auto [It, Inserted] =
+      VendorSymbols.try_emplace(VendorIdentifier, VendorSymbol);
+
+  if (Inserted) {
+    // Setup the just-created symbol
+    VendorSymbol->setVariableValue(MCConstantExpr::create(0, Ctx));
+    Asm->registerSymbol(*VendorSymbol);
+  } else {
+    // Fetch the existing symbol
+    VendorSymbol = It->getValue();
+  }
+
+  MCFixup VendorFixup =
+      MCFixup::create(Fixup.getOffset(), nullptr, ELF::R_RISCV_VENDOR);
+  // Explicitly create MCValue rather than using an MCExpr and evaluating it so
+  // that the absolute vendor symbol is not evaluated to constant 0.
+  MCValue VendorTarget = MCValue::get(VendorSymbol);
+  uint64_t VendorValue;
+  Asm->getWriter().recordRelocation(F, VendorFixup, VendorTarget, VendorValue);
+}
+
 bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
                                const MCValue &Target, uint64_t &FixedValue,
                                bool IsResolved) {
@@ -660,7 +778,14 @@ bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
   if (IsResolved &&
       (getFixupKindInfo(Fixup.getKind()).Flags & MCFixupKindInfo::FKF_IsPCRel))
     IsResolved = isPCRelFixupResolved(Target.getAddSym(), F);
-  IsResolved = MCAsmBackend::addReloc(F, Fixup, Target, FixedValue, IsResolved);
+
+  if (!IsResolved) {
+    // Some Fixups require a vendor relocation, record it (directly) before we
+    // add the relocation.
+    maybeAddVendorReloc(F, Fixup);
+
+    Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue);
+  }
 
   if (Fixup.isLinkerRelaxable()) {
     auto FA = MCFixup::create(Fixup.getOffset(), nullptr, ELF::R_RISCV_RELAX);
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index 4dffbb82e07c..91efd4454750 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -12,6 +12,7 @@
 #include "MCTargetDesc/RISCVBaseInfo.h"
 #include "MCTargetDesc/RISCVFixupKinds.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -31,6 +32,8 @@ class RISCVAsmBackend : public MCAsmBackend {
 
   bool isPCRelFixupResolved(const MCSymbol *SymA, const MCFragment &F);
 
+  StringMap<MCSymbol *> VendorSymbols;
+
 public:
   RISCVAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI, bool Is64Bit,
                   const MCTargetOptions &Options);
@@ -50,6 +53,8 @@ class RISCVAsmBackend : public MCAsmBackend {
   bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
                 uint64_t &FixedValue, bool IsResolved) override;
 
+  void maybeAddVendorReloc(const MCFragment &, const MCFixup &);
+
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
index 378554adc803..1d81096d6b60 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/RISCVFixupKinds.h"
-#include "MCTargetDesc/RISCVMCExpr.h"
+#include "MCTargetDesc/RISCVMCAsmInfo.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
@@ -48,9 +48,8 @@ RISCVELFObjectWriter::~RISCVELFObjectWriter() = default;
 unsigned RISCVELFObjectWriter::getRelocType(const MCFixup &Fixup,
                                             const MCValue &Target,
                                             bool IsPCRel) const {
-  const MCExpr *Expr = Fixup.getValue();
   unsigned Kind = Fixup.getTargetKind();
-  auto Spec = RISCVMCExpr::Specifier(Target.getSpecifier());
+  auto Spec = Target.getSpecifier();
   switch (Spec) {
   case ELF::R_RISCV_TPREL_HI20:
   case ELF::R_RISCV_TLS_GOT_HI20:
@@ -63,7 +62,7 @@ unsigned RISCVELFObjectWriter::getRelocType(const MCFixup &Fixup,
   case ELF::R_RISCV_GOT32_PCREL:
     if (Kind == FK_Data_4)
       break;
-    reportError(Fixup.getLoc(), "%" + RISCVMCExpr::getSpecifierName(Spec) +
+    reportError(Fixup.getLoc(), "%" + RISCV::getSpecifierName(Spec) +
                                     " can only be used in a .word directive");
     return ELF::R_RISCV_NONE;
   default:
@@ -119,14 +118,11 @@ unsigned RISCVELFObjectWriter::getRelocType(const MCFixup &Fixup,
     reportError(Fixup.getLoc(), "2-byte data relocations not supported");
     return ELF::R_RISCV_NONE;
   case FK_Data_4:
-    if (Expr->getKind() == MCExpr::Target) {
-      auto Spec = cast<RISCVMCExpr>(Expr)->getSpecifier();
-      switch (Spec) {
-      case ELF::R_RISCV_32_PCREL:
-      case ELF::R_RISCV_GOT32_PCREL:
-      case ELF::R_RISCV_PLT32:
-        return Spec;
-      }
+    switch (Spec) {
+    case ELF::R_RISCV_32_PCREL:
+    case ELF::R_RISCV_GOT32_PCREL:
+    case ELF::R_RISCV_PLT32:
+      return Spec;
     }
     return ELF::R_RISCV_32;
   case FK_Data_8:
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 1f4a77414db6..8c9ab8effa71 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -12,7 +12,6 @@
 
 #include "RISCVInstPrinter.h"
 #include "RISCVBaseInfo.h"
-#include "RISCVMCExpr.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
@@ -102,7 +101,7 @@ void RISCVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
 }
 
 void RISCVInstPrinter::printBranchOperand(const MCInst *MI, uint64_t Address,
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
index e75bc521d47c..090d331d99ca 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "RISCVMCAsmInfo.h"
-#include "MCTargetDesc/RISCVMCExpr.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCExpr.h"
@@ -45,5 +44,16 @@ const MCExpr *RISCVMCAsmInfo::getExprForFDESymbol(const MCSymbol *Sym,
   MCContext &Ctx = Streamer.getContext();
   const MCExpr *ME = MCSymbolRefExpr::create(Sym, Ctx);
   assert(Encoding & dwarf::DW_EH_PE_sdata4 && "Unexpected encoding");
-  return RISCVMCExpr::create(ME, ELF::R_RISCV_32_PCREL, Ctx);
+  return MCSpecifierExpr::create(ME, ELF::R_RISCV_32_PCREL, Ctx);
+}
+
+void RISCVMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                        const MCSpecifierExpr &Expr) const {
+  auto S = Expr.getSpecifier();
+  bool HasSpecifier = S != 0 && S != ELF::R_RISCV_CALL_PLT;
+  if (HasSpecifier)
+    OS << '%' << RISCV::getSpecifierName(S) << '(';
+  printExpr(OS, *Expr.getSubExpr());
+  if (HasSpecifier)
+    OS << ')';
 }
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h
index bceeb1256471..097e94b6117c 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h
@@ -14,6 +14,7 @@
 #define LLVM_LIB_TARGET_RISCV_MCTARGETDESC_RISCVMCASMINFO_H
 
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCFixup.h"
 
 namespace llvm {
 class Triple;
@@ -26,8 +27,29 @@ class RISCVMCAsmInfo : public MCAsmInfoELF {
 
   const MCExpr *getExprForFDESymbol(const MCSymbol *Sym, unsigned Encoding,
                                     MCStreamer &Streamer) const override;
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
 };
 
+namespace RISCV {
+using Specifier = uint16_t;
+// Specifiers mapping to relocation types below FirstTargetFixupKind are
+// encoded literally, with these exceptions:
+enum {
+  S_None,
+  // Specifiers mapping to distinct relocation types.
+  S_LO = FirstTargetFixupKind,
+  S_PCREL_LO,
+  S_TPREL_LO,
+  // Vendor-specific relocation types might conflict across vendors.
+  // Refer to them using Specifier constants.
+  S_QC_ABS20,
+};
+
+Specifier parseSpecifierName(StringRef name);
+StringRef getSpecifierName(Specifier Kind);
+} // namespace RISCV
+
 } // namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index 8f9ce37a4de2..1185e3558b00 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -12,7 +12,7 @@
 
 #include "MCTargetDesc/RISCVBaseInfo.h"
 #include "MCTargetDesc/RISCVFixupKinds.h"
-#include "MCTargetDesc/RISCVMCExpr.h"
+#include "MCTargetDesc/RISCVMCAsmInfo.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -177,7 +177,7 @@ void RISCVMCCodeEmitter::expandTLSDESCCall(const MCInst &MI,
   MCOperand SrcSymbol = MI.getOperand(3);
   assert(SrcSymbol.isExpr() &&
          "Expected expression as first input to TLSDESCCALL");
-  const RISCVMCExpr *Expr = dyn_cast<RISCVMCExpr>(SrcSymbol.getExpr());
+  const auto *Expr = dyn_cast<MCSpecifierExpr>(SrcSymbol.getExpr());
   MCRegister Link = MI.getOperand(0).getReg();
   MCRegister Dest = MI.getOperand(1).getReg();
   int64_t Imm = MI.getOperand(2).getImm();
@@ -205,7 +205,7 @@ void RISCVMCCodeEmitter::expandAddTPRel(const MCInst &MI,
   assert(SrcSymbol.isExpr() &&
          "Expected expression as third input to TP-relative add");
 
-  const RISCVMCExpr *Expr = dyn_cast<RISCVMCExpr>(SrcSymbol.getExpr());
+  const auto *Expr = dyn_cast<MCSpecifierExpr>(SrcSymbol.getExpr());
   assert(Expr && Expr->getSpecifier() == ELF::R_RISCV_TPREL_ADD &&
          "Expected tprel_add relocation on TP-relative symbol");
 
@@ -565,8 +565,8 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
   MCExpr::ExprKind Kind = Expr->getKind();
   unsigned FixupKind = RISCV::fixup_riscv_invalid;
   bool RelaxCandidate = false;
-  if (Kind == MCExpr::Target) {
-    const RISCVMCExpr *RVExpr = cast<RISCVMCExpr>(Expr);
+  if (Kind == MCExpr::Specifier) {
+    const auto *RVExpr = cast<MCSpecifierExpr>(Expr);
     FixupKind = RVExpr->getSpecifier();
     switch (RVExpr->getSpecifier()) {
     default:
@@ -580,7 +580,7 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       // encounter it here is an error.
       llvm_unreachable(
           "ELF::R_RISCV_TPREL_ADD should not represent an instruction operand");
-    case RISCVMCExpr::VK_LO:
+    case RISCV::S_LO:
       if (MIFrm == RISCVII::InstFormatI)
         FixupKind = RISCV::fixup_riscv_lo12_i;
       else if (MIFrm == RISCVII::InstFormatS)
@@ -593,7 +593,7 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       FixupKind = RISCV::fixup_riscv_hi20;
       RelaxCandidate = true;
       break;
-    case RISCVMCExpr::VK_PCREL_LO:
+    case RISCV::S_PCREL_LO:
       if (MIFrm == RISCVII::InstFormatI)
         FixupKind = RISCV::fixup_riscv_pcrel_lo12_i;
       else if (MIFrm == RISCVII::InstFormatS)
@@ -606,7 +606,7 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       FixupKind = RISCV::fixup_riscv_pcrel_hi20;
       RelaxCandidate = true;
       break;
-    case RISCVMCExpr::VK_TPREL_LO:
+    case RISCV::S_TPREL_LO:
       if (MIFrm == RISCVII::InstFormatI)
         FixupKind = ELF::R_RISCV_TPREL_LO12_I;
       else if (MIFrm == RISCVII::InstFormatS)
@@ -622,8 +622,9 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       FixupKind = RISCV::fixup_riscv_call_plt;
       RelaxCandidate = true;
       break;
-    case RISCVMCExpr::VK_QC_ABS20:
+    case RISCV::S_QC_ABS20:
       FixupKind = RISCV::fixup_riscv_qc_abs20_u;
+      RelaxCandidate = true;
       break;
     }
   } else if (Kind == MCExpr::SymbolRef || Kind == MCExpr::Binary) {
@@ -642,8 +643,10 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       FixupKind = RISCV::fixup_riscv_qc_e_branch;
     } else if (MIFrm == RISCVII::InstFormatQC_EAI) {
       FixupKind = RISCV::fixup_riscv_qc_e_32;
+      RelaxCandidate = true;
     } else if (MIFrm == RISCVII::InstFormatQC_EJ) {
       FixupKind = RISCV::fixup_riscv_qc_e_jump_plt;
+      RelaxCandidate = true;
     }
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index e4ccf5029231..baa508ad3a68 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "RISCVMCExpr.h"
 #include "MCTargetDesc/RISCVAsmBackend.h"
+#include "MCTargetDesc/RISCVMCAsmInfo.h"
 #include "RISCVFixupKinds.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAssembler.h"
@@ -26,90 +26,14 @@ using namespace llvm;
 
 #define DEBUG_TYPE "riscvmcexpr"
 
-const RISCVMCExpr *RISCVMCExpr::create(const MCExpr *Expr, Specifier S,
-                                       MCContext &Ctx) {
-  return new (Ctx) RISCVMCExpr(Expr, S);
-}
-
-void RISCVMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  Specifier S = getSpecifier();
-  bool HasVariant = S != VK_None && S != ELF::R_RISCV_CALL_PLT;
-
-  if (HasVariant)
-    OS << '%' << getSpecifierName(S) << '(';
-  Expr->print(OS, MAI);
-  if (HasVariant)
-    OS << ')';
-}
-
-const MCFixup *RISCVMCExpr::getPCRelHiFixup(const MCFragment **DFOut) const {
-  MCValue AUIPCLoc;
-  if (!getSubExpr()->evaluateAsRelocatable(AUIPCLoc, nullptr))
-    return nullptr;
-
-  const MCSymbol *AUIPCSymbol = AUIPCLoc.getAddSym();
-  if (!AUIPCSymbol)
-    return nullptr;
-  const auto *DF = dyn_cast_or_null<MCDataFragment>(AUIPCSymbol->getFragment());
-
-  if (!DF)
-    return nullptr;
-
-  uint64_t Offset = AUIPCSymbol->getOffset();
-  if (DF->getContents().size() == Offset) {
-    DF = dyn_cast_or_null<MCDataFragment>(DF->getNext());
-    if (!DF)
-      return nullptr;
-    Offset = 0;
-  }
-
-  for (const MCFixup &F : DF->getFixups()) {
-    if (F.getOffset() != Offset)
-      continue;
-    auto Kind = F.getTargetKind();
-    if (!mc::isRelocation(F.getKind())) {
-      if (Kind == RISCV::fixup_riscv_pcrel_hi20) {
-        *DFOut = DF;
-        return &F;
-      }
-      break;
-    }
-    switch (Kind) {
-    case ELF::R_RISCV_GOT_HI20:
-    case ELF::R_RISCV_TLS_GOT_HI20:
-    case ELF::R_RISCV_TLS_GD_HI20:
-    case ELF::R_RISCV_TLSDESC_HI20:
-      *DFOut = DF;
-      return &F;
-    }
-  }
-
-  return nullptr;
-}
-
-bool RISCVMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                            const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-
-  // Custom fixup types are not valid with symbol difference expressions.
-  return !Res.getSubSym();
-}
-
-void RISCVMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-std::optional<RISCVMCExpr::Specifier>
-RISCVMCExpr::getSpecifierForName(StringRef name) {
-  return StringSwitch<std::optional<RISCVMCExpr::Specifier>>(name)
-      .Case("lo", VK_LO)
+RISCV::Specifier RISCV::parseSpecifierName(StringRef name) {
+  return StringSwitch<RISCV::Specifier>(name)
+      .Case("lo", RISCV::S_LO)
       .Case("hi", ELF::R_RISCV_HI20)
-      .Case("pcrel_lo", VK_PCREL_LO)
+      .Case("pcrel_lo", RISCV::S_PCREL_LO)
       .Case("pcrel_hi", ELF::R_RISCV_PCREL_HI20)
       .Case("got_pcrel_hi", ELF::R_RISCV_GOT_HI20)
-      .Case("tprel_lo", VK_TPREL_LO)
+      .Case("tprel_lo", RISCV::S_TPREL_LO)
       .Case("tprel_hi", ELF::R_RISCV_TPREL_HI20)
       .Case("tprel_add", ELF::R_RISCV_TPREL_ADD)
       .Case("tls_ie_pcrel_hi", ELF::R_RISCV_TLS_GOT_HI20)
@@ -118,28 +42,28 @@ RISCVMCExpr::getSpecifierForName(StringRef name) {
       .Case("tlsdesc_load_lo", ELF::R_RISCV_TLSDESC_LOAD_LO12)
       .Case("tlsdesc_add_lo", ELF::R_RISCV_TLSDESC_ADD_LO12)
       .Case("tlsdesc_call", ELF::R_RISCV_TLSDESC_CALL)
-      .Case("qc.abs20", VK_QC_ABS20)
+      .Case("qc.abs20", RISCV::S_QC_ABS20)
       // Used in data directives
       .Case("pltpcrel", ELF::R_RISCV_PLT32)
       .Case("gotpcrel", ELF::R_RISCV_GOT32_PCREL)
-      .Default(std::nullopt);
+      .Default(0);
 }
 
-StringRef RISCVMCExpr::getSpecifierName(Specifier S) {
+StringRef RISCV::getSpecifierName(Specifier S) {
   switch (S) {
-  case VK_None:
+  case RISCV::S_None:
     llvm_unreachable("not used as %specifier()");
-  case VK_LO:
+  case RISCV::S_LO:
     return "lo";
   case ELF::R_RISCV_HI20:
     return "hi";
-  case VK_PCREL_LO:
+  case RISCV::S_PCREL_LO:
     return "pcrel_lo";
   case ELF::R_RISCV_PCREL_HI20:
     return "pcrel_hi";
   case ELF::R_RISCV_GOT_HI20:
     return "got_pcrel_hi";
-  case VK_TPREL_LO:
+  case RISCV::S_TPREL_LO:
     return "tprel_lo";
   case ELF::R_RISCV_TPREL_HI20:
     return "tprel_hi";
@@ -165,7 +89,7 @@ StringRef RISCVMCExpr::getSpecifierName(Specifier S) {
     return "gotpcrel";
   case ELF::R_RISCV_PLT32:
     return "pltpcrel";
-  case VK_QC_ABS20:
+  case RISCV::S_QC_ABS20:
     return "qc.abs20";
   }
   llvm_unreachable("Invalid ELF symbol kind");
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
deleted file mode 100644
index 917c59ab3961..000000000000
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
+++ /dev/null
@@ -1,79 +0,0 @@
-//===-- RISCVMCExpr.h - RISC-V specific MC expression classes----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes RISC-V specific MCExprs, used for modifiers like
-// "%hi" or "%lo" etc.,
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_RISCV_MCTARGETDESC_RISCVMCEXPR_H
-#define LLVM_LIB_TARGET_RISCV_MCTARGETDESC_RISCVMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixup.h"
-
-namespace llvm {
-
-class StringRef;
-
-class RISCVMCExpr : public MCTargetExpr {
-public:
-  using Specifier = uint16_t;
-  // Specifiers mapping to relocation types below FirstTargetFixupKind are
-  // encoded literally, with these exceptions:
-  enum {
-    VK_None,
-    // Specifiers mapping to distinct relocation types.
-    VK_LO = FirstTargetFixupKind,
-    VK_PCREL_LO,
-    VK_TPREL_LO,
-    // Vendor-specific relocation types might conflict across vendors.
-    // Refer to them using Specifier constants.
-    VK_QC_ABS20,
-  };
-
-private:
-  const MCExpr *Expr;
-  const Specifier specifier;
-
-  explicit RISCVMCExpr(const MCExpr *Expr, Specifier S)
-      : Expr(Expr), specifier(S) {}
-
-public:
-  static const RISCVMCExpr *create(const MCExpr *Expr, Specifier S,
-                                   MCContext &Ctx);
-
-  Specifier getSpecifier() const { return specifier; }
-
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  /// Get the corresponding PC-relative HI fixup that a VK_PCREL_LO
-  /// points to, and optionally the fragment containing it.
-  ///
-  /// \returns nullptr if this isn't a VK_PCREL_LO pointing to a
-  /// known PC-relative HI fixup.
-  const MCFixup *getPCRelHiFixup(const MCFragment **DFOut) const;
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-
-  static std::optional<Specifier> getSpecifierForName(StringRef name);
-  static StringRef getSpecifierName(Specifier Kind);
-};
-} // end namespace llvm.
-
-#endif
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
index 8ea2548258fd..747e462ee9d5 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
@@ -94,7 +94,15 @@ static void generateInstSeqImpl(int64_t Val, const MCSubtargetInfo &STI,
       Res.emplace_back(RISCV::LUI, Hi20);
 
     if (Lo12 || Hi20 == 0) {
-      unsigned AddiOpc = (IsRV64 && Hi20) ? RISCV::ADDIW : RISCV::ADDI;
+      unsigned AddiOpc = RISCV::ADDI;
+      if (IsRV64 && Hi20) {
+        // Use ADDIW rather than ADDI only when necessary for correctness. As
+        // noted in RISCVOptWInstrs, this helps reduce test differences vs
+        // RV32 without being a pessimization.
+        int64_t LuiRes = SignExtend64<32>(Hi20 << 12);
+        if (!isInt<32>(LuiRes + Lo12))
+          AddiOpc = RISCV::ADDIW;
+      }
       Res.emplace_back(AddiOpc, Lo12);
     }
     return;
@@ -353,6 +361,13 @@ InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI) {
       } while (Hi != 0);
       Res = TmpSeq;
     }
+
+    // Fold LI 1 + SLLI into BSETI.
+    if (Res[0].getOpcode() == RISCV::ADDI && Res[0].getImm() == 1 &&
+        Res[1].getOpcode() == RISCV::SLLI) {
+      Res.erase(Res.begin());                                 // Remove ADDI.
+      Res.front() = Inst(RISCV::BSETI, Res.front().getImm()); // Patch SLLI.
+    }
   }
 
   // Perform optimization with BCLRI in the Zbs extension.
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h
index ae94f3778b21..a82cd650f42f 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h
@@ -20,15 +20,15 @@ class APInt;
 namespace RISCVMatInt {
 
 enum OpndKind {
-  RegImm, // ADDI/ADDIW/SLLI/SRLI/BSETI/BCLRI
-  Imm,    // LUI
-  RegReg, // SH1ADD/SH2ADD/SH3ADD
+  RegImm, // ADDI/ADDIW/XORI/SLLI/SRLI/SLLI_UW/RORI/BSETI/BCLRI/TH_SRRI
+  Imm,    // LUI/QC_LI/QC_E_LI
+  RegReg, // SH1ADD/SH2ADD/SH3ADD/PACK
   RegX0,  // ADD_UW
 };
 
 class Inst {
   unsigned Opc;
-  int32_t Imm; // The largest value we need to store is 20 bits.
+  int32_t Imm; // The largest value we need to store is 32 bits for QC_E_LI.
 
 public:
   Inst(unsigned Opc, int64_t I) : Opc(Opc), Imm(I) {
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCV.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCV.td
index e322ae340349..b24d8637cb27 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCV.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCV.td
@@ -51,6 +51,7 @@ include "RISCVMacroFusion.td"
 //===----------------------------------------------------------------------===//
 // RISC-V Scheduling Models
 //===----------------------------------------------------------------------===//
+include "RISCVSchedAndes45.td"
 include "RISCVSchedGenericOOO.td"
 include "RISCVSchedMIPSP8700.td"
 include "RISCVSchedRocket.td"
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index 72f1596d79a0..83e9b4b4d7c5 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -13,7 +13,7 @@
 
 #include "MCTargetDesc/RISCVBaseInfo.h"
 #include "MCTargetDesc/RISCVInstPrinter.h"
-#include "MCTargetDesc/RISCVMCExpr.h"
+#include "MCTargetDesc/RISCVMCAsmInfo.h"
 #include "MCTargetDesc/RISCVMatInt.h"
 #include "MCTargetDesc/RISCVTargetStreamer.h"
 #include "RISCV.h"
@@ -432,7 +432,7 @@ bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
   if (Offset.isImm())
     OS << MCO.getImm();
   else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol())
-    OS << *MCO.getExpr();
+    MAI->printExpr(OS, *MCO.getExpr());
 
   if (Offset.isMCSymbol())
     MMI->getContext().registerInlineAsmLabel(Offset.getMCSymbol());
@@ -630,7 +630,7 @@ void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
     Sym = OutContext.getOrCreateSymbol(SymName);
   }
   auto Res = MCSymbolRefExpr::create(Sym, OutContext);
-  auto Expr = RISCVMCExpr::create(Res, ELF::R_RISCV_CALL_PLT, OutContext);
+  auto Expr = MCSpecifierExpr::create(Res, ELF::R_RISCV_CALL_PLT, OutContext);
 
   EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
 }
@@ -741,8 +741,8 @@ void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
 
   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
-  auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref, ELF::R_RISCV_CALL_PLT,
-                                  OutContext);
+  auto Expr = MCSpecifierExpr::create(HwasanTagMismatchV2Ref,
+                                      ELF::R_RISCV_CALL_PLT, OutContext);
 
   for (auto &P : HwasanMemaccessSymbols) {
     unsigned Reg = std::get<0>(P.first);
@@ -957,25 +957,25 @@ void RISCVAsmPrinter::emitNoteGnuProperty(const Module &M) {
 static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
                                     const AsmPrinter &AP) {
   MCContext &Ctx = AP.OutContext;
-  RISCVMCExpr::Specifier Kind;
+  RISCV::Specifier Kind;
 
   switch (MO.getTargetFlags()) {
   default:
     llvm_unreachable("Unknown target flag on GV operand");
   case RISCVII::MO_None:
-    Kind = RISCVMCExpr::VK_None;
+    Kind = RISCV::S_None;
     break;
   case RISCVII::MO_CALL:
     Kind = ELF::R_RISCV_CALL_PLT;
     break;
   case RISCVII::MO_LO:
-    Kind = RISCVMCExpr::VK_LO;
+    Kind = RISCV::S_LO;
     break;
   case RISCVII::MO_HI:
     Kind = ELF::R_RISCV_HI20;
     break;
   case RISCVII::MO_PCREL_LO:
-    Kind = RISCVMCExpr::VK_PCREL_LO;
+    Kind = RISCV::S_PCREL_LO;
     break;
   case RISCVII::MO_PCREL_HI:
     Kind = ELF::R_RISCV_PCREL_HI20;
@@ -984,7 +984,7 @@ static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
     Kind = ELF::R_RISCV_GOT_HI20;
     break;
   case RISCVII::MO_TPREL_LO:
-    Kind = RISCVMCExpr::VK_TPREL_LO;
+    Kind = RISCV::S_TPREL_LO;
     break;
   case RISCVII::MO_TPREL_HI:
     Kind = ELF::R_RISCV_TPREL_HI20;
@@ -1018,8 +1018,8 @@ static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
     ME = MCBinaryExpr::createAdd(
         ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
 
-  if (Kind != RISCVMCExpr::VK_None)
-    ME = RISCVMCExpr::create(ME, Kind, Ctx);
+  if (Kind != RISCV::S_None)
+    ME = MCSpecifierExpr::create(ME, Kind, Ctx);
   return MCOperand::createExpr(ME);
 }
 
@@ -1101,7 +1101,7 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
   if (RISCVII::hasRoundModeOp(TSFlags))
     --NumOps;
 
-  bool hasVLOutput = RISCV::isFaultFirstLoad(*MI);
+  bool hasVLOutput = RISCVInstrInfo::isFaultOnlyFirstLoad(*MI);
   for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {
     const MachineOperand &MO = MI->getOperand(OpNo);
     // Skip vl output. It should be the second output.
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVCallingConv.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
index d3bfbb094376..cb6117eb0917 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
@@ -13,6 +13,7 @@
 #include "RISCVCallingConv.h"
 #include "RISCVSubtarget.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Module.h"
 #include "llvm/MC/MCRegister.h"
 
 using namespace llvm;
@@ -332,10 +333,24 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
   unsigned XLen = Subtarget.getXLen();
   MVT XLenVT = Subtarget.getXLenVT();
 
-  // Static chain parameter must not be passed in normal argument registers,
-  // so we assign t2 for it as done in GCC's __builtin_call_with_static_chain
   if (ArgFlags.isNest()) {
-    if (MCRegister Reg = State.AllocateReg(RISCV::X7)) {
+    // Static chain parameter must not be passed in normal argument registers,
+    // so we assign t2/t3 for it as done in GCC's
+    // __builtin_call_with_static_chain
+    bool HasCFBranch =
+        Subtarget.hasStdExtZicfilp() &&
+        MF.getFunction().getParent()->getModuleFlag("cf-protection-branch");
+
+    // Normal: t2, Branch control flow protection: t3
+    const auto StaticChainReg = HasCFBranch ? RISCV::X28 : RISCV::X7;
+
+    RISCVABI::ABI ABI = Subtarget.getTargetABI();
+    if (HasCFBranch &&
+        (ABI == RISCVABI::ABI_ILP32E || ABI == RISCVABI::ABI_LP64E))
+      reportFatalUsageError(
+          "Nested functions with control flow protection are not "
+          "usable with ILP32E or LP64E ABI.");
+    if (MCRegister Reg = State.AllocateReg(StaticChainReg)) {
       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
       return false;
     }
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVFeatures.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVFeatures.td
index 690068d05aaa..940caa4f4044 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -75,7 +75,8 @@ def FeatureStdExtI
       RISCVExtensionBitmask<0, 8>;
 
 def FeatureStdExtE
-    : RISCVExtension<2, 0, "Embedded Instruction Set with 16 GPRs">;
+    : RISCVExtension<2, 0, "Embedded Instruction Set with 16 GPRs">,
+      RISCVExtensionBitmask<0, 4>;
 
 def FeatureStdExtZic64b
     : RISCVExtension<1, 0, "Cache Block Size Is 64 Bytes">;
@@ -510,7 +511,8 @@ def HasStdExtZbs : Predicate<"Subtarget->hasStdExtZbs()">,
 
 def FeatureStdExtB
     : RISCVExtension<1, 0, "the collection of the Zba, Zbb, Zbs extensions",
-                     [FeatureStdExtZba, FeatureStdExtZbb, FeatureStdExtZbs]>;
+                     [FeatureStdExtZba, FeatureStdExtZbb, FeatureStdExtZbs]>,
+      RISCVExtensionBitmask<0, 1>;
 
 def FeatureStdExtZbkb
     : RISCVExtension<1, 0, "Bitmanip instructions for Cryptography">,
@@ -887,8 +889,8 @@ def HasVInstructionsFullMultiply : Predicate<"Subtarget->hasVInstructionsFullMul
 
 // Hypervisor Extensions
 
-def FeatureStdExtH : RISCVExtension<1, 0, "Hypervisor">;
-
+def FeatureStdExtH : RISCVExtension<1, 0, "Hypervisor">,
+                     RISCVExtensionBitmask<0, 7>;
 def HasStdExtH : Predicate<"Subtarget->hasStdExtH()">,
                  AssemblerPredicate<(all_of FeatureStdExtH),
                                     "'H' (Hypervisor)">;
@@ -1740,6 +1742,9 @@ def TuneSiFive7 : SubtargetFeature<"sifive7", "RISCVProcFamily", "SiFive7",
 def TuneVentanaVeyron : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "VentanaVeyron",
                                          "Ventana Veyron-Series processors">;
 
+def TuneAndes45 : SubtargetFeature<"andes45", "RISCVProcFamily", "Andes45",
+                                   "Andes 45-Series processors">;
+
 def TuneVXRMPipelineFlush : SubtargetFeature<"vxrm-pipeline-flush", "HasVXRMPipelineFlush",
                                              "true", "VXRM writes causes pipeline flush">;
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 2de22ee165ae..4539efd591c8 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -601,8 +601,16 @@ bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
 }
 
 bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
-  // Only supported with XTHeadBb at the moment.
-  if (!Subtarget->hasVendorXTHeadBb())
+  unsigned Opc;
+
+  if (Subtarget->hasVendorXTHeadBb())
+    Opc = RISCV::TH_EXT;
+  else if (Subtarget->hasVendorXAndesPerf())
+    Opc = RISCV::NDS_BFOS;
+  else if (Subtarget->hasVendorXqcibm())
+    Opc = RISCV::QC_EXT;
+  else
+    // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
     return false;
 
   auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
@@ -613,9 +621,15 @@ bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
   if (!N0.hasOneUse())
     return false;
 
-  auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb, SDLoc DL,
-                             MVT VT) {
-    return CurDAG->getMachineNode(RISCV::TH_EXT, DL, VT, N0.getOperand(0),
+  auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb,
+                             const SDLoc &DL, MVT VT) {
+    if (Opc == RISCV::QC_EXT) {
+      // QC.EXT X, width, shamt
+      // shamt is the same as Lsb
+      // width is the number of bits to extract from the Lsb
+      Msb = Msb - Lsb + 1;
+    }
+    return CurDAG->getMachineNode(Opc, DL, VT, N0.getOperand(0),
                                   CurDAG->getTargetConstant(Msb, DL, VT),
                                   CurDAG->getTargetConstant(Lsb, DL, VT));
   };
@@ -625,7 +639,7 @@ bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
   const unsigned RightShAmt = N1C->getZExtValue();
 
   // Transform (sra (shl X, C1) C2) with C1 < C2
-  //        -> (TH.EXT X, msb, lsb)
+  //        -> (SignedBitfieldExtract X, msb, lsb)
   if (N0.getOpcode() == ISD::SHL) {
     auto *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
     if (!N01C)
@@ -641,13 +655,13 @@ bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
     const unsigned Msb = MsbPlusOne - 1;
     const unsigned Lsb = RightShAmt - LeftShAmt;
 
-    SDNode *TH_EXT = BitfieldExtract(N0, Msb, Lsb, DL, VT);
-    ReplaceNode(Node, TH_EXT);
+    SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
+    ReplaceNode(Node, Sbe);
     return true;
   }
 
   // Transform (sra (sext_inreg X, _), C) ->
-  //           (TH.EXT X, msb, lsb)
+  //           (SignedBitfieldExtract X, msb, lsb)
   if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
     unsigned ExtSize =
         cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
@@ -661,23 +675,87 @@ bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
     // the X[Msb] bit and sign-extend it.
     const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
 
-    SDNode *TH_EXT = BitfieldExtract(N0, Msb, Lsb, DL, VT);
-    ReplaceNode(Node, TH_EXT);
+    SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
+    ReplaceNode(Node, Sbe);
     return true;
   }
 
   return false;
 }
 
-bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node, SDLoc DL,
-                                                   MVT VT, SDValue X,
-                                                   unsigned Msb, unsigned Lsb) {
-  // Only supported with XTHeadBb/XAndesPerf at the moment.
-  if (!Subtarget->hasVendorXTHeadBb() && !Subtarget->hasVendorXAndesPerf())
+bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
+  // Only supported with XAndesPerf at the moment.
+  if (!Subtarget->hasVendorXAndesPerf())
     return false;
 
-  unsigned Opc =
-      Subtarget->hasVendorXTHeadBb() ? RISCV::TH_EXTU : RISCV::NDS_BFOZ;
+  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
+  if (!N1C)
+    return false;
+
+  SDValue N0 = Node->getOperand(0);
+  if (!N0.hasOneUse())
+    return false;
+
+  auto BitfieldInsert = [&](SDValue N0, unsigned Msb, unsigned Lsb,
+                            const SDLoc &DL, MVT VT) {
+    unsigned Opc = RISCV::NDS_BFOS;
+    // If the Lsb is equal to the Msb, then the Lsb should be 0.
+    if (Lsb == Msb)
+      Lsb = 0;
+    return CurDAG->getMachineNode(Opc, DL, VT, N0.getOperand(0),
+                                  CurDAG->getTargetConstant(Lsb, DL, VT),
+                                  CurDAG->getTargetConstant(Msb, DL, VT));
+  };
+
+  SDLoc DL(Node);
+  MVT VT = Node->getSimpleValueType(0);
+  const unsigned RightShAmt = N1C->getZExtValue();
+
+  // Transform (sra (shl X, C1) C2) with C1 > C2
+  //        -> (NDS.BFOS X, lsb, msb)
+  if (N0.getOpcode() == ISD::SHL) {
+    auto *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
+    if (!N01C)
+      return false;
+
+    const unsigned LeftShAmt = N01C->getZExtValue();
+    // Make sure that this is a bitfield insertion (i.e., the shift-right
+    // amount should be less than the left-shift).
+    if (LeftShAmt <= RightShAmt)
+      return false;
+
+    const unsigned MsbPlusOne = VT.getSizeInBits() - RightShAmt;
+    const unsigned Msb = MsbPlusOne - 1;
+    const unsigned Lsb = LeftShAmt - RightShAmt;
+
+    SDNode *Sbi = BitfieldInsert(N0, Msb, Lsb, DL, VT);
+    ReplaceNode(Node, Sbi);
+    return true;
+  }
+
+  return false;
+}
+
+bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node,
+                                                   const SDLoc &DL, MVT VT,
+                                                   SDValue X, unsigned Msb,
+                                                   unsigned Lsb) {
+  unsigned Opc;
+
+  if (Subtarget->hasVendorXTHeadBb()) {
+    Opc = RISCV::TH_EXTU;
+  } else if (Subtarget->hasVendorXAndesPerf()) {
+    Opc = RISCV::NDS_BFOZ;
+  } else if (Subtarget->hasVendorXqcibm()) {
+    Opc = RISCV::QC_EXTU;
+    // QC.EXTU X, width, shamt
+    // shamt is the same as Lsb
+    // width is the number of bits to extract from the Lsb
+    Msb = Msb - Lsb + 1;
+  } else {
+    // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
+    return false;
+  }
 
   SDNode *Ube = CurDAG->getMachineNode(Opc, DL, VT, X,
                                        CurDAG->getTargetConstant(Msb, DL, VT),
@@ -686,6 +764,26 @@ bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node, SDLoc DL,
   return true;
 }
 
+bool RISCVDAGToDAGISel::tryUnsignedBitfieldInsertInZero(SDNode *Node,
+                                                        const SDLoc &DL, MVT VT,
+                                                        SDValue X, unsigned Msb,
+                                                        unsigned Lsb) {
+  // Only supported with XAndesPerf at the moment.
+  if (!Subtarget->hasVendorXAndesPerf())
+    return false;
+
+  unsigned Opc = RISCV::NDS_BFOZ;
+
+  // If the Lsb is equal to the Msb, then the Lsb should be 0.
+  if (Lsb == Msb)
+    Lsb = 0;
+  SDNode *Ubi = CurDAG->getMachineNode(Opc, DL, VT, X,
+                                       CurDAG->getTargetConstant(Lsb, DL, VT),
+                                       CurDAG->getTargetConstant(Msb, DL, VT));
+  ReplaceNode(Node, Ubi);
+  return true;
+}
+
 bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
   // Target does not support indexed loads.
   if (!Subtarget->hasVendorXTHeadMemIdx())
@@ -1049,11 +1147,11 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     unsigned ShAmt = N1C->getZExtValue();
     uint64_t Mask = N0.getConstantOperandVal(1);
 
-    if (ShAmt <= 32 && isShiftedMask_64(Mask)) {
+    if (isShiftedMask_64(Mask)) {
       unsigned XLen = Subtarget->getXLen();
       unsigned LeadingZeros = XLen - llvm::bit_width(Mask);
       unsigned TrailingZeros = llvm::countr_zero(Mask);
-      if (TrailingZeros > 0 && LeadingZeros == 32) {
+      if (ShAmt <= 32 && TrailingZeros > 0 && LeadingZeros == 32) {
         // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C)
         // where C2 has 32 leading zeros and C3 trailing zeros.
         SDNode *SRLIW = CurDAG->getMachineNode(
@@ -1169,6 +1267,9 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     if (trySignedBitfieldExtract(Node))
       return;
 
+    if (trySignedBitfieldInsertInSign(Node))
+      return;
+
     // Optimize (sra (sext_inreg X, i16), C) ->
     //          (srai (slli X, (XLen-16), (XLen-16) + C)
     // And      (sra (sext_inreg X, i8), C) ->
@@ -1322,6 +1423,19 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
             return;
           }
 
+          // Try to use an unsigned bitfield insert (e.g., nds.bfoz) if
+          // available.
+          // Transform (and (shl x, c2), c1)
+          //        -> (<bfinsert> x, msb, lsb)
+          // e.g.
+          //     (and (shl x, 12), 0x00fff000)
+          //     If XLen = 32 and C2 = 12, then
+          //     Msb = 32 - 8 - 1 = 23 and Lsb = 12
+          const unsigned Msb = XLen - Leading - 1;
+          const unsigned Lsb = C2;
+          if (tryUnsignedBitfieldInsertInZero(Node, DL, VT, X, Msb, Lsb))
+            return;
+
           // (srli (slli c2+c3), c3)
           if (OneUseOrZExtW && !IsCANDI) {
             SDNode *SLLI = CurDAG->getMachineNode(
@@ -3114,34 +3228,44 @@ bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt,
       else
         Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
 
-      // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
-      // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
-      // followed by a SHXADD with c3 for the X amount.
       if (isShiftedMask_64(Mask)) {
         unsigned Leading = XLen - llvm::bit_width(Mask);
         unsigned Trailing = llvm::countr_zero(Mask);
-        if (LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
-          SDLoc DL(N);
-          EVT VT = N.getValueType();
-          Val = SDValue(CurDAG->getMachineNode(
-                            RISCV::SRLI, DL, VT, N0.getOperand(0),
-                            CurDAG->getTargetConstant(Trailing - C2, DL, VT)),
-                        0);
-          return true;
-        }
+        if (Trailing != ShAmt)
+          return false;
+
+        unsigned Opcode;
+        // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
+        // leading zeros and c3 trailing zeros. We can use an SRLI by c3-c2
+        // followed by a SHXADD with c3 for the X amount.
+        if (LeftShift && Leading == 0 && C2 < Trailing)
+          Opcode = RISCV::SRLI;
+        // Look for (and (shl y, c2), c1) where c1 is a shifted mask with 32-c2
+        // leading zeros and c3 trailing zeros. We can use an SRLIW by c3-c2
+        // followed by a SHXADD with c3 for the X amount.
+        else if (LeftShift && Leading == 32 - C2 && C2 < Trailing)
+          Opcode = RISCV::SRLIW;
         // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
-        // leading zeros and c3 trailing zeros. We can use an SRLI by C3
+        // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
         // followed by a SHXADD using c3 for the X amount.
-        if (!LeftShift && Leading == C2 && Trailing == ShAmt) {
-          SDLoc DL(N);
-          EVT VT = N.getValueType();
-          Val = SDValue(
-              CurDAG->getMachineNode(
-                  RISCV::SRLI, DL, VT, N0.getOperand(0),
-                  CurDAG->getTargetConstant(Leading + Trailing, DL, VT)),
-              0);
-          return true;
-        }
+        else if (!LeftShift && Leading == C2)
+          Opcode = RISCV::SRLI;
+        // Look for (and (shr y, c2), c1) where c1 is a shifted mask with 32+c2
+        // leading zeros and c3 trailing zeros. We can use an SRLIW by c2+c3
+        // followed by a SHXADD using c3 for the X amount.
+        else if (!LeftShift && Leading == 32 + C2)
+          Opcode = RISCV::SRLIW;
+        else
+          return false;
+
+        SDLoc DL(N);
+        EVT VT = N.getValueType();
+        ShAmt = LeftShift ? Trailing - C2 : Trailing + C2;
+        Val = SDValue(
+            CurDAG->getMachineNode(Opcode, DL, VT, N0.getOperand(0),
+                                   CurDAG->getTargetConstant(ShAmt, DL, VT)),
+            0);
+        return true;
       }
     } else if (N0.getOpcode() == ISD::SRA && N0.hasOneUse() &&
                isa<ConstantSDNode>(N0.getOperand(1))) {
@@ -3251,6 +3375,50 @@ bool RISCVDAGToDAGISel::selectSHXADD_UWOp(SDValue N, unsigned ShAmt,
   return false;
 }
 
+bool RISCVDAGToDAGISel::orIsAdd(const SDNode *N) const {
+  if (N->getFlags().hasDisjoint())
+    return true;
+  KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
+  KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
+  return KnownBits::haveNoCommonBitsSet(Known0, Known1);
+}
+
+bool RISCVDAGToDAGISel::selectImm64IfCheaper(int64_t Imm, int64_t OrigImm,
+                                             SDValue N, SDValue &Val) {
+  int OrigCost = RISCVMatInt::getIntMatCost(APInt(64, OrigImm), 64, *Subtarget,
+                                            /*CompressionCost=*/true);
+  int Cost = RISCVMatInt::getIntMatCost(APInt(64, Imm), 64, *Subtarget,
+                                        /*CompressionCost=*/true);
+  if (OrigCost <= Cost)
+    return false;
+
+  Val = selectImm(CurDAG, SDLoc(N), N->getSimpleValueType(0), Imm, *Subtarget);
+  return true;
+}
+
+bool RISCVDAGToDAGISel::selectZExtImm32(SDValue N, SDValue &Val) {
+  if (!isa<ConstantSDNode>(N))
+    return false;
+  int64_t Imm = cast<ConstantSDNode>(N)->getSExtValue();
+  if ((Imm >> 31) != 1)
+    return false;
+
+  for (const SDNode *U : N->users()) {
+    switch (U->getOpcode()) {
+    case ISD::ADD:
+      break;
+    case ISD::OR:
+      if (orIsAdd(U))
+        break;
+      return false;
+    default:
+      return false;
+    }
+  }
+
+  return selectImm64IfCheaper(0xffffffff00000000 | Imm, Imm, N, Val);
+}
+
 bool RISCVDAGToDAGISel::selectNegImm(SDValue N, SDValue &Val) {
   if (!isa<ConstantSDNode>(N))
     return false;
@@ -3274,15 +3442,7 @@ bool RISCVDAGToDAGISel::selectNegImm(SDValue N, SDValue &Val) {
     }
   }
 
-  int OrigImmCost = RISCVMatInt::getIntMatCost(APInt(64, Imm), 64, *Subtarget,
-                                               /*CompressionCost=*/true);
-  int NegImmCost = RISCVMatInt::getIntMatCost(APInt(64, -Imm), 64, *Subtarget,
-                                              /*CompressionCost=*/true);
-  if (OrigImmCost <= NegImmCost)
-    return false;
-
-  Val = selectImm(CurDAG, SDLoc(N), N->getSimpleValueType(0), -Imm, *Subtarget);
-  return true;
+  return selectImm64IfCheaper(-Imm, Imm, N, Val);
 }
 
 bool RISCVDAGToDAGISel::selectInvLogicImm(SDValue N, SDValue &Val) {
@@ -3317,19 +3477,15 @@ bool RISCVDAGToDAGISel::selectInvLogicImm(SDValue N, SDValue &Val) {
     }
   }
 
-  // For 64-bit constants, the instruction sequences get complex,
-  // so we select inverted only if it's cheaper.
-  if (!isInt<32>(Imm)) {
-    int OrigImmCost = RISCVMatInt::getIntMatCost(APInt(64, Imm), 64, *Subtarget,
-                                                 /*CompressionCost=*/true);
-    int NegImmCost = RISCVMatInt::getIntMatCost(APInt(64, ~Imm), 64, *Subtarget,
-                                                /*CompressionCost=*/true);
-    if (OrigImmCost <= NegImmCost)
-      return false;
+  if (isInt<32>(Imm)) {
+    Val =
+        selectImm(CurDAG, SDLoc(N), N->getSimpleValueType(0), ~Imm, *Subtarget);
+    return true;
   }
 
-  Val = selectImm(CurDAG, SDLoc(N), N->getSimpleValueType(0), ~Imm, *Subtarget);
-  return true;
+  // For 64-bit constants, the instruction sequences get complex,
+  // so we select inverted only if it's cheaper.
+  return selectImm64IfCheaper(~Imm, Imm, N, Val);
 }
 
 static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index 11d62e5edad3..cb63c21fd8fc 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -77,8 +77,11 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
 
   bool tryShrinkShlLogicImm(SDNode *Node);
   bool trySignedBitfieldExtract(SDNode *Node);
-  bool tryUnsignedBitfieldExtract(SDNode *Node, SDLoc DL, MVT VT, SDValue X,
-                                  unsigned Msb, unsigned Lsb);
+  bool trySignedBitfieldInsertInSign(SDNode *Node);
+  bool tryUnsignedBitfieldExtract(SDNode *Node, const SDLoc &DL, MVT VT,
+                                  SDValue X, unsigned Msb, unsigned Lsb);
+  bool tryUnsignedBitfieldInsertInZero(SDNode *Node, const SDLoc &DL, MVT VT,
+                                       SDValue X, unsigned Msb, unsigned Lsb);
   bool tryIndexedLoad(SDNode *Node);
 
   bool selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt);
@@ -116,9 +119,11 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
     return selectSHXADD_UWOp(N, ShAmt, Val);
   }
 
+  bool selectZExtImm32(SDValue N, SDValue &Val);
   bool selectNegImm(SDValue N, SDValue &Val);
   bool selectInvLogicImm(SDValue N, SDValue &Val);
 
+  bool orIsAdd(const SDNode *Node) const;
   bool hasAllNBitUsers(SDNode *Node, unsigned Bits,
                        const unsigned Depth = 0) const;
   bool hasAllBUsers(SDNode *Node) const { return hasAllNBitUsers(Node, 8); }
@@ -197,6 +202,8 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
   bool doPeepholeMergeVVMFold();
   bool doPeepholeNoRegPassThru();
   bool performCombineVMergeAndVOps(SDNode *N);
+  bool selectImm64IfCheaper(int64_t Imm, int64_t OrigImm, SDValue N,
+                            SDValue &Val);
 };
 
 class RISCVDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 6e8e4ac1c6a9..7cfada6c0601 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -434,9 +434,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::ABS, MVT::i32, Custom);
   }
 
-  if (Subtarget.useCCMovInsn())
-    setOperationAction(ISD::SELECT, XLenVT, Legal);
-  else if (!Subtarget.hasVendorXTHeadCondMov())
+  if (!Subtarget.useCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
     setOperationAction(ISD::SELECT, XLenVT, Custom);
 
   if (Subtarget.hasVendorXqcia() && !Subtarget.is64Bit()) {
@@ -651,6 +649,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
     setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
+    setOperationAction(ISD::GET_FPENV, XLenVT, Custom);
+    setOperationAction(ISD::SET_FPENV, XLenVT, Custom);
+    setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
   }
 
   setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
@@ -1576,7 +1577,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   // zve32x is broken for partial_reduce_umla, but let's not make it worse.
   if (Subtarget.hasStdExtZvqdotq() && Subtarget.getELen() >= 64) {
     static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
-                                      ISD::PARTIAL_REDUCE_UMLA};
+                                      ISD::PARTIAL_REDUCE_UMLA,
+                                      ISD::PARTIAL_REDUCE_SUMLA};
     setPartialReduceMLAAction(MLAOps, MVT::nxv1i32, MVT::nxv4i8, Custom);
     setPartialReduceMLAAction(MLAOps, MVT::nxv2i32, MVT::nxv8i8, Custom);
     setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv16i8, Custom);
@@ -5110,9 +5112,23 @@ static SDValue lowerVZIP(unsigned Opc, SDValue Op0, SDValue Op1,
     Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
   }
 
-  auto [Mask, VL] = getDefaultVLOps(IntVT, ContainerVT, DL, DAG, Subtarget);
-  SDValue Passthru = DAG.getUNDEF(ContainerVT);
-  SDValue Res = DAG.getNode(Opc, DL, ContainerVT, Op0, Op1, Passthru, Mask, VL);
+  MVT InnerVT = ContainerVT;
+  auto [Mask, VL] = getDefaultVLOps(IntVT, InnerVT, DL, DAG, Subtarget);
+  if (Op1.isUndef() && ContainerVT.bitsGT(getLMUL1VT(ContainerVT)) &&
+      (RISCVISD::RI_VUNZIP2A_VL == Opc || RISCVISD::RI_VUNZIP2B_VL == Opc)) {
+    InnerVT = ContainerVT.getHalfNumVectorElementsVT();
+    VL = DAG.getConstant(VT.getVectorNumElements() / 2, DL,
+                         Subtarget.getXLenVT());
+    Mask = getAllOnesMask(InnerVT, VL, DL, DAG);
+    unsigned HighIdx = InnerVT.getVectorElementCount().getKnownMinValue();
+    Op1 = DAG.getExtractSubvector(DL, InnerVT, Op0, HighIdx);
+    Op0 = DAG.getExtractSubvector(DL, InnerVT, Op0, 0);
+  }
+
+  SDValue Passthru = DAG.getUNDEF(InnerVT);
+  SDValue Res = DAG.getNode(Opc, DL, InnerVT, Op0, Op1, Passthru, Mask, VL);
+  if (InnerVT.bitsLT(ContainerVT))
+    Res = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), Res, 0);
   if (IntVT.isFixedLengthVector())
     Res = convertFromScalableVector(IntVT, Res, DAG, Subtarget);
   Res = DAG.getBitcast(VT, Res);
@@ -5808,6 +5824,53 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     }
   }
 
+  // If this is a deinterleave(2), try using vunzip{a,b}.  This mostly catches
+  // e64 which can't match above.
+  unsigned Index = 0;
+  if (Subtarget.hasVendorXRivosVizip() &&
+      ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index) &&
+      1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
+    unsigned Opc =
+        Index == 0 ? RISCVISD::RI_VUNZIP2A_VL : RISCVISD::RI_VUNZIP2B_VL;
+    if (V2.isUndef())
+      return lowerVZIP(Opc, V1, V2, DL, DAG, Subtarget);
+    if (auto VLEN = Subtarget.getRealVLen();
+        VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0)
+      return lowerVZIP(Opc, V1, V2, DL, DAG, Subtarget);
+    if (SDValue Src = foldConcatVector(V1, V2)) {
+      EVT NewVT = VT.getDoubleNumVectorElementsVT();
+      Src = DAG.getExtractSubvector(DL, NewVT, Src, 0);
+      SDValue Res =
+          lowerVZIP(Opc, Src, DAG.getUNDEF(NewVT), DL, DAG, Subtarget);
+      return DAG.getExtractSubvector(DL, VT, Res, 0);
+    }
+    // Deinterleave each source and concatenate them, or concat first, then
+    // deinterleave.
+    if (1 < count_if(Mask,
+                     [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
+        1 < count_if(Mask,
+                     [&Mask](int Idx) { return Idx >= (int)Mask.size(); })) {
+
+      const unsigned EltSize = VT.getScalarSizeInBits();
+      const unsigned MinVLMAX = Subtarget.getRealMinVLen() / EltSize;
+      if (NumElts < MinVLMAX) {
+        MVT ConcatVT = VT.getDoubleNumVectorElementsVT();
+        SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
+        SDValue Res =
+            lowerVZIP(Opc, Concat, DAG.getUNDEF(ConcatVT), DL, DAG, Subtarget);
+        return DAG.getExtractSubvector(DL, VT, Res, 0);
+      }
+
+      SDValue Lo = lowerVZIP(Opc, V1, DAG.getUNDEF(VT), DL, DAG, Subtarget);
+      SDValue Hi = lowerVZIP(Opc, V2, DAG.getUNDEF(VT), DL, DAG, Subtarget);
+
+      MVT SubVT = VT.getHalfNumVectorElementsVT();
+      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
+                         DAG.getExtractSubvector(DL, SubVT, Lo, 0),
+                         DAG.getExtractSubvector(DL, SubVT, Hi, 0));
+    }
+  }
+
   if (SDValue V =
           lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
     return V;
@@ -8099,6 +8162,12 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerGET_ROUNDING(Op, DAG);
   case ISD::SET_ROUNDING:
     return lowerSET_ROUNDING(Op, DAG);
+  case ISD::GET_FPENV:
+    return lowerGET_FPENV(Op, DAG);
+  case ISD::SET_FPENV:
+    return lowerSET_FPENV(Op, DAG);
+  case ISD::RESET_FPENV:
+    return lowerRESET_FPENV(Op, DAG);
   case ISD::EH_DWARF_CFA:
     return lowerEH_DWARF_CFA(Op, DAG);
   case ISD::VP_MERGE:
@@ -8259,6 +8328,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerADJUST_TRAMPOLINE(Op, DAG);
   case ISD::PARTIAL_REDUCE_UMLA:
   case ISD::PARTIAL_REDUCE_SMLA:
+  case ISD::PARTIAL_REDUCE_SUMLA:
     return lowerPARTIAL_REDUCE_MLA(Op, DAG);
   }
 }
@@ -8303,9 +8373,23 @@ SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
   //     16: <StaticChainOffset>
   //     24: <FunctionAddressOffset>
   //     32:
-
-  constexpr unsigned StaticChainOffset = 16;
-  constexpr unsigned FunctionAddressOffset = 24;
+  // Offset with branch control flow protection enabled:
+  //      0: lpad    <imm20>
+  //      4: auipc   t3, 0
+  //      8: ld      t2, 28(t3)
+  //     12: ld      t3, 20(t3)
+  //     16: jalr    t2
+  //     20: <StaticChainOffset>
+  //     28: <FunctionAddressOffset>
+  //     36:
+
+  const bool HasCFBranch =
+      Subtarget.hasStdExtZicfilp() &&
+      DAG.getMachineFunction().getFunction().getParent()->getModuleFlag(
+          "cf-protection-branch");
+  const unsigned StaticChainIdx = HasCFBranch ? 5 : 4;
+  const unsigned StaticChainOffset = StaticChainIdx * 4;
+  const unsigned FunctionAddressOffset = StaticChainOffset + 8;
 
   const MCSubtargetInfo *STI = getTargetMachine().getMCSubtargetInfo();
   assert(STI);
@@ -8317,38 +8401,70 @@ SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
     return Encoding;
   };
 
-  SDValue OutChains[6];
-
-  uint32_t Encodings[] = {
-      // auipc t2, 0
-      // Loads the current PC into t2.
-      GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X7).addImm(0)),
-      // ld t0, 24(t2)
-      // Loads the function address into t0. Note that we are using offsets
-      // pc-relative to the first instruction of the trampoline.
-      GetEncoding(
-          MCInstBuilder(RISCV::LD).addReg(RISCV::X5).addReg(RISCV::X7).addImm(
-              FunctionAddressOffset)),
-      // ld t2, 16(t2)
-      // Load the value of the static chain.
-      GetEncoding(
-          MCInstBuilder(RISCV::LD).addReg(RISCV::X7).addReg(RISCV::X7).addImm(
-              StaticChainOffset)),
-      // jalr t0
-      // Jump to the function.
-      GetEncoding(MCInstBuilder(RISCV::JALR)
-                      .addReg(RISCV::X0)
-                      .addReg(RISCV::X5)
-                      .addImm(0))};
+  SmallVector<SDValue> OutChains;
+
+  SmallVector<uint32_t> Encodings;
+  if (!HasCFBranch) {
+    Encodings.append(
+        {// auipc t2, 0
+         // Loads the current PC into t2.
+         GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X7).addImm(0)),
+         // ld t0, 24(t2)
+         // Loads the function address into t0. Note that we are using offsets
+         // pc-relative to the first instruction of the trampoline.
+         GetEncoding(MCInstBuilder(RISCV::LD)
+                         .addReg(RISCV::X5)
+                         .addReg(RISCV::X7)
+                         .addImm(FunctionAddressOffset)),
+         // ld t2, 16(t2)
+         // Load the value of the static chain.
+         GetEncoding(MCInstBuilder(RISCV::LD)
+                         .addReg(RISCV::X7)
+                         .addReg(RISCV::X7)
+                         .addImm(StaticChainOffset)),
+         // jalr t0
+         // Jump to the function.
+         GetEncoding(MCInstBuilder(RISCV::JALR)
+                         .addReg(RISCV::X0)
+                         .addReg(RISCV::X5)
+                         .addImm(0))});
+  } else {
+    Encodings.append(
+        {// auipc x0, <imm20> (lpad <imm20>)
+         // Landing pad.
+         GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X0).addImm(0)),
+         // auipc t3, 0
+         // Loads the current PC into t3.
+         GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X28).addImm(0)),
+         // ld t2, (FunctionAddressOffset - 4)(t3)
+         // Loads the function address into t2. Note that we are using offsets
+         // pc-relative to the SECOND instruction of the trampoline.
+         GetEncoding(MCInstBuilder(RISCV::LD)
+                         .addReg(RISCV::X7)
+                         .addReg(RISCV::X28)
+                         .addImm(FunctionAddressOffset - 4)),
+         // ld t3, (StaticChainOffset - 4)(t3)
+         // Load the value of the static chain.
+         GetEncoding(MCInstBuilder(RISCV::LD)
+                         .addReg(RISCV::X28)
+                         .addReg(RISCV::X28)
+                         .addImm(StaticChainOffset - 4)),
+         // jalr t2
+         // Software-guarded jump to the function.
+         GetEncoding(MCInstBuilder(RISCV::JALR)
+                         .addReg(RISCV::X0)
+                         .addReg(RISCV::X7)
+                         .addImm(0))});
+  }
 
   // Store encoded instructions.
   for (auto [Idx, Encoding] : llvm::enumerate(Encodings)) {
     SDValue Addr = Idx > 0 ? DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
                                          DAG.getConstant(Idx * 4, dl, MVT::i64))
                            : Trmp;
-    OutChains[Idx] = DAG.getTruncStore(
+    OutChains.push_back(DAG.getTruncStore(
         Root, dl, DAG.getConstant(Encoding, dl, MVT::i64), Addr,
-        MachinePointerInfo(TrmpAddr, Idx * 4), MVT::i32);
+        MachinePointerInfo(TrmpAddr, Idx * 4), MVT::i32));
   }
 
   // Now store the variable part of the trampoline.
@@ -8364,16 +8480,18 @@ SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
       {StaticChainOffset, StaticChain},
       {FunctionAddressOffset, FunctionAddress},
   };
-  for (auto [Idx, OffsetValue] : llvm::enumerate(OffsetValues)) {
+  for (auto &OffsetValue : OffsetValues) {
     SDValue Addr =
         DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
                     DAG.getConstant(OffsetValue.Offset, dl, MVT::i64));
     OffsetValue.Addr = Addr;
-    OutChains[Idx + 4] =
+    OutChains.push_back(
         DAG.getStore(Root, dl, OffsetValue.Value, Addr,
-                     MachinePointerInfo(TrmpAddr, OffsetValue.Offset));
+                     MachinePointerInfo(TrmpAddr, OffsetValue.Offset)));
   }
 
+  assert(OutChains.size() == StaticChainIdx + 2 &&
+         "Size of OutChains mismatch");
   SDValue StoreToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
 
   // The end of instructions of trampoline is the same as the static chain
@@ -8400,7 +8518,6 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
   // Currently, only the vqdot and vqdotu case (from zvqdotq) should be legal.
   // TODO: There are many other sub-cases we could potentially lower, are
   // any of them worthwhile?  Ex: via vredsum, vwredsum, vwwmaccu, etc..
-  // TODO: PARTIAL_REDUCE_*MLA can't represent a vqdotsu currently.
   SDLoc DL(Op);
   MVT VT = Op.getSimpleValueType();
   SDValue Accum = Op.getOperand(0);
@@ -8427,8 +8544,20 @@ SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
     B = convertToScalableVector(ContainerVT, B, DAG, Subtarget);
   }
 
-  bool IsSigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_SMLA;
-  unsigned Opc = IsSigned ? RISCVISD::VQDOT_VL : RISCVISD::VQDOTU_VL;
+  unsigned Opc;
+  switch (Op.getOpcode()) {
+  case ISD::PARTIAL_REDUCE_SMLA:
+    Opc = RISCVISD::VQDOT_VL;
+    break;
+  case ISD::PARTIAL_REDUCE_UMLA:
+    Opc = RISCVISD::VQDOTU_VL;
+    break;
+  case ISD::PARTIAL_REDUCE_SUMLA:
+    Opc = RISCVISD::VQDOTSU_VL;
+    break;
+  default:
+    llvm_unreachable("Unexpected opcode");
+  }
   auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
   SDValue Res = DAG.getNode(Opc, DL, ContainerVT, {A, B, Accum, Mask, VL});
   if (VT.isFixedLengthVector())
@@ -8967,6 +9096,32 @@ SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
       return DAG.getNode(ISD::ADD, DL, VT, CMOV, RHSVal);
     }
 
+    // (select c, c1, t) -> (add (czero_nez t - c1, c), c1)
+    // (select c, t, c1) -> (add (czero_eqz t - c1, c), c1)
+    if (isa<ConstantSDNode>(TrueV) != isa<ConstantSDNode>(FalseV)) {
+      bool IsCZERO_NEZ = isa<ConstantSDNode>(TrueV);
+      SDValue ConstVal = IsCZERO_NEZ ? TrueV : FalseV;
+      SDValue RegV = IsCZERO_NEZ ? FalseV : TrueV;
+      int64_t RawConstVal = cast<ConstantSDNode>(ConstVal)->getSExtValue();
+      // Fall back to XORI if Const == -0x800
+      if (RawConstVal == -0x800) {
+        SDValue XorOp = DAG.getNode(ISD::XOR, DL, VT, RegV, ConstVal);
+        SDValue CMOV =
+            DAG.getNode(IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
+                        DL, VT, XorOp, CondV);
+        return DAG.getNode(ISD::XOR, DL, VT, CMOV, ConstVal);
+      }
+      // Efficient only if the constant and its negation fit into `ADDI`
+      // Prefer Add/Sub over Xor since can be compressed for small immediates
+      if (isInt<12>(RawConstVal)) {
+        SDValue SubOp = DAG.getNode(ISD::SUB, DL, VT, RegV, ConstVal);
+        SDValue CMOV =
+            DAG.getNode(IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
+                        DL, VT, SubOp, CondV);
+        return DAG.getNode(ISD::ADD, DL, VT, CMOV, ConstVal);
+      }
+    }
+
     // (select c, t, f) -> (or (czero_eqz t, c), (czero_nez f, c))
     // Unless we have the short forward branch optimization.
     if (!Subtarget.hasConditionalMoveFusion())
@@ -13679,6 +13834,41 @@ SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
                      RMValue);
 }
 
+SDValue RISCVTargetLowering::lowerGET_FPENV(SDValue Op,
+                                            SelectionDAG &DAG) const {
+  const MVT XLenVT = Subtarget.getXLenVT();
+  SDLoc DL(Op);
+  SDValue Chain = Op->getOperand(0);
+  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
+  SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
+  return DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
+}
+
+SDValue RISCVTargetLowering::lowerSET_FPENV(SDValue Op,
+                                            SelectionDAG &DAG) const {
+  const MVT XLenVT = Subtarget.getXLenVT();
+  SDLoc DL(Op);
+  SDValue Chain = Op->getOperand(0);
+  SDValue EnvValue = Op->getOperand(1);
+  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
+
+  EnvValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EnvValue);
+  return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
+                     EnvValue);
+}
+
+SDValue RISCVTargetLowering::lowerRESET_FPENV(SDValue Op,
+                                              SelectionDAG &DAG) const {
+  const MVT XLenVT = Subtarget.getXLenVT();
+  SDLoc DL(Op);
+  SDValue Chain = Op->getOperand(0);
+  SDValue EnvValue = DAG.getRegister(RISCV::X0, XLenVT);
+  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
+
+  return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
+                     EnvValue);
+}
+
 SDValue RISCVTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
                                                SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
@@ -14777,7 +14967,7 @@ static SDValue combineShlAddIAddImpl(SDNode *N, SDValue AddI, SDValue Other,
     return SDValue();
 
   APInt VShift;
-  if (!sd_match(SHLVal, m_BinOp(ISD::SHL, m_Value(), m_ConstInt(VShift))))
+  if (!sd_match(SHLVal, m_OneUse(m_Shl(m_Value(), m_ConstInt(VShift)))))
     return SDValue();
 
   if (VShift.slt(1) || VShift.sgt(3))
@@ -15983,17 +16173,80 @@ static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &D
   return true;
 }
 
+/// Try to map an integer comparison with size > XLEN to vector instructions
+/// before type legalization splits it up into chunks.
+static SDValue
+combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
+                                const SDLoc &DL, SelectionDAG &DAG,
+                                const RISCVSubtarget &Subtarget) {
+  assert(ISD::isIntEqualitySetCC(CC) && "Bad comparison predicate");
+
+  if (!Subtarget.hasVInstructions())
+    return SDValue();
+
+  MVT XLenVT = Subtarget.getXLenVT();
+  EVT OpVT = X.getValueType();
+  // We're looking for an oversized integer equality comparison.
+  if (!OpVT.isScalarInteger())
+    return SDValue();
+
+  unsigned OpSize = OpVT.getSizeInBits();
+  // TODO: Support non-power-of-2 types.
+  if (!isPowerOf2_32(OpSize))
+    return SDValue();
+
+  // The size should be larger than XLen and smaller than the maximum vector
+  // size.
+  if (OpSize <= Subtarget.getXLen() ||
+      OpSize > Subtarget.getRealMinVLen() *
+                   Subtarget.getMaxLMULForFixedLengthVectors())
+    return SDValue();
+
+  // Don't perform this combine if constructing the vector will be expensive.
+  auto IsVectorBitCastCheap = [](SDValue X) {
+    X = peekThroughBitcasts(X);
+    return isa<ConstantSDNode>(X) || X.getValueType().isVector() ||
+           X.getOpcode() == ISD::LOAD;
+  };
+  if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
+    return SDValue();
+
+  if (DAG.getMachineFunction().getFunction().hasFnAttribute(
+          Attribute::NoImplicitFloat))
+    return SDValue();
+
+  unsigned VecSize = OpSize / 8;
+  EVT VecVT = MVT::getVectorVT(MVT::i8, VecSize);
+  EVT CmpVT = MVT::getVectorVT(MVT::i1, VecSize);
+
+  SDValue VecX = DAG.getBitcast(VecVT, X);
+  SDValue VecY = DAG.getBitcast(VecVT, Y);
+  SDValue Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
+  return DAG.getSetCC(DL, VT, DAG.getNode(ISD::VECREDUCE_OR, DL, XLenVT, Cmp),
+                      DAG.getConstant(0, DL, XLenVT), CC);
+}
+
 // Replace (seteq (i64 (and X, 0xffffffff)), C1) with
 // (seteq (i64 (sext_inreg (X, i32)), C1')) where C1' is C1 sign extended from
 // bit 31. Same for setne. C1' may be cheaper to materialize and the sext_inreg
 // can become a sext.w instead of a shift pair.
 static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
+  SDLoc dl(N);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
   EVT OpVT = N0.getValueType();
 
+  ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
+  // Looking for an equality compare.
+  if (!isIntEqualitySetCC(Cond))
+    return SDValue();
+
+  if (SDValue V =
+          combineVectorSizedSetCCEquality(VT, N0, N1, Cond, dl, DAG, Subtarget))
+    return V;
+
   if (OpVT != MVT::i64 || !Subtarget.is64Bit())
     return SDValue();
 
@@ -16008,11 +16261,6 @@ static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
       N0.getConstantOperandVal(1) != UINT64_C(0xffffffff))
     return SDValue();
 
-  // Looking for an equality compare.
-  ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
-  if (!isIntEqualitySetCC(Cond))
-    return SDValue();
-
   // Don't do this if the sign bit is provably zero, it will be turned back into
   // an AND.
   APInt SignMask = APInt::getOneBitSet(64, 31);
@@ -16021,7 +16269,6 @@ static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
 
   const APInt &C1 = N1C->getAPIntValue();
 
-  SDLoc dl(N);
   // If the constant is larger than 2^32 - 1 it is impossible for both sides
   // to be equal.
   if (C1.getActiveBits() > 32)
@@ -18251,31 +18498,6 @@ static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
                      DAG.getBuildVector(VT, DL, RHSOps));
 }
 
-static SDValue lowerVQDOT(unsigned Opc, SDValue Op0, SDValue Op1,
-                          const SDLoc &DL, SelectionDAG &DAG,
-                          const RISCVSubtarget &Subtarget) {
-  assert(RISCVISD::VQDOT_VL == Opc || RISCVISD::VQDOTU_VL == Opc ||
-         RISCVISD::VQDOTSU_VL == Opc);
-  MVT VT = Op0.getSimpleValueType();
-  assert(VT == Op1.getSimpleValueType() &&
-         VT.getVectorElementType() == MVT::i32);
-
-  SDValue Passthru = DAG.getConstant(0, DL, VT);
-  MVT ContainerVT = VT;
-  if (VT.isFixedLengthVector()) {
-    ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
-    Passthru = convertToScalableVector(ContainerVT, Passthru, DAG, Subtarget);
-    Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
-    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
-  }
-  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
-  SDValue LocalAccum = DAG.getNode(Opc, DL, ContainerVT,
-                                   {Op0, Op1, Passthru, Mask, VL});
-  if (VT.isFixedLengthVector())
-    return convertFromScalableVector(VT, LocalAccum, DAG, Subtarget);
-  return LocalAccum;
-}
-
 static MVT getQDOTXResultType(MVT OpVT) {
   ElementCount OpEC = OpVT.getVectorElementCount();
   assert(OpEC.isKnownMultipleOf(4) && OpVT.getVectorElementType() == MVT::i8);
@@ -18334,61 +18556,62 @@ static SDValue foldReduceOperandViaVQDOT(SDValue InVec, const SDLoc &DL,
     }
   }
 
-  // reduce (zext a) <--> reduce (mul zext a. zext 1)
-  // reduce (sext a) <--> reduce (mul sext a. sext 1)
+  // zext a <--> partial_reduce_umla 0, a, 1
+  // sext a <--> partial_reduce_smla 0, a, 1
   if (InVec.getOpcode() == ISD::ZERO_EXTEND ||
       InVec.getOpcode() == ISD::SIGN_EXTEND) {
     SDValue A = InVec.getOperand(0);
-    if (A.getValueType().getVectorElementType() != MVT::i8 ||
-        !TLI.isTypeLegal(A.getValueType()))
+    EVT OpVT = A.getValueType();
+    if (OpVT.getVectorElementType() != MVT::i8 || !TLI.isTypeLegal(OpVT))
       return SDValue();
 
     MVT ResVT = getQDOTXResultType(A.getSimpleValueType());
-    A = DAG.getBitcast(ResVT, A);
-    SDValue B = DAG.getConstant(0x01010101, DL, ResVT);
-
+    SDValue B = DAG.getConstant(0x1, DL, OpVT);
     bool IsSigned = InVec.getOpcode() == ISD::SIGN_EXTEND;
-    unsigned Opc = IsSigned ? RISCVISD::VQDOT_VL : RISCVISD::VQDOTU_VL;
-    return lowerVQDOT(Opc, A, B, DL, DAG, Subtarget);
+    unsigned Opc =
+        IsSigned ? ISD::PARTIAL_REDUCE_SMLA : ISD::PARTIAL_REDUCE_UMLA;
+    return DAG.getNode(Opc, DL, ResVT, {DAG.getConstant(0, DL, ResVT), A, B});
   }
 
-  // mul (sext, sext) -> vqdot
-  // mul (zext, zext) -> vqdotu
-  // mul (sext, zext) -> vqdotsu
-  // mul (zext, sext) -> vqdotsu (swapped)
-  // TODO: Improve .vx handling - we end up with a sub-vector insert
-  // which confuses the splat pattern matching.  Also, match vqdotus.vx
+  // mul (sext a, sext b) -> partial_reduce_smla 0, a, b
+  // mul (zext a, zext b) -> partial_reduce_umla 0, a, b
+  // mul (sext a, zext b) -> partial_reduce_ssmla 0, a, b
+  // mul (zext a, sext b) -> partial_reduce_smla 0, b, a (swapped)
   if (InVec.getOpcode() != ISD::MUL)
     return SDValue();
 
   SDValue A = InVec.getOperand(0);
   SDValue B = InVec.getOperand(1);
-  unsigned Opc = 0;
-  if (A.getOpcode() == B.getOpcode()) {
-    if (A.getOpcode() == ISD::SIGN_EXTEND)
-      Opc = RISCVISD::VQDOT_VL;
-    else if (A.getOpcode() == ISD::ZERO_EXTEND)
-      Opc = RISCVISD::VQDOTU_VL;
-    else
-      return SDValue();
-  } else {
-    if (B.getOpcode() != ISD::ZERO_EXTEND)
-      std::swap(A, B);
-    if (A.getOpcode() != ISD::SIGN_EXTEND || B.getOpcode() != ISD::ZERO_EXTEND)
-      return SDValue();
-    Opc = RISCVISD::VQDOTSU_VL;
-  }
-  assert(Opc);
 
-  if (A.getOperand(0).getValueType().getVectorElementType() != MVT::i8 ||
-      A.getOperand(0).getValueType() != B.getOperand(0).getValueType() ||
+  if (!ISD::isExtOpcode(A.getOpcode()))
+    return SDValue();
+
+  EVT OpVT = A.getOperand(0).getValueType();
+  if (OpVT.getVectorElementType() != MVT::i8 ||
+      OpVT != B.getOperand(0).getValueType() ||
       !TLI.isTypeLegal(A.getValueType()))
     return SDValue();
 
-  MVT ResVT = getQDOTXResultType(A.getOperand(0).getSimpleValueType());
-  A = DAG.getBitcast(ResVT, A.getOperand(0));
-  B = DAG.getBitcast(ResVT, B.getOperand(0));
-  return lowerVQDOT(Opc, A, B, DL, DAG, Subtarget);
+  unsigned Opc;
+  if (A.getOpcode() == ISD::SIGN_EXTEND && B.getOpcode() == ISD::SIGN_EXTEND)
+    Opc = ISD::PARTIAL_REDUCE_SMLA;
+  else if (A.getOpcode() == ISD::ZERO_EXTEND &&
+           B.getOpcode() == ISD::ZERO_EXTEND)
+    Opc = ISD::PARTIAL_REDUCE_UMLA;
+  else if (A.getOpcode() == ISD::SIGN_EXTEND &&
+           B.getOpcode() == ISD::ZERO_EXTEND)
+    Opc = ISD::PARTIAL_REDUCE_SUMLA;
+  else if (A.getOpcode() == ISD::ZERO_EXTEND &&
+           B.getOpcode() == ISD::SIGN_EXTEND) {
+    Opc = ISD::PARTIAL_REDUCE_SUMLA;
+    std::swap(A, B);
+  } else
+    return SDValue();
+
+  MVT ResVT = getQDOTXResultType(OpVT.getSimpleVT());
+  return DAG.getNode(
+      Opc, DL, ResVT,
+      {DAG.getConstant(0, DL, ResVT), A.getOperand(0), B.getOperand(0)});
 }
 
 static SDValue performVECREDUCECombine(SDNode *N, SelectionDAG &DAG,
@@ -20354,8 +20577,8 @@ bool RISCVTargetLowering::isDesirableToCommuteWithShift(
     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
 
     // Bail if we might break a sh{1,2,3}add pattern.
-    if (Subtarget.hasStdExtZba() && C2 && C2->getZExtValue() >= 1 &&
-        C2->getZExtValue() <= 3 && N->hasOneUse() &&
+    if ((Subtarget.hasStdExtZba() || Subtarget.hasVendorXAndesPerf()) && C2 &&
+        C2->getZExtValue() >= 1 && C2->getZExtValue() <= 3 && N->hasOneUse() &&
         N->user_begin()->getOpcode() == ISD::ADD &&
         !isUsedByLdSt(*N->user_begin(), nullptr) &&
         !isa<ConstantSDNode>(N->user_begin()->getOperand(1)))
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.h b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 2ea2bf656ffd..417d684a6238 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -544,6 +544,9 @@ class RISCVTargetLowering : public TargetLowering {
                                             unsigned ExtendOpc) const;
   SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerRESET_FPENV(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) const;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index e1c69c69a99c..9a513891b765 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -1253,7 +1253,7 @@ void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
     return;
   }
 
-  if (RISCV::isFaultFirstLoad(MI)) {
+  if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
     // Update AVL to vl-output of the fault first load.
     assert(MI.getOperand(1).getReg().isVirtual());
     if (LIS) {
@@ -1698,13 +1698,24 @@ void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
           MI.getOperand(0).setReg(DefReg);
           MI.getOperand(0).setIsDead(false);
 
+          // Move the AVL from NextMI to MI
+          dropAVLUse(MI.getOperand(1));
+          if (NextMI->getOperand(1).isImm())
+            MI.getOperand(1).ChangeToImmediate(NextMI->getOperand(1).getImm());
+          else
+            MI.getOperand(1).ChangeToRegister(NextMI->getOperand(1).getReg(),
+                                              false);
+          dropAVLUse(NextMI->getOperand(1));
+
           // The def of DefReg moved to MI, so extend the LiveInterval up to
           // it.
           if (DefReg.isVirtual() && LIS) {
             LiveInterval &DefLI = LIS->getInterval(DefReg);
             SlotIndex MISlot = LIS->getInstructionIndex(MI).getRegSlot();
-            VNInfo *DefVNI = DefLI.getVNInfoAt(DefLI.beginIndex());
-            LiveInterval::Segment S(MISlot, DefLI.beginIndex(), DefVNI);
+            SlotIndex NextMISlot =
+                LIS->getInstructionIndex(*NextMI).getRegSlot();
+            VNInfo *DefVNI = DefLI.getVNInfoAt(NextMISlot);
+            LiveInterval::Segment S(MISlot, NextMISlot, DefVNI);
             DefLI.addSegment(S);
             DefVNI->def = MISlot;
             // Mark DefLI as spillable if it was previously unspillable
@@ -1715,13 +1726,6 @@ void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
             LIS->shrinkToUses(&DefLI);
           }
 
-          dropAVLUse(MI.getOperand(1));
-          if (NextMI->getOperand(1).isImm())
-            MI.getOperand(1).ChangeToImmediate(NextMI->getOperand(1).getImm());
-          else
-            MI.getOperand(1).ChangeToRegister(NextMI->getOperand(1).getReg(),
-                                              false);
-
           MI.setDesc(NextMI->getDesc());
         }
         MI.getOperand(2).setImm(NextMI->getOperand(2).getImm());
@@ -1752,7 +1756,7 @@ void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
 void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
   for (auto I = MBB.begin(), E = MBB.end(); I != E;) {
     MachineInstr &MI = *I++;
-    if (RISCV::isFaultFirstLoad(MI)) {
+    if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
       Register VLOutput = MI.getOperand(1).getReg();
       assert(VLOutput.isVirtual());
       if (!MI.getOperand(1).isDead()) {
@@ -1770,6 +1774,7 @@ void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
       }
       // We don't use the vl output of the VLEFF/VLSEGFF anymore.
       MI.getOperand(1).setReg(RISCV::X0);
+      MI.addRegisterDefined(RISCV::VL, MRI->getTargetRegisterInfo());
     }
   }
 }
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index 8d56637d8b28..088a6923fadb 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -99,6 +99,8 @@ def VMConstraint  : RISCVVConstraint<0b100>;
 //
 // * vcompress: The destination vector register group cannot overlap the
 // source vector register group or the source mask register
+def WidenVNoMask : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VS1Constraint.Value)>;
 def WidenV       : RISCVVConstraint<!or(VS2Constraint.Value,
                                         VS1Constraint.Value,
                                         VMConstraint.Value)>;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 9dc434b2c919..d9ef911b9a32 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2694,7 +2694,11 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         CASE_OPERAND_UIMM(8)
         CASE_OPERAND_UIMM(10)
         CASE_OPERAND_UIMM(12)
+        CASE_OPERAND_UIMM(16)
         CASE_OPERAND_UIMM(20)
+        CASE_OPERAND_UIMM(32)
+        CASE_OPERAND_UIMM(48)
+        CASE_OPERAND_UIMM(64)
           // clang-format on
         case RISCVOp::OPERAND_UIMM2_LSB0:
           Ok = isShiftedUInt<1, 1>(Imm);
@@ -3569,24 +3573,15 @@ std::string RISCVInstrInfo::createMIROperandComment(
 #define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL)                                 \
   RISCV::PseudoV##OP##_##TYPE##_##LMUL
 
-#define CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)                                     \
-  CASE_VMA_OPCODE_COMMON(OP, TYPE, M1):                                        \
+#define CASE_VMA_OPCODE_LMULS(OP, TYPE)                                        \
+  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8):                                       \
+  case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4):                                  \
+  case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2):                                  \
+  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1):                                   \
   case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2):                                   \
   case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4):                                   \
   case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
 
-#define CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)                                    \
-  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2):                                       \
-  case CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)
-
-#define CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)                                    \
-  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4):                                       \
-  case CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)
-
-#define CASE_VMA_OPCODE_LMULS(OP, TYPE)                                        \
-  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8):                                       \
-  case CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)
-
 // VFMA instructions are SEW specific.
 #define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW)                           \
   RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
@@ -3692,7 +3687,8 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
     // If the tail policy is undisturbed we can't commute.
     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
-    if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
+    if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
+         1) == 0)
       return false;
 
     // For these instructions we can only swap operand 1 and operand 3 by
@@ -3712,7 +3708,8 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
     // If the tail policy is undisturbed we can't commute.
     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
-    if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
+    if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
+         1) == 0)
       return false;
 
     // For these instructions we have more freedom. We can commute with the
@@ -3784,29 +3781,15 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
     Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL;                             \
     break;
 
-#define CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)                    \
+#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)                       \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8)                       \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4)                       \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2)                       \
   CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1)                        \
   CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2)                        \
   CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4)                        \
   CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
 
-#define CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)                   \
-  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2)                       \
-  CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
-
-#define CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)                   \
-  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4)                       \
-  CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
-
-#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)                       \
-  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8)                       \
-  CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
-
-#define CASE_VMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)                            \
-  CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16)                       \
-  CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32)                       \
-  CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64)
-
 // VFMA depends on SEW.
 #define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW)          \
   case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW:                        \
@@ -3823,18 +3806,14 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW)                 \
   CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
 
-#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)                               \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16)                     \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32)                     \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
-
 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)             \
   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW)                 \
   CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
 
-#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE, SEW)                 \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8, SEW)                 \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)
+#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)                               \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16)                     \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32)                     \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
 
 #define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)                           \
   CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16)                 \
@@ -3957,6 +3936,15 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
 }
 
+#undef CASE_VMA_CHANGE_OPCODE_COMMON
+#undef CASE_VMA_CHANGE_OPCODE_LMULS
+#undef CASE_VFMA_CHANGE_OPCODE_COMMON
+#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
+#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
+#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
+#undef CASE_VFMA_CHANGE_OPCODE_VV
+#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
+
 #undef CASE_RVV_OPCODE_UNMASK_LMUL
 #undef CASE_RVV_OPCODE_MASK_LMUL
 #undef CASE_RVV_OPCODE_LMUL
@@ -3968,9 +3956,6 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
 #undef CASE_RVV_OPCODE
 
 #undef CASE_VMA_OPCODE_COMMON
-#undef CASE_VMA_OPCODE_LMULS_M1
-#undef CASE_VMA_OPCODE_LMULS_MF2
-#undef CASE_VMA_OPCODE_LMULS_MF4
 #undef CASE_VMA_OPCODE_LMULS
 #undef CASE_VFMA_OPCODE_COMMON
 #undef CASE_VFMA_OPCODE_LMULS_M1
@@ -4220,38 +4205,32 @@ bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
 #define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL)                                    \
   RISCV::PseudoV##OP##_##LMUL##_TIED
 
-#define CASE_WIDEOP_OPCODE_LMULS_MF4(OP)                                       \
-  CASE_WIDEOP_OPCODE_COMMON(OP, MF4):                                          \
+#define CASE_WIDEOP_OPCODE_LMULS(OP)                                           \
+  CASE_WIDEOP_OPCODE_COMMON(OP, MF8):                                          \
+  case CASE_WIDEOP_OPCODE_COMMON(OP, MF4):                                     \
   case CASE_WIDEOP_OPCODE_COMMON(OP, MF2):                                     \
   case CASE_WIDEOP_OPCODE_COMMON(OP, M1):                                      \
   case CASE_WIDEOP_OPCODE_COMMON(OP, M2):                                      \
   case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
 
-#define CASE_WIDEOP_OPCODE_LMULS(OP)                                           \
-  CASE_WIDEOP_OPCODE_COMMON(OP, MF8):                                          \
-  case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
-
 #define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL)                             \
   case RISCV::PseudoV##OP##_##LMUL##_TIED:                                     \
     NewOpc = RISCV::PseudoV##OP##_##LMUL;                                      \
     break;
 
-#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)                                \
+#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                    \
+  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8)                                    \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4)                                    \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2)                                    \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1)                                     \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2)                                     \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
 
-#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                    \
-  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8)                                    \
-  CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
-
 // FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
 #define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW)                            \
   RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
 
-#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP)                                    \
+#define CASE_FP_WIDEOP_OPCODE_LMULS(OP)                                        \
   CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16):                                  \
   case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16):                             \
   case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32):                             \
@@ -4267,7 +4246,7 @@ bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
     NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW;                              \
     break;
 
-#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)                             \
+#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                 \
   CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16)                            \
   CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16)                            \
   CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32)                            \
@@ -4277,9 +4256,6 @@ bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
   CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32)                             \
   CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)                             \
   CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32)                             \
-
-#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                 \
-  CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
 // clang-format on
 
 MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
@@ -4289,8 +4265,8 @@ MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     return nullptr;
-  case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV):
-  case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV): {
+  case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
+  case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
            MI.getNumExplicitOperands() == 7 &&
            "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
@@ -4303,8 +4279,8 @@ MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
     switch (MI.getOpcode()) {
     default:
       llvm_unreachable("Unexpected opcode");
-    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWADD_WV)
-    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWSUB_WV)
+    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV)
+    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV)
     }
     // clang-format on
 
@@ -4327,7 +4303,8 @@ MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
     // If the tail policy is undisturbed we can't convert.
     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
            MI.getNumExplicitOperands() == 6);
-    if ((MI.getOperand(5).getImm() & 1) == 0)
+    if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
+         1) == 0)
       return nullptr;
 
     // clang-format off
@@ -4383,15 +4360,12 @@ MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
 }
 
 #undef CASE_WIDEOP_OPCODE_COMMON
-#undef CASE_WIDEOP_OPCODE_LMULS_MF4
 #undef CASE_WIDEOP_OPCODE_LMULS
 #undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
-#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4
 #undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
 #undef CASE_FP_WIDEOP_OPCODE_COMMON
-#undef CASE_FP_WIDEOP_OPCODE_LMULS_MF4
+#undef CASE_FP_WIDEOP_OPCODE_LMULS
 #undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
-#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4
 #undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
 
 void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -4564,11 +4538,6 @@ RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
   }
 }
 
-bool RISCV::isFaultFirstLoad(const MachineInstr &MI) {
-  return MI.getNumExplicitDefs() == 2 &&
-         MI.modifiesRegister(RISCV::VL, /*TRI=*/nullptr) && !MI.isInlineAsm();
-}
-
 bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
   int16_t MI1FrmOpIdx =
       RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);
@@ -4582,7 +4551,7 @@ bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
 }
 
 std::optional<unsigned>
-RISCV::getVectorLowDemandedScalarBits(uint16_t Opcode, unsigned Log2SEW) {
+RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
   switch (Opcode) {
   default:
     return std::nullopt;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index b099acd81e99..020be91e90e0 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -357,8 +357,6 @@ bool isRVVSpill(const MachineInstr &MI);
 std::optional<std::pair<unsigned, unsigned>>
 isRVVSpillForZvlsseg(unsigned Opcode);
 
-bool isFaultFirstLoad(const MachineInstr &MI);
-
 // Return true if both input instructions have equal rounding mode. If at least
 // one of the instructions does not have rounding mode, false will be returned.
 bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2);
@@ -366,7 +364,7 @@ bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2);
 // If \p Opcode is a .vx vector instruction, returns the lower number of bits
 // that are used from the scalar .x operand for a given \p Log2SEW. Otherwise
 // returns null.
-std::optional<unsigned> getVectorLowDemandedScalarBits(uint16_t Opcode,
+std::optional<unsigned> getVectorLowDemandedScalarBits(unsigned Opcode,
                                                        unsigned Log2SEW);
 
 // Returns the MC opcode of RVV pseudo instruction.
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index de424b3be852..70fad925cf07 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -341,21 +341,33 @@ def simm12 : RISCVSImmLeafOp<12> {
 def simm12_no6 : ImmLeaf<XLenVT, [{
   return isInt<12>(Imm) && !isInt<6>(Imm) && isInt<12>(-Imm);}]>;
 
-// A 13-bit signed immediate where the least significant bit is zero.
-def bare_simm13_lsb0 : Operand<OtherVT> {
-  let ParserMatchClass = BareSImmNLsb0AsmOperand<13>;
-  let PrintMethod = "printBranchOperand";
-  let EncoderMethod = "getImmOpValueAsrN<1>";
-  let DecoderMethod = "decodeSImmOperandAndLslN<13, 1>";
+class BareSImm13Lsb0MaybeSym : Operand<OtherVT> {
   let MCOperandPredicate = [{
     int64_t Imm;
     if (MCOp.evaluateAsConstantImm(Imm))
       return isShiftedInt<12, 1>(Imm);
     return MCOp.isBareSymbolRef();
   }];
+}
+
+// A 13-bit signed immediate where the least significant bit is zero. The ImmLeaf
+// is needed so that the CompressInstEmitter can correctly add checks for the
+// compress patterns that involve instructions that use this operand. Similar to
+// bare_simm9_lsb0 in RISCVInstrInfoC.td.
+def bare_simm13_lsb0 : BareSImm13Lsb0MaybeSym,
+                       ImmLeaf<XLenVT, [{return isShiftedInt<12, 1>(Imm);}]> {
+  let ParserMatchClass = BareSImmNLsb0AsmOperand<13>;
+  let PrintMethod = "printBranchOperand";
+  let EncoderMethod = "getImmOpValueAsrN<1>";
+  let DecoderMethod = "decodeSImmOperandAndLslN<13, 1>";
   let OperandType = "OPERAND_PCREL";
 }
 
+// We need this (sort of) duplicate definition since adding ImmLeaf to
+// bare_simm13_lsb0 above makes it not sit well with codegen patterns where it
+// is used to match with a basic block (eg. BccPat<>).
+def bare_simm13_lsb0_bb : BareSImm13Lsb0MaybeSym;
+
 class UImm20OperandMaybeSym : RISCVUImmOp<20> {
   let MCOperandPredicate = [{
     int64_t Imm;
@@ -1433,11 +1445,7 @@ def : PatGprUimmLog2XLen<sra, SRAI>;
 // Select 'or' as ADDI if the immediate bits are known to be 0 in $rs1. This
 // can improve compressibility.
 def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
-  if (N->getFlags().hasDisjoint())
-    return true;
-  KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
-  KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
-  return KnownBits::haveNoCommonBitsSet(Known0, Known1);
+  return orIsAdd(N);
 }]>;
 def : PatGprSimm12<or_is_add, ADDI>;
 
@@ -1660,10 +1668,10 @@ multiclass SelectCC_GPR_riirr<DAGOperand valty, DAGOperand imm> {
 // Match `riscv_brcc` and lower to the appropriate RISC-V branch instruction.
 multiclass BccPat<CondCode Cond, RVInstB Inst> {
   def : Pat<(riscv_brcc (XLenVT GPR:$rs1), GPR:$rs2, Cond, bb:$imm12),
-            (Inst GPR:$rs1, GPR:$rs2, bare_simm13_lsb0:$imm12)>;
+            (Inst GPR:$rs1, GPR:$rs2, bare_simm13_lsb0_bb:$imm12)>;
   // Explicitly select 0 to X0. The register coalescer doesn't always do it.
   def : Pat<(riscv_brcc (XLenVT GPR:$rs1), 0, Cond, bb:$imm12),
-            (Inst GPR:$rs1, (XLenVT X0), bare_simm13_lsb0:$imm12)>;
+            (Inst GPR:$rs1, (XLenVT X0), bare_simm13_lsb0_bb:$imm12)>;
 }
 
 class BrccCompressOpt<CondCode Cond, RVInstB Inst>
@@ -2029,6 +2037,13 @@ let hasSideEffects = true in {
 def ReadFFLAGS : ReadSysReg<SysRegFFLAGS, [FFLAGS]>;
 def WriteFFLAGS : WriteSysReg<SysRegFFLAGS, [FFLAGS]>;
 }
+
+let hasPostISelHook = 1 in {
+def ReadFCSR : ReadSysReg<SysRegFCSR, [FRM, FFLAGS]>;
+def WriteFCSR : WriteSysReg<SysRegFCSR, [FRM, FFLAGS]>;
+def WriteFCSRImm : WriteSysRegImm<SysRegFCSR, [FRM, FFLAGS]>;
+}
+
 /// Other pseudo-instructions
 
 // Pessimistically assume the stack pointer will be clobbered
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 281f8d55932b..9c03c7c83af0 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -783,6 +783,15 @@ class GetVTypePredicates<VTypeInfo vti> {
                                      true : [HasVInstructions]);
 }
 
+class GetVTypeMinimalPredicates<VTypeInfo vti> {
+  list<Predicate> Predicates = !cond(!eq(vti.Scalar, f16) : [HasVInstructionsF16Minimal],
+                                     !eq(vti.Scalar, bf16) : [HasVInstructionsBF16Minimal],
+                                     !eq(vti.Scalar, f32) : [HasVInstructionsAnyF],
+                                     !eq(vti.Scalar, f64) : [HasVInstructionsF64],
+                                     !eq(vti.SEW, 64) : [HasVInstructionsI64],
+                                     true : [HasVInstructions]);
+}
+
 class VPseudoUSLoadNoMask<VReg RetClass,
                           int EEW,
                           DAGOperand sewop = sew> :
@@ -4568,7 +4577,7 @@ multiclass VPatUnaryS_M<string intrinsic_name,
 multiclass VPatUnaryV_V_AnyMask<string intrinsic, string instruction,
                                 list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
-    let Predicates = GetVTypePredicates<vti>.Predicates in
+    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
     def : VPatUnaryAnyMask<intrinsic, instruction, "VM",
                            vti.Vector, vti.Vector, vti.Mask,
                            vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
@@ -4887,7 +4896,7 @@ multiclass VPatBinaryV_VV_INT<string intrinsic, string instruction,
                               list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
     defvar ivti = GetIntVTypeInfo<vti>.Vti;
-    let Predicates = GetVTypePredicates<vti>.Predicates in
+    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
     defm : VPatBinary<intrinsic,
                       instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW,
                       vti.Vector, vti.Vector, ivti.Vector, vti.Mask,
@@ -4907,8 +4916,8 @@ multiclass VPatBinaryV_VV_INT_EEW<string intrinsic, string instruction,
       defvar emul_str = octuple_to_str<octuple_emul>.ret;
       defvar ivti = !cast<VTypeInfo>("VI" # eew # emul_str);
       defvar inst = instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str;
-      let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
-                                   GetVTypePredicates<ivti>.Predicates) in
+      let Predicates = !listconcat(GetVTypeMinimalPredicates<vti>.Predicates,
+                                   GetVTypeMinimalPredicates<ivti>.Predicates) in
       defm : VPatBinary<intrinsic, inst,
                         vti.Vector, vti.Vector, ivti.Vector, vti.Mask,
                         vti.Log2SEW, vti.RegClass,
@@ -4950,7 +4959,7 @@ multiclass VPatBinaryV_VX_RM<string intrinsic, string instruction,
 multiclass VPatBinaryV_VX_INT<string intrinsic, string instruction,
                           list<VTypeInfo> vtilist> {
   foreach vti = vtilist in
-    let Predicates = GetVTypePredicates<vti>.Predicates in
+    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
     defm : VPatBinary<intrinsic, instruction # "_VX_" # vti.LMul.MX,
                       vti.Vector, vti.Vector, XLenVT, vti.Mask,
                       vti.Log2SEW, vti.RegClass,
@@ -4979,6 +4988,16 @@ multiclass VPatBinaryV_VI_RM<string intrinsic, string instruction,
                                   vti.RegClass, imm_type>;
 }
 
+multiclass VPatBinaryV_VI_INT<string intrinsic, string instruction,
+                              list<VTypeInfo> vtilist, Operand imm_type> {
+  foreach vti = vtilist in
+    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+    defm : VPatBinary<intrinsic, instruction # "_VI_" # vti.LMul.MX,
+                      vti.Vector, vti.Vector, XLenVT, vti.Mask,
+                      vti.Log2SEW, vti.RegClass,
+                      vti.RegClass, imm_type>;
+}
+
 multiclass VPatBinaryM_MM<string intrinsic, string instruction> {
   foreach mti = AllMasks in
     let Predicates = [HasVInstructions] in
@@ -5565,7 +5584,7 @@ multiclass VPatTernaryV_VV_AAXA_RM<string intrinsic, string instruction,
 multiclass VPatTernaryV_VX<string intrinsic, string instruction,
                            list<VTypeInfo> vtilist> {
   foreach vti = vtilist in
-    let Predicates = GetVTypePredicates<vti>.Predicates in
+    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
     defm : VPatTernaryWithPolicy<intrinsic, instruction, "VX",
                                  vti.Vector, vti.Vector, XLenVT, vti.Mask,
                                  vti.Log2SEW, vti.LMul, vti.RegClass,
@@ -5597,7 +5616,7 @@ multiclass VPatTernaryV_VX_AAXA_RM<string intrinsic, string instruction,
 multiclass VPatTernaryV_VI<string intrinsic, string instruction,
                            list<VTypeInfo> vtilist, Operand Imm_type> {
   foreach vti = vtilist in
-    let Predicates = GetVTypePredicates<vti>.Predicates in
+    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
     defm : VPatTernaryWithPolicy<intrinsic, instruction, "VI",
                                  vti.Vector, vti.Vector, XLenVT, vti.Mask,
                                  vti.Log2SEW, vti.LMul, vti.RegClass,
@@ -5709,7 +5728,7 @@ multiclass VPatBinaryV_VV_VX_VI_INT<string intrinsic, string instruction,
                                     list<VTypeInfo> vtilist, Operand ImmType>
     : VPatBinaryV_VV_INT<intrinsic#"_vv", instruction, vtilist>,
       VPatBinaryV_VX_INT<intrinsic#"_vx", instruction, vtilist>,
-      VPatBinaryV_VI<intrinsic#"_vx", instruction, vtilist, ImmType>;
+      VPatBinaryV_VI_INT<intrinsic#"_vx", instruction, vtilist, ImmType>;
 
 multiclass VPatReductionV_VS<string intrinsic, string instruction, bit IsFloat = 0> {
   foreach vti = !if(IsFloat, NoGroupFloatVectors, NoGroupIntegerVectors) in {
@@ -5887,12 +5906,11 @@ multiclass VPatConversionWF_VF<string intrinsic, string instruction,
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
     // Define vfwcvt.f.f.v for f16 when Zvfhmin is enable.
-    let Predicates = !if(!eq(fvti.Scalar, f16), [HasVInstructionsF16Minimal],
-                         !listconcat(GetVTypePredicates<fvti>.Predicates,
-                                     GetVTypePredicates<fwti>.Predicates)) in
-      defm : VPatConversion<intrinsic, instruction, "V",
-                            fwti.Vector, fvti.Vector, fwti.Mask, fvti.Log2SEW,
-                            fvti.LMul, fwti.RegClass, fvti.RegClass, isSEWAware>;
+    let Predicates = !listconcat(GetVTypeMinimalPredicates<fvti>.Predicates,
+                                 GetVTypeMinimalPredicates<fwti>.Predicates) in
+    defm : VPatConversion<intrinsic, instruction, "V",
+                          fwti.Vector, fvti.Vector, fwti.Mask, fvti.Log2SEW,
+                          fvti.LMul, fwti.RegClass, fvti.RegClass, isSEWAware>;
   }
 }
 
@@ -5979,8 +5997,9 @@ multiclass VPatConversionVF_WF_RM<string intrinsic, string instruction,
   foreach fvtiToFWti = wlist in {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
-    let Predicates = !listconcat(GetVTypePredicates<fvti>.Predicates,
-                                 GetVTypePredicates<fwti>.Predicates) in
+    // Define vfncvt.f.f.w for f16 when Zvfhmin is enable.
+    let Predicates = !listconcat(GetVTypeMinimalPredicates<fvti>.Predicates,
+                                 GetVTypeMinimalPredicates<fwti>.Predicates) in
     defm : VPatConversionRoundingMode<intrinsic, instruction, "W",
                                       fvti.Vector, fwti.Vector, fvti.Mask, fvti.Log2SEW,
                                       fvti.LMul, fvti.RegClass, fwti.RegClass,
@@ -6144,8 +6163,6 @@ defm PseudoVSUX : VPseudoIStore<Ordered=false>;
 // 7.7. Unit-stride Fault-Only-First Loads
 //===----------------------------------------------------------------------===//
 
-// vleff may update VL register
-let Defs = [VL] in
 defm PseudoVL : VPseudoFFLoad;
 
 //===----------------------------------------------------------------------===//
@@ -6159,11 +6176,7 @@ defm PseudoVSSEG : VPseudoUSSegStore;
 defm PseudoVSSSEG : VPseudoSSegStore;
 defm PseudoVSOXSEG : VPseudoISegStore<Ordered=true>;
 defm PseudoVSUXSEG : VPseudoISegStore<Ordered=false>;
-
-// vlseg<nf>e<eew>ff.v may update VL register
-let Defs = [VL] in {
 defm PseudoVLSEG : VPseudoUSSegLoadFF;
-}
 
 //===----------------------------------------------------------------------===//
 // 11. Vector Integer Arithmetic Instructions
@@ -7005,8 +7018,7 @@ defm : VPatBinaryV_VM_XM_IM<"int_riscv_vmerge", "PseudoVMERGE">;
 // 11.16. Vector Integer Move Instructions
 //===----------------------------------------------------------------------===//
 foreach vti = AllVectors in {
-  let Predicates = !if(!eq(vti.Scalar, f16), [HasVInstructionsF16Minimal],
-                       GetVTypePredicates<vti>.Predicates) in {
+  let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in {
     def : Pat<(vti.Vector (int_riscv_vmv_v_v (vti.Vector vti.RegClass:$passthru),
                                              (vti.Vector vti.RegClass:$rs1),
                                              VLOpFrag)),
@@ -7201,8 +7213,7 @@ defm : VPatConversionVI_VF<"int_riscv_vfclass", "PseudoVFCLASS">;
 // NOTE: Clang previously used int_riscv_vfmerge for vector-vector, but now uses
 // int_riscv_vmerge. Support both for compatibility.
 foreach vti = AllFloatVectors in {
-  let Predicates = !if(!eq(vti.Scalar, f16), [HasVInstructionsF16Minimal],
-                       GetVTypePredicates<vti>.Predicates) in
+  let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
     defm : VPatBinaryCarryInTAIL<"int_riscv_vmerge", "PseudoVMERGE", "VVM",
                                  vti.Vector,
                                  vti.Vector, vti.Vector, vti.Mask,
@@ -7281,16 +7292,8 @@ defm : VPatConversionVF_WI_RM<"int_riscv_vfncvt_f_xu_w", "PseudoVFNCVT_F_XU",
                               isSEWAware=1>;
 defm : VPatConversionVF_WI_RM<"int_riscv_vfncvt_f_x_w", "PseudoVFNCVT_F_X",
                               isSEWAware=1>;
-defvar WidenableFloatVectorsExceptF16 = !filter(fvtiToFWti, AllWidenableFloatVectors,
-                                                !ne(fvtiToFWti.Vti.Scalar, f16));
-defm : VPatConversionVF_WF_RM<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F",
-                           WidenableFloatVectorsExceptF16, isSEWAware=1>;
-// Define vfncvt.f.f.w for f16 when Zvfhmin is enable.
-defvar F16WidenableFloatVectors = !filter(fvtiToFWti, AllWidenableFloatVectors,
-                                          !eq(fvtiToFWti.Vti.Scalar, f16));
-let Predicates = [HasVInstructionsF16Minimal] in
 defm : VPatConversionVF_WF_RM<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F",
-                           F16WidenableFloatVectors, isSEWAware=1>;
+                              AllWidenableFloatVectors, isSEWAware=1>;
 defm : VPatConversionVF_WF_BF_RM<"int_riscv_vfncvtbf16_f_f_w", 
                                  "PseudoVFNCVTBF16_F_F", isSEWAware=1>;
 defm : VPatConversionVF_WF<"int_riscv_vfncvt_rod_f_f_w", "PseudoVFNCVT_ROD_F_F",
@@ -7425,10 +7428,7 @@ defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
                               eew=16, vtilist=AllIntegerVectors>;
 
 defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER",
-                                AllFloatVectorsExceptFP16, uimm5>;
-let Predicates = [HasVInstructionsF16Minimal] in
-  defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER",
-                                  AllFP16Vectors, uimm5>;
+                                AllFloatVectors, uimm5>;
 defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER",
                                 AllBFloatVectors, uimm5>;
 defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
@@ -7437,9 +7437,7 @@ defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
 // 16.5. Vector Compress Instruction
 //===----------------------------------------------------------------------===//
 defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>;
-defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectorsExceptFP16>;
-let Predicates = [HasVInstructionsF16Minimal] in
-  defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFP16Vectors>;
+defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>;
 defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllBFloatVectors>;
 
 // Include the non-intrinsic ISel patterns
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index e318a78285a2..520959b0896f 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -864,8 +864,7 @@ multiclass VPatAVGADD_VV_VX_RM<SDNode vop, int vxrm, string suffix = ""> {
 
 // 7.4. Vector Unit-Stride Instructions
 foreach vti = AllVectors in
-  let Predicates = !if(!eq(vti.Scalar, f16), [HasVInstructionsF16Minimal],
-                       GetVTypePredicates<vti>.Predicates) in 
+  let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
   defm : VPatUSLoadStoreSDNode<vti.Vector, vti.RegClass, vti.Log2SEW, vti.LMul,
                                vti.AVL, vti.RegClass>;
 foreach mti = AllMasks in
@@ -1449,9 +1448,8 @@ defm : VPatNConvertI2FPSDNode_W_RM<any_uint_to_fp, "PseudoVFNCVT_F_XU_W">;
 foreach fvtiToFWti = AllWidenableFloatVectors in {
   defvar fvti = fvtiToFWti.Vti;
   defvar fwti = fvtiToFWti.Wti;
-  let Predicates = !if(!eq(fvti.Scalar, f16), [HasVInstructionsF16Minimal],
-                       !listconcat(GetVTypePredicates<fvti>.Predicates,
-                                   GetVTypePredicates<fwti>.Predicates)) in
+  let Predicates = !listconcat(GetVTypeMinimalPredicates<fvti>.Predicates,
+                               GetVTypeMinimalPredicates<fwti>.Predicates) in
   def : Pat<(fvti.Vector (fpround (fwti.Vector fwti.RegClass:$rs1))),
             (!cast<Instruction>("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW)
                 (fvti.Vector (IMPLICIT_DEF)),
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 1da4adc8c312..6328e6c860f7 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -653,11 +653,7 @@ let HasOneUse = 1 in {
                                                node:$E),
                                           (riscv_or_vl node:$A, node:$B, node:$C,
                                                        node:$D, node:$E), [{
-    if (N->getFlags().hasDisjoint())
-      return true;
-    KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
-    KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
-    return KnownBits::haveNoCommonBitsSet(Known0, Known1);
+    return orIsAdd(N);
   }]>;
   def riscv_sub_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
                                          node:$E),
@@ -2687,9 +2683,9 @@ defm : VPatWConvertI2FPVL_V<any_riscv_sint_to_fp_vl, "PseudoVFWCVT_F_X_V">;
 foreach fvtiToFWti = AllWidenableFloatVectors in {
   defvar fvti = fvtiToFWti.Vti;
   defvar fwti = fvtiToFWti.Wti;
-  let Predicates = !if(!eq(fvti.Scalar, f16), [HasVInstructionsF16Minimal],
-                       !listconcat(GetVTypePredicates<fvti>.Predicates,
-                                   GetVTypePredicates<fwti>.Predicates)) in
+  // Define vfwcvt.f.f.v for f16 when Zvfhmin is enable.
+  let Predicates = !listconcat(GetVTypeMinimalPredicates<fvti>.Predicates,
+                               GetVTypeMinimalPredicates<fwti>.Predicates) in
   def : Pat<(fwti.Vector (any_riscv_fpextend_vl
                              (fvti.Vector fvti.RegClass:$rs1),
                              (fvti.Mask VMV0:$vm),
@@ -2730,10 +2726,9 @@ defm : VPatNConvertI2FP_RM_VL_W<riscv_vfcvt_rm_f_x_vl, "PseudoVFNCVT_F_X_W">;
 foreach fvtiToFWti = AllWidenableFloatVectors in {
   defvar fvti = fvtiToFWti.Vti;
   defvar fwti = fvtiToFWti.Wti;
-  // Define vfwcvt.f.f.v for f16 when Zvfhmin is enable.
-  let Predicates = !if(!eq(fvti.Scalar, f16), [HasVInstructionsF16Minimal],
-                       !listconcat(GetVTypePredicates<fvti>.Predicates,
-                                   GetVTypePredicates<fwti>.Predicates)) in {
+  // Define vfncvt.f.f.w for f16 when Zvfhmin is enable.
+  let Predicates = !listconcat(GetVTypeMinimalPredicates<fvti>.Predicates,
+                               GetVTypeMinimalPredicates<fwti>.Predicates) in {
     def : Pat<(fvti.Vector (any_riscv_fpround_vl
                                (fwti.Vector fwti.RegClass:$rs1),
                                (fwti.Mask VMV0:$vm), VLOpFrag)),
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index 5d7dad458cf5..942dde4bab31 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -798,9 +798,9 @@ def IntCCtoRISCVCCCV : SDNodeXForm<riscv_selectcc, [{
 
 let Predicates = [HasVendorXCVbi, IsRV32], AddedComplexity = 2 in {
   def : Pat<(riscv_brcc GPR:$rs1, simm5:$imm5, SETEQ, bb:$imm12),
-            (CV_BEQIMM GPR:$rs1, simm5:$imm5, bare_simm13_lsb0:$imm12)>;
+            (CV_BEQIMM GPR:$rs1, simm5:$imm5, bare_simm13_lsb0_bb:$imm12)>;
   def : Pat<(riscv_brcc GPR:$rs1, simm5:$imm5, SETNE, bb:$imm12),
-            (CV_BNEIMM GPR:$rs1, simm5:$imm5, bare_simm13_lsb0:$imm12)>;
+            (CV_BNEIMM GPR:$rs1, simm5:$imm5, bare_simm13_lsb0_bb:$imm12)>;
 
   defm CC_SImm5_CV : SelectCC_GPR_riirr<GPR, simm5>;
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index 6981daa75d09..affe3a847675 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -208,7 +208,7 @@ let Predicates = [HasVendorXSfvqmaccdod], DecoderNamespace = "XSfvector",
 }
 
 let Predicates = [HasVendorXSfvqmaccqoq], DecoderNamespace = "XSfvector",
-    DestEEW = EEWSEWx4, RVVConstraint=WidenV in {
+    DestEEW = EEWSEWx4, RVVConstraint=WidenVNoMask in {
   def VQMACCU_4x8x4  : CustomSiFiveVMACC<0b111100, OPMVV, "sf.vqmaccu.4x8x4">;
   def VQMACC_4x8x4   : CustomSiFiveVMACC<0b111101, OPMVV, "sf.vqmacc.4x8x4">;
   def VQMACCUS_4x8x4 : CustomSiFiveVMACC<0b111110, OPMVV, "sf.vqmaccus.4x8x4">;
@@ -216,7 +216,7 @@ let Predicates = [HasVendorXSfvqmaccqoq], DecoderNamespace = "XSfvector",
 }
 
 let Predicates = [HasVendorXSfvfwmaccqqq], DecoderNamespace = "XSfvector",
-    DestEEW = EEWSEWx2, RVVConstraint=WidenV in {
+    DestEEW = EEWSEWx2, RVVConstraint=WidenVNoMask in {
   def VFWMACC_4x4x4 : CustomSiFiveVMACC<0b111100, OPFVV, "sf.vfwmacc.4x4x4">;
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index 2fccbcaf2cf3..89441444a994 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -660,10 +660,12 @@ def : Pat<(i32 (sub GPR:$rd, (mul (sexti16 (i32 GPR:$rs1)),
           (TH_MULSH GPR:$rd, GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasVendorXTHeadMac, IsRV32]
 
+let Predicates = [HasVendorXTHeadVdot] in {
 defm PseudoTHVdotVMAQA      : VPseudoVMAQA_VV_VX;
 defm PseudoTHVdotVMAQAU     : VPseudoVMAQA_VV_VX;
 defm PseudoTHVdotVMAQASU    : VPseudoVMAQA_VV_VX;
 defm PseudoTHVdotVMAQAUS    : VPseudoVMAQA_VX;
+}
 
 let Predicates = [HasVendorXTHeadVdot] in {
 defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqa",  "PseudoTHVdotVMAQA",
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index f292c9376574..b94fee3c6e57 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -19,6 +19,12 @@ def uimm5nonzero : RISCVOp<XLenVT>,
   let ParserMatchClass = UImmAsmOperand<5, "NonZero">;
   let DecoderMethod = "decodeUImmNonZeroOperand<5>";
   let OperandType = "OPERAND_UIMM5_NONZERO";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (!MCOp.evaluateAsConstantImm(Imm))
+      return false;
+    return (Imm != 0) && isUInt<5>(Imm);;
+  }];
 }
 
 def uimm5gt3 : RISCVOp<XLenVT>, ImmLeaf<XLenVT,
@@ -111,6 +117,12 @@ def simm5nonzero : RISCVOp<XLenVT>,
   let ParserMatchClass = SImmAsmOperand<5, "NonZero">;
   let DecoderMethod = "decodeSImmNonZeroOperand<5>";
   let OperandType = "OPERAND_SIMM5_NONZERO";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (!MCOp.evaluateAsConstantImm(Imm))
+      return false;
+    return (Imm != 0) && isInt<5>(Imm);
+  }];
 }
 
 def simm11 : RISCVSImmLeafOp<11>;
@@ -562,9 +574,10 @@ class QCILoadMultiple<bits<2> funct2, DAGOperand InTyRs2, string opcodestr>
 
 // rd corresponds to the source for the store 'rs3' described in the spec.
 let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
-class QCIStoreMultiple<bits<2> funct2, DAGOperand InTyRs2, string opcodestr>
+class QCIStoreMultiple<bits<2> funct2, DAGOperand InTyRd, DAGOperand InTyRs2,
+                       string opcodestr>
     : RVInstRBase<0b111, OPC_CUSTOM_1, (outs),
-                  (ins GPR:$rd, GPR:$rs1, InTyRs2:$rs2, uimm7_lsb00:$imm),
+                  (ins InTyRd:$rd, GPR:$rs1, InTyRs2:$rs2, uimm7_lsb00:$imm),
                   opcodestr, "$rd, $rs2, ${imm}(${rs1})"> {
   bits<7> imm;
   let Inst{31-25} = {funct2, imm{6-2}};
@@ -955,10 +968,10 @@ let Predicates = [HasVendorXqcics, IsRV32] in {
 } // Predicates = [HasVendorXqcics, IsRV32]
 
 let Predicates = [HasVendorXqcilsm, IsRV32] in {
-    def QC_SWM : QCIStoreMultiple<0b00, GPRNoX0, "qc.swm">;
-    def QC_SWMI : QCIStoreMultiple<0b01, uimm5nonzero, "qc.swmi">;
-    def QC_SETWM : QCIStoreMultiple<0b10, GPRNoX0, "qc.setwm">;
-    def QC_SETWMI : QCIStoreMultiple<0b11, uimm5nonzero, "qc.setwmi">;
+    def QC_SWM : QCIStoreMultiple<0b00, GPRNoX0, GPRNoX0, "qc.swm">;
+    def QC_SWMI : QCIStoreMultiple<0b01, GPRNoX0, uimm5nonzero, "qc.swmi">;
+    def QC_SETWM : QCIStoreMultiple<0b10, GPR, GPRNoX0, "qc.setwm">;
+    def QC_SETWMI : QCIStoreMultiple<0b11, GPR, uimm5nonzero, "qc.setwmi">;
 
     def QC_LWM : QCILoadMultiple<0b00, GPRNoX0, "qc.lwm">;
     def QC_LWMI : QCILoadMultiple<0b01, uimm5nonzero, "qc.lwmi">;
@@ -1199,9 +1212,9 @@ let EmitPriority = 0 in {
 let Predicates = [HasVendorXqcilsm, IsRV32] in {
 let EmitPriority = 0 in {
   def : InstAlias<"qc.swm $rs3, $rs2, (${rs1})",
-                  (QC_SWM GPR:$rs3, GPR:$rs1, GPRNoX0:$rs2, 0)>;
+                  (QC_SWM GPRNoX0:$rs3, GPR:$rs1, GPRNoX0:$rs2, 0)>;
   def : InstAlias<"qc.swmi $rs3, $length, (${rs1})",
-                  (QC_SWMI GPR:$rs3, GPR:$rs1, uimm5nonzero:$length, 0)>;
+                  (QC_SWMI GPRNoX0:$rs3, GPR:$rs1, uimm5nonzero:$length, 0)>;
   def : InstAlias<"qc.setwm $rs3, $rs2, (${rs1})",
                   (QC_SETWM GPR:$rs3, GPR:$rs1, GPRNoX0:$rs2, 0)>;
   def : InstAlias<"qc.setwmi $rs3, $length, (${rs1})",
@@ -1322,11 +1335,11 @@ class QCScaledStPat<PatFrag StoreOp, RVInst Inst>
 // Match `riscv_brcc` and lower to the appropriate XQCIBI branch instruction.
 class BcciPat<CondCode Cond, QCIBranchInst_rii Inst, DAGOperand InTyImm>
     : Pat<(riscv_brcc (XLenVT GPRNoX0:$rs1), InTyImm:$rs2, Cond, bb:$imm12),
-          (Inst GPRNoX0:$rs1, InTyImm:$rs2, bare_simm13_lsb0:$imm12)>;
+          (Inst GPRNoX0:$rs1, InTyImm:$rs2, bare_simm13_lsb0_bb:$imm12)>;
 
 class Bcci48Pat<CondCode Cond, QCIBranchInst48_rii Inst, DAGOperand InTyImm>
     : Pat<(riscv_brcc (XLenVT GPRNoX0:$rs1), InTyImm:$rs2, Cond, bb:$imm12),
-          (Inst GPRNoX0:$rs1, InTyImm:$rs2, bare_simm13_lsb0:$imm12)>;
+          (Inst GPRNoX0:$rs1, InTyImm:$rs2, bare_simm13_lsb0_bb:$imm12)>;
 
 defm CC_SImm5NonZero_QC  : SelectCC_GPR_riirr<GPRNoX0, simm5nonzero>;
 defm CC_UImm5NonZero_QC  : SelectCC_GPR_riirr<GPRNoX0, uimm5nonzero>;
@@ -1429,6 +1442,14 @@ let Predicates = [HasVendorXqcibm, IsRV32] in {
 def : Pat<(sext_inreg (i32 GPR:$rs1), i16), (QC_EXT GPR:$rs1, 16, 0)>;
 def : Pat<(sext_inreg (i32 GPR:$rs1), i8), (QC_EXT GPR:$rs1, 8, 0)>;
 def : Pat<(sext_inreg (i32 GPR:$rs1), i1), (QC_EXT GPR:$rs1, 1, 0)>;
+
+// Prefer qc.extu to andi for the following cases since the former can be compressed
+def : Pat<(i32 (and GPRNoX0:$rs, 63)), (QC_EXTU GPRNoX0:$rs, 6, 0)>;
+def : Pat<(i32 (and GPRNoX0:$rs, 127)), (QC_EXTU GPRNoX0:$rs, 7, 0)>;
+def : Pat<(i32 (and GPRNoX0:$rs, 255)), (QC_EXTU GPRNoX0:$rs, 8, 0)>;
+def : Pat<(i32 (and GPRNoX0:$rs, 511)), (QC_EXTU GPRNoX0:$rs, 9, 0)>;
+def : Pat<(i32 (and GPRNoX0:$rs, 1023)), (QC_EXTU GPRNoX0:$rs, 10, 0)>;
+def : Pat<(i32 (and GPRNoX0:$rs, 2047)), (QC_EXTU GPRNoX0:$rs, 11, 0)>;
 } // Predicates = [HasVendorXqcibm, IsRV32]
 
 let Predicates = [HasVendorXqciint, IsRV32] in
@@ -1543,3 +1564,18 @@ def : CompressPat<(QC_E_ADDI GPRNoX0:$rs1, GPRNoX0:$rs2, 0),
 def : CompressPat<(QC_E_ADDI X2, X2, simm10_lsb0000nonzero:$imm),
                   (C_ADDI16SP X2, simm10_lsb0000nonzero:$imm)>;
 } // let isCompressOnly = true, Predicates = [HasVendorXqcilia, IsRV32]
+
+let isCompressOnly = true, Predicates = [HasVendorXqcibi, IsRV32] in {
+def : CompressPat<(QC_E_BEQI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12),
+                  (QC_BEQI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12)>;
+def : CompressPat<(QC_E_BNEI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12),
+                  (QC_BNEI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12)>;
+def : CompressPat<(QC_E_BGEI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12),
+                  (QC_BGEI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12)>;
+def : CompressPat<(QC_E_BLTI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12),
+                  (QC_BLTI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12)>;
+def : CompressPat<(QC_E_BGEUI GPRNoX0:$rs1, uimm5nonzero:$imm5, bare_simm13_lsb0:$imm12),
+                  (QC_BGEUI GPRNoX0:$rs1, uimm5nonzero:$imm5, bare_simm13_lsb0:$imm12)>;
+def : CompressPat<(QC_E_BLTUI GPRNoX0:$rs1, uimm5nonzero:$imm5, bare_simm13_lsb0:$imm12),
+                  (QC_BLTUI GPRNoX0:$rs1, uimm5nonzero:$imm5, bare_simm13_lsb0:$imm12)>;
+} // let isCompressOnly = true, Predicates = [HasVendorXqcibi, IsRV32]
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 4353e94bdb1d..4806bcc1d63d 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -704,9 +704,13 @@ def : CSImm12MulBy4Pat<SH2ADD>;
 def : CSImm12MulBy8Pat<SH3ADD>;
 } // Predicates = [HasStdExtZba]
 
+def zExtImm32 : ComplexPattern<i64, 1, "selectZExtImm32", [], [], 0>;
+
 multiclass ADD_UWPat<Instruction add_uw> {
   def : Pat<(i64 (add_like_non_imm12 (and GPR:$rs1, 0xFFFFFFFF), GPR:$rs2)),
             (add_uw GPR:$rs1, GPR:$rs2)>;
+  def : Pat<(i64 (add_like zExtImm32:$rs1, GPR:$rs2)),
+            (add_uw zExtImm32:$rs1, GPR:$rs2)>;
   def : Pat<(i64 (and GPR:$rs, 0xFFFFFFFF)), (add_uw GPR:$rs, (XLenVT X0))>;
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZvqdotq.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZvqdotq.td
index ea3c53cb0a5d..27959eaccd90 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZvqdotq.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfoZvqdotq.td
@@ -37,11 +37,11 @@ let HasPassthruOp = true, HasMaskOp = true in {
 multiclass VPseudoVQDOT_VV_VX {
   foreach m = MxSet<32>.m in {
     defm "" : VPseudoBinaryV_VV<m>,
-            SchedBinary<"WriteVIALUV", "ReadVIALUV", "ReadVIALUV", m.MX,
-                        forcePassthruRead=true>;
+              SchedBinary<"WriteVIMulAddV", "ReadVIMulAddV", "ReadVIMulAddV", m.MX,
+                          forcePassthruRead=true>;
     defm "" : VPseudoBinaryV_VX<m>,
-            SchedBinary<"WriteVIALUX", "ReadVIALUV", "ReadVIALUX", m.MX,
-                        forcePassthruRead=true>;
+              SchedBinary<"WriteVIMulAddX", "ReadVIMulAddV", "ReadVIMulAddX", m.MX,
+                          forcePassthruRead=true>;
   }
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrPredicates.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
index 4c37cb7e393b..1057eeee31d6 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
@@ -129,6 +129,13 @@ def isVSlideInstr
                       !instances<Pseudo>("^PseudoVSLIDEUP_VI.*")
                      ])>>>;
 
+def isFaultOnlyFirstLoad
+    : TIIPredicate<"isFaultOnlyFirstLoad",
+                    MCReturnStatement<
+                      CheckOpcode<
+                       !instances<Pseudo>(
+                          "^PseudoVL(SEG[2-8])?E(8|16|32|64)FF_V.*")>>>;
+
 def isNonZeroLoadImmediate
     : TIIPredicate<"isNonZeroLoadImmediate",
                    MCReturnStatement<CheckAll<[
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
index 3b601bb43bf2..ed61236415cc 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
@@ -591,6 +591,27 @@ static bool isSignExtendedW(Register SrcReg, const RISCVSubtarget &ST,
         return false;
       break;
 
+    case RISCV::ADDI: {
+      if (MI->getOperand(1).isReg() && MI->getOperand(1).getReg().isVirtual()) {
+        if (MachineInstr *SrcMI = MRI.getVRegDef(MI->getOperand(1).getReg())) {
+          if (SrcMI->getOpcode() == RISCV::LUI &&
+              SrcMI->getOperand(1).isImm()) {
+            uint64_t Imm = SrcMI->getOperand(1).getImm();
+            Imm = SignExtend64<32>(Imm << 12);
+            Imm += (uint64_t)MI->getOperand(2).getImm();
+            if (isInt<32>(Imm))
+              continue;
+          }
+        }
+      }
+
+      if (hasAllWUsers(*MI, ST, MRI)) {
+        FixableDef.insert(MI);
+        break;
+      }
+      return false;
+    }
+
     // With these opcode, we can "fix" them with the W-version
     // if we know all users of the result only rely on bits 31:0
     case RISCV::SLLI:
@@ -598,7 +619,6 @@ static bool isSignExtendedW(Register SrcReg, const RISCVSubtarget &ST,
       if (MI->getOperand(2).getImm() >= 32)
         return false;
       [[fallthrough]];
-    case RISCV::ADDI:
     case RISCV::ADD:
     case RISCV::LD:
     case RISCV::LWU:
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVProcessors.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVProcessors.td
index dc0ae384d6b0..32f4ab607a34 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -271,10 +271,10 @@ def SIFIVE_U74 : RISCVProcessorModel<"sifive-u74",
                                       FeatureStdExtC],
                                      SiFive7TuneFeatures>;
 
-defvar SiFiveX280TuneFeatures = !listconcat(SiFive7TuneFeatures,
-                                            [TuneDLenFactor2,
-                                             TuneOptimizedZeroStrideLoad,
-                                             TuneOptimizedNF2SegmentLoadStore]);
+defvar SiFiveIntelligenceTuneFeatures = !listconcat(SiFive7TuneFeatures,
+                                                    [TuneDLenFactor2,
+                                                     TuneOptimizedZeroStrideLoad,
+                                                     TuneOptimizedNF2SegmentLoadStore]);
 def SIFIVE_X280 : RISCVProcessorModel<"sifive-x280", SiFive7Model,
                                       [Feature64Bit,
                                        FeatureStdExtI,
@@ -290,7 +290,51 @@ def SIFIVE_X280 : RISCVProcessorModel<"sifive-x280", SiFive7Model,
                                        FeatureStdExtZvfh,
                                        FeatureStdExtZba,
                                        FeatureStdExtZbb],
-                                      SiFiveX280TuneFeatures>;
+                                      SiFiveIntelligenceTuneFeatures>;
+
+def SIFIVE_X390 : RISCVProcessorModel<"sifive-x390", NoSchedModel,
+                                      [Feature64Bit,
+                                       FeatureStdExtI,
+                                       FeatureStdExtM,
+                                       FeatureStdExtA,
+                                       FeatureStdExtF,
+                                       FeatureStdExtD,
+                                       FeatureStdExtC,
+                                       FeatureStdExtB,
+                                       FeatureStdExtV,
+                                       FeatureStdExtZic64b,
+                                       FeatureStdExtZicbom,
+                                       FeatureStdExtZicbop,
+                                       FeatureStdExtZicboz,
+                                       FeatureStdExtZiccamoa,
+                                       FeatureStdExtZiccif,
+                                       FeatureStdExtZiccrse,
+                                       FeatureStdExtZicfilp,
+                                       FeatureStdExtZicfiss,
+                                       FeatureStdExtZicntr,
+                                       FeatureStdExtZicond,
+                                       FeatureStdExtZifencei,
+                                       FeatureStdExtZihintntl,
+                                       FeatureStdExtZihintpause,
+                                       FeatureStdExtZihpm,
+                                       FeatureStdExtZimop,
+                                       FeatureStdExtZa64rs,
+                                       FeatureStdExtZawrs,
+                                       FeatureStdExtZfa,
+                                       FeatureStdExtZfh,
+                                       FeatureStdExtZcb,
+                                       FeatureStdExtZcmop,
+                                       FeatureStdExtZkr,
+                                       FeatureStdExtZkt,
+                                       FeatureStdExtZvbb,
+                                       FeatureStdExtZvfbfmin,
+                                       FeatureStdExtZvfbfwma,
+                                       FeatureStdExtZvfh,
+                                       FeatureStdExtZvkt,
+                                       FeatureStdExtZvl1024b,
+                                       FeatureVendorXSiFivecdiscarddlone,
+                                       FeatureVendorXSiFivecflushdlone],
+                                      SiFiveIntelligenceTuneFeatures>;
 
 defvar SiFiveP400TuneFeatures = [TuneNoDefaultUnroll,
                                  TuneConditionalCompressedMoveFusion,
@@ -678,8 +722,16 @@ def ANDES_AX25 : RISCVProcessorModel<"andes-ax25",
                                       FeatureStdExtZbc,
                                       FeatureVendorXAndesPerf]>;
 
+defvar Andes45TuneFeatures = [TuneAndes45,
+                              TuneNoDefaultUnroll,
+                              TuneShortForwardBranchOpt,
+                              TunePostRAScheduler];
+
+def ANDES_45 : RISCVTuneProcessorModel<"andes-45-series",
+                                       Andes45Model, Andes45TuneFeatures>;
+
 def ANDES_N45 : RISCVProcessorModel<"andes-n45",
-                                    NoSchedModel,
+                                    Andes45Model,
                                     [Feature32Bit,
                                      FeatureStdExtI,
                                      FeatureStdExtZicsr,
@@ -690,10 +742,11 @@ def ANDES_N45 : RISCVProcessorModel<"andes-n45",
                                      FeatureStdExtD,
                                      FeatureStdExtC,
                                      FeatureStdExtB,
-                                     FeatureVendorXAndesPerf]>;
+                                     FeatureVendorXAndesPerf],
+                                    Andes45TuneFeatures>;
 
 def ANDES_NX45 : RISCVProcessorModel<"andes-nx45",
-                                     NoSchedModel,
+                                     Andes45Model,
                                      [Feature64Bit,
                                       FeatureStdExtI,
                                       FeatureStdExtZicsr,
@@ -704,10 +757,11 @@ def ANDES_NX45 : RISCVProcessorModel<"andes-nx45",
                                       FeatureStdExtD,
                                       FeatureStdExtC,
                                       FeatureStdExtB,
-                                      FeatureVendorXAndesPerf]>;
+                                      FeatureVendorXAndesPerf],
+                                     Andes45TuneFeatures>;
 
 def ANDES_A45 : RISCVProcessorModel<"andes-a45",
-                                    NoSchedModel,
+                                    Andes45Model,
                                     [Feature32Bit,
                                      FeatureStdExtI,
                                      FeatureStdExtZicsr,
@@ -718,10 +772,11 @@ def ANDES_A45 : RISCVProcessorModel<"andes-a45",
                                      FeatureStdExtD,
                                      FeatureStdExtC,
                                      FeatureStdExtB,
-                                     FeatureVendorXAndesPerf]>;
+                                     FeatureVendorXAndesPerf],
+                                    Andes45TuneFeatures>;
 
 def ANDES_AX45 : RISCVProcessorModel<"andes-ax45",
-                                     NoSchedModel,
+                                     Andes45Model,
                                      [Feature64Bit,
                                       FeatureStdExtI,
                                       FeatureStdExtZicsr,
@@ -732,4 +787,5 @@ def ANDES_AX45 : RISCVProcessorModel<"andes-ax45",
                                       FeatureStdExtD,
                                       FeatureStdExtC,
                                       FeatureStdExtB,
-                                      FeatureVendorXAndesPerf]>;
+                                      FeatureVendorXAndesPerf],
+                                     Andes45TuneFeatures>;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 80213e1503b0..e87f4523a84f 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -846,6 +846,7 @@ foreach m = LMULList in {
 
 def FFLAGS : RISCVReg<0, "fflags">;
 def FRM    : RISCVReg<0, "frm">;
+def FCSR   : RISCVReg<0, "fcsr">;
 
 // Shadow Stack register
 def SSP    : RISCVReg<0, "ssp">;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVSchedAndes45.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVSchedAndes45.td
new file mode 100644
index 000000000000..f42b48a1d8b8
--- /dev/null
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVSchedAndes45.td
@@ -0,0 +1,339 @@
+//==- RISCVSchedAndes45.td - Andes45 Scheduling Definitions --*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+
+// FIXME: Implement sheduling model for V and other extensions.
+def Andes45Model : SchedMachineModel {
+  let MicroOpBufferSize = 0;  // Andes45 is in-order processor
+  let IssueWidth = 2;         // 2 micro-ops dispatched per cycle
+  let LoadLatency = 2;
+  let MispredictPenalty = 5;
+  let CompleteModel = 0;
+}
+
+let SchedModel = Andes45Model in {
+
+//===----------------------------------------------------------------------===//
+// Define each kind of processor resource and number available.
+
+//===----------------------------------------------------------------------===//
+// Andes 45 series CPU
+//   - 2 Interger Arithmetic and Logical Units (ALU)
+//   - Multiply / Divide Unit (MDU)
+//   - Load Store Unit (LSU)
+//   - Control and Status Register Unit (CSR)
+//   - Floating Point Multiply-Accumulate Unit (FMAC)
+//   - Floating Point Divide / SQRT Unit (FDIV)
+//   - Floating Point Move Unit (FMV)
+//   - Floating Point Misc Unit (FMISC)
+//===----------------------------------------------------------------------===//
+
+let BufferSize = 0 in {
+def Andes45ALU : ProcResource<2>;
+def Andes45MDU : ProcResource<1>;
+def Andes45LSU : ProcResource<1>;
+def Andes45CSR : ProcResource<1>;
+
+def Andes45FMAC  : ProcResource<1>;
+def Andes45FDIV  : ProcResource<1>;
+def Andes45FMV   : ProcResource<1>;
+def Andes45FMISC : ProcResource<1>;
+}
+
+// Integer arithmetic and logic
+def : WriteRes<WriteIALU, [Andes45ALU]>;
+def : WriteRes<WriteIALU32, [Andes45ALU]>;
+def : WriteRes<WriteShiftImm, [Andes45ALU]>;
+def : WriteRes<WriteShiftImm32, [Andes45ALU]>;
+def : WriteRes<WriteShiftReg, [Andes45ALU]>;
+def : WriteRes<WriteShiftReg32, [Andes45ALU]>;
+
+// Branching
+def : WriteRes<WriteJmp, [Andes45ALU]>;
+def : WriteRes<WriteJal, [Andes45ALU]>;
+def : WriteRes<WriteJalr, [Andes45ALU]>;
+
+// Integer multiplication
+let Latency = 3 in {
+def : WriteRes<WriteIMul, [Andes45MDU]>;
+def : WriteRes<WriteIMul32, [Andes45MDU]>;
+}
+
+// Integer division
+let Latency = 39, ReleaseAtCycles = [39] in {
+def : WriteRes<WriteIDiv, [Andes45MDU]>;
+def : WriteRes<WriteIDiv32, [Andes45MDU]>;
+}
+
+// Integer remainder
+let Latency = 39, ReleaseAtCycles = [39] in {
+def : WriteRes<WriteIRem, [Andes45MDU]>;
+def : WriteRes<WriteIRem32, [Andes45MDU]>;
+}
+
+// Memory
+let Latency = 5 in {
+def : WriteRes<WriteLDB, [Andes45LSU]>;
+def : WriteRes<WriteLDH, [Andes45LSU]>;
+def : WriteRes<WriteFLD16, [Andes45LSU]>;
+}
+
+let Latency = 3 in {
+def : WriteRes<WriteLDW, [Andes45LSU]>;
+def : WriteRes<WriteLDD, [Andes45LSU]>;
+def : WriteRes<WriteFLD32, [Andes45LSU]>;
+def : WriteRes<WriteFLD64, [Andes45LSU]>;
+}
+
+let Latency = 1 in {
+def : WriteRes<WriteSTB, [Andes45LSU]>;
+def : WriteRes<WriteSTH, [Andes45LSU]>;
+def : WriteRes<WriteSTW, [Andes45LSU]>;
+def : WriteRes<WriteSTD, [Andes45LSU]>;
+def : WriteRes<WriteFST16, [Andes45LSU]>;
+def : WriteRes<WriteFST32, [Andes45LSU]>;
+def : WriteRes<WriteFST64, [Andes45LSU]>;
+}
+
+// Atomic Memory
+let Latency = 9 in {
+def : WriteRes<WriteAtomicW, [Andes45LSU]>;
+def : WriteRes<WriteAtomicD, [Andes45LSU]>;
+def : WriteRes<WriteAtomicLDW, [Andes45LSU]>;
+def : WriteRes<WriteAtomicLDD, [Andes45LSU]>;
+}
+
+let Latency = 3 in {
+def : WriteRes<WriteAtomicSTW, [Andes45LSU]>;
+def : WriteRes<WriteAtomicSTD, [Andes45LSU]>;
+}
+
+// FMAC
+let Latency = 4 in  {
+def : WriteRes<WriteFAdd16, [Andes45FMAC]>;
+def : WriteRes<WriteFAdd32, [Andes45FMAC]>;
+def : WriteRes<WriteFAdd64, [Andes45FMAC]>;
+def : WriteRes<WriteFMul16, [Andes45FMAC]>;
+def : WriteRes<WriteFMul32, [Andes45FMAC]>;
+def : WriteRes<WriteFMul64, [Andes45FMAC]>;
+def : WriteRes<WriteFMA16, [Andes45FMAC]>;
+def : WriteRes<WriteFMA32, [Andes45FMAC]>;
+def : WriteRes<WriteFMA64, [Andes45FMAC]>;
+}
+
+// FDIV
+let Latency = 12, ReleaseAtCycles = [12] in
+def : WriteRes<WriteFDiv16, [Andes45FDIV]>;
+let Latency = 11, ReleaseAtCycles = [11] in
+def : WriteRes<WriteFSqrt16, [Andes45FDIV]>;
+
+let Latency = 19, ReleaseAtCycles = [19] in
+def : WriteRes<WriteFDiv32, [Andes45FDIV]>;
+let Latency = 18, ReleaseAtCycles = [18] in
+def : WriteRes<WriteFSqrt32, [Andes45FDIV]>;
+
+let Latency = 33, ReleaseAtCycles = [33] in
+def : WriteRes<WriteFDiv64, [Andes45FDIV]>;
+let Latency = 32, ReleaseAtCycles = [32] in
+def : WriteRes<WriteFSqrt64, [Andes45FDIV]>;
+
+// FMV
+def : WriteRes<WriteFSGNJ16, [Andes45FMV]>;
+def : WriteRes<WriteFSGNJ32, [Andes45FMV]>;
+def : WriteRes<WriteFSGNJ64, [Andes45FMV]>;
+def : WriteRes<WriteFMovF16ToI16, [Andes45FMV]>;
+def : WriteRes<WriteFMovI16ToF16, [Andes45FMV]>;
+def : WriteRes<WriteFMovF32ToI32, [Andes45FMV]>;
+def : WriteRes<WriteFMovI32ToF32, [Andes45FMV]>;
+def : WriteRes<WriteFMovF64ToI64, [Andes45FMV]>;
+def : WriteRes<WriteFMovI64ToF64, [Andes45FMV]>;
+
+// FMISC
+let Latency = 2 in  {
+def : WriteRes<WriteFMinMax16, [Andes45FMISC]>;
+def : WriteRes<WriteFMinMax32, [Andes45FMISC]>;
+def : WriteRes<WriteFMinMax64, [Andes45FMISC]>;
+def : WriteRes<WriteFClass16, [Andes45FMISC]>;
+def : WriteRes<WriteFClass32, [Andes45FMISC]>;
+def : WriteRes<WriteFClass64, [Andes45FMISC]>;
+def : WriteRes<WriteFCmp16, [Andes45FMISC]>;
+def : WriteRes<WriteFCmp32, [Andes45FMISC]>;
+def : WriteRes<WriteFCmp64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF16ToI32, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF16ToI64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF32ToI32, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF32ToI64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF64ToI32, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF64ToI64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtI32ToF16, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtI32ToF32, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtI32ToF64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtI64ToF16, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtI64ToF32, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtI64ToF64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF16ToF32, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF16ToF64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF32ToF16, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF32ToF64, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF64ToF16, [Andes45FMISC]>;
+def : WriteRes<WriteFCvtF64ToF32, [Andes45FMISC]>;
+}
+
+// Bitmanip
+// Zba extension
+def : WriteRes<WriteSHXADD, [Andes45ALU]>;
+def : WriteRes<WriteSHXADD32, [Andes45ALU]>;
+
+// Zbb extension
+def : WriteRes<WriteRotateImm, [Andes45ALU]>;
+def : WriteRes<WriteRotateImm32, [Andes45ALU]>;
+def : WriteRes<WriteRotateReg, [Andes45ALU]>;
+def : WriteRes<WriteRotateReg32, [Andes45ALU]>;
+def : WriteRes<WriteREV8, [Andes45ALU]>;
+def : WriteRes<WriteORCB, [Andes45ALU]>;
+def : WriteRes<WriteIMinMax, [Andes45ALU]>;
+
+let Latency = 3 in  {
+def : WriteRes<WriteCLZ, [Andes45ALU]>;
+def : WriteRes<WriteCLZ32, [Andes45ALU]>;
+def : WriteRes<WriteCTZ, [Andes45ALU]>;
+def : WriteRes<WriteCTZ32, [Andes45ALU]>;
+def : WriteRes<WriteCPOP, [Andes45ALU]>;
+def : WriteRes<WriteCPOP32, [Andes45ALU]>;
+}
+
+// Zbc extension
+let Latency = 3 in
+def : WriteRes<WriteCLMUL, [Andes45ALU]>;
+
+// Zbs extension
+def : WriteRes<WriteSingleBit, [Andes45ALU]>;
+def : WriteRes<WriteSingleBitImm, [Andes45ALU]>;
+def : WriteRes<WriteBEXT, [Andes45ALU]>;
+def : WriteRes<WriteBEXTI, [Andes45ALU]>;
+
+// Others
+def : WriteRes<WriteCSR, [Andes45CSR]>;
+def : WriteRes<WriteNop, []>;
+
+//===----------------------------------------------------------------------===//
+
+// Bypass and advance
+def : ReadAdvance<ReadIALU, 0>;
+def : ReadAdvance<ReadIALU32, 0>;
+def : ReadAdvance<ReadShiftImm, 0>;
+def : ReadAdvance<ReadShiftImm32, 0>;
+def : ReadAdvance<ReadShiftReg, 0>;
+def : ReadAdvance<ReadShiftReg32, 0>;
+def : ReadAdvance<ReadJalr, 0>;
+def : ReadAdvance<ReadJmp, 0>;
+def : ReadAdvance<ReadIMul, 0>;
+def : ReadAdvance<ReadIMul32, 0>;
+def : ReadAdvance<ReadIDiv, 0>;
+def : ReadAdvance<ReadIDiv32, 0>;
+def : ReadAdvance<ReadIRem, 0>;
+def : ReadAdvance<ReadIRem32, 0>;
+def : ReadAdvance<ReadStoreData, 0>;
+def : ReadAdvance<ReadMemBase, 0>;
+def : ReadAdvance<ReadAtomicWA, 0>;
+def : ReadAdvance<ReadAtomicWD, 0>;
+def : ReadAdvance<ReadAtomicDA, 0>;
+def : ReadAdvance<ReadAtomicDD, 0>;
+def : ReadAdvance<ReadAtomicLDW, 0>;
+def : ReadAdvance<ReadAtomicLDD, 0>;
+def : ReadAdvance<ReadAtomicSTW, 0>;
+def : ReadAdvance<ReadAtomicSTD, 0>;
+def : ReadAdvance<ReadFStoreData, 0>;
+def : ReadAdvance<ReadFMemBase, 0>;
+def : ReadAdvance<ReadFAdd16, 0>;
+def : ReadAdvance<ReadFAdd32, 0>;
+def : ReadAdvance<ReadFAdd64, 0>;
+def : ReadAdvance<ReadFMul16, 0>;
+def : ReadAdvance<ReadFMul32, 0>;
+def : ReadAdvance<ReadFMul64, 0>;
+def : ReadAdvance<ReadFMA16, 0>;
+def : ReadAdvance<ReadFMA32, 0>;
+def : ReadAdvance<ReadFMA64, 0>;
+def : ReadAdvance<ReadFMA16Addend, 0>;
+def : ReadAdvance<ReadFMA32Addend, 0>;
+def : ReadAdvance<ReadFMA64Addend, 0>;
+def : ReadAdvance<ReadFDiv16, 0>;
+def : ReadAdvance<ReadFDiv32, 0>;
+def : ReadAdvance<ReadFDiv64, 0>;
+def : ReadAdvance<ReadFSqrt16, 0>;
+def : ReadAdvance<ReadFSqrt32, 0>;
+def : ReadAdvance<ReadFSqrt64, 0>;
+def : ReadAdvance<ReadFSGNJ16, 0>;
+def : ReadAdvance<ReadFSGNJ32, 0>;
+def : ReadAdvance<ReadFSGNJ64, 0>;
+def : ReadAdvance<ReadFMovF16ToI16, 0>;
+def : ReadAdvance<ReadFMovI16ToF16, 0>;
+def : ReadAdvance<ReadFMovF32ToI32, 0>;
+def : ReadAdvance<ReadFMovI32ToF32, 0>;
+def : ReadAdvance<ReadFMovF64ToI64, 0>;
+def : ReadAdvance<ReadFMovI64ToF64, 0>;
+def : ReadAdvance<ReadFMinMax16, 0>;
+def : ReadAdvance<ReadFMinMax32, 0>;
+def : ReadAdvance<ReadFMinMax64, 0>;
+def : ReadAdvance<ReadFClass16, 0>;
+def : ReadAdvance<ReadFClass32, 0>;
+def : ReadAdvance<ReadFClass64, 0>;
+def : ReadAdvance<ReadFCmp16, 0>;
+def : ReadAdvance<ReadFCmp32, 0>;
+def : ReadAdvance<ReadFCmp64, 0>;
+def : ReadAdvance<ReadFCvtF16ToI32, 0>;
+def : ReadAdvance<ReadFCvtF16ToI64, 0>;
+def : ReadAdvance<ReadFCvtF32ToI32, 0>;
+def : ReadAdvance<ReadFCvtF32ToI64, 0>;
+def : ReadAdvance<ReadFCvtF64ToI32, 0>;
+def : ReadAdvance<ReadFCvtF64ToI64, 0>;
+def : ReadAdvance<ReadFCvtI32ToF16, 0>;
+def : ReadAdvance<ReadFCvtI32ToF32, 0>;
+def : ReadAdvance<ReadFCvtI32ToF64, 0>;
+def : ReadAdvance<ReadFCvtI64ToF16, 0>;
+def : ReadAdvance<ReadFCvtI64ToF32, 0>;
+def : ReadAdvance<ReadFCvtI64ToF64, 0>;
+def : ReadAdvance<ReadFCvtF16ToF32, 0>;
+def : ReadAdvance<ReadFCvtF16ToF64, 0>;
+def : ReadAdvance<ReadFCvtF32ToF16, 0>;
+def : ReadAdvance<ReadFCvtF32ToF64, 0>;
+def : ReadAdvance<ReadFCvtF64ToF16, 0>;
+def : ReadAdvance<ReadFCvtF64ToF32, 0>;
+def : ReadAdvance<ReadSHXADD, 0>;
+def : ReadAdvance<ReadSHXADD32, 0>;
+def : ReadAdvance<ReadRotateImm, 1>;
+def : ReadAdvance<ReadRotateImm32, 1>;
+def : ReadAdvance<ReadRotateReg, 1>;
+def : ReadAdvance<ReadRotateReg32, 1>;
+def : ReadAdvance<ReadCLZ, 0>;
+def : ReadAdvance<ReadCLZ32, 0>;
+def : ReadAdvance<ReadCTZ, 0>;
+def : ReadAdvance<ReadCTZ32, 0>;
+def : ReadAdvance<ReadCPOP, 0>;
+def : ReadAdvance<ReadCPOP32, 0>;
+def : ReadAdvance<ReadREV8, 0>;
+def : ReadAdvance<ReadORCB, 0>;
+def : ReadAdvance<ReadIMinMax, 0>;
+def : ReadAdvance<ReadCLMUL, 0>;
+def : ReadAdvance<ReadSingleBit, 0>;
+def : ReadAdvance<ReadSingleBitImm, 0>;
+def : ReadAdvance<ReadCSR, 0>;
+
+//===----------------------------------------------------------------------===//
+// Unsupported extensions
+defm : UnsupportedSchedQ;
+defm : UnsupportedSchedSFB;
+defm : UnsupportedSchedV;
+defm : UnsupportedSchedXsfvcp;
+defm : UnsupportedSchedZabha;
+defm : UnsupportedSchedZbkb;
+defm : UnsupportedSchedZbkx;
+defm : UnsupportedSchedZfa;
+defm : UnsupportedSchedZvk;
+}
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/external/llvm-project/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index af64a871a929..c1d7cd4a716e 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -88,9 +88,8 @@ class SiFive7GetCyclesSegmentedSeg2<string mx> {
 
 // Cycles for segmented loads and stores are calculated using the
 // formula vl * ceil((SEW * nf) / DLEN), where SEW * nf is the segment size.
-class SiFive7GetCyclesSegmented<string mx, int sew, int nf> {
-  defvar VLEN = 512;
-  defvar DLEN = 256;
+class SiFive7GetCyclesSegmented<string mx, int sew, int nf, int VLEN> {
+  defvar DLEN = !div(VLEN, 2);
   // (VLEN * LMUL) / SEW
   defvar VLUpperBound  = !cond(
     !eq(mx, "M1") : !div(VLEN, sew),
@@ -107,23 +106,20 @@ class SiFive7GetCyclesSegmented<string mx, int sew, int nf> {
   int c = !mul(VLUpperBound, !div(!sub(!add(a, b), 1), b));
 }
 
-class SiFive7GetCyclesOnePerElement<string mx, int sew> {
-  // FIXME: On SiFive7, VLEN is 512. Although a user can request the compiler
-  // to use a different VLEN, this model will not make scheduling decisions
-  // based on the user specified VLEN.
+class SiFive7GetCyclesOnePerElement<string mx, int sew, int VLEN> {
   // c = ceil(VLEN / SEW) * LMUL
   // Note: c >= 1 since the smallest VLEN is 512 / 8 = 8, and the
   // largest division performed on VLEN is in MF8 case with division
   // by 8. Therefore, there is no need to ceil the result.
-  int VLEN = !div(512, sew);
+  int numElements = !div(VLEN, sew);
   int c = !cond(
-    !eq(mx, "M1")  : VLEN,
-    !eq(mx, "M2")  : !mul(VLEN, 2),
-    !eq(mx, "M4")  : !mul(VLEN, 4),
-    !eq(mx, "M8")  : !mul(VLEN, 8),
-    !eq(mx, "MF2") : !div(VLEN, 2),
-    !eq(mx, "MF4") : !div(VLEN, 4),
-    !eq(mx, "MF8") : !div(VLEN, 8)
+    !eq(mx, "M1")  : numElements,
+    !eq(mx, "M2")  : !mul(numElements, 2),
+    !eq(mx, "M4")  : !mul(numElements, 4),
+    !eq(mx, "M8")  : !mul(numElements, 8),
+    !eq(mx, "MF2") : !div(numElements, 2),
+    !eq(mx, "MF4") : !div(numElements, 4),
+    !eq(mx, "MF8") : !div(numElements, 8)
   );
 }
 
@@ -139,10 +135,9 @@ class SiFive7GetDivOrSqrtFactor<int sew> {
 
 /// Cycles for reductions take approximately VL*SEW/DLEN + 5(4 + log(DLEN/SEW))
 /// cycles.
-class SiFive7GetReductionCycles<string mx, int sew> {
+class SiFive7GetReductionCycles<string mx, int sew, int VLEN> {
   // VLUpperBound*SEW/DLEN is equivalent to 2*LMUL since
   // VLUpperBound=(VLEN*LMUL)/SEW.
-  defvar VLEN = 512;
   defvar DLEN = !div(VLEN, 2);
   defvar TwoTimesLMUL = !cond(
     !eq(mx, "M1") : 2,
@@ -160,8 +155,7 @@ class SiFive7GetReductionCycles<string mx, int sew> {
 }
 
 /// Cycles for ordered reductions take approximately 6*VL cycles
-class SiFive7GetOrderedReductionCycles<string mx, int sew> {
-  defvar VLEN = 512;
+class SiFive7GetOrderedReductionCycles<string mx, int sew, int VLEN> {
   // (VLEN * LMUL) / SEW
   defvar VLUpperBound  = !cond(
     !eq(mx, "M1") : !div(VLEN, sew),
@@ -234,6 +228,8 @@ def SiFive7VCQ         : ProcResource<1>; // Vector Command Queue
 
 def SiFive7PipeAB : ProcResGroup<[SiFive7PipeA, SiFive7PipeB]>;
 
+defvar SiFive7VLEN = 512;
+
 // Branching
 let Latency = 3 in {
 def : WriteRes<WriteJmp, [SiFive7PipeB]>;
@@ -481,7 +477,7 @@ foreach mx = SchedMxList in {
 
 foreach mx = SchedMxList in {
   defvar VLDSX0Cycles = SiFive7GetCyclesDefault<mx>.c;
-  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 8>.c;
+  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 8, SiFive7VLEN>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   defm SiFive7 : LMULWriteResMXVariant<"WriteVLDS8",  VLDSX0Pred, [SiFive7VCQ, SiFive7VL],
                                        4, [0, 1], [1, !add(1, VLDSX0Cycles)], !add(3, Cycles),
@@ -501,7 +497,7 @@ foreach mx = SchedMxList in {
 // since LMUL >= 16/64.
 foreach mx = ["MF4", "MF2", "M1", "M2", "M4", "M8"] in {
   defvar VLDSX0Cycles = SiFive7GetCyclesDefault<mx>.c;
-  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 16>.c;
+  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 16, SiFive7VLEN>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   defm SiFive7 : LMULWriteResMXVariant<"WriteVLDS16",  VLDSX0Pred, [SiFive7VCQ, SiFive7VL],
                                        4, [0, 1], [1, !add(1, VLDSX0Cycles)], !add(3, Cycles),
@@ -518,7 +514,7 @@ foreach mx = ["MF4", "MF2", "M1", "M2", "M4", "M8"] in {
 }
 foreach mx = ["MF2", "M1", "M2", "M4", "M8"] in {
   defvar VLDSX0Cycles = SiFive7GetCyclesDefault<mx>.c;
-  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 32>.c;
+  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 32, SiFive7VLEN>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   defm SiFive7 : LMULWriteResMXVariant<"WriteVLDS32",  VLDSX0Pred, [SiFive7VCQ, SiFive7VL],
                                        4, [0, 1], [1, !add(1, VLDSX0Cycles)], !add(3, Cycles),
@@ -535,7 +531,7 @@ foreach mx = ["MF2", "M1", "M2", "M4", "M8"] in {
 }
 foreach mx = ["M1", "M2", "M4", "M8"] in {
   defvar VLDSX0Cycles = SiFive7GetCyclesDefault<mx>.c;
-  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 64>.c;
+  defvar Cycles = SiFive7GetCyclesOnePerElement<mx, 64, SiFive7VLEN>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   defm SiFive7 : LMULWriteResMXVariant<"WriteVLDS64",  VLDSX0Pred, [SiFive7VCQ, SiFive7VL],
                                        4, [0, 1], [1, !add(1, VLDSX0Cycles)], !add(3, Cycles),
@@ -588,7 +584,7 @@ foreach mx = SchedMxList in {
     let Latency = 1, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in
     defm "" : LMULWriteResMX<"WriteVSSEG2e" # eew,   [SiFive7VCQ, SiFive7VS], mx, IsWorstCase>;
     foreach nf=3-8 in {
-      defvar Cycles = SiFive7GetCyclesSegmented<mx, eew, nf>.c;
+      defvar Cycles = SiFive7GetCyclesSegmented<mx, eew, nf, SiFive7VLEN>.c;
       defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
       // Does not chain so set latency high
       let Latency = !add(3, Cycles), AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
@@ -603,7 +599,7 @@ foreach mx = SchedMxList in {
 foreach mx = SchedMxList in {
   foreach nf=2-8 in {
     foreach eew = [8, 16, 32, 64] in {
-      defvar Cycles = SiFive7GetCyclesSegmented<mx, eew, nf>.c;
+      defvar Cycles = SiFive7GetCyclesSegmented<mx, eew, nf, SiFive7VLEN>.c;
       defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
       // Does not chain so set latency high
       let Latency = !add(3, Cycles), AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
@@ -669,7 +665,7 @@ foreach mx = SchedMxList in {
 foreach mx = SchedMxList in {
   foreach sew = SchedSEWSet<mx>.val in {
     defvar Cycles = !mul(SiFive7GetDivOrSqrtFactor<sew>.c,
-                         !div(SiFive7GetCyclesOnePerElement<mx, sew>.c, 4));
+                         !div(SiFive7GetCyclesOnePerElement<mx, sew, SiFive7VLEN>.c, 4));
     defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
     let Latency = Cycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVIDivV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
@@ -774,7 +770,7 @@ foreach mx = SchedMxList in {
 foreach mx = SchedMxListF in {
   foreach sew = SchedSEWSet<mx, isF=1>.val in {
     defvar Cycles = !mul(SiFive7GetDivOrSqrtFactor<sew>.c,
-                         !div(SiFive7GetCyclesOnePerElement<mx, sew>.c, 4));
+                         !div(SiFive7GetCyclesOnePerElement<mx, sew, SiFive7VLEN>.c, 4));
     defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
     let Latency = Cycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFSqrtV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
@@ -834,7 +830,7 @@ foreach mx = SchedMxListFW in {
 // 14. Vector Reduction Operations
 foreach mx = SchedMxList in {
   foreach sew = SchedSEWSet<mx>.val in {
-    defvar Cycles = SiFive7GetReductionCycles<mx, sew>.c;
+    defvar Cycles = SiFive7GetReductionCycles<mx, sew, SiFive7VLEN>.c;
     defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
     let Latency = Cycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVIRedV_From", [SiFive7VCQ, SiFive7VA],
@@ -847,7 +843,7 @@ foreach mx = SchedMxList in {
 
 foreach mx = SchedMxListWRed in {
   foreach sew = SchedSEWSet<mx, 0, 1>.val in {
-    defvar Cycles = SiFive7GetReductionCycles<mx, sew>.c;
+    defvar Cycles = SiFive7GetReductionCycles<mx, sew, SiFive7VLEN>.c;
     defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListWRed>.c;
     let Latency = Cycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in
     defm "" : LMULSEWWriteResMXSEW<"WriteVIWRedV_From", [SiFive7VCQ, SiFive7VA],
@@ -857,7 +853,7 @@ foreach mx = SchedMxListWRed in {
 
 foreach mx = SchedMxListF in {
   foreach sew = SchedSEWSet<mx, 1>.val in {
-    defvar RedCycles = SiFive7GetReductionCycles<mx, sew>.c;
+    defvar RedCycles = SiFive7GetReductionCycles<mx, sew, SiFive7VLEN>.c;
     defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
     let Latency = RedCycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, RedCycles)] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFRedV_From", [SiFive7VCQ, SiFive7VA],
@@ -865,7 +861,7 @@ foreach mx = SchedMxListF in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFRedMinMaxV_From", [SiFive7VCQ, SiFive7VA],
                                      mx, sew, IsWorstCase>;
     }
-    defvar OrdRedCycles = SiFive7GetOrderedReductionCycles<mx, sew>.c;
+    defvar OrdRedCycles = SiFive7GetOrderedReductionCycles<mx, sew, SiFive7VLEN>.c;
     let Latency = OrdRedCycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, OrdRedCycles)] in
     defm "" : LMULSEWWriteResMXSEW<"WriteVFRedOV_From", [SiFive7VCQ, SiFive7VA],
                                    mx, sew, IsWorstCase>;
@@ -874,12 +870,12 @@ foreach mx = SchedMxListF in {
 
 foreach mx = SchedMxListFWRed in {
   foreach sew = SchedSEWSet<mx, 1, 1>.val in {
-    defvar RedCycles = SiFive7GetReductionCycles<mx, sew>.c;
+    defvar RedCycles = SiFive7GetReductionCycles<mx, sew, SiFive7VLEN>.c;
     defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListFWRed, 1>.c;
     let Latency = RedCycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, RedCycles)] in
     defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedV_From", [SiFive7VCQ, SiFive7VA],
                                    mx, sew, IsWorstCase>;
-    defvar OrdRedCycles = SiFive7GetOrderedReductionCycles<mx, sew>.c;
+    defvar OrdRedCycles = SiFive7GetOrderedReductionCycles<mx, sew, SiFive7VLEN>.c;
     let Latency = OrdRedCycles, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, OrdRedCycles)] in
     defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedOV_From", [SiFive7VCQ, SiFive7VA],
                                    mx, sew, IsWorstCase>;
@@ -924,7 +920,7 @@ foreach mx = SchedMxList in {
 
 foreach mx = SchedMxList in {
   foreach sew = SchedSEWSet<mx>.val in {
-    defvar Cycles = SiFive7GetCyclesOnePerElement<mx, sew>.c;
+    defvar Cycles = SiFive7GetCyclesOnePerElement<mx, sew, SiFive7VLEN>.c;
     defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
     let Latency = !add(Cycles, 3), AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherVV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVSubtarget.h b/external/llvm-project/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 0eef7b1feaf5..04c7ca7d0572 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -83,6 +83,7 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
     SiFive7,
     VentanaVeyron,
     MIPSP8700,
+    Andes45,
   };
   enum RISCVVRGatherCostModelEnum : uint8_t {
     Quadratic,
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 8342845ec6f2..8a47453cedcd 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -298,9 +298,8 @@ bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
 
 ScheduleDAGInstrs *
 RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
-  ScheduleDAGMILive *DAG = nullptr;
+  ScheduleDAGMILive *DAG = createSchedLive(C);
   if (EnableMISchedLoadStoreClustering) {
-    DAG = createGenericSchedLive(C);
     DAG->addMutation(createLoadClusterDAGMutation(
         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
     DAG->addMutation(createStoreClusterDAGMutation(
@@ -308,18 +307,16 @@ RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
   }
 
   const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
-  if (!DisableVectorMaskMutation && ST.hasVInstructions()) {
-    DAG = DAG ? DAG : createGenericSchedLive(C);
+  if (!DisableVectorMaskMutation && ST.hasVInstructions())
     DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
-  }
+
   return DAG;
 }
 
 ScheduleDAGInstrs *
 RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
-  ScheduleDAGMI *DAG = nullptr;
+  ScheduleDAGMI *DAG = createSchedPostRA(C);
   if (EnablePostMISchedLoadStoreClustering) {
-    DAG = createGenericSchedPostRA(C);
     DAG->addMutation(createLoadClusterDAGMutation(
         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
     DAG->addMutation(createStoreClusterDAGMutation(
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.cpp
index 3cb5c7e13dd7..bc90cf8f53ab 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "RISCVTargetObjectFile.h"
-#include "MCTargetDesc/RISCVMCExpr.h"
 #include "MCTargetDesc/RISCVMCObjectFileInfo.h"
 #include "RISCVTargetMachine.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -53,7 +52,7 @@ const MCExpr *RISCVELFTargetObjectFile::getIndirectSymViaGOTPCRel(
   const MCExpr *Res = MCSymbolRefExpr::create(Sym, Ctx);
   Res = MCBinaryExpr::createAdd(
       Res, MCConstantExpr::create(Offset + MV.getConstant(), Ctx), Ctx);
-  return RISCVMCExpr::create(Res, ELF::R_RISCV_GOT32_PCREL, Ctx);
+  return MCSpecifierExpr::create(Res, ELF::R_RISCV_GOT32_PCREL, Ctx);
 }
 
 // A address must be loaded from a small section if its size is less than the
@@ -180,10 +179,3 @@ MCSection *RISCVELFTargetObjectFile::getSectionForConstant(
   return TargetLoweringObjectFileELF::getSectionForConstant(DL, Kind, C,
                                                             Alignment);
 }
-
-const MCExpr *
-RISCVELFTargetObjectFile::createTargetMCExpr(const MCExpr *Expr,
-                                             uint8_t Specifier) const {
-  return RISCVMCExpr::create(Expr, RISCVMCExpr::Specifier(Specifier),
-                             getContext());
-}
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
index b6da3f4721f4..ff7e3e4c752c 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
@@ -48,9 +48,6 @@ class RISCVELFTargetObjectFile : public TargetLoweringObjectFileELF {
 
   bool isInSmallSection(uint64_t Size) const;
 
-  const MCExpr *createTargetMCExpr(const MCExpr *Expr,
-                                   uint8_t Specifier) const override;
-
   const MCExpr *getIndirectSymViaGOTPCRel(const GlobalValue *GV,
                                           const MCSymbol *Sym,
                                           const MCValue &MV, int64_t Offset,
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index ff822dec232a..bee47527cf42 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -305,8 +305,7 @@ InstructionCost RISCVTTIImpl::getPartialReductionCost(
   if (!ST->hasStdExtZvqdotq() || ST->getELen() < 64 ||
       Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
       InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(8) ||
-      OpAExtend != OpBExtend || !AccumType->isIntegerTy(32) ||
-      !VF.isKnownMultipleOf(4))
+      !AccumType->isIntegerTy(32) || !VF.isKnownMultipleOf(4))
     return InstructionCost::getInvalid();
 
   Type *Tp = VectorType::get(AccumType, VF.divideCoefficientBy(4));
@@ -2953,5 +2952,22 @@ RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
     Options.LoadSizes = {4, 2, 1};
     Options.AllowedTailExpansions = {3};
   }
+
+  if (IsZeroCmp && ST->hasVInstructions()) {
+    unsigned RealMinVLen = ST->getRealMinVLen();
+    // Support Fractional LMULs if the lengths are larger than XLen.
+    // TODO: Support non-power-of-2 types.
+    for (unsigned FLMUL = 8; FLMUL >= 2; FLMUL /= 2) {
+      unsigned Len = RealMinVLen / FLMUL;
+      if (Len > ST->getXLen())
+        Options.LoadSizes.insert(Options.LoadSizes.begin(), Len / 8);
+    }
+    for (unsigned LMUL = 1; LMUL <= ST->getMaxLMULForFixedLengthVectors();
+         LMUL *= 2) {
+      unsigned Len = RealMinVLen * LMUL;
+      if (Len > ST->getXLen())
+        Options.LoadSizes.insert(Options.LoadSizes.begin(), Len / 8);
+    }
+  }
   return Options;
 }
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 0a784461d67b..dd7e9f7709f8 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -25,7 +25,7 @@
 
 namespace llvm {
 
-class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
+class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
   using BaseT = BasicTTIImplBase<RISCVTTIImpl>;
   using TTI = TargetTransformInfo;
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
index 0bb81a58c29a..d7aa85048eed 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -1330,8 +1330,8 @@ RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const {
         UserMI.getOperand(0).getSubReg() == RISCV::NoSubRegister &&
         UserMI.getOperand(1).getSubReg() == RISCV::NoSubRegister) {
       LLVM_DEBUG(dbgs() << "    Peeking through uses of COPY\n");
-      for (auto &CopyUse : MRI->use_operands(UserMI.getOperand(0).getReg()))
-        Worklist.insert(&CopyUse);
+      Worklist.insert_range(llvm::make_pointer_range(
+          MRI->use_operands(UserMI.getOperand(0).getReg())));
       continue;
     }
 
@@ -1340,8 +1340,8 @@ RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const {
       if (!PHISeen.insert(&UserMI).second)
         continue;
       LLVM_DEBUG(dbgs() << "    Peeking through uses of PHI\n");
-      for (auto &PhiUse : MRI->use_operands(UserMI.getOperand(0).getReg()))
-        Worklist.insert(&PhiUse);
+      Worklist.insert_range(llvm::make_pointer_range(
+          MRI->use_operands(UserMI.getOperand(0).getReg())));
       continue;
     }
 
diff --git a/external/llvm-project/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/external/llvm-project/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 6bb026378274..c9c2413d009b 100644
--- a/external/llvm-project/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/external/llvm-project/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -611,7 +611,7 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
 
   MachineInstr *Src = MRI->getVRegDef(MI.getOperand(2).getReg());
   if (!Src || Src->hasUnmodeledSideEffects() ||
-      Src->getParent() != MI.getParent() || Src->getNumDefs() != 1 ||
+      Src->getParent() != MI.getParent() ||
       !RISCVII::isFirstDefTiedToFirstUse(Src->getDesc()) ||
       !RISCVII::hasVLOp(Src->getDesc().TSFlags) ||
       !RISCVII::hasVecPolicyOp(Src->getDesc().TSFlags))
@@ -622,7 +622,7 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
     return false;
 
   // Src needs to have the same passthru as VMV_V_V
-  MachineOperand &SrcPassthru = Src->getOperand(1);
+  MachineOperand &SrcPassthru = Src->getOperand(Src->getNumExplicitDefs());
   if (SrcPassthru.getReg() != RISCV::NoRegister &&
       SrcPassthru.getReg() != Passthru.getReg())
     return false;
@@ -643,7 +643,8 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
     // If Src is masked then its passthru needs to be in VRNoV0.
     if (Passthru.getReg() != RISCV::NoRegister)
       MRI->constrainRegClass(Passthru.getReg(),
-                             TII->getRegClass(Src->getDesc(), 1, TRI,
+                             TII->getRegClass(Src->getDesc(),
+                                              SrcPassthru.getOperandNo(), TRI,
                                               *Src->getParent()->getParent()));
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
index 342456757409..0ed97f5b41c5 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
@@ -68,7 +68,8 @@ getSymbolicOperandMnemonic(SPIRV::OperandCategory::OperandCategory Category,
       Category != SPIRV::OperandCategory::FunctionControlOperand &&
       Category != SPIRV::OperandCategory::MemorySemanticsOperand &&
       Category != SPIRV::OperandCategory::MemoryOperandOperand &&
-      Category != SPIRV::OperandCategory::KernelProfilingInfoOperand)
+      Category != SPIRV::OperandCategory::KernelProfilingInfoOperand &&
+      Category != SPIRV::OperandCategory::SpecConstantOpOperandsOperand)
     return "UNKNOWN";
   // Value that encodes many enum values (one bit per enum value).
   std::string Name;
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
index 083c7f8460bf..b8c467fef8e8 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
@@ -222,6 +222,11 @@ namespace CooperativeMatrixOperands {
 #include "SPIRVGenTables.inc"
 } // namespace CooperativeMatrixOperands
 
+namespace SpecConstantOpOperands {
+#define GET_SpecConstantOpOperands_DECL
+#include "SPIRVGenTables.inc"
+} // namespace SpecConstantOpOperands
+
 struct ExtendedBuiltin {
   StringRef Name;
   InstructionSet::InstructionSet Set;
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 1b39a789ffdd..26b94788b810 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -510,6 +510,22 @@ void SPIRVAsmPrinter::outputExecutionMode(const Module &M) {
       continue;
     MCRegister FReg = MAI->getFuncReg(&F);
     assert(FReg.isValid());
+
+    if (Attribute Attr = F.getFnAttribute("hlsl.shader"); Attr.isValid()) {
+      // SPIR-V common validation: Fragment requires OriginUpperLeft or
+      // OriginLowerLeft.
+      // VUID-StandaloneSpirv-OriginLowerLeft-04653: Fragment must declare
+      // OriginUpperLeft.
+      if (Attr.getValueAsString() == "pixel") {
+        MCInst Inst;
+        Inst.setOpcode(SPIRV::OpExecutionMode);
+        Inst.addOperand(MCOperand::createReg(FReg));
+        unsigned EM =
+            static_cast<unsigned>(SPIRV::ExecutionMode::OriginUpperLeft);
+        Inst.addOperand(MCOperand::createImm(EM));
+        outputMCInst(Inst);
+      }
+    }
     if (MDNode *Node = F.getMetadata("reqd_work_group_size"))
       outputExecutionModeFromMDNode(FReg, Node, SPIRV::ExecutionMode::LocalSize,
                                     3, 1);
@@ -532,7 +548,7 @@ void SPIRVAsmPrinter::outputExecutionMode(const Module &M) {
       Inst.addOperand(MCOperand::createImm(TypeCode));
       outputMCInst(Inst);
     }
-    if (ST->isOpenCLEnv() && !M.getNamedMetadata("spirv.ExecutionMode") &&
+    if (ST->isKernel() && !M.getNamedMetadata("spirv.ExecutionMode") &&
         !M.getNamedMetadata("opencl.enable.FP_CONTRACT")) {
       MCInst Inst;
       Inst.setOpcode(SPIRV::OpExecutionMode);
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 5991a9af6364..36cc5cbe655b 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -267,18 +267,49 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx,
 
 static SPIRV::ExecutionModel::ExecutionModel
 getExecutionModel(const SPIRVSubtarget &STI, const Function &F) {
-  if (STI.isOpenCLEnv())
+  if (STI.isKernel())
     return SPIRV::ExecutionModel::Kernel;
 
+  if (STI.isShader()) {
+    auto attribute = F.getFnAttribute("hlsl.shader");
+    if (!attribute.isValid()) {
+      report_fatal_error(
+          "This entry point lacks mandatory hlsl.shader attribute.");
+    }
+
+    const auto value = attribute.getValueAsString();
+    if (value == "compute")
+      return SPIRV::ExecutionModel::GLCompute;
+    if (value == "vertex")
+      return SPIRV::ExecutionModel::Vertex;
+    if (value == "pixel")
+      return SPIRV::ExecutionModel::Fragment;
+
+    report_fatal_error(
+        "This HLSL entry point is not supported by this backend.");
+  }
+
+  assert(STI.getEnv() == SPIRVSubtarget::Unknown);
+  // "hlsl.shader" attribute is mandatory for Vulkan, so we can set Env to
+  // Shader whenever we find it, and to Kernel otherwise.
+
+  // We will now change the Env based on the attribute, so we need to strip
+  // `const` out of the ref to STI.
+  SPIRVSubtarget *NonConstSTI = const_cast<SPIRVSubtarget *>(&STI);
   auto attribute = F.getFnAttribute("hlsl.shader");
   if (!attribute.isValid()) {
-    report_fatal_error(
-        "This entry point lacks mandatory hlsl.shader attribute.");
+    NonConstSTI->setEnv(SPIRVSubtarget::Kernel);
+    return SPIRV::ExecutionModel::Kernel;
   }
+  NonConstSTI->setEnv(SPIRVSubtarget::Shader);
 
   const auto value = attribute.getValueAsString();
   if (value == "compute")
     return SPIRV::ExecutionModel::GLCompute;
+  if (value == "vertex")
+    return SPIRV::ExecutionModel::Vertex;
+  if (value == "pixel")
+    return SPIRV::ExecutionModel::Fragment;
 
   report_fatal_error("This HLSL entry point is not supported by this backend.");
 }
@@ -319,7 +350,7 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
           buildOpDecorate(VRegs[i][0], MIRBuilder,
                           SPIRV::Decoration::MaxByteOffset, {DerefBytes});
       }
-      if (Arg.hasAttribute(Attribute::Alignment) && !ST->isVulkanEnv()) {
+      if (Arg.hasAttribute(Attribute::Alignment) && !ST->isShader()) {
         auto Alignment = static_cast<unsigned>(
             Arg.getAttribute(Attribute::Alignment).getValueAsInt());
         buildOpDecorate(VRegs[i][0], MIRBuilder, SPIRV::Decoration::Alignment,
@@ -439,6 +470,11 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
 
   // Handle entry points and function linkage.
   if (isEntryPoint(F)) {
+    // EntryPoints can help us to determine the environment we're working on.
+    // Therefore, we need a non-const pointer to SPIRVSubtarget to update the
+    // environment if we need to.
+    const SPIRVSubtarget *ST =
+        static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
     auto MIB = MIRBuilder.buildInstr(SPIRV::OpEntryPoint)
                    .addImm(static_cast<uint32_t>(getExecutionModel(*ST, F)))
                    .addUse(FuncVReg);
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 51fe725c8618..292b83e05b56 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -772,10 +772,10 @@ Register SPIRVGlobalRegistry::buildGlobalVariable(
   // TODO: maybe move to GenerateDecorations pass.
   const SPIRVSubtarget &ST =
       cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());
-  if (IsConst && ST.isOpenCLEnv())
+  if (IsConst && !ST.isShader())
     buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Constant, {});
 
-  if (GVar && GVar->getAlign().valueOrOne().value() != 1 && !ST.isVulkanEnv()) {
+  if (GVar && GVar->getAlign().valueOrOne().value() != 1 && !ST.isShader()) {
     unsigned Alignment = (unsigned)GVar->getAlign().valueOrOne().value();
     buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Alignment, {Alignment});
   }
@@ -799,107 +799,15 @@ Register SPIRVGlobalRegistry::buildGlobalVariable(
   return Reg;
 }
 
-static std::string GetSpirvImageTypeName(const SPIRVType *Type,
-                                         MachineIRBuilder &MIRBuilder,
-                                         const std::string &Prefix,
-                                         SPIRVGlobalRegistry &GR);
-
 // Returns a name based on the Type. Notes that this does not look at
 // decorations, and will return the same string for two types that are the same
 // except for decorations.
-static std::string buildSpirvTypeName(const SPIRVType *Type,
-                                      MachineIRBuilder &MIRBuilder,
-                                      SPIRVGlobalRegistry &GR) {
-  switch (Type->getOpcode()) {
-  case SPIRV::OpTypeSampledImage: {
-    return GetSpirvImageTypeName(Type, MIRBuilder, "sampled_image_", GR);
-  }
-  case SPIRV::OpTypeImage: {
-    return GetSpirvImageTypeName(Type, MIRBuilder, "image_", GR);
-  }
-  case SPIRV::OpTypeArray: {
-    MachineRegisterInfo *MRI = MIRBuilder.getMRI();
-    Register ElementTypeReg = Type->getOperand(1).getReg();
-    auto *ElementType = MRI->getUniqueVRegDef(ElementTypeReg);
-    uint32_t ArraySize = getArrayComponentCount(MRI, Type);
-    return (buildSpirvTypeName(ElementType, MIRBuilder, GR) + Twine("[") +
-            Twine(ArraySize) + Twine("]"))
-        .str();
-  }
-  case SPIRV::OpTypeFloat:
-    return ("f" + Twine(Type->getOperand(1).getImm())).str();
-  case SPIRV::OpTypeSampler:
-    return ("sampler");
-  case SPIRV::OpTypeInt:
-    if (Type->getOperand(2).getImm())
-      return ("i" + Twine(Type->getOperand(1).getImm())).str();
-    return ("u" + Twine(Type->getOperand(1).getImm())).str();
-  case SPIRV::OpTypePointer: {
-    uint32_t StorageClass = GR.getPointerStorageClass(Type);
-    SPIRVType *PointeeType = GR.getPointeeType(Type);
-    return ("p_" + Twine(StorageClass) + Twine("_") +
-            buildSpirvTypeName(PointeeType, MIRBuilder, GR))
-        .str();
-  }
-  case SPIRV::OpTypeStruct: {
-    std::string TypeName = "{";
-    for (uint32_t I = 1; I < Type->getNumOperands(); ++I) {
-      SPIRVType *MemberType =
-          GR.getSPIRVTypeForVReg(Type->getOperand(I).getReg());
-      TypeName += '_' + buildSpirvTypeName(MemberType, MIRBuilder, GR);
-    }
-    return TypeName + "}";
-  }
-  case SPIRV::OpTypeVector: {
-    MachineRegisterInfo *MRI = MIRBuilder.getMRI();
-    Register ElementTypeReg = Type->getOperand(1).getReg();
-    auto *ElementType = MRI->getUniqueVRegDef(ElementTypeReg);
-    uint32_t VectorSize = GR.getScalarOrVectorComponentCount(Type);
-    return (buildSpirvTypeName(ElementType, MIRBuilder, GR) + Twine("[") +
-            Twine(VectorSize) + Twine("]"))
-        .str();
-  }
-  case SPIRV::OpTypeRuntimeArray: {
-    MachineRegisterInfo *MRI = MIRBuilder.getMRI();
-    Register ElementTypeReg = Type->getOperand(1).getReg();
-    auto *ElementType = MRI->getUniqueVRegDef(ElementTypeReg);
-    uint32_t ArraySize = 0;
-    return (buildSpirvTypeName(ElementType, MIRBuilder, GR) + Twine("[") +
-            Twine(ArraySize) + Twine("]"))
-        .str();
-  }
-  default:
-    llvm_unreachable("Trying to the the name of an unknown type.");
-  }
-}
-
-static std::string GetSpirvImageTypeName(const SPIRVType *Type,
-                                         MachineIRBuilder &MIRBuilder,
-                                         const std::string &Prefix,
-                                         SPIRVGlobalRegistry &GR) {
-  Register SampledTypeReg = Type->getOperand(1).getReg();
-  auto *SampledType = MIRBuilder.getMRI()->getUniqueVRegDef(SampledTypeReg);
-  std::string TypeName =
-      Prefix + buildSpirvTypeName(SampledType, MIRBuilder, GR);
-  for (uint32_t I = 2; I < Type->getNumOperands(); ++I) {
-    TypeName = (TypeName + '_' + Twine(Type->getOperand(I).getImm())).str();
-  }
-  return TypeName;
-}
-
 Register SPIRVGlobalRegistry::getOrCreateGlobalVariableWithBinding(
-    const SPIRVType *VarType, uint32_t Set, uint32_t Binding,
+    const SPIRVType *VarType, uint32_t Set, uint32_t Binding, StringRef Name,
     MachineIRBuilder &MIRBuilder) {
   Register VarReg =
       MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
 
-  // TODO(138533): The name should come from the llvm-ir, but how that name will
-  // be passed from the HLSL to the backend has not been decided. Using this
-  // place holder for now.
-  std::string Name =
-      ("__resource_" + buildSpirvTypeName(VarType, MIRBuilder, *this) + "_" +
-       Twine(Set) + "_" + Twine(Binding))
-          .str();
   buildGlobalVariable(VarReg, VarType, Name, nullptr,
                       getPointerStorageClass(VarType), nullptr, false, false,
                       SPIRV::LinkageType::Import, MIRBuilder, false);
@@ -988,7 +896,7 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeStruct(
   Register ResVReg = createTypeVReg(MIRBuilder);
   if (Ty->hasName())
     buildOpName(ResVReg, Ty->getName(), MIRBuilder);
-  if (Ty->isPacked() && !ST.isVulkanEnv())
+  if (Ty->isPacked() && !ST.isShader())
     buildOpDecorate(ResVReg, MIRBuilder, SPIRV::Decoration::CPacked, {});
 
   SPIRVType *SPVType =
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 3b481b3aba0c..35f616a1981d 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -548,6 +548,7 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
                                bool IsInstSelector);
   Register getOrCreateGlobalVariableWithBinding(const SPIRVType *VarType,
                                                 uint32_t Set, uint32_t Binding,
+                                                StringRef Name,
                                                 MachineIRBuilder &MIRBuilder);
 
   // Convenient helpers for getting types with check for duplicates.
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 338f6809a3e4..049ba0275f22 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -245,7 +245,7 @@ def OpSpecConstantComposite: Op<51, (outs ID:$res), (ins TYPE:$type, variable_op
                   "$res = OpSpecConstantComposite $type">;
 def OpSpecConstantCompositeContinuedINTEL: Op<6092, (outs), (ins variable_ops),
                   "OpSpecConstantCompositeContinuedINTEL">;
-def OpSpecConstantOp: Op<52, (outs ID:$res), (ins TYPE:$t, i32imm:$c, ID:$o, variable_ops),
+def OpSpecConstantOp: Op<52, (outs ID:$res), (ins TYPE:$t, SpecConstantOpOperands:$c, ID:$o, variable_ops),
                   "$res = OpSpecConstantOp $t $c $o">;
 
 // 3.42.8 Memory Instructions
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 5258f07d2f71..911a6966aaef 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -322,7 +322,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
                                   SPIRV::StorageClass::StorageClass SC,
                                   uint32_t Set, uint32_t Binding,
                                   uint32_t ArraySize, Register IndexReg,
-                                  bool IsNonUniform,
+                                  bool IsNonUniform, StringRef Name,
                                   MachineIRBuilder MIRBuilder) const;
   SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const;
   bool extractSubvector(Register &ResVReg, const SPIRVType *ResType,
@@ -860,7 +860,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
                 .addUse(GV);
         return MIB.constrainAllUses(TII, TRI, RBI) &&
                BuildMI(BB, I, I.getDebugLoc(),
-                       TII.get(STI.isVulkanEnv()
+                       TII.get(STI.isLogicalSPIRV()
                                    ? SPIRV::OpInBoundsAccessChain
                                    : SPIRV::OpInBoundsPtrAccessChain))
                    .addDef(ResVReg)
@@ -1034,7 +1034,7 @@ bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
                                           const SPIRVType *ResType,
                                           MachineInstr &I,
                                           unsigned Opcode) const {
-  if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) {
+  if (STI.isPhysicalSPIRV() && I.getOperand(1).isReg()) {
     Register SrcReg = I.getOperand(1).getReg();
     bool IsGV = false;
     for (MachineRegisterInfo::def_instr_iterator DefIt =
@@ -1564,15 +1564,14 @@ static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) {
   bool IsGRef = false;
   bool IsAllowedRefs =
-      std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(),
-                  [&IsGRef](auto const &It) {
-                    unsigned Opcode = It.getOpcode();
-                    if (Opcode == SPIRV::OpConstantComposite ||
-                        Opcode == SPIRV::OpVariable ||
-                        isSpvIntrinsic(It, Intrinsic::spv_init_global))
-                      return IsGRef = true;
-                    return Opcode == SPIRV::OpName;
-                  });
+      llvm::all_of(MRI->use_instructions(ResVReg), [&IsGRef](auto const &It) {
+        unsigned Opcode = It.getOpcode();
+        if (Opcode == SPIRV::OpConstantComposite ||
+            Opcode == SPIRV::OpVariable ||
+            isSpvIntrinsic(It, Intrinsic::spv_init_global))
+          return IsGRef = true;
+        return Opcode == SPIRV::OpName;
+      });
   return IsAllowedRefs && IsGRef;
 }
 
@@ -2062,7 +2061,7 @@ bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
   auto ExtractOp =
       Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
 
-  bool ZeroAsNull = STI.isOpenCLEnv();
+  bool ZeroAsNull = !STI.isShader();
   // Extract the i8 element, multiply and add it to the accumulator
   for (unsigned i = 0; i < 4; i++) {
     // A[i]
@@ -2202,7 +2201,7 @@ bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
                  .addDef(ResVReg)
                  .addUse(GR.getSPIRVTypeID(ResType))
                  .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I,
-                                                IntTy, TII, STI.isOpenCLEnv()));
+                                                IntTy, TII, !STI.isShader()));
 
   for (unsigned J = 2; J < I.getNumOperands(); J++) {
     BMI.addUse(I.getOperand(J).getReg());
@@ -2226,7 +2225,7 @@ bool SPIRVInstructionSelector::selectWaveActiveCountBits(
                 .addDef(ResVReg)
                 .addUse(GR.getSPIRVTypeID(ResType))
                 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy,
-                                               TII, STI.isOpenCLEnv()))
+                                               TII, !STI.isShader()))
                 .addImm(SPIRV::GroupOperation::Reduce)
                 .addUse(BallotReg)
                 .constrainAllUses(TII, TRI, RBI);
@@ -2257,7 +2256,7 @@ bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
       .addDef(ResVReg)
       .addUse(GR.getSPIRVTypeID(ResType))
       .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
-                                     STI.isOpenCLEnv()))
+                                     !STI.isShader()))
       .addImm(SPIRV::GroupOperation::Reduce)
       .addUse(I.getOperand(2).getReg())
       .constrainAllUses(TII, TRI, RBI);
@@ -2284,7 +2283,7 @@ bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
       .addDef(ResVReg)
       .addUse(GR.getSPIRVTypeID(ResType))
       .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
-                                     STI.isOpenCLEnv()))
+                                     !STI.isShader()))
       .addImm(SPIRV::GroupOperation::Reduce)
       .addUse(I.getOperand(2).getReg());
 }
@@ -2506,7 +2505,7 @@ bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
                                                  MachineInstr &I) const {
   // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
-  bool ZeroAsNull = STI.isOpenCLEnv();
+  bool ZeroAsNull = !STI.isShader();
   if (ResType->getOpcode() == SPIRV::OpTypeVector)
     return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
   return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
@@ -2515,7 +2514,7 @@ Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType,
                                                   MachineInstr &I) const {
   // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
-  bool ZeroAsNull = STI.isOpenCLEnv();
+  bool ZeroAsNull = !STI.isShader();
   APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType));
   if (ResType->getOpcode() == SPIRV::OpTypeVector)
     return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
@@ -2525,7 +2524,7 @@ Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType,
 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType,
                                                  MachineInstr &I) const {
   // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
-  bool ZeroAsNull = STI.isOpenCLEnv();
+  bool ZeroAsNull = !STI.isShader();
   APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType));
   if (ResType->getOpcode() == SPIRV::OpTypeVector)
     return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull);
@@ -2713,10 +2712,10 @@ bool SPIRVInstructionSelector::selectConst(Register ResVReg,
     Reg = GR.getOrCreateConstNullPtr(MIRBuilder, ResType);
   } else if (Opcode == TargetOpcode::G_FCONSTANT) {
     Reg = GR.getOrCreateConstFP(I.getOperand(1).getFPImm()->getValue(), I,
-                                ResType, TII, STI.isOpenCLEnv());
+                                ResType, TII, !STI.isShader());
   } else {
     Reg = GR.getOrCreateConstInt(I.getOperand(1).getCImm()->getZExtValue(), I,
-                                 ResType, TII, STI.isOpenCLEnv());
+                                 ResType, TII, !STI.isShader());
   }
   return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I);
 }
@@ -2796,7 +2795,7 @@ bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
   // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
   // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
   // we have to use Op[InBounds]AccessChain.
-  const unsigned Opcode = STI.isVulkanEnv()
+  const unsigned Opcode = STI.isLogicalSPIRV()
                               ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
                                                : SPIRV::OpAccessChain)
                               : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
@@ -3380,14 +3379,14 @@ bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
 Register SPIRVInstructionSelector::buildPointerToResource(
     const SPIRVType *SpirvResType, SPIRV::StorageClass::StorageClass SC,
     uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
-    bool IsNonUniform, MachineIRBuilder MIRBuilder) const {
+    bool IsNonUniform, StringRef Name, MachineIRBuilder MIRBuilder) const {
   const Type *ResType = GR.getTypeForSPIRVType(SpirvResType);
   if (ArraySize == 1) {
     SPIRVType *PtrType =
         GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
     assert(GR.getPointeeType(PtrType) == SpirvResType &&
            "SpirvResType did not have an explicit layout.");
-    return GR.getOrCreateGlobalVariableWithBinding(PtrType, Set, Binding,
+    return GR.getOrCreateGlobalVariableWithBinding(PtrType, Set, Binding, Name,
                                                    MIRBuilder);
   }
 
@@ -3395,7 +3394,7 @@ Register SPIRVInstructionSelector::buildPointerToResource(
   SPIRVType *VarPointerType =
       GR.getOrCreateSPIRVPointerType(VarType, MIRBuilder, SC);
   Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
-      VarPointerType, Set, Binding, MIRBuilder);
+      VarPointerType, Set, Binding, Name, MIRBuilder);
 
   SPIRVType *ResPointerType =
       GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
@@ -3493,7 +3492,7 @@ bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
 
   // On odd component counts we need to handle one more component
   if (CurrentComponent != ComponentCount) {
-    bool ZeroAsNull = STI.isOpenCLEnv();
+    bool ZeroAsNull = !STI.isShader();
     Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type));
     Register ConstIntLastIdx = GR.getOrCreateConstInt(
         ComponentCount - 1, I, BaseType, TII, ZeroAsNull);
@@ -3523,7 +3522,7 @@ bool SPIRVInstructionSelector::selectFirstBitSet64(
     Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const {
   unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
   SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType);
-  bool ZeroAsNull = STI.isOpenCLEnv();
+  bool ZeroAsNull = !STI.isShader();
   Register ConstIntZero =
       GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull);
   Register ConstIntOne =
@@ -3725,7 +3724,7 @@ bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
                  .addUse(GR.getSPIRVTypeID(ResType))
                  .addUse(I.getOperand(2).getReg())
                  .constrainAllUses(TII, TRI, RBI);
-  if (!STI.isVulkanEnv()) {
+  if (!STI.isShader()) {
     unsigned Alignment = I.getOperand(3).getImm();
     buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment});
   }
@@ -3744,7 +3743,7 @@ bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
                  .addUse(GR.getSPIRVTypeID(ResType))
                  .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
                  .constrainAllUses(TII, TRI, RBI);
-  if (!STI.isVulkanEnv()) {
+  if (!STI.isShader()) {
     unsigned Alignment = I.getOperand(2).getImm();
     buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment,
                     {Alignment});
@@ -4081,6 +4080,9 @@ bool SPIRVInstructionSelector::loadHandleBeforePosition(
   uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI);
   Register IndexReg = HandleDef.getOperand(5).getReg();
   bool IsNonUniform = ArraySize > 1 && foldImm(HandleDef.getOperand(6), MRI);
+  std::string Name =
+      getStringValueFromReg(HandleDef.getOperand(7).getReg(), *MRI);
+
   bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
   MachineIRBuilder MIRBuilder(HandleDef);
   SPIRVType *VarType = ResType;
@@ -4091,8 +4093,9 @@ bool SPIRVInstructionSelector::loadHandleBeforePosition(
     SC = GR.getPointerStorageClass(ResType);
   }
 
-  Register VarReg = buildPointerToResource(VarType, SC, Set, Binding, ArraySize,
-                                           IndexReg, IsNonUniform, MIRBuilder);
+  Register VarReg =
+      buildPointerToResource(VarType, SC, Set, Binding, ArraySize, IndexReg,
+                             IsNonUniform, Name, MIRBuilder);
 
   if (IsNonUniform)
     buildOpDecorate(HandleReg, HandleDef, TII, SPIRV::Decoration::NonUniformEXT,
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 2fdd54fdfc39..b71a9dd68dd4 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -68,7 +68,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
                                SPIRV::RequirementHandler &Reqs) {
   // A set of capabilities to avoid if there is another option.
   AvoidCapabilitiesSet AvoidCaps;
-  if (ST.isOpenCLEnv())
+  if (!ST.isShader())
     AvoidCaps.S.insert(SPIRV::Capability::Shader);
 
   VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, i);
@@ -144,8 +144,8 @@ void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
         static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MemMD, 1));
   } else {
     // TODO: Add support for VulkanMemoryModel.
-    MAI.Mem = ST->isOpenCLEnv() ? SPIRV::MemoryModel::OpenCL
-                                : SPIRV::MemoryModel::GLSL450;
+    MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
+                             : SPIRV::MemoryModel::OpenCL;
     if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
       unsigned PtrSize = ST->getPointerSize();
       MAI.Addr = PtrSize == 32   ? SPIRV::AddressingModel::Physical32
@@ -175,7 +175,7 @@ void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
     // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling
     // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV
     // Translator avoids potential issues with run-times in a similar manner.
-    if (ST->isOpenCLEnv()) {
+    if (!ST->isShader()) {
       MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
       MAI.SrcLangVersion = 100000;
     } else {
@@ -203,7 +203,7 @@ void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
   MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand,
                                  MAI.Addr, *ST);
 
-  if (ST->isOpenCLEnv()) {
+  if (!ST->isShader()) {
     // TODO: check if it's required by default.
     MAI.ExtInstSetMap[static_cast<unsigned>(
         SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
@@ -804,12 +804,12 @@ void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) {
     addAvailableCaps(EnabledCapabilities);
   }
 
-  if (ST.isOpenCLEnv()) {
+  if (!ST.isShader()) {
     initAvailableCapabilitiesForOpenCL(ST);
     return;
   }
 
-  if (ST.isVulkanEnv()) {
+  if (ST.isShader()) {
     initAvailableCapabilitiesForVulkan(ST);
     return;
   }
@@ -969,7 +969,7 @@ static void addOpTypeImageReqs(const MachineInstr &MI,
   }
 
   // Has optional access qualifier.
-  if (ST.isOpenCLEnv()) {
+  if (!ST.isShader()) {
     if (MI.getNumOperands() > 8 &&
         MI.getOperand(8).getImm() == SPIRV::AccessQualifier::ReadWrite)
       Reqs.addRequirements(SPIRV::Capability::ImageReadWrite);
@@ -1267,7 +1267,7 @@ void addInstrRequirements(const MachineInstr &MI,
                                ST);
     // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer
     // capability.
-    if (!ST.isOpenCLEnv())
+    if (ST.isShader())
       break;
     assert(MI.getOperand(2).isReg());
     const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
@@ -1342,7 +1342,7 @@ void addInstrRequirements(const MachineInstr &MI,
     addOpTypeImageReqs(MI, Reqs, ST);
     break;
   case SPIRV::OpTypeSampler:
-    if (!ST.isVulkanEnv()) {
+    if (!ST.isShader()) {
       Reqs.addCapability(SPIRV::Capability::ImageBasic);
     }
     break;
@@ -1793,7 +1793,8 @@ void addInstrRequirements(const MachineInstr &MI,
     // not allowed to produce
     // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
     // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
-    if (isImageTypeWithUnknownFormat(TypeDef) && !ST.isOpenCLEnv())
+
+    if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())
       Reqs.addCapability(SPIRV::Capability::StorageImageReadWithoutFormat);
     break;
   }
@@ -1806,7 +1807,8 @@ void addInstrRequirements(const MachineInstr &MI,
     // not allowed to produce
     // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
     // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
-    if (isImageTypeWithUnknownFormat(TypeDef) && !ST.isOpenCLEnv())
+
+    if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())
       Reqs.addCapability(SPIRV::Capability::StorageImageWriteWithoutFormat);
     break;
   }
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 65c7b9aafe7d..2bffbf73b574 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -405,13 +405,13 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
         Changed = true;
         break;
       case Intrinsic::lifetime_start:
-        if (STI.isOpenCLEnv()) {
+        if (!STI.isShader()) {
           Changed |= toSpvOverloadedIntrinsic(
               II, Intrinsic::SPVIntrinsics::spv_lifetime_start, {1});
         }
         break;
       case Intrinsic::lifetime_end:
-        if (STI.isOpenCLEnv()) {
+        if (!STI.isShader()) {
           Changed |= toSpvOverloadedIntrinsic(
               II, Intrinsic::SPVIntrinsics::spv_lifetime_end, {1});
         }
@@ -438,10 +438,9 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
 
   IRBuilder<> B(F->getContext());
 
-  bool HasAggrArg =
-      std::any_of(F->arg_begin(), F->arg_end(), [](Argument &Arg) {
-        return Arg.getType()->isAggregateType();
-      });
+  bool HasAggrArg = llvm::any_of(F->args(), [](Argument &Arg) {
+    return Arg.getType()->isAggregateType();
+  });
   bool DoClone = IsRetAggr || HasAggrArg;
   if (!DoClone)
     return F;
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
index 9224deb040f4..cdf3c6224d4c 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
@@ -82,6 +82,14 @@ SPIRVSubtarget::SPIRVSubtarget(const Triple &TT, const std::string &CPU,
   }
   OpenCLVersion = VersionTuple(2, 2);
 
+  // Set the environment based on the target triple.
+  if (TargetTriple.getOS() == Triple::Vulkan)
+    Env = Shader;
+  else if (TargetTriple.getEnvironment() == Triple::OpenCL)
+    Env = Kernel;
+  else
+    Env = Unknown;
+
   // The order of initialization is important.
   initAvailableExtensions(Extensions);
   initAvailableExtInstSets();
@@ -111,10 +119,10 @@ bool SPIRVSubtarget::canUseExtInstSet(
 
 SPIRV::InstructionSet::InstructionSet
 SPIRVSubtarget::getPreferredInstructionSet() const {
-  if (isOpenCLEnv())
-    return SPIRV::InstructionSet::OpenCL_std;
-  else
+  if (isShader())
     return SPIRV::InstructionSet::GLSL_std_450;
+  else
+    return SPIRV::InstructionSet::OpenCL_std;
 }
 
 bool SPIRVSubtarget::isAtLeastSPIRVVer(VersionTuple VerToCompareTo) const {
@@ -122,7 +130,7 @@ bool SPIRVSubtarget::isAtLeastSPIRVVer(VersionTuple VerToCompareTo) const {
 }
 
 bool SPIRVSubtarget::isAtLeastOpenCLVer(VersionTuple VerToCompareTo) const {
-  if (!isOpenCLEnv())
+  if (isShader())
     return false;
   return isAtLeastVer(OpenCLVersion, VerToCompareTo);
 }
@@ -145,7 +153,7 @@ void SPIRVSubtarget::accountForAMDShaderTrinaryMinmax() {
 // Must have called initAvailableExtensions first.
 void SPIRVSubtarget::initAvailableExtInstSets() {
   AvailableExtInstSets.clear();
-  if (!isOpenCLEnv())
+  if (isShader())
     AvailableExtInstSets.insert(SPIRV::InstructionSet::GLSL_std_450);
   else
     AvailableExtInstSets.insert(SPIRV::InstructionSet::OpenCL_std);
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.h b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.h
index e4484f6508b6..ad3e38d296ed 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.h
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSubtarget.h
@@ -36,6 +36,10 @@ class StringRef;
 class SPIRVTargetMachine;
 
 class SPIRVSubtarget : public SPIRVGenSubtargetInfo {
+public:
+  // Enum for the SPIR-V environment: Kernel, Shader or Unknown.
+  enum SPIRVEnvType { Kernel, Shader, Unknown };
+
 private:
   const unsigned PointerSize;
   VersionTuple SPIRVVersion;
@@ -49,6 +53,7 @@ class SPIRVSubtarget : public SPIRVGenSubtargetInfo {
   SPIRVFrameLowering FrameLowering;
   SPIRVTargetLowering TLInfo;
   Triple TargetTriple;
+  SPIRVEnvType Env;
 
   // GlobalISel related APIs.
   std::unique_ptr<CallLowering> CallLoweringInfo;
@@ -78,14 +83,24 @@ class SPIRVSubtarget : public SPIRVGenSubtargetInfo {
   unsigned getPointerSize() const { return PointerSize; }
   unsigned getBound() const { return GR->getBound(); }
   bool canDirectlyComparePointers() const;
-  // TODO: this environment is not implemented in Triple, we need to decide
-  // how to standardize its support. For now, let's assume SPIR-V with physical
-  // addressing is OpenCL, and Logical addressing is Vulkan.
-  bool isOpenCLEnv() const {
+  void setEnv(SPIRVEnvType E) {
+    if (E == Unknown)
+      report_fatal_error("Unknown environment is not allowed.");
+    if (Env != Unknown)
+      report_fatal_error("Environment is already set.");
+
+    Env = E;
+  }
+  SPIRVEnvType getEnv() const { return Env; }
+  bool isKernel() const { return getEnv() == Kernel; }
+  bool isShader() const { return getEnv() == Shader; }
+  bool isLogicalSPIRV() const {
+    return TargetTriple.getArch() == Triple::spirv;
+  }
+  bool isPhysicalSPIRV() const {
     return TargetTriple.getArch() == Triple::spirv32 ||
            TargetTriple.getArch() == Triple::spirv64;
   }
-  bool isVulkanEnv() const { return TargetTriple.getArch() == Triple::spirv; }
   const std::string &getTargetTripleAsStr() const { return TargetTriple.str(); }
   VersionTuple getSPIRVVersion() const { return SPIRVVersion; };
   bool isAtLeastSPIRVVer(VersionTuple VerToCompareTo) const;
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index ca8a9a9997a8..f1aae42ea2be 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -172,6 +172,7 @@ def KernelProfilingInfoOperand : OperandCategory;
 def OpcodeOperand : OperandCategory;
 def CooperativeMatrixLayoutOperand : OperandCategory;
 def CooperativeMatrixOperandsOperand : OperandCategory;
+def SpecConstantOpOperandsOperand : OperandCategory;
 def MatrixMultiplyAccumulateOperandsOperand : OperandCategory;
 
 //===----------------------------------------------------------------------===//
@@ -1755,6 +1756,95 @@ defm MatrixAAndBBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x40,
 defm MatrixCBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x80, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
 defm MatrixResultBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x100, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
 
+//===----------------------------------------------------------------------===//
+// Multiclass used to define SpecConstant Operands enum values and at the
+// same time SymbolicOperand.
+//===----------------------------------------------------------------------===//
+
+def SpecConstantOpOperands : GenericEnum, Operand<i32> {
+  let FilterClass = "SpecConstantOpOperands";
+  let NameField = "Name";
+  let ValueField = "Value";
+  let PrintMethod = !strconcat("printSymbolicOperand<OperandCategory::", FilterClass, "Operand>");
+}
+
+class SpecConstantOpOperands<string name, bits<32> value> {
+  string Name = name;
+  bits<32> Value = value;
+}
+
+multiclass SpecConstantOpOperandsOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
+  def : SpecConstantOpOperands<NAME, value>;
+  defm : SymbolicOperandWithRequirements<SpecConstantOpOperandsOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+// Conversion
+defm SConvert :  SpecConstantOpOperandsOperand<114, [], []>;
+defm FConvert :  SpecConstantOpOperandsOperand<115, [], []>;
+defm ConvertFToS :  SpecConstantOpOperandsOperand<110, [], [Kernel]>;
+defm ConvertSToF :  SpecConstantOpOperandsOperand<111, [], [Kernel]>;
+defm ConvertFToU :  SpecConstantOpOperandsOperand<109, [], [Kernel]>;
+defm ConvertUToF :  SpecConstantOpOperandsOperand<112, [], [Kernel]>;
+defm UConvert :  SpecConstantOpOperandsOperand<113, [], [Kernel]>;
+defm ConvertPtrToU :  SpecConstantOpOperandsOperand<117, [], [Kernel]>;
+defm ConvertUToPtr :  SpecConstantOpOperandsOperand<120, [], [Kernel]>;
+defm GenericCastToPtr :  SpecConstantOpOperandsOperand<122, [], [Kernel]>;
+defm PtrCastToGeneric :  SpecConstantOpOperandsOperand<121, [], [Kernel]>;
+defm Bitcast :  SpecConstantOpOperandsOperand<124, [], []>;
+defm QuantizeToF16 :  SpecConstantOpOperandsOperand<116, [], [Shader]>;
+// Arithmetic 
+defm SNegate :  SpecConstantOpOperandsOperand<126, [], []>;
+defm Not :  SpecConstantOpOperandsOperand<200, [], []>;
+defm IAdd :  SpecConstantOpOperandsOperand<128, [], []>;
+defm ISub :  SpecConstantOpOperandsOperand<130, [], []>;
+defm IMul :  SpecConstantOpOperandsOperand<132, [], []>;
+defm UDiv :  SpecConstantOpOperandsOperand<134, [], []>;
+defm SDiv :  SpecConstantOpOperandsOperand<135, [], []>;
+defm UMod :  SpecConstantOpOperandsOperand<137, [], []>;
+defm SRem :  SpecConstantOpOperandsOperand<138, [], []>;
+defm SMod :  SpecConstantOpOperandsOperand<139, [], []>;
+defm ShiftRightLogical :  SpecConstantOpOperandsOperand<194, [], []>;
+defm ShiftRightArithmetic :  SpecConstantOpOperandsOperand<195, [], []>;
+defm ShiftLeftLogical :  SpecConstantOpOperandsOperand<196, [], []>;
+defm BitwiseOr :  SpecConstantOpOperandsOperand<197, [], []>;
+defm BitwiseAnd :  SpecConstantOpOperandsOperand<199, [], []>;
+defm BitwiseXor :  SpecConstantOpOperandsOperand<198, [], []>;
+defm FNegate :  SpecConstantOpOperandsOperand<127, [], [Kernel]>;
+defm FAdd :  SpecConstantOpOperandsOperand<129, [], [Kernel]>;
+defm FSub :  SpecConstantOpOperandsOperand<131, [], [Kernel]>;
+defm FMul :  SpecConstantOpOperandsOperand<133, [], [Kernel]>;
+defm FDiv :  SpecConstantOpOperandsOperand<136, [], [Kernel]>;
+defm FRem :  SpecConstantOpOperandsOperand<140, [], [Kernel]>;
+defm FMod :  SpecConstantOpOperandsOperand<141, [], [Kernel]>;
+// Composite;
+defm VectorShuffle :  SpecConstantOpOperandsOperand<79, [], []>;
+defm CompositeExtract :  SpecConstantOpOperandsOperand<81, [], []>;
+defm CompositeInsert :  SpecConstantOpOperandsOperand<82, [], []>;
+// Logical;
+defm LogicalOr :  SpecConstantOpOperandsOperand<166, [], []>;
+defm LogicalAnd :  SpecConstantOpOperandsOperand<167, [], []>;
+defm LogicalNot :  SpecConstantOpOperandsOperand<168, [], []>;
+defm LogicalEqual :  SpecConstantOpOperandsOperand<164, [], []>;
+defm LogicalNotEqual :  SpecConstantOpOperandsOperand<165, [], []>;
+defm Select :  SpecConstantOpOperandsOperand<169, [], []>;
+// Comparison;
+defm IEqual :  SpecConstantOpOperandsOperand<170, [], []>;
+defm INotEqual :  SpecConstantOpOperandsOperand<171, [], []>;
+defm ULessThan :  SpecConstantOpOperandsOperand<176, [], []>;
+defm SLessThan :  SpecConstantOpOperandsOperand<177, [], []>;
+defm UGreaterThan :  SpecConstantOpOperandsOperand<172, [], []>;
+defm SGreaterThan :  SpecConstantOpOperandsOperand<173, [], []>;
+defm ULessThanEqual :  SpecConstantOpOperandsOperand<178, [], []>;
+defm SLessThanEqual :  SpecConstantOpOperandsOperand<179, [], []>;
+defm UGreaterThanEqual :  SpecConstantOpOperandsOperand<174, [], []>;
+defm SGreaterThanEqual :  SpecConstantOpOperandsOperand<175, [], []>;
+// Memory
+defm AccessChain :  SpecConstantOpOperandsOperand<65, [], [Kernel]>;
+defm InBoundsAccessChain :  SpecConstantOpOperandsOperand<66, [], [Kernel]>;
+defm PtrAccessChain :  SpecConstantOpOperandsOperand<67, [], [Kernel]>;
+defm InBoundsPtrAccessChain :  SpecConstantOpOperandsOperand<70, [], [Kernel]>;
+defm CooperativeMatrixLengthKHR : SpecConstantOpOperandsOperand<4460, [], []>;
+
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Matrix Multiply Accumulate Operands enum values and at the same time
 // SymbolicOperand entries with string mnemonics and capabilities.
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index a858b901439c..82fe23a22b60 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -188,7 +188,7 @@ TargetPassConfig *SPIRVTargetMachine::createPassConfig(PassManagerBase &PM) {
 void SPIRVPassConfig::addIRPasses() {
   TargetPassConfig::addIRPasses();
 
-  if (TM.getSubtargetImpl()->isVulkanEnv()) {
+  if (TM.getSubtargetImpl()->isShader()) {
     // Vulkan does not allow address space casts. This pass is run to remove
     // address space casts that can be removed.
     // If an address space cast is not removed while targeting Vulkan, lowering
@@ -226,7 +226,7 @@ void SPIRVPassConfig::addIRPasses() {
 
 void SPIRVPassConfig::addISelPrepare() {
   addPass(createSPIRVEmitIntrinsicsPass(&getTM<SPIRVTargetMachine>()));
-  if (TM.getSubtargetImpl()->isVulkanEnv())
+  if (TM.getSubtargetImpl()->isLogicalSPIRV())
     addPass(createSPIRVLegalizePointerCastPass(&getTM<SPIRVTargetMachine>()));
   TargetPassConfig::addISelPrepare();
 }
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
index 911c9a19a8d1..40e561ba3888 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVTargetTransformInfo.h
@@ -22,7 +22,7 @@
 #include "llvm/CodeGen/BasicTTIImpl.h"
 
 namespace llvm {
-class SPIRVTTIImpl : public BasicTTIImplBase<SPIRVTTIImpl> {
+class SPIRVTTIImpl final : public BasicTTIImplBase<SPIRVTTIImpl> {
   using BaseT = BasicTTIImplBase<SPIRVTTIImpl>;
   using TTI = TargetTransformInfo;
 
@@ -50,7 +50,7 @@ class SPIRVTTIImpl : public BasicTTIImplBase<SPIRVTTIImpl> {
   }
 
   unsigned getFlatAddressSpace() const override {
-    if (ST->isVulkanEnv())
+    if (ST->isShader())
       return 0;
     // FIXME: Clang has 2 distinct address space maps. One where
     // default=4=Generic, and one with default=0=Function. This depends on the
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 725a7979d3e5..768efb96a53e 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -80,6 +80,16 @@ std::string getStringImm(const MachineInstr &MI, unsigned StartIndex) {
   return getSPIRVStringOperand(MI, StartIndex);
 }
 
+std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI) {
+  MachineInstr *Def = getVRegDef(MRI, Reg);
+  assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&
+         "Expected G_GLOBAL_VALUE");
+  const GlobalValue *GV = Def->getOperand(1).getGlobal();
+  Value *V = GV->getOperand(0);
+  const ConstantDataArray *CDA = cast<ConstantDataArray>(V);
+  return CDA->getAsCString().str();
+}
+
 void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB) {
   const auto Bitwidth = Imm.getBitWidth();
   if (Bitwidth == 1)
diff --git a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.h b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.h
index f14a7d356ea5..d732188f9289 100644
--- a/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/external/llvm-project/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -125,6 +125,10 @@ void addStringImm(const StringRef &Str, IRBuilder<> &B,
 // the reverse of the logic in addStringImm.
 std::string getStringImm(const MachineInstr &MI, unsigned StartIndex);
 
+// Returns the string constant that the register refers to. It is assumed that
+// Reg is a global value that contains a string.
+std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI);
+
 // Add the given numerical immediate to MIB.
 void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB);
 
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/external/llvm-project/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index 46db96b5f893..90aacacd8ed2 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -848,14 +848,14 @@ bool SparcAsmParser::expandSETX(MCInst &Inst, SMLoc IDLoc,
   // sethi %hh(val), tmp
   Instructions.push_back(MCInstBuilder(SP::SETHIi)
                              .addReg(MCTmpOp.getReg())
-                             .addExpr(SparcMCExpr::create(
-                                 ELF::R_SPARC_HH22, ValExpr, getContext())));
+                             .addExpr(MCSpecifierExpr::create(
+                                 ValExpr, ELF::R_SPARC_HH22, getContext())));
   // or    tmp, %hm(val), tmp
   Instructions.push_back(MCInstBuilder(SP::ORri)
                              .addReg(MCTmpOp.getReg())
                              .addReg(MCTmpOp.getReg())
-                             .addExpr(SparcMCExpr::create(
-                                 ELF::R_SPARC_HM10, ValExpr, getContext())));
+                             .addExpr(MCSpecifierExpr::create(
+                                 ValExpr, ELF::R_SPARC_HM10, getContext())));
   // sllx  tmp, 32, tmp
   Instructions.push_back(MCInstBuilder(SP::SLLXri)
                              .addReg(MCTmpOp.getReg())
@@ -1165,7 +1165,7 @@ ParseStatus SparcAsmParser::parseTailRelocSym(OperandVector &Operands) {
     return Error(getLoc(), "expected valid identifier for operand modifier");
 
   StringRef Name = getParser().getTok().getIdentifier();
-  uint16_t RelType = SparcMCExpr::parseSpecifier(Name);
+  uint16_t RelType = Sparc::parseSpecifier(Name);
   if (RelType == 0)
     return Error(getLoc(), "invalid relocation specifier");
 
@@ -1661,6 +1661,9 @@ static bool hasGOTReference(const MCExpr *Expr) {
 
   case MCExpr::Unary:
     return hasGOTReference(cast<MCUnaryExpr>(Expr)->getSubExpr());
+
+  case MCExpr::Specifier:
+    llvm_unreachable("unused by this backend");
   }
   return false;
 }
@@ -1686,7 +1689,7 @@ const SparcMCExpr *SparcAsmParser::adjustPICRelocation(uint16_t RelType,
     }
   }
 
-  return SparcMCExpr::create(RelType, subExpr, getContext());
+  return MCSpecifierExpr::create(subExpr, RelType, getContext());
 }
 
 bool SparcAsmParser::matchSparcAsmModifiers(const MCExpr *&EVal,
@@ -1697,7 +1700,7 @@ bool SparcAsmParser::matchSparcAsmModifiers(const MCExpr *&EVal,
 
   StringRef name = Tok.getString();
 
-  auto VK = SparcMCExpr::parseSpecifier(name);
+  auto VK = Sparc::parseSpecifier(name);
   switch (VK) {
   case 0:
     Error(getLoc(), "invalid relocation specifier");
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index 0e29d1702150..cc1b7e93b31c 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -112,167 +112,164 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
 }
 
 namespace {
-  class SparcAsmBackend : public MCAsmBackend {
-  protected:
-    bool Is64Bit;
-    bool IsV8Plus;
-
-  public:
-    SparcAsmBackend(const MCSubtargetInfo &STI)
-        : MCAsmBackend(STI.getTargetTriple().isLittleEndian()
-                           ? llvm::endianness::little
-                           : llvm::endianness::big),
-          Is64Bit(STI.getTargetTriple().isArch64Bit()),
-          IsV8Plus(STI.hasFeature(Sparc::FeatureV8Plus)) {}
-
-
-    std::optional<MCFixupKind> getFixupKind(StringRef Name) const override {
-      unsigned Type;
-      Type = llvm::StringSwitch<unsigned>(Name)
+class SparcAsmBackend : public MCAsmBackend {
+protected:
+  bool Is64Bit;
+  bool IsV8Plus;
+
+public:
+  SparcAsmBackend(const MCSubtargetInfo &STI)
+      : MCAsmBackend(STI.getTargetTriple().isLittleEndian()
+                         ? llvm::endianness::little
+                         : llvm::endianness::big),
+        Is64Bit(STI.getTargetTriple().isArch64Bit()),
+        IsV8Plus(STI.hasFeature(Sparc::FeatureV8Plus)) {}
+
+  std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
+  void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
+                  MutableArrayRef<char> Data, uint64_t Value,
+                  bool IsResolved) override;
+
+  bool writeNopData(raw_ostream &OS, uint64_t Count,
+                    const MCSubtargetInfo *STI) const override {
+
+    // If the count is not 4-byte aligned, we must be writing data into the
+    // text section (otherwise we have unaligned instructions, and thus have
+    // far bigger problems), so just write zeros instead.
+    OS.write_zeros(Count % 4);
+
+    uint64_t NumNops = Count / 4;
+    for (uint64_t i = 0; i != NumNops; ++i)
+      support::endian::write<uint32_t>(OS, 0x01000000, Endian);
+
+    return true;
+  }
+};
+
+class ELFSparcAsmBackend : public SparcAsmBackend {
+  Triple::OSType OSType;
+
+public:
+  ELFSparcAsmBackend(const MCSubtargetInfo &STI, Triple::OSType OSType)
+      : SparcAsmBackend(STI), OSType(OSType) {}
+
+  std::unique_ptr<MCObjectTargetWriter>
+  createObjectTargetWriter() const override {
+    uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(OSType);
+    return createSparcELFObjectWriter(Is64Bit, IsV8Plus, OSABI);
+  }
+};
+} // end anonymous namespace
+
+std::optional<MCFixupKind> SparcAsmBackend::getFixupKind(StringRef Name) const {
+  unsigned Type;
+  Type = llvm::StringSwitch<unsigned>(Name)
 #define ELF_RELOC(X, Y) .Case(#X, Y)
 #include "llvm/BinaryFormat/ELFRelocs/Sparc.def"
 #undef ELF_RELOC
-                 .Case("BFD_RELOC_NONE", ELF::R_SPARC_NONE)
-                 .Case("BFD_RELOC_8", ELF::R_SPARC_8)
-                 .Case("BFD_RELOC_16", ELF::R_SPARC_16)
-                 .Case("BFD_RELOC_32", ELF::R_SPARC_32)
-                 .Case("BFD_RELOC_64", ELF::R_SPARC_64)
-                 .Default(-1u);
-      if (Type == -1u)
-        return std::nullopt;
-      return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
-    }
-
-    MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
-      // clang-format off
-      const static MCFixupKindInfo InfosBE[Sparc::NumTargetFixupKinds] = {
-        // name                    offset bits  flags
-        { "fixup_sparc_call30",     2,     30,  MCFixupKindInfo::FKF_IsPCRel },
-        { "fixup_sparc_13",        19,     13,  0 },
-      };
-
-      const static MCFixupKindInfo InfosLE[Sparc::NumTargetFixupKinds] = {
-        // name                    offset bits  flags
-        { "fixup_sparc_call30",     0,     30,  MCFixupKindInfo::FKF_IsPCRel },
-        { "fixup_sparc_13",         0,     13,  0 },
-      };
-      // clang-format on
-
-      if (!mc::isRelocation(Kind)) {
-        if (Kind < FirstTargetFixupKind)
-          return MCAsmBackend::getFixupKindInfo(Kind);
-        assert(unsigned(Kind - FirstTargetFixupKind) <
-                   Sparc::NumTargetFixupKinds &&
-               "Invalid kind!");
-        if (Endian == llvm::endianness::little)
-          return InfosLE[Kind - FirstTargetFixupKind];
-
-        return InfosBE[Kind - FirstTargetFixupKind];
-      }
-
-      MCFixupKindInfo Info{};
-      switch (uint16_t(Kind)) {
-      case ELF::R_SPARC_PC10:
-        Info = {"", 22, 10, MCFixupKindInfo::FKF_IsPCRel};
-        break;
-      case ELF::R_SPARC_PC22:
-        Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
-        break;
-      case ELF::R_SPARC_WDISP10:
-        Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
-        break;
-      case ELF::R_SPARC_WDISP16:
-        Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
-        break;
-      case ELF::R_SPARC_WDISP19:
-        Info = {"", 13, 19, MCFixupKindInfo::FKF_IsPCRel};
-        break;
-      case ELF::R_SPARC_WDISP22:
-        Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
-        break;
-
-      case ELF::R_SPARC_HI22:
-        Info = {"", 10, 22, 0};
-        break;
-      case ELF::R_SPARC_LO10:
-        Info = {"", 22, 10, 0};
-        break;
-      case ELF::R_SPARC_HH22:
-        Info = {"", 10, 22, 0};
-        break;
-      case ELF::R_SPARC_HM10:
-        Info = {"", 22, 10, 0};
-        break;
-      case ELF::R_SPARC_LM22:
-        Info = {"", 10, 22, 0};
-        break;
-      case ELF::R_SPARC_HIX22:
-        Info = {"", 10, 22, 0};
-        break;
-      case ELF::R_SPARC_LOX10:
-        Info = {"", 19, 13, 0};
-        break;
-      }
-      if (Endian == llvm::endianness::little)
-        Info.TargetOffset = 32 - Info.TargetOffset - Info.TargetSize;
-      return Info;
-    }
-
-    void relaxInstruction(MCInst &Inst,
-                          const MCSubtargetInfo &STI) const override {
-      // FIXME.
-      llvm_unreachable("relaxInstruction() unimplemented");
-    }
-
-    bool writeNopData(raw_ostream &OS, uint64_t Count,
-                      const MCSubtargetInfo *STI) const override {
-
-      // If the count is not 4-byte aligned, we must be writing data into the
-      // text section (otherwise we have unaligned instructions, and thus have
-      // far bigger problems), so just write zeros instead.
-      OS.write_zeros(Count % 4);
-
-      uint64_t NumNops = Count / 4;
-      for (uint64_t i = 0; i != NumNops; ++i)
-        support::endian::write<uint32_t>(OS, 0x01000000, Endian);
-
-      return true;
-    }
+             .Case("BFD_RELOC_NONE", ELF::R_SPARC_NONE)
+             .Case("BFD_RELOC_8", ELF::R_SPARC_8)
+             .Case("BFD_RELOC_16", ELF::R_SPARC_16)
+             .Case("BFD_RELOC_32", ELF::R_SPARC_32)
+             .Case("BFD_RELOC_64", ELF::R_SPARC_64)
+             .Default(-1u);
+  if (Type == -1u)
+    return std::nullopt;
+  return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
+}
+
+MCFixupKindInfo SparcAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+  // clang-format off
+  const static MCFixupKindInfo InfosBE[Sparc::NumTargetFixupKinds] = {
+    // name                    offset bits  flags
+    { "fixup_sparc_call30",     2,     30,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_sparc_13",        19,     13,  0 },
   };
 
-  class ELFSparcAsmBackend : public SparcAsmBackend {
-    Triple::OSType OSType;
-  public:
-    ELFSparcAsmBackend(const MCSubtargetInfo &STI, Triple::OSType OSType)
-        : SparcAsmBackend(STI), OSType(OSType) {}
-
-    void applyFixup(const MCFragment &, const MCFixup &Fixup,
-                    const MCValue &Target, MutableArrayRef<char> Data,
-                    uint64_t Value, bool IsResolved) override {
-      if (!IsResolved)
-        return;
-      Value = adjustFixupValue(Fixup.getKind(), Value);
-
-      unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
-      unsigned Offset = Fixup.getOffset();
-      // For each byte of the fragment that the fixup touches, mask in the bits
-      // from the fixup value. The Value has been "split up" into the
-      // appropriate bitfields above.
-      for (unsigned i = 0; i != NumBytes; ++i) {
-        unsigned Idx =
-            Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
-        Data[Offset + Idx] |= uint8_t((Value >> (i * 8)) & 0xff);
-      }
-    }
-
-    std::unique_ptr<MCObjectTargetWriter>
-    createObjectTargetWriter() const override {
-      uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(OSType);
-      return createSparcELFObjectWriter(Is64Bit, IsV8Plus, OSABI);
-    }
+  const static MCFixupKindInfo InfosLE[Sparc::NumTargetFixupKinds] = {
+    // name                    offset bits  flags
+    { "fixup_sparc_call30",     0,     30,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_sparc_13",         0,     13,  0 },
   };
+  // clang-format on
 
-} // end anonymous namespace
+  if (!mc::isRelocation(Kind)) {
+    if (Kind < FirstTargetFixupKind)
+      return MCAsmBackend::getFixupKindInfo(Kind);
+    assert(unsigned(Kind - FirstTargetFixupKind) < Sparc::NumTargetFixupKinds &&
+           "Invalid kind!");
+    if (Endian == llvm::endianness::little)
+      return InfosLE[Kind - FirstTargetFixupKind];
+
+    return InfosBE[Kind - FirstTargetFixupKind];
+  }
+
+  MCFixupKindInfo Info{};
+  switch (uint16_t(Kind)) {
+  case ELF::R_SPARC_PC10:
+    Info = {"", 22, 10, MCFixupKindInfo::FKF_IsPCRel};
+    break;
+  case ELF::R_SPARC_PC22:
+    Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
+    break;
+  case ELF::R_SPARC_WDISP10:
+    Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
+    break;
+  case ELF::R_SPARC_WDISP16:
+    Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
+    break;
+  case ELF::R_SPARC_WDISP19:
+    Info = {"", 13, 19, MCFixupKindInfo::FKF_IsPCRel};
+    break;
+  case ELF::R_SPARC_WDISP22:
+    Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
+    break;
+
+  case ELF::R_SPARC_HI22:
+    Info = {"", 10, 22, 0};
+    break;
+  case ELF::R_SPARC_LO10:
+    Info = {"", 22, 10, 0};
+    break;
+  case ELF::R_SPARC_HH22:
+    Info = {"", 10, 22, 0};
+    break;
+  case ELF::R_SPARC_HM10:
+    Info = {"", 22, 10, 0};
+    break;
+  case ELF::R_SPARC_LM22:
+    Info = {"", 10, 22, 0};
+    break;
+  case ELF::R_SPARC_HIX22:
+    Info = {"", 10, 22, 0};
+    break;
+  case ELF::R_SPARC_LOX10:
+    Info = {"", 19, 13, 0};
+    break;
+  }
+  if (Endian == llvm::endianness::little)
+    Info.TargetOffset = 32 - Info.TargetOffset - Info.TargetSize;
+  return Info;
+}
+
+void SparcAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+                                 const MCValue &Target,
+                                 MutableArrayRef<char> Data, uint64_t Value,
+                                 bool IsResolved) {
+  if (!IsResolved)
+    return;
+  Value = adjustFixupValue(Fixup.getKind(), Value);
+
+  unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
+  unsigned Offset = Fixup.getOffset();
+  // For each byte of the fragment that the fixup touches, mask in the
+  // bits from the fixup value.
+  for (unsigned i = 0; i != NumBytes; ++i) {
+    unsigned Idx = Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
+    Data[Offset + Idx] |= uint8_t((Value >> (i * 8)) & 0xff);
+  }
+}
 
 MCAsmBackend *llvm::createSparcAsmBackend(const Target &T,
                                           const MCSubtargetInfo &STI,
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
index 936518da3511..2d1a4fe94d18 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
@@ -12,6 +12,7 @@
 
 #include "SparcInstPrinter.h"
 #include "Sparc.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -142,7 +143,7 @@ void SparcInstPrinter::printOperand(const MCInst *MI, int opNum,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
 }
 
 void SparcInstPrinter::printMemOperand(const MCInst *MI, int opNum,
@@ -288,5 +289,5 @@ void SparcInstPrinter::printCTILabel(const MCInst *MI, uint64_t Address,
   }
 
   // Otherwise, just print the expression.
-  Op.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *Op.getExpr());
 }
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
index 8de9a789a63b..800567bf58ff 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
@@ -50,8 +50,7 @@ SparcELFMCAsmInfo::getExprForPersonalitySymbol(const MCSymbol *Sym,
                                                MCStreamer &Streamer) const {
   if (Encoding & dwarf::DW_EH_PE_pcrel) {
     MCContext &Ctx = Streamer.getContext();
-    return SparcMCExpr::create(ELF::R_SPARC_DISP32,
-                               MCSymbolRefExpr::create(Sym, Ctx), Ctx);
+    return MCSpecifierExpr::create(Sym, ELF::R_SPARC_DISP32, Ctx);
   }
 
   return MCAsmInfo::getExprForPersonalitySymbol(Sym, Encoding, Streamer);
@@ -63,8 +62,17 @@ SparcELFMCAsmInfo::getExprForFDESymbol(const MCSymbol *Sym,
                                        MCStreamer &Streamer) const {
   if (Encoding & dwarf::DW_EH_PE_pcrel) {
     MCContext &Ctx = Streamer.getContext();
-    return SparcMCExpr::create(ELF::R_SPARC_DISP32,
-                               MCSymbolRefExpr::create(Sym, Ctx), Ctx);
+    return MCSpecifierExpr::create(Sym, ELF::R_SPARC_DISP32, Ctx);
   }
   return MCAsmInfo::getExprForFDESymbol(Sym, Encoding, Streamer);
 }
+
+void SparcELFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                           const MCSpecifierExpr &Expr) const {
+  StringRef S = Sparc::getSpecifierName(Expr.getSpecifier());
+  if (!S.empty())
+    OS << '%' << S << '(';
+  printExpr(OS, *Expr.getSubExpr());
+  if (!S.empty())
+    OS << ')';
+}
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
index c9162f2dc8a5..7ea800f11917 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
@@ -32,6 +32,8 @@ class SparcELFMCAsmInfo : public MCAsmInfoELF {
                                     unsigned Encoding,
                                     MCStreamer &Streamer) const override;
 
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
 };
 
 } // end namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
index 2c8dbaa5aba6..4ce9bea5d795 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
@@ -135,7 +135,7 @@ getMachineOpValue(const MCInst &MI, const MCOperand &MO,
   assert(MO.isExpr());
   const MCExpr *Expr = MO.getExpr();
   if (const SparcMCExpr *SExpr = dyn_cast<SparcMCExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getFixupKind()));
+    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
     return 0;
   }
 
@@ -165,7 +165,7 @@ unsigned SparcMCCodeEmitter::getSImm5OpValue(const MCInst &MI, unsigned OpNo,
     return CE->getValue();
 
   if (const SparcMCExpr *SExpr = dyn_cast<SparcMCExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getFixupKind()));
+    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
     return 0;
   }
   Fixups.push_back(MCFixup::create(0, Expr, ELF::R_SPARC_5));
@@ -191,7 +191,7 @@ SparcMCCodeEmitter::getSImm13OpValue(const MCInst &MI, unsigned OpNo,
     return CE->getValue();
 
   if (const SparcMCExpr *SExpr = dyn_cast<SparcMCExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getFixupKind()));
+    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
     return 0;
   }
   Fixups.push_back(MCFixup::create(0, Expr, Sparc::fixup_sparc_13));
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
index 5dab9191d0cf..6d43b9371390 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
@@ -22,21 +22,7 @@ using namespace llvm;
 
 #define DEBUG_TYPE "sparcmcexpr"
 
-const SparcMCExpr *SparcMCExpr::create(uint16_t S, const MCExpr *Expr,
-                                       MCContext &Ctx) {
-  return new (Ctx) SparcMCExpr(S, Expr);
-}
-
-void SparcMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  StringRef S = getSpecifierName(specifier);
-  if (!S.empty())
-    OS << '%' << S << '(';
-  getSubExpr()->print(OS, MAI);
-  if (!S.empty())
-    OS << ')';
-}
-
-StringRef SparcMCExpr::getSpecifierName(uint16_t S) {
+StringRef Sparc::getSpecifierName(uint16_t S) {
   // clang-format off
   switch (uint16_t(S)) {
   case 0:                          return {};
@@ -83,7 +69,7 @@ StringRef SparcMCExpr::getSpecifierName(uint16_t S) {
   llvm_unreachable("Unhandled SparcMCExpr::Specifier");
 }
 
-uint16_t SparcMCExpr::parseSpecifier(StringRef name) {
+uint16_t Sparc::parseSpecifier(StringRef name) {
   return StringSwitch<uint16_t>(name)
       .Case("lo", ELF::R_SPARC_LO10)
       .Case("hi", ELF::R_SPARC_HI22)
@@ -128,21 +114,3 @@ uint16_t SparcMCExpr::parseSpecifier(StringRef name) {
       .Case("gdop", ELF::R_SPARC_GOTDATA_OP)
       .Default(0);
 }
-
-uint16_t SparcMCExpr::getFixupKind() const {
-  assert(uint16_t(specifier) < FirstTargetFixupKind);
-  return specifier;
-}
-
-bool SparcMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                            const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-
-  Res.setSpecifier(specifier);
-  return true;
-}
-
-void SparcMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
index 095b115ea5a0..8e7c173c70cc 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
@@ -20,36 +20,12 @@
 namespace llvm {
 
 class StringRef;
-class SparcMCExpr : public MCTargetExpr {
-private:
-  const uint16_t specifier;
-  const MCExpr *Expr;
+using SparcMCExpr = MCSpecifierExpr;
 
-  explicit SparcMCExpr(uint16_t S, const MCExpr *Expr)
-      : specifier(S), Expr(Expr) {}
-
-public:
-  static const SparcMCExpr *create(uint16_t S, const MCExpr *Expr,
-                                   MCContext &Ctx);
-  uint16_t getSpecifier() const { return specifier; }
-  const MCExpr *getSubExpr() const { return Expr; }
-  uint16_t getFixupKind() const;
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-
-  static uint16_t parseSpecifier(StringRef name);
-  static StringRef getSpecifierName(uint16_t S);
-};
+namespace Sparc {
+uint16_t parseSpecifier(StringRef name);
+StringRef getSpecifierName(uint16_t S);
+} // namespace Sparc
 
 } // end namespace llvm.
 
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
index a30cf5a661bb..f4201f9a8dc1 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
@@ -82,7 +82,7 @@ class SparcAsmPrinter : public AsmPrinter {
 static MCOperand createSparcMCOperand(uint16_t Kind, MCSymbol *Sym,
                                       MCContext &OutContext) {
   const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Sym, OutContext);
-  const SparcMCExpr *expr = SparcMCExpr::create(Kind, MCSym, OutContext);
+  const SparcMCExpr *expr = MCSpecifierExpr::create(MCSym, Kind, OutContext);
   return MCOperand::createExpr(expr);
 }
 static MCOperand createPCXCallOP(MCSymbol *Label,
@@ -101,7 +101,7 @@ static MCOperand createPCXRelExprOp(uint16_t Spec, MCSymbol *GOTLabel,
 
   const MCBinaryExpr *Sub = MCBinaryExpr::createSub(Cur, Start, OutContext);
   const MCBinaryExpr *Add = MCBinaryExpr::createAdd(GOT, Sub, OutContext);
-  const SparcMCExpr *expr = SparcMCExpr::create(Spec, Add, OutContext);
+  const SparcMCExpr *expr = MCSpecifierExpr::create(Add, Spec, OutContext);
   return MCOperand::createExpr(expr);
 }
 
@@ -302,7 +302,7 @@ MCOperand SparcAsmPrinter::lowerOperand(const MachineOperand &MO) const {
 
     const MCExpr *expr = MCSymbolRefExpr::create(Symbol, OutContext);
     if (RelType)
-      expr = SparcMCExpr::create(RelType, expr, OutContext);
+      expr = MCSpecifierExpr::create(expr, RelType, OutContext);
     return MCOperand::createExpr(expr);
   }
 
@@ -371,11 +371,7 @@ void SparcAsmPrinter::emitFunctionBodyStart() {
 void SparcAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
                                    raw_ostream &O) {
   const DataLayout &DL = getDataLayout();
-  const MachineOperand &MO = MI->getOperand (opNum);
-  auto TF = MO.getTargetFlags();
-
-  StringRef Spec = SparcMCExpr::getSpecifierName(TF);
-  O << Spec;
+  const MachineOperand &MO = MI->getOperand(opNum);
   switch (MO.getType()) {
   case MachineOperand::MO_Register:
     O << "%" << StringRef(getRegisterName(MO.getReg())).lower();
@@ -406,8 +402,6 @@ void SparcAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
   default:
     llvm_unreachable("<unknown operand type>");
   }
-  if (!Spec.empty())
-    O << ")";
 }
 
 void SparcAsmPrinter::printMemOperand(const MachineInstr *MI, int opNum,
diff --git a/external/llvm-project/llvm/lib/Target/Sparc/SparcTargetObjectFile.cpp b/external/llvm-project/llvm/lib/Target/Sparc/SparcTargetObjectFile.cpp
index 668e6eab4e1b..a42a67d91d84 100644
--- a/external/llvm-project/llvm/lib/Target/Sparc/SparcTargetObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Target/Sparc/SparcTargetObjectFile.cpp
@@ -39,8 +39,7 @@ const MCExpr *SparcELFTargetObjectFile::getTTypeGlobalReference(
     }
 
     MCContext &Ctx = getContext();
-    return SparcMCExpr::create(ELF::R_SPARC_DISP32,
-                               MCSymbolRefExpr::create(SSym, Ctx), Ctx);
+    return MCSpecifierExpr::create(SSym, ELF::R_SPARC_DISP32, Ctx);
   }
 
   return TargetLoweringObjectFileELF::getTTypeGlobalReference(GV, Encoding, TM,
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 83c74c8a976d..6ee2a87565ba 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -8,7 +8,6 @@
 
 #include "MCTargetDesc/SystemZGNUInstPrinter.h"
 #include "MCTargetDesc/SystemZMCAsmInfo.h"
-#include "MCTargetDesc/SystemZMCExpr.h"
 #include "MCTargetDesc/SystemZMCTargetDesc.h"
 #include "MCTargetDesc/SystemZTargetStreamer.h"
 #include "TargetInfo/SystemZTargetInfo.h"
@@ -1707,12 +1706,12 @@ ParseStatus SystemZAsmParser::parsePCRel(OperandVector &Operands,
     if (Parser.getTok().isNot(AsmToken::Identifier))
       return Error(Parser.getTok().getLoc(), "unexpected token");
 
-    SystemZMCExpr::Specifier Kind = SystemZMCExpr::VK_None;
+    auto Kind = SystemZ::S_None;
     StringRef Name = Parser.getTok().getString();
     if (Name == "tls_gdcall")
-      Kind = SystemZMCExpr::VK_TLSGD;
+      Kind = SystemZ::S_TLSGD;
     else if (Name == "tls_ldcall")
-      Kind = SystemZMCExpr::VK_TLSLDM;
+      Kind = SystemZ::S_TLSLDM;
     else
       return Error(Parser.getTok().getLoc(), "unknown TLS tag");
     Parser.Lex();
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
index c95445637d0b..28f7ced8d7ce 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
@@ -8,7 +8,6 @@ add_llvm_component_library(LLVMSystemZDesc
   SystemZMCAsmBackend.cpp
   SystemZMCAsmInfo.cpp
   SystemZMCCodeEmitter.cpp
-  SystemZMCExpr.cpp
   SystemZMCTargetDesc.cpp
   SystemZTargetStreamer.cpp
 
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZELFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZELFObjectWriter.cpp
index b44859d75df0..8b5587ab7125 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZELFObjectWriter.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/SystemZMCExpr.h"
+#include "MCTargetDesc/SystemZMCAsmInfo.h"
 #include "MCTargetDesc/SystemZMCFixups.h"
 #include "MCTargetDesc/SystemZMCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -103,14 +103,14 @@ unsigned SystemZELFObjectWriter::getRelocType(const MCFixup &Fixup,
                                               bool IsPCRel) const {
   SMLoc Loc = Fixup.getLoc();
   unsigned Kind = Fixup.getKind();
-  auto Specifier = SystemZMCExpr::Specifier(Target.getSpecifier());
+  auto Specifier = SystemZ::Specifier(Target.getSpecifier());
   switch (Specifier) {
-  case SystemZMCExpr::VK_INDNTPOFF:
-  case SystemZMCExpr::VK_NTPOFF:
-  case SystemZMCExpr::VK_TLSGD:
-  case SystemZMCExpr::VK_TLSLD:
-  case SystemZMCExpr::VK_TLSLDM:
-  case SystemZMCExpr::VK_DTPOFF:
+  case SystemZ::S_INDNTPOFF:
+  case SystemZ::S_NTPOFF:
+  case SystemZ::S_TLSGD:
+  case SystemZ::S_TLSLD:
+  case SystemZ::S_TLSLDM:
+  case SystemZ::S_DTPOFF:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
@@ -119,12 +119,12 @@ unsigned SystemZELFObjectWriter::getRelocType(const MCFixup &Fixup,
   }
 
   switch (Specifier) {
-  case SystemZMCExpr::VK_None:
+  case SystemZ::S_None:
     if (IsPCRel)
       return getPCRelReloc(Loc, Kind);
     return getAbsoluteReloc(Loc, Kind);
 
-  case SystemZMCExpr::VK_NTPOFF:
+  case SystemZ::S_NTPOFF:
     assert(!IsPCRel && "NTPOFF shouldn't be PC-relative");
     switch (Kind) {
     case FK_Data_4:
@@ -135,14 +135,14 @@ unsigned SystemZELFObjectWriter::getRelocType(const MCFixup &Fixup,
     reportError(Loc, "Unsupported thread-local address (local-exec)");
     return 0;
 
-  case SystemZMCExpr::VK_INDNTPOFF:
+  case SystemZ::S_INDNTPOFF:
     if (IsPCRel && Kind == SystemZ::FK_390_PC32DBL)
       return ELF::R_390_TLS_IEENT;
     reportError(Loc,
                 "Only PC-relative INDNTPOFF accesses are supported for now");
     return 0;
 
-  case SystemZMCExpr::VK_DTPOFF:
+  case SystemZ::S_DTPOFF:
     assert(!IsPCRel && "DTPOFF shouldn't be PC-relative");
     switch (Kind) {
     case FK_Data_4:
@@ -153,7 +153,7 @@ unsigned SystemZELFObjectWriter::getRelocType(const MCFixup &Fixup,
     reportError(Loc, "Unsupported thread-local address (local-dynamic)");
     return 0;
 
-  case SystemZMCExpr::VK_TLSLDM:
+  case SystemZ::S_TLSLDM:
     assert(!IsPCRel && "TLSLDM shouldn't be PC-relative");
     switch (Kind) {
     case FK_Data_4:
@@ -166,7 +166,7 @@ unsigned SystemZELFObjectWriter::getRelocType(const MCFixup &Fixup,
     reportError(Loc, "Unsupported thread-local address (local-dynamic)");
     return 0;
 
-  case SystemZMCExpr::VK_TLSGD:
+  case SystemZ::S_TLSGD:
     assert(!IsPCRel && "TLSGD shouldn't be PC-relative");
     switch (Kind) {
     case FK_Data_4:
@@ -179,14 +179,14 @@ unsigned SystemZELFObjectWriter::getRelocType(const MCFixup &Fixup,
     reportError(Loc, "Unsupported thread-local address (general-dynamic)");
     return 0;
 
-  case SystemZMCExpr::VK_GOT:
-  case SystemZMCExpr::VK_GOTENT:
+  case SystemZ::S_GOT:
+  case SystemZ::S_GOTENT:
     if (IsPCRel && Kind == SystemZ::FK_390_PC32DBL)
       return ELF::R_390_GOTENT;
     reportError(Loc, "Only PC-relative GOT accesses are supported for now");
     return 0;
 
-  case SystemZMCExpr::VK_PLT:
+  case SystemZ::S_PLT:
     assert(IsPCRel && "@PLT shouldn't be PC-relative");
     switch (Kind) {
     case SystemZ::FK_390_PC12DBL:
@@ -209,8 +209,8 @@ unsigned SystemZELFObjectWriter::getRelocType(const MCFixup &Fixup,
 bool SystemZELFObjectWriter::needsRelocateWithSymbol(const MCValue &V,
                                                      unsigned Type) const {
   switch (V.getSpecifier()) {
-  case SystemZMCExpr::VK_GOT:
-  case SystemZMCExpr::VK_PLT:
+  case SystemZ::S_GOT:
+  case SystemZ::S_PLT:
     return true;
   default:
     return false;
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
index 165feec7a7d4..ec8c81080930 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
@@ -209,7 +209,7 @@ void SystemZHLASMAsmStreamer::emitHLASMValueImpl(const MCExpr *Value,
   switch (Value->getKind()) {
   case MCExpr::Constant: {
     OS << "XL" << Size << '\'';
-    Value->print(OS, MAI);
+    MAI->printExpr(OS, *Value);
     OS << '\'';
     return;
   }
@@ -258,12 +258,12 @@ void SystemZHLASMAsmStreamer::emitHLASMValueImpl(const MCExpr *Value,
     return;
   }
   case MCExpr::Target:
-    Value->print(OS, MAI);
+    MAI->printExpr(OS, *Value);
     return;
   default:
     if (Parens)
       OS << "A(";
-    Value->print(OS, MAI);
+    MAI->printExpr(OS, *Value);
     if (Parens)
       OS << ')';
     return;
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
index b03ad98f2c38..297fdc832592 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZInstPrinterCommon.h"
-#include "MCTargetDesc/SystemZMCExpr.h"
+#include "MCTargetDesc/SystemZMCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCRegister.h"
@@ -53,7 +53,7 @@ void SystemZInstPrinterCommon::printOperand(const MCOperand &MO,
   } else if (MO.isImm())
     markup(O, Markup::Immediate) << MO.getImm();
   else if (MO.isExpr())
-    MO.getExpr()->print(O, MAI);
+    MAI->printExpr(O, *MO.getExpr());
   else
     llvm_unreachable("Invalid operand");
 }
@@ -171,7 +171,7 @@ void SystemZInstPrinterCommon::printPCRelOperand(const MCInst *MI,
     markup(O, Markup::Target) << formatHex((uint64_t)TargetAddress);
   } else {
     // Otherwise, just print the expression.
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
   }
 }
 
@@ -185,11 +185,11 @@ void SystemZInstPrinterCommon::printPCRelTLSOperand(const MCInst *MI,
   if ((unsigned)OpNum + 1 < MI->getNumOperands()) {
     const MCOperand &MO = MI->getOperand(OpNum + 1);
     const MCSymbolRefExpr &refExp = cast<MCSymbolRefExpr>(*MO.getExpr());
-    switch (getSpecifier(&refExp)) {
-    case SystemZMCExpr::VK_TLSGD:
+    switch (refExp.getSpecifier()) {
+    case SystemZ::S_TLSGD:
       O << ":tls_gdcall:";
       break;
-    case SystemZMCExpr::VK_TLSLDM:
+    case SystemZ::S_TLSLDM:
       O << ":tls_ldcall:";
       break;
     default:
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
index 27272cdbbd23..052875bf0d3f 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
@@ -7,22 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZMCAsmInfo.h"
-#include "MCTargetDesc/SystemZMCExpr.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCValue.h"
 
 using namespace llvm;
 
 const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
-    {SystemZMCExpr::VK_DTPOFF, "DTPOFF"},
-    {SystemZMCExpr::VK_GOT, "GOT"},
-    {SystemZMCExpr::VK_GOTENT, "GOTENT"},
-    {SystemZMCExpr::VK_INDNTPOFF, "INDNTPOFF"},
-    {SystemZMCExpr::VK_NTPOFF, "NTPOFF"},
-    {SystemZMCExpr::VK_PLT, "PLT"},
-    {SystemZMCExpr::VK_TLSGD, "TLSGD"},
-    {SystemZMCExpr::VK_TLSLD, "TLSLD"},
-    {SystemZMCExpr::VK_TLSLDM, "TLSLDM"},
+    {SystemZ::S_DTPOFF, "DTPOFF"}, {SystemZ::S_GOT, "GOT"},
+    {SystemZ::S_GOTENT, "GOTENT"}, {SystemZ::S_INDNTPOFF, "INDNTPOFF"},
+    {SystemZ::S_NTPOFF, "NTPOFF"}, {SystemZ::S_PLT, "PLT"},
+    {SystemZ::S_TLSGD, "TLSGD"},   {SystemZ::S_TLSLD, "TLSLD"},
+    {SystemZ::S_TLSLDM, "TLSLDM"},
 };
 
 SystemZMCAsmInfoELF::SystemZMCAsmInfoELF(const Triple &TT) {
@@ -49,6 +45,7 @@ SystemZMCAsmInfoGOFF::SystemZMCAsmInfoGOFF(const Triple &TT) {
   CalleeSaveStackSlotSize = 8;
   CodePointerSize = 8;
   CommentString = "*";
+  UsesSetToEquateSymbol = true;
   ExceptionsType = ExceptionHandling::ZOS;
   IsHLASM = true;
   IsLittleEndian = false;
@@ -61,3 +58,31 @@ SystemZMCAsmInfoGOFF::SystemZMCAsmInfoGOFF(const Triple &TT) {
 bool SystemZMCAsmInfoGOFF::isAcceptableChar(char C) const {
   return MCAsmInfo::isAcceptableChar(C) || C == '#';
 }
+
+void SystemZMCAsmInfoGOFF::printSpecifierExpr(
+    raw_ostream &OS, const MCSpecifierExpr &Expr) const {
+  switch (Expr.getSpecifier()) {
+  case SystemZ::S_None:
+    OS << "A";
+    break;
+  case SystemZ::S_RCon:
+    OS << "R";
+    break;
+  case SystemZ::S_VCon:
+    OS << "V";
+    break;
+  default:
+    llvm_unreachable("Invalid kind");
+  }
+  OS << '(';
+  printExpr(OS, *Expr.getSubExpr());
+  OS << ')';
+}
+
+bool SystemZMCAsmInfoGOFF::evaluateAsRelocatableImpl(
+    const MCSpecifierExpr &Expr, MCValue &Res, const MCAssembler *Asm) const {
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(Res, Asm))
+    return false;
+  Res.setSpecifier(Expr.getSpecifier());
+  return true;
+}
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
index 58b9a3dd652e..11c2833b8ada 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
@@ -26,8 +26,34 @@ class SystemZMCAsmInfoGOFF : public MCAsmInfoGOFF {
 public:
   explicit SystemZMCAsmInfoGOFF(const Triple &TT);
   bool isAcceptableChar(char C) const override;
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                                 const MCAssembler *Asm) const override;
 };
 
+namespace SystemZ {
+using Specifier = uint16_t;
+enum {
+  S_None,
+
+  S_DTPOFF,
+  S_GOT,
+  S_GOTENT,
+  S_INDNTPOFF,
+  S_NTPOFF,
+  S_PLT,
+  S_TLSGD,
+  S_TLSLD,
+  S_TLSLDM,
+
+  // HLASM docs for address constants:
+  // https://www.ibm.com/docs/en/hla-and-tf/1.6?topic=value-address-constants
+  S_RCon, // Address of ADA of symbol.
+  S_VCon, // Address of external function symbol.
+};
+} // namespace SystemZ
+
 } // end namespace llvm
 
 #endif
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCExpr.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCExpr.cpp
deleted file mode 100644
index 41e41a1a7b66..000000000000
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCExpr.cpp
+++ /dev/null
@@ -1,45 +0,0 @@
-//===-- SystemZMCExpr.cpp - SystemZ specific MC expression classes --------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "SystemZMCExpr.h"
-#include "llvm/MC/MCContext.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "systemzmcexpr"
-
-const SystemZMCExpr *SystemZMCExpr::create(SystemZMCExpr::Specifier Kind,
-                                           const MCExpr *Expr, MCContext &Ctx) {
-  return new (Ctx) SystemZMCExpr(Kind, Expr);
-}
-
-StringRef SystemZMCExpr::getVariantKindName() const {
-  switch (getSpecifier()) {
-  case VK_None:
-    return "A";
-  case VK_SystemZ_RCon:
-    return "R";
-  case VK_SystemZ_VCon:
-    return "V";
-  default:
-    llvm_unreachable("Invalid kind");
-  }
-}
-
-void SystemZMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  OS << getVariantKindName() << '(';
-  Expr->print(OS, MAI);
-  OS << ')';
-}
-
-bool SystemZMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                              const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-  return true;
-}
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCExpr.h b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCExpr.h
deleted file mode 100644
index 39b8ea14de0d..000000000000
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCExpr.h
+++ /dev/null
@@ -1,76 +0,0 @@
-//===-- SystemZMCExpr.h - SystemZ specific MC expression classes -*- C++-*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZMCEXPR_H
-#define LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCValue.h"
-
-namespace llvm {
-
-class SystemZMCExpr : public MCTargetExpr {
-public:
-  enum Specifier : uint8_t {
-    VK_None,
-
-    VK_DTPOFF = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_GOT,
-    VK_GOTENT,
-    VK_INDNTPOFF,
-    VK_NTPOFF,
-    VK_PLT,
-    VK_TLSGD,
-    VK_TLSLD,
-    VK_TLSLDM,
-
-    // HLASM docs for address constants:
-    // https://www.ibm.com/docs/en/hla-and-tf/1.6?topic=value-address-constants
-    VK_SystemZ_RCon, // Address of ADA of symbol.
-    VK_SystemZ_VCon, // Address of external function symbol.
-  };
-
-private:
-  const Specifier specifier;
-  const MCExpr *Expr;
-
-  explicit SystemZMCExpr(Specifier S, const MCExpr *Expr)
-      : specifier(S), Expr(Expr) {}
-
-public:
-  static const SystemZMCExpr *create(Specifier Kind, const MCExpr *Expr,
-                                     MCContext &Ctx);
-
-  Specifier getSpecifier() const { return specifier; }
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  StringRef getVariantKindName() const;
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override {
-    Streamer.visitUsedExpr(*getSubExpr());
-  }
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-};
-
-static inline SystemZMCExpr::Specifier
-getSpecifier(const MCSymbolRefExpr *SRE) {
-  return SystemZMCExpr::Specifier(SRE->getKind());
-}
-} // end namespace llvm
-
-#endif
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
index a4506eddaa69..772067809744 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZTargetStreamer.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 
 using namespace llvm;
@@ -44,7 +45,7 @@ const MCExpr *SystemZTargetHLASMStreamer::createWordDiffExpr(
   OS << Temp->getName() << " EQU ";
   const MCBinaryExpr *TempExpr = MCBinaryExpr::createSub(
       MCSymbolRefExpr::create(Hi, Ctx), MCSymbolRefExpr::create(Lo, Ctx), Ctx);
-  TempExpr->print(OS, Ctx.getAsmInfo());
+  Ctx.getAsmInfo()->printExpr(OS, *TempExpr);
   OS << "\n";
   return MCBinaryExpr::createLShr(MCSymbolRefExpr::create(Temp, Ctx),
                                   MCConstantExpr::create(1, Ctx), Ctx);
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index 57911ac1ec2f..aaf12b88de13 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -14,7 +14,7 @@
 #include "SystemZAsmPrinter.h"
 #include "MCTargetDesc/SystemZGNUInstPrinter.h"
 #include "MCTargetDesc/SystemZHLASMInstPrinter.h"
-#include "MCTargetDesc/SystemZMCExpr.h"
+#include "MCTargetDesc/SystemZMCAsmInfo.h"
 #include "MCTargetDesc/SystemZMCTargetDesc.h"
 #include "SystemZConstantPoolValue.h"
 #include "SystemZMCInstLower.h"
@@ -79,7 +79,7 @@ static MCInst lowerRIEfLow(const MachineInstr *MI, unsigned Opcode) {
 static const MCSymbolRefExpr *getTLSGetOffset(MCContext &Context) {
   StringRef Name = "__tls_get_offset";
   return MCSymbolRefExpr::create(Context.getOrCreateSymbol(Name),
-                                 SystemZMCExpr::VK_PLT, Context);
+                                 SystemZ::S_PLT, Context);
 }
 
 static const MCSymbolRefExpr *getGlobalOffsetTable(MCContext &Context) {
@@ -319,11 +319,10 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     break;
 
   case SystemZ::CallBRASL_XPLINK64:
-    EmitToStreamer(
-        *OutStreamer,
-        MCInstBuilder(SystemZ::BRASL)
-            .addReg(SystemZ::R7D)
-            .addExpr(Lower.getExpr(MI->getOperand(0), SystemZMCExpr::VK_PLT)));
+    EmitToStreamer(*OutStreamer, MCInstBuilder(SystemZ::BRASL)
+                                     .addReg(SystemZ::R7D)
+                                     .addExpr(Lower.getExpr(MI->getOperand(0),
+                                                            SystemZ::S_PLT)));
     emitCallInformation(CallType::BRASL7);
     return;
 
@@ -380,10 +379,9 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     return;
   }
   case SystemZ::CallBRASL:
-    LoweredMI =
-        MCInstBuilder(SystemZ::BRASL)
-            .addReg(SystemZ::R14D)
-            .addExpr(Lower.getExpr(MI->getOperand(0), SystemZMCExpr::VK_PLT));
+    LoweredMI = MCInstBuilder(SystemZ::BRASL)
+                    .addReg(SystemZ::R14D)
+                    .addExpr(Lower.getExpr(MI->getOperand(0), SystemZ::S_PLT));
     break;
 
   case SystemZ::CallBASR:
@@ -393,17 +391,15 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     break;
 
   case SystemZ::CallJG:
-    LoweredMI =
-        MCInstBuilder(SystemZ::JG)
-            .addExpr(Lower.getExpr(MI->getOperand(0), SystemZMCExpr::VK_PLT));
+    LoweredMI = MCInstBuilder(SystemZ::JG)
+                    .addExpr(Lower.getExpr(MI->getOperand(0), SystemZ::S_PLT));
     break;
 
   case SystemZ::CallBRCL:
-    LoweredMI =
-        MCInstBuilder(SystemZ::BRCL)
-            .addImm(MI->getOperand(0).getImm())
-            .addImm(MI->getOperand(1).getImm())
-            .addExpr(Lower.getExpr(MI->getOperand(2), SystemZMCExpr::VK_PLT));
+    LoweredMI = MCInstBuilder(SystemZ::BRCL)
+                    .addImm(MI->getOperand(0).getImm())
+                    .addImm(MI->getOperand(1).getImm())
+                    .addExpr(Lower.getExpr(MI->getOperand(2), SystemZ::S_PLT));
     break;
 
   case SystemZ::CallBR:
@@ -495,15 +491,15 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
         MCInstBuilder(SystemZ::BRASL)
             .addReg(SystemZ::R14D)
             .addExpr(getTLSGetOffset(MF->getContext()))
-            .addExpr(Lower.getExpr(MI->getOperand(0), SystemZMCExpr::VK_TLSGD));
+            .addExpr(Lower.getExpr(MI->getOperand(0), SystemZ::S_TLSGD));
     break;
 
   case SystemZ::TLS_LDCALL:
-    LoweredMI = MCInstBuilder(SystemZ::BRASL)
-                    .addReg(SystemZ::R14D)
-                    .addExpr(getTLSGetOffset(MF->getContext()))
-                    .addExpr(Lower.getExpr(MI->getOperand(0),
-                                           SystemZMCExpr::VK_TLSLDM));
+    LoweredMI =
+        MCInstBuilder(SystemZ::BRASL)
+            .addReg(SystemZ::R14D)
+            .addExpr(getTLSGetOffset(MF->getContext()))
+            .addExpr(Lower.getExpr(MI->getOperand(0), SystemZ::S_TLSLDM));
     break;
 
   case SystemZ::GOT:
@@ -798,7 +794,7 @@ void SystemZAsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
 
   MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");
   const MCSymbolRefExpr *Op =
-      MCSymbolRefExpr::create(fentry, SystemZMCExpr::VK_PLT, Ctx);
+      MCSymbolRefExpr::create(fentry, SystemZ::S_PLT, Ctx);
   OutStreamer->emitInstruction(
       MCInstBuilder(SystemZ::BRASL).addReg(SystemZ::R0D).addExpr(Op),
       getSubtargetInfo());
@@ -880,7 +876,7 @@ void SystemZAsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
       EncodedBytes += 2;
     }
   } else if (CalleeMO.isGlobal()) {
-    const MCExpr *Expr = Lower.getExpr(CalleeMO, SystemZMCExpr::VK_PLT);
+    const MCExpr *Expr = Lower.getExpr(CalleeMO, SystemZ::S_PLT);
     EmitToStreamer(*OutStreamer, MCInstBuilder(SystemZ::BRASL)
                                    .addReg(SystemZ::R14D)
                                    .addExpr(Expr));
@@ -923,11 +919,10 @@ void SystemZAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(
   EmitNop(OutContext, *OutStreamer, 2, getSubtargetInfo());
   EmitToStreamer(*OutStreamer,
                  MCInstBuilder(SystemZ::LLILF).addReg(SystemZ::R2D).addImm(0));
-  EmitToStreamer(*OutStreamer,
-                 MCInstBuilder(SystemZ::BRASL)
-                     .addReg(SystemZ::R14D)
-                     .addExpr(MCSymbolRefExpr::create(
-                         FuncEntry, SystemZMCExpr::VK_PLT, OutContext)));
+  EmitToStreamer(*OutStreamer, MCInstBuilder(SystemZ::BRASL)
+                                   .addReg(SystemZ::R14D)
+                                   .addExpr(MCSymbolRefExpr::create(
+                                       FuncEntry, SystemZ::S_PLT, OutContext)));
   OutStreamer->emitLabel(EndOfSled);
   recordSled(BeginOfSled, MI, SledKind::FUNCTION_ENTER, 2);
 }
@@ -967,10 +962,9 @@ void SystemZAsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
   EmitNop(OutContext, *OutStreamer, 4, getSubtargetInfo());
   EmitToStreamer(*OutStreamer,
                  MCInstBuilder(SystemZ::LLILF).addReg(SystemZ::R2D).addImm(0));
-  EmitToStreamer(*OutStreamer,
-                 MCInstBuilder(SystemZ::J)
-                     .addExpr(MCSymbolRefExpr::create(
-                         FuncExit, SystemZMCExpr::VK_PLT, OutContext)));
+  EmitToStreamer(*OutStreamer, MCInstBuilder(SystemZ::J)
+                                   .addExpr(MCSymbolRefExpr::create(
+                                       FuncExit, SystemZ::S_PLT, OutContext)));
   if (FallthroughLabel)
     OutStreamer->emitLabel(FallthroughLabel);
   recordSled(BeginOfSled, MI, SledKind::FUNCTION_EXIT, 2);
@@ -992,13 +986,13 @@ void SystemZAsmPrinter::emitAttributes(Module &M) {
 static uint8_t getSpecifierFromModifier(SystemZCP::SystemZCPModifier Modifier) {
   switch (Modifier) {
   case SystemZCP::TLSGD:
-    return SystemZMCExpr::VK_TLSGD;
+    return SystemZ::S_TLSGD;
   case SystemZCP::TLSLDM:
-    return SystemZMCExpr::VK_TLSLDM;
+    return SystemZ::S_TLSLDM;
   case SystemZCP::DTPOFF:
-    return SystemZMCExpr::VK_DTPOFF;
+    return SystemZ::S_DTPOFF;
   case SystemZCP::NTPOFF:
-    return SystemZMCExpr::VK_NTPOFF;
+    return SystemZ::S_NTPOFF;
   }
   llvm_unreachable("Invalid SystemCPModifier!");
 }
@@ -1043,7 +1037,7 @@ static void printOperand(const MCOperand &MCOp, const MCAsmInfo *MAI,
   else if (MCOp.isImm())
     OS << MCOp.getImm();
   else if (MCOp.isExpr())
-    MCOp.getExpr()->print(OS, MAI);
+    MAI->printExpr(OS, *MCOp.getExpr());
   else
     llvm_unreachable("Invalid operand");
 }
@@ -1145,23 +1139,20 @@ void SystemZAsmPrinter::emitADASection() {
       // imported functions, that are placed in the ADA to be 8 byte aligned.
       EMIT_COMMENT("function descriptor of");
       OutStreamer->emitValue(
-          SystemZMCExpr::create(SystemZMCExpr::VK_SystemZ_RCon,
-                                MCSymbolRefExpr::create(Sym, OutContext),
-                                OutContext),
+          MCSpecifierExpr::create(MCSymbolRefExpr::create(Sym, OutContext),
+                                  SystemZ::S_RCon, OutContext),
           PointerSize);
       OutStreamer->emitValue(
-          SystemZMCExpr::create(SystemZMCExpr::VK_SystemZ_VCon,
-                                MCSymbolRefExpr::create(Sym, OutContext),
-                                OutContext),
+          MCSpecifierExpr::create(MCSymbolRefExpr::create(Sym, OutContext),
+                                  SystemZ::S_VCon, OutContext),
           PointerSize);
       EmittedBytes += PointerSize * 2;
       break;
     case SystemZII::MO_ADA_DATA_SYMBOL_ADDR:
       EMIT_COMMENT("pointer to data symbol");
       OutStreamer->emitValue(
-          SystemZMCExpr::create(SystemZMCExpr::VK_None,
-                                MCSymbolRefExpr::create(Sym, OutContext),
-                                OutContext),
+          MCSpecifierExpr::create(MCSymbolRefExpr::create(Sym, OutContext),
+                                  SystemZ::S_None, OutContext),
           PointerSize);
       EmittedBytes += PointerSize;
       break;
@@ -1174,9 +1165,8 @@ void SystemZAsmPrinter::emitADASection() {
 
       EMIT_COMMENT("pointer to function descriptor");
       OutStreamer->emitValue(
-          SystemZMCExpr::create(SystemZMCExpr::VK_SystemZ_VCon,
-                                MCSymbolRefExpr::create(Alias, OutContext),
-                                OutContext),
+          MCSpecifierExpr::create(MCSymbolRefExpr::create(Alias, OutContext),
+                                  SystemZ::S_VCon, OutContext),
           PointerSize);
       EmittedBytes += PointerSize;
       break;
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 3b86a9bc5879..1c59b1e63b7b 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -828,19 +828,6 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
 
   // Default to having -disable-strictnode-mutation on
   IsStrictFPEnabled = true;
-
-  if (Subtarget.isTargetzOS()) {
-    struct RTLibCallMapping {
-      RTLIB::Libcall Code;
-      const char *Name;
-    };
-    static RTLibCallMapping RTLibCallCommon[] = {
-#define HANDLE_LIBCALL(code, name) {RTLIB::code, name},
-#include "ZOSLibcallNames.def"
-    };
-    for (auto &E : RTLibCallCommon)
-      setLibcallName(E.Code, E.Name);
-  }
 }
 
 bool SystemZTargetLowering::useSoftFloat() const {
@@ -1296,6 +1283,20 @@ bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
   return true;
 }
 
+bool SystemZTargetLowering::hasAndNot(SDValue Y) const {
+  EVT VT = Y.getValueType();
+
+  // We can use NC(G)RK for types in GPRs ...
+  if (VT == MVT::i32 || VT == MVT::i64)
+    return Subtarget.hasMiscellaneousExtensions3();
+
+  // ... or VNC for types in VRs.
+  if (VT.isVector() || VT == MVT::i128)
+    return Subtarget.hasVector();
+
+  return false;
+}
+
 // Information about the addressing mode for a memory access.
 struct AddressingMode {
   // True if a long displacement is supported.
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index f3536a840fda..f2f0bf6d8b41 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -671,6 +671,7 @@ class SystemZTargetLowering : public TargetLowering {
   }
 
   unsigned getStackProbeSize(const MachineFunction &MF) const;
+  bool hasAndNot(SDValue Y) const override;
 
 private:
   const SystemZSubtarget &Subtarget;
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.cpp
index 1aa71618082f..c1d0994a9e17 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZMCInstLower.h"
-#include "MCTargetDesc/SystemZMCExpr.h"
+#include "MCTargetDesc/SystemZMCAsmInfo.h"
 #include "SystemZAsmPrinter.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/MC/MCExpr.h"
@@ -16,15 +16,15 @@
 
 using namespace llvm;
 
-// Return the VK_* enumeration for MachineOperand target flags Flags.
-static SystemZMCExpr::Specifier getSpecifierForTFlags(unsigned Flags) {
+// Return the S_* enumeration for MachineOperand target flags Flags.
+static SystemZ::Specifier getSpecifierForTFlags(unsigned Flags) {
   switch (Flags & SystemZII::MO_SYMBOL_MODIFIER) {
     case 0:
-      return SystemZMCExpr::VK_None;
+      return SystemZ::S_None;
     case SystemZII::MO_GOT:
-      return SystemZMCExpr::VK_GOT;
+      return SystemZ::S_GOT;
     case SystemZII::MO_INDNTPOFF:
-      return SystemZMCExpr::VK_INDNTPOFF;
+      return SystemZ::S_INDNTPOFF;
   }
   llvm_unreachable("Unrecognised MO_ACCESS_MODEL");
 }
@@ -34,7 +34,7 @@ SystemZMCInstLower::SystemZMCInstLower(MCContext &ctx,
   : Ctx(ctx), AsmPrinter(asmprinter) {}
 
 const MCExpr *SystemZMCInstLower::getExpr(const MachineOperand &MO,
-                                          SystemZMCExpr::Specifier Spec) const {
+                                          SystemZ::Specifier Spec) const {
   const MCSymbol *Symbol;
   bool HasOffset = true;
   switch (MO.getType()) {
@@ -85,7 +85,7 @@ MCOperand SystemZMCInstLower::lowerOperand(const MachineOperand &MO) const {
     return MCOperand::createImm(MO.getImm());
 
   default: {
-    SystemZMCExpr::Specifier Kind = getSpecifierForTFlags(MO.getTargetFlags());
+    auto Kind = getSpecifierForTFlags(MO.getTargetFlags());
     return MCOperand::createExpr(getExpr(MO, Kind));
   }
   }
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.h b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.h
index 90526882c853..3187d7726c31 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.h
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZMCInstLower.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZMCINSTLOWER_H
 #define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZMCINSTLOWER_H
 
-#include "MCTargetDesc/SystemZMCExpr.h"
+#include "MCTargetDesc/SystemZMCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
@@ -35,8 +35,7 @@ class LLVM_LIBRARY_VISIBILITY SystemZMCInstLower {
   MCOperand lowerOperand(const MachineOperand& MO) const;
 
   // Return an MCExpr for symbolic operand MO with variant kind Kind.
-  const MCExpr *getExpr(const MachineOperand &MO,
-                        SystemZMCExpr::Specifier) const;
+  const MCExpr *getExpr(const MachineOperand &MO, SystemZ::Specifier) const;
 };
 } // end namespace llvm
 
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
index 6c376e4bf622..942ef8838562 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
@@ -8,7 +8,7 @@
 
 #include "SystemZSubtarget.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
-#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
 #include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
@@ -83,9 +83,9 @@ bool SystemZSubtarget::isAddressedViaADA(const GlobalValue *GV) const {
     // least two byte alignment, then generated code can use relative
     // instructions to address the variable. Otherwise, use the ADA to address
     // the variable.
-    if (GO->getAlignment() & 0x1) {
-      return true;
-    }
+    if (auto *GV = dyn_cast<GlobalVariable>(GO))
+      if (GV->getAlign() && (*GV->getAlign()).value() & 0x1)
+        return true;
 
     // getKindForGlobal only works with definitions
     if (GO->isDeclaration()) {
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
index 0f790ca3a64b..ddb5a730a6fd 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -209,8 +209,7 @@ SystemZTargetMachine::getSubtargetImpl(const Function &F) const {
 
 ScheduleDAGInstrs *
 SystemZTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
-  return new ScheduleDAGMI(C, std::make_unique<SystemZPostRASchedStrategy>(C),
-                           /*RemoveKillFlags=*/true);
+  return createSchedPostRA<SystemZPostRASchedStrategy>(C);
 }
 
 namespace {
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp
index 7d22c26ff9a8..ae90c51432fe 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZTargetObjectFile.h"
-#include "MCTargetDesc/SystemZMCExpr.h"
+#include "MCTargetDesc/SystemZMCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/Target/TargetMachine.h"
 
@@ -15,5 +15,5 @@ using namespace llvm;
 
 const MCExpr *SystemZELFTargetObjectFile::getDebugThreadLocalSymbol(
     const MCSymbol *Sym) const {
-  return MCSymbolRefExpr::create(Sym, SystemZMCExpr::VK_DTPOFF, getContext());
+  return MCSymbolRefExpr::create(Sym, SystemZ::S_DTPOFF, getContext());
 }
diff --git a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index b4bc41974b70..368a4af768b3 100644
--- a/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -15,7 +15,7 @@
 
 namespace llvm {
 
-class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
+class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
   typedef BasicTTIImplBase<SystemZTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
diff --git a/external/llvm-project/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/external/llvm-project/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
index ab499b55ef9b..c54ce40de45f 100644
--- a/external/llvm-project/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
@@ -73,7 +73,7 @@ class VEAsmParser : public MCTargetAsmParser {
   ParseStatus parseVEAsmOperand(std::unique_ptr<VEOperand> &Operand);
 
   // Helper function to parse expression with a symbol.
-  const MCExpr *extractSpecifier(const MCExpr *E, VEMCExpr::Specifier &Variant);
+  const MCExpr *extractSpecifier(const MCExpr *E, VE::Specifier &Variant);
   bool parseExpression(const MCExpr *&EVal);
 
   // Split the mnemonic stripping conditional code and quantifiers
@@ -1036,68 +1036,70 @@ bool VEAsmParser::parseLiteralValues(unsigned Size, SMLoc L) {
 /// Extract \code @lo32/@hi32/etc \endcode specifier from expression.
 /// Recursively scan the expression and check for VK_HI32/LO32/etc
 /// symbol variants.  If all symbols with modifier use the same
-/// variant, return the corresponding VEMCExpr::Specifier,
+/// variant, return the corresponding VE::Specifier,
 /// and a modified expression using the default symbol variant.
 /// Otherwise, return NULL.
 const MCExpr *VEAsmParser::extractSpecifier(const MCExpr *E,
-                                            VEMCExpr::Specifier &Variant) {
+                                            VE::Specifier &Variant) {
   MCContext &Context = getParser().getContext();
-  Variant = VEMCExpr::VK_None;
+  Variant = VE::S_None;
 
   switch (E->getKind()) {
   case MCExpr::Target:
   case MCExpr::Constant:
     return nullptr;
+  case MCExpr::Specifier:
+    llvm_unreachable("unused by this backend");
 
   case MCExpr::SymbolRef: {
     const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(E);
 
-    switch (getSpecifier(SRE)) {
-    case VEMCExpr::VK_None:
+    switch (SRE->getSpecifier()) {
+    case VE::S_None:
       // Use VK_REFLONG to a symbol without modifiers.
-      Variant = VEMCExpr::VK_REFLONG;
+      Variant = VE::S_REFLONG;
       break;
-    case VEMCExpr::VK_HI32:
-      Variant = VEMCExpr::VK_HI32;
+    case VE::S_HI32:
+      Variant = VE::S_HI32;
       break;
-    case VEMCExpr::VK_LO32:
-      Variant = VEMCExpr::VK_LO32;
+    case VE::S_LO32:
+      Variant = VE::S_LO32;
       break;
-    case VEMCExpr::VK_PC_HI32:
-      Variant = VEMCExpr::VK_PC_HI32;
+    case VE::S_PC_HI32:
+      Variant = VE::S_PC_HI32;
       break;
-    case VEMCExpr::VK_PC_LO32:
-      Variant = VEMCExpr::VK_PC_LO32;
+    case VE::S_PC_LO32:
+      Variant = VE::S_PC_LO32;
       break;
-    case VEMCExpr::VK_GOT_HI32:
-      Variant = VEMCExpr::VK_GOT_HI32;
+    case VE::S_GOT_HI32:
+      Variant = VE::S_GOT_HI32;
       break;
-    case VEMCExpr::VK_GOT_LO32:
-      Variant = VEMCExpr::VK_GOT_LO32;
+    case VE::S_GOT_LO32:
+      Variant = VE::S_GOT_LO32;
       break;
-    case VEMCExpr::VK_GOTOFF_HI32:
-      Variant = VEMCExpr::VK_GOTOFF_HI32;
+    case VE::S_GOTOFF_HI32:
+      Variant = VE::S_GOTOFF_HI32;
       break;
-    case VEMCExpr::VK_GOTOFF_LO32:
-      Variant = VEMCExpr::VK_GOTOFF_LO32;
+    case VE::S_GOTOFF_LO32:
+      Variant = VE::S_GOTOFF_LO32;
       break;
-    case VEMCExpr::VK_PLT_HI32:
-      Variant = VEMCExpr::VK_PLT_HI32;
+    case VE::S_PLT_HI32:
+      Variant = VE::S_PLT_HI32;
       break;
-    case VEMCExpr::VK_PLT_LO32:
-      Variant = VEMCExpr::VK_PLT_LO32;
+    case VE::S_PLT_LO32:
+      Variant = VE::S_PLT_LO32;
       break;
-    case VEMCExpr::VK_TLS_GD_HI32:
-      Variant = VEMCExpr::VK_TLS_GD_HI32;
+    case VE::S_TLS_GD_HI32:
+      Variant = VE::S_TLS_GD_HI32;
       break;
-    case VEMCExpr::VK_TLS_GD_LO32:
-      Variant = VEMCExpr::VK_TLS_GD_LO32;
+    case VE::S_TLS_GD_LO32:
+      Variant = VE::S_TLS_GD_LO32;
       break;
-    case VEMCExpr::VK_TPOFF_HI32:
-      Variant = VEMCExpr::VK_TPOFF_HI32;
+    case VE::S_TPOFF_HI32:
+      Variant = VE::S_TPOFF_HI32;
       break;
-    case VEMCExpr::VK_TPOFF_LO32:
-      Variant = VEMCExpr::VK_TPOFF_LO32;
+    case VE::S_TPOFF_LO32:
+      Variant = VE::S_TPOFF_LO32;
       break;
     default:
       return nullptr;
@@ -1116,7 +1118,7 @@ const MCExpr *VEAsmParser::extractSpecifier(const MCExpr *E,
 
   case MCExpr::Binary: {
     const MCBinaryExpr *BE = cast<MCBinaryExpr>(E);
-    VEMCExpr::Specifier LHSVariant, RHSVariant;
+    VE::Specifier LHSVariant, RHSVariant;
     const MCExpr *LHS = extractSpecifier(BE->getLHS(), LHSVariant);
     const MCExpr *RHS = extractSpecifier(BE->getRHS(), RHSVariant);
 
@@ -1128,9 +1130,9 @@ const MCExpr *VEAsmParser::extractSpecifier(const MCExpr *E,
     if (!RHS)
       RHS = BE->getRHS();
 
-    if (LHSVariant == VEMCExpr::VK_None)
+    if (LHSVariant == VE::S_None)
       Variant = RHSVariant;
-    else if (RHSVariant == VEMCExpr::VK_None)
+    else if (RHSVariant == VE::S_None)
       Variant = LHSVariant;
     else if (LHSVariant == RHSVariant)
       Variant = LHSVariant;
@@ -1151,11 +1153,11 @@ bool VEAsmParser::parseExpression(const MCExpr *&EVal) {
   if (getParser().parseExpression(EVal))
     return true;
 
-  // Convert MCSymbolRefExpr with VK_* to MCExpr with VK_*.
-  VEMCExpr::Specifier Specifier;
+  // Convert MCSymbolRefExpr with specifier to MCSpecifierExpr.
+  VE::Specifier Specifier;
   const MCExpr *E = extractSpecifier(EVal, Specifier);
   if (E)
-    EVal = VEMCExpr::create(Specifier, E, getParser().getContext());
+    EVal = MCSpecifierExpr::create(E, Specifier, getParser().getContext());
 
   return false;
 }
diff --git a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
index e707bb2fe3e1..0e3f5d18de07 100644
--- a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
@@ -40,18 +40,18 @@ unsigned VEELFObjectWriter::getRelocType(const MCFixup &Fixup,
                                          const MCValue &Target,
                                          bool IsPCRel) const {
   switch (Target.getSpecifier()) {
-  case VEMCExpr::VK_TLS_GD_HI32:
-  case VEMCExpr::VK_TLS_GD_LO32:
-  case VEMCExpr::VK_TPOFF_HI32:
-  case VEMCExpr::VK_TPOFF_LO32:
+  case VE::S_TLS_GD_HI32:
+  case VE::S_TLS_GD_LO32:
+  case VE::S_TPOFF_HI32:
+  case VE::S_TPOFF_LO32:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
   default:
     break;
   }
-  if (const VEMCExpr *SExpr = dyn_cast<VEMCExpr>(Fixup.getValue())) {
-    if (SExpr->getSpecifier() == VEMCExpr::VK_PC_LO32)
+  if (auto *SExpr = dyn_cast<MCSpecifierExpr>(Fixup.getValue())) {
+    if (SExpr->getSpecifier() == VE::S_PC_LO32)
       return ELF::R_VE_PC_LO32;
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
index 77bd30e96f7b..b78b86f70f39 100644
--- a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
@@ -12,6 +12,7 @@
 
 #include "VEInstPrinter.h"
 #include "VE.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -61,7 +62,7 @@ void VEInstPrinter::printOperand(const MCInst *MI, int OpNum,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
 }
 
 void VEInstPrinter::printMemASXOperand(const MCInst *MI, int OpNum,
diff --git a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp
index f290804ae449..8eb3aedd668e 100644
--- a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp
@@ -14,25 +14,26 @@
 #include "VEMCExpr.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
 
 const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
-    {VEMCExpr::VK_HI32, "hi"},
-    {VEMCExpr::VK_LO32, "lo"},
-    {VEMCExpr::VK_PC_HI32, "pc_hi"},
-    {VEMCExpr::VK_PC_LO32, "pc_lo"},
-    {VEMCExpr::VK_GOT_HI32, "got_hi"},
-    {VEMCExpr::VK_GOT_LO32, "got_lo"},
-    {VEMCExpr::VK_GOTOFF_HI32, "gotoff_hi"},
-    {VEMCExpr::VK_GOTOFF_LO32, "gotoff_lo"},
-    {VEMCExpr::VK_PLT_HI32, "plt_hi"},
-    {VEMCExpr::VK_PLT_LO32, "plt_lo"},
-    {VEMCExpr::VK_TLS_GD_HI32, "tls_gd_hi"},
-    {VEMCExpr::VK_TLS_GD_LO32, "tls_gd_lo"},
-    {VEMCExpr::VK_TPOFF_HI32, "tpoff_hi"},
-    {VEMCExpr::VK_TPOFF_LO32, "tpoff_lo"},
+    {VE::S_HI32, "hi"},
+    {VE::S_LO32, "lo"},
+    {VE::S_PC_HI32, "pc_hi"},
+    {VE::S_PC_LO32, "pc_lo"},
+    {VE::S_GOT_HI32, "got_hi"},
+    {VE::S_GOT_LO32, "got_lo"},
+    {VE::S_GOTOFF_HI32, "gotoff_hi"},
+    {VE::S_GOTOFF_LO32, "gotoff_lo"},
+    {VE::S_PLT_HI32, "plt_hi"},
+    {VE::S_PLT_LO32, "plt_lo"},
+    {VE::S_TLS_GD_HI32, "tls_gd_hi"},
+    {VE::S_TLS_GD_LO32, "tls_gd_lo"},
+    {VE::S_TPOFF_HI32, "tpoff_hi"},
+    {VE::S_TPOFF_LO32, "tpoff_lo"},
 };
 
 void VEELFMCAsmInfo::anchor() {}
@@ -56,3 +57,20 @@ VEELFMCAsmInfo::VEELFMCAsmInfo(const Triple &TheTriple) {
 
   initializeVariantKinds(variantKindDescs);
 }
+
+void VEELFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                        const MCSpecifierExpr &Expr) const {
+  printExpr(OS, *Expr.getSubExpr());
+  auto specifier = Expr.getSpecifier();
+  if (specifier && specifier != VE::S_REFLONG)
+    OS << '@' << getSpecifierName(specifier);
+}
+
+bool VEELFMCAsmInfo::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr,
+                                               MCValue &Res,
+                                               const MCAssembler *Asm) const {
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(Res, Asm))
+    return false;
+  Res.setSpecifier(Expr.getSpecifier());
+  return true;
+}
diff --git a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h
index 6557d68b383c..2d73c94e2113 100644
--- a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h
@@ -24,6 +24,10 @@ class VEELFMCAsmInfo : public MCAsmInfoELF {
 
 public:
   explicit VEELFMCAsmInfo(const Triple &TheTriple);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
+  bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
+                                 const MCAssembler *Asm) const override;
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
index 77fea1cb1756..712de5accce5 100644
--- a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
@@ -98,8 +98,8 @@ unsigned VEMCCodeEmitter::getMachineOpValue(const MCInst &MI,
   assert(MO.isExpr());
 
   const MCExpr *Expr = MO.getExpr();
-  if (const VEMCExpr *SExpr = dyn_cast<VEMCExpr>(Expr)) {
-    MCFixupKind Kind = (MCFixupKind)SExpr->getFixupKind();
+  if (const auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
+    auto Kind = VE::getFixupKind(SExpr->getSpecifier());
     Fixups.push_back(MCFixup::create(0, Expr, Kind));
     return 0;
   }
diff --git a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
index 62f5ae67fb01..ca13aba095e2 100644
--- a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
@@ -22,64 +22,39 @@ using namespace llvm;
 
 #define DEBUG_TYPE "vemcexpr"
 
-const VEMCExpr *VEMCExpr::create(Specifier Kind, const MCExpr *Expr,
-                                 MCContext &Ctx) {
-  return new (Ctx) VEMCExpr(Kind, Expr);
-}
-
-void VEMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-
-  const MCExpr *Expr = getSubExpr();
-  Expr->print(OS, MAI);
-  if (specifier != VK_None && specifier != VK_REFLONG)
-    OS << '@' << MAI->getSpecifierName(specifier);
-}
-
-VE::Fixups VEMCExpr::getFixupKind(VEMCExpr::Specifier S) {
+VE::Fixups VE::getFixupKind(uint8_t S) {
   switch (S) {
   default:
     llvm_unreachable("Unhandled VEMCExpr::Specifier");
-  case VK_REFLONG:
+  case VE::S_REFLONG:
     return VE::fixup_ve_reflong;
-  case VK_HI32:
+  case VE::S_HI32:
     return VE::fixup_ve_hi32;
-  case VK_LO32:
+  case VE::S_LO32:
     return VE::fixup_ve_lo32;
-  case VK_PC_HI32:
+  case VE::S_PC_HI32:
     return VE::fixup_ve_pc_hi32;
-  case VK_PC_LO32:
+  case VE::S_PC_LO32:
     return VE::fixup_ve_pc_lo32;
-  case VK_GOT_HI32:
+  case VE::S_GOT_HI32:
     return VE::fixup_ve_got_hi32;
-  case VK_GOT_LO32:
+  case VE::S_GOT_LO32:
     return VE::fixup_ve_got_lo32;
-  case VK_GOTOFF_HI32:
+  case VE::S_GOTOFF_HI32:
     return VE::fixup_ve_gotoff_hi32;
-  case VK_GOTOFF_LO32:
+  case VE::S_GOTOFF_LO32:
     return VE::fixup_ve_gotoff_lo32;
-  case VK_PLT_HI32:
+  case VE::S_PLT_HI32:
     return VE::fixup_ve_plt_hi32;
-  case VK_PLT_LO32:
+  case VE::S_PLT_LO32:
     return VE::fixup_ve_plt_lo32;
-  case VK_TLS_GD_HI32:
+  case VE::S_TLS_GD_HI32:
     return VE::fixup_ve_tls_gd_hi32;
-  case VK_TLS_GD_LO32:
+  case VE::S_TLS_GD_LO32:
     return VE::fixup_ve_tls_gd_lo32;
-  case VK_TPOFF_HI32:
+  case VE::S_TPOFF_HI32:
     return VE::fixup_ve_tpoff_hi32;
-  case VK_TPOFF_LO32:
+  case VE::S_TPOFF_LO32:
     return VE::fixup_ve_tpoff_lo32;
   }
 }
-
-bool VEMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                         const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-  return true;
-}
-
-void VEMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
diff --git a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h
index a176cd3f0971..b7913513bd51 100644
--- a/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h
@@ -20,73 +20,30 @@
 namespace llvm {
 
 class StringRef;
-class VEMCExpr : public MCTargetExpr {
-public:
-  enum Specifier {
-    VK_None,
 
-    VK_REFLONG = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_HI32,        // @hi
-    VK_LO32,        // @lo
-    VK_PC_HI32,     // @pc_hi
-    VK_PC_LO32,     // @pc_lo
-    VK_GOT_HI32,    // @got_hi
-    VK_GOT_LO32,    // @got_lo
-    VK_GOTOFF_HI32, // @gotoff_hi
-    VK_GOTOFF_LO32, // @gotoff_lo
-    VK_PLT_HI32,    // @plt_hi
-    VK_PLT_LO32,    // @plt_lo
-    VK_TLS_GD_HI32, // @tls_gd_hi
-    VK_TLS_GD_LO32, // @tls_gd_lo
-    VK_TPOFF_HI32,  // @tpoff_hi
-    VK_TPOFF_LO32,  // @tpoff_lo
-  };
-
-private:
-  const Specifier specifier;
-  const MCExpr *Expr;
-
-  explicit VEMCExpr(Specifier S, const MCExpr *Expr)
-      : specifier(S), Expr(Expr) {}
-
-public:
-  /// @name Construction
-  /// @{
-
-  static const VEMCExpr *create(Specifier Kind, const MCExpr *Expr,
-                                MCContext &Ctx);
-  /// @}
-  /// @name Accessors
-  /// @{
-
-  /// getOpcode - Get the kind of this expression.
-  Specifier getSpecifier() const { return specifier; }
-
-  /// getSubExpr - Get the child of this expression.
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  /// getFixupKind - Get the fixup kind of this expression.
-  VE::Fixups getFixupKind() const { return getFixupKind(specifier); }
-
-  /// @}
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-
-  static VE::Fixups getFixupKind(Specifier S);
+namespace VE {
+enum Specifier {
+  S_None,
+
+  S_REFLONG = MCSymbolRefExpr::FirstTargetSpecifier,
+  S_HI32,        // @hi
+  S_LO32,        // @lo
+  S_PC_HI32,     // @pc_hi
+  S_PC_LO32,     // @pc_lo
+  S_GOT_HI32,    // @got_hi
+  S_GOT_LO32,    // @got_lo
+  S_GOTOFF_HI32, // @gotoff_hi
+  S_GOTOFF_LO32, // @gotoff_lo
+  S_PLT_HI32,    // @plt_hi
+  S_PLT_LO32,    // @plt_lo
+  S_TLS_GD_HI32, // @tls_gd_hi
+  S_TLS_GD_LO32, // @tls_gd_lo
+  S_TPOFF_HI32,  // @tpoff_hi
+  S_TPOFF_LO32,  // @tpoff_lo
 };
 
-static inline VEMCExpr::Specifier getSpecifier(const MCSymbolRefExpr *SRE) {
-  return VEMCExpr::Specifier(SRE->getKind());
-}
+VE::Fixups getFixupKind(uint8_t S);
+} // namespace VE
 
 } // namespace llvm
 
diff --git a/external/llvm-project/llvm/lib/Target/VE/VEAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/VE/VEAsmPrinter.cpp
index ee347cda0521..af0dc0404d3c 100644
--- a/external/llvm-project/llvm/lib/Target/VE/VEAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/VEAsmPrinter.cpp
@@ -67,18 +67,17 @@ class VEAsmPrinter : public AsmPrinter {
 };
 } // end of anonymous namespace
 
-static MCOperand createVEMCOperand(VEMCExpr::Specifier Kind, MCSymbol *Sym,
+static MCOperand createVEMCOperand(VE::Specifier Kind, MCSymbol *Sym,
                                    MCContext &OutContext) {
   const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Sym, OutContext);
-  const VEMCExpr *expr = VEMCExpr::create(Kind, MCSym, OutContext);
-  return MCOperand::createExpr(expr);
+  return MCOperand::createExpr(
+      MCSpecifierExpr::create(MCSym, Kind, OutContext));
 }
 
-static MCOperand createGOTRelExprOp(VEMCExpr::Specifier Kind,
-                                    MCSymbol *GOTLabel, MCContext &OutContext) {
+static MCOperand createGOTRelExprOp(VE::Specifier Kind, MCSymbol *GOTLabel,
+                                    MCContext &OutContext) {
   const MCSymbolRefExpr *GOT = MCSymbolRefExpr::create(GOTLabel, OutContext);
-  const VEMCExpr *expr = VEMCExpr::create(Kind, GOT, OutContext);
-  return MCOperand::createExpr(expr);
+  return MCOperand::createExpr(MCSpecifierExpr::create(GOT, Kind, OutContext));
 }
 
 static void emitSIC(MCStreamer &OutStreamer, MCOperand &RD,
@@ -166,9 +165,8 @@ static void emitANDrm(MCStreamer &OutStreamer, MCOperand &RS1, MCOperand &Imm,
 }
 
 static void emitHiLo(MCStreamer &OutStreamer, MCSymbol *GOTSym,
-                     VEMCExpr::Specifier HiKind, VEMCExpr::Specifier LoKind,
-                     MCOperand &RD, MCContext &OutContext,
-                     const MCSubtargetInfo &STI) {
+                     VE::Specifier HiKind, VE::Specifier LoKind, MCOperand &RD,
+                     MCContext &OutContext, const MCSubtargetInfo &STI) {
 
   MCOperand hi = createVEMCOperand(HiKind, GOTSym, OutContext);
   MCOperand lo = createVEMCOperand(LoKind, GOTSym, OutContext);
@@ -194,8 +192,8 @@ void VEAsmPrinter::lowerGETGOTAndEmitMCInsts(const MachineInstr *MI,
     case CodeModel::Small:
     case CodeModel::Medium:
     case CodeModel::Large:
-      emitHiLo(*OutStreamer, GOTLabel, VEMCExpr::VK_HI32, VEMCExpr::VK_LO32,
-               MCRegOP, OutContext, STI);
+      emitHiLo(*OutStreamer, GOTLabel, VE::S_HI32, VE::S_LO32, MCRegOP,
+               OutContext, STI);
       break;
     }
     return;
@@ -209,14 +207,12 @@ void VEAsmPrinter::lowerGETGOTAndEmitMCInsts(const MachineInstr *MI,
   // sic %plt
   // lea.sl %got, _GLOBAL_OFFSET_TABLE_@PC_HI(%plt, %got)
   MCOperand cim24 = MCOperand::createImm(-24);
-  MCOperand loImm =
-      createGOTRelExprOp(VEMCExpr::VK_PC_LO32, GOTLabel, OutContext);
+  MCOperand loImm = createGOTRelExprOp(VE::S_PC_LO32, GOTLabel, OutContext);
   emitLEAzii(*OutStreamer, cim24, loImm, MCRegOP, STI);
   MCOperand M032 = MCOperand::createImm(M0(32));
   emitANDrm(*OutStreamer, MCRegOP, M032, MCRegOP, STI);
   emitSIC(*OutStreamer, RegPLT, STI);
-  MCOperand hiImm =
-      createGOTRelExprOp(VEMCExpr::VK_PC_HI32, GOTLabel, OutContext);
+  MCOperand hiImm = createGOTRelExprOp(VE::S_PC_HI32, GOTLabel, OutContext);
   emitLEASLrri(*OutStreamer, RegGOT, RegPLT, hiImm, MCRegOP, STI);
 }
 
@@ -257,14 +253,12 @@ void VEAsmPrinter::lowerGETFunPLTAndEmitMCInsts(const MachineInstr *MI,
   // sic %plt                            ; FIXME: is it safe to use %plt here?
   // lea.sl %dst, func@plt_hi(%plt, %dst)
   MCOperand cim24 = MCOperand::createImm(-24);
-  MCOperand loImm =
-      createGOTRelExprOp(VEMCExpr::VK_PLT_LO32, AddrSym, OutContext);
+  MCOperand loImm = createGOTRelExprOp(VE::S_PLT_LO32, AddrSym, OutContext);
   emitLEAzii(*OutStreamer, cim24, loImm, MCRegOP, STI);
   MCOperand M032 = MCOperand::createImm(M0(32));
   emitANDrm(*OutStreamer, MCRegOP, M032, MCRegOP, STI);
   emitSIC(*OutStreamer, RegPLT, STI);
-  MCOperand hiImm =
-      createGOTRelExprOp(VEMCExpr::VK_PLT_HI32, AddrSym, OutContext);
+  MCOperand hiImm = createGOTRelExprOp(VE::S_PLT_HI32, AddrSym, OutContext);
   emitLEASLrri(*OutStreamer, MCRegOP, RegPLT, hiImm, MCRegOP, STI);
 }
 
@@ -305,22 +299,20 @@ void VEAsmPrinter::lowerGETTLSAddrAndEmitMCInsts(const MachineInstr *MI,
   // lea.sl %s12, __tls_get_addr@plt_hi(%s12, %lr)
   // bsic %lr, (, %s12)
   MCOperand cim24 = MCOperand::createImm(-24);
-  MCOperand loImm =
-      createGOTRelExprOp(VEMCExpr::VK_TLS_GD_LO32, AddrSym, OutContext);
+  MCOperand loImm = createGOTRelExprOp(VE::S_TLS_GD_LO32, AddrSym, OutContext);
   emitLEAzii(*OutStreamer, cim24, loImm, RegS0, STI);
   MCOperand M032 = MCOperand::createImm(M0(32));
   emitANDrm(*OutStreamer, RegS0, M032, RegS0, STI);
   emitSIC(*OutStreamer, RegLR, STI);
-  MCOperand hiImm =
-      createGOTRelExprOp(VEMCExpr::VK_TLS_GD_HI32, AddrSym, OutContext);
+  MCOperand hiImm = createGOTRelExprOp(VE::S_TLS_GD_HI32, AddrSym, OutContext);
   emitLEASLrri(*OutStreamer, RegS0, RegLR, hiImm, RegS0, STI);
   MCOperand ci8 = MCOperand::createImm(8);
   MCOperand loImm2 =
-      createGOTRelExprOp(VEMCExpr::VK_PLT_LO32, GetTLSLabel, OutContext);
+      createGOTRelExprOp(VE::S_PLT_LO32, GetTLSLabel, OutContext);
   emitLEAzii(*OutStreamer, ci8, loImm2, RegS12, STI);
   emitANDrm(*OutStreamer, RegS12, M032, RegS12, STI);
   MCOperand hiImm2 =
-      createGOTRelExprOp(VEMCExpr::VK_PLT_HI32, GetTLSLabel, OutContext);
+      createGOTRelExprOp(VE::S_PLT_HI32, GetTLSLabel, OutContext);
   emitLEASLrri(*OutStreamer, RegS12, RegLR, hiImm2, RegS12, STI);
   emitBSIC(*OutStreamer, RegLR, RegS12, STI);
 }
diff --git a/external/llvm-project/llvm/lib/Target/VE/VEISelLowering.cpp b/external/llvm-project/llvm/lib/Target/VE/VEISelLowering.cpp
index 313c894cafa8..b5a0d26abbf8 100644
--- a/external/llvm-project/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -664,7 +664,7 @@ SDValue VETargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
       Callee = DAG.getNode(VEISD::GETFUNPLT, DL, PtrVT, Callee);
     } else {
-      Callee = makeHiLoPair(Callee, VEMCExpr::VK_HI32, VEMCExpr::VK_LO32, DAG);
+      Callee = makeHiLoPair(Callee, VE::S_HI32, VE::S_LO32, DAG);
     }
   } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
     if (IsPICCall) {
@@ -673,7 +673,7 @@ SDValue VETargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
       Callee = DAG.getNode(VEISD::GETFUNPLT, DL, PtrVT, Callee);
     } else {
-      Callee = makeHiLoPair(Callee, VEMCExpr::VK_HI32, VEMCExpr::VK_LO32, DAG);
+      Callee = makeHiLoPair(Callee, VE::S_HI32, VE::S_LO32, DAG);
     }
   }
 
@@ -1020,8 +1020,8 @@ SDValue VETargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
       //     lea %reg, label@gotoff_lo
       //     and %reg, %reg, (32)0
       //     lea.sl %reg, label@gotoff_hi(%reg, %got)
-      SDValue HiLo = makeHiLoPair(Op, VEMCExpr::VK_GOTOFF_HI32,
-                                  VEMCExpr::VK_GOTOFF_LO32, DAG);
+      SDValue HiLo =
+          makeHiLoPair(Op, VE::S_GOTOFF_HI32, VE::S_GOTOFF_LO32, DAG);
       SDValue GlobalBase = DAG.getNode(VEISD::GLOBAL_BASE_REG, DL, PtrVT);
       return DAG.getNode(ISD::ADD, DL, PtrVT, GlobalBase, HiLo);
     }
@@ -1030,8 +1030,7 @@ SDValue VETargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
     //     and %reg, %reg, (32)0
     //     lea.sl %reg, label@got_hi(%reg)
     //     ld %reg, (%reg, %got)
-    SDValue HiLo =
-        makeHiLoPair(Op, VEMCExpr::VK_GOT_HI32, VEMCExpr::VK_GOT_LO32, DAG);
+    SDValue HiLo = makeHiLoPair(Op, VE::S_GOT_HI32, VE::S_GOT_LO32, DAG);
     SDValue GlobalBase = DAG.getNode(VEISD::GLOBAL_BASE_REG, DL, PtrVT);
     SDValue AbsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, GlobalBase, HiLo);
     return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), AbsAddr,
@@ -1046,7 +1045,7 @@ SDValue VETargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
   case CodeModel::Medium:
   case CodeModel::Large:
     // abs64.
-    return makeHiLoPair(Op, VEMCExpr::VK_HI32, VEMCExpr::VK_LO32, DAG);
+    return makeHiLoPair(Op, VE::S_HI32, VE::S_LO32, DAG);
   }
 }
 
@@ -1782,12 +1781,11 @@ SDValue VETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
     SDValue Addr =
         DAG.getTargetExternalSymbol(TM->getStrList()->back().c_str(), VT, 0);
     if (isPositionIndependent()) {
-      Addr = makeHiLoPair(Addr, VEMCExpr::VK_GOTOFF_HI32,
-                          VEMCExpr::VK_GOTOFF_LO32, DAG);
+      Addr = makeHiLoPair(Addr, VE::S_GOTOFF_HI32, VE::S_GOTOFF_LO32, DAG);
       SDValue GlobalBase = DAG.getNode(VEISD::GLOBAL_BASE_REG, DL, VT);
       return DAG.getNode(ISD::ADD, DL, VT, GlobalBase, Addr);
     }
-    return makeHiLoPair(Addr, VEMCExpr::VK_HI32, VEMCExpr::VK_LO32, DAG);
+    return makeHiLoPair(Addr, VE::S_HI32, VE::S_LO32, DAG);
   }
   }
 }
@@ -2011,8 +2009,7 @@ SDValue VETargetLowering::getPICJumpTableRelocBase(SDValue Table,
   // In order to do so, we need to genarate correctly marked DAG node using
   // makeHiLoPair.
   SDValue Op = DAG.getGlobalAddress(Function, DL, PtrTy);
-  SDValue HiLo =
-      makeHiLoPair(Op, VEMCExpr::VK_GOTOFF_HI32, VEMCExpr::VK_GOTOFF_LO32, DAG);
+  SDValue HiLo = makeHiLoPair(Op, VE::S_GOTOFF_HI32, VE::S_GOTOFF_LO32, DAG);
   SDValue GlobalBase = DAG.getNode(VEISD::GLOBAL_BASE_REG, DL, PtrTy);
   return DAG.getNode(ISD::ADD, DL, PtrTy, GlobalBase, HiLo);
 }
@@ -2038,14 +2035,14 @@ Register VETargetLowering::prepareMBB(MachineBasicBlock &MBB,
     BuildMI(MBB, I, DL, TII->get(VE::LEAzii), Tmp1)
         .addImm(0)
         .addImm(0)
-        .addMBB(TargetBB, VEMCExpr::VK_GOTOFF_LO32);
+        .addMBB(TargetBB, VE::S_GOTOFF_LO32);
     BuildMI(MBB, I, DL, TII->get(VE::ANDrm), Tmp2)
         .addReg(Tmp1, getKillRegState(true))
         .addImm(M0(32));
     BuildMI(MBB, I, DL, TII->get(VE::LEASLrri), Result)
         .addReg(VE::SX15)
         .addReg(Tmp2, getKillRegState(true))
-        .addMBB(TargetBB, VEMCExpr::VK_GOTOFF_HI32);
+        .addMBB(TargetBB, VE::S_GOTOFF_HI32);
   } else {
     // Create following instructions for non-PIC code.
     //     lea     %Tmp1, TargetBB@lo
@@ -2054,14 +2051,14 @@ Register VETargetLowering::prepareMBB(MachineBasicBlock &MBB,
     BuildMI(MBB, I, DL, TII->get(VE::LEAzii), Tmp1)
         .addImm(0)
         .addImm(0)
-        .addMBB(TargetBB, VEMCExpr::VK_LO32);
+        .addMBB(TargetBB, VE::S_LO32);
     BuildMI(MBB, I, DL, TII->get(VE::ANDrm), Tmp2)
         .addReg(Tmp1, getKillRegState(true))
         .addImm(M0(32));
     BuildMI(MBB, I, DL, TII->get(VE::LEASLrii), Result)
         .addReg(Tmp2, getKillRegState(true))
         .addImm(0)
-        .addMBB(TargetBB, VEMCExpr::VK_HI32);
+        .addMBB(TargetBB, VE::S_HI32);
   }
   return Result;
 }
@@ -2099,14 +2096,14 @@ Register VETargetLowering::prepareSymbol(MachineBasicBlock &MBB,
       BuildMI(MBB, I, DL, TII->get(VE::LEAzii), Tmp1)
           .addImm(0)
           .addImm(0)
-          .addExternalSymbol(Symbol.data(), VEMCExpr::VK_GOTOFF_LO32);
+          .addExternalSymbol(Symbol.data(), VE::S_GOTOFF_LO32);
       BuildMI(MBB, I, DL, TII->get(VE::ANDrm), Tmp2)
           .addReg(Tmp1, getKillRegState(true))
           .addImm(M0(32));
       BuildMI(MBB, I, DL, TII->get(VE::LEASLrri), Result)
           .addReg(VE::SX15)
           .addReg(Tmp2, getKillRegState(true))
-          .addExternalSymbol(Symbol.data(), VEMCExpr::VK_GOTOFF_HI32);
+          .addExternalSymbol(Symbol.data(), VE::S_GOTOFF_HI32);
     } else {
       Register Tmp1 = MRI.createVirtualRegister(RC);
       Register Tmp2 = MRI.createVirtualRegister(RC);
@@ -2119,14 +2116,14 @@ Register VETargetLowering::prepareSymbol(MachineBasicBlock &MBB,
       BuildMI(MBB, I, DL, TII->get(VE::LEAzii), Tmp1)
           .addImm(0)
           .addImm(0)
-          .addExternalSymbol(Symbol.data(), VEMCExpr::VK_GOT_LO32);
+          .addExternalSymbol(Symbol.data(), VE::S_GOT_LO32);
       BuildMI(MBB, I, DL, TII->get(VE::ANDrm), Tmp2)
           .addReg(Tmp1, getKillRegState(true))
           .addImm(M0(32));
       BuildMI(MBB, I, DL, TII->get(VE::LEASLrri), Tmp3)
           .addReg(VE::SX15)
           .addReg(Tmp2, getKillRegState(true))
-          .addExternalSymbol(Symbol.data(), VEMCExpr::VK_GOT_HI32);
+          .addExternalSymbol(Symbol.data(), VE::S_GOT_HI32);
       BuildMI(MBB, I, DL, TII->get(VE::LDrii), Result)
           .addReg(Tmp3, getKillRegState(true))
           .addImm(0)
@@ -2142,14 +2139,14 @@ Register VETargetLowering::prepareSymbol(MachineBasicBlock &MBB,
     BuildMI(MBB, I, DL, TII->get(VE::LEAzii), Tmp1)
         .addImm(0)
         .addImm(0)
-        .addExternalSymbol(Symbol.data(), VEMCExpr::VK_LO32);
+        .addExternalSymbol(Symbol.data(), VE::S_LO32);
     BuildMI(MBB, I, DL, TII->get(VE::ANDrm), Tmp2)
         .addReg(Tmp1, getKillRegState(true))
         .addImm(M0(32));
     BuildMI(MBB, I, DL, TII->get(VE::LEASLrii), Result)
         .addReg(Tmp2, getKillRegState(true))
         .addImm(0)
-        .addExternalSymbol(Symbol.data(), VEMCExpr::VK_HI32);
+        .addExternalSymbol(Symbol.data(), VE::S_HI32);
   }
   return Result;
 }
@@ -2528,14 +2525,14 @@ VETargetLowering::emitSjLjDispatchBlock(MachineInstr &MI,
     BuildMI(DispContBB, DL, TII->get(VE::LEAzii), Tmp1)
         .addImm(0)
         .addImm(0)
-        .addJumpTableIndex(MJTI, VEMCExpr::VK_GOTOFF_LO32);
+        .addJumpTableIndex(MJTI, VE::S_GOTOFF_LO32);
     BuildMI(DispContBB, DL, TII->get(VE::ANDrm), Tmp2)
         .addReg(Tmp1, getKillRegState(true))
         .addImm(M0(32));
     BuildMI(DispContBB, DL, TII->get(VE::LEASLrri), BReg)
         .addReg(VE::SX15)
         .addReg(Tmp2, getKillRegState(true))
-        .addJumpTableIndex(MJTI, VEMCExpr::VK_GOTOFF_HI32);
+        .addJumpTableIndex(MJTI, VE::S_GOTOFF_HI32);
   } else {
     // Create following instructions for non-PIC code.
     //     lea     %Tmp1, .LJTI0_0@lo
@@ -2544,14 +2541,14 @@ VETargetLowering::emitSjLjDispatchBlock(MachineInstr &MI,
     BuildMI(DispContBB, DL, TII->get(VE::LEAzii), Tmp1)
         .addImm(0)
         .addImm(0)
-        .addJumpTableIndex(MJTI, VEMCExpr::VK_LO32);
+        .addJumpTableIndex(MJTI, VE::S_LO32);
     BuildMI(DispContBB, DL, TII->get(VE::ANDrm), Tmp2)
         .addReg(Tmp1, getKillRegState(true))
         .addImm(M0(32));
     BuildMI(DispContBB, DL, TII->get(VE::LEASLrii), BReg)
         .addReg(Tmp2, getKillRegState(true))
         .addImm(0)
-        .addJumpTableIndex(MJTI, VEMCExpr::VK_HI32);
+        .addJumpTableIndex(MJTI, VE::S_HI32);
   }
 
   switch (JTE) {
diff --git a/external/llvm-project/llvm/lib/Target/VE/VEMCInstLower.cpp b/external/llvm-project/llvm/lib/Target/VE/VEMCInstLower.cpp
index bed71df3921c..a438d8740cd0 100644
--- a/external/llvm-project/llvm/lib/Target/VE/VEMCInstLower.cpp
+++ b/external/llvm-project/llvm/lib/Target/VE/VEMCInstLower.cpp
@@ -28,7 +28,7 @@ using namespace llvm;
 static MCOperand LowerSymbolOperand(const MachineInstr *MI,
                                     const MachineOperand &MO,
                                     const MCSymbol *Symbol, AsmPrinter &AP) {
-  VEMCExpr::Specifier Kind = (VEMCExpr::Specifier)MO.getTargetFlags();
+  auto Kind = (VE::Specifier)MO.getTargetFlags();
 
   const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, AP.OutContext);
   // Add offset iff MO is not jump table info or machine basic block.
@@ -36,7 +36,7 @@ static MCOperand LowerSymbolOperand(const MachineInstr *MI,
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(MO.getOffset(), AP.OutContext),
         AP.OutContext);
-  Expr = VEMCExpr::create(Kind, Expr, AP.OutContext);
+  Expr = MCSpecifierExpr::create(Expr, Kind, AP.OutContext);
   return MCOperand::createExpr(Expr);
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/VE/VETargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/VE/VETargetTransformInfo.h
index 5cb028608782..5c0ddca62c76 100644
--- a/external/llvm-project/llvm/lib/Target/VE/VETargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/VE/VETargetTransformInfo.h
@@ -49,7 +49,7 @@ static bool isVectorLaneType(llvm::Type &ElemTy) {
 
 namespace llvm {
 
-class VETTIImpl : public BasicTTIImplBase<VETTIImpl> {
+class VETTIImpl final : public BasicTTIImplBase<VETTIImpl> {
   using BaseT = BasicTTIImplBase<VETTIImpl>;
   friend BaseT;
 
diff --git a/external/llvm-project/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
index 344ccec58aff..321aee472082 100644
--- a/external/llvm-project/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -344,7 +345,7 @@ void WebAssemblyInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
       auto &Sym = static_cast<const MCSymbolWasm &>(SRE->getSymbol());
       O << WebAssembly::signatureToString(Sym.getSignature());
     } else {
-      Op.getExpr()->print(O, &MAI);
+      MAI.printExpr(O, *Op.getExpr());
     }
   }
 }
diff --git a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
index f02725efc7e0..344a3636b431 100644
--- a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
+++ b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
@@ -135,7 +135,6 @@ bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) {
         Function::Create(NewType, F.getLinkage(), F.getName() + ".fixed_sig");
     NewF->setAttributes(F.getAttributes());
     NewF->removeFnAttr("no-prototype");
-    NewF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat;
     Replacements.emplace_back(&F, NewF);
   }
 
diff --git a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
index 6c46673c36bf..07171d472dc2 100644
--- a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
+++ b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
@@ -375,7 +375,7 @@ void WebAssemblyFixIrreducibleControlFlow::makeSingleEntryLoop(
   // add them as successors.
   DenseMap<MachineBasicBlock *, unsigned> Indices;
   for (auto *Entry : SortedEntries) {
-    auto Pair = Indices.insert(std::make_pair(Entry, 0));
+    auto Pair = Indices.try_emplace(Entry);
     assert(Pair.second);
 
     unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1;
diff --git a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index aac347331119..3cd923c0ba05 100644
--- a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -385,11 +385,6 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
 
   setMaxAtomicSizeInBitsSupported(64);
 
-  // Define the emscripten name for return address helper.
-  // TODO: when implementing other Wasm backends, make this generic or only do
-  // this on emscripten depending on what they end up doing.
-  setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
-
   // Always convert switches to br_tables unless there is only one case, which
   // is equivalent to a simple branch. This reduces code size for wasm, and we
   // defer possible jump table optimizations to the VM.
diff --git a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index ce795d3dedc6..d5c4532824c0 100644
--- a/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/external/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -531,8 +531,10 @@ struct StaticLibcallNameMap {
   StaticLibcallNameMap() {
     static const std::pair<const char *, RTLIB::Libcall> NameLibcalls[] = {
 #define HANDLE_LIBCALL(code, name) {(const char *)name, RTLIB::code},
+#define LIBCALL_NO_NAME nullptr
 #include "llvm/IR/RuntimeLibcalls.def"
 #undef HANDLE_LIBCALL
+#undef LIBCALL_NO_NAME
     };
     for (const auto &NameLibcall : NameLibcalls) {
       if (NameLibcall.first != nullptr &&
diff --git a/external/llvm-project/llvm/lib/Target/X86/CMakeLists.txt b/external/llvm-project/llvm/lib/Target/X86/CMakeLists.txt
index 6627e97dd094..1bf9f8b46799 100644
--- a/external/llvm-project/llvm/lib/Target/X86/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Target/X86/CMakeLists.txt
@@ -85,7 +85,6 @@ set(sources
   X86VZeroUpper.cpp
   X86WinEHState.cpp
   X86WinEHUnwindV2.cpp
-  X86WinFixupBufferSecurityCheck.cpp
   X86InsertWait.cpp
   GISel/X86CallLowering.cpp
   GISel/X86InstructionSelector.cpp
diff --git a/external/llvm-project/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/external/llvm-project/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
index ea08f71b8af4..3090ad313b90 100644
--- a/external/llvm-project/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
@@ -364,7 +364,7 @@ bool X86InstructionSelector::select(MachineInstr &I) {
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
   unsigned Opcode = I.getOpcode();
-  if (!isPreISelGenericOpcode(Opcode)) {
+  if (!isPreISelGenericOpcode(Opcode) && !I.isPreISelOpcode()) {
     // Certain non-generic instructions also need some special handling.
 
     if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
diff --git a/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index f008cb1bea83..11dd05c58498 100644
--- a/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -490,31 +490,25 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
       });
 
   getActionDefinitionsBuilder(G_SITOFP)
-      .legalIf([=](const LegalityQuery &Query) {
-        return (HasSSE1 &&
-                (typePairInSet(0, 1, {{s32, s32}})(Query) ||
-                 (Is64Bit && typePairInSet(0, 1, {{s32, s64}})(Query)))) ||
-               (HasSSE2 &&
-                (typePairInSet(0, 1, {{s64, s32}})(Query) ||
-                 (Is64Bit && typePairInSet(0, 1, {{s64, s64}})(Query))));
-      })
-      .clampScalar(1, s32, sMaxScalar)
+      .legalFor(HasSSE1, {{s32, s32}})
+      .legalFor(HasSSE1 && Is64Bit, {{s32, s64}})
+      .legalFor(HasSSE2, {{s64, s32}})
+      .legalFor(HasSSE2 && Is64Bit, {{s64, s64}})
+      .clampScalar(1, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)
       .widenScalarToNextPow2(1)
+      .customForCartesianProduct(UseX87, {s32, s64, s80}, {s16, s32, s64})
       .clampScalar(0, s32, HasSSE2 ? s64 : s32)
       .widenScalarToNextPow2(0);
 
   getActionDefinitionsBuilder(G_FPTOSI)
-      .legalIf([=](const LegalityQuery &Query) {
-        return (HasSSE1 &&
-                (typePairInSet(0, 1, {{s32, s32}})(Query) ||
-                 (Is64Bit && typePairInSet(0, 1, {{s64, s32}})(Query)))) ||
-               (HasSSE2 &&
-                (typePairInSet(0, 1, {{s32, s64}})(Query) ||
-                 (Is64Bit && typePairInSet(0, 1, {{s64, s64}})(Query))));
-      })
-      .clampScalar(1, s32, HasSSE2 ? s64 : s32)
+      .legalFor(HasSSE1, {{s32, s32}})
+      .legalFor(HasSSE1 && Is64Bit, {{s64, s32}})
+      .legalFor(HasSSE2, {{s32, s64}})
+      .legalFor(HasSSE2 && Is64Bit, {{s64, s64}})
+      .clampScalar(0, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)
       .widenScalarToNextPow2(0)
-      .clampScalar(0, s32, sMaxScalar)
+      .customForCartesianProduct(UseX87, {s16, s32, s64}, {s32, s64, s80})
+      .clampScalar(1, s32, HasSSE2 ? s64 : s32)
       .widenScalarToNextPow2(1);
 
   // For G_UITOFP and G_FPTOUI without AVX512, we have to custom legalize types
@@ -671,10 +665,70 @@ bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
     return legalizeUITOFP(MI, MRI, Helper);
   case TargetOpcode::G_STORE:
     return legalizeNarrowingStore(MI, MRI, Helper);
+  case TargetOpcode::G_SITOFP:
+    return legalizeSITOFP(MI, MRI, Helper);
+  case TargetOpcode::G_FPTOSI:
+    return legalizeFPTOSI(MI, MRI, Helper);
   }
   llvm_unreachable("expected switch to return");
 }
 
+bool X86LegalizerInfo::legalizeSITOFP(MachineInstr &MI,
+                                      MachineRegisterInfo &MRI,
+                                      LegalizerHelper &Helper) const {
+  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
+  MachineFunction &MF = *MI.getMF();
+  auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+
+  assert((SrcTy.getSizeInBits() == 16 || SrcTy.getSizeInBits() == 32 ||
+          SrcTy.getSizeInBits() == 64) &&
+         "Unexpected source type for SITOFP in X87 mode.");
+
+  TypeSize MemSize = SrcTy.getSizeInBytes();
+  MachinePointerInfo PtrInfo;
+  Align Alignmt = Helper.getStackTemporaryAlignment(SrcTy);
+  auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);
+  MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));
+
+  // Store the integer value on the FPU stack.
+  MIRBuilder.buildStore(Src, SlotPointer, *StoreMMO);
+
+  MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOLoad, MemSize, Align(MemSize));
+  MIRBuilder.buildInstr(X86::G_FILD)
+      .addDef(Dst)
+      .addUse(SlotPointer.getReg(0))
+      .addMemOperand(LoadMMO);
+
+  MI.eraseFromParent();
+  return true;
+}
+
+bool X86LegalizerInfo::legalizeFPTOSI(MachineInstr &MI,
+                                      MachineRegisterInfo &MRI,
+                                      LegalizerHelper &Helper) const {
+  MachineFunction &MF = *MI.getMF();
+  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
+  auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+
+  TypeSize MemSize = DstTy.getSizeInBytes();
+  MachinePointerInfo PtrInfo;
+  Align Alignmt = Helper.getStackTemporaryAlignment(DstTy);
+  auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);
+  MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));
+
+  MIRBuilder.buildInstr(X86::G_FIST)
+      .addUse(Src)
+      .addUse(SlotPointer.getReg(0))
+      .addMemOperand(StoreMMO);
+
+  MIRBuilder.buildLoad(Dst, SlotPointer, PtrInfo, Align(MemSize));
+  MI.eraseFromParent();
+  return true;
+}
+
 bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,
                                            MachineRegisterInfo &MRI,
                                            LegalizerHelper &Helper) const {
diff --git a/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h b/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
index 54f776456397..1ba82674ed4c 100644
--- a/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
+++ b/external/llvm-project/llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
@@ -48,6 +48,12 @@ class X86LegalizerInfo : public LegalizerInfo {
 
   bool legalizeNarrowingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
                               LegalizerHelper &Helper) const;
+
+  bool legalizeSITOFP(MachineInstr &MI, MachineRegisterInfo &MRI,
+                      LegalizerHelper &Helper) const;
+
+  bool legalizeFPTOSI(MachineInstr &MI, MachineRegisterInfo &MRI,
+                      LegalizerHelper &Helper) const;
 };
 } // namespace llvm
 #endif
diff --git a/external/llvm-project/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp b/external/llvm-project/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
index 0baca8149469..74fe5fcaab6c 100644
--- a/external/llvm-project/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
@@ -111,6 +111,7 @@ bool X86RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
   case TargetOpcode::G_FPTOSI:
   case TargetOpcode::G_FPTOUI:
   case TargetOpcode::G_FCMP:
+  case X86::G_FIST:
   case TargetOpcode::G_LROUND:
   case TargetOpcode::G_LLROUND:
   case TargetOpcode::G_INTRINSIC_TRUNC:
@@ -129,6 +130,7 @@ bool X86RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
   switch (MI.getOpcode()) {
   case TargetOpcode::G_SITOFP:
   case TargetOpcode::G_UITOFP:
+  case X86::G_FILD:
     return true;
   default:
     break;
@@ -296,6 +298,16 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     // VECRReg)
     getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ true, OpRegBankIdx);
     break;
+  case X86::G_FIST:
+  case X86::G_FILD: {
+    auto &Op0 = MI.getOperand(0);
+    auto &Op1 = MI.getOperand(1);
+    const LLT Ty0 = MRI.getType(Op0.getReg());
+    const LLT Ty1 = MRI.getType(Op1.getReg());
+    OpRegBankIdx[0] = getPartialMappingIdx(MI, Ty0, /* isFP= */ true);
+    OpRegBankIdx[1] = getPartialMappingIdx(MI, Ty1, /* isFP= */ false);
+    break;
+  }
   case TargetOpcode::G_SITOFP:
   case TargetOpcode::G_FPTOSI:
   case TargetOpcode::G_UITOFP:
diff --git a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
index cd89b88f4619..6614eea3901b 100644
--- a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
@@ -14,6 +14,7 @@
 #include "X86ATTInstPrinter.h"
 #include "X86BaseInfo.h"
 #include "X86InstComments.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrAnalysis.h"
@@ -414,7 +415,7 @@ void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     assert(Op.isExpr() && "unknown operand kind in printOperand");
     WithMarkup M = markup(O, Markup::Immediate);
     O << '$';
-    Op.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -445,7 +446,7 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op,
       O << formatImm(DispVal);
   } else {
     assert(DispSpec.isExpr() && "non-immediate displacement for LEA?");
-    DispSpec.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *DispSpec.getExpr());
   }
 
   if (IndexReg.getReg() || BaseReg.getReg()) {
@@ -500,7 +501,7 @@ void X86ATTInstPrinter::printMemOffset(const MCInst *MI, unsigned Op,
     O << formatImm(DispSpec.getImm());
   } else {
     assert(DispSpec.isExpr() && "non-immediate displacement?");
-    DispSpec.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *DispSpec.getExpr());
   }
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index ac60543e50ef..2d0f021eac5a 100644
--- a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -194,7 +194,7 @@ class X86AsmBackend : public MCAsmBackend {
   bool padInstructionEncoding(MCRelaxableFragment &RF, MCCodeEmitter &Emitter,
                               unsigned &RemainingSize) const;
 
-  void finishLayout(const MCAssembler &Asm) const override;
+  bool finishLayout(const MCAssembler &Asm) const override;
 
   unsigned getMaximumNopSize(const MCSubtargetInfo &STI) const override;
 
@@ -863,7 +863,7 @@ bool X86AsmBackend::padInstructionEncoding(MCRelaxableFragment &RF,
   return Changed;
 }
 
-void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
+bool X86AsmBackend::finishLayout(const MCAssembler &Asm) const {
   // See if we can further relax some instructions to cut down on the number of
   // nop bytes required for code alignment.  The actual win is in reducing
   // instruction count, not number of bytes.  Modern X86-64 can easily end up
@@ -871,7 +871,7 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
   // (i.e. eliminate nops) even at the cost of increasing the size and
   // complexity of others.
   if (!X86PadForAlign && !X86PadForBranchAlign)
-    return;
+    return false;
 
   // The processed regions are delimitered by LabeledFragments. -g may have more
   // MCSymbols and therefore different relaxation results. X86PadForAlign is
@@ -880,6 +880,7 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
   for (const MCSymbol &S : Asm.symbols())
     LabeledFragments.insert(S.getFragment());
 
+  bool Changed = false;
   for (MCSection &Sec : Asm) {
     if (!Sec.isText())
       continue;
@@ -928,8 +929,7 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
         // Give the backend a chance to play any tricks it wishes to increase
         // the encoding size of the given instruction.  Target independent code
         // will try further relaxation, but target's may play further tricks.
-        if (padInstructionEncoding(RF, Asm.getEmitter(), RemainingSize))
-          Sec.setHasLayout(false);
+        Changed |= padInstructionEncoding(RF, Asm.getEmitter(), RemainingSize);
 
         // If we have an instruction which hasn't been fully relaxed, we can't
         // skip past it and insert bytes before it.  Changing its starting
@@ -942,14 +942,12 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
       }
       Relaxable.clear();
 
-      // BoundaryAlign explicitly tracks it's size (unlike align)
-      if (F.getKind() == MCFragment::FT_BoundaryAlign)
-        cast<MCBoundaryAlignFragment>(F).setSize(RemainingSize);
-
       // If we're looking at a boundary align, make sure we don't try to pad
       // its target instructions for some following directive.  Doing so would
       // break the alignment of the current boundary align.
       if (auto *BF = dyn_cast<MCBoundaryAlignFragment>(&F)) {
+        cast<MCBoundaryAlignFragment>(F).setSize(RemainingSize);
+        Changed = true;
         const MCFragment *LastFragment = BF->getLastFragment();
         if (!LastFragment)
           continue;
@@ -959,9 +957,7 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm) const {
     }
   }
 
-  // The layout is done. Mark every fragment as valid.
-  for (MCSection &Section : Asm)
-    Asm.getFragmentOffset(*Section.curFragList()->Tail);
+  return Changed;
 }
 
 unsigned X86AsmBackend::getMaximumNopSize(const MCSubtargetInfo &STI) const {
diff --git a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
index 4c26fc86f954..547745fdba9d 100644
--- a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
@@ -931,10 +931,18 @@ static bool printPTERNLOGComments(const MCInst *MI, raw_ostream &OS,
     // dest, src1, mask, src2, memory, tbl
     CASE_PTERNLOG(PTERNLOGD, m)
     CASE_PTERNLOG(PTERNLOGQ, m)
+    Src2Idx = NumOperands - 7;
+    Src3Idx = -1;
+    break;
+
     CASE_PTERNLOG(PTERNLOGD, mb)
+    Src2Idx = NumOperands - 7;
+    Src3Idx = -2;
+    break;
+
     CASE_PTERNLOG(PTERNLOGQ, mb)
     Src2Idx = NumOperands - 7;
-    Src3Idx = -1;
+    Src3Idx = -3;
     break;
 
   default:
@@ -943,8 +951,21 @@ static bool printPTERNLOGComments(const MCInst *MI, raw_ostream &OS,
   StringRef DestName = getRegName(MI->getOperand(0).getReg());
   StringRef Src1Name = getRegName(MI->getOperand(1).getReg());
   StringRef Src2Name = getRegName(MI->getOperand(Src2Idx).getReg());
-  StringRef Src3Name =
-      Src3Idx != -1 ? getRegName(MI->getOperand(Src3Idx).getReg()) : "mem";
+  StringRef Src3Name;
+  switch (Src3Idx) {
+  case -1:
+    Src3Name = "mem";
+    break;
+  case -2:
+    Src3Name = "m32bcst";
+    break;
+  case -3:
+    Src3Name = "m64bcst";
+    break;
+  default:
+    Src3Name = getRegName(MI->getOperand(Src3Idx).getReg());
+    break;
+  }
   uint8_t TruthTable = MI->getOperand(NumOperands - 1).getImm();
 
   StringRef SrcNames[] = {Src1Name, Src2Name, Src3Name};
diff --git a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp
index 8470d26011cd..7523d2aedcce 100644
--- a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp
@@ -374,7 +374,7 @@ void X86InstPrinterCommon::printPCRelImm(const MCInst *MI, uint64_t Address,
       markup(O, Markup::Immediate) << formatHex((uint64_t)Address);
     } else {
       // Otherwise, just print the expression.
-      Op.getExpr()->print(O, &MAI);
+      MAI.printExpr(O, *Op.getExpr());
     }
   }
 }
diff --git a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
index 0d92609b3a63..b8e117be465e 100644
--- a/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
@@ -14,6 +14,7 @@
 #include "X86IntelInstPrinter.h"
 #include "X86BaseInfo.h"
 #include "X86InstComments.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrAnalysis.h"
@@ -372,7 +373,7 @@ void X86IntelInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
     O << "offset ";
-    Op.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -415,7 +416,7 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op,
   if (!DispSpec.isImm()) {
     if (NeedPlus) O << " + ";
     assert(DispSpec.isExpr() && "non-immediate displacement for LEA?");
-    DispSpec.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *DispSpec.getExpr());
   } else {
     int64_t DispVal = DispSpec.getImm();
     if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) {
@@ -470,7 +471,7 @@ void X86IntelInstPrinter::printMemOffset(const MCInst *MI, unsigned Op,
     markup(O, Markup::Immediate) << formatImm(DispSpec.getImm());
   } else {
     assert(DispSpec.isExpr() && "non-immediate displacement?");
-    DispSpec.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *DispSpec.getExpr());
   }
 
   O << ']';
@@ -479,7 +480,7 @@ void X86IntelInstPrinter::printMemOffset(const MCInst *MI, unsigned Op,
 void X86IntelInstPrinter::printU8Imm(const MCInst *MI, unsigned Op,
                                      raw_ostream &O) {
   if (MI->getOperand(Op).isExpr())
-    return MI->getOperand(Op).getExpr()->print(O, &MAI);
+    return MAI.printExpr(O, *MI->getOperand(Op).getExpr());
 
   markup(O, Markup::Immediate) << formatImm(MI->getOperand(Op).getImm() & 0xff);
 }
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86.h b/external/llvm-project/llvm/lib/Target/X86/X86.h
index be2ddac35cab..6261fadf10a7 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86.h
+++ b/external/llvm-project/llvm/lib/Target/X86/X86.h
@@ -73,9 +73,6 @@ FunctionPass *createX86OptimizeLEAs();
 /// Return a pass that transforms setcc + movzx pairs into xor + setcc.
 FunctionPass *createX86FixupSetCC();
 
-/// Return a pass that transform inline buffer security check into seperate bb
-FunctionPass *createX86WinFixupBufferSecurityCheckPass();
-
 /// Return a pass that avoids creating store forward block issues in the hardware.
 FunctionPass *createX86AvoidStoreForwardingBlocks();
 
@@ -195,7 +192,6 @@ void initializeX86ExpandPseudoPass(PassRegistry &);
 void initializeX86FastPreTileConfigPass(PassRegistry &);
 void initializeX86FastTileConfigPass(PassRegistry &);
 void initializeX86FixupSetCCPassPass(PassRegistry &);
-void initializeX86WinFixupBufferSecurityCheckPassPass(PassRegistry &);
 void initializeX86FlagsCopyLoweringPassPass(PassRegistry &);
 void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &);
 void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &);
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86CompressEVEX.cpp b/external/llvm-project/llvm/lib/Target/X86/X86CompressEVEX.cpp
index fe593aa307df..4ea30de78402 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86CompressEVEX.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86CompressEVEX.cpp
@@ -300,6 +300,7 @@ static bool CompressEVEXImpl(MachineInstr &MI, const X86Subtarget &ST) {
 }
 
 bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "Start X86CompressEVEXPass\n";);
 #ifndef NDEBUG
   // Make sure the tables are sorted.
   static std::atomic<bool> TableChecked(false);
@@ -320,7 +321,7 @@ bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) {
     for (MachineInstr &MI : MBB)
       Changed |= CompressEVEXImpl(MI, ST);
   }
-
+  LLVM_DEBUG(dbgs() << "End X86CompressEVEXPass\n";);
   return Changed;
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/external/llvm-project/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 7f06e0f3a38b..c7abb367fad2 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -301,8 +301,9 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
       X86FL->emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue=*/true);
     }
 
+    // Use this predicate to set REX prefix for X86_64 targets.
+    bool IsX64 = STI->isTargetWin64() || STI->isTargetUEFI64();
     // Jump to label or value in register.
-    bool IsWin64 = STI->isTargetWin64();
     if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||
         Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {
       unsigned Op;
@@ -341,7 +342,7 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
     } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) {
       unsigned Op = (Opcode == X86::TCRETURNmi)
                         ? X86::TAILJMPm
-                        : (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
+                        : (IsX64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
       MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
       for (unsigned i = 0; i != X86::AddrNumOperands; ++i)
         MIB.add(MBBI->getOperand(i));
@@ -349,10 +350,10 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
                (Opcode == X86::TCRETURNri64_ImpCall)) {
       JumpTarget.setIsKill();
       BuildMI(MBB, MBBI, DL,
-              TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
+              TII->get(IsX64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
           .add(JumpTarget);
     } else {
-      assert(!IsWin64 && "Win64 requires REX for indirect jumps.");
+      assert(!IsX64 && "Win64 and UEFI64 require REX for indirect jumps.");
       JumpTarget.setIsKill();
       BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr))
           .add(JumpTarget);
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86FixupInstTuning.cpp b/external/llvm-project/llvm/lib/Target/X86/X86FixupInstTuning.cpp
index 6bb7600dedca..33dc0a232815 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86FixupInstTuning.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86FixupInstTuning.cpp
@@ -81,6 +81,7 @@ bool X86FixupInstTuningPass::processInstruction(
   MachineInstr &MI = *I;
   unsigned Opc = MI.getOpcode();
   unsigned NumOperands = MI.getDesc().getNumOperands();
+  bool OptSize = MF.getFunction().hasOptSize();
 
   auto GetInstTput = [&](unsigned Opcode) -> std::optional<double> {
     // We already checked that SchedModel exists in `NewOpcPreferable`.
@@ -131,11 +132,15 @@ bool X86FixupInstTuningPass::processInstruction(
   auto ProcessVPERMILPDri = [&](unsigned NewOpc) -> bool {
     if (!NewOpcPreferable(NewOpc))
       return false;
-    unsigned MaskImm = MI.getOperand(NumOperands - 1).getImm();
-    MI.removeOperand(NumOperands - 1);
-    MI.addOperand(MI.getOperand(NumOperands - 2));
-    MI.setDesc(TII->get(NewOpc));
-    MI.addOperand(MachineOperand::CreateImm(MaskImm));
+    LLVM_DEBUG(dbgs() << "Replacing: " << MI);
+    {
+      unsigned MaskImm = MI.getOperand(NumOperands - 1).getImm();
+      MI.removeOperand(NumOperands - 1);
+      MI.addOperand(MI.getOperand(NumOperands - 2));
+      MI.setDesc(TII->get(NewOpc));
+      MI.addOperand(MachineOperand::CreateImm(MaskImm));
+    }
+    LLVM_DEBUG(dbgs() << "     With: " << MI);
     return true;
   };
 
@@ -146,11 +151,15 @@ bool X86FixupInstTuningPass::processInstruction(
   auto ProcessVPERMILPSri = [&](unsigned NewOpc) -> bool {
     if (!NewOpcPreferable(NewOpc))
       return false;
-    unsigned MaskImm = MI.getOperand(NumOperands - 1).getImm();
-    MI.removeOperand(NumOperands - 1);
-    MI.addOperand(MI.getOperand(NumOperands - 2));
-    MI.setDesc(TII->get(NewOpc));
-    MI.addOperand(MachineOperand::CreateImm(MaskImm));
+    LLVM_DEBUG(dbgs() << "Replacing: " << MI);
+    {
+      unsigned MaskImm = MI.getOperand(NumOperands - 1).getImm();
+      MI.removeOperand(NumOperands - 1);
+      MI.addOperand(MI.getOperand(NumOperands - 2));
+      MI.setDesc(TII->get(NewOpc));
+      MI.addOperand(MachineOperand::CreateImm(MaskImm));
+    }
+    LLVM_DEBUG(dbgs() << "     With: " << MI);
     return true;
   };
 
@@ -163,7 +172,11 @@ bool X86FixupInstTuningPass::processInstruction(
     if (!ST->hasNoDomainDelayShuffle() ||
         !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
       return false;
-    MI.setDesc(TII->get(NewOpc));
+    LLVM_DEBUG(dbgs() << "Replacing: " << MI);
+    {
+      MI.setDesc(TII->get(NewOpc));
+    }
+    LLVM_DEBUG(dbgs() << "     With: " << MI);
     return true;
   };
 
@@ -184,9 +197,12 @@ bool X86FixupInstTuningPass::processInstruction(
   auto ProcessUNPCK = [&](unsigned NewOpc, unsigned MaskImm) -> bool {
     if (!NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
       return false;
-
-    MI.setDesc(TII->get(NewOpc));
-    MI.addOperand(MachineOperand::CreateImm(MaskImm));
+    LLVM_DEBUG(dbgs() << "Replacing: " << MI);
+    {
+      MI.setDesc(TII->get(NewOpc));
+      MI.addOperand(MachineOperand::CreateImm(MaskImm));
+    }
+    LLVM_DEBUG(dbgs() << "     With: " << MI);
     return true;
   };
 
@@ -197,7 +213,11 @@ bool X86FixupInstTuningPass::processInstruction(
     if (!ST->hasNoDomainDelayShuffle() ||
         !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
       return false;
-    MI.setDesc(TII->get(NewOpc));
+    LLVM_DEBUG(dbgs() << "Replacing: " << MI);
+    {
+      MI.setDesc(TII->get(NewOpc));
+    }
+    LLVM_DEBUG(dbgs() << "     With: " << MI);
     return true;
   };
 
@@ -222,7 +242,60 @@ bool X86FixupInstTuningPass::processInstruction(
     return ProcessUNPCKToIntDomain(NewOpc);
   };
 
+  auto ProcessBLENDWToBLENDD = [&](unsigned MovOpc, unsigned NumElts) -> bool {
+    if (!ST->hasAVX2() || !NewOpcPreferable(MovOpc))
+      return false;
+    // Convert to VPBLENDD if scaling the VPBLENDW mask down/up loses no bits.
+    APInt MaskW =
+        APInt(8, MI.getOperand(NumOperands - 1).getImm(), /*IsSigned=*/false);
+    APInt MaskD = APIntOps::ScaleBitMask(MaskW, 4, /*MatchAllBits=*/true);
+    if (MaskW != APIntOps::ScaleBitMask(MaskD, 8, /*MatchAllBits=*/true))
+      return false;
+    APInt NewMaskD = APInt::getSplat(NumElts, MaskD);
+    LLVM_DEBUG(dbgs() << "Replacing: " << MI);
+    {
+      MI.setDesc(TII->get(MovOpc));
+      MI.removeOperand(NumOperands - 1);
+      MI.addOperand(MachineOperand::CreateImm(NewMaskD.getZExtValue()));
+    }
+    LLVM_DEBUG(dbgs() << "     With: " << MI);
+    return true;
+  };
+
+  auto ProcessBLENDToMOV = [&](unsigned MovOpc, unsigned Mask,
+                               unsigned MovImm) -> bool {
+    if ((MI.getOperand(NumOperands - 1).getImm() & Mask) != MovImm)
+      return false;
+    if (!OptSize && !NewOpcPreferable(MovOpc))
+      return false;
+    LLVM_DEBUG(dbgs() << "Replacing: " << MI);
+    {
+      MI.setDesc(TII->get(MovOpc));
+      MI.removeOperand(NumOperands - 1);
+    }
+    LLVM_DEBUG(dbgs() << "     With: " << MI);
+    return true;
+  };
+
   switch (Opc) {
+  case X86::BLENDPDrri:
+    return ProcessBLENDToMOV(X86::MOVSDrr, 0x3, 0x1);
+  case X86::VBLENDPDrri:
+    return ProcessBLENDToMOV(X86::VMOVSDrr, 0x3, 0x1);
+
+  case X86::BLENDPSrri:
+    return ProcessBLENDToMOV(X86::MOVSSrr, 0xF, 0x1) ||
+           ProcessBLENDToMOV(X86::MOVSDrr, 0xF, 0x3);
+  case X86::VBLENDPSrri:
+    return ProcessBLENDToMOV(X86::VMOVSSrr, 0xF, 0x1) ||
+           ProcessBLENDToMOV(X86::VMOVSDrr, 0xF, 0x3);
+
+  case X86::VPBLENDWrri:
+    // TODO: Add X86::VPBLENDWrmi handling
+    // TODO: Add X86::VPBLENDWYrri handling
+    // TODO: Add X86::VPBLENDWYrmi handling
+    return ProcessBLENDWToBLENDD(X86::VPBLENDDrri, 4);
+
   case X86::VPERMILPDri:
     return ProcessVPERMILPDri(X86::VSHUFPDrri);
   case X86::VPERMILPDYri:
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86FrameLowering.cpp b/external/llvm-project/llvm/lib/Target/X86/X86FrameLowering.cpp
index 75f49beee27c..42a09106b3a3 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -2915,13 +2915,14 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
   // 1. Use push2 when
   //       a) number of CSR > 1 if no need padding
   //       b) number of CSR > 2 if need padding
+  //       c) stack alignment >= 16 bytes
   // 2. When the number of CSR push is odd
   //    a. Start to use push2 from the 1st push if stack is 16B aligned.
   //    b. Start to use push2 from the 2nd push if stack is not 16B aligned.
   // 3. When the number of CSR push is even, start to use push2 from the 1st
   //    push and make the stack 16B aligned before the push
   unsigned NumRegsForPush2 = 0;
-  if (STI.hasPush2Pop2()) {
+  if (STI.hasPush2Pop2() && getStackAlignment() >= 16) {
     unsigned NumCSGPR = llvm::count_if(CSI, [](const CalleeSavedInfo &I) {
       return X86::GR64RegClass.contains(I.getReg());
     });
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp b/external/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp
index b1a3e3c006bb..290fad07be4f 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -166,26 +166,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       addBypassSlowDiv(64, 32);
   }
 
-  // Setup Windows compiler runtime calls.
-  if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) {
-    static const struct {
-      const RTLIB::Libcall Op;
-      const char * const Name;
-      const CallingConv::ID CC;
-    } LibraryCalls[] = {
-      { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall },
-      { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall },
-      { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall },
-      { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall },
-      { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall },
-    };
-
-    for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
-      setLibcallCallingConv(LC.Op, LC.CC);
-    }
-  }
-
   if (Subtarget.canUseCMPXCHG16B())
     setMaxAtomicSizeInBitsSupported(128);
   else if (Subtarget.canUseCMPXCHG8B())
@@ -532,6 +512,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
+
+  // FIXME: This should be set in RuntimeLibcallsInfo
   if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
     setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
 
@@ -2238,7 +2220,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i8, Custom);
     setOperationAction(ISD::ANY_EXTEND,         MVT::v32i8, Custom);
 
-    for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
+    for (auto VT : {MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16,
+                    MVT::v16f16, MVT::v8f16}) {
       setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);
       setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
     }
@@ -2371,6 +2354,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::LLRINT, MVT::v8f16, Legal);
     }
 
+    setOperationAction(ISD::FP_TO_SINT,         MVT::v8i16, Custom);
+    setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v8i16, Custom);
+    setOperationAction(ISD::FP_TO_UINT,         MVT::v8i16, Custom);
+    setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i16, Custom);
+
     if (Subtarget.hasVLX()) {
       setGroup(MVT::v8f16);
       setGroup(MVT::v16f16);
@@ -2386,10 +2374,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::UINT_TO_FP,         MVT::v8i16,  Legal);
       setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v8i16,  Legal);
 
-      setOperationAction(ISD::FP_TO_SINT,         MVT::v8i16, Custom);
-      setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v8i16, Custom);
-      setOperationAction(ISD::FP_TO_UINT,         MVT::v8i16, Custom);
-      setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i16, Custom);
       setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Legal);
       setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v8f16, Legal);
       setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Custom);
@@ -4152,8 +4136,7 @@ static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
       isNullConstant(Vec.getOperand(2)))
     return DAG.getUNDEF(ResultVT);
 
-  SDValue VecIdx = DAG.getVectorIdxConstant(IdxVal, dl);
-  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
+  return DAG.getExtractSubvector(dl, ResultVT, Vec, IdxVal);
 }
 
 /// Generate a DAG to grab 128-bits from a vector > 128 bits.  This
@@ -4165,7 +4148,8 @@ static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
 static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
                                    SelectionDAG &DAG, const SDLoc &dl) {
   assert((Vec.getValueType().is256BitVector() ||
-          Vec.getValueType().is512BitVector()) && "Unexpected vector size!");
+          Vec.getValueType().is512BitVector()) &&
+         "Unexpected vector size!");
   return extractSubVector(Vec, IdxVal, DAG, dl, 128);
 }
 
@@ -4184,20 +4168,16 @@ static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
   // Inserting UNDEF is Result
   if (Vec.isUndef())
     return Result;
-  EVT VT = Vec.getValueType();
-  EVT ElVT = VT.getVectorElementType();
-  EVT ResultVT = Result.getValueType();
 
   // Insert the relevant vectorWidth bits.
-  unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
+  EVT VT = Vec.getValueType();
+  unsigned ElemsPerChunk = vectorWidth / VT.getScalarSizeInBits();
   assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
 
   // This is the index of the first element of the vectorWidth-bit chunk
   // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
   IdxVal &= ~(ElemsPerChunk - 1);
-
-  SDValue VecIdx = DAG.getVectorIdxConstant(IdxVal, dl);
-  return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
+  return DAG.getInsertSubvector(dl, Result, Vec, IdxVal);
 }
 
 /// Generate a DAG to put 128-bits into a vector > 128 bits.  This
@@ -4233,8 +4213,7 @@ static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
   }
   SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
                                 : DAG.getUNDEF(VT);
-  return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec,
-                     DAG.getVectorIdxConstant(0, dl));
+  return DAG.getInsertSubvector(dl, Res, Vec, 0);
 }
 
 /// Widen a vector to a larger size with the same scalar type, with the new
@@ -4332,6 +4311,26 @@ static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops,
     }
   }
 
+  if (N->getOpcode() == ISD::EXTRACT_SUBVECTOR) {
+    EVT VT = N->getValueType(0);
+    SDValue Src = N->getOperand(0);
+    uint64_t Idx = N->getConstantOperandVal(1);
+
+    // Collect all the subvectors from the source vector and slice off the
+    // extraction.
+    SmallVector<SDValue, 4> SrcOps;
+    if (collectConcatOps(Src.getNode(), SrcOps, DAG) &&
+        VT.getSizeInBits() > SrcOps[0].getValueSizeInBits() &&
+        (VT.getSizeInBits() % SrcOps[0].getValueSizeInBits()) == 0 &&
+        (Idx % SrcOps[0].getValueType().getVectorNumElements()) == 0) {
+      unsigned SubIdx = Idx / SrcOps[0].getValueType().getVectorNumElements();
+      unsigned NumSubs = VT.getSizeInBits() / SrcOps[0].getValueSizeInBits();
+      Ops.append(SrcOps.begin() + SubIdx, SrcOps.begin() + SubIdx + NumSubs);
+      return true;
+    }
+  }
+
+  assert(Ops.empty() && "Expected an empty ops vector");
   return false;
 }
 
@@ -4371,13 +4370,13 @@ static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
   assert((NumElems % 2) == 0 && (SizeInBits % 2) == 0 &&
          "Can't split odd sized vector");
 
-  if (Op.getOpcode() == ISD::CONCAT_VECTORS) {
-    assert((Op.getNumOperands() % 2) == 0 &&
-           "Can't split odd sized vector concat");
-    unsigned HalfOps = Op.getNumOperands() / 2;
+  SmallVector<SDValue, 4> SubOps;
+  if (collectConcatOps(Op.getNode(), SubOps, DAG)) {
+    assert((SubOps.size() % 2) == 0 && "Can't split odd sized vector concat");
+    unsigned HalfOps = SubOps.size() / 2;
     EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
-    SmallVector<SDValue, 2> LoOps(Op->op_begin(), Op->op_begin() + HalfOps);
-    SmallVector<SDValue, 2> HiOps(Op->op_begin() + HalfOps, Op->op_end());
+    SmallVector<SDValue, 2> LoOps(SubOps.begin(), SubOps.begin() + HalfOps);
+    SmallVector<SDValue, 2> HiOps(SubOps.begin() + HalfOps, SubOps.end());
     SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, LoOps);
     SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, HiOps);
     return std::make_pair(Lo, Hi);
@@ -5243,25 +5242,25 @@ static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
   }
 
   // Extract constant bits from a subvector's source.
-  if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
-    // TODO - support extract_subvector through bitcasts.
-    if (EltSizeInBits != VT.getScalarSizeInBits())
-      return false;
-
-    if (getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
-                                      UndefElts, EltBits, AllowWholeUndefs,
-                                      AllowPartialUndefs)) {
-      EVT SrcVT = Op.getOperand(0).getValueType();
-      unsigned NumSrcElts = SrcVT.getVectorNumElements();
-      unsigned NumSubElts = VT.getVectorNumElements();
-      unsigned BaseIdx = Op.getConstantOperandVal(1);
-      UndefElts = UndefElts.extractBits(NumSubElts, BaseIdx);
-      if ((BaseIdx + NumSubElts) != NumSrcElts)
-        EltBits.erase(EltBits.begin() + BaseIdx + NumSubElts, EltBits.end());
-      if (BaseIdx != 0)
-        EltBits.erase(EltBits.begin(), EltBits.begin() + BaseIdx);
-      return true;
-    }
+  if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits, UndefElts,
+                                    EltBits, AllowWholeUndefs,
+                                    AllowPartialUndefs)) {
+    EVT SrcVT = Op.getOperand(0).getValueType();
+    unsigned NumSrcElts = SrcVT.getSizeInBits() / EltSizeInBits;
+    unsigned NumSubElts = VT.getSizeInBits() / EltSizeInBits;
+    unsigned BaseOfs = Op.getConstantOperandVal(1) * VT.getScalarSizeInBits();
+    unsigned BaseIdx = BaseOfs / EltSizeInBits;
+    assert((SrcVT.getSizeInBits() % EltSizeInBits) == 0 &&
+           (VT.getSizeInBits() % EltSizeInBits) == 0 &&
+           (BaseOfs % EltSizeInBits) == 0 && "Bad subvector index");
+
+    UndefElts = UndefElts.extractBits(NumSubElts, BaseIdx);
+    if ((BaseIdx + NumSubElts) != NumSrcElts)
+      EltBits.erase(EltBits.begin() + BaseIdx + NumSubElts, EltBits.end());
+    if (BaseIdx != 0)
+      EltBits.erase(EltBits.begin(), EltBits.begin() + BaseIdx);
+    return true;
   }
 
   // Extract constant bits from shuffle node sources.
@@ -9615,13 +9614,12 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
 // 256-bit AVX can use the vinsertf128 instruction
 // to create 256-bit vectors from two other 128-bit ones.
 // TODO: Detect subvector broadcast here instead of DAG combine?
-static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
+static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, const SDLoc &dl,
+                                      SelectionDAG &DAG,
                                       const X86Subtarget &Subtarget) {
-  SDLoc dl(Op);
   MVT ResVT = Op.getSimpleValueType();
-
-  assert((ResVT.is256BitVector() ||
-          ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");
+  assert((ResVT.is256BitVector() || ResVT.is512BitVector()) &&
+         "Value type must be 256-/512-bit wide");
 
   unsigned NumOperands = Op.getNumOperands();
   unsigned NumFreezeUndef = 0;
@@ -9689,13 +9687,11 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
 // zeros) of the result of a node that already zeros all upper bits of
 // k-register.
 // TODO: Merge this with LowerAVXCONCAT_VECTORS?
-static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
+static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op, const SDLoc &dl,
                                        const X86Subtarget &Subtarget,
                                        SelectionDAG & DAG) {
-  SDLoc dl(Op);
   MVT ResVT = Op.getSimpleValueType();
   unsigned NumOperands = Op.getNumOperands();
-
   assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&
          "Unexpected number of operands in CONCAT_VECTORS");
 
@@ -9767,19 +9763,18 @@ static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
 static SDValue LowerCONCAT_VECTORS(SDValue Op,
                                    const X86Subtarget &Subtarget,
                                    SelectionDAG &DAG) {
+  SDLoc DL(Op);
   MVT VT = Op.getSimpleValueType();
   if (VT.getVectorElementType() == MVT::i1)
-    return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG);
-
-  assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
-         (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
-          Op.getNumOperands() == 4)));
+    return LowerCONCAT_VECTORSvXi1(Op, DL, Subtarget, DAG);
 
   // AVX can use the vinsertf128 instruction to create 256-bit vectors
   // from two other 128-bit ones.
-
   // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
-  return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget);
+  assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
+         (VT.is512BitVector() &&
+          (Op.getNumOperands() == 2 || Op.getNumOperands() == 4)));
+  return LowerAVXCONCAT_VECTORS(Op, DL, DAG, Subtarget);
 }
 
 //===----------------------------------------------------------------------===//
@@ -9803,20 +9798,27 @@ static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
     return false;
 
   EVT VT = Op.getValueType();
+  EVT ExpectedVT = ExpectedOp.getValueType();
+
+  // Sources must be vectors and match the mask's element count.
+  if (!VT.isVector() || !ExpectedVT.isVector() ||
+      (int)VT.getVectorNumElements() != MaskSize ||
+      (int)ExpectedVT.getVectorNumElements() != MaskSize)
+    return false;
+
+  // Exact match.
+  if (Idx == ExpectedIdx && Op == ExpectedOp)
+    return true;
+
   switch (Op.getOpcode()) {
   case ISD::BUILD_VECTOR:
     // If the values are build vectors, we can look through them to find
     // equivalent inputs that make the shuffles equivalent.
-    // TODO: Handle MaskSize != Op.getNumOperands()?
-    if (MaskSize == (int)Op.getNumOperands() &&
-        MaskSize == (int)ExpectedOp.getNumOperands())
-      return Op.getOperand(Idx) == ExpectedOp.getOperand(ExpectedIdx);
-    break;
+    return Op.getOperand(Idx) == ExpectedOp.getOperand(ExpectedIdx);
   case ISD::BITCAST: {
     SDValue Src = peekThroughBitcasts(Op);
     EVT SrcVT = Src.getValueType();
-    if (Op == ExpectedOp && SrcVT.isVector() &&
-        (int)VT.getVectorNumElements() == MaskSize) {
+    if (Op == ExpectedOp && SrcVT.isVector()) {
       if ((SrcVT.getScalarSizeInBits() % VT.getScalarSizeInBits()) == 0) {
         unsigned Scale = SrcVT.getScalarSizeInBits() / VT.getScalarSizeInBits();
         return (Idx % Scale) == (ExpectedIdx % Scale) &&
@@ -9837,21 +9839,29 @@ static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
   }
   case ISD::VECTOR_SHUFFLE: {
     auto *SVN = cast<ShuffleVectorSDNode>(Op);
-    return Op == ExpectedOp && (int)VT.getVectorNumElements() == MaskSize &&
+    return Op == ExpectedOp &&
            SVN->getMaskElt(Idx) == SVN->getMaskElt(ExpectedIdx);
   }
   case X86ISD::VBROADCAST:
   case X86ISD::VBROADCAST_LOAD:
-    // TODO: Handle MaskSize != VT.getVectorNumElements()?
-    return (Op == ExpectedOp && (int)VT.getVectorNumElements() == MaskSize);
+    return Op == ExpectedOp;
   case X86ISD::SUBV_BROADCAST_LOAD:
-    // TODO: Handle MaskSize != VT.getVectorNumElements()?
-    if (Op == ExpectedOp && (int)VT.getVectorNumElements() == MaskSize) {
+    if (Op == ExpectedOp) {
       auto *MemOp = cast<MemSDNode>(Op);
       unsigned NumMemElts = MemOp->getMemoryVT().getVectorNumElements();
       return (Idx % NumMemElts) == (ExpectedIdx % NumMemElts);
     }
     break;
+  case X86ISD::VPERMI: {
+    if (Op == ExpectedOp) {
+      SmallVector<int, 8> Mask;
+      DecodeVPERMMask(MaskSize, Op.getConstantOperandVal(1), Mask);
+      SDValue Src = Op.getOperand(0);
+      return IsElementEquivalent(MaskSize, Src, Src, Mask[Idx],
+                                 Mask[ExpectedIdx]);
+    }
+    break;
+  }
   case X86ISD::HADD:
   case X86ISD::HSUB:
   case X86ISD::FHADD:
@@ -9859,20 +9869,16 @@ static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
   case X86ISD::PACKSS:
   case X86ISD::PACKUS:
     // HOP(X,X) can refer to the elt from the lower/upper half of a lane.
-    // TODO: Handle MaskSize != NumElts?
     // TODO: Handle HOP(X,Y) vs HOP(Y,X) equivalence cases.
     if (Op == ExpectedOp && Op.getOperand(0) == Op.getOperand(1)) {
       int NumElts = VT.getVectorNumElements();
-      if (MaskSize == NumElts) {
-        int NumLanes = VT.getSizeInBits() / 128;
-        int NumEltsPerLane = NumElts / NumLanes;
-        int NumHalfEltsPerLane = NumEltsPerLane / 2;
-        bool SameLane =
-            (Idx / NumEltsPerLane) == (ExpectedIdx / NumEltsPerLane);
-        bool SameElt =
-            (Idx % NumHalfEltsPerLane) == (ExpectedIdx % NumHalfEltsPerLane);
-        return SameLane && SameElt;
-      }
+      int NumLanes = VT.getSizeInBits() / 128;
+      int NumEltsPerLane = NumElts / NumLanes;
+      int NumHalfEltsPerLane = NumEltsPerLane / 2;
+      bool SameLane = (Idx / NumEltsPerLane) == (ExpectedIdx / NumEltsPerLane);
+      bool SameElt =
+          (Idx % NumHalfEltsPerLane) == (ExpectedIdx % NumHalfEltsPerLane);
+      return SameLane && SameElt;
     }
     break;
   }
@@ -14319,9 +14325,9 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
 
   // At this point, each half should contain all its inputs, and we can then
   // just shuffle them into their final position.
-  assert(count_if(LoMask, [](int M) { return M >= 4; }) == 0 &&
+  assert(none_of(LoMask, [](int M) { return M >= 4; }) &&
          "Failed to lift all the high half inputs to the low mask!");
-  assert(count_if(HiMask, [](int M) { return M >= 0 && M < 4; }) == 0 &&
+  assert(none_of(HiMask, [](int M) { return M >= 0 && M < 4; }) &&
          "Failed to lift all the low half inputs to the high mask!");
 
   // Do a half shuffle for the low mask.
@@ -15092,6 +15098,17 @@ static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
   assert(V1.getSimpleValueType() == VT && "Bad operand type!");
   assert(V2.getSimpleValueType() == VT && "Bad operand type!");
 
+  // If this came from the AVX1 v8i32 -> v8f32 bitcast, split using v4i32.
+  if (VT == MVT::v8f32) {
+    SDValue BC1 = peekThroughBitcasts(V1);
+    SDValue BC2 = peekThroughBitcasts(V2);
+    if (BC1.getValueType() == MVT::v8i32 && BC2.getValueType() == MVT::v8i32) {
+      if (SDValue Split = splitAndLowerShuffle(DL, MVT::v8i32, BC1, BC2, Mask,
+                                               DAG, SimpleOnly))
+        return DAG.getBitcast(VT, Split);
+    }
+  }
+
   ArrayRef<int> LoMask = Mask.slice(0, Mask.size() / 2);
   ArrayRef<int> HiMask = Mask.slice(Mask.size() / 2);
 
@@ -18546,7 +18563,9 @@ SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
     return SDValue();
 
   case MVT::v8i16:
-  case MVT::v16i16: {
+  case MVT::v16i16:
+  case MVT::v8f16:
+  case MVT::v16f16: {
     // Bitcast everything to the vXi8 type and use a vXi8 vselect.
     MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2);
     Cond = DAG.getBitcast(CastVT, Cond);
@@ -20010,10 +20029,12 @@ static SDValue promoteXINT_TO_FP(SDValue Op, const SDLoc &dl,
 
 static bool isLegalConversion(MVT VT, MVT FloatVT, bool IsSigned,
                               const X86Subtarget &Subtarget) {
-  if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned)
-    return true;
-  if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned)
-    return true;
+  if (FloatVT.getScalarType() != MVT::f16 || Subtarget.hasVLX()) {
+    if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned)
+      return true;
+    if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned)
+      return true;
+  }
   if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32))
     return true;
   if (Subtarget.useAVX512Regs()) {
@@ -20369,10 +20390,16 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
     if (VT == MVT::v8f64)
       return Op;
 
-    assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&
+    assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64 ||
+            VT == MVT::v8f16) &&
            "Unexpected VT!");
-    MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
-    MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
+    MVT WideVT = VT == MVT::v8f16 ? MVT::v16f16 : MVT::v16f32;
+    MVT WideIntVT = MVT::v16i32;
+    if (VT == MVT::v4f64) {
+      WideVT = MVT::v8f64;
+      WideIntVT = MVT::v8i32;
+    }
+
     // Need to concat with zero vector for strict fp to avoid spurious
     // exceptions.
     SDValue Tmp =
@@ -21552,6 +21579,7 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
   bool IsStrict = Op->isStrictFPOpcode();
   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
                   Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
+  bool HasVLX = Subtarget.hasVLX();
   MVT VT = Op->getSimpleValueType(0);
   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
   SDValue Chain = IsStrict ? Op->getOperand(0) : SDValue();
@@ -21582,7 +21610,7 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
       else
         Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
 
-      if (!IsSigned && !Subtarget.hasVLX()) {
+      if (!IsSigned && !HasVLX) {
         assert(Subtarget.useAVX512Regs() && "Unexpected features!");
         // Widen to 512-bits.
         ResVT = MVT::v8i32;
@@ -21612,7 +21640,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
     }
 
     if (Subtarget.hasFP16() && SrcVT.getVectorElementType() == MVT::f16) {
-      if (VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16)
+      if ((HasVLX && (VT == MVT::v8i16 || VT == MVT::v16i16)) ||
+          VT == MVT::v32i16)
         return Op;
 
       MVT ResVT = VT;
@@ -21620,7 +21649,7 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
       if (EleVT != MVT::i64)
         ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
 
-      if (SrcVT != MVT::v8f16) {
+      if (SrcVT == MVT::v2f16 || SrcVT == MVT::v4f16) {
         SDValue Tmp =
             IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
         SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
@@ -21628,6 +21657,16 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
       }
 
+      if (!HasVLX) {
+        assert(Subtarget.useAVX512Regs() && "Unexpected features!");
+        // Widen to 512-bits.
+        unsigned IntSize = EleVT.getSizeInBits();
+        unsigned Num = IntSize > 16 ? 512 / IntSize : 32;
+        ResVT = MVT::getVectorVT(EleVT, Num);
+        Src = widenSubVector(MVT::getVectorVT(MVT::f16, Num), Src, IsStrict,
+                             Subtarget, DAG, dl);
+      }
+
       if (IsStrict) {
         Res = DAG.getNode(IsSigned ? X86ISD::STRICT_CVTTP2SI
                                    : X86ISD::STRICT_CVTTP2UI,
@@ -21640,7 +21679,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
 
       // TODO: Need to add exception check code for strict FP.
       if (EleVT.getSizeInBits() < 16) {
-        ResVT = MVT::getVectorVT(EleVT, 8);
+        if (HasVLX)
+          ResVT = MVT::getVectorVT(EleVT, 8);
         Res = DAG.getNode(ISD::TRUNCATE, dl, ResVT, Res);
       }
 
@@ -22204,7 +22244,7 @@ SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
     } else {
       In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In);
       In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
-                       DAG.getUNDEF(MVT::v4f32), In,
+                       DAG.getUNDEF(MVT::v4i32), In,
                        DAG.getVectorIdxConstant(0, DL));
       In = DAG.getBitcast(MVT::v8i16, In);
       Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In,
@@ -25519,14 +25559,12 @@ static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
   if (St->isTruncatingStore())
     return SDValue();
 
-  // If this is a 256-bit store of concatenated ops, we are better off splitting
-  // that store into two 128-bit stores. This avoids spurious use of 256-bit ops
-  // and each half can execute independently. Some cores would split the op into
-  // halves anyway, so the concat (vinsertf128) is purely an extra op.
+  // If this is a 256/512-bit store of concatenated ops, we are better off
+  // splitting that store into two half-size stores. This avoids spurious use of
+  // concatenated ops and each half can execute independently. Some cores would
+  // split the op into halves anyway, so the concat is purely an extra op.
   MVT StoreVT = StoredVal.getSimpleValueType();
-  if (StoreVT.is256BitVector() ||
-      ((StoreVT == MVT::v32i16 || StoreVT == MVT::v64i8) &&
-       !Subtarget.hasBWI())) {
+  if (StoreVT.is256BitVector() || StoreVT.is512BitVector()) {
     if (StoredVal.hasOneUse() && isFreeToSplitVector(StoredVal, DAG))
       return splitVectorStore(St, DAG);
     return SDValue();
@@ -28999,6 +29037,30 @@ static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
   return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
 }
 
+static SDValue LowerVectorCTLZ_GFNI(SDValue Op, const SDLoc &DL,
+                                    SelectionDAG &DAG,
+                                    const X86Subtarget &Subtarget) {
+  MVT VT = Op.getSimpleValueType();
+  SDValue Input = Op.getOperand(0);
+
+  assert(VT.isVector() && VT.getVectorElementType() == MVT::i8 &&
+         "Expected vXi8 input for GFNI-based CTLZ lowering");
+
+  SDValue Reversed = DAG.getNode(ISD::BITREVERSE, DL, VT, Input);
+
+  SDValue Neg = DAG.getNegative(Reversed, DL, VT);
+  SDValue Filtered = DAG.getNode(ISD::AND, DL, VT, Reversed, Neg);
+
+  MVT VT64 = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
+  SDValue CTTZConst = DAG.getConstant(0xAACCF0FF00000000ULL, DL, VT64);
+  SDValue CTTZMatrix = DAG.getBitcast(VT, CTTZConst);
+
+  SDValue LZCNT =
+      DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, Filtered, CTTZMatrix,
+                  DAG.getTargetConstant(8, DL, MVT::i8));
+  return LZCNT;
+}
+
 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
                          SelectionDAG &DAG) {
   MVT VT = Op.getSimpleValueType();
@@ -29007,6 +29069,9 @@ static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
   SDLoc dl(Op);
   unsigned Opc = Op.getOpcode();
 
+  if (VT.isVector() && VT.getScalarType() == MVT::i8 && Subtarget.hasGFNI())
+    return LowerVectorCTLZ_GFNI(Op, dl, DAG, Subtarget);
+
   if (VT.isVector())
     return LowerVectorCTLZ(Op, dl, Subtarget, DAG);
 
@@ -33192,8 +33257,8 @@ static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
          "Cannot lower masked load op.");
 
   assert((ScalarVT.getSizeInBits() >= 32 ||
-          (Subtarget.hasBWI() &&
-              (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
+          (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16 ||
+                                  ScalarVT == MVT::f16))) &&
          "Unsupported masked load op.");
 
   // This operation is legal for targets with VLX, but without
@@ -33240,9 +33305,9 @@ static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
          "Cannot lower masked store op.");
 
   assert((ScalarVT.getSizeInBits() >= 32 ||
-          (Subtarget.hasBWI() &&
-              (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
-          "Unsupported masked store op.");
+          (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16 ||
+                                  ScalarVT == MVT::f16))) &&
+         "Unsupported masked store op.");
 
   // This operation is legal for targets with VLX, but without
   // VLX the vector should be widened to 512 bit
@@ -34123,12 +34188,10 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       }
 
       if (IsStrict) {
-        Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
         Res =
             DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
         Chain = Res.getValue(1);
       } else {
-        Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
         Res = DAG.getNode(Opc, dl, ResVT, Src);
       }
 
@@ -35552,8 +35615,29 @@ bool X86TargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
 bool X86TargetLowering::shouldFoldSelectWithIdentityConstant(
     unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
     SDValue Y) const {
-  if (SelectOpcode != ISD::VSELECT)
+  if (SelectOpcode == ISD::SELECT) {
+    if (VT.isVector())
+      return false;
+    if (!Subtarget.hasBMI() || (VT != MVT::i32 && VT != MVT::i64))
+      return false;
+    using namespace llvm::SDPatternMatch;
+    // BLSI
+    if (BinOpcode == ISD::AND && (sd_match(Y, m_Neg(m_Specific(X))) ||
+                                  sd_match(X, m_Neg(m_Specific(Y)))))
+      return true;
+    // BLSR
+    if (BinOpcode == ISD::AND &&
+        (sd_match(Y, m_Add(m_Specific(X), m_AllOnes())) ||
+         sd_match(X, m_Add(m_Specific(Y), m_AllOnes()))))
+      return true;
+    // BLSMSK
+    if (BinOpcode == ISD::XOR &&
+        (sd_match(Y, m_Add(m_Specific(X), m_AllOnes())) ||
+         sd_match(X, m_Add(m_Specific(Y), m_AllOnes()))))
+      return true;
+
     return false;
+  }
   // TODO: This is too general. There are cases where pre-AVX512 codegen would
   //       benefit. The transform may also be profitable for scalar code.
   if (!Subtarget.hasAVX512())
@@ -40627,7 +40711,8 @@ static SDValue combineX86ShuffleChainWithExtract(
         continue;
       }
       if (Input.getOpcode() == ISD::INSERT_SUBVECTOR &&
-          Input.getOperand(0).isUndef()) {
+          Input.getOperand(0).isUndef() &&
+          isNullConstant(Input.getOperand(2))) {
         Input = peekThroughBitcasts(Input.getOperand(1));
         continue;
       }
@@ -41519,8 +41604,8 @@ static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
                                              const X86Subtarget &Subtarget) {
   return combineX86ShufflesRecursively(
       {Op}, 0, Op.getOpcode(), Op.getSimpleValueType(), {0}, {}, /*Depth=*/0,
-      X86::MaxShuffleCombineDepth, /*AllowCrossLaneVarMask=*/true,
-      /*AllowPerLaneVarMask=*/true, isMaskableNode(Op, Subtarget), DAG,
+      X86::MaxShuffleCombineDepth, /*AllowVariableCrossLaneMask=*/true,
+      /*AllowVariablePerLaneMask=*/true, isMaskableNode(Op, Subtarget), DAG,
       SDLoc(Op), Subtarget);
 }
 
@@ -42174,7 +42259,8 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
       if (SDValue Res = combineX86ShufflesRecursively(
               {BC}, 0, BC.getOpcode(), BC.getSimpleValueType(), DemandedMask,
               {}, /*Depth=*/0, X86::MaxShuffleCombineDepth,
-              /*AllowCrossLaneVarMask=*/true, /*AllowPerLaneVarMask=*/true,
+              /*AllowVariableCrossLaneMask=*/true,
+              /*AllowVariablePerLaneMask=*/true,
               /*IsMaskedShuffle=*/false, DAG, DL, Subtarget))
         return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
                            DAG.getBitcast(SrcVT, Res));
@@ -42617,9 +42703,19 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
     return SDValue();
   }
   case X86ISD::VPERM2X128: {
-    // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)).
     SDValue LHS = N->getOperand(0);
     SDValue RHS = N->getOperand(1);
+    unsigned Imm = N.getConstantOperandVal(2) & 255;
+
+    // Canonicalize unary/repeated operands to LHS.
+    if (LHS.isUndef() && !RHS.isUndef())
+      return DAG.getNode(X86ISD::VPERM2X128, DL, VT, RHS, LHS,
+                         DAG.getTargetConstant(Imm ^ 0x22, DL, MVT::i8));
+    if (LHS == RHS)
+      return DAG.getNode(X86ISD::VPERM2X128, DL, VT, LHS, DAG.getUNDEF(VT),
+                         DAG.getTargetConstant(Imm & ~0x22, DL, MVT::i8));
+
+    // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)).
     if (LHS.getOpcode() == ISD::BITCAST &&
         (RHS.getOpcode() == ISD::BITCAST || RHS.isUndef())) {
       EVT SrcVT = LHS.getOperand(0).getValueType();
@@ -42652,7 +42748,6 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
       }
       return SDValue();
     };
-    unsigned Imm = N.getConstantOperandVal(2);
     if (SDValue SubLo = FindSubVector128(Imm & 0x0F)) {
       if (SDValue SubHi = FindSubVector128((Imm & 0xF0) >> 4)) {
         MVT SubVT = VT.getHalfNumVectorElementsVT();
@@ -42661,6 +42756,24 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
         return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi);
       }
     }
+
+    // Attempt to match VBROADCAST*128 subvector broadcast load.
+    if (RHS.isUndef()) {
+      SmallVector<int, 4> Mask;
+      DecodeVPERM2X128Mask(4, Imm, Mask);
+      if (isUndefOrInRange(Mask, 0, 4)) {
+        bool SplatLo = isShuffleEquivalent(Mask, {0, 1, 0, 1}, LHS);
+        bool SplatHi = isShuffleEquivalent(Mask, {2, 3, 2, 3}, LHS);
+        if ((SplatLo || SplatHi) && !Subtarget.hasAVX512() &&
+            X86::mayFoldLoad(LHS, Subtarget, /*AssumeSingleUse=*/true)) {
+          MVT MemVT = VT.getHalfNumVectorElementsVT();
+          unsigned Ofs = SplatLo ? 0 : MemVT.getStoreSize();
+          return getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, DL, VT, MemVT,
+                                   cast<LoadSDNode>(LHS), Ofs, DAG);
+        }
+      }
+    }
+
     return SDValue();
   }
   case X86ISD::PSHUFD:
@@ -42850,7 +42963,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
           (SubOps.size() == 2 || (SubOps[2].isUndef() && SubOps[3].isUndef())))
         return SDValue();
       // Bail if any subops would have folded into the concat.
-      if (any_of(SubOps, [](SDValue Op) { return isShuffleFoldableLoad(Op); }))
+      if (any_of(SubOps, isShuffleFoldableLoad))
         return SDValue();
       // Concat 4x128 back to 2x256.
       if (SubOps.size() == 4) {
@@ -44126,7 +44239,12 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       // Conversions.
       // TODO: Add more CVT opcodes when we have test coverage.
     case X86ISD::CVTTP2SI:
-    case X86ISD::CVTTP2UI:
+    case X86ISD::CVTTP2UI: {
+      if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f16 &&
+          !Subtarget.hasVLX())
+        break;
+      [[fallthrough]];
+    }
     case X86ISD::CVTPH2PS: {
       SDLoc DL(Op);
       unsigned Scale = SizeInBits / ExtSizeInBits;
@@ -44289,9 +44407,10 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
 
     SDValue NewShuffle = combineX86ShufflesRecursively(
         {Op}, 0, Op.getOpcode(), Op.getSimpleValueType(), DemandedMask, {}, 0,
-        X86::MaxShuffleCombineDepth - Depth, /*AllowCrossLaneVarMask=*/true,
-        /*AllowPerLaneVarMask=*/true, isMaskableNode(Op, Subtarget), TLO.DAG,
-        SDLoc(Op), Subtarget);
+        X86::MaxShuffleCombineDepth - Depth,
+        /*AllowVariableCrossLaneMask=*/true,
+        /*AllowVariablePerLaneMask=*/true, isMaskableNode(Op, Subtarget),
+        TLO.DAG, SDLoc(Op), Subtarget);
     if (NewShuffle)
       return TLO.CombineTo(Op, NewShuffle);
   }
@@ -46701,9 +46820,8 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
   // need to convert vector bool to a scalar bool.
   if (DCI.isBeforeLegalize() && Vec.getOpcode() == ISD::VSELECT &&
       Vec.getOperand(0).getOpcode() == ISD::SETCC &&
-      Vec.getOperand(0).getOperand(0).getValueType() == VecVT) {
-    assert(Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&
-           "Unexpected cond type for combine");
+      Vec.getOperand(0).getOperand(0).getValueType() == VecVT &&
+      Vec.getOperand(0).getValueType().getScalarType() == MVT::i1) {
     // ext (sel Cond, X, Y), 0 --> sel (ext Cond, 0), (ext X, 0), (ext Y, 0)
     SDLoc DL(ExtElt);
     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
@@ -47663,13 +47781,19 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
                                                            DL, DAG, Subtarget))
       return V;
 
-  // Convert vselects with constant condition into shuffles.
-  if (CondConstantVector && DCI.isBeforeLegalizeOps() &&
-      (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV)) {
+  if (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV) {
     SmallVector<int, 64> Mask;
     if (createShuffleMaskFromVSELECT(Mask, Cond,
-                                     N->getOpcode() == X86ISD::BLENDV))
-      return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
+                                     N->getOpcode() == X86ISD::BLENDV)) {
+      // Convert vselects with constant condition into shuffles.
+      if (DCI.isBeforeLegalizeOps())
+        return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
+
+      // Attempt to combine as shuffle.
+      SDValue Op(N, 0);
+      if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
+        return Res;
+    }
   }
 
   // fold vselect(cond, pshufb(x), pshufb(y)) -> or (pshufb(x), pshufb(y))
@@ -47718,6 +47842,7 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
        Cond.getOpcode() == ISD::STRICT_FSETCCS) &&
       VT.isFloatingPoint() && VT != MVT::f80 && VT != MVT::f128 &&
       !isSoftF16(VT, Subtarget) && (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&
+      ((VT != MVT::v8f16 && VT != MVT::v16f16) || Subtarget.hasVLX()) &&
       (Subtarget.hasSSE2() ||
        (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) {
     bool IsStrict = Cond->isStrictFPOpcode();
@@ -49269,6 +49394,8 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
   //      (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2)
   // Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) ->
   //    (ADD (CMOV (CTTZ X), C1-C2, (X == 0)), C2)
+  // Or (CMOV (BSR ?, X), Y, (X == 0)) -> (BSR Y, X)
+  // TODO: Or (CMOV (BSF ?, X), Y, (X == 0)) -> (BSF Y, X)
   if ((CC == X86::COND_NE || CC == X86::COND_E) &&
       Cond.getOpcode() == X86ISD::CMP && isNullConstant(Cond.getOperand(1))) {
     SDValue Add = TrueOp;
@@ -49277,6 +49404,14 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
     if (CC == X86::COND_E)
       std::swap(Add, Const);
 
+    // TODO: ADD BSF support, but requires changes to the "REP BSF" CTTZ hack.
+    if (Subtarget.hasBitScanPassThrough() && Add.getOpcode() == X86ISD::BSR &&
+        Add.getResNo() == 0 && Add.hasOneUse() &&
+        Add.getOperand(1) == Cond.getOperand(0)) {
+      return DAG.getNode(Add.getOpcode(), DL, Add->getVTList(), Const,
+                         Add.getOperand(1));
+    }
+
     // We might have replaced the constant in the cmov with the LHS of the
     // compare. If so change it to the RHS of the compare.
     if (Const == Cond.getOperand(0))
@@ -51956,7 +52091,8 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
       if (SDValue Shuffle = combineX86ShufflesRecursively(
               {SrcVec}, 0, SrcVec.getOpcode(), SrcVec.getSimpleValueType(),
               ShuffleMask, {}, /*Depth=*/1, X86::MaxShuffleCombineDepth,
-              /*AllowVarCrossLaneMask=*/true, /*AllowVarPerLaneMask=*/true,
+              /*AllowVariableCrossLaneMask=*/true,
+              /*AllowVariablePerLaneMask=*/true,
               /*IsMaskedShuffle=*/false, DAG, SDLoc(SrcVec), Subtarget))
         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
                            N0.getOperand(1));
@@ -52226,59 +52362,6 @@ static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
   return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
 }
 
-static SDValue foldMaskedMergeImpl(SDValue And0_L, SDValue And0_R,
-                                   SDValue And1_L, SDValue And1_R,
-                                   const SDLoc &DL, SelectionDAG &DAG) {
-  if (!isBitwiseNot(And0_L, true) || !And0_L->hasOneUse())
-    return SDValue();
-  SDValue NotOp = And0_L->getOperand(0);
-  if (NotOp == And1_R)
-    std::swap(And1_R, And1_L);
-  if (NotOp != And1_L)
-    return SDValue();
-
-  // (~(NotOp) & And0_R) | (NotOp & And1_R)
-  // --> ((And0_R ^ And1_R) & NotOp) ^ And1_R
-  EVT VT = And1_L->getValueType(0);
-  SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R);
-  SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R);
-  SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp);
-  SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R);
-  return Xor1;
-}
-
-/// Fold "masked merge" expressions like `(m & x) | (~m & y)` into the
-/// equivalent `((x ^ y) & m) ^ y)` pattern.
-/// This is typically a better representation for  targets without a fused
-/// "and-not" operation. This function is intended to be called from a
-/// `TargetLowering::PerformDAGCombine` callback on `ISD::OR` nodes.
-static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG) {
-  // Note that masked-merge variants using XOR or ADD expressions are
-  // normalized to OR by InstCombine so we only check for OR.
-  assert(Node->getOpcode() == ISD::OR && "Must be called with ISD::OR node");
-  SDValue N0 = Node->getOperand(0);
-  if (N0->getOpcode() != ISD::AND || !N0->hasOneUse())
-    return SDValue();
-  SDValue N1 = Node->getOperand(1);
-  if (N1->getOpcode() != ISD::AND || !N1->hasOneUse())
-    return SDValue();
-
-  SDLoc DL(Node);
-  SDValue N00 = N0->getOperand(0);
-  SDValue N01 = N0->getOperand(1);
-  SDValue N10 = N1->getOperand(0);
-  SDValue N11 = N1->getOperand(1);
-  if (SDValue Result = foldMaskedMergeImpl(N00, N01, N10, N11, DL, DAG))
-    return Result;
-  if (SDValue Result = foldMaskedMergeImpl(N01, N00, N10, N11, DL, DAG))
-    return Result;
-  if (SDValue Result = foldMaskedMergeImpl(N10, N11, N00, N01, DL, DAG))
-    return Result;
-  if (SDValue Result = foldMaskedMergeImpl(N11, N10, N00, N01, DL, DAG))
-    return Result;
-  return SDValue();
-}
-
 /// If this is an add or subtract where one operand is produced by a cmp+setcc,
 /// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
 /// with CMP+{ADC, SBB}.
@@ -52682,11 +52765,6 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-  // We should fold "masked merge" patterns when `andn` is not available.
-  if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1)
-    if (SDValue R = foldMaskedMerge(N, DAG))
-      return R;
-
   if (SDValue R = combineOrXorWithSETCC(N->getOpcode(), dl, VT, N0, N1, DAG))
     return R;
 
@@ -53015,7 +53093,7 @@ static SDValue combineConstantPoolLoads(SDNode *N, const SDLoc &dl,
         (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD ||
          User->getOpcode() == X86ISD::VBROADCAST_LOAD ||
          ISD::isNormalLoad(User)) &&
-        UserLd->getChain() == Chain && !User->hasAnyUseOfValue(1) &&
+        UserLd->getChain() == Chain && User->hasAnyUseOfValue(0) &&
         User->getValueSizeInBits(0).getFixedValue() >
             RegVT.getFixedSizeInBits()) {
       EVT UserVT = User->getValueType(0);
@@ -53037,6 +53115,7 @@ static SDValue combineConstantPoolLoads(SDNode *N, const SDLoc &dl,
               getTargetConstantBitsFromNode(SDValue(User, 0), NumBits,
                                             UserUndefs, UserBits)) {
             if (MatchingBits(Undefs, UserUndefs, Bits, UserBits)) {
+              DAG.makeEquivalentMemoryOrdering(SDValue(N, 1), SDValue(User, 1));
               SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, dl,
                                                  RegVT.getSizeInBits());
               Extract = DAG.getBitcast(RegVT, Extract);
@@ -53123,9 +53202,10 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
           User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
           UserLd->getChain() == Chain && UserLd->getBasePtr() == Ptr &&
           UserLd->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits() &&
-          !User->hasAnyUseOfValue(1) &&
+          User->hasAnyUseOfValue(0) &&
           User->getValueSizeInBits(0).getFixedValue() >
               RegVT.getFixedSizeInBits()) {
+        DAG.makeEquivalentMemoryOrdering(SDValue(N, 1), SDValue(User, 1));
         SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, dl,
                                            RegVT.getSizeInBits());
         Extract = DAG.getBitcast(RegVT, Extract);
@@ -55286,10 +55366,17 @@ static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
+  auto IsMinMaxLegal = [&](EVT VT) {
+    if (!TLI.isTypeLegal(VT))
+      return false;
+    return VT.getScalarType() != MVT::f16 ||
+           (Subtarget.hasFP16() && (VT == MVT::v32f16 || Subtarget.hasVLX()));
+  };
+
   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
         (Subtarget.hasSSE2() && VT == MVT::f64) ||
         (Subtarget.hasFP16() && VT == MVT::f16) ||
-        (VT.isVector() && TLI.isTypeLegal(VT))))
+        (VT.isVector() && IsMinMaxLegal(VT))))
     return SDValue();
 
   SDValue Op0 = N->getOperand(0);
@@ -58011,6 +58098,23 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
     return V;
 
+  // Canonicalize hidden LEA pattern:
+  // Fold (add (sub (shl x, c), y), z) -> (sub (add (shl x, c), z), y)
+  // iff c < 4
+  if (VT == MVT::i32 || VT == MVT::i64) {
+    SDValue Y, Z, Shift;
+    APInt Amt;
+    if (sd_match(
+            N, m_Add(m_OneUse(m_Sub(m_AllOf(m_Value(Shift),
+                                            m_Shl(m_Value(), m_ConstInt(Amt))),
+                                    m_Value(Y))),
+                     m_Value(Z))) &&
+        Amt.ult(4) && !isa<ConstantSDNode>(Z)) {
+      return DAG.getNode(ISD::SUB, DL, VT,
+                         DAG.getNode(ISD::ADD, DL, VT, Shift, Z), Y);
+    }
+  }
+
   // add(psadbw(X,0),psadbw(Y,0)) -> psadbw(add(X,Y),0)
   // iff X and Y won't overflow.
   if (Op0.getOpcode() == X86ISD::PSADBW && Op1.getOpcode() == X86ISD::PSADBW &&
@@ -58025,21 +58129,31 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-  // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into
-  // (sub Y, (sext (vXi1 X))).
-  // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y) in
-  // generic DAG combine without a legal type check, but adding this there
-  // caused regressions.
   if (VT.isVector()) {
     SDValue X, Y;
     EVT BoolVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
                                   VT.getVectorElementCount());
+
+    // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into
+    // (sub Y, (sext (vXi1 X))).
+    // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y)
+    // in generic DAG combine without a legal type check, but adding this there
+    // caused regressions.
     if (DAG.getTargetLoweringInfo().isTypeLegal(BoolVT) &&
         sd_match(N, m_Add(m_ZExt(m_AllOf(m_SpecificVT(BoolVT), m_Value(X))),
                           m_Value(Y)))) {
       SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, X);
       return DAG.getNode(ISD::SUB, DL, VT, Y, SExt);
     }
+
+    // Fold (add X, (srl Y, 7)) -> (sub X, (icmp_sgt 0, Y)) to undo instcombine
+    // canonicalisation as we don't have good vXi8 shifts.
+    if (VT.getScalarType() == MVT::i8 &&
+        sd_match(N, m_Add(m_Value(X), m_Srl(m_Value(Y), m_SpecificInt(7))))) {
+      SDValue Cmp =
+          DAG.getSetCC(DL, BoolVT, DAG.getConstant(0, DL, VT), Y, ISD::SETGT);
+      return DAG.getNode(ISD::SUB, DL, VT, X, DAG.getSExtOrTrunc(Cmp, DL, VT));
+    }
   }
 
   // Peephole for 512-bit VPDPBSSD on non-VLX targets.
@@ -58402,26 +58516,34 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       EVT SrcVT1 = Src1.getOperand(0).getValueType();
       unsigned NumSrcElts0 = SrcVT0.getVectorNumElements();
       unsigned NumSrcElts1 = SrcVT1.getVectorNumElements();
+      const APInt &SrcIdx0 = Src0.getConstantOperandAPInt(1);
+      const APInt &SrcIdx1 = Src1.getConstantOperandAPInt(1);
       // concat(extract_subvector(v0), extract_subvector(v1)) -> vperm2x128.
-      // Only concat of subvector high halves which vperm2x128 is best at.
+      // Only concat of subvector high halves which vperm2x128 is best at or if
+      // it should fold into a subvector broadcast.
       if (VT.is256BitVector() && SrcVT0.is256BitVector() &&
-          SrcVT1.is256BitVector() &&
-          Src0.getConstantOperandAPInt(1) == (NumSrcElts0 / 2) &&
-          Src1.getConstantOperandAPInt(1) == (NumSrcElts1 / 2)) {
-        return DAG.getNode(X86ISD::VPERM2X128, DL, VT,
-                           DAG.getBitcast(VT, Src0.getOperand(0)),
-                           DAG.getBitcast(VT, Src1.getOperand(0)),
-                           DAG.getTargetConstant(0x31, DL, MVT::i8));
+          SrcVT1.is256BitVector()) {
+        assert((SrcIdx0 == 0 || SrcIdx0 == (NumSrcElts0 / 2)) &&
+               (SrcIdx1 == 0 || SrcIdx1 == (NumSrcElts1 / 2)) &&
+               "Bad subvector index");
+        if ((SrcIdx0 == (NumSrcElts0 / 2) && SrcIdx1 == (NumSrcElts1 / 2)) ||
+            (IsSplat && ISD::isNormalLoad(Src0.getOperand(0).getNode()))) {
+          unsigned Index = 0;
+          Index |= SrcIdx0 == 0 ? 0x00 : 0x01;
+          Index |= SrcIdx1 == 0 ? 0x20 : 0x30;
+          return DAG.getNode(X86ISD::VPERM2X128, DL, VT,
+                             DAG.getBitcast(VT, Src0.getOperand(0)),
+                             DAG.getBitcast(VT, Src1.getOperand(0)),
+                             DAG.getTargetConstant(Index, DL, MVT::i8));
+        }
       }
       // Widen extract_subvector
       // concat(extract_subvector(x,lo), extract_subvector(x,hi))
       // --> extract_subvector(x,lo)
       unsigned NumSubElts0 = Src0.getValueType().getVectorNumElements();
       if (Src0.getOperand(0) == Src1.getOperand(0) &&
-          (Src0.getConstantOperandAPInt(1) == 0 ||
-           Src0.getConstantOperandAPInt(1) == (NumSrcElts0 / 2)) &&
-          Src1.getConstantOperandAPInt(1) ==
-              (Src0.getConstantOperandAPInt(1) + NumSubElts0)) {
+          (SrcIdx0 == 0 || SrcIdx0 == (NumSrcElts0 / 2)) &&
+          SrcIdx1 == (SrcIdx0 + NumSubElts0)) {
         return DAG.getBitcast(VT,
                               extractSubVector(Src0.getOperand(0),
                                                Src0.getConstantOperandVal(1),
@@ -58575,7 +58697,6 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       break;
     }
     case X86ISD::SHUFP: {
-      // TODO: Add SHUFPD support if/when necessary.
       if (!IsSplat &&
           (VT == MVT::v8f32 ||
            (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) &&
@@ -58644,18 +58765,6 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
             DAG.getNode(X86ISD::VPERMILPI, DL, FloatVT, Res, Op0.getOperand(1));
         return DAG.getBitcast(VT, Res);
       }
-      if (!IsSplat && (VT == MVT::v4f64 || VT == MVT::v8f64)) {
-        unsigned NumSubElts = Op0.getValueType().getVectorNumElements();
-        uint64_t Mask = (1ULL << NumSubElts) - 1;
-        uint64_t Idx = 0;
-        for (unsigned I = 0; I != NumOps; ++I) {
-          uint64_t SubIdx = Ops[I].getConstantOperandVal(1);
-          Idx |= (SubIdx & Mask) << (I * NumSubElts);
-        }
-        return DAG.getNode(X86ISD::VPERMILPI, DL, VT,
-                           ConcatSubOperand(VT, Ops, 0),
-                           DAG.getTargetConstant(Idx, DL, MVT::i8));
-      }
       break;
     case X86ISD::VPERMILPV:
       if (!IsSplat && (VT.is256BitVector() ||
@@ -59226,6 +59335,54 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     return DAG.getBitcast(VT, Res);
   }
 
+  // We can always convert per-lane vXf64 shuffles into VSHUFPD.
+  if (!IsSplat &&
+      ((NumOps == 2 && VT == MVT::v4f64) ||
+       (NumOps == 4 && VT == MVT::v8f64 && Subtarget.useAVX512Regs())) &&
+      all_of(Ops, [](SDValue Op) { return Op.hasOneUse(); })) {
+    // Collect the individual per-lane v2f64/v4f64 shuffles.
+    MVT OpVT = Ops[0].getSimpleValueType();
+    unsigned NumOpElts = OpVT.getVectorNumElements();
+    SmallVector<SmallVector<SDValue, 2>, 4> SrcOps(NumOps);
+    SmallVector<SmallVector<int, 8>, 4> SrcMasks(NumOps);
+    if (all_of(seq<int>(NumOps), [&](int I) {
+          return getTargetShuffleInputs(Ops[I], SrcOps[I], SrcMasks[I], DAG,
+                                        Depth + 1) &&
+                 !is128BitLaneCrossingShuffleMask(OpVT, SrcMasks[I]) &&
+                 none_of(SrcMasks[I], isUndefOrZero) &&
+                 SrcMasks[I].size() == NumOpElts &&
+                 all_of(SrcOps[I], [&OpVT](SDValue V) {
+                   return V.getValueType() == OpVT;
+                 });
+        })) {
+      // Concatenate the shuffle masks into SHUFPD mask and collect subops.
+      bool Unary = true;
+      unsigned SHUFPDMask = 0;
+      SmallVector<SDValue, 4> LHS(NumOps), RHS(NumOps);
+      for (unsigned I = 0; I != NumOps; ++I) {
+        LHS[I] = SrcOps[I][SrcMasks[I][0] / NumOpElts];
+        RHS[I] = SrcOps[I][SrcMasks[I][1] / NumOpElts];
+        Unary &= LHS[I] == RHS[I];
+        for (unsigned J = 0; J != NumOpElts; ++J)
+          SHUFPDMask |= (SrcMasks[I][J] & 1) << ((I * NumOpElts) + J);
+      }
+      // Concat SHUFPD LHS/RHS operands - if they match then it will become a
+      // PERMILPD mask and we can always profitably concatenate them.
+      SDValue Concat0 =
+          combineConcatVectorOps(DL, VT, LHS, DAG, Subtarget, Depth + 1);
+      SDValue Concat1 =
+          combineConcatVectorOps(DL, VT, RHS, DAG, Subtarget, Depth + 1);
+      if (Unary || Concat0 || Concat1) {
+        Concat0 =
+            Concat0 ? Concat0 : DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS);
+        Concat1 =
+            Concat1 ? Concat1 : DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, RHS);
+        return DAG.getNode(X86ISD::SHUFP, DL, VT, Concat0, Concat1,
+                           DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
+      }
+    }
+  }
+
   return SDValue();
 }
 
@@ -59421,10 +59578,8 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
   // If we're splatting the lower half subvector of a full vector load into the
   // upper half, just splat the subvector directly, potentially creating a
   // subvector broadcast.
-  // TODO: Drop hasOneUse checks.
   if ((int)IdxVal == (VecNumElts / 2) &&
-      Vec.getValueSizeInBits() == (2 * SubVec.getValueSizeInBits()) &&
-      (Vec.hasOneUse() || SubVec.hasOneUse())) {
+      Vec.getValueSizeInBits() == (2 * SubVec.getValueSizeInBits())) {
     auto *VecLd = dyn_cast<LoadSDNode>(Vec);
     auto *SubLd = dyn_cast<LoadSDNode>(SubVec);
     if (VecLd && SubLd &&
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp b/external/llvm-project/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
index 7740a174af4f..52be14228e55 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
@@ -147,7 +147,7 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) {
   for (auto &MBB : MF) {
     // Find all basic blocks that their address was taken (for example
     // in the case of indirect jump) and add ENDBR instruction.
-    if (MBB.hasAddressTaken())
+    if (MBB.isMachineBlockAddressTaken() || MBB.isIRBlockAddressTaken())
       Changed |= addENDBR(MBB, MBB.begin());
 
     for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86InstrFragments.td b/external/llvm-project/llvm/lib/Target/X86/X86InstrFragments.td
index fe95b8c20a8f..116986a0fffe 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/external/llvm-project/llvm/lib/Target/X86/X86InstrFragments.td
@@ -842,15 +842,20 @@ def X86fldf80 : PatFrag<(ops node:$ptr), (X86fld node:$ptr), [{
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f80;
 }]>;
 
-def X86fild16 : PatFrag<(ops node:$ptr), (X86fild node:$ptr), [{
-  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
-}]>;
-def X86fild32 : PatFrag<(ops node:$ptr), (X86fild node:$ptr), [{
-  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
-}]>;
-def X86fild64 : PatFrag<(ops node:$ptr), (X86fild node:$ptr), [{
-  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
-}]>;
+def X86fild16 : PatFrag<(ops node:$ptr), (X86fild node:$ptr)> {
+  let IsLoad = true;
+  let MemoryVT = i16;
+}
+
+def X86fild32 : PatFrag<(ops node:$ptr), (X86fild node:$ptr)> {
+  let IsLoad = true;
+  let MemoryVT = i32;
+}
+
+def X86fild64 : PatFrag<(ops node:$ptr), (X86fild node:$ptr)> {
+  let IsLoad = true;
+  let MemoryVT = i64;
+}
 
 def X86fist32 : PatFrag<(ops node:$val, node:$ptr),
                         (X86fist node:$val, node:$ptr), [{
@@ -862,18 +867,23 @@ def X86fist64 : PatFrag<(ops node:$val, node:$ptr),
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 }]>;
 
-def X86fp_to_i16mem : PatFrag<(ops node:$val, node:$ptr),
-                              (X86fp_to_mem node:$val, node:$ptr), [{
-  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
-}]>;
-def X86fp_to_i32mem : PatFrag<(ops node:$val, node:$ptr),
-                              (X86fp_to_mem node:$val, node:$ptr), [{
-  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
-}]>;
-def X86fp_to_i64mem : PatFrag<(ops node:$val, node:$ptr),
-                              (X86fp_to_mem node:$val, node:$ptr), [{
-  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
-}]>;
+def X86fp_to_i16mem
+    : PatFrag<(ops node:$val, node:$ptr), (X86fp_to_mem node:$val, node:$ptr)> {
+  let IsStore = true;
+  let MemoryVT = i16;
+}
+
+def X86fp_to_i32mem
+    : PatFrag<(ops node:$val, node:$ptr), (X86fp_to_mem node:$val, node:$ptr)> {
+  let IsStore = true;
+  let MemoryVT = i32;
+}
+
+def X86fp_to_i64mem
+    : PatFrag<(ops node:$val, node:$ptr), (X86fp_to_mem node:$val, node:$ptr)> {
+  let IsStore = true;
+  let MemoryVT = i64;
+}
 
 //===----------------------------------------------------------------------===//
 // FPStack pattern fragments
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86InstrGISel.td b/external/llvm-project/llvm/lib/Target/X86/X86InstrGISel.td
new file mode 100644
index 000000000000..f4fa33807cd9
--- /dev/null
+++ b/external/llvm-project/llvm/lib/Target/X86/X86InstrGISel.td
@@ -0,0 +1,31 @@
+//===- X86InstrGISel.td - X86 GISel target specific opcodes -*- tablegen -*===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// X86 GlobalISel target pseudo instruction definitions. This is kept
+// separately from the other tablegen files for organizational purposes, but
+// share the same infrastructure.
+//
+//===----------------------------------------------------------------------===//
+
+class X86GenericInstruction : GenericInstruction { let Namespace = "X86"; }
+
+def G_FILD : X86GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins ptype1:$src);
+  let hasSideEffects = false;
+  let mayLoad = true;
+}
+def G_FIST : X86GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins type0:$src1, ptype1:$src2);
+  let hasSideEffects = false;
+  let mayStore = true;
+}
+
+def : GINodeEquiv<G_FILD, X86fild>;
+def : GINodeEquiv<G_FIST, X86fp_to_mem>;
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.cpp b/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.cpp
index 7d9fa759ad81..abf365eedec3 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -2353,33 +2353,9 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
     break;
   case X86::BLENDPDrri:
   case X86::BLENDPSrri:
+  case X86::PBLENDWrri:
   case X86::VBLENDPDrri:
   case X86::VBLENDPSrri:
-    // If we're optimizing for size, try to use MOVSD/MOVSS.
-    if (MI.getParent()->getParent()->getFunction().hasOptSize()) {
-      unsigned Mask = (Opc == X86::BLENDPDrri || Opc == X86::VBLENDPDrri) ? 0x03: 0x0F;
-      if ((MI.getOperand(3).getImm() ^ Mask) == 1) {
-#define FROM_TO(FROM, TO)                                                      \
-  case X86::FROM:                                                              \
-    Opc = X86::TO;                                                             \
-    break;
-        switch (Opc) {
-        default:
-          llvm_unreachable("Unreachable!");
-        FROM_TO(BLENDPDrri, MOVSDrr)
-        FROM_TO(BLENDPSrri, MOVSSrr)
-        FROM_TO(VBLENDPDrri, VMOVSDrr)
-        FROM_TO(VBLENDPSrri, VMOVSSrr)
-        }
-        WorkingMI = CloneIfNew(MI);
-        WorkingMI->setDesc(get(Opc));
-        WorkingMI->removeOperand(3);
-        break;
-      }
-#undef FROM_TO
-    }
-    [[fallthrough]];
-  case X86::PBLENDWrri:
   case X86::VBLENDPDYrri:
   case X86::VBLENDPSYrri:
   case X86::VPBLENDDrri:
@@ -2487,6 +2463,7 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
       break;
     }
 
+    assert(Opc == X86::MOVSDrr && "Only MOVSD can commute to SHUFPD");
     WorkingMI = CloneIfNew(MI);
     WorkingMI->setDesc(get(X86::SHUFPDrri));
     WorkingMI->addOperand(MachineOperand::CreateImm(0x02));
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.td b/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.td
index e75d6743f927..7f6c5614847e 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/external/llvm-project/llvm/lib/Target/X86/X86InstrInfo.td
@@ -37,6 +37,11 @@ include "X86InstrFormats.td"
 //
 include "X86InstrUtils.td"
 
+//===----------------------------------------------------------------------===//
+// Global ISel
+//
+include "X86InstrGISel.td"
+
 //===----------------------------------------------------------------------===//
 // Subsystems.
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/external/llvm-project/llvm/lib/Target/X86/X86MachineFunctionInfo.h
index 24371369d4a4..5f974e5de9a1 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86MachineFunctionInfo.h
+++ b/external/llvm-project/llvm/lib/Target/X86/X86MachineFunctionInfo.h
@@ -149,7 +149,7 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
   /// other tools to detect the extended record.
   bool HasSwiftAsyncContext = false;
 
-  /// Ajust stack for push2/pop2
+  /// Adjust stack for push2/pop2
   bool PadForPush2Pop2 = false;
 
   /// Candidate registers for push2/pop2
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86PadShortFunction.cpp b/external/llvm-project/llvm/lib/Target/X86/X86PadShortFunction.cpp
index 170ca2a93250..049384eefa18 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86PadShortFunction.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86PadShortFunction.cpp
@@ -100,6 +100,7 @@ FunctionPass *llvm::createX86PadShortFunctions() {
 /// runOnMachineFunction - Loop over all of the basic blocks, inserting
 /// NOOP instructions before early exits.
 bool PadShortFunc::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "Start X86PadShortFunctionPass\n";);
   if (skipFunction(MF.getFunction()))
     return false;
 
@@ -149,7 +150,7 @@ bool PadShortFunc::runOnMachineFunction(MachineFunction &MF) {
       MadeChange = true;
     }
   }
-
+  LLVM_DEBUG(dbgs() << "End X86PadShortFunctionPass\n";);
   return MadeChange;
 }
 
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86PartialReduction.cpp b/external/llvm-project/llvm/lib/Target/X86/X86PartialReduction.cpp
index 1d0815ee830b..a25e4e0f464a 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -157,8 +157,7 @@ bool X86PartialReduction::tryMAddReplacement(Instruction *Op,
 
     // If the operation can be freely truncated and has enough sign bits we
     // can shrink.
-    if (IsFreeTruncation(Op) &&
-        ComputeNumSignBits(Op, *DL, 0, nullptr, Mul) > 16)
+    if (IsFreeTruncation(Op) && ComputeNumSignBits(Op, *DL, nullptr, Mul) > 16)
       return true;
 
     // SelectionDAG has limited support for truncating through an add or sub if
@@ -167,7 +166,7 @@ bool X86PartialReduction::tryMAddReplacement(Instruction *Op,
       if (BO->getParent() == Mul->getParent() &&
           IsFreeTruncation(BO->getOperand(0)) &&
           IsFreeTruncation(BO->getOperand(1)) &&
-          ComputeNumSignBits(Op, *DL, 0, nullptr, Mul) > 16)
+          ComputeNumSignBits(Op, *DL, nullptr, Mul) > 16)
         return true;
     }
 
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.cpp b/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 71d36594afae..9adac067aace 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -1280,4 +1280,4 @@ bool X86RegisterInfo::isNonRex2RegClass(const TargetRegisterClass *RC) const {
   case X86::GR64_with_sub_16bit_in_GR16_NOREX2RegClassID:
     return true;
   }
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.td b/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.td
index 3f9af5639a68..e9ca25d808a5 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/external/llvm-project/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -806,17 +806,37 @@ def VR512_0_15 : RegisterClass<"X86", [v16f32, v8f64, v64i8, v32i16, v16i32, v8i
                                512, (sequence "ZMM%u", 0, 15)>;
 
 // Scalar AVX-512 floating point registers.
-def FR32X : RegisterClass<"X86", [f32], 32, (sequence "XMM%u", 0, 31)>;
+def FR32X : RegisterClass<"X86", [f32], 32, (sequence "XMM%u", 0, 31)> {
+  let AltOrders = [(add (sequence "XMM%u", 16, 31), (sequence "XMM%u", 0, 15))];
+  let AltOrderSelect = [{
+    return Rev;
+  }];
+}
 
-def FR64X : RegisterClass<"X86", [f64], 64, (add FR32X)>;
+def FR64X : RegisterClass<"X86", [f64], 64, (add FR32X)> {
+  let AltOrders = [(add (sequence "XMM%u", 16, 31), (sequence "XMM%u", 0, 15))];
+  let AltOrderSelect = [{
+    return Rev;
+  }];
+}
 
 def FR16X : RegisterClass<"X86", [f16], 16, (add FR32X)> {let Size = 32;}
 
 // Extended VR128 and VR256 for AVX-512 instructions
 def VR128X : RegisterClass<"X86", [v4f32, v2f64, v8f16, v8bf16, v16i8, v8i16, v4i32, v2i64, f128],
-                           128, (add FR32X)>;
+                           128, (add FR32X)> {
+  let AltOrders = [(add (sequence "XMM%u", 16, 31), (sequence "XMM%u", 0, 15))];
+  let AltOrderSelect = [{
+    return Rev;
+  }];
+}
 def VR256X : RegisterClass<"X86", [v8f32, v4f64, v16f16, v16bf16, v32i8, v16i16, v8i32, v4i64],
-                           256, (sequence "YMM%u", 0, 31)>;
+                           256, (sequence "YMM%u", 0, 31)> {
+  let AltOrders = [(add (sequence "YMM%u", 16, 31), (sequence "YMM%u", 0, 15))];
+  let AltOrderSelect = [{
+    return Rev;
+  }];
+}
 
 // Mask registers
 def VK1     : RegisterClass<"X86", [v1i1],  16,  (sequence "K%u", 0, 7)> {let Size = 16;}
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86SuppressAPXForReloc.cpp b/external/llvm-project/llvm/lib/Target/X86/X86SuppressAPXForReloc.cpp
index 5851262c8e7b..b40e1f208bda 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86SuppressAPXForReloc.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86SuppressAPXForReloc.cpp
@@ -252,4 +252,4 @@ bool X86SuppressAPXForRelocationPass::runOnMachineFunction(
   Changed |= handleNDDOrNFInstructions(MF, ST);
 
   return Changed;
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86TargetMachine.cpp b/external/llvm-project/llvm/lib/Target/X86/X86TargetMachine.cpp
index 6c27b7211263..2d4afc23f1a4 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -378,14 +378,14 @@ void X86TargetMachine::reset() { SubtargetMap.clear(); }
 
 ScheduleDAGInstrs *
 X86TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
-  ScheduleDAGMILive *DAG = createGenericSchedLive(C);
+  ScheduleDAGMILive *DAG = createSchedLive(C);
   DAG->addMutation(createX86MacroFusionDAGMutation());
   return DAG;
 }
 
 ScheduleDAGInstrs *
 X86TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
-  ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
+  ScheduleDAGMI *DAG = createSchedPostRA(C);
   DAG->addMutation(createX86MacroFusionDAGMutation());
   return DAG;
 }
@@ -554,7 +554,6 @@ bool X86PassConfig::addPreISel() {
 void X86PassConfig::addPreRegAlloc() {
   if (getOptLevel() != CodeGenOptLevel::None) {
     addPass(&LiveRangeShrinkID);
-    addPass(createX86WinFixupBufferSecurityCheckPass());
     addPass(createX86FixupSetCC());
     addPass(createX86OptimizeLEAs());
     addPass(createX86CallFrameOptimization());
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index ee4798c6cafb..e13f38911780 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -123,7 +123,7 @@ std::optional<unsigned> X86TTIImpl::getCacheSize(
     //   - Broadwell
     //   - Skylake
     //   - Kabylake
-    return 32 * 1024;  //  32 KByte
+    return 32 * 1024;  //  32 KiB
   case TargetTransformInfo::CacheLevel::L2D:
     //   - Penryn
     //   - Nehalem
@@ -134,7 +134,7 @@ std::optional<unsigned> X86TTIImpl::getCacheSize(
     //   - Broadwell
     //   - Skylake
     //   - Kabylake
-    return 256 * 1024; // 256 KByte
+    return 256 * 1024; // 256 KiB
   }
 
   llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
@@ -5998,12 +5998,19 @@ InstructionCost X86TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
     // This is an imperfect hack to prevent constant hoisting of
     // compares that might be trying to check if a 64-bit value fits in
     // 32-bits. The backend can optimize these cases using a right shift by 32.
-    // Ideally we would check the compare predicate here. There also other
-    // similar immediates the backend can use shifts for.
+    // There are other predicates and immediates the backend can use shifts for.
     if (Idx == 1 && ImmBitWidth == 64) {
       uint64_t ImmVal = Imm.getZExtValue();
       if (ImmVal == 0x100000000ULL || ImmVal == 0xffffffff)
         return TTI::TCC_Free;
+
+      if (auto *Cmp = dyn_cast_or_null<CmpInst>(Inst)) {
+        if (Cmp->isEquality()) {
+          KnownBits Known = computeKnownBits(Cmp->getOperand(0), DL);
+          if (Known.countMinTrailingZeros() >= 32)
+            return TTI::TCC_Free;
+        }
+      }
     }
     ImmIdx = 1;
     break;
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 72673d6fbd80..8045f1b1d663 100644
--- a/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -25,7 +25,7 @@ namespace llvm {
 
 class InstCombiner;
 
-class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
+class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
   typedef BasicTTIImplBase<X86TTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
diff --git a/external/llvm-project/llvm/lib/Target/X86/X86WinFixupBufferSecurityCheck.cpp b/external/llvm-project/llvm/lib/Target/X86/X86WinFixupBufferSecurityCheck.cpp
deleted file mode 100644
index 5c12af1fee63..000000000000
--- a/external/llvm-project/llvm/lib/Target/X86/X86WinFixupBufferSecurityCheck.cpp
+++ /dev/null
@@ -1,245 +0,0 @@
-//===- X86WinFixupBufferSecurityCheck.cpp Fix Buffer Security Check Call -===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// Buffer Security Check implementation inserts windows specific callback into
-// code. On windows, __security_check_cookie call gets call everytime function
-// is return without fixup. Since this function is defined in runtime library,
-// it incures cost of call in dll which simply does comparison and returns most
-// time. With Fixup, We selective move to call in DLL only if comparison fails.
-//===----------------------------------------------------------------------===//
-
-#include "X86.h"
-#include "X86FrameLowering.h"
-#include "X86InstrInfo.h"
-#include "X86Subtarget.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/IR/Module.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "x86-win-fixup-bscheck"
-
-namespace {
-
-class X86WinFixupBufferSecurityCheckPass : public MachineFunctionPass {
-public:
-  static char ID;
-
-  X86WinFixupBufferSecurityCheckPass() : MachineFunctionPass(ID) {}
-
-  StringRef getPassName() const override {
-    return "X86 Windows Fixup Buffer Security Check";
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  std::pair<MachineBasicBlock *, MachineInstr *>
-  getSecurityCheckerBasicBlock(MachineFunction &MF);
-
-  void getGuardCheckSequence(MachineBasicBlock *CurMBB, MachineInstr *CheckCall,
-                             MachineInstr *SeqMI[5]);
-
-  void SplitBasicBlock(MachineBasicBlock *CurMBB, MachineBasicBlock *NewRetMBB,
-                       MachineBasicBlock::iterator SplitIt);
-
-  void FinishBlock(MachineBasicBlock *MBB);
-
-  void FinishFunction(MachineBasicBlock *FailMBB, MachineBasicBlock *NewRetMBB);
-
-  std::pair<MachineInstr *, MachineInstr *>
-  CreateFailCheckSequence(MachineBasicBlock *CurMBB, MachineBasicBlock *FailMBB,
-                          MachineInstr *SeqMI[5]);
-};
-} // end anonymous namespace
-
-char X86WinFixupBufferSecurityCheckPass::ID = 0;
-
-INITIALIZE_PASS(X86WinFixupBufferSecurityCheckPass, DEBUG_TYPE, DEBUG_TYPE,
-                false, false)
-
-FunctionPass *llvm::createX86WinFixupBufferSecurityCheckPass() {
-  return new X86WinFixupBufferSecurityCheckPass();
-}
-
-void X86WinFixupBufferSecurityCheckPass::SplitBasicBlock(
-    MachineBasicBlock *CurMBB, MachineBasicBlock *NewRetMBB,
-    MachineBasicBlock::iterator SplitIt) {
-  NewRetMBB->splice(NewRetMBB->end(), CurMBB, SplitIt, CurMBB->end());
-}
-
-std::pair<MachineBasicBlock *, MachineInstr *>
-X86WinFixupBufferSecurityCheckPass::getSecurityCheckerBasicBlock(
-    MachineFunction &MF) {
-  MachineBasicBlock::reverse_iterator RBegin, REnd;
-
-  for (auto &MBB : llvm::reverse(MF)) {
-    for (RBegin = MBB.rbegin(), REnd = MBB.rend(); RBegin != REnd; ++RBegin) {
-      auto &MI = *RBegin;
-      if (MI.getOpcode() == X86::CALL64pcrel32 &&
-          MI.getNumExplicitOperands() == 1) {
-        auto MO = MI.getOperand(0);
-        if (MO.isGlobal()) {
-          auto Callee = dyn_cast<Function>(MO.getGlobal());
-          if (Callee && Callee->getName() == "__security_check_cookie") {
-            return std::make_pair(&MBB, &MI);
-            break;
-          }
-        }
-      }
-    }
-  }
-  return std::make_pair(nullptr, nullptr);
-}
-
-void X86WinFixupBufferSecurityCheckPass::getGuardCheckSequence(
-    MachineBasicBlock *CurMBB, MachineInstr *CheckCall,
-    MachineInstr *SeqMI[5]) {
-
-  MachineBasicBlock::iterator UIt(CheckCall);
-  MachineBasicBlock::reverse_iterator DIt(CheckCall);
-  // Seq From StackUp to Stack Down Is fixed.
-  // ADJCALLSTACKUP64
-  ++UIt;
-  SeqMI[4] = &*UIt;
-
-  // CALL __security_check_cookie
-  SeqMI[3] = CheckCall;
-
-  // COPY function slot cookie
-  ++DIt;
-  SeqMI[2] = &*DIt;
-
-  // ADJCALLSTACKDOWN64
-  ++DIt;
-  SeqMI[1] = &*DIt;
-
-  MachineBasicBlock::reverse_iterator XIt(SeqMI[1]);
-  for (; XIt != CurMBB->rbegin(); ++XIt) {
-    auto &CI = *XIt;
-    if ((CI.getOpcode() == X86::XOR64_FP) || (CI.getOpcode() == X86::XOR32_FP))
-      break;
-  }
-  SeqMI[0] = &*XIt;
-}
-
-std::pair<MachineInstr *, MachineInstr *>
-X86WinFixupBufferSecurityCheckPass::CreateFailCheckSequence(
-    MachineBasicBlock *CurMBB, MachineBasicBlock *FailMBB,
-    MachineInstr *SeqMI[5]) {
-
-  auto MF = CurMBB->getParent();
-
-  Module &M = *MF->getFunction().getParent();
-  GlobalVariable *GV = M.getGlobalVariable("__security_cookie");
-  assert(GV && " Security Cookie was not installed!");
-
-  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
-
-  MachineInstr *GuardXor = SeqMI[0];
-  MachineBasicBlock::iterator InsertPt(GuardXor);
-  ++InsertPt;
-
-  // Compare security_Cookie with XOR_Val, if not same, we have violation
-  auto CMI = BuildMI(*CurMBB, InsertPt, DebugLoc(), TII->get(X86::CMP64rm))
-                 .addReg(GuardXor->getOperand(0).getReg())
-                 .addReg(X86::RIP)
-                 .addImm(1)
-                 .addReg(X86::NoRegister)
-                 .addGlobalAddress(GV)
-                 .addReg(X86::NoRegister);
-
-  BuildMI(*CurMBB, InsertPt, DebugLoc(), TII->get(X86::JCC_1))
-      .addMBB(FailMBB)
-      .addImm(X86::COND_NE);
-
-  auto JMI = BuildMI(*CurMBB, InsertPt, DebugLoc(), TII->get(X86::JMP_1));
-
-  return std::make_pair(CMI.getInstr(), JMI.getInstr());
-}
-
-void X86WinFixupBufferSecurityCheckPass::FinishBlock(MachineBasicBlock *MBB) {
-  LivePhysRegs LiveRegs;
-  computeAndAddLiveIns(LiveRegs, *MBB);
-}
-
-void X86WinFixupBufferSecurityCheckPass::FinishFunction(
-    MachineBasicBlock *FailMBB, MachineBasicBlock *NewRetMBB) {
-  FailMBB->getParent()->RenumberBlocks();
-  // FailMBB includes call to MSCV RT  where is __security_check_cookie
-  // function is called. This function uses regcall and it expects cookie
-  // value from stack slot.( even if this is modified)
-  // Before going further we compute back livein for this block to make sure
-  // it is live and provided.
-  FinishBlock(FailMBB);
-  FinishBlock(NewRetMBB);
-}
-
-bool X86WinFixupBufferSecurityCheckPass::runOnMachineFunction(
-    MachineFunction &MF) {
-  bool Changed = false;
-  const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
-
-  if (!(STI.isTargetWindowsItanium() || STI.isTargetWindowsMSVC()))
-    return Changed;
-
-  // Check if security cookie was installed or not
-  Module &M = *MF.getFunction().getParent();
-  GlobalVariable *GV = M.getGlobalVariable("__security_cookie");
-  if (!GV)
-    return Changed;
-
-  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
-
-  // Check if security check cookie was installed or not
-  auto [CurMBB, CheckCall] = getSecurityCheckerBasicBlock(MF);
-
-  if (!CheckCall)
-    return Changed;
-
-  MachineBasicBlock *FailMBB = MF.CreateMachineBasicBlock();
-  MachineBasicBlock *NewRetMBB = MF.CreateMachineBasicBlock();
-
-  MF.insert(MF.end(), NewRetMBB);
-  MF.insert(MF.end(), FailMBB);
-
-  MachineInstr *SeqMI[5];
-  getGuardCheckSequence(CurMBB, CheckCall, SeqMI);
-  // MachineInstr * GuardXor  = SeqMI[0];
-
-  auto FailSeqRange = CreateFailCheckSequence(CurMBB, FailMBB, SeqMI);
-  MachineInstrBuilder JMI(MF, FailSeqRange.second);
-
-  // After Inserting JMP_1, we can not have two terminators
-  // in same block, split CurrentMBB after JMP_1
-  MachineBasicBlock::iterator SplitIt(SeqMI[4]);
-  ++SplitIt;
-  SplitBasicBlock(CurMBB, NewRetMBB, SplitIt);
-
-  // Fill up Failure Routine, move Fail Check Squence from CurMBB to FailMBB
-  MachineBasicBlock::iterator U1It(SeqMI[1]);
-  MachineBasicBlock::iterator U2It(SeqMI[4]);
-  ++U2It;
-  FailMBB->splice(FailMBB->end(), CurMBB, U1It, U2It);
-  BuildMI(*FailMBB, FailMBB->end(), DebugLoc(), TII->get(X86::INT3));
-
-  // Move left over instruction after StackUp
-  // from Current Basic BLocks into New Return Block
-  JMI.addMBB(NewRetMBB);
-  MachineBasicBlock::iterator SplicePt(JMI.getInstr());
-  ++SplicePt;
-  if (SplicePt != CurMBB->end())
-    NewRetMBB->splice(NewRetMBB->end(), CurMBB, SplicePt);
-
-  // Restructure Basic Blocks
-  CurMBB->addSuccessor(NewRetMBB);
-  CurMBB->addSuccessor(FailMBB);
-
-  FinishFunction(FailMBB, NewRetMBB);
-  return !Changed;
-}
diff --git a/external/llvm-project/llvm/lib/Target/XCore/XCoreTargetTransformInfo.h b/external/llvm-project/llvm/lib/Target/XCore/XCoreTargetTransformInfo.h
index f2c10518109d..cb809b992396 100644
--- a/external/llvm-project/llvm/lib/Target/XCore/XCoreTargetTransformInfo.h
+++ b/external/llvm-project/llvm/lib/Target/XCore/XCoreTargetTransformInfo.h
@@ -24,7 +24,7 @@
 
 namespace llvm {
 
-class XCoreTTIImpl : public BasicTTIImplBase<XCoreTTIImpl> {
+class XCoreTTIImpl final : public BasicTTIImplBase<XCoreTTIImpl> {
   typedef BasicTTIImplBase<XCoreTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/external/llvm-project/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
index e0bbbc79b201..1f6cfec8edf4 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
@@ -393,9 +393,7 @@ bool XtensaAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
   case Xtensa::L32R: {
     const MCSymbolRefExpr *OpExpr =
         static_cast<const MCSymbolRefExpr *>(Inst.getOperand(1).getExpr());
-    XtensaMCExpr::Specifier Kind = XtensaMCExpr::VK_None;
-    const MCExpr *NewOpExpr = XtensaMCExpr::create(OpExpr, Kind, getContext());
-    Inst.getOperand(1).setExpr(NewOpExpr);
+    Inst.getOperand(1).setExpr(OpExpr);
     break;
   }
   case Xtensa::MOVI: {
@@ -413,10 +411,8 @@ bool XtensaAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
         const MCExpr *Value = MCConstantExpr::create(ImmOp64, getContext());
         MCSymbol *Sym = getContext().createTempSymbol();
         const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
-        const MCExpr *OpExpr =
-            XtensaMCExpr::create(Expr, XtensaMCExpr::VK_None, getContext());
         TmpInst.addOperand(Inst.getOperand(0));
-        MCOperand Op1 = MCOperand::createExpr(OpExpr);
+        MCOperand Op1 = MCOperand::createExpr(Expr);
         TmpInst.addOperand(Op1);
         TS.emitLiteral(Sym, Value, true, IDLoc);
         Inst = TmpInst;
@@ -428,10 +424,8 @@ bool XtensaAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
       const MCExpr *Value = Inst.getOperand(1).getExpr();
       MCSymbol *Sym = getContext().createTempSymbol();
       const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
-      const MCExpr *OpExpr =
-          XtensaMCExpr::create(Expr, XtensaMCExpr::VK_None, getContext());
       TmpInst.addOperand(Inst.getOperand(0));
-      MCOperand Op1 = MCOperand::createExpr(OpExpr);
+      MCOperand Op1 = MCOperand::createExpr(Expr);
       TmpInst.addOperand(Op1);
       Inst = TmpInst;
       TS.emitLiteral(Sym, Value, true, IDLoc);
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/CMakeLists.txt b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/CMakeLists.txt
index dc12863394c7..6c5a6bef5e24 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/CMakeLists.txt
@@ -4,7 +4,6 @@ add_llvm_component_library(LLVMXtensaDesc
   XtensaInstPrinter.cpp
   XtensaMCAsmInfo.cpp
   XtensaMCCodeEmitter.cpp
-  XtensaMCExpr.cpp
   XtensaMCTargetDesc.cpp
   XtensaTargetStreamer.cpp
 
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
index fc5e1780de2e..6f9f29765452 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
@@ -15,6 +15,7 @@
 #include "XtensaInstPrinter.h"
 #include "MCTargetDesc/XtensaMCExpr.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegister.h"
@@ -35,14 +36,7 @@ static void printExpr(const MCExpr *Expr, raw_ostream &OS) {
   if (!(SRE = cast<MCSymbolRefExpr>(Expr)))
     assert(false && "Unexpected MCExpr type.");
 
-  auto Spec = XtensaMCExpr::Specifier(SRE->getKind());
-  switch (Spec) {
-  case XtensaMCExpr::VK_None:
-    break;
-  // TODO
-  default:
-    report_fatal_error("Invalid kind!");
-  }
+  assert(SRE->getSpecifier() == 0);
 
   OS << SRE->getSymbol();
 
@@ -51,9 +45,6 @@ static void printExpr(const MCExpr *Expr, raw_ostream &OS) {
       OS << '+';
     OS << Offset;
   }
-
-  if (Spec != XtensaMCExpr::VK_None)
-    OS << ')';
 }
 
 void XtensaInstPrinter::printOperand(const MCOperand &MC, raw_ostream &O) {
@@ -97,7 +88,7 @@ void XtensaInstPrinter::printBranchTarget(const MCInst *MI, uint64_t Address,
     int64_t Val = MC.getImm() + 4;
     printPCRelImm(Address, Val, O);
   } else if (MC.isExpr())
-    MC.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MC.getExpr());
   else
     llvm_unreachable("Invalid operand");
 }
@@ -109,7 +100,7 @@ void XtensaInstPrinter::printLoopTarget(const MCInst *MI, uint64_t Address,
     int64_t Val = MC.getImm() + 4;
     printPCRelImm(Address, Val, O);
   } else if (MC.isExpr())
-    MC.getExpr()->print(O, &MAI, true);
+    MAI.printExpr(O, *MC.getExpr());
   else
     llvm_unreachable("Invalid operand");
 }
@@ -121,7 +112,7 @@ void XtensaInstPrinter::printJumpTarget(const MCInst *MI, uint64_t Address,
     int64_t Val = MC.getImm() + 4;
     printPCRelImm(Address, Val, O);
   } else if (MC.isExpr())
-    MC.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MC.getExpr());
   else
     llvm_unreachable("Invalid operand");
   ;
@@ -144,7 +135,7 @@ void XtensaInstPrinter::printCallOperand(const MCInst *MI, uint64_t Address,
       O << Val;
     }
   } else if (MC.isExpr())
-    MC.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MC.getExpr());
   else
     llvm_unreachable("Invalid operand");
 }
@@ -168,7 +159,7 @@ void XtensaInstPrinter::printL32RTarget(const MCInst *MI, uint64_t Address,
       printPCRelImm(Address, Value, O);
     }
   } else if (MC.isExpr())
-    MC.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MC.getExpr());
   else
     llvm_unreachable("Invalid operand");
 }
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp
index 28764d369247..0b20f2e14a84 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp
@@ -13,6 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "XtensaMCAsmInfo.h"
+#include "XtensaMCExpr.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
@@ -30,3 +32,22 @@ XtensaMCAsmInfo::XtensaMCAsmInfo(const Triple &TT) {
   ExceptionsType = ExceptionHandling::DwarfCFI;
   AlignmentIsInBytes = false;
 }
+
+void XtensaMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                         const MCSpecifierExpr &Expr) const {
+  StringRef S = Xtensa::getSpecifierName(Expr.getSpecifier());
+  if (!S.empty())
+    OS << '%' << S << '(';
+  printExpr(OS, *Expr.getSubExpr());
+  if (!S.empty())
+    OS << ')';
+}
+
+uint8_t Xtensa::parseSpecifier(StringRef name) { return 0; }
+
+StringRef Xtensa::getSpecifierName(uint8_t S) {
+  switch (S) {
+  default:
+    llvm_unreachable("Invalid ELF symbol kind");
+  }
+}
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h
index a86a95f6be37..6f6f4bcb7047 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h
@@ -23,6 +23,9 @@ class Triple;
 class XtensaMCAsmInfo : public MCAsmInfoELF {
 public:
   explicit XtensaMCAsmInfo(const Triple &TT);
+
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.cpp b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.cpp
deleted file mode 100644
index 4c4035b32af3..000000000000
--- a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-//===-- XtensaMCExpr.cpp - Xtensa specific MC expression classes ----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the implementation of the assembly expression modifiers
-// accepted by the Xtensa architecture
-//
-//===----------------------------------------------------------------------===//
-
-#include "XtensaMCExpr.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbolELF.h"
-#include "llvm/MC/MCValue.h"
-#include "llvm/Object/ELF.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "xtensamcexpr"
-
-const XtensaMCExpr *XtensaMCExpr::create(const MCExpr *Expr, Specifier S,
-                                         MCContext &Ctx) {
-  return new (Ctx) XtensaMCExpr(Expr, S);
-}
-
-void XtensaMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  bool HasSpecifier = getSpecifier() != VK_None;
-  if (HasSpecifier)
-    OS << '%' << getSpecifierName(getSpecifier()) << '(';
-  Expr->print(OS, MAI);
-  if (HasSpecifier)
-    OS << ')';
-}
-
-bool XtensaMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                             const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-  return !Res.getSubSym();
-}
-
-void XtensaMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-XtensaMCExpr::Specifier XtensaMCExpr::parseSpecifier(StringRef name) {
-  return StringSwitch<XtensaMCExpr::Specifier>(name).Default(VK_None);
-}
-
-StringRef XtensaMCExpr::getSpecifierName(Specifier S) {
-  switch (S) {
-  default:
-    llvm_unreachable("Invalid ELF symbol kind");
-  }
-}
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h
index 32354fef3ede..5a7b1ee9880f 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h
@@ -20,36 +20,13 @@
 namespace llvm {
 
 class StringRef;
-class XtensaMCExpr : public MCTargetExpr {
-public:
-  enum Specifier { VK_None, VK_TPOFF };
 
-private:
-  const MCExpr *Expr;
-  const Specifier specifier;
+namespace Xtensa {
+enum Specifier { S_None, S_TPOFF };
 
-  explicit XtensaMCExpr(const MCExpr *Expr, Specifier S)
-      : Expr(Expr), specifier(S) {}
-
-public:
-  static const XtensaMCExpr *create(const MCExpr *Expr, Specifier,
-                                    MCContext &Ctx);
-
-  Specifier getSpecifier() const { return specifier; }
-
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static Specifier parseSpecifier(StringRef name);
-  static StringRef getSpecifierName(Specifier Kind);
-};
+uint8_t parseSpecifier(StringRef name);
+StringRef getSpecifierName(uint8_t S);
+} // namespace Xtensa
 
 } // end namespace llvm.
 
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp b/external/llvm-project/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp
index 9182ea272bef..4e3ed4b9e8ee 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp
@@ -32,13 +32,13 @@
 
 using namespace llvm;
 
-static XtensaMCExpr::Specifier
+static Xtensa::Specifier
 getModifierSpecifier(XtensaCP::XtensaCPModifier Modifier) {
   switch (Modifier) {
   case XtensaCP::no_modifier:
-    return XtensaMCExpr::VK_None;
+    return Xtensa::S_None;
   case XtensaCP::TPOFF:
-    return XtensaMCExpr::VK_TPOFF;
+    return Xtensa::S_TPOFF;
   }
   report_fatal_error("Invalid XtensaCPModifier!");
 }
@@ -92,7 +92,7 @@ void XtensaAsmPrinter::emitMachineConstantPoolValue(
   MCSymbol *LblSym = GetCPISymbol(ACPV->getLabelId());
   auto *TS =
       static_cast<XtensaTargetStreamer *>(OutStreamer->getTargetStreamer());
-  XtensaMCExpr::Specifier VK = getModifierSpecifier(ACPV->getModifier());
+  auto Spec = getModifierSpecifier(ACPV->getModifier());
 
   if (ACPV->getModifier() != XtensaCP::no_modifier) {
     std::string SymName(MCSym->getName());
@@ -101,7 +101,7 @@ void XtensaAsmPrinter::emitMachineConstantPoolValue(
     MCSym = OutContext.getOrCreateSymbol(SymName);
   }
 
-  const MCExpr *Expr = MCSymbolRefExpr::create(MCSym, VK, OutContext);
+  const MCExpr *Expr = MCSymbolRefExpr::create(MCSym, Spec, OutContext);
   TS->emitLiteral(LblSym, Expr, false);
 }
 
@@ -227,8 +227,6 @@ XtensaAsmPrinter::LowerSymbolOperand(const MachineOperand &MO,
                                      MachineOperand::MachineOperandType MOTy,
                                      unsigned Offset) const {
   const MCSymbol *Symbol;
-  XtensaMCExpr::Specifier Kind = XtensaMCExpr::VK_None;
-
   switch (MOTy) {
   case MachineOperand::MO_GlobalAddress:
     Symbol = getSymbol(MO.getGlobal());
@@ -257,8 +255,6 @@ XtensaAsmPrinter::LowerSymbolOperand(const MachineOperand &MO,
   }
 
   const MCExpr *ME = MCSymbolRefExpr::create(Symbol, OutContext);
-  ME = XtensaMCExpr::create(ME, Kind, OutContext);
-
   if (Offset) {
     // Assume offset is never negative.
     assert(Offset > 0);
diff --git a/external/llvm-project/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp b/external/llvm-project/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp
index 06cccd4831bf..959553de9939 100644
--- a/external/llvm-project/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp
+++ b/external/llvm-project/llvm/lib/Target/Xtensa/XtensaISelDAGToDAG.cpp
@@ -59,10 +59,9 @@ class XtensaDAGToDAGISel : public SelectionDAGISel {
     }
 
     if (TM.isPositionIndependent()) {
-      DiagnosticInfoUnsupported Diag(CurDAG->getMachineFunction().getFunction(),
-                                     "PIC relocations are not supported ",
-                                     Addr.getDebugLoc());
-      CurDAG->getContext()->diagnose(Diag);
+      CurDAG->getContext()->diagnose(DiagnosticInfoUnsupported(
+          CurDAG->getMachineFunction().getFunction(),
+          "PIC relocations are not supported", Addr.getDebugLoc()));
     }
 
     if ((Addr.getOpcode() == ISD::TargetExternalSymbol ||
diff --git a/external/llvm-project/llvm/lib/TargetParser/Host.cpp b/external/llvm-project/llvm/lib/TargetParser/Host.cpp
index 09d298c03d1a..b96a06544888 100644
--- a/external/llvm-project/llvm/lib/TargetParser/Host.cpp
+++ b/external/llvm-project/llvm/lib/TargetParser/Host.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/TargetParser/X86TargetParser.h"
 #include <string.h>
@@ -1669,8 +1670,32 @@ StringRef sys::getHostCPUName() {
   return "generic";
 }
 #elif defined(__riscv)
+#if defined(__linux__)
+// struct riscv_hwprobe
+struct RISCVHwProbe {
+  int64_t Key;
+  uint64_t Value;
+};
+#endif
+
 StringRef sys::getHostCPUName() {
 #if defined(__linux__)
+  // Try the hwprobe way first.
+  RISCVHwProbe Query[]{{/*RISCV_HWPROBE_KEY_MVENDORID=*/0, 0},
+                       {/*RISCV_HWPROBE_KEY_MARCHID=*/1, 0},
+                       {/*RISCV_HWPROBE_KEY_MIMPID=*/2, 0}};
+  int Ret = syscall(/*__NR_riscv_hwprobe=*/258, /*pairs=*/Query,
+                    /*pair_count=*/std::size(Query), /*cpu_count=*/0,
+                    /*cpus=*/0, /*flags=*/0);
+  if (Ret == 0) {
+    RISCV::CPUModel Model{static_cast<uint32_t>(Query[0].Value), Query[1].Value,
+                          Query[2].Value};
+    StringRef Name = RISCV::getCPUNameFromCPUModel(Model);
+    if (!Name.empty())
+      return Name;
+  }
+
+  // Then try the cpuinfo way.
   std::unique_ptr<llvm::MemoryBuffer> P = getProcCpuinfoContent();
   StringRef Content = P ? P->getBuffer() : "";
   StringRef Name = detail::getHostCPUNameForRISCV(Content);
@@ -2141,11 +2166,6 @@ const StringMap<bool> sys::getHostCPUFeatures() {
   return Features;
 }
 #elif defined(__linux__) && defined(__riscv)
-// struct riscv_hwprobe
-struct RISCVHwProbe {
-  int64_t Key;
-  uint64_t Value;
-};
 const StringMap<bool> sys::getHostCPUFeatures() {
   RISCVHwProbe Query[]{{/*RISCV_HWPROBE_KEY_BASE_BEHAVIOR=*/3, 0},
                        {/*RISCV_HWPROBE_KEY_IMA_EXT_0=*/4, 0},
diff --git a/external/llvm-project/llvm/lib/TargetParser/LoongArchTargetParser.cpp b/external/llvm-project/llvm/lib/TargetParser/LoongArchTargetParser.cpp
index e394c0c15b20..db68498609b5 100644
--- a/external/llvm-project/llvm/lib/TargetParser/LoongArchTargetParser.cpp
+++ b/external/llvm-project/llvm/lib/TargetParser/LoongArchTargetParser.cpp
@@ -34,6 +34,18 @@ bool LoongArch::isValidArchName(StringRef Arch) {
   return false;
 }
 
+bool LoongArch::isValidFeatureName(StringRef Feature) {
+  if (Feature.starts_with("+") || Feature.starts_with("-"))
+    return false;
+  for (const auto &F : AllFeatures) {
+    StringRef CanonicalName =
+        F.Name.starts_with("+") ? F.Name.drop_front() : F.Name;
+    if (CanonicalName == Feature)
+      return true;
+  }
+  return false;
+}
+
 bool LoongArch::getArchFeatures(StringRef Arch,
                                 std::vector<StringRef> &Features) {
   for (const auto A : AllArchs) {
diff --git a/external/llvm-project/llvm/lib/TargetParser/RISCVISAInfo.cpp b/external/llvm-project/llvm/lib/TargetParser/RISCVISAInfo.cpp
index e76ddd4b648d..17c98332ab0a 100644
--- a/external/llvm-project/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/external/llvm-project/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -449,8 +449,7 @@ RISCVISAInfo::parseFeatures(unsigned XLen,
   assert(XLen == 32 || XLen == 64);
   std::unique_ptr<RISCVISAInfo> ISAInfo(new RISCVISAInfo(XLen));
 
-  for (auto &Feature : Features) {
-    StringRef ExtName = Feature;
+  for (StringRef ExtName : Features) {
     assert(ExtName.size() > 1 && (ExtName[0] == '+' || ExtName[0] == '-'));
     bool Add = ExtName[0] == '+';
     ExtName = ExtName.drop_front(1); // Drop '+' or '-'
diff --git a/external/llvm-project/llvm/lib/TargetParser/RISCVTargetParser.cpp b/external/llvm-project/llvm/lib/TargetParser/RISCVTargetParser.cpp
index 2e5e8f4e50c9..9957ec0c28d8 100644
--- a/external/llvm-project/llvm/lib/TargetParser/RISCVTargetParser.cpp
+++ b/external/llvm-project/llvm/lib/TargetParser/RISCVTargetParser.cpp
@@ -57,10 +57,7 @@ bool hasFastVectorUnalignedAccess(StringRef CPU) {
   return Info && Info->FastVectorUnalignedAccess;
 }
 
-bool hasValidCPUModel(StringRef CPU) {
-  const CPUModel Model = getCPUModel(CPU);
-  return Model.MVendorID != 0 && Model.MArchID != 0 && Model.MImpID != 0;
-}
+bool hasValidCPUModel(StringRef CPU) { return getCPUModel(CPU).isValid(); }
 
 CPUModel getCPUModel(StringRef CPU) {
   const CPUInfo *Info = getCPUInfoByName(CPU);
@@ -69,6 +66,16 @@ CPUModel getCPUModel(StringRef CPU) {
   return Info->Model;
 }
 
+StringRef getCPUNameFromCPUModel(const CPUModel &Model) {
+  if (!Model.isValid())
+    return "";
+
+  for (auto &C : RISCVCPUInfo)
+    if (C.Model == Model)
+      return C.Name;
+  return "";
+}
+
 bool parseCPU(StringRef CPU, bool IsRV64) {
   const CPUInfo *Info = getCPUInfoByName(CPU);
 
diff --git a/external/llvm-project/llvm/lib/TargetParser/Triple.cpp b/external/llvm-project/llvm/lib/TargetParser/Triple.cpp
index 6a559ff023ca..5718ae385bac 100644
--- a/external/llvm-project/llvm/lib/TargetParser/Triple.cpp
+++ b/external/llvm-project/llvm/lib/TargetParser/Triple.cpp
@@ -1446,9 +1446,12 @@ bool Triple::getMacOSXVersion(VersionTuple &Version) const {
     }
     if (Version.getMajor() <= 19) {
       Version = VersionTuple(10, Version.getMajor() - 4);
-    } else {
-      // darwin20+ corresponds to macOS 11+.
+    } else if (Version.getMajor() < 25) {
+      // darwin20-24 corresponds to macOS 11-15.
       Version = VersionTuple(11 + Version.getMajor() - 20);
+    } else {
+      // darwin25 corresponds with macOS26+.
+      Version = VersionTuple(Version.getMajor() + 1);
     }
     break;
   case MacOSX:
@@ -1492,14 +1495,33 @@ VersionTuple Triple::getiOSVersion() const {
     // Default to 5.0 (or 7.0 for arm64).
     if (Version.getMajor() == 0)
       return (getArch() == aarch64) ? VersionTuple(7) : VersionTuple(5);
-    return Version;
+    if (Version.getMajor() == 19)
+      // tvOS 19 corresponds to ios26.
+      return VersionTuple(26);
+    return getCanonicalVersionForOS(OSType::IOS, Version,
+                                    isValidVersionForOS(OSType::IOS, Version));
   }
   case XROS: {
+    VersionTuple Version = getOSVersion();
     // xrOS 1 is aligned with iOS 17.
+    if (Version.getMajor() < 3)
+      return Version.withMajorReplaced(Version.getMajor() + 16);
+    // visionOS 3 corresponds to ios 26+.
+    if (Version.getMajor() == 3)
+      return VersionTuple(26);
+    return getCanonicalVersionForOS(OSType::XROS, Version,
+                                    isValidVersionForOS(OSType::XROS, Version));
+  }
+  case WatchOS: {
     VersionTuple Version = getOSVersion();
-    return Version.withMajorReplaced(Version.getMajor() + 16);
+    // watchOS 12 corresponds to ios 26.
+    if (Version.getMajor() == 12)
+      return VersionTuple(26);
+    return getCanonicalVersionForOS(
+        OSType::WatchOS, Version,
+        isValidVersionForOS(OSType::WatchOS, Version));
   }
-  case WatchOS:
+  case BridgeOS:
     llvm_unreachable("conflicting triple info");
   case DriverKit:
     llvm_unreachable("DriverKit doesn't have an iOS version");
@@ -1732,8 +1754,6 @@ unsigned Triple::getTrampolineSize() const {
     if (isOSLinux())
       return 48;
     break;
-  case Triple::aarch64:
-    return 36;
   }
   return 32;
 }
@@ -2116,12 +2136,12 @@ bool Triple::isMacOSXVersionLT(unsigned Major, unsigned Minor,
     return isOSVersionLT(Major, Minor, Micro);
 
   // Otherwise, compare to the "Darwin" number.
-  if (Major == 10) {
+  if (Major == 10)
     return isOSVersionLT(Minor + 4, Micro, 0);
-  } else {
-    assert(Major >= 11 && "Unexpected major version");
+  assert(Major >= 11 && "Unexpected major version");
+  if (Major < 25)
     return isOSVersionLT(Major - 11 + 20, Minor, Micro);
-  }
+  return isOSVersionLT(Major + 1, Minor, Micro);
 }
 
 VersionTuple Triple::getMinimumSupportedOSVersion() const {
@@ -2149,7 +2169,10 @@ VersionTuple Triple::getMinimumSupportedOSVersion() const {
     // ARM64 simulators are supported for watchOS 7+.
     if (isSimulatorEnvironment())
       return VersionTuple(7, 0, 0);
-    break;
+    // ARM64/ARM64e slices are supported starting from watchOS 26.
+    // ARM64_32 is older though.
+    assert(getArch() != Triple::aarch64_32);
+    return VersionTuple(26, 0, 0);
   case Triple::DriverKit:
     return VersionTuple(20, 0, 0);
   default:
@@ -2159,16 +2182,85 @@ VersionTuple Triple::getMinimumSupportedOSVersion() const {
 }
 
 VersionTuple Triple::getCanonicalVersionForOS(OSType OSKind,
-                                              const VersionTuple &Version) {
+                                              const VersionTuple &Version,
+                                              bool IsInValidRange) {
+  const unsigned MacOSRangeBump = 10;
+  const unsigned IOSRangeBump = 7;
+  const unsigned XROSRangeBump = 23;
+  const unsigned WatchOSRangeBump = 14;
   switch (OSKind) {
-  case MacOSX:
+  case MacOSX: {
     // macOS 10.16 is canonicalized to macOS 11.
     if (Version == VersionTuple(10, 16))
       return VersionTuple(11, 0);
-    [[fallthrough]];
+    // macOS 16 is canonicalized to macOS 26.
+    if (Version == VersionTuple(16, 0))
+      return VersionTuple(26, 0);
+    if (!IsInValidRange)
+      return Version.withMajorReplaced(Version.getMajor() + MacOSRangeBump);
+    break;
+  }
+  case IOS:
+  case TvOS: {
+    // Both iOS & tvOS 19.0 canonicalize to 26.
+    if (Version == VersionTuple(19, 0))
+      return VersionTuple(26, 0);
+    if (!IsInValidRange)
+      return Version.withMajorReplaced(Version.getMajor() + IOSRangeBump);
+    break;
+  }
+  case XROS: {
+    // visionOS3 is canonicalized to 26.
+    if (Version == VersionTuple(3, 0))
+      return VersionTuple(26, 0);
+    if (!IsInValidRange)
+      return Version.withMajorReplaced(Version.getMajor() + XROSRangeBump);
+    break;
+  }
+  case WatchOS: {
+    // watchOS 12 is canonicalized to 26.
+    if (Version == VersionTuple(12, 0))
+      return VersionTuple(26, 0);
+    if (!IsInValidRange)
+      return Version.withMajorReplaced(Version.getMajor() + WatchOSRangeBump);
+    break;
+  }
   default:
     return Version;
   }
+
+  return Version;
+}
+
+bool Triple::isValidVersionForOS(OSType OSKind, const VersionTuple &Version) {
+  /// This constant is used to capture gaps in versioning.
+  const VersionTuple CommonVersion(26);
+  auto IsValid = [&](const VersionTuple &StartingVersion) {
+    return !((Version > StartingVersion) && (Version < CommonVersion));
+  };
+  switch (OSKind) {
+  case WatchOS: {
+    const VersionTuple StartingWatchOS(12);
+    return IsValid(StartingWatchOS);
+  }
+  case IOS:
+  case TvOS: {
+    const VersionTuple StartingIOS(19);
+    return IsValid(StartingIOS);
+  }
+  case MacOSX: {
+    const VersionTuple StartingMacOS(16);
+    return IsValid(StartingMacOS);
+  }
+  case XROS: {
+    const VersionTuple StartingXROS(3);
+    return IsValid(StartingXROS);
+  }
+  default:
+    return true;
+  }
+
+  llvm_unreachable("unexpected or invalid os version");
 }
 
 // HLSL triple environment orders are relied on in the front end
diff --git a/external/llvm-project/llvm/lib/Testing/Support/SupportHelpers.cpp b/external/llvm-project/llvm/lib/Testing/Support/SupportHelpers.cpp
index 3c12edfa9d99..e9e44fd11510 100644
--- a/external/llvm-project/llvm/lib/Testing/Support/SupportHelpers.cpp
+++ b/external/llvm-project/llvm/lib/Testing/Support/SupportHelpers.cpp
@@ -3,7 +3,6 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
diff --git a/external/llvm-project/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp b/external/llvm-project/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp
index ec0271f774ec..cda07e81faf1 100644
--- a/external/llvm-project/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp
+++ b/external/llvm-project/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp
@@ -22,7 +22,6 @@
 #include "llvm/TextAPI/RecordsSlice.h"
 #include "llvm/TextAPI/TextAPIError.h"
 #include <iomanip>
-#include <set>
 #include <sstream>
 #include <string>
 #include <tuple>
diff --git a/external/llvm-project/llvm/lib/TextAPI/InterfaceFile.cpp b/external/llvm-project/llvm/lib/TextAPI/InterfaceFile.cpp
index 49e6fa673fd8..a201554de9e6 100644
--- a/external/llvm-project/llvm/lib/TextAPI/InterfaceFile.cpp
+++ b/external/llvm-project/llvm/lib/TextAPI/InterfaceFile.cpp
@@ -13,8 +13,6 @@
 #include "llvm/TextAPI/InterfaceFile.h"
 #include "llvm/TextAPI/RecordsSlice.h"
 #include "llvm/TextAPI/TextAPIError.h"
-#include <iomanip>
-#include <sstream>
 
 using namespace llvm;
 using namespace llvm::MachO;
diff --git a/external/llvm-project/llvm/lib/TextAPI/Platform.cpp b/external/llvm-project/llvm/lib/TextAPI/Platform.cpp
index d7d391102ce6..d5c4e0a974d0 100644
--- a/external/llvm-project/llvm/lib/TextAPI/Platform.cpp
+++ b/external/llvm-project/llvm/lib/TextAPI/Platform.cpp
@@ -124,7 +124,9 @@ VersionTuple mapToSupportedOSVersion(const Triple &Triple) {
   const VersionTuple MinSupportedOS = Triple.getMinimumSupportedOSVersion();
   if (MinSupportedOS > Triple.getOSVersion())
     return MinSupportedOS;
-  return Triple.getOSVersion();
+  return Triple::getCanonicalVersionForOS(
+      Triple.getOS(), Triple.getOSVersion(),
+      Triple::isValidVersionForOS(Triple.getOS(), Triple.getOSVersion()));
 }
 
 } // end namespace MachO.
diff --git a/external/llvm-project/llvm/lib/TextAPI/RecordsSlice.cpp b/external/llvm-project/llvm/lib/TextAPI/RecordsSlice.cpp
index 1500b8752115..1d04b6360652 100644
--- a/external/llvm-project/llvm/lib/TextAPI/RecordsSlice.cpp
+++ b/external/llvm-project/llvm/lib/TextAPI/RecordsSlice.cpp
@@ -259,7 +259,7 @@ ObjCInterfaceRecord::getObjCCategories() const {
 
 ObjCIVarRecord *ObjCContainerRecord::addObjCIVar(StringRef IVar,
                                                  RecordLinkage Linkage) {
-  auto Result = IVars.insert({IVar, nullptr});
+  auto Result = IVars.try_emplace(IVar);
   if (Result.second)
     Result.first->second = std::make_unique<ObjCIVarRecord>(IVar, Linkage);
   return Result.first->second.get();
diff --git a/external/llvm-project/llvm/lib/TextAPI/TextStub.cpp b/external/llvm-project/llvm/lib/TextAPI/TextStub.cpp
index d903ba409360..be9be07a2aa0 100644
--- a/external/llvm-project/llvm/lib/TextAPI/TextStub.cpp
+++ b/external/llvm-project/llvm/lib/TextAPI/TextStub.cpp
@@ -25,7 +25,6 @@
 #include "llvm/TextAPI/PackedVersion.h"
 #include "llvm/TextAPI/TextAPIReader.h"
 #include "llvm/TextAPI/TextAPIWriter.h"
-#include <algorithm>
 #include <set>
 
 // clang-format off
diff --git a/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index c128687062ad..f62361d33470 100644
--- a/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -328,15 +328,33 @@ static bool tryToRecognizePopCount(Instruction &I) {
                               m_SpecificInt(Mask33))))) {
         Value *Root, *SubOp1;
         // Matching "i - ((i >> 1) & 0x55555555...)".
+        const APInt *AndMask;
         if (match(AndOp0, m_Sub(m_Value(Root), m_Value(SubOp1))) &&
             match(SubOp1, m_And(m_LShr(m_Specific(Root), m_SpecificInt(1)),
-                                m_SpecificInt(Mask55)))) {
-          LLVM_DEBUG(dbgs() << "Recognized popcount intrinsic\n");
-          IRBuilder<> Builder(&I);
-          I.replaceAllUsesWith(
-              Builder.CreateIntrinsic(Intrinsic::ctpop, I.getType(), {Root}));
-          ++NumPopCountRecognized;
-          return true;
+                                m_APInt(AndMask)))) {
+          auto CheckAndMask = [&]() {
+            if (*AndMask == Mask55)
+              return true;
+
+            // Exact match failed, see if any bits are known to be 0 where we
+            // expect a 1 in the mask.
+            if (!AndMask->isSubsetOf(Mask55))
+              return false;
+
+            APInt NeededMask = Mask55 & ~*AndMask;
+            return MaskedValueIsZero(cast<Instruction>(SubOp1)->getOperand(0),
+                                     NeededMask,
+                                     SimplifyQuery(I.getDataLayout()));
+          };
+
+          if (CheckAndMask()) {
+            LLVM_DEBUG(dbgs() << "Recognized popcount intrinsic\n");
+            IRBuilder<> Builder(&I);
+            I.replaceAllUsesWith(
+                Builder.CreateIntrinsic(Intrinsic::ctpop, I.getType(), {Root}));
+            ++NumPopCountRecognized;
+            return true;
+          }
         }
       }
     }
@@ -422,8 +440,7 @@ static bool foldSqrt(CallInst *Call, LibFunc Func, TargetTransformInfo &TTI,
   if (TTI.haveFastSqrt(Ty) &&
       (Call->hasNoNaNs() ||
        cannotBeOrderedLessThanZero(
-           Arg, 0,
-           SimplifyQuery(Call->getDataLayout(), &TLI, &DT, &AC, Call)))) {
+           Arg, SimplifyQuery(Call->getDataLayout(), &TLI, &DT, &AC, Call)))) {
     IRBuilder<> Builder(Call);
     Value *NewSqrt =
         Builder.CreateIntrinsic(Intrinsic::sqrt, Ty, Arg, Call, "sqrt");
diff --git a/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h b/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
index 28a0181392ef..0b8a9fa48e34 100644
--- a/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
+++ b/external/llvm-project/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
@@ -107,15 +107,14 @@ class TruncInstCombine {
   Type *getBestTruncatedType();
 
   KnownBits computeKnownBits(const Value *V) const {
-    return llvm::computeKnownBits(V, DL, /*Depth=*/0, &AC,
+    return llvm::computeKnownBits(V, DL, &AC,
                                   /*CtxI=*/cast<Instruction>(CurrentTruncInst),
                                   &DT);
   }
 
   unsigned ComputeNumSignBits(const Value *V) const {
     return llvm::ComputeNumSignBits(
-        V, DL, /*Depth=*/0, &AC, /*CtxI=*/cast<Instruction>(CurrentTruncInst),
-        &DT);
+        V, DL, &AC, /*CtxI=*/cast<Instruction>(CurrentTruncInst), &DT);
   }
 
   /// Given a \p V value and a \p SclTy scalar type return the generated reduced
diff --git a/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index eea6dfba14e3..9ac145b10fe4 100644
--- a/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -131,9 +131,10 @@ void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
         StructType::create({FnPtrTy, FnPtrTy}, "NoopCoro.Frame");
 
     // Create a Noop function that does nothing.
-    Function *NoopFn =
-        Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
-                         "__NoopCoro_ResumeDestroy", &M);
+    Function *NoopFn = Function::createWithDefaultAttr(
+        FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
+        M.getDataLayout().getProgramAddressSpace(), "__NoopCoro_ResumeDestroy",
+        &M);
     NoopFn->setCallingConv(CallingConv::Fast);
     buildDebugInfoForNoopResumeDestroyFunc(NoopFn);
     auto *Entry = BasicBlock::Create(C, "entry", NoopFn);
diff --git a/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 0f7e04b53edf..9210c81cb835 100644
--- a/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -26,6 +26,7 @@
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/OptimizedStructLayout.h"
 #include "llvm/Transforms/Coroutines/ABI.h"
@@ -41,8 +42,6 @@
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 #define DEBUG_TYPE "coro-frame"
 
 namespace {
@@ -842,18 +841,12 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
       DILocation::get(DIS->getContext(), LineNum, /*Column=*/1, DIS);
   assert(FrameDIVar->isValidLocationForIntrinsic(DILoc));
 
-  if (UseNewDbgInfoFormat) {
-    DbgVariableRecord *NewDVR =
-        new DbgVariableRecord(ValueAsMetadata::get(Shape.FramePtr), FrameDIVar,
-                              DBuilder.createExpression(), DILoc,
-                              DbgVariableRecord::LocationType::Declare);
-    BasicBlock::iterator It = Shape.getInsertPtAfterFramePtr();
-    It->getParent()->insertDbgRecordBefore(NewDVR, It);
-  } else {
-    DBuilder.insertDeclare(Shape.FramePtr, FrameDIVar,
-                           DBuilder.createExpression(), DILoc,
-                           Shape.getInsertPtAfterFramePtr());
-  }
+  DbgVariableRecord *NewDVR =
+      new DbgVariableRecord(ValueAsMetadata::get(Shape.FramePtr), FrameDIVar,
+                            DBuilder.createExpression(), DILoc,
+                            DbgVariableRecord::LocationType::Declare);
+  BasicBlock::iterator It = Shape.getInsertPtAfterFramePtr();
+  It->getParent()->insertDbgRecordBefore(NewDVR, It);
 }
 
 // Build a struct that will keep state for an active coroutine.
@@ -1132,23 +1125,15 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
         }
 
         auto SalvageOne = [&](auto *DDI) {
-          bool AllowUnresolved = false;
           // This dbg.declare is preserved for all coro-split function
           // fragments. It will be unreachable in the main function, and
           // processed by coro::salvageDebugInfo() by the Cloner.
-          if (UseNewDbgInfoFormat) {
-            DbgVariableRecord *NewDVR = new DbgVariableRecord(
-                ValueAsMetadata::get(CurrentReload), DDI->getVariable(),
-                DDI->getExpression(), DDI->getDebugLoc(),
-                DbgVariableRecord::LocationType::Declare);
-            Builder.GetInsertPoint()->getParent()->insertDbgRecordBefore(
-                NewDVR, Builder.GetInsertPoint());
-          } else {
-            DIBuilder(*CurrentBlock->getParent()->getParent(), AllowUnresolved)
-                .insertDeclare(CurrentReload, DDI->getVariable(),
-                               DDI->getExpression(), DDI->getDebugLoc(),
-                               Builder.GetInsertPoint());
-          }
+          DbgVariableRecord *NewDVR = new DbgVariableRecord(
+              ValueAsMetadata::get(CurrentReload), DDI->getVariable(),
+              DDI->getExpression(), DDI->getDebugLoc(),
+              DbgVariableRecord::LocationType::Declare);
+          Builder.GetInsertPoint()->getParent()->insertDbgRecordBefore(
+              NewDVR, Builder.GetInsertPoint());
           // This dbg.declare is for the main function entry point.  It
           // will be deleted in all coro-split functions.
           coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
@@ -1215,11 +1200,17 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
   for (const auto &A : FrameData.Allocas) {
     AllocaInst *Alloca = A.Alloca;
     UsersToUpdate.clear();
-    for (User *U : Alloca->users()) {
+    for (User *U : make_early_inc_range(Alloca->users())) {
       auto *I = cast<Instruction>(U);
-      if (DT.dominates(Shape.CoroBegin, I))
+      // It is meaningless to retain the lifetime intrinsics refer for the
+      // member of coroutine frames and the meaningless lifetime intrinsics
+      // are possible to block further optimizations.
+      if (I->isLifetimeStartOrEnd())
+        I->eraseFromParent();
+      else if (DT.dominates(Shape.CoroBegin, I))
         UsersToUpdate.push_back(I);
     }
+
     if (UsersToUpdate.empty())
       continue;
     auto *G = GetFramePointer(Alloca);
@@ -1233,17 +1224,8 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
     for (auto *DVR : DbgVariableRecords)
       DVR->replaceVariableLocationOp(Alloca, G);
 
-    for (Instruction *I : UsersToUpdate) {
-      // It is meaningless to retain the lifetime intrinsics refer for the
-      // member of coroutine frames and the meaningless lifetime intrinsics
-      // are possible to block further optimizations.
-      if (I->isLifetimeStartOrEnd()) {
-        I->eraseFromParent();
-        continue;
-      }
-
+    for (Instruction *I : UsersToUpdate)
       I->replaceUsesOfWith(Alloca, G);
-    }
   }
   Builder.SetInsertPoint(&*Shape.getInsertPtAfterFramePtr());
   for (const auto &A : FrameData.Allocas) {
diff --git a/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index f9a6c70fedc2..8813f91e9060 100644
--- a/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -703,6 +703,7 @@ void coro::BaseCloner::replaceEntryBlock() {
     auto *SwitchBB =
         cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
     Builder.CreateBr(SwitchBB);
+    SwitchBB->moveAfter(Entry);
     break;
   }
   case coro::ABI::Async:
@@ -912,29 +913,14 @@ void coro::BaseCloner::create() {
     assert(SP != OrigF.getSubprogram() && SP->isDistinct());
     updateScopeLine(ActiveSuspend, *SP);
 
-    // Update the linkage name to reflect the modified symbol name. It
-    // is necessary to update the linkage name in Swift, since the
-    // mangling changes for resume functions. It might also be the
-    // right thing to do in C++, but due to a limitation in LLVM's
-    // AsmPrinter we can only do this if the function doesn't have an
-    // abstract specification, since the DWARF backend expects the
-    // abstract specification to contain the linkage name and asserts
-    // that they are identical.
-    if (SP->getUnit() &&
-        SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift) {
-      SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
-      if (auto *Decl = SP->getDeclaration()) {
-        auto *NewDecl = DISubprogram::get(
-            Decl->getContext(), Decl->getScope(), Decl->getName(),
-            NewF->getName(), Decl->getFile(), Decl->getLine(), Decl->getType(),
-            Decl->getScopeLine(), Decl->getContainingType(),
-            Decl->getVirtualIndex(), Decl->getThisAdjustment(),
-            Decl->getFlags(), Decl->getSPFlags(), Decl->getUnit(),
-            Decl->getTemplateParams(), nullptr, Decl->getRetainedNodes(),
-            Decl->getThrownTypes(), Decl->getAnnotations(),
-            Decl->getTargetFuncName());
-        SP->replaceDeclaration(NewDecl);
-      }
+    // Update the linkage name and the function name to reflect the modified
+    // name.
+    MDString *NewLinkageName = MDString::get(Context, NewF->getName());
+    SP->replaceLinkageName(NewLinkageName);
+    if (DISubprogram *Decl = SP->getDeclaration()) {
+      TempDISubprogram NewDecl = Decl->clone();
+      NewDecl->replaceLinkageName(NewLinkageName);
+      SP->replaceDeclaration(MDNode::replaceWithUniqued(std::move(NewDecl)));
     }
   }
 
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 0ec5202b8cfe..262c902d40d2 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -179,7 +179,6 @@ doPromotion(Function *F, FunctionAnalysisManager &FAM,
                                   F->getName());
   NF->copyAttributesFrom(F);
   NF->copyMetadata(F, 0);
-  NF->setIsNewDbgInfoFormat(F->IsNewDbgInfoFormat);
 
   // The new function will have the !dbg metadata copied from the original
   // function. The original function may not be deleted, and dbg metadata need
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/Attributor.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/Attributor.cpp
index cbdbf9ae1494..dac1f7a30c37 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -242,8 +242,6 @@ Constant *
 AA::getInitialValueForObj(Attributor &A, const AbstractAttribute &QueryingAA,
                           Value &Obj, Type &Ty, const TargetLibraryInfo *TLI,
                           const DataLayout &DL, AA::RangeTy *RangePtr) {
-  if (isa<AllocaInst>(Obj))
-    return UndefValue::get(&Ty);
   if (Constant *Init = getInitialValueOfAllocation(&Obj, TLI, &Ty))
     return Init;
   auto *GV = dyn_cast<GlobalVariable>(&Obj);
@@ -2726,8 +2724,6 @@ void Attributor::createShallowWrapper(Function &F) {
       Function::Create(FnTy, F.getLinkage(), F.getAddressSpace(), F.getName());
   F.setName(""); // set the inside function anonymous
   M.getFunctionList().insert(F.getIterator(), Wrapper);
-  // Flag whether the function is using new-debug-info or not.
-  Wrapper->IsNewDbgInfoFormat = M.IsNewDbgInfoFormat;
 
   F.setLinkage(GlobalValue::InternalLinkage);
 
@@ -2808,8 +2804,6 @@ bool Attributor::internalizeFunctions(SmallPtrSetImpl<Function *> &FnSet,
       VMap[&Arg] = &(*NewFArgIt++);
     }
     SmallVector<ReturnInst *, 8> Returns;
-    // Flag whether the function is using new-debug-info or not.
-    Copied->IsNewDbgInfoFormat = F->IsNewDbgInfoFormat;
 
     // Copy the body of the original function to the new one
     CloneFunctionInto(Copied, F, VMap,
@@ -3027,8 +3021,6 @@ ChangeStatus Attributor::rewriteFunctionSignatures(
     OldFn->getParent()->getFunctionList().insert(OldFn->getIterator(), NewFn);
     NewFn->takeName(OldFn);
     NewFn->copyAttributesFrom(OldFn);
-    // Flag whether the function is using new-debug-info or not.
-    NewFn->IsNewDbgInfoFormat = OldFn->IsNewDbgInfoFormat;
 
     // Patch the pointer to LLVM function in debug info descriptor.
     NewFn->setSubprogram(OldFn->getSubprogram());
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 482ac7b3a8cf..bf1a48936e7d 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -626,8 +626,7 @@ static void followUsesInContext(AAType &AA, Attributor &A,
     if (const Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {
       bool Found = Explorer.findInContextOf(UserI, EIt, EEnd);
       if (Found && AA.followUseInMBEC(A, U, UserI, State))
-        for (const Use &Us : UserI->uses())
-          Uses.insert(&Us);
+        Uses.insert_range(llvm::make_pointer_range(UserI->uses()));
     }
   }
 }
@@ -12603,19 +12602,38 @@ struct AAAddressSpaceImpl : public AAAddressSpace {
 
   ChangeStatus updateImpl(Attributor &A) override {
     uint32_t OldAddressSpace = AssumedAddressSpace;
+    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
 
     auto CheckAddressSpace = [&](Value &Obj) {
+      // Ignore undef.
       if (isa<UndefValue>(&Obj))
         return true;
-      if (auto *Arg = dyn_cast<Argument>(&Obj)) {
+
+      // If the object already has a non-flat address space, we simply take it.
+      unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
+      if (ObjAS != FlatAS)
+        return takeAddressSpace(ObjAS);
+
+      // At this point, we know Obj is in the flat address space. For a final
+      // attempt, we want to use getAssumedAddrSpace, but first we must get the
+      // associated function, if possible.
+      Function *F = nullptr;
+      if (auto *Arg = dyn_cast<Argument>(&Obj))
+        F = Arg->getParent();
+      else if (auto *I = dyn_cast<Instruction>(&Obj))
+        F = I->getFunction();
+
+      // Use getAssumedAddrSpace if the associated function exists.
+      if (F) {
         auto *TTI =
-            A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
-                *Arg->getParent());
-        unsigned AssumedAS = TTI->getAssumedAddrSpace(Arg);
+            A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(*F);
+        unsigned AssumedAS = TTI->getAssumedAddrSpace(&Obj);
         if (AssumedAS != ~0U)
           return takeAddressSpace(AssumedAS);
       }
-      return takeAddressSpace(Obj.getType()->getPointerAddressSpace());
+
+      // Now we can't do anything else but to take the flat AS.
+      return takeAddressSpace(FlatAS);
     };
 
     auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(), this,
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
index 2e2687a5ff6e..d32b829e2ad7 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
@@ -170,7 +170,6 @@ bool DeadArgumentEliminationPass::deleteDeadVarargs(Function &F) {
   NF->setComdat(F.getComdat());
   F.getParent()->getFunctionList().insert(F.getIterator(), NF);
   NF->takeName(&F);
-  NF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat;
 
   // Loop over all the callers of the function, transforming the call sites
   // to pass in a smaller number of arguments into the new function.
@@ -884,7 +883,6 @@ bool DeadArgumentEliminationPass::removeDeadStuffFromFunction(Function *F) {
   // it again.
   F->getParent()->getFunctionList().insert(F->getIterator(), NF);
   NF->takeName(F);
-  NF->IsNewDbgInfoFormat = F->IsNewDbgInfoFormat;
 
   // Loop over all the callers of the function, transforming the call sites to
   // pass in a smaller number of arguments into the new function.
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
index e25f23107966..16ffd503300e 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
@@ -508,7 +508,6 @@ ExpandVariadics::replaceAllUsesWithNewDeclaration(Module &M,
   Function *NF = Function::Create(FTy, F.getLinkage(), F.getAddressSpace());
 
   NF->setName(F.getName() + ".varargs");
-  NF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat;
 
   F.getParent()->getFunctionList().insert(F.getIterator(), NF);
 
@@ -550,7 +549,6 @@ ExpandVariadics::deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
   NF->setComdat(F.getComdat());
   F.getParent()->getFunctionList().insert(F.getIterator(), NF);
   NF->setName(F.getName() + ".valist");
-  NF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat;
 
   AttrBuilder ParamAttrs(Ctx);
 
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index b3fe0ab8b5cb..7db058638650 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -1494,8 +1494,14 @@ processInternalGlobal(GlobalVariable *GV, const GlobalStatus &GS,
     // FIXME: Pass Global's alignment when globals have alignment
     AllocaInst *Alloca = new AllocaInst(ElemTy, DL.getAllocaAddrSpace(),
                                         nullptr, GV->getName(), FirstI);
-    if (!isa<UndefValue>(GV->getInitializer()))
-      new StoreInst(GV->getInitializer(), Alloca, FirstI);
+    Alloca->setDebugLoc(DebugLoc::getCompilerGenerated());
+    if (!isa<UndefValue>(GV->getInitializer())) {
+      auto *SI = new StoreInst(GV->getInitializer(), Alloca, FirstI);
+      // FIXME: We're localizing a global and creating a store instruction for
+      // the initial value of that global. Could we logically use the global
+      // variable's (if one exists) line for this?
+      SI->setDebugLoc(DebugLoc::getCompilerGenerated());
+    }
 
     GV->replaceAllUsesWith(Alloca);
     GV->eraseFromParent();
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/IROutliner.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/IROutliner.cpp
index ff66a518be75..cb18b55ae218 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/IROutliner.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/IROutliner.cpp
@@ -730,7 +730,7 @@ static void moveFunctionData(Function &Old, Function &New,
       // other outlined instructions.
       if (!isa<CallInst>(&Val)) {
         // Remove the debug information for outlined functions.
-        Val.setDebugLoc(DebugLoc());
+        Val.setDebugLoc(DebugLoc::getDropped());
 
         // Loop info metadata may contain line locations. Update them to have no
         // value in the new subprogram since the outlined code could be from
@@ -1864,7 +1864,7 @@ replaceArgumentUses(OutlinableRegion &Region,
       Value *ValueOperand = SI->getValueOperand();
 
       StoreInst *NewI = cast<StoreInst>(I->clone());
-      NewI->setDebugLoc(DebugLoc());
+      NewI->setDebugLoc(DebugLoc::getDropped());
       BasicBlock *OutputBB = VBBIt->second;
       NewI->insertInto(OutputBB, OutputBB->end());
       LLVM_DEBUG(dbgs() << "Move store for instruction " << *I << " to "
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index de9efc5b25b4..ab67a0980e0c 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -19,11 +19,14 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TypeMetadataUtils.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -140,7 +143,7 @@ bool BitSetInfo::containsGlobalOffset(uint64_t Offset) const {
   if (BitOffset >= BitSize)
     return false;
 
-  return Bits.count(BitOffset);
+  return Bits.count(BitSize - 1 - BitOffset);
 }
 
 void BitSetInfo::print(raw_ostream &OS) const {
@@ -185,7 +188,11 @@ BitSetInfo BitSetBuilder::build() {
   BSI.BitSize = ((Max - Min) >> BSI.AlignLog2) + 1;
   for (uint64_t Offset : Offsets) {
     Offset >>= BSI.AlignLog2;
-    BSI.Bits.insert(Offset);
+    // We invert the order of bits when adding them to the bitset. This is
+    // because the offset that we test against is computed by subtracting the
+    // address that we are testing from the global's address, which means that
+    // the offset increases as the tested address decreases.
+    BSI.Bits.insert(BSI.BitSize - 1 - Offset);
   }
 
   return BSI;
@@ -462,7 +469,8 @@ class LowerTypeTestsModule {
   struct TypeIdLowering {
     TypeTestResolution::Kind TheKind = TypeTestResolution::Unsat;
 
-    /// All except Unsat: the start address within the combined global.
+    /// All except Unsat: the address of the last element within the combined
+    /// global.
     Constant *OffsetedGlobal;
 
     /// ByteArray, Inline, AllOnes: log2 of the required global alignment
@@ -769,7 +777,11 @@ Value *LowerTypeTestsModule::lowerTypeTestCall(Metadata *TypeId, CallInst *CI,
   if (TIL.TheKind == TypeTestResolution::Single)
     return B.CreateICmpEQ(PtrAsInt, OffsetedGlobalAsInt);
 
-  Value *PtrOffset = B.CreateSub(PtrAsInt, OffsetedGlobalAsInt);
+  // Here we compute `last element - address`. The reason why we do this instead
+  // of computing `address - first element` is that it leads to a slightly
+  // shorter instruction sequence on x86. Because it doesn't matter how we do
+  // the subtraction on other architectures, we do so unconditionally.
+  Value *PtrOffset = B.CreateSub(OffsetedGlobalAsInt, PtrAsInt);
 
   // We need to check that the offset both falls within our range and is
   // suitably aligned. We can check both properties at the same time by
@@ -779,9 +791,8 @@ Value *LowerTypeTestsModule::lowerTypeTestCall(Metadata *TypeId, CallInst *CI,
   // result, causing the comparison to fail if they are nonzero. The rotate
   // also conveniently gives us a bit offset to use during the load from
   // the bitset.
-  Value *BitOffset = B.CreateIntrinsic(
-      IntPtrTy, Intrinsic::fshr,
-      {PtrOffset, PtrOffset, B.CreateZExt(TIL.AlignLog2, IntPtrTy)});
+  Value *BitOffset = B.CreateIntrinsic(IntPtrTy, Intrinsic::fshr,
+                                       {PtrOffset, PtrOffset, TIL.AlignLog2});
 
   Value *OffsetInRange = B.CreateICmpULE(BitOffset, TIL.SizeM1);
 
@@ -1033,7 +1044,7 @@ LowerTypeTestsModule::importTypeId(StringRef TypeId) {
   if (TIL.TheKind == TypeTestResolution::ByteArray ||
       TIL.TheKind == TypeTestResolution::Inline ||
       TIL.TheKind == TypeTestResolution::AllOnes) {
-    TIL.AlignLog2 = ImportConstant("align", TTRes.AlignLog2, 8, Int8Ty);
+    TIL.AlignLog2 = ImportConstant("align", TTRes.AlignLog2, 8, IntPtrTy);
     TIL.SizeM1 =
         ImportConstant("size_m1", TTRes.SizeM1, TTRes.SizeM1BitWidth, IntPtrTy);
   }
@@ -1152,14 +1163,17 @@ void LowerTypeTestsModule::lowerTypeTestCalls(
 
     ByteArrayInfo *BAI = nullptr;
     TypeIdLowering TIL;
+
+    uint64_t GlobalOffset =
+        BSI.ByteOffset + ((BSI.BitSize - 1) << BSI.AlignLog2);
     TIL.OffsetedGlobal = ConstantExpr::getGetElementPtr(
-        Int8Ty, CombinedGlobalAddr, ConstantInt::get(IntPtrTy, BSI.ByteOffset)),
-    TIL.AlignLog2 = ConstantInt::get(Int8Ty, BSI.AlignLog2);
+        Int8Ty, CombinedGlobalAddr, ConstantInt::get(IntPtrTy, GlobalOffset)),
+    TIL.AlignLog2 = ConstantInt::get(IntPtrTy, BSI.AlignLog2);
     TIL.SizeM1 = ConstantInt::get(IntPtrTy, BSI.BitSize - 1);
     if (BSI.isAllOnes()) {
       TIL.TheKind = (BSI.BitSize == 1) ? TypeTestResolution::Single
                                        : TypeTestResolution::AllOnes;
-    } else if (BSI.BitSize <= 64) {
+    } else if (BSI.BitSize <= IntPtrTy->getBitWidth()) {
       TIL.TheKind = TypeTestResolution::Inline;
       uint64_t InlineBits = 0;
       for (auto Bit : BSI.Bits)
@@ -2472,3 +2486,95 @@ PreservedAnalyses LowerTypeTestsPass::run(Module &M,
     return PreservedAnalyses::all();
   return PreservedAnalyses::none();
 }
+
+PreservedAnalyses SimplifyTypeTestsPass::run(Module &M,
+                                             ModuleAnalysisManager &AM) {
+  bool Changed = false;
+  // Figure out whether inlining has exposed a constant address to a lowered
+  // type test, and remove the test if so and the address is known to pass the
+  // test. Unfortunately this pass ends up needing to reverse engineer what
+  // LowerTypeTests did; this is currently inherent to the design of ThinLTO
+  // importing where LowerTypeTests needs to run at the start.
+  //
+  // We look for things like:
+  //
+  // sub (i64 ptrtoint (ptr @_Z2fpv to i64), i64 ptrtoint (ptr
+  // @__typeid__ZTSFvvE_global_addr to i64))
+  //
+  // which gets replaced with 0 if _Z2fpv (more specifically _Z2fpv.cfi, the
+  // function referred to by the jump table) is a member of the type _ZTSFvv, as
+  // well as things like
+  //
+  // icmp eq ptr @_Z2fpv, @__typeid__ZTSFvvE_global_addr
+  //
+  // which gets replaced with true if _Z2fpv is a member.
+  for (auto &GV : M.globals()) {
+    if (!GV.getName().starts_with("__typeid_") ||
+        !GV.getName().ends_with("_global_addr"))
+      continue;
+    // __typeid_foo_global_addr -> foo
+    auto *MD = MDString::get(M.getContext(),
+                             GV.getName().substr(9, GV.getName().size() - 21));
+    auto MaySimplifyPtr = [&](Value *Ptr) {
+      if (auto *GV = dyn_cast<GlobalValue>(Ptr))
+        if (auto *CFIGV = M.getNamedValue((GV->getName() + ".cfi").str()))
+          Ptr = CFIGV;
+      return isKnownTypeIdMember(MD, M.getDataLayout(), Ptr, 0);
+    };
+    auto MaySimplifyInt = [&](Value *Op) {
+      auto *PtrAsInt = dyn_cast<ConstantExpr>(Op);
+      if (!PtrAsInt || PtrAsInt->getOpcode() != Instruction::PtrToInt)
+        return false;
+      return MaySimplifyPtr(PtrAsInt->getOperand(0));
+    };
+    for (User *U : make_early_inc_range(GV.users())) {
+      if (auto *CI = dyn_cast<ICmpInst>(U)) {
+        if (CI->getPredicate() == CmpInst::ICMP_EQ &&
+            MaySimplifyPtr(CI->getOperand(0))) {
+          // This is an equality comparison (TypeTestResolution::Single case in
+          // lowerTypeTestCall). In this case we just replace the comparison
+          // with true.
+          CI->replaceAllUsesWith(ConstantInt::getTrue(M.getContext()));
+          CI->eraseFromParent();
+          Changed = true;
+          continue;
+        }
+      }
+      auto *CE = dyn_cast<ConstantExpr>(U);
+      if (!CE || CE->getOpcode() != Instruction::PtrToInt)
+        continue;
+      for (Use &U : make_early_inc_range(CE->uses())) {
+        auto *CE = dyn_cast<ConstantExpr>(U.getUser());
+        if (U.getOperandNo() == 0 && CE &&
+            CE->getOpcode() == Instruction::Sub &&
+            MaySimplifyInt(CE->getOperand(1))) {
+          // This is a computation of PtrOffset as generated by
+          // LowerTypeTestsModule::lowerTypeTestCall above. If
+          // isKnownTypeIdMember passes we just pretend it evaluated to 0. This
+          // should cause later passes to remove the range and alignment checks.
+          // The bitset checks won't be removed but those are uncommon.
+          CE->replaceAllUsesWith(ConstantInt::get(CE->getType(), 0));
+          Changed = true;
+        }
+        auto *CI = dyn_cast<ICmpInst>(U.getUser());
+        if (U.getOperandNo() == 1 && CI &&
+            CI->getPredicate() == CmpInst::ICMP_EQ &&
+            MaySimplifyInt(CI->getOperand(0))) {
+          // This is an equality comparison. Unlike in the case above it
+          // remained as an integer compare.
+          CI->replaceAllUsesWith(ConstantInt::getTrue(M.getContext()));
+          CI->eraseFromParent();
+          Changed = true;
+        }
+      }
+    }
+  }
+
+  if (!Changed)
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA = PreservedAnalyses::none();
+  PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<PostDominatorTreeAnalysis>();
+  PA.preserve<LoopAnalysis>();
+  return PA;
+}
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 5b4350845b72..10120dd0e10c 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -1642,7 +1642,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
   // this entry.
   DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
 
-  bool PrevIterCreatedNode = false;
+  [[maybe_unused]] bool PrevIterCreatedNode = false;
   bool CreatedNode = false;
   for (unsigned I = 0; I < Calls.size();
        I++, PrevIterCreatedNode = CreatedNode) {
@@ -2183,6 +2183,9 @@ ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
 
   updateStackNodes();
 
+  if (ExportToDot)
+    exportToDot("poststackupdate");
+
   handleCallsitesWithMultipleTargets();
 
   markBackedges();
@@ -2232,9 +2235,8 @@ IndexCallsiteContextGraph::IndexCallsiteContextGraph(
           CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
               EmptyContext;
           unsigned I = 0;
-          assert(
-              (!MemProfReportHintedSizes && MinClonedColdBytePercent >= 100) ||
-              AN.ContextSizeInfos.size() == AN.MIBs.size());
+          assert(!metadataMayIncludeContextSizeInfo() ||
+                 AN.ContextSizeInfos.size() == AN.MIBs.size());
           // Now add all of the MIBs and their stack nodes.
           for (auto &MIB : AN.MIBs) {
             CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
@@ -2286,6 +2288,9 @@ IndexCallsiteContextGraph::IndexCallsiteContextGraph(
 
   updateStackNodes();
 
+  if (ExportToDot)
+    exportToDot("poststackupdate");
+
   handleCallsitesWithMultipleTargets();
 
   markBackedges();
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index e5397e94c792..d4555e9435f1 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -751,7 +751,6 @@ void MergeFunctions::writeThunk(Function *F, Function *G) {
     NewG = Function::Create(G->getFunctionType(), G->getLinkage(),
                             G->getAddressSpace(), "", G->getParent());
     NewG->setComdat(G->getComdat());
-    NewG->IsNewDbgInfoFormat = G->IsNewDbgInfoFormat;
     BB = BasicBlock::Create(F->getContext(), "", NewG);
   }
 
@@ -897,7 +896,6 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
     NewF->takeName(F);
     NewF->setComdat(F->getComdat());
     F->setComdat(nullptr);
-    NewF->IsNewDbgInfoFormat = F->IsNewDbgInfoFormat;
     // Ensure CFI type metadata is propagated to the new function.
     copyMetadataIfPresent(F, NewF, "type");
     copyMetadataIfPresent(F, NewF, "kcfi_type");
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index c088e261c81f..57a5c3b1e6bc 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -5259,8 +5259,7 @@ struct AAFoldRuntimeCallCallSiteReturned : AAFoldRuntimeCall {
         IRPosition::callsite_returned(CB),
         [&](const IRPosition &IRP, const AbstractAttribute *AA,
             bool &UsedAssumedInformation) -> std::optional<Value *> {
-          assert((isValidState() ||
-                  (SimplifiedValue && *SimplifiedValue == nullptr)) &&
+          assert((isValidState() || SimplifiedValue == nullptr) &&
                  "Unexpected invalid state!");
 
           if (!isAtFixpoint()) {
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/SCCP.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/SCCP.cpp
index 9d15dd0c1038..43c5df357500 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/SCCP.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/SCCP.cpp
@@ -76,10 +76,8 @@ static void findReturnsToZap(Function &F,
                if (!isa<CallBase>(U))
                  return true;
                if (U->getType()->isStructTy()) {
-                 return all_of(Solver.getStructLatticeValueFor(U),
-                               [](const ValueLatticeElement &LV) {
-                                 return !SCCPSolver::isOverdefined(LV);
-                               });
+                 return none_of(Solver.getStructLatticeValueFor(U),
+                                SCCPSolver::isOverdefined);
                }
 
                // We don't consider assume-like intrinsics to be actual address
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/StripSymbols.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/StripSymbols.cpp
index 78bcd8b83bd2..ecd0ade13146 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/StripSymbols.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/StripSymbols.cpp
@@ -316,4 +316,4 @@ PreservedAnalyses StripDeadCGProfilePass::run(Module &M,
   M.setModuleFlag(Module::Append, "CG Profile",
                   MDTuple::getDistinct(M.getContext(), ValidCGEdges));
   return PreservedAnalyses::none();
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index 06f5d78d77e0..e276376f2158 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -584,9 +584,7 @@ llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) {
   FunctionAnalysisManager &FAM =
       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 
-  ScopedDbgInfoFormatSetter FormatSetter(M, M.IsNewDbgInfoFormat);
-  if (M.IsNewDbgInfoFormat)
-    M.removeDebugIntrinsicDeclarations();
+  M.removeDebugIntrinsicDeclarations();
 
   bool Changed = writeThinLTOBitcode(
       OS, ThinLinkOS,
diff --git a/external/llvm-project/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/external/llvm-project/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 3a25255d0a4c..30e1dc7167a3 100644
--- a/external/llvm-project/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -494,28 +494,28 @@ struct CallSiteInfo {
   /// Whether all call sites represented by this CallSiteInfo, including those
   /// in summaries, have been devirtualized. This starts off as true because a
   /// default constructed CallSiteInfo represents no call sites.
+  ///
+  /// If at the end of the pass there are still undevirtualized calls, we will
+  /// need to add a use of llvm.type.test to each of the function summaries in
+  /// the vector.
   bool AllCallSitesDevirted = true;
 
   // These fields are used during the export phase of ThinLTO and reflect
   // information collected from function summaries.
 
-  /// Whether any function summary contains an llvm.assume(llvm.type.test) for
-  /// this slot.
-  bool SummaryHasTypeTestAssumeUsers = false;
-
   /// CFI-specific: a vector containing the list of function summaries that use
   /// the llvm.type.checked.load intrinsic and therefore will require
   /// resolutions for llvm.type.test in order to implement CFI checks if
-  /// devirtualization was unsuccessful. If devirtualization was successful, the
-  /// pass will clear this vector by calling markDevirt(). If at the end of the
-  /// pass the vector is non-empty, we will need to add a use of llvm.type.test
-  /// to each of the function summaries in the vector.
+  /// devirtualization was unsuccessful.
   std::vector<FunctionSummary *> SummaryTypeCheckedLoadUsers;
+
+  /// A vector containing the list of function summaries that use
+  /// assume(llvm.type.test).
   std::vector<FunctionSummary *> SummaryTypeTestAssumeUsers;
 
   bool isExported() const {
-    return SummaryHasTypeTestAssumeUsers ||
-           !SummaryTypeCheckedLoadUsers.empty();
+    return !SummaryTypeCheckedLoadUsers.empty() ||
+           !SummaryTypeTestAssumeUsers.empty();
   }
 
   void addSummaryTypeCheckedLoadUser(FunctionSummary *FS) {
@@ -525,16 +525,10 @@ struct CallSiteInfo {
 
   void addSummaryTypeTestAssumeUser(FunctionSummary *FS) {
     SummaryTypeTestAssumeUsers.push_back(FS);
-    SummaryHasTypeTestAssumeUsers = true;
     AllCallSitesDevirted = false;
   }
 
-  void markDevirt() {
-    AllCallSitesDevirted = true;
-
-    // As explained in the comment for SummaryTypeCheckedLoadUsers.
-    SummaryTypeCheckedLoadUsers.clear();
-  }
+  void markDevirt() { AllCallSitesDevirted = true; }
 };
 
 // Call site information collected for a specific VTableSlot.
@@ -1845,7 +1839,7 @@ bool DevirtModule::tryVirtualConstProp(
   if (BitWidth > 64)
     return false;
 
-  Align TypeAlignment = M.getDataLayout().getPrefTypeAlign(RetType);
+  Align TypeAlignment = M.getDataLayout().getABIIntegerTypeAlignment(BitWidth);
 
   // Make sure that each function is defined, does not access memory, takes at
   // least one argument, does not use its first argument (which we assume is
@@ -2465,11 +2459,14 @@ bool DevirtModule::run() {
     if (ExportSummary && isa<MDString>(S.first.TypeID)) {
       auto GUID = GlobalValue::getGUIDAssumingExternalLinkage(
           cast<MDString>(S.first.TypeID)->getString());
-      for (auto *FS : S.second.CSInfo.SummaryTypeCheckedLoadUsers)
-        FS->addTypeTest(GUID);
+      auto AddTypeTestsForTypeCheckedLoads = [&](CallSiteInfo &CSI) {
+        if (!CSI.AllCallSitesDevirted)
+          for (auto *FS : CSI.SummaryTypeCheckedLoadUsers)
+            FS->addTypeTest(GUID);
+      };
+      AddTypeTestsForTypeCheckedLoads(S.second.CSInfo);
       for (auto &CCS : S.second.ConstCSInfo)
-        for (auto *FS : CCS.second.SummaryTypeCheckedLoadUsers)
-          FS->addTypeTest(GUID);
+        AddTypeTestsForTypeCheckedLoads(CCS.second);
     }
   }
 
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 658bbbc56976..c1ce364eb179 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -949,7 +949,7 @@ Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) {
     // If X has no high-bits set above an xor mask:
     // add (xor X, LowMaskC), C --> sub (LowMaskC + C), X
     if (C2->isMask()) {
-      KnownBits LHSKnown = computeKnownBits(X, 0, &Add);
+      KnownBits LHSKnown = computeKnownBits(X, &Add);
       if ((*C2 | LHSKnown.Zero).isAllOnes())
         return BinaryOperator::CreateSub(ConstantInt::get(Ty, *C2 + *C), X);
     }
@@ -965,8 +965,8 @@ Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) {
         ShAmt = BitWidth - C->logBase2() - 1;
       else if (C2->isPowerOf2())
         ShAmt = BitWidth - C2->logBase2() - 1;
-      if (ShAmt && MaskedValueIsZero(X, APInt::getHighBitsSet(BitWidth, ShAmt),
-                                     0, &Add)) {
+      if (ShAmt &&
+          MaskedValueIsZero(X, APInt::getHighBitsSet(BitWidth, ShAmt), &Add)) {
         Constant *ShAmtC = ConstantInt::get(Ty, ShAmt);
         Value *NewShl = Builder.CreateShl(X, ShAmtC, "sext");
         return BinaryOperator::CreateAShr(NewShl, ShAmtC);
@@ -2068,72 +2068,90 @@ Instruction *InstCombinerImpl::visitFAdd(BinaryOperator &I) {
   return nullptr;
 }
 
+CommonPointerBase CommonPointerBase::compute(Value *LHS, Value *RHS) {
+  CommonPointerBase Base;
+
+  if (LHS->getType() != RHS->getType())
+    return Base;
+
+  // Collect all base pointers of LHS.
+  SmallPtrSet<Value *, 16> Ptrs;
+  Value *Ptr = LHS;
+  while (true) {
+    Ptrs.insert(Ptr);
+    if (auto *GEP = dyn_cast<GEPOperator>(Ptr))
+      Ptr = GEP->getPointerOperand();
+    else
+      break;
+  }
+
+  // Find common base and collect RHS GEPs.
+  while (true) {
+    if (Ptrs.contains(RHS)) {
+      Base.Ptr = RHS;
+      break;
+    }
+
+    if (auto *GEP = dyn_cast<GEPOperator>(RHS)) {
+      Base.RHSGEPs.push_back(GEP);
+      Base.RHSNW &= GEP->getNoWrapFlags();
+      RHS = GEP->getPointerOperand();
+    } else {
+      // No common base.
+      return Base;
+    }
+  }
+
+  // Collect LHS GEPs.
+  while (true) {
+    if (LHS == Base.Ptr)
+      break;
+
+    auto *GEP = cast<GEPOperator>(LHS);
+    Base.LHSGEPs.push_back(GEP);
+    Base.LHSNW &= GEP->getNoWrapFlags();
+    LHS = GEP->getPointerOperand();
+  }
+
+  return Base;
+}
+
 /// Optimize pointer differences into the same array into a size.  Consider:
 ///  &A[10] - &A[0]: we should compile this to "10".  LHS/RHS are the pointer
 /// operands to the ptrtoint instructions for the LHS/RHS of the subtract.
 Value *InstCombinerImpl::OptimizePointerDifference(Value *LHS, Value *RHS,
                                                    Type *Ty, bool IsNUW) {
-  // If LHS is a gep based on RHS or RHS is a gep based on LHS, we can optimize
-  // this.
-  bool Swapped = false;
-  GEPOperator *GEP1 = nullptr, *GEP2 = nullptr;
-  if (!isa<GEPOperator>(LHS) && isa<GEPOperator>(RHS)) {
-    std::swap(LHS, RHS);
-    Swapped = true;
-  }
-
-  // Require at least one GEP with a common base pointer on both sides.
-  if (auto *LHSGEP = dyn_cast<GEPOperator>(LHS)) {
-    // (gep X, ...) - X
-    if (LHSGEP->getOperand(0)->stripPointerCasts() ==
-        RHS->stripPointerCasts()) {
-      GEP1 = LHSGEP;
-    } else if (auto *RHSGEP = dyn_cast<GEPOperator>(RHS)) {
-      // (gep X, ...) - (gep X, ...)
-      if (LHSGEP->getOperand(0)->stripPointerCasts() ==
-          RHSGEP->getOperand(0)->stripPointerCasts()) {
-        GEP1 = LHSGEP;
-        GEP2 = RHSGEP;
-      }
-    }
-  }
-
-  if (!GEP1)
+  CommonPointerBase Base = CommonPointerBase::compute(LHS, RHS);
+  if (!Base.Ptr)
     return nullptr;
 
   // To avoid duplicating the offset arithmetic, rewrite the GEP to use the
-  // computed offset. This may erase the original GEP, so be sure to cache the
-  // nowrap flags before emitting the offset.
+  // computed offset.
   // TODO: We should probably do this even if there is only one GEP.
-  bool RewriteGEPs = GEP2 != nullptr;
+  bool RewriteGEPs = !Base.LHSGEPs.empty() && !Base.RHSGEPs.empty();
 
-  // Emit the offset of the GEP and an intptr_t.
-  GEPNoWrapFlags GEP1NW = GEP1->getNoWrapFlags();
-  Value *Result = EmitGEPOffset(GEP1, RewriteGEPs);
+  Type *IdxTy = DL.getIndexType(LHS->getType());
+  Value *Result = EmitGEPOffsets(Base.LHSGEPs, Base.LHSNW, IdxTy, RewriteGEPs);
+  Value *Offset2 = EmitGEPOffsets(Base.RHSGEPs, Base.RHSNW, IdxTy, RewriteGEPs);
 
   // If this is a single inbounds GEP and the original sub was nuw,
   // then the final multiplication is also nuw.
   if (auto *I = dyn_cast<Instruction>(Result))
-    if (IsNUW && !GEP2 && !Swapped && GEP1NW.isInBounds() &&
+    if (IsNUW && match(Offset2, m_Zero()) && Base.LHSNW.isInBounds() &&
         I->getOpcode() == Instruction::Mul)
       I->setHasNoUnsignedWrap();
 
   // If we have a 2nd GEP of the same base pointer, subtract the offsets.
   // If both GEPs are inbounds, then the subtract does not have signed overflow.
   // If both GEPs are nuw and the original sub is nuw, the new sub is also nuw.
-  if (GEP2) {
-    GEPNoWrapFlags GEP2NW = GEP2->getNoWrapFlags();
-    Value *Offset = EmitGEPOffset(GEP2, RewriteGEPs);
-    Result = Builder.CreateSub(Result, Offset, "gepdiff",
-                               IsNUW && GEP1NW.hasNoUnsignedWrap() &&
-                                   GEP2NW.hasNoUnsignedWrap(),
-                               GEP1NW.isInBounds() && GEP2NW.isInBounds());
+  if (!match(Offset2, m_Zero())) {
+    Result =
+        Builder.CreateSub(Result, Offset2, "gepdiff",
+                          IsNUW && Base.LHSNW.hasNoUnsignedWrap() &&
+                              Base.RHSNW.hasNoUnsignedWrap(),
+                          Base.LHSNW.isInBounds() && Base.RHSNW.isInBounds());
   }
 
-  // If we have p - gep(p, ...)  then we have to negate the result.
-  if (Swapped)
-    Result = Builder.CreateNeg(Result, "diff.neg");
-
   return Builder.CreateIntCast(Result, Ty, true);
 }
 
@@ -2438,7 +2456,7 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
       // zero. We don't use information from dominating conditions so this
       // transform is easier to reverse if necessary.
       KnownBits RHSKnown = llvm::computeKnownBits(
-          Op1, 0, SQ.getWithInstruction(&I).getWithoutDomCondCache());
+          Op1, SQ.getWithInstruction(&I).getWithoutDomCondCache());
       if ((*Op0C | RHSKnown.Zero).isAllOnes())
         return BinaryOperator::CreateXor(Op1, Op0);
     }
@@ -2808,8 +2826,14 @@ static Instruction *foldFNegIntoConstant(Instruction &I, const DataLayout &DL) {
   // Fold negation into constant operand.
   // -(X * C) --> X * (-C)
   if (match(FNegOp, m_FMul(m_Value(X), m_Constant(C))))
-    if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL))
-      return BinaryOperator::CreateFMulFMF(X, NegC, &I);
+    if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL)) {
+      FastMathFlags FNegF = I.getFastMathFlags();
+      FastMathFlags OpF = FNegOp->getFastMathFlags();
+      FastMathFlags FMF = FastMathFlags::unionValue(FNegF, OpF) |
+                          FastMathFlags::intersectRewrite(FNegF, OpF);
+      FMF.setNoInfs(FNegF.noInfs() && OpF.noInfs());
+      return BinaryOperator::CreateFMulFMF(X, NegC, FMF);
+    }
   // -(X / C) --> X / (-C)
   if (match(FNegOp, m_FDiv(m_Value(X), m_Constant(C))))
     if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL))
@@ -2990,7 +3014,7 @@ Instruction *InstCombinerImpl::visitFSub(BinaryOperator &I) {
   // killed later. We still limit that particular transform with 'hasOneUse'
   // because an fneg is assumed better/cheaper than a generic fsub.
   if (I.hasNoSignedZeros() ||
-      cannotBeNegativeZero(Op0, 0, getSimplifyQuery().getWithInstruction(&I))) {
+      cannotBeNegativeZero(Op0, getSimplifyQuery().getWithInstruction(&I))) {
     if (match(Op1, m_OneUse(m_FSub(m_Value(X), m_Value(Y))))) {
       Value *NewSub = Builder.CreateFSubFMF(Y, X, &I);
       return BinaryOperator::CreateFAddFMF(Op0, NewSub, &I);
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 59b46ebdb72e..dce695a03600 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -697,8 +697,8 @@ static Value *foldLogOpOfMaskedICmps(Value *LHS, Value *RHS, bool IsAnd,
   // -> (icmp eq (A & (B|D)), (B|D))
   // iff B and D is known to be a power of two
   if (Mask & Mask_NotAllZeros &&
-      isKnownToBeAPowerOfTwo(B, /*OrZero=*/false, /*Depth=*/0, Q) &&
-      isKnownToBeAPowerOfTwo(D, /*OrZero=*/false, /*Depth=*/0, Q)) {
+      isKnownToBeAPowerOfTwo(B, /*OrZero=*/false, Q) &&
+      isKnownToBeAPowerOfTwo(D, /*OrZero=*/false, Q)) {
     // If this is a logical and/or, then we must prevent propagation of a
     // poison value from the RHS by inserting freeze.
     if (IsLogical)
@@ -759,7 +759,7 @@ Value *InstCombinerImpl::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1,
   }
 
   // This simplification is only valid if the upper range is not negative.
-  KnownBits Known = computeKnownBits(RangeEnd, /*Depth=*/0, Cmp1);
+  KnownBits Known = computeKnownBits(RangeEnd, Cmp1);
   if (!Known.isNonNegative())
     return nullptr;
 
@@ -800,8 +800,7 @@ foldAndOrOfICmpsWithPow2AndWithZero(InstCombiner::BuilderTy &Builder,
   if (!match(LHS, m_OneUse(m_ICmp(Pred, m_Value(Op), m_Zero()))) ||
       !match(RHS, m_OneUse(m_c_ICmp(Pred, m_Specific(Op), m_Value(Pow2)))) ||
       match(Pow2, m_One()) ||
-      !isKnownToBeAPowerOfTwo(Pow2, Q.DL, /*OrZero=*/true, /*Depth=*/0, Q.AC,
-                              Q.CxtI, Q.DT))
+      !isKnownToBeAPowerOfTwo(Pow2, Q.DL, /*OrZero=*/true, Q.AC, Q.CxtI, Q.DT))
     return nullptr;
 
   Value *And = Builder.CreateAnd(Op, Pow2);
@@ -2447,7 +2446,7 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
   if (match(&I, m_c_And(m_Value(Y), m_OneUse(m_CombineOr(
                                         m_c_Add(m_Value(X), m_Deferred(Y)),
                                         m_Sub(m_Value(X), m_Deferred(Y)))))) &&
-      isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, /*Depth*/ 0, &I))
+      isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, &I))
     return BinaryOperator::CreateAnd(Builder.CreateNot(X), Y);
 
   if (match(Op1, m_APInt(C))) {
@@ -2572,13 +2571,13 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
         match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) {
       APInt NotAndMask(~(*C));
       BinaryOperator::BinaryOps BinOp = cast<BinaryOperator>(Op0)->getOpcode();
-      if (MaskedValueIsZero(X, NotAndMask, 0, &I)) {
+      if (MaskedValueIsZero(X, NotAndMask, &I)) {
         // Not masking anything out for the LHS, move mask to RHS.
         // and ({x}or X, Y), C --> {x}or X, (and Y, C)
         Value *NewRHS = Builder.CreateAnd(Y, Op1, Y->getName() + ".masked");
         return BinaryOperator::Create(BinOp, X, NewRHS);
       }
-      if (!isa<Constant>(Y) && MaskedValueIsZero(Y, NotAndMask, 0, &I)) {
+      if (!isa<Constant>(Y) && MaskedValueIsZero(Y, NotAndMask, &I)) {
         // Not masking anything out for the RHS, move mask to LHS.
         // and ({x}or X, Y), C --> {x}or (and X, C), Y
         Value *NewLHS = Builder.CreateAnd(X, Op1, X->getName() + ".masked");
@@ -2613,7 +2612,7 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
           match(C1, m_Power2())) {
         Constant *Log2C1 = ConstantExpr::getExactLogBase2(C1);
         Constant *LshrC = ConstantExpr::getAdd(C2, Log2C3);
-        KnownBits KnownLShrc = computeKnownBits(LshrC, 0, nullptr);
+        KnownBits KnownLShrc = computeKnownBits(LshrC, nullptr);
         if (KnownLShrc.getMaxValue().ult(Width)) {
           // iff C1,C3 is pow2 and C2 + cttz(C3) < BitWidth:
           // ((C1 << X) >> C2) & C3 -> X == (cttz(C3)+C2-cttz(C1)) ? C3 : 0
@@ -2814,7 +2813,7 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
                         m_Value(B)))) {
     if (A->getType()->isIntOrIntVectorTy(1))
       return SelectInst::Create(A, Constant::getNullValue(Ty), B);
-    if (computeKnownBits(A, /* Depth */ 0, &I).countMaxActiveBits() <= 1) {
+    if (computeKnownBits(A, &I).countMaxActiveBits() <= 1) {
       return SelectInst::Create(
           Builder.CreateICmpEQ(A, Constant::getNullValue(A->getType())), B,
           Constant::getNullValue(Ty));
@@ -2954,7 +2953,7 @@ InstCombinerImpl::convertOrOfShiftsToFunnelShift(Instruction &Or) {
       // might remove it after this fold). This still doesn't guarantee that the
       // final codegen will match this original pattern.
       if (match(R, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(L))))) {
-        KnownBits KnownL = computeKnownBits(L, /*Depth*/ 0, &Or);
+        KnownBits KnownL = computeKnownBits(L, &Or);
         return KnownL.getMaxValue().ult(Width) ? L : nullptr;
       }
 
@@ -3593,6 +3592,73 @@ static Value *foldOrOfInversions(BinaryOperator &I,
   return nullptr;
 }
 
+// A decomposition of ((X & Mask) * Factor). The NUW / NSW bools
+// track these properities for preservation. Note that we can decompose
+// equivalent select form of this expression (e.g. (!(X & Mask) ? 0 : Mask *
+// Factor))
+struct DecomposedBitMaskMul {
+  Value *X;
+  APInt Factor;
+  APInt Mask;
+  bool NUW;
+  bool NSW;
+};
+
+static std::optional<DecomposedBitMaskMul> matchBitmaskMul(Value *V) {
+  Instruction *Op = dyn_cast<Instruction>(V);
+  if (!Op)
+    return std::nullopt;
+
+  // Decompose (A & N) * C) into BitMaskMul
+  Value *Original = nullptr;
+  const APInt *Mask = nullptr;
+  const APInt *MulConst = nullptr;
+  if (match(Op, m_Mul(m_And(m_Value(Original), m_APInt(Mask)),
+                      m_APInt(MulConst)))) {
+    if (MulConst->isZero() || Mask->isZero())
+      return std::nullopt;
+
+    return std::optional<DecomposedBitMaskMul>(
+        {Original, *MulConst, *Mask,
+         cast<BinaryOperator>(Op)->hasNoUnsignedWrap(),
+         cast<BinaryOperator>(Op)->hasNoSignedWrap()});
+  }
+
+  Value *Cond = nullptr;
+  const APInt *EqZero = nullptr, *NeZero = nullptr;
+
+  // Decompose ((A & N) ? 0 : N * C) into BitMaskMul
+  if (match(Op, m_Select(m_Value(Cond), m_APInt(EqZero), m_APInt(NeZero)))) {
+    auto ICmpDecompose =
+        decomposeBitTest(Cond, /*LookThruTrunc=*/true,
+                         /*AllowNonZeroC=*/false, /*DecomposeBitMask=*/true);
+    if (!ICmpDecompose.has_value())
+      return std::nullopt;
+
+    assert(ICmpInst::isEquality(ICmpDecompose->Pred) &&
+           ICmpDecompose->C.isZero());
+
+    if (ICmpDecompose->Pred == ICmpInst::ICMP_NE)
+      std::swap(EqZero, NeZero);
+
+    if (!EqZero->isZero() || NeZero->isZero())
+      return std::nullopt;
+
+    if (!ICmpDecompose->Mask.isPowerOf2() || ICmpDecompose->Mask.isZero() ||
+        NeZero->getBitWidth() != ICmpDecompose->Mask.getBitWidth())
+      return std::nullopt;
+
+    if (!NeZero->urem(ICmpDecompose->Mask).isZero())
+      return std::nullopt;
+
+    return std::optional<DecomposedBitMaskMul>(
+        {ICmpDecompose->X, NeZero->udiv(ICmpDecompose->Mask),
+         ICmpDecompose->Mask, /*NUW=*/false, /*NSW=*/false});
+  }
+
+  return std::nullopt;
+}
+
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
@@ -3675,49 +3741,26 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
                                    /*NSW=*/true, /*NUW=*/true))
       return R;
 
-    Value *Cond0 = nullptr, *Cond1 = nullptr;
-    const APInt *Op0Eq = nullptr, *Op0Ne = nullptr;
-    const APInt *Op1Eq = nullptr, *Op1Ne = nullptr;
-
-    //  (!(A & N) ? 0 : N * C) + (!(A & M) ? 0 : M * C) -> A & (N + M) * C
-    if (match(I.getOperand(0),
-              m_Select(m_Value(Cond0), m_APInt(Op0Eq), m_APInt(Op0Ne))) &&
-        match(I.getOperand(1),
-              m_Select(m_Value(Cond1), m_APInt(Op1Eq), m_APInt(Op1Ne)))) {
-
-      auto LHSDecompose =
-          decomposeBitTest(Cond0, /*LookThruTrunc=*/true,
-                           /*AllowNonZeroC=*/false, /*DecomposeAnd=*/true);
-      auto RHSDecompose =
-          decomposeBitTest(Cond1, /*LookThruTrunc=*/true,
-                           /*AllowNonZeroC=*/false, /*DecomposeAnd=*/true);
-
-      if (LHSDecompose && RHSDecompose && LHSDecompose->X == RHSDecompose->X &&
-          RHSDecompose->Mask.isPowerOf2() && LHSDecompose->Mask.isPowerOf2() &&
-          LHSDecompose->Mask != RHSDecompose->Mask &&
-          LHSDecompose->Mask.getBitWidth() == Op0Ne->getBitWidth() &&
-          RHSDecompose->Mask.getBitWidth() == Op1Ne->getBitWidth()) {
-        assert(Op0Ne->getBitWidth() == Op1Ne->getBitWidth());
-        assert(ICmpInst::isEquality(LHSDecompose->Pred));
-        if (LHSDecompose->Pred == ICmpInst::ICMP_NE)
-          std::swap(Op0Eq, Op0Ne);
-        if (RHSDecompose->Pred == ICmpInst::ICMP_NE)
-          std::swap(Op1Eq, Op1Ne);
-
-        if (!Op0Ne->isZero() && !Op1Ne->isZero() && Op0Eq->isZero() &&
-            Op1Eq->isZero() && Op0Ne->urem(LHSDecompose->Mask).isZero() &&
-            Op1Ne->urem(RHSDecompose->Mask).isZero() &&
-            Op0Ne->udiv(LHSDecompose->Mask) ==
-                Op1Ne->udiv(RHSDecompose->Mask)) {
-          auto NewAnd = Builder.CreateAnd(
-              LHSDecompose->X,
-              ConstantInt::get(LHSDecompose->X->getType(),
-                               (LHSDecompose->Mask + RHSDecompose->Mask)));
-
-          return BinaryOperator::CreateMul(
-              NewAnd, ConstantInt::get(NewAnd->getType(),
-                                       Op0Ne->udiv(LHSDecompose->Mask)));
-        }
+    // (A & N) * C + (A & M) * C -> (A & (N + M)) & C
+    // This also accepts the equivalent select form of (A & N) * C
+    // expressions i.e. !(A & N) ? 0 : N * C)
+    auto Decomp1 = matchBitmaskMul(I.getOperand(1));
+    if (Decomp1) {
+      auto Decomp0 = matchBitmaskMul(I.getOperand(0));
+      if (Decomp0 && Decomp0->X == Decomp1->X &&
+          (Decomp0->Mask & Decomp1->Mask).isZero() &&
+          Decomp0->Factor == Decomp1->Factor) {
+
+        Value *NewAnd = Builder.CreateAnd(
+            Decomp0->X, ConstantInt::get(Decomp0->X->getType(),
+                                         (Decomp0->Mask + Decomp1->Mask)));
+
+        auto *Combined = BinaryOperator::CreateMul(
+            NewAnd, ConstantInt::get(NewAnd->getType(), Decomp1->Factor));
+
+        Combined->setHasNoUnsignedWrap(Decomp0->NUW && Decomp1->NUW);
+        Combined->setHasNoSignedWrap(Decomp0->NSW && Decomp1->NSW);
+        return Combined;
       }
     }
   }
@@ -3725,7 +3768,7 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
   Value *X, *Y;
   const APInt *CV;
   if (match(&I, m_c_Or(m_OneUse(m_Xor(m_Value(X), m_APInt(CV))), m_Value(Y))) &&
-      !CV->isAllOnes() && MaskedValueIsZero(Y, *CV, 0, &I)) {
+      !CV->isAllOnes() && MaskedValueIsZero(Y, *CV, &I)) {
     // (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0
     // The check for a 'not' op is for efficiency (if Y is known zero --> ~X).
     Value *Or = Builder.CreateOr(X, Y);
@@ -3769,14 +3812,14 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
         // ((X | B) & C0) | (B & C1) --> (X | B) & (C0 | C1)
         // iff (C0 & C1) == 0 and (X & ~C0) == 0
         if (match(A, m_c_Or(m_Value(X), m_Specific(B))) &&
-            MaskedValueIsZero(X, ~*C0, 0, &I)) {
+            MaskedValueIsZero(X, ~*C0, &I)) {
           Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
           return BinaryOperator::CreateAnd(A, C01);
         }
         // (A & C0) | ((X | A) & C1) --> (X | A) & (C0 | C1)
         // iff (C0 & C1) == 0 and (X & ~C1) == 0
         if (match(B, m_c_Or(m_Value(X), m_Specific(A))) &&
-            MaskedValueIsZero(X, ~*C1, 0, &I)) {
+            MaskedValueIsZero(X, ~*C1, &I)) {
           Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
           return BinaryOperator::CreateAnd(B, C01);
         }
@@ -4156,7 +4199,7 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
   // (X & C1) | C2 -> X & (C1 | C2) iff (X & C2) == C2
   if (match(Op0, m_OneUse(m_And(m_Value(X), m_APInt(C1)))) &&
       match(Op1, m_APInt(C2))) {
-    KnownBits KnownX = computeKnownBits(X, /*Depth*/ 0, &I);
+    KnownBits KnownX = computeKnownBits(X, &I);
     if ((KnownX.One & *C2) == *C2)
       return BinaryOperator::CreateAnd(X, ConstantInt::get(Ty, *C1 | *C2));
   }
@@ -4310,7 +4353,7 @@ Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
         LC->isZero() && RC->isZero() && LHS->hasOneUse() && RHS->hasOneUse() &&
         match(LHS0, m_And(m_Value(X), m_Value(Pow2))) &&
         match(RHS0, m_And(m_Value(Y), m_Specific(Pow2))) &&
-        isKnownToBeAPowerOfTwo(Pow2, /*OrZero=*/true, /*Depth=*/0, &I)) {
+        isKnownToBeAPowerOfTwo(Pow2, /*OrZero=*/true, &I)) {
       Value *Xor = Builder.CreateXor(X, Y);
       Value *And = Builder.CreateAnd(Xor, Pow2);
       return Builder.CreateICmp(PredL == PredR ? ICmpInst::ICMP_NE
@@ -4514,6 +4557,12 @@ bool InstCombinerImpl::sinkNotIntoLogicalOp(Instruction &I) {
   if (Op0 == Op1)
     return false;
 
+  // If one of the operands is a user of the other,
+  // freelyInvert->freelyInvertAllUsersOf will change the operands of I, which
+  // may cause miscompilation.
+  if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0))))
+    return false;
+
   Instruction::BinaryOps NewOpc =
       match(&I, m_LogicalAnd()) ? Instruction::Or : Instruction::And;
   bool IsBinaryOp = isa<BinaryOperator>(I);
@@ -4896,7 +4945,7 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
 
       // (X | C) ^ RHSC --> X ^ (C ^ RHSC) iff X & C == 0
       if (match(Op0, m_Or(m_Value(X), m_APInt(C))) &&
-          MaskedValueIsZero(X, *C, 0, &I))
+          MaskedValueIsZero(X, *C, &I))
         return BinaryOperator::CreateXor(X, ConstantInt::get(Ty, *C ^ *RHSC));
 
       // When X is a power-of-two or zero and zero input is poison:
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index e101edf4a620..8c8cc0859e4a 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -601,7 +601,7 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {
     return BO;
   }
 
-  KnownBits Known = IC.computeKnownBits(Op0, 0, &II);
+  KnownBits Known = IC.computeKnownBits(Op0, &II);
 
   // Create a mask for bits above (ctlz) or below (cttz) the first known one.
   unsigned PossibleZeros = IsTZ ? Known.countMaxTrailingZeros()
@@ -684,7 +684,7 @@ static Instruction *foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC) {
   }
 
   KnownBits Known(BitWidth);
-  IC.computeKnownBits(Op0, Known, 0, &II);
+  IC.computeKnownBits(Op0, Known, &II);
 
   // If all bits are zero except for exactly one fixed bit, then the result
   // must be 0 or 1, and we can get that answer by shifting to LSB:
@@ -1099,7 +1099,7 @@ Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) {
 }
 
 static std::optional<bool> getKnownSign(Value *Op, const SimplifyQuery &SQ) {
-  KnownBits Known = computeKnownBits(Op, /*Depth=*/0, SQ);
+  KnownBits Known = computeKnownBits(Op, SQ);
   if (Known.isNonNegative())
     return false;
   if (Known.isNegative())
@@ -1222,9 +1222,8 @@ Instruction *InstCombinerImpl::matchSAddSubSat(IntrinsicInst &MinMax1) {
 
   // The two operands of the add/sub must be nsw-truncatable to the NewTy. This
   // is usually achieved via a sext from a smaller type.
-  if (ComputeMaxSignificantBits(AddSub->getOperand(0), 0, AddSub) >
-          NewBitWidth ||
-      ComputeMaxSignificantBits(AddSub->getOperand(1), 0, AddSub) > NewBitWidth)
+  if (ComputeMaxSignificantBits(AddSub->getOperand(0), AddSub) > NewBitWidth ||
+      ComputeMaxSignificantBits(AddSub->getOperand(1), AddSub) > NewBitWidth)
     return nullptr;
 
   // Finally create and return the sat intrinsic, truncated to the new type
@@ -1401,42 +1400,46 @@ static Instruction *factorizeMinMaxTree(IntrinsicInst *II) {
 /// try to shuffle after the intrinsic.
 Instruction *
 InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
-  // TODO: This should be extended to handle other intrinsics like fshl, ctpop,
-  //       etc. Use llvm::isTriviallyVectorizable() and related to determine
-  //       which intrinsics are safe to shuffle?
-  switch (II->getIntrinsicID()) {
-  case Intrinsic::smax:
-  case Intrinsic::smin:
-  case Intrinsic::umax:
-  case Intrinsic::umin:
-  case Intrinsic::fma:
-  case Intrinsic::fshl:
-  case Intrinsic::fshr:
-    break;
-  default:
+  if (!isTriviallyVectorizable(II->getIntrinsicID()) ||
+      !II->getCalledFunction()->isSpeculatable())
+    return nullptr;
+
+  // fabs is canonicalized to fabs (shuffle ...) in foldShuffleOfUnaryOps, so
+  // avoid undoing it.
+  if (match(II, m_FAbs(m_Value())))
     return nullptr;
-  }
 
   Value *X;
   Constant *C;
   ArrayRef<int> Mask;
-  auto *NonConstArg = find_if_not(II->args(), IsaPred<Constant>);
+  auto *NonConstArg = find_if_not(II->args(), [&II](Use &Arg) {
+    return isa<Constant>(Arg.get()) ||
+           isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+                                              Arg.getOperandNo(), nullptr);
+  });
   if (!NonConstArg ||
       !match(NonConstArg, m_Shuffle(m_Value(X), m_Poison(), m_Mask(Mask))))
     return nullptr;
 
-  // At least 1 operand must have 1 use because we are creating 2 instructions.
-  if (none_of(II->args(), [](Value *V) { return V->hasOneUse(); }))
+  // At least 1 operand must be a shuffle with 1 use because we are creating 2
+  // instructions.
+  if (none_of(II->args(), [](Value *V) {
+        return isa<ShuffleVectorInst>(V) && V->hasOneUse();
+      }))
     return nullptr;
 
   // See if all arguments are shuffled with the same mask.
   SmallVector<Value *, 4> NewArgs;
   Type *SrcTy = X->getType();
-  for (Value *Arg : II->args()) {
-    if (match(Arg, m_Shuffle(m_Value(X), m_Poison(), m_SpecificMask(Mask))) &&
-        X->getType() == SrcTy)
+  for (Use &Arg : II->args()) {
+    if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+                                           Arg.getOperandNo(), nullptr))
+      NewArgs.push_back(Arg);
+    else if (match(&Arg,
+                   m_Shuffle(m_Value(X), m_Poison(), m_SpecificMask(Mask))) &&
+             X->getType() == SrcTy)
       NewArgs.push_back(X);
-    else if (match(Arg, m_ImmConstant(C))) {
+    else if (match(&Arg, m_ImmConstant(C))) {
       // If it's a constant, try find the constant that would be shuffled to C.
       if (Constant *ShuffledC =
               unshuffleConstant(Mask, C, cast<VectorType>(SrcTy)))
@@ -1449,8 +1452,12 @@ InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
 
   // intrinsic (shuf X, M), (shuf Y, M), ... --> shuf (intrinsic X, Y, ...), M
   Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
+  // Result type might be a different vector width.
+  // TODO: Check that the result type isn't widened?
+  VectorType *ResTy =
+      VectorType::get(II->getType()->getScalarType(), cast<VectorType>(SrcTy));
   Value *NewIntrinsic =
-      Builder.CreateIntrinsic(II->getIntrinsicID(), SrcTy, NewArgs, FPI);
+      Builder.CreateIntrinsic(ResTy, II->getIntrinsicID(), NewArgs, FPI);
   return new ShuffleVectorInst(NewIntrinsic, Mask);
 }
 
@@ -2160,7 +2167,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       }
     }
 
-    KnownBits Known = computeKnownBits(IIOperand, 0, II);
+    KnownBits Known = computeKnownBits(IIOperand, II);
     uint64_t LZ = alignDown(Known.countMinLeadingZeros(), 8);
     uint64_t TZ = alignDown(Known.countMinTrailingZeros(), 8);
     unsigned BW = Known.getBitWidth();
@@ -2752,7 +2759,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   case Intrinsic::copysign: {
     Value *Mag = II->getArgOperand(0), *Sign = II->getArgOperand(1);
     if (std::optional<bool> KnownSignBit = computeKnownFPSignBit(
-            Sign, /*Depth=*/0, getSimplifyQuery().getWithInstruction(II))) {
+            Sign, getSimplifyQuery().getWithInstruction(II))) {
       if (*KnownSignBit) {
         // If we know that the sign argument is negative, reduce to FNABS:
         // copysign Mag, -Sign --> fneg (fabs Mag)
@@ -3077,10 +3084,16 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   case Intrinsic::arm_neon_aesd:
   case Intrinsic::arm_neon_aese:
   case Intrinsic::aarch64_crypto_aesd:
-  case Intrinsic::aarch64_crypto_aese: {
+  case Intrinsic::aarch64_crypto_aese:
+  case Intrinsic::aarch64_sve_aesd:
+  case Intrinsic::aarch64_sve_aese: {
     Value *DataArg = II->getArgOperand(0);
     Value *KeyArg  = II->getArgOperand(1);
 
+    // Accept zero on either operand.
+    if (!match(KeyArg, m_ZeroInt()))
+      std::swap(KeyArg, DataArg);
+
     // Try to use the builtin XOR in AESE and AESD to eliminate a prior XOR
     Value *Data, *Key;
     if (match(KeyArg, m_ZeroInt()) &&
@@ -3100,8 +3113,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
           ID0 != Intrinsic::hexagon_V6_vandqrt_128B)
         break;
       Value *Bytes = Op0->getArgOperand(1), *Mask = II->getArgOperand(1);
-      uint64_t Bytes1 = computeKnownBits(Bytes, 0, Op0).One.getZExtValue();
-      uint64_t Mask1 = computeKnownBits(Mask, 0, II).One.getZExtValue();
+      uint64_t Bytes1 = computeKnownBits(Bytes, Op0).One.getZExtValue();
+      uint64_t Mask1 = computeKnownBits(Mask, II).One.getZExtValue();
       // Check if every byte has common bits in Bytes and Mask.
       uint64_t C = Bytes1 & Mask1;
       if ((C & 0xFF) && (C & 0xFF00) && (C & 0xFF0000) && (C & 0xFF000000))
@@ -3392,7 +3405,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     // If there is a dominating assume with the same condition as this one,
     // then this one is redundant, and should be removed.
     KnownBits Known(1);
-    computeKnownBits(IIOperand, Known, 0, II);
+    computeKnownBits(IIOperand, Known, II);
     if (Known.isAllOnes() && isAssumeWithEmptyBundle(cast<AssumeInst>(*II)))
       return eraseInstFromFunction(*II);
 
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 9088b5bdec78..033ef8be700e 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -295,8 +295,8 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombinerImpl &IC,
     APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
     // Do not preserve the original context instruction. Simplifying div/rem
     // based on later context may introduce a trap.
-    if (IC.MaskedValueIsZero(I->getOperand(0), Mask, 0, I) &&
-        IC.MaskedValueIsZero(I->getOperand(1), Mask, 0, I)) {
+    if (IC.MaskedValueIsZero(I->getOperand(0), Mask, I) &&
+        IC.MaskedValueIsZero(I->getOperand(1), Mask, I)) {
       return canEvaluateTruncated(I->getOperand(0), Ty, IC, I) &&
              canEvaluateTruncated(I->getOperand(1), Ty, IC, I);
     }
@@ -321,7 +321,7 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombinerImpl &IC,
     //       zero - use AmtKnownBits.getMaxValue().
     uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
     uint32_t BitWidth = Ty->getScalarSizeInBits();
-    KnownBits AmtKnownBits = IC.computeKnownBits(I->getOperand(1), 0, CxtI);
+    KnownBits AmtKnownBits = IC.computeKnownBits(I->getOperand(1), CxtI);
     APInt MaxShiftAmt = AmtKnownBits.getMaxValue();
     APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
     if (MaxShiftAmt.ult(BitWidth)) {
@@ -333,7 +333,7 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombinerImpl &IC,
           return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
                  canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
       }
-      if (IC.MaskedValueIsZero(I->getOperand(0), ShiftedBits, 0, CxtI))
+      if (IC.MaskedValueIsZero(I->getOperand(0), ShiftedBits, CxtI))
         return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
                canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
     }
@@ -351,7 +351,7 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombinerImpl &IC,
         llvm::computeKnownBits(I->getOperand(1), IC.getDataLayout());
     unsigned ShiftedBits = OrigBitWidth - BitWidth;
     if (AmtKnownBits.getMaxValue().ult(BitWidth) &&
-        ShiftedBits < IC.ComputeNumSignBits(I->getOperand(0), 0, CxtI))
+        ShiftedBits < IC.ComputeNumSignBits(I->getOperand(0), CxtI))
       return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
              canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
     break;
@@ -595,7 +595,7 @@ Instruction *InstCombinerImpl::narrowFunnelShift(TruncInst &Trunc) {
   // from 'zext', 'and' or 'shift'). High bits of the left-shifted value are
   // truncated, so those do not matter.
   APInt HiBitMask = APInt::getHighBitsSet(WideWidth, WideWidth - NarrowWidth);
-  if (!MaskedValueIsZero(ShVal1, HiBitMask, 0, &Trunc))
+  if (!MaskedValueIsZero(ShVal1, HiBitMask, &Trunc))
     return nullptr;
 
   // Adjust the width of ShAmt for narrowed funnel shift operation:
@@ -815,6 +815,12 @@ Instruction *InstCombinerImpl::visitTrunc(TruncInst &Trunc) {
       return new ICmpInst(ICmpInst::ICMP_EQ, X, CmpC);
     }
 
+    if (match(Src, m_Shr(m_Value(X), m_SpecificInt(SrcWidth - 1)))) {
+      // trunc (ashr X, BW-1) to i1 --> icmp slt X, 0
+      // trunc (lshr X, BW-1) to i1 --> icmp slt X, 0
+      return new ICmpInst(ICmpInst::ICMP_SLT, X, Zero);
+    }
+
     Constant *C;
     if (match(Src, m_OneUse(m_LShr(m_Value(X), m_ImmConstant(C))))) {
       // trunc (lshr X, C) to i1 --> icmp ne (and X, C'), 0
@@ -935,12 +941,9 @@ Instruction *InstCombinerImpl::visitTrunc(TruncInst &Trunc) {
         Trunc.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
       Attribute Attr =
           Trunc.getFunction()->getFnAttribute(Attribute::VScaleRange);
-      if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax()) {
-        if (Log2_32(*MaxVScale) < DestWidth) {
-          Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
-          return replaceInstUsesWith(Trunc, VScale);
-        }
-      }
+      if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax())
+        if (Log2_32(*MaxVScale) < DestWidth)
+          return replaceInstUsesWith(Trunc, Builder.CreateVScale(DestTy));
     }
   }
 
@@ -951,13 +954,13 @@ Instruction *InstCombinerImpl::visitTrunc(TruncInst &Trunc) {
 
   bool Changed = false;
   if (!Trunc.hasNoSignedWrap() &&
-      ComputeMaxSignificantBits(Src, /*Depth=*/0, &Trunc) <= DestWidth) {
+      ComputeMaxSignificantBits(Src, &Trunc) <= DestWidth) {
     Trunc.setHasNoSignedWrap(true);
     Changed = true;
   }
   if (!Trunc.hasNoUnsignedWrap() &&
       MaskedValueIsZero(Src, APInt::getBitsSetFrom(SrcWidth, DestWidth),
-                        /*Depth=*/0, &Trunc)) {
+                        &Trunc)) {
     Trunc.setHasNoUnsignedWrap(true);
     Changed = true;
   }
@@ -1000,7 +1003,7 @@ Instruction *InstCombinerImpl::transformZExtICmp(ICmpInst *Cmp,
     if (Op1CV->isZero() && Cmp->isEquality()) {
       // Exactly 1 possible 1? But not the high-bit because that is
       // canonicalized to this form.
-      KnownBits Known = computeKnownBits(Cmp->getOperand(0), 0, &Zext);
+      KnownBits Known = computeKnownBits(Cmp->getOperand(0), &Zext);
       APInt KnownZeroMask(~Known.Zero);
       uint32_t ShAmt = KnownZeroMask.logBase2();
       bool IsExpectShAmt = KnownZeroMask.isPowerOf2() &&
@@ -1109,7 +1112,7 @@ static bool canEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear,
       unsigned VSize = V->getType()->getScalarSizeInBits();
       if (IC.MaskedValueIsZero(I->getOperand(1),
                                APInt::getHighBitsSet(VSize, BitsToClear),
-                               0, CxtI)) {
+                               CxtI)) {
         // If this is an And instruction and all of the BitsToClear are
         // known to be zero we can reset BitsToClear.
         if (I->getOpcode() == Instruction::And)
@@ -1229,10 +1232,9 @@ Instruction *InstCombinerImpl::visitZExt(ZExtInst &Zext) {
 
     // If the high bits are already filled with zeros, just replace this
     // cast with the result.
-    if (MaskedValueIsZero(Res,
-                          APInt::getHighBitsSet(DestBitSize,
-                                                DestBitSize - SrcBitsKept),
-                             0, &Zext))
+    if (MaskedValueIsZero(
+            Res, APInt::getHighBitsSet(DestBitSize, DestBitSize - SrcBitsKept),
+            &Zext))
       return replaceInstUsesWith(Zext, Res);
 
     // We need to emit an AND to clear the high bits.
@@ -1315,10 +1317,8 @@ Instruction *InstCombinerImpl::visitZExt(ZExtInst &Zext) {
           Zext.getFunction()->getFnAttribute(Attribute::VScaleRange);
       if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax()) {
         unsigned TypeWidth = Src->getType()->getScalarSizeInBits();
-        if (Log2_32(*MaxVScale) < TypeWidth) {
-          Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
-          return replaceInstUsesWith(Zext, VScale);
-        }
+        if (Log2_32(*MaxVScale) < TypeWidth)
+          return replaceInstUsesWith(Zext, Builder.CreateVScale(DestTy));
       }
     }
   }
@@ -1369,7 +1369,7 @@ Instruction *InstCombinerImpl::transformSExtICmp(ICmpInst *Cmp,
     // the icmp and sext into bitwise/integer operations.
     if (Cmp->hasOneUse() &&
         Cmp->isEquality() && (Op1C->isZero() || Op1C->getValue().isPowerOf2())){
-      KnownBits Known = computeKnownBits(Op0, 0, &Sext);
+      KnownBits Known = computeKnownBits(Op0, &Sext);
 
       APInt KnownZeroMask(~Known.Zero);
       if (KnownZeroMask.isPowerOf2()) {
@@ -1509,7 +1509,7 @@ Instruction *InstCombinerImpl::visitSExt(SExtInst &Sext) {
 
     // If the high bits are already filled with sign bit, just replace this
     // cast with the result.
-    if (ComputeNumSignBits(Res, 0, &Sext) > DestBitSize - SrcBitSize)
+    if (ComputeNumSignBits(Res, &Sext) > DestBitSize - SrcBitSize)
       return replaceInstUsesWith(Sext, Res);
 
     // We need to emit a shl + ashr to do the sign extend.
@@ -1523,7 +1523,7 @@ Instruction *InstCombinerImpl::visitSExt(SExtInst &Sext) {
     // If the input has more sign bits than bits truncated, then convert
     // directly to final type.
     unsigned XBitSize = X->getType()->getScalarSizeInBits();
-    if (ComputeNumSignBits(X, 0, &Sext) > XBitSize - SrcBitSize)
+    if (ComputeNumSignBits(X, &Sext) > XBitSize - SrcBitSize)
       return CastInst::CreateIntegerCast(X, DestTy, /* isSigned */ true);
 
     // If input is a trunc from the destination type, then convert into shifts.
@@ -1605,12 +1605,9 @@ Instruction *InstCombinerImpl::visitSExt(SExtInst &Sext) {
         Sext.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
       Attribute Attr =
           Sext.getFunction()->getFnAttribute(Attribute::VScaleRange);
-      if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax()) {
-        if (Log2_32(*MaxVScale) < (SrcBitSize - 1)) {
-          Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
-          return replaceInstUsesWith(Sext, VScale);
-        }
-      }
+      if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax())
+        if (Log2_32(*MaxVScale) < (SrcBitSize - 1))
+          return replaceInstUsesWith(Sext, Builder.CreateVScale(DestTy));
     }
   }
 
@@ -1748,7 +1745,7 @@ static bool isKnownExactCastIntToFP(CastInst &I, InstCombinerImpl &IC) {
   // TODO:
   // Try harder to find if the source integer type has less significant bits.
   // For example, compute number of sign bits.
-  KnownBits SrcKnown = IC.computeKnownBits(Src, 0, &I);
+  KnownBits SrcKnown = IC.computeKnownBits(Src, &I);
   int SigBits = (int)SrcTy->getScalarSizeInBits() -
                 SrcKnown.countMinLeadingZeros() -
                 SrcKnown.countMinTrailingZeros();
@@ -1926,7 +1923,9 @@ Instruction *InstCombinerImpl::visitFPTrunc(FPTruncInst &FPT) {
       II->getOperandBundlesAsDefs(OpBundles);
       CallInst *NewCI =
           CallInst::Create(Overload, {InnerTrunc}, OpBundles, II->getName());
-      NewCI->copyFastMathFlags(II);
+      // A normal value may be converted to an infinity. It means that we cannot
+      // propagate ninf from the intrinsic. So we propagate FMF from fptrunc.
+      NewCI->copyFastMathFlags(&FPT);
       return NewCI;
     }
     }
@@ -2007,9 +2006,8 @@ static Instruction *foldFPtoI(Instruction &FI, InstCombiner &IC) {
   // fpto{u/s}i non-norm --> 0
   FPClassTest Mask =
       FI.getOpcode() == Instruction::FPToUI ? fcPosNormal : fcNormal;
-  KnownFPClass FPClass =
-      computeKnownFPClass(FI.getOperand(0), Mask, /*Depth=*/0,
-                          IC.getSimplifyQuery().getWithInstruction(&FI));
+  KnownFPClass FPClass = computeKnownFPClass(
+      FI.getOperand(0), Mask, IC.getSimplifyQuery().getWithInstruction(&FI));
   if (FPClass.isKnownNever(Mask))
     return IC.replaceInstUsesWith(FI, ConstantInt::getNullValue(FI.getType()));
 
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index bdc7a49700cf..084e7fbaa268 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1119,8 +1119,8 @@ static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
   // This is only really a signed overflow check if the inputs have been
   // sign-extended; check for that condition. For example, if CI2 is 2^31 and
   // the operands of the add are 64 bits wide, we need at least 33 sign bits.
-  if (IC.ComputeMaxSignificantBits(A, 0, &I) > NewWidth ||
-      IC.ComputeMaxSignificantBits(B, 0, &I) > NewWidth)
+  if (IC.ComputeMaxSignificantBits(A, &I) > NewWidth ||
+      IC.ComputeMaxSignificantBits(B, &I) > NewWidth)
     return nullptr;
 
   // In order to replace the original add with a narrower
@@ -1183,7 +1183,7 @@ Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) {
   if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))),
                         m_CombineAnd(m_Zero(), m_Value(Zero)))))
     return nullptr;
-  if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I))
+  if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, &I))
     return nullptr;
   // This may increase instruction count, we don't enforce that Y is a constant.
   Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType()));
@@ -1250,8 +1250,8 @@ Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
   Value *X, *Y;
   if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) &&
       ICmpInst::isEquality(Pred)) {
-    KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
-    KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
+    KnownBits XKnown = computeKnownBits(X, &Cmp);
+    KnownBits YKnown = computeKnownBits(Y, &Cmp);
     if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
       return new ICmpInst(Pred, X, Cmp.getOperand(1));
   }
@@ -1261,13 +1261,13 @@ Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
   if (match(Cmp.getOperand(0), m_Mul(m_Value(X), m_Value(Y))) &&
       ICmpInst::isEquality(Pred)) {
 
-    KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
+    KnownBits XKnown = computeKnownBits(X, &Cmp);
     // if X % 2 != 0
     //    (icmp eq/ne Y)
     if (XKnown.countMaxTrailingZeros() == 0)
       return new ICmpInst(Pred, Y, Cmp.getOperand(1));
 
-    KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
+    KnownBits YKnown = computeKnownBits(Y, &Cmp);
     // if Y % 2 != 0
     //    (icmp eq/ne X)
     if (YKnown.countMaxTrailingZeros() == 0)
@@ -1487,7 +1487,7 @@ Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
 
     // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all
     // of the high bits truncated out of x are known.
-    KnownBits Known = computeKnownBits(X, 0, &Cmp);
+    KnownBits Known = computeKnownBits(X, &Cmp);
 
     // If all the high bits are known, we can do this xform.
     if ((Known.Zero | Known.One).countl_one() >= SrcBits - DstBits) {
@@ -1808,7 +1808,7 @@ Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
     }
 
     APInt NewC2 = *C2;
-    KnownBits Know = computeKnownBits(And->getOperand(0), 0, And);
+    KnownBits Know = computeKnownBits(And->getOperand(0), And);
     // Set high zeros of C2 to allow matching negated power-of-2.
     NewC2 = *C2 | APInt::getHighBitsSet(C2->getBitWidth(),
                                         Know.countMinLeadingZeros());
@@ -3770,8 +3770,7 @@ static Instruction *foldCtpopPow2Test(ICmpInst &I, IntrinsicInst *CtpopLhs,
   if (((I.isEquality() || Pred == ICmpInst::ICMP_UGT) && CRhs == 1) ||
       (Pred == ICmpInst::ICMP_ULT && CRhs == 2)) {
     Value *Op = CtpopLhs->getArgOperand(0);
-    KnownBits OpKnown = computeKnownBits(Op, Q.DL,
-                                         /*Depth*/ 0, Q.AC, Q.CxtI, Q.DT);
+    KnownBits OpKnown = computeKnownBits(Op, Q.DL, Q.AC, Q.CxtI, Q.DT);
     // No need to check for count > 1, that should be already constant folded.
     if (OpKnown.countMinPopulation() == 1) {
       Value *And = Builder.CreateAnd(
@@ -4441,13 +4440,13 @@ static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q,
     // Pow2 - 1 is a Mask.
     if (!Not && match(I->getOperand(1), m_AllOnes()))
       return isKnownToBeAPowerOfTwo(I->getOperand(0), Q.DL, /*OrZero*/ true,
-                                    Depth, Q.AC, Q.CxtI, Q.DT);
+                                    Q.AC, Q.CxtI, Q.DT, Depth);
     break;
   case Instruction::Sub:
     // -Pow2 is a ~Mask.
     if (Not && match(I->getOperand(0), m_Zero()))
       return isKnownToBeAPowerOfTwo(I->getOperand(1), Q.DL, /*OrZero*/ true,
-                                    Depth, Q.AC, Q.CxtI, Q.DT);
+                                    Q.AC, Q.CxtI, Q.DT, Depth);
     break;
   case Instruction::Call: {
     if (auto *II = dyn_cast<IntrinsicInst>(I)) {
@@ -5008,7 +5007,7 @@ static Instruction *foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q,
   if (!ICmpInst::isSigned(Pred))
     return nullptr;
 
-  KnownBits KnownY = IC.computeKnownBits(A, /*Depth=*/0, &I);
+  KnownBits KnownY = IC.computeKnownBits(A, &I);
   // (X & NegY) spred X --> (X & NegY) upred X
   if (KnownY.isNegative())
     return new ICmpInst(ICmpInst::getUnsignedPredicate(Pred), Op0, Op1);
@@ -5440,7 +5439,7 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
          match(Op1, m_c_Mul(m_Specific(Z), m_Value(Y))))) {
       if (ICmpInst::isSigned(Pred)) {
         if (Op0HasNSW && Op1HasNSW) {
-          KnownBits ZKnown = computeKnownBits(Z, 0, &I);
+          KnownBits ZKnown = computeKnownBits(Z, &I);
           if (ZKnown.isStrictlyPositive())
             return new ICmpInst(Pred, X, Y);
           if (ZKnown.isNegative())
@@ -5463,7 +5462,7 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
               isKnownNonEqual(X, Y, SQ))
             return new ICmpInst(Pred, Z, Constant::getNullValue(Z->getType()));
 
-          KnownBits ZKnown = computeKnownBits(Z, 0, &I);
+          KnownBits ZKnown = computeKnownBits(Z, &I);
           // if Z % 2 != 0
           //    X * Z eq/ne Y * Z -> X eq/ne Y
           if (ZKnown.countMaxTrailingZeros() == 0)
@@ -6144,12 +6143,12 @@ Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
   // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0
   // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0
   if (match(Op0, m_c_And(m_Specific(Op1), m_Value())) &&
-      isKnownToBeAPowerOfTwo(Op1, /* OrZero */ false, 0, &I))
+      isKnownToBeAPowerOfTwo(Op1, /* OrZero */ false, &I))
     return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
                         ConstantInt::getNullValue(Op0->getType()));
 
   if (match(Op1, m_c_And(m_Specific(Op0), m_Value())) &&
-      isKnownToBeAPowerOfTwo(Op0, /* OrZero */ false, 0, &I))
+      isKnownToBeAPowerOfTwo(Op0, /* OrZero */ false, &I))
     return new ICmpInst(CmpInst::getInversePredicate(Pred), Op1,
                         ConstantInt::getNullValue(Op1->getType()));
 
@@ -6187,7 +6186,7 @@ Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
                    m_ICmp(m_OneUse(m_c_And(m_Value(A), m_Matcher)), m_Zero())))
       IsZero = true;
 
-    if (IsZero && isKnownToBeAPowerOfTwo(A, /* OrZero */ true, /*Depth*/ 0, &I))
+    if (IsZero && isKnownToBeAPowerOfTwo(A, /* OrZero */ true, &I))
       // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
       //    -> (icmp eq/ne (and X, P2), 0)
       // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
@@ -6840,11 +6839,10 @@ Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) {
     // (especially IndVarSimplify) may not be able to reliably undo.
     SimplifyQuery Q = SQ.getWithoutDomCondCache().getWithInstruction(&I);
     if (SimplifyDemandedBits(&I, 0, getDemandedBitsLHSMask(I, BitWidth),
-                             Op0Known, /*Depth=*/0, Q))
+                             Op0Known, Q))
       return &I;
 
-    if (SimplifyDemandedBits(&I, 1, APInt::getAllOnes(BitWidth), Op1Known,
-                             /*Depth=*/0, Q))
+    if (SimplifyDemandedBits(&I, 1, APInt::getAllOnes(BitWidth), Op1Known, Q))
       return &I;
   }
 
@@ -7730,6 +7728,30 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
     }
   }
 
+  // icmp slt (sub nsw x, y), (add nsw x, y)  -->  icmp sgt y, 0
+  // icmp ult (sub nuw x, y), (add nuw x, y)  -->  icmp ugt y, 0
+  // icmp eq (sub nsw/nuw x, y), (add nsw/nuw x, y)   -->  icmp eq y, 0
+  {
+    Value *A, *B;
+    CmpPredicate CmpPred;
+    if (match(&I, m_c_ICmp(CmpPred, m_Sub(m_Value(A), m_Value(B)),
+                           m_c_Add(m_Deferred(A), m_Deferred(B))))) {
+      auto *I0 = cast<OverflowingBinaryOperator>(Op0);
+      auto *I1 = cast<OverflowingBinaryOperator>(Op1);
+      bool I0NUW = I0->hasNoUnsignedWrap();
+      bool I1NUW = I1->hasNoUnsignedWrap();
+      bool I0NSW = I0->hasNoSignedWrap();
+      bool I1NSW = I1->hasNoSignedWrap();
+      if ((ICmpInst::isUnsigned(Pred) && I0NUW && I1NUW) ||
+          (ICmpInst::isSigned(Pred) && I0NSW && I1NSW) ||
+          (ICmpInst::isEquality(Pred) &&
+           ((I0NUW || I0NSW) && (I1NUW || I1NSW)))) {
+        return new ICmpInst(CmpPredicate::getSwapped(CmpPred), B,
+                            ConstantInt::get(Op0->getType(), 0));
+      }
+    }
+  }
+
   // Try to optimize equality comparisons against alloca-based pointers.
   if (Op0->getType()->isPointerTy() && I.isEquality()) {
     assert(Op1->getType()->isPointerTy() &&
@@ -7755,7 +7777,7 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
     // and       (X & ~Y) != 0 --> (X & Y) == 0
     // if A is a power of 2.
     if (match(Op0, m_And(m_Value(X), m_Not(m_Value(Y)))) &&
-        match(Op1, m_Zero()) && isKnownToBeAPowerOfTwo(X, false, 0, &I) &&
+        match(Op1, m_Zero()) && isKnownToBeAPowerOfTwo(X, false, &I) &&
         I.isEquality())
       return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(X, Y),
                           Op1);
@@ -8378,10 +8400,9 @@ static Instruction *foldFCmpFSubIntoFCmp(FCmpInst &I, Instruction *LHSI,
     // flag then we can assume we do not have that case. Otherwise we might be
     // able to prove that either X or Y is not infinity.
     if (!LHSI->hasNoNaNs() && !LHSI->hasNoInfs() &&
-        !isKnownNeverInfinity(Y, /*Depth=*/0,
+        !isKnownNeverInfinity(Y,
                               CI.getSimplifyQuery().getWithInstruction(&I)) &&
-        !isKnownNeverInfinity(X, /*Depth=*/0,
-                              CI.getSimplifyQuery().getWithInstruction(&I)))
+        !isKnownNeverInfinity(X, CI.getSimplifyQuery().getWithInstruction(&I)))
       break;
 
     [[fallthrough]];
@@ -8531,11 +8552,11 @@ Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) {
   // then canonicalize the operand to 0.0.
   if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
     if (!match(Op0, m_PosZeroFP()) &&
-        isKnownNeverNaN(Op0, 0, getSimplifyQuery().getWithInstruction(&I)))
+        isKnownNeverNaN(Op0, getSimplifyQuery().getWithInstruction(&I)))
       return replaceOperand(I, 0, ConstantFP::getZero(OpType));
 
     if (!match(Op1, m_PosZeroFP()) &&
-        isKnownNeverNaN(Op1, 0, getSimplifyQuery().getWithInstruction(&I)))
+        isKnownNeverNaN(Op1, getSimplifyQuery().getWithInstruction(&I)))
       return replaceOperand(I, 1, ConstantFP::getZero(OpType));
   }
 
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 5e0cd17fb192..8c9de862fe8f 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -204,8 +204,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
                                    const Instruction *CtxI = nullptr,
                                    unsigned Depth = 0) const {
     return llvm::computeKnownFPClass(
-        Val, FMF, Interested, Depth,
-        getSimplifyQuery().getWithInstruction(CtxI));
+        Val, FMF, Interested, getSimplifyQuery().getWithInstruction(CtxI),
+        Depth);
   }
 
   KnownFPClass computeKnownFPClass(Value *Val,
@@ -213,7 +213,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
                                    const Instruction *CtxI = nullptr,
                                    unsigned Depth = 0) const {
     return llvm::computeKnownFPClass(
-        Val, Interested, Depth, getSimplifyQuery().getWithInstruction(CtxI));
+        Val, Interested, getSimplifyQuery().getWithInstruction(CtxI), Depth);
   }
 
   /// Check if fmul \p MulVal, +0.0 will yield +0.0 (or signed zero is
@@ -378,6 +378,10 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   }
 
   Value *EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP = false);
+  /// Emit sum of multiple GEP offsets. The GEPs are processed in reverse
+  /// order.
+  Value *EmitGEPOffsets(ArrayRef<GEPOperator *> GEPs, GEPNoWrapFlags NW,
+                        Type *IdxTy, bool RewriteGEPs);
   Instruction *scalarizePHI(ExtractElementInst &EI, PHINode *PN);
   Instruction *foldBitcastExtElt(ExtractElementInst &ExtElt);
   Instruction *foldCastedBitwiseLogic(BinaryOperator &I);
@@ -558,20 +562,22 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   /// Attempts to replace I with a simpler value based on the demanded
   /// bits.
   Value *SimplifyDemandedUseBits(Instruction *I, const APInt &DemandedMask,
-                                 KnownBits &Known, unsigned Depth,
-                                 const SimplifyQuery &Q);
+                                 KnownBits &Known, const SimplifyQuery &Q,
+                                 unsigned Depth = 0);
   using InstCombiner::SimplifyDemandedBits;
   bool SimplifyDemandedBits(Instruction *I, unsigned Op,
                             const APInt &DemandedMask, KnownBits &Known,
-                            unsigned Depth, const SimplifyQuery &Q) override;
+                            const SimplifyQuery &Q,
+                            unsigned Depth = 0) override;
 
   /// Helper routine of SimplifyDemandedUseBits. It computes KnownZero/KnownOne
   /// bits. It also tries to handle simplifications that can be done based on
   /// DemandedMask, but without modifying the Instruction.
   Value *SimplifyMultipleUseDemandedBits(Instruction *I,
                                          const APInt &DemandedMask,
-                                         KnownBits &Known, unsigned Depth,
-                                         const SimplifyQuery &Q);
+                                         KnownBits &Known,
+                                         const SimplifyQuery &Q,
+                                         unsigned Depth = 0);
 
   /// Helper routine of SimplifyDemandedUseBits. It tries to simplify demanded
   /// bit for "r1 = shr x, c1; r2 = shl r1, c2" instruction sequence.
@@ -591,8 +597,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   /// Attempts to replace V with a simpler value based on the demanded
   /// floating-point classes
   Value *SimplifyDemandedUseFPClass(Value *V, FPClassTest DemandedMask,
-                                    KnownFPClass &Known, unsigned Depth,
-                                    Instruction *CxtI);
+                                    KnownFPClass &Known, Instruction *CxtI,
+                                    unsigned Depth = 0);
   bool SimplifyDemandedFPClass(Instruction *I, unsigned Op,
                                FPClassTest DemandedMask, KnownFPClass &Known,
                                unsigned Depth = 0);
@@ -765,6 +771,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
                             Value *A, Value *B, Instruction &Outer,
                             SelectPatternFlavor SPF2, Value *C);
   Instruction *foldSelectInstWithICmp(SelectInst &SI, ICmpInst *ICI);
+  Value *foldSelectWithConstOpToBinOp(ICmpInst *Cmp, Value *TrueVal,
+                                      Value *FalseVal);
   Instruction *foldSelectValueEquivalence(SelectInst &SI, CmpInst &CI);
   bool replaceInInstruction(Value *V, Value *Old, Value *New,
                             unsigned Depth = 0);
@@ -860,6 +868,21 @@ class Negator final {
                                      InstCombinerImpl &IC);
 };
 
+struct CommonPointerBase {
+  /// Common base pointer.
+  Value *Ptr = nullptr;
+  /// LHS GEPs until common base.
+  SmallVector<GEPOperator *> LHSGEPs;
+  /// RHS GEPs until common base.
+  SmallVector<GEPOperator *> RHSGEPs;
+  /// LHS GEP NoWrapFlags until common base.
+  GEPNoWrapFlags LHSNW = GEPNoWrapFlags::all();
+  /// RHS GEP NoWrapFlags until common base.
+  GEPNoWrapFlags RHSNW = GEPNoWrapFlags::all();
+
+  static CommonPointerBase compute(Value *LHS, Value *RHS);
+};
+
 } // end namespace llvm
 
 #undef DEBUG_TYPE
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index c29cba6f675c..1d208de75db3 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -918,7 +918,7 @@ static bool canReplaceGEPIdxWithZero(InstCombinerImpl &IC,
   // first non-zero index.
   auto IsAllNonNegative = [&]() {
     for (unsigned i = Idx+1, e = GEPI->getNumOperands(); i != e; ++i) {
-      KnownBits Known = IC.computeKnownBits(GEPI->getOperand(i), 0, MemI);
+      KnownBits Known = IC.computeKnownBits(GEPI->getOperand(i), MemI);
       if (Known.isNonNegative())
         continue;
       return false;
@@ -1581,8 +1581,8 @@ bool InstCombinerImpl::mergeStoreIntoSuccessor(StoreInst &SI) {
   // Insert a PHI node now if we need it.
   Value *MergedVal = OtherStore->getValueOperand();
   // The debug locations of the original instructions might differ. Merge them.
-  DebugLoc MergedLoc = DILocation::getMergedLocation(SI.getDebugLoc(),
-                                                     OtherStore->getDebugLoc());
+  DebugLoc MergedLoc =
+      DebugLoc::getMergedLocation(SI.getDebugLoc(), OtherStore->getDebugLoc());
   if (MergedVal != SI.getValueOperand()) {
     PHINode *PN =
         PHINode::Create(SI.getValueOperand()->getType(), 2, "storemerge");
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index c7023eb79b04..457199a72510 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -66,7 +66,7 @@ static Value *simplifyValueKnownNonZero(Value *V, InstCombinerImpl &IC,
   // inexact.  Similarly for <<.
   BinaryOperator *I = dyn_cast<BinaryOperator>(V);
   if (I && I->isLogicalShift() &&
-      IC.isKnownToBeAPowerOfTwo(I->getOperand(0), false, 0, &CxtI)) {
+      IC.isKnownToBeAPowerOfTwo(I->getOperand(0), false, &CxtI)) {
     // We know that this is an exact/nuw shift and that the input is a
     // non-zero context as well.
     if (Value *V2 = simplifyValueKnownNonZero(I->getOperand(0), IC, CxtI)) {
@@ -987,9 +987,8 @@ Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) {
   // X * -0.0 --> copysign(0.0, -X)
   const APFloat *FPC;
   if (match(Op1, m_APFloatAllowPoison(FPC)) && FPC->isZero() &&
-      ((I.hasNoInfs() &&
-        isKnownNeverNaN(Op0, /*Depth=*/0, SQ.getWithInstruction(&I))) ||
-       isKnownNeverNaN(&I, /*Depth=*/0, SQ.getWithInstruction(&I)))) {
+      ((I.hasNoInfs() && isKnownNeverNaN(Op0, SQ.getWithInstruction(&I))) ||
+       isKnownNeverNaN(&I, SQ.getWithInstruction(&I)))) {
     if (FPC->isNegative())
       Op0 = Builder.CreateFNegFMF(Op0, &I);
     CallInst *CopySign = Builder.CreateIntrinsic(Intrinsic::copysign,
@@ -1720,7 +1719,7 @@ Instruction *InstCombinerImpl::visitUDiv(BinaryOperator &I) {
       return Log2;
 
     // Op0 udiv Op1 -> Op0 lshr cttz(Op1), if Op1 is a power of 2.
-    if (isKnownToBeAPowerOfTwo(Denom, /*OrZero=*/true, /*Depth=*/0, &I))
+    if (isKnownToBeAPowerOfTwo(Denom, /*OrZero=*/true, &I))
       // This will increase instruction count but it's okay
       // since bitwise operations are substantially faster than
       // division.
@@ -1830,7 +1829,7 @@ Instruction *InstCombinerImpl::visitSDiv(BinaryOperator &I) {
                               ConstantInt::getAllOnesValue(Ty));
   }
 
-  KnownBits KnownDividend = computeKnownBits(Op0, 0, &I);
+  KnownBits KnownDividend = computeKnownBits(Op0, &I);
   if (!I.isExact() &&
       (match(Op1, m_Power2(Op1C)) || match(Op1, m_NegatedPower2(Op1C))) &&
       KnownDividend.countMinTrailingZeros() >= Op1C->countr_zero()) {
@@ -1855,7 +1854,7 @@ Instruction *InstCombinerImpl::visitSDiv(BinaryOperator &I) {
       return BinaryOperator::CreateNeg(Shr);
     }
 
-    if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, &I)) {
+    if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, &I)) {
       // X sdiv (1 << Y) -> X udiv (1 << Y) ( -> X u>> Y)
       // Safe because the only negative value (1 << Y) can take on is
       // INT_MIN, and X sdiv INT_MIN == X udiv INT_MIN == 0 if X doesn't have
@@ -2409,7 +2408,7 @@ Instruction *InstCombinerImpl::visitURem(BinaryOperator &I) {
   // X urem Y -> X and Y-1, where Y is a power of 2,
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Type *Ty = I.getType();
-  if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, &I)) {
+  if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, &I)) {
     // This may increase instruction count, we don't enforce that Y is a
     // constant.
     Constant *N1 = Constant::getAllOnesValue(Ty);
@@ -2492,8 +2491,7 @@ Instruction *InstCombinerImpl::visitSRem(BinaryOperator &I) {
   // If the sign bits of both operands are zero (i.e. we can prove they are
   // unsigned inputs), turn this into a urem.
   APInt Mask(APInt::getSignMask(I.getType()->getScalarSizeInBits()));
-  if (MaskedValueIsZero(Op1, Mask, 0, &I) &&
-      MaskedValueIsZero(Op0, Mask, 0, &I)) {
+  if (MaskedValueIsZero(Op1, Mask, &I) && MaskedValueIsZero(Op0, Mask, &I)) {
     // X srem Y -> X urem Y, iff X and Y don't have sign bit set
     return BinaryOperator::CreateURem(Op0, Op1, I.getName());
   }
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index a842a5edcb8a..6477141ab095 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -870,7 +870,14 @@ Instruction *InstCombinerImpl::foldPHIArgZextsIntoPHI(PHINode &Phi) {
     NewPhi->addIncoming(NewIncoming[I], Phi.getIncomingBlock(I));
 
   InsertNewInstBefore(NewPhi, Phi.getIterator());
-  return CastInst::CreateZExtOrBitCast(NewPhi, Phi.getType());
+  auto *CI = CastInst::CreateZExtOrBitCast(NewPhi, Phi.getType());
+
+  // We use a dropped location here because the new ZExt is necessarily a merge
+  // of ZExtInsts and at least one constant from incoming branches; the presence
+  // of the constant means we have no viable DebugLoc from that branch, and
+  // therefore we must use a dropped location.
+  CI->setDebugLoc(DebugLoc::getDropped());
+  return CI;
 }
 
 /// If all operands to a PHI node are the same "unary" operator and they all are
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index cca5705f6b58..73ba0f78e805 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -100,7 +100,7 @@ static Instruction *foldSelectBinOpIdentity(SelectInst &Sel,
   // transform. Bail out if we can not exclude that possibility.
   if (isa<FPMathOperator>(BO))
     if (!BO->hasNoSignedZeros() &&
-        !cannotBeNegativeZero(Y, 0,
+        !cannotBeNegativeZero(Y,
                               IC.getSimplifyQuery().getWithInstruction(&Sel)))
       return nullptr;
 
@@ -565,6 +565,62 @@ Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal,
   return nullptr;
 }
 
+/// Try to fold a select to a min/max intrinsic. Many cases are already handled
+/// by matchDecomposedSelectPattern but here we handle the cases where more
+/// extensive modification of the IR is required.
+static Value *foldSelectICmpMinMax(const ICmpInst *Cmp, Value *TVal,
+                                   Value *FVal,
+                                   InstCombiner::BuilderTy &Builder,
+                                   const SimplifyQuery &SQ) {
+  const Value *CmpLHS = Cmp->getOperand(0);
+  const Value *CmpRHS = Cmp->getOperand(1);
+  ICmpInst::Predicate Pred = Cmp->getPredicate();
+
+  // (X > Y) ? X : (Y - 1) ==> MIN(X, Y - 1)
+  // (X < Y) ? X : (Y + 1) ==> MAX(X, Y + 1)
+  // This transformation is valid when overflow corresponding to the sign of
+  // the comparison is poison and we must drop the non-matching overflow flag.
+  if (CmpRHS == TVal) {
+    std::swap(CmpLHS, CmpRHS);
+    Pred = CmpInst::getSwappedPredicate(Pred);
+  }
+
+  // TODO: consider handling 'or disjoint' as well, though these would need to
+  // be converted to 'add' instructions.
+  if (!(CmpLHS == TVal && isa<Instruction>(FVal)))
+    return nullptr;
+
+  if (Pred == CmpInst::ICMP_SGT &&
+      match(FVal, m_NSWAdd(m_Specific(CmpRHS), m_One()))) {
+    cast<Instruction>(FVal)->setHasNoUnsignedWrap(false);
+    return Builder.CreateBinaryIntrinsic(Intrinsic::smax, TVal, FVal);
+  }
+
+  if (Pred == CmpInst::ICMP_SLT &&
+      match(FVal, m_NSWAdd(m_Specific(CmpRHS), m_AllOnes()))) {
+    cast<Instruction>(FVal)->setHasNoUnsignedWrap(false);
+    return Builder.CreateBinaryIntrinsic(Intrinsic::smin, TVal, FVal);
+  }
+
+  if (Pred == CmpInst::ICMP_UGT &&
+      match(FVal, m_NUWAdd(m_Specific(CmpRHS), m_One()))) {
+    cast<Instruction>(FVal)->setHasNoSignedWrap(false);
+    return Builder.CreateBinaryIntrinsic(Intrinsic::umax, TVal, FVal);
+  }
+
+  // Note: We must use isKnownNonZero here because "sub nuw %x, 1" will be
+  // canonicalized to "add %x, -1" discarding the nuw flag.
+  if (Pred == CmpInst::ICMP_ULT &&
+      match(FVal, m_Add(m_Specific(CmpRHS), m_AllOnes())) &&
+      isKnownNonZero(CmpRHS, SQ)) {
+    cast<Instruction>(FVal)->setHasNoSignedWrap(false);
+    cast<Instruction>(FVal)->setHasNoUnsignedWrap(false);
+    return Builder.CreateBinaryIntrinsic(Intrinsic::umin, TVal, FVal);
+  }
+
+  return nullptr;
+}
+
 /// We want to turn:
 ///   (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1)
 /// into:
@@ -1312,7 +1368,11 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
 
       // If NewOp is a constant and OldOp is not replace iff NewOp doesn't
       // contain and undef elements.
-      if (match(NewOp, m_ImmConstant()) || NewOp == V) {
+      // Make sure that V is always simpler than TrueVal, otherwise we might
+      // end up in an infinite loop.
+      if (match(NewOp, m_ImmConstant()) ||
+          (isa<Instruction>(TrueVal) &&
+           is_contained(cast<Instruction>(TrueVal)->operands(), V))) {
         if (isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT))
           return replaceOperand(Sel, Swapped ? 2 : 1, V);
         return nullptr;
@@ -1819,10 +1879,9 @@ static Instruction *foldSelectICmpEq(SelectInst &SI, ICmpInst *ICI,
 
 /// Fold `X Pred C1 ? X BOp C2 : C1 BOp C2` to `min/max(X, C1) BOp C2`.
 /// This allows for better canonicalization.
-static Value *foldSelectWithConstOpToBinOp(ICmpInst *Cmp, Value *TrueVal,
-                                           Value *FalseVal,
-                                           IRBuilderBase &Builder) {
-  BinaryOperator *BOp;
+Value *InstCombinerImpl::foldSelectWithConstOpToBinOp(ICmpInst *Cmp,
+                                                      Value *TrueVal,
+                                                      Value *FalseVal) {
   Constant *C1, *C2, *C3;
   Value *X;
   CmpPredicate Predicate;
@@ -1838,39 +1897,77 @@ static Value *foldSelectWithConstOpToBinOp(ICmpInst *Cmp, Value *TrueVal,
     Predicate = ICmpInst::getInversePredicate(Predicate);
   }
 
-  if (!match(TrueVal, m_BinOp(BOp)) || !match(FalseVal, m_Constant(C3)))
+  if (!match(FalseVal, m_Constant(C3)) || !TrueVal->hasOneUse())
     return nullptr;
 
-  unsigned Opcode = BOp->getOpcode();
+  bool IsIntrinsic;
+  unsigned Opcode;
+  if (BinaryOperator *BOp = dyn_cast<BinaryOperator>(TrueVal)) {
+    Opcode = BOp->getOpcode();
+    IsIntrinsic = false;
 
-  // This fold causes some regressions and is primarily intended for
-  // add and sub. So we early exit for div and rem to minimize the
-  // regressions.
-  if (Instruction::isIntDivRem(Opcode))
-    return nullptr;
+    // This fold causes some regressions and is primarily intended for
+    // add and sub. So we early exit for div and rem to minimize the
+    // regressions.
+    if (Instruction::isIntDivRem(Opcode))
+      return nullptr;
 
-  if (!match(BOp, m_OneUse(m_BinOp(m_Specific(X), m_Constant(C2)))))
+    if (!match(BOp, m_BinOp(m_Specific(X), m_Constant(C2))))
+      return nullptr;
+
+  } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(TrueVal)) {
+    if (!match(II, m_MaxOrMin(m_Specific(X), m_Constant(C2))))
+      return nullptr;
+    Opcode = II->getIntrinsicID();
+    IsIntrinsic = true;
+  } else {
     return nullptr;
+  }
 
   Value *RHS;
   SelectPatternFlavor SPF;
-  const DataLayout &DL = BOp->getDataLayout();
+  const DataLayout &DL = Cmp->getDataLayout();
   auto Flipped = getFlippedStrictnessPredicateAndConstant(Predicate, C1);
 
-  if (C3 == ConstantFoldBinaryOpOperands(Opcode, C1, C2, DL)) {
+  auto FoldBinaryOpOrIntrinsic = [&](Constant *LHS, Constant *RHS) {
+    return IsIntrinsic ? ConstantFoldBinaryIntrinsic(Opcode, LHS, RHS,
+                                                     LHS->getType(), nullptr)
+                       : ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
+  };
+
+  if (C3 == FoldBinaryOpOrIntrinsic(C1, C2)) {
     SPF = getSelectPattern(Predicate).Flavor;
     RHS = C1;
-  } else if (Flipped && C3 == ConstantFoldBinaryOpOperands(
-                                  Opcode, Flipped->second, C2, DL)) {
+  } else if (Flipped && C3 == FoldBinaryOpOrIntrinsic(Flipped->second, C2)) {
     SPF = getSelectPattern(Flipped->first).Flavor;
     RHS = Flipped->second;
   } else {
     return nullptr;
   }
 
-  Intrinsic::ID IntrinsicID = getMinMaxIntrinsic(SPF);
-  Value *Intrinsic = Builder.CreateBinaryIntrinsic(IntrinsicID, X, RHS);
-  return Builder.CreateBinOp(BOp->getOpcode(), Intrinsic, C2);
+  Intrinsic::ID MinMaxID = getMinMaxIntrinsic(SPF);
+  Value *MinMax = Builder.CreateBinaryIntrinsic(MinMaxID, X, RHS);
+  if (IsIntrinsic)
+    return Builder.CreateBinaryIntrinsic(Opcode, MinMax, C2);
+
+  const auto BinOpc = Instruction::BinaryOps(Opcode);
+  Value *BinOp = Builder.CreateBinOp(BinOpc, MinMax, C2);
+
+  // If we can attach no-wrap flags to the new instruction, do so if the
+  // old instruction had them and C1 BinOp C2 does not overflow.
+  if (Instruction *BinOpInst = dyn_cast<Instruction>(BinOp)) {
+    if (BinOpc == Instruction::Add || BinOpc == Instruction::Sub ||
+        BinOpc == Instruction::Mul) {
+      Instruction *OldBinOp = cast<BinaryOperator>(TrueVal);
+      if (OldBinOp->hasNoSignedWrap() &&
+          willNotOverflow(BinOpc, RHS, C2, *BinOpInst, /*IsSigned=*/true))
+        BinOpInst->setHasNoSignedWrap();
+      if (OldBinOp->hasNoUnsignedWrap() &&
+          willNotOverflow(BinOpc, RHS, C2, *BinOpInst, /*IsSigned=*/false))
+        BinOpInst->setHasNoUnsignedWrap();
+    }
+  }
+  return BinOp;
 }
 
 /// Visit a SelectInst that has an ICmpInst as its first operand.
@@ -1917,6 +2014,9 @@ Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
     return &SI;
   }
 
+  if (Value *V = foldSelectICmpMinMax(ICI, TrueVal, FalseVal, Builder, SQ))
+    return replaceInstUsesWith(SI, V);
+
   if (Instruction *V =
           foldSelectICmpAndAnd(SI.getType(), ICI, TrueVal, FalseVal, Builder))
     return V;
@@ -1945,7 +2045,7 @@ Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
   if (Value *V = foldAbsDiff(ICI, TrueVal, FalseVal, Builder))
     return replaceInstUsesWith(SI, V);
 
-  if (Value *V = foldSelectWithConstOpToBinOp(ICI, TrueVal, FalseVal, Builder))
+  if (Value *V = foldSelectWithConstOpToBinOp(ICI, TrueVal, FalseVal))
     return replaceInstUsesWith(SI, V);
 
   return Changed ? &SI : nullptr;
@@ -2800,9 +2900,8 @@ static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI,
     if (match(TrueVal, m_FSub(m_PosZeroFP(), m_Specific(X))) &&
         (cast<FPMathOperator>(CondVal)->hasNoNaNs() || SI.hasNoNaNs() ||
          (SI.hasOneUse() && canIgnoreSignBitOfNaN(*SI.use_begin())) ||
-         isKnownNeverNaN(X, /*Depth=*/0,
-                         IC.getSimplifyQuery().getWithInstruction(
-                             cast<Instruction>(CondVal))))) {
+         isKnownNeverNaN(X, IC.getSimplifyQuery().getWithInstruction(
+                                cast<Instruction>(CondVal))))) {
       if (!Swap && (Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE)) {
         Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
         return IC.replaceInstUsesWith(SI, Fabs);
@@ -3550,6 +3649,28 @@ Instruction *InstCombinerImpl::foldSelectToCmp(SelectInst &SI) {
   if (!LHS->getType()->isIntOrIntVectorTy())
     return nullptr;
 
+  // If there is no -1, 0 or 1 at TV, then invert the select statement and try
+  // to canonicalize to one of the forms above
+  if (!isa<Constant>(TV)) {
+    if (!isa<Constant>(FV))
+      return nullptr;
+    Pred = ICmpInst::getInverseCmpPredicate(Pred);
+    std::swap(TV, FV);
+  }
+
+  if (ICmpInst::isNonStrictPredicate(Pred)) {
+    if (Constant *C = dyn_cast<Constant>(RHS)) {
+      auto FlippedPredAndConst =
+          getFlippedStrictnessPredicateAndConstant(Pred, C);
+      if (!FlippedPredAndConst)
+        return nullptr;
+      Pred = FlippedPredAndConst->first;
+      RHS = FlippedPredAndConst->second;
+    } else {
+      return nullptr;
+    }
+  }
+
   // Try to swap operands and the predicate. We need to be careful when doing
   // so because two of the patterns have opposite predicates, so use the
   // constant inside select to determine if swapping operands would be
@@ -3757,8 +3878,7 @@ static Value *foldSelectBitTest(SelectInst &Sel, Value *CondVal, Value *TrueVal,
       assert(ICmpInst::isEquality(Res->Pred) && "Not equality test?");
       AndMask = Res->Mask;
       V = Res->X;
-      KnownBits Known =
-          computeKnownBits(V, /*Depth=*/0, SQ.getWithInstruction(&Sel));
+      KnownBits Known = computeKnownBits(V, SQ.getWithInstruction(&Sel));
       AndMask &= Known.getMaxValue();
       if (!AndMask.isPowerOf2())
         return nullptr;
@@ -4185,7 +4305,7 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   // the assumption cache, so make sure that is populated.
   if (!CondVal->getType()->isVectorTy() && !AC.assumptions().empty()) {
     KnownBits Known(1);
-    computeKnownBits(CondVal, Known, 0, &SI);
+    computeKnownBits(CondVal, Known, &SI);
     if (Known.One.isOne())
       return replaceInstUsesWith(SI, TrueVal);
     if (Known.Zero.isOne())
@@ -4340,7 +4460,7 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
     if (!CC.AffectedValues.empty()) {
       if (!isa<Constant>(TrueVal) &&
           hasAffectedValue(TrueVal, CC.AffectedValues, /*Depth=*/0)) {
-        KnownBits Known = llvm::computeKnownBits(TrueVal, /*Depth=*/0, Q);
+        KnownBits Known = llvm::computeKnownBits(TrueVal, Q);
         if (Known.isConstant())
           return replaceOperand(SI, 1,
                                 ConstantInt::get(SelType, Known.getConstant()));
@@ -4349,7 +4469,7 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
       CC.Invert = true;
       if (!isa<Constant>(FalseVal) &&
           hasAffectedValue(FalseVal, CC.AffectedValues, /*Depth=*/0)) {
-        KnownBits Known = llvm::computeKnownBits(FalseVal, /*Depth=*/0, Q);
+        KnownBits Known = llvm::computeKnownBits(FalseVal, Q);
         if (Known.isConstant())
           return replaceOperand(SI, 2,
                                 ConstantInt::get(SelType, Known.getConstant()));
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 7d369a1639a3..550f095b26ba 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -567,7 +567,7 @@ static bool canEvaluateShiftedShift(unsigned OuterShAmt, bool IsOuterShl,
     unsigned MaskShift =
         IsInnerShl ? TypeWidth - InnerShAmt : InnerShAmt - OuterShAmt;
     APInt Mask = APInt::getLowBitsSet(TypeWidth, OuterShAmt) << MaskShift;
-    if (IC.MaskedValueIsZero(InnerShift->getOperand(0), Mask, 0, CxtI))
+    if (IC.MaskedValueIsZero(InnerShift->getOperand(0), Mask, CxtI))
       return true;
   }
 
@@ -1003,13 +1003,13 @@ static bool setShiftFlags(BinaryOperator &I, const SimplifyQuery &Q) {
   }
 
   // Compute what we know about shift count.
-  KnownBits KnownCnt = computeKnownBits(I.getOperand(1), /* Depth */ 0, Q);
+  KnownBits KnownCnt = computeKnownBits(I.getOperand(1), Q);
   unsigned BitWidth = KnownCnt.getBitWidth();
   // Since shift produces a poison value if RHS is equal to or larger than the
   // bit width, we can safely assume that RHS is less than the bit width.
   uint64_t MaxCnt = KnownCnt.getMaxValue().getLimitedValue(BitWidth - 1);
 
-  KnownBits KnownAmt = computeKnownBits(I.getOperand(0), /* Depth */ 0, Q);
+  KnownBits KnownAmt = computeKnownBits(I.getOperand(0), Q);
   bool Changed = false;
 
   if (I.getOpcode() == Instruction::Shl) {
@@ -1021,8 +1021,8 @@ static bool setShiftFlags(BinaryOperator &I, const SimplifyQuery &Q) {
     // If we have more sign bits than maximum shift cnt we have nsw.
     if (!I.hasNoSignedWrap()) {
       if (MaxCnt < KnownAmt.countMinSignBits() ||
-          MaxCnt < ComputeNumSignBits(I.getOperand(0), Q.DL, /*Depth*/ 0, Q.AC,
-                                      Q.CxtI, Q.DT)) {
+          MaxCnt <
+              ComputeNumSignBits(I.getOperand(0), Q.DL, Q.AC, Q.CxtI, Q.DT)) {
         I.setHasNoSignedWrap();
         Changed = true;
       }
@@ -1068,7 +1068,7 @@ Instruction *InstCombinerImpl::visitShl(BinaryOperator &I) {
     if (match(Op0, m_OneUse(m_ZExt(m_Value(X))))) {
       unsigned SrcWidth = X->getType()->getScalarSizeInBits();
       if (ShAmtC < SrcWidth &&
-          MaskedValueIsZero(X, APInt::getHighBitsSet(SrcWidth, ShAmtC), 0, &I))
+          MaskedValueIsZero(X, APInt::getHighBitsSet(SrcWidth, ShAmtC), &I))
         return new ZExtInst(Builder.CreateShl(X, ShAmtC), Ty);
     }
 
@@ -1311,8 +1311,8 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
 
   // Fold (X + Y) / 2 --> (X & Y) iff (X u<= 1) && (Y u<= 1)
   if (match(Op0, m_Add(m_Value(X), m_Value(Y))) && match(Op1, m_One()) &&
-      computeKnownBits(X, /*Depth=*/0, &I).countMaxActiveBits() <= 1 &&
-      computeKnownBits(Y, /*Depth=*/0, &I).countMaxActiveBits() <= 1)
+      computeKnownBits(X, &I).countMaxActiveBits() <= 1 &&
+      computeKnownBits(Y, &I).countMaxActiveBits() <= 1)
     return BinaryOperator::CreateAnd(X, Y);
 
   // (sub nuw X, (Y << nuw Z)) >>u exact Z --> (X >>u exact Z) sub nuw Y
@@ -1625,7 +1625,7 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
   if (match(Op0, m_Shl(m_Value(Shl0_Op0), m_Value(Shl0_Op1))) &&
       match(Op1, m_Intrinsic<Intrinsic::cttz>(m_BinOp(Shl1))) &&
       match(Shl1, m_Shl(m_Specific(Shl0_Op0), m_Value(Shl1_Op1))) &&
-      isKnownToBeAPowerOfTwo(Shl0_Op0, /*OrZero=*/true, 0, &I)) {
+      isKnownToBeAPowerOfTwo(Shl0_Op0, /*OrZero=*/true, &I)) {
     auto *Shl0 = cast<BinaryOperator>(Op0);
     bool HasNUW = Shl0->hasNoUnsignedWrap() && Shl1->hasNoUnsignedWrap();
     bool HasNSW = Shl0->hasNoSignedWrap() && Shl1->hasNoSignedWrap();
@@ -1829,7 +1829,7 @@ Instruction *InstCombinerImpl::visitAShr(BinaryOperator &I) {
     return R;
 
   // See if we can turn a signed shr into an unsigned shr.
-  if (MaskedValueIsZero(Op0, APInt::getSignMask(BitWidth), 0, &I)) {
+  if (MaskedValueIsZero(Op0, APInt::getSignMask(BitWidth), &I)) {
     Instruction *Lshr = BinaryOperator::CreateLShr(Op0, Op1);
     Lshr->setIsExact(I.isExact());
     return Lshr;
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index b5c1ee0e01cd..0e3436d12702 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -75,7 +75,7 @@ bool InstCombinerImpl::SimplifyDemandedInstructionBits(Instruction &Inst,
                                                        KnownBits &Known) {
   APInt DemandedMask(APInt::getAllOnes(Known.getBitWidth()));
   Value *V = SimplifyDemandedUseBits(&Inst, DemandedMask, Known,
-                                     0, SQ.getWithInstruction(&Inst));
+                                     SQ.getWithInstruction(&Inst));
   if (!V) return false;
   if (V == &Inst) return true;
   replaceInstUsesWith(Inst, V);
@@ -94,12 +94,13 @@ bool InstCombinerImpl::SimplifyDemandedInstructionBits(Instruction &Inst) {
 /// change and false otherwise.
 bool InstCombinerImpl::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
                                             const APInt &DemandedMask,
-                                            KnownBits &Known, unsigned Depth,
-                                            const SimplifyQuery &Q) {
+                                            KnownBits &Known,
+                                            const SimplifyQuery &Q,
+                                            unsigned Depth) {
   Use &U = I->getOperandUse(OpNo);
   Value *V = U.get();
   if (isa<Constant>(V)) {
-    llvm::computeKnownBits(V, Known, Depth, Q);
+    llvm::computeKnownBits(V, Known, Q, Depth);
     return false;
   }
 
@@ -112,7 +113,7 @@ bool InstCombinerImpl::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
 
   Instruction *VInst = dyn_cast<Instruction>(V);
   if (!VInst) {
-    llvm::computeKnownBits(V, Known, Depth, Q);
+    llvm::computeKnownBits(V, Known, Q, Depth);
     return false;
   }
 
@@ -122,12 +123,12 @@ bool InstCombinerImpl::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
   Value *NewVal;
   if (VInst->hasOneUse()) {
     // If the instruction has one use, we can directly simplify it.
-    NewVal = SimplifyDemandedUseBits(VInst, DemandedMask, Known, Depth, Q);
+    NewVal = SimplifyDemandedUseBits(VInst, DemandedMask, Known, Q, Depth);
   } else {
     // If there are multiple uses of this instruction, then we can simplify
     // VInst to some other value, but not modify the instruction.
     NewVal =
-        SimplifyMultipleUseDemandedBits(VInst, DemandedMask, Known, Depth, Q);
+        SimplifyMultipleUseDemandedBits(VInst, DemandedMask, Known, Q, Depth);
   }
   if (!NewVal) return false;
   if (Instruction* OpInst = dyn_cast<Instruction>(U))
@@ -163,8 +164,8 @@ bool InstCombinerImpl::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
 Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
                                                  const APInt &DemandedMask,
                                                  KnownBits &Known,
-                                                 unsigned Depth,
-                                                 const SimplifyQuery &Q) {
+                                                 const SimplifyQuery &Q,
+                                                 unsigned Depth) {
   assert(I != nullptr && "Null pointer of Value???");
   assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
   uint32_t BitWidth = DemandedMask.getBitWidth();
@@ -198,9 +199,9 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     // significant bit and all those below it.
     DemandedFromOps = APInt::getLowBitsSet(BitWidth, BitWidth - NLZ);
     if (ShrinkDemandedConstant(I, 0, DemandedFromOps) ||
-        SimplifyDemandedBits(I, 0, DemandedFromOps, LHSKnown, Depth + 1, Q) ||
+        SimplifyDemandedBits(I, 0, DemandedFromOps, LHSKnown, Q, Depth + 1) ||
         ShrinkDemandedConstant(I, 1, DemandedFromOps) ||
-        SimplifyDemandedBits(I, 1, DemandedFromOps, RHSKnown, Depth + 1, Q)) {
+        SimplifyDemandedBits(I, 1, DemandedFromOps, RHSKnown, Q, Depth + 1)) {
       disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
       return true;
     }
@@ -209,17 +210,17 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
 
   switch (I->getOpcode()) {
   default:
-    llvm::computeKnownBits(I, Known, Depth, Q);
+    llvm::computeKnownBits(I, Known, Q, Depth);
     break;
   case Instruction::And: {
     // If either the LHS or the RHS are Zero, the result is zero.
-    if (SimplifyDemandedBits(I, 1, DemandedMask, RHSKnown, Depth + 1, Q) ||
-        SimplifyDemandedBits(I, 0, DemandedMask & ~RHSKnown.Zero, LHSKnown,
-                             Depth + 1, Q))
+    if (SimplifyDemandedBits(I, 1, DemandedMask, RHSKnown, Q, Depth + 1) ||
+        SimplifyDemandedBits(I, 0, DemandedMask & ~RHSKnown.Zero, LHSKnown, Q,
+                             Depth + 1))
       return I;
 
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
-                                         Depth, Q);
+                                         Q, Depth);
 
     // If the client is only demanding bits that we know, return the known
     // constant.
@@ -241,16 +242,16 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
   }
   case Instruction::Or: {
     // If either the LHS or the RHS are One, the result is One.
-    if (SimplifyDemandedBits(I, 1, DemandedMask, RHSKnown, Depth + 1, Q) ||
-        SimplifyDemandedBits(I, 0, DemandedMask & ~RHSKnown.One, LHSKnown,
-                             Depth + 1, Q)) {
+    if (SimplifyDemandedBits(I, 1, DemandedMask, RHSKnown, Q, Depth + 1) ||
+        SimplifyDemandedBits(I, 0, DemandedMask & ~RHSKnown.One, LHSKnown, Q,
+                             Depth + 1)) {
       // Disjoint flag may not longer hold.
       I->dropPoisonGeneratingFlags();
       return I;
     }
 
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
-                                         Depth, Q);
+                                         Q, Depth);
 
     // If the client is only demanding bits that we know, return the known
     // constant.
@@ -281,8 +282,8 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     break;
   }
   case Instruction::Xor: {
-    if (SimplifyDemandedBits(I, 1, DemandedMask, RHSKnown, Depth + 1, Q) ||
-        SimplifyDemandedBits(I, 0, DemandedMask, LHSKnown, Depth + 1, Q))
+    if (SimplifyDemandedBits(I, 1, DemandedMask, RHSKnown, Q, Depth + 1) ||
+        SimplifyDemandedBits(I, 0, DemandedMask, LHSKnown, Q, Depth + 1))
       return I;
     Value *LHS, *RHS;
     if (DemandedMask == 1 &&
@@ -296,7 +297,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     }
 
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
-                                         Depth, Q);
+                                         Q, Depth);
 
     // If the client is only demanding bits that we know, return the known
     // constant.
@@ -373,8 +374,8 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     break;
   }
   case Instruction::Select: {
-    if (SimplifyDemandedBits(I, 2, DemandedMask, RHSKnown, Depth + 1, Q) ||
-        SimplifyDemandedBits(I, 1, DemandedMask, LHSKnown, Depth + 1, Q))
+    if (SimplifyDemandedBits(I, 2, DemandedMask, RHSKnown, Q, Depth + 1) ||
+        SimplifyDemandedBits(I, 1, DemandedMask, LHSKnown, Q, Depth + 1))
       return I;
 
     // If the operands are constants, see if we can simplify them.
@@ -415,9 +416,9 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
 
     // Only known if known in both the LHS and RHS.
     adjustKnownBitsForSelectArm(LHSKnown, I->getOperand(0), I->getOperand(1),
-                                /*Invert=*/false, Depth, Q);
+                                /*Invert=*/false, Q, Depth);
     adjustKnownBitsForSelectArm(RHSKnown, I->getOperand(0), I->getOperand(2),
-                                /*Invert=*/true, Depth, Q);
+                                /*Invert=*/true, Q, Depth);
     Known = LHSKnown.intersectWith(RHSKnown);
     break;
   }
@@ -445,8 +446,8 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
 
     APInt InputDemandedMask = DemandedMask.zextOrTrunc(SrcBitWidth);
     KnownBits InputKnown(SrcBitWidth);
-    if (SimplifyDemandedBits(I, 0, InputDemandedMask, InputKnown, Depth + 1,
-                             Q)) {
+    if (SimplifyDemandedBits(I, 0, InputDemandedMask, InputKnown, Q,
+                             Depth + 1)) {
       // For zext nneg, we may have dropped the instruction which made the
       // input non-negative.
       I->dropPoisonGeneratingFlags();
@@ -472,7 +473,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       InputDemandedBits.setBit(SrcBitWidth-1);
 
     KnownBits InputKnown(SrcBitWidth);
-    if (SimplifyDemandedBits(I, 0, InputDemandedBits, InputKnown, Depth + 1, Q))
+    if (SimplifyDemandedBits(I, 0, InputDemandedBits, InputKnown, Q, Depth + 1))
       return I;
 
     // If the input sign bit is known zero, or if the NewBits are not demanded
@@ -533,7 +534,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     unsigned NLZ = DemandedMask.countl_zero();
     APInt DemandedFromOps = APInt::getLowBitsSet(BitWidth, BitWidth - NLZ);
     if (ShrinkDemandedConstant(I, 1, DemandedFromOps) ||
-        SimplifyDemandedBits(I, 1, DemandedFromOps, RHSKnown, Depth + 1, Q))
+        SimplifyDemandedBits(I, 1, DemandedFromOps, RHSKnown, Q, Depth + 1))
       return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
 
     // If low order bits are not demanded and known to be zero in one operand,
@@ -543,7 +544,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     APInt DemandedFromLHS = DemandedFromOps;
     DemandedFromLHS.clearLowBits(NTZ);
     if (ShrinkDemandedConstant(I, 0, DemandedFromLHS) ||
-        SimplifyDemandedBits(I, 0, DemandedFromLHS, LHSKnown, Depth + 1, Q))
+        SimplifyDemandedBits(I, 0, DemandedFromLHS, LHSKnown, Q, Depth + 1))
       return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
 
     // If we are known to be adding zeros to every bit below
@@ -576,7 +577,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     unsigned NLZ = DemandedMask.countl_zero();
     APInt DemandedFromOps = APInt::getLowBitsSet(BitWidth, BitWidth - NLZ);
     if (ShrinkDemandedConstant(I, 1, DemandedFromOps) ||
-        SimplifyDemandedBits(I, 1, DemandedFromOps, RHSKnown, Depth + 1, Q))
+        SimplifyDemandedBits(I, 1, DemandedFromOps, RHSKnown, Q, Depth + 1))
       return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
 
     // If low order bits are not demanded and are known to be zero in RHS,
@@ -586,7 +587,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     APInt DemandedFromLHS = DemandedFromOps;
     DemandedFromLHS.clearLowBits(NTZ);
     if (ShrinkDemandedConstant(I, 0, DemandedFromLHS) ||
-        SimplifyDemandedBits(I, 0, DemandedFromLHS, LHSKnown, Depth + 1, Q))
+        SimplifyDemandedBits(I, 0, DemandedFromLHS, LHSKnown, Q, Depth + 1))
       return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
 
     // If we are known to be subtracting zeros from every bit below
@@ -639,7 +640,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       return InsertNewInstWith(And1, I->getIterator());
     }
 
-    llvm::computeKnownBits(I, Known, Depth, Q);
+    llvm::computeKnownBits(I, Known, Q, Depth);
     break;
   }
   case Instruction::Shl: {
@@ -660,7 +661,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
             auto [IID, FShiftArgs] = *Opt;
             if ((IID == Intrinsic::fshl || IID == Intrinsic::fshr) &&
                 FShiftArgs[0] == FShiftArgs[1]) {
-              llvm::computeKnownBits(I, Known, Depth, Q);
+              llvm::computeKnownBits(I, Known, Q, Depth);
               break;
             }
           }
@@ -674,7 +675,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
         if (I->hasNoSignedWrap()) {
           unsigned NumHiDemandedBits = BitWidth - DemandedMask.countr_zero();
           unsigned SignBits =
-              ComputeNumSignBits(I->getOperand(0), Depth + 1, Q.CxtI);
+              ComputeNumSignBits(I->getOperand(0), Q.CxtI, Depth + 1);
           if (SignBits > ShiftAmt && SignBits - ShiftAmt >= NumHiDemandedBits)
             return I->getOperand(0);
         }
@@ -706,7 +707,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       else if (IOp->hasNoUnsignedWrap())
         DemandedMaskIn.setHighBits(ShiftAmt);
 
-      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Depth + 1, Q))
+      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Q, Depth + 1))
         return I;
 
       Known = KnownBits::shl(Known,
@@ -719,13 +720,13 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       // demanding those bits from the pre-shifted operand either.
       if (unsigned CTLZ = DemandedMask.countl_zero()) {
         APInt DemandedFromOp(APInt::getLowBitsSet(BitWidth, BitWidth - CTLZ));
-        if (SimplifyDemandedBits(I, 0, DemandedFromOp, Known, Depth + 1, Q)) {
+        if (SimplifyDemandedBits(I, 0, DemandedFromOp, Known, Q, Depth + 1)) {
           // We can't guarantee that nsw/nuw hold after simplifying the operand.
           I->dropPoisonGeneratingFlags();
           return I;
         }
       }
-      llvm::computeKnownBits(I, Known, Depth, Q);
+      llvm::computeKnownBits(I, Known, Q, Depth);
     }
     break;
   }
@@ -742,7 +743,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
             auto [IID, FShiftArgs] = *Opt;
             if ((IID == Intrinsic::fshl || IID == Intrinsic::fshr) &&
                 FShiftArgs[0] == FShiftArgs[1]) {
-              llvm::computeKnownBits(I, Known, Depth, Q);
+              llvm::computeKnownBits(I, Known, Q, Depth);
               break;
             }
           }
@@ -756,7 +757,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
         // need to shift.
         unsigned NumHiDemandedBits = BitWidth - DemandedMask.countr_zero();
         unsigned SignBits =
-            ComputeNumSignBits(I->getOperand(0), Depth + 1, Q.CxtI);
+            ComputeNumSignBits(I->getOperand(0), Q.CxtI, Depth + 1);
         if (SignBits >= NumHiDemandedBits)
           return I->getOperand(0);
 
@@ -789,7 +790,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
 
       // Unsigned shift right.
       APInt DemandedMaskIn(DemandedMask.shl(ShiftAmt));
-      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Depth + 1, Q)) {
+      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Q, Depth + 1)) {
         // exact flag may not longer hold.
         I->dropPoisonGeneratingFlags();
         return I;
@@ -799,12 +800,12 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       if (ShiftAmt)
         Known.Zero.setHighBits(ShiftAmt);  // high bits known zero.
     } else {
-      llvm::computeKnownBits(I, Known, Depth, Q);
+      llvm::computeKnownBits(I, Known, Q, Depth);
     }
     break;
   }
   case Instruction::AShr: {
-    unsigned SignBits = ComputeNumSignBits(I->getOperand(0), Depth + 1, Q.CxtI);
+    unsigned SignBits = ComputeNumSignBits(I->getOperand(0), Q.CxtI, Depth + 1);
 
     // If we only want bits that already match the signbit then we don't need
     // to shift.
@@ -834,7 +835,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       bool ShiftedInBitsDemanded = DemandedMask.countl_zero() < ShiftAmt;
       if (ShiftedInBitsDemanded)
         DemandedMaskIn.setSignBit();
-      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Depth + 1, Q)) {
+      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, Known, Q, Depth + 1)) {
         // exact flag may not longer hold.
         I->dropPoisonGeneratingFlags();
         return I;
@@ -854,7 +855,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
           Known, KnownBits::makeConstant(APInt(BitWidth, ShiftAmt)),
           ShiftAmt != 0, I->isExact());
     } else {
-      llvm::computeKnownBits(I, Known, Depth, Q);
+      llvm::computeKnownBits(I, Known, Q, Depth);
     }
     break;
   }
@@ -866,7 +867,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       unsigned RHSTrailingZeros = SA->countr_zero();
       APInt DemandedMaskIn =
           APInt::getHighBitsSet(BitWidth, BitWidth - RHSTrailingZeros);
-      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, LHSKnown, Depth + 1, Q)) {
+      if (SimplifyDemandedBits(I, 0, DemandedMaskIn, LHSKnown, Q, Depth + 1)) {
         // We can't guarantee that "exact" is still true after changing the
         // the dividend.
         I->dropPoisonGeneratingFlags();
@@ -876,7 +877,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
       Known = KnownBits::udiv(LHSKnown, KnownBits::makeConstant(*SA),
                               cast<BinaryOperator>(I)->isExact());
     } else {
-      llvm::computeKnownBits(I, Known, Depth, Q);
+      llvm::computeKnownBits(I, Known, Q, Depth);
     }
     break;
   }
@@ -888,13 +889,13 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
 
       APInt LowBits = *Rem - 1;
       APInt Mask2 = LowBits | APInt::getSignMask(BitWidth);
-      if (SimplifyDemandedBits(I, 0, Mask2, LHSKnown, Depth + 1, Q))
+      if (SimplifyDemandedBits(I, 0, Mask2, LHSKnown, Q, Depth + 1))
         return I;
       Known = KnownBits::srem(LHSKnown, KnownBits::makeConstant(*Rem));
       break;
     }
 
-    llvm::computeKnownBits(I, Known, Depth, Q);
+    llvm::computeKnownBits(I, Known, Q, Depth);
     break;
   }
   case Instruction::Call: {
@@ -950,10 +951,10 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
         unsigned MaskWidth = I->getOperand(1)->getType()->getScalarSizeInBits();
         RHSKnown = KnownBits(MaskWidth);
         // If either the LHS or the RHS are Zero, the result is zero.
-        if (SimplifyDemandedBits(I, 0, DemandedMask, LHSKnown, Depth + 1, Q) ||
+        if (SimplifyDemandedBits(I, 0, DemandedMask, LHSKnown, Q, Depth + 1) ||
             SimplifyDemandedBits(
                 I, 1, (DemandedMask & ~LHSKnown.Zero).zextOrTrunc(MaskWidth),
-                RHSKnown, Depth + 1, Q))
+                RHSKnown, Q, Depth + 1))
           return I;
 
         // TODO: Should be 1-extend
@@ -994,7 +995,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
                          m_PtrAdd(m_Value(InnerPtr), m_ConstantInt(GEPIndex)),
                          m_ConstantInt(PtrMaskImmediate)))) {
 
-          LHSKnown = computeKnownBits(InnerPtr, Depth + 1, I);
+          LHSKnown = computeKnownBits(InnerPtr, I, Depth + 1);
           if (!LHSKnown.isZero()) {
             const unsigned trailingZeros = LHSKnown.countMinTrailingZeros();
             uint64_t PointerAlignBits = (uint64_t(1) << trailingZeros) - 1;
@@ -1039,10 +1040,10 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
         APInt DemandedMaskLHS(DemandedMask.lshr(ShiftAmt));
         APInt DemandedMaskRHS(DemandedMask.shl(BitWidth - ShiftAmt));
         if (I->getOperand(0) != I->getOperand(1)) {
-          if (SimplifyDemandedBits(I, 0, DemandedMaskLHS, LHSKnown, Depth + 1,
-                                   Q) ||
-              SimplifyDemandedBits(I, 1, DemandedMaskRHS, RHSKnown, Depth + 1,
-                                   Q)) {
+          if (SimplifyDemandedBits(I, 0, DemandedMaskLHS, LHSKnown, Q,
+                                   Depth + 1) ||
+              SimplifyDemandedBits(I, 1, DemandedMaskRHS, RHSKnown, Q,
+                                   Depth + 1)) {
             // Range attribute may no longer hold.
             I->dropPoisonGeneratingReturnAttributes();
             return I;
@@ -1050,14 +1051,14 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
         } else { // fshl is a rotate
           // Avoid converting rotate into funnel shift.
           // Only simplify if one operand is constant.
-          LHSKnown = computeKnownBits(I->getOperand(0), Depth + 1, I);
+          LHSKnown = computeKnownBits(I->getOperand(0), I, Depth + 1);
           if (DemandedMaskLHS.isSubsetOf(LHSKnown.Zero | LHSKnown.One) &&
               !match(I->getOperand(0), m_SpecificInt(LHSKnown.One))) {
             replaceOperand(*I, 0, Constant::getIntegerValue(VTy, LHSKnown.One));
             return I;
           }
 
-          RHSKnown = computeKnownBits(I->getOperand(1), Depth + 1, I);
+          RHSKnown = computeKnownBits(I->getOperand(1), I, Depth + 1);
           if (DemandedMaskRHS.isSubsetOf(RHSKnown.Zero | RHSKnown.One) &&
               !match(I->getOperand(1), m_SpecificInt(RHSKnown.One))) {
             replaceOperand(*I, 1, Constant::getIntegerValue(VTy, RHSKnown.One));
@@ -1107,7 +1108,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     }
 
     if (!KnownBitsComputed)
-      llvm::computeKnownBits(I, Known, Depth, Q);
+      llvm::computeKnownBits(I, Known, Q, Depth);
     break;
   }
   }
@@ -1126,7 +1127,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     return Constant::getIntegerValue(VTy, Known.One);
 
   if (VerifyKnownBits) {
-    KnownBits ReferenceKnown = llvm::computeKnownBits(I, Depth, Q);
+    KnownBits ReferenceKnown = llvm::computeKnownBits(I, Q, Depth);
     if (Known != ReferenceKnown) {
       errs() << "Mismatched known bits for " << *I << " in "
              << I->getFunction()->getName() << "\n";
@@ -1143,8 +1144,8 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
 /// bits. It also tries to handle simplifications that can be done based on
 /// DemandedMask, but without modifying the Instruction.
 Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
-    Instruction *I, const APInt &DemandedMask, KnownBits &Known, unsigned Depth,
-    const SimplifyQuery &Q) {
+    Instruction *I, const APInt &DemandedMask, KnownBits &Known,
+    const SimplifyQuery &Q, unsigned Depth) {
   unsigned BitWidth = DemandedMask.getBitWidth();
   Type *ITy = I->getType();
 
@@ -1157,11 +1158,11 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
   // this instruction has a simpler value in that context.
   switch (I->getOpcode()) {
   case Instruction::And: {
-    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Depth + 1, Q);
-    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Depth + 1, Q);
+    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Q, Depth + 1);
+    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Q, Depth + 1);
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
-                                         Depth, Q);
-    computeKnownBitsFromContext(I, Known, Depth, Q);
+                                         Q, Depth);
+    computeKnownBitsFromContext(I, Known, Q, Depth);
 
     // If the client is only demanding bits that we know, return the known
     // constant.
@@ -1178,11 +1179,11 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
     break;
   }
   case Instruction::Or: {
-    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Depth + 1, Q);
-    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Depth + 1, Q);
+    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Q, Depth + 1);
+    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Q, Depth + 1);
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
-                                         Depth, Q);
-    computeKnownBitsFromContext(I, Known, Depth, Q);
+                                         Q, Depth);
+    computeKnownBitsFromContext(I, Known, Q, Depth);
 
     // If the client is only demanding bits that we know, return the known
     // constant.
@@ -1201,11 +1202,11 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
     break;
   }
   case Instruction::Xor: {
-    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Depth + 1, Q);
-    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Depth + 1, Q);
+    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Q, Depth + 1);
+    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Q, Depth + 1);
     Known = analyzeKnownBitsFromAndXorOr(cast<Operator>(I), LHSKnown, RHSKnown,
-                                         Depth, Q);
-    computeKnownBitsFromContext(I, Known, Depth, Q);
+                                         Q, Depth);
+    computeKnownBitsFromContext(I, Known, Q, Depth);
 
     // If the client is only demanding bits that we know, return the known
     // constant.
@@ -1228,18 +1229,18 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
 
     // If an operand adds zeros to every bit below the highest demanded bit,
     // that operand doesn't change the result. Return the other side.
-    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Depth + 1, Q);
+    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Q, Depth + 1);
     if (DemandedFromOps.isSubsetOf(RHSKnown.Zero))
       return I->getOperand(0);
 
-    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Depth + 1, Q);
+    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Q, Depth + 1);
     if (DemandedFromOps.isSubsetOf(LHSKnown.Zero))
       return I->getOperand(1);
 
     bool NSW = cast<OverflowingBinaryOperator>(I)->hasNoSignedWrap();
     bool NUW = cast<OverflowingBinaryOperator>(I)->hasNoUnsignedWrap();
     Known = KnownBits::add(LHSKnown, RHSKnown, NSW, NUW);
-    computeKnownBitsFromContext(I, Known, Depth, Q);
+    computeKnownBitsFromContext(I, Known, Q, Depth);
     break;
   }
   case Instruction::Sub: {
@@ -1248,20 +1249,20 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
 
     // If an operand subtracts zeros from every bit below the highest demanded
     // bit, that operand doesn't change the result. Return the other side.
-    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Depth + 1, Q);
+    llvm::computeKnownBits(I->getOperand(1), RHSKnown, Q, Depth + 1);
     if (DemandedFromOps.isSubsetOf(RHSKnown.Zero))
       return I->getOperand(0);
 
     bool NSW = cast<OverflowingBinaryOperator>(I)->hasNoSignedWrap();
     bool NUW = cast<OverflowingBinaryOperator>(I)->hasNoUnsignedWrap();
-    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Depth + 1, Q);
+    llvm::computeKnownBits(I->getOperand(0), LHSKnown, Q, Depth + 1);
     Known = KnownBits::sub(LHSKnown, RHSKnown, NSW, NUW);
-    computeKnownBitsFromContext(I, Known, Depth, Q);
+    computeKnownBitsFromContext(I, Known, Q, Depth);
     break;
   }
   case Instruction::AShr: {
     // Compute the Known bits to simplify things downstream.
-    llvm::computeKnownBits(I, Known, Depth, Q);
+    llvm::computeKnownBits(I, Known, Q, Depth);
 
     // If this user is only demanding bits that we know, return the known
     // constant.
@@ -1288,7 +1289,7 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
   }
   default:
     // Compute the Known bits to simplify things downstream.
-    llvm::computeKnownBits(I, Known, Depth, Q);
+    llvm::computeKnownBits(I, Known, Q, Depth);
 
     // If this user is only demanding bits that we know, return the known
     // constant.
@@ -1967,8 +1968,8 @@ static Constant *getFPClassConstant(Type *Ty, FPClassTest Mask) {
 Value *InstCombinerImpl::SimplifyDemandedUseFPClass(Value *V,
                                                     FPClassTest DemandedMask,
                                                     KnownFPClass &Known,
-                                                    unsigned Depth,
-                                                    Instruction *CxtI) {
+                                                    Instruction *CxtI,
+                                                    unsigned Depth) {
   assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
   Type *VTy = V->getType();
 
@@ -2078,7 +2079,7 @@ bool InstCombinerImpl::SimplifyDemandedFPClass(Instruction *I, unsigned OpNo,
                                                unsigned Depth) {
   Use &U = I->getOperandUse(OpNo);
   Value *NewVal =
-      SimplifyDemandedUseFPClass(U.get(), DemandedMask, Known, Depth, I);
+      SimplifyDemandedUseFPClass(U.get(), DemandedMask, Known, I, Depth);
   if (!NewVal)
     return false;
   if (Instruction *OpInst = dyn_cast<Instruction>(U))
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 116b86222841..f946c3856948 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2244,7 +2244,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf,
   // have preserved the exact NaN bit-pattern.
   // Avoid the folding if X can have NaN elements.
   if (Shuf.getType()->getElementType()->isFloatingPointTy() &&
-      !isKnownNeverNaN(X, 0, SQ))
+      !isKnownNeverNaN(X, SQ))
     return nullptr;
 
   // Shuffle identity constants into the lanes that return the original value.
diff --git a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 3dc89772676d..4fe900e9421f 100644
--- a/external/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -217,6 +217,26 @@ Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
   return Offset;
 }
 
+Value *InstCombinerImpl::EmitGEPOffsets(ArrayRef<GEPOperator *> GEPs,
+                                        GEPNoWrapFlags NW, Type *IdxTy,
+                                        bool RewriteGEPs) {
+  Value *Sum = nullptr;
+  for (GEPOperator *GEP : reverse(GEPs)) {
+    Value *Offset = EmitGEPOffset(GEP, RewriteGEPs);
+    if (Offset->getType() != IdxTy)
+      Offset = Builder.CreateVectorSplat(
+          cast<VectorType>(IdxTy)->getElementCount(), Offset);
+    if (Sum)
+      Sum = Builder.CreateAdd(Sum, Offset, "", NW.hasNoUnsignedWrap(),
+                              NW.isInBounds());
+    else
+      Sum = Offset;
+  }
+  if (!Sum)
+    return Constant::getNullValue(IdxTy);
+  return Sum;
+}
+
 /// Legal integers and common types are considered desirable. This is used to
 /// avoid creating instructions with types that may not be supported well by the
 /// the backend.
@@ -1719,6 +1739,15 @@ Instruction *InstCombinerImpl::FoldOpIntoSelect(Instruction &Op, SelectInst *SI,
   if (SI->getType()->isIntOrIntVectorTy(1))
     return nullptr;
 
+  // Avoid breaking min/max reduction pattern,
+  // which is necessary for vectorization later.
+  if (isa<MinMaxIntrinsic>(&Op))
+    for (Value *IntrinOp : Op.operands())
+      if (auto *PN = dyn_cast<PHINode>(IntrinOp))
+        for (Value *PhiOp : PN->operands())
+          if (PhiOp == &Op)
+            return nullptr;
+
   // Test if a FCmpInst instruction is used exclusively by a select as
   // part of a minimum or maximum operation. If so, refrain from doing
   // any other folding. This helps out other analyses which understand
@@ -1729,7 +1758,8 @@ Instruction *InstCombinerImpl::FoldOpIntoSelect(Instruction &Op, SelectInst *SI,
   if (auto *CI = dyn_cast<FCmpInst>(SI->getCondition())) {
     if (CI->hasOneUse()) {
       Value *Op0 = CI->getOperand(0), *Op1 = CI->getOperand(1);
-      if ((TV == Op0 && FV == Op1) || (FV == Op0 && TV == Op1))
+      if (((TV == Op0 && FV == Op1) || (FV == Op0 && TV == Op1)) &&
+          !CI->isCommutative())
         return nullptr;
     }
   }
@@ -3620,7 +3650,7 @@ Instruction *InstCombinerImpl::visitReturnInst(ReturnInst &RI) {
 
   KnownFPClass KnownClass;
   Value *Simplified =
-      SimplifyDemandedUseFPClass(RetVal, ~ReturnClass, KnownClass, 0, &RI);
+      SimplifyDemandedUseFPClass(RetVal, ~ReturnClass, KnownClass, &RI);
   if (!Simplified)
     return nullptr;
 
@@ -3957,7 +3987,7 @@ Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) {
       return replaceOperand(SI, 0, V);
   }
 
-  KnownBits Known = computeKnownBits(Cond, 0, &SI);
+  KnownBits Known = computeKnownBits(Cond, &SI);
   unsigned LeadingKnownZeros = Known.countMinLeadingZeros();
   unsigned LeadingKnownOnes = Known.countMinLeadingOnes();
 
@@ -5339,8 +5369,7 @@ bool InstCombinerImpl::run() {
         // We copy the old instruction's DebugLoc to the new instruction, unless
         // InstCombine already assigned a DebugLoc to it, in which case we
         // should trust the more specifically selected DebugLoc.
-        if (!Result->getDebugLoc())
-          Result->setDebugLoc(I->getDebugLoc());
+        Result->setDebugLoc(Result->getDebugLoc().orElse(I->getDebugLoc()));
         // We also copy annotation metadata to the new instruction.
         Result->copyMetadata(*I, LLVMContext::MD_annotation);
         // Everything uses the new instruction now.
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/CMakeLists.txt b/external/llvm-project/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
index 0b863f6fef46..15fd421a41b0 100644
--- a/external/llvm-project/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/CMakeLists.txt
@@ -6,7 +6,8 @@ add_llvm_component_library(LLVMInstrumentation
   DataFlowSanitizer.cpp
   GCOVProfiling.cpp
   BlockCoverageInference.cpp
-  MemProfiler.cpp
+  MemProfInstrumentation.cpp
+  MemProfUse.cpp
   MemorySanitizer.cpp
   NumericalStabilitySanitizer.cpp
   IndirectCallPromotion.cpp
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/external/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index d94a2fbb23d2..61fef1387d82 100644
--- a/external/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1975,12 +1975,10 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2,
   auto V1Elems = ShadowElements.find(V1);
   auto V2Elems = ShadowElements.find(V2);
   if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) {
-    if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
-                      V2Elems->second.begin(), V2Elems->second.end())) {
+    if (llvm::includes(V1Elems->second, V2Elems->second)) {
       return collapseToPrimitiveShadow(V1, Pos);
     }
-    if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
-                      V1Elems->second.begin(), V1Elems->second.end())) {
+    if (llvm::includes(V2Elems->second, V1Elems->second)) {
       return collapseToPrimitiveShadow(V2, Pos);
     }
   } else if (V1Elems != ShadowElements.end()) {
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/external/llvm-project/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index e81a725c62ea..77db686f8229 100644
--- a/external/llvm-project/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -419,7 +419,8 @@ class HWAddressSanitizer {
     }
 
   public:
-    void init(Triple &TargetTriple, bool InstrumentWithCalls);
+    void init(Triple &TargetTriple, bool InstrumentWithCalls,
+              bool CompileKernel);
     Align getObjectAlignment() const { return Align(1ULL << Scale); }
     bool isInGlobal() const { return Kind == OffsetKind::kGlobal; }
     bool isInIfunc() const { return Kind == OffsetKind::kIfunc; }
@@ -642,7 +643,7 @@ void HWAddressSanitizer::initializeModule() {
   PointerTagShift = IsX86_64 ? 57 : 56;
   TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
 
-  Mapping.init(TargetTriple, InstrumentWithCalls);
+  Mapping.init(TargetTriple, InstrumentWithCalls, CompileKernel);
 
   C = &(M.getContext());
   IRBuilder<> IRB(*C);
@@ -1874,7 +1875,8 @@ void HWAddressSanitizer::instrumentPersonalityFunctions() {
 }
 
 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
-                                             bool InstrumentWithCalls) {
+                                             bool InstrumentWithCalls,
+                                             bool CompileKernel) {
   // Start with defaults.
   Scale = kDefaultShadowScale;
   Kind = OffsetKind::kTls;
@@ -1885,7 +1887,7 @@ void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
     // Fuchsia is always PIE, which means that the beginning of the address
     // space is always available.
     SetFixed(0);
-  } else if (ClEnableKhwasan || InstrumentWithCalls) {
+  } else if (CompileKernel || InstrumentWithCalls) {
     SetFixed(0);
     WithFrameRecord = false;
   }
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/external/llvm-project/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index a99f8607418c..b951b09865a9 100644
--- a/external/llvm-project/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -48,6 +48,7 @@
 #include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/Triple.h"
@@ -77,7 +78,7 @@ cl::opt<bool> DebugInfoCorrelate(
              "-profile-correlate=debug-info)"),
     cl::init(false));
 
-cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
+LLVM_ABI cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
     "profile-correlate",
     cl::desc("Use debug info or binary file to correlate profiles."),
     cl::init(InstrProfCorrelator::NONE),
@@ -1955,12 +1956,6 @@ void InstrLowerer::emitNameData() {
                                 GlobalValue::PrivateLinkage, NamesVal,
                                 getInstrProfNamesVarName());
 
-  // Make names variable public if current target is a GPU
-  if (isGPUProfTarget(M)) {
-    NamesVar->setLinkage(GlobalValue::ExternalLinkage);
-    NamesVar->setVisibility(GlobalValue::VisibilityTypes::ProtectedVisibility);
-  }
-
   NamesSize = CompressedNameStr.size();
   setGlobalVariableLargeSection(TT, *NamesVar);
   NamesVar->setSection(
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp b/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
new file mode 100644
index 000000000000..3ae771a575f0
--- /dev/null
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
@@ -0,0 +1,656 @@
+//===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProf. Memory accesses are instrumented
+// to increment the access count held in a shadow memory location, or
+// alternatively to call into the runtime. Memory intrinsic calls (memmove,
+// memcpy, memset) are changed to call the memory profiling runtime version
+// instead.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/MemProf.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+using namespace llvm::memprof;
+
+#define DEBUG_TYPE "memprof"
+
+constexpr int LLVM_MEM_PROFILER_VERSION = 1;
+
+// Size of memory mapped to a single shadow location.
+constexpr uint64_t DefaultMemGranularity = 64;
+
+// Size of memory mapped to a single histogram bucket.
+constexpr uint64_t HistogramGranularity = 8;
+
+// Scale from granularity down to shadow size.
+constexpr uint64_t DefaultShadowScale = 3;
+
+constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
+constexpr uint64_t MemProfCtorAndDtorPriority = 1;
+// On Emscripten, the system needs more than one priorities for constructors.
+constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
+constexpr char MemProfInitName[] = "__memprof_init";
+constexpr char MemProfVersionCheckNamePrefix[] =
+    "__memprof_version_mismatch_check_v";
+
+constexpr char MemProfShadowMemoryDynamicAddress[] =
+    "__memprof_shadow_memory_dynamic_address";
+
+constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
+
+constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
+
+// Command-line flags.
+
+static cl::opt<bool> ClInsertVersionCheck(
+    "memprof-guard-against-version-mismatch",
+    cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
+    cl::init(true));
+
+// This flag may need to be replaced with -f[no-]memprof-reads.
+static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
+                                       cl::desc("instrument read instructions"),
+                                       cl::Hidden, cl::init(true));
+
+static cl::opt<bool>
+    ClInstrumentWrites("memprof-instrument-writes",
+                       cl::desc("instrument write instructions"), cl::Hidden,
+                       cl::init(true));
+
+static cl::opt<bool> ClInstrumentAtomics(
+    "memprof-instrument-atomics",
+    cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
+    cl::init(true));
+
+static cl::opt<bool> ClUseCalls(
+    "memprof-use-callbacks",
+    cl::desc("Use callbacks instead of inline instrumentation sequences."),
+    cl::Hidden, cl::init(false));
+
+static cl::opt<std::string>
+    ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
+                                 cl::desc("Prefix for memory access callbacks"),
+                                 cl::Hidden, cl::init("__memprof_"));
+
+// These flags allow to change the shadow mapping.
+// The shadow mapping looks like
+//    Shadow = ((Mem & mask) >> scale) + offset
+
+static cl::opt<int> ClMappingScale("memprof-mapping-scale",
+                                   cl::desc("scale of memprof shadow mapping"),
+                                   cl::Hidden, cl::init(DefaultShadowScale));
+
+static cl::opt<int>
+    ClMappingGranularity("memprof-mapping-granularity",
+                         cl::desc("granularity of memprof shadow mapping"),
+                         cl::Hidden, cl::init(DefaultMemGranularity));
+
+static cl::opt<bool> ClStack("memprof-instrument-stack",
+                             cl::desc("Instrument scalar stack variables"),
+                             cl::Hidden, cl::init(false));
+
+// Debug flags.
+
+static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
+                            cl::init(0));
+
+static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
+                                        cl::desc("Debug func"));
+
+static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
+                               cl::Hidden, cl::init(-1));
+
+static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
+                               cl::Hidden, cl::init(-1));
+
+static cl::opt<bool> ClHistogram("memprof-histogram",
+                                 cl::desc("Collect access count histograms"),
+                                 cl::Hidden, cl::init(false));
+
+static cl::opt<std::string>
+    MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
+                                 cl::desc("The default memprof options"),
+                                 cl::Hidden, cl::init(""));
+
+// Instrumentation statistics
+STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
+STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
+STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
+STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
+
+namespace {
+
+/// This struct defines the shadow mapping using the rule:
+///   shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
+struct ShadowMapping {
+  ShadowMapping() {
+    Scale = ClMappingScale;
+    Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
+    Mask = ~(Granularity - 1);
+  }
+
+  int Scale;
+  int Granularity;
+  uint64_t Mask; // Computed as ~(Granularity-1)
+};
+
+static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
+  return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
+                                       : MemProfCtorAndDtorPriority;
+}
+
+struct InterestingMemoryAccess {
+  Value *Addr = nullptr;
+  bool IsWrite;
+  Type *AccessTy;
+  Value *MaybeMask = nullptr;
+};
+
+/// Instrument the code in module to profile memory accesses.
+class MemProfiler {
+public:
+  MemProfiler(Module &M) {
+    C = &(M.getContext());
+    LongSize = M.getDataLayout().getPointerSizeInBits();
+    IntptrTy = Type::getIntNTy(*C, LongSize);
+    PtrTy = PointerType::getUnqual(*C);
+  }
+
+  /// If it is an interesting memory access, populate information
+  /// about the access and return a InterestingMemoryAccess struct.
+  /// Otherwise return std::nullopt.
+  std::optional<InterestingMemoryAccess>
+  isInterestingMemoryAccess(Instruction *I) const;
+
+  void instrumentMop(Instruction *I, const DataLayout &DL,
+                     InterestingMemoryAccess &Access);
+  void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
+                         Value *Addr, bool IsWrite);
+  void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
+                                   Instruction *I, Value *Addr, Type *AccessTy,
+                                   bool IsWrite);
+  void instrumentMemIntrinsic(MemIntrinsic *MI);
+  Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
+  bool instrumentFunction(Function &F);
+  bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
+  bool insertDynamicShadowAtFunctionEntry(Function &F);
+
+private:
+  void initializeCallbacks(Module &M);
+
+  LLVMContext *C;
+  int LongSize;
+  Type *IntptrTy;
+  PointerType *PtrTy;
+  ShadowMapping Mapping;
+
+  // These arrays is indexed by AccessIsWrite
+  FunctionCallee MemProfMemoryAccessCallback[2];
+
+  FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
+  Value *DynamicShadowOffset = nullptr;
+};
+
+class ModuleMemProfiler {
+public:
+  ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
+
+  bool instrumentModule(Module &);
+
+private:
+  Triple TargetTriple;
+  ShadowMapping Mapping;
+  Function *MemProfCtorFunction = nullptr;
+};
+
+} // end anonymous namespace
+
+MemProfilerPass::MemProfilerPass() = default;
+
+PreservedAnalyses MemProfilerPass::run(Function &F,
+                                       AnalysisManager<Function> &AM) {
+  assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
+         "Memprof with histogram only supports default mapping granularity");
+  Module &M = *F.getParent();
+  MemProfiler Profiler(M);
+  if (Profiler.instrumentFunction(F))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
+
+ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
+
+PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
+                                             AnalysisManager<Module> &AM) {
+
+  ModuleMemProfiler Profiler(M);
+  if (Profiler.instrumentModule(M))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
+
+Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
+  // (Shadow & mask) >> scale
+  Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
+  Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
+  // (Shadow >> scale) | offset
+  assert(DynamicShadowOffset);
+  return IRB.CreateAdd(Shadow, DynamicShadowOffset);
+}
+
+// Instrument memset/memmove/memcpy
+void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
+  IRBuilder<> IRB(MI);
+  if (isa<MemTransferInst>(MI)) {
+    IRB.CreateCall(isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
+                   {MI->getOperand(0), MI->getOperand(1),
+                    IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+  } else if (isa<MemSetInst>(MI)) {
+    IRB.CreateCall(
+        MemProfMemset,
+        {MI->getOperand(0),
+         IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
+         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+  }
+  MI->eraseFromParent();
+}
+
+std::optional<InterestingMemoryAccess>
+MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
+  // Do not instrument the load fetching the dynamic shadow address.
+  if (DynamicShadowOffset == I)
+    return std::nullopt;
+
+  InterestingMemoryAccess Access;
+
+  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
+    if (!ClInstrumentReads)
+      return std::nullopt;
+    Access.IsWrite = false;
+    Access.AccessTy = LI->getType();
+    Access.Addr = LI->getPointerOperand();
+  } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
+    if (!ClInstrumentWrites)
+      return std::nullopt;
+    Access.IsWrite = true;
+    Access.AccessTy = SI->getValueOperand()->getType();
+    Access.Addr = SI->getPointerOperand();
+  } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
+    if (!ClInstrumentAtomics)
+      return std::nullopt;
+    Access.IsWrite = true;
+    Access.AccessTy = RMW->getValOperand()->getType();
+    Access.Addr = RMW->getPointerOperand();
+  } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
+    if (!ClInstrumentAtomics)
+      return std::nullopt;
+    Access.IsWrite = true;
+    Access.AccessTy = XCHG->getCompareOperand()->getType();
+    Access.Addr = XCHG->getPointerOperand();
+  } else if (auto *CI = dyn_cast<CallInst>(I)) {
+    auto *F = CI->getCalledFunction();
+    if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
+              F->getIntrinsicID() == Intrinsic::masked_store)) {
+      unsigned OpOffset = 0;
+      if (F->getIntrinsicID() == Intrinsic::masked_store) {
+        if (!ClInstrumentWrites)
+          return std::nullopt;
+        // Masked store has an initial operand for the value.
+        OpOffset = 1;
+        Access.AccessTy = CI->getArgOperand(0)->getType();
+        Access.IsWrite = true;
+      } else {
+        if (!ClInstrumentReads)
+          return std::nullopt;
+        Access.AccessTy = CI->getType();
+        Access.IsWrite = false;
+      }
+
+      auto *BasePtr = CI->getOperand(0 + OpOffset);
+      Access.MaybeMask = CI->getOperand(2 + OpOffset);
+      Access.Addr = BasePtr;
+    }
+  }
+
+  if (!Access.Addr)
+    return std::nullopt;
+
+  // Do not instrument accesses from different address spaces; we cannot deal
+  // with them.
+  Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
+  if (PtrTy->getPointerAddressSpace() != 0)
+    return std::nullopt;
+
+  // Ignore swifterror addresses.
+  // swifterror memory addresses are mem2reg promoted by instruction
+  // selection. As such they cannot have regular uses like an instrumentation
+  // function and it makes no sense to track them as memory.
+  if (Access.Addr->isSwiftError())
+    return std::nullopt;
+
+  // Peel off GEPs and BitCasts.
+  auto *Addr = Access.Addr->stripInBoundsOffsets();
+
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
+    // Do not instrument PGO counter updates.
+    if (GV->hasSection()) {
+      StringRef SectionName = GV->getSection();
+      // Check if the global is in the PGO counters section.
+      auto OF = I->getModule()->getTargetTriple().getObjectFormat();
+      if (SectionName.ends_with(
+              getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
+        return std::nullopt;
+    }
+
+    // Do not instrument accesses to LLVM internal variables.
+    if (GV->getName().starts_with("__llvm"))
+      return std::nullopt;
+  }
+
+  return Access;
+}
+
+void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
+                                              Instruction *I, Value *Addr,
+                                              Type *AccessTy, bool IsWrite) {
+  auto *VTy = cast<FixedVectorType>(AccessTy);
+  unsigned Num = VTy->getNumElements();
+  auto *Zero = ConstantInt::get(IntptrTy, 0);
+  for (unsigned Idx = 0; Idx < Num; ++Idx) {
+    Value *InstrumentedAddress = nullptr;
+    Instruction *InsertBefore = I;
+    if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
+      // dyn_cast as we might get UndefValue
+      if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
+        if (Masked->isZero())
+          // Mask is constant false, so no instrumentation needed.
+          continue;
+        // If we have a true or undef value, fall through to instrumentAddress.
+        // with InsertBefore == I
+      }
+    } else {
+      IRBuilder<> IRB(I);
+      Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
+      Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
+      InsertBefore = ThenTerm;
+    }
+
+    IRBuilder<> IRB(InsertBefore);
+    InstrumentedAddress =
+        IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
+    instrumentAddress(I, InsertBefore, InstrumentedAddress, IsWrite);
+  }
+}
+
+void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
+                                InterestingMemoryAccess &Access) {
+  // Skip instrumentation of stack accesses unless requested.
+  if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
+    if (Access.IsWrite)
+      ++NumSkippedStackWrites;
+    else
+      ++NumSkippedStackReads;
+    return;
+  }
+
+  if (Access.IsWrite)
+    NumInstrumentedWrites++;
+  else
+    NumInstrumentedReads++;
+
+  if (Access.MaybeMask) {
+    instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
+                                Access.AccessTy, Access.IsWrite);
+  } else {
+    // Since the access counts will be accumulated across the entire allocation,
+    // we only update the shadow access count for the first location and thus
+    // don't need to worry about alignment and type size.
+    instrumentAddress(I, I, Access.Addr, Access.IsWrite);
+  }
+}
+
+void MemProfiler::instrumentAddress(Instruction *OrigIns,
+                                    Instruction *InsertBefore, Value *Addr,
+                                    bool IsWrite) {
+  IRBuilder<> IRB(InsertBefore);
+  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
+
+  if (ClUseCalls) {
+    IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
+    return;
+  }
+
+  Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);
+  Type *ShadowPtrTy = PointerType::get(*C, 0);
+
+  Value *ShadowPtr = memToShadow(AddrLong, IRB);
+  Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
+  Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
+  // If we are profiling with histograms, add overflow protection at 255.
+  if (ClHistogram) {
+    Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
+    Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
+    Instruction *IncBlock =
+        SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
+    IRB.SetInsertPoint(IncBlock);
+  }
+  Value *Inc = ConstantInt::get(ShadowTy, 1);
+  ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
+  IRB.CreateStore(ShadowValue, ShadowAddr);
+}
+
+// Create the variable for the profile file name.
+void createProfileFileNameVar(Module &M) {
+  const MDString *MemProfFilename =
+      dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
+  if (!MemProfFilename)
+    return;
+  assert(!MemProfFilename->getString().empty() &&
+         "Unexpected MemProfProfileFilename metadata with empty string");
+  Constant *ProfileNameConst = ConstantDataArray::getString(
+      M.getContext(), MemProfFilename->getString(), true);
+  GlobalVariable *ProfileNameVar = new GlobalVariable(
+      M, ProfileNameConst->getType(), /*isConstant=*/true,
+      GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
+  const Triple &TT = M.getTargetTriple();
+  if (TT.supportsCOMDAT()) {
+    ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
+    ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
+  }
+}
+
+// Set MemprofHistogramFlag as a Global veriable in IR. This makes it accessible
+// to the runtime, changing shadow count behavior.
+void createMemprofHistogramFlagVar(Module &M) {
+  const StringRef VarName(MemProfHistogramFlagVar);
+  Type *IntTy1 = Type::getInt1Ty(M.getContext());
+  auto MemprofHistogramFlag = new GlobalVariable(
+      M, IntTy1, true, GlobalValue::WeakAnyLinkage,
+      Constant::getIntegerValue(IntTy1, APInt(1, ClHistogram)), VarName);
+  const Triple &TT = M.getTargetTriple();
+  if (TT.supportsCOMDAT()) {
+    MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
+    MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));
+  }
+  appendToCompilerUsed(M, MemprofHistogramFlag);
+}
+
+void createMemprofDefaultOptionsVar(Module &M) {
+  Constant *OptionsConst = ConstantDataArray::getString(
+      M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);
+  GlobalVariable *OptionsVar =
+      new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
+                         GlobalValue::WeakAnyLinkage, OptionsConst,
+                         memprof::getMemprofOptionsSymbolName());
+  const Triple &TT = M.getTargetTriple();
+  if (TT.supportsCOMDAT()) {
+    OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
+    OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));
+  }
+}
+
+bool ModuleMemProfiler::instrumentModule(Module &M) {
+
+  // Create a module constructor.
+  std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
+  std::string VersionCheckName =
+      ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
+                           : "";
+  std::tie(MemProfCtorFunction, std::ignore) =
+      createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
+                                          MemProfInitName, /*InitArgTypes=*/{},
+                                          /*InitArgs=*/{}, VersionCheckName);
+
+  const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
+  appendToGlobalCtors(M, MemProfCtorFunction, Priority);
+
+  createProfileFileNameVar(M);
+
+  createMemprofHistogramFlagVar(M);
+
+  createMemprofDefaultOptionsVar(M);
+
+  return true;
+}
+
+void MemProfiler::initializeCallbacks(Module &M) {
+  IRBuilder<> IRB(*C);
+
+  for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
+    const std::string TypeStr = AccessIsWrite ? "store" : "load";
+    const std::string HistPrefix = ClHistogram ? "hist_" : "";
+
+    SmallVector<Type *, 2> Args1{1, IntptrTy};
+    MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
+        ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
+        FunctionType::get(IRB.getVoidTy(), Args1, false));
+  }
+  MemProfMemmove = M.getOrInsertFunction(
+      ClMemoryAccessCallbackPrefix + "memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
+  MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
+                                        PtrTy, PtrTy, PtrTy, IntptrTy);
+  MemProfMemset =
+      M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset", PtrTy,
+                            PtrTy, IRB.getInt32Ty(), IntptrTy);
+}
+
+bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
+  // For each NSObject descendant having a +load method, this method is invoked
+  // by the ObjC runtime before any of the static constructors is called.
+  // Therefore we need to instrument such methods with a call to __memprof_init
+  // at the beginning in order to initialize our runtime before any access to
+  // the shadow memory.
+  // We cannot just ignore these methods, because they may call other
+  // instrumented functions.
+  if (F.getName().contains(" load]")) {
+    FunctionCallee MemProfInitFunction =
+        declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
+    IRBuilder<> IRB(&F.front(), F.front().begin());
+    IRB.CreateCall(MemProfInitFunction, {});
+    return true;
+  }
+  return false;
+}
+
+bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
+  IRBuilder<> IRB(&F.front().front());
+  Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
+      MemProfShadowMemoryDynamicAddress, IntptrTy);
+  if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
+    cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
+  DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
+  return true;
+}
+
+bool MemProfiler::instrumentFunction(Function &F) {
+  if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
+    return false;
+  if (ClDebugFunc == F.getName())
+    return false;
+  if (F.getName().starts_with("__memprof_"))
+    return false;
+
+  bool FunctionModified = false;
+
+  // If needed, insert __memprof_init.
+  // This function needs to be called even if the function body is not
+  // instrumented.
+  if (maybeInsertMemProfInitAtFunctionEntry(F))
+    FunctionModified = true;
+
+  LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
+
+  initializeCallbacks(*F.getParent());
+
+  SmallVector<Instruction *, 16> ToInstrument;
+
+  // Fill the set of memory operations to instrument.
+  for (auto &BB : F) {
+    for (auto &Inst : BB) {
+      if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
+        ToInstrument.push_back(&Inst);
+    }
+  }
+
+  if (ToInstrument.empty()) {
+    LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
+                      << " " << F << "\n");
+
+    return FunctionModified;
+  }
+
+  FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
+
+  int NumInstrumented = 0;
+  for (auto *Inst : ToInstrument) {
+    if (ClDebugMin < 0 || ClDebugMax < 0 ||
+        (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
+      std::optional<InterestingMemoryAccess> Access =
+          isInterestingMemoryAccess(Inst);
+      if (Access)
+        instrumentMop(Inst, F.getDataLayout(), *Access);
+      else
+        instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
+    }
+    NumInstrumented++;
+  }
+
+  if (NumInstrumented > 0)
+    FunctionModified = true;
+
+  LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
+                    << F << "\n");
+
+  return FunctionModified;
+}
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
similarity index 53%
rename from external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
rename to external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
index a64dfc02f6bf..e5b357fc1bfb 100644
--- a/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
@@ -1,4 +1,4 @@
-//===- MemProfiler.cpp - memory allocation and access profiler ------------===//
+//===- MemProfUse.cpp - memory allocation profile use pass --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,34 +6,22 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file is a part of MemProfiler. Memory accesses are instrumented
-// to increment the access count held in a shadow memory location, or
-// alternatively to call into the runtime. Memory intrinsic calls (memmove,
-// memcpy, memset) are changed to call the memory profiling runtime version
-// instead.
+// This file implements the MemProfUsePass which reads memory profiling data
+// and uses it to add metadata to instructions to guide optimization.
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemoryProfileInfo.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Constant.h"
-#include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
-#include "llvm/IR/Type.h"
-#include "llvm/IR/Value.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/ProfileData/MemProfCommon.h"
@@ -42,10 +30,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/HashBuilder.h"
 #include "llvm/Support/VirtualFileSystem.h"
-#include "llvm/TargetParser/Triple.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/LongestCommonSequence.h"
-#include "llvm/Transforms/Utils/ModuleUtils.h"
 #include <map>
 #include <set>
 
@@ -60,95 +45,6 @@ extern cl::opt<bool> NoPGOWarnMismatch;
 extern cl::opt<bool> NoPGOWarnMismatchComdatWeak;
 } // namespace llvm
 
-constexpr int LLVM_MEM_PROFILER_VERSION = 1;
-
-// Size of memory mapped to a single shadow location.
-constexpr uint64_t DefaultMemGranularity = 64;
-
-// Size of memory mapped to a single histogram bucket.
-constexpr uint64_t HistogramGranularity = 8;
-
-// Scale from granularity down to shadow size.
-constexpr uint64_t DefaultShadowScale = 3;
-
-constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
-constexpr uint64_t MemProfCtorAndDtorPriority = 1;
-// On Emscripten, the system needs more than one priorities for constructors.
-constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
-constexpr char MemProfInitName[] = "__memprof_init";
-constexpr char MemProfVersionCheckNamePrefix[] =
-    "__memprof_version_mismatch_check_v";
-
-constexpr char MemProfShadowMemoryDynamicAddress[] =
-    "__memprof_shadow_memory_dynamic_address";
-
-constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
-
-constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
-
-// Command-line flags.
-
-static cl::opt<bool> ClInsertVersionCheck(
-    "memprof-guard-against-version-mismatch",
-    cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
-    cl::init(true));
-
-// This flag may need to be replaced with -f[no-]memprof-reads.
-static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
-                                       cl::desc("instrument read instructions"),
-                                       cl::Hidden, cl::init(true));
-
-static cl::opt<bool>
-    ClInstrumentWrites("memprof-instrument-writes",
-                       cl::desc("instrument write instructions"), cl::Hidden,
-                       cl::init(true));
-
-static cl::opt<bool> ClInstrumentAtomics(
-    "memprof-instrument-atomics",
-    cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
-    cl::init(true));
-
-static cl::opt<bool> ClUseCalls(
-    "memprof-use-callbacks",
-    cl::desc("Use callbacks instead of inline instrumentation sequences."),
-    cl::Hidden, cl::init(false));
-
-static cl::opt<std::string>
-    ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
-                                 cl::desc("Prefix for memory access callbacks"),
-                                 cl::Hidden, cl::init("__memprof_"));
-
-// These flags allow to change the shadow mapping.
-// The shadow mapping looks like
-//    Shadow = ((Mem & mask) >> scale) + offset
-
-static cl::opt<int> ClMappingScale("memprof-mapping-scale",
-                                   cl::desc("scale of memprof shadow mapping"),
-                                   cl::Hidden, cl::init(DefaultShadowScale));
-
-static cl::opt<int>
-    ClMappingGranularity("memprof-mapping-granularity",
-                         cl::desc("granularity of memprof shadow mapping"),
-                         cl::Hidden, cl::init(DefaultMemGranularity));
-
-static cl::opt<bool> ClStack("memprof-instrument-stack",
-                             cl::desc("Instrument scalar stack variables"),
-                             cl::Hidden, cl::init(false));
-
-// Debug flags.
-
-static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
-                            cl::init(0));
-
-static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
-                                        cl::desc("Debug func"));
-
-static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
-                               cl::Hidden, cl::init(-1));
-
-static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
-                               cl::Hidden, cl::init(-1));
-
 // By default disable matching of allocation profiles onto operator new that
 // already explicitly pass a hot/cold hint, since we don't currently
 // override these hints anyway.
@@ -158,21 +54,12 @@ static cl::opt<bool> ClMemProfMatchHotColdNew(
         "Match allocation profiles onto existing hot/cold operator new calls"),
     cl::Hidden, cl::init(false));
 
-static cl::opt<bool> ClHistogram("memprof-histogram",
-                                 cl::desc("Collect access count histograms"),
-                                 cl::Hidden, cl::init(false));
-
 static cl::opt<bool>
     ClPrintMemProfMatchInfo("memprof-print-match-info",
                             cl::desc("Print matching stats for each allocation "
                                      "context in this module's profiles"),
                             cl::Hidden, cl::init(false));
 
-static cl::opt<std::string>
-    MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
-                                 cl::desc("The default memprof options"),
-                                 cl::Hidden, cl::init(""));
-
 static cl::opt<bool>
     SalvageStaleProfile("memprof-salvage-stale-profile",
                         cl::desc("Salvage stale MemProf profile"),
@@ -184,20 +71,10 @@ static cl::opt<bool> ClMemProfAttachCalleeGuids(
         "Attach calleeguids as value profile metadata for indirect calls."),
     cl::init(true), cl::Hidden);
 
-extern cl::opt<bool> MemProfReportHintedSizes;
-extern cl::opt<unsigned> MinClonedColdBytePercent;
-extern cl::opt<unsigned> MinCallsiteColdBytePercent;
-
 static cl::opt<unsigned> MinMatchedColdBytePercent(
     "memprof-matching-cold-threshold", cl::init(100), cl::Hidden,
     cl::desc("Min percent of cold bytes matched to hint allocation cold"));
 
-// Instrumentation statistics
-STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
-STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
-STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
-STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
-
 // Matching statistics
 STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");
 STATISTIC(NumOfMemProfMismatch,
@@ -214,518 +91,6 @@ STATISTIC(NumOfMemProfMatchedAllocs,
 STATISTIC(NumOfMemProfMatchedCallSites,
           "Number of matched memory profile callsites.");
 
-namespace {
-
-/// This struct defines the shadow mapping using the rule:
-///   shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
-struct ShadowMapping {
-  ShadowMapping() {
-    Scale = ClMappingScale;
-    Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
-    Mask = ~(Granularity - 1);
-  }
-
-  int Scale;
-  int Granularity;
-  uint64_t Mask; // Computed as ~(Granularity-1)
-};
-
-static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
-  return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
-                                       : MemProfCtorAndDtorPriority;
-}
-
-struct InterestingMemoryAccess {
-  Value *Addr = nullptr;
-  bool IsWrite;
-  Type *AccessTy;
-  Value *MaybeMask = nullptr;
-};
-
-/// Instrument the code in module to profile memory accesses.
-class MemProfiler {
-public:
-  MemProfiler(Module &M) {
-    C = &(M.getContext());
-    LongSize = M.getDataLayout().getPointerSizeInBits();
-    IntptrTy = Type::getIntNTy(*C, LongSize);
-    PtrTy = PointerType::getUnqual(*C);
-  }
-
-  /// If it is an interesting memory access, populate information
-  /// about the access and return a InterestingMemoryAccess struct.
-  /// Otherwise return std::nullopt.
-  std::optional<InterestingMemoryAccess>
-  isInterestingMemoryAccess(Instruction *I) const;
-
-  void instrumentMop(Instruction *I, const DataLayout &DL,
-                     InterestingMemoryAccess &Access);
-  void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
-                         Value *Addr, bool IsWrite);
-  void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
-                                   Instruction *I, Value *Addr, Type *AccessTy,
-                                   bool IsWrite);
-  void instrumentMemIntrinsic(MemIntrinsic *MI);
-  Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
-  bool instrumentFunction(Function &F);
-  bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
-  bool insertDynamicShadowAtFunctionEntry(Function &F);
-
-private:
-  void initializeCallbacks(Module &M);
-
-  LLVMContext *C;
-  int LongSize;
-  Type *IntptrTy;
-  PointerType *PtrTy;
-  ShadowMapping Mapping;
-
-  // These arrays is indexed by AccessIsWrite
-  FunctionCallee MemProfMemoryAccessCallback[2];
-
-  FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
-  Value *DynamicShadowOffset = nullptr;
-};
-
-class ModuleMemProfiler {
-public:
-  ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
-
-  bool instrumentModule(Module &);
-
-private:
-  Triple TargetTriple;
-  ShadowMapping Mapping;
-  Function *MemProfCtorFunction = nullptr;
-};
-
-// Options under which we need to record the context size info in the alloc trie
-// used to build metadata.
-bool recordContextSizeInfo() {
-  return MemProfReportHintedSizes || MinClonedColdBytePercent < 100 ||
-         MinCallsiteColdBytePercent < 100;
-}
-
-} // end anonymous namespace
-
-MemProfilerPass::MemProfilerPass() = default;
-
-PreservedAnalyses MemProfilerPass::run(Function &F,
-                                       AnalysisManager<Function> &AM) {
-  assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
-         "Memprof with histogram only supports default mapping granularity");
-  Module &M = *F.getParent();
-  MemProfiler Profiler(M);
-  if (Profiler.instrumentFunction(F))
-    return PreservedAnalyses::none();
-  return PreservedAnalyses::all();
-}
-
-ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
-
-PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
-                                             AnalysisManager<Module> &AM) {
-
-  ModuleMemProfiler Profiler(M);
-  if (Profiler.instrumentModule(M))
-    return PreservedAnalyses::none();
-  return PreservedAnalyses::all();
-}
-
-Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
-  // (Shadow & mask) >> scale
-  Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
-  Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
-  // (Shadow >> scale) | offset
-  assert(DynamicShadowOffset);
-  return IRB.CreateAdd(Shadow, DynamicShadowOffset);
-}
-
-// Instrument memset/memmove/memcpy
-void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
-  IRBuilder<> IRB(MI);
-  if (isa<MemTransferInst>(MI)) {
-    IRB.CreateCall(isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
-                   {MI->getOperand(0), MI->getOperand(1),
-                    IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
-  } else if (isa<MemSetInst>(MI)) {
-    IRB.CreateCall(
-        MemProfMemset,
-        {MI->getOperand(0),
-         IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
-         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
-  }
-  MI->eraseFromParent();
-}
-
-std::optional<InterestingMemoryAccess>
-MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
-  // Do not instrument the load fetching the dynamic shadow address.
-  if (DynamicShadowOffset == I)
-    return std::nullopt;
-
-  InterestingMemoryAccess Access;
-
-  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-    if (!ClInstrumentReads)
-      return std::nullopt;
-    Access.IsWrite = false;
-    Access.AccessTy = LI->getType();
-    Access.Addr = LI->getPointerOperand();
-  } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
-    if (!ClInstrumentWrites)
-      return std::nullopt;
-    Access.IsWrite = true;
-    Access.AccessTy = SI->getValueOperand()->getType();
-    Access.Addr = SI->getPointerOperand();
-  } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
-    if (!ClInstrumentAtomics)
-      return std::nullopt;
-    Access.IsWrite = true;
-    Access.AccessTy = RMW->getValOperand()->getType();
-    Access.Addr = RMW->getPointerOperand();
-  } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
-    if (!ClInstrumentAtomics)
-      return std::nullopt;
-    Access.IsWrite = true;
-    Access.AccessTy = XCHG->getCompareOperand()->getType();
-    Access.Addr = XCHG->getPointerOperand();
-  } else if (auto *CI = dyn_cast<CallInst>(I)) {
-    auto *F = CI->getCalledFunction();
-    if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
-              F->getIntrinsicID() == Intrinsic::masked_store)) {
-      unsigned OpOffset = 0;
-      if (F->getIntrinsicID() == Intrinsic::masked_store) {
-        if (!ClInstrumentWrites)
-          return std::nullopt;
-        // Masked store has an initial operand for the value.
-        OpOffset = 1;
-        Access.AccessTy = CI->getArgOperand(0)->getType();
-        Access.IsWrite = true;
-      } else {
-        if (!ClInstrumentReads)
-          return std::nullopt;
-        Access.AccessTy = CI->getType();
-        Access.IsWrite = false;
-      }
-
-      auto *BasePtr = CI->getOperand(0 + OpOffset);
-      Access.MaybeMask = CI->getOperand(2 + OpOffset);
-      Access.Addr = BasePtr;
-    }
-  }
-
-  if (!Access.Addr)
-    return std::nullopt;
-
-  // Do not instrument accesses from different address spaces; we cannot deal
-  // with them.
-  Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
-  if (PtrTy->getPointerAddressSpace() != 0)
-    return std::nullopt;
-
-  // Ignore swifterror addresses.
-  // swifterror memory addresses are mem2reg promoted by instruction
-  // selection. As such they cannot have regular uses like an instrumentation
-  // function and it makes no sense to track them as memory.
-  if (Access.Addr->isSwiftError())
-    return std::nullopt;
-
-  // Peel off GEPs and BitCasts.
-  auto *Addr = Access.Addr->stripInBoundsOffsets();
-
-  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
-    // Do not instrument PGO counter updates.
-    if (GV->hasSection()) {
-      StringRef SectionName = GV->getSection();
-      // Check if the global is in the PGO counters section.
-      auto OF = I->getModule()->getTargetTriple().getObjectFormat();
-      if (SectionName.ends_with(
-              getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
-        return std::nullopt;
-    }
-
-    // Do not instrument accesses to LLVM internal variables.
-    if (GV->getName().starts_with("__llvm"))
-      return std::nullopt;
-  }
-
-  return Access;
-}
-
-void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
-                                              Instruction *I, Value *Addr,
-                                              Type *AccessTy, bool IsWrite) {
-  auto *VTy = cast<FixedVectorType>(AccessTy);
-  unsigned Num = VTy->getNumElements();
-  auto *Zero = ConstantInt::get(IntptrTy, 0);
-  for (unsigned Idx = 0; Idx < Num; ++Idx) {
-    Value *InstrumentedAddress = nullptr;
-    Instruction *InsertBefore = I;
-    if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
-      // dyn_cast as we might get UndefValue
-      if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
-        if (Masked->isZero())
-          // Mask is constant false, so no instrumentation needed.
-          continue;
-        // If we have a true or undef value, fall through to instrumentAddress.
-        // with InsertBefore == I
-      }
-    } else {
-      IRBuilder<> IRB(I);
-      Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
-      Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
-      InsertBefore = ThenTerm;
-    }
-
-    IRBuilder<> IRB(InsertBefore);
-    InstrumentedAddress =
-        IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
-    instrumentAddress(I, InsertBefore, InstrumentedAddress, IsWrite);
-  }
-}
-
-void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
-                                InterestingMemoryAccess &Access) {
-  // Skip instrumentation of stack accesses unless requested.
-  if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
-    if (Access.IsWrite)
-      ++NumSkippedStackWrites;
-    else
-      ++NumSkippedStackReads;
-    return;
-  }
-
-  if (Access.IsWrite)
-    NumInstrumentedWrites++;
-  else
-    NumInstrumentedReads++;
-
-  if (Access.MaybeMask) {
-    instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
-                                Access.AccessTy, Access.IsWrite);
-  } else {
-    // Since the access counts will be accumulated across the entire allocation,
-    // we only update the shadow access count for the first location and thus
-    // don't need to worry about alignment and type size.
-    instrumentAddress(I, I, Access.Addr, Access.IsWrite);
-  }
-}
-
-void MemProfiler::instrumentAddress(Instruction *OrigIns,
-                                    Instruction *InsertBefore, Value *Addr,
-                                    bool IsWrite) {
-  IRBuilder<> IRB(InsertBefore);
-  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
-
-  if (ClUseCalls) {
-    IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
-    return;
-  }
-
-  Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);
-  Type *ShadowPtrTy = PointerType::get(*C, 0);
-
-  Value *ShadowPtr = memToShadow(AddrLong, IRB);
-  Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
-  Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
-  // If we are profiling with histograms, add overflow protection at 255.
-  if (ClHistogram) {
-    Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
-    Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
-    Instruction *IncBlock =
-        SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
-    IRB.SetInsertPoint(IncBlock);
-  }
-  Value *Inc = ConstantInt::get(ShadowTy, 1);
-  ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
-  IRB.CreateStore(ShadowValue, ShadowAddr);
-}
-
-// Create the variable for the profile file name.
-void createProfileFileNameVar(Module &M) {
-  const MDString *MemProfFilename =
-      dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
-  if (!MemProfFilename)
-    return;
-  assert(!MemProfFilename->getString().empty() &&
-         "Unexpected MemProfProfileFilename metadata with empty string");
-  Constant *ProfileNameConst = ConstantDataArray::getString(
-      M.getContext(), MemProfFilename->getString(), true);
-  GlobalVariable *ProfileNameVar = new GlobalVariable(
-      M, ProfileNameConst->getType(), /*isConstant=*/true,
-      GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
-  const Triple &TT = M.getTargetTriple();
-  if (TT.supportsCOMDAT()) {
-    ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
-    ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
-  }
-}
-
-// Set MemprofHistogramFlag as a Global veriable in IR. This makes it accessible
-// to the runtime, changing shadow count behavior.
-void createMemprofHistogramFlagVar(Module &M) {
-  const StringRef VarName(MemProfHistogramFlagVar);
-  Type *IntTy1 = Type::getInt1Ty(M.getContext());
-  auto MemprofHistogramFlag = new GlobalVariable(
-      M, IntTy1, true, GlobalValue::WeakAnyLinkage,
-      Constant::getIntegerValue(IntTy1, APInt(1, ClHistogram)), VarName);
-  const Triple &TT = M.getTargetTriple();
-  if (TT.supportsCOMDAT()) {
-    MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
-    MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));
-  }
-  appendToCompilerUsed(M, MemprofHistogramFlag);
-}
-
-void createMemprofDefaultOptionsVar(Module &M) {
-  Constant *OptionsConst = ConstantDataArray::getString(
-      M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);
-  GlobalVariable *OptionsVar = new GlobalVariable(
-      M, OptionsConst->getType(), /*isConstant=*/true,
-      GlobalValue::WeakAnyLinkage, OptionsConst, getMemprofOptionsSymbolName());
-  const Triple &TT = M.getTargetTriple();
-  if (TT.supportsCOMDAT()) {
-    OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
-    OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));
-  }
-}
-
-bool ModuleMemProfiler::instrumentModule(Module &M) {
-
-  // Create a module constructor.
-  std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
-  std::string VersionCheckName =
-      ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
-                           : "";
-  std::tie(MemProfCtorFunction, std::ignore) =
-      createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
-                                          MemProfInitName, /*InitArgTypes=*/{},
-                                          /*InitArgs=*/{}, VersionCheckName);
-
-  const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
-  appendToGlobalCtors(M, MemProfCtorFunction, Priority);
-
-  createProfileFileNameVar(M);
-
-  createMemprofHistogramFlagVar(M);
-
-  createMemprofDefaultOptionsVar(M);
-
-  return true;
-}
-
-void MemProfiler::initializeCallbacks(Module &M) {
-  IRBuilder<> IRB(*C);
-
-  for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
-    const std::string TypeStr = AccessIsWrite ? "store" : "load";
-    const std::string HistPrefix = ClHistogram ? "hist_" : "";
-
-    SmallVector<Type *, 2> Args1{1, IntptrTy};
-    MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
-        ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
-        FunctionType::get(IRB.getVoidTy(), Args1, false));
-  }
-  MemProfMemmove = M.getOrInsertFunction(
-      ClMemoryAccessCallbackPrefix + "memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
-  MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
-                                        PtrTy, PtrTy, PtrTy, IntptrTy);
-  MemProfMemset =
-      M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset", PtrTy,
-                            PtrTy, IRB.getInt32Ty(), IntptrTy);
-}
-
-bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
-  // For each NSObject descendant having a +load method, this method is invoked
-  // by the ObjC runtime before any of the static constructors is called.
-  // Therefore we need to instrument such methods with a call to __memprof_init
-  // at the beginning in order to initialize our runtime before any access to
-  // the shadow memory.
-  // We cannot just ignore these methods, because they may call other
-  // instrumented functions.
-  if (F.getName().contains(" load]")) {
-    FunctionCallee MemProfInitFunction =
-        declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
-    IRBuilder<> IRB(&F.front(), F.front().begin());
-    IRB.CreateCall(MemProfInitFunction, {});
-    return true;
-  }
-  return false;
-}
-
-bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
-  IRBuilder<> IRB(&F.front().front());
-  Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
-      MemProfShadowMemoryDynamicAddress, IntptrTy);
-  if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
-    cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
-  DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
-  return true;
-}
-
-bool MemProfiler::instrumentFunction(Function &F) {
-  if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
-    return false;
-  if (ClDebugFunc == F.getName())
-    return false;
-  if (F.getName().starts_with("__memprof_"))
-    return false;
-
-  bool FunctionModified = false;
-
-  // If needed, insert __memprof_init.
-  // This function needs to be called even if the function body is not
-  // instrumented.
-  if (maybeInsertMemProfInitAtFunctionEntry(F))
-    FunctionModified = true;
-
-  LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
-
-  initializeCallbacks(*F.getParent());
-
-  SmallVector<Instruction *, 16> ToInstrument;
-
-  // Fill the set of memory operations to instrument.
-  for (auto &BB : F) {
-    for (auto &Inst : BB) {
-      if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
-        ToInstrument.push_back(&Inst);
-    }
-  }
-
-  if (ToInstrument.empty()) {
-    LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
-                      << " " << F << "\n");
-
-    return FunctionModified;
-  }
-
-  FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
-
-  int NumInstrumented = 0;
-  for (auto *Inst : ToInstrument) {
-    if (ClDebugMin < 0 || ClDebugMax < 0 ||
-        (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
-      std::optional<InterestingMemoryAccess> Access =
-          isInterestingMemoryAccess(Inst);
-      if (Access)
-        instrumentMop(Inst, F.getDataLayout(), *Access);
-      else
-        instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
-    }
-    NumInstrumented++;
-  }
-
-  if (NumInstrumented > 0)
-    FunctionModified = true;
-
-  LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
-                    << F << "\n");
-
-  return FunctionModified;
-}
-
 static void addCallsiteMetadata(Instruction &I,
                                 ArrayRef<uint64_t> InlinedCallStack,
                                 LLVMContext &Ctx) {
@@ -758,7 +123,7 @@ static AllocationType addCallStack(CallStackTrie &AllocTrie,
                                 AllocInfo->Info.getAllocCount(),
                                 AllocInfo->Info.getTotalLifetime());
   std::vector<ContextTotalSize> ContextSizeInfo;
-  if (recordContextSizeInfo()) {
+  if (recordContextSizeInfoForAnalysis()) {
     auto TotalSize = AllocInfo->Info.getTotalSize();
     assert(TotalSize);
     assert(FullStackId != 0);
@@ -816,6 +181,11 @@ static bool isAllocationWithHotColdVariant(const Function *Callee,
   }
 }
 
+struct AllocMatchInfo {
+  uint64_t TotalSize = 0;
+  AllocationType AllocType = AllocationType::None;
+};
+
 DenseMap<uint64_t, SmallVector<CallEdgeTy, 0>>
 memprof::extractCallsFromIR(Module &M, const TargetLibraryInfo &TLI,
                             function_ref<bool(uint64_t)> IsPresentInProfile) {
@@ -994,9 +364,11 @@ static void addVPMetadata(Module &M, Instruction &I,
 static void readMemprof(Module &M, Function &F,
                         IndexedInstrProfReader *MemProfReader,
                         const TargetLibraryInfo &TLI,
+                        std::map<std::pair<uint64_t, unsigned>, AllocMatchInfo>
+                            &FullStackIdToAllocMatchInfo,
                         std::set<std::vector<uint64_t>> &MatchedCallSites,
                         DenseMap<uint64_t, LocToLocMap> &UndriftMaps,
-                        OptimizationRemarkEmitter &ORE) {
+                        OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize) {
   auto &Ctx = M.getContext();
   // Previously we used getIRPGOFuncName() here. If F is local linkage,
   // getIRPGOFuncName() returns FuncName with prefix 'FileName;'. But
@@ -1185,7 +557,7 @@ static void readMemprof(Module &M, Function &F,
         // We may match this instruction's location list to multiple MIB
         // contexts. Add them to a Trie specialized for trimming the contexts to
         // the minimal needed to disambiguate contexts with unique behavior.
-        CallStackTrie AllocTrie(&ORE);
+        CallStackTrie AllocTrie(&ORE, MaxColdSize);
         uint64_t TotalSize = 0;
         uint64_t TotalColdSize = 0;
         for (auto *AllocInfo : AllocInfoIter->second) {
@@ -1196,7 +568,7 @@ static void readMemprof(Module &M, Function &F,
                                                  InlinedCallStack)) {
             NumOfMemProfMatchedAllocContexts++;
             uint64_t FullStackId = 0;
-            if (ClPrintMemProfMatchInfo || recordContextSizeInfo())
+            if (ClPrintMemProfMatchInfo || recordContextSizeInfoForAnalysis())
               FullStackId = computeFullStackId(AllocInfo->CallStack);
             auto AllocType = addCallStack(AllocTrie, AllocInfo, FullStackId);
             TotalSize += AllocInfo->Info.getTotalSize();
@@ -1206,11 +578,9 @@ static void readMemprof(Module &M, Function &F,
             // was requested.
             if (ClPrintMemProfMatchInfo) {
               assert(FullStackId != 0);
-              errs() << "MemProf " << getAllocTypeAttributeString(AllocType)
-                     << " context with id " << FullStackId
-                     << " has total profiled size "
-                     << AllocInfo->Info.getTotalSize() << " is matched with "
-                     << InlinedCallStack.size() << " frames\n";
+              FullStackIdToAllocMatchInfo[std::make_pair(
+                  FullStackId, InlinedCallStack.size())] = {
+                  AllocInfo->Info.getTotalSize(), AllocType};
             }
           }
         }
@@ -1325,21 +695,38 @@ PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) {
   if (SalvageStaleProfile)
     UndriftMaps = computeUndriftMap(M, MemProfReader.get(), TLI);
 
+  // Map from the stack hash and matched frame count of each allocation context
+  // in the function profiles to the total profiled size (bytes) and allocation
+  // type.
+  std::map<std::pair<uint64_t, unsigned>, AllocMatchInfo>
+      FullStackIdToAllocMatchInfo;
+
   // Set of the matched call sites, each expressed as a sequence of an inline
   // call stack.
   std::set<std::vector<uint64_t>> MatchedCallSites;
 
+  uint64_t MaxColdSize = 0;
+  if (auto *MemProfSum = MemProfReader->getMemProfSummary())
+    MaxColdSize = MemProfSum->getMaxColdTotalSize();
+
   for (auto &F : M) {
     if (F.isDeclaration())
       continue;
 
     const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
     auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
-    readMemprof(M, F, MemProfReader.get(), TLI, MatchedCallSites, UndriftMaps,
-                ORE);
+    readMemprof(M, F, MemProfReader.get(), TLI, FullStackIdToAllocMatchInfo,
+                MatchedCallSites, UndriftMaps, ORE, MaxColdSize);
   }
 
   if (ClPrintMemProfMatchInfo) {
+    for (const auto &[IdLengthPair, Info] : FullStackIdToAllocMatchInfo) {
+      auto [Id, Length] = IdLengthPair;
+      errs() << "MemProf " << getAllocTypeAttributeString(Info.AllocType)
+             << " context with id " << Id << " has total profiled size "
+             << Info.TotalSize << " is matched with " << Length << " frames\n";
+    }
+
     for (const auto &CallStack : MatchedCallSites) {
       errs() << "MemProf callsite match for inline call stack";
       for (uint64_t StackId : CallStack)
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 232983c9e5a9..fb55bd7bfe56 100644
--- a/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -316,14 +316,22 @@ static cl::opt<bool> ClEagerChecks(
 
 static cl::opt<bool> ClDumpStrictInstructions(
     "msan-dump-strict-instructions",
-    cl::desc("print out instructions with default strict semantics"),
+    cl::desc("print out instructions with default strict semantics i.e.,"
+             "check that all the inputs are fully initialized, and mark "
+             "the output as fully initialized. These semantics are applied "
+             "to instructions that could not be handled explicitly nor "
+             "heuristically."),
     cl::Hidden, cl::init(false));
 
-static cl::opt<bool> ClDumpStrictIntrinsics(
-    "msan-dump-strict-intrinsics",
-    cl::desc("Prints 'unknown' intrinsics that were handled heuristically. "
-             "Use -msan-dump-strict-instructions to print intrinsics that "
-             "could not be handled exactly nor heuristically."),
+// Currently, all the heuristically handled instructions are specifically
+// IntrinsicInst. However, we use the broader "HeuristicInstructions" name
+// to parallel 'msan-dump-strict-instructions', and to keep the door open to
+// handling non-intrinsic instructions heuristically.
+static cl::opt<bool> ClDumpHeuristicInstructions(
+    "msan-dump-heuristic-instructions",
+    cl::desc("Prints 'unknown' instructions that were handled heuristically. "
+             "Use -msan-dump-strict-instructions to print instructions that "
+             "could not be handled explicitly nor heuristically."),
     cl::Hidden, cl::init(false));
 
 static cl::opt<int> ClInstrumentationWithCallThreshold(
@@ -2077,6 +2085,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       assert(ShadowPtr && "Could not find shadow for an argument");
       return ShadowPtr;
     }
+
+    // TODO: Partially undefined vectors are handled by the fall-through case
+    //       below (see partial-poison.ll); this causes false negatives.
+
     // For everything else the shadow is zero.
     return getCleanShadow(V);
   }
@@ -3197,10 +3209,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
   bool handleUnknownIntrinsic(IntrinsicInst &I) {
     if (handleUnknownIntrinsicUnlogged(I)) {
-      if (ClDumpStrictIntrinsics)
+      if (ClDumpHeuristicInstructions)
         dumpInst(I);
 
-      LLVM_DEBUG(dbgs() << "UNKNOWN INTRINSIC HANDLED HEURISTICALLY: " << I
+      LLVM_DEBUG(dbgs() << "UNKNOWN INSTRUCTION HANDLED HEURISTICALLY: " << I
                         << "\n");
       return true;
     } else
@@ -4159,6 +4171,63 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOrigin(&I, PtrSrcOrigin);
   }
 
+  // Instrument AVX permutation intrinsic.
+  // We apply the same permutation (argument index 1) to the shadow.
+  void handleAVXPermutation(IntrinsicInst &I) {
+    assert(I.arg_size() == 2);
+    assert(isa<FixedVectorType>(I.getArgOperand(0)->getType()));
+    assert(isa<FixedVectorType>(I.getArgOperand(1)->getType()));
+    [[maybe_unused]] auto ArgVectorSize =
+        cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
+    assert(cast<FixedVectorType>(I.getArgOperand(1)->getType())
+               ->getNumElements() == ArgVectorSize);
+    assert(I.getType() == I.getArgOperand(0)->getType());
+    IRBuilder<> IRB(&I);
+    Value *Shadow = getShadow(&I, 0);
+    insertShadowCheck(I.getArgOperand(1), &I);
+
+    // Shadows are integer-ish types but some intrinsics require a
+    // different (e.g., floating-point) type.
+    Shadow = IRB.CreateBitCast(Shadow, I.getArgOperand(0)->getType());
+    CallInst *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+                                       {Shadow, I.getArgOperand(1)});
+
+    setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
+    setOriginForNaryOp(I);
+  }
+  // Instrument AVX permutation intrinsic.
+  // We apply the same permutation (argument index 1) to the shadows.
+  void handleAVXVpermil2var(IntrinsicInst &I) {
+    assert(I.arg_size() == 3);
+    assert(isa<FixedVectorType>(I.getArgOperand(0)->getType()));
+    assert(isa<FixedVectorType>(I.getArgOperand(1)->getType()));
+    assert(isa<FixedVectorType>(I.getArgOperand(2)->getType()));
+    [[maybe_unused]] auto ArgVectorSize =
+        cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
+    assert(cast<FixedVectorType>(I.getArgOperand(1)->getType())
+               ->getNumElements() == ArgVectorSize);
+    assert(cast<FixedVectorType>(I.getArgOperand(2)->getType())
+               ->getNumElements() == ArgVectorSize);
+    assert(I.getArgOperand(0)->getType() == I.getArgOperand(2)->getType());
+    assert(I.getType() == I.getArgOperand(0)->getType());
+    assert(I.getArgOperand(1)->getType()->isIntOrIntVectorTy());
+    IRBuilder<> IRB(&I);
+    Value *AShadow = getShadow(&I, 0);
+    Value *Idx = I.getArgOperand(1);
+    Value *BShadow = getShadow(&I, 2);
+    insertShadowCheck(Idx, &I);
+
+    // Shadows are integer-ish types but some intrinsics require a
+    // different (e.g., floating-point) type.
+    AShadow = IRB.CreateBitCast(AShadow, I.getArgOperand(0)->getType());
+    BShadow = IRB.CreateBitCast(BShadow, I.getArgOperand(2)->getType());
+    CallInst *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+                                       {AShadow, Idx, BShadow});
+
+    setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
+    setOriginForNaryOp(I);
+  }
+
   // Instrument BMI / BMI2 intrinsics.
   // All of these intrinsics are Z = I(X, Y)
   // where the types of all operands and the result match, and are either i32 or
@@ -5103,6 +5172,52 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       assert(Success);
       break;
     }
+    case Intrinsic::x86_avx2_permd:
+    case Intrinsic::x86_avx2_permps:
+    case Intrinsic::x86_ssse3_pshuf_b_128:
+    case Intrinsic::x86_avx2_pshuf_b:
+    case Intrinsic::x86_avx512_pshuf_b_512:
+    case Intrinsic::x86_avx512_permvar_df_256:
+    case Intrinsic::x86_avx512_permvar_df_512:
+    case Intrinsic::x86_avx512_permvar_di_256:
+    case Intrinsic::x86_avx512_permvar_di_512:
+    case Intrinsic::x86_avx512_permvar_hi_128:
+    case Intrinsic::x86_avx512_permvar_hi_256:
+    case Intrinsic::x86_avx512_permvar_hi_512:
+    case Intrinsic::x86_avx512_permvar_qi_128:
+    case Intrinsic::x86_avx512_permvar_qi_256:
+    case Intrinsic::x86_avx512_permvar_qi_512:
+    case Intrinsic::x86_avx512_permvar_sf_512:
+    case Intrinsic::x86_avx512_permvar_si_512:
+    case Intrinsic::x86_avx_vpermilvar_pd:
+    case Intrinsic::x86_avx_vpermilvar_pd_256:
+    case Intrinsic::x86_avx512_vpermilvar_pd_512:
+    case Intrinsic::x86_avx_vpermilvar_ps:
+    case Intrinsic::x86_avx_vpermilvar_ps_256:
+    case Intrinsic::x86_avx512_vpermilvar_ps_512: {
+      handleAVXPermutation(I);
+      break;
+    }
+    case Intrinsic::x86_avx512_vpermi2var_d_128:
+    case Intrinsic::x86_avx512_vpermi2var_d_256:
+    case Intrinsic::x86_avx512_vpermi2var_d_512:
+    case Intrinsic::x86_avx512_vpermi2var_hi_128:
+    case Intrinsic::x86_avx512_vpermi2var_hi_256:
+    case Intrinsic::x86_avx512_vpermi2var_hi_512:
+    case Intrinsic::x86_avx512_vpermi2var_pd_128:
+    case Intrinsic::x86_avx512_vpermi2var_pd_256:
+    case Intrinsic::x86_avx512_vpermi2var_pd_512:
+    case Intrinsic::x86_avx512_vpermi2var_ps_128:
+    case Intrinsic::x86_avx512_vpermi2var_ps_256:
+    case Intrinsic::x86_avx512_vpermi2var_ps_512:
+    case Intrinsic::x86_avx512_vpermi2var_q_128:
+    case Intrinsic::x86_avx512_vpermi2var_q_256:
+    case Intrinsic::x86_avx512_vpermi2var_q_512:
+    case Intrinsic::x86_avx512_vpermi2var_qi_128:
+    case Intrinsic::x86_avx512_vpermi2var_qi_256:
+    case Intrinsic::x86_avx512_vpermi2var_qi_512:
+      handleAVXVpermil2var(I);
+      break;
 
     case Intrinsic::x86_avx512fp16_mask_add_sh_round:
     case Intrinsic::x86_avx512fp16_mask_sub_sh_round:
diff --git a/external/llvm-project/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/external/llvm-project/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 3347f3d376c9..9c92a7feb6e6 100644
--- a/external/llvm-project/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -100,6 +100,7 @@
 #include "llvm/Support/CRC.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DOTGraphTraits.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
@@ -359,7 +360,8 @@ extern cl::opt<std::string> ViewBlockFreqFuncName;
 // ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
 extern cl::opt<bool> EnableVTableValueProfiling;
 extern cl::opt<bool> EnableVTableProfileUse;
-extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
+LLVM_ABI extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind>
+    ProfileCorrelate;
 } // namespace llvm
 
 namespace {
diff --git a/external/llvm-project/llvm/lib/Transforms/ObjCARC/BlotMapVector.h b/external/llvm-project/llvm/lib/Transforms/ObjCARC/BlotMapVector.h
index 2fa07cfb32c0..0e64ccabe575 100644
--- a/external/llvm-project/llvm/lib/Transforms/ObjCARC/BlotMapVector.h
+++ b/external/llvm-project/llvm/lib/Transforms/ObjCARC/BlotMapVector.h
@@ -53,8 +53,7 @@ template <class KeyT, class ValueT> class BlotMapVector {
   const_iterator end() const { return Vector.end(); }
 
   ValueT &operator[](const KeyT &Arg) {
-    std::pair<typename MapTy::iterator, bool> Pair =
-        Map.insert(std::make_pair(Arg, size_t(0)));
+    std::pair<typename MapTy::iterator, bool> Pair = Map.try_emplace(Arg);
     if (Pair.second) {
       size_t Num = Vector.size();
       Pair.first->second = Num;
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index dd4d4efb7fec..839f5933e09b 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -381,7 +381,7 @@ void ConstantHoistingPass::collectConstantCandidates(
     ConstCandMapType::iterator Itr;
     bool Inserted;
     ConstPtrUnionType Cand = ConstInt;
-    std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
+    std::tie(Itr, Inserted) = ConstCandMap.try_emplace(Cand);
     if (Inserted) {
       ConstIntCandVec.push_back(ConstantCandidate(ConstInt));
       Itr->second = ConstIntCandVec.size() - 1;
@@ -439,7 +439,7 @@ void ConstantHoistingPass::collectConstantCandidates(
   ConstCandMapType::iterator Itr;
   bool Inserted;
   ConstPtrUnionType Cand = ConstExpr;
-  std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
+  std::tie(Itr, Inserted) = ConstCandMap.try_emplace(Cand);
   if (Inserted) {
     ExprCandVec.push_back(ConstantCandidate(
         ConstantInt::get(Type::getInt32Ty(*Ctx), Offset.getLimitedValue()),
@@ -883,7 +883,7 @@ bool ConstantHoistingPass::emitBaseConstants(GlobalVariable *BaseGV) {
         emitBaseConstants(Base, &R);
         ReBasesNum++;
         // Use the same debug location as the last user of the constant.
-        Base->setDebugLoc(DILocation::getMergedLocation(
+        Base->setDebugLoc(DebugLoc::getMergedLocation(
             Base->getDebugLoc(), R.User.Inst->getDebugLoc()));
       }
       assert(!Base->use_empty() && "The use list is empty!?");
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index cbad5dd35768..2786d81773ed 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1010,9 +1010,9 @@ void State::addInfoForInductions(BasicBlock &BB) {
   auto IncUnsigned = SE.getMonotonicPredicateType(AR, CmpInst::ICMP_UGT);
   auto IncSigned = SE.getMonotonicPredicateType(AR, CmpInst::ICMP_SGT);
   bool MonotonicallyIncreasingUnsigned =
-      IncUnsigned && *IncUnsigned == ScalarEvolution::MonotonicallyIncreasing;
+      IncUnsigned == ScalarEvolution::MonotonicallyIncreasing;
   bool MonotonicallyIncreasingSigned =
-      IncSigned && *IncSigned == ScalarEvolution::MonotonicallyIncreasing;
+      IncSigned == ScalarEvolution::MonotonicallyIncreasing;
   // If SCEV guarantees that AR does not wrap, PN >= StartValue can be added
   // unconditionally.
   if (MonotonicallyIncreasingUnsigned)
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 5226aeb66f65..4627f537dc16 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -370,15 +370,30 @@ static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI,
   { // Scope for SwitchInstProfUpdateWrapper. It must not live during
     // ConstantFoldTerminator() as the underlying SwitchInst can be changed.
     SwitchInstProfUpdateWrapper SI(*I);
+    ConstantRange CR =
+        LVI->getConstantRangeAtUse(I->getOperandUse(0), /*UndefAllowed=*/false);
     unsigned ReachableCaseCount = 0;
 
     for (auto CI = SI->case_begin(), CE = SI->case_end(); CI != CE;) {
       ConstantInt *Case = CI->getCaseValue();
-      auto *Res = dyn_cast_or_null<ConstantInt>(
-          LVI->getPredicateAt(CmpInst::ICMP_EQ, Cond, Case, I,
-                              /* UseBlockValue */ true));
+      std::optional<bool> Predicate = std::nullopt;
+      if (!CR.contains(Case->getValue()))
+        Predicate = false;
+      else if (CR.isSingleElement() &&
+               *CR.getSingleElement() == Case->getValue())
+        Predicate = true;
+      if (!Predicate) {
+        // Handle missing cases, e.g., the range has a hole.
+        auto *Res = dyn_cast_or_null<ConstantInt>(
+            LVI->getPredicateAt(CmpInst::ICMP_EQ, Cond, Case, I,
+                                /* UseBlockValue=*/true));
+        if (Res && Res->isZero())
+          Predicate = false;
+        else if (Res && Res->isOne())
+          Predicate = true;
+      }
 
-      if (Res && Res->isZero()) {
+      if (Predicate && !*Predicate) {
         // This case never fires - remove it.
         BasicBlock *Succ = CI->getCaseSuccessor();
         Succ->removePredecessor(BB);
@@ -395,7 +410,7 @@ static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI,
           DTU.applyUpdatesPermissive({{DominatorTree::Delete, BB, Succ}});
         continue;
       }
-      if (Res && Res->isOne()) {
+      if (Predicate && *Predicate) {
         // This case always fires.  Arrange for the switch to be turned into an
         // unconditional branch by replacing the switch condition with the case
         // value.
@@ -410,27 +425,25 @@ static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI,
       ++ReachableCaseCount;
     }
 
-    if (ReachableCaseCount > 1 && !SI->defaultDestUnreachable()) {
+    // The default dest is unreachable if all cases are covered.
+    if (!SI->defaultDestUnreachable() &&
+        !CR.isSizeLargerThan(ReachableCaseCount)) {
       BasicBlock *DefaultDest = SI->getDefaultDest();
-      ConstantRange CR = LVI->getConstantRangeAtUse(I->getOperandUse(0),
-                                                    /*UndefAllowed*/ false);
-      // The default dest is unreachable if all cases are covered.
-      if (!CR.isSizeLargerThan(ReachableCaseCount)) {
-        BasicBlock *NewUnreachableBB =
-            BasicBlock::Create(BB->getContext(), "default.unreachable",
-                               BB->getParent(), DefaultDest);
-        new UnreachableInst(BB->getContext(), NewUnreachableBB);
+      BasicBlock *NewUnreachableBB =
+          BasicBlock::Create(BB->getContext(), "default.unreachable",
+                             BB->getParent(), DefaultDest);
+      auto *UI = new UnreachableInst(BB->getContext(), NewUnreachableBB);
+      UI->setDebugLoc(DebugLoc::getTemporary());
 
-        DefaultDest->removePredecessor(BB);
-        SI->setDefaultDest(NewUnreachableBB);
+      DefaultDest->removePredecessor(BB);
+      SI->setDefaultDest(NewUnreachableBB);
 
-        if (SuccessorsCount[DefaultDest] == 1)
-          DTU.applyUpdates({{DominatorTree::Delete, BB, DefaultDest}});
-        DTU.applyUpdates({{DominatorTree::Insert, BB, NewUnreachableBB}});
+      if (SuccessorsCount[DefaultDest] == 1)
+        DTU.applyUpdates({{DominatorTree::Delete, BB, DefaultDest}});
+      DTU.applyUpdates({{DominatorTree::Insert, BB, NewUnreachableBB}});
 
-        ++NumDeadCases;
-        Changed = true;
-      }
+      ++NumDeadCases;
+      Changed = true;
     }
   }
 
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 15192edc4d57..4a2eb9284a6e 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -2029,10 +2029,18 @@ struct DSEState {
     auto *InnerCallee = Malloc->getCalledFunction();
     if (!InnerCallee)
       return false;
-    LibFunc Func;
+    LibFunc Func = NotLibFunc;
+    StringRef ZeroedVariantName;
     if (!TLI.getLibFunc(*InnerCallee, Func) || !TLI.has(Func) ||
-        Func != LibFunc_malloc)
-      return false;
+        Func != LibFunc_malloc) {
+      Attribute Attr = Malloc->getFnAttr("alloc-variant-zeroed");
+      if (!Attr.isValid())
+        return false;
+      ZeroedVariantName = Attr.getValueAsString();
+      if (ZeroedVariantName.empty())
+        return false;
+    }
+
     // Gracefully handle malloc with unexpected memory attributes.
     auto *MallocDef = dyn_cast_or_null<MemoryDef>(MSSA.getMemoryAccess(Malloc));
     if (!MallocDef)
@@ -2059,15 +2067,32 @@ struct DSEState {
 
     if (Malloc->getOperand(0) != MemSet->getLength())
       return false;
-    if (!shouldCreateCalloc(Malloc, MemSet) ||
-        !DT.dominates(Malloc, MemSet) ||
+    if (!shouldCreateCalloc(Malloc, MemSet) || !DT.dominates(Malloc, MemSet) ||
         !memoryIsNotModifiedBetween(Malloc, MemSet, BatchAA, DL, &DT))
       return false;
     IRBuilder<> IRB(Malloc);
-    Type *SizeTTy = Malloc->getArgOperand(0)->getType();
-    auto *Calloc =
-        emitCalloc(ConstantInt::get(SizeTTy, 1), Malloc->getArgOperand(0), IRB,
-                   TLI, Malloc->getType()->getPointerAddressSpace());
+    assert(Func == LibFunc_malloc || !ZeroedVariantName.empty());
+    Value *Calloc = nullptr;
+    if (!ZeroedVariantName.empty()) {
+      LLVMContext &Ctx = Malloc->getContext();
+      AttributeList Attrs = InnerCallee->getAttributes();
+      AllocFnKind AllocKind =
+          Attrs.getFnAttr(Attribute::AllocKind).getAllocKind() |
+          AllocFnKind::Zeroed;
+      Attrs =
+          Attrs.addFnAttribute(Ctx, Attribute::getWithAllocKind(Ctx, AllocKind))
+              .removeFnAttribute(Ctx, "alloc-variant-zeroed");
+      FunctionCallee ZeroedVariant = Malloc->getModule()->getOrInsertFunction(
+          ZeroedVariantName, InnerCallee->getFunctionType(), Attrs);
+      SmallVector<Value *, 3> Args;
+      Args.append(Malloc->arg_begin(), Malloc->arg_end());
+      Calloc = IRB.CreateCall(ZeroedVariant, Args, ZeroedVariantName);
+    } else {
+      Type *SizeTTy = Malloc->getArgOperand(0)->getType();
+      Calloc =
+          emitCalloc(ConstantInt::get(SizeTTy, 1), Malloc->getArgOperand(0),
+                     IRB, TLI, Malloc->getType()->getPointerAddressSpace());
+    }
     if (!Calloc)
       return false;
 
@@ -2320,7 +2345,8 @@ bool isFuncLocalAndNotCaptured(Value *Arg, const CallBase *CB,
                                EarliestEscapeAnalysis &EA) {
   const Value *UnderlyingObj = getUnderlyingObject(Arg);
   return isIdentifiedFunctionLocal(UnderlyingObj) &&
-         EA.isNotCapturedBefore(UnderlyingObj, CB, /*OrAt*/ true);
+         capturesNothing(
+             EA.getCapturesBefore(UnderlyingObj, CB, /*OrAt*/ true));
 }
 
 SmallVector<MemoryLocation, 1>
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 09cb2f4cb010..5c62a2cf526e 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -400,7 +400,9 @@ static bool isEqualImpl(SimpleValue LHS, SimpleValue RHS) {
     return LII->getArgOperand(0) == RII->getArgOperand(1) &&
            LII->getArgOperand(1) == RII->getArgOperand(0) &&
            std::equal(LII->arg_begin() + 2, LII->arg_end(),
-                      RII->arg_begin() + 2, RII->arg_end());
+                      RII->arg_begin() + 2, RII->arg_end()) &&
+           LII->hasSameSpecialState(RII, /*IgnoreAlignment=*/false,
+                                    /*IntersectAttrs=*/true);
   }
 
   // See comment above in `getHashValue()`.
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/GVN.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/GVN.cpp
index 77ca14f88a83..c8a0479358ea 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -1165,7 +1165,7 @@ static bool isLifetimeStart(const Instruction *Inst) {
 /// Assuming To can be reached from both From and Between, does Between lie on
 /// every path from From to To?
 static bool liesBetween(const Instruction *From, Instruction *Between,
-                        const Instruction *To, DominatorTree *DT) {
+                        const Instruction *To, const DominatorTree *DT) {
   if (From->getParent() == Between->getParent())
     return DT->dominates(From, Between);
   SmallSet<BasicBlock *, 1> Exclusion;
@@ -1173,20 +1173,15 @@ static bool liesBetween(const Instruction *From, Instruction *Between,
   return !isPotentiallyReachable(From, To, &Exclusion, DT);
 }
 
-/// Try to locate the three instruction involved in a missed
-/// load-elimination case that is due to an intervening store.
-static void reportMayClobberedLoad(LoadInst *Load, MemDepResult DepInfo,
-                                   DominatorTree *DT,
-                                   OptimizationRemarkEmitter *ORE) {
-  using namespace ore;
+static const Instruction *findMayClobberedPtrAccess(LoadInst *Load,
+                                                    const DominatorTree *DT) {
+  Value *PtrOp = Load->getPointerOperand();
+  if (!PtrOp->hasUseList())
+    return nullptr;
 
   Instruction *OtherAccess = nullptr;
 
-  OptimizationRemarkMissed R(DEBUG_TYPE, "LoadClobbered", Load);
-  R << "load of type " << NV("Type", Load->getType()) << " not eliminated"
-    << setExtraArgs();
-
-  for (auto *U : Load->getPointerOperand()->users()) {
+  for (auto *U : PtrOp->users()) {
     if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
       auto *I = cast<Instruction>(U);
       if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) {
@@ -1202,32 +1197,48 @@ static void reportMayClobberedLoad(LoadInst *Load, MemDepResult DepInfo,
     }
   }
 
-  if (!OtherAccess) {
-    // There is no dominating use, check if we can find a closest non-dominating
-    // use that lies between any other potentially available use and Load.
-    for (auto *U : Load->getPointerOperand()->users()) {
-      if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
-        auto *I = cast<Instruction>(U);
-        if (I->getFunction() == Load->getFunction() &&
-            isPotentiallyReachable(I, Load, nullptr, DT)) {
-          if (OtherAccess) {
-            if (liesBetween(OtherAccess, I, Load, DT)) {
-              OtherAccess = I;
-            } else if (!liesBetween(I, OtherAccess, Load, DT)) {
-              // These uses are both partially available at Load were it not for
-              // the clobber, but neither lies strictly after the other.
-              OtherAccess = nullptr;
-              break;
-            } // else: keep current OtherAccess since it lies between U and
-              // Load.
-          } else {
+  if (OtherAccess)
+    return OtherAccess;
+
+  // There is no dominating use, check if we can find a closest non-dominating
+  // use that lies between any other potentially available use and Load.
+  for (auto *U : PtrOp->users()) {
+    if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
+      auto *I = cast<Instruction>(U);
+      if (I->getFunction() == Load->getFunction() &&
+          isPotentiallyReachable(I, Load, nullptr, DT)) {
+        if (OtherAccess) {
+          if (liesBetween(OtherAccess, I, Load, DT)) {
             OtherAccess = I;
-          }
+          } else if (!liesBetween(I, OtherAccess, Load, DT)) {
+            // These uses are both partially available at Load were it not for
+            // the clobber, but neither lies strictly after the other.
+            OtherAccess = nullptr;
+            break;
+          } // else: keep current OtherAccess since it lies between U and
+          // Load.
+        } else {
+          OtherAccess = I;
         }
       }
     }
   }
 
+  return OtherAccess;
+}
+
+/// Try to locate the three instruction involved in a missed
+/// load-elimination case that is due to an intervening store.
+static void reportMayClobberedLoad(LoadInst *Load, MemDepResult DepInfo,
+                                   const DominatorTree *DT,
+                                   OptimizationRemarkEmitter *ORE) {
+  using namespace ore;
+
+  OptimizationRemarkMissed R(DEBUG_TYPE, "LoadClobbered", Load);
+  R << "load of type " << NV("Type", Load->getType()) << " not eliminated"
+    << setExtraArgs();
+
+  const Instruction *OtherAccess = findMayClobberedPtrAccess(Load, DT);
   if (OtherAccess)
     R << " in favor of " << NV("OtherAccess", OtherAccess);
 
@@ -2579,6 +2590,14 @@ bool GVNPass::propagateEquality(Value *LHS, Value *RHS,
 
       continue;
     }
+
+    // Propagate equalities that results from truncation with no unsigned wrap
+    // like (trunc nuw i64 %v to i1) == "true" or (trunc nuw i64 %v to i1) ==
+    // "false"
+    if (match(LHS, m_NUWTrunc(m_Value(A)))) {
+      Worklist.emplace_back(A, ConstantInt::get(A->getType(), IsKnownTrue));
+      continue;
+    }
   }
 
   return Changed;
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 95d52b9b4e18..334c911191cb 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1506,6 +1506,9 @@ bool IndVarSimplify::canonicalizeExitCondition(Loop *L) {
       auto *NewRHS = CastInst::Create(
           Instruction::Trunc, RHS, LHSOp->getType(), "",
           L->getLoopPreheader()->getTerminator()->getIterator());
+      // NewRHS is an operation that has been hoisted out of the loop, and
+      // therefore should have a dropped location.
+      NewRHS->setDebugLoc(DebugLoc::getDropped());
       ICmp->setOperand(Swapped ? 1 : 0, LHSOp);
       ICmp->setOperand(Swapped ? 0 : 1, NewRHS);
       // Samesign flag cannot be preserved after narrowing the compare.
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index 66836ef05d5d..d5569c469690 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -174,6 +174,7 @@ class InferAddressSpaces : public FunctionPass {
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
     AU.addPreserved<DominatorTreeWrapperPass>();
+    AU.addRequired<DominatorTreeWrapperPass>();
     AU.addRequired<AssumptionCacheTracker>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
   }
@@ -261,6 +262,7 @@ INITIALIZE_PASS_BEGIN(InferAddressSpaces, DEBUG_TYPE, "Infer address spaces",
                       false, false)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_END(InferAddressSpaces, DEBUG_TYPE, "Infer address spaces",
                     false, false)
 
@@ -869,6 +871,11 @@ Value *InferAddressSpacesImpl::cloneValueWithNewAddressSpace(
         NewI->setDebugLoc(I->getDebugLoc());
       }
     }
+    if (NewV) {
+      Instruction *DomPoint =
+          isa<Instruction>(NewV) ? cast<Instruction>(NewV) : I;
+      replaceAllDbgUsesWith(*I, *NewV, *DomPoint, *DT);
+    }
     return NewV;
   }
 
@@ -1430,10 +1437,9 @@ bool InferAddressSpaces::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
-  auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
-  DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
   return InferAddressSpacesImpl(
-             getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F), DT,
+             getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
+             &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
              &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F),
              FlatAddrSpace)
       .run(F);
@@ -1452,7 +1458,7 @@ PreservedAnalyses InferAddressSpacesPass::run(Function &F,
                                               FunctionAnalysisManager &AM) {
   bool Changed =
       InferAddressSpacesImpl(AM.getResult<AssumptionAnalysis>(F),
-                             AM.getCachedResult<DominatorTreeAnalysis>(F),
+                             &AM.getResult<DominatorTreeAnalysis>(F),
                              &AM.getResult<TargetIRAnalysis>(F), FlatAddrSpace)
           .run(F);
   if (Changed) {
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/InferAlignment.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/InferAlignment.cpp
index ced04fdb5b4e..0ddc23152d84 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/InferAlignment.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/InferAlignment.cpp
@@ -62,7 +62,7 @@ bool inferAlignment(Function &F, AssumptionCache &AC, DominatorTree &DT) {
     for (Instruction &I : BB) {
       Changed |= tryToImproveAlign(
           DL, &I, [&](Value *PtrOp, Align OldAlign, Align PrefAlign) {
-            KnownBits Known = computeKnownBits(PtrOp, DL, 0, &AC, &I, &DT);
+            KnownBits Known = computeKnownBits(PtrOp, DL, &AC, &I, &DT);
             unsigned TrailZ = std::min(Known.countMinTrailingZeros(),
                                        +Value::MaxAlignmentExponent);
             return Align(1ull << std::min(Known.getBitWidth() - 1, TrailZ));
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 9449b4cb35b9..37b85bf9de81 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -3001,8 +3001,10 @@ bool JumpThreadingPass::tryToUnfoldSelectInCurrBB(BasicBlock *BB) {
       continue;
     // Expand the select.
     Value *Cond = SI->getCondition();
-    if (!isGuaranteedNotToBeUndefOrPoison(Cond, nullptr, SI))
+    if (!isGuaranteedNotToBeUndefOrPoison(Cond, nullptr, SI)) {
       Cond = new FreezeInst(Cond, "cond.fr", SI->getIterator());
+      cast<FreezeInst>(Cond)->setDebugLoc(DebugLoc::getTemporary());
+    }
     MDNode *BranchWeights = getBranchWeightMDNode(*SI);
     Instruction *Term =
         SplitBlockAndInsertIfThen(Cond, SI, false, BranchWeights);
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/LICM.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/LICM.cpp
index 7965ed76a81b..bd59caa6a959 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -786,8 +786,8 @@ class ControlFlowHoister {
       return InitialPreheader;
     }
     BranchInst *BI = It->first;
-    assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==
-               HoistableBranches.end() &&
+    assert(std::none_of(std::next(It), HoistableBranches.end(),
+                        HasBBAsSuccessor) &&
            "BB is expected to be the target of at most one branch");
 
     LLVMContext &C = BB->getContext();
@@ -2224,10 +2224,10 @@ bool llvm::promoteLoopAccessesToScalars(
   });
 
   // Look at all the loop uses, and try to merge their locations.
-  std::vector<DILocation *> LoopUsesLocs;
-  for (auto *U : LoopUses)
-    LoopUsesLocs.push_back(U->getDebugLoc().get());
-  auto DL = DebugLoc(DILocation::getMergedLocations(LoopUsesLocs));
+  std::vector<DebugLoc> LoopUsesLocs;
+  for (auto U : LoopUses)
+    LoopUsesLocs.push_back(U->getDebugLoc());
+  auto DL = DebugLoc::getMergedLocations(LoopUsesLocs);
 
   // We use the SSAUpdater interface to insert phi nodes as required.
   SmallVector<PHINode *, 16> NewPHIs;
@@ -2248,7 +2248,7 @@ bool llvm::promoteLoopAccessesToScalars(
     if (SawUnorderedAtomic)
       PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
     PreheaderLoad->setAlignment(Alignment);
-    PreheaderLoad->setDebugLoc(DebugLoc());
+    PreheaderLoad->setDebugLoc(DebugLoc::getDropped());
     if (AATags && LoadIsGuaranteedToExecute)
       PreheaderLoad->setAAMetadata(AATags);
 
@@ -2808,6 +2808,7 @@ static bool hoistMulAddAssociation(Instruction &I, Loop &L,
     auto *NewBO =
         BinaryOperator::Create(Ins->getOpcode(), LHS, RHS,
                                Ins->getName() + ".reass", Ins->getIterator());
+    NewBO->setDebugLoc(DebugLoc::getDropped());
     NewBO->copyIRFlags(Ins);
     if (VariantOp == Ins)
       VariantOp = NewBO;
@@ -2864,6 +2865,7 @@ static bool hoistBOAssociation(Instruction &I, Loop &L,
 
   auto *NewBO = BinaryOperator::Create(
       Opcode, LV, Inv, BO->getName() + ".reass", BO->getIterator());
+  NewBO->setDebugLoc(DebugLoc::getDropped());
 
   if (Opcode == Instruction::FAdd || Opcode == Instruction::FMul) {
     // Intersect FMF flags for FADD and FMUL.
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp
index 3d3f22d686e3..7253c7f22250 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp
@@ -28,7 +28,7 @@ PreservedAnalyses LoopAccessInfoPrinterPass::run(Function &F,
   while (!Worklist.empty()) {
     Loop *L = Worklist.pop_back_val();
     OS.indent(2) << L->getHeader()->getName() << ":\n";
-    LAIs.getInfo(*L).print(OS, 4);
+    LAIs.getInfo(*L, AllowPartial).print(OS, 4);
   }
   return PreservedAnalyses::all();
 }
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index f1457a275041..1e16677e5a56 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -209,6 +209,14 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
             Direction = '*';
           Dep.push_back(Direction);
         }
+
+        // If the Dependence object doesn't have any information, fill the
+        // dependency vector with '*'.
+        if (D->isConfused()) {
+          assert(Dep.empty() && "Expected empty dependency vector");
+          Dep.assign(Level, '*');
+        }
+
         while (Dep.size() != Level) {
           Dep.push_back('I');
         }
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index 39e8d702a692..6bdf76f789a4 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -442,7 +442,7 @@ class LoadEliminationForLoop {
     assert(PH && "Preheader should exist!");
     Value *InitialPtr = SEE.expandCodeFor(PtrSCEV->getStart(), Ptr->getType(),
                                           PH->getTerminator());
-    Value *Initial =
+    Instruction *Initial =
         new LoadInst(Cand.Load->getType(), InitialPtr, "load_initial",
                      /* isVolatile */ false, Cand.Load->getAlign(),
                      PH->getTerminator()->getIterator());
@@ -450,6 +450,7 @@ class LoadEliminationForLoop {
     // into the loop's preheader. A debug location inside the loop will cause
     // a misleading stepping when debugging. The test update-debugloc-store
     // -forwarded.ll checks this.
+    Initial->setDebugLoc(DebugLoc::getDropped());
 
     PHINode *PHI = PHINode::Create(Initial->getType(), 2, "store_forwarded");
     PHI->insertBefore(L->getHeader()->begin());
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index afac49ad03ba..e4f35e4b2108 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -2788,7 +2788,7 @@ std::pair<size_t, Immediate> LSRInstance::getUse(const SCEV *&Expr,
   }
 
   std::pair<UseMapTy::iterator, bool> P =
-    UseMap.insert(std::make_pair(LSRUse::SCEVUseKindPair(Expr, Kind), 0));
+      UseMap.try_emplace(LSRUse::SCEVUseKindPair(Expr, Kind));
   if (!P.second) {
     // A use already existed with this base.
     size_t LUIdx = P.first->second;
@@ -6008,9 +6008,8 @@ static bool canHoistIVInc(const TargetTransformInfo &TTI, const LSRFixup &Fixup,
 
   Instruction *I = Fixup.UserInst;
   Type *Ty = I->getType();
-  return Ty->isIntegerTy() &&
-         ((isa<LoadInst>(I) && TTI.isIndexedLoadLegal(TTI.MIM_PostInc, Ty)) ||
-          (isa<StoreInst>(I) && TTI.isIndexedStoreLegal(TTI.MIM_PostInc, Ty)));
+  return (isa<LoadInst>(I) && TTI.isIndexedLoadLegal(TTI.MIM_PostInc, Ty)) ||
+         (isa<StoreInst>(I) && TTI.isIndexedStoreLegal(TTI.MIM_PostInc, Ty));
 }
 
 /// Rewrite all the fixup locations with new values, following the chosen
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
index 6b4fc88cd3a7..4f2bfb073baf 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
@@ -368,7 +368,7 @@ bool LoopVersioningLICM::legalLoopInstructions() {
   IsReadOnlyLoop = true;
   using namespace ore;
   // Get LoopAccessInfo from current loop via the proxy.
-  LAI = &LAIs.getInfo(*CurLoop);
+  LAI = &LAIs.getInfo(*CurLoop, /*AllowPartial=*/true);
   // Check LoopAccessInfo for need of runtime check.
   if (LAI->getRuntimePointerChecking()->getChecks().empty()) {
     LLVM_DEBUG(dbgs() << "    LAA: Runtime check not found !!\n");
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index 756a72e6d97b..1e37f40fa9d5 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -19,9 +19,11 @@
 
 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
 #include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -32,14 +34,17 @@
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/MatrixBuilder.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
@@ -52,6 +57,10 @@ using namespace PatternMatch;
 
 #define DEBUG_TYPE "lower-matrix-intrinsics"
 
+STATISTIC(FlattenedMatrices, "Number of matrix flattenings");
+STATISTIC(ReshapedMatrices, "Number of matrix reshapes");
+STATISTIC(SplitMatrices, "Number of matrix splits");
+
 static cl::opt<bool>
     FuseMatrix("fuse-matrix", cl::init(true), cl::Hidden,
                cl::desc("Enable/disable fusing matrix instructions."));
@@ -221,7 +230,16 @@ struct ShapeInfo {
 
   /// Returns the transposed shape.
   ShapeInfo t() const { return ShapeInfo(NumColumns, NumRows); }
+
+  friend raw_ostream &operator<<(raw_ostream &OS, ShapeInfo SI);
+
+  LLVM_DUMP_METHOD void dump() const { dbgs() << *this << '\n'; }
 };
+
+raw_ostream &operator<<(raw_ostream &OS, ShapeInfo SI) {
+  return OS << SI.NumRows << 'x' << SI.NumColumns;
+}
+
 } // namespace
 
 static bool isUniformShape(Value *V) {
@@ -232,6 +250,43 @@ static bool isUniformShape(Value *V) {
   if (I->isBinaryOp())
     return true;
 
+  if (auto *Cast = dyn_cast<CastInst>(V)) {
+    switch (Cast->getOpcode()) {
+    case llvm::Instruction::Trunc:
+    case llvm::Instruction::ZExt:
+    case llvm::Instruction::SExt:
+    case llvm::Instruction::FPToUI:
+    case llvm::Instruction::FPToSI:
+    case llvm::Instruction::UIToFP:
+    case llvm::Instruction::SIToFP:
+    case llvm::Instruction::FPTrunc:
+    case llvm::Instruction::FPExt:
+      return true;
+    case llvm::Instruction::AddrSpaceCast:
+    case CastInst::PtrToInt:
+    case CastInst::IntToPtr:
+      return false;
+    case CastInst::BitCast: {
+      if (auto *SrcVTy = dyn_cast<FixedVectorType>(Cast->getSrcTy()))
+        if (auto *DestVTy = dyn_cast<FixedVectorType>(Cast->getDestTy()))
+          return SrcVTy->getNumElements() == DestVTy->getNumElements();
+      return false;
+    }
+    case llvm::Instruction::CastOpsEnd:
+      llvm_unreachable("not an actual cast op");
+    }
+    llvm_unreachable("unhandled cast opcode");
+  }
+
+  if (auto *II = dyn_cast<IntrinsicInst>(V))
+    switch (II->getIntrinsicID()) {
+    case Intrinsic::abs:
+    case Intrinsic::fabs:
+      return true;
+    default:
+      return false;
+    }
+
   switch (I->getOpcode()) {
   case Instruction::FNeg:
     return true;
@@ -269,9 +324,11 @@ computeShapeInfoForInst(Instruction *I,
       return OpShape->second;
   }
 
-  if (isUniformShape(I)) {
+  if (isUniformShape(I) || isa<SelectInst>(I)) {
+    auto Ops = I->operands();
+    auto ShapedOps = isa<SelectInst>(I) ? drop_begin(Ops) : Ops;
     // Find the first operand that has a known shape and use that.
-    for (auto &Op : I->operands()) {
+    for (auto &Op : ShapedOps) {
       auto OpShape = ShapeMap.find(Op.get());
       if (OpShape != ShapeMap.end())
         return OpShape->second;
@@ -383,25 +440,25 @@ class LowerMatrixIntrinsics {
         return Vectors.size();
       else {
         assert(Vectors.size() > 0 && "Cannot call getNumRows without columns");
-        return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
+        return getVectorTy()->getNumElements();
       }
     }
     unsigned getNumRows() const {
       if (isColumnMajor()) {
         assert(Vectors.size() > 0 && "Cannot call getNumRows without columns");
-        return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
+        return getVectorTy()->getNumElements();
       } else
         return Vectors.size();
     }
 
     void addVector(Value *V) { Vectors.push_back(V); }
-    VectorType *getColumnTy() {
+    FixedVectorType *getColumnTy() {
       assert(isColumnMajor() && "only supported for column-major matrixes");
       return getVectorTy();
     }
 
-    VectorType *getVectorTy() const {
-      return cast<VectorType>(Vectors[0]->getType());
+    FixedVectorType *getVectorTy() const {
+      return cast<FixedVectorType>(Vectors[0]->getType());
     }
 
     iterator_range<SmallVector<Value *, 8>::iterator> columns() {
@@ -457,6 +514,8 @@ class LowerMatrixIntrinsics {
       return getNumColumns();
     }
 
+    ShapeInfo shape() const { return {getNumRows(), getNumColumns()}; }
+
     /// Extract a vector of \p NumElts starting at index (\p I, \p J). If the
     /// matrix is column-major, the result vector is extracted from a column
     /// vector, otherwise from a row vector.
@@ -514,7 +573,7 @@ class LowerMatrixIntrinsics {
       : Func(F), DL(F.getDataLayout()), TTI(TTI), AM(AM) {}
 
   unsigned getNumOps(Type *VT) {
-    assert(isa<VectorType>(VT) && "Expected vector type");
+    assert(isa<FixedVectorType>(VT) && "Expected vector type");
     return getNumOps(VT->getScalarType(),
                      cast<FixedVectorType>(VT)->getNumElements());
   }
@@ -540,10 +599,8 @@ class LowerMatrixIntrinsics {
   /// into vectors.
   MatrixTy getMatrix(Value *MatrixVal, const ShapeInfo &SI,
                      IRBuilder<> &Builder) {
-    VectorType *VType = dyn_cast<VectorType>(MatrixVal->getType());
-    assert(VType && "MatrixVal must be a vector type");
-    assert(cast<FixedVectorType>(VType)->getNumElements() ==
-               SI.NumRows * SI.NumColumns &&
+    FixedVectorType *VType = cast<FixedVectorType>(MatrixVal->getType());
+    assert(VType->getNumElements() == SI.NumRows * SI.NumColumns &&
            "The vector size must match the number of matrix elements");
 
     // Check if we lowered MatrixVal using shape information. In that case,
@@ -563,8 +620,7 @@ class LowerMatrixIntrinsics {
 
     // Otherwise split MatrixVal.
     SmallVector<Value *, 16> SplitVecs;
-    for (unsigned MaskStart = 0;
-         MaskStart < cast<FixedVectorType>(VType)->getNumElements();
+    for (unsigned MaskStart = 0; MaskStart < VType->getNumElements();
          MaskStart += SI.getStride()) {
       Value *V = Builder.CreateShuffleVector(
           MatrixVal, createSequentialMask(MaskStart, SI.getStride(), 0),
@@ -572,6 +628,31 @@ class LowerMatrixIntrinsics {
       SplitVecs.push_back(V);
     }
 
+    if (Instruction *Inst = dyn_cast<Instruction>(MatrixVal)) {
+      if (Found != Inst2ColumnMatrix.end()) {
+        // FIXME: re: "at least": SplitVecs.size() doesn't count the shuffles
+        // that embedInVector created.
+        LLVM_DEBUG(dbgs() << "matrix reshape from " << Found->second.shape()
+                          << " to " << SI << " using at least "
+                          << SplitVecs.size() << " shuffles on behalf of:\n"
+                          << *Inst << '\n');
+        ReshapedMatrices++;
+      } else if (!ShapeMap.contains(MatrixVal)) {
+        LLVM_DEBUG(
+            dbgs()
+            << "splitting a " << SI << " matrix with " << SplitVecs.size()
+            << " shuffles beacuse we do not have a shape-aware lowering for "
+               "its def:\n"
+            << *Inst << '\n');
+        (void)Inst;
+        SplitMatrices++;
+      } else {
+        // The ShapeMap has it, so it's a case where we're being lowered
+        // before the def, and we expect that InstCombine will clean things up
+        // afterward.
+      }
+    }
+
     return {SplitVecs};
   }
 
@@ -621,9 +702,10 @@ class LowerMatrixIntrinsics {
       case Intrinsic::matrix_column_major_store:
         return true;
       default:
-        return false;
+        return isUniformShape(II);
       }
-    return isUniformShape(V) || isa<StoreInst>(V) || isa<LoadInst>(V);
+    return isUniformShape(V) || isa<StoreInst>(V) || isa<LoadInst>(V) ||
+           isa<SelectInst>(V);
   }
 
   /// Propagate the shape information of instructions to their users.
@@ -710,10 +792,12 @@ class LowerMatrixIntrinsics {
       } else if (isa<StoreInst>(V)) {
         // Nothing to do.  We forward-propagated to this so we would just
         // backward propagate to an instruction with an already known shape.
-      } else if (isUniformShape(V)) {
+      } else if (isUniformShape(V) || isa<SelectInst>(V)) {
+        auto Ops = cast<Instruction>(V)->operands();
+        auto ShapedOps = isa<SelectInst>(V) ? drop_begin(Ops) : Ops;
         // Propagate to all operands.
         ShapeInfo Shape = ShapeMap[V];
-        for (Use &U : cast<Instruction>(V)->operands()) {
+        for (Use &U : ShapedOps) {
           if (setShapeInfo(U.get(), Shape))
             pushInstruction(U.get(), WorkList);
         }
@@ -1057,21 +1141,31 @@ class LowerMatrixIntrinsics {
       if (FusedInsts.count(Inst))
         continue;
 
-      IRBuilder<> Builder(Inst);
-
-      if (CallInst *CInst = dyn_cast<CallInst>(Inst))
-        Changed |= VisitCallInst(CInst);
+      const ShapeInfo &SI = ShapeMap.at(Inst);
 
       Value *Op1;
       Value *Op2;
+      MatrixTy Result;
       if (auto *BinOp = dyn_cast<BinaryOperator>(Inst))
-        Changed |= VisitBinaryOperator(BinOp);
-      if (auto *UnOp = dyn_cast<UnaryOperator>(Inst))
-        Changed |= VisitUnaryOperator(UnOp);
-      if (match(Inst, m_Load(m_Value(Op1))))
-        Changed |= VisitLoad(cast<LoadInst>(Inst), Op1, Builder);
+        Result = VisitBinaryOperator(BinOp, SI);
+      else if (auto *Cast = dyn_cast<CastInst>(Inst))
+        Result = VisitCastInstruction(Cast, SI);
+      else if (auto *UnOp = dyn_cast<UnaryOperator>(Inst))
+        Result = VisitUnaryOperator(UnOp, SI);
+      else if (auto *Intr = dyn_cast<IntrinsicInst>(Inst))
+        Result = VisitIntrinsicInst(Intr, SI);
+      else if (auto *Select = dyn_cast<SelectInst>(Inst))
+        Result = VisitSelectInst(Select, SI);
+      else if (match(Inst, m_Load(m_Value(Op1))))
+        Result = VisitLoad(cast<LoadInst>(Inst), SI, Op1);
       else if (match(Inst, m_Store(m_Value(Op1), m_Value(Op2))))
-        Changed |= VisitStore(cast<StoreInst>(Inst), Op1, Op2, Builder);
+        Result = VisitStore(cast<StoreInst>(Inst), SI, Op1, Op2);
+      else
+        continue;
+
+      IRBuilder<> Builder(Inst);
+      finalizeLowering(Inst, Result, Builder);
+      Changed = true;
     }
 
     if (ORE) {
@@ -1109,28 +1203,50 @@ class LowerMatrixIntrinsics {
     return Changed;
   }
 
-  /// Replace intrinsic calls
-  bool VisitCallInst(CallInst *Inst) {
-    if (!Inst->getCalledFunction() || !Inst->getCalledFunction()->isIntrinsic())
-      return false;
+  /// Replace intrinsic calls.
+  MatrixTy VisitIntrinsicInst(IntrinsicInst *Inst, const ShapeInfo &SI) {
+    assert(Inst->getCalledFunction() &&
+           Inst->getCalledFunction()->isIntrinsic());
 
     switch (Inst->getCalledFunction()->getIntrinsicID()) {
     case Intrinsic::matrix_multiply:
-      LowerMultiply(Inst);
-      break;
+      return LowerMultiply(Inst);
     case Intrinsic::matrix_transpose:
-      LowerTranspose(Inst);
-      break;
+      return LowerTranspose(Inst);
     case Intrinsic::matrix_column_major_load:
-      LowerColumnMajorLoad(Inst);
-      break;
+      return LowerColumnMajorLoad(Inst);
     case Intrinsic::matrix_column_major_store:
-      LowerColumnMajorStore(Inst);
-      break;
+      return LowerColumnMajorStore(Inst);
+    case Intrinsic::abs:
+    case Intrinsic::fabs: {
+      IRBuilder<> Builder(Inst);
+      MatrixTy Result;
+      MatrixTy M = getMatrix(Inst->getOperand(0), SI, Builder);
+      Builder.setFastMathFlags(getFastMathFlags(Inst));
+
+      for (auto *Vector : M.vectors()) {
+        switch (Inst->getIntrinsicID()) {
+        case Intrinsic::abs:
+          Result.addVector(Builder.CreateBinaryIntrinsic(Intrinsic::abs, Vector,
+                                                         Inst->getOperand(1)));
+          continue;
+        case Intrinsic::fabs:
+          Result.addVector(
+              Builder.CreateUnaryIntrinsic(Inst->getIntrinsicID(), Vector));
+          continue;
+        default:
+          llvm_unreachable("unexpected intrinsic");
+        }
+      }
+
+      return Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
+                                     Result.getNumVectors());
+    }
     default:
-      return false;
+      break;
     }
-    return true;
+    llvm_unreachable(
+        "only intrinsics supporting shape info should be seen here");
   }
 
   /// Compute the alignment for a column/row \p Idx with \p Stride between them.
@@ -1157,7 +1273,7 @@ class LowerMatrixIntrinsics {
   /// vectors.
   MatrixTy loadMatrix(Type *Ty, Value *Ptr, MaybeAlign MAlign, Value *Stride,
                       bool IsVolatile, ShapeInfo Shape, IRBuilder<> &Builder) {
-    auto *VType = cast<VectorType>(Ty);
+    auto *VType = cast<FixedVectorType>(Ty);
     Type *EltTy = VType->getElementType();
     Type *VecTy = FixedVectorType::get(EltTy, Shape.getStride());
     Value *EltPtr = Ptr;
@@ -1196,26 +1312,24 @@ class LowerMatrixIntrinsics {
   }
 
   /// Lower a load instruction with shape information.
-  void LowerLoad(Instruction *Inst, Value *Ptr, MaybeAlign Align, Value *Stride,
-                 bool IsVolatile, ShapeInfo Shape) {
+  MatrixTy LowerLoad(Instruction *Inst, Value *Ptr, MaybeAlign Align,
+                     Value *Stride, bool IsVolatile, ShapeInfo Shape) {
     IRBuilder<> Builder(Inst);
-    finalizeLowering(Inst,
-                     loadMatrix(Inst->getType(), Ptr, Align, Stride, IsVolatile,
-                                Shape, Builder),
-                     Builder);
+    return loadMatrix(Inst->getType(), Ptr, Align, Stride, IsVolatile, Shape,
+                      Builder);
   }
 
   /// Lowers llvm.matrix.column.major.load.
   ///
   /// The intrinsic loads a matrix from memory using a stride between columns.
-  void LowerColumnMajorLoad(CallInst *Inst) {
+  MatrixTy LowerColumnMajorLoad(CallInst *Inst) {
     assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&
            "Intrinsic only supports column-major layout!");
     Value *Ptr = Inst->getArgOperand(0);
     Value *Stride = Inst->getArgOperand(1);
-    LowerLoad(Inst, Ptr, Inst->getParamAlign(0), Stride,
-              cast<ConstantInt>(Inst->getArgOperand(2))->isOne(),
-              {Inst->getArgOperand(3), Inst->getArgOperand(4)});
+    return LowerLoad(Inst, Ptr, Inst->getParamAlign(0), Stride,
+                     cast<ConstantInt>(Inst->getArgOperand(2))->isOne(),
+                     {Inst->getArgOperand(3), Inst->getArgOperand(4)});
   }
 
   /// Stores a sub-matrix \p StoreVal into the \p R x \p C matrix starting at \p
@@ -1239,7 +1353,7 @@ class LowerMatrixIntrinsics {
   MatrixTy storeMatrix(Type *Ty, MatrixTy StoreVal, Value *Ptr,
                        MaybeAlign MAlign, Value *Stride, bool IsVolatile,
                        IRBuilder<> &Builder) {
-    auto VType = cast<VectorType>(Ty);
+    auto *VType = cast<FixedVectorType>(Ty);
     Value *EltPtr = Ptr;
     for (auto Vec : enumerate(StoreVal.vectors())) {
       Value *GEP = computeVectorAddr(
@@ -1258,28 +1372,27 @@ class LowerMatrixIntrinsics {
   }
 
   /// Lower a store instruction with shape information.
-  void LowerStore(Instruction *Inst, Value *Matrix, Value *Ptr, MaybeAlign A,
-                  Value *Stride, bool IsVolatile, ShapeInfo Shape) {
+  MatrixTy LowerStore(Instruction *Inst, Value *Matrix, Value *Ptr,
+                      MaybeAlign A, Value *Stride, bool IsVolatile,
+                      ShapeInfo Shape) {
     IRBuilder<> Builder(Inst);
     auto StoreVal = getMatrix(Matrix, Shape, Builder);
-    finalizeLowering(Inst,
-                     storeMatrix(Matrix->getType(), StoreVal, Ptr, A, Stride,
-                                 IsVolatile, Builder),
-                     Builder);
+    return storeMatrix(Matrix->getType(), StoreVal, Ptr, A, Stride, IsVolatile,
+                       Builder);
   }
 
   /// Lowers llvm.matrix.column.major.store.
   ///
   /// The intrinsic store a matrix back memory using a stride between columns.
-  void LowerColumnMajorStore(CallInst *Inst) {
+  MatrixTy LowerColumnMajorStore(CallInst *Inst) {
     assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&
            "Intrinsic only supports column-major layout!");
     Value *Matrix = Inst->getArgOperand(0);
     Value *Ptr = Inst->getArgOperand(1);
     Value *Stride = Inst->getArgOperand(2);
-    LowerStore(Inst, Matrix, Ptr, Inst->getParamAlign(1), Stride,
-               cast<ConstantInt>(Inst->getArgOperand(3))->isOne(),
-               {Inst->getArgOperand(4), Inst->getArgOperand(5)});
+    return LowerStore(Inst, Matrix, Ptr, Inst->getParamAlign(1), Stride,
+                      cast<ConstantInt>(Inst->getArgOperand(3))->isOne(),
+                      {Inst->getArgOperand(4), Inst->getArgOperand(5)});
   }
 
   // Set elements I..I+NumElts-1 to Block
@@ -1351,11 +1464,21 @@ class LowerMatrixIntrinsics {
     ToRemove.push_back(Inst);
     Value *Flattened = nullptr;
     for (Use &U : llvm::make_early_inc_range(Inst->uses())) {
-      if (!ShapeMap.contains(U.getUser())) {
-        if (!Flattened)
-          Flattened = Matrix.embedInVector(Builder);
-        U.set(Flattened);
+      if (ShapeMap.contains(U.getUser()))
+        continue;
+
+      if (!Flattened) {
+        Flattened = Matrix.embedInVector(Builder);
+        LLVM_DEBUG(
+            if (Instruction *User = dyn_cast<Instruction>(U.getUser())) dbgs()
+                << "flattening a " << Matrix.shape() << " matrix:\n"
+                << *Inst
+                << "\nbecause we do not have a shape-aware lowering for its "
+                   "user:\n"
+                << *User << '\n';);
+        FlattenedMatrices++;
       }
+      U.set(Flattened);
     }
   }
 
@@ -1377,7 +1500,7 @@ class LowerMatrixIntrinsics {
     Value *LHS = MatMul->getArgOperand(0);
     Value *RHS = MatMul->getArgOperand(1);
 
-    Type *ElementType = cast<VectorType>(LHS->getType())->getElementType();
+    Type *ElementType = cast<FixedVectorType>(LHS->getType())->getElementType();
     bool IsIntVec = ElementType->isIntegerTy();
 
     // Floating point reductions require reassocation.
@@ -1475,7 +1598,7 @@ class LowerMatrixIntrinsics {
     int MulOpCode = IsIntVec ? Instruction::Mul : Instruction::FMul;
     InstructionCost ReductionCost =
         TTI.getArithmeticReductionCost(
-            AddOpCode, cast<VectorType>(LHS->getType()),
+            AddOpCode, cast<FixedVectorType>(LHS->getType()),
             IsIntVec ? std::nullopt : std::optional(FMF)) +
         TTI.getArithmeticInstrCost(MulOpCode, LHS->getType());
     InstructionCost SequentialAddCost =
@@ -1535,8 +1658,8 @@ class LowerMatrixIntrinsics {
       Result = Builder.CreateAddReduce(Mul);
     else {
       Result = Builder.CreateFAddReduce(
-          ConstantFP::get(cast<VectorType>(LHS->getType())->getElementType(),
-                          0.0),
+          ConstantFP::get(
+              cast<FixedVectorType>(LHS->getType())->getElementType(), 0.0),
           Mul);
       cast<Instruction>(Result)->setFastMathFlags(FMF);
     }
@@ -1735,7 +1858,7 @@ class LowerMatrixIntrinsics {
     const unsigned R = LShape.NumRows;
     const unsigned C = RShape.NumColumns;
     const unsigned M = LShape.NumColumns;
-    auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
+    auto *EltType = cast<FixedVectorType>(MatMul->getType())->getElementType();
 
     const unsigned VF = std::max<unsigned>(
         TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
@@ -1771,7 +1894,7 @@ class LowerMatrixIntrinsics {
 
   void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape,
                         Value *RPtr, ShapeInfo RShape, StoreInst *Store) {
-    auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
+    auto *EltType = cast<FixedVectorType>(MatMul->getType())->getElementType();
 
     // Create the main tiling loop nest.
     TileInfo TI(LShape.NumRows, RShape.NumColumns, LShape.NumColumns, TileSize);
@@ -1842,7 +1965,7 @@ class LowerMatrixIntrinsics {
     const unsigned R = LShape.NumRows;
     const unsigned C = RShape.NumColumns;
     const unsigned M = LShape.NumColumns;
-    auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
+    auto *EltType = cast<FixedVectorType>(MatMul->getType())->getElementType();
 
     Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul);
     Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul);
@@ -1914,7 +2037,8 @@ class LowerMatrixIntrinsics {
             ? match(B, m_Intrinsic<Intrinsic::matrix_transpose>(m_Value(T)))
             : match(A, m_Intrinsic<Intrinsic::matrix_transpose>(m_Value(T)))) {
       IRBuilder<> Builder(MatMul);
-      auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
+      auto *EltType =
+          cast<FixedVectorType>(MatMul->getType())->getElementType();
       ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
       ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
       const unsigned R = LShape.NumRows;
@@ -2043,9 +2167,9 @@ class LowerMatrixIntrinsics {
   }
 
   /// Lowers llvm.matrix.multiply.
-  void LowerMultiply(CallInst *MatMul) {
+  MatrixTy LowerMultiply(CallInst *MatMul) {
     IRBuilder<> Builder(MatMul);
-    auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
+    auto *EltType = cast<FixedVectorType>(MatMul->getType())->getElementType();
     ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
     ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
 
@@ -2065,15 +2189,15 @@ class LowerMatrixIntrinsics {
 
     emitMatrixMultiply(Result, Lhs, Rhs, Builder, false, false,
                        getFastMathFlags(MatMul));
-    finalizeLowering(MatMul, Result, Builder);
+    return Result;
   }
 
   /// Lowers llvm.matrix.transpose.
-  void LowerTranspose(CallInst *Inst) {
+  MatrixTy LowerTranspose(CallInst *Inst) {
     MatrixTy Result;
     IRBuilder<> Builder(Inst);
     Value *InputVal = Inst->getArgOperand(0);
-    VectorType *VectorTy = cast<VectorType>(InputVal->getType());
+    FixedVectorType *VectorTy = cast<FixedVectorType>(InputVal->getType());
     ShapeInfo ArgShape(Inst->getArgOperand(1), Inst->getArgOperand(2));
     MatrixTy InputMatrix = getMatrix(InputVal, ArgShape, Builder);
 
@@ -2099,82 +2223,55 @@ class LowerMatrixIntrinsics {
     // TODO: Improve estimate of operations needed for transposes. Currently we
     // just count the insertelement/extractelement instructions, but do not
     // account for later simplifications/combines.
-    finalizeLowering(
-        Inst,
-        Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns)
-            .addNumExposedTransposes(1),
-        Builder);
+    return Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns)
+        .addNumExposedTransposes(1);
   }
 
-  /// Lower load instructions, if shape information is available.
-  bool VisitLoad(LoadInst *Inst, Value *Ptr, IRBuilder<> &Builder) {
-    auto I = ShapeMap.find(Inst);
-    if (I == ShapeMap.end())
-      return false;
-
-    LowerLoad(Inst, Ptr, Inst->getAlign(),
-              Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
-              I->second);
-    return true;
+  /// Lower load instructions.
+  MatrixTy VisitLoad(LoadInst *Inst, const ShapeInfo &SI, Value *Ptr) {
+    IRBuilder<> Builder(Inst);
+    return LowerLoad(Inst, Ptr, Inst->getAlign(),
+                     Builder.getInt64(SI.getStride()), Inst->isVolatile(), SI);
   }
 
-  bool VisitStore(StoreInst *Inst, Value *StoredVal, Value *Ptr,
-                  IRBuilder<> &Builder) {
-    auto I = ShapeMap.find(StoredVal);
-    if (I == ShapeMap.end())
-      return false;
-
-    LowerStore(Inst, StoredVal, Ptr, Inst->getAlign(),
-               Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
-               I->second);
-    return true;
+  MatrixTy VisitStore(StoreInst *Inst, const ShapeInfo &SI, Value *StoredVal,
+                      Value *Ptr) {
+    IRBuilder<> Builder(Inst);
+    return LowerStore(Inst, StoredVal, Ptr, Inst->getAlign(),
+                      Builder.getInt64(SI.getStride()), Inst->isVolatile(), SI);
   }
 
-  /// Lower binary operators, if shape information is available.
-  bool VisitBinaryOperator(BinaryOperator *Inst) {
-    auto I = ShapeMap.find(Inst);
-    if (I == ShapeMap.end())
-      return false;
-
+  /// Lower binary operators.
+  MatrixTy VisitBinaryOperator(BinaryOperator *Inst, const ShapeInfo &SI) {
     Value *Lhs = Inst->getOperand(0);
     Value *Rhs = Inst->getOperand(1);
 
     IRBuilder<> Builder(Inst);
-    ShapeInfo &Shape = I->second;
 
     MatrixTy Result;
-    MatrixTy A = getMatrix(Lhs, Shape, Builder);
-    MatrixTy B = getMatrix(Rhs, Shape, Builder);
+    MatrixTy A = getMatrix(Lhs, SI, Builder);
+    MatrixTy B = getMatrix(Rhs, SI, Builder);
     assert(A.isColumnMajor() == B.isColumnMajor() &&
            Result.isColumnMajor() == A.isColumnMajor() &&
            "operands must agree on matrix layout");
 
     Builder.setFastMathFlags(getFastMathFlags(Inst));
 
-    for (unsigned I = 0; I < Shape.getNumVectors(); ++I)
-      Result.addVector(Builder.CreateBinOp(Inst->getOpcode(), A.getVector(I),
-                                           B.getVector(I)));
+    for (auto [AV, BV] : llvm::zip_equal(A.vectors(), B.vectors()))
+      Result.addVector(Builder.CreateBinOp(Inst->getOpcode(), AV, BV));
 
-    finalizeLowering(Inst,
-                     Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
-                                             Result.getNumVectors()),
-                     Builder);
-    return true;
+    return Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
+                                   Result.getNumVectors());
   }
 
-  /// Lower unary operators, if shape information is available.
-  bool VisitUnaryOperator(UnaryOperator *Inst) {
-    auto I = ShapeMap.find(Inst);
-    if (I == ShapeMap.end())
-      return false;
-
+  /// Lower unary operators.
+  MatrixTy VisitUnaryOperator(UnaryOperator *Inst, const ShapeInfo &SI) {
     Value *Op = Inst->getOperand(0);
 
     IRBuilder<> Builder(Inst);
-    ShapeInfo &Shape = I->second;
 
     MatrixTy Result;
-    MatrixTy M = getMatrix(Op, Shape, Builder);
+    MatrixTy M = getMatrix(Op, SI, Builder);
 
     Builder.setFastMathFlags(getFastMathFlags(Inst));
 
@@ -2188,14 +2285,61 @@ class LowerMatrixIntrinsics {
       }
     };
 
-    for (unsigned I = 0; I < Shape.getNumVectors(); ++I)
-      Result.addVector(BuildVectorOp(M.getVector(I)));
+    for (auto *Vector : M.vectors())
+      Result.addVector(BuildVectorOp(Vector));
 
-    finalizeLowering(Inst,
-                     Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
-                                             Result.getNumVectors()),
-                     Builder);
-    return true;
+    return Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
+                                   Result.getNumVectors());
+  }
+
+  /// Lower cast instructions.
+  MatrixTy VisitCastInstruction(CastInst *Inst, const ShapeInfo &Shape) {
+    Value *Op = Inst->getOperand(0);
+
+    IRBuilder<> Builder(Inst);
+
+    MatrixTy Result;
+    MatrixTy M = getMatrix(Op, Shape, Builder);
+
+    Builder.setFastMathFlags(getFastMathFlags(Inst));
+
+    auto *OrigVTy = cast<VectorType>(Inst->getType());
+    auto *NewVTy = VectorType::get(OrigVTy->getElementType(),
+                                   ElementCount::getFixed(M.getStride()));
+
+    for (auto *Vector : M.vectors())
+      Result.addVector(Builder.CreateCast(Inst->getOpcode(), Vector, NewVTy));
+
+    return Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
+                                   Result.getNumVectors());
+  }
+
+  /// Lower selects.
+  MatrixTy VisitSelectInst(SelectInst *Inst, const ShapeInfo &Shape) {
+    Value *Cond = Inst->getOperand(0);
+    Value *OpA = Inst->getOperand(1);
+    Value *OpB = Inst->getOperand(2);
+
+    IRBuilder<> Builder(Inst);
+
+    MatrixTy Result;
+    MatrixTy A = getMatrix(OpA, Shape, Builder);
+    MatrixTy B = getMatrix(OpB, Shape, Builder);
+
+    SmallVector<Value*> CondV;
+    if (isa<FixedVectorType>(Cond->getType())) {
+      MatrixTy C = getMatrix(Cond, Shape, Builder);
+      llvm::copy(C.vectors(), std::back_inserter(CondV));
+    } else {
+      CondV.resize(A.getNumVectors());
+      std::fill(CondV.begin(), CondV.end(), Cond);
+    }
+
+    for (auto [CV, AV, BV] : llvm::zip_equal(CondV, A.vectors(), B.vectors()))
+      Result.addVector(Builder.CreateSelect(CV, AV, BV));
+
+    return Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
+                                   Result.getNumVectors());
   }
 
   /// Helper to linearize a matrix expression tree into a string. Currently
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index a78e3770384a..1c4ec6aa08b4 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -1364,8 +1364,9 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
   return true;
 }
 
-/// Determine whether the instruction has undefined content for the given Size,
-/// either because it was freshly alloca'd or started its lifetime.
+/// Determine whether the pointer V had only undefined content (due to Def) up
+/// to the given Size, either because it was freshly alloca'd or started its
+/// lifetime.
 static bool hasUndefContents(MemorySSA *MSSA, BatchAAResults &AA, Value *V,
                              MemoryDef *Def, Value *Size) {
   if (MSSA->isLiveOnEntryDef(Def))
@@ -1400,6 +1401,24 @@ static bool hasUndefContents(MemorySSA *MSSA, BatchAAResults &AA, Value *V,
   return false;
 }
 
+// If the memcpy is larger than the previous, but the memory was undef prior to
+// that, we can just ignore the tail. Technically we're only interested in the
+// bytes from 0..MemSrcOffset and MemSrcLength+MemSrcOffset..CopySize here, but
+// as we can't easily represent this location (hasUndefContents uses mustAlias
+// which cannot deal with offsets), we use the full 0..CopySize range.
+static bool overreadUndefContents(MemorySSA *MSSA, MemCpyInst *MemCpy,
+                                  MemIntrinsic *MemSrc, BatchAAResults &BAA) {
+  Value *CopySize = MemCpy->getLength();
+  MemoryLocation MemCpyLoc = MemoryLocation::getForSource(MemCpy);
+  MemoryUseOrDef *MemSrcAccess = MSSA->getMemoryAccess(MemSrc);
+  MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
+      MemSrcAccess->getDefiningAccess(), MemCpyLoc, BAA);
+  if (auto *MD = dyn_cast<MemoryDef>(Clobber))
+    if (hasUndefContents(MSSA, BAA, MemCpy->getSource(), MD, CopySize))
+      return true;
+  return false;
+}
+
 /// Transform memcpy to memset when its source was just memset.
 /// In other words, turn:
 /// \code
@@ -1415,44 +1434,45 @@ static bool hasUndefContents(MemorySSA *MSSA, BatchAAResults &AA, Value *V,
 bool MemCpyOptPass::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
                                                MemSetInst *MemSet,
                                                BatchAAResults &BAA) {
-  // Make sure that memcpy(..., memset(...), ...), that is we are memsetting and
-  // memcpying from the same address. Otherwise it is hard to reason about.
-  if (!BAA.isMustAlias(MemSet->getRawDest(), MemCpy->getRawSource()))
-    return false;
-
   Value *MemSetSize = MemSet->getLength();
   Value *CopySize = MemCpy->getLength();
 
-  if (MemSetSize != CopySize) {
-    // Make sure the memcpy doesn't read any more than what the memset wrote.
-    // Don't worry about sizes larger than i64.
-
-    // A known memset size is required.
-    auto *CMemSetSize = dyn_cast<ConstantInt>(MemSetSize);
-    if (!CMemSetSize)
+  int64_t MOffset = 0;
+  const DataLayout &DL = MemCpy->getModule()->getDataLayout();
+  // We can only transforms memcpy's where the dest of one is the source of the
+  // other, or they have a known offset.
+  if (MemCpy->getSource() != MemSet->getDest()) {
+    std::optional<int64_t> Offset =
+        MemCpy->getSource()->getPointerOffsetFrom(MemSet->getDest(), DL);
+    if (!Offset || *Offset < 0)
       return false;
+    MOffset = *Offset;
+  }
 
-    // A known memcpy size is also required.
+  if (MOffset != 0 || MemSetSize != CopySize) {
+    // Make sure the memcpy doesn't read any more than what the memset wrote,
+    // other than undef. Don't worry about sizes larger than i64.
+    auto *CMemSetSize = dyn_cast<ConstantInt>(MemSetSize);
     auto *CCopySize = dyn_cast<ConstantInt>(CopySize);
-    if (!CCopySize)
-      return false;
-    if (CCopySize->getZExtValue() > CMemSetSize->getZExtValue()) {
-      // If the memcpy is larger than the memset, but the memory was undef prior
-      // to the memset, we can just ignore the tail. Technically we're only
-      // interested in the bytes from MemSetSize..CopySize here, but as we can't
-      // easily represent this location, we use the full 0..CopySize range.
-      MemoryLocation MemCpyLoc = MemoryLocation::getForSource(MemCpy);
-      bool CanReduceSize = false;
-      MemoryUseOrDef *MemSetAccess = MSSA->getMemoryAccess(MemSet);
-      MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
-          MemSetAccess->getDefiningAccess(), MemCpyLoc, BAA);
-      if (auto *MD = dyn_cast<MemoryDef>(Clobber))
-        if (hasUndefContents(MSSA, BAA, MemCpy->getSource(), MD, CopySize))
-          CanReduceSize = true;
-
-      if (!CanReduceSize)
+    if (!CMemSetSize || !CCopySize ||
+        CCopySize->getZExtValue() + MOffset > CMemSetSize->getZExtValue()) {
+      if (!overreadUndefContents(MSSA, MemCpy, MemSet, BAA))
         return false;
-      CopySize = MemSetSize;
+
+      if (CMemSetSize && CCopySize) {
+        // If both have constant sizes and offsets, clip the memcpy to the
+        // bounds of the memset if applicable.
+        assert(CCopySize->getZExtValue() + MOffset >
+               CMemSetSize->getZExtValue());
+        if (MOffset == 0)
+          CopySize = MemSetSize;
+        else
+          CopySize =
+              ConstantInt::get(CopySize->getType(),
+                               CMemSetSize->getZExtValue() <= (uint64_t)MOffset
+                                   ? 0
+                                   : CMemSetSize->getZExtValue() - MOffset);
+      }
     }
   }
 
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp
index a6c93e416121..b6e7f4aebeb8 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -1120,8 +1120,13 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
       return PI.setEscapedReadOnly(&LI);
 
     TypeSize Size = DL.getTypeStoreSize(LI.getType());
-    if (Size.isScalable())
-      return PI.setAborted(&LI);
+    if (Size.isScalable()) {
+      unsigned VScale = LI.getFunction()->getVScaleValue();
+      if (!VScale)
+        return PI.setAborted(&LI);
+
+      Size = TypeSize::getFixed(Size.getKnownMinValue() * VScale);
+    }
 
     return handleLoadOrStore(LI.getType(), LI, Offset, Size.getFixedValue(),
                              LI.isVolatile());
@@ -1135,8 +1140,13 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
       return PI.setAborted(&SI);
 
     TypeSize StoreSize = DL.getTypeStoreSize(ValOp->getType());
-    if (StoreSize.isScalable())
-      return PI.setAborted(&SI);
+    if (StoreSize.isScalable()) {
+      unsigned VScale = SI.getFunction()->getVScaleValue();
+      if (!VScale)
+        return PI.setAborted(&SI);
+
+      StoreSize = TypeSize::getFixed(StoreSize.getKnownMinValue() * VScale);
+    }
 
     uint64_t Size = StoreSize.getFixedValue();
 
@@ -1927,7 +1937,8 @@ static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
 /// ensure that we only try to convert viable values. The strategy is that we
 /// will peel off single element struct and array wrappings to get to an
 /// underlying value, and convert that value.
-static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
+static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy,
+                            unsigned VScale = 0) {
   if (OldTy == NewTy)
     return true;
 
@@ -1941,8 +1952,35 @@ static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
     return false;
   }
 
-  if (DL.getTypeSizeInBits(NewTy).getFixedValue() !=
-      DL.getTypeSizeInBits(OldTy).getFixedValue())
+  TypeSize NewSize = DL.getTypeSizeInBits(NewTy);
+  TypeSize OldSize = DL.getTypeSizeInBits(OldTy);
+
+  if ((isa<ScalableVectorType>(NewTy) && isa<FixedVectorType>(OldTy)) ||
+      (isa<ScalableVectorType>(OldTy) && isa<FixedVectorType>(NewTy))) {
+    // Conversion is only possible when the size of scalable vectors is known.
+    if (!VScale)
+      return false;
+
+    // For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
+    // a single domain (either fixed or scalable). Any additional conversion
+    // between fixed and scalable types is handled through integer types.
+    auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
+    auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
+
+    if (isa<ScalableVectorType>(NewTy)) {
+      if (!VectorType::getWithSizeAndScalar(cast<VectorType>(NewVTy), OldVTy))
+        return false;
+
+      NewSize = TypeSize::getFixed(NewSize.getKnownMinValue() * VScale);
+    } else {
+      if (!VectorType::getWithSizeAndScalar(cast<VectorType>(OldVTy), NewVTy))
+        return false;
+
+      OldSize = TypeSize::getFixed(OldSize.getKnownMinValue() * VScale);
+    }
+  }
+
+  if (NewSize != OldSize)
     return false;
   if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
     return false;
@@ -1992,7 +2030,14 @@ static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
 static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
                            Type *NewTy) {
   Type *OldTy = V->getType();
-  assert(canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type");
+
+#ifndef NDEBUG
+  BasicBlock *BB = IRB.GetInsertBlock();
+  assert(BB && BB->getParent() && "VScale unknown!");
+  unsigned VScale = BB->getParent()->getVScaleValue();
+  assert(canConvertValue(DL, OldTy, NewTy, VScale) &&
+         "Value not convertable to type");
+#endif
 
   if (OldTy == NewTy)
     return V;
@@ -2000,13 +2045,41 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
   assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
          "Integer types must be the exact same to convert.");
 
+  // A variant of bitcast that supports a mixture of fixed and scalable types
+  // that are know to have the same size.
+  auto CreateBitCastLike = [&IRB](Value *In, Type *Ty) -> Value * {
+    Type *InTy = In->getType();
+    if (InTy == Ty)
+      return In;
+
+    if (isa<FixedVectorType>(InTy) && isa<ScalableVectorType>(Ty)) {
+      // For vscale_range(2) expand <4 x i32> to <vscale x 4 x i16> -->
+      //   <4 x i32> to <vscale x 2 x i32> to <vscale x 4 x i16>
+      auto *VTy = VectorType::getWithSizeAndScalar(cast<VectorType>(Ty), InTy);
+      return IRB.CreateBitCast(IRB.CreateInsertVector(VTy,
+                                                      PoisonValue::get(VTy), In,
+                                                      IRB.getInt64(0)),
+                               Ty);
+    }
+
+    if (isa<ScalableVectorType>(InTy) && isa<FixedVectorType>(Ty)) {
+      // For vscale_range(2) expand <vscale x 4 x i16> to <4 x i32> -->
+      //   <vscale x 4 x i16> to <vscale x 2 x i32> to <4 x i32>
+      auto *VTy = VectorType::getWithSizeAndScalar(cast<VectorType>(InTy), Ty);
+      return IRB.CreateExtractVector(Ty, IRB.CreateBitCast(In, VTy),
+                                     IRB.getInt64(0));
+    }
+
+    return IRB.CreateBitCast(In, Ty);
+  };
+
   // See if we need inttoptr for this type pair. May require additional bitcast.
   if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
     // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
     // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
     // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
     // Directly handle i64 to i8*
-    return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)),
+    return IRB.CreateIntToPtr(CreateBitCastLike(V, DL.getIntPtrType(NewTy)),
                               NewTy);
   }
 
@@ -2016,7 +2089,7 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
     // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
     // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
     // Expand i8* to i64 --> i8* to i64 to i64
-    return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
+    return CreateBitCastLike(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
                              NewTy);
   }
 
@@ -2031,12 +2104,14 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
     // size.
     if (OldAS != NewAS) {
       assert(DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS));
-      return IRB.CreateIntToPtr(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
-                                NewTy);
+      return IRB.CreateIntToPtr(
+          CreateBitCastLike(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
+                            DL.getIntPtrType(NewTy)),
+          NewTy);
     }
   }
 
-  return IRB.CreateBitCast(V, NewTy);
+  return CreateBitCastLike(V, NewTy);
 }
 
 /// Test whether the given slice use can be promoted to a vector.
@@ -2046,7 +2121,8 @@ static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
 static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
                                             VectorType *Ty,
                                             uint64_t ElementSize,
-                                            const DataLayout &DL) {
+                                            const DataLayout &DL,
+                                            unsigned VScale) {
   // First validate the slice offsets.
   uint64_t BeginOffset =
       std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset();
@@ -2090,7 +2166,7 @@ static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
       assert(LTy->isIntegerTy());
       LTy = SplitIntTy;
     }
-    if (!canConvertValue(DL, SliceTy, LTy))
+    if (!canConvertValue(DL, SliceTy, LTy, VScale))
       return false;
   } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
     if (SI->isVolatile())
@@ -2103,7 +2179,7 @@ static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
       assert(STy->isIntegerTy());
       STy = SplitIntTy;
     }
-    if (!canConvertValue(DL, STy, SliceTy))
+    if (!canConvertValue(DL, STy, SliceTy, VScale))
       return false;
   } else {
     return false;
@@ -2118,7 +2194,7 @@ static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
 /// (and thus isVectorPromotionViable) over all slices of the alloca for the
 /// given VectorType.
 static bool checkVectorTypeForPromotion(Partition &P, VectorType *VTy,
-                                        const DataLayout &DL) {
+                                        const DataLayout &DL, unsigned VScale) {
   uint64_t ElementSize =
       DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue();
 
@@ -2131,11 +2207,11 @@ static bool checkVectorTypeForPromotion(Partition &P, VectorType *VTy,
   ElementSize /= 8;
 
   for (const Slice &S : P)
-    if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL))
+    if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL, VScale))
       return false;
 
   for (const Slice *S : P.splitSliceTails())
-    if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL))
+    if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL, VScale))
       return false;
 
   return true;
@@ -2150,7 +2226,7 @@ checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
                              SmallVectorImpl<VectorType *> &CandidateTys,
                              bool HaveCommonEltTy, Type *CommonEltTy,
                              bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
-                             VectorType *CommonVecPtrTy) {
+                             VectorType *CommonVecPtrTy, unsigned VScale) {
   // If we didn't find a vector type, nothing to do here.
   if (CandidateTys.empty())
     return nullptr;
@@ -2226,7 +2302,7 @@ checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
   });
 
   for (VectorType *VTy : CandidateTys)
-    if (checkVectorTypeForPromotion(P, VTy, DL))
+    if (checkVectorTypeForPromotion(P, VTy, DL, VScale))
       return VTy;
 
   return nullptr;
@@ -2237,7 +2313,7 @@ static VectorType *createAndCheckVectorTypesForPromotion(
     function_ref<void(Type *)> CheckCandidateType, Partition &P,
     const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
     bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy,
-    bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy) {
+    bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale) {
   [[maybe_unused]] VectorType *OriginalElt =
       CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr;
   // Consider additional vector types where the element type size is a
@@ -2262,9 +2338,9 @@ static VectorType *createAndCheckVectorTypesForPromotion(
     }
   }
 
-  return checkVectorTypesForPromotion(P, DL, CandidateTys, HaveCommonEltTy,
-                                      CommonEltTy, HaveVecPtrTy,
-                                      HaveCommonVecPtrTy, CommonVecPtrTy);
+  return checkVectorTypesForPromotion(
+      P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
+      HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
 }
 
 /// Test whether the given alloca partitioning and range of slices can be
@@ -2276,7 +2352,8 @@ static VectorType *createAndCheckVectorTypesForPromotion(
 /// SSA value. We only can ensure this for a limited set of operations, and we
 /// don't want to do the rewrites unless we are confident that the result will
 /// be promotable, so we have an early test here.
-static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
+static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL,
+                                           unsigned VScale) {
   // Collect the candidate types for vector-based promotion. Also track whether
   // we have different element types.
   SmallVector<VectorType *, 4> CandidateTys;
@@ -2288,7 +2365,7 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
   bool HaveCommonEltTy = true;
   bool HaveCommonVecPtrTy = true;
   auto CheckCandidateType = [&](Type *Ty) {
-    if (auto *VTy = dyn_cast<VectorType>(Ty)) {
+    if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
       // Return if bitcast to vectors is different for total size in bits.
       if (!CandidateTys.empty()) {
         VectorType *V = CandidateTys[0];
@@ -2343,14 +2420,14 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
   if (auto *VTy = createAndCheckVectorTypesForPromotion(
           LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
           CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
-          HaveCommonVecPtrTy, CommonVecPtrTy))
+          HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
     return VTy;
 
   CandidateTys.clear();
   return createAndCheckVectorTypesForPromotion(
       DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
       HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
-      CommonVecPtrTy);
+      CommonVecPtrTy, VScale);
 }
 
 /// Test whether a slice of an alloca is valid for integer widening.
@@ -2387,7 +2464,8 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (LI->isVolatile())
       return false;
     // We can't handle loads that extend past the allocated memory.
-    if (DL.getTypeStoreSize(LI->getType()).getFixedValue() > Size)
+    TypeSize LoadSize = DL.getTypeStoreSize(LI->getType());
+    if (!LoadSize.isFixed() || LoadSize.getFixedValue() > Size)
       return false;
     // So far, AllocaSliceRewriter does not support widening split slice tails
     // in rewriteIntegerLoad.
@@ -2412,7 +2490,8 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (SI->isVolatile())
       return false;
     // We can't handle stores that extend past the allocated memory.
-    if (DL.getTypeStoreSize(ValueTy).getFixedValue() > Size)
+    TypeSize StoreSize = DL.getTypeStoreSize(ValueTy);
+    if (!StoreSize.isFixed() || StoreSize.getFixedValue() > Size)
       return false;
     // So far, AllocaSliceRewriter does not support widening split slice tails
     // in rewriteIntegerStore.
@@ -2885,8 +2964,6 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
 
     Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
                              : LI.getType();
-    const bool IsLoadPastEnd =
-        DL.getTypeStoreSize(TargetTy).getFixedValue() > SliceSize;
     bool IsPtrAdjusted = false;
     Value *V;
     if (VecTy) {
@@ -2896,8 +2973,9 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     } else if (NewBeginOffset == NewAllocaBeginOffset &&
                NewEndOffset == NewAllocaEndOffset &&
                (canConvertValue(DL, NewAllocaTy, TargetTy) ||
-                (IsLoadPastEnd && NewAllocaTy->isIntegerTy() &&
-                 TargetTy->isIntegerTy() && !LI.isVolatile()))) {
+                (NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
+                 DL.getTypeStoreSize(TargetTy).getFixedValue() > SliceSize &&
+                 !LI.isVolatile()))) {
       Value *NewPtr =
           getPtrToNewAI(LI.getPointerAddressSpace(), LI.isVolatile());
       LoadInst *NewLI = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), NewPtr,
@@ -3070,7 +3148,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets()))
         Pass.PostPromotionWorklist.insert(AI);
 
-    if (SliceSize < DL.getTypeStoreSize(V->getType()).getFixedValue()) {
+    TypeSize StoreSize = DL.getTypeStoreSize(V->getType());
+    if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
       assert(!SI.isVolatile());
       assert(V->getType()->isIntegerTy() &&
              "Only integer type loads and stores are split");
@@ -4846,14 +4925,18 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
   Type *SliceTy = nullptr;
   VectorType *SliceVecTy = nullptr;
   const DataLayout &DL = AI.getDataLayout();
+  unsigned VScale = AI.getFunction()->getVScaleValue();
+
   std::pair<Type *, IntegerType *> CommonUseTy =
       findCommonType(P.begin(), P.end(), P.endOffset());
   // Do all uses operate on the same type?
-  if (CommonUseTy.first)
-    if (DL.getTypeAllocSize(CommonUseTy.first).getFixedValue() >= P.size()) {
+  if (CommonUseTy.first) {
+    TypeSize CommonUseSize = DL.getTypeAllocSize(CommonUseTy.first);
+    if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
       SliceTy = CommonUseTy.first;
       SliceVecTy = dyn_cast<VectorType>(SliceTy);
     }
+  }
   // If not, can we find an appropriate subtype in the original allocated type?
   if (!SliceTy)
     if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(),
@@ -4874,12 +4957,12 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
 
   // If the common use types are not viable for promotion then attempt to find
   // another type that is viable.
-  if (SliceVecTy && !checkVectorTypeForPromotion(P, SliceVecTy, DL))
+  if (SliceVecTy && !checkVectorTypeForPromotion(P, SliceVecTy, DL, VScale))
     if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(),
                                                  P.beginOffset(), P.size())) {
       VectorType *TypePartitionVecTy = dyn_cast<VectorType>(TypePartitionTy);
       if (TypePartitionVecTy &&
-          checkVectorTypeForPromotion(P, TypePartitionVecTy, DL))
+          checkVectorTypeForPromotion(P, TypePartitionVecTy, DL, VScale))
         SliceTy = TypePartitionTy;
     }
 
@@ -4890,7 +4973,7 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
   bool IsIntegerPromotable = isIntegerWideningViable(P, SliceTy, DL);
 
   VectorType *VecTy =
-      IsIntegerPromotable ? nullptr : isVectorPromotionViable(P, DL);
+      IsIntegerPromotable ? nullptr : isVectorPromotionViable(P, DL, VScale);
   if (VecTy)
     SliceTy = VecTy;
 
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index e24088c29498..42d6680c3cb7 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -968,6 +968,29 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
 
   // FIXME: Do we need to add an alignment parameter to the intrinsic?
   unsigned VectorWidth = AddrType->getNumElements();
+  auto CreateHistogramUpdateValue = [&](IntrinsicInst *CI, Value *Load,
+                                        Value *Inc) -> Value * {
+    Value *UpdateOp;
+    switch (CI->getIntrinsicID()) {
+    case Intrinsic::experimental_vector_histogram_add:
+      UpdateOp = Builder.CreateAdd(Load, Inc);
+      break;
+    case Intrinsic::experimental_vector_histogram_uadd_sat:
+      UpdateOp =
+          Builder.CreateIntrinsic(Intrinsic::uadd_sat, {EltTy}, {Load, Inc});
+      break;
+    case Intrinsic::experimental_vector_histogram_umin:
+      UpdateOp = Builder.CreateIntrinsic(Intrinsic::umin, {EltTy}, {Load, Inc});
+      break;
+    case Intrinsic::experimental_vector_histogram_umax:
+      UpdateOp = Builder.CreateIntrinsic(Intrinsic::umax, {EltTy}, {Load, Inc});
+      break;
+
+    default:
+      llvm_unreachable("Unexpected histogram intrinsic");
+    }
+    return UpdateOp;
+  };
 
   // Shorten the way if the mask is a vector of constants.
   if (isConstantIntVector(Mask)) {
@@ -976,8 +999,9 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
         continue;
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
       LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
-      Value *Add = Builder.CreateAdd(Load, Inc);
-      Builder.CreateStore(Add, Ptr);
+      Value *Update =
+          CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
+      Builder.CreateStore(Update, Ptr);
     }
     CI->eraseFromParent();
     return;
@@ -997,8 +1021,9 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
     Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
-    Value *Add = Builder.CreateAdd(Load, Inc);
-    Builder.CreateStore(Add, Ptr);
+    Value *UpdateOp =
+        CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
+    Builder.CreateStore(UpdateOp, Ptr);
 
     // Create "else" block, fill it in the next iteration
     BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -1089,6 +1114,9 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
     default:
       break;
     case Intrinsic::experimental_vector_histogram_add:
+    case Intrinsic::experimental_vector_histogram_uadd_sat:
+    case Intrinsic::experimental_vector_histogram_umin:
+    case Intrinsic::experimental_vector_histogram_umax:
       if (TTI.isLegalMaskedVectorHistogram(CI->getArgOperand(0)->getType(),
                                            CI->getArgOperand(1)->getType()))
         return false;
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 0bf90036b8b8..9b40fc03da6b 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -274,6 +274,7 @@ static void buildPartialUnswitchConditionalBranch(
     BasicBlock &UnswitchedSucc, BasicBlock &NormalSucc, bool InsertFreeze,
     const Instruction *I, AssumptionCache *AC, const DominatorTree &DT) {
   IRBuilder<> IRB(&BB);
+  IRB.SetCurrentDebugLocation(DebugLoc::getCompilerGenerated());
 
   SmallVector<Value *> FrozenInvariants;
   for (Value *Inv : Invariants) {
@@ -330,6 +331,7 @@ static void buildPartialInvariantUnswitchConditionalBranch(
   }
 
   IRBuilder<> IRB(&BB);
+  IRB.SetCurrentDebugLocation(DebugLoc::getCompilerGenerated());
   Value *Cond = VMap[ToDuplicate[0]];
   IRB.CreateCondBr(Cond, Direction ? &UnswitchedSucc : &NormalSucc,
                    Direction ? &NormalSucc : &UnswitchedSucc);
@@ -2369,6 +2371,7 @@ static void unswitchNontrivialInvariants(
         // BI (`dyn_cast<BranchInst>(TI)`) is an in-loop instruction hoisted
         // out of the loop.
         Cond = new FreezeInst(Cond, Cond->getName() + ".fr", BI->getIterator());
+        cast<Instruction>(Cond)->setDebugLoc(DebugLoc::getDropped());
       }
       BI->setCondition(Cond);
       DTUpdates.push_back({DominatorTree::Insert, SplitBB, ClonedPH});
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index 4e437e9abeb4..d20378ece4ee 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -128,7 +128,7 @@ performBlockTailMerging(Function &F, ArrayRef<BasicBlock *> BBs,
 
   // Now, go through each block (with the current terminator type)
   // we've recorded, and rewrite it to branch to the new common block.
-  DILocation *CommonDebugLoc = nullptr;
+  DebugLoc CommonDebugLoc;
   for (BasicBlock *BB : BBs) {
     auto *Term = BB->getTerminator();
     assert(Term->getOpcode() == CanonicalTerm->getOpcode() &&
@@ -145,7 +145,7 @@ performBlockTailMerging(Function &F, ArrayRef<BasicBlock *> BBs,
       CommonDebugLoc = Term->getDebugLoc();
     else
       CommonDebugLoc =
-          DILocation::getMergedLocation(CommonDebugLoc, Term->getDebugLoc());
+          DebugLoc::getMergedLocation(CommonDebugLoc, Term->getDebugLoc());
 
     // And turn BB into a block that just unconditionally branches
     // to the canonical block.
diff --git a/external/llvm-project/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/external/llvm-project/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
index 7dd6c60370ed..c71c5a70a12f 100644
--- a/external/llvm-project/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
@@ -515,7 +515,8 @@ void TailRecursionEliminator::createTailRecurseLoopHeader(CallInst *CI) {
   BasicBlock *NewEntry = BasicBlock::Create(F.getContext(), "", &F, HeaderBB);
   NewEntry->takeName(HeaderBB);
   HeaderBB->setName("tailrecurse");
-  BranchInst::Create(HeaderBB, NewEntry);
+  auto *BI = BranchInst::Create(HeaderBB, NewEntry);
+  BI->setDebugLoc(DebugLoc::getCompilerGenerated());
   // If the new branch preserves the debug location of CI, it could result in
   // misleading stepping, if CI is located in a conditional branch.
   // So, here we don't give any debug location to the new branch.
@@ -801,6 +802,7 @@ void TailRecursionEliminator::cleanupAndFinalize() {
         SelectInst *SI =
             SelectInst::Create(RetKnownPN, RetPN, RI->getOperand(0),
                                "current.ret.tr", RI->getIterator());
+        SI->setDebugLoc(DebugLoc::getCompilerGenerated());
         RetSelects.push_back(SI);
         RI->setOperand(0, SI);
       }
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index 7210fee9c7b7..4aebeeda7f26 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -20,13 +20,14 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DebugCounter.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 using namespace llvm;
 
 namespace llvm {
-cl::opt<bool> ShouldPreserveAllAttributes(
+LLVM_ABI cl::opt<bool> ShouldPreserveAllAttributes(
     "assume-preserve-all", cl::init(false), cl::Hidden,
     cl::desc("enable preservation of all attributes. even those that are "
              "unlikely to be useful"));
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 6608515e1cbb..1feed14b4fed 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -437,45 +437,7 @@ DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
 }
 
 static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
-  if (BB->IsNewDbgInfoFormat)
-    return DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BB);
-
-  SmallVector<DbgValueInst *, 8> ToBeRemoved;
-  SmallDenseSet<DebugVariable> VariableSet;
-  for (auto &I : reverse(*BB)) {
-    if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
-      DebugVariable Key(DVI->getVariable(),
-                        DVI->getExpression(),
-                        DVI->getDebugLoc()->getInlinedAt());
-      auto R = VariableSet.insert(Key);
-      // If the variable fragment hasn't been seen before then we don't want
-      // to remove this dbg intrinsic.
-      if (R.second)
-        continue;
-
-      if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI)) {
-        // Don't delete dbg.assign intrinsics that are linked to instructions.
-        if (!at::getAssignmentInsts(DAI).empty())
-          continue;
-        // Unlinked dbg.assign intrinsics can be treated like dbg.values.
-      }
-
-      // If the same variable fragment is described more than once it is enough
-      // to keep the last one (i.e. the first found since we for reverse
-      // iteration).
-      ToBeRemoved.push_back(DVI);
-      continue;
-    }
-    // Sequence with consecutive dbg.value instrs ended. Clear the map to
-    // restart identifying redundant instructions if case we find another
-    // dbg.value sequence.
-    VariableSet.clear();
-  }
-
-  for (auto &Instr : ToBeRemoved)
-    Instr->eraseFromParent();
-
-  return !ToBeRemoved.empty();
+  return DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BB);
 }
 
 /// Remove redundant dbg.value instructions using a forward scan. This can
@@ -578,49 +540,7 @@ DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
 }
 
 static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
-  if (BB->IsNewDbgInfoFormat)
-    return DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BB);
-
-  SmallVector<DbgValueInst *, 8> ToBeRemoved;
-  SmallDenseMap<DebugVariable,
-                std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
-      VariableMap;
-  for (auto &I : *BB) {
-    if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
-      DebugVariable Key(DVI->getVariable(), std::nullopt,
-                        DVI->getDebugLoc()->getInlinedAt());
-      auto [VMI, Inserted] = VariableMap.try_emplace(Key);
-      auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
-      // A dbg.assign with no linked instructions can be treated like a
-      // dbg.value (i.e. can be deleted).
-      bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
-
-      // Update the map if we found a new value/expression describing the
-      // variable, or if the variable wasn't mapped already.
-      SmallVector<Value *, 4> Values(DVI->getValues());
-      if (Inserted || VMI->second.first != Values ||
-          VMI->second.second != DVI->getExpression()) {
-        // Use a sentinel value (nullptr) for the DIExpression when we see a
-        // linked dbg.assign so that the next debug intrinsic will never match
-        // it (i.e. always treat linked dbg.assigns as if they're unique).
-        if (IsDbgValueKind)
-          VMI->second = {Values, DVI->getExpression()};
-        else
-          VMI->second = {Values, nullptr};
-        continue;
-      }
-
-      // Don't delete dbg.assign intrinsics that are linked to instructions.
-      if (!IsDbgValueKind)
-        continue;
-      ToBeRemoved.push_back(DVI);
-    }
-  }
-
-  for (auto &Instr : ToBeRemoved)
-    Instr->eraseFromParent();
-
-  return !ToBeRemoved.empty();
+  return DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BB);
 }
 
 /// Remove redundant undef dbg.assign intrinsic from an entry block using a
@@ -643,41 +563,7 @@ static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
 /// Possible improvements:
 /// - Keep track of non-overlapping fragments.
 static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
-  if (BB->IsNewDbgInfoFormat)
-    return DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BB);
-
-  assert(BB->isEntryBlock() && "expected entry block");
-  SmallVector<DbgAssignIntrinsic *, 8> ToBeRemoved;
-  DenseSet<DebugVariable> SeenDefForAggregate;
-  // Returns the DebugVariable for DVI with no fragment info.
-  auto GetAggregateVariable = [](DbgValueInst *DVI) {
-    return DebugVariable(DVI->getVariable(), std::nullopt,
-                         DVI->getDebugLoc()->getInlinedAt());
-  };
-
-  // Remove undef dbg.assign intrinsics that are encountered before
-  // any non-undef intrinsics from the entry block.
-  for (auto &I : *BB) {
-    DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I);
-    if (!DVI)
-      continue;
-    auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
-    bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
-    DebugVariable Aggregate = GetAggregateVariable(DVI);
-    if (!SeenDefForAggregate.contains(Aggregate)) {
-      bool IsKill = DVI->isKillLocation() && IsDbgValueKind;
-      if (!IsKill) {
-        SeenDefForAggregate.insert(Aggregate);
-      } else if (DAI) {
-        ToBeRemoved.push_back(DAI);
-      }
-    }
-  }
-
-  for (DbgAssignIntrinsic *DAI : ToBeRemoved)
-    DAI->eraseFromParent();
-
-  return !ToBeRemoved.empty();
+  return DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BB);
 }
 
 bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 5487dbef8a43..fccb73a36b18 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -114,7 +114,6 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
                                   const Twine &NameSuffix, Function *F,
                                   ClonedCodeInfo *CodeInfo, bool MapAtoms) {
   BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "", F);
-  NewBB->IsNewDbgInfoFormat = BB->IsNewDbgInfoFormat;
   if (BB->hasName())
     NewBB->setName(BB->getName() + NameSuffix);
 
@@ -286,7 +285,6 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
                              const char *NameSuffix, ClonedCodeInfo *CodeInfo,
                              ValueMapTypeRemapper *TypeMapper,
                              ValueMaterializer *Materializer) {
-  NewFunc->setIsNewDbgInfoFormat(OldFunc->IsNewDbgInfoFormat);
   assert(NameSuffix && "NameSuffix cannot be null!");
 
 #ifndef NDEBUG
@@ -391,7 +389,6 @@ Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,
   // Create the new function...
   Function *NewF = Function::Create(FTy, F->getLinkage(), F->getAddressSpace(),
                                     F->getName(), F->getParent());
-  NewF->setIsNewDbgInfoFormat(F->IsNewDbgInfoFormat);
 
   // Loop over the arguments, copying the names of the mapped arguments over...
   Function::arg_iterator DestI = NewF->arg_begin();
@@ -525,7 +522,6 @@ void PruningFunctionCloner::CloneBlock(
   BasicBlock *NewBB;
   Twine NewName(BB->hasName() ? Twine(BB->getName()) + NameSuffix : "");
   BBEntry = NewBB = BasicBlock::Create(BB->getContext(), NewName, NewFunc);
-  NewBB->IsNewDbgInfoFormat = BB->IsNewDbgInfoFormat;
 
   // It is only legal to clone a function if a block address within that
   // function is never referenced outside of the function.  Given that, we
@@ -548,10 +544,7 @@ void PruningFunctionCloner::CloneBlock(
   // Keep a cursor pointing at the last place we cloned debug-info records from.
   BasicBlock::const_iterator DbgCursor = StartingInst;
   auto CloneDbgRecordsToHere =
-      [NewBB, &DbgCursor](Instruction *NewInst, BasicBlock::const_iterator II) {
-        if (!NewBB->IsNewDbgInfoFormat)
-          return;
-
+      [&DbgCursor](Instruction *NewInst, BasicBlock::const_iterator II) {
         // Clone debug-info records onto this instruction. Iterate through any
         // source-instructions we've cloned and then subsequently optimised
         // away, so that their debug-info doesn't go missing.
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/CloneModule.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/CloneModule.cpp
index 88e2bfe45d2c..55fb0acd39ea 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/CloneModule.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/CloneModule.cpp
@@ -61,7 +61,6 @@ std::unique_ptr<Module> llvm::CloneModule(
   New->setDataLayout(M.getDataLayout());
   New->setTargetTriple(M.getTargetTriple());
   New->setModuleInlineAsm(M.getModuleInlineAsm());
-  New->IsNewDbgInfoFormat = M.IsNewDbgInfoFormat;
 
   // Loop over all of the global variables, making corresponding globals in the
   // new module.  Here we add them to the VMap and to the new Module.  We
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 7475a2ec2303..79c7245fcf91 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -792,7 +792,6 @@ void CodeExtractor::severSplitPHINodesOfExits() {
         NewBB = BasicBlock::Create(ExitBB->getContext(),
                                    ExitBB->getName() + ".split",
                                    ExitBB->getParent(), ExitBB);
-        NewBB->IsNewDbgInfoFormat = ExitBB->IsNewDbgInfoFormat;
         SmallVector<BasicBlock *, 4> Preds(predecessors(ExitBB));
         for (BasicBlock *PredBB : Preds)
           if (Blocks.count(PredBB))
@@ -1561,7 +1560,6 @@ CodeExtractor::extractCodeRegion(const CodeExtractorAnalysisCache &CEAC,
   Function *newFunction = constructFunctionDeclaration(
       inputs, outputs, EntryFreq, oldFunction->getName() + "." + SuffixToUse,
       StructValues, StructTy);
-  newFunction->IsNewDbgInfoFormat = oldFunction->IsNewDbgInfoFormat;
   SmallVector<Value *> NewValues;
 
   emitFunctionBody(inputs, outputs, StructValues, newFunction, StructTy, header,
@@ -1650,7 +1648,6 @@ void CodeExtractor::emitFunctionBody(
   // head of the region, but the entry node of a function cannot have preds.
   BasicBlock *newFuncRoot =
       BasicBlock::Create(Context, "newFuncRoot", newFunction);
-  newFuncRoot->IsNewDbgInfoFormat = oldFunction->IsNewDbgInfoFormat;
 
   // Now sink all instructions which only have non-phi uses inside the region.
   // Group the allocas at the start of the block, so that any bitcast uses of
@@ -1884,10 +1881,8 @@ CallInst *CodeExtractor::emitReplacerCall(
   // This takes place of the original loop
   BasicBlock *codeReplacer =
       BasicBlock::Create(Context, "codeRepl", oldFunction, ReplIP);
-  codeReplacer->IsNewDbgInfoFormat = oldFunction->IsNewDbgInfoFormat;
   BasicBlock *AllocaBlock =
       AllocationBlock ? AllocationBlock : &oldFunction->getEntryBlock();
-  AllocaBlock->IsNewDbgInfoFormat = oldFunction->IsNewDbgInfoFormat;
 
   // Update the entry count of the function.
   if (BFI)
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/Debugify.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/Debugify.cpp
index 1c3d1bc9771f..57e8c499551d 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -321,7 +321,7 @@ bool llvm::stripDebugifyMetadata(Module &M) {
 
 bool hasLoc(const Instruction &I) {
   const DILocation *Loc = I.getDebugLoc().get();
-#if LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
   DebugLocKind Kind = I.getDebugLoc().getKind();
   return Loc || Kind != DebugLocKind::Normal;
 #else
@@ -727,7 +727,7 @@ bool diagnoseMisSizedDbgValue(Module &M, DbgValTy *DbgVal) {
   bool HasBadSize = false;
   if (Ty->isIntegerTy()) {
     auto Signedness = DbgVal->getVariable()->getSignedness();
-    if (Signedness && *Signedness == DIBasicType::Signedness::Signed)
+    if (Signedness == DIBasicType::Signedness::Signed)
       HasBadSize = ValueOperandSize < *DbgVarSize;
   } else {
     HasBadSize = ValueOperandSize != *DbgVarSize;
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/Evaluator.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/Evaluator.cpp
index 2af447aadce2..d1db2ee29f3a 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/Evaluator.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/Evaluator.cpp
@@ -77,7 +77,9 @@ isSimpleEnoughValueToCommitHelper(Constant *C,
   // We don't know exactly what relocations are allowed in constant expressions,
   // so we allow &global+constantoffset, which is safe and uniformly supported
   // across targets.
-  ConstantExpr *CE = cast<ConstantExpr>(C);
+  ConstantExpr *CE = dyn_cast<ConstantExpr>(C);
+  if (!CE)
+    return false;
   switch (CE->getOpcode()) {
   case Instruction::BitCast:
     // Bitcast is fine if the casted value is fine.
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 7a9605bf5f8d..f47c467d1514 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1775,6 +1775,7 @@ static Value *HandleByValArgument(Type *ByValType, Value *Arg,
   AllocaInst *NewAlloca =
       new AllocaInst(ByValType, Arg->getType()->getPointerAddressSpace(),
                      nullptr, Alignment, Arg->getName());
+  NewAlloca->setDebugLoc(DebugLoc::getCompilerGenerated());
   NewAlloca->insertBefore(Caller->begin()->begin());
   IFI.StaticAllocas.push_back(NewAlloca);
 
@@ -3258,6 +3259,8 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
 
     // Add an unconditional branch to make this look like the CallInst case...
     CreatedBranchToNormalDest = BranchInst::Create(II->getNormalDest(), CB.getIterator());
+    // We intend to replace this DebugLoc with another later.
+    CreatedBranchToNormalDest->setDebugLoc(DebugLoc::getTemporary());
 
     // Split the basic block.  This guarantees that no PHI nodes will have to be
     // updated due to new incoming edges, and make the invoke case more
@@ -3359,6 +3362,12 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
     Returns[0]->eraseFromParent();
     ReturnBB->eraseFromParent();
   } else if (!CB.use_empty()) {
+    // In this case there are no returns to use, so there is no clear source
+    // location for the "return".
+    // FIXME: It may be correct to use the scope end line of the function here,
+    // since this likely means we are falling out of the function.
+    if (CreatedBranchToNormalDest)
+      CreatedBranchToNormalDest->setDebugLoc(DebugLoc::getUnknown());
     // No returns, but something is using the return value of the call.  Just
     // nuke the result.
     CB.replaceAllUsesWith(PoisonValue::get(CB.getType()));
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/Local.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/Local.cpp
index edac2e0c93e1..fe40da534241 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/Local.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/Local.cpp
@@ -71,6 +71,7 @@
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
@@ -88,8 +89,6 @@
 using namespace llvm;
 using namespace llvm::PatternMatch;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 #define DEBUG_TYPE "local"
 
 STATISTIC(NumRemoved, "Number of unreachable basic blocks removed");
@@ -1548,9 +1547,9 @@ Align llvm::tryEnforceAlignment(Value *V, Align PrefAlign,
     return PrefAlign;
   }
 
-  if (auto *GO = dyn_cast<GlobalObject>(V)) {
+  if (auto *GV = dyn_cast<GlobalVariable>(V)) {
     // TODO: as above, this shouldn't be necessary.
-    Align CurrentAlign = GO->getPointerAlignment(DL);
+    Align CurrentAlign = GV->getPointerAlignment(DL);
     if (PrefAlign <= CurrentAlign)
       return CurrentAlign;
 
@@ -1558,16 +1557,16 @@ Align llvm::tryEnforceAlignment(Value *V, Align PrefAlign,
     // of the global.  If the memory we set aside for the global may not be the
     // memory used by the final program then it is impossible for us to reliably
     // enforce the preferred alignment.
-    if (!GO->canIncreaseAlignment())
+    if (!GV->canIncreaseAlignment())
       return CurrentAlign;
 
-    if (GO->isThreadLocal()) {
-      unsigned MaxTLSAlign = GO->getParent()->getMaxTLSAlignment() / CHAR_BIT;
+    if (GV->isThreadLocal()) {
+      unsigned MaxTLSAlign = GV->getParent()->getMaxTLSAlignment() / CHAR_BIT;
       if (MaxTLSAlign && PrefAlign > Align(MaxTLSAlign))
         PrefAlign = Align(MaxTLSAlign);
     }
 
-    GO->setAlignment(PrefAlign);
+    GV->setAlignment(PrefAlign);
     return PrefAlign;
   }
 
@@ -1582,7 +1581,7 @@ Align llvm::getOrEnforceKnownAlignment(Value *V, MaybeAlign PrefAlign,
   assert(V->getType()->isPointerTy() &&
          "getOrEnforceKnownAlignment expects a pointer!");
 
-  KnownBits Known = computeKnownBits(V, DL, 0, AC, CxtI, DT);
+  KnownBits Known = computeKnownBits(V, DL, AC, CxtI, DT);
   unsigned TrailZ = Known.countMinTrailingZeros();
 
   // Avoid trouble with ridiculously large TrailZ values, such as
@@ -1691,16 +1690,10 @@ static void insertDbgValueOrDbgVariableRecord(DIBuilder &Builder, Value *DV,
                                               DIExpression *DIExpr,
                                               const DebugLoc &NewLoc,
                                               BasicBlock::iterator Instr) {
-  if (!UseNewDbgInfoFormat) {
-    Builder.insertDbgValueIntrinsic(DV, DIVar, DIExpr, NewLoc, Instr);
-  } else {
-    // RemoveDIs: if we're using the new debug-info format, allocate a
-    // DbgVariableRecord directly instead of a dbg.value intrinsic.
-    ValueAsMetadata *DVAM = ValueAsMetadata::get(DV);
-    DbgVariableRecord *DV =
-        new DbgVariableRecord(DVAM, DIVar, DIExpr, NewLoc.get());
-    Instr->getParent()->insertDbgRecordBefore(DV, Instr);
-  }
+  ValueAsMetadata *DVAM = ValueAsMetadata::get(DV);
+  DbgVariableRecord *DVRec =
+      new DbgVariableRecord(DVAM, DIVar, DIExpr, NewLoc.get());
+  Instr->getParent()->insertDbgRecordBefore(DVRec, Instr);
 }
 
 static void insertDbgValueOrDbgVariableRecordAfter(
@@ -1896,7 +1889,6 @@ void llvm::ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR,
   // then we want to insert a dbg.value for the corresponding fragment.
   LLVM_DEBUG(dbgs() << "Failed to convert dbg.declare to dbg.value: " << *DVR
                     << '\n');
-  assert(UseNewDbgInfoFormat);
 
   // For now, when there is a store to parts of the variable (but we do not
   // know which part) we insert an dbg.value intrinsic to indicate that we
@@ -1987,7 +1979,6 @@ void llvm::ConvertDebugDeclareToDebugValue(DbgVariableRecord *DVR, LoadInst *LI,
   // future if multi-location support is added to the IR, it might be
   // preferable to keep tracking both the loaded value and the original
   // address in case the alloca can not be elided.
-  assert(UseNewDbgInfoFormat);
 
   // Create a DbgVariableRecord directly and insert.
   ValueAsMetadata *LIVAM = ValueAsMetadata::get(LI);
@@ -2927,7 +2918,8 @@ using DbgValReplacement = std::optional<DIExpression *>;
 /// possibly moving/undefing users to prevent use-before-def. Returns true if
 /// changes are made.
 static bool rewriteDebugUsers(
-    Instruction &From, Value &To, Instruction &DomPoint, DominatorTree &DT,
+    Instruction &From, Value &To, Instruction &DomPoint,
+    const DominatorTree &DT,
     function_ref<DbgValReplacement(DbgVariableIntrinsic &DII)> RewriteExpr,
     function_ref<DbgValReplacement(DbgVariableRecord &DVR)> RewriteDVRExpr) {
   // Find debug users of From.
@@ -3062,6 +3054,11 @@ static bool getNewDIConversionOps(const DataLayout &DL, Type *SourceTy,
     return true;
   }
 
+  if (SourceTy->isPointerTy() && DestTy->isPointerTy()) {
+    Ops.emplace_back(DIOp::Convert(DestTy));
+    return true;
+  }
+
   if (!SourceTy->isIntegerTy() || !DestTy->isIntegerTy())
     return false;
 
@@ -3130,7 +3127,8 @@ updateNewDIExpressionArgType(IntrinsicOrRecord &DII, Value *LocOp,
 }
 
 bool llvm::replaceAllDbgUsesWith(Instruction &From, Value &To,
-                                 Instruction &DomPoint, DominatorTree &DT) {
+                                 Instruction &DomPoint,
+                                 const DominatorTree &DT) {
   // Exit early if From has no debug users.
   if (!From.isUsedByMetadata())
     return false;
@@ -3207,6 +3205,23 @@ bool llvm::replaceAllDbgUsesWith(Instruction &From, Value &To,
                              SignOrZeroExtDVR);
   }
 
+  if (FromTy->isPointerTy() && ToTy->isPointerTy()) {
+    // Non-bitcast address space conversions are only supported on
+    // DIOp-DIExpressions.
+    auto IdentityNew = [&](DbgVariableIntrinsic &DII) -> DbgValReplacement {
+      if (DII.getExpression()->holdsNewElements())
+        return updateNewDIExpressionArgType(DII, &From, ToTy);
+      return std::nullopt;
+    };
+    auto IdentityNewDVR = [&](DbgVariableRecord &DVR) -> DbgValReplacement {
+      if (DVR.getExpression()->holdsNewElements())
+        return updateNewDIExpressionArgType(DVR, &From, ToTy);
+      return std::nullopt;
+    };
+    return rewriteDebugUsers(From, To, DomPoint, DT, IdentityNew,
+                             IdentityNewDVR);
+  }
+
   // TODO: Floating-point conversions, vectors.
   return false;
 }
@@ -3506,7 +3521,8 @@ static bool markAliveBlocks(Function &F,
           BasicBlock *UnreachableNormalDest = BasicBlock::Create(
               Ctx, OrigNormalDest->getName() + ".unreachable",
               II->getFunction(), OrigNormalDest);
-          new UnreachableInst(Ctx, UnreachableNormalDest);
+          auto *UI = new UnreachableInst(Ctx, UnreachableNormalDest);
+          UI->setDebugLoc(DebugLoc::getTemporary());
           II->setNormalDest(UnreachableNormalDest);
           if (DTU)
             DTU->applyUpdates(
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp
index bd025fddd0cf..f34396254825 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -374,6 +374,9 @@ static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
                                    L.getLoopPredecessor()->getTerminator()))
     return false;
 
+  auto Guards = ScalarEvolution::LoopGuards::collect(&L, SE);
+  BTC = SE.applyLoopGuards(BTC, Guards);
+  RightSCEV = SE.applyLoopGuards(RightSCEV, Guards);
   const SCEV *ValAtLastIter = LeftAR->evaluateAtIteration(BTC, SE);
   const SCEV *ValAtSecondToLastIter = LeftAR->evaluateAtIteration(
       SE.getMinusSCEV(BTC, SE.getOne(BTC->getType())), SE);
@@ -1257,7 +1260,11 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
     // Now adjust users of the original exit values by replacing them with the
     // exit value from the peeled iteration and remove them.
     for (const auto &[P, E] : ExitValues) {
-      P->replaceAllUsesWith(isa<Constant>(E) ? E : &*VMap.lookup(E));
+      Instruction *ExitInst = dyn_cast<Instruction>(E);
+      if (ExitInst && L->contains(ExitInst))
+        P->replaceAllUsesWith(&*VMap[ExitInst]);
+      else
+        P->replaceAllUsesWith(E);
       P->eraseFromParent();
     }
     formLCSSA(*L, DT, LI, SE);
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 693b1f517f8d..6b42503b2e01 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -634,8 +634,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
           // memory access in coroutines.
           !Inst->getFunction()->isPresplitCoroutine()) {
 
-        if (LoopEntryBranch->getParent()->IsNewDbgInfoFormat &&
-            !NextDbgInsts.empty()) {
+        if (!NextDbgInsts.empty()) {
           auto DbgValueRange =
               LoopEntryBranch->cloneDebugInfoFrom(Inst, NextDbgInsts.begin());
           RemapDbgRecordRange(M, DbgValueRange, ValueMap,
@@ -664,8 +663,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
 
       ++NumInstrsDuplicated;
 
-      if (LoopEntryBranch->getParent()->IsNewDbgInfoFormat &&
-          !NextDbgInsts.empty()) {
+      if (!NextDbgInsts.empty()) {
         auto Range = C->cloneDebugInfoFrom(Inst, NextDbgInsts.begin());
         RemapDbgRecordRange(M, Range, ValueMap,
                             RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/LoopUtils.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 3621989424e8..cf6b183c78ac 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -41,6 +41,7 @@
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
@@ -605,7 +606,6 @@ void llvm::deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE,
 
   // Use a map to unique and a vector to guarantee deterministic ordering.
   llvm::SmallDenseSet<DebugVariable, 4> DeadDebugSet;
-  llvm::SmallVector<DbgVariableIntrinsic *, 4> DeadDebugInst;
   llvm::SmallVector<DbgVariableRecord *, 4> DeadDbgVariableRecords;
 
   if (ExitBlock) {
@@ -632,29 +632,19 @@ void llvm::deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE,
           U.set(Poison);
         }
 
-        // RemoveDIs: do the same as below for DbgVariableRecords.
-        if (Block->IsNewDbgInfoFormat) {
-          for (DbgVariableRecord &DVR : llvm::make_early_inc_range(
-                   filterDbgVars(I.getDbgRecordRange()))) {
-            DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
-                              DVR.getDebugLoc().get());
-            if (!DeadDebugSet.insert(Key).second)
-              continue;
-            // Unlinks the DVR from it's container, for later insertion.
-            DVR.removeFromParent();
-            DeadDbgVariableRecords.push_back(&DVR);
-          }
-        }
-
-        // For one of each variable encountered, preserve a debug intrinsic (set
+        // For one of each variable encountered, preserve a debug record (set
         // to Poison) and transfer it to the loop exit. This terminates any
         // variable locations that were set during the loop.
-        auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I);
-        if (!DVI)
-          continue;
-        if (!DeadDebugSet.insert(DebugVariable(DVI)).second)
-          continue;
-        DeadDebugInst.push_back(DVI);
+        for (DbgVariableRecord &DVR :
+             llvm::make_early_inc_range(filterDbgVars(I.getDbgRecordRange()))) {
+          DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
+                            DVR.getDebugLoc().get());
+          if (!DeadDebugSet.insert(Key).second)
+            continue;
+          // Unlinks the DVR from it's container, for later insertion.
+          DVR.removeFromParent();
+          DeadDbgVariableRecords.push_back(&DVR);
+        }
       }
 
     // After the loop has been deleted all the values defined and modified
@@ -670,9 +660,6 @@ void llvm::deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE,
            "There should be a non-PHI instruction in exit block, else these "
            "instructions will have no parent.");
 
-    for (auto *DVI : DeadDebugInst)
-      DVI->moveBefore(*ExitBlock, InsertDbgValueBefore);
-
     // Due to the "head" bit in BasicBlock::iterator, we're going to insert
     // each DbgVariableRecord right at the start of the block, wheras dbg.values
     // would be repeatedly inserted before the first instruction. To replicate
@@ -1332,18 +1319,19 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
   }
 }
 
-Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src,
-                                   RecurKind Kind) {
+Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
+                                   RecurKind Kind, Value *Mask, Value *EVL) {
   assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
          !RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
          "AnyOf or FindLastIV reductions are not supported.");
   Intrinsic::ID Id = getReductionIntrinsicID(Kind);
-  auto *SrcTy = cast<VectorType>(Src->getType());
-  Type *SrcEltTy = SrcTy->getElementType();
-  Value *Iden =
-      getRecurrenceIdentity(Kind, SrcEltTy, VBuilder.getFastMathFlags());
-  Value *Ops[] = {Iden, Src};
-  return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
+  auto VPID = VPIntrinsic::getForIntrinsic(Id);
+  assert(VPReductionIntrinsic::isVPReduction(VPID) &&
+         "No VPIntrinsic for this reduction");
+  auto *EltTy = cast<VectorType>(Src->getType())->getElementType();
+  Value *Iden = getRecurrenceIdentity(Kind, EltTy, Builder.getFastMathFlags());
+  Value *Ops[] = {Iden, Src, Mask, EVL};
+  return Builder.CreateIntrinsic(EltTy, VPID, Ops);
 }
 
 Value *llvm::createOrderedReduction(IRBuilderBase &B, RecurKind Kind,
@@ -1356,17 +1344,21 @@ Value *llvm::createOrderedReduction(IRBuilderBase &B, RecurKind Kind,
   return B.CreateFAddReduce(Start, Src);
 }
 
-Value *llvm::createOrderedReduction(VectorBuilder &VBuilder, RecurKind Kind,
-                                    Value *Src, Value *Start) {
+Value *llvm::createOrderedReduction(IRBuilderBase &Builder, RecurKind Kind,
+                                    Value *Src, Value *Start, Value *Mask,
+                                    Value *EVL) {
   assert((Kind == RecurKind::FAdd || Kind == RecurKind::FMulAdd) &&
          "Unexpected reduction kind");
   assert(Src->getType()->isVectorTy() && "Expected a vector type");
   assert(!Start->getType()->isVectorTy() && "Expected a scalar type");
 
   Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd);
-  auto *SrcTy = cast<VectorType>(Src->getType());
-  Value *Ops[] = {Start, Src};
-  return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
+  auto VPID = VPIntrinsic::getForIntrinsic(Id);
+  assert(VPReductionIntrinsic::isVPReduction(VPID) &&
+         "No VPIntrinsic for this reduction");
+  auto *EltTy = cast<VectorType>(Src->getType())->getElementType();
+  Value *Ops[] = {Start, Src, Mask, EVL};
+  return Builder.CreateIntrinsic(EltTy, VPID, Ops);
 }
 
 void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue,
@@ -1820,10 +1812,11 @@ void llvm::appendLoopsToWorklist(RangeT &&Loops,
   appendReversedLoopsToWorklist(reverse(Loops), Worklist);
 }
 
-template void llvm::appendLoopsToWorklist<ArrayRef<Loop *> &>(
+template LLVM_EXPORT_TEMPLATE void
+llvm::appendLoopsToWorklist<ArrayRef<Loop *> &>(
     ArrayRef<Loop *> &Loops, SmallPriorityWorklist<Loop *, 4> &Worklist);
 
-template void
+template LLVM_EXPORT_TEMPLATE void
 llvm::appendLoopsToWorklist<Loop &>(Loop &L,
                                     SmallPriorityWorklist<Loop *, 4> &Worklist);
 
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/LowerSwitch.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/LowerSwitch.cpp
index b70310b36459..6acbce884fcc 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/LowerSwitch.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/LowerSwitch.cpp
@@ -406,7 +406,7 @@ void ProcessSwitchInst(SwitchInst *SI,
     //    roughly C icmp's per switch, where C is the number of cases in the
     //    switch, while LowerSwitch only needs to call LVI once per switch.
     const DataLayout &DL = F->getDataLayout();
-    KnownBits Known = computeKnownBits(Val, DL, /*Depth=*/0, AC, SI);
+    KnownBits Known = computeKnownBits(Val, DL, AC, SI);
     // TODO Shouldn't this create a signed range?
     ConstantRange KnownBitsRange =
         ConstantRange::fromKnownBits(Known, /*IsSigned=*/false);
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/LowerVectorIntrinsics.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/LowerVectorIntrinsics.cpp
index a34ef260dc24..71c10f5b157c 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/LowerVectorIntrinsics.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/LowerVectorIntrinsics.cpp
@@ -21,6 +21,7 @@ bool llvm::lowerUnaryVectorIntrinsicAsLoop(Module &M, CallInst *CI) {
   BasicBlock *PostLoopBB = nullptr;
   Function *ParentFunc = PreLoopBB->getParent();
   LLVMContext &Ctx = PreLoopBB->getContext();
+  Type *Int64Ty = IntegerType::get(Ctx, 64);
 
   PostLoopBB = PreLoopBB->splitBasicBlock(CI);
   BasicBlock *LoopBB = BasicBlock::Create(Ctx, "", ParentFunc, PostLoopBB);
@@ -28,22 +29,11 @@ bool llvm::lowerUnaryVectorIntrinsicAsLoop(Module &M, CallInst *CI) {
 
   // Loop preheader
   IRBuilder<> PreLoopBuilder(PreLoopBB->getTerminator());
-  Value *LoopEnd = nullptr;
-  if (auto *ScalableVecTy = dyn_cast<ScalableVectorType>(VecTy)) {
-    Value *VScale = PreLoopBuilder.CreateVScale(
-        ConstantInt::get(PreLoopBuilder.getInt64Ty(), 1));
-    Value *N = ConstantInt::get(PreLoopBuilder.getInt64Ty(),
-                                ScalableVecTy->getMinNumElements());
-    LoopEnd = PreLoopBuilder.CreateMul(VScale, N);
-  } else {
-    FixedVectorType *FixedVecTy = cast<FixedVectorType>(VecTy);
-    LoopEnd = ConstantInt::get(PreLoopBuilder.getInt64Ty(),
-                               FixedVecTy->getNumElements());
-  }
+  Value *LoopEnd =
+      PreLoopBuilder.CreateElementCount(Int64Ty, VecTy->getElementCount());
 
   // Loop body
   IRBuilder<> LoopBuilder(LoopBB);
-  Type *Int64Ty = LoopBuilder.getInt64Ty();
 
   PHINode *LoopIndex = LoopBuilder.CreatePHI(Int64Ty, 2);
   LoopIndex->addIncoming(ConstantInt::get(Int64Ty, 0U), PreLoopBB);
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/ModuleUtils.cpp
index 10efdd61d455..596849ecab74 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/ModuleUtils.cpp
@@ -18,6 +18,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
@@ -346,10 +347,11 @@ void llvm::filterDeadComdatFunctions(
 std::string llvm::getUniqueModuleId(Module *M) {
   MD5 Md5;
 
-  auto *UniqueSourceFileNames = mdconst::extract_or_null<ConstantInt>(
-      M->getModuleFlag("Unique Source File Names"));
-  if (UniqueSourceFileNames && UniqueSourceFileNames->getZExtValue()) {
-    Md5.update(M->getSourceFileName());
+  auto *UniqueSourceFileIdentifier = dyn_cast_or_null<MDNode>(
+      M->getModuleFlag("Unique Source File Identifier"));
+  if (UniqueSourceFileIdentifier) {
+    Md5.update(
+        cast<MDString>(UniqueSourceFileIdentifier->getOperand(0))->getString());
   } else {
     bool ExportsSymbols = false;
     for (auto &GV : M->global_values()) {
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 6ba64968193c..46808a818cb2 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -281,18 +281,48 @@ struct AllocaInfo {
   }
 };
 
+template <typename T> class VectorWithUndo {
+  SmallVector<T, 8> Vals;
+  SmallVector<std::pair<size_t, T>, 8> Undo;
+
+public:
+  void undo(size_t S) {
+    assert(S <= Undo.size());
+    while (S < Undo.size()) {
+      Vals[Undo.back().first] = Undo.back().second;
+      Undo.pop_back();
+    }
+  }
+
+  void resize(size_t Sz) { Vals.resize(Sz); }
+
+  size_t undoSize() const { return Undo.size(); }
+
+  const T &operator[](size_t Idx) const { return Vals[Idx]; }
+
+  void set(size_t Idx, const T &Val) {
+    if (Vals[Idx] == Val)
+      return;
+    Undo.emplace_back(Idx, Vals[Idx]);
+    Vals[Idx] = Val;
+  }
+
+  void init(size_t Idx, const T &Val) {
+    assert(Undo.empty());
+    Vals[Idx] = Val;
+  }
+};
+
 /// Data package used by RenamePass().
 struct RenamePassData {
-  using ValVector = std::vector<Value *>;
-  using LocationVector = std::vector<DebugLoc>;
-
-  RenamePassData(BasicBlock *B, BasicBlock *P, ValVector V, LocationVector L)
-      : BB(B), Pred(P), Values(std::move(V)), Locations(std::move(L)) {}
+  RenamePassData(BasicBlock *B, BasicBlock *P, size_t V, size_t L)
+      : BB(B), Pred(P), UndoVals(V), UndoLocs(L) {}
 
   BasicBlock *BB;
   BasicBlock *Pred;
-  ValVector Values;
-  LocationVector Locations;
+
+  size_t UndoVals;
+  size_t UndoLocs;
 };
 
 /// This assigns and keeps a per-bb relative ordering of load/store
@@ -392,6 +422,15 @@ struct PromoteMem2Reg {
   /// number.
   SmallVector<unsigned> BBNumPreds;
 
+  /// The state of incoming values for the current DFS step.
+  VectorWithUndo<Value *> IncomingVals;
+
+  /// The state of incoming locations for the current DFS step.
+  VectorWithUndo<DebugLoc> IncomingLocs;
+
+  // DFS work stack.
+  SmallVector<RenamePassData, 8> Worklist;
+
   /// Whether the function has the no-signed-zeros-fp-math attribute set.
   bool NoSignedZeros = false;
 
@@ -423,10 +462,7 @@ struct PromoteMem2Reg {
   void ComputeLiveInBlocks(AllocaInst *AI, AllocaInfo &Info,
                            const SmallPtrSetImpl<BasicBlock *> &DefBlocks,
                            SmallPtrSetImpl<BasicBlock *> &LiveInBlocks);
-  void RenamePass(BasicBlock *BB, BasicBlock *Pred,
-                  RenamePassData::ValVector &IncVals,
-                  RenamePassData::LocationVector &IncLocs,
-                  std::vector<RenamePassData> &Worklist);
+  void RenamePass(BasicBlock *BB, BasicBlock *Pred);
   bool QueuePhiNode(BasicBlock *BB, unsigned AllocaIdx, unsigned &Version);
 
   /// Delete dbg.assigns that have been demoted to dbg.values.
@@ -438,6 +474,19 @@ struct PromoteMem2Reg {
       DVR->eraseFromParent();
     DVRAssignsToDelete.clear();
   }
+
+  void pushToWorklist(BasicBlock *BB, BasicBlock *Pred) {
+    Worklist.emplace_back(BB, Pred, IncomingVals.undoSize(),
+                          IncomingLocs.undoSize());
+  }
+
+  RenamePassData popFromWorklist() {
+    RenamePassData R = Worklist.back();
+    Worklist.pop_back();
+    IncomingVals.undo(R.UndoVals);
+    IncomingLocs.undo(R.UndoLocs);
+    return R;
+  }
 };
 
 } // end anonymous namespace
@@ -849,28 +898,24 @@ void PromoteMem2Reg::run() {
   // Set the incoming values for the basic block to be null values for all of
   // the alloca's.  We do this in case there is a load of a value that has not
   // been stored yet.  In this case, it will get this null value.
-  RenamePassData::ValVector Values(Allocas.size());
+  IncomingVals.resize(Allocas.size());
   for (unsigned i = 0, e = Allocas.size(); i != e; ++i)
-    Values[i] = UndefValue::get(Allocas[i]->getAllocatedType());
+    IncomingVals.init(i, UndefValue::get(Allocas[i]->getAllocatedType()));
 
   // When handling debug info, treat all incoming values as if they have unknown
   // locations until proven otherwise.
-  RenamePassData::LocationVector Locations(Allocas.size());
+  IncomingLocs.resize(Allocas.size());
 
   // The renamer uses the Visited set to avoid infinite loops.
-  Visited.resize(F.getMaxBlockNumber());
+  Visited.resize(F.getMaxBlockNumber(), false);
+
+  // Add the entry block to the worklist, with a null predecessor.
+  pushToWorklist(&F.front(), nullptr);
 
-  // Walks all basic blocks in the function performing the SSA rename algorithm
-  // and inserting the phi nodes we marked as necessary
-  std::vector<RenamePassData> RenamePassWorkList;
-  RenamePassWorkList.emplace_back(&F.front(), nullptr, std::move(Values),
-                                  std::move(Locations));
   do {
-    RenamePassData RPD = std::move(RenamePassWorkList.back());
-    RenamePassWorkList.pop_back();
-    // RenamePass may add new worklist entries.
-    RenamePass(RPD.BB, RPD.Pred, RPD.Values, RPD.Locations, RenamePassWorkList);
-  } while (!RenamePassWorkList.empty());
+    RenamePassData RPD = popFromWorklist();
+    RenamePass(RPD.BB, RPD.Pred);
+  } while (!Worklist.empty());
 
   // Remove the allocas themselves from the function.
   for (Instruction *A : Allocas) {
@@ -1095,11 +1140,7 @@ static void updateForIncomingValueLocation(PHINode *PN, DebugLoc DL,
 ///
 /// IncomingVals indicates what value each Alloca contains on exit from the
 /// predecessor block Pred.
-void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred,
-                                RenamePassData::ValVector &IncomingVals,
-                                RenamePassData::LocationVector &IncomingLocs,
-                                std::vector<RenamePassData> &Worklist) {
-NextIteration:
+void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred) {
   // If we are inserting any phi nodes into this BB, they will already be in the
   // block.
   if (PHINode *APN = dyn_cast<PHINode>(BB->begin())) {
@@ -1139,7 +1180,7 @@ void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred,
           APN->setHasNoSignedZeros(true);
 
         // The currently active variable for this block is now the PHI.
-        IncomingVals[AllocaNo] = APN;
+        IncomingVals.set(AllocaNo, APN);
         AllocaATInfo[AllocaNo].updateForNewPhi(APN, DIB);
         auto ConvertDbgDeclares = [&](auto &Container) {
           for (auto *DbgItem : Container)
@@ -1197,10 +1238,10 @@ void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred,
 
       // what value were we writing?
       unsigned AllocaNo = ai->second;
-      IncomingVals[AllocaNo] = SI->getOperand(0);
+      IncomingVals.set(AllocaNo, SI->getOperand(0));
 
       // Record debuginfo for the store before removing it.
-      IncomingLocs[AllocaNo] = SI->getDebugLoc();
+      IncomingLocs.set(AllocaNo, SI->getDebugLoc());
       AllocaATInfo[AllocaNo].updateForDeletedStore(SI, DIB, &DbgAssignsToDelete,
                                                    &DVRAssignsToDelete);
       auto ConvertDbgDeclares = [&](auto &Container) {
@@ -1215,24 +1256,13 @@ void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred,
   }
 
   // 'Recurse' to our successors.
-  succ_iterator I = succ_begin(BB), E = succ_end(BB);
-  if (I == E)
-    return;
 
   // Keep track of the successors so we don't visit the same successor twice
   SmallPtrSet<BasicBlock *, 8> VisitedSuccs;
 
-  // Handle the first successor without using the worklist.
-  VisitedSuccs.insert(*I);
-  Pred = BB;
-  BB = *I;
-  ++I;
-
-  for (; I != E; ++I)
-    if (VisitedSuccs.insert(*I).second)
-      Worklist.emplace_back(*I, Pred, IncomingVals, IncomingLocs);
-
-  goto NextIteration;
+  for (BasicBlock *S : reverse(successors(BB)))
+    if (VisitedSuccs.insert(S).second)
+      pushToWorklist(S, BB);
 }
 
 void llvm::PromoteMemToReg(ArrayRef<AllocaInst *> Allocas, DominatorTree &DT,
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index 4535f86f9d68..f4b378b82dae 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Analysis/ValueLatticeUtils.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -30,6 +31,7 @@
 #include <vector>
 
 using namespace llvm;
+using namespace PatternMatch;
 
 #define DEBUG_TYPE "sccp"
 
@@ -83,20 +85,28 @@ bool SCCPSolver::tryToReplaceWithConstant(Value *V) {
   return true;
 }
 
+/// Helper for getting ranges from \p Solver. Instructions inserted during
+/// simplification are unavailable in the solver, so we return a full range for
+/// them.
+static ConstantRange getRange(Value *Op, SCCPSolver &Solver,
+                              const SmallPtrSetImpl<Value *> &InsertedValues) {
+  if (auto *Const = dyn_cast<Constant>(Op))
+    return Const->toConstantRange();
+  if (InsertedValues.contains(Op)) {
+    unsigned Bitwidth = Op->getType()->getScalarSizeInBits();
+    return ConstantRange::getFull(Bitwidth);
+  }
+  return Solver.getLatticeValueFor(Op).asConstantRange(Op->getType(),
+                                                       /*UndefAllowed=*/false);
+}
+
 /// Try to use \p Inst's value range from \p Solver to infer the NUW flag.
 static bool refineInstruction(SCCPSolver &Solver,
                               const SmallPtrSetImpl<Value *> &InsertedValues,
                               Instruction &Inst) {
   bool Changed = false;
   auto GetRange = [&Solver, &InsertedValues](Value *Op) {
-    if (auto *Const = dyn_cast<Constant>(Op))
-      return Const->toConstantRange();
-    if (InsertedValues.contains(Op)) {
-      unsigned Bitwidth = Op->getType()->getScalarSizeInBits();
-      return ConstantRange::getFull(Bitwidth);
-    }
-    return Solver.getLatticeValueFor(Op).asConstantRange(
-        Op->getType(), /*UndefAllowed=*/false);
+    return getRange(Op, Solver, InsertedValues);
   };
 
   if (isa<OverflowingBinaryOperator>(Inst)) {
@@ -167,16 +177,8 @@ static bool replaceSignedInst(SCCPSolver &Solver,
                               SmallPtrSetImpl<Value *> &InsertedValues,
                               Instruction &Inst) {
   // Determine if a signed value is known to be >= 0.
-  auto isNonNegative = [&Solver](Value *V) {
-    // If this value was constant-folded, it may not have a solver entry.
-    // Handle integers. Otherwise, return false.
-    if (auto *C = dyn_cast<Constant>(V)) {
-      auto *CInt = dyn_cast<ConstantInt>(C);
-      return CInt && !CInt->isNegative();
-    }
-    const ValueLatticeElement &IV = Solver.getLatticeValueFor(V);
-    return IV.isConstantRange(/*UndefAllowed=*/false) &&
-           IV.getConstantRange().isAllNonNegative();
+  auto isNonNegative = [&Solver, &InsertedValues](Value *V) {
+    return getRange(V, Solver, InsertedValues).isAllNonNegative();
   };
 
   Instruction *NewInst = nullptr;
@@ -185,7 +187,7 @@ static bool replaceSignedInst(SCCPSolver &Solver,
   case Instruction::SExt: {
     // If the source value is not negative, this is a zext/uitofp.
     Value *Op0 = Inst.getOperand(0);
-    if (InsertedValues.count(Op0) || !isNonNegative(Op0))
+    if (!isNonNegative(Op0))
       return false;
     NewInst = CastInst::Create(Inst.getOpcode() == Instruction::SExt
                                    ? Instruction::ZExt
@@ -197,7 +199,7 @@ static bool replaceSignedInst(SCCPSolver &Solver,
   case Instruction::AShr: {
     // If the shifted value is not negative, this is a logical shift right.
     Value *Op0 = Inst.getOperand(0);
-    if (InsertedValues.count(Op0) || !isNonNegative(Op0))
+    if (!isNonNegative(Op0))
       return false;
     NewInst = BinaryOperator::CreateLShr(Op0, Inst.getOperand(1), "", Inst.getIterator());
     NewInst->setIsExact(Inst.isExact());
@@ -207,8 +209,7 @@ static bool replaceSignedInst(SCCPSolver &Solver,
   case Instruction::SRem: {
     // If both operands are not negative, this is the same as udiv/urem.
     Value *Op0 = Inst.getOperand(0), *Op1 = Inst.getOperand(1);
-    if (InsertedValues.count(Op0) || InsertedValues.count(Op1) ||
-        !isNonNegative(Op0) || !isNonNegative(Op1))
+    if (!isNonNegative(Op0) || !isNonNegative(Op1))
       return false;
     auto NewOpcode = Inst.getOpcode() == Instruction::SDiv ? Instruction::UDiv
                                                            : Instruction::URem;
@@ -232,6 +233,26 @@ static bool replaceSignedInst(SCCPSolver &Solver,
   return true;
 }
 
+/// Try to use \p Inst's value range from \p Solver to simplify it.
+static Value *simplifyInstruction(SCCPSolver &Solver,
+                                  SmallPtrSetImpl<Value *> &InsertedValues,
+                                  Instruction &Inst) {
+  auto GetRange = [&Solver, &InsertedValues](Value *Op) {
+    return getRange(Op, Solver, InsertedValues);
+  };
+
+  Value *X;
+  const APInt *RHSC;
+  // Remove masking operations.
+  if (match(&Inst, m_And(m_Value(X), m_LowBitMask(RHSC)))) {
+    ConstantRange LRange = GetRange(Inst.getOperand(0));
+    if (LRange.getUnsignedMax().ule(*RHSC))
+      return X;
+  }
+
+  return nullptr;
+}
+
 bool SCCPSolver::simplifyInstsInBlock(BasicBlock &BB,
                                       SmallPtrSetImpl<Value *> &InsertedValues,
                                       Statistic &InstRemovedStat,
@@ -251,6 +272,11 @@ bool SCCPSolver::simplifyInstsInBlock(BasicBlock &BB,
       ++InstReplacedStat;
     } else if (refineInstruction(*this, InsertedValues, Inst)) {
       MadeChanges = true;
+    } else if (auto *V = simplifyInstruction(*this, InsertedValues, Inst)) {
+      Inst.replaceAllUsesWith(V);
+      Inst.eraseFromParent();
+      ++InstRemovedStat;
+      MadeChanges = true;
     }
   }
   return MadeChanges;
@@ -322,7 +348,9 @@ bool SCCPSolver::removeNonFeasibleEdges(BasicBlock *BB, DomTreeUpdater &DTU,
         NewUnreachableBB =
             BasicBlock::Create(DefaultDest->getContext(), "default.unreachable",
                                DefaultDest->getParent(), DefaultDest);
-        new UnreachableInst(DefaultDest->getContext(), NewUnreachableBB);
+        auto *UI =
+            new UnreachableInst(DefaultDest->getContext(), NewUnreachableBB);
+        UI->setDebugLoc(DebugLoc::getTemporary());
       }
 
       DefaultDest->removePredecessor(BB);
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/SSAUpdater.cpp
index 48d9528f0c3d..5db7fc956c49 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/SSAUpdater.cpp
@@ -318,6 +318,11 @@ class SSAUpdaterTraits<SSAUpdater> {
                                SSAUpdater *Updater) {
     PHINode *PHI =
         PHINode::Create(Updater->ProtoType, NumPreds, Updater->ProtoName);
+    // FIXME: Ordinarily we don't care about or try to assign DebugLocs to PHI
+    // nodes, but loop optimizations may try to use a PHI node as a DebugLoc
+    // source (e.g. if this is an induction variable), and it's not clear what
+    // location we could attach here, so mark this unknown for now.
+    PHI->setDebugLoc(DebugLoc::getUnknown());
     PHI->insertBefore(BB->begin());
     return PHI;
   }
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 88d2eca36ca5..70afd4133df7 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1440,7 +1440,7 @@ Value *SCEVExpander::visitSequentialUMinExpr(const SCEVSequentialUMinExpr *S) {
 }
 
 Value *SCEVExpander::visitVScale(const SCEVVScale *S) {
-  return Builder.CreateVScale(ConstantInt::get(S->getType(), 1));
+  return Builder.CreateVScale(S->getType());
 }
 
 Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty,
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 7840601d341b..0980f0e57aa6 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1137,7 +1137,7 @@ static void cloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(
         // branch, drop it. When we fold the bonus instructions we want to make
         // sure we reset their debug locations in order to avoid stepping on
         // dead code caused by folding dead branches.
-        NewBonusInst->setDebugLoc(DebugLoc());
+        NewBonusInst->setDebugLoc(DebugLoc::getDropped());
       } else if (const DebugLoc &DL = NewBonusInst->getDebugLoc()) {
         mapAtomInstance(DL, VMap);
       }
@@ -2095,11 +2095,11 @@ bool SimplifyCFGOpt::hoistSuccIdenticalTerminatorToSwitchOrIf(
 
   // Ensure terminator gets a debug location, even an unknown one, in case
   // it involves inlinable calls.
-  SmallVector<DILocation *, 4> Locs;
+  SmallVector<DebugLoc, 4> Locs;
   Locs.push_back(I1->getDebugLoc());
   for (auto *OtherSuccTI : OtherSuccTIs)
     Locs.push_back(OtherSuccTI->getDebugLoc());
-  NT->setDebugLoc(DILocation::getMergedLocations(Locs));
+  NT->setDebugLoc(DebugLoc::getMergedLocations(Locs));
 
   // PHIs created below will adopt NT's merged DebugLoc.
   IRBuilder<NoFolder> Builder(NT);
@@ -2821,7 +2821,8 @@ static void mergeCompatibleInvokesImpl(ArrayRef<InvokeInst *> Invokes,
       // so just form a new block with unreachable terminator.
       BasicBlock *MergedNormalDest = BasicBlock::Create(
           Ctx, II0BB->getName() + ".cont", Func, InsertBeforeBlock);
-      new UnreachableInst(Ctx, MergedNormalDest);
+      auto *UI = new UnreachableInst(Ctx, MergedNormalDest);
+      UI->setDebugLoc(DebugLoc::getTemporary());
       MergedInvoke->setNormalDest(MergedNormalDest);
     }
 
@@ -2895,7 +2896,7 @@ static void mergeCompatibleInvokesImpl(ArrayRef<InvokeInst *> Invokes,
       MergedDebugLoc = II->getDebugLoc();
     else
       MergedDebugLoc =
-          DILocation::getMergedLocation(MergedDebugLoc, II->getDebugLoc());
+          DebugLoc::getMergedLocation(MergedDebugLoc, II->getDebugLoc());
 
     // And replace the old `invoke` with an unconditionally branch
     // to the block with the merged `invoke`.
@@ -3389,7 +3390,7 @@ bool SimplifyCFGOpt::speculativelyExecuteBB(BranchInst *BI,
     if (!SpeculatedStoreValue || &I != SpeculatedStore) {
       // Don't update the DILocation of dbg.assign intrinsics.
       if (!isa<DbgAssignIntrinsic>(&I))
-        I.setDebugLoc(DebugLoc());
+        I.setDebugLoc(DebugLoc::getDropped());
     }
     I.dropUBImplyingAttrsAndMetadata();
 
@@ -4054,13 +4055,11 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
 
   Module *M = BB->getModule();
 
-  if (PredBlock->IsNewDbgInfoFormat) {
-    PredBlock->getTerminator()->cloneDebugInfoFrom(BB->getTerminator());
-    for (DbgVariableRecord &DVR :
-         filterDbgVars(PredBlock->getTerminator()->getDbgRecordRange())) {
-      RemapDbgRecord(M, &DVR, VMap,
-                     RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
-    }
+  PredBlock->getTerminator()->cloneDebugInfoFrom(BB->getTerminator());
+  for (DbgVariableRecord &DVR :
+       filterDbgVars(PredBlock->getTerminator()->getDbgRecordRange())) {
+    RemapDbgRecord(M, &DVR, VMap,
+                   RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
   }
 
   // Now that the Cond was cloned into the predecessor basic block,
@@ -5707,7 +5706,8 @@ static void createUnreachableSwitchDefault(SwitchInst *Switch,
   BasicBlock *NewDefaultBlock = BasicBlock::Create(
       BB->getContext(), BB->getName() + ".unreachabledefault", BB->getParent(),
       OrigDefaultBlock);
-  new UnreachableInst(Switch->getContext(), NewDefaultBlock);
+  auto *UI = new UnreachableInst(Switch->getContext(), NewDefaultBlock);
+  UI->setDebugLoc(DebugLoc::getTemporary());
   Switch->setDefaultDest(&*NewDefaultBlock);
   if (DTU) {
     SmallVector<DominatorTree::UpdateType, 2> Updates;
@@ -5854,13 +5854,13 @@ static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU,
                                      AssumptionCache *AC,
                                      const DataLayout &DL) {
   Value *Cond = SI->getCondition();
-  KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI);
+  KnownBits Known = computeKnownBits(Cond, DL, AC, SI);
 
   // We can also eliminate cases by determining that their values are outside of
   // the limited range of the condition based on how many significant (non-sign)
   // bits are in the condition value.
   unsigned MaxSignificantBitsInCond =
-      ComputeMaxSignificantBits(Cond, DL, 0, AC, SI);
+      ComputeMaxSignificantBits(Cond, DL, AC, SI);
 
   // Gather dead cases.
   SmallVector<ConstantInt *, 8> DeadCases;
@@ -7557,8 +7557,7 @@ bool SimplifyCFGOpt::simplifyDuplicateSwitchArms(SwitchInst *SI,
     if (Seen.insert(BB).second) {
       // Keep track of which PHIs we need as keys in PhiPredIVs below.
       for (BasicBlock *Succ : BI->successors())
-        for (PHINode &Phi : Succ->phis())
-          Phis.insert(&Phi);
+        Phis.insert_range(llvm::make_pointer_range(Succ->phis()));
       // Add the successor only if not previously visited.
       Cases.emplace_back(SwitchSuccWrapper{BB, &PhiPredIVs});
     }
@@ -8109,6 +8108,7 @@ bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
 
 /// Check if passing a value to an instruction will cause undefined behavior.
 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified) {
+  assert(V->getType() == I->getType() && "Mismatched types");
   Constant *C = dyn_cast<Constant>(V);
   if (!C)
     return false;
@@ -8166,6 +8166,10 @@ static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValu
     // Look through GEPs. A load from a GEP derived from NULL is still undefined
     if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(User))
       if (GEP->getPointerOperand() == I) {
+        // The type of GEP may differ from the type of base pointer.
+        // Bail out on vector GEPs, as they are not handled by other checks.
+        if (GEP->getType()->isVectorTy())
+          return false;
         // The current base address is null, there are four cases to consider:
         // getelementptr (TY, null, 0)                 -> null
         // getelementptr (TY, null, not zero)          -> may be modified
@@ -8224,8 +8228,8 @@ static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValu
       if (CB->isArgOperand(&Use)) {
         unsigned ArgIdx = CB->getArgOperandNo(&Use);
         // Passing null to a nonnnull+noundef argument is undefined.
-        if (C->isNullValue() && CB->isPassingUndefUB(ArgIdx) &&
-            CB->paramHasAttr(ArgIdx, Attribute::NonNull))
+        if (isa<ConstantPointerNull>(C) &&
+            CB->paramHasNonNullAttr(ArgIdx, /*AllowUndefOrPoison=*/false))
           return !PtrValueMayBeModified;
         // Passing undef to a noundef argument is undefined.
         if (isa<UndefValue>(C) && CB->isPassingUndefUB(ArgIdx))
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 3fe4621eca70..43264cce7371 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -1605,13 +1605,13 @@ bool WidenIV::widenLoopCompare(WidenIV::NarrowIVDefUse DU) {
   //
   //  - The signedness of the IV extension and comparison match
   //
-  //  - The narrow IV is always positive (and thus its sign extension is equal
-  //    to its zero extension).  For instance, let's say we're zero extending
-  //    %narrow for the following use
+  //  - The narrow IV is always non-negative (and thus its sign extension is
+  //    equal to its zero extension).  For instance, let's say we're zero
+  //    extending %narrow for the following use
   //
   //      icmp slt i32 %narrow, %val   ... (A)
   //
-  //    and %narrow is always positive.  Then
+  //    and %narrow is always non-negative.  Then
   //
   //      (A) == icmp slt i32 sext(%narrow), sext(%val)
   //          == icmp slt i32 zext(%narrow), sext(%val)
@@ -1630,6 +1630,12 @@ bool WidenIV::widenLoopCompare(WidenIV::NarrowIVDefUse DU) {
 
   // Widen the other operand of the compare, if necessary.
   if (CastWidth < IVWidth) {
+    // If the narrow IV is always non-negative and the other operand is sext,
+    // widen using sext so we can combine them. This works for all non-signed
+    // comparison predicates.
+    if (DU.NeverNegative && isa<SExtInst>(Op) && !Cmp->isSigned())
+      CmpPreferredSign = true;
+
     Value *ExtOp = createExtendInst(Op, WideType, CmpPreferredSign, Cmp);
     DU.NarrowUse->replaceUsesOfWith(Op, ExtOp);
   }
diff --git a/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 94a79ad82437..737321daa910 100644
--- a/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1001,7 +1001,7 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
       }
 
       Value *Offset = GEP->getOperand(2);
-      KnownBits Known = computeKnownBits(Offset, DL, 0, nullptr, CI, nullptr);
+      KnownBits Known = computeKnownBits(Offset, DL, nullptr, CI, nullptr);
       uint64_t ArrSize =
              cast<ArrayType>(GEP->getSourceElementType())->getNumElements();
 
@@ -2262,7 +2262,7 @@ Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
   // errno), but sqrt(-Inf) is required by various standards to set errno.
   if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
       !isKnownNeverInfinity(
-          Base, 0, SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
+          Base, SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
     return nullptr;
 
   Sqrt = getSqrtCall(Base, AttributeList(), Pow->doesNotAccessMemory(), Mod, B,
@@ -2574,8 +2574,7 @@ Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
       SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
       KnownFPClass Known = computeKnownFPClass(
           Log->getOperand(0),
-          KnownFPClass::OrderedLessThanZeroMask | fcSubnormal,
-          /*Depth=*/0, SQ);
+          KnownFPClass::OrderedLessThanZeroMask | fcSubnormal, SQ);
       Function *F = Log->getParent()->getParent();
       const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
       IsKnownNoErrno =
@@ -2803,12 +2802,10 @@ Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
   bool IsNoNan = CI->hasNoNaNs();
   if (!IsNoNan) {
     SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
-    KnownFPClass Known0 = computeKnownFPClass(CI->getOperand(0), fcInf,
-                                              /*Depth=*/0, SQ);
+    KnownFPClass Known0 = computeKnownFPClass(CI->getOperand(0), fcInf, SQ);
     if (Known0.isKnownNeverInfinity()) {
       KnownFPClass Known1 =
-          computeKnownFPClass(CI->getOperand(1), fcZero | fcSubnormal,
-                              /*Depth=*/0, SQ);
+          computeKnownFPClass(CI->getOperand(1), fcZero | fcSubnormal, SQ);
       Function *F = CI->getParent()->getParent();
       const fltSemantics &FltSem =
           CI->getType()->getScalarType()->getFltSemantics();
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/external/llvm-project/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 63cf1a5e3f7c..96670fe3ea19 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMVectorize
   SandboxVectorizer/Interval.cpp
   SandboxVectorizer/Legality.cpp
   SandboxVectorizer/Passes/BottomUpVec.cpp
+  SandboxVectorizer/Passes/PackReuse.cpp
   SandboxVectorizer/Passes/RegionsFromBBs.cpp
   SandboxVectorizer/Passes/RegionsFromMetadata.cpp
   SandboxVectorizer/Passes/SeedCollection.cpp
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 53c83a8c256e..89f63c3b66aa 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1269,8 +1269,8 @@ std::optional<APInt> Vectorizer::getConstantOffsetComplexAddrs(
     // When computing known bits, use the GEPs as context instructions, since
     // they likely are in the same BB as the load/store.
     KnownBits Known(BitWidth);
-    computeKnownBits((IdxDiff.sge(0) ? ValA : OpB), Known, DL, 0, &AC,
-                     ContextInst, &DT);
+    computeKnownBits((IdxDiff.sge(0) ? ValA : OpB), Known, DL, &AC, ContextInst,
+                     &DT);
     APInt BitsAllowedToBeSet = Known.Zero.zext(IdxDiff.getBitWidth());
     if (Signed)
       BitsAllowedToBeSet.clearBit(BitWidth - 1);
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
index 02ffc67c774d..491f0b76f4ae 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
@@ -437,7 +437,7 @@ Value *LoopIdiomVectorize::createMaskedFindMismatch(
   Value *InitialPred = Builder.CreateIntrinsic(
       Intrinsic::get_active_lane_mask, {PredVTy, I64Type}, {ExtStart, ExtEnd});
 
-  Value *VecLen = Builder.CreateIntrinsic(Intrinsic::vscale, {I64Type}, {});
+  Value *VecLen = Builder.CreateVScale(I64Type);
   VecLen =
       Builder.CreateMul(VecLen, ConstantInt::get(I64Type, ByteCompareVF), "",
                         /*HasNUW=*/true, /*HasNSW=*/true);
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index b81d582f07e8..70f541d64b30 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -153,7 +153,7 @@ class VPBuilder {
   VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
                               Instruction *Inst = nullptr,
                               const Twine &Name = "") {
-    DebugLoc DL;
+    DebugLoc DL = DebugLoc::getUnknown();
     if (Inst)
       DL = Inst->getDebugLoc();
     VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name);
@@ -165,7 +165,8 @@ class VPBuilder {
     return createInstruction(Opcode, Operands, DL, Name);
   }
   VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
-                              const VPIRFlags &Flags, DebugLoc DL = {},
+                              const VPIRFlags &Flags,
+                              DebugLoc DL = DebugLoc::getUnknown(),
                               const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstruction(Opcode, Operands, Flags, DL, Name));
@@ -174,7 +175,8 @@ class VPBuilder {
   VPInstruction *createNaryOp(unsigned Opcode,
                               std::initializer_list<VPValue *> Operands,
                               Type *ResultTy, const VPIRFlags &Flags = {},
-                              DebugLoc DL = {}, const Twine &Name = "") {
+                              DebugLoc DL = DebugLoc::getUnknown(),
+                              const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name));
   }
@@ -182,22 +184,25 @@ class VPBuilder {
   VPInstruction *createOverflowingOp(unsigned Opcode,
                                      std::initializer_list<VPValue *> Operands,
                                      VPRecipeWithIRFlags::WrapFlagsTy WrapFlags,
-                                     DebugLoc DL = {}, const Twine &Name = "") {
+                                     DebugLoc DL = DebugLoc::getUnknown(),
+                                     const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstruction(Opcode, Operands, WrapFlags, DL, Name));
   }
 
-  VPValue *createNot(VPValue *Operand, DebugLoc DL = {},
+  VPValue *createNot(VPValue *Operand, DebugLoc DL = DebugLoc::getUnknown(),
                      const Twine &Name = "") {
     return createInstruction(VPInstruction::Not, {Operand}, DL, Name);
   }
 
-  VPValue *createAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
+  VPValue *createAnd(VPValue *LHS, VPValue *RHS,
+                     DebugLoc DL = DebugLoc::getUnknown(),
                      const Twine &Name = "") {
     return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name);
   }
 
-  VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
+  VPValue *createOr(VPValue *LHS, VPValue *RHS,
+                    DebugLoc DL = DebugLoc::getUnknown(),
                     const Twine &Name = "") {
 
     return tryInsertInstruction(new VPInstruction(
@@ -205,14 +210,16 @@ class VPBuilder {
         VPRecipeWithIRFlags::DisjointFlagsTy(false), DL, Name));
   }
 
-  VPValue *createLogicalAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
+  VPValue *createLogicalAnd(VPValue *LHS, VPValue *RHS,
+                            DebugLoc DL = DebugLoc::getUnknown(),
                             const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
   }
 
   VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
-                        DebugLoc DL = {}, const Twine &Name = "",
+                        DebugLoc DL = DebugLoc::getUnknown(),
+                        const Twine &Name = "",
                         std::optional<FastMathFlags> FMFs = std::nullopt) {
     auto *Select =
         FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
@@ -226,20 +233,23 @@ class VPBuilder {
   /// and \p B.
   /// TODO: add createFCmp when needed.
   VPValue *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
-                      DebugLoc DL = {}, const Twine &Name = "") {
+                      DebugLoc DL = DebugLoc::getUnknown(),
+                      const Twine &Name = "") {
     assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
            Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
     return tryInsertInstruction(
         new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name));
   }
 
-  VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {},
+  VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
+                              DebugLoc DL = DebugLoc::getUnknown(),
                               const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
                           GEPNoWrapFlags::none(), DL, Name));
   }
-  VPValue *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {},
+  VPValue *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset,
+                                DebugLoc DL = DebugLoc::getUnknown(),
                                 const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e9ace195684b..bd0a2ec3986d 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -505,9 +505,6 @@ class InnerLoopVectorizer {
   /// Fix the vectorized code, taking care of header phi's, and more.
   void fixVectorizedLoop(VPTransformState &State);
 
-  // Return true if any runtime check is added.
-  bool areSafetyChecksAdded() { return AddedSafetyChecks; }
-
   /// Fix the non-induction PHIs in \p Plan.
   void fixNonInductionPHIs(VPTransformState &State);
 
@@ -533,6 +530,9 @@ class InnerLoopVectorizer {
   /// Returns (and creates if needed) the trip count of the widened loop.
   Value *getOrCreateVectorTripCount(BasicBlock *InsertBlock);
 
+  // Create a check to see if the vector loop should be executed
+  Value *createIterationCountCheck(ElementCount VF, unsigned UF) const;
+
   /// Emit a bypass check to see if the vector trip count is zero, including if
   /// it overflows.
   void emitIterationCountCheck(BasicBlock *Bypass);
@@ -611,9 +611,6 @@ class InnerLoopVectorizer {
   /// Middle Block between the vector and the scalar.
   BasicBlock *LoopMiddleBlock = nullptr;
 
-  /// A list of all bypass blocks. The first block is the entry of the loop.
-  SmallVector<BasicBlock *, 4> LoopBypassBlocks;
-
   /// Trip count of the original loop.
   Value *TripCount = nullptr;
 
@@ -623,9 +620,6 @@ class InnerLoopVectorizer {
   /// The profitablity analysis.
   LoopVectorizationCostModel *Cost;
 
-  // Record whether runtime checks are added.
-  bool AddedSafetyChecks = false;
-
   /// BFI and PSI are used to check for profile guided size optimizations.
   BlockFrequencyInfo *BFI;
   ProfileSummaryInfo *PSI;
@@ -775,7 +769,7 @@ class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
 /// Look for a meaningful debug location on the instruction or its operands.
 static DebugLoc getDebugLocFromInstOrOperands(Instruction *I) {
   if (!I)
-    return DebugLoc();
+    return DebugLoc::getUnknown();
 
   DebugLoc Empty;
   if (I->getDebugLoc() != Empty)
@@ -959,6 +953,14 @@ class LoopVectorizationCostModel {
     return expectedCost(UserVF).isValid();
   }
 
+  /// \return True if maximizing vector bandwidth is enabled by the target or
+  /// user options, for the given register kind.
+  bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind);
+
+  /// \return True if maximizing vector bandwidth is enabled by the target or
+  /// user options, for the given vector factor.
+  bool useMaxBandwidth(ElementCount VF);
+
   /// \return The size (in bits) of the smallest and widest types in the code
   /// that needs to be vectorized. We ignore values that remain scalar such as
   /// 64 bit loop indices.
@@ -987,25 +989,6 @@ class LoopVectorizationCostModel {
   /// decision in a map for use in planning and plan execution.
   void setVectorizedCallDecision(ElementCount VF);
 
-  /// A struct that represents some properties of the register usage
-  /// of a loop.
-  struct RegisterUsage {
-    /// Holds the number of loop invariant values that are used in the loop.
-    /// The key is ClassID of target-provided register class.
-    SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs;
-    /// Holds the maximum number of concurrent live intervals in the loop.
-    /// The key is ClassID of target-provided register class.
-    SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
-
-    /// Check if any of the tracked live intervals exceeds the number of
-    /// available registers for the target.
-    bool exceedsMaxNumRegs(const TargetTransformInfo &TTI) const {
-      return any_of(MaxLocalUsers, [&TTI](auto &LU) {
-        return LU.second > TTI.getNumberOfRegisters(LU.first);
-      });
-    }
-  };
-
   /// Collect values we want to ignore in the cost model.
   void collectValuesToIgnore();
 
@@ -1105,7 +1088,7 @@ class LoopVectorizationCostModel {
   void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
                            InstructionCost Cost) {
     assert(VF.isVector() && "Expected VF >=2");
-    WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
+    WideningDecisions[{I, VF}] = {W, Cost};
   }
 
   /// Save vectorization decision \p W and \p Cost taken by the cost model for
@@ -1127,11 +1110,9 @@ class LoopVectorizationCostModel {
     for (unsigned Idx = 0; Idx < Grp->getFactor(); ++Idx) {
       if (auto *I = Grp->getMember(Idx)) {
         if (Grp->getInsertPos() == I)
-          WideningDecisions[std::make_pair(I, VF)] =
-              std::make_pair(W, InsertPosCost);
+          WideningDecisions[{I, VF}] = {W, InsertPosCost};
         else
-          WideningDecisions[std::make_pair(I, VF)] =
-              std::make_pair(W, OtherMemberCost);
+          WideningDecisions[{I, VF}] = {W, OtherMemberCost};
       }
     }
   }
@@ -1145,7 +1126,7 @@ class LoopVectorizationCostModel {
         TheLoop->isInnermost() &&
         "cost-model should not be used for outer loops (in VPlan-native path)");
 
-    std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
+    std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
     auto Itr = WideningDecisions.find(InstOnVF);
     if (Itr == WideningDecisions.end())
       return CM_Unknown;
@@ -1156,7 +1137,7 @@ class LoopVectorizationCostModel {
   /// width \p VF.
   InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
     assert(VF.isVector() && "Expected VF >=2");
-    std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
+    std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
     assert(WideningDecisions.contains(InstOnVF) &&
            "The cost is not calculated");
     return WideningDecisions[InstOnVF].second;
@@ -1175,8 +1156,7 @@ class LoopVectorizationCostModel {
                                std::optional<unsigned> MaskPos,
                                InstructionCost Cost) {
     assert(!VF.isScalar() && "Expected vector VF");
-    CallWideningDecisions[std::make_pair(CI, VF)] = {Kind, Variant, IID,
-                                                     MaskPos, Cost};
+    CallWideningDecisions[{CI, VF}] = {Kind, Variant, IID, MaskPos, Cost};
   }
 
   CallWideningDecision getCallWideningDecision(CallInst *CI,
@@ -1373,21 +1353,20 @@ class LoopVectorizationCostModel {
   void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC) {
     assert(!ChosenTailFoldingStyle && "Tail folding must not be selected yet.");
     if (!Legal->canFoldTailByMasking()) {
-      ChosenTailFoldingStyle =
-          std::make_pair(TailFoldingStyle::None, TailFoldingStyle::None);
+      ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
       return;
     }
 
     if (!ForceTailFoldingStyle.getNumOccurrences()) {
-      ChosenTailFoldingStyle = std::make_pair(
+      ChosenTailFoldingStyle = {
           TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true),
-          TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false));
+          TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false)};
       return;
     }
 
     // Set styles when forced.
-    ChosenTailFoldingStyle = std::make_pair(ForceTailFoldingStyle.getValue(),
-                                            ForceTailFoldingStyle.getValue());
+    ChosenTailFoldingStyle = {ForceTailFoldingStyle.getValue(),
+                              ForceTailFoldingStyle.getValue()};
     if (ForceTailFoldingStyle != TailFoldingStyle::DataWithEVL)
       return;
     // Override forced styles if needed.
@@ -1400,9 +1379,8 @@ class LoopVectorizationCostModel {
       // If for some reason EVL mode is unsupported, fallback to
       // DataWithoutLaneMask to try to vectorize the loop with folded tail
       // in a generic way.
-      ChosenTailFoldingStyle =
-          std::make_pair(TailFoldingStyle::DataWithoutLaneMask,
-                         TailFoldingStyle::DataWithoutLaneMask);
+      ChosenTailFoldingStyle = {TailFoldingStyle::DataWithoutLaneMask,
+                                TailFoldingStyle::DataWithoutLaneMask};
       LLVM_DEBUG(
           dbgs()
           << "LV: Preference for VP intrinsics indicated. Will "
@@ -1799,6 +1777,9 @@ class GeneratedRTChecks {
   /// they have been used.
   Value *MemRuntimeCheckCond = nullptr;
 
+  /// True if any checks have been added.
+  bool AddedAnyChecks = false;
+
   DominatorTree *DT;
   LoopInfo *LI;
   TargetTransformInfo *TTI;
@@ -1903,13 +1884,15 @@ class GeneratedRTChecks {
     if (SCEVCheckBlock) {
       SCEVCheckBlock->getTerminator()->moveBefore(
           Preheader->getTerminator()->getIterator());
-      new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
+      auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
+      UI->setDebugLoc(DebugLoc::getTemporary());
       Preheader->getTerminator()->eraseFromParent();
     }
     if (MemCheckBlock) {
       MemCheckBlock->getTerminator()->moveBefore(
           Preheader->getTerminator()->getIterator());
-      new UnreachableInst(Preheader->getContext(), MemCheckBlock);
+      auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
+      UI->setDebugLoc(DebugLoc::getTemporary());
       Preheader->getTerminator()->eraseFromParent();
     }
 
@@ -2058,9 +2041,9 @@ class GeneratedRTChecks {
     if (AddBranchWeights)
       setBranchWeights(BI, SCEVCheckBypassWeights, /*IsExpected=*/false);
     ReplaceInstWithInst(SCEVCheckBlock->getTerminator(), &BI);
-
     // Mark the check as used, to prevent it from being removed during cleanup.
     SCEVCheckCond = nullptr;
+    AddedAnyChecks = true;
     return SCEVCheckBlock;
   }
 
@@ -2090,8 +2073,12 @@ class GeneratedRTChecks {
 
     // Mark the check as used, to prevent it from being removed during cleanup.
     MemRuntimeCheckCond = nullptr;
+    AddedAnyChecks = true;
     return MemCheckBlock;
   }
+
+  /// Return true if any runtime checks have been added
+  bool hasChecks() const { return AddedAnyChecks; }
 };
 } // namespace
 
@@ -2389,13 +2376,8 @@ void InnerLoopVectorizer::introduceCheckBlockInVPlan(BasicBlock *CheckIRBB) {
   }
 }
 
-void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
-  Value *Count = getTripCount();
-  // Reuse existing vector loop preheader for TC checks.
-  // Note that new preheader block is generated for vector loop.
-  BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
-  IRBuilder<> Builder(TCCheckBlock->getTerminator());
-
+Value *InnerLoopVectorizer::createIterationCountCheck(ElementCount VF,
+                                                      unsigned UF) const {
   // Generate code to check if the loop's trip count is less than VF * UF, or
   // equal to it in case a scalar epilogue is required; this implies that the
   // vector trip count is zero. This check also covers the case where adding one
@@ -2404,7 +2386,13 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
   auto P = Cost->requiresScalarEpilogue(VF.isVector()) ? ICmpInst::ICMP_ULE
                                                        : ICmpInst::ICMP_ULT;
 
+  // Reuse existing vector loop preheader for TC checks.
+  // Note that new preheader block is generated for vector loop.
+  BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
+  IRBuilder<> Builder(TCCheckBlock->getTerminator());
+
   // If tail is to be folded, vector loop takes care of all iterations.
+  Value *Count = getTripCount();
   Type *CountTy = Count->getType();
   Value *CheckMinIters = Builder.getFalse();
   auto CreateStep = [&]() -> Value * {
@@ -2453,7 +2441,12 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
     // Don't execute the vector loop if (UMax - n) < (VF * UF).
     CheckMinIters = Builder.CreateICmp(ICmpInst::ICMP_ULT, LHS, CreateStep());
   }
+  return CheckMinIters;
+}
 
+void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
+  BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
+  Value *CheckMinIters = createIterationCountCheck(VF, UF);
   // Create new preheader for vector loop.
   LoopVectorPreHeader = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
                                    static_cast<DominatorTree *>(nullptr), LI,
@@ -2464,7 +2457,6 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
   if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
     setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
   ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
-  LoopBypassBlocks.push_back(TCCheckBlock);
 
   assert(cast<VPIRBasicBlock>(Plan.getEntry())->getIRBasicBlock() ==
              TCCheckBlock &&
@@ -2480,10 +2472,6 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
   assert((!Cost->OptForSize ||
           Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
          "Cannot SCEV check stride or overflow when optimizing for size");
-  assert(!LoopBypassBlocks.empty() &&
-         "Should already be a bypass block due to iteration count check");
-  LoopBypassBlocks.push_back(SCEVCheckBlock);
-  AddedSafetyChecks = true;
 
   introduceCheckBlockInVPlan(SCEVCheckBlock);
   return SCEVCheckBlock;
@@ -2518,10 +2506,6 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
     });
   }
 
-  LoopBypassBlocks.push_back(MemCheckBlock);
-
-  AddedSafetyChecks = true;
-
   introduceCheckBlockInVPlan(MemCheckBlock);
   return MemCheckBlock;
 }
@@ -2567,9 +2551,9 @@ static Value *getExpandedStep(const InductionDescriptor &ID,
     return C->getValue();
   if (auto *U = dyn_cast<SCEVUnknown>(Step))
     return U->getValue();
-  auto I = ExpandedSCEVs.find(Step);
-  assert(I != ExpandedSCEVs.end() && "SCEV must be expanded at this point");
-  return I->second;
+  Value *V = ExpandedSCEVs.lookup(Step);
+  assert(V && "SCEV must be expanded at this point");
+  return V;
 }
 
 /// Knowing that loop \p L executes a single vector iteration, add instructions
@@ -2707,6 +2691,20 @@ static void cse(BasicBlock *BB) {
   }
 }
 
+/// This function attempts to return a value that represents the vectorization
+/// factor at runtime. For fixed-width VFs we know this precisely at compile
+/// time, but for scalable VFs we calculate it based on an estimate of the
+/// vscale value.
+static unsigned getEstimatedRuntimeVF(ElementCount VF,
+                                      std::optional<unsigned> VScale) {
+  unsigned EstimatedVF = VF.getKnownMinValue();
+  if (VF.isScalable())
+    if (VScale)
+      EstimatedVF *= *VScale;
+  assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
+  return EstimatedVF;
+}
+
 InstructionCost
 LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
                                               ElementCount VF) const {
@@ -2806,10 +2804,11 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
   //
   // For scalable vectorization we can't know at compile time how many
   // iterations of the loop are handled in one vector iteration, so instead
-  // assume a pessimistic vscale of '1'.
+  // use the value of vscale used for tuning.
   Loop *VectorLoop = LI->getLoopFor(HeaderBB);
-  setProfileInfoAfterUnrolling(OrigLoop, VectorLoop, OrigLoop,
-                               VF.getKnownMinValue() * UF);
+  unsigned EstimatedVFxUF =
+      getEstimatedRuntimeVF(VF * UF, Cost->getVScaleForTuning());
+  setProfileInfoAfterUnrolling(OrigLoop, VectorLoop, OrigLoop, EstimatedVFxUF);
 }
 
 void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
@@ -3135,12 +3134,13 @@ LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
     // that we will create. This cost is likely to be zero. The phi node
     // cost, if any, should be scaled by the block probability because it
     // models a copy at the end of each predicated block.
-    ScalarizationCost += VF.getKnownMinValue() *
-      TTI.getCFInstrCost(Instruction::PHI, CostKind);
+    ScalarizationCost +=
+        VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
 
     // The cost of the non-predicated instruction.
-    ScalarizationCost += VF.getKnownMinValue() *
-      TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind);
+    ScalarizationCost +=
+        VF.getFixedValue() *
+        TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind);
 
     // The cost of insertelement and extractelement instructions needed for
     // scalarization.
@@ -3194,10 +3194,9 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
   if (hasIrregularType(ScalarTy, DL))
     return false;
 
-  // For scalable vectors, the only interleave factor currently supported
-  // must be power of 2 since we require the (de)interleave2 intrinsics
-  // instead of shufflevectors.
-  if (VF.isScalable() && !isPowerOf2_32(InterleaveFactor))
+  // For scalable vectors, the interleave factors must be <= 8 since we require
+  // the (de)interleaveN intrinsics instead of shufflevectors.
+  if (VF.isScalable() && InterleaveFactor > 8)
     return false;
 
   // If the group involves a non-integral pointer, we may not be able to
@@ -3945,6 +3944,20 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   return FixedScalableVFPair::getNone();
 }
 
+bool LoopVectorizationCostModel::useMaxBandwidth(ElementCount VF) {
+  return useMaxBandwidth(VF.isScalable()
+                             ? TargetTransformInfo::RGK_ScalableVector
+                             : TargetTransformInfo::RGK_FixedWidthVector);
+}
+
+bool LoopVectorizationCostModel::useMaxBandwidth(
+    TargetTransformInfo::RegisterKind RegKind) {
+  return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 &&
+                               (TTI.shouldMaximizeVectorBandwidth(RegKind) ||
+                                (UseWiderVFIfCallVariantsPresent &&
+                                 Legal->hasVectorCallVariants())));
+}
+
 ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
     unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType,
     ElementCount MaxSafeVF, bool FoldTailByMasking) {
@@ -4010,10 +4023,7 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
       ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
                            : TargetTransformInfo::RGK_FixedWidthVector;
   ElementCount MaxVF = MaxVectorElementCount;
-  if (MaximizeBandwidth ||
-      (MaximizeBandwidth.getNumOccurrences() == 0 &&
-       (TTI.shouldMaximizeVectorBandwidth(RegKind) ||
-        (UseWiderVFIfCallVariantsPresent && Legal->hasVectorCallVariants())))) {
+  if (useMaxBandwidth(RegKind)) {
     auto MaxVectorElementCountMaxBW = ElementCount::get(
         llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
         ComputeScalableMaxVF);
@@ -4036,20 +4046,6 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
   return MaxVF;
 }
 
-/// This function attempts to return a value that represents the vectorization
-/// factor at runtime. For fixed-width VFs we know this precisely at compile
-/// time, but for scalable VFs we calculate it based on an estimate of the
-/// vscale value.
-static unsigned getEstimatedRuntimeVF(ElementCount VF,
-                                      std::optional<unsigned> VScale) {
-  unsigned EstimatedVF = VF.getKnownMinValue();
-  if (VF.isScalable())
-    if (VScale)
-      EstimatedVF *= *VScale;
-  assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
-  return EstimatedVF;
-}
-
 bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
                                                 const VectorizationFactor &B,
                                                 const unsigned MaxTripCount,
@@ -4309,7 +4305,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
           return NumLegalParts <= VF.getKnownMinValue();
         }
         // Two or more elements that share a register - are vectorized.
-        return NumLegalParts < VF.getKnownMinValue();
+        return NumLegalParts < VF.getFixedValue();
       };
 
       // If no def nor is a store, e.g., branches, continue - no value to check.
@@ -4343,15 +4339,6 @@ static bool hasReplicatorRegion(VPlan &Plan) {
 }
 
 #ifndef NDEBUG
-/// Estimate the register usage for \p Plan and vectorization factors in \p VFs
-/// by calculating the highest number of values that are live at a single
-/// location as a rough estimate. Returns the register usage for each VF in \p
-/// VFs.
-static SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>
-calculateRegisterUsage(VPlan &Plan, ArrayRef<ElementCount> VFs,
-                       const TargetTransformInfo &TTI,
-                       const SmallPtrSetImpl<const Value *> &ValuesToIgnore);
-
 VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
   InstructionCost ExpectedCost = CM.expectedCost(ElementCount::getFixed(1));
   LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n");
@@ -4377,15 +4364,21 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
   for (auto &P : VPlans) {
     ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
                                P->vectorFactors().end());
-    auto RUs = ::calculateRegisterUsage(*P, VFs, TTI, CM.ValuesToIgnore);
-    for (auto [VF, RU] : zip_equal(VFs, RUs)) {
+
+    SmallVector<VPRegisterUsage, 8> RUs;
+    if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) ||
+        CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector))
+      RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
+
+    for (unsigned I = 0; I < VFs.size(); I++) {
+      ElementCount VF = VFs[I];
       // The cost for scalar VF=1 is already calculated, so ignore it.
       if (VF.isScalar())
         continue;
 
       /// Don't consider the VF if it exceeds the number of registers for the
       /// target.
-      if (RU.exceedsMaxNumRegs(TTI))
+      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI))
         continue;
 
       InstructionCost C = CM.expectedCost(VF);
@@ -4603,8 +4596,8 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
         assert(!isa<SCEVCouldNotCompute>(TC) &&
                "Trip count SCEV must be computable");
         RemainingIterations = SE.getURemExpr(
-            TC, SE.getConstant(TCType, MainLoopVF.getKnownMinValue() * IC));
-        MaxTripCount = MainLoopVF.getKnownMinValue() * IC - 1;
+            TC, SE.getConstant(TCType, MainLoopVF.getFixedValue() * IC));
+        MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
         if (SE.isKnownPredicate(CmpInst::ICMP_ULT, RemainingIterations,
                                 SE.getConstant(TCType, MaxTripCount))) {
           MaxTripCount =
@@ -4615,7 +4608,7 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
       }
       if (SE.isKnownPredicate(
               CmpInst::ICMP_UGT,
-              SE.getConstant(TCType, NextVF.Width.getKnownMinValue()),
+              SE.getConstant(TCType, NextVF.Width.getFixedValue()),
               RemainingIterations))
         continue;
     }
@@ -4704,254 +4697,6 @@ void LoopVectorizationCostModel::collectElementTypesForWidening() {
   }
 }
 
-/// Get the VF scaling factor applied to the recipe's output, if the recipe has
-/// one.
-static unsigned getVFScaleFactor(VPRecipeBase *R) {
-  if (auto *RR = dyn_cast<VPReductionPHIRecipe>(R))
-    return RR->getVFScaleFactor();
-  if (auto *RR = dyn_cast<VPPartialReductionRecipe>(R))
-    return RR->getVFScaleFactor();
-  return 1;
-}
-
-/// Estimate the register usage for \p Plan and vectorization factors in \p VFs
-/// by calculating the highest number of values that are live at a single
-/// location as a rough estimate. Returns the register usage for each VF in \p
-/// VFs.
-static SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>
-calculateRegisterUsage(VPlan &Plan, ArrayRef<ElementCount> VFs,
-                       const TargetTransformInfo &TTI,
-                       const SmallPtrSetImpl<const Value *> &ValuesToIgnore) {
-  // Each 'key' in the map opens a new interval. The values
-  // of the map are the index of the 'last seen' usage of the
-  // recipe that is the key.
-  using IntervalMap = SmallDenseMap<VPRecipeBase *, unsigned, 16>;
-
-  // Maps indices to recipes.
-  SmallVector<VPRecipeBase *, 64> Idx2Recipe;
-  // Marks the end of each interval.
-  IntervalMap EndPoint;
-  // Saves the list of recipe indices that are used in the loop.
-  SmallPtrSet<VPRecipeBase *, 8> Ends;
-  // Saves the list of values that are used in the loop but are defined outside
-  // the loop (not including non-recipe values such as arguments and
-  // constants).
-  SmallSetVector<VPValue *, 8> LoopInvariants;
-  LoopInvariants.insert(&Plan.getVectorTripCount());
-
-  // We scan the loop in a topological order in order and assign a number to
-  // each recipe. We use RPO to ensure that defs are met before their users. We
-  // assume that each recipe that has in-loop users starts an interval. We
-  // record every time that an in-loop value is used, so we have a list of the
-  // first and last occurrences of each recipe.
-  ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
-      Plan.getVectorLoopRegion());
-  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
-    if (!VPBB->getParent())
-      break;
-    for (VPRecipeBase &R : *VPBB) {
-      Idx2Recipe.push_back(&R);
-
-      // Save the end location of each USE.
-      for (VPValue *U : R.operands()) {
-        auto *DefR = U->getDefiningRecipe();
-
-        // Ignore non-recipe values such as arguments, constants, etc.
-        // FIXME: Might need some motivation why these values are ignored. If
-        // for example an argument is used inside the loop it will increase the
-        // register pressure (so shouldn't we add it to LoopInvariants).
-        if (!DefR && (!U->getLiveInIRValue() ||
-                      !isa<Instruction>(U->getLiveInIRValue())))
-          continue;
-
-        // If this recipe is outside the loop then record it and continue.
-        if (!DefR) {
-          LoopInvariants.insert(U);
-          continue;
-        }
-
-        // Overwrite previous end points.
-        EndPoint[DefR] = Idx2Recipe.size();
-        Ends.insert(DefR);
-      }
-    }
-    if (VPBB == Plan.getVectorLoopRegion()->getExiting()) {
-      // VPWidenIntOrFpInductionRecipes are used implicitly at the end of the
-      // exiting block, where their increment will get materialized eventually.
-      for (auto &R : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
-        if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
-          EndPoint[&R] = Idx2Recipe.size();
-          Ends.insert(&R);
-        }
-      }
-    }
-  }
-
-  // Saves the list of intervals that end with the index in 'key'.
-  using RecipeList = SmallVector<VPRecipeBase *, 2>;
-  SmallDenseMap<unsigned, RecipeList, 16> TransposeEnds;
-
-  // Next, we transpose the EndPoints into a multi map that holds the list of
-  // intervals that *end* at a specific location.
-  for (auto &Interval : EndPoint)
-    TransposeEnds[Interval.second].push_back(Interval.first);
-
-  SmallPtrSet<VPRecipeBase *, 8> OpenIntervals;
-  SmallVector<LoopVectorizationCostModel::RegisterUsage, 8> RUs(VFs.size());
-  SmallVector<SmallMapVector<unsigned, unsigned, 4>, 8> MaxUsages(VFs.size());
-
-  LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n");
-
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
-
-  const auto &TTICapture = TTI;
-  auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) -> unsigned {
-    if (Ty->isTokenTy() || !VectorType::isValidElementType(Ty) ||
-        (VF.isScalable() &&
-         !TTICapture.isElementTypeLegalForScalableVector(Ty)))
-      return 0;
-    return TTICapture.getRegUsageForType(VectorType::get(Ty, VF));
-  };
-
-  // We scan the instructions linearly and record each time that a new interval
-  // starts, by placing it in a set. If we find this value in TransposEnds then
-  // we remove it from the set. The max register usage is the maximum register
-  // usage of the recipes of the set.
-  for (unsigned int Idx = 0, Sz = Idx2Recipe.size(); Idx < Sz; ++Idx) {
-    VPRecipeBase *R = Idx2Recipe[Idx];
-
-    // Remove all of the recipes that end at this location.
-    RecipeList &List = TransposeEnds[Idx];
-    for (VPRecipeBase *ToRemove : List)
-      OpenIntervals.erase(ToRemove);
-
-    // Ignore recipes that are never used within the loop and do not have side
-    // effects.
-    if (!Ends.count(R) && !R->mayHaveSideEffects())
-      continue;
-
-    // Skip recipes for ignored values.
-    // TODO: Should mark recipes for ephemeral values that cannot be removed
-    // explictly in VPlan.
-    if (isa<VPSingleDefRecipe>(R) &&
-        ValuesToIgnore.contains(
-            cast<VPSingleDefRecipe>(R)->getUnderlyingValue()))
-      continue;
-
-    // For each VF find the maximum usage of registers.
-    for (unsigned J = 0, E = VFs.size(); J < E; ++J) {
-      // Count the number of registers used, per register class, given all open
-      // intervals.
-      // Note that elements in this SmallMapVector will be default constructed
-      // as 0. So we can use "RegUsage[ClassID] += n" in the code below even if
-      // there is no previous entry for ClassID.
-      SmallMapVector<unsigned, unsigned, 4> RegUsage;
-
-      for (auto *R : OpenIntervals) {
-        // Skip recipes that weren't present in the original loop.
-        // TODO: Remove after removing the legacy
-        // LoopVectorizationCostModel::calculateRegisterUsage
-        if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe,
-                VPBranchOnMaskRecipe>(R))
-          continue;
-
-        if (VFs[J].isScalar() ||
-            isa<VPCanonicalIVPHIRecipe, VPReplicateRecipe, VPDerivedIVRecipe,
-                VPScalarIVStepsRecipe>(R) ||
-            (isa<VPInstruction>(R) &&
-             all_of(cast<VPSingleDefRecipe>(R)->users(),
-                    [&](VPUser *U) {
-                      return cast<VPRecipeBase>(U)->usesScalars(
-                          R->getVPSingleValue());
-                    })) ||
-            (isa<VPReductionPHIRecipe>(R) &&
-             (cast<VPReductionPHIRecipe>(R))->isInLoop())) {
-          unsigned ClassID = TTI.getRegisterClassForType(
-              false, TypeInfo.inferScalarType(R->getVPSingleValue()));
-          // FIXME: The target might use more than one register for the type
-          // even in the scalar case.
-          RegUsage[ClassID] += 1;
-        } else {
-          // The output from scaled phis and scaled reductions actually has
-          // fewer lanes than the VF.
-          unsigned ScaleFactor = getVFScaleFactor(R);
-          ElementCount VF = VFs[J].divideCoefficientBy(ScaleFactor);
-          LLVM_DEBUG(if (VF != VFs[J]) {
-            dbgs() << "LV(REG): Scaled down VF from " << VFs[J] << " to " << VF
-                   << " for " << *R << "\n";
-          });
-
-          for (VPValue *DefV : R->definedValues()) {
-            Type *ScalarTy = TypeInfo.inferScalarType(DefV);
-            unsigned ClassID = TTI.getRegisterClassForType(true, ScalarTy);
-            RegUsage[ClassID] += GetRegUsage(ScalarTy, VF);
-          }
-        }
-      }
-
-      for (const auto &Pair : RegUsage) {
-        auto &Entry = MaxUsages[J][Pair.first];
-        Entry = std::max(Entry, Pair.second);
-      }
-    }
-
-    LLVM_DEBUG(dbgs() << "LV(REG): At #" << Idx << " Interval # "
-                      << OpenIntervals.size() << '\n');
-
-    // Add the current recipe to the list of open intervals.
-    OpenIntervals.insert(R);
-  }
-
-  // We also search for instructions that are defined outside the loop, but are
-  // used inside the loop. We need this number separately from the max-interval
-  // usage number because when we unroll, loop-invariant values do not take
-  // more register.
-  LoopVectorizationCostModel::RegisterUsage RU;
-  for (unsigned Idx = 0, End = VFs.size(); Idx < End; ++Idx) {
-    // Note that elements in this SmallMapVector will be default constructed
-    // as 0. So we can use "Invariant[ClassID] += n" in the code below even if
-    // there is no previous entry for ClassID.
-    SmallMapVector<unsigned, unsigned, 4> Invariant;
-
-    for (auto *In : LoopInvariants) {
-      // FIXME: The target might use more than one register for the type
-      // even in the scalar case.
-      bool IsScalar = all_of(In->users(), [&](VPUser *U) {
-        return cast<VPRecipeBase>(U)->usesScalars(In);
-      });
-
-      ElementCount VF = IsScalar ? ElementCount::getFixed(1) : VFs[Idx];
-      unsigned ClassID = TTI.getRegisterClassForType(
-          VF.isVector(), TypeInfo.inferScalarType(In));
-      Invariant[ClassID] += GetRegUsage(TypeInfo.inferScalarType(In), VF);
-    }
-
-    LLVM_DEBUG({
-      dbgs() << "LV(REG): VF = " << VFs[Idx] << '\n';
-      dbgs() << "LV(REG): Found max usage: " << MaxUsages[Idx].size()
-             << " item\n";
-      for (const auto &pair : MaxUsages[Idx]) {
-        dbgs() << "LV(REG): RegisterClass: "
-               << TTI.getRegisterClassName(pair.first) << ", " << pair.second
-               << " registers\n";
-      }
-      dbgs() << "LV(REG): Found invariant usage: " << Invariant.size()
-             << " item\n";
-      for (const auto &pair : Invariant) {
-        dbgs() << "LV(REG): RegisterClass: "
-               << TTI.getRegisterClassName(pair.first) << ", " << pair.second
-               << " registers\n";
-      }
-    });
-
-    RU.LoopInvariantRegs = Invariant;
-    RU.MaxLocalUsers = MaxUsages[Idx];
-    RUs[Idx] = RU;
-  }
-
-  return RUs;
-}
-
 unsigned
 LoopVectorizationCostModel::selectInterleaveCount(VPlan &Plan, ElementCount VF,
                                                   InstructionCost LoopCost) {
@@ -5002,8 +4747,8 @@ LoopVectorizationCostModel::selectInterleaveCount(VPlan &Plan, ElementCount VF,
       return 1;
   }
 
-  RegisterUsage R =
-      ::calculateRegisterUsage(Plan, {VF}, TTI, ValuesToIgnore)[0];
+  VPRegisterUsage R =
+      calculateRegisterUsageForPlan(Plan, {VF}, TTI, ValuesToIgnore)[0];
   // We divide by these constants so assume that we have at least one
   // instruction that uses at least one register.
   for (auto &Pair : R.MaxLocalUsers) {
@@ -5534,14 +5279,14 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
 
   // Get the cost of the scalar memory instruction and address computation.
   InstructionCost Cost =
-      VF.getKnownMinValue() * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV);
+      VF.getFixedValue() * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV);
 
   // Don't pass *I here, since it is scalar but will actually be part of a
   // vectorized loop where the user of it is a vectorized instruction.
   const Align Alignment = getLoadStoreAlignment(I);
-  Cost += VF.getKnownMinValue() * TTI.getMemoryOpCost(I->getOpcode(),
-                                                      ValTy->getScalarType(),
-                                                      Alignment, AS, CostKind);
+  Cost += VF.getFixedValue() * TTI.getMemoryOpCost(I->getOpcode(),
+                                                   ValTy->getScalarType(),
+                                                   Alignment, AS, CostKind);
 
   // Get the overhead of the extractelement and insertelement instructions
   // we might create due to scalarization.
@@ -5557,7 +5302,7 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
     auto *VecI1Ty =
         VectorType::get(IntegerType::getInt1Ty(ValTy->getContext()), VF);
     Cost += TTI.getScalarizationOverhead(
-        VecI1Ty, APInt::getAllOnes(VF.getKnownMinValue()),
+        VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
         /*Insert=*/false, /*Extract=*/true, CostKind);
     Cost += TTI.getCFInstrCost(Instruction::Br, CostKind);
 
@@ -5618,6 +5363,10 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
   StoreInst *SI = cast<StoreInst>(I);
 
   bool IsLoopInvariantStoreValue = Legal->isInvariant(SI->getValueOperand());
+  // TODO: We have existing tests that request the cost of extracting element
+  // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
+  // the actual generated code, which involves extracting the last element of
+  // a scalable vector where the lane to extract is unknown at compile time.
   return TTI.getAddressComputationCost(ValTy) +
          TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS,
                              CostKind) +
@@ -5900,7 +5649,7 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
 
     for (Type *VectorTy : getContainedTypes(RetTy)) {
       Cost += TTI.getScalarizationOverhead(
-          cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getKnownMinValue()),
+          cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getFixedValue()),
           /*Insert=*/true,
           /*Extract=*/false, CostKind);
     }
@@ -7333,7 +7082,10 @@ static bool planContainsAdditionalSimplifications(VPlan &Plan,
   return any_of(TheLoop->blocks(), [&SeenInstrs, &CostCtx,
                                     TheLoop](BasicBlock *BB) {
     return any_of(*BB, [&SeenInstrs, &CostCtx, TheLoop, BB](Instruction &I) {
-      if (isa<PHINode>(&I) && BB == TheLoop->getHeader())
+      // Skip induction phis when checking for simplifications, as they may not
+      // be lowered directly be lowered to a corresponding PHI recipe.
+      if (isa<PHINode>(&I) && BB == TheLoop->getHeader() &&
+          CostCtx.CM.Legal->isInductionPhi(cast<PHINode>(&I)))
         return false;
       return !SeenInstrs.contains(&I) && !CostCtx.skipCostComputation(&I, true);
     });
@@ -7380,8 +7132,14 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
   for (auto &P : VPlans) {
     ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
                                P->vectorFactors().end());
-    auto RUs = ::calculateRegisterUsage(*P, VFs, TTI, CM.ValuesToIgnore);
-    for (auto [VF, RU] : zip_equal(VFs, RUs)) {
+
+    SmallVector<VPRegisterUsage, 8> RUs;
+    if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) ||
+        CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector))
+      RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
+
+    for (unsigned I = 0; I < VFs.size(); I++) {
+      ElementCount VF = VFs[I];
       if (VF.isScalar())
         continue;
       if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
@@ -7403,7 +7161,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
       InstructionCost Cost = cost(*P, VF);
       VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
 
-      if (RU.exceedsMaxNumRegs(TTI)) {
+      if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI)) {
         LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
                           << VF << " because it uses too many registers\n");
         continue;
@@ -7485,15 +7243,27 @@ static void addRuntimeUnrollDisableMetaData(Loop *L) {
   }
 }
 
-// If \p R is a ComputeReductionResult when vectorizing the epilog loop,
-// fix the reduction's scalar PHI node by adding the incoming value from the
-// main vector loop.
+static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
+  using namespace VPlanPatternMatch;
+  assert(RdxResult->getOpcode() == VPInstruction::ComputeFindLastIVResult &&
+         "RdxResult must be ComputeFindLastIVResult");
+  VPValue *StartVPV = RdxResult->getOperand(1);
+  match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
+  return StartVPV->getLiveInIRValue();
+}
+
+// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
+// epilog loop, fix the reduction's scalar PHI node by adding the incoming value
+// from the main vector loop.
 static void fixReductionScalarResumeWhenVectorizingEpilog(
-    VPRecipeBase *R, VPTransformState &State, BasicBlock *LoopMiddleBlock,
-    BasicBlock *BypassBlock) {
-  auto *EpiRedResult = dyn_cast<VPInstruction>(R);
+    VPPhi *EpiResumePhiR, VPTransformState &State, BasicBlock *BypassBlock) {
+  // Get the VPInstruction computing the reduction result in the middle block.
+  // The first operand may not be from the middle block if it is not connected
+  // to the scalar preheader. In that case, there's nothing to fix.
+  auto *EpiRedResult = dyn_cast<VPInstruction>(EpiResumePhiR->getOperand(0));
   if (!EpiRedResult ||
-      (EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
+      (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
+       EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
        EpiRedResult->getOpcode() != VPInstruction::ComputeFindLastIVResult))
     return;
 
@@ -7501,31 +7271,38 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
       cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
   const RecurrenceDescriptor &RdxDesc =
       EpiRedHeaderPhi->getRecurrenceDescriptor();
-  Value *MainResumeValue =
-      EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
+  Value *MainResumeValue;
+  if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
+    assert((VPI->getOpcode() == VPInstruction::Broadcast ||
+            VPI->getOpcode() == VPInstruction::ReductionStartVector) &&
+           "unexpected start recipe");
+    MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
+  } else
+    MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
   if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
           RdxDesc.getRecurrenceKind())) {
+    Value *StartV = EpiRedResult->getOperand(1)->getLiveInIRValue();
+    (void)StartV;
     auto *Cmp = cast<ICmpInst>(MainResumeValue);
     assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
            "AnyOf expected to start with ICMP_NE");
-    assert(Cmp->getOperand(1) == RdxDesc.getRecurrenceStartValue() &&
+    assert(Cmp->getOperand(1) == StartV &&
            "AnyOf expected to start by comparing main resume value to original "
            "start value");
     MainResumeValue = Cmp->getOperand(0);
   } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
                  RdxDesc.getRecurrenceKind())) {
+    Value *StartV = getStartValueFromReductionResult(EpiRedResult);
+    Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
     using namespace llvm::PatternMatch;
     Value *Cmp, *OrigResumeV, *CmpOp;
     bool IsExpectedPattern =
-        match(MainResumeValue, m_Select(m_OneUse(m_Value(Cmp)),
-                                        m_Specific(RdxDesc.getSentinelValue()),
-                                        m_Value(OrigResumeV))) &&
+        match(MainResumeValue,
+              m_Select(m_OneUse(m_Value(Cmp)), m_Specific(SentinelV),
+                       m_Value(OrigResumeV))) &&
         (match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
                                    m_Value(CmpOp))) &&
-         (match(CmpOp,
-                m_Freeze(m_Specific(RdxDesc.getRecurrenceStartValue()))) ||
-          (CmpOp == RdxDesc.getRecurrenceStartValue() &&
-           isGuaranteedNotToBeUndefOrPoison(CmpOp))));
+         ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
     assert(IsExpectedPattern && "Unexpected reduction resume pattern");
     (void)IsExpectedPattern;
     MainResumeValue = OrigResumeV;
@@ -7535,12 +7312,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
   // When fixing reductions in the epilogue loop we should already have
   // created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
   // over the incoming values correctly.
-  using namespace VPlanPatternMatch;
-  assert(count_if(EpiRedResult->users(), IsaPred<VPPhi>) == 1 &&
-         "ResumePhi must have a single user");
-  auto *EpiResumePhiVPI =
-      cast<VPInstruction>(*find_if(EpiRedResult->users(), IsaPred<VPPhi>));
-  auto *EpiResumePhi = cast<PHINode>(State.get(EpiResumePhiVPI, true));
+  auto *EpiResumePhi = cast<PHINode>(State.get(EpiResumePhiR, true));
   EpiResumePhi->setIncomingValueForBlock(
       BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
 }
@@ -7558,6 +7330,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
                            OrigLoop->getHeader()->getContext());
   VPlanTransforms::runPass(VPlanTransforms::materializeBroadcasts, BestVPlan);
+  if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
+    VPlanTransforms::runPass(VPlanTransforms::addBranchWeightToMiddleTerminator,
+                             BestVPlan, BestVF);
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
   VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
   VPlanTransforms::narrowInterleaveGroups(
@@ -7565,14 +7340,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
       TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector));
   VPlanTransforms::removeDeadRecipes(BestVPlan);
 
-  // Retrieve and store the middle block before dissolving regions. Regions are
-  // dissolved after optimizing for VF and UF, which completely removes unneeded
-  // loop regions first.
-  VPBasicBlock *MiddleVPBB =
-      BestVPlan.getVectorLoopRegion() ? BestVPlan.getMiddleBlock() : nullptr;
-  VPlanTransforms::dissolveLoopRegions(BestVPlan);
   VPlanTransforms::convertToConcreteRecipes(BestVPlan,
                                             *Legal->getWidestInductionType());
+  // Regions are dissolved after optimizing for VF and UF, which completely
+  // removes unneeded loop regions first.
+  VPlanTransforms::dissolveLoopRegions(BestVPlan);
   // Perform the actual loop transformation.
   VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
                          OrigLoop->getParentLoop(),
@@ -7649,18 +7421,12 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
       }
     }
     VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader();
-    ArrayRef<VPBlockBase *> ScalarPreds = ScalarPH->getPredecessors();
-    if (!ScalarPreds.empty()) {
+    if (ScalarPH->getNumPredecessors() > 0) {
       // If ScalarPH has predecessors, we may need to update its reduction
-      // resume values. If there is a middle block, it must be the first
-      // predecessor. Note that the first predecessor may not be the middle
-      // block, if the middle block doesn't branch to the scalar preheader. In
-      // that case, fixReductionScalarResumeWhenVectorizingEpilog will be a
-      // no-op.
-      auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPreds[0]);
-      for (VPRecipeBase &R : *MiddleVPBB) {
-        fixReductionScalarResumeWhenVectorizingEpilog(
-            &R, State, State.CFG.VPBB2IRBB[MiddleVPBB], BypassBlock);
+      // resume values.
+      for (VPRecipeBase &R : ScalarPH->phis()) {
+        fixReductionScalarResumeWhenVectorizingEpilog(cast<VPPhi>(&R), State,
+                                                      BypassBlock);
       }
     }
   }
@@ -7720,20 +7486,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
 
   ILV.printDebugTracesAtEnd();
 
-  // 4. Adjust branch weight of the branch in the middle block.
-  if (HeaderVPBB) {
-    auto *MiddleTerm =
-        cast<BranchInst>(State.CFG.VPBB2IRBB[MiddleVPBB]->getTerminator());
-    if (MiddleTerm->isConditional() &&
-        hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
-      // Assume that `Count % VectorTripCount` is equally distributed.
-      unsigned TripCount = BestVPlan.getUF() * State.VF.getKnownMinValue();
-      assert(TripCount > 0 && "trip count should not be zero");
-      const uint32_t Weights[] = {1, TripCount - 1};
-      setBranchWeights(*MiddleTerm, Weights, /*IsExpected=*/false);
-    }
-  }
-
   return ExpandedSCEVs;
 }
 
@@ -7798,25 +7550,12 @@ BasicBlock *
 EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
                                                     bool ForEpilogue) {
   assert(Bypass && "Expected valid bypass basic block.");
-  ElementCount VFactor = ForEpilogue ? EPI.EpilogueVF : VF;
-  unsigned UFactor = ForEpilogue ? EPI.EpilogueUF : UF;
   Value *Count = getTripCount();
-  // Reuse existing vector loop preheader for TC checks.
-  // Note that new preheader block is generated for vector loop.
-  BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
-  IRBuilder<> Builder(TCCheckBlock->getTerminator());
-
-  // Generate code to check if the loop's trip count is less than VF * UF of the
-  // main vector loop.
-  auto P = Cost->requiresScalarEpilogue(ForEpilogue ? EPI.EpilogueVF.isVector()
-                                                    : VF.isVector())
-               ? ICmpInst::ICMP_ULE
-               : ICmpInst::ICMP_ULT;
-
-  Value *CheckMinIters = Builder.CreateICmp(
-      P, Count, createStepForVF(Builder, Count->getType(), VFactor, UFactor),
-      "min.iters.check");
+  MinProfitableTripCount = ElementCount::getFixed(0);
+  Value *CheckMinIters = createIterationCountCheck(
+      ForEpilogue ? EPI.EpilogueVF : VF, ForEpilogue ? EPI.EpilogueUF : UF);
 
+  BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
   if (!ForEpilogue)
     TCCheckBlock->setName("vector.main.loop.iter.check");
 
@@ -7826,8 +7565,6 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
                                    nullptr, "vector.ph");
 
   if (ForEpilogue) {
-    LoopBypassBlocks.push_back(TCCheckBlock);
-
     // Save the trip count so we don't have to regenerate it in the
     // vec.epilog.iter.check. This is safe to do because the trip count
     // generated here dominates the vector epilog iter check.
@@ -7888,13 +7625,6 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
 
   DT->changeImmediateDominator(LoopScalarPreHeader,
                                EPI.EpilogueIterationCountCheck);
-  // Keep track of bypass blocks, as they feed start values to the induction and
-  // reduction phis in the scalar loop preheader.
-  if (EPI.SCEVSafetyCheck)
-    LoopBypassBlocks.push_back(EPI.SCEVSafetyCheck);
-  if (EPI.MemSafetyCheck)
-    LoopBypassBlocks.push_back(EPI.MemSafetyCheck);
-  LoopBypassBlocks.push_back(EPI.EpilogueIterationCountCheck);
 
   // The vec.epilog.iter.check block may contain Phi nodes from inductions or
   // reductions which merge control-flow from the latch block and the middle
@@ -7965,7 +7695,6 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
     setBranchWeights(BI, Weights, /*IsExpected=*/false);
   }
   ReplaceInstWithInst(Insert->getTerminator(), &BI);
-  LoopBypassBlocks.push_back(Insert);
 
   // A new entry block has been created for the epilogue VPlan. Hook it in, as
   // otherwise we would try to modify the entry to the main vector loop.
@@ -8436,7 +8165,7 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
     PartialReductionChain Chain = Pair.first;
     if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
         ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB))
-      ScaledReductionMap.insert(std::make_pair(Chain.Reduction, Pair.second));
+      ScaledReductionMap.try_emplace(Chain.Reduction, Pair.second);
   }
 }
 
@@ -8508,12 +8237,11 @@ bool VPRecipeBuilder::getScaledReductions(
           [&](ElementCount VF) {
             InstructionCost Cost = TTI->getPartialReductionCost(
                 Update->getOpcode(), A->getType(), B->getType(), PHI->getType(),
-                VF, OpAExtend, OpBExtend,
-                std::make_optional(BinOp->getOpcode()));
+                VF, OpAExtend, OpBExtend, BinOp->getOpcode());
             return Cost.isValid();
           },
           Range)) {
-    Chains.push_back(std::make_pair(Chain, TargetScaleFactor));
+    Chains.emplace_back(Chain, TargetScaleFactor);
     return true;
   }
 
@@ -8996,10 +8724,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
       bool Result = (VF.isVector() && // Query is illegal for VF == 1
                      CM.getWideningDecision(IG->getInsertPos(), VF) ==
                          LoopVectorizationCostModel::CM_Interleave);
-      // For scalable vectors, the only interleave factor currently supported
-      // must be power of 2 since we require the (de)interleave2 intrinsics
-      // instead of shufflevectors.
-      assert((!Result || !VF.isScalable() || isPowerOf2_32(IG->getFactor())) &&
+      // For scalable vectors, the interleave factors must be <= 8 since we
+      // require the (de)interleaveN intrinsics instead of shufflevectors.
+      assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
              "Unsupported interleave factor for scalable vectors");
       return Result;
     };
@@ -9439,7 +9166,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       continue;
 
     const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    Type *PhiTy = PhiR->getOperand(0)->getLiveInIRValue()->getType();
+    Type *PhiTy = PhiR->getUnderlyingValue()->getType();
     // If tail is folded by masking, introduce selects between the phi
     // and the users outside the vector region of each reduction, at the
     // beginning of the dedicated latch block.
@@ -9460,6 +9187,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
         return isa<VPInstruction>(&U) &&
                (cast<VPInstruction>(&U)->getOpcode() ==
+                    VPInstruction::ComputeAnyOfResult ||
+                cast<VPInstruction>(&U)->getOpcode() ==
                     VPInstruction::ComputeReductionResult ||
                 cast<VPInstruction>(&U)->getOpcode() ==
                     VPInstruction::ComputeFindLastIVResult);
@@ -9509,8 +9238,15 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
             RdxDesc.getRecurrenceKind())) {
       VPValue *Start = PhiR->getStartValue();
+      VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
       FinalReductionResult =
           Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
+                               {PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
+    } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
+                   RdxDesc.getRecurrenceKind())) {
+      VPValue *Start = PhiR->getStartValue();
+      FinalReductionResult =
+          Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
                                {PhiR, Start, NewExitingVPV}, ExitDL);
     } else {
       VPIRFlags Flags = RecurrenceDescriptor::isFloatingPointRecurrenceKind(
@@ -9569,6 +9305,27 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       // start value.
       PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
     }
+    RecurKind RK = RdxDesc.getRecurrenceKind();
+    if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) &&
+         !RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
+         !RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))) {
+      VPBuilder PHBuilder(Plan->getVectorPreheader());
+      VPValue *Iden = Plan->getOrAddLiveIn(
+          getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
+      // If the PHI is used by a partial reduction, set the scale factor.
+      unsigned ScaleFactor =
+          RecipeBuilder.getScalingForReduction(RdxDesc.getLoopExitInstr())
+              .value_or(1);
+      Type *I32Ty = IntegerType::getInt32Ty(PhiTy->getContext());
+      auto *ScaleFactorVPV =
+          Plan->getOrAddLiveIn(ConstantInt::get(I32Ty, ScaleFactor));
+      VPValue *StartV = PHBuilder.createNaryOp(
+          VPInstruction::ReductionStartVector,
+          {PhiR->getStartValue(), Iden, ScaleFactorVPV},
+          PhiTy->isFloatingPointTy() ? RdxDesc.getFastMathFlags()
+                                     : FastMathFlags());
+      PhiR->setOperand(0, StartV);
+    }
   }
   for (VPRecipeBase *R : ToDelete)
     R->eraseFromParent();
@@ -9999,25 +9756,16 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
       // VPlan.
       // FIXME: Improve modeling for canonical IV start values in the epilogue
       // loop.
-      BasicBlock *MainMiddle = find_singleton<BasicBlock>(
-          predecessors(L->getLoopPreheader()),
-          [&EPI](BasicBlock *BB, bool) -> BasicBlock * {
-            if (BB != EPI.MainLoopIterationCountCheck &&
-                BB != EPI.EpilogueIterationCountCheck &&
-                BB != EPI.SCEVSafetyCheck && BB != EPI.MemSafetyCheck)
-              return BB;
-            return nullptr;
-          });
       using namespace llvm::PatternMatch;
       Type *IdxTy = IV->getScalarType();
       PHINode *EPResumeVal = find_singleton<PHINode>(
           L->getLoopPreheader()->phis(),
-          [&EPI, IdxTy, MainMiddle](PHINode &P, bool) -> PHINode * {
+          [&EPI, IdxTy](PHINode &P, bool) -> PHINode * {
             if (P.getType() == IdxTy &&
-                P.getIncomingValueForBlock(MainMiddle) == EPI.VectorTripCount &&
                 match(
                     P.getIncomingValueForBlock(EPI.MainLoopIterationCountCheck),
-                    m_SpecificInt(0)))
+                    m_SpecificInt(0)) &&
+                is_contained(P.incoming_values(), EPI.VectorTripCount))
               return &P;
             return nullptr;
           });
@@ -10040,23 +9788,37 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
     Value *ResumeV = nullptr;
     // TODO: Move setting of resume values to prepareToExecute.
     if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
+      auto *RdxResult =
+          cast<VPInstruction>(*find_if(ReductionPhi->users(), [](VPUser *U) {
+            auto *VPI = dyn_cast<VPInstruction>(U);
+            return VPI &&
+                   (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
+                    VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
+                    VPI->getOpcode() == VPInstruction::ComputeFindLastIVResult);
+          }));
       ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
                     ->getIncomingValueForBlock(L->getLoopPreheader());
       const RecurrenceDescriptor &RdxDesc =
           ReductionPhi->getRecurrenceDescriptor();
       RecurKind RK = RdxDesc.getRecurrenceKind();
       if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
+        Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
+        assert(RdxDesc.getRecurrenceStartValue() == StartV &&
+               "start value from ComputeAnyOfResult must match");
+
         // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
         // start value; compare the final value from the main vector loop
         // to the start value.
         BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
-        ResumeV =
-            Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue());
+        ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
       } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
-        ToFrozen[RdxDesc.getRecurrenceStartValue()] =
-            cast<PHINode>(ResumeV)->getIncomingValueForBlock(
-                EPI.MainLoopIterationCountCheck);
+        Value *StartV = getStartValueFromReductionResult(RdxResult);
+        assert(RdxDesc.getRecurrenceStartValue() == StartV &&
+               "start value from ComputeFindLastIVResult must match");
+
+        ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
+            EPI.MainLoopIterationCountCheck);
 
         // VPReductionPHIRecipe for FindLastIV reductions requires an adjustment
         // to the resume value. The resume value is adjusted to the sentinel
@@ -10066,10 +9828,18 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
         // variable.
         BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
-        Value *Cmp = Builder.CreateICmpEQ(
-            ResumeV, ToFrozen[RdxDesc.getRecurrenceStartValue()]);
-        ResumeV =
-            Builder.CreateSelect(Cmp, RdxDesc.getSentinelValue(), ResumeV);
+        Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
+        Value *Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
+        ResumeV = Builder.CreateSelect(Cmp, Sentinel, ResumeV);
+      } else {
+        VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
+        auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
+        if (auto *VPI = dyn_cast<VPInstruction>(PhiR->getStartValue())) {
+          assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
+                 "unexpected start value");
+          VPI->setOperand(0, StartVal);
+          continue;
+        }
       }
     } else {
       // Retrieve the induction resume values for wide inductions from
@@ -10105,8 +9875,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
     auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
     if (!ExpandR)
       continue;
-    auto *ExpandedVal =
-        Plan.getOrAddLiveIn(ExpandedSCEVs.find(ExpandR->getSCEV())->second);
+    VPValue *ExpandedVal =
+        Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR->getSCEV()));
     ExpandR->replaceAllUsesWith(ExpandedVal);
     if (Plan.getTripCount() == ExpandR)
       Plan.resetTripCount(ExpandedVal);
@@ -10364,9 +10134,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   bool VectorizeLoop = true, InterleaveLoop = true;
   if (VF.Width.isScalar()) {
     LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
-    VecDiagMsg = std::make_pair(
+    VecDiagMsg = {
         "VectorizationNotBeneficial",
-        "the cost-model indicates that vectorization is not beneficial");
+        "the cost-model indicates that vectorization is not beneficial"};
     VectorizeLoop = false;
   }
 
@@ -10375,16 +10145,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     // requested.
     LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
                          "interleaving should be avoided up front\n");
-    IntDiagMsg = std::make_pair(
-        "InterleavingAvoided",
-        "Ignoring UserIC, because interleaving was avoided up front");
+    IntDiagMsg = {"InterleavingAvoided",
+                  "Ignoring UserIC, because interleaving was avoided up front"};
     InterleaveLoop = false;
   } else if (IC == 1 && UserIC <= 1) {
     // Tell the user interleaving is not beneficial.
     LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
-    IntDiagMsg = std::make_pair(
+    IntDiagMsg = {
         "InterleavingNotBeneficial",
-        "the cost-model indicates that interleaving is not beneficial");
+        "the cost-model indicates that interleaving is not beneficial"};
     InterleaveLoop = false;
     if (UserIC == 1) {
       IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
@@ -10395,10 +10164,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     // Tell the user interleaving is beneficial, but it explicitly disabled.
     LLVM_DEBUG(
         dbgs() << "LV: Interleaving is beneficial but is explicitly disabled.");
-    IntDiagMsg = std::make_pair(
-        "InterleavingBeneficialButDisabled",
-        "the cost-model indicates that interleaving is beneficial "
-        "but is explicitly disabled or interleave count is set to 1");
+    IntDiagMsg = {"InterleavingBeneficialButDisabled",
+                  "the cost-model indicates that interleaving is beneficial "
+                  "but is explicitly disabled or interleave count is set to 1"};
     InterleaveLoop = false;
   }
 
@@ -10408,10 +10176,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
     LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
                       << "to histogram operations.\n");
-    IntDiagMsg = std::make_pair(
+    IntDiagMsg = {
         "HistogramPreventsScalarInterleaving",
         "Unable to interleave without vectorization due to constraints on "
-        "the order of histogram operations");
+        "the order of histogram operations"};
     InterleaveLoop = false;
   }
 
@@ -10530,7 +10298,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         }
         ++LoopsEpilogueVectorized;
 
-        if (!MainILV.areSafetyChecksAdded())
+        if (!Checks.hasChecks())
           DisableRuntimeUnroll = true;
       } else {
         InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width,
@@ -10542,7 +10310,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         // Add metadata to disable runtime unrolling a scalar loop when there
         // are no runtime checks about strides and memory. A scalar loop that is
         // rarely used is not worth unrolling.
-        if (!LB.areSafetyChecksAdded())
+        if (!Checks.hasChecks())
           DisableRuntimeUnroll = true;
       }
       // Report the vectorization decision.
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3e7f4cb56c0b..c3ca22dce0cc 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6885,8 +6885,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
     int Sz = TE.Scalars.size();
     if (isSplat(TE.Scalars) && !allConstant(TE.Scalars) &&
         count_if(TE.Scalars, IsaPred<UndefValue>) == Sz - 1) {
-      const auto *It =
-          find_if(TE.Scalars, [](Value *V) { return !isConstant(V); });
+      const auto *It = find_if_not(TE.Scalars, isConstant);
       if (It == TE.Scalars.begin())
         return OrdersType();
       auto *Ty = getWidenedType(TE.Scalars.front()->getType(), Sz);
@@ -10488,7 +10487,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
       } else if (ShuffleOrOp == Instruction::SIToFP ||
                  ShuffleOrOp == Instruction::UIToFP) {
         unsigned NumSignBits =
-            ComputeNumSignBits(VL0->getOperand(0), *DL, 0, AC, nullptr, DT);
+            ComputeNumSignBits(VL0->getOperand(0), *DL, AC, nullptr, DT);
         if (auto *OpI = dyn_cast<Instruction>(VL0->getOperand(0))) {
           APInt Mask = DB->getDemandedBits(OpI);
           NumSignBits = std::max(NumSignBits, Mask.countl_zero());
@@ -10532,12 +10531,12 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
       buildTreeRec(Operands.back(), Depth + 1, {TE, 1});
       if (ShuffleOrOp == Instruction::ICmp) {
         unsigned NumSignBits0 =
-            ComputeNumSignBits(VL0->getOperand(0), *DL, 0, AC, nullptr, DT);
+            ComputeNumSignBits(VL0->getOperand(0), *DL, AC, nullptr, DT);
         if (NumSignBits0 * 2 >=
             DL->getTypeSizeInBits(VL0->getOperand(0)->getType()))
           ExtraBitWidthNodes.insert(getOperandEntry(TE, 0)->Idx);
         unsigned NumSignBits1 =
-            ComputeNumSignBits(VL0->getOperand(1), *DL, 0, AC, nullptr, DT);
+            ComputeNumSignBits(VL0->getOperand(1), *DL, AC, nullptr, DT);
         if (NumSignBits1 * 2 >=
             DL->getTypeSizeInBits(VL0->getOperand(1)->getType()))
           ExtraBitWidthNodes.insert(getOperandEntry(TE, 1)->Idx);
@@ -15321,13 +15320,11 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
         if (TEUseEI.UserTE->State == TreeEntry::Vectorize &&
             (TEUseEI.UserTE->getOpcode() != Instruction::PHI ||
              TEUseEI.UserTE->isAltShuffle()) &&
-            all_of(TEUseEI.UserTE->Scalars,
-                   [](Value *V) { return isUsedOutsideBlock(V); })) {
+            all_of(TEUseEI.UserTE->Scalars, isUsedOutsideBlock)) {
           if (UseEI.UserTE->State != TreeEntry::Vectorize ||
               (UseEI.UserTE->getOpcode() == Instruction::PHI &&
                !UseEI.UserTE->isAltShuffle()) ||
-              any_of(UseEI.UserTE->Scalars,
-                     [](Value *V) { return !isUsedOutsideBlock(V); }))
+              !all_of(UseEI.UserTE->Scalars, isUsedOutsideBlock))
             continue;
         }
 
@@ -17437,6 +17434,12 @@ static Instruction *propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
   return llvm::propagateMetadata(Inst, Insts);
 }
 
+static DebugLoc getDebugLocFromPHI(PHINode &PN) {
+  if (DebugLoc DL = PN.getDebugLoc())
+    return DL;
+  return DebugLoc::getUnknown();
+}
+
 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
   IRBuilderBase::InsertPointGuard Guard(Builder);
 
@@ -17602,14 +17605,14 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       auto *PH = cast<PHINode>(VL0);
       Builder.SetInsertPoint(PH->getParent(),
                              PH->getParent()->getFirstNonPHIIt());
-      Builder.SetCurrentDebugLocation(PH->getDebugLoc());
+      Builder.SetCurrentDebugLocation(getDebugLocFromPHI(*PH));
       PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
       Value *V = NewPhi;
 
       // Adjust insertion point once all PHI's have been generated.
       Builder.SetInsertPoint(PH->getParent(),
                              PH->getParent()->getFirstInsertionPt());
-      Builder.SetCurrentDebugLocation(PH->getDebugLoc());
+      Builder.SetCurrentDebugLocation(getDebugLocFromPHI(*PH));
 
       V = FinalShuffle(V, E);
 
@@ -17641,7 +17644,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         }
 
         Builder.SetInsertPoint(IBB->getTerminator());
-        Builder.SetCurrentDebugLocation(PH->getDebugLoc());
+        Builder.SetCurrentDebugLocation(getDebugLocFromPHI(*PH));
         Value *Vec = vectorizeOperand(E, I);
         if (VecTy != Vec->getType()) {
           assert((It != MinBWs.end() || getOperandEntry(E, I)->isGather() ||
@@ -20024,7 +20027,7 @@ bool BoUpSLP::collectValuesToDemote(
       if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL)))
         return true;
     }
-    unsigned NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT);
+    unsigned NumSignBits = ComputeNumSignBits(V, *DL, AC, nullptr, DT);
     unsigned BitWidth1 = OrigBitWidth - NumSignBits;
     if (IsSignedNode)
       ++BitWidth1;
@@ -20260,8 +20263,8 @@ bool BoUpSLP::collectValuesToDemote(
         KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
         unsigned ShiftedBits = OrigBitWidth - BitWidth;
         return AmtKnownBits.getMaxValue().ult(BitWidth) &&
-               ShiftedBits < ComputeNumSignBits(I->getOperand(0), *DL, 0, AC,
-                                                nullptr, DT);
+               ShiftedBits <
+                   ComputeNumSignBits(I->getOperand(0), *DL, AC, nullptr, DT);
       });
     };
     return TryProcessInstruction(
@@ -20324,10 +20327,10 @@ bool BoUpSLP::collectValuesToDemote(
                "Expected min/max intrinsics only.");
         unsigned SignBits = OrigBitWidth - BitWidth;
         APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth - 1);
-        unsigned Op0SignBits = ComputeNumSignBits(I->getOperand(0), *DL, 0, AC,
-                                              nullptr, DT);
-        unsigned Op1SignBits = ComputeNumSignBits(I->getOperand(1), *DL, 0, AC,
-                                              nullptr, DT);
+        unsigned Op0SignBits =
+            ComputeNumSignBits(I->getOperand(0), *DL, AC, nullptr, DT);
+        unsigned Op1SignBits =
+            ComputeNumSignBits(I->getOperand(1), *DL, AC, nullptr, DT);
         return SignBits <= Op0SignBits &&
                ((SignBits != Op0SignBits &&
                  !isKnownNonNegative(I->getOperand(0), SimplifyQuery(*DL))) ||
@@ -20346,7 +20349,7 @@ bool BoUpSLP::collectValuesToDemote(
         unsigned SignBits = OrigBitWidth - BitWidth;
         APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth - 1);
         unsigned Op0SignBits =
-            ComputeNumSignBits(I->getOperand(0), *DL, 0, AC, nullptr, DT);
+            ComputeNumSignBits(I->getOperand(0), *DL, AC, nullptr, DT);
         return SignBits <= Op0SignBits &&
                ((SignBits != Op0SignBits &&
                  !isKnownNonNegative(I->getOperand(0), SimplifyQuery(*DL))) ||
@@ -20532,7 +20535,7 @@ void BoUpSLP::computeMinimumValueSizes() {
     for (Value *Root : E.Scalars) {
       if (isa<PoisonValue>(Root))
         continue;
-      unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT);
+      unsigned NumSignBits = ComputeNumSignBits(Root, *DL, AC, nullptr, DT);
       TypeSize NumTypeBits =
           DL->getTypeSizeInBits(Root->getType()->getScalarType());
       unsigned BitWidth1 = NumTypeBits - NumSignBits;
@@ -20623,7 +20626,7 @@ void BoUpSLP::computeMinimumValueSizes() {
       for (Value *V : *UserIgnoreList) {
         if (isa<PoisonValue>(V))
           continue;
-        unsigned NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT);
+        unsigned NumSignBits = ComputeNumSignBits(V, *DL, AC, nullptr, DT);
         TypeSize NumTypeBits = DL->getTypeSizeInBits(V->getType());
         unsigned BitWidth1 = NumTypeBits - NumSignBits;
         if (!isKnownNonNegative(V, SimplifyQuery(*DL)))
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Interval.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Interval.cpp
index b2bfbf5c2e8b..e651e92f8b27 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Interval.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Interval.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h"
 #include "llvm/SandboxIR/Instruction.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h"
 
@@ -42,7 +43,7 @@ template <typename T> void Interval<T>::print(raw_ostream &OS) const {
 template <typename T> void Interval<T>::dump() const { print(dbgs()); }
 #endif
 
-template class Interval<Instruction>;
-template class Interval<MemDGNode>;
+template class LLVM_EXPORT_TEMPLATE Interval<Instruction>;
+template class LLVM_EXPORT_TEMPLATE Interval<MemDGNode>;
 
 } // namespace llvm::sandboxir
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 9781d4cf146e..86dbd2171a56 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -278,13 +278,14 @@ void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
 }
 
 Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
-                                  ArrayRef<Value *> UserBndl, unsigned Depth) {
+                                  ArrayRef<Value *> UserBndl, unsigned Depth,
+                                  LegalityAnalysis &Legality) {
   bool StopForDebug =
       DebugBndlCnt++ >= StopBundle && StopBundle != StopBundleDisabled;
   LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
              VecUtils::dump(Bndl));
-  const auto &LegalityRes = StopForDebug ? Legality->getForcedPackForDebugging()
-                                         : Legality->canVectorize(Bndl);
+  const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
+                                         : Legality.canVectorize(Bndl);
   LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
   auto ActionPtr =
       std::make_unique<Action>(&LegalityRes, Bndl, UserBndl, Depth);
@@ -297,14 +298,16 @@ Action *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
       break;
     case Instruction::Opcode::Store: {
       // Don't recurse towards the pointer operand.
-      Action *OpA = vectorizeRec(getOperand(Bndl, 0), Bndl, Depth + 1);
+      Action *OpA =
+          vectorizeRec(getOperand(Bndl, 0), Bndl, Depth + 1, Legality);
       Operands.push_back(OpA);
       break;
     }
     default:
       // Visit all operands.
       for (auto OpIdx : seq<unsigned>(I->getNumOperands())) {
-        Action *OpA = vectorizeRec(getOperand(Bndl, OpIdx), Bndl, Depth + 1);
+        Action *OpA =
+            vectorizeRec(getOperand(Bndl, OpIdx), Bndl, Depth + 1, Legality);
         Operands.push_back(OpA);
       }
       break;
@@ -476,16 +479,17 @@ Value *BottomUpVec::emitVectors() {
   return NewVec;
 }
 
-bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) {
+bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl,
+                               LegalityAnalysis &Legality) {
   Change = false;
   if (LLVM_UNLIKELY(BottomUpInvocationCnt++ >= StopAt &&
                     StopAt != StopAtDisabled))
     return false;
   DeadInstrCandidates.clear();
-  Legality->clear();
+  Legality.clear();
   Actions.clear();
   DebugBndlCnt = 0;
-  vectorizeRec(Bndl, {}, /*Depth=*/0);
+  vectorizeRec(Bndl, {}, /*Depth=*/0, Legality);
   LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "BottomUpVec: Vectorization Actions:\n";
              Actions.dump());
   emitVectors();
@@ -498,16 +502,16 @@ bool BottomUpVec::runOnRegion(Region &Rgn, const Analyses &A) {
   assert(SeedSlice.size() >= 2 && "Bad slice!");
   Function &F = *SeedSlice[0]->getParent()->getParent();
   IMaps = std::make_unique<InstrMaps>();
-  Legality = std::make_unique<LegalityAnalysis>(
-      A.getAA(), A.getScalarEvolution(), F.getParent()->getDataLayout(),
-      F.getContext(), *IMaps);
+  LegalityAnalysis Legality(A.getAA(), A.getScalarEvolution(),
+                            F.getParent()->getDataLayout(), F.getContext(),
+                            *IMaps);
 
   // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
   SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
   // Try to vectorize starting from the seed slice. The returned value
   // is true if we found vectorizable code and generated some vector
   // code for it. It does not mean that the code is profitable.
-  return tryVectorize(SeedSliceVals);
+  return tryVectorize(SeedSliceVals, Legality);
 }
 
 } // namespace sandboxir
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.cpp
new file mode 100644
index 000000000000..14a6de30bf39
--- /dev/null
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.cpp
@@ -0,0 +1,53 @@
+//===- PackReuse.cpp - A pack de-duplication pass -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+namespace llvm::sandboxir {
+
+bool PackReuse::runOnRegion(Region &Rgn, const Analyses &A) {
+  if (Rgn.empty())
+    return Change;
+  // The key to the map is the ordered operands of the pack.
+  // The value is a vector of all Pack Instrs with the same operands.
+  DenseMap<std::pair<BasicBlock *, SmallVector<Value *>>,
+           SmallVector<SmallVector<Instruction *>>>
+      PacksMap;
+  // Go over the region and look for pack patterns.
+  for (auto *I : Rgn) {
+    auto PackOpt = VecUtils::matchPack(I);
+    if (PackOpt) {
+      // TODO: For now limit pack reuse within a BB.
+      BasicBlock *BB = (*PackOpt->Instrs.front()).getParent();
+      PacksMap[{BB, PackOpt->Operands}].push_back(PackOpt->Instrs);
+    }
+  }
+  for (auto &Pair : PacksMap) {
+    auto &Packs = Pair.second;
+    if (Packs.size() <= 1)
+      continue;
+    // Sort packs by program order.
+    sort(Packs, [](const auto &PackInstrs1, const auto &PackInstrs2) {
+      return PackInstrs1.front()->comesBefore(PackInstrs2.front());
+    });
+    Instruction *TopMostPack = Packs[0].front();
+    // Replace duplicate packs with the first one.
+    for (const auto &PackInstrs :
+         make_range(std::next(Packs.begin()), Packs.end())) {
+      PackInstrs.front()->replaceAllUsesWith(TopMostPack);
+      // Delete the pack instrs bottom-up since they are now dead.
+      for (auto *PackI : PackInstrs)
+        PackI->eraseFromParent();
+    }
+    Change = true;
+  }
+  return Change;
+}
+
+} // namespace llvm::sandboxir
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index c12bd9199794..02b973926854 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -18,6 +18,7 @@
 #endif
 
 REGION_PASS("null", ::llvm::sandboxir::NullPass)
+REGION_PASS("pack-reuse", ::llvm::sandboxir::PackReuse)
 REGION_PASS("print-instruction-count", ::llvm::sandboxir::PrintInstructionCount)
 REGION_PASS("print-region", ::llvm::sandboxir::PrintRegion)
 REGION_PASS("tr-save", ::llvm::sandboxir::TransactionSave)
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
index 121a195f45ee..73d5b938261d 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.cpp
@@ -20,10 +20,11 @@ RegionsFromMetadata::RegionsFromMetadata(StringRef Pipeline)
 bool RegionsFromMetadata::runOnFunction(Function &F, const Analyses &A) {
   SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
       sandboxir::Region::createRegionsFromMD(F, A.getTTI());
+  bool Change = false;
   for (auto &R : Regions) {
-    RPM.runOnRegion(*R, A);
+    Change |= RPM.runOnRegion(*R, A);
   }
-  return false;
+  return Change;
 }
 
 } // namespace llvm::sandboxir
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index feeda65b3f6e..8432b066f966 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -2,6 +2,7 @@
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PackReuse.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintRegion.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h"
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
index ee7063437a8f..008976ac9749 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
@@ -12,6 +12,7 @@
 #include "llvm/IR/Type.h"
 #include "llvm/SandboxIR/Instruction.h"
 #include "llvm/SandboxIR/Utils.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 
 using namespace llvm;
@@ -119,8 +120,9 @@ template <typename LoadOrStoreT> void SeedContainer::insert(LoadOrStoreT *LSI) {
 }
 
 // Explicit instantiations
-template void SeedContainer::insert<LoadInst>(LoadInst *);
-template void SeedContainer::insert<StoreInst>(StoreInst *);
+template LLVM_EXPORT_TEMPLATE void SeedContainer::insert<LoadInst>(LoadInst *);
+template LLVM_EXPORT_TEMPLATE void
+SeedContainer::insert<StoreInst>(StoreInst *);
 
 #ifndef NDEBUG
 void SeedContainer::print(raw_ostream &OS) const {
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 165b57c87beb..cca3d32c0783 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -331,7 +331,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
 
   bool IsSingleScalar = vputils::isSingleScalar(Def);
 
-  VPLane LastLane(IsSingleScalar ? 0 : VF.getKnownMinValue() - 1);
+  VPLane LastLane(IsSingleScalar ? 0 : VF.getFixedValue() - 1);
   // Check if there is a scalar value for the selected lane.
   if (!hasScalarValue(Def, LastLane)) {
     // At the moment, VPWidenIntOrFpInductionRecipes, VPScalarIVStepsRecipes and
@@ -368,7 +368,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
     assert(!VF.isScalable() && "VF is assumed to be non scalable.");
     Value *Undef = PoisonValue::get(toVectorizedTy(LastInst->getType(), VF));
     set(Def, Undef);
-    for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
+    for (unsigned Lane = 0; Lane < VF.getFixedValue(); ++Lane)
       packScalarIntoVectorizedValue(Def, Lane);
     VectorValue = get(Def);
   }
@@ -395,7 +395,7 @@ void VPTransformState::setDebugLocFrom(DebugLoc DL) {
       LLVM_DEBUG(dbgs() << "Failed to create new discriminator: "
                         << DIL->getFilename() << " Line: " << DIL->getLine());
   } else
-    Builder.SetCurrentDebugLocation(DIL);
+    Builder.SetCurrentDebugLocation(DL);
 }
 
 void VPTransformState::packScalarIntoVectorizedValue(const VPValue *Def,
@@ -789,8 +789,7 @@ void VPRegionBlock::execute(VPTransformState *State) {
   ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
       Entry);
   State->Lane = VPLane(0);
-  for (unsigned Lane = 0, VF = State->VF.getKnownMinValue(); Lane < VF;
-       ++Lane) {
+  for (unsigned Lane = 0, VF = State->VF.getFixedValue(); Lane < VF; ++Lane) {
     State->Lane = VPLane(Lane, VPLane::Kind::First);
     // Visit the VPBlocks connected to \p this, starting from it.
     for (VPBlockBase *Block : RPOT) {
@@ -883,6 +882,16 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
 
 void VPRegionBlock::dissolveToCFGLoop() {
   auto *Header = cast<VPBasicBlock>(getEntry());
+  if (auto *CanIV = dyn_cast<VPCanonicalIVPHIRecipe>(&Header->front())) {
+    assert(this == getPlan()->getVectorLoopRegion() &&
+           "Canonical IV must be in the entry of the top-level loop region");
+    auto *ScalarR = VPBuilder(CanIV).createScalarPhi(
+        {CanIV->getStartValue(), CanIV->getBackedgeValue()},
+        CanIV->getDebugLoc(), "index");
+    CanIV->replaceAllUsesWith(ScalarR);
+    CanIV->eraseFromParent();
+  }
+
   VPBlockBase *Preheader = getSinglePredecessor();
   auto *ExitingLatch = cast<VPBasicBlock>(getExiting());
   VPBlockBase *Middle = getSingleSuccessor();
@@ -1006,8 +1015,6 @@ void VPlan::execute(VPTransformState *State) {
   for (VPBlockBase *Block : RPOT)
     Block->execute(State);
 
-  State->CFG.DTU.flush();
-
   VPBasicBlock *Header = vputils::getFirstLoopHeader(*this, State->VPDT);
   if (!Header)
     return;
@@ -1064,7 +1071,14 @@ void VPlan::execute(VPTransformState *State) {
 InstructionCost VPlan::cost(ElementCount VF, VPCostContext &Ctx) {
   // For now only return the cost of the vector loop region, ignoring any other
   // blocks, like the preheader or middle blocks.
-  return getVectorLoopRegion()->cost(VF, Ctx);
+  InstructionCost Cost = getVectorLoopRegion()->cost(VF, Ctx);
+
+  // If any instructions in the middle block are invalid return invalid.
+  // TODO: Remove once no VPlans with VF == vscale x 1 and first-order recurrences are created.
+  if (!getMiddleBlock()->cost(VF, Ctx).isValid())
+    return InstructionCost::getInvalid();
+
+  return Cost;
 }
 
 VPRegionBlock *VPlan::getVectorLoopRegion() {
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.h b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.h
index 44f0b6d964a6..5a3c4a514a5d 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -882,11 +882,40 @@ template <unsigned PartOpIdx> class VPUnrollPartAccessor {
   unsigned getUnrollPart(VPUser &U) const;
 };
 
+/// Helper to manage IR metadata for recipes. It filters out metadata that
+/// cannot be propagated.
+class VPIRMetadata {
+  SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+
+public:
+  VPIRMetadata() {}
+
+  /// Adds metatadata that can be preserved from the original instruction
+  /// \p I.
+  VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
+
+  /// Adds metatadata that can be preserved from the original instruction
+  /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
+  VPIRMetadata(Instruction &I, LoopVersioning *LVer);
+
+  /// Copy constructor for cloning.
+  VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
+
+  /// Add all metadata to \p I.
+  void applyMetadata(Instruction &I) const;
+
+  /// Add metadata with kind \p Kind and \p Node.
+  void addMetadata(unsigned Kind, MDNode *Node) {
+    Metadata.emplace_back(Kind, Node);
+  }
+};
+
 /// This is a concrete Recipe that models a single VPlan-level instruction.
 /// While as any Recipe it may generate a sequence of IR instructions when
 /// executed, these instructions would always form a single-def expression as
 /// the VPInstruction is also a single def-use vertex.
 class VPInstruction : public VPRecipeWithIRFlags,
+                      public VPIRMetadata,
                       public VPUnrollPartAccessor<1> {
   friend class VPlanSlp;
 
@@ -907,6 +936,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
     BranchOnCount,
     BranchOnCond,
     Broadcast,
+    ComputeAnyOfResult,
     ComputeFindLastIVResult,
     ComputeReductionResult,
     // Extracts the last lane from its operand if it is a vector, or the last
@@ -933,6 +963,10 @@ class VPInstruction : public VPRecipeWithIRFlags,
     /// Scale the first operand (vector step) by the second operand
     /// (scalar-step).  Casts both operands to the result type if needed.
     WideIVStep,
+    /// Start vector for reductions with 3 operands: the original start value,
+    /// the identity value for the reduction and an integer indicating the
+    /// scaling factor.
+    ReductionStartVector,
     // Creates a step vector starting from 0 to VF with a step of 1.
     StepVector,
 
@@ -971,7 +1005,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
   VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL = {},
                 const Twine &Name = "")
       : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
-        Opcode(Opcode), Name(Name.str()) {}
+        VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {}
 
   VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags, DebugLoc DL = {},
@@ -1263,29 +1297,6 @@ struct VPIRPhi : public VPIRInstruction, public VPPhiAccessors {
   const VPRecipeBase *getAsRecipe() const override { return this; }
 };
 
-/// Helper to manage IR metadata for recipes. It filters out metadata that
-/// cannot be propagated.
-class VPIRMetadata {
-  SmallVector<std::pair<unsigned, MDNode *>> Metadata;
-
-public:
-  VPIRMetadata() {}
-
-  /// Adds metatadata that can be preserved from the original instruction
-  /// \p I.
-  VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
-
-  /// Adds metatadata that can be preserved from the original instruction
-  /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
-  VPIRMetadata(Instruction &I, LoopVersioning *LVer);
-
-  /// Copy constructor for cloning.
-  VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
-
-  /// Add all metadata to \p I.
-  void applyMetadata(Instruction &I) const;
-};
-
 /// VPWidenRecipe is a recipe for producing a widened instruction using the
 /// opcode and operands of the recipe. This recipe covers most of the
 /// traditional vectorization cases where each recipe transforms into a
@@ -1811,9 +1822,9 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags,
 class VPHeaderPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors {
 protected:
   VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr,
-                    VPValue *Start, DebugLoc DL = {})
-      : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>({Start}), UnderlyingInstr, DL) {
-  }
+                    VPValue *Start, DebugLoc DL = DebugLoc::getUnknown())
+      : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>({Start}),
+                          UnderlyingInstr, DL) {}
 
   const VPRecipeBase *getAsRecipe() const override { return this; }
 
@@ -2109,7 +2120,7 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors {
   VPWidenPHIRecipe *clone() override {
     auto *C = new VPWidenPHIRecipe(cast<PHINode>(getUnderlyingValue()),
                                    getOperand(0), getDebugLoc(), Name);
-    for (VPValue *Op : make_range(std::next(op_begin()), op_end()))
+    for (VPValue *Op : llvm::drop_begin(operands()))
       C->addOperand(Op);
     return C;
   }
@@ -2230,7 +2241,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
   bool onlyFirstLaneUsed(const VPValue *Op) const override {
     assert(is_contained(operands(), Op) &&
            "Op must be an operand of the recipe");
-    return Op == getStartValue();
+    return isOrdered() || isInLoop();
   }
 };
 
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 926490bfad7d..76da5b0314a8 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -10,8 +10,10 @@
 #include "VPlan.h"
 #include "VPlanCFG.h"
 #include "VPlanDominatorTree.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/GenericDomTreeConstruction.h"
@@ -72,6 +74,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
   switch (Opcode) {
   case Instruction::ExtractElement:
   case Instruction::Freeze:
+  case VPInstruction::ReductionStartVector:
     return inferScalarType(R->getOperand(0));
   case Instruction::Select: {
     Type *ResTy = inferScalarType(R->getOperand(1));
@@ -87,6 +90,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
                inferScalarType(R->getOperand(1)) &&
            "different types inferred for different operands");
     return IntegerType::get(Ctx, 1);
+  case VPInstruction::ComputeAnyOfResult:
   case VPInstruction::ComputeFindLastIVResult:
   case VPInstruction::ComputeReductionResult: {
     auto *PhiR = cast<VPReductionPHIRecipe>(R->getOperand(0));
@@ -384,3 +388,256 @@ bool VPDominatorTree::properlyDominates(const VPRecipeBase *A,
 #endif
   return Base::properlyDominates(ParentA, ParentB);
 }
+
+/// Get the VF scaling factor applied to the recipe's output, if the recipe has
+/// one.
+static unsigned getVFScaleFactor(VPRecipeBase *R) {
+  if (auto *RR = dyn_cast<VPReductionPHIRecipe>(R))
+    return RR->getVFScaleFactor();
+  if (auto *RR = dyn_cast<VPPartialReductionRecipe>(R))
+    return RR->getVFScaleFactor();
+  assert(
+      (!isa<VPInstruction>(R) || cast<VPInstruction>(R)->getOpcode() !=
+                                     VPInstruction::ReductionStartVector) &&
+      "getting scaling factor of reduction-start-vector not implemented yet");
+  return 1;
+}
+
+bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI) const {
+  return any_of(MaxLocalUsers, [&TTI](auto &LU) {
+    return LU.second > TTI.getNumberOfRegisters(LU.first);
+  });
+}
+
+SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
+    VPlan &Plan, ArrayRef<ElementCount> VFs, const TargetTransformInfo &TTI,
+    const SmallPtrSetImpl<const Value *> &ValuesToIgnore) {
+  // Each 'key' in the map opens a new interval. The values
+  // of the map are the index of the 'last seen' usage of the
+  // recipe that is the key.
+  using IntervalMap = SmallDenseMap<VPRecipeBase *, unsigned, 16>;
+
+  // Maps indices to recipes.
+  SmallVector<VPRecipeBase *, 64> Idx2Recipe;
+  // Marks the end of each interval.
+  IntervalMap EndPoint;
+  // Saves the list of recipe indices that are used in the loop.
+  SmallPtrSet<VPRecipeBase *, 8> Ends;
+  // Saves the list of values that are used in the loop but are defined outside
+  // the loop (not including non-recipe values such as arguments and
+  // constants).
+  SmallSetVector<VPValue *, 8> LoopInvariants;
+  LoopInvariants.insert(&Plan.getVectorTripCount());
+
+  // We scan the loop in a topological order in order and assign a number to
+  // each recipe. We use RPO to ensure that defs are met before their users. We
+  // assume that each recipe that has in-loop users starts an interval. We
+  // record every time that an in-loop value is used, so we have a list of the
+  // first and last occurrences of each recipe.
+  ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
+      Plan.getVectorLoopRegion());
+  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+    if (!VPBB->getParent())
+      break;
+    for (VPRecipeBase &R : *VPBB) {
+      Idx2Recipe.push_back(&R);
+
+      // Save the end location of each USE.
+      for (VPValue *U : R.operands()) {
+        auto *DefR = U->getDefiningRecipe();
+
+        // Ignore non-recipe values such as arguments, constants, etc.
+        // FIXME: Might need some motivation why these values are ignored. If
+        // for example an argument is used inside the loop it will increase the
+        // register pressure (so shouldn't we add it to LoopInvariants).
+        if (!DefR && (!U->getLiveInIRValue() ||
+                      !isa<Instruction>(U->getLiveInIRValue())))
+          continue;
+
+        // If this recipe is outside the loop then record it and continue.
+        if (!DefR) {
+          LoopInvariants.insert(U);
+          continue;
+        }
+
+        // Overwrite previous end points.
+        EndPoint[DefR] = Idx2Recipe.size();
+        Ends.insert(DefR);
+      }
+    }
+    if (VPBB == Plan.getVectorLoopRegion()->getExiting()) {
+      // VPWidenIntOrFpInductionRecipes are used implicitly at the end of the
+      // exiting block, where their increment will get materialized eventually.
+      for (auto &R : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
+        if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
+          EndPoint[&R] = Idx2Recipe.size();
+          Ends.insert(&R);
+        }
+      }
+    }
+  }
+
+  // Saves the list of intervals that end with the index in 'key'.
+  using RecipeList = SmallVector<VPRecipeBase *, 2>;
+  SmallDenseMap<unsigned, RecipeList, 16> TransposeEnds;
+
+  // Next, we transpose the EndPoints into a multi map that holds the list of
+  // intervals that *end* at a specific location.
+  for (auto &Interval : EndPoint)
+    TransposeEnds[Interval.second].push_back(Interval.first);
+
+  SmallPtrSet<VPRecipeBase *, 8> OpenIntervals;
+  SmallVector<VPRegisterUsage, 8> RUs(VFs.size());
+  SmallVector<SmallMapVector<unsigned, unsigned, 4>, 8> MaxUsages(VFs.size());
+
+  LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n");
+
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+
+  const auto &TTICapture = TTI;
+  auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) -> unsigned {
+    if (Ty->isTokenTy() || !VectorType::isValidElementType(Ty) ||
+        (VF.isScalable() &&
+         !TTICapture.isElementTypeLegalForScalableVector(Ty)))
+      return 0;
+    return TTICapture.getRegUsageForType(VectorType::get(Ty, VF));
+  };
+
+  // We scan the instructions linearly and record each time that a new interval
+  // starts, by placing it in a set. If we find this value in TransposEnds then
+  // we remove it from the set. The max register usage is the maximum register
+  // usage of the recipes of the set.
+  for (unsigned int Idx = 0, Sz = Idx2Recipe.size(); Idx < Sz; ++Idx) {
+    VPRecipeBase *R = Idx2Recipe[Idx];
+
+    // Remove all of the recipes that end at this location.
+    RecipeList &List = TransposeEnds[Idx];
+    for (VPRecipeBase *ToRemove : List)
+      OpenIntervals.erase(ToRemove);
+
+    // Ignore recipes that are never used within the loop and do not have side
+    // effects.
+    if (!Ends.count(R) && !R->mayHaveSideEffects())
+      continue;
+
+    // Skip recipes for ignored values.
+    // TODO: Should mark recipes for ephemeral values that cannot be removed
+    // explictly in VPlan.
+    if (isa<VPSingleDefRecipe>(R) &&
+        ValuesToIgnore.contains(
+            cast<VPSingleDefRecipe>(R)->getUnderlyingValue()))
+      continue;
+
+    // For each VF find the maximum usage of registers.
+    for (unsigned J = 0, E = VFs.size(); J < E; ++J) {
+      // Count the number of registers used, per register class, given all open
+      // intervals.
+      // Note that elements in this SmallMapVector will be default constructed
+      // as 0. So we can use "RegUsage[ClassID] += n" in the code below even if
+      // there is no previous entry for ClassID.
+      SmallMapVector<unsigned, unsigned, 4> RegUsage;
+
+      for (auto *R : OpenIntervals) {
+        // Skip recipes that weren't present in the original loop.
+        // TODO: Remove after removing the legacy
+        // LoopVectorizationCostModel::calculateRegisterUsage
+        if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe,
+                VPBranchOnMaskRecipe>(R))
+          continue;
+
+        if (VFs[J].isScalar() ||
+            isa<VPCanonicalIVPHIRecipe, VPReplicateRecipe, VPDerivedIVRecipe,
+                VPScalarIVStepsRecipe>(R) ||
+            (isa<VPInstruction>(R) &&
+             all_of(cast<VPSingleDefRecipe>(R)->users(),
+                    [&](VPUser *U) {
+                      return cast<VPRecipeBase>(U)->usesScalars(
+                          R->getVPSingleValue());
+                    })) ||
+            (isa<VPReductionPHIRecipe>(R) &&
+             (cast<VPReductionPHIRecipe>(R))->isInLoop())) {
+          unsigned ClassID = TTI.getRegisterClassForType(
+              false, TypeInfo.inferScalarType(R->getVPSingleValue()));
+          // FIXME: The target might use more than one register for the type
+          // even in the scalar case.
+          RegUsage[ClassID] += 1;
+        } else {
+          // The output from scaled phis and scaled reductions actually has
+          // fewer lanes than the VF.
+          unsigned ScaleFactor = getVFScaleFactor(R);
+          ElementCount VF = VFs[J].divideCoefficientBy(ScaleFactor);
+          LLVM_DEBUG(if (VF != VFs[J]) {
+            dbgs() << "LV(REG): Scaled down VF from " << VFs[J] << " to " << VF
+                   << " for " << *R << "\n";
+          });
+
+          for (VPValue *DefV : R->definedValues()) {
+            Type *ScalarTy = TypeInfo.inferScalarType(DefV);
+            unsigned ClassID = TTI.getRegisterClassForType(true, ScalarTy);
+            RegUsage[ClassID] += GetRegUsage(ScalarTy, VF);
+          }
+        }
+      }
+
+      for (const auto &Pair : RegUsage) {
+        auto &Entry = MaxUsages[J][Pair.first];
+        Entry = std::max(Entry, Pair.second);
+      }
+    }
+
+    LLVM_DEBUG(dbgs() << "LV(REG): At #" << Idx << " Interval # "
+                      << OpenIntervals.size() << '\n');
+
+    // Add the current recipe to the list of open intervals.
+    OpenIntervals.insert(R);
+  }
+
+  // We also search for instructions that are defined outside the loop, but are
+  // used inside the loop. We need this number separately from the max-interval
+  // usage number because when we unroll, loop-invariant values do not take
+  // more register.
+  VPRegisterUsage RU;
+  for (unsigned Idx = 0, End = VFs.size(); Idx < End; ++Idx) {
+    // Note that elements in this SmallMapVector will be default constructed
+    // as 0. So we can use "Invariant[ClassID] += n" in the code below even if
+    // there is no previous entry for ClassID.
+    SmallMapVector<unsigned, unsigned, 4> Invariant;
+
+    for (auto *In : LoopInvariants) {
+      // FIXME: The target might use more than one register for the type
+      // even in the scalar case.
+      bool IsScalar = all_of(In->users(), [&](VPUser *U) {
+        return cast<VPRecipeBase>(U)->usesScalars(In);
+      });
+
+      ElementCount VF = IsScalar ? ElementCount::getFixed(1) : VFs[Idx];
+      unsigned ClassID = TTI.getRegisterClassForType(
+          VF.isVector(), TypeInfo.inferScalarType(In));
+      Invariant[ClassID] += GetRegUsage(TypeInfo.inferScalarType(In), VF);
+    }
+
+    LLVM_DEBUG({
+      dbgs() << "LV(REG): VF = " << VFs[Idx] << '\n';
+      dbgs() << "LV(REG): Found max usage: " << MaxUsages[Idx].size()
+             << " item\n";
+      for (const auto &pair : MaxUsages[Idx]) {
+        dbgs() << "LV(REG): RegisterClass: "
+               << TTI.getRegisterClassName(pair.first) << ", " << pair.second
+               << " registers\n";
+      }
+      dbgs() << "LV(REG): Found invariant usage: " << Invariant.size()
+             << " item\n";
+      for (const auto &pair : Invariant) {
+        dbgs() << "LV(REG): RegisterClass: "
+               << TTI.getRegisterClassName(pair.first) << ", " << pair.second
+               << " registers\n";
+      }
+    });
+
+    RU.LoopInvariantRegs = Invariant;
+    RU.MaxLocalUsers = MaxUsages[Idx];
+    RUs[Idx] = RU;
+  }
+
+  return RUs;
+}
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 941e13959c23..7bcf9dba8c31 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/IR/Type.h"
 
 namespace llvm {
@@ -27,6 +28,8 @@ struct VPWidenSelectRecipe;
 class VPReplicateRecipe;
 class VPRecipeBase;
 class VPlan;
+class Value;
+class TargetTransformInfo;
 class Type;
 
 /// An analysis for type-inference for VPValues.
@@ -70,6 +73,30 @@ class VPTypeAnalysis {
 // Collect a VPlan's ephemeral recipes (those used only by an assume).
 void collectEphemeralRecipesForVPlan(VPlan &Plan,
                                      DenseSet<VPRecipeBase *> &EphRecipes);
+
+/// A struct that represents some properties of the register usage
+/// of a loop.
+struct VPRegisterUsage {
+  /// Holds the number of loop invariant values that are used in the loop.
+  /// The key is ClassID of target-provided register class.
+  SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs;
+  /// Holds the maximum number of concurrent live intervals in the loop.
+  /// The key is ClassID of target-provided register class.
+  SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
+
+  /// Check if any of the tracked live intervals exceeds the number of
+  /// available registers for the target.
+  bool exceedsMaxNumRegs(const TargetTransformInfo &TTI) const;
+};
+
+/// Estimate the register usage for \p Plan and vectorization factors in \p VFs
+/// by calculating the highest number of values that are live at a single
+/// location as a rough estimate. Returns the register usage for each VF in \p
+/// VFs.
+SmallVector<VPRegisterUsage, 8> calculateRegisterUsageForPlan(
+    VPlan &Plan, ArrayRef<ElementCount> VFs, const TargetTransformInfo &TTI,
+    const SmallPtrSetImpl<const Value *> &ValuesToIgnore);
+
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLANANALYSIS_H
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index f2a7f16e19a7..b2535fe3aa57 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -318,6 +318,31 @@ m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
       {Op0, Op1, Op2});
 }
 
+template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
+          unsigned Opcode, bool Commutative, typename... RecipeTys>
+using Recipe4Op_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t, Op3_t>,
+                                     Opcode, Commutative, RecipeTys...>;
+
+template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
+          unsigned Opcode>
+using VPInstruction4Op_match =
+    Recipe4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode, /*Commutative*/ false,
+                    VPInstruction>;
+
+template <unsigned Opcode, typename Op0_t, typename Op1_t, typename Op2_t,
+          typename Op3_t>
+inline VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>
+m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2,
+                const Op3_t &Op3) {
+  return VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>(
+      {Op0, Op1, Op2, Op3});
+}
+template <typename Op0_t>
+inline UnaryVPInstruction_match<Op0_t, Instruction::Freeze>
+m_Freeze(const Op0_t &Op0) {
+  return m_VPInstruction<Instruction::Freeze>(Op0);
+}
+
 template <typename Op0_t>
 inline UnaryVPInstruction_match<Op0_t, VPInstruction::Not>
 m_Not(const Op0_t &Op0) {
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a4831ea7c11f..048286d7a97b 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -30,7 +30,6 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
-#include "llvm/IR/VectorBuilder.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -67,6 +66,7 @@ bool VPRecipeBase::mayWriteToMemory() const {
   case VPWidenIntrinsicSC:
     return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory();
   case VPBranchOnMaskSC:
+  case VPFirstOrderRecurrencePHISC:
   case VPScalarIVStepsSC:
   case VPPredInstPHISC:
     return false;
@@ -114,6 +114,7 @@ bool VPRecipeBase::mayReadFromMemory() const {
   case VPWidenIntrinsicSC:
     return cast<VPWidenIntrinsicRecipe>(this)->mayReadFromMemory();
   case VPBranchOnMaskSC:
+  case VPFirstOrderRecurrencePHISC:
   case VPPredInstPHISC:
   case VPScalarIVStepsSC:
   case VPWidenStoreEVLSC:
@@ -147,6 +148,7 @@ bool VPRecipeBase::mayReadFromMemory() const {
 bool VPRecipeBase::mayHaveSideEffects() const {
   switch (getVPDefID()) {
   case VPDerivedIVSC:
+  case VPFirstOrderRecurrencePHISC:
   case VPPredInstPHISC:
   case VPVectorEndPointerSC:
     return false;
@@ -410,7 +412,7 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
                              const VPIRFlags &Flags, DebugLoc DL,
                              const Twine &Name)
     : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, Flags, DL),
-      Opcode(Opcode), Name(Name.str()) {
+      VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {
   assert(flagsValidForOpcode(getOpcode()) &&
          "Set flags not supported for the provided opcode");
 }
@@ -591,7 +593,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
   }
   case VPInstruction::BranchOnCond: {
     Value *Cond = State.get(getOperand(0), VPLane(0));
-    return createCondBranch(Cond, getParent(), State);
+    auto *Br = createCondBranch(Cond, getParent(), State);
+    applyMetadata(*Br);
+    return Br;
   }
   case VPInstruction::BranchOnCount: {
     // First create the compare.
@@ -604,6 +608,34 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return Builder.CreateVectorSplat(
         State.VF, State.get(getOperand(0), /*IsScalar*/ true), "broadcast");
   }
+  case VPInstruction::ReductionStartVector: {
+    if (State.VF.isScalar())
+      return State.get(getOperand(0), true);
+    IRBuilderBase::FastMathFlagGuard FMFG(Builder);
+    Builder.setFastMathFlags(getFastMathFlags());
+    // If this start vector is scaled then it should produce a vector with fewer
+    // elements than the VF.
+    ElementCount VF = State.VF.divideCoefficientBy(
+        cast<ConstantInt>(getOperand(2)->getLiveInIRValue())->getZExtValue());
+    auto *Iden = Builder.CreateVectorSplat(VF, State.get(getOperand(1), true));
+    Constant *Zero = Builder.getInt32(0);
+    return Builder.CreateInsertElement(Iden, State.get(getOperand(0), true),
+                                       Zero);
+  }
+  case VPInstruction::ComputeAnyOfResult: {
+    // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
+    // and will be removed by breaking up the recipe further.
+    auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
+    auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
+    Value *ReducedPartRdx = State.get(getOperand(2));
+    for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx)
+      ReducedPartRdx = Builder.CreateBinOp(
+          (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(
+              RecurKind::AnyOf),
+          State.get(getOperand(Idx)), ReducedPartRdx, "bin.rdx");
+    return createAnyOfReduction(Builder, ReducedPartRdx,
+                                State.get(getOperand(1), VPLane(0)), OrigPhi);
+  }
   case VPInstruction::ComputeFindLastIVResult: {
     // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
     // and will be removed by breaking up the recipe further.
@@ -616,24 +648,23 @@ Value *VPInstruction::generate(VPTransformState &State) {
     assert(!PhiR->isInLoop() &&
            "In-loop FindLastIV reduction is not supported yet");
 
-    // The recipe's operands are the reduction phi, followed by one operand for
-    // each part of the reduction.
-    unsigned UF = getNumOperands() - 2;
-    Value *ReducedPartRdx = State.get(getOperand(2));
+    // The recipe's operands are the reduction phi, the start value, the
+    // sentinel value, followed by one operand for each part of the reduction.
+    unsigned UF = getNumOperands() - 3;
+    Value *ReducedPartRdx = State.get(getOperand(3));
     for (unsigned Part = 1; Part < UF; ++Part) {
       ReducedPartRdx = createMinMaxOp(Builder, RecurKind::SMax, ReducedPartRdx,
-                                      State.get(getOperand(2 + Part)));
+                                      State.get(getOperand(3 + Part)));
     }
 
-    return createFindLastIVReduction(Builder, ReducedPartRdx,
-                                     State.get(getOperand(1), true),
-                                     RdxDesc.getSentinelValue());
+    Value *Start = State.get(getOperand(1), true);
+    Value *Sentinel = getOperand(2)->getLiveInIRValue();
+    return createFindLastIVReduction(Builder, ReducedPartRdx, Start, Sentinel);
   }
   case VPInstruction::ComputeReductionResult: {
     // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
     // and will be removed by breaking up the recipe further.
     auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
-    auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
     // Get its reduction variable descriptor.
     const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
 
@@ -641,7 +672,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     assert(!RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
            "should be handled by ComputeFindLastIVResult");
 
-    Type *PhiTy = OrigPhi->getType();
+    Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
     // The recipe's operands are the reduction phi, followed by one operand for
     // each part of the reduction.
     unsigned UF = getNumOperands() - 1;
@@ -657,7 +688,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     // then extend the loop exit value to enable InstCombine to evaluate the
     // entire expression in the smaller type.
     // TODO: Handle this in truncateToMinBW.
-    if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
+    if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
       Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
       for (unsigned Part = 0; Part < UF; ++Part)
         RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
@@ -681,25 +712,18 @@ Value *VPInstruction::generate(VPTransformState &State) {
 
     // Create the reduction after the loop. Note that inloop reductions create
     // the target reduction in the loop using a Reduction recipe.
-    if ((State.VF.isVector() ||
-         RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) &&
-        !PhiR->isInLoop()) {
+    if (State.VF.isVector() && !PhiR->isInLoop()) {
       // TODO: Support in-order reductions based on the recurrence descriptor.
       // All ops in the reduction inherit fast-math-flags from the recurrence
       // descriptor.
-      if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK))
-        ReducedPartRdx =
-            createAnyOfReduction(Builder, ReducedPartRdx,
-                                 RdxDesc.getRecurrenceStartValue(), OrigPhi);
-      else
-        ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
+      ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
 
       // If the reduction can be performed in a smaller type, we need to extend
       // the reduction to the wider type before we branch to the original loop.
-      if (PhiTy != RdxDesc.getRecurrenceType())
+      if (ResultTy != RdxDesc.getRecurrenceType())
         ReducedPartRdx = RdxDesc.isSigned()
-                             ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
-                             : Builder.CreateZExt(ReducedPartRdx, PhiTy);
+                             ? Builder.CreateSExt(ReducedPartRdx, ResultTy)
+                             : Builder.CreateZExt(ReducedPartRdx, ResultTy);
     }
 
     return ReducedPartRdx;
@@ -816,6 +840,10 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
                                   I32Ty, {Arg0Ty, I32Ty, I1Ty});
     return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
   }
+  case VPInstruction::ExtractPenultimateElement:
+    if (VF == ElementCount::getScalable(1))
+      return InstructionCost::getInvalid();
+  LLVM_FALLTHROUGH;
   default:
     // TODO: Compute cost other VPInstructions once the legacy cost model has
     // been retired.
@@ -830,6 +858,7 @@ bool VPInstruction::isVectorToScalar() const {
          getOpcode() == VPInstruction::ExtractPenultimateElement ||
          getOpcode() == Instruction::ExtractElement ||
          getOpcode() == VPInstruction::FirstActiveLane ||
+         getOpcode() == VPInstruction::ComputeAnyOfResult ||
          getOpcode() == VPInstruction::ComputeFindLastIVResult ||
          getOpcode() == VPInstruction::ComputeReductionResult ||
          getOpcode() == VPInstruction::AnyOf;
@@ -851,7 +880,7 @@ void VPInstruction::execute(VPTransformState &State) {
                                     isVectorToScalar() || isSingleScalar());
   bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
   if (GeneratesPerAllLanes) {
-    for (unsigned Lane = 0, NumLanes = State.VF.getKnownMinValue();
+    for (unsigned Lane = 0, NumLanes = State.VF.getFixedValue();
          Lane != NumLanes; ++Lane) {
       Value *GeneratedValue = generatePerLane(State, VPLane(Lane));
       assert(GeneratedValue && "generatePerLane must produce a value");
@@ -892,6 +921,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   case VPInstruction::PtrAdd:
   case VPInstruction::WideIVStep:
   case VPInstruction::StepVector:
+  case VPInstruction::ReductionStartVector:
     return false;
   default:
     return true;
@@ -922,9 +952,11 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
   case VPInstruction::CanonicalIVIncrementForPart:
   case VPInstruction::BranchOnCount:
   case VPInstruction::BranchOnCond:
+  case VPInstruction::ReductionStartVector:
     return true;
   case VPInstruction::PtrAdd:
     return Op == getOperand(0) || vputils::onlyFirstLaneUsed(this);
+  case VPInstruction::ComputeAnyOfResult:
   case VPInstruction::ComputeFindLastIVResult:
     return Op == getOperand(1);
   };
@@ -1005,6 +1037,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::ExtractPenultimateElement:
     O << "extract-penultimate-element";
     break;
+  case VPInstruction::ComputeAnyOfResult:
+    O << "compute-anyof-result";
+    break;
   case VPInstruction::ComputeFindLastIVResult:
     O << "compute-find-last-iv-result";
     break;
@@ -1023,6 +1058,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::FirstActiveLane:
     O << "first-active-lane";
     break;
+  case VPInstruction::ReductionStartVector:
+    O << "reduction-start-vector";
+    break;
   default:
     O << Instruction::getOpcodeName(getOpcode());
   }
@@ -1106,15 +1144,8 @@ void VPPhi::print(raw_ostream &O, const Twine &Indent,
                   VPSlotTracker &SlotTracker) const {
   O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";
   printAsOperand(O, SlotTracker);
-  const VPBasicBlock *Parent = getParent();
-  if (Parent == Parent->getPlan()->getScalarPreheader()) {
-    // TODO: Use regular printing for resume-phis as well
-    O << " = resume-phi ";
-    printOperands(O, SlotTracker);
-  } else {
-    O << " = phi ";
-    printPhiOperands(O, SlotTracker);
-  }
+  O << " = phi ";
+  printPhiOperands(O, SlotTracker);
 }
 #endif
 
@@ -1613,6 +1644,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
            Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
            Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
            Opcode == VPInstruction::WideIVStep ||
+           Opcode == VPInstruction::ReductionStartVector ||
            Opcode == VPInstruction::ComputeReductionResult;
   case OperationType::NonNegOp:
     return Opcode == Instruction::ZExt;
@@ -2500,21 +2532,17 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
   Value *VecOp = State.get(getVecOp());
   Value *EVL = State.get(getEVL(), VPLane(0));
 
-  VectorBuilder VBuilder(Builder);
-  VBuilder.setEVL(EVL);
   Value *Mask;
-  // TODO: move the all-true mask generation into VectorBuilder.
   if (VPValue *CondOp = getCondOp())
     Mask = State.get(CondOp);
   else
     Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
-  VBuilder.setMask(Mask);
 
   Value *NewRed;
   if (isOrdered()) {
-    NewRed = createOrderedReduction(VBuilder, Kind, VecOp, Prev);
+    NewRed = createOrderedReduction(Builder, Kind, VecOp, Prev, Mask, EVL);
   } else {
-    NewRed = createSimpleReduction(VBuilder, VecOp, Kind);
+    NewRed = createSimpleReduction(Builder, VecOp, Kind, Mask, EVL);
     if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
       NewRed = createMinMaxOp(Builder, Kind, NewRed, Prev);
     else
@@ -2768,8 +2796,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
   }
 
   // Generate scalar instances for all VF lanes.
-  assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
-  const unsigned EndLane = State.VF.getKnownMinValue();
+  const unsigned EndLane = State.VF.getFixedValue();
   for (unsigned Lane = 0; Lane < EndLane; ++Lane)
     scalarizeInstruction(UI, this, VPLane(Lane), State);
 }
@@ -2822,7 +2849,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
                UI->getOpcode(), ResultTy, CostKind,
                {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
                Op2Info, Operands, UI, &Ctx.TLI) *
-           (isSingleScalar() ? 1 : VF.getKnownMinValue());
+           (isSingleScalar() ? 1 : VF.getFixedValue());
   }
   }
 
@@ -3062,10 +3089,8 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
         Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},
                                 nullptr, "wide.masked.gather");
   } else {
-    VectorBuilder VBuilder(Builder);
-    VBuilder.setEVL(EVL).setMask(Mask);
-    NewLI = cast<CallInst>(VBuilder.createVectorInstruction(
-        Instruction::Load, DataTy, Addr, "vp.op.load"));
+    NewLI = Builder.CreateIntrinsic(DataTy, Intrinsic::vp_load,
+                                    {Addr, Mask, EVL}, nullptr, "vp.op.load");
   }
   NewLI->addParamAttr(
       0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
@@ -3180,11 +3205,9 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
                                     Intrinsic::vp_scatter,
                                     {StoredVal, Addr, Mask, EVL});
   } else {
-    VectorBuilder VBuilder(Builder);
-    VBuilder.setEVL(EVL).setMask(Mask);
-    NewSI = cast<CallInst>(VBuilder.createVectorInstruction(
-        Instruction::Store, Type::getVoidTy(EVL->getContext()),
-        {StoredVal, Addr}));
+    NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
+                                    Intrinsic::vp_store,
+                                    {StoredVal, Addr, Mask, EVL});
   }
   NewSI->addParamAttr(
       1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
@@ -3270,21 +3293,13 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
   // Scalable vectors cannot use arbitrary shufflevectors (only splats), so
   // must use intrinsics to interleave.
   if (VecTy->isScalableTy()) {
-    assert(isPowerOf2_32(Factor) && "Unsupported interleave factor for "
-                                    "scalable vectors, must be power of 2");
-    SmallVector<Value *> InterleavingValues(Vals);
-    // When interleaving, the number of values will be shrunk until we have the
-    // single final interleaved value.
-    auto *InterleaveTy = cast<VectorType>(InterleavingValues[0]->getType());
-    for (unsigned Midpoint = Factor / 2; Midpoint > 0; Midpoint /= 2) {
-      InterleaveTy = VectorType::getDoubleElementsVectorType(InterleaveTy);
-      for (unsigned I = 0; I < Midpoint; ++I)
-        InterleavingValues[I] = Builder.CreateIntrinsic(
-            InterleaveTy, Intrinsic::vector_interleave2,
-            {InterleavingValues[I], InterleavingValues[Midpoint + I]},
-            /*FMFSource=*/nullptr, Name);
-    }
-    return InterleavingValues[0];
+    assert(Factor <= 8 && "Unsupported interleave factor for scalable vectors");
+    VectorType *InterleaveTy =
+        VectorType::get(VecTy->getElementType(),
+                        VecTy->getElementCount().multiplyCoefficientBy(Factor));
+    return Builder.CreateIntrinsic(InterleaveTy,
+                                   getInterleaveIntrinsicID(Factor), Vals,
+                                   /*FMFSource=*/nullptr, Name);
   }
 
   // Fixed length. Start by concatenating all vectors into a wide vector.
@@ -3370,7 +3385,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
                           &InterleaveFactor](Value *MaskForGaps) -> Value * {
     if (State.VF.isScalable()) {
       assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
-      assert(isPowerOf2_32(InterleaveFactor) &&
+      assert(InterleaveFactor <= 8 &&
              "Unsupported deinterleave factor for scalable vectors");
       auto *ResBlockInMask = State.get(BlockInMask);
       SmallVector<Value *> Ops(InterleaveFactor, ResBlockInMask);
@@ -3383,7 +3398,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
     Value *ResBlockInMask = State.get(BlockInMask);
     Value *ShuffledMask = State.Builder.CreateShuffleVector(
         ResBlockInMask,
-        createReplicatedMask(InterleaveFactor, State.VF.getKnownMinValue()),
+        createReplicatedMask(InterleaveFactor, State.VF.getFixedValue()),
         "interleaved.mask");
     return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,
                                                    ShuffledMask, MaskForGaps)
@@ -3395,8 +3410,8 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
   if (isa<LoadInst>(Instr)) {
     Value *MaskForGaps = nullptr;
     if (NeedsMaskForGaps) {
-      MaskForGaps = createBitMaskForGaps(State.Builder,
-                                         State.VF.getKnownMinValue(), *Group);
+      MaskForGaps =
+          createBitMaskForGaps(State.Builder, State.VF.getFixedValue(), *Group);
       assert(MaskForGaps && "Mask for Gaps is required but it is null");
     }
 
@@ -3414,43 +3429,18 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
     ArrayRef<VPValue *> VPDefs = definedValues();
     const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
     if (VecTy->isScalableTy()) {
-      assert(isPowerOf2_32(InterleaveFactor) &&
-             "Unsupported deinterleave factor for scalable vectors");
-
       // Scalable vectors cannot use arbitrary shufflevectors (only splats),
       // so must use intrinsics to deinterleave.
-      SmallVector<Value *> DeinterleavedValues(InterleaveFactor);
-      DeinterleavedValues[0] = NewLoad;
-      // For the case of InterleaveFactor > 2, we will have to do recursive
-      // deinterleaving, because the current available deinterleave intrinsic
-      // supports only Factor of 2, otherwise it will bailout after first
-      // iteration.
-      // When deinterleaving, the number of values will double until we
-      // have "InterleaveFactor".
-      for (unsigned NumVectors = 1; NumVectors < InterleaveFactor;
-           NumVectors *= 2) {
-        // Deinterleave the elements within the vector
-        SmallVector<Value *> TempDeinterleavedValues(NumVectors);
-        for (unsigned I = 0; I < NumVectors; ++I) {
-          auto *DiTy = DeinterleavedValues[I]->getType();
-          TempDeinterleavedValues[I] = State.Builder.CreateIntrinsic(
-              Intrinsic::vector_deinterleave2, DiTy, DeinterleavedValues[I],
-              /*FMFSource=*/nullptr, "strided.vec");
-        }
-        // Extract the deinterleaved values:
-        for (unsigned I = 0; I < 2; ++I)
-          for (unsigned J = 0; J < NumVectors; ++J)
-            DeinterleavedValues[NumVectors * I + J] =
-                State.Builder.CreateExtractValue(TempDeinterleavedValues[J], I);
-      }
+      assert(InterleaveFactor <= 8 &&
+             "Unsupported deinterleave factor for scalable vectors");
+      Value *Deinterleave = State.Builder.CreateIntrinsic(
+          getDeinterleaveIntrinsicID(InterleaveFactor), NewLoad->getType(),
+          NewLoad,
+          /*FMFSource=*/nullptr, "strided.vec");
 
-#ifndef NDEBUG
-      for (Value *Val : DeinterleavedValues)
-        assert(Val && "NULL Deinterleaved Value");
-#endif
       for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
         Instruction *Member = Group->getMember(I);
-        Value *StridedVec = DeinterleavedValues[I];
+        Value *StridedVec = State.Builder.CreateExtractValue(Deinterleave, I);
         if (!Member) {
           // This value is not needed as it's not used
           cast<Instruction>(StridedVec)->eraseFromParent();
@@ -3472,6 +3462,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
 
       return;
     }
+    assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
 
     // For each member in the group, shuffle out the appropriate data from the
     // wide loads.
@@ -3484,13 +3475,12 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
         continue;
 
       auto StrideMask =
-          createStrideMask(I, InterleaveFactor, State.VF.getKnownMinValue());
+          createStrideMask(I, InterleaveFactor, State.VF.getFixedValue());
       Value *StridedVec =
           State.Builder.CreateShuffleVector(NewLoad, StrideMask, "strided.vec");
 
       // If this member has different type, cast the result type.
       if (Member->getType() != ScalarTy) {
-        assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
         VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
         StridedVec =
             createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
@@ -3826,7 +3816,7 @@ VPFirstOrderRecurrencePHIRecipe::computeCost(ElementCount VF,
   if (VF.isScalar())
     return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
 
-  if (VF.isScalable() && VF.getKnownMinValue() == 1)
+  if (VF == ElementCount::getScalable(1))
     return InstructionCost::getInvalid();
 
   return 0;
@@ -3843,17 +3833,19 @@ void VPFirstOrderRecurrencePHIRecipe::print(raw_ostream &O, const Twine &Indent,
 #endif
 
 void VPReductionPHIRecipe::execute(VPTransformState &State) {
-  // If this phi is fed by a scaled reduction then it should output a
-  // vector with fewer elements than the VF.
-  ElementCount VF = State.VF.divideCoefficientBy(VFScaleFactor);
+  // Reductions do not have to start at zero. They can start with
+  // any loop invariant values.
+  VPValue *StartVPV = getStartValue();
 
   // In order to support recurrences we need to be able to vectorize Phi nodes.
   // Phi nodes have cycles, so we need to vectorize them in two stages. This is
   // stage #1: We create a new vector PHI node with no incoming edges. We'll use
   // this value when we vectorize all of the instructions that use the PHI.
-  auto *ScalarTy = State.TypeAnalysis.inferScalarType(this);
+  BasicBlock *VectorPH =
+      State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));
   bool ScalarPHI = State.VF.isScalar() || IsInLoop;
-  Type *VecTy = ScalarPHI ? ScalarTy : VectorType::get(ScalarTy, VF);
+  Value *StartV = State.get(StartVPV, ScalarPHI);
+  Type *VecTy = StartV->getType();
 
   BasicBlock *HeaderBB = State.CFG.PrevBB;
   assert(State.CurrentParentLoop->getHeader() == HeaderBB &&
@@ -3862,49 +3854,7 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
   Phi->insertBefore(HeaderBB->getFirstInsertionPt());
   State.set(this, Phi, IsInLoop);
 
-  BasicBlock *VectorPH =
-      State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));
-  // Create start and identity vector values for the reduction in the preheader.
-  // TODO: Introduce recipes in VPlan preheader to create initial values.
-  IRBuilderBase::InsertPointGuard IPBuilder(State.Builder);
-  State.Builder.SetInsertPoint(VectorPH->getTerminator());
-
-  // Reductions do not have to start at zero. They can start with
-  // any loop invariant values.
-  VPValue *StartVPV = getStartValue();
-  RecurKind RK = RdxDesc.getRecurrenceKind();
-  if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK) ||
-      RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
-      RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
-    // [I|F]FindLastIV will use a sentinel value to initialize the reduction
-    // phi or the resume value from the main vector loop when vectorizing the
-    // epilogue loop. In the exit block, ComputeReductionResult will generate
-    // checks to verify if the reduction result is the sentinel value. If the
-    // result is the sentinel value, it will be corrected back to the start
-    // value.
-    // TODO: The sentinel value is not always necessary. When the start value is
-    // a constant, and smaller than the start value of the induction variable,
-    // the start value can be directly used to initialize the reduction phi.
-    Phi->addIncoming(State.get(StartVPV, ScalarPHI), VectorPH);
-    return;
-  }
-
-  Value *Iden = getRecurrenceIdentity(RK, VecTy->getScalarType(),
-                                      RdxDesc.getFastMathFlags());
-  unsigned CurrentPart = getUnrollPart(*this);
-  Value *StartV = StartVPV->getLiveInIRValue();
-  if (!ScalarPHI) {
-    if (CurrentPart == 0) {
-      Iden = State.Builder.CreateVectorSplat(VF, Iden);
-      Constant *Zero = State.Builder.getInt32(0);
-      StartV = State.Builder.CreateInsertElement(Iden, StartV, Zero);
-    } else {
-      Iden = State.Builder.CreateVectorSplat(VF, Iden);
-    }
-  }
-
-  Value *StartVal = (CurrentPart == 0) ? StartV : Iden;
-  Phi->addIncoming(StartVal, VectorPH);
+  Phi->addIncoming(StartV, VectorPH);
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5b42a9056b69..44a72755b9cf 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -31,6 +31,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/TypeSize.h"
@@ -1015,13 +1016,11 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
           .Default([](auto *) { return false; }))
     return;
 
-  // Fold PredPHI constant -> constant.
+  // Fold PredPHI LiveIn -> LiveIn.
   if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(&R)) {
     VPValue *Op = PredPHI->getOperand(0);
-    if (!Op->isLiveIn() || !Op->getLiveInIRValue())
-      return;
-    if (auto *C = dyn_cast<Constant>(Op->getLiveInIRValue()))
-      PredPHI->replaceAllUsesWith(Plan->getOrAddLiveIn(C));
+    if (Op->isLiveIn())
+      PredPHI->replaceAllUsesWith(Op);
   }
 
   VPValue *A;
@@ -1155,6 +1154,16 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
       return;
     }
   }
+  // Simplify redundant ReductionStartVector recipes after unrolling.
+  VPValue *StartV;
+  if (match(Def, m_VPInstruction<VPInstruction::ReductionStartVector>(
+                     m_VPValue(StartV), m_VPValue(), m_VPValue()))) {
+    Def->replaceUsesWithIf(StartV, [](const VPUser &U, unsigned Idx) {
+      auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&U);
+      return PhiR && PhiR->isInLoop();
+    });
+    return;
+  }
 }
 
 void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
@@ -2601,14 +2610,10 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getEntry()))) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
-      if (isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(&R)) {
-        auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
-        StringRef Name =
-            isa<VPCanonicalIVPHIRecipe>(PhiR) ? "index" : "evl.based.iv";
-        VPBuilder Builder(PhiR);
-        auto *ScalarR = Builder.createScalarPhi(
+      if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
+        auto *ScalarR = VPBuilder(PhiR).createScalarPhi(
             {PhiR->getStartValue(), PhiR->getBackedgeValue()},
-            PhiR->getDebugLoc(), Name);
+            PhiR->getDebugLoc(), "evl.based.iv");
         PhiR->replaceAllUsesWith(ScalarR);
         ToRemove.push_back(PhiR);
         continue;
@@ -3199,3 +3204,25 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
       Plan.getOrAddLiveIn(ConstantInt::get(CanIV->getScalarType(), 1)));
   removeDeadRecipes(Plan);
 }
+
+/// Add branch weight metadata, if the \p Plan's middle block is terminated by a
+/// BranchOnCond recipe.
+void VPlanTransforms::addBranchWeightToMiddleTerminator(VPlan &Plan,
+                                                        ElementCount VF) {
+  VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
+  auto *MiddleTerm =
+      dyn_cast_or_null<VPInstruction>(MiddleVPBB->getTerminator());
+  // Only add branch metadata if there is a (conditional) terminator.
+  if (!MiddleTerm)
+    return;
+
+  assert(MiddleTerm->getOpcode() == VPInstruction::BranchOnCond &&
+         "must have a BranchOnCond");
+  // Assume that `TripCount % VectorStep ` is equally distributed.
+  unsigned VectorStep = Plan.getUF() * VF.getKnownMinValue();
+  assert(VectorStep > 0 && "trip count should not be zero");
+  MDBuilder MDB(Plan.getScalarHeader()->getIRBasicBlock()->getContext());
+  MDNode *BranchWeights =
+      MDB.createBranchWeights({1, VectorStep - 1}, /*IsExpected=*/false);
+  MiddleTerm->addMetadata(LLVMContext::MD_prof, BranchWeights);
+}
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 34e2de4eb3b7..5a03bdb7c688 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -234,6 +234,10 @@ struct VPlanTransforms {
   /// removed in the future.
   static DenseMap<VPBasicBlock *, VPValue *>
   introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
+
+  /// Add branch weight metadata, if the \p Plan's middle block is terminated by
+  /// a BranchOnCond recipe.
+  static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF);
 };
 
 } // namespace llvm
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index e1fb3d476c58..dfb5bfabd22b 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -25,6 +25,7 @@
 #include "llvm/IR/Intrinsics.h"
 
 using namespace llvm;
+using namespace llvm::VPlanPatternMatch;
 
 namespace {
 
@@ -223,6 +224,22 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
       Copy->addOperand(R);
       Copy->addOperand(getConstantVPV(Part));
     } else if (RdxPhi) {
+      // If the start value is a ReductionStartVector, use the identity value
+      // (second operand) for unrolled parts. If the scaling factor is > 1,
+      // create a new ReductionStartVector with the scale factor and both
+      // operands set to the identity value.
+      if (auto *VPI = dyn_cast<VPInstruction>(RdxPhi->getStartValue())) {
+        assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
+               "unexpected start VPInstruction");
+        if (match(VPI->getOperand(2), m_SpecificInt(1))) {
+          Copy->setOperand(0, VPI->getOperand(1));
+        } else if (Part == 1) {
+          auto *C = VPI->clone();
+          C->setOperand(0, C->getOperand(1));
+          C->insertAfter(VPI);
+          addUniformForAllParts(C);
+        }
+      }
       Copy->addOperand(getConstantVPV(Part));
     } else {
       assert(isa<VPActiveLaneMaskPHIRecipe>(R) &&
@@ -233,7 +250,6 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
 
 /// Handle non-header-phi recipes.
 void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
-  using namespace llvm::VPlanPatternMatch;
   if (match(&R, m_BranchOnCond(m_VPValue())) ||
       match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
     return;
@@ -301,7 +317,6 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
   }
 }
 
-using namespace llvm::VPlanPatternMatch;
 void UnrollState::unrollBlock(VPBlockBase *VPB) {
   auto *VPR = dyn_cast<VPRegionBlock>(VPB);
   if (VPR) {
@@ -327,10 +342,12 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
     // Add all VPValues for all parts to ComputeReductionResult which combines
     // the parts to compute the final reduction value.
     VPValue *Op1;
-    if (match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(
+    if (match(&R, m_VPInstruction<VPInstruction::ComputeAnyOfResult>(
+                      m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||
+        match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(
                       m_VPValue(), m_VPValue(Op1))) ||
         match(&R, m_VPInstruction<VPInstruction::ComputeFindLastIVResult>(
-                      m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
+                      m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
       addUniformForAllParts(cast<VPInstruction>(&R));
       for (unsigned Part = 1; Part != UF; ++Part)
         R.addOperand(getValueForPart(Op1, Part));
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanValue.h b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanValue.h
index ad347ed6f50f..a0d3dc9b934c 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -45,16 +45,9 @@ class VPPhiAccessors;
 // coming from the input IR and instructions which VPlan will generate if
 // executed.
 class VPValue {
-  friend class VPBuilder;
   friend class VPDef;
   friend struct VPDoubleValueDef;
-  friend class VPInstruction;
   friend class VPInterleaveRecipe;
-  friend struct VPlanTransforms;
-  friend class VPBasicBlock;
-  friend class VPInterleavedAccessInfo;
-  friend class VPSlotTracker;
-  friend class VPRecipeBase;
   friend class VPlan;
 
   const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
diff --git a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 45010d002158..fba4a68f4a27 100644
--- a/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/external/llvm-project/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -429,8 +429,7 @@ bool VPlanVerifier::verify(const VPlan &Plan) {
     return false;
   }
 
-  // TODO: Remove once loop regions are dissolved before execution.
-  if (!VerifyLate && !isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
+  if (!isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
     errs() << "VPlan vector loop header does not start with a "
               "VPCanonicalIVPHIRecipe\n";
     return false;
diff --git a/external/llvm-project/llvm/lib/XRay/InstrumentationMap.cpp b/external/llvm-project/llvm/lib/XRay/InstrumentationMap.cpp
index 0ebdcd5bac75..5da09a57596a 100644
--- a/external/llvm-project/llvm/lib/XRay/InstrumentationMap.cpp
+++ b/external/llvm-project/llvm/lib/XRay/InstrumentationMap.cpp
@@ -24,7 +24,6 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/TargetParser/Triple.h"
-#include <algorithm>
 #include <cstddef>
 #include <cstdint>
 #include <system_error>
diff --git a/external/llvm-project/llvm/lib/XRay/Profile.cpp b/external/llvm-project/llvm/lib/XRay/Profile.cpp
index 4d1ab10e1d1b..1b340e54c9db 100644
--- a/external/llvm-project/llvm/lib/XRay/Profile.cpp
+++ b/external/llvm-project/llvm/lib/XRay/Profile.cpp
@@ -16,7 +16,6 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/XRay/Trace.h"
-#include <deque>
 #include <memory>
 
 namespace llvm {
diff --git a/external/llvm-project/llvm/runtimes/CMakeLists.txt b/external/llvm-project/llvm/runtimes/CMakeLists.txt
index c018aa905a18..9d12634a3a47 100644
--- a/external/llvm-project/llvm/runtimes/CMakeLists.txt
+++ b/external/llvm-project/llvm/runtimes/CMakeLists.txt
@@ -193,7 +193,7 @@ endif()
 
 function(_get_runtime_name name out_var)
   string(FIND ${name} "lib" idx)
-  if(idx EQUAL 0 AND NOT ${name} STREQUAL "libc")
+  if(idx EQUAL 0 AND NOT (${name} STREQUAL "libc" OR ${name} STREQUAL "libclc"))
     string(SUBSTRING ${name} 3 -1 name)
   endif()
   set(${out_var} ${name} PARENT_SCOPE)
diff --git a/external/llvm-project/llvm/test/Analysis/BasicAA/captures.ll b/external/llvm-project/llvm/test/Analysis/BasicAA/captures.ll
index fcc51b54b71b..c9ed1ea74be8 100644
--- a/external/llvm-project/llvm/test/Analysis/BasicAA/captures.ll
+++ b/external/llvm-project/llvm/test/Analysis/BasicAA/captures.ll
@@ -17,8 +17,7 @@ define void @address_capture() {
 
 ; CHECK-LABEL: read_only_capture
 ; CHECK: MayAlias:	i32* %a, i32* %p
-; CHECK: Both ModRef:  Ptr: i32* %a	<->  %p = call ptr @get_ptr()
-; TODO: The ModRef could be just Ref.
+; CHECK: Just Ref:  Ptr: i32* %a	<->  %p = call ptr @get_ptr()
 define void @read_only_capture() {
   %a = alloca i32
   call void @capture(ptr captures(address, read_provenance) %a)
@@ -40,3 +39,42 @@ define void @address_capture_and_full_capture() {
   load i32, ptr %a
   ret void
 }
+
+; CHECK-LABEL: address_capture_and_full_capture_commuted
+; CHECK: MayAlias:	i32* %a, i32* %p
+; CHECK: Both ModRef:  Ptr: i32* %a	<->  %p = call ptr @get_ptr()
+define void @address_capture_and_full_capture_commuted() {
+  %a = alloca i32
+  call void @capture(ptr %a)
+  call void @capture(ptr captures(address) %a)
+  %p = call ptr @get_ptr()
+  store i32 0, ptr %p
+  load i32, ptr %a
+  ret void
+}
+
+; CHECK-LABEL: read_only_capture_and_full_capture
+; CHECK: MayAlias:	i32* %a, i32* %p
+; CHECK: Both ModRef:  Ptr: i32* %a	<->  %p = call ptr @get_ptr()
+define void @read_only_capture_and_full_capture() {
+  %a = alloca i32
+  call void @capture(ptr captures(address, read_provenance) %a)
+  call void @capture(ptr %a)
+  %p = call ptr @get_ptr()
+  store i32 0, ptr %p
+  load i32, ptr %a
+  ret void
+}
+
+; CHECK-LABEL: read_only_capture_and_full_capture_commuted
+; CHECK: MayAlias:	i32* %a, i32* %p
+; CHECK: Both ModRef:  Ptr: i32* %a	<->  %p = call ptr @get_ptr()
+define void @read_only_capture_and_full_capture_commuted() {
+  %a = alloca i32
+  call void @capture(ptr %a)
+  call void @capture(ptr captures(address, read_provenance) %a)
+  %p = call ptr @get_ptr()
+  store i32 0, ptr %p
+  load i32, ptr %a
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Analysis/BasicAA/gep-decomposition-limit.ll b/external/llvm-project/llvm/test/Analysis/BasicAA/gep-decomposition-limit.ll
index 23a96ebca848..a256ececbe56 100644
--- a/external/llvm-project/llvm/test/Analysis/BasicAA/gep-decomposition-limit.ll
+++ b/external/llvm-project/llvm/test/Analysis/BasicAA/gep-decomposition-limit.ll
@@ -2,22 +2,22 @@
 
 ; CHECK-LABEL: Function: test
 ;; Before limit:
-; CHECK-DAG: MustAlias: i8* %gep.add5, i8* %gep.inc5
-; CHECK-DAG: NoAlias: i8* %gep.inc3, i8* %gep.inc5
-; CHECK-DAG: NoAlias: i8* %gep.inc4, i8* %gep.inc5
+; CHECK-DAG: MustAlias: i8* %gep.add9, i8* %gep.inc9
+; CHECK-DAG: NoAlias: i8* %gep.inc7, i8* %gep.inc9
+; CHECK-DAG: NoAlias: i8* %gep.inc8, i8* %gep.inc9
 ;; At limit:
-; CHECK-DAG: MustAlias: i8* %gep.add6, i8* %gep.inc6
-; CHECK-DAG: NoAlias: i8* %gep.inc4, i8* %gep.inc6
-; CHECK-DAG: NoAlias: i8* %gep.inc5, i8* %gep.inc6
+; CHECK-DAG: MustAlias: i8* %gep.add10, i8* %gep.inc10
+; CHECK-DAG: NoAlias: i8* %gep.inc10, i8* %gep.inc8
+; CHECK-DAG: NoAlias: i8* %gep.inc10, i8* %gep.inc9
 ;; After limit:
-; CHECK-DAG: MayAlias: i8* %gep.add7, i8* %gep.inc7
-; CHECK-DAG: MayAlias: i8* %gep.inc5, i8* %gep.inc7
-; CHECK-DAG: NoAlias: i8* %gep.inc6, i8* %gep.inc7
+; CHECK-DAG: MayAlias: i8* %gep.add11, i8* %gep.inc11
+; CHECK-DAG: MayAlias: i8* %gep.inc11, i8* %gep.inc9
+; CHECK-DAG: NoAlias: i8* %gep.inc10, i8* %gep.inc11
 
 define void @test(ptr %base) {
-  %gep.add5 = getelementptr i8, ptr %base, i64 5
-  %gep.add6 = getelementptr i8, ptr %base, i64 6
-  %gep.add7 = getelementptr i8, ptr %base, i64 7
+  %gep.add9 = getelementptr i8, ptr %base, i64 9
+  %gep.add10 = getelementptr i8, ptr %base, i64 10
+  %gep.add11 = getelementptr i8, ptr %base, i64 11
 
   %gep.inc1 = getelementptr i8, ptr %base, i64 1
   %gep.inc2 = getelementptr i8, ptr %gep.inc1, i64 1
@@ -26,15 +26,23 @@ define void @test(ptr %base) {
   %gep.inc5 = getelementptr i8, ptr %gep.inc4, i64 1
   %gep.inc6 = getelementptr i8, ptr %gep.inc5, i64 1
   %gep.inc7 = getelementptr i8, ptr %gep.inc6, i64 1
+  %gep.inc8 = getelementptr i8, ptr %gep.inc7, i64 1
+  %gep.inc9 = getelementptr i8, ptr %gep.inc8, i64 1
+  %gep.inc10 = getelementptr i8, ptr %gep.inc9, i64 1
+  %gep.inc11 = getelementptr i8, ptr %gep.inc10, i64 1
 
-  load i8, ptr %gep.add5
-  load i8, ptr %gep.add6
-  load i8, ptr %gep.add7
+  load i8, ptr %gep.add9
+  load i8, ptr %gep.add10
+  load i8, ptr %gep.add11
   load i8, ptr %gep.inc3
   load i8, ptr %gep.inc4
   load i8, ptr %gep.inc5
   load i8, ptr %gep.inc6
   load i8, ptr %gep.inc7
+  load i8, ptr %gep.inc8
+  load i8, ptr %gep.inc9
+  load i8, ptr %gep.inc10
+  load i8, ptr %gep.inc11
 
   ret void
 }
diff --git a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/div.ll b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/div.ll
index 5367344ce573..3a2358dba51b 100644
--- a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/div.ll
+++ b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/div.ll
@@ -870,27 +870,27 @@ define void @sdiv_uniformconstnegpow2() {
 ; CHECK-LABEL: 'sdiv_uniformconstnegpow2'
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = sdiv i128 undef, -16
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I64 = sdiv i64 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I32 = sdiv i32 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I16 = sdiv i16 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I8 = sdiv i8 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %I128 = sdiv i128 undef, -16
diff --git a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/rem.ll b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/rem.ll
index d684e3af00b8..2fa62f170591 100644
--- a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/rem.ll
+++ b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/rem.ll
@@ -870,27 +870,27 @@ define void @srem_uniformconstnegpow2() {
 ; CHECK-LABEL: 'srem_uniformconstnegpow2'
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = srem i128 undef, -16
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I64 = srem i64 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I32 = srem i32 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I16 = srem i16 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I8 = srem i8 undef, -16
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %I128 = srem i128 undef, -16
diff --git a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-div.ll b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
index 480c3146a210..c055d3218f65 100644
--- a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
+++ b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
@@ -479,42 +479,42 @@ define void @udiv_uniformconstpow2() {
 
 define void @sdiv_uniformconstnegpow2() {
 ; CHECK-LABEL: 'sdiv_uniformconstnegpow2'
-; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
diff --git a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll
index e2488735de4b..eac8b66bcd21 100644
--- a/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll
+++ b/external/llvm-project/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll
@@ -491,43 +491,43 @@ define void @urem_uniformconstpow2() {
 
 define void @srem_uniformconstnegpow2() {
 ; CHECK-LABEL: 'srem_uniformconstnegpow2'
-; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:Invalid CodeSize:4 Lat:4 SizeLat:4 for: %NV2i128 = srem <vscale x 2 x i128> undef, splat (i128 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %V2i64 = srem <2 x i64> undef, splat (i64 -16)
diff --git a/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll b/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll
index e068ab638d3a..e1bca7161412 100644
--- a/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll
+++ b/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll
@@ -86,6 +86,8 @@ define void @vector_reverse() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %reverse_nxv16i32 = call <vscale x 16 x i32> @llvm.vector.reverse.nxv16i32(<vscale x 16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 166 for instruction: %reverse_nxv32i32 = call <vscale x 32 x i32> @llvm.vector.reverse.nxv32i32(<vscale x 32 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %reverse_nxv8i64 = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> undef)
@@ -106,6 +108,8 @@ define void @vector_reverse() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv16i32 = call <vscale x 16 x i32> @llvm.vector.reverse.nxv16i32(<vscale x 16 x i32> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %reverse_nxv32i32 = call <vscale x 32 x i32> @llvm.vector.reverse.nxv32i32(<vscale x 32 x i32> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %reverse_nxv8i64 = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> undef)
@@ -125,6 +129,8 @@ define void @vector_reverse() {
   %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
   %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
   %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+  %reverse_nxv16i32 = call <vscale x 16 x i32> @llvm.vector.reverse.nxv16i32(<vscale x 16 x i32> undef)
+  %reverse_nxv32i32 = call <vscale x 32 x i32> @llvm.vector.reverse.nxv32i32(<vscale x 32 x i32> undef)
   %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
   %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
   %reverse_nxv8i64 = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> undef)
diff --git a/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll b/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll
index 7e92d8203a13..8f3219861f2f 100644
--- a/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll
+++ b/external/llvm-project/llvm/test/Analysis/CostModel/RISCV/shuffle-reverse.ll
@@ -26,8 +26,11 @@ define void @reverse() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
@@ -55,8 +58,11 @@ define void @reverse() {
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
@@ -85,9 +91,12 @@ define void @reverse() {
   %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
   %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 
   %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
   %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 
   %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
   %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
diff --git a/external/llvm-project/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/external/llvm-project/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
new file mode 100644
index 000000000000..7a3082056ad2
--- /dev/null
+++ b/external/llvm-project/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -0,0 +1,927 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<hash-recognize>' -disable-output %s 2>&1 | FileCheck %s
+
+define i16 @crc16.le.tc8(i8 %msg, i16 %checksum) {
+; CHECK-LABEL: 'crc16.le.tc8'
+; CHECK-NEXT:  Found little-endian CRC-16 loop with trip count 8
+; CHECK-NEXT:    Initial CRC: i16 %checksum
+; CHECK-NEXT:    Generating polynomial: 40961
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+; CHECK-NEXT:    Auxiliary data: i8 %msg
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 49345 49537 320 49921 960 640 49729 50689 1728 1920 51009 1280 50625 50305 1088
+; CHECK-NEXT:  52225 3264 3456 52545 3840 53185 52865 3648 2560 51905 52097 2880 51457 2496 2176 51265
+; CHECK-NEXT:  55297 6336 6528 55617 6912 56257 55937 6720 7680 57025 57217 8000 56577 7616 7296 56385
+; CHECK-NEXT:  5120 54465 54657 5440 55041 6080 5760 54849 53761 4800 4992 54081 4352 53697 53377 4160
+; CHECK-NEXT:  61441 12480 12672 61761 13056 62401 62081 12864 13824 63169 63361 14144 62721 13760 13440 62529
+; CHECK-NEXT:  15360 64705 64897 15680 65281 16320 16000 65089 64001 15040 15232 64321 14592 63937 63617 14400
+; CHECK-NEXT:  10240 59585 59777 10560 60161 11200 10880 59969 60929 11968 12160 61249 11520 60865 60545 11328
+; CHECK-NEXT:  58369 9408 9600 58689 9984 59329 59009 9792 8704 58049 58241 9024 57601 8640 8320 57409
+; CHECK-NEXT:  40961 24768 24960 41281 25344 41921 41601 25152 26112 42689 42881 26432 42241 26048 25728 42049
+; CHECK-NEXT:  27648 44225 44417 27968 44801 28608 28288 44609 43521 27328 27520 43841 26880 43457 43137 26688
+; CHECK-NEXT:  30720 47297 47489 31040 47873 31680 31360 47681 48641 32448 32640 48961 32000 48577 48257 31808
+; CHECK-NEXT:  46081 29888 30080 46401 30464 47041 46721 30272 29184 45761 45953 29504 45313 29120 28800 45121
+; CHECK-NEXT:  20480 37057 37249 20800 37633 21440 21120 37441 38401 22208 22400 38721 21760 38337 38017 21568
+; CHECK-NEXT:  39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
+; CHECK-NEXT:  34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
+; CHECK-NEXT:  17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.trunc = trunc i16 %crc to i8
+  %xor.data.crc = xor i8 %data, %crc.trunc
+  %and.data.crc = and i8 %xor.data.crc, 1
+  %data.next = lshr i8 %data, 1
+  %check.sb = icmp eq i8 %and.data.crc, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @crc16.le.tc8.udiv(i8 %msg, i16 %checksum) {
+; CHECK-LABEL: 'crc16.le.tc8.udiv'
+; CHECK-NEXT:  Found little-endian CRC-16 loop with trip count 8
+; CHECK-NEXT:    Initial CRC: i16 %checksum
+; CHECK-NEXT:    Generating polynomial: 40961
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+; CHECK-NEXT:    Auxiliary data: i8 %msg
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 49345 49537 320 49921 960 640 49729 50689 1728 1920 51009 1280 50625 50305 1088
+; CHECK-NEXT:  52225 3264 3456 52545 3840 53185 52865 3648 2560 51905 52097 2880 51457 2496 2176 51265
+; CHECK-NEXT:  55297 6336 6528 55617 6912 56257 55937 6720 7680 57025 57217 8000 56577 7616 7296 56385
+; CHECK-NEXT:  5120 54465 54657 5440 55041 6080 5760 54849 53761 4800 4992 54081 4352 53697 53377 4160
+; CHECK-NEXT:  61441 12480 12672 61761 13056 62401 62081 12864 13824 63169 63361 14144 62721 13760 13440 62529
+; CHECK-NEXT:  15360 64705 64897 15680 65281 16320 16000 65089 64001 15040 15232 64321 14592 63937 63617 14400
+; CHECK-NEXT:  10240 59585 59777 10560 60161 11200 10880 59969 60929 11968 12160 61249 11520 60865 60545 11328
+; CHECK-NEXT:  58369 9408 9600 58689 9984 59329 59009 9792 8704 58049 58241 9024 57601 8640 8320 57409
+; CHECK-NEXT:  40961 24768 24960 41281 25344 41921 41601 25152 26112 42689 42881 26432 42241 26048 25728 42049
+; CHECK-NEXT:  27648 44225 44417 27968 44801 28608 28288 44609 43521 27328 27520 43841 26880 43457 43137 26688
+; CHECK-NEXT:  30720 47297 47489 31040 47873 31680 31360 47681 48641 32448 32640 48961 32000 48577 48257 31808
+; CHECK-NEXT:  46081 29888 30080 46401 30464 47041 46721 30272 29184 45761 45953 29504 45313 29120 28800 45121
+; CHECK-NEXT:  20480 37057 37249 20800 37633 21440 21120 37441 38401 22208 22400 38721 21760 38337 38017 21568
+; CHECK-NEXT:  39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
+; CHECK-NEXT:  34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
+; CHECK-NEXT:  17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.trunc = trunc i16 %crc to i8
+  %xor.data.crc = xor i8 %data, %crc.trunc
+  %and.data.crc = and i8 %xor.data.crc, 1
+  %data.next = udiv i8 %data, 2
+  %check.sb = icmp eq i8 %and.data.crc, 0
+  %crc.lshr = udiv i16 %crc, 2
+  %xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @crc16.le.tc16(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'crc16.le.tc16'
+; CHECK-NEXT:  Found little-endian CRC-16 loop with trip count 16
+; CHECK-NEXT:    Initial CRC: i16 %checksum
+; CHECK-NEXT:    Generating polynomial: 40961
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+; CHECK-NEXT:    Auxiliary data: i16 %msg
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 49345 49537 320 49921 960 640 49729 50689 1728 1920 51009 1280 50625 50305 1088
+; CHECK-NEXT:  52225 3264 3456 52545 3840 53185 52865 3648 2560 51905 52097 2880 51457 2496 2176 51265
+; CHECK-NEXT:  55297 6336 6528 55617 6912 56257 55937 6720 7680 57025 57217 8000 56577 7616 7296 56385
+; CHECK-NEXT:  5120 54465 54657 5440 55041 6080 5760 54849 53761 4800 4992 54081 4352 53697 53377 4160
+; CHECK-NEXT:  61441 12480 12672 61761 13056 62401 62081 12864 13824 63169 63361 14144 62721 13760 13440 62529
+; CHECK-NEXT:  15360 64705 64897 15680 65281 16320 16000 65089 64001 15040 15232 64321 14592 63937 63617 14400
+; CHECK-NEXT:  10240 59585 59777 10560 60161 11200 10880 59969 60929 11968 12160 61249 11520 60865 60545 11328
+; CHECK-NEXT:  58369 9408 9600 58689 9984 59329 59009 9792 8704 58049 58241 9024 57601 8640 8320 57409
+; CHECK-NEXT:  40961 24768 24960 41281 25344 41921 41601 25152 26112 42689 42881 26432 42241 26048 25728 42049
+; CHECK-NEXT:  27648 44225 44417 27968 44801 28608 28288 44609 43521 27328 27520 43841 26880 43457 43137 26688
+; CHECK-NEXT:  30720 47297 47489 31040 47873 31680 31360 47681 48641 32448 32640 48961 32000 48577 48257 31808
+; CHECK-NEXT:  46081 29888 30080 46401 30464 47041 46721 30272 29184 45761 45953 29504 45313 29120 28800 45121
+; CHECK-NEXT:  20480 37057 37249 20800 37633 21440 21120 37441 38401 22208 22400 38721 21760 38337 38017 21568
+; CHECK-NEXT:  39937 23744 23936 40257 24320 40897 40577 24128 23040 39617 39809 23360 39169 22976 22656 38977
+; CHECK-NEXT:  34817 18624 18816 35137 19200 35777 35457 19008 19968 36545 36737 20288 36097 19904 19584 35905
+; CHECK-NEXT:  17408 33985 34177 17728 34561 18368 18048 34369 33281 17088 17280 33601 16640 33217 32897 16448
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %and.crc.data = and i16 %xor.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @crc16.be.tc8.crc.init.li(i16 %checksum, i8 %msg) {
+; CHECK-LABEL: 'crc16.be.tc8.crc.init.li'
+; CHECK-NEXT:  Found big-endian CRC-16 loop with trip count 8
+; CHECK-NEXT:    Initial CRC: %crc.init = xor i16 %msg.shl, %checksum
+; CHECK-NEXT:    Generating polynomial: 4129
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 256 512 768 1024 1280 1536 1792 2048 2304 2560 2816 3072 3328 3584 3840
+; CHECK-NEXT:  4096 4352 4608 4864 5120 5376 5632 5888 6144 6400 6656 6912 7168 7424 7680 7936
+; CHECK-NEXT:  8192 8448 8704 8960 9216 9472 9728 9984 10240 10496 10752 11008 11264 11520 11776 12032
+; CHECK-NEXT:  12288 12544 12800 13056 13312 13568 13824 14080 14336 14592 14848 15104 15360 15616 15872 16128
+; CHECK-NEXT:  16384 16640 16896 17152 17408 17664 17920 18176 18432 18688 18944 19200 19456 19712 19968 20224
+; CHECK-NEXT:  20480 20736 20992 21248 21504 21760 22016 22272 22528 22784 23040 23296 23552 23808 24064 24320
+; CHECK-NEXT:  24576 24832 25088 25344 25600 25856 26112 26368 26624 26880 27136 27392 27648 27904 28160 28416
+; CHECK-NEXT:  28672 28928 29184 29440 29696 29952 30208 30464 30720 30976 31232 31488 31744 32000 32256 32512
+; CHECK-NEXT:  32768 33024 33280 33536 33792 34048 34304 34560 34816 35072 35328 35584 35840 36096 36352 36608
+; CHECK-NEXT:  36864 37120 37376 37632 37888 38144 38400 38656 38912 39168 39424 39680 39936 40192 40448 40704
+; CHECK-NEXT:  40960 41216 41472 41728 41984 42240 42496 42752 43008 43264 43520 43776 44032 44288 44544 44800
+; CHECK-NEXT:  45056 45312 45568 45824 46080 46336 46592 46848 47104 47360 47616 47872 48128 48384 48640 48896
+; CHECK-NEXT:  49152 49408 49664 49920 50176 50432 50688 50944 51200 51456 51712 51968 52224 52480 52736 52992
+; CHECK-NEXT:  53248 53504 53760 54016 54272 54528 54784 55040 55296 55552 55808 56064 56320 56576 56832 57088
+; CHECK-NEXT:  57344 57600 57856 58112 58368 58624 58880 59136 59392 59648 59904 60160 60416 60672 60928 61184
+; CHECK-NEXT:  61440 61696 61952 62208 62464 62720 62976 63232 63488 63744 64000 64256 64512 64768 65024 65280
+;
+entry:
+  %msg.ext = zext i8 %msg to i16
+  %msg.shl = shl nuw i16 %msg.ext, 8
+  %crc.init = xor i16 %msg.shl, %checksum
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @crc16.be.tc8.crc.init.arg(i16 %crc.init) {
+; CHECK-LABEL: 'crc16.be.tc8.crc.init.arg'
+; CHECK-NEXT:  Found big-endian CRC-16 loop with trip count 8
+; CHECK-NEXT:    Initial CRC: i16 %crc.init
+; CHECK-NEXT:    Generating polynomial: 4129
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 256 512 768 1024 1280 1536 1792 2048 2304 2560 2816 3072 3328 3584 3840
+; CHECK-NEXT:  4096 4352 4608 4864 5120 5376 5632 5888 6144 6400 6656 6912 7168 7424 7680 7936
+; CHECK-NEXT:  8192 8448 8704 8960 9216 9472 9728 9984 10240 10496 10752 11008 11264 11520 11776 12032
+; CHECK-NEXT:  12288 12544 12800 13056 13312 13568 13824 14080 14336 14592 14848 15104 15360 15616 15872 16128
+; CHECK-NEXT:  16384 16640 16896 17152 17408 17664 17920 18176 18432 18688 18944 19200 19456 19712 19968 20224
+; CHECK-NEXT:  20480 20736 20992 21248 21504 21760 22016 22272 22528 22784 23040 23296 23552 23808 24064 24320
+; CHECK-NEXT:  24576 24832 25088 25344 25600 25856 26112 26368 26624 26880 27136 27392 27648 27904 28160 28416
+; CHECK-NEXT:  28672 28928 29184 29440 29696 29952 30208 30464 30720 30976 31232 31488 31744 32000 32256 32512
+; CHECK-NEXT:  32768 33024 33280 33536 33792 34048 34304 34560 34816 35072 35328 35584 35840 36096 36352 36608
+; CHECK-NEXT:  36864 37120 37376 37632 37888 38144 38400 38656 38912 39168 39424 39680 39936 40192 40448 40704
+; CHECK-NEXT:  40960 41216 41472 41728 41984 42240 42496 42752 43008 43264 43520 43776 44032 44288 44544 44800
+; CHECK-NEXT:  45056 45312 45568 45824 46080 46336 46592 46848 47104 47360 47616 47872 48128 48384 48640 48896
+; CHECK-NEXT:  49152 49408 49664 49920 50176 50432 50688 50944 51200 51456 51712 51968 52224 52480 52736 52992
+; CHECK-NEXT:  53248 53504 53760 54016 54272 54528 54784 55040 55296 55552 55808 56064 56320 56576 56832 57088
+; CHECK-NEXT:  57344 57600 57856 58112 58368 58624 58880 59136 59392 59648 59904 60160 60416 60672 60928 61184
+; CHECK-NEXT:  61440 61696 61952 62208 62464 62720 62976 63232 63488 63744 64000 64256 64512 64768 65024 65280
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @crc16.be.tc8.crc.init.arg.flipped.sb.check(i16 %crc.init) {
+; CHECK-LABEL: 'crc16.be.tc8.crc.init.arg.flipped.sb.check'
+; CHECK-NEXT:  Found big-endian CRC-16 loop with trip count 8
+; CHECK-NEXT:    Initial CRC: i16 %crc.init
+; CHECK-NEXT:    Generating polynomial: 4129
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 256 512 768 1024 1280 1536 1792 2048 2304 2560 2816 3072 3328 3584 3840
+; CHECK-NEXT:  4096 4352 4608 4864 5120 5376 5632 5888 6144 6400 6656 6912 7168 7424 7680 7936
+; CHECK-NEXT:  8192 8448 8704 8960 9216 9472 9728 9984 10240 10496 10752 11008 11264 11520 11776 12032
+; CHECK-NEXT:  12288 12544 12800 13056 13312 13568 13824 14080 14336 14592 14848 15104 15360 15616 15872 16128
+; CHECK-NEXT:  16384 16640 16896 17152 17408 17664 17920 18176 18432 18688 18944 19200 19456 19712 19968 20224
+; CHECK-NEXT:  20480 20736 20992 21248 21504 21760 22016 22272 22528 22784 23040 23296 23552 23808 24064 24320
+; CHECK-NEXT:  24576 24832 25088 25344 25600 25856 26112 26368 26624 26880 27136 27392 27648 27904 28160 28416
+; CHECK-NEXT:  28672 28928 29184 29440 29696 29952 30208 30464 30720 30976 31232 31488 31744 32000 32256 32512
+; CHECK-NEXT:  32768 33024 33280 33536 33792 34048 34304 34560 34816 35072 35328 35584 35840 36096 36352 36608
+; CHECK-NEXT:  36864 37120 37376 37632 37888 38144 38400 38656 38912 39168 39424 39680 39936 40192 40448 40704
+; CHECK-NEXT:  40960 41216 41472 41728 41984 42240 42496 42752 43008 43264 43520 43776 44032 44288 44544 44800
+; CHECK-NEXT:  45056 45312 45568 45824 46080 46336 46592 46848 47104 47360 47616 47872 48128 48384 48640 48896
+; CHECK-NEXT:  49152 49408 49664 49920 50176 50432 50688 50944 51200 51456 51712 51968 52224 52480 52736 52992
+; CHECK-NEXT:  53248 53504 53760 54016 54272 54528 54784 55040 55296 55552 55808 56064 56320 56576 56832 57088
+; CHECK-NEXT:  57344 57600 57856 58112 58368 58624 58880 59136 59392 59648 59904 60160 60416 60672 60928 61184
+; CHECK-NEXT:  61440 61696 61952 62208 62464 62720 62976 63232 63488 63744 64000 64256 64512 64768 65024 65280
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp sge i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i8 @crc8.be.tc8.ptr.nested.loop(ptr %msg, i32 %loop.limit) {
+; CHECK-LABEL: 'crc8.be.tc8.ptr.nested.loop'
+; CHECK-NEXT:  Found big-endian CRC-8 loop with trip count 8
+; CHECK-NEXT:    Initial CRC: %crc.init = xor i8 %msg.load, %crc.outer
+; CHECK-NEXT:    Generating polynomial: 29
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i8 %crc.xor, i8 %crc.shl
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 29 58 39 116 105 78 83 232 245 210 207 156 129 166 187
+; CHECK-NEXT:  205 208 247 234 185 164 131 158 37 56 31 2 81 76 107 118
+; CHECK-NEXT:  135 154 189 160 243 238 201 212 111 114 85 72 27 6 33 60
+; CHECK-NEXT:  74 87 112 109 62 35 4 25 162 191 152 133 214 203 236 241
+; CHECK-NEXT:  19 14 41 52 103 122 93 64 251 230 193 220 143 146 181 168
+; CHECK-NEXT:  222 195 228 249 170 183 144 141 54 43 12 17 66 95 120 101
+; CHECK-NEXT:  148 137 174 179 224 253 218 199 124 97 70 91 8 21 50 47
+; CHECK-NEXT:  89 68 99 126 45 48 23 10 177 172 139 150 197 216 255 226
+; CHECK-NEXT:  38 59 28 1 82 79 104 117 206 211 244 233 186 167 128 157
+; CHECK-NEXT:  235 246 209 204 159 130 165 184 3 30 57 36 119 106 77 80
+; CHECK-NEXT:  161 188 155 134 213 200 239 242 73 84 115 110 61 32 7 26
+; CHECK-NEXT:  108 113 86 75 24 5 34 63 132 153 190 163 240 237 202 215
+; CHECK-NEXT:  53 40 15 18 65 92 123 102 221 192 231 250 169 180 147 142
+; CHECK-NEXT:  248 229 194 223 140 145 182 171 16 13 42 55 100 121 94 67
+; CHECK-NEXT:  178 175 136 149 198 219 252 225 90 71 96 125 46 51 20 9
+; CHECK-NEXT:  127 98 69 88 11 22 49 44 151 138 173 176 227 254 217 196
+;
+entry:
+  br label %outer.loop
+
+outer.loop:                                        ; preds = %inner.exit, %entry
+  %crc.outer = phi i8 [ 0, %entry ], [ %crc.next, %inner.exit ]
+  %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %inner.exit ]
+  %outer.exit.cond = icmp ult i32 %outer.iv, %loop.limit
+  br i1 %outer.exit.cond, label %ph, label %exit
+
+ph:                                                ; preds = %outer.loop
+  %outer.iv.ext = sext i32 %outer.iv to i64
+  %msg.outer.iv = getelementptr inbounds i8, ptr %msg, i64 %outer.iv.ext
+  %msg.load = load i8, ptr %msg.outer.iv, align 1
+  %crc.init = xor i8 %msg.load, %crc.outer
+  br label %inner.loop
+
+inner.loop:                                        ; preds = %inner.loop, %ph
+  %inner.iv = phi i32 [ 0, %ph ], [ %inner.iv.next, %inner.loop ]
+  %crc = phi i8 [ %crc.init, %ph ], [ %crc.next, %inner.loop ]
+  %crc.shl = shl i8 %crc, 1
+  %crc.xor = xor i8 %crc.shl, 29
+  %check.sb = icmp slt i8 %crc, 0
+  %crc.next = select i1 %check.sb, i8 %crc.xor, i8 %crc.shl
+  %inner.iv.next = add nuw nsw i32 %inner.iv, 1
+  %exit.cond = icmp samesign ult i32 %inner.iv, 7
+  br i1 %exit.cond, label %inner.loop, label %inner.exit
+
+inner.exit:                                        ; preds = %inner.loop
+  %outer.iv.next = add i32 %outer.iv, 1
+  br label %outer.loop
+
+exit:                                              ; preds = %outer.loop
+  ret i8 %crc.outer
+}
+
+define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) {
+; CHECK-LABEL: 'crc32.le.tc8.data32'
+; CHECK-NEXT:  Found little-endian CRC-32 loop with trip count 8
+; CHECK-NEXT:    Initial CRC: i32 %checksum
+; CHECK-NEXT:    Generating polynomial: 33800
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+; CHECK-NEXT:    Auxiliary data: i32 %msg
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 4489 8978 12955 17956 22445 25910 29887 35912 40385 44890 48851 51820 56293 59774 63735
+; CHECK-NEXT:  4225 264 13203 8730 22181 18220 30135 25662 40137 36160 49115 44626 56045 52068 63999 59510
+; CHECK-NEXT:  8450 12427 528 5017 26406 30383 17460 21949 44362 48323 36440 40913 60270 64231 51324 55797
+; CHECK-NEXT:  12675 8202 4753 792 30631 26158 21685 17724 48587 44098 40665 36688 64495 60006 55549 51572
+; CHECK-NEXT:  16900 21389 24854 28831 1056 5545 10034 14011 52812 57285 60766 64727 34920 39393 43898 47859
+; CHECK-NEXT:  21125 17164 29079 24606 5281 1320 14259 9786 57037 53060 64991 60502 39145 35168 48123 43634
+; CHECK-NEXT:  25350 29327 16404 20893 9506 13483 1584 6073 61262 65223 52316 56789 43370 47331 35448 39921
+; CHECK-NEXT:  29575 25102 20629 16668 13731 9258 5809 1848 65487 60998 56541 52564 47595 43106 39673 35696
+; CHECK-NEXT:  33800 38273 42778 46739 49708 54181 57662 61623 2112 6601 11090 15067 20068 24557 28022 31999
+; CHECK-NEXT:  38025 34048 47003 42514 53933 49956 61887 57398 6337 2376 15315 10842 24293 20332 32247 27774
+; CHECK-NEXT:  42250 46211 34328 38801 58158 62119 49212 53685 10562 14539 2640 7129 28518 32495 19572 24061
+; CHECK-NEXT:  46475 41986 38553 34576 62383 57894 53437 49460 14787 10314 6865 2904 32743 28270 23797 19836
+; CHECK-NEXT:  50700 55173 58654 62615 32808 37281 41786 45747 19012 23501 26966 30943 3168 7657 12146 16123
+; CHECK-NEXT:  54925 50948 62879 58390 37033 33056 46011 41522 23237 19276 31191 26718 7393 3432 16371 11898
+; CHECK-NEXT:  59150 63111 50204 54677 41258 45219 33336 37809 27462 31439 18516 23005 11618 15595 3696 8185
+; CHECK-NEXT:  63375 58886 54429 50452 45483 40994 37561 33584 31687 27214 22741 18780 15843 11370 7921 3960
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %xor.crc.data = xor i32 %crc, %data
+  %sb.crc.data = and i32 %xor.crc.data, 1
+  %check.sb = icmp eq i32 %sb.crc.data, 0
+  %crc.lshr = lshr i32 %crc, 1
+  %crc.xor = xor i32 %crc.lshr, 33800
+  %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %data.next = lshr i32 %data, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i32 %crc.next
+}
+
+; Negative tests
+
+define i16 @not.crc.non.const.tc(i16 %crc.init, i32 %loop.limit) {
+; CHECK-LABEL: 'not.crc.non.const.tc'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Unable to find a small constant trip count
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp sge i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, %loop.limit
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.non.canonical.loop(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.non.canonical.loop'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop not in canonical form
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 7, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = sub nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign eq i32 %iv, 0
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.tc.limit(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.tc.limit'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Unable to find a small constant trip count
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 512
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.no.conditional.recurrence(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.no.conditional.recurrence'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Unable to find conditional recurrence
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %shl = shl i16 %crc, 1
+  %crc.next = xor i16 %shl, 258
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+
+define i16 @not.crc.bad.shift.recurrence(i16 %checksum, i8 %msg) {
+; CHECK-LABEL: 'not.crc.bad.shift.recurrence'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop with non-unit bitshifts
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %crc.lshr = lshr i16 %crc, 8
+  %data.ext = zext i8 %data to i16
+  %xor.crc.data = xor i16 %crc.lshr, %data.ext
+  %check.sb = icmp samesign ult i16 %xor.crc.data, 128
+  %crc.and = and i16 %crc, 32767
+  %crc.xor = xor i16 %crc.and, 258
+  %crc.next = select i1 %check.sb, i16 %crc.and, i16 %crc.xor
+  %data.next = shl i8 %data, 1
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.nonunit.shifts(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.nonunit.shifts'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop with non-unit bitshifts
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 2
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.result.unused(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.result.unused'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Unable to find use of computed value in loop exit block
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc
+}
+
+define i16 @not.crc.wrong.sb.check.const(i8 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.wrong.sb.check.const'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %crc.lshr = lshr i16 %crc, 8
+  %data.ext = zext i8 %data to i16
+  %xor.crc.data = xor i16 %crc.lshr, %data.ext
+  %check.sb = icmp samesign ult i16 %xor.crc.data, 128
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 258
+  %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+  %data.next = shl i8 %data, 1
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.wrong.sb.check.pred(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.wrong.sb.check.pred'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Bad RHS of significant-bit-check
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp sgt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.excess.tc(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.excess.tc'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop iterations exceed bitwidth of result
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %and.crc.data = and i16 %xor.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 20
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i32 @not.crc.unknown.icmp.rhs(i32 %checksum, i32 %msg, i32 %unknown) {
+; CHECK-LABEL: 'not.crc.unknown.icmp.rhs'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Bad LHS of significant-bit-check
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %xor.crc.data = xor i32 %crc, %data
+  %sb.crc.data = or i32 %xor.crc.data, 1
+  %check.sb = icmp eq i32 %sb.crc.data, %unknown
+  %crc.lshr = lshr i32 %crc, 1
+  %crc.xor = xor i32 %crc.lshr, 33800
+  %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %data.next = lshr i32 %data, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i32 %crc.next
+}
+
+define i32 @not.crc.unknown.icmp.lhs(i32 %checksum, i32 %msg, i32 %unknown) {
+; CHECK-LABEL: 'not.crc.unknown.icmp.lhs'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Bad LHS of significant-bit-check
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %xor.crc.data = xor i32 %crc, %data
+  %sb.crc.data = or i32 %xor.crc.data, %unknown
+  %check.sb = icmp eq i32 %sb.crc.data, 0
+  %crc.lshr = lshr i32 %crc, 1
+  %crc.xor = xor i32 %crc.lshr, 33800
+  %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %data.next = lshr i32 %data, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i32 %crc.next
+}
+
+
+define i16 @not.crc.stray.or(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.stray.or'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Bad LHS of significant-bit-check
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %and.crc.data = and i16 %xor.crc.data, 1
+  %crc.corrupt = or i16 %and.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp ne i16 %crc.corrupt, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.inverse.sb.check(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.inverse.sb.check'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Expected top 16 bits zero (1100000000000001)
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %and.crc.data = and i16 %xor.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp ne i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @crc1.tc8.sb.check.endian.mismatch(i8 %msg, i16 %checksum) {
+; CHECK-LABEL: 'crc1.tc8.sb.check.endian.mismatch'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Bad RHS of significant-bit-check
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.trunc = trunc i16 %crc to i8
+  %xor.data.crc = xor i8 %data, %crc.trunc
+  %and.data.crc = and i8 %xor.data.crc, 1
+  %data.next = mul i8 %data, 2
+  %check.sb = icmp eq i8 %and.data.crc, 0
+  %crc.lshr = mul i16 %crc, 2
+  %xor = xor i16 %crc.lshr, 0
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.init.arg.inverted.select(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.init.arg.inverted.select'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Expected top 8 bits zero (11000000????????)
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %sb.crc = and i16 %crc, 1
+  %check.sb = icmp eq i16 %sb.crc, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.lshr
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i32 @not.crc.dead.msg.bad.use(i32 %checksum, i32 %msg) {
+; CHECK-LABEL: 'not.crc.dead.msg.bad.use'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %data.or = or i32 %data, -1
+  %xor.crc.data = xor i32 %crc, %data.or
+  %sb.crc.data = and i32 %xor.crc.data, 1
+  %check.sb = icmp eq i32 %sb.crc.data, 0
+  %crc.lshr = lshr i32 %crc, 1
+  %crc.xor = xor i32 %crc.lshr, 33800
+  %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %data.next = lshr i32 %data, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i32 %crc.next
+}
+
+define i16 @not.crc.dead.msg.no.use(i8 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.dead.msg.no.use'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.trunc = trunc i16 %crc to i8
+  %and.crc = and i8 %crc.trunc, 1
+  %data.next = lshr i8 %data, 1
+  %check.sb = icmp eq i8 %and.crc, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  %data.zext = zext i8 %data.next to i16
+  %ret = xor i16 %crc.next, %data.zext
+  ret i16 %ret
+}
+
+define i32 @not.crc.dead.msg.wrong.op(i32 %checksum, i32 %msg) {
+; CHECK-LABEL: 'not.crc.dead.msg.wrong.op'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %or.crc.data = or i32 %crc, %data
+  %sb.crc.data = and i32 %or.crc.data, 1
+  %check.sb = icmp eq i32 %sb.crc.data, 0
+  %crc.lshr = lshr i32 %crc, 1
+  %crc.xor = xor i32 %crc.lshr, 33800
+  %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %data.next = lshr i32 %data, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i32 %crc.next
+}
+
+define i16 @not.crc.float.simple.recurrence(float %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.float.simple.recurrence'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop with non-unit bitshifts
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi float [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.conv = sitofp i16 %crc to float
+  %frem.data.crc = frem float %data, %crc.conv
+  %and.data.crc = fdiv float %frem.data.crc, 2.0
+  %data.next = fdiv float %data, 2.0
+  %check.sb = fcmp oeq float %and.data.crc, 0.0
+  %crc.lshr = lshr i16 %crc, 1
+  %xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.stray.phi(i8 %msg, i16 %checksum, i1 %c) {
+; CHECK-LABEL: 'not.crc.stray.phi'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Found stray PHI
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.trunc = trunc i16 %crc to i8
+  %xor.data.crc = xor i8 %data, %crc.trunc
+  %and.data.crc = and i8 %xor.data.crc, 1
+  %data.next = select i1 %c, i8 %data, i8 1
+  %check.sb = icmp eq i8 %and.data.crc, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
diff --git a/external/llvm-project/llvm/test/Analysis/KernelInfo/openmp/amdgpu.ll b/external/llvm-project/llvm/test/Analysis/KernelInfo/openmp/amdgpu.ll
index 049142732aa1..a84e261357de 100644
--- a/external/llvm-project/llvm/test/Analysis/KernelInfo/openmp/amdgpu.ll
+++ b/external/llvm-project/llvm/test/Analysis/KernelInfo/openmp/amdgpu.ll
@@ -182,7 +182,7 @@ attributes #5 = { nounwind }
 !10 = !{i32 7, !"frame-pointer", i32 2}
 !11 = !{i32 4, !"amdgpu_hostcall", i32 1}
 !12 = !{!"clang version 20.0.0git (/tmp/llvm/clang b9447c03a9ef2eed55b685a33511df86f7f94e89)"}
-!13 = !{!"AMD clang version 17.0.0 (https://github.com/RadeonOpenCompute/llvm-project roc-6.0.2 24012 af27734ed982b52a9f1be0f035ac91726fc697e4)"}
+!13 = !{!"AMD clang version 17.0.0 (https://github.com/ROCm/llvm-project roc-6.0.2 24012 af27734ed982b52a9f1be0f035ac91726fc697e4)"}
 !14 = !{i32 2, i32 0}
 !15 = distinct !DISubprogram(name: "__omp_offloading_fd02_727e9_h_l12_debug__", scope: !16, file: !16, line: 13, type: !17, scopeLine: 13, flags: DIFlagArtificial | DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, retainedNodes: !22)
 !16 = !DIFile(filename: "test.c", directory: "/tmp")
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/allow-partial.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/allow-partial.ll
new file mode 100644
index 000000000000..3096d545eb6d
--- /dev/null
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/allow-partial.ll
@@ -0,0 +1,99 @@
+; RUN: opt -disable-output -passes='print<access-info><allow-partial>,print<access-info>' %s 2>&1 | FileCheck %s --check-prefixes=ALLOW-BEFORE
+; RUN: opt -disable-output -passes='print<access-info>,print<access-info><allow-partial>' %s 2>&1 | FileCheck %s --check-prefixes=ALLOW-AFTER
+
+; Check that we get the right results when loop access analysis is run twice,
+; once without partial results and once with.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+
+define void @gep_loaded_offset(ptr %p, ptr %q, ptr %r, i32 %n) {
+; ALLOW-BEFORE-LABEL: 'gep_loaded_offset'
+; ALLOW-BEFORE-NEXT:    while.body:
+; ALLOW-BEFORE-NEXT:      Report: cannot identify array bounds
+; ALLOW-BEFORE-NEXT:      Dependences:
+; ALLOW-BEFORE-NEXT:      Run-time memory checks:
+; ALLOW-BEFORE-NEXT:      Check 0:
+; ALLOW-BEFORE-NEXT:        Comparing group GRP0:
+; ALLOW-BEFORE-NEXT:          %p.addr = phi ptr [ %incdec.ptr, %while.body ], [ %p, %entry ]
+; ALLOW-BEFORE-NEXT:        Against group GRP1:
+; ALLOW-BEFORE-NEXT:        ptr %r
+; ALLOW-BEFORE-NEXT:      Grouped accesses:
+; ALLOW-BEFORE-NEXT:        Group GRP0:
+; ALLOW-BEFORE-NEXT:          (Low: %p High: (4 + (4 * (zext i32 (-1 + %n)<nsw> to i64))<nuw><nsw> + %p))
+; ALLOW-BEFORE-NEXT:            Member: {%p,+,4}<nuw><%while.body>
+; ALLOW-BEFORE-NEXT:        Group GRP1:
+; ALLOW-BEFORE-NEXT:          (Low: %r High: (8 + %r))
+; ALLOW-BEFORE-NEXT:            Member: %r
+; ALLOW-BEFORE-NEXT:      Generated run-time checks are incomplete
+; ALLOW-BEFORE-EMPTY:
+; ALLOW-BEFORE-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; ALLOW-BEFORE-NEXT:      SCEV assumptions:
+; ALLOW-BEFORE-EMPTY:
+; ALLOW-BEFORE-NEXT:      Expressions re-written:
+;
+; ALLOW-BEFORE-LABEL: 'gep_loaded_offset'
+; ALLOW-BEFORE-NEXT:    while.body:
+; ALLOW-BEFORE-NEXT:      Report: cannot identify array bounds
+; ALLOW-BEFORE-NEXT:      Dependences:
+; ALLOW-BEFORE-NEXT:      Run-time memory checks:
+; ALLOW-BEFORE-NEXT:      Grouped accesses:
+; ALLOW-BEFORE-EMPTY:
+; ALLOW-BEFORE-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; ALLOW-BEFORE-NEXT:      SCEV assumptions:
+; ALLOW-BEFORE-EMPTY:
+; ALLOW-BEFORE-NEXT:      Expressions re-written:
+;
+; ALLOW-AFTER-LABEL: 'gep_loaded_offset'
+; ALLOW-AFTER-NEXT:    while.body:
+; ALLOW-AFTER-NEXT:      Report: cannot identify array bounds
+; ALLOW-AFTER-NEXT:      Dependences:
+; ALLOW-AFTER-NEXT:      Run-time memory checks:
+; ALLOW-AFTER-NEXT:      Grouped accesses:
+; ALLOW-AFTER-EMPTY:
+; ALLOW-AFTER-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; ALLOW-AFTER-NEXT:      SCEV assumptions:
+; ALLOW-AFTER-EMPTY:
+; ALLOW-AFTER-NEXT:      Expressions re-written:
+;
+; ALLOW-AFTER-LABEL: 'gep_loaded_offset'
+; ALLOW-AFTER-NEXT:    while.body:
+; ALLOW-AFTER-NEXT:      Report: cannot identify array bounds
+; ALLOW-AFTER-NEXT:      Dependences:
+; ALLOW-AFTER-NEXT:      Run-time memory checks:
+; ALLOW-AFTER-NEXT:      Check 0:
+; ALLOW-AFTER-NEXT:        Comparing group GRP0:
+; ALLOW-AFTER-NEXT:          %p.addr = phi ptr [ %incdec.ptr, %while.body ], [ %p, %entry ]
+; ALLOW-AFTER-NEXT:        Against group GRP1:
+; ALLOW-AFTER-NEXT:        ptr %r
+; ALLOW-AFTER-NEXT:      Grouped accesses:
+; ALLOW-AFTER-NEXT:        Group GRP0:
+; ALLOW-AFTER-NEXT:          (Low: %p High: (4 + (4 * (zext i32 (-1 + %n)<nsw> to i64))<nuw><nsw> + %p))
+; ALLOW-AFTER-NEXT:            Member: {%p,+,4}<nuw><%while.body>
+; ALLOW-AFTER-NEXT:        Group GRP1:
+; ALLOW-AFTER-NEXT:          (Low: %r High: (8 + %r))
+; ALLOW-AFTER-NEXT:            Member: %r
+; ALLOW-AFTER-NEXT:      Generated run-time checks are incomplete
+; ALLOW-AFTER-EMPTY:
+; ALLOW-AFTER-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; ALLOW-AFTER-NEXT:      SCEV assumptions:
+; ALLOW-AFTER-EMPTY:
+; ALLOW-AFTER-NEXT:      Expressions re-written:
+;
+entry:
+  br label %while.body
+
+while.body:
+  %n.addr = phi i32 [ %dec, %while.body ], [ %n, %entry ]
+  %p.addr = phi ptr [ %incdec.ptr, %while.body ], [ %p, %entry ]
+  %dec = add nsw i32 %n.addr, -1
+  %rval = load i64, ptr %r, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %q, i64 %rval
+  %val = load i32, ptr %arrayidx, align 4
+  %incdec.ptr = getelementptr inbounds nuw i8, ptr %p.addr, i64 4
+  store i32 %val, ptr %p.addr, align 4
+  %tobool.not = icmp eq i32 %dec, 0
+  br i1 %tobool.not, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/forked-pointers.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/forked-pointers.ll
index c28dd58fcb20..fa72ae910cbb 100644
--- a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/forked-pointers.ll
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/forked-pointers.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 3
-; RUN: opt -disable-output -passes='print<access-info>' %s 2>&1 | FileCheck %s --check-prefixes=CHECK,FULLDEPTH
-; RUN: opt -disable-output -passes='print<access-info>' -max-forked-scev-depth=2 %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DEPTH2
+; RUN: opt -disable-output -passes='print<access-info><allow-partial>' %s 2>&1 | FileCheck %s --check-prefixes=CHECK,FULLDEPTH
+; RUN: opt -disable-output -passes='print<access-info><allow-partial>' -max-forked-scev-depth=2 %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DEPTH2
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -404,7 +404,19 @@ define dso_local void @forked_ptrs_same_base_different_offset(ptr nocapture read
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %arrayidx5 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (400 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %Preds High: (400 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -475,7 +487,19 @@ define dso_local void @forked_ptrs_add_to_offset(ptr nocapture readonly %Base, p
 ; DEPTH2-NEXT:      Report: cannot identify array bounds
 ; DEPTH2-NEXT:      Dependences:
 ; DEPTH2-NEXT:      Run-time memory checks:
+; DEPTH2-NEXT:      Check 0:
+; DEPTH2-NEXT:        Comparing group GRP0:
+; DEPTH2-NEXT:          %arrayidx5 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; DEPTH2-NEXT:        Against group GRP1:
+; DEPTH2-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; DEPTH2-NEXT:      Grouped accesses:
+; DEPTH2-NEXT:        Group GRP0:
+; DEPTH2-NEXT:          (Low: %Dest High: (400 + %Dest))
+; DEPTH2-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; DEPTH2-NEXT:        Group GRP1:
+; DEPTH2-NEXT:          (Low: %Preds High: (400 + %Preds))
+; DEPTH2-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; DEPTH2-NEXT:      Generated run-time checks are incomplete
 ; DEPTH2-EMPTY:
 ; DEPTH2-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; DEPTH2-NEXT:      SCEV assumptions:
@@ -543,7 +567,19 @@ define dso_local void @forked_ptrs_sub_from_offset(ptr nocapture readonly %Base,
 ; DEPTH2-NEXT:      Report: cannot identify array bounds
 ; DEPTH2-NEXT:      Dependences:
 ; DEPTH2-NEXT:      Run-time memory checks:
+; DEPTH2-NEXT:      Check 0:
+; DEPTH2-NEXT:        Comparing group GRP0:
+; DEPTH2-NEXT:          %arrayidx5 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; DEPTH2-NEXT:        Against group GRP1:
+; DEPTH2-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; DEPTH2-NEXT:      Grouped accesses:
+; DEPTH2-NEXT:        Group GRP0:
+; DEPTH2-NEXT:          (Low: %Dest High: (400 + %Dest))
+; DEPTH2-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; DEPTH2-NEXT:        Group GRP1:
+; DEPTH2-NEXT:          (Low: %Preds High: (400 + %Preds))
+; DEPTH2-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; DEPTH2-NEXT:      Generated run-time checks are incomplete
 ; DEPTH2-EMPTY:
 ; DEPTH2-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; DEPTH2-NEXT:      SCEV assumptions:
@@ -611,7 +647,19 @@ define dso_local void @forked_ptrs_add_sub_offset(ptr nocapture readonly %Base,
 ; DEPTH2-NEXT:      Report: cannot identify array bounds
 ; DEPTH2-NEXT:      Dependences:
 ; DEPTH2-NEXT:      Run-time memory checks:
+; DEPTH2-NEXT:      Check 0:
+; DEPTH2-NEXT:        Comparing group GRP0:
+; DEPTH2-NEXT:          %arrayidx5 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; DEPTH2-NEXT:        Against group GRP1:
+; DEPTH2-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; DEPTH2-NEXT:      Grouped accesses:
+; DEPTH2-NEXT:        Group GRP0:
+; DEPTH2-NEXT:          (Low: %Dest High: (400 + %Dest))
+; DEPTH2-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; DEPTH2-NEXT:        Group GRP1:
+; DEPTH2-NEXT:          (Low: %Preds High: (400 + %Preds))
+; DEPTH2-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; DEPTH2-NEXT:      Generated run-time checks are incomplete
 ; DEPTH2-EMPTY:
 ; DEPTH2-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; DEPTH2-NEXT:      SCEV assumptions:
@@ -649,7 +697,19 @@ define dso_local void @forked_ptrs_mul_by_offset(ptr nocapture readonly %Base, p
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %arrayidx5 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (400 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %Preds High: (400 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -687,7 +747,19 @@ define dso_local void @forked_ptrs_uniform_and_strided_forks(ptr nocapture reado
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %arrayidx5 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (400 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %Preds High: (400 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -730,7 +802,19 @@ define dso_local void @forked_ptrs_gather_and_contiguous_forks(ptr nocapture rea
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %1 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (400 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %Preds High: (400 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -769,7 +853,19 @@ define dso_local void @forked_ptrs_two_forks_gep(ptr nocapture readonly %Base1,
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %1 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (400 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %Preds High: (400 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -807,6 +903,11 @@ define void @forked_ptrs_two_select(ptr nocapture readonly %Base1, ptr nocapture
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (400 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%loop>
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -845,7 +946,19 @@ define void @forked_ptrs_too_many_gep_ops(ptr nocapture readonly %Base1, ptr noc
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %1 = getelementptr inbounds float, ptr %Dest, i64 %indvars.iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (400 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,4}<nuw><%for.body>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %Preds High: (400 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -881,7 +994,19 @@ define void @forked_ptrs_vector_gep(ptr nocapture readonly %Base1, ptr nocapture
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %1 = getelementptr inbounds <4 x float>, ptr %Dest, i64 %indvars.iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %arrayidx = getelementptr inbounds i32, ptr %Preds, i64 %indvars.iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Dest High: (1552 + %Dest))
+; CHECK-NEXT:            Member: {%Dest,+,64}<nuw><%for.body>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %Preds High: (388 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,16}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -1056,6 +1181,10 @@ define void @forked_ptrs_with_different_base3(ptr nocapture readonly %Preds, ptr
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %Preds High: (31996 + %Preds))
+; CHECK-NEXT:            Member: {%Preds,+,4}<nuw><%for.body>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-backward.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-backward.ll
index d263749ea1f4..3160a77e0ba5 100644
--- a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-backward.ll
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-backward.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info><allow-partial>' -disable-output %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
@@ -237,6 +237,10 @@ define void @different_non_constant_strides_known_backward_via_assume(ptr %A, i6
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (1024 + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -271,6 +275,10 @@ define void @different_non_constant_strides_known_backward_via_assume_distance_l
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (1024 + %A)<nuw>)
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -306,6 +314,10 @@ define void @different_non_constant_strides_known_backward_via_assume_min_distan
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (1024 + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -341,6 +353,10 @@ define void @different_non_constant_strides_not_known_backward(ptr %A, i64 %scal
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (1024 + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-forward.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-forward.ll
index 5f4c732dc19d..9e0a1e3a03dd 100644
--- a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-forward.ll
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-forward.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info><allow-partial>' -disable-output %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
@@ -85,6 +85,10 @@ define void @different_non_constant_strides_known_forward_via_assume(ptr %A, i64
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (1024 + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -119,6 +123,10 @@ define void @different_non_constant_strides_known_forward_via_assume_min_distanc
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (1024 + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -154,6 +162,10 @@ define void @different_non_constant_strides_not_known_forward(ptr %A, i64 %scale
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (1024 + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
index bd38c50cb15d..6fbe0e45976b 100644
--- a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 3
-; RUN: opt -passes='print<access-info>' -disable-output  < %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info><allow-partial>' -disable-output  < %s 2>&1 | FileCheck %s
 
 %s1 = type { [32000 x double], [32000 x double], [32000 x double] }
 
@@ -462,6 +462,10 @@ define i32 @store_with_pointer_phi_in_same_bb_use_other_phi(ptr %A, ptr %B, ptr
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %A High: (256000 + %A))
+; CHECK-NEXT:            Member: {%A,+,8}<nuw><%loop.header>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll
index 0763fd220d02..d1d1ecb2af88 100644
--- a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info><allow-partial>' -disable-output %s 2>&1 | FileCheck %s
 
 
 define void @dependency_check_and_runtime_checks_needed_select_of_invariant_ptrs(ptr %a, ptr %b, ptr %c, i64 %offset, i64 %n) {
@@ -167,6 +167,14 @@ define void @dependency_check_and_runtime_checks_needed_select_of_ptr_add_recs_m
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
+; CHECK-NEXT:            Member: {((4 * %offset) + %a),+,4}<%loop>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %a High: ((4 * %n) + %a))
+; CHECK-NEXT:            Member: {%a,+,4}<nuw><%loop>
+; CHECK-NEXT:            Member: {%a,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -208,6 +216,14 @@ define void @dependency_check_and_runtime_checks_needed_select_of_ptr_add_recs_m
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
+; CHECK-NEXT:            Member: {((4 * %offset) + %a),+,4}<%loop>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %a High: ((4 * %n) + %a))
+; CHECK-NEXT:            Member: {%a,+,4}<nuw><%loop>
+; CHECK-NEXT:            Member: {%a,+,4}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll
index 402081fb939f..0708f908211e 100644
--- a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info><allow-partial>' -disable-output %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
@@ -80,6 +80,10 @@ define void @indirect_ptr_recurrences_read_write_may_alias_no_tbaa(ptr %A, ptr %
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: (8 + %B)<nuw> High: (40 + %B))
+; CHECK-NEXT:            Member: {(8 + %B)<nuw>,+,8}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -111,7 +115,19 @@ define void @indirect_ptr_recurrences_read_write_may_alias_different_obj(ptr %A,
 ; CHECK-NEXT:      Report: cannot identify array bounds
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %gep.C = getelementptr inbounds ptr, ptr %C, i64 %iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %gep.B = getelementptr inbounds ptr, ptr %B, i64 %iv
 ; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: (8 + %C)<nuw> High: (36 + %C))
+; CHECK-NEXT:            Member: {(8 + %C)<nuw>,+,8}<nuw><%loop>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: (8 + %B)<nuw> High: (40 + %B))
+; CHECK-NEXT:            Member: {(8 + %B)<nuw>,+,8}<nuw><%loop>
+; CHECK-NEXT:      Generated run-time checks are incomplete
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
index abfdff79dc11..1d3512128678 100644
--- a/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
+++ b/external/llvm-project/llvm/test/Analysis/LoopAccessAnalysis/underlying-objects-2.ll
@@ -127,9 +127,12 @@ for_j.body:
   %gepB7 = getelementptr inbounds i8, ptr %gepB6, i64 0
   %gepB8 = getelementptr inbounds i8, ptr %gepB7, i64 0
   %gepB9 = getelementptr inbounds i8, ptr %gepB8, i64 0
+  %gepB10 = getelementptr inbounds i8, ptr %gepB9, i64 0
+  %gepB11 = getelementptr inbounds i8, ptr %gepB10, i64 0
+  %gepB12 = getelementptr inbounds i8, ptr %gepB11, i64 0
 
   %loadPrev = load i8, ptr %gepPrev, align 1
-  %loadB = load i8, ptr %gepB9, align 1
+  %loadB = load i8, ptr %gepB12, align 1
 
   %mul = mul i8 %loadPrev, %loadB
 
diff --git a/external/llvm-project/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll b/external/llvm-project/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
index 989049ab67a0..840a5172561d 100644
--- a/external/llvm-project/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
+++ b/external/llvm-project/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
@@ -1,17 +1,20 @@
 ; RUN: opt < %s -S -passes=memcpyopt | FileCheck --match-full-lines %s
 
+declare void @use(ptr)
+
 ; Alias scopes are merged by taking the intersection of domains, then the union of the scopes within those domains
 define i8 @test(i8 %input) {
   %tmp = alloca i8
   %dst = alloca i8
   %src = alloca i8
 ; CHECK:   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 1, i1 false), !alias.scope ![[SCOPE:[0-9]+]]
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %src), !noalias !4
+  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %src), !noalias !4
   store i8 %input, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %src, i64 1, i1 false), !alias.scope !0
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %src), !noalias !4
+  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %src), !noalias !4
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 1, i1 false), !alias.scope !4
   %ret_value = load i8, ptr %dst
+  call void @use(ptr %src)
   ret i8 %ret_value
 }
 
diff --git a/external/llvm-project/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/external/llvm-project/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index b7bdca42d559..a17f11a680aa 100644
--- a/external/llvm-project/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/external/llvm-project/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -307,9 +307,9 @@ define void @nvvm_cp_async_bulk_intrinsics(ptr addrspace(3) %dst, ptr addrspace(
 
 ; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_im2col
 define void @nvvm_cp_async_bulk_tensor_g2s_im2col(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 0, i64 0, i1 false, i1 false)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 0, i64 0, i1 false, i1 false)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 0, i64 0, i1 false, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 0, i64 0, i1 false, i1 false, i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 0, i64 0, i1 false, i1 false, i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 0, i64 0, i1 false, i1 false, i32 0)
   call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 0, i64 0, i1 0, i1 0)
   call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 0, i64 0, i1 0, i1 0)
   call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 0, i64 0, i1 0, i1 0)
@@ -318,11 +318,11 @@ define void @nvvm_cp_async_bulk_tensor_g2s_im2col(ptr addrspace(3) %d, ptr addrs
 
 ; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_tile
 define void @nvvm_cp_async_bulk_tensor_g2s_tile(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 false, i1 false)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 0, i64 0, i1 false, i1 false)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 0, i64 0, i1 false, i1 false)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %4, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 0, i64 0, i1 false, i1 false)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %5, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 0, i64 0, i1 false, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 false, i1 false, i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 0, i64 0, i1 false, i1 false, i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 0, i64 0, i1 false, i1 false, i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %4, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 0, i64 0, i1 false, i1 false, i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %5, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 0, i64 0, i1 false, i1 false, i32 0)
   call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 0, i1 0)
   call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 0, i64 0, i1 0, i1 0)
   call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 0, i64 0, i1 0, i1 0)
diff --git a/external/llvm-project/llvm/test/Assembler/drop-debug-info-nonzero-alloca.ll b/external/llvm-project/llvm/test/Assembler/drop-debug-info-nonzero-alloca.ll
index 2b089d263937..c8b235757afb 100644
--- a/external/llvm-project/llvm/test/Assembler/drop-debug-info-nonzero-alloca.ll
+++ b/external/llvm-project/llvm/test/Assembler/drop-debug-info-nonzero-alloca.ll
@@ -12,8 +12,12 @@ entry:
       metadata ptr undef,
       metadata !DILocalVariable(scope: !1),
       metadata !DIExpression())
-; AS: llvm.dbg.value intrinsic requires a !dbg attachment
+; AS: invalid #dbg record DILocation
+; AS: #dbg_value(ptr undef, !{{[0-9]+}}, !DIExpression(), (null))
+; AS: label %entry
+; AS: ptr @foo
 ; AS: warning: ignoring invalid debug info in <stdin>
+
 ret void
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index d5ead2bff2f1..e0f2dd728c5e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -451,11 +451,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -470,11 +469,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -489,11 +487,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -508,11 +505,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -527,11 +523,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
index 282b0591d174..4a1b7642eff4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
@@ -77,11 +77,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -96,11 +95,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -115,11 +113,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -134,11 +131,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -153,11 +149,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -557,11 +552,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -576,11 +570,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -595,11 +588,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -614,11 +606,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -633,11 +624,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -1037,11 +1027,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -1056,11 +1045,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -1075,11 +1063,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -1094,11 +1081,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -1113,11 +1099,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -1517,11 +1502,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -1536,11 +1520,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -1555,11 +1538,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -1574,11 +1556,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -1593,11 +1574,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -1997,11 +1977,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, b
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -2016,11 +1995,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -2035,11 +2013,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -2054,11 +2031,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -2073,11 +2049,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_seq_cst(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -2477,11 +2452,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, b
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -2496,11 +2470,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -2515,11 +2488,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -2534,11 +2506,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -2553,11 +2524,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_seq_cst(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll
index d92899d9aa3e..ea6ebe08936f 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2.ll
@@ -9,7 +9,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -24,7 +23,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -39,7 +37,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -54,7 +51,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -69,7 +65,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -84,7 +79,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
 ; -O1:    ldxrh w0, [x8]
@@ -99,7 +93,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
 ; -O1:    ldaxrh w0, [x8]
@@ -114,7 +107,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
 ; -O1:    ldxrh w0, [x8]
@@ -129,7 +121,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w0, [x8]
@@ -144,7 +135,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w0, [x8]
@@ -159,7 +149,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_monotonic:
 ; -O1:    ldxr w0, [x8]
@@ -174,7 +163,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acquire:
 ; -O1:    ldaxr w0, [x8]
@@ -189,7 +177,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_release:
 ; -O1:    ldxr w0, [x8]
@@ -204,7 +191,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acq_rel:
 ; -O1:    ldaxr w0, [x8]
@@ -219,7 +205,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_seq_cst:
 ; -O1:    ldaxr w0, [x8]
@@ -234,7 +219,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -249,7 +233,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -264,7 +247,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -279,7 +261,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -294,7 +275,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -394,7 +374,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -409,7 +388,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -424,7 +402,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -439,7 +416,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -454,7 +430,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -610,7 +585,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -627,7 +601,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -644,7 +617,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -661,7 +633,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -678,7 +649,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -695,7 +665,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -712,7 +681,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -729,7 +697,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -746,7 +713,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -763,7 +729,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -780,7 +745,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -797,7 +761,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -814,7 +777,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -831,7 +793,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -848,7 +809,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -865,7 +825,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -882,7 +841,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -899,7 +857,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -916,7 +873,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -933,7 +889,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1045,7 +1000,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1062,7 +1016,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1079,7 +1032,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1096,7 +1048,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1113,7 +1064,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1375,7 +1325,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1392,7 +1341,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1409,7 +1357,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1426,7 +1373,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1443,7 +1389,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1460,7 +1405,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -1477,7 +1421,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -1494,7 +1437,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -1511,7 +1453,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -1528,7 +1469,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -1545,7 +1485,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -1562,7 +1501,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -1579,7 +1517,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -1596,7 +1533,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -1613,7 +1549,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -1630,7 +1565,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -1647,7 +1581,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -1664,7 +1597,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -1681,7 +1613,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -1698,7 +1629,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1810,7 +1740,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1827,7 +1756,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1844,7 +1772,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1861,7 +1788,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1878,7 +1804,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2140,7 +2065,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2157,7 +2081,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2174,7 +2097,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2191,7 +2113,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2208,7 +2129,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2225,7 +2145,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -2242,7 +2161,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -2259,7 +2177,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -2276,7 +2193,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -2293,7 +2209,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -2310,7 +2225,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -2327,7 +2241,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -2344,7 +2257,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -2361,7 +2273,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -2378,7 +2289,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -2395,7 +2305,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -2412,7 +2321,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -2429,7 +2337,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -2446,7 +2353,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -2463,7 +2369,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -2585,7 +2490,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2602,7 +2506,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2619,7 +2522,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2636,7 +2538,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2653,7 +2554,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2926,7 +2826,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2945,7 +2844,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2964,7 +2862,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2983,7 +2880,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3002,7 +2898,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3021,7 +2916,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3040,7 +2934,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3059,7 +2952,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3078,7 +2970,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3097,7 +2988,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -3116,7 +3006,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -3135,7 +3024,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -3154,7 +3042,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -3173,7 +3060,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -3192,7 +3078,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -3211,7 +3096,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -3230,7 +3114,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -3249,7 +3132,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -3268,7 +3150,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -3287,7 +3168,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -3431,7 +3311,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3450,7 +3329,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3469,7 +3347,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3488,7 +3365,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3507,7 +3383,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3830,7 +3705,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3847,7 +3721,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3864,7 +3737,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3881,7 +3753,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3898,7 +3769,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3915,7 +3785,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_monotonic(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3932,7 +3801,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3949,7 +3817,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3966,7 +3833,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3983,7 +3849,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4000,7 +3865,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_monotonic(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4017,7 +3881,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4034,7 +3897,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4051,7 +3913,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4068,7 +3929,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4085,7 +3945,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_monotonic(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4102,7 +3961,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4119,7 +3977,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4136,7 +3993,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4153,7 +4009,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -4275,7 +4130,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4292,7 +4146,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4309,7 +4162,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4326,7 +4178,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4343,7 +4194,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4615,7 +4465,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4632,7 +4481,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4649,7 +4497,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4666,7 +4513,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4683,7 +4529,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4700,7 +4545,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -4717,7 +4561,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -4734,7 +4577,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -4751,7 +4593,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -4768,7 +4609,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4785,7 +4625,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4802,7 +4641,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4819,7 +4657,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4836,7 +4673,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4853,7 +4689,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4870,7 +4705,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4887,7 +4721,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4904,7 +4737,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4921,7 +4753,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4938,7 +4769,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5060,7 +4890,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -5077,7 +4906,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -5094,7 +4922,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -5111,7 +4938,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -5128,7 +4954,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -5402,7 +5227,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5423,7 +5247,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5444,7 +5267,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5465,7 +5287,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -5486,7 +5307,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -5507,7 +5327,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -5528,7 +5347,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -5549,7 +5367,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -5570,7 +5387,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -5591,7 +5407,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -5611,7 +5426,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -5630,7 +5444,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -5649,7 +5462,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -5668,7 +5480,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -5687,7 +5498,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -5706,7 +5516,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -5725,7 +5534,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -5744,7 +5552,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -5763,7 +5570,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -5782,7 +5588,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5917,7 +5722,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5938,7 +5742,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5959,7 +5762,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5980,7 +5782,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6001,7 +5802,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6327,7 +6127,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6348,7 +6147,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6369,7 +6167,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6390,7 +6187,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6411,7 +6207,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6432,7 +6227,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -6453,7 +6247,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -6474,7 +6267,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -6495,7 +6287,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -6516,7 +6307,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -6536,7 +6326,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -6555,7 +6344,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -6574,7 +6362,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -6593,7 +6380,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -6612,7 +6398,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -6631,7 +6416,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -6650,7 +6434,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -6669,7 +6452,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -6688,7 +6470,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -6707,7 +6488,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -6842,7 +6622,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6863,7 +6642,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6884,7 +6662,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6905,7 +6682,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6926,7 +6702,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -7252,7 +7027,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7273,7 +7047,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7294,7 +7067,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7315,7 +7087,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7336,7 +7107,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -7357,7 +7127,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -7378,7 +7147,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -7399,7 +7167,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -7420,7 +7187,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -7441,7 +7207,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -7461,7 +7226,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -7480,7 +7244,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -7499,7 +7262,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -7518,7 +7280,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -7537,7 +7298,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -7556,7 +7316,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -7575,7 +7334,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -7594,7 +7352,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -7613,7 +7370,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -7632,7 +7388,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -7767,7 +7522,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7788,7 +7542,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7809,7 +7562,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7830,7 +7582,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7851,7 +7602,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8177,7 +7927,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8198,7 +7947,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8219,7 +7967,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8240,7 +7987,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8261,7 +8007,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8282,7 +8027,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -8303,7 +8047,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -8324,7 +8067,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -8345,7 +8087,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -8366,7 +8107,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -8386,7 +8126,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -8405,7 +8144,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -8424,7 +8162,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -8443,7 +8180,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -8462,7 +8198,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -8481,7 +8216,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -8500,7 +8234,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -8519,7 +8252,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -8538,7 +8270,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -8557,7 +8288,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -8692,7 +8422,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8713,7 +8442,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8734,7 +8462,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8755,7 +8482,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8776,7 +8502,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2_lse128.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2_lse128.ll
index 83e383f33563..29663f263385 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2_lse128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lse2_lse128.ll
@@ -2009,7 +2009,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2026,7 +2025,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casab w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2043,7 +2041,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    caslb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_release:
 ; -O1:    and w10, w8, w1
@@ -2060,7 +2057,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2077,7 +2073,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst:
 ; -O1:    and w10, w8, w1
@@ -2094,7 +2089,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    mvn w10, w8
 ; -O0:    cash w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2111,7 +2105,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casah w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2128,7 +2121,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    caslh w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_release:
 ; -O1:    and w10, w8, w1
@@ -2145,7 +2137,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalh w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2162,7 +2153,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalh w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst:
 ; -O1:    and w10, w8, w1
@@ -2179,7 +2169,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    mvn w10, w8
 ; -O0:    cas w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2196,7 +2185,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casa w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2213,7 +2201,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casl w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_release:
 ; -O1:    and w10, w8, w1
@@ -2230,7 +2217,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casal w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2247,7 +2233,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casal w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst:
 ; -O1:    and w10, w8, w1
@@ -2264,7 +2249,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    mvn x10, x8
 ; -O0:    cas x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic:
 ; -O1:    and x10, x8, x1
@@ -2281,7 +2265,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casa x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire:
 ; -O1:    and x10, x8, x1
@@ -2298,7 +2281,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casl x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_release:
 ; -O1:    and x10, x8, x1
@@ -2315,7 +2297,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casal x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel:
 ; -O1:    and x10, x8, x1
@@ -2332,7 +2313,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casal x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst:
 ; -O1:    and x10, x8, x1
@@ -2464,7 +2444,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    mvn w10, w8
 ; -O0:    casb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2481,7 +2460,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casab w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2498,7 +2476,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    caslb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release:
 ; -O1:    and w10, w8, w1
@@ -2515,7 +2492,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2532,7 +2508,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst:
 ; -O1:    and w10, w8, w1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
index 4d1f3e9117a4..9f0c6b07d7b7 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -379,11 +379,10 @@ define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -394,11 +393,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -409,11 +407,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -424,11 +421,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -439,11 +435,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -456,11 +451,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -475,11 +469,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -494,11 +487,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -513,11 +505,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -532,11 +523,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -549,11 +539,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 
 define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -564,11 +553,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -579,11 +567,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -594,11 +581,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -609,11 +595,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -624,11 +609,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %va
 
 define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
 ; -O1:    ldxr x8, [x0]
@@ -639,11 +623,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
 ; -O1:    ldaxr x8, [x0]
@@ -654,11 +637,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double
 
 define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
 ; -O1:    ldxr x8, [x0]
@@ -669,11 +651,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double
 
 define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
 ; -O1:    ldaxr x8, [x0]
@@ -684,11 +665,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double
 
 define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
 ; -O1:    ldaxr x8, [x0]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll
index f9c1a2216dc2..45737a1ae6be 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-outline_atomics.ll
@@ -2119,7 +2119,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2136,7 +2135,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2153,7 +2151,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2170,7 +2167,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2187,7 +2183,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2204,7 +2199,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas2_relax
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -2221,7 +2215,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas2_acq
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -2238,7 +2231,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas2_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -2255,7 +2247,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -2272,7 +2263,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -2289,7 +2279,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas4_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -2306,7 +2295,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas4_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -2323,7 +2311,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas4_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -2340,7 +2327,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -2357,7 +2343,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -2374,7 +2359,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    mvn x1, x8
 ; -O0:    bl __aarch64_cas8_relax
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -2391,7 +2375,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    mvn x1, x8
 ; -O0:    bl __aarch64_cas8_acq
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -2408,7 +2391,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    mvn x1, x8
 ; -O0:    bl __aarch64_cas8_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -2425,7 +2407,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    mvn x1, x8
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -2442,7 +2423,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    mvn x1, x8
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -2564,7 +2544,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2581,7 +2560,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2598,7 +2576,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2615,7 +2592,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2632,7 +2608,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    mvn w1, w8
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3985,7 +3960,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -4004,7 +3978,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -4023,7 +3996,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -4042,7 +4014,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -4061,7 +4032,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -4080,7 +4050,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas2_relax
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -4099,7 +4068,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas2_acq
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -4118,7 +4086,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas2_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -4137,7 +4104,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -4156,7 +4122,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -4174,7 +4139,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas4_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4191,7 +4155,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas4_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4208,7 +4171,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas4_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4225,7 +4187,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4242,7 +4203,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4259,7 +4219,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    csel x1, x0, x8, gt
 ; -O0:    bl __aarch64_cas8_relax
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4276,7 +4235,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, gt
 ; -O0:    bl __aarch64_cas8_acq
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4293,7 +4251,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, gt
 ; -O0:    bl __aarch64_cas8_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4310,7 +4267,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, gt
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4327,7 +4283,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, gt
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -4440,7 +4395,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -4459,7 +4413,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -4478,7 +4431,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -4497,7 +4449,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -4516,7 +4467,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, gt
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -4840,7 +4790,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -4859,7 +4808,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -4878,7 +4826,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -4897,7 +4844,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -4916,7 +4862,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -4935,7 +4880,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas2_relax
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -4954,7 +4898,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas2_acq
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -4973,7 +4916,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas2_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -4992,7 +4934,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -5011,7 +4952,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8, uxth
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -5029,7 +4969,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas4_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -5046,7 +4985,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas4_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -5063,7 +5001,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas4_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -5080,7 +5017,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -5097,7 +5033,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -5114,7 +5049,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    csel x1, x0, x8, le
 ; -O0:    bl __aarch64_cas8_relax
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -5131,7 +5065,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, le
 ; -O0:    bl __aarch64_cas8_acq
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -5148,7 +5081,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, le
 ; -O0:    bl __aarch64_cas8_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -5165,7 +5097,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, le
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -5182,7 +5113,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, le
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5295,7 +5225,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5314,7 +5243,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5333,7 +5261,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5352,7 +5279,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -5371,7 +5297,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, le
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8, uxtb
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -5695,7 +5620,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -5714,7 +5638,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -5733,7 +5656,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -5752,7 +5674,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -5771,7 +5692,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -5790,7 +5710,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas2_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -5809,7 +5728,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas2_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -5828,7 +5746,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas2_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -5847,7 +5764,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -5866,7 +5782,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -5884,7 +5799,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas4_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -5901,7 +5815,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas4_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -5918,7 +5831,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas4_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -5935,7 +5847,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -5952,7 +5863,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -5969,7 +5879,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    csel x1, x0, x8, hi
 ; -O0:    bl __aarch64_cas8_relax
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -5986,7 +5895,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, hi
 ; -O0:    bl __aarch64_cas8_acq
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -6003,7 +5911,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, hi
 ; -O0:    bl __aarch64_cas8_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -6020,7 +5927,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, hi
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -6037,7 +5943,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, hi
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -6150,7 +6055,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -6169,7 +6073,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -6188,7 +6091,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -6207,7 +6109,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -6226,7 +6127,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, hi
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -6550,7 +6450,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -6569,7 +6468,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -6588,7 +6486,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -6607,7 +6504,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -6626,7 +6522,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -6645,7 +6540,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas2_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -6664,7 +6558,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas2_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -6683,7 +6576,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas2_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -6702,7 +6594,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -6721,7 +6612,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas2_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -6739,7 +6629,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas4_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -6756,7 +6645,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas4_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -6773,7 +6661,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas4_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -6790,7 +6677,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -6807,7 +6693,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas4_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -6824,7 +6709,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    csel x1, x0, x8, ls
 ; -O0:    bl __aarch64_cas8_relax
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -6841,7 +6725,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, ls
 ; -O0:    bl __aarch64_cas8_acq
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -6858,7 +6741,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, ls
 ; -O0:    bl __aarch64_cas8_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -6875,7 +6757,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, ls
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -6892,7 +6773,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    csel x1, x0, x8, ls
 ; -O0:    bl __aarch64_cas8_acq_rel
 ; -O0:    subs x8, x0, x8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -7005,7 +6885,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_relax
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7024,7 +6903,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_acq
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7043,7 +6921,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7062,7 +6939,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7081,7 +6957,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    csel w1, w0, w8, ls
 ; -O0:    bl __aarch64_cas1_acq_rel
 ; -O0:    subs w8, w0, w8
-; -O0:    subs w8, w8, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll
index c09a2c593f0f..8200e2c2c2b3 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc.ll
@@ -9,7 +9,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -24,7 +23,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -39,7 +37,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -54,7 +51,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -69,7 +65,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -84,7 +79,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
 ; -O1:    ldxrh w0, [x8]
@@ -99,7 +93,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
 ; -O1:    ldaxrh w0, [x8]
@@ -114,7 +107,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
 ; -O1:    ldxrh w0, [x8]
@@ -129,7 +121,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w0, [x8]
@@ -144,7 +135,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w0, [x8]
@@ -159,7 +149,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_monotonic:
 ; -O1:    ldxr w0, [x8]
@@ -174,7 +163,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acquire:
 ; -O1:    ldaxr w0, [x8]
@@ -189,7 +177,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_release:
 ; -O1:    ldxr w0, [x8]
@@ -204,7 +191,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acq_rel:
 ; -O1:    ldaxr w0, [x8]
@@ -219,7 +205,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_seq_cst:
 ; -O1:    ldaxr w0, [x8]
@@ -234,7 +219,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -249,7 +233,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -264,7 +247,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -279,7 +261,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -294,7 +275,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -394,7 +374,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -409,7 +388,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -424,7 +402,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -439,7 +416,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -454,7 +430,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -610,7 +585,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -627,7 +601,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -644,7 +617,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -661,7 +633,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -678,7 +649,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -695,7 +665,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -712,7 +681,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -729,7 +697,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -746,7 +713,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -763,7 +729,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -780,7 +745,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -797,7 +761,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -814,7 +777,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -831,7 +793,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -848,7 +809,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -865,7 +825,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -882,7 +841,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -899,7 +857,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -916,7 +873,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -933,7 +889,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1045,7 +1000,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1062,7 +1016,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1079,7 +1032,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1096,7 +1048,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1113,7 +1064,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1375,7 +1325,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1392,7 +1341,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1409,7 +1357,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1426,7 +1373,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1443,7 +1389,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1460,7 +1405,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -1477,7 +1421,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -1494,7 +1437,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -1511,7 +1453,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -1528,7 +1469,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -1545,7 +1485,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -1562,7 +1501,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -1579,7 +1517,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -1596,7 +1533,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -1613,7 +1549,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -1630,7 +1565,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -1647,7 +1581,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -1664,7 +1597,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -1681,7 +1613,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -1698,7 +1629,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1810,7 +1740,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1827,7 +1756,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1844,7 +1772,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1861,7 +1788,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1878,7 +1804,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2140,7 +2065,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2157,7 +2081,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2174,7 +2097,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2191,7 +2113,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2208,7 +2129,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2225,7 +2145,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -2242,7 +2161,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -2259,7 +2177,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -2276,7 +2193,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -2293,7 +2209,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -2310,7 +2225,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -2327,7 +2241,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -2344,7 +2257,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -2361,7 +2273,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -2378,7 +2289,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -2395,7 +2305,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -2412,7 +2321,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -2429,7 +2337,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -2446,7 +2353,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -2463,7 +2369,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -2585,7 +2490,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2602,7 +2506,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2619,7 +2522,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2636,7 +2538,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2653,7 +2554,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2926,7 +2826,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2945,7 +2844,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2964,7 +2862,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2983,7 +2880,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3002,7 +2898,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3021,7 +2916,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3040,7 +2934,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3059,7 +2952,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3078,7 +2970,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3097,7 +2988,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -3116,7 +3006,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -3135,7 +3024,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -3154,7 +3042,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -3173,7 +3060,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -3192,7 +3078,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -3211,7 +3096,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -3230,7 +3114,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -3249,7 +3132,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -3268,7 +3150,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -3287,7 +3168,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -3431,7 +3311,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3450,7 +3329,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3469,7 +3347,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3488,7 +3365,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3507,7 +3383,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3830,7 +3705,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3847,7 +3721,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3864,7 +3737,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3881,7 +3753,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3898,7 +3769,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3915,7 +3785,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_monotonic(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3932,7 +3801,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3949,7 +3817,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3966,7 +3833,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3983,7 +3849,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4000,7 +3865,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_monotonic(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4017,7 +3881,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4034,7 +3897,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4051,7 +3913,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4068,7 +3929,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4085,7 +3945,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_monotonic(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4102,7 +3961,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4119,7 +3977,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4136,7 +3993,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4153,7 +4009,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -4275,7 +4130,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4292,7 +4146,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4309,7 +4162,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4326,7 +4178,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4343,7 +4194,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4615,7 +4465,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4632,7 +4481,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4649,7 +4497,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4666,7 +4513,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4683,7 +4529,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4700,7 +4545,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -4717,7 +4561,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -4734,7 +4577,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -4751,7 +4593,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -4768,7 +4609,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4785,7 +4625,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4802,7 +4641,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4819,7 +4657,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4836,7 +4673,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4853,7 +4689,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4870,7 +4705,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4887,7 +4721,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4904,7 +4737,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4921,7 +4753,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4938,7 +4769,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5060,7 +4890,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -5077,7 +4906,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -5094,7 +4922,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -5111,7 +4938,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -5128,7 +4954,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -5402,7 +5227,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5423,7 +5247,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5444,7 +5267,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5465,7 +5287,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -5486,7 +5307,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -5507,7 +5327,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -5528,7 +5347,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -5549,7 +5367,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -5570,7 +5387,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -5591,7 +5407,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -5611,7 +5426,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -5630,7 +5444,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -5649,7 +5462,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -5668,7 +5480,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -5687,7 +5498,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -5706,7 +5516,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -5725,7 +5534,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -5744,7 +5552,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -5763,7 +5570,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -5782,7 +5588,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5917,7 +5722,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5938,7 +5742,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5959,7 +5762,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5980,7 +5782,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6001,7 +5802,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6327,7 +6127,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6348,7 +6147,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6369,7 +6167,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6390,7 +6187,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6411,7 +6207,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6432,7 +6227,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -6453,7 +6247,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -6474,7 +6267,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -6495,7 +6287,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -6516,7 +6307,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -6536,7 +6326,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -6555,7 +6344,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -6574,7 +6362,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -6593,7 +6380,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -6612,7 +6398,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -6631,7 +6416,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -6650,7 +6434,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -6669,7 +6452,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -6688,7 +6470,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -6707,7 +6488,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -6842,7 +6622,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6863,7 +6642,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6884,7 +6662,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6905,7 +6682,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6926,7 +6702,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -7252,7 +7027,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7273,7 +7047,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7294,7 +7067,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7315,7 +7087,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7336,7 +7107,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -7357,7 +7127,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -7378,7 +7147,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -7399,7 +7167,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -7420,7 +7187,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -7441,7 +7207,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -7461,7 +7226,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -7480,7 +7244,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -7499,7 +7262,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -7518,7 +7280,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -7537,7 +7298,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -7556,7 +7316,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -7575,7 +7334,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -7594,7 +7352,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -7613,7 +7370,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -7632,7 +7388,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -7767,7 +7522,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7788,7 +7542,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7809,7 +7562,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7830,7 +7582,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7851,7 +7602,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8177,7 +7927,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8198,7 +7947,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8219,7 +7967,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8240,7 +7987,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8261,7 +8007,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8282,7 +8027,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -8303,7 +8047,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -8324,7 +8067,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -8345,7 +8087,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -8366,7 +8107,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -8386,7 +8126,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -8405,7 +8144,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -8424,7 +8162,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -8443,7 +8180,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -8462,7 +8198,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -8481,7 +8216,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -8500,7 +8234,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -8519,7 +8252,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -8538,7 +8270,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -8557,7 +8288,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -8692,7 +8422,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8713,7 +8442,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8734,7 +8462,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8755,7 +8482,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8776,7 +8502,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll
index 30166b4e7411..6df602200ebb 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-rcpc3.ll
@@ -9,7 +9,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -24,7 +23,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -39,7 +37,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -54,7 +51,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -69,7 +65,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -84,7 +79,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
 ; -O1:    ldxrh w0, [x8]
@@ -99,7 +93,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
 ; -O1:    ldaxrh w0, [x8]
@@ -114,7 +107,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
 ; -O1:    ldxrh w0, [x8]
@@ -129,7 +121,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w0, [x8]
@@ -144,7 +135,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w0, [x8]
@@ -159,7 +149,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_monotonic:
 ; -O1:    ldxr w0, [x8]
@@ -174,7 +163,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acquire:
 ; -O1:    ldaxr w0, [x8]
@@ -189,7 +177,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_release:
 ; -O1:    ldxr w0, [x8]
@@ -204,7 +191,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acq_rel:
 ; -O1:    ldaxr w0, [x8]
@@ -219,7 +205,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_seq_cst:
 ; -O1:    ldaxr w0, [x8]
@@ -234,7 +219,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -249,7 +233,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -264,7 +247,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -279,7 +261,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -294,7 +275,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -394,7 +374,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -409,7 +388,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -424,7 +402,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -439,7 +416,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -454,7 +430,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -610,7 +585,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -627,7 +601,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -644,7 +617,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -661,7 +633,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -678,7 +649,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -695,7 +665,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -712,7 +681,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -729,7 +697,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -746,7 +713,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -763,7 +729,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -780,7 +745,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -797,7 +761,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -814,7 +777,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -831,7 +793,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -848,7 +809,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -865,7 +825,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -882,7 +841,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -899,7 +857,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -916,7 +873,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -933,7 +889,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1045,7 +1000,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1062,7 +1016,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1079,7 +1032,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1096,7 +1048,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1113,7 +1064,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1375,7 +1325,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1392,7 +1341,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1409,7 +1357,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1426,7 +1373,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1443,7 +1389,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1460,7 +1405,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -1477,7 +1421,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -1494,7 +1437,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -1511,7 +1453,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -1528,7 +1469,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -1545,7 +1485,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -1562,7 +1501,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -1579,7 +1517,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -1596,7 +1533,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -1613,7 +1549,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -1630,7 +1565,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -1647,7 +1581,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -1664,7 +1597,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -1681,7 +1613,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -1698,7 +1629,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1810,7 +1740,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1827,7 +1756,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1844,7 +1772,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1861,7 +1788,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1878,7 +1804,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2140,7 +2065,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2157,7 +2081,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2174,7 +2097,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2191,7 +2113,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2208,7 +2129,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2225,7 +2145,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -2242,7 +2161,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -2259,7 +2177,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -2276,7 +2193,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -2293,7 +2209,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -2310,7 +2225,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -2327,7 +2241,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -2344,7 +2257,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -2361,7 +2273,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -2378,7 +2289,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -2395,7 +2305,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -2412,7 +2321,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -2429,7 +2337,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -2446,7 +2353,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -2463,7 +2369,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -2585,7 +2490,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2602,7 +2506,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2619,7 +2522,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2636,7 +2538,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2653,7 +2554,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2926,7 +2826,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2945,7 +2844,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2964,7 +2862,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2983,7 +2880,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3002,7 +2898,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3021,7 +2916,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3040,7 +2934,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3059,7 +2952,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3078,7 +2970,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3097,7 +2988,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -3116,7 +3006,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -3135,7 +3024,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -3154,7 +3042,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -3173,7 +3060,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -3192,7 +3078,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -3211,7 +3096,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -3230,7 +3114,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -3249,7 +3132,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -3268,7 +3150,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -3287,7 +3168,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -3431,7 +3311,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3450,7 +3329,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3469,7 +3347,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3488,7 +3365,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3507,7 +3383,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3830,7 +3705,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3847,7 +3721,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3864,7 +3737,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3881,7 +3753,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3898,7 +3769,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3915,7 +3785,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_monotonic(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3932,7 +3801,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3949,7 +3817,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3966,7 +3833,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3983,7 +3849,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4000,7 +3865,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_monotonic(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4017,7 +3881,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4034,7 +3897,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4051,7 +3913,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4068,7 +3929,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4085,7 +3945,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_monotonic(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4102,7 +3961,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4119,7 +3977,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4136,7 +3993,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4153,7 +4009,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -4275,7 +4130,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4292,7 +4146,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4309,7 +4162,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4326,7 +4178,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4343,7 +4194,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4615,7 +4465,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4632,7 +4481,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4649,7 +4497,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4666,7 +4513,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4683,7 +4529,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4700,7 +4545,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -4717,7 +4561,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -4734,7 +4577,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -4751,7 +4593,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -4768,7 +4609,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4785,7 +4625,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4802,7 +4641,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4819,7 +4657,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4836,7 +4673,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4853,7 +4689,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4870,7 +4705,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4887,7 +4721,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4904,7 +4737,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4921,7 +4753,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4938,7 +4769,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5060,7 +4890,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -5077,7 +4906,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -5094,7 +4922,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -5111,7 +4938,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -5128,7 +4954,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -5402,7 +5227,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5423,7 +5247,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5444,7 +5267,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5465,7 +5287,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -5486,7 +5307,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -5507,7 +5327,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -5528,7 +5347,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -5549,7 +5367,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -5570,7 +5387,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -5591,7 +5407,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -5611,7 +5426,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -5630,7 +5444,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -5649,7 +5462,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -5668,7 +5480,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -5687,7 +5498,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -5706,7 +5516,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -5725,7 +5534,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -5744,7 +5552,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -5763,7 +5570,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -5782,7 +5588,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5917,7 +5722,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5938,7 +5742,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5959,7 +5762,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5980,7 +5782,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6001,7 +5802,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6327,7 +6127,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6348,7 +6147,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6369,7 +6167,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6390,7 +6187,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6411,7 +6207,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6432,7 +6227,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -6453,7 +6247,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -6474,7 +6267,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -6495,7 +6287,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -6516,7 +6307,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -6536,7 +6326,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -6555,7 +6344,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -6574,7 +6362,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -6593,7 +6380,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -6612,7 +6398,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -6631,7 +6416,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -6650,7 +6434,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -6669,7 +6452,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -6688,7 +6470,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -6707,7 +6488,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -6842,7 +6622,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6863,7 +6642,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6884,7 +6662,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6905,7 +6682,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6926,7 +6702,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -7252,7 +7027,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7273,7 +7047,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7294,7 +7067,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7315,7 +7087,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7336,7 +7107,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -7357,7 +7127,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -7378,7 +7147,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -7399,7 +7167,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -7420,7 +7187,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -7441,7 +7207,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -7461,7 +7226,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -7480,7 +7244,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -7499,7 +7262,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -7518,7 +7280,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -7537,7 +7298,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -7556,7 +7316,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -7575,7 +7334,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -7594,7 +7352,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -7613,7 +7370,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -7632,7 +7388,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -7767,7 +7522,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7788,7 +7542,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7809,7 +7562,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7830,7 +7582,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7851,7 +7602,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8177,7 +7927,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8198,7 +7947,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8219,7 +7967,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8240,7 +7987,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8261,7 +8007,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8282,7 +8027,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -8303,7 +8047,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -8324,7 +8067,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -8345,7 +8087,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -8366,7 +8107,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -8386,7 +8126,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -8405,7 +8144,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -8424,7 +8162,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -8443,7 +8180,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -8462,7 +8198,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -8481,7 +8216,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -8500,7 +8234,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -8519,7 +8252,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -8538,7 +8270,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -8557,7 +8288,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -8692,7 +8422,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8713,7 +8442,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8734,7 +8462,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8755,7 +8482,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8776,7 +8502,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8_1a.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8_1a.ll
index 0c3ed9b0f1de..34cd68c94c6a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8_1a.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8_1a.ll
@@ -2059,7 +2059,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2076,7 +2075,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casab w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2093,7 +2091,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    caslb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_release:
 ; -O1:    and w10, w8, w1
@@ -2110,7 +2107,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2127,7 +2123,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst:
 ; -O1:    and w10, w8, w1
@@ -2144,7 +2139,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    mvn w10, w8
 ; -O0:    cash w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2161,7 +2155,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casah w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2178,7 +2171,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    caslh w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_release:
 ; -O1:    and w10, w8, w1
@@ -2195,7 +2187,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalh w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2212,7 +2203,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalh w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst:
 ; -O1:    and w10, w8, w1
@@ -2229,7 +2219,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    mvn w10, w8
 ; -O0:    cas w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2246,7 +2235,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casa w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2263,7 +2251,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casl w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_release:
 ; -O1:    and w10, w8, w1
@@ -2280,7 +2267,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casal w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2297,7 +2283,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casal w8, w10, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst:
 ; -O1:    and w10, w8, w1
@@ -2314,7 +2299,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    mvn x10, x8
 ; -O0:    cas x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic:
 ; -O1:    and x10, x8, x1
@@ -2331,7 +2315,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casa x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire:
 ; -O1:    and x10, x8, x1
@@ -2348,7 +2331,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casl x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_release:
 ; -O1:    and x10, x8, x1
@@ -2365,7 +2347,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casal x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel:
 ; -O1:    and x10, x8, x1
@@ -2382,7 +2363,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    mvn x10, x8
 ; -O0:    casal x8, x10, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst:
 ; -O1:    and x10, x8, x1
@@ -2514,7 +2494,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    mvn w10, w8
 ; -O0:    casb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic:
 ; -O1:    and w10, w8, w1
@@ -2531,7 +2510,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casab w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire:
 ; -O1:    and w10, w8, w1
@@ -2548,7 +2526,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    caslb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release:
 ; -O1:    and w10, w8, w1
@@ -2565,7 +2542,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel:
 ; -O1:    and w10, w8, w1
@@ -2582,7 +2558,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    mvn w10, w8
 ; -O0:    casalb w8, w10, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst:
 ; -O1:    and w10, w8, w1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll
index e9c1b33dc3d2..60fd7dab5822 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a.ll
@@ -9,7 +9,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -24,7 +23,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -39,7 +37,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -54,7 +51,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -69,7 +65,6 @@ define dso_local i8 @atomicrmw_xchg_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -84,7 +79,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_monotonic:
 ; -O1:    ldxrh w0, [x8]
@@ -99,7 +93,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acquire:
 ; -O1:    ldaxrh w0, [x8]
@@ -114,7 +107,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_release:
 ; -O1:    ldxrh w0, [x8]
@@ -129,7 +121,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w0, [x8]
@@ -144,7 +135,6 @@ define dso_local i16 @atomicrmw_xchg_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w0, [x8]
@@ -159,7 +149,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_monotonic:
 ; -O1:    ldxr w0, [x8]
@@ -174,7 +163,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acquire:
 ; -O1:    ldaxr w0, [x8]
@@ -189,7 +177,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_release:
 ; -O1:    ldxr w0, [x8]
@@ -204,7 +191,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_acq_rel:
 ; -O1:    ldaxr w0, [x8]
@@ -219,7 +205,6 @@ define dso_local i32 @atomicrmw_xchg_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i32_aligned_seq_cst:
 ; -O1:    ldaxr w0, [x8]
@@ -234,7 +219,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -249,7 +233,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -264,7 +247,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -279,7 +261,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -294,7 +275,6 @@ define dso_local i64 @atomicrmw_xchg_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -394,7 +374,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_monotonic:
 ; -O1:    ldxrb w0, [x8]
@@ -409,7 +388,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acquire:
 ; -O1:    ldaxrb w0, [x8]
@@ -424,7 +402,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_release:
 ; -O1:    ldxrb w0, [x8]
@@ -439,7 +416,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w0, [x8]
@@ -454,7 +430,6 @@ define dso_local i8 @atomicrmw_xchg_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xchg_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w0, [x8]
@@ -610,7 +585,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -627,7 +601,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -644,7 +617,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -661,7 +633,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -678,7 +649,6 @@ define dso_local i8 @atomicrmw_add_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -695,7 +665,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -712,7 +681,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -729,7 +697,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -746,7 +713,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -763,7 +729,6 @@ define dso_local i16 @atomicrmw_add_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -780,7 +745,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -797,7 +761,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -814,7 +777,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -831,7 +793,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -848,7 +809,6 @@ define dso_local i32 @atomicrmw_add_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -865,7 +825,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -882,7 +841,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -899,7 +857,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -916,7 +873,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -933,7 +889,6 @@ define dso_local i64 @atomicrmw_add_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1045,7 +1000,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1062,7 +1016,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1079,7 +1032,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1096,7 +1048,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1113,7 +1064,6 @@ define dso_local i8 @atomicrmw_add_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_add_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1375,7 +1325,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1392,7 +1341,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1409,7 +1357,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1426,7 +1373,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1443,7 +1389,6 @@ define dso_local i8 @atomicrmw_sub_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -1460,7 +1405,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -1477,7 +1421,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -1494,7 +1437,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -1511,7 +1453,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -1528,7 +1469,6 @@ define dso_local i16 @atomicrmw_sub_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -1545,7 +1485,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -1562,7 +1501,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -1579,7 +1517,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -1596,7 +1533,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -1613,7 +1549,6 @@ define dso_local i32 @atomicrmw_sub_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -1630,7 +1565,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -1647,7 +1581,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -1664,7 +1597,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -1681,7 +1613,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -1698,7 +1629,6 @@ define dso_local i64 @atomicrmw_sub_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -1810,7 +1740,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -1827,7 +1756,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -1844,7 +1772,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -1861,7 +1788,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -1878,7 +1804,6 @@ define dso_local i8 @atomicrmw_sub_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_sub_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2140,7 +2065,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2157,7 +2081,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2174,7 +2097,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2191,7 +2113,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2208,7 +2129,6 @@ define dso_local i8 @atomicrmw_and_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2225,7 +2145,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -2242,7 +2161,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -2259,7 +2177,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -2276,7 +2193,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -2293,7 +2209,6 @@ define dso_local i16 @atomicrmw_and_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -2310,7 +2225,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -2327,7 +2241,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -2344,7 +2257,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -2361,7 +2273,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -2378,7 +2289,6 @@ define dso_local i32 @atomicrmw_and_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -2395,7 +2305,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -2412,7 +2321,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -2429,7 +2337,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -2446,7 +2353,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -2463,7 +2369,6 @@ define dso_local i64 @atomicrmw_and_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -2585,7 +2490,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2602,7 +2506,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2619,7 +2522,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2636,7 +2538,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -2653,7 +2554,6 @@ define dso_local i8 @atomicrmw_and_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_and_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -2926,7 +2826,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -2945,7 +2844,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -2964,7 +2862,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -2983,7 +2880,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3002,7 +2898,6 @@ define dso_local i8 @atomicrmw_nand_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3021,7 +2916,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3040,7 +2934,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3059,7 +2952,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3078,7 +2970,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3097,7 +2988,6 @@ define dso_local i16 @atomicrmw_nand_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -3116,7 +3006,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -3135,7 +3024,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -3154,7 +3042,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -3173,7 +3060,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -3192,7 +3078,6 @@ define dso_local i32 @atomicrmw_nand_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -3211,7 +3096,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -3230,7 +3114,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -3249,7 +3132,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -3268,7 +3150,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -3287,7 +3168,6 @@ define dso_local i64 @atomicrmw_nand_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -3431,7 +3311,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3450,7 +3329,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3469,7 +3347,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3488,7 +3365,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3507,7 +3383,6 @@ define dso_local i8 @atomicrmw_nand_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_nand_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3830,7 +3705,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -3847,7 +3721,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -3864,7 +3737,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -3881,7 +3753,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -3898,7 +3769,6 @@ define dso_local i8 @atomicrmw_or_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -3915,7 +3785,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_monotonic(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -3932,7 +3801,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -3949,7 +3817,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -3966,7 +3833,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -3983,7 +3849,6 @@ define dso_local i16 @atomicrmw_or_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4000,7 +3865,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_monotonic(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4017,7 +3881,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4034,7 +3897,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4051,7 +3913,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4068,7 +3929,6 @@ define dso_local i32 @atomicrmw_or_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4085,7 +3945,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_monotonic(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4102,7 +3961,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4119,7 +3977,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4136,7 +3993,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4153,7 +4009,6 @@ define dso_local i64 @atomicrmw_or_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -4275,7 +4130,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4292,7 +4146,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4309,7 +4162,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4326,7 +4178,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4343,7 +4194,6 @@ define dso_local i8 @atomicrmw_or_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_or_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4615,7 +4465,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -4632,7 +4481,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -4649,7 +4497,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -4666,7 +4513,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -4683,7 +4529,6 @@ define dso_local i8 @atomicrmw_xor_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -4700,7 +4545,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -4717,7 +4561,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -4734,7 +4577,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -4751,7 +4593,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -4768,7 +4609,6 @@ define dso_local i16 @atomicrmw_xor_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -4785,7 +4625,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -4802,7 +4641,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -4819,7 +4657,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -4836,7 +4673,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -4853,7 +4689,6 @@ define dso_local i32 @atomicrmw_xor_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -4870,7 +4705,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -4887,7 +4721,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -4904,7 +4737,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -4921,7 +4753,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -4938,7 +4769,6 @@ define dso_local i64 @atomicrmw_xor_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5060,7 +4890,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_monotonic:
 ; -O1:    ldxrb w8, [x0]
@@ -5077,7 +4906,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acquire:
 ; -O1:    ldaxrb w8, [x0]
@@ -5094,7 +4922,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_release:
 ; -O1:    ldxrb w8, [x0]
@@ -5111,7 +4938,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w8, [x0]
@@ -5128,7 +4954,6 @@ define dso_local i8 @atomicrmw_xor_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_xor_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w8, [x0]
@@ -5402,7 +5227,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5423,7 +5247,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5444,7 +5267,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5465,7 +5287,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -5486,7 +5307,6 @@ define dso_local i8 @atomicrmw_max_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -5507,7 +5327,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -5528,7 +5347,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -5549,7 +5367,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -5570,7 +5387,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -5591,7 +5407,6 @@ define dso_local i16 @atomicrmw_max_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -5611,7 +5426,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -5630,7 +5444,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -5649,7 +5462,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -5668,7 +5480,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -5687,7 +5498,6 @@ define dso_local i32 @atomicrmw_max_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -5706,7 +5516,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -5725,7 +5534,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -5744,7 +5552,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -5763,7 +5570,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -5782,7 +5588,6 @@ define dso_local i64 @atomicrmw_max_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -5917,7 +5722,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -5938,7 +5742,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -5959,7 +5762,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -5980,7 +5782,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6001,7 +5802,6 @@ define dso_local i8 @atomicrmw_max_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_max_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6327,7 +6127,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6348,7 +6147,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6369,7 +6167,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6390,7 +6187,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6411,7 +6207,6 @@ define dso_local i8 @atomicrmw_min_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_aligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -6432,7 +6227,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -6453,7 +6247,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -6474,7 +6267,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -6495,7 +6287,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -6516,7 +6307,6 @@ define dso_local i16 @atomicrmw_min_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxth
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i16_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -6536,7 +6326,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -6555,7 +6344,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -6574,7 +6362,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -6593,7 +6380,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -6612,7 +6398,6 @@ define dso_local i32 @atomicrmw_min_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -6631,7 +6416,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -6650,7 +6434,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -6669,7 +6452,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -6688,7 +6470,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -6707,7 +6488,6 @@ define dso_local i64 @atomicrmw_min_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -6842,7 +6622,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_monotonic:
 ; -O1:    ldxrb w9, [x0]
@@ -6863,7 +6642,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acquire:
 ; -O1:    ldaxrb w9, [x0]
@@ -6884,7 +6662,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_release:
 ; -O1:    ldxrb w9, [x0]
@@ -6905,7 +6682,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_acq_rel:
 ; -O1:    ldaxrb w9, [x0]
@@ -6926,7 +6702,6 @@ define dso_local i8 @atomicrmw_min_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w9, uxtb
 ; -O0:    stlxrb w10, w12, [x11]
 ; -O0:    subs w9, w8, w9, uxtb
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_min_i8_unaligned_seq_cst:
 ; -O1:    ldaxrb w9, [x0]
@@ -7252,7 +7027,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7273,7 +7047,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7294,7 +7067,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7315,7 +7087,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7336,7 +7107,6 @@ define dso_local i8 @atomicrmw_umax_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -7357,7 +7127,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -7378,7 +7147,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -7399,7 +7167,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -7420,7 +7187,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -7441,7 +7207,6 @@ define dso_local i16 @atomicrmw_umax_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -7461,7 +7226,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -7480,7 +7244,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -7499,7 +7262,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -7518,7 +7280,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -7537,7 +7298,6 @@ define dso_local i32 @atomicrmw_umax_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -7556,7 +7316,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -7575,7 +7334,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -7594,7 +7352,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -7613,7 +7370,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -7632,7 +7388,6 @@ define dso_local i64 @atomicrmw_umax_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -7767,7 +7522,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -7788,7 +7542,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -7809,7 +7562,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -7830,7 +7582,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -7851,7 +7602,6 @@ define dso_local i8 @atomicrmw_umax_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umax_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8177,7 +7927,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_monotonic(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8198,7 +7947,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8219,7 +7967,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8240,7 +7987,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8261,7 +8007,6 @@ define dso_local i8 @atomicrmw_umin_i8_aligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
@@ -8282,7 +8027,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_monotonic(ptr %ptr, i16 %value)
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_monotonic:
 ; -O1:    and w9, w1, #0xffff
@@ -8303,7 +8047,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acquire(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acquire:
 ; -O1:    and w9, w1, #0xffff
@@ -8324,7 +8067,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_release(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_release:
 ; -O1:    and w9, w1, #0xffff
@@ -8345,7 +8087,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_acq_rel(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_acq_rel:
 ; -O1:    and w9, w1, #0xffff
@@ -8366,7 +8107,6 @@ define dso_local i16 @atomicrmw_umin_i16_aligned_seq_cst(ptr %ptr, i16 %value) {
 ; -O0:    cmp w8, w12, uxth
 ; -O0:    stlxrh w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i16_aligned_seq_cst:
 ; -O1:    and w9, w1, #0xffff
@@ -8386,7 +8126,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_monotonic(ptr %ptr, i32 %value)
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -8405,7 +8144,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acquire(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -8424,7 +8162,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_release(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -8443,7 +8180,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_acq_rel(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -8462,7 +8198,6 @@ define dso_local i32 @atomicrmw_umin_i32_aligned_seq_cst(ptr %ptr, i32 %value) {
 ; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i32_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -8481,7 +8216,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_monotonic(ptr %ptr, i64 %value)
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_monotonic:
 ; -O1:    ldxr x0, [x8]
@@ -8500,7 +8234,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acquire(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acquire:
 ; -O1:    ldaxr x0, [x8]
@@ -8519,7 +8252,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_release(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_release:
 ; -O1:    ldxr x0, [x8]
@@ -8538,7 +8270,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_acq_rel(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_acq_rel:
 ; -O1:    ldaxr x0, [x8]
@@ -8557,7 +8288,6 @@ define dso_local i64 @atomicrmw_umin_i64_aligned_seq_cst(ptr %ptr, i64 %value) {
 ; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
 ; -O0:    subs x9, x8, x9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i64_aligned_seq_cst:
 ; -O1:    ldaxr x0, [x8]
@@ -8692,7 +8422,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_monotonic(ptr %ptr, i8 %value)
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_monotonic:
 ; -O1:    and w9, w1, #0xff
@@ -8713,7 +8442,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acquire(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acquire:
 ; -O1:    and w9, w1, #0xff
@@ -8734,7 +8462,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_release(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_release:
 ; -O1:    and w9, w1, #0xff
@@ -8755,7 +8482,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_acq_rel(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_acq_rel:
 ; -O1:    and w9, w1, #0xff
@@ -8776,7 +8502,6 @@ define dso_local i8 @atomicrmw_umin_i8_unaligned_seq_cst(ptr %ptr, i8 %value) {
 ; -O0:    cmp w8, w12, uxtb
 ; -O0:    stlxrb w10, w13, [x11]
 ; -O0:    subs w9, w8, w9
-; -O0:    subs w9, w9, #1
 ;
 ; -O1-LABEL: atomicrmw_umin_i8_unaligned_seq_cst:
 ; -O1:    and w9, w1, #0xff
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
index 31074f619cad..fc3ed5680b23 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
@@ -5,11 +5,10 @@
 
 define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -20,11 +19,10 @@ define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -35,11 +33,10 @@ define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -50,11 +47,10 @@ define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -65,11 +61,10 @@ define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -82,11 +77,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -101,11 +95,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -120,11 +113,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -139,11 +131,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -158,11 +149,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -175,11 +165,10 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 
 define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -190,11 +179,10 @@ define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -205,11 +193,10 @@ define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -220,11 +207,10 @@ define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -235,11 +221,10 @@ define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -250,11 +235,10 @@ define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %va
 
 define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
 ; -O1:    ldxr x8, [x0]
@@ -265,11 +249,10 @@ define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
 ; -O1:    ldaxr x8, [x0]
@@ -280,11 +263,10 @@ define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double
 
 define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
 ; -O1:    ldxr x8, [x0]
@@ -295,11 +277,10 @@ define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double
 
 define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
 ; -O1:    ldaxr x8, [x0]
@@ -310,11 +291,10 @@ define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double
 
 define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
 ; -O1:    ldaxr x8, [x0]
@@ -500,11 +480,10 @@ define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -515,11 +494,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -530,11 +508,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -545,11 +522,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -560,11 +536,10 @@ define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -577,11 +552,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -596,11 +570,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -615,11 +588,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -634,11 +606,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -653,11 +624,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -670,11 +640,10 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 
 define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -685,11 +654,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -700,11 +668,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -715,11 +682,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -730,11 +696,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -745,11 +710,10 @@ define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %va
 
 define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
 ; -O1:    ldxr x8, [x0]
@@ -760,11 +724,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
 ; -O1:    ldaxr x8, [x0]
@@ -775,11 +738,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double
 
 define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
 ; -O1:    ldxr x8, [x0]
@@ -790,11 +752,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double
 
 define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
 ; -O1:    ldaxr x8, [x0]
@@ -805,11 +766,10 @@ define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double
 
 define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
 ; -O1:    ldaxr x8, [x0]
@@ -995,11 +955,10 @@ define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -1010,11 +969,10 @@ define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -1025,11 +983,10 @@ define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -1040,11 +997,10 @@ define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -1055,11 +1011,10 @@ define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -1072,11 +1027,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -1091,11 +1045,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -1110,11 +1063,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -1129,11 +1081,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -1148,11 +1099,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -1165,11 +1115,10 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 
 define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -1180,11 +1129,10 @@ define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -1195,11 +1143,10 @@ define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -1210,11 +1157,10 @@ define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -1225,11 +1171,10 @@ define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -1240,11 +1185,10 @@ define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %va
 
 define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
 ; -O1:    ldxr x8, [x0]
@@ -1255,11 +1199,10 @@ define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
 ; -O1:    ldaxr x8, [x0]
@@ -1270,11 +1213,10 @@ define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double
 
 define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
 ; -O1:    ldxr x8, [x0]
@@ -1285,11 +1227,10 @@ define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double
 
 define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
 ; -O1:    ldaxr x8, [x0]
@@ -1300,11 +1241,10 @@ define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double
 
 define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
 ; -O1:    ldaxr x8, [x0]
@@ -1490,11 +1430,10 @@ define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -1505,11 +1444,10 @@ define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -1520,11 +1458,10 @@ define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -1535,11 +1472,10 @@ define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -1550,11 +1486,10 @@ define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value
 
 define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -1567,11 +1502,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -1586,11 +1520,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -1605,11 +1538,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -1624,11 +1556,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -1643,11 +1574,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 ; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -1660,11 +1590,10 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat
 
 define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -1675,11 +1604,10 @@ define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -1690,11 +1618,10 @@ define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -1705,11 +1632,10 @@ define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -1720,11 +1646,10 @@ define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %va
 
 define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -1735,11 +1660,10 @@ define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %va
 
 define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
 ; -O1:    ldxr x8, [x0]
@@ -1750,11 +1674,10 @@ define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
 ; -O1:    ldaxr x8, [x0]
@@ -1765,11 +1688,10 @@ define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double
 
 define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
 ; -O1:    ldxr x8, [x0]
@@ -1780,11 +1702,10 @@ define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double
 
 define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
 ; -O1:    ldaxr x8, [x0]
@@ -1795,11 +1716,10 @@ define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double
 
 define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
 ; -O1:    ldaxr x8, [x0]
@@ -1985,11 +1905,10 @@ define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_monotonic(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -2000,11 +1919,10 @@ define dso_local half @atomicrmw_fmaximum_half_aligned_monotonic(ptr %ptr, half
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_acquire(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -2015,11 +1933,10 @@ define dso_local half @atomicrmw_fmaximum_half_aligned_acquire(ptr %ptr, half %v
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_release(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_half_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -2030,11 +1947,10 @@ define dso_local half @atomicrmw_fmaximum_half_aligned_release(ptr %ptr, half %v
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -2045,11 +1961,10 @@ define dso_local half @atomicrmw_fmaximum_half_aligned_acq_rel(ptr %ptr, half %v
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -2062,11 +1977,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, b
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -2081,11 +1995,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -2100,11 +2013,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -2119,11 +2031,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -2138,11 +2049,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_seq_cst(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -2155,11 +2065,10 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_seq_cst(ptr %ptr, bfl
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_monotonic(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -2170,11 +2079,10 @@ define dso_local float @atomicrmw_fmaximum_float_aligned_monotonic(ptr %ptr, flo
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_acquire(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -2185,11 +2093,10 @@ define dso_local float @atomicrmw_fmaximum_float_aligned_acquire(ptr %ptr, float
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_release(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_float_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -2200,11 +2107,10 @@ define dso_local float @atomicrmw_fmaximum_float_aligned_release(ptr %ptr, float
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -2215,11 +2121,10 @@ define dso_local float @atomicrmw_fmaximum_float_aligned_acq_rel(ptr %ptr, float
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -2230,11 +2135,10 @@ define dso_local float @atomicrmw_fmaximum_float_aligned_seq_cst(ptr %ptr, float
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
 ; -O1:    ldxr x8, [x0]
@@ -2245,11 +2149,10 @@ define dso_local double @atomicrmw_fmaximum_double_aligned_monotonic(ptr %ptr, d
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_acquire(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
 ; -O1:    ldaxr x8, [x0]
@@ -2260,11 +2163,10 @@ define dso_local double @atomicrmw_fmaximum_double_aligned_acquire(ptr %ptr, dou
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_release(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_double_aligned_release:
 ; -O1:    ldxr x8, [x0]
@@ -2275,11 +2177,10 @@ define dso_local double @atomicrmw_fmaximum_double_aligned_release(ptr %ptr, dou
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
 ; -O1:    ldaxr x8, [x0]
@@ -2290,11 +2191,10 @@ define dso_local double @atomicrmw_fmaximum_double_aligned_acq_rel(ptr %ptr, dou
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
 ; -O1:    ldaxr x8, [x0]
@@ -2480,11 +2380,10 @@ define dso_local double @atomicrmw_fmaximum_double_unaligned_seq_cst(ptr %ptr, d
 
 define dso_local half @atomicrmw_fminimum_half_aligned_monotonic(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
 ; -O1:    ldxrh w8, [x0]
@@ -2495,11 +2394,10 @@ define dso_local half @atomicrmw_fminimum_half_aligned_monotonic(ptr %ptr, half
 
 define dso_local half @atomicrmw_fminimum_half_aligned_acquire(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fminimum_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_half_aligned_acquire:
 ; -O1:    ldaxrh w8, [x0]
@@ -2510,11 +2408,10 @@ define dso_local half @atomicrmw_fminimum_half_aligned_acquire(ptr %ptr, half %v
 
 define dso_local half @atomicrmw_fminimum_half_aligned_release(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fminimum_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_half_aligned_release:
 ; -O1:    ldxrh w8, [x0]
@@ -2525,11 +2422,10 @@ define dso_local half @atomicrmw_fminimum_half_aligned_release(ptr %ptr, half %v
 
 define dso_local half @atomicrmw_fminimum_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
 ; -O1:    ldaxrh w8, [x0]
@@ -2540,11 +2436,10 @@ define dso_local half @atomicrmw_fminimum_half_aligned_acq_rel(ptr %ptr, half %v
 
 define dso_local half @atomicrmw_fminimum_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; -O0-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
 ; -O1:    ldaxrh w8, [x0]
@@ -2557,11 +2452,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, b
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
 ; -O1:    ldxrh w9, [x0]
@@ -2576,11 +2470,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
 ; -O1:    ldaxrh w9, [x0]
@@ -2595,11 +2488,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
 ; -O1:    ldxrh w9, [x0]
@@ -2614,11 +2506,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
 ; -O1:    ldaxrh w9, [x0]
@@ -2633,11 +2524,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_seq_cst(ptr %ptr, bfl
 ; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
 ; -O0:    add w8, w8, w9
 ; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
+; -O0:    ldaxrh w8, [x11]
+; -O0:    cmp w8, w9, uxth
 ; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9, uxth
 ;
 ; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
 ; -O1:    ldaxrh w9, [x0]
@@ -2650,11 +2540,10 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_seq_cst(ptr %ptr, bfl
 
 define dso_local float @atomicrmw_fminimum_float_aligned_monotonic(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
 ; -O1:    ldxr w8, [x0]
@@ -2665,11 +2554,10 @@ define dso_local float @atomicrmw_fminimum_float_aligned_monotonic(ptr %ptr, flo
 
 define dso_local float @atomicrmw_fminimum_float_aligned_acquire(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fminimum_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_float_aligned_acquire:
 ; -O1:    ldaxr w8, [x0]
@@ -2680,11 +2568,10 @@ define dso_local float @atomicrmw_fminimum_float_aligned_acquire(ptr %ptr, float
 
 define dso_local float @atomicrmw_fminimum_float_aligned_release(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fminimum_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_float_aligned_release:
 ; -O1:    ldxr w8, [x0]
@@ -2695,11 +2582,10 @@ define dso_local float @atomicrmw_fminimum_float_aligned_release(ptr %ptr, float
 
 define dso_local float @atomicrmw_fminimum_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
 ; -O1:    ldaxr w8, [x0]
@@ -2710,11 +2596,10 @@ define dso_local float @atomicrmw_fminimum_float_aligned_acq_rel(ptr %ptr, float
 
 define dso_local float @atomicrmw_fminimum_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; -O0-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
+; -O0:    ldaxr w8, [x11]
+; -O0:    cmp w8, w9
 ; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
+; -O0:    subs w9, w8, w9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
 ; -O1:    ldaxr w8, [x0]
@@ -2725,11 +2610,10 @@ define dso_local float @atomicrmw_fminimum_float_aligned_seq_cst(ptr %ptr, float
 
 define dso_local double @atomicrmw_fminimum_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
 ; -O1:    ldxr x8, [x0]
@@ -2740,11 +2624,10 @@ define dso_local double @atomicrmw_fminimum_double_aligned_monotonic(ptr %ptr, d
 
 define dso_local double @atomicrmw_fminimum_double_aligned_acquire(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fminimum_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_double_aligned_acquire:
 ; -O1:    ldaxr x8, [x0]
@@ -2755,11 +2638,10 @@ define dso_local double @atomicrmw_fminimum_double_aligned_acquire(ptr %ptr, dou
 
 define dso_local double @atomicrmw_fminimum_double_aligned_release(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fminimum_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_double_aligned_release:
 ; -O1:    ldxr x8, [x0]
@@ -2770,11 +2652,10 @@ define dso_local double @atomicrmw_fminimum_double_aligned_release(ptr %ptr, dou
 
 define dso_local double @atomicrmw_fminimum_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
 ; -O1:    ldaxr x8, [x0]
@@ -2785,11 +2666,10 @@ define dso_local double @atomicrmw_fminimum_double_aligned_acq_rel(ptr %ptr, dou
 
 define dso_local double @atomicrmw_fminimum_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
+; -O0:    ldaxr x8, [x11]
+; -O0:    cmp x8, x9
 ; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
+; -O0:    subs x9, x8, x9
 ;
 ; -O1-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
 ; -O1:    ldaxr x8, [x0]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir
index 93f6051c3bd3..5189582d0b6a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir
@@ -55,13 +55,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: %arg1:_(s64) = COPY $x0
     ; CHECK-NEXT: %arg2:_(s64) = COPY $x1
-    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: %one:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: %bv:_(<2 x s64>) = G_BUILD_VECTOR %arg1(s64), %arg2(s64)
-    ; CHECK-NEXT: %extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<2 x s64>), %zero(s64)
-    ; CHECK-NEXT: %extract2:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<2 x s64>), %one(s64)
-    ; CHECK-NEXT: $x0 = COPY %extract(s64)
-    ; CHECK-NEXT: $x1 = COPY %extract2(s64)
+    ; CHECK-NEXT: $x0 = COPY %arg1(s64)
+    ; CHECK-NEXT: $x1 = COPY %arg2(s64)
     ; CHECK-NEXT: $q0 = COPY %bv(<2 x s64>)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %arg1:_(s64) = COPY $x0
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir
index e2933690c7c5..e81447a1de4b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir
@@ -175,10 +175,8 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: %arg1:_(s64) = COPY $x0
     ; CHECK-NEXT: %arg2:_(s64) = COPY $x1
-    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: %bv:_(<2 x s64>) = G_BUILD_VECTOR %arg1(s64), %arg2(s64)
-    ; CHECK-NEXT: %extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<2 x s64>), %zero(s64)
-    ; CHECK-NEXT: $x0 = COPY %extract(s64)
+    ; CHECK-NEXT: $x0 = COPY %arg1(s64)
     ; CHECK-NEXT: $q0 = COPY %bv(<2 x s64>)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %arg1:_(s64) = COPY $x0
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir
index c000a8e635bc..86c0575961a1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir
@@ -278,8 +278,8 @@ body:             |
     ; CHECK: liveins: $x0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>))
     ; CHECK-NEXT: RET_ReallyLR
     %3:_(s8) = G_CONSTANT i8 127
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-stack-protector-windows.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-stack-protector-windows.ll
index 6aefc5341da0..e7f4785d01df 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-stack-protector-windows.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-stack-protector-windows.ll
@@ -17,8 +17,12 @@ define void @caller() sspreq {
 ; CHECK-NEXT:    ldr x8, [x8, :lo12:__security_cookie]
 ; CHECK-NEXT:    str x8, [sp, #8]
 ; CHECK-NEXT:    bl callee
-; CHECK-NEXT:    ldr x0, [sp, #8]
-; CHECK-NEXT:    bl __security_check_cookie
+; CHECK-NEXT:    adrp x8, __security_cookie
+; CHECK-NEXT:    ldr x9, [sp, #8]
+; CHECK-NEXT:    ldr x8, [x8, :lo12:__security_cookie]
+; CHECK-NEXT:    cmp x8, x9
+; CHECK-NEXT:    b.ne .LBB0_2
+; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    .seh_save_reg x30, 16
@@ -26,6 +30,10 @@ define void @caller() sspreq {
 ; CHECK-NEXT:    .seh_stackalloc 32
 ; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: // %entry
+; CHECK-NEXT:    ldr x0, [sp, #8]
+; CHECK-NEXT:    bl __security_check_cookie
+; CHECK-NEXT:    brk #0x1
 ; CHECK-NEXT:    .seh_endfunclet
 ; CHECK-NEXT:    .seh_endproc
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-concat.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-concat.mir
index 85129e9639a6..03f7533c7d60 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-concat.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-concat.mir
@@ -42,7 +42,7 @@ body: |
   ; CHECK-NEXT: %1:_ KnownBits:???????? SignBits:1
   ; CHECK-NEXT: %sext0:_ KnownBits:???????????????? SignBits:9
   ; CHECK-NEXT: %sext1:_ KnownBits:???????????????? SignBits:9
-  ; CHECK-NEXT: %res:_ KnownBits:???????????????? SignBits:1
+  ; CHECK-NEXT: %res:_ KnownBits:???????????????? SignBits:9
     %0:_(<2 x s8>) = COPY $h0
     %1:_(<2 x s8>) = COPY $h1
     %sext0:_(<2 x s16>) = G_SEXT %0
@@ -60,7 +60,7 @@ body: |
   ; CHECK-NEXT: %1:_ KnownBits:???????? SignBits:1
   ; CHECK-NEXT: %zext0:_ KnownBits:00000000???????? SignBits:8
   ; CHECK-NEXT: %sext1:_ KnownBits:???????????????? SignBits:9
-  ; CHECK-NEXT: %res:_ KnownBits:???????????????? SignBits:1
+  ; CHECK-NEXT: %res:_ KnownBits:???????????????? SignBits:8
     %0:_(<2 x s8>) = COPY $h0
     %1:_(<2 x s8>) = COPY $h1
     %zext0:_(<2 x s16>) = G_ZEXT %0
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-min-max.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-min-max.mir
index 4a3457a35256..fae979d38c6e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-min-max.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-min-max.mir
@@ -249,14 +249,12 @@ body: |
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(slt), [[DEF]](<2 x s64>), [[DEF]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(slt), [[DEF]](<2 x s64>), [[DEF]]
-    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP]], 1
-    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP1]], 1
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG1]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG]]
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP1]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP1]]
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND2]]
@@ -521,14 +519,12 @@ body: |
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ult), [[DEF]](<2 x s64>), [[DEF]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ult), [[DEF]](<2 x s64>), [[DEF]]
-    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP]], 1
-    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP1]], 1
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG1]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG]]
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP1]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP1]]
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND2]]
@@ -793,14 +789,12 @@ body: |
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(sgt), [[DEF]](<2 x s64>), [[DEF]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(sgt), [[DEF]](<2 x s64>), [[DEF]]
-    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP]], 1
-    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP1]], 1
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG1]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG]]
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP1]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP1]]
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND2]]
@@ -1065,14 +1059,12 @@ body: |
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ugt), [[DEF]](<2 x s64>), [[DEF]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(<2 x s64>) = G_ICMP intpred(ugt), [[DEF]](<2 x s64>), [[DEF]]
-    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP]], 1
-    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(<2 x s64>) = G_SEXT_INREG [[ICMP1]], 1
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[SEXT_INREG1]], [[BUILD_VECTOR]]
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG]]
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<2 x s64>) = G_XOR [[ICMP1]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[ICMP1]]
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR]]
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s64>) = G_AND [[DEF]], [[XOR1]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[AND]], [[AND2]]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
index be80886ed3ef..ba53cb57c2ef 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
@@ -11,7 +11,7 @@ define i32 @bar() {
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
 ; CHECK-NEXT:    umov.b w8, v0[0]
 ; CHECK-NEXT:    umov.b w9, v0[1]
-; CHECK-NEXT:    mov.s v1[0], w8
+; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    umov.b w8, v0[2]
 ; CHECK-NEXT:    mov.s v1[1], w9
 ; CHECK-NEXT:    umov.b w9, v0[3]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-unmergedup.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-unmergedup.mir
new file mode 100644
index 000000000000..acfbec0dd0ef
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-unmergedup.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            unmerge_dup8
+legalized:       true
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: unmerge_dup8
+    ; CHECK: [[COPY:%[0-9]+]]:_(<16 x s8>) = COPY $q0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[DUPLANE8_:%[0-9]+]]:_(<8 x s8>) = G_DUPLANE8 [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: $d0 = COPY [[DUPLANE8_]](<8 x s8>)
+    ; CHECK-NEXT: $d1 = COPY [[DUPLANE8_]](<8 x s8>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:_(<16 x s8>) = COPY $q0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<16 x s8>) = G_DUPLANE8 %0, %1
+    %3:_(<8 x s8>), %4:_(<8 x s8>) = G_UNMERGE_VALUES %2
+    $d0 = COPY %3
+    $d1 = COPY %4
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            unmerge_dup16
+legalized:       true
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: unmerge_dup16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[DUPLANE16_:%[0-9]+]]:_(<4 x s16>) = G_DUPLANE16 [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: $d0 = COPY [[DUPLANE16_]](<4 x s16>)
+    ; CHECK-NEXT: $d1 = COPY [[DUPLANE16_]](<4 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<8 x s16>) = G_DUPLANE16 %0, %1
+    %3:_(<4 x s16>), %4:_(<4 x s16>) = G_UNMERGE_VALUES %2
+    $d0 = COPY %3
+    $d1 = COPY %4
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            unmerge_dup32
+legalized:       true
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: unmerge_dup32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[DUPLANE32_:%[0-9]+]]:_(<2 x s32>) = G_DUPLANE32 [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: $d0 = COPY [[DUPLANE32_]](<2 x s32>)
+    ; CHECK-NEXT: $d1 = COPY [[DUPLANE32_]](<2 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<4 x s32>) = G_DUPLANE32 %0, %1
+    %3:_(<2 x s32>), %4:_(<2 x s32>) = G_UNMERGE_VALUES %2
+    $d0 = COPY %3
+    $d1 = COPY %4
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            unmerge_dup64
+legalized:       true
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: unmerge_dup64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[DUPLANE64_:%[0-9]+]]:_(<2 x s64>) = G_DUPLANE64 [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DUPLANE64_]](<2 x s64>), [[C1]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DUPLANE64_]](<2 x s64>), [[C2]](s64)
+    ; CHECK-NEXT: $d0 = COPY [[EVEC]](s64)
+    ; CHECK-NEXT: $d1 = COPY [[EVEC1]](s64)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<2 x s64>) = G_DUPLANE64 %0, %1
+    %3:_(s64), %4:_(s64) = G_UNMERGE_VALUES %2
+    $d0 = COPY %3
+    $d1 = COPY %4
+    RET_ReallyLR implicit $x0
+
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir
index 0115531dfb09..22d1ccc056eb 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir
@@ -57,15 +57,12 @@ body:             |
     ; SELECT-NEXT: %r:gpr32 = COPY $w0
     ; SELECT-NEXT: %q:gpr32 = COPY $w1
     ; SELECT-NEXT: [[DEF:%[0-9]+]]:fpr64 = IMPLICIT_DEF
+    ; SELECT-NEXT: [[INSERT_SUBREG:%[0-9]+]]:fpr64 = INSERT_SUBREG [[DEF]], %r, %subreg.ssub
     ; SELECT-NEXT: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; SELECT-NEXT: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[DEF]], %subreg.dsub
-    ; SELECT-NEXT: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 0, %r
+    ; SELECT-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[INSERT_SUBREG]], %subreg.dsub
+    ; SELECT-NEXT: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, %q
     ; SELECT-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
-    ; SELECT-NEXT: [[DEF2:%[0-9]+]]:fpr128 = IMPLICIT_DEF
-    ; SELECT-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF2]], [[COPY]], %subreg.dsub
-    ; SELECT-NEXT: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, %q
-    ; SELECT-NEXT: [[COPY1:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
-    ; SELECT-NEXT: $d0 = COPY [[COPY1]]
+    ; SELECT-NEXT: $d0 = COPY [[COPY]]
     ; SELECT-NEXT: RET_ReallyLR implicit $d0
     %r:_(s32) = COPY $w0
     %q:_(s32) = COPY $w1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-ext.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-ext.mir
index 14d44d85e06f..8dedb26dac2e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-ext.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-ext.mir
@@ -255,7 +255,8 @@ body:             |
     ; CHECK: liveins: $q0, $q1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: %v1:_(<8 x s16>) = COPY $q0
-    ; CHECK-NEXT: %shuf:_(<8 x s16>) = G_REV64 %v1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %shuf:_(<8 x s16>) = G_DUPLANE16 %v1, [[C]](s64)
     ; CHECK-NEXT: $q0 = COPY %shuf(<8 x s16>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %v1:_(<8 x s16>) = COPY $q0
@@ -298,8 +299,8 @@ body:             |
     ; CHECK: liveins: $q0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: %v1:_(<2 x s64>) = COPY $q0
-    ; CHECK-NEXT: %v2:_(<2 x s64>) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: %shuf:_(<2 x s64>) = G_TRN2 %v1, %v2
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %shuf:_(<2 x s64>) = G_DUPLANE64 %v1, [[C]](s64)
     ; CHECK-NEXT: $q0 = COPY %shuf(<2 x s64>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %v1:_(<2 x s64>) = COPY $q0
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-rev.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-rev.mir
index c5a603015549..1d24f8acfbc5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-rev.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-rev.mir
@@ -38,8 +38,11 @@ body:             |
     ; CHECK: liveins: $d0, $d1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
-    ; CHECK-NEXT: [[REV64_:%[0-9]+]]:_(<2 x s32>) = G_REV64 [[COPY]]
-    ; CHECK-NEXT: $d0 = COPY [[REV64_]](<2 x s32>)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[COPY]](<2 x s32>), [[DEF]](<2 x s32>)
+    ; CHECK-NEXT: [[DUPLANE32_:%[0-9]+]]:_(<2 x s32>) = G_DUPLANE32 [[CONCAT_VECTORS]], [[C]](s64)
+    ; CHECK-NEXT: $d0 = COPY [[DUPLANE32_]](<2 x s32>)
     ; CHECK-NEXT: RET_ReallyLR implicit $d0
     %0:_(<2 x s32>) = COPY $d0
     %1:_(<2 x s32>) = COPY $d1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll
index 9734ab35bd6b..7f922c004755 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll
@@ -76,8 +76,7 @@ define <1 x i32> @test_bitf_v1i32(<1 x i32> %A, <1 x i32> %B, <1 x i32> %C) {
 ; CHECK-GI-NEXT:    bic w9, w9, w8
 ; CHECK-GI-NEXT:    and w8, w8, w10
 ; CHECK-GI-NEXT:    orr w8, w9, w8
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
   %neg = xor <1 x i32> %C, <i32 -1>
   %and = and <1 x i32> %neg, %B
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
index 45ad4b07ff66..b8eb8269d605 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
@@ -76,8 +76,7 @@ define <1 x i32> @test_bit_v1i32(<1 x i32> %A, <1 x i32> %B, <1 x i32> %C) {
 ; CHECK-GI-NEXT:    and w9, w8, w9
 ; CHECK-GI-NEXT:    bic w8, w10, w8
 ; CHECK-GI-NEXT:    orr w8, w9, w8
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
   %and = and <1 x i32> %C, %B
   %neg = xor <1 x i32> %C, <i32 -1>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bitwisenot-fold.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bitwisenot-fold.ll
new file mode 100644
index 000000000000..5fbf38b2560d
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-bitwisenot-fold.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-linux | FileCheck %s
+
+define i8 @andnot_add_with_neg_i8(i8 %a0, i8 %a1) {
+; CHECK-LABEL: andnot_add_with_neg_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w0, w1
+; CHECK-NEXT:    bic w0, w0, w8
+; CHECK-NEXT:    ret
+  %not = xor i8 %a0, -1
+  %sum = add i8 %not, %a1
+  %and = and i8 %sum, %a0
+  ret i8 %and
+}
+
+define i8 @andnot_sub_with_neg_i8(i8 %a0, i8 %a1) {
+; CHECK-LABEL: andnot_sub_with_neg_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    bic w0, w0, w8
+; CHECK-NEXT:    ret
+  %not = xor i8 %a0, -1
+  %diff = sub i8 %not, %a1
+  %and = and i8 %diff, %a0
+  ret i8 %and
+}
+
+define i16 @andnot_add_with_neg_i16(i16 %a0, i16 %a1) {
+; CHECK-LABEL: andnot_add_with_neg_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w0, w1
+; CHECK-NEXT:    bic w0, w0, w8
+; CHECK-NEXT:    ret
+  %not = xor i16 %a0, -1
+  %sum = add i16 %not, %a1
+  %and = and i16 %sum, %a0
+  ret i16 %and
+}
+
+define i16 @andnot_sub_with_neg_i16(i16 %a0, i16 %a1) {
+; CHECK-LABEL: andnot_sub_with_neg_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    bic w0, w0, w8
+; CHECK-NEXT:    ret
+  %not = xor i16 %a0, -1
+  %diff = sub i16 %not, %a1
+  %and = and i16 %diff, %a0
+  ret i16 %and
+}
+
+define i32 @andnot_add_with_neg_i32(i32 %a0, i32 %a1) {
+; CHECK-LABEL: andnot_add_with_neg_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w8, w0, w1
+; CHECK-NEXT:    bic w0, w0, w8
+; CHECK-NEXT:    ret
+  %not = xor i32 %a0, -1
+  %sum = add i32 %not, %a1
+  %and = and i32 %sum, %a0
+  ret i32 %and
+}
+
+define i32 @andnot_sub_with_neg_i32(i32 %a0, i32 %a1) {
+; CHECK-LABEL: andnot_sub_with_neg_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    bic w0, w0, w8
+; CHECK-NEXT:    ret
+  %not = xor i32 %a0, -1
+  %diff = sub i32 %not, %a1
+  %and = and i32 %diff, %a0
+  ret i32 %and
+}
+
+define i64 @andnot_add_with_neg_i64(i64 %a0, i64 %a1) {
+; CHECK-LABEL: andnot_add_with_neg_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub x8, x0, x1
+; CHECK-NEXT:    bic x0, x0, x8
+; CHECK-NEXT:    ret
+  %not = xor i64 %a0, -1
+  %sum = add i64 %not, %a1
+  %and = and i64 %sum, %a0
+  ret i64 %and
+}
+
+define i64 @andnot_sub_with_neg_i64(i64 %a0, i64 %a1) {
+; CHECK-LABEL: andnot_sub_with_neg_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, x1
+; CHECK-NEXT:    bic x0, x0, x8
+; CHECK-NEXT:    ret
+  %not = xor i64 %a0, -1
+  %diff = sub i64 %not, %a1
+  %and = and i64 %diff, %a0
+  ret i64 %and
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
index 295863f18fd4..be79135c8b83 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
@@ -335,8 +335,6 @@ define void @typei1_orig(i64 %a, ptr %p, ptr %q) {
 ; CHECK-GI-NEXT:    cmtst v0.8h, v0.8h, v0.8h
 ; CHECK-GI-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-GI-NEXT:    sshr v0.16b, v0.16b, #7
 ; CHECK-GI-NEXT:    str q0, [x1]
 ; CHECK-GI-NEXT:    ret
   %tmp = xor <16 x i1> zeroinitializer, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
index 418113a4e4e0..8655bb1292ef 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
@@ -204,7 +204,7 @@ define void @matrix_mul_double_shuffle(i32 %N, ptr nocapture %C, ptr nocapture r
 ; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-GI-NEXT:    ldrh w9, [x2], #16
 ; CHECK-GI-NEXT:    subs x8, x8, #8
-; CHECK-GI-NEXT:    mov v2.s[0], w9
+; CHECK-GI-NEXT:    fmov s2, w9
 ; CHECK-GI-NEXT:    mov w9, w0
 ; CHECK-GI-NEXT:    add w0, w0, #8
 ; CHECK-GI-NEXT:    lsl x9, x9, #2
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
index 8de1fc5762c1..bf73aeb855c0 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
@@ -1291,21 +1291,13 @@ entry:
 }
 
 define i64 @umull_ldrb_h(ptr %x0, i16 %x1) {
-; CHECK-SD-LABEL: umull_ldrb_h:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umull x0, w8, w9
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldrb_h:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldrb_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umull x0, w8, w9
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1315,21 +1307,13 @@ entry:
 }
 
 define i64 @umull_ldrb_h_commuted(ptr %x0, i16 %x1) {
-; CHECK-SD-LABEL: umull_ldrb_h_commuted:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umull x0, w9, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldrb_h_commuted:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    mul x0, x9, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldrb_h_commuted:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umull x0, w9, w8
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1339,18 +1323,11 @@ entry:
 }
 
 define i64 @umull_ldrh_w(ptr %x0, i32 %x1) {
-; CHECK-SD-LABEL: umull_ldrh_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrh w8, [x0]
-; CHECK-SD-NEXT:    umull x0, w8, w1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldrh_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrh w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldrh_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    umull x0, w8, w1
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -1360,21 +1337,13 @@ entry:
 }
 
 define i64 @umull_ldr_b(ptr %x0, i8 %x1) {
-; CHECK-SD-LABEL: umull_ldr_b:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xff
-; CHECK-SD-NEXT:    umull x0, w8, w9
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldr_b:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xff
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldr_b:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xff
+; CHECK-NEXT:    umull x0, w8, w9
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -1384,18 +1353,11 @@ entry:
 }
 
 define i64 @umull_ldr2_w(ptr %x0, i32 %x1) {
-; CHECK-SD-LABEL: umull_ldr2_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umull x0, w8, w1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldr2_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldr2_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umull x0, w8, w1
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1405,19 +1367,12 @@ entry:
 }
 
 define i64 @umull_ldr2_ldr2(ptr %x0, ptr %x1) {
-; CHECK-SD-LABEL: umull_ldr2_ldr2:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    ldr w9, [x1]
-; CHECK-SD-NEXT:    umull x0, w8, w9
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldr2_ldr2:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    ldr w9, [x1]
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldr2_ldr2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    ldr w9, [x1]
+; CHECK-NEXT:    umull x0, w8, w9
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1428,18 +1383,11 @@ entry:
 }
 
 define i64 @umull_ldr2_d(ptr %x0, i64 %x1) {
-; CHECK-SD-LABEL: umull_ldr2_d:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umull x0, w8, w1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldr2_d:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldr2_d:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umull x0, w8, w1
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1449,21 +1397,13 @@ entry:
 }
 
 define i64 @umaddl_ldrb_h(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umaddl_ldrb_h:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umaddl x0, w8, w9, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_ldrb_h:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    madd x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_ldrb_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umaddl x0, w8, w9, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1474,21 +1414,13 @@ entry:
 }
 
 define i64 @umaddl_ldrb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umaddl_ldrb_h_commuted:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umaddl x0, w9, w8, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_ldrb_h_commuted:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    madd x0, x9, x8, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_ldrb_h_commuted:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umaddl x0, w9, w8, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1499,18 +1431,11 @@ entry:
 }
 
 define i64 @umaddl_ldrh_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umaddl_ldrh_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrh w8, [x0]
-; CHECK-SD-NEXT:    umaddl x0, w8, w1, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_ldrh_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrh w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    madd x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_ldrh_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    umaddl x0, w8, w1, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -1521,21 +1446,13 @@ entry:
 }
 
 define i64 @umaddl_ldr_b(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umaddl_ldr_b:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xff
-; CHECK-SD-NEXT:    umaddl x0, w8, w9, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_ldr_b:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xff
-; CHECK-GI-NEXT:    madd x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_ldr_b:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xff
+; CHECK-NEXT:    umaddl x0, w8, w9, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -1546,18 +1463,11 @@ entry:
 }
 
 define i64 @umaddl_ldr2_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umaddl_ldr2_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umaddl x0, w8, w1, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_ldr2_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    madd x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_ldr2_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umaddl x0, w8, w1, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1568,19 +1478,12 @@ entry:
 }
 
 define i64 @umaddl_ldr2_ldr2(ptr %x0, ptr %x1, i64 %x2) {
-; CHECK-SD-LABEL: umaddl_ldr2_ldr2:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    ldr w9, [x1]
-; CHECK-SD-NEXT:    umaddl x0, w8, w9, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_ldr2_ldr2:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    ldr w9, [x1]
-; CHECK-GI-NEXT:    madd x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_ldr2_ldr2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    ldr w9, [x1]
+; CHECK-NEXT:    umaddl x0, w8, w9, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1592,18 +1495,11 @@ entry:
 }
 
 define i64 @umaddl_ldr2_d(ptr %x0, i64 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umaddl_ldr2_d:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umaddl x0, w8, w1, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_ldr2_d:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    madd x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_ldr2_d:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umaddl x0, w8, w1, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1614,21 +1510,13 @@ entry:
 }
 
 define i64 @umnegl_ldrb_h(ptr %x0, i16 %x1) {
-; CHECK-SD-LABEL: umnegl_ldrb_h:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umnegl x0, w8, w9
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umnegl_ldrb_h:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    mneg x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umnegl_ldrb_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umnegl x0, w8, w9
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1639,21 +1527,13 @@ entry:
 }
 
 define i64 @umnegl_ldrb_h_commuted(ptr %x0, i16 %x1) {
-; CHECK-SD-LABEL: umnegl_ldrb_h_commuted:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umnegl x0, w9, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umnegl_ldrb_h_commuted:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    mneg x0, x9, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umnegl_ldrb_h_commuted:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umnegl x0, w9, w8
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1664,18 +1544,11 @@ entry:
 }
 
 define i64 @umnegl_ldrh_w(ptr %x0, i32 %x1) {
-; CHECK-SD-LABEL: umnegl_ldrh_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrh w8, [x0]
-; CHECK-SD-NEXT:    umnegl x0, w8, w1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umnegl_ldrh_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrh w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    mneg x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umnegl_ldrh_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    umnegl x0, w8, w1
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -1686,21 +1559,13 @@ entry:
 }
 
 define i64 @umnegl_ldr_b(ptr %x0, i8 %x1) {
-; CHECK-SD-LABEL: umnegl_ldr_b:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xff
-; CHECK-SD-NEXT:    umnegl x0, w8, w9
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umnegl_ldr_b:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xff
-; CHECK-GI-NEXT:    mneg x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umnegl_ldr_b:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xff
+; CHECK-NEXT:    umnegl x0, w8, w9
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -1711,18 +1576,11 @@ entry:
 }
 
 define i64 @umnegl_ldr2_w(ptr %x0, i32 %x1) {
-; CHECK-SD-LABEL: umnegl_ldr2_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umnegl x0, w8, w1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umnegl_ldr2_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    mneg x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umnegl_ldr2_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umnegl x0, w8, w1
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1733,19 +1591,12 @@ entry:
 }
 
 define i64 @umnegl_ldr2_ldr2(ptr %x0, ptr %x1) {
-; CHECK-SD-LABEL: umnegl_ldr2_ldr2:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    ldr w9, [x1]
-; CHECK-SD-NEXT:    umnegl x0, w8, w9
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umnegl_ldr2_ldr2:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    ldr w9, [x1]
-; CHECK-GI-NEXT:    mneg x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umnegl_ldr2_ldr2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    ldr w9, [x1]
+; CHECK-NEXT:    umnegl x0, w8, w9
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1757,18 +1608,11 @@ entry:
 }
 
 define i64 @umnegl_ldr2_d(ptr %x0, i64 %x1) {
-; CHECK-SD-LABEL: umnegl_ldr2_d:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umnegl x0, w8, w1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umnegl_ldr2_d:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    mneg x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umnegl_ldr2_d:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umnegl x0, w8, w1
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1779,21 +1623,13 @@ entry:
 }
 
 define i64 @umsubl_ldrb_h(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umsubl_ldrb_h:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umsubl x0, w8, w9, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umsubl_ldrb_h:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    msub x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umsubl_ldrb_h:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umsubl x0, w8, w9, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1804,21 +1640,13 @@ entry:
 }
 
 define i64 @umsubl_ldrb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umsubl_ldrb_h_commuted:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrb w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xffff
-; CHECK-SD-NEXT:    umsubl x0, w9, w8, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umsubl_ldrb_h_commuted:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrb w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xffff
-; CHECK-GI-NEXT:    msub x0, x9, x8, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umsubl_ldrb_h_commuted:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xffff
+; CHECK-NEXT:    umsubl x0, w9, w8, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1829,18 +1657,11 @@ entry:
 }
 
 define i64 @umsubl_ldrh_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umsubl_ldrh_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldrh w8, [x0]
-; CHECK-SD-NEXT:    umsubl x0, w8, w1, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umsubl_ldrh_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldrh w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    msub x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umsubl_ldrh_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    umsubl x0, w8, w1, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -1851,21 +1672,13 @@ entry:
 }
 
 define i64 @umsubl_ldr_b(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umsubl_ldr_b:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-SD-NEXT:    and x9, x1, #0xff
-; CHECK-SD-NEXT:    umsubl x0, w8, w9, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umsubl_ldr_b:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-GI-NEXT:    and x9, x1, #0xff
-; CHECK-GI-NEXT:    msub x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umsubl_ldr_b:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    and x9, x1, #0xff
+; CHECK-NEXT:    umsubl x0, w8, w9, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -1876,18 +1689,11 @@ entry:
 }
 
 define i64 @umsubl_ldr2_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umsubl_ldr2_w:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umsubl x0, w8, w1, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umsubl_ldr2_w:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    msub x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umsubl_ldr2_w:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umsubl x0, w8, w1, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1898,19 +1704,12 @@ entry:
 }
 
 define i64 @umsubl_ldr2_ldr2(ptr %x0, ptr %x1, i64 %x2) {
-; CHECK-SD-LABEL: umsubl_ldr2_ldr2:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    ldr w9, [x1]
-; CHECK-SD-NEXT:    umsubl x0, w8, w9, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umsubl_ldr2_ldr2:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    ldr w9, [x1]
-; CHECK-GI-NEXT:    msub x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umsubl_ldr2_ldr2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    ldr w9, [x1]
+; CHECK-NEXT:    umsubl x0, w8, w9, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1922,18 +1721,11 @@ entry:
 }
 
 define i64 @umsubl_ldr2_d(ptr %x0, i64 %x1, i64 %x2) {
-; CHECK-SD-LABEL: umsubl_ldr2_d:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr w8, [x0]
-; CHECK-SD-NEXT:    umsubl x0, w8, w1, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umsubl_ldr2_d:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr w8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    msub x0, x8, x9, x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umsubl_ldr2_d:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    umsubl x0, w8, w1, x2
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1944,20 +1736,12 @@ entry:
 }
 
 define i64 @umull_ldr2_w_cc1(ptr %x0, i32 %x1) {
-; CHECK-SD-LABEL: umull_ldr2_w_cc1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr x8, [x0]
-; CHECK-SD-NEXT:    and x8, x8, #0x7fffffff
-; CHECK-SD-NEXT:    umull x0, w8, w1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_ldr2_w_cc1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr x8, [x0]
-; CHECK-GI-NEXT:    mov w9, w1
-; CHECK-GI-NEXT:    and x8, x8, #0x7fffffff
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_ldr2_w_cc1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr x8, [x0]
+; CHECK-NEXT:    and x8, x8, #0x7fffffff
+; CHECK-NEXT:    umull x0, w8, w1
+; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 2147483647
@@ -1998,18 +1782,11 @@ entry:
 }
 
 define i64 @umull_and_lshr(i64 %x) {
-; CHECK-SD-LABEL: umull_and_lshr:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    lsr x8, x0, #32
-; CHECK-SD-NEXT:    umull x0, w0, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umull_and_lshr:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    lsr x8, x0, #32
-; CHECK-GI-NEXT:    mov w9, w0
-; CHECK-GI-NEXT:    mul x0, x9, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umull_and_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr x8, x0, #32
+; CHECK-NEXT:    umull x0, w0, w8
+; CHECK-NEXT:    ret
     %lo = and i64 %x, u0xffffffff
     %hi = lshr i64 %x, 32
     %mul = mul i64 %lo, %hi
@@ -2028,18 +1805,11 @@ define i64 @umull_and_and(i64 %x, i64 %y) {
 }
 
 define i64 @umaddl_and_lshr(i64 %x, i64 %a) {
-; CHECK-SD-LABEL: umaddl_and_lshr:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    lsr x8, x0, #32
-; CHECK-SD-NEXT:    umaddl x0, w0, w8, x1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_and_lshr:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    lsr x8, x0, #32
-; CHECK-GI-NEXT:    mov w9, w0
-; CHECK-GI-NEXT:    madd x0, x9, x8, x1
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_and_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr x8, x0, #32
+; CHECK-NEXT:    umaddl x0, w0, w8, x1
+; CHECK-NEXT:    ret
     %lo = and i64 %x, u0xffffffff
     %hi = lshr i64 %x, 32
     %mul = mul i64 %lo, %hi
@@ -2048,16 +1818,10 @@ define i64 @umaddl_and_lshr(i64 %x, i64 %a) {
 }
 
 define i64 @umaddl_and_and(i64 %x, i64 %y, i64 %a) {
-; CHECK-SD-LABEL: umaddl_and_and:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    umaddl x0, w0, w1, x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: umaddl_and_and:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    umull x8, w0, w1
-; CHECK-GI-NEXT:    add x0, x2, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: umaddl_and_and:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umaddl x0, w0, w1, x2
+; CHECK-NEXT:    ret
     %lo = and i64 %x, u0xffffffff
     %hi = and i64 %y, u0xffffffff
     %mul = mul i64 %lo, %hi
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators-sve.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators-sve.ll
index 0d4c05355101..554756709290 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators-sve.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators-sve.ll
@@ -1,29 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "aba" --filter "abd" --filter "add" --version 5
 ; RUN: opt -passes=loop-unroll %s -o - | llc -O3 - -mtriple=aarch64-unknown-unknown -mcpu=neoverse-v2 -o - | FileCheck %s
 
 define i64 @sabalb_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: sabalb_i32_to_i64_accumulation
+; CHECK-LABEL: sabalb_i32_to_i64_accumulation:
+; CHECK:    sabdlb z1.d, z1.s, z2.s
+; CHECK:    sabalb z0.d, z3.s, z4.s
+; CHECK:    sabalb z1.d, z4.s, z2.s
+; CHECK:    sabdlb z2.d, z3.s, z2.s
+; CHECK:    sabalb z2.d, z4.s, z3.s
+; CHECK:    sabalb z0.d, z3.s, z4.s
+; CHECK:    sabalb z1.d, z3.s, z4.s
+; CHECK:    sabalb z2.d, z3.s, z4.s
+; CHECK:    sabalb z0.d, z3.s, z4.s
+; CHECK:    sabalb z1.d, z3.s, z4.s
+; CHECK:    sabalb z2.d, z3.s, z4.s
+; CHECK:    sabalb z0.d, z3.s, z4.s
+; CHECK:    sabalb z1.d, z3.s, z4.s
+; CHECK:    sabalb z2.d, z3.s, z4.s
+; CHECK:    sabalb z0.d, z3.s, z4.s
+; CHECK:    add z0.d, z2.d, z0.d
+; CHECK:    sabalb z1.d, z3.s, z4.s
+; CHECK:    add z0.d, z0.d, z1.d
+; CHECK:    uaddv d0, p0, z0.d
 entry:
   br label %loop
 loop:
-; CHECK: sabdlb
-; CHECK: sabalb z0.d
-; CHECK: sabalb z1.d
-; CHECK: sabalb z2.d
-; CHECK: add	z0.d, z2.d, z0.d
-; CHECK: add	z0.d, z0.d, z1.d
-; CHECK: uaddv	d0, p0, z0.d
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i32, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i32, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
   %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
   %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.sabalb.nxv2i64(<vscale x 2 x i64> %acc_phi,
                                                                        <vscale x 4 x i32> %a,
                                                                        <vscale x 4 x i32> %b)
-  
-  %next_i = add i32 %i, 4
-  %cmp = icmp slt i32 %next_i, 64
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
@@ -34,29 +47,41 @@ declare <vscale x  2 x i64> @llvm.aarch64.sve.sabalb.nxv2i64(<vscale x 2 x i64>,
 declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)
 
 define i32 @sabalb_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: sabalb_i16_to_i32_accumulation
+; CHECK-LABEL: sabalb_i16_to_i32_accumulation:
+; CHECK:    sabdlb z1.s, z1.h, z2.h
+; CHECK:    sabalb z0.s, z3.h, z4.h
+; CHECK:    sabalb z1.s, z4.h, z2.h
+; CHECK:    sabdlb z2.s, z3.h, z2.h
+; CHECK:    sabalb z2.s, z4.h, z3.h
+; CHECK:    sabalb z0.s, z3.h, z4.h
+; CHECK:    sabalb z1.s, z3.h, z4.h
+; CHECK:    sabalb z2.s, z3.h, z4.h
+; CHECK:    sabalb z0.s, z3.h, z4.h
+; CHECK:    sabalb z1.s, z3.h, z4.h
+; CHECK:    sabalb z2.s, z3.h, z4.h
+; CHECK:    sabalb z0.s, z3.h, z4.h
+; CHECK:    sabalb z1.s, z3.h, z4.h
+; CHECK:    sabalb z2.s, z3.h, z4.h
+; CHECK:    sabalb z0.s, z3.h, z4.h
+; CHECK:    add z0.s, z2.s, z0.s
+; CHECK:    sabalb z1.s, z3.h, z4.h
+; CHECK:    add z0.s, z0.s, z1.s
+; CHECK:    uaddv d0, p0, z0.s
 entry:
   br label %loop
 loop:
-; CHECK: sabdlb
-; CHECK: sabalb z0.s
-; CHECK: sabalb z1.s
-; CHECK: sabalb z2.s
-; CHECK: add	z0.s, z2.s, z0.s
-; CHECK: add	z0.s, z0.s, z1.s
-; CHECK: uaddv	d0, p0, z0.s
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
   %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
   %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.sabalb.nxv4i32(<vscale x 4 x i32> %acc_phi,
                                                                        <vscale x 8 x i16> %a,
                                                                        <vscale x 8 x i16> %b)
-  
-  %next_i = add i32 %i, 8
-  %cmp = icmp slt i32 %next_i, 128
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
@@ -67,29 +92,41 @@ declare <vscale x 4 x i32> @llvm.aarch64.sve.sabalb.nxv4i32(<vscale x 4 x i32>,
 declare i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32>)
 
 define i16 @sabalb_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: sabalb_i8_to_i16_accumulation
+; CHECK-LABEL: sabalb_i8_to_i16_accumulation:
+; CHECK:    sabdlb z1.h, z1.b, z2.b
+; CHECK:    sabalb z0.h, z3.b, z4.b
+; CHECK:    sabalb z1.h, z4.b, z2.b
+; CHECK:    sabdlb z2.h, z3.b, z2.b
+; CHECK:    sabalb z2.h, z4.b, z3.b
+; CHECK:    sabalb z0.h, z3.b, z4.b
+; CHECK:    sabalb z1.h, z3.b, z4.b
+; CHECK:    sabalb z2.h, z3.b, z4.b
+; CHECK:    sabalb z0.h, z3.b, z4.b
+; CHECK:    sabalb z1.h, z3.b, z4.b
+; CHECK:    sabalb z2.h, z3.b, z4.b
+; CHECK:    sabalb z0.h, z3.b, z4.b
+; CHECK:    sabalb z1.h, z3.b, z4.b
+; CHECK:    sabalb z2.h, z3.b, z4.b
+; CHECK:    sabalb z0.h, z3.b, z4.b
+; CHECK:    add z0.h, z2.h, z0.h
+; CHECK:    sabalb z1.h, z3.b, z4.b
+; CHECK:    add z0.h, z0.h, z1.h
+; CHECK:    uaddv d0, p0, z0.h
 entry:
   br label %loop
 loop:
-; CHECK: sabdlb
-; CHECK: sabalb z0.h
-; CHECK: sabalb z1.h
-; CHECK: sabalb z2.h
-; CHECK: add	z0.h, z2.h, z0.h
-; CHECK: add	z0.h, z0.h, z1.h
-; CHECK: uaddv	d0, p0, z0.h
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
   %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
   %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.sabalb.nxv8i16(<vscale x 8 x i16> %acc_phi,
                                                                        <vscale x 16 x i8> %a,
                                                                        <vscale x 16 x i8> %b)
-  
-  %next_i = add i32 %i, 16
-  %cmp = icmp slt i32 %next_i, 256
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
@@ -100,29 +137,41 @@ declare <vscale x 8 x i16> @llvm.aarch64.sve.sabalb.nxv8i16(<vscale x 8 x i16>,
 declare i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16>)
 
 define i64 @sabalt_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: sabalt_i32_to_i64_accumulation
+; CHECK-LABEL: sabalt_i32_to_i64_accumulation:
+; CHECK:    sabdlt z1.d, z1.s, z2.s
+; CHECK:    sabalt z0.d, z3.s, z4.s
+; CHECK:    sabalt z1.d, z4.s, z2.s
+; CHECK:    sabdlt z2.d, z3.s, z2.s
+; CHECK:    sabalt z2.d, z4.s, z3.s
+; CHECK:    sabalt z0.d, z3.s, z4.s
+; CHECK:    sabalt z1.d, z3.s, z4.s
+; CHECK:    sabalt z2.d, z3.s, z4.s
+; CHECK:    sabalt z0.d, z3.s, z4.s
+; CHECK:    sabalt z1.d, z3.s, z4.s
+; CHECK:    sabalt z2.d, z3.s, z4.s
+; CHECK:    sabalt z0.d, z3.s, z4.s
+; CHECK:    sabalt z1.d, z3.s, z4.s
+; CHECK:    sabalt z2.d, z3.s, z4.s
+; CHECK:    sabalt z0.d, z3.s, z4.s
+; CHECK:    add z0.d, z2.d, z0.d
+; CHECK:    sabalt z1.d, z3.s, z4.s
+; CHECK:    add z0.d, z0.d, z1.d
+; CHECK:    uaddv d0, p0, z0.d
 entry:
   br label %loop
 loop:
-; CHECK: sabdlt
-; CHECK: sabalt z0.d
-; CHECK: sabalt z1.d
-; CHECK: sabalt z2.d
-; CHECK: add	z0.d, z2.d, z0.d
-; CHECK: add	z0.d, z0.d, z1.d
-; CHECK: uaddv	d0, p0, z0.d
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i32, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i32, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
   %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
   %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.sabalt.nxv2i64(<vscale x 2 x i64> %acc_phi,
                                                                        <vscale x 4 x i32> %a,
                                                                        <vscale x 4 x i32> %b)
-  
-  %next_i = add i32 %i, 4
-  %cmp = icmp slt i32 %next_i, 64
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
@@ -132,29 +181,41 @@ exit:
 declare <vscale x  2 x i64> @llvm.aarch64.sve.sabalt.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 
 define i32 @sabalt_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: sabalt_i16_to_i32_accumulation
+; CHECK-LABEL: sabalt_i16_to_i32_accumulation:
+; CHECK:    sabdlt z1.s, z1.h, z2.h
+; CHECK:    sabalt z0.s, z3.h, z4.h
+; CHECK:    sabalt z1.s, z4.h, z2.h
+; CHECK:    sabdlt z2.s, z3.h, z2.h
+; CHECK:    sabalt z2.s, z4.h, z3.h
+; CHECK:    sabalt z0.s, z3.h, z4.h
+; CHECK:    sabalt z1.s, z3.h, z4.h
+; CHECK:    sabalt z2.s, z3.h, z4.h
+; CHECK:    sabalt z0.s, z3.h, z4.h
+; CHECK:    sabalt z1.s, z3.h, z4.h
+; CHECK:    sabalt z2.s, z3.h, z4.h
+; CHECK:    sabalt z0.s, z3.h, z4.h
+; CHECK:    sabalt z1.s, z3.h, z4.h
+; CHECK:    sabalt z2.s, z3.h, z4.h
+; CHECK:    sabalt z0.s, z3.h, z4.h
+; CHECK:    add z0.s, z2.s, z0.s
+; CHECK:    sabalt z1.s, z3.h, z4.h
+; CHECK:    add z0.s, z0.s, z1.s
+; CHECK:    uaddv d0, p0, z0.s
 entry:
   br label %loop
 loop:
-; CHECK: sabdlt
-; CHECK: sabalt z0.s
-; CHECK: sabalt z1.s
-; CHECK: sabalt z2.s
-; CHECK: add	z0.s, z2.s, z0.s
-; CHECK: add	z0.s, z0.s, z1.s
-; CHECK: uaddv	d0, p0, z0.s
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
   %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
   %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.sabalt.nxv4i32(<vscale x 4 x i32> %acc_phi,
                                                                        <vscale x 8 x i16> %a,
                                                                        <vscale x 8 x i16> %b)
-  
-  %next_i = add i32 %i, 8
-  %cmp = icmp slt i32 %next_i, 128
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
@@ -164,29 +225,41 @@ exit:
 declare <vscale x 4 x i32> @llvm.aarch64.sve.sabalt.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 
 define i16 @sabalt_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: sabalt_i8_to_i16_accumulation
+; CHECK-LABEL: sabalt_i8_to_i16_accumulation:
+; CHECK:    sabdlt z1.h, z1.b, z2.b
+; CHECK:    sabalt z0.h, z3.b, z4.b
+; CHECK:    sabalt z1.h, z4.b, z2.b
+; CHECK:    sabdlt z2.h, z3.b, z2.b
+; CHECK:    sabalt z2.h, z4.b, z3.b
+; CHECK:    sabalt z0.h, z3.b, z4.b
+; CHECK:    sabalt z1.h, z3.b, z4.b
+; CHECK:    sabalt z2.h, z3.b, z4.b
+; CHECK:    sabalt z0.h, z3.b, z4.b
+; CHECK:    sabalt z1.h, z3.b, z4.b
+; CHECK:    sabalt z2.h, z3.b, z4.b
+; CHECK:    sabalt z0.h, z3.b, z4.b
+; CHECK:    sabalt z1.h, z3.b, z4.b
+; CHECK:    sabalt z2.h, z3.b, z4.b
+; CHECK:    sabalt z0.h, z3.b, z4.b
+; CHECK:    add z0.h, z2.h, z0.h
+; CHECK:    sabalt z1.h, z3.b, z4.b
+; CHECK:    add z0.h, z0.h, z1.h
+; CHECK:    uaddv d0, p0, z0.h
 entry:
   br label %loop
 loop:
-; CHECK: sabdlt
-; CHECK: sabalt z0.h
-; CHECK: sabalt z1.h
-; CHECK: sabalt z2.h
-; CHECK: add	z0.h, z2.h, z0.h
-; CHECK: add	z0.h, z0.h, z1.h
-; CHECK: uaddv	d0, p0, z0.h
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
   %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
   %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.sabalt.nxv8i16(<vscale x 8 x i16> %acc_phi,
                                                                        <vscale x 16 x i8> %a,
                                                                        <vscale x 16 x i8> %b)
-  
-  %next_i = add i32 %i, 16
-  %cmp = icmp slt i32 %next_i, 256
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
@@ -196,29 +269,41 @@ exit:
 declare <vscale x 8 x i16> @llvm.aarch64.sve.sabalt.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)
 
 define i64 @uabalb_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: uabalb_i32_to_i64_accumulation
+; CHECK-LABEL: uabalb_i32_to_i64_accumulation:
+; CHECK:    uabdlb z1.d, z1.s, z2.s
+; CHECK:    uabalb z0.d, z3.s, z4.s
+; CHECK:    uabalb z1.d, z4.s, z2.s
+; CHECK:    uabdlb z2.d, z3.s, z2.s
+; CHECK:    uabalb z2.d, z4.s, z3.s
+; CHECK:    uabalb z0.d, z3.s, z4.s
+; CHECK:    uabalb z1.d, z3.s, z4.s
+; CHECK:    uabalb z2.d, z3.s, z4.s
+; CHECK:    uabalb z0.d, z3.s, z4.s
+; CHECK:    uabalb z1.d, z3.s, z4.s
+; CHECK:    uabalb z2.d, z3.s, z4.s
+; CHECK:    uabalb z0.d, z3.s, z4.s
+; CHECK:    uabalb z1.d, z3.s, z4.s
+; CHECK:    uabalb z2.d, z3.s, z4.s
+; CHECK:    uabalb z0.d, z3.s, z4.s
+; CHECK:    add z0.d, z2.d, z0.d
+; CHECK:    uabalb z1.d, z3.s, z4.s
+; CHECK:    add z0.d, z0.d, z1.d
+; CHECK:    uaddv d0, p0, z0.d
 entry:
   br label %loop
 loop:
-; CHECK: uabdlb
-; CHECK: uabalb z0.d
-; CHECK: uabalb z1.d
-; CHECK: uabalb z2.d
-; CHECK: add	z0.d, z2.d, z0.d
-; CHECK: add	z0.d, z0.d, z1.d
-; CHECK: uaddv	d0, p0, z0.d
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i32, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i32, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
   %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
   %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.uabalb.nxv2i64(<vscale x 2 x i64> %acc_phi,
                                                                        <vscale x 4 x i32> %a,
                                                                        <vscale x 4 x i32> %b)
-  
-  %next_i = add i32 %i, 4
-  %cmp = icmp slt i32 %next_i, 64
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
@@ -228,29 +313,41 @@ exit:
 declare <vscale x  2 x i64> @llvm.aarch64.sve.uabalb.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 
 define i32 @uabalb_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: uabalb_i16_to_i32_accumulation
+; CHECK-LABEL: uabalb_i16_to_i32_accumulation:
+; CHECK:    uabdlb z1.s, z1.h, z2.h
+; CHECK:    uabalb z0.s, z3.h, z4.h
+; CHECK:    uabalb z1.s, z4.h, z2.h
+; CHECK:    uabdlb z2.s, z3.h, z2.h
+; CHECK:    uabalb z2.s, z4.h, z3.h
+; CHECK:    uabalb z0.s, z3.h, z4.h
+; CHECK:    uabalb z1.s, z3.h, z4.h
+; CHECK:    uabalb z2.s, z3.h, z4.h
+; CHECK:    uabalb z0.s, z3.h, z4.h
+; CHECK:    uabalb z1.s, z3.h, z4.h
+; CHECK:    uabalb z2.s, z3.h, z4.h
+; CHECK:    uabalb z0.s, z3.h, z4.h
+; CHECK:    uabalb z1.s, z3.h, z4.h
+; CHECK:    uabalb z2.s, z3.h, z4.h
+; CHECK:    uabalb z0.s, z3.h, z4.h
+; CHECK:    add z0.s, z2.s, z0.s
+; CHECK:    uabalb z1.s, z3.h, z4.h
+; CHECK:    add z0.s, z0.s, z1.s
+; CHECK:    uaddv d0, p0, z0.s
 entry:
   br label %loop
 loop:
-; CHECK: uabdlb
-; CHECK: uabalb z0.s
-; CHECK: uabalb z1.s
-; CHECK: uabalb z2.s
-; CHECK: add	z0.s, z2.s, z0.s
-; CHECK: add	z0.s, z0.s, z1.s
-; CHECK: uaddv	d0, p0, z0.s
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
   %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
   %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.uabalb.nxv4i32(<vscale x 4 x i32> %acc_phi,
                                                                        <vscale x 8 x i16> %a,
                                                                        <vscale x 8 x i16> %b)
-  
-  %next_i = add i32 %i, 8
-  %cmp = icmp slt i32 %next_i, 128
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
@@ -260,29 +357,41 @@ exit:
 declare <vscale x 4 x i32> @llvm.aarch64.sve.uabalb.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 
 define i16 @uabalb_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: uabalb_i8_to_i16_accumulation
+; CHECK-LABEL: uabalb_i8_to_i16_accumulation:
+; CHECK:    uabdlb z1.h, z1.b, z2.b
+; CHECK:    uabalb z0.h, z3.b, z4.b
+; CHECK:    uabalb z1.h, z4.b, z2.b
+; CHECK:    uabdlb z2.h, z3.b, z2.b
+; CHECK:    uabalb z2.h, z4.b, z3.b
+; CHECK:    uabalb z0.h, z3.b, z4.b
+; CHECK:    uabalb z1.h, z3.b, z4.b
+; CHECK:    uabalb z2.h, z3.b, z4.b
+; CHECK:    uabalb z0.h, z3.b, z4.b
+; CHECK:    uabalb z1.h, z3.b, z4.b
+; CHECK:    uabalb z2.h, z3.b, z4.b
+; CHECK:    uabalb z0.h, z3.b, z4.b
+; CHECK:    uabalb z1.h, z3.b, z4.b
+; CHECK:    uabalb z2.h, z3.b, z4.b
+; CHECK:    uabalb z0.h, z3.b, z4.b
+; CHECK:    add z0.h, z2.h, z0.h
+; CHECK:    uabalb z1.h, z3.b, z4.b
+; CHECK:    add z0.h, z0.h, z1.h
+; CHECK:    uaddv d0, p0, z0.h
 entry:
   br label %loop
 loop:
-; CHECK: uabdlb
-; CHECK: uabalb z0.h
-; CHECK: uabalb z1.h
-; CHECK: uabalb z2.h
-; CHECK: add	z0.h, z2.h, z0.h
-; CHECK: add	z0.h, z0.h, z1.h
-; CHECK: uaddv	d0, p0, z0.h
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
   %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
   %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16> %acc_phi,
                                                                        <vscale x 16 x i8> %a,
                                                                        <vscale x 16 x i8> %b)
-  
-  %next_i = add i32 %i, 16
-  %cmp = icmp slt i32 %next_i, 256
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
@@ -292,29 +401,41 @@ exit:
 declare <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)
 
 define i64 @uabalt_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: uabalt_i32_to_i64_accumulation
+; CHECK-LABEL: uabalt_i32_to_i64_accumulation:
+; CHECK:    uabdlt z1.d, z1.s, z2.s
+; CHECK:    uabalt z0.d, z3.s, z4.s
+; CHECK:    uabalt z1.d, z4.s, z2.s
+; CHECK:    uabdlt z2.d, z3.s, z2.s
+; CHECK:    uabalt z2.d, z4.s, z3.s
+; CHECK:    uabalt z0.d, z3.s, z4.s
+; CHECK:    uabalt z1.d, z3.s, z4.s
+; CHECK:    uabalt z2.d, z3.s, z4.s
+; CHECK:    uabalt z0.d, z3.s, z4.s
+; CHECK:    uabalt z1.d, z3.s, z4.s
+; CHECK:    uabalt z2.d, z3.s, z4.s
+; CHECK:    uabalt z0.d, z3.s, z4.s
+; CHECK:    uabalt z1.d, z3.s, z4.s
+; CHECK:    uabalt z2.d, z3.s, z4.s
+; CHECK:    uabalt z0.d, z3.s, z4.s
+; CHECK:    add z0.d, z2.d, z0.d
+; CHECK:    uabalt z1.d, z3.s, z4.s
+; CHECK:    add z0.d, z0.d, z1.d
+; CHECK:    uaddv d0, p0, z0.d
 entry:
   br label %loop
 loop:
-; CHECK: uabdlt
-; CHECK: uabalt z0.d
-; CHECK: uabalt z1.d
-; CHECK: uabalt z2.d
-; CHECK: add	z0.d, z2.d, z0.d
-; CHECK: add	z0.d, z0.d, z1.d
-; CHECK: uaddv	d0, p0, z0.d
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i32, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i32, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
   %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
   %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.uabalt.nxv2i64(<vscale x 2 x i64> %acc_phi,
                                                                        <vscale x 4 x i32> %a,
                                                                        <vscale x 4 x i32> %b)
-  
-  %next_i = add i32 %i, 4
-  %cmp = icmp slt i32 %next_i, 64
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
@@ -324,29 +445,41 @@ exit:
 declare <vscale x  2 x i64> @llvm.aarch64.sve.uabalt.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 
 define i32 @uabalt_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: uabalt_i16_to_i32_accumulation
+; CHECK-LABEL: uabalt_i16_to_i32_accumulation:
+; CHECK:    uabdlt z1.s, z1.h, z2.h
+; CHECK:    uabalt z0.s, z3.h, z4.h
+; CHECK:    uabalt z1.s, z4.h, z2.h
+; CHECK:    uabdlt z2.s, z3.h, z2.h
+; CHECK:    uabalt z2.s, z4.h, z3.h
+; CHECK:    uabalt z0.s, z3.h, z4.h
+; CHECK:    uabalt z1.s, z3.h, z4.h
+; CHECK:    uabalt z2.s, z3.h, z4.h
+; CHECK:    uabalt z0.s, z3.h, z4.h
+; CHECK:    uabalt z1.s, z3.h, z4.h
+; CHECK:    uabalt z2.s, z3.h, z4.h
+; CHECK:    uabalt z0.s, z3.h, z4.h
+; CHECK:    uabalt z1.s, z3.h, z4.h
+; CHECK:    uabalt z2.s, z3.h, z4.h
+; CHECK:    uabalt z0.s, z3.h, z4.h
+; CHECK:    add z0.s, z2.s, z0.s
+; CHECK:    uabalt z1.s, z3.h, z4.h
+; CHECK:    add z0.s, z0.s, z1.s
+; CHECK:    uaddv d0, p0, z0.s
 entry:
   br label %loop
 loop:
-; CHECK: uabdlt
-; CHECK: uabalt z0.s
-; CHECK: uabalt z1.s
-; CHECK: uabalt z2.s
-; CHECK: add	z0.s, z2.s, z0.s
-; CHECK: add	z0.s, z0.s, z1.s
-; CHECK: uaddv	d0, p0, z0.s
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
   %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
   %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.uabalt.nxv4i32(<vscale x 4 x i32> %acc_phi,
                                                                        <vscale x 8 x i16> %a,
                                                                        <vscale x 8 x i16> %b)
-  
-  %next_i = add i32 %i, 8
-  %cmp = icmp slt i32 %next_i, 128
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
@@ -356,29 +489,41 @@ exit:
 declare <vscale x 4 x i32> @llvm.aarch64.sve.uabalt.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 
 define i16 @uabalt_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: uabalt_i8_to_i16_accumulation
+; CHECK-LABEL: uabalt_i8_to_i16_accumulation:
+; CHECK:    uabdlt z1.h, z1.b, z2.b
+; CHECK:    uabalt z0.h, z3.b, z4.b
+; CHECK:    uabalt z1.h, z4.b, z2.b
+; CHECK:    uabdlt z2.h, z3.b, z2.b
+; CHECK:    uabalt z2.h, z4.b, z3.b
+; CHECK:    uabalt z0.h, z3.b, z4.b
+; CHECK:    uabalt z1.h, z3.b, z4.b
+; CHECK:    uabalt z2.h, z3.b, z4.b
+; CHECK:    uabalt z0.h, z3.b, z4.b
+; CHECK:    uabalt z1.h, z3.b, z4.b
+; CHECK:    uabalt z2.h, z3.b, z4.b
+; CHECK:    uabalt z0.h, z3.b, z4.b
+; CHECK:    uabalt z1.h, z3.b, z4.b
+; CHECK:    uabalt z2.h, z3.b, z4.b
+; CHECK:    uabalt z0.h, z3.b, z4.b
+; CHECK:    add z0.h, z2.h, z0.h
+; CHECK:    uabalt z1.h, z3.b, z4.b
+; CHECK:    add z0.h, z0.h, z1.h
+; CHECK:    uaddv d0, p0, z0.h
 entry:
   br label %loop
 loop:
-; CHECK: uabdlt
-; CHECK: uabalt z0.h
-; CHECK: uabalt z1.h
-; CHECK: uabalt z2.h
-; CHECK: add	z0.h, z2.h, z0.h
-; CHECK: add	z0.h, z0.h, z1.h
-; CHECK: uaddv	d0, p0, z0.h
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
-  %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
   %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
   %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
   %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16> %acc_phi,
                                                                        <vscale x 16 x i8> %a,
                                                                        <vscale x 16 x i8> %b)
-  
-  %next_i = add i32 %i, 16
-  %cmp = icmp slt i32 %next_i, 256
+
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
@@ -388,23 +533,53 @@ exit:
 declare <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)
 
 define i16 @uabalt_and_uabalb_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: uabalt_and_uabalb_accumulation
+; CHECK-LABEL: uabalt_and_uabalb_accumulation:
+; CHECK:    uabalb z1.h, z2.b, z3.b
+; CHECK:    uabalt z0.h, z2.b, z3.b
+; CHECK:    uabalb z1.h, z6.b, z7.b
+; CHECK:    uabalt z0.h, z6.b, z7.b
+; CHECK:    uabdlb z7.h, z4.b, z5.b
+; CHECK:    uabdlt z4.h, z4.b, z5.b
+; CHECK:    uabalb z7.h, z6.b, z5.b
+; CHECK:    uabalt z4.h, z6.b, z5.b
+; CHECK:    uabdlb z6.h, z2.b, z3.b
+; CHECK:    uabdlt z2.h, z2.b, z3.b
+; CHECK:    uabalb z6.h, z5.b, z3.b
+; CHECK:    uabalt z2.h, z5.b, z3.b
+; CHECK:    uabalb z1.h, z3.b, z5.b
+; CHECK:    uabalt z0.h, z3.b, z5.b
+; CHECK:    uabalb z7.h, z3.b, z5.b
+; CHECK:    uabalt z4.h, z3.b, z5.b
+; CHECK:    uabalb z6.h, z3.b, z5.b
+; CHECK:    uabalt z2.h, z3.b, z5.b
+; CHECK:    uabalb z1.h, z3.b, z5.b
+; CHECK:    uabalt z0.h, z3.b, z5.b
+; CHECK:    uabalb z7.h, z3.b, z5.b
+; CHECK:    uabalt z4.h, z3.b, z5.b
+; CHECK:    uabalb z6.h, z3.b, z5.b
+; CHECK:    uabalt z2.h, z3.b, z5.b
+; CHECK:    uabalb z1.h, z3.b, z5.b
+; CHECK:    uabalt z0.h, z3.b, z5.b
+; CHECK:    uabalb z7.h, z3.b, z5.b
+; CHECK:    uabalt z4.h, z3.b, z5.b
+; CHECK:    uabalb z6.h, z3.b, z5.b
+; CHECK:    uabalt z2.h, z3.b, z5.b
+; CHECK:    add z2.h, z4.h, z2.h
+; CHECK:    uabalb z1.h, z3.b, z5.b
+; CHECK:    uabalt z0.h, z3.b, z5.b
+; CHECK:    add z3.h, z7.h, z6.h
+; CHECK:    add z1.h, z3.h, z1.h
+; CHECK:    add z0.h, z2.h, z0.h
+; CHECK:    add z1.h, p0/m, z1.h, z0.h
+; CHECK:    uaddv d0, p0, z1.h
 entry:
   br label %loop
 loop:
-; CHECK: uabdlt
-; CHECK: uabdlb
-; CHECK: uabalt z0.h
-; CHECK: uabalt z2.h
-; CHECK: uabalt z4.h
-; CHECK: uabalb z1.h
-; CHECK: uabalb z6.h
-; CHECK: uabalb z5.h
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_hi_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next_hi, %loop ]
   %acc_lo_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next_lo, %loop ]
-  %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
-  %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
+  %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
+  %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
   %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
   %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
   %acc_next_lo = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16> %acc_lo_phi,
@@ -413,8 +588,8 @@ loop:
   %acc_next_hi = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16> %acc_hi_phi,
                                                                          <vscale x 16 x i8> %a,
                                                                          <vscale x 16 x i8> %b)
-  %next_i = add i32 %i, 16
-  %cmp = icmp slt i32 %next_i, 256
+  %next_i = add i32 %i, 1
+  %cmp = icmp slt i32 %next_i, 16
   br i1 %cmp, label %loop, label %exit
 exit:
   %mask = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
@@ -423,4 +598,4 @@ exit:
   ret i16 %reduce
 }
 
-declare <vscale x 8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
\ No newline at end of file
+declare <vscale x 8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators.ll
index 86150a8d3d3c..a84d666c1be6 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators.ll
@@ -1,23 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "aba" --filter "abd" --filter "add" --version 5
 ; RUN: opt -passes=loop-unroll %s -o - | llc -O3 - -mtriple=arm64e-apple-darwin -o - | FileCheck %s
 
 
 define i16 @sabal_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
-  ; CHECK-LABEL: sabal_i8_to_i16_accumulation
+; CHECK-LABEL: sabal_i8_to_i16_accumulation:
+; CHECK:    sabdl.8h v1, v1, v3
+; CHECK:    sabdl.8h v0, v0, v2
+; CHECK:    sabdl.8h v2, v3, v5
+; CHECK:    sabal.8h v1, v4, v6
+; CHECK:    sabal.8h v0, v7, v17
+; CHECK:    sabal.8h v2, v16, v18
+; CHECK:    sabal.8h v1, v19, v21
+; CHECK:    sabal.8h v0, v20, v22
+; CHECK:    add.8h v1, v2, v1
+; CHECK:    add.8h v0, v1, v0
+; CHECK:    addv.8h h0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: sabdl.8h v1
-; CHECK: sabdl.8h v0
-; CHECK: sabdl.8h v2
-; CHECK: sabal.8h v1
-; CHECK: sabal.8h v0
-; CHECK: sabal.8h v2
-; CHECK: sabal.8h v1
-; CHECK: sabal.8h v0
-; CHECK: add.8h v1, v2, v1
-; CHECK: add.8h v0, v1, v0
-; CHECK: addv.8h
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -43,22 +44,22 @@ declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
 
 
 define i32 @sabal_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: sabal_i16_to_i32_accumulation
+; CHECK-LABEL: sabal_i16_to_i32_accumulation:
+; CHECK:    sabdl.4s v1, v1, v3
+; CHECK:    sabdl.4s v0, v0, v2
+; CHECK:    sabdl.4s v2, v3, v5
+; CHECK:    sabal.4s v1, v4, v6
+; CHECK:    sabal.4s v0, v7, v17
+; CHECK:    sabal.4s v2, v16, v18
+; CHECK:    sabal.4s v1, v19, v21
+; CHECK:    sabal.4s v0, v20, v22
+; CHECK:    add.4s v1, v2, v1
+; CHECK:    add.4s v0, v1, v0
+; CHECK:    addv.4s s0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: sabdl.4s v1
-; CHECK: sabdl.4s v0
-; CHECK: sabdl.4s v2
-; CHECK: sabal.4s v1
-; CHECK: sabal.4s v0
-; CHECK: sabal.4s v2
-; CHECK: sabal.4s v1
-; CHECK: sabal.4s v0
-; CHECK: add.4s v1, v2, v1
-; CHECK: add.4s v0, v1, v0
-; CHECK: addv.4s
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -82,33 +83,33 @@ declare <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16>, <4 x i16>)
 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
 
 define i16 @uabal2_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uabal2_accumulation
+; CHECK-LABEL: uabal2_accumulation:
+; CHECK:    uabdl2.8h v4, v1, v3
+; CHECK:    uabdl.8h v1, v1, v3
+; CHECK:    uabdl2.8h v24, v0, v2
+; CHECK:    uabdl2.8h v25, v3, v6
+; CHECK:    uabal2.8h v4, v5, v7
+; CHECK:    uabal2.8h v24, v16, v18
+; CHECK:    uabal2.8h v25, v17, v19
+; CHECK:    uabal2.8h v4, v20, v22
+; CHECK:    uabal2.8h v24, v21, v23
+; CHECK:    add.8h v4, v25, v4
+; CHECK:    add.8h v4, v4, v24
+; CHECK:    uabdl.8h v0, v0, v2
+; CHECK:    uabdl.8h v2, v3, v6
+; CHECK:    uabal.8h v1, v5, v7
+; CHECK:    uabal.8h v0, v16, v18
+; CHECK:    uabal.8h v2, v17, v19
+; CHECK:    uabal.8h v1, v20, v22
+; CHECK:    uabal.8h v0, v21, v23
+; CHECK:    add.8h v1, v2, v1
+; CHECK:    add.8h v0, v1, v0
+; CHECK:    add.8h v0, v4, v0
+; CHECK:    addv.8h h0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK:  uabdl2.8h       v4
-; CHECK:  uabdl.8h        v1 
-; CHECK:  uabdl2.8h       v24
-; CHECK:  uabdl2.8h       v25
-; CHECK:  uabal2.8h       v4 
-; CHECK:  uabal2.8h       v24
-; CHECK:  uabal2.8h       v25
-; CHECK:  uabal2.8h       v4
-; CHECK:  uabal2.8h       v24
-; CHECK:  add.8h          v4, v25, v4
-; CHECK:  add.8h          v4, v4, v24
-; CHECK:  uabdl.8h        v0
-; CHECK:  uabdl.8h        v2
-; CHECK:  uabal.8h        v1
-; CHECK:  uabal.8h        v0
-; CHECK:  uabal.8h        v2
-; CHECK:  uabal.8h        v1
-; CHECK:  uabal.8h        v0
-; CHECK:  add.8h          v1, v2, v1
-; CHECK:  add.8h          v0, v1, v0
-; CHECK:  add.8h          v0, v4, v0
-; CHECK:  addv.8h         h0, v0
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi_hi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next_hi, %loop ]
@@ -138,22 +139,22 @@ exit:
 }
 
 define i32 @uaba_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uaba_accumulation
+; CHECK-LABEL: uaba_accumulation:
+; CHECK:    uabd.4s v0, v0, v2
+; CHECK:    uabd.4s v1, v1, v3
+; CHECK:    uabd.4s v2, v2, v5
+; CHECK:    uaba.4s v0, v4, v6
+; CHECK:    uaba.4s v1, v7, v17
+; CHECK:    uaba.4s v2, v16, v18
+; CHECK:    uaba.4s v0, v19, v21
+; CHECK:    uaba.4s v1, v20, v22
+; CHECK:    add.4s v0, v2, v0
+; CHECK:    add.4s v0, v0, v1
+; CHECK:    addv.4s s0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: uabd.4s v0
-; CHECK: uabd.4s v1
-; CHECK: uabd.4s v2
-; CHECK: uaba.4s v0
-; CHECK: uaba.4s v1
-; CHECK: uaba.4s v2
-; CHECK: uaba.4s v0
-; CHECK: uaba.4s v1
-; CHECK: add.4s v0, v2, v0
-; CHECK: add.4s v0, v0, v1
-; CHECK: addv.4s 
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -175,22 +176,22 @@ exit:
 declare <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 
 define i32 @saba_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: saba_accumulation
+; CHECK-LABEL: saba_accumulation:
+; CHECK:    sabd.4s v0, v0, v2
+; CHECK:    sabd.4s v1, v1, v3
+; CHECK:    sabd.4s v2, v2, v5
+; CHECK:    saba.4s v0, v4, v6
+; CHECK:    saba.4s v1, v7, v17
+; CHECK:    saba.4s v2, v16, v18
+; CHECK:    saba.4s v0, v19, v21
+; CHECK:    saba.4s v1, v20, v22
+; CHECK:    add.4s v0, v2, v0
+; CHECK:    add.4s v0, v0, v1
+; CHECK:    addv.4s s0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: sabd.4s v0
-; CHECK: sabd.4s v1
-; CHECK: sabd.4s v2
-; CHECK: saba.4s v0
-; CHECK: saba.4s v1
-; CHECK: saba.4s v2
-; CHECK: saba.4s v0
-; CHECK: saba.4s v1
-; CHECK: add.4s v0, v2, v0
-; CHECK: add.4s v0, v0, v1
-; CHECK: addv.4s
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -203,11 +204,11 @@ loop:
   %vabd = tail call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %a, <4 x i32> %b)
   %acc_next = add <4 x i32> %acc_phi, %vabd
   ; Increment loop counter and check the bound
-  %next_i = add i32 %i, 4 
-  %cmp = icmp slt i32 %next_i, 32  
+  %next_i = add i32 %i, 4
+  %cmp = icmp slt i32 %next_i, 32
   br i1 %cmp, label %loop, label %exit
 
-exit:                            
+exit:
   %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)
   ret i32 %reduce
 }
@@ -215,22 +216,22 @@ exit:
 declare <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 
 define i32 @uaba_v2i32_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uaba_v2i32_accumulation
+; CHECK-LABEL: uaba_v2i32_accumulation:
+; CHECK:    uabd.2s v0, v0, v2
+; CHECK:    uabd.2s v1, v1, v3
+; CHECK:    uabd.2s v2, v2, v5
+; CHECK:    uaba.2s v0, v4, v6
+; CHECK:    uaba.2s v1, v7, v17
+; CHECK:    uaba.2s v2, v16, v18
+; CHECK:    uaba.2s v0, v19, v21
+; CHECK:    uaba.2s v1, v20, v22
+; CHECK:    add.2s v0, v2, v0
+; CHECK:    add.2s v0, v0, v1
+; CHECK:    addp.2s v0, v0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: uabd.2s v0
-; CHECK: uabd.2s v1
-; CHECK: uabd.2s v2
-; CHECK: uaba.2s v0
-; CHECK: uaba.2s v1
-; CHECK: uaba.2s v2
-; CHECK: uaba.2s v0
-; CHECK: uaba.2s v1
-; CHECK: add.2s v0, v2, v0
-; CHECK: add.2s v0, v0, v1
-; CHECK: addp.2s
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <2 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -250,22 +251,22 @@ exit:
 }
 
 define i8 @uaba_v8i8_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uaba_v8i8_accumulation
+; CHECK-LABEL: uaba_v8i8_accumulation:
+; CHECK:    uabd.8b v0, v0, v2
+; CHECK:    uabd.8b v1, v1, v3
+; CHECK:    uabd.8b v2, v2, v5
+; CHECK:    uaba.8b v0, v4, v6
+; CHECK:    uaba.8b v1, v7, v17
+; CHECK:    uaba.8b v2, v16, v18
+; CHECK:    uaba.8b v0, v19, v21
+; CHECK:    uaba.8b v1, v20, v22
+; CHECK:    add.8b v0, v2, v0
+; CHECK:    add.8b v0, v0, v1
+; CHECK:    addv.8b b0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: uabd.8b v0
-; CHECK: uabd.8b v1
-; CHECK: uabd.8b v2
-; CHECK: uaba.8b v0
-; CHECK: uaba.8b v1
-; CHECK: uaba.8b v2
-; CHECK: uaba.8b v0
-; CHECK: uaba.8b v1
-; CHECK: add.8b v0, v2, v0
-; CHECK: add.8b v0, v0, v1
-; CHECK: addv.8b
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <8 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -285,22 +286,22 @@ exit:
 }
 
 define i8 @uaba_v16i8_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uaba_v16i8_accumulation
+; CHECK-LABEL: uaba_v16i8_accumulation:
+; CHECK:    uabd.16b v0, v0, v2
+; CHECK:    uabd.16b v1, v1, v3
+; CHECK:    uabd.16b v2, v2, v5
+; CHECK:    uaba.16b v0, v4, v6
+; CHECK:    uaba.16b v1, v7, v17
+; CHECK:    uaba.16b v2, v16, v18
+; CHECK:    uaba.16b v0, v19, v21
+; CHECK:    uaba.16b v1, v20, v22
+; CHECK:    add.16b v0, v2, v0
+; CHECK:    add.16b v0, v0, v1
+; CHECK:    addv.16b b0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: uabd.16b v0
-; CHECK: uabd.16b v1
-; CHECK: uabd.16b v2
-; CHECK: uaba.16b v0
-; CHECK: uaba.16b v1
-; CHECK: uaba.16b v2
-; CHECK: uaba.16b v0
-; CHECK: uaba.16b v1
-; CHECK: add.16b v0, v2, v0
-; CHECK: add.16b v0, v0, v1
-; CHECK: addv.16b
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <16 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -320,22 +321,22 @@ exit:
 }
 
 define i16 @uaba_v8i16_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uaba_v8i16_accumulation
+; CHECK-LABEL: uaba_v8i16_accumulation:
+; CHECK:    uabd.8h v0, v0, v2
+; CHECK:    uabd.8h v1, v1, v3
+; CHECK:    uabd.8h v2, v2, v5
+; CHECK:    uaba.8h v0, v4, v6
+; CHECK:    uaba.8h v1, v7, v17
+; CHECK:    uaba.8h v2, v16, v18
+; CHECK:    uaba.8h v0, v19, v21
+; CHECK:    uaba.8h v1, v20, v22
+; CHECK:    add.8h v0, v2, v0
+; CHECK:    add.8h v0, v0, v1
+; CHECK:    addv.8h h0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: uabd.8h v0
-; CHECK: uabd.8h v1
-; CHECK: uabd.8h v2
-; CHECK: uaba.8h v0
-; CHECK: uaba.8h v1
-; CHECK: uaba.8h v2
-; CHECK: uaba.8h v0
-; CHECK: uaba.8h v1
-; CHECK: add.8h v0, v2, v0
-; CHECK: add.8h v0, v0, v1
-; CHECK: addv.8h
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -355,22 +356,22 @@ exit:
 }
 
 define i8 @saba_v8i8_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: saba_v8i8_accumulation
+; CHECK-LABEL: saba_v8i8_accumulation:
+; CHECK:    sabd.8b v0, v0, v2
+; CHECK:    sabd.8b v1, v1, v3
+; CHECK:    sabd.8b v2, v2, v5
+; CHECK:    saba.8b v0, v4, v6
+; CHECK:    saba.8b v1, v7, v17
+; CHECK:    saba.8b v2, v16, v18
+; CHECK:    saba.8b v0, v19, v21
+; CHECK:    saba.8b v1, v20, v22
+; CHECK:    add.8b v0, v2, v0
+; CHECK:    add.8b v0, v0, v1
+; CHECK:    addv.8b b0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: sabd.8b v0
-; CHECK: sabd.8b v1
-; CHECK: sabd.8b v2
-; CHECK: saba.8b v0
-; CHECK: saba.8b v1
-; CHECK: saba.8b v2
-; CHECK: saba.8b v0
-; CHECK: saba.8b v1
-; CHECK: add.8b v0, v2, v0
-; CHECK: add.8b v0, v0, v1
-; CHECK: addv.8b
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <8 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -390,21 +391,21 @@ exit:
 }
 
 define i16 @saba_v4i16_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: saba_v4i16_accumulation
+; CHECK-LABEL: saba_v4i16_accumulation:
+; CHECK:    sabd.4h v0, v0, v2
+; CHECK:    sabd.4h v1, v1, v3
+; CHECK:    sabd.4h v2, v2, v5
+; CHECK:    saba.4h v0, v4, v6
+; CHECK:    saba.4h v1, v7, v17
+; CHECK:    saba.4h v2, v16, v18
+; CHECK:    saba.4h v0, v19, v21
+; CHECK:    saba.4h v1, v20, v22
+; CHECK:    add.4h v0, v2, v0
+; CHECK:    add.4h v0, v0, v1
+; CHECK:    addv.4h h0, v0
 entry:
   br label %loop
 loop:
-; CHECK: sabd.4h v0
-; CHECK: sabd.4h v1
-; CHECK: sabd.4h v2
-; CHECK: saba.4h v0
-; CHECK: saba.4h v1
-; CHECK: saba.4h v2
-; CHECK: saba.4h v0
-; CHECK: saba.4h v1
-; CHECK: add.4h v0, v2, v0
-; CHECK: add.4h v0, v0, v1
-; CHECK: addv.4h
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <4 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -423,22 +424,22 @@ exit:
 }
 
 define i16 @saba_v8i16_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: saba_v8i16_accumulation
+; CHECK-LABEL: saba_v8i16_accumulation:
+; CHECK:    sabd.8h v0, v0, v2
+; CHECK:    sabd.8h v1, v1, v3
+; CHECK:    sabd.8h v2, v2, v5
+; CHECK:    saba.8h v0, v4, v6
+; CHECK:    saba.8h v1, v7, v17
+; CHECK:    saba.8h v2, v16, v18
+; CHECK:    saba.8h v0, v19, v21
+; CHECK:    saba.8h v1, v20, v22
+; CHECK:    add.8h v0, v2, v0
+; CHECK:    add.8h v0, v0, v1
+; CHECK:    addv.8h h0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: sabd.8h v0
-; CHECK: sabd.8h v1
-; CHECK: sabd.8h v2
-; CHECK: saba.8h v0
-; CHECK: saba.8h v1
-; CHECK: saba.8h v2
-; CHECK: saba.8h v0
-; CHECK: saba.8h v1
-; CHECK: add.8h v0, v2, v0
-; CHECK: add.8h v0, v0, v1
-; CHECK: addv.8h
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -458,22 +459,22 @@ exit:
 }
 
 define i16 @uabal_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uabal_i8_to_i16_accumulation
+; CHECK-LABEL: uabal_i8_to_i16_accumulation:
+; CHECK:    uabdl.8h v1, v1, v3
+; CHECK:    uabdl.8h v0, v0, v2
+; CHECK:    uabdl.8h v2, v3, v5
+; CHECK:    uabal.8h v1, v4, v6
+; CHECK:    uabal.8h v0, v7, v17
+; CHECK:    uabal.8h v2, v16, v18
+; CHECK:    uabal.8h v1, v19, v21
+; CHECK:    uabal.8h v0, v20, v22
+; CHECK:    add.8h v1, v2, v1
+; CHECK:    add.8h v0, v1, v0
+; CHECK:    addv.8h h0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: uabdl.8h v1
-; CHECK: uabdl.8h v0
-; CHECK: uabdl.8h v2
-; CHECK: uabal.8h v1
-; CHECK: uabal.8h v0
-; CHECK: uabal.8h v2
-; CHECK: uabal.8h v1
-; CHECK: uabal.8h v0
-; CHECK: add.8h v1, v2, v1
-; CHECK: add.8h v0, v1, v0
-; CHECK: addv.8h
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
@@ -494,22 +495,22 @@ exit:
 }
 
 define i32 @uabal_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: uabal_i16_to_i32_accumulation
+; CHECK-LABEL: uabal_i16_to_i32_accumulation:
+; CHECK:    uabdl.4s v1, v1, v3
+; CHECK:    uabdl.4s v0, v0, v2
+; CHECK:    uabdl.4s v2, v3, v5
+; CHECK:    uabal.4s v1, v4, v6
+; CHECK:    uabal.4s v0, v7, v17
+; CHECK:    uabal.4s v2, v16, v18
+; CHECK:    uabal.4s v1, v19, v21
+; CHECK:    uabal.4s v0, v20, v22
+; CHECK:    add.4s v1, v2, v1
+; CHECK:    add.4s v0, v1, v0
+; CHECK:    addv.4s s0, v0
 entry:
   br label %loop
 
 loop:
-; CHECK: uabdl.4s v1
-; CHECK: uabdl.4s v0
-; CHECK: uabdl.4s v2
-; CHECK: uabal.4s v1
-; CHECK: uabal.4s v0
-; CHECK: uabal.4s v2
-; CHECK: uabal.4s v1
-; CHECK: uabal.4s v0
-; CHECK: add.4s v1, v2, v1
-; CHECK: add.4s v0, v1, v0
-; CHECK: addv.4s
 
   %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
   %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 63cc061cb618..b92a52403698 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -mtriple aarch64-none-linux-gnu -mattr=+sme2 -force-streaming -stop-after=finalize-isel | FileCheck %s
+; RUN: llc < %s -mtriple aarch64-none-linux-gnu -stop-after=finalize-isel | FileCheck %s
 
-define void @UphPNR(target("aarch64.svcount") %predcnt) {
+define void @UphPNR(target("aarch64.svcount") %predcnt) "target-features"="+sme2" "aarch64_pstate_sm_enabled" {
 entry:
 ; CHECK:  %0:ppr = COPY $p0
 ; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
@@ -14,7 +14,7 @@ entry:
   ret void
 }
 
-define void @UpaPNR(target("aarch64.svcount") %predcnt) {
+define void @UpaPNR(target("aarch64.svcount") %predcnt) "target-features"="+sme2" "aarch64_pstate_sm_enabled" {
 entry:
 ; CHECK:  %0:ppr = COPY $p0
 ; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
@@ -28,7 +28,7 @@ entry:
   ret void
 }
 
-define void @UplPNR(target("aarch64.svcount") %predcnt) {
+define void @UplPNR(target("aarch64.svcount") %predcnt) "target-features"="+sme2" "aarch64_pstate_sm_enabled" {
 entry:
 ; CHECK:  %0:ppr = COPY $p0
 ; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
@@ -41,3 +41,86 @@ entry:
   call void asm sideeffect "fadd z0.h, $0/m, z0.h, #0.5", "@3Upl"(target("aarch64.svcount") %0)
   ret void
 }
+
+; Test that the z-register clobbers result in preserving %0 across the inline asm call.
+define <2 x float> @sme_nosve_nonstreaming(ptr %in) "target-features"="+sme,-sve" {
+entry:
+; CHECK-LABEL: name: sme_nosve_nonstreaming
+; CHECK:  INLINEASM &"smstart sm; smstop sm;"
+; CHECK-SAME: implicit-def early-clobber $q0
+; CHECK-SAME: implicit-def early-clobber $q1
+; CHECK-SAME: implicit-def early-clobber $q2
+; CHECK-SAME: implicit-def early-clobber $q3
+; CHECK-SAME: implicit-def early-clobber $q4
+; CHECK-SAME: implicit-def early-clobber $q5
+; CHECK-SAME: implicit-def early-clobber $q6
+; CHECK-SAME: implicit-def early-clobber $q7
+; CHECK-SAME: implicit-def early-clobber $q8
+; CHECK-SAME: implicit-def early-clobber $q9
+; CHECK-SAME: implicit-def early-clobber $q10
+; CHECK-SAME: implicit-def early-clobber $q11
+; CHECK-SAME: implicit-def early-clobber $q12
+; CHECK-SAME: implicit-def early-clobber $q13
+; CHECK-SAME: implicit-def early-clobber $q14
+; CHECK-SAME: implicit-def early-clobber $q15
+; CHECK-SAME: implicit-def early-clobber $q16
+; CHECK-SAME: implicit-def early-clobber $q17
+; CHECK-SAME: implicit-def early-clobber $q18
+; CHECK-SAME: implicit-def early-clobber $q19
+; CHECK-SAME: implicit-def early-clobber $q20
+; CHECK-SAME: implicit-def early-clobber $q21
+; CHECK-SAME: implicit-def early-clobber $q22
+; CHECK-SAME: implicit-def early-clobber $q23
+; CHECK-SAME: implicit-def early-clobber $q24
+; CHECK-SAME: implicit-def early-clobber $q25
+; CHECK-SAME: implicit-def early-clobber $q26
+; CHECK-SAME: implicit-def early-clobber $q27
+; CHECK-SAME: implicit-def early-clobber $q28
+; CHECK-SAME: implicit-def early-clobber $q29
+; CHECK-SAME: implicit-def early-clobber $q30
+; CHECK-SAME: implicit-def early-clobber $q31
+  %0 = load <2 x float>, ptr %in, align 8
+  call void asm sideeffect "smstart sm; smstop sm;", "~{z0},~{z1},~{z2},~{z3},~{z4},~{z5},~{z6},~{z7},~{z8},~{z9},~{z10},~{z11},~{z12},~{z13},~{z14},~{z15},~{z16},~{z17},~{z18},~{z19},~{z20},~{z21},~{z22},~{z23},~{z24},~{z25},~{z26},~{z27},~{z28},~{z29},~{z30},~{z31}"()
+  ret <2 x float> %0
+}
+
+define <2 x float> @sme_nosve_streaming(ptr %in) "target-features"="+sme,-sve" "aarch64_pstate_sm_enabled" {
+entry:
+; CHECK-LABEL: name: sme_nosve_streaming
+; CHECK:  INLINEASM &"smstart sm; smstop sm;"
+; CHECK-SAME: implicit-def early-clobber $z0
+; CHECK-SAME: implicit-def early-clobber $z1
+; CHECK-SAME: implicit-def early-clobber $z2
+; CHECK-SAME: implicit-def early-clobber $z3
+; CHECK-SAME: implicit-def early-clobber $z4
+; CHECK-SAME: implicit-def early-clobber $z5
+; CHECK-SAME: implicit-def early-clobber $z6
+; CHECK-SAME: implicit-def early-clobber $z7
+; CHECK-SAME: implicit-def early-clobber $z8
+; CHECK-SAME: implicit-def early-clobber $z9
+; CHECK-SAME: implicit-def early-clobber $z10
+; CHECK-SAME: implicit-def early-clobber $z11
+; CHECK-SAME: implicit-def early-clobber $z12
+; CHECK-SAME: implicit-def early-clobber $z13
+; CHECK-SAME: implicit-def early-clobber $z14
+; CHECK-SAME: implicit-def early-clobber $z15
+; CHECK-SAME: implicit-def early-clobber $z16
+; CHECK-SAME: implicit-def early-clobber $z17
+; CHECK-SAME: implicit-def early-clobber $z18
+; CHECK-SAME: implicit-def early-clobber $z19
+; CHECK-SAME: implicit-def early-clobber $z20
+; CHECK-SAME: implicit-def early-clobber $z21
+; CHECK-SAME: implicit-def early-clobber $z22
+; CHECK-SAME: implicit-def early-clobber $z23
+; CHECK-SAME: implicit-def early-clobber $z24
+; CHECK-SAME: implicit-def early-clobber $z25
+; CHECK-SAME: implicit-def early-clobber $z26
+; CHECK-SAME: implicit-def early-clobber $z27
+; CHECK-SAME: implicit-def early-clobber $z28
+; CHECK-SAME: implicit-def early-clobber $z29
+; CHECK-SAME: implicit-def early-clobber $z30
+; CHECK-SAME: implicit-def early-clobber $z31
+  %0 = load <2 x float>, ptr %in, align 8
+  call void asm sideeffect "smstart sm; smstop sm;", "~{z0},~{z1},~{z2},~{z3},~{z4},~{z5},~{z6},~{z7},~{z8},~{z9},~{z10},~{z11},~{z12},~{z13},~{z14},~{z15},~{z16},~{z17},~{z18},~{z19},~{z20},~{z21},~{z22},~{z23},~{z24},~{z25},~{z26},~{z27},~{z28},~{z29},~{z30},~{z31}"()
+  ret <2 x float> %0
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index d6fd4c4110a1..2f23a32c36a9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -2282,14 +2282,14 @@ define <2 x i64> @asr(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #32
 ; CHECK-GI-NEXT:    sshr v1.2d, v1.2d, #32
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
     %x = ashr <2 x i64> %a, <i64 32, i64 32>
     %y = ashr <2 x i64> %b, <i64 32, i64 32>
@@ -2317,14 +2317,14 @@ define <2 x i64> @asr_const(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-GI-NEXT:    adrp x8, .LCPI81_0
 ; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #32
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI81_0]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
     %x = ashr <2 x i64> %a, <i64 32, i64 32>
     %z = mul nsw <2 x i64> %x, <i64 31, i64 31>
@@ -2799,14 +2799,14 @@ define <2 x i64> @sdistribute_v2i32(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    sshll v2.2d, v2.2s, #0
 ; CHECK-GI-NEXT:    saddl v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v2.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %4 = sext <2 x i32> %src1 to <2 x i64>
@@ -2838,14 +2838,14 @@ define <2 x i64> @sdistribute_const1_v2i32(<2 x i32> %src1, <2 x i32> %mul) {
 ; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
 ; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI101_0]
 ; CHECK-GI-NEXT:    saddw v0.2d, v2.2d, v0.2s
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %4 = sext <2 x i32> %src1 to <2 x i64>
@@ -2875,14 +2875,14 @@ define <2 x i64> @sdistribute_const2_v2i32(<2 x i32> %src1, <2 x i32> %src2) {
 ; CHECK-GI-NEXT:    adrp x8, .LCPI102_0
 ; CHECK-GI-NEXT:    saddl v0.2d, v0.2s, v1.2s
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI102_0]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %4 = sext <2 x i32> %src1 to <2 x i64>
@@ -2909,14 +2909,14 @@ define <2 x i64> @udistribute_v2i32(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
 ; CHECK-GI-NEXT:    uaddl v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v2.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %4 = zext <2 x i32> %src1 to <2 x i64>
@@ -2948,14 +2948,14 @@ define <2 x i64> @udistribute_const1_v2i32(<2 x i32> %src1, <2 x i32> %mul) {
 ; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
 ; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI104_0]
 ; CHECK-GI-NEXT:    uaddw v0.2d, v2.2d, v0.2s
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %4 = zext <2 x i32> %src1 to <2 x i64>
@@ -2985,14 +2985,14 @@ define <2 x i64> @udistribute_const2_v2i32(<2 x i32> %src1, <2 x i32> %src2) {
 ; CHECK-GI-NEXT:    adrp x8, .LCPI105_0
 ; CHECK-GI-NEXT:    uaddl v0.2d, v0.2s, v1.2s
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI105_0]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %4 = zext <2 x i32> %src1 to <2 x i64>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/abs.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/abs.ll
index d501d9ed2454..0f56d25a47b2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/abs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/abs.ll
@@ -247,8 +247,7 @@ define <1 x i32> @abs_v1i32(<1 x i32> %a){
 ; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    cmp w8, #0
 ; CHECK-GI-NEXT:    cneg w8, w9, le
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
 entry:
   %res = call <1 x i32> @llvm.abs.v1i32(<1 x i32> %a, i1 0)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/argument-blocks-array-of-struct.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/argument-blocks-array-of-struct.ll
index f364429b86c3..6019a62f4925 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/argument-blocks-array-of-struct.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/argument-blocks-array-of-struct.ll
@@ -581,9 +581,8 @@ define void @callee_in_memory(%T_IN_MEMORY %a) {
 ; CHECK-SD-NEXT:    add x8, x8, :lo12:in_memory_store
 ; CHECK-SD-NEXT:    ldr d0, [sp, #64]
 ; CHECK-SD-NEXT:    str d0, [x8, #64]
-; CHECK-SD-NEXT:    ldr q0, [sp, #16]
 ; CHECK-SD-NEXT:    str q2, [x8, #48]
-; CHECK-SD-NEXT:    ldr q2, [sp]
+; CHECK-SD-NEXT:    ldp q2, q0, [sp]
 ; CHECK-SD-NEXT:    stp q0, q1, [x8, #16]
 ; CHECK-SD-NEXT:    str q2, [x8]
 ; CHECK-SD-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dagcombiner-load-slicing.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dagcombiner-load-slicing.ll
index 7e72e8de01f4..3bada9d5b3bb 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dagcombiner-load-slicing.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dagcombiner-load-slicing.ll
@@ -7,8 +7,8 @@
 
 ; CHECK-LABEL: @test
 ; CHECK: add [[BASE:x[0-9]+]], x0, x1, lsl #3
-; CHECK: ldp [[CPLX1_I:s[0-9]+]], [[CPLX1_R:s[0-9]+]], [[[BASE]]]
-; CHECK: ldp [[CPLX2_I:s[0-9]+]], [[CPLX2_R:s[0-9]+]], [[[BASE]], #64]
+; CHECK-DAG: ldp [[CPLX1_I:s[0-9]+]], [[CPLX1_R:s[0-9]+]], [[[BASE]]]
+; CHECK-DAG: ldp [[CPLX2_I:s[0-9]+]], [[CPLX2_R:s[0-9]+]], [[[BASE]], #64]
 ; CHECK: fadd {{s[0-9]+}}, [[CPLX2_I]], [[CPLX1_I]]
 ; CHECK: fadd {{s[0-9]+}}, [[CPLX2_R]], [[CPLX1_R]]
 ; CHECK: ret
@@ -36,8 +36,8 @@ entry:
 
 ; CHECK-LABEL: @test_int
 ; CHECK: add [[BASE:x[0-9]+]], x0, x1, lsl #3
-; CHECK: ldp [[CPLX1_I:w[0-9]+]], [[CPLX1_R:w[0-9]+]], [[[BASE]]]
-; CHECK: ldp [[CPLX2_I:w[0-9]+]], [[CPLX2_R:w[0-9]+]], [[[BASE]], #64]
+; CHECK-DAG: ldp [[CPLX1_I:w[0-9]+]], [[CPLX1_R:w[0-9]+]], [[[BASE]]]
+; CHECK-DAG: ldp [[CPLX2_I:w[0-9]+]], [[CPLX2_R:w[0-9]+]], [[[BASE]], #64]
 ; CHECK: add {{w[0-9]+}}, [[CPLX2_I]], [[CPLX1_I]]
 ; CHECK: add {{w[0-9]+}}, [[CPLX2_R]], [[CPLX1_R]]
 ; CHECK: ret
@@ -65,8 +65,8 @@ entry:
 
 ; CHECK-LABEL: @test_long
 ; CHECK: add [[BASE:x[0-9]+]], x0, x1, lsl #4
-; CHECK: ldp [[CPLX1_I:x[0-9]+]], [[CPLX1_R:x[0-9]+]], [[[BASE]]]
-; CHECK: ldp [[CPLX2_I:x[0-9]+]], [[CPLX2_R:x[0-9]+]], [[[BASE]], #128]
+; CHECK-DAG: ldp [[CPLX1_I:x[0-9]+]], [[CPLX1_R:x[0-9]+]], [[[BASE]]]
+; CHECK-DAG: ldp [[CPLX2_I:x[0-9]+]], [[CPLX2_R:x[0-9]+]], [[[BASE]], #128]
 ; CHECK: add {{x[0-9]+}}, [[CPLX2_I]], [[CPLX1_I]]
 ; CHECK: add {{x[0-9]+}}, [[CPLX2_R]], [[CPLX1_R]]
 ; CHECK: ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dup.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dup.ll
index 4c28ea759220..49fb6c98e223 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dup.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-dup.ll
@@ -334,40 +334,25 @@ entry:
 }
 
 define <2 x i32> @f(i32 %a, i32 %b) nounwind readnone  {
-; CHECK-SD-LABEL: f:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    mov.s v0[1], w1
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: f:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov.s v0[0], w0
-; CHECK-GI-NEXT:    mov.s v0[1], w1
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: f:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov.s v0[1], w1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
   %vecinit = insertelement <2 x i32> undef, i32 %a, i32 0
   %vecinit1 = insertelement <2 x i32> %vecinit, i32 %b, i32 1
   ret <2 x i32> %vecinit1
 }
 
 define <4 x i32> @g(i32 %a, i32 %b) nounwind readnone  {
-; CHECK-SD-LABEL: g:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    mov.s v0[1], w1
-; CHECK-SD-NEXT:    mov.s v0[2], w1
-; CHECK-SD-NEXT:    mov.s v0[3], w0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: g:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov.s v0[0], w0
-; CHECK-GI-NEXT:    mov.s v0[1], w1
-; CHECK-GI-NEXT:    mov.s v0[2], w1
-; CHECK-GI-NEXT:    mov.s v0[3], w0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: g:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov.s v0[1], w1
+; CHECK-NEXT:    mov.s v0[2], w1
+; CHECK-NEXT:    mov.s v0[3], w0
+; CHECK-NEXT:    ret
   %vecinit = insertelement <4 x i32> undef, i32 %a, i32 0
   %vecinit1 = insertelement <4 x i32> %vecinit, i32 %b, i32 1
   %vecinit2 = insertelement <4 x i32> %vecinit1, i32 %b, i32 2
@@ -376,17 +361,11 @@ define <4 x i32> @g(i32 %a, i32 %b) nounwind readnone  {
 }
 
 define <2 x i64> @h(i64 %a, i64 %b) nounwind readnone  {
-; CHECK-SD-LABEL: h:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    mov.d v0[1], x1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: h:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov.d v0[0], x0
-; CHECK-GI-NEXT:    mov.d v0[1], x1
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    mov.d v0[1], x1
+; CHECK-NEXT:    ret
   %vecinit = insertelement <2 x i64> undef, i64 %a, i32 0
   %vecinit1 = insertelement <2 x i64> %vecinit, i64 %b, i32 1
   ret <2 x i64> %vecinit1
@@ -422,16 +401,10 @@ define <4 x i16> @test_build_illegal(<4 x i32> %in) {
 ; SelectionDAGBuilder here. We then added a DUPLANE on top of that, preventing
 ; the formation of an indexed-by-7 MLS.
 define <4 x i16> @test_high_splat(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) #0 {
-; CHECK-SD-LABEL: test_high_splat:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mls.4h v0, v1, v2[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_high_splat:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup.8h v2, v2[7]
-; CHECK-GI-NEXT:    mls.4h v0, v2, v1
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_high_splat:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mls.4h v0, v1, v2[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %mul = mul <4 x i16> %shuffle, %b
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
index 0f085566b0d8..69b30108ed53 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
@@ -174,7 +174,7 @@ define <2 x i16> @test_varidx_extract_v4s16(<4 x i16> %x, i32 %idx) {
 ; CHECK-GISEL-NEXT:    and x9, x9, #0x3
 ; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GISEL-NEXT:    str d0, [sp, #8]
-; CHECK-GISEL-NEXT:    madd x8, x9, x8, x10
+; CHECK-GISEL-NEXT:    umaddl x8, w9, w8, x10
 ; CHECK-GISEL-NEXT:    umov w9, v0.h[1]
 ; CHECK-GISEL-NEXT:    ld1 { v0.h }[0], [x8]
 ; CHECK-GISEL-NEXT:    mov v0.s[1], w9
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll
index 550e89f4a27f..aec31eced397 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll
@@ -53,3 +53,22 @@ define i64 @test_integer(i64  %in) {
   %val = select i1 %cmp, i64 0, i64 %in
   ret i64 %val
 }
+
+; Make sure we don't translate it into fminnm when the nsz flag is set on the fcmp.
+define float @minnum_fcmp_nsz(float %x, float %y) {
+; CHECK-LABEL: minnum_fcmp_nsz:
+  %cmp = fcmp nnan nsz ole float %x, %y
+  %sel = select i1 %cmp, float %x, float %y
+  ret float %sel
+; CHECK-NOT: fminnm
+; CHECK: fcsel s0, s0, s1, le
+}
+
+; Make sure we translate it into fminnm when the nsz flag is set on the select.
+define float @minnum_select_nsz(float %x, float %y) {
+; CHECK-LABEL: minnum_select_nsz:
+  %cmp = fcmp nnan ole float %x, %y
+  %sel = select nsz i1 %cmp, float %x, float %y
+  ret float %sel
+; CHECK: fminnm s0, s0, s1
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fp128.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fp128.ll
index 7eb26096ed15..a75f6419d5a5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fp128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-fp128.ll
@@ -618,7 +618,7 @@ define <2 x i32> @vec_fptosi_32(<2 x fp128> %val) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -661,7 +661,7 @@ define <2 x i64> @vec_fptosi_64(<2 x fp128> %val) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov x19, x0
 ; CHECK-GI-NEXT:    bl __fixtfdi
-; CHECK-GI-NEXT:    mov v0.d[0], x19
+; CHECK-GI-NEXT:    fmov d0, x19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.d[1], x0
 ; CHECK-GI-NEXT:    add sp, sp, #32
@@ -702,7 +702,7 @@ define <2 x i32> @vec_fptoui_32(<2 x fp128> %val) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -745,7 +745,7 @@ define <2 x i64> @vec_fptoui_64(<2 x fp128> %val) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov x19, x0
 ; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    mov v0.d[0], x19
+; CHECK-GI-NEXT:    fmov d0, x19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.d[1], x0
 ; CHECK-GI-NEXT:    add sp, sp, #32
@@ -977,7 +977,7 @@ define <2 x i1> @vec_setcc1(<2 x fp128> %lhs, <2 x fp128> %rhs) {
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w19, le
 ; CHECK-GI-NEXT:    bl __letf2
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w8, le
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
@@ -1032,7 +1032,7 @@ define <2 x i1> @vec_setcc2(<2 x fp128> %lhs, <2 x fp128> %rhs) {
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w19, gt
 ; CHECK-GI-NEXT:    bl __letf2
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w8, gt
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
@@ -1109,7 +1109,7 @@ define <2 x i1> @vec_setcc3(<2 x fp128> %lhs, <2 x fp128> %rhs) {
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w20, eq
 ; CHECK-GI-NEXT:    bl __unordtf2
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    cset w8, ne
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
index c3ad3b4192cf..85d8b7c3e286 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
@@ -159,16 +159,10 @@ entry:
 }
 
 define <4 x i16> @test_vmla_laneq_s16(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmla_laneq_s16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mla v0.4h, v1.4h, v2.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmla_laneq_s16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    mla v0.4h, v2.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmla_laneq_s16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla v0.4h, v1.4h, v2.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %mul = mul <4 x i16> %shuffle, %b
@@ -189,16 +183,10 @@ entry:
 }
 
 define <2 x i32> @test_vmla_laneq_s32(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmla_laneq_s32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mla v0.2s, v1.2s, v2.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmla_laneq_s32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    mla v0.2s, v2.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmla_laneq_s32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla v0.2s, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %mul = mul <2 x i32> %shuffle, %b
@@ -271,16 +259,10 @@ entry:
 }
 
 define <4 x i16> @test_vmls_laneq_s16(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmls_laneq_s16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmls_laneq_s16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    mls v0.4h, v2.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmls_laneq_s16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mls v0.4h, v1.4h, v2.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %mul = mul <4 x i16> %shuffle, %b
@@ -301,16 +283,10 @@ entry:
 }
 
 define <2 x i32> @test_vmls_laneq_s32(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmls_laneq_s32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmls_laneq_s32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmls_laneq_s32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mls v0.2s, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %mul = mul <2 x i32> %shuffle, %b
@@ -427,16 +403,10 @@ entry:
 }
 
 define <4 x i16> @test_vmul_laneq_s16(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_s16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_s16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-GI-NEXT:    mul v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_s16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.4h, v0.4h, v1.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %mul = mul <4 x i16> %shuffle, %a
@@ -455,16 +425,10 @@ entry:
 }
 
 define <2 x i32> @test_vmul_laneq_s32(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_s32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_s32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    mul v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_s32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %mul = mul <2 x i32> %shuffle, %a
@@ -483,16 +447,10 @@ entry:
 }
 
 define <4 x i16> @test_vmul_laneq_u16(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_u16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_u16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-GI-NEXT:    mul v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_u16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.4h, v0.4h, v1.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %mul = mul <4 x i16> %shuffle, %a
@@ -511,16 +469,10 @@ entry:
 }
 
 define <2 x i32> @test_vmul_laneq_u32(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_u32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_u32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    mul v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_u32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %mul = mul <2 x i32> %shuffle, %a
@@ -567,16 +519,10 @@ entry:
 declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
 
 define <2 x float> @test_vfma_laneq_f32(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
-; CHECK-SD-LABEL: test_vfma_laneq_f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmla v0.2s, v1.2s, v2.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vfma_laneq_f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vfma_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -834,16 +780,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlal_laneq_s16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_s16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlal v0.4s, v1.4h, v2.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_s16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    smlal v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_s16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlal v0.4s, v1.4h, v2.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -852,16 +792,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlal_laneq_s32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_s32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlal v0.2d, v1.2s, v2.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_s32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    smlal v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_s32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlal v0.2d, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -920,8 +854,7 @@ define <4 x i32> @test_vmlal_high_laneq_s16(<4 x i32> %a, <8 x i16> %b, <8 x i16
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_s16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    smlal v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    smlal v0.4s, v1.4h, v2.h[7]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -940,8 +873,7 @@ define <2 x i64> @test_vmlal_high_laneq_s32(<2 x i64> %a, <4 x i32> %b, <4 x i32
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_s32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    smlal v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    smlal v0.2d, v1.2s, v2.s[3]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -978,16 +910,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlsl_laneq_s16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_s16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlsl v0.4s, v1.4h, v2.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_s16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    smlsl v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_s16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlsl v0.4s, v1.4h, v2.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -996,16 +922,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlsl_laneq_s32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_s32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlsl v0.2d, v1.2s, v2.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_s32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    smlsl v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_s32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlsl v0.2d, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -1064,8 +984,7 @@ define <4 x i32> @test_vmlsl_high_laneq_s16(<4 x i32> %a, <8 x i16> %b, <8 x i16
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_s16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    smlsl v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    smlsl v0.4s, v1.4h, v2.h[7]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -1084,8 +1003,7 @@ define <2 x i64> @test_vmlsl_high_laneq_s32(<2 x i64> %a, <4 x i32> %b, <4 x i32
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_s32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    smlsl v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    smlsl v0.2d, v1.2s, v2.s[3]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -1122,16 +1040,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlal_laneq_u16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_u16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlal v0.4s, v1.4h, v2.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_u16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    umlal v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_u16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlal v0.4s, v1.4h, v2.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -1140,16 +1052,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlal_laneq_u32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_u32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlal v0.2d, v1.2s, v2.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_u32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    umlal v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_u32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlal v0.2d, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -1208,8 +1114,7 @@ define <4 x i32> @test_vmlal_high_laneq_u16(<4 x i32> %a, <8 x i16> %b, <8 x i16
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_u16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    umlal v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    umlal v0.4s, v1.4h, v2.h[7]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -1228,8 +1133,7 @@ define <2 x i64> @test_vmlal_high_laneq_u32(<2 x i64> %a, <4 x i32> %b, <4 x i32
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_u32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    umlal v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    umlal v0.2d, v1.2s, v2.s[3]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -1266,16 +1170,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlsl_laneq_u16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_u16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlsl v0.4s, v1.4h, v2.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_u16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    umlsl v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_u16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlsl v0.4s, v1.4h, v2.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -1284,16 +1182,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlsl_laneq_u32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_u32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlsl v0.2d, v1.2s, v2.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_u32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    umlsl v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_u32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlsl v0.2d, v1.2s, v2.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -1352,8 +1244,7 @@ define <4 x i32> @test_vmlsl_high_laneq_u16(<4 x i32> %a, <8 x i16> %b, <8 x i16
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_u16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[7]
-; CHECK-GI-NEXT:    umlsl v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    umlsl v0.4s, v1.4h, v2.h[7]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -1372,8 +1263,7 @@ define <2 x i64> @test_vmlsl_high_laneq_u32(<2 x i64> %a, <4 x i32> %b, <4 x i32
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_u32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[3]
-; CHECK-GI-NEXT:    umlsl v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    umlsl v0.2d, v1.2s, v2.s[3]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -1512,16 +1402,10 @@ entry:
 }
 
 define <4 x i32> @test_vmull_laneq_s16(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_s16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smull v0.4s, v0.4h, v1.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_s16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-GI-NEXT:    smull v0.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_s16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smull v0.4s, v0.4h, v1.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)
@@ -1529,16 +1413,10 @@ entry:
 }
 
 define <2 x i64> @test_vmull_laneq_s32(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_s32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smull v0.2d, v0.2s, v1.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_s32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    smull v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_s32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smull v0.2d, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)
@@ -1546,16 +1424,10 @@ entry:
 }
 
 define <4 x i32> @test_vmull_laneq_u16(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_u16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umull v0.4s, v0.4h, v1.h[7]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_u16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-GI-NEXT:    umull v0.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_u16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umull v0.4s, v0.4h, v1.h[7]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)
@@ -1563,16 +1435,10 @@ entry:
 }
 
 define <2 x i64> @test_vmull_laneq_u32(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_u32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umull v0.2d, v0.2s, v1.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_u32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    umull v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_u32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umull v0.2d, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)
@@ -1588,8 +1454,7 @@ define <4 x i32> @test_vmull_high_laneq_s16(<8 x i16> %a, <8 x i16> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_s16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-GI-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    smull v0.4s, v0.4h, v1.h[7]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -1607,8 +1472,7 @@ define <2 x i64> @test_vmull_high_laneq_s32(<4 x i32> %a, <4 x i32> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_s32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    smull v0.2d, v0.2s, v1.s[3]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -1626,8 +1490,7 @@ define <4 x i32> @test_vmull_high_laneq_u16(<8 x i16> %a, <8 x i16> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_u16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-GI-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    umull v0.4s, v0.4h, v1.h[7]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -1645,8 +1508,7 @@ define <2 x i64> @test_vmull_high_laneq_u32(<4 x i32> %a, <4 x i32> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_u32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    umull v0.2d, v0.2s, v1.s[3]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -1816,16 +1678,10 @@ entry:
 }
 
 define <4 x i32> @test_vqdmull_laneq_s16(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vqdmull_laneq_s16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sqdmull v0.4s, v0.4h, v1.h[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vqdmull_laneq_s16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[3]
-; CHECK-GI-NEXT:    sqdmull v0.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vqdmull_laneq_s16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqdmull v0.4s, v0.4h, v1.h[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)
@@ -1833,16 +1689,10 @@ entry:
 }
 
 define <2 x i64> @test_vqdmull_laneq_s32(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vqdmull_laneq_s32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sqdmull v0.2d, v0.2s, v1.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vqdmull_laneq_s32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    sqdmull v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vqdmull_laneq_s32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqdmull v0.2d, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>
   %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)
@@ -1898,8 +1748,7 @@ define <4 x i32> @test_vqdmull_high_laneq_s16(<8 x i16> %a, <8 x i16> %v) {
 ; CHECK-GI-LABEL: test_vqdmull_high_laneq_s16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[7]
-; CHECK-GI-NEXT:    sqdmull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    sqdmull v0.4s, v0.4h, v1.h[7]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -1917,8 +1766,7 @@ define <2 x i64> @test_vqdmull_high_laneq_s32(<4 x i32> %a, <4 x i32> %v) {
 ; CHECK-GI-LABEL: test_vqdmull_high_laneq_s32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    sqdmull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    sqdmull v0.2d, v0.2s, v1.s[3]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -2322,16 +2170,10 @@ entry:
 }
 
 define <2 x float> @test_vmul_laneq_f32(<2 x float> %a, <4 x float> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmul v0.2s, v0.2s, v1.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    fmul v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>
   %mul = fmul <2 x float> %shuffle, %a
@@ -2553,16 +2395,10 @@ entry:
 }
 
 define <2 x float> @test_vmulx_laneq_f32(<2 x float> %a, <4 x float> %v) {
-; CHECK-SD-LABEL: test_vmulx_laneq_f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmulx v0.2s, v0.2s, v1.s[3]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmulx_laneq_f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[3]
-; CHECK-GI-NEXT:    fmulx v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmulx_laneq_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.2s, v0.2s, v1.s[3]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>
   %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)
@@ -2657,16 +2493,10 @@ entry:
 }
 
 define <4 x i16> @test_vmla_laneq_s16_0(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmla_laneq_s16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mla v0.4h, v1.4h, v2.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmla_laneq_s16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    mla v0.4h, v2.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmla_laneq_s16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla v0.4h, v1.4h, v2.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %mul = mul <4 x i16> %shuffle, %b
@@ -2687,16 +2517,10 @@ entry:
 }
 
 define <2 x i32> @test_vmla_laneq_s32_0(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmla_laneq_s32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mla v0.2s, v1.2s, v2.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmla_laneq_s32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    mla v0.2s, v2.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmla_laneq_s32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mla v0.2s, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %mul = mul <2 x i32> %shuffle, %b
@@ -2769,16 +2593,10 @@ entry:
 }
 
 define <4 x i16> @test_vmls_laneq_s16_0(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmls_laneq_s16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmls_laneq_s16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    mls v0.4h, v2.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmls_laneq_s16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mls v0.4h, v1.4h, v2.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %mul = mul <4 x i16> %shuffle, %b
@@ -2799,16 +2617,10 @@ entry:
 }
 
 define <2 x i32> @test_vmls_laneq_s32_0(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmls_laneq_s32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmls_laneq_s32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmls_laneq_s32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mls v0.2s, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %mul = mul <2 x i32> %shuffle, %b
@@ -2925,16 +2737,10 @@ entry:
 }
 
 define <4 x i16> @test_vmul_laneq_s16_0(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_s16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_s16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    mul v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_s16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.4h, v0.4h, v1.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %mul = mul <4 x i16> %shuffle, %a
@@ -2953,16 +2759,10 @@ entry:
 }
 
 define <2 x i32> @test_vmul_laneq_s32_0(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_s32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_s32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    mul v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_s32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %mul = mul <2 x i32> %shuffle, %a
@@ -2981,16 +2781,10 @@ entry:
 }
 
 define <4 x i16> @test_vmul_laneq_u16_0(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_u16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_u16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    mul v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_u16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.4h, v0.4h, v1.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %mul = mul <4 x i16> %shuffle, %a
@@ -3009,16 +2803,10 @@ entry:
 }
 
 define <2 x i32> @test_vmul_laneq_u32_0(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_u32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_u32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    mul v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_u32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %mul = mul <2 x i32> %shuffle, %a
@@ -3061,16 +2849,10 @@ entry:
 }
 
 define <2 x float> @test_vfma_laneq_f32_0(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
-; CHECK-SD-LABEL: test_vfma_laneq_f32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vfma_laneq_f32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    fmla v0.2s, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vfma_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v0.2s, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %lane = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer
   %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -3188,16 +2970,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlal_laneq_s16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_s16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlal v0.4s, v1.4h, v2.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_s16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    smlal v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_s16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlal v0.4s, v1.4h, v2.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -3206,16 +2982,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlal_laneq_s32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_s32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlal v0.2d, v1.2s, v2.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_s32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    smlal v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_s32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlal v0.2d, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -3274,8 +3044,7 @@ define <4 x i32> @test_vmlal_high_laneq_s16_0(<4 x i32> %a, <8 x i16> %b, <8 x i
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_s16_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    smlal v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    smlal v0.4s, v1.4h, v2.h[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -3294,8 +3063,7 @@ define <2 x i64> @test_vmlal_high_laneq_s32_0(<2 x i64> %a, <4 x i32> %b, <4 x i
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_s32_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    smlal v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    smlal v0.2d, v1.2s, v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -3332,16 +3100,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlsl_laneq_s16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_s16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlsl v0.4s, v1.4h, v2.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_s16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    smlsl v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_s16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlsl v0.4s, v1.4h, v2.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -3350,16 +3112,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlsl_laneq_s32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_s32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smlsl v0.2d, v1.2s, v2.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_s32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    smlsl v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_s32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smlsl v0.2d, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -3418,8 +3174,7 @@ define <4 x i32> @test_vmlsl_high_laneq_s16_0(<4 x i32> %a, <8 x i16> %b, <8 x i
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_s16_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    smlsl v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    smlsl v0.4s, v1.4h, v2.h[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -3438,8 +3193,7 @@ define <2 x i64> @test_vmlsl_high_laneq_s32_0(<2 x i64> %a, <4 x i32> %b, <4 x i
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_s32_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    smlsl v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    smlsl v0.2d, v1.2s, v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -3476,16 +3230,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlal_laneq_u16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_u16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlal v0.4s, v1.4h, v2.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_u16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    umlal v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_u16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlal v0.4s, v1.4h, v2.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -3494,16 +3242,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlal_laneq_u32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlal_laneq_u32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlal v0.2d, v1.2s, v2.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlal_laneq_u32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    umlal v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlal_laneq_u32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlal v0.2d, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -3562,8 +3304,7 @@ define <4 x i32> @test_vmlal_high_laneq_u16_0(<4 x i32> %a, <8 x i16> %b, <8 x i
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_u16_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    umlal v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    umlal v0.4s, v1.4h, v2.h[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -3582,8 +3323,7 @@ define <2 x i64> @test_vmlal_high_laneq_u32_0(<2 x i64> %a, <4 x i32> %b, <4 x i
 ; CHECK-GI-LABEL: test_vmlal_high_laneq_u32_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    umlal v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    umlal v0.2d, v1.2s, v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -3620,16 +3360,10 @@ entry:
 }
 
 define <4 x i32> @test_vmlsl_laneq_u16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_u16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlsl v0.4s, v1.4h, v2.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_u16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    umlsl v0.4s, v1.4h, v2.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_u16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlsl v0.4s, v1.4h, v2.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)
@@ -3638,16 +3372,10 @@ entry:
 }
 
 define <2 x i64> @test_vmlsl_laneq_u32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmlsl_laneq_u32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umlsl v0.2d, v1.2s, v2.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmlsl_laneq_u32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    umlsl v0.2d, v1.2s, v2.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmlsl_laneq_u32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umlsl v0.2d, v1.2s, v2.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)
@@ -3706,8 +3434,7 @@ define <4 x i32> @test_vmlsl_high_laneq_u16_0(<4 x i32> %a, <8 x i16> %b, <8 x i
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_u16_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.8h, v2.h[0]
-; CHECK-GI-NEXT:    umlsl v0.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    umlsl v0.4s, v1.4h, v2.h[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -3726,8 +3453,7 @@ define <2 x i64> @test_vmlsl_high_laneq_u32_0(<2 x i64> %a, <4 x i32> %b, <4 x i
 ; CHECK-GI-LABEL: test_vmlsl_high_laneq_u32_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v1.d[1]
-; CHECK-GI-NEXT:    dup v2.4s, v2.s[0]
-; CHECK-GI-NEXT:    umlsl v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    umlsl v0.2d, v1.2s, v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -3866,16 +3592,10 @@ entry:
 }
 
 define <4 x i32> @test_vmull_laneq_s16_0(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_s16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smull v0.4s, v0.4h, v1.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_s16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    smull v0.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_s16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smull v0.4s, v0.4h, v1.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)
@@ -3883,16 +3603,10 @@ entry:
 }
 
 define <2 x i64> @test_vmull_laneq_s32_0(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_s32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    smull v0.2d, v0.2s, v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_s32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    smull v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_s32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    smull v0.2d, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)
@@ -3900,16 +3614,10 @@ entry:
 }
 
 define <4 x i32> @test_vmull_laneq_u16_0(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_u16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umull v0.4s, v0.4h, v1.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_u16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    umull v0.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_u16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umull v0.4s, v0.4h, v1.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)
@@ -3917,16 +3625,10 @@ entry:
 }
 
 define <2 x i64> @test_vmull_laneq_u32_0(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vmull_laneq_u32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    umull v0.2d, v0.2s, v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmull_laneq_u32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    umull v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmull_laneq_u32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    umull v0.2d, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)
@@ -3942,8 +3644,7 @@ define <4 x i32> @test_vmull_high_laneq_s16_0(<8 x i16> %a, <8 x i16> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_s16_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    smull v0.4s, v0.4h, v1.h[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -3961,8 +3662,7 @@ define <2 x i64> @test_vmull_high_laneq_s32_0(<4 x i32> %a, <4 x i32> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_s32_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    smull v0.2d, v0.2s, v1.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -3980,8 +3680,7 @@ define <4 x i32> @test_vmull_high_laneq_u16_0(<8 x i16> %a, <8 x i16> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_u16_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    umull v0.4s, v0.4h, v1.h[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -3999,8 +3698,7 @@ define <2 x i64> @test_vmull_high_laneq_u32_0(<4 x i32> %a, <4 x i32> %v) {
 ; CHECK-GI-LABEL: test_vmull_high_laneq_u32_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    umull v0.2d, v0.2s, v1.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -4170,16 +3868,10 @@ entry:
 }
 
 define <4 x i32> @test_vqdmull_laneq_s16_0(<4 x i16> %a, <8 x i16> %v) {
-; CHECK-SD-LABEL: test_vqdmull_laneq_s16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sqdmull v0.4s, v0.4h, v1.h[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vqdmull_laneq_s16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    sqdmull v0.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vqdmull_laneq_s16_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqdmull v0.4s, v0.4h, v1.h[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer
   %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)
@@ -4187,16 +3879,10 @@ entry:
 }
 
 define <2 x i64> @test_vqdmull_laneq_s32_0(<2 x i32> %a, <4 x i32> %v) {
-; CHECK-SD-LABEL: test_vqdmull_laneq_s32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sqdmull v0.2d, v0.2s, v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vqdmull_laneq_s32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    sqdmull v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vqdmull_laneq_s32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sqdmull v0.2d, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer
   %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)
@@ -4252,8 +3938,7 @@ define <4 x i32> @test_vqdmull_high_laneq_s16_0(<8 x i16> %a, <8 x i16> %v) {
 ; CHECK-GI-LABEL: test_vqdmull_high_laneq_s16_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
-; CHECK-GI-NEXT:    sqdmull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    sqdmull v0.4s, v0.4h, v1.h[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -4271,8 +3956,7 @@ define <2 x i64> @test_vqdmull_high_laneq_s32_0(<4 x i32> %a, <4 x i32> %v) {
 ; CHECK-GI-LABEL: test_vqdmull_high_laneq_s32_0:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    sqdmull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    sqdmull v0.2d, v0.2s, v1.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
@@ -4402,16 +4086,10 @@ entry:
 }
 
 define <2 x float> @test_vmul_laneq_f32_0(<2 x float> %a, <4 x float> %v) {
-; CHECK-SD-LABEL: test_vmul_laneq_f32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmul_laneq_f32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    fmul v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmul_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer
   %mul = fmul <2 x float> %shuffle, %a
@@ -4498,16 +4176,10 @@ entry:
 }
 
 define <2 x float> @test_vmulx_laneq_f32_0(<2 x float> %a, <4 x float> %v) {
-; CHECK-SD-LABEL: test_vmulx_laneq_f32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vmulx_laneq_f32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    fmulx v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vmulx_laneq_f32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmulx v0.2s, v0.2s, v1.s[0]
+; CHECK-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer
   %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-add-pairwise.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-add-pairwise.ll
index 17fb312c6375..0ede4bc7a4d6 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-add-pairwise.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-add-pairwise.ll
@@ -139,7 +139,7 @@ define i32 @addp_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    addp v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    rev64 v1.2s, v0.2s
+; CHECK-GI-NEXT:    dup v1.2s, v0.s[1]
 ; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
 ; CHECK-GI-NEXT:    fmov w0, s0
 ; CHECK-GI-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index 51f1351a5edf..367105f78381 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -807,46 +807,28 @@ define <2 x i64> @test_vdupq_lane_s64(<1 x i64> %v1) #0 {
 }
 
 define <8 x i8> @test_vdup_laneq_s8(<16 x i8> %v1) #0 {
-; CHECK-SD-LABEL: test_vdup_laneq_s8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    dup v0.8b, v0.b[5]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vdup_laneq_s8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    dup v0.16b, v0.b[5]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vdup_laneq_s8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v0.8b, v0.b[5]
+; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i8> %v1, <16 x i8> undef, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
   ret <8 x i8> %shuffle
 }
 
 define <4 x i16> @test_vdup_laneq_s16(<8 x i16> %v1) #0 {
-; CHECK-SD-LABEL: test_vdup_laneq_s16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    dup v0.4h, v0.h[2]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vdup_laneq_s16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    dup v0.8h, v0.h[2]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vdup_laneq_s16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v0.4h, v0.h[2]
+; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x i16> %v1, <8 x i16> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i16> %shuffle
 }
 
 define <2 x i32> @test_vdup_laneq_s32(<4 x i32> %v1) #0 {
-; CHECK-SD-LABEL: test_vdup_laneq_s32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    dup v0.2s, v0.s[1]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vdup_laneq_s32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    dup v0.4s, v0.s[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_vdup_laneq_s32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v0.2s, v0.s[1]
+; CHECK-NEXT:    ret
   %shuffle = shufflevector <4 x i32> %v1, <4 x i32> undef, <2 x i32> <i32 1, i32 1>
   ret <2 x i32> %shuffle
 }
@@ -1198,44 +1180,28 @@ define <8 x i16> @scalar_to_vector.v8i16(i16 %a) {
 }
 
 define <2 x i32> @scalar_to_vector.v2i32(i32 %a) {
-; CHECK-SD-LABEL: scalar_to_vector.v2i32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: scalar_to_vector.v2i32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: scalar_to_vector.v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <2 x i32> undef, i32 %a, i32 0
   ret <2 x i32> %b
 }
 
 define <4 x i32> @scalar_to_vector.v4i32(i32 %a) {
-; CHECK-SD-LABEL: scalar_to_vector.v4i32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: scalar_to_vector.v4i32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: scalar_to_vector.v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ret
   %b = insertelement <4 x i32> undef, i32 %a, i32 0
   ret <4 x i32> %b
 }
 
 define <2 x i64> @scalar_to_vector.v2i64(i64 %a) {
-; CHECK-SD-LABEL: scalar_to_vector.v2i64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: scalar_to_vector.v2i64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov v0.d[0], x0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: scalar_to_vector.v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ret
   %b = insertelement <2 x i64> undef, i64 %a, i32 0
   ret <2 x i64> %b
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll
index ca6bb8360de5..bdbebd8726fd 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll
@@ -139,7 +139,7 @@ define <4 x i32> @div32xi4(<4 x i32> %x) {
 ; CHECK-GI-NEXT:    mov w12, v0.s[3]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w8
 ; CHECK-GI-NEXT:    sdiv w10, w10, w8
-; CHECK-GI-NEXT:    mov v0.s[0], w9
+; CHECK-GI-NEXT:    fmov s0, w9
 ; CHECK-GI-NEXT:    sdiv w11, w11, w8
 ; CHECK-GI-NEXT:    mov v0.s[1], w10
 ; CHECK-GI-NEXT:    sdiv w8, w12, w8
@@ -240,14 +240,14 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %a) {
 ; CHECK-GI-LABEL: udiv_v2i64:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov x8, #9363 // =0x2493
-; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
 ; CHECK-GI-NEXT:    movk x8, #37449, lsl #16
 ; CHECK-GI-NEXT:    movk x8, #18724, lsl #32
 ; CHECK-GI-NEXT:    movk x8, #9362, lsl #48
-; CHECK-GI-NEXT:    umulh x9, x9, x8
-; CHECK-GI-NEXT:    umulh x8, x10, x8
-; CHECK-GI-NEXT:    mov v1.d[0], x9
+; CHECK-GI-NEXT:    umulh x10, x10, x8
+; CHECK-GI-NEXT:    umulh x8, x9, x8
+; CHECK-GI-NEXT:    fmov d1, x10
 ; CHECK-GI-NEXT:    mov v1.d[1], x8
 ; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
 ; CHECK-GI-NEXT:    usra v1.2d, v0.2d, #1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
index 6c70d19a977a..ce8bd7774cf1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v1i1-setcc.ll
@@ -174,9 +174,9 @@ define <1 x i16> @test_select_f16_i16(half %i105, half %in, <1 x i16> %x, <1 x i
 ; CHECK-LABEL: test_select_f16_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fcmp s0, s0
-; CHECK-NEXT:    csetm w8, vs
-; CHECK-NEXT:    dup v0.4h, w8
+; CHECK-NEXT:    fcmeq s0, s0, s0
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    dup v0.4h, v0.h[0]
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    ret
   %i179 = fcmp uno half %i105, zeroinitializer
@@ -249,3 +249,19 @@ if.then:
 if.end:
   ret i32 1;
 }
+
+define <1 x i64> @test_zext_half(<1 x half> %v1) {
+; CHECK-LABEL: test_zext_half:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcmgt v0.4s, v0.4s, #0.0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %1 = fcmp ogt <1 x half> %v1, zeroinitializer
+  %2 = zext <1 x i1> %1 to <1 x i64>
+  ret <1 x i64> %2
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll
index 3d52a2c044a0..cb14adc00df0 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+rdm | FileCheck %s
-; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+v8.1a | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+rdm | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+v8.1a | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+rdm -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16>, <4 x i16>)
 declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16>, <8 x i16>)
@@ -662,14 +663,24 @@ entry:
 }
 
 define i32 @test_vqrdmlahs_lane_s32(i32 %a, i32 %b, <2 x i32> %c) {
-; CHECK-LABEL: test_vqrdmlahs_lane_s32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, w0
-; CHECK-NEXT:    fmov s2, w1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    sqrdmlah s1, s2, v0.s[1]
-; CHECK-NEXT:    fmov w0, s1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_vqrdmlahs_lane_s32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s2, w1
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    sqrdmlah s1, s2, v0.s[1]
+; CHECK-SD-NEXT:    fmov w0, s1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_vqrdmlahs_lane_s32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    fmov s1, w0
+; CHECK-GI-NEXT:    fmov s2, w1
+; CHECK-GI-NEXT:    mov s0, v0.s[1]
+; CHECK-GI-NEXT:    sqrdmlah s1, s2, s0
+; CHECK-GI-NEXT:    fmov w0, s1
+; CHECK-GI-NEXT:    ret
 entry:
   %vget_lane = extractelement <2 x i32> %c, i64 1
   %vqrdmlahs_s32.i = tail call i32 @llvm.aarch64.neon.sqrdmlah.i32(i32 %a, i32 %b, i32 %vget_lane) #4
@@ -802,14 +813,24 @@ entry:
 }
 
 define i32 @test_vqrdmlshs_lane_s32(i32 %a, i32 %b, <2 x i32> %c) {
-; CHECK-LABEL: test_vqrdmlshs_lane_s32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, w0
-; CHECK-NEXT:    fmov s2, w1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    sqrdmlsh s1, s2, v0.s[1]
-; CHECK-NEXT:    fmov w0, s1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_vqrdmlshs_lane_s32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    fmov s2, w1
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    sqrdmlsh s1, s2, v0.s[1]
+; CHECK-SD-NEXT:    fmov w0, s1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_vqrdmlshs_lane_s32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    fmov s1, w0
+; CHECK-GI-NEXT:    fmov s2, w1
+; CHECK-GI-NEXT:    mov s0, v0.s[1]
+; CHECK-GI-NEXT:    sqrdmlsh s1, s2, s0
+; CHECK-GI-NEXT:    fmov w0, s1
+; CHECK-GI-NEXT:    ret
 entry:
   %vget_lane = extractelement <2 x i32> %c, i64 1
   %vqrdmlshs_s32.i = tail call i32 @llvm.aarch64.neon.sqrdmlsh.i32(i32 %a, i32 %b, i32 %vget_lane) #4
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-rev.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-rev.ll
index 6bdd5f998a3b..84557b441853 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-rev.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-rev.ll
@@ -34,8 +34,7 @@ define i32 @test_rev_w_srl16(i16 %a) {
 ; CHECK-GI-LABEL: test_rev_w_srl16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    and w8, w0, #0xffff
-; CHECK-GI-NEXT:    rev w8, w8
-; CHECK-GI-NEXT:    lsr w0, w8, #16
+; CHECK-GI-NEXT:    rev16 w0, w8
 ; CHECK-GI-NEXT:    ret
 entry:
   %0 = zext i16 %a to i32
@@ -45,12 +44,18 @@ entry:
 }
 
 define i32 @test_rev_w_srl16_load(ptr %a) {
-; CHECK-LABEL: test_rev_w_srl16_load:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    rev w8, w8
-; CHECK-NEXT:    lsr w0, w8, #16
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_rev_w_srl16_load:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    rev w8, w8
+; CHECK-SD-NEXT:    lsr w0, w8, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_rev_w_srl16_load:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrh w8, [x0]
+; CHECK-GI-NEXT:    rev16 w0, w8
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load i16, ptr %a
   %1 = zext i16 %0 to i32
@@ -71,8 +76,7 @@ define i32 @test_rev_w_srl16_add(i8 %a, i8 %b) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    and w8, w1, #0xff
 ; CHECK-GI-NEXT:    add w8, w8, w0, uxtb
-; CHECK-GI-NEXT:    rev w8, w8
-; CHECK-GI-NEXT:    lsr w0, w8, #16
+; CHECK-GI-NEXT:    rev16 w0, w8
 ; CHECK-GI-NEXT:    ret
 entry:
   %0 = zext i8 %a to i32
@@ -96,8 +100,7 @@ define i64 @test_rev_x_srl32(i32 %a) {
 ; CHECK-GI-LABEL: test_rev_x_srl32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov w8, w0
-; CHECK-GI-NEXT:    rev x8, x8
-; CHECK-GI-NEXT:    lsr x0, x8, #32
+; CHECK-GI-NEXT:    rev32 x0, x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %0 = zext i32 %a to i64
@@ -107,12 +110,18 @@ entry:
 }
 
 define i64 @test_rev_x_srl32_load(ptr %a) {
-; CHECK-LABEL: test_rev_x_srl32_load:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    rev x8, x8
-; CHECK-NEXT:    lsr x0, x8, #32
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_rev_x_srl32_load:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    rev x8, x8
+; CHECK-SD-NEXT:    lsr x0, x8, #32
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_rev_x_srl32_load:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    rev32 x0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load i32, ptr %a
   %1 = zext i32 %0 to i64
@@ -122,18 +131,11 @@ entry:
 }
 
 define i64 @test_rev_x_srl32_shift(i64 %a) {
-; CHECK-SD-LABEL: test_rev_x_srl32_shift:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ubfx x8, x0, #2, #29
-; CHECK-SD-NEXT:    rev32 x0, x8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_rev_x_srl32_shift:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ubfx x8, x0, #2, #29
-; CHECK-GI-NEXT:    rev x8, x8
-; CHECK-GI-NEXT:    lsr x0, x8, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_rev_x_srl32_shift:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ubfx x8, x0, #2, #29
+; CHECK-NEXT:    rev32 x0, x8
+; CHECK-NEXT:    ret
 entry:
   %0 = shl i64 %a, 33
   %1 = lshr i64 %0, 35
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vector-insertion.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vector-insertion.ll
index 94074d1689f6..ff28c7817d14 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vector-insertion.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vector-insertion.ll
@@ -66,6 +66,35 @@ define <16 x i8> @test_insert_v16i8_insert_2_undef_base(i8 %a) {
   %v.6 = insertelement <16 x i8> %v.4, i8 %a, i32 6
   %v.7 = insertelement <16 x i8> %v.6, i8 %a, i32 7
   %v.8 = insertelement <16 x i8> %v.7, i8 %a, i32 8
+  %v.10 = insertelement <16 x i8> %v.8, i8 %a, i32 10
+  %v.11 = insertelement <16 x i8> %v.10, i8 %a, i32 11
+  %v.12 = insertelement <16 x i8> %v.11, i8 %a, i32 12
+  %v.13 = insertelement <16 x i8> %v.12, i8 %a, i32 13
+  %v.14 = insertelement <16 x i8> %v.13, i8 %a, i32 14
+  %v.15 = insertelement <16 x i8> %v.14, i8 %a, i32 15
+  ret <16 x i8> %v.15
+}
+
+; Similar to above, but we leave element 8 as undef. One interesting part with
+; this test case is that %a may be poison, so simply inserting %a also at
+; index 8 would make the result vector more poisonous.
+define <16 x i8> @test_insert_v16i8_insert_2_undef_base_skip8(i32 %a0) {
+; CHECK-LABEL: test_insert_v16i8_insert_2_undef_base_skip8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr w8, w0, #5
+; CHECK-NEXT:    dup.16b v0, w8
+; CHECK-NEXT:    mov.b v0[5], wzr
+; CHECK-NEXT:    mov.b v0[9], wzr
+; CHECK-NEXT:    ret
+  %a1 = lshr exact i32 %a0, 5
+  %a = trunc i32 %a1 to i8
+  %v.0 = insertelement <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>  , i8 %a, i32 0
+  %v.1 = insertelement <16 x i8> %v.0, i8 %a, i32 1
+  %v.2 = insertelement <16 x i8> %v.1, i8 %a, i32 2
+  %v.3 = insertelement <16 x i8> %v.2, i8 %a, i32 3
+  %v.4 = insertelement <16 x i8> %v.3, i8 %a, i32 4
+  %v.6 = insertelement <16 x i8> %v.4, i8 %a, i32 6
+  %v.7 = insertelement <16 x i8> %v.6, i8 %a, i32 7
   %v.10 = insertelement <16 x i8> %v.7, i8 %a, i32 10
   %v.11 = insertelement <16 x i8> %v.10, i8 %a, i32 11
   %v.12 = insertelement <16 x i8> %v.11, i8 %a, i32 12
@@ -75,8 +104,8 @@ define <16 x i8> @test_insert_v16i8_insert_2_undef_base(i8 %a) {
   ret <16 x i8> %v.15
 }
 
-define <16 x i8> @test_insert_v16i8_insert_2_undef_base_different_valeus(i8 %a, i8 %b) {
-; CHECK-LABEL: test_insert_v16i8_insert_2_undef_base_different_valeus:
+define <16 x i8> @test_insert_v16i8_insert_2_undef_base_different_values(i8 %a, i8 %b) {
+; CHECK-LABEL: test_insert_v16i8_insert_2_undef_base_different_values:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    dup.16b v0, w0
 ; CHECK-NEXT:    mov.b v0[2], w1
@@ -94,6 +123,42 @@ define <16 x i8> @test_insert_v16i8_insert_2_undef_base_different_valeus(i8 %a,
   %v.6 = insertelement <16 x i8> %v.4, i8 %a, i32 6
   %v.7 = insertelement <16 x i8> %v.6, i8 %b, i32 7
   %v.8 = insertelement <16 x i8> %v.7, i8 %a, i32 8
+  %v.10 = insertelement <16 x i8> %v.8, i8 %a, i32 10
+  %v.11 = insertelement <16 x i8> %v.10, i8 %a, i32 11
+  %v.12 = insertelement <16 x i8> %v.11, i8 %b, i32 12
+  %v.13 = insertelement <16 x i8> %v.12, i8 %a, i32 13
+  %v.14 = insertelement <16 x i8> %v.13, i8 %a, i32 14
+  %v.15 = insertelement <16 x i8> %v.14, i8 %b, i32 15
+  ret <16 x i8> %v.15
+}
+
+; Similar to above, but we leave element 8 as undef. One interesting part with
+; this test case is that %a and %b may be poison, so simply inserting %a or %b
+; at index 8 would make the result vector more poisonous.
+define <16 x i8> @test_insert_v16i8_insert_2_undef_base_different_values_skip8(i32 %a0, i32 %b0) {
+; CHECK-LABEL: test_insert_v16i8_insert_2_undef_base_different_values_skip8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr w8, w0, #5
+; CHECK-NEXT:    dup.16b v0, w8
+; CHECK-NEXT:    lsr w8, w1, #5
+; CHECK-NEXT:    mov.b v0[2], w8
+; CHECK-NEXT:    mov.b v0[5], wzr
+; CHECK-NEXT:    mov.b v0[7], w8
+; CHECK-NEXT:    mov.b v0[9], wzr
+; CHECK-NEXT:    mov.b v0[12], w8
+; CHECK-NEXT:    mov.b v0[15], w8
+; CHECK-NEXT:    ret
+  %a1 = lshr exact i32 %a0, 5
+  %a = trunc i32 %a1 to i8
+  %b1 = lshr exact i32 %b0, 5
+  %b = trunc i32 %b1 to i8
+  %v.0 = insertelement <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>  , i8 %a, i32 0
+  %v.1 = insertelement <16 x i8> %v.0, i8 %a, i32 1
+  %v.2 = insertelement <16 x i8> %v.1, i8 %b, i32 2
+  %v.3 = insertelement <16 x i8> %v.2, i8 %a, i32 3
+  %v.4 = insertelement <16 x i8> %v.3, i8 %a, i32 4
+  %v.6 = insertelement <16 x i8> %v.4, i8 %a, i32 6
+  %v.7 = insertelement <16 x i8> %v.6, i8 %b, i32 7
   %v.10 = insertelement <16 x i8> %v.7, i8 %a, i32 10
   %v.11 = insertelement <16 x i8> %v.10, i8 %a, i32 11
   %v.12 = insertelement <16 x i8> %v.11, i8 %b, i32 12
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqadd.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqadd.ll
index df8864fe6ea3..ff1fedad4339 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqadd.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqadd.ll
@@ -1,129 +1,215 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
 
 define <8 x i8> @sqadd8b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqadd8b:
-;CHECK: sqadd.8b
-	%tmp1 = load <8 x i8>, ptr %A
-	%tmp2 = load <8 x i8>, ptr %B
-	%tmp3 = call <8 x i8> @llvm.aarch64.neon.sqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
-	ret <8 x i8> %tmp3
+; CHECK-LABEL: sqadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqadd.8b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @sqadd4h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqadd4h:
-;CHECK: sqadd.4h
-	%tmp1 = load <4 x i16>, ptr %A
-	%tmp2 = load <4 x i16>, ptr %B
-	%tmp3 = call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
-	ret <4 x i16> %tmp3
+; CHECK-LABEL: sqadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqadd.4h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i16>, ptr %A
+  %tmp2 = load <4 x i16>, ptr %B
+  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+  ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @sqadd2s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqadd2s:
-;CHECK: sqadd.2s
-	%tmp1 = load <2 x i32>, ptr %A
-	%tmp2 = load <2 x i32>, ptr %B
-	%tmp3 = call <2 x i32> @llvm.aarch64.neon.sqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
-	ret <2 x i32> %tmp3
+; CHECK-LABEL: sqadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqadd.2s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i32>, ptr %A
+  %tmp2 = load <2 x i32>, ptr %B
+  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+  ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @sqadd1d(ptr %A, ptr %B) nounwind {
+; CHECK-LABEL: sqadd1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqadd d0, d0, d1
+; CHECK-NEXT:    ret
+  %tmp1 = load <1 x i64>, ptr %A
+  %tmp2 = load <1 x i64>, ptr %B
+  %tmp3 = call <1 x i64> @llvm.aarch64.neon.sqadd.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+  ret <1 x i64> %tmp3
 }
 
+
 define <8 x i8> @uqadd8b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqadd8b:
-;CHECK: uqadd.8b
-	%tmp1 = load <8 x i8>, ptr %A
-	%tmp2 = load <8 x i8>, ptr %B
-	%tmp3 = call <8 x i8> @llvm.aarch64.neon.uqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
-	ret <8 x i8> %tmp3
+; CHECK-LABEL: uqadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqadd.8b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.uqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @uqadd4h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqadd4h:
-;CHECK: uqadd.4h
-	%tmp1 = load <4 x i16>, ptr %A
-	%tmp2 = load <4 x i16>, ptr %B
-	%tmp3 = call <4 x i16> @llvm.aarch64.neon.uqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
-	ret <4 x i16> %tmp3
+; CHECK-LABEL: uqadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqadd.4h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i16>, ptr %A
+  %tmp2 = load <4 x i16>, ptr %B
+  %tmp3 = call <4 x i16> @llvm.aarch64.neon.uqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+  ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @uqadd2s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqadd2s:
-;CHECK: uqadd.2s
-	%tmp1 = load <2 x i32>, ptr %A
-	%tmp2 = load <2 x i32>, ptr %B
-	%tmp3 = call <2 x i32> @llvm.aarch64.neon.uqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
-	ret <2 x i32> %tmp3
+; CHECK-LABEL: uqadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqadd.2s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i32>, ptr %A
+  %tmp2 = load <2 x i32>, ptr %B
+  %tmp3 = call <2 x i32> @llvm.aarch64.neon.uqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+  ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @uqadd1d(ptr %A, ptr %B) nounwind {
+; CHECK-LABEL: uqadd1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqadd d0, d0, d1
+; CHECK-NEXT:    ret
+  %tmp1 = load <1 x i64>, ptr %A
+  %tmp2 = load <1 x i64>, ptr %B
+  %tmp3 = call <1 x i64> @llvm.aarch64.neon.uqadd.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+  ret <1 x i64> %tmp3
 }
 
+
 define <16 x i8> @sqadd16b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqadd16b:
-;CHECK: sqadd.16b
-	%tmp1 = load <16 x i8>, ptr %A
-	%tmp2 = load <16 x i8>, ptr %B
-	%tmp3 = call <16 x i8> @llvm.aarch64.neon.sqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
-	ret <16 x i8> %tmp3
+; CHECK-LABEL: sqadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqadd.16b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <16 x i8>, ptr %A
+  %tmp2 = load <16 x i8>, ptr %B
+  %tmp3 = call <16 x i8> @llvm.aarch64.neon.sqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+  ret <16 x i8> %tmp3
 }
 
 define <8 x i16> @sqadd8h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqadd8h:
-;CHECK: sqadd.8h
-	%tmp1 = load <8 x i16>, ptr %A
-	%tmp2 = load <8 x i16>, ptr %B
-	%tmp3 = call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
-	ret <8 x i16> %tmp3
+; CHECK-LABEL: sqadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqadd.8h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i16>, ptr %A
+  %tmp2 = load <8 x i16>, ptr %B
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
 }
 
 define <4 x i32> @sqadd4s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqadd4s:
-;CHECK: sqadd.4s
-	%tmp1 = load <4 x i32>, ptr %A
-	%tmp2 = load <4 x i32>, ptr %B
-	%tmp3 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
-	ret <4 x i32> %tmp3
+; CHECK-LABEL: sqadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqadd.4s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i32>, ptr %A
+  %tmp2 = load <4 x i32>, ptr %B
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+  ret <4 x i32> %tmp3
 }
 
 define <2 x i64> @sqadd2d(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqadd2d:
-;CHECK: sqadd.2d
-	%tmp1 = load <2 x i64>, ptr %A
-	%tmp2 = load <2 x i64>, ptr %B
-	%tmp3 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
-	ret <2 x i64> %tmp3
+; CHECK-LABEL: sqadd2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqadd.2d v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i64>, ptr %A
+  %tmp2 = load <2 x i64>, ptr %B
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+  ret <2 x i64> %tmp3
 }
 
+
 define <16 x i8> @uqadd16b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqadd16b:
-;CHECK: uqadd.16b
-	%tmp1 = load <16 x i8>, ptr %A
-	%tmp2 = load <16 x i8>, ptr %B
-	%tmp3 = call <16 x i8> @llvm.aarch64.neon.uqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
-	ret <16 x i8> %tmp3
+; CHECK-LABEL: uqadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqadd.16b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <16 x i8>, ptr %A
+  %tmp2 = load <16 x i8>, ptr %B
+  %tmp3 = call <16 x i8> @llvm.aarch64.neon.uqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+  ret <16 x i8> %tmp3
 }
 
 define <8 x i16> @uqadd8h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqadd8h:
-;CHECK: uqadd.8h
-	%tmp1 = load <8 x i16>, ptr %A
-	%tmp2 = load <8 x i16>, ptr %B
-	%tmp3 = call <8 x i16> @llvm.aarch64.neon.uqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
-	ret <8 x i16> %tmp3
+; CHECK-LABEL: uqadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqadd.8h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i16>, ptr %A
+  %tmp2 = load <8 x i16>, ptr %B
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.uqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
 }
 
 define <4 x i32> @uqadd4s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqadd4s:
-;CHECK: uqadd.4s
-	%tmp1 = load <4 x i32>, ptr %A
-	%tmp2 = load <4 x i32>, ptr %B
-	%tmp3 = call <4 x i32> @llvm.aarch64.neon.uqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
-	ret <4 x i32> %tmp3
+; CHECK-LABEL: uqadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqadd.4s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i32>, ptr %A
+  %tmp2 = load <4 x i32>, ptr %B
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.uqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+  ret <4 x i32> %tmp3
 }
 
 define <2 x i64> @uqadd2d(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqadd2d:
-;CHECK: uqadd.2d
-	%tmp1 = load <2 x i64>, ptr %A
-	%tmp2 = load <2 x i64>, ptr %B
-	%tmp3 = call <2 x i64> @llvm.aarch64.neon.uqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
-	ret <2 x i64> %tmp3
+; CHECK-LABEL: uqadd2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqadd.2d v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i64>, ptr %A
+  %tmp2 = load <2 x i64>, ptr %B
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.uqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+  ret <2 x i64> %tmp3
 }
 
 declare <8 x i8>  @llvm.aarch64.neon.sqadd.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
@@ -147,78 +233,130 @@ declare <4 x i32> @llvm.aarch64.neon.uqadd.v4i32(<4 x i32>, <4 x i32>) nounwind
 declare <2 x i64> @llvm.aarch64.neon.uqadd.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
 
 define <8 x i8> @usqadd8b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: usqadd8b:
-;CHECK: usqadd.8b
-	%tmp1 = load <8 x i8>, ptr %A
-	%tmp2 = load <8 x i8>, ptr %B
-	%tmp3 = call <8 x i8> @llvm.aarch64.neon.usqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
-	ret <8 x i8> %tmp3
+; CHECK-LABEL: usqadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    usqadd.8b v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.usqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @usqadd4h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: usqadd4h:
-;CHECK: usqadd.4h
-	%tmp1 = load <4 x i16>, ptr %A
-	%tmp2 = load <4 x i16>, ptr %B
-	%tmp3 = call <4 x i16> @llvm.aarch64.neon.usqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
-	ret <4 x i16> %tmp3
+; CHECK-LABEL: usqadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    usqadd.4h v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i16>, ptr %A
+  %tmp2 = load <4 x i16>, ptr %B
+  %tmp3 = call <4 x i16> @llvm.aarch64.neon.usqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+  ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @usqadd2s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: usqadd2s:
-;CHECK: usqadd.2s
-	%tmp1 = load <2 x i32>, ptr %A
-	%tmp2 = load <2 x i32>, ptr %B
-	%tmp3 = call <2 x i32> @llvm.aarch64.neon.usqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
-	ret <2 x i32> %tmp3
+; CHECK-LABEL: usqadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    usqadd.2s v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i32>, ptr %A
+  %tmp2 = load <2 x i32>, ptr %B
+  %tmp3 = call <2 x i32> @llvm.aarch64.neon.usqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+  ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @usqadd1d(ptr %A, ptr %B) nounwind {
+; CHECK-LABEL: usqadd1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    usqadd d0, d1
+; CHECK-NEXT:    ret
+  %tmp1 = load <1 x i64>, ptr %A
+  %tmp2 = load <1 x i64>, ptr %B
+  %tmp3 = call <1 x i64> @llvm.aarch64.neon.usqadd.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+  ret <1 x i64> %tmp3
 }
 
+
 define <16 x i8> @usqadd16b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: usqadd16b:
-;CHECK: usqadd.16b
-	%tmp1 = load <16 x i8>, ptr %A
-	%tmp2 = load <16 x i8>, ptr %B
-	%tmp3 = call <16 x i8> @llvm.aarch64.neon.usqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
-	ret <16 x i8> %tmp3
+; CHECK-LABEL: usqadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    usqadd.16b v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <16 x i8>, ptr %A
+  %tmp2 = load <16 x i8>, ptr %B
+  %tmp3 = call <16 x i8> @llvm.aarch64.neon.usqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+  ret <16 x i8> %tmp3
 }
 
 define <8 x i16> @usqadd8h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: usqadd8h:
-;CHECK: usqadd.8h
-	%tmp1 = load <8 x i16>, ptr %A
-	%tmp2 = load <8 x i16>, ptr %B
-	%tmp3 = call <8 x i16> @llvm.aarch64.neon.usqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
-	ret <8 x i16> %tmp3
+; CHECK-LABEL: usqadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    usqadd.8h v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i16>, ptr %A
+  %tmp2 = load <8 x i16>, ptr %B
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.usqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
 }
 
 define <4 x i32> @usqadd4s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: usqadd4s:
-;CHECK: usqadd.4s
-	%tmp1 = load <4 x i32>, ptr %A
-	%tmp2 = load <4 x i32>, ptr %B
-	%tmp3 = call <4 x i32> @llvm.aarch64.neon.usqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
-	ret <4 x i32> %tmp3
+; CHECK-LABEL: usqadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    usqadd.4s v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i32>, ptr %A
+  %tmp2 = load <4 x i32>, ptr %B
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.usqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+  ret <4 x i32> %tmp3
 }
 
 define <2 x i64> @usqadd2d(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: usqadd2d:
-;CHECK: usqadd.2d
-	%tmp1 = load <2 x i64>, ptr %A
-	%tmp2 = load <2 x i64>, ptr %B
-	%tmp3 = call <2 x i64> @llvm.aarch64.neon.usqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
-	ret <2 x i64> %tmp3
+; CHECK-LABEL: usqadd2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    usqadd.2d v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i64>, ptr %A
+  %tmp2 = load <2 x i64>, ptr %B
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.usqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+  ret <2 x i64> %tmp3
 }
 
 define i64 @usqadd_d(i64 %l, i64 %r) nounwind {
 ; CHECK-LABEL: usqadd_d:
-; CHECK: usqadd {{d[0-9]+}}, {{d[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    fmov d1, x1
+; CHECK-NEXT:    usqadd d0, d1
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    ret
   %sum = call i64 @llvm.aarch64.neon.usqadd.i64(i64 %l, i64 %r)
   ret i64 %sum
 }
 
 define i32 @usqadd_s(i32 %l, i32 %r) nounwind {
 ; CHECK-LABEL: usqadd_s:
-; CHECK: usqadd {{s[0-9]+}}, {{s[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    fmov s1, w1
+; CHECK-NEXT:    usqadd s0, s1
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
   %sum = call i32 @llvm.aarch64.neon.usqadd.i32(i32 %l, i32 %r)
   ret i32 %sum
 }
@@ -235,86 +373,132 @@ declare <8 x i16> @llvm.aarch64.neon.usqadd.v8i16(<8 x i16>, <8 x i16>) nounwind
 declare <4 x i32> @llvm.aarch64.neon.usqadd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.usqadd.v2i64(<2 x i64>, <2 x i64>) nounwind readnone
 
+
 define <8 x i8> @suqadd8b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: suqadd8b:
-;CHECK: suqadd.8b
-	%tmp1 = load <8 x i8>, ptr %A
-	%tmp2 = load <8 x i8>, ptr %B
-	%tmp3 = call <8 x i8> @llvm.aarch64.neon.suqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
-	ret <8 x i8> %tmp3
+; CHECK-LABEL: suqadd8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    suqadd.8b v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.suqadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @suqadd4h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: suqadd4h:
-;CHECK: suqadd.4h
-	%tmp1 = load <4 x i16>, ptr %A
-	%tmp2 = load <4 x i16>, ptr %B
-	%tmp3 = call <4 x i16> @llvm.aarch64.neon.suqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
-	ret <4 x i16> %tmp3
+; CHECK-LABEL: suqadd4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    suqadd.4h v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i16>, ptr %A
+  %tmp2 = load <4 x i16>, ptr %B
+  %tmp3 = call <4 x i16> @llvm.aarch64.neon.suqadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+  ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @suqadd2s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: suqadd2s:
-;CHECK: suqadd.2s
-	%tmp1 = load <2 x i32>, ptr %A
-	%tmp2 = load <2 x i32>, ptr %B
-	%tmp3 = call <2 x i32> @llvm.aarch64.neon.suqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
-	ret <2 x i32> %tmp3
+; CHECK-LABEL: suqadd2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    suqadd.2s v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i32>, ptr %A
+  %tmp2 = load <2 x i32>, ptr %B
+  %tmp3 = call <2 x i32> @llvm.aarch64.neon.suqadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+  ret <2 x i32> %tmp3
+}
+
+define <1 x i64> @suqadd1d(ptr %A, ptr %B) nounwind {
+; CHECK-LABEL: suqadd1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    suqadd d0, d1
+; CHECK-NEXT:    ret
+  %tmp1 = load <1 x i64>, ptr %A
+  %tmp2 = load <1 x i64>, ptr %B
+  %tmp3 = call <1 x i64> @llvm.aarch64.neon.suqadd.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+  ret <1 x i64> %tmp3
 }
 
+
 define <16 x i8> @suqadd16b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: suqadd16b:
-;CHECK: suqadd.16b
-	%tmp1 = load <16 x i8>, ptr %A
-	%tmp2 = load <16 x i8>, ptr %B
-	%tmp3 = call <16 x i8> @llvm.aarch64.neon.suqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
-	ret <16 x i8> %tmp3
+; CHECK-LABEL: suqadd16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    suqadd.16b v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <16 x i8>, ptr %A
+  %tmp2 = load <16 x i8>, ptr %B
+  %tmp3 = call <16 x i8> @llvm.aarch64.neon.suqadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+  ret <16 x i8> %tmp3
 }
 
 define <8 x i16> @suqadd8h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: suqadd8h:
-;CHECK: suqadd.8h
-	%tmp1 = load <8 x i16>, ptr %A
-	%tmp2 = load <8 x i16>, ptr %B
-	%tmp3 = call <8 x i16> @llvm.aarch64.neon.suqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
-	ret <8 x i16> %tmp3
+; CHECK-LABEL: suqadd8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    suqadd.8h v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i16>, ptr %A
+  %tmp2 = load <8 x i16>, ptr %B
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.suqadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
 }
 
 define <4 x i32> @suqadd4s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: suqadd4s:
-;CHECK: suqadd.4s
-	%tmp1 = load <4 x i32>, ptr %A
-	%tmp2 = load <4 x i32>, ptr %B
-	%tmp3 = call <4 x i32> @llvm.aarch64.neon.suqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
-	ret <4 x i32> %tmp3
+; CHECK-LABEL: suqadd4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    suqadd.4s v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i32>, ptr %A
+  %tmp2 = load <4 x i32>, ptr %B
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.suqadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+  ret <4 x i32> %tmp3
 }
 
 define <2 x i64> @suqadd2d(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: suqadd2d:
-;CHECK: suqadd.2d
-	%tmp1 = load <2 x i64>, ptr %A
-	%tmp2 = load <2 x i64>, ptr %B
-	%tmp3 = call <2 x i64> @llvm.aarch64.neon.suqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
-	ret <2 x i64> %tmp3
-}
-
-define <1 x i64> @suqadd_1d(<1 x i64> %l, <1 x i64> %r) nounwind {
-; CHECK-LABEL: suqadd_1d:
-; CHECK: suqadd {{d[0-9]+}}, {{d[0-9]+}}
-  %sum = call <1 x i64> @llvm.aarch64.neon.suqadd.v1i64(<1 x i64> %l, <1 x i64> %r)
-  ret <1 x i64> %sum
+; CHECK-LABEL: suqadd2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    suqadd.2d v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i64>, ptr %A
+  %tmp2 = load <2 x i64>, ptr %B
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.suqadd.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+  ret <2 x i64> %tmp3
 }
 
 define i64 @suqadd_d(i64 %l, i64 %r) nounwind {
 ; CHECK-LABEL: suqadd_d:
-; CHECK: suqadd {{d[0-9]+}}, {{d[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    fmov d1, x1
+; CHECK-NEXT:    suqadd d0, d1
+; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    ret
   %sum = call i64 @llvm.aarch64.neon.suqadd.i64(i64 %l, i64 %r)
   ret i64 %sum
 }
 
 define i32 @suqadd_s(i32 %l, i32 %r) nounwind {
 ; CHECK-LABEL: suqadd_s:
-; CHECK: suqadd {{s[0-9]+}}, {{s[0-9]+}}
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    fmov s1, w1
+; CHECK-NEXT:    suqadd s0, s1
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
   %sum = call i32 @llvm.aarch64.neon.suqadd.i32(i32 %l, i32 %r)
   ret i32 %sum
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqsub.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqsub.ll
index dee21291fa14..b8168eba8ceb 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqsub.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64-vqsub.ll
@@ -1,129 +1,215 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
 
 define <8 x i8> @sqsub8b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqsub8b:
-;CHECK: sqsub.8b
-	%tmp1 = load <8 x i8>, ptr %A
-	%tmp2 = load <8 x i8>, ptr %B
-	%tmp3 = call <8 x i8> @llvm.aarch64.neon.sqsub.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
-	ret <8 x i8> %tmp3
+; CHECK-LABEL: sqsub8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqsub.8b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqsub.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @sqsub4h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqsub4h:
-;CHECK: sqsub.4h
-	%tmp1 = load <4 x i16>, ptr %A
-	%tmp2 = load <4 x i16>, ptr %B
-	%tmp3 = call <4 x i16> @llvm.aarch64.neon.sqsub.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
-	ret <4 x i16> %tmp3
+; CHECK-LABEL: sqsub4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqsub.4h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i16>, ptr %A
+  %tmp2 = load <4 x i16>, ptr %B
+  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqsub.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+  ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @sqsub2s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqsub2s:
-;CHECK: sqsub.2s
-	%tmp1 = load <2 x i32>, ptr %A
-	%tmp2 = load <2 x i32>, ptr %B
-	%tmp3 = call <2 x i32> @llvm.aarch64.neon.sqsub.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
-	ret <2 x i32> %tmp3
+; CHECK-LABEL: sqsub2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqsub.2s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i32>, ptr %A
+  %tmp2 = load <2 x i32>, ptr %B
+  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqsub.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+  ret <2 x i32> %tmp3
 }
 
+define <1 x i64> @sqsub1d(ptr %A, ptr %B) nounwind {
+; CHECK-LABEL: sqsub1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    sqsub d0, d0, d1
+; CHECK-NEXT:    ret
+  %tmp1 = load <1 x i64>, ptr %A
+  %tmp2 = load <1 x i64>, ptr %B
+  %tmp3 = call <1 x i64> @llvm.aarch64.neon.sqsub.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+  ret <1 x i64> %tmp3
+}
+
+
 define <8 x i8> @uqsub8b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqsub8b:
-;CHECK: uqsub.8b
-	%tmp1 = load <8 x i8>, ptr %A
-	%tmp2 = load <8 x i8>, ptr %B
-	%tmp3 = call <8 x i8> @llvm.aarch64.neon.uqsub.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
-	ret <8 x i8> %tmp3
+; CHECK-LABEL: uqsub8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqsub.8b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.uqsub.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @uqsub4h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqsub4h:
-;CHECK: uqsub.4h
-	%tmp1 = load <4 x i16>, ptr %A
-	%tmp2 = load <4 x i16>, ptr %B
-	%tmp3 = call <4 x i16> @llvm.aarch64.neon.uqsub.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
-	ret <4 x i16> %tmp3
+; CHECK-LABEL: uqsub4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqsub.4h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i16>, ptr %A
+  %tmp2 = load <4 x i16>, ptr %B
+  %tmp3 = call <4 x i16> @llvm.aarch64.neon.uqsub.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
+  ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @uqsub2s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqsub2s:
-;CHECK: uqsub.2s
-	%tmp1 = load <2 x i32>, ptr %A
-	%tmp2 = load <2 x i32>, ptr %B
-	%tmp3 = call <2 x i32> @llvm.aarch64.neon.uqsub.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
-	ret <2 x i32> %tmp3
+; CHECK-LABEL: uqsub2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqsub.2s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i32>, ptr %A
+  %tmp2 = load <2 x i32>, ptr %B
+  %tmp3 = call <2 x i32> @llvm.aarch64.neon.uqsub.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
+  ret <2 x i32> %tmp3
 }
 
+define <1 x i64> @uqsub1d(ptr %A, ptr %B) nounwind {
+; CHECK-LABEL: uqsub1d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    uqsub d0, d0, d1
+; CHECK-NEXT:    ret
+  %tmp1 = load <1 x i64>, ptr %A
+  %tmp2 = load <1 x i64>, ptr %B
+  %tmp3 = call <1 x i64> @llvm.aarch64.neon.uqsub.v1i64(<1 x i64> %tmp1, <1 x i64> %tmp2)
+  ret <1 x i64> %tmp3
+}
+
+
 define <16 x i8> @sqsub16b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqsub16b:
-;CHECK: sqsub.16b
-	%tmp1 = load <16 x i8>, ptr %A
-	%tmp2 = load <16 x i8>, ptr %B
-	%tmp3 = call <16 x i8> @llvm.aarch64.neon.sqsub.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
-	ret <16 x i8> %tmp3
+; CHECK-LABEL: sqsub16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqsub.16b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <16 x i8>, ptr %A
+  %tmp2 = load <16 x i8>, ptr %B
+  %tmp3 = call <16 x i8> @llvm.aarch64.neon.sqsub.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+  ret <16 x i8> %tmp3
 }
 
 define <8 x i16> @sqsub8h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqsub8h:
-;CHECK: sqsub.8h
-	%tmp1 = load <8 x i16>, ptr %A
-	%tmp2 = load <8 x i16>, ptr %B
-	%tmp3 = call <8 x i16> @llvm.aarch64.neon.sqsub.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
-	ret <8 x i16> %tmp3
+; CHECK-LABEL: sqsub8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqsub.8h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i16>, ptr %A
+  %tmp2 = load <8 x i16>, ptr %B
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqsub.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
 }
 
 define <4 x i32> @sqsub4s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqsub4s:
-;CHECK: sqsub.4s
-	%tmp1 = load <4 x i32>, ptr %A
-	%tmp2 = load <4 x i32>, ptr %B
-	%tmp3 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
-	ret <4 x i32> %tmp3
+; CHECK-LABEL: sqsub4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqsub.4s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i32>, ptr %A
+  %tmp2 = load <4 x i32>, ptr %B
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+  ret <4 x i32> %tmp3
 }
 
 define <2 x i64> @sqsub2d(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: sqsub2d:
-;CHECK: sqsub.2d
-	%tmp1 = load <2 x i64>, ptr %A
-	%tmp2 = load <2 x i64>, ptr %B
-	%tmp3 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
-	ret <2 x i64> %tmp3
+; CHECK-LABEL: sqsub2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sqsub.2d v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i64>, ptr %A
+  %tmp2 = load <2 x i64>, ptr %B
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+  ret <2 x i64> %tmp3
 }
 
+
 define <16 x i8> @uqsub16b(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqsub16b:
-;CHECK: uqsub.16b
-	%tmp1 = load <16 x i8>, ptr %A
-	%tmp2 = load <16 x i8>, ptr %B
-	%tmp3 = call <16 x i8> @llvm.aarch64.neon.uqsub.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
-	ret <16 x i8> %tmp3
+; CHECK-LABEL: uqsub16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqsub.16b v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <16 x i8>, ptr %A
+  %tmp2 = load <16 x i8>, ptr %B
+  %tmp3 = call <16 x i8> @llvm.aarch64.neon.uqsub.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
+  ret <16 x i8> %tmp3
 }
 
 define <8 x i16> @uqsub8h(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqsub8h:
-;CHECK: uqsub.8h
-	%tmp1 = load <8 x i16>, ptr %A
-	%tmp2 = load <8 x i16>, ptr %B
-	%tmp3 = call <8 x i16> @llvm.aarch64.neon.uqsub.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
-	ret <8 x i16> %tmp3
+; CHECK-LABEL: uqsub8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqsub.8h v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <8 x i16>, ptr %A
+  %tmp2 = load <8 x i16>, ptr %B
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.uqsub.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
 }
 
 define <4 x i32> @uqsub4s(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqsub4s:
-;CHECK: uqsub.4s
-	%tmp1 = load <4 x i32>, ptr %A
-	%tmp2 = load <4 x i32>, ptr %B
-	%tmp3 = call <4 x i32> @llvm.aarch64.neon.uqsub.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
-	ret <4 x i32> %tmp3
+; CHECK-LABEL: uqsub4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqsub.4s v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i32>, ptr %A
+  %tmp2 = load <4 x i32>, ptr %B
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.uqsub.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
+  ret <4 x i32> %tmp3
 }
 
 define <2 x i64> @uqsub2d(ptr %A, ptr %B) nounwind {
-;CHECK-LABEL: uqsub2d:
-;CHECK: uqsub.2d
-	%tmp1 = load <2 x i64>, ptr %A
-	%tmp2 = load <2 x i64>, ptr %B
-	%tmp3 = call <2 x i64> @llvm.aarch64.neon.uqsub.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
-	ret <2 x i64> %tmp3
+; CHECK-LABEL: uqsub2d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uqsub.2d v0, v0, v1
+; CHECK-NEXT:    ret
+  %tmp1 = load <2 x i64>, ptr %A
+  %tmp2 = load <2 x i64>, ptr %B
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.uqsub.v2i64(<2 x i64> %tmp1, <2 x i64> %tmp2)
+  ret <2 x i64> %tmp3
 }
 
 declare <8 x i8>  @llvm.aarch64.neon.sqsub.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-alias.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-alias.ll
index 03cc87313694..18023a95a5d2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-alias.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-alias.ll
@@ -13,30 +13,30 @@ define dso_local void @patchable_func() hybrid_patchable {
 @patchable_alias = alias void (), ptr @patchable_func
 
 ; CHECK:              .weak_anti_dep  func_alias
-; CHECK-NEXT: .set func_alias, "#func_alias"
+; CHECK-NEXT: func_alias = "#func_alias"
 ; CHECK-NEXT:         .weak_anti_dep  func_alias2
-; CHECK-NEXT: .set func_alias2, "#func_alias2"
+; CHECK-NEXT: func_alias2 = "#func_alias2"
 ; CHECK-NEXT:         .weak_anti_dep  func
-; CHECK-NEXT: .set func, "#func"
+; CHECK-NEXT: func = "#func"
 ; CHECK:              .weak_anti_dep  patchable_alias
-; CHECK-NEXT: .set patchable_alias, "#patchable_alias"
+; CHECK-NEXT: patchable_alias = "#patchable_alias"
 
 ; CHECK:              .globl  "#func_alias"
 ; CHECK-NEXT:         .def    "#func_alias";
 ; CHECK-NEXT:         .scl    2;
 ; CHECK-NEXT:         .type   32;
 ; CHECK-NEXT:         .endef
-; CHECK-NEXT: .set "#func_alias", "#func"
+; CHECK-NEXT: "#func_alias" = "#func"
 ; CHECK-NEXT:         .globl  "#func_alias2"
 ; CHECK-NEXT:         .def    "#func_alias2";
 ; CHECK-NEXT:         .scl    2;
 ; CHECK-NEXT:         .type   32;
 ; CHECK-NEXT:         .endef
-; CHECK-NEXT: .set "#func_alias2", "#func_alias"
+; CHECK-NEXT: "#func_alias2" = "#func_alias"
 
 ; CHECK:              .globl  "#patchable_alias"
 ; CHECK-NEXT:         .def    "#patchable_alias";
 ; CHECK-NEXT:         .scl    2;
 ; CHECK-NEXT:         .type   32;
 ; CHECK-NEXT:         .endef
-; CHECK-NEXT: .set "#patchable_alias", "#patchable_func"
+; CHECK-NEXT: "#patchable_alias" = "#patchable_func"
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
new file mode 100644
index 000000000000..92b95a90d89a
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
@@ -0,0 +1,45 @@
+; RUN: llc -mtriple=arm64ec-pc-windows-msvc < %s | FileCheck %s
+
+define void @f1(ptr %p, i64 %n) {
+; CHECK-LABEL: "#f1":
+; CHECK: bl "#memset"
+  call void @llvm.memset.p0.i64(ptr %p, i8 0, i64 %n, i1 false)
+  ret void
+}
+
+define void @f2(ptr %p1, ptr %p2, i64 %n) {
+; CHECK-LABEL: "#f2":
+; CHECK: bl "#memcpy"
+  call void @llvm.memcpy.p0.i64(ptr %p1, ptr %p2, i64 %n, i1 false)
+  ret void
+}
+
+define double @f3(double %x, double %y) {
+; CHECK-LABEL: "#f3":
+; CHECK: b "#fmod"
+  %r = frem double %x, %y
+  ret double %r
+}
+
+define i128 @f4(i128 %x, i128 %y) {
+; CHECK-LABEL: "#f4":
+; CHECK: bl "#__divti3"
+  %r = sdiv i128 %x, %y
+  ret i128 %r
+}
+
+; FIXME: This is wrong; should be "#__aarch64_cas1_relax"
+define i8 @f5(i8 %expected, i8 %new, ptr %ptr) "target-features"="+outline-atomics" {
+; CHECK-LABEL: "#f5":
+; CHECK: bl __aarch64_cas1_relax
+    %pair = cmpxchg ptr %ptr, i8 %expected, i8 %new monotonic monotonic, align 1
+   %r = extractvalue { i8, i1 } %pair, 0
+    ret i8 %r
+}
+
+define float @f6(float %val, i32 %a) {
+; CHECK-LABEL: "#f6":
+; CHECK: bl "#ldexp"
+  %call = tail call fast float @llvm.ldexp.f32(float %val, i32 %a)
+  ret float %call
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
index f964484c0c2d..7c77832a9d9a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
@@ -76,7 +76,7 @@ define dso_local void @caller() nounwind {
 ; CHECK-NEXT:      .p2align        2
 ; CHECK-NEXT:  "#caller":                              // @"#caller"
 ; CHECK-NEXT:      .weak_anti_dep  caller
-; CHECK-NEXT:  .set caller, "#caller"{{$}}
+; CHECK-NEXT:  caller = "#caller"{{$}}
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:      str     x30, [sp, #-16]!                // 8-byte Folded Spill
 ; CHECK-NEXT:      bl      "#func"
@@ -253,13 +253,13 @@ define dso_local void @caller() nounwind {
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
 ; CHECK-NEXT:      .weak  func
-; CHECK-NEXT:  .set func, "EXP+#func"{{$}}
+; CHECK-NEXT:  func = "EXP+#func"{{$}}
 ; CHECK-NEXT:      .weak  "#func"
 ; CHECK-NEXT:      .def    "#func";
 ; CHECK-NEXT:      .scl    2;
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
-; CHECK-NEXT:  .set "#func", "#func$hybpatch_thunk"{{$}}
+; CHECK-NEXT:  "#func" = "#func$hybpatch_thunk"{{$}}
 ; CHECK-NEXT:      .def    "EXP+#has_varargs";
 ; CHECK-NEXT:      .scl    2;
 ; CHECK-NEXT:      .type   32;
@@ -269,13 +269,13 @@ define dso_local void @caller() nounwind {
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
 ; CHECK-NEXT:      .weak   has_varargs
-; CHECK-NEXT:  .set has_varargs, "EXP+#has_varargs"
+; CHECK-NEXT:  has_varargs = "EXP+#has_varargs"
 ; CHECK-NEXT:      .weak   "#has_varargs"
 ; CHECK-NEXT:      .def    "#has_varargs";
 ; CHECK-NEXT:      .scl    2;
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
-; CHECK-NEXT:  .set "#has_varargs", "#has_varargs$hybpatch_thunk"
+; CHECK-NEXT:  "#has_varargs" = "#has_varargs$hybpatch_thunk"
 ; CHECK-NEXT:      .def    "EXP+#has_sret";
 ; CHECK-NEXT:      .scl    2;
 ; CHECK-NEXT:      .type   32;
@@ -285,13 +285,13 @@ define dso_local void @caller() nounwind {
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
 ; CHECK-NEXT:      .weak   has_sret
-; CHECK-NEXT:  .set has_sret, "EXP+#has_sret"
+; CHECK-NEXT:  has_sret = "EXP+#has_sret"
 ; CHECK-NEXT:      .weak   "#has_sret"
 ; CHECK-NEXT:      .def    "#has_sret";
 ; CHECK-NEXT:      .scl    2;
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
-; CHECK-NEXT:  .set "#has_sret", "#has_sret$hybpatch_thunk"
+; CHECK-NEXT:  "#has_sret" = "#has_sret$hybpatch_thunk"
 ; CHECK-NEXT:      .def    "EXP+#exp";
 ; CHECK-NEXT:      .scl    2;
 ; CHECK-NEXT:      .type   32;
@@ -301,13 +301,13 @@ define dso_local void @caller() nounwind {
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
 ; CHECK-NEXT:      .weak   exp
-; CHECK-NEXT:  .set exp, "EXP+#exp"
+; CHECK-NEXT:  exp = "EXP+#exp"
 ; CHECK-NEXT:      .weak   "#exp"
 ; CHECK-NEXT:      .def    "#exp";
 ; CHECK-NEXT:      .scl    2;
 ; CHECK-NEXT:      .type   32;
 ; CHECK-NEXT:      .endef
-; CHECK-NEXT:  .set "#exp", "#exp$hybpatch_thunk"
+; CHECK-NEXT:  "#exp" = "#exp$hybpatch_thunk"
 
 ; SYM:      [53](sec 15)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #func$hybpatch_thunk
 ; SYM:      [58](sec 16)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #has_varargs$hybpatch_thunk
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
index b79dd7d61dd6..b44f39ad7b73 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
@@ -10,12 +10,12 @@ define void @caller() nounwind {
 }
 
 ; CHECK:      .weak_anti_dep  caller
-; CHECK-NEXT: .set caller, "#caller"{{$}}
+; CHECK-NEXT: caller = "#caller"{{$}}
 
 ; CHECK:      .weak_anti_dep  func
-; CHECK-NEXT: .set func, "#func"{{$}}
+; CHECK-NEXT: func = "#func"{{$}}
 ; CHECK-NEXT: .weak_anti_dep  "#func"
-; CHECK-NEXT: .set "#func", "#func$exit_thunk"{{$}}
+; CHECK-NEXT: "#func" = "#func$exit_thunk"{{$}}
 
 ; SYM:       [ 8](sec  4)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #caller
 ; SYM:       [21](sec  7)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #func$exit_thunk
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-varargs.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-varargs.ll
index 5fab5738078d..389969bebaea 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-varargs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/arm64ec-varargs.ll
@@ -45,9 +45,9 @@ define void @varargs_caller() nounwind {
 ; CHECK-NEXT:    stp x9, x8, [sp]
 ; CHECK-NEXT:    str xzr, [sp, #16]
 ; CHECK-NEXT:    .weak_anti_dep varargs_callee
-; CHECK-NEXT:  .set varargs_callee, "#varargs_callee"
+; CHECK-NEXT:  varargs_callee = "#varargs_callee"
 ; CHECK-NEXT:    .weak_anti_dep "#varargs_callee"
-; CHECK-NEXT:  .set "#varargs_callee", varargs_callee
+; CHECK-NEXT:  "#varargs_callee" = varargs_callee
 ; CHECK-NEXT:    bl "#varargs_callee"
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
@@ -86,9 +86,9 @@ define void @varargs_many_argscalleer() nounwind {
 ; CHECK-NEXT:    stp x9, x8, [sp]
 ; CHECK-NEXT:    stp q0, q0, [sp, #16]
 ; CHECK-NEXT:    .weak_anti_dep varargs_many_argscallee
-; CHECK-NEXT:  .set varargs_many_argscallee, "#varargs_many_argscallee"
+; CHECK-NEXT:  varargs_many_argscallee = "#varargs_many_argscallee"
 ; CHECK-NEXT:    .weak_anti_dep "#varargs_many_argscallee"
-; CHECK-NEXT:  .set "#varargs_many_argscallee", varargs_many_argscallee
+; CHECK-NEXT:  "#varargs_many_argscallee" = varargs_many_argscallee
 ; CHECK-NEXT:    bl "#varargs_many_argscallee"
 ; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #64
@@ -116,9 +116,9 @@ define void @varargs_caller_tail() nounwind {
 ; CHECK-NEXT:    stp x9, x8, [sp]
 ; CHECK-NEXT:    str xzr, [sp, #16]
 ; CHECK-NEXT:    .weak_anti_dep varargs_callee
-; CHECK-NEXT:  .set varargs_callee, "#varargs_callee"
+; CHECK-NEXT:  varargs_callee = "#varargs_callee"
 ; CHECK-NEXT:    .weak_anti_dep "#varargs_callee"
-; CHECK-NEXT:  .set "#varargs_callee", varargs_callee
+; CHECK-NEXT:  "#varargs_callee" = varargs_callee
 ; CHECK-NEXT:    bl "#varargs_callee"
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add x4, sp, #48
@@ -129,9 +129,9 @@ define void @varargs_caller_tail() nounwind {
 ; CHECK-NEXT:    mov x5, xzr
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    .weak_anti_dep varargs_callee
-; CHECK-NEXT:  .set varargs_callee, "#varargs_callee"
+; CHECK-NEXT:  varargs_callee = "#varargs_callee"
 ; CHECK-NEXT:    .weak_anti_dep "#varargs_callee"
-; CHECK-NEXT:  .set "#varargs_callee", varargs_callee
+; CHECK-NEXT:  "#varargs_callee" = varargs_callee
 ; CHECK-NEXT:    b "#varargs_callee"
   call void (double, ...) @varargs_callee(double 1.0, i32 2, double 3.0, i32 4, double 5.0, <2 x double> <double 0.0, double 0.0>)
   tail call void (double, ...) @varargs_callee(double 1.0, i32 4, i32 3, i32 2)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll
index 37a7782caeed..71e0250b3697 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/atomicrmw-O0.ll
@@ -32,9 +32,7 @@ define i8 @test_rmw_add_8(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB0_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB0_1 Depth=1
 ; NOLSE-NEXT:    subs w9, w8, w9, uxtb
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB0_1
 ; NOLSE-NEXT:    b .LBB0_5
@@ -45,7 +43,7 @@ define i8 @test_rmw_add_8(ptr %dst)   {
 ;
 ; LSE-LABEL: test_rmw_add_8:
 ; LSE:       // %bb.0: // %entry
-; LSE-NEXT:    mov w8, #1
+; LSE-NEXT:    mov w8, #1 // =0x1
 ; LSE-NEXT:    ldaddalb w8, w0, [x0]
 ; LSE-NEXT:    ret
 entry:
@@ -81,9 +79,7 @@ define i16 @test_rmw_add_16(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB1_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB1_1 Depth=1
 ; NOLSE-NEXT:    subs w9, w8, w9, uxth
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB1_1
 ; NOLSE-NEXT:    b .LBB1_5
@@ -94,7 +90,7 @@ define i16 @test_rmw_add_16(ptr %dst)   {
 ;
 ; LSE-LABEL: test_rmw_add_16:
 ; LSE:       // %bb.0: // %entry
-; LSE-NEXT:    mov w8, #1
+; LSE-NEXT:    mov w8, #1 // =0x1
 ; LSE-NEXT:    ldaddalh w8, w0, [x0]
 ; LSE-NEXT:    ret
 entry:
@@ -130,9 +126,7 @@ define i32 @test_rmw_add_32(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB2_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB2_1 Depth=1
 ; NOLSE-NEXT:    subs w9, w8, w9
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB2_1
 ; NOLSE-NEXT:    b .LBB2_5
@@ -143,7 +137,7 @@ define i32 @test_rmw_add_32(ptr %dst)   {
 ;
 ; LSE-LABEL: test_rmw_add_32:
 ; LSE:       // %bb.0: // %entry
-; LSE-NEXT:    mov w8, #1
+; LSE-NEXT:    mov w8, #1 // =0x1
 ; LSE-NEXT:    ldaddal w8, w0, [x0]
 ; LSE-NEXT:    ret
 entry:
@@ -179,9 +173,7 @@ define i64 @test_rmw_add_64(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB3_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB3_1 Depth=1
 ; NOLSE-NEXT:    subs x9, x8, x9
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str x8, [sp, #8] // 8-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str x8, [sp, #24] // 8-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB3_1
 ; NOLSE-NEXT:    b .LBB3_5
@@ -192,7 +184,7 @@ define i64 @test_rmw_add_64(ptr %dst)   {
 ;
 ; LSE-LABEL: test_rmw_add_64:
 ; LSE:       // %bb.0: // %entry
-; LSE-NEXT:    mov w8, #1
+; LSE-NEXT:    mov w8, #1 // =0x1
 ; LSE-NEXT:    // kill: def $x8 killed $w8
 ; LSE-NEXT:    ldaddal x8, x0, [x0]
 ; LSE-NEXT:    ret
@@ -326,9 +318,7 @@ define i8 @test_rmw_nand_8(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB5_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB5_1 Depth=1
 ; NOLSE-NEXT:    subs w9, w8, w9, uxtb
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB5_1
 ; NOLSE-NEXT:    b .LBB5_5
@@ -354,9 +344,7 @@ define i8 @test_rmw_nand_8(ptr %dst)   {
 ; LSE-NEXT:    mov w8, w9
 ; LSE-NEXT:    casalb w8, w10, [x11]
 ; LSE-NEXT:    subs w9, w8, w9, uxtb
-; LSE-NEXT:    cset w9, eq
 ; LSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB5_1
 ; LSE-NEXT:    b .LBB5_2
@@ -398,9 +386,7 @@ define i16 @test_rmw_nand_16(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB6_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB6_1 Depth=1
 ; NOLSE-NEXT:    subs w9, w8, w9, uxth
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB6_1
 ; NOLSE-NEXT:    b .LBB6_5
@@ -426,9 +412,7 @@ define i16 @test_rmw_nand_16(ptr %dst)   {
 ; LSE-NEXT:    mov w8, w9
 ; LSE-NEXT:    casalh w8, w10, [x11]
 ; LSE-NEXT:    subs w9, w8, w9, uxth
-; LSE-NEXT:    cset w9, eq
 ; LSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB6_1
 ; LSE-NEXT:    b .LBB6_2
@@ -470,9 +454,7 @@ define i32 @test_rmw_nand_32(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB7_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB7_1 Depth=1
 ; NOLSE-NEXT:    subs w9, w8, w9
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB7_1
 ; NOLSE-NEXT:    b .LBB7_5
@@ -498,9 +480,7 @@ define i32 @test_rmw_nand_32(ptr %dst)   {
 ; LSE-NEXT:    mov w8, w9
 ; LSE-NEXT:    casal w8, w10, [x11]
 ; LSE-NEXT:    subs w9, w8, w9
-; LSE-NEXT:    cset w9, eq
 ; LSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
-; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB7_1
 ; LSE-NEXT:    b .LBB7_2
@@ -545,9 +525,7 @@ define i64 @test_rmw_nand_64(ptr %dst)   {
 ; NOLSE-NEXT:  .LBB8_4: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB8_1 Depth=1
 ; NOLSE-NEXT:    subs x9, x8, x9
-; NOLSE-NEXT:    cset w9, eq
 ; NOLSE-NEXT:    str x8, [sp, #8] // 8-byte Folded Spill
-; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str x8, [sp, #24] // 8-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB8_1
 ; NOLSE-NEXT:    b .LBB8_5
@@ -576,9 +554,7 @@ define i64 @test_rmw_nand_64(ptr %dst)   {
 ; LSE-NEXT:    mov x8, x9
 ; LSE-NEXT:    casal x8, x10, [x11]
 ; LSE-NEXT:    subs x9, x8, x9
-; LSE-NEXT:    cset w9, eq
 ; LSE-NEXT:    str x8, [sp, #8] // 8-byte Folded Spill
-; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str x8, [sp, #24] // 8-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB8_1
 ; LSE-NEXT:    b .LBB8_2
@@ -613,7 +589,7 @@ define i128 @test_rmw_nand_128(ptr %dst)   {
 ; NOLSE-NEXT:    // implicit-def: $x8
 ; NOLSE-NEXT:    mov w8, w10
 ; NOLSE-NEXT:    orr x14, x8, #0xfffffffffffffffe
-; NOLSE-NEXT:    mov x15, #-1
+; NOLSE-NEXT:    mov x15, #-1 // =0xffffffffffffffff
 ; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
 ; NOLSE-NEXT:    // Parent Loop BB9_1 Depth=1
 ; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
@@ -672,7 +648,7 @@ define i128 @test_rmw_nand_128(ptr %dst)   {
 ; LSE-NEXT:    // implicit-def: $x9
 ; LSE-NEXT:    mov w9, w12
 ; LSE-NEXT:    orr x2, x9, #0xfffffffffffffffe
-; LSE-NEXT:    mov x9, #-1
+; LSE-NEXT:    mov x9, #-1 // =0xffffffffffffffff
 ; LSE-NEXT:    // kill: def $x2 killed $x2 def $x2_x3
 ; LSE-NEXT:    mov x3, x9
 ; LSE-NEXT:    caspal x0, x1, x2, x3, [x8]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/bcmp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/bcmp.ll
index fee52ead9896..e70ddc3415ca 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/bcmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/bcmp.ll
@@ -494,13 +494,14 @@ define i1 @bcmp_i128(i128 %a0, i128 %b0, i128 %a1, i128 %b1, i128 %a2, i128 %b2)
 ; CHECK-LABEL: bcmp_i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmp x2, x0
-; CHECK-NEXT:    ldp x8, x10, [sp]
+; CHECK-NEXT:    ldp x10, x8, [sp, #8]
 ; CHECK-NEXT:    ccmp x3, x1, #0, eq
-; CHECK-NEXT:    ldp x9, x11, [sp, #16]
+; CHECK-NEXT:    ldr x9, [sp]
+; CHECK-NEXT:    ldr x11, [sp, #24]
 ; CHECK-NEXT:    ccmp x6, x4, #0, eq
 ; CHECK-NEXT:    ccmp x7, x5, #0, eq
 ; CHECK-NEXT:    cset w12, ne
-; CHECK-NEXT:    cmp x9, x8
+; CHECK-NEXT:    cmp x8, x9
 ; CHECK-NEXT:    ccmp x11, x10, #0, eq
 ; CHECK-NEXT:    csinc w0, w12, wzr, eq
 ; CHECK-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast-extend.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast-extend.ll
index 3133d0efb4b9..f62303ecea66 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast-extend.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast-extend.ll
@@ -84,20 +84,20 @@ define <4 x i64> @z_i32_v4i64(i32 %x) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    mov b1, v0.b[2]
-; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    mov b2, v0.b[1]
 ; CHECK-GI-NEXT:    mov b3, v0.b[3]
-; CHECK-GI-NEXT:    ubfx x8, x8, #0, #8
+; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    fmov w8, s2
-; CHECK-GI-NEXT:    ubfx x9, x9, #0, #8
 ; CHECK-GI-NEXT:    ubfx x8, x8, #0, #8
-; CHECK-GI-NEXT:    mov v1.d[0], x9
-; CHECK-GI-NEXT:    fmov w9, s3
-; CHECK-GI-NEXT:    mov v0.d[1], x8
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov d0, x8
 ; CHECK-GI-NEXT:    ubfx x9, x9, #0, #8
-; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    ubfx x10, x10, #0, #8
+; CHECK-GI-NEXT:    ubfx x11, x11, #0, #8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v0.d[1], x10
+; CHECK-GI-NEXT:    mov v1.d[1], x11
 ; CHECK-GI-NEXT:    ret
   %b = bitcast i32 %x to <4 x i8>
   %e = zext <4 x i8> %b to <4 x i64>
@@ -188,20 +188,20 @@ define <4 x i64> @s_i32_v4i64(i32 %x) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    mov b1, v0.b[2]
-; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    mov b2, v0.b[1]
 ; CHECK-GI-NEXT:    mov b3, v0.b[3]
-; CHECK-GI-NEXT:    sxtb x8, w8
+; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    fmov w8, s2
-; CHECK-GI-NEXT:    sxtb x9, w9
 ; CHECK-GI-NEXT:    sxtb x8, w8
-; CHECK-GI-NEXT:    mov v1.d[0], x9
-; CHECK-GI-NEXT:    fmov w9, s3
-; CHECK-GI-NEXT:    mov v0.d[1], x8
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov d0, x8
 ; CHECK-GI-NEXT:    sxtb x9, w9
-; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    sxtb x10, w10
+; CHECK-GI-NEXT:    sxtb x11, w11
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v0.d[1], x10
+; CHECK-GI-NEXT:    mov v1.d[1], x11
 ; CHECK-GI-NEXT:    ret
   %b = bitcast i32 %x to <4 x i8>
   %e = sext <4 x i8> %b to <4 x i64>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast.ll
index d54cc4adb81b..d2f72ecacc86 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/bitcast.ll
@@ -13,9 +13,9 @@ define <4 x i16> @foo1(<2 x i32> %a) {
 ; CHECK-GI-LABEL: foo1:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #58712 // =0xe558
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    zip1 v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    rev32 v0.4h, v0.4h
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[1]
 ; CHECK-GI-NEXT:    ret
   %1 = shufflevector <2 x i32> <i32 58712, i32 undef>, <2 x i32> %a, <2 x i32> <i32 0, i32 2>
 ; Can't optimize the following bitcast to scalar_to_vector.
@@ -33,9 +33,9 @@ define <4 x i16> @foo2(<2 x i32> %a) {
 ; CHECK-GI-LABEL: foo2:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #712 // =0x2c8
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    zip1 v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    rev32 v0.4h, v0.4h
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[1]
 ; CHECK-GI-NEXT:    ret
   %1 = shufflevector <2 x i32> <i32 712, i32 undef>, <2 x i32> %a, <2 x i32> <i32 0, i32 2>
 ; Can't optimize the following bitcast to scalar_to_vector.
@@ -133,7 +133,6 @@ define <2 x i16> @bitcast_i32_v2i16(i32 %a, i32 %b){
 ; CHECK-GI-NEXT:    add w8, w0, w1
 ; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-NEXT:    mov v0.s[0], w8
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -449,8 +448,6 @@ define <2 x i16> @bitcast_v4i8_v2i16(<4 x i8> %a, <4 x i8> %b){
 ; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
 ; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-GI-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -503,12 +500,10 @@ define <4 x i64> @bitcast_v8i32_v4i64(<8 x i32> %a, <8 x i32> %b){
 ;
 ; CHECK-GI-LABEL: bitcast_v8i32_v4i64:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    add v2.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT:    add v3.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT:    mov x8, v2.d[1]
-; CHECK-GI-NEXT:    mov x9, v3.d[1]
-; CHECK-GI-NEXT:    mov v0.d[0], v2.d[0]
-; CHECK-GI-NEXT:    mov v1.d[0], v3.d[0]
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    ret
@@ -564,12 +559,10 @@ define <4 x i64> @bitcast_v16i16_v4i64(<16 x i16> %a, <16 x i16> %b){
 ;
 ; CHECK-GI-LABEL: bitcast_v16i16_v4i64:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    add v2.8h, v0.8h, v2.8h
-; CHECK-GI-NEXT:    add v3.8h, v1.8h, v3.8h
-; CHECK-GI-NEXT:    mov x8, v2.d[1]
-; CHECK-GI-NEXT:    mov x9, v3.d[1]
-; CHECK-GI-NEXT:    mov v0.d[0], v2.d[0]
-; CHECK-GI-NEXT:    mov v1.d[0], v3.d[0]
+; CHECK-GI-NEXT:    add v0.8h, v0.8h, v2.8h
+; CHECK-GI-NEXT:    add v1.8h, v1.8h, v3.8h
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    ret
@@ -606,18 +599,14 @@ define <8 x i64> @bitcast_v16i32_v8i64(<16 x i32> %a, <16 x i32> %b){
 ;
 ; CHECK-GI-LABEL: bitcast_v16i32_v8i64:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    add v4.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT:    add v5.4s, v1.4s, v5.4s
-; CHECK-GI-NEXT:    add v6.4s, v2.4s, v6.4s
-; CHECK-GI-NEXT:    add v7.4s, v3.4s, v7.4s
-; CHECK-GI-NEXT:    mov x8, v4.d[1]
-; CHECK-GI-NEXT:    mov x9, v5.d[1]
-; CHECK-GI-NEXT:    mov x10, v6.d[1]
-; CHECK-GI-NEXT:    mov x11, v7.d[1]
-; CHECK-GI-NEXT:    mov v0.d[0], v4.d[0]
-; CHECK-GI-NEXT:    mov v1.d[0], v5.d[0]
-; CHECK-GI-NEXT:    mov v2.d[0], v6.d[0]
-; CHECK-GI-NEXT:    mov v3.d[0], v7.d[0]
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v5.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v6.4s
+; CHECK-GI-NEXT:    add v3.4s, v3.4s, v7.4s
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mov x10, v2.d[1]
+; CHECK-GI-NEXT:    mov x11, v3.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    mov v2.d[1], x10
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/bitreverse.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/bitreverse.ll
index c1ca39d87105..9e2228aa47c6 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/bitreverse.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/bitreverse.ll
@@ -72,6 +72,19 @@ define i64 @g_64(i64 %a) {
   ret i64 %b
 }
 
+declare i128 @llvm.bitreverse.i128(i128) readnone
+
+define i128 @g_128(i128 %a) {
+; CHECK-LABEL: g_128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rbit x8, x1
+; CHECK-NEXT:    rbit x1, x0
+; CHECK-NEXT:    mov x0, x8
+; CHECK-NEXT:    ret
+  %b = call i128 @llvm.bitreverse.i128(i128 %a)
+  ret i128 %b
+}
+
 declare <16 x i3> @llvm.bitreverse.v16i3(<16 x i3>) readnone
 
 define <16 x i3> @g_vec_16x3(<16 x i3> %a) {
@@ -282,3 +295,19 @@ define <4 x i64> @g_vec_4x64(<4 x i64> %a) {
   %b = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a)
   ret <4 x i64> %b
 }
+
+declare <2 x i128> @llvm.bitreverse.v2i128(<2 x i128>) readnone
+
+define <2 x i128> @g_vec_2x128(<2 x i128> %a) {
+; CHECK-LABEL: g_vec_2x128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rbit x8, x1
+; CHECK-NEXT:    rbit x9, x3
+; CHECK-NEXT:    rbit x1, x0
+; CHECK-NEXT:    rbit x3, x2
+; CHECK-NEXT:    mov x0, x8
+; CHECK-NEXT:    mov x2, x9
+; CHECK-NEXT:    ret
+  %b = call <2 x i128> @llvm.bitreverse.v2i128(<2 x i128> %a)
+  ret <2 x i128> %b
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/bswap.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/bswap.ll
index e86f55d63f75..898958fb4993 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/bswap.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/bswap.ll
@@ -209,8 +209,7 @@ define <1 x i32> @bswap_v1i32(<1 x i32> %a){
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    rev w8, w8
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
 entry:
   %res = call <1 x i32> @llvm.bswap.v1i32(<1 x i32> %a)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/build-vector-dup-simd.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/build-vector-dup-simd.ll
new file mode 100644
index 000000000000..ac0b8e89519d
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/build-vector-dup-simd.ll
@@ -0,0 +1,378 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-NOFULLFP16
+; RUN: llc < %s -mtriple=aarch64 --enable-no-nans-fp-math | FileCheck %s --check-prefixes=CHECK,CHECK-NONANS
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16  | FileCheck %s --check-prefixes=CHECK,CHECK-FULLFP16
+
+define <1 x float> @dup_v1i32_oeq(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_oeq:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmeq s0, s0, s1
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp oeq float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_ogt(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_ogt:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmgt s0, s0, s1
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp ogt float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_oge(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_oge:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmge s0, s0, s1
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp oge float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_olt(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_olt:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmgt s0, s1, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp olt float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_ole(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_ole:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmge s0, s1, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp ole float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_one(float %a, float %b) {
+; CHECK-NOFULLFP16-LABEL: dup_v1i32_one:
+; CHECK-NOFULLFP16:       // %bb.0: // %entry
+; CHECK-NOFULLFP16-NEXT:    fcmgt s2, s0, s1
+; CHECK-NOFULLFP16-NEXT:    fcmgt s0, s1, s0
+; CHECK-NOFULLFP16-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NOFULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFULLFP16-NEXT:    ret
+;
+; CHECK-NONANS-LABEL: dup_v1i32_one:
+; CHECK-NONANS:       // %bb.0: // %entry
+; CHECK-NONANS-NEXT:    fcmeq s0, s0, s1
+; CHECK-NONANS-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NONANS-NEXT:    ret
+;
+; CHECK-FULLFP16-LABEL: dup_v1i32_one:
+; CHECK-FULLFP16:       // %bb.0: // %entry
+; CHECK-FULLFP16-NEXT:    fcmgt s2, s0, s1
+; CHECK-FULLFP16-NEXT:    fcmgt s0, s1, s0
+; CHECK-FULLFP16-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-FULLFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FULLFP16-NEXT:    ret
+entry:
+  %0 = fcmp one float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_ord(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_ord:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmge s2, s0, s1
+; CHECK-NEXT:    fcmgt s0, s1, s0
+; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp ord float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_ueq(float %a, float %b) {
+; CHECK-NOFULLFP16-LABEL: dup_v1i32_ueq:
+; CHECK-NOFULLFP16:       // %bb.0: // %entry
+; CHECK-NOFULLFP16-NEXT:    fcmgt s2, s0, s1
+; CHECK-NOFULLFP16-NEXT:    fcmgt s0, s1, s0
+; CHECK-NOFULLFP16-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NOFULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NOFULLFP16-NEXT:    ret
+;
+; CHECK-NONANS-LABEL: dup_v1i32_ueq:
+; CHECK-NONANS:       // %bb.0: // %entry
+; CHECK-NONANS-NEXT:    fcmeq s0, s0, s1
+; CHECK-NONANS-NEXT:    ret
+;
+; CHECK-FULLFP16-LABEL: dup_v1i32_ueq:
+; CHECK-FULLFP16:       // %bb.0: // %entry
+; CHECK-FULLFP16-NEXT:    fcmgt s2, s0, s1
+; CHECK-FULLFP16-NEXT:    fcmgt s0, s1, s0
+; CHECK-FULLFP16-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-FULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-FULLFP16-NEXT:    ret
+entry:
+  %0 = fcmp ueq float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_ugt(float %a, float %b) {
+; CHECK-NOFULLFP16-LABEL: dup_v1i32_ugt:
+; CHECK-NOFULLFP16:       // %bb.0: // %entry
+; CHECK-NOFULLFP16-NEXT:    fcmge s0, s1, s0
+; CHECK-NOFULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NOFULLFP16-NEXT:    ret
+;
+; CHECK-NONANS-LABEL: dup_v1i32_ugt:
+; CHECK-NONANS:       // %bb.0: // %entry
+; CHECK-NONANS-NEXT:    fcmgt s0, s0, s1
+; CHECK-NONANS-NEXT:    ret
+;
+; CHECK-FULLFP16-LABEL: dup_v1i32_ugt:
+; CHECK-FULLFP16:       // %bb.0: // %entry
+; CHECK-FULLFP16-NEXT:    fcmge s0, s1, s0
+; CHECK-FULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-FULLFP16-NEXT:    ret
+entry:
+  %0 = fcmp ugt float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_uge(float %a, float %b) {
+; CHECK-NOFULLFP16-LABEL: dup_v1i32_uge:
+; CHECK-NOFULLFP16:       // %bb.0: // %entry
+; CHECK-NOFULLFP16-NEXT:    fcmgt s0, s1, s0
+; CHECK-NOFULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NOFULLFP16-NEXT:    ret
+;
+; CHECK-NONANS-LABEL: dup_v1i32_uge:
+; CHECK-NONANS:       // %bb.0: // %entry
+; CHECK-NONANS-NEXT:    fcmge s0, s0, s1
+; CHECK-NONANS-NEXT:    ret
+;
+; CHECK-FULLFP16-LABEL: dup_v1i32_uge:
+; CHECK-FULLFP16:       // %bb.0: // %entry
+; CHECK-FULLFP16-NEXT:    fcmgt s0, s1, s0
+; CHECK-FULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-FULLFP16-NEXT:    ret
+entry:
+  %0 = fcmp uge float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_ult(float %a, float %b) {
+; CHECK-NOFULLFP16-LABEL: dup_v1i32_ult:
+; CHECK-NOFULLFP16:       // %bb.0: // %entry
+; CHECK-NOFULLFP16-NEXT:    fcmge s0, s0, s1
+; CHECK-NOFULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NOFULLFP16-NEXT:    ret
+;
+; CHECK-NONANS-LABEL: dup_v1i32_ult:
+; CHECK-NONANS:       // %bb.0: // %entry
+; CHECK-NONANS-NEXT:    fcmgt s0, s1, s0
+; CHECK-NONANS-NEXT:    ret
+;
+; CHECK-FULLFP16-LABEL: dup_v1i32_ult:
+; CHECK-FULLFP16:       // %bb.0: // %entry
+; CHECK-FULLFP16-NEXT:    fcmge s0, s0, s1
+; CHECK-FULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-FULLFP16-NEXT:    ret
+entry:
+  %0 = fcmp ult float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_ule(float %a, float %b) {
+; CHECK-NOFULLFP16-LABEL: dup_v1i32_ule:
+; CHECK-NOFULLFP16:       // %bb.0: // %entry
+; CHECK-NOFULLFP16-NEXT:    fcmgt s0, s0, s1
+; CHECK-NOFULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NOFULLFP16-NEXT:    ret
+;
+; CHECK-NONANS-LABEL: dup_v1i32_ule:
+; CHECK-NONANS:       // %bb.0: // %entry
+; CHECK-NONANS-NEXT:    fcmge s0, s1, s0
+; CHECK-NONANS-NEXT:    ret
+;
+; CHECK-FULLFP16-LABEL: dup_v1i32_ule:
+; CHECK-FULLFP16:       // %bb.0: // %entry
+; CHECK-FULLFP16-NEXT:    fcmgt s0, s0, s1
+; CHECK-FULLFP16-NEXT:    mvn v0.8b, v0.8b
+; CHECK-FULLFP16-NEXT:    ret
+entry:
+  %0 = fcmp ule float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_une(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_une:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmeq s0, s0, s1
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp une float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <1 x float> @dup_v1i32_uno(float %a, float %b) {
+; CHECK-LABEL: dup_v1i32_uno:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmge s2, s0, s1
+; CHECK-NEXT:    fcmgt s0, s1, s0
+; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp uno float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <1 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <1 x i32> %vecinit.i to <1 x float>
+  ret <1 x float> %1
+}
+
+define <4 x float> @dup_v4i32(float %a, float %b) {
+; CHECK-LABEL: dup_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmge s0, s0, s1
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp oge float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <4 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <4 x i32> %vecinit.i to <4 x float>
+  %2 = shufflevector <4 x float> %1, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %2
+}
+
+define <4 x float> @dup_v4i32_reversed(float %a, float %b) {
+; CHECK-LABEL: dup_v4i32_reversed:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmgt s0, s1, s0
+; CHECK-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp ogt float %b, %a
+  %vcmpd.i = sext i1 %0 to i32
+  %vecinit.i = insertelement <4 x i32> poison, i32 %vcmpd.i, i64 0
+  %1 = bitcast <4 x i32> %vecinit.i to <4 x float>
+  %2 = shufflevector <4 x float> %1, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %2
+}
+
+define <2 x double> @dup_v2i64(double %a, double %b) {
+; CHECK-LABEL: dup_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmgt d0, d0, d1
+; CHECK-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp ogt double %a, %b
+  %vcmpd.i = sext i1 %0 to i64
+  %vecinit.i = insertelement <2 x i64> poison, i64 %vcmpd.i, i64 0
+  %1 = bitcast <2 x i64> %vecinit.i to <2 x double>
+  %2 = shufflevector <2 x double> %1, <2 x double> poison, <2 x i32> zeroinitializer
+  ret <2 x double> %2
+}
+
+define <8 x half> @dup_v8i16(half %a, half %b) {
+; CHECK-NOFULLFP16-LABEL: dup_v8i16:
+; CHECK-NOFULLFP16:       // %bb.0: // %entry
+; CHECK-NOFULLFP16-NEXT:    fcvt s1, h1
+; CHECK-NOFULLFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFULLFP16-NEXT:    fcmeq s0, s0, s1
+; CHECK-NOFULLFP16-NEXT:    ret
+;
+; CHECK-NONANS-LABEL: dup_v8i16:
+; CHECK-NONANS:       // %bb.0: // %entry
+; CHECK-NONANS-NEXT:    fcvt s1, h1
+; CHECK-NONANS-NEXT:    fcvt s0, h0
+; CHECK-NONANS-NEXT:    fcmeq s0, s0, s1
+; CHECK-NONANS-NEXT:    ret
+;
+; CHECK-FULLFP16-LABEL: dup_v8i16:
+; CHECK-FULLFP16:       // %bb.0: // %entry
+; CHECK-FULLFP16-NEXT:    fcmp h0, h1
+; CHECK-FULLFP16-NEXT:    csetm w8, eq
+; CHECK-FULLFP16-NEXT:    fmov s0, w8
+; CHECK-FULLFP16-NEXT:    ret
+; FIXME: Could be replaced with fcmeq + dup but the type of the former is
+; promoted to i32 during selection and then the optimization does not apply.
+
+  entry:
+  %0 = fcmp oeq half %a, %b
+  %vcmpd.i = sext i1 %0 to i16
+  %vecinit.i = insertelement <8 x i16> poison, i16 %vcmpd.i, i64 0
+  %1 = bitcast <8 x i16> %vecinit.i to <8 x half>
+  ret <8 x half> %1
+}
+
+; Check that a mask is not generated for non-vectorized users.
+define i32 @mask_i32(float %a, float %b) {
+; CHECK-LABEL: mask_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    csetm w0, eq
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp oeq float %a, %b
+  %vcmpd.i = sext i1 %0 to i32
+  ret i32 %vcmpd.i
+}
+
+; Verify that a mask is not emitted when (allOnes, allZeros) are not the
+; operands for the SELECT_CC.
+define i32 @bool_i32(float %a, float %b) {
+; CHECK-LABEL: bool_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+entry:
+  %0 = fcmp oeq float %a, %b
+  %vcmpd.i = zext i1 %0 to i32
+  ret i32 %vcmpd.i
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-label-bti.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-label-bti.ll
new file mode 100644
index 000000000000..657b5e304c7c
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-label-bti.ll
@@ -0,0 +1,40 @@
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+
+; Test function which compares two integers and returns the value of
+; the overflow flag, by using an asm goto to make the asm block branch
+; based on that flag, and then a phi to set the return value based on
+; whether the branch was taken.
+define i32 @overflow(i64 %a, i64 %b) #0 {
+asm:
+  callbr void asm sideeffect "cmp $0, $1 \0A\09 b.vs ${2:l}",
+          "r,r,!i,~{cc}"(i64 %a, i64 %b)
+          to label %fallthrough [label %indirect]
+
+indirect:
+  br label %fallthrough
+
+fallthrough:
+  ; Return 1 if we came via the 'indirect' block (because the b.vs was
+  ; taken), and 0 if we came straight from the asm block (because it
+  ; was untaken).
+  %retval = phi i32 [0, %asm], [1, %indirect]
+  ret i32 %retval
+}
+
+; CHECK: overflow:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.{{[0-9]+}}:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: //APP
+; CHECK-NEXT: cmp x0, x1
+; CHECK-NEXT: b.vs [[LABEL:\.[A-Za-z0-9_]+]]
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: // %bb.{{[0-9]+}}:
+; CHECK-NEXT: mov w0, wzr
+; CHECK-NEXT: ret
+; CHECK-NEXT: [[LABEL]]:
+; CHECK-NOT:  bti
+; CHECK:      mov w0, #1
+; CHECK-NEXT: ret
+
+attributes #0 = { "branch-target-enforcement" "target-features"="+bti" }
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-label.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-label.ll
index 1818f94a831b..0a49cfa11afe 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-label.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-label.ll
@@ -7,7 +7,7 @@ define i32 @test1() {
 ; CHECK:         .word b
 ; CHECK-NEXT:    .word .LBB0_2
 ; CHECK: // %bb.1:
-; CHECK: .LBB0_2: // Block address taken
+; CHECK: .LBB0_2: // Inline asm indirect target
 entry:
   callbr void asm sideeffect "1:\0A\09.word b, ${0:l}\0A\09", "!i"()
           to label %cleanup [label %indirect]
@@ -31,7 +31,7 @@ entry:
 if.then:
 ; CHECK:       .word b
 ; CHECK-NEXT:  .word .LBB1_3
-; CHECK:       .LBB1_3: // Block address taken
+; CHECK:       .LBB1_3: // Inline asm indirect target
   callbr void asm sideeffect "1:\0A\09.word b, ${0:l}\0A\09", "!i"()
           to label %if.then4 [label %if.end6]
 
@@ -46,7 +46,7 @@ if.end6:
   br i1 %phitmp, label %if.end10, label %if.then9
 
 if.then9:
-; CHECK: .LBB1_5: // Block address taken
+; CHECK: .LBB1_5: // Inline asm indirect target
   callbr void asm sideeffect "", "!i"()
           to label %if.end10 [label %l_yes]
 
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll
index fbe89e70e4d8..00d5d20de8fe 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll
@@ -22,7 +22,7 @@ define i32 @test0() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32all = COPY %5
   ; CHECK-NEXT:   B %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.entry.indirect_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.entry.indirect_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.5(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %5
@@ -35,7 +35,7 @@ define i32 @test0() {
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY %7
   ; CHECK-NEXT:   B %bb.4
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.3.direct.indirect_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.3.direct.indirect_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.5(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr32all = COPY %7
@@ -87,7 +87,7 @@ define i32 @dont_split0() {
   ; CHECK-NEXT:   $w0 = COPY [[MOVi32imm]]
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.y (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.y (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32all = COPY $wzr
   ; CHECK-NEXT:   $w0 = COPY [[COPY]]
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
@@ -116,7 +116,7 @@ define i32 @dont_split1() {
   ; CHECK-NEXT:   $w0 = COPY [[MOVi32imm]]
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.y (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.y (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   $w0 = COPY %1
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
 entry:
@@ -147,7 +147,7 @@ define i32 @dont_split2() {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY $wzr
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY [[COPY1]]
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.y (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.y (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr32all = PHI [[COPY]], %bb.0, [[COPY2]], %bb.1
   ; CHECK-NEXT:   $w0 = COPY [[PHI]]
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
@@ -174,7 +174,7 @@ define i32 @dont_split3() {
   ; CHECK-NEXT: bb.1.x:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.v (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.v (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 42
   ; CHECK-NEXT:   $w0 = COPY [[MOVi32imm]]
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
@@ -198,7 +198,7 @@ define i32 @split_me0() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32all = COPY %3
   ; CHECK-NEXT:   B %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.entry.y_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.entry.y_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %3
@@ -248,7 +248,7 @@ define i32 @split_me1(i1 %z) {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %5
   ; CHECK-NEXT:   B %bb.3
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.w.v_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.w.v_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.4(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY %5
@@ -301,7 +301,7 @@ define i32 @split_me2(i1 %z) {
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY %6
   ; CHECK-NEXT:   B %bb.3
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.w.v_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.w.v_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.4(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr32all = COPY %6
@@ -349,7 +349,7 @@ define i32 @dont_split4() {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   B %bb.3
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.y (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.y (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %3
@@ -383,7 +383,7 @@ define i32 @dont_split5() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32all = COPY %3
   ; CHECK-NEXT:   B %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.y (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.y (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %3
@@ -414,7 +414,7 @@ define i32 @split_me3() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32all = COPY %3
   ; CHECK-NEXT:   B %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.entry.out_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.entry.out_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %3
@@ -460,7 +460,7 @@ define i32 @dont_split6(i32 %0) {
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32all = COPY %4
   ; CHECK-NEXT:   B %bb.3
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.loop.loop_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.loop.loop_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr32all = COPY %4
@@ -495,7 +495,7 @@ define i32 @split_me4() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32all = COPY %3
   ; CHECK-NEXT:   B %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.entry.same_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.entry.same_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %3
@@ -526,7 +526,7 @@ define i32 @split_me5() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32all = COPY %3
   ; CHECK-NEXT:   B %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.entry.same_crit_edge (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.entry.same_crit_edge (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32all = COPY %3
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll
index c5fd9b63cce9..5765e0acae26 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll
@@ -602,3 +602,49 @@ define i1 @almost_immediate_neg_ugt_64(i64 %x) {
   %cmp = icmp ugt i64 %x, -16773121
   ret i1 %cmp
 }
+
+define i1 @cmn_nsw(i32 %a, i32 %b) {
+; CHECK-LABEL: cmn_nsw:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmn w0, w1
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+  %sub = sub nsw i32 0, %b
+  %cmp = icmp sgt i32 %a, %sub
+  ret i1 %cmp
+}
+
+define i1 @cmn_nsw_64(i64 %a, i64 %b) {
+; CHECK-LABEL: cmn_nsw_64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmn x0, x1
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+  %sub = sub nsw i64 0, %b
+  %cmp = icmp sgt i64 %a, %sub
+  ret i1 %cmp
+}
+
+define i1 @cmn_nsw_neg(i32 %a, i32 %b) {
+; CHECK-LABEL: cmn_nsw_neg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w8, w1
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+  %sub = sub i32 0, %b
+  %cmp = icmp sgt i32 %a, %sub
+  ret i1 %cmp
+}
+
+define i1 @cmn_nsw_neg_64(i64 %a, i64 %b) {
+; CHECK-LABEL: cmn_nsw_neg_64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg x8, x1
+; CHECK-NEXT:    cmp x0, x8
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+  %sub = sub i64 0, %b
+  %cmp = icmp sgt i64 %a, %sub
+  ret i1 %cmp
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/concat-vector.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/concat-vector.ll
index 05a40453833e..acf15f1bd117 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/concat-vector.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/concat-vector.ll
@@ -32,10 +32,8 @@ define <8 x i8> @concat2(<4 x i8> %A, <4 x i8> %B) {
 ;
 ; CHECK-GI-LABEL: concat2:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -63,10 +61,8 @@ define <4 x i16> @concat4(<2 x i16> %A, <2 x i16> %B) {
 ;
 ; CHECK-GI-LABEL: concat4:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
 ; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -129,9 +125,8 @@ define <4 x half> @concat9(<2 x half> %A, <2 x half> %B) {
 ;
 ; CHECK-GI-LABEL: concat9:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
 ; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    ret
@@ -168,9 +163,8 @@ define <8 x i16> @concat_v8s16_v2s16(ptr %ptr) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    ldrh w8, [x0]
 ; CHECK-GI-NEXT:    ldrh w9, [x0, #2]
-; CHECK-GI-NEXT:    fmov s1, w8
-; CHECK-GI-NEXT:    mov v1.h[1], w9
-; CHECK-GI-NEXT:    mov v0.s[0], v1.s[0]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov v0.h[1], w9
 ; CHECK-GI-NEXT:    ret
     %a = load <2 x i16>, ptr %ptr
     %b = shufflevector <2 x i16> %a, <2 x i16> %a, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -220,15 +214,13 @@ define <16 x i8> @concat_v16s8_v4s8_reg(<4 x i8> %A, <4 x i8> %B, <4 x i8> %C, <
 ;
 ; CHECK-GI-LABEL: concat_v16s8_v4s8_reg:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    uzp1 v2.8b, v2.8b, v0.8b
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
-; CHECK-GI-NEXT:    uzp1 v1.8b, v3.8b, v0.8b
 ; CHECK-GI-NEXT:    fmov w8, s2
+; CHECK-GI-NEXT:    uzp1 v1.8b, v3.8b, v0.8b
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    mov v0.s[3], w8
@@ -253,15 +245,13 @@ define <8 x i16> @concat_v8s16_v2s16_reg(<2 x i16> %A, <2 x i16> %B, <2 x i16> %
 ;
 ; CHECK-GI-LABEL: concat_v8s16_v2s16_reg:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
 ; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    uzp1 v2.4h, v2.4h, v0.4h
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
-; CHECK-GI-NEXT:    uzp1 v1.4h, v3.4h, v0.4h
 ; CHECK-GI-NEXT:    fmov w8, s2
+; CHECK-GI-NEXT:    uzp1 v1.4h, v3.4h, v0.4h
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    mov v0.s[3], w8
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/ctlz.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/ctlz.ll
index a4863d1f7420..b1c6e24c30a7 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/ctlz.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/ctlz.ll
@@ -294,11 +294,11 @@ define <2 x i64> @v2i64(<2 x i64> %d) {
 ;
 ; CHECK-GI-LABEL: v2i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    clz x9, x9
+; CHECK-GI-NEXT:    fmov d0, x9
 ; CHECK-GI-NEXT:    clz x8, x8
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    clz x8, x9
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -411,16 +411,16 @@ define <4 x i64> @v4i64(<4 x i64> %d) {
 ;
 ; CHECK-GI-LABEL: v4i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
 ; CHECK-GI-NEXT:    mov x8, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
-; CHECK-GI-NEXT:    clz x9, x9
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
 ; CHECK-GI-NEXT:    clz x10, x10
-; CHECK-GI-NEXT:    mov v0.d[0], x9
-; CHECK-GI-NEXT:    mov v1.d[0], x10
+; CHECK-GI-NEXT:    clz x11, x11
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    fmov d1, x11
 ; CHECK-GI-NEXT:    clz x8, x8
-; CHECK-GI-NEXT:    clz x9, x11
+; CHECK-GI-NEXT:    clz x9, x9
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/cttz.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/cttz.ll
index 60125f8a1981..93ac97e20dab 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/cttz.ll
@@ -235,15 +235,15 @@ define void @v2i16(ptr %p1) {
 ; CHECK-GI-LABEL: v2i16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov w8, #65535 // =0xffff
-; CHECK-GI-NEXT:    ld1 { v1.h }[0], [x0]
-; CHECK-GI-NEXT:    ldr h2, [x0, #2]
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
-; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    ld1 { v0.h }[0], [x0]
+; CHECK-GI-NEXT:    ldr h1, [x0, #2]
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], w8
 ; CHECK-GI-NEXT:    add x8, x0, #2
-; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    add v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    and v0.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    eor v1.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v2.2s
+; CHECK-GI-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
 ; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
 ; CHECK-GI-NEXT:    uaddlp v0.4h, v0.8b
@@ -418,7 +418,7 @@ define <3 x i32> @v3i32(<3 x i32> %d) {
 ; CHECK-GI-LABEL: v3i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    mov v1.s[1], w8
 ; CHECK-GI-NEXT:    mov v1.s[2], w8
 ; CHECK-GI-NEXT:    eor v2.16b, v0.16b, v1.16b
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/darwinpcs-tail.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/darwinpcs-tail.ll
index 5d3c755d0d73..61c25edded2a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/darwinpcs-tail.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/darwinpcs-tail.ll
@@ -5,11 +5,11 @@
 
 ; CHECK-LABEL: __ZThn16_N1C3addEPKcz:
 ; CHECK:       b __ZN1C3addEPKcz
+
 ; CHECK-LABEL: _tailTest:
 ; CHECK:       b __ZN1C3addEPKcz
+
 ; CHECK-LABEL: __ZThn8_N1C1fEiiiiiiiiiz:
-; CHECK:       ldr     w9, [sp, #4]
-; CHECK:       str     w9, [sp, #4]
 ; CHECK:       b __ZN1C1fEiiiiiiiiiz
 
 %class.C = type { %class.A.base, [4 x i8], %class.B.base, [4 x i8] }
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/dup.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/dup.ll
index bfc0ef0826f6..079ff1076b11 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/dup.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/dup.ll
@@ -5,6 +5,7 @@
 ; CHECK-GI:       warning: Instruction selection used fallback path for dup_v2i8
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for duplane0_v2i8
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for loaddup_str_v2i8
 
 define <2 x i8> @dup_v2i8(i8 %a) {
 ; CHECK-LABEL: dup_v2i8:
@@ -41,6 +42,21 @@ entry:
   ret <2 x i8> %c
 }
 
+define <2 x i8> @loaddup_str_v2i8(ptr %p) {
+; CHECK-LABEL: loaddup_str_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    strb wzr, [x0]
+; CHECK-NEXT:    dup v0.2s, w8
+; CHECK-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <2 x i8> poison, i8 %a, i64 0
+  store i8 0, ptr %p
+  %c = shufflevector <2 x i8> %b, <2 x i8> poison, <2 x i32> zeroinitializer
+  ret <2 x i8> %c
+}
+
 define <3 x i8> @dup_v3i8(i8 %a) {
 ; CHECK-SD-LABEL: dup_v3i8:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -105,6 +121,34 @@ entry:
   ret <3 x i8> %c
 }
 
+define <3 x i8> @loaddup_str_v3i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, x0
+; CHECK-SD-NEXT:    ldrb w0, [x0]
+; CHECK-SD-NEXT:    strb wzr, [x8]
+; CHECK-SD-NEXT:    mov w1, w0
+; CHECK-SD-NEXT:    mov w2, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr b0, [x0]
+; CHECK-GI-NEXT:    mov x8, x0
+; CHECK-GI-NEXT:    strb wzr, [x8]
+; CHECK-GI-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-GI-NEXT:    umov w0, v0.b[0]
+; CHECK-GI-NEXT:    umov w1, v0.b[1]
+; CHECK-GI-NEXT:    umov w2, v0.b[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <3 x i8> poison, i8 %a, i64 0
+  store i8 0, ptr %p
+  %c = shufflevector <3 x i8> %b, <3 x i8> poison, <3 x i32> zeroinitializer
+  ret <3 x i8> %c
+}
+
 define <4 x i8> @dup_v4i8(i8 %a) {
 ; CHECK-SD-LABEL: dup_v4i8:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -162,6 +206,30 @@ entry:
   ret <4 x i8> %c
 }
 
+define <4 x i8> @loaddup_str_v4i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    strb wzr, [x0]
+; CHECK-SD-NEXT:    dup v0.4h, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr b0, [x0]
+; CHECK-GI-NEXT:    strb wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <4 x i8> poison, i8 %a, i64 0
+  store i8 0, ptr %p
+  %c = shufflevector <4 x i8> %b, <4 x i8> poison, <4 x i32> zeroinitializer
+  ret <4 x i8> %c
+}
+
 define <8 x i8> @dup_v8i8(i8 %a) {
 ; CHECK-LABEL: dup_v8i8:
 ; CHECK:       // %bb.0: // %entry
@@ -196,6 +264,27 @@ entry:
   ret <8 x i8> %c
 }
 
+define <8 x i8> @loaddup_str_v8i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v8i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8b }, [x0]
+; CHECK-SD-NEXT:    strb wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v8i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr b0, [x0]
+; CHECK-GI-NEXT:    strb wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <8 x i8> poison, i8 %a, i64 0
+  store i8 0, ptr %p
+  %c = shufflevector <8 x i8> %b, <8 x i8> poison, <8 x i32> zeroinitializer
+  ret <8 x i8> %c
+}
+
 define <16 x i8> @dup_v16i8(i8 %a) {
 ; CHECK-LABEL: dup_v16i8:
 ; CHECK:       // %bb.0: // %entry
@@ -229,6 +318,27 @@ entry:
   ret <16 x i8> %c
 }
 
+define <16 x i8> @loaddup_str_v16i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v16i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.16b }, [x0]
+; CHECK-SD-NEXT:    strb wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v16i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr b0, [x0]
+; CHECK-GI-NEXT:    strb wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.16b, v0.b[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <16 x i8> poison, i8 %a, i64 0
+  store i8 0, ptr %p
+  %c = shufflevector <16 x i8> %b, <16 x i8> poison, <16 x i32> zeroinitializer
+  ret <16 x i8> %c
+}
+
 define <32 x i8> @dup_v32i8(i8 %a) {
 ; CHECK-LABEL: dup_v32i8:
 ; CHECK:       // %bb.0: // %entry
@@ -271,6 +381,29 @@ entry:
   ret <32 x i8> %c
 }
 
+define <32 x i8> @loaddup_str_v32i8(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.16b }, [x0]
+; CHECK-SD-NEXT:    strb wzr, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr b1, [x0]
+; CHECK-GI-NEXT:    strb wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.16b, v1.b[0]
+; CHECK-GI-NEXT:    dup v1.16b, v1.b[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %b = insertelement <32 x i8> poison, i8 %a, i64 0
+  store i8 0, ptr %p
+  %c = shufflevector <32 x i8> %b, <32 x i8> poison, <32 x i32> zeroinitializer
+  ret <32 x i8> %c
+}
+
 define <2 x i16> @dup_v2i16(i16 %a) {
 ; CHECK-SD-LABEL: dup_v2i16:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -328,6 +461,30 @@ entry:
   ret <2 x i16> %c
 }
 
+define <2 x i16> @loaddup_str_v2i16(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    strh wzr, [x0]
+; CHECK-SD-NEXT:    dup v0.2s, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <2 x i16> poison, i16 %a, i64 0
+  store i16 0, ptr %p
+  %c = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> zeroinitializer
+  ret <2 x i16> %c
+}
+
 define <3 x i16> @dup_v3i16(i16 %a) {
 ; CHECK-LABEL: dup_v3i16:
 ; CHECK:       // %bb.0: // %entry
@@ -362,6 +519,27 @@ entry:
   ret <3 x i16> %c
 }
 
+define <3 x i16> @loaddup_str_v3i16(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    strh wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <3 x i16> poison, i16 %a, i64 0
+  store i16 0, ptr %p
+  %c = shufflevector <3 x i16> %b, <3 x i16> poison, <3 x i32> zeroinitializer
+  ret <3 x i16> %c
+}
+
 define <4 x i16> @dup_v4i16(i16 %a) {
 ; CHECK-LABEL: dup_v4i16:
 ; CHECK:       // %bb.0: // %entry
@@ -396,6 +574,27 @@ entry:
   ret <4 x i16> %c
 }
 
+define <4 x i16> @loaddup_str_v4i16(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    strh wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <4 x i16> poison, i16 %a, i64 0
+  store i16 0, ptr %p
+  %c = shufflevector <4 x i16> %b, <4 x i16> poison, <4 x i32> zeroinitializer
+  ret <4 x i16> %c
+}
+
 define <8 x i16> @dup_v8i16(i16 %a) {
 ; CHECK-LABEL: dup_v8i16:
 ; CHECK:       // %bb.0: // %entry
@@ -429,6 +628,27 @@ entry:
   ret <8 x i16> %c
 }
 
+define <8 x i16> @loaddup_str_v8i16(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    strh wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <8 x i16> poison, i16 %a, i64 0
+  store i16 0, ptr %p
+  %c = shufflevector <8 x i16> %b, <8 x i16> poison, <8 x i32> zeroinitializer
+  ret <8 x i16> %c
+}
+
 define <16 x i16> @dup_v16i16(i16 %a) {
 ; CHECK-LABEL: dup_v16i16:
 ; CHECK:       // %bb.0: // %entry
@@ -471,6 +691,29 @@ entry:
   ret <16 x i16> %c
 }
 
+define <16 x i16> @loaddup_str_v16i16(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    strh wzr, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h1, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8h, v1.h[0]
+; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %b = insertelement <16 x i16> poison, i16 %a, i64 0
+  store i16 0, ptr %p
+  %c = shufflevector <16 x i16> %b, <16 x i16> poison, <16 x i32> zeroinitializer
+  ret <16 x i16> %c
+}
+
 define <2 x i32> @dup_v2i32(i32 %a) {
 ; CHECK-LABEL: dup_v2i32:
 ; CHECK:       // %bb.0: // %entry
@@ -505,6 +748,27 @@ entry:
   ret <2 x i32> %c
 }
 
+define <2 x i32> @loaddup_str_v2i32(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2s, v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <2 x i32> poison, i32 %a, i64 0
+  store i32 0, ptr %p
+  %c = shufflevector <2 x i32> %b, <2 x i32> poison, <2 x i32> zeroinitializer
+  ret <2 x i32> %c
+}
+
 define <3 x i32> @dup_v3i32(i32 %a) {
 ; CHECK-LABEL: dup_v3i32:
 ; CHECK:       // %bb.0: // %entry
@@ -538,6 +802,27 @@ entry:
   ret <3 x i32> %c
 }
 
+define <3 x i32> @loaddup_str_v3i32(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <3 x i32> poison, i32 %a, i64 0
+  store i32 0, ptr %p
+  %c = shufflevector <3 x i32> %b, <3 x i32> poison, <3 x i32> zeroinitializer
+  ret <3 x i32> %c
+}
+
 define <4 x i32> @dup_v4i32(i32 %a) {
 ; CHECK-LABEL: dup_v4i32:
 ; CHECK:       // %bb.0: // %entry
@@ -571,6 +856,27 @@ entry:
   ret <4 x i32> %c
 }
 
+define <4 x i32> @loaddup_str_v4i32(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <4 x i32> poison, i32 %a, i64 0
+  store i32 0, ptr %p
+  %c = shufflevector <4 x i32> %b, <4 x i32> poison, <4 x i32> zeroinitializer
+  ret <4 x i32> %c
+}
+
 define <8 x i32> @dup_v8i32(i32 %a) {
 ; CHECK-LABEL: dup_v8i32:
 ; CHECK:       // %bb.0: // %entry
@@ -613,6 +919,29 @@ entry:
   ret <8 x i32> %c
 }
 
+define <8 x i32> @loaddup_str_v8i32(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s1, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4s, v1.s[0]
+; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %b = insertelement <8 x i32> poison, i32 %a, i64 0
+  store i32 0, ptr %p
+  %c = shufflevector <8 x i32> %b, <8 x i32> poison, <8 x i32> zeroinitializer
+  ret <8 x i32> %c
+}
+
 define <2 x i64> @dup_v2i64(i64 %a) {
 ; CHECK-LABEL: dup_v2i64:
 ; CHECK:       // %bb.0: // %entry
@@ -646,6 +975,27 @@ entry:
   ret <2 x i64> %c
 }
 
+define <2 x i64> @loaddup_str_v2i64(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-SD-NEXT:    str xzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr d0, [x0]
+; CHECK-GI-NEXT:    str xzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %b = insertelement <2 x i64> poison, i64 %a, i64 0
+  store i64 0, ptr %p
+  %c = shufflevector <2 x i64> %b, <2 x i64> poison, <2 x i32> zeroinitializer
+  ret <2 x i64> %c
+}
+
 define <3 x i64> @dup_v3i64(i64 %a) {
 ; CHECK-SD-LABEL: dup_v3i64:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -675,13 +1025,12 @@ define <3 x i64> @duplane0_v3i64(<3 x i64> %b) {
 ;
 ; CHECK-GI-LABEL: duplane0_v3i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov d2, d0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    mov v2.d[1], v1.d[0]
-; CHECK-GI-NEXT:    dup v0.2d, v2.d[0]
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    dup v0.2d, v0.d[0]
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov d2, d0
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = shufflevector <3 x i64> %b, <3 x i64> poison, <3 x i32> zeroinitializer
@@ -711,6 +1060,33 @@ entry:
   ret <3 x i64> %c
 }
 
+define <3 x i64> @loaddup_str_v3i64(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    str xzr, [x0]
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr d2, [x0]
+; CHECK-GI-NEXT:    str xzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2d, v2.d[0]
+; CHECK-GI-NEXT:    dup v2.2d, v2.d[0]
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %b = insertelement <3 x i64> poison, i64 %a, i64 0
+  store i64 0, ptr %p
+  %c = shufflevector <3 x i64> %b, <3 x i64> poison, <3 x i32> zeroinitializer
+  ret <3 x i64> %c
+}
+
 define <4 x i64> @dup_v4i64(i64 %a) {
 ; CHECK-LABEL: dup_v4i64:
 ; CHECK:       // %bb.0: // %entry
@@ -753,6 +1129,29 @@ entry:
   ret <4 x i64> %c
 }
 
+define <4 x i64> @loaddup_str_v4i64(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-SD-NEXT:    str xzr, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr d1, [x0]
+; CHECK-GI-NEXT:    str xzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2d, v1.d[0]
+; CHECK-GI-NEXT:    dup v1.2d, v1.d[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %b = insertelement <4 x i64> poison, i64 %a, i64 0
+  store i64 0, ptr %p
+  %c = shufflevector <4 x i64> %b, <4 x i64> poison, <4 x i32> zeroinitializer
+  ret <4 x i64> %c
+}
+
 define <2 x i128> @dup_v2i128(i128 %a) {
 ; CHECK-LABEL: dup_v2i128:
 ; CHECK:       // %bb.0: // %entry
@@ -800,6 +1199,38 @@ entry:
   ret <2 x i128> %c
 }
 
+define <2 x i128> @loaddup_str_v2i128(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, x0
+; CHECK-SD-NEXT:    ldr x0, [x0]
+; CHECK-SD-NEXT:    ldr x1, [x8, #8]
+; CHECK-SD-NEXT:    stp xzr, xzr, [x8]
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    mov v0.d[0], xzr
+; CHECK-GI-NEXT:    mov x8, x0
+; CHECK-GI-NEXT:    mov d2, v1.d[1]
+; CHECK-GI-NEXT:    fmov x0, d1
+; CHECK-GI-NEXT:    fmov x2, d1
+; CHECK-GI-NEXT:    mov v0.d[1], xzr
+; CHECK-GI-NEXT:    fmov x1, d2
+; CHECK-GI-NEXT:    fmov x3, d2
+; CHECK-GI-NEXT:    str q0, [x8]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i128, ptr %p
+  %b = insertelement <2 x i128> poison, i128 %a, i64 0
+  store i128 0, ptr %p
+  %c = shufflevector <2 x i128> %b, <2 x i128> poison, <2 x i32> zeroinitializer
+  ret <2 x i128> %c
+}
+
 define <3 x i128> @dup_v3i128(i128 %a) {
 ; CHECK-LABEL: dup_v3i128:
 ; CHECK:       // %bb.0: // %entry
@@ -855,6 +1286,41 @@ entry:
   ret <3 x i128> %c
 }
 
+define <3 x i128> @loaddup_str_v3i128(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, x0
+; CHECK-SD-NEXT:    ldp x0, x1, [x0]
+; CHECK-SD-NEXT:    stp xzr, xzr, [x8]
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x4, x0
+; CHECK-SD-NEXT:    mov x5, x1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    mov v0.d[0], xzr
+; CHECK-GI-NEXT:    mov x8, x0
+; CHECK-GI-NEXT:    mov d2, v1.d[1]
+; CHECK-GI-NEXT:    fmov x0, d1
+; CHECK-GI-NEXT:    fmov x2, d1
+; CHECK-GI-NEXT:    fmov x4, d1
+; CHECK-GI-NEXT:    mov v0.d[1], xzr
+; CHECK-GI-NEXT:    fmov x1, d2
+; CHECK-GI-NEXT:    fmov x3, d2
+; CHECK-GI-NEXT:    fmov x5, d2
+; CHECK-GI-NEXT:    str q0, [x8]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i128, ptr %p
+  %b = insertelement <3 x i128> poison, i128 %a, i64 0
+  store i128 0, ptr %p
+  %c = shufflevector <3 x i128> %b, <3 x i128> poison, <3 x i32> zeroinitializer
+  ret <3 x i128> %c
+}
+
 define <4 x i128> @dup_v4i128(i128 %a) {
 ; CHECK-LABEL: dup_v4i128:
 ; CHECK:       // %bb.0: // %entry
@@ -918,6 +1384,45 @@ entry:
   ret <4 x i128> %c
 }
 
+define <4 x i128> @loaddup_str_v4i128(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, x0
+; CHECK-SD-NEXT:    ldp x0, x1, [x0]
+; CHECK-SD-NEXT:    stp xzr, xzr, [x8]
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x4, x0
+; CHECK-SD-NEXT:    mov x5, x1
+; CHECK-SD-NEXT:    mov x6, x0
+; CHECK-SD-NEXT:    mov x7, x1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    mov v0.d[0], xzr
+; CHECK-GI-NEXT:    mov x8, x0
+; CHECK-GI-NEXT:    mov d2, v1.d[1]
+; CHECK-GI-NEXT:    fmov x0, d1
+; CHECK-GI-NEXT:    fmov x2, d1
+; CHECK-GI-NEXT:    fmov x4, d1
+; CHECK-GI-NEXT:    fmov x6, d1
+; CHECK-GI-NEXT:    mov v0.d[1], xzr
+; CHECK-GI-NEXT:    fmov x1, d2
+; CHECK-GI-NEXT:    fmov x3, d2
+; CHECK-GI-NEXT:    fmov x5, d2
+; CHECK-GI-NEXT:    fmov x7, d2
+; CHECK-GI-NEXT:    str q0, [x8]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load i128, ptr %p
+  %b = insertelement <4 x i128> poison, i128 %a, i64 0
+  store i128 0, ptr %p
+  %c = shufflevector <4 x i128> %b, <4 x i128> poison, <4 x i32> zeroinitializer
+  ret <4 x i128> %c
+}
+
 define <2 x half> @dup_v2half(half %a) {
 ; CHECK-LABEL: dup_v2half:
 ; CHECK:       // %bb.0: // %entry
@@ -953,6 +1458,28 @@ entry:
   ret <2 x half> %c
 }
 
+define <2 x half> @loaddup_str_v2half(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2half:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2half:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <2 x half> poison, half %a, i64 0
+  store half 0.0, ptr %p
+  %c = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> zeroinitializer
+  ret <2 x half> %c
+}
+
 define <3 x half> @dup_v3half(half %a) {
 ; CHECK-LABEL: dup_v3half:
 ; CHECK:       // %bb.0: // %entry
@@ -988,6 +1515,28 @@ entry:
   ret <3 x half> %c
 }
 
+define <3 x half> @loaddup_str_v3half(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3half:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3half:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <3 x half> poison, half %a, i64 0
+  store half 0.0, ptr %p
+  %c = shufflevector <3 x half> %b, <3 x half> poison, <3 x i32> zeroinitializer
+  ret <3 x half> %c
+}
+
 define <4 x half> @dup_v4half(half %a) {
 ; CHECK-LABEL: dup_v4half:
 ; CHECK:       // %bb.0: // %entry
@@ -1023,6 +1572,28 @@ entry:
   ret <4 x half> %c
 }
 
+define <4 x half> @loaddup_str_v4half(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4half:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4half:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <4 x half> poison, half %a, i64 0
+  store half 0.0, ptr %p
+  %c = shufflevector <4 x half> %b, <4 x half> poison, <4 x i32> zeroinitializer
+  ret <4 x half> %c
+}
+
 define <8 x half> @dup_v8half(half %a) {
 ; CHECK-LABEL: dup_v8half:
 ; CHECK:       // %bb.0: // %entry
@@ -1057,6 +1628,28 @@ entry:
   ret <8 x half> %c
 }
 
+define <8 x half> @loaddup_str_v8half(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v8half:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v8half:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <8 x half> poison, half %a, i64 0
+  store half 0.0, ptr %p
+  %c = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> zeroinitializer
+  ret <8 x half> %c
+}
+
 define <16 x half> @dup_v16half(half %a) {
 ; CHECK-SD-LABEL: dup_v16half:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -1108,6 +1701,30 @@ entry:
   ret <16 x half> %c
 }
 
+define <16 x half> @loaddup_str_v16half(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v16half:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    movi d2, #0000000000000000
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    str h2, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v16half:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h1, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8h, v1.h[0]
+; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load half, ptr %p
+  %b = insertelement <16 x half> poison, half %a, i64 0
+  store half 0.0, ptr %p
+  %c = shufflevector <16 x half> %b, <16 x half> poison, <16 x i32> zeroinitializer
+  ret <16 x half> %c
+}
+
 define <2 x bfloat> @dup_v2bfloat(bfloat %a) {
 ; CHECK-LABEL: dup_v2bfloat:
 ; CHECK:       // %bb.0: // %entry
@@ -1143,6 +1760,28 @@ entry:
   ret <2 x bfloat> %c
 }
 
+define <2 x bfloat> @loaddup_str_v2bfloat(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2bfloat:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2bfloat:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <2 x bfloat> poison, bfloat %a, i64 0
+  store bfloat 0.0, ptr %p
+  %c = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> zeroinitializer
+  ret <2 x bfloat> %c
+}
+
 define <3 x bfloat> @dup_v3bfloat(bfloat %a) {
 ; CHECK-LABEL: dup_v3bfloat:
 ; CHECK:       // %bb.0: // %entry
@@ -1178,6 +1817,28 @@ entry:
   ret <3 x bfloat> %c
 }
 
+define <3 x bfloat> @loaddup_str_v3bfloat(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3bfloat:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3bfloat:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <3 x bfloat> poison, bfloat %a, i64 0
+  store bfloat 0.0, ptr %p
+  %c = shufflevector <3 x bfloat> %b, <3 x bfloat> poison, <3 x i32> zeroinitializer
+  ret <3 x bfloat> %c
+}
+
 define <4 x bfloat> @dup_v4bfloat(bfloat %a) {
 ; CHECK-LABEL: dup_v4bfloat:
 ; CHECK:       // %bb.0: // %entry
@@ -1213,6 +1874,28 @@ entry:
   ret <4 x bfloat> %c
 }
 
+define <4 x bfloat> @loaddup_str_v4bfloat(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4bfloat:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.4h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4bfloat:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <4 x bfloat> poison, bfloat %a, i64 0
+  store bfloat 0.0, ptr %p
+  %c = shufflevector <4 x bfloat> %b, <4 x bfloat> poison, <4 x i32> zeroinitializer
+  ret <4 x bfloat> %c
+}
+
 define <8 x bfloat> @dup_v8bfloat(bfloat %a) {
 ; CHECK-LABEL: dup_v8bfloat:
 ; CHECK:       // %bb.0: // %entry
@@ -1247,6 +1930,28 @@ entry:
   ret <8 x bfloat> %c
 }
 
+define <8 x bfloat> @loaddup_str_v8bfloat(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v8bfloat:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0000000000000000
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    str h1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v8bfloat:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8h, v0.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <8 x bfloat> poison, bfloat %a, i64 0
+  store bfloat 0.0, ptr %p
+  %c = shufflevector <8 x bfloat> %b, <8 x bfloat> poison, <8 x i32> zeroinitializer
+  ret <8 x bfloat> %c
+}
+
 define <16 x bfloat> @dup_v16bfloat(bfloat %a) {
 ; CHECK-SD-LABEL: dup_v16bfloat:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -1298,6 +2003,30 @@ entry:
   ret <16 x bfloat> %c
 }
 
+define <16 x bfloat> @loaddup_str_v16bfloat(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v16bfloat:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.8h }, [x0]
+; CHECK-SD-NEXT:    movi d2, #0000000000000000
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    str h2, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v16bfloat:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h1, [x0]
+; CHECK-GI-NEXT:    strh wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.8h, v1.h[0]
+; CHECK-GI-NEXT:    dup v1.8h, v1.h[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load bfloat, ptr %p
+  %b = insertelement <16 x bfloat> poison, bfloat %a, i64 0
+  store bfloat 0.0, ptr %p
+  %c = shufflevector <16 x bfloat> %b, <16 x bfloat> poison, <16 x i32> zeroinitializer
+  ret <16 x bfloat> %c
+}
+
 define <2 x float> @dup_v2float(float %a) {
 ; CHECK-LABEL: dup_v2float:
 ; CHECK:       // %bb.0: // %entry
@@ -1333,6 +2062,27 @@ entry:
   ret <2 x float> %c
 }
 
+define <2 x float> @loaddup_str_v2float(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2float:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2float:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2s, v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <2 x float> poison, float %a, i64 0
+  store float 0.0, ptr %p
+  %c = shufflevector <2 x float> %b, <2 x float> poison, <2 x i32> zeroinitializer
+  ret <2 x float> %c
+}
+
 define <3 x float> @dup_v3float(float %a) {
 ; CHECK-LABEL: dup_v3float:
 ; CHECK:       // %bb.0: // %entry
@@ -1367,6 +2117,27 @@ entry:
   ret <3 x float> %c
 }
 
+define <3 x float> @loaddup_str_v3float(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3float:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3float:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <3 x float> poison, float %a, i64 0
+  store float 0.0, ptr %p
+  %c = shufflevector <3 x float> %b, <3 x float> poison, <3 x i32> zeroinitializer
+  ret <3 x float> %c
+}
+
 define <4 x float> @dup_v4float(float %a) {
 ; CHECK-LABEL: dup_v4float:
 ; CHECK:       // %bb.0: // %entry
@@ -1401,6 +2172,27 @@ entry:
   ret <4 x float> %c
 }
 
+define <4 x float> @loaddup_str_v4float(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4float:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4float:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s0, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <4 x float> poison, float %a, i64 0
+  store float 0.0, ptr %p
+  %c = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> zeroinitializer
+  ret <4 x float> %c
+}
+
 define <8 x float> @dup_v8float(float %a) {
 ; CHECK-SD-LABEL: dup_v8float:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -1452,6 +2244,29 @@ entry:
   ret <8 x float> %c
 }
 
+define <8 x float> @loaddup_str_v8float(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v8float:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.4s }, [x0]
+; CHECK-SD-NEXT:    str wzr, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v8float:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s1, [x0]
+; CHECK-GI-NEXT:    str wzr, [x0]
+; CHECK-GI-NEXT:    dup v0.4s, v1.s[0]
+; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load float, ptr %p
+  %b = insertelement <8 x float> poison, float %a, i64 0
+  store float 0.0, ptr %p
+  %c = shufflevector <8 x float> %b, <8 x float> poison, <8 x i32> zeroinitializer
+  ret <8 x float> %c
+}
+
 define <2 x double> @dup_v2double(double %a) {
 ; CHECK-LABEL: dup_v2double:
 ; CHECK:       // %bb.0: // %entry
@@ -1486,6 +2301,27 @@ entry:
   ret <2 x double> %c
 }
 
+define <2 x double> @loaddup_str_v2double(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v2double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-SD-NEXT:    str xzr, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v2double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr d0, [x0]
+; CHECK-GI-NEXT:    str xzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2d, v0.d[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load double, ptr %p
+  %b = insertelement <2 x double> poison, double %a, i64 0
+  store double 0.0, ptr %p
+  %c = shufflevector <2 x double> %b, <2 x double> poison, <2 x i32> zeroinitializer
+  ret <2 x double> %c
+}
+
 define <3 x double> @dup_v3double(double %a) {
 ; CHECK-SD-LABEL: dup_v3double:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -1517,13 +2353,12 @@ define <3 x double> @duplane0_v3double(<3 x double> %b) {
 ;
 ; CHECK-GI-LABEL: duplane0_v3double:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov d2, d0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    mov v2.d[1], v1.d[0]
-; CHECK-GI-NEXT:    dup v0.2d, v2.d[0]
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    dup v0.2d, v0.d[0]
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov d2, d0
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = shufflevector <3 x double> %b, <3 x double> poison, <3 x i32> zeroinitializer
@@ -1553,6 +2388,33 @@ entry:
   ret <3 x double> %c
 }
 
+define <3 x double> @loaddup_str_v3double(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v3double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    str xzr, [x0]
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v3double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr d2, [x0]
+; CHECK-GI-NEXT:    str xzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2d, v2.d[0]
+; CHECK-GI-NEXT:    dup v2.2d, v2.d[0]
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load double, ptr %p
+  %b = insertelement <3 x double> poison, double %a, i64 0
+  store double 0.0, ptr %p
+  %c = shufflevector <3 x double> %b, <3 x double> poison, <3 x i32> zeroinitializer
+  ret <3 x double> %c
+}
+
 define <4 x double> @dup_v4double(double %a) {
 ; CHECK-SD-LABEL: dup_v4double:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -1604,6 +2466,29 @@ entry:
   ret <4 x double> %c
 }
 
+define <4 x double> @loaddup_str_v4double(ptr %p) {
+; CHECK-SD-LABEL: loaddup_str_v4double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1r { v0.2d }, [x0]
+; CHECK-SD-NEXT:    str xzr, [x0]
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: loaddup_str_v4double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr d1, [x0]
+; CHECK-GI-NEXT:    str xzr, [x0]
+; CHECK-GI-NEXT:    dup v0.2d, v1.d[0]
+; CHECK-GI-NEXT:    dup v1.2d, v1.d[0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = load double, ptr %p
+  %b = insertelement <4 x double> poison, double %a, i64 0
+  store double 0.0, ptr %p
+  %c = shufflevector <4 x double> %b, <4 x double> poison, <4 x i32> zeroinitializer
+  ret <4 x double> %c
+}
+
 define <2 x fp128> @dup_v2fp128(fp128 %a) {
 ; CHECK-LABEL: dup_v2fp128:
 ; CHECK:       // %bb.0: // %entry
@@ -1638,6 +2523,23 @@ entry:
   ret <2 x fp128> %c
 }
 
+define <2 x fp128> @loaddup_str_v2fp128(ptr %p) {
+; CHECK-LABEL: loaddup_str_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    adrp x8, .LCPI155_0
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI155_0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    str q2, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load fp128, ptr %p
+  %b = insertelement <2 x fp128> poison, fp128 %a, i64 0
+  store fp128 0xL00000000000000000000000000000000, ptr %p
+  %c = shufflevector <2 x fp128> %b, <2 x fp128> poison, <2 x i32> zeroinitializer
+  ret <2 x fp128> %c
+}
+
 define <3 x fp128> @dup_v3fp128(fp128 %a) {
 ; CHECK-LABEL: dup_v3fp128:
 ; CHECK:       // %bb.0: // %entry
@@ -1675,6 +2577,24 @@ entry:
   ret <3 x fp128> %c
 }
 
+define <3 x fp128> @loaddup_str_v3fp128(ptr %p) {
+; CHECK-LABEL: loaddup_str_v3fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    adrp x8, .LCPI159_0
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI159_0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    str q3, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load fp128, ptr %p
+  %b = insertelement <3 x fp128> poison, fp128 %a, i64 0
+  store fp128 0xL00000000000000000000000000000000, ptr %p
+  %c = shufflevector <3 x fp128> %b, <3 x fp128> poison, <3 x i32> zeroinitializer
+  ret <3 x fp128> %c
+}
+
 define <4 x fp128> @dup_v4fp128(fp128 %a) {
 ; CHECK-LABEL: dup_v4fp128:
 ; CHECK:       // %bb.0: // %entry
@@ -1714,3 +2634,22 @@ entry:
   %c = shufflevector <4 x fp128> %b, <4 x fp128> poison, <4 x i32> zeroinitializer
   ret <4 x fp128> %c
 }
+
+define <4 x fp128> @loaddup_str_v4fp128(ptr %p) {
+; CHECK-LABEL: loaddup_str_v4fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    adrp x8, .LCPI163_0
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI163_0]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    mov v3.16b, v0.16b
+; CHECK-NEXT:    str q4, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %a = load fp128, ptr %p
+  %b = insertelement <4 x fp128> poison, fp128 %a, i64 0
+  store fp128 0xL00000000000000000000000000000000, ptr %p
+  %c = shufflevector <4 x fp128> %b, <4 x fp128> poison, <4 x i32> zeroinitializer
+  ret <4 x fp128> %c
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/ehcontguard.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/ehcontguard.ll
index eecff391d0f8..cb603a482d22 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/ehcontguard.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/ehcontguard.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=aarch64-windows | FileCheck %s
 ; EHCont Guard is currently only available on Windows
 
-; CHECK: .set "@feat.00", 16384
+; CHECK: "@feat.00" = 16384
 
 ; CHECK: .section .gehcont$y
 
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/extbinopload.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/extbinopload.ll
index 75f3ffc9515e..cabb0e7278e4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/extbinopload.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/extbinopload.ll
@@ -861,7 +861,7 @@ define <16 x i32> @extrause_shuffle(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) {
 define <16 x i32> @extrause_ext(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) {
 ; CHECK-LABEL: extrause_ext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp s0, s3, [x2]
+; CHECK-NEXT:    ldp s0, s5, [x2]
 ; CHECK-NEXT:    add x8, x3, #8
 ; CHECK-NEXT:    add x9, x3, #12
 ; CHECK-NEXT:    add x10, x1, #8
@@ -871,26 +871,26 @@ define <16 x i32> @extrause_ext(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) {
 ; CHECK-NEXT:    ld1 { v1.s }[1], [x1], #4
 ; CHECK-NEXT:    ld1 { v2.s }[1], [x1]
 ; CHECK-NEXT:    ldp s7, s4, [x0, #8]
-; CHECK-NEXT:    ld1 { v3.s }[1], [x3]
-; CHECK-NEXT:    ldp s6, s5, [x2, #8]
+; CHECK-NEXT:    ld1 { v5.s }[1], [x3]
+; CHECK-NEXT:    ldp s6, s3, [x2, #8]
 ; CHECK-NEXT:    ld1 { v4.s }[1], [x11]
 ; CHECK-NEXT:    ld1 { v7.s }[1], [x10]
-; CHECK-NEXT:    ld1 { v5.s }[1], [x9]
+; CHECK-NEXT:    ld1 { v3.s }[1], [x9]
 ; CHECK-NEXT:    ld1 { v6.s }[1], [x8]
 ; CHECK-NEXT:    uaddl v2.8h, v2.8b, v4.8b
 ; CHECK-NEXT:    uaddl v1.8h, v1.8b, v7.8b
 ; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
-; CHECK-NEXT:    uaddl v3.8h, v3.8b, v5.8b
+; CHECK-NEXT:    uaddl v5.8h, v5.8b, v3.8b
 ; CHECK-NEXT:    uaddl v6.8h, v0.8b, v6.8b
-; CHECK-NEXT:    ushll v5.8h, v5.8b, #0
+; CHECK-NEXT:    ushll v16.8h, v3.8b, #0
 ; CHECK-NEXT:    ushll v0.4s, v2.4h, #3
 ; CHECK-NEXT:    ushll2 v2.4s, v2.8h, #3
-; CHECK-NEXT:    ushll v7.4s, v3.4h, #3
-; CHECK-NEXT:    ushll2 v3.4s, v3.8h, #3
-; CHECK-NEXT:    stp q4, q5, [x4]
+; CHECK-NEXT:    ushll v7.4s, v5.4h, #3
+; CHECK-NEXT:    ushll2 v5.4s, v5.8h, #3
+; CHECK-NEXT:    stp q4, q16, [x4]
 ; CHECK-NEXT:    uaddw v0.4s, v0.4s, v1.4h
 ; CHECK-NEXT:    uaddw2 v1.4s, v2.4s, v1.8h
-; CHECK-NEXT:    uaddw2 v3.4s, v3.4s, v6.8h
+; CHECK-NEXT:    uaddw2 v3.4s, v5.4s, v6.8h
 ; CHECK-NEXT:    uaddw v2.4s, v7.4s, v6.4h
 ; CHECK-NEXT:    ret
   %lp1 = load <4 x i8>, ptr %p
@@ -960,7 +960,7 @@ define <16 x i32> @extrause_ext(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) {
 define <16 x i32> @extrause_add(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) {
 ; CHECK-LABEL: extrause_add:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp s0, s4, [x2]
+; CHECK-NEXT:    ldp s0, s5, [x2]
 ; CHECK-NEXT:    add x8, x3, #8
 ; CHECK-NEXT:    add x9, x3, #12
 ; CHECK-NEXT:    add x10, x1, #8
@@ -970,15 +970,15 @@ define <16 x i32> @extrause_add(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) {
 ; CHECK-NEXT:    ld1 { v1.s }[1], [x1], #4
 ; CHECK-NEXT:    ld1 { v2.s }[1], [x1]
 ; CHECK-NEXT:    ldp s7, s3, [x0, #8]
-; CHECK-NEXT:    ld1 { v4.s }[1], [x3]
-; CHECK-NEXT:    ldp s6, s5, [x2, #8]
+; CHECK-NEXT:    ld1 { v5.s }[1], [x3]
+; CHECK-NEXT:    ldp s6, s4, [x2, #8]
 ; CHECK-NEXT:    ld1 { v3.s }[1], [x11]
 ; CHECK-NEXT:    ld1 { v7.s }[1], [x10]
-; CHECK-NEXT:    ld1 { v5.s }[1], [x9]
+; CHECK-NEXT:    ld1 { v4.s }[1], [x9]
 ; CHECK-NEXT:    ld1 { v6.s }[1], [x8]
 ; CHECK-NEXT:    uaddl v16.8h, v2.8b, v3.8b
 ; CHECK-NEXT:    uaddl v1.8h, v1.8b, v7.8b
-; CHECK-NEXT:    uaddl v4.8h, v4.8b, v5.8b
+; CHECK-NEXT:    uaddl v4.8h, v5.8b, v4.8b
 ; CHECK-NEXT:    uaddl v2.8h, v0.8b, v6.8b
 ; CHECK-NEXT:    ushll v0.4s, v16.4h, #3
 ; CHECK-NEXT:    ushll2 v6.4s, v16.8h, #3
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fcmp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fcmp.ll
index e16ea9883a0c..f33f57e8751c 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fcmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fcmp.ll
@@ -654,7 +654,7 @@ define <2 x double> @v2f128_double(<2 x fp128> %a, <2 x fp128> %b, <2 x double>
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w19, lt
 ; CHECK-GI-NEXT:    bl __lttf2
-; CHECK-GI-NEXT:    mov v0.d[0], x19
+; CHECK-GI-NEXT:    fmov d0, x19
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w8, lt
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #32] // 32-byte Folded Reload
@@ -761,29 +761,29 @@ define <3 x double> @v3f128_double(<3 x fp128> %a, <3 x fp128> %b, <3 x double>
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    cset w22, lt
 ; CHECK-GI-NEXT:    bl __lttf2
+; CHECK-GI-NEXT:    ldp q0, q2, [sp, #64] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    sbfx x8, x21, #0, #1
-; CHECK-GI-NEXT:    ldp q3, q2, [sp, #64] // 32-byte Folded Reload
+; CHECK-GI-NEXT:    ldp q4, q3, [sp, #96] // 32-byte Folded Reload
+; CHECK-GI-NEXT:    sbfx x9, x22, #0, #1
+; CHECK-GI-NEXT:    fmov d1, x8
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v1.d[0], x8
-; CHECK-GI-NEXT:    sbfx x8, x22, #0, #1
-; CHECK-GI-NEXT:    mov v2.d[1], v3.d[0]
-; CHECK-GI-NEXT:    ldp q4, q3, [sp, #96] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.d[1], x8
-; CHECK-GI-NEXT:    mov v1.d[1], x8
-; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT:    mov v2.d[1], v0.d[0]
+; CHECK-GI-NEXT:    fmov d0, x8
 ; CHECK-GI-NEXT:    cset w8, lt
+; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
 ; CHECK-GI-NEXT:    sbfx x8, x8, #0, #1
-; CHECK-GI-NEXT:    and v1.16b, v2.16b, v1.16b
-; CHECK-GI-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v0.d[1], x9
 ; CHECK-GI-NEXT:    and x9, x19, x8
 ; CHECK-GI-NEXT:    bic x8, x20, x8
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    orr x8, x9, x8
-; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    bic v1.16b, v3.16b, v1.16b
+; CHECK-GI-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    add sp, sp, #176
@@ -929,22 +929,22 @@ define <3 x i32> @v3f64_i32(<3 x double> %a, <3 x double> %b, <3 x i32> %d, <3 x
 ; CHECK-GI-NEXT:    fcmp d2, d5
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    cset w9, mi
-; CHECK-GI-NEXT:    mov v2.s[0], w9
-; CHECK-GI-NEXT:    mov w9, #-1 // =0xffffffff
-; CHECK-GI-NEXT:    fcmgt v0.2d, v3.2d, v0.2d
 ; CHECK-GI-NEXT:    mov v1.s[1], w8
-; CHECK-GI-NEXT:    mov v3.s[0], w9
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fcmgt v0.2d, v3.2d, v0.2d
 ; CHECK-GI-NEXT:    mov v1.s[2], w8
-; CHECK-GI-NEXT:    mov v3.s[1], w9
+; CHECK-GI-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
 ; CHECK-GI-NEXT:    mov v0.d[1], v2.d[0]
-; CHECK-GI-NEXT:    mov v3.s[2], w9
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w8
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v1.4s
 ; CHECK-GI-NEXT:    neg v1.4s, v1.4s
+; CHECK-GI-NEXT:    mov v2.s[2], w8
 ; CHECK-GI-NEXT:    sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v3.16b
+; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v2.16b
 ; CHECK-GI-NEXT:    and v0.16b, v6.16b, v0.16b
 ; CHECK-GI-NEXT:    and v1.16b, v7.16b, v1.16b
 ; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
@@ -969,8 +969,6 @@ define <4 x i32> @v4f64_i32(<4 x double> %a, <4 x double> %b, <4 x i32> %d, <4 x
 ; CHECK-GI-NEXT:    fcmgt v0.2d, v2.2d, v0.2d
 ; CHECK-GI-NEXT:    fcmgt v1.2d, v3.2d, v1.2d
 ; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #31
-; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #31
 ; CHECK-GI-NEXT:    bsl v0.16b, v4.16b, v5.16b
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -1001,18 +999,18 @@ define <3 x float> @v3f32_float(<3 x float> %a, <3 x float> %b, <3 x float> %d,
 ; CHECK-GI-LABEL: v3f32_float:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
-; CHECK-GI-NEXT:    mov w9, #-1 // =0xffffffff
 ; CHECK-GI-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov v5.s[0], w9
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    mov v4.s[1], w8
-; CHECK-GI-NEXT:    mov v5.s[1], w9
 ; CHECK-GI-NEXT:    mov v4.s[2], w8
-; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w8
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT:    neg v1.4s, v4.4s
-; CHECK-GI-NEXT:    sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v5.16b
+; CHECK-GI-NEXT:    neg v4.4s, v4.4s
+; CHECK-GI-NEXT:    sshl v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-GI-NEXT:    and v1.16b, v3.16b, v1.16b
 ; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
@@ -1079,18 +1077,18 @@ define <3 x i32> @v3f32_i32(<3 x float> %a, <3 x float> %b, <3 x i32> %d, <3 x i
 ; CHECK-GI-LABEL: v3f32_i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
-; CHECK-GI-NEXT:    mov w9, #-1 // =0xffffffff
 ; CHECK-GI-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov v5.s[0], w9
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    mov v4.s[1], w8
-; CHECK-GI-NEXT:    mov v5.s[1], w9
 ; CHECK-GI-NEXT:    mov v4.s[2], w8
-; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w8
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT:    neg v1.4s, v4.4s
-; CHECK-GI-NEXT:    sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v5.16b
+; CHECK-GI-NEXT:    neg v4.4s, v4.4s
+; CHECK-GI-NEXT:    sshl v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-GI-NEXT:    and v1.16b, v3.16b, v1.16b
 ; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
@@ -1291,8 +1289,6 @@ define <8 x half> @v8f16_half(<8 x half> %a, <8 x half> %b, <8 x half> %d, <8 x
 ; CHECK-GI-NOFP16-NEXT:    fcmgt v4.4s, v5.4s, v4.4s
 ; CHECK-GI-NOFP16-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v4.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
-; CHECK-GI-NOFP16-NEXT:    sshr v0.8h, v0.8h, #15
 ; CHECK-GI-NOFP16-NEXT:    bsl v0.16b, v2.16b, v3.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
@@ -1341,9 +1337,9 @@ define <16 x half> @v16f16_half(<16 x half> %a, <16 x half> %b, <16 x half> %d,
 ; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v0.4h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl v18.4s, v2.4h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl v19.4s, v3.4h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
 ; CHECK-GI-NOFP16-NEXT:    fcmgt v16.4s, v18.4s, v16.4s
@@ -1352,12 +1348,12 @@ define <16 x half> @v16f16_half(<16 x half> %a, <16 x half> %b, <16 x half> %d,
 ; CHECK-GI-NOFP16-NEXT:    fcmgt v1.4s, v3.4s, v1.4s
 ; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v16.8h, v0.8h
 ; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v2.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    shl v0.8h, v0.8h, #15
-; CHECK-GI-NOFP16-NEXT:    shl v1.8h, v1.8h, #15
-; CHECK-GI-NOFP16-NEXT:    sshr v0.8h, v0.8h, #15
-; CHECK-GI-NOFP16-NEXT:    sshr v1.8h, v1.8h, #15
-; CHECK-GI-NOFP16-NEXT:    bsl v0.16b, v4.16b, v6.16b
-; CHECK-GI-NOFP16-NEXT:    bsl v1.16b, v5.16b, v7.16b
+; CHECK-GI-NOFP16-NEXT:    and v2.16b, v4.16b, v0.16b
+; CHECK-GI-NOFP16-NEXT:    bic v0.16b, v6.16b, v0.16b
+; CHECK-GI-NOFP16-NEXT:    and v3.16b, v5.16b, v1.16b
+; CHECK-GI-NOFP16-NEXT:    bic v1.16b, v7.16b, v1.16b
+; CHECK-GI-NOFP16-NEXT:    orr v0.16b, v2.16b, v0.16b
+; CHECK-GI-NOFP16-NEXT:    orr v1.16b, v3.16b, v1.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: v16f16_half:
@@ -1453,49 +1449,49 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32
 ; CHECK-GI-NOFP16-NEXT:    mov v2.h[0], v0.h[4]
 ; CHECK-GI-NOFP16-NEXT:    mov v3.h[0], v1.h[4]
 ; CHECK-GI-NOFP16-NEXT:    mov w8, #31 // =0x1f
-; CHECK-GI-NOFP16-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NOFP16-NEXT:    mov w9, #-1 // =0xffffffff
-; CHECK-GI-NOFP16-NEXT:    mov v5.s[0], w0
-; CHECK-GI-NOFP16-NEXT:    mov v6.s[0], w9
-; CHECK-GI-NOFP16-NEXT:    mov v7.s[0], w7
-; CHECK-GI-NOFP16-NEXT:    ldr s16, [sp]
-; CHECK-GI-NOFP16-NEXT:    ldr s17, [sp, #24]
-; CHECK-GI-NOFP16-NEXT:    ldr s18, [sp, #32]
+; CHECK-GI-NOFP16-NEXT:    fmov s4, w8
+; CHECK-GI-NOFP16-NEXT:    fmov s7, w0
+; CHECK-GI-NOFP16-NEXT:    ldr s5, [sp]
+; CHECK-GI-NOFP16-NEXT:    fmov s16, w7
+; CHECK-GI-NOFP16-NEXT:    fmov s18, w4
+; CHECK-GI-NOFP16-NEXT:    ldr s17, [sp, #32]
+; CHECK-GI-NOFP16-NEXT:    ldr s6, [sp, #8]
 ; CHECK-GI-NOFP16-NEXT:    mov v2.h[1], v0.h[5]
 ; CHECK-GI-NOFP16-NEXT:    mov v3.h[1], v1.h[5]
 ; CHECK-GI-NOFP16-NEXT:    mov v4.s[1], w8
-; CHECK-GI-NOFP16-NEXT:    mov v5.s[1], w1
-; CHECK-GI-NOFP16-NEXT:    mov v17.s[1], v18.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v6.s[1], w9
-; CHECK-GI-NOFP16-NEXT:    mov v7.s[1], v16.s[0]
-; CHECK-GI-NOFP16-NEXT:    ldr s16, [sp, #8]
+; CHECK-GI-NOFP16-NEXT:    mov v7.s[1], w1
+; CHECK-GI-NOFP16-NEXT:    mov v16.s[1], v5.s[0]
+; CHECK-GI-NOFP16-NEXT:    ldr s5, [sp, #24]
+; CHECK-GI-NOFP16-NEXT:    mov v18.s[1], w5
+; CHECK-GI-NOFP16-NEXT:    mov v5.s[1], v17.s[0]
 ; CHECK-GI-NOFP16-NEXT:    mov v2.h[2], v0.h[6]
 ; CHECK-GI-NOFP16-NEXT:    mov v3.h[2], v1.h[6]
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
 ; CHECK-GI-NOFP16-NEXT:    mov v4.s[2], w8
+; CHECK-GI-NOFP16-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    mov v5.s[2], w2
-; CHECK-GI-NOFP16-NEXT:    mov v6.s[2], w9
-; CHECK-GI-NOFP16-NEXT:    mov v7.s[2], v16.s[0]
-; CHECK-GI-NOFP16-NEXT:    ldr s16, [sp, #40]
+; CHECK-GI-NOFP16-NEXT:    mov v7.s[2], w2
+; CHECK-GI-NOFP16-NEXT:    mov v16.s[2], v6.s[0]
+; CHECK-GI-NOFP16-NEXT:    ldr s6, [sp, #40]
+; CHECK-GI-NOFP16-NEXT:    mov v18.s[2], w6
 ; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
 ; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT:    mov v17.s[2], v16.s[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.s[2], v6.s[0]
 ; CHECK-GI-NOFP16-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    mov v5.s[3], w3
+; CHECK-GI-NOFP16-NEXT:    mov v7.s[3], w3
 ; CHECK-GI-NOFP16-NEXT:    fcmgt v2.4s, v3.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    mov v3.s[0], w4
+; CHECK-GI-NOFP16-NEXT:    fmov s3, w8
+; CHECK-GI-NOFP16-NEXT:    mov v3.s[1], w8
 ; CHECK-GI-NOFP16-NEXT:    ushl v2.4s, v2.4s, v4.4s
 ; CHECK-GI-NOFP16-NEXT:    neg v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    mov v3.s[1], w5
+; CHECK-GI-NOFP16-NEXT:    mov v3.s[2], w8
 ; CHECK-GI-NOFP16-NEXT:    sshl v2.4s, v2.4s, v4.4s
 ; CHECK-GI-NOFP16-NEXT:    ldr s4, [sp, #16]
-; CHECK-GI-NOFP16-NEXT:    mov v3.s[2], w6
-; CHECK-GI-NOFP16-NEXT:    mov v7.s[3], v4.s[0]
-; CHECK-GI-NOFP16-NEXT:    eor v1.16b, v2.16b, v6.16b
-; CHECK-GI-NOFP16-NEXT:    and v2.16b, v3.16b, v2.16b
-; CHECK-GI-NOFP16-NEXT:    and v1.16b, v17.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    bsl v0.16b, v5.16b, v7.16b
+; CHECK-GI-NOFP16-NEXT:    mov v16.s[3], v4.s[0]
+; CHECK-GI-NOFP16-NEXT:    eor v1.16b, v2.16b, v3.16b
+; CHECK-GI-NOFP16-NEXT:    and v2.16b, v18.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT:    bsl v0.16b, v7.16b, v16.16b
+; CHECK-GI-NOFP16-NEXT:    and v1.16b, v5.16b, v1.16b
 ; CHECK-GI-NOFP16-NEXT:    orr v1.16b, v2.16b, v1.16b
 ; CHECK-GI-NOFP16-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NOFP16-NEXT:    mov s3, v0.s[2]
@@ -1514,60 +1510,60 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32
 ; CHECK-GI-FP16-LABEL: v7f16_i32:
 ; CHECK-GI-FP16:       // %bb.0: // %entry
 ; CHECK-GI-FP16-NEXT:    fcmgt v0.8h, v1.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    mov w9, #31 // =0x1f
-; CHECK-GI-FP16-NEXT:    mov v4.s[0], w0
-; CHECK-GI-FP16-NEXT:    mov v2.s[0], w9
-; CHECK-GI-FP16-NEXT:    mov v5.s[0], w7
-; CHECK-GI-FP16-NEXT:    ldr s6, [sp]
-; CHECK-GI-FP16-NEXT:    mov v7.s[0], w4
+; CHECK-GI-FP16-NEXT:    mov w10, #31 // =0x1f
+; CHECK-GI-FP16-NEXT:    fmov s5, w0
+; CHECK-GI-FP16-NEXT:    fmov s2, w10
+; CHECK-GI-FP16-NEXT:    fmov s6, w7
+; CHECK-GI-FP16-NEXT:    ldr s3, [sp]
+; CHECK-GI-FP16-NEXT:    fmov s17, w4
+; CHECK-GI-FP16-NEXT:    ldr s7, [sp, #24]
 ; CHECK-GI-FP16-NEXT:    ldr s16, [sp, #32]
-; CHECK-GI-FP16-NEXT:    ldr s17, [sp, #8]
+; CHECK-GI-FP16-NEXT:    mov v5.s[1], w1
 ; CHECK-GI-FP16-NEXT:    umov w8, v0.h[4]
-; CHECK-GI-FP16-NEXT:    umov w10, v0.h[5]
-; CHECK-GI-FP16-NEXT:    mov v4.s[1], w1
-; CHECK-GI-FP16-NEXT:    mov v2.s[1], w9
-; CHECK-GI-FP16-NEXT:    mov v5.s[1], v6.s[0]
-; CHECK-GI-FP16-NEXT:    ldr s6, [sp, #24]
-; CHECK-GI-FP16-NEXT:    mov v7.s[1], w5
-; CHECK-GI-FP16-NEXT:    mov v6.s[1], v16.s[0]
-; CHECK-GI-FP16-NEXT:    ldr s16, [sp, #40]
-; CHECK-GI-FP16-NEXT:    mov v1.s[0], w8
+; CHECK-GI-FP16-NEXT:    umov w9, v0.h[5]
+; CHECK-GI-FP16-NEXT:    mov v2.s[1], w10
+; CHECK-GI-FP16-NEXT:    mov v6.s[1], v3.s[0]
+; CHECK-GI-FP16-NEXT:    ldr s3, [sp, #8]
+; CHECK-GI-FP16-NEXT:    mov v17.s[1], w5
+; CHECK-GI-FP16-NEXT:    mov v7.s[1], v16.s[0]
+; CHECK-GI-FP16-NEXT:    mov v5.s[2], w2
+; CHECK-GI-FP16-NEXT:    fmov s1, w8
 ; CHECK-GI-FP16-NEXT:    umov w8, v0.h[6]
+; CHECK-GI-FP16-NEXT:    mov v2.s[2], w10
 ; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-FP16-NEXT:    mov v2.s[2], w9
-; CHECK-GI-FP16-NEXT:    mov v4.s[2], w2
-; CHECK-GI-FP16-NEXT:    mov v5.s[2], v17.s[0]
-; CHECK-GI-FP16-NEXT:    mov v7.s[2], w6
+; CHECK-GI-FP16-NEXT:    mov v6.s[2], v3.s[0]
+; CHECK-GI-FP16-NEXT:    ldr s3, [sp, #40]
+; CHECK-GI-FP16-NEXT:    mov v17.s[2], w6
+; CHECK-GI-FP16-NEXT:    mov v1.s[1], w9
+; CHECK-GI-FP16-NEXT:    mov v7.s[2], v3.s[0]
+; CHECK-GI-FP16-NEXT:    mov v5.s[3], w3
 ; CHECK-GI-FP16-NEXT:    shl v0.4s, v0.4s, #31
-; CHECK-GI-FP16-NEXT:    mov v6.s[2], v16.s[0]
-; CHECK-GI-FP16-NEXT:    mov v1.s[1], w10
-; CHECK-GI-FP16-NEXT:    mov w10, #-1 // =0xffffffff
-; CHECK-GI-FP16-NEXT:    mov v3.s[0], w10
-; CHECK-GI-FP16-NEXT:    mov v4.s[3], w3
-; CHECK-GI-FP16-NEXT:    sshr v0.4s, v0.4s, #31
 ; CHECK-GI-FP16-NEXT:    mov v1.s[2], w8
-; CHECK-GI-FP16-NEXT:    mov v3.s[1], w10
+; CHECK-GI-FP16-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-GI-FP16-NEXT:    sshr v0.4s, v0.4s, #31
+; CHECK-GI-FP16-NEXT:    fmov s4, w8
+; CHECK-GI-FP16-NEXT:    mov v4.s[1], w8
 ; CHECK-GI-FP16-NEXT:    ushl v1.4s, v1.4s, v2.4s
 ; CHECK-GI-FP16-NEXT:    neg v2.4s, v2.4s
-; CHECK-GI-FP16-NEXT:    mov v3.s[2], w10
 ; CHECK-GI-FP16-NEXT:    sshl v1.4s, v1.4s, v2.4s
 ; CHECK-GI-FP16-NEXT:    ldr s2, [sp, #16]
-; CHECK-GI-FP16-NEXT:    mov v5.s[3], v2.s[0]
-; CHECK-GI-FP16-NEXT:    eor v3.16b, v1.16b, v3.16b
-; CHECK-GI-FP16-NEXT:    and v1.16b, v7.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    and v2.16b, v6.16b, v3.16b
-; CHECK-GI-FP16-NEXT:    bsl v0.16b, v4.16b, v5.16b
-; CHECK-GI-FP16-NEXT:    orr v1.16b, v1.16b, v2.16b
-; CHECK-GI-FP16-NEXT:    mov s2, v0.s[1]
+; CHECK-GI-FP16-NEXT:    mov v4.s[2], w8
+; CHECK-GI-FP16-NEXT:    mov v6.s[3], v2.s[0]
+; CHECK-GI-FP16-NEXT:    eor v3.16b, v1.16b, v4.16b
+; CHECK-GI-FP16-NEXT:    and v1.16b, v17.16b, v1.16b
+; CHECK-GI-FP16-NEXT:    bsl v0.16b, v5.16b, v6.16b
+; CHECK-GI-FP16-NEXT:    and v2.16b, v7.16b, v3.16b
 ; CHECK-GI-FP16-NEXT:    mov s3, v0.s[2]
 ; CHECK-GI-FP16-NEXT:    mov s4, v0.s[3]
 ; CHECK-GI-FP16-NEXT:    fmov w0, s0
+; CHECK-GI-FP16-NEXT:    orr v1.16b, v1.16b, v2.16b
+; CHECK-GI-FP16-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-FP16-NEXT:    mov s5, v1.s[1]
 ; CHECK-GI-FP16-NEXT:    mov s6, v1.s[2]
-; CHECK-GI-FP16-NEXT:    fmov w4, s1
-; CHECK-GI-FP16-NEXT:    fmov w1, s2
 ; CHECK-GI-FP16-NEXT:    fmov w2, s3
+; CHECK-GI-FP16-NEXT:    fmov w1, s2
 ; CHECK-GI-FP16-NEXT:    fmov w3, s4
+; CHECK-GI-FP16-NEXT:    fmov w4, s1
 ; CHECK-GI-FP16-NEXT:    fmov w5, s5
 ; CHECK-GI-FP16-NEXT:    fmov w6, s6
 ; CHECK-GI-FP16-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fcopysign.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fcopysign.ll
index a42ec8e253be..3a5f7e2cd6b2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fcopysign.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fcopysign.ll
@@ -156,8 +156,8 @@ define <3 x float> @copysign_v3f32(<3 x float> %a, <3 x float> %b) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov w8, #-2147483648 // =0x80000000
 ; CHECK-GI-NEXT:    mov w9, #2147483647 // =0x7fffffff
-; CHECK-GI-NEXT:    mov v2.s[0], w9
-; CHECK-GI-NEXT:    mov v3.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fmov s3, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mov v3.s[1], w8
 ; CHECK-GI-NEXT:    mov v2.s[2], w9
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fdiv-combine.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fdiv-combine.ll
index 0627250d0779..d8f7f0a30668 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fdiv-combine.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fdiv-combine.ll
@@ -1,19 +1,33 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-unknown-unknown < %s -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:       warning: Instruction selection used fallback path for splat_fdiv_nxv4f32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for splat_three_fdiv_nxv4f32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for splat_fdiv_nxv2f64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for splat_two_fdiv_nxv2f64
 
 ; Following test cases check:
 ;   a / D; b / D; c / D;
 ;                =>
 ;   recip = 1.0 / D; a * recip; b * recip; c * recip;
 define void @three_fdiv_float(float %D, float %a, float %b, float %c) #0 {
-; CHECK-LABEL: three_fdiv_float:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov s4, #1.00000000
-; CHECK-NEXT:    fdiv s4, s4, s0
-; CHECK-NEXT:    fmul s0, s1, s4
-; CHECK-NEXT:    fmul s1, s2, s4
-; CHECK-NEXT:    fmul s2, s3, s4
-; CHECK-NEXT:    b foo_3f
+; CHECK-SD-LABEL: three_fdiv_float:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmov s4, #1.00000000
+; CHECK-SD-NEXT:    fdiv s4, s4, s0
+; CHECK-SD-NEXT:    fmul s0, s1, s4
+; CHECK-SD-NEXT:    fmul s1, s2, s4
+; CHECK-SD-NEXT:    fmul s2, s3, s4
+; CHECK-SD-NEXT:    b foo_3f
+;
+; CHECK-GI-LABEL: three_fdiv_float:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fdiv s4, s1, s0
+; CHECK-GI-NEXT:    fdiv s1, s2, s0
+; CHECK-GI-NEXT:    fdiv s2, s3, s0
+; CHECK-GI-NEXT:    fmov s0, s4
+; CHECK-GI-NEXT:    b foo_3f
   %div = fdiv float %a, %D
   %div1 = fdiv float %b, %D
   %div2 = fdiv float %c, %D
@@ -22,14 +36,22 @@ define void @three_fdiv_float(float %D, float %a, float %b, float %c) #0 {
 }
 
 define void @three_fdiv_double(double %D, double %a, double %b, double %c) #0 {
-; CHECK-LABEL: three_fdiv_double:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov d4, #1.00000000
-; CHECK-NEXT:    fdiv d4, d4, d0
-; CHECK-NEXT:    fmul d0, d1, d4
-; CHECK-NEXT:    fmul d1, d2, d4
-; CHECK-NEXT:    fmul d2, d3, d4
-; CHECK-NEXT:    b foo_3d
+; CHECK-SD-LABEL: three_fdiv_double:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmov d4, #1.00000000
+; CHECK-SD-NEXT:    fdiv d4, d4, d0
+; CHECK-SD-NEXT:    fmul d0, d1, d4
+; CHECK-SD-NEXT:    fmul d1, d2, d4
+; CHECK-SD-NEXT:    fmul d2, d3, d4
+; CHECK-SD-NEXT:    b foo_3d
+;
+; CHECK-GI-LABEL: three_fdiv_double:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fdiv d4, d1, d0
+; CHECK-GI-NEXT:    fdiv d1, d2, d0
+; CHECK-GI-NEXT:    fdiv d2, d3, d0
+; CHECK-GI-NEXT:    fmov d0, d4
+; CHECK-GI-NEXT:    b foo_3d
   %div = fdiv double %a, %D
   %div1 = fdiv double %b, %D
   %div2 = fdiv double %c, %D
@@ -38,14 +60,22 @@ define void @three_fdiv_double(double %D, double %a, double %b, double %c) #0 {
 }
 
 define void @three_fdiv_4xfloat(<4 x float> %D, <4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
-; CHECK-LABEL: three_fdiv_4xfloat:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov v4.4s, #1.00000000
-; CHECK-NEXT:    fdiv v4.4s, v4.4s, v0.4s
-; CHECK-NEXT:    fmul v0.4s, v1.4s, v4.4s
-; CHECK-NEXT:    fmul v1.4s, v2.4s, v4.4s
-; CHECK-NEXT:    fmul v2.4s, v3.4s, v4.4s
-; CHECK-NEXT:    b foo_3_4xf
+; CHECK-SD-LABEL: three_fdiv_4xfloat:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmov v4.4s, #1.00000000
+; CHECK-SD-NEXT:    fdiv v4.4s, v4.4s, v0.4s
+; CHECK-SD-NEXT:    fmul v0.4s, v1.4s, v4.4s
+; CHECK-SD-NEXT:    fmul v1.4s, v2.4s, v4.4s
+; CHECK-SD-NEXT:    fmul v2.4s, v3.4s, v4.4s
+; CHECK-SD-NEXT:    b foo_3_4xf
+;
+; CHECK-GI-LABEL: three_fdiv_4xfloat:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fdiv v4.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    fdiv v1.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    fdiv v2.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT:    mov v0.16b, v4.16b
+; CHECK-GI-NEXT:    b foo_3_4xf
   %div = fdiv <4 x float> %a, %D
   %div1 = fdiv <4 x float> %b, %D
   %div2 = fdiv <4 x float> %c, %D
@@ -54,14 +84,22 @@ define void @three_fdiv_4xfloat(<4 x float> %D, <4 x float> %a, <4 x float> %b,
 }
 
 define void @three_fdiv_2xdouble(<2 x double> %D, <2 x double> %a, <2 x double> %b, <2 x double> %c) #0 {
-; CHECK-LABEL: three_fdiv_2xdouble:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov v4.2d, #1.00000000
-; CHECK-NEXT:    fdiv v4.2d, v4.2d, v0.2d
-; CHECK-NEXT:    fmul v0.2d, v1.2d, v4.2d
-; CHECK-NEXT:    fmul v1.2d, v2.2d, v4.2d
-; CHECK-NEXT:    fmul v2.2d, v3.2d, v4.2d
-; CHECK-NEXT:    b foo_3_2xd
+; CHECK-SD-LABEL: three_fdiv_2xdouble:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmov v4.2d, #1.00000000
+; CHECK-SD-NEXT:    fdiv v4.2d, v4.2d, v0.2d
+; CHECK-SD-NEXT:    fmul v0.2d, v1.2d, v4.2d
+; CHECK-SD-NEXT:    fmul v1.2d, v2.2d, v4.2d
+; CHECK-SD-NEXT:    fmul v2.2d, v3.2d, v4.2d
+; CHECK-SD-NEXT:    b foo_3_2xd
+;
+; CHECK-GI-LABEL: three_fdiv_2xdouble:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fdiv v4.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT:    fdiv v1.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT:    fdiv v2.2d, v3.2d, v0.2d
+; CHECK-GI-NEXT:    mov v0.16b, v4.16b
+; CHECK-GI-NEXT:    b foo_3_2xd
   %div = fdiv <2 x double> %a, %D
   %div1 = fdiv <2 x double> %b, %D
   %div2 = fdiv <2 x double> %c, %D
@@ -98,16 +136,25 @@ define void @two_fdiv_double(double %D, double %a, double %b) #0 {
 }
 
 define void @splat_three_fdiv_4xfloat(float %D, <4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
-; CHECK-LABEL: splat_three_fdiv_4xfloat:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-NEXT:    fmov v4.4s, #1.00000000
-; CHECK-NEXT:    dup v0.4s, v0.s[0]
-; CHECK-NEXT:    fdiv v4.4s, v4.4s, v0.4s
-; CHECK-NEXT:    fmul v0.4s, v1.4s, v4.4s
-; CHECK-NEXT:    fmul v1.4s, v2.4s, v4.4s
-; CHECK-NEXT:    fmul v2.4s, v3.4s, v4.4s
-; CHECK-NEXT:    b foo_3_4xf
+; CHECK-SD-LABEL: splat_three_fdiv_4xfloat:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    fmov v4.4s, #1.00000000
+; CHECK-SD-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-SD-NEXT:    fdiv v4.4s, v4.4s, v0.4s
+; CHECK-SD-NEXT:    fmul v0.4s, v1.4s, v4.4s
+; CHECK-SD-NEXT:    fmul v1.4s, v2.4s, v4.4s
+; CHECK-SD-NEXT:    fmul v2.4s, v3.4s, v4.4s
+; CHECK-SD-NEXT:    b foo_3_4xf
+;
+; CHECK-GI-LABEL: splat_three_fdiv_4xfloat:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    dup v4.4s, v0.s[0]
+; CHECK-GI-NEXT:    fdiv v0.4s, v1.4s, v4.4s
+; CHECK-GI-NEXT:    fdiv v1.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    fdiv v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    b foo_3_4xf
   %D.ins = insertelement <4 x float> poison, float %D, i64 0
   %splat = shufflevector <4 x float> %D.ins, <4 x float> poison, <4 x i32> zeroinitializer
   %div = fdiv <4 x float> %a, %splat
@@ -118,14 +165,21 @@ define void @splat_three_fdiv_4xfloat(float %D, <4 x float> %a, <4 x float> %b,
 }
 
 define <4 x float> @splat_fdiv_v4f32(float %D, <4 x float> %a) #1 {
-; CHECK-LABEL: splat_fdiv_v4f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-NEXT:    fmov v2.4s, #1.00000000
-; CHECK-NEXT:    dup v0.4s, v0.s[0]
-; CHECK-NEXT:    fdiv v0.4s, v2.4s, v0.4s
-; CHECK-NEXT:    fmul v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: splat_fdiv_v4f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    fmov v2.4s, #1.00000000
+; CHECK-SD-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-SD-NEXT:    fdiv v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    fmul v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: splat_fdiv_v4f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    dup v0.4s, v0.s[0]
+; CHECK-GI-NEXT:    fdiv v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %D.ins = insertelement <4 x float> poison, float %D, i64 0
   %splat = shufflevector <4 x float> %D.ins, <4 x float> poison, <4 x i32> zeroinitializer
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll
index d9d80f1cb50e..1fbca7ca2c27 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll
@@ -118,12 +118,12 @@ define void @fptoui_2x_v8f32_to_v8i8_in_loop(ptr %A, ptr %B, ptr %dst) {
 ; CHECK-NEXT:    add x10, x0, x9
 ; CHECK-NEXT:    add x9, x1, x9
 ; CHECK-NEXT:    ldp q2, q1, [x10]
-; CHECK-NEXT:    fcvtzu.4s v5, v1
-; CHECK-NEXT:    ldp q1, q3, [x9]
-; CHECK-NEXT:    fcvtzu.4s v4, v2
-; CHECK-NEXT:    fcvtzu.4s v7, v3
+; CHECK-NEXT:    fcvtzu.4s v4, v1
+; CHECK-NEXT:    ldp q7, q1, [x9]
+; CHECK-NEXT:    fcvtzu.4s v3, v2
 ; CHECK-NEXT:    fcvtzu.4s v6, v1
-; CHECK-NEXT:    tbl.16b v1, { v4, v5, v6, v7 }, v0
+; CHECK-NEXT:    fcvtzu.4s v5, v7
+; CHECK-NEXT:    tbl.16b v1, { v3, v4, v5, v6 }, v0
 ; CHECK-NEXT:    str q1, [x2, x8, lsl #4]
 ; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    cmp x8, #1000
@@ -185,12 +185,12 @@ define void @fptoui_2x_v8f32_to_v8i8_in_loop_no_concat_shuffle(ptr %A, ptr %B, p
 ; CHECK-NEXT:    add x10, x0, x9
 ; CHECK-NEXT:    add x9, x1, x9
 ; CHECK-NEXT:    ldp q2, q1, [x10]
-; CHECK-NEXT:    fcvtzu.4s v5, v1
-; CHECK-NEXT:    ldp q1, q3, [x9]
-; CHECK-NEXT:    fcvtzu.4s v4, v2
-; CHECK-NEXT:    fcvtzu.4s v7, v3
+; CHECK-NEXT:    fcvtzu.4s v4, v1
+; CHECK-NEXT:    ldp q7, q1, [x9]
+; CHECK-NEXT:    fcvtzu.4s v3, v2
 ; CHECK-NEXT:    fcvtzu.4s v6, v1
-; CHECK-NEXT:    tbl.16b v1, { v4, v5, v6, v7 }, v0
+; CHECK-NEXT:    fcvtzu.4s v5, v7
+; CHECK-NEXT:    tbl.16b v1, { v3, v4, v5, v6 }, v0
 ; CHECK-NEXT:    str q1, [x2, x8, lsl #4]
 ; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    cmp x8, #1000
@@ -252,12 +252,12 @@ define void @fptoui_v16f32_to_v16i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    cmp x8, #1000
 ; CHECK-NEXT:    ldp q2, q1, [x9, #32]
-; CHECK-NEXT:    fcvtzu.4s v7, v1
-; CHECK-NEXT:    ldp q1, q3, [x9]
-; CHECK-NEXT:    fcvtzu.4s v6, v2
-; CHECK-NEXT:    fcvtzu.4s v5, v3
+; CHECK-NEXT:    fcvtzu.4s v6, v1
+; CHECK-NEXT:    ldp q7, q1, [x9]
+; CHECK-NEXT:    fcvtzu.4s v5, v2
 ; CHECK-NEXT:    fcvtzu.4s v4, v1
-; CHECK-NEXT:    tbl.16b v1, { v4, v5, v6, v7 }, v0
+; CHECK-NEXT:    fcvtzu.4s v3, v7
+; CHECK-NEXT:    tbl.16b v1, { v3, v4, v5, v6 }, v0
 ; CHECK-NEXT:    str q1, [x1], #32
 ; CHECK-NEXT:    b.eq LBB4_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fptoi.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fptoi.ll
index ae3b6a54a1f7..6a792958ee6b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fptoi.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fptoi.ll
@@ -7225,7 +7225,7 @@ define <2 x i64> @fptos_v2f128_v2i64(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov x19, x0
 ; CHECK-GI-NEXT:    bl __fixtfdi
-; CHECK-GI-NEXT:    mov v0.d[0], x19
+; CHECK-GI-NEXT:    fmov d0, x19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.d[1], x0
 ; CHECK-GI-NEXT:    add sp, sp, #32
@@ -7268,7 +7268,7 @@ define <2 x i64> @fptou_v2f128_v2i64(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov x19, x0
 ; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    mov v0.d[0], x19
+; CHECK-GI-NEXT:    fmov d0, x19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.d[1], x0
 ; CHECK-GI-NEXT:    add sp, sp, #32
@@ -7424,7 +7424,7 @@ define <2 x i32> @fptos_v2f128_v2i32(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -7467,7 +7467,7 @@ define <2 x i32> @fptou_v2f128_v2i32(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -7519,7 +7519,7 @@ define <3 x i32> @fptos_v3f128_v3i32(<3 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w20, w0
 ; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w20
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
@@ -7572,7 +7572,7 @@ define <3 x i32> @fptou_v3f128_v3i32(<3 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w20, w0
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w20
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
@@ -7616,7 +7616,7 @@ define <2 x i16> @fptos_v2f128_v2i16(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -7659,7 +7659,7 @@ define <2 x i16> @fptou_v2f128_v2i16(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -7816,7 +7816,7 @@ define <2 x i8> @fptos_v2f128_v2i8(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -7859,7 +7859,7 @@ define <2 x i8> @fptou_v2f128_v2i8(<2 x fp128> %a) {
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index a33b1ef569fc..9c21d2bf083a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -32,8 +32,7 @@ define <1 x i32> @test_signed_v1f32_v1i32(<1 x float> %f) {
 ; CHECK-GI-LABEL: test_signed_v1f32_v1i32:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    fcvtzs w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
     %x = call <1 x i32> @llvm.fptosi.sat.v1f32.v1i32(<1 x float> %f)
     ret <1 x i32> %x
@@ -244,18 +243,11 @@ declare <5 x i32> @llvm.fptosi.sat.v5f64.v5i32 (<5 x double>)
 declare <6 x i32> @llvm.fptosi.sat.v6f64.v6i32 (<6 x double>)
 
 define <1 x i32> @test_signed_v1f64_v1i32(<1 x double> %f) {
-; CHECK-SD-LABEL: test_signed_v1f64_v1i32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fcvtzs w8, d0
-; CHECK-SD-NEXT:    fmov s0, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_signed_v1f64_v1i32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcvtzs w8, d0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_signed_v1f64_v1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs w8, d0
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    ret
     %x = call <1 x i32> @llvm.fptosi.sat.v1f64.v1i32(<1 x double> %f)
     ret <1 x i32> %x
 }
@@ -565,8 +557,7 @@ define <1 x i32> @test_signed_v1f128_v1i32(<1 x fp128> %f) {
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    add sp, sp, #48
 ; CHECK-GI-NEXT:    ret
     %x = call <1 x i32> @llvm.fptosi.sat.v1f128.v1i32(<1 x fp128> %f)
@@ -708,7 +699,7 @@ define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    mov v1.16b, v0.16b
 ; CHECK-GI-NEXT:    bl __unordtf2
-; CHECK-GI-NEXT:    mov v0.s[0], w21
+; CHECK-GI-NEXT:    fmov s0, w21
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    csel w8, wzr, w19, ne
@@ -726,7 +717,7 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-LABEL: test_signed_v3f128_v3i32:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    sub sp, sp, #128
-; CHECK-SD-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x30, x23, [sp, #80] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    stp x22, x21, [sp, #96] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    stp x20, x19, [sp, #112] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 128
@@ -734,13 +725,13 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    .cfi_offset w20, -16
 ; CHECK-SD-NEXT:    .cfi_offset w21, -24
 ; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w23, -40
 ; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    stp q0, q2, [sp, #48] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v2.16b, v1.16b
+; CHECK-SD-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    adrp x8, .LCPI16_0
-; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
 ; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    stp q2, q1, [sp, #32] // 32-byte Folded Spill
+; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
 ; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __getf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -755,15 +746,15 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov w21, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT:    mov w22, #2147483647 // =0x7fffffff
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w19, w21, w19, gt
+; CHECK-SD-NEXT:    csel w19, w22, w19, gt
 ; CHECK-SD-NEXT:    mov v1.16b, v0.16b
 ; CHECK-SD-NEXT:    bl __unordtf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w22, wzr, w19, ne
+; CHECK-SD-NEXT:    csel w21, wzr, w19, ne
 ; CHECK-SD-NEXT:    bl __getf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov w19, w0
@@ -775,16 +766,13 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w19, w21, w19, gt
+; CHECK-SD-NEXT:    csel w19, w22, w19, gt
 ; CHECK-SD-NEXT:    mov v1.16b, v0.16b
 ; CHECK-SD-NEXT:    bl __unordtf2
-; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    csel w8, wzr, w19, ne
-; CHECK-SD-NEXT:    fmov s0, w8
-; CHECK-SD-NEXT:    mov v0.s[1], w22
-; CHECK-SD-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    cmp w0, #0
+; CHECK-SD-NEXT:    csel w23, wzr, w19, ne
 ; CHECK-SD-NEXT:    bl __getf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov w19, w0
@@ -796,16 +784,17 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w19, w21, w19, gt
+; CHECK-SD-NEXT:    csel w19, w22, w19, gt
 ; CHECK-SD-NEXT:    mov v1.16b, v0.16b
 ; CHECK-SD-NEXT:    bl __unordtf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-SD-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-SD-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    fmov s0, w8
+; CHECK-SD-NEXT:    mov v0.s[1], w23
+; CHECK-SD-NEXT:    ldp x30, x23, [sp, #80] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov v0.s[2], w21
 ; CHECK-SD-NEXT:    ldp x22, x21, [sp, #96] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[2], w8
 ; CHECK-SD-NEXT:    add sp, sp, #128
 ; CHECK-SD-NEXT:    ret
 ;
@@ -903,7 +892,7 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    mov v1.16b, v0.16b
 ; CHECK-GI-NEXT:    bl __unordtf2
-; CHECK-GI-NEXT:    mov v0.s[0], w21
+; CHECK-GI-NEXT:    fmov s0, w21
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
@@ -1143,7 +1132,7 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov w19, w0
 ; CHECK-GI-NEXT:    mov v1.16b, v0.16b
 ; CHECK-GI-NEXT:    bl __unordtf2
-; CHECK-GI-NEXT:    mov v0.s[0], w21
+; CHECK-GI-NEXT:    fmov s0, w21
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    csel w8, wzr, w19, ne
@@ -1190,15 +1179,13 @@ define <1 x i32> @test_signed_v1f16_v1i32(<1 x half> %f) {
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
 ; CHECK-GI-CVT-NEXT:    fcvtzs w8, s0
-; CHECK-GI-CVT-NEXT:    mov v0.s[0], w8
-; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-CVT-NEXT:    fmov s0, w8
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_v1f16_v1i32:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    fcvtzs w8, h0
-; CHECK-GI-FP16-NEXT:    mov v0.s[0], w8
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call <1 x i32> @llvm.fptosi.sat.v1f16.v1i32(<1 x half> %f)
     ret <1 x i32> %x
@@ -4890,63 +4877,61 @@ define <16 x i16> @test_signed_v16f16_v16i16(<16 x half> %f) {
 define <8 x i8> @test_signed_v8f64_v8i8(<8 x double> %f) {
 ; CHECK-SD-LABEL: test_signed_v8f64_v8i8:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov d4, v3.d[1]
-; CHECK-SD-NEXT:    fcvtzs w11, d3
-; CHECK-SD-NEXT:    mov w9, #127 // =0x7f
-; CHECK-SD-NEXT:    mov d3, v1.d[1]
-; CHECK-SD-NEXT:    fcvtzs w13, d2
-; CHECK-SD-NEXT:    fcvtzs w15, d1
-; CHECK-SD-NEXT:    fcvtzs w17, d0
-; CHECK-SD-NEXT:    fcvtzs w8, d4
-; CHECK-SD-NEXT:    mov d4, v2.d[1]
-; CHECK-SD-NEXT:    mov d2, v0.d[1]
-; CHECK-SD-NEXT:    fcvtzs w14, d3
-; CHECK-SD-NEXT:    cmp w8, #127
-; CHECK-SD-NEXT:    fcvtzs w12, d4
-; CHECK-SD-NEXT:    fcvtzs w16, d2
-; CHECK-SD-NEXT:    csel w10, w8, w9, lt
-; CHECK-SD-NEXT:    mov w8, #-128 // =0xffffff80
+; CHECK-SD-NEXT:    mov d4, v0.d[1]
+; CHECK-SD-NEXT:    fcvtzs w10, d0
+; CHECK-SD-NEXT:    mov w8, #127 // =0x7f
+; CHECK-SD-NEXT:    fcvtzs w12, d1
+; CHECK-SD-NEXT:    mov d1, v1.d[1]
+; CHECK-SD-NEXT:    fcvtzs w9, d4
+; CHECK-SD-NEXT:    cmp w9, #127
+; CHECK-SD-NEXT:    csel w11, w9, w8, lt
+; CHECK-SD-NEXT:    mov w9, #-128 // =0xffffff80
+; CHECK-SD-NEXT:    cmn w11, #128
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    cmn w10, #128
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    cmp w12, #127
+; CHECK-SD-NEXT:    fmov s0, w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    mov d1, v2.d[1]
+; CHECK-SD-NEXT:    mov v0.b[1], w11
+; CHECK-SD-NEXT:    csel w11, w12, w8, lt
+; CHECK-SD-NEXT:    cmn w11, #128
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    mov v0.b[2], w11
+; CHECK-SD-NEXT:    fcvtzs w11, d2
 ; CHECK-SD-NEXT:    cmn w10, #128
-; CHECK-SD-NEXT:    csel w10, w10, w8, gt
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
 ; CHECK-SD-NEXT:    cmp w11, #127
-; CHECK-SD-NEXT:    csel w11, w11, w9, lt
+; CHECK-SD-NEXT:    mov v0.b[3], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    mov d1, v3.d[1]
 ; CHECK-SD-NEXT:    cmn w11, #128
-; CHECK-SD-NEXT:    csel w11, w11, w8, gt
-; CHECK-SD-NEXT:    cmp w12, #127
-; CHECK-SD-NEXT:    csel w12, w12, w9, lt
-; CHECK-SD-NEXT:    fmov s3, w11
-; CHECK-SD-NEXT:    cmn w12, #128
-; CHECK-SD-NEXT:    csel w12, w12, w8, gt
-; CHECK-SD-NEXT:    cmp w13, #127
-; CHECK-SD-NEXT:    csel w13, w13, w9, lt
-; CHECK-SD-NEXT:    mov v3.s[1], w10
-; CHECK-SD-NEXT:    cmn w13, #128
-; CHECK-SD-NEXT:    csel w13, w13, w8, gt
-; CHECK-SD-NEXT:    cmp w14, #127
-; CHECK-SD-NEXT:    csel w14, w14, w9, lt
-; CHECK-SD-NEXT:    fmov s2, w13
-; CHECK-SD-NEXT:    cmn w14, #128
-; CHECK-SD-NEXT:    csel w14, w14, w8, gt
-; CHECK-SD-NEXT:    cmp w15, #127
-; CHECK-SD-NEXT:    csel w15, w15, w9, lt
-; CHECK-SD-NEXT:    mov v2.s[1], w12
-; CHECK-SD-NEXT:    cmn w15, #128
-; CHECK-SD-NEXT:    csel w15, w15, w8, gt
-; CHECK-SD-NEXT:    cmp w16, #127
-; CHECK-SD-NEXT:    csel w11, w16, w9, lt
-; CHECK-SD-NEXT:    fmov s1, w15
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[4], w11
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    fcvtzs w11, d3
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    cmn w10, #128
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[5], w10
+; CHECK-SD-NEXT:    cmp w11, #127
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
 ; CHECK-SD-NEXT:    cmn w11, #128
-; CHECK-SD-NEXT:    csel w10, w11, w8, gt
-; CHECK-SD-NEXT:    cmp w17, #127
-; CHECK-SD-NEXT:    csel w9, w17, w9, lt
-; CHECK-SD-NEXT:    mov v1.s[1], w14
-; CHECK-SD-NEXT:    cmn w9, #128
-; CHECK-SD-NEXT:    csel w8, w9, w8, gt
-; CHECK-SD-NEXT:    fmov s0, w8
-; CHECK-SD-NEXT:    adrp x8, .LCPI82_0
-; CHECK-SD-NEXT:    ldr d4, [x8, :lo12:.LCPI82_0]
-; CHECK-SD-NEXT:    mov v0.s[1], w10
-; CHECK-SD-NEXT:    tbl v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v4.8b
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[6], w11
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w8, w10, w8, lt
+; CHECK-SD-NEXT:    cmn w8, #128
+; CHECK-SD-NEXT:    csel w8, w8, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[7], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_signed_v8f64_v8i8:
@@ -4990,11 +4975,9 @@ define <16 x i8> @test_signed_v16f64_v16i8(<16 x double> %f) {
 ; CHECK-SD-NEXT:    mov d16, v0.d[1]
 ; CHECK-SD-NEXT:    fcvtzs w10, d0
 ; CHECK-SD-NEXT:    mov w8, #127 // =0x7f
-; CHECK-SD-NEXT:    mov d0, v1.d[1]
-; CHECK-SD-NEXT:    fcvtzs w13, d1
-; CHECK-SD-NEXT:    mov d1, v2.d[1]
+; CHECK-SD-NEXT:    fcvtzs w12, d1
+; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    fcvtzs w9, d16
-; CHECK-SD-NEXT:    fcvtzs w12, d0
 ; CHECK-SD-NEXT:    cmp w9, #127
 ; CHECK-SD-NEXT:    csel w11, w9, w8, lt
 ; CHECK-SD-NEXT:    mov w9, #-128 // =0xffffff80
@@ -5006,115 +4989,94 @@ define <16 x i8> @test_signed_v16f64_v16i8(<16 x double> %f) {
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
 ; CHECK-SD-NEXT:    cmp w12, #127
 ; CHECK-SD-NEXT:    fmov s0, w10
-; CHECK-SD-NEXT:    csel w10, w12, w8, lt
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    mov d1, v2.d[1]
+; CHECK-SD-NEXT:    mov v0.b[1], w11
+; CHECK-SD-NEXT:    csel w11, w12, w8, lt
+; CHECK-SD-NEXT:    cmn w11, #128
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    mov v0.b[2], w11
+; CHECK-SD-NEXT:    fcvtzs w11, d2
 ; CHECK-SD-NEXT:    cmn w10, #128
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
-; CHECK-SD-NEXT:    cmp w13, #127
-; CHECK-SD-NEXT:    csel w12, w13, w8, lt
-; CHECK-SD-NEXT:    mov v0.s[1], w11
-; CHECK-SD-NEXT:    fcvtzs w11, d1
-; CHECK-SD-NEXT:    cmn w12, #128
-; CHECK-SD-NEXT:    csel w12, w12, w9, gt
-; CHECK-SD-NEXT:    fmov s1, w12
-; CHECK-SD-NEXT:    fcvtzs w12, d2
-; CHECK-SD-NEXT:    mov d2, v3.d[1]
 ; CHECK-SD-NEXT:    cmp w11, #127
-; CHECK-SD-NEXT:    mov w13, v0.s[1]
-; CHECK-SD-NEXT:    mov v1.s[1], w10
-; CHECK-SD-NEXT:    csel w10, w11, w8, lt
+; CHECK-SD-NEXT:    mov v0.b[3], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    mov d1, v3.d[1]
+; CHECK-SD-NEXT:    cmn w11, #128
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[4], w11
+; CHECK-SD-NEXT:    fcvtzs w11, d3
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w10, #128
-; CHECK-SD-NEXT:    fcvtzs w11, d2
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
-; CHECK-SD-NEXT:    cmp w12, #127
-; CHECK-SD-NEXT:    mov v0.b[1], w13
-; CHECK-SD-NEXT:    csel w12, w12, w8, lt
-; CHECK-SD-NEXT:    cmn w12, #128
-; CHECK-SD-NEXT:    mov w13, v1.s[1]
-; CHECK-SD-NEXT:    csel w12, w12, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[5], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
 ; CHECK-SD-NEXT:    cmp w11, #127
-; CHECK-SD-NEXT:    fmov s2, w12
-; CHECK-SD-NEXT:    fcvtzs w12, d3
-; CHECK-SD-NEXT:    mov d3, v4.d[1]
-; CHECK-SD-NEXT:    mov v0.b[2], v1.b[0]
-; CHECK-SD-NEXT:    mov v2.s[1], w10
-; CHECK-SD-NEXT:    csel w10, w11, w8, lt
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    mov d1, v4.d[1]
+; CHECK-SD-NEXT:    cmn w11, #128
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[6], w11
+; CHECK-SD-NEXT:    fcvtzs w11, d4
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w10, #128
-; CHECK-SD-NEXT:    fcvtzs w11, d3
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
-; CHECK-SD-NEXT:    cmp w12, #127
-; CHECK-SD-NEXT:    mov v0.b[3], w13
-; CHECK-SD-NEXT:    csel w12, w12, w8, lt
-; CHECK-SD-NEXT:    cmn w12, #128
-; CHECK-SD-NEXT:    mov w13, v2.s[1]
-; CHECK-SD-NEXT:    csel w12, w12, w9, gt
 ; CHECK-SD-NEXT:    cmp w11, #127
-; CHECK-SD-NEXT:    fmov s3, w12
-; CHECK-SD-NEXT:    fcvtzs w12, d4
-; CHECK-SD-NEXT:    mov v0.b[4], v2.b[0]
-; CHECK-SD-NEXT:    mov d4, v5.d[1]
-; CHECK-SD-NEXT:    mov v3.s[1], w10
-; CHECK-SD-NEXT:    csel w10, w11, w8, lt
+; CHECK-SD-NEXT:    mov v0.b[7], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    mov d1, v5.d[1]
+; CHECK-SD-NEXT:    cmn w11, #128
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[8], w11
+; CHECK-SD-NEXT:    fcvtzs w11, d5
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w10, #128
-; CHECK-SD-NEXT:    mov v0.b[5], w13
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
-; CHECK-SD-NEXT:    cmp w12, #127
-; CHECK-SD-NEXT:    fcvtzs w11, d4
-; CHECK-SD-NEXT:    csel w12, w12, w8, lt
-; CHECK-SD-NEXT:    cmn w12, #128
-; CHECK-SD-NEXT:    mov w13, v3.s[1]
-; CHECK-SD-NEXT:    csel w12, w12, w9, gt
-; CHECK-SD-NEXT:    mov v0.b[6], v3.b[0]
-; CHECK-SD-NEXT:    fmov s4, w12
-; CHECK-SD-NEXT:    fcvtzs w12, d5
+; CHECK-SD-NEXT:    mov v0.b[9], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
 ; CHECK-SD-NEXT:    cmp w11, #127
-; CHECK-SD-NEXT:    mov d5, v6.d[1]
-; CHECK-SD-NEXT:    mov v4.s[1], w10
-; CHECK-SD-NEXT:    csel w10, w11, w8, lt
-; CHECK-SD-NEXT:    mov v0.b[7], w13
-; CHECK-SD-NEXT:    cmn w10, #128
-; CHECK-SD-NEXT:    csel w10, w10, w9, gt
-; CHECK-SD-NEXT:    cmp w12, #127
-; CHECK-SD-NEXT:    fcvtzs w13, d5
-; CHECK-SD-NEXT:    csel w11, w12, w8, lt
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    mov d1, v6.d[1]
 ; CHECK-SD-NEXT:    cmn w11, #128
-; CHECK-SD-NEXT:    mov w12, v4.s[1]
-; CHECK-SD-NEXT:    mov v0.b[8], v4.b[0]
 ; CHECK-SD-NEXT:    csel w11, w11, w9, gt
-; CHECK-SD-NEXT:    fmov s5, w11
+; CHECK-SD-NEXT:    mov v0.b[10], w11
 ; CHECK-SD-NEXT:    fcvtzs w11, d6
-; CHECK-SD-NEXT:    cmp w13, #127
-; CHECK-SD-NEXT:    mov d6, v7.d[1]
-; CHECK-SD-NEXT:    mov v0.b[9], w12
-; CHECK-SD-NEXT:    mov v5.s[1], w10
-; CHECK-SD-NEXT:    csel w10, w13, w8, lt
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w10, #128
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
 ; CHECK-SD-NEXT:    cmp w11, #127
-; CHECK-SD-NEXT:    fcvtzs w13, d6
+; CHECK-SD-NEXT:    mov v0.b[11], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
 ; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    mov d1, v7.d[1]
 ; CHECK-SD-NEXT:    cmn w11, #128
-; CHECK-SD-NEXT:    mov v0.b[10], v5.b[0]
-; CHECK-SD-NEXT:    mov w12, v5.s[1]
 ; CHECK-SD-NEXT:    csel w11, w11, w9, gt
-; CHECK-SD-NEXT:    fmov s6, w11
+; CHECK-SD-NEXT:    mov v0.b[12], w11
+; CHECK-SD-NEXT:    cmp w10, #127
 ; CHECK-SD-NEXT:    fcvtzs w11, d7
-; CHECK-SD-NEXT:    cmp w13, #127
-; CHECK-SD-NEXT:    mov v0.b[11], w12
-; CHECK-SD-NEXT:    mov v6.s[1], w10
-; CHECK-SD-NEXT:    csel w10, w13, w8, lt
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w10, #128
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[13], w10
 ; CHECK-SD-NEXT:    cmp w11, #127
-; CHECK-SD-NEXT:    csel w8, w11, w8, lt
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    cmn w11, #128
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v0.b[14], w11
+; CHECK-SD-NEXT:    cmp w10, #127
+; CHECK-SD-NEXT:    csel w8, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w8, #128
-; CHECK-SD-NEXT:    mov v0.b[12], v6.b[0]
-; CHECK-SD-NEXT:    mov w11, v6.s[1]
 ; CHECK-SD-NEXT:    csel w8, w8, w9, gt
-; CHECK-SD-NEXT:    fmov s7, w8
-; CHECK-SD-NEXT:    mov v0.b[13], w11
-; CHECK-SD-NEXT:    mov v7.s[1], w10
-; CHECK-SD-NEXT:    mov v0.b[14], v7.b[0]
-; CHECK-SD-NEXT:    mov w8, v7.s[1]
 ; CHECK-SD-NEXT:    mov v0.b[15], w8
 ; CHECK-SD-NEXT:    ret
 ;
@@ -5179,63 +5141,60 @@ define <16 x i8> @test_signed_v16f64_v16i8(<16 x double> %f) {
 define <8 x i16> @test_signed_v8f64_v8i16(<8 x double> %f) {
 ; CHECK-SD-LABEL: test_signed_v8f64_v8i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov d4, v3.d[1]
+; CHECK-SD-NEXT:    mov d4, v0.d[1]
 ; CHECK-SD-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-SD-NEXT:    fcvtzs w11, d3
-; CHECK-SD-NEXT:    mov d3, v1.d[1]
-; CHECK-SD-NEXT:    fcvtzs w13, d2
-; CHECK-SD-NEXT:    fcvtzs w15, d1
-; CHECK-SD-NEXT:    fcvtzs w17, d0
+; CHECK-SD-NEXT:    fcvtzs w10, d0
+; CHECK-SD-NEXT:    fcvtzs w12, d1
+; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    fcvtzs w9, d4
-; CHECK-SD-NEXT:    mov d4, v2.d[1]
-; CHECK-SD-NEXT:    mov d2, v0.d[1]
-; CHECK-SD-NEXT:    fcvtzs w14, d3
 ; CHECK-SD-NEXT:    cmp w9, w8
-; CHECK-SD-NEXT:    fcvtzs w12, d4
-; CHECK-SD-NEXT:    fcvtzs w16, d2
-; CHECK-SD-NEXT:    csel w10, w9, w8, lt
+; CHECK-SD-NEXT:    csel w11, w9, w8, lt
 ; CHECK-SD-NEXT:    mov w9, #-32768 // =0xffff8000
+; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    cmn w10, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    cmp w12, w8
+; CHECK-SD-NEXT:    fmov s0, w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    mov d1, v2.d[1]
+; CHECK-SD-NEXT:    mov v0.h[1], w11
+; CHECK-SD-NEXT:    csel w11, w12, w8, lt
+; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    mov v0.h[2], w11
+; CHECK-SD-NEXT:    fcvtzs w11, d2
 ; CHECK-SD-NEXT:    cmn w10, #8, lsl #12 // =32768
 ; CHECK-SD-NEXT:    csel w10, w10, w9, gt
 ; CHECK-SD-NEXT:    cmp w11, w8
+; CHECK-SD-NEXT:    mov v0.h[3], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d1
 ; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    mov d1, v3.d[1]
 ; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
 ; CHECK-SD-NEXT:    csel w11, w11, w9, gt
-; CHECK-SD-NEXT:    cmp w12, w8
-; CHECK-SD-NEXT:    csel w12, w12, w8, lt
-; CHECK-SD-NEXT:    fmov s3, w11
-; CHECK-SD-NEXT:    cmn w12, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w12, w12, w9, gt
-; CHECK-SD-NEXT:    cmp w13, w8
-; CHECK-SD-NEXT:    csel w13, w13, w8, lt
-; CHECK-SD-NEXT:    mov v3.s[1], w10
-; CHECK-SD-NEXT:    cmn w13, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w13, w13, w9, gt
-; CHECK-SD-NEXT:    cmp w14, w8
-; CHECK-SD-NEXT:    csel w14, w14, w8, lt
-; CHECK-SD-NEXT:    fmov s2, w13
-; CHECK-SD-NEXT:    cmn w14, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w14, w14, w9, gt
-; CHECK-SD-NEXT:    cmp w15, w8
-; CHECK-SD-NEXT:    csel w15, w15, w8, lt
-; CHECK-SD-NEXT:    mov v2.s[1], w12
-; CHECK-SD-NEXT:    cmn w15, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w15, w15, w9, gt
-; CHECK-SD-NEXT:    cmp w16, w8
-; CHECK-SD-NEXT:    csel w11, w16, w8, lt
-; CHECK-SD-NEXT:    fmov s1, w15
+; CHECK-SD-NEXT:    mov v0.h[4], w11
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    fcvtzs w11, d3
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    cmn w10, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    mov v0.h[5], w10
+; CHECK-SD-NEXT:    cmp w11, w8
+; CHECK-SD-NEXT:    fcvtzs w10, d1
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
 ; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w10, w11, w9, gt
-; CHECK-SD-NEXT:    cmp w17, w8
-; CHECK-SD-NEXT:    csel w8, w17, w8, lt
-; CHECK-SD-NEXT:    mov v1.s[1], w14
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v0.h[6], w11
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    csel w8, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w8, #8, lsl #12 // =32768
 ; CHECK-SD-NEXT:    csel w8, w8, w9, gt
-; CHECK-SD-NEXT:    fmov s0, w8
-; CHECK-SD-NEXT:    adrp x8, .LCPI84_0
-; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI84_0]
-; CHECK-SD-NEXT:    mov v0.s[1], w10
-; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v4.16b
+; CHECK-SD-NEXT:    mov v0.h[7], w8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_signed_v8f64_v8i16:
@@ -5275,116 +5234,112 @@ define <8 x i16> @test_signed_v8f64_v8i16(<8 x double> %f) {
 define <16 x i16> @test_signed_v16f64_v16i16(<16 x double> %f) {
 ; CHECK-SD-LABEL: test_signed_v16f64_v16i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov d16, v3.d[1]
-; CHECK-SD-NEXT:    mov w9, #32767 // =0x7fff
-; CHECK-SD-NEXT:    fcvtzs w11, d3
-; CHECK-SD-NEXT:    mov d3, v1.d[1]
-; CHECK-SD-NEXT:    fcvtzs w14, d2
-; CHECK-SD-NEXT:    fcvtzs w15, d1
-; CHECK-SD-NEXT:    mov d1, v7.d[1]
-; CHECK-SD-NEXT:    fcvtzs w18, d0
-; CHECK-SD-NEXT:    fcvtzs w1, d7
-; CHECK-SD-NEXT:    fcvtzs w2, d6
-; CHECK-SD-NEXT:    fcvtzs w4, d5
-; CHECK-SD-NEXT:    fcvtzs w6, d4
-; CHECK-SD-NEXT:    fcvtzs w8, d16
-; CHECK-SD-NEXT:    mov d16, v2.d[1]
-; CHECK-SD-NEXT:    mov d2, v0.d[1]
-; CHECK-SD-NEXT:    mov d0, v6.d[1]
+; CHECK-SD-NEXT:    mov d16, v0.d[1]
+; CHECK-SD-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-SD-NEXT:    fcvtzs w11, d0
+; CHECK-SD-NEXT:    mov d0, v1.d[1]
+; CHECK-SD-NEXT:    fcvtzs w12, d1
+; CHECK-SD-NEXT:    fcvtzs w15, d3
+; CHECK-SD-NEXT:    mov d1, v4.d[1]
+; CHECK-SD-NEXT:    fcvtzs w1, d5
+; CHECK-SD-NEXT:    fcvtzs w9, d16
+; CHECK-SD-NEXT:    fcvtzs w14, d0
+; CHECK-SD-NEXT:    mov d0, v2.d[1]
 ; CHECK-SD-NEXT:    fcvtzs w0, d1
-; CHECK-SD-NEXT:    cmp w8, w9
-; CHECK-SD-NEXT:    fcvtzs w13, d16
-; CHECK-SD-NEXT:    fcvtzs w17, d2
-; CHECK-SD-NEXT:    csel w10, w8, w9, lt
-; CHECK-SD-NEXT:    mov w8, #-32768 // =0xffff8000
+; CHECK-SD-NEXT:    cmp w9, w8
+; CHECK-SD-NEXT:    csel w10, w9, w8, lt
+; CHECK-SD-NEXT:    mov w9, #-32768 // =0xffff8000
 ; CHECK-SD-NEXT:    cmn w10, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w10, w10, w8, gt
-; CHECK-SD-NEXT:    cmp w11, w9
-; CHECK-SD-NEXT:    csel w11, w11, w9, lt
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    cmp w11, w8
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
 ; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w12, w11, w8, gt
-; CHECK-SD-NEXT:    cmp w13, w9
-; CHECK-SD-NEXT:    csel w11, w13, w9, lt
-; CHECK-SD-NEXT:    fcvtzs w13, d3
+; CHECK-SD-NEXT:    csel w13, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w12, w8
+; CHECK-SD-NEXT:    csel w11, w12, w8, lt
+; CHECK-SD-NEXT:    fcvtzs w12, d2
+; CHECK-SD-NEXT:    mov d2, v5.d[1]
 ; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w11, w11, w8, gt
-; CHECK-SD-NEXT:    cmp w14, w9
-; CHECK-SD-NEXT:    csel w14, w14, w9, lt
-; CHECK-SD-NEXT:    cmn w14, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w14, w14, w8, gt
-; CHECK-SD-NEXT:    cmp w13, w9
-; CHECK-SD-NEXT:    csel w13, w13, w9, lt
-; CHECK-SD-NEXT:    cmn w13, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w13, w13, w8, gt
-; CHECK-SD-NEXT:    cmp w15, w9
-; CHECK-SD-NEXT:    csel w15, w15, w9, lt
-; CHECK-SD-NEXT:    cmn w15, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w16, w15, w8, gt
-; CHECK-SD-NEXT:    cmp w17, w9
-; CHECK-SD-NEXT:    csel w15, w17, w9, lt
+; CHECK-SD-NEXT:    csel w17, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w14, w8
+; CHECK-SD-NEXT:    csel w11, w14, w8, lt
+; CHECK-SD-NEXT:    fcvtzs w14, d0
+; CHECK-SD-NEXT:    mov d0, v3.d[1]
+; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    cmp w12, w8
+; CHECK-SD-NEXT:    csel w12, w12, w8, lt
+; CHECK-SD-NEXT:    cmn w12, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    fcvtzs w18, d0
+; CHECK-SD-NEXT:    fmov s0, w13
+; CHECK-SD-NEXT:    csel w16, w12, w9, gt
+; CHECK-SD-NEXT:    cmp w14, w8
+; CHECK-SD-NEXT:    csel w12, w14, w8, lt
+; CHECK-SD-NEXT:    cmn w12, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d2
+; CHECK-SD-NEXT:    csel w14, w12, w9, gt
+; CHECK-SD-NEXT:    cmp w15, w8
+; CHECK-SD-NEXT:    mov d2, v6.d[1]
+; CHECK-SD-NEXT:    csel w12, w15, w8, lt
+; CHECK-SD-NEXT:    cmn w12, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w12, w12, w9, gt
+; CHECK-SD-NEXT:    cmp w18, w8
+; CHECK-SD-NEXT:    mov v0.h[2], w17
+; CHECK-SD-NEXT:    csel w15, w18, w8, lt
+; CHECK-SD-NEXT:    fcvtzs w18, d4
 ; CHECK-SD-NEXT:    cmn w15, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w15, w15, w8, gt
-; CHECK-SD-NEXT:    cmp w18, w9
-; CHECK-SD-NEXT:    csel w17, w18, w9, lt
-; CHECK-SD-NEXT:    cmn w17, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w17, w17, w8, gt
-; CHECK-SD-NEXT:    cmp w0, w9
-; CHECK-SD-NEXT:    csel w18, w0, w9, lt
-; CHECK-SD-NEXT:    fcvtzs w0, d0
-; CHECK-SD-NEXT:    mov d0, v5.d[1]
-; CHECK-SD-NEXT:    cmn w18, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w18, w18, w8, gt
-; CHECK-SD-NEXT:    cmp w1, w9
-; CHECK-SD-NEXT:    csel w1, w1, w9, lt
-; CHECK-SD-NEXT:    cmn w1, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    fcvtzs w3, d0
-; CHECK-SD-NEXT:    mov d0, v4.d[1]
-; CHECK-SD-NEXT:    csel w1, w1, w8, gt
-; CHECK-SD-NEXT:    cmp w0, w9
-; CHECK-SD-NEXT:    csel w0, w0, w9, lt
-; CHECK-SD-NEXT:    fmov s7, w1
+; CHECK-SD-NEXT:    csel w15, w15, w9, gt
+; CHECK-SD-NEXT:    cmp w0, w8
+; CHECK-SD-NEXT:    csel w0, w0, w8, lt
+; CHECK-SD-NEXT:    mov v0.h[3], w11
 ; CHECK-SD-NEXT:    cmn w0, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w0, w0, w8, gt
-; CHECK-SD-NEXT:    cmp w2, w9
-; CHECK-SD-NEXT:    fcvtzs w5, d0
-; CHECK-SD-NEXT:    csel w2, w2, w9, lt
-; CHECK-SD-NEXT:    fmov s3, w12
-; CHECK-SD-NEXT:    mov v7.s[1], w18
-; CHECK-SD-NEXT:    cmn w2, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w2, w2, w8, gt
-; CHECK-SD-NEXT:    cmp w3, w9
-; CHECK-SD-NEXT:    csel w3, w3, w9, lt
-; CHECK-SD-NEXT:    mov v3.s[1], w10
-; CHECK-SD-NEXT:    fmov s6, w2
-; CHECK-SD-NEXT:    cmn w3, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    fmov s2, w14
-; CHECK-SD-NEXT:    csel w3, w3, w8, gt
-; CHECK-SD-NEXT:    cmp w4, w9
-; CHECK-SD-NEXT:    csel w4, w4, w9, lt
-; CHECK-SD-NEXT:    mov v6.s[1], w0
-; CHECK-SD-NEXT:    cmn w4, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    mov v2.s[1], w11
-; CHECK-SD-NEXT:    csel w12, w4, w8, gt
-; CHECK-SD-NEXT:    cmp w5, w9
-; CHECK-SD-NEXT:    fmov s1, w16
-; CHECK-SD-NEXT:    csel w10, w5, w9, lt
-; CHECK-SD-NEXT:    fmov s5, w12
+; CHECK-SD-NEXT:    csel w0, w0, w9, gt
+; CHECK-SD-NEXT:    cmp w18, w8
+; CHECK-SD-NEXT:    csel w18, w18, w8, lt
+; CHECK-SD-NEXT:    cmn w18, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    mov v0.h[4], w16
+; CHECK-SD-NEXT:    csel w13, w18, w9, gt
+; CHECK-SD-NEXT:    cmp w1, w8
+; CHECK-SD-NEXT:    fmov s1, w13
+; CHECK-SD-NEXT:    csel w13, w1, w8, lt
+; CHECK-SD-NEXT:    cmn w13, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w13, w13, w9, gt
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    mov v0.h[5], w14
+; CHECK-SD-NEXT:    mov v1.h[1], w0
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
 ; CHECK-SD-NEXT:    cmn w10, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w10, w10, w8, gt
-; CHECK-SD-NEXT:    cmp w6, w9
-; CHECK-SD-NEXT:    mov v1.s[1], w13
-; CHECK-SD-NEXT:    csel w9, w6, w9, lt
-; CHECK-SD-NEXT:    mov v5.s[1], w3
-; CHECK-SD-NEXT:    fmov s0, w17
-; CHECK-SD-NEXT:    cmn w9, #8, lsl #12 // =32768
-; CHECK-SD-NEXT:    csel w8, w9, w8, gt
-; CHECK-SD-NEXT:    fmov s4, w8
-; CHECK-SD-NEXT:    mov v0.s[1], w15
-; CHECK-SD-NEXT:    adrp x8, .LCPI85_0
-; CHECK-SD-NEXT:    ldr q16, [x8, :lo12:.LCPI85_0]
-; CHECK-SD-NEXT:    mov v4.s[1], w10
-; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v16.16b
-; CHECK-SD-NEXT:    tbl v1.16b, { v4.16b, v5.16b, v6.16b, v7.16b }, v16.16b
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    mov v0.h[6], w12
+; CHECK-SD-NEXT:    mov v1.h[2], w13
+; CHECK-SD-NEXT:    fcvtzs w13, d6
+; CHECK-SD-NEXT:    mov v0.h[7], w15
+; CHECK-SD-NEXT:    cmp w13, w8
+; CHECK-SD-NEXT:    mov v1.h[3], w10
+; CHECK-SD-NEXT:    fcvtzs w10, d2
+; CHECK-SD-NEXT:    csel w11, w13, w8, lt
+; CHECK-SD-NEXT:    mov d2, v7.d[1]
+; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v1.h[4], w11
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    fcvtzs w11, d7
+; CHECK-SD-NEXT:    csel w10, w10, w8, lt
+; CHECK-SD-NEXT:    cmn w10, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w10, w10, w9, gt
+; CHECK-SD-NEXT:    mov v1.h[5], w10
+; CHECK-SD-NEXT:    cmp w11, w8
+; CHECK-SD-NEXT:    fcvtzs w10, d2
+; CHECK-SD-NEXT:    csel w11, w11, w8, lt
+; CHECK-SD-NEXT:    cmn w11, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w11, w11, w9, gt
+; CHECK-SD-NEXT:    mov v1.h[6], w11
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    csel w8, w10, w8, lt
+; CHECK-SD-NEXT:    cmn w8, #8, lsl #12 // =32768
+; CHECK-SD-NEXT:    csel w8, w8, w9, gt
+; CHECK-SD-NEXT:    mov v1.h[7], w8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_signed_v16f64_v16i16:
@@ -5579,7 +5534,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov x19, x0
 ; CHECK-GI-NEXT:    mov v1.16b, v0.16b
 ; CHECK-GI-NEXT:    bl __unordtf2
-; CHECK-GI-NEXT:    mov v0.d[0], x21
+; CHECK-GI-NEXT:    fmov d0, x21
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    csel x8, xzr, x19, ne
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index b1b5154a57b4..44847a41287d 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -32,8 +32,7 @@ define <1 x i32> @test_unsigned_v1f32_v1i32(<1 x float> %f) {
 ; CHECK-GI-LABEL: test_unsigned_v1f32_v1i32:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    fcvtzu w8, s0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
     %x = call <1 x i32> @llvm.fptoui.sat.v1f32.v1i32(<1 x float> %f)
     ret <1 x i32> %x
@@ -244,18 +243,11 @@ declare <5 x i32> @llvm.fptoui.sat.v5f64.v5i32 (<5 x double>)
 declare <6 x i32> @llvm.fptoui.sat.v6f64.v6i32 (<6 x double>)
 
 define <1 x i32> @test_unsigned_v1f64_v1i32(<1 x double> %f) {
-; CHECK-SD-LABEL: test_unsigned_v1f64_v1i32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fcvtzu w8, d0
-; CHECK-SD-NEXT:    fmov s0, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_unsigned_v1f64_v1i32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fcvtzu w8, d0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_unsigned_v1f64_v1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu w8, d0
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    ret
     %x = call <1 x i32> @llvm.fptoui.sat.v1f64.v1i32(<1 x double> %f)
     ret <1 x i32> %x
 }
@@ -513,9 +505,8 @@ define <1 x i32> @test_unsigned_v1f128_v1i32(<1 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[0], w0
+; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    add sp, sp, #48
 ; CHECK-GI-NEXT:    ret
     %x = call <1 x i32> @llvm.fptoui.sat.v1f128.v1i32(<1 x fp128> %f)
@@ -629,7 +620,7 @@ define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    csel x8, x22, x21, lt
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
@@ -645,18 +636,18 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-LABEL: test_unsigned_v3f128_v3i32:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    sub sp, sp, #112
-; CHECK-SD-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x30, x21, [sp, #80] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-SD-NEXT:    .cfi_offset w19, -8
 ; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
 ; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    stp q0, q2, [sp, #48] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v2.16b, v1.16b
+; CHECK-SD-NEXT:    stp q1, q0, [sp, #48] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    adrp x8, .LCPI16_0
-; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
 ; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
+; CHECK-SD-NEXT:    str q2, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __getf2
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -681,13 +672,10 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
 ; CHECK-SD-NEXT:    bl __gttf2
-; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    csinv w8, w19, wzr, le
-; CHECK-SD-NEXT:    fmov s0, w8
-; CHECK-SD-NEXT:    mov v0.s[1], w20
-; CHECK-SD-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    cmp w0, #0
+; CHECK-SD-NEXT:    csinv w21, w19, wzr, le
 ; CHECK-SD-NEXT:    bl __getf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov w19, w0
@@ -698,11 +686,12 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-SD-NEXT:    csinv w8, w19, wzr, le
+; CHECK-SD-NEXT:    fmov s0, w8
+; CHECK-SD-NEXT:    mov v0.s[1], w21
+; CHECK-SD-NEXT:    ldp x30, x21, [sp, #80] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov v0.s[2], w20
 ; CHECK-SD-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[2], w8
 ; CHECK-SD-NEXT:    add sp, sp, #112
 ; CHECK-SD-NEXT:    ret
 ;
@@ -782,7 +771,7 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    csel x8, x23, x21, lt
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x30, x23, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w20
@@ -976,7 +965,7 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    csel x8, x24, x22, lt
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov v0.s[0], w19
+; CHECK-GI-NEXT:    fmov s0, w19
 ; CHECK-GI-NEXT:    ldp x24, x23, [sp, #96] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.s[1], w20
@@ -1021,15 +1010,13 @@ define <1 x i32> @test_unsigned_v1f16_v1i32(<1 x half> %f) {
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
 ; CHECK-GI-CVT-NEXT:    fcvtzu w8, s0
-; CHECK-GI-CVT-NEXT:    mov v0.s[0], w8
-; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-CVT-NEXT:    fmov s0, w8
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_v1f16_v1i32:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    fcvtzu w8, h0
-; CHECK-GI-FP16-NEXT:    mov v0.s[0], w8
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call <1 x i32> @llvm.fptoui.sat.v1f16.v1i32(<1 x half> %f)
     ret <1 x i32> %x
@@ -4048,46 +4035,44 @@ define <16 x i16> @test_unsigned_v16f16_v16i16(<16 x half> %f) {
 define <8 x i8> @test_unsigned_v8f64_v8i8(<8 x double> %f) {
 ; CHECK-SD-LABEL: test_unsigned_v8f64_v8i8:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov d4, v3.d[1]
-; CHECK-SD-NEXT:    mov d5, v2.d[1]
-; CHECK-SD-NEXT:    mov w11, #255 // =0xff
-; CHECK-SD-NEXT:    fcvtzu w9, d3
-; CHECK-SD-NEXT:    mov d3, v1.d[1]
-; CHECK-SD-NEXT:    fcvtzu w12, d2
-; CHECK-SD-NEXT:    fcvtzu w14, d1
-; CHECK-SD-NEXT:    fcvtzu w8, d4
 ; CHECK-SD-NEXT:    mov d4, v0.d[1]
-; CHECK-SD-NEXT:    fcvtzu w10, d5
-; CHECK-SD-NEXT:    fcvtzu w13, d3
-; CHECK-SD-NEXT:    cmp w8, #255
-; CHECK-SD-NEXT:    fcvtzu w15, d4
-; CHECK-SD-NEXT:    csel w8, w8, w11, lo
+; CHECK-SD-NEXT:    fcvtzu w10, d0
+; CHECK-SD-NEXT:    mov w8, #255 // =0xff
+; CHECK-SD-NEXT:    fcvtzu w11, d1
+; CHECK-SD-NEXT:    mov d1, v1.d[1]
+; CHECK-SD-NEXT:    fcvtzu w9, d4
+; CHECK-SD-NEXT:    cmp w9, #255
+; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    cmp w10, #255
+; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    cmp w11, #255
+; CHECK-SD-NEXT:    fmov s0, w10
+; CHECK-SD-NEXT:    csel w10, w11, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[1], w9
+; CHECK-SD-NEXT:    fcvtzu w9, d1
+; CHECK-SD-NEXT:    mov d1, v2.d[1]
+; CHECK-SD-NEXT:    mov v0.b[2], w10
 ; CHECK-SD-NEXT:    cmp w9, #255
-; CHECK-SD-NEXT:    csel w9, w9, w11, lo
+; CHECK-SD-NEXT:    fcvtzu w10, d2
+; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[3], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
-; CHECK-SD-NEXT:    fmov s4, w9
-; CHECK-SD-NEXT:    csel w9, w10, w11, lo
-; CHECK-SD-NEXT:    cmp w12, #255
-; CHECK-SD-NEXT:    fcvtzu w10, d0
-; CHECK-SD-NEXT:    mov v4.s[1], w8
-; CHECK-SD-NEXT:    csel w8, w12, w11, lo
-; CHECK-SD-NEXT:    cmp w13, #255
-; CHECK-SD-NEXT:    fmov s3, w8
-; CHECK-SD-NEXT:    csel w8, w13, w11, lo
-; CHECK-SD-NEXT:    cmp w14, #255
-; CHECK-SD-NEXT:    mov v3.s[1], w9
-; CHECK-SD-NEXT:    csel w9, w14, w11, lo
-; CHECK-SD-NEXT:    cmp w15, #255
-; CHECK-SD-NEXT:    fmov s2, w9
-; CHECK-SD-NEXT:    csel w9, w15, w11, lo
+; CHECK-SD-NEXT:    fcvtzu w9, d1
+; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    mov d1, v3.d[1]
+; CHECK-SD-NEXT:    mov v0.b[4], w10
+; CHECK-SD-NEXT:    cmp w9, #255
+; CHECK-SD-NEXT:    fcvtzu w10, d3
+; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[5], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
-; CHECK-SD-NEXT:    mov v2.s[1], w8
-; CHECK-SD-NEXT:    csel w8, w10, w11, lo
-; CHECK-SD-NEXT:    fmov s1, w8
-; CHECK-SD-NEXT:    adrp x8, .LCPI82_0
-; CHECK-SD-NEXT:    ldr d0, [x8, :lo12:.LCPI82_0]
-; CHECK-SD-NEXT:    mov v1.s[1], w9
-; CHECK-SD-NEXT:    tbl v0.8b, { v1.16b, v2.16b, v3.16b, v4.16b }, v0.8b
+; CHECK-SD-NEXT:    fcvtzu w9, d1
+; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[6], w10
+; CHECK-SD-NEXT:    cmp w9, #255
+; CHECK-SD-NEXT:    csel w8, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[7], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_unsigned_v8f64_v8i8:
@@ -4120,98 +4105,75 @@ define <16 x i8> @test_unsigned_v16f64_v16i8(<16 x double> %f) {
 ; CHECK-SD-NEXT:    mov d16, v0.d[1]
 ; CHECK-SD-NEXT:    fcvtzu w10, d0
 ; CHECK-SD-NEXT:    mov w8, #255 // =0xff
+; CHECK-SD-NEXT:    fcvtzu w11, d1
+; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    fcvtzu w9, d16
-; CHECK-SD-NEXT:    mov d16, v1.d[1]
 ; CHECK-SD-NEXT:    cmp w9, #255
 ; CHECK-SD-NEXT:    csel w9, w9, w8, lo
 ; CHECK-SD-NEXT:    cmp w10, #255
 ; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    cmp w11, #255
 ; CHECK-SD-NEXT:    fmov s0, w10
-; CHECK-SD-NEXT:    fcvtzu w10, d16
-; CHECK-SD-NEXT:    mov d16, v2.d[1]
-; CHECK-SD-NEXT:    mov v0.s[1], w9
+; CHECK-SD-NEXT:    csel w10, w11, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[1], w9
 ; CHECK-SD-NEXT:    fcvtzu w9, d1
-; CHECK-SD-NEXT:    cmp w10, #255
-; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    mov d1, v2.d[1]
+; CHECK-SD-NEXT:    mov v0.b[2], w10
 ; CHECK-SD-NEXT:    cmp w9, #255
-; CHECK-SD-NEXT:    mov w11, v0.s[1]
-; CHECK-SD-NEXT:    csel w9, w9, w8, lo
-; CHECK-SD-NEXT:    fmov s1, w9
-; CHECK-SD-NEXT:    fcvtzu w9, d16
-; CHECK-SD-NEXT:    mov d16, v3.d[1]
-; CHECK-SD-NEXT:    mov v0.b[1], w11
-; CHECK-SD-NEXT:    mov v1.s[1], w10
 ; CHECK-SD-NEXT:    fcvtzu w10, d2
-; CHECK-SD-NEXT:    cmp w9, #255
 ; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[3], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
-; CHECK-SD-NEXT:    mov w11, v1.s[1]
-; CHECK-SD-NEXT:    mov v0.b[2], v1.b[0]
+; CHECK-SD-NEXT:    fcvtzu w9, d1
 ; CHECK-SD-NEXT:    csel w10, w10, w8, lo
-; CHECK-SD-NEXT:    fmov s2, w10
-; CHECK-SD-NEXT:    fcvtzu w10, d16
-; CHECK-SD-NEXT:    mov d16, v4.d[1]
-; CHECK-SD-NEXT:    mov v0.b[3], w11
-; CHECK-SD-NEXT:    mov v2.s[1], w9
-; CHECK-SD-NEXT:    fcvtzu w9, d3
+; CHECK-SD-NEXT:    mov d1, v3.d[1]
+; CHECK-SD-NEXT:    mov v0.b[4], w10
+; CHECK-SD-NEXT:    cmp w9, #255
+; CHECK-SD-NEXT:    fcvtzu w10, d3
+; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[5], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
+; CHECK-SD-NEXT:    fcvtzu w9, d1
 ; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    mov d1, v4.d[1]
+; CHECK-SD-NEXT:    mov v0.b[6], w10
 ; CHECK-SD-NEXT:    cmp w9, #255
-; CHECK-SD-NEXT:    mov w11, v2.s[1]
-; CHECK-SD-NEXT:    mov v0.b[4], v2.b[0]
-; CHECK-SD-NEXT:    csel w9, w9, w8, lo
-; CHECK-SD-NEXT:    fmov s3, w9
-; CHECK-SD-NEXT:    fcvtzu w9, d16
-; CHECK-SD-NEXT:    mov d16, v5.d[1]
-; CHECK-SD-NEXT:    mov v0.b[5], w11
-; CHECK-SD-NEXT:    mov v3.s[1], w10
 ; CHECK-SD-NEXT:    fcvtzu w10, d4
-; CHECK-SD-NEXT:    cmp w9, #255
 ; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[7], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
-; CHECK-SD-NEXT:    mov w11, v3.s[1]
-; CHECK-SD-NEXT:    mov v0.b[6], v3.b[0]
+; CHECK-SD-NEXT:    fcvtzu w9, d1
 ; CHECK-SD-NEXT:    csel w10, w10, w8, lo
-; CHECK-SD-NEXT:    fmov s4, w10
-; CHECK-SD-NEXT:    fcvtzu w10, d16
-; CHECK-SD-NEXT:    mov v0.b[7], w11
-; CHECK-SD-NEXT:    mov v4.s[1], w9
-; CHECK-SD-NEXT:    fcvtzu w9, d5
-; CHECK-SD-NEXT:    mov d5, v6.d[1]
+; CHECK-SD-NEXT:    mov d1, v5.d[1]
+; CHECK-SD-NEXT:    mov v0.b[8], w10
+; CHECK-SD-NEXT:    cmp w9, #255
+; CHECK-SD-NEXT:    fcvtzu w10, d5
+; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[9], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
+; CHECK-SD-NEXT:    fcvtzu w9, d1
 ; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    mov d1, v6.d[1]
+; CHECK-SD-NEXT:    mov v0.b[10], w10
 ; CHECK-SD-NEXT:    cmp w9, #255
-; CHECK-SD-NEXT:    mov w11, v4.s[1]
-; CHECK-SD-NEXT:    mov v0.b[8], v4.b[0]
-; CHECK-SD-NEXT:    csel w9, w9, w8, lo
-; CHECK-SD-NEXT:    fmov s16, w9
-; CHECK-SD-NEXT:    fcvtzu w9, d5
-; CHECK-SD-NEXT:    mov d5, v7.d[1]
-; CHECK-SD-NEXT:    mov v0.b[9], w11
-; CHECK-SD-NEXT:    mov v16.s[1], w10
 ; CHECK-SD-NEXT:    fcvtzu w10, d6
-; CHECK-SD-NEXT:    cmp w9, #255
 ; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[11], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
-; CHECK-SD-NEXT:    mov v0.b[10], v16.b[0]
-; CHECK-SD-NEXT:    mov w11, v16.s[1]
+; CHECK-SD-NEXT:    fcvtzu w9, d1
 ; CHECK-SD-NEXT:    csel w10, w10, w8, lo
-; CHECK-SD-NEXT:    fmov s6, w10
-; CHECK-SD-NEXT:    fcvtzu w10, d7
-; CHECK-SD-NEXT:    mov v0.b[11], w11
-; CHECK-SD-NEXT:    mov v6.s[1], w9
-; CHECK-SD-NEXT:    fcvtzu w9, d5
+; CHECK-SD-NEXT:    mov d1, v7.d[1]
+; CHECK-SD-NEXT:    mov v0.b[12], w10
 ; CHECK-SD-NEXT:    cmp w9, #255
-; CHECK-SD-NEXT:    mov v0.b[12], v6.b[0]
-; CHECK-SD-NEXT:    mov w11, v6.s[1]
+; CHECK-SD-NEXT:    fcvtzu w10, d7
 ; CHECK-SD-NEXT:    csel w9, w9, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[13], w9
 ; CHECK-SD-NEXT:    cmp w10, #255
-; CHECK-SD-NEXT:    csel w8, w10, w8, lo
-; CHECK-SD-NEXT:    fmov s5, w8
-; CHECK-SD-NEXT:    mov v0.b[13], w11
-; CHECK-SD-NEXT:    mov v5.s[1], w9
-; CHECK-SD-NEXT:    mov v0.b[14], v5.b[0]
-; CHECK-SD-NEXT:    mov w8, v5.s[1]
+; CHECK-SD-NEXT:    fcvtzu w9, d1
+; CHECK-SD-NEXT:    csel w10, w10, w8, lo
+; CHECK-SD-NEXT:    mov v0.b[14], w10
+; CHECK-SD-NEXT:    cmp w9, #255
+; CHECK-SD-NEXT:    csel w8, w9, w8, lo
 ; CHECK-SD-NEXT:    mov v0.b[15], w8
 ; CHECK-SD-NEXT:    ret
 ;
@@ -4257,46 +4219,43 @@ define <16 x i8> @test_unsigned_v16f64_v16i8(<16 x double> %f) {
 define <8 x i16> @test_unsigned_v8f64_v8i16(<8 x double> %f) {
 ; CHECK-SD-LABEL: test_unsigned_v8f64_v8i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov d4, v3.d[1]
-; CHECK-SD-NEXT:    mov d5, v2.d[1]
+; CHECK-SD-NEXT:    mov d4, v0.d[1]
+; CHECK-SD-NEXT:    fcvtzu w9, d0
 ; CHECK-SD-NEXT:    mov w10, #65535 // =0xffff
-; CHECK-SD-NEXT:    fcvtzu w9, d3
-; CHECK-SD-NEXT:    mov d3, v1.d[1]
-; CHECK-SD-NEXT:    fcvtzu w12, d2
-; CHECK-SD-NEXT:    fcvtzu w14, d1
+; CHECK-SD-NEXT:    fcvtzu w11, d1
+; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    fcvtzu w8, d4
-; CHECK-SD-NEXT:    mov d4, v0.d[1]
-; CHECK-SD-NEXT:    fcvtzu w11, d5
-; CHECK-SD-NEXT:    fcvtzu w13, d3
 ; CHECK-SD-NEXT:    cmp w8, w10
-; CHECK-SD-NEXT:    fcvtzu w15, d4
 ; CHECK-SD-NEXT:    csel w8, w8, w10, lo
 ; CHECK-SD-NEXT:    cmp w9, w10
 ; CHECK-SD-NEXT:    csel w9, w9, w10, lo
 ; CHECK-SD-NEXT:    cmp w11, w10
-; CHECK-SD-NEXT:    fmov s4, w9
+; CHECK-SD-NEXT:    fmov s0, w9
 ; CHECK-SD-NEXT:    csel w9, w11, w10, lo
-; CHECK-SD-NEXT:    cmp w12, w10
-; CHECK-SD-NEXT:    fcvtzu w11, d0
-; CHECK-SD-NEXT:    mov v4.s[1], w8
-; CHECK-SD-NEXT:    csel w8, w12, w10, lo
-; CHECK-SD-NEXT:    cmp w13, w10
-; CHECK-SD-NEXT:    fmov s3, w8
-; CHECK-SD-NEXT:    csel w8, w13, w10, lo
-; CHECK-SD-NEXT:    cmp w14, w10
-; CHECK-SD-NEXT:    mov v3.s[1], w9
-; CHECK-SD-NEXT:    csel w9, w14, w10, lo
-; CHECK-SD-NEXT:    cmp w15, w10
-; CHECK-SD-NEXT:    fmov s2, w9
-; CHECK-SD-NEXT:    csel w9, w15, w10, lo
-; CHECK-SD-NEXT:    cmp w11, w10
-; CHECK-SD-NEXT:    mov v2.s[1], w8
-; CHECK-SD-NEXT:    csel w8, w11, w10, lo
-; CHECK-SD-NEXT:    fmov s1, w8
-; CHECK-SD-NEXT:    adrp x8, .LCPI84_0
-; CHECK-SD-NEXT:    ldr q0, [x8, :lo12:.LCPI84_0]
-; CHECK-SD-NEXT:    mov v1.s[1], w9
-; CHECK-SD-NEXT:    tbl v0.16b, { v1.16b, v2.16b, v3.16b, v4.16b }, v0.16b
+; CHECK-SD-NEXT:    mov v0.h[1], w8
+; CHECK-SD-NEXT:    fcvtzu w8, d1
+; CHECK-SD-NEXT:    mov d1, v2.d[1]
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    cmp w8, w10
+; CHECK-SD-NEXT:    fcvtzu w9, d2
+; CHECK-SD-NEXT:    csel w8, w8, w10, lo
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    cmp w9, w10
+; CHECK-SD-NEXT:    fcvtzu w8, d1
+; CHECK-SD-NEXT:    csel w9, w9, w10, lo
+; CHECK-SD-NEXT:    mov d1, v3.d[1]
+; CHECK-SD-NEXT:    mov v0.h[4], w9
+; CHECK-SD-NEXT:    cmp w8, w10
+; CHECK-SD-NEXT:    fcvtzu w9, d3
+; CHECK-SD-NEXT:    csel w8, w8, w10, lo
+; CHECK-SD-NEXT:    mov v0.h[5], w8
+; CHECK-SD-NEXT:    cmp w9, w10
+; CHECK-SD-NEXT:    fcvtzu w8, d1
+; CHECK-SD-NEXT:    csel w9, w9, w10, lo
+; CHECK-SD-NEXT:    mov v0.h[6], w9
+; CHECK-SD-NEXT:    cmp w8, w10
+; CHECK-SD-NEXT:    csel w8, w8, w10, lo
+; CHECK-SD-NEXT:    mov v0.h[7], w8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_unsigned_v8f64_v8i16:
@@ -4325,83 +4284,79 @@ define <8 x i16> @test_unsigned_v8f64_v8i16(<8 x double> %f) {
 define <16 x i16> @test_unsigned_v16f64_v16i16(<16 x double> %f) {
 ; CHECK-SD-LABEL: test_unsigned_v16f64_v16i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov d16, v3.d[1]
-; CHECK-SD-NEXT:    mov d17, v2.d[1]
+; CHECK-SD-NEXT:    mov d16, v0.d[1]
+; CHECK-SD-NEXT:    mov d17, v1.d[1]
 ; CHECK-SD-NEXT:    mov w8, #65535 // =0xffff
-; CHECK-SD-NEXT:    fcvtzu w9, d3
-; CHECK-SD-NEXT:    mov d3, v1.d[1]
-; CHECK-SD-NEXT:    fcvtzu w10, d1
-; CHECK-SD-NEXT:    mov d1, v0.d[1]
-; CHECK-SD-NEXT:    fcvtzu w11, d2
-; CHECK-SD-NEXT:    fcvtzu w12, d0
-; CHECK-SD-NEXT:    mov d0, v7.d[1]
-; CHECK-SD-NEXT:    mov d2, v6.d[1]
-; CHECK-SD-NEXT:    fcvtzu w14, d7
-; CHECK-SD-NEXT:    fcvtzu w13, d16
-; CHECK-SD-NEXT:    fcvtzu w16, d17
-; CHECK-SD-NEXT:    fcvtzu w15, d6
-; CHECK-SD-NEXT:    fcvtzu w17, d3
-; CHECK-SD-NEXT:    mov d6, v5.d[1]
-; CHECK-SD-NEXT:    mov d3, v4.d[1]
-; CHECK-SD-NEXT:    fcvtzu w18, d1
-; CHECK-SD-NEXT:    cmp w13, w8
-; CHECK-SD-NEXT:    csel w13, w13, w8, lo
+; CHECK-SD-NEXT:    fcvtzu w10, d0
+; CHECK-SD-NEXT:    mov d0, v2.d[1]
+; CHECK-SD-NEXT:    fcvtzu w11, d1
+; CHECK-SD-NEXT:    mov d1, v3.d[1]
+; CHECK-SD-NEXT:    fcvtzu w13, d2
+; CHECK-SD-NEXT:    mov d2, v4.d[1]
+; CHECK-SD-NEXT:    fcvtzu w18, d4
+; CHECK-SD-NEXT:    fcvtzu w9, d16
+; CHECK-SD-NEXT:    fcvtzu w12, d17
+; CHECK-SD-NEXT:    fcvtzu w16, d1
+; CHECK-SD-NEXT:    fcvtzu w17, d2
+; CHECK-SD-NEXT:    mov d2, v5.d[1]
 ; CHECK-SD-NEXT:    cmp w9, w8
-; CHECK-SD-NEXT:    csel w9, w9, w8, lo
-; CHECK-SD-NEXT:    cmp w16, w8
-; CHECK-SD-NEXT:    fmov s19, w9
-; CHECK-SD-NEXT:    csel w9, w16, w8, lo
+; CHECK-SD-NEXT:    csel w14, w9, w8, lo
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    fcvtzu w9, d0
+; CHECK-SD-NEXT:    csel w15, w10, w8, lo
 ; CHECK-SD-NEXT:    cmp w11, w8
-; CHECK-SD-NEXT:    fcvtzu w16, d0
+; CHECK-SD-NEXT:    fcvtzu w10, d3
 ; CHECK-SD-NEXT:    csel w11, w11, w8, lo
-; CHECK-SD-NEXT:    cmp w17, w8
-; CHECK-SD-NEXT:    mov v19.s[1], w13
-; CHECK-SD-NEXT:    csel w13, w17, w8, lo
-; CHECK-SD-NEXT:    cmp w10, w8
-; CHECK-SD-NEXT:    csel w10, w10, w8, lo
-; CHECK-SD-NEXT:    cmp w18, w8
-; CHECK-SD-NEXT:    fmov s18, w11
-; CHECK-SD-NEXT:    csel w11, w18, w8, lo
 ; CHECK-SD-NEXT:    cmp w12, w8
-; CHECK-SD-NEXT:    fcvtzu w17, d2
+; CHECK-SD-NEXT:    fmov s0, w15
 ; CHECK-SD-NEXT:    csel w12, w12, w8, lo
+; CHECK-SD-NEXT:    cmp w13, w8
+; CHECK-SD-NEXT:    csel w13, w13, w8, lo
+; CHECK-SD-NEXT:    cmp w9, w8
+; CHECK-SD-NEXT:    csel w0, w9, w8, lo
+; CHECK-SD-NEXT:    cmp w10, w8
+; CHECK-SD-NEXT:    mov v0.h[1], w14
+; CHECK-SD-NEXT:    csel w10, w10, w8, lo
 ; CHECK-SD-NEXT:    cmp w16, w8
-; CHECK-SD-NEXT:    fcvtzu w18, d6
-; CHECK-SD-NEXT:    mov v18.s[1], w9
+; CHECK-SD-NEXT:    fcvtzu w14, d2
 ; CHECK-SD-NEXT:    csel w9, w16, w8, lo
-; CHECK-SD-NEXT:    cmp w14, w8
-; CHECK-SD-NEXT:    fmov s17, w10
-; CHECK-SD-NEXT:    csel w10, w14, w8, lo
-; CHECK-SD-NEXT:    fcvtzu w16, d5
-; CHECK-SD-NEXT:    fmov s23, w10
 ; CHECK-SD-NEXT:    cmp w17, w8
-; CHECK-SD-NEXT:    fcvtzu w14, d3
-; CHECK-SD-NEXT:    csel w10, w17, w8, lo
-; CHECK-SD-NEXT:    cmp w15, w8
-; CHECK-SD-NEXT:    fcvtzu w17, d4
-; CHECK-SD-NEXT:    mov v17.s[1], w13
-; CHECK-SD-NEXT:    mov v23.s[1], w9
-; CHECK-SD-NEXT:    csel w9, w15, w8, lo
+; CHECK-SD-NEXT:    mov d2, v6.d[1]
+; CHECK-SD-NEXT:    csel w16, w17, w8, lo
 ; CHECK-SD-NEXT:    cmp w18, w8
-; CHECK-SD-NEXT:    fmov s22, w9
-; CHECK-SD-NEXT:    csel w9, w18, w8, lo
-; CHECK-SD-NEXT:    cmp w16, w8
-; CHECK-SD-NEXT:    fmov s16, w12
-; CHECK-SD-NEXT:    mov v22.s[1], w10
-; CHECK-SD-NEXT:    csel w10, w16, w8, lo
+; CHECK-SD-NEXT:    csel w17, w18, w8, lo
+; CHECK-SD-NEXT:    fcvtzu w18, d5
+; CHECK-SD-NEXT:    mov v0.h[2], w11
+; CHECK-SD-NEXT:    fmov s1, w17
+; CHECK-SD-NEXT:    fcvtzu w11, d6
+; CHECK-SD-NEXT:    mov v1.h[1], w16
+; CHECK-SD-NEXT:    cmp w18, w8
+; CHECK-SD-NEXT:    mov v0.h[3], w12
+; CHECK-SD-NEXT:    csel w15, w18, w8, lo
 ; CHECK-SD-NEXT:    cmp w14, w8
-; CHECK-SD-NEXT:    fmov s21, w10
-; CHECK-SD-NEXT:    csel w10, w14, w8, lo
-; CHECK-SD-NEXT:    cmp w17, w8
-; CHECK-SD-NEXT:    csel w8, w17, w8, lo
-; CHECK-SD-NEXT:    mov v16.s[1], w11
-; CHECK-SD-NEXT:    mov v21.s[1], w9
-; CHECK-SD-NEXT:    fmov s20, w8
-; CHECK-SD-NEXT:    adrp x8, .LCPI85_0
-; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI85_0]
-; CHECK-SD-NEXT:    mov v20.s[1], w10
-; CHECK-SD-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
-; CHECK-SD-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
+; CHECK-SD-NEXT:    fcvtzu w12, d2
+; CHECK-SD-NEXT:    csel w14, w14, w8, lo
+; CHECK-SD-NEXT:    cmp w11, w8
+; CHECK-SD-NEXT:    mov d2, v7.d[1]
+; CHECK-SD-NEXT:    csel w11, w11, w8, lo
+; CHECK-SD-NEXT:    mov v1.h[2], w15
+; CHECK-SD-NEXT:    mov v0.h[4], w13
+; CHECK-SD-NEXT:    cmp w12, w8
+; CHECK-SD-NEXT:    csel w12, w12, w8, lo
+; CHECK-SD-NEXT:    mov v1.h[3], w14
+; CHECK-SD-NEXT:    mov v0.h[5], w0
+; CHECK-SD-NEXT:    mov v1.h[4], w11
+; CHECK-SD-NEXT:    fcvtzu w11, d7
+; CHECK-SD-NEXT:    mov v0.h[6], w10
+; CHECK-SD-NEXT:    mov v1.h[5], w12
+; CHECK-SD-NEXT:    cmp w11, w8
+; CHECK-SD-NEXT:    fcvtzu w12, d2
+; CHECK-SD-NEXT:    csel w11, w11, w8, lo
+; CHECK-SD-NEXT:    mov v0.h[7], w9
+; CHECK-SD-NEXT:    mov v1.h[6], w11
+; CHECK-SD-NEXT:    cmp w12, w8
+; CHECK-SD-NEXT:    csel w8, w12, w8, lo
+; CHECK-SD-NEXT:    mov v1.h[7], w8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: test_unsigned_v16f64_v16i16:
@@ -4549,7 +4504,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    csel x8, x23, x22, lt
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    mov v0.d[0], x19
+; CHECK-GI-NEXT:    fmov d0, x19
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x30, x23, [sp, #48] // 16-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fptrunc.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fptrunc.ll
index b4c38e9f2df3..1f84c944d7c1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fptrunc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fptrunc.ll
@@ -304,25 +304,15 @@ entry:
 }
 
 define <3 x half> @fptrunc_v3f64_v3f16(<3 x double> %a) {
-; CHECK-SD-LABEL: fptrunc_v3f64_v3f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvt h1, d1
-; CHECK-SD-NEXT:    fcvt h0, d0
-; CHECK-SD-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NEXT:    fcvt h1, d2
-; CHECK-SD-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptrunc_v3f64_v3f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvt h0, d0
-; CHECK-GI-NEXT:    fcvt h1, d1
-; CHECK-GI-NEXT:    fcvt h2, d2
-; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: fptrunc_v3f64_v3f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvt h0, d0
+; CHECK-NEXT:    fcvt h1, d1
+; CHECK-NEXT:    fcvt h2, d2
+; CHECK-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %c = fptrunc <3 x double> %a to <3 x half>
   ret <3 x half> %c
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/framelayout-scavengingslot-stack-hazard.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/framelayout-scavengingslot-stack-hazard.mir
new file mode 100644
index 000000000000..52ac36f80185
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/framelayout-scavengingslot-stack-hazard.mir
@@ -0,0 +1,99 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -aarch64-stack-hazard-size=1024 -run-pass=prologepilog %s -o - | FileCheck %s
+--- |
+
+  define void @stack_hazard_streaming_compat() "aarch64_pstate_sm_compatible" { entry: unreachable }
+  define void @stack_hazard_streaming_compat_emergency_spill_slot() "aarch64_pstate_sm_compatible" { entry: unreachable }
+
+...
+
+# +------------------+
+# | GPR callee-saves |
+# +------------------+ <- FP
+# | <hazard padding> |
+# +------------------+
+# | FPR locals       |
+# | %stack.1         |
+# +------------------+
+# | <hazard padding> |
+# +------------------+
+# | GPR locals       |
+# | %stack.2         |
+# | <emergency spill>|
+# +------------------+ <- BP
+# | <VLA>            |
+# +------------------+ <- SP (can't be used due to VLA)
+
+# In this case without the base pointer we'd need the emergency spill slot to
+# access both %stack.1 and %stack.2. With the base pointer we can reach both
+# without spilling.
+
+name: stack_hazard_streaming_compat
+# CHECK-LABEL: name: stack_hazard_streaming_compat
+# CHECK: bb.0:
+# CHECK:      STRDui $d0, $x19, 131
+# CHECK-NEXT: STRXui $x0, $x19, 1
+# CHECK: bb.1:
+tracksRegLiveness: true
+frameInfo:
+  isFrameAddressTaken: true
+stack:
+  - { id: 0, type: variable-sized,  alignment: 1 }
+  - { id: 1, size: 8, alignment: 8 }
+  - { id: 2, size: 8, alignment: 8 }
+body: |
+  bb.0:
+    liveins: $x0, $x8, $d0
+    $x9 = LDRXui $x0, 0 :: (load (s64))
+    STRDui $d0, %stack.1, 0 :: (store (s64) into %stack.1)
+    STRXui $x0, %stack.2, 0 :: (store (s64) into %stack.2)
+    B %bb.1
+  bb.1:
+    liveins: $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr
+    RET_ReallyLR implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27, implicit $x28, implicit $lr
+...
+---
+# +------------------+
+# | GPR callee-saves |
+# +------------------+ <- FP
+# | <hazard padding> |
+# +------------------+
+# | FPR locals       |
+# | %stack.1         |
+# +------------------+
+# | <hazard padding> |
+# +------------------+
+# | GPR locals       |
+# | %stack.2         | (very large)
+# | <emergency spill>|
+# +------------------+ <- BP
+# | <VLA>            |
+# +------------------+ <- SP (can't be used due to VLA)
+
+# In this case we need to use the emergency spill slot to access %stack.1 as it
+# is too far from the frame pointer and the base pointer to directly address.
+# Note: This also tests that the <emergency spill> located near the SP/BP.
+
+name: stack_hazard_streaming_compat_emergency_spill_slot
+# CHECK-LABEL: name: stack_hazard_streaming_compat_emergency_spill_slot
+# CHECK: bb.0:
+# CHECK:      STRXui killed $[[SCRATCH:x[0-9]+]], $x19, 0
+# CHECK-NEXT: $[[SCRATCH]] = ADDXri $x19, 1056, 0
+# CHECK-NEXT: STRDui $d0, killed $[[SCRATCH]], 4095
+# CHECK-NEXT: $[[SCRATCH]] = LDRXui $x19, 0
+# CHECK: bb.1:
+tracksRegLiveness: true
+frameInfo:
+  isFrameAddressTaken: true
+stack:
+  - { id: 0, type: variable-sized,  alignment: 1 }
+  - { id: 1, size: 8, alignment: 8 }
+  - { id: 2, size: 32761, alignment: 8 }
+body: |
+  bb.0:
+    liveins: $x0, $x8, $d0
+    $x9 = LDRXui $x0, 0 :: (load (s64))
+    STRDui $d0, %stack.1, 0 :: (store (s64) into %stack.1)
+    B %bb.1
+  bb.1:
+    liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr
+    RET_ReallyLR implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27, implicit $x28, implicit $lr
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/framelayout-sve-win.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/framelayout-sve-win.mir
new file mode 100644
index 000000000000..5933c5daa67e
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/framelayout-sve-win.mir
@@ -0,0 +1,894 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --print-fixed-stack --version 5
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 --print-fixed-stack
+# RUN: llc -mattr=+sve -mtriple=aarch64-windows-msvc -run-pass=prologepilog %s -o - | FileCheck %s
+
+--- |
+
+  define void @test_allocate_sve() uwtable { entry: unreachable }
+  define void @test_allocate_sve_gpr_callee_saves() uwtable { entry: unreachable }
+  define void @test_allocate_sve_gpr_realigned() uwtable { entry: unreachable }
+  define void @test_address_sve() uwtable { entry: unreachable }
+  define void @test_address_sve_fp() uwtable { entry: unreachable }
+  define void @test_stack_arg_sve() uwtable { entry: unreachable }
+  define void @test_address_sve_out_of_range() uwtable { entry: unreachable }
+  define void @test_address_gpr_vla() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_pregs_sve() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_zregs_sve() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_sve() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_sve_realign() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @frame_layout() uwtable { entry: unreachable }
+...
+---
+name:            test_allocate_sve
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2 }
+  - { id: 1, stack-id: default, size: 16, alignment: 8 }
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_allocate_sve
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.2)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: frame-setup SEH_StackAlloc 16
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 2
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 2
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    RET_ReallyLR
+...
+---
+name:            test_allocate_sve_gpr_callee_saves
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2 }
+  - { id: 1, stack-id: default, size: 16, alignment: 8 }
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_allocate_sve_gpr_callee_saves
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $x21, $x20, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $x20, killed $x21, $sp, -4 :: (store (s64) into %stack.3), (store (s64) into %stack.4)
+    ; CHECK-NEXT: frame-setup SEH_SaveRegP_X 20, 21, -32
+    ; CHECK-NEXT: frame-setup STRXui killed $lr, $sp, 2 :: (store (s64) into %stack.2)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg 30, 16
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: frame-setup SEH_StackAlloc 16
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 2
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $x20 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x21 = IMPLICIT_DEF
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 2
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: $lr = frame-destroy LDRXui $sp, 2 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg 30, 16
+    ; CHECK-NEXT: early-clobber $sp, $x20, $x21 = frame-destroy LDPXpost $sp, 4 :: (load (s64) from %stack.3), (load (s64) from %stack.4)
+    ; CHECK-NEXT: frame-destroy SEH_SaveRegP_X 20, 21, -32
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    $x20 = IMPLICIT_DEF
+    $x21 = IMPLICIT_DEF
+    RET_ReallyLR
+...
+---
+name:            test_allocate_sve_gpr_realigned
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2  }
+  - { id: 1, stack-id: default, size: 16, alignment: 32 }
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_allocate_sve_gpr_realigned
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2 :: (store (s64) into %stack.2), (store (s64) into %stack.3)
+    ; CHECK-NEXT: frame-setup SEH_SaveFPLR_X -16
+    ; CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
+    ; CHECK-NEXT: frame-setup SEH_SetFP
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $x9 = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: $x9 = frame-setup ADDVL_XXI $x9, -2, implicit $vg
+    ; CHECK-NEXT: $sp = frame-setup ANDXri killed $x9, 7930
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
+    ; CHECK-NEXT: frame-destroy SEH_SetFP
+    ; CHECK-NEXT: early-clobber $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2 :: (load (s64) from %stack.2), (load (s64) from %stack.3)
+    ; CHECK-NEXT: frame-destroy SEH_SaveFPLR_X -16
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    RET_ReallyLR
+...
+---
+name:            test_address_sve
+frameInfo:
+  maxAlignment:  16
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
+  - { id: 1, stack-id: scalable-vector, size: 16, alignment: 8 }
+  - { id: 2, stack-id: scalable-vector, size:  2, alignment: 2 }
+  - { id: 3, stack-id: default, size: 16, alignment: 8 }
+body:             |
+  bb.0.entry:
+    liveins: $z0, $z1, $p0
+
+    ; CHECK-LABEL: name: test_address_sve
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $z0, $z1, $p0, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.4)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: frame-setup SEH_StackAlloc 16
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 3
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $x8 = ADDXri $sp, 16, 0
+    ; CHECK-NEXT: STR_ZXI $z0, killed $x8, 2
+    ; CHECK-NEXT: $x8 = ADDXri $sp, 16, 0
+    ; CHECK-NEXT: STR_ZXI $z1, killed $x8, 1
+    ; CHECK-NEXT: $x8 = ADDXri $sp, 16, 0
+    ; CHECK-NEXT: STR_PXI $p0, killed $x8, 7
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 3
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.4)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    STR_ZXI $z0, %stack.0, 0
+    STR_ZXI $z1, %stack.1, 0
+    STR_PXI $p0, %stack.2, 0
+
+    RET_ReallyLR
+...
+---
+name:            test_address_sve_fp
+frameInfo:
+  maxAlignment:  16
+  isFrameAddressTaken: true
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
+  - { id: 1, stack-id: scalable-vector, size: 16, alignment: 8 }
+  - { id: 2, stack-id: scalable-vector, size:  2, alignment: 2 }
+  - { id: 3, stack-id: default, size: 16, alignment: 8 }
+body:             |
+  bb.0.entry:
+    liveins: $z0, $z1, $p0
+
+    ; CHECK-LABEL: name: test_address_sve_fp
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $z0, $z1, $p0, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2 :: (store (s64) into %stack.4), (store (s64) into %stack.5)
+    ; CHECK-NEXT: frame-setup SEH_SaveFPLR_X -16
+    ; CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
+    ; CHECK-NEXT: frame-setup SEH_SetFP
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3, implicit $vg
+    ; CHECK-NEXT: STR_ZXI $z0, $fp, -1
+    ; CHECK-NEXT: STR_ZXI $z1, $fp, -2
+    ; CHECK-NEXT: STR_PXI $p0, $fp, -17
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 3
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: early-clobber $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2 :: (load (s64) from %stack.4), (load (s64) from %stack.5)
+    ; CHECK-NEXT: frame-destroy SEH_SaveFPLR_X -16
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    STR_ZXI $z0, %stack.0, 0
+    STR_ZXI $z1, %stack.1, 0
+    STR_PXI $p0, %stack.2, 0
+
+    RET_ReallyLR
+...
+---
+name:             test_stack_arg_sve
+fixedStack:
+  - { id: 0, stack-id: default, size: 16, alignment: 16, offset: 0 }
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
+  - { id: 1, stack-id: default, size: 16, alignment: 16 }
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: test_stack_arg_sve
+    ; CHECK: fixedStack:
+    ; CHECK-NEXT:   - { id: 0, type: default, offset: 0, size: 16, alignment: 16, stack-id: default, 
+    ; CHECK-NEXT:       isImmutable: false, isAliased: false, callee-saved-register: '', 
+    ; CHECK-NEXT:       callee-saved-restored: true, debug-info-variable: '', debug-info-expression: '', 
+    ; CHECK-NEXT:       debug-info-location: '' }
+    ; CHECK: liveins: $x0, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.2)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: frame-setup SEH_StackAlloc 16
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 1
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 1, implicit $vg
+    ; CHECK-NEXT: $x0 = LDRXui killed $x8, 4
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 1
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    $x0 = LDRXui %fixed-stack.0, 0
+    RET_ReallyLR
+...
+---
+name:            test_address_sve_out_of_range
+frameInfo:
+  maxAlignment:  16
+stack:
+  - { id: 0, stack-id: scalable-vector, size:   16, alignment: 16 }
+  - { id: 1, stack-id: scalable-vector, size: 3584, alignment: 16 }
+  - { id: 2, stack-id: scalable-vector, size:  512, alignment: 16 }
+
+body:             |
+  bb.0.entry:
+    liveins: $z0, $p0
+
+    ; CHECK-LABEL: name: test_address_sve_out_of_range
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $z0, $p0, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.3)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 1
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 1, implicit $vg
+    ; CHECK-NEXT: STR_ZXI $z0, killed $x8, 255
+    ; CHECK-NEXT: $x8 = ADDPL_XXI $sp, 1, implicit $vg
+    ; CHECK-NEXT: STR_PXI $p0, killed $x8, 255
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 31
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 9, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 9
+    ; CHECK-NEXT: early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.3)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    STR_ZXI $z0, %stack.0, 0
+    STR_PXI $p0, %stack.1, 0
+
+    RET_ReallyLR
+...
+---
+name:            test_address_gpr_vla
+frameInfo:
+  maxAlignment:  16
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
+  - { id: 1, stack-id: default, size: 16, alignment: 8 }
+  - { id: 2, stack-id: default, type: variable-sized }
+body:             |
+  bb.0.entry:
+    liveins: $xzr
+
+    ; CHECK-LABEL: name: test_address_gpr_vla
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $xzr, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $x19, $sp, -32 :: (store (s64) into %stack.5)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 19, -32
+    ; CHECK-NEXT: frame-setup STPXi killed $fp, killed $lr, $sp, 1 :: (store (s64) into %stack.3), (store (s64) into %stack.4)
+    ; CHECK-NEXT: frame-setup SEH_SaveFPLR 8
+    ; CHECK-NEXT: $fp = frame-setup ADDXri $sp, 8, 0
+    ; CHECK-NEXT: frame-setup SEH_AddFP 8
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ; CHECK-NEXT: $x19 = ADDXri $sp, 0, 0
+    ; CHECK-NEXT: STRXui $xzr, $x19, 0
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy SUBXri $fp, 8, 0
+    ; CHECK-NEXT: frame-destroy SEH_AddFP 8
+    ; CHECK-NEXT: $fp, $lr = frame-destroy LDPXi $sp, 1 :: (load (s64) from %stack.3), (load (s64) from %stack.4)
+    ; CHECK-NEXT: frame-destroy SEH_SaveFPLR 8
+    ; CHECK-NEXT: early-clobber $sp, $x19 = frame-destroy LDRXpost $sp, 32 :: (load (s64) from %stack.5)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg_X 19, -32
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    STRXui $xzr, %stack.1, 0
+
+    RET_ReallyLR
+...
+---
+name: save_restore_pregs_sve
+stack:
+  - { id: 0, stack-id: default, size: 32, alignment: 16 }
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: save_restore_pregs_sve
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $p4, $p5, $p6, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 1
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p4, $sp, 0 :: (store (s16) into %stack.4)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 4, 0
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p5, $sp, 1 :: (store (s16) into %stack.3)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 5, 1
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p6, $sp, 2 :: (store (s16) into %stack.2)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 6, 2
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.1)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 32, 0
+    ; CHECK-NEXT: frame-setup SEH_StackAlloc 32
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 32, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 32
+    ; CHECK-NEXT: $lr = frame-destroy LDRXui $sp, 0 :: (load (s64) from %stack.1)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg 30, 0
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 0 :: (load (s16) from %stack.4)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 4, 0
+    ; CHECK-NEXT: $p5 = frame-destroy LDR_PXI $sp, 1 :: (load (s16) from %stack.3)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 5, 1
+    ; CHECK-NEXT: $p6 = frame-destroy LDR_PXI $sp, 2 :: (load (s16) from %stack.2)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 6, 2
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 1
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+
+    RET_ReallyLR
+...
+---
+name: save_restore_zregs_sve
+stack:
+  - { id: 0, stack-id: default, size: 32, alignment: 16 }
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: save_restore_zregs_sve
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $z8, $z9, $z10, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 3
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z8, $sp, 0 :: (store (s128) into %stack.4)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 8, 0
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z9, $sp, 1 :: (store (s128) into %stack.3)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 9, 1
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z10, $sp, 2 :: (store (s128) into %stack.2)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 10, 2
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.1)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 32, 0
+    ; CHECK-NEXT: frame-setup SEH_StackAlloc 32
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $z8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z10 = IMPLICIT_DEF
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 32, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 32
+    ; CHECK-NEXT: $lr = frame-destroy LDRXui $sp, 0 :: (load (s64) from %stack.1)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg 30, 0
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 0 :: (load (s128) from %stack.4)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 8, 0
+    ; CHECK-NEXT: $z9 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.3)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 9, 1
+    ; CHECK-NEXT: $z10 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.2)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 10, 2
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 3
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    $z8 = IMPLICIT_DEF
+    $z9 = IMPLICIT_DEF
+    $z10 = IMPLICIT_DEF
+
+    RET_ReallyLR
+...
+---
+name: save_restore_sve
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
+  - { id: 1, stack-id: default, size: 32, alignment: 16 }
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: save_restore_sve
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $p4, $p5, $p6, $p7, $p8, $p9, $p10, $p11, $p12, $p13, $p14, $p15, $z8, $z9, $z10, $z11, $z12, $z13, $z14, $z15, $z16, $z17, $z18, $z19, $z20, $z21, $z22, $z23, $x20, $x19, $lr, $x21
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -18, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 18
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p4, $sp, 0 :: (store (s16) into %stack.33)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 4, 0
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p5, $sp, 1 :: (store (s16) into %stack.32)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 5, 1
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p6, $sp, 2 :: (store (s16) into %stack.31)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 6, 2
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p7, $sp, 3 :: (store (s16) into %stack.30)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 7, 3
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p8, $sp, 4 :: (store (s16) into %stack.29)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 8, 4
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p9, $sp, 5 :: (store (s16) into %stack.28)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 9, 5
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p10, $sp, 6 :: (store (s16) into %stack.27)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 10, 6
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p11, $sp, 7 :: (store (s16) into %stack.26)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 11, 7
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p12, $sp, 8 :: (store (s16) into %stack.25)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 12, 8
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p13, $sp, 9 :: (store (s16) into %stack.24)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 13, 9
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p14, $sp, 10 :: (store (s16) into %stack.23)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 14, 10
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p15, $sp, 11 :: (store (s16) into %stack.22)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 15, 11
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z8, $sp, 2 :: (store (s128) into %stack.21)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 8, 2
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z9, $sp, 3 :: (store (s128) into %stack.20)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 9, 3
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z10, $sp, 4 :: (store (s128) into %stack.19)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 10, 4
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z11, $sp, 5 :: (store (s128) into %stack.18)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 11, 5
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z12, $sp, 6 :: (store (s128) into %stack.17)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 12, 6
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z13, $sp, 7 :: (store (s128) into %stack.16)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 13, 7
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z14, $sp, 8 :: (store (s128) into %stack.15)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 14, 8
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z15, $sp, 9 :: (store (s128) into %stack.14)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 15, 9
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z16, $sp, 10 :: (store (s128) into %stack.13)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 16, 10
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z17, $sp, 11 :: (store (s128) into %stack.12)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 17, 11
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z18, $sp, 12 :: (store (s128) into %stack.11)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 18, 12
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z19, $sp, 13 :: (store (s128) into %stack.10)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 19, 13
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z20, $sp, 14 :: (store (s128) into %stack.9)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 20, 14
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z21, $sp, 15 :: (store (s128) into %stack.8)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 21, 15
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z22, $sp, 16 :: (store (s128) into %stack.7)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 22, 16
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z23, $sp, 17 :: (store (s128) into %stack.6)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 23, 17
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $x19, killed $x20, $sp, -4 :: (store (s64) into %stack.4), (store (s64) into %stack.5)
+    ; CHECK-NEXT: frame-setup SEH_SaveRegP_X 19, 20, -32
+    ; CHECK-NEXT: frame-setup STPXi killed $x21, killed $lr, $sp, 2 :: (store (s64) into %stack.2), (store (s64) into %stack.3)
+    ; CHECK-NEXT: frame-setup SEH_SaveRegP 21, 30, 16
+    ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 32, 0
+    ; CHECK-NEXT: frame-setup SEH_StackAlloc 32
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 1
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $z8_z9_z10_z11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z12_z13_z14_z15 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z16_z17_z18_z19 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z20_z21_z22_z23 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z24_z25_z26_z27 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z28_z29_z30_z31 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p7 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x19 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x20 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x21 = IMPLICIT_DEF
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 1
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 32, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 32
+    ; CHECK-NEXT: $x21, $lr = frame-destroy LDPXi $sp, 2 :: (load (s64) from %stack.2), (load (s64) from %stack.3)
+    ; CHECK-NEXT: frame-destroy SEH_SaveRegP 21, 30, 16
+    ; CHECK-NEXT: $x19, $x20 = frame-destroy LDPXi $sp, 0 :: (load (s64) from %stack.4), (load (s64) from %stack.5)
+    ; CHECK-NEXT: frame-destroy SEH_SaveRegP 19, 20, 0
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 32, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 32
+    ; CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.21)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 8, 2
+    ; CHECK-NEXT: $z9 = frame-destroy LDR_ZXI $sp, 3 :: (load (s128) from %stack.20)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 9, 3
+    ; CHECK-NEXT: $z10 = frame-destroy LDR_ZXI $sp, 4 :: (load (s128) from %stack.19)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 10, 4
+    ; CHECK-NEXT: $z11 = frame-destroy LDR_ZXI $sp, 5 :: (load (s128) from %stack.18)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 11, 5
+    ; CHECK-NEXT: $z12 = frame-destroy LDR_ZXI $sp, 6 :: (load (s128) from %stack.17)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 12, 6
+    ; CHECK-NEXT: $z13 = frame-destroy LDR_ZXI $sp, 7 :: (load (s128) from %stack.16)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 13, 7
+    ; CHECK-NEXT: $z14 = frame-destroy LDR_ZXI $sp, 8 :: (load (s128) from %stack.15)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 14, 8
+    ; CHECK-NEXT: $z15 = frame-destroy LDR_ZXI $sp, 9 :: (load (s128) from %stack.14)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 15, 9
+    ; CHECK-NEXT: $z16 = frame-destroy LDR_ZXI $sp, 10 :: (load (s128) from %stack.13)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 16, 10
+    ; CHECK-NEXT: $z17 = frame-destroy LDR_ZXI $sp, 11 :: (load (s128) from %stack.12)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 17, 11
+    ; CHECK-NEXT: $z18 = frame-destroy LDR_ZXI $sp, 12 :: (load (s128) from %stack.11)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 18, 12
+    ; CHECK-NEXT: $z19 = frame-destroy LDR_ZXI $sp, 13 :: (load (s128) from %stack.10)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 19, 13
+    ; CHECK-NEXT: $z20 = frame-destroy LDR_ZXI $sp, 14 :: (load (s128) from %stack.9)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 20, 14
+    ; CHECK-NEXT: $z21 = frame-destroy LDR_ZXI $sp, 15 :: (load (s128) from %stack.8)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 21, 15
+    ; CHECK-NEXT: $z22 = frame-destroy LDR_ZXI $sp, 16 :: (load (s128) from %stack.7)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 22, 16
+    ; CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 17 :: (load (s128) from %stack.6)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 23, 17
+    ; CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 0 :: (load (s16) from %stack.33)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 4, 0
+    ; CHECK-NEXT: $p5 = frame-destroy LDR_PXI $sp, 1 :: (load (s16) from %stack.32)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 5, 1
+    ; CHECK-NEXT: $p6 = frame-destroy LDR_PXI $sp, 2 :: (load (s16) from %stack.31)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 6, 2
+    ; CHECK-NEXT: $p7 = frame-destroy LDR_PXI $sp, 3 :: (load (s16) from %stack.30)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 7, 3
+    ; CHECK-NEXT: $p8 = frame-destroy LDR_PXI $sp, 4 :: (load (s16) from %stack.29)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 8, 4
+    ; CHECK-NEXT: $p9 = frame-destroy LDR_PXI $sp, 5 :: (load (s16) from %stack.28)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 9, 5
+    ; CHECK-NEXT: $p10 = frame-destroy LDR_PXI $sp, 6 :: (load (s16) from %stack.27)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 10, 6
+    ; CHECK-NEXT: $p11 = frame-destroy LDR_PXI $sp, 7 :: (load (s16) from %stack.26)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 11, 7
+    ; CHECK-NEXT: $p12 = frame-destroy LDR_PXI $sp, 8 :: (load (s16) from %stack.25)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 12, 8
+    ; CHECK-NEXT: $p13 = frame-destroy LDR_PXI $sp, 9 :: (load (s16) from %stack.24)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 13, 9
+    ; CHECK-NEXT: $p14 = frame-destroy LDR_PXI $sp, 10 :: (load (s16) from %stack.23)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 14, 10
+    ; CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 11 :: (load (s16) from %stack.22)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 15, 11
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 18, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 18
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    $z8_z9_z10_z11   = IMPLICIT_DEF
+    $z12_z13_z14_z15 = IMPLICIT_DEF
+    $z16_z17_z18_z19 = IMPLICIT_DEF
+    $z20_z21_z22_z23 = IMPLICIT_DEF
+    $z24_z25_z26_z27 = IMPLICIT_DEF
+    $z28_z29_z30_z31 = IMPLICIT_DEF
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+    $p7 = IMPLICIT_DEF
+    $p8 = IMPLICIT_DEF
+    $p9 = IMPLICIT_DEF
+    $p10 = IMPLICIT_DEF
+    $p11 = IMPLICIT_DEF
+    $p12 = IMPLICIT_DEF
+    $p13 = IMPLICIT_DEF
+    $p14 = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    $x19 = IMPLICIT_DEF
+    $x20 = IMPLICIT_DEF
+    $x21 = IMPLICIT_DEF
+
+    RET_ReallyLR
+...
+---
+name: save_restore_sve_realign
+stack:
+  - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
+  - { id: 1, stack-id: default, size: 16, alignment: 32 }
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: save_restore_sve_realign
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $p4, $p5, $p6, $p7, $p8, $p9, $p10, $p11, $p12, $p13, $p14, $p15, $z8, $z9, $z10, $z11, $z12, $z13, $z14, $z15, $z16, $z17, $z18, $z19, $z20, $z21, $z22, $z23, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -18, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 18
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p4, $sp, 0 :: (store (s16) into %stack.31)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 4, 0
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p5, $sp, 1 :: (store (s16) into %stack.30)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 5, 1
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p6, $sp, 2 :: (store (s16) into %stack.29)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 6, 2
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p7, $sp, 3 :: (store (s16) into %stack.28)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 7, 3
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p8, $sp, 4 :: (store (s16) into %stack.27)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 8, 4
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p9, $sp, 5 :: (store (s16) into %stack.26)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 9, 5
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p10, $sp, 6 :: (store (s16) into %stack.25)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 10, 6
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p11, $sp, 7 :: (store (s16) into %stack.24)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 11, 7
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p12, $sp, 8 :: (store (s16) into %stack.23)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 12, 8
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p13, $sp, 9 :: (store (s16) into %stack.22)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 13, 9
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p14, $sp, 10 :: (store (s16) into %stack.21)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 14, 10
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p15, $sp, 11 :: (store (s16) into %stack.20)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 15, 11
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z8, $sp, 2 :: (store (s128) into %stack.19)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 8, 2
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z9, $sp, 3 :: (store (s128) into %stack.18)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 9, 3
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z10, $sp, 4 :: (store (s128) into %stack.17)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 10, 4
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z11, $sp, 5 :: (store (s128) into %stack.16)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 11, 5
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z12, $sp, 6 :: (store (s128) into %stack.15)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 12, 6
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z13, $sp, 7 :: (store (s128) into %stack.14)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 13, 7
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z14, $sp, 8 :: (store (s128) into %stack.13)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 14, 8
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z15, $sp, 9 :: (store (s128) into %stack.12)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 15, 9
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z16, $sp, 10 :: (store (s128) into %stack.11)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 16, 10
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z17, $sp, 11 :: (store (s128) into %stack.10)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 17, 11
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z18, $sp, 12 :: (store (s128) into %stack.9)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 18, 12
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z19, $sp, 13 :: (store (s128) into %stack.8)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 19, 13
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z20, $sp, 14 :: (store (s128) into %stack.7)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 20, 14
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z21, $sp, 15 :: (store (s128) into %stack.6)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 21, 15
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z22, $sp, 16 :: (store (s128) into %stack.5)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 22, 16
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z23, $sp, 17 :: (store (s128) into %stack.4)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 23, 17
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2 :: (store (s64) into %stack.2), (store (s64) into %stack.3)
+    ; CHECK-NEXT: frame-setup SEH_SaveFPLR_X -16
+    ; CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
+    ; CHECK-NEXT: frame-setup SEH_SetFP
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $x9 = frame-setup SUBXri $sp, 16, 0
+    ; CHECK-NEXT: $x9 = frame-setup ADDVL_XXI $x9, -1, implicit $vg
+    ; CHECK-NEXT: $sp = frame-setup ANDXri killed $x9, 7930
+    ; CHECK-NEXT: $z8_z9_z10_z11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z12_z13_z14_z15 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z16_z17_z18_z19 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z20_z21_z22_z23 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z24_z25_z26_z27 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z28_z29_z30_z31 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p5 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p6 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p7 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p9 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p10 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p11 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p12 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p13 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p14 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
+    ; CHECK-NEXT: frame-destroy SEH_SetFP
+    ; CHECK-NEXT: $fp, $lr = frame-destroy LDPXi $sp, 0 :: (load (s64) from %stack.2), (load (s64) from %stack.3)
+    ; CHECK-NEXT: frame-destroy SEH_SaveFPLR 0
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.19)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 8, 2
+    ; CHECK-NEXT: $z9 = frame-destroy LDR_ZXI $sp, 3 :: (load (s128) from %stack.18)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 9, 3
+    ; CHECK-NEXT: $z10 = frame-destroy LDR_ZXI $sp, 4 :: (load (s128) from %stack.17)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 10, 4
+    ; CHECK-NEXT: $z11 = frame-destroy LDR_ZXI $sp, 5 :: (load (s128) from %stack.16)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 11, 5
+    ; CHECK-NEXT: $z12 = frame-destroy LDR_ZXI $sp, 6 :: (load (s128) from %stack.15)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 12, 6
+    ; CHECK-NEXT: $z13 = frame-destroy LDR_ZXI $sp, 7 :: (load (s128) from %stack.14)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 13, 7
+    ; CHECK-NEXT: $z14 = frame-destroy LDR_ZXI $sp, 8 :: (load (s128) from %stack.13)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 14, 8
+    ; CHECK-NEXT: $z15 = frame-destroy LDR_ZXI $sp, 9 :: (load (s128) from %stack.12)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 15, 9
+    ; CHECK-NEXT: $z16 = frame-destroy LDR_ZXI $sp, 10 :: (load (s128) from %stack.11)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 16, 10
+    ; CHECK-NEXT: $z17 = frame-destroy LDR_ZXI $sp, 11 :: (load (s128) from %stack.10)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 17, 11
+    ; CHECK-NEXT: $z18 = frame-destroy LDR_ZXI $sp, 12 :: (load (s128) from %stack.9)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 18, 12
+    ; CHECK-NEXT: $z19 = frame-destroy LDR_ZXI $sp, 13 :: (load (s128) from %stack.8)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 19, 13
+    ; CHECK-NEXT: $z20 = frame-destroy LDR_ZXI $sp, 14 :: (load (s128) from %stack.7)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 20, 14
+    ; CHECK-NEXT: $z21 = frame-destroy LDR_ZXI $sp, 15 :: (load (s128) from %stack.6)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 21, 15
+    ; CHECK-NEXT: $z22 = frame-destroy LDR_ZXI $sp, 16 :: (load (s128) from %stack.5)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 22, 16
+    ; CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 17 :: (load (s128) from %stack.4)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 23, 17
+    ; CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 0 :: (load (s16) from %stack.31)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 4, 0
+    ; CHECK-NEXT: $p5 = frame-destroy LDR_PXI $sp, 1 :: (load (s16) from %stack.30)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 5, 1
+    ; CHECK-NEXT: $p6 = frame-destroy LDR_PXI $sp, 2 :: (load (s16) from %stack.29)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 6, 2
+    ; CHECK-NEXT: $p7 = frame-destroy LDR_PXI $sp, 3 :: (load (s16) from %stack.28)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 7, 3
+    ; CHECK-NEXT: $p8 = frame-destroy LDR_PXI $sp, 4 :: (load (s16) from %stack.27)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 8, 4
+    ; CHECK-NEXT: $p9 = frame-destroy LDR_PXI $sp, 5 :: (load (s16) from %stack.26)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 9, 5
+    ; CHECK-NEXT: $p10 = frame-destroy LDR_PXI $sp, 6 :: (load (s16) from %stack.25)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 10, 6
+    ; CHECK-NEXT: $p11 = frame-destroy LDR_PXI $sp, 7 :: (load (s16) from %stack.24)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 11, 7
+    ; CHECK-NEXT: $p12 = frame-destroy LDR_PXI $sp, 8 :: (load (s16) from %stack.23)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 12, 8
+    ; CHECK-NEXT: $p13 = frame-destroy LDR_PXI $sp, 9 :: (load (s16) from %stack.22)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 13, 9
+    ; CHECK-NEXT: $p14 = frame-destroy LDR_PXI $sp, 10 :: (load (s16) from %stack.21)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 14, 10
+    ; CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 11 :: (load (s16) from %stack.20)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 15, 11
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 18, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 18
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    $z8_z9_z10_z11   = IMPLICIT_DEF
+    $z12_z13_z14_z15 = IMPLICIT_DEF
+    $z16_z17_z18_z19 = IMPLICIT_DEF
+    $z20_z21_z22_z23 = IMPLICIT_DEF
+    $z24_z25_z26_z27 = IMPLICIT_DEF
+    $z28_z29_z30_z31 = IMPLICIT_DEF
+    $p4 = IMPLICIT_DEF
+    $p5 = IMPLICIT_DEF
+    $p6 = IMPLICIT_DEF
+    $p7 = IMPLICIT_DEF
+    $p8 = IMPLICIT_DEF
+    $p9 = IMPLICIT_DEF
+    $p10 = IMPLICIT_DEF
+    $p11 = IMPLICIT_DEF
+    $p12 = IMPLICIT_DEF
+    $p13 = IMPLICIT_DEF
+    $p14 = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    RET_ReallyLR
+...
+---
+name: frame_layout
+stack:
+  - { id: 0, type: default,    size:  32, alignment: 16, stack-id: scalable-vector }
+  - { id: 1, type: default,    size:   4, alignment:  2, stack-id: scalable-vector }
+  - { id: 2, type: default,    size:  16, alignment: 16, stack-id: scalable-vector }
+  - { id: 3, type: default,    size:   2, alignment:  2, stack-id: scalable-vector }
+  - { id: 4, type: spill-slot, size:  16, alignment: 16, stack-id: scalable-vector }
+  - { id: 5, type: spill-slot, size:   2, alignment:  2, stack-id: scalable-vector }
+body:             |
+  bb.0.entry:
+
+    ; Trigger some callee saves
+    ; CHECK-LABEL: name: frame_layout
+    ; CHECK: fixedStack:
+    ; CHECK: liveins: $p4, $p15, $z8, $z23, $lr
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 3
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p4, $sp, 0 :: (store (s16) into %stack.10)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 4, 0
+    ; CHECK-NEXT: frame-setup STR_PXI killed $p15, $sp, 1 :: (store (s16) into %stack.9)
+    ; CHECK-NEXT: frame-setup SEH_SavePReg 15, 1
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z8, $sp, 1 :: (store (s128) into %stack.8)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 8, 1
+    ; CHECK-NEXT: frame-setup STR_ZXI killed $z23, $sp, 2 :: (store (s128) into %stack.7)
+    ; CHECK-NEXT: frame-setup SEH_SaveZReg 23, 2
+    ; CHECK-NEXT: early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.6)
+    ; CHECK-NEXT: frame-setup SEH_SaveReg_X 30, -16
+    ; CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -7, implicit $vg
+    ; CHECK-NEXT: frame-setup SEH_AllocZ 7
+    ; CHECK-NEXT: frame-setup SEH_PrologEnd
+    ; CHECK-NEXT: $z8 = IMPLICIT_DEF
+    ; CHECK-NEXT: $z23 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p4 = IMPLICIT_DEF
+    ; CHECK-NEXT: $p15 = IMPLICIT_DEF
+    ; CHECK-NEXT: frame-destroy SEH_EpilogStart
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 7, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 7
+    ; CHECK-NEXT: $lr = frame-destroy LDRXui $sp, 0 :: (load (s64) from %stack.6)
+    ; CHECK-NEXT: frame-destroy SEH_SaveReg 30, 0
+    ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+    ; CHECK-NEXT: frame-destroy SEH_StackAlloc 16
+    ; CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 1 :: (load (s128) from %stack.8)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 8, 1
+    ; CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 2 :: (load (s128) from %stack.7)
+    ; CHECK-NEXT: frame-destroy SEH_SaveZReg 23, 2
+    ; CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 0 :: (load (s16) from %stack.10)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 4, 0
+    ; CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 1 :: (load (s16) from %stack.9)
+    ; CHECK-NEXT: frame-destroy SEH_SavePReg 15, 1
+    ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3, implicit $vg
+    ; CHECK-NEXT: frame-destroy SEH_AllocZ 3
+    ; CHECK-NEXT: frame-destroy SEH_EpilogEnd
+    ; CHECK-NEXT: RET_ReallyLR
+    $z8  = IMPLICIT_DEF
+    $z23 = IMPLICIT_DEF
+    $p4  = IMPLICIT_DEF
+    $p15 = IMPLICIT_DEF
+
+    RET_ReallyLR
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/freeze.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/freeze.ll
index 6efd9f40f006..0c56e1b66e81 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/freeze.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/freeze.ll
@@ -137,10 +137,9 @@ define <2 x i16> @freeze_v2i16() {
 ; CHECK-GI-LABEL: freeze_v2i16:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov h0, v0.h[1]
-; CHECK-GI-NEXT:    mov v1.s[0], w8
 ; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov v1.s[1], w8
-; CHECK-GI-NEXT:    add v0.2s, v1.2s, v1.2s
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v0.2s
 ; CHECK-GI-NEXT:    ret
   %y1 = freeze <2 x i16> undef
   %t1 = add <2 x i16> %y1, %y1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fsh.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/fsh.ll
index e29f35cfc217..4c28c9082402 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/fsh.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fsh.ll
@@ -1378,77 +1378,77 @@ define <7 x i32> @rotl_v7i32(<7 x i32> %a, <7 x i32> %c) {
 ;
 ; CHECK-GI-LABEL: rotl_v7i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[0], w7
 ; CHECK-GI-NEXT:    ldr s0, [sp, #24]
-; CHECK-GI-NEXT:    mov v2.s[0], wzr
-; CHECK-GI-NEXT:    mov v3.s[0], w7
-; CHECK-GI-NEXT:    mov x9, sp
-; CHECK-GI-NEXT:    ldr s5, [sp, #32]
-; CHECK-GI-NEXT:    mov v4.16b, v0.16b
-; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    fmov s3, w7
+; CHECK-GI-NEXT:    ldr s2, [sp, #32]
+; CHECK-GI-NEXT:    mov x8, sp
+; CHECK-GI-NEXT:    fmov s4, w7
+; CHECK-GI-NEXT:    mov v6.16b, v0.16b
 ; CHECK-GI-NEXT:    ldr s7, [sp]
-; CHECK-GI-NEXT:    mov v16.s[0], w8
-; CHECK-GI-NEXT:    mov v6.s[0], w0
-; CHECK-GI-NEXT:    ldr s18, [sp, #40]
-; CHECK-GI-NEXT:    ld1 { v1.s }[1], [x9]
-; CHECK-GI-NEXT:    mov v2.s[1], wzr
-; CHECK-GI-NEXT:    add x9, sp, #8
-; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
-; CHECK-GI-NEXT:    mov v3.s[1], v7.s[0]
-; CHECK-GI-NEXT:    mov v7.s[0], w0
-; CHECK-GI-NEXT:    mov v19.s[0], w8
-; CHECK-GI-NEXT:    ldr s17, [sp, #8]
-; CHECK-GI-NEXT:    mov v20.s[0], w4
-; CHECK-GI-NEXT:    ld1 { v1.s }[2], [x9]
-; CHECK-GI-NEXT:    mov v16.s[1], w8
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    ld1 { v3.s }[1], [x8]
+; CHECK-GI-NEXT:    add x8, sp, #8
+; CHECK-GI-NEXT:    mov v4.s[1], v7.s[0]
+; CHECK-GI-NEXT:    ldr s7, [sp, #8]
+; CHECK-GI-NEXT:    fmov s16, w0
+; CHECK-GI-NEXT:    mov v6.s[1], v2.s[0]
+; CHECK-GI-NEXT:    fmov s17, w0
 ; CHECK-GI-NEXT:    add x9, sp, #16
-; CHECK-GI-NEXT:    mov v2.s[2], wzr
-; CHECK-GI-NEXT:    mov v6.s[1], w1
-; CHECK-GI-NEXT:    mov v21.s[0], w4
-; CHECK-GI-NEXT:    mov v4.s[2], v18.s[0]
-; CHECK-GI-NEXT:    mov v7.s[1], w1
-; CHECK-GI-NEXT:    mov v3.s[2], v17.s[0]
-; CHECK-GI-NEXT:    ld1 { v1.s }[3], [x9]
-; CHECK-GI-NEXT:    mov v0.s[1], v5.s[0]
-; CHECK-GI-NEXT:    mov v19.s[1], w8
-; CHECK-GI-NEXT:    movi v5.4s, #31
-; CHECK-GI-NEXT:    mov v16.s[2], w8
-; CHECK-GI-NEXT:    ldr s17, [sp, #16]
-; CHECK-GI-NEXT:    mov v6.s[2], w2
-; CHECK-GI-NEXT:    mov v20.s[1], w5
-; CHECK-GI-NEXT:    mov v21.s[1], w5
-; CHECK-GI-NEXT:    sub v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT:    neg v1.4s, v1.4s
-; CHECK-GI-NEXT:    mov v7.s[2], w2
-; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
-; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
-; CHECK-GI-NEXT:    mov v19.s[2], w8
-; CHECK-GI-NEXT:    and v1.16b, v1.16b, v5.16b
-; CHECK-GI-NEXT:    and v2.16b, v2.16b, v16.16b
-; CHECK-GI-NEXT:    mov v6.s[3], w3
-; CHECK-GI-NEXT:    mov v7.s[3], w3
-; CHECK-GI-NEXT:    mov v20.s[2], w6
-; CHECK-GI-NEXT:    mov v21.s[2], w6
-; CHECK-GI-NEXT:    and v3.16b, v3.16b, v5.16b
-; CHECK-GI-NEXT:    and v0.16b, v0.16b, v19.16b
+; CHECK-GI-NEXT:    ld1 { v3.s }[2], [x8]
+; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
+; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    fmov s18, w8
+; CHECK-GI-NEXT:    mov v16.s[1], w1
+; CHECK-GI-NEXT:    mov v4.s[2], v7.s[0]
+; CHECK-GI-NEXT:    ldr s7, [sp, #16]
+; CHECK-GI-NEXT:    mov v17.s[1], w1
+; CHECK-GI-NEXT:    mov v6.s[2], v5.s[0]
+; CHECK-GI-NEXT:    ld1 { v3.s }[3], [x9]
+; CHECK-GI-NEXT:    fmov s2, w4
+; CHECK-GI-NEXT:    mov v18.s[1], w8
+; CHECK-GI-NEXT:    movi v19.4s, #31
+; CHECK-GI-NEXT:    mov v0.s[2], v5.s[0]
+; CHECK-GI-NEXT:    mov v16.s[2], w2
+; CHECK-GI-NEXT:    mov v4.s[3], v7.s[0]
+; CHECK-GI-NEXT:    fmov s7, w4
+; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    sub v1.4s, v1.4s, v6.4s
+; CHECK-GI-NEXT:    fmov s6, w8
+; CHECK-GI-NEXT:    mov v17.s[2], w2
+; CHECK-GI-NEXT:    mov v18.s[2], w8
+; CHECK-GI-NEXT:    mov v2.s[1], w5
+; CHECK-GI-NEXT:    mov v7.s[1], w5
+; CHECK-GI-NEXT:    and v3.16b, v3.16b, v19.16b
+; CHECK-GI-NEXT:    mov v16.s[3], w3
+; CHECK-GI-NEXT:    mov v6.s[1], w8
+; CHECK-GI-NEXT:    and v4.16b, v4.16b, v19.16b
+; CHECK-GI-NEXT:    mov v17.s[3], w3
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v18.16b
+; CHECK-GI-NEXT:    mov v2.s[2], w6
+; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    mov v7.s[2], w6
+; CHECK-GI-NEXT:    mov v6.s[2], w8
 ; CHECK-GI-NEXT:    neg v1.4s, v1.4s
-; CHECK-GI-NEXT:    neg v2.4s, v2.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v6.4s, v3.4s
-; CHECK-GI-NEXT:    ushl v0.4s, v20.4s, v0.4s
-; CHECK-GI-NEXT:    ushl v1.4s, v7.4s, v1.4s
-; CHECK-GI-NEXT:    ushl v2.4s, v21.4s, v2.4s
-; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    mov s2, v1.s[1]
-; CHECK-GI-NEXT:    mov s3, v1.s[2]
-; CHECK-GI-NEXT:    mov s4, v1.s[3]
+; CHECK-GI-NEXT:    ushl v4.4s, v17.4s, v4.4s
+; CHECK-GI-NEXT:    ushl v3.4s, v16.4s, v3.4s
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v6.16b
+; CHECK-GI-NEXT:    ushl v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    orr v2.16b, v4.16b, v3.16b
+; CHECK-GI-NEXT:    ushl v0.4s, v7.4s, v0.4s
+; CHECK-GI-NEXT:    mov s3, v2.s[2]
+; CHECK-GI-NEXT:    mov s4, v2.s[3]
+; CHECK-GI-NEXT:    fmov w0, s2
+; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    mov s1, v2.s[1]
+; CHECK-GI-NEXT:    fmov w2, s3
+; CHECK-GI-NEXT:    fmov w3, s4
 ; CHECK-GI-NEXT:    mov s5, v0.s[1]
 ; CHECK-GI-NEXT:    mov s6, v0.s[2]
-; CHECK-GI-NEXT:    fmov w0, s1
 ; CHECK-GI-NEXT:    fmov w4, s0
-; CHECK-GI-NEXT:    fmov w1, s2
-; CHECK-GI-NEXT:    fmov w2, s3
-; CHECK-GI-NEXT:    fmov w3, s4
+; CHECK-GI-NEXT:    fmov w1, s1
 ; CHECK-GI-NEXT:    fmov w5, s5
 ; CHECK-GI-NEXT:    fmov w6, s6
 ; CHECK-GI-NEXT:    ret
@@ -1505,74 +1505,74 @@ define <7 x i32> @rotr_v7i32(<7 x i32> %a, <7 x i32> %c) {
 ;
 ; CHECK-GI-LABEL: rotr_v7i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w7
-; CHECK-GI-NEXT:    ldr s2, [sp]
-; CHECK-GI-NEXT:    mov v1.s[0], w7
+; CHECK-GI-NEXT:    fmov s1, w7
+; CHECK-GI-NEXT:    ldr s3, [sp]
+; CHECK-GI-NEXT:    fmov s2, w7
+; CHECK-GI-NEXT:    mov x8, sp
+; CHECK-GI-NEXT:    ldr s6, [sp, #8]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    ldr s7, [sp, #32]
+; CHECK-GI-NEXT:    fmov s16, w0
+; CHECK-GI-NEXT:    fmov s17, w0
+; CHECK-GI-NEXT:    mov v1.s[1], v3.s[0]
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    ld1 { v2.s }[1], [x8]
 ; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
-; CHECK-GI-NEXT:    ldr s7, [sp, #8]
-; CHECK-GI-NEXT:    mov v3.s[0], wzr
-; CHECK-GI-NEXT:    mov v5.s[0], w8
-; CHECK-GI-NEXT:    mov x11, sp
-; CHECK-GI-NEXT:    ldr s16, [sp, #32]
-; CHECK-GI-NEXT:    mov v4.s[0], w0
-; CHECK-GI-NEXT:    mov v17.s[0], w0
-; CHECK-GI-NEXT:    ldr s6, [sp, #16]
-; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-GI-NEXT:    ldr s2, [sp, #24]
-; CHECK-GI-NEXT:    ld1 { v1.s }[1], [x11]
-; CHECK-GI-NEXT:    mov v3.s[1], wzr
-; CHECK-GI-NEXT:    add x10, sp, #8
-; CHECK-GI-NEXT:    ldr s18, [sp, #40]
-; CHECK-GI-NEXT:    mov v19.16b, v2.16b
-; CHECK-GI-NEXT:    mov v2.s[1], v16.s[0]
-; CHECK-GI-NEXT:    mov v5.s[1], w8
-; CHECK-GI-NEXT:    ld1 { v1.s }[2], [x10]
-; CHECK-GI-NEXT:    mov v4.s[1], w1
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    add x9, sp, #8
+; CHECK-GI-NEXT:    mov v4.16b, v3.16b
+; CHECK-GI-NEXT:    mov v3.s[1], v7.s[0]
+; CHECK-GI-NEXT:    fmov s18, w8
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    ld1 { v2.s }[2], [x9]
 ; CHECK-GI-NEXT:    mov v17.s[1], w1
-; CHECK-GI-NEXT:    mov v0.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v7.s[0], w8
-; CHECK-GI-NEXT:    mov v20.s[0], w4
-; CHECK-GI-NEXT:    mov v19.s[1], v16.s[0]
-; CHECK-GI-NEXT:    add x9, sp, #16
-; CHECK-GI-NEXT:    movi v16.4s, #31
-; CHECK-GI-NEXT:    mov v2.s[2], v18.s[0]
-; CHECK-GI-NEXT:    mov v3.s[2], wzr
-; CHECK-GI-NEXT:    mov v5.s[2], w8
-; CHECK-GI-NEXT:    ld1 { v1.s }[3], [x9]
-; CHECK-GI-NEXT:    mov v4.s[2], w2
+; CHECK-GI-NEXT:    mov v1.s[2], v6.s[0]
+; CHECK-GI-NEXT:    fmov s6, w8
+; CHECK-GI-NEXT:    mov v16.s[1], w1
+; CHECK-GI-NEXT:    mov v4.s[1], v7.s[0]
+; CHECK-GI-NEXT:    ldr s7, [sp, #16]
+; CHECK-GI-NEXT:    mov v18.s[1], w8
+; CHECK-GI-NEXT:    mov v3.s[2], v5.s[0]
+; CHECK-GI-NEXT:    fmov s19, w4
+; CHECK-GI-NEXT:    add x10, sp, #16
+; CHECK-GI-NEXT:    mov v6.s[1], w8
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    ld1 { v2.s }[3], [x10]
+; CHECK-GI-NEXT:    mov v1.s[3], v7.s[0]
+; CHECK-GI-NEXT:    movi v7.4s, #31
 ; CHECK-GI-NEXT:    mov v17.s[2], w2
-; CHECK-GI-NEXT:    mov v0.s[3], v6.s[0]
-; CHECK-GI-NEXT:    mov v6.s[0], w4
-; CHECK-GI-NEXT:    mov v7.s[1], w8
-; CHECK-GI-NEXT:    mov v19.s[2], v18.s[0]
-; CHECK-GI-NEXT:    mov v20.s[1], w5
-; CHECK-GI-NEXT:    neg v1.4s, v1.4s
-; CHECK-GI-NEXT:    and v2.16b, v2.16b, v5.16b
-; CHECK-GI-NEXT:    mov v4.s[3], w3
-; CHECK-GI-NEXT:    mov v17.s[3], w3
-; CHECK-GI-NEXT:    mov v6.s[1], w5
-; CHECK-GI-NEXT:    mov v7.s[2], w8
-; CHECK-GI-NEXT:    and v0.16b, v0.16b, v16.16b
-; CHECK-GI-NEXT:    sub v3.4s, v3.4s, v19.4s
-; CHECK-GI-NEXT:    mov v20.s[2], w6
-; CHECK-GI-NEXT:    and v1.16b, v1.16b, v16.16b
+; CHECK-GI-NEXT:    mov v4.s[2], v5.s[0]
+; CHECK-GI-NEXT:    fmov s5, w4
+; CHECK-GI-NEXT:    mov v16.s[2], w2
+; CHECK-GI-NEXT:    mov v19.s[1], w5
+; CHECK-GI-NEXT:    mov v18.s[2], w8
 ; CHECK-GI-NEXT:    neg v2.4s, v2.4s
-; CHECK-GI-NEXT:    neg v0.4s, v0.4s
-; CHECK-GI-NEXT:    mov v6.s[2], w6
-; CHECK-GI-NEXT:    and v3.16b, v3.16b, v7.16b
+; CHECK-GI-NEXT:    mov v6.s[2], w8
+; CHECK-GI-NEXT:    mov v5.s[1], w5
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v7.16b
+; CHECK-GI-NEXT:    mov v17.s[3], w3
+; CHECK-GI-NEXT:    sub v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mov v16.s[3], w3
+; CHECK-GI-NEXT:    and v2.16b, v2.16b, v7.16b
+; CHECK-GI-NEXT:    mov v19.s[2], w6
+; CHECK-GI-NEXT:    and v3.16b, v3.16b, v6.16b
+; CHECK-GI-NEXT:    neg v1.4s, v1.4s
+; CHECK-GI-NEXT:    mov v5.s[2], w6
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v18.16b
+; CHECK-GI-NEXT:    ushl v2.4s, v16.4s, v2.4s
+; CHECK-GI-NEXT:    neg v3.4s, v3.4s
 ; CHECK-GI-NEXT:    ushl v1.4s, v17.4s, v1.4s
-; CHECK-GI-NEXT:    ushl v2.4s, v20.4s, v2.4s
-; CHECK-GI-NEXT:    ushl v0.4s, v4.4s, v0.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v6.4s, v3.4s
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    orr v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT:    mov s2, v0.s[1]
-; CHECK-GI-NEXT:    mov s3, v0.s[2]
-; CHECK-GI-NEXT:    mov s4, v0.s[3]
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    mov s5, v1.s[1]
-; CHECK-GI-NEXT:    mov s6, v1.s[2]
-; CHECK-GI-NEXT:    fmov w4, s1
+; CHECK-GI-NEXT:    ushl v0.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT:    ushl v3.4s, v19.4s, v3.4s
+; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    orr v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT:    mov s2, v1.s[1]
+; CHECK-GI-NEXT:    mov s3, v1.s[2]
+; CHECK-GI-NEXT:    mov s4, v1.s[3]
+; CHECK-GI-NEXT:    fmov w0, s1
+; CHECK-GI-NEXT:    mov s5, v0.s[1]
+; CHECK-GI-NEXT:    mov s6, v0.s[2]
+; CHECK-GI-NEXT:    fmov w4, s0
 ; CHECK-GI-NEXT:    fmov w1, s2
 ; CHECK-GI-NEXT:    fmov w2, s3
 ; CHECK-GI-NEXT:    fmov w3, s4
@@ -2509,78 +2509,78 @@ define <7 x i32> @fshl_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) {
 ;
 ; CHECK-GI-LABEL: fshl_v7i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov w11, #-1 // =0xffffffff
 ; CHECK-GI-NEXT:    ldr s3, [sp, #48]
-; CHECK-GI-NEXT:    ldr s19, [sp, #56]
-; CHECK-GI-NEXT:    mov v20.s[0], w11
-; CHECK-GI-NEXT:    mov v21.s[0], w7
-; CHECK-GI-NEXT:    ldr s16, [sp, #80]
-; CHECK-GI-NEXT:    ldr s22, [sp, #88]
-; CHECK-GI-NEXT:    mov w12, #31 // =0x1f
+; CHECK-GI-NEXT:    ldr s20, [sp, #56]
+; CHECK-GI-NEXT:    add x9, sp, #56
+; CHECK-GI-NEXT:    ldr s4, [sp, #48]
+; CHECK-GI-NEXT:    ldr s7, [sp, #80]
+; CHECK-GI-NEXT:    mov w12, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    ldr s21, [sp, #88]
+; CHECK-GI-NEXT:    mov v3.s[1], v20.s[0]
+; CHECK-GI-NEXT:    fmov s20, w12
+; CHECK-GI-NEXT:    ld1 { v4.s }[1], [x9]
+; CHECK-GI-NEXT:    ldr s17, [sp]
+; CHECK-GI-NEXT:    add x13, sp, #64
+; CHECK-GI-NEXT:    mov v7.s[1], v21.s[0]
+; CHECK-GI-NEXT:    fmov s21, w7
+; CHECK-GI-NEXT:    ldr s19, [sp, #64]
+; CHECK-GI-NEXT:    mov w11, #31 // =0x1f
+; CHECK-GI-NEXT:    mov v20.s[1], w12
+; CHECK-GI-NEXT:    ldr s18, [sp, #96]
+; CHECK-GI-NEXT:    ld1 { v4.s }[2], [x13]
 ; CHECK-GI-NEXT:    mov w13, #1 // =0x1
-; CHECK-GI-NEXT:    ldr s6, [sp]
-; CHECK-GI-NEXT:    mov v3.s[1], v19.s[0]
-; CHECK-GI-NEXT:    mov v19.s[0], w12
-; CHECK-GI-NEXT:    mov v23.s[0], w13
-; CHECK-GI-NEXT:    mov v16.s[1], v22.s[0]
-; CHECK-GI-NEXT:    ldr s7, [sp, #64]
-; CHECK-GI-NEXT:    mov v20.s[1], w11
-; CHECK-GI-NEXT:    mov v22.s[0], w0
-; CHECK-GI-NEXT:    mov v21.s[1], v6.s[0]
-; CHECK-GI-NEXT:    ldr s24, [sp, #96]
-; CHECK-GI-NEXT:    mov v6.s[0], w12
-; CHECK-GI-NEXT:    ldr s5, [sp, #8]
-; CHECK-GI-NEXT:    ldr s18, [sp, #48]
-; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v19.s[1], w12
-; CHECK-GI-NEXT:    mov v23.s[1], w13
+; CHECK-GI-NEXT:    mov v3.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v21.s[1], v17.s[0]
+; CHECK-GI-NEXT:    fmov s17, w11
+; CHECK-GI-NEXT:    fmov s19, w13
+; CHECK-GI-NEXT:    fmov s23, w0
+; CHECK-GI-NEXT:    fmov s24, w11
+; CHECK-GI-NEXT:    ldr s6, [sp, #8]
 ; CHECK-GI-NEXT:    ldr s0, [sp, #24]
-; CHECK-GI-NEXT:    ldr s4, [sp, #32]
-; CHECK-GI-NEXT:    add x10, sp, #56
-; CHECK-GI-NEXT:    mov v16.s[2], v24.s[0]
-; CHECK-GI-NEXT:    mov v20.s[2], w11
-; CHECK-GI-NEXT:    ldr s17, [sp, #72]
-; CHECK-GI-NEXT:    ld1 { v18.s }[1], [x10]
-; CHECK-GI-NEXT:    mov v22.s[1], w1
-; CHECK-GI-NEXT:    mov v21.s[2], v5.s[0]
-; CHECK-GI-NEXT:    mov v5.s[0], w4
-; CHECK-GI-NEXT:    ldr s24, [sp, #80]
-; CHECK-GI-NEXT:    mov v6.s[1], w12
-; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
-; CHECK-GI-NEXT:    add x9, sp, #64
+; CHECK-GI-NEXT:    ldr s5, [sp, #32]
+; CHECK-GI-NEXT:    mov v7.s[2], v18.s[0]
+; CHECK-GI-NEXT:    mov v17.s[1], w11
+; CHECK-GI-NEXT:    mov v19.s[1], w13
+; CHECK-GI-NEXT:    mov v20.s[2], w12
+; CHECK-GI-NEXT:    ldr s16, [sp, #72]
+; CHECK-GI-NEXT:    mov v23.s[1], w1
+; CHECK-GI-NEXT:    ldr s18, [sp, #80]
+; CHECK-GI-NEXT:    mov v21.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v24.s[1], w11
+; CHECK-GI-NEXT:    mov v0.s[1], v5.s[0]
+; CHECK-GI-NEXT:    fmov s6, w4
 ; CHECK-GI-NEXT:    add x10, sp, #88
-; CHECK-GI-NEXT:    movi v7.4s, #31
-; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
-; CHECK-GI-NEXT:    mov v19.s[2], w12
-; CHECK-GI-NEXT:    mov v23.s[2], w13
+; CHECK-GI-NEXT:    movi v22.4s, #31
+; CHECK-GI-NEXT:    mov v3.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v17.s[2], w11
+; CHECK-GI-NEXT:    mov v19.s[2], w13
 ; CHECK-GI-NEXT:    ldr s2, [sp, #16]
 ; CHECK-GI-NEXT:    ldr s1, [sp, #40]
-; CHECK-GI-NEXT:    ld1 { v18.s }[2], [x9]
-; CHECK-GI-NEXT:    ld1 { v24.s }[1], [x10]
-; CHECK-GI-NEXT:    eor v4.16b, v16.16b, v20.16b
-; CHECK-GI-NEXT:    mov v22.s[2], w2
-; CHECK-GI-NEXT:    mov v5.s[1], w5
+; CHECK-GI-NEXT:    ld1 { v18.s }[1], [x10]
+; CHECK-GI-NEXT:    eor v5.16b, v7.16b, v20.16b
+; CHECK-GI-NEXT:    mov v23.s[2], w2
+; CHECK-GI-NEXT:    mov v6.s[1], w5
 ; CHECK-GI-NEXT:    add x8, sp, #72
 ; CHECK-GI-NEXT:    add x9, sp, #96
 ; CHECK-GI-NEXT:    mov v21.s[3], v2.s[0]
-; CHECK-GI-NEXT:    mov v6.s[2], w12
+; CHECK-GI-NEXT:    mov v24.s[2], w11
 ; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-GI-NEXT:    ld1 { v18.s }[3], [x8]
-; CHECK-GI-NEXT:    bic v2.16b, v7.16b, v3.16b
-; CHECK-GI-NEXT:    ld1 { v24.s }[2], [x9]
-; CHECK-GI-NEXT:    and v1.16b, v4.16b, v19.16b
-; CHECK-GI-NEXT:    neg v3.4s, v23.4s
-; CHECK-GI-NEXT:    mov v22.s[3], w3
-; CHECK-GI-NEXT:    mov v5.s[2], w6
-; CHECK-GI-NEXT:    and v4.16b, v18.16b, v7.16b
-; CHECK-GI-NEXT:    ushr v7.4s, v21.4s, #1
+; CHECK-GI-NEXT:    ld1 { v4.s }[3], [x8]
+; CHECK-GI-NEXT:    bic v2.16b, v22.16b, v3.16b
+; CHECK-GI-NEXT:    ld1 { v18.s }[2], [x9]
+; CHECK-GI-NEXT:    and v1.16b, v5.16b, v17.16b
+; CHECK-GI-NEXT:    neg v3.4s, v19.4s
+; CHECK-GI-NEXT:    mov v23.s[3], w3
+; CHECK-GI-NEXT:    mov v6.s[2], w6
+; CHECK-GI-NEXT:    and v4.16b, v4.16b, v22.16b
+; CHECK-GI-NEXT:    ushr v5.4s, v21.4s, #1
 ; CHECK-GI-NEXT:    neg v2.4s, v2.4s
-; CHECK-GI-NEXT:    and v6.16b, v24.16b, v6.16b
+; CHECK-GI-NEXT:    and v7.16b, v18.16b, v24.16b
 ; CHECK-GI-NEXT:    neg v1.4s, v1.4s
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v22.4s, v4.4s
-; CHECK-GI-NEXT:    ushl v2.4s, v7.4s, v2.4s
-; CHECK-GI-NEXT:    ushl v4.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT:    ushl v3.4s, v23.4s, v4.4s
+; CHECK-GI-NEXT:    ushl v2.4s, v5.4s, v2.4s
+; CHECK-GI-NEXT:    ushl v4.4s, v6.4s, v7.4s
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v1.4s
 ; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v2.16b
 ; CHECK-GI-NEXT:    orr v0.16b, v4.16b, v0.16b
@@ -2662,86 +2662,86 @@ define <7 x i32> @fshr_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) {
 ;
 ; CHECK-GI-LABEL: fshr_v7i32:
 ; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr s4, [sp, #48]
+; CHECK-GI-NEXT:    add x8, sp, #56
 ; CHECK-GI-NEXT:    ldr s2, [sp, #48]
-; CHECK-GI-NEXT:    ldr s16, [sp, #56]
-; CHECK-GI-NEXT:    add x10, sp, #56
-; CHECK-GI-NEXT:    ldr s4, [sp, #80]
-; CHECK-GI-NEXT:    ldr s17, [sp, #88]
-; CHECK-GI-NEXT:    mov w11, #-1 // =0xffffffff
-; CHECK-GI-NEXT:    ldr s7, [sp, #48]
-; CHECK-GI-NEXT:    mov v2.s[1], v16.s[0]
-; CHECK-GI-NEXT:    mov v16.s[0], w0
-; CHECK-GI-NEXT:    mov v4.s[1], v17.s[0]
-; CHECK-GI-NEXT:    ldr s19, [sp, #64]
-; CHECK-GI-NEXT:    mov v18.s[0], w11
-; CHECK-GI-NEXT:    ld1 { v7.s }[1], [x10]
-; CHECK-GI-NEXT:    mov w10, #31 // =0x1f
-; CHECK-GI-NEXT:    add x9, sp, #64
-; CHECK-GI-NEXT:    mov v17.s[0], w10
+; CHECK-GI-NEXT:    ldr s18, [sp, #56]
+; CHECK-GI-NEXT:    ldr s5, [sp, #80]
+; CHECK-GI-NEXT:    add x9, sp, #72
+; CHECK-GI-NEXT:    ld1 { v4.s }[1], [x8]
+; CHECK-GI-NEXT:    ldr s19, [sp, #88]
+; CHECK-GI-NEXT:    add x8, sp, #64
+; CHECK-GI-NEXT:    mov v2.s[1], v18.s[0]
+; CHECK-GI-NEXT:    ldr s20, [sp, #64]
 ; CHECK-GI-NEXT:    ldr s21, [sp, #96]
-; CHECK-GI-NEXT:    mov v22.s[0], w4
-; CHECK-GI-NEXT:    mov v2.s[2], v19.s[0]
-; CHECK-GI-NEXT:    mov v19.s[0], w7
-; CHECK-GI-NEXT:    mov v16.s[1], w1
-; CHECK-GI-NEXT:    ld1 { v7.s }[2], [x9]
-; CHECK-GI-NEXT:    mov w9, #1 // =0x1
-; CHECK-GI-NEXT:    mov v4.s[2], v21.s[0]
-; CHECK-GI-NEXT:    mov v21.s[0], w10
-; CHECK-GI-NEXT:    mov v23.s[0], w9
-; CHECK-GI-NEXT:    ldr s6, [sp]
-; CHECK-GI-NEXT:    ldr s24, [sp, #80]
-; CHECK-GI-NEXT:    mov v17.s[1], w10
-; CHECK-GI-NEXT:    mov v18.s[1], w11
-; CHECK-GI-NEXT:    add x12, sp, #88
-; CHECK-GI-NEXT:    mov v19.s[1], v6.s[0]
-; CHECK-GI-NEXT:    add x8, sp, #72
-; CHECK-GI-NEXT:    ld1 { v24.s }[1], [x12]
-; CHECK-GI-NEXT:    mov v16.s[2], w2
-; CHECK-GI-NEXT:    mov v22.s[1], w5
-; CHECK-GI-NEXT:    mov v21.s[1], w10
-; CHECK-GI-NEXT:    mov v23.s[1], w9
-; CHECK-GI-NEXT:    ldr s5, [sp, #8]
+; CHECK-GI-NEXT:    mov v5.s[1], v19.s[0]
+; CHECK-GI-NEXT:    fmov s19, w0
+; CHECK-GI-NEXT:    ldr s7, [sp, #80]
+; CHECK-GI-NEXT:    ld1 { v4.s }[2], [x8]
+; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
+; CHECK-GI-NEXT:    ldr s16, [sp]
+; CHECK-GI-NEXT:    fmov s22, w8
+; CHECK-GI-NEXT:    add x10, sp, #88
+; CHECK-GI-NEXT:    fmov s23, w8
+; CHECK-GI-NEXT:    mov v2.s[2], v20.s[0]
+; CHECK-GI-NEXT:    ld1 { v7.s }[1], [x10]
+; CHECK-GI-NEXT:    mov v19.s[1], w1
+; CHECK-GI-NEXT:    ld1 { v4.s }[3], [x9]
+; CHECK-GI-NEXT:    mov w9, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    mov v5.s[2], v21.s[0]
+; CHECK-GI-NEXT:    fmov s20, w9
+; CHECK-GI-NEXT:    fmov s21, w7
+; CHECK-GI-NEXT:    mov w10, #1 // =0x1
+; CHECK-GI-NEXT:    mov v22.s[1], w8
+; CHECK-GI-NEXT:    fmov s24, w10
+; CHECK-GI-NEXT:    mov v23.s[1], w8
+; CHECK-GI-NEXT:    mov v19.s[2], w2
+; CHECK-GI-NEXT:    ldr s6, [sp, #8]
 ; CHECK-GI-NEXT:    ldr s0, [sp, #24]
+; CHECK-GI-NEXT:    mov v20.s[1], w9
+; CHECK-GI-NEXT:    mov v21.s[1], v16.s[0]
+; CHECK-GI-NEXT:    fmov s16, w4
+; CHECK-GI-NEXT:    mov v24.s[1], w10
 ; CHECK-GI-NEXT:    ldr s3, [sp, #32]
-; CHECK-GI-NEXT:    ld1 { v7.s }[3], [x8]
-; CHECK-GI-NEXT:    movi v6.4s, #31
-; CHECK-GI-NEXT:    add x8, sp, #96
-; CHECK-GI-NEXT:    mov v17.s[2], w10
-; CHECK-GI-NEXT:    mov v18.s[2], w11
-; CHECK-GI-NEXT:    ldr s20, [sp, #72]
-; CHECK-GI-NEXT:    ld1 { v24.s }[2], [x8]
-; CHECK-GI-NEXT:    mov v19.s[2], v5.s[0]
+; CHECK-GI-NEXT:    movi v18.4s, #31
+; CHECK-GI-NEXT:    add x11, sp, #96
+; CHECK-GI-NEXT:    mov v22.s[2], w8
+; CHECK-GI-NEXT:    ldr s17, [sp, #72]
+; CHECK-GI-NEXT:    mov v16.s[1], w5
+; CHECK-GI-NEXT:    ld1 { v7.s }[2], [x11]
 ; CHECK-GI-NEXT:    mov v0.s[1], v3.s[0]
-; CHECK-GI-NEXT:    mov v16.s[3], w3
-; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
-; CHECK-GI-NEXT:    mov v21.s[2], w10
-; CHECK-GI-NEXT:    mov v22.s[2], w6
-; CHECK-GI-NEXT:    mov v23.s[2], w9
+; CHECK-GI-NEXT:    mov v20.s[2], w9
+; CHECK-GI-NEXT:    mov v21.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v17.s[0]
+; CHECK-GI-NEXT:    mov v19.s[3], w3
+; CHECK-GI-NEXT:    mov v23.s[2], w8
+; CHECK-GI-NEXT:    mov v24.s[2], w10
 ; CHECK-GI-NEXT:    ldr s1, [sp, #16]
-; CHECK-GI-NEXT:    and v5.16b, v7.16b, v6.16b
+; CHECK-GI-NEXT:    and v4.16b, v4.16b, v18.16b
 ; CHECK-GI-NEXT:    ldr s3, [sp, #40]
-; CHECK-GI-NEXT:    and v7.16b, v24.16b, v17.16b
-; CHECK-GI-NEXT:    eor v4.16b, v4.16b, v18.16b
-; CHECK-GI-NEXT:    mov v19.s[3], v1.s[0]
-; CHECK-GI-NEXT:    shl v1.4s, v16.4s, #1
+; CHECK-GI-NEXT:    mov v16.s[2], w6
+; CHECK-GI-NEXT:    and v6.16b, v7.16b, v22.16b
+; CHECK-GI-NEXT:    eor v5.16b, v5.16b, v20.16b
+; CHECK-GI-NEXT:    mov v21.s[3], v1.s[0]
 ; CHECK-GI-NEXT:    mov v0.s[2], v3.s[0]
-; CHECK-GI-NEXT:    bic v2.16b, v6.16b, v2.16b
-; CHECK-GI-NEXT:    neg v5.4s, v5.4s
-; CHECK-GI-NEXT:    and v3.16b, v4.16b, v21.16b
-; CHECK-GI-NEXT:    ushl v4.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    neg v6.4s, v7.4s
+; CHECK-GI-NEXT:    bic v2.16b, v18.16b, v2.16b
+; CHECK-GI-NEXT:    shl v1.4s, v19.4s, #1
+; CHECK-GI-NEXT:    neg v4.4s, v4.4s
+; CHECK-GI-NEXT:    neg v6.4s, v6.4s
+; CHECK-GI-NEXT:    and v3.16b, v5.16b, v23.16b
+; CHECK-GI-NEXT:    ushl v5.4s, v16.4s, v24.4s
 ; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    ushl v2.4s, v19.4s, v5.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v3.4s
+; CHECK-GI-NEXT:    ushl v2.4s, v21.4s, v4.4s
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v6.4s
+; CHECK-GI-NEXT:    ushl v3.4s, v5.4s, v3.4s
 ; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v2.16b
 ; CHECK-GI-NEXT:    orr v0.16b, v3.16b, v0.16b
 ; CHECK-GI-NEXT:    mov s2, v1.s[1]
 ; CHECK-GI-NEXT:    mov s3, v1.s[2]
 ; CHECK-GI-NEXT:    mov s4, v1.s[3]
+; CHECK-GI-NEXT:    fmov w0, s1
 ; CHECK-GI-NEXT:    mov s5, v0.s[1]
 ; CHECK-GI-NEXT:    mov s6, v0.s[2]
-; CHECK-GI-NEXT:    fmov w0, s1
 ; CHECK-GI-NEXT:    fmov w4, s0
 ; CHECK-GI-NEXT:    fmov w1, s2
 ; CHECK-GI-NEXT:    fmov w2, s3
@@ -3587,35 +3587,35 @@ define <7 x i32> @rotl_v7i32_c(<7 x i32> %a) {
 ;
 ; CHECK-GI-LABEL: rotl_v7i32_c:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v1.s[0], w0
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w0
 ; CHECK-GI-NEXT:    mov w8, #29 // =0x1d
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov w9, #3 // =0x3
-; CHECK-GI-NEXT:    mov v3.s[0], w4
-; CHECK-GI-NEXT:    mov v4.s[0], w9
-; CHECK-GI-NEXT:    mov v5.s[0], w4
-; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    fmov s3, w4
+; CHECK-GI-NEXT:    fmov s4, w4
+; CHECK-GI-NEXT:    fmov s5, w9
 ; CHECK-GI-NEXT:    mov v1.s[1], w1
+; CHECK-GI-NEXT:    mov v0.s[1], w1
 ; CHECK-GI-NEXT:    mov v2.s[1], w8
 ; CHECK-GI-NEXT:    mov v3.s[1], w5
-; CHECK-GI-NEXT:    mov v4.s[1], w9
-; CHECK-GI-NEXT:    mov v5.s[1], w5
-; CHECK-GI-NEXT:    mov v0.s[2], w2
+; CHECK-GI-NEXT:    mov v4.s[1], w5
+; CHECK-GI-NEXT:    mov v5.s[1], w9
 ; CHECK-GI-NEXT:    mov v1.s[2], w2
+; CHECK-GI-NEXT:    mov v0.s[2], w2
 ; CHECK-GI-NEXT:    mov v2.s[2], w8
 ; CHECK-GI-NEXT:    mov v3.s[2], w6
-; CHECK-GI-NEXT:    mov v4.s[2], w9
-; CHECK-GI-NEXT:    mov v5.s[2], w6
-; CHECK-GI-NEXT:    mov v0.s[3], w3
+; CHECK-GI-NEXT:    mov v4.s[2], w6
+; CHECK-GI-NEXT:    mov v5.s[2], w9
 ; CHECK-GI-NEXT:    mov v1.s[3], w3
+; CHECK-GI-NEXT:    mov v0.s[3], w3
 ; CHECK-GI-NEXT:    neg v2.4s, v2.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
-; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #29
-; CHECK-GI-NEXT:    ushl v2.4s, v5.4s, v2.4s
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v2.16b
+; CHECK-GI-NEXT:    ushl v4.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #3
+; CHECK-GI-NEXT:    ushr v0.4s, v0.4s, #29
+; CHECK-GI-NEXT:    ushl v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    orr v1.16b, v4.16b, v2.16b
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v0.s[2]
 ; CHECK-GI-NEXT:    mov s4, v0.s[3]
@@ -3664,35 +3664,35 @@ define <7 x i32> @rotr_v7i32_c(<7 x i32> %a) {
 ;
 ; CHECK-GI-LABEL: rotr_v7i32_c:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v1.s[0], w0
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w0
 ; CHECK-GI-NEXT:    mov w8, #3 // =0x3
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov w9, #29 // =0x1d
-; CHECK-GI-NEXT:    mov v3.s[0], w4
-; CHECK-GI-NEXT:    mov v4.s[0], w4
-; CHECK-GI-NEXT:    mov v5.s[0], w9
-; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    fmov s3, w4
+; CHECK-GI-NEXT:    fmov s4, w4
+; CHECK-GI-NEXT:    fmov s5, w9
 ; CHECK-GI-NEXT:    mov v1.s[1], w1
+; CHECK-GI-NEXT:    mov v0.s[1], w1
 ; CHECK-GI-NEXT:    mov v2.s[1], w8
 ; CHECK-GI-NEXT:    mov v3.s[1], w5
 ; CHECK-GI-NEXT:    mov v4.s[1], w5
 ; CHECK-GI-NEXT:    mov v5.s[1], w9
-; CHECK-GI-NEXT:    mov v0.s[2], w2
 ; CHECK-GI-NEXT:    mov v1.s[2], w2
+; CHECK-GI-NEXT:    mov v0.s[2], w2
 ; CHECK-GI-NEXT:    mov v2.s[2], w8
 ; CHECK-GI-NEXT:    mov v3.s[2], w6
 ; CHECK-GI-NEXT:    mov v4.s[2], w6
 ; CHECK-GI-NEXT:    mov v5.s[2], w9
-; CHECK-GI-NEXT:    mov v0.s[3], w3
 ; CHECK-GI-NEXT:    mov v1.s[3], w3
+; CHECK-GI-NEXT:    mov v0.s[3], w3
 ; CHECK-GI-NEXT:    neg v2.4s, v2.4s
-; CHECK-GI-NEXT:    ushl v4.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    ushr v0.4s, v0.4s, #3
-; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #29
-; CHECK-GI-NEXT:    ushl v2.4s, v3.4s, v2.4s
-; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    orr v1.16b, v2.16b, v4.16b
+; CHECK-GI-NEXT:    ushl v3.4s, v3.4s, v5.4s
+; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #3
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
+; CHECK-GI-NEXT:    ushl v2.4s, v4.4s, v2.4s
+; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    orr v1.16b, v2.16b, v3.16b
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v0.s[2]
 ; CHECK-GI-NEXT:    mov s4, v0.s[3]
@@ -4200,39 +4200,39 @@ define <7 x i32> @fshl_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
 ;
 ; CHECK-GI-LABEL: fshl_v7i32_c:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
+; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    mov w8, #29 // =0x1d
-; CHECK-GI-NEXT:    mov v2.s[0], w7
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s5, w7
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ldr s1, [sp]
 ; CHECK-GI-NEXT:    mov w9, #3 // =0x3
-; CHECK-GI-NEXT:    mov v4.s[0], w4
-; CHECK-GI-NEXT:    mov v5.s[0], w9
-; CHECK-GI-NEXT:    ldr s3, [sp]
-; CHECK-GI-NEXT:    ldr s6, [sp, #24]
-; CHECK-GI-NEXT:    ldr s7, [sp, #32]
+; CHECK-GI-NEXT:    fmov s7, w9
+; CHECK-GI-NEXT:    ldr s4, [sp, #24]
+; CHECK-GI-NEXT:    ldr s6, [sp, #32]
 ; CHECK-GI-NEXT:    mov v0.s[1], w1
-; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
-; CHECK-GI-NEXT:    ldr s3, [sp, #8]
-; CHECK-GI-NEXT:    mov v1.s[1], w8
-; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT:    mov v4.s[1], w5
-; CHECK-GI-NEXT:    mov v5.s[1], w9
-; CHECK-GI-NEXT:    ldr s7, [sp, #40]
+; CHECK-GI-NEXT:    mov v5.s[1], v1.s[0]
+; CHECK-GI-NEXT:    fmov s1, w4
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v4.s[1], v6.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #8]
+; CHECK-GI-NEXT:    mov v7.s[1], w9
+; CHECK-GI-NEXT:    ldr s6, [sp, #40]
+; CHECK-GI-NEXT:    mov v1.s[1], w5
 ; CHECK-GI-NEXT:    mov v0.s[2], w2
-; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
-; CHECK-GI-NEXT:    ldr s3, [sp, #16]
-; CHECK-GI-NEXT:    mov v1.s[2], w8
-; CHECK-GI-NEXT:    mov v6.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v4.s[2], w6
-; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    mov v5.s[2], v2.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], w9
+; CHECK-GI-NEXT:    mov v1.s[2], w6
 ; CHECK-GI-NEXT:    mov v0.s[3], w3
-; CHECK-GI-NEXT:    mov v2.s[3], v3.s[0]
-; CHECK-GI-NEXT:    neg v1.4s, v1.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    mov v5.s[3], v2.s[0]
+; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v7.4s
 ; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
-; CHECK-GI-NEXT:    ushl v1.4s, v6.4s, v1.4s
-; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #29
-; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
+; CHECK-GI-NEXT:    ushl v2.4s, v4.4s, v3.4s
+; CHECK-GI-NEXT:    usra v0.4s, v5.4s, #29
+; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v2.16b
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v0.s[2]
 ; CHECK-GI-NEXT:    mov s4, v0.s[3]
@@ -4293,39 +4293,39 @@ define <7 x i32> @fshr_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
 ;
 ; CHECK-GI-LABEL: fshr_v7i32_c:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
+; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    mov w8, #3 // =0x3
-; CHECK-GI-NEXT:    mov v2.s[0], w7
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s5, w7
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ldr s1, [sp]
 ; CHECK-GI-NEXT:    mov w9, #29 // =0x1d
-; CHECK-GI-NEXT:    mov v4.s[0], w4
-; CHECK-GI-NEXT:    mov v5.s[0], w9
-; CHECK-GI-NEXT:    ldr s3, [sp]
-; CHECK-GI-NEXT:    ldr s6, [sp, #24]
-; CHECK-GI-NEXT:    ldr s7, [sp, #32]
+; CHECK-GI-NEXT:    fmov s7, w9
+; CHECK-GI-NEXT:    ldr s4, [sp, #24]
+; CHECK-GI-NEXT:    ldr s6, [sp, #32]
 ; CHECK-GI-NEXT:    mov v0.s[1], w1
-; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
-; CHECK-GI-NEXT:    ldr s3, [sp, #8]
-; CHECK-GI-NEXT:    mov v1.s[1], w8
-; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT:    mov v4.s[1], w5
-; CHECK-GI-NEXT:    mov v5.s[1], w9
-; CHECK-GI-NEXT:    ldr s7, [sp, #40]
+; CHECK-GI-NEXT:    mov v5.s[1], v1.s[0]
+; CHECK-GI-NEXT:    fmov s1, w4
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v4.s[1], v6.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #8]
+; CHECK-GI-NEXT:    mov v7.s[1], w9
+; CHECK-GI-NEXT:    ldr s6, [sp, #40]
+; CHECK-GI-NEXT:    mov v1.s[1], w5
 ; CHECK-GI-NEXT:    mov v0.s[2], w2
-; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
-; CHECK-GI-NEXT:    ldr s3, [sp, #16]
-; CHECK-GI-NEXT:    mov v1.s[2], w8
-; CHECK-GI-NEXT:    mov v6.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v4.s[2], w6
-; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    mov v5.s[2], v2.s[0]
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], w9
+; CHECK-GI-NEXT:    mov v1.s[2], w6
 ; CHECK-GI-NEXT:    mov v0.s[3], w3
-; CHECK-GI-NEXT:    mov v2.s[3], v3.s[0]
-; CHECK-GI-NEXT:    neg v1.4s, v1.4s
-; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    mov v5.s[3], v2.s[0]
+; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v7.4s
 ; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
-; CHECK-GI-NEXT:    ushl v1.4s, v6.4s, v1.4s
-; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #3
-; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
+; CHECK-GI-NEXT:    ushl v2.4s, v4.4s, v3.4s
+; CHECK-GI-NEXT:    usra v0.4s, v5.4s, #3
+; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v2.16b
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v0.s[2]
 ; CHECK-GI-NEXT:    mov s4, v0.s[3]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/fuse-addr-mode.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/fuse-addr-mode.mir
new file mode 100644
index 000000000000..5d218d0a2f24
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/fuse-addr-mode.mir
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64 -o - %s -run-pass machine-sink | FileCheck %s
+
+# we want to fuse an addition with lsl into an ldr but we have to be careful with
+# the shift distance: we can only represent specific shift distances: e.g: 3
+# but nothing large like 32
+
+--- |
+  define dso_local i64 @fuse_shift_add_into_addr_mode()  {
+  entry:
+    ret i64 0
+  }
+
+---
+name:            fuse_shift_add_into_addr_mode
+body:             |
+  bb.1.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: fuse_shift_add_into_addr_mode
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY]], [[COPY1]], 0, 1 :: (load (s64))
+    ; CHECK-NEXT: [[ADDXrs:%[0-9]+]]:gpr64common = ADDXrs [[COPY]], [[COPY1]], 5
+    ; CHECK-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs]], 0 :: (load (s64))
+    ; CHECK-NEXT: [[ADDXrs1:%[0-9]+]]:gpr64common = ADDXrs [[COPY]], [[COPY1]], 32
+    ; CHECK-NEXT: [[LDRXui1:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs1]], 0 :: (load (s64))
+    ; CHECK-NEXT: [[ADDXrs2:%[0-9]+]]:gpr64common = ADDXrs [[COPY]], [[COPY1]], 35
+    ; CHECK-NEXT: [[LDRXui2:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs1]], 0 :: (load (s64))
+    ; CHECK-NEXT: [[ADDXrs3:%[0-9]+]]:gpr64common = ADDXrs [[COPY]], [[COPY1]], 63
+    ; CHECK-NEXT: [[LDRXui3:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs1]], 0 :: (load (s64))
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:gpr64 = COPY $x0
+    %1:gpr64 = COPY $x1
+    %2:gpr64common = ADDXrs %0, %1, 3
+    %3:gpr64 = LDRXui %2, 0 :: (load (s64))
+    %4:gpr64common = ADDXrs %0, %1, 5
+    %5:gpr64 = LDRXui %4, 0 :: (load (s64))
+    %6:gpr64common = ADDXrs %0, %1, 32
+    %7:gpr64 = LDRXui %6, 0 :: (load (s64))
+    %8:gpr64common = ADDXrs %0, %1, 35
+    %9:gpr64 = LDRXui %6, 0 :: (load (s64))
+    %10:gpr64common = ADDXrs %0, %1, 63
+    %11:gpr64 = LDRXui %6, 0 :: (load (s64))
+    RET_ReallyLR implicit $x0
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
index 2d84a69f3144..c76b50d69b87 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mattr=+sve    < %s | FileCheck %s -check-prefix CHECK-SVE
-; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s -check-prefix CHECK-SVE2p1
+; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s -check-prefix CHECK-SVE2p1-SME2 -check-prefix CHECK-SVE2p1
+; RUN: llc -mattr=+sve -mattr=+sme2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-SVE2p1-SME2 -check-prefix CHECK-SME2
 target triple = "aarch64-linux"
 
 ; Test combining of getActiveLaneMask with a pair of extract_vector operations.
@@ -13,12 +14,12 @@ define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0
 ; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
 ; CHECK-SVE-NEXT:    b use
 ;
-; CHECK-SVE2p1-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
-; CHECK-SVE2p1:       // %bb.0:
-; CHECK-SVE2p1-NEXT:    mov w8, w1
-; CHECK-SVE2p1-NEXT:    mov w9, w0
-; CHECK-SVE2p1-NEXT:    whilelo { p0.h, p1.h }, x9, x8
-; CHECK-SVE2p1-NEXT:    b use
+; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE2p1-SME2:       // %bb.0:
+; CHECK-SVE2p1-SME2-NEXT:    mov w8, w1
+; CHECK-SVE2p1-SME2-NEXT:    mov w9, w0
+; CHECK-SVE2p1-SME2-NEXT:    whilelo { p0.h, p1.h }, x9, x8
+; CHECK-SVE2p1-SME2-NEXT:    b use
     %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 %i, i32 %n)
     %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
     %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
@@ -34,10 +35,10 @@ define void @test_2x8bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n) #0
 ; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
 ; CHECK-SVE-NEXT:    b use
 ;
-; CHECK-SVE2p1-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
-; CHECK-SVE2p1:       // %bb.0:
-; CHECK-SVE2p1-NEXT:    whilelo { p0.h, p1.h }, x0, x1
-; CHECK-SVE2p1-NEXT:    b use
+; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
+; CHECK-SVE2p1-SME2:       // %bb.0:
+; CHECK-SVE2p1-SME2-NEXT:    whilelo { p0.h, p1.h }, x0, x1
+; CHECK-SVE2p1-SME2-NEXT:    b use
     %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 %i, i64 %n)
     %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
     %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
@@ -53,12 +54,12 @@ define void @test_edge_case_2x1bit_mask(i64 %i, i64 %n) #0 {
 ; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
 ; CHECK-SVE-NEXT:    b use
 ;
-; CHECK-SVE2p1-LABEL: test_edge_case_2x1bit_mask:
-; CHECK-SVE2p1:       // %bb.0:
-; CHECK-SVE2p1-NEXT:    whilelo p1.d, x0, x1
-; CHECK-SVE2p1-NEXT:    punpklo p0.h, p1.b
-; CHECK-SVE2p1-NEXT:    punpkhi p1.h, p1.b
-; CHECK-SVE2p1-NEXT:    b use
+; CHECK-SVE2p1-SME2-LABEL: test_edge_case_2x1bit_mask:
+; CHECK-SVE2p1-SME2:       // %bb.0:
+; CHECK-SVE2p1-SME2-NEXT:    whilelo p1.d, x0, x1
+; CHECK-SVE2p1-SME2-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE2p1-SME2-NEXT:    punpkhi p1.h, p1.b
+; CHECK-SVE2p1-SME2-NEXT:    b use
     %r = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 %i, i64 %n)
     %v0 = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1.i64(<vscale x 2 x i1> %r, i64 0)
     %v1 = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1.i64(<vscale x 2 x i1> %r, i64 1)
@@ -74,10 +75,10 @@ define void @test_boring_case_2x2bit_mask(i64 %i, i64 %n) #0 {
 ; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
 ; CHECK-SVE-NEXT:    b use
 ;
-; CHECK-SVE2p1-LABEL: test_boring_case_2x2bit_mask:
-; CHECK-SVE2p1:       // %bb.0:
-; CHECK-SVE2p1-NEXT:    whilelo { p0.d, p1.d }, x0, x1
-; CHECK-SVE2p1-NEXT:    b use
+; CHECK-SVE2p1-SME2-LABEL: test_boring_case_2x2bit_mask:
+; CHECK-SVE2p1-SME2:       // %bb.0:
+; CHECK-SVE2p1-SME2-NEXT:    whilelo { p0.d, p1.d }, x0, x1
+; CHECK-SVE2p1-SME2-NEXT:    b use
     %r = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 %i, i64 %n)
     %v0 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1.i64(<vscale x 4 x i1> %r, i64 0)
     %v1 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1.i64(<vscale x 4 x i1> %r, i64 2)
@@ -96,14 +97,14 @@ define void @test_partial_extract(i64 %i, i64 %n) #0 {
 ; CHECK-SVE-NEXT:    punpklo p1.h, p2.b
 ; CHECK-SVE-NEXT:    b use
 ;
-; CHECK-SVE2p1-LABEL: test_partial_extract:
-; CHECK-SVE2p1:       // %bb.0:
-; CHECK-SVE2p1-NEXT:    whilelo p0.h, x0, x1
-; CHECK-SVE2p1-NEXT:    punpklo p1.h, p0.b
-; CHECK-SVE2p1-NEXT:    punpkhi p2.h, p0.b
-; CHECK-SVE2p1-NEXT:    punpklo p0.h, p1.b
-; CHECK-SVE2p1-NEXT:    punpklo p1.h, p2.b
-; CHECK-SVE2p1-NEXT:    b use
+; CHECK-SVE2p1-SME2-LABEL: test_partial_extract:
+; CHECK-SVE2p1-SME2:       // %bb.0:
+; CHECK-SVE2p1-SME2-NEXT:    whilelo p0.h, x0, x1
+; CHECK-SVE2p1-SME2-NEXT:    punpklo p1.h, p0.b
+; CHECK-SVE2p1-SME2-NEXT:    punpkhi p2.h, p0.b
+; CHECK-SVE2p1-SME2-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE2p1-SME2-NEXT:    punpklo p1.h, p2.b
+; CHECK-SVE2p1-SME2-NEXT:    b use
     %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
     %v0 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
     %v1 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
@@ -111,7 +112,7 @@ define void @test_partial_extract(i64 %i, i64 %n) #0 {
     ret void
 }
 
-;; Negative test for when extracting a fixed-length vector.
+; Negative test for when extracting a fixed-length vector.
 define void @test_fixed_extract(i64 %i, i64 %n) #0 {
 ; CHECK-SVE-LABEL: test_fixed_extract:
 ; CHECK-SVE:       // %bb.0:
@@ -144,6 +145,21 @@ define void @test_fixed_extract(i64 %i, i64 %n) #0 {
 ; CHECK-SVE2p1-NEXT:    mov v1.s[1], w11
 ; CHECK-SVE2p1-NEXT:    // kill: def $d1 killed $d1 killed $q1
 ; CHECK-SVE2p1-NEXT:    b use
+;
+; CHECK-SME2-LABEL: test_fixed_extract:
+; CHECK-SME2:       // %bb.0:
+; CHECK-SME2-NEXT:    whilelo p0.h, x0, x1
+; CHECK-SME2-NEXT:    cset w8, mi
+; CHECK-SME2-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-SME2-NEXT:    mov z1.h, z0.h[1]
+; CHECK-SME2-NEXT:    mov z2.h, z0.h[5]
+; CHECK-SME2-NEXT:    mov z3.h, z0.h[4]
+; CHECK-SME2-NEXT:    fmov s0, w8
+; CHECK-SME2-NEXT:    zip1 z0.s, z0.s, z1.s
+; CHECK-SME2-NEXT:    zip1 z1.s, z3.s, z2.s
+; CHECK-SME2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-SME2-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-SME2-NEXT:    b use
     %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
     %v0 = call <2 x i1> @llvm.vector.extract.v2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
     %v1 = call <2 x i1> @llvm.vector.extract.v2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
@@ -151,6 +167,67 @@ define void @test_fixed_extract(i64 %i, i64 %n) #0 {
     ret void
 }
 
+; Illegal Types
+
+define void @test_2x16bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
+; CHECK-SVE-LABEL: test_2x16bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    rdvl x8, #1
+; CHECK-SVE-NEXT:    adds w8, w0, w8
+; CHECK-SVE-NEXT:    csinv w8, w8, wzr, lo
+; CHECK-SVE-NEXT:    whilelo p0.b, w0, w1
+; CHECK-SVE-NEXT:    whilelo p1.b, w8, w1
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-SME2-LABEL: test_2x16bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE2p1-SME2:       // %bb.0:
+; CHECK-SVE2p1-SME2-NEXT:    mov w8, w1
+; CHECK-SVE2p1-SME2-NEXT:    mov w9, w0
+; CHECK-SVE2p1-SME2-NEXT:    whilelo { p0.b, p1.b }, x9, x8
+; CHECK-SVE2p1-SME2-NEXT:    b use
+  %r = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i32(i32 %i, i32 %n)
+  %v0 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1.i64(<vscale x 32 x i1> %r, i64 0)
+  %v1 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1.i64(<vscale x 32 x i1> %r, i64 16)
+  tail call void @use(<vscale x 16 x i1> %v0, <vscale x 16 x i1> %v1)
+  ret void
+}
+
+define void @test_2x32bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
+; CHECK-SVE-LABEL: test_2x32bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    rdvl x8, #2
+; CHECK-SVE-NEXT:    rdvl x9, #1
+; CHECK-SVE-NEXT:    adds w8, w0, w8
+; CHECK-SVE-NEXT:    csinv w8, w8, wzr, lo
+; CHECK-SVE-NEXT:    adds w10, w8, w9
+; CHECK-SVE-NEXT:    csinv w10, w10, wzr, lo
+; CHECK-SVE-NEXT:    whilelo p3.b, w10, w1
+; CHECK-SVE-NEXT:    adds w9, w0, w9
+; CHECK-SVE-NEXT:    csinv w9, w9, wzr, lo
+; CHECK-SVE-NEXT:    whilelo p0.b, w0, w1
+; CHECK-SVE-NEXT:    whilelo p1.b, w9, w1
+; CHECK-SVE-NEXT:    whilelo p2.b, w8, w1
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-SME2-LABEL: test_2x32bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE2p1-SME2:       // %bb.0:
+; CHECK-SVE2p1-SME2-NEXT:    rdvl x8, #2
+; CHECK-SVE2p1-SME2-NEXT:    mov w9, w1
+; CHECK-SVE2p1-SME2-NEXT:    mov w10, w0
+; CHECK-SVE2p1-SME2-NEXT:    adds w8, w0, w8
+; CHECK-SVE2p1-SME2-NEXT:    csinv w8, w8, wzr, lo
+; CHECK-SVE2p1-SME2-NEXT:    whilelo { p0.b, p1.b }, x10, x9
+; CHECK-SVE2p1-SME2-NEXT:    whilelo { p2.b, p3.b }, x8, x9
+; CHECK-SVE2p1-SME2-NEXT:    b use
+  %r = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i32(i32 %i, i32 %n)
+  %v0 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1.i64(<vscale x 64 x i1> %r, i64 0)
+  %v1 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1.i64(<vscale x 64 x i1> %r, i64 16)
+  %v2 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1.i64(<vscale x 64 x i1> %r, i64 32)
+  %v3 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1.i64(<vscale x 64 x i1> %r, i64 48)
+  tail call void @use(<vscale x 16 x i1> %v0, <vscale x 16 x i1> %v1, <vscale x 16 x i1> %v2, <vscale x 16 x i1> %v3)
+  ret void
+}
+
 declare void @use(...)
 
 attributes #0 = { nounwind }
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-1.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-1.ll
index cc17e344c211..626310fc4ec2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-1.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-1.ll
@@ -23,9 +23,9 @@ define void @f1(i32 %a1, i32 %a2) {
 ;CHECK:	.type	.L_MergedGlobals,@object  // @_MergedGlobals
 ;CHECK:	.local	.L_MergedGlobals
 ;CHECK:	.comm	.L_MergedGlobals,8,4
-;CHECK: .set m, .L_MergedGlobals
-;CHECK: .set n, .L_MergedGlobals+4
+;CHECK: m = .L_MergedGlobals
+;CHECK: n = .L_MergedGlobals+4
 
 ;CHECK-APPLE-IOS: .zerofill __DATA,__bss,__MergedGlobals,8,2 ; @_MergedGlobals
-;CHECK-APPLE-IOS-NOT: .set _m, l__MergedGlobals
-;CHECK-APPLE-IOS-NOT: .set _n, l__MergedGlobals+4
+;CHECK-APPLE-IOS-NOT: _m = l__MergedGlobals
+;CHECK-APPLE-IOS-NOT: _n = l__MergedGlobals+4
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-2.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-2.ll
index 85d814c3177b..1b5333b907d2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-2.ll
@@ -32,21 +32,21 @@ define dso_local void @g1(i32 %a1, i32 %a2) {
 ;CHECK:	.comm	.L_MergedGlobals,12,4
 
 ;CHECK:	.globl	x
-;CHECK: .set x, .L_MergedGlobals
+;CHECK: x = .L_MergedGlobals
 ;CHECK: .size x, 4
 ;CHECK:	.globl	y
-;CHECK: .set y, .L_MergedGlobals+4
+;CHECK: y = .L_MergedGlobals+4
 ;CHECK: .size y, 4
 ;CHECK:	.globl	z
-;CHECK: .set z, .L_MergedGlobals+8
+;CHECK: z = .L_MergedGlobals+8
 ;CHECK: .size z, 4
 
 ;CHECK-APPLE-IOS: .zerofill __DATA,__common,__MergedGlobals_x,12,2
 
 ;CHECK-APPLE-IOS: .globl	_x
-;CHECK-APPLE-IOS: .set {{.*}}, __MergedGlobals_x
+;CHECK-APPLE-IOS: {{.*}} = __MergedGlobals_x
 ;CHECK-APPLE-IOS: .globl	_y
-;CHECK-APPLE-IOS: .set _y, __MergedGlobals_x+4
+;CHECK-APPLE-IOS: _y = __MergedGlobals_x+4
 ;CHECK-APPLE-IOS: .globl	_z
-;CHECK-APPLE-IOS: .set _z, __MergedGlobals_x+8
+;CHECK-APPLE-IOS: _z = __MergedGlobals_x+8
 ;CHECK-APPLE-IOS: .subsections_via_symbols
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-3.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-3.ll
index b3f58887139f..2a0ae1227455 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-3.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-3.ll
@@ -40,14 +40,14 @@ define dso_local void @f1(i32 %a1, i32 %a2, i32 %a3) {
 
 ;CHECK-APPLE-IOS: .globl  __MergedGlobals_x
 ;CHECK-APPLE-IOS: .zerofill __DATA,__common,__MergedGlobals_x,800,2
-;CHECK-APPLE-IOS: .set _x, __MergedGlobals_x
-;CHECK-APPLE-IOS: .set _y, __MergedGlobals_x+400
+;CHECK-APPLE-IOS: _x = __MergedGlobals_x
+;CHECK-APPLE-IOS: _y = __MergedGlobals_x+400
 
 ;CHECK: .type   .L_MergedGlobals,@object // @_MergedGlobals
 ;CHECK: .local  .L_MergedGlobals
 ;CHECK: .comm   .L_MergedGlobals,800,4
 ;CHECK: globl  x
-;CHECK: .set x, .L_MergedGlobals
+;CHECK: x = .L_MergedGlobals
 ;CHECK: globl  y
-;CHECK: .set y, .L_MergedGlobals+400
-;CHECK-NOT: .set z, .L_MergedGlobals
+;CHECK: y = .L_MergedGlobals+400
+;CHECK-NOT: z = .L_MergedGlobals
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll
index 9c694fc4d289..5292aa91fc38 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/global-merge-hidden-minsize.ll
@@ -16,10 +16,10 @@ attributes #0 = { minsize optsize }
 
 ; CHECK: .globl x
 ; CHECK: .hidden x
-; CHECK: .set x, .L_MergedGlobals
+; CHECK: x = .L_MergedGlobals
 ; CHECK: .size x, 4
 
 ; CHECK: .globl y
 ; CHECK: .hidden y
-; CHECK: .set y, .L_MergedGlobals+4
+; CHECK: y = .L_MergedGlobals+4
 ; CHECK: .size y, 4
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/i128_with_overflow.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/i128_with_overflow.ll
index e3a8feebaa60..9924b7c63f76 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/i128_with_overflow.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/i128_with_overflow.ll
@@ -10,9 +10,7 @@ define i128 @test_uadd_i128(i128 noundef %x, i128 noundef %y) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    adds x0, x0, x2
 ; CHECK-SD-NEXT:    adcs x1, x1, x3
-; CHECK-SD-NEXT:    cset w8, hs
-; CHECK-SD-NEXT:    cmp w8, #1
-; CHECK-SD-NEXT:    b.ne .LBB0_2
+; CHECK-SD-NEXT:    b.lo .LBB0_2
 ; CHECK-SD-NEXT:  // %bb.1: // %if.then
 ; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
@@ -66,9 +64,7 @@ define i128 @test_sadd_i128(i128 noundef %x, i128 noundef %y) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    adds x0, x0, x2
 ; CHECK-SD-NEXT:    adcs x1, x1, x3
-; CHECK-SD-NEXT:    cset w8, vs
-; CHECK-SD-NEXT:    cmp w8, #1
-; CHECK-SD-NEXT:    b.ne .LBB1_2
+; CHECK-SD-NEXT:    b.vc .LBB1_2
 ; CHECK-SD-NEXT:  // %bb.1: // %if.then
 ; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
@@ -122,9 +118,7 @@ define i128 @test_usub_i128(i128 noundef %x, i128 noundef %y) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    subs x0, x0, x2
 ; CHECK-SD-NEXT:    sbcs x1, x1, x3
-; CHECK-SD-NEXT:    cset w8, lo
-; CHECK-SD-NEXT:    cmp w8, #1
-; CHECK-SD-NEXT:    b.ne .LBB2_2
+; CHECK-SD-NEXT:    b.hs .LBB2_2
 ; CHECK-SD-NEXT:  // %bb.1: // %if.then
 ; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
@@ -178,9 +172,7 @@ define i128 @test_ssub_i128(i128 noundef %x, i128 noundef %y) {
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    subs x0, x0, x2
 ; CHECK-SD-NEXT:    sbcs x1, x1, x3
-; CHECK-SD-NEXT:    cset w8, vs
-; CHECK-SD-NEXT:    cmp w8, #1
-; CHECK-SD-NEXT:    b.ne .LBB3_2
+; CHECK-SD-NEXT:    b.vc .LBB3_2
 ; CHECK-SD-NEXT:  // %bb.1: // %if.then
 ; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/icmp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/icmp.ll
index e284795760c5..18665bcbeae8 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/icmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/icmp.ll
@@ -1228,18 +1228,18 @@ define <3 x i32> @v3i32_i32(<3 x i32> %a, <3 x i32> %b, <3 x i32> %d, <3 x i32>
 ; CHECK-GI-LABEL: v3i32_i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
-; CHECK-GI-NEXT:    mov w9, #-1 // =0xffffffff
 ; CHECK-GI-NEXT:    cmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov v5.s[0], w9
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    mov v4.s[1], w8
-; CHECK-GI-NEXT:    mov v5.s[1], w9
 ; CHECK-GI-NEXT:    mov v4.s[2], w8
-; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w8
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT:    neg v1.4s, v4.4s
-; CHECK-GI-NEXT:    sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v5.16b
+; CHECK-GI-NEXT:    neg v4.4s, v4.4s
+; CHECK-GI-NEXT:    sshl v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    eor v1.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-GI-NEXT:    and v1.16b, v3.16b, v1.16b
 ; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/ifunc-asm.ll
index 57fc2f0c9d7f..7aad6cce09cf 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/ifunc-asm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/ifunc-asm.ll
@@ -16,7 +16,7 @@ entry:
 @global_ifunc = ifunc i32 (i32), ptr @the_resolver
 ; ELF:             .globl global_ifunc
 ; ELF-NEXT:        .type global_ifunc,@gnu_indirect_function
-; ELF-NEXT:        .set global_ifunc, the_resolver
+; ELF-NEXT:        global_ifunc = the_resolver
 
 ; MACHO:           .section __DATA,__data
 ; MACHO-NEXT:      .p2align 3, 0x0
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/insertextract.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/insertextract.ll
index aa4f31fb5f53..5c89316e5f57 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/insertextract.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/insertextract.ll
@@ -630,7 +630,7 @@ define <8 x i8> @insert_v8i8_c(<8 x i8> %a, i8 %b, i32 %c) {
 ; CHECK-GI-NEXT:    mov w8, #1 // =0x1
 ; CHECK-GI-NEXT:    str d0, [sp, #8]
 ; CHECK-GI-NEXT:    and x9, x9, #0x7
-; CHECK-GI-NEXT:    mul x8, x9, x8
+; CHECK-GI-NEXT:    umull x8, w9, w8
 ; CHECK-GI-NEXT:    add x9, sp, #8
 ; CHECK-GI-NEXT:    strb w0, [x9, x8]
 ; CHECK-GI-NEXT:    ldr d0, [sp, #8]
@@ -682,7 +682,7 @@ define <16 x i8> @insert_v16i8_c(<16 x i8> %a, i8 %b, i32 %c) {
 ; CHECK-GI-NEXT:    mov w8, #1 // =0x1
 ; CHECK-GI-NEXT:    str q0, [sp]
 ; CHECK-GI-NEXT:    and x9, x9, #0xf
-; CHECK-GI-NEXT:    mul x8, x9, x8
+; CHECK-GI-NEXT:    umull x8, w9, w8
 ; CHECK-GI-NEXT:    mov x9, sp
 ; CHECK-GI-NEXT:    strb w0, [x9, x8]
 ; CHECK-GI-NEXT:    ldr q0, [sp], #16
@@ -964,21 +964,13 @@ entry:
 }
 
 define <3 x i32> @insert_v3i32_0(<3 x i32> %a, i32 %b, i32 %c) {
-; CHECK-SD-LABEL: insert_v3i32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s1, w0
-; CHECK-SD-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-SD-NEXT:    mov v1.s[2], v0.s[2]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: insert_v3i32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[0], w0
-; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT:    mov v1.s[2], v0.s[2]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: insert_v3i32_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NEXT:    mov v1.s[2], v0.s[2]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
 entry:
   %d = insertelement <3 x i32> %a, i32 %b, i32 0
   ret <3 x i32> %d
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/itofp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/itofp.ll
index 71b53c662bb2..e8194b9bd9b2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/itofp.ll
@@ -3315,11 +3315,11 @@ define <3 x double> @stofp_v3i8_v3f64(<3 x i8> %a) {
 ; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-GI-NEXT:    smov x8, v0.h[0]
 ; CHECK-GI-NEXT:    smov x9, v0.h[1]
-; CHECK-GI-NEXT:    mov v1.d[0], x8
+; CHECK-GI-NEXT:    fmov d1, x8
 ; CHECK-GI-NEXT:    smov x8, v0.h[2]
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    smov x9, v0.h[3]
-; CHECK-GI-NEXT:    mov v2.d[0], x8
+; CHECK-GI-NEXT:    fmov d2, x8
 ; CHECK-GI-NEXT:    scvtf v0.2d, v1.2d
 ; CHECK-GI-NEXT:    mov v2.d[1], x9
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
@@ -3360,11 +3360,11 @@ define <3 x double> @utofp_v3i8_v3f64(<3 x i8> %a) {
 ; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
 ; CHECK-GI-NEXT:    umov w8, v0.h[0]
 ; CHECK-GI-NEXT:    umov w9, v0.h[1]
-; CHECK-GI-NEXT:    mov v1.d[0], x8
+; CHECK-GI-NEXT:    fmov d1, x8
 ; CHECK-GI-NEXT:    umov w8, v0.h[2]
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    umov w9, v0.h[3]
-; CHECK-GI-NEXT:    mov v2.d[0], x8
+; CHECK-GI-NEXT:    fmov d2, x8
 ; CHECK-GI-NEXT:    ucvtf v0.2d, v1.2d
 ; CHECK-GI-NEXT:    mov v2.d[1], x9
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
@@ -4143,11 +4143,11 @@ entry:
 define <3 x float> @stofp_v3i128_v3f32(<3 x i128> %a) {
 ; CHECK-SD-LABEL: stofp_v3i128_v3f32:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-NEXT:    sub sp, sp, #80
+; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-SD-NEXT:    .cfi_offset w19, -8
 ; CHECK-SD-NEXT:    .cfi_offset w20, -16
 ; CHECK-SD-NEXT:    .cfi_offset w21, -24
@@ -4155,31 +4155,31 @@ define <3 x float> @stofp_v3i128_v3f32(<3 x i128> %a) {
 ; CHECK-SD-NEXT:    .cfi_offset w30, -48
 ; CHECK-SD-NEXT:    mov x21, x1
 ; CHECK-SD-NEXT:    mov x22, x0
-; CHECK-SD-NEXT:    mov x0, x2
-; CHECK-SD-NEXT:    mov x1, x3
-; CHECK-SD-NEXT:    mov x19, x5
-; CHECK-SD-NEXT:    mov x20, x4
+; CHECK-SD-NEXT:    mov x0, x4
+; CHECK-SD-NEXT:    mov x1, x5
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
 ; CHECK-SD-NEXT:    bl __floattisf
 ; CHECK-SD-NEXT:    mov x0, x22
 ; CHECK-SD-NEXT:    mov x1, x21
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __floattisf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
 ; CHECK-SD-NEXT:    mov x0, x20
 ; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __floattisf
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    mov v1.s[1], v0.s[0]
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
 ; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    add sp, sp, #64
+; CHECK-SD-NEXT:    add sp, sp, #80
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: stofp_v3i128_v3f32:
@@ -4227,11 +4227,11 @@ entry:
 define <3 x float> @utofp_v3i128_v3f32(<3 x i128> %a) {
 ; CHECK-SD-LABEL: utofp_v3i128_v3f32:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-NEXT:    sub sp, sp, #80
+; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-SD-NEXT:    .cfi_offset w19, -8
 ; CHECK-SD-NEXT:    .cfi_offset w20, -16
 ; CHECK-SD-NEXT:    .cfi_offset w21, -24
@@ -4239,31 +4239,31 @@ define <3 x float> @utofp_v3i128_v3f32(<3 x i128> %a) {
 ; CHECK-SD-NEXT:    .cfi_offset w30, -48
 ; CHECK-SD-NEXT:    mov x21, x1
 ; CHECK-SD-NEXT:    mov x22, x0
-; CHECK-SD-NEXT:    mov x0, x2
-; CHECK-SD-NEXT:    mov x1, x3
-; CHECK-SD-NEXT:    mov x19, x5
-; CHECK-SD-NEXT:    mov x20, x4
+; CHECK-SD-NEXT:    mov x0, x4
+; CHECK-SD-NEXT:    mov x1, x5
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
 ; CHECK-SD-NEXT:    bl __floatuntisf
 ; CHECK-SD-NEXT:    mov x0, x22
 ; CHECK-SD-NEXT:    mov x1, x21
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __floatuntisf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
 ; CHECK-SD-NEXT:    mov x0, x20
 ; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __floatuntisf
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    mov v1.s[1], v0.s[0]
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
 ; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    add sp, sp, #64
+; CHECK-SD-NEXT:    add sp, sp, #80
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: utofp_v3i128_v3f32:
@@ -6035,11 +6035,11 @@ entry:
 define <3 x half> @stofp_v3i128_v3f16(<3 x i128> %a) {
 ; CHECK-SD-NOFP16-LABEL: stofp_v3i128_v3f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #64
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #80
+; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w19, -8
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w20, -16
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w21, -24
@@ -6047,40 +6047,41 @@ define <3 x half> @stofp_v3i128_v3f16(<3 x i128> %a) {
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -48
 ; CHECK-SD-NOFP16-NEXT:    mov x21, x1
 ; CHECK-SD-NOFP16-NEXT:    mov x22, x0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x2
-; CHECK-SD-NOFP16-NEXT:    mov x1, x3
-; CHECK-SD-NOFP16-NEXT:    mov x19, x5
-; CHECK-SD-NOFP16-NEXT:    mov x20, x4
+; CHECK-SD-NOFP16-NEXT:    mov x0, x4
+; CHECK-SD-NOFP16-NEXT:    mov x1, x5
+; CHECK-SD-NOFP16-NEXT:    mov x19, x3
+; CHECK-SD-NOFP16-NEXT:    mov x20, x2
 ; CHECK-SD-NOFP16-NEXT:    bl __floattisf
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    mov x0, x22
 ; CHECK-SD-NOFP16-NEXT:    mov x1, x21
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NOFP16-NEXT:    bl __floattisf
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NOFP16-NEXT:    mov x0, x20
 ; CHECK-SD-NOFP16-NEXT:    mov x1, x19
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
 ; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NOFP16-NEXT:    bl __floattisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
 ; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    add sp, sp, #64
+; CHECK-SD-NOFP16-NEXT:    add sp, sp, #80
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: stofp_v3i128_v3f16:
 ; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sub sp, sp, #64
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-FP16-NEXT:    sub sp, sp, #80
+; CHECK-SD-FP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w19, -8
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w20, -16
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w21, -24
@@ -6088,31 +6089,32 @@ define <3 x half> @stofp_v3i128_v3f16(<3 x i128> %a) {
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -48
 ; CHECK-SD-FP16-NEXT:    mov x21, x1
 ; CHECK-SD-FP16-NEXT:    mov x22, x0
-; CHECK-SD-FP16-NEXT:    mov x0, x2
-; CHECK-SD-FP16-NEXT:    mov x1, x3
-; CHECK-SD-FP16-NEXT:    mov x19, x5
-; CHECK-SD-FP16-NEXT:    mov x20, x4
+; CHECK-SD-FP16-NEXT:    mov x0, x4
+; CHECK-SD-FP16-NEXT:    mov x1, x5
+; CHECK-SD-FP16-NEXT:    mov x19, x3
+; CHECK-SD-FP16-NEXT:    mov x20, x2
 ; CHECK-SD-FP16-NEXT:    bl __floattihf
 ; CHECK-SD-FP16-NEXT:    mov x0, x22
 ; CHECK-SD-FP16-NEXT:    mov x1, x21
 ; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-FP16-NEXT:    bl __floattihf
-; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
 ; CHECK-SD-FP16-NEXT:    mov x0, x20
 ; CHECK-SD-FP16-NEXT:    mov x1, x19
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
 ; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-FP16-NEXT:    bl __floattihf
 ; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    mov v1.h[1], v0.h[0]
+; CHECK-SD-FP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-FP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-SD-FP16-NEXT:    fmov d0, d1
-; CHECK-SD-FP16-NEXT:    add sp, sp, #64
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-FP16-NEXT:    add sp, sp, #80
 ; CHECK-SD-FP16-NEXT:    ret
 ;
 ; CHECK-GI-NOFP16-LABEL: stofp_v3i128_v3f16:
@@ -6200,11 +6202,11 @@ entry:
 define <3 x half> @utofp_v3i128_v3f16(<3 x i128> %a) {
 ; CHECK-SD-NOFP16-LABEL: utofp_v3i128_v3f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #64
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #80
+; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w19, -8
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w20, -16
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w21, -24
@@ -6212,40 +6214,41 @@ define <3 x half> @utofp_v3i128_v3f16(<3 x i128> %a) {
 ; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -48
 ; CHECK-SD-NOFP16-NEXT:    mov x21, x1
 ; CHECK-SD-NOFP16-NEXT:    mov x22, x0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x2
-; CHECK-SD-NOFP16-NEXT:    mov x1, x3
-; CHECK-SD-NOFP16-NEXT:    mov x19, x5
-; CHECK-SD-NOFP16-NEXT:    mov x20, x4
+; CHECK-SD-NOFP16-NEXT:    mov x0, x4
+; CHECK-SD-NOFP16-NEXT:    mov x1, x5
+; CHECK-SD-NOFP16-NEXT:    mov x19, x3
+; CHECK-SD-NOFP16-NEXT:    mov x20, x2
 ; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    mov x0, x22
 ; CHECK-SD-NOFP16-NEXT:    mov x1, x21
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NOFP16-NEXT:    mov x0, x20
 ; CHECK-SD-NOFP16-NEXT:    mov x1, x19
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
 ; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
 ; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    add sp, sp, #64
+; CHECK-SD-NOFP16-NEXT:    add sp, sp, #80
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: utofp_v3i128_v3f16:
 ; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sub sp, sp, #64
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-FP16-NEXT:    sub sp, sp, #80
+; CHECK-SD-FP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w19, -8
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w20, -16
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w21, -24
@@ -6253,31 +6256,32 @@ define <3 x half> @utofp_v3i128_v3f16(<3 x i128> %a) {
 ; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -48
 ; CHECK-SD-FP16-NEXT:    mov x21, x1
 ; CHECK-SD-FP16-NEXT:    mov x22, x0
-; CHECK-SD-FP16-NEXT:    mov x0, x2
-; CHECK-SD-FP16-NEXT:    mov x1, x3
-; CHECK-SD-FP16-NEXT:    mov x19, x5
-; CHECK-SD-FP16-NEXT:    mov x20, x4
+; CHECK-SD-FP16-NEXT:    mov x0, x4
+; CHECK-SD-FP16-NEXT:    mov x1, x5
+; CHECK-SD-FP16-NEXT:    mov x19, x3
+; CHECK-SD-FP16-NEXT:    mov x20, x2
 ; CHECK-SD-FP16-NEXT:    bl __floatuntihf
 ; CHECK-SD-FP16-NEXT:    mov x0, x22
 ; CHECK-SD-FP16-NEXT:    mov x1, x21
 ; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-FP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-FP16-NEXT:    bl __floatuntihf
-; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
 ; CHECK-SD-FP16-NEXT:    mov x0, x20
 ; CHECK-SD-FP16-NEXT:    mov x1, x19
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
 ; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-FP16-NEXT:    bl __floatuntihf
 ; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-FP16-NEXT:    mov v1.h[1], v0.h[0]
+; CHECK-SD-FP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-FP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-SD-FP16-NEXT:    fmov d0, d1
-; CHECK-SD-FP16-NEXT:    add sp, sp, #64
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-FP16-NEXT:    add sp, sp, #80
 ; CHECK-SD-FP16-NEXT:    ret
 ;
 ; CHECK-GI-NOFP16-LABEL: utofp_v3i128_v3f16:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/loh-adrp-add-ldr-clobber.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/loh-adrp-add-ldr-clobber.mir
index ce2d8f02f4cc..a1d8bf375a19 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/loh-adrp-add-ldr-clobber.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/loh-adrp-add-ldr-clobber.mir
@@ -1,16 +1,34 @@
-# RUN: llc -o /dev/null %s -mtriple=aarch64-apple-ios -run-pass=aarch64-collect-loh -debug-only=aarch64-collect-loh 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -mtriple=aarch64-apple-ios -run-pass=aarch64-collect-loh -debug-only=aarch64-collect-loh 2>&1 | FileCheck %s --implicit-check-not=MCLOH_
 # REQUIRES: asserts
+
+# Check that we don't emit LOHs when there is a clobbering def of x8.
 --- |
   @sym2 = local_unnamed_addr global [10000000 x i32] zeroinitializer, align 8
   @sym = local_unnamed_addr global i32 zeroinitializer, align 8
 
-  define i32 @main() {
-    ret i32 0
-  }
+  define i32 @adrp_add_ldr() { ret i32 0 }
+  define i32 @adrp_add_str() { ret i32 0 }
+...
+
+---
+name:            adrp_add_ldr
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x21', virtual-reg: '' }
+body:             |
+  bb.0:
+    liveins: $x21
+    renamable $x8 = ADRP target-flags(aarch64-page) @sym
+    renamable $x9 = ADDXri killed renamable $x8, target-flags(aarch64-pageoff, aarch64-nc) @sym, 0
+    renamable $x8 = ADDXri killed renamable $x21, 1, 0
+    $x9 = LDRXui $x9, 0
 
+    RET undef $lr
 ...
+
 ---
-name:            main
+name:            adrp_add_str
 alignment:       4
 tracksRegLiveness: true
 liveins:
@@ -19,13 +37,10 @@ liveins:
 body:             |
   bb.0:
     liveins: $x21, $x22
-    ; Check we don't emit an loh here because there's a clobbering def of x8 before the ldr.
-    ; CHECK-LABEL: main
-    ; CHECK-NOT: MCLOH_AdrpAddLdr
     renamable $x8 = ADRP target-flags(aarch64-page) @sym
     renamable $x9 = ADDXri killed renamable $x8, target-flags(aarch64-pageoff, aarch64-nc) @sym, 0
-    renamable $x8 = ADDXri killed renamable $x22, 1, 0
-    $x9 = LDRXui $x9, 0
-    RET undef $lr
+    renamable $x8 = ADDXri killed renamable $x21, 1, 0
+    STRXui $x22, $x9, 0
 
+    RET undef $lr
 ...
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/mul.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/mul.ll
index 8d9a6e6b9291..1558043f7f40 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/mul.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/mul.ll
@@ -404,14 +404,14 @@ define <2 x i64> @v2i64(<2 x i64> %d, <2 x i64> %e) {
 ;
 ; CHECK-GI-LABEL: v2i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %s = mul <2 x i64> %d, %e
@@ -449,16 +449,16 @@ define <3 x i64> @v3i64(<3 x i64> %d, <3 x i64> %e) {
 ; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d3
-; CHECK-GI-NEXT:    mov x10, v0.d[1]
-; CHECK-GI-NEXT:    mov x11, v3.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d3
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x10, x11
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    mov v0.d[1], x9
 ; CHECK-GI-NEXT:    fmov x9, d5
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
+; CHECK-GI-NEXT:    fmov x8, d2
 ; CHECK-GI-NEXT:    mul x8, x8, x9
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -503,10 +503,10 @@ define <4 x i64> @v4i64(<4 x i64> %d, <4 x i64> %e) {
 ; CHECK-GI-NEXT:    fmov x9, d1
 ; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x9, x9, x12
-; CHECK-GI-NEXT:    mov v0.d[0], x8
+; CHECK-GI-NEXT:    fmov d0, x8
 ; CHECK-GI-NEXT:    mul x11, x13, x14
-; CHECK-GI-NEXT:    mov v1.d[0], x9
 ; CHECK-GI-NEXT:    mov v0.d[1], x10
+; CHECK-GI-NEXT:    fmov d1, x9
 ; CHECK-GI-NEXT:    mov v1.d[1], x11
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -576,17 +576,17 @@ define <3 x i128> @v3i128(<3 x i128> %d, <3 x i128> %e) {
 ; CHECK-GI-NEXT:    mul x9, x2, x10
 ; CHECK-GI-NEXT:    umulh x14, x2, x10
 ; CHECK-GI-NEXT:    madd x10, x3, x10, x13
-; CHECK-GI-NEXT:    ldp x13, x15, [sp, #16]
+; CHECK-GI-NEXT:    ldp x15, x13, [sp, #16]
 ; CHECK-GI-NEXT:    mov x2, x9
 ; CHECK-GI-NEXT:    umulh x11, x0, x6
 ; CHECK-GI-NEXT:    mov x0, x8
-; CHECK-GI-NEXT:    mul x15, x4, x15
+; CHECK-GI-NEXT:    mul x13, x4, x13
 ; CHECK-GI-NEXT:    add x3, x10, x14
-; CHECK-GI-NEXT:    umulh x16, x4, x13
+; CHECK-GI-NEXT:    umulh x16, x4, x15
 ; CHECK-GI-NEXT:    add x1, x12, x11
-; CHECK-GI-NEXT:    madd x15, x5, x13, x15
-; CHECK-GI-NEXT:    mul x4, x4, x13
-; CHECK-GI-NEXT:    add x5, x15, x16
+; CHECK-GI-NEXT:    madd x13, x5, x15, x13
+; CHECK-GI-NEXT:    mul x4, x4, x15
+; CHECK-GI-NEXT:    add x5, x13, x16
 ; CHECK-GI-NEXT:    ret
 entry:
   %s = mul <3 x i128> %d, %e
@@ -638,14 +638,14 @@ define <4 x i128> @v4i128(<4 x i128> %d, <4 x i128> %e) {
 ; CHECK-GI-NEXT:    umulh x17, x4, x15
 ; CHECK-GI-NEXT:    add x3, x13, x14
 ; CHECK-GI-NEXT:    madd x15, x5, x15, x16
-; CHECK-GI-NEXT:    ldp x16, x18, [sp, #48]
+; CHECK-GI-NEXT:    ldp x18, x16, [sp, #48]
 ; CHECK-GI-NEXT:    mov x4, x10
-; CHECK-GI-NEXT:    mul x18, x6, x18
-; CHECK-GI-NEXT:    umulh x0, x6, x16
+; CHECK-GI-NEXT:    mul x16, x6, x16
+; CHECK-GI-NEXT:    umulh x0, x6, x18
 ; CHECK-GI-NEXT:    add x5, x15, x17
-; CHECK-GI-NEXT:    madd x18, x7, x16, x18
-; CHECK-GI-NEXT:    mul x6, x6, x16
-; CHECK-GI-NEXT:    add x7, x18, x0
+; CHECK-GI-NEXT:    madd x16, x7, x18, x16
+; CHECK-GI-NEXT:    mul x6, x6, x18
+; CHECK-GI-NEXT:    add x7, x16, x0
 ; CHECK-GI-NEXT:    mov x0, x8
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
index fb65a748c865..7e6f3548bdaf 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
@@ -1143,7 +1143,7 @@ define <4 x i32> @vselect_constant_cond_zero_v4i32(<4 x i32> %a) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #1 // =0x1
 ; CHECK-GI-NEXT:    mov w9, #0 // =0x0
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    mov v1.s[1], w9
 ; CHECK-GI-NEXT:    mov v1.s[2], w9
 ; CHECK-GI-NEXT:    mov v1.s[3], w8
@@ -1215,7 +1215,7 @@ define <4 x i32> @vselect_constant_cond_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #1 // =0x1
 ; CHECK-GI-NEXT:    mov w9, #0 // =0x0
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mov v2.s[2], w9
 ; CHECK-GI-NEXT:    mov v2.s[3], w8
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
index 2c2cb7211287..fb8b721769b2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
@@ -2585,7 +2585,7 @@ define <4 x i32> @fcmnv4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-GI-LABEL: fcmnv4xfloat:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #0 // =0x0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    mov v0.s[1], w8
 ; CHECK-GI-NEXT:    mov v0.d[1], v0.d[0]
 ; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #31
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 2d3fda704908..a534112b7c55 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -415,17 +415,17 @@ define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
 ; CHECK-GI-NEXT:    ldrb w8, [x0, #4]
 ; CHECK-GI-NEXT:    ldrb w9, [x1, #4]
 ; CHECK-GI-NEXT:    ldrb w10, [x1]
+; CHECK-GI-NEXT:    ldrb w11, [x0, #1]
+; CHECK-GI-NEXT:    ldrb w12, [x1, #1]
 ; CHECK-GI-NEXT:    mul w8, w9, w8
 ; CHECK-GI-NEXT:    ldrb w9, [x0]
-; CHECK-GI-NEXT:    mov v0.s[0], w10
-; CHECK-GI-NEXT:    mov v1.s[0], w9
-; CHECK-GI-NEXT:    ldrb w9, [x1, #1]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
-; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
-; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    fmov s0, w10
+; CHECK-GI-NEXT:    fmov s1, w9
 ; CHECK-GI-NEXT:    ldrb w9, [x1, #2]
-; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w12
 ; CHECK-GI-NEXT:    ldrb w8, [x0, #2]
+; CHECK-GI-NEXT:    mov v1.s[1], w11
 ; CHECK-GI-NEXT:    mov v2.s[1], wzr
 ; CHECK-GI-NEXT:    mov v0.s[2], w9
 ; CHECK-GI-NEXT:    ldrb w9, [x1, #3]
@@ -468,12 +468,12 @@ define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    ldrb w8, [x0]
 ; CHECK-GI-NEXT:    ldrb w9, [x0, #4]
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    mov v1.s[0], w9
-; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
-; CHECK-GI-NEXT:    mov v0.s[1], w8
-; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    ldrb w10, [x0, #1]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov s1, w9
 ; CHECK-GI-NEXT:    ldrb w8, [x0, #2]
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    mov v1.s[2], wzr
 ; CHECK-GI-NEXT:    ldrb w8, [x0, #3]
@@ -509,17 +509,17 @@ define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
 ; CHECK-GI-NEXT:    ldrsb w8, [x0, #4]
 ; CHECK-GI-NEXT:    ldrsb w9, [x1, #4]
 ; CHECK-GI-NEXT:    ldrsb w10, [x1]
+; CHECK-GI-NEXT:    ldrsb w11, [x0, #1]
+; CHECK-GI-NEXT:    ldrsb w12, [x1, #1]
 ; CHECK-GI-NEXT:    mul w8, w9, w8
 ; CHECK-GI-NEXT:    ldrsb w9, [x0]
-; CHECK-GI-NEXT:    mov v0.s[0], w10
-; CHECK-GI-NEXT:    mov v1.s[0], w9
-; CHECK-GI-NEXT:    ldrsb w9, [x1, #1]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
-; CHECK-GI-NEXT:    ldrsb w8, [x0, #1]
-; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    fmov s0, w10
+; CHECK-GI-NEXT:    fmov s1, w9
 ; CHECK-GI-NEXT:    ldrsb w9, [x1, #2]
-; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w12
 ; CHECK-GI-NEXT:    ldrsb w8, [x0, #2]
+; CHECK-GI-NEXT:    mov v1.s[1], w11
 ; CHECK-GI-NEXT:    mov v2.s[1], wzr
 ; CHECK-GI-NEXT:    mov v0.s[2], w9
 ; CHECK-GI-NEXT:    ldrsb w9, [x1, #3]
@@ -569,55 +569,55 @@ define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
 ; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
-; CHECK-GI-NEXT:    smov w8, v0.b[4]
-; CHECK-GI-NEXT:    smov w9, v1.b[4]
-; CHECK-GI-NEXT:    smov w10, v2.b[4]
-; CHECK-GI-NEXT:    smov w11, v3.b[4]
-; CHECK-GI-NEXT:    smov w12, v0.b[0]
-; CHECK-GI-NEXT:    smov w13, v1.b[0]
-; CHECK-GI-NEXT:    smov w14, v2.b[0]
-; CHECK-GI-NEXT:    smov w15, v3.b[0]
-; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    smov w9, v0.b[1]
-; CHECK-GI-NEXT:    mul w10, w10, w11
-; CHECK-GI-NEXT:    smov w11, v1.b[1]
-; CHECK-GI-NEXT:    mov v4.s[0], w12
-; CHECK-GI-NEXT:    smov w12, v2.b[1]
-; CHECK-GI-NEXT:    mov v5.s[0], w13
+; CHECK-GI-NEXT:    smov w9, v1.b[0]
+; CHECK-GI-NEXT:    smov w10, v0.b[4]
+; CHECK-GI-NEXT:    smov w11, v1.b[4]
+; CHECK-GI-NEXT:    smov w12, v2.b[0]
+; CHECK-GI-NEXT:    smov w13, v2.b[4]
+; CHECK-GI-NEXT:    smov w14, v3.b[4]
+; CHECK-GI-NEXT:    smov w8, v0.b[0]
+; CHECK-GI-NEXT:    smov w16, v3.b[0]
+; CHECK-GI-NEXT:    smov w15, v0.b[1]
+; CHECK-GI-NEXT:    fmov s5, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    smov w10, v1.b[1]
+; CHECK-GI-NEXT:    fmov s6, w12
+; CHECK-GI-NEXT:    mul w12, w13, w14
+; CHECK-GI-NEXT:    smov w11, v2.b[1]
 ; CHECK-GI-NEXT:    smov w13, v3.b[1]
-; CHECK-GI-NEXT:    mov v6.s[0], w8
-; CHECK-GI-NEXT:    mov v7.s[0], w14
-; CHECK-GI-NEXT:    mov v16.s[0], w15
-; CHECK-GI-NEXT:    mov v17.s[0], w10
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    fmov s7, w16
+; CHECK-GI-NEXT:    fmov s16, w9
 ; CHECK-GI-NEXT:    smov w8, v0.b[2]
-; CHECK-GI-NEXT:    smov w10, v1.b[2]
-; CHECK-GI-NEXT:    smov w14, v2.b[2]
-; CHECK-GI-NEXT:    smov w15, v3.b[2]
-; CHECK-GI-NEXT:    mov v4.s[1], w9
-; CHECK-GI-NEXT:    mov v5.s[1], w11
-; CHECK-GI-NEXT:    smov w9, v0.b[3]
-; CHECK-GI-NEXT:    smov w11, v1.b[3]
-; CHECK-GI-NEXT:    mov v6.s[1], wzr
-; CHECK-GI-NEXT:    mov v7.s[1], w12
-; CHECK-GI-NEXT:    mov v16.s[1], w13
+; CHECK-GI-NEXT:    smov w14, v1.b[2]
+; CHECK-GI-NEXT:    fmov s17, w12
+; CHECK-GI-NEXT:    smov w9, v3.b[2]
+; CHECK-GI-NEXT:    mov v5.s[1], w10
+; CHECK-GI-NEXT:    mov v4.s[1], w15
+; CHECK-GI-NEXT:    smov w15, v2.b[2]
+; CHECK-GI-NEXT:    mov v6.s[1], w11
+; CHECK-GI-NEXT:    mov v16.s[1], wzr
+; CHECK-GI-NEXT:    mov v7.s[1], w13
+; CHECK-GI-NEXT:    smov w10, v0.b[3]
 ; CHECK-GI-NEXT:    mov v17.s[1], wzr
+; CHECK-GI-NEXT:    smov w11, v1.b[3]
 ; CHECK-GI-NEXT:    smov w12, v2.b[3]
 ; CHECK-GI-NEXT:    smov w13, v3.b[3]
+; CHECK-GI-NEXT:    mov v5.s[2], w14
 ; CHECK-GI-NEXT:    mov v4.s[2], w8
-; CHECK-GI-NEXT:    mov v5.s[2], w10
-; CHECK-GI-NEXT:    mov v6.s[2], wzr
-; CHECK-GI-NEXT:    mov v7.s[2], w14
-; CHECK-GI-NEXT:    mov v16.s[2], w15
+; CHECK-GI-NEXT:    mov v6.s[2], w15
+; CHECK-GI-NEXT:    mov v16.s[2], wzr
+; CHECK-GI-NEXT:    mov v7.s[2], w9
 ; CHECK-GI-NEXT:    mov v17.s[2], wzr
-; CHECK-GI-NEXT:    mov v4.s[3], w9
 ; CHECK-GI-NEXT:    mov v5.s[3], w11
-; CHECK-GI-NEXT:    mov v6.s[3], wzr
-; CHECK-GI-NEXT:    mov v7.s[3], w12
-; CHECK-GI-NEXT:    mov v16.s[3], w13
+; CHECK-GI-NEXT:    mov v4.s[3], w10
+; CHECK-GI-NEXT:    mov v6.s[3], w12
+; CHECK-GI-NEXT:    mov v16.s[3], wzr
+; CHECK-GI-NEXT:    mov v7.s[3], w13
 ; CHECK-GI-NEXT:    mov v17.s[3], wzr
-; CHECK-GI-NEXT:    mla v6.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    mla v17.4s, v7.4s, v16.4s
-; CHECK-GI-NEXT:    addv s0, v6.4s
+; CHECK-GI-NEXT:    mla v16.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    mla v17.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    addv s0, v16.4s
 ; CHECK-GI-NEXT:    addv s1, v17.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
@@ -661,26 +661,26 @@ define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <
 ; CHECK-GI-NEXT:    smov w8, v0.b[0]
 ; CHECK-GI-NEXT:    smov w9, v0.b[4]
 ; CHECK-GI-NEXT:    smov w10, v2.b[0]
-; CHECK-GI-NEXT:    smov w11, v2.b[4]
-; CHECK-GI-NEXT:    smov w12, v0.b[1]
-; CHECK-GI-NEXT:    mov v1.s[0], w8
-; CHECK-GI-NEXT:    smov w8, v2.b[1]
-; CHECK-GI-NEXT:    mov v3.s[0], w9
-; CHECK-GI-NEXT:    mov v4.s[0], w10
-; CHECK-GI-NEXT:    mov v5.s[0], w11
-; CHECK-GI-NEXT:    smov w9, v0.b[2]
-; CHECK-GI-NEXT:    smov w10, v2.b[2]
-; CHECK-GI-NEXT:    smov w11, v2.b[3]
-; CHECK-GI-NEXT:    mov v1.s[1], w12
+; CHECK-GI-NEXT:    smov w12, v2.b[4]
+; CHECK-GI-NEXT:    smov w11, v0.b[1]
+; CHECK-GI-NEXT:    smov w13, v2.b[1]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    fmov s4, w10
+; CHECK-GI-NEXT:    fmov s5, w12
+; CHECK-GI-NEXT:    smov w8, v0.b[2]
+; CHECK-GI-NEXT:    smov w9, v2.b[2]
+; CHECK-GI-NEXT:    smov w10, v0.b[3]
+; CHECK-GI-NEXT:    mov v1.s[1], w11
 ; CHECK-GI-NEXT:    mov v3.s[1], wzr
-; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    mov v4.s[1], w13
 ; CHECK-GI-NEXT:    mov v5.s[1], wzr
-; CHECK-GI-NEXT:    smov w8, v0.b[3]
-; CHECK-GI-NEXT:    mov v1.s[2], w9
+; CHECK-GI-NEXT:    smov w11, v2.b[3]
+; CHECK-GI-NEXT:    mov v1.s[2], w8
 ; CHECK-GI-NEXT:    mov v3.s[2], wzr
-; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    mov v4.s[2], w9
 ; CHECK-GI-NEXT:    mov v5.s[2], wzr
-; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mov v1.s[3], w10
 ; CHECK-GI-NEXT:    mov v3.s[3], wzr
 ; CHECK-GI-NEXT:    mov v4.s[3], w11
 ; CHECK-GI-NEXT:    mov v5.s[3], wzr
@@ -2298,125 +2298,125 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_udot_v25i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr q1, [x1]
-; CHECK-GI-NEXT:    ldrb w11, [x1, #16]!
-; CHECK-GI-NEXT:    ldrb w12, [x1, #4]
-; CHECK-GI-NEXT:    ldr q0, [x0]
-; CHECK-GI-NEXT:    mov v23.s[0], wzr
-; CHECK-GI-NEXT:    umov w9, v1.b[4]
-; CHECK-GI-NEXT:    umov w10, v1.b[12]
-; CHECK-GI-NEXT:    umov w13, v1.b[0]
-; CHECK-GI-NEXT:    umov w14, v1.b[5]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov v3.s[0], w11
-; CHECK-GI-NEXT:    umov w11, v0.b[0]
-; CHECK-GI-NEXT:    umov w12, v1.b[1]
-; CHECK-GI-NEXT:    umov w15, v1.b[8]
-; CHECK-GI-NEXT:    ldrb w8, [x0, #16]!
-; CHECK-GI-NEXT:    mov v23.s[1], wzr
-; CHECK-GI-NEXT:    mov v2.s[0], w9
-; CHECK-GI-NEXT:    mov v4.s[0], w10
+; CHECK-GI-NEXT:    ldr q2, [x1]
+; CHECK-GI-NEXT:    ldrb w9, [x1, #16]!
+; CHECK-GI-NEXT:    ldrb w11, [x1, #4]
+; CHECK-GI-NEXT:    ldrb w12, [x1, #5]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    umov w13, v2.b[4]
+; CHECK-GI-NEXT:    umov w14, v2.b[5]
+; CHECK-GI-NEXT:    umov w10, v2.b[0]
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    umov w9, v2.b[8]
+; CHECK-GI-NEXT:    fmov s5, w11
+; CHECK-GI-NEXT:    umov w11, v2.b[12]
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    ldrb w8, [x1, #1]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    fmov s7, w13
+; CHECK-GI-NEXT:    fmov s4, w10
+; CHECK-GI-NEXT:    umov w10, v2.b[13]
+; CHECK-GI-NEXT:    mov v5.s[1], w12
+; CHECK-GI-NEXT:    umov w13, v2.b[9]
+; CHECK-GI-NEXT:    fmov s6, w9
+; CHECK-GI-NEXT:    fmov s16, w11
+; CHECK-GI-NEXT:    umov w9, v1.b[0]
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v7.s[1], w14
+; CHECK-GI-NEXT:    umov w14, v2.b[6]
+; CHECK-GI-NEXT:    ldrb w12, [x1, #6]
+; CHECK-GI-NEXT:    umov w8, v2.b[1]
+; CHECK-GI-NEXT:    umov w11, v2.b[2]
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    mov v16.s[1], w10
+; CHECK-GI-NEXT:    umov w10, v2.b[14]
+; CHECK-GI-NEXT:    mov v5.s[2], w12
+; CHECK-GI-NEXT:    umov w12, v1.b[5]
+; CHECK-GI-NEXT:    mov v6.s[1], w13
+; CHECK-GI-NEXT:    fmov s17, w9
+; CHECK-GI-NEXT:    mov v7.s[2], w14
+; CHECK-GI-NEXT:    umov w14, v1.b[4]
+; CHECK-GI-NEXT:    umov w9, v2.b[10]
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    umov w8, v1.b[1]
+; CHECK-GI-NEXT:    umov w13, v2.b[7]
+; CHECK-GI-NEXT:    mov v16.s[2], w10
+; CHECK-GI-NEXT:    umov w10, v2.b[15]
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    fmov s18, w14
+; CHECK-GI-NEXT:    mov v6.s[2], w9
+; CHECK-GI-NEXT:    umov w9, v1.b[12]
+; CHECK-GI-NEXT:    mov v4.s[2], w11
+; CHECK-GI-NEXT:    ldrb w11, [x1, #7]
+; CHECK-GI-NEXT:    mov v17.s[1], w8
+; CHECK-GI-NEXT:    ldrb w8, [x1, #2]
+; CHECK-GI-NEXT:    mov v16.s[3], w10
 ; CHECK-GI-NEXT:    umov w10, v1.b[13]
-; CHECK-GI-NEXT:    ldrb w9, [x1, #5]
-; CHECK-GI-NEXT:    mov v6.s[0], w13
-; CHECK-GI-NEXT:    umov w13, v1.b[6]
-; CHECK-GI-NEXT:    mov v16.s[0], w11
-; CHECK-GI-NEXT:    umov w11, v1.b[2]
-; CHECK-GI-NEXT:    mov v7.s[0], w15
-; CHECK-GI-NEXT:    mov v5.s[1], w9
-; CHECK-GI-NEXT:    ldrb w9, [x1, #6]
-; CHECK-GI-NEXT:    umov w15, v1.b[9]
-; CHECK-GI-NEXT:    mov v2.s[1], w14
-; CHECK-GI-NEXT:    ldrb w14, [x1, #1]
-; CHECK-GI-NEXT:    mov v4.s[1], w10
-; CHECK-GI-NEXT:    umov w10, v1.b[14]
-; CHECK-GI-NEXT:    mov v6.s[1], w12
-; CHECK-GI-NEXT:    umov w12, v0.b[1]
-; CHECK-GI-NEXT:    mov v3.s[1], w14
-; CHECK-GI-NEXT:    umov w14, v0.b[12]
-; CHECK-GI-NEXT:    mov v21.s[0], w8
-; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
-; CHECK-GI-NEXT:    mov v5.s[2], w9
-; CHECK-GI-NEXT:    umov w9, v0.b[4]
-; CHECK-GI-NEXT:    mov v2.s[2], w13
-; CHECK-GI-NEXT:    umov w13, v1.b[7]
-; CHECK-GI-NEXT:    mov v7.s[1], w15
-; CHECK-GI-NEXT:    mov v4.s[2], w10
-; CHECK-GI-NEXT:    umov w10, v1.b[15]
-; CHECK-GI-NEXT:    mov v16.s[1], w12
-; CHECK-GI-NEXT:    ldrb w12, [x1, #2]
-; CHECK-GI-NEXT:    mov v6.s[2], w11
-; CHECK-GI-NEXT:    umov w11, v0.b[2]
-; CHECK-GI-NEXT:    mov v17.s[0], w9
-; CHECK-GI-NEXT:    umov w9, v0.b[8]
-; CHECK-GI-NEXT:    mov v18.s[0], w14
-; CHECK-GI-NEXT:    mov v2.s[3], w13
-; CHECK-GI-NEXT:    ldrb w13, [x1, #7]
-; CHECK-GI-NEXT:    mov v3.s[2], w12
+; CHECK-GI-NEXT:    mov v18.s[1], w12
+; CHECK-GI-NEXT:    umov w12, v1.b[6]
+; CHECK-GI-NEXT:    mov v5.s[3], w11
+; CHECK-GI-NEXT:    ldrb w11, [x0, #16]!
+; CHECK-GI-NEXT:    mov v7.s[3], w13
+; CHECK-GI-NEXT:    umov w13, v1.b[2]
+; CHECK-GI-NEXT:    fmov s20, w9
+; CHECK-GI-NEXT:    ldrb w9, [x0, #5]
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    umov w8, v1.b[8]
+; CHECK-GI-NEXT:    fmov s22, w11
+; CHECK-GI-NEXT:    mov v18.s[2], w12
 ; CHECK-GI-NEXT:    ldrb w12, [x0, #4]
-; CHECK-GI-NEXT:    mov v4.s[3], w10
-; CHECK-GI-NEXT:    umov w10, v0.b[5]
-; CHECK-GI-NEXT:    mov v5.s[3], w13
-; CHECK-GI-NEXT:    ldrb w13, [x0, #8]
-; CHECK-GI-NEXT:    mov v16.s[2], w11
-; CHECK-GI-NEXT:    umov w11, v0.b[13]
-; CHECK-GI-NEXT:    mov v20.s[0], w12
-; CHECK-GI-NEXT:    ldrb w12, [x1, #8]
-; CHECK-GI-NEXT:    mov v19.s[0], w9
-; CHECK-GI-NEXT:    umov w9, v0.b[6]
-; CHECK-GI-NEXT:    umov w15, v1.b[10]
-; CHECK-GI-NEXT:    mul w12, w12, w13
-; CHECK-GI-NEXT:    mov v17.s[1], w10
-; CHECK-GI-NEXT:    ldrb w10, [x0, #5]
-; CHECK-GI-NEXT:    umov w13, v0.b[9]
-; CHECK-GI-NEXT:    mov v21.s[1], w8
-; CHECK-GI-NEXT:    umov w8, v1.b[11]
-; CHECK-GI-NEXT:    mov v18.s[1], w11
-; CHECK-GI-NEXT:    umov w11, v0.b[14]
+; CHECK-GI-NEXT:    umov w11, v2.b[3]
 ; CHECK-GI-NEXT:    mov v20.s[1], w10
-; CHECK-GI-NEXT:    ldrb w10, [x0, #6]
-; CHECK-GI-NEXT:    mov v22.s[0], w12
-; CHECK-GI-NEXT:    umov w12, v0.b[7]
-; CHECK-GI-NEXT:    mov v17.s[2], w9
-; CHECK-GI-NEXT:    umov w9, v0.b[10]
-; CHECK-GI-NEXT:    mov v7.s[2], w15
+; CHECK-GI-NEXT:    ldrb w10, [x0, #8]
+; CHECK-GI-NEXT:    fmov s21, w12
+; CHECK-GI-NEXT:    ldrb w12, [x1, #8]
+; CHECK-GI-NEXT:    mov v17.s[2], w13
+; CHECK-GI-NEXT:    umov w13, v1.b[9]
+; CHECK-GI-NEXT:    fmov s19, w8
+; CHECK-GI-NEXT:    umov w8, v1.b[14]
+; CHECK-GI-NEXT:    mul w10, w12, w10
+; CHECK-GI-NEXT:    umov w12, v1.b[7]
+; CHECK-GI-NEXT:    mov v4.s[3], w11
+; CHECK-GI-NEXT:    mov v21.s[1], w9
+; CHECK-GI-NEXT:    ldrb w9, [x0, #6]
 ; CHECK-GI-NEXT:    mov v19.s[1], w13
-; CHECK-GI-NEXT:    umov w13, v1.b[3]
-; CHECK-GI-NEXT:    mov v23.s[2], wzr
-; CHECK-GI-NEXT:    mov v18.s[2], w11
-; CHECK-GI-NEXT:    umov w11, v0.b[15]
-; CHECK-GI-NEXT:    mov v20.s[2], w10
-; CHECK-GI-NEXT:    ldrb w10, [x0, #2]
-; CHECK-GI-NEXT:    mov v22.s[1], wzr
-; CHECK-GI-NEXT:    mov v17.s[3], w12
+; CHECK-GI-NEXT:    ldrb w13, [x0, #1]
+; CHECK-GI-NEXT:    mov v20.s[2], w8
+; CHECK-GI-NEXT:    umov w8, v1.b[10]
+; CHECK-GI-NEXT:    mov v18.s[3], w12
 ; CHECK-GI-NEXT:    ldrb w12, [x0, #7]
-; CHECK-GI-NEXT:    mov v7.s[3], w8
-; CHECK-GI-NEXT:    ldrb w8, [x0, #3]
-; CHECK-GI-NEXT:    mov v19.s[2], w9
-; CHECK-GI-NEXT:    umov w9, v0.b[3]
-; CHECK-GI-NEXT:    mov v18.s[3], w11
-; CHECK-GI-NEXT:    umov w11, v0.b[11]
-; CHECK-GI-NEXT:    mov v21.s[2], w10
+; CHECK-GI-NEXT:    mov v21.s[2], w9
+; CHECK-GI-NEXT:    umov w9, v2.b[11]
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    ldrb w10, [x0, #2]
+; CHECK-GI-NEXT:    mov v22.s[1], w13
+; CHECK-GI-NEXT:    umov w13, v1.b[15]
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    mov v19.s[2], w8
+; CHECK-GI-NEXT:    umov w8, v1.b[3]
+; CHECK-GI-NEXT:    mov v21.s[3], w12
+; CHECK-GI-NEXT:    mov v6.s[3], w9
+; CHECK-GI-NEXT:    ldrb w9, [x0, #3]
+; CHECK-GI-NEXT:    mov v20.s[3], w13
+; CHECK-GI-NEXT:    umov w13, v1.b[11]
+; CHECK-GI-NEXT:    mov v22.s[2], w10
 ; CHECK-GI-NEXT:    ldrb w10, [x1, #3]
-; CHECK-GI-NEXT:    mov v20.s[3], w12
-; CHECK-GI-NEXT:    mov v22.s[2], wzr
-; CHECK-GI-NEXT:    mov v6.s[3], w13
-; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v17.4s
-; CHECK-GI-NEXT:    mov v23.s[3], wzr
+; CHECK-GI-NEXT:    mul v1.4s, v7.4s, v18.4s
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v17.s[3], w8
 ; CHECK-GI-NEXT:    mov v3.s[3], w10
-; CHECK-GI-NEXT:    mov v16.s[3], w9
-; CHECK-GI-NEXT:    mov v19.s[3], w11
-; CHECK-GI-NEXT:    mul v1.4s, v4.4s, v18.4s
-; CHECK-GI-NEXT:    mov v21.s[3], w8
-; CHECK-GI-NEXT:    mul v2.4s, v5.4s, v20.4s
-; CHECK-GI-NEXT:    mov v22.s[3], wzr
-; CHECK-GI-NEXT:    mla v0.4s, v6.4s, v16.4s
-; CHECK-GI-NEXT:    mla v1.4s, v7.4s, v19.4s
-; CHECK-GI-NEXT:    mla v2.4s, v3.4s, v21.4s
-; CHECK-GI-NEXT:    add v3.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v5.4s, v5.4s, v21.4s
+; CHECK-GI-NEXT:    mov v19.s[3], w13
+; CHECK-GI-NEXT:    mul v7.4s, v16.4s, v20.4s
+; CHECK-GI-NEXT:    mov v22.s[3], w9
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    mla v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT:    mla v7.4s, v6.4s, v19.4s
+; CHECK-GI-NEXT:    mla v5.4s, v3.4s, v22.4s
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v7.4s
+; CHECK-GI-NEXT:    add v0.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    add w0, w8, w2
@@ -2456,66 +2456,66 @@ define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-GI-LABEL: test_udot_v25i8_nomla:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    ldr q1, [x0]
-; CHECK-GI-NEXT:    ldrb w11, [x0, #16]!
-; CHECK-GI-NEXT:    ldrb w14, [x0, #4]
-; CHECK-GI-NEXT:    ldrb w17, [x0, #8]
-; CHECK-GI-NEXT:    mov v0.s[0], wzr
-; CHECK-GI-NEXT:    umov w12, v1.b[0]
-; CHECK-GI-NEXT:    umov w13, v1.b[4]
-; CHECK-GI-NEXT:    umov w15, v1.b[8]
-; CHECK-GI-NEXT:    umov w16, v1.b[12]
-; CHECK-GI-NEXT:    umov w18, v1.b[1]
-; CHECK-GI-NEXT:    umov w1, v1.b[5]
-; CHECK-GI-NEXT:    umov w2, v1.b[9]
-; CHECK-GI-NEXT:    umov w3, v1.b[13]
-; CHECK-GI-NEXT:    mov v4.s[0], w11
-; CHECK-GI-NEXT:    mov v7.s[0], w14
-; CHECK-GI-NEXT:    mov v16.s[0], w17
-; CHECK-GI-NEXT:    ldrb w8, [x0, #1]
-; CHECK-GI-NEXT:    mov v2.s[0], w12
-; CHECK-GI-NEXT:    mov v3.s[0], w13
-; CHECK-GI-NEXT:    ldrb w10, [x0, #5]
-; CHECK-GI-NEXT:    mov v5.s[0], w15
-; CHECK-GI-NEXT:    mov v6.s[0], w16
+; CHECK-GI-NEXT:    ldrb w17, [x0, #16]!
+; CHECK-GI-NEXT:    ldrb w16, [x0, #4]
+; CHECK-GI-NEXT:    ldrb w14, [x0, #8]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    umov w15, v1.b[0]
+; CHECK-GI-NEXT:    umov w2, v1.b[4]
+; CHECK-GI-NEXT:    umov w4, v1.b[8]
+; CHECK-GI-NEXT:    umov w5, v1.b[12]
+; CHECK-GI-NEXT:    umov w1, v1.b[1]
+; CHECK-GI-NEXT:    umov w3, v1.b[5]
+; CHECK-GI-NEXT:    umov w6, v1.b[9]
+; CHECK-GI-NEXT:    umov w7, v1.b[13]
+; CHECK-GI-NEXT:    fmov s6, w17
+; CHECK-GI-NEXT:    fmov s7, w16
+; CHECK-GI-NEXT:    fmov s16, w14
+; CHECK-GI-NEXT:    ldrb w18, [x0, #1]
+; CHECK-GI-NEXT:    fmov s2, w15
+; CHECK-GI-NEXT:    fmov s3, w2
+; CHECK-GI-NEXT:    ldrb w11, [x0, #5]
+; CHECK-GI-NEXT:    fmov s4, w4
+; CHECK-GI-NEXT:    fmov s5, w5
 ; CHECK-GI-NEXT:    ldrb w16, [x0, #2]
 ; CHECK-GI-NEXT:    umov w9, v1.b[2]
 ; CHECK-GI-NEXT:    umov w12, v1.b[6]
 ; CHECK-GI-NEXT:    ldrb w17, [x0, #6]
-; CHECK-GI-NEXT:    umov w14, v1.b[10]
+; CHECK-GI-NEXT:    umov w13, v1.b[10]
 ; CHECK-GI-NEXT:    umov w15, v1.b[14]
-; CHECK-GI-NEXT:    mov v4.s[1], w8
-; CHECK-GI-NEXT:    mov v2.s[1], w18
-; CHECK-GI-NEXT:    mov v3.s[1], w1
-; CHECK-GI-NEXT:    mov v7.s[1], w10
-; CHECK-GI-NEXT:    mov v5.s[1], w2
-; CHECK-GI-NEXT:    mov v6.s[1], w3
+; CHECK-GI-NEXT:    mov v2.s[1], w1
+; CHECK-GI-NEXT:    mov v3.s[1], w3
+; CHECK-GI-NEXT:    mov v4.s[1], w6
+; CHECK-GI-NEXT:    mov v5.s[1], w7
+; CHECK-GI-NEXT:    mov v6.s[1], w18
+; CHECK-GI-NEXT:    mov v7.s[1], w11
 ; CHECK-GI-NEXT:    mov v16.s[1], wzr
 ; CHECK-GI-NEXT:    mov v0.s[1], wzr
-; CHECK-GI-NEXT:    umov w11, v1.b[3]
-; CHECK-GI-NEXT:    umov w13, v1.b[7]
-; CHECK-GI-NEXT:    umov w8, v1.b[11]
-; CHECK-GI-NEXT:    umov w10, v1.b[15]
-; CHECK-GI-NEXT:    mov v4.s[2], w16
+; CHECK-GI-NEXT:    umov w8, v1.b[3]
+; CHECK-GI-NEXT:    umov w10, v1.b[7]
+; CHECK-GI-NEXT:    umov w11, v1.b[11]
+; CHECK-GI-NEXT:    umov w14, v1.b[15]
 ; CHECK-GI-NEXT:    mov v2.s[2], w9
 ; CHECK-GI-NEXT:    ldrb w9, [x0, #3]
 ; CHECK-GI-NEXT:    mov v3.s[2], w12
 ; CHECK-GI-NEXT:    ldrb w12, [x0, #7]
-; CHECK-GI-NEXT:    mov v5.s[2], w14
-; CHECK-GI-NEXT:    mov v6.s[2], w15
+; CHECK-GI-NEXT:    mov v4.s[2], w13
+; CHECK-GI-NEXT:    mov v5.s[2], w15
+; CHECK-GI-NEXT:    mov v6.s[2], w16
 ; CHECK-GI-NEXT:    mov v7.s[2], w17
 ; CHECK-GI-NEXT:    mov v16.s[2], wzr
 ; CHECK-GI-NEXT:    mov v0.s[2], wzr
-; CHECK-GI-NEXT:    mov v4.s[3], w9
-; CHECK-GI-NEXT:    mov v2.s[3], w11
-; CHECK-GI-NEXT:    mov v3.s[3], w13
-; CHECK-GI-NEXT:    mov v5.s[3], w8
-; CHECK-GI-NEXT:    mov v6.s[3], w10
+; CHECK-GI-NEXT:    mov v2.s[3], w8
+; CHECK-GI-NEXT:    mov v3.s[3], w10
+; CHECK-GI-NEXT:    mov v4.s[3], w11
+; CHECK-GI-NEXT:    mov v5.s[3], w14
+; CHECK-GI-NEXT:    mov v6.s[3], w9
 ; CHECK-GI-NEXT:    mov v7.s[3], w12
 ; CHECK-GI-NEXT:    mov v16.s[3], wzr
 ; CHECK-GI-NEXT:    mov v0.s[3], wzr
 ; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s
-; CHECK-GI-NEXT:    add v3.4s, v4.4s, v7.4s
+; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
 ; CHECK-GI-NEXT:    add v0.4s, v16.4s, v0.4s
 ; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
 ; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
@@ -2554,125 +2554,125 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_sdot_v25i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr q1, [x1]
-; CHECK-GI-NEXT:    ldrsb w11, [x1, #16]!
-; CHECK-GI-NEXT:    ldrsb w12, [x1, #4]
-; CHECK-GI-NEXT:    ldr q0, [x0]
-; CHECK-GI-NEXT:    mov v23.s[0], wzr
-; CHECK-GI-NEXT:    smov w9, v1.b[4]
-; CHECK-GI-NEXT:    smov w10, v1.b[12]
-; CHECK-GI-NEXT:    smov w13, v1.b[0]
-; CHECK-GI-NEXT:    smov w14, v1.b[5]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov v3.s[0], w11
-; CHECK-GI-NEXT:    smov w11, v0.b[0]
-; CHECK-GI-NEXT:    smov w12, v1.b[1]
-; CHECK-GI-NEXT:    smov w15, v1.b[8]
-; CHECK-GI-NEXT:    ldrsb w8, [x0, #16]!
-; CHECK-GI-NEXT:    mov v23.s[1], wzr
-; CHECK-GI-NEXT:    mov v2.s[0], w9
-; CHECK-GI-NEXT:    mov v4.s[0], w10
+; CHECK-GI-NEXT:    ldr q2, [x1]
+; CHECK-GI-NEXT:    ldrsb w9, [x1, #16]!
+; CHECK-GI-NEXT:    ldrsb w11, [x1, #4]
+; CHECK-GI-NEXT:    ldrsb w12, [x1, #5]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    smov w13, v2.b[4]
+; CHECK-GI-NEXT:    smov w14, v2.b[5]
+; CHECK-GI-NEXT:    smov w10, v2.b[0]
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    smov w9, v2.b[8]
+; CHECK-GI-NEXT:    fmov s5, w11
+; CHECK-GI-NEXT:    smov w11, v2.b[12]
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    ldrsb w8, [x1, #1]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    fmov s7, w13
+; CHECK-GI-NEXT:    fmov s4, w10
+; CHECK-GI-NEXT:    smov w10, v2.b[13]
+; CHECK-GI-NEXT:    mov v5.s[1], w12
+; CHECK-GI-NEXT:    smov w13, v2.b[9]
+; CHECK-GI-NEXT:    fmov s6, w9
+; CHECK-GI-NEXT:    fmov s16, w11
+; CHECK-GI-NEXT:    smov w9, v1.b[0]
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v7.s[1], w14
+; CHECK-GI-NEXT:    smov w14, v2.b[6]
+; CHECK-GI-NEXT:    ldrsb w12, [x1, #6]
+; CHECK-GI-NEXT:    smov w8, v2.b[1]
+; CHECK-GI-NEXT:    smov w11, v2.b[2]
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    mov v16.s[1], w10
+; CHECK-GI-NEXT:    smov w10, v2.b[14]
+; CHECK-GI-NEXT:    mov v5.s[2], w12
+; CHECK-GI-NEXT:    smov w12, v1.b[5]
+; CHECK-GI-NEXT:    mov v6.s[1], w13
+; CHECK-GI-NEXT:    fmov s17, w9
+; CHECK-GI-NEXT:    mov v7.s[2], w14
+; CHECK-GI-NEXT:    smov w14, v1.b[4]
+; CHECK-GI-NEXT:    smov w9, v2.b[10]
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    smov w8, v1.b[1]
+; CHECK-GI-NEXT:    smov w13, v2.b[7]
+; CHECK-GI-NEXT:    mov v16.s[2], w10
+; CHECK-GI-NEXT:    smov w10, v2.b[15]
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    fmov s18, w14
+; CHECK-GI-NEXT:    mov v6.s[2], w9
+; CHECK-GI-NEXT:    smov w9, v1.b[12]
+; CHECK-GI-NEXT:    mov v4.s[2], w11
+; CHECK-GI-NEXT:    ldrsb w11, [x1, #7]
+; CHECK-GI-NEXT:    mov v17.s[1], w8
+; CHECK-GI-NEXT:    ldrsb w8, [x1, #2]
+; CHECK-GI-NEXT:    mov v16.s[3], w10
 ; CHECK-GI-NEXT:    smov w10, v1.b[13]
-; CHECK-GI-NEXT:    ldrsb w9, [x1, #5]
-; CHECK-GI-NEXT:    mov v6.s[0], w13
-; CHECK-GI-NEXT:    smov w13, v1.b[6]
-; CHECK-GI-NEXT:    mov v16.s[0], w11
-; CHECK-GI-NEXT:    smov w11, v1.b[2]
-; CHECK-GI-NEXT:    mov v7.s[0], w15
-; CHECK-GI-NEXT:    mov v5.s[1], w9
-; CHECK-GI-NEXT:    ldrsb w9, [x1, #6]
-; CHECK-GI-NEXT:    smov w15, v1.b[9]
-; CHECK-GI-NEXT:    mov v2.s[1], w14
-; CHECK-GI-NEXT:    ldrsb w14, [x1, #1]
-; CHECK-GI-NEXT:    mov v4.s[1], w10
-; CHECK-GI-NEXT:    smov w10, v1.b[14]
-; CHECK-GI-NEXT:    mov v6.s[1], w12
-; CHECK-GI-NEXT:    smov w12, v0.b[1]
-; CHECK-GI-NEXT:    mov v3.s[1], w14
-; CHECK-GI-NEXT:    smov w14, v0.b[12]
-; CHECK-GI-NEXT:    mov v21.s[0], w8
-; CHECK-GI-NEXT:    ldrsb w8, [x0, #1]
-; CHECK-GI-NEXT:    mov v5.s[2], w9
-; CHECK-GI-NEXT:    smov w9, v0.b[4]
-; CHECK-GI-NEXT:    mov v2.s[2], w13
-; CHECK-GI-NEXT:    smov w13, v1.b[7]
-; CHECK-GI-NEXT:    mov v7.s[1], w15
-; CHECK-GI-NEXT:    mov v4.s[2], w10
-; CHECK-GI-NEXT:    smov w10, v1.b[15]
-; CHECK-GI-NEXT:    mov v16.s[1], w12
-; CHECK-GI-NEXT:    ldrsb w12, [x1, #2]
-; CHECK-GI-NEXT:    mov v6.s[2], w11
-; CHECK-GI-NEXT:    smov w11, v0.b[2]
-; CHECK-GI-NEXT:    mov v17.s[0], w9
-; CHECK-GI-NEXT:    smov w9, v0.b[8]
-; CHECK-GI-NEXT:    mov v18.s[0], w14
-; CHECK-GI-NEXT:    mov v2.s[3], w13
-; CHECK-GI-NEXT:    ldrsb w13, [x1, #7]
-; CHECK-GI-NEXT:    mov v3.s[2], w12
+; CHECK-GI-NEXT:    mov v18.s[1], w12
+; CHECK-GI-NEXT:    smov w12, v1.b[6]
+; CHECK-GI-NEXT:    mov v5.s[3], w11
+; CHECK-GI-NEXT:    ldrsb w11, [x0, #16]!
+; CHECK-GI-NEXT:    mov v7.s[3], w13
+; CHECK-GI-NEXT:    smov w13, v1.b[2]
+; CHECK-GI-NEXT:    fmov s20, w9
+; CHECK-GI-NEXT:    ldrsb w9, [x0, #5]
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    smov w8, v1.b[8]
+; CHECK-GI-NEXT:    fmov s22, w11
+; CHECK-GI-NEXT:    mov v18.s[2], w12
 ; CHECK-GI-NEXT:    ldrsb w12, [x0, #4]
-; CHECK-GI-NEXT:    mov v4.s[3], w10
-; CHECK-GI-NEXT:    smov w10, v0.b[5]
-; CHECK-GI-NEXT:    mov v5.s[3], w13
-; CHECK-GI-NEXT:    ldrsb w13, [x0, #8]
-; CHECK-GI-NEXT:    mov v16.s[2], w11
-; CHECK-GI-NEXT:    smov w11, v0.b[13]
-; CHECK-GI-NEXT:    mov v20.s[0], w12
-; CHECK-GI-NEXT:    ldrsb w12, [x1, #8]
-; CHECK-GI-NEXT:    mov v19.s[0], w9
-; CHECK-GI-NEXT:    smov w9, v0.b[6]
-; CHECK-GI-NEXT:    smov w15, v1.b[10]
-; CHECK-GI-NEXT:    mul w12, w12, w13
-; CHECK-GI-NEXT:    mov v17.s[1], w10
-; CHECK-GI-NEXT:    ldrsb w10, [x0, #5]
-; CHECK-GI-NEXT:    smov w13, v0.b[9]
-; CHECK-GI-NEXT:    mov v21.s[1], w8
-; CHECK-GI-NEXT:    smov w8, v1.b[11]
-; CHECK-GI-NEXT:    mov v18.s[1], w11
-; CHECK-GI-NEXT:    smov w11, v0.b[14]
+; CHECK-GI-NEXT:    smov w11, v2.b[3]
 ; CHECK-GI-NEXT:    mov v20.s[1], w10
-; CHECK-GI-NEXT:    ldrsb w10, [x0, #6]
-; CHECK-GI-NEXT:    mov v22.s[0], w12
-; CHECK-GI-NEXT:    smov w12, v0.b[7]
-; CHECK-GI-NEXT:    mov v17.s[2], w9
-; CHECK-GI-NEXT:    smov w9, v0.b[10]
-; CHECK-GI-NEXT:    mov v7.s[2], w15
+; CHECK-GI-NEXT:    ldrsb w10, [x0, #8]
+; CHECK-GI-NEXT:    fmov s21, w12
+; CHECK-GI-NEXT:    ldrsb w12, [x1, #8]
+; CHECK-GI-NEXT:    mov v17.s[2], w13
+; CHECK-GI-NEXT:    smov w13, v1.b[9]
+; CHECK-GI-NEXT:    fmov s19, w8
+; CHECK-GI-NEXT:    smov w8, v1.b[14]
+; CHECK-GI-NEXT:    mul w10, w12, w10
+; CHECK-GI-NEXT:    smov w12, v1.b[7]
+; CHECK-GI-NEXT:    mov v4.s[3], w11
+; CHECK-GI-NEXT:    mov v21.s[1], w9
+; CHECK-GI-NEXT:    ldrsb w9, [x0, #6]
 ; CHECK-GI-NEXT:    mov v19.s[1], w13
-; CHECK-GI-NEXT:    smov w13, v1.b[3]
-; CHECK-GI-NEXT:    mov v23.s[2], wzr
-; CHECK-GI-NEXT:    mov v18.s[2], w11
-; CHECK-GI-NEXT:    smov w11, v0.b[15]
-; CHECK-GI-NEXT:    mov v20.s[2], w10
-; CHECK-GI-NEXT:    ldrsb w10, [x0, #2]
-; CHECK-GI-NEXT:    mov v22.s[1], wzr
-; CHECK-GI-NEXT:    mov v17.s[3], w12
+; CHECK-GI-NEXT:    ldrsb w13, [x0, #1]
+; CHECK-GI-NEXT:    mov v20.s[2], w8
+; CHECK-GI-NEXT:    smov w8, v1.b[10]
+; CHECK-GI-NEXT:    mov v18.s[3], w12
 ; CHECK-GI-NEXT:    ldrsb w12, [x0, #7]
-; CHECK-GI-NEXT:    mov v7.s[3], w8
-; CHECK-GI-NEXT:    ldrsb w8, [x0, #3]
-; CHECK-GI-NEXT:    mov v19.s[2], w9
-; CHECK-GI-NEXT:    smov w9, v0.b[3]
-; CHECK-GI-NEXT:    mov v18.s[3], w11
-; CHECK-GI-NEXT:    smov w11, v0.b[11]
-; CHECK-GI-NEXT:    mov v21.s[2], w10
+; CHECK-GI-NEXT:    mov v21.s[2], w9
+; CHECK-GI-NEXT:    smov w9, v2.b[11]
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    ldrsb w10, [x0, #2]
+; CHECK-GI-NEXT:    mov v22.s[1], w13
+; CHECK-GI-NEXT:    smov w13, v1.b[15]
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    mov v19.s[2], w8
+; CHECK-GI-NEXT:    smov w8, v1.b[3]
+; CHECK-GI-NEXT:    mov v21.s[3], w12
+; CHECK-GI-NEXT:    mov v6.s[3], w9
+; CHECK-GI-NEXT:    ldrsb w9, [x0, #3]
+; CHECK-GI-NEXT:    mov v20.s[3], w13
+; CHECK-GI-NEXT:    smov w13, v1.b[11]
+; CHECK-GI-NEXT:    mov v22.s[2], w10
 ; CHECK-GI-NEXT:    ldrsb w10, [x1, #3]
-; CHECK-GI-NEXT:    mov v20.s[3], w12
-; CHECK-GI-NEXT:    mov v22.s[2], wzr
-; CHECK-GI-NEXT:    mov v6.s[3], w13
-; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v17.4s
-; CHECK-GI-NEXT:    mov v23.s[3], wzr
+; CHECK-GI-NEXT:    mul v1.4s, v7.4s, v18.4s
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v17.s[3], w8
 ; CHECK-GI-NEXT:    mov v3.s[3], w10
-; CHECK-GI-NEXT:    mov v16.s[3], w9
-; CHECK-GI-NEXT:    mov v19.s[3], w11
-; CHECK-GI-NEXT:    mul v1.4s, v4.4s, v18.4s
-; CHECK-GI-NEXT:    mov v21.s[3], w8
-; CHECK-GI-NEXT:    mul v2.4s, v5.4s, v20.4s
-; CHECK-GI-NEXT:    mov v22.s[3], wzr
-; CHECK-GI-NEXT:    mla v0.4s, v6.4s, v16.4s
-; CHECK-GI-NEXT:    mla v1.4s, v7.4s, v19.4s
-; CHECK-GI-NEXT:    mla v2.4s, v3.4s, v21.4s
-; CHECK-GI-NEXT:    add v3.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v5.4s, v5.4s, v21.4s
+; CHECK-GI-NEXT:    mov v19.s[3], w13
+; CHECK-GI-NEXT:    mul v7.4s, v16.4s, v20.4s
+; CHECK-GI-NEXT:    mov v22.s[3], w9
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    mla v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT:    mla v7.4s, v6.4s, v19.4s
+; CHECK-GI-NEXT:    mla v5.4s, v3.4s, v22.4s
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v7.4s
+; CHECK-GI-NEXT:    add v0.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    add w0, w8, w2
@@ -2905,344 +2905,349 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25
 ;
 ; CHECK-GI-LABEL: test_sdot_v25i8_double:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str d10, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #8] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x29, [sp, #24] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w29, -8
-; CHECK-GI-NEXT:    .cfi_offset b8, -16
-; CHECK-GI-NEXT:    .cfi_offset b9, -24
-; CHECK-GI-NEXT:    .cfi_offset b10, -32
-; CHECK-GI-NEXT:    ldr w8, [sp, #32]
-; CHECK-GI-NEXT:    sxtb w9, w0
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    str x29, [sp, #32] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    .cfi_offset b8, -24
+; CHECK-GI-NEXT:    .cfi_offset b9, -32
+; CHECK-GI-NEXT:    .cfi_offset b10, -40
+; CHECK-GI-NEXT:    .cfi_offset b11, -48
+; CHECK-GI-NEXT:    sxtb w8, w0
 ; CHECK-GI-NEXT:    sxtb w10, w4
-; CHECK-GI-NEXT:    sxtb w11, w5
-; CHECK-GI-NEXT:    sxtb w12, w3
-; CHECK-GI-NEXT:    sxtb w13, w7
-; CHECK-GI-NEXT:    mov v0.s[0], w9
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v2.s[0], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #40]
-; CHECK-GI-NEXT:    sxtb w10, w1
-; CHECK-GI-NEXT:    ldr w14, [sp, #152]
-; CHECK-GI-NEXT:    mov v1.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #64]
-; CHECK-GI-NEXT:    mov v9.s[0], wzr
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr x29, [sp, #24] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w10
-; CHECK-GI-NEXT:    mov v2.s[1], w11
+; CHECK-GI-NEXT:    sxtb w9, w1
+; CHECK-GI-NEXT:    sxtb w11, w2
+; CHECK-GI-NEXT:    sxtb w13, w6
+; CHECK-GI-NEXT:    ldr w12, [sp, #72]
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #48]
+; CHECK-GI-NEXT:    fmov s4, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #80]
+; CHECK-GI-NEXT:    ldr w14, [sp, #128]
+; CHECK-GI-NEXT:    ldr w15, [sp, #152]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    sxtb w10, w2
-; CHECK-GI-NEXT:    sxtb w11, w6
-; CHECK-GI-NEXT:    mov v1.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #48]
-; CHECK-GI-NEXT:    mov v3.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #72]
-; CHECK-GI-NEXT:    mov v9.s[1], wzr
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v0.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #56]
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    mov v2.s[1], w9
+; CHECK-GI-NEXT:    sxtb w9, w5
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #88]
+; CHECK-GI-NEXT:    ldr x29, [sp, #32] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    mov v4.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #56]
+; CHECK-GI-NEXT:    fmov s5, w10
 ; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w10, w3
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
 ; CHECK-GI-NEXT:    mov v2.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #80]
-; CHECK-GI-NEXT:    mov v1.s[2], w9
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w11, [sp, #64]
+; CHECK-GI-NEXT:    mov v5.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #104]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.s[1], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #96]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v3.s[1], w8
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #88]
-; CHECK-GI-NEXT:    mov v0.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #128]
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v2.s[3], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #168]
+; CHECK-GI-NEXT:    mov v4.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #120]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v1.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #104]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v4.s[0], w9
+; CHECK-GI-NEXT:    mov v2.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #112]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
 ; CHECK-GI-NEXT:    mov v3.s[2], w11
-; CHECK-GI-NEXT:    sxtb w11, w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #136]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #112]
-; CHECK-GI-NEXT:    ldr w12, [sp, #176]
-; CHECK-GI-NEXT:    mov v6.s[0], w11
+; CHECK-GI-NEXT:    sxtb w11, w10
+; CHECK-GI-NEXT:    mov v5.s[2], w9
+; CHECK-GI-NEXT:    sxtb w9, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #144]
+; CHECK-GI-NEXT:    ldr w10, [sp, #136]
+; CHECK-GI-NEXT:    fmov s6, w11
+; CHECK-GI-NEXT:    sxtb w11, w7
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    ldr w11, [sp, #120]
-; CHECK-GI-NEXT:    mov v9.s[2], wzr
-; CHECK-GI-NEXT:    mov v4.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #200]
-; CHECK-GI-NEXT:    mov v3.s[3], w8
-; CHECK-GI-NEXT:    sxtb w8, w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #144]
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v5.s[1], w13
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w13, [sp, #184]
-; CHECK-GI-NEXT:    mov v6.s[1], w8
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #160]
-; CHECK-GI-NEXT:    mov v4.s[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #208]
-; CHECK-GI-NEXT:    mov v7.s[0], w10
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    mov v5.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #184]
+; CHECK-GI-NEXT:    mov v4.s[3], w11
+; CHECK-GI-NEXT:    mov v6.s[1], w9
+; CHECK-GI-NEXT:    fmov s7, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #216]
+; CHECK-GI-NEXT:    sxtb w9, w12
+; CHECK-GI-NEXT:    sxtb w12, w14
+; CHECK-GI-NEXT:    sxtb w14, w15
+; CHECK-GI-NEXT:    sxtb w8, w8
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    sxtb w10, w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #192]
-; CHECK-GI-NEXT:    mov v5.s[2], w12
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w12, [sp, #232]
-; CHECK-GI-NEXT:    mov v6.s[2], w13
-; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    ldr w11, [sp, #160]
+; CHECK-GI-NEXT:    mov v7.s[1], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #224]
+; CHECK-GI-NEXT:    mov v3.s[3], w9
+; CHECK-GI-NEXT:    mov v6.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #192]
+; CHECK-GI-NEXT:    fmov s16, w8
+; CHECK-GI-NEXT:    fmov s18, w13
+; CHECK-GI-NEXT:    sxtb w14, w14
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v4.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #216]
-; CHECK-GI-NEXT:    mov v7.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #264]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    ldr w14, [sp, #368]
-; CHECK-GI-NEXT:    mov v5.s[3], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #296]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v6.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #240]
-; CHECK-GI-NEXT:    mov v16.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #224]
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v7.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #272]
-; CHECK-GI-NEXT:    mov v18.s[0], w9
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #304]
+; CHECK-GI-NEXT:    ldr w9, [sp, #168]
+; CHECK-GI-NEXT:    ldr w13, [sp, #208]
+; CHECK-GI-NEXT:    mov v7.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #256]
+; CHECK-GI-NEXT:    ldr w8, [sp, #176]
+; CHECK-GI-NEXT:    mov v16.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #200]
+; CHECK-GI-NEXT:    mov v18.s[1], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #232]
+; CHECK-GI-NEXT:    mov v6.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #248]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v17.s[0], w13
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w13, [sp, #248]
-; CHECK-GI-NEXT:    mov v16.s[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #328]
+; CHECK-GI-NEXT:    sxtb w11, w11
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v7.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #280]
-; CHECK-GI-NEXT:    mov v18.s[1], w10
+; CHECK-GI-NEXT:    sxtb w14, w14
+; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    ldr w10, [sp, #312]
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v17.s[1], w9
+; CHECK-GI-NEXT:    mov v16.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #240]
+; CHECK-GI-NEXT:    mov v7.s[3], w9
+; CHECK-GI-NEXT:    mov v18.s[2], w14
+; CHECK-GI-NEXT:    fmov s17, w10
+; CHECK-GI-NEXT:    ldr w14, [sp, #264]
 ; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #288]
+; CHECK-GI-NEXT:    ldr w10, [sp, #272]
+; CHECK-GI-NEXT:    sxtb w14, w14
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v16.s[2], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #336]
+; CHECK-GI-NEXT:    ldr w15, [sp, #392]
+; CHECK-GI-NEXT:    mov v17.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #280]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v18.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #312]
+; CHECK-GI-NEXT:    mov v16.s[3], w13
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w13, [sp, #296]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v19.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #288]
-; CHECK-GI-NEXT:    mov v18.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #320]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    mov v17.s[2], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #320]
+; CHECK-GI-NEXT:    fmov s20, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #344]
+; CHECK-GI-NEXT:    fmov s19, w12
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    ldr w9, [sp, #256]
-; CHECK-GI-NEXT:    mov v17.s[2], w10
+; CHECK-GI-NEXT:    sxtb w14, w14
+; CHECK-GI-NEXT:    ldr w12, [sp, #304]
+; CHECK-GI-NEXT:    mul v4.4s, v4.4s, v18.4s
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w10, [sp, #344]
+; CHECK-GI-NEXT:    mov v20.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #352]
+; CHECK-GI-NEXT:    mov v19.s[1], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #328]
 ; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    fmov s21, w11
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mul v2.4s, v2.4s, v7.4s
-; CHECK-GI-NEXT:    mov v19.s[1], w13
-; CHECK-GI-NEXT:    mov v18.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #360]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w13, [sp, #400]
-; CHECK-GI-NEXT:    mov v16.s[3], w9
-; CHECK-GI-NEXT:    mov v17.s[3], w12
-; CHECK-GI-NEXT:    sxtb w12, w14
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #352]
-; CHECK-GI-NEXT:    mov v9.s[3], wzr
-; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v6.4s
-; CHECK-GI-NEXT:    mov v19.s[2], w10
+; CHECK-GI-NEXT:    ldr w11, [sp, #336]
+; CHECK-GI-NEXT:    sxtb w14, w14
+; CHECK-GI-NEXT:    mov v17.s[3], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #376]
-; CHECK-GI-NEXT:    mov v20.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #432]
-; CHECK-GI-NEXT:    mul w8, w8, w11
-; CHECK-GI-NEXT:    sxtb w11, w13
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w13, [sp, #384]
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v23.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #408]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v20.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #440]
-; CHECK-GI-NEXT:    mov v21.s[0], w8
+; CHECK-GI-NEXT:    mov v20.s[2], w13
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v22.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #464]
-; CHECK-GI-NEXT:    sxtb w8, w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #416]
+; CHECK-GI-NEXT:    ldr w13, [sp, #368]
+; CHECK-GI-NEXT:    mov v21.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #360]
+; CHECK-GI-NEXT:    mov v19.s[2], w14
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v23.s[1], w11
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v19.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #392]
-; CHECK-GI-NEXT:    mov v20.s[2], w8
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v22.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #472]
-; CHECK-GI-NEXT:    mov v24.s[0], w12
-; CHECK-GI-NEXT:    ldr w11, [sp, #424]
-; CHECK-GI-NEXT:    ldr w8, [sp, #448]
+; CHECK-GI-NEXT:    ldr w14, [sp, #384]
+; CHECK-GI-NEXT:    mla v4.4s, v2.4s, v16.4s
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v23.s[2], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #496]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v20.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #480]
+; CHECK-GI-NEXT:    mov v20.s[3], w12
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    mul w10, w8, w10
+; CHECK-GI-NEXT:    mov v21.s[2], w9
+; CHECK-GI-NEXT:    mov v19.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #416]
+; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    sxtb w14, w15
+; CHECK-GI-NEXT:    ldr w9, [sp, #400]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v24.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #528]
-; CHECK-GI-NEXT:    ldr w12, [sp, #456]
-; CHECK-GI-NEXT:    mov v22.s[2], w8
-; CHECK-GI-NEXT:    mov v23.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #504]
+; CHECK-GI-NEXT:    fmov s22, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #432]
+; CHECK-GI-NEXT:    fmov s23, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #448]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v25.s[0], w13
+; CHECK-GI-NEXT:    mov v21.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #424]
+; CHECK-GI-NEXT:    fmov s25, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #480]
+; CHECK-GI-NEXT:    sxtb w13, w13
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w8, [sp, #488]
 ; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v23.s[1], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #456]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v24.s[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #536]
-; CHECK-GI-NEXT:    mov v26.s[0], w10
-; CHECK-GI-NEXT:    ldr w13, [sp, #568]
-; CHECK-GI-NEXT:    mov v22.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #512]
+; CHECK-GI-NEXT:    fmov s24, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #440]
+; CHECK-GI-NEXT:    mov v25.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #488]
+; CHECK-GI-NEXT:    sxtb w14, w14
+; CHECK-GI-NEXT:    fmov s26, w11
+; CHECK-GI-NEXT:    ldr w15, [sp, #504]
+; CHECK-GI-NEXT:    ldr w11, [sp, #472]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v24.s[1], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #464]
+; CHECK-GI-NEXT:    mov v23.s[2], w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w8, [sp, #408]
+; CHECK-GI-NEXT:    mov v26.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #496]
+; CHECK-GI-NEXT:    mov v25.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #512]
+; CHECK-GI-NEXT:    sxtb w9, w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #520]
+; CHECK-GI-NEXT:    sxtb w12, w12
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v25.s[1], w11
+; CHECK-GI-NEXT:    mov v22.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v24.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #528]
+; CHECK-GI-NEXT:    mov v26.s[2], w12
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    sxtb w13, w15
+; CHECK-GI-NEXT:    fmov s27, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #584]
+; CHECK-GI-NEXT:    ldr w15, [sp, #552]
+; CHECK-GI-NEXT:    mov v25.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #544]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    sxtb w10, w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v24.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #560]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v24.s[3], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #544]
-; CHECK-GI-NEXT:    mov v26.s[1], w9
-; CHECK-GI-NEXT:    ldr w13, [sp, #520]
-; CHECK-GI-NEXT:    ldr w11, [sp, #576]
-; CHECK-GI-NEXT:    mov v27.s[0], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #600]
+; CHECK-GI-NEXT:    mov v26.s[3], w13
+; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    sxtb w14, w15
+; CHECK-GI-NEXT:    fmov s29, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    fmov s28, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #616]
+; CHECK-GI-NEXT:    mov v27.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #592]
+; CHECK-GI-NEXT:    ldr w15, [sp, #568]
+; CHECK-GI-NEXT:    mov v23.s[3], w8
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #536]
+; CHECK-GI-NEXT:    ldr w10, [sp, #576]
+; CHECK-GI-NEXT:    mov v28.s[1], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #624]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    fmov s30, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #600]
+; CHECK-GI-NEXT:    mov v27.s[2], w9
+; CHECK-GI-NEXT:    mov v29.s[1], w13
+; CHECK-GI-NEXT:    sxtb w13, w14
+; CHECK-GI-NEXT:    sxtb w14, w15
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #608]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v25.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #584]
-; CHECK-GI-NEXT:    sxtb w9, w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #552]
+; CHECK-GI-NEXT:    mov v30.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #632]
+; CHECK-GI-NEXT:    mov v28.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #640]
+; CHECK-GI-NEXT:    sxtb w9, w9
 ; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v26.s[2], w8
-; CHECK-GI-NEXT:    sxtb w8, w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #664]
-; CHECK-GI-NEXT:    mov v27.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #608]
-; CHECK-GI-NEXT:    mov v28.s[0], w10
-; CHECK-GI-NEXT:    mov v25.s[3], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #592]
+; CHECK-GI-NEXT:    mov v29.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #648]
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v27.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #664]
+; CHECK-GI-NEXT:    mov v30.s[2], w13
+; CHECK-GI-NEXT:    mov v28.s[3], w14
+; CHECK-GI-NEXT:    ldr w14, [sp, #680]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #560]
-; CHECK-GI-NEXT:    mov v26.s[3], w11
-; CHECK-GI-NEXT:    sxtb w11, w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #672]
-; CHECK-GI-NEXT:    mov v30.s[0], w12
-; CHECK-GI-NEXT:    mov v27.s[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #616]
-; CHECK-GI-NEXT:    mov v28.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #632]
-; CHECK-GI-NEXT:    ldr w12, [sp, #728]
-; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #656]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w14, w14
+; CHECK-GI-NEXT:    mov v29.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #688]
+; CHECK-GI-NEXT:    fmov s31, w12
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w12, [sp, #752]
+; CHECK-GI-NEXT:    mov v30.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #744]
+; CHECK-GI-NEXT:    fmov s8, w14
 ; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w14, [sp, #712]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v21.s[1], wzr
-; CHECK-GI-NEXT:    mov v30.s[1], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #760]
-; CHECK-GI-NEXT:    mov v27.s[3], w11
-; CHECK-GI-NEXT:    mov v28.s[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #696]
-; CHECK-GI-NEXT:    mov v29.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #736]
-; CHECK-GI-NEXT:    mov v8.s[0], w12
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v31.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #696]
+; CHECK-GI-NEXT:    mov v8.s[1], w9
+; CHECK-GI-NEXT:    sxtb w14, w14
+; CHECK-GI-NEXT:    ldr w9, [sp, #720]
+; CHECK-GI-NEXT:    fmov s9, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #776]
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    ldr w11, [sp, #640]
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w12, [sp, #680]
+; CHECK-GI-NEXT:    fmov s10, w14
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mul w10, w10, w13
-; CHECK-GI-NEXT:    mov v21.s[2], wzr
+; CHECK-GI-NEXT:    mov v22.s[2], wzr
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v31.s[0], w8
+; CHECK-GI-NEXT:    mov v31.s[2], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #704]
-; CHECK-GI-NEXT:    mov v8.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #744]
-; CHECK-GI-NEXT:    mul v3.4s, v3.4s, v18.4s
-; CHECK-GI-NEXT:    mov v10.s[0], w10
-; CHECK-GI-NEXT:    mov v29.s[1], w11
+; CHECK-GI-NEXT:    mov v9.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #760]
+; CHECK-GI-NEXT:    mov v8.s[2], w13
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    mov v10.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #728]
 ; CHECK-GI-NEXT:    sxtb w11, w12
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w12, [sp, #624]
-; CHECK-GI-NEXT:    mov v30.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #648]
-; CHECK-GI-NEXT:    ldr w10, [sp, #688]
-; CHECK-GI-NEXT:    mov v31.s[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #712]
-; CHECK-GI-NEXT:    mov v8.s[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #752]
-; CHECK-GI-NEXT:    mov v10.s[1], wzr
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mul v5.4s, v5.4s, v20.4s
+; CHECK-GI-NEXT:    mul v7.4s, v7.4s, v21.4s
+; CHECK-GI-NEXT:    mul v18.4s, v25.4s, v30.4s
+; CHECK-GI-NEXT:    mov v22.s[3], wzr
+; CHECK-GI-NEXT:    fmov s11, w10
+; CHECK-GI-NEXT:    mov v9.s[2], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #768]
+; CHECK-GI-NEXT:    mov v8.s[3], w8
+; CHECK-GI-NEXT:    sxtb w8, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #672]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mla v5.4s, v3.4s, v17.4s
+; CHECK-GI-NEXT:    mov v11.s[1], wzr
+; CHECK-GI-NEXT:    mov v10.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #736]
+; CHECK-GI-NEXT:    mov v9.s[3], w10
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v28.s[3], w12
-; CHECK-GI-NEXT:    mul v5.4s, v5.4s, v19.4s
-; CHECK-GI-NEXT:    mov v29.s[2], w11
-; CHECK-GI-NEXT:    mov v30.s[3], w10
-; CHECK-GI-NEXT:    mov v31.s[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #656]
-; CHECK-GI-NEXT:    mov v8.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #720]
-; CHECK-GI-NEXT:    mov v10.s[2], wzr
-; CHECK-GI-NEXT:    mov v21.s[3], wzr
-; CHECK-GI-NEXT:    mla v3.4s, v1.4s, v16.4s
+; CHECK-GI-NEXT:    mla v7.4s, v6.4s, v19.4s
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mul v7.4s, v23.4s, v28.4s
-; CHECK-GI-NEXT:    mul v18.4s, v24.4s, v30.4s
-; CHECK-GI-NEXT:    mla v5.4s, v4.4s, v17.4s
-; CHECK-GI-NEXT:    mul v19.4s, v26.4s, v8.4s
-; CHECK-GI-NEXT:    mov v29.s[3], w8
+; CHECK-GI-NEXT:    mul v20.4s, v26.4s, v8.4s
+; CHECK-GI-NEXT:    mla v18.4s, v23.4s, v29.4s
 ; CHECK-GI-NEXT:    mov v31.s[3], w9
-; CHECK-GI-NEXT:    mov v10.s[3], wzr
-; CHECK-GI-NEXT:    add v0.4s, v21.4s, v9.4s
-; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    mla v7.4s, v20.4s, v27.4s
-; CHECK-GI-NEXT:    mla v18.4s, v22.4s, v29.4s
-; CHECK-GI-NEXT:    mla v19.4s, v25.4s, v31.4s
-; CHECK-GI-NEXT:    add v0.4s, v5.4s, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v10.4s, v9.4s
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #8] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
-; CHECK-GI-NEXT:    add v3.4s, v7.4s, v18.4s
-; CHECK-GI-NEXT:    add v1.4s, v19.4s, v1.4s
-; CHECK-GI-NEXT:    addv s0, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v3.4s, v1.4s
-; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add v1.4s, v22.4s, v1.4s
+; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    mov v11.s[2], wzr
+; CHECK-GI-NEXT:    mov v10.s[3], w8
+; CHECK-GI-NEXT:    mul v21.4s, v28.4s, v9.4s
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add v1.4s, v7.4s, v1.4s
+; CHECK-GI-NEXT:    mla v20.4s, v24.4s, v31.4s
+; CHECK-GI-NEXT:    mov v11.s[3], wzr
+; CHECK-GI-NEXT:    mla v21.4s, v27.4s, v10.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    add v3.4s, v18.4s, v20.4s
+; CHECK-GI-NEXT:    add v0.4s, v11.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add v0.4s, v21.4s, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    add w0, w8, w9
-; CHECK-GI-NEXT:    ldr d10, [sp], #32 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d11, d10, [sp], #48 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <25 x i8> %a to <25 x i32>
@@ -3386,175 +3391,179 @@ define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    sxtb w8, w0
-; CHECK-GI-NEXT:    sxtb w9, w4
-; CHECK-GI-NEXT:    ldr w10, [sp, #48]
-; CHECK-GI-NEXT:    sxtb w11, w5
-; CHECK-GI-NEXT:    sxtb w12, w6
-; CHECK-GI-NEXT:    sxtb w13, w7
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #16]
-; CHECK-GI-NEXT:    mov v1.s[0], w9
 ; CHECK-GI-NEXT:    sxtb w9, w1
-; CHECK-GI-NEXT:    ldr w14, [sp, #104]
-; CHECK-GI-NEXT:    mov v20.s[0], wzr
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v0.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #80]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
-; CHECK-GI-NEXT:    sxtb w8, w10
-; CHECK-GI-NEXT:    mov v1.s[1], w11
-; CHECK-GI-NEXT:    ldr w10, [sp, #24]
-; CHECK-GI-NEXT:    sxtb w11, w2
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v20.s[1], wzr
-; CHECK-GI-NEXT:    mov v3.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #56]
-; CHECK-GI-NEXT:    mov v4.s[0], w9
-; CHECK-GI-NEXT:    sxtb w9, w10
-; CHECK-GI-NEXT:    mov v0.s[2], w11
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #32]
-; CHECK-GI-NEXT:    mov v1.s[2], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #48]
+; CHECK-GI-NEXT:    sxtb w11, w6
 ; CHECK-GI-NEXT:    sxtb w12, w3
+; CHECK-GI-NEXT:    sxtb w13, w7
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    sxtb w8, w4
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w14, [sp, #448]
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    sxtb w8, w2
+; CHECK-GI-NEXT:    fmov s5, w10
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #112]
-; CHECK-GI-NEXT:    mov v3.s[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #88]
+; CHECK-GI-NEXT:    sxtb w9, w5
+; CHECK-GI-NEXT:    ldr w10, [sp, #80]
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #16]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v0.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #64]
+; CHECK-GI-NEXT:    mov v2.s[2], w8
 ; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #24]
+; CHECK-GI-NEXT:    fmov s6, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #64]
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    mov v3.s[2], w11
+; CHECK-GI-NEXT:    fmov s4, w9
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w11, [sp, #40]
-; CHECK-GI-NEXT:    mov v1.s[3], w13
-; CHECK-GI-NEXT:    mov v2.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #120]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v5.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #96]
+; CHECK-GI-NEXT:    ldr w9, [sp, #56]
+; CHECK-GI-NEXT:    ldr w11, [sp, #32]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #88]
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
 ; CHECK-GI-NEXT:    mov v4.s[1], w8
-; CHECK-GI-NEXT:    ldr w13, [sp, #72]
-; CHECK-GI-NEXT:    sxtb w8, w10
-; CHECK-GI-NEXT:    mov v3.s[2], w12
-; CHECK-GI-NEXT:    ldr w10, [sp, #352]
+; CHECK-GI-NEXT:    ldr w8, [sp, #120]
 ; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v3.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #112]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w8, w8
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    sxtb w12, w13
-; CHECK-GI-NEXT:    sxtb w13, w14
-; CHECK-GI-NEXT:    mov v20.s[2], wzr
-; CHECK-GI-NEXT:    mov v4.s[2], w9
-; CHECK-GI-NEXT:    sxtb w9, w10
-; CHECK-GI-NEXT:    mov v2.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #128]
-; CHECK-GI-NEXT:    mov v5.s[1], w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #384]
-; CHECK-GI-NEXT:    mov v3.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #360]
-; CHECK-GI-NEXT:    mov v6.s[0], w9
+; CHECK-GI-NEXT:    mov v5.s[1], w9
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v6.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #128]
+; CHECK-GI-NEXT:    mov v4.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #104]
+; CHECK-GI-NEXT:    ldr w9, [sp, #40]
+; CHECK-GI-NEXT:    fmov s7, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #96]
+; CHECK-GI-NEXT:    sxtb w12, w12
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #144]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v4.s[3], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #392]
-; CHECK-GI-NEXT:    mov v5.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #368]
-; CHECK-GI-NEXT:    mov v7.s[0], w10
+; CHECK-GI-NEXT:    mov v5.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #136]
+; CHECK-GI-NEXT:    sxtb w13, w13
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v6.s[1], w12
-; CHECK-GI-NEXT:    ldr w10, [sp, #416]
-; CHECK-GI-NEXT:    sxtb w12, w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #448]
-; CHECK-GI-NEXT:    ldr w8, [sp, #136]
-; CHECK-GI-NEXT:    mov v16.s[0], w9
-; CHECK-GI-NEXT:    sxtb w9, w11
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    mov v7.s[1], w8
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w11, [sp, #400]
-; CHECK-GI-NEXT:    mov v7.s[1], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #480]
-; CHECK-GI-NEXT:    mov v6.s[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #424]
-; CHECK-GI-NEXT:    mov v17.s[0], w10
+; CHECK-GI-NEXT:    ldr w8, [sp, #72]
+; CHECK-GI-NEXT:    mov v6.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #144]
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #360]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    mov v7.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #352]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #456]
-; CHECK-GI-NEXT:    mov v16.s[1], wzr
-; CHECK-GI-NEXT:    mov v18.s[0], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #488]
-; CHECK-GI-NEXT:    mov v7.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #512]
-; CHECK-GI-NEXT:    mov v19.s[0], w12
-; CHECK-GI-NEXT:    mov v17.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #544]
+; CHECK-GI-NEXT:    mov v6.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #384]
+; CHECK-GI-NEXT:    fmov s16, w13
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w13, [sp, #416]
+; CHECK-GI-NEXT:    mov v5.s[3], w8
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w8, [sp, #368]
+; CHECK-GI-NEXT:    mov v7.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #392]
+; CHECK-GI-NEXT:    fmov s17, w12
+; CHECK-GI-NEXT:    fmov s18, w11
+; CHECK-GI-NEXT:    ldr w12, [sp, #424]
+; CHECK-GI-NEXT:    ldr w11, [sp, #456]
 ; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w8, w8
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    sxtb w14, w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #376]
-; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v17.s[1], w9
+; CHECK-GI-NEXT:    mov v16.s[1], wzr
 ; CHECK-GI-NEXT:    mov v18.s[1], w10
+; CHECK-GI-NEXT:    sxtb w10, w12
+; CHECK-GI-NEXT:    sxtb w12, w14
+; CHECK-GI-NEXT:    fmov s19, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #400]
+; CHECK-GI-NEXT:    ldr w9, [sp, #376]
+; CHECK-GI-NEXT:    fmov s20, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #464]
+; CHECK-GI-NEXT:    add v3.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v17.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #488]
+; CHECK-GI-NEXT:    mov v19.s[1], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #432]
-; CHECK-GI-NEXT:    mov v19.s[1], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #520]
-; CHECK-GI-NEXT:    mov v21.s[0], w11
-; CHECK-GI-NEXT:    mov v22.s[0], w9
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v20.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #480]
+; CHECK-GI-NEXT:    mov v18.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #512]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w11, [sp, #464]
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    ldr w9, [sp, #496]
-; CHECK-GI-NEXT:    ldr w12, [sp, #408]
-; CHECK-GI-NEXT:    mov v17.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #528]
+; CHECK-GI-NEXT:    sxtb w8, w8
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v21.s[1], w13
+; CHECK-GI-NEXT:    mov v16.s[2], wzr
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w13, [sp, #440]
-; CHECK-GI-NEXT:    mov v22.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v19.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #520]
+; CHECK-GI-NEXT:    fmov s21, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #544]
+; CHECK-GI-NEXT:    mov v20.s[2], w12
+; CHECK-GI-NEXT:    fmov s22, w13
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v18.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #472]
-; CHECK-GI-NEXT:    mov v19.s[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #504]
-; CHECK-GI-NEXT:    mov v16.s[2], wzr
+; CHECK-GI-NEXT:    ldr w13, [sp, #528]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w12, [sp, #408]
+; CHECK-GI-NEXT:    mov v17.s[3], w9
+; CHECK-GI-NEXT:    mov v21.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #496]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    fmov s23, w11
+; CHECK-GI-NEXT:    mov v22.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #440]
 ; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w11, [sp, #472]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v21.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #536]
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v22.s[2], wzr
-; CHECK-GI-NEXT:    mov v6.s[3], w8
-; CHECK-GI-NEXT:    sxtb w8, w11
-; CHECK-GI-NEXT:    sxtb w9, w9
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v5.s[3], w14
-; CHECK-GI-NEXT:    mov v7.s[3], w12
-; CHECK-GI-NEXT:    mov v17.s[3], w13
-; CHECK-GI-NEXT:    mov v18.s[3], w8
 ; CHECK-GI-NEXT:    mov v16.s[3], wzr
-; CHECK-GI-NEXT:    mov v20.s[3], wzr
-; CHECK-GI-NEXT:    mov v19.s[3], w9
-; CHECK-GI-NEXT:    mov v21.s[3], w10
-; CHECK-GI-NEXT:    mov v22.s[3], wzr
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    add v4.4s, v17.4s, v18.4s
-; CHECK-GI-NEXT:    add v6.4s, v16.4s, v20.4s
-; CHECK-GI-NEXT:    add v5.4s, v19.4s, v21.4s
-; CHECK-GI-NEXT:    add v7.4s, v22.4s, v20.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v6.4s
-; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT:    add v3.4s, v5.4s, v7.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    add v4.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    mov v23.s[1], wzr
+; CHECK-GI-NEXT:    mov v21.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #504]
+; CHECK-GI-NEXT:    mov v22.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #536]
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v18.s[3], w12
+; CHECK-GI-NEXT:    mov v19.s[3], w10
+; CHECK-GI-NEXT:    sxtb w9, w13
+; CHECK-GI-NEXT:    mov v20.s[3], w11
+; CHECK-GI-NEXT:    add v1.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT:    mov v23.s[2], wzr
+; CHECK-GI-NEXT:    mov v21.s[3], w8
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    mov v22.s[3], w9
+; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT:    add v1.4s, v4.4s, v1.4s
+; CHECK-GI-NEXT:    add v6.4s, v19.4s, v20.4s
+; CHECK-GI-NEXT:    mov v23.s[3], wzr
+; CHECK-GI-NEXT:    add v7.4s, v21.4s, v22.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT:    add v0.4s, v23.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add v0.4s, v7.4s, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    add w0, w8, w9
 ; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
@@ -3920,167 +3929,197 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_udot_v33i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q19, q4, [x1]
-; CHECK-GI-NEXT:    mov v25.s[0], wzr
-; CHECK-GI-NEXT:    ldp q7, q5, [x0]
-; CHECK-GI-NEXT:    umov w8, v19.b[0]
-; CHECK-GI-NEXT:    umov w9, v19.b[4]
-; CHECK-GI-NEXT:    umov w10, v19.b[8]
-; CHECK-GI-NEXT:    umov w11, v19.b[1]
-; CHECK-GI-NEXT:    umov w12, v19.b[6]
-; CHECK-GI-NEXT:    umov w13, v19.b[12]
-; CHECK-GI-NEXT:    umov w14, v4.b[0]
-; CHECK-GI-NEXT:    umov w15, v4.b[4]
-; CHECK-GI-NEXT:    umov w16, v4.b[12]
-; CHECK-GI-NEXT:    mov v25.s[1], wzr
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    umov w8, v19.b[5]
-; CHECK-GI-NEXT:    mov v2.s[0], w9
-; CHECK-GI-NEXT:    umov w9, v19.b[9]
-; CHECK-GI-NEXT:    mov v1.s[0], w10
-; CHECK-GI-NEXT:    umov w10, v19.b[2]
-; CHECK-GI-NEXT:    mov v6.s[0], w13
-; CHECK-GI-NEXT:    umov w13, v19.b[3]
-; CHECK-GI-NEXT:    mov v3.s[0], w14
-; CHECK-GI-NEXT:    umov w14, v19.b[13]
-; CHECK-GI-NEXT:    mov v16.s[0], w15
-; CHECK-GI-NEXT:    umov w15, v4.b[8]
-; CHECK-GI-NEXT:    mov v0.s[1], w11
-; CHECK-GI-NEXT:    umov w11, v19.b[10]
-; CHECK-GI-NEXT:    mov v2.s[1], w8
-; CHECK-GI-NEXT:    ldrb w8, [x0, #32]
-; CHECK-GI-NEXT:    mov v1.s[1], w9
-; CHECK-GI-NEXT:    ldrb w9, [x1, #32]
-; CHECK-GI-NEXT:    mov v17.s[0], w16
-; CHECK-GI-NEXT:    umov w16, v19.b[14]
-; CHECK-GI-NEXT:    mov v25.s[2], wzr
-; CHECK-GI-NEXT:    mul w8, w9, w8
-; CHECK-GI-NEXT:    mov v6.s[1], w14
-; CHECK-GI-NEXT:    umov w14, v4.b[1]
-; CHECK-GI-NEXT:    mov v0.s[2], w10
-; CHECK-GI-NEXT:    umov w10, v19.b[7]
-; CHECK-GI-NEXT:    mov v2.s[2], w12
-; CHECK-GI-NEXT:    umov w12, v19.b[11]
-; CHECK-GI-NEXT:    mov v1.s[2], w11
-; CHECK-GI-NEXT:    umov w11, v4.b[5]
-; CHECK-GI-NEXT:    mov v18.s[0], w15
-; CHECK-GI-NEXT:    umov w15, v19.b[15]
-; CHECK-GI-NEXT:    umov w9, v5.b[2]
-; CHECK-GI-NEXT:    mov v6.s[2], w16
-; CHECK-GI-NEXT:    umov w16, v7.b[0]
-; CHECK-GI-NEXT:    mov v3.s[1], w14
-; CHECK-GI-NEXT:    mov v0.s[3], w13
-; CHECK-GI-NEXT:    umov w13, v7.b[4]
-; CHECK-GI-NEXT:    mov v2.s[3], w10
-; CHECK-GI-NEXT:    umov w10, v4.b[6]
-; CHECK-GI-NEXT:    mov v1.s[3], w12
-; CHECK-GI-NEXT:    umov w12, v4.b[13]
-; CHECK-GI-NEXT:    mov v16.s[1], w11
-; CHECK-GI-NEXT:    umov w11, v4.b[9]
-; CHECK-GI-NEXT:    umov w14, v7.b[5]
-; CHECK-GI-NEXT:    mov v19.s[0], w16
-; CHECK-GI-NEXT:    umov w16, v7.b[1]
-; CHECK-GI-NEXT:    mov v6.s[3], w15
-; CHECK-GI-NEXT:    mov v20.s[0], w13
-; CHECK-GI-NEXT:    umov w13, v4.b[2]
-; CHECK-GI-NEXT:    umov w15, v7.b[6]
-; CHECK-GI-NEXT:    mov v17.s[1], w12
-; CHECK-GI-NEXT:    umov w12, v4.b[14]
-; CHECK-GI-NEXT:    mov v27.s[0], w8
-; CHECK-GI-NEXT:    mov v16.s[2], w10
-; CHECK-GI-NEXT:    umov w10, v4.b[7]
-; CHECK-GI-NEXT:    mov v18.s[1], w11
-; CHECK-GI-NEXT:    umov w11, v4.b[10]
-; CHECK-GI-NEXT:    mov v19.s[1], w16
-; CHECK-GI-NEXT:    umov w16, v5.b[4]
+; CHECK-GI-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-GI-NEXT:    ldp q21, q25, [x1]
+; CHECK-GI-NEXT:    fmov s5, wzr
+; CHECK-GI-NEXT:    ldp q26, q22, [x0]
+; CHECK-GI-NEXT:    fmov s6, wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    fmov s3, wzr
+; CHECK-GI-NEXT:    umov w8, v21.b[0]
+; CHECK-GI-NEXT:    umov w9, v21.b[4]
+; CHECK-GI-NEXT:    umov w10, v21.b[1]
+; CHECK-GI-NEXT:    umov w13, v21.b[8]
+; CHECK-GI-NEXT:    umov w11, v21.b[5]
+; CHECK-GI-NEXT:    umov w14, v21.b[9]
+; CHECK-GI-NEXT:    umov w15, v25.b[0]
+; CHECK-GI-NEXT:    umov w12, v21.b[2]
+; CHECK-GI-NEXT:    fmov s2, wzr
+; CHECK-GI-NEXT:    fmov s4, wzr
+; CHECK-GI-NEXT:    mov v5.s[1], wzr
+; CHECK-GI-NEXT:    mov v6.s[1], wzr
+; CHECK-GI-NEXT:    fmov s7, w8
+; CHECK-GI-NEXT:    fmov s17, w9
+; CHECK-GI-NEXT:    umov w8, v21.b[6]
+; CHECK-GI-NEXT:    fmov s16, w13
+; CHECK-GI-NEXT:    umov w9, v21.b[3]
+; CHECK-GI-NEXT:    umov w13, v21.b[7]
+; CHECK-GI-NEXT:    fmov s18, w15
+; CHECK-GI-NEXT:    umov w15, v25.b[4]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    mov v7.s[1], w10
+; CHECK-GI-NEXT:    umov w10, v21.b[12]
+; CHECK-GI-NEXT:    mov v17.s[1], w11
+; CHECK-GI-NEXT:    umov w11, v21.b[13]
+; CHECK-GI-NEXT:    mov v16.s[1], w14
+; CHECK-GI-NEXT:    umov w14, v25.b[1]
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.s[1], wzr
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    fmov s20, w15
+; CHECK-GI-NEXT:    umov w15, v25.b[13]
+; CHECK-GI-NEXT:    mov v4.s[1], wzr
+; CHECK-GI-NEXT:    fmov s19, w10
+; CHECK-GI-NEXT:    mov v7.s[2], w12
+; CHECK-GI-NEXT:    umov w12, v21.b[10]
+; CHECK-GI-NEXT:    mov v18.s[1], w14
+; CHECK-GI-NEXT:    umov w14, v25.b[5]
+; CHECK-GI-NEXT:    mov v17.s[2], w8
+; CHECK-GI-NEXT:    umov w8, v21.b[11]
+; CHECK-GI-NEXT:    umov w10, v21.b[14]
+; CHECK-GI-NEXT:    mov v5.s[2], wzr
+; CHECK-GI-NEXT:    mov v19.s[1], w11
+; CHECK-GI-NEXT:    umov w11, v25.b[2]
+; CHECK-GI-NEXT:    mov v6.s[2], wzr
+; CHECK-GI-NEXT:    mov v16.s[2], w12
+; CHECK-GI-NEXT:    umov w12, v25.b[8]
+; CHECK-GI-NEXT:    mov v7.s[3], w9
 ; CHECK-GI-NEXT:    mov v20.s[1], w14
-; CHECK-GI-NEXT:    umov w14, v4.b[15]
-; CHECK-GI-NEXT:    mov v3.s[2], w13
-; CHECK-GI-NEXT:    mov v17.s[2], w12
-; CHECK-GI-NEXT:    umov w12, v7.b[12]
-; CHECK-GI-NEXT:    umov w13, v7.b[7]
-; CHECK-GI-NEXT:    mov v16.s[3], w10
-; CHECK-GI-NEXT:    umov w10, v7.b[8]
-; CHECK-GI-NEXT:    umov w8, v7.b[3]
+; CHECK-GI-NEXT:    umov w14, v21.b[15]
+; CHECK-GI-NEXT:    umov w9, v25.b[9]
+; CHECK-GI-NEXT:    mov v17.s[3], w13
+; CHECK-GI-NEXT:    umov w13, v25.b[12]
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
 ; CHECK-GI-NEXT:    mov v18.s[2], w11
-; CHECK-GI-NEXT:    umov w11, v7.b[2]
-; CHECK-GI-NEXT:    mov v23.s[0], w16
-; CHECK-GI-NEXT:    mov v20.s[2], w15
-; CHECK-GI-NEXT:    umov w15, v5.b[12]
-; CHECK-GI-NEXT:    umov w16, v7.b[14]
-; CHECK-GI-NEXT:    mov v17.s[3], w14
-; CHECK-GI-NEXT:    umov w14, v7.b[13]
-; CHECK-GI-NEXT:    mov v22.s[0], w12
-; CHECK-GI-NEXT:    umov w12, v7.b[9]
-; CHECK-GI-NEXT:    mov v21.s[0], w10
-; CHECK-GI-NEXT:    umov w10, v4.b[3]
-; CHECK-GI-NEXT:    mov v19.s[2], w11
-; CHECK-GI-NEXT:    umov w11, v5.b[0]
-; CHECK-GI-NEXT:    mov v27.s[1], wzr
-; CHECK-GI-NEXT:    mov v20.s[3], w13
-; CHECK-GI-NEXT:    umov w13, v5.b[5]
-; CHECK-GI-NEXT:    mov v24.s[0], w15
-; CHECK-GI-NEXT:    mov v22.s[1], w14
-; CHECK-GI-NEXT:    umov w14, v5.b[8]
-; CHECK-GI-NEXT:    umov w15, v4.b[11]
-; CHECK-GI-NEXT:    mov v21.s[1], w12
-; CHECK-GI-NEXT:    umov w12, v5.b[13]
-; CHECK-GI-NEXT:    mov v25.s[3], wzr
-; CHECK-GI-NEXT:    mov v4.s[0], w11
-; CHECK-GI-NEXT:    umov w11, v5.b[1]
-; CHECK-GI-NEXT:    mov v3.s[3], w10
-; CHECK-GI-NEXT:    mov v23.s[1], w13
-; CHECK-GI-NEXT:    umov w13, v5.b[6]
-; CHECK-GI-NEXT:    mov v19.s[3], w8
-; CHECK-GI-NEXT:    mov v22.s[2], w16
-; CHECK-GI-NEXT:    umov w16, v5.b[9]
-; CHECK-GI-NEXT:    mov v26.s[0], w14
+; CHECK-GI-NEXT:    umov w11, v26.b[0]
+; CHECK-GI-NEXT:    mov v19.s[2], w10
+; CHECK-GI-NEXT:    fmov s21, w12
+; CHECK-GI-NEXT:    umov w12, v26.b[1]
+; CHECK-GI-NEXT:    mov v16.s[3], w8
+; CHECK-GI-NEXT:    umov w8, v26.b[5]
+; CHECK-GI-NEXT:    umov w10, v25.b[6]
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    fmov s23, w13
+; CHECK-GI-NEXT:    umov w13, v25.b[3]
+; CHECK-GI-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-NEXT:    fmov s24, w11
+; CHECK-GI-NEXT:    mov v21.s[1], w9
+; CHECK-GI-NEXT:    umov w9, v25.b[10]
+; CHECK-GI-NEXT:    umov w11, v26.b[2]
+; CHECK-GI-NEXT:    mov v19.s[3], w14
+; CHECK-GI-NEXT:    umov w14, v26.b[13]
+; CHECK-GI-NEXT:    mov v23.s[1], w15
+; CHECK-GI-NEXT:    umov w15, v25.b[14]
+; CHECK-GI-NEXT:    mov v20.s[2], w10
 ; CHECK-GI-NEXT:    mov v24.s[1], w12
-; CHECK-GI-NEXT:    umov w12, v5.b[14]
-; CHECK-GI-NEXT:    umov w14, v7.b[10]
-; CHECK-GI-NEXT:    mov v4.s[1], w11
-; CHECK-GI-NEXT:    umov w11, v7.b[15]
-; CHECK-GI-NEXT:    mov v18.s[3], w15
-; CHECK-GI-NEXT:    mov v23.s[2], w13
-; CHECK-GI-NEXT:    umov w13, v5.b[7]
-; CHECK-GI-NEXT:    mul v2.4s, v2.4s, v20.4s
-; CHECK-GI-NEXT:    mov v26.s[1], w16
-; CHECK-GI-NEXT:    umov w16, v5.b[10]
-; CHECK-GI-NEXT:    mov v27.s[2], wzr
-; CHECK-GI-NEXT:    mov v24.s[2], w12
-; CHECK-GI-NEXT:    umov w12, v5.b[15]
-; CHECK-GI-NEXT:    mov v21.s[2], w14
-; CHECK-GI-NEXT:    umov w14, v7.b[11]
-; CHECK-GI-NEXT:    mov v4.s[2], w9
-; CHECK-GI-NEXT:    umov w9, v5.b[3]
-; CHECK-GI-NEXT:    mov v22.s[3], w11
-; CHECK-GI-NEXT:    umov w11, v5.b[11]
-; CHECK-GI-NEXT:    mov v23.s[3], w13
-; CHECK-GI-NEXT:    mov v26.s[2], w16
-; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v19.4s
-; CHECK-GI-NEXT:    mov v27.s[3], wzr
-; CHECK-GI-NEXT:    mov v24.s[3], w12
-; CHECK-GI-NEXT:    mov v21.s[3], w14
-; CHECK-GI-NEXT:    mov v4.s[3], w9
-; CHECK-GI-NEXT:    mul v5.4s, v6.4s, v22.4s
-; CHECK-GI-NEXT:    mul v6.4s, v16.4s, v23.4s
-; CHECK-GI-NEXT:    add v16.4s, v25.4s, v25.4s
-; CHECK-GI-NEXT:    mov v26.s[3], w11
-; CHECK-GI-NEXT:    mul v7.4s, v17.4s, v24.4s
-; CHECK-GI-NEXT:    add v0.4s, v25.4s, v16.4s
-; CHECK-GI-NEXT:    mla v5.4s, v1.4s, v21.4s
-; CHECK-GI-NEXT:    mla v6.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT:    add v3.4s, v16.4s, v16.4s
-; CHECK-GI-NEXT:    mla v7.4s, v18.4s, v26.4s
-; CHECK-GI-NEXT:    add v0.4s, v27.4s, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v5.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT:    add v2.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    umov w12, v26.b[4]
+; CHECK-GI-NEXT:    umov w10, v25.b[7]
+; CHECK-GI-NEXT:    mov v21.s[2], w9
+; CHECK-GI-NEXT:    umov w9, v25.b[11]
+; CHECK-GI-NEXT:    mov v18.s[3], w13
+; CHECK-GI-NEXT:    umov w13, v26.b[9]
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[2], wzr
+; CHECK-GI-NEXT:    mov v23.s[2], w15
+; CHECK-GI-NEXT:    umov w15, v25.b[15]
+; CHECK-GI-NEXT:    mov v5.s[3], wzr
+; CHECK-GI-NEXT:    fmov s27, w12
+; CHECK-GI-NEXT:    mov v24.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v26.b[6]
+; CHECK-GI-NEXT:    umov w12, v26.b[8]
+; CHECK-GI-NEXT:    mov v21.s[3], w9
+; CHECK-GI-NEXT:    umov w9, v26.b[12]
+; CHECK-GI-NEXT:    mov v20.s[3], w10
+; CHECK-GI-NEXT:    umov w10, v26.b[3]
+; CHECK-GI-NEXT:    mov v6.s[3], wzr
+; CHECK-GI-NEXT:    mov v27.s[1], w8
+; CHECK-GI-NEXT:    mov v23.s[3], w15
+; CHECK-GI-NEXT:    umov w15, v22.b[0]
+; CHECK-GI-NEXT:    umov w8, v26.b[7]
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    fmov s25, w12
+; CHECK-GI-NEXT:    fmov s29, w9
+; CHECK-GI-NEXT:    umov w9, v22.b[5]
+; CHECK-GI-NEXT:    mov v24.s[3], w10
+; CHECK-GI-NEXT:    umov w10, v22.b[1]
+; CHECK-GI-NEXT:    umov w12, v26.b[10]
+; CHECK-GI-NEXT:    mov v27.s[2], w11
+; CHECK-GI-NEXT:    umov w11, v22.b[4]
+; CHECK-GI-NEXT:    fmov s28, w15
+; CHECK-GI-NEXT:    mov v25.s[1], w13
+; CHECK-GI-NEXT:    umov w13, v26.b[14]
+; CHECK-GI-NEXT:    mov v29.s[1], w14
+; CHECK-GI-NEXT:    umov w15, v22.b[12]
+; CHECK-GI-NEXT:    umov w14, v22.b[2]
+; CHECK-GI-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-NEXT:    mov v28.s[1], w10
+; CHECK-GI-NEXT:    umov w10, v22.b[13]
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    fmov s30, w11
+; CHECK-GI-NEXT:    umov w11, v22.b[6]
+; CHECK-GI-NEXT:    mov v27.s[3], w8
+; CHECK-GI-NEXT:    mov v25.s[2], w12
+; CHECK-GI-NEXT:    mov v29.s[2], w13
+; CHECK-GI-NEXT:    umov w13, v26.b[11]
+; CHECK-GI-NEXT:    fmov s31, w15
+; CHECK-GI-NEXT:    umov w15, v26.b[15]
+; CHECK-GI-NEXT:    umov w12, v22.b[9]
+; CHECK-GI-NEXT:    mov v30.s[1], w9
+; CHECK-GI-NEXT:    umov w9, v22.b[8]
+; CHECK-GI-NEXT:    mov v28.s[2], w14
+; CHECK-GI-NEXT:    ldrb w14, [x1, #32]
+; CHECK-GI-NEXT:    umov w8, v22.b[15]
+; CHECK-GI-NEXT:    mul v17.4s, v17.4s, v27.4s
+; CHECK-GI-NEXT:    mov v31.s[1], w10
+; CHECK-GI-NEXT:    umov w10, v22.b[14]
+; CHECK-GI-NEXT:    mov v25.s[3], w13
+; CHECK-GI-NEXT:    ldrb w13, [x0, #32]
+; CHECK-GI-NEXT:    mov v29.s[3], w15
+; CHECK-GI-NEXT:    mov v4.s[3], wzr
+; CHECK-GI-NEXT:    mov v30.s[2], w11
+; CHECK-GI-NEXT:    fmov s26, w9
+; CHECK-GI-NEXT:    umov w9, v22.b[7]
+; CHECK-GI-NEXT:    umov w11, v22.b[3]
+; CHECK-GI-NEXT:    add v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT:    mla v17.4s, v7.4s, v24.4s
+; CHECK-GI-NEXT:    mov v31.s[2], w10
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mov v26.s[1], w12
+; CHECK-GI-NEXT:    umov w12, v22.b[10]
+; CHECK-GI-NEXT:    mul v19.4s, v19.4s, v29.4s
+; CHECK-GI-NEXT:    mov v30.s[3], w9
+; CHECK-GI-NEXT:    mul w9, w14, w13
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mov v28.s[3], w11
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v5.4s
+; CHECK-GI-NEXT:    mov v31.s[3], w8
+; CHECK-GI-NEXT:    umov w8, v22.b[11]
+; CHECK-GI-NEXT:    fmov s8, w9
+; CHECK-GI-NEXT:    mov v26.s[2], w12
+; CHECK-GI-NEXT:    mla v19.4s, v16.4s, v25.4s
+; CHECK-GI-NEXT:    mul v20.4s, v20.4s, v30.4s
 ; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    mov v8.s[1], wzr
+; CHECK-GI-NEXT:    mul v22.4s, v23.4s, v31.4s
+; CHECK-GI-NEXT:    mov v26.s[3], w8
+; CHECK-GI-NEXT:    add v3.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT:    mla v20.4s, v18.4s, v28.4s
+; CHECK-GI-NEXT:    mov v8.s[2], wzr
+; CHECK-GI-NEXT:    mla v22.4s, v21.4s, v26.4s
+; CHECK-GI-NEXT:    mov v8.s[3], wzr
+; CHECK-GI-NEXT:    add v4.4s, v20.4s, v22.4s
+; CHECK-GI-NEXT:    add v0.4s, v8.4s, v0.4s
+; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <33 x i8>, ptr %a
@@ -4120,115 +4159,126 @@ define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
 ;
 ; CHECK-GI-LABEL: test_udot_v33i8_nomla:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str x27, [sp, #-80]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GI-NEXT:    .cfi_offset w19, -8
 ; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w23, -40
-; CHECK-GI-NEXT:    .cfi_offset w24, -48
-; CHECK-GI-NEXT:    .cfi_offset w25, -56
-; CHECK-GI-NEXT:    .cfi_offset w26, -64
-; CHECK-GI-NEXT:    .cfi_offset w27, -80
-; CHECK-GI-NEXT:    ldp q2, q1, [x0]
-; CHECK-GI-NEXT:    mov v0.s[0], wzr
-; CHECK-GI-NEXT:    ldrb w2, [x0, #32]
-; CHECK-GI-NEXT:    umov w20, v2.b[0]
-; CHECK-GI-NEXT:    umov w21, v2.b[4]
-; CHECK-GI-NEXT:    umov w22, v2.b[8]
-; CHECK-GI-NEXT:    umov w23, v2.b[12]
-; CHECK-GI-NEXT:    umov w24, v1.b[0]
-; CHECK-GI-NEXT:    umov w25, v1.b[4]
-; CHECK-GI-NEXT:    umov w26, v1.b[8]
-; CHECK-GI-NEXT:    umov w27, v1.b[12]
-; CHECK-GI-NEXT:    umov w0, v2.b[1]
-; CHECK-GI-NEXT:    umov w12, v2.b[2]
-; CHECK-GI-NEXT:    umov w8, v2.b[3]
-; CHECK-GI-NEXT:    umov w3, v2.b[5]
-; CHECK-GI-NEXT:    umov w14, v2.b[6]
-; CHECK-GI-NEXT:    umov w9, v2.b[7]
-; CHECK-GI-NEXT:    umov w4, v2.b[9]
-; CHECK-GI-NEXT:    umov w15, v2.b[10]
-; CHECK-GI-NEXT:    umov w10, v2.b[11]
-; CHECK-GI-NEXT:    umov w5, v2.b[13]
-; CHECK-GI-NEXT:    umov w16, v2.b[14]
-; CHECK-GI-NEXT:    umov w11, v2.b[15]
-; CHECK-GI-NEXT:    umov w6, v1.b[1]
-; CHECK-GI-NEXT:    umov w7, v1.b[5]
-; CHECK-GI-NEXT:    umov w19, v1.b[9]
-; CHECK-GI-NEXT:    mov v2.s[0], w20
-; CHECK-GI-NEXT:    mov v3.s[0], w21
-; CHECK-GI-NEXT:    mov v4.s[0], w22
-; CHECK-GI-NEXT:    mov v5.s[0], w23
-; CHECK-GI-NEXT:    mov v6.s[0], w24
-; CHECK-GI-NEXT:    ldp x24, x23, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v7.s[0], w25
-; CHECK-GI-NEXT:    mov v16.s[0], w26
-; CHECK-GI-NEXT:    umov w20, v1.b[13]
-; CHECK-GI-NEXT:    ldp x26, x25, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v17.s[0], w27
-; CHECK-GI-NEXT:    mov v18.s[0], w2
-; CHECK-GI-NEXT:    umov w17, v1.b[2]
-; CHECK-GI-NEXT:    umov w1, v1.b[6]
-; CHECK-GI-NEXT:    umov w2, v1.b[10]
-; CHECK-GI-NEXT:    umov w21, v1.b[14]
-; CHECK-GI-NEXT:    mov v2.s[1], w0
-; CHECK-GI-NEXT:    mov v3.s[1], w3
-; CHECK-GI-NEXT:    mov v4.s[1], w4
-; CHECK-GI-NEXT:    mov v5.s[1], w5
-; CHECK-GI-NEXT:    mov v6.s[1], w6
-; CHECK-GI-NEXT:    mov v7.s[1], w7
-; CHECK-GI-NEXT:    mov v16.s[1], w19
-; CHECK-GI-NEXT:    mov v17.s[1], w20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v18.s[1], wzr
+; CHECK-GI-NEXT:    ldp q7, q19, [x0]
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    ldrb w10, [x0, #32]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    fmov s3, wzr
+; CHECK-GI-NEXT:    fmov s2, wzr
+; CHECK-GI-NEXT:    fmov s5, wzr
+; CHECK-GI-NEXT:    fmov s4, wzr
+; CHECK-GI-NEXT:    umov w15, v7.b[8]
+; CHECK-GI-NEXT:    umov w2, v7.b[12]
+; CHECK-GI-NEXT:    umov w16, v7.b[9]
+; CHECK-GI-NEXT:    umov w4, v7.b[13]
+; CHECK-GI-NEXT:    umov w8, v7.b[0]
+; CHECK-GI-NEXT:    umov w12, v7.b[4]
+; CHECK-GI-NEXT:    umov w9, v7.b[1]
+; CHECK-GI-NEXT:    umov w14, v7.b[5]
+; CHECK-GI-NEXT:    umov w3, v7.b[10]
+; CHECK-GI-NEXT:    umov w5, v7.b[14]
+; CHECK-GI-NEXT:    umov w11, v7.b[2]
+; CHECK-GI-NEXT:    umov w17, v7.b[3]
+; CHECK-GI-NEXT:    fmov s17, w15
+; CHECK-GI-NEXT:    fmov s18, w2
+; CHECK-GI-NEXT:    umov w13, v7.b[6]
+; CHECK-GI-NEXT:    umov w18, v7.b[7]
+; CHECK-GI-NEXT:    umov w0, v7.b[11]
+; CHECK-GI-NEXT:    umov w1, v7.b[15]
+; CHECK-GI-NEXT:    fmov s7, w8
+; CHECK-GI-NEXT:    fmov s16, w12
+; CHECK-GI-NEXT:    umov w2, v19.b[0]
+; CHECK-GI-NEXT:    mov v17.s[1], w16
+; CHECK-GI-NEXT:    mov v18.s[1], w4
+; CHECK-GI-NEXT:    umov w4, v19.b[4]
+; CHECK-GI-NEXT:    umov w6, v19.b[1]
+; CHECK-GI-NEXT:    umov w7, v19.b[5]
+; CHECK-GI-NEXT:    umov w19, v19.b[9]
+; CHECK-GI-NEXT:    mov v7.s[1], w9
+; CHECK-GI-NEXT:    mov v16.s[1], w14
+; CHECK-GI-NEXT:    umov w20, v19.b[13]
+; CHECK-GI-NEXT:    fmov s6, wzr
+; CHECK-GI-NEXT:    umov w12, v19.b[2]
+; CHECK-GI-NEXT:    umov w8, v19.b[3]
+; CHECK-GI-NEXT:    mov v17.s[2], w3
+; CHECK-GI-NEXT:    umov w3, v19.b[8]
+; CHECK-GI-NEXT:    mov v18.s[2], w5
+; CHECK-GI-NEXT:    umov w5, v19.b[12]
+; CHECK-GI-NEXT:    umov w14, v19.b[6]
+; CHECK-GI-NEXT:    umov w9, v19.b[7]
+; CHECK-GI-NEXT:    mov v7.s[2], w11
+; CHECK-GI-NEXT:    mov v16.s[2], w13
+; CHECK-GI-NEXT:    umov w15, v19.b[10]
+; CHECK-GI-NEXT:    umov w11, v19.b[11]
+; CHECK-GI-NEXT:    umov w16, v19.b[14]
+; CHECK-GI-NEXT:    umov w13, v19.b[15]
+; CHECK-GI-NEXT:    fmov s19, w2
+; CHECK-GI-NEXT:    fmov s20, w4
+; CHECK-GI-NEXT:    fmov s21, w3
+; CHECK-GI-NEXT:    fmov s22, w5
+; CHECK-GI-NEXT:    fmov s23, w10
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
 ; CHECK-GI-NEXT:    mov v0.s[1], wzr
-; CHECK-GI-NEXT:    umov w13, v1.b[3]
-; CHECK-GI-NEXT:    umov w18, v1.b[7]
-; CHECK-GI-NEXT:    umov w0, v1.b[11]
-; CHECK-GI-NEXT:    umov w3, v1.b[15]
-; CHECK-GI-NEXT:    mov v2.s[2], w12
-; CHECK-GI-NEXT:    mov v3.s[2], w14
-; CHECK-GI-NEXT:    mov v4.s[2], w15
-; CHECK-GI-NEXT:    mov v5.s[2], w16
-; CHECK-GI-NEXT:    mov v6.s[2], w17
-; CHECK-GI-NEXT:    mov v7.s[2], w1
-; CHECK-GI-NEXT:    mov v16.s[2], w2
-; CHECK-GI-NEXT:    mov v17.s[2], w21
-; CHECK-GI-NEXT:    mov v18.s[2], wzr
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v3.s[1], wzr
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    mov v19.s[1], w6
+; CHECK-GI-NEXT:    mov v20.s[1], w7
+; CHECK-GI-NEXT:    mov v21.s[1], w19
+; CHECK-GI-NEXT:    mov v22.s[1], w20
+; CHECK-GI-NEXT:    mov v23.s[1], wzr
+; CHECK-GI-NEXT:    mov v5.s[1], wzr
+; CHECK-GI-NEXT:    mov v4.s[1], wzr
+; CHECK-GI-NEXT:    mov v6.s[1], wzr
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
 ; CHECK-GI-NEXT:    mov v0.s[2], wzr
-; CHECK-GI-NEXT:    mov v2.s[3], w8
-; CHECK-GI-NEXT:    mov v3.s[3], w9
-; CHECK-GI-NEXT:    mov v4.s[3], w10
-; CHECK-GI-NEXT:    mov v5.s[3], w11
-; CHECK-GI-NEXT:    mov v6.s[3], w13
-; CHECK-GI-NEXT:    mov v7.s[3], w18
-; CHECK-GI-NEXT:    mov v16.s[3], w0
-; CHECK-GI-NEXT:    mov v17.s[3], w3
-; CHECK-GI-NEXT:    mov v18.s[3], wzr
+; CHECK-GI-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v19.s[2], w12
+; CHECK-GI-NEXT:    mov v20.s[2], w14
+; CHECK-GI-NEXT:    mov v21.s[2], w15
+; CHECK-GI-NEXT:    mov v22.s[2], w16
+; CHECK-GI-NEXT:    mov v23.s[2], wzr
+; CHECK-GI-NEXT:    mov v5.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[2], wzr
+; CHECK-GI-NEXT:    mov v6.s[2], wzr
+; CHECK-GI-NEXT:    mov v7.s[3], w17
+; CHECK-GI-NEXT:    mov v16.s[3], w18
+; CHECK-GI-NEXT:    mov v17.s[3], w0
+; CHECK-GI-NEXT:    mov v18.s[3], w1
+; CHECK-GI-NEXT:    mov v19.s[3], w8
+; CHECK-GI-NEXT:    mov v20.s[3], w9
+; CHECK-GI-NEXT:    mov v21.s[3], w11
+; CHECK-GI-NEXT:    mov v22.s[3], w13
+; CHECK-GI-NEXT:    mov v23.s[3], wzr
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
 ; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    mov v5.s[3], wzr
+; CHECK-GI-NEXT:    mov v4.s[3], wzr
+; CHECK-GI-NEXT:    mov v6.s[3], wzr
+; CHECK-GI-NEXT:    add v7.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    add v16.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT:    add v17.4s, v19.4s, v20.4s
+; CHECK-GI-NEXT:    add v18.4s, v21.4s, v22.4s
+; CHECK-GI-NEXT:    add v1.4s, v23.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    add v3.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
 ; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    add v5.4s, v18.4s, v0.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT:    add v3.4s, v5.4s, v0.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
-; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
 ; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    ldr x27, [sp], #80 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ldp x20, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <33 x i8>, ptr %a1
@@ -4266,169 +4316,199 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_sdot_v33i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q19, q4, [x1]
-; CHECK-GI-NEXT:    mov v25.s[0], wzr
-; CHECK-GI-NEXT:    ldp q7, q5, [x0]
-; CHECK-GI-NEXT:    smov w8, v19.b[0]
-; CHECK-GI-NEXT:    smov w9, v19.b[4]
-; CHECK-GI-NEXT:    smov w10, v19.b[8]
-; CHECK-GI-NEXT:    smov w11, v19.b[1]
-; CHECK-GI-NEXT:    smov w12, v19.b[6]
-; CHECK-GI-NEXT:    smov w13, v19.b[12]
-; CHECK-GI-NEXT:    smov w14, v4.b[0]
-; CHECK-GI-NEXT:    smov w15, v4.b[4]
-; CHECK-GI-NEXT:    smov w16, v4.b[12]
-; CHECK-GI-NEXT:    mov v25.s[1], wzr
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    smov w8, v19.b[5]
-; CHECK-GI-NEXT:    mov v2.s[0], w9
-; CHECK-GI-NEXT:    smov w9, v19.b[9]
-; CHECK-GI-NEXT:    mov v1.s[0], w10
-; CHECK-GI-NEXT:    smov w10, v19.b[2]
-; CHECK-GI-NEXT:    mov v6.s[0], w13
-; CHECK-GI-NEXT:    smov w13, v19.b[3]
-; CHECK-GI-NEXT:    mov v3.s[0], w14
-; CHECK-GI-NEXT:    smov w14, v19.b[13]
-; CHECK-GI-NEXT:    mov v16.s[0], w15
-; CHECK-GI-NEXT:    smov w15, v4.b[8]
-; CHECK-GI-NEXT:    mov v0.s[1], w11
-; CHECK-GI-NEXT:    smov w11, v19.b[10]
-; CHECK-GI-NEXT:    mov v2.s[1], w8
-; CHECK-GI-NEXT:    ldrsb w8, [x0, #32]
-; CHECK-GI-NEXT:    mov v1.s[1], w9
-; CHECK-GI-NEXT:    ldrsb w9, [x1, #32]
-; CHECK-GI-NEXT:    mov v17.s[0], w16
-; CHECK-GI-NEXT:    smov w16, v19.b[14]
-; CHECK-GI-NEXT:    mov v25.s[2], wzr
-; CHECK-GI-NEXT:    mul w8, w9, w8
-; CHECK-GI-NEXT:    mov v6.s[1], w14
-; CHECK-GI-NEXT:    smov w14, v4.b[1]
-; CHECK-GI-NEXT:    mov v0.s[2], w10
-; CHECK-GI-NEXT:    smov w10, v19.b[7]
-; CHECK-GI-NEXT:    mov v2.s[2], w12
-; CHECK-GI-NEXT:    smov w12, v19.b[11]
-; CHECK-GI-NEXT:    mov v1.s[2], w11
-; CHECK-GI-NEXT:    smov w11, v4.b[5]
-; CHECK-GI-NEXT:    mov v18.s[0], w15
-; CHECK-GI-NEXT:    smov w15, v19.b[15]
-; CHECK-GI-NEXT:    smov w9, v5.b[2]
-; CHECK-GI-NEXT:    mov v6.s[2], w16
-; CHECK-GI-NEXT:    smov w16, v7.b[0]
-; CHECK-GI-NEXT:    mov v3.s[1], w14
-; CHECK-GI-NEXT:    mov v0.s[3], w13
-; CHECK-GI-NEXT:    smov w13, v7.b[4]
-; CHECK-GI-NEXT:    mov v2.s[3], w10
-; CHECK-GI-NEXT:    smov w10, v4.b[6]
-; CHECK-GI-NEXT:    mov v1.s[3], w12
-; CHECK-GI-NEXT:    smov w12, v4.b[13]
-; CHECK-GI-NEXT:    mov v16.s[1], w11
-; CHECK-GI-NEXT:    smov w11, v4.b[9]
-; CHECK-GI-NEXT:    smov w14, v7.b[5]
-; CHECK-GI-NEXT:    mov v19.s[0], w16
-; CHECK-GI-NEXT:    smov w16, v7.b[1]
-; CHECK-GI-NEXT:    mov v6.s[3], w15
-; CHECK-GI-NEXT:    mov v20.s[0], w13
-; CHECK-GI-NEXT:    smov w13, v4.b[2]
-; CHECK-GI-NEXT:    smov w15, v7.b[6]
-; CHECK-GI-NEXT:    mov v17.s[1], w12
-; CHECK-GI-NEXT:    smov w12, v4.b[14]
-; CHECK-GI-NEXT:    mov v27.s[0], w8
-; CHECK-GI-NEXT:    mov v16.s[2], w10
-; CHECK-GI-NEXT:    smov w10, v4.b[7]
-; CHECK-GI-NEXT:    mov v18.s[1], w11
-; CHECK-GI-NEXT:    smov w11, v4.b[10]
-; CHECK-GI-NEXT:    mov v19.s[1], w16
-; CHECK-GI-NEXT:    smov w16, v5.b[4]
+; CHECK-GI-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-GI-NEXT:    ldp q21, q25, [x1]
+; CHECK-GI-NEXT:    fmov s5, wzr
+; CHECK-GI-NEXT:    ldp q26, q22, [x0]
+; CHECK-GI-NEXT:    fmov s6, wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    fmov s3, wzr
+; CHECK-GI-NEXT:    smov w8, v21.b[0]
+; CHECK-GI-NEXT:    smov w9, v21.b[4]
+; CHECK-GI-NEXT:    smov w10, v21.b[1]
+; CHECK-GI-NEXT:    smov w13, v21.b[8]
+; CHECK-GI-NEXT:    smov w11, v21.b[5]
+; CHECK-GI-NEXT:    smov w14, v21.b[9]
+; CHECK-GI-NEXT:    smov w15, v25.b[0]
+; CHECK-GI-NEXT:    smov w12, v21.b[2]
+; CHECK-GI-NEXT:    fmov s2, wzr
+; CHECK-GI-NEXT:    fmov s4, wzr
+; CHECK-GI-NEXT:    mov v5.s[1], wzr
+; CHECK-GI-NEXT:    mov v6.s[1], wzr
+; CHECK-GI-NEXT:    fmov s7, w8
+; CHECK-GI-NEXT:    fmov s17, w9
+; CHECK-GI-NEXT:    smov w8, v21.b[6]
+; CHECK-GI-NEXT:    fmov s16, w13
+; CHECK-GI-NEXT:    smov w9, v21.b[3]
+; CHECK-GI-NEXT:    smov w13, v21.b[7]
+; CHECK-GI-NEXT:    fmov s18, w15
+; CHECK-GI-NEXT:    smov w15, v25.b[4]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    mov v7.s[1], w10
+; CHECK-GI-NEXT:    smov w10, v21.b[12]
+; CHECK-GI-NEXT:    mov v17.s[1], w11
+; CHECK-GI-NEXT:    smov w11, v21.b[13]
+; CHECK-GI-NEXT:    mov v16.s[1], w14
+; CHECK-GI-NEXT:    smov w14, v25.b[1]
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.s[1], wzr
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    fmov s20, w15
+; CHECK-GI-NEXT:    smov w15, v25.b[13]
+; CHECK-GI-NEXT:    mov v4.s[1], wzr
+; CHECK-GI-NEXT:    fmov s19, w10
+; CHECK-GI-NEXT:    mov v7.s[2], w12
+; CHECK-GI-NEXT:    smov w12, v21.b[10]
+; CHECK-GI-NEXT:    mov v18.s[1], w14
+; CHECK-GI-NEXT:    smov w14, v25.b[5]
+; CHECK-GI-NEXT:    mov v17.s[2], w8
+; CHECK-GI-NEXT:    smov w8, v21.b[11]
+; CHECK-GI-NEXT:    smov w10, v21.b[14]
+; CHECK-GI-NEXT:    mov v5.s[2], wzr
+; CHECK-GI-NEXT:    mov v19.s[1], w11
+; CHECK-GI-NEXT:    smov w11, v25.b[2]
+; CHECK-GI-NEXT:    mov v6.s[2], wzr
+; CHECK-GI-NEXT:    mov v16.s[2], w12
+; CHECK-GI-NEXT:    smov w12, v25.b[8]
+; CHECK-GI-NEXT:    mov v7.s[3], w9
 ; CHECK-GI-NEXT:    mov v20.s[1], w14
-; CHECK-GI-NEXT:    smov w14, v4.b[15]
-; CHECK-GI-NEXT:    mov v3.s[2], w13
-; CHECK-GI-NEXT:    mov v17.s[2], w12
-; CHECK-GI-NEXT:    smov w12, v7.b[12]
-; CHECK-GI-NEXT:    smov w13, v7.b[7]
-; CHECK-GI-NEXT:    mov v16.s[3], w10
-; CHECK-GI-NEXT:    smov w10, v7.b[8]
-; CHECK-GI-NEXT:    smov w8, v7.b[3]
+; CHECK-GI-NEXT:    smov w14, v21.b[15]
+; CHECK-GI-NEXT:    smov w9, v25.b[9]
+; CHECK-GI-NEXT:    mov v17.s[3], w13
+; CHECK-GI-NEXT:    smov w13, v25.b[12]
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
 ; CHECK-GI-NEXT:    mov v18.s[2], w11
-; CHECK-GI-NEXT:    smov w11, v7.b[2]
-; CHECK-GI-NEXT:    mov v23.s[0], w16
-; CHECK-GI-NEXT:    mov v20.s[2], w15
-; CHECK-GI-NEXT:    smov w15, v5.b[12]
-; CHECK-GI-NEXT:    smov w16, v7.b[14]
-; CHECK-GI-NEXT:    mov v17.s[3], w14
-; CHECK-GI-NEXT:    smov w14, v7.b[13]
-; CHECK-GI-NEXT:    mov v22.s[0], w12
-; CHECK-GI-NEXT:    smov w12, v7.b[9]
-; CHECK-GI-NEXT:    mov v21.s[0], w10
-; CHECK-GI-NEXT:    smov w10, v4.b[3]
-; CHECK-GI-NEXT:    mov v19.s[2], w11
-; CHECK-GI-NEXT:    smov w11, v5.b[0]
-; CHECK-GI-NEXT:    mov v27.s[1], wzr
-; CHECK-GI-NEXT:    mov v20.s[3], w13
-; CHECK-GI-NEXT:    smov w13, v5.b[5]
-; CHECK-GI-NEXT:    mov v24.s[0], w15
-; CHECK-GI-NEXT:    mov v22.s[1], w14
-; CHECK-GI-NEXT:    smov w14, v5.b[8]
-; CHECK-GI-NEXT:    smov w15, v4.b[11]
-; CHECK-GI-NEXT:    mov v21.s[1], w12
-; CHECK-GI-NEXT:    smov w12, v5.b[13]
-; CHECK-GI-NEXT:    mov v25.s[3], wzr
-; CHECK-GI-NEXT:    mov v4.s[0], w11
-; CHECK-GI-NEXT:    smov w11, v5.b[1]
-; CHECK-GI-NEXT:    mov v3.s[3], w10
-; CHECK-GI-NEXT:    mov v23.s[1], w13
-; CHECK-GI-NEXT:    smov w13, v5.b[6]
-; CHECK-GI-NEXT:    mov v19.s[3], w8
-; CHECK-GI-NEXT:    mov v22.s[2], w16
-; CHECK-GI-NEXT:    smov w16, v5.b[9]
-; CHECK-GI-NEXT:    mov v26.s[0], w14
+; CHECK-GI-NEXT:    smov w11, v26.b[0]
+; CHECK-GI-NEXT:    mov v19.s[2], w10
+; CHECK-GI-NEXT:    fmov s21, w12
+; CHECK-GI-NEXT:    smov w12, v26.b[1]
+; CHECK-GI-NEXT:    mov v16.s[3], w8
+; CHECK-GI-NEXT:    smov w8, v26.b[5]
+; CHECK-GI-NEXT:    smov w10, v25.b[6]
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    fmov s23, w13
+; CHECK-GI-NEXT:    smov w13, v25.b[3]
+; CHECK-GI-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-NEXT:    fmov s24, w11
+; CHECK-GI-NEXT:    mov v21.s[1], w9
+; CHECK-GI-NEXT:    smov w9, v25.b[10]
+; CHECK-GI-NEXT:    smov w11, v26.b[2]
+; CHECK-GI-NEXT:    mov v19.s[3], w14
+; CHECK-GI-NEXT:    smov w14, v26.b[13]
+; CHECK-GI-NEXT:    mov v23.s[1], w15
+; CHECK-GI-NEXT:    smov w15, v25.b[14]
+; CHECK-GI-NEXT:    mov v20.s[2], w10
 ; CHECK-GI-NEXT:    mov v24.s[1], w12
-; CHECK-GI-NEXT:    smov w12, v5.b[14]
-; CHECK-GI-NEXT:    smov w14, v7.b[10]
-; CHECK-GI-NEXT:    mov v4.s[1], w11
-; CHECK-GI-NEXT:    smov w11, v7.b[15]
-; CHECK-GI-NEXT:    mov v18.s[3], w15
-; CHECK-GI-NEXT:    mov v23.s[2], w13
-; CHECK-GI-NEXT:    smov w13, v5.b[7]
-; CHECK-GI-NEXT:    mul v2.4s, v2.4s, v20.4s
-; CHECK-GI-NEXT:    mov v26.s[1], w16
-; CHECK-GI-NEXT:    smov w16, v5.b[10]
-; CHECK-GI-NEXT:    mov v27.s[2], wzr
-; CHECK-GI-NEXT:    mov v24.s[2], w12
-; CHECK-GI-NEXT:    smov w12, v5.b[15]
-; CHECK-GI-NEXT:    mov v21.s[2], w14
-; CHECK-GI-NEXT:    smov w14, v7.b[11]
-; CHECK-GI-NEXT:    mov v4.s[2], w9
-; CHECK-GI-NEXT:    smov w9, v5.b[3]
-; CHECK-GI-NEXT:    mov v22.s[3], w11
-; CHECK-GI-NEXT:    smov w11, v5.b[11]
-; CHECK-GI-NEXT:    mov v23.s[3], w13
-; CHECK-GI-NEXT:    mov v26.s[2], w16
-; CHECK-GI-NEXT:    mla v2.4s, v0.4s, v19.4s
-; CHECK-GI-NEXT:    mov v27.s[3], wzr
-; CHECK-GI-NEXT:    mov v24.s[3], w12
-; CHECK-GI-NEXT:    mov v21.s[3], w14
-; CHECK-GI-NEXT:    mov v4.s[3], w9
-; CHECK-GI-NEXT:    mul v5.4s, v6.4s, v22.4s
-; CHECK-GI-NEXT:    mul v6.4s, v16.4s, v23.4s
-; CHECK-GI-NEXT:    add v16.4s, v25.4s, v25.4s
-; CHECK-GI-NEXT:    mov v26.s[3], w11
-; CHECK-GI-NEXT:    mul v7.4s, v17.4s, v24.4s
-; CHECK-GI-NEXT:    add v0.4s, v25.4s, v16.4s
-; CHECK-GI-NEXT:    mla v5.4s, v1.4s, v21.4s
-; CHECK-GI-NEXT:    mla v6.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT:    add v3.4s, v16.4s, v16.4s
-; CHECK-GI-NEXT:    mla v7.4s, v18.4s, v26.4s
-; CHECK-GI-NEXT:    add v0.4s, v27.4s, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v5.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT:    add v2.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    addv s0, v0.4s
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    add w0, w8, w2
-; CHECK-GI-NEXT:    ret
-entry:
+; CHECK-GI-NEXT:    smov w12, v26.b[4]
+; CHECK-GI-NEXT:    smov w10, v25.b[7]
+; CHECK-GI-NEXT:    mov v21.s[2], w9
+; CHECK-GI-NEXT:    smov w9, v25.b[11]
+; CHECK-GI-NEXT:    mov v18.s[3], w13
+; CHECK-GI-NEXT:    smov w13, v26.b[9]
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[2], wzr
+; CHECK-GI-NEXT:    mov v23.s[2], w15
+; CHECK-GI-NEXT:    smov w15, v25.b[15]
+; CHECK-GI-NEXT:    mov v5.s[3], wzr
+; CHECK-GI-NEXT:    fmov s27, w12
+; CHECK-GI-NEXT:    mov v24.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v26.b[6]
+; CHECK-GI-NEXT:    smov w12, v26.b[8]
+; CHECK-GI-NEXT:    mov v21.s[3], w9
+; CHECK-GI-NEXT:    smov w9, v26.b[12]
+; CHECK-GI-NEXT:    mov v20.s[3], w10
+; CHECK-GI-NEXT:    smov w10, v26.b[3]
+; CHECK-GI-NEXT:    mov v6.s[3], wzr
+; CHECK-GI-NEXT:    mov v27.s[1], w8
+; CHECK-GI-NEXT:    mov v23.s[3], w15
+; CHECK-GI-NEXT:    smov w15, v22.b[0]
+; CHECK-GI-NEXT:    smov w8, v26.b[7]
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    fmov s25, w12
+; CHECK-GI-NEXT:    fmov s29, w9
+; CHECK-GI-NEXT:    smov w9, v22.b[5]
+; CHECK-GI-NEXT:    mov v24.s[3], w10
+; CHECK-GI-NEXT:    smov w10, v22.b[1]
+; CHECK-GI-NEXT:    smov w12, v26.b[10]
+; CHECK-GI-NEXT:    mov v27.s[2], w11
+; CHECK-GI-NEXT:    smov w11, v22.b[4]
+; CHECK-GI-NEXT:    fmov s28, w15
+; CHECK-GI-NEXT:    mov v25.s[1], w13
+; CHECK-GI-NEXT:    smov w13, v26.b[14]
+; CHECK-GI-NEXT:    mov v29.s[1], w14
+; CHECK-GI-NEXT:    smov w15, v22.b[12]
+; CHECK-GI-NEXT:    smov w14, v22.b[2]
+; CHECK-GI-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-NEXT:    mov v28.s[1], w10
+; CHECK-GI-NEXT:    smov w10, v22.b[13]
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    fmov s30, w11
+; CHECK-GI-NEXT:    smov w11, v22.b[6]
+; CHECK-GI-NEXT:    mov v27.s[3], w8
+; CHECK-GI-NEXT:    mov v25.s[2], w12
+; CHECK-GI-NEXT:    mov v29.s[2], w13
+; CHECK-GI-NEXT:    smov w13, v26.b[11]
+; CHECK-GI-NEXT:    fmov s31, w15
+; CHECK-GI-NEXT:    smov w15, v26.b[15]
+; CHECK-GI-NEXT:    smov w12, v22.b[9]
+; CHECK-GI-NEXT:    mov v30.s[1], w9
+; CHECK-GI-NEXT:    smov w9, v22.b[8]
+; CHECK-GI-NEXT:    mov v28.s[2], w14
+; CHECK-GI-NEXT:    ldrsb w14, [x1, #32]
+; CHECK-GI-NEXT:    smov w8, v22.b[15]
+; CHECK-GI-NEXT:    mul v17.4s, v17.4s, v27.4s
+; CHECK-GI-NEXT:    mov v31.s[1], w10
+; CHECK-GI-NEXT:    smov w10, v22.b[14]
+; CHECK-GI-NEXT:    mov v25.s[3], w13
+; CHECK-GI-NEXT:    ldrsb w13, [x0, #32]
+; CHECK-GI-NEXT:    mov v29.s[3], w15
+; CHECK-GI-NEXT:    mov v4.s[3], wzr
+; CHECK-GI-NEXT:    mov v30.s[2], w11
+; CHECK-GI-NEXT:    fmov s26, w9
+; CHECK-GI-NEXT:    smov w9, v22.b[7]
+; CHECK-GI-NEXT:    smov w11, v22.b[3]
+; CHECK-GI-NEXT:    add v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT:    mla v17.4s, v7.4s, v24.4s
+; CHECK-GI-NEXT:    mov v31.s[2], w10
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mov v26.s[1], w12
+; CHECK-GI-NEXT:    smov w12, v22.b[10]
+; CHECK-GI-NEXT:    mul v19.4s, v19.4s, v29.4s
+; CHECK-GI-NEXT:    mov v30.s[3], w9
+; CHECK-GI-NEXT:    mul w9, w14, w13
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mov v28.s[3], w11
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v5.4s
+; CHECK-GI-NEXT:    mov v31.s[3], w8
+; CHECK-GI-NEXT:    smov w8, v22.b[11]
+; CHECK-GI-NEXT:    fmov s8, w9
+; CHECK-GI-NEXT:    mov v26.s[2], w12
+; CHECK-GI-NEXT:    mla v19.4s, v16.4s, v25.4s
+; CHECK-GI-NEXT:    mul v20.4s, v20.4s, v30.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    mov v8.s[1], wzr
+; CHECK-GI-NEXT:    mul v22.4s, v23.4s, v31.4s
+; CHECK-GI-NEXT:    mov v26.s[3], w8
+; CHECK-GI-NEXT:    add v3.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT:    mla v20.4s, v18.4s, v28.4s
+; CHECK-GI-NEXT:    mov v8.s[2], wzr
+; CHECK-GI-NEXT:    mla v22.4s, v21.4s, v26.4s
+; CHECK-GI-NEXT:    mov v8.s[3], wzr
+; CHECK-GI-NEXT:    add v4.4s, v20.4s, v22.4s
+; CHECK-GI-NEXT:    add v0.4s, v8.4s, v0.4s
+; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
   %0 = load <33 x i8>, ptr %a
   %1 = sext <33 x i8> %0 to <33 x i32>
   %2 = load <33 x i8>, ptr %b
@@ -4722,12 +4802,13 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33
 ;
 ; CHECK-GI-LABEL: test_sdot_v33i8_double:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT:    sub sp, sp, #96
+; CHECK-GI-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    str x29, [sp, #80] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    .cfi_offset b8, -24
 ; CHECK-GI-NEXT:    .cfi_offset b9, -32
@@ -4737,444 +4818,508 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33
 ; CHECK-GI-NEXT:    .cfi_offset b13, -64
 ; CHECK-GI-NEXT:    .cfi_offset b14, -72
 ; CHECK-GI-NEXT:    .cfi_offset b15, -80
-; CHECK-GI-NEXT:    ldr w8, [sp, #80]
-; CHECK-GI-NEXT:    sxtb w9, w0
-; CHECK-GI-NEXT:    ldr w10, [sp, #112]
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    sxtb w9, w1
+; CHECK-GI-NEXT:    sxtb w10, w2
 ; CHECK-GI-NEXT:    sxtb w11, w4
+; CHECK-GI-NEXT:    sxtb w12, w5
 ; CHECK-GI-NEXT:    sxtb w13, w7
-; CHECK-GI-NEXT:    sxtb w12, w3
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v0.s[0], w9
-; CHECK-GI-NEXT:    sxtb w9, w10
-; CHECK-GI-NEXT:    mov v3.s[0], w11
-; CHECK-GI-NEXT:    sxtb w10, w1
-; CHECK-GI-NEXT:    sxtb w11, w5
-; CHECK-GI-NEXT:    mov v1.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #88]
-; CHECK-GI-NEXT:    mov v5.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #120]
-; CHECK-GI-NEXT:    ldr w14, [sp, #168]
-; CHECK-GI-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v0.s[1], w10
-; CHECK-GI-NEXT:    sxtb w10, w2
-; CHECK-GI-NEXT:    mov v3.s[1], w11
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    sxtb w11, w6
-; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    fmov s28, w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #96]
-; CHECK-GI-NEXT:    mov v5.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #128]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    fmov s25, w11
+; CHECK-GI-NEXT:    sxtb w11, w6
+; CHECK-GI-NEXT:    ldr w14, [sp, #528]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v0.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #104]
-; CHECK-GI-NEXT:    mov v3.s[2], w11
+; CHECK-GI-NEXT:    fmov s18, wzr
+; CHECK-GI-NEXT:    fmov s20, wzr
+; CHECK-GI-NEXT:    mov v28.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #104]
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s24, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #112]
+; CHECK-GI-NEXT:    mov v25.s[1], w12
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w11, [sp, #136]
-; CHECK-GI-NEXT:    mov v1.s[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #144]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v5.s[2], w9
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #152]
+; CHECK-GI-NEXT:    ldr w12, [sp, #136]
+; CHECK-GI-NEXT:    mov v18.s[1], wzr
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v0.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #160]
-; CHECK-GI-NEXT:    mov v3.s[3], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #176]
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v1.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #208]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
-; CHECK-GI-NEXT:    mov v5.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #240]
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    sxtb w8, w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #184]
+; CHECK-GI-NEXT:    mov v20.s[1], wzr
+; CHECK-GI-NEXT:    fmov s19, wzr
+; CHECK-GI-NEXT:    mov v28.s[2], w10
+; CHECK-GI-NEXT:    sxtb w10, w3
+; CHECK-GI-NEXT:    mov v24.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #128]
+; CHECK-GI-NEXT:    mov v25.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #168]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v6.s[0], w13
+; CHECK-GI-NEXT:    mov v18.s[2], wzr
+; CHECK-GI-NEXT:    fmov s21, wzr
+; CHECK-GI-NEXT:    sxtb w9, w9
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w13, [sp, #264]
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #216]
+; CHECK-GI-NEXT:    mov v20.s[2], wzr
+; CHECK-GI-NEXT:    mov v28.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #160]
+; CHECK-GI-NEXT:    mov v24.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #120]
+; CHECK-GI-NEXT:    fmov s30, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #144]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v7.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #248]
-; CHECK-GI-NEXT:    mov v2.s[1], w9
+; CHECK-GI-NEXT:    mov v25.s[3], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #200]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w9, [sp, #192]
-; CHECK-GI-NEXT:    mov v6.s[1], w10
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w10, [sp, #200]
-; CHECK-GI-NEXT:    mov v4.s[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #224]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v7.s[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #256]
-; CHECK-GI-NEXT:    mov v2.s[2], w12
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w12, [sp, #232]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v6.s[2], w9
-; CHECK-GI-NEXT:    sxtb w9, w11
-; CHECK-GI-NEXT:    sxtb w11, w14
-; CHECK-GI-NEXT:    mov v4.s[2], w8
-; CHECK-GI-NEXT:    ldr w14, [sp, #280]
-; CHECK-GI-NEXT:    ldr w8, [sp, #272]
-; CHECK-GI-NEXT:    mov v7.s[2], w9
-; CHECK-GI-NEXT:    mov v2.s[3], w11
-; CHECK-GI-NEXT:    sxtb w11, w12
-; CHECK-GI-NEXT:    sxtb w12, w13
-; CHECK-GI-NEXT:    sxtb w13, w14
-; CHECK-GI-NEXT:    ldr w9, [sp, #288]
-; CHECK-GI-NEXT:    mov v6.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #312]
-; CHECK-GI-NEXT:    ldr w14, [sp, #544]
-; CHECK-GI-NEXT:    mov v4.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #344]
-; CHECK-GI-NEXT:    mov v16.s[0], w13
-; CHECK-GI-NEXT:    mov v7.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #376]
+; CHECK-GI-NEXT:    mov v19.s[1], wzr
+; CHECK-GI-NEXT:    fmov s22, w10
+; CHECK-GI-NEXT:    mov v30.s[1], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #176]
+; CHECK-GI-NEXT:    mov v24.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #224]
+; CHECK-GI-NEXT:    ldr w12, [sp, #152]
 ; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v21.s[1], wzr
+; CHECK-GI-NEXT:    mov v22.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #192]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v30.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #232]
+; CHECK-GI-NEXT:    sxtb w12, w12
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    fmov s23, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #240]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w13, [sp, #296]
-; CHECK-GI-NEXT:    mov v19.s[0], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #320]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v17.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #352]
-; CHECK-GI-NEXT:    mov v16.s[1], w9
-; CHECK-GI-NEXT:    mov v21.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #384]
+; CHECK-GI-NEXT:    mov v18.s[3], wzr
+; CHECK-GI-NEXT:    mov v20.s[3], wzr
+; CHECK-GI-NEXT:    mov v22.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #184]
+; CHECK-GI-NEXT:    fmov s26, w11
+; CHECK-GI-NEXT:    mov v23.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #256]
+; CHECK-GI-NEXT:    ldr w11, [sp, #208]
 ; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v30.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #264]
+; CHECK-GI-NEXT:    mov v26.s[1], w13
+; CHECK-GI-NEXT:    sxtb w9, w9
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    ldr w9, [sp, #304]
-; CHECK-GI-NEXT:    mov v19.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #328]
+; CHECK-GI-NEXT:    mov v22.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #296]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    fmov s29, w9
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v17.s[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #360]
-; CHECK-GI-NEXT:    mov v16.s[2], w13
-; CHECK-GI-NEXT:    mov v21.s[1], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #392]
-; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w13, [sp, #216]
+; CHECK-GI-NEXT:    sxtb w9, w10
+; CHECK-GI-NEXT:    mov v23.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #248]
+; CHECK-GI-NEXT:    mov v26.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #304]
+; CHECK-GI-NEXT:    ldr w10, [sp, #272]
+; CHECK-GI-NEXT:    fmov s31, w9
+; CHECK-GI-NEXT:    mov v29.s[1], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #312]
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    ldr w12, [sp, #280]
+; CHECK-GI-NEXT:    fmov s16, wzr
+; CHECK-GI-NEXT:    mov v31.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #328]
+; CHECK-GI-NEXT:    mov v23.s[3], w8
+; CHECK-GI-NEXT:    sxtb w8, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #360]
+; CHECK-GI-NEXT:    mov v29.s[2], w10
+; CHECK-GI-NEXT:    sxtb w10, w11
+; CHECK-GI-NEXT:    mov v26.s[3], w13
 ; CHECK-GI-NEXT:    ldr w13, [sp, #336]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v19.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #368]
+; CHECK-GI-NEXT:    ldr w11, [sp, #368]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v17.s[2], w11
+; CHECK-GI-NEXT:    mov v31.s[2], w8
+; CHECK-GI-NEXT:    fmov s0, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #320]
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    ldr w11, [sp, #400]
-; CHECK-GI-NEXT:    mov v21.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #408]
+; CHECK-GI-NEXT:    fmov s12, w9
+; CHECK-GI-NEXT:    sxtb w11, w11
 ; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v29.s[3], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #376]
+; CHECK-GI-NEXT:    mov v0.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #344]
+; CHECK-GI-NEXT:    ldr w8, [sp, #288]
+; CHECK-GI-NEXT:    mov v12.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #392]
+; CHECK-GI-NEXT:    mov v31.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #424]
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #400]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v16.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #416]
-; CHECK-GI-NEXT:    mov v19.s[3], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #440]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v17.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #472]
 ; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v0.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #432]
+; CHECK-GI-NEXT:    fmov s13, w11
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v18.s[0], w12
-; CHECK-GI-NEXT:    mov v21.s[3], w11
+; CHECK-GI-NEXT:    mov v12.s[2], w9
+; CHECK-GI-NEXT:    fmov s8, w10
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #440]
+; CHECK-GI-NEXT:    ldr w11, [sp, #384]
+; CHECK-GI-NEXT:    ldr w9, [sp, #352]
+; CHECK-GI-NEXT:    fmov s17, wzr
+; CHECK-GI-NEXT:    mov v13.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #408]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w11, [sp, #504]
-; CHECK-GI-NEXT:    ldr w12, [sp, #424]
-; CHECK-GI-NEXT:    mov v22.s[0], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #448]
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v20.s[0], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #480]
+; CHECK-GI-NEXT:    mov v8.s[1], w12
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w12, [sp, #456]
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v18.s[1], w9
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    fmov s3, wzr
+; CHECK-GI-NEXT:    mov v12.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #488]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v23.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #512]
-; CHECK-GI-NEXT:    mov v22.s[1], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #456]
-; CHECK-GI-NEXT:    ldr w9, [sp, #432]
-; CHECK-GI-NEXT:    mov v20.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #488]
+; CHECK-GI-NEXT:    mov v13.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #496]
+; CHECK-GI-NEXT:    mov v0.s[3], w9
+; CHECK-GI-NEXT:    mov v8.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #416]
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #464]
+; CHECK-GI-NEXT:    fmov s14, w12
 ; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v18.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #464]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v23.s[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #520]
-; CHECK-GI-NEXT:    mov v22.s[2], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #496]
+; CHECK-GI-NEXT:    fmov s9, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #504]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v20.s[2], w10
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v18.s[3], w9
-; CHECK-GI-NEXT:    sxtb w9, w13
-; CHECK-GI-NEXT:    ldr w10, [sp, #528]
-; CHECK-GI-NEXT:    mov v23.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #536]
-; CHECK-GI-NEXT:    sxtb w13, w14
-; CHECK-GI-NEXT:    mov v22.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #576]
+; CHECK-GI-NEXT:    ldr w12, [sp, #448]
+; CHECK-GI-NEXT:    mul v27.4s, v25.4s, v0.4s
+; CHECK-GI-NEXT:    mov v13.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #560]
+; CHECK-GI-NEXT:    sxtb w15, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #568]
+; CHECK-GI-NEXT:    mov v9.s[1], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #520]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v20.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #608]
+; CHECK-GI-NEXT:    mov v14.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #472]
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w13, w13
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v24.s[0], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #560]
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mul w8, w8, w11
-; CHECK-GI-NEXT:    mov v23.s[3], w10
+; CHECK-GI-NEXT:    fmov s10, w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #552]
-; CHECK-GI-NEXT:    ldr w11, [sp, #584]
-; CHECK-GI-NEXT:    mov v27.s[0], w12
-; CHECK-GI-NEXT:    mov v26.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #616]
-; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    fmov s15, w13
+; CHECK-GI-NEXT:    mov v8.s[3], w12
+; CHECK-GI-NEXT:    sxtb w12, w14
 ; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v14.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #480]
+; CHECK-GI-NEXT:    mov v10.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #576]
+; CHECK-GI-NEXT:    mov v9.s[2], w15
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mov v15.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #512]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v25.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #640]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w12, [sp, #568]
-; CHECK-GI-NEXT:    mov v24.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #592]
-; CHECK-GI-NEXT:    mov v27.s[1], w11
-; CHECK-GI-NEXT:    mov v26.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #624]
-; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #584]
+; CHECK-GI-NEXT:    ldr w13, [sp, #536]
+; CHECK-GI-NEXT:    mla v27.4s, v28.4s, v31.4s
+; CHECK-GI-NEXT:    mul v30.4s, v30.4s, v13.4s
+; CHECK-GI-NEXT:    mov v10.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #592]
+; CHECK-GI-NEXT:    fmov s25, w8
+; CHECK-GI-NEXT:    mov v14.s[3], w9
+; CHECK-GI-NEXT:    sxtb w9, w12
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w11, [sp, #600]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v28.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #648]
+; CHECK-GI-NEXT:    sxtb w8, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #624]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v9.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #600]
+; CHECK-GI-NEXT:    mla v30.4s, v24.4s, v12.4s
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v10.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #632]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #656]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v24.s[2], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #632]
-; CHECK-GI-NEXT:    mov v27.s[2], w10
-; CHECK-GI-NEXT:    mov v26.s[2], w9
+; CHECK-GI-NEXT:    fmov s28, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #688]
+; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #656]
+; CHECK-GI-NEXT:    mov v15.s[2], w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #544]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #664]
-; CHECK-GI-NEXT:    mov v28.s[1], w8
+; CHECK-GI-NEXT:    mov v28.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #696]
+; CHECK-GI-NEXT:    fmov s11, w8
+; CHECK-GI-NEXT:    fmov s31, w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w12, w13
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w8, [sp, #680]
-; CHECK-GI-NEXT:    mov v24.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #672]
-; CHECK-GI-NEXT:    mov v27.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #704]
-; CHECK-GI-NEXT:    mov v26.s[3], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #736]
+; CHECK-GI-NEXT:    ldr w11, [sp, #672]
+; CHECK-GI-NEXT:    ldr w8, [sp, #616]
+; CHECK-GI-NEXT:    mov v11.s[1], w9
+; CHECK-GI-NEXT:    mov v15.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #608]
+; CHECK-GI-NEXT:    mov v31.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #704]
+; CHECK-GI-NEXT:    ldr w9, [sp, #640]
+; CHECK-GI-NEXT:    sxtb w11, w11
 ; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mul v24.4s, v26.4s, v14.4s
+; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w9, w9
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v28.s[2], w10
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v29.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #688]
-; CHECK-GI-NEXT:    ldr w10, [sp, #696]
-; CHECK-GI-NEXT:    mov v31.s[0], w11
+; CHECK-GI-NEXT:    mov v11.s[2], w11
 ; CHECK-GI-NEXT:    ldr w11, [sp, #712]
-; CHECK-GI-NEXT:    mov v30.s[0], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #744]
+; CHECK-GI-NEXT:    mov v0.s[2], w12
+; CHECK-GI-NEXT:    mov v31.s[2], w10
+; CHECK-GI-NEXT:    ldr w12, [sp, #648]
+; CHECK-GI-NEXT:    mov v28.s[2], w9
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #720]
+; CHECK-GI-NEXT:    ldr w9, [sp, #680]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    ldr w14, [sp, #776]
+; CHECK-GI-NEXT:    mul v26.4s, v29.4s, v15.4s
+; CHECK-GI-NEXT:    mla v24.4s, v22.4s, v8.4s
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v31.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #784]
+; CHECK-GI-NEXT:    mov v28.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #752]
+; CHECK-GI-NEXT:    fmov s13, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #792]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v28.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #768]
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v29.s[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #720]
-; CHECK-GI-NEXT:    mov v31.s[1], w11
+; CHECK-GI-NEXT:    mov v11.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #760]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w8, [sp, #728]
+; CHECK-GI-NEXT:    fmov s14, w11
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w11, [sp, #728]
-; CHECK-GI-NEXT:    mov v30.s[1], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #752]
+; CHECK-GI-NEXT:    ldr w11, [sp, #744]
+; CHECK-GI-NEXT:    fmov s12, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #824]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v8.s[0], w9
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mul v3.4s, v3.4s, v19.4s
-; CHECK-GI-NEXT:    sxtb w9, w13
-; CHECK-GI-NEXT:    mov v29.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #760]
-; CHECK-GI-NEXT:    mov v31.s[2], w8
-; CHECK-GI-NEXT:    sxtb w8, w10
-; CHECK-GI-NEXT:    sxtb w10, w14
-; CHECK-GI-NEXT:    mov v30.s[2], w9
-; CHECK-GI-NEXT:    ldr w14, [sp, #808]
-; CHECK-GI-NEXT:    ldr w13, [sp, #784]
-; CHECK-GI-NEXT:    mov v8.s[1], w10
+; CHECK-GI-NEXT:    mla v26.4s, v23.4s, v9.4s
+; CHECK-GI-NEXT:    ldr w13, [sp, #984]
+; CHECK-GI-NEXT:    mov v14.s[1], w10
 ; CHECK-GI-NEXT:    sxtb w10, w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #792]
-; CHECK-GI-NEXT:    sxtb w12, w14
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mul v5.4s, v5.4s, v21.4s
-; CHECK-GI-NEXT:    mov v31.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #840]
+; CHECK-GI-NEXT:    mov v13.s[1], w8
+; CHECK-GI-NEXT:    mov v12.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #832]
+; CHECK-GI-NEXT:    ldr w8, [sp, #736]
+; CHECK-GI-NEXT:    fmov s29, w10
+; CHECK-GI-NEXT:    ldr w12, [sp, #768]
+; CHECK-GI-NEXT:    ldr w10, [sp, #800]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v30.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #872]
-; CHECK-GI-NEXT:    mov v9.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #816]
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v8.s[2], w13
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w13, [sp, #824]
-; CHECK-GI-NEXT:    mov v21.s[0], wzr
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    fmov s6, wzr
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v11.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #848]
-; CHECK-GI-NEXT:    mov v10.s[0], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #880]
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v9.s[1], w12
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v8.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #904]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w12, [sp, #832]
-; CHECK-GI-NEXT:    mov v11.s[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #856]
-; CHECK-GI-NEXT:    mov v29.s[3], w8
-; CHECK-GI-NEXT:    mov v10.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #888]
+; CHECK-GI-NEXT:    fmov s2, wzr
+; CHECK-GI-NEXT:    mov v29.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #840]
+; CHECK-GI-NEXT:    mov v13.s[2], w8
+; CHECK-GI-NEXT:    mov v12.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #808]
+; CHECK-GI-NEXT:    mov v14.s[2], w10
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v9.s[2], w13
+; CHECK-GI-NEXT:    ldr w8, [sp, #776]
+; CHECK-GI-NEXT:    ldr w10, [sp, #848]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    fmov s5, wzr
+; CHECK-GI-NEXT:    fmov s4, wzr
+; CHECK-GI-NEXT:    mov v29.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #856]
+; CHECK-GI-NEXT:    mov v13.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #864]
+; CHECK-GI-NEXT:    mov v14.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #888]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w11, w11
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v12.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #912]
+; CHECK-GI-NEXT:    fmov s7, wzr
+; CHECK-GI-NEXT:    fmov s15, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #920]
+; CHECK-GI-NEXT:    mov v12.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #872]
+; CHECK-GI-NEXT:    mov v29.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #896]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    fmov s22, w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #928]
+; CHECK-GI-NEXT:    mov v15.s[1], w11
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v11.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #896]
-; CHECK-GI-NEXT:    ldr w13, [sp, #864]
-; CHECK-GI-NEXT:    mov v10.s[2], w10
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    fmov s8, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #952]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    ldr w11, [sp, #904]
+; CHECK-GI-NEXT:    mov v22.s[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #936]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #920]
-; CHECK-GI-NEXT:    mov v9.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #968]
+; CHECK-GI-NEXT:    mov v19.s[2], wzr
+; CHECK-GI-NEXT:    mov v21.s[2], wzr
+; CHECK-GI-NEXT:    mov v15.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #960]
+; CHECK-GI-NEXT:    mov v8.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #880]
+; CHECK-GI-NEXT:    fmov s23, w9
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v12.s[1], w9
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    ldr w9, [sp, #944]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #800]
-; CHECK-GI-NEXT:    ldr w9, [sp, #928]
-; CHECK-GI-NEXT:    mov v10.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1032]
-; CHECK-GI-NEXT:    mov v11.s[3], w13
-; CHECK-GI-NEXT:    mov v14.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #976]
-; CHECK-GI-NEXT:    ldr w13, [sp, #936]
-; CHECK-GI-NEXT:    mov v12.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1000]
+; CHECK-GI-NEXT:    mov v22.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #912]
+; CHECK-GI-NEXT:    mov v8.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #968]
+; CHECK-GI-NEXT:    mov v23.s[1], w8
+; CHECK-GI-NEXT:    mov v15.s[3], w12
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    add v18.4s, v18.4s, v20.4s
+; CHECK-GI-NEXT:    mov v22.s[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #992]
+; CHECK-GI-NEXT:    fmov s9, w12
+; CHECK-GI-NEXT:    mov v23.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1048]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1056]
+; CHECK-GI-NEXT:    mul v0.4s, v0.4s, v15.4s
+; CHECK-GI-NEXT:    sxtb w13, w11
+; CHECK-GI-NEXT:    mov v8.s[3], w9
+; CHECK-GI-NEXT:    sxtb w11, w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #1000]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v21.s[1], wzr
-; CHECK-GI-NEXT:    mov v15.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1040]
+; CHECK-GI-NEXT:    mov v9.s[1], w13
+; CHECK-GI-NEXT:    ldr w10, [sp, #1016]
+; CHECK-GI-NEXT:    ldr w8, [sp, #816]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    mov v16.s[1], wzr
+; CHECK-GI-NEXT:    mla v0.4s, v10.4s, v29.4s
+; CHECK-GI-NEXT:    fmov s10, w11
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v14.s[1], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #984]
-; CHECK-GI-NEXT:    mov v13.s[0], w13
-; CHECK-GI-NEXT:    mov v19.s[0], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1008]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1024]
+; CHECK-GI-NEXT:    mul v20.4s, v11.4s, v8.4s
+; CHECK-GI-NEXT:    ldr q8, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v9.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1008]
+; CHECK-GI-NEXT:    fmov s29, w10
+; CHECK-GI-NEXT:    mov v10.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #1064]
 ; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v17.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.s[1], wzr
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    ldr w13, [sp, #944]
+; CHECK-GI-NEXT:    mov v6.s[1], wzr
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    mov v5.s[1], wzr
+; CHECK-GI-NEXT:    mov v4.s[1], wzr
+; CHECK-GI-NEXT:    mov v7.s[1], wzr
+; CHECK-GI-NEXT:    mov v10.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #1080]
+; CHECK-GI-NEXT:    mov v8.s[1], wzr
+; CHECK-GI-NEXT:    mov v9.s[3], w9
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v15.s[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1048]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1032]
+; CHECK-GI-NEXT:    sxtb w9, w12
+; CHECK-GI-NEXT:    mov v29.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1072]
+; CHECK-GI-NEXT:    mov v19.s[3], wzr
+; CHECK-GI-NEXT:    mov v21.s[3], wzr
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    mul w8, w8, w9
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v14.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #1064]
-; CHECK-GI-NEXT:    sxtb w13, w13
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v19.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #992]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v13.s[1], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #952]
-; CHECK-GI-NEXT:    mov v15.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
+; CHECK-GI-NEXT:    mov v16.s[2], wzr
+; CHECK-GI-NEXT:    mov v17.s[2], wzr
+; CHECK-GI-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-NEXT:    mov v6.s[2], wzr
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v5.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[2], wzr
+; CHECK-GI-NEXT:    mov v7.s[2], wzr
+; CHECK-GI-NEXT:    mov v8.s[2], wzr
+; CHECK-GI-NEXT:    mov v29.s[2], w10
+; CHECK-GI-NEXT:    mov v10.s[3], w11
+; CHECK-GI-NEXT:    add v19.4s, v19.4s, v21.4s
+; CHECK-GI-NEXT:    ldr w9, [sp, #976]
+; CHECK-GI-NEXT:    fmov s21, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1040]
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    mov v16.s[3], wzr
+; CHECK-GI-NEXT:    mov v17.s[3], wzr
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mul w8, w8, w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #1056]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    sxtb w13, w13
-; CHECK-GI-NEXT:    mov v12.s[3], w9
-; CHECK-GI-NEXT:    sxtb w9, w11
-; CHECK-GI-NEXT:    mov v14.s[3], w10
-; CHECK-GI-NEXT:    mov v21.s[2], wzr
-; CHECK-GI-NEXT:    sxtb w10, w12
-; CHECK-GI-NEXT:    mul v6.4s, v6.4s, v22.4s
-; CHECK-GI-NEXT:    mov v22.s[0], w8
-; CHECK-GI-NEXT:    mov v13.s[2], w13
-; CHECK-GI-NEXT:    mov v19.s[2], w9
-; CHECK-GI-NEXT:    ldr w8, [sp, #960]
-; CHECK-GI-NEXT:    mov v15.s[3], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #1024]
-; CHECK-GI-NEXT:    mov v25.s[1], wzr
-; CHECK-GI-NEXT:    mul v7.4s, v7.4s, v23.4s
-; CHECK-GI-NEXT:    mov v21.s[3], wzr
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mul v23.4s, v27.4s, v11.4s
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v22.s[1], wzr
-; CHECK-GI-NEXT:    mul v27.4s, v28.4s, v12.4s
-; CHECK-GI-NEXT:    mul v28.4s, v31.4s, v14.4s
-; CHECK-GI-NEXT:    mul v31.4s, v8.4s, v15.4s
-; CHECK-GI-NEXT:    mov v13.s[3], w8
-; CHECK-GI-NEXT:    mov v19.s[3], w9
-; CHECK-GI-NEXT:    mla v3.4s, v0.4s, v16.4s
+; CHECK-GI-NEXT:    mov v11.16b, v8.16b
+; CHECK-GI-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-NEXT:    mov v6.s[3], wzr
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    mov v5.s[3], wzr
+; CHECK-GI-NEXT:    mov v4.s[3], wzr
+; CHECK-GI-NEXT:    mov v7.s[3], wzr
+; CHECK-GI-NEXT:    mov v25.s[1], wzr
+; CHECK-GI-NEXT:    mov v21.s[1], wzr
+; CHECK-GI-NEXT:    mul v8.4s, v13.4s, v9.4s
+; CHECK-GI-NEXT:    mul v9.4s, v14.4s, v10.4s
+; CHECK-GI-NEXT:    mov v23.s[3], w9
+; CHECK-GI-NEXT:    mov v29.s[3], w8
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    mov v11.s[3], wzr
+; CHECK-GI-NEXT:    add v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT:    add v3.4s, v3.4s, v6.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    add v4.4s, v4.4s, v7.4s
 ; CHECK-GI-NEXT:    mov v25.s[2], wzr
-; CHECK-GI-NEXT:    add v0.4s, v21.4s, v21.4s
-; CHECK-GI-NEXT:    mla v5.4s, v1.4s, v17.4s
-; CHECK-GI-NEXT:    mla v6.4s, v2.4s, v18.4s
-; CHECK-GI-NEXT:    mov v22.s[2], wzr
-; CHECK-GI-NEXT:    mla v7.4s, v4.4s, v20.4s
-; CHECK-GI-NEXT:    mla v23.4s, v24.4s, v9.4s
-; CHECK-GI-NEXT:    mla v27.4s, v26.4s, v10.4s
-; CHECK-GI-NEXT:    mla v28.4s, v29.4s, v13.4s
-; CHECK-GI-NEXT:    mla v31.4s, v30.4s, v19.4s
-; CHECK-GI-NEXT:    add v1.4s, v21.4s, v0.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    mov v21.s[2], wzr
+; CHECK-GI-NEXT:    mla v20.4s, v28.4s, v22.4s
+; CHECK-GI-NEXT:    mla v8.4s, v31.4s, v23.4s
+; CHECK-GI-NEXT:    mla v9.4s, v12.4s, v29.4s
+; CHECK-GI-NEXT:    add v5.4s, v19.4s, v16.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT:    add v3.4s, v11.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    add v4.4s, v27.4s, v30.4s
+; CHECK-GI-NEXT:    add v6.4s, v24.4s, v26.4s
+; CHECK-GI-NEXT:    ldr x29, [sp, #80] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v25.s[3], wzr
-; CHECK-GI-NEXT:    add v2.4s, v3.4s, v5.4s
-; CHECK-GI-NEXT:    mov v22.s[3], wzr
-; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v23.4s, v27.4s
-; CHECK-GI-NEXT:    add v4.4s, v28.4s, v31.4s
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    add v3.4s, v25.4s, v0.4s
-; CHECK-GI-NEXT:    add v0.4s, v22.4s, v0.4s
-; CHECK-GI-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-GI-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    addv s1, v2.4s
+; CHECK-GI-NEXT:    mov v21.s[3], wzr
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v20.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v5.4s
+; CHECK-GI-NEXT:    add v5.4s, v8.4s, v9.4s
+; CHECK-GI-NEXT:    add v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT:    add v3.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add v1.4s, v25.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v5.4s
+; CHECK-GI-NEXT:    add v2.4s, v21.4s, v2.4s
+; CHECK-GI-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
 ; CHECK-GI-NEXT:    addv s0, v0.4s
 ; CHECK-GI-NEXT:    fmov w8, s1
 ; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    add w0, w8, w9
-; CHECK-GI-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #96
 ; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <33 x i8> %a to <33 x i32>
@@ -5350,238 +5495,309 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %
 ;
 ; CHECK-GI-LABEL: test_sdot_v33i8_double_nomla:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    stp d13, d12, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    str x29, [sp, #48] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
-; CHECK-GI-NEXT:    sxtb w10, w0
-; CHECK-GI-NEXT:    ldr w9, [sp, #16]
-; CHECK-GI-NEXT:    ldr w8, [sp, #48]
-; CHECK-GI-NEXT:    sxtb w11, w4
-; CHECK-GI-NEXT:    sxtb w12, w6
-; CHECK-GI-NEXT:    ldr w13, [sp, #592]
-; CHECK-GI-NEXT:    mov v0.s[0], w10
-; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    .cfi_offset b8, -24
+; CHECK-GI-NEXT:    .cfi_offset b9, -32
+; CHECK-GI-NEXT:    .cfi_offset b10, -40
+; CHECK-GI-NEXT:    .cfi_offset b11, -48
+; CHECK-GI-NEXT:    .cfi_offset b12, -56
+; CHECK-GI-NEXT:    .cfi_offset b13, -64
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    sxtb w9, w1
+; CHECK-GI-NEXT:    sxtb w12, w4
+; CHECK-GI-NEXT:    ldr w10, [sp, #72]
+; CHECK-GI-NEXT:    ldr w11, [sp, #96]
+; CHECK-GI-NEXT:    sxtb w13, w5
+; CHECK-GI-NEXT:    fmov s22, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #64]
+; CHECK-GI-NEXT:    fmov s23, w12
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w12, w7
+; CHECK-GI-NEXT:    fmov s18, wzr
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v1.s[0], w11
-; CHECK-GI-NEXT:    sxtb w10, w1
-; CHECK-GI-NEXT:    sxtb w11, w5
-; CHECK-GI-NEXT:    mov v2.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #24]
-; CHECK-GI-NEXT:    mov v3.s[0], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #56]
-; CHECK-GI-NEXT:    mov v22.s[0], wzr
-; CHECK-GI-NEXT:    mov v0.s[1], w10
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w10, [sp, #80]
+; CHECK-GI-NEXT:    fmov s19, wzr
+; CHECK-GI-NEXT:    fmov s20, wzr
+; CHECK-GI-NEXT:    mov v22.s[1], w9
+; CHECK-GI-NEXT:    sxtb w9, w2
+; CHECK-GI-NEXT:    mov v23.s[1], w13
+; CHECK-GI-NEXT:    fmov s24, w8
+; CHECK-GI-NEXT:    sxtb w8, w11
+; CHECK-GI-NEXT:    sxtb w11, w6
+; CHECK-GI-NEXT:    ldr w13, [sp, #232]
+; CHECK-GI-NEXT:    mov v18.s[1], wzr
+; CHECK-GI-NEXT:    mov v19.s[1], wzr
+; CHECK-GI-NEXT:    fmov s25, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #80]
+; CHECK-GI-NEXT:    fmov s21, wzr
+; CHECK-GI-NEXT:    mov v22.s[2], w9
+; CHECK-GI-NEXT:    mov v24.s[1], w10
+; CHECK-GI-NEXT:    sxtb w10, w3
+; CHECK-GI-NEXT:    ldr w9, [sp, #104]
+; CHECK-GI-NEXT:    mov v23.s[2], w11
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v1.s[1], w11
-; CHECK-GI-NEXT:    sxtb w11, w2
-; CHECK-GI-NEXT:    mov v2.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #32]
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v3.s[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #64]
-; CHECK-GI-NEXT:    mov v22.s[1], wzr
-; CHECK-GI-NEXT:    mov v0.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #136]
+; CHECK-GI-NEXT:    mov v18.s[2], wzr
+; CHECK-GI-NEXT:    mov v19.s[2], wzr
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    sxtb w11, w3
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v1.s[2], w12
-; CHECK-GI-NEXT:    mov v4.s[0], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #40]
-; CHECK-GI-NEXT:    mov v2.s[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #88]
-; CHECK-GI-NEXT:    sxtb w12, w7
-; CHECK-GI-NEXT:    mov v3.s[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #72]
-; CHECK-GI-NEXT:    mov v0.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #112]
+; CHECK-GI-NEXT:    fmov s6, wzr
+; CHECK-GI-NEXT:    fmov s7, wzr
+; CHECK-GI-NEXT:    mov v22.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #128]
+; CHECK-GI-NEXT:    mov v24.s[2], w8
+; CHECK-GI-NEXT:    mov v25.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #112]
+; CHECK-GI-NEXT:    ldr w8, [sp, #88]
 ; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    mov v23.s[3], w12
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v1.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #144]
+; CHECK-GI-NEXT:    ldr w12, [sp, #160]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v2.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #96]
-; CHECK-GI-NEXT:    mov v4.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #176]
+; CHECK-GI-NEXT:    fmov s26, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #144]
+; CHECK-GI-NEXT:    mov v18.s[3], wzr
+; CHECK-GI-NEXT:    mov v25.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #120]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v3.s[3], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #120]
-; CHECK-GI-NEXT:    mov v5.s[0], w11
+; CHECK-GI-NEXT:    mov v24.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #168]
 ; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v26.s[1], w11
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    ldr w11, [sp, #152]
-; CHECK-GI-NEXT:    mov v6.s[0], w12
+; CHECK-GI-NEXT:    fmov s27, w12
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w12, [sp, #104]
-; CHECK-GI-NEXT:    mov v4.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #184]
-; CHECK-GI-NEXT:    mov v7.s[0], w9
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #128]
-; CHECK-GI-NEXT:    mov v5.s[1], w8
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w8, [sp, #160]
-; CHECK-GI-NEXT:    mov v6.s[1], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #480]
+; CHECK-GI-NEXT:    ldr w12, [sp, #224]
+; CHECK-GI-NEXT:    ldr w11, [sp, #152]
+; CHECK-GI-NEXT:    mov v25.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #192]
+; CHECK-GI-NEXT:    add v22.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT:    mov v27.s[1], w8
+; CHECK-GI-NEXT:    mov v19.s[3], wzr
+; CHECK-GI-NEXT:    fmov s5, wzr
+; CHECK-GI-NEXT:    mov v26.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #200]
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v4.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #192]
-; CHECK-GI-NEXT:    mov v7.s[1], w10
+; CHECK-GI-NEXT:    fmov s16, wzr
+; CHECK-GI-NEXT:    fmov s17, wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    sxtb w8, w10
+; CHECK-GI-NEXT:    sxtb w10, w12
+; CHECK-GI-NEXT:    fmov s28, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #176]
+; CHECK-GI-NEXT:    sxtb w12, w13
+; CHECK-GI-NEXT:    ldr w13, [sp, #208]
+; CHECK-GI-NEXT:    fmov s29, w10
+; CHECK-GI-NEXT:    sxtb w10, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #536]
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v28.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #184]
+; CHECK-GI-NEXT:    mov v26.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #528]
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    mov v29.s[1], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #240]
+; CHECK-GI-NEXT:    mov v27.s[2], w9
+; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #136]
-; CHECK-GI-NEXT:    mov v5.s[2], w9
-; CHECK-GI-NEXT:    sxtb w9, w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #216]
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    ldr w11, [sp, #168]
-; CHECK-GI-NEXT:    mov v6.s[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #488]
-; CHECK-GI-NEXT:    mov v16.s[0], w9
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #200]
-; CHECK-GI-NEXT:    mov v7.s[2], w12
+; CHECK-GI-NEXT:    mov v28.s[2], w13
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w12, [sp, #512]
-; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v5.s[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #544]
+; CHECK-GI-NEXT:    fmov s30, w10
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v6.s[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #496]
+; CHECK-GI-NEXT:    ldr w10, [sp, #256]
+; CHECK-GI-NEXT:    mov v29.s[2], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #248]
+; CHECK-GI-NEXT:    mov v27.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #560]
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    add v23.4s, v24.4s, v25.4s
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v16.s[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #208]
-; CHECK-GI-NEXT:    mov v7.s[3], w9
-; CHECK-GI-NEXT:    sxtb w9, w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #520]
+; CHECK-GI-NEXT:    mov v30.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #544]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v28.s[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #568]
+; CHECK-GI-NEXT:    mov v29.s[3], w12
+; CHECK-GI-NEXT:    ldr w12, [sp, #592]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v17.s[0], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #552]
+; CHECK-GI-NEXT:    fmov s31, w8
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #552]
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v30.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #600]
+; CHECK-GI-NEXT:    fmov s8, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #624]
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v19.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #504]
+; CHECK-GI-NEXT:    mov v31.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #576]
+; CHECK-GI-NEXT:    fmov s9, w12
+; CHECK-GI-NEXT:    sxtb w11, w11
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v16.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #576]
-; CHECK-GI-NEXT:    mov v18.s[0], w8
-; CHECK-GI-NEXT:    sxtb w8, w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #528]
+; CHECK-GI-NEXT:    fmov s1, wzr
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v17.s[1], w10
+; CHECK-GI-NEXT:    mov v30.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #632]
+; CHECK-GI-NEXT:    mov v9.s[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #608]
+; CHECK-GI-NEXT:    fmov s10, w10
+; CHECK-GI-NEXT:    mov v31.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #656]
+; CHECK-GI-NEXT:    ldr w10, [sp, #688]
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    ldr w10, [sp, #560]
-; CHECK-GI-NEXT:    mov v19.s[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #608]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v16.s[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #584]
-; CHECK-GI-NEXT:    mov v20.s[0], w11
-; CHECK-GI-NEXT:    sxtb w10, w10
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    ldr w11, [sp, #536]
-; CHECK-GI-NEXT:    mov v17.s[2], w12
+; CHECK-GI-NEXT:    fmov s3, wzr
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    mov v19.s[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #616]
-; CHECK-GI-NEXT:    mov v21.s[0], w8
-; CHECK-GI-NEXT:    ldr w12, [sp, #568]
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v22.s[2], wzr
-; CHECK-GI-NEXT:    mov v20.s[1], w9
-; CHECK-GI-NEXT:    sxtb w9, w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #640]
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v17.s[3], w11
-; CHECK-GI-NEXT:    sxtb w11, w13
-; CHECK-GI-NEXT:    mov v21.s[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #672]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v19.s[3], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #624]
-; CHECK-GI-NEXT:    ldr w8, [sp, #600]
-; CHECK-GI-NEXT:    mov v20.s[2], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #704]
-; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v23.s[0], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #648]
+; CHECK-GI-NEXT:    fmov s2, wzr
+; CHECK-GI-NEXT:    mov v9.s[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #664]
+; CHECK-GI-NEXT:    mov v10.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #696]
+; CHECK-GI-NEXT:    fmov s11, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #640]
+; CHECK-GI-NEXT:    fmov s12, w10
 ; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v24.s[0], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #736]
-; CHECK-GI-NEXT:    mov v21.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #680]
+; CHECK-GI-NEXT:    ldr w10, [sp, #672]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    fmov s4, wzr
+; CHECK-GI-NEXT:    mov v11.s[1], w11
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v25.s[0], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #712]
+; CHECK-GI-NEXT:    mov v20.s[1], wzr
+; CHECK-GI-NEXT:    mov v12.s[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #704]
+; CHECK-GI-NEXT:    mov v10.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #648]
+; CHECK-GI-NEXT:    mov v21.s[1], wzr
+; CHECK-GI-NEXT:    mov v6.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v7.s[1], wzr
+; CHECK-GI-NEXT:    mov v5.s[1], wzr
 ; CHECK-GI-NEXT:    sxtb w9, w9
-; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    mov v23.s[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #656]
-; CHECK-GI-NEXT:    sxtb w11, w11
-; CHECK-GI-NEXT:    mov v26.s[0], w9
+; CHECK-GI-NEXT:    mov v11.s[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #680]
+; CHECK-GI-NEXT:    mov v12.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #712]
+; CHECK-GI-NEXT:    mov v16.s[1], wzr
+; CHECK-GI-NEXT:    mov v10.s[3], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #720]
-; CHECK-GI-NEXT:    mov v24.s[1], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #688]
 ; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    mov v25.s[1], w11
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v17.s[1], wzr
+; CHECK-GI-NEXT:    add v19.4s, v18.4s, v19.4s
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v11.s[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #728]
+; CHECK-GI-NEXT:    mov v12.s[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #752]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    fmov s13, w9
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #760]
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    mov v3.s[1], wzr
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-NEXT:    mov v4.s[1], wzr
+; CHECK-GI-NEXT:    mov v13.s[1], w10
+; CHECK-GI-NEXT:    fmov s24, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #736]
+; CHECK-GI-NEXT:    mov v20.s[2], wzr
+; CHECK-GI-NEXT:    mov v21.s[2], wzr
+; CHECK-GI-NEXT:    mov v6.s[2], wzr
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    mov v7.s[2], wzr
+; CHECK-GI-NEXT:    mov v8.s[1], wzr
+; CHECK-GI-NEXT:    mov v24.s[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #768]
+; CHECK-GI-NEXT:    ldr w12, [sp, #584]
+; CHECK-GI-NEXT:    mov v13.s[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #784]
+; CHECK-GI-NEXT:    ldr w11, [sp, #616]
 ; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v5.s[2], wzr
+; CHECK-GI-NEXT:    mov v16.s[2], wzr
+; CHECK-GI-NEXT:    sxtb w10, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #744]
+; CHECK-GI-NEXT:    mov v17.s[2], wzr
+; CHECK-GI-NEXT:    mov v24.s[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #776]
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    fmov s18, w10
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-NEXT:    mov v4.s[2], wzr
+; CHECK-GI-NEXT:    mov v20.s[3], wzr
+; CHECK-GI-NEXT:    mov v21.s[3], wzr
+; CHECK-GI-NEXT:    mov v6.s[3], wzr
+; CHECK-GI-NEXT:    mov v7.s[3], wzr
 ; CHECK-GI-NEXT:    mov v18.s[1], wzr
 ; CHECK-GI-NEXT:    sxtb w12, w12
-; CHECK-GI-NEXT:    ldr w11, [sp, #632]
-; CHECK-GI-NEXT:    mov v23.s[2], w10
-; CHECK-GI-NEXT:    mov v26.s[1], wzr
-; CHECK-GI-NEXT:    ldr w10, [sp, #664]
+; CHECK-GI-NEXT:    sxtb w11, w11
 ; CHECK-GI-NEXT:    sxtb w8, w8
-; CHECK-GI-NEXT:    mov v24.s[2], w12
-; CHECK-GI-NEXT:    ldr w12, [sp, #696]
-; CHECK-GI-NEXT:    mov v22.s[3], wzr
-; CHECK-GI-NEXT:    mov v25.s[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #728]
-; CHECK-GI-NEXT:    mov v20.s[3], w8
-; CHECK-GI-NEXT:    sxtb w8, w11
-; CHECK-GI-NEXT:    sxtb w10, w10
-; CHECK-GI-NEXT:    sxtb w11, w12
 ; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov v8.s[2], wzr
+; CHECK-GI-NEXT:    mov v31.s[3], w12
+; CHECK-GI-NEXT:    mov v9.s[3], w11
+; CHECK-GI-NEXT:    mov v5.s[3], wzr
+; CHECK-GI-NEXT:    mov v16.s[3], wzr
+; CHECK-GI-NEXT:    mov v17.s[3], wzr
+; CHECK-GI-NEXT:    mov v13.s[3], w8
 ; CHECK-GI-NEXT:    mov v18.s[2], wzr
-; CHECK-GI-NEXT:    mov v26.s[2], wzr
-; CHECK-GI-NEXT:    mov v21.s[3], w8
-; CHECK-GI-NEXT:    mov v23.s[3], w10
-; CHECK-GI-NEXT:    mov v24.s[3], w11
-; CHECK-GI-NEXT:    mov v25.s[3], w9
-; CHECK-GI-NEXT:    add v27.4s, v22.4s, v22.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    mov v24.s[3], w9
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-NEXT:    mov v4.s[3], wzr
+; CHECK-GI-NEXT:    add v20.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT:    add v6.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT:    mov v8.s[3], wzr
+; CHECK-GI-NEXT:    add v25.4s, v26.4s, v27.4s
+; CHECK-GI-NEXT:    add v26.4s, v28.4s, v29.4s
 ; CHECK-GI-NEXT:    mov v18.s[3], wzr
-; CHECK-GI-NEXT:    mov v26.s[3], wzr
-; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    add v5.4s, v22.4s, v27.4s
-; CHECK-GI-NEXT:    add v6.4s, v19.4s, v20.4s
-; CHECK-GI-NEXT:    add v7.4s, v21.4s, v23.4s
-; CHECK-GI-NEXT:    add v16.4s, v24.4s, v25.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v3.4s, v27.4s, v27.4s
-; CHECK-GI-NEXT:    add v2.4s, v18.4s, v5.4s
-; CHECK-GI-NEXT:    add v4.4s, v4.4s, v6.4s
-; CHECK-GI-NEXT:    add v5.4s, v26.4s, v5.4s
-; CHECK-GI-NEXT:    add v6.4s, v7.4s, v16.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    add v3.4s, v5.4s, v3.4s
-; CHECK-GI-NEXT:    add v2.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT:    add v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT:    add v7.4s, v30.4s, v31.4s
+; CHECK-GI-NEXT:    add v5.4s, v5.4s, v20.4s
+; CHECK-GI-NEXT:    add v17.4s, v9.4s, v10.4s
+; CHECK-GI-NEXT:    add v20.4s, v11.4s, v12.4s
+; CHECK-GI-NEXT:    add v21.4s, v13.4s, v24.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v6.4s
+; CHECK-GI-NEXT:    add v3.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT:    add v4.4s, v25.4s, v26.4s
+; CHECK-GI-NEXT:    add v5.4s, v8.4s, v5.4s
+; CHECK-GI-NEXT:    add v6.4s, v19.4s, v16.4s
+; CHECK-GI-NEXT:    add v7.4s, v7.4s, v17.4s
+; CHECK-GI-NEXT:    add v16.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ldr x29, [sp, #48] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    add v0.4s, v18.4s, v0.4s
+; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    add v0.4s, v4.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    add w0, w8, w9
-; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d13, d12, [sp], #64 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <33 x i8> %a to <33 x i32>
@@ -5610,15 +5826,15 @@ define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_udot_v48i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
 ; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-NEXT:    ldr q7, [x0, #32]
 ; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-GI-NEXT:    ldr q17, [x1, #32]
 ; CHECK-GI-NEXT:    ldp q4, q5, [x0]
-; CHECK-GI-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-NEXT:    udot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-NEXT:    udot v1.4s, v6.16b, v4.16b
 ; CHECK-GI-NEXT:    udot v3.4s, v16.16b, v5.16b
@@ -5658,7 +5874,7 @@ define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
 ;
 ; CHECK-GI-LABEL: test_udot_v48i8_nomla:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
 ; CHECK-GI-NEXT:    movi v1.16b, #1
 ; CHECK-GI-NEXT:    ldr q7, [x0, #32]
 ; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
@@ -5701,15 +5917,15 @@ define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ;
 ; CHECK-GI-LABEL: test_sdot_v48i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-NEXT:    fmov s0, wzr
 ; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-NEXT:    ldr q7, [x0, #32]
 ; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-GI-NEXT:    ldr q17, [x1, #32]
 ; CHECK-GI-NEXT:    ldp q4, q5, [x0]
-; CHECK-GI-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-NEXT:    sdot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-NEXT:    sdot v1.4s, v6.16b, v4.16b
 ; CHECK-GI-NEXT:    sdot v3.4s, v16.16b, v5.16b
@@ -6123,408 +6339,412 @@ define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48
 ; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
-; CHECK-GI-NEXT:    ldr w11, [sp, #80]
-; CHECK-GI-NEXT:    ldr w8, [sp, #208]
-; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    ldr w12, [sp, #336]
+; CHECK-GI-NEXT:    ldr w10, [sp, #80]
 ; CHECK-GI-NEXT:    ldr w9, [sp, #88]
-; CHECK-GI-NEXT:    mov v20.s[0], wzr
-; CHECK-GI-NEXT:    fmov s1, w11
-; CHECK-GI-NEXT:    fmov s2, w8
-; CHECK-GI-NEXT:    ldr w11, [sp, #464]
-; CHECK-GI-NEXT:    ldr w8, [sp, #592]
-; CHECK-GI-NEXT:    ldr w10, [sp, #216]
-; CHECK-GI-NEXT:    fmov s3, w12
+; CHECK-GI-NEXT:    fmov s2, w0
+; CHECK-GI-NEXT:    ldr w11, [sp, #208]
+; CHECK-GI-NEXT:    ldr w8, [sp, #216]
+; CHECK-GI-NEXT:    fmov s1, wzr
+; CHECK-GI-NEXT:    fmov s3, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #336]
+; CHECK-GI-NEXT:    ldr w12, [sp, #720]
 ; CHECK-GI-NEXT:    fmov s4, w11
-; CHECK-GI-NEXT:    mov v0.b[1], w1
-; CHECK-GI-NEXT:    ldr w11, [sp, #600]
-; CHECK-GI-NEXT:    fmov s5, w8
-; CHECK-GI-NEXT:    mov v1.b[1], w9
-; CHECK-GI-NEXT:    mov v2.b[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #344]
-; CHECK-GI-NEXT:    ldr w9, [sp, #472]
-; CHECK-GI-NEXT:    ldr w8, [sp, #96]
-; CHECK-GI-NEXT:    ldr w12, [sp, #848]
-; CHECK-GI-NEXT:    ldr w13, [sp, #728]
-; CHECK-GI-NEXT:    mov v20.s[1], wzr
-; CHECK-GI-NEXT:    mov v3.b[1], w10
-; CHECK-GI-NEXT:    mov v4.b[1], w9
+; CHECK-GI-NEXT:    mov v2.b[1], w1
+; CHECK-GI-NEXT:    ldr w11, [sp, #344]
+; CHECK-GI-NEXT:    fmov s5, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #592]
+; CHECK-GI-NEXT:    fmov s16, w12
+; CHECK-GI-NEXT:    mov v3.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #464]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1112]
+; CHECK-GI-NEXT:    fmov s7, w10
+; CHECK-GI-NEXT:    mov v4.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #472]
+; CHECK-GI-NEXT:    fmov s6, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #600]
 ; CHECK-GI-NEXT:    mov v5.b[1], w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #224]
-; CHECK-GI-NEXT:    mov v0.b[2], w2
-; CHECK-GI-NEXT:    ldr w10, [sp, #352]
-; CHECK-GI-NEXT:    ldr w11, [sp, #480]
-; CHECK-GI-NEXT:    mov v1.b[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #608]
-; CHECK-GI-NEXT:    mov v2.b[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #104]
-; CHECK-GI-NEXT:    fmov s7, w12
-; CHECK-GI-NEXT:    mov v3.b[2], w10
-; CHECK-GI-NEXT:    mov v4.b[2], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #224]
+; CHECK-GI-NEXT:    mov v2.b[2], w2
+; CHECK-GI-NEXT:    ldr w11, [sp, #16]
+; CHECK-GI-NEXT:    mov v7.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #480]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    mov v6.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #96]
+; CHECK-GI-NEXT:    mov v4.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #608]
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #352]
+; CHECK-GI-NEXT:    mov v2.b[3], w3
+; CHECK-GI-NEXT:    mov v7.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #488]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
 ; CHECK-GI-NEXT:    mov v5.b[2], w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #232]
-; CHECK-GI-NEXT:    mov v0.b[3], w3
+; CHECK-GI-NEXT:    mov v6.b[2], w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #104]
+; CHECK-GI-NEXT:    ldr w9, [sp, #232]
+; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v3.b[3], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #360]
-; CHECK-GI-NEXT:    ldr w11, [sp, #488]
-; CHECK-GI-NEXT:    mov v1.b[3], w9
+; CHECK-GI-NEXT:    mov v2.b[4], w4
+; CHECK-GI-NEXT:    mov v4.b[3], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #616]
-; CHECK-GI-NEXT:    mov v2.b[3], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #240]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1232]
-; CHECK-GI-NEXT:    mov v3.b[3], w8
-; CHECK-GI-NEXT:    mov v4.b[3], w11
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    mov v5.b[3], w8
+; CHECK-GI-NEXT:    mov v6.b[3], w10
 ; CHECK-GI-NEXT:    ldr w8, [sp, #112]
-; CHECK-GI-NEXT:    mov v5.b[3], w9
-; CHECK-GI-NEXT:    mov v0.b[4], w4
-; CHECK-GI-NEXT:    ldr w9, [sp, #368]
-; CHECK-GI-NEXT:    ldr w11, [sp, #496]
-; CHECK-GI-NEXT:    mov v1.b[4], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #624]
-; CHECK-GI-NEXT:    mov v2.b[4], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #248]
-; CHECK-GI-NEXT:    fmov s18, w12
-; CHECK-GI-NEXT:    mov v3.b[4], w9
-; CHECK-GI-NEXT:    mov v4.b[4], w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #120]
+; CHECK-GI-NEXT:    ldr w10, [sp, #240]
+; CHECK-GI-NEXT:    mov v7.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #496]
+; CHECK-GI-NEXT:    mov v3.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #368]
+; CHECK-GI-NEXT:    mov v2.b[5], w5
+; CHECK-GI-NEXT:    mov v4.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #624]
+; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
 ; CHECK-GI-NEXT:    mov v5.b[4], w8
-; CHECK-GI-NEXT:    mov v0.b[5], w5
+; CHECK-GI-NEXT:    mov v6.b[4], w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #120]
+; CHECK-GI-NEXT:    ldr w9, [sp, #248]
+; CHECK-GI-NEXT:    mov v7.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #504]
+; CHECK-GI-NEXT:    mov v3.b[5], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #376]
-; CHECK-GI-NEXT:    ldr w11, [sp, #504]
-; CHECK-GI-NEXT:    mov v1.b[5], w9
+; CHECK-GI-NEXT:    mov v2.b[6], w6
+; CHECK-GI-NEXT:    mov v4.b[5], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #632]
-; CHECK-GI-NEXT:    mov v2.b[5], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #256]
-; CHECK-GI-NEXT:    ldr w12, [sp, #744]
-; CHECK-GI-NEXT:    mov v3.b[5], w8
-; CHECK-GI-NEXT:    mov v4.b[5], w11
+; CHECK-GI-NEXT:    movi v26.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v5.b[5], w8
+; CHECK-GI-NEXT:    mov v6.b[5], w10
 ; CHECK-GI-NEXT:    ldr w8, [sp, #128]
-; CHECK-GI-NEXT:    mov v5.b[5], w9
-; CHECK-GI-NEXT:    mov v0.b[6], w6
-; CHECK-GI-NEXT:    ldr w9, [sp, #384]
-; CHECK-GI-NEXT:    ldr w11, [sp, #512]
-; CHECK-GI-NEXT:    mov v1.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #640]
-; CHECK-GI-NEXT:    mov v2.b[6], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #264]
-; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v3.b[6], w9
-; CHECK-GI-NEXT:    mov v4.b[6], w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #136]
+; CHECK-GI-NEXT:    ldr w10, [sp, #256]
+; CHECK-GI-NEXT:    mov v7.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #512]
+; CHECK-GI-NEXT:    mov v3.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #384]
+; CHECK-GI-NEXT:    mov v2.b[7], w7
+; CHECK-GI-NEXT:    mov v4.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #640]
+; CHECK-GI-NEXT:    movi v27.2d, #0000000000000000
 ; CHECK-GI-NEXT:    mov v5.b[6], w8
-; CHECK-GI-NEXT:    mov v0.b[7], w7
-; CHECK-GI-NEXT:    ldr w8, [sp, #392]
-; CHECK-GI-NEXT:    ldr w11, [sp, #520]
-; CHECK-GI-NEXT:    mov v1.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #648]
-; CHECK-GI-NEXT:    mov v2.b[7], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #16]
-; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr w8, [sp, #136]
+; CHECK-GI-NEXT:    mov v6.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #264]
+; CHECK-GI-NEXT:    mov v7.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #520]
 ; CHECK-GI-NEXT:    mov v3.b[7], w8
-; CHECK-GI-NEXT:    mov v4.b[7], w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #144]
-; CHECK-GI-NEXT:    mov v5.b[7], w9
-; CHECK-GI-NEXT:    ldr w11, [sp, #272]
-; CHECK-GI-NEXT:    mov v0.b[8], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #400]
-; CHECK-GI-NEXT:    ldr w10, [sp, #528]
-; CHECK-GI-NEXT:    mov v1.b[8], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #656]
+; CHECK-GI-NEXT:    ldr w8, [sp, #392]
 ; CHECK-GI-NEXT:    mov v2.b[8], w11
+; CHECK-GI-NEXT:    mov v4.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #648]
 ; CHECK-GI-NEXT:    ldr w11, [sp, #24]
-; CHECK-GI-NEXT:    mov v3.b[8], w9
+; CHECK-GI-NEXT:    mov v5.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #144]
+; CHECK-GI-NEXT:    mov v6.b[7], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #272]
+; CHECK-GI-NEXT:    mov v7.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #528]
+; CHECK-GI-NEXT:    mov v3.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #400]
+; CHECK-GI-NEXT:    mov v2.b[9], w11
 ; CHECK-GI-NEXT:    mov v4.b[8], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #152]
+; CHECK-GI-NEXT:    ldr w10, [sp, #656]
+; CHECK-GI-NEXT:    ldr w11, [sp, #32]
 ; CHECK-GI-NEXT:    mov v5.b[8], w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #280]
-; CHECK-GI-NEXT:    mov v0.b[9], w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #408]
-; CHECK-GI-NEXT:    ldr w11, [sp, #536]
-; CHECK-GI-NEXT:    mov v1.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #664]
-; CHECK-GI-NEXT:    mov v2.b[9], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #32]
+; CHECK-GI-NEXT:    ldr w8, [sp, #152]
+; CHECK-GI-NEXT:    mov v6.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #280]
+; CHECK-GI-NEXT:    mov v7.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #536]
 ; CHECK-GI-NEXT:    mov v3.b[9], w8
-; CHECK-GI-NEXT:    mov v4.b[9], w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #160]
-; CHECK-GI-NEXT:    mov v5.b[9], w9
-; CHECK-GI-NEXT:    ldr w11, [sp, #288]
-; CHECK-GI-NEXT:    mov v0.b[10], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #416]
-; CHECK-GI-NEXT:    ldr w10, [sp, #544]
-; CHECK-GI-NEXT:    mov v1.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #672]
+; CHECK-GI-NEXT:    ldr w8, [sp, #408]
 ; CHECK-GI-NEXT:    mov v2.b[10], w11
+; CHECK-GI-NEXT:    mov v4.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #664]
 ; CHECK-GI-NEXT:    ldr w11, [sp, #40]
-; CHECK-GI-NEXT:    mov v3.b[10], w9
+; CHECK-GI-NEXT:    mov v5.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #160]
+; CHECK-GI-NEXT:    mov v6.b[9], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #288]
+; CHECK-GI-NEXT:    mov v7.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #544]
+; CHECK-GI-NEXT:    mov v3.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #416]
+; CHECK-GI-NEXT:    mov v2.b[11], w11
 ; CHECK-GI-NEXT:    mov v4.b[10], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #168]
+; CHECK-GI-NEXT:    ldr w10, [sp, #672]
+; CHECK-GI-NEXT:    ldr w11, [sp, #48]
 ; CHECK-GI-NEXT:    mov v5.b[10], w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #296]
-; CHECK-GI-NEXT:    mov v0.b[11], w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #424]
-; CHECK-GI-NEXT:    ldr w11, [sp, #552]
-; CHECK-GI-NEXT:    mov v1.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #680]
-; CHECK-GI-NEXT:    mov v2.b[11], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #48]
+; CHECK-GI-NEXT:    ldr w8, [sp, #168]
+; CHECK-GI-NEXT:    mov v6.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #296]
+; CHECK-GI-NEXT:    mov v7.b[10], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #552]
 ; CHECK-GI-NEXT:    mov v3.b[11], w8
-; CHECK-GI-NEXT:    mov v4.b[11], w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #176]
-; CHECK-GI-NEXT:    mov v5.b[11], w9
-; CHECK-GI-NEXT:    ldr w11, [sp, #304]
-; CHECK-GI-NEXT:    mov v0.b[12], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #432]
-; CHECK-GI-NEXT:    ldr w10, [sp, #560]
-; CHECK-GI-NEXT:    mov v1.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #688]
+; CHECK-GI-NEXT:    ldr w8, [sp, #424]
 ; CHECK-GI-NEXT:    mov v2.b[12], w11
+; CHECK-GI-NEXT:    mov v4.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #680]
 ; CHECK-GI-NEXT:    ldr w11, [sp, #56]
-; CHECK-GI-NEXT:    mov v3.b[12], w9
+; CHECK-GI-NEXT:    mov v5.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #176]
+; CHECK-GI-NEXT:    mov v6.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #304]
+; CHECK-GI-NEXT:    mov v7.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #560]
+; CHECK-GI-NEXT:    mov v3.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #432]
+; CHECK-GI-NEXT:    mov v2.b[13], w11
 ; CHECK-GI-NEXT:    mov v4.b[12], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #184]
+; CHECK-GI-NEXT:    ldr w10, [sp, #688]
+; CHECK-GI-NEXT:    ldr w11, [sp, #64]
 ; CHECK-GI-NEXT:    mov v5.b[12], w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #312]
-; CHECK-GI-NEXT:    mov v0.b[13], w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #440]
-; CHECK-GI-NEXT:    ldr w11, [sp, #568]
-; CHECK-GI-NEXT:    mov v1.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #696]
-; CHECK-GI-NEXT:    mov v2.b[13], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #64]
+; CHECK-GI-NEXT:    ldr w8, [sp, #184]
+; CHECK-GI-NEXT:    mov v6.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #312]
+; CHECK-GI-NEXT:    mov v7.b[12], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #568]
 ; CHECK-GI-NEXT:    mov v3.b[13], w8
-; CHECK-GI-NEXT:    mov v4.b[13], w11
-; CHECK-GI-NEXT:    ldr w8, [sp, #192]
-; CHECK-GI-NEXT:    mov v5.b[13], w9
-; CHECK-GI-NEXT:    ldr w11, [sp, #320]
-; CHECK-GI-NEXT:    mov v0.b[14], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #448]
-; CHECK-GI-NEXT:    ldr w10, [sp, #576]
-; CHECK-GI-NEXT:    mov v1.b[14], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #704]
+; CHECK-GI-NEXT:    ldr w8, [sp, #440]
 ; CHECK-GI-NEXT:    mov v2.b[14], w11
+; CHECK-GI-NEXT:    mov v4.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #696]
 ; CHECK-GI-NEXT:    ldr w11, [sp, #72]
-; CHECK-GI-NEXT:    mov v3.b[14], w9
+; CHECK-GI-NEXT:    mov v5.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #192]
+; CHECK-GI-NEXT:    mov v6.b[13], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #320]
+; CHECK-GI-NEXT:    mov v7.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #576]
+; CHECK-GI-NEXT:    mov v3.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #448]
+; CHECK-GI-NEXT:    mov v2.b[15], w11
 ; CHECK-GI-NEXT:    mov v4.b[14], w10
-; CHECK-GI-NEXT:    ldr w9, [sp, #200]
-; CHECK-GI-NEXT:    mov v5.b[14], w8
-; CHECK-GI-NEXT:    ldr w10, [sp, #328]
-; CHECK-GI-NEXT:    mov v0.b[15], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #584]
-; CHECK-GI-NEXT:    mov v1.b[15], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #712]
-; CHECK-GI-NEXT:    mov v2.b[15], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #720]
-; CHECK-GI-NEXT:    ldr w8, [sp, #456]
-; CHECK-GI-NEXT:    mov v4.b[15], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #976]
-; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v5.b[15], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #856]
-; CHECK-GI-NEXT:    fmov s6, w10
-; CHECK-GI-NEXT:    fmov s16, w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1104]
-; CHECK-GI-NEXT:    ldr w10, [sp, #984]
-; CHECK-GI-NEXT:    mov v7.b[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1360]
+; CHECK-GI-NEXT:    ldr w10, [sp, #704]
+; CHECK-GI-NEXT:    ldr w11, [sp, #848]
+; CHECK-GI-NEXT:    mov v5.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #200]
+; CHECK-GI-NEXT:    mov v6.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #328]
+; CHECK-GI-NEXT:    mov v7.b[14], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #584]
 ; CHECK-GI-NEXT:    mov v3.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #456]
 ; CHECK-GI-NEXT:    fmov s17, w11
-; CHECK-GI-NEXT:    mov v6.b[1], w13
-; CHECK-GI-NEXT:    ldr w13, [sp, #1112]
-; CHECK-GI-NEXT:    fmov s19, w9
+; CHECK-GI-NEXT:    mov v4.b[15], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #712]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1104]
+; CHECK-GI-NEXT:    mov v5.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #856]
+; CHECK-GI-NEXT:    mov v6.b[15], w10
+; CHECK-GI-NEXT:    mov v7.b[15], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #976]
+; CHECK-GI-NEXT:    ldr w10, [sp, #728]
+; CHECK-GI-NEXT:    mov v17.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1232]
+; CHECK-GI-NEXT:    fmov s19, w11
+; CHECK-GI-NEXT:    fmov s18, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1360]
 ; CHECK-GI-NEXT:    mov v16.b[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1240]
-; CHECK-GI-NEXT:    ldr w11, [sp, #1368]
-; CHECK-GI-NEXT:    ldr w8, [sp, #736]
-; CHECK-GI-NEXT:    ldr w9, [sp, #864]
-; CHECK-GI-NEXT:    mov v17.b[1], w13
+; CHECK-GI-NEXT:    ldr w10, [sp, #984]
+; CHECK-GI-NEXT:    fmov s20, w8
+; CHECK-GI-NEXT:    ldr w11, [sp, #1240]
+; CHECK-GI-NEXT:    fmov s21, w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #1368]
+; CHECK-GI-NEXT:    mov v19.b[1], w12
 ; CHECK-GI-NEXT:    mov v18.b[1], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #992]
-; CHECK-GI-NEXT:    mov v19.b[1], w11
-; CHECK-GI-NEXT:    mov v6.b[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1120]
-; CHECK-GI-NEXT:    mov v7.b[2], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1248]
-; CHECK-GI-NEXT:    ldr w11, [sp, #1376]
+; CHECK-GI-NEXT:    ldr w10, [sp, #736]
+; CHECK-GI-NEXT:    ldr w9, [sp, #992]
+; CHECK-GI-NEXT:    mov v20.b[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #864]
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    mov v21.b[1], w8
 ; CHECK-GI-NEXT:    mov v16.b[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #752]
-; CHECK-GI-NEXT:    mov v20.s[2], wzr
-; CHECK-GI-NEXT:    mov v17.b[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #872]
+; CHECK-GI-NEXT:    ldr w8, [sp, #1120]
+; CHECK-GI-NEXT:    mov v17.b[2], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #1248]
+; CHECK-GI-NEXT:    ldr w11, [sp, #744]
 ; CHECK-GI-NEXT:    mov v18.b[2], w9
-; CHECK-GI-NEXT:    mov v19.b[2], w11
-; CHECK-GI-NEXT:    ldr w9, [sp, #1000]
-; CHECK-GI-NEXT:    mov v6.b[3], w12
-; CHECK-GI-NEXT:    ldr w11, [sp, #1128]
-; CHECK-GI-NEXT:    mov v7.b[3], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1256]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1384]
-; CHECK-GI-NEXT:    mov v16.b[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #760]
-; CHECK-GI-NEXT:    mov v17.b[3], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #880]
-; CHECK-GI-NEXT:    mov v18.b[3], w8
-; CHECK-GI-NEXT:    mov v19.b[3], w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #1008]
-; CHECK-GI-NEXT:    mov v6.b[4], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1136]
-; CHECK-GI-NEXT:    mov v7.b[4], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1264]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1392]
-; CHECK-GI-NEXT:    mov v16.b[4], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #768]
-; CHECK-GI-NEXT:    mov v17.b[4], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #888]
-; CHECK-GI-NEXT:    mov v18.b[4], w11
-; CHECK-GI-NEXT:    mov v19.b[4], w12
-; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
-; CHECK-GI-NEXT:    mov v6.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1144]
-; CHECK-GI-NEXT:    mov v7.b[5], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1272]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1400]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1376]
+; CHECK-GI-NEXT:    mov v19.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #872]
+; CHECK-GI-NEXT:    mov v20.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1000]
+; CHECK-GI-NEXT:    mov v21.b[2], w9
+; CHECK-GI-NEXT:    mov v16.b[3], w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #1256]
+; CHECK-GI-NEXT:    mov v17.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1128]
+; CHECK-GI-NEXT:    ldr w11, [sp, #752]
+; CHECK-GI-NEXT:    mov v18.b[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1384]
+; CHECK-GI-NEXT:    sdot v22.4s, v2.16b, v5.16b
+; CHECK-GI-NEXT:    mov v19.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #880]
+; CHECK-GI-NEXT:    mov v20.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1008]
+; CHECK-GI-NEXT:    mov v21.b[3], w10
+; CHECK-GI-NEXT:    mov v16.b[4], w11
+; CHECK-GI-NEXT:    mov v17.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1136]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1264]
+; CHECK-GI-NEXT:    mov v18.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1392]
+; CHECK-GI-NEXT:    ldr w11, [sp, #760]
+; CHECK-GI-NEXT:    mov v19.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #888]
+; CHECK-GI-NEXT:    mov v20.b[4], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1016]
+; CHECK-GI-NEXT:    mov v21.b[4], w9
 ; CHECK-GI-NEXT:    mov v16.b[5], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #776]
-; CHECK-GI-NEXT:    mov v17.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #896]
+; CHECK-GI-NEXT:    mov v17.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1144]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1272]
 ; CHECK-GI-NEXT:    mov v18.b[5], w10
-; CHECK-GI-NEXT:    mov v19.b[5], w12
-; CHECK-GI-NEXT:    ldr w10, [sp, #1024]
-; CHECK-GI-NEXT:    mov v6.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1152]
-; CHECK-GI-NEXT:    mov v7.b[6], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1280]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1408]
-; CHECK-GI-NEXT:    mov v16.b[6], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #784]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1400]
+; CHECK-GI-NEXT:    ldr w11, [sp, #768]
+; CHECK-GI-NEXT:    mov v19.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #896]
+; CHECK-GI-NEXT:    mov v20.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1024]
+; CHECK-GI-NEXT:    mov v21.b[5], w10
+; CHECK-GI-NEXT:    mov v16.b[6], w11
 ; CHECK-GI-NEXT:    mov v17.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #904]
+; CHECK-GI-NEXT:    ldr w8, [sp, #1152]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1280]
 ; CHECK-GI-NEXT:    mov v18.b[6], w9
-; CHECK-GI-NEXT:    mov v19.b[6], w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #1032]
-; CHECK-GI-NEXT:    mov v6.b[7], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1160]
-; CHECK-GI-NEXT:    mov v7.b[7], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1288]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1416]
-; CHECK-GI-NEXT:    mov v16.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #792]
-; CHECK-GI-NEXT:    mov v17.b[7], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #912]
-; CHECK-GI-NEXT:    mov v18.b[7], w8
-; CHECK-GI-NEXT:    mov v19.b[7], w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #1040]
-; CHECK-GI-NEXT:    mov v6.b[8], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1168]
-; CHECK-GI-NEXT:    mov v7.b[8], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1296]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1424]
-; CHECK-GI-NEXT:    mov v16.b[8], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #800]
-; CHECK-GI-NEXT:    mov v17.b[8], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #920]
-; CHECK-GI-NEXT:    mov v18.b[8], w11
-; CHECK-GI-NEXT:    mov v19.b[8], w12
-; CHECK-GI-NEXT:    ldr w11, [sp, #1048]
-; CHECK-GI-NEXT:    mov v6.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1176]
-; CHECK-GI-NEXT:    mov v7.b[9], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1304]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1432]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1408]
+; CHECK-GI-NEXT:    ldr w11, [sp, #776]
+; CHECK-GI-NEXT:    mov v19.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #904]
+; CHECK-GI-NEXT:    mov v20.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1032]
+; CHECK-GI-NEXT:    mov v21.b[6], w9
+; CHECK-GI-NEXT:    mov v16.b[7], w11
+; CHECK-GI-NEXT:    mov v17.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1160]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1288]
+; CHECK-GI-NEXT:    mov v18.b[7], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1416]
+; CHECK-GI-NEXT:    ldr w11, [sp, #784]
+; CHECK-GI-NEXT:    mov v19.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #912]
+; CHECK-GI-NEXT:    mov v20.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1040]
+; CHECK-GI-NEXT:    mov v21.b[7], w10
+; CHECK-GI-NEXT:    mov v16.b[8], w11
+; CHECK-GI-NEXT:    mov v17.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1168]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1296]
+; CHECK-GI-NEXT:    mov v18.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1424]
+; CHECK-GI-NEXT:    ldr w11, [sp, #792]
+; CHECK-GI-NEXT:    mov v19.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #920]
+; CHECK-GI-NEXT:    mov v20.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1048]
+; CHECK-GI-NEXT:    mov v21.b[8], w9
 ; CHECK-GI-NEXT:    mov v16.b[9], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #808]
-; CHECK-GI-NEXT:    mov v17.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #928]
+; CHECK-GI-NEXT:    mov v17.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1176]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1304]
 ; CHECK-GI-NEXT:    mov v18.b[9], w10
-; CHECK-GI-NEXT:    mov v19.b[9], w12
-; CHECK-GI-NEXT:    ldr w10, [sp, #1056]
-; CHECK-GI-NEXT:    mov v6.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1184]
-; CHECK-GI-NEXT:    mov v7.b[10], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1312]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1440]
-; CHECK-GI-NEXT:    mov v16.b[10], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #816]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1432]
+; CHECK-GI-NEXT:    ldr w11, [sp, #800]
+; CHECK-GI-NEXT:    mov v19.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #928]
+; CHECK-GI-NEXT:    mov v20.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1056]
+; CHECK-GI-NEXT:    mov v21.b[9], w10
+; CHECK-GI-NEXT:    mov v16.b[10], w11
 ; CHECK-GI-NEXT:    mov v17.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #936]
+; CHECK-GI-NEXT:    ldr w8, [sp, #1184]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1312]
 ; CHECK-GI-NEXT:    mov v18.b[10], w9
-; CHECK-GI-NEXT:    mov v19.b[10], w12
-; CHECK-GI-NEXT:    ldr w9, [sp, #1064]
-; CHECK-GI-NEXT:    mov v6.b[11], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1192]
-; CHECK-GI-NEXT:    mov v7.b[11], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1320]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1448]
-; CHECK-GI-NEXT:    mov v16.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #824]
-; CHECK-GI-NEXT:    mov v17.b[11], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #944]
-; CHECK-GI-NEXT:    mov v18.b[11], w8
-; CHECK-GI-NEXT:    mov v19.b[11], w12
-; CHECK-GI-NEXT:    ldr w8, [sp, #1072]
-; CHECK-GI-NEXT:    mov v6.b[12], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1200]
-; CHECK-GI-NEXT:    mov v7.b[12], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #1328]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1456]
-; CHECK-GI-NEXT:    mov v16.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #832]
-; CHECK-GI-NEXT:    mov v17.b[12], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #952]
-; CHECK-GI-NEXT:    mov v18.b[12], w11
-; CHECK-GI-NEXT:    mov v19.b[12], w12
-; CHECK-GI-NEXT:    ldr w11, [sp, #1080]
-; CHECK-GI-NEXT:    mov v6.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1208]
-; CHECK-GI-NEXT:    mov v7.b[13], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1336]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1464]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1440]
+; CHECK-GI-NEXT:    ldr w11, [sp, #808]
+; CHECK-GI-NEXT:    mov v19.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #936]
+; CHECK-GI-NEXT:    mov v20.b[10], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1064]
+; CHECK-GI-NEXT:    mov v21.b[10], w9
+; CHECK-GI-NEXT:    mov v16.b[11], w11
+; CHECK-GI-NEXT:    mov v17.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1192]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1320]
+; CHECK-GI-NEXT:    mov v18.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1448]
+; CHECK-GI-NEXT:    ldr w11, [sp, #816]
+; CHECK-GI-NEXT:    mov v19.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #944]
+; CHECK-GI-NEXT:    mov v20.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1072]
+; CHECK-GI-NEXT:    mov v21.b[11], w10
+; CHECK-GI-NEXT:    mov v16.b[12], w11
+; CHECK-GI-NEXT:    mov v17.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1200]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1328]
+; CHECK-GI-NEXT:    mov v18.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1456]
+; CHECK-GI-NEXT:    ldr w11, [sp, #824]
+; CHECK-GI-NEXT:    mov v19.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #952]
+; CHECK-GI-NEXT:    mov v20.b[12], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1080]
+; CHECK-GI-NEXT:    mov v21.b[12], w9
 ; CHECK-GI-NEXT:    mov v16.b[13], w11
-; CHECK-GI-NEXT:    ldr w11, [sp, #960]
-; CHECK-GI-NEXT:    mov v17.b[13], w9
+; CHECK-GI-NEXT:    mov v17.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1208]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1336]
 ; CHECK-GI-NEXT:    mov v18.b[13], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1464]
+; CHECK-GI-NEXT:    ldr w11, [sp, #832]
+; CHECK-GI-NEXT:    mov v19.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #960]
+; CHECK-GI-NEXT:    mov v20.b[13], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #1088]
-; CHECK-GI-NEXT:    mov v19.b[13], w12
-; CHECK-GI-NEXT:    mov v6.b[14], w8
+; CHECK-GI-NEXT:    mov v21.b[13], w10
+; CHECK-GI-NEXT:    mov v16.b[14], w11
+; CHECK-GI-NEXT:    mov v17.b[14], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #1216]
-; CHECK-GI-NEXT:    mov v7.b[14], w11
 ; CHECK-GI-NEXT:    ldr w10, [sp, #1344]
-; CHECK-GI-NEXT:    ldr w11, [sp, #1472]
-; CHECK-GI-NEXT:    ldr w12, [sp, #840]
-; CHECK-GI-NEXT:    mov v16.b[14], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1096]
-; CHECK-GI-NEXT:    mov v17.b[14], w8
-; CHECK-GI-NEXT:    mov v18.b[14], w10
+; CHECK-GI-NEXT:    mov v18.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1472]
+; CHECK-GI-NEXT:    ldr w11, [sp, #840]
+; CHECK-GI-NEXT:    mov v19.b[14], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #968]
-; CHECK-GI-NEXT:    mov v19.b[14], w11
-; CHECK-GI-NEXT:    ldr w10, [sp, #1224]
-; CHECK-GI-NEXT:    mov v6.b[15], w12
+; CHECK-GI-NEXT:    mov v20.b[14], w10
+; CHECK-GI-NEXT:    mov v21.b[14], w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #1096]
+; CHECK-GI-NEXT:    mov v16.b[15], w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #1224]
 ; CHECK-GI-NEXT:    ldr w11, [sp, #1352]
-; CHECK-GI-NEXT:    ldr w12, [sp, #1480]
-; CHECK-GI-NEXT:    mov v7.b[15], w8
-; CHECK-GI-NEXT:    mov v16.b[15], w9
-; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
-; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v17.b[15], w10
-; CHECK-GI-NEXT:    mov v18.b[15], w11
-; CHECK-GI-NEXT:    movi v26.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v19.b[15], w12
-; CHECK-GI-NEXT:    sdot v21.4s, v0.16b, v3.16b
-; CHECK-GI-NEXT:    sdot v22.4s, v1.16b, v4.16b
-; CHECK-GI-NEXT:    sdot v23.4s, v2.16b, v5.16b
-; CHECK-GI-NEXT:    mov v20.s[3], wzr
-; CHECK-GI-NEXT:    sdot v25.4s, v6.16b, v17.16b
-; CHECK-GI-NEXT:    sdot v26.4s, v7.16b, v18.16b
-; CHECK-GI-NEXT:    sdot v24.4s, v16.16b, v19.16b
-; CHECK-GI-NEXT:    add v0.4s, v21.4s, v22.4s
-; CHECK-GI-NEXT:    add v1.4s, v23.4s, v20.4s
-; CHECK-GI-NEXT:    add v2.4s, v25.4s, v26.4s
-; CHECK-GI-NEXT:    add v3.4s, v24.4s, v20.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    mov v17.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1480]
+; CHECK-GI-NEXT:    mov v18.b[15], w10
+; CHECK-GI-NEXT:    sdot v23.4s, v3.16b, v6.16b
+; CHECK-GI-NEXT:    mov v19.b[15], w9
+; CHECK-GI-NEXT:    mov v20.b[15], w11
+; CHECK-GI-NEXT:    sdot v24.4s, v4.16b, v7.16b
+; CHECK-GI-NEXT:    mov v21.b[15], w8
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    add v2.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT:    sdot v25.4s, v16.16b, v19.16b
+; CHECK-GI-NEXT:    sdot v27.4s, v17.16b, v20.16b
+; CHECK-GI-NEXT:    sdot v26.4s, v18.16b, v21.16b
+; CHECK-GI-NEXT:    add v1.4s, v24.4s, v1.4s
+; CHECK-GI-NEXT:    add v3.4s, v25.4s, v27.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v26.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    add w0, w8, w9
 ; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
@@ -6746,217 +6966,221 @@ define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %
 ; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
-; CHECK-GI-NEXT:    ldr w10, [sp, #80]
-; CHECK-GI-NEXT:    ldr w9, [sp, #88]
-; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    ldr w11, [sp, #208]
-; CHECK-GI-NEXT:    ldr w8, [sp, #216]
-; CHECK-GI-NEXT:    mov v6.s[0], wzr
-; CHECK-GI-NEXT:    fmov s1, w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #720]
-; CHECK-GI-NEXT:    movi v7.16b, #1
-; CHECK-GI-NEXT:    fmov s2, w11
-; CHECK-GI-NEXT:    mov v0.b[1], w1
-; CHECK-GI-NEXT:    ldr w11, [sp, #728]
-; CHECK-GI-NEXT:    fmov s3, w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #976]
-; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v1.b[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #848]
-; CHECK-GI-NEXT:    mov v6.s[1], wzr
-; CHECK-GI-NEXT:    fmov s5, w10
-; CHECK-GI-NEXT:    mov v2.b[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #856]
+; CHECK-GI-NEXT:    ldr w9, [sp, #208]
+; CHECK-GI-NEXT:    ldr w11, [sp, #80]
+; CHECK-GI-NEXT:    fmov s2, w0
+; CHECK-GI-NEXT:    ldr w10, [sp, #216]
+; CHECK-GI-NEXT:    ldr w12, [sp, #848]
+; CHECK-GI-NEXT:    fmov s1, wzr
 ; CHECK-GI-NEXT:    fmov s4, w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #984]
-; CHECK-GI-NEXT:    mov v3.b[1], w11
-; CHECK-GI-NEXT:    ldr w10, [sp, #224]
-; CHECK-GI-NEXT:    mov v0.b[2], w2
-; CHECK-GI-NEXT:    ldr w11, [sp, #16]
-; CHECK-GI-NEXT:    mov v5.b[1], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #864]
-; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v4.b[1], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #96]
-; CHECK-GI-NEXT:    mov v2.b[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #992]
-; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
-; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v1.b[2], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #736]
-; CHECK-GI-NEXT:    mov v0.b[3], w3
-; CHECK-GI-NEXT:    mov v5.b[2], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #872]
-; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v3.b[2], w8
-; CHECK-GI-NEXT:    mov v4.b[2], w9
+; CHECK-GI-NEXT:    fmov s3, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #720]
+; CHECK-GI-NEXT:    ldr w8, [sp, #88]
+; CHECK-GI-NEXT:    fmov s6, w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #856]
+; CHECK-GI-NEXT:    fmov s5, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #984]
+; CHECK-GI-NEXT:    mov v2.b[1], w1
+; CHECK-GI-NEXT:    mov v4.b[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #976]
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #728]
+; CHECK-GI-NEXT:    mov v6.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #736]
+; CHECK-GI-NEXT:    fmov s7, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #96]
+; CHECK-GI-NEXT:    ldr w12, [sp, #16]
+; CHECK-GI-NEXT:    mov v5.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #224]
+; CHECK-GI-NEXT:    mov v2.b[2], w2
+; CHECK-GI-NEXT:    mov v3.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #864]
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    mov v7.b[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #992]
+; CHECK-GI-NEXT:    mov v4.b[2], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #104]
+; CHECK-GI-NEXT:    mov v6.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #872]
+; CHECK-GI-NEXT:    mov v5.b[2], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #232]
-; CHECK-GI-NEXT:    mov v6.s[2], wzr
-; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
-; CHECK-GI-NEXT:    mov v1.b[3], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #744]
-; CHECK-GI-NEXT:    mov v0.b[4], w4
-; CHECK-GI-NEXT:    mov v2.b[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1000]
+; CHECK-GI-NEXT:    mov v2.b[3], w3
 ; CHECK-GI-NEXT:    mov v3.b[3], w8
-; CHECK-GI-NEXT:    mov v4.b[3], w10
-; CHECK-GI-NEXT:    ldr w8, [sp, #112]
-; CHECK-GI-NEXT:    ldr w10, [sp, #240]
-; CHECK-GI-NEXT:    mov v5.b[3], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #880]
-; CHECK-GI-NEXT:    mov v1.b[4], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #752]
-; CHECK-GI-NEXT:    mov v0.b[5], w5
-; CHECK-GI-NEXT:    mov v2.b[4], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1008]
-; CHECK-GI-NEXT:    mov v6.s[3], wzr
-; CHECK-GI-NEXT:    mov v3.b[4], w8
-; CHECK-GI-NEXT:    mov v4.b[4], w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #744]
+; CHECK-GI-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-NEXT:    mov v7.b[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1000]
+; CHECK-GI-NEXT:    mov v4.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #112]
+; CHECK-GI-NEXT:    mov v6.b[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #880]
+; CHECK-GI-NEXT:    mov v5.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #240]
+; CHECK-GI-NEXT:    mov v2.b[4], w4
+; CHECK-GI-NEXT:    mov v3.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #752]
+; CHECK-GI-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-NEXT:    mov v7.b[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1008]
+; CHECK-GI-NEXT:    mov v4.b[4], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #120]
-; CHECK-GI-NEXT:    ldr w9, [sp, #248]
-; CHECK-GI-NEXT:    mov v5.b[4], w10
+; CHECK-GI-NEXT:    mov v6.b[4], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #888]
-; CHECK-GI-NEXT:    mov v1.b[5], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #760]
-; CHECK-GI-NEXT:    mov v0.b[6], w6
-; CHECK-GI-NEXT:    mov v2.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1016]
+; CHECK-GI-NEXT:    mov v5.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #248]
+; CHECK-GI-NEXT:    mov v2.b[5], w5
 ; CHECK-GI-NEXT:    mov v3.b[5], w8
-; CHECK-GI-NEXT:    mov v4.b[5], w10
-; CHECK-GI-NEXT:    ldr w8, [sp, #128]
-; CHECK-GI-NEXT:    ldr w10, [sp, #256]
-; CHECK-GI-NEXT:    mov v5.b[5], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #896]
-; CHECK-GI-NEXT:    mov v1.b[6], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #768]
-; CHECK-GI-NEXT:    mov v0.b[7], w7
-; CHECK-GI-NEXT:    mov v2.b[6], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1024]
-; CHECK-GI-NEXT:    mov v3.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #760]
+; CHECK-GI-NEXT:    movi v16.16b, #1
+; CHECK-GI-NEXT:    mov v7.b[4], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
+; CHECK-GI-NEXT:    mov v4.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #128]
+; CHECK-GI-NEXT:    mov v6.b[5], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #896]
+; CHECK-GI-NEXT:    mov v5.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #256]
+; CHECK-GI-NEXT:    mov v2.b[6], w6
+; CHECK-GI-NEXT:    mov v3.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #768]
+; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v7.b[5], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1024]
+; CHECK-GI-NEXT:    mov v4.b[6], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #136]
-; CHECK-GI-NEXT:    mov v4.b[6], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #264]
-; CHECK-GI-NEXT:    mov v5.b[6], w10
+; CHECK-GI-NEXT:    mov v6.b[6], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #904]
-; CHECK-GI-NEXT:    mov v1.b[7], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #776]
-; CHECK-GI-NEXT:    mov v0.b[8], w11
-; CHECK-GI-NEXT:    mov v2.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1032]
-; CHECK-GI-NEXT:    ldr w11, [sp, #24]
+; CHECK-GI-NEXT:    mov v5.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #264]
+; CHECK-GI-NEXT:    mov v2.b[7], w7
 ; CHECK-GI-NEXT:    mov v3.b[7], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #144]
-; CHECK-GI-NEXT:    mov v4.b[7], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #272]
-; CHECK-GI-NEXT:    mov v5.b[7], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #912]
-; CHECK-GI-NEXT:    mov v1.b[8], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #784]
-; CHECK-GI-NEXT:    mov v0.b[9], w11
-; CHECK-GI-NEXT:    mov v2.b[8], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1040]
-; CHECK-GI-NEXT:    ldr w11, [sp, #32]
-; CHECK-GI-NEXT:    mov v3.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #776]
+; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v7.b[6], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1032]
+; CHECK-GI-NEXT:    mov v4.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #144]
+; CHECK-GI-NEXT:    mov v6.b[7], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #912]
+; CHECK-GI-NEXT:    mov v5.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #272]
+; CHECK-GI-NEXT:    mov v2.b[8], w12
+; CHECK-GI-NEXT:    mov v3.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #784]
+; CHECK-GI-NEXT:    ldr w12, [sp, #24]
+; CHECK-GI-NEXT:    mov v7.b[7], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1040]
+; CHECK-GI-NEXT:    mov v4.b[8], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #152]
-; CHECK-GI-NEXT:    mov v4.b[8], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #280]
-; CHECK-GI-NEXT:    mov v5.b[8], w10
+; CHECK-GI-NEXT:    mov v6.b[8], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #920]
-; CHECK-GI-NEXT:    mov v1.b[9], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #792]
-; CHECK-GI-NEXT:    mov v0.b[10], w11
-; CHECK-GI-NEXT:    mov v2.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1048]
-; CHECK-GI-NEXT:    ldr w11, [sp, #40]
+; CHECK-GI-NEXT:    mov v5.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #280]
+; CHECK-GI-NEXT:    mov v2.b[9], w12
 ; CHECK-GI-NEXT:    mov v3.b[9], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #160]
-; CHECK-GI-NEXT:    mov v4.b[9], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #288]
-; CHECK-GI-NEXT:    mov v5.b[9], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #928]
-; CHECK-GI-NEXT:    mov v1.b[10], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #800]
-; CHECK-GI-NEXT:    mov v0.b[11], w11
-; CHECK-GI-NEXT:    mov v2.b[10], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1056]
-; CHECK-GI-NEXT:    ldr w11, [sp, #48]
-; CHECK-GI-NEXT:    mov v3.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #792]
+; CHECK-GI-NEXT:    ldr w12, [sp, #32]
+; CHECK-GI-NEXT:    mov v7.b[8], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1048]
+; CHECK-GI-NEXT:    mov v4.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #160]
+; CHECK-GI-NEXT:    mov v6.b[9], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #928]
+; CHECK-GI-NEXT:    mov v5.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #288]
+; CHECK-GI-NEXT:    mov v2.b[10], w12
+; CHECK-GI-NEXT:    mov v3.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #800]
+; CHECK-GI-NEXT:    ldr w12, [sp, #40]
+; CHECK-GI-NEXT:    mov v7.b[9], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1056]
+; CHECK-GI-NEXT:    mov v4.b[10], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #168]
-; CHECK-GI-NEXT:    mov v4.b[10], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #296]
-; CHECK-GI-NEXT:    mov v5.b[10], w10
+; CHECK-GI-NEXT:    mov v6.b[10], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #936]
-; CHECK-GI-NEXT:    mov v1.b[11], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #808]
-; CHECK-GI-NEXT:    mov v0.b[12], w11
-; CHECK-GI-NEXT:    mov v2.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1064]
-; CHECK-GI-NEXT:    ldr w11, [sp, #56]
+; CHECK-GI-NEXT:    mov v5.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #296]
+; CHECK-GI-NEXT:    mov v2.b[11], w12
 ; CHECK-GI-NEXT:    mov v3.b[11], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #176]
-; CHECK-GI-NEXT:    mov v4.b[11], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #304]
-; CHECK-GI-NEXT:    mov v5.b[11], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #944]
-; CHECK-GI-NEXT:    mov v1.b[12], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #816]
-; CHECK-GI-NEXT:    mov v0.b[13], w11
-; CHECK-GI-NEXT:    mov v2.b[12], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1072]
-; CHECK-GI-NEXT:    ldr w11, [sp, #64]
-; CHECK-GI-NEXT:    mov v3.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #808]
+; CHECK-GI-NEXT:    ldr w12, [sp, #48]
+; CHECK-GI-NEXT:    mov v7.b[10], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1064]
+; CHECK-GI-NEXT:    mov v4.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #176]
+; CHECK-GI-NEXT:    mov v6.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #944]
+; CHECK-GI-NEXT:    mov v5.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #304]
+; CHECK-GI-NEXT:    mov v2.b[12], w12
+; CHECK-GI-NEXT:    mov v3.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #816]
+; CHECK-GI-NEXT:    ldr w12, [sp, #56]
+; CHECK-GI-NEXT:    mov v7.b[11], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1072]
+; CHECK-GI-NEXT:    mov v4.b[12], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #184]
-; CHECK-GI-NEXT:    mov v4.b[12], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #312]
-; CHECK-GI-NEXT:    mov v5.b[12], w10
+; CHECK-GI-NEXT:    mov v6.b[12], w10
 ; CHECK-GI-NEXT:    ldr w10, [sp, #952]
-; CHECK-GI-NEXT:    mov v1.b[13], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #824]
-; CHECK-GI-NEXT:    mov v0.b[14], w11
-; CHECK-GI-NEXT:    mov v2.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #1080]
-; CHECK-GI-NEXT:    ldr w11, [sp, #72]
+; CHECK-GI-NEXT:    mov v5.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #312]
+; CHECK-GI-NEXT:    mov v2.b[13], w12
 ; CHECK-GI-NEXT:    mov v3.b[13], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #192]
-; CHECK-GI-NEXT:    mov v4.b[13], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #320]
-; CHECK-GI-NEXT:    mov v5.b[13], w9
-; CHECK-GI-NEXT:    ldr w9, [sp, #960]
-; CHECK-GI-NEXT:    mov v1.b[14], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #832]
-; CHECK-GI-NEXT:    mov v0.b[15], w11
-; CHECK-GI-NEXT:    mov v2.b[14], w10
-; CHECK-GI-NEXT:    ldr w10, [sp, #1088]
-; CHECK-GI-NEXT:    ldr w11, [sp, #968]
-; CHECK-GI-NEXT:    mov v3.b[14], w8
-; CHECK-GI-NEXT:    mov v4.b[14], w9
+; CHECK-GI-NEXT:    ldr w8, [sp, #824]
+; CHECK-GI-NEXT:    ldr w12, [sp, #64]
+; CHECK-GI-NEXT:    mov v7.b[12], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1080]
+; CHECK-GI-NEXT:    mov v4.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #192]
+; CHECK-GI-NEXT:    mov v6.b[13], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #960]
+; CHECK-GI-NEXT:    mov v5.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #320]
+; CHECK-GI-NEXT:    mov v2.b[14], w12
+; CHECK-GI-NEXT:    mov v3.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #832]
+; CHECK-GI-NEXT:    ldr w12, [sp, #72]
+; CHECK-GI-NEXT:    mov v7.b[13], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1088]
+; CHECK-GI-NEXT:    mov v4.b[14], w8
 ; CHECK-GI-NEXT:    ldr w8, [sp, #200]
-; CHECK-GI-NEXT:    mov v5.b[14], w10
+; CHECK-GI-NEXT:    mov v6.b[14], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #968]
+; CHECK-GI-NEXT:    mov v5.b[14], w9
 ; CHECK-GI-NEXT:    ldr w9, [sp, #328]
-; CHECK-GI-NEXT:    ldr w10, [sp, #840]
-; CHECK-GI-NEXT:    mov v1.b[15], w8
-; CHECK-GI-NEXT:    ldr w8, [sp, #1096]
-; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v7.16b
-; CHECK-GI-NEXT:    mov v2.b[15], w9
-; CHECK-GI-NEXT:    mov v3.b[15], w10
-; CHECK-GI-NEXT:    mov v4.b[15], w11
+; CHECK-GI-NEXT:    mov v2.b[15], w12
+; CHECK-GI-NEXT:    mov v3.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #840]
+; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v7.b[14], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1096]
+; CHECK-GI-NEXT:    mov v4.b[15], w9
+; CHECK-GI-NEXT:    mov v6.b[15], w10
+; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
 ; CHECK-GI-NEXT:    mov v5.b[15], w8
-; CHECK-GI-NEXT:    sdot v17.4s, v1.16b, v7.16b
-; CHECK-GI-NEXT:    sdot v18.4s, v2.16b, v7.16b
-; CHECK-GI-NEXT:    sdot v19.4s, v3.16b, v7.16b
-; CHECK-GI-NEXT:    sdot v21.4s, v4.16b, v7.16b
-; CHECK-GI-NEXT:    sdot v20.4s, v5.16b, v7.16b
-; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    add v1.4s, v18.4s, v6.4s
-; CHECK-GI-NEXT:    add v2.4s, v19.4s, v21.4s
-; CHECK-GI-NEXT:    add v3.4s, v20.4s, v6.4s
-; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-NEXT:    mov v0.s[2], wzr
+; CHECK-GI-NEXT:    sdot v17.4s, v2.16b, v16.16b
+; CHECK-GI-NEXT:    sdot v18.4s, v3.16b, v16.16b
+; CHECK-GI-NEXT:    mov v7.b[15], w11
+; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v16.16b
+; CHECK-GI-NEXT:    sdot v21.4s, v6.16b, v16.16b
+; CHECK-GI-NEXT:    sdot v20.4s, v5.16b, v16.16b
+; CHECK-GI-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-NEXT:    mov v0.s[3], wzr
+; CHECK-GI-NEXT:    add v2.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT:    sdot v22.4s, v7.16b, v16.16b
+; CHECK-GI-NEXT:    add v1.4s, v19.4s, v1.4s
+; CHECK-GI-NEXT:    add v3.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT:    add v0.4s, v22.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
 ; CHECK-GI-NEXT:    addv s1, v1.4s
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    fmov w9, s0
 ; CHECK-GI-NEXT:    add w0, w8, w9
 ; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index d977d8fc9cf2..0c7b3c7d3c13 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -181,14 +181,7 @@ define <4 x i32> @usdot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {
 ;
 ; CHECK-NEWLOWERING-I8MM-LABEL: usdot:
 ; CHECK-NEWLOWERING-I8MM:       // %bb.0:
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v3.8h, v1.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v4.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v4.4h, v3.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.4s, v4.8h, v3.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v2.4h, v1.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.4s, v2.8h, v1.8h
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v0.4s, v1.16b, v2.16b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
   %u.wide = zext <16 x i8> %u to <16 x i32>
   %s.wide = sext <16 x i8> %s to <16 x i32>
@@ -247,15 +240,8 @@ define <4 x i32> @usdot_in_loop(ptr %p1, ptr %p2){
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ldr q3, [x1, x8]
 ; CHECK-NEWLOWERING-I8MM-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    add x8, x8, #16
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v4.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v5.8h, v3.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v1.4s, v3.16b, v2.16b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    cmp x8, #16
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.4s, v4.4h, v5.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.4s, v4.8h, v5.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.4s, v2.4h, v3.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.4s, v2.8h, v3.8h
 ; CHECK-NEWLOWERING-I8MM-NEXT:    b.ne .LBB6_1
 ; CHECK-NEWLOWERING-I8MM-NEXT:  // %bb.2: // %end
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
@@ -306,19 +292,7 @@ define <2 x i32> @usdot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
 ;
 ; CHECK-NEWLOWERING-I8MM-LABEL: usdot_narrow:
 ; CHECK-NEWLOWERING-I8MM:       // %bb.0:
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEWLOWERING-I8MM-NEXT:    smull v3.4s, v2.4h, v1.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v2.4h, v1.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    smull2 v1.4s, v2.8h, v1.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v3.16b, v3.16b, v3.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    add v0.2s, v3.2s, v0.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v5.4h, v4.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    add v0.2s, v1.2s, v0.2s
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v0.2s, v1.8b, v2.8b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
   %u.wide = zext <8 x i8> %u to <8 x i32>
   %s.wide = sext <8 x i8> %s to <8 x i32>
@@ -347,14 +321,7 @@ define <4 x i32> @sudot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) #0{
 ;
 ; CHECK-NEWLOWERING-I8MM-LABEL: sudot:
 ; CHECK-NEWLOWERING-I8MM:       // %bb.0:
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v3.8h, v1.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v4.4h, v3.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.4s, v4.8h, v3.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v2.4h, v1.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.4s, v2.8h, v1.8h
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v0.4s, v2.16b, v1.16b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
   %s.wide = sext <16 x i8> %u to <16 x i32>
   %u.wide = zext <16 x i8> %s to <16 x i32>
@@ -413,15 +380,8 @@ define <4 x i32> @sudot_in_loop(ptr %p1, ptr %p2){
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ldr q3, [x1, x8]
 ; CHECK-NEWLOWERING-I8MM-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    add x8, x8, #16
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v5.8h, v3.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v1.4s, v2.16b, v3.16b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    cmp x8, #16
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.4s, v4.4h, v5.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.4s, v4.8h, v5.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.4s, v2.4h, v3.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.4s, v2.8h, v3.8h
 ; CHECK-NEWLOWERING-I8MM-NEXT:    b.ne .LBB9_1
 ; CHECK-NEWLOWERING-I8MM-NEXT:  // %bb.2: // %end
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
@@ -472,19 +432,7 @@ define <2 x i32> @sudot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
 ;
 ; CHECK-NEWLOWERING-I8MM-LABEL: sudot_narrow:
 ; CHECK-NEWLOWERING-I8MM:       // %bb.0:
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEWLOWERING-I8MM-NEXT:    smull v3.4s, v2.4h, v1.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v2.4h, v1.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    smull2 v1.4s, v2.8h, v1.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v3.16b, v3.16b, v3.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-NEWLOWERING-I8MM-NEXT:    add v0.2s, v3.2s, v0.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v5.4h, v4.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    add v0.2s, v1.2s, v0.2s
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v0.2s, v2.8b, v1.8b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
   %u.wide = sext <8 x i8> %u to <8 x i32>
   %s.wide = zext <8 x i8> %s to <8 x i32>
@@ -614,26 +562,10 @@ define <4 x i64> @usdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
 ;
 ; CHECK-NEWLOWERING-I8MM-LABEL: usdot_8to64:
 ; CHECK-NEWLOWERING-I8MM:       // %bb.0: // %entry
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v5.8h, v3.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v6.4s, v4.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v7.4s, v2.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v16.4s, v5.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v17.4s, v3.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v4.4s, v4.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v5.4s, v5.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v3.4s, v3.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.2d, v6.2s, v16.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.2d, v7.2s, v17.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.2d, v6.4s, v16.4s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.2d, v7.4s, v17.4s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.2d, v4.2s, v5.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.2d, v2.2s, v3.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.2d, v4.4s, v5.4s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.2d, v2.4s, v3.4s
+; CHECK-NEWLOWERING-I8MM-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v4.4s, v2.16b, v3.16b
+; CHECK-NEWLOWERING-I8MM-NEXT:    saddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-I8MM-NEXT:    saddw2 v0.2d, v0.2d, v4.4s
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
 entry:
   %a.wide = zext <16 x i8> %a to <16 x i64>
@@ -679,26 +611,10 @@ define <4 x i64> @sudot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
 ;
 ; CHECK-NEWLOWERING-I8MM-LABEL: sudot_8to64:
 ; CHECK-NEWLOWERING-I8MM:       // %bb.0: // %entry
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v4.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v5.8h, v3.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v6.4s, v4.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v7.4s, v2.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v16.4s, v5.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v17.4s, v3.4h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v4.4s, v4.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v5.4s, v5.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v3.4s, v3.8h, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.2d, v6.2s, v16.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.2d, v7.2s, v17.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.2d, v6.4s, v16.4s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.2d, v7.4s, v17.4s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.2d, v4.2s, v5.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.2d, v2.2s, v3.2s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.2d, v4.4s, v5.4s
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.2d, v2.4s, v3.4s
+; CHECK-NEWLOWERING-I8MM-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v4.4s, v3.16b, v2.16b
+; CHECK-NEWLOWERING-I8MM-NEXT:    saddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-I8MM-NEXT:    saddw2 v0.2d, v0.2d, v4.4s
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ret
 entry:
   %a.wide = sext <16 x i8> %a to <16 x i64>
@@ -1147,21 +1063,9 @@ define <4 x i32> @usdot_multiple_zext_users(ptr %p1, ptr %p2, ptr %p3) {
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ldr q3, [x1, x8]
 ; CHECK-NEWLOWERING-I8MM-NEXT:    ldr q4, [x2, x8]
 ; CHECK-NEWLOWERING-I8MM-NEXT:    add x8, x8, #16
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v5.8h, v2.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll v6.8h, v4.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll v7.8h, v3.8b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    ushll2 v4.8h, v4.16b, #0
-; CHECK-NEWLOWERING-I8MM-NEXT:    sshll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v0.4s, v4.16b, v2.16b
+; CHECK-NEWLOWERING-I8MM-NEXT:    usdot v1.4s, v4.16b, v3.16b
 ; CHECK-NEWLOWERING-I8MM-NEXT:    cmp x8, #1024
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v5.4h, v6.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.4s, v7.4h, v6.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.4s, v5.8h, v6.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.4s, v7.8h, v6.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v0.4s, v2.4h, v4.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal v1.4s, v3.4h, v4.4h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v0.4s, v2.8h, v4.8h
-; CHECK-NEWLOWERING-I8MM-NEXT:    smlal2 v1.4s, v3.8h, v4.8h
 ; CHECK-NEWLOWERING-I8MM-NEXT:    b.ne .LBB28_1
 ; CHECK-NEWLOWERING-I8MM-NEXT:  // %bb.2: // %end
 ; CHECK-NEWLOWERING-I8MM-NEXT:    add v0.4s, v1.4s, v0.4s
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
index e59d9098a30d..ca74b4e95b0a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
@@ -112,3 +112,357 @@ define void @histogram_i32_literal_alltruemask(ptr %base, <4 x i32> %indices) {
   call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret void
 }
+
+define void @histogram_uadd_sat_i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask) {
+; CHECK-LABEL: histogram_uadd_sat_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    tbnz w8, #0, .LBB3_3
+; CHECK-NEXT:  // %bb.1: // %else
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbnz w8, #0, .LBB3_4
+; CHECK-NEXT:  .LBB3_2: // %else2
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB3_3: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    adds x9, x9, x0
+; CHECK-NEXT:    csinv x9, x9, xzr, lo
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbz w8, #0, .LBB3_2
+; CHECK-NEXT:  .LBB3_4: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    adds x9, x9, x0
+; CHECK-NEXT:    csinv x9, x9, xzr, lo
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    ret
+  call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask)
+  ret void
+}
+
+define void @histogram_uadd_sat_i32_literal(ptr %base, <4 x i32> %indices, <4 x i1> %mask) {
+; CHECK-LABEL: histogram_uadd_sat_i32_literal:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v2.2d, x0
+; CHECK-NEXT:    sshll v3.2d, v0.2s, #2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    umov w8, v1.h[0]
+; CHECK-NEXT:    add v3.2d, v2.2d, v3.2d
+; CHECK-NEXT:    tbz w8, #0, .LBB4_2
+; CHECK-NEXT:  // %bb.1: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d3
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    adds w9, w9, #1
+; CHECK-NEXT:    csinv w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB4_2: // %else
+; CHECK-NEXT:    umov w8, v1.h[1]
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    tbz w8, #0, .LBB4_4
+; CHECK-NEXT:  // %bb.3: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v3.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    adds w9, w9, #1
+; CHECK-NEXT:    csinv w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB4_4: // %else2
+; CHECK-NEXT:    umov w8, v1.h[2]
+; CHECK-NEXT:    add v0.2d, v2.2d, v0.2d
+; CHECK-NEXT:    tbnz w8, #0, .LBB4_7
+; CHECK-NEXT:  // %bb.5: // %else4
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbnz w8, #0, .LBB4_8
+; CHECK-NEXT:  .LBB4_6: // %else6
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB4_7: // %cond.histogram.update3
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    adds w9, w9, #1
+; CHECK-NEXT:    csinv w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbz w8, #0, .LBB4_6
+; CHECK-NEXT:  .LBB4_8: // %cond.histogram.update5
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    adds w9, w9, #1
+; CHECK-NEXT:    csinv w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ret
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+  ret void
+}
+
+define void @histogram_uadd_sat_i32_literal_alltruemask(ptr %base, <4 x i32> %indices) {
+; CHECK-LABEL: histogram_uadd_sat_i32_literal_alltruemask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v1.2d, x0
+; CHECK-NEXT:    sshll v2.2d, v0.2s, #2
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    add v2.2d, v1.2d, v2.2d
+; CHECK-NEXT:    add v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    fmov x8, d2
+; CHECK-NEXT:    mov x10, v2.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    adds w9, w9, #1
+; CHECK-NEXT:    csinv w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ldr w8, [x10]
+; CHECK-NEXT:    adds w8, w8, #1
+; CHECK-NEXT:    csinv w8, w8, wzr, lo
+; CHECK-NEXT:    str w8, [x10]
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    mov x10, v0.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    adds w9, w9, #1
+; CHECK-NEXT:    csinv w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ldr w8, [x10]
+; CHECK-NEXT:    adds w8, w8, #1
+; CHECK-NEXT:    csinv w8, w8, wzr, lo
+; CHECK-NEXT:    str w8, [x10]
+; CHECK-NEXT:    ret
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
+define void @histogram_umax_i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask) {
+; CHECK-LABEL: histogram_umax_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    tbnz w8, #0, .LBB6_3
+; CHECK-NEXT:  // %bb.1: // %else
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbnz w8, #0, .LBB6_4
+; CHECK-NEXT:  .LBB6_2: // %else2
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB6_3: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    cmp x9, x0
+; CHECK-NEXT:    csel x9, x9, x0, hi
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbz w8, #0, .LBB6_2
+; CHECK-NEXT:  .LBB6_4: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    cmp x9, x0
+; CHECK-NEXT:    csel x9, x9, x0, hi
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    ret
+  call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask)
+  ret void
+}
+
+define void @histogram_umax_i32_literal(ptr %base, <4 x i32> %indices, <4 x i1> %mask) {
+; CHECK-LABEL: histogram_umax_i32_literal:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v2.2d, x0
+; CHECK-NEXT:    sshll v3.2d, v0.2s, #2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    umov w8, v1.h[0]
+; CHECK-NEXT:    add v3.2d, v2.2d, v3.2d
+; CHECK-NEXT:    tbz w8, #0, .LBB7_2
+; CHECK-NEXT:  // %bb.1: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d3
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, hi
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB7_2: // %else
+; CHECK-NEXT:    umov w8, v1.h[1]
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    tbz w8, #0, .LBB7_4
+; CHECK-NEXT:  // %bb.3: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v3.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, hi
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB7_4: // %else2
+; CHECK-NEXT:    umov w8, v1.h[2]
+; CHECK-NEXT:    add v0.2d, v2.2d, v0.2d
+; CHECK-NEXT:    tbnz w8, #0, .LBB7_7
+; CHECK-NEXT:  // %bb.5: // %else4
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbnz w8, #0, .LBB7_8
+; CHECK-NEXT:  .LBB7_6: // %else6
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB7_7: // %cond.histogram.update3
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, hi
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbz w8, #0, .LBB7_6
+; CHECK-NEXT:  .LBB7_8: // %cond.histogram.update5
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, hi
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ret
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+  ret void
+}
+
+define void @histogram_umax_i32_literal_alltruemask(ptr %base, <4 x i32> %indices) {
+; CHECK-LABEL: histogram_umax_i32_literal_alltruemask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v1.2d, x0
+; CHECK-NEXT:    sshll v2.2d, v0.2s, #2
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    add v2.2d, v1.2d, v2.2d
+; CHECK-NEXT:    add v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    fmov x8, d2
+; CHECK-NEXT:    mov x10, v2.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, hi
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ldr w8, [x10]
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    csinc w8, w8, wzr, hi
+; CHECK-NEXT:    str w8, [x10]
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    mov x10, v0.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, hi
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ldr w8, [x10]
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    csinc w8, w8, wzr, hi
+; CHECK-NEXT:    str w8, [x10]
+; CHECK-NEXT:    ret
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
+define void @histogram_umin_i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask) {
+; CHECK-LABEL: histogram_umin_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    tbnz w8, #0, .LBB9_3
+; CHECK-NEXT:  // %bb.1: // %else
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbnz w8, #0, .LBB9_4
+; CHECK-NEXT:  .LBB9_2: // %else2
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB9_3: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    cmp x9, x0
+; CHECK-NEXT:    csel x9, x9, x0, lo
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbz w8, #0, .LBB9_2
+; CHECK-NEXT:  .LBB9_4: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    cmp x9, x0
+; CHECK-NEXT:    csel x9, x9, x0, lo
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    ret
+  call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask)
+  ret void
+}
+
+define void @histogram_umin_i32_literal(ptr %base, <4 x i32> %indices, <4 x i1> %mask) {
+; CHECK-LABEL: histogram_umin_i32_literal:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v2.2d, x0
+; CHECK-NEXT:    sshll v3.2d, v0.2s, #2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    umov w8, v1.h[0]
+; CHECK-NEXT:    add v3.2d, v2.2d, v3.2d
+; CHECK-NEXT:    tbz w8, #0, .LBB10_2
+; CHECK-NEXT:  // %bb.1: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d3
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB10_2: // %else
+; CHECK-NEXT:    umov w8, v1.h[1]
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    tbz w8, #0, .LBB10_4
+; CHECK-NEXT:  // %bb.3: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v3.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB10_4: // %else2
+; CHECK-NEXT:    umov w8, v1.h[2]
+; CHECK-NEXT:    add v0.2d, v2.2d, v0.2d
+; CHECK-NEXT:    tbnz w8, #0, .LBB10_7
+; CHECK-NEXT:  // %bb.5: // %else4
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbnz w8, #0, .LBB10_8
+; CHECK-NEXT:  .LBB10_6: // %else6
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB10_7: // %cond.histogram.update3
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbz w8, #0, .LBB10_6
+; CHECK-NEXT:  .LBB10_8: // %cond.histogram.update5
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ret
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+  ret void
+}
+
+define void @histogram_umin_i32_literal_alltruemask(ptr %base, <4 x i32> %indices) {
+; CHECK-LABEL: histogram_umin_i32_literal_alltruemask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v1.2d, x0
+; CHECK-NEXT:    sshll v2.2d, v0.2s, #2
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    add v2.2d, v1.2d, v2.2d
+; CHECK-NEXT:    add v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    fmov x8, d2
+; CHECK-NEXT:    mov x10, v2.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ldr w8, [x10]
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    csinc w8, w8, wzr, lo
+; CHECK-NEXT:    str w8, [x10]
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    mov x10, v0.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    csinc w9, w9, wzr, lo
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ldr w8, [x10]
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    csinc w8, w8, wzr, lo
+; CHECK-NEXT:    str w8, [x10]
+; CHECK-NEXT:    ret
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/nest-register.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/nest-register.ll
index 1e1c1b044bab..2e94dfba1fa5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/nest-register.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/nest-register.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -disable-post-ra -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu | FileCheck %s
 
 ; Tests that the 'nest' parameter attribute causes the relevant parameter to be
@@ -5,18 +6,21 @@
 
 define ptr @nest_receiver(ptr nest %arg) nounwind {
 ; CHECK-LABEL: nest_receiver:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: mov x0, x18
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x0, x15
+; CHECK-NEXT:    ret
 
   ret ptr %arg
 }
 
 define ptr @nest_caller(ptr %arg) nounwind {
 ; CHECK-LABEL: nest_caller:
-; CHECK: mov x18, x0
-; CHECK-NEXT: bl nest_receiver
-; CHECK: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    mov x15, x0
+; CHECK-NEXT:    bl nest_receiver
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
 
   %result = call ptr @nest_receiver(ptr nest %arg)
   ret ptr %result
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/nzcv-save.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/nzcv-save.ll
index c40e529ccab1..cc666dd8d34e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/nzcv-save.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/nzcv-save.ll
@@ -6,19 +6,19 @@
 define void @f(ptr nocapture %a, ptr nocapture %b, ptr nocapture %cc, ptr nocapture %dd) nounwind uwtable noinline ssp {
 ; CHECK-LABEL: f:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp x8, x10, [x2]
-; CHECK-NEXT:    ldp x9, x11, [x3]
+; CHECK-NEXT:    ldp x8, x11, [x3]
+; CHECK-NEXT:    ldp x9, x10, [x2]
 ; CHECK-NEXT:    ldp x13, x12, [x2, #16]
-; CHECK-NEXT:    adds x8, x8, x9
-; CHECK-NEXT:    ldp x14, x9, [x3, #16]
+; CHECK-NEXT:    adds x8, x9, x8
+; CHECK-NEXT:    ldp x9, x14, [x3, #16]
 ; CHECK-NEXT:    adcs x10, x10, x11
 ; CHECK-NEXT:    stp x8, x10, [x0]
-; CHECK-NEXT:    adcs x11, x13, x14
-; CHECK-NEXT:    adc x13, x12, x9
+; CHECK-NEXT:    adcs x9, x13, x9
+; CHECK-NEXT:    adc x11, x12, x14
 ; CHECK-NEXT:    orr x12, x12, #0x100
-; CHECK-NEXT:    adc x9, x12, x9
-; CHECK-NEXT:    stp x11, x13, [x0, #16]
-; CHECK-NEXT:    stp x11, x9, [x1, #16]
+; CHECK-NEXT:    stp x9, x11, [x0, #16]
+; CHECK-NEXT:    adc x11, x12, x14
+; CHECK-NEXT:    stp x9, x11, [x1, #16]
 ; CHECK-NEXT:    stp x8, x10, [x1]
 ; CHECK-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/peephole-opt-analyzeCompare-subreg-use.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/peephole-opt-analyzeCompare-subreg-use.mir
new file mode 100644
index 000000000000..c9d6bc7cc8f8
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/peephole-opt-analyzeCompare-subreg-use.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64-- -run-pass=peephole-opt -o - %s | FileCheck %s
+
+# Make sure that analyzeCompare doesn't produce illegal folds due to
+# ignoring the subregister index on the use operands.
+
+---
+name:            analyze_compare_subreg_use_lhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: analyze_compare_subreg_use_lhs
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr
+    ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[COPY]].sub_32, [[COPY2]], implicit-def dead $nzcv
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF
+    ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:gpr64 = INSERT_SUBREG [[DEF]], killed [[SUBSWrr]], %subreg.sub_32
+    ; CHECK-NEXT: [[RORVXr:%[0-9]+]]:gpr64 = RORVXr [[COPY1]], killed [[INSERT_SUBREG]]
+    ; CHECK-NEXT: $x0 = COPY [[RORVXr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:gpr64 = COPY $x1
+    %1:gpr64 = COPY $x0
+    %2:gpr32 = COPY $wzr
+    %3:gpr32 = SUBSWrr %0.sub_32, %2, implicit-def dead $nzcv
+    %4:gpr64all = IMPLICIT_DEF
+    %5:gpr64 = INSERT_SUBREG %4, killed %3, %subreg.sub_32
+    %6:gpr64 = RORVXr %1, killed %5
+    $x0 = COPY %6
+    RET_ReallyLR implicit $x0
+
+...
+
+---
+name:            analyze_compare_subreg_use_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: analyze_compare_subreg_use_rhs
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr
+    ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[COPY2]], [[COPY]].sub_32, implicit-def dead $nzcv
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF
+    ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:gpr64 = INSERT_SUBREG [[DEF]], killed [[SUBSWrr]], %subreg.sub_32
+    ; CHECK-NEXT: [[RORVXr:%[0-9]+]]:gpr64 = RORVXr [[COPY1]], killed [[INSERT_SUBREG]]
+    ; CHECK-NEXT: $x0 = COPY [[RORVXr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:gpr64 = COPY $x1
+    %1:gpr64 = COPY $x0
+    %2:gpr32 = COPY $wzr
+    %3:gpr32 = SUBSWrr %2, %0.sub_32, implicit-def dead $nzcv
+    %4:gpr64all = IMPLICIT_DEF
+    %5:gpr64 = INSERT_SUBREG %4, killed %3, %subreg.sub_32
+    %6:gpr64 = RORVXr %1, killed %5
+    $x0 = COPY %6
+    RET_ReallyLR implicit $x0
+
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/phi.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/phi.ll
index 55942d0e421b..02842c04bf7b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/phi.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/phi.ll
@@ -317,7 +317,7 @@ define <2 x i8> @tv2i8(i1 %c, ptr %p, <2 x i8> %a, <2 x i8> %b) {
 ; CHECK-GI-NEXT:  .LBB10_3: // %e
 ; CHECK-GI-NEXT:    umov w8, v1.b[0]
 ; CHECK-GI-NEXT:    umov w9, v1.b[1]
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    ret
@@ -895,26 +895,27 @@ define <3 x ptr> @tv3p0(i1 %c, ptr %p, <3 x ptr> %a, <3 x ptr> %b) {
 ;
 ; CHECK-GI-LABEL: tv3p0:
 ; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d5 killed $d5 def $q5
 ; CHECK-GI-NEXT:    tbz w0, #0, .LBB30_2
 ; CHECK-GI-NEXT:  // %bb.1: // %t
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    str wzr, [x1]
-; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    mov v5.16b, v2.16b
+; CHECK-GI-NEXT:    str wzr, [x1]
+; CHECK-GI-NEXT:    mov v0.d[1], x8
+; CHECK-GI-NEXT:    mov v3.16b, v0.16b
 ; CHECK-GI-NEXT:    b .LBB30_3
 ; CHECK-GI-NEXT:  .LBB30_2:
-; CHECK-GI-NEXT:    fmov x8, d3
-; CHECK-GI-NEXT:    fmov x9, d5
-; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    fmov x8, d4
+; CHECK-GI-NEXT:    mov v3.d[1], x8
 ; CHECK-GI-NEXT:  .LBB30_3: // %e
-; CHECK-GI-NEXT:    mov v1.d[0], x9
-; CHECK-GI-NEXT:    mov v0.d[1], x8
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    fmov x10, d1
-; CHECK-GI-NEXT:    fmov d1, d2
-; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    mov d0, v3.d[1]
+; CHECK-GI-NEXT:    fmov d2, d5
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    fmov d0, d3
+; CHECK-GI-NEXT:    fmov d1, x9
 ; CHECK-GI-NEXT:    ret
 entry:
     br i1 %c, label %t, label %e
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/popcount.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/popcount.ll
index 61f221988777..c158d8ad93b0 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/popcount.ll
@@ -652,11 +652,8 @@ define i32 @ctpop_into_extract(ptr %p) {
 ; CHECKO0-LABEL: ctpop_into_extract:
 ; CHECKO0:       // %bb.0:
 ; CHECKO0-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECKO0-NEXT:    // implicit-def: $d1
-; CHECKO0-NEXT:    // implicit-def: $q0
-; CHECKO0-NEXT:    fmov d0, d1
-; CHECKO0-NEXT:    mov v0.s[0], w8
-; CHECKO0-NEXT:    fmov d2, d0
+; CHECKO0-NEXT:    // implicit-def: $d2
+; CHECKO0-NEXT:    fmov s2, w8
 ; CHECKO0-NEXT:    ldr d0, [x0]
 ; CHECKO0-NEXT:    fmov s1, s0
 ; CHECKO0-NEXT:    fmov w8, s1
@@ -709,12 +706,12 @@ define i32 @ctpop_into_extract(ptr %p) {
 ; GISEL-LABEL: ctpop_into_extract:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr d0, [x0]
-; GISEL-NEXT:    mov w9, #-1 // =0xffffffff
 ; GISEL-NEXT:    mov x8, x0
-; GISEL-NEXT:    mov v2.s[0], w9
 ; GISEL-NEXT:    mov w0, wzr
-; GISEL-NEXT:    fmov w10, s0
-; GISEL-NEXT:    fmov s1, w10
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    fmov s1, w9
+; GISEL-NEXT:    mov w9, #-1 // =0xffffffff
+; GISEL-NEXT:    fmov s2, w9
 ; GISEL-NEXT:    cnt v1.8b, v1.8b
 ; GISEL-NEXT:    uaddlv h1, v1.8b
 ; GISEL-NEXT:    mov v2.s[1], v1.s[0]
@@ -725,11 +722,8 @@ define i32 @ctpop_into_extract(ptr %p) {
 ; GISELO0-LABEL: ctpop_into_extract:
 ; GISELO0:       // %bb.0:
 ; GISELO0-NEXT:    mov w8, #-1 // =0xffffffff
-; GISELO0-NEXT:    // implicit-def: $d1
-; GISELO0-NEXT:    // implicit-def: $q0
-; GISELO0-NEXT:    fmov d0, d1
-; GISELO0-NEXT:    mov v0.s[0], w8
-; GISELO0-NEXT:    fmov d2, d0
+; GISELO0-NEXT:    // implicit-def: $d2
+; GISELO0-NEXT:    fmov s2, w8
 ; GISELO0-NEXT:    ldr d0, [x0]
 ; GISELO0-NEXT:    fmov s1, s0
 ; GISELO0-NEXT:    fmov w8, s1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/ptradd.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/ptradd.ll
index 427542377bfd..28a8f4303765 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/ptradd.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/ptradd.ll
@@ -90,18 +90,17 @@ define <3 x ptr> @vector_gep_v3i32(<3 x ptr> %b, <3 x i32> %off) {
 ; CHECK-GI-LABEL: vector_gep_v3i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    smov x9, v3.s[0]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    smov x8, v3.s[1]
-; CHECK-GI-NEXT:    mov v4.d[0], x9
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov v0.d[1], x9
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    mov v4.d[1], x8
+; CHECK-GI-NEXT:    smov x10, v3.s[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov w8, v3.s[2]
-; CHECK-GI-NEXT:    add v0.2d, v0.2d, v4.2d
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    mov v1.d[1], x10
 ; CHECK-GI-NEXT:    add x8, x9, w8, sxtw
 ; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    add v0.2d, v0.2d, v1.2d
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -164,13 +163,12 @@ define <3 x ptr> @vector_gep_v3i64(<3 x ptr> %b, <3 x i64> %off) {
 ;
 ; CHECK-GI-LABEL: vector_gep_v3i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
 ; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
 ; CHECK-GI-NEXT:    fmov x9, d5
 ; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    fmov x8, d1
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    fmov x8, d2
 ; CHECK-GI-NEXT:    add x8, x8, x9
@@ -201,19 +199,12 @@ entry:
 }
 
 define <2 x ptr> @vector_gep_v4i128(<2 x ptr> %b, <2 x i128> %off) {
-; CHECK-SD-LABEL: vector_gep_v4i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov d1, x0
-; CHECK-SD-NEXT:    mov v1.d[1], x2
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v1.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vector_gep_v4i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.d[0], x0
-; CHECK-GI-NEXT:    mov v1.d[1], x2
-; CHECK-GI-NEXT:    add v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vector_gep_v4i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d1, x0
+; CHECK-NEXT:    mov v1.d[1], x2
+; CHECK-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    ret
 entry:
   %g = getelementptr i8, <2 x ptr> %b, <2 x i128> %off
   ret <2 x ptr> %g
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/rem.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/rem.ll
index d807635f5d87..7477d33f9aa4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/rem.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/rem.ll
@@ -190,7 +190,7 @@ define <2 x i8> @sv2i8(<2 x i8> %d, <2 x i8> %e) {
 ; CHECK-GI-NEXT:    sdiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -284,7 +284,7 @@ define <4 x i8> @sv4i8(<4 x i8> %d, <4 x i8> %e) {
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    sdiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
@@ -363,12 +363,12 @@ define <8 x i8> @sv8i8(<8 x i8> %d, <8 x i8> %e) {
 ; CHECK-GI-NEXT:    fmov w13, s1
 ; CHECK-GI-NEXT:    mov w14, v1.s[1]
 ; CHECK-GI-NEXT:    mov w15, v1.s[2]
+; CHECK-GI-NEXT:    mov w16, v1.s[3]
 ; CHECK-GI-NEXT:    sdiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v2.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v2.s[2]
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov w8, v0.s[3]
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    sdiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v2.s[3]
 ; CHECK-GI-NEXT:    mov v4.s[1], w9
@@ -381,11 +381,11 @@ define <8 x i8> @sv8i8(<8 x i8> %d, <8 x i8> %e) {
 ; CHECK-GI-NEXT:    mls v2.4s, v4.4s, v3.4s
 ; CHECK-GI-NEXT:    sdiv w13, w13, w14
 ; CHECK-GI-NEXT:    mov w14, v0.s[2]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    fmov s5, w12
 ; CHECK-GI-NEXT:    sdiv w14, w14, w15
+; CHECK-GI-NEXT:    mov w15, v0.s[3]
 ; CHECK-GI-NEXT:    mov v5.s[1], w13
-; CHECK-GI-NEXT:    sdiv w8, w8, w12
+; CHECK-GI-NEXT:    sdiv w8, w15, w16
 ; CHECK-GI-NEXT:    mov v5.s[2], w14
 ; CHECK-GI-NEXT:    mov v5.s[3], w8
 ; CHECK-GI-NEXT:    mls v0.4s, v5.4s, v1.4s
@@ -527,20 +527,20 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    mov w18, v1.s[1]
 ; CHECK-GI-NEXT:    mov w0, v1.s[2]
 ; CHECK-GI-NEXT:    mov w1, v1.s[3]
-; CHECK-GI-NEXT:    sdiv w11, w8, w9
+; CHECK-GI-NEXT:    sdiv w10, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v2.s[1]
 ; CHECK-GI-NEXT:    mov w9, v3.s[1]
 ; CHECK-GI-NEXT:    fmov w2, s7
 ; CHECK-GI-NEXT:    mov w3, v7.s[1]
 ; CHECK-GI-NEXT:    mov w4, v7.s[2]
-; CHECK-GI-NEXT:    sdiv w10, w8, w9
+; CHECK-GI-NEXT:    mov w5, v7.s[3]
+; CHECK-GI-NEXT:    sdiv w11, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v2.s[2]
 ; CHECK-GI-NEXT:    mov w9, v3.s[2]
-; CHECK-GI-NEXT:    mov v16.s[0], w11
-; CHECK-GI-NEXT:    mov w11, v6.s[3]
+; CHECK-GI-NEXT:    fmov s16, w10
 ; CHECK-GI-NEXT:    sdiv w9, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v2.s[3]
-; CHECK-GI-NEXT:    mov v16.s[1], w10
+; CHECK-GI-NEXT:    mov v16.s[1], w11
 ; CHECK-GI-NEXT:    sdiv w8, w8, w12
 ; CHECK-GI-NEXT:    fmov w12, s4
 ; CHECK-GI-NEXT:    mov v16.s[2], w9
@@ -552,8 +552,7 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    sdiv w15, w12, w13
 ; CHECK-GI-NEXT:    mov w12, v4.s[2]
 ; CHECK-GI-NEXT:    mov w13, v5.s[2]
-; CHECK-GI-NEXT:    mov v17.s[0], w14
-; CHECK-GI-NEXT:    mov w14, v7.s[3]
+; CHECK-GI-NEXT:    fmov s17, w14
 ; CHECK-GI-NEXT:    sdiv w13, w12, w13
 ; CHECK-GI-NEXT:    mov w12, v4.s[3]
 ; CHECK-GI-NEXT:    mov v17.s[1], w15
@@ -566,7 +565,7 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    mls v4.4s, v17.4s, v5.4s
 ; CHECK-GI-NEXT:    sdiv w17, w17, w18
 ; CHECK-GI-NEXT:    mov w18, v0.s[2]
-; CHECK-GI-NEXT:    mov v18.s[0], w16
+; CHECK-GI-NEXT:    fmov s18, w16
 ; CHECK-GI-NEXT:    sdiv w18, w18, w0
 ; CHECK-GI-NEXT:    mov w0, v0.s[3]
 ; CHECK-GI-NEXT:    mov v18.s[1], w17
@@ -580,10 +579,11 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v4.8h
 ; CHECK-GI-NEXT:    sdiv w2, w2, w3
 ; CHECK-GI-NEXT:    mov w3, v6.s[2]
-; CHECK-GI-NEXT:    mov v19.s[0], w1
+; CHECK-GI-NEXT:    fmov s19, w1
 ; CHECK-GI-NEXT:    sdiv w3, w3, w4
+; CHECK-GI-NEXT:    mov w4, v6.s[3]
 ; CHECK-GI-NEXT:    mov v19.s[1], w2
-; CHECK-GI-NEXT:    sdiv w10, w11, w14
+; CHECK-GI-NEXT:    sdiv w10, w4, w5
 ; CHECK-GI-NEXT:    mov v19.s[2], w3
 ; CHECK-GI-NEXT:    mov v19.s[3], w10
 ; CHECK-GI-NEXT:    mls v6.4s, v19.4s, v7.4s
@@ -866,13 +866,14 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
 ;
 ; CHECK-GI-LABEL: sv32i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-GI-NEXT:    sub sp, sp, #112
+; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-GI-NEXT:    .cfi_offset w19, -8
 ; CHECK-GI-NEXT:    .cfi_offset w20, -16
 ; CHECK-GI-NEXT:    .cfi_offset w21, -24
@@ -901,41 +902,43 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    fmov w9, s7
 ; CHECK-GI-NEXT:    mov w12, v7.s[3]
 ; CHECK-GI-NEXT:    fmov w13, s5
+; CHECK-GI-NEXT:    mov w14, v5.s[1]
 ; CHECK-GI-NEXT:    mov w16, v5.s[3]
 ; CHECK-GI-NEXT:    fmov w6, s19
 ; CHECK-GI-NEXT:    mov w7, v19.s[3]
 ; CHECK-GI-NEXT:    fmov w21, s17
-; CHECK-GI-NEXT:    mov w23, v17.s[3]
-; CHECK-GI-NEXT:    sdiv w11, w8, w9
+; CHECK-GI-NEXT:    sdiv w10, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v6.s[1]
 ; CHECK-GI-NEXT:    mov w9, v7.s[1]
-; CHECK-GI-NEXT:    sdiv w10, w8, w9
+; CHECK-GI-NEXT:    mov w22, v17.s[3]
+; CHECK-GI-NEXT:    sdiv w11, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v6.s[2]
 ; CHECK-GI-NEXT:    mov w9, v7.s[2]
-; CHECK-GI-NEXT:    mov v20.s[0], w11
+; CHECK-GI-NEXT:    fmov s20, w10
 ; CHECK-GI-NEXT:    sdiv w9, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v6.s[3]
 ; CHECK-GI-NEXT:    sshll2 v6.8h, v0.16b, #0
-; CHECK-GI-NEXT:    mov v20.s[1], w10
+; CHECK-GI-NEXT:    mov v20.s[1], w11
 ; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
 ; CHECK-GI-NEXT:    sshll v28.4s, v0.4h, #0
 ; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
 ; CHECK-GI-NEXT:    sdiv w8, w8, w12
 ; CHECK-GI-NEXT:    fmov w12, s4
 ; CHECK-GI-NEXT:    mov v20.s[2], w9
-; CHECK-GI-NEXT:    sdiv w15, w12, w13
+; CHECK-GI-NEXT:    sdiv w13, w12, w13
 ; CHECK-GI-NEXT:    mov w12, v4.s[1]
-; CHECK-GI-NEXT:    mov w13, v5.s[1]
-; CHECK-GI-NEXT:    mov v20.s[3], w8
-; CHECK-GI-NEXT:    sdiv w14, w12, w13
+; CHECK-GI-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
+; CHECK-GI-NEXT:    ldr w11, [sp, #12] // 4-byte Folded Reload
+; CHECK-GI-NEXT:    mov v20.s[3], w11
+; CHECK-GI-NEXT:    sdiv w15, w12, w14
 ; CHECK-GI-NEXT:    mov w12, v4.s[2]
-; CHECK-GI-NEXT:    mov w13, v5.s[2]
+; CHECK-GI-NEXT:    mov w14, v5.s[2]
 ; CHECK-GI-NEXT:    sshll v5.4s, v6.4h, #0
-; CHECK-GI-NEXT:    mov v21.s[0], w15
-; CHECK-GI-NEXT:    sdiv w13, w12, w13
+; CHECK-GI-NEXT:    fmov s21, w13
+; CHECK-GI-NEXT:    sdiv w14, w12, w14
 ; CHECK-GI-NEXT:    mov w12, v4.s[3]
 ; CHECK-GI-NEXT:    sshll2 v4.8h, v2.16b, #0
-; CHECK-GI-NEXT:    mov v21.s[1], w14
+; CHECK-GI-NEXT:    mov v21.s[1], w15
 ; CHECK-GI-NEXT:    sshll v2.8h, v2.8b, #0
 ; CHECK-GI-NEXT:    sshll v7.4s, v4.4h, #0
 ; CHECK-GI-NEXT:    sshll v30.4s, v2.4h, #0
@@ -944,72 +947,72 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    mls v28.4s, v20.4s, v30.4s
 ; CHECK-GI-NEXT:    sdiv w12, w12, w16
 ; CHECK-GI-NEXT:    fmov w16, s5
-; CHECK-GI-NEXT:    mov v21.s[2], w13
-; CHECK-GI-NEXT:    sdiv w1, w16, w17
+; CHECK-GI-NEXT:    mov v21.s[2], w14
+; CHECK-GI-NEXT:    sdiv w18, w16, w17
 ; CHECK-GI-NEXT:    mov w16, v5.s[1]
 ; CHECK-GI-NEXT:    mov w17, v7.s[1]
 ; CHECK-GI-NEXT:    mov v21.s[3], w12
 ; CHECK-GI-NEXT:    mls v0.4s, v21.4s, v2.4s
-; CHECK-GI-NEXT:    sdiv w0, w16, w17
+; CHECK-GI-NEXT:    sdiv w1, w16, w17
 ; CHECK-GI-NEXT:    mov w16, v5.s[2]
 ; CHECK-GI-NEXT:    mov w17, v7.s[2]
-; CHECK-GI-NEXT:    mov v22.s[0], w1
+; CHECK-GI-NEXT:    fmov s22, w18
 ; CHECK-GI-NEXT:    uzp1 v0.8h, v28.8h, v0.8h
-; CHECK-GI-NEXT:    sdiv w18, w16, w17
+; CHECK-GI-NEXT:    sdiv w0, w16, w17
 ; CHECK-GI-NEXT:    mov w16, v5.s[3]
 ; CHECK-GI-NEXT:    mov w17, v7.s[3]
 ; CHECK-GI-NEXT:    sshll2 v5.4s, v6.8h, #0
 ; CHECK-GI-NEXT:    sshll2 v7.4s, v4.8h, #0
-; CHECK-GI-NEXT:    mov v22.s[1], w0
+; CHECK-GI-NEXT:    mov v22.s[1], w1
 ; CHECK-GI-NEXT:    sshll v6.4s, v6.4h, #0
 ; CHECK-GI-NEXT:    sshll v4.4s, v4.4h, #0
 ; CHECK-GI-NEXT:    fmov w2, s7
-; CHECK-GI-NEXT:    mov w4, v7.s[3]
+; CHECK-GI-NEXT:    mov w3, v7.s[3]
 ; CHECK-GI-NEXT:    sdiv w16, w16, w17
 ; CHECK-GI-NEXT:    fmov w17, s5
-; CHECK-GI-NEXT:    mov v22.s[2], w18
+; CHECK-GI-NEXT:    mov v22.s[2], w0
 ; CHECK-GI-NEXT:    sdiv w5, w17, w2
 ; CHECK-GI-NEXT:    mov w17, v5.s[1]
 ; CHECK-GI-NEXT:    mov w2, v7.s[1]
 ; CHECK-GI-NEXT:    mov v22.s[3], w16
 ; CHECK-GI-NEXT:    mls v6.4s, v22.4s, v4.4s
-; CHECK-GI-NEXT:    sdiv w3, w17, w2
+; CHECK-GI-NEXT:    sdiv w4, w17, w2
 ; CHECK-GI-NEXT:    mov w17, v5.s[2]
 ; CHECK-GI-NEXT:    mov w2, v7.s[2]
-; CHECK-GI-NEXT:    mov v23.s[0], w5
+; CHECK-GI-NEXT:    fmov s23, w5
 ; CHECK-GI-NEXT:    sdiv w2, w17, w2
 ; CHECK-GI-NEXT:    mov w17, v5.s[3]
-; CHECK-GI-NEXT:    mov v23.s[1], w3
-; CHECK-GI-NEXT:    sdiv w17, w17, w4
-; CHECK-GI-NEXT:    fmov w4, s18
+; CHECK-GI-NEXT:    mov v23.s[1], w4
+; CHECK-GI-NEXT:    sdiv w17, w17, w3
+; CHECK-GI-NEXT:    fmov w3, s18
 ; CHECK-GI-NEXT:    mov v23.s[2], w2
-; CHECK-GI-NEXT:    sdiv w20, w4, w6
-; CHECK-GI-NEXT:    mov w4, v18.s[1]
+; CHECK-GI-NEXT:    sdiv w20, w3, w6
+; CHECK-GI-NEXT:    mov w3, v18.s[1]
 ; CHECK-GI-NEXT:    mov w6, v19.s[1]
 ; CHECK-GI-NEXT:    mov v23.s[3], w17
 ; CHECK-GI-NEXT:    mls v5.4s, v23.4s, v7.4s
-; CHECK-GI-NEXT:    sdiv w19, w4, w6
-; CHECK-GI-NEXT:    mov w4, v18.s[2]
+; CHECK-GI-NEXT:    sdiv w19, w3, w6
+; CHECK-GI-NEXT:    mov w3, v18.s[2]
 ; CHECK-GI-NEXT:    mov w6, v19.s[2]
-; CHECK-GI-NEXT:    mov v24.s[0], w20
+; CHECK-GI-NEXT:    fmov s24, w20
 ; CHECK-GI-NEXT:    uzp1 v2.8h, v6.8h, v5.8h
 ; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    sdiv w6, w4, w6
-; CHECK-GI-NEXT:    mov w4, v18.s[3]
+; CHECK-GI-NEXT:    sdiv w6, w3, w6
+; CHECK-GI-NEXT:    mov w3, v18.s[3]
 ; CHECK-GI-NEXT:    mov v24.s[1], w19
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    sdiv w4, w4, w7
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    sdiv w3, w3, w7
 ; CHECK-GI-NEXT:    fmov w7, s16
 ; CHECK-GI-NEXT:    mov v24.s[2], w6
-; CHECK-GI-NEXT:    sdiv w24, w7, w21
+; CHECK-GI-NEXT:    sdiv w23, w7, w21
 ; CHECK-GI-NEXT:    mov w7, v16.s[1]
 ; CHECK-GI-NEXT:    mov w21, v17.s[1]
-; CHECK-GI-NEXT:    mov v24.s[3], w4
-; CHECK-GI-NEXT:    sdiv w22, w7, w21
+; CHECK-GI-NEXT:    mov v24.s[3], w3
+; CHECK-GI-NEXT:    sdiv w24, w7, w21
 ; CHECK-GI-NEXT:    mov w7, v16.s[2]
 ; CHECK-GI-NEXT:    mov w21, v17.s[2]
 ; CHECK-GI-NEXT:    sshll2 v17.8h, v1.16b, #0
-; CHECK-GI-NEXT:    mov v25.s[0], w24
+; CHECK-GI-NEXT:    fmov s25, w23
 ; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
 ; CHECK-GI-NEXT:    sshll v18.4s, v17.4h, #0
 ; CHECK-GI-NEXT:    sshll v29.4s, v1.4h, #0
@@ -1017,8 +1020,9 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    sdiv w21, w7, w21
 ; CHECK-GI-NEXT:    mov w7, v16.s[3]
 ; CHECK-GI-NEXT:    sshll2 v16.8h, v3.16b, #0
-; CHECK-GI-NEXT:    mov v25.s[1], w22
+; CHECK-GI-NEXT:    mov v25.s[1], w24
 ; CHECK-GI-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    sshll v19.4s, v16.4h, #0
 ; CHECK-GI-NEXT:    sshll v31.4s, v3.4h, #0
 ; CHECK-GI-NEXT:    sshll2 v3.4s, v3.8h, #0
@@ -1028,51 +1032,51 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    mov w28, v19.s[3]
 ; CHECK-GI-NEXT:    sshll2 v19.4s, v16.8h, #0
 ; CHECK-GI-NEXT:    sshll v16.4s, v16.4h, #0
-; CHECK-GI-NEXT:    sdiv w7, w7, w23
-; CHECK-GI-NEXT:    fmov w23, s18
+; CHECK-GI-NEXT:    sdiv w7, w7, w22
+; CHECK-GI-NEXT:    fmov w22, s18
 ; CHECK-GI-NEXT:    mov v25.s[2], w21
 ; CHECK-GI-NEXT:    mls v29.4s, v24.4s, v31.4s
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    fmov w29, s19
 ; CHECK-GI-NEXT:    mov w30, v19.s[1]
-; CHECK-GI-NEXT:    mov w15, v19.s[2]
-; CHECK-GI-NEXT:    sdiv w25, w23, w25
-; CHECK-GI-NEXT:    mov w23, v18.s[1]
+; CHECK-GI-NEXT:    mov w8, v19.s[2]
+; CHECK-GI-NEXT:    mov w10, v19.s[3]
+; CHECK-GI-NEXT:    sdiv w25, w22, w25
+; CHECK-GI-NEXT:    mov w22, v18.s[1]
 ; CHECK-GI-NEXT:    mov v25.s[3], w7
 ; CHECK-GI-NEXT:    mls v1.4s, v25.4s, v3.4s
-; CHECK-GI-NEXT:    sdiv w26, w23, w26
-; CHECK-GI-NEXT:    mov w23, v18.s[2]
-; CHECK-GI-NEXT:    mov v26.s[0], w25
+; CHECK-GI-NEXT:    sdiv w26, w22, w26
+; CHECK-GI-NEXT:    mov w22, v18.s[2]
+; CHECK-GI-NEXT:    fmov s26, w25
 ; CHECK-GI-NEXT:    uzp1 v1.8h, v29.8h, v1.8h
-; CHECK-GI-NEXT:    sdiv w27, w23, w27
-; CHECK-GI-NEXT:    mov w23, v18.s[3]
+; CHECK-GI-NEXT:    sdiv w27, w22, w27
+; CHECK-GI-NEXT:    mov w22, v18.s[3]
 ; CHECK-GI-NEXT:    sshll2 v18.4s, v17.8h, #0
 ; CHECK-GI-NEXT:    mov v26.s[1], w26
 ; CHECK-GI-NEXT:    sshll v17.4s, v17.4h, #0
-; CHECK-GI-NEXT:    ldp x26, x25, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w11, v18.s[2]
+; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w9, v18.s[3]
-; CHECK-GI-NEXT:    sdiv w23, w23, w28
+; CHECK-GI-NEXT:    sdiv w22, w22, w28
 ; CHECK-GI-NEXT:    fmov w28, s18
 ; CHECK-GI-NEXT:    mov v26.s[2], w27
 ; CHECK-GI-NEXT:    sdiv w28, w28, w29
 ; CHECK-GI-NEXT:    mov w29, v18.s[1]
-; CHECK-GI-NEXT:    mov v26.s[3], w23
-; CHECK-GI-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v26.s[3], w22
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mls v17.4s, v26.4s, v16.4s
 ; CHECK-GI-NEXT:    sdiv w29, w29, w30
-; CHECK-GI-NEXT:    mov v27.s[0], w28
-; CHECK-GI-NEXT:    ldp x28, x27, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    sdiv w10, w11, w15
-; CHECK-GI-NEXT:    mov w11, v19.s[3]
+; CHECK-GI-NEXT:    mov w30, v18.s[2]
+; CHECK-GI-NEXT:    fmov s27, w28
+; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    sdiv w8, w30, w8
 ; CHECK-GI-NEXT:    mov v27.s[1], w29
-; CHECK-GI-NEXT:    sdiv w8, w9, w11
-; CHECK-GI-NEXT:    mov v27.s[2], w10
-; CHECK-GI-NEXT:    mov v27.s[3], w8
+; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    sdiv w9, w9, w10
+; CHECK-GI-NEXT:    mov v27.s[2], w8
+; CHECK-GI-NEXT:    mov v27.s[3], w9
 ; CHECK-GI-NEXT:    mls v18.4s, v27.4s, v19.4s
 ; CHECK-GI-NEXT:    uzp1 v3.8h, v17.8h, v18.8h
 ; CHECK-GI-NEXT:    uzp1 v1.16b, v1.16b, v3.16b
-; CHECK-GI-NEXT:    ldp x29, x30, [sp], #96 // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #112
 ; CHECK-GI-NEXT:    ret
 entry:
   %s = srem <32 x i8> %d, %e
@@ -1109,7 +1113,7 @@ define <2 x i8> @uv2i8(<2 x i8> %d, <2 x i8> %e) {
 ; CHECK-GI-NEXT:    udiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -1202,7 +1206,7 @@ define <4 x i8> @uv4i8(<4 x i8> %d, <4 x i8> %e) {
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    udiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
@@ -1281,12 +1285,12 @@ define <8 x i8> @uv8i8(<8 x i8> %d, <8 x i8> %e) {
 ; CHECK-GI-NEXT:    fmov w13, s1
 ; CHECK-GI-NEXT:    mov w14, v1.s[1]
 ; CHECK-GI-NEXT:    mov w15, v1.s[2]
+; CHECK-GI-NEXT:    mov w16, v1.s[3]
 ; CHECK-GI-NEXT:    udiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v2.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v2.s[2]
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov w8, v0.s[3]
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    udiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v2.s[3]
 ; CHECK-GI-NEXT:    mov v4.s[1], w9
@@ -1299,11 +1303,11 @@ define <8 x i8> @uv8i8(<8 x i8> %d, <8 x i8> %e) {
 ; CHECK-GI-NEXT:    mls v2.4s, v4.4s, v3.4s
 ; CHECK-GI-NEXT:    udiv w13, w13, w14
 ; CHECK-GI-NEXT:    mov w14, v0.s[2]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    fmov s5, w12
 ; CHECK-GI-NEXT:    udiv w14, w14, w15
+; CHECK-GI-NEXT:    mov w15, v0.s[3]
 ; CHECK-GI-NEXT:    mov v5.s[1], w13
-; CHECK-GI-NEXT:    udiv w8, w8, w12
+; CHECK-GI-NEXT:    udiv w8, w15, w16
 ; CHECK-GI-NEXT:    mov v5.s[2], w14
 ; CHECK-GI-NEXT:    mov v5.s[3], w8
 ; CHECK-GI-NEXT:    mls v0.4s, v5.4s, v1.4s
@@ -1445,20 +1449,20 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    mov w18, v1.s[1]
 ; CHECK-GI-NEXT:    mov w0, v1.s[2]
 ; CHECK-GI-NEXT:    mov w1, v1.s[3]
-; CHECK-GI-NEXT:    udiv w11, w8, w9
+; CHECK-GI-NEXT:    udiv w10, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v2.s[1]
 ; CHECK-GI-NEXT:    mov w9, v3.s[1]
 ; CHECK-GI-NEXT:    fmov w2, s7
 ; CHECK-GI-NEXT:    mov w3, v7.s[1]
 ; CHECK-GI-NEXT:    mov w4, v7.s[2]
-; CHECK-GI-NEXT:    udiv w10, w8, w9
+; CHECK-GI-NEXT:    mov w5, v7.s[3]
+; CHECK-GI-NEXT:    udiv w11, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v2.s[2]
 ; CHECK-GI-NEXT:    mov w9, v3.s[2]
-; CHECK-GI-NEXT:    mov v16.s[0], w11
-; CHECK-GI-NEXT:    mov w11, v6.s[3]
+; CHECK-GI-NEXT:    fmov s16, w10
 ; CHECK-GI-NEXT:    udiv w9, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v2.s[3]
-; CHECK-GI-NEXT:    mov v16.s[1], w10
+; CHECK-GI-NEXT:    mov v16.s[1], w11
 ; CHECK-GI-NEXT:    udiv w8, w8, w12
 ; CHECK-GI-NEXT:    fmov w12, s4
 ; CHECK-GI-NEXT:    mov v16.s[2], w9
@@ -1470,8 +1474,7 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    udiv w15, w12, w13
 ; CHECK-GI-NEXT:    mov w12, v4.s[2]
 ; CHECK-GI-NEXT:    mov w13, v5.s[2]
-; CHECK-GI-NEXT:    mov v17.s[0], w14
-; CHECK-GI-NEXT:    mov w14, v7.s[3]
+; CHECK-GI-NEXT:    fmov s17, w14
 ; CHECK-GI-NEXT:    udiv w13, w12, w13
 ; CHECK-GI-NEXT:    mov w12, v4.s[3]
 ; CHECK-GI-NEXT:    mov v17.s[1], w15
@@ -1484,7 +1487,7 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    mls v4.4s, v17.4s, v5.4s
 ; CHECK-GI-NEXT:    udiv w17, w17, w18
 ; CHECK-GI-NEXT:    mov w18, v0.s[2]
-; CHECK-GI-NEXT:    mov v18.s[0], w16
+; CHECK-GI-NEXT:    fmov s18, w16
 ; CHECK-GI-NEXT:    udiv w18, w18, w0
 ; CHECK-GI-NEXT:    mov w0, v0.s[3]
 ; CHECK-GI-NEXT:    mov v18.s[1], w17
@@ -1498,10 +1501,11 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
 ; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v4.8h
 ; CHECK-GI-NEXT:    udiv w2, w2, w3
 ; CHECK-GI-NEXT:    mov w3, v6.s[2]
-; CHECK-GI-NEXT:    mov v19.s[0], w1
+; CHECK-GI-NEXT:    fmov s19, w1
 ; CHECK-GI-NEXT:    udiv w3, w3, w4
+; CHECK-GI-NEXT:    mov w4, v6.s[3]
 ; CHECK-GI-NEXT:    mov v19.s[1], w2
-; CHECK-GI-NEXT:    udiv w10, w11, w14
+; CHECK-GI-NEXT:    udiv w10, w4, w5
 ; CHECK-GI-NEXT:    mov v19.s[2], w3
 ; CHECK-GI-NEXT:    mov v19.s[3], w10
 ; CHECK-GI-NEXT:    mls v6.4s, v19.4s, v7.4s
@@ -1784,13 +1788,14 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
 ;
 ; CHECK-GI-LABEL: uv32i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-GI-NEXT:    sub sp, sp, #112
+; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-GI-NEXT:    .cfi_offset w19, -8
 ; CHECK-GI-NEXT:    .cfi_offset w20, -16
 ; CHECK-GI-NEXT:    .cfi_offset w21, -24
@@ -1819,41 +1824,43 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    fmov w9, s7
 ; CHECK-GI-NEXT:    mov w12, v7.s[3]
 ; CHECK-GI-NEXT:    fmov w13, s5
+; CHECK-GI-NEXT:    mov w14, v5.s[1]
 ; CHECK-GI-NEXT:    mov w16, v5.s[3]
 ; CHECK-GI-NEXT:    fmov w6, s19
 ; CHECK-GI-NEXT:    mov w7, v19.s[3]
 ; CHECK-GI-NEXT:    fmov w21, s17
-; CHECK-GI-NEXT:    mov w23, v17.s[3]
-; CHECK-GI-NEXT:    udiv w11, w8, w9
+; CHECK-GI-NEXT:    udiv w10, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v6.s[1]
 ; CHECK-GI-NEXT:    mov w9, v7.s[1]
-; CHECK-GI-NEXT:    udiv w10, w8, w9
+; CHECK-GI-NEXT:    mov w22, v17.s[3]
+; CHECK-GI-NEXT:    udiv w11, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v6.s[2]
 ; CHECK-GI-NEXT:    mov w9, v7.s[2]
-; CHECK-GI-NEXT:    mov v20.s[0], w11
+; CHECK-GI-NEXT:    fmov s20, w10
 ; CHECK-GI-NEXT:    udiv w9, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v6.s[3]
 ; CHECK-GI-NEXT:    ushll2 v6.8h, v0.16b, #0
-; CHECK-GI-NEXT:    mov v20.s[1], w10
+; CHECK-GI-NEXT:    mov v20.s[1], w11
 ; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-GI-NEXT:    ushll v28.4s, v0.4h, #0
 ; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
 ; CHECK-GI-NEXT:    udiv w8, w8, w12
 ; CHECK-GI-NEXT:    fmov w12, s4
 ; CHECK-GI-NEXT:    mov v20.s[2], w9
-; CHECK-GI-NEXT:    udiv w15, w12, w13
+; CHECK-GI-NEXT:    udiv w13, w12, w13
 ; CHECK-GI-NEXT:    mov w12, v4.s[1]
-; CHECK-GI-NEXT:    mov w13, v5.s[1]
-; CHECK-GI-NEXT:    mov v20.s[3], w8
-; CHECK-GI-NEXT:    udiv w14, w12, w13
+; CHECK-GI-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
+; CHECK-GI-NEXT:    ldr w11, [sp, #12] // 4-byte Folded Reload
+; CHECK-GI-NEXT:    mov v20.s[3], w11
+; CHECK-GI-NEXT:    udiv w15, w12, w14
 ; CHECK-GI-NEXT:    mov w12, v4.s[2]
-; CHECK-GI-NEXT:    mov w13, v5.s[2]
+; CHECK-GI-NEXT:    mov w14, v5.s[2]
 ; CHECK-GI-NEXT:    ushll v5.4s, v6.4h, #0
-; CHECK-GI-NEXT:    mov v21.s[0], w15
-; CHECK-GI-NEXT:    udiv w13, w12, w13
+; CHECK-GI-NEXT:    fmov s21, w13
+; CHECK-GI-NEXT:    udiv w14, w12, w14
 ; CHECK-GI-NEXT:    mov w12, v4.s[3]
 ; CHECK-GI-NEXT:    ushll2 v4.8h, v2.16b, #0
-; CHECK-GI-NEXT:    mov v21.s[1], w14
+; CHECK-GI-NEXT:    mov v21.s[1], w15
 ; CHECK-GI-NEXT:    ushll v2.8h, v2.8b, #0
 ; CHECK-GI-NEXT:    ushll v7.4s, v4.4h, #0
 ; CHECK-GI-NEXT:    ushll v30.4s, v2.4h, #0
@@ -1862,72 +1869,72 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    mls v28.4s, v20.4s, v30.4s
 ; CHECK-GI-NEXT:    udiv w12, w12, w16
 ; CHECK-GI-NEXT:    fmov w16, s5
-; CHECK-GI-NEXT:    mov v21.s[2], w13
-; CHECK-GI-NEXT:    udiv w1, w16, w17
+; CHECK-GI-NEXT:    mov v21.s[2], w14
+; CHECK-GI-NEXT:    udiv w18, w16, w17
 ; CHECK-GI-NEXT:    mov w16, v5.s[1]
 ; CHECK-GI-NEXT:    mov w17, v7.s[1]
 ; CHECK-GI-NEXT:    mov v21.s[3], w12
 ; CHECK-GI-NEXT:    mls v0.4s, v21.4s, v2.4s
-; CHECK-GI-NEXT:    udiv w0, w16, w17
+; CHECK-GI-NEXT:    udiv w1, w16, w17
 ; CHECK-GI-NEXT:    mov w16, v5.s[2]
 ; CHECK-GI-NEXT:    mov w17, v7.s[2]
-; CHECK-GI-NEXT:    mov v22.s[0], w1
+; CHECK-GI-NEXT:    fmov s22, w18
 ; CHECK-GI-NEXT:    uzp1 v0.8h, v28.8h, v0.8h
-; CHECK-GI-NEXT:    udiv w18, w16, w17
+; CHECK-GI-NEXT:    udiv w0, w16, w17
 ; CHECK-GI-NEXT:    mov w16, v5.s[3]
 ; CHECK-GI-NEXT:    mov w17, v7.s[3]
 ; CHECK-GI-NEXT:    ushll2 v5.4s, v6.8h, #0
 ; CHECK-GI-NEXT:    ushll2 v7.4s, v4.8h, #0
-; CHECK-GI-NEXT:    mov v22.s[1], w0
+; CHECK-GI-NEXT:    mov v22.s[1], w1
 ; CHECK-GI-NEXT:    ushll v6.4s, v6.4h, #0
 ; CHECK-GI-NEXT:    ushll v4.4s, v4.4h, #0
 ; CHECK-GI-NEXT:    fmov w2, s7
-; CHECK-GI-NEXT:    mov w4, v7.s[3]
+; CHECK-GI-NEXT:    mov w3, v7.s[3]
 ; CHECK-GI-NEXT:    udiv w16, w16, w17
 ; CHECK-GI-NEXT:    fmov w17, s5
-; CHECK-GI-NEXT:    mov v22.s[2], w18
+; CHECK-GI-NEXT:    mov v22.s[2], w0
 ; CHECK-GI-NEXT:    udiv w5, w17, w2
 ; CHECK-GI-NEXT:    mov w17, v5.s[1]
 ; CHECK-GI-NEXT:    mov w2, v7.s[1]
 ; CHECK-GI-NEXT:    mov v22.s[3], w16
 ; CHECK-GI-NEXT:    mls v6.4s, v22.4s, v4.4s
-; CHECK-GI-NEXT:    udiv w3, w17, w2
+; CHECK-GI-NEXT:    udiv w4, w17, w2
 ; CHECK-GI-NEXT:    mov w17, v5.s[2]
 ; CHECK-GI-NEXT:    mov w2, v7.s[2]
-; CHECK-GI-NEXT:    mov v23.s[0], w5
+; CHECK-GI-NEXT:    fmov s23, w5
 ; CHECK-GI-NEXT:    udiv w2, w17, w2
 ; CHECK-GI-NEXT:    mov w17, v5.s[3]
-; CHECK-GI-NEXT:    mov v23.s[1], w3
-; CHECK-GI-NEXT:    udiv w17, w17, w4
-; CHECK-GI-NEXT:    fmov w4, s18
+; CHECK-GI-NEXT:    mov v23.s[1], w4
+; CHECK-GI-NEXT:    udiv w17, w17, w3
+; CHECK-GI-NEXT:    fmov w3, s18
 ; CHECK-GI-NEXT:    mov v23.s[2], w2
-; CHECK-GI-NEXT:    udiv w20, w4, w6
-; CHECK-GI-NEXT:    mov w4, v18.s[1]
+; CHECK-GI-NEXT:    udiv w20, w3, w6
+; CHECK-GI-NEXT:    mov w3, v18.s[1]
 ; CHECK-GI-NEXT:    mov w6, v19.s[1]
 ; CHECK-GI-NEXT:    mov v23.s[3], w17
 ; CHECK-GI-NEXT:    mls v5.4s, v23.4s, v7.4s
-; CHECK-GI-NEXT:    udiv w19, w4, w6
-; CHECK-GI-NEXT:    mov w4, v18.s[2]
+; CHECK-GI-NEXT:    udiv w19, w3, w6
+; CHECK-GI-NEXT:    mov w3, v18.s[2]
 ; CHECK-GI-NEXT:    mov w6, v19.s[2]
-; CHECK-GI-NEXT:    mov v24.s[0], w20
+; CHECK-GI-NEXT:    fmov s24, w20
 ; CHECK-GI-NEXT:    uzp1 v2.8h, v6.8h, v5.8h
 ; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT:    udiv w6, w4, w6
-; CHECK-GI-NEXT:    mov w4, v18.s[3]
+; CHECK-GI-NEXT:    udiv w6, w3, w6
+; CHECK-GI-NEXT:    mov w3, v18.s[3]
 ; CHECK-GI-NEXT:    mov v24.s[1], w19
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    udiv w4, w4, w7
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    udiv w3, w3, w7
 ; CHECK-GI-NEXT:    fmov w7, s16
 ; CHECK-GI-NEXT:    mov v24.s[2], w6
-; CHECK-GI-NEXT:    udiv w24, w7, w21
+; CHECK-GI-NEXT:    udiv w23, w7, w21
 ; CHECK-GI-NEXT:    mov w7, v16.s[1]
 ; CHECK-GI-NEXT:    mov w21, v17.s[1]
-; CHECK-GI-NEXT:    mov v24.s[3], w4
-; CHECK-GI-NEXT:    udiv w22, w7, w21
+; CHECK-GI-NEXT:    mov v24.s[3], w3
+; CHECK-GI-NEXT:    udiv w24, w7, w21
 ; CHECK-GI-NEXT:    mov w7, v16.s[2]
 ; CHECK-GI-NEXT:    mov w21, v17.s[2]
 ; CHECK-GI-NEXT:    ushll2 v17.8h, v1.16b, #0
-; CHECK-GI-NEXT:    mov v25.s[0], w24
+; CHECK-GI-NEXT:    fmov s25, w23
 ; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
 ; CHECK-GI-NEXT:    ushll v18.4s, v17.4h, #0
 ; CHECK-GI-NEXT:    ushll v29.4s, v1.4h, #0
@@ -1935,8 +1942,9 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    udiv w21, w7, w21
 ; CHECK-GI-NEXT:    mov w7, v16.s[3]
 ; CHECK-GI-NEXT:    ushll2 v16.8h, v3.16b, #0
-; CHECK-GI-NEXT:    mov v25.s[1], w22
+; CHECK-GI-NEXT:    mov v25.s[1], w24
 ; CHECK-GI-NEXT:    ushll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ushll v19.4s, v16.4h, #0
 ; CHECK-GI-NEXT:    ushll v31.4s, v3.4h, #0
 ; CHECK-GI-NEXT:    ushll2 v3.4s, v3.8h, #0
@@ -1946,51 +1954,51 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
 ; CHECK-GI-NEXT:    mov w28, v19.s[3]
 ; CHECK-GI-NEXT:    ushll2 v19.4s, v16.8h, #0
 ; CHECK-GI-NEXT:    ushll v16.4s, v16.4h, #0
-; CHECK-GI-NEXT:    udiv w7, w7, w23
-; CHECK-GI-NEXT:    fmov w23, s18
+; CHECK-GI-NEXT:    udiv w7, w7, w22
+; CHECK-GI-NEXT:    fmov w22, s18
 ; CHECK-GI-NEXT:    mov v25.s[2], w21
 ; CHECK-GI-NEXT:    mls v29.4s, v24.4s, v31.4s
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    fmov w29, s19
 ; CHECK-GI-NEXT:    mov w30, v19.s[1]
-; CHECK-GI-NEXT:    mov w15, v19.s[2]
-; CHECK-GI-NEXT:    udiv w25, w23, w25
-; CHECK-GI-NEXT:    mov w23, v18.s[1]
+; CHECK-GI-NEXT:    mov w8, v19.s[2]
+; CHECK-GI-NEXT:    mov w10, v19.s[3]
+; CHECK-GI-NEXT:    udiv w25, w22, w25
+; CHECK-GI-NEXT:    mov w22, v18.s[1]
 ; CHECK-GI-NEXT:    mov v25.s[3], w7
 ; CHECK-GI-NEXT:    mls v1.4s, v25.4s, v3.4s
-; CHECK-GI-NEXT:    udiv w26, w23, w26
-; CHECK-GI-NEXT:    mov w23, v18.s[2]
-; CHECK-GI-NEXT:    mov v26.s[0], w25
+; CHECK-GI-NEXT:    udiv w26, w22, w26
+; CHECK-GI-NEXT:    mov w22, v18.s[2]
+; CHECK-GI-NEXT:    fmov s26, w25
 ; CHECK-GI-NEXT:    uzp1 v1.8h, v29.8h, v1.8h
-; CHECK-GI-NEXT:    udiv w27, w23, w27
-; CHECK-GI-NEXT:    mov w23, v18.s[3]
+; CHECK-GI-NEXT:    udiv w27, w22, w27
+; CHECK-GI-NEXT:    mov w22, v18.s[3]
 ; CHECK-GI-NEXT:    ushll2 v18.4s, v17.8h, #0
 ; CHECK-GI-NEXT:    mov v26.s[1], w26
 ; CHECK-GI-NEXT:    ushll v17.4s, v17.4h, #0
-; CHECK-GI-NEXT:    ldp x26, x25, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w11, v18.s[2]
+; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov w9, v18.s[3]
-; CHECK-GI-NEXT:    udiv w23, w23, w28
+; CHECK-GI-NEXT:    udiv w22, w22, w28
 ; CHECK-GI-NEXT:    fmov w28, s18
 ; CHECK-GI-NEXT:    mov v26.s[2], w27
 ; CHECK-GI-NEXT:    udiv w28, w28, w29
 ; CHECK-GI-NEXT:    mov w29, v18.s[1]
-; CHECK-GI-NEXT:    mov v26.s[3], w23
-; CHECK-GI-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v26.s[3], w22
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mls v17.4s, v26.4s, v16.4s
 ; CHECK-GI-NEXT:    udiv w29, w29, w30
-; CHECK-GI-NEXT:    mov v27.s[0], w28
-; CHECK-GI-NEXT:    ldp x28, x27, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    udiv w10, w11, w15
-; CHECK-GI-NEXT:    mov w11, v19.s[3]
+; CHECK-GI-NEXT:    mov w30, v18.s[2]
+; CHECK-GI-NEXT:    fmov s27, w28
+; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    udiv w8, w30, w8
 ; CHECK-GI-NEXT:    mov v27.s[1], w29
-; CHECK-GI-NEXT:    udiv w8, w9, w11
-; CHECK-GI-NEXT:    mov v27.s[2], w10
-; CHECK-GI-NEXT:    mov v27.s[3], w8
+; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    udiv w9, w9, w10
+; CHECK-GI-NEXT:    mov v27.s[2], w8
+; CHECK-GI-NEXT:    mov v27.s[3], w9
 ; CHECK-GI-NEXT:    mls v18.4s, v27.4s, v19.4s
 ; CHECK-GI-NEXT:    uzp1 v3.8h, v17.8h, v18.8h
 ; CHECK-GI-NEXT:    uzp1 v1.16b, v1.16b, v3.16b
-; CHECK-GI-NEXT:    ldp x29, x30, [sp], #96 // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #112
 ; CHECK-GI-NEXT:    ret
 entry:
   %s = urem <32 x i8> %d, %e
@@ -2029,7 +2037,7 @@ define <2 x i16> @sv2i16(<2 x i16> %d, <2 x i16> %e) {
 ; CHECK-GI-NEXT:    sdiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -2129,7 +2137,7 @@ define <4 x i16> @sv4i16(<4 x i16> %d, <4 x i16> %e) {
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    sdiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
@@ -2204,12 +2212,12 @@ define <8 x i16> @sv8i16(<8 x i16> %d, <8 x i16> %e) {
 ; CHECK-GI-NEXT:    fmov w13, s1
 ; CHECK-GI-NEXT:    mov w14, v1.s[1]
 ; CHECK-GI-NEXT:    mov w15, v1.s[2]
+; CHECK-GI-NEXT:    mov w16, v1.s[3]
 ; CHECK-GI-NEXT:    sdiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v2.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v2.s[2]
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov w8, v0.s[3]
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    sdiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v2.s[3]
 ; CHECK-GI-NEXT:    mov v4.s[1], w9
@@ -2222,11 +2230,11 @@ define <8 x i16> @sv8i16(<8 x i16> %d, <8 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v2.4s, v4.4s, v3.4s
 ; CHECK-GI-NEXT:    sdiv w13, w13, w14
 ; CHECK-GI-NEXT:    mov w14, v0.s[2]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    fmov s5, w12
 ; CHECK-GI-NEXT:    sdiv w14, w14, w15
+; CHECK-GI-NEXT:    mov w15, v0.s[3]
 ; CHECK-GI-NEXT:    mov v5.s[1], w13
-; CHECK-GI-NEXT:    sdiv w8, w8, w12
+; CHECK-GI-NEXT:    sdiv w8, w15, w16
 ; CHECK-GI-NEXT:    mov v5.s[2], w14
 ; CHECK-GI-NEXT:    mov v5.s[3], w8
 ; CHECK-GI-NEXT:    mls v0.4s, v5.4s, v1.4s
@@ -2387,17 +2395,18 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mov w1, v7.s[3]
 ; CHECK-GI-NEXT:    sshll2 v7.4s, v3.8h, #0
 ; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
-; CHECK-GI-NEXT:    sdiv w11, w8, w9
+; CHECK-GI-NEXT:    sdiv w10, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v4.s[1]
 ; CHECK-GI-NEXT:    mov w9, v5.s[1]
 ; CHECK-GI-NEXT:    fmov w2, s7
 ; CHECK-GI-NEXT:    mov w3, v7.s[1]
 ; CHECK-GI-NEXT:    mov w4, v7.s[2]
-; CHECK-GI-NEXT:    sdiv w10, w8, w9
+; CHECK-GI-NEXT:    mov w5, v7.s[3]
+; CHECK-GI-NEXT:    sdiv w11, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v4.s[2]
 ; CHECK-GI-NEXT:    mov w9, v5.s[2]
 ; CHECK-GI-NEXT:    sshll2 v5.4s, v2.8h, #0
-; CHECK-GI-NEXT:    mov v16.s[0], w11
+; CHECK-GI-NEXT:    fmov s16, w10
 ; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
 ; CHECK-GI-NEXT:    fmov w13, s5
 ; CHECK-GI-NEXT:    mov w14, v5.s[1]
@@ -2406,7 +2415,7 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    sdiv w9, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v4.s[3]
 ; CHECK-GI-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-GI-NEXT:    mov v16.s[1], w10
+; CHECK-GI-NEXT:    mov v16.s[1], w11
 ; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
 ; CHECK-GI-NEXT:    sdiv w8, w8, w12
 ; CHECK-GI-NEXT:    fmov w12, s4
@@ -2417,8 +2426,7 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v0.4s, v16.4s, v2.4s
 ; CHECK-GI-NEXT:    sdiv w14, w12, w14
 ; CHECK-GI-NEXT:    mov w12, v4.s[2]
-; CHECK-GI-NEXT:    mov v17.s[0], w13
-; CHECK-GI-NEXT:    mov w13, v7.s[3]
+; CHECK-GI-NEXT:    fmov s17, w13
 ; CHECK-GI-NEXT:    sdiv w15, w12, w15
 ; CHECK-GI-NEXT:    mov w12, v4.s[3]
 ; CHECK-GI-NEXT:    mov v17.s[1], w14
@@ -2431,14 +2439,13 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v4.4s, v17.4s, v5.4s
 ; CHECK-GI-NEXT:    sdiv w17, w17, w18
 ; CHECK-GI-NEXT:    mov w18, v6.s[2]
-; CHECK-GI-NEXT:    mov v18.s[0], w16
+; CHECK-GI-NEXT:    fmov s18, w16
 ; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
 ; CHECK-GI-NEXT:    sdiv w18, w18, w0
 ; CHECK-GI-NEXT:    mov w0, v6.s[3]
 ; CHECK-GI-NEXT:    sshll2 v6.4s, v1.8h, #0
 ; CHECK-GI-NEXT:    mov v18.s[1], w17
 ; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-GI-NEXT:    mov w11, v6.s[3]
 ; CHECK-GI-NEXT:    sdiv w0, w0, w1
 ; CHECK-GI-NEXT:    fmov w1, s6
 ; CHECK-GI-NEXT:    mov v18.s[2], w18
@@ -2448,10 +2455,11 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v1.4s, v18.4s, v3.4s
 ; CHECK-GI-NEXT:    sdiv w2, w2, w3
 ; CHECK-GI-NEXT:    mov w3, v6.s[2]
-; CHECK-GI-NEXT:    mov v19.s[0], w1
+; CHECK-GI-NEXT:    fmov s19, w1
 ; CHECK-GI-NEXT:    sdiv w3, w3, w4
+; CHECK-GI-NEXT:    mov w4, v6.s[3]
 ; CHECK-GI-NEXT:    mov v19.s[1], w2
-; CHECK-GI-NEXT:    sdiv w10, w11, w13
+; CHECK-GI-NEXT:    sdiv w10, w4, w5
 ; CHECK-GI-NEXT:    mov v19.s[2], w3
 ; CHECK-GI-NEXT:    mov v19.s[3], w10
 ; CHECK-GI-NEXT:    mls v6.4s, v19.4s, v7.4s
@@ -2492,7 +2500,7 @@ define <2 x i16> @uv2i16(<2 x i16> %d, <2 x i16> %e) {
 ; CHECK-GI-NEXT:    udiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -2597,7 +2605,7 @@ define <4 x i16> @uv4i16(<4 x i16> %d, <4 x i16> %e) {
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    udiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
@@ -2672,12 +2680,12 @@ define <8 x i16> @uv8i16(<8 x i16> %d, <8 x i16> %e) {
 ; CHECK-GI-NEXT:    fmov w13, s1
 ; CHECK-GI-NEXT:    mov w14, v1.s[1]
 ; CHECK-GI-NEXT:    mov w15, v1.s[2]
+; CHECK-GI-NEXT:    mov w16, v1.s[3]
 ; CHECK-GI-NEXT:    udiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v2.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v2.s[2]
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov w8, v0.s[3]
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    udiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v2.s[3]
 ; CHECK-GI-NEXT:    mov v4.s[1], w9
@@ -2690,11 +2698,11 @@ define <8 x i16> @uv8i16(<8 x i16> %d, <8 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v2.4s, v4.4s, v3.4s
 ; CHECK-GI-NEXT:    udiv w13, w13, w14
 ; CHECK-GI-NEXT:    mov w14, v0.s[2]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    fmov s5, w12
 ; CHECK-GI-NEXT:    udiv w14, w14, w15
+; CHECK-GI-NEXT:    mov w15, v0.s[3]
 ; CHECK-GI-NEXT:    mov v5.s[1], w13
-; CHECK-GI-NEXT:    udiv w8, w8, w12
+; CHECK-GI-NEXT:    udiv w8, w15, w16
 ; CHECK-GI-NEXT:    mov v5.s[2], w14
 ; CHECK-GI-NEXT:    mov v5.s[3], w8
 ; CHECK-GI-NEXT:    mls v0.4s, v5.4s, v1.4s
@@ -2855,17 +2863,18 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mov w1, v7.s[3]
 ; CHECK-GI-NEXT:    ushll2 v7.4s, v3.8h, #0
 ; CHECK-GI-NEXT:    ushll v3.4s, v3.4h, #0
-; CHECK-GI-NEXT:    udiv w11, w8, w9
+; CHECK-GI-NEXT:    udiv w10, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v4.s[1]
 ; CHECK-GI-NEXT:    mov w9, v5.s[1]
 ; CHECK-GI-NEXT:    fmov w2, s7
 ; CHECK-GI-NEXT:    mov w3, v7.s[1]
 ; CHECK-GI-NEXT:    mov w4, v7.s[2]
-; CHECK-GI-NEXT:    udiv w10, w8, w9
+; CHECK-GI-NEXT:    mov w5, v7.s[3]
+; CHECK-GI-NEXT:    udiv w11, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v4.s[2]
 ; CHECK-GI-NEXT:    mov w9, v5.s[2]
 ; CHECK-GI-NEXT:    ushll2 v5.4s, v2.8h, #0
-; CHECK-GI-NEXT:    mov v16.s[0], w11
+; CHECK-GI-NEXT:    fmov s16, w10
 ; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
 ; CHECK-GI-NEXT:    fmov w13, s5
 ; CHECK-GI-NEXT:    mov w14, v5.s[1]
@@ -2874,7 +2883,7 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    udiv w9, w8, w9
 ; CHECK-GI-NEXT:    mov w8, v4.s[3]
 ; CHECK-GI-NEXT:    ushll2 v4.4s, v0.8h, #0
-; CHECK-GI-NEXT:    mov v16.s[1], w10
+; CHECK-GI-NEXT:    mov v16.s[1], w11
 ; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-GI-NEXT:    udiv w8, w8, w12
 ; CHECK-GI-NEXT:    fmov w12, s4
@@ -2885,8 +2894,7 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v0.4s, v16.4s, v2.4s
 ; CHECK-GI-NEXT:    udiv w14, w12, w14
 ; CHECK-GI-NEXT:    mov w12, v4.s[2]
-; CHECK-GI-NEXT:    mov v17.s[0], w13
-; CHECK-GI-NEXT:    mov w13, v7.s[3]
+; CHECK-GI-NEXT:    fmov s17, w13
 ; CHECK-GI-NEXT:    udiv w15, w12, w15
 ; CHECK-GI-NEXT:    mov w12, v4.s[3]
 ; CHECK-GI-NEXT:    mov v17.s[1], w14
@@ -2899,14 +2907,13 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v4.4s, v17.4s, v5.4s
 ; CHECK-GI-NEXT:    udiv w17, w17, w18
 ; CHECK-GI-NEXT:    mov w18, v6.s[2]
-; CHECK-GI-NEXT:    mov v18.s[0], w16
+; CHECK-GI-NEXT:    fmov s18, w16
 ; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
 ; CHECK-GI-NEXT:    udiv w18, w18, w0
 ; CHECK-GI-NEXT:    mov w0, v6.s[3]
 ; CHECK-GI-NEXT:    ushll2 v6.4s, v1.8h, #0
 ; CHECK-GI-NEXT:    mov v18.s[1], w17
 ; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-GI-NEXT:    mov w11, v6.s[3]
 ; CHECK-GI-NEXT:    udiv w0, w0, w1
 ; CHECK-GI-NEXT:    fmov w1, s6
 ; CHECK-GI-NEXT:    mov v18.s[2], w18
@@ -2916,10 +2923,11 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
 ; CHECK-GI-NEXT:    mls v1.4s, v18.4s, v3.4s
 ; CHECK-GI-NEXT:    udiv w2, w2, w3
 ; CHECK-GI-NEXT:    mov w3, v6.s[2]
-; CHECK-GI-NEXT:    mov v19.s[0], w1
+; CHECK-GI-NEXT:    fmov s19, w1
 ; CHECK-GI-NEXT:    udiv w3, w3, w4
+; CHECK-GI-NEXT:    mov w4, v6.s[3]
 ; CHECK-GI-NEXT:    mov v19.s[1], w2
-; CHECK-GI-NEXT:    udiv w10, w11, w13
+; CHECK-GI-NEXT:    udiv w10, w4, w5
 ; CHECK-GI-NEXT:    mov v19.s[2], w3
 ; CHECK-GI-NEXT:    mov v19.s[3], w10
 ; CHECK-GI-NEXT:    mls v6.4s, v19.4s, v7.4s
@@ -2958,7 +2966,7 @@ define <2 x i32> @sv2i32(<2 x i32> %d, <2 x i32> %e) {
 ; CHECK-GI-NEXT:    sdiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -2990,10 +2998,10 @@ define <3 x i32> @sv3i32(<3 x i32> %d, <3 x i32> %e) {
 ;
 ; CHECK-GI-LABEL: sv3i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v1.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    mov s0, v0.s[2]
 ; CHECK-GI-NEXT:    mov s1, v1.s[2]
 ; CHECK-GI-NEXT:    sdiv w10, w8, w9
@@ -3003,11 +3011,11 @@ define <3 x i32> @sv3i32(<3 x i32> %d, <3 x i32> %e) {
 ; CHECK-GI-NEXT:    fmov w15, s1
 ; CHECK-GI-NEXT:    sdiv w13, w11, w12
 ; CHECK-GI-NEXT:    msub w8, w10, w9, w8
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    sdiv w9, w14, w15
-; CHECK-GI-NEXT:    msub w8, w13, w12, w11
-; CHECK-GI-NEXT:    mov v0.s[1], w8
-; CHECK-GI-NEXT:    msub w8, w9, w15, w14
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    sdiv w16, w14, w15
+; CHECK-GI-NEXT:    msub w9, w13, w12, w11
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    msub w8, w16, w15, w14
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -3051,7 +3059,7 @@ define <4 x i32> @sv4i32(<4 x i32> %d, <4 x i32> %e) {
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    sdiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
@@ -3129,12 +3137,12 @@ define <8 x i32> @sv8i32(<8 x i32> %d, <8 x i32> %e) {
 ; CHECK-GI-NEXT:    fmov w13, s3
 ; CHECK-GI-NEXT:    mov w14, v3.s[1]
 ; CHECK-GI-NEXT:    mov w15, v3.s[2]
+; CHECK-GI-NEXT:    mov w16, v3.s[3]
 ; CHECK-GI-NEXT:    sdiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    sdiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov w8, v1.s[3]
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    sdiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v4.s[1], w9
@@ -3147,11 +3155,11 @@ define <8 x i32> @sv8i32(<8 x i32> %d, <8 x i32> %e) {
 ; CHECK-GI-NEXT:    mls v0.4s, v4.4s, v2.4s
 ; CHECK-GI-NEXT:    sdiv w13, w13, w14
 ; CHECK-GI-NEXT:    mov w14, v1.s[2]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov w12, v3.s[3]
+; CHECK-GI-NEXT:    fmov s5, w12
 ; CHECK-GI-NEXT:    sdiv w14, w14, w15
+; CHECK-GI-NEXT:    mov w15, v1.s[3]
 ; CHECK-GI-NEXT:    mov v5.s[1], w13
-; CHECK-GI-NEXT:    sdiv w8, w8, w12
+; CHECK-GI-NEXT:    sdiv w8, w15, w16
 ; CHECK-GI-NEXT:    mov v5.s[2], w14
 ; CHECK-GI-NEXT:    mov v5.s[3], w8
 ; CHECK-GI-NEXT:    mls v1.4s, v5.4s, v3.4s
@@ -3189,7 +3197,7 @@ define <2 x i32> @uv2i32(<2 x i32> %d, <2 x i32> %e) {
 ; CHECK-GI-NEXT:    udiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -3221,10 +3229,10 @@ define <3 x i32> @uv3i32(<3 x i32> %d, <3 x i32> %e) {
 ;
 ; CHECK-GI-LABEL: uv3i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    mov s2, v0.s[1]
 ; CHECK-GI-NEXT:    mov s3, v1.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    mov s0, v0.s[2]
 ; CHECK-GI-NEXT:    mov s1, v1.s[2]
 ; CHECK-GI-NEXT:    udiv w10, w8, w9
@@ -3234,11 +3242,11 @@ define <3 x i32> @uv3i32(<3 x i32> %d, <3 x i32> %e) {
 ; CHECK-GI-NEXT:    fmov w15, s1
 ; CHECK-GI-NEXT:    udiv w13, w11, w12
 ; CHECK-GI-NEXT:    msub w8, w10, w9, w8
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    udiv w9, w14, w15
-; CHECK-GI-NEXT:    msub w8, w13, w12, w11
-; CHECK-GI-NEXT:    mov v0.s[1], w8
-; CHECK-GI-NEXT:    msub w8, w9, w15, w14
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    udiv w16, w14, w15
+; CHECK-GI-NEXT:    msub w9, w13, w12, w11
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    msub w8, w16, w15, w14
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -3282,7 +3290,7 @@ define <4 x i32> @uv4i32(<4 x i32> %d, <4 x i32> %e) {
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    udiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
@@ -3360,12 +3368,12 @@ define <8 x i32> @uv8i32(<8 x i32> %d, <8 x i32> %e) {
 ; CHECK-GI-NEXT:    fmov w13, s3
 ; CHECK-GI-NEXT:    mov w14, v3.s[1]
 ; CHECK-GI-NEXT:    mov w15, v3.s[2]
+; CHECK-GI-NEXT:    mov w16, v3.s[3]
 ; CHECK-GI-NEXT:    udiv w8, w8, w9
 ; CHECK-GI-NEXT:    mov w9, v0.s[1]
 ; CHECK-GI-NEXT:    udiv w9, w9, w10
 ; CHECK-GI-NEXT:    mov w10, v0.s[2]
-; CHECK-GI-NEXT:    mov v4.s[0], w8
-; CHECK-GI-NEXT:    mov w8, v1.s[3]
+; CHECK-GI-NEXT:    fmov s4, w8
 ; CHECK-GI-NEXT:    udiv w10, w10, w11
 ; CHECK-GI-NEXT:    mov w11, v0.s[3]
 ; CHECK-GI-NEXT:    mov v4.s[1], w9
@@ -3378,11 +3386,11 @@ define <8 x i32> @uv8i32(<8 x i32> %d, <8 x i32> %e) {
 ; CHECK-GI-NEXT:    mls v0.4s, v4.4s, v2.4s
 ; CHECK-GI-NEXT:    udiv w13, w13, w14
 ; CHECK-GI-NEXT:    mov w14, v1.s[2]
-; CHECK-GI-NEXT:    mov v5.s[0], w12
-; CHECK-GI-NEXT:    mov w12, v3.s[3]
+; CHECK-GI-NEXT:    fmov s5, w12
 ; CHECK-GI-NEXT:    udiv w14, w14, w15
+; CHECK-GI-NEXT:    mov w15, v1.s[3]
 ; CHECK-GI-NEXT:    mov v5.s[1], w13
-; CHECK-GI-NEXT:    udiv w8, w8, w12
+; CHECK-GI-NEXT:    udiv w8, w15, w16
 ; CHECK-GI-NEXT:    mov v5.s[2], w14
 ; CHECK-GI-NEXT:    mov v5.s[3], w8
 ; CHECK-GI-NEXT:    mls v1.4s, v5.4s, v3.4s
@@ -3415,14 +3423,14 @@ define <2 x i64> @sv2i64(<2 x i64> %d, <2 x i64> %e) {
 ; CHECK-GI-NEXT:    mov x11, v0.d[1]
 ; CHECK-GI-NEXT:    sdiv x8, x8, x9
 ; CHECK-GI-NEXT:    sdiv x11, x11, x10
-; CHECK-GI-NEXT:    mov v1.d[0], x8
+; CHECK-GI-NEXT:    fmov d1, x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x11
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
-; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x11, x10
-; CHECK-GI-NEXT:    mov v1.d[0], x8
-; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x9, x11, x9
+; CHECK-GI-NEXT:    mul x8, x8, x10
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v1.d[1], x8
 ; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -3469,21 +3477,21 @@ define <3 x i64> @sv3i64(<3 x i64> %d, <3 x i64> %e) {
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NEXT:    sdiv x8, x8, x9
 ; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    fmov x11, d3
-; CHECK-GI-NEXT:    mov x14, v3.d[1]
+; CHECK-GI-NEXT:    fmov x14, d3
+; CHECK-GI-NEXT:    mov x12, v3.d[1]
 ; CHECK-GI-NEXT:    sdiv x9, x9, x10
-; CHECK-GI-NEXT:    mov v6.d[0], x8
+; CHECK-GI-NEXT:    fmov d6, x8
 ; CHECK-GI-NEXT:    fmov x8, d2
 ; CHECK-GI-NEXT:    mov v6.d[1], x9
 ; CHECK-GI-NEXT:    fmov x9, d5
-; CHECK-GI-NEXT:    sdiv x12, x8, x9
-; CHECK-GI-NEXT:    fmov x10, d6
-; CHECK-GI-NEXT:    mov x13, v6.d[1]
-; CHECK-GI-NEXT:    mul x10, x10, x11
-; CHECK-GI-NEXT:    mul x11, x13, x14
-; CHECK-GI-NEXT:    mov v2.d[0], x10
+; CHECK-GI-NEXT:    sdiv x10, x8, x9
+; CHECK-GI-NEXT:    fmov x13, d6
+; CHECK-GI-NEXT:    mov x11, v6.d[1]
+; CHECK-GI-NEXT:    mul x13, x13, x14
+; CHECK-GI-NEXT:    mul x11, x11, x12
+; CHECK-GI-NEXT:    fmov d2, x13
 ; CHECK-GI-NEXT:    mov v2.d[1], x11
-; CHECK-GI-NEXT:    msub x8, x12, x9, x8
+; CHECK-GI-NEXT:    msub x8, x10, x9, x8
 ; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v2.2d
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -3530,26 +3538,26 @@ define <4 x i64> @sv4i64(<4 x i64> %d, <4 x i64> %e) {
 ; CHECK-GI-NEXT:    mov x14, v3.d[1]
 ; CHECK-GI-NEXT:    mov x15, v1.d[1]
 ; CHECK-GI-NEXT:    sdiv x8, x8, x9
-; CHECK-GI-NEXT:    sdiv x12, x12, x13
-; CHECK-GI-NEXT:    mov v2.d[0], x8
 ; CHECK-GI-NEXT:    sdiv x11, x11, x10
-; CHECK-GI-NEXT:    mov v3.d[0], x12
-; CHECK-GI-NEXT:    sdiv x15, x15, x14
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    sdiv x12, x12, x13
 ; CHECK-GI-NEXT:    mov v2.d[1], x11
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    mov x11, v2.d[1]
-; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x10, x11, x10
-; CHECK-GI-NEXT:    mov v2.d[0], x8
-; CHECK-GI-NEXT:    mov v3.d[1], x15
-; CHECK-GI-NEXT:    mov v2.d[1], x10
-; CHECK-GI-NEXT:    fmov x9, d3
-; CHECK-GI-NEXT:    mov x12, v3.d[1]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x8, v2.d[1]
+; CHECK-GI-NEXT:    mul x9, x11, x9
+; CHECK-GI-NEXT:    mul x8, x8, x10
+; CHECK-GI-NEXT:    fmov d2, x9
+; CHECK-GI-NEXT:    mov v2.d[1], x8
+; CHECK-GI-NEXT:    sdiv x15, x15, x14
+; CHECK-GI-NEXT:    fmov d3, x12
 ; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v2.2d
-; CHECK-GI-NEXT:    mul x9, x9, x13
-; CHECK-GI-NEXT:    mul x11, x12, x14
-; CHECK-GI-NEXT:    mov v3.d[0], x9
-; CHECK-GI-NEXT:    mov v3.d[1], x11
+; CHECK-GI-NEXT:    mov v3.d[1], x15
+; CHECK-GI-NEXT:    fmov x11, d3
+; CHECK-GI-NEXT:    mov x10, v3.d[1]
+; CHECK-GI-NEXT:    mul x11, x11, x13
+; CHECK-GI-NEXT:    mul x10, x10, x14
+; CHECK-GI-NEXT:    fmov d3, x11
+; CHECK-GI-NEXT:    mov v3.d[1], x10
 ; CHECK-GI-NEXT:    sub v1.2d, v1.2d, v3.2d
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -3580,14 +3588,14 @@ define <2 x i64> @uv2i64(<2 x i64> %d, <2 x i64> %e) {
 ; CHECK-GI-NEXT:    mov x11, v0.d[1]
 ; CHECK-GI-NEXT:    udiv x8, x8, x9
 ; CHECK-GI-NEXT:    udiv x11, x11, x10
-; CHECK-GI-NEXT:    mov v1.d[0], x8
+; CHECK-GI-NEXT:    fmov d1, x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x11
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    mov x11, v1.d[1]
-; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x9, x11, x10
-; CHECK-GI-NEXT:    mov v1.d[0], x8
-; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x9, x11, x9
+; CHECK-GI-NEXT:    mul x8, x8, x10
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v1.d[1], x8
 ; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -3634,21 +3642,21 @@ define <3 x i64> @uv3i64(<3 x i64> %d, <3 x i64> %e) {
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NEXT:    udiv x8, x8, x9
 ; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    fmov x11, d3
-; CHECK-GI-NEXT:    mov x14, v3.d[1]
+; CHECK-GI-NEXT:    fmov x14, d3
+; CHECK-GI-NEXT:    mov x12, v3.d[1]
 ; CHECK-GI-NEXT:    udiv x9, x9, x10
-; CHECK-GI-NEXT:    mov v6.d[0], x8
+; CHECK-GI-NEXT:    fmov d6, x8
 ; CHECK-GI-NEXT:    fmov x8, d2
 ; CHECK-GI-NEXT:    mov v6.d[1], x9
 ; CHECK-GI-NEXT:    fmov x9, d5
-; CHECK-GI-NEXT:    udiv x12, x8, x9
-; CHECK-GI-NEXT:    fmov x10, d6
-; CHECK-GI-NEXT:    mov x13, v6.d[1]
-; CHECK-GI-NEXT:    mul x10, x10, x11
-; CHECK-GI-NEXT:    mul x11, x13, x14
-; CHECK-GI-NEXT:    mov v2.d[0], x10
+; CHECK-GI-NEXT:    udiv x10, x8, x9
+; CHECK-GI-NEXT:    fmov x13, d6
+; CHECK-GI-NEXT:    mov x11, v6.d[1]
+; CHECK-GI-NEXT:    mul x13, x13, x14
+; CHECK-GI-NEXT:    mul x11, x11, x12
+; CHECK-GI-NEXT:    fmov d2, x13
 ; CHECK-GI-NEXT:    mov v2.d[1], x11
-; CHECK-GI-NEXT:    msub x8, x12, x9, x8
+; CHECK-GI-NEXT:    msub x8, x10, x9, x8
 ; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v2.2d
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -3695,26 +3703,26 @@ define <4 x i64> @uv4i64(<4 x i64> %d, <4 x i64> %e) {
 ; CHECK-GI-NEXT:    mov x14, v3.d[1]
 ; CHECK-GI-NEXT:    mov x15, v1.d[1]
 ; CHECK-GI-NEXT:    udiv x8, x8, x9
-; CHECK-GI-NEXT:    udiv x12, x12, x13
-; CHECK-GI-NEXT:    mov v2.d[0], x8
 ; CHECK-GI-NEXT:    udiv x11, x11, x10
-; CHECK-GI-NEXT:    mov v3.d[0], x12
-; CHECK-GI-NEXT:    udiv x15, x15, x14
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    udiv x12, x12, x13
 ; CHECK-GI-NEXT:    mov v2.d[1], x11
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    mov x11, v2.d[1]
-; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    mul x10, x11, x10
-; CHECK-GI-NEXT:    mov v2.d[0], x8
-; CHECK-GI-NEXT:    mov v3.d[1], x15
-; CHECK-GI-NEXT:    mov v2.d[1], x10
-; CHECK-GI-NEXT:    fmov x9, d3
-; CHECK-GI-NEXT:    mov x12, v3.d[1]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x8, v2.d[1]
+; CHECK-GI-NEXT:    mul x9, x11, x9
+; CHECK-GI-NEXT:    mul x8, x8, x10
+; CHECK-GI-NEXT:    fmov d2, x9
+; CHECK-GI-NEXT:    mov v2.d[1], x8
+; CHECK-GI-NEXT:    udiv x15, x15, x14
+; CHECK-GI-NEXT:    fmov d3, x12
 ; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v2.2d
-; CHECK-GI-NEXT:    mul x9, x9, x13
-; CHECK-GI-NEXT:    mul x11, x12, x14
-; CHECK-GI-NEXT:    mov v3.d[0], x9
-; CHECK-GI-NEXT:    mov v3.d[1], x11
+; CHECK-GI-NEXT:    mov v3.d[1], x15
+; CHECK-GI-NEXT:    fmov x11, d3
+; CHECK-GI-NEXT:    mov x10, v3.d[1]
+; CHECK-GI-NEXT:    mul x11, x11, x13
+; CHECK-GI-NEXT:    mul x10, x10, x14
+; CHECK-GI-NEXT:    fmov d3, x11
+; CHECK-GI-NEXT:    mov v3.d[1], x10
 ; CHECK-GI-NEXT:    sub v1.2d, v1.2d, v3.2d
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat.ll
index d07fcbc29806..290031ac50ea 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat.ll
@@ -2,13 +2,6 @@
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -mtriple=aarch64-- -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-declare i4 @llvm.sadd.sat.i4(i4, i4)
-declare i8 @llvm.sadd.sat.i8(i8, i8)
-declare i16 @llvm.sadd.sat.i16(i16, i16)
-declare i32 @llvm.sadd.sat.i32(i32, i32)
-declare i64 @llvm.sadd.sat.i64(i64, i64)
-declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
-
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-SD-LABEL: func:
 ; CHECK-SD:       // %bb.0:
@@ -135,12 +128,5 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
   %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
-
-define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
-; CHECK-LABEL: vec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sqadd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
-  %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
-  ret <4 x i32> %tmp;
-}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
index e1018bbee789..1c4a504d0ab7 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
@@ -5,37 +5,6 @@
 ; CHECK-GI:       warning: Instruction selection used fallback path for v16i4
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i1
 
-declare <1 x i8> @llvm.sadd.sat.v1i8(<1 x i8>, <1 x i8>)
-declare <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <12 x i8> @llvm.sadd.sat.v12i8(<12 x i8>, <12 x i8>)
-declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>)
-declare <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)
-
-declare <1 x i16> @llvm.sadd.sat.v1i16(<1 x i16>, <1 x i16>)
-declare <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <12 x i16> @llvm.sadd.sat.v12i16(<12 x i16>, <12 x i16>)
-declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16>, <32 x i16>)
-
-declare <16 x i1> @llvm.sadd.sat.v16i1(<16 x i1>, <16 x i1>)
-declare <16 x i4> @llvm.sadd.sat.v16i4(<16 x i4>, <16 x i4>)
-
-declare <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64>, <8 x i64>)
-
-declare <4 x i24> @llvm.sadd.sat.v4i24(<4 x i24>, <4 x i24>)
-declare <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128>, <2 x i128>)
-
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
@@ -393,6 +362,34 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
   ret <16 x i1> %z
 }
 
+define void @v1i32(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ldr s1, [x1]
+; CHECK-SD-NEXT:    sqadd v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    str s0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    adds w8, w8, w9
+; CHECK-GI-NEXT:    mov w9, #-2147483648 // =0x80000000
+; CHECK-GI-NEXT:    cset w10, vs
+; CHECK-GI-NEXT:    add w9, w9, w8, asr #31
+; CHECK-GI-NEXT:    tst w10, #0x1
+; CHECK-GI-NEXT:    csel w8, w9, w8, ne
+; CHECK-GI-NEXT:    str w8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i32>, ptr %px
+  %y = load <1 x i32>, ptr %py
+  %z = call <1 x i32> @llvm.sadd.sat.v1i32(<1 x i32> %x, <1 x i32> %y)
+  store <1 x i32> %z, ptr %pz
+  ret void
+}
+
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
@@ -447,6 +444,34 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
   ret <16 x i32> %z
 }
 
+define void @v1i64(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    sqadd d0, d0, d1
+; CHECK-SD-NEXT:    str d0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr x8, [x0]
+; CHECK-GI-NEXT:    ldr x9, [x1]
+; CHECK-GI-NEXT:    adds x8, x8, x9
+; CHECK-GI-NEXT:    mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT:    cset w10, vs
+; CHECK-GI-NEXT:    add x9, x9, x8, asr #63
+; CHECK-GI-NEXT:    tst w10, #0x1
+; CHECK-GI-NEXT:    csel x8, x9, x8, ne
+; CHECK-GI-NEXT:    str x8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i64>, ptr %px
+  %y = load <1 x i64>, ptr %py
+  %z = call <1 x i64> @llvm.sadd.sat.v1i64(<1 x i64> %x, <1 x i64> %y)
+  store <1 x i64> %z, ptr %pz
+  ret void
+}
+
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/scavenge-large-call.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/scavenge-large-call.ll
index 0c9bdd098aa2..0cbdd087a5b9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/scavenge-large-call.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/scavenge-large-call.ll
@@ -4,7 +4,7 @@
 ; CHECK: add {{x[0-9]+}}, sp,
 
 define void @caller(ptr %0, i16 %1, i16 %2, i8 %3, double %4, i16 %5, i8 %6, ptr %7, double %8, i32 %9, ptr %10, double %11, double %12, [2 x i64] %13, [2 x i64] %14, [2 x i64] %15, double %16, double %17, [2 x i64] %18, [2 x i64] %19, i16 %20, i32 %21, double %22, i8 %23, [2 x i64] %24, [2 x i64] %25, [2 x i64] %26, i8 %27, i16 %28, i16 %29, i16 %30, i32 %31, [2 x i64] %32, [2 x i64] %33, [2 x i64] %34, [2 x i64] %35, [2 x i64] %36, i32 %37, i32 %38) {
-  tail call void @callee(ptr %0, i16 %1, i16 %2, i8 %3, double 0.000000e+00, i16 %5, i8 %6, ptr %7, double 0.000000e+00, i32 %9, ptr %10, double 0.000000e+00, double 0.000000e+00, [2 x i64] %13, [2 x i64] %14, [2 x i64] %15, double 0.000000e+00, double 0.000000e+00, [2 x i64] %18, [2 x i64] %19, i16 %20, i32 %21, double 0.000000e+00, i8 %23, [2 x i64] %24, [2 x i64] %25, [2 x i64] zeroinitializer, i8 %27, i16 0, i16 0, i16 %28, i32 0, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] %35, [2 x i64] %36, i32 0, i32 0)
+  call void @callee(ptr %0, i16 %1, i16 %2, i8 %3, double 0.000000e+00, i16 %5, i8 %6, ptr %7, double 0.000000e+00, i32 %9, ptr %10, double 0.000000e+00, double 0.000000e+00, [2 x i64] %13, [2 x i64] %14, [2 x i64] %15, double 0.000000e+00, double 0.000000e+00, [2 x i64] %18, [2 x i64] %19, i16 %20, i32 %21, double 0.000000e+00, i8 %23, [2 x i64] %24, [2 x i64] %25, [2 x i64] zeroinitializer, i8 %27, i16 0, i16 0, i16 %28, i32 0, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] %35, [2 x i64] %36, i32 0, i32 0)
   ret void
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/scmp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/scmp.ll
index 7a73578f43e8..73cbd2cbeee4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/scmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/scmp.ll
@@ -310,10 +310,6 @@ define <16 x i8> @signOf_neon_scmp(<8 x i16> %s0_lo, <8 x i16> %s0_hi, <8 x i16>
 ; CHECK-GI-NEXT:    cmgt v1.8h, v3.8h, v1.8h
 ; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    uzp1 v1.16b, v4.16b, v5.16b
-; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-GI-NEXT:    shl v1.16b, v1.16b, #7
-; CHECK-GI-NEXT:    sshr v0.16b, v0.16b, #7
-; CHECK-GI-NEXT:    sshr v1.16b, v1.16b, #7
 ; CHECK-GI-NEXT:    sub v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/seh-finally.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/seh-finally.ll
index 04a30800d929..fd6b3fd0bc1f 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/seh-finally.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/seh-finally.ll
@@ -38,7 +38,7 @@ entry:
 ; CHECK: add     x29, sp, #16
 ; CHECK: mov     x0, #-2
 ; CHECK: stur    x0, [x29, #16]
-; CHECK: .set .Lsimple_seh$frame_escape_0, -8
+; CHECK: .Lsimple_seh$frame_escape_0 = -8
 ; CHECK: ldur    w0, [x29, #-8]
 ; CHECK: bl      foo
 
@@ -89,7 +89,7 @@ entry:
 ; CHECK: mov     x19, sp
 ; CHECK: mov     x0, #-2
 ; CHECK: stur    x0, [x29, #24]
-; CHECK: .set .Lstack_realign$frame_escape_0, 0
+; CHECK: .Lstack_realign$frame_escape_0 = 0
 ; CHECK: ldr     w0, [x19]
 ; CHECK: bl      foo
 
@@ -137,7 +137,7 @@ entry:
 ; CHECK: add     x29, sp, #32
 ; CHECK: mov     x1, #-2
 ; CHECK: stur    x1, [x29, #16]
-; CHECK: .set .Lvla_present$frame_escape_0, -4
+; CHECK: .Lvla_present$frame_escape_0 = -4
 ; CHECK: stur    w0, [x29, #-4]
 ; CHECK: ldur    w8, [x29, #-4]
 ; CHECK: mov     x9, sp
@@ -204,7 +204,7 @@ entry:
 ; CHECK: mov     x19, sp
 ; CHECK: mov     x1, #-2
 ; CHECK: stur    x1, [x29, #24]
-; CHECK: .set .Lvla_and_realign$frame_escape_0, 32
+; CHECK: .Lvla_and_realign$frame_escape_0 = 32
 ; CHECK: str     w0, [x29, #36]
 ; CHECK: ldr     w8, [x29, #36]
 ; CHECK: mov     x9, sp
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/select_cc.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/select_cc.ll
index 66dbd4ed5a4d..73e4d4c7f0ae 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/select_cc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/select_cc.ll
@@ -129,7 +129,7 @@ define <4 x i32> @select_icmp_sgt(<4 x i32> %a, <4 x i8> %b) {
 ; CHECK-GI-NEXT:    cmgt v1.8b, v1.8b, v2.8b
 ; CHECK-GI-NEXT:    umov w8, v1.b[0]
 ; CHECK-GI-NEXT:    umov w9, v1.b[1]
-; CHECK-GI-NEXT:    mov v2.s[0], w8
+; CHECK-GI-NEXT:    fmov s2, w8
 ; CHECK-GI-NEXT:    umov w8, v1.b[2]
 ; CHECK-GI-NEXT:    mov v2.s[1], w9
 ; CHECK-GI-NEXT:    umov w9, v1.b[3]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sext.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sext.ll
index ce5b80f3e2be..ef6b65cd50a1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sext.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sext.ll
@@ -249,10 +249,10 @@ define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) {
 ; CHECK-GI-LABEL: sext_v3i8_v3i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    sxtb w8, w0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    sxtb w8, w1
-; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    sxtb w9, w1
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    sxtb w8, w2
+; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -306,7 +306,7 @@ define <3 x i32> @sext_v3i16_v3i32(<3 x i16> %a) {
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    smov w8, v0.h[0]
 ; CHECK-GI-NEXT:    smov w9, v0.h[1]
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    smov w8, v0.h[2]
 ; CHECK-GI-NEXT:    mov v1.s[1], w9
 ; CHECK-GI-NEXT:    mov v1.s[2], w8
@@ -411,10 +411,10 @@ define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) {
 ; CHECK-GI-LABEL: sext_v3i10_v3i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    sbfx w8, w0, #0, #10
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    sbfx w8, w1, #0, #10
-; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    sbfx w9, w1, #0, #10
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    sbfx w8, w2, #0, #10
+; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -1192,50 +1192,50 @@ define <16 x i64> @sext_v16i10_v16i64(<16 x i10> %a) {
 ;
 ; CHECK-GI-LABEL: sext_v16i10_v16i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[0], w0
-; CHECK-GI-NEXT:    mov v2.s[0], w2
+; CHECK-GI-NEXT:    fmov s7, w0
+; CHECK-GI-NEXT:    fmov s17, w2
 ; CHECK-GI-NEXT:    ldr s0, [sp]
-; CHECK-GI-NEXT:    mov v3.s[0], w4
-; CHECK-GI-NEXT:    mov v4.s[0], w6
-; CHECK-GI-NEXT:    ldr s5, [sp, #8]
-; CHECK-GI-NEXT:    ldr s6, [sp, #16]
-; CHECK-GI-NEXT:    ldr s7, [sp, #24]
-; CHECK-GI-NEXT:    ldr s16, [sp, #32]
-; CHECK-GI-NEXT:    ldr s17, [sp, #40]
-; CHECK-GI-NEXT:    ldr s18, [sp, #48]
-; CHECK-GI-NEXT:    ldr s19, [sp, #56]
-; CHECK-GI-NEXT:    mov v1.s[1], w1
-; CHECK-GI-NEXT:    mov v0.s[1], v5.s[0]
-; CHECK-GI-NEXT:    mov v2.s[1], w3
-; CHECK-GI-NEXT:    mov v3.s[1], w5
-; CHECK-GI-NEXT:    mov v4.s[1], w7
-; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT:    mov v16.s[1], v17.s[0]
-; CHECK-GI-NEXT:    mov v18.s[1], v19.s[0]
+; CHECK-GI-NEXT:    fmov s18, w4
+; CHECK-GI-NEXT:    fmov s19, w6
+; CHECK-GI-NEXT:    ldr s1, [sp, #8]
+; CHECK-GI-NEXT:    ldr s2, [sp, #16]
+; CHECK-GI-NEXT:    ldr s3, [sp, #24]
+; CHECK-GI-NEXT:    ldr s4, [sp, #32]
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    ldr s6, [sp, #48]
+; CHECK-GI-NEXT:    ldr s16, [sp, #56]
+; CHECK-GI-NEXT:    mov v7.s[1], w1
+; CHECK-GI-NEXT:    mov v17.s[1], w3
+; CHECK-GI-NEXT:    mov v18.s[1], w5
+; CHECK-GI-NEXT:    mov v19.s[1], w7
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT:    mov v6.s[1], v16.s[0]
+; CHECK-GI-NEXT:    ushll v1.2d, v7.2s, #0
+; CHECK-GI-NEXT:    ushll v3.2d, v17.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v19.2s, #0
 ; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
 ; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll v3.2d, v3.2s, #0
 ; CHECK-GI-NEXT:    ushll v4.2d, v4.2s, #0
-; CHECK-GI-NEXT:    ushll v5.2d, v6.2s, #0
-; CHECK-GI-NEXT:    ushll v6.2d, v16.2s, #0
-; CHECK-GI-NEXT:    ushll v7.2d, v18.2s, #0
-; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT:    ushll v6.2d, v6.2s, #0
 ; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #54
-; CHECK-GI-NEXT:    shl v2.2d, v2.2d, #54
 ; CHECK-GI-NEXT:    shl v3.2d, v3.2d, #54
-; CHECK-GI-NEXT:    shl v16.2d, v4.2d, #54
 ; CHECK-GI-NEXT:    shl v5.2d, v5.2d, #54
-; CHECK-GI-NEXT:    shl v6.2d, v6.2d, #54
 ; CHECK-GI-NEXT:    shl v7.2d, v7.2d, #54
-; CHECK-GI-NEXT:    sshr v4.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v16.2d, v0.2d, #54
+; CHECK-GI-NEXT:    shl v17.2d, v2.2d, #54
+; CHECK-GI-NEXT:    shl v18.2d, v4.2d, #54
+; CHECK-GI-NEXT:    shl v19.2d, v6.2d, #54
 ; CHECK-GI-NEXT:    sshr v0.2d, v1.2d, #54
-; CHECK-GI-NEXT:    sshr v1.2d, v2.2d, #54
-; CHECK-GI-NEXT:    sshr v2.2d, v3.2d, #54
-; CHECK-GI-NEXT:    sshr v3.2d, v16.2d, #54
-; CHECK-GI-NEXT:    sshr v5.2d, v5.2d, #54
-; CHECK-GI-NEXT:    sshr v6.2d, v6.2d, #54
-; CHECK-GI-NEXT:    sshr v7.2d, v7.2d, #54
+; CHECK-GI-NEXT:    sshr v1.2d, v3.2d, #54
+; CHECK-GI-NEXT:    sshr v2.2d, v5.2d, #54
+; CHECK-GI-NEXT:    sshr v3.2d, v7.2d, #54
+; CHECK-GI-NEXT:    sshr v4.2d, v16.2d, #54
+; CHECK-GI-NEXT:    sshr v5.2d, v17.2d, #54
+; CHECK-GI-NEXT:    sshr v6.2d, v18.2d, #54
+; CHECK-GI-NEXT:    sshr v7.2d, v19.2d, #54
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sext <16 x i10> %a to <16 x i64>
@@ -1245,12 +1245,10 @@ entry:
 define <2 x i128> @sext_v2i64_v2i128(<2 x i64> %a) {
 ; CHECK-SD-LABEL: sext_v2i64_v2i128:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    dup v1.2d, v0.d[1]
+; CHECK-SD-NEXT:    mov x2, v0.d[1]
 ; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    fmov x2, d1
 ; CHECK-SD-NEXT:    asr x1, x0, #63
-; CHECK-SD-NEXT:    asr x3, x8, #63
+; CHECK-SD-NEXT:    asr x3, x2, #63
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: sext_v2i64_v2i128:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/shift.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/shift.ll
index 5f24af82d834..9827cb3526f9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/shift.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/shift.ll
@@ -598,8 +598,7 @@ define <1 x i32> @shl_v1i32(<1 x i32> %0, <1 x i32> %1){
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    lsl w8, w8, w9
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
     %3 = shl <1 x i32> %0, %1
     ret <1 x i32> %3
@@ -775,8 +774,7 @@ define <1 x i32> @ashr_v1i32(<1 x i32> %0, <1 x i32> %1){
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    asr w8, w8, w9
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
     %3 = ashr <1 x i32> %0, %1
     ret <1 x i32> %3
@@ -948,8 +946,7 @@ define <1 x i32> @lshr_v1i32(<1 x i32> %0, <1 x i32> %1){
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    lsr w8, w8, w9
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    ret
     %3 = lshr <1 x i32> %0, %1
     ret <1 x i32> %3
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/shufflevector.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/shufflevector.ll
index e5c07e0c81e3..9fd5e6508678 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/shufflevector.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/shufflevector.ll
@@ -217,7 +217,7 @@ define <2 x i1> @shufflevector_v2i1(<2 x i1> %a, <2 x i1> %b){
 ; CHECK-GI-NEXT:    mov v0.b[1], v1.b[1]
 ; CHECK-GI-NEXT:    umov w8, v0.b[0]
 ; CHECK-GI-NEXT:    umov w9, v0.b[1]
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    ret
@@ -395,21 +395,16 @@ define <3 x ptr> @shufflevector_v3p0(<3 x ptr> %a, <3 x ptr> %b) {
 ;
 ; CHECK-GI-LABEL: shufflevector_v3p0:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d3
-; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    mov v2.d[0], x9
 ; CHECK-GI-NEXT:    fmov x8, d1
 ; CHECK-GI-NEXT:    fmov x9, d4
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-GI-NEXT:    dup v2.2d, v5.d[0]
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
-; CHECK-GI-NEXT:    mov v2.d[1], x9
-; CHECK-GI-NEXT:    fmov x8, d5
-; CHECK-GI-NEXT:    mov v1.d[0], x8
-; CHECK-GI-NEXT:    ext v0.16b, v0.16b, v2.16b, #8
-; CHECK-GI-NEXT:    fmov x10, d1
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    fmov d1, d2
-; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    mov v3.d[1], x9
+; CHECK-GI-NEXT:    ext v0.16b, v0.16b, v3.16b, #8
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    ret
     %c = shufflevector <3 x ptr> %a, <3 x ptr> %b, <3 x i32> <i32 1, i32 3, i32 5>
     ret <3 x ptr> %c
@@ -449,7 +444,7 @@ define <2 x i1> @shufflevector_v2i1_zeroes(<2 x i1> %a, <2 x i1> %b){
 ; CHECK-GI-NEXT:    dup v0.8b, v0.b[0]
 ; CHECK-GI-NEXT:    umov w8, v0.b[0]
 ; CHECK-GI-NEXT:    umov w9, v0.b[1]
-; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
new file mode 100644
index 000000000000..bcc6a3ea9b28
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
@@ -0,0 +1,109 @@
+# RUN: llc -mtriple=aarch64 -run-pass=pipeliner -debug-only=pipeliner -aarch64-enable-pipeliner -pipeliner-mve-cg %s -o /dev/null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# Test a case where fenv is enabled, there are instructions that may raise a
+# floating-point exception, and there is an instruction for barrier event. In
+# this case the order of them must not change.
+#
+# FIXME: Currently the following dependencies are missed.
+#
+# Loop carried edges from SU(7)
+#   Order
+#     SU(2)
+#     SU(3)
+#     SU(4)
+#     SU(5)
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  @x = dso_local global i32 0, align 4
+
+  define dso_local void @f(ptr nocapture noundef writeonly %a, float noundef %y, i32 noundef %n) {
+  entry:
+    %cmp6 = icmp sgt i32 %n, 0
+    br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %wide.trip.count = zext nneg i32 %n to i64
+    br label %for.body
+
+  for.cond.cleanup:
+    ret void
+
+  for.body:
+    %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+    %tmp9 = trunc i64 %indvars.iv to i32
+    %conv = tail call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %tmp9, metadata !"round.dynamic", metadata !"fpexcept.strict") #2
+    %add = tail call float @llvm.experimental.constrained.fadd.f32(float %conv, float %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #2
+    %0 = shl nuw nsw i64 %indvars.iv, 2
+    %scevgep = getelementptr i8, ptr %a, i64 %0
+    store float %add, ptr %scevgep, align 4, !tbaa !6
+    %1 = load volatile i32, ptr @x, align 4, !tbaa !10
+    %2 = zext i32 %1 to i64
+    %3 = add i64 %indvars.iv, %2
+    %tmp = trunc i64 %3 to i32
+    store volatile i32 %tmp, ptr @x, align 4, !tbaa !10
+    %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+    %exitcond.not = icmp eq i64 %wide.trip.count, %indvars.iv.next
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+
+  declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+
+  attributes #2 = { strictfp }
+
+  !6 = !{!7, !7, i64 0}
+  !7 = !{!"float", !8, i64 0}
+  !8 = !{!"omnipotent char", !9, i64 0}
+  !9 = !{!"Simple C/C++ TBAA"}
+  !10 = !{!11, !11, i64 0}
+  !11 = !{!"int", !8, i64 0}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    liveins: $x0, $s0, $w1
+  
+    %5:gpr32common = COPY $w1
+    %4:fpr32 = COPY $s0
+    %3:gpr64common = COPY $x0
+    dead $wzr = SUBSWri %5, 1, 0, implicit-def $nzcv
+    Bcc 11, %bb.2, implicit $nzcv
+    B %bb.1
+  
+  bb.1.for.body.preheader:
+    %8:gpr32 = ORRWrs $wzr, %5, 0
+    %0:gpr64 = SUBREG_TO_REG 0, killed %8, %subreg.sub_32
+    %9:gpr64all = COPY $xzr
+    %7:gpr64all = COPY %9
+    %13:gpr64common = ADRP target-flags(aarch64-page) @x
+    B %bb.3
+  
+  bb.2.for.cond.cleanup:
+    RET_ReallyLR
+  
+  bb.3.for.body:
+    successors: %bb.2, %bb.3
+  
+    %1:gpr64common = PHI %7, %bb.1, %2, %bb.3
+    %10:gpr32 = COPY %1.sub_32
+    %11:fpr32 = SCVTFUWSri %10, implicit $fpcr
+    %12:fpr32 = FADDSrr killed %11, %4, implicit $fpcr
+    STRSroX killed %12, %3, %1, 0, 1 :: (store (s32) into %ir.scevgep, !tbaa !6)
+    %14:gpr32 = LDRWui %13, target-flags(aarch64-pageoff, aarch64-nc) @x :: (volatile dereferenceable load (s32) from @x, !tbaa !10)
+    %15:gpr32 = ADDWrr %10, killed %14
+    STRWui killed %15, %13, target-flags(aarch64-pageoff, aarch64-nc) @x :: (volatile store (s32) into @x, !tbaa !10)
+    %16:gpr64common = nuw nsw ADDXri %1, 1, 0
+    %2:gpr64all = COPY %16
+    dead $xzr = SUBSXrr %0, %16, implicit-def $nzcv
+    Bcc 0, %bb.2, implicit $nzcv
+    B %bb.3
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions2.mir b/external/llvm-project/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions2.mir
new file mode 100644
index 000000000000..6116f15811ec
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions2.mir
@@ -0,0 +1,99 @@
+# RUN: llc -mtriple=aarch64 -run-pass=pipeliner -debug-only=pipeliner -aarch64-enable-pipeliner -pipeliner-mve-cg %s -o /dev/null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# Test a case where fenv is enabled, there are instructions that may raise a
+# floatin-point exception, but there is no instruction for barrier event. In
+# this case no loop-carried dependencies are necessary.
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  define dso_local float @f(ptr nocapture noundef writeonly %a, float noundef %y, i32 noundef %n) local_unnamed_addr {
+  entry:
+    %conv = tail call float @llvm.experimental.constrained.fptrunc.f32.f64(double 1.000000e+00, metadata !"round.dynamic", metadata !"fpexcept.strict")
+    %cmp8 = icmp sgt i32 %n, 0
+    br i1 %cmp8, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %wide.trip.count = zext nneg i32 %n to i64
+    br label %for.body
+
+  for.cond.cleanup:
+    %acc.0.lcssa = phi float [ %conv, %entry ], [ %mul, %for.body ]
+    ret float %acc.0.lcssa
+
+  for.body:
+    %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+    %acc.010 = phi float [ %conv, %for.body.preheader ], [ %mul, %for.body ]
+    %tmp = trunc i64 %indvars.iv to i32
+    %conv2 = tail call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %tmp, metadata !"round.dynamic", metadata !"fpexcept.strict")
+    %add = tail call float @llvm.experimental.constrained.fadd.f32(float %conv2, float %y, metadata !"round.dynamic", metadata !"fpexcept.strict")
+    %mul = tail call float @llvm.experimental.constrained.fmul.f32(float %acc.010, float %add, metadata !"round.dynamic", metadata !"fpexcept.strict")
+    %0 = shl nuw nsw i64 %indvars.iv, 2
+    %scevgep = getelementptr i8, ptr %a, i64 %0
+    store float %add, ptr %scevgep, align 4, !tbaa !6
+    %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+    %exitcond.not = icmp eq i64 %wide.trip.count, %indvars.iv.next
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+
+  declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+
+  declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+
+  declare float @llvm.experimental.constrained.fmul.f32(float, float, metadata, metadata)
+
+  !6 = !{!7, !7, i64 0}
+  !7 = !{!"float", !8, i64 0}
+  !8 = !{!"omnipotent char", !9, i64 0}
+  !9 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    liveins: $x0, $s0, $w1
+  
+    %9:gpr32common = COPY $w1
+    %8:fpr32 = COPY $s0
+    %7:gpr64common = COPY $x0
+    %10:fpr64 = FMOVDi 112
+    %0:fpr32 = FCVTSDr killed %10, implicit $fpcr
+    dead $wzr = SUBSWri %9, 1, 0, implicit-def $nzcv
+    Bcc 11, %bb.2, implicit $nzcv
+    B %bb.1
+  
+  bb.1.for.body.preheader:
+    %13:gpr32 = ORRWrs $wzr, %9, 0
+    %1:gpr64 = SUBREG_TO_REG 0, killed %13, %subreg.sub_32
+    %14:gpr64all = COPY $xzr
+    %12:gpr64all = COPY %14
+    B %bb.3
+  
+  bb.2.for.cond.cleanup:
+    %2:fpr32 = PHI %0, %bb.0, %5, %bb.3
+    $s0 = COPY %2
+    RET_ReallyLR implicit $s0
+  
+  bb.3.for.body:
+    successors: %bb.2, %bb.3
+  
+    %3:gpr64common = PHI %12, %bb.1, %6, %bb.3
+    %4:fpr32 = PHI %0, %bb.1, %5, %bb.3
+    %15:gpr32 = COPY %3.sub_32
+    %16:fpr32 = SCVTFUWSri killed %15, implicit $fpcr
+    %17:fpr32 = FADDSrr killed %16, %8, implicit $fpcr
+    %5:fpr32 = FMULSrr %4, %17, implicit $fpcr
+    STRSroX %17, %7, %3, 0, 1 :: (store (s32) into %ir.scevgep, !tbaa !6)
+    %18:gpr64common = nuw nsw ADDXri %3, 1, 0
+    %6:gpr64all = COPY %18
+    dead $xzr = SUBSXrr %1, %18, implicit-def $nzcv
+    Bcc 0, %bb.2, implicit $nzcv
+    B %bb.3
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat.ll
index 23550d3c41cc..4740803fc762 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat.ll
@@ -2,13 +2,6 @@
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -mtriple=aarch64-- -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-declare i4 @llvm.ssub.sat.i4(i4, i4)
-declare i8 @llvm.ssub.sat.i8(i8, i8)
-declare i16 @llvm.ssub.sat.i16(i16, i16)
-declare i32 @llvm.ssub.sat.i32(i32, i32)
-declare i64 @llvm.ssub.sat.i64(i64, i64)
-declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
-
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-SD-LABEL: func:
 ; CHECK-SD:       // %bb.0:
@@ -135,12 +128,5 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
   %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
-
-define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
-; CHECK-LABEL: vec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sqsub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
-  %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
-  ret <4 x i32> %tmp;
-}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
index 085857c0c542..3af858713525 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
@@ -5,38 +5,6 @@
 ; CHECK-GI:       warning: Instruction selection used fallback path for v16i4
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i1
 
-declare <1 x i8> @llvm.ssub.sat.v1i8(<1 x i8>, <1 x i8>)
-declare <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <12 x i8> @llvm.ssub.sat.v12i8(<12 x i8>, <12 x i8>)
-declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>)
-declare <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)
-
-declare <1 x i16> @llvm.ssub.sat.v1i16(<1 x i16>, <1 x i16>)
-declare <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <12 x i16> @llvm.ssub.sat.v12i16(<12 x i16>, <12 x i16>)
-declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16>, <32 x i16>)
-
-declare <16 x i1> @llvm.ssub.sat.v16i1(<16 x i1>, <16 x i1>)
-declare <16 x i4> @llvm.ssub.sat.v16i4(<16 x i4>, <16 x i4>)
-
-declare <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64>, <8 x i64>)
-
-declare <4 x i24> @llvm.ssub.sat.v4i24(<4 x i24>, <4 x i24>)
-declare <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128>, <2 x i128>)
-
-
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
@@ -396,6 +364,34 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
   ret <16 x i1> %z
 }
 
+define void @v1i32(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ldr s1, [x1]
+; CHECK-SD-NEXT:    sqsub v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    str s0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    subs w8, w8, w9
+; CHECK-GI-NEXT:    mov w9, #-2147483648 // =0x80000000
+; CHECK-GI-NEXT:    cset w10, vs
+; CHECK-GI-NEXT:    add w9, w9, w8, asr #31
+; CHECK-GI-NEXT:    tst w10, #0x1
+; CHECK-GI-NEXT:    csel w8, w9, w8, ne
+; CHECK-GI-NEXT:    str w8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i32>, ptr %px
+  %y = load <1 x i32>, ptr %py
+  %z = call <1 x i32> @llvm.ssub.sat.v1i32(<1 x i32> %x, <1 x i32> %y)
+  store <1 x i32> %z, ptr %pz
+  ret void
+}
+
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
@@ -450,6 +446,34 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
   ret <16 x i32> %z
 }
 
+define void @v1i64(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    sqsub d0, d0, d1
+; CHECK-SD-NEXT:    str d0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr x8, [x0]
+; CHECK-GI-NEXT:    ldr x9, [x1]
+; CHECK-GI-NEXT:    subs x8, x8, x9
+; CHECK-GI-NEXT:    mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT:    cset w10, vs
+; CHECK-GI-NEXT:    add x9, x9, x8, asr #63
+; CHECK-GI-NEXT:    tst w10, #0x1
+; CHECK-GI-NEXT:    csel x8, x9, x8, ne
+; CHECK-GI-NEXT:    str x8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i64>, ptr %px
+  %y = load <1 x i64>, ptr %py
+  %z = call <1 x i64> @llvm.ssub.sat.v1i64(<1 x i64> %x, <1 x i64> %y)
+  store <1 x i64> %z, ptr %pz
+  ret void
+}
+
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/stack-hazard.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/stack-hazard.ll
index 791d7580c327..e169b199733b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/stack-hazard.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/stack-hazard.ll
@@ -2905,12 +2905,13 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    mov x9, sp
 ; CHECK64-NEXT:    mov w20, w0
 ; CHECK64-NEXT:    msub x9, x8, x8, x9
+; CHECK64-NEXT:    mov x19, sp
 ; CHECK64-NEXT:    mov sp, x9
-; CHECK64-NEXT:    stur x9, [x29, #-208]
-; CHECK64-NEXT:    sub x9, x29, #208
-; CHECK64-NEXT:    sturh wzr, [x29, #-198]
-; CHECK64-NEXT:    stur wzr, [x29, #-196]
-; CHECK64-NEXT:    sturh w8, [x29, #-200]
+; CHECK64-NEXT:    str x9, [x19]
+; CHECK64-NEXT:    add x9, x19, #0
+; CHECK64-NEXT:    strh wzr, [x19, #10]
+; CHECK64-NEXT:    str wzr, [x19, #12]
+; CHECK64-NEXT:    strh w8, [x19, #8]
 ; CHECK64-NEXT:    msr TPIDR2_EL0, x9
 ; CHECK64-NEXT:    .cfi_offset vg, -32
 ; CHECK64-NEXT:    smstop sm
@@ -2919,7 +2920,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    smstart za
 ; CHECK64-NEXT:    mrs x8, TPIDR2_EL0
-; CHECK64-NEXT:    sub x0, x29, #208
+; CHECK64-NEXT:    add x0, x19, #0
 ; CHECK64-NEXT:    cbnz x8, .LBB33_2
 ; CHECK64-NEXT:  // %bb.1: // %entry
 ; CHECK64-NEXT:    bl __arm_tpidr2_restore
@@ -2985,16 +2986,13 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    mov x9, sp
 ; CHECK1024-NEXT:    mov w20, w0
 ; CHECK1024-NEXT:    msub x9, x8, x8, x9
+; CHECK1024-NEXT:    mov x19, sp
 ; CHECK1024-NEXT:    mov sp, x9
-; CHECK1024-NEXT:    sub x10, x29, #1872
-; CHECK1024-NEXT:    stur x9, [x10, #-256]
-; CHECK1024-NEXT:    sub x9, x29, #1862
-; CHECK1024-NEXT:    sub x10, x29, #1860
-; CHECK1024-NEXT:    sturh wzr, [x9, #-256]
-; CHECK1024-NEXT:    sub x9, x29, #2128
-; CHECK1024-NEXT:    stur wzr, [x10, #-256]
-; CHECK1024-NEXT:    sub x10, x29, #1864
-; CHECK1024-NEXT:    sturh w8, [x10, #-256]
+; CHECK1024-NEXT:    str x9, [x19]
+; CHECK1024-NEXT:    add x9, x19, #0
+; CHECK1024-NEXT:    strh wzr, [x19, #10]
+; CHECK1024-NEXT:    str wzr, [x19, #12]
+; CHECK1024-NEXT:    strh w8, [x19, #8]
 ; CHECK1024-NEXT:    msr TPIDR2_EL0, x9
 ; CHECK1024-NEXT:    .cfi_offset vg, -32
 ; CHECK1024-NEXT:    smstop sm
@@ -3003,7 +3001,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    smstart za
 ; CHECK1024-NEXT:    mrs x8, TPIDR2_EL0
-; CHECK1024-NEXT:    sub x0, x29, #2128
+; CHECK1024-NEXT:    add x0, x19, #0
 ; CHECK1024-NEXT:    cbnz x8, .LBB33_2
 ; CHECK1024-NEXT:  // %bb.1: // %entry
 ; CHECK1024-NEXT:    bl __arm_tpidr2_restore
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/stackguard-internal.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/stackguard-internal.ll
index a70c8874edba..7b32e8c0caab 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/stackguard-internal.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/stackguard-internal.ll
@@ -6,7 +6,7 @@ target triple = "aarch64-linux-gnu"
 ; is an alias.  (The alias is created by GlobalMerge.)
 ; CHECK: adrp {{.*}}, __stack_chk_guard
 ; CHECK: ldr {{.*}}, [{{.*}}, :lo12:__stack_chk_guard]
-; CHECK: .set __stack_chk_guard, .L_MergedGlobals+4
+; CHECK: __stack_chk_guard = .L_MergedGlobals+4
 
 @__stack_chk_guard = internal global [8 x i32] zeroinitializer, align 4
 @x = internal global i32 0, align 4
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll
index 9619895c450c..32c3eaeb9c87 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll
@@ -207,7 +207,7 @@ define void @test_attributes(ptr byval(%struct2) %s) gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    ldr x8, [sp, #64]
 ; CHECK-NEXT:    ldr q0, [sp, #48]
-; CHECK-NEXT:    mov x18, xzr
+; CHECK-NEXT:    mov x15, xzr
 ; CHECK-NEXT:    mov w0, #42 // =0x2a
 ; CHECK-NEXT:    mov w1, #17 // =0x11
 ; CHECK-NEXT:    str x8, [sp, #16]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll
index 72686c3f418e..c3fca4c18ee7 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll
@@ -11,66 +11,64 @@ target triple = "aarch64-unknown-linux-gnu"
 define dso_local void @func1(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v6, ptr %v7, ptr %v8,
 ; CHECK-LABEL: func1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-48]! // 8-byte Folded Spill
-; CHECK-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w21, -24
-; CHECK-NEXT:    .cfi_offset w22, -32
-; CHECK-NEXT:    .cfi_offset w29, -48
-; CHECK-NEXT:    add x10, sp, #176
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    add x9, sp, #144
-; CHECK-NEXT:    ldr z3, [x10]
+; CHECK-NEXT:    sub sp, sp, #368
+; CHECK-NEXT:    stp x29, x30, [sp, #336] // 16-byte Folded Spill
+; CHECK-NEXT:    str x28, [sp, #352] // 8-byte Folded Spill
+; CHECK-NEXT:    add x29, sp, #336
+; CHECK-NEXT:    .cfi_def_cfa w29, 32
+; CHECK-NEXT:    .cfi_offset w28, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    add x8, x29, #32
+; CHECK-NEXT:    add x9, x29, #72
+; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ldr z0, [x8]
-; CHECK-NEXT:    add x8, sp, #112
-; CHECK-NEXT:    ldr z2, [x9]
+; CHECK-NEXT:    add x8, x29, #256
+; CHECK-NEXT:    ldr z3, [x9]
 ; CHECK-NEXT:    ldr z1, [x8]
-; CHECK-NEXT:    add x20, sp, #176
-; CHECK-NEXT:    ldp x9, x8, [sp, #328]
-; CHECK-NEXT:    ldr x15, [sp, #104]
-; CHECK-NEXT:    ldp x11, x10, [sp, #312]
-; CHECK-NEXT:    ldur q4, [sp, #88]
-; CHECK-NEXT:    ldp x13, x12, [sp, #296]
-; CHECK-NEXT:    ldr x19, [sp, #272]
-; CHECK-NEXT:    ldp x18, x14, [sp, #280]
-; CHECK-NEXT:    ldp x16, x17, [sp, #208]
-; CHECK-NEXT:    ldp x21, x22, [sp, #352]
-; CHECK-NEXT:    str z3, [x20]
-; CHECK-NEXT:    add x20, sp, #144
-; CHECK-NEXT:    str z2, [x20]
-; CHECK-NEXT:    add x20, sp, #112
-; CHECK-NEXT:    str z1, [x20]
-; CHECK-NEXT:    add x20, sp, #48
-; CHECK-NEXT:    str z0, [x20]
-; CHECK-NEXT:    stp x21, x22, [sp, #352]
-; CHECK-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    stp x19, x18, [sp, #272]
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    stp x16, x17, [sp, #208]
-; CHECK-NEXT:    stur q4, [sp, #88]
-; CHECK-NEXT:    str x15, [sp, #104]
-; CHECK-NEXT:    stp x14, x13, [sp, #288]
-; CHECK-NEXT:    stp x12, x11, [sp, #304]
-; CHECK-NEXT:    stp x10, x9, [sp, #320]
-; CHECK-NEXT:    str x8, [sp, #336]
-; CHECK-NEXT:    ldr x29, [sp], #48 // 8-byte Folded Reload
-; CHECK-NEXT:    b func2
+; CHECK-NEXT:    add x8, x29, #288
+; CHECK-NEXT:    add x9, x29, #168
+; CHECK-NEXT:    ldr z2, [x8]
+; CHECK-NEXT:    add x8, x29, #104
+; CHECK-NEXT:    ldr z6, [x9]
+; CHECK-NEXT:    ldr z4, [x8]
+; CHECK-NEXT:    add x8, x29, #136
+; CHECK-NEXT:    mov x12, #17 // =0x11
+; CHECK-NEXT:    ldr z5, [x8]
+; CHECK-NEXT:    ldp x10, x11, [x29, #336]
+; CHECK-NEXT:    st1d { z6.d }, p0, [sp, x12, lsl #3]
+; CHECK-NEXT:    mov x12, #13 // =0xd
+; CHECK-NEXT:    ldr x8, [x29, #200]
+; CHECK-NEXT:    ldr x9, [x29, #320]
+; CHECK-NEXT:    st1d { z5.d }, p0, [sp, x12, lsl #3]
+; CHECK-NEXT:    mov x12, #9 // =0x9
+; CHECK-NEXT:    st1d { z4.d }, p0, [sp, x12, lsl #3]
+; CHECK-NEXT:    mov x12, #5 // =0x5
+; CHECK-NEXT:    st1d { z3.d }, p0, [sp, x12, lsl #3]
+; CHECK-NEXT:    stp x10, x11, [sp, #304]
+; CHECK-NEXT:    str x9, [sp, #288]
+; CHECK-NEXT:    str z2, [sp, #8, mul vl]
+; CHECK-NEXT:    str z1, [sp, #7, mul vl]
+; CHECK-NEXT:    str x8, [sp, #168]
+; CHECK-NEXT:    str z0, [sp]
+; CHECK-NEXT:    bl func2
+; CHECK-NEXT:    ldp x29, x30, [sp, #336] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x28, [sp, #352] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #368
+; CHECK-NEXT:    ret
                              ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr %v13, ptr %v14,  ptr %v15, ptr %v16,
                              ptr %v17, ptr %v18, ptr %v19, ptr %v20, ptr %v21, ptr %v22, ptr %v23, ptr %v24,
                              ptr %v25, ptr %v26, ptr %v27, ptr %v28, ptr %v29, ptr %v30, ptr %v31, ptr %v32,
                              ptr %v33, ptr %v34, ptr %v35, ptr %v36, ptr %v37, ptr %v38, ptr %v39, ptr %v40,
                              ptr %v41, ptr %v42, ptr %v43, ptr %v44, ptr %v45, ptr %v46, ptr %v47, ptr %v48,
                              i64 %v49) #0 {
-  tail call void @func2(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v6, ptr %v7, ptr %v8,
-                        ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr undef, ptr %v14, ptr %v15, ptr %v16,
-                        ptr %v17, ptr %v18, ptr %v19, ptr %v20, ptr %v21, ptr %v22, ptr %v23, ptr %v24,
-                        ptr %v25, ptr %v26, ptr %v27, ptr %v28, ptr %v29, ptr %v30, ptr undef, ptr undef,
-                        ptr undef, ptr undef, ptr undef, ptr undef, ptr %v37, ptr %v38, ptr %v39, ptr %v40,
-                        ptr %v41, ptr %v42, ptr %v43, ptr %v44, ptr %v45, ptr undef, ptr %v47, ptr %v48,
-                        i64 undef)
+  call void @func2(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v6, ptr %v7, ptr %v8,
+                   ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr undef, ptr %v14, ptr %v15, ptr %v16,
+                   ptr %v17, ptr %v18, ptr %v19, ptr %v20, ptr %v21, ptr %v22, ptr %v23, ptr %v24,
+                   ptr %v25, ptr %v26, ptr %v27, ptr %v28, ptr %v29, ptr %v30, ptr undef, ptr undef,
+                   ptr undef, ptr undef, ptr undef, ptr undef, ptr %v37, ptr %v38, ptr %v39, ptr %v40,
+                   ptr %v41, ptr %v42, ptr %v43, ptr %v44, ptr %v45, ptr undef, ptr %v47, ptr %v48,
+                   i64 undef)
   ret void
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index 79d766d1b990..af813ff16a20 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mattr=+dotprod -aarch64-enable-partial-reduce-nodes=true < %s | FileCheck %s --check-prefixes=COMMON,NEON
-; RUN: llc -mattr=+sve,+dotprod -aarch64-enable-partial-reduce-nodes=true < %s | FileCheck %s --check-prefixes=COMMON,SVE
-; RUN: llc -mattr=+sme -aarch64-enable-partial-reduce-nodes=true -force-streaming < %s | FileCheck %s --check-prefix=SME
+; RUN: llc -mattr=+dotprod,+i8mm -aarch64-enable-partial-reduce-nodes=true < %s | FileCheck %s --check-prefixes=COMMON,NEON
+; RUN: llc -mattr=+sve,+dotprod,+i8mm -aarch64-enable-partial-reduce-nodes=true < %s | FileCheck %s --check-prefixes=COMMON,SVE
+; RUN: llc -mattr=+sme,+i8mm -aarch64-enable-partial-reduce-nodes=true -force-streaming < %s | FileCheck %s --check-prefix=SME
 
 target triple = "aarch64"
 
@@ -407,6 +407,154 @@ define <4 x i32> @four_way_i8_i32_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
   ret <4 x i32> %partial.reduce
 }
 
+define <4 x i32> @four_way_i8_i32_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {
+; COMMON-LABEL: four_way_i8_i32_vl128_usdot:
+; COMMON:       // %bb.0:
+; COMMON-NEXT:    ldr q0, [x0]
+; COMMON-NEXT:    ldr q1, [x1]
+; COMMON-NEXT:    ldr q2, [x2]
+; COMMON-NEXT:    usdot v0.4s, v1.16b, v2.16b
+; COMMON-NEXT:    ret
+;
+; SME-LABEL: four_way_i8_i32_vl128_usdot:
+; SME:       // %bb.0:
+; SME-NEXT:    ldr q0, [x0]
+; SME-NEXT:    ldr q1, [x1]
+; SME-NEXT:    ldr q2, [x2]
+; SME-NEXT:    usdot z0.s, z1.b, z2.b
+; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SME-NEXT:    ret
+  %acc = load <4 x i32>, ptr %accptr
+  %u = load <16 x i8>, ptr %uptr
+  %s = load <16 x i8>, ptr %sptr
+  %u.wide = zext <16 x i8> %u to <16 x i32>
+  %s.wide = sext <16 x i8> %s to <16 x i32>
+  %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide
+  %partial.reduce = tail call <4 x i32> @llvm.experimental.vector.partial.reduce.add(<4 x i32> %acc, <16 x i32> %mult)
+  ret <4 x i32> %partial.reduce
+}
+
+define <4 x i32> @four_way_i8_i32_vl128_sudot(ptr %accptr, ptr %uptr, ptr %sptr) {
+; COMMON-LABEL: four_way_i8_i32_vl128_sudot:
+; COMMON:       // %bb.0:
+; COMMON-NEXT:    ldr q0, [x0]
+; COMMON-NEXT:    ldr q1, [x1]
+; COMMON-NEXT:    ldr q2, [x2]
+; COMMON-NEXT:    usdot v0.4s, v2.16b, v1.16b
+; COMMON-NEXT:    ret
+;
+; SME-LABEL: four_way_i8_i32_vl128_sudot:
+; SME:       // %bb.0:
+; SME-NEXT:    ldr q0, [x0]
+; SME-NEXT:    ldr q1, [x1]
+; SME-NEXT:    ldr q2, [x2]
+; SME-NEXT:    usdot z0.s, z2.b, z1.b
+; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SME-NEXT:    ret
+  %acc = load <4 x i32>, ptr %accptr
+  %u = load <16 x i8>, ptr %uptr
+  %s = load <16 x i8>, ptr %sptr
+  %u.wide = sext <16 x i8> %u to <16 x i32>
+  %s.wide = zext <16 x i8> %s to <16 x i32>
+  %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide
+  %partial.reduce = tail call <4 x i32> @llvm.experimental.vector.partial.reduce.add(<4 x i32> %acc, <16 x i32> %mult)
+  ret <4 x i32> %partial.reduce
+}
+
+define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {
+; NEON-LABEL: four_way_i8_i64_vl128_usdot:
+; NEON:       // %bb.0:
+; NEON-NEXT:    movi v0.2d, #0000000000000000
+; NEON-NEXT:    ldr q1, [x1]
+; NEON-NEXT:    ldr q2, [x2]
+; NEON-NEXT:    usdot v0.4s, v1.16b, v2.16b
+; NEON-NEXT:    ldr q1, [x0]
+; NEON-NEXT:    saddw v1.2d, v1.2d, v0.2s
+; NEON-NEXT:    saddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT:    ret
+;
+; SVE-LABEL: four_way_i8_i64_vl128_usdot:
+; SVE:       // %bb.0:
+; SVE-NEXT:    movi v0.2d, #0000000000000000
+; SVE-NEXT:    ldr q1, [x1]
+; SVE-NEXT:    ldr q2, [x2]
+; SVE-NEXT:    usdot z0.s, z1.b, z2.b
+; SVE-NEXT:    ldr q2, [x0]
+; SVE-NEXT:    sunpklo z1.d, z0.s
+; SVE-NEXT:    sunpkhi z0.d, z0.s
+; SVE-NEXT:    add z1.d, z2.d, z1.d
+; SVE-NEXT:    add z0.d, z1.d, z0.d
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT:    ret
+;
+; SME-LABEL: four_way_i8_i64_vl128_usdot:
+; SME:       // %bb.0:
+; SME-NEXT:    mov z0.s, #0 // =0x0
+; SME-NEXT:    ldr q1, [x1]
+; SME-NEXT:    ldr q2, [x2]
+; SME-NEXT:    usdot z0.s, z1.b, z2.b
+; SME-NEXT:    ldr q1, [x0]
+; SME-NEXT:    saddwb z1.d, z1.d, z0.s
+; SME-NEXT:    saddwt z0.d, z1.d, z0.s
+; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SME-NEXT:    ret
+  %acc = load <2 x i64>, ptr %accptr
+  %u = load <16 x i8>, ptr %uptr
+  %s = load <16 x i8>, ptr %sptr
+  %u.wide = zext <16 x i8> %u to <16 x i64>
+  %s.wide = sext <16 x i8> %s to <16 x i64>
+  %mult = mul nuw nsw <16 x i64> %s.wide, %u.wide
+  %partial.reduce = tail call <2 x i64> @llvm.experimental.vector.partial.reduce.add(<2 x i64> %acc, <16 x i64> %mult)
+  ret <2 x i64> %partial.reduce
+}
+
+define <2 x i64> @four_way_i16_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {
+; COMMON-LABEL: four_way_i16_i64_vl128_usdot:
+; COMMON:       // %bb.0:
+; COMMON-NEXT:    ldr q1, [x1]
+; COMMON-NEXT:    ldr q2, [x2]
+; COMMON-NEXT:    ldr q0, [x0]
+; COMMON-NEXT:    ushll v3.4s, v1.4h, #0
+; COMMON-NEXT:    sshll v4.4s, v2.4h, #0
+; COMMON-NEXT:    ushll2 v1.4s, v1.8h, #0
+; COMMON-NEXT:    sshll2 v2.4s, v2.8h, #0
+; COMMON-NEXT:    smlal v0.2d, v4.2s, v3.2s
+; COMMON-NEXT:    smlal2 v0.2d, v4.4s, v3.4s
+; COMMON-NEXT:    smlal v0.2d, v2.2s, v1.2s
+; COMMON-NEXT:    smlal2 v0.2d, v2.4s, v1.4s
+; COMMON-NEXT:    ret
+;
+; SME-LABEL: four_way_i16_i64_vl128_usdot:
+; SME:       // %bb.0:
+; SME-NEXT:    ptrue p0.d, vl2
+; SME-NEXT:    ldr q2, [x0]
+; SME-NEXT:    mov x8, #2 // =0x2
+; SME-NEXT:    ld1h { z0.d }, p0/z, [x1]
+; SME-NEXT:    ld1sh { z1.d }, p0/z, [x2]
+; SME-NEXT:    mad z0.d, p0/m, z1.d, z2.d
+; SME-NEXT:    ld1h { z1.d }, p0/z, [x1, x8, lsl #1]
+; SME-NEXT:    ld1sh { z2.d }, p0/z, [x2, x8, lsl #1]
+; SME-NEXT:    mov x8, #4 // =0x4
+; SME-NEXT:    mla z0.d, p0/m, z2.d, z1.d
+; SME-NEXT:    ld1h { z1.d }, p0/z, [x1, x8, lsl #1]
+; SME-NEXT:    ld1sh { z2.d }, p0/z, [x2, x8, lsl #1]
+; SME-NEXT:    mov x8, #6 // =0x6
+; SME-NEXT:    mla z0.d, p0/m, z2.d, z1.d
+; SME-NEXT:    ld1h { z1.d }, p0/z, [x1, x8, lsl #1]
+; SME-NEXT:    ld1sh { z2.d }, p0/z, [x2, x8, lsl #1]
+; SME-NEXT:    mla z0.d, p0/m, z2.d, z1.d
+; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SME-NEXT:    ret
+  %acc = load <2 x i64>, ptr %accptr
+  %u = load <8 x i16>, ptr %uptr
+  %s = load <8 x i16>, ptr %sptr
+  %u.wide = zext <8 x i16> %u to <8 x i64>
+  %s.wide = sext <8 x i16> %s to <8 x i64>
+  %mult = mul nuw nsw <8 x i64> %s.wide, %u.wide
+  %partial.reduce = tail call <2 x i64> @llvm.experimental.vector.partial.reduce.add(<2 x i64> %acc, <8 x i64> %mult)
+  ret <2 x i64> %partial.reduce
+}
+
 define <8 x i32> @four_way_i8_i32_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {
 ;
 ; COMMON-LABEL: four_way_i8_i32_vl128_double_width:
@@ -438,6 +586,37 @@ define <8 x i32> @four_way_i8_i32_vl128_double_width(ptr %accptr, ptr %uptr, ptr
   ret <8 x i32> %partial.reduce
 }
 
+define <8 x i32> @four_way_i8_i32_vl128_double_width_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {
+;
+; COMMON-LABEL: four_way_i8_i32_vl128_double_width_usdot:
+; COMMON:       // %bb.0:
+; COMMON-NEXT:    ldp q0, q1, [x0]
+; COMMON-NEXT:    ldp q3, q2, [x1]
+; COMMON-NEXT:    ldp q5, q4, [x2]
+; COMMON-NEXT:    usdot v0.4s, v3.16b, v5.16b
+; COMMON-NEXT:    usdot v1.4s, v2.16b, v4.16b
+; COMMON-NEXT:    ret
+;
+; SME-LABEL: four_way_i8_i32_vl128_double_width_usdot:
+; SME:       // %bb.0:
+; SME-NEXT:    ldp q0, q1, [x0]
+; SME-NEXT:    ldp q3, q2, [x1]
+; SME-NEXT:    ldp q5, q4, [x2]
+; SME-NEXT:    usdot z0.s, z3.b, z5.b
+; SME-NEXT:    usdot z1.s, z2.b, z4.b
+; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; SME-NEXT:    ret
+  %acc = load <8 x i32>, ptr %accptr
+  %u = load <32 x i8>, ptr %uptr
+  %s = load <32 x i8>, ptr %sptr
+  %u.wide = zext <32 x i8> %u to <32 x i32>
+  %s.wide = sext <32 x i8> %s to <32 x i32>
+  %mult = mul nuw nsw <32 x i32> %s.wide, %u.wide
+  %partial.reduce = tail call <8 x i32> @llvm.experimental.vector.partial.reduce.add(<8 x i32> %acc, <32 x i32> %mult)
+  ret <8 x i32> %partial.reduce
+}
+
 define <8 x i32> @four_way_i8_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {
 ;
 ;
@@ -483,6 +662,51 @@ define <8 x i32> @four_way_i8_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
   ret <8 x i32> %partial.reduce
 }
 
+define <8 x i32> @four_way_i8_i32_vl256_usdot(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {
+;
+;
+; NEON-LABEL: four_way_i8_i32_vl256_usdot:
+; NEON:       // %bb.0:
+; NEON-NEXT:    ldp q0, q1, [x0]
+; NEON-NEXT:    ldp q3, q2, [x1]
+; NEON-NEXT:    ldp q5, q4, [x2]
+; NEON-NEXT:    usdot v0.4s, v3.16b, v5.16b
+; NEON-NEXT:    usdot v1.4s, v2.16b, v4.16b
+; NEON-NEXT:    ret
+;
+; SVE-LABEL: four_way_i8_i32_vl256_usdot:
+; SVE:       // %bb.0:
+; SVE-NEXT:    ldr z0, [x0]
+; SVE-NEXT:    ldr z1, [x1]
+; SVE-NEXT:    ldr z2, [x2]
+; SVE-NEXT:    usdot z0.s, z1.b, z2.b
+; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; SVE-NEXT:    ret
+;
+; SME-LABEL: four_way_i8_i32_vl256_usdot:
+; SME:       // %bb.0:
+; SME-NEXT:    ldr z0, [x0]
+; SME-NEXT:    ldr z1, [x1]
+; SME-NEXT:    ldr z2, [x2]
+; SME-NEXT:    usdot z0.s, z1.b, z2.b
+; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
+; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; SME-NEXT:    ret
+  %acc = load <8 x i32>, ptr %accptr
+  %u = load <32 x i8>, ptr %uptr
+  %s = load <32 x i8>, ptr %sptr
+  %u.wide = zext <32 x i8> %u to <32 x i32>
+  %s.wide = sext <32 x i8> %s to <32 x i32>
+  %mult = mul nuw nsw <32 x i32> %s.wide, %u.wide
+  %partial.reduce = tail call <8 x i32> @llvm.experimental.vector.partial.reduce.add(<8 x i32> %acc, <32 x i32> %mult)
+  ret <8 x i32> %partial.reduce
+}
+
 ;
 ; Four-way dot (i16 -> i64)
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
index d3ccfaaf20a2..221a15e5c8fe 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
@@ -3,7 +3,7 @@
 ; RUN: llc -mtriple=aarch64 -mattr=+sve2 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOI8MM
 ; RUN: llc -mtriple=aarch64 -mattr=+sve,+i8mm -aarch64-enable-partial-reduce-nodes %s -o - | FileCheck %s --check-prefixes=CHECK-NEWLOWERING,CHECK-NEWLOWERING-SVE
 ; RUN: llc -mtriple=aarch64 -mattr=+sve2,+i8mm -aarch64-enable-partial-reduce-nodes %s -o - | FileCheck %s --check-prefixes=CHECK-NEWLOWERING,CHECK-NEWLOWERING-SVE2
-; RUN: llc -mtriple=aarch64 -mattr=+sme -force-streaming -aarch64-enable-partial-reduce-nodes %s -o - | FileCheck %s --check-prefixes=CHECK-NEWLOWERING,CHECK-NEWLOWERING-SME
+; RUN: llc -mtriple=aarch64 -mattr=+sve,+sme,+i8mm -force-streaming -aarch64-enable-partial-reduce-nodes %s -o - | FileCheck %s --check-prefixes=CHECK-NEWLOWERING,CHECK-NEWLOWERING-SME
 
 define <vscale x 4 x i32> @udot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: udot:
@@ -106,23 +106,7 @@ define <vscale x 4 x i32> @usdot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a,
 ;
 ; CHECK-NEWLOWERING-LABEL: usdot:
 ; CHECK-NEWLOWERING:       // %bb.0: // %entry
-; CHECK-NEWLOWERING-NEXT:    uunpklo z3.h, z1.b
-; CHECK-NEWLOWERING-NEXT:    sunpklo z4.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    ptrue p0.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z1.h, z1.b
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z2.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    uunpklo z5.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    sunpklo z6.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z3.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z4.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z5.s, z6.s
-; CHECK-NEWLOWERING-NEXT:    uunpklo z5.s, z1.h
-; CHECK-NEWLOWERING-NEXT:    sunpklo z6.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z1.s, z1.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z2.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z3.s, z4.s
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z5.s, z6.s
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z1.s, z2.s
+; CHECK-NEWLOWERING-NEXT:    usdot z0.s, z1.b, z2.b
 ; CHECK-NEWLOWERING-NEXT:    ret
 entry:
   %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>
@@ -161,23 +145,7 @@ define <vscale x 4 x i32> @sudot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a,
 ;
 ; CHECK-NEWLOWERING-LABEL: sudot:
 ; CHECK-NEWLOWERING:       // %bb.0: // %entry
-; CHECK-NEWLOWERING-NEXT:    sunpklo z3.h, z1.b
-; CHECK-NEWLOWERING-NEXT:    uunpklo z4.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    ptrue p0.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z1.h, z1.b
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z2.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    sunpklo z5.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    uunpklo z6.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z3.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z4.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z5.s, z6.s
-; CHECK-NEWLOWERING-NEXT:    sunpklo z5.s, z1.h
-; CHECK-NEWLOWERING-NEXT:    uunpklo z6.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z1.s, z1.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z2.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z3.s, z4.s
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z5.s, z6.s
-; CHECK-NEWLOWERING-NEXT:    mla z0.s, p0/m, z1.s, z2.s
+; CHECK-NEWLOWERING-NEXT:    usdot z0.s, z2.b, z1.b
 ; CHECK-NEWLOWERING-NEXT:    ret
 entry:
   %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>
@@ -329,46 +297,31 @@ define <vscale x 4 x i64> @usdot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i
 ; CHECK-NOI8MM-NEXT:    mla z0.d, p0/m, z2.d, z3.d
 ; CHECK-NOI8MM-NEXT:    ret
 ;
-; CHECK-NEWLOWERING-LABEL: usdot_8to64:
-; CHECK-NEWLOWERING:       // %bb.0: // %entry
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z4.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    uunpklo z2.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z5.h, z3.b
-; CHECK-NEWLOWERING-NEXT:    sunpklo z3.h, z3.b
-; CHECK-NEWLOWERING-NEXT:    ptrue p0.d
-; CHECK-NEWLOWERING-NEXT:    uunpklo z6.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    uunpklo z7.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    sunpklo z24.s, z5.h
-; CHECK-NEWLOWERING-NEXT:    sunpklo z25.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z4.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z2.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z5.s, z5.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z3.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    uunpklo z26.d, z6.s
-; CHECK-NEWLOWERING-NEXT:    uunpklo z27.d, z7.s
-; CHECK-NEWLOWERING-NEXT:    sunpklo z28.d, z24.s
-; CHECK-NEWLOWERING-NEXT:    sunpklo z29.d, z25.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z6.d, z6.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z7.d, z7.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z24.d, z24.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z25.d, z25.s
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z26.d, z28.d
-; CHECK-NEWLOWERING-NEXT:    uunpklo z26.d, z4.s
-; CHECK-NEWLOWERING-NEXT:    sunpklo z28.d, z5.s
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z27.d, z29.d
-; CHECK-NEWLOWERING-NEXT:    uunpklo z27.d, z2.s
-; CHECK-NEWLOWERING-NEXT:    sunpklo z29.d, z3.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z4.d, z4.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z2.d, z2.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z5.d, z5.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z3.d, z3.s
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z6.d, z24.d
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z7.d, z25.d
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z26.d, z28.d
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z27.d, z29.d
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z4.d, z5.d
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z2.d, z3.d
-; CHECK-NEWLOWERING-NEXT:    ret
+; CHECK-NEWLOWERING-SVE-LABEL: usdot_8to64:
+; CHECK-NEWLOWERING-SVE:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-SVE-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-SVE-NEXT:    usdot z4.s, z2.b, z3.b
+; CHECK-NEWLOWERING-SVE-NEXT:    sunpklo z2.d, z4.s
+; CHECK-NEWLOWERING-SVE-NEXT:    sunpkhi z3.d, z4.s
+; CHECK-NEWLOWERING-SVE-NEXT:    add z0.d, z0.d, z2.d
+; CHECK-NEWLOWERING-SVE-NEXT:    add z0.d, z0.d, z3.d
+; CHECK-NEWLOWERING-SVE-NEXT:    ret
+;
+; CHECK-NEWLOWERING-SVE2-LABEL: usdot_8to64:
+; CHECK-NEWLOWERING-SVE2:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-SVE2-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-SVE2-NEXT:    usdot z4.s, z2.b, z3.b
+; CHECK-NEWLOWERING-SVE2-NEXT:    saddwb z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SVE2-NEXT:    saddwt z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SVE2-NEXT:    ret
+;
+; CHECK-NEWLOWERING-SME-LABEL: usdot_8to64:
+; CHECK-NEWLOWERING-SME:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-SME-NEXT:    mov z4.s, #0 // =0x0
+; CHECK-NEWLOWERING-SME-NEXT:    usdot z4.s, z2.b, z3.b
+; CHECK-NEWLOWERING-SME-NEXT:    saddwb z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SME-NEXT:    saddwt z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SME-NEXT:    ret
 entry:
   %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i64>
   %b.wide = sext <vscale x 16 x i8> %b to <vscale x 16 x i64>
@@ -430,46 +383,31 @@ define <vscale x 4 x i64> @sudot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i
 ; CHECK-NOI8MM-NEXT:    mla z0.d, p0/m, z2.d, z3.d
 ; CHECK-NOI8MM-NEXT:    ret
 ;
-; CHECK-NEWLOWERING-LABEL: sudot_8to64:
-; CHECK-NEWLOWERING:       // %bb.0: // %entry
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z4.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    sunpklo z2.h, z2.b
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z5.h, z3.b
-; CHECK-NEWLOWERING-NEXT:    uunpklo z3.h, z3.b
-; CHECK-NEWLOWERING-NEXT:    ptrue p0.d
-; CHECK-NEWLOWERING-NEXT:    sunpklo z6.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    sunpklo z7.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    uunpklo z24.s, z5.h
-; CHECK-NEWLOWERING-NEXT:    uunpklo z25.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z4.s, z4.h
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z2.s, z2.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z5.s, z5.h
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z3.s, z3.h
-; CHECK-NEWLOWERING-NEXT:    sunpklo z26.d, z6.s
-; CHECK-NEWLOWERING-NEXT:    sunpklo z27.d, z7.s
-; CHECK-NEWLOWERING-NEXT:    uunpklo z28.d, z24.s
-; CHECK-NEWLOWERING-NEXT:    uunpklo z29.d, z25.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z6.d, z6.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z7.d, z7.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z24.d, z24.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z25.d, z25.s
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z26.d, z28.d
-; CHECK-NEWLOWERING-NEXT:    sunpklo z26.d, z4.s
-; CHECK-NEWLOWERING-NEXT:    uunpklo z28.d, z5.s
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z27.d, z29.d
-; CHECK-NEWLOWERING-NEXT:    sunpklo z27.d, z2.s
-; CHECK-NEWLOWERING-NEXT:    uunpklo z29.d, z3.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z4.d, z4.s
-; CHECK-NEWLOWERING-NEXT:    sunpkhi z2.d, z2.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z5.d, z5.s
-; CHECK-NEWLOWERING-NEXT:    uunpkhi z3.d, z3.s
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z6.d, z24.d
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z7.d, z25.d
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z26.d, z28.d
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z27.d, z29.d
-; CHECK-NEWLOWERING-NEXT:    mla z1.d, p0/m, z4.d, z5.d
-; CHECK-NEWLOWERING-NEXT:    mla z0.d, p0/m, z2.d, z3.d
-; CHECK-NEWLOWERING-NEXT:    ret
+; CHECK-NEWLOWERING-SVE-LABEL: sudot_8to64:
+; CHECK-NEWLOWERING-SVE:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-SVE-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-SVE-NEXT:    usdot z4.s, z3.b, z2.b
+; CHECK-NEWLOWERING-SVE-NEXT:    sunpklo z2.d, z4.s
+; CHECK-NEWLOWERING-SVE-NEXT:    sunpkhi z3.d, z4.s
+; CHECK-NEWLOWERING-SVE-NEXT:    add z0.d, z0.d, z2.d
+; CHECK-NEWLOWERING-SVE-NEXT:    add z0.d, z0.d, z3.d
+; CHECK-NEWLOWERING-SVE-NEXT:    ret
+;
+; CHECK-NEWLOWERING-SVE2-LABEL: sudot_8to64:
+; CHECK-NEWLOWERING-SVE2:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-SVE2-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-SVE2-NEXT:    usdot z4.s, z3.b, z2.b
+; CHECK-NEWLOWERING-SVE2-NEXT:    saddwb z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SVE2-NEXT:    saddwt z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SVE2-NEXT:    ret
+;
+; CHECK-NEWLOWERING-SME-LABEL: sudot_8to64:
+; CHECK-NEWLOWERING-SME:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-SME-NEXT:    mov z4.s, #0 // =0x0
+; CHECK-NEWLOWERING-SME-NEXT:    usdot z4.s, z3.b, z2.b
+; CHECK-NEWLOWERING-SME-NEXT:    saddwb z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SME-NEXT:    saddwt z0.d, z0.d, z4.s
+; CHECK-NEWLOWERING-SME-NEXT:    ret
 entry:
   %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>
   %b.wide = zext <vscale x 16 x i8> %b to <vscale x 16 x i64>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-stack-frame-layout-win.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-stack-frame-layout-win.ll
new file mode 100644
index 000000000000..8f732bd93312
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-stack-frame-layout-win.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple=aarch64-windows-msvc -mattr=+sve -pass-remarks-analysis=stack-frame-layout < %s 2>&1 -o /dev/null | FileCheck %s
+
+; CHECK: Function: f10
+; CHECK: Offset: [SP+0-16 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-32 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-48 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-64 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-80 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-96 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-112 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-128 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-144 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-160 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-176 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-192 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-208 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-224 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-240 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-256 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP+0-258 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-260 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-262 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-264 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-266 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-268 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-270 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-272 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-274 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-276 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-278 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP+0-280 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK: Offset: [SP-16-288 x vscale], Type: Spill, Align: 16, Size: 8
+; CHECK: Offset: [SP-24-288 x vscale], Type: Spill, Align: 8, Size: 8
+; CHECK: Offset: [SP-32-288 x vscale], Type: Spill, Align: 8, Size: 8
+; CHECK: Offset: [SP-32-304 x vscale], Type: Variable, Align: 16, Size: vscale x 16
+; CHECK: Offset: [SP-48-304 x vscale], Type: Variable, Align: 8, Size: 16
+
+declare void @g10(ptr,ptr)
+define void @f10(i64 %n, <vscale x 2 x i64> %x) "frame-pointer"="all" {
+  %p1 = alloca [2 x i64]
+  %p2 = alloca <vscale x 2 x i64>
+  call void @g10(ptr %p1, ptr %p2)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll
index ffef6f74f2d3..e4eda2e3a1e3 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll
@@ -100,22 +100,17 @@ define void @bitcast_v2i16(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: bitcast_v2i16:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #32
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldrh w8, [x0, #2]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #4]
+; NONEON-NOSVE-NEXT:    sub sp, sp, #16
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
 ; NONEON-NOSVE-NEXT:    ldrh w8, [x0]
-; NONEON-NOSVE-NEXT:    str w8, [sp]
+; NONEON-NOSVE-NEXT:    ldrh w9, [x0, #2]
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #2]
+; NONEON-NOSVE-NEXT:    strh w8, [sp]
 ; NONEON-NOSVE-NEXT:    ldr d0, [sp]
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #8]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #18]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
 ; NONEON-NOSVE-NEXT:    str w8, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
+; NONEON-NOSVE-NEXT:    add sp, sp, #16
 ; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <2 x i16>, ptr %a
   %cast = bitcast <2 x i16> %load to <2 x half>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll
index 7d6336a43a4f..4d524bc848de 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll
@@ -148,75 +148,38 @@ define <16 x i32> @load_sext_v16i8i32(ptr %ap)  {
 ;
 ; NONEON-NOSVE-LABEL: load_sext_v16i8i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp]
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #27]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #27]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #25]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #25]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #31]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #31]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #29]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #29]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #19]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #18]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #17]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #17]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #23]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #23]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #22]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #21]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #21]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #128]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #32]
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i8>, ptr %ap
   %val = sext <16 x i8> %a to <16 x i32>
@@ -291,18 +254,12 @@ define <4 x i256> @load_sext_v4i32i256(ptr %ap) {
 ; NONEON-NOSVE-LABEL: load_sext_v4i32i256:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #24]
-; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldp x11, x9, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldp x12, x13, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldpsw x11, x9, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldpsw x12, x13, [sp, #24]
 ; NONEON-NOSVE-NEXT:    asr x10, x9, #63
 ; NONEON-NOSVE-NEXT:    asr x14, x11, #63
 ; NONEON-NOSVE-NEXT:    stp x10, x10, [x8, #112]
@@ -315,7 +272,7 @@ define <4 x i256> @load_sext_v4i32i256(ptr %ap) {
 ; NONEON-NOSVE-NEXT:    stp x13, x9, [x8, #32]
 ; NONEON-NOSVE-NEXT:    stp x10, x10, [x8, #16]
 ; NONEON-NOSVE-NEXT:    stp x12, x10, [x8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
+; NONEON-NOSVE-NEXT:    add sp, sp, #32
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i32>, ptr %ap
   %val = sext <4 x i32> %a to <4 x i256>
@@ -327,52 +284,28 @@ define <2 x i256> @load_sext_v2i64i256(ptr %ap) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    mov z1.d, z0.d[1]
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    fmov x9, d1
-; CHECK-NEXT:    asr x8, x8, #63
-; CHECK-NEXT:    fmov d3, x8
-; CHECK-NEXT:    mov z2.d, x8
-; CHECK-NEXT:    asr x9, x9, #63
-; CHECK-NEXT:    fmov d4, x9
-; CHECK-NEXT:    zip1 z0.d, z0.d, z3.d
-; CHECK-NEXT:    mov z3.d, x9
-; CHECK-NEXT:    fmov x2, d2
-; CHECK-NEXT:    zip1 z1.d, z1.d, z4.d
-; CHECK-NEXT:    mov z4.d, z2.d[1]
-; CHECK-NEXT:    mov z5.d, z0.d[1]
-; CHECK-NEXT:    mov z6.d, z3.d[1]
 ; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    fmov x6, d3
-; CHECK-NEXT:    mov z2.d, z1.d[1]
-; CHECK-NEXT:    fmov x3, d4
-; CHECK-NEXT:    fmov x1, d5
 ; CHECK-NEXT:    fmov x4, d1
-; CHECK-NEXT:    fmov x7, d6
-; CHECK-NEXT:    fmov x5, d2
+; CHECK-NEXT:    asr x1, x0, #63
+; CHECK-NEXT:    mov x2, x1
+; CHECK-NEXT:    mov x3, x1
+; CHECK-NEXT:    asr x5, x4, #63
+; CHECK-NEXT:    mov x6, x5
+; CHECK-NEXT:    mov x7, x5
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: load_sext_v2i64i256:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #144
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 144
 ; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp]
-; NONEON-NOSVE-NEXT:    ldp x8, x10, [sp]
-; NONEON-NOSVE-NEXT:    asr x9, x8, #63
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
-; NONEON-NOSVE-NEXT:    asr x8, x10, #63
-; NONEON-NOSVE-NEXT:    stp x9, x9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp x10, x8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp x8, x8, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp x0, x1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp x2, x3, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldp x4, x5, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp x6, x7, [sp, #112]
-; NONEON-NOSVE-NEXT:    add sp, sp, #144
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT:    ldp x0, x4, [sp], #16
+; NONEON-NOSVE-NEXT:    asr x1, x0, #63
+; NONEON-NOSVE-NEXT:    asr x5, x4, #63
+; NONEON-NOSVE-NEXT:    mov x2, x1
+; NONEON-NOSVE-NEXT:    mov x3, x1
+; NONEON-NOSVE-NEXT:    mov x6, x5
+; NONEON-NOSVE-NEXT:    mov x7, x5
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i64>, ptr %ap
   %val = sext <2 x i64> %a to <2 x i256>
@@ -410,88 +343,51 @@ define <16 x i64> @load_zext_v16i16i64(ptr %ap)  {
 ;
 ; NONEON-NOSVE-LABEL: load_zext_v16i16i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #336
-; NONEON-NOSVE-NEXT:    str x29, [sp, #320] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 336
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
+; NONEON-NOSVE-NEXT:    sub sp, sp, #192
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 192
 ; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #316]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #308]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #320] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #300]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #292]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #284]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #276]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #40]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #50]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #268]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #260]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #54]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #42]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #120]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #112]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #104]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #58]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #34]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #88]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #38]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #72]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #36]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #184]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldp d2, d1, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #62]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #176]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #62]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #168]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #160]
-; NONEON-NOSVE-NEXT:    str d1, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #104]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #248]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp d1, d2, [sp, #176]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #168]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #232]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #224]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #216]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #208]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #192]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #332]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #200]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #192]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #184]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #296]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #176]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [sp, #288]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #280]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #168]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #264]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #256]
-; NONEON-NOSVE-NEXT:    add sp, sp, #336
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #160]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    ldp q5, q4, [sp, #160]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #152]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #144]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #54]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #136]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #52]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #128]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #128]
+; NONEON-NOSVE-NEXT:    add sp, sp, #192
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %ap
   %val = zext <16 x i16> %a to <16 x i64>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll
index 5f6b60a767f9..4fe303b9bbf4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll
@@ -26,18 +26,18 @@ define void @fp_convert_combine_crash(ptr %a, ptr %b) {
 ; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
 ; NONEON-NOSVE-NEXT:    ldp s1, s0, [sp, #24]
 ; NONEON-NOSVE-NEXT:    fcvtzs w8, s0, #3
-; NONEON-NOSVE-NEXT:    ldp s0, s2, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp s2, s0, [sp, #16]
 ; NONEON-NOSVE-NEXT:    fcvtzs w9, s1, #3
-; NONEON-NOSVE-NEXT:    fcvtzs w10, s2, #3
-; NONEON-NOSVE-NEXT:    fcvtzs w11, s0, #3
-; NONEON-NOSVE-NEXT:    ldp s2, s1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldp s0, s3, [sp]
+; NONEON-NOSVE-NEXT:    fcvtzs w10, s0, #3
+; NONEON-NOSVE-NEXT:    fcvtzs w11, s2, #3
+; NONEON-NOSVE-NEXT:    ldp s1, s0, [sp, #8]
+; NONEON-NOSVE-NEXT:    ldp s2, s3, [sp]
 ; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    fcvtzs w12, s1, #3
-; NONEON-NOSVE-NEXT:    fcvtzs w8, s2, #3
+; NONEON-NOSVE-NEXT:    fcvtzs w12, s0, #3
+; NONEON-NOSVE-NEXT:    fcvtzs w8, s1, #3
 ; NONEON-NOSVE-NEXT:    stp w11, w10, [sp, #48]
 ; NONEON-NOSVE-NEXT:    fcvtzs w9, s3, #3
-; NONEON-NOSVE-NEXT:    fcvtzs w10, s0, #3
+; NONEON-NOSVE-NEXT:    fcvtzs w10, s2, #3
 ; NONEON-NOSVE-NEXT:    stp w8, w12, [sp, #40]
 ; NONEON-NOSVE-NEXT:    stp w10, w9, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
index c96189b96026..56149e99b15f 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
@@ -440,25 +440,20 @@ define void @fcvt_v2f16_v2f64(ptr %a, ptr %b) {
 ; NONEON-NOSVE-LABEL: fcvt_v2f16_v2f64:
 ; NONEON-NOSVE:       // %bb.0:
 ; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
+; NONEON-NOSVE-NEXT:    str w8, [sp, #-32]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
 ; NONEON-NOSVE-NEXT:    ldr d0, [sp]
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #10]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #8]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #28]
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #8]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
 ; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
+; NONEON-NOSVE-NEXT:    add sp, sp, #32
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x half>, ptr %a
   %res = fpext <2 x half> %op1 to <2 x double>
@@ -480,35 +475,27 @@ define void @fcvt_v4f16_v4f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: fcvt_v4f16_v4f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    ldr d0, [x0]
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #10]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
+; NONEON-NOSVE-NEXT:    fcvt d1, s0
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #8]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #24]
+; NONEON-NOSVE-NEXT:    fcvt d0, s0
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #14]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #12]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #44]
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #40]
-; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #36]
-; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #12]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x half>, ptr %a
   %res = fpext <4 x half> %op1 to <4 x double>
@@ -537,61 +524,44 @@ define void @fcvt_v8f16_v8f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: fcvt_v8f16_v8f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp]
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #26]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
+; NONEON-NOSVE-NEXT:    fcvt d1, s0
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #24]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #56]
+; NONEON-NOSVE-NEXT:    fcvt d0, s0
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #30]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
+; NONEON-NOSVE-NEXT:    fcvt d1, s0
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #28]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #48]
+; NONEON-NOSVE-NEXT:    fcvt d0, s0
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #18]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
+; NONEON-NOSVE-NEXT:    fcvt d1, s0
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #16]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #40]
+; NONEON-NOSVE-NEXT:    fcvt d0, s0
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #22]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #20]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #92]
-; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #88]
-; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #84]
-; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #80]
-; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #76]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #72]
-; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #68]
-; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #20]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fpext <8 x half> %op1 to <8 x double>
@@ -637,115 +607,79 @@ define void @fcvt_v16f16_v16f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: fcvt_v16f16_v16f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #336
-; NONEON-NOSVE-NEXT:    str x29, [sp, #320] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 336
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
+; NONEON-NOSVE-NEXT:    sub sp, sp, #192
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 192
 ; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #320] // 8-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #40]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #50]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #48]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #54]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #52]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #88]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #42]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
+; NONEON-NOSVE-NEXT:    fcvt d1, s0
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #40]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #80]
+; NONEON-NOSVE-NEXT:    fcvt d0, s0
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
 ; NONEON-NOSVE-NEXT:    ldr h0, [sp, #46]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #44]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #66]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #64]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #70]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #68]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #58]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #56]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #62]
-; NONEON-NOSVE-NEXT:    fcvt s1, h0
-; NONEON-NOSVE-NEXT:    ldr h0, [sp, #60]
 ; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldp d2, d1, [sp, #120]
-; NONEON-NOSVE-NEXT:    str d1, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #104]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #164]
-; NONEON-NOSVE-NEXT:    stp d1, d2, [sp, #176]
-; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #160]
-; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #156]
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #152]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #44]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #148]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #34]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #144]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #32]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #140]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #38]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #136]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #36]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #192]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #332]
-; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #192]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #58]
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #64]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #328]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #56]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #188]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #62]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #184]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #60]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #180]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #288]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #160]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #50]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #160]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #176]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #48]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #172]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #144]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #54]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d1, s0
-; NONEON-NOSVE-NEXT:    ldr s0, [sp, #168]
+; NONEON-NOSVE-NEXT:    ldr h0, [sp, #52]
+; NONEON-NOSVE-NEXT:    fcvt s0, h0
 ; NONEON-NOSVE-NEXT:    fcvt d0, s0
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #256]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #128]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #128]
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #336
+; NONEON-NOSVE-NEXT:    add sp, sp, #192
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fpext <16 x half> %op1 to <16 x double>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll
index ad5f91a5f39a..ec0693a541e4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll
@@ -194,12 +194,12 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x i1> %mask
 define void @select_v16f16(ptr %a, ptr %b) {
 ; CHECK-LABEL: select_v16f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.h, vl8
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    fcmeq p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    fcmeq p1.h, p0/z, z1.h, z0.h
 ; CHECK-NEXT:    fcmeq p0.h, p0/z, z2.h, z3.h
-; CHECK-NEXT:    sel z0.h, p1, z0.h, z1.h
+; CHECK-NEXT:    mov z0.h, p1/m, z1.h
 ; CHECK-NEXT:    sel z1.h, p0, z2.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
@@ -429,12 +429,12 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x i1> %m
 define void @select_v8f32(ptr %a, ptr %b) {
 ; CHECK-LABEL: select_v8f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    fcmeq p1.s, p0/z, z1.s, z0.s
 ; CHECK-NEXT:    fcmeq p0.s, p0/z, z2.s, z3.s
-; CHECK-NEXT:    sel z0.s, p1, z0.s, z1.s
+; CHECK-NEXT:    mov z0.s, p1/m, z1.s
 ; CHECK-NEXT:    sel z1.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
@@ -553,12 +553,12 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x i1>
 define void @select_v4f64(ptr %a, ptr %b) {
 ; CHECK-LABEL: select_v4f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    fcmeq p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    fcmeq p1.d, p0/z, z1.d, z0.d
 ; CHECK-NEXT:    fcmeq p0.d, p0/z, z2.d, z3.d
-; CHECK-NEXT:    sel z0.d, p1, z0.d, z1.d
+; CHECK-NEXT:    mov z0.d, p1/m, z1.d
 ; CHECK-NEXT:    sel z1.d, p0, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
index 40c8ab27c0b0..75911e5ff156 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
@@ -30,50 +30,32 @@ define void @sext_v8i1_v8i32(<8 x i1> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i1_v8i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #44]
-; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #34]
+; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #10]
+; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #9]
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #8]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
+; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #13]
 ; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
 ; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
-; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #36]
-; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #38]
+; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #14]
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #15]
 ; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
 ; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
-; NONEON-NOSVE-NEXT:    stp w10, w8, [sp, #72]
+; NONEON-NOSVE-NEXT:    stp w10, w8, [sp, #40]
 ; NONEON-NOSVE-NEXT:    sbfx w8, w14, #0, #1
 ; NONEON-NOSVE-NEXT:    sbfx w10, w15, #0, #1
-; NONEON-NOSVE-NEXT:    stp w8, w12, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp w8, w12, [sp, #32]
 ; NONEON-NOSVE-NEXT:    sbfx w12, w13, #0, #1
 ; NONEON-NOSVE-NEXT:    sbfx w8, w11, #0, #1
-; NONEON-NOSVE-NEXT:    stp w12, w10, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w12, w10, [sp, #24]
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i1> %a to <8 x i32>
   store <8 x i32> %b, ptr %out
@@ -206,14 +188,14 @@ define void @sext_v32i8_v32i16(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v32i8_v32i16:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #272
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #176] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #192] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #208] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #224] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #240] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #256] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 272
+; NONEON-NOSVE-NEXT:    sub sp, sp, #208
+; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #112] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #128] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #144] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #160] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #176] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #192] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 208
 ; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
 ; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
 ; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
@@ -230,182 +212,146 @@ define void @sext_v32i8_v32i16(ptr %in, ptr %out) {
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
 ; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
-; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #19]
+; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
+; NONEON-NOSVE-NEXT:    add w5, w14, w14
+; NONEON-NOSVE-NEXT:    sxtb w6, w17
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #17]
+; NONEON-NOSVE-NEXT:    sxtb w5, w5
 ; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    add w8, w29, w29
-; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
-; NONEON-NOSVE-NEXT:    add w8, w27, w27
-; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w8, w25, w25
+; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #30]
+; NONEON-NOSVE-NEXT:    strh w6, [sp, #78]
+; NONEON-NOSVE-NEXT:    add w6, w30, w30
+; NONEON-NOSVE-NEXT:    sxtb w8, w8
+; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #31]
+; NONEON-NOSVE-NEXT:    sxtb w6, w6
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w8, w23, w23
-; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w8, w21, w21
-; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w8, w19, w19
-; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
-; NONEON-NOSVE-NEXT:    add w9, w28, w28
-; NONEON-NOSVE-NEXT:    add w18, w16, w16
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w8, w6, w6
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
+; NONEON-NOSVE-NEXT:    strh w5, [sp, #72]
+; NONEON-NOSVE-NEXT:    add w5, w29, w29
+; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #29]
+; NONEON-NOSVE-NEXT:    sxtb w9, w9
+; NONEON-NOSVE-NEXT:    sxtb w5, w5
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
+; NONEON-NOSVE-NEXT:    add w8, w27, w27
+; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #26]
+; NONEON-NOSVE-NEXT:    strh w6, [sp, #70]
+; NONEON-NOSVE-NEXT:    add w6, w28, w28
+; NONEON-NOSVE-NEXT:    sxtb w8, w8
+; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #27]
+; NONEON-NOSVE-NEXT:    sxtb w6, w6
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #66]
 ; NONEON-NOSVE-NEXT:    add w9, w26, w26
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
-; NONEON-NOSVE-NEXT:    add w8, w4, w4
+; NONEON-NOSVE-NEXT:    strh w5, [sp, #64]
+; NONEON-NOSVE-NEXT:    add w5, w25, w25
+; NONEON-NOSVE-NEXT:    sxtb w9, w9
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
+; NONEON-NOSVE-NEXT:    sxtb w8, w5
+; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #25]
+; NONEON-NOSVE-NEXT:    strh w6, [sp, #62]
+; NONEON-NOSVE-NEXT:    add w6, w24, w24
+; NONEON-NOSVE-NEXT:    add w5, w23, w23
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
+; NONEON-NOSVE-NEXT:    sxtb w9, w6
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
+; NONEON-NOSVE-NEXT:    sxtb w8, w5
+; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #38]
+; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #39]
+; NONEON-NOSVE-NEXT:    add w6, w22, w22
+; NONEON-NOSVE-NEXT:    add w5, w21, w21
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #21]
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #54]
+; NONEON-NOSVE-NEXT:    sxtb w9, w6
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
+; NONEON-NOSVE-NEXT:    sxtb w8, w5
+; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #36]
+; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #37]
+; NONEON-NOSVE-NEXT:    add w6, w20, w20
+; NONEON-NOSVE-NEXT:    add w4, w4, w4
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
+; NONEON-NOSVE-NEXT:    add w18, w15, w15
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #50]
+; NONEON-NOSVE-NEXT:    sxtb w9, w6
+; NONEON-NOSVE-NEXT:    sxtb w19, w18
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
+; NONEON-NOSVE-NEXT:    sxtb w8, w4
+; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #34]
+; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #35]
+; NONEON-NOSVE-NEXT:    add w3, w3, w3
+; NONEON-NOSVE-NEXT:    add w2, w2, w2
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #110]
+; NONEON-NOSVE-NEXT:    sxtb w9, w3
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #108]
+; NONEON-NOSVE-NEXT:    sxtb w8, w2
+; NONEON-NOSVE-NEXT:    sxtb w7, w16
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #33]
+; NONEON-NOSVE-NEXT:    add w0, w0, w0
+; NONEON-NOSVE-NEXT:    add w18, w18, w18
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #106]
+; NONEON-NOSVE-NEXT:    sxtb w9, w0
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #104]
+; NONEON-NOSVE-NEXT:    sxtb w8, w18
 ; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
-; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
-; NONEON-NOSVE-NEXT:    add w9, w24, w24
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    add w8, w2, w2
-; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w17, w17, w17
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
-; NONEON-NOSVE-NEXT:    add w9, w22, w22
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
-; NONEON-NOSVE-NEXT:    add w8, w16, w16
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
-; NONEON-NOSVE-NEXT:    add w17, w30, w30
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
-; NONEON-NOSVE-NEXT:    add w9, w20, w20
-; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
-; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
-; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
 ; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #102]
+; NONEON-NOSVE-NEXT:    sxtb w9, w17
+; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #100]
+; NONEON-NOSVE-NEXT:    sxtb w8, w16
 ; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
-; NONEON-NOSVE-NEXT:    add w9, w7, w7
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    add w14, w14, w14
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #98]
+; NONEON-NOSVE-NEXT:    sxtb w9, w15
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #96]
+; NONEON-NOSVE-NEXT:    sxtb w8, w14
+; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
 ; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
-; NONEON-NOSVE-NEXT:    add w9, w5, w5
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
-; NONEON-NOSVE-NEXT:    add w9, w3, w3
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #95]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
-; NONEON-NOSVE-NEXT:    add w9, w0, w0
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #94]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
-; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #93]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
-; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
-; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #91]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #90]
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #256] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #89]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #240] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #87]
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #224] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #86]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #208] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #85]
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #192] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #176] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #83]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #82]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #81]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #111]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #112]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #110]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #109]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #108]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #107]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #106]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #105]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #103]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #102]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #101]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #99]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #98]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #97]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #144]
+; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    add w12, w12, w12
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #94]
+; NONEON-NOSVE-NEXT:    sxtb w9, w13
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #92]
+; NONEON-NOSVE-NEXT:    sxtb w8, w12
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #90]
+; NONEON-NOSVE-NEXT:    sxtb w9, w11
+; NONEON-NOSVE-NEXT:    ldr w11, [sp, #8] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #88]
+; NONEON-NOSVE-NEXT:    sxtb w8, w10
+; NONEON-NOSVE-NEXT:    ldr w10, [sp, #12] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #86]
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #84]
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    sxtb w9, w11
+; NONEON-NOSVE-NEXT:    strh w7, [sp, #76]
+; NONEON-NOSVE-NEXT:    sxtb w8, w10
+; NONEON-NOSVE-NEXT:    strh w19, [sp, #74]
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #82]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #48]
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #192] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #176] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
+; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #160] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #144] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #272
+; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #128] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #112] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add sp, sp, #208
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
@@ -431,42 +377,24 @@ define void @sext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i8_v8i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #11]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #11]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #9]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #9]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #15]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #15]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #14]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #13]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #13]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #38]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #36]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #34]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i8> %a to <8 x i32>
   store <8 x i32>%b, ptr %out
@@ -492,75 +420,39 @@ define void @sext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i8_v16i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #27]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #27]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #25]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #25]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #31]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #31]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #29]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #29]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #19]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #18]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #17]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #17]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #23]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #23]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #22]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #21]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #21]
 ; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %b = sext <16 x i8> %a to <16 x i32>
   store <16 x i32> %b, ptr %out
@@ -599,14 +491,14 @@ define void @sext_v32i8_v32i32(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v32i8_v32i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #464
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #368] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #384] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #400] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #416] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #432] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #448] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 464
+; NONEON-NOSVE-NEXT:    sub sp, sp, #272
+; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #176] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #192] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #208] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #224] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #240] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #256] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 272
 ; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
 ; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
 ; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
@@ -621,258 +513,136 @@ define void @sext_v32i8_v32i32(ptr %in, ptr %out) {
 ; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
-; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #18]
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #19]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #44]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #45]
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #17]
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #23]
 ; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
+; NONEON-NOSVE-NEXT:    sxtb w6, w17
+; NONEON-NOSVE-NEXT:    sxtb w7, w16
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
+; NONEON-NOSVE-NEXT:    add w5, w14, w14
+; NONEON-NOSVE-NEXT:    add w18, w15, w15
+; NONEON-NOSVE-NEXT:    sxtb w19, w18
+; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #21]
+; NONEON-NOSVE-NEXT:    stp w7, w6, [sp, #104]
+; NONEON-NOSVE-NEXT:    add w6, w30, w30
+; NONEON-NOSVE-NEXT:    sxtb w5, w5
 ; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    add w8, w29, w29
-; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
-; NONEON-NOSVE-NEXT:    add w8, w27, w27
-; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w8, w25, w25
+; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #27]
+; NONEON-NOSVE-NEXT:    sxtb w6, w6
+; NONEON-NOSVE-NEXT:    sxtb w8, w8
+; NONEON-NOSVE-NEXT:    stp w5, w19, [sp, #96]
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w8, w23, w23
-; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w8, w21, w21
-; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w8, w19, w19
-; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
-; NONEON-NOSVE-NEXT:    add w9, w28, w28
-; NONEON-NOSVE-NEXT:    add w18, w16, w16
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w8, w6, w6
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
+; NONEON-NOSVE-NEXT:    add w5, w29, w29
+; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #25]
+; NONEON-NOSVE-NEXT:    sxtb w9, w9
+; NONEON-NOSVE-NEXT:    stp w8, w6, [sp, #88]
+; NONEON-NOSVE-NEXT:    add w6, w28, w28
+; NONEON-NOSVE-NEXT:    sxtb w5, w5
+; NONEON-NOSVE-NEXT:    add w8, w27, w27
+; NONEON-NOSVE-NEXT:    sxtb w6, w6
+; NONEON-NOSVE-NEXT:    sxtb w8, w8
+; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #30]
+; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #31]
+; NONEON-NOSVE-NEXT:    stp w5, w9, [sp, #80]
 ; NONEON-NOSVE-NEXT:    add w9, w26, w26
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
-; NONEON-NOSVE-NEXT:    add w8, w4, w4
-; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
-; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
-; NONEON-NOSVE-NEXT:    add w9, w24, w24
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    add w8, w2, w2
-; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
+; NONEON-NOSVE-NEXT:    add w5, w25, w25
+; NONEON-NOSVE-NEXT:    stp w8, w6, [sp, #72]
+; NONEON-NOSVE-NEXT:    sxtb w9, w9
+; NONEON-NOSVE-NEXT:    sxtb w8, w5
+; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #29]
+; NONEON-NOSVE-NEXT:    add w6, w24, w24
+; NONEON-NOSVE-NEXT:    add w5, w23, w23
+; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #34]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
+; NONEON-NOSVE-NEXT:    sxtb w9, w6
+; NONEON-NOSVE-NEXT:    sxtb w8, w5
+; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #35]
+; NONEON-NOSVE-NEXT:    add w6, w22, w22
+; NONEON-NOSVE-NEXT:    add w5, w21, w21
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT:    sxtb w9, w6
+; NONEON-NOSVE-NEXT:    sxtb w8, w5
+; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #33]
+; NONEON-NOSVE-NEXT:    add w6, w20, w20
+; NONEON-NOSVE-NEXT:    add w4, w4, w4
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT:    sxtb w9, w6
+; NONEON-NOSVE-NEXT:    sxtb w8, w4
+; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #38]
+; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #39]
+; NONEON-NOSVE-NEXT:    add w3, w3, w3
+; NONEON-NOSVE-NEXT:    add w2, w2, w2
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #36]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #168]
+; NONEON-NOSVE-NEXT:    sxtb w9, w3
+; NONEON-NOSVE-NEXT:    sxtb w8, w2
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #37]
+; NONEON-NOSVE-NEXT:    add w0, w0, w0
+; NONEON-NOSVE-NEXT:    add w18, w18, w18
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
+; NONEON-NOSVE-NEXT:    sxtb w9, w0
+; NONEON-NOSVE-NEXT:    sxtb w8, w18
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #43]
 ; NONEON-NOSVE-NEXT:    add w17, w17, w17
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
-; NONEON-NOSVE-NEXT:    add w9, w22, w22
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
-; NONEON-NOSVE-NEXT:    add w8, w16, w16
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
-; NONEON-NOSVE-NEXT:    add w17, w30, w30
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
-; NONEON-NOSVE-NEXT:    add w9, w20, w20
-; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
-; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
-; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
-; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
-; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
-; NONEON-NOSVE-NEXT:    add w9, w7, w7
-; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
-; NONEON-NOSVE-NEXT:    add w9, w5, w5
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
-; NONEON-NOSVE-NEXT:    add w9, w3, w3
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #91]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
-; NONEON-NOSVE-NEXT:    add w9, w0, w0
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #90]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
-; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #89]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
-; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
-; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #95]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #94]
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #448] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #93]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #432] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #83]
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #416] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #82]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #81]
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #400] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #384] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #87]
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #368] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #86]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #85]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #107]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #106]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #105]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #182]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #111]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #110]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #109]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #108]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #99]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #98]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #97]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #103]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #102]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #101]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #198]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #196]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #194]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #192]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #206]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #204]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #202]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #200]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #180]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #272]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #178]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #176]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #190]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #188]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #186]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #184]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #230]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #240]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #228]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #226]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #224]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #238]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #236]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #234]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #232]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #214]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #336]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #212]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #210]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #222]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #220]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #218]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #216]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #304]
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
+; NONEON-NOSVE-NEXT:    sxtb w9, w17
+; NONEON-NOSVE-NEXT:    sxtb w8, w16
+; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #41]
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    add w14, w14, w14
+; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
+; NONEON-NOSVE-NEXT:    sxtb w9, w15
+; NONEON-NOSVE-NEXT:    sxtb w8, w14
+; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #47]
+; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    add w12, w12, w12
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
+; NONEON-NOSVE-NEXT:    sxtb w9, w13
+; NONEON-NOSVE-NEXT:    sxtb w8, w12
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
+; NONEON-NOSVE-NEXT:    sxtb w9, w11
+; NONEON-NOSVE-NEXT:    ldr w11, [sp, #8] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    sxtb w8, w10
+; NONEON-NOSVE-NEXT:    ldr w10, [sp, #12] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #80]
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    sxtb w11, w11
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
+; NONEON-NOSVE-NEXT:    sxtb w8, w10
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #144]
+; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #112]
+; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #256] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #112]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
+; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #240] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
+; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #224] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
+; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #208] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #464
+; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #192] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #176] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add sp, sp, #272
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
@@ -944,57 +714,26 @@ define void @sext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i8_v8i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #176
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 176
+; NONEON-NOSVE-NEXT:    sub sp, sp, #80
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add x8, sp, #144
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #11]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #9]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #8]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #15]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #22]
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #14]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #13]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #12]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #44]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #34]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #38]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #36]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #96]
-; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #104]
-; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x8]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #9]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #8]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #11]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #10]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #13]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #12]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #15]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #176
+; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
+; NONEON-NOSVE-NEXT:    add sp, sp, #80
 ; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i8> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
@@ -1034,109 +773,43 @@ define void @sext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i8_v16i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #368
-; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
-; NONEON-NOSVE-NEXT:    str q0, [sp]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #35]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #34]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #33]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #32]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #39]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #38]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #37]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #36]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #27]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #25]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #98]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #31]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #29]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #102]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #192]
-; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #104]
-; NONEON-NOSVE-NEXT:    str d2, [sp, #168]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #216]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldrsw x9, [sp, #364]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #360]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #200]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #320]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #208]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #184]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #288]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #192]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #256]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #176]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #224]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #25]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #24]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #144]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #27]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #26]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #128]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #29]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #128]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #112]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #31]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #30]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #17]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #19]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #21]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #23]
+; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x0, #32]
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x0, #64]
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #368
+; NONEON-NOSVE-NEXT:    add sp, sp, #160
 ; NONEON-NOSVE-NEXT:    ret
   %b = sext <16 x i8> %a to <16 x i64>
   store <16 x i64> %b, ptr %out
@@ -1208,14 +881,14 @@ define void @sext_v32i8_v32i64(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v32i8_v32i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sub sp, sp, #752
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 848
+; NONEON-NOSVE-NEXT:    sub sp, sp, #400
+; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #304] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #320] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #336] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #352] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #368] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #384] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 400
 ; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
 ; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
 ; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
@@ -1230,345 +903,144 @@ define void @sext_v32i8_v32i64(ptr %in, ptr %out) {
 ; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
-; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #17]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #46]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #47]
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #18]
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #19]
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
 ; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT:    sxtb x19, w17
+; NONEON-NOSVE-NEXT:    sxtb x20, w16
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
+; NONEON-NOSVE-NEXT:    add w7, w14, w14
+; NONEON-NOSVE-NEXT:    add w18, w15, w15
+; NONEON-NOSVE-NEXT:    sxtb x21, w18
+; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #22]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #23]
+; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #160]
+; NONEON-NOSVE-NEXT:    add w19, w30, w30
+; NONEON-NOSVE-NEXT:    sxtb x7, w7
 ; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    add w8, w29, w29
-; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
-; NONEON-NOSVE-NEXT:    add w8, w27, w27
-; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w8, w25, w25
+; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #25]
+; NONEON-NOSVE-NEXT:    sxtb x19, w19
+; NONEON-NOSVE-NEXT:    sxtb x8, w8
+; NONEON-NOSVE-NEXT:    stp x7, x21, [sp, #144]
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w8, w23, w23
-; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w8, w21, w21
+; NONEON-NOSVE-NEXT:    add w7, w29, w29
+; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #27]
+; NONEON-NOSVE-NEXT:    sxtb x9, w9
+; NONEON-NOSVE-NEXT:    stp x8, x19, [sp, #128]
+; NONEON-NOSVE-NEXT:    add w19, w28, w28
+; NONEON-NOSVE-NEXT:    sxtb x7, w7
+; NONEON-NOSVE-NEXT:    add w8, w27, w27
+; NONEON-NOSVE-NEXT:    sxtb x19, w19
+; NONEON-NOSVE-NEXT:    sxtb x8, w8
+; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
 ; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w8, w19, w19
-; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
-; NONEON-NOSVE-NEXT:    add w9, w28, w28
-; NONEON-NOSVE-NEXT:    add w18, w16, w16
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w8, w6, w6
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
+; NONEON-NOSVE-NEXT:    stp x7, x9, [sp, #112]
 ; NONEON-NOSVE-NEXT:    add w9, w26, w26
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
-; NONEON-NOSVE-NEXT:    add w8, w4, w4
-; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
-; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
-; NONEON-NOSVE-NEXT:    add w9, w24, w24
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    add w8, w2, w2
-; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w17, w17, w17
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
-; NONEON-NOSVE-NEXT:    add w9, w22, w22
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
-; NONEON-NOSVE-NEXT:    add w8, w16, w16
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
-; NONEON-NOSVE-NEXT:    add w17, w30, w30
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
-; NONEON-NOSVE-NEXT:    add w9, w20, w20
-; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
-; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
+; NONEON-NOSVE-NEXT:    add w7, w25, w25
+; NONEON-NOSVE-NEXT:    stp x8, x19, [sp, #96]
+; NONEON-NOSVE-NEXT:    sxtb x9, w9
+; NONEON-NOSVE-NEXT:    sxtb x8, w7
+; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #30]
+; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #31]
+; NONEON-NOSVE-NEXT:    add w19, w24, w24
+; NONEON-NOSVE-NEXT:    add w7, w23, w23
+; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #80]
+; NONEON-NOSVE-NEXT:    sxtb x9, w19
+; NONEON-NOSVE-NEXT:    sxtb x8, w7
+; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #33]
+; NONEON-NOSVE-NEXT:    add w19, w22, w22
+; NONEON-NOSVE-NEXT:    add w6, w6, w6
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #64]
+; NONEON-NOSVE-NEXT:    sxtb x9, w19
+; NONEON-NOSVE-NEXT:    sxtb x8, w6
+; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
 ; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
-; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
-; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
-; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
-; NONEON-NOSVE-NEXT:    add w9, w7, w7
-; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
-; NONEON-NOSVE-NEXT:    add w9, w5, w5
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
-; NONEON-NOSVE-NEXT:    add w9, w3, w3
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #91]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
-; NONEON-NOSVE-NEXT:    add w9, w0, w0
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #90]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
-; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #89]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
-; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
-; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #95]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #94]
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #93]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #83]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #82]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #81]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #87]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #86]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #85]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #107]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #106]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #105]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #178]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #111]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #110]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #109]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #108]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #99]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #98]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #97]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #103]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #102]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #101]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
-; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #194]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #192]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #198]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #196]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #202]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #272]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #200]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #206]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #400]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #204]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #176]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #288]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #182]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #180]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #186]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #416]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #240]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #184]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #190]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #368]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #188]
-; NONEON-NOSVE-NEXT:    ldrsw x9, [sp, #372]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #226]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #256]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #224]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #230]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #384]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #228]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #234]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #336]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #232]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #238]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #464]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #236]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #210]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #352]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #214]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #480]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #212]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #218]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #304]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #216]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #222]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #432]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #220]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #404]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #320]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #568]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #400]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #560]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #412]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #448]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #584]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #408]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #576]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #420]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #560]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #600]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #416]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #592]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #428]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #616]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #424]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #608]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #368]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [sp, #592]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #496]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #380]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #520]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #376]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #512]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #388]
-; NONEON-NOSVE-NEXT:    ldp q4, q5, [sp, #496]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #536]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #384]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #528]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #396]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #552]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #392]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #544]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #468]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #528]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #696]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #464]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #688]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #476]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #712]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #472]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #704]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #484]
-; NONEON-NOSVE-NEXT:    ldp q16, q17, [sp, #688]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #728]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #480]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #720]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #492]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #744]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #488]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #736]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #436]
-; NONEON-NOSVE-NEXT:    ldp q19, q20, [sp, #720]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #632]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #432]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #624]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #444]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #648]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #440]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #640]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #452]
-; NONEON-NOSVE-NEXT:    ldp q22, q23, [sp, #624]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #664]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #448]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #656]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #460]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #680]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #456]
-; NONEON-NOSVE-NEXT:    str x8, [sp, #672]
-; NONEON-NOSVE-NEXT:    ldp q21, q18, [sp, #656]
+; NONEON-NOSVE-NEXT:    add w5, w5, w5
+; NONEON-NOSVE-NEXT:    add w4, w4, w4
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
+; NONEON-NOSVE-NEXT:    sxtb x9, w5
+; NONEON-NOSVE-NEXT:    sxtb x8, w4
+; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #36]
+; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #37]
+; NONEON-NOSVE-NEXT:    add w3, w3, w3
+; NONEON-NOSVE-NEXT:    add w2, w2, w2
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #38]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #288]
+; NONEON-NOSVE-NEXT:    sxtb x9, w3
+; NONEON-NOSVE-NEXT:    sxtb x8, w2
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #39]
+; NONEON-NOSVE-NEXT:    add w0, w0, w0
+; NONEON-NOSVE-NEXT:    add w18, w18, w18
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #272]
+; NONEON-NOSVE-NEXT:    sxtb x9, w0
+; NONEON-NOSVE-NEXT:    sxtb x8, w18
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #41]
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #256]
+; NONEON-NOSVE-NEXT:    sxtb x9, w17
+; NONEON-NOSVE-NEXT:    sxtb x8, w16
+; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #43]
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    add w14, w14, w14
+; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #44]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #240]
+; NONEON-NOSVE-NEXT:    sxtb x9, w15
+; NONEON-NOSVE-NEXT:    sxtb x8, w14
+; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #45]
+; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    add w12, w12, w12
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #224]
+; NONEON-NOSVE-NEXT:    sxtb x9, w13
+; NONEON-NOSVE-NEXT:    sxtb x8, w12
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #208]
+; NONEON-NOSVE-NEXT:    sxtb x9, w11
+; NONEON-NOSVE-NEXT:    ldr w11, [sp, #8] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    sxtb x8, w10
+; NONEON-NOSVE-NEXT:    ldr w10, [sp, #12] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #144]
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    sxtb x11, w11
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #192]
+; NONEON-NOSVE-NEXT:    sxtb x8, w10
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #112]
+; NONEON-NOSVE-NEXT:    ldp q5, q4, [sp, #80]
+; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #176]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldp q17, q16, [sp, #272]
+; NONEON-NOSVE-NEXT:    ldp q18, q21, [sp, #176]
+; NONEON-NOSVE-NEXT:    ldp q20, q19, [sp, #240]
+; NONEON-NOSVE-NEXT:    ldp q23, q22, [sp, #208]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
+; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #384] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1, #32]
+; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #368] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q4, q5, [x1, #64]
+; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #352] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #96]
+; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #336] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q16, q17, [x1, #128]
+; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #320] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q19, q20, [x1, #160]
+; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #304] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q22, q23, [x1, #192]
 ; NONEON-NOSVE-NEXT:    stp q21, q18, [x1, #224]
-; NONEON-NOSVE-NEXT:    add sp, sp, #752
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp], #96 // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add sp, sp, #400
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
@@ -1636,91 +1108,70 @@ define void @sext_v16i16_v16i32(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i16_v16i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
+; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
-; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
-; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
-; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
-; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
-; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
+; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #6]
+; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #2]
+; NONEON-NOSVE-NEXT:    ldrh w3, [sp]
+; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #12]
+; NONEON-NOSVE-NEXT:    ldrh w5, [sp, #14]
 ; NONEON-NOSVE-NEXT:    add w13, w13, w13
-; NONEON-NOSVE-NEXT:    add w14, w14, w14
-; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    add w14, w3, w3
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w13, w5, w5
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    add w12, w12, w12
+; NONEON-NOSVE-NEXT:    sxth w16, w16
+; NONEON-NOSVE-NEXT:    sxth w13, w13
+; NONEON-NOSVE-NEXT:    add w3, w3, w3
+; NONEON-NOSVE-NEXT:    sxth w12, w12
+; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #8]
+; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #10]
+; NONEON-NOSVE-NEXT:    stp w13, w16, [sp, #56]
+; NONEON-NOSVE-NEXT:    sxth w13, w3
+; NONEON-NOSVE-NEXT:    add w16, w5, w5
+; NONEON-NOSVE-NEXT:    add w3, w4, w4
+; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp w13, w12, [sp, #48]
+; NONEON-NOSVE-NEXT:    sxth w12, w16
+; NONEON-NOSVE-NEXT:    sxth w13, w3
+; NONEON-NOSVE-NEXT:    add w16, w2, w2
+; NONEON-NOSVE-NEXT:    add w0, w0, w0
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
-; NONEON-NOSVE-NEXT:    add w14, w4, w4
+; NONEON-NOSVE-NEXT:    stp w13, w12, [sp, #40]
+; NONEON-NOSVE-NEXT:    sxth w12, w16
+; NONEON-NOSVE-NEXT:    sxth w13, w0
 ; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
-; NONEON-NOSVE-NEXT:    add w13, w2, w2
-; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #38]
-; NONEON-NOSVE-NEXT:    add w14, w0, w0
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #36]
-; NONEON-NOSVE-NEXT:    add w13, w18, w18
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #34]
 ; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
 ; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #16]
 ; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w14, w17, w17
-; NONEON-NOSVE-NEXT:    add w12, w12, w12
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w13, w16, w16
+; NONEON-NOSVE-NEXT:    add w16, w18, w18
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    stp w13, w12, [sp, #32]
+; NONEON-NOSVE-NEXT:    sxth w12, w16
+; NONEON-NOSVE-NEXT:    sxth w13, w17
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    add w14, w14, w14
 ; NONEON-NOSVE-NEXT:    add w11, w11, w11
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    add w10, w10, w10
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #62]
-; NONEON-NOSVE-NEXT:    add w14, w15, w15
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #60]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #58]
-; NONEON-NOSVE-NEXT:    strh w12, [sp, #56]
-; NONEON-NOSVE-NEXT:    strh w11, [sp, #54]
-; NONEON-NOSVE-NEXT:    strh w10, [sp, #52]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
+; NONEON-NOSVE-NEXT:    add w9, w9, w9
+; NONEON-NOSVE-NEXT:    add w8, w8, w8
+; NONEON-NOSVE-NEXT:    stp w13, w12, [sp, #88]
+; NONEON-NOSVE-NEXT:    sxth w12, w15
+; NONEON-NOSVE-NEXT:    sxth w13, w14
+; NONEON-NOSVE-NEXT:    sxth w11, w11
+; NONEON-NOSVE-NEXT:    sxth w10, w10
+; NONEON-NOSVE-NEXT:    sxth w9, w9
+; NONEON-NOSVE-NEXT:    sxth w8, w8
+; NONEON-NOSVE-NEXT:    stp w13, w12, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
@@ -1746,24 +1197,18 @@ define void @sext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v4i16_v4i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #10]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #14]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrsh x9, [sp, #10]
+; NONEON-NOSVE-NEXT:    ldrsh x8, [sp, #8]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrsh x9, [sp, #14]
+; NONEON-NOSVE-NEXT:    ldrsh x8, [sp, #12]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %b = sext <4 x i16> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
@@ -1789,39 +1234,27 @@ define void @sext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i16_v8i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #22]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldrsh x9, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrsh x8, [sp, #24]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrsh x9, [sp, #30]
+; NONEON-NOSVE-NEXT:    ldrsh x8, [sp, #28]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrsh x9, [sp, #18]
+; NONEON-NOSVE-NEXT:    ldrsh x8, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrsh x9, [sp, #22]
+; NONEON-NOSVE-NEXT:    ldrsh x8, [sp, #20]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i16> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
@@ -1860,124 +1293,75 @@ define void @sext_v16i16_v16i64(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i16_v16i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #368
-; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
+; NONEON-NOSVE-NEXT:    sub sp, sp, #160
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
-; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
-; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
-; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
-; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
-; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
-; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
+; NONEON-NOSVE-NEXT:    ldrh w13, [sp]
+; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #2]
+; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #6]
+; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #4]
+; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #8]
+; NONEON-NOSVE-NEXT:    ldrh w5, [sp, #10]
 ; NONEON-NOSVE-NEXT:    add w13, w13, w13
-; NONEON-NOSVE-NEXT:    add w14, w14, w14
-; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w14, w3, w3
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w13, w5, w5
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w14, w4, w4
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w13, w2, w2
-; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    add w14, w0, w0
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w13, w18, w18
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
-; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
-; NONEON-NOSVE-NEXT:    add w14, w17, w17
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
 ; NONEON-NOSVE-NEXT:    add w12, w12, w12
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w13, w16, w16
+; NONEON-NOSVE-NEXT:    sxth x16, w16
+; NONEON-NOSVE-NEXT:    sxth x13, w13
+; NONEON-NOSVE-NEXT:    add w3, w3, w3
+; NONEON-NOSVE-NEXT:    sxth x12, w12
+; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #12]
+; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp x13, x16, [sp, #80]
+; NONEON-NOSVE-NEXT:    sxth x13, w3
+; NONEON-NOSVE-NEXT:    add w16, w5, w5
+; NONEON-NOSVE-NEXT:    add w3, w4, w4
+; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp x13, x12, [sp, #64]
+; NONEON-NOSVE-NEXT:    sxth x12, w16
+; NONEON-NOSVE-NEXT:    sxth x13, w3
+; NONEON-NOSVE-NEXT:    add w16, w2, w2
+; NONEON-NOSVE-NEXT:    add w0, w0, w0
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
+; NONEON-NOSVE-NEXT:    stp x13, x12, [sp, #48]
+; NONEON-NOSVE-NEXT:    sxth x12, w16
+; NONEON-NOSVE-NEXT:    sxth x13, w0
+; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]
+; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #22]
+; NONEON-NOSVE-NEXT:    add w16, w18, w18
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    stp x13, x12, [sp, #32]
+; NONEON-NOSVE-NEXT:    sxth x12, w16
+; NONEON-NOSVE-NEXT:    sxth x13, w17
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    add w14, w14, w14
 ; NONEON-NOSVE-NEXT:    add w11, w11, w11
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
 ; NONEON-NOSVE-NEXT:    add w10, w10, w10
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w14, w15, w15
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #66]
-; NONEON-NOSVE-NEXT:    strh w12, [sp, #64]
-; NONEON-NOSVE-NEXT:    strh w11, [sp, #62]
-; NONEON-NOSVE-NEXT:    strh w10, [sp, #60]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #98]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #102]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #184]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #104]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
-; NONEON-NOSVE-NEXT:    str d2, [sp, #200]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #184]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #192]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #176]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #216]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #224]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldrsw x9, [sp, #364]
-; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #360]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #200]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #320]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #208]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #288]
+; NONEON-NOSVE-NEXT:    add w9, w9, w9
+; NONEON-NOSVE-NEXT:    add w8, w8, w8
+; NONEON-NOSVE-NEXT:    stp x13, x12, [sp, #144]
+; NONEON-NOSVE-NEXT:    sxth x12, w15
+; NONEON-NOSVE-NEXT:    sxth x13, w14
+; NONEON-NOSVE-NEXT:    sxth x11, w11
+; NONEON-NOSVE-NEXT:    sxth x10, w10
+; NONEON-NOSVE-NEXT:    sxth x9, w9
+; NONEON-NOSVE-NEXT:    sxth x8, w8
+; NONEON-NOSVE-NEXT:    stp x13, x12, [sp, #128]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp x10, x11, [sp, #112]
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #128]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #96]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #368
+; NONEON-NOSVE-NEXT:    add sp, sp, #160
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
@@ -2037,43 +1421,38 @@ define void @sext_v8i32_v8i64(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i32_v8i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp]
-; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #8]
+; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
+; NONEON-NOSVE-NEXT:    ldp w9, w8, [sp]
 ; NONEON-NOSVE-NEXT:    ldp w14, w15, [sp, #16]
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
 ; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #24]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
+; NONEON-NOSVE-NEXT:    add w8, w8, w8
+; NONEON-NOSVE-NEXT:    add w9, w9, w9
+; NONEON-NOSVE-NEXT:    sxtw x8, w8
+; NONEON-NOSVE-NEXT:    sxtw x9, w9
+; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #8]
+; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #48]
+; NONEON-NOSVE-NEXT:    add w8, w15, w15
+; NONEON-NOSVE-NEXT:    add w9, w14, w14
+; NONEON-NOSVE-NEXT:    sxtw x8, w8
+; NONEON-NOSVE-NEXT:    sxtw x9, w9
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    add w12, w12, w12
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    sxtw x13, w13
+; NONEON-NOSVE-NEXT:    sxtw x12, w12
+; NONEON-NOSVE-NEXT:    sxtw x11, w11
+; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #80]
+; NONEON-NOSVE-NEXT:    sxtw x8, w10
+; NONEON-NOSVE-NEXT:    stp x12, x13, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i32>, ptr %in
   %b = add <8 x i32> %a, %a
@@ -2162,14 +1541,14 @@ define void @zext_v32i8_v32i16(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v32i8_v32i16:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #272
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #176] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #192] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #208] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #224] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #240] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #256] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 272
+; NONEON-NOSVE-NEXT:    sub sp, sp, #208
+; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #112] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #128] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #144] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #160] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #176] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #192] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 208
 ; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
 ; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
 ; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
@@ -2186,182 +1565,146 @@ define void @zext_v32i8_v32i16(ptr %in, ptr %out) {
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
 ; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
-; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #21]
+; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #17]
+; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #31]
 ; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #18]
+; NONEON-NOSVE-NEXT:    add w5, w17, w17
+; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #29]
+; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #30]
 ; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    add w8, w29, w29
-; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
-; NONEON-NOSVE-NEXT:    add w8, w27, w27
-; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w8, w25, w25
+; NONEON-NOSVE-NEXT:    and w5, w5, #0xff
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w8, w23, w23
-; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w8, w21, w21
-; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w8, w19, w19
-; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
-; NONEON-NOSVE-NEXT:    add w9, w28, w28
-; NONEON-NOSVE-NEXT:    add w18, w16, w16
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w8, w6, w6
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    strh w5, [sp, #74]
+; NONEON-NOSVE-NEXT:    add w5, w29, w29
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
+; NONEON-NOSVE-NEXT:    add w8, w28, w28
+; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #28]
+; NONEON-NOSVE-NEXT:    and w5, w5, #0xff
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #68]
+; NONEON-NOSVE-NEXT:    add w9, w27, w27
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #27]
+; NONEON-NOSVE-NEXT:    strh w5, [sp, #66]
+; NONEON-NOSVE-NEXT:    add w5, w26, w26
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
+; NONEON-NOSVE-NEXT:    add w8, w25, w25
+; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #26]
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #62]
+; NONEON-NOSVE-NEXT:    and w9, w5, #0xff
+; NONEON-NOSVE-NEXT:    add w5, w24, w24
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #25]
+; NONEON-NOSVE-NEXT:    strh w9, [sp, #60]
+; NONEON-NOSVE-NEXT:    add w9, w23, w23
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    and w8, w5, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #24]
+; NONEON-NOSVE-NEXT:    add w5, w22, w22
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #39]
+; NONEON-NOSVE-NEXT:    add w9, w21, w21
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
+; NONEON-NOSVE-NEXT:    and w8, w5, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #23]
+; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #38]
+; NONEON-NOSVE-NEXT:    add w5, w20, w20
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #22]
+; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #37]
+; NONEON-NOSVE-NEXT:    add w9, w19, w19
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    and w8, w5, #0xff
+; NONEON-NOSVE-NEXT:    add w0, w16, w16
+; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #36]
+; NONEON-NOSVE-NEXT:    add w4, w4, w4
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
+; NONEON-NOSVE-NEXT:    add w18, w15, w15
+; NONEON-NOSVE-NEXT:    and w6, w0, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #35]
+; NONEON-NOSVE-NEXT:    add w9, w3, w3
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #110]
+; NONEON-NOSVE-NEXT:    and w8, w4, #0xff
+; NONEON-NOSVE-NEXT:    and w7, w18, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #34]
+; NONEON-NOSVE-NEXT:    add w2, w2, w2
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #108]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #33]
+; NONEON-NOSVE-NEXT:    add w9, w0, w0
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #106]
+; NONEON-NOSVE-NEXT:    and w8, w2, #0xff
 ; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
-; NONEON-NOSVE-NEXT:    add w9, w26, w26
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
-; NONEON-NOSVE-NEXT:    add w8, w4, w4
-; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
-; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
-; NONEON-NOSVE-NEXT:    add w9, w24, w24
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    add w8, w2, w2
-; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w17, w17, w17
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
-; NONEON-NOSVE-NEXT:    add w9, w22, w22
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
-; NONEON-NOSVE-NEXT:    add w8, w16, w16
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
-; NONEON-NOSVE-NEXT:    add w17, w30, w30
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
-; NONEON-NOSVE-NEXT:    add w9, w20, w20
-; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
-; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
-; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
+; NONEON-NOSVE-NEXT:    add w18, w18, w18
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #104]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
 ; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
+; NONEON-NOSVE-NEXT:    add w9, w17, w17
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #102]
+; NONEON-NOSVE-NEXT:    and w8, w18, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #100]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
 ; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
-; NONEON-NOSVE-NEXT:    add w9, w7, w7
-; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
-; NONEON-NOSVE-NEXT:    add w9, w5, w5
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
-; NONEON-NOSVE-NEXT:    add w9, w3, w3
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #95]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
-; NONEON-NOSVE-NEXT:    add w9, w0, w0
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #94]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
 ; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #93]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #98]
+; NONEON-NOSVE-NEXT:    and w8, w16, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
+; NONEON-NOSVE-NEXT:    add w14, w14, w14
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #96]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
 ; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #94]
+; NONEON-NOSVE-NEXT:    and w8, w14, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
+; NONEON-NOSVE-NEXT:    add w12, w12, w12
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #92]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
 ; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #91]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #90]
+; NONEON-NOSVE-NEXT:    and w8, w12, #0xff
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #88]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
 ; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #90]
+; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #20]
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #86]
+; NONEON-NOSVE-NEXT:    and w8, w10, #0xff
+; NONEON-NOSVE-NEXT:    ldr w10, [sp, #12] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    strh w6, [sp, #78]
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #256] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #89]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #240] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #87]
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #224] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #86]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #208] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #85]
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #192] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #176] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #83]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #82]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #81]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #111]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #112]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #110]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #109]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #108]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #107]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #106]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #105]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #103]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #102]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #101]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #99]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #98]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #97]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #144]
+; NONEON-NOSVE-NEXT:    add w6, w30, w30
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #84]
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
+; NONEON-NOSVE-NEXT:    and w6, w6, #0xff
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #82]
+; NONEON-NOSVE-NEXT:    and w8, w10, #0xff
+; NONEON-NOSVE-NEXT:    strh w7, [sp, #76]
+; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #192] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    strh w6, [sp, #72]
+; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #176] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    strh w8, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #160] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #144] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
+; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #128] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #272
+; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #112] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add sp, sp, #208
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
@@ -2387,42 +1730,24 @@ define void @zext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i8_v8i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #38]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #36]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #34]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %b = zext <8 x i8> %a to <8 x i32>
   store <8 x i32>%b, ptr %out
@@ -2448,75 +1773,39 @@ define void @zext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i8_v16i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #27]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #25]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #31]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #29]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #17]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #23]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #21]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %b = zext <16 x i8> %a to <16 x i32>
   store <16 x i32> %b, ptr %out
@@ -2555,14 +1844,14 @@ define void @zext_v32i8_v32i32(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v32i8_v32i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #464
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #368] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #384] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #400] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #416] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #432] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #448] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 464
+; NONEON-NOSVE-NEXT:    sub sp, sp, #272
+; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #176] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #192] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #208] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #224] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #240] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #256] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 272
 ; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
 ; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
 ; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
@@ -2577,258 +1866,136 @@ define void @zext_v32i8_v32i32(ptr %in, ptr %out) {
 ; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
-; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #44]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #45]
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #18]
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #19]
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #17]
+; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
+; NONEON-NOSVE-NEXT:    add w18, w15, w15
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #22]
+; NONEON-NOSVE-NEXT:    add w0, w16, w16
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
+; NONEON-NOSVE-NEXT:    and w19, w18, #0xff
+; NONEON-NOSVE-NEXT:    and w7, w0, #0xff
+; NONEON-NOSVE-NEXT:    add w6, w17, w17
+; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #21]
+; NONEON-NOSVE-NEXT:    stp w19, w7, [sp, #104]
+; NONEON-NOSVE-NEXT:    add w7, w30, w30
 ; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    add w8, w29, w29
-; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
-; NONEON-NOSVE-NEXT:    add w8, w27, w27
-; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w8, w25, w25
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w8, w23, w23
-; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w8, w21, w21
-; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w8, w19, w19
-; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
-; NONEON-NOSVE-NEXT:    add w9, w28, w28
-; NONEON-NOSVE-NEXT:    add w18, w16, w16
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w8, w6, w6
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
-; NONEON-NOSVE-NEXT:    add w9, w26, w26
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
-; NONEON-NOSVE-NEXT:    add w8, w4, w4
-; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
-; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
+; NONEON-NOSVE-NEXT:    and w6, w6, #0xff
+; NONEON-NOSVE-NEXT:    and w7, w7, #0xff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #27]
+; NONEON-NOSVE-NEXT:    stp w7, w6, [sp, #96]
+; NONEON-NOSVE-NEXT:    add w6, w29, w29
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #88]
+; NONEON-NOSVE-NEXT:    add w8, w28, w28
+; NONEON-NOSVE-NEXT:    and w6, w6, #0xff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #25]
+; NONEON-NOSVE-NEXT:    add w9, w27, w27
+; NONEON-NOSVE-NEXT:    stp w8, w6, [sp, #80]
+; NONEON-NOSVE-NEXT:    add w6, w26, w26
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #30]
+; NONEON-NOSVE-NEXT:    and w6, w6, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #31]
+; NONEON-NOSVE-NEXT:    add w8, w25, w25
+; NONEON-NOSVE-NEXT:    stp w6, w9, [sp, #72]
 ; NONEON-NOSVE-NEXT:    add w9, w24, w24
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    add w8, w2, w2
-; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w17, w17, w17
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
-; NONEON-NOSVE-NEXT:    add w9, w22, w22
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
-; NONEON-NOSVE-NEXT:    add w8, w16, w16
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
-; NONEON-NOSVE-NEXT:    add w17, w30, w30
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #29]
+; NONEON-NOSVE-NEXT:    add w6, w23, w23
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #64]
+; NONEON-NOSVE-NEXT:    add w8, w22, w22
+; NONEON-NOSVE-NEXT:    and w9, w6, #0xff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #34]
+; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #35]
+; NONEON-NOSVE-NEXT:    add w6, w21, w21
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
 ; NONEON-NOSVE-NEXT:    add w9, w20, w20
-; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
+; NONEON-NOSVE-NEXT:    and w8, w6, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #32]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #33]
+; NONEON-NOSVE-NEXT:    add w5, w5, w5
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #48]
+; NONEON-NOSVE-NEXT:    add w8, w4, w4
+; NONEON-NOSVE-NEXT:    and w9, w5, #0xff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #38]
+; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #39]
+; NONEON-NOSVE-NEXT:    add w3, w3, w3
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #168]
+; NONEON-NOSVE-NEXT:    add w9, w2, w2
+; NONEON-NOSVE-NEXT:    and w8, w3, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #36]
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #37]
+; NONEON-NOSVE-NEXT:    add w0, w0, w0
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #160]
+; NONEON-NOSVE-NEXT:    add w8, w18, w18
+; NONEON-NOSVE-NEXT:    and w9, w0, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #42]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #43]
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
+; NONEON-NOSVE-NEXT:    add w9, w16, w16
+; NONEON-NOSVE-NEXT:    and w8, w17, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #41]
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #144]
 ; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
-; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
-; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
-; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
-; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
-; NONEON-NOSVE-NEXT:    add w9, w7, w7
-; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
+; NONEON-NOSVE-NEXT:    and w9, w15, #0xff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #46]
+; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #47]
+; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
+; NONEON-NOSVE-NEXT:    add w9, w12, w12
+; NONEON-NOSVE-NEXT:    and w8, w13, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #128]
 ; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
-; NONEON-NOSVE-NEXT:    add w9, w5, w5
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
-; NONEON-NOSVE-NEXT:    add w9, w3, w3
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #91]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
-; NONEON-NOSVE-NEXT:    add w9, w0, w0
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #90]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
-; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #89]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
-; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
-; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #95]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #94]
+; NONEON-NOSVE-NEXT:    and w9, w11, #0xff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldr w10, [sp, #8] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
+; NONEON-NOSVE-NEXT:    ldr w9, [sp, #12] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #80]
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #448] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #93]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #432] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #83]
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #416] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #82]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #81]
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #400] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #384] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #87]
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #368] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #86]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #85]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #107]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #106]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #105]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #182]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #111]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #110]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #109]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #108]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #99]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #98]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #97]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #103]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #102]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #101]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #198]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #196]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #194]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #192]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #206]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #204]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #202]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #200]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #180]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #272]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #178]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #176]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #190]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #188]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #186]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #184]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #230]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #240]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #228]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #226]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #224]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #238]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #236]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #234]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #232]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #214]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #336]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #212]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #210]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #222]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #220]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #218]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #216]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #304]
+; NONEON-NOSVE-NEXT:    and w8, w10, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #112]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #144]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #112]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
+; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #256] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
+; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #240] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
+; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #224] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #464
+; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #208] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #192] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #176] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add sp, sp, #272
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
@@ -2858,26 +2025,20 @@ define void @zext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v4i8_v4i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #40]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %b = zext <4 x i8> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
@@ -2904,61 +2065,30 @@ define void @zext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i8_v8i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #176
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 176
+; NONEON-NOSVE-NEXT:    sub sp, sp, #80
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add x8, sp, #144
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #22]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #44]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #34]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #38]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #36]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #96]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #104]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #168]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x8]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #72]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #176
+; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
+; NONEON-NOSVE-NEXT:    add sp, sp, #80
 ; NONEON-NOSVE-NEXT:    ret
   %b = zext <8 x i8> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
@@ -2998,129 +2128,51 @@ define void @zext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i8_v16i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #368
-; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
-; NONEON-NOSVE-NEXT:    str q0, [sp]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #332]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #324]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #348]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #35]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #340]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #300]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #34]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #292]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #33]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #316]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #32]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #308]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #39]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #268]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #38]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #260]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #37]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #284]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #36]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #276]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #152]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #98]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #144]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #136]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #128]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #128]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #120]
 ; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #102]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #216]
-; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #192]
-; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #104]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #364]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #328]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #360]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #200]
-; NONEON-NOSVE-NEXT:    str d2, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #320]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #296]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #312]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #184]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #288]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #264]
-; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #252]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #192]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #280]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #256]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #232]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #176]
-; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #244]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #224]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #112]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #104]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #88]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #72]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #40]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x0, #32]
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x0, #64]
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #368
+; NONEON-NOSVE-NEXT:    add sp, sp, #160
 ; NONEON-NOSVE-NEXT:    ret
   %b = zext <16 x i8> %a to <16 x i64>
   store <16 x i64> %b, ptr %out
@@ -3192,14 +2244,14 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v32i8_v32i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sub sp, sp, #752
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 848
+; NONEON-NOSVE-NEXT:    sub sp, sp, #400
+; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #304] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #320] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #336] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #352] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #368] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #384] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 400
 ; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
 ; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
 ; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
@@ -3213,379 +2265,168 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
 ; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
 ; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #572]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #564]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #588]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #172]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #292]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #300]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
-; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #46]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #47]
+; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
+; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #22]
+; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #23]
 ; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
-; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
 ; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
-; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
+; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #25]
+; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #26]
 ; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    add w8, w29, w29
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #27]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #29]
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #140]
+; NONEON-NOSVE-NEXT:    add w8, w30, w30
+; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #28]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #148]
+; NONEON-NOSVE-NEXT:    add w9, w29, w29
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #132]
 ; NONEON-NOSVE-NEXT:    add w8, w27, w27
-; NONEON-NOSVE-NEXT:    add w18, w16, w16
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w8, w25, w25
-; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w8, w23, w23
-; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w8, w21, w21
-; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w8, w19, w19
-; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #124]
 ; NONEON-NOSVE-NEXT:    add w9, w28, w28
-; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w8, w6, w6
-; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
-; NONEON-NOSVE-NEXT:    add w9, w26, w26
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
-; NONEON-NOSVE-NEXT:    add w8, w4, w4
-; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
-; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
-; NONEON-NOSVE-NEXT:    add w17, w17, w17
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
-; NONEON-NOSVE-NEXT:    add w9, w24, w24
-; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    add w8, w2, w2
-; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
-; NONEON-NOSVE-NEXT:    add w9, w22, w22
-; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
-; NONEON-NOSVE-NEXT:    add w8, w16, w16
-; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
-; NONEON-NOSVE-NEXT:    add w17, w30, w30
-; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #108]
+; NONEON-NOSVE-NEXT:    add w8, w26, w26
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #116]
+; NONEON-NOSVE-NEXT:    add w9, w25, w25
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #100]
+; NONEON-NOSVE-NEXT:    add w8, w24, w24
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #92]
+; NONEON-NOSVE-NEXT:    add w9, w21, w21
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #30]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #76]
+; NONEON-NOSVE-NEXT:    add w8, w19, w19
+; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #31]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #84]
 ; NONEON-NOSVE-NEXT:    add w9, w20, w20
-; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
-; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
-; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
-; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
-; NONEON-NOSVE-NEXT:    add w9, w7, w7
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #33]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #68]
+; NONEON-NOSVE-NEXT:    add w8, w7, w7
+; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #60]
+; NONEON-NOSVE-NEXT:    add w9, w6, w6
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #17]
+; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #34]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    str w8, [sp, #48]
+; NONEON-NOSVE-NEXT:    add w8, w4, w4
+; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #35]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #52]
 ; NONEON-NOSVE-NEXT:    add w9, w5, w5
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
-; NONEON-NOSVE-NEXT:    add w9, w3, w3
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #91]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    add w0, w16, w16
+; NONEON-NOSVE-NEXT:    add w22, w18, w18
+; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #37]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
+; NONEON-NOSVE-NEXT:    add w8, w3, w3
+; NONEON-NOSVE-NEXT:    and w23, w0, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #36]
+; NONEON-NOSVE-NEXT:    str w9, [sp, #288]
+; NONEON-NOSVE-NEXT:    add w9, w2, w2
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #38]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
+; NONEON-NOSVE-NEXT:    add w8, w18, w18
+; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #39]
+; NONEON-NOSVE-NEXT:    str w9, [sp, #280]
 ; NONEON-NOSVE-NEXT:    add w9, w0, w0
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #90]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
-; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #89]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #41]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
+; NONEON-NOSVE-NEXT:    add w8, w17, w17
+; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #40]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #252]
+; NONEON-NOSVE-NEXT:    add w9, w16, w16
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #42]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #236]
+; NONEON-NOSVE-NEXT:    add w8, w15, w15
+; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #43]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #244]
+; NONEON-NOSVE-NEXT:    add w9, w14, w14
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #45]
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #228]
+; NONEON-NOSVE-NEXT:    add w8, w12, w12
+; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #44]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #220]
 ; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
-; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #95]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #94]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #204]
+; NONEON-NOSVE-NEXT:    add w8, w11, w11
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #212]
+; NONEON-NOSVE-NEXT:    add w9, w10, w10
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #196]
+; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #188]
+; NONEON-NOSVE-NEXT:    ldr w9, [sp, #12] // 4-byte Folded Reload
+; NONEON-NOSVE-NEXT:    and w22, w22, #0xff
+; NONEON-NOSVE-NEXT:    add w8, w8, w8
+; NONEON-NOSVE-NEXT:    stp wzr, w22, [sp, #164]
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #580]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #93]
-; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #604]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #83]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #596]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #82]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #81]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #620]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #87]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #612]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #86]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #508]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #85]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #500]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #524]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #107]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #516]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #106]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #540]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #105]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #178]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #532]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #111]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #556]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #110]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #548]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #109]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #700]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #108]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #692]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #99]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #716]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #98]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #708]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #97]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #732]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #103]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #724]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #102]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #748]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #101]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #740]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #636]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #194]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #628]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #192]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #652]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #198]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #196]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #644]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #202]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #272]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #668]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #200]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #660]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #206]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #400]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #204]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #684]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #176]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #288]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #676]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #182]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #180]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #186]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #416]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #240]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #184]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #190]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #368]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #188]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #226]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #256]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #224]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #230]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #384]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #228]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #234]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #336]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #232]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #238]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #464]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #236]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #210]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #352]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #214]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #480]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #212]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #218]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #304]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #216]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #222]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #432]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #220]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #404]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #320]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #568]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #400]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #560]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #412]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #448]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #584]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #408]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #576]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #420]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #560]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #600]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #416]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #592]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #428]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #616]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #424]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #608]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #372]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [sp, #592]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #504]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #368]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #496]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #380]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #520]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #376]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #512]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #388]
-; NONEON-NOSVE-NEXT:    ldp q4, q5, [sp, #496]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #536]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #384]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #528]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #396]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #552]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #392]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #544]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #468]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #528]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #696]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #464]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #688]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #476]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #712]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #472]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #704]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #484]
-; NONEON-NOSVE-NEXT:    ldp q16, q17, [sp, #688]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #728]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #480]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #720]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #492]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #744]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #488]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #736]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #436]
-; NONEON-NOSVE-NEXT:    ldp q19, q20, [sp, #720]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #632]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #432]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #624]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #444]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #648]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #440]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #640]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #452]
-; NONEON-NOSVE-NEXT:    ldp q22, q23, [sp, #624]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #664]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #448]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #656]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #460]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #680]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #456]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #672]
-; NONEON-NOSVE-NEXT:    ldp q21, q18, [sp, #656]
+; NONEON-NOSVE-NEXT:    stp wzr, w23, [sp, #156]
+; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #180]
+; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #276]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #144]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #284]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #112]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #260]
+; NONEON-NOSVE-NEXT:    ldp q5, q4, [sp, #80]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #268]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #48]
+; NONEON-NOSVE-NEXT:    str w8, [sp, #176]
+; NONEON-NOSVE-NEXT:    ldp q17, q16, [sp, #272]
+; NONEON-NOSVE-NEXT:    ldp q18, q21, [sp, #176]
+; NONEON-NOSVE-NEXT:    ldp q20, q19, [sp, #240]
+; NONEON-NOSVE-NEXT:    ldp q23, q22, [sp, #208]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
+; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #384] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1, #32]
+; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #368] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q4, q5, [x1, #64]
+; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #352] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #96]
+; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #336] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q16, q17, [x1, #128]
+; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #320] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q19, q20, [x1, #160]
+; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #304] // 16-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q22, q23, [x1, #192]
 ; NONEON-NOSVE-NEXT:    stp q21, q18, [x1, #224]
-; NONEON-NOSVE-NEXT:    add sp, sp, #752
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp], #96 // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add sp, sp, #400
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
@@ -3653,91 +2494,70 @@ define void @zext_v16i16_v16i32(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i16_v16i32:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
-; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
+; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
-; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
-; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
-; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
-; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
-; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #4]
+; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #2]
+; NONEON-NOSVE-NEXT:    ldrh w3, [sp]
+; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #12]
+; NONEON-NOSVE-NEXT:    ldrh w5, [sp, #14]
 ; NONEON-NOSVE-NEXT:    add w14, w14, w14
-; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
+; NONEON-NOSVE-NEXT:    add w17, w17, w17
+; NONEON-NOSVE-NEXT:    add w16, w16, w16
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    and w17, w17, #0xffff
+; NONEON-NOSVE-NEXT:    and w16, w16, #0xffff
+; NONEON-NOSVE-NEXT:    stp w17, w14, [sp, #56]
 ; NONEON-NOSVE-NEXT:    add w14, w3, w3
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w13, w5, w5
+; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #8]
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #10]
+; NONEON-NOSVE-NEXT:    add w17, w5, w5
+; NONEON-NOSVE-NEXT:    stp w14, w16, [sp, #48]
+; NONEON-NOSVE-NEXT:    add w16, w4, w4
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
-; NONEON-NOSVE-NEXT:    add w14, w4, w4
 ; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
-; NONEON-NOSVE-NEXT:    add w13, w2, w2
-; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #38]
-; NONEON-NOSVE-NEXT:    add w14, w0, w0
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #36]
-; NONEON-NOSVE-NEXT:    add w13, w18, w18
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #34]
 ; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
 ; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w14, w17, w17
+; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #18]
+; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #22]
+; NONEON-NOSVE-NEXT:    and w14, w17, #0xffff
+; NONEON-NOSVE-NEXT:    and w16, w16, #0xffff
+; NONEON-NOSVE-NEXT:    add w17, w2, w2
+; NONEON-NOSVE-NEXT:    stp w16, w14, [sp, #40]
+; NONEON-NOSVE-NEXT:    add w14, w0, w0
+; NONEON-NOSVE-NEXT:    and w16, w17, #0xffff
+; NONEON-NOSVE-NEXT:    add w17, w18, w18
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    add w13, w13, w13
 ; NONEON-NOSVE-NEXT:    add w12, w12, w12
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w13, w16, w16
 ; NONEON-NOSVE-NEXT:    add w11, w11, w11
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    add w10, w10, w10
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #62]
-; NONEON-NOSVE-NEXT:    add w14, w15, w15
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #60]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #58]
-; NONEON-NOSVE-NEXT:    strh w12, [sp, #56]
-; NONEON-NOSVE-NEXT:    strh w11, [sp, #54]
-; NONEON-NOSVE-NEXT:    strh w10, [sp, #52]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
+; NONEON-NOSVE-NEXT:    add w9, w9, w9
+; NONEON-NOSVE-NEXT:    add w8, w8, w8
+; NONEON-NOSVE-NEXT:    stp w14, w16, [sp, #32]
+; NONEON-NOSVE-NEXT:    and w14, w17, #0xffff
+; NONEON-NOSVE-NEXT:    and w15, w15, #0xffff
+; NONEON-NOSVE-NEXT:    and w13, w13, #0xffff
+; NONEON-NOSVE-NEXT:    and w12, w12, #0xffff
+; NONEON-NOSVE-NEXT:    and w11, w11, #0xffff
+; NONEON-NOSVE-NEXT:    and w10, w10, #0xffff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xffff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xffff
+; NONEON-NOSVE-NEXT:    stp w15, w14, [sp, #88]
+; NONEON-NOSVE-NEXT:    stp w12, w13, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]
+; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
@@ -3763,26 +2583,20 @@ define void @zext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v4i16_v4i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #40]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #24]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %b = zext <4 x i16> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
@@ -3808,43 +2622,31 @@ define void @zext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i16_v8i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #88]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #72]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #56]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #40]
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #104]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %b = zext <8 x i16> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
@@ -3883,144 +2685,85 @@ define void @zext_v16i16_v16i64(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i16_v16i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #368
-; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
+; NONEON-NOSVE-NEXT:    sub sp, sp, #160
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #268]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #260]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #284]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #92]
+; NONEON-NOSVE-NEXT:    str wzr, [sp, #156]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
-; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
-; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
-; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
-; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
-; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
-; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
-; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    ldrh w14, [sp]
+; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #2]
+; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #4]
+; NONEON-NOSVE-NEXT:    ldrh w5, [sp, #6]
+; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #8]
+; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #10]
 ; NONEON-NOSVE-NEXT:    add w14, w14, w14
-; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #54]
-; NONEON-NOSVE-NEXT:    add w14, w3, w3
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #52]
-; NONEON-NOSVE-NEXT:    add w13, w5, w5
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #50]
-; NONEON-NOSVE-NEXT:    add w14, w4, w4
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #48]
-; NONEON-NOSVE-NEXT:    add w13, w2, w2
-; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
+; NONEON-NOSVE-NEXT:    add w15, w15, w15
+; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #12]
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    and w15, w15, #0xffff
+; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp wzr, w15, [sp, #84]
+; NONEON-NOSVE-NEXT:    add w15, w4, w4
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
+; NONEON-NOSVE-NEXT:    stp wzr, w14, [sp, #76]
+; NONEON-NOSVE-NEXT:    add w14, w5, w5
+; NONEON-NOSVE-NEXT:    and w15, w15, #0xffff
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    stp wzr, w15, [sp, #60]
+; NONEON-NOSVE-NEXT:    add w15, w3, w3
+; NONEON-NOSVE-NEXT:    stp wzr, w14, [sp, #68]
+; NONEON-NOSVE-NEXT:    add w14, w2, w2
+; NONEON-NOSVE-NEXT:    and w15, w15, #0xffff
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]
+; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #20]
+; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #22]
+; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp wzr, w15, [sp, #52]
+; NONEON-NOSVE-NEXT:    add w15, w18, w18
+; NONEON-NOSVE-NEXT:    stp wzr, w14, [sp, #44]
 ; NONEON-NOSVE-NEXT:    add w14, w0, w0
-; NONEON-NOSVE-NEXT:    add w9, w9, w9
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
-; NONEON-NOSVE-NEXT:    add w13, w18, w18
-; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
-; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
-; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
-; NONEON-NOSVE-NEXT:    add w14, w17, w17
-; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
-; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
+; NONEON-NOSVE-NEXT:    and w15, w15, #0xffff
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    add w13, w13, w13
+; NONEON-NOSVE-NEXT:    stp wzr, w14, [sp, #36]
+; NONEON-NOSVE-NEXT:    add w14, w16, w16
 ; NONEON-NOSVE-NEXT:    add w12, w12, w12
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    str w15, [sp, #32]
+; NONEON-NOSVE-NEXT:    add w15, w17, w17
 ; NONEON-NOSVE-NEXT:    add w10, w10, w10
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    add w13, w16, w16
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #82]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #80]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #70]
-; NONEON-NOSVE-NEXT:    add w14, w15, w15
-; NONEON-NOSVE-NEXT:    strh w13, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #86]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #84]
-; NONEON-NOSVE-NEXT:    strh w14, [sp, #66]
-; NONEON-NOSVE-NEXT:    strh w12, [sp, #64]
-; NONEON-NOSVE-NEXT:    strh w11, [sp, #62]
-; NONEON-NOSVE-NEXT:    strh w10, [sp, #60]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #74]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #276]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #332]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #78]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #76]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #98]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #324]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #102]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #100]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #184]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #104]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #90]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #348]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #94]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #92]
-; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #184]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
-; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #340]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #264]
-; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #252]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #192]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
-; NONEON-NOSVE-NEXT:    str d2, [sp, #200]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #280]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #168]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #300]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #292]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #232]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #176]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #316]
-; NONEON-NOSVE-NEXT:    str wzr, [sp, #308]
-; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #244]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #240]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #216]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #224]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #364]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #328]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #360]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #200]
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #320]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #296]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #208]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #312]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
-; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #288]
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    add w9, w9, w9
+; NONEON-NOSVE-NEXT:    add w8, w8, w8
+; NONEON-NOSVE-NEXT:    and w14, w14, #0xffff
+; NONEON-NOSVE-NEXT:    and w15, w15, #0xffff
+; NONEON-NOSVE-NEXT:    and w13, w13, #0xffff
+; NONEON-NOSVE-NEXT:    and w12, w12, #0xffff
+; NONEON-NOSVE-NEXT:    and w10, w10, #0xffff
+; NONEON-NOSVE-NEXT:    and w11, w11, #0xffff
+; NONEON-NOSVE-NEXT:    and w9, w9, #0xffff
+; NONEON-NOSVE-NEXT:    and w8, w8, #0xffff
+; NONEON-NOSVE-NEXT:    stp wzr, w15, [sp, #148]
+; NONEON-NOSVE-NEXT:    stp wzr, w14, [sp, #140]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #64]
+; NONEON-NOSVE-NEXT:    stp wzr, w12, [sp, #132]
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp wzr, w13, [sp, #124]
+; NONEON-NOSVE-NEXT:    stp wzr, w11, [sp, #116]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #128]
+; NONEON-NOSVE-NEXT:    stp wzr, w10, [sp, #108]
+; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #100]
+; NONEON-NOSVE-NEXT:    str w8, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #96]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #368
+; NONEON-NOSVE-NEXT:    add sp, sp, #160
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
@@ -4082,47 +2825,34 @@ define void @zext_v8i32_v8i64(ptr %in, ptr %out) {
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i32_v8i64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp]
-; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldp w14, w15, [sp, #16]
+; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
+; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp]
+; NONEON-NOSVE-NEXT:    ldp w14, w15, [sp, #8]
+; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #16]
+; NONEON-NOSVE-NEXT:    add w10, w10, w10
+; NONEON-NOSVE-NEXT:    add w11, w11, w11
+; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #24]
+; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #48]
+; NONEON-NOSVE-NEXT:    add w10, w15, w15
+; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #40]
+; NONEON-NOSVE-NEXT:    add w10, w14, w14
+; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #32]
+; NONEON-NOSVE-NEXT:    add w10, w13, w13
 ; NONEON-NOSVE-NEXT:    add w9, w9, w9
+; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #88]
+; NONEON-NOSVE-NEXT:    add w10, w12, w12
 ; NONEON-NOSVE-NEXT:    add w8, w8, w8
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    add w9, w13, w13
-; NONEON-NOSVE-NEXT:    add w8, w12, w12
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    add w9, w15, w15
-; NONEON-NOSVE-NEXT:    add w8, w14, w14
-; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    add w9, w11, w11
-; NONEON-NOSVE-NEXT:    add w8, w10, w10
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #104]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #144]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
+; NONEON-NOSVE-NEXT:    stp w11, wzr, [sp, #56]
+; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #80]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #32]
+; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #72]
+; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i32>, ptr %in
   %b = add <8 x i32> %a, %a
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
index 97f2e7a1e66c..0c97eedd4362 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
@@ -973,11 +973,11 @@ define <4 x i32> @smulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
 ; NONEON-NOSVE-NEXT:    ldpsw x13, x12, [sp, #48]
 ; NONEON-NOSVE-NEXT:    smull x11, w11, w12
-; NONEON-NOSVE-NEXT:    ldpsw x12, x14, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldpsw x14, x12, [sp, #56]
 ; NONEON-NOSVE-NEXT:    smull x10, w10, w13
 ; NONEON-NOSVE-NEXT:    lsr x11, x11, #32
-; NONEON-NOSVE-NEXT:    smull x9, w9, w14
-; NONEON-NOSVE-NEXT:    smull x8, w8, w12
+; NONEON-NOSVE-NEXT:    smull x9, w9, w12
+; NONEON-NOSVE-NEXT:    smull x8, w8, w14
 ; NONEON-NOSVE-NEXT:    lsr x10, x10, #32
 ; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
 ; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]
@@ -1038,12 +1038,12 @@ define void @smulh_v8i32(ptr %a, ptr %b) {
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
 ; NONEON-NOSVE-NEXT:    ldpsw x17, x16, [sp, #112]
 ; NONEON-NOSVE-NEXT:    smull x15, w15, w16
-; NONEON-NOSVE-NEXT:    ldpsw x16, x18, [sp, #120]
+; NONEON-NOSVE-NEXT:    ldpsw x18, x16, [sp, #120]
 ; NONEON-NOSVE-NEXT:    smull x14, w14, w17
 ; NONEON-NOSVE-NEXT:    ldpsw x17, x1, [sp, #80]
-; NONEON-NOSVE-NEXT:    smull x13, w13, w18
+; NONEON-NOSVE-NEXT:    smull x13, w13, w16
 ; NONEON-NOSVE-NEXT:    lsr x15, x15, #32
-; NONEON-NOSVE-NEXT:    smull x12, w12, w16
+; NONEON-NOSVE-NEXT:    smull x12, w12, w18
 ; NONEON-NOSVE-NEXT:    lsr x14, x14, #32
 ; NONEON-NOSVE-NEXT:    ldpsw x16, x18, [sp, #88]
 ; NONEON-NOSVE-NEXT:    smull x11, w11, w1
@@ -2172,11 +2172,11 @@ define <4 x i32> @umulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
 ; NONEON-NOSVE-NEXT:    ldp w13, w12, [sp, #48]
 ; NONEON-NOSVE-NEXT:    umull x11, w11, w12
-; NONEON-NOSVE-NEXT:    ldp w12, w14, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldp w14, w12, [sp, #56]
 ; NONEON-NOSVE-NEXT:    umull x10, w10, w13
 ; NONEON-NOSVE-NEXT:    lsr x11, x11, #32
-; NONEON-NOSVE-NEXT:    umull x9, w9, w14
-; NONEON-NOSVE-NEXT:    umull x8, w8, w12
+; NONEON-NOSVE-NEXT:    umull x9, w9, w12
+; NONEON-NOSVE-NEXT:    umull x8, w8, w14
 ; NONEON-NOSVE-NEXT:    lsr x10, x10, #32
 ; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
 ; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]
@@ -2237,12 +2237,12 @@ define void @umulh_v8i32(ptr %a, ptr %b) {
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
 ; NONEON-NOSVE-NEXT:    ldp w17, w16, [sp, #112]
 ; NONEON-NOSVE-NEXT:    umull x15, w15, w16
-; NONEON-NOSVE-NEXT:    ldp w16, w18, [sp, #120]
+; NONEON-NOSVE-NEXT:    ldp w18, w16, [sp, #120]
 ; NONEON-NOSVE-NEXT:    umull x14, w14, w17
 ; NONEON-NOSVE-NEXT:    ldp w17, w1, [sp, #80]
-; NONEON-NOSVE-NEXT:    umull x13, w13, w18
+; NONEON-NOSVE-NEXT:    umull x13, w13, w16
 ; NONEON-NOSVE-NEXT:    lsr x15, x15, #32
-; NONEON-NOSVE-NEXT:    umull x12, w12, w16
+; NONEON-NOSVE-NEXT:    umull x12, w12, w18
 ; NONEON-NOSVE-NEXT:    lsr x14, x14, #32
 ; NONEON-NOSVE-NEXT:    ldp w16, w18, [sp, #88]
 ; NONEON-NOSVE-NEXT:    umull x11, w11, w1
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
index 46a2ce6ed710..7df362826d05 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
@@ -452,29 +452,23 @@ define void @ucvtf_v4i16_v4f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v4i16_v4f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    ldr d0, [x0]
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i16>, ptr %a
   %res = uitofp <4 x i16> %op1 to <4 x double>
@@ -506,49 +500,36 @@ define void @ucvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v8i16_v8f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp]
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #22]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #144]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #128]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = uitofp <8 x i16> %op1 to <8 x double>
@@ -602,92 +583,63 @@ define void @ucvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v16i16_v16f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #336
-; NONEON-NOSVE-NEXT:    str x29, [sp, #320] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 336
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
+; NONEON-NOSVE-NEXT:    sub sp, sp, #192
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 192
 ; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #320] // 8-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #40]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #50]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #54]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #42]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
 ; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #58]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldp d2, d1, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    str d1, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #160]
-; NONEON-NOSVE-NEXT:    stp d1, d2, [sp, #176]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #168]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #240]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #44]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #224]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #34]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #38]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #36]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #332]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #192]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    ucvtf d1, w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #192]
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #64]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #184]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #304]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #62]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #60]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #176]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #288]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #160]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #160]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #288]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #272]
-; NONEON-NOSVE-NEXT:    ucvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #54]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #144]
+; NONEON-NOSVE-NEXT:    ucvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #52]
 ; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #256]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #128]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #128]
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #336
+; NONEON-NOSVE-NEXT:    add sp, sp, #192
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = uitofp <16 x i16> %op1 to <16 x double>
@@ -1891,29 +1843,23 @@ define void @scvtf_v4i16_v4f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v4i16_v4f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #80
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
+; NONEON-NOSVE-NEXT:    sub sp, sp, #48
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
 ; NONEON-NOSVE-NEXT:    ldr d0, [x0]
 ; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #10]
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #10]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
 ; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #14]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #14]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #16]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #80
+; NONEON-NOSVE-NEXT:    add sp, sp, #48
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i16>, ptr %a
   %res = sitofp <4 x i16> %op1 to <4 x double>
@@ -1945,49 +1891,36 @@ define void @scvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v8i16_v8f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #160
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
 ; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp]
+; NONEON-NOSVE-NEXT:    str q0, [sp, #-96]!
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
 ; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #26]
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #26]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
 ; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #24]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #30]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #18]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #22]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #144]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #30]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #28]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #128]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #18]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #64]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #22]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #20]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
 ; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #160
+; NONEON-NOSVE-NEXT:    add sp, sp, #96
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = sitofp <8 x i16> %op1 to <8 x double>
@@ -2041,92 +1974,63 @@ define void @scvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v16i16_v16f64:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #336
-; NONEON-NOSVE-NEXT:    str x29, [sp, #320] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 336
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
+; NONEON-NOSVE-NEXT:    sub sp, sp, #192
+; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 192
 ; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr x29, [sp, #320] // 8-byte Folded Reload
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [sp]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #40]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #50]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #48]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #54]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #52]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #42]
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #42]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
 ; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #88]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #46]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #44]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #66]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #64]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #152]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #70]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #68]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #58]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #56]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #136]
-; NONEON-NOSVE-NEXT:    ldp d2, d1, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
-; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #62]
-; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #60]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
-; NONEON-NOSVE-NEXT:    str d1, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #160]
-; NONEON-NOSVE-NEXT:    stp d1, d2, [sp, #176]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #168]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #240]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #46]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #44]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #144]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #224]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #34]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #32]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #38]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #36]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #332]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #192]
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #58]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
 ; NONEON-NOSVE-NEXT:    scvtf d1, w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #328]
-; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #192]
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #56]
+; NONEON-NOSVE-NEXT:    ldp q4, q3, [sp, #64]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #184]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #304]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #62]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #60]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #176]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #288]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #50]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #160]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #160]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #168]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [sp, #288]
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #272]
-; NONEON-NOSVE-NEXT:    scvtf d1, w9
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #54]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #144]
+; NONEON-NOSVE-NEXT:    scvtf d1, w8
+; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #52]
 ; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #256]
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #224]
-; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #256]
+; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #128]
+; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #96]
+; NONEON-NOSVE-NEXT:    ldp q2, q5, [sp, #128]
 ; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
 ; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
 ; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
 ; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #336
+; NONEON-NOSVE-NEXT:    add sp, sp, #192
 ; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = sitofp <16 x i16> %op1 to <16 x double>
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll
index 41eb731fd66d..39701131d7db 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll
@@ -288,12 +288,12 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask)
 define void @select_v32i8(ptr %a, ptr %b) {
 ; CHECK-LABEL: select_v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    cmpeq p1.b, p0/z, z1.b, z0.b
 ; CHECK-NEXT:    cmpeq p0.b, p0/z, z2.b, z3.b
-; CHECK-NEXT:    sel z0.b, p1, z0.b, z1.b
+; CHECK-NEXT:    mov z0.b, p1/m, z1.b
 ; CHECK-NEXT:    sel z1.b, p0, z2.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
@@ -692,12 +692,12 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) {
 define void @select_v16i16(ptr %a, ptr %b) {
 ; CHECK-LABEL: select_v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.h, vl8
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    cmpeq p1.h, p0/z, z1.h, z0.h
 ; CHECK-NEXT:    cmpeq p0.h, p0/z, z2.h, z3.h
-; CHECK-NEXT:    sel z0.h, p1, z0.h, z1.h
+; CHECK-NEXT:    mov z0.h, p1/m, z1.h
 ; CHECK-NEXT:    sel z1.h, p0, z2.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
@@ -906,12 +906,12 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask) {
 define void @select_v8i32(ptr %a, ptr %b) {
 ; CHECK-LABEL: select_v8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    cmpeq p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    cmpeq p1.s, p0/z, z1.s, z0.s
 ; CHECK-NEXT:    cmpeq p0.s, p0/z, z2.s, z3.s
-; CHECK-NEXT:    sel z0.s, p1, z0.s, z1.s
+; CHECK-NEXT:    mov z0.s, p1/m, z1.s
 ; CHECK-NEXT:    sel z1.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
@@ -930,16 +930,16 @@ define void @select_v8i32(ptr %a, ptr %b) {
 ; NONEON-NOSVE-NEXT:    cmp w10, w9
 ; NONEON-NOSVE-NEXT:    csel w9, w10, w9, eq
 ; NONEON-NOSVE-NEXT:    cmp w13, w12
-; NONEON-NOSVE-NEXT:    ldp w15, w16, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldp w10, w16, [sp, #48]
 ; NONEON-NOSVE-NEXT:    csel w12, w13, w12, eq
 ; NONEON-NOSVE-NEXT:    cmp w14, w11
-; NONEON-NOSVE-NEXT:    ldp w10, w13, [sp, #32]
+; NONEON-NOSVE-NEXT:    ldp w15, w13, [sp, #32]
 ; NONEON-NOSVE-NEXT:    csel w11, w14, w11, eq
 ; NONEON-NOSVE-NEXT:    ldp w17, w14, [sp, #56]
 ; NONEON-NOSVE-NEXT:    ldp w18, w1, [sp, #40]
-; NONEON-NOSVE-NEXT:    cmp w10, w15
+; NONEON-NOSVE-NEXT:    cmp w15, w10
 ; NONEON-NOSVE-NEXT:    stp w12, w11, [sp, #72]
-; NONEON-NOSVE-NEXT:    csel w10, w10, w15, eq
+; NONEON-NOSVE-NEXT:    csel w10, w15, w10, eq
 ; NONEON-NOSVE-NEXT:    cmp w13, w16
 ; NONEON-NOSVE-NEXT:    ldr w15, [sp]
 ; NONEON-NOSVE-NEXT:    csel w13, w13, w16, eq
@@ -1039,12 +1039,12 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) {
 define void @select_v4i64(ptr %a, ptr %b) {
 ; CHECK-LABEL: select_v4i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    cmpeq p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    cmpeq p1.d, p0/z, z1.d, z0.d
 ; CHECK-NEXT:    cmpeq p0.d, p0/z, z2.d, z3.d
-; CHECK-NEXT:    sel z0.d, p1, z0.d, z1.d
+; CHECK-NEXT:    mov z0.d, p1/m, z1.d
 ; CHECK-NEXT:    sel z1.d, p0, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
@@ -1057,13 +1057,13 @@ define void @select_v4i64(ptr %a, ptr %b) {
 ; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
 ; NONEON-NOSVE-NEXT:    stp q2, q3, [sp, #32]
 ; NONEON-NOSVE-NEXT:    ldr x9, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldp x8, x11, [sp, #24]
+; NONEON-NOSVE-NEXT:    ldp x8, x10, [sp, #24]
 ; NONEON-NOSVE-NEXT:    ldr x13, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldp x10, x12, [sp, #48]
+; NONEON-NOSVE-NEXT:    ldp x11, x12, [sp, #48]
 ; NONEON-NOSVE-NEXT:    cmp x9, x8
 ; NONEON-NOSVE-NEXT:    csel x8, x9, x8, eq
-; NONEON-NOSVE-NEXT:    cmp x11, x10
-; NONEON-NOSVE-NEXT:    csel x9, x11, x10, eq
+; NONEON-NOSVE-NEXT:    cmp x10, x11
+; NONEON-NOSVE-NEXT:    csel x9, x10, x11, eq
 ; NONEON-NOSVE-NEXT:    ldr x10, [sp, #16]
 ; NONEON-NOSVE-NEXT:    ldr x11, [sp]
 ; NONEON-NOSVE-NEXT:    cmp x13, x12
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
index e8c9704940c7..e6c6003ee6c6 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
@@ -28,23 +28,17 @@ define void @alloc_v4i8(ptr %st_ptr) nounwind {
 ;
 ; NONEON-NOSVE-LABEL: alloc_v4i8:
 ; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #48
-; NONEON-NOSVE-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT:    sub sp, sp, #32
+; NONEON-NOSVE-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
 ; NONEON-NOSVE-NEXT:    mov x19, x0
-; NONEON-NOSVE-NEXT:    add x0, sp, #28
+; NONEON-NOSVE-NEXT:    add x0, sp, #12
 ; NONEON-NOSVE-NEXT:    bl def
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]
-; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
-; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    str d0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
-; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #16]
+; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
+; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
 ; NONEON-NOSVE-NEXT:    strb w8, [x19, #1]
 ; NONEON-NOSVE-NEXT:    strb w9, [x19]
-; NONEON-NOSVE-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
+; NONEON-NOSVE-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT:    add sp, sp, #32
 ; NONEON-NOSVE-NEXT:    ret
   %alloc = alloca [4 x i8]
   call void @def(ptr %alloc)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
index 3d9f407c3064..e0e88c47fb55 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
@@ -151,20 +151,20 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    .cfi_offset b13, -48
 ; CHECK-NEXT:    .cfi_offset b14, -56
 ; CHECK-NEXT:    .cfi_offset b15, -64
-; CHECK-NEXT:    ldp q1, q0, [x0]
-; CHECK-NEXT:    ldp q2, q3, [x1]
-; CHECK-NEXT:    mov z5.h, z0.h[7]
-; CHECK-NEXT:    mov z7.h, z0.h[6]
-; CHECK-NEXT:    mov z17.h, z0.h[5]
-; CHECK-NEXT:    mov z4.h, z3.h[7]
-; CHECK-NEXT:    mov z6.h, z3.h[6]
-; CHECK-NEXT:    mov z16.h, z3.h[5]
-; CHECK-NEXT:    mov z18.h, z3.h[4]
-; CHECK-NEXT:    mov z19.h, z0.h[4]
-; CHECK-NEXT:    mov z20.h, z2.h[7]
-; CHECK-NEXT:    mov z21.h, z1.h[7]
-; CHECK-NEXT:    mov z22.h, z2.h[6]
-; CHECK-NEXT:    mov z23.h, z1.h[6]
+; CHECK-NEXT:    ldp q1, q2, [x1]
+; CHECK-NEXT:    ldp q0, q3, [x0]
+; CHECK-NEXT:    mov z4.h, z2.h[7]
+; CHECK-NEXT:    mov z6.h, z2.h[6]
+; CHECK-NEXT:    mov z16.h, z2.h[5]
+; CHECK-NEXT:    mov z5.h, z3.h[7]
+; CHECK-NEXT:    mov z7.h, z3.h[6]
+; CHECK-NEXT:    mov z17.h, z3.h[5]
+; CHECK-NEXT:    mov z18.h, z2.h[4]
+; CHECK-NEXT:    mov z19.h, z3.h[4]
+; CHECK-NEXT:    mov z20.h, z1.h[7]
+; CHECK-NEXT:    mov z21.h, z0.h[7]
+; CHECK-NEXT:    mov z22.h, z1.h[6]
+; CHECK-NEXT:    mov z23.h, z0.h[6]
 ; CHECK-NEXT:    zip1 z24.h, z5.h, z4.h
 ; CHECK-NEXT:    zip1 z25.h, z7.h, z6.h
 ; CHECK-NEXT:    zip1 z16.h, z17.h, z16.h
@@ -174,12 +174,12 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    zip1 z18.h, z21.h, z20.h
 ; CHECK-NEXT:    zip1 z21.s, z25.s, z24.s
 ; CHECK-NEXT:    zip1 z22.h, z23.h, z22.h
-; CHECK-NEXT:    mov z23.h, z2.h[5]
+; CHECK-NEXT:    mov z23.h, z1.h[5]
 ; CHECK-NEXT:    mov z20.h, z6.h[7]
-; CHECK-NEXT:    mov z24.h, z1.h[5]
-; CHECK-NEXT:    mov z25.h, z2.h[4]
+; CHECK-NEXT:    mov z24.h, z0.h[5]
+; CHECK-NEXT:    mov z25.h, z1.h[4]
 ; CHECK-NEXT:    mov z19.h, z7.h[7]
-; CHECK-NEXT:    mov z26.h, z1.h[4]
+; CHECK-NEXT:    mov z26.h, z0.h[4]
 ; CHECK-NEXT:    mov z27.h, z6.h[6]
 ; CHECK-NEXT:    mov z28.h, z7.h[5]
 ; CHECK-NEXT:    mov z29.h, z6.h[5]
@@ -212,22 +212,22 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    zip1 z19.s, z28.s, z27.s
 ; CHECK-NEXT:    zip1 z18.s, z22.s, z18.s
 ; CHECK-NEXT:    zip1 z20.s, z24.s, z23.s
-; CHECK-NEXT:    zip1 z0.h, z0.h, z3.h
+; CHECK-NEXT:    zip1 z2.h, z3.h, z2.h
 ; CHECK-NEXT:    zip1 z3.s, z26.s, z25.s
 ; CHECK-NEXT:    zip1 z22.s, z30.s, z29.s
 ; CHECK-NEXT:    zip1 z6.h, z6.h, z7.h
 ; CHECK-NEXT:    zip1 z7.d, z16.d, z21.d
 ; CHECK-NEXT:    zip1 z16.d, z19.d, z17.d
-; CHECK-NEXT:    zip1 z1.h, z1.h, z2.h
-; CHECK-NEXT:    zip1 z2.h, z4.h, z5.h
+; CHECK-NEXT:    zip1 z0.h, z0.h, z1.h
+; CHECK-NEXT:    zip1 z1.h, z4.h, z5.h
 ; CHECK-NEXT:    zip1 z4.d, z20.d, z18.d
 ; CHECK-NEXT:    zip1 z3.d, z22.d, z3.d
-; CHECK-NEXT:    add z0.h, z0.h, z6.h
+; CHECK-NEXT:    add z2.h, z2.h, z6.h
 ; CHECK-NEXT:    add z5.h, z7.h, z16.h
-; CHECK-NEXT:    add z1.h, z1.h, z2.h
-; CHECK-NEXT:    add z2.h, z4.h, z3.h
-; CHECK-NEXT:    stp q0, q5, [x0, #32]
-; CHECK-NEXT:    stp q1, q2, [x0]
+; CHECK-NEXT:    add z0.h, z0.h, z1.h
+; CHECK-NEXT:    add z1.h, z4.h, z3.h
+; CHECK-NEXT:    stp q2, q5, [x0, #32]
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 ;
@@ -659,10 +659,10 @@ define void @zip1_v8i32_undef(ptr %a) {
 define void @trn_v32i8(ptr %a, ptr %b) {
 ; CHECK-LABEL: trn_v32i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    trn1 z4.b, z0.b, z1.b
-; CHECK-NEXT:    trn2 z0.b, z0.b, z1.b
+; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    trn1 z4.b, z1.b, z0.b
+; CHECK-NEXT:    trn2 z0.b, z1.b, z0.b
 ; CHECK-NEXT:    trn1 z1.b, z2.b, z3.b
 ; CHECK-NEXT:    trn2 z2.b, z2.b, z3.b
 ; CHECK-NEXT:    add z0.b, z4.b, z0.b
@@ -862,10 +862,10 @@ define void @trn_v8i16(ptr %a, ptr %b) {
 define void @trn_v16i16(ptr %a, ptr %b) {
 ; CHECK-LABEL: trn_v16i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    trn1 z4.h, z0.h, z1.h
-; CHECK-NEXT:    trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    trn1 z4.h, z1.h, z0.h
+; CHECK-NEXT:    trn2 z0.h, z1.h, z0.h
 ; CHECK-NEXT:    trn1 z1.h, z2.h, z3.h
 ; CHECK-NEXT:    trn2 z2.h, z2.h, z3.h
 ; CHECK-NEXT:    add z0.h, z4.h, z0.h
@@ -961,10 +961,10 @@ define void @trn_v16i16(ptr %a, ptr %b) {
 define void @trn_v8i32(ptr %a, ptr %b) {
 ; CHECK-LABEL: trn_v8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    zip1 z4.s, z0.s, z1.s
-; CHECK-NEXT:    trn2 z0.s, z0.s, z1.s
+; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    zip1 z4.s, z1.s, z0.s
+; CHECK-NEXT:    trn2 z0.s, z1.s, z0.s
 ; CHECK-NEXT:    trn1 z1.s, z2.s, z3.s
 ; CHECK-NEXT:    trn2 z2.s, z2.s, z3.s
 ; CHECK-NEXT:    add z0.s, z4.s, z0.s
@@ -1006,11 +1006,11 @@ define void @trn_v8i32(ptr %a, ptr %b) {
 define void @trn_v4f64(ptr %a, ptr %b) {
 ; CHECK-LABEL: trn_v4f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q2, [x0]
+; CHECK-NEXT:    ldp q0, q3, [x1]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ldp q1, q3, [x1]
-; CHECK-NEXT:    zip1 z4.d, z0.d, z1.d
-; CHECK-NEXT:    trn2 z0.d, z0.d, z1.d
+; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    zip1 z4.d, z1.d, z0.d
+; CHECK-NEXT:    trn2 z0.d, z1.d, z0.d
 ; CHECK-NEXT:    zip1 z1.d, z2.d, z3.d
 ; CHECK-NEXT:    trn2 z2.d, z2.d, z3.d
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, z4.d
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
index e07036f2a1ac..90466e3cebd5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
@@ -125,14 +125,14 @@ define i1 @ptest_or_v16i1(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldp q2, q3, [x1, #32]
 ; CHECK-NEXT:    ldp q4, q5, [x0]
 ; CHECK-NEXT:    fcmne p1.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    ldp q1, q6, [x1]
+; CHECK-NEXT:    ldp q6, q1, [x1]
 ; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    fcmne p2.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fcmne p5.s, p0/z, z2.s, #0.0
 ; CHECK-NEXT:    fcmne p4.s, p0/z, z5.s, #0.0
 ; CHECK-NEXT:    fcmne p7.s, p0/z, z4.s, #0.0
-; CHECK-NEXT:    fcmne p6.s, p0/z, z6.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    fcmne p6.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z6.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
@@ -334,14 +334,14 @@ define i1 @ptest_and_v16i1(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldp q2, q3, [x1, #32]
 ; CHECK-NEXT:    ldp q4, q5, [x0]
 ; CHECK-NEXT:    fcmne p1.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    ldp q1, q6, [x1]
+; CHECK-NEXT:    ldp q6, q1, [x1]
 ; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    fcmne p2.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fcmne p5.s, p0/z, z2.s, #0.0
 ; CHECK-NEXT:    fcmne p4.s, p0/z, z5.s, #0.0
 ; CHECK-NEXT:    fcmne p7.s, p0/z, z4.s, #0.0
-; CHECK-NEXT:    fcmne p6.s, p0/z, z6.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    fcmne p6.s, p0/z, z1.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z6.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
index adf1b48b6998..139ecafaff0e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
@@ -1,106 +1,166 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+sve2 | FileCheck %s
+; RUN: llc < %s -mattr=+sve | FileCheck %s -check-prefixes=CHECK,SVE
+; RUN: llc < %s -mattr=+sve,+sme2 | FileCheck %s -check-prefixes=CHECK,SVE
+; RUN: llc < %s -mattr=+sme2 -force-streaming | FileCheck %s -check-prefixes=CHECK,SME2,SME2-ALL
+; RUN: llc < %s -mattr=+sme2 -force-streaming -aarch64-sve-vector-bits-min=256 | FileCheck %s -check-prefixes=CHECK,SME2,SME2-256
+
+target triple = "aarch64-unknown-linux-gnu"
 
 define {<vscale x 2 x half>, <vscale x 2 x half>} @vector_deinterleave_nxv2f16_nxv4f16(<vscale x 4 x half> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2f16_nxv4f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
-; CHECK-NEXT:    uzp2 z2.s, z0.s, z0.s
-; CHECK-NEXT:    uunpklo z0.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z2.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv2f16_nxv4f16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z1.s, z0.s, z0.s
+; SVE-NEXT:    uzp2 z2.s, z0.s, z0.s
+; SVE-NEXT:    uunpklo z0.d, z1.s
+; SVE-NEXT:    uunpklo z1.d, z2.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv2f16_nxv4f16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.d, z0.s
+; SME2-NEXT:    uunpklo z0.d, z0.s
+; SME2-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 2 x half>, <vscale x 2 x half>} @llvm.vector.deinterleave2.nxv4f16(<vscale x 4 x half> %vec)
   ret {<vscale x 2 x half>, <vscale x 2 x half>} %retval
 }
 
 define {<vscale x 4 x half>, <vscale x 4 x half>} @vector_deinterleave_nxv4f16_nxv8f16(<vscale x 8 x half> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv4f16_nxv8f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z1.h, z0.h, z0.h
-; CHECK-NEXT:    uzp2 z2.h, z0.h, z0.h
-; CHECK-NEXT:    uunpklo z0.s, z1.h
-; CHECK-NEXT:    uunpklo z1.s, z2.h
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv4f16_nxv8f16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z1.h, z0.h, z0.h
+; SVE-NEXT:    uzp2 z2.h, z0.h, z0.h
+; SVE-NEXT:    uunpklo z0.s, z1.h
+; SVE-NEXT:    uunpklo z1.s, z2.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv4f16_nxv8f16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.s, z0.h
+; SME2-NEXT:    uunpklo z0.s, z0.h
+; SME2-NEXT:    uzp { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call {<vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave2.nxv8f16(<vscale x 8 x half> %vec)
   ret {<vscale x 4 x half>, <vscale x 4 x half>} %retval
 }
 
 define {<vscale x 8 x half>, <vscale x 8 x half>} @vector_deinterleave_nxv8f16_nxv16f16(<vscale x 16 x half> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv8f16_nxv16f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.h, z0.h, z1.h
-; CHECK-NEXT:    uzp2 z1.h, z0.h, z1.h
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv8f16_nxv16f16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.h, z0.h, z1.h
+; SVE-NEXT:    uzp2 z1.h, z0.h, z1.h
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv8f16_nxv16f16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.vector.deinterleave2.nxv16f16(<vscale x 16 x half> %vec)
   ret {<vscale x 8 x half>, <vscale x 8 x half>} %retval
 }
 
 define {<vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32_nxv4f32(<vscale x 4 x float> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2f32_nxv4f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
-; CHECK-NEXT:    uzp2 z2.s, z0.s, z0.s
-; CHECK-NEXT:    uunpklo z0.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z2.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv2f32_nxv4f32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z1.s, z0.s, z0.s
+; SVE-NEXT:    uzp2 z2.s, z0.s, z0.s
+; SVE-NEXT:    uunpklo z0.d, z1.s
+; SVE-NEXT:    uunpklo z1.d, z2.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv2f32_nxv4f32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.d, z0.s
+; SME2-NEXT:    uunpklo z0.d, z0.s
+; SME2-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave2.nxv4f32(<vscale x 4 x float> %vec)
   ret {<vscale x 2 x float>, <vscale x 2 x float>} %retval
 }
 
 define {<vscale x 4 x float>, <vscale x 4 x float>} @vector_deinterleave_nxv4f32_nxv8f32(<vscale x 8 x float> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv4f32_nxv8f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.s, z0.s, z1.s
-; CHECK-NEXT:    uzp2 z1.s, z0.s, z1.s
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv4f32_nxv8f32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.s, z0.s, z1.s
+; SVE-NEXT:    uzp2 z1.s, z0.s, z1.s
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv4f32_nxv8f32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %vec)
   ret {<vscale x 4 x float>, <vscale x 4 x float>} %retval
 }
 
 define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f64_nxv4f64(<vscale x 4 x double> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2f64_nxv4f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.d, z0.d, z1.d
-; CHECK-NEXT:    uzp2 z1.d, z0.d, z1.d
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv2f64_nxv4f64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.d, z0.d, z1.d
+; SVE-NEXT:    uzp2 z1.d, z0.d, z1.d
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv2f64_nxv4f64:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %vec)
   ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval
 }
 
 define {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @vector_deinterleave_nxv2bf16_nxv4bf16(<vscale x 4 x bfloat> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
-; CHECK-NEXT:    uzp2 z2.s, z0.s, z0.s
-; CHECK-NEXT:    uunpklo z0.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z2.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z1.s, z0.s, z0.s
+; SVE-NEXT:    uzp2 z2.s, z0.s, z0.s
+; SVE-NEXT:    uunpklo z0.d, z1.s
+; SVE-NEXT:    uunpklo z1.d, z2.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.d, z0.s
+; SME2-NEXT:    uunpklo z0.d, z0.s
+; SME2-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @llvm.vector.deinterleave2.nxv4bf16(<vscale x 4 x bfloat> %vec)
   ret {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} %retval
 }
 
 define {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @vector_deinterleave_nxv4bf16_nxv8bf16(<vscale x 8 x bfloat> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z1.h, z0.h, z0.h
-; CHECK-NEXT:    uzp2 z2.h, z0.h, z0.h
-; CHECK-NEXT:    uunpklo z0.s, z1.h
-; CHECK-NEXT:    uunpklo z1.s, z2.h
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z1.h, z0.h, z0.h
+; SVE-NEXT:    uzp2 z2.h, z0.h, z0.h
+; SVE-NEXT:    uunpklo z0.s, z1.h
+; SVE-NEXT:    uunpklo z1.s, z2.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.s, z0.h
+; SME2-NEXT:    uunpklo z0.s, z0.h
+; SME2-NEXT:    uzp { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @llvm.vector.deinterleave2.nxv8bf16(<vscale x 8 x bfloat> %vec)
   ret {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} %retval
 }
 
 define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @vector_deinterleave_nxv8bf16_nxv16bf16(<vscale x 16 x bfloat> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.h, z0.h, z1.h
-; CHECK-NEXT:    uzp2 z1.h, z0.h, z1.h
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.h, z0.h, z1.h
+; SVE-NEXT:    uzp2 z1.h, z0.h, z1.h
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.vector.deinterleave2.nxv16bf16(<vscale x 16 x bfloat> %vec)
   ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %retval
 }
@@ -108,49 +168,263 @@ define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @vector_deinterleave_nxv8b
 ; Integers
 
 define {<vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv32i8(<vscale x 32 x i8> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv16i8_nxv32i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.b, z0.b, z1.b
-; CHECK-NEXT:    uzp2 z1.b, z0.b, z1.b
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv16i8_nxv32i8:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.b, z0.b, z1.b
+; SVE-NEXT:    uzp2 z1.b, z0.b, z1.b
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv16i8_nxv32i8:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.b, z1.b }, z0.b, z1.b
+; SME2-NEXT:    ret
   %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %vec)
   ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %retval
 }
 
 define {<vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv16i16(<vscale x 16 x i16> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv8i16_nxv16i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.h, z0.h, z1.h
-; CHECK-NEXT:    uzp2 z1.h, z0.h, z1.h
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv8i16_nxv16i16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.h, z0.h, z1.h
+; SVE-NEXT:    uzp2 z1.h, z0.h, z1.h
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv8i16_nxv16i16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> %vec)
   ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %retval
 }
 
 define {<vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxvv8i32(<vscale x 8 x i32> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv4i32_nxvv8i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.s, z0.s, z1.s
-; CHECK-NEXT:    uzp2 z1.s, z0.s, z1.s
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv4i32_nxvv8i32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.s, z0.s, z1.s
+; SVE-NEXT:    uzp2 z1.s, z0.s, z1.s
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv4i32_nxvv8i32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %vec)
   ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %retval
 }
 
 define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv4i64(<vscale x 4 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z2.d, z0.d, z1.d
-; CHECK-NEXT:    uzp2 z1.d, z0.d, z1.d
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z2.d, z0.d, z1.d
+; SVE-NEXT:    uzp2 z1.d, z0.d, z1.d
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> %vec)
   ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %retval
 }
 
+define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv64i8(<vscale x 64 x i8> %vec) {
+; SVE-LABEL: vector_deinterleave_nxv16i8_nxv64i8:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z4.b, z2.b, z3.b
+; SVE-NEXT:    uzp1 z5.b, z0.b, z1.b
+; SVE-NEXT:    uzp2 z3.b, z2.b, z3.b
+; SVE-NEXT:    uzp2 z6.b, z0.b, z1.b
+; SVE-NEXT:    uzp1 z0.b, z5.b, z4.b
+; SVE-NEXT:    uzp2 z2.b, z5.b, z4.b
+; SVE-NEXT:    uzp1 z1.b, z6.b, z3.b
+; SVE-NEXT:    uzp2 z3.b, z6.b, z3.b
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv16i8_nxv64i8:
+; SME2:       // %bb.0:
+; SME2-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    uzp { z0.b - z3.b }, { z0.b - z3.b }
+; SME2-NEXT:    ret
+  %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> %vec)
+  ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %retval
+}
+
+define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv32i16(<vscale x 32 x i16> %vec) {
+; SVE-LABEL: vector_deinterleave_nxv8i16_nxv32i16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z4.h, z2.h, z3.h
+; SVE-NEXT:    uzp1 z5.h, z0.h, z1.h
+; SVE-NEXT:    uzp2 z3.h, z2.h, z3.h
+; SVE-NEXT:    uzp2 z6.h, z0.h, z1.h
+; SVE-NEXT:    uzp1 z0.h, z5.h, z4.h
+; SVE-NEXT:    uzp2 z2.h, z5.h, z4.h
+; SVE-NEXT:    uzp1 z1.h, z6.h, z3.h
+; SVE-NEXT:    uzp2 z3.h, z6.h, z3.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv8i16_nxv32i16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    uzp { z0.h - z3.h }, { z0.h - z3.h }
+; SME2-NEXT:    ret
+  %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave4.nxv32i16(<vscale x 32 x i16> %vec)
+  ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %retval
+}
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv16i32(<vscale x 16 x i32> %vec) {
+; SVE-LABEL: vector_deinterleave_nxv4i32_nxv16i32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z4.s, z2.s, z3.s
+; SVE-NEXT:    uzp1 z5.s, z0.s, z1.s
+; SVE-NEXT:    uzp2 z3.s, z2.s, z3.s
+; SVE-NEXT:    uzp2 z6.s, z0.s, z1.s
+; SVE-NEXT:    uzp1 z0.s, z5.s, z4.s
+; SVE-NEXT:    uzp2 z2.s, z5.s, z4.s
+; SVE-NEXT:    uzp1 z1.s, z6.s, z3.s
+; SVE-NEXT:    uzp2 z3.s, z6.s, z3.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv4i32_nxv16i32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    uzp { z0.s - z3.s }, { z0.s - z3.s }
+; SME2-NEXT:    ret
+  %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %vec)
+  ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %retval
+}
+
+define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv8i64(<vscale x 8 x i64> %vec) {
+; SVE-LABEL: vector_deinterleave_nxv2i64_nxv8i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z4.d, z2.d, z3.d
+; SVE-NEXT:    uzp1 z5.d, z0.d, z1.d
+; SVE-NEXT:    uzp2 z3.d, z2.d, z3.d
+; SVE-NEXT:    uzp2 z6.d, z0.d, z1.d
+; SVE-NEXT:    uzp1 z0.d, z5.d, z4.d
+; SVE-NEXT:    uzp2 z2.d, z5.d, z4.d
+; SVE-NEXT:    uzp1 z1.d, z6.d, z3.d
+; SVE-NEXT:    uzp2 z3.d, z6.d, z3.d
+; SVE-NEXT:    ret
+;
+; SME2-ALL-LABEL: vector_deinterleave_nxv2i64_nxv8i64:
+; SME2-ALL:       // %bb.0:
+; SME2-ALL-NEXT:    uzp { z4.d, z5.d }, z2.d, z3.d
+; SME2-ALL-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-ALL-NEXT:    uzp { z2.d, z3.d }, z0.d, z4.d
+; SME2-ALL-NEXT:    uzp { z4.d, z5.d }, z1.d, z5.d
+; SME2-ALL-NEXT:    mov z0.d, z2.d
+; SME2-ALL-NEXT:    mov z1.d, z4.d
+; SME2-ALL-NEXT:    mov z2.d, z3.d
+; SME2-ALL-NEXT:    mov z3.d, z5.d
+; SME2-ALL-NEXT:    ret
+;
+; SME2-256-LABEL: vector_deinterleave_nxv2i64_nxv8i64:
+; SME2-256:       // %bb.0:
+; SME2-256-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    uzp { z0.d - z3.d }, { z0.d - z3.d }
+; SME2-256-NEXT:    ret
+  %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> %vec)
+  ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval
+}
+
+define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv16i64(<vscale x 16 x i64> %vec) {
+; SVE-LABEL: vector_deinterleave_nxv2i64_nxv16i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z24.d, z6.d, z7.d
+; SVE-NEXT:    uzp1 z25.d, z4.d, z5.d
+; SVE-NEXT:    uzp1 z26.d, z2.d, z3.d
+; SVE-NEXT:    uzp1 z27.d, z0.d, z1.d
+; SVE-NEXT:    uzp2 z6.d, z6.d, z7.d
+; SVE-NEXT:    uzp2 z4.d, z4.d, z5.d
+; SVE-NEXT:    uzp2 z2.d, z2.d, z3.d
+; SVE-NEXT:    uzp2 z0.d, z0.d, z1.d
+; SVE-NEXT:    uzp1 z5.d, z25.d, z24.d
+; SVE-NEXT:    uzp2 z24.d, z25.d, z24.d
+; SVE-NEXT:    uzp1 z7.d, z27.d, z26.d
+; SVE-NEXT:    uzp1 z28.d, z4.d, z6.d
+; SVE-NEXT:    uzp2 z25.d, z27.d, z26.d
+; SVE-NEXT:    uzp1 z29.d, z0.d, z2.d
+; SVE-NEXT:    uzp2 z26.d, z4.d, z6.d
+; SVE-NEXT:    uzp2 z27.d, z0.d, z2.d
+; SVE-NEXT:    uzp1 z0.d, z7.d, z5.d
+; SVE-NEXT:    uzp1 z2.d, z25.d, z24.d
+; SVE-NEXT:    uzp2 z4.d, z7.d, z5.d
+; SVE-NEXT:    uzp1 z1.d, z29.d, z28.d
+; SVE-NEXT:    uzp1 z3.d, z27.d, z26.d
+; SVE-NEXT:    uzp2 z5.d, z29.d, z28.d
+; SVE-NEXT:    uzp2 z6.d, z25.d, z24.d
+; SVE-NEXT:    uzp2 z7.d, z27.d, z26.d
+; SVE-NEXT:    ret
+;
+; SME2-ALL-LABEL: vector_deinterleave_nxv2i64_nxv16i64:
+; SME2-ALL:       // %bb.0:
+; SME2-ALL-NEXT:    uzp { z6.d, z7.d }, z6.d, z7.d
+; SME2-ALL-NEXT:    uzp { z24.d, z25.d }, z4.d, z5.d
+; SME2-ALL-NEXT:    uzp { z26.d, z27.d }, z24.d, z6.d
+; SME2-ALL-NEXT:    uzp { z2.d, z3.d }, z2.d, z3.d
+; SME2-ALL-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-ALL-NEXT:    uzp { z28.d, z29.d }, z0.d, z2.d
+; SME2-ALL-NEXT:    uzp { z4.d, z5.d }, z28.d, z26.d
+; SME2-ALL-NEXT:    uzp { z30.d, z31.d }, z25.d, z7.d
+; SME2-ALL-NEXT:    uzp { z0.d, z1.d }, z1.d, z3.d
+; SME2-ALL-NEXT:    uzp { z6.d, z7.d }, z0.d, z30.d
+; SME2-ALL-NEXT:    uzp { z24.d, z25.d }, z29.d, z27.d
+; SME2-ALL-NEXT:    uzp { z26.d, z27.d }, z1.d, z31.d
+; SME2-ALL-NEXT:    mov z0.d, z4.d
+; SME2-ALL-NEXT:    mov z1.d, z6.d
+; SME2-ALL-NEXT:    mov z2.d, z24.d
+; SME2-ALL-NEXT:    mov z3.d, z26.d
+; SME2-ALL-NEXT:    mov z4.d, z5.d
+; SME2-ALL-NEXT:    mov z5.d, z7.d
+; SME2-ALL-NEXT:    mov z6.d, z25.d
+; SME2-ALL-NEXT:    mov z7.d, z27.d
+; SME2-ALL-NEXT:    ret
+;
+; SME2-256-LABEL: vector_deinterleave_nxv2i64_nxv16i64:
+; SME2-256:       // %bb.0:
+; SME2-256-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; SME2-256-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; SME2-256-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; SME2-256-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; SME2-256-NEXT:    uzp { z28.d - z31.d }, { z4.d - z7.d }
+; SME2-256-NEXT:    uzp { z0.d - z3.d }, { z0.d - z3.d }
+; SME2-256-NEXT:    uzp { z4.d, z5.d }, z0.d, z28.d
+; SME2-256-NEXT:    uzp { z6.d, z7.d }, z1.d, z29.d
+; SME2-256-NEXT:    uzp { z24.d, z25.d }, z2.d, z30.d
+; SME2-256-NEXT:    uzp { z26.d, z27.d }, z3.d, z31.d
+; SME2-256-NEXT:    mov z0.d, z4.d
+; SME2-256-NEXT:    mov z1.d, z6.d
+; SME2-256-NEXT:    mov z2.d, z24.d
+; SME2-256-NEXT:    mov z3.d, z26.d
+; SME2-256-NEXT:    mov z4.d, z5.d
+; SME2-256-NEXT:    mov z5.d, z7.d
+; SME2-256-NEXT:    mov z6.d, z25.d
+; SME2-256-NEXT:    mov z7.d, z27.d
+; SME2-256-NEXT:    ret
+  %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave8.nxv16i64(<vscale x 16 x i64> %vec)
+  ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval
+}
+
 ; Predicated
 define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv32i1(<vscale x 32 x i1> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv32i1:
@@ -203,39 +477,65 @@ define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1
 ; Split illegal types
 
 define {<vscale x 4 x i64>, <vscale x 4 x i64>} @vector_deinterleave_nxv4i64_nxv8i64(<vscale x 8 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv4i64_nxv8i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z4.d, z2.d, z3.d
-; CHECK-NEXT:    uzp1 z5.d, z0.d, z1.d
-; CHECK-NEXT:    uzp2 z6.d, z0.d, z1.d
-; CHECK-NEXT:    uzp2 z3.d, z2.d, z3.d
-; CHECK-NEXT:    mov z0.d, z5.d
-; CHECK-NEXT:    mov z1.d, z4.d
-; CHECK-NEXT:    mov z2.d, z6.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv4i64_nxv8i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z4.d, z2.d, z3.d
+; SVE-NEXT:    uzp1 z5.d, z0.d, z1.d
+; SVE-NEXT:    uzp2 z6.d, z0.d, z1.d
+; SVE-NEXT:    uzp2 z3.d, z2.d, z3.d
+; SVE-NEXT:    mov z0.d, z5.d
+; SVE-NEXT:    mov z1.d, z4.d
+; SVE-NEXT:    mov z2.d, z6.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv4i64_nxv8i64:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z4.d, z5.d }, z0.d, z1.d
+; SME2-NEXT:    uzp { z6.d, z7.d }, z2.d, z3.d
+; SME2-NEXT:    mov z0.d, z4.d
+; SME2-NEXT:    mov z1.d, z6.d
+; SME2-NEXT:    mov z2.d, z5.d
+; SME2-NEXT:    mov z3.d, z7.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 4 x i64>, <vscale x 4 x i64>} @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> %vec)
   ret {<vscale x 4 x i64>, <vscale x 4 x i64>} %retval
 }
 
 define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv16i64(<vscale x 16 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv8i64_nxv16i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uzp1 z24.d, z2.d, z3.d
-; CHECK-NEXT:    uzp1 z25.d, z0.d, z1.d
-; CHECK-NEXT:    uzp1 z26.d, z4.d, z5.d
-; CHECK-NEXT:    uzp1 z27.d, z6.d, z7.d
-; CHECK-NEXT:    uzp2 z28.d, z0.d, z1.d
-; CHECK-NEXT:    uzp2 z29.d, z2.d, z3.d
-; CHECK-NEXT:    uzp2 z30.d, z4.d, z5.d
-; CHECK-NEXT:    uzp2 z7.d, z6.d, z7.d
-; CHECK-NEXT:    mov z0.d, z25.d
-; CHECK-NEXT:    mov z1.d, z24.d
-; CHECK-NEXT:    mov z2.d, z26.d
-; CHECK-NEXT:    mov z3.d, z27.d
-; CHECK-NEXT:    mov z4.d, z28.d
-; CHECK-NEXT:    mov z5.d, z29.d
-; CHECK-NEXT:    mov z6.d, z30.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv8i64_nxv16i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uzp1 z24.d, z2.d, z3.d
+; SVE-NEXT:    uzp1 z25.d, z0.d, z1.d
+; SVE-NEXT:    uzp1 z26.d, z4.d, z5.d
+; SVE-NEXT:    uzp1 z27.d, z6.d, z7.d
+; SVE-NEXT:    uzp2 z28.d, z0.d, z1.d
+; SVE-NEXT:    uzp2 z29.d, z2.d, z3.d
+; SVE-NEXT:    uzp2 z30.d, z4.d, z5.d
+; SVE-NEXT:    uzp2 z7.d, z6.d, z7.d
+; SVE-NEXT:    mov z0.d, z25.d
+; SVE-NEXT:    mov z1.d, z24.d
+; SVE-NEXT:    mov z2.d, z26.d
+; SVE-NEXT:    mov z3.d, z27.d
+; SVE-NEXT:    mov z4.d, z28.d
+; SVE-NEXT:    mov z5.d, z29.d
+; SVE-NEXT:    mov z6.d, z30.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv8i64_nxv16i64:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uzp { z24.d, z25.d }, z0.d, z1.d
+; SME2-NEXT:    uzp { z26.d, z27.d }, z2.d, z3.d
+; SME2-NEXT:    uzp { z28.d, z29.d }, z4.d, z5.d
+; SME2-NEXT:    uzp { z30.d, z31.d }, z6.d, z7.d
+; SME2-NEXT:    mov z0.d, z24.d
+; SME2-NEXT:    mov z1.d, z26.d
+; SME2-NEXT:    mov z2.d, z28.d
+; SME2-NEXT:    mov z3.d, z30.d
+; SME2-NEXT:    mov z4.d, z25.d
+; SME2-NEXT:    mov z5.d, z27.d
+; SME2-NEXT:    mov z6.d, z29.d
+; SME2-NEXT:    mov z7.d, z31.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 8 x i64>, <vscale x 8 x i64>} @llvm.vector.deinterleave2.nxv16i64(<vscale x 16 x i64> %vec)
   ret {<vscale x 8 x i64>, <vscale x 8 x i64>} %retval
 }
@@ -244,42 +544,62 @@ define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv
 ; Promote illegal type size
 
 define {<vscale x 8 x i8>, <vscale x 8 x i8>} @vector_deinterleave_nxv8i8_nxv16i8(<vscale x 16 x i8> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv8i8_nxv16i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpkhi z1.h, z0.b
-; CHECK-NEXT:    uunpklo z2.h, z0.b
-; CHECK-NEXT:    uzp1 z0.h, z2.h, z1.h
-; CHECK-NEXT:    uzp2 z1.h, z2.h, z1.h
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv8i8_nxv16i8:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uunpkhi z1.h, z0.b
+; SVE-NEXT:    uunpklo z2.h, z0.b
+; SVE-NEXT:    uzp1 z0.h, z2.h, z1.h
+; SVE-NEXT:    uzp2 z1.h, z2.h, z1.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv8i8_nxv16i8:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.h, z0.b
+; SME2-NEXT:    uunpklo z0.h, z0.b
+; SME2-NEXT:    uzp { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call {<vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> %vec)
   ret {<vscale x 8 x i8>, <vscale x 8 x i8>} %retval
 }
 
 define {<vscale x 4 x i16>, <vscale x 4 x i16>} @vector_deinterleave_nxv4i16_nxv8i16(<vscale x 8 x i16> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv4i16_nxv8i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpkhi z1.s, z0.h
-; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    uzp1 z0.s, z2.s, z1.s
-; CHECK-NEXT:    uzp2 z1.s, z2.s, z1.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv4i16_nxv8i16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uunpkhi z1.s, z0.h
+; SVE-NEXT:    uunpklo z2.s, z0.h
+; SVE-NEXT:    uzp1 z0.s, z2.s, z1.s
+; SVE-NEXT:    uzp2 z1.s, z2.s, z1.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv4i16_nxv8i16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.s, z0.h
+; SME2-NEXT:    uunpklo z0.s, z0.h
+; SME2-NEXT:    uzp { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call {<vscale x 4 x i16>, <vscale x 4 x i16>} @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %vec)
   ret {<vscale x 4 x i16>, <vscale x 4 x i16>} %retval
 }
 
 define {<vscale x 2 x i32>, <vscale x 2 x i32>} @vector_deinterleave_nxv2i32_nxv4i32(<vscale x 4 x i32> %vec) {
-; CHECK-LABEL: vector_deinterleave_nxv2i32_nxv4i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpkhi z1.d, z0.s
-; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    uzp1 z0.d, z2.d, z1.d
-; CHECK-NEXT:    uzp2 z1.d, z2.d, z1.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: vector_deinterleave_nxv2i32_nxv4i32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    uunpkhi z1.d, z0.s
+; SVE-NEXT:    uunpklo z2.d, z0.s
+; SVE-NEXT:    uzp1 z0.d, z2.d, z1.d
+; SVE-NEXT:    uzp2 z1.d, z2.d, z1.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: vector_deinterleave_nxv2i32_nxv4i32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    uunpkhi z1.d, z0.s
+; SME2-NEXT:    uunpklo z0.d, z0.s
+; SME2-NEXT:    uzp { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %vec)
   ret {<vscale x 2 x i32>, <vscale x 2 x i32>} %retval
 }
 
-
 ; Floating declarations
 declare {<vscale x 2 x half>,<vscale x 2 x half>} @llvm.vector.deinterleave2.nxv4f16(<vscale x 4 x half>)
 declare {<vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave2.nxv8f16(<vscale x 8 x half>)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index 288034422d9c..52cb2d9ebe34 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -1,101 +1,156 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+; RUN: llc < %s -mattr=+sve | FileCheck %s -check-prefixes=CHECK,SVE
+; RUN: llc < %s -mattr=+sve,+sme2 | FileCheck %s -check-prefixes=CHECK,SVE
+; RUN: llc < %s -mattr=+sme2 -force-streaming | FileCheck %s -check-prefixes=CHECK,SME2,SME-ALL
+; RUN: llc < %s -mattr=+sme2 -force-streaming -aarch64-sve-vector-bits-min=256 | FileCheck %s -check-prefixes=CHECK,SME2,SME2-256
+
+target triple = "aarch64-unknown-linux-gnu"
 
 define <vscale x 4 x half> @interleave2_nxv4f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1) {
-; CHECK-LABEL: interleave2_nxv4f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.d, z0.d, z1.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z1.d
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv4f16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.d, z0.d, z1.d
+; SVE-NEXT:    zip1 z0.d, z0.d, z1.d
+; SVE-NEXT:    uzp1 z0.s, z0.s, z2.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv4f16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    uzp1 z0.s, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call <vscale x 4 x half> @llvm.vector.interleave2.nxv4f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1)
   ret <vscale x 4 x half> %retval
 }
 
 define <vscale x 8 x half> @interleave2_nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1) {
-; CHECK-LABEL: interleave2_nxv8f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.s, z0.s, z1.s
-; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z2.h
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv8f16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.s, z0.s, z1.s
+; SVE-NEXT:    zip1 z0.s, z0.s, z1.s
+; SVE-NEXT:    uzp1 z0.h, z0.h, z2.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv8f16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    uzp1 z0.h, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call <vscale x 8 x half> @llvm.vector.interleave2.nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1)
   ret <vscale x 8 x half> %retval
 }
 
 define <vscale x 16 x half> @interleave2_nxv16f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1) {
-; CHECK-LABEL: interleave2_nxv16f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.h, z0.h, z1.h
-; CHECK-NEXT:    zip2 z1.h, z0.h, z1.h
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv16f16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.h, z0.h, z1.h
+; SVE-NEXT:    zip2 z1.h, z0.h, z1.h
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv16f16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call <vscale x 16 x half> @llvm.vector.interleave2.nxv16f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1)
   ret <vscale x 16 x half> %retval
 }
 
 define <vscale x 4 x float> @interleave2_nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1) {
-; CHECK-LABEL: interleave2_nxv4f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.d, z0.d, z1.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z1.d
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv4f32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.d, z0.d, z1.d
+; SVE-NEXT:    zip1 z0.d, z0.d, z1.d
+; SVE-NEXT:    uzp1 z0.s, z0.s, z2.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv4f32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    uzp1 z0.s, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1)
   ret <vscale x 4 x float> %retval
 }
 
 define <vscale x 8 x float> @interleave2_nxv8f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1) {
-; CHECK-LABEL: interleave2_nxv8f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.s, z0.s, z1.s
-; CHECK-NEXT:    zip2 z1.s, z0.s, z1.s
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv8f32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.s, z0.s, z1.s
+; SVE-NEXT:    zip2 z1.s, z0.s, z1.s
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv8f32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1)
   ret <vscale x 8 x float> %retval
 }
 
 define <vscale x 4 x double> @interleave2_nxv4f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1) {
-; CHECK-LABEL: interleave2_nxv4f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.d, z0.d, z1.d
-; CHECK-NEXT:    zip2 z1.d, z0.d, z1.d
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv4f64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.d, z0.d, z1.d
+; SVE-NEXT:    zip2 z1.d, z0.d, z1.d
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv4f64:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call <vscale x 4 x double>@llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1)
   ret <vscale x 4 x double> %retval
 }
 
 define <vscale x 4 x bfloat> @interleave2_nxv4bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1) {
-; CHECK-LABEL: interleave2_nxv4bf16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.d, z0.d, z1.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z1.d
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv4bf16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.d, z0.d, z1.d
+; SVE-NEXT:    zip1 z0.d, z0.d, z1.d
+; SVE-NEXT:    uzp1 z0.s, z0.s, z2.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv4bf16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    uzp1 z0.s, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call <vscale x 4 x bfloat> @llvm.vector.interleave2.nxv4bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1)
   ret <vscale x 4 x bfloat> %retval
 }
 
 define <vscale x 8 x bfloat> @interleave2_nxv8bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1) {
-; CHECK-LABEL: interleave2_nxv8bf16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.s, z0.s, z1.s
-; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z2.h
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv8bf16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.s, z0.s, z1.s
+; SVE-NEXT:    zip1 z0.s, z0.s, z1.s
+; SVE-NEXT:    uzp1 z0.h, z0.h, z2.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv8bf16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    uzp1 z0.h, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call <vscale x 8 x bfloat> @llvm.vector.interleave2.nxv8bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1)
   ret <vscale x 8 x bfloat> %retval
 }
 
 define <vscale x 16 x bfloat> @interleave2_nxv16bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1) {
-; CHECK-LABEL: interleave2_nxv16bf16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.h, z0.h, z1.h
-; CHECK-NEXT:    zip2 z1.h, z0.h, z1.h
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv16bf16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.h, z0.h, z1.h
+; SVE-NEXT:    zip2 z1.h, z0.h, z1.h
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv16bf16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call <vscale x 16 x bfloat> @llvm.vector.interleave2.nxv16bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1)
   ret <vscale x 16 x bfloat> %retval
 }
@@ -103,49 +158,241 @@ define <vscale x 16 x bfloat> @interleave2_nxv16bf16(<vscale x 8 x bfloat> %vec0
 ; Integers
 
 define <vscale x 32 x i8> @interleave2_nxv32i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1) {
-; CHECK-LABEL: interleave2_nxv32i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.b, z0.b, z1.b
-; CHECK-NEXT:    zip2 z1.b, z0.b, z1.b
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv32i8:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.b, z0.b, z1.b
+; SVE-NEXT:    zip2 z1.b, z0.b, z1.b
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv32i8:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.b, z1.b }, z0.b, z1.b
+; SME2-NEXT:    ret
   %retval = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1)
   ret <vscale x 32 x i8> %retval
 }
 
 define <vscale x 16 x i16> @interleave2_nxv16i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1) {
-; CHECK-LABEL: interleave2_nxv16i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.h, z0.h, z1.h
-; CHECK-NEXT:    zip2 z1.h, z0.h, z1.h
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv16i16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.h, z0.h, z1.h
+; SVE-NEXT:    zip2 z1.h, z0.h, z1.h
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv16i16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1)
   ret <vscale x 16 x i16> %retval
 }
 
 define <vscale x 8 x i32> @interleave2_nxv8i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1) {
-; CHECK-LABEL: interleave2_nxv8i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.s, z0.s, z1.s
-; CHECK-NEXT:    zip2 z1.s, z0.s, z1.s
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv8i32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.s, z0.s, z1.s
+; SVE-NEXT:    zip2 z1.s, z0.s, z1.s
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv8i32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1)
   ret <vscale x 8 x i32> %retval
 }
 
 define <vscale x 4 x i64> @interleave2_nxv4i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1) {
-; CHECK-LABEL: interleave2_nxv4i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z2.d, z0.d, z1.d
-; CHECK-NEXT:    zip2 z1.d, z0.d, z1.d
-; CHECK-NEXT:    mov z0.d, z2.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv4i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z2.d, z0.d, z1.d
+; SVE-NEXT:    zip2 z1.d, z0.d, z1.d
+; SVE-NEXT:    mov z0.d, z2.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv4i64:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    ret
   %retval = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1)
   ret <vscale x 4 x i64> %retval
 }
 
+define <vscale x 64 x i8> @interleave4_nxv16i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2, <vscale x 16 x i8> %vec3) {
+; SVE-LABEL: interleave4_nxv16i8:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z4.b, z1.b, z3.b
+; SVE-NEXT:    zip1 z5.b, z0.b, z2.b
+; SVE-NEXT:    zip2 z3.b, z1.b, z3.b
+; SVE-NEXT:    zip2 z6.b, z0.b, z2.b
+; SVE-NEXT:    zip1 z0.b, z5.b, z4.b
+; SVE-NEXT:    zip2 z1.b, z5.b, z4.b
+; SVE-NEXT:    zip1 z2.b, z6.b, z3.b
+; SVE-NEXT:    zip2 z3.b, z6.b, z3.b
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave4_nxv16i8:
+; SME2:       // %bb.0:
+; SME2-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    zip { z0.b - z3.b }, { z0.b - z3.b }
+; SME2-NEXT:    ret
+  %retval = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2, <vscale x 16 x i8> %vec3)
+  ret <vscale x 64 x i8> %retval
+}
+
+define <vscale x 32 x i16> @interleave4_nxv8i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3) {
+; SVE-LABEL: interleave4_nxv8i16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z4.h, z1.h, z3.h
+; SVE-NEXT:    zip1 z5.h, z0.h, z2.h
+; SVE-NEXT:    zip2 z3.h, z1.h, z3.h
+; SVE-NEXT:    zip2 z6.h, z0.h, z2.h
+; SVE-NEXT:    zip1 z0.h, z5.h, z4.h
+; SVE-NEXT:    zip2 z1.h, z5.h, z4.h
+; SVE-NEXT:    zip1 z2.h, z6.h, z3.h
+; SVE-NEXT:    zip2 z3.h, z6.h, z3.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave4_nxv8i16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    zip { z0.h - z3.h }, { z0.h - z3.h }
+; SME2-NEXT:    ret
+  %retval = call <vscale x 32 x i16> @llvm.vector.interleave4.nxv8i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3)
+  ret <vscale x 32 x i16> %retval
+}
+
+define <vscale x 16 x i32> @interleave4_nxv4i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2, <vscale x 4 x i32> %vec3) {
+; SVE-LABEL: interleave4_nxv4i32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z4.s, z1.s, z3.s
+; SVE-NEXT:    zip1 z5.s, z0.s, z2.s
+; SVE-NEXT:    zip2 z3.s, z1.s, z3.s
+; SVE-NEXT:    zip2 z6.s, z0.s, z2.s
+; SVE-NEXT:    zip1 z0.s, z5.s, z4.s
+; SVE-NEXT:    zip2 z1.s, z5.s, z4.s
+; SVE-NEXT:    zip1 z2.s, z6.s, z3.s
+; SVE-NEXT:    zip2 z3.s, z6.s, z3.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave4_nxv4i32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-NEXT:    zip { z0.s - z3.s }, { z0.s - z3.s }
+; SME2-NEXT:    ret
+  %retval = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv4i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2, <vscale x 4 x i32> %vec3)
+  ret <vscale x 16 x i32> %retval
+}
+
+define <vscale x 8 x i64> @interleave4_nxv8i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3) {
+; SVE-LABEL: interleave4_nxv8i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z4.d, z1.d, z3.d
+; SVE-NEXT:    zip1 z5.d, z0.d, z2.d
+; SVE-NEXT:    zip2 z3.d, z1.d, z3.d
+; SVE-NEXT:    zip2 z6.d, z0.d, z2.d
+; SVE-NEXT:    zip1 z0.d, z5.d, z4.d
+; SVE-NEXT:    zip2 z1.d, z5.d, z4.d
+; SVE-NEXT:    zip1 z2.d, z6.d, z3.d
+; SVE-NEXT:    zip2 z3.d, z6.d, z3.d
+; SVE-NEXT:    ret
+;
+; SME-ALL-LABEL: interleave4_nxv8i64:
+; SME-ALL:       // %bb.0:
+; SME-ALL-NEXT:    zip { z4.d, z5.d }, z1.d, z3.d
+; SME-ALL-NEXT:    zip { z2.d, z3.d }, z0.d, z2.d
+; SME-ALL-NEXT:    zip { z0.d, z1.d }, z2.d, z4.d
+; SME-ALL-NEXT:    zip { z2.d, z3.d }, z3.d, z5.d
+; SME-ALL-NEXT:    ret
+;
+; SME2-256-LABEL: interleave4_nxv8i64:
+; SME2-256:       // %bb.0:
+; SME2-256-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; SME2-256-NEXT:    zip { z0.d - z3.d }, { z0.d - z3.d }
+; SME2-256-NEXT:    ret
+  %retval = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3)
+  ret <vscale x 8 x i64> %retval
+}
+
+define <vscale x 16 x i64> @interleave8_nxv16i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3, <vscale x 2 x i64> %vec4, <vscale x 2 x i64> %vec5, <vscale x 2 x i64> %vec6, <vscale x 2 x i64> %vec7) {
+; SVE-LABEL: interleave8_nxv16i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z24.d, z3.d, z7.d
+; SVE-NEXT:    zip1 z25.d, z1.d, z5.d
+; SVE-NEXT:    zip1 z26.d, z2.d, z6.d
+; SVE-NEXT:    zip1 z27.d, z0.d, z4.d
+; SVE-NEXT:    zip2 z3.d, z3.d, z7.d
+; SVE-NEXT:    zip2 z1.d, z1.d, z5.d
+; SVE-NEXT:    zip2 z2.d, z2.d, z6.d
+; SVE-NEXT:    zip2 z0.d, z0.d, z4.d
+; SVE-NEXT:    zip1 z4.d, z25.d, z24.d
+; SVE-NEXT:    zip2 z6.d, z25.d, z24.d
+; SVE-NEXT:    zip1 z5.d, z27.d, z26.d
+; SVE-NEXT:    zip2 z7.d, z27.d, z26.d
+; SVE-NEXT:    zip1 z24.d, z1.d, z3.d
+; SVE-NEXT:    zip1 z25.d, z0.d, z2.d
+; SVE-NEXT:    zip2 z26.d, z1.d, z3.d
+; SVE-NEXT:    zip2 z27.d, z0.d, z2.d
+; SVE-NEXT:    zip1 z0.d, z5.d, z4.d
+; SVE-NEXT:    zip2 z1.d, z5.d, z4.d
+; SVE-NEXT:    zip1 z2.d, z7.d, z6.d
+; SVE-NEXT:    zip2 z3.d, z7.d, z6.d
+; SVE-NEXT:    zip1 z4.d, z25.d, z24.d
+; SVE-NEXT:    zip2 z5.d, z25.d, z24.d
+; SVE-NEXT:    zip1 z6.d, z27.d, z26.d
+; SVE-NEXT:    zip2 z7.d, z27.d, z26.d
+; SVE-NEXT:    ret
+;
+; SME-ALL-LABEL: interleave8_nxv16i64:
+; SME-ALL:       // %bb.0:
+; SME-ALL-NEXT:    zip { z24.d, z25.d }, z3.d, z7.d
+; SME-ALL-NEXT:    zip { z26.d, z27.d }, z1.d, z5.d
+; SME-ALL-NEXT:    zip { z28.d, z29.d }, z26.d, z24.d
+; SME-ALL-NEXT:    zip { z6.d, z7.d }, z2.d, z6.d
+; SME-ALL-NEXT:    zip { z4.d, z5.d }, z0.d, z4.d
+; SME-ALL-NEXT:    zip { z2.d, z3.d }, z4.d, z6.d
+; SME-ALL-NEXT:    zip { z0.d, z1.d }, z2.d, z28.d
+; SME-ALL-NEXT:    zip { z2.d, z3.d }, z3.d, z29.d
+; SME-ALL-NEXT:    zip { z24.d, z25.d }, z27.d, z25.d
+; SME-ALL-NEXT:    zip { z6.d, z7.d }, z5.d, z7.d
+; SME-ALL-NEXT:    zip { z4.d, z5.d }, z6.d, z24.d
+; SME-ALL-NEXT:    zip { z6.d, z7.d }, z7.d, z25.d
+; SME-ALL-NEXT:    ret
+;
+; SME2-256-LABEL: interleave8_nxv16i64:
+; SME2-256:       // %bb.0:
+; SME2-256-NEXT:    zip { z26.d, z27.d }, z3.d, z7.d
+; SME2-256-NEXT:    zip { z6.d, z7.d }, z2.d, z6.d
+; SME2-256-NEXT:    zip { z24.d, z25.d }, z1.d, z5.d
+; SME2-256-NEXT:    zip { z0.d, z1.d }, z0.d, z4.d
+; SME2-256-NEXT:    mov z28.d, z0.d
+; SME2-256-NEXT:    mov z29.d, z24.d
+; SME2-256-NEXT:    mov z30.d, z6.d
+; SME2-256-NEXT:    mov z31.d, z26.d
+; SME2-256-NEXT:    mov z24.d, z1.d
+; SME2-256-NEXT:    mov z26.d, z7.d
+; SME2-256-NEXT:    zip { z0.d - z3.d }, { z28.d - z31.d }
+; SME2-256-NEXT:    zip { z4.d - z7.d }, { z24.d - z27.d }
+; SME2-256-NEXT:    ret
+  %retval = call <vscale x 16 x i64> @llvm.vector.interleave8.nxv16i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3, <vscale x 2 x i64> %vec4, <vscale x 2 x i64> %vec5, <vscale x 2 x i64> %vec6, <vscale x 2 x i64> %vec7)
+  ret <vscale x 16 x i64> %retval
+}
+
 ; Predicated
 
 define <vscale x 32 x i1> @interleave2_nxv32i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1) {
@@ -195,31 +442,47 @@ define <vscale x 4 x i1> @interleave2_nxv4i1(<vscale x 2 x i1> %vec0, <vscale x
 ; Split illegal type size
 
 define <vscale x 16 x i32> @interleave2_nxv16i32(<vscale x 8 x i32> %vec0, <vscale x 8 x i32> %vec1) {
-; CHECK-LABEL: interleave2_nxv16i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z4.s, z1.s, z3.s
-; CHECK-NEXT:    zip1 z5.s, z0.s, z2.s
-; CHECK-NEXT:    zip2 z2.s, z0.s, z2.s
-; CHECK-NEXT:    zip2 z3.s, z1.s, z3.s
-; CHECK-NEXT:    mov z0.d, z5.d
-; CHECK-NEXT:    mov z1.d, z2.d
-; CHECK-NEXT:    mov z2.d, z4.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv16i32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z4.s, z1.s, z3.s
+; SVE-NEXT:    zip1 z5.s, z0.s, z2.s
+; SVE-NEXT:    zip2 z2.s, z0.s, z2.s
+; SVE-NEXT:    zip2 z3.s, z1.s, z3.s
+; SVE-NEXT:    mov z0.d, z5.d
+; SVE-NEXT:    mov z1.d, z2.d
+; SVE-NEXT:    mov z2.d, z4.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv16i32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z4.s, z5.s }, z0.s, z2.s
+; SME2-NEXT:    zip { z2.s, z3.s }, z1.s, z3.s
+; SME2-NEXT:    mov z0.d, z4.d
+; SME2-NEXT:    mov z1.d, z5.d
+; SME2-NEXT:    ret
   %retval = call <vscale x 16 x i32>@llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %vec0, <vscale x 8 x i32> %vec1)
   ret <vscale x 16 x i32> %retval
 }
 
 define <vscale x 8 x i64> @interleave2_nxv8i64(<vscale x 4 x i64> %vec0, <vscale x 4 x i64> %vec1) {
-; CHECK-LABEL: interleave2_nxv8i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip1 z4.d, z1.d, z3.d
-; CHECK-NEXT:    zip1 z5.d, z0.d, z2.d
-; CHECK-NEXT:    zip2 z2.d, z0.d, z2.d
-; CHECK-NEXT:    zip2 z3.d, z1.d, z3.d
-; CHECK-NEXT:    mov z0.d, z5.d
-; CHECK-NEXT:    mov z1.d, z2.d
-; CHECK-NEXT:    mov z2.d, z4.d
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv8i64:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip1 z4.d, z1.d, z3.d
+; SVE-NEXT:    zip1 z5.d, z0.d, z2.d
+; SVE-NEXT:    zip2 z2.d, z0.d, z2.d
+; SVE-NEXT:    zip2 z3.d, z1.d, z3.d
+; SVE-NEXT:    mov z0.d, z5.d
+; SVE-NEXT:    mov z1.d, z2.d
+; SVE-NEXT:    mov z2.d, z4.d
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv8i64:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z4.d, z5.d }, z0.d, z2.d
+; SME2-NEXT:    zip { z2.d, z3.d }, z1.d, z3.d
+; SME2-NEXT:    mov z0.d, z4.d
+; SME2-NEXT:    mov z1.d, z5.d
+; SME2-NEXT:    ret
   %retval = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> %vec0, <vscale x 4 x i64> %vec1)
   ret <vscale x 8 x i64> %retval
 }
@@ -227,34 +490,52 @@ define <vscale x 8 x i64> @interleave2_nxv8i64(<vscale x 4 x i64> %vec0, <vscale
 ; Promote illegal type size
 
 define <vscale x 16 x i8> @interleave2_nxv8i8(<vscale x 8 x i8> %vec0, <vscale x 8 x i8> %vec1) {
-; CHECK-LABEL: interleave2_nxv8i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.h, z0.h, z1.h
-; CHECK-NEXT:    zip1 z0.h, z0.h, z1.h
-; CHECK-NEXT:    uzp1 z0.b, z0.b, z2.b
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv8i8:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.h, z0.h, z1.h
+; SVE-NEXT:    zip1 z0.h, z0.h, z1.h
+; SVE-NEXT:    uzp1 z0.b, z0.b, z2.b
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv8i8:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.h, z1.h }, z0.h, z1.h
+; SME2-NEXT:    uzp1 z0.b, z0.b, z1.b
+; SME2-NEXT:    ret
   %retval = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> %vec0, <vscale x 8 x i8> %vec1)
   ret <vscale x 16 x i8> %retval
 }
 
 define <vscale x 8 x i16> @interleave2_nxv4i16(<vscale x 4 x i16> %vec0, <vscale x 4 x i16> %vec1) {
-; CHECK-LABEL: interleave2_nxv4i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.s, z0.s, z1.s
-; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT:    uzp1 z0.h, z0.h, z2.h
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv4i16:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.s, z0.s, z1.s
+; SVE-NEXT:    zip1 z0.s, z0.s, z1.s
+; SVE-NEXT:    uzp1 z0.h, z0.h, z2.h
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv4i16:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.s, z1.s }, z0.s, z1.s
+; SME2-NEXT:    uzp1 z0.h, z0.h, z1.h
+; SME2-NEXT:    ret
   %retval = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %vec0, <vscale x 4 x i16> %vec1)
   ret <vscale x 8 x i16> %retval
 }
 
 define <vscale x 4 x i32> @interleave2_nxv2i32(<vscale x 2 x i32> %vec0, <vscale x 2 x i32> %vec1) {
-; CHECK-LABEL: interleave2_nxv2i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    zip2 z2.d, z0.d, z1.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z1.d
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-NEXT:    ret
+; SVE-LABEL: interleave2_nxv2i32:
+; SVE:       // %bb.0:
+; SVE-NEXT:    zip2 z2.d, z0.d, z1.d
+; SVE-NEXT:    zip1 z0.d, z0.d, z1.d
+; SVE-NEXT:    uzp1 z0.s, z0.s, z2.s
+; SVE-NEXT:    ret
+;
+; SME2-LABEL: interleave2_nxv2i32:
+; SME2:       // %bb.0:
+; SME2-NEXT:    zip { z0.d, z1.d }, z0.d, z1.d
+; SME2-NEXT:    uzp1 z0.s, z0.s, z1.s
+; SME2-NEXT:    ret
   %retval = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %vec0, <vscale x 2 x i32> %vec1)
   ret <vscale x 4 x i32> %retval
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll
index fe8271cdf054..f477fcbe1eb5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll
@@ -16,6 +16,17 @@ define <vscale x 16 x i8> @aesd_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
   ret <vscale x 16 x i8> %out
 }
 
+define <vscale x 16 x i8> @aesd_i8_commuted(<vscale x 16 x i8> %a,
+; CHECK-LABEL: aesd_i8_commuted:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    aesd z0.b, z0.b, z1.b
+; CHECK-NEXT:    ret
+                                            <vscale x 16 x i8> %b) {
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.aesd(<vscale x 16 x i8> %b,
+                                                        <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i8> %out
+}
+
 ;
 ; AESIMC
 ;
@@ -43,6 +54,17 @@ define <vscale x 16 x i8> @aese_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
   ret <vscale x 16 x i8> %out
 }
 
+define <vscale x 16 x i8> @aese_i8_commuted(<vscale x 16 x i8> %a,
+; CHECK-LABEL: aese_i8_commuted:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    aese z0.b, z0.b, z1.b
+; CHECK-NEXT:    ret
+                                            <vscale x 16 x i8> %b) {
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.aese(<vscale x 16 x i8> %b,
+                                                        <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i8> %out
+}
+
 ;
 ; AESMC
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/sve2p2-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/sve2p2-intrinsics.ll
new file mode 100644
index 000000000000..8d863dab7b47
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/sve2p2-intrinsics.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK
+
+;
+; COMPACT
+;
+
+define <vscale x 16 x i8> @compact_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: compact_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.b, p0, z0.b
+; CHECK-NEXT:    ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.compact.nxv16i8(<vscale x 16 x i1> %pg,
+                                                                   <vscale x 16 x i8> %a)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @compact_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: compact_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.h, p0, z0.h
+; CHECK-NEXT:    ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.compact.nxv8i16(<vscale x 8 x i1> %pg,
+                                                                   <vscale x 8 x i16> %a)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 8 x half> @compact_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: compact_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.h, p0, z0.h
+; CHECK-NEXT:    ret
+  %out = call <vscale x 8 x half> @llvm.aarch64.sve.compact.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                    <vscale x 8 x half> %a)
+  ret <vscale x 8 x half> %out
+}
+
+define <vscale x 8 x bfloat> @compact_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a) {
+; CHECK-LABEL: compact_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.h, p0, z0.h
+; CHECK-NEXT:    ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.compact.nxv8bf16(<vscale x 8 x i1> %pg,
+                                                                       <vscale x 8 x bfloat> %a)
+  ret <vscale x 8 x bfloat> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.compact.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.compact.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 8 x half> @llvm.aarch64.sve.compact.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.compact.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/tail-call-stack-args.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/tail-call-stack-args.ll
new file mode 100644
index 000000000000..9ea516f93072
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/tail-call-stack-args.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc %s -mtriple=aarch64 -o - | FileCheck %s --check-prefixes=CHECK,SDAG
+; RUN: llc %s -mtriple=aarch64 -global-isel -o - | FileCheck %s --check-prefixes=CHECK,GI
+
+; Tail calls which have stack arguments in the same offsets as the caller do not
+; need to load and store the arguments from the stack.
+
+declare void @func(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j)
+
+define void @wrapper_func(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j) {
+; CHECK-LABEL: wrapper_func:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    b func
+
+  tail call void @func(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j)
+  ret void
+}
+
+define void @wrapper_func_zero_arg(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j) {
+; CHECK-LABEL: wrapper_func_zero_arg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str wzr, [sp, #8]
+; CHECK-NEXT:    b func
+  tail call void @func(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 0)
+  ret void
+}
+
+define void @wrapper_func_overriden_arg(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j) {
+; CHECK-LABEL: wrapper_func_overriden_arg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr w8, [sp]
+; CHECK-NEXT:    str wzr, [sp]
+; CHECK-NEXT:    str w8, [sp, #8]
+; CHECK-NEXT:    b func
+  tail call void @func(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 0, i32 %i)
+  ret void
+}
+
+declare void @func_i1(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i1 %j)
+
+; FIXME: Support i1 passthrough stack arguments in GlobalISel.
+define void @wrapper_func_i1(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i1 %j) {
+; SDAG-LABEL: wrapper_func_i1:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    b func_i1
+;
+; GI-LABEL: wrapper_func_i1:
+; GI:       // %bb.0:
+; GI-NEXT:    ldrb w8, [sp, #8]
+; GI-NEXT:    strb w8, [sp, #8]
+; GI-NEXT:    b func_i1
+  tail call void @func_i1(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i1 %j)
+  ret void
+}
+
+declare void @func_signext_i1(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i1 signext %j)
+
+; FIXME: Support zero/sign-extended stack arguments.
+define void @wrapper_func_i8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i1 signext %j) {
+; CHECK-LABEL: wrapper_func_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrsb w8, [sp, #8]
+; CHECK-NEXT:    strb w8, [sp, #8]
+; CHECK-NEXT:    b func_signext_i1
+  tail call void @func_signext_i1(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i1 signext %j)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/trampoline.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/trampoline.ll
index 30ac2aa283b3..d9016b02a0f8 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/trampoline.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/trampoline.ll
@@ -1,32 +1,265 @@
-; RUN: llc -mtriple=aarch64-- < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK-LINUX
+; RUN: llc -mtriple=aarch64-none-eabi < %s | FileCheck %s --check-prefixes=CHECK-LINUX
+; RUN: llc -mtriple=aarch64-pc-windows-msvc < %s | FileCheck %s --check-prefix=CHECK-PC
+; RUN: llc -mtriple=aarch64-apple-darwin < %s | FileCheck %s --check-prefixes=CHECK-APPLE
 
 @trampg = internal global [36 x i8] zeroinitializer, align 8
 
 declare void @llvm.init.trampoline(ptr, ptr, ptr);
 declare ptr @llvm.adjust.trampoline(ptr);
 
-define i64 @f(ptr nest %c, i64 %x, i64 %y) {
-  %sum = add i64 %x, %y
-  ret i64 %sum
+define ptr @f(ptr nest %x, i64 %y) {
+; CHECK-LINUX-LABEL: f:
+; CHECK-LINUX:       // %bb.0:
+; CHECK-LINUX-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-LINUX-NEXT:    sub sp, sp, #237, lsl #12 // =970752
+; CHECK-LINUX-NEXT:    sub sp, sp, #3264
+; CHECK-LINUX-NEXT:    .cfi_def_cfa_offset 974032
+; CHECK-LINUX-NEXT:    .cfi_offset w29, -16
+; CHECK-LINUX-NEXT:    add x0, x15, x0
+; CHECK-LINUX-NEXT:    add sp, sp, #237, lsl #12 // =970752
+; CHECK-LINUX-NEXT:    add sp, sp, #3264
+; CHECK-LINUX-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-LINUX-NEXT:    ret
+;
+; CHECK-PC-LABEL: f:
+; CHECK-PC:       .seh_proc f
+; CHECK-PC-NEXT:  // %bb.0:
+; CHECK-PC-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-PC-NEXT:    .seh_save_fplr_x 16
+; CHECK-PC-NEXT:    mov x9, x15
+; CHECK-PC-NEXT:    mov x15, #60876 // =0xedcc
+; CHECK-PC-NEXT:    .seh_nop
+; CHECK-PC-NEXT:    bl __chkstk
+; CHECK-PC-NEXT:    .seh_nop
+; CHECK-PC-NEXT:    sub sp, sp, x15, lsl #4
+; CHECK-PC-NEXT:    .seh_stackalloc 974016
+; CHECK-PC-NEXT:    mov x15, x9
+; CHECK-PC-NEXT:    .seh_endprologue
+; CHECK-PC-NEXT:    add x0, x15, x0
+; CHECK-PC-NEXT:    .seh_startepilogue
+; CHECK-PC-NEXT:    add sp, sp, #237, lsl #12 // =970752
+; CHECK-PC-NEXT:    .seh_stackalloc 970752
+; CHECK-PC-NEXT:    add sp, sp, #3264
+; CHECK-PC-NEXT:    .seh_stackalloc 3264
+; CHECK-PC-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-PC-NEXT:    .seh_save_fplr_x 16
+; CHECK-PC-NEXT:    .seh_endepilogue
+; CHECK-PC-NEXT:    ret
+; CHECK-PC-NEXT:    .seh_endfunclet
+; CHECK-PC-NEXT:    .seh_endproc
+;
+; CHECK-APPLE-LABEL: f:
+; CHECK-APPLE:       ; %bb.0:
+; CHECK-APPLE-NEXT:    stp x28, x27, [sp, #-16]! ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    sub sp, sp, #237, lsl #12 ; =970752
+; CHECK-APPLE-NEXT:    sub sp, sp, #3264
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 974032
+; CHECK-APPLE-NEXT:    .cfi_offset w27, -8
+; CHECK-APPLE-NEXT:    .cfi_offset w28, -16
+; CHECK-APPLE-NEXT:    add x0, x15, x0
+; CHECK-APPLE-NEXT:    add sp, sp, #237, lsl #12 ; =970752
+; CHECK-APPLE-NEXT:    add sp, sp, #3264
+; CHECK-APPLE-NEXT:    ldp x28, x27, [sp], #16 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    ret
+  %chkstack = alloca [u0xedcba x i8]
+  %sum = getelementptr i8, ptr %x, i64 %y
+  ret ptr %sum
 }
 
 define i64 @func1() {
+; CHECK-LINUX-LABEL: func1:
+; CHECK-LINUX:       // %bb.0:
+; CHECK-LINUX-NEXT:    sub sp, sp, #64
+; CHECK-LINUX-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-LINUX-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-LINUX-NEXT:    .cfi_offset w30, -16
+; CHECK-LINUX-NEXT:    adrp x8, :got:f
+; CHECK-LINUX-NEXT:    mov w9, #544 // =0x220
+; CHECK-LINUX-NEXT:    add x0, sp, #8
+; CHECK-LINUX-NEXT:    ldr x8, [x8, :got_lo12:f]
+; CHECK-LINUX-NEXT:    movk w9, #54815, lsl #16
+; CHECK-LINUX-NEXT:    str w9, [sp, #16]
+; CHECK-LINUX-NEXT:    add x9, sp, #56
+; CHECK-LINUX-NEXT:    stp x9, x8, [sp, #24]
+; CHECK-LINUX-NEXT:    mov x8, #132 // =0x84
+; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #16
+; CHECK-LINUX-NEXT:    movk x8, #177, lsl #32
+; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #48
+; CHECK-LINUX-NEXT:    str x8, [sp, #8]
+; CHECK-LINUX-NEXT:    add x8, sp, #8
+; CHECK-LINUX-NEXT:    add x1, x8, #12
+; CHECK-LINUX-NEXT:    bl __clear_cache
+; CHECK-LINUX-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-LINUX-NEXT:    mov x0, xzr
+; CHECK-LINUX-NEXT:    add sp, sp, #64
+; CHECK-LINUX-NEXT:    ret
+;
+; CHECK-PC-LABEL: func1:
+; CHECK-PC:       .seh_proc func1
+; CHECK-PC-NEXT:  // %bb.0:
+; CHECK-PC-NEXT:    sub sp, sp, #64
+; CHECK-PC-NEXT:    .seh_stackalloc 64
+; CHECK-PC-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-PC-NEXT:    .seh_save_reg x30, 48
+; CHECK-PC-NEXT:    .seh_endprologue
+; CHECK-PC-NEXT:    adrp x8, f
+; CHECK-PC-NEXT:    add x8, x8, :lo12:f
+; CHECK-PC-NEXT:    add x9, sp, #56
+; CHECK-PC-NEXT:    stp x9, x8, [sp, #24]
+; CHECK-PC-NEXT:    mov w8, #544 // =0x220
+; CHECK-PC-NEXT:    add x0, sp, #8
+; CHECK-PC-NEXT:    movk w8, #54815, lsl #16
+; CHECK-PC-NEXT:    str w8, [sp, #16]
+; CHECK-PC-NEXT:    mov x8, #132 // =0x84
+; CHECK-PC-NEXT:    movk x8, #22528, lsl #16
+; CHECK-PC-NEXT:    movk x8, #177, lsl #32
+; CHECK-PC-NEXT:    movk x8, #22528, lsl #48
+; CHECK-PC-NEXT:    str x8, [sp, #8]
+; CHECK-PC-NEXT:    add x8, sp, #8
+; CHECK-PC-NEXT:    add x1, x8, #12
+; CHECK-PC-NEXT:    bl __clear_cache
+; CHECK-PC-NEXT:    mov x0, xzr
+; CHECK-PC-NEXT:    .seh_startepilogue
+; CHECK-PC-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-PC-NEXT:    .seh_save_reg x30, 48
+; CHECK-PC-NEXT:    add sp, sp, #64
+; CHECK-PC-NEXT:    .seh_stackalloc 64
+; CHECK-PC-NEXT:    .seh_endepilogue
+; CHECK-PC-NEXT:    ret
+; CHECK-PC-NEXT:    .seh_endfunclet
+; CHECK-PC-NEXT:    .seh_endproc
+;
+; CHECK-APPLE-LABEL: func1:
+; CHECK-APPLE:       ; %bb.0:
+; CHECK-APPLE-NEXT:    sub sp, sp, #64
+; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #48] ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
+; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
+; CHECK-APPLE-NEXT:  Lloh0:
+; CHECK-APPLE-NEXT:    adrp x8, _f@PAGE
+; CHECK-APPLE-NEXT:  Lloh1:
+; CHECK-APPLE-NEXT:    add x8, x8, _f@PAGEOFF
+; CHECK-APPLE-NEXT:    add x9, sp, #40
+; CHECK-APPLE-NEXT:    stp x9, x8, [sp, #16]
+; CHECK-APPLE-NEXT:    mov w8, #544 ; =0x220
+; CHECK-APPLE-NEXT:    mov x0, sp
+; CHECK-APPLE-NEXT:    movk w8, #54815, lsl #16
+; CHECK-APPLE-NEXT:    str w8, [sp, #8]
+; CHECK-APPLE-NEXT:    mov x8, #132 ; =0x84
+; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #16
+; CHECK-APPLE-NEXT:    movk x8, #177, lsl #32
+; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #48
+; CHECK-APPLE-NEXT:    str x8, [sp]
+; CHECK-APPLE-NEXT:    mov x8, sp
+; CHECK-APPLE-NEXT:    add x1, x8, #12
+; CHECK-APPLE-NEXT:    bl ___clear_cache
+; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #48] ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    mov x0, xzr
+; CHECK-APPLE-NEXT:    add sp, sp, #64
+; CHECK-APPLE-NEXT:    ret
+; CHECK-APPLE-NEXT:    .loh AdrpAdd Lloh0, Lloh1
   %val = alloca i64
-  %nval = bitcast ptr %val to ptr
   %tramp = alloca [36 x i8], align 8
-  ; CHECK:	mov	w1, #36
-  ; CHECK:	bl	__trampoline_setup
-  call void @llvm.init.trampoline(ptr %tramp, ptr @f, ptr %nval)
+  call void @llvm.init.trampoline(ptr %tramp, ptr @f, ptr %val)
   %fp = call ptr @llvm.adjust.trampoline(ptr %tramp)
   ret i64 0
 }
 
 define i64 @func2() {
+; CHECK-LINUX-LABEL: func2:
+; CHECK-LINUX:       // %bb.0:
+; CHECK-LINUX-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-LINUX-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-LINUX-NEXT:    .cfi_offset w30, -16
+; CHECK-LINUX-NEXT:    adrp x8, :got:f
+; CHECK-LINUX-NEXT:    mov w9, #544 // =0x220
+; CHECK-LINUX-NEXT:    adrp x0, trampg
+; CHECK-LINUX-NEXT:    add x0, x0, :lo12:trampg
+; CHECK-LINUX-NEXT:    ldr x8, [x8, :got_lo12:f]
+; CHECK-LINUX-NEXT:    movk w9, #54815, lsl #16
+; CHECK-LINUX-NEXT:    str w9, [x0, #8]
+; CHECK-LINUX-NEXT:    add x9, sp, #8
+; CHECK-LINUX-NEXT:    add x1, x0, #12
+; CHECK-LINUX-NEXT:    stp x9, x8, [x0, #16]
+; CHECK-LINUX-NEXT:    mov x8, #132 // =0x84
+; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #16
+; CHECK-LINUX-NEXT:    movk x8, #177, lsl #32
+; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #48
+; CHECK-LINUX-NEXT:    str x8, [x0]
+; CHECK-LINUX-NEXT:    bl __clear_cache
+; CHECK-LINUX-NEXT:    mov x0, xzr
+; CHECK-LINUX-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-LINUX-NEXT:    ret
+;
+; CHECK-PC-LABEL: func2:
+; CHECK-PC:       .seh_proc func2
+; CHECK-PC-NEXT:  // %bb.0:
+; CHECK-PC-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-PC-NEXT:    .seh_save_reg_x x30, 16
+; CHECK-PC-NEXT:    .seh_endprologue
+; CHECK-PC-NEXT:    adrp x0, trampg
+; CHECK-PC-NEXT:    add x0, x0, :lo12:trampg
+; CHECK-PC-NEXT:    adrp x8, f
+; CHECK-PC-NEXT:    add x8, x8, :lo12:f
+; CHECK-PC-NEXT:    add x9, sp, #8
+; CHECK-PC-NEXT:    add x1, x0, #12
+; CHECK-PC-NEXT:    stp x9, x8, [x0, #16]
+; CHECK-PC-NEXT:    mov w8, #544 // =0x220
+; CHECK-PC-NEXT:    movk w8, #54815, lsl #16
+; CHECK-PC-NEXT:    str w8, [x0, #8]
+; CHECK-PC-NEXT:    mov x8, #132 // =0x84
+; CHECK-PC-NEXT:    movk x8, #22528, lsl #16
+; CHECK-PC-NEXT:    movk x8, #177, lsl #32
+; CHECK-PC-NEXT:    movk x8, #22528, lsl #48
+; CHECK-PC-NEXT:    str x8, [x0]
+; CHECK-PC-NEXT:    bl __clear_cache
+; CHECK-PC-NEXT:    mov x0, xzr
+; CHECK-PC-NEXT:    .seh_startepilogue
+; CHECK-PC-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-PC-NEXT:    .seh_save_reg_x x30, 16
+; CHECK-PC-NEXT:    .seh_endepilogue
+; CHECK-PC-NEXT:    ret
+; CHECK-PC-NEXT:    .seh_endfunclet
+; CHECK-PC-NEXT:    .seh_endproc
+;
+; CHECK-APPLE-LABEL: func2:
+; CHECK-APPLE:       ; %bb.0:
+; CHECK-APPLE-NEXT:    sub sp, sp, #32
+; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
+; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
+; CHECK-APPLE-NEXT:  Lloh2:
+; CHECK-APPLE-NEXT:    adrp x0, _trampg@PAGE
+; CHECK-APPLE-NEXT:  Lloh3:
+; CHECK-APPLE-NEXT:    add x0, x0, _trampg@PAGEOFF
+; CHECK-APPLE-NEXT:  Lloh4:
+; CHECK-APPLE-NEXT:    adrp x8, _f@PAGE
+; CHECK-APPLE-NEXT:  Lloh5:
+; CHECK-APPLE-NEXT:    add x8, x8, _f@PAGEOFF
+; CHECK-APPLE-NEXT:    add x9, sp, #8
+; CHECK-APPLE-NEXT:    add x1, x0, #12
+; CHECK-APPLE-NEXT:    stp x9, x8, [x0, #16]
+; CHECK-APPLE-NEXT:    mov w8, #544 ; =0x220
+; CHECK-APPLE-NEXT:    movk w8, #54815, lsl #16
+; CHECK-APPLE-NEXT:    str w8, [x0, #8]
+; CHECK-APPLE-NEXT:    mov x8, #132 ; =0x84
+; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #16
+; CHECK-APPLE-NEXT:    movk x8, #177, lsl #32
+; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #48
+; CHECK-APPLE-NEXT:    str x8, [x0]
+; CHECK-APPLE-NEXT:    bl ___clear_cache
+; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    mov x0, xzr
+; CHECK-APPLE-NEXT:    add sp, sp, #32
+; CHECK-APPLE-NEXT:    ret
+; CHECK-APPLE-NEXT:    .loh AdrpAdd Lloh4, Lloh5
+; CHECK-APPLE-NEXT:    .loh AdrpAdd Lloh2, Lloh3
   %val = alloca i64
-  %nval = bitcast ptr %val to ptr
-  ; CHECK:	mov	w1, #36
-  ; CHECK:	bl	__trampoline_setup
-  call void @llvm.init.trampoline(ptr @trampg, ptr @f, ptr %nval)
+  call void @llvm.init.trampoline(ptr @trampg, ptr @f, ptr %val)
   %fp = call ptr @llvm.adjust.trampoline(ptr @trampg)
   ret i64 0
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/trunc.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/trunc.ll
index d510dc83cfd8..18a35b85eb41 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/trunc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/trunc.ll
@@ -297,10 +297,10 @@ define <3 x i32> @trunc_v3i32_v3i64(<3 x i64> %a) {
 ; CHECK-GI-LABEL: trunc_v3i32_v3i64:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -867,17 +867,11 @@ entry:
 }
 
 define <2 x i64> @trunc_v2i64_v2i128(<2 x i128> %a) {
-; CHECK-SD-LABEL: trunc_v2i64_v2i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    mov v0.d[1], x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: trunc_v2i64_v2i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.d[0], x0
-; CHECK-GI-NEXT:    mov v0.d[1], x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: trunc_v2i64_v2i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    mov v0.d[1], x2
+; CHECK-NEXT:    ret
 entry:
   %c = trunc <2 x i128> %a to <2 x i64>
   ret <2 x i64> %c
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat.ll
index e9d22c7be52e..7a7906622fb9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat.ll
@@ -2,12 +2,6 @@
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -mtriple=aarch64-- -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-declare i4 @llvm.uadd.sat.i4(i4, i4)
-declare i8 @llvm.uadd.sat.i8(i8, i8)
-declare i16 @llvm.uadd.sat.i16(i16, i16)
-declare i32 @llvm.uadd.sat.i32(i32, i32)
-declare i64 @llvm.uadd.sat.i64(i64, i64)
-
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-SD-LABEL: func:
 ; CHECK-SD:       // %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
index b0b3198fda0e..3cfb24aaccb1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll
@@ -5,37 +5,6 @@
 ; CHECK-GI:       warning: Instruction selection used fallback path for v16i4
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i1
 
-declare <1 x i8> @llvm.uadd.sat.v1i8(<1 x i8>, <1 x i8>)
-declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <12 x i8> @llvm.uadd.sat.v12i8(<12 x i8>, <12 x i8>)
-declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>)
-declare <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8>, <64 x i8>)
-
-declare <1 x i16> @llvm.uadd.sat.v1i16(<1 x i16>, <1 x i16>)
-declare <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <12 x i16> @llvm.uadd.sat.v12i16(<12 x i16>, <12 x i16>)
-declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16>, <32 x i16>)
-
-declare <16 x i1> @llvm.uadd.sat.v16i1(<16 x i1>, <16 x i1>)
-declare <16 x i4> @llvm.uadd.sat.v16i4(<16 x i4>, <16 x i4>)
-
-declare <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64>, <8 x i64>)
-
-declare <4 x i24> @llvm.uadd.sat.v4i24(<4 x i24>, <4 x i24>)
-declare <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128>, <2 x i128>)
-
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
@@ -387,6 +356,32 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
   ret <16 x i1> %z
 }
 
+define void @v1i32(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ldr s1, [x1]
+; CHECK-SD-NEXT:    uqadd v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    str s0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    adds w8, w8, w9
+; CHECK-GI-NEXT:    cset w9, hs
+; CHECK-GI-NEXT:    tst w9, #0x1
+; CHECK-GI-NEXT:    csinv w8, w8, wzr, eq
+; CHECK-GI-NEXT:    str w8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i32>, ptr %px
+  %y = load <1 x i32>, ptr %py
+  %z = call <1 x i32> @llvm.uadd.sat.v1i32(<1 x i32> %x, <1 x i32> %y)
+  store <1 x i32> %z, ptr %pz
+  ret void
+}
+
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
@@ -441,6 +436,32 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
   ret <16 x i32> %z
 }
 
+define void @v1i64(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    uqadd d0, d0, d1
+; CHECK-SD-NEXT:    str d0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr x8, [x0]
+; CHECK-GI-NEXT:    ldr x9, [x1]
+; CHECK-GI-NEXT:    adds x8, x8, x9
+; CHECK-GI-NEXT:    cset w9, hs
+; CHECK-GI-NEXT:    tst w9, #0x1
+; CHECK-GI-NEXT:    csinv x8, x8, xzr, eq
+; CHECK-GI-NEXT:    str x8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i64>, ptr %px
+  %y = load <1 x i64>, ptr %py
+  %z = call <1 x i64> @llvm.uadd.sat.v1i64(<1 x i64> %x, <1 x i64> %y)
+  store <1 x i64> %z, ptr %pz
+  ret void
+}
+
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/ucmp.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/ucmp.ll
index ad46e4abc477..af8225307fed 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/ucmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/ucmp.ll
@@ -348,10 +348,6 @@ define <16 x i8> @signOf_neon(<8 x i16> %s0_lo, <8 x i16> %s0_hi, <8 x i16> %s1_
 ; CHECK-GI-NEXT:    cmhi v1.8h, v3.8h, v1.8h
 ; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    uzp1 v1.16b, v4.16b, v5.16b
-; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-GI-NEXT:    shl v1.16b, v1.16b, #7
-; CHECK-GI-NEXT:    sshr v0.16b, v0.16b, #7
-; CHECK-GI-NEXT:    sshr v1.16b, v1.16b, #7
 ; CHECK-GI-NEXT:    sub v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat.ll
index 54d7fc5a63b1..045bb762ef79 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat.ll
@@ -2,12 +2,6 @@
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -mtriple=aarch64-- -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-declare i4 @llvm.usub.sat.i4(i4, i4)
-declare i8 @llvm.usub.sat.i8(i8, i8)
-declare i16 @llvm.usub.sat.i16(i16, i16)
-declare i32 @llvm.usub.sat.i32(i32, i32)
-declare i64 @llvm.usub.sat.i64(i64, i64)
-
 define i32 @func(i32 %x, i32 %y) nounwind {
 ; CHECK-SD-LABEL: func:
 ; CHECK-SD:       // %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat_vec.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
index 54754e7fbaed..a71cf95a728d 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/usub_sat_vec.ll
@@ -5,38 +5,6 @@
 ; CHECK-GI:       warning: Instruction selection used fallback path for v16i4
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i1
 
-declare <1 x i8> @llvm.usub.sat.v1i8(<1 x i8>, <1 x i8>)
-declare <2 x i8> @llvm.usub.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.usub.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.usub.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <12 x i8> @llvm.usub.sat.v12i8(<12 x i8>, <12 x i8>)
-declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <32 x i8> @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>)
-declare <64 x i8> @llvm.usub.sat.v64i8(<64 x i8>, <64 x i8>)
-
-declare <1 x i16> @llvm.usub.sat.v1i16(<1 x i16>, <1 x i16>)
-declare <2 x i16> @llvm.usub.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.usub.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <12 x i16> @llvm.usub.sat.v12i16(<12 x i16>, <12 x i16>)
-declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <32 x i16> @llvm.usub.sat.v32i16(<32 x i16>, <32 x i16>)
-
-declare <16 x i1> @llvm.usub.sat.v16i1(<16 x i1>, <16 x i1>)
-declare <16 x i4> @llvm.usub.sat.v16i4(<16 x i4>, <16 x i4>)
-
-declare <2 x i32> @llvm.usub.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.usub.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <2 x i64> @llvm.usub.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.usub.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.usub.sat.v8i64(<8 x i64>, <8 x i64>)
-
-declare <4 x i24> @llvm.usub.sat.v4i24(<4 x i24>, <4 x i24>)
-declare <2 x i128> @llvm.usub.sat.v2i128(<2 x i128>, <2 x i128>)
-
-
 define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
@@ -385,6 +353,32 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
   ret <16 x i1> %z
 }
 
+define void @v1i32(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ldr s1, [x1]
+; CHECK-SD-NEXT:    uqsub v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    str s0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    subs w8, w8, w9
+; CHECK-GI-NEXT:    cset w9, lo
+; CHECK-GI-NEXT:    tst w9, #0x1
+; CHECK-GI-NEXT:    csel w8, wzr, w8, ne
+; CHECK-GI-NEXT:    str w8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i32>, ptr %px
+  %y = load <1 x i32>, ptr %py
+  %z = call <1 x i32> @llvm.usub.sat.v1i32(<1 x i32> %x, <1 x i32> %y)
+  store <1 x i32> %z, ptr %pz
+  ret void
+}
+
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
@@ -439,6 +433,32 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
   ret <16 x i32> %z
 }
 
+define void @v1i64(ptr %px, ptr %py, ptr %pz) nounwind {
+; CHECK-SD-LABEL: v1i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    uqsub d0, d0, d1
+; CHECK-SD-NEXT:    str d0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v1i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr x8, [x0]
+; CHECK-GI-NEXT:    ldr x9, [x1]
+; CHECK-GI-NEXT:    subs x8, x8, x9
+; CHECK-GI-NEXT:    cset w9, lo
+; CHECK-GI-NEXT:    tst w9, #0x1
+; CHECK-GI-NEXT:    csel x8, xzr, x8, ne
+; CHECK-GI-NEXT:    str x8, [x2]
+; CHECK-GI-NEXT:    ret
+  %x = load <1 x i64>, ptr %px
+  %y = load <1 x i64>, ptr %py
+  %z = call <1 x i64> @llvm.usub.sat.v1i64(<1 x i64> %x, <1 x i64> %y)
+  store <1 x i64> %z, ptr %pz
+  ret void
+}
+
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index 4e2ca082e28b..d6d323530946 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -596,23 +596,15 @@ define i4 @convert_to_bitmask_4xi8(<4 x i8> %vec) {
 ; CHECK-GI-NEXT:    mov.b v1[3], w8
 ; CHECK-GI-NEXT:    cmeq.8b v0, v0, v1
 ; CHECK-GI-NEXT:    mvn.8b v0, v0
-; CHECK-GI-NEXT:    umov.b w8, v0[0]
-; CHECK-GI-NEXT:    umov.b w9, v0[1]
-; CHECK-GI-NEXT:    mov.s v1[0], w8
-; CHECK-GI-NEXT:    umov.b w8, v0[2]
-; CHECK-GI-NEXT:    mov.s v1[1], w9
-; CHECK-GI-NEXT:    umov.b w9, v0[3]
-; CHECK-GI-NEXT:    mov.s v1[2], w8
-; CHECK-GI-NEXT:    mov.s v1[3], w9
-; CHECK-GI-NEXT:    mov.s w8, v1[1]
-; CHECK-GI-NEXT:    mov.s w9, v1[2]
-; CHECK-GI-NEXT:    fmov w11, s1
-; CHECK-GI-NEXT:    mov.s w10, v1[3]
+; CHECK-GI-NEXT:    umov.b w8, v0[1]
+; CHECK-GI-NEXT:    umov.b w9, v0[0]
+; CHECK-GI-NEXT:    umov.b w10, v0[2]
+; CHECK-GI-NEXT:    umov.b w11, v0[3]
 ; CHECK-GI-NEXT:    and w8, w8, #0x1
-; CHECK-GI-NEXT:    bfi w11, w8, #1, #31
-; CHECK-GI-NEXT:    and w8, w9, #0x1
-; CHECK-GI-NEXT:    and w9, w10, #0x1
-; CHECK-GI-NEXT:    orr w8, w11, w8, lsl #2
+; CHECK-GI-NEXT:    bfi w9, w8, #1, #31
+; CHECK-GI-NEXT:    and w8, w10, #0x1
+; CHECK-GI-NEXT:    orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT:    and w9, w11, #0x1
 ; CHECK-GI-NEXT:    orr w8, w8, w9, lsl #3
 ; CHECK-GI-NEXT:    strb w8, [sp, #15]
 ; CHECK-GI-NEXT:    and w0, w8, #0xff
@@ -858,41 +850,32 @@ define i6 @no_combine_illegal_num_elements(<6 x i32> %vec) {
 ; CHECK-GI:       ; %bb.0:
 ; CHECK-GI-NEXT:    sub sp, sp, #16
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT:    mov.s v0[0], w0
-; CHECK-GI-NEXT:    mov.s v1[0], w4
-; CHECK-GI-NEXT:    mov.s v2[0], wzr
-; CHECK-GI-NEXT:    mov.s v0[1], w1
-; CHECK-GI-NEXT:    mov.s v1[1], w5
-; CHECK-GI-NEXT:    mov.s v2[1], wzr
-; CHECK-GI-NEXT:    mov.s v0[2], w2
-; CHECK-GI-NEXT:    cmeq.4s v1, v1, v2
-; CHECK-GI-NEXT:    mvn.16b v1, v1
-; CHECK-GI-NEXT:    mov.s v0[3], w3
-; CHECK-GI-NEXT:    cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT:    mov.s w8, v0[1]
-; CHECK-GI-NEXT:    mov.s w9, v0[2]
-; CHECK-GI-NEXT:    mov.s w10, v0[3]
-; CHECK-GI-NEXT:    mov.h v0[1], w8
+; CHECK-GI-NEXT:    fmov s1, w0
+; CHECK-GI-NEXT:    fmov s0, wzr
+; CHECK-GI-NEXT:    fmov s2, w4
+; CHECK-GI-NEXT:    mov.s v1[1], w1
+; CHECK-GI-NEXT:    mov.s v2[1], w5
+; CHECK-GI-NEXT:    mov.s v0[1], wzr
+; CHECK-GI-NEXT:    mov.s v1[2], w2
+; CHECK-GI-NEXT:    cmeq.4s v0, v2, v0
+; CHECK-GI-NEXT:    mvn.16b v0, v0
+; CHECK-GI-NEXT:    mov.s v1[3], w3
+; CHECK-GI-NEXT:    cmtst.4s v1, v1, v1
 ; CHECK-GI-NEXT:    mov.s w8, v1[1]
-; CHECK-GI-NEXT:    mov.h v0[2], w9
-; CHECK-GI-NEXT:    mov.h v0[3], w10
-; CHECK-GI-NEXT:    mov.h v0[4], v1[0]
-; CHECK-GI-NEXT:    mov.h v0[5], w8
-; CHECK-GI-NEXT:    umov.h w8, v0[1]
-; CHECK-GI-NEXT:    umov.h w9, v0[0]
-; CHECK-GI-NEXT:    umov.h w10, v0[2]
-; CHECK-GI-NEXT:    umov.h w11, v0[3]
+; CHECK-GI-NEXT:    mov.s w9, v1[2]
+; CHECK-GI-NEXT:    fmov w11, s1
+; CHECK-GI-NEXT:    mov.s w10, v1[3]
 ; CHECK-GI-NEXT:    and w8, w8, #0x1
-; CHECK-GI-NEXT:    bfi w9, w8, #1, #31
-; CHECK-GI-NEXT:    and w8, w10, #0x1
-; CHECK-GI-NEXT:    umov.h w10, v0[4]
-; CHECK-GI-NEXT:    orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT:    and w9, w11, #0x1
-; CHECK-GI-NEXT:    umov.h w11, v0[5]
-; CHECK-GI-NEXT:    orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT:    bfi w11, w8, #1, #31
+; CHECK-GI-NEXT:    and w8, w9, #0x1
 ; CHECK-GI-NEXT:    and w9, w10, #0x1
+; CHECK-GI-NEXT:    mov.s w10, v0[1]
+; CHECK-GI-NEXT:    orr w8, w11, w8, lsl #2
+; CHECK-GI-NEXT:    orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    and w9, w9, #0x1
 ; CHECK-GI-NEXT:    orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT:    and w9, w11, #0x1
+; CHECK-GI-NEXT:    and w9, w10, #0x1
 ; CHECK-GI-NEXT:    orr w8, w8, w9, lsl #5
 ; CHECK-GI-NEXT:    and w8, w8, #0x3f
 ; CHECK-GI-NEXT:    strb w8, [sp, #15]
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index fd2428236628..290a473df6bf 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -2018,29 +2018,33 @@ define i32 @test_udot_v24i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_udot_v24i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT:    ldr q1, [x0]
-; CHECK-GI-BASE-NEXT:    ldr d2, [x0, #16]
-; CHECK-GI-BASE-NEXT:    ldr q3, [x1]
-; CHECK-GI-BASE-NEXT:    ldr d4, [x1, #16]
-; CHECK-GI-BASE-NEXT:    ushll v5.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll v6.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v4.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    fmov s0, wzr
+; CHECK-GI-BASE-NEXT:    fmov s1, wzr
+; CHECK-GI-BASE-NEXT:    ldr q2, [x0]
+; CHECK-GI-BASE-NEXT:    ldr d3, [x0, #16]
+; CHECK-GI-BASE-NEXT:    ldr q4, [x1]
+; CHECK-GI-BASE-NEXT:    ldr d5, [x1, #16]
+; CHECK-GI-BASE-NEXT:    ushll v6.8h, v2.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v2.8h, v2.16b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
-; CHECK-GI-BASE-NEXT:    umull v7.4s, v6.4h, v5.4h
-; CHECK-GI-BASE-NEXT:    umull v16.4s, v3.4h, v1.4h
-; CHECK-GI-BASE-NEXT:    umull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT:    ushll v3.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll v7.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v5.8h, v5.8b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT:    umlal2 v7.4s, v6.8h, v5.8h
-; CHECK-GI-BASE-NEXT:    umlal2 v16.4s, v3.8h, v1.8h
-; CHECK-GI-BASE-NEXT:    umlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT:    umull v16.4s, v7.4h, v6.4h
+; CHECK-GI-BASE-NEXT:    umull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    umull v18.4s, v5.4h, v3.4h
 ; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT:    add v1.4s, v7.4s, v16.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v17.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT:    umlal2 v16.4s, v7.8h, v6.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v18.4s, v5.8h, v3.8h
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v16.4s, v17.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v18.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
 ; CHECK-GI-BASE-NEXT:    fmov w0, s0
@@ -2114,45 +2118,58 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_udot_v48i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT:    ldp q1, q5, [x1]
-; CHECK-GI-BASE-NEXT:    ldp q2, q3, [x0]
-; CHECK-GI-BASE-NEXT:    ldr q4, [x0, #32]
-; CHECK-GI-BASE-NEXT:    ldr q6, [x1, #32]
-; CHECK-GI-BASE-NEXT:    ushll v7.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v16.8h, v5.8b, #0
+; CHECK-GI-BASE-NEXT:    fmov s0, wzr
+; CHECK-GI-BASE-NEXT:    fmov s2, wzr
+; CHECK-GI-BASE-NEXT:    ldr q16, [x0, #32]
+; CHECK-GI-BASE-NEXT:    fmov s1, wzr
+; CHECK-GI-BASE-NEXT:    fmov s3, wzr
+; CHECK-GI-BASE-NEXT:    ldr q19, [x1, #32]
+; CHECK-GI-BASE-NEXT:    ldp q5, q7, [x1]
+; CHECK-GI-BASE-NEXT:    ushll v23.8h, v16.8b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-BASE-NEXT:    ushll v20.8h, v19.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT:    mov v3.s[1], wzr
+; CHECK-GI-BASE-NEXT:    ushll2 v19.8h, v19.16b, #0
+; CHECK-GI-BASE-NEXT:    ldp q18, q17, [x0]
+; CHECK-GI-BASE-NEXT:    ushll v4.8h, v5.8b, #0
 ; CHECK-GI-BASE-NEXT:    ushll2 v5.8h, v5.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v17.8h, v6.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v6.8h, v6.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v18.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v19.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT:    ushll v20.8h, v4.8b, #0
-; CHECK-GI-BASE-NEXT:    ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v6.8h, v7.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v7.8h, v7.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v16.8h, v16.16b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT:    umull v21.4s, v7.4h, v18.4h
-; CHECK-GI-BASE-NEXT:    umull v22.4s, v1.4h, v2.4h
-; CHECK-GI-BASE-NEXT:    umull v23.4s, v16.4h, v19.4h
-; CHECK-GI-BASE-NEXT:    umull v24.4s, v5.4h, v3.4h
-; CHECK-GI-BASE-NEXT:    umull v25.4s, v17.4h, v20.4h
-; CHECK-GI-BASE-NEXT:    umull v26.4s, v6.4h, v4.4h
+; CHECK-GI-BASE-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-BASE-NEXT:    ushll v21.8h, v18.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v18.8h, v18.16b, #0
+; CHECK-GI-BASE-NEXT:    ushll v22.8h, v17.8b, #0
+; CHECK-GI-BASE-NEXT:    ushll2 v17.8h, v17.16b, #0
+; CHECK-GI-BASE-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-BASE-NEXT:    umull v28.4s, v20.4h, v23.4h
+; CHECK-GI-BASE-NEXT:    umull v29.4s, v19.4h, v16.4h
+; CHECK-GI-BASE-NEXT:    umull v24.4s, v4.4h, v21.4h
+; CHECK-GI-BASE-NEXT:    umull v25.4s, v5.4h, v18.4h
+; CHECK-GI-BASE-NEXT:    umull v26.4s, v6.4h, v22.4h
+; CHECK-GI-BASE-NEXT:    umull v27.4s, v7.4h, v17.4h
 ; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT:    umlal2 v21.4s, v7.8h, v18.8h
-; CHECK-GI-BASE-NEXT:    umlal2 v22.4s, v1.8h, v2.8h
-; CHECK-GI-BASE-NEXT:    umlal2 v23.4s, v16.8h, v19.8h
-; CHECK-GI-BASE-NEXT:    umlal2 v24.4s, v5.8h, v3.8h
-; CHECK-GI-BASE-NEXT:    umlal2 v25.4s, v17.8h, v20.8h
-; CHECK-GI-BASE-NEXT:    umlal2 v26.4s, v6.8h, v4.8h
-; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT:    add v1.4s, v21.4s, v22.4s
-; CHECK-GI-BASE-NEXT:    add v2.4s, v23.4s, v24.4s
-; CHECK-GI-BASE-NEXT:    add v3.4s, v25.4s, v26.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-BASE-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-BASE-NEXT:    umlal2 v28.4s, v20.8h, v23.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v29.4s, v19.8h, v16.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v24.4s, v4.8h, v21.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v25.4s, v5.8h, v18.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v26.4s, v6.8h, v22.8h
+; CHECK-GI-BASE-NEXT:    umlal2 v27.4s, v7.8h, v17.8h
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-BASE-NEXT:    add v4.4s, v28.4s, v29.4s
+; CHECK-GI-BASE-NEXT:    add v2.4s, v24.4s, v25.4s
+; CHECK-GI-BASE-NEXT:    add v3.4s, v26.4s, v27.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v4.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
 ; CHECK-GI-BASE-NEXT:    fmov w0, s0
@@ -2160,15 +2177,15 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-DOT-LABEL: test_udot_v48i8:
 ; CHECK-GI-DOT:       // %bb.0: // %entry
-; CHECK-GI-DOT-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-DOT-NEXT:    fmov s0, wzr
 ; CHECK-GI-DOT-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    ldr q7, [x0, #32]
 ; CHECK-GI-DOT-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    ldr q17, [x1, #32]
 ; CHECK-GI-DOT-NEXT:    ldp q4, q5, [x0]
-; CHECK-GI-DOT-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-DOT-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-DOT-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-DOT-NEXT:    udot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-DOT-NEXT:    udot v1.4s, v6.16b, v4.16b
 ; CHECK-GI-DOT-NEXT:    udot v3.4s, v16.16b, v5.16b
@@ -2319,29 +2336,33 @@ define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_sdot_v24i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT:    ldr q1, [x0]
-; CHECK-GI-BASE-NEXT:    ldr d2, [x0, #16]
-; CHECK-GI-BASE-NEXT:    ldr q3, [x1]
-; CHECK-GI-BASE-NEXT:    ldr d4, [x1, #16]
-; CHECK-GI-BASE-NEXT:    sshll v5.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll v6.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    fmov s0, wzr
+; CHECK-GI-BASE-NEXT:    fmov s1, wzr
+; CHECK-GI-BASE-NEXT:    ldr q2, [x0]
+; CHECK-GI-BASE-NEXT:    ldr d3, [x0, #16]
+; CHECK-GI-BASE-NEXT:    ldr q4, [x1]
+; CHECK-GI-BASE-NEXT:    ldr d5, [x1, #16]
+; CHECK-GI-BASE-NEXT:    sshll v6.8h, v2.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v2.8h, v2.16b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
-; CHECK-GI-BASE-NEXT:    smull v7.4s, v6.4h, v5.4h
-; CHECK-GI-BASE-NEXT:    smull v16.4s, v3.4h, v1.4h
-; CHECK-GI-BASE-NEXT:    smull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT:    sshll v3.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll v7.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v5.8h, v5.8b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT:    smlal2 v7.4s, v6.8h, v5.8h
-; CHECK-GI-BASE-NEXT:    smlal2 v16.4s, v3.8h, v1.8h
-; CHECK-GI-BASE-NEXT:    smlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT:    smull v16.4s, v7.4h, v6.4h
+; CHECK-GI-BASE-NEXT:    smull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT:    smull v18.4s, v5.4h, v3.4h
 ; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT:    add v1.4s, v7.4s, v16.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v17.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT:    smlal2 v16.4s, v7.8h, v6.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v18.4s, v5.8h, v3.8h
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v16.4s, v17.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v18.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
 ; CHECK-GI-BASE-NEXT:    fmov w0, s0
@@ -2415,45 +2436,58 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-BASE-LABEL: test_sdot_v48i8:
 ; CHECK-GI-BASE:       // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT:    mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT:    ldp q1, q5, [x1]
-; CHECK-GI-BASE-NEXT:    ldp q2, q3, [x0]
-; CHECK-GI-BASE-NEXT:    ldr q4, [x0, #32]
-; CHECK-GI-BASE-NEXT:    ldr q6, [x1, #32]
-; CHECK-GI-BASE-NEXT:    sshll v7.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v16.8h, v5.8b, #0
+; CHECK-GI-BASE-NEXT:    fmov s0, wzr
+; CHECK-GI-BASE-NEXT:    fmov s2, wzr
+; CHECK-GI-BASE-NEXT:    ldr q16, [x0, #32]
+; CHECK-GI-BASE-NEXT:    fmov s1, wzr
+; CHECK-GI-BASE-NEXT:    fmov s3, wzr
+; CHECK-GI-BASE-NEXT:    ldr q19, [x1, #32]
+; CHECK-GI-BASE-NEXT:    ldp q5, q7, [x1]
+; CHECK-GI-BASE-NEXT:    sshll v23.8h, v16.8b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT:    mov v2.s[1], wzr
+; CHECK-GI-BASE-NEXT:    sshll v20.8h, v19.8b, #0
+; CHECK-GI-BASE-NEXT:    mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT:    mov v3.s[1], wzr
+; CHECK-GI-BASE-NEXT:    sshll2 v19.8h, v19.16b, #0
+; CHECK-GI-BASE-NEXT:    ldp q18, q17, [x0]
+; CHECK-GI-BASE-NEXT:    sshll v4.8h, v5.8b, #0
 ; CHECK-GI-BASE-NEXT:    sshll2 v5.8h, v5.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v17.8h, v6.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v6.8h, v6.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v18.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v19.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT:    sshll v20.8h, v4.8b, #0
-; CHECK-GI-BASE-NEXT:    sshll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v6.8h, v7.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v7.8h, v7.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v16.8h, v16.16b, #0
 ; CHECK-GI-BASE-NEXT:    mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT:    smull v21.4s, v7.4h, v18.4h
-; CHECK-GI-BASE-NEXT:    smull v22.4s, v1.4h, v2.4h
-; CHECK-GI-BASE-NEXT:    smull v23.4s, v16.4h, v19.4h
-; CHECK-GI-BASE-NEXT:    smull v24.4s, v5.4h, v3.4h
-; CHECK-GI-BASE-NEXT:    smull v25.4s, v17.4h, v20.4h
-; CHECK-GI-BASE-NEXT:    smull v26.4s, v6.4h, v4.4h
+; CHECK-GI-BASE-NEXT:    mov v2.s[2], wzr
+; CHECK-GI-BASE-NEXT:    sshll v21.8h, v18.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v18.8h, v18.16b, #0
+; CHECK-GI-BASE-NEXT:    sshll v22.8h, v17.8b, #0
+; CHECK-GI-BASE-NEXT:    sshll2 v17.8h, v17.16b, #0
+; CHECK-GI-BASE-NEXT:    mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT:    mov v3.s[2], wzr
+; CHECK-GI-BASE-NEXT:    smull v28.4s, v20.4h, v23.4h
+; CHECK-GI-BASE-NEXT:    smull v29.4s, v19.4h, v16.4h
+; CHECK-GI-BASE-NEXT:    smull v24.4s, v4.4h, v21.4h
+; CHECK-GI-BASE-NEXT:    smull v25.4s, v5.4h, v18.4h
+; CHECK-GI-BASE-NEXT:    smull v26.4s, v6.4h, v22.4h
+; CHECK-GI-BASE-NEXT:    smull v27.4s, v7.4h, v17.4h
 ; CHECK-GI-BASE-NEXT:    mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT:    smlal2 v21.4s, v7.8h, v18.8h
-; CHECK-GI-BASE-NEXT:    smlal2 v22.4s, v1.8h, v2.8h
-; CHECK-GI-BASE-NEXT:    smlal2 v23.4s, v16.8h, v19.8h
-; CHECK-GI-BASE-NEXT:    smlal2 v24.4s, v5.8h, v3.8h
-; CHECK-GI-BASE-NEXT:    smlal2 v25.4s, v17.8h, v20.8h
-; CHECK-GI-BASE-NEXT:    smlal2 v26.4s, v6.8h, v4.8h
-; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT:    add v1.4s, v21.4s, v22.4s
-; CHECK-GI-BASE-NEXT:    add v2.4s, v23.4s, v24.4s
-; CHECK-GI-BASE-NEXT:    add v3.4s, v25.4s, v26.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-GI-BASE-NEXT:    add v0.4s, v3.4s, v0.4s
+; CHECK-GI-BASE-NEXT:    mov v2.s[3], wzr
+; CHECK-GI-BASE-NEXT:    mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT:    mov v3.s[3], wzr
+; CHECK-GI-BASE-NEXT:    smlal2 v28.4s, v20.8h, v23.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v29.4s, v19.8h, v16.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v24.4s, v4.8h, v21.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v25.4s, v5.8h, v18.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v26.4s, v6.8h, v22.8h
+; CHECK-GI-BASE-NEXT:    smlal2 v27.4s, v7.8h, v17.8h
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-GI-BASE-NEXT:    add v4.4s, v28.4s, v29.4s
+; CHECK-GI-BASE-NEXT:    add v2.4s, v24.4s, v25.4s
+; CHECK-GI-BASE-NEXT:    add v3.4s, v26.4s, v27.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-BASE-NEXT:    add v0.4s, v4.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-GI-BASE-NEXT:    addv s0, v0.4s
 ; CHECK-GI-BASE-NEXT:    fmov w0, s0
@@ -2461,15 +2495,15 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) {
 ;
 ; CHECK-GI-DOT-LABEL: test_sdot_v48i8:
 ; CHECK-GI-DOT:       // %bb.0: // %entry
-; CHECK-GI-DOT-NEXT:    mov v0.s[0], wzr
+; CHECK-GI-DOT-NEXT:    fmov s0, wzr
 ; CHECK-GI-DOT-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    ldr q7, [x0, #32]
 ; CHECK-GI-DOT-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-GI-DOT-NEXT:    ldr q17, [x1, #32]
 ; CHECK-GI-DOT-NEXT:    ldp q4, q5, [x0]
-; CHECK-GI-DOT-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-DOT-NEXT:    mov v0.s[1], wzr
+; CHECK-GI-DOT-NEXT:    ldp q6, q16, [x1]
 ; CHECK-GI-DOT-NEXT:    sdot v2.4s, v17.16b, v7.16b
 ; CHECK-GI-DOT-NEXT:    sdot v1.4s, v6.16b, v4.16b
 ; CHECK-GI-DOT-NEXT:    sdot v3.4s, v16.16b, v5.16b
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/win-sve.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/win-sve.ll
new file mode 100644
index 000000000000..691230af3e67
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/win-sve.ll
@@ -0,0 +1,1580 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-windows-msvc -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-windows-msvc -mattr=+sve -filetype=obj < %s -o /dev/null
+
+declare void @g()
+define i32 @f(<vscale x 2 x i64> %x) {
+; CHECK-LABEL: f:
+; CHECK:       .seh_proc f
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    bl g
+; CHECK-NEXT:    mov w0, #3 // =0x3
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    ldp x29, x30, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  call void @g()
+  ret i32 3
+}
+
+declare void @g2(ptr,ptr)
+define void @f2(i64 %n, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: f2:
+; CHECK:       .seh_proc f2
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    str x19, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x19, 32
+; CHECK-NEXT:    str x28, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    add x29, sp, #16
+; CHECK-NEXT:    .seh_add_fp 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    lsl x8, x0, #3
+; CHECK-NEXT:    mov x19, sp
+; CHECK-NEXT:    add x8, x8, #15
+; CHECK-NEXT:    lsr x15, x8, #4
+; CHECK-NEXT:    bl __chkstk
+; CHECK-NEXT:    sub x1, sp, x15, lsl #4
+; CHECK-NEXT:    mov sp, x1
+; CHECK-NEXT:    add x0, x19, #0
+; CHECK-NEXT:    bl g2
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    sub sp, x29, #16
+; CHECK-NEXT:    .seh_add_fp 16
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    ldr x28, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    ldr x19, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x19, 0
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %p1 = alloca [4 x i64]
+  %p2 = alloca i64, i64 %n
+  call void @g2(ptr %p1, ptr %p2)
+  ret void
+}
+
+declare void @g3(ptr,ptr)
+define void @f3(i64 %n, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: f3:
+; CHECK:       .seh_proc f3
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    add x0, sp, #8
+; CHECK-NEXT:    mov x1, sp
+; CHECK-NEXT:    bl g3
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldp x29, x30, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %p1 = alloca i64
+  %p2 = alloca i64
+  call void @g3(ptr %p1, ptr %p2)
+  ret void
+}
+
+declare void @g4(ptr,ptr)
+define void @f4(i64 %n, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: f4:
+; CHECK:       .seh_proc f4
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    add x1, sp, #16
+; CHECK-NEXT:    bl g4
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldp x29, x30, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %p1 = alloca [2 x i64]
+  %p2 = alloca <vscale x 2 x i64>
+  call void @g4(ptr %p1, ptr %p2)
+  ret void
+}
+
+declare void @g5(ptr,ptr,ptr)
+define void @f5(i64 %n, <vscale x 2 x i64> %x) {
+; CHECK-LABEL: f5:
+; CHECK:       .seh_proc f5
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    str x19, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x19, 32
+; CHECK-NEXT:    str x28, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    add x29, sp, #16
+; CHECK-NEXT:    .seh_add_fp 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    lsl x8, x0, #3
+; CHECK-NEXT:    mov x19, sp
+; CHECK-NEXT:    add x8, x8, #15
+; CHECK-NEXT:    lsr x15, x8, #4
+; CHECK-NEXT:    bl __chkstk
+; CHECK-NEXT:    sub x2, sp, x15, lsl #4
+; CHECK-NEXT:    mov sp, x2
+; CHECK-NEXT:    sub x1, x29, #16
+; CHECK-NEXT:    add x0, x19, #0
+; CHECK-NEXT:    addvl x1, x1, #-1
+; CHECK-NEXT:    bl g5
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    sub sp, x29, #16
+; CHECK-NEXT:    .seh_add_fp 16
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    ldr x28, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    ldr x19, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x19, 0
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %p1 = alloca [8 x i64]
+  %p2 = alloca <vscale x 2 x i64>
+  %p3 = alloca i64, i64 %n
+  call void @g5(ptr %p1, ptr %p2, ptr %p3)
+  ret void
+}
+
+declare dso_local i32 @__CxxFrameHandler3(...)
+declare void @g6(ptr, i64)
+declare void @cleanup()
+define void @f6(<vscale x 2 x i64> %x, [8 x i64] %pad, i64 %n9) personality ptr @__CxxFrameHandler3 {
+; CHECK-LABEL: f6:
+; CHECK:       .Lfunc_begin0:
+; CHECK-NEXT:  .seh_proc f6
+; CHECK-NEXT:    .seh_handler __CxxFrameHandler3, @unwind, @except
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    str x19, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x19, 32
+; CHECK-NEXT:    str x28, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    add x29, sp, #16
+; CHECK-NEXT:    .seh_add_fp 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    mov x0, #-2 // =0xfffffffffffffffe
+; CHECK-NEXT:    addvl x8, x29, #18
+; CHECK-NEXT:    mov x19, sp
+; CHECK-NEXT:    stur x0, [x8, #16]
+; CHECK-NEXT:    addvl x8, x29, #18
+; CHECK-NEXT:    ldr x1, [x8, #32]
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    add x0, x19, #0
+; CHECK-NEXT:    bl g6
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:  // %bb.1: // %invoke.cont
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .seh_stackalloc 64
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    ldr x28, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    ldr x19, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x19, 0
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_handlerdata
+; CHECK-NEXT:    .word $cppxdata$f6@IMGREL
+; CHECK-NEXT:    .text
+; CHECK-NEXT:    .seh_endproc
+; CHECK-NEXT:    .def "?dtor$2@?0?f6@4HA";
+; CHECK-NEXT:    .scl 3;
+; CHECK-NEXT:    .type 32;
+; CHECK-NEXT:    .endef
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  "?dtor$2@?0?f6@4HA":
+; CHECK-NEXT:  .seh_proc "?dtor$2@?0?f6@4HA"
+; CHECK-NEXT:  .LBB5_2: // %ehcleanup
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    str x19, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x19, 32
+; CHECK-NEXT:    str x28, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    bl cleanup
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 16
+; CHECK-NEXT:    ldr x28, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 8
+; CHECK-NEXT:    ldr x19, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x19, 0
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+  %p1 = alloca [8 x i64]
+  invoke void @g6(ptr %p1, i64 %n9) to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %c = cleanuppad within none []
+  call void @cleanup() [ "funclet"(token %c) ]
+  cleanupret from %c unwind to caller
+}
+
+declare void @g7(ptr)
+define void @f7(i64 %n) {
+; CHECK-LABEL: f7:
+; CHECK:       .seh_proc f7
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    bl g7
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %p = alloca <vscale x 2 x i64>
+  call void @g7(ptr %p)
+  ret void
+}
+
+define void @f8(<vscale x 2 x i64> %v) {
+; CHECK-LABEL: f8:
+; CHECK:       .seh_proc f8
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x30, 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    ldr x30, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  call void asm "", "~{d8}"()
+  ret void
+}
+
+define void @f9(<vscale x 2 x i64> %v, ...) {
+; CHECK-LABEL: f9:
+; CHECK:       .seh_proc f9
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    .seh_stackalloc 64
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x30, 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    add x9, sp, #8
+; CHECK-NEXT:    str x2, [x8, #32]
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    str x0, [x8, #16]
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    str x1, [x8, #24]
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    str x3, [x8, #40]
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    str x4, [x8, #48]
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    str x5, [x8, #56]
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    str x6, [x8, #64]
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    str x7, [x8, #72]
+; CHECK-NEXT:    add x8, sp, #16
+; CHECK-NEXT:    addvl x8, x8, #1
+; CHECK-NEXT:    str x8, [sp, #8]
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    ldr x30, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .seh_stackalloc 64
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .seh_stackalloc 64
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %va_list = alloca ptr
+  call void @llvm.va_start.p0(ptr %va_list)
+  call void asm "", "r,~{d8},~{memory}"(ptr %va_list)
+  ret void
+}
+
+declare void @g10(ptr,ptr)
+define void @f10(i64 %n, <vscale x 2 x i64> %x) "frame-pointer"="all" {
+; CHECK-LABEL: f10:
+; CHECK:       .seh_proc f10
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    str p4, [sp] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    str p5, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    str p6, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    str p7, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    str p8, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    str p9, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    str p11, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    str p12, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    str p13, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    str p14, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    str p15, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    str z9, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    str z10, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    str z12, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    str z13, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    str z14, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    str z16, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    str z17, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    str z18, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    str z19, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    str z20, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    str z21, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    str z22, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    str x28, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 32
+; CHECK-NEXT:    stp x29, x30, [sp, #8] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr 8
+; CHECK-NEXT:    add x29, sp, #8
+; CHECK-NEXT:    .seh_add_fp 8
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    sub x1, x29, #8
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    addvl x1, x1, #-1
+; CHECK-NEXT:    bl g10
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldp x29, x30, [sp, #8] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 8
+; CHECK-NEXT:    ldr x28, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 0
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 2
+; CHECK-NEXT:    ldr z9, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z9, 3
+; CHECK-NEXT:    ldr z10, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z10, 4
+; CHECK-NEXT:    ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z11, 5
+; CHECK-NEXT:    ldr z12, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z12, 6
+; CHECK-NEXT:    ldr z13, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z13, 7
+; CHECK-NEXT:    ldr z14, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z14, 8
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z15, 9
+; CHECK-NEXT:    ldr z16, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z16, 10
+; CHECK-NEXT:    ldr z17, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z17, 11
+; CHECK-NEXT:    ldr z18, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z18, 12
+; CHECK-NEXT:    ldr z19, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z19, 13
+; CHECK-NEXT:    ldr z20, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z20, 14
+; CHECK-NEXT:    ldr z21, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z21, 15
+; CHECK-NEXT:    ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z22, 16
+; CHECK-NEXT:    ldr z23, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z23, 17
+; CHECK-NEXT:    ldr p4, [sp] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p4, 0
+; CHECK-NEXT:    ldr p5, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p5, 1
+; CHECK-NEXT:    ldr p6, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p6, 2
+; CHECK-NEXT:    ldr p7, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p7, 3
+; CHECK-NEXT:    ldr p8, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p8, 4
+; CHECK-NEXT:    ldr p9, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p9, 5
+; CHECK-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p10, 6
+; CHECK-NEXT:    ldr p11, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p11, 7
+; CHECK-NEXT:    ldr p12, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p12, 8
+; CHECK-NEXT:    ldr p13, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p13, 9
+; CHECK-NEXT:    ldr p14, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p14, 10
+; CHECK-NEXT:    ldr p15, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    .seh_save_preg p15, 11
+; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .seh_allocz 18
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %p1 = alloca [2 x i64]
+  %p2 = alloca <vscale x 2 x i64>
+  call void @g10(ptr %p1, ptr %p2)
+  ret void
+}
+
+; Check handling of alloca allocated into CSR space, no frame pointer.
+define i32 @f11(double %d, <vscale x 4 x i32> %vs) "aarch64_pstate_sm_compatible" {
+; CHECK-LABEL: f11:
+; CHECK:       .seh_proc f11
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x30, 16
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    str d0, [sp, #8]
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    ldr x30, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+entry:
+  %a = alloca double
+  tail call void asm sideeffect "", "~{d8}"() #1
+  store double %d, ptr %a
+  ret i32 0
+}
+
+define i32 @f12(double %d, <vscale x 4 x i32> %vs) "aarch64_pstate_sm_compatible" {
+; CHECK-LABEL: f12:
+; CHECK:       .seh_proc f12
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x30, 16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    addvl x8, sp, #1
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    str d0, [x8, #8]
+; CHECK-NEXT:    str d0, [sp]
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    ldr x30, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+entry:
+  %a = alloca double
+  %b = alloca <vscale x 16 x i8>
+  tail call void asm sideeffect "", "~{d8}"() #1
+  store double %d, ptr %a
+  store double %d, ptr %b
+  ret i32 0
+}
+
+; Check handling of alloca allocated into CSR space, with frame pointer.
+define i32 @f13(double %d, <vscale x 4 x i32> %vs) "frame-pointer"="all" {
+; CHECK-LABEL: f13:
+; CHECK:       .seh_proc f13
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    str x28, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 32
+; CHECK-NEXT:    stp x29, x30, [sp, #8] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr 8
+; CHECK-NEXT:    add x29, sp, #8
+; CHECK-NEXT:    .seh_add_fp 8
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    str d0, [x29, #16]
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    ldp x29, x30, [sp, #8] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 8
+; CHECK-NEXT:    ldr x28, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 0
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+entry:
+  %a = alloca double
+  tail call void asm sideeffect "", "~{d8},~{x28}"() #1
+  store double %d, ptr %a
+  ret i32 0
+}
+
+define i32 @f14(double %d, <vscale x 4 x i32> %vs) "frame-pointer"="all" {
+; CHECK-LABEL: f14:
+; CHECK:       .seh_proc f14
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    str x28, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 32
+; CHECK-NEXT:    stp x29, x30, [sp, #8] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_fplr 8
+; CHECK-NEXT:    add x29, sp, #8
+; CHECK-NEXT:    .seh_add_fp 8
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    addvl x8, x29, #-1
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    str d0, [x29, #16]
+; CHECK-NEXT:    stur d0, [x8, #-8]
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    ldp x29, x30, [sp, #8] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_fplr 8
+; CHECK-NEXT:    ldr x28, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 0
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .seh_stackalloc 32
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+entry:
+  %a = alloca double
+  %b = alloca <vscale x 16 x i8>
+  tail call void asm sideeffect "", "~{d8},~{x28}"() #1
+  store double %d, ptr %a
+  store double %d, ptr %b
+  ret i32 0
+}
+
+define tailcc void @f15(double %d, <vscale x 4 x i32> %vs, [9 x i64], i32 %i) {
+; CHECK-LABEL: f15:
+; CHECK:       .seh_proc f15
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    str x28, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 16
+; CHECK-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x30, 8
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    .seh_endprologue
+; CHECK-NEXT:    addvl x8, sp, #2
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    stp d0, d0, [sp, #8]
+; CHECK-NEXT:    ldr w8, [x8, #104]
+; CHECK-NEXT:    str w8, [sp, #8]
+; CHECK-NEXT:    .seh_startepilogue
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 8
+; CHECK-NEXT:    ldr x28, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 0
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .seh_stackalloc 16
+; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    .seh_save_zreg z8, 0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .seh_allocz 1
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    .seh_stackalloc 80
+; CHECK-NEXT:    .seh_endepilogue
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .seh_endfunclet
+; CHECK-NEXT:    .seh_endproc
+  %a = alloca double
+  %b = alloca <vscale x 16 x i8>
+  tail call void asm sideeffect "", "~{d8},~{x28}"() #1
+  store double %d, ptr %a
+  store double %d, ptr %b
+  store i32 %i, ptr %a
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/win64cc-x18.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/win64cc-x18.ll
index b3e78cc9bbb8..4b45c300e9c1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/win64cc-x18.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/win64cc-x18.ll
@@ -1,35 +1,26 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+;; Testing that nest uses x15 on all calling conventions (except Arm64EC)
 
-;; Testing that x18 is not clobbered when passing pointers with the nest
-;; attribute on windows
-
-; RUN: llc < %s -mtriple=aarch64-pc-windows-msvc | FileCheck %s --check-prefixes=CHECK,CHECK-NO-X18
-; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK-X18
+; RUN: llc < %s -mtriple=aarch64-pc-windows-msvc | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64-apple-darwin- | FileCheck %s
 
 define dso_local i64 @other(ptr nest %p) #0 {
 ; CHECK-LABEL: other:
-; CHECK-X18: ldr x0, [x18]
-; CHECK-NO-X18: ldr x0, [x0]
+; CHECK:    ldr x0, [x15]
+; CHECK:    ret
   %r = load i64, ptr %p
-; CHECK: ret
   ret i64 %r
 }
 
 define dso_local void @func() #0 {
 ; CHECK-LABEL: func:
-
-
+; CHECK:    add x15, sp, #8
+; CHECK:    bl {{_?other}}
+; CHECK:    ret
 entry:
   %p = alloca i64
-; CHECK: mov w8, #1
-; CHECK: stp x30, x8, [sp, #-16]
-; CHECK-X18: add x18, sp, #8
   store i64 1, ptr %p
-; CHECK-NO-X18: add x0, sp, #8
-; CHECK: bl other
   call void @other(ptr nest %p)
-; CHECK: ldr x30, [sp], #16
-; CHECK: ret
   ret void
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/xar.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/xar.ll
index d682f4f4a1bf..652617b58eaf 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/xar.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/xar.ll
@@ -1,6 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=aarch64 -mattr=+sha3 < %s | FileCheck --check-prefix=SHA3 %s
 ; RUN: llc -mtriple=aarch64 -mattr=-sha3 < %s | FileCheck --check-prefix=NOSHA3 %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck --check-prefix=SVE2 %s
+
+/* 128-bit vectors */
 
 define <2 x i64> @xar(<2 x i64> %x, <2 x i64> %y) {
 ; SHA3-LABEL: xar:
@@ -14,6 +17,14 @@ define <2 x i64> @xar(<2 x i64> %x, <2 x i64> %y) {
 ; NOSHA3-NEXT:    shl v0.2d, v1.2d, #10
 ; NOSHA3-NEXT:    usra v0.2d, v1.2d, #54
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    xar z0.d, z0.d, z1.d, #54
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
     %a = xor <2 x i64> %x, %y
     %b = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %a, <2 x i64> <i64 10, i64 10>)
     ret <2 x i64> %b
@@ -34,24 +45,40 @@ define <1 x i64> @xar_v1i64(<1 x i64> %a, <1 x i64> %b) {
 ; NOSHA3-NEXT:    shl d0, d1, #1
 ; NOSHA3-NEXT:    usra d0, d1, #63
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_v1i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    xar z0.d, z0.d, z1.d, #63
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
   %v.val = xor <1 x i64> %a, %b
   %fshl = tail call <1 x i64> @llvm.fshl.v1i64(<1 x i64> %v.val, <1 x i64> %v.val, <1 x i64> splat (i64 1))
   ret <1 x i64> %fshl
 }
 
-define <2 x i64> @xar_instead_of_or1(<2 x i64> %r) {
-; SHA3-LABEL: xar_instead_of_or1:
+define <2 x i64> @xar_instead_of_or_v2i64(<2 x i64> %r) {
+; SHA3-LABEL: xar_instead_of_or_v2i64:
 ; SHA3:       // %bb.0: // %entry
 ; SHA3-NEXT:    movi v1.2d, #0000000000000000
 ; SHA3-NEXT:    xar v0.2d, v0.2d, v1.2d, #39
 ; SHA3-NEXT:    ret
 ;
-; NOSHA3-LABEL: xar_instead_of_or1:
+; NOSHA3-LABEL: xar_instead_of_or_v2i64:
 ; NOSHA3:       // %bb.0: // %entry
 ; NOSHA3-NEXT:    shl v1.2d, v0.2d, #25
 ; NOSHA3-NEXT:    usra v1.2d, v0.2d, #39
 ; NOSHA3-NEXT:    mov v0.16b, v1.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v2i64:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    xar z0.d, z0.d, z1.d, #39
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
 entry:
   %or = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %r, <2 x i64> %r, <2 x i64> splat (i64 25))
   ret <2 x i64> %or
@@ -72,67 +99,266 @@ define <1 x i64> @xar_instead_of_or_v1i64(<1 x i64> %v.val) {
 ; NOSHA3-NEXT:    usra d1, d0, #63
 ; NOSHA3-NEXT:    fmov d0, d1
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v1i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    xar z0.d, z0.d, z1.d, #63
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
   %fshl = tail call <1 x i64> @llvm.fshl.v1i64(<1 x i64> %v.val, <1 x i64> %v.val, <1 x i64> splat (i64 1))
   ret <1 x i64> %fshl
 }
 
-define <4 x i32> @xar_instead_of_or2(<4 x i32> %r) {
-; SHA3-LABEL: xar_instead_of_or2:
+define <4 x i32> @xar_instead_of_or_v4i32(<4 x i32> %r) {
+; SHA3-LABEL: xar_instead_of_or_v4i32:
 ; SHA3:       // %bb.0: // %entry
 ; SHA3-NEXT:    shl v1.4s, v0.4s, #25
 ; SHA3-NEXT:    usra v1.4s, v0.4s, #7
 ; SHA3-NEXT:    mov v0.16b, v1.16b
 ; SHA3-NEXT:    ret
 ;
-; NOSHA3-LABEL: xar_instead_of_or2:
+; NOSHA3-LABEL: xar_instead_of_or_v4i32:
 ; NOSHA3:       // %bb.0: // %entry
 ; NOSHA3-NEXT:    shl v1.4s, v0.4s, #25
 ; NOSHA3-NEXT:    usra v1.4s, v0.4s, #7
 ; NOSHA3-NEXT:    mov v0.16b, v1.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v4i32:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    xar z0.s, z0.s, z1.s, #7
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
 entry:
   %or = call <4 x i32> @llvm.fshl.v2i32(<4 x i32> %r, <4 x i32> %r, <4 x i32> splat (i32 25))
   ret <4 x i32> %or
 }
 
-define <8 x i16> @xar_instead_of_or3(<8 x i16> %r) {
-; SHA3-LABEL: xar_instead_of_or3:
+define <8 x i16> @xar_instead_of_or_v8i16(<8 x i16> %r) {
+; SHA3-LABEL: xar_instead_of_or_v8i16:
 ; SHA3:       // %bb.0: // %entry
 ; SHA3-NEXT:    shl v1.8h, v0.8h, #9
 ; SHA3-NEXT:    usra v1.8h, v0.8h, #7
 ; SHA3-NEXT:    mov v0.16b, v1.16b
 ; SHA3-NEXT:    ret
 ;
-; NOSHA3-LABEL: xar_instead_of_or3:
+; NOSHA3-LABEL: xar_instead_of_or_v8i16:
 ; NOSHA3:       // %bb.0: // %entry
 ; NOSHA3-NEXT:    shl v1.8h, v0.8h, #9
 ; NOSHA3-NEXT:    usra v1.8h, v0.8h, #7
 ; NOSHA3-NEXT:    mov v0.16b, v1.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v8i16:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    xar z0.h, z0.h, z1.h, #7
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
 entry:
   %or = call <8 x i16> @llvm.fshl.v2i16(<8 x i16> %r, <8 x i16> %r, <8 x i16> splat (i16 25))
   ret <8 x i16> %or
 }
 
-define <16 x i8> @xar_instead_of_or4(<16 x i8> %r) {
-; SHA3-LABEL: xar_instead_of_or4:
+define <16 x i8> @xar_instead_of_or_v16i8(<16 x i8> %r) {
+; SHA3-LABEL: xar_instead_of_or_v16i8:
 ; SHA3:       // %bb.0: // %entry
 ; SHA3-NEXT:    add v1.16b, v0.16b, v0.16b
 ; SHA3-NEXT:    usra v1.16b, v0.16b, #7
 ; SHA3-NEXT:    mov v0.16b, v1.16b
 ; SHA3-NEXT:    ret
 ;
-; NOSHA3-LABEL: xar_instead_of_or4:
+; NOSHA3-LABEL: xar_instead_of_or_v16i8:
 ; NOSHA3:       // %bb.0: // %entry
 ; NOSHA3-NEXT:    add v1.16b, v0.16b, v0.16b
 ; NOSHA3-NEXT:    usra v1.16b, v0.16b, #7
 ; NOSHA3-NEXT:    mov v0.16b, v1.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v16i8:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    xar z0.b, z0.b, z1.b, #7
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
 entry:
   %or = call <16 x i8> @llvm.fshl.v2i8(<16 x i8> %r, <16 x i8> %r, <16 x i8> splat (i8 25))
   ret <16 x i8> %or
 }
 
+/* 64 bit vectors */
+
+define <2 x i32> @xar_v2i32(<2 x i32> %x, <2 x i32> %y) {
+; SHA3-LABEL: xar_v2i32:
+; SHA3:       // %bb.0: // %entry
+; SHA3-NEXT:    eor v1.8b, v0.8b, v1.8b
+; SHA3-NEXT:    shl v0.2s, v1.2s, #25
+; SHA3-NEXT:    usra v0.2s, v1.2s, #7
+; SHA3-NEXT:    ret
+;
+; NOSHA3-LABEL: xar_v2i32:
+; NOSHA3:       // %bb.0: // %entry
+; NOSHA3-NEXT:    eor v1.8b, v0.8b, v1.8b
+; NOSHA3-NEXT:    shl v0.2s, v1.2s, #25
+; NOSHA3-NEXT:    usra v0.2s, v1.2s, #7
+; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_v2i32:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    xar z0.s, z0.s, z1.s, #7
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+entry:
+  %a = xor <2 x i32> %x, %y
+  %b = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %a, <2 x i32> <i32 25, i32 25>)
+  ret <2 x i32> %b
+}
+
+define <2 x i32> @xar_instead_of_or_v2i32(<2 x i32> %r) {
+; SHA3-LABEL: xar_instead_of_or_v2i32:
+; SHA3:       // %bb.0: // %entry
+; SHA3-NEXT:    shl v1.2s, v0.2s, #25
+; SHA3-NEXT:    usra v1.2s, v0.2s, #7
+; SHA3-NEXT:    fmov d0, d1
+; SHA3-NEXT:    ret
+;
+; NOSHA3-LABEL: xar_instead_of_or_v2i32:
+; NOSHA3:       // %bb.0: // %entry
+; NOSHA3-NEXT:    shl v1.2s, v0.2s, #25
+; NOSHA3-NEXT:    usra v1.2s, v0.2s, #7
+; NOSHA3-NEXT:    fmov d0, d1
+; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v2i32:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    xar z0.s, z0.s, z1.s, #7
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+entry:
+  %or = call <2 x i32> @llvm.fshl(<2 x i32> %r, <2 x i32> %r, <2 x i32> splat (i32 25))
+  ret <2 x i32> %or
+}
+
+define <4 x i16> @xar_v4i16(<4 x i16> %x, <4 x i16> %y) {
+; SHA3-LABEL: xar_v4i16:
+; SHA3:       // %bb.0: // %entry
+; SHA3-NEXT:    eor v1.8b, v0.8b, v1.8b
+; SHA3-NEXT:    shl v0.4h, v1.4h, #9
+; SHA3-NEXT:    usra v0.4h, v1.4h, #7
+; SHA3-NEXT:    ret
+;
+; NOSHA3-LABEL: xar_v4i16:
+; NOSHA3:       // %bb.0: // %entry
+; NOSHA3-NEXT:    eor v1.8b, v0.8b, v1.8b
+; NOSHA3-NEXT:    shl v0.4h, v1.4h, #9
+; NOSHA3-NEXT:    usra v0.4h, v1.4h, #7
+; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_v4i16:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    xar z0.h, z0.h, z1.h, #7
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+entry:
+  %a = xor <4 x i16> %x, %y
+  %b = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %a, <4 x i16> splat (i16 25))
+  ret <4 x i16> %b
+}
+
+define <4 x i16> @xar_instead_of_or_v4i16(<4 x i16> %r) {
+; SHA3-LABEL: xar_instead_of_or_v4i16:
+; SHA3:       // %bb.0: // %entry
+; SHA3-NEXT:    shl v1.4h, v0.4h, #9
+; SHA3-NEXT:    usra v1.4h, v0.4h, #7
+; SHA3-NEXT:    fmov d0, d1
+; SHA3-NEXT:    ret
+;
+; NOSHA3-LABEL: xar_instead_of_or_v4i16:
+; NOSHA3:       // %bb.0: // %entry
+; NOSHA3-NEXT:    shl v1.4h, v0.4h, #9
+; NOSHA3-NEXT:    usra v1.4h, v0.4h, #7
+; NOSHA3-NEXT:    fmov d0, d1
+; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v4i16:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    xar z0.h, z0.h, z1.h, #7
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+entry:
+  %or = call <4 x i16> @llvm.fshl(<4 x i16> %r, <4 x i16> %r, <4 x i16> splat (i16 25))
+  ret <4 x i16> %or
+}
+
+define <8 x i8> @xar_v8i8(<8 x i8> %x, <8 x i8> %y) {
+; SHA3-LABEL: xar_v8i8:
+; SHA3:       // %bb.0: // %entry
+; SHA3-NEXT:    eor v1.8b, v0.8b, v1.8b
+; SHA3-NEXT:    add v0.8b, v1.8b, v1.8b
+; SHA3-NEXT:    usra v0.8b, v1.8b, #7
+; SHA3-NEXT:    ret
+;
+; NOSHA3-LABEL: xar_v8i8:
+; NOSHA3:       // %bb.0: // %entry
+; NOSHA3-NEXT:    eor v1.8b, v0.8b, v1.8b
+; NOSHA3-NEXT:    add v0.8b, v1.8b, v1.8b
+; NOSHA3-NEXT:    usra v0.8b, v1.8b, #7
+; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_v8i8:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    xar z0.b, z0.b, z1.b, #7
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+entry:
+  %a = xor <8 x i8> %x, %y
+  %b = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %a, <8 x i8> splat (i8 25))
+  ret <8 x i8> %b
+}
+
+define <8 x i8> @xar_instead_of_or_v8i8(<8 x i8> %r) {
+; SHA3-LABEL: xar_instead_of_or_v8i8:
+; SHA3:       // %bb.0: // %entry
+; SHA3-NEXT:    add v1.8b, v0.8b, v0.8b
+; SHA3-NEXT:    usra v1.8b, v0.8b, #7
+; SHA3-NEXT:    fmov d0, d1
+; SHA3-NEXT:    ret
+;
+; NOSHA3-LABEL: xar_instead_of_or_v8i8:
+; NOSHA3:       // %bb.0: // %entry
+; NOSHA3-NEXT:    add v1.8b, v0.8b, v0.8b
+; NOSHA3-NEXT:    usra v1.8b, v0.8b, #7
+; NOSHA3-NEXT:    fmov d0, d1
+; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: xar_instead_of_or_v8i8:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    movi v1.2d, #0000000000000000
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    xar z0.b, z0.b, z1.b, #7
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+entry:
+  %or = call <8 x i8> @llvm.fshl(<8 x i8> %r, <8 x i8> %r, <8 x i8> splat (i8 25))
+  ret <8 x i8> %or
+}
+
 declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
 declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
 declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/xtn.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/xtn.ll
index 8a4d6b8c7b78..3ae829245372 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/xtn.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/xtn.ll
@@ -127,19 +127,12 @@ entry:
 }
 
 define <2 x i8> @xtn_v2i128_v2i8(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    mov v0.s[1], w2
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v0.s[1], w2
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: xtn_v2i128_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.s[1], w2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %arg1 = trunc <2 x i128> %a to <2 x i8>
   ret <2 x i8> %arg1
@@ -165,19 +158,12 @@ entry:
 }
 
 define <2 x i16> @xtn_v2i128_v2i16(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    mov v0.s[1], w2
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v0.s[1], w2
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: xtn_v2i128_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.s[1], w2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %arg1 = trunc <2 x i128> %a to <2 x i16>
   ret <2 x i16> %arg1
@@ -194,36 +180,23 @@ entry:
 }
 
 define <2 x i32> @xtn_v2i128_v2i32(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    mov v0.s[1], w2
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v0.s[1], w2
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: xtn_v2i128_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    mov v0.s[1], w2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %arg1 = trunc <2 x i128> %a to <2 x i32>
   ret <2 x i32> %arg1
 }
 
 define <2 x i64> @xtn_v2i128_v2i64(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    mov v0.d[1], x2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.d[0], x0
-; CHECK-GI-NEXT:    mov v0.d[1], x2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: xtn_v2i128_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    mov v0.d[1], x2
+; CHECK-NEXT:    ret
 entry:
   %arg1 = trunc <2 x i128> %a to <2 x i64>
   ret <2 x i64> %arg1
@@ -341,10 +314,10 @@ define <3 x i32> @xtn_v3i64_v3i32(<3 x i64> %a) {
 ; CHECK-GI-LABEL: xtn_v3i64_v3i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/zero-call-used-regs.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/zero-call-used-regs.ll
index 4799ea3bcd19..986666e015e9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/zero-call-used-regs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/zero-call-used-regs.ll
@@ -93,7 +93,7 @@ define dso_local i32 @all_gpr_arg(i32 noundef %a, i32 noundef %b, i32 noundef %c
 ; CHECK-NEXT:    mov x5, #0 // =0x0
 ; CHECK-NEXT:    mov x6, #0 // =0x0
 ; CHECK-NEXT:    mov x7, #0 // =0x0
-; CHECK-NEXT:    mov x18, #0 // =0x0
+; CHECK-NEXT:    mov x15, #0 // =0x0
 ; CHECK-NEXT:    orr w0, w8, w2
 ; CHECK-NEXT:    mov x2, #0 // =0x0
 ; CHECK-NEXT:    mov x8, #0 // =0x0
@@ -146,7 +146,7 @@ define dso_local i32 @all_arg(i32 noundef %a, i32 noundef %b, i32 noundef %c) lo
 ; DEFAULT-NEXT:    mov x5, #0 // =0x0
 ; DEFAULT-NEXT:    mov x6, #0 // =0x0
 ; DEFAULT-NEXT:    mov x7, #0 // =0x0
-; DEFAULT-NEXT:    mov x18, #0 // =0x0
+; DEFAULT-NEXT:    mov x15, #0 // =0x0
 ; DEFAULT-NEXT:    movi v0.2d, #0000000000000000
 ; DEFAULT-NEXT:    orr w0, w8, w2
 ; DEFAULT-NEXT:    mov x2, #0 // =0x0
@@ -169,7 +169,7 @@ define dso_local i32 @all_arg(i32 noundef %a, i32 noundef %b, i32 noundef %c) lo
 ; SVE-OR-SME-NEXT:    mov x5, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov x6, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov x7, #0 // =0x0
-; SVE-OR-SME-NEXT:    mov x18, #0 // =0x0
+; SVE-OR-SME-NEXT:    mov x15, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov z0.d, #0 // =0x0
 ; SVE-OR-SME-NEXT:    orr w0, w8, w2
 ; SVE-OR-SME-NEXT:    mov x2, #0 // =0x0
@@ -196,7 +196,7 @@ define dso_local i32 @all_arg(i32 noundef %a, i32 noundef %b, i32 noundef %c) lo
 ; STREAMING-COMPAT-NEXT:    mov x5, #0 // =0x0
 ; STREAMING-COMPAT-NEXT:    mov x6, #0 // =0x0
 ; STREAMING-COMPAT-NEXT:    mov x7, #0 // =0x0
-; STREAMING-COMPAT-NEXT:    mov x18, #0 // =0x0
+; STREAMING-COMPAT-NEXT:    mov x15, #0 // =0x0
 ; STREAMING-COMPAT-NEXT:    fmov d0, xzr
 ; STREAMING-COMPAT-NEXT:    orr w0, w8, w2
 ; STREAMING-COMPAT-NEXT:    mov x2, #0 // =0x0
@@ -492,7 +492,7 @@ define dso_local double @all_gpr_arg_float(double noundef %a, float noundef %b)
 ; CHECK-NEXT:    mov x6, #0 // =0x0
 ; CHECK-NEXT:    mov x7, #0 // =0x0
 ; CHECK-NEXT:    mov x8, #0 // =0x0
-; CHECK-NEXT:    mov x18, #0 // =0x0
+; CHECK-NEXT:    mov x15, #0 // =0x0
 ; CHECK-NEXT:    ret
 
 entry:
@@ -547,7 +547,7 @@ define dso_local double @all_arg_float(double noundef %a, float noundef %b) loca
 ; DEFAULT-NEXT:    mov x6, #0 // =0x0
 ; DEFAULT-NEXT:    mov x7, #0 // =0x0
 ; DEFAULT-NEXT:    mov x8, #0 // =0x0
-; DEFAULT-NEXT:    mov x18, #0 // =0x0
+; DEFAULT-NEXT:    mov x15, #0 // =0x0
 ; DEFAULT-NEXT:    movi v1.2d, #0000000000000000
 ; DEFAULT-NEXT:    movi v2.2d, #0000000000000000
 ; DEFAULT-NEXT:    movi v3.2d, #0000000000000000
@@ -570,7 +570,7 @@ define dso_local double @all_arg_float(double noundef %a, float noundef %b) loca
 ; SVE-OR-SME-NEXT:    mov x6, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov x7, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov x8, #0 // =0x0
-; SVE-OR-SME-NEXT:    mov x18, #0 // =0x0
+; SVE-OR-SME-NEXT:    mov x15, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov z1.d, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov z2.d, #0 // =0x0
 ; SVE-OR-SME-NEXT:    mov z3.d, #0 // =0x0
@@ -597,7 +597,7 @@ define dso_local double @all_arg_float(double noundef %a, float noundef %b) loca
 ; STREAMING-COMPAT-NEXT:    mov x6, #0 // =0x0
 ; STREAMING-COMPAT-NEXT:    mov x7, #0 // =0x0
 ; STREAMING-COMPAT-NEXT:    mov x8, #0 // =0x0
-; STREAMING-COMPAT-NEXT:    mov x18, #0 // =0x0
+; STREAMING-COMPAT-NEXT:    mov x15, #0 // =0x0
 ; STREAMING-COMPAT-NEXT:    fmov d1, xzr
 ; STREAMING-COMPAT-NEXT:    fmov d2, xzr
 ; STREAMING-COMPAT-NEXT:    fmov d3, xzr
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index 46a245948598..68a9dff81232 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1245,54 +1245,48 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    ldr x9, [x0, x8]
 ; CHECK-NEXT:    add x8, x8, #16
 ; CHECK-NEXT:    cmp x8, #128
-; CHECK-NEXT:    and w11, w9, #0xf
-; CHECK-NEXT:    ubfx w10, w9, #4, #4
-; CHECK-NEXT:    fmov s1, w11
-; CHECK-NEXT:    mov.b v1[1], w10
-; CHECK-NEXT:    ubfx w10, w9, #8, #4
-; CHECK-NEXT:    mov.b v1[2], w10
-; CHECK-NEXT:    ubfx w10, w9, #12, #4
-; CHECK-NEXT:    mov.b v1[3], w10
-; CHECK-NEXT:    ubfx w10, w9, #16, #4
-; CHECK-NEXT:    mov.b v1[4], w10
-; CHECK-NEXT:    ubfx w10, w9, #20, #4
-; CHECK-NEXT:    mov.b v1[5], w10
-; CHECK-NEXT:    ubfx w10, w9, #24, #4
-; CHECK-NEXT:    mov.b v1[6], w10
-; CHECK-NEXT:    lsr w10, w9, #28
-; CHECK-NEXT:    mov.b v1[7], w10
-; CHECK-NEXT:    ubfx x10, x9, #32, #4
-; CHECK-NEXT:    mov.b v1[8], w10
-; CHECK-NEXT:    ubfx x10, x9, #36, #4
-; CHECK-NEXT:    mov.b v1[9], w10
-; CHECK-NEXT:    ubfx x10, x9, #40, #4
-; CHECK-NEXT:    mov.b v1[10], w10
-; CHECK-NEXT:    ubfx x10, x9, #44, #4
-; CHECK-NEXT:    mov.b v1[11], w10
-; CHECK-NEXT:    ubfx x10, x9, #48, #4
-; CHECK-NEXT:    mov.b v1[12], w10
+; CHECK-NEXT:    ubfx x12, x9, #48, #4
 ; CHECK-NEXT:    ubfx x10, x9, #52, #4
-; CHECK-NEXT:    mov.b v1[13], w10
-; CHECK-NEXT:    ubfx x10, x9, #56, #4
-; CHECK-NEXT:    lsr x9, x9, #60
-; CHECK-NEXT:    mov.b v1[14], w10
-; CHECK-NEXT:    mov.b v1[15], w9
-; CHECK-NEXT:    ext.16b v2, v1, v1, #8
-; CHECK-NEXT:    zip2.8b v3, v1, v0
-; CHECK-NEXT:    zip1.8b v1, v1, v0
-; CHECK-NEXT:    zip2.8b v4, v2, v0
-; CHECK-NEXT:    zip1.8b v2, v2, v0
-; CHECK-NEXT:    ushll.4s v3, v3, #0
+; CHECK-NEXT:    ubfx x14, x9, #32, #4
+; CHECK-NEXT:    ubfx w15, w9, #16, #4
+; CHECK-NEXT:    ubfx x11, x9, #36, #4
+; CHECK-NEXT:    ubfx w13, w9, #20, #4
+; CHECK-NEXT:    fmov s1, w12
+; CHECK-NEXT:    fmov s2, w14
+; CHECK-NEXT:    ubfx w12, w9, #4, #4
+; CHECK-NEXT:    fmov s3, w15
+; CHECK-NEXT:    mov.h v1[1], w10
+; CHECK-NEXT:    and w10, w9, #0xf
+; CHECK-NEXT:    mov.h v2[1], w11
+; CHECK-NEXT:    fmov s4, w10
+; CHECK-NEXT:    ubfx x11, x9, #56, #4
+; CHECK-NEXT:    mov.h v3[1], w13
+; CHECK-NEXT:    ubfx x10, x9, #40, #4
+; CHECK-NEXT:    mov.h v4[1], w12
+; CHECK-NEXT:    ubfx w12, w9, #24, #4
+; CHECK-NEXT:    mov.h v1[2], w11
+; CHECK-NEXT:    ubfx w11, w9, #8, #4
+; CHECK-NEXT:    mov.h v2[2], w10
+; CHECK-NEXT:    lsr x10, x9, #60
+; CHECK-NEXT:    mov.h v3[2], w12
+; CHECK-NEXT:    ubfx x12, x9, #44, #4
+; CHECK-NEXT:    mov.h v4[2], w11
+; CHECK-NEXT:    lsr w11, w9, #28
+; CHECK-NEXT:    ubfx w9, w9, #12, #4
+; CHECK-NEXT:    mov.h v1[3], w10
+; CHECK-NEXT:    mov.h v2[3], w12
+; CHECK-NEXT:    mov.h v3[3], w11
+; CHECK-NEXT:    mov.h v4[3], w9
 ; CHECK-NEXT:    ushll.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v3, v3, v0
-; CHECK-NEXT:    ushll.4s v4, v4, #0
 ; CHECK-NEXT:    ushll.4s v2, v2, #0
+; CHECK-NEXT:    ushll.4s v3, v3, #0
+; CHECK-NEXT:    ushll.4s v4, v4, #0
 ; CHECK-NEXT:    and.16b v1, v1, v0
-; CHECK-NEXT:    and.16b v4, v4, v0
 ; CHECK-NEXT:    and.16b v2, v2, v0
-; CHECK-NEXT:    stp q1, q3, [x1]
-; CHECK-NEXT:    stp q2, q4, [x1, #32]
-; CHECK-NEXT:    add x1, x1, #64
+; CHECK-NEXT:    and.16b v3, v3, v0
+; CHECK-NEXT:    and.16b v4, v4, v0
+; CHECK-NEXT:    stp q2, q1, [x1, #32]
+; CHECK-NEXT:    stp q4, q3, [x1], #64
 ; CHECK-NEXT:    b.ne LBB13_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
@@ -1306,59 +1300,54 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
 ; CHECK-BE-NEXT:    ldr x9, [x0, x8]
 ; CHECK-BE-NEXT:    add x8, x8, #16
 ; CHECK-BE-NEXT:    cmp x8, #128
-; CHECK-BE-NEXT:    lsr x10, x9, #60
-; CHECK-BE-NEXT:    ubfx x11, x9, #56, #4
-; CHECK-BE-NEXT:    fmov s1, w10
-; CHECK-BE-NEXT:    ubfx x10, x9, #52, #4
-; CHECK-BE-NEXT:    mov v1.b[1], w11
-; CHECK-BE-NEXT:    mov v1.b[2], w10
-; CHECK-BE-NEXT:    ubfx x10, x9, #48, #4
-; CHECK-BE-NEXT:    mov v1.b[3], w10
-; CHECK-BE-NEXT:    ubfx x10, x9, #44, #4
-; CHECK-BE-NEXT:    mov v1.b[4], w10
-; CHECK-BE-NEXT:    ubfx x10, x9, #40, #4
-; CHECK-BE-NEXT:    mov v1.b[5], w10
-; CHECK-BE-NEXT:    ubfx x10, x9, #36, #4
-; CHECK-BE-NEXT:    mov v1.b[6], w10
-; CHECK-BE-NEXT:    ubfx x10, x9, #32, #4
-; CHECK-BE-NEXT:    mov v1.b[7], w10
-; CHECK-BE-NEXT:    lsr w10, w9, #28
-; CHECK-BE-NEXT:    mov v1.b[8], w10
-; CHECK-BE-NEXT:    ubfx w10, w9, #24, #4
-; CHECK-BE-NEXT:    mov v1.b[9], w10
-; CHECK-BE-NEXT:    ubfx w10, w9, #20, #4
-; CHECK-BE-NEXT:    mov v1.b[10], w10
-; CHECK-BE-NEXT:    ubfx w10, w9, #16, #4
-; CHECK-BE-NEXT:    mov v1.b[11], w10
-; CHECK-BE-NEXT:    ubfx w10, w9, #12, #4
-; CHECK-BE-NEXT:    mov v1.b[12], w10
+; CHECK-BE-NEXT:    ubfx w12, w9, #12, #4
+; CHECK-BE-NEXT:    lsr w14, w9, #28
 ; CHECK-BE-NEXT:    ubfx w10, w9, #8, #4
-; CHECK-BE-NEXT:    mov v1.b[13], w10
-; CHECK-BE-NEXT:    ubfx w10, w9, #4, #4
-; CHECK-BE-NEXT:    and w9, w9, #0xf
-; CHECK-BE-NEXT:    mov v1.b[14], w10
+; CHECK-BE-NEXT:    ubfx x15, x9, #44, #4
+; CHECK-BE-NEXT:    ubfx w11, w9, #24, #4
+; CHECK-BE-NEXT:    ubfx x13, x9, #40, #4
+; CHECK-BE-NEXT:    fmov s1, w12
+; CHECK-BE-NEXT:    lsr x12, x9, #60
+; CHECK-BE-NEXT:    fmov s2, w14
+; CHECK-BE-NEXT:    fmov s3, w15
+; CHECK-BE-NEXT:    fmov s4, w12
+; CHECK-BE-NEXT:    ubfx w12, w9, #20, #4
+; CHECK-BE-NEXT:    mov v1.h[1], w10
+; CHECK-BE-NEXT:    ubfx x10, x9, #56, #4
+; CHECK-BE-NEXT:    mov v2.h[1], w11
+; CHECK-BE-NEXT:    ubfx w11, w9, #4, #4
+; CHECK-BE-NEXT:    mov v3.h[1], w13
+; CHECK-BE-NEXT:    mov v4.h[1], w10
+; CHECK-BE-NEXT:    ubfx x10, x9, #36, #4
+; CHECK-BE-NEXT:    mov v1.h[2], w11
+; CHECK-BE-NEXT:    ubfx x11, x9, #52, #4
+; CHECK-BE-NEXT:    mov v2.h[2], w12
+; CHECK-BE-NEXT:    mov v3.h[2], w10
+; CHECK-BE-NEXT:    and w10, w9, #0xf
+; CHECK-BE-NEXT:    ubfx w12, w9, #16, #4
+; CHECK-BE-NEXT:    mov v4.h[2], w11
+; CHECK-BE-NEXT:    ubfx x11, x9, #32, #4
+; CHECK-BE-NEXT:    ubfx x9, x9, #48, #4
+; CHECK-BE-NEXT:    mov v1.h[3], w10
+; CHECK-BE-NEXT:    mov v2.h[3], w12
 ; CHECK-BE-NEXT:    add x10, x1, #32
-; CHECK-BE-NEXT:    mov v1.b[15], w9
-; CHECK-BE-NEXT:    add x9, x1, #16
-; CHECK-BE-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT:    zip2 v3.8b, v1.8b, v0.8b
-; CHECK-BE-NEXT:    zip1 v1.8b, v1.8b, v0.8b
-; CHECK-BE-NEXT:    zip2 v4.8b, v2.8b, v0.8b
-; CHECK-BE-NEXT:    zip1 v2.8b, v2.8b, v0.8b
-; CHECK-BE-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-BE-NEXT:    mov v3.h[3], w11
+; CHECK-BE-NEXT:    mov v4.h[3], w9
+; CHECK-BE-NEXT:    add x9, x1, #48
 ; CHECK-BE-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-BE-NEXT:    and v3.16b, v3.16b, v0.16b
-; CHECK-BE-NEXT:    ushll v4.4s, v4.4h, #0
 ; CHECK-BE-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-BE-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-BE-NEXT:    ushll v4.4s, v4.4h, #0
 ; CHECK-BE-NEXT:    and v1.16b, v1.16b, v0.16b
-; CHECK-BE-NEXT:    st1 { v3.4s }, [x9]
-; CHECK-BE-NEXT:    add x9, x1, #48
-; CHECK-BE-NEXT:    and v4.16b, v4.16b, v0.16b
 ; CHECK-BE-NEXT:    and v2.16b, v2.16b, v0.16b
-; CHECK-BE-NEXT:    st1 { v1.4s }, [x1]
-; CHECK-BE-NEXT:    add x1, x1, #64
-; CHECK-BE-NEXT:    st1 { v4.4s }, [x9]
+; CHECK-BE-NEXT:    and v3.16b, v3.16b, v0.16b
+; CHECK-BE-NEXT:    and v4.16b, v4.16b, v0.16b
+; CHECK-BE-NEXT:    st1 { v1.4s }, [x9]
+; CHECK-BE-NEXT:    add x9, x1, #16
 ; CHECK-BE-NEXT:    st1 { v2.4s }, [x10]
+; CHECK-BE-NEXT:    st1 { v4.4s }, [x1]
+; CHECK-BE-NEXT:    add x1, x1, #64
+; CHECK-BE-NEXT:    st1 { v3.4s }, [x9]
 ; CHECK-BE-NEXT:    b.ne .LBB13_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit
 ; CHECK-BE-NEXT:    ret
@@ -2913,28 +2902,28 @@ define i32 @test_widening_instr_mull_64(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-NEXT:  LBB25_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr q4, [x0]
-; CHECK-NEXT:    ldp q17, q7, [x1, #32]
+; CHECK-NEXT:    ldp q17, q16, [x1, #32]
 ; CHECK-NEXT:    ldr q18, [x1]
 ; CHECK-NEXT:    subs w2, w2, #1
 ; CHECK-NEXT:    tbl.16b v5, { v4 }, v3
 ; CHECK-NEXT:    tbl.16b v6, { v4 }, v0
-; CHECK-NEXT:    tbl.16b v16, { v4 }, v1
+; CHECK-NEXT:    tbl.16b v7, { v4 }, v1
 ; CHECK-NEXT:    tbl.16b v4, { v4 }, v2
 ; CHECK-NEXT:    ldr q21, [x8, #16]!
 ; CHECK-NEXT:    mov x1, x8
-; CHECK-NEXT:    umull2.2d v19, v5, v7
+; CHECK-NEXT:    umull2.2d v19, v5, v16
 ; CHECK-NEXT:    umull2.2d v20, v6, v17
-; CHECK-NEXT:    umull2.2d v22, v16, v18
-; CHECK-NEXT:    umull.2d v5, v5, v7
-; CHECK-NEXT:    umull2.2d v7, v4, v21
+; CHECK-NEXT:    umull2.2d v22, v7, v18
+; CHECK-NEXT:    umull.2d v5, v5, v16
+; CHECK-NEXT:    umull2.2d v16, v4, v21
 ; CHECK-NEXT:    umull.2d v4, v4, v21
-; CHECK-NEXT:    umull.2d v16, v16, v18
+; CHECK-NEXT:    umull.2d v7, v7, v18
 ; CHECK-NEXT:    umull.2d v6, v6, v17
 ; CHECK-NEXT:    str q20, [x0, #80]
 ; CHECK-NEXT:    stp q22, q4, [x0, #16]
 ; CHECK-NEXT:    stp q5, q19, [x0, #96]
-; CHECK-NEXT:    str q7, [x0, #48]
-; CHECK-NEXT:    str q16, [x0]
+; CHECK-NEXT:    str q16, [x0, #48]
+; CHECK-NEXT:    str q7, [x0]
 ; CHECK-NEXT:    str q6, [x0, #64]!
 ; CHECK-NEXT:    b.ne LBB25_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
diff --git a/external/llvm-project/llvm/test/CodeGen/AArch64/zext.ll b/external/llvm-project/llvm/test/CodeGen/AArch64/zext.ll
index 962486afa3bb..7078d9b2586a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AArch64/zext.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AArch64/zext.ll
@@ -269,10 +269,10 @@ define <3 x i32> @zext_v3i8_v3i32(<3 x i8> %a) {
 ; CHECK-GI-LABEL: zext_v3i8_v3i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    and w8, w0, #0xff
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    and w8, w1, #0xff
-; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    and w9, w1, #0xff
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    and w8, w2, #0xff
+; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -326,7 +326,7 @@ define <3 x i32> @zext_v3i16_v3i32(<3 x i16> %a) {
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    umov w8, v0.h[0]
 ; CHECK-GI-NEXT:    umov w9, v0.h[1]
-; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    fmov s1, w8
 ; CHECK-GI-NEXT:    umov w8, v0.h[2]
 ; CHECK-GI-NEXT:    mov v1.s[1], w9
 ; CHECK-GI-NEXT:    mov v1.s[2], w8
@@ -428,10 +428,10 @@ define <3 x i32> @zext_v3i10_v3i32(<3 x i10> %a) {
 ; CHECK-GI-LABEL: zext_v3i10_v3i32:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    and w8, w0, #0x3ff
-; CHECK-GI-NEXT:    mov v0.s[0], w8
-; CHECK-GI-NEXT:    and w8, w1, #0x3ff
-; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    and w9, w1, #0x3ff
+; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    and w8, w2, #0x3ff
+; CHECK-GI-NEXT:    mov v0.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[2], w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -1169,44 +1169,44 @@ define <16 x i64> @zext_v16i10_v16i64(<16 x i10> %a) {
 ;
 ; CHECK-GI-LABEL: zext_v16i10_v16i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v0.s[0], w0
-; CHECK-GI-NEXT:    mov v1.s[0], w2
-; CHECK-GI-NEXT:    ldr s3, [sp]
-; CHECK-GI-NEXT:    mov v2.s[0], w4
-; CHECK-GI-NEXT:    mov v5.s[0], w6
-; CHECK-GI-NEXT:    ldr s4, [sp, #8]
-; CHECK-GI-NEXT:    ldr s6, [sp, #16]
-; CHECK-GI-NEXT:    ldr s7, [sp, #24]
-; CHECK-GI-NEXT:    ldr s16, [sp, #32]
-; CHECK-GI-NEXT:    ldr s17, [sp, #40]
-; CHECK-GI-NEXT:    ldr s18, [sp, #48]
-; CHECK-GI-NEXT:    ldr s19, [sp, #56]
-; CHECK-GI-NEXT:    mov v0.s[1], w1
-; CHECK-GI-NEXT:    mov v1.s[1], w3
-; CHECK-GI-NEXT:    mov v3.s[1], v4.s[0]
-; CHECK-GI-NEXT:    mov v2.s[1], w5
-; CHECK-GI-NEXT:    mov v5.s[1], w7
+; CHECK-GI-NEXT:    fmov s16, w0
+; CHECK-GI-NEXT:    fmov s17, w2
+; CHECK-GI-NEXT:    ldr s1, [sp]
+; CHECK-GI-NEXT:    fmov s18, w4
+; CHECK-GI-NEXT:    fmov s19, w6
+; CHECK-GI-NEXT:    ldr s3, [sp, #8]
+; CHECK-GI-NEXT:    ldr s0, [sp, #16]
+; CHECK-GI-NEXT:    ldr s4, [sp, #24]
+; CHECK-GI-NEXT:    ldr s2, [sp, #32]
+; CHECK-GI-NEXT:    ldr s5, [sp, #40]
+; CHECK-GI-NEXT:    ldr s6, [sp, #48]
+; CHECK-GI-NEXT:    ldr s7, [sp, #56]
+; CHECK-GI-NEXT:    mov v16.s[1], w1
+; CHECK-GI-NEXT:    mov v17.s[1], w3
+; CHECK-GI-NEXT:    mov v18.s[1], w5
+; CHECK-GI-NEXT:    mov v19.s[1], w7
+; CHECK-GI-NEXT:    mov v1.s[1], v3.s[0]
+; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v5.s[0]
 ; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT:    mov v16.s[1], v17.s[0]
-; CHECK-GI-NEXT:    mov v18.s[1], v19.s[0]
 ; CHECK-GI-NEXT:    adrp x8, .LCPI54_0
-; CHECK-GI-NEXT:    ldr q7, [x8, :lo12:.LCPI54_0]
-; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll v4.2d, v5.2s, #0
-; CHECK-GI-NEXT:    ushll v5.2d, v3.2s, #0
-; CHECK-GI-NEXT:    ushll v6.2d, v6.2s, #0
-; CHECK-GI-NEXT:    ushll v16.2d, v16.2s, #0
-; CHECK-GI-NEXT:    ushll v17.2d, v18.2s, #0
-; CHECK-GI-NEXT:    and v0.16b, v0.16b, v7.16b
-; CHECK-GI-NEXT:    and v1.16b, v1.16b, v7.16b
-; CHECK-GI-NEXT:    and v2.16b, v2.16b, v7.16b
-; CHECK-GI-NEXT:    and v3.16b, v4.16b, v7.16b
-; CHECK-GI-NEXT:    and v4.16b, v5.16b, v7.16b
-; CHECK-GI-NEXT:    and v5.16b, v6.16b, v7.16b
-; CHECK-GI-NEXT:    and v6.16b, v16.16b, v7.16b
-; CHECK-GI-NEXT:    and v7.16b, v17.16b, v7.16b
+; CHECK-GI-NEXT:    ushll v3.2d, v16.2s, #0
+; CHECK-GI-NEXT:    ushll v4.2d, v17.2s, #0
+; CHECK-GI-NEXT:    ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT:    ushll v7.2d, v19.2s, #0
+; CHECK-GI-NEXT:    ushll v16.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ushll v18.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v19.2d, v2.2s, #0
+; CHECK-GI-NEXT:    ushll v20.2d, v6.2s, #0
+; CHECK-GI-NEXT:    ldr q17, [x8, :lo12:.LCPI54_0]
+; CHECK-GI-NEXT:    and v0.16b, v3.16b, v17.16b
+; CHECK-GI-NEXT:    and v1.16b, v4.16b, v17.16b
+; CHECK-GI-NEXT:    and v2.16b, v5.16b, v17.16b
+; CHECK-GI-NEXT:    and v3.16b, v7.16b, v17.16b
+; CHECK-GI-NEXT:    and v4.16b, v16.16b, v17.16b
+; CHECK-GI-NEXT:    and v5.16b, v18.16b, v17.16b
+; CHECK-GI-NEXT:    and v6.16b, v19.16b, v17.16b
+; CHECK-GI-NEXT:    and v7.16b, v20.16b, v17.16b
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = zext <16 x i10> %a to <16 x i64>
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
index 27b93872b9f1..b67080bd4798 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
@@ -40,17 +40,17 @@ define void @add_v3i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_ushort v6, v[0:1], off
-; GFX9-NEXT:    global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_ushort v8, v[2:3], off
-; GFX9-NEXT:    global_load_ushort v9, v[2:3], off offset:4
+; GFX9-NEXT:    global_load_ushort v7, v[2:3], off
+; GFX9-NEXT:    global_load_ushort v8, v[2:3], off offset:4
+; GFX9-NEXT:    global_load_ushort v9, v[0:1], off offset:4
 ; GFX9-NEXT:    global_load_ushort v10, v[0:1], off offset:2
 ; GFX9-NEXT:    global_load_ushort v11, v[2:3], off offset:2
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v8
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v7
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_pk_add_u16 v2, v7, v9
+; GFX9-NEXT:    v_pk_add_u16 v2, v9, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v10, 16, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -206,10 +206,10 @@ define void @add_v5i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_ushort v6, v[0:1], off
 ; GFX9-NEXT:    global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT:    global_load_ushort v8, v[0:1], off offset:8
-; GFX9-NEXT:    global_load_ushort v9, v[2:3], off
-; GFX9-NEXT:    global_load_ushort v10, v[2:3], off offset:4
-; GFX9-NEXT:    global_load_ushort v11, v[2:3], off offset:8
+; GFX9-NEXT:    global_load_ushort v8, v[2:3], off
+; GFX9-NEXT:    global_load_ushort v9, v[2:3], off offset:4
+; GFX9-NEXT:    global_load_ushort v10, v[2:3], off offset:8
+; GFX9-NEXT:    global_load_ushort v11, v[0:1], off offset:8
 ; GFX9-NEXT:    global_load_ushort v12, v[0:1], off offset:2
 ; GFX9-NEXT:    global_load_ushort v13, v[0:1], off offset:6
 ; GFX9-NEXT:    global_load_ushort v14, v[2:3], off offset:2
@@ -218,12 +218,12 @@ define void @add_v5i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v9
-; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_and_b32_e32 v3, 0xffff, v10
+; GFX9-NEXT:    v_and_b32_e32 v3, 0xffff, v9
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_pk_add_u16 v6, v8, v11
+; GFX9-NEXT:    v_pk_add_u16 v6, v11, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v12, 16, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
@@ -421,11 +421,11 @@ define void @addv_7i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
 ; GFX9-NEXT:    global_load_ushort v6, v[0:1], off
 ; GFX9-NEXT:    global_load_ushort v7, v[0:1], off offset:4
 ; GFX9-NEXT:    global_load_ushort v8, v[0:1], off offset:8
-; GFX9-NEXT:    global_load_ushort v9, v[0:1], off offset:12
-; GFX9-NEXT:    global_load_ushort v10, v[2:3], off
-; GFX9-NEXT:    global_load_ushort v11, v[2:3], off offset:4
-; GFX9-NEXT:    global_load_ushort v12, v[2:3], off offset:8
-; GFX9-NEXT:    global_load_ushort v13, v[2:3], off offset:12
+; GFX9-NEXT:    global_load_ushort v9, v[2:3], off
+; GFX9-NEXT:    global_load_ushort v10, v[2:3], off offset:4
+; GFX9-NEXT:    global_load_ushort v11, v[2:3], off offset:8
+; GFX9-NEXT:    global_load_ushort v12, v[2:3], off offset:12
+; GFX9-NEXT:    global_load_ushort v13, v[0:1], off offset:12
 ; GFX9-NEXT:    global_load_ushort v14, v[0:1], off offset:2
 ; GFX9-NEXT:    global_load_ushort v15, v[0:1], off offset:6
 ; GFX9-NEXT:    global_load_ushort v16, v[0:1], off offset:10
@@ -438,14 +438,14 @@ define void @addv_7i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v7
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
 ; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_and_b32_e32 v3, 0xffff, v9
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_and_b32_e32 v3, 0xffff, v10
+; GFX9-NEXT:    v_and_b32_e32 v6, 0xffff, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_and_b32_e32 v6, 0xffff, v11
-; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_and_b32_e32 v7, 0xffff, v12
+; GFX9-NEXT:    v_and_b32_e32 v7, 0xffff, v11
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_pk_add_u16 v8, v9, v13
+; GFX9-NEXT:    v_pk_add_u16 v8, v13, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v14, 16, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
@@ -720,8 +720,8 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
 ; GFX9-NEXT:    global_load_ushort v14, v[0:1], off offset:16
 ; GFX9-NEXT:    global_load_ushort v15, v[2:3], off offset:16
 ; GFX9-NEXT:    global_load_dwordx4 v[10:13], v[2:3], off
-; GFX9-NEXT:    global_load_ushort v16, v[0:1], off offset:20
-; GFX9-NEXT:    global_load_ushort v17, v[2:3], off offset:20
+; GFX9-NEXT:    global_load_ushort v16, v[2:3], off offset:20
+; GFX9-NEXT:    global_load_ushort v17, v[0:1], off offset:20
 ; GFX9-NEXT:    global_load_ushort v18, v[0:1], off offset:18
 ; GFX9-NEXT:    global_load_ushort v19, v[2:3], off offset:18
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
@@ -738,7 +738,7 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_lshl_or_b32 v8, v19, 16, v15
 ; GFX9-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
-; GFX9-NEXT:    v_pk_add_u16 v6, v16, v17
+; GFX9-NEXT:    v_pk_add_u16 v6, v17, v16
 ; GFX9-NEXT:    v_pk_add_u16 v0, v7, v8
 ; GFX9-NEXT:    global_store_short v[4:5], v0, off offset:16
 ; GFX9-NEXT:    global_store_short_d16_hi v[4:5], v0, off offset:18
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/knownbits-ptrtoint.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/knownbits-ptrtoint.mir
new file mode 100644
index 000000000000..4073568fd421
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/knownbits-ptrtoint.mir
@@ -0,0 +1,110 @@
+# NOTE: Assertions have been autogenerated by utils/update_givaluetracking_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -passes="print<gisel-value-tracking>" %s -filetype=null 2>&1 | FileCheck %s
+## Check that we don't incorrectly assume known zeroes for and extend of a truncated ptrtoint
+## Test case for https://github.com/llvm/llvm-project/issues/139598
+---
+## We should see 128 unknown bits.
+name:            PtrToInt
+body:             |
+  bb.0:
+  ; CHECK-LABEL: name: @PtrToInt
+  ; CHECK-NEXT: %0:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %1:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %2:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %3:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %4:_ KnownBits:???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %5:_ KnownBits:???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:1
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p8) = G_MERGE_VALUES %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s128) = G_PTRTOINT %4(p8)
+...
+---
+## We should see 128 high zeroes followed by 128 unknown bits for extending ptrtoint.
+name:            PtrToIntExt
+body:             |
+  bb.0:
+  ; CHECK-LABEL: name: @PtrToIntExt
+  ; CHECK-NEXT: %0:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %1:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %2:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %3:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %4:_ KnownBits:???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %5:_ KnownBits:00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:128
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p8) = G_MERGE_VALUES %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s256) = G_PTRTOINT %4(p8)
+...
+---
+## We should see 48 unknown bits for truncating ptrtoint.
+name:            PtrToIntTrunc
+body:             |
+  bb.0:
+  ; CHECK-LABEL: name: @PtrToIntTrunc
+  ; CHECK-NEXT: %0:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %1:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %2:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %3:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %4:_ KnownBits:???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %5:_ KnownBits:???????????????????????????????????????????????? SignBits:1
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p8) = G_MERGE_VALUES %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s48) = G_PTRTOINT %4(p8)
+...
+---
+## This is the test for issue 139598: Truncating and then extending the
+## G_PTRTOINT result was filling all bits above the index bitwidth with known
+## zeroes even though the incoming value is completely unknown and G_PTRTOINT.
+## is lowered to a bitwise copy.
+## We should see all zero high bits with 48 unknown bits.
+name:            PtrToIntTruncExplicitExt
+body:             |
+  bb.0:
+  ; CHECK-LABEL: name: @PtrToIntTruncExplicitExt
+  ; CHECK-NEXT: %0:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %1:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %2:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %3:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %4:_ KnownBits:???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %5:_ KnownBits:???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %6:_ KnownBits:???????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %7:_ KnownBits:0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000???????????????????????????????????????????????? SignBits:208
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p8) = G_MERGE_VALUES %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s128) = G_PTRTOINT %4(p8)
+    %6:_(s48) = G_TRUNC %5(s128)
+    %7:_(s256) = G_ZEXT %6(s48)
+...
+---
+## Same test again but this time have the G_PTRTOINT do the truncation.
+## We should see all zero high bits with 48 unknown bits.
+name:            PtrToIntTruncImplicitExt
+body:             |
+  bb.0:
+  ; CHECK-LABEL: name: @PtrToIntTruncImplicitExt
+  ; CHECK-NEXT: %0:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %1:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %2:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %3:_ KnownBits:???????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %4:_ KnownBits:???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %5:_ KnownBits:???????????????????????????????????????????????? SignBits:1
+  ; CHECK-NEXT: %6:_ KnownBits:0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000???????????????????????????????????????????????? SignBits:208
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p8) = G_MERGE_VALUES %0(s32), %1(s32), %2(s32), %3(s32)
+    %5:_(s48) = G_PTRTOINT %4(p8)
+    %6:_(s256) = G_ZEXT %5(s48)
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/shufflevector-pointer-crash.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/shufflevector-pointer-crash.mir
new file mode 100644
index 000000000000..ac903adba9a4
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/shufflevector-pointer-crash.mir
@@ -0,0 +1,36 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -march=amdgcn -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            test
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: test
+    ; CHECK: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[C]](p0) :: (load (s32), align 16)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 4
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[C1]](p0) :: (load (s32) from unknown-address + 4)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(p0) = G_CONSTANT i64 8
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[C2]](p0) :: (load (s32) from unknown-address + 8, align 8)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(p0) = G_CONSTANT i64 12
+    ; CHECK-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[C3]](p0) :: (load (s32) from unknown-address + 12)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
+    ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p0>) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[BITCAST]](<2 x p0>)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY [[UV]](p0)
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](p0)
+    ; CHECK-NEXT: $vgpr0 = COPY [[UV2]](s32)
+    ; CHECK-NEXT: $vgpr1 = COPY [[UV3]](s32)
+    ; CHECK-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+    %0:_(p0) = G_CONSTANT i64 0
+    %1:_(<2 x p0>) = G_BUILD_VECTOR %0:_(p0), %0:_(p0)
+    %2:_(<2 x p0>) = G_LOAD %0:_(p0) :: (load (<2 x p0>))
+    %3:_(p0) = G_SHUFFLE_VECTOR %2:_(<2 x p0>), %1:_, shufflemask(0)
+    %4:_(s32), %5:_(s32) = G_UNMERGE_VALUES %3:_(p0)
+    $vgpr0 = COPY %4:_(s32)
+    $vgpr1 = COPY %5:_(s32)
+    SI_RETURN implicit $vgpr0, implicit $vgpr1
+...
+
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
index cc2c80060231..78766b4e8eb0 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
@@ -276,3 +276,28 @@ define void @kernel_argument_promotion_pattern_inter_procedure(ptr %p, i32 %val)
   call void @use_argument_after_promotion(ptr %p.cast.1, i32 %val)
   ret void
 }
+
+define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, ptr addrspace(3) %p3, i32 %val) {
+; CHECK-LABEL: define amdgpu_kernel void @kernel_argument_with_known_as(
+; CHECK-SAME: ptr addrspace(1) [[P1:%.*]], ptr addrspace(3) [[P3:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[P1_CAST:%.*]] = addrspacecast ptr addrspace(1) [[P1]] to ptr
+; CHECK-NEXT:    [[P3_CAST:%.*]] = addrspacecast ptr addrspace(3) [[P3]] to ptr
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[B]], ptr [[P1_CAST]], ptr [[P3_CAST]]
+; CHECK-NEXT:    [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]], !amdgpu.no.remote.memory [[META1]]
+; CHECK-NEXT:    ret void
+;
+  %p1.cast = addrspacecast ptr addrspace(1) %p1 to ptr
+  %p3.cast = addrspacecast ptr addrspace(3) %p3 to ptr
+  %b = icmp eq i32 %val, 0
+  %p = select i1 %b, ptr %p1.cast, ptr %p3.cast
+  %atomic.add = atomicrmw add ptr %p, i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace !0, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1
+  ret void
+}
+
+!0 = !{i32 5, i32 6}
+!1 = !{}
+;.
+; CHECK: [[META0]] = !{i32 5, i32 6}
+; CHECK: [[META1]] = !{}
+;.
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 3b6900aa6478..8c7a46232642 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -3121,8 +3121,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; kill: killed $vgpr36
@@ -3284,7 +3284,7 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; kill: killed $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -3522,7 +3522,6 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB12_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
@@ -4334,8 +4333,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    ; kill: killed $vgpr39
@@ -4990,13 +4989,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 40, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 44, v0
@@ -5016,13 +5016,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 52, v0
@@ -5042,13 +5043,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 56, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v16, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 60, v0
@@ -5068,13 +5070,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 64, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v18, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x44, v0
@@ -5094,13 +5097,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x4c, v0
@@ -5119,13 +5123,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x50, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x54, v0
@@ -5145,13 +5150,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x58, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x5c, v0
@@ -5171,13 +5177,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x60, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x64, v0
@@ -5195,13 +5202,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x68, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x6c, v0
@@ -5221,13 +5229,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x70, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x74, v0
@@ -5293,8 +5302,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    ; implicit-def: $vgpr40
 ; GFX9-NEXT:    ; kill: killed $vgpr40
@@ -5484,7 +5493,7 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(30)
+; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    ; kill: killed $vgpr33
@@ -5493,7 +5502,6 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB12_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
-; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 24, v32
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
@@ -5689,7 +5697,6 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB12_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
-; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_add_u32_e32 v32, 3, v32
 ; GFX9-NEXT:    s_waitcnt vmcnt(28)
 ; GFX9-NEXT:    v_add_u32_e32 v31, 3, v31
@@ -6106,13 +6113,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
@@ -6131,13 +6139,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
@@ -6156,13 +6165,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
@@ -6181,13 +6191,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
@@ -6206,13 +6217,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
@@ -6231,13 +6243,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
@@ -6256,13 +6269,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
@@ -12466,10 +12480,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:388
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
@@ -12489,33 +12503,35 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:136
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:144
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:152
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:160
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:168
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:176
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:184
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v1
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:136
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:144
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:152
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:160
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:168
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:176
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:184
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 8, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 8, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 24, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 8, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 24, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v6
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -12536,10 +12552,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v18
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v20
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v22
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -12550,69 +12566,69 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v28
-; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:132
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:140
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v31
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:140
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v33
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v36
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v35
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v36
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v39
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:156
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:164
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:172
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:180
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v40
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v50
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v51
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:192
@@ -12627,14 +12643,14 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:196
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -12656,16 +12672,16 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -12684,19 +12700,19 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -12715,10 +12731,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:284
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -12787,7 +12803,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:356
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -12805,121 +12821,119 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:92
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:68
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v43
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v0, v0, v54
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
-; SI-NEXT:    v_or_b32_e32 v3, v53, v3
+; SI-NEXT:    v_or_b32_e32 v3, v40, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v35
+; SI-NEXT:    v_or_b32_e32 v2, v2, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v34, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
+; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v48
@@ -12929,11 +12943,11 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v61
+; SI-NEXT:    v_or_b32_e32 v4, v4, v62
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v38
@@ -12943,18 +12957,18 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v6, v58
+; SI-NEXT:    v_or_b32_e32 v6, v6, v56
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v46
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v40
-; SI-NEXT:    v_or_b32_e32 v8, v8, v50
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; SI-NEXT:    v_or_b32_e32 v8, v8, v44
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v33
@@ -12965,24 +12979,24 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v62
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v63
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v63
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v60
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v59
@@ -12991,86 +13005,85 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v57
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v51
 ; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v56
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v47
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v55
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v50
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v44
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v45
+; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v19
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
@@ -13083,7 +13096,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v20
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
@@ -13104,7 +13117,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13117,7 +13130,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v22
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13130,7 +13143,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v23
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v23, 0xff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13143,7 +13156,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v24
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13156,7 +13169,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13169,7 +13182,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13239,7 +13252,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
-; SI-NEXT:    v_or_b32_e32 v31, v31, v36
+; SI-NEXT:    v_or_b32_e32 v31, v31, v43
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
 ; SI-NEXT:    v_or_b32_e32 v32, v33, v32
@@ -13253,249 +13266,249 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:  .LBB14_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_movk_i32 s6, 0x300
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_or_b32_e32 v0, v43, v0
+; SI-NEXT:    v_or_b32_e32 v0, v54, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
+; SI-NEXT:    v_or_b32_e32 v2, v40, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
@@ -13504,16 +13517,18 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
-; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
+; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
@@ -13521,7 +13536,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v41, v1
@@ -13534,11 +13549,11 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v2, v35, v2
+; SI-NEXT:    v_or_b32_e32 v2, v53, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -13556,7 +13571,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v61, v4
+; SI-NEXT:    v_or_b32_e32 v4, v62, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
@@ -13574,7 +13589,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    v_or_b32_e32 v6, v56, v6
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
@@ -13587,9 +13602,9 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v40
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v35
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v50, v8
+; SI-NEXT:    v_or_b32_e32 v8, v44, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
@@ -13604,7 +13619,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v62
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v63
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
@@ -13612,7 +13627,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v63
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v39
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
@@ -13620,7 +13635,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v60
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13633,12 +13648,12 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v57
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v56
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
@@ -13651,7 +13666,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v55
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v50
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
@@ -13659,19 +13674,17 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v44
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v36
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
@@ -13681,7 +13694,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13690,7 +13703,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v15
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -13698,7 +13711,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13707,7 +13720,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -13715,7 +13728,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13724,7 +13737,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v17
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -13732,7 +13745,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13749,7 +13762,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v20, v19
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -13776,7 +13789,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v21, v20
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -13793,7 +13806,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v21
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
@@ -13810,7 +13823,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
@@ -13827,7 +13840,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, s7, v23
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
@@ -13844,7 +13857,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s7, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
@@ -13861,7 +13874,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s7, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
@@ -13953,7 +13966,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
-; SI-NEXT:    v_or_b32_e32 v31, v36, v31
+; SI-NEXT:    v_or_b32_e32 v31, v43, v31
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x300, v31
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
@@ -14034,19 +14047,18 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -14078,59 +14090,61 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
@@ -14139,25 +14153,25 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -14165,25 +14179,25 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -14191,25 +14205,25 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -14217,10 +14231,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -14232,10 +14246,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -14246,22 +14260,22 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -14272,14 +14286,14 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -14291,34 +14305,33 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB14_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -14326,19 +14339,19 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -14349,23 +14362,23 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr61
-; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
 ; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr38
@@ -14406,147 +14419,217 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; kill: killed $vgpr33
 ; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -14556,16 +14639,16 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -14574,7 +14657,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -14583,253 +14666,190 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr33
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:  .LBB14_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB14_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(13)
+; VI-NEXT:    v_add_u16_e32 v9, 3, v61
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v31, 0x300
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v61
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v1, 3, v1
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; VI-NEXT:    v_add_u16_sdwa v1, v1, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -14838,31 +14858,25 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
 ; VI-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v4, v4, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v5, 3, v5
 ; VI-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(13)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
 ; VI-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(11)
+; VI-NEXT:    s_waitcnt vmcnt(10)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
 ; VI-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
-; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -14899,13 +14913,13 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v63
+; VI-NEXT:    v_add_u16_e32 v8, 3, v62
 ; VI-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v59
+; VI-NEXT:    v_add_u16_e32 v9, 3, v32
 ; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v10, 3, v62
+; VI-NEXT:    v_add_u16_e32 v10, 3, v60
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
@@ -14913,27 +14927,28 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
 ; VI-NEXT:    v_add_u16_e32 v10, 3, v58
 ; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v11, 3, v60
+; VI-NEXT:    v_add_u16_e32 v11, 3, v56
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v57
+; VI-NEXT:    v_add_u16_e32 v11, 3, v59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v12, 3, v56
+; VI-NEXT:    v_add_u16_e32 v12, 3, v57
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v46
+; VI-NEXT:    v_add_u16_e32 v12, 3, v47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_add_u16_e32 v13, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -14944,148 +14959,149 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 3, v44
 ; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v54
+; VI-NEXT:    v_add_u16_e32 v14, 3, v42
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v15, 3, v40
-; VI-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_add_u16_sdwa v15, v15, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v15
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v15, v15, v16
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v17, v17, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v16, v16, v17
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v18, v18, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v17, v17, v18
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v19, v19, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v18, v18, v19
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v20, v20, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v19, v19, v20
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v21, v21, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v20, v20, v21
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v22, v22, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v21, v21, v22
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v23, v23, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v22, v22, v23
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v24, v24, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v23, v23, v24
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
 ; VI-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v25, v25, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v24, v24, v25
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -15099,19 +15115,19 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v26, v26, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v26, 3, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
 ; VI-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v27, v27, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v26, v26, v27
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -15125,20 +15141,20 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v28, v28, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v27, v27, v28
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v28, 3, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v29, v29, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v28, v28, v29
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -15151,7 +15167,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v29, v29, v30
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v30, 3, v30
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -15164,7 +15180,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v32, v32, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v30, v30, v32
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v32, 3, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -15173,7 +15189,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v32, 0x300, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v33, 3, v33
-; VI-NEXT:    v_or_b32_sdwa v33, v34, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v33, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v31, v33, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v31, v32, v31
 ; VI-NEXT:  .LBB14_4: ; %end
@@ -15250,19 +15266,18 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -15274,93 +15289,95 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v35, 8, v29
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v10
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v14
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; GFX9-NEXT:    s_nop 0
@@ -15370,25 +15387,25 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -15397,25 +15414,25 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -15424,25 +15441,25 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -15451,10 +15468,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -15466,10 +15483,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -15481,22 +15498,22 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -15508,14 +15525,14 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -15527,34 +15544,33 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB14_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -15562,20 +15578,20 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -15586,23 +15602,23 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr61
-; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr38
@@ -15643,147 +15659,217 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; kill: killed $vgpr33
 ; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -15793,16 +15879,16 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -15811,7 +15897,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -15820,258 +15906,196 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr33
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:  .LBB14_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB14_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(33)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(30)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(28)
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
+; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    s_waitcnt vmcnt(27)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
-; GFX9-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(23)
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; GFX9-NEXT:    v_add_u16_sdwa v1, v1, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -16080,32 +16104,26 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
 ; GFX9-NEXT:    v_add_u16_e32 v4, 3, v4
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v4, v4, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_add_u16_e32 v5, 3, v5
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -16142,41 +16160,41 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v63
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v62
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v59
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v32
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v60
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
 ; GFX9-NEXT:    v_add_u16_e32 v10, 3, v58
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v60
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v56
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v57
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v56
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v57
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v46
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v46
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -16187,148 +16205,149 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 3, v44
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v54
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v40
-; GFX9-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v17, v17, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v16, v16, v17
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v18, v18, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v17, v17, v18
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v19, v19, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v18, v18, v19
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v20, v20, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v19, v19, v20
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v21, v21, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v20, v20, v21
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v22, v22, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v21, v21, v22
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v23, v23, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v22, v22, v23
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v24, v24, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v23, v23, v24
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v25, v25, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v24, v24, v25
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -16342,19 +16361,19 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v26, v26, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v25, v25, v26
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v26, 3, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v27, v27, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v26, v26, v27
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -16368,20 +16387,20 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v28, v28, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v27, v27, v28
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v28, 3, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v29, v29, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v28, v28, v29
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -16394,7 +16413,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v30, v30, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v29, v29, v30
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v30, 3, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -16407,7 +16426,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v31, v31, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v30, v30, v31
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v31, 3, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -16416,7 +16435,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v31, 0x300, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v32, 3, v32
-; GFX9-NEXT:    v_or_b32_sdwa v32, v33, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v32, v63, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v32, v32, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX9-NEXT:  .LBB14_4: ; %end
@@ -19196,8 +19215,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    s_cbranch_vccnz .LBB15_5
 ; SI-NEXT:  ; %bb.4: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_add_i32 s28, s28, 3
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v40
@@ -19252,7 +19271,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
@@ -19374,12 +19393,12 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 0x3000000, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
@@ -20294,8 +20313,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    s_cbranch_vccnz .LBB15_5
 ; VI-NEXT:  ; %bb.4: ; %cmp.true
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_add_i32 s28, s28, 3
 ; VI-NEXT:    s_and_b32 s4, s28, 0xff
 ; VI-NEXT:    s_lshl_b32 s5, s29, 8
@@ -20347,7 +20366,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    s_add_i32 s6, s6, 0x3000000
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x300, v1
@@ -20442,11 +20461,11 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -20569,11 +20588,11 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -21358,11 +21377,11 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -21574,11 +21593,11 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -21762,134 +21781,119 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:324
 ; GFX11-TRUE16-NEXT:    ; meta instruction
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:320
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v86, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:312
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:308
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:304
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:300
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:264
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:260
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:256
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:252
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:248
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:240
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:200
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:192
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:312
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:308
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:304
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:300
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:296
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:260
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:252
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:248
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:244
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:236
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:208
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:200
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:184
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:176
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:168
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:160
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:152
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:144
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:136
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:132
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:128
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:116
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:184
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:176
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:168
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:160
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:152
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:144
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:136
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v183, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v40, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v41, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v42, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v43, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v45, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v44, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v46, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v47, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v57, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v56, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v58, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v59, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v61, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v60, off, s32 offset:64
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v183.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v40.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v42.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v41.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v43.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v44.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v46.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v45.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v47.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v56.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v57.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v58.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v59.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v60.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v62.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v61.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v63.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v72.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v74.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v73.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v75.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v76.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v78.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v77.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v79.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v88.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v90.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v89.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v91.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v92.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v93.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v62, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v63, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v73, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v72, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v74, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v75, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v76, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v77, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v78, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v79, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v89, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v88, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v90, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v91, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v93, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v92, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v30 :: v_dual_mov_b32 v33, v29
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v28 :: v_dual_mov_b32 v34, v27
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v36, v26 :: v_dual_mov_b32 v37, v25
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v24 :: v_dual_mov_b32 v38, v23
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v22 :: v_dual_mov_b32 v49, v21
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v20 :: v_dual_mov_b32 v51, v19
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v18 :: v_dual_mov_b32 v53, v17
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v55, v16 :: v_dual_mov_b32 v54, v15
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v64, v14 :: v_dual_mov_b32 v65, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v67, v12 :: v_dual_mov_b32 v66, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v68, v10 :: v_dual_mov_b32 v69, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v71, v8 :: v_dual_mov_b32 v70, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v80, v6 :: v_dual_mov_b32 v81, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v83, v4 :: v_dual_mov_b32 v82, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v84, v2 :: v_dual_mov_b32 v85, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB15_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v91
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v89
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v88
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v79
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v77
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v78
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v76
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v69
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
@@ -21902,305 +21906,305 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s5, s6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v75
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v73
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v66
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v2, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v72
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v65
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s1, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v67
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s2, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s3, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v60
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s19, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s27, 8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v63
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v61
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v54
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v55
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v56
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v49
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v59
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v58
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v51
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v44
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v37
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v47
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v45
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v39
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v33
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v43
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v34
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v180
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v91
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v183
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v32
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v93
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v176
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v79
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v179
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v177
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v90
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v88
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v89
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v164
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v75
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v167
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v166
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v78
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v77
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v76
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v160
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v63
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v163
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v161
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v72
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v59
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v151
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v60
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v61
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v144
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v47
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v147
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v145
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v56
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v43
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v135
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v133
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v44
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v45
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v183
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v131
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v130
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v40
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v116
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v179
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v119
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v117
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v180
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v181
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v167
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v176
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v114
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v177
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v100
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v163
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v101
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v164
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v102
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v151
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v97
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v160
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v161
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v147
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v86
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v148
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v135
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v83
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v144
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v131
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v132
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v133
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v119
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v67
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v128
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v114
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v116
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v54
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v117
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v102
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v49
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v112
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v98
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v100
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v101
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v87
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v33
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v96
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v97
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v93
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
@@ -22287,31 +22291,31 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v86
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v91
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v88
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v84
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v81
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v80
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v89
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v78
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v71
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v77
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v70
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v76
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -22327,61 +22331,61 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v68
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v74
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v67
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v73
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v72
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v63
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v65
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v64
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v0, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v8, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v61
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v60
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v59
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v52
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v8, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v51
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v8, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v57
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v50
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v2, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v56
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v47
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v49
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v46
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v45
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v44
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v43
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v35
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v41
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -22392,63 +22396,63 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v10
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v3, v0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v182
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v11
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v92
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v180
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v179
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v91
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v90
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v89
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v13, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v177
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v88
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v176
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v167
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v78
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v13, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v77
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v13, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v165
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v76
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v164
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v163
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v74
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v162
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v14, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v161
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v72
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v160
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v151
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v63
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v61
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v15, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v149
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v60
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v59
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -22459,61 +22463,61 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v15
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v58
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v146
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v145
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v56
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v144
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v135
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v47
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v46
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v45
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v18, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v44
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v132
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v43
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v42
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v18, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v41
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v18, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v129
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v40
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v128
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v182
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v181
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v19, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v180
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v116
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v115
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v179
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v177
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v19, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v20, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v176
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xff, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v167
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -22524,61 +22528,61 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v20
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v166
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v102
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v165
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v101
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v164
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v100
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v163
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v162
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v98
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v161
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v23, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v160
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v96
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v87
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v151
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v150
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v23, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v149
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v23, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v148
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v146
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v82
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v144
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v80
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v71
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v135
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v133
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v25, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v131
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -22589,52 +22593,52 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v25
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v67
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v130
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v3, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v128
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v129
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v113
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v27, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v115
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v114
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v28, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v53
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v116
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v30, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v36, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v101
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v32, 3, v99
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v28, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v29, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 0x300, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v49
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v112
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v100
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v30, 0xffff, v27
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v102
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xff, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v31, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v33
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 8, v96
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xff, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v87
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v28, v27
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v37, v29
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v33, v29
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v36
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v32, v34
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v36, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v37, v34
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -23713,9 +23717,9 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -23748,7 +23752,7 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; kill: killed $vgpr31
@@ -23867,7 +23871,6 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v62
 ; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -23972,7 +23975,6 @@ define <64 x bfloat> @bitcast_v32i32_to_v64bf16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB16_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, 3, v62
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v63
@@ -25671,6 +25673,8 @@ define <32 x i32> @bitcast_v64bf16_to_v32i32(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v26, 16, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr33
@@ -25747,8 +25751,6 @@ define <32 x i32> @bitcast_v64bf16_to_v32i32(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_alignbit_b32 v5, v5, v6, 16
@@ -32152,9 +32154,9 @@ define <64 x half> @bitcast_v32i32_to_v64f16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -32187,7 +32189,7 @@ define <64 x half> @bitcast_v32i32_to_v64f16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; kill: killed $vgpr31
@@ -32260,7 +32262,6 @@ define <64 x half> @bitcast_v32i32_to_v64f16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB20_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v62
 ; SI-NEXT:    v_cvt_f32_f16_e32 v32, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -35341,9 +35342,9 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
@@ -35393,9 +35394,8 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
 ; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
@@ -36198,8 +36198,8 @@ define <64 x i16> @bitcast_v32i32_to_v64i16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -36233,7 +36233,7 @@ define <64 x i16> @bitcast_v32i32_to_v64i16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; kill: killed $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -36284,7 +36284,6 @@ define <64 x i16> @bitcast_v32i32_to_v64i16(<32 x i32> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB24_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
@@ -37400,26 +37399,25 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:132
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
@@ -37432,102 +37430,107 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v14
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v26
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v30
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v33
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:100
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v31
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:28
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v16
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB26_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v41
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -37544,17 +37547,7 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v50
-; SI-NEXT:    v_or_b32_e32 v23, v23, v32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v32
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
@@ -37584,75 +37577,71 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v48
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v35
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    v_or_b32_e32 v16, v16, v53
+; SI-NEXT:    v_or_b32_e32 v17, v17, v51
+; SI-NEXT:    v_or_b32_e32 v18, v18, v50
 ; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v43
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v55
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v53
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_or_b32_e32 v19, v19, v48
-; SI-NEXT:    v_or_b32_e32 v21, v21, v36
-; SI-NEXT:    v_or_b32_e32 v22, v22, v34
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v18, v18, v49
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_or_b32_e32 v22, v22, v23
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v54
+; SI-NEXT:    v_or_b32_e32 v19, v19, v39
+; SI-NEXT:    v_or_b32_e32 v23, v23, v24
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v52
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v39
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v35
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v37
 ; SI-NEXT:    v_or_b32_e32 v26, v26, v27
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v33
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v28
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v63
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v62
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v61
@@ -37665,13 +37654,15 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v46
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v45
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v44
-; SI-NEXT:    v_or_b32_e32 v12, v12, v42
-; SI-NEXT:    v_or_b32_e32 v13, v13, v41
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v40
-; SI-NEXT:    v_or_b32_e32 v15, v15, v54
-; SI-NEXT:    v_or_b32_e32 v20, v20, v38
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    v_or_b32_e32 v15, v15, v55
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -37685,63 +37676,79 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v29
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v16, v52
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v20, v36
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v30
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v30, v31
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v17, v51
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v21, v34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v31, v37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    v_or_b32_e32 v31, v31, v38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:  .LBB26_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB26_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v41
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -37758,69 +37765,68 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v48
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    s_mov_b32 s6, 0x30000
+; SI-NEXT:    v_or_b32_e32 v16, v53, v16
+; SI-NEXT:    v_or_b32_e32 v17, v51, v17
+; SI-NEXT:    v_or_b32_e32 v18, v50, v18
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v43
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v55
-; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v53
-; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v50
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT:    v_or_b32_e32 v22, v23, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v54
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v19, v48, v19
-; SI-NEXT:    v_or_b32_e32 v21, v36, v21
-; SI-NEXT:    v_or_b32_e32 v22, v34, v22
-; SI-NEXT:    v_or_b32_e32 v23, v32, v23
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
-; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    v_or_b32_e32 v23, v24, v23
+; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
-; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v39
+; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v49
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
-; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v37
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_or_b32_e32 v27, v28, v27
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -37837,8 +37843,7 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_or_b32_e32 v0, v63, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v62, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v61, v2
@@ -37851,28 +37856,25 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v46, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v45, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v44, v11
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_or_b32_e32 v13, v41, v13
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v40, v14
-; SI-NEXT:    v_or_b32_e32 v15, v54, v15
-; SI-NEXT:    v_or_b32_e32 v18, v49, v18
-; SI-NEXT:    v_or_b32_e32 v20, v38, v20
+; SI-NEXT:    v_or_b32_e32 v15, v55, v15
+; SI-NEXT:    v_or_b32_e32 v19, v39, v19
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v29, v28
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
@@ -37882,34 +37884,37 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, s6, v28
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v36, v20
+; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v30, v29
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v31, v30
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v51, v17
-; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v34, v21
+; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 0x30000, v30
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v37, v31
+; SI-NEXT:    v_or_b32_e32 v31, v38, v31
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x30000, v31
 ; SI-NEXT:  .LBB26_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -37929,7 +37934,7 @@ define <32 x i32> @bitcast_v64i16_to_v32i32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i16_to_v32i32:
@@ -41268,8 +41273,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; kill: killed $vgpr36
@@ -41431,7 +41436,7 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; kill: killed $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -41669,7 +41674,6 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB36_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_f32_e32 v31, 1.0, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_f32_e32 v32, 1.0, v32
@@ -42481,8 +42485,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    ; kill: killed $vgpr39
@@ -43137,13 +43141,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 40, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 44, v0
@@ -43163,13 +43168,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 52, v0
@@ -43189,13 +43195,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 56, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v16, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 60, v0
@@ -43215,13 +43222,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 64, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v18, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x44, v0
@@ -43241,13 +43249,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x4c, v0
@@ -43266,13 +43275,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x50, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x54, v0
@@ -43292,13 +43302,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x58, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x5c, v0
@@ -43318,13 +43329,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x60, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x64, v0
@@ -43342,13 +43354,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x68, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x6c, v0
@@ -43368,13 +43381,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x70, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x74, v0
@@ -43440,8 +43454,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    ; implicit-def: $vgpr40
 ; GFX9-NEXT:    ; kill: killed $vgpr40
@@ -43631,7 +43645,7 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(30)
+; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    ; kill: killed $vgpr33
@@ -43640,7 +43654,6 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB36_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
-; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 24, v32
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
@@ -43836,7 +43849,6 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB36_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
-; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_add_f32_e32 v32, 1.0, v32
 ; GFX9-NEXT:    s_waitcnt vmcnt(28)
 ; GFX9-NEXT:    v_add_f32_e32 v31, 1.0, v31
@@ -44253,13 +44265,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
@@ -44278,13 +44291,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
@@ -44303,13 +44317,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
@@ -44328,13 +44343,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
@@ -44353,13 +44369,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
@@ -44378,13 +44395,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
@@ -44403,13 +44421,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
@@ -51635,10 +51654,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:388
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
@@ -51658,33 +51677,35 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:136
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:144
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:152
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:160
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:168
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:176
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:184
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v1
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:136
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:144
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:152
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:160
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:168
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:176
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:184
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 8, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 8, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 24, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 8, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 24, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v6
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -51705,10 +51726,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v18
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v20
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v22
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -51719,69 +51740,69 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v28
-; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:132
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:140
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v31
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:140
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v33
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v36
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v35
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v36
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v39
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:156
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:164
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:172
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:180
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v40
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v50
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v51
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:192
@@ -51796,14 +51817,14 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:196
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -51825,16 +51846,16 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -51853,19 +51874,19 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -51884,10 +51905,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:284
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -51956,7 +51977,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:356
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -51974,121 +51995,119 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:92
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:68
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB38_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v43
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v0, v0, v54
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
-; SI-NEXT:    v_or_b32_e32 v3, v53, v3
+; SI-NEXT:    v_or_b32_e32 v3, v40, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v35
+; SI-NEXT:    v_or_b32_e32 v2, v2, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v34, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
+; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v48
@@ -52098,11 +52117,11 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v61
+; SI-NEXT:    v_or_b32_e32 v4, v4, v62
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v38
@@ -52112,18 +52131,18 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v6, v58
+; SI-NEXT:    v_or_b32_e32 v6, v6, v56
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v46
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v40
-; SI-NEXT:    v_or_b32_e32 v8, v8, v50
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; SI-NEXT:    v_or_b32_e32 v8, v8, v44
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v33
@@ -52134,24 +52153,24 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v62
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v63
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v63
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v60
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v59
@@ -52160,86 +52179,85 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v57
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v51
 ; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v56
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v47
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v55
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v50
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v44
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v45
+; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v19
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
@@ -52252,7 +52270,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v20
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
@@ -52273,7 +52291,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52286,7 +52304,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v22
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52299,7 +52317,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v23
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v23, 0xff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52312,7 +52330,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v24
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52325,7 +52343,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52338,7 +52356,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52408,7 +52426,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
-; SI-NEXT:    v_or_b32_e32 v31, v31, v36
+; SI-NEXT:    v_or_b32_e32 v31, v31, v43
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
 ; SI-NEXT:    v_or_b32_e32 v32, v33, v32
@@ -52422,249 +52440,249 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:  .LBB38_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB38_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_movk_i32 s6, 0x300
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_or_b32_e32 v0, v43, v0
+; SI-NEXT:    v_or_b32_e32 v0, v54, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
+; SI-NEXT:    v_or_b32_e32 v2, v40, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
@@ -52673,16 +52691,18 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
-; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
+; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
@@ -52690,7 +52710,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v41, v1
@@ -52703,11 +52723,11 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v2, v35, v2
+; SI-NEXT:    v_or_b32_e32 v2, v53, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -52725,7 +52745,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v61, v4
+; SI-NEXT:    v_or_b32_e32 v4, v62, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
@@ -52743,7 +52763,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    v_or_b32_e32 v6, v56, v6
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
@@ -52756,9 +52776,9 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v40
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v35
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v50, v8
+; SI-NEXT:    v_or_b32_e32 v8, v44, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
@@ -52773,7 +52793,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v62
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v63
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
@@ -52781,7 +52801,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v63
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v39
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
@@ -52789,7 +52809,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v60
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52802,12 +52822,12 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v57
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v56
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
@@ -52820,7 +52840,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v55
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v50
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
@@ -52828,19 +52848,17 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v44
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v36
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
@@ -52850,7 +52868,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52859,7 +52877,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v15
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -52867,7 +52885,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52876,7 +52894,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -52884,7 +52902,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52893,7 +52911,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v17
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -52901,7 +52919,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52918,7 +52936,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v20, v19
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -52945,7 +52963,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v21, v20
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -52962,7 +52980,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v21
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
@@ -52979,7 +52997,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
@@ -52996,7 +53014,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, s7, v23
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
@@ -53013,7 +53031,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s7, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
@@ -53030,7 +53048,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s7, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
@@ -53122,7 +53140,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
-; SI-NEXT:    v_or_b32_e32 v31, v36, v31
+; SI-NEXT:    v_or_b32_e32 v31, v43, v31
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x300, v31
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
@@ -53203,19 +53221,18 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -53247,59 +53264,61 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
@@ -53308,25 +53327,25 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -53334,25 +53353,25 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -53360,25 +53379,25 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -53386,10 +53405,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -53401,10 +53420,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -53415,22 +53434,22 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -53441,14 +53460,14 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -53460,34 +53479,33 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB38_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -53495,19 +53513,19 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -53518,23 +53536,23 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr61
-; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
 ; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr38
@@ -53575,147 +53593,217 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; kill: killed $vgpr33
 ; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -53725,16 +53813,16 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -53743,7 +53831,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -53752,253 +53840,190 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr33
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:  .LBB38_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB38_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(13)
+; VI-NEXT:    v_add_u16_e32 v9, 3, v61
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v31, 0x300
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v61
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v1, 3, v1
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; VI-NEXT:    v_add_u16_sdwa v1, v1, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -54007,31 +54032,25 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
 ; VI-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v4, v4, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v5, 3, v5
 ; VI-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(13)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
 ; VI-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(11)
+; VI-NEXT:    s_waitcnt vmcnt(10)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
 ; VI-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
-; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -54068,13 +54087,13 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v63
+; VI-NEXT:    v_add_u16_e32 v8, 3, v62
 ; VI-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v59
+; VI-NEXT:    v_add_u16_e32 v9, 3, v32
 ; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v10, 3, v62
+; VI-NEXT:    v_add_u16_e32 v10, 3, v60
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
@@ -54082,27 +54101,28 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
 ; VI-NEXT:    v_add_u16_e32 v10, 3, v58
 ; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v11, 3, v60
+; VI-NEXT:    v_add_u16_e32 v11, 3, v56
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v57
+; VI-NEXT:    v_add_u16_e32 v11, 3, v59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v12, 3, v56
+; VI-NEXT:    v_add_u16_e32 v12, 3, v57
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v46
+; VI-NEXT:    v_add_u16_e32 v12, 3, v47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_add_u16_e32 v13, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -54113,148 +54133,149 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 3, v44
 ; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v54
+; VI-NEXT:    v_add_u16_e32 v14, 3, v42
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v15, 3, v40
-; VI-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_add_u16_sdwa v15, v15, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v15
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v15, v15, v16
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v17, v17, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v16, v16, v17
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v18, v18, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v17, v17, v18
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v19, v19, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v18, v18, v19
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v20, v20, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v19, v19, v20
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v21, v21, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v20, v20, v21
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v22, v22, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v21, v21, v22
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v23, v23, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v22, v22, v23
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v24, v24, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v23, v23, v24
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
 ; VI-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v25, v25, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v24, v24, v25
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -54268,19 +54289,19 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v26, v26, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v26, 3, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
 ; VI-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v27, v27, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v26, v26, v27
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -54294,20 +54315,20 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v28, v28, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v27, v27, v28
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v28, 3, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v29, v29, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v28, v28, v29
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -54320,7 +54341,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v29, v29, v30
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v30, 3, v30
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -54333,7 +54354,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v32, v32, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v30, v30, v32
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v32, 3, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -54342,7 +54363,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v32, 0x300, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v33, 3, v33
-; VI-NEXT:    v_or_b32_sdwa v33, v34, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v33, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v31, v33, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v31, v32, v31
 ; VI-NEXT:  .LBB38_4: ; %end
@@ -54419,19 +54440,18 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -54443,93 +54463,95 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v35, 8, v29
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v10
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v14
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; GFX9-NEXT:    s_nop 0
@@ -54539,25 +54561,25 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -54566,25 +54588,25 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -54593,25 +54615,25 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -54620,10 +54642,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -54635,10 +54657,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -54650,22 +54672,22 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -54677,14 +54699,14 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -54696,34 +54718,33 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB38_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -54731,20 +54752,20 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -54755,23 +54776,23 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr61
-; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr38
@@ -54812,147 +54833,217 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; kill: killed $vgpr33
 ; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -54962,16 +55053,16 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -54980,7 +55071,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -54989,258 +55080,196 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr33
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:  .LBB38_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB38_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(33)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(30)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(28)
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
+; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    s_waitcnt vmcnt(27)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
-; GFX9-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(23)
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; GFX9-NEXT:    v_add_u16_sdwa v1, v1, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -55249,32 +55278,26 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
 ; GFX9-NEXT:    v_add_u16_e32 v4, 3, v4
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v4, v4, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_add_u16_e32 v5, 3, v5
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -55311,41 +55334,41 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v63
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v62
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v59
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v32
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v60
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
 ; GFX9-NEXT:    v_add_u16_e32 v10, 3, v58
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v60
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v56
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v57
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v56
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v57
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v46
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v46
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -55356,148 +55379,149 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 3, v44
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v54
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v40
-; GFX9-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v17, v17, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v16, v16, v17
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v18, v18, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v17, v17, v18
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v19, v19, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v18, v18, v19
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v20, v20, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v19, v19, v20
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v21, v21, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v20, v20, v21
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v22, v22, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v21, v21, v22
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v23, v23, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v22, v22, v23
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v24, v24, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v23, v23, v24
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v25, v25, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v24, v24, v25
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -55511,19 +55535,19 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v26, v26, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v25, v25, v26
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v26, 3, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v27, v27, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v26, v26, v27
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -55537,20 +55561,20 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v28, v28, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v27, v27, v28
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v28, 3, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v29, v29, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v28, v28, v29
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -55563,7 +55587,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v30, v30, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v29, v29, v30
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v30, 3, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -55576,7 +55600,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v31, v31, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v30, v30, v31
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v31, 3, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -55585,7 +55609,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v31, 0x300, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v32, 3, v32
-; GFX9-NEXT:    v_or_b32_sdwa v32, v33, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v32, v63, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v32, v32, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX9-NEXT:  .LBB38_4: ; %end
@@ -58365,8 +58389,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; SI-NEXT:    s_cbranch_vccnz .LBB39_5
 ; SI-NEXT:  ; %bb.4: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_add_i32 s28, s28, 3
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v40
@@ -58421,7 +58445,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
@@ -58543,12 +58567,12 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 0x3000000, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
@@ -59463,8 +59487,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    s_cbranch_vccnz .LBB39_5
 ; VI-NEXT:  ; %bb.4: ; %cmp.true
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_add_i32 s28, s28, 3
 ; VI-NEXT:    s_and_b32 s4, s28, 0xff
 ; VI-NEXT:    s_lshl_b32 s5, s29, 8
@@ -59516,7 +59540,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    s_add_i32 s6, s6, 0x3000000
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x300, v1
@@ -59611,11 +59635,11 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -59738,11 +59762,11 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -60527,11 +60551,11 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -60743,11 +60767,11 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -60931,134 +60955,119 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:324
 ; GFX11-TRUE16-NEXT:    ; meta instruction
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:320
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v86, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:312
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:308
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:304
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:300
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:264
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:260
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:256
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:252
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:248
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:240
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:200
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:192
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:312
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:308
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:304
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:300
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:296
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:260
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:252
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:248
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:244
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:236
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:208
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:200
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:184
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:176
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:168
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:160
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:152
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:144
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:136
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:132
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:128
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:116
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:184
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:176
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:168
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:160
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:152
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:144
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:136
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v183, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v40, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v41, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v42, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v43, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v45, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v44, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v46, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v47, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v57, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v56, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v58, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v59, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v61, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v60, off, s32 offset:64
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v183.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v40.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v42.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v41.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v43.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v44.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v46.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v45.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v47.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v56.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v57.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v58.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v59.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v60.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v62.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v61.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v63.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v72.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v74.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v73.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v75.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v76.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v78.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v77.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v79.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v88.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v90.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v89.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v91.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v92.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v93.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v62, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v63, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v73, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v72, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v74, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v75, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v76, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v77, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v78, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v79, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v89, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v88, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v90, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v91, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v93, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v92, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v30 :: v_dual_mov_b32 v33, v29
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v28 :: v_dual_mov_b32 v34, v27
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v36, v26 :: v_dual_mov_b32 v37, v25
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v24 :: v_dual_mov_b32 v38, v23
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v22 :: v_dual_mov_b32 v49, v21
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v20 :: v_dual_mov_b32 v51, v19
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v18 :: v_dual_mov_b32 v53, v17
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v55, v16 :: v_dual_mov_b32 v54, v15
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v64, v14 :: v_dual_mov_b32 v65, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v67, v12 :: v_dual_mov_b32 v66, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v68, v10 :: v_dual_mov_b32 v69, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v71, v8 :: v_dual_mov_b32 v70, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v80, v6 :: v_dual_mov_b32 v81, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v83, v4 :: v_dual_mov_b32 v82, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v84, v2 :: v_dual_mov_b32 v85, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB39_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v91
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v89
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v88
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v79
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v77
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v78
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v76
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v69
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
@@ -61071,305 +61080,305 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s5, s6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v75
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v73
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v66
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v2, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v72
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v65
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s1, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v67
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s2, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s3, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v60
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s19, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s27, 8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v63
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v61
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v54
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v55
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v56
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v49
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v59
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v58
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v51
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v44
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v37
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v47
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v45
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v39
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v33
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v43
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v34
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v180
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v91
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v183
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v32
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v93
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v176
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v79
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v179
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v177
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v90
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v88
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v89
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v164
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v75
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v167
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v166
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v78
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v77
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v76
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v160
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v63
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v163
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v161
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v72
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v59
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v151
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v60
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v61
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v144
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v47
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v147
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v145
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v56
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v43
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v135
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v133
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v44
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v45
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v183
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v131
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v130
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v40
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v116
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v179
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v119
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v117
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v180
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v181
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v167
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v176
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v114
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v177
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v100
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v163
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v101
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v164
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v102
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v151
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v97
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v160
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v161
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v147
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v86
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v148
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v135
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v83
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v144
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v131
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v132
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v133
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v119
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v67
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v128
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v114
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v116
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v54
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v117
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v102
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v49
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v112
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v98
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v100
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v101
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v87
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v33
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v96
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v97
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v93
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
@@ -61456,31 +61465,31 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v86
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v91
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v88
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v84
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v81
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v80
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v89
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v78
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v71
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v77
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v70
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v76
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -61496,61 +61505,61 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v68
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v74
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v67
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v73
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v72
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v63
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v65
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v64
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v0, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v8, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v61
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v60
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v59
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v52
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v8, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v51
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v8, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v57
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v50
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v2, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v56
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v47
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v49
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v46
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v45
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v44
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v43
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v35
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v41
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -61561,63 +61570,63 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v10
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v3, v0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v182
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v11
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v92
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v180
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v179
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v91
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v90
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v89
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v13, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v177
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v88
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v176
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v167
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v78
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v13, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v77
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v13, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v165
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v76
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v164
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v163
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v74
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v162
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v14, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v161
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v72
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v160
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v151
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v63
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v61
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v15, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v149
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v60
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v59
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -61628,61 +61637,61 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v15
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v58
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v146
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v145
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v56
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v144
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v135
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v47
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v46
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v45
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v18, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v44
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v132
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v43
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v42
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v18, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v41
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v18, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v129
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v40
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v128
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v182
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v181
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v19, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v180
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v116
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v115
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v179
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v177
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v19, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v20, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v176
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xff, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v167
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -61693,61 +61702,61 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v20
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v166
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v102
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v165
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v101
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v164
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v100
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v163
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v162
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v98
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v161
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v23, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v160
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v96
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v87
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v151
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v150
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v23, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v149
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v23, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v148
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v146
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v82
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v144
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v80
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v71
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v135
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v133
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v25, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v131
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -61758,52 +61767,52 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v25
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v67
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v130
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v3, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v128
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v129
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v113
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v27, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v115
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v114
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v28, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v53
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v116
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v30, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v36, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v101
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v32, 3, v99
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v28, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v29, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 0x300, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v49
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v112
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v100
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v30, 0xffff, v27
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v102
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xff, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v31, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v33
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 8, v96
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xff, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v87
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v28, v27
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v37, v29
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v33, v29
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v36
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v32, v34
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v36, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v37, v34
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -62882,9 +62891,9 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -62917,7 +62926,7 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; kill: killed $vgpr31
@@ -63036,7 +63045,6 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v62
 ; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -63141,7 +63149,6 @@ define <64 x bfloat> @bitcast_v32f32_to_v64bf16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB40_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_f32_e32 v32, 1.0, v62
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_f32_e32 v31, 1.0, v63
@@ -64885,6 +64892,8 @@ define <32 x float> @bitcast_v64bf16_to_v32f32(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v26, 16, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr33
@@ -64961,8 +64970,6 @@ define <32 x float> @bitcast_v64bf16_to_v32f32(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_alignbit_b32 v5, v5, v6, 16
@@ -71366,9 +71373,9 @@ define <64 x half> @bitcast_v32f32_to_v64f16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -71401,7 +71408,7 @@ define <64 x half> @bitcast_v32f32_to_v64f16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; kill: killed $vgpr31
@@ -71474,7 +71481,6 @@ define <64 x half> @bitcast_v32f32_to_v64f16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB44_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v62
 ; SI-NEXT:    v_cvt_f32_f16_e32 v32, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -74529,9 +74535,9 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
@@ -74581,9 +74587,8 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
 ; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
@@ -75386,8 +75391,8 @@ define <64 x i16> @bitcast_v32f32_to_v64i16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -75421,7 +75426,7 @@ define <64 x i16> @bitcast_v32f32_to_v64i16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; kill: killed $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -75472,7 +75477,6 @@ define <64 x i16> @bitcast_v32f32_to_v64i16(<32 x float> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB48_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_f32_e32 v31, 1.0, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_f32_e32 v32, 1.0, v32
@@ -76568,26 +76572,25 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:132
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
@@ -76600,102 +76603,107 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v14
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v26
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v30
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v33
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:100
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v31
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:28
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v16
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v41
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -76712,17 +76720,7 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v50
-; SI-NEXT:    v_or_b32_e32 v23, v23, v32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v32
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
@@ -76752,75 +76750,71 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v48
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v35
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    v_or_b32_e32 v16, v16, v53
+; SI-NEXT:    v_or_b32_e32 v17, v17, v51
+; SI-NEXT:    v_or_b32_e32 v18, v18, v50
 ; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v43
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v55
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v53
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_or_b32_e32 v19, v19, v48
-; SI-NEXT:    v_or_b32_e32 v21, v21, v36
-; SI-NEXT:    v_or_b32_e32 v22, v22, v34
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v18, v18, v49
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_or_b32_e32 v22, v22, v23
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v54
+; SI-NEXT:    v_or_b32_e32 v19, v19, v39
+; SI-NEXT:    v_or_b32_e32 v23, v23, v24
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v52
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v39
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v35
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v37
 ; SI-NEXT:    v_or_b32_e32 v26, v26, v27
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v33
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v28
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v63
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v62
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v61
@@ -76833,13 +76827,15 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v46
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v45
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v44
-; SI-NEXT:    v_or_b32_e32 v12, v12, v42
-; SI-NEXT:    v_or_b32_e32 v13, v13, v41
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v40
-; SI-NEXT:    v_or_b32_e32 v15, v15, v54
-; SI-NEXT:    v_or_b32_e32 v20, v20, v38
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    v_or_b32_e32 v15, v15, v55
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -76853,63 +76849,79 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v29
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v16, v52
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v20, v36
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v30
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v30, v31
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v17, v51
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v21, v34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v31, v37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    v_or_b32_e32 v31, v31, v38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:  .LBB50_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v41
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -76926,69 +76938,68 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v48
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    s_mov_b32 s6, 0x30000
+; SI-NEXT:    v_or_b32_e32 v16, v53, v16
+; SI-NEXT:    v_or_b32_e32 v17, v51, v17
+; SI-NEXT:    v_or_b32_e32 v18, v50, v18
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v43
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v55
-; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v53
-; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v50
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT:    v_or_b32_e32 v22, v23, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v54
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v19, v48, v19
-; SI-NEXT:    v_or_b32_e32 v21, v36, v21
-; SI-NEXT:    v_or_b32_e32 v22, v34, v22
-; SI-NEXT:    v_or_b32_e32 v23, v32, v23
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
-; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    v_or_b32_e32 v23, v24, v23
+; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
-; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v39
+; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v49
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
-; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v37
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_or_b32_e32 v27, v28, v27
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -77005,8 +77016,7 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_or_b32_e32 v0, v63, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v62, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v61, v2
@@ -77019,28 +77029,25 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v46, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v45, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v44, v11
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_or_b32_e32 v13, v41, v13
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v40, v14
-; SI-NEXT:    v_or_b32_e32 v15, v54, v15
-; SI-NEXT:    v_or_b32_e32 v18, v49, v18
-; SI-NEXT:    v_or_b32_e32 v20, v38, v20
+; SI-NEXT:    v_or_b32_e32 v15, v55, v15
+; SI-NEXT:    v_or_b32_e32 v19, v39, v19
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v29, v28
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
@@ -77050,34 +77057,37 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, s6, v28
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v36, v20
+; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v30, v29
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v31, v30
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v51, v17
-; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v34, v21
+; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 0x30000, v30
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v37, v31
+; SI-NEXT:    v_or_b32_e32 v31, v38, v31
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x30000, v31
 ; SI-NEXT:  .LBB50_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -77097,7 +77107,7 @@ define <32 x float> @bitcast_v64i16_to_v32f32(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i16_to_v32f32:
@@ -79422,8 +79432,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; kill: killed $vgpr36
@@ -79585,7 +79595,7 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; kill: killed $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -80634,8 +80644,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    ; kill: killed $vgpr39
@@ -81290,13 +81300,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 40, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 44, v0
@@ -81316,13 +81327,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 52, v0
@@ -81342,13 +81354,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 56, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v16, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 60, v0
@@ -81368,13 +81381,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 64, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v18, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x44, v0
@@ -81394,13 +81408,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x4c, v0
@@ -81419,13 +81434,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x50, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x54, v0
@@ -81445,13 +81461,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x58, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x5c, v0
@@ -81471,13 +81488,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x60, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x64, v0
@@ -81495,13 +81513,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x68, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x6c, v0
@@ -81521,13 +81540,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x70, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x74, v0
@@ -81593,8 +81613,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    ; implicit-def: $vgpr40
 ; GFX9-NEXT:    ; kill: killed $vgpr40
@@ -81784,7 +81804,7 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(30)
+; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    ; kill: killed $vgpr33
@@ -81793,7 +81813,6 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB56_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
-; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 24, v32
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
@@ -82405,13 +82424,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
@@ -82430,13 +82450,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
@@ -82455,13 +82476,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
@@ -82480,13 +82502,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
@@ -82505,13 +82528,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
@@ -82530,13 +82554,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
@@ -82555,13 +82580,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
@@ -88788,10 +88814,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:388
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
@@ -88811,33 +88837,35 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:136
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:144
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:152
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:160
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:168
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:176
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:184
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v1
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:136
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:144
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:152
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:160
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:168
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:176
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:184
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 8, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 8, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 24, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 8, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 24, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v6
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -88858,10 +88886,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v18
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v20
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v22
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -88872,69 +88900,69 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v28
-; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:132
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:140
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v31
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:140
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v33
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v36
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v35
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v36
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v39
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:156
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:164
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:172
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:180
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v40
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v50
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v51
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:192
@@ -88949,14 +88977,14 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:196
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -88978,16 +89006,16 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -89006,19 +89034,19 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -89037,10 +89065,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:284
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -89109,7 +89137,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:356
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -89127,121 +89155,119 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:92
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:68
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB58_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v43
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v0, v0, v54
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
-; SI-NEXT:    v_or_b32_e32 v3, v53, v3
+; SI-NEXT:    v_or_b32_e32 v3, v40, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v35
+; SI-NEXT:    v_or_b32_e32 v2, v2, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v34, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
+; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v48
@@ -89251,11 +89277,11 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v61
+; SI-NEXT:    v_or_b32_e32 v4, v4, v62
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v38
@@ -89265,18 +89291,18 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v6, v58
+; SI-NEXT:    v_or_b32_e32 v6, v6, v56
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v46
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v40
-; SI-NEXT:    v_or_b32_e32 v8, v8, v50
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; SI-NEXT:    v_or_b32_e32 v8, v8, v44
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v33
@@ -89287,24 +89313,24 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v62
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v63
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v63
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v60
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v59
@@ -89313,86 +89339,85 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v57
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v51
 ; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v56
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v47
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v55
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v50
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v44
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v45
+; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v19
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
@@ -89405,7 +89430,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v20
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
@@ -89426,7 +89451,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -89439,7 +89464,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v22
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -89452,7 +89477,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v23
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v23, 0xff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -89465,7 +89490,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v24
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -89478,7 +89503,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -89491,7 +89516,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -89561,7 +89586,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
-; SI-NEXT:    v_or_b32_e32 v31, v31, v36
+; SI-NEXT:    v_or_b32_e32 v31, v31, v43
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
 ; SI-NEXT:    v_or_b32_e32 v32, v33, v32
@@ -89575,249 +89600,249 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:  .LBB58_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB58_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_movk_i32 s6, 0x300
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_or_b32_e32 v0, v43, v0
+; SI-NEXT:    v_or_b32_e32 v0, v54, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
+; SI-NEXT:    v_or_b32_e32 v2, v40, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
@@ -89826,16 +89851,18 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
-; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
+; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
@@ -89843,7 +89870,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v41, v1
@@ -89856,11 +89883,11 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v2, v35, v2
+; SI-NEXT:    v_or_b32_e32 v2, v53, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -89878,7 +89905,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v61, v4
+; SI-NEXT:    v_or_b32_e32 v4, v62, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
@@ -89896,7 +89923,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    v_or_b32_e32 v6, v56, v6
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
@@ -89909,9 +89936,9 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v40
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v35
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v50, v8
+; SI-NEXT:    v_or_b32_e32 v8, v44, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
@@ -89926,7 +89953,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v62
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v63
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
@@ -89934,7 +89961,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v63
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v39
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
@@ -89942,7 +89969,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v60
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -89955,12 +89982,12 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v57
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v56
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
@@ -89973,7 +90000,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v55
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v50
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
@@ -89981,19 +90008,17 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v44
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v36
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
@@ -90003,7 +90028,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -90012,7 +90037,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v15
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -90020,7 +90045,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -90029,7 +90054,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -90037,7 +90062,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -90046,7 +90071,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v17
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -90054,7 +90079,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -90071,7 +90096,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v20, v19
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -90098,7 +90123,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v21, v20
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -90115,7 +90140,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v21
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
@@ -90132,7 +90157,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
@@ -90149,7 +90174,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, s7, v23
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
@@ -90166,7 +90191,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s7, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
@@ -90183,7 +90208,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s7, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
@@ -90275,7 +90300,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
-; SI-NEXT:    v_or_b32_e32 v31, v36, v31
+; SI-NEXT:    v_or_b32_e32 v31, v43, v31
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x300, v31
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
@@ -90356,19 +90381,18 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -90400,59 +90424,61 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
@@ -90461,25 +90487,25 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -90487,25 +90513,25 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -90513,25 +90539,25 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -90539,10 +90565,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -90554,10 +90580,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -90568,22 +90594,22 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -90594,14 +90620,14 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -90613,34 +90639,33 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB58_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -90648,19 +90673,19 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -90671,23 +90696,23 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr61
-; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
 ; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr38
@@ -90728,147 +90753,217 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; kill: killed $vgpr33
 ; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -90878,16 +90973,16 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -90896,7 +90991,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -90905,253 +91000,190 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr33
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:  .LBB58_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB58_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(13)
+; VI-NEXT:    v_add_u16_e32 v9, 3, v61
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v31, 0x300
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v61
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v1, 3, v1
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; VI-NEXT:    v_add_u16_sdwa v1, v1, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -91160,31 +91192,25 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
 ; VI-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v4, v4, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v5, 3, v5
 ; VI-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(13)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
 ; VI-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(11)
+; VI-NEXT:    s_waitcnt vmcnt(10)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
 ; VI-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
-; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -91221,13 +91247,13 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v63
+; VI-NEXT:    v_add_u16_e32 v8, 3, v62
 ; VI-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v59
+; VI-NEXT:    v_add_u16_e32 v9, 3, v32
 ; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v10, 3, v62
+; VI-NEXT:    v_add_u16_e32 v10, 3, v60
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
@@ -91235,27 +91261,28 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
 ; VI-NEXT:    v_add_u16_e32 v10, 3, v58
 ; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v11, 3, v60
+; VI-NEXT:    v_add_u16_e32 v11, 3, v56
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v57
+; VI-NEXT:    v_add_u16_e32 v11, 3, v59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v12, 3, v56
+; VI-NEXT:    v_add_u16_e32 v12, 3, v57
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v46
+; VI-NEXT:    v_add_u16_e32 v12, 3, v47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_add_u16_e32 v13, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -91266,148 +91293,149 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 3, v44
 ; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v54
+; VI-NEXT:    v_add_u16_e32 v14, 3, v42
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v15, 3, v40
-; VI-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_add_u16_sdwa v15, v15, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v15
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v15, v15, v16
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v17, v17, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v16, v16, v17
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v18, v18, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v17, v17, v18
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v19, v19, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v18, v18, v19
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v20, v20, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v19, v19, v20
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v21, v21, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v20, v20, v21
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v22, v22, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v21, v21, v22
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v23, v23, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v22, v22, v23
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v24, v24, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v23, v23, v24
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
 ; VI-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v25, v25, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v24, v24, v25
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -91421,19 +91449,19 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v26, v26, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v26, 3, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
 ; VI-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v27, v27, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v26, v26, v27
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -91447,20 +91475,20 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v28, v28, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v27, v27, v28
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v28, 3, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v29, v29, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v28, v28, v29
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -91473,7 +91501,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v29, v29, v30
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v30, 3, v30
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -91486,7 +91514,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v32, v32, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v30, v30, v32
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v32, 3, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -91495,7 +91523,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v32, 0x300, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v33, 3, v33
-; VI-NEXT:    v_or_b32_sdwa v33, v34, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v33, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v31, v33, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v31, v32, v31
 ; VI-NEXT:  .LBB58_4: ; %end
@@ -91572,19 +91600,18 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -91596,93 +91623,95 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v35, 8, v29
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v10
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v14
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; GFX9-NEXT:    s_nop 0
@@ -91692,25 +91721,25 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -91719,25 +91748,25 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -91746,25 +91775,25 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -91773,10 +91802,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -91788,10 +91817,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -91803,22 +91832,22 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -91830,14 +91859,14 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -91849,34 +91878,33 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB58_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -91884,20 +91912,20 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -91908,23 +91936,23 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr61
-; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr38
@@ -91965,147 +91993,217 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; kill: killed $vgpr33
 ; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -92115,16 +92213,16 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -92133,7 +92231,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -92142,258 +92240,196 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr33
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:  .LBB58_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB58_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(33)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(30)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(28)
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
+; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    s_waitcnt vmcnt(27)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
-; GFX9-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(23)
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; GFX9-NEXT:    v_add_u16_sdwa v1, v1, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -92402,32 +92438,26 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
 ; GFX9-NEXT:    v_add_u16_e32 v4, 3, v4
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v4, v4, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_add_u16_e32 v5, 3, v5
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -92464,41 +92494,41 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v63
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v62
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v59
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v32
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v60
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
 ; GFX9-NEXT:    v_add_u16_e32 v10, 3, v58
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v60
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v56
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v57
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v56
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v57
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v46
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v46
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -92509,148 +92539,149 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 3, v44
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v54
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v40
-; GFX9-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v17, v17, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v16, v16, v17
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v18, v18, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v17, v17, v18
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v19, v19, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v18, v18, v19
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v20, v20, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v19, v19, v20
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v21, v21, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v20, v20, v21
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v22, v22, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v21, v21, v22
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v23, v23, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v22, v22, v23
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v24, v24, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v23, v23, v24
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v25, v25, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v24, v24, v25
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -92664,19 +92695,19 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v26, v26, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v25, v25, v26
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v26, 3, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v27, v27, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v26, v26, v27
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -92690,20 +92721,20 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v28, v28, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v27, v27, v28
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v28, 3, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v29, v29, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v28, v28, v29
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -92716,7 +92747,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v30, v30, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v29, v29, v30
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v30, 3, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -92729,7 +92760,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v31, v31, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v30, v30, v31
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v31, 3, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -92738,7 +92769,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v31, 0x300, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v32, 3, v32
-; GFX9-NEXT:    v_or_b32_sdwa v32, v33, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v32, v63, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v32, v32, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX9-NEXT:  .LBB58_4: ; %end
@@ -95518,8 +95549,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    s_cbranch_vccnz .LBB59_5
 ; SI-NEXT:  ; %bb.4: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_add_i32 s28, s28, 3
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v40
@@ -95574,7 +95605,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
@@ -95696,12 +95727,12 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 0x3000000, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
@@ -96616,8 +96647,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    s_cbranch_vccnz .LBB59_5
 ; VI-NEXT:  ; %bb.4: ; %cmp.true
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_add_i32 s28, s28, 3
 ; VI-NEXT:    s_and_b32 s4, s28, 0xff
 ; VI-NEXT:    s_lshl_b32 s5, s29, 8
@@ -96669,7 +96700,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    s_add_i32 s6, s6, 0x3000000
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x300, v1
@@ -96764,11 +96795,11 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -96891,11 +96922,11 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -97680,11 +97711,11 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -97896,11 +97927,11 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -98084,134 +98115,119 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:324
 ; GFX11-TRUE16-NEXT:    ; meta instruction
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:320
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v86, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:312
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:308
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:304
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:300
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:264
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:260
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:256
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:252
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:248
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:240
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:200
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:192
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:312
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:308
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:304
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:300
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:296
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:260
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:252
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:248
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:244
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:236
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:208
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:200
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:184
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:176
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:168
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:160
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:152
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:144
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:136
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:132
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:128
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:116
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:184
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:176
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:168
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:160
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:152
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:144
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:136
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v183, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v40, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v41, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v42, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v43, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v45, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v44, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v46, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v47, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v57, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v56, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v58, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v59, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v61, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v60, off, s32 offset:64
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v183.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v40.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v42.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v41.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v43.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v44.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v46.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v45.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v47.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v56.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v57.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v58.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v59.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v60.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v62.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v61.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v63.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v72.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v74.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v73.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v75.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v76.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v78.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v77.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v79.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v88.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v90.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v89.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v91.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v92.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v93.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v62, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v63, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v73, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v72, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v74, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v75, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v76, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v77, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v78, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v79, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v89, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v88, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v90, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v91, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v93, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v92, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v30 :: v_dual_mov_b32 v33, v29
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v28 :: v_dual_mov_b32 v34, v27
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v36, v26 :: v_dual_mov_b32 v37, v25
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v24 :: v_dual_mov_b32 v38, v23
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v22 :: v_dual_mov_b32 v49, v21
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v20 :: v_dual_mov_b32 v51, v19
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v18 :: v_dual_mov_b32 v53, v17
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v55, v16 :: v_dual_mov_b32 v54, v15
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v64, v14 :: v_dual_mov_b32 v65, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v67, v12 :: v_dual_mov_b32 v66, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v68, v10 :: v_dual_mov_b32 v69, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v71, v8 :: v_dual_mov_b32 v70, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v80, v6 :: v_dual_mov_b32 v81, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v83, v4 :: v_dual_mov_b32 v82, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v84, v2 :: v_dual_mov_b32 v85, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB59_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v91
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v89
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v88
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v79
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v77
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v78
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v76
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v69
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
@@ -98224,305 +98240,305 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s5, s6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v75
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v73
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v66
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v2, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v72
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v65
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s1, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v67
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s2, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s3, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v60
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s19, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s27, 8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v63
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v61
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v54
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v55
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v56
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v49
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v59
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v58
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v51
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v44
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v37
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v47
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v45
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v39
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v33
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v43
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v34
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v180
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v91
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v183
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v32
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v93
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v176
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v79
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v179
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v177
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v90
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v88
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v89
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v164
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v75
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v167
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v166
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v78
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v77
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v76
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v160
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v63
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v163
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v161
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v72
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v59
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v151
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v60
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v61
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v144
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v47
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v147
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v145
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v56
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v43
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v135
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v133
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v44
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v45
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v183
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v131
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v130
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v40
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v116
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v179
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v119
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v117
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v180
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v181
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v167
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v176
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v114
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v177
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v100
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v163
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v101
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v164
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v102
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v151
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v97
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v160
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v161
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v147
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v86
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v148
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v135
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v83
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v144
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v131
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v132
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v133
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v119
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v67
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v128
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v114
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v116
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v54
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v117
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v102
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v49
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v112
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v98
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v100
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v101
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v87
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v33
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v96
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v97
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v93
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
@@ -98609,31 +98625,31 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v86
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v91
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v88
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v84
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v81
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v80
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v89
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v78
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v71
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v77
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v70
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v76
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -98649,61 +98665,61 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v68
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v74
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v67
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v73
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v72
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v63
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v65
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v64
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v0, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v8, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v61
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v60
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v59
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v52
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v8, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v51
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v8, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v57
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v50
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v2, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v56
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v47
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v49
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v46
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v45
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v44
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v43
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v35
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v41
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -98714,63 +98730,63 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v10
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v3, v0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v182
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v11
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v92
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v180
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v179
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v91
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v90
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v89
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v13, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v177
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v88
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v176
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v167
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v78
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v13, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v77
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v13, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v165
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v76
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v164
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v163
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v74
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v162
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v14, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v161
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v72
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v160
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v151
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v63
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v61
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v15, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v149
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v60
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v59
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -98781,61 +98797,61 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v15
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v58
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v146
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v145
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v56
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v144
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v135
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v47
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v46
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v45
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v18, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v44
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v132
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v43
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v42
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v18, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v41
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v18, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v129
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v40
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v128
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v182
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v181
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v19, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v180
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v116
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v115
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v179
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v177
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v19, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v20, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v176
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xff, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v167
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -98846,61 +98862,61 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v20
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v166
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v102
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v165
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v101
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v164
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v100
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v163
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v162
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v98
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v161
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v23, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v160
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v96
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v87
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v151
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v150
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v23, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v149
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v23, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v148
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v146
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v82
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v144
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v80
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v71
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v135
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v133
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v25, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v131
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -98911,52 +98927,52 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v25
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v67
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v130
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v3, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v128
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v129
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v113
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v27, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v115
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v114
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v28, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v53
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v116
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v30, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v36, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v101
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v32, 3, v99
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v28, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v29, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 0x300, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v49
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v112
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v100
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v30, 0xffff, v27
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v102
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xff, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v31, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v33
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 8, v96
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xff, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v87
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v28, v27
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v37, v29
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v33, v29
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v36
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v32, v34
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v36, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v37, v34
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -100035,9 +100051,9 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -100070,7 +100086,7 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; kill: killed $vgpr31
@@ -100189,7 +100205,6 @@ define <64 x bfloat> @bitcast_v16i64_to_v64bf16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v62
 ; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -101980,6 +101995,8 @@ define <16 x i64> @bitcast_v64bf16_to_v16i64(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v26, 16, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr33
@@ -102056,8 +102073,6 @@ define <16 x i64> @bitcast_v64bf16_to_v16i64(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_alignbit_b32 v5, v5, v6, 16
@@ -108461,9 +108476,9 @@ define <64 x half> @bitcast_v16i64_to_v64f16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -108496,7 +108511,7 @@ define <64 x half> @bitcast_v16i64_to_v64f16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; kill: killed $vgpr31
@@ -108569,7 +108584,6 @@ define <64 x half> @bitcast_v16i64_to_v64f16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB64_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v62
 ; SI-NEXT:    v_cvt_f32_f16_e32 v32, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -111665,9 +111679,9 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
@@ -111717,9 +111731,8 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
 ; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
@@ -112522,8 +112535,8 @@ define <64 x i16> @bitcast_v16i64_to_v64i16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -112557,7 +112570,7 @@ define <64 x i16> @bitcast_v16i64_to_v64i16(<16 x i64> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -113737,26 +113750,25 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:132
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
@@ -113769,102 +113781,107 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v14
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v26
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v30
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v33
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:100
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v31
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:28
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v16
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB70_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v41
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -113881,25 +113898,7 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v50
-; SI-NEXT:    v_or_b32_e32 v23, v23, v32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v32
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
@@ -113929,67 +113928,71 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v48
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v35
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v43
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v55
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v53
+; SI-NEXT:    v_or_b32_e32 v16, v16, v53
+; SI-NEXT:    v_or_b32_e32 v17, v17, v51
+; SI-NEXT:    v_or_b32_e32 v18, v18, v50
 ; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_or_b32_e32 v19, v19, v48
-; SI-NEXT:    v_or_b32_e32 v21, v21, v36
-; SI-NEXT:    v_or_b32_e32 v22, v22, v34
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v18, v18, v49
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_or_b32_e32 v22, v22, v23
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v54
+; SI-NEXT:    v_or_b32_e32 v19, v19, v39
+; SI-NEXT:    v_or_b32_e32 v23, v23, v24
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v52
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v39
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v35
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v37
 ; SI-NEXT:    v_or_b32_e32 v26, v26, v27
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v33
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v28
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v63
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v62
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v61
@@ -114002,13 +114005,15 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v46
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v45
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v44
-; SI-NEXT:    v_or_b32_e32 v12, v12, v42
-; SI-NEXT:    v_or_b32_e32 v13, v13, v41
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v40
-; SI-NEXT:    v_or_b32_e32 v15, v15, v54
-; SI-NEXT:    v_or_b32_e32 v20, v20, v38
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    v_or_b32_e32 v15, v15, v55
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -114022,63 +114027,79 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v29
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v16, v52
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v20, v36
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v30
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v30, v31
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v17, v51
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v21, v34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v31, v37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    v_or_b32_e32 v31, v31, v38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:  .LBB70_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB70_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v41
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -114095,69 +114116,68 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v48
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    s_mov_b32 s6, 0x30000
+; SI-NEXT:    v_or_b32_e32 v16, v53, v16
+; SI-NEXT:    v_or_b32_e32 v17, v51, v17
+; SI-NEXT:    v_or_b32_e32 v18, v50, v18
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v43
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v55
-; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v53
-; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v50
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT:    v_or_b32_e32 v22, v23, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v54
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v19, v48, v19
-; SI-NEXT:    v_or_b32_e32 v21, v36, v21
-; SI-NEXT:    v_or_b32_e32 v22, v34, v22
-; SI-NEXT:    v_or_b32_e32 v23, v32, v23
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
-; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    v_or_b32_e32 v23, v24, v23
+; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
-; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v39
+; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v49
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
-; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v37
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_or_b32_e32 v27, v28, v27
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -114174,8 +114194,7 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_or_b32_e32 v0, v63, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v62, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v61, v2
@@ -114188,28 +114207,25 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v46, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v45, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v44, v11
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_or_b32_e32 v13, v41, v13
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v40, v14
-; SI-NEXT:    v_or_b32_e32 v15, v54, v15
-; SI-NEXT:    v_or_b32_e32 v18, v49, v18
-; SI-NEXT:    v_or_b32_e32 v20, v38, v20
+; SI-NEXT:    v_or_b32_e32 v15, v55, v15
+; SI-NEXT:    v_or_b32_e32 v19, v39, v19
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v29, v28
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
@@ -114219,34 +114235,37 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, s6, v28
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v36, v20
+; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v30, v29
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v31, v30
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v51, v17
-; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v34, v21
+; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 0x30000, v30
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v37, v31
+; SI-NEXT:    v_or_b32_e32 v31, v38, v31
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x30000, v31
 ; SI-NEXT:  .LBB70_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -114266,7 +114285,7 @@ define <16 x i64> @bitcast_v64i16_to_v16i64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i16_to_v16i64:
@@ -115622,8 +115641,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; kill: killed $vgpr36
@@ -115785,7 +115804,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; kill: killed $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -116818,8 +116837,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    ; kill: killed $vgpr39
@@ -117459,13 +117478,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 40, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 44, v0
@@ -117485,13 +117505,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 52, v0
@@ -117511,13 +117532,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 56, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v16, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 60, v0
@@ -117537,13 +117559,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 64, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v18, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x44, v0
@@ -117563,13 +117586,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x4c, v0
@@ -117588,13 +117612,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x50, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x54, v0
@@ -117614,13 +117639,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x58, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x5c, v0
@@ -117640,13 +117666,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x60, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x64, v0
@@ -117691,13 +117718,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x70, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x74, v0
@@ -117765,8 +117793,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; kill: killed $vgpr41
@@ -117960,14 +117988,13 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(32)
+; GFX9-NEXT:    s_waitcnt vmcnt(31)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB72_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
-; GFX9-NEXT:    s_waitcnt vmcnt(31)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 24, v32
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
@@ -118569,13 +118596,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
@@ -118594,13 +118622,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
@@ -118619,13 +118648,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
@@ -118644,13 +118674,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
@@ -118669,13 +118700,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
@@ -118694,13 +118726,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
@@ -118719,13 +118752,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
@@ -123736,13 +123770,14 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
@@ -125920,10 +125955,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:388
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
@@ -125943,33 +125978,35 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:136
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:144
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:152
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:160
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:168
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:176
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:184
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v1
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:136
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:144
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:152
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:160
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:168
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:176
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:184
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 8, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 8, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 24, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 8, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 24, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v6
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -125990,10 +126027,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v18
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v20
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v22
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -126004,69 +126041,69 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v28
-; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:132
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:140
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v31
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:140
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:148
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v33
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v36
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v35
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v36
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v39
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:156
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:164
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:172
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:180
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v49
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v40
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v50
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v51
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:192
@@ -126081,14 +126118,14 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:196
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -126110,16 +126147,16 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -126138,19 +126175,19 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -126169,10 +126206,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:284
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -126241,7 +126278,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:356
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v2
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -126259,121 +126296,119 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:92
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:68
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB74_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v43
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v0, v0, v54
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
-; SI-NEXT:    v_or_b32_e32 v3, v53, v3
+; SI-NEXT:    v_or_b32_e32 v3, v40, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v35
+; SI-NEXT:    v_or_b32_e32 v2, v2, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v34, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
+; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v48
@@ -126383,11 +126418,11 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v61
+; SI-NEXT:    v_or_b32_e32 v4, v4, v62
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v38
@@ -126397,18 +126432,18 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v6, v58
+; SI-NEXT:    v_or_b32_e32 v6, v6, v56
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v46
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v40
-; SI-NEXT:    v_or_b32_e32 v8, v8, v50
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; SI-NEXT:    v_or_b32_e32 v8, v8, v44
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v33
@@ -126419,24 +126454,24 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v62
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v63
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v63
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v60
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v59
@@ -126445,86 +126480,85 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v57
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v51
 ; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v56
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v47
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v55
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v50
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v44
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v45
+; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v19
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
@@ -126537,7 +126571,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v20
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
@@ -126558,7 +126592,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -126571,7 +126605,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v22
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -126584,7 +126618,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v23
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v23, 0xff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -126597,7 +126631,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v24
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -126610,7 +126644,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -126623,7 +126657,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -126693,7 +126727,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
-; SI-NEXT:    v_or_b32_e32 v31, v31, v36
+; SI-NEXT:    v_or_b32_e32 v31, v31, v43
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
 ; SI-NEXT:    v_or_b32_e32 v32, v33, v32
@@ -126707,249 +126741,249 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; kill: killed $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:  .LBB74_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB74_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_movk_i32 s6, 0x300
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v54, v9
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_or_b32_e32 v0, v43, v0
+; SI-NEXT:    v_or_b32_e32 v0, v54, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v42, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
+; SI-NEXT:    v_or_b32_e32 v2, v40, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
@@ -126958,16 +126992,18 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v32, v4
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
-; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_or_b32_e32 v5, v60, v5
+; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
@@ -126975,7 +127011,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v41, v1
@@ -126988,11 +127024,11 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
+; SI-NEXT:    v_or_b32_e32 v8, v55, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v2, v35, v2
+; SI-NEXT:    v_or_b32_e32 v2, v53, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -127010,7 +127046,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v61, v4
+; SI-NEXT:    v_or_b32_e32 v4, v62, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
@@ -127028,7 +127064,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v58, v6
+; SI-NEXT:    v_or_b32_e32 v6, v56, v6
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
@@ -127041,9 +127077,9 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v40
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v35
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v50, v8
+; SI-NEXT:    v_or_b32_e32 v8, v44, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
@@ -127058,7 +127094,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v62
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v63
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
@@ -127066,7 +127102,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v63
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v39
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
@@ -127074,7 +127110,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v60
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -127087,12 +127123,12 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v57
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v56
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
@@ -127105,7 +127141,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v55
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v50
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
@@ -127113,19 +127149,17 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v44
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v36
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
@@ -127135,7 +127169,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -127144,7 +127178,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v15
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -127152,7 +127186,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -127161,7 +127195,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -127169,7 +127203,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -127178,7 +127212,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v17
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -127186,7 +127220,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -127203,7 +127237,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v19, v20, v19
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -127230,7 +127264,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    v_or_b32_e32 v20, v21, v20
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -127247,7 +127281,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v21
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
@@ -127264,7 +127298,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
@@ -127281,7 +127315,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    v_or_b32_e32 v23, v24, v23
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, s7, v23
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
@@ -127298,7 +127332,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s7, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
@@ -127315,7 +127349,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s7, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
@@ -127407,7 +127441,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, 3, v32
-; SI-NEXT:    v_or_b32_e32 v31, v36, v31
+; SI-NEXT:    v_or_b32_e32 v31, v43, v31
 ; SI-NEXT:    v_and_b32_e32 v32, 0xff, v32
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x300, v31
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
@@ -127488,19 +127522,18 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -127532,59 +127565,61 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
@@ -127593,25 +127628,25 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -127619,25 +127654,25 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -127645,25 +127680,25 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -127671,10 +127706,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -127686,10 +127721,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -127700,22 +127735,22 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -127726,14 +127761,14 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -127745,34 +127780,33 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB74_2
 ; VI-NEXT:  ; %bb.1: ; %cmp.false
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -127780,19 +127814,19 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -127803,23 +127837,23 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr61
-; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr55
+; VI-NEXT:    ; implicit-def: $vgpr54
 ; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr38
@@ -127860,147 +127894,217 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; kill: killed $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; kill: killed $vgpr33
 ; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr54
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -128010,16 +128114,16 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -128028,7 +128132,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -128037,253 +128141,190 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr33
-; VI-NEXT:    ; kill: killed $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr32
+; VI-NEXT:    ; kill: killed $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:  .LBB74_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB74_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(13)
+; VI-NEXT:    v_add_u16_e32 v9, 3, v61
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v31, 0x300
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v61
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_add_u16_e32 v8, 3, v8
+; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v1, 3, v1
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 3, v1
+; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; VI-NEXT:    v_add_u16_sdwa v1, v1, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -128292,31 +128333,25 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v1, v2, v3
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
 ; VI-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v4, v4, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v5, 3, v5
 ; VI-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(13)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
 ; VI-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(11)
+; VI-NEXT:    s_waitcnt vmcnt(10)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
 ; VI-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_sdwa v9, v9, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
-; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -128353,13 +128388,13 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v63
+; VI-NEXT:    v_add_u16_e32 v8, 3, v62
 ; VI-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v59
+; VI-NEXT:    v_add_u16_e32 v9, 3, v32
 ; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v10, 3, v62
+; VI-NEXT:    v_add_u16_e32 v10, 3, v60
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
@@ -128367,27 +128402,28 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
 ; VI-NEXT:    v_add_u16_e32 v10, 3, v58
 ; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v11, 3, v60
+; VI-NEXT:    v_add_u16_e32 v11, 3, v56
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v57
+; VI-NEXT:    v_add_u16_e32 v11, 3, v59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v12, 3, v56
+; VI-NEXT:    v_add_u16_e32 v12, 3, v57
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v46
+; VI-NEXT:    v_add_u16_e32 v12, 3, v47
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_add_u16_e32 v13, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -128398,148 +128434,149 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 3, v44
 ; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v54
+; VI-NEXT:    v_add_u16_e32 v14, 3, v42
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v15, 3, v40
-; VI-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_add_u16_sdwa v15, v15, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v15
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v15, 3, v15
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v15, v15, v16
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v17, v17, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v16, v16, v17
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v18, v18, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v17, v17, v18
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v18, 3, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v19, v19, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v18, v18, v19
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v20, v20, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v19, v19, v20
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v21, v21, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v20, v20, v21
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v22, v22, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v21, v21, v22
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v23, v23, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v22, v22, v23
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v23, 3, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v24, v24, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v23, v23, v24
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v24, 3, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
 ; VI-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v25, v25, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v24, v24, v25
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -128553,19 +128590,19 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v26, v26, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v26, 3, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
 ; VI-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v27, v27, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v26, v26, v27
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -128579,20 +128616,20 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v28, v28, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v27, v27, v28
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v28, 3, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v29, v29, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v28, v28, v29
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v29, 3, v29
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -128605,7 +128642,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v29, v29, v30
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v30, 3, v30
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -128618,7 +128655,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v32, v32, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v30, v30, v32
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v32, 3, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -128627,7 +128664,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v32, 0x300, v32
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v33, 3, v33
-; VI-NEXT:    v_or_b32_sdwa v33, v34, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v33, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v31, v33, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v31, v32, v31
 ; VI-NEXT:  .LBB74_4: ; %end
@@ -128704,19 +128741,18 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:136
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:144
-; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:152
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:160
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:168
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:176
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:184
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:144
+; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:152
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:160
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:176
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:184
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v7
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v9
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v13
@@ -128728,93 +128764,95 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v35, 8, v29
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v10
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v14
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(21)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v54
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v32
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v40
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v42
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:172
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:180
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v45
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v46
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v47
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v56
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; GFX9-NEXT:    s_nop 0
@@ -128824,25 +128862,25 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:204
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:212
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:224
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:232
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:240
@@ -128851,25 +128889,25 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:220
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -128878,25 +128916,25 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:252
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -128905,10 +128943,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
@@ -128920,10 +128958,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:308
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:320
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:328
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:336
@@ -128935,22 +128973,22 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:340
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:352
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:360
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:368
@@ -128962,14 +129000,14 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
@@ -128981,34 +129019,33 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
 ; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB74_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -129016,20 +129053,20 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v55 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v41 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
@@ -129040,23 +129077,23 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v57, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr61
-; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
+; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr38
@@ -129097,147 +129134,217 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; kill: killed $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; kill: killed $vgpr33
 ; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v32, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v54, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -129247,16 +129354,16 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
@@ -129265,7 +129372,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
@@ -129274,258 +129381,196 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v63 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; kill: killed $vgpr32
 ; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr33
-; GFX9-NEXT:    ; kill: killed $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; kill: killed $vgpr63
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:  .LBB74_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB74_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(33)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(30)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(28)
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
+; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    s_waitcnt vmcnt(27)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
-; GFX9-NEXT:    v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v3, v55, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(23)
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v1
+; GFX9-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; GFX9-NEXT:    v_add_u16_sdwa v1, v1, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v2
@@ -129534,32 +129579,26 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
 ; GFX9-NEXT:    v_add_u16_e32 v4, 3, v4
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v50, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v4, v4, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_add_u16_e32 v5, 3, v5
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v48, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v38, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(12)
+; GFX9-NEXT:    s_waitcnt vmcnt(11)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v34, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -129596,41 +129635,41 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v63
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v62
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v59
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v32
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v60
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
 ; GFX9-NEXT:    v_add_u16_e32 v10, 3, v58
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v60
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v56
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v57
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v56
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v57
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v46
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v46
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
@@ -129641,148 +129680,149 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 3, v44
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v54
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v40
-; GFX9-NEXT:    v_or_b32_sdwa v14, v32, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v17, v17, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v16, v16, v17
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v18, v18, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v17, v17, v18
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v19, v19, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v18, v18, v19
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v20, v20, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v19, v19, v20
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v21, v21, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v20, v20, v21
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v22, v22, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v21, v21, v22
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v23, v23, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v22, v22, v23
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v23, 3, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v23, 0x300, v23
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v24, v24, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v23, v23, v24
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v25, v25, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v24, v24, v25
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v25, 3, v25
@@ -129796,19 +129836,19 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v26, v26, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v25, v25, v26
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v26, 3, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v27, v27, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v26, v26, v27
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -129822,20 +129862,20 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v28, v28, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v27, v27, v28
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v28, 3, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v28, v29, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v29, v29, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v28, v28, v29
-; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v29, 3, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -129848,7 +129888,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v30, v30, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v29, v29, v30
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v30, 3, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -129861,7 +129901,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v31, v31, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v30, v30, v31
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v31, 3, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -129870,7 +129910,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v31, 0x300, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v32, 3, v32
-; GFX9-NEXT:    v_or_b32_sdwa v32, v33, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v32, v63, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v32, v32, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX9-NEXT:  .LBB74_4: ; %end
@@ -132650,8 +132690,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; SI-NEXT:    s_cbranch_vccnz .LBB75_5
 ; SI-NEXT:  ; %bb.4: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_add_i32 s28, s28, 3
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v40
@@ -132706,7 +132746,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
@@ -132828,12 +132868,12 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 0x3000000, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
@@ -133748,8 +133788,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    s_cbranch_vccnz .LBB75_5
 ; VI-NEXT:  ; %bb.4: ; %cmp.true
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    s_add_i32 s28, s28, 3
 ; VI-NEXT:    s_and_b32 s4, s28, 0xff
 ; VI-NEXT:    s_lshl_b32 s5, s29, 8
@@ -133801,7 +133841,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    s_add_i32 s6, s6, 0x3000000
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 3, v1
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x300, v1
@@ -133896,11 +133936,11 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -134023,11 +134063,11 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; VI-NEXT:    v_add_u32_e32 v20, vcc, 0x3000000, v0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v0
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x300, v0
@@ -134812,11 +134852,11 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -135028,11 +135068,11 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX9-NEXT:    v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -135216,134 +135256,119 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:324
 ; GFX11-TRUE16-NEXT:    ; meta instruction
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:320
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v86, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:312
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:308
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:304
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:300
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:264
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:260
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:256
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:252
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:248
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:240
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:200
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:192
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:312
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:308
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:304
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:300
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:296
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:260
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:252
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:248
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:244
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:236
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:208
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:200
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:184
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:176
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:168
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:160
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:152
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:144
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:136
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:132
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:128
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:116
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:184
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:176
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:168
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:160
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:152
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:144
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:136
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v183, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v40, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v41, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v42, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v43, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v45, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v44, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v46, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v47, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v57, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v56, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v58, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v59, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v61, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v60, off, s32 offset:64
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v163, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v164, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v176, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v178, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v177, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v179, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v180, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v182, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v181, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v183.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v40.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v42.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v41.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v43.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v44.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v46.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v45.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v47.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v56.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v57.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v58.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v59.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v60.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v62.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v61.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v63.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v72.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v74.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v73.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v75.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v76.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v78.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v77.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v79.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v88.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v90.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v89.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v91.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v92.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v93.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v62, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v63, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v73, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v72, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v74, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v75, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v76, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v77, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v78, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v79, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v89, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v88, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v90, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v91, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v93, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v92, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v30 :: v_dual_mov_b32 v33, v29
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v28 :: v_dual_mov_b32 v34, v27
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v36, v26 :: v_dual_mov_b32 v37, v25
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v24 :: v_dual_mov_b32 v38, v23
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v22 :: v_dual_mov_b32 v49, v21
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v20 :: v_dual_mov_b32 v51, v19
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v18 :: v_dual_mov_b32 v53, v17
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v55, v16 :: v_dual_mov_b32 v54, v15
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v64, v14 :: v_dual_mov_b32 v65, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v67, v12 :: v_dual_mov_b32 v66, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v68, v10 :: v_dual_mov_b32 v69, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v71, v8 :: v_dual_mov_b32 v70, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v80, v6 :: v_dual_mov_b32 v81, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v83, v4 :: v_dual_mov_b32 v82, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v84, v2 :: v_dual_mov_b32 v85, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB75_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v91
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v89
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v88
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v79
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v77
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v78
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v76
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v69
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
@@ -135356,305 +135381,305 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s5, s6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v75
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v73
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v66
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v2, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v72
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v65
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s1, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v67
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s2, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s3, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v60
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s19, 8
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s27, 8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v63
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v61
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v54
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v55
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v56
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v49
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v59
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v58
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v51
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v44
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v37
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v47
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v45
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v39
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v33
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v43
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v34
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v180
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v91
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v183
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v32
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v93
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v176
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v79
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v179
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v177
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v90
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v88
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v89
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v164
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v75
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v167
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v166
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v78
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v77
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v76
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v160
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v63
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v163
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v161
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v74
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v72
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v59
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v151
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v62
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v60
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v61
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v144
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v47
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v147
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v145
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v56
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v43
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v135
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v133
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v46
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v44
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v45
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v183
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v131
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v130
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v41
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v40
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v116
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v179
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v119
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v117
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v182
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v180
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v181
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v167
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v178
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v176
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v114
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v177
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v100
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v163
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v101
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v164
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v102
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v165
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v151
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v97
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v162
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v160
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v161
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v147
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v86
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v149
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v148
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v135
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v83
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v144
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v82
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v131
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v132
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v133
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v119
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v67
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v128
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v129
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v114
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v116
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v54
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v117
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v102
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v113
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v49
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v112
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v98
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v103
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v100
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v101
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v87
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v33
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v96
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v97
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v93
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
@@ -135741,31 +135766,31 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v86
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v92
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v85
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v91
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v90
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v88
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v84
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v81
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v80
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v89
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v78
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v71
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v77
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v70
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v76
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -135781,61 +135806,61 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v68
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v74
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v67
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v73
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v72
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v63
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v65
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v64
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v0, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v8, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v61
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v60
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v59
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v52
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v8, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v58
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v51
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v8, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v57
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v50
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v2, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v56
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v47
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v49
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v46
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v45
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v44
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v43
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v42
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v35
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v41
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -135846,63 +135871,63 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v10
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v3, v0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v182
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v93
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v11
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v181
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v92
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v180
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v179
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v91
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v90
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v89
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v13, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v177
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v88
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v176
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v167
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v79
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v78
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v13, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v166
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v77
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v13, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v165
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 3, v76
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v164
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v163
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v75
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v74
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v162
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v73
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v14, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v161
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v72
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v160
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v151
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v150
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v63
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v62
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v61
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v15, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v149
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v60
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v148
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v59
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -135913,61 +135938,61 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v15
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v58
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v146
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v57
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v145
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v56
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v144
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v135
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v47
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v46
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v45
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v18, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v44
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v132
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v43
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v42
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v17, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v18, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v41
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v18, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v129
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v40
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v128
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v183
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 3, v182
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v181
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v19, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v180
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v116
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v115
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v179
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v178
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 3, v177
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v19, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v20, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v176
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xff, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v112
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v167
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -135978,61 +136003,61 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v20
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v21, 3, v166
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v3, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v102
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v165
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v101
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v164
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v100
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v163
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v162
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v98
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v161
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v23, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v160
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v96
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v87
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v151
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v150
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v22, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v23, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v149
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v23, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v148
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v147
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 3, v146
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v0, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v82
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v145
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v144
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v80
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v71
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v135
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v134
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v133
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v25, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v131
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -136043,52 +136068,52 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v1, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v25
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v67
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 3, v130
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v3, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 8, v128
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v129
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 8, v113
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v27, v3
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v115
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 8, v114
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v28, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v53
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v116
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v30, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v36, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v101
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v32, 3, v99
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v28, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v29, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 0x300, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v49
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v28, 3, v112
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v34, 3, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v100
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v30, 0xffff, v27
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xff, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v102
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xff, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v31, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v33
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 8, v96
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xff, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v87
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v28, v27
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v37, v29
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v33, v29
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v36
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v32, v34
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v36, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v37, v34
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
@@ -137167,8 +137192,8 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; kill: killed $vgpr35
@@ -137266,14 +137291,13 @@ define <64 x bfloat> @bitcast_v16f64_to_v64bf16(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; kill: killed $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB76_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v35, 0xffff0000, v32
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
 ; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
@@ -139059,6 +139083,8 @@ define <16 x double> @bitcast_v64bf16_to_v16f64(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v26, 16, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; kill: killed $vgpr33
@@ -139135,8 +139161,6 @@ define <16 x double> @bitcast_v64bf16_to_v16f64(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_alignbit_b32 v5, v5, v6, 16
@@ -145540,8 +145564,8 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; kill: killed $vgpr35
@@ -145639,7 +145663,7 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -145686,7 +145710,6 @@ define <64 x half> @bitcast_v16f64_to_v64f16(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v52
-; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshrrev_b32_e32 v34, 16, v31
@@ -148612,9 +148635,9 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
@@ -148664,9 +148687,8 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
 ; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
@@ -149469,8 +149491,8 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr60
@@ -149504,7 +149526,7 @@ define <64 x i16> @bitcast_v16f64_to_v64i16(<16 x double> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -150557,26 +150579,25 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:132
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:112
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v5
@@ -150589,102 +150610,107 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v14
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v18
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v20
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v22
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(8) expcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v24
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v26
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v30
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v33
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v28
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:100
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v31
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:28
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v16
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB86_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v41
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -150701,25 +150727,7 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v50
-; SI-NEXT:    v_or_b32_e32 v23, v23, v32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v32
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
@@ -150749,67 +150757,71 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v48
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v35
 ; SI-NEXT:    ; kill: killed $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v43
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v55
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v53
+; SI-NEXT:    v_or_b32_e32 v16, v16, v53
+; SI-NEXT:    v_or_b32_e32 v17, v17, v51
+; SI-NEXT:    v_or_b32_e32 v18, v18, v50
 ; SI-NEXT:    ; kill: killed $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    v_or_b32_e32 v19, v19, v48
-; SI-NEXT:    v_or_b32_e32 v21, v21, v36
-; SI-NEXT:    v_or_b32_e32 v22, v22, v34
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; kill: killed $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v18, v18, v49
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_or_b32_e32 v22, v22, v23
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v54
+; SI-NEXT:    v_or_b32_e32 v19, v19, v39
+; SI-NEXT:    v_or_b32_e32 v23, v23, v24
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v52
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v25
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v39
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v35
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v37
 ; SI-NEXT:    v_or_b32_e32 v26, v26, v27
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v33
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v28
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v63
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v62
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v61
@@ -150822,13 +150834,15 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v46
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v45
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v44
-; SI-NEXT:    v_or_b32_e32 v12, v12, v42
-; SI-NEXT:    v_or_b32_e32 v13, v13, v41
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v40
-; SI-NEXT:    v_or_b32_e32 v15, v15, v54
-; SI-NEXT:    v_or_b32_e32 v20, v20, v38
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    v_or_b32_e32 v15, v15, v55
+; SI-NEXT:    ; kill: killed $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -150842,63 +150856,79 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v29
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v16, v52
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v20, v36
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v30
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v30, v31
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v17, v51
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v21, v34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; kill: killed $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v31, v37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; kill: killed $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    v_or_b32_e32 v31, v31, v38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; kill: killed $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:  .LBB86_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB86_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v41
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
@@ -150915,69 +150945,68 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v48
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v32
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    s_mov_b32 s6, 0x30000
+; SI-NEXT:    v_or_b32_e32 v16, v53, v16
+; SI-NEXT:    v_or_b32_e32 v17, v51, v17
+; SI-NEXT:    v_or_b32_e32 v18, v50, v18
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v43
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v55
-; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v53
-; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v50
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT:    v_or_b32_e32 v22, v23, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v54
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v19, v48, v19
-; SI-NEXT:    v_or_b32_e32 v21, v36, v21
-; SI-NEXT:    v_or_b32_e32 v22, v34, v22
-; SI-NEXT:    v_or_b32_e32 v23, v32, v23
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
-; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    v_or_b32_e32 v23, v24, v23
+; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
-; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v39
+; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v49
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
-; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v37
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_or_b32_e32 v27, v28, v27
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -150994,8 +151023,7 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_or_b32_e32 v0, v63, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v62, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v61, v2
@@ -151008,28 +151036,25 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v9, v46, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v45, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v44, v11
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_or_b32_e32 v13, v41, v13
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v40, v14
-; SI-NEXT:    v_or_b32_e32 v15, v54, v15
-; SI-NEXT:    v_or_b32_e32 v18, v49, v18
-; SI-NEXT:    v_or_b32_e32 v20, v38, v20
+; SI-NEXT:    v_or_b32_e32 v15, v55, v15
+; SI-NEXT:    v_or_b32_e32 v19, v39, v19
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_or_b32_e32 v28, v29, v28
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
@@ -151039,34 +151064,37 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
+; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, s6, v28
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v20, v36, v20
+; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    v_or_b32_e32 v29, v30, v29
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 0x30000, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff, v30
 ; SI-NEXT:    v_or_b32_e32 v30, v31, v30
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; SI-NEXT:    v_or_b32_e32 v17, v51, v17
-; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v21, v34, v21
+; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, 0x30000, v30
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v37, v31
+; SI-NEXT:    v_or_b32_e32 v31, v38, v31
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 0x30000, v31
 ; SI-NEXT:  .LBB86_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -151086,7 +151114,7 @@ define <16 x double> @bitcast_v64i16_to_v16f64(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i16_to_v16f64:
@@ -152486,31 +152514,31 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v6
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v8
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v12
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v14
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v16
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v20
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v22
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v24
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v28
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v30
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
@@ -152522,36 +152550,35 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr20
-; SI-NEXT:    ; implicit-def: $vgpr14
+; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr6
+; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v7
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v9
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v10
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v11
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
@@ -152578,63 +152605,64 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v25
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr11
 ; SI-NEXT:    ; implicit-def: $vgpr10
-; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr9
 ; SI-NEXT:    ; implicit-def: $vgpr13
+; SI-NEXT:    ; implicit-def: $vgpr23
 ; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    ; implicit-def: $vgpr17
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:176
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:160
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:140
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:172
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:176
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:136
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:172
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:168
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:160
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:96
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v26
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v27
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v29
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:96
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:88
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:64
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:80
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:196
@@ -152643,27 +152671,27 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:192
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(3) expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:208
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:204
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:200
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:244
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252
@@ -152673,35 +152701,35 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 24, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240
 ; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:260
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:276
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:256
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v29, 24, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v31, 24, v1
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 24, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:272
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -152713,8 +152741,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:316
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:288
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:304
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:300
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:296
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:300
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:296
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v1
 ; SI-NEXT:    s_waitcnt vmcnt(5)
@@ -152751,10 +152779,10 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 24, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:40
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -152762,15 +152790,15 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -152782,44 +152810,44 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:184
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:184
 ; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:216
 ; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:248
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:280
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:312
 ; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:344
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:376
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:376
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:72
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:64
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:40
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr5
 ; SI-NEXT:    ; kill: killed $vgpr5
 ; SI-NEXT:    ; implicit-def: $vgpr5
@@ -152907,276 +152935,280 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 24, v54
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 24, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v54
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v60
 ; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
-; SI-NEXT:    v_or_b32_e32 v5, v6, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v6, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v6, v5, v6
+; SI-NEXT:    v_or_b32_e32 v12, v5, v9
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v55, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v55, 16, v12
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v12, v5, v10
+; SI-NEXT:    v_or_b32_e32 v16, v5, v9
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v12
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v14, v5, v10
+; SI-NEXT:    v_or_b32_e32 v20, v5, v9
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v14
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v20, v5, v10
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v20
+; SI-NEXT:    v_or_b32_e32 v22, v5, v9
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v22, v5, v10
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v22
+; SI-NEXT:    v_or_b32_e32 v24, v5, v9
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v24, v5, v10
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v24
+; SI-NEXT:    v_or_b32_e32 v28, v5, v9
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v28
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v28, v5, v10
+; SI-NEXT:    v_or_b32_e32 v29, v5, v9
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v28
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v29
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
@@ -153261,140 +153293,147 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v27
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v27, v5, v10
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v27
+; SI-NEXT:    v_or_b32_e32 v27, v5, v9
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
 ; SI-NEXT:    ; implicit-def: $vgpr27
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v8
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v6
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v8, v5, v8
+; SI-NEXT:    v_or_b32_e32 v6, v5, v6
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v8
-; SI-NEXT:    ; implicit-def: $vgpr8
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
+; SI-NEXT:    ; implicit-def: $vgpr6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v30
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v30, v5, v10
+; SI-NEXT:    v_or_b32_e32 v30, v5, v9
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v30
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v30
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v31
-; SI-NEXT:    ; implicit-def: $vgpr31
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v40
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    v_or_b32_e32 v10, v10, v9
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    v_or_b32_e32 v9, v57, v9
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v9, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v5, v19, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v53
-; SI-NEXT:    v_or_b32_e32 v9, v5, v26
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v9
+; SI-NEXT:    v_or_b32_e32 v19, v5, v26
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
 ; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr9
+; SI-NEXT:    ; implicit-def: $vgpr19
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v29, v5
+; SI-NEXT:    v_or_b32_e32 v5, v31, v5
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v15
@@ -153403,43 +153442,15 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v16, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v42
-; SI-NEXT:    v_or_b32_e32 v7, v5, v38
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v51, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v32
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v40
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v13, v10
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v49
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v19, v57, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v60
-; SI-NEXT:    v_or_b32_e32 v16, v13, v61
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v43
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v23, v62, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v58
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v17, v13
-; SI-NEXT:    v_and_b32_e32 v17, 0xff, v46
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT:    v_or_b32_e32 v18, v3, v17
-; SI-NEXT:    v_and_b32_e32 v3, 0xff, v4
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v58
+; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_or_b32_e32 v23, v17, v15
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v46
+; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_or_b32_e32 v18, v3, v15
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v63
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v29, 16, v2
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
@@ -153531,78 +153542,96 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v25
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    v_or_b32_e32 v11, v41, v5
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v47
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    v_or_b32_e32 v15, v5, v44
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v36
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v59
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    v_or_b32_e32 v5, v45, v5
-; SI-NEXT:    v_or_b32_e32 v17, v63, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
+; SI-NEXT:    v_or_b32_e32 v17, v4, v3
 ; SI-NEXT:    ; implicit-def: $vgpr15
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; kill: killed $vgpr2
+; SI-NEXT:    ; implicit-def: $vgpr31
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr2
+; SI-NEXT:    ; implicit-def: $vgpr4
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_or_b32_e32 v5, v33, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v42
+; SI-NEXT:    v_or_b32_e32 v7, v5, v38
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_or_b32_e32 v5, v51, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_or_b32_e32 v5, v8, v5
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_or_b32_e32 v11, v41, v5
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v8, v5, v44
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v36
+; SI-NEXT:    v_or_b32_e32 v14, v13, v61
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v43
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT:    v_or_b32_e32 v5, v45, v5
+; SI-NEXT:    v_or_b32_e32 v13, v62, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v14
 ; SI-NEXT:    ; implicit-def: $vgpr7
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr21
-; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr14
+; SI-NEXT:    ; implicit-def: $vgpr8
 ; SI-NEXT:    ; implicit-def: $vgpr25
 ; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; kill: killed $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr62
-; SI-NEXT:    ; implicit-def: $vgpr3
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:  .LBB88_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB88_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v2, v2, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v59
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v63
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_or_b32_e32 v2, v2, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v59
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x300, v2
-; SI-NEXT:    v_or_b32_e32 v4, v63, v4
+; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v4, v2
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v58
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v56
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v46
@@ -153625,359 +153654,358 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v40
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v54
-; SI-NEXT:    v_or_b32_e32 v5, v6, v5
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 8, v54
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v49
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT:    v_or_b32_e32 v6, v57, v6
+; SI-NEXT:    v_or_b32_e32 v9, v57, v9
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT:    v_or_b32_e32 v5, v6, v5
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v47
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_or_b32_e32 v5, v9, v5
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v47
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v36
-; SI-NEXT:    v_or_b32_e32 v6, v44, v6
+; SI-NEXT:    v_or_b32_e32 v9, v44, v9
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v45, v10
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v10, v6
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v32
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_or_b32_e32 v9, v10, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v14
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v11, 8, v31
-; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; SI-NEXT:    v_or_b32_e32 v8, v8, v10
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v25
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
+; SI-NEXT:    v_or_b32_e32 v10, v41, v10
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT:    v_or_b32_e32 v8, v10, v8
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v42
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v21
+; SI-NEXT:    v_or_b32_e32 v10, v38, v10
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
-; SI-NEXT:    v_or_b32_e32 v11, v41, v11
+; SI-NEXT:    v_or_b32_e32 v11, v51, v11
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v42
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v21
-; SI-NEXT:    v_or_b32_e32 v11, v38, v11
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
-; SI-NEXT:    v_or_b32_e32 v12, v51, v12
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v15
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT:    v_or_b32_e32 v7, v7, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v7, v7, v11
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s7, v6
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT:    v_or_b32_e32 v11, v33, v11
+; SI-NEXT:    v_or_b32_e32 v11, v11, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v53
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_or_b32_e32 v7, v26, v7
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v16, v12
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; SI-NEXT:    v_or_b32_e32 v12, v31, v12
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v7
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v53
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 8, v13
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_or_b32_e32 v7, v13, v7
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    v_or_b32_e32 v7, v26, v7
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT:    v_or_b32_e32 v13, v29, v13
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_or_b32_e32 v13, v19, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 8, v14
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v30
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v14, v7
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT:    v_or_b32_e32 v9, v9, v14
-; SI-NEXT:    v_or_b32_e32 v14, v9, v7
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v30
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v7, v9, v7
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v15, v9
-; SI-NEXT:    v_or_b32_e32 v15, v9, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v14, v15, v14
+; SI-NEXT:    v_or_b32_e32 v14, v14, v7
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 8, v9
-; SI-NEXT:    v_or_b32_e32 v7, v9, v7
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v15, 8, v15
+; SI-NEXT:    v_or_b32_e32 v7, v15, v7
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v16, v9
-; SI-NEXT:    v_or_b32_e32 v16, v9, v7
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v8
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_or_b32_e32 v15, v15, v7
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_or_b32_e32 v17, v8, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_or_b32_e32 v7, v16, v7
+; SI-NEXT:    v_or_b32_e32 v16, v7, v6
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_or_b32_e32 v18, v8, v7
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_or_b32_e32 v7, v17, v7
+; SI-NEXT:    v_or_b32_e32 v17, v7, v6
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v27
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_or_b32_e32 v19, v8, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_or_b32_e32 v7, v18, v7
+; SI-NEXT:    v_or_b32_e32 v18, v7, v6
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_or_b32_e32 v20, v8, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v13
-; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v17
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v1, v7, v1
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_or_b32_e32 v21, v7, v1
+; SI-NEXT:    v_or_b32_e32 v7, v19, v7
+; SI-NEXT:    v_or_b32_e32 v19, v7, v6
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_or_b32_e32 v1, v6, v1
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_or_b32_e32 v20, v6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v21
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT:    v_or_b32_e32 v1, v7, v1
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; SI-NEXT:    v_or_b32_e32 v1, v6, v1
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_or_b32_e32 v22, v7, v1
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_or_b32_e32 v21, v6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v1, v7, v1
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v1, v6, v1
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_or_b32_e32 v23, v7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_or_b32_e32 v22, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT:    v_or_b32_e32 v1, v7, v1
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; SI-NEXT:    v_or_b32_e32 v1, v6, v1
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_or_b32_e32 v24, v7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_or_b32_e32 v23, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v1, v7, v1
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v1, v6, v1
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_or_b32_e32 v25, v7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v25
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v21
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_or_b32_e32 v24, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT:    v_or_b32_e32 v1, v7, v1
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; SI-NEXT:    v_or_b32_e32 v1, v6, v1
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_or_b32_e32 v26, v7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v12
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v16
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v20
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v24
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_or_b32_e32 v25, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v10
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v13
+; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v16
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v19
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s7, v22
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v25
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v19
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v27, v7, v1
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v26, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
-; SI-NEXT:    v_or_b32_e32 v24, v25, v24
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v28, v7, v1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v27, v6, v1
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s7, v9
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v12
+; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v15
+; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v18
+; SI-NEXT:    v_add_i32_e32 v18, vcc, s7, v21
+; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v24
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s7, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s7, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s7, v10
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v14
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s7, v18
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s7, v22
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v26
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s7, v8
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v11
+; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v14
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s7, v17
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v20
+; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v23
+; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v26
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
-; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v11
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v15
-; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v19
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s7, v23
-; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v27
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_or_b32_e32 v23, v28, v23
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v23, v27, v23
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, s7, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v21
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
+; SI-NEXT:    v_or_b32_e32 v24, v25, v24
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
 ; SI-NEXT:    v_or_b32_e32 v24, v25, v24
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
@@ -153988,7 +154016,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v25, v26, v25
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -154017,7 +154045,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v27, v28, v27
 ; SI-NEXT:    v_or_b32_e32 v26, v27, v26
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s7, v26
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
@@ -154053,7 +154081,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v29, v30, v29
 ; SI-NEXT:    v_or_b32_e32 v28, v29, v28
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, s7, v28
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
@@ -154089,7 +154117,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v31, v32, v31
 ; SI-NEXT:    v_or_b32_e32 v30, v31, v30
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, s7, v30
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
@@ -154127,7 +154155,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, s7, v32
 ; SI-NEXT:    v_and_b32_e32 v33, 0xffff0000, v32
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v32, 0xffff0000, v31
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v30
@@ -154137,27 +154165,27 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v30, 0xffff0000, v29
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff0000, v28
 ; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff0000, v27
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff0000, v26
 ; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff0000, v25
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff0000, v24
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff0000, v23
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v22, 0xffff0000, v21
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff0000, v18
@@ -154168,78 +154196,78 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v17
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff0000, v16
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff0000, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff0000, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v7
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff0000, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff0000, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff0000, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v3
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff0000, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff0000, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v29, 16, v1
+; SI-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB88_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154257,7 +154285,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 4, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154268,7 +154296,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v52
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154277,8 +154305,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 12, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154288,7 +154316,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 16, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v50
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154297,8 +154325,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 20, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154308,7 +154336,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 24, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154317,8 +154345,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 28, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154328,7 +154356,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154337,8 +154365,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 36, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154348,7 +154376,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 40, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v37
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154358,7 +154386,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154377,7 +154405,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 52, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154388,7 +154416,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 56, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v34
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154397,8 +154425,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154408,8 +154436,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v33
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154417,8 +154445,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154429,7 +154457,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v28
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154437,7 +154465,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -154448,8 +154476,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v24
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154457,8 +154485,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154469,7 +154497,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v22
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154489,7 +154517,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v20
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -154508,35 +154536,35 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v5
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v14
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v12
 ; SI-NEXT:    v_alignbit_b32 v1, v1, v2, 16
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v19
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v9
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v10
 ; SI-NEXT:    v_alignbit_b32 v1, v1, v2, 16
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v23
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v13
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v12
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v32
 ; SI-NEXT:    v_alignbit_b32 v1, v1, v2, 16
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x74, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v18
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v13
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v23
 ; SI-NEXT:    v_alignbit_b32 v1, v1, v2, 16
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x78, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v17
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v6
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v29
 ; SI-NEXT:    v_alignbit_b32 v1, v1, v2, 16
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x7c, v0
 ; SI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
@@ -154611,39 +154639,39 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:144
-; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:152
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:136
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:144
+; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:152
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:160
 ; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:168
 ; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:176
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:184
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
 ; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v1
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v25
 ; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
-; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v3
 ; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v5
 ; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v7
-; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v9
 ; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
-; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v13
-; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v35, 8, v17
 ; VI-NEXT:    v_lshlrev_b16_e32 v36, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v21
 ; VI-NEXT:    v_lshlrev_b16_e32 v34, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v27
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
@@ -154657,46 +154685,47 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v27
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
 ; VI-NEXT:    ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v37
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v38
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v39
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v48
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v49
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v50
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v51
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
@@ -154713,7 +154742,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v43
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:208
@@ -154721,15 +154750,15 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
@@ -154752,20 +154781,20 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -154778,20 +154807,20 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -154799,17 +154828,17 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
@@ -154825,17 +154854,17 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:316
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
@@ -154850,44 +154879,44 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v3, off, s[0:3], s32 offset:376
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:348
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v1
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v3
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:372
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
 ; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -154913,31 +154942,31 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v4, v4, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr54
 ; VI-NEXT:    ; implicit-def: $vgpr55
 ; VI-NEXT:    ; implicit-def: $vgpr40
 ; VI-NEXT:    ; implicit-def: $vgpr41
-; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr36
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v34 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr34
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr52
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr37
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -154956,35 +154985,35 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_or_b32_sdwa v10, v63, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v61, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_or_b32_sdwa v12, v58, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v59, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v14, v45, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    ; implicit-def: $vgpr45
 ; VI-NEXT:    s_waitcnt vmcnt(2)
@@ -154993,26 +155022,26 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v63, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr62
-; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v60, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v10, v57, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v56, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v58, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -155021,35 +155050,35 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v13, v44, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -155064,20 +155093,20 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -155085,189 +155114,189 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v31, v31, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    v_or_b32_sdwa v31, v31, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr49
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v32, v32, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v30, v30, v37 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v30, v30, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v30, v30, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v30, v30, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v31, v31, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v31, v31, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    ; kill: killed $vgpr32
@@ -155372,7 +155401,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr51
 ; VI-NEXT:  .LBB88_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB88_4
@@ -155380,53 +155409,51 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v18, 0x300
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
 ; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    v_add_u16_sdwa v4, v0, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(12)
 ; VI-NEXT:    v_or_b32_sdwa v29, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(13)
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v4, v0, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_sdwa v0, v2, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v29, 0x300, v29
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v2, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v0, v2, v0
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v2, v52, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v51, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v50, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v2, v2, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -155436,19 +155463,19 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v1, 0x300, v1
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v4
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v3, v49, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v3, v3, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
-; VI-NEXT:    v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v4, v37, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v4, 0x300, v4
 ; VI-NEXT:    v_or_b32_e32 v3, v4, v3
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
@@ -155456,14 +155483,13 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
 ; VI-NEXT:    v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v5, 3, v5
 ; VI-NEXT:    v_or_b32_sdwa v5, v35, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v5, 0x300, v5
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
 ; VI-NEXT:    v_or_b32_sdwa v4, v36, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v4, v4, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -155480,7 +155506,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v32, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_sdwa v32, v32, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -155488,7 +155514,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; VI-NEXT:    v_or_b32_e32 v28, v28, v32
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -155496,79 +155522,94 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v33, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v33, v33, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v27, 0x300, v27
 ; VI-NEXT:    v_or_b32_e32 v27, v27, v33
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v34, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v34, v34, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; VI-NEXT:    v_or_b32_e32 v26, v26, v34
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v35, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v35, v35, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v25, 0x300, v25
+; VI-NEXT:    v_or_b32_e32 v25, v25, v35
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v36, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v25, 0x300, v25
-; VI-NEXT:    v_or_b32_e32 v25, v25, v35
+; VI-NEXT:    v_add_u16_sdwa v36, v36, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v36, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_sdwa v36, v36, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
+; VI-NEXT:    v_or_b32_e32 v24, v24, v36
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v24, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v37, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v37, v37, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
 ; VI-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
-; VI-NEXT:    v_or_b32_e32 v24, v24, v36
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
+; VI-NEXT:    v_or_b32_e32 v23, v23, v37
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v8, 3, v8
 ; VI-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v7, v8, v7
-; VI-NEXT:    v_add_u16_e32 v8, 3, v61
+; VI-NEXT:    v_add_u16_e32 v8, 3, v63
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v9, 3, v62
@@ -155577,30 +155618,30 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; VI-NEXT:    v_or_b32_e32 v8, v9, v8
-; VI-NEXT:    v_add_u16_e32 v9, 3, v63
+; VI-NEXT:    v_add_u16_e32 v9, 3, v61
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v10, 3, v59
+; VI-NEXT:    v_add_u16_e32 v10, 3, v60
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v9, v9, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:    v_add_u16_e32 v10, 3, v60
+; VI-NEXT:    v_add_u16_e32 v10, 3, v57
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v11, 3, v57
+; VI-NEXT:    v_add_u16_e32 v11, 3, v56
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v10, v10, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_or_b32_e32 v10, v11, v10
-; VI-NEXT:    v_add_u16_e32 v11, 3, v58
+; VI-NEXT:    v_add_u16_e32 v11, 3, v59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v12, 3, v56
+; VI-NEXT:    v_add_u16_e32 v12, 3, v58
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; VI-NEXT:    v_or_b32_e32 v11, v12, v11
@@ -155609,7 +155650,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v13, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_or_b32_e32 v12, v13, v12
@@ -155618,7 +155659,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 3, v44
 ; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v13, v13, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_or_b32_e32 v13, v14, v13
@@ -155630,35 +155671,35 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; VI-NEXT:    v_or_b32_e32 v14, v15, v14
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v15, 3, v15
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v15, v15, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    v_or_b32_e32 v15, v16, v15
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    v_or_b32_sdwa v17, v19, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -155667,54 +155708,43 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v16, v19, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v19, 0x300, v20
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_e32 v16, v19, v16
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
-; VI-NEXT:    v_or_b32_sdwa v30, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v30, v39, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v30, 0x300, v30
-; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
-; VI-NEXT:    v_or_b32_sdwa v31, v50, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v31, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v31, 0x300, v31
-; VI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
-; VI-NEXT:    v_or_b32_sdwa v21, v37, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v37, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v40, v21, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_sdwa v37, v37, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v29, v29, v40
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v23, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
-; VI-NEXT:    v_or_b32_e32 v23, v23, v37
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v38, v38, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v20, 3, v20
-; VI-NEXT:    v_or_b32_sdwa v20, v48, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_sdwa v55, v20, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v38, v38, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v30, v30, v55
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v55, v20, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; VI-NEXT:    v_or_b32_e32 v22, v22, v38
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v30, v30, v55
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v39, 3, v39
 ; VI-NEXT:    v_or_b32_sdwa v39, v48, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
@@ -155736,7 +155766,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v53, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v39, 3, v39
 ; VI-NEXT:    v_or_b32_sdwa v39, v49, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -155838,17 +155868,16 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
 ; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:136
+; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:136
 ; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:144
 ; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:152
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:160
 ; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:168
 ; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:176
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:184
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v27
-; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v3
@@ -155856,81 +155885,81 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v7
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v11
-; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v13
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v38, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v17
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v35, 8, v19
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v21
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v23
-; GFX9-NEXT:    s_waitcnt vmcnt(27)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v25
 ; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v10
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v14
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v25
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v37
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v39
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v48
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v49
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
@@ -155938,7 +155967,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
@@ -155955,7 +155984,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v43
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
@@ -155964,15 +155993,15 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
@@ -155996,20 +156025,20 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -156023,20 +156052,20 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -156045,17 +156074,17 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
@@ -156082,7 +156111,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
@@ -156102,48 +156131,48 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:372
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:100
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB88_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -156152,9 +156181,9 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s6
@@ -156183,10 +156212,10 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr51
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v3, v4, v3, s6
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
@@ -156197,7 +156226,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v5, v6, v5, s6
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr34
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -156212,93 +156241,93 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v7, v8, v7, s6
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v58, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v15, v43, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    v_or_b32_sdwa v15, v42, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v63, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr61
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v57, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v43, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v16, v17, v16, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -156313,20 +156342,20 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v18, v19, v18, s6
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s6
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v20, v21, v20, s6
@@ -156334,58 +156363,58 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s6
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v22, v23, v22, s6
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
@@ -156403,22 +156432,22 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
 ; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr53
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr37
 ; GFX9-NEXT:    ; kill: killed $vgpr37
 ; GFX9-NEXT:    ; implicit-def: $vgpr37
@@ -156515,7 +156544,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v30, v31, v30, s6
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr49
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -156627,27 +156656,27 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_cbranch_execz .LBB88_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
@@ -156655,19 +156684,20 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
 ; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    s_waitcnt vmcnt(20)
+; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v2
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v3
-; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    v_perm_b32 v0, v2, v0, s6
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(17)
+; GFX9-NEXT:    s_waitcnt vmcnt(16)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
@@ -156692,7 +156722,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
-; GFX9-NEXT:    v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v4, 3, v4
@@ -156718,7 +156748,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v36, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
@@ -156742,11 +156772,11 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v37, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v37, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
-; GFX9-NEXT:    v_or_b32_sdwa v21, v39, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v33, 0x300, v21
 ; GFX9-NEXT:    v_add_u16_e32 v34, 0x300, v23
 ; GFX9-NEXT:    v_perm_b32 v29, v34, v29, s6
@@ -156754,16 +156784,17 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_perm_b32 v6, v7, v6, s6
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v38, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
@@ -156773,7 +156804,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v39, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
@@ -156783,7 +156814,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v48, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -156796,45 +156827,45 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v62
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v59
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v60
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v58
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v57
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v60
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v56
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v57
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v56
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v58
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v46
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v46
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
@@ -156843,63 +156874,63 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 3, v44
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v43
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v15, 3, v43
+; GFX9-NEXT:    v_add_u16_e32 v15, 3, v42
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v16
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v18
 ; GFX9-NEXT:    v_perm_b32 v17, v17, v20, s6
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v19
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v16, v18, v16, s6
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v49, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v20
 ; GFX9-NEXT:    v_perm_b32 v30, v33, v30, s6
@@ -156907,7 +156938,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v50, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v52, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -156917,14 +156948,14 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v51, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v23, 0x300, v51
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v52, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v53, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -156934,7 +156965,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v53, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v53
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -156947,7 +156978,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v55, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v55
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -156955,7 +156986,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v40, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v50, 0x300, v40
 ; GFX9-NEXT:    v_perm_b32 v21, v50, v21, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -156963,14 +156994,14 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v41, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v41
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v42, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v51, 0x300, v42
 ; GFX9-NEXT:    v_perm_b32 v20, v51, v20, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -156978,7 +157009,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v43, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v43
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -160935,17 +160966,17 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; VI-NEXT:    v_mov_b32_e32 v45, v62
 ; VI-NEXT:    v_or_b32_sdwa v16, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v3, v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v17, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_mov_b32_e32 v54, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_mov_b32_e32 v32, v1
 ; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v54, v22
 ; VI-NEXT:    v_mov_b32_e32 v41, v24
 ; VI-NEXT:    v_or_b32_sdwa v18, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
@@ -162182,8 +162213,8 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX9-NEXT:    s_cbranch_vccnz .LBB89_5
 ; GFX9-NEXT:  ; %bb.4: ; %cmp.true
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_add_i32 s28, s28, 3
 ; GFX9-NEXT:    s_and_b32 s4, s28, 0xff
 ; GFX9-NEXT:    s_lshl_b32 s5, s29, 8
@@ -162216,7 +162247,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -162266,9 +162297,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX9-NEXT:    s_or_b32 s7, s8, s7
 ; GFX9-NEXT:    s_and_b32 s8, s16, 0xff
 ; GFX9-NEXT:    s_or_b32 s8, s9, s8
+; GFX9-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
-; GFX9-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX9-NEXT:    s_or_b32 s9, s10, s9
 ; GFX9-NEXT:    s_addk_i32 s4, 0x300
 ; GFX9-NEXT:    s_addk_i32 s5, 0x300
@@ -162301,11 +162332,11 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX9-NEXT:    v_add_u32_e32 v1, 0x300, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v6, v1, 16, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -162316,11 +162347,11 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX9-NEXT:    v_add_u32_e32 v1, 0x300, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v7, v1, 16, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -162451,9 +162482,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v2, 0x300, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
 ; GFX9-NEXT:    v_or_b32_sdwa v51, v34, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -162530,22 +162561,21 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v43, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v31, 0x300, v0
 ; GFX9-NEXT:    v_add_u32_e32 v52, 0x300, v43
 ; GFX9-NEXT:    v_and_b32_e32 v31, 0xffff, v31
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v44, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v33
@@ -162682,8 +162712,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:324
 ; GFX11-TRUE16-NEXT:    ; meta instruction
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:320
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v6 :: v_dual_mov_b32 v33, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:316
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v89, off, s32 offset:312
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v92, off, s32 offset:308
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v91, off, s32 offset:304
@@ -162733,72 +162764,56 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:120
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:112
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:64
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:44
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:24
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v86, v30 :: v_dual_mov_b32 v81, v29
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v84, v28 :: v_dual_mov_b32 v83, v27
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v85, v26 :: v_dual_mov_b32 v80, v24
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v69, v25 :: v_dual_mov_b32 v82, v22
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v71, v23 :: v_dual_mov_b32 v64, v21
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v67, v20 :: v_dual_mov_b32 v68, v19
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v70, v18 :: v_dual_mov_b32 v55, v16
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v17 :: v_dual_mov_b32 v65, v15
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v66, v14 :: v_dual_mov_b32 v49, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v12 :: v_dual_mov_b32 v54, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v11 :: v_dual_mov_b32 v36, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v8 :: v_dual_mov_b32 v37, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v38, v7 :: v_dual_mov_b32 v35, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, v5 :: v_dual_mov_b32 v39, v2
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v32, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB89_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -162856,20 +162871,20 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v86
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v85
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v83
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v86
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v98
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v84
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v85
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v103
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v87
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v102
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v101
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -162877,48 +162892,48 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v13, v14
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v99
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v96
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v116
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v97
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v112
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v3, 16, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v12, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v14, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v16, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v112
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v103
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v100
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v131
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v118
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v128
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v115
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v146
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v133
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v115
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v116
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v113
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v119
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v14, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v130
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v129
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v133
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v19
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v135
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v102
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v117
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v13
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v145
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v144
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v145
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v18, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v14
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v14, v3, 16, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v132
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v132
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xff, v117
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v114
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v135
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xff, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v129
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v15, v15, 16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v144
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v20, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v16
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v16, v18, 16, v19
@@ -163263,84 +163278,84 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v147
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 0x300, v0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v145
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v146
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v144
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v144
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v145
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v135
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v134
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v129
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v117
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v117, 0x300, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v119, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v133
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v131
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v131, 0x300, v2
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v132, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v129
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v146
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v114, 0x300, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v135
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v129, 0x300, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v130
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v129, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v128
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v128, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v119
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v102
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v117
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v131
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v102, 0x300, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v128
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v117, 0x300, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v116
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v115
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v113
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v116
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v114
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v112
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v112
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v103
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v102
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v101
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v99
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v100
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v98
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v86
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v87
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v96
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v55, 3, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v97
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v9
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v98
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v85
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v87
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v85
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v84
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 0x300, v5
@@ -163446,16 +163461,16 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v70, 16, v32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v13, 16, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v119
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v17
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v20, v20, 16, v34
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v16, v16, 16, v32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v129
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v18, v114, 16, v33
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v128
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v18, v129, 16, v33
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v21, v21, 16, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v176
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v17, v102, 16, v32
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v17, v117, 16, v32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v164
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v22
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
@@ -163474,7 +163489,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v15, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v19, v117, 16, v19
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v19, v131, 16, v19
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v24, v167, 16, v24
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v27, v45, 16, v32
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v28, v57, 16, v33
@@ -166444,8 +166459,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; VI-NEXT:    ; implicit-def: $vgpr35
 ; VI-NEXT:    ; implicit-def: $vgpr45
@@ -167623,13 +167638,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 8, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 12, v0
@@ -167721,13 +167737,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 52, v0
@@ -167747,13 +167764,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 56, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v16, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 60, v0
@@ -167773,13 +167791,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 64, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v18, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x44, v0
@@ -167799,13 +167818,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x48, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v20, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x4c, v0
@@ -167824,13 +167844,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x50, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x54, v0
@@ -167849,13 +167870,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x58, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x5c, v0
@@ -167872,13 +167894,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x60, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x64, v0
@@ -167898,13 +167921,14 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x68, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x6c, v0
@@ -167982,8 +168006,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_mov_b32_e32 v46, v15
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    ; kill: killed $vgpr33
@@ -168168,7 +168192,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(34)
+; GFX9-NEXT:    s_waitcnt vmcnt(33)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v15
 ; GFX9-NEXT:    ; implicit-def: $vgpr15
 ; GFX9-NEXT:    ; kill: killed $vgpr15
@@ -168193,7 +168217,6 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(36)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 24, v32
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
@@ -168622,7 +168645,6 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
 ; GFX9-NEXT:    v_cndmask_b32_e32 v15, v16, v17, vcc
 ; GFX9-NEXT:    v_perm_b32 v33, v15, v25, s7
-; GFX9-NEXT:    s_waitcnt vmcnt(52)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v15, 16, v32
 ; GFX9-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
 ; GFX9-NEXT:    v_bfe_u32 v16, v15, 16, 1
@@ -180598,15 +180620,18 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v32, v30
 ; SI-NEXT:    v_mov_b32_e32 v44, v19
 ; SI-NEXT:    v_mov_b32_e32 v43, v17
+; SI-NEXT:    v_mov_b32_e32 v32, v14
+; SI-NEXT:    v_mov_b32_e32 v14, v12
+; SI-NEXT:    v_mov_b32_e32 v12, v10
 ; SI-NEXT:    v_mov_b32_e32 v41, v7
 ; SI-NEXT:    v_mov_b32_e32 v55, v5
 ; SI-NEXT:    v_mov_b32_e32 v54, v3
 ; SI-NEXT:    v_mov_b32_e32 v51, v1
-; SI-NEXT:    v_mov_b32_e32 v30, v0
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:392
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    s_waitcnt expcnt(1)
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:392
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:20
@@ -180616,13 +180641,13 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:60
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:116
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:124
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:132
 ; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:140
@@ -180633,153 +180658,147 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:180
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:188
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:128
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:136
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v4
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v18
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v20
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v22
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v24
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v26
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v28
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v30
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 8, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v14, 8, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 8, v32
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v17
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v49
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v50
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v52
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v42
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v25
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v27
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v40
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v5
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v42
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v33
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:144
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v17
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v34
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:136
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v23
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v35
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v31
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v36
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v31
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
+; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    ; implicit-def: $vgpr31
+; SI-NEXT:    ; implicit-def: $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr27
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:160
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:168
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:176
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v38
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v23
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v39
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v29
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v48
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:176
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 8, v53
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v40
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v4
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v10
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v14
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v16
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:144
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v18
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v20
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v22
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v24
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v26
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v28
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:160
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:196
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:220
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:192
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:200
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:208
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
@@ -180799,7 +180818,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:224
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:232
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -180810,13 +180829,13 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v4
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
@@ -180857,15 +180876,15 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v45, 8, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:296
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:304
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
@@ -180908,7 +180927,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v1
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v2
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -180917,7 +180936,6 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:360
-; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -180925,35 +180943,35 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:388
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:384
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:384
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:40
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:152
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -180968,14 +180986,16 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:312
 ; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:344
 ; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:376
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:56
+; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
@@ -181080,75 +181100,83 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB92_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v51
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v2, v17
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v54
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v51
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_and_b32_e32 v48, 0xff, v50
+; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
+; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
+; SI-NEXT:    v_and_b32_e32 v28, 0xff, v28
+; SI-NEXT:    v_and_b32_e32 v30, 0xff, v30
+; SI-NEXT:    v_and_b32_e32 v34, 0xff, v34
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
+; SI-NEXT:    v_and_b32_e32 v39, 0xff, v50
+; SI-NEXT:    v_and_b32_e32 v48, 0xff, v40
 ; SI-NEXT:    v_and_b32_e32 v49, 0xff, v49
 ; SI-NEXT:    v_and_b32_e32 v52, 0xff, v52
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v42, 0xff, v42
-; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v46, 0xff, v46
 ; SI-NEXT:    v_or_b32_e32 v45, v46, v45
-; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v56, 0xff, v56
 ; SI-NEXT:    v_or_b32_e32 v56, v56, v61
-; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v57, 0xff, v57
+; SI-NEXT:    v_and_b32_e32 v47, 0xff, v47
 ; SI-NEXT:    v_or_b32_e32 v1, v57, v1
+; SI-NEXT:    v_or_b32_e32 v3, v47, v3
 ; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v2, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v54
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v2, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v55
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v2, v6
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v41
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v2, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v9
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v35
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v2, v0
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v11
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v2, v12
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v11
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v0, v2, v14
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v13
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v24
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v17, v2, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT:    v_mov_b32_e32 v8, v7
+; SI-NEXT:    v_mov_b32_e32 v7, v19
+; SI-NEXT:    v_or_b32_e32 v19, v2, v32
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v15
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    v_and_b32_e32 v35, 0xff, v7
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v53
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr19
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_or_b32_e32 v31, v2, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v17, v2, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v43
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v31
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v45
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v23, v2, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v44
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v23
@@ -181156,11 +181184,11 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v29, v2, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v31, v2, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v21
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v29
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v31
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:976 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
@@ -181172,231 +181200,227 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v25, v2, v0
+; SI-NEXT:    v_or_b32_e32 v27, v2, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v25
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v27
+; SI-NEXT:    v_cvt_f32_f16_e32 v27, v3
+; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v27, v2, v0
+; SI-NEXT:    v_or_b32_e32 v29, v2, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v27
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v29
 ; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr17
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v2, v0
+; SI-NEXT:    v_or_b32_e32 v2, v2, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
-; SI-NEXT:    v_or_b32_e32 v2, v2, v32
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v5, v5, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v5
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v1
-; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v33, v6, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v33
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v33, v45
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT:    v_or_b32_e32 v6, v6, v7
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v6, v6, v8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v6
 ; SI-NEXT:    ; implicit-def: $vgpr6
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v35, v8, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v10
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v35
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v10, v8, v0
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v8, v8, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v10
-; SI-NEXT:    ; implicit-def: $vgpr10
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v8
+; SI-NEXT:    ; implicit-def: $vgpr8
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v34, v8, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v14
-; SI-NEXT:    v_or_b32_e32 v14, v8, v53
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v16
-; SI-NEXT:    v_or_b32_e32 v16, v8, v40
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v20
+; SI-NEXT:    v_or_b32_e32 v12, v12, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v34
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v12
+; SI-NEXT:    ; implicit-def: $vgpr12
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v14, v14, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v14
 ; SI-NEXT:    ; implicit-def: $vgpr14
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v32, 0xff, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v32, v32, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_or_b32_e32 v20, v8, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v32
+; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_or_b32_e32 v18, v18, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v20
-; SI-NEXT:    ; implicit-def: $vgpr20
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v18
+; SI-NEXT:    ; implicit-def: $vgpr18
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v8, v8, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v22, v22, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v8
-; SI-NEXT:    ; implicit-def: $vgpr8
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v24, v12, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v26
+; SI-NEXT:    v_or_b32_e32 v24, v24, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v24
 ; SI-NEXT:    ; implicit-def: $vgpr24
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v26, v12, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v28
+; SI-NEXT:    v_or_b32_e32 v26, v26, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v26
 ; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v28, v12, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v28, v28, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v28
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v12, v12, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v30, v30, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v30
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v18, v18, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v34, v34, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v18
-; SI-NEXT:    ; implicit-def: $vgpr18
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v34
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_or_b32_e32 v16, v16, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v16
+; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v22, v22, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v20, v20, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v22
-; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v20
+; SI-NEXT:    ; implicit-def: $vgpr20
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v32, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v32, v32, v0
+; SI-NEXT:    v_or_b32_e32 v35, v35, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v32
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v35
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v36, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v36, v36, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v36
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v37, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v37, v37, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v37
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v38, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v38, v38, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v38
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v39, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v39, v39, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v39
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v7
 ; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v48, v48, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v48
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v49, v49, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v49
 ; SI-NEXT:    ; implicit-def: $vgpr49
@@ -181406,21 +181430,21 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v50, v50, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v50
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v9
 ; SI-NEXT:    ; implicit-def: $vgpr9
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v54, 0xff, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
@@ -181433,24 +181457,24 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v52, v52, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v52
 ; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v11
 ; SI-NEXT:    ; implicit-def: $vgpr11
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v53, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v53, v53, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
@@ -181459,7 +181483,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v55, 0xff, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v55, v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
@@ -181481,29 +181505,33 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v40, v40, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v39, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v48, v40
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v15
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v42, v42, v15
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v42
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v38, v42
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v43, 0xff, v15
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v43, v43, v15
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v43
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v21
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v34, v15
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v36, v15
 ; SI-NEXT:    ; implicit-def: $vgpr15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v44, 0xff, v21
@@ -181511,96 +181539,69 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v44, v44, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v44
-; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v46, 0xff, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v46, v46, v58
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v46
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v46
 ; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v58, 0xff, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v58, v58, v59
-; SI-NEXT:    v_cvt_f32_f16_e32 v27, v58
+; SI-NEXT:    v_cvt_f32_f16_e32 v29, v58
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v59, 0xff, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v59, v59, v60
-; SI-NEXT:    v_cvt_f32_f16_e32 v36, v59
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v59
 ; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v60, 0xff, v21
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v60, v60, v62
-; SI-NEXT:    v_cvt_f32_f16_e32 v33, v60
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v60
 ; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v61, 0xff, v21
-; SI-NEXT:    v_or_b32_e32 v21, v61, v47
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v21
-; SI-NEXT:    ; implicit-def: $vgpr21
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v25, v61, v25
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v25
+; SI-NEXT:    ; implicit-def: $vgpr25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v61, 0xff, v47
-; SI-NEXT:    v_and_b32_e32 v47, 0xff, v19
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v61, 0xff, v21
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v61, v61, v63
-; SI-NEXT:    v_or_b32_e32 v3, v47, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v29, v61
-; SI-NEXT:    v_cvt_f32_f16_e32 v25, v3
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v61
 ; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v55
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v13
-; SI-NEXT:    ; implicit-def: $vgpr13
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v41
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v0, v43
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v44
+; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
@@ -181748,68 +181749,92 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v55
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v13
+; SI-NEXT:    ; implicit-def: $vgpr13
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v41
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:  .LBB92_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB92_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v19
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v47
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v4, v3, v2
+; SI-NEXT:    v_or_b32_e32 v5, v3, v2
 ; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v5, v1, v2
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v29, vcc, 3, v19
+; SI-NEXT:    v_and_b32_e32 v29, 0xff, v29
+; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
+; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v34, vcc, 3, v34
+; SI-NEXT:    v_and_b32_e32 v34, 0xff, v34
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; SI-NEXT:    v_and_b32_e32 v30, 0xff, v30
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
 ; SI-NEXT:    v_and_b32_e32 v28, 0xff, v28
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
 ; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
 ; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v35, vcc, 3, v35
-; SI-NEXT:    v_and_b32_e32 v35, 0xff, v35
-; SI-NEXT:    v_mov_b32_e32 v1, v17
+; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
+; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_mov_b32_e32 v17, v43
 ; SI-NEXT:    v_mov_b32_e32 v19, v44
+; SI-NEXT:    v_add_i32_e32 v47, vcc, 3, v19
+; SI-NEXT:    v_and_b32_e32 v47, 0xff, v47
 ; SI-NEXT:    v_add_i32_e32 v57, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v57, 0xff, v57
-; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v51
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
@@ -181821,17 +181846,15 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v25, v47, v2
-; SI-NEXT:    v_add_i32_e32 v47, vcc, 3, v19
-; SI-NEXT:    v_and_b32_e32 v47, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v25, v25, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v27, v62, v2
+; SI-NEXT:    v_or_b32_e32 v31, v62, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v56
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v29, v61, v2
+; SI-NEXT:    v_or_b32_e32 v33, v61, v2
 ; SI-NEXT:    v_add_i32_e32 v56, vcc, 3, v17
 ; SI-NEXT:    v_and_b32_e32 v56, 0xff, v56
 ; SI-NEXT:    v_add_i32_e32 v61, vcc, 3, v41
@@ -181842,56 +181865,66 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v31, v60, v2
+; SI-NEXT:    v_or_b32_e32 v35, v60, v2
 ; SI-NEXT:    v_add_i32_e32 v60, vcc, 3, v9
 ; SI-NEXT:    v_and_b32_e32 v60, 0xff, v60
+; SI-NEXT:    v_or_b32_e32 v12, v12, v60
+; SI-NEXT:    v_add_i32_e32 v60, vcc, s6, v25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v33, v59, v2
+; SI-NEXT:    v_or_b32_e32 v37, v59, v2
 ; SI-NEXT:    v_add_i32_e32 v59, vcc, 3, v11
 ; SI-NEXT:    v_and_b32_e32 v59, 0xff, v59
-; SI-NEXT:    v_or_b32_e32 v11, v12, v59
-; SI-NEXT:    v_add_i32_e32 v59, vcc, s6, v27
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; SI-NEXT:    v_or_b32_e32 v14, v14, v59
+; SI-NEXT:    v_add_i32_e32 v59, vcc, s6, v31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v34, v58, v2
+; SI-NEXT:    v_or_b32_e32 v38, v58, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v46
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_or_b32_e32 v36, v45, v2
+; SI-NEXT:    v_or_b32_e32 v39, v45, v2
 ; SI-NEXT:    v_add_i32_e32 v46, vcc, 3, v21
 ; SI-NEXT:    v_and_b32_e32 v46, 0xff, v46
 ; SI-NEXT:    v_add_i32_e32 v58, vcc, 3, v13
 ; SI-NEXT:    v_and_b32_e32 v58, 0xff, v58
+; SI-NEXT:    v_or_b32_e32 v32, v32, v58
+; SI-NEXT:    v_add_i32_e32 v58, vcc, s6, v33
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:980 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v37, v0, v2
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v48, v0, v2
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v38, v0, v2
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v39, v0, v2
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v42
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v48, v0, v2
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
@@ -181899,7 +181932,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -181908,7 +181941,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -181917,7 +181950,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -181926,618 +181959,622 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
-; SI-NEXT:    v_mov_b32_e32 v2, v8
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v52
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v52
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v49
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v49
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v50
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v40
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v50
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v2, v0, v2
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v27, 0xff, v27
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
+; SI-NEXT:    v_or_b32_e32 v0, v0, v27
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v29
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v22
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v20, v0, v20
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v16, v0, v16
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v34
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v18
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
+; SI-NEXT:    v_or_b32_e32 v0, v0, v30
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v8, v0, v8
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v8
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v28
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v26
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v24
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v49, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v49, 0xff, v49
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v49
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v22
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v20
+; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v20
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v18
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v40, v16
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v36, vcc, 3, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v36, 0xff, v36
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v36
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v53, v14
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v49, vcc, 3, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v49, 0xff, v49
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v49
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v50, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v50, 0xff, v50
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v50
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v10
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v52, vcc, 3, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v52, 0xff, v52
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v35
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v0, v0, v52
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v52, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v52, 0xff, v52
-; SI-NEXT:    v_or_b32_e32 v7, v7, v52
-; SI-NEXT:    v_add_i32_e32 v54, vcc, s6, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v53, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v53, 0xff, v53
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v53
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v53, 0xff, v53
+; SI-NEXT:    v_or_b32_e32 v7, v7, v53
+; SI-NEXT:    v_add_i32_e32 v54, vcc, s6, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v40, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v40, 0xff, v40
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v40
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v42, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v42, 0xff, v42
-; SI-NEXT:    v_or_b32_e32 v32, v32, v42
-; SI-NEXT:    v_add_i32_e32 v42, vcc, s6, v48
-; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v32
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v42
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v43, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v43, 0xff, v43
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v43
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v43, vcc, s6, v39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v44, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v44, 0xff, v44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v44
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v44, vcc, s6, v38
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v45, vcc, 3, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v45, 0xff, v45
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v45
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v45, vcc, s6, v37
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v45, vcc, s6, v48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v46
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v46, vcc, s6, v36
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v19, v0, v47
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v47, vcc, s6, v34
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v17, v0, v56
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v56, vcc, s6, v33
-; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v15, v0, v57
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v57, vcc, s6, v31
-; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v13, v0, v58
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v58, vcc, s6, v29
-; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
+; SI-NEXT:    v_add_i32_e32 v46, vcc, s6, v39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v9, v0, v60
-; SI-NEXT:    v_or_b32_e32 v0, v2, v61
+; SI-NEXT:    v_or_b32_e32 v0, v0, v47
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v47, vcc, s6, v38
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v56
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v56, vcc, s6, v37
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v57
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v8, v61
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v0, v6, v62
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v62, vcc, s6, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v60, vcc, s6, v25
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v57, vcc, s6, v35
 ; SI-NEXT:    v_add_i32_e32 v61, vcc, s6, v23
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
+; SI-NEXT:    v_add_i32_e32 v62, vcc, s6, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v41, vcc, s6, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v63, v0, v63
-; SI-NEXT:    v_or_b32_e32 v0, v1, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v0, v3
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v4
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v32, vcc, s6, v63
-; SI-NEXT:    v_cvt_f32_f16_e32 v32, v32
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_add_i32_e32 v55, vcc, s6, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v40, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v63, vcc, s6, v63
-; SI-NEXT:    v_cvt_f32_f16_e32 v63, v63
-; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v41, vcc, s6, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v55, vcc, s6, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v51, vcc, s6, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v53, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v31, vcc, s6, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v44, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v52, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v29, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v50, vcc, s6, v0
+; SI-NEXT:    v_add_i32_e32 v43, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v42, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v40, vcc, s6, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v49, vcc, s6, v0
+; SI-NEXT:    v_add_i32_e32 v53, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v52, vcc, s6, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v50, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v49, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v48, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v39, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v38, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v37, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v36, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v35, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v34, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v33, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v30, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v29, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v1
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v2
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v3
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v62
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v6
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v8
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v0, v10
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v0, v12
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v0, v14
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v16
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v18
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v20
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v22
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v24
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v26
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v27
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v28
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v29
+; SI-NEXT:    v_cvt_f32_f16_e32 v29, v56
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v30
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v33
-; SI-NEXT:    v_cvt_f32_f16_e32 v33, v59
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v33, v46
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v34
-; SI-NEXT:    v_cvt_f32_f16_e32 v34, v44
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v35
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v59
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v36
-; SI-NEXT:    v_cvt_f32_f16_e32 v36, v57
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v36, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v37
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v42
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v57
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v38
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v47
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v38, v42
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v39
-; SI-NEXT:    v_cvt_f32_f16_e32 v39, v53
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v47
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v48
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v45
+; SI-NEXT:    v_cvt_f32_f16_e32 v48, v53
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v49
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v50
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v40
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v43
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v45
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v27, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v31, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v12
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v7
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v9
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v11
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v13
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v15
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v17
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v60
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v19
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v31
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v46
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v21
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v23
 ; SI-NEXT:    v_cvt_f32_f16_e32 v23, v58
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v29
-; SI-NEXT:    v_cvt_f32_f16_e32 v29, v61
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v27
-; SI-NEXT:    v_cvt_f32_f16_e32 v27, v56
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v31
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v61
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v25
-; SI-NEXT:    v_cvt_f32_f16_e32 v25, v0
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v51
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v54
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v55
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v41
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_add_i32_e32 v32, vcc, s6, v12
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v60
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v62
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v12
+; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v63
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v14
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v32
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v63, vcc, s6, v63
+; SI-NEXT:    v_cvt_f32_f16_e32 v63, v63
+; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB92_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 12, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 12, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    buffer_store_dword v0, v30, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_dword v0, v10, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
@@ -182547,7 +182584,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 4, v30
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 4, v10
 ; SI-NEXT:    buffer_store_dword v0, v1, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
@@ -182558,7 +182595,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 8, v30
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 8, v10
 ; SI-NEXT:    buffer_store_dword v0, v1, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
@@ -182573,7 +182610,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 16, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182584,7 +182621,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 20, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 20, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182595,7 +182632,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 24, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 24, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182606,7 +182643,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 28, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 28, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182616,8 +182653,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v30
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182626,9 +182663,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 36, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 36, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182637,9 +182674,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 40, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 40, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182649,8 +182686,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 44, v30
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 44, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182659,9 +182696,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 48, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 48, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182670,9 +182707,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 52, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 52, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182682,8 +182719,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 56, v30
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 56, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182692,9 +182729,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 60, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 60, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182703,9 +182740,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 64, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 64, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182715,8 +182752,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v30
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182725,9 +182762,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x48, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x48, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182736,9 +182773,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182748,8 +182785,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v30
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182758,9 +182795,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182769,9 +182806,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x58, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x58, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182781,8 +182818,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x5c, v30
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x5c, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -182791,62 +182828,64 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v39
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x60, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v48
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x60, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v37
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x64, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v38
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x64, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v48
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v34
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x68, v30
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v36
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x68, v10
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v31
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v30
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v39
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v33
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v36
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v27
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v30
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v37
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v29
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v35
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v23
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x74, v30
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x74, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v29
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v5
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x78, v30
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v17
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x78, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v4
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x7c, v30
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v27
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v5
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x7c, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
@@ -182921,39 +182960,39 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:144
-; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:152
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:136
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:144
+; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:152
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:160
 ; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:168
 ; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:176
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:184
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
 ; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v1
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v25
 ; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
-; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v3
 ; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v5
 ; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v7
-; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v9
 ; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
-; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v13
-; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v35, 8, v17
 ; VI-NEXT:    v_lshlrev_b16_e32 v36, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v21
 ; VI-NEXT:    v_lshlrev_b16_e32 v34, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v27
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
@@ -182967,46 +183006,47 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v27
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
 ; VI-NEXT:    ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v37
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v38
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v39
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v48
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v49
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v50
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v51
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
@@ -183023,7 +183063,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v43
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:208
@@ -183031,15 +183071,15 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
@@ -183062,20 +183102,20 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -183088,20 +183128,20 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -183109,17 +183149,17 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
@@ -183135,17 +183175,17 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:316
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
@@ -183160,44 +183200,44 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v3, off, s[0:3], s32 offset:376
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:348
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v1
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v3
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:372
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
 ; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -183223,31 +183263,31 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v4, v4, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr54
 ; VI-NEXT:    ; implicit-def: $vgpr55
 ; VI-NEXT:    ; implicit-def: $vgpr40
 ; VI-NEXT:    ; implicit-def: $vgpr41
-; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr36
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v34 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr34
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr52
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr37
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -183266,35 +183306,35 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_or_b32_sdwa v10, v63, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v61, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_or_b32_sdwa v12, v58, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v59, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v14, v45, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    ; implicit-def: $vgpr45
 ; VI-NEXT:    s_waitcnt vmcnt(2)
@@ -183303,26 +183343,26 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v63, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr62
-; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v60, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v10, v57, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v56, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v58, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -183331,35 +183371,35 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v13, v44, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -183374,20 +183414,20 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -183395,189 +183435,189 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v31, v31, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    v_or_b32_sdwa v31, v31, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr49
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v32, v32, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v30, v30, v37 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v30, v30, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v30, v30, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v30, v30, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v31, v31, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v31, v31, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    ; kill: killed $vgpr32
@@ -183682,7 +183722,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr51
 ; VI-NEXT:  .LBB92_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB92_4
@@ -183690,53 +183730,51 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v18, 0x300
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
 ; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    v_add_u16_sdwa v4, v0, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(12)
 ; VI-NEXT:    v_or_b32_sdwa v29, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(13)
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v4, v0, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_sdwa v0, v2, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v29, 0x300, v29
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v2, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v0, v2, v0
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v2, v52, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v51, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v50, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v2, v2, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -183746,19 +183784,19 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v1, 0x300, v1
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v4
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v3, v49, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v3, v3, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
-; VI-NEXT:    v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v4, v37, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v4, 0x300, v4
 ; VI-NEXT:    v_or_b32_e32 v3, v4, v3
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
@@ -183766,14 +183804,13 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
 ; VI-NEXT:    v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v5, 3, v5
 ; VI-NEXT:    v_or_b32_sdwa v5, v35, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v5, 0x300, v5
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
 ; VI-NEXT:    v_or_b32_sdwa v4, v36, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v4, v4, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -183790,7 +183827,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v32, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_sdwa v32, v32, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -183798,7 +183835,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; VI-NEXT:    v_or_b32_e32 v28, v28, v32
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -183806,79 +183843,94 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v33, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v33, v33, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v27, 0x300, v27
 ; VI-NEXT:    v_or_b32_e32 v27, v27, v33
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v34, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v34, v34, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; VI-NEXT:    v_or_b32_e32 v26, v26, v34
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v35, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v35, v35, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v25, 0x300, v25
+; VI-NEXT:    v_or_b32_e32 v25, v25, v35
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v36, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v25, 0x300, v25
-; VI-NEXT:    v_or_b32_e32 v25, v25, v35
+; VI-NEXT:    v_add_u16_sdwa v36, v36, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v36, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_sdwa v36, v36, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
+; VI-NEXT:    v_or_b32_e32 v24, v24, v36
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v24, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v37, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v37, v37, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
 ; VI-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
-; VI-NEXT:    v_or_b32_e32 v24, v24, v36
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
+; VI-NEXT:    v_or_b32_e32 v23, v23, v37
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v8, 3, v8
 ; VI-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v7, v8, v7
-; VI-NEXT:    v_add_u16_e32 v8, 3, v61
+; VI-NEXT:    v_add_u16_e32 v8, 3, v63
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v9, 3, v62
@@ -183887,30 +183939,30 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; VI-NEXT:    v_or_b32_e32 v8, v9, v8
-; VI-NEXT:    v_add_u16_e32 v9, 3, v63
+; VI-NEXT:    v_add_u16_e32 v9, 3, v61
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v10, 3, v59
+; VI-NEXT:    v_add_u16_e32 v10, 3, v60
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v9, v9, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:    v_add_u16_e32 v10, 3, v60
+; VI-NEXT:    v_add_u16_e32 v10, 3, v57
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v11, 3, v57
+; VI-NEXT:    v_add_u16_e32 v11, 3, v56
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v10, v10, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_or_b32_e32 v10, v11, v10
-; VI-NEXT:    v_add_u16_e32 v11, 3, v58
+; VI-NEXT:    v_add_u16_e32 v11, 3, v59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v12, 3, v56
+; VI-NEXT:    v_add_u16_e32 v12, 3, v58
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; VI-NEXT:    v_or_b32_e32 v11, v12, v11
@@ -183919,7 +183971,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v13, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_or_b32_e32 v12, v13, v12
@@ -183928,7 +183980,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 3, v44
 ; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v13, v13, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_or_b32_e32 v13, v14, v13
@@ -183940,35 +183992,35 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; VI-NEXT:    v_or_b32_e32 v14, v15, v14
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v15, 3, v15
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v15, v15, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    v_or_b32_e32 v15, v16, v15
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    v_or_b32_sdwa v17, v19, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -183977,54 +184029,43 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v16, v19, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v19, 0x300, v20
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_e32 v16, v19, v16
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
-; VI-NEXT:    v_or_b32_sdwa v30, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v30, v39, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v30, 0x300, v30
-; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
-; VI-NEXT:    v_or_b32_sdwa v31, v50, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v31, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v31, 0x300, v31
-; VI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
-; VI-NEXT:    v_or_b32_sdwa v21, v37, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v37, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v40, v21, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_sdwa v37, v37, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v29, v29, v40
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v23, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
-; VI-NEXT:    v_or_b32_e32 v23, v23, v37
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v38, v38, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v20, 3, v20
-; VI-NEXT:    v_or_b32_sdwa v20, v48, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_sdwa v55, v20, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v38, v38, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v30, v30, v55
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v55, v20, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; VI-NEXT:    v_or_b32_e32 v22, v22, v38
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v30, v30, v55
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v39, 3, v39
 ; VI-NEXT:    v_or_b32_sdwa v39, v48, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
@@ -184046,7 +184087,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v53, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v39, 3, v39
 ; VI-NEXT:    v_or_b32_sdwa v39, v49, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -184148,17 +184189,16 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
 ; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:136
+; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:136
 ; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:144
 ; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:152
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:160
 ; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:168
 ; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:176
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:184
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v27
-; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v3
@@ -184166,81 +184206,81 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v7
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v11
-; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v13
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v38, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v17
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v35, 8, v19
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v21
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v23
-; GFX9-NEXT:    s_waitcnt vmcnt(27)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v25
 ; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v10
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v14
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v25
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v37
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v39
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v48
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v49
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
@@ -184248,7 +184288,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
@@ -184265,7 +184305,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v43
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
@@ -184274,15 +184314,15 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
@@ -184306,20 +184346,20 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -184333,20 +184373,20 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -184355,17 +184395,17 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
@@ -184392,7 +184432,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
@@ -184412,48 +184452,48 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:372
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:100
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB92_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -184462,9 +184502,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s6
@@ -184493,10 +184533,10 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr51
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v3, v4, v3, s6
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
@@ -184507,7 +184547,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v5, v6, v5, s6
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr34
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -184522,93 +184562,93 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v7, v8, v7, s6
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v58, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v15, v43, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    v_or_b32_sdwa v15, v42, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v63, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr61
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v57, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v43, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v16, v17, v16, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -184623,20 +184663,20 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v18, v19, v18, s6
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s6
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v20, v21, v20, s6
@@ -184644,58 +184684,58 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s6
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v22, v23, v22, s6
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
@@ -184713,22 +184753,22 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
 ; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr53
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr37
 ; GFX9-NEXT:    ; kill: killed $vgpr37
 ; GFX9-NEXT:    ; implicit-def: $vgpr37
@@ -184825,7 +184865,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v30, v31, v30, s6
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr49
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -184937,27 +184977,27 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_cbranch_execz .LBB92_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
@@ -184965,19 +185005,20 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
 ; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    s_waitcnt vmcnt(20)
+; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v2
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v3
-; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    v_perm_b32 v0, v2, v0, s6
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(17)
+; GFX9-NEXT:    s_waitcnt vmcnt(16)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
@@ -185002,7 +185043,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
-; GFX9-NEXT:    v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v4, 3, v4
@@ -185028,7 +185069,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v36, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
@@ -185052,11 +185093,11 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v37, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v37, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
-; GFX9-NEXT:    v_or_b32_sdwa v21, v39, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v33, 0x300, v21
 ; GFX9-NEXT:    v_add_u16_e32 v34, 0x300, v23
 ; GFX9-NEXT:    v_perm_b32 v29, v34, v29, s6
@@ -185064,16 +185105,17 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_perm_b32 v6, v7, v6, s6
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v38, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
@@ -185083,7 +185125,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v39, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
@@ -185093,7 +185135,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v48, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -185106,45 +185148,45 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v62
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v59
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v60
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v58
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v57
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v60
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v56
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v57
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v56
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v58
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v46
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v46
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
@@ -185153,63 +185195,63 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 3, v44
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v43
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v15, 3, v43
+; GFX9-NEXT:    v_add_u16_e32 v15, 3, v42
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v16
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v18
 ; GFX9-NEXT:    v_perm_b32 v17, v17, v20, s6
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v19
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v16, v18, v16, s6
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v49, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v20
 ; GFX9-NEXT:    v_perm_b32 v30, v33, v30, s6
@@ -185217,7 +185259,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v50, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v52, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -185227,14 +185269,14 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v51, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v23, 0x300, v51
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v52, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v53, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -185244,7 +185286,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v53, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v53
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -185257,7 +185299,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v55, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v55
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -185265,7 +185307,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v40, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v50, 0x300, v40
 ; GFX9-NEXT:    v_perm_b32 v21, v50, v21, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -185273,14 +185315,14 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v41, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v41
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v42, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v51, 0x300, v42
 ; GFX9-NEXT:    v_perm_b32 v20, v51, v20, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -185288,7 +185330,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v43, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v43
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -189146,17 +189188,17 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; VI-NEXT:    v_mov_b32_e32 v45, v62
 ; VI-NEXT:    v_or_b32_sdwa v16, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v3, v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v17, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_mov_b32_e32 v54, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_mov_b32_e32 v32, v1
 ; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v54, v22
 ; VI-NEXT:    v_mov_b32_e32 v41, v24
 ; VI-NEXT:    v_or_b32_sdwa v18, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
@@ -190393,8 +190435,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX9-NEXT:    s_cbranch_vccnz .LBB93_5
 ; GFX9-NEXT:  ; %bb.4: ; %cmp.true
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_add_i32 s28, s28, 3
 ; GFX9-NEXT:    s_and_b32 s4, s28, 0xff
 ; GFX9-NEXT:    s_lshl_b32 s5, s29, 8
@@ -190427,7 +190469,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -190477,9 +190519,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX9-NEXT:    s_or_b32 s7, s8, s7
 ; GFX9-NEXT:    s_and_b32 s8, s16, 0xff
 ; GFX9-NEXT:    s_or_b32 s8, s9, s8
+; GFX9-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
-; GFX9-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX9-NEXT:    s_or_b32 s9, s10, s9
 ; GFX9-NEXT:    s_addk_i32 s4, 0x300
 ; GFX9-NEXT:    s_addk_i32 s5, 0x300
@@ -190512,11 +190554,11 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX9-NEXT:    v_add_u32_e32 v1, 0x300, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v6, v1, 16, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -190527,11 +190569,11 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX9-NEXT:    v_add_u32_e32 v1, 0x300, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v7, v1, 16, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -190662,9 +190704,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v2, 0x300, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
 ; GFX9-NEXT:    v_or_b32_sdwa v51, v34, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -190741,22 +190783,21 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v43, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v31, 0x300, v0
 ; GFX9-NEXT:    v_add_u32_e32 v52, 0x300, v43
 ; GFX9-NEXT:    v_and_b32_e32 v31, 0xffff, v31
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v44, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v33
@@ -190893,8 +190934,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:324
 ; GFX11-TRUE16-NEXT:    ; meta instruction
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:320
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v6 :: v_dual_mov_b32 v33, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:316
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v89, off, s32 offset:312
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v92, off, s32 offset:308
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v91, off, s32 offset:304
@@ -190944,72 +190986,56 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:120
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:112
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:64
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:44
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:24
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v86, v30 :: v_dual_mov_b32 v81, v29
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v84, v28 :: v_dual_mov_b32 v83, v27
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v85, v26 :: v_dual_mov_b32 v80, v24
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v69, v25 :: v_dual_mov_b32 v82, v22
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v71, v23 :: v_dual_mov_b32 v64, v21
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v67, v20 :: v_dual_mov_b32 v68, v19
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v70, v18 :: v_dual_mov_b32 v55, v16
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v17 :: v_dual_mov_b32 v65, v15
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v66, v14 :: v_dual_mov_b32 v49, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v12 :: v_dual_mov_b32 v54, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v11 :: v_dual_mov_b32 v36, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v8 :: v_dual_mov_b32 v37, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v38, v7 :: v_dual_mov_b32 v35, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, v5 :: v_dual_mov_b32 v39, v2
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v32, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB93_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -191067,20 +191093,20 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v86
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v85
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v83
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v86
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v98
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v84
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v85
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v103
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v87
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v102
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v101
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -191088,48 +191114,48 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v13, v14
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v99
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v96
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v116
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v97
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v112
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v3, 16, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v12, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v14, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v16, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v112
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v103
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v100
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v131
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v118
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v128
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v115
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v146
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v133
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v115
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v116
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v113
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v119
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v14, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v130
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v129
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v133
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v19
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v135
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v102
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v117
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v13
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v145
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v144
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v145
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v18, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v14
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v14, v3, 16, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v132
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v132
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xff, v117
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v114
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v135
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xff, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v129
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v15, v15, 16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v144
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v20, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v16
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v16, v18, 16, v19
@@ -191474,84 +191500,84 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v147
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 0x300, v0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v145
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v146
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v144
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v144
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v145
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v135
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v134
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v129
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v117
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v117, 0x300, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v119, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v133
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v131
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v131, 0x300, v2
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v132, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v129
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v146
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v114, 0x300, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v135
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v129, 0x300, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v130
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v129, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v128
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v128, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v119
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v102
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v117
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v131
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v102, 0x300, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v128
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v117, 0x300, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v116
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v115
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v113
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v116
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v114
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v112
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v112
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v103
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v102
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v101
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v99
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v100
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v98
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v86
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v87
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v96
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v55, 3, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v97
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v9
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v98
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v85
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v87
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v85
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v84
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 0x300, v5
@@ -191657,16 +191683,16 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v70, 16, v32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v13, 16, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v119
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v17
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v20, v20, 16, v34
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v16, v16, 16, v32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v129
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v18, v114, 16, v33
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v128
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v18, v129, 16, v33
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v21, v21, 16, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v176
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v17, v102, 16, v32
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v17, v117, 16, v32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v164
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v22
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
@@ -191685,7 +191711,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v15, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v19, v117, 16, v19
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v19, v131, 16, v19
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v24, v167, 16, v24
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v27, v45, 16, v32
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v28, v57, 16, v33
@@ -193595,8 +193621,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v28, v4, v2
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; SI-NEXT:    v_or_b32_e32 v27, v3, v1
@@ -193660,8 +193686,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v37, v4, v2
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; SI-NEXT:    v_or_b32_e32 v36, v3, v1
@@ -193725,8 +193751,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v51, v4, v2
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; SI-NEXT:    v_or_b32_e32 v50, v3, v1
@@ -195160,8 +195186,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; VI-NEXT:    v_lshrrev_b64 v[45:46], 24, v[45:46]
 ; VI-NEXT:    v_lshrrev_b64 v[52:53], 24, v[52:53]
 ; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshrrev_b32_e32 v31, 8, v49
@@ -195679,8 +195705,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    ; kill: killed $vgpr50
@@ -195805,7 +195831,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    ; kill: killed $vgpr33
@@ -195924,7 +195950,6 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 24, v2
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(45)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 8, v32
@@ -196063,7 +196088,6 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b64 v[33:34], 24, v[13:14]
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
 ; GFX9-NEXT:    v_pk_add_f16 v32, v32, s6 op_sel_hi:[1,0]
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
 ; GFX9-NEXT:    v_pk_add_f16 v31, v31, s6 op_sel_hi:[1,0]
@@ -196479,13 +196503,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
@@ -196504,13 +196529,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
@@ -196529,13 +196555,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
@@ -196554,13 +196581,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
@@ -196579,13 +196607,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
@@ -200876,13 +200905,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 44, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 8, v10
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v2, v6, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 48, v0
@@ -201099,12 +201129,12 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x78, v0
 ; VI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0x7c, v0
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -204611,19 +204641,19 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:392
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:4
@@ -204644,16 +204674,13 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:148
 ; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:188
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:128
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v14
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v16
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v12
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -204661,266 +204688,254 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v28
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v16
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v24
+; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    ; implicit-def: $vgpr20
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr28
+; SI-NEXT:    ; implicit-def: $vgpr52
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; kill: killed $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; kill: killed $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; kill: killed $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:360
+; SI-NEXT:    ; kill: killed $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:360
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v30, 24, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 24, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v7
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v9
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v5
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v13
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v15
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v11
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v21
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:152
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v17
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v23
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 24, v32
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:332
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr27
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v32
 ; SI-NEXT:    ; implicit-def: $vgpr21
-; SI-NEXT:    ; kill: killed $vgpr23
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:160
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr27
+; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:88
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:64
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:80
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:140
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:128
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:152
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v29
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v31
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:108
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:160
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:144
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:184
-; SI-NEXT:    v_lshlrev_b32_e32 v29, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v18
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:108
-; SI-NEXT:    ; implicit-def: $vgpr18
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:180
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:176
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:140
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 24, v33
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:180
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:176
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v34
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 8, v10
-; SI-NEXT:    ; implicit-def: $vgpr10
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v33
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v34
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:172
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:196
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:220
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:192
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:172
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:192
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 24, v7
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:216
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:208
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:208
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v8
 ; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:252
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:204
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:204
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:252
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:248
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:244
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:244
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:240
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v9
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v9
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v7
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v8
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:260
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:284
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:256
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:236
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:284
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:256
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:280
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:276
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:272
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v8
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v11
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v7
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v8
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:316
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:288
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:268
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:316
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:288
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 8, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 8, v8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:312
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:312
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:308
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:308
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:304
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v9
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 24, v9
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v11
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:324
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:348
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:320
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:300
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v6
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:348
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:320
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v8
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:344
@@ -204930,123 +204945,135 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:340
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:336
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 24, v9
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:356
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:380
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:352
-; SI-NEXT:    ; implicit-def: $vgpr9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 24, v6
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v9
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:332
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:380
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:352
 ; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:376
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:372
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:368
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:364
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v4
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v8
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:388
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:384
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:372
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:368
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:364
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:388
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:384
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v10
+; SI-NEXT:    ; implicit-def: $vgpr10
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v18
+; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v26
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr26
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 24, v11
+; SI-NEXT:    ; implicit-def: $vgpr11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 24, v14
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v19, 24, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v31, 8, v15
+; SI-NEXT:    ; implicit-def: $vgpr15
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 24, v8
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:120
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:40
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr3
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:48
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr2
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:56
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr5
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v11
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v6
+; SI-NEXT:    ; implicit-def: $vgpr6
+; SI-NEXT:    ; kill: killed $vgpr6
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v7
+; SI-NEXT:    ; implicit-def: $vgpr6
+; SI-NEXT:    ; kill: killed $vgpr6
+; SI-NEXT:    ; implicit-def: $vgpr7
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v12
+; SI-NEXT:    ; implicit-def: $vgpr6
+; SI-NEXT:    ; implicit-def: $vgpr12
+; SI-NEXT:    ; kill: killed $vgpr6
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    ; implicit-def: $vgpr11
-; SI-NEXT:    ; kill: killed $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24
-; SI-NEXT:    v_lshlrev_b32_e32 v15, 8, v15
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr6
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:80
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:40
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 8, v13
+; SI-NEXT:    ; implicit-def: $vgpr13
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:16
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 8, v13
-; SI-NEXT:    ; implicit-def: $vgpr13
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr8
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:104
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v14
+; SI-NEXT:    ; implicit-def: $vgpr14
+; SI-NEXT:    ; kill: killed $vgpr14
+; SI-NEXT:    ; implicit-def: $vgpr14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -205056,23 +205083,23 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:168
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:200
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:232
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:264
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:296
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:944 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:328
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -205116,29 +205143,27 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_or_b32_e32 v20, v1, v2
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_or_b32_e32 v12, v1, v2
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v18, v2, v1
+; SI-NEXT:    v_or_b32_e32 v42, v2, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v3, v2, v1
+; SI-NEXT:    v_or_b32_e32 v5, v2, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -205148,848 +205173,855 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v2, v1
+; SI-NEXT:    v_or_b32_e32 v7, v16, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v9, v2, v1
+; SI-NEXT:    v_or_b32_e32 v11, v2, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v41, v1, v2
+; SI-NEXT:    v_or_b32_e32 v40, v1, v2
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v55, v24, v1
+; SI-NEXT:    v_or_b32_e32 v10, v24, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT:    v_mov_b32_e32 v24, v33
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v13, v2, v1
+; SI-NEXT:    v_or_b32_e32 v15, v2, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v1, v2
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v26, v1, v2
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v26, v30, v1
+; SI-NEXT:    v_or_b32_e32 v18, v25, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v21, v2, v1
+; SI-NEXT:    v_or_b32_e32 v19, v2, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v32, v1, v2
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v28, v2, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v30, v2, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v27, v2, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v21, v2, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v1, v2
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v29, v1, v2
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v35, v2, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v33, v2, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v31, v2, v1
+; SI-NEXT:    v_or_b32_e32 v27, v2, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v63, v1, v2
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v48, v2, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v50, v2, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v11, v1, v2
+; SI-NEXT:    v_or_b32_e32 v38, v1, v2
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v2, v2, v1
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v52, v1, v2
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v1, v2, v1
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v2, v1, v2
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v53, v4, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v6, v1, v4
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v55, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v4, v1
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v43, v10, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v44, v4, v10
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v17, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v16, v10, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v46, v10, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_or_b32_e32 v14, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v56, v4, v10
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v17, v10, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v44, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v57, v10, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v61, v4, v10
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v45, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v30, v10, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v49, v10, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_or_b32_e32 v25, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v4, v4, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT:    v_or_b32_e32 v47, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v23, v10, v22
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT:    v_or_b32_e32 v58, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v33, v10, v22
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v16, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    v_mov_b32_e32 v10, v29
-; SI-NEXT:    v_or_b32_e32 v29, v22, v34
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
+; SI-NEXT:    v_or_b32_e32 v59, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v62, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v24, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v13, v1, v6
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v6, v6, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v23, v1, v8
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v36, v1, v8
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v8, v8, v35
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT:    v_or_b32_e32 v34, v37, v22
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v35, v37, v22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; SI-NEXT:    v_or_b32_e32 v37, v51, v22
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_or_b32_e32 v51, v22, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v51, v22, v63
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT:    v_or_b32_e32 v34, v56, v22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT:    v_or_b32_e32 v7, v47, v7
-; SI-NEXT:    v_or_b32_e32 v47, v59, v22
+; SI-NEXT:    v_or_b32_e32 v56, v60, v22
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v54
-; SI-NEXT:    v_or_b32_e32 v54, v22, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v60
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v19, v8
-; SI-NEXT:    v_and_b32_e32 v19, 0xff, v42
-; SI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; SI-NEXT:    v_or_b32_e32 v14, v14, v19
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v54, v22, v4
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT:    v_or_b32_e32 v9, v9, v22
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v57
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v4, v32, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
-; SI-NEXT:    v_or_b32_e32 v19, v19, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT:    v_or_b32_e32 v25, v22, v24
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v28, v22, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v32, v22, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v39, v22, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v39, v22, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v42, v22, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v41, v22, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v59, v22, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v57, v22, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v60, v22, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v60, v22, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v36, v22, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v63, v22, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v10, v22, v10
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_or_b32_e32 v38, v22, v24
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v1, v22, v1
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v1, v1, v55
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
+; SI-NEXT:    v_or_b32_e32 v48, v22, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v58, v22, v24
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v52, v22, v24
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
+; SI-NEXT:    v_or_b32_e32 v46, v22, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v24, v22, v24
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_or_b32_e32 v40, v22, v40
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v49, v22, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v50, v22, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v20, v22, v20
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_or_b32_e32 v12, v22, v12
+; SI-NEXT:    v_or_b32_e32 v53, v22, v53
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_or_b32_e32 v15, v22, v15
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v45
-; SI-NEXT:    v_or_b32_e32 v45, v22, v62
-; SI-NEXT:    v_mov_b32_e32 v62, v18
-; SI-NEXT:    v_or_b32_e32 v20, v20, v62
-; SI-NEXT:    v_or_b32_e32 v22, v19, v3
-; SI-NEXT:    v_alignbit_b32 v3, v20, v3, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v3, v22, v3
+; SI-NEXT:    v_and_b32_e32 v22, 0xff, v61
+; SI-NEXT:    v_mov_b32_e32 v61, v42
+; SI-NEXT:    v_or_b32_e32 v31, v22, v31
+; SI-NEXT:    v_or_b32_e32 v22, v12, v61
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v28
+; SI-NEXT:    v_or_b32_e32 v43, v12, v5
+; SI-NEXT:    v_alignbit_b32 v5, v22, v5, 16
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    v_mov_b32_e32 v19, v5
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT:    v_mov_b32_e32 v12, v7
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v32
+; SI-NEXT:    v_or_b32_e32 v7, v7, v11
+; SI-NEXT:    v_and_b32_e32 v32, 0xffff, v49
+; SI-NEXT:    v_or_b32_e32 v32, v32, v59
+; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v5, v3, v19
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v25
-; SI-NEXT:    v_or_b32_e32 v25, v3, v9
-; SI-NEXT:    v_alignbit_b32 v3, v5, v9, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v41
-; SI-NEXT:    v_or_b32_e32 v41, v3, v55
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v39
-; SI-NEXT:    v_or_b32_e32 v18, v3, v13
-; SI-NEXT:    v_alignbit_b32 v3, v41, v13, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_or_b32_e32 v5, v5, v12
+; SI-NEXT:    v_alignbit_b32 v11, v5, v11, 16
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v40
+; SI-NEXT:    v_or_b32_e32 v42, v11, v10
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v39
+; SI-NEXT:    v_or_b32_e32 v40, v11, v15
+; SI-NEXT:    v_alignbit_b32 v11, v42, v15, 16
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v26
+; SI-NEXT:    v_or_b32_e32 v26, v11, v18
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v41
+; SI-NEXT:    v_or_b32_e32 v39, v11, v19
+; SI-NEXT:    v_alignbit_b32 v11, v26, v19, 16
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v13, v3, v26
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v42
-; SI-NEXT:    v_or_b32_e32 v39, v3, v21
-; SI-NEXT:    v_alignbit_b32 v3, v13, v21, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v32
-; SI-NEXT:    v_or_b32_e32 v32, v3, v28
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v59
-; SI-NEXT:    v_or_b32_e32 v42, v3, v27
-; SI-NEXT:    v_alignbit_b32 v3, v32, v27, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_or_b32_e32 v28, v11, v30
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v57
+; SI-NEXT:    v_or_b32_e32 v11, v11, v21
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_alignbit_b32 v11, v28, v21, 16
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v29
+; SI-NEXT:    v_or_b32_e32 v29, v11, v33
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v60
+; SI-NEXT:    v_or_b32_e32 v21, v11, v27
+; SI-NEXT:    v_alignbit_b32 v11, v29, v27, 16
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v27, v3, v35
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v60
-; SI-NEXT:    v_or_b32_e32 v3, v3, v31
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v3, v27, v31, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v63
-; SI-NEXT:    v_or_b32_e32 v21, v3, v48
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v36
-; SI-NEXT:    v_or_b32_e32 v31, v3, v50
-; SI-NEXT:    v_alignbit_b32 v3, v21, v50, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v3, v2
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v10
-; SI-NEXT:    v_or_b32_e32 v3, v3, v53
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v3, v11, v53, 16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v6
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v38
-; SI-NEXT:    v_or_b32_e32 v3, v3, v1
-; SI-NEXT:    v_or_b32_e32 v6, v6, v43
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v6, v3, v43, 16
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v44
-; SI-NEXT:    v_or_b32_e32 v9, v6, v16
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v58
-; SI-NEXT:    v_or_b32_e32 v6, v6, v46
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v6, v9, v46, 16
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v56
-; SI-NEXT:    v_or_b32_e32 v63, v6, v17
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v52
-; SI-NEXT:    v_or_b32_e32 v6, v6, v57
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_or_b32_e32 v19, v11, v38
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v63
+; SI-NEXT:    v_or_b32_e32 v27, v11, v52
+; SI-NEXT:    v_alignbit_b32 v11, v19, v52, 16
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v52, 16, v30
+; SI-NEXT:    v_lshrrev_b32_e32 v30, 16, v23
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_or_b32_e32 v11, v11, v2
+; SI-NEXT:    v_alignbit_b32 v1, v11, v55, 16
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v6, v63, v57, 16
-; SI-NEXT:    v_or_b32_e32 v57, v4, v23
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v40
-; SI-NEXT:    v_or_b32_e32 v52, v4, v33
-; SI-NEXT:    v_alignbit_b32 v4, v57, v33, 16
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v17
+; SI-NEXT:    v_or_b32_e32 v15, v1, v14
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v48
+; SI-NEXT:    v_or_b32_e32 v1, v1, v44
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v29
-; SI-NEXT:    v_or_b32_e32 v46, v4, v34
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v12
-; SI-NEXT:    v_or_b32_e32 v38, v4, v37
-; SI-NEXT:    v_alignbit_b32 v4, v46, v37, 16
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v51
-; SI-NEXT:    v_or_b32_e32 v44, v4, v7
-; SI-NEXT:    v_lshrrev_b32_e32 v40, 16, v7
-; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v8
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v15
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    v_or_b32_e32 v36, v4, v47
-; SI-NEXT:    v_alignbit_b32 v4, v44, v47, 16
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v1, v15, v44, 16
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v54
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    v_or_b32_e32 v43, v4, v8
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v45
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    v_or_b32_e32 v12, v4, v14
-; SI-NEXT:    v_alignbit_b32 v4, v43, v14, 16
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v45
+; SI-NEXT:    v_or_b32_e32 v17, v1, v25
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v46
+; SI-NEXT:    v_or_b32_e32 v1, v1, v47
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v28
-; SI-NEXT:    v_lshrrev_b32_e32 v28, 16, v23
-; SI-NEXT:    v_mov_b32_e32 v23, v12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v1, v17, v47, 16
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v61
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    v_or_b32_e32 v61, v6, v30
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v24
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    v_or_b32_e32 v58, v6, v49
-; SI-NEXT:    v_alignbit_b32 v6, v61, v49, 16
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v58
+; SI-NEXT:    v_or_b32_e32 v1, v1, v16
+; SI-NEXT:    v_alignbit_b32 v32, v1, v59, 16
+; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v32, 0xffff, v62
+; SI-NEXT:    v_or_b32_e32 v59, v6, v23
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v62, v32, v24
+; SI-NEXT:    v_and_b32_e32 v32, 0xffff, v50
+; SI-NEXT:    v_or_b32_e32 v50, v6, v36
+; SI-NEXT:    v_alignbit_b32 v6, v59, v36, 16
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v8
+; SI-NEXT:    v_or_b32_e32 v47, v6, v35
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v53
+; SI-NEXT:    v_or_b32_e32 v49, v6, v37
+; SI-NEXT:    v_alignbit_b32 v6, v47, v37, 16
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v51
+; SI-NEXT:    v_or_b32_e32 v45, v6, v34
+; SI-NEXT:    v_or_b32_e32 v48, v3, v56
+; SI-NEXT:    v_alignbit_b32 v3, v45, v56, 16
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v54
+; SI-NEXT:    v_or_b32_e32 v44, v3, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v31
+; SI-NEXT:    v_or_b32_e32 v3, v3, v9
+; SI-NEXT:    v_lshrrev_b32_e32 v20, 16, v18
+; SI-NEXT:    v_lshrrev_b32_e32 v18, 16, v14
+; SI-NEXT:    v_mov_b32_e32 v14, v3
+; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v4
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    v_or_b32_e32 v46, v32, v13
+; SI-NEXT:    v_alignbit_b32 v13, v62, v13, 16
+; SI-NEXT:    v_alignbit_b32 v6, v44, v9, 16
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v58, 16, v61
+; SI-NEXT:    v_lshrrev_b32_e32 v55, 16, v10
+; SI-NEXT:    s_waitcnt expcnt(1)
+; SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v33
+; SI-NEXT:    v_lshrrev_b32_e32 v8, 16, v38
+; SI-NEXT:    v_lshrrev_b32_e32 v38, 16, v2
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v25
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v16
+; SI-NEXT:    v_lshrrev_b32_e32 v33, 16, v24
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v62
-; SI-NEXT:    v_lshrrev_b32_e32 v56, 16, v19
-; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v55
-; SI-NEXT:    v_lshrrev_b32_e32 v53, 16, v26
-; SI-NEXT:    v_mov_b32_e32 v26, v42
-; SI-NEXT:    v_lshrrev_b32_e32 v50, 16, v35
-; SI-NEXT:    v_lshrrev_b32_e32 v49, 16, v48
-; SI-NEXT:    v_lshrrev_b32_e32 v48, 16, v2
-; SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v17
-; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v30
-; SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v34
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v35
+; SI-NEXT:    v_lshrrev_b32_e32 v36, 16, v34
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr24
-; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    ; kill: killed $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr25
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr8
-; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    ; implicit-def: $vgpr14
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; kill: killed $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr15
-; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr4
+; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr9
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:  .LBB96_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB96_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v45
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v42
-; SI-NEXT:    v_or_b32_e32 v1, v62, v1
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v41
+; SI-NEXT:    v_or_b32_e32 v1, v31, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
-; SI-NEXT:    v_or_b32_e32 v2, v14, v2
+; SI-NEXT:    v_or_b32_e32 v2, v9, v2
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v54
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v60
-; SI-NEXT:    v_or_b32_e32 v2, v8, v2
-; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_or_b32_e32 v2, v4, v2
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v57
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    s_movk_i32 s6, 0x300
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT:    v_or_b32_e32 v3, v19, v3
+; SI-NEXT:    v_or_b32_e32 v4, v32, v4
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v2, v4, v2
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:968 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v34
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_or_b32_e32 v5, v56, v5
+; SI-NEXT:    v_mov_b32_e32 v30, v16
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
-; SI-NEXT:    v_add_i32_e32 v43, vcc, s7, v2
-; SI-NEXT:    v_mov_b32_e32 v28, v24
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
+; SI-NEXT:    v_mov_b32_e32 v31, v24
+; SI-NEXT:    v_add_i32_e32 v44, vcc, s7, v2
+; SI-NEXT:    v_add_i32_e32 v27, vcc, s7, v1
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:972 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; SI-NEXT:    v_or_b32_e32 v3, v15, v3
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:964 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT:    v_or_b32_e32 v4, v59, v4
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v4, v3
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v47, v5
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v51, v6
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_or_b32_e32 v7, v37, v7
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v4, v60, v4
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:956 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v32, vcc, s7, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT:    v_mov_b32_e32 v48, v32
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v7, v4
+; SI-NEXT:    v_or_b32_e32 v4, v63, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v44, vcc, s7, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshrrev_b32_e32 v40, 16, v44
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v45, vcc, s7, v4
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v36, 16, v45
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_or_b32_e32 v5, v12, v5
+; SI-NEXT:    v_or_b32_e32 v5, v53, v5
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v35, vcc, s7, v5
-; SI-NEXT:    v_mov_b32_e32 v38, v35
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v33, vcc, s7, v5
+; SI-NEXT:    v_mov_b32_e32 v49, v33
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v34, v6
+; SI-NEXT:    v_or_b32_e32 v6, v35, v6
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v46, vcc, s7, v6
-; SI-NEXT:    v_add_i32_e32 v34, vcc, s7, v3
-; SI-NEXT:    v_mov_b32_e32 v36, v34
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v47, vcc, s7, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -205998,16 +206030,16 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s7, v7
-; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    v_mov_b32_e32 v50, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -206016,15 +206048,15 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v57, vcc, s7, v8
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:960 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v59, vcc, s7, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -206033,16 +206065,16 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v9
-; SI-NEXT:    v_mov_b32_e32 v58, v7
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v9
+; SI-NEXT:    v_mov_b32_e32 v46, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -206051,9 +206083,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v61, vcc, s7, v10
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v62, vcc, s7, v10
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
@@ -206070,7 +206102,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v11
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v11
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
@@ -206085,15 +206117,15 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v63, vcc, s7, v12
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v12
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -206102,50 +206134,45 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v13
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    v_or_b32_e32 v15, v17, v15
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v14
-; SI-NEXT:    v_lshrrev_b32_e32 v55, 16, v9
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v17
+; SI-NEXT:    v_or_b32_e32 v15, v17, v15
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v15
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -206155,9 +206182,8 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v17, v18, v17
 ; SI-NEXT:    v_or_b32_e32 v16, v17, v16
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v3, vcc, s7, v16
-; SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v46
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
@@ -206171,468 +206197,476 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
-; SI-NEXT:    v_or_b32_e32 v17, v18, v17
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s7, v17
+; SI-NEXT:    v_or_b32_e32 v18, v18, v17
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v18, v19, v18
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
+; SI-NEXT:    v_or_b32_e32 v17, v19, v17
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v23
 ; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_or_b32_e32 v19, v20, v19
-; SI-NEXT:    v_or_b32_e32 v18, v19, v18
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_or_b32_e32 v19, v19, v17
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v18
-; SI-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v19
+; SI-NEXT:    v_lshrrev_b32_e32 v38, 16, v11
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
-; SI-NEXT:    v_and_b32_e32 v19, 0xff, v19
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v19, v20, v19
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_or_b32_e32 v17, v20, v17
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
 ; SI-NEXT:    v_or_b32_e32 v20, v21, v20
-; SI-NEXT:    v_or_b32_e32 v19, v20, v19
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v31, vcc, s7, v19
+; SI-NEXT:    v_or_b32_e32 v20, v20, v17
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v20, v21, v20
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v20
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_or_b32_e32 v17, v21, v17
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; SI-NEXT:    v_or_b32_e32 v21, v22, v21
-; SI-NEXT:    v_or_b32_e32 v20, v21, v20
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v21, v21, v17
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s7, v21
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v21, v22, v21
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v17, v22, v17
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
 ; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; SI-NEXT:    v_or_b32_e32 v22, v23, v22
-; SI-NEXT:    v_or_b32_e32 v22, v22, v21
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v15, vcc, s7, v22
+; SI-NEXT:    v_or_b32_e32 v22, v22, v17
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v21, v23, v21
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v21
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_or_b32_e32 v17, v23, v17
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; SI-NEXT:    v_and_b32_e32 v23, 0xff, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT:    v_or_b32_e32 v23, v25, v23
-; SI-NEXT:    v_or_b32_e32 v23, v23, v21
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v27, vcc, s7, v23
-; SI-NEXT:    v_mov_b32_e32 v23, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v50, 16, v27
+; SI-NEXT:    v_or_b32_e32 v23, v26, v23
+; SI-NEXT:    v_or_b32_e32 v23, v23, v17
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    v_mov_b32_e32 v26, v25
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v29, vcc, s7, v23
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
-; SI-NEXT:    v_or_b32_e32 v24, v24, v21
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v24
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v21, 0xff, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT:    v_or_b32_e32 v25, v25, v21
-; SI-NEXT:    v_or_b32_e32 v1, v25, v1
-; SI-NEXT:    v_add_i32_e32 v26, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s7, v20
-; SI-NEXT:    v_lshrrev_b32_e32 v49, 16, v21
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT:    v_or_b32_e32 v24, v24, v17
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v24
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT:    v_or_b32_e32 v25, v25, v17
+; SI-NEXT:    v_or_b32_e32 v2, v25, v2
+; SI-NEXT:    v_add_i32_e32 v18, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v14
+; SI-NEXT:    v_mov_b32_e32 v14, v27
+; SI-NEXT:    v_add_i32_e32 v27, vcc, s7, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v4, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v32, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v28, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v52, 16, v28
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v4, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v39, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v39, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v30, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshrrev_b32_e32 v53, 16, v13
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v26, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v26, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v20, 16, v26
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v4, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v40, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v28, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v41, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshrrev_b32_e32 v28, 16, v57
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v31, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v42, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v55, 16, v42
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_or_b32_e32 v1, v33, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v4, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v25, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v4, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshrrev_b32_e32 v56, 16, v5
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v30, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT:    v_lshrrev_b32_e32 v30, 16, v59
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_or_b32_e32 v1, v29, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v4, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v43, vcc, s7, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v4, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v1
-; SI-NEXT:    v_alignbit_b32 v1, v20, v22, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v5, v25, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:944 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s7, v2
+; SI-NEXT:    v_alignbit_b32 v2, v22, v43, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v41, v18, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v5, v7, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v13, v39, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v42, v40, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v32, v26, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v26, v39, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v1, v27, v15, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v28, v18, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_alignbit_b32 v2, v29, v21, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v21, v31, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v19, v27, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v1, v11, v14, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v11, v16, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v1, v3, v12, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v15, v13, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v1, v9, v10, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v17, v10, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v1, v63, v8, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v1, v9, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v61, v7, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v62, v8, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v57, v6, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v59, v6, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v46, v35, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v47, v33, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v44, v34, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v2, v45, v32, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v1, v43, v23, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v20
-; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v41
-; SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v32
-; SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; SI-NEXT:    v_alignbit_b32 v2, v44, v14, 16
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v58, 16, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v29
+; SI-NEXT:    v_lshrrev_b32_e32 v8, 16, v19
+; SI-NEXT:    v_lshrrev_b32_e32 v18, 16, v15
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v17
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v63
-; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v61
-; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v43
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; SI-NEXT:    v_lshrrev_b32_e32 v33, 16, v62
+; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v47
+; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v44
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:940 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB96_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v22
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v43
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v20
-; SI-NEXT:    v_or_b32_e32 v6, v7, v6
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 4, v0
-; SI-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v22
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v58
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v25
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v6, v6, v7
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 8, v0
-; SI-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 8, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v56
-; SI-NEXT:    v_or_b32_e32 v5, v5, v6
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 12, v0
-; SI-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v12
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 12, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v18
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v40
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v5, v5, v6
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 16, v0
-; SI-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v41
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v10
-; SI-NEXT:    v_or_b32_e32 v5, v5, v6
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
-; SI-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v42
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v55
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 20, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v39
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v39
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v5, v5, v6
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 24, v0
-; SI-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 24, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v53
-; SI-NEXT:    v_or_b32_e32 v5, v5, v6
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 28, v0
-; SI-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v20
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 28, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v26
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v5, v5, v6
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 32, v0
-; SI-NEXT:    buffer_store_dword v5, v6, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v32
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 36, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v52
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 36, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v4, v4, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 40, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 40, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v50
-; SI-NEXT:    v_or_b32_e32 v4, v4, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 44, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v13
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 44, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v31
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v27
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v4, v4, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 48, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 48, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v49
-; SI-NEXT:    v_or_b32_e32 v4, v4, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 52, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v8
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 52, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v4, v4, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 56, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 56, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v48
-; SI-NEXT:    v_or_b32_e32 v4, v4, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 60, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v38
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 60, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v4, v4, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 64, v0
-; SI-NEXT:    buffer_store_dword v4, v5, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 64, v0
+; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v35
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x44, v0
 ; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -206641,14 +206675,14 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x48, v0
 ; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v55
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x4c, v0
 ; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -206656,77 +206690,74 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x50, v0
 ; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v63
-; SI-NEXT:    v_or_b32_e32 v1, v3, v1
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 0x54, v0
-; SI-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v58
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v46
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 0x58, v0
-; SI-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x58, v0
+; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v62
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v33
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x5c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v52
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v50
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v57
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v28
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v59
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v30
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v38
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v49
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x68, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v46
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v16
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v36
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v44
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v40
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v45
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v36
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x74, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x78, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:940 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v43
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v44
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x7c, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -206803,39 +206834,39 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:144
-; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:152
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:136
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:144
+; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:152
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:160
 ; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:168
 ; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:176
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:184
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
 ; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v1
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v25
 ; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
-; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v3
 ; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v5
 ; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v7
-; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v9
 ; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
-; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v13
-; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v35, 8, v17
 ; VI-NEXT:    v_lshlrev_b16_e32 v36, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v21
 ; VI-NEXT:    v_lshlrev_b16_e32 v34, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v27
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
@@ -206849,46 +206880,47 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v27
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
 ; VI-NEXT:    ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v37
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v38
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v39
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v48
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v49
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v50
+; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v51
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
@@ -206905,7 +206937,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v43
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:208
@@ -206913,15 +206945,15 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
@@ -206944,20 +206976,20 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -206970,20 +207002,20 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -206991,17 +207023,17 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
@@ -207017,17 +207049,17 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:316
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:324
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; VI-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
@@ -207042,44 +207074,44 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_ushort v3, off, s[0:3], s32 offset:376
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:348
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v1
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v2
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v3
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:372
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; VI-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
 ; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
 ; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -207105,31 +207137,31 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v4, v4, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr54
 ; VI-NEXT:    ; implicit-def: $vgpr55
 ; VI-NEXT:    ; implicit-def: $vgpr40
 ; VI-NEXT:    ; implicit-def: $vgpr41
-; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr36
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v34 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr34
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr50
 ; VI-NEXT:    ; implicit-def: $vgpr52
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr37
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -207148,35 +207180,35 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_or_b32_sdwa v10, v63, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v61, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_or_b32_sdwa v12, v58, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v59, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v14, v45, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    ; implicit-def: $vgpr45
 ; VI-NEXT:    s_waitcnt vmcnt(2)
@@ -207185,26 +207217,26 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v63, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr62
-; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v60, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr59
+; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v10, v57, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v56, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr56
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v58, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr56
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -207213,35 +207245,35 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v13, v44, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -207256,20 +207288,20 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -207277,189 +207309,189 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v28, v28, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v31, v31, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    v_or_b32_sdwa v31, v31, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr49
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v32, v32, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v30, v30, v37 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v30, v30, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
-; VI-NEXT:    ; kill: killed $vgpr37
-; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; kill: killed $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v30, v30, v38 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v30, v30, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v30, v30, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr39
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v31, v31, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v31, v31, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v31, v31, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    ; kill: killed $vgpr32
@@ -207564,7 +207596,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    ; kill: killed $vgpr32
 ; VI-NEXT:    ; implicit-def: $vgpr32
-; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr51
 ; VI-NEXT:  .LBB96_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB96_4
@@ -207572,53 +207604,51 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v18, 0x300
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
 ; VI-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    v_add_u16_sdwa v4, v0, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(12)
 ; VI-NEXT:    v_or_b32_sdwa v29, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(13)
+; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v4, v0, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_sdwa v0, v2, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v29, 0x300, v29
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v2, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v0, v2, v0
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v2, 3, v2
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v2, 3, v2
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
 ; VI-NEXT:    v_or_b32_sdwa v2, v52, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v3, v51, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v50, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v2, v2, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -207628,19 +207658,19 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v1, 0x300, v1
 ; VI-NEXT:    v_or_b32_e32 v1, v1, v4
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v3, v49, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v3, v3, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
-; VI-NEXT:    v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v4, v37, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v4, 0x300, v4
 ; VI-NEXT:    v_or_b32_e32 v3, v4, v3
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
@@ -207648,14 +207678,13 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
 ; VI-NEXT:    v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v5, 3, v5
 ; VI-NEXT:    v_or_b32_sdwa v5, v35, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v5, 0x300, v5
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v4, 3, v4
 ; VI-NEXT:    v_or_b32_sdwa v4, v36, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v4, v4, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -207672,7 +207701,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_or_b32_sdwa v32, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_sdwa v32, v32, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -207680,7 +207709,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v28, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v28, 0x300, v28
 ; VI-NEXT:    v_or_b32_e32 v28, v28, v32
 ; VI-NEXT:    s_waitcnt vmcnt(1)
@@ -207688,79 +207717,94 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v33, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v33, v33, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v27, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v27, 0x300, v27
 ; VI-NEXT:    v_or_b32_e32 v27, v27, v33
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v34, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v34, v34, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v26, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v26, 0x300, v26
 ; VI-NEXT:    v_or_b32_e32 v26, v26, v34
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v35, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v35, v35, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v25, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v25, 0x300, v25
+; VI-NEXT:    v_or_b32_e32 v25, v25, v35
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v36, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v25, 0x300, v25
-; VI-NEXT:    v_or_b32_e32 v25, v25, v35
+; VI-NEXT:    v_add_u16_sdwa v36, v36, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v36, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_sdwa v36, v36, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v24, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
+; VI-NEXT:    v_or_b32_e32 v24, v24, v36
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v24, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v37, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v37, v37, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v23, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
 ; VI-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v24, 0x300, v24
-; VI-NEXT:    v_or_b32_e32 v24, v24, v36
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
+; VI-NEXT:    v_or_b32_e32 v23, v23, v37
 ; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_add_u16_e32 v22, 3, v22
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v8, 3, v8
 ; VI-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v7, v8, v7
-; VI-NEXT:    v_add_u16_e32 v8, 3, v61
+; VI-NEXT:    v_add_u16_e32 v8, 3, v63
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v9, 3, v62
@@ -207769,30 +207813,30 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; VI-NEXT:    v_or_b32_e32 v8, v9, v8
-; VI-NEXT:    v_add_u16_e32 v9, 3, v63
+; VI-NEXT:    v_add_u16_e32 v9, 3, v61
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v10, 3, v59
+; VI-NEXT:    v_add_u16_e32 v10, 3, v60
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v9, v9, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_or_b32_e32 v9, v10, v9
-; VI-NEXT:    v_add_u16_e32 v10, 3, v60
+; VI-NEXT:    v_add_u16_e32 v10, 3, v57
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v11, 3, v57
+; VI-NEXT:    v_add_u16_e32 v11, 3, v56
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v10, v10, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_or_b32_e32 v10, v11, v10
-; VI-NEXT:    v_add_u16_e32 v11, 3, v58
+; VI-NEXT:    v_add_u16_e32 v11, 3, v59
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v12, 3, v56
+; VI-NEXT:    v_add_u16_e32 v12, 3, v58
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; VI-NEXT:    v_or_b32_e32 v11, v12, v11
@@ -207801,7 +207845,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v13, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_or_b32_e32 v12, v13, v12
@@ -207810,7 +207854,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 3, v44
 ; VI-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v13, v13, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_or_b32_e32 v13, v14, v13
@@ -207822,35 +207866,35 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; VI-NEXT:    v_or_b32_e32 v14, v15, v14
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v15, 3, v15
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v15, v15, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    v_or_b32_e32 v15, v16, v15
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
 ; VI-NEXT:    v_or_b32_sdwa v17, v19, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
 ; VI-NEXT:    v_or_b32_sdwa v20, v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -207859,54 +207903,43 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v16, v19, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v19, 0x300, v20
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; VI-NEXT:    v_or_b32_e32 v16, v19, v16
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
-; VI-NEXT:    v_or_b32_sdwa v30, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v30, v39, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v30, 0x300, v30
-; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v20, 3, v20
-; VI-NEXT:    v_or_b32_sdwa v31, v50, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v31, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v31, 0x300, v31
-; VI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v21, 3, v21
-; VI-NEXT:    v_or_b32_sdwa v21, v37, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v37, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; VI-NEXT:    v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v40, v21, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_sdwa v37, v37, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v29, v29, v40
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v23, v23, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v23, 0x300, v23
-; VI-NEXT:    v_or_b32_e32 v23, v23, v37
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v38, v38, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_e32 v20, 3, v20
-; VI-NEXT:    v_or_b32_sdwa v20, v48, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; VI-NEXT:    v_add_u16_sdwa v55, v20, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_sdwa v38, v38, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v30, v30, v55
-; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v22, 3, v22
 ; VI-NEXT:    v_or_b32_sdwa v22, v39, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v20, 3, v20
+; VI-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_sdwa v55, v20, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v22, 0x300, v22
 ; VI-NEXT:    v_or_b32_e32 v22, v22, v38
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v30, v30, v55
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v39, 3, v39
 ; VI-NEXT:    v_or_b32_sdwa v39, v48, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
@@ -207928,7 +207961,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; VI-NEXT:    v_add_u16_e32 v19, 3, v19
 ; VI-NEXT:    v_or_b32_sdwa v19, v53, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v39, 3, v39
 ; VI-NEXT:    v_or_b32_sdwa v39, v49, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -208030,17 +208063,16 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:112
 ; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
 ; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:136
+; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:136
 ; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:144
 ; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:152
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:160
 ; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:168
 ; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:176
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:184
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v27
-; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v29
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v3
@@ -208048,81 +208080,81 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v7
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v11
-; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v13
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v38, 8, v15
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v17
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v35, 8, v19
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v21
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v23
-; GFX9-NEXT:    s_waitcnt vmcnt(27)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v25
 ; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v30
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v6
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v8
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v10
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v14
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v16
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v18
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v20
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v22
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v24
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v26
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(26)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v28
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:140
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v31
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v25
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:140
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:148
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v37
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v39
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v48
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v49
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
@@ -208130,7 +208162,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:156
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:164
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
@@ -208147,7 +208179,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v43
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v44
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:192
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:200
@@ -208156,15 +208188,15 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:188
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:196
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
@@ -208188,20 +208220,20 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:228
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:236
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:244
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:256
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:264
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:272
@@ -208215,20 +208247,20 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:260
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:268
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:276
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:288
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:296
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:304
@@ -208237,17 +208269,17 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:284
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:292
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:300
@@ -208274,7 +208306,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v2
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v3
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:332
@@ -208294,48 +208326,48 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:356
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:364
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:372
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:384
 ; GFX9-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:380
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:100
 ; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:92
 ; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v56, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v59, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB96_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
@@ -208344,9 +208376,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s6
@@ -208375,10 +208407,10 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; implicit-def: $vgpr51
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v3, v4, v3, s6
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr48
+; GFX9-NEXT:    ; implicit-def: $vgpr39
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
@@ -208389,7 +208421,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v5, v6, v5, s6
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr34
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -208404,93 +208436,93 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v7, v8, v7, s6
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v60, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v60, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v56, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v56, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v58, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v46, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v44, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v15, v43, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    v_or_b32_sdwa v15, v42, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v63, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v59, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v61, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr59
+; GFX9-NEXT:    ; implicit-def: $vgpr61
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v58, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v57, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v11, v57, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v59, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v12, v46, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v47, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr46
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    ; implicit-def: $vgpr47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v45, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v43, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr42
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v16, v17, v16, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v18, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -208505,20 +208537,20 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v18, v19, v18, s6
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v19, v20, v19, s6
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v20, v21, v20, s6
@@ -208526,58 +208558,58 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v21, v22, v21, s6
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v22, v22, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v22, v23, v22, s6
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v23, v24, v23, s6
-; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v24, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v24, v25, v24, s6
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v25, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v25, v26, v25, s6
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v26, v26, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v26, v27, v26, s6
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
@@ -208595,22 +208627,22 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v28, v29, v28, s6
 ; GFX9-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v32, v32, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr53
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v37 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v29, v30, v29, s6
-; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr37
 ; GFX9-NEXT:    ; kill: killed $vgpr37
 ; GFX9-NEXT:    ; implicit-def: $vgpr37
@@ -208707,7 +208739,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v30, v30, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v30, v31, v30, s6
-; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr49
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v31, v31, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -208819,27 +208851,27 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_cbranch_execz .LBB96_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
@@ -208847,19 +208879,20 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
+; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
 ; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v55, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    s_waitcnt vmcnt(20)
+; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
 ; GFX9-NEXT:    v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v2
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v3
-; GFX9-NEXT:    v_add_u16_e32 v24, 3, v24
 ; GFX9-NEXT:    v_perm_b32 v0, v2, v0, s6
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(17)
+; GFX9-NEXT:    s_waitcnt vmcnt(16)
 ; GFX9-NEXT:    v_or_b32_sdwa v24, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
@@ -208884,7 +208917,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u16_e32 v3, 3, v3
-; GFX9-NEXT:    v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v4, 3, v4
@@ -208910,7 +208943,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v36, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 3, v22
@@ -208934,11 +208967,11 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v23, v37, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v37, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 3, v21
-; GFX9-NEXT:    v_or_b32_sdwa v21, v39, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v21, v48, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v33, 0x300, v21
 ; GFX9-NEXT:    v_add_u16_e32 v34, 0x300, v23
 ; GFX9-NEXT:    v_perm_b32 v29, v34, v29, s6
@@ -208946,16 +208979,17 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_perm_b32 v6, v7, v6, s6
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v38, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
@@ -208965,7 +208999,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v39, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
@@ -208975,7 +209009,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v48, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -208988,45 +209022,45 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v62
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v59
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v61
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v60
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v58
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v57
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v60
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v56
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v57
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v59
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v56
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v58
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v46
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v47
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v46
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
@@ -209035,63 +209069,63 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 3, v44
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v43
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v15, 3, v43
+; GFX9-NEXT:    v_add_u16_e32 v15, 3, v42
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v17, 3, v17
 ; GFX9-NEXT:    v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v17, 0x300, v17
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v19, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v16
 ; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v18
 ; GFX9-NEXT:    v_perm_b32 v17, v17, v20, s6
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v19
 ; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v16, v18, v16, s6
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v20, 3, v20
 ; GFX9-NEXT:    v_or_b32_sdwa v20, v49, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_or_b32_sdwa v49, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v20
 ; GFX9-NEXT:    v_perm_b32 v30, v33, v30, s6
@@ -209099,7 +209133,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v50, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v18, 3, v18
 ; GFX9-NEXT:    v_or_b32_sdwa v18, v52, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -209109,14 +209143,14 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v51, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v23, 0x300, v51
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v52, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 3, v19
 ; GFX9-NEXT:    v_or_b32_sdwa v19, v53, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -209126,7 +209160,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v53, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v53
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -209139,7 +209173,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v55, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v55
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -209147,7 +209181,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v40, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v50, 0x300, v40
 ; GFX9-NEXT:    v_perm_b32 v21, v50, v21, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -209155,14 +209189,14 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v41, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v41
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v42, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v51, 0x300, v42
 ; GFX9-NEXT:    v_perm_b32 v20, v51, v20, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
@@ -209170,7 +209204,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v43, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u16_e32 v19, 0x300, v43
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v27, 3, v27
@@ -210980,8 +211014,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[4:5]
 ; SI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
@@ -211033,650 +211067,648 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    v_writelane_b32 v63, s99, 33
 ; SI-NEXT:    v_writelane_b32 v63, s30, 34
 ; SI-NEXT:    v_writelane_b32 v63, s31, 35
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v29, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:332
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:328
-; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:324
-; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:320
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:308
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:304
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:332
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:328
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:324
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:320
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:308
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:304
 ; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:300
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:296
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:292
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:288
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:276
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:272
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:268
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:264
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:260
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:292
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:288
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:276
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:272
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:268
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:264
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:260
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:256
 ; SI-NEXT:    s_waitcnt expcnt(6)
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:256
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:244
 ; SI-NEXT:    s_waitcnt expcnt(5)
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:244
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:240
 ; SI-NEXT:    s_waitcnt expcnt(4)
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:240
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:236
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:232
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:228
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:224
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:212
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:236
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:232
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:228
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:224
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:212
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:208
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:204
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:200
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:196
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:204
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:200
+; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:196
 ; SI-NEXT:    ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; SI-NEXT:    s_waitcnt expcnt(2)
-; SI-NEXT:    v_mov_b32_e32 v29, v5
+; SI-NEXT:    s_waitcnt expcnt(1)
+; SI-NEXT:    v_mov_b32_e32 v29, v26
 ; SI-NEXT:    v_writelane_b32 v62, s28, 0
-; SI-NEXT:    v_writelane_b32 v62, s27, 1
-; SI-NEXT:    v_writelane_b32 v62, s25, 2
-; SI-NEXT:    v_writelane_b32 v62, s24, 3
-; SI-NEXT:    v_writelane_b32 v62, s23, 4
-; SI-NEXT:    v_writelane_b32 v62, s22, 5
-; SI-NEXT:    v_writelane_b32 v62, s21, 6
-; SI-NEXT:    v_writelane_b32 v62, s20, 7
-; SI-NEXT:    v_writelane_b32 v62, s19, 8
-; SI-NEXT:    v_writelane_b32 v62, s18, 9
-; SI-NEXT:    v_writelane_b32 v62, s16, 10
-; SI-NEXT:    v_readfirstlane_b32 s76, v18
-; SI-NEXT:    v_readfirstlane_b32 s40, v25
-; SI-NEXT:    v_readfirstlane_b32 s16, v24
-; SI-NEXT:    v_readfirstlane_b32 s42, v23
-; SI-NEXT:    v_readfirstlane_b32 s52, v20
-; SI-NEXT:    v_readfirstlane_b32 s8, v19
-; SI-NEXT:    v_readfirstlane_b32 s35, v27
-; SI-NEXT:    v_readfirstlane_b32 s28, v28
-; SI-NEXT:    v_readfirstlane_b32 s87, v26
+; SI-NEXT:    v_writelane_b32 v62, s25, 1
+; SI-NEXT:    v_writelane_b32 v62, s24, 2
+; SI-NEXT:    v_writelane_b32 v62, s23, 3
+; SI-NEXT:    v_writelane_b32 v62, s22, 4
+; SI-NEXT:    v_writelane_b32 v62, s21, 5
+; SI-NEXT:    v_writelane_b32 v62, s18, 6
+; SI-NEXT:    v_writelane_b32 v62, s16, 7
+; SI-NEXT:    v_readfirstlane_b32 s15, v16
+; SI-NEXT:    v_readfirstlane_b32 s18, v25
+; SI-NEXT:    v_readfirstlane_b32 s43, v15
+; SI-NEXT:    v_readfirstlane_b32 s42, v24
+; SI-NEXT:    v_readfirstlane_b32 s44, v23
+; SI-NEXT:    v_readfirstlane_b32 s49, v12
+; SI-NEXT:    v_readfirstlane_b32 s8, v11
+; SI-NEXT:    v_readfirstlane_b32 s53, v20
+; SI-NEXT:    v_readfirstlane_b32 s40, v19
+; SI-NEXT:    v_readfirstlane_b32 s23, v28
+; SI-NEXT:    v_readfirstlane_b32 s34, v27
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v21, v13
+; SI-NEXT:    v_mov_b32_e32 v13, v5
+; SI-NEXT:    v_readfirstlane_b32 s97, v29
+; SI-NEXT:    v_readfirstlane_b32 s80, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 24, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v22
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 24, v30
 ; SI-NEXT:    v_readfirstlane_b32 s96, v17
-; SI-NEXT:    v_readfirstlane_b32 s99, v16
-; SI-NEXT:    v_readfirstlane_b32 s89, v15
-; SI-NEXT:    v_readfirstlane_b32 s88, v12
-; SI-NEXT:    v_readfirstlane_b32 s30, v11
-; SI-NEXT:    v_readfirstlane_b32 s64, v10
-; SI-NEXT:    v_readfirstlane_b32 s55, v9
-; SI-NEXT:    v_readfirstlane_b32 s65, v8
-; SI-NEXT:    v_readfirstlane_b32 s80, v7
-; SI-NEXT:    v_readfirstlane_b32 s21, v2
-; SI-NEXT:    v_readfirstlane_b32 s74, v1
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:156
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:220
+; SI-NEXT:    v_readfirstlane_b32 s64, v9
+; SI-NEXT:    v_readfirstlane_b32 s25, v8
+; SI-NEXT:    v_readfirstlane_b32 s83, v7
+; SI-NEXT:    v_readfirstlane_b32 s84, v4
+; SI-NEXT:    v_readfirstlane_b32 s93, v3
+; SI-NEXT:    v_readfirstlane_b32 s76, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_readfirstlane_b32 s50, v35
-; SI-NEXT:    v_readfirstlane_b32 s31, v36
-; SI-NEXT:    v_readfirstlane_b32 s53, v37
-; SI-NEXT:    v_readfirstlane_b32 s82, v48
-; SI-NEXT:    v_readfirstlane_b32 s7, v49
-; SI-NEXT:    v_readfirstlane_b32 s79, v52
-; SI-NEXT:    v_readfirstlane_b32 s78, v55
-; SI-NEXT:    v_readfirstlane_b32 s4, v41
+; SI-NEXT:    v_readfirstlane_b32 s4, v34
+; SI-NEXT:    v_writelane_b32 v62, s4, 8
+; SI-NEXT:    v_readfirstlane_b32 s4, v38
+; SI-NEXT:    v_writelane_b32 v62, s4, 9
+; SI-NEXT:    v_readfirstlane_b32 s4, v49
+; SI-NEXT:    v_writelane_b32 v62, s4, 10
+; SI-NEXT:    v_readfirstlane_b32 s4, v50
 ; SI-NEXT:    v_writelane_b32 v62, s4, 11
-; SI-NEXT:    v_readfirstlane_b32 s4, v43
+; SI-NEXT:    v_readfirstlane_b32 s79, v52
+; SI-NEXT:    v_readfirstlane_b32 s88, v54
+; SI-NEXT:    v_readfirstlane_b32 s4, v55
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:192
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:180
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:176
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:172
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:168
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:164
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:160
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:148
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:144
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:140
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:136
 ; SI-NEXT:    v_writelane_b32 v62, s4, 12
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:192
-; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:180
-; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:176
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:172
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:168
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:164
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:160
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:148
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:144
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:140
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:136
-; SI-NEXT:    v_readfirstlane_b32 s4, v45
-; SI-NEXT:    v_writelane_b32 v62, s4, 13
-; SI-NEXT:    v_readfirstlane_b32 s4, v56
-; SI-NEXT:    v_writelane_b32 v62, s4, 14
-; SI-NEXT:    v_readfirstlane_b32 s4, v57
-; SI-NEXT:    v_writelane_b32 v62, s4, 15
+; SI-NEXT:    v_readfirstlane_b32 s77, v41
+; SI-NEXT:    v_readfirstlane_b32 s4, v42
+; SI-NEXT:    v_readfirstlane_b32 s94, v31
+; SI-NEXT:    v_readfirstlane_b32 s70, v32
+; SI-NEXT:    v_readfirstlane_b32 s51, v33
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_readfirstlane_b32 s4, v60
-; SI-NEXT:    v_readfirstlane_b32 s86, v31
-; SI-NEXT:    v_readfirstlane_b32 s36, v32
-; SI-NEXT:    v_readfirstlane_b32 s71, v33
-; SI-NEXT:    v_readfirstlane_b32 s77, v59
-; SI-NEXT:    v_writelane_b32 v62, s4, 16
-; SI-NEXT:    v_readfirstlane_b32 s4, v61
+; SI-NEXT:    v_readfirstlane_b32 s37, v45
+; SI-NEXT:    v_readfirstlane_b32 s24, v56
+; SI-NEXT:    v_readfirstlane_b32 s7, v57
+; SI-NEXT:    v_readfirstlane_b32 s92, v58
+; SI-NEXT:    v_readfirstlane_b32 s28, v59
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:132
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:128
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:100
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:84
-; SI-NEXT:    v_readfirstlane_b32 s98, v50
-; SI-NEXT:    v_readfirstlane_b32 s67, v53
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:64
-; SI-NEXT:    v_writelane_b32 v62, s4, 17
-; SI-NEXT:    v_readfirstlane_b32 s4, v38
-; SI-NEXT:    v_writelane_b32 v62, s4, 18
-; SI-NEXT:    v_readfirstlane_b32 s81, v34
-; SI-NEXT:    v_readfirstlane_b32 s75, v39
-; SI-NEXT:    v_readfirstlane_b32 s68, v42
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:80
-; SI-NEXT:    v_readfirstlane_b32 s49, v40
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:40
-; SI-NEXT:    v_readfirstlane_b32 s51, v54
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:80
+; SI-NEXT:    v_readfirstlane_b32 s35, v43
+; SI-NEXT:    v_readfirstlane_b32 s55, v46
+; SI-NEXT:    v_readfirstlane_b32 s68, v35
+; SI-NEXT:    v_readfirstlane_b32 s87, v37
+; SI-NEXT:    v_readfirstlane_b32 s67, v39
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_readfirstlane_b32 s97, v51
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:72
-; SI-NEXT:    v_readfirstlane_b32 s93, v36
-; SI-NEXT:    v_readfirstlane_b32 s24, v37
-; SI-NEXT:    v_readfirstlane_b32 s27, v48
-; SI-NEXT:    v_readfirstlane_b32 s84, v43
-; SI-NEXT:    v_readfirstlane_b32 s83, v44
-; SI-NEXT:    v_readfirstlane_b32 s85, v46
-; SI-NEXT:    v_readfirstlane_b32 s20, v47
-; SI-NEXT:    v_readfirstlane_b32 s4, v58
-; SI-NEXT:    v_writelane_b32 v62, s4, 19
-; SI-NEXT:    v_readfirstlane_b32 s23, v49
-; SI-NEXT:    v_readfirstlane_b32 s92, v52
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:336
+; SI-NEXT:    v_readfirstlane_b32 s74, v53
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:64
+; SI-NEXT:    v_readfirstlane_b32 s85, v48
+; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:40
+; SI-NEXT:    v_writelane_b32 v62, s4, 13
+; SI-NEXT:    v_readfirstlane_b32 s98, v40
+; SI-NEXT:    v_readfirstlane_b32 s69, v51
+; SI-NEXT:    v_readfirstlane_b32 s21, v36
+; SI-NEXT:    v_readfirstlane_b32 s58, v38
+; SI-NEXT:    v_readfirstlane_b32 s65, v49
+; SI-NEXT:    v_readfirstlane_b32 s62, v54
+; SI-NEXT:    v_readfirstlane_b32 s81, v44
+; SI-NEXT:    v_readfirstlane_b32 s71, v47
+; SI-NEXT:    v_readfirstlane_b32 s38, v60
+; SI-NEXT:    v_readfirstlane_b32 s86, v61
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:156
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:220
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_readfirstlane_b32 s90, v50
+; SI-NEXT:    v_readfirstlane_b32 s31, v52
+; SI-NEXT:    v_readfirstlane_b32 s4, v55
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:336
 ; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:4
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32
-; SI-NEXT:    v_readfirstlane_b32 s90, v35
-; SI-NEXT:    v_readfirstlane_b32 s4, v31
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:92
-; SI-NEXT:    v_readfirstlane_b32 s38, v32
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; SI-NEXT:    v_readfirstlane_b32 s72, v31
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:316
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:72
+; SI-NEXT:    v_readfirstlane_b32 s82, v56
+; SI-NEXT:    v_readfirstlane_b32 s95, v57
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_readfirstlane_b32 s70, v33
-; SI-NEXT:    v_readfirstlane_b32 s54, v59
-; SI-NEXT:    v_readfirstlane_b32 s57, v60
-; SI-NEXT:    v_readfirstlane_b32 s56, v61
-; SI-NEXT:    v_readfirstlane_b32 s59, v55
-; SI-NEXT:    v_readfirstlane_b32 s61, v41
-; SI-NEXT:    v_readfirstlane_b32 s19, v45
-; SI-NEXT:    v_readfirstlane_b32 s34, v50
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:316
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:284
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:252
+; SI-NEXT:    v_readfirstlane_b32 s39, v58
+; SI-NEXT:    v_readfirstlane_b32 s56, v59
+; SI-NEXT:    v_readfirstlane_b32 s57, v41
+; SI-NEXT:    v_readfirstlane_b32 s36, v42
+; SI-NEXT:    v_readfirstlane_b32 s73, v45
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:284
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:252
 ; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:188
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:124
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:28
-; SI-NEXT:    v_writelane_b32 v62, s4, 20
-; SI-NEXT:    v_readfirstlane_b32 s25, v53
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 24, v5
-; SI-NEXT:    v_readfirstlane_b32 s91, v40
-; SI-NEXT:    v_readfirstlane_b32 s37, v34
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:124
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:92
+; SI-NEXT:    v_readfirstlane_b32 s16, v34
+; SI-NEXT:    v_readfirstlane_b32 s48, v32
+; SI-NEXT:    v_readfirstlane_b32 s52, v33
+; SI-NEXT:    v_writelane_b32 v62, s4, 14
+; SI-NEXT:    v_readfirstlane_b32 s47, v35
+; SI-NEXT:    v_readfirstlane_b32 s60, v37
+; SI-NEXT:    v_readfirstlane_b32 s61, v39
+; SI-NEXT:    v_readfirstlane_b32 s89, v43
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_readfirstlane_b32 s47, v56
-; SI-NEXT:    v_readfirstlane_b32 s46, v57
-; SI-NEXT:    v_readfirstlane_b32 s22, v42
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 24, v22
-; SI-NEXT:    v_readfirstlane_b32 s72, v39
-; SI-NEXT:    v_readfirstlane_b32 s94, v51
-; SI-NEXT:    v_readfirstlane_b32 s48, v54
-; SI-NEXT:    v_readfirstlane_b32 s66, v43
-; SI-NEXT:    v_readfirstlane_b32 s69, v44
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_readfirstlane_b32 s45, v46
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v49
-; SI-NEXT:    s_and_b64 s[4:5], vcc, exec
-; SI-NEXT:    v_readfirstlane_b32 s4, v4
-; SI-NEXT:    v_writelane_b32 v62, s4, 21
-; SI-NEXT:    v_readfirstlane_b32 s4, v3
-; SI-NEXT:    v_writelane_b32 v62, s4, 22
-; SI-NEXT:    v_writelane_b32 v62, s17, 23
-; SI-NEXT:    v_writelane_b32 v62, s40, 24
-; SI-NEXT:    v_writelane_b32 v62, s16, 25
-; SI-NEXT:    v_writelane_b32 v62, s42, 26
-; SI-NEXT:    v_writelane_b32 v62, s46, 27
-; SI-NEXT:    v_writelane_b32 v62, s47, 28
-; SI-NEXT:    v_writelane_b32 v62, s56, 29
-; SI-NEXT:    v_writelane_b32 v62, s57, 30
-; SI-NEXT:    v_writelane_b32 v62, s45, 31
-; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_readfirstlane_b32 s58, v52
-; SI-NEXT:    v_writelane_b32 v62, s49, 32
-; SI-NEXT:    v_writelane_b32 v62, s58, 33
-; SI-NEXT:    v_writelane_b32 v62, s59, 34
-; SI-NEXT:    v_writelane_b32 v62, s52, 35
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_readfirstlane_b32 s60, v38
+; SI-NEXT:    v_readfirstlane_b32 s99, v46
 ; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:312
-; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:280
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:248
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:280
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:248
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:216
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:184
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:184
 ; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:152
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; SI-NEXT:    v_writelane_b32 v62, s60, 36
-; SI-NEXT:    v_writelane_b32 v62, s61, 37
-; SI-NEXT:    v_writelane_b32 v62, s93, 38
-; SI-NEXT:    v_writelane_b32 v62, s8, 39
-; SI-NEXT:    v_readfirstlane_b32 s62, v47
-; SI-NEXT:    v_writelane_b32 v62, s72, 40
-; SI-NEXT:    v_readfirstlane_b32 s73, v58
-; SI-NEXT:    v_writelane_b32 v62, s62, 41
-; SI-NEXT:    v_writelane_b32 v62, s73, 42
-; SI-NEXT:    v_writelane_b32 v62, s35, 43
-; SI-NEXT:    v_writelane_b32 v62, s94, 44
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24
+; SI-NEXT:    v_readfirstlane_b32 s54, v48
+; SI-NEXT:    v_readfirstlane_b32 s50, v53
+; SI-NEXT:    v_readfirstlane_b32 s78, v49
+; SI-NEXT:    v_readfirstlane_b32 s30, v51
+; SI-NEXT:    v_readfirstlane_b32 s66, v54
+; SI-NEXT:    v_readfirstlane_b32 s91, v40
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_readfirstlane_b32 s6, v44
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v50
+; SI-NEXT:    s_and_b64 s[4:5], vcc, exec
+; SI-NEXT:    v_readfirstlane_b32 s4, v10
+; SI-NEXT:    v_writelane_b32 v62, s4, 15
+; SI-NEXT:    v_readfirstlane_b32 s4, v2
+; SI-NEXT:    v_writelane_b32 v62, s4, 16
+; SI-NEXT:    v_writelane_b32 v62, s17, 17
+; SI-NEXT:    v_writelane_b32 v62, s15, 18
+; SI-NEXT:    v_writelane_b32 v62, s18, 19
+; SI-NEXT:    v_writelane_b32 v62, s43, 20
+; SI-NEXT:    v_writelane_b32 v62, s42, 21
+; SI-NEXT:    v_writelane_b32 v62, s44, 22
+; SI-NEXT:    v_writelane_b32 v62, s16, 23
+; SI-NEXT:    v_writelane_b32 v62, s49, 24
+; SI-NEXT:    v_writelane_b32 v62, s8, 25
+; SI-NEXT:    v_writelane_b32 v62, s6, 26
+; SI-NEXT:    v_readfirstlane_b32 s45, v52
+; SI-NEXT:    v_writelane_b32 v62, s56, 27
+; SI-NEXT:    v_writelane_b32 v62, s45, 28
+; SI-NEXT:    v_writelane_b32 v62, s53, 29
+; SI-NEXT:    v_writelane_b32 v62, s94, 30
+; SI-NEXT:    v_writelane_b32 v62, s57, 31
+; SI-NEXT:    v_writelane_b32 v62, s58, 32
+; SI-NEXT:    v_writelane_b32 v62, s47, 33
+; SI-NEXT:    v_readfirstlane_b32 s46, v55
+; SI-NEXT:    v_writelane_b32 v62, s40, 34
+; SI-NEXT:    v_readfirstlane_b32 s59, v47
+; SI-NEXT:    v_writelane_b32 v62, s46, 35
+; SI-NEXT:    v_writelane_b32 v62, s59, 36
+; SI-NEXT:    v_writelane_b32 v62, s60, 37
+; SI-NEXT:    v_writelane_b32 v62, s36, 38
+; SI-NEXT:    v_writelane_b32 v62, s65, 39
+; SI-NEXT:    v_writelane_b32 v62, s61, 40
+; SI-NEXT:    v_writelane_b32 v62, s73, 41
+; SI-NEXT:    v_writelane_b32 v62, s62, 42
+; SI-NEXT:    v_writelane_b32 v62, s72, 43
+; SI-NEXT:    v_writelane_b32 v62, s23, 44
 ; SI-NEXT:    v_writelane_b32 v62, s48, 45
-; SI-NEXT:    v_writelane_b32 v62, s91, 46
+; SI-NEXT:    v_writelane_b32 v62, s34, 46
+; SI-NEXT:    v_writelane_b32 v62, s78, 47
+; SI-NEXT:    v_writelane_b32 v62, s30, 48
+; SI-NEXT:    v_writelane_b32 v62, s54, 49
+; SI-NEXT:    v_writelane_b32 v62, s50, 50
+; SI-NEXT:    v_writelane_b32 v62, s52, 51
+; SI-NEXT:    v_writelane_b32 v62, s82, 52
+; SI-NEXT:    v_writelane_b32 v62, s66, 53
+; SI-NEXT:    v_readfirstlane_b32 s22, v36
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 24, v57
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 24, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v60
-; SI-NEXT:    v_lshlrev_b32_e32 v31, 24, v31
-; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v59
-; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v45
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 24, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 24, v41
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 24, v55
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 24, v50
-; SI-NEXT:    v_writelane_b32 v62, s66, 47
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v18
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT:    s_cbranch_scc0 .LBB97_2
+; SI-NEXT:    v_lshlrev_b32_e32 v23, 24, v58
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 24, v59
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 24, v60
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 24, v45
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 24, v61
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v42
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 24, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v31
+; SI-NEXT:    v_writelane_b32 v62, s91, 54
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT:    s_cbranch_scc0 .LBB97_4
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT:    v_mov_b32_e32 v21, v29
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v29
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT:    v_readlane_b32 s4, v62, 7
-; SI-NEXT:    v_readlane_b32 s5, v62, 6
-; SI-NEXT:    s_and_b32 s4, s4, 0xff
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; SI-NEXT:    v_mov_b32_e32 v5, v13
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v13
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; SI-NEXT:    v_readlane_b32 s5, v62, 5
+; SI-NEXT:    s_and_b32 s4, s20, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_or_b32 s4, s4, s5
-; SI-NEXT:    v_writelane_b32 v62, s4, 48
-; SI-NEXT:    v_readlane_b32 s4, v62, 5
+; SI-NEXT:    v_writelane_b32 v62, s4, 55
+; SI-NEXT:    v_readlane_b32 s4, v62, 4
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
-; SI-NEXT:    v_readlane_b32 s5, v62, 4
+; SI-NEXT:    v_readlane_b32 s5, v62, 3
 ; SI-NEXT:    s_lshl_b32 s4, s4, 16
 ; SI-NEXT:    s_lshl_b32 s5, s5, 24
 ; SI-NEXT:    s_or_b32 s63, s5, s4
-; SI-NEXT:    v_readlane_b32 s4, v62, 9
+; SI-NEXT:    v_readlane_b32 s4, v62, 6
 ; SI-NEXT:    s_and_b32 s5, s4, 0xff
-; SI-NEXT:    v_readlane_b32 s4, v62, 8
 ; SI-NEXT:    s_lshl_b32 s5, s5, 16
-; SI-NEXT:    s_lshl_b32 s9, s4, 24
+; SI-NEXT:    s_lshl_b32 s9, s19, 24
 ; SI-NEXT:    v_readlane_b32 s4, v62, 0
 ; SI-NEXT:    s_or_b32 s9, s9, s5
 ; SI-NEXT:    s_and_b32 s5, s4, 0xff
 ; SI-NEXT:    s_lshl_b32 s10, s29, 8
 ; SI-NEXT:    s_or_b32 s4, s5, s10
-; SI-NEXT:    v_writelane_b32 v62, s4, 49
-; SI-NEXT:    s_and_b32 s5, s74, 0xff
+; SI-NEXT:    v_writelane_b32 v62, s4, 56
+; SI-NEXT:    s_and_b32 s5, s76, 0xff
+; SI-NEXT:    v_readlane_b32 s10, v62, 16
 ; SI-NEXT:    s_lshl_b32 s5, s5, 16
-; SI-NEXT:    v_writelane_b32 v62, s54, 50
-; SI-NEXT:    s_lshl_b32 s11, s21, 24
-; SI-NEXT:    s_mov_b32 s18, s22
-; SI-NEXT:    s_mov_b32 s22, s21
-; SI-NEXT:    s_or_b32 s21, s11, s5
+; SI-NEXT:    s_lshl_b32 s11, s10, 24
+; SI-NEXT:    s_or_b32 s5, s11, s5
 ; SI-NEXT:    s_and_b32 s11, s26, 0xff
-; SI-NEXT:    v_readlane_b32 s4, v62, 1
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s12, s4, 24
+; SI-NEXT:    s_lshl_b32 s12, s27, 24
 ; SI-NEXT:    s_or_b32 s14, s12, s11
-; SI-NEXT:    s_and_b32 s11, s80, 0xff
-; SI-NEXT:    s_lshl_b32 s12, s65, 8
-; SI-NEXT:    s_or_b32 s12, s11, s12
-; SI-NEXT:    s_and_b32 s11, s55, 0xff
+; SI-NEXT:    s_and_b32 s11, s83, 0xff
+; SI-NEXT:    s_lshl_b32 s12, s25, 8
+; SI-NEXT:    s_or_b32 s10, s11, s12
+; SI-NEXT:    v_writelane_b32 v62, s10, 57
+; SI-NEXT:    s_and_b32 s11, s64, 0xff
+; SI-NEXT:    v_readlane_b32 s10, v62, 15
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s13, s64, 24
+; SI-NEXT:    s_lshl_b32 s13, s10, 24
 ; SI-NEXT:    s_or_b32 s41, s13, s11
-; SI-NEXT:    s_and_b32 s11, s89, 0xff
-; SI-NEXT:    s_lshl_b32 s13, s99, 8
-; SI-NEXT:    s_or_b32 s13, s11, s13
+; SI-NEXT:    s_and_b32 s11, s43, 0xff
+; SI-NEXT:    s_lshl_b32 s13, s15, 8
+; SI-NEXT:    s_or_b32 s10, s11, s13
 ; SI-NEXT:    s_and_b32 s11, s96, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s76, 24
+; SI-NEXT:    s_lshl_b32 s15, s80, 24
 ; SI-NEXT:    s_or_b32 s43, s15, s11
-; SI-NEXT:    s_and_b32 s11, s42, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s16, 8
-; SI-NEXT:    s_or_b32 s16, s11, s15
-; SI-NEXT:    s_and_b32 s11, s40, 0xff
+; SI-NEXT:    s_and_b32 s11, s44, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s42, 8
+; SI-NEXT:    s_or_b32 s13, s11, s15
+; SI-NEXT:    s_and_b32 s11, s18, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s87, 24
+; SI-NEXT:    s_lshl_b32 s15, s97, 24
 ; SI-NEXT:    s_or_b32 s44, s15, s11
-; SI-NEXT:    s_and_b32 s11, s73, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s62, 8
-; SI-NEXT:    s_or_b32 s62, s11, s15
-; SI-NEXT:    s_and_b32 s11, s58, 0xff
+; SI-NEXT:    s_and_b32 s11, s59, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s46, 8
+; SI-NEXT:    s_or_b32 s12, s11, s15
+; SI-NEXT:    s_and_b32 s11, s45, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s45, 24
+; SI-NEXT:    s_lshl_b32 s15, s6, 24
 ; SI-NEXT:    s_or_b32 s45, s15, s11
-; SI-NEXT:    s_and_b32 s11, s48, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s94, 8
+; SI-NEXT:    s_and_b32 s11, s30, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s78, 8
+; SI-NEXT:    v_writelane_b32 v62, s10, 58
 ; SI-NEXT:    s_or_b32 s10, s11, s15
-; SI-NEXT:    s_and_b32 s11, s46, 0xff
+; SI-NEXT:    s_and_b32 s11, s99, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s47, 24
+; SI-NEXT:    s_lshl_b32 s15, s89, 24
 ; SI-NEXT:    s_or_b32 s46, s15, s11
-; SI-NEXT:    s_and_b32 s11, s25, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s34, 8
-; SI-NEXT:    s_or_b32 s94, s11, s15
-; SI-NEXT:    s_and_b32 s11, s72, 0xff
+; SI-NEXT:    s_and_b32 s11, s61, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s60, 8
+; SI-NEXT:    s_or_b32 s6, s11, s15
+; SI-NEXT:    s_and_b32 s11, s22, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s60, 24
+; SI-NEXT:    s_lshl_b32 s15, s47, 24
 ; SI-NEXT:    s_or_b32 s47, s15, s11
-; SI-NEXT:    s_and_b32 s11, s61, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s59, 8
-; SI-NEXT:    s_or_b32 s73, s11, s15
-; SI-NEXT:    s_and_b32 s11, s56, 0xff
+; SI-NEXT:    s_and_b32 s11, s57, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s56, 8
+; SI-NEXT:    v_writelane_b32 v62, s6, 59
+; SI-NEXT:    s_or_b32 s6, s11, s15
+; SI-NEXT:    s_and_b32 s11, s39, 0xff
+; SI-NEXT:    v_writelane_b32 v62, s6, 60
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s57, 24
-; SI-NEXT:    v_readlane_b32 s4, v62, 20
+; SI-NEXT:    s_lshl_b32 s15, s95, 24
 ; SI-NEXT:    s_or_b32 s56, s15, s11
-; SI-NEXT:    s_and_b32 s11, s38, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s4, 8
+; SI-NEXT:    s_and_b32 s11, s48, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s72, 8
+; SI-NEXT:    v_readlane_b32 s6, v62, 14
 ; SI-NEXT:    s_or_b32 s48, s11, s15
-; SI-NEXT:    s_and_b32 s11, s92, 0xff
+; SI-NEXT:    s_and_b32 s11, s6, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s23, 24
+; SI-NEXT:    s_lshl_b32 s15, s31, 24
 ; SI-NEXT:    s_or_b32 vcc_lo, s15, s11
-; SI-NEXT:    s_and_b32 s11, s20, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s85, 8
+; SI-NEXT:    s_and_b32 s11, s86, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s38, 8
 ; SI-NEXT:    s_or_b32 s72, s11, s15
-; SI-NEXT:    s_and_b32 s11, s83, 0xff
+; SI-NEXT:    s_and_b32 s11, s71, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s84, 24
+; SI-NEXT:    s_lshl_b32 s15, s81, 24
 ; SI-NEXT:    s_or_b32 vcc_hi, s15, s11
-; SI-NEXT:    s_and_b32 s11, s93, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s97, 8
+; SI-NEXT:    s_and_b32 s11, s58, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s85, 8
 ; SI-NEXT:    s_or_b32 s57, s11, s15
-; SI-NEXT:    s_and_b32 s11, s67, 0xff
+; SI-NEXT:    s_and_b32 s11, s69, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s51, 24
-; SI-NEXT:    v_writelane_b32 v62, s67, 51
-; SI-NEXT:    s_mov_b32 s67, s51
-; SI-NEXT:    s_mov_b32 s51, s74
+; SI-NEXT:    s_lshl_b32 s15, s74, 24
+; SI-NEXT:    v_writelane_b32 v62, s74, 61
 ; SI-NEXT:    s_or_b32 s74, s15, s11
-; SI-NEXT:    s_and_b32 s11, s98, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s75, 8
-; SI-NEXT:    v_readlane_b32 s4, v62, 18
-; SI-NEXT:    v_writelane_b32 v62, s87, 52
+; SI-NEXT:    s_and_b32 s11, s87, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s21, 8
 ; SI-NEXT:    s_or_b32 s58, s11, s15
-; SI-NEXT:    s_and_b32 s11, s4, 0xff
-; SI-NEXT:    v_writelane_b32 v62, s25, 53
+; SI-NEXT:    s_and_b32 s11, s68, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s81, 24
-; SI-NEXT:    v_readlane_b32 s4, v62, 15
-; SI-NEXT:    s_mov_b32 s54, s75
+; SI-NEXT:    s_lshl_b32 s15, s28, 24
 ; SI-NEXT:    s_or_b32 s75, s15, s11
-; SI-NEXT:    s_and_b32 s11, s77, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s4, 8
-; SI-NEXT:    v_readlane_b32 s4, v62, 14
+; SI-NEXT:    s_and_b32 s11, s24, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s55, 8
+; SI-NEXT:    v_writelane_b32 v62, s25, 62
 ; SI-NEXT:    s_or_b32 s59, s11, s15
-; SI-NEXT:    s_and_b32 s11, s4, 0xff
-; SI-NEXT:    v_readlane_b32 s4, v62, 13
+; SI-NEXT:    s_and_b32 s11, s37, 0xff
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s4, 24
-; SI-NEXT:    v_readlane_b32 s4, v62, 12
-; SI-NEXT:    s_mov_b32 s95, s69
-; SI-NEXT:    s_mov_b32 s69, s76
+; SI-NEXT:    s_lshl_b32 s15, s51, 24
+; SI-NEXT:    v_readlane_b32 s4, v62, 13
+; SI-NEXT:    s_mov_b32 s18, s21
+; SI-NEXT:    s_mov_b32 s21, s97
+; SI-NEXT:    s_mov_b32 s97, s37
+; SI-NEXT:    s_mov_b32 s37, s76
 ; SI-NEXT:    s_or_b32 s76, s15, s11
-; SI-NEXT:    s_and_b32 s11, s86, 0xff
+; SI-NEXT:    s_and_b32 s11, s35, 0xff
 ; SI-NEXT:    s_lshl_b32 s15, s4, 8
-; SI-NEXT:    v_readlane_b32 s4, v62, 11
 ; SI-NEXT:    s_or_b32 s60, s11, s15
-; SI-NEXT:    s_and_b32 s11, s4, 0xff
+; SI-NEXT:    s_and_b32 s11, s77, 0xff
+; SI-NEXT:    v_readlane_b32 s4, v62, 12
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s78, 24
-; SI-NEXT:    s_mov_b32 s93, s99
-; SI-NEXT:    s_mov_b32 s99, s84
-; SI-NEXT:    s_mov_b32 s84, s77
+; SI-NEXT:    s_lshl_b32 s15, s4, 24
+; SI-NEXT:    v_readlane_b32 s4, v62, 11
+; SI-NEXT:    s_mov_b32 s6, s95
+; SI-NEXT:    s_mov_b32 s95, s39
+; SI-NEXT:    s_mov_b32 s39, s89
+; SI-NEXT:    s_mov_b32 s89, s99
+; SI-NEXT:    s_mov_b32 s99, s83
+; SI-NEXT:    s_mov_b32 s83, s55
+; SI-NEXT:    s_mov_b32 s55, s64
+; SI-NEXT:    s_mov_b32 s64, s35
+; SI-NEXT:    s_mov_b32 s35, s77
 ; SI-NEXT:    s_or_b32 s77, s15, s11
-; SI-NEXT:    s_and_b32 s11, s82, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s53, 8
+; SI-NEXT:    s_and_b32 s11, s4, 0xff
+; SI-NEXT:    v_readlane_b32 s4, v62, 10
+; SI-NEXT:    s_lshl_b32 s15, s4, 8
+; SI-NEXT:    v_readlane_b32 s4, v62, 9
 ; SI-NEXT:    s_or_b32 s61, s11, s15
-; SI-NEXT:    s_and_b32 s11, s31, 0xff
+; SI-NEXT:    s_and_b32 s11, s4, 0xff
+; SI-NEXT:    v_readlane_b32 s4, v62, 8
 ; SI-NEXT:    s_lshl_b32 s11, s11, 16
-; SI-NEXT:    s_lshl_b32 s15, s50, 24
-; SI-NEXT:    s_mov_b32 s4, s85
-; SI-NEXT:    s_mov_b32 s85, s83
-; SI-NEXT:    s_mov_b32 s83, s82
-; SI-NEXT:    s_mov_b32 s82, s53
-; SI-NEXT:    s_mov_b32 s53, s50
-; SI-NEXT:    s_mov_b32 s50, s31
-; SI-NEXT:    s_mov_b32 s31, s78
+; SI-NEXT:    s_lshl_b32 s15, s4, 24
 ; SI-NEXT:    s_or_b32 s78, s15, s11
-; SI-NEXT:    v_readlane_b32 s11, v62, 10
+; SI-NEXT:    v_readlane_b32 s11, v62, 7
 ; SI-NEXT:    s_and_b32 s11, s11, 0xff
 ; SI-NEXT:    s_lshl_b32 s15, s17, 8
 ; SI-NEXT:    s_or_b32 s11, s11, s15
 ; SI-NEXT:    s_and_b32 s11, s11, 0xffff
 ; SI-NEXT:    v_mov_b32_e32 v51, s9
-; SI-NEXT:    s_or_b32 s6, s11, s9
-; SI-NEXT:    v_readlane_b32 s9, v62, 3
-; SI-NEXT:    v_readlane_b32 s11, v62, 2
+; SI-NEXT:    s_or_b32 s17, s11, s9
+; SI-NEXT:    v_readlane_b32 s9, v62, 2
+; SI-NEXT:    v_readlane_b32 s11, v62, 1
 ; SI-NEXT:    s_and_b32 s9, s9, 0xff
 ; SI-NEXT:    s_lshl_b32 s15, s11, 8
 ; SI-NEXT:    s_or_b32 s9, s9, s15
 ; SI-NEXT:    s_and_b32 s9, s9, 0xffff
-; SI-NEXT:    s_or_b32 s17, s9, s14
-; SI-NEXT:    v_readlane_b32 s9, v62, 22
+; SI-NEXT:    s_mov_b32 s4, s96
+; SI-NEXT:    s_mov_b32 s96, s24
 ; SI-NEXT:    v_mov_b32_e32 v52, s14
-; SI-NEXT:    s_and_b32 s14, s9, 0xff
-; SI-NEXT:    v_readlane_b32 s9, v62, 21
-; SI-NEXT:    s_lshl_b32 s15, s9, 8
+; SI-NEXT:    s_or_b32 s24, s9, s14
+; SI-NEXT:    s_and_b32 s14, s93, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s84, 8
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_or_b32_e32 v53, v6, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
 ; SI-NEXT:    v_or_b32_e32 v50, s14, v53
-; SI-NEXT:    s_and_b32 s14, s30, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s88, 8
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v20
+; SI-NEXT:    s_and_b32 s14, s8, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s49, 8
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v21
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_or_b32_e32 v54, v14, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
 ; SI-NEXT:    v_or_b32_e32 v17, s14, v54
-; SI-NEXT:    s_and_b32 s14, s8, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s52, 8
+; SI-NEXT:    s_and_b32 s14, s40, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s53, 8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v19
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v29
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v55, v40, v1
+; SI-NEXT:    v_or_b32_e32 v55, v18, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
 ; SI-NEXT:    v_or_b32_e32 v16, s14, v55
-; SI-NEXT:    s_and_b32 s14, s35, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s28, 8
+; SI-NEXT:    s_and_b32 s14, s34, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s23, 8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v29
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v13
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    s_waitcnt expcnt(4)
-; SI-NEXT:    v_or_b32_e32 v40, v42, v1
+; SI-NEXT:    v_or_b32_e32 v40, v19, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
 ; SI-NEXT:    v_or_b32_e32 v15, s14, v40
-; SI-NEXT:    s_and_b32 s14, s95, 0xff
+; SI-NEXT:    s_and_b32 s14, s91, 0xff
 ; SI-NEXT:    s_lshl_b32 s15, s66, 8
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v56
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v43
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v41, v61, v1
+; SI-NEXT:    v_or_b32_e32 v41, v22, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
 ; SI-NEXT:    v_or_b32_e32 v12, s14, v41
-; SI-NEXT:    s_and_b32 s14, s18, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s91, 8
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v57
+; SI-NEXT:    s_and_b32 s14, s50, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s54, 8
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    s_waitcnt expcnt(3)
-; SI-NEXT:    v_or_b32_e32 v42, v60, v1
+; SI-NEXT:    v_or_b32_e32 v42, v23, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
 ; SI-NEXT:    v_or_b32_e32 v11, s14, v42
-; SI-NEXT:    s_and_b32 s14, s37, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s19, 8
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v33
+; SI-NEXT:    s_and_b32 s14, s73, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s36, 8
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v46
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    s_mov_b32 s91, s6
-; SI-NEXT:    v_or_b32_e32 v59, v31, v1
+; SI-NEXT:    v_or_b32_e32 v28, v59, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
-; SI-NEXT:    v_readlane_b32 s6, v62, 50
-; SI-NEXT:    v_or_b32_e32 v10, s14, v59
-; SI-NEXT:    s_and_b32 s14, s6, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s70, 8
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v30
+; SI-NEXT:    v_or_b32_e32 v10, s14, v28
+; SI-NEXT:    s_and_b32 s14, s82, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s52, 8
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v26
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v5, v24, v1
+; SI-NEXT:    v_or_b32_e32 v60, v24, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
-; SI-NEXT:    v_readlane_b32 s6, v62, 19
-; SI-NEXT:    v_or_b32_e32 v9, s14, v5
+; SI-NEXT:    v_or_b32_e32 v9, s14, v60
 ; SI-NEXT:    s_and_b32 s14, s90, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s6, 8
+; SI-NEXT:    s_lshl_b32 s15, s16, 8
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v35
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v13, v25, v1
+; SI-NEXT:    v_or_b32_e32 v31, v44, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
-; SI-NEXT:    v_or_b32_e32 v8, s14, v13
-; SI-NEXT:    s_and_b32 s14, s27, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s24, 8
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v32
+; SI-NEXT:    v_or_b32_e32 v8, s14, v31
+; SI-NEXT:    s_and_b32 s14, s62, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s65, 8
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v37
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_mov_b32_e32 v26, v31
-; SI-NEXT:    v_or_b32_e32 v31, v27, v1
+; SI-NEXT:    v_or_b32_e32 v61, v45, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
-; SI-NEXT:    v_or_b32_e32 v7, s14, v31
-; SI-NEXT:    s_and_b32 s14, s49, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s68, 8
+; SI-NEXT:    v_or_b32_e32 v7, s14, v61
+; SI-NEXT:    s_and_b32 s14, s98, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s67, 8
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v38
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_mov_b32_e32 v28, v25
-; SI-NEXT:    v_mov_b32_e32 v25, v60
-; SI-NEXT:    v_or_b32_e32 v60, v43, v1
+; SI-NEXT:    s_waitcnt expcnt(5)
+; SI-NEXT:    v_or_b32_e32 v6, v47, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
-; SI-NEXT:    v_readlane_b32 s6, v62, 17
-; SI-NEXT:    v_or_b32_e32 v4, s14, v60
-; SI-NEXT:    s_and_b32 s14, s6, 0xff
-; SI-NEXT:    v_readlane_b32 s6, v62, 16
-; SI-NEXT:    s_lshl_b32 s15, s6, 8
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v39
+; SI-NEXT:    v_or_b32_e32 v4, s14, v6
+; SI-NEXT:    s_and_b32 s14, s92, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s7, 8
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v33
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_mov_b32_e32 v23, v27
-; SI-NEXT:    v_mov_b32_e32 v27, v24
-; SI-NEXT:    v_mov_b32_e32 v24, v61
-; SI-NEXT:    v_or_b32_e32 v61, v44, v1
+; SI-NEXT:    v_readlane_b32 s8, v62, 55
+; SI-NEXT:    s_waitcnt expcnt(4)
+; SI-NEXT:    v_mov_b32_e32 v22, v14
+; SI-NEXT:    v_or_b32_e32 v14, v56, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
-; SI-NEXT:    v_or_b32_e32 v2, s14, v61
-; SI-NEXT:    s_and_b32 s14, s71, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s36, 8
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v48
+; SI-NEXT:    s_and_b32 s8, s8, 0xffff
+; SI-NEXT:    v_or_b32_e32 v2, s14, v14
+; SI-NEXT:    s_and_b32 s14, s70, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s94, 8
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v39
+; SI-NEXT:    s_or_b32 s42, s8, s63
+; SI-NEXT:    v_readlane_b32 s8, v62, 56
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_readlane_b32 s6, v62, 48
-; SI-NEXT:    v_or_b32_e32 v6, v45, v1
+; SI-NEXT:    s_and_b32 s8, s8, 0xffff
+; SI-NEXT:    s_waitcnt expcnt(2)
+; SI-NEXT:    v_mov_b32_e32 v32, v23
+; SI-NEXT:    v_mov_b32_e32 v23, v18
+; SI-NEXT:    v_or_b32_e32 v18, v57, v1
 ; SI-NEXT:    s_and_b32 s14, s14, 0xffff
-; SI-NEXT:    s_and_b32 s8, s6, 0xffff
-; SI-NEXT:    v_readlane_b32 s6, v62, 49
-; SI-NEXT:    v_or_b32_e32 v1, s14, v6
-; SI-NEXT:    s_and_b32 s14, s79, 0xff
-; SI-NEXT:    s_lshl_b32 s15, s7, 8
+; SI-NEXT:    s_or_b32 s40, s8, s5
+; SI-NEXT:    v_readlane_b32 s8, v62, 57
+; SI-NEXT:    v_or_b32_e32 v1, s14, v18
+; SI-NEXT:    s_and_b32 s14, s88, 0xff
+; SI-NEXT:    s_lshl_b32 s15, s79, 8
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v34
-; SI-NEXT:    s_or_b32 s42, s8, s63
-; SI-NEXT:    s_and_b32 s8, s6, 0xffff
+; SI-NEXT:    s_and_b32 s8, s8, 0xffff
+; SI-NEXT:    v_readlane_b32 s9, v62, 60
 ; SI-NEXT:    s_or_b32 s14, s14, s15
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    s_or_b32 s40, s8, s21
-; SI-NEXT:    s_and_b32 s8, s12, 0xffff
-; SI-NEXT:    v_or_b32_e32 v14, v46, v3
-; SI-NEXT:    s_and_b32 s14, s14, 0xffff
 ; SI-NEXT:    s_or_b32 s15, s8, s41
-; SI-NEXT:    s_and_b32 s8, s13, 0xffff
-; SI-NEXT:    v_or_b32_e32 v3, s14, v14
-; SI-NEXT:    s_or_b32 s14, s8, s43
-; SI-NEXT:    s_and_b32 s8, s16, 0xffff
-; SI-NEXT:    s_and_b32 s16, s73, 0xffff
-; SI-NEXT:    s_or_b32 s13, s8, s44
-; SI-NEXT:    s_and_b32 s8, s62, 0xffff
-; SI-NEXT:    s_or_b32 s35, s16, s56
+; SI-NEXT:    v_readlane_b32 s8, v62, 58
+; SI-NEXT:    s_and_b32 s16, s9, 0xffff
+; SI-NEXT:    v_mov_b32_e32 v27, v26
+; SI-NEXT:    v_mov_b32_e32 v26, v24
+; SI-NEXT:    v_mov_b32_e32 v24, v19
+; SI-NEXT:    v_or_b32_e32 v19, v58, v3
+; SI-NEXT:    s_and_b32 s14, s14, 0xffff
+; SI-NEXT:    s_and_b32 s8, s8, 0xffff
+; SI-NEXT:    s_or_b32 s36, s16, s56
 ; SI-NEXT:    s_and_b32 s16, s48, 0xffff
-; SI-NEXT:    s_or_b32 s12, s8, s45
-; SI-NEXT:    s_and_b32 s8, s10, 0xffff
-; SI-NEXT:    s_or_b32 s52, s16, vcc_lo
+; SI-NEXT:    v_or_b32_e32 v3, s14, v19
+; SI-NEXT:    s_or_b32 s14, s8, s43
+; SI-NEXT:    s_and_b32 s8, s13, 0xffff
+; SI-NEXT:    s_or_b32 s53, s16, vcc_lo
 ; SI-NEXT:    s_and_b32 s16, s72, 0xffff
-; SI-NEXT:    s_or_b32 s10, s8, s46
-; SI-NEXT:    s_and_b32 s8, s94, 0xffff
+; SI-NEXT:    s_or_b32 s13, s8, s44
+; SI-NEXT:    s_and_b32 s8, s12, 0xffff
 ; SI-NEXT:    s_or_b32 s94, s16, vcc_hi
 ; SI-NEXT:    s_and_b32 s16, s57, 0xffff
+; SI-NEXT:    s_or_b32 s12, s8, s45
+; SI-NEXT:    s_and_b32 s8, s10, 0xffff
 ; SI-NEXT:    s_or_b32 s49, s16, s74
 ; SI-NEXT:    s_and_b32 s16, s58, 0xffff
+; SI-NEXT:    s_or_b32 s10, s8, s46
+; SI-NEXT:    v_readlane_b32 s8, v62, 59
 ; SI-NEXT:    s_or_b32 s48, s16, s75
 ; SI-NEXT:    s_and_b32 s16, s59, 0xffff
-; SI-NEXT:    s_mov_b32 s25, s23
+; SI-NEXT:    s_and_b32 s8, s8, 0xffff
 ; SI-NEXT:    s_or_b32 s11, s16, s76
 ; SI-NEXT:    s_and_b32 s16, s60, 0xffff
 ; SI-NEXT:    s_and_b32 s23, s61, 0xffff
-; SI-NEXT:    s_mov_b32 s87, s34
-; SI-NEXT:    s_mov_b32 s34, s55
-; SI-NEXT:    s_mov_b32 s55, s22
+; SI-NEXT:    s_mov_b32 s30, s87
+; SI-NEXT:    s_mov_b32 s87, s85
 ; SI-NEXT:    s_or_b32 s8, s8, s47
 ; SI-NEXT:    s_or_b32 s9, s16, s77
 ; SI-NEXT:    s_or_b32 s16, s23, s78
-; SI-NEXT:    s_mov_b32 s22, s18
-; SI-NEXT:    v_mov_b32_e32 v49, v30
 ; SI-NEXT:    v_mov_b32_e32 v36, v35
-; SI-NEXT:    v_mov_b32_e32 v37, v32
-; SI-NEXT:    v_mov_b32_e32 v18, v43
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v33, v44
+; SI-NEXT:    v_mov_b32_e32 v30, v37
 ; SI-NEXT:    v_mov_b32_e32 v35, v45
-; SI-NEXT:    v_mov_b32_e32 v22, v46
+; SI-NEXT:    v_mov_b32_e32 v20, v47
+; SI-NEXT:    v_mov_b32_e32 v49, v56
+; SI-NEXT:    v_mov_b32_e32 v48, v39
+; SI-NEXT:    v_mov_b32_e32 v39, v57
+; SI-NEXT:    v_mov_b32_e32 v25, v58
 ; SI-NEXT:    v_alignbit_b32 v57, s42, v51, 16
 ; SI-NEXT:    v_alignbit_b32 v58, s40, v52, 16
 ; SI-NEXT:    v_alignbit_b32 v56, s15, v53, 16
 ; SI-NEXT:    v_alignbit_b32 v47, s14, v54, 16
+; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    v_alignbit_b32 v46, s13, v55, 16
 ; SI-NEXT:    v_alignbit_b32 v45, s12, v40, 16
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v44, s10, v41, 16
 ; SI-NEXT:    v_alignbit_b32 v43, s8, v42, 16
-; SI-NEXT:    v_alignbit_b32 v42, s35, v59, 16
-; SI-NEXT:    v_alignbit_b32 v41, s52, v5, 16
-; SI-NEXT:    v_alignbit_b32 v40, s94, v13, 16
-; SI-NEXT:    v_alignbit_b32 v55, s49, v31, 16
-; SI-NEXT:    v_mov_b32_e32 v31, v26
-; SI-NEXT:    v_alignbit_b32 v54, s48, v60, 16
-; SI-NEXT:    v_mov_b32_e32 v60, v25
-; SI-NEXT:    v_mov_b32_e32 v25, v28
-; SI-NEXT:    v_alignbit_b32 v53, s11, v61, 16
-; SI-NEXT:    v_mov_b32_e32 v61, v24
-; SI-NEXT:    v_mov_b32_e32 v24, v27
-; SI-NEXT:    v_alignbit_b32 v52, s9, v6, 16
-; SI-NEXT:    v_alignbit_b32 v51, s16, v14, 16
+; SI-NEXT:    v_alignbit_b32 v42, s36, v28, 16
+; SI-NEXT:    v_alignbit_b32 v41, s53, v60, 16
+; SI-NEXT:    v_alignbit_b32 v40, s94, v31, 16
+; SI-NEXT:    v_alignbit_b32 v55, s49, v61, 16
+; SI-NEXT:    v_alignbit_b32 v54, s48, v6, 16
+; SI-NEXT:    v_alignbit_b32 v53, s11, v14, 16
+; SI-NEXT:    v_mov_b32_e32 v14, v22
+; SI-NEXT:    v_alignbit_b32 v52, s9, v18, 16
+; SI-NEXT:    v_mov_b32_e32 v18, v23
+; SI-NEXT:    v_alignbit_b32 v51, s16, v19, 16
+; SI-NEXT:    v_mov_b32_e32 v19, v24
+; SI-NEXT:    v_mov_b32_e32 v24, v26
 ; SI-NEXT:    s_lshr_b32 s73, s63, 16
-; SI-NEXT:    s_lshr_b32 s72, s21, 16
+; SI-NEXT:    s_lshr_b32 s72, s5, 16
 ; SI-NEXT:    s_lshr_b32 s63, s41, 16
 ; SI-NEXT:    s_lshr_b32 s62, s43, 16
 ; SI-NEXT:    s_lshr_b32 s61, s44, 16
@@ -211687,573 +211719,477 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    s_lshr_b32 s56, vcc_lo, 16
 ; SI-NEXT:    s_lshr_b32 s47, vcc_hi, 16
 ; SI-NEXT:    s_lshr_b32 s46, s74, 16
-; SI-NEXT:    s_mov_b32 s74, s51
-; SI-NEXT:    s_mov_b32 s51, s67
-; SI-NEXT:    v_readlane_b32 s67, v62, 51
+; SI-NEXT:    v_readlane_b32 s25, v62, 62
+; SI-NEXT:    v_readlane_b32 s74, v62, 61
 ; SI-NEXT:    s_lshr_b32 s45, s75, 16
-; SI-NEXT:    s_mov_b32 s23, s25
-; SI-NEXT:    s_mov_b32 s21, s55
-; SI-NEXT:    s_mov_b32 s55, s34
-; SI-NEXT:    s_mov_b32 s75, s54
-; SI-NEXT:    s_mov_b32 s34, s87
-; SI-NEXT:    v_readlane_b32 s25, v62, 53
-; SI-NEXT:    v_readlane_b32 s87, v62, 52
 ; SI-NEXT:    s_lshr_b32 s44, s76, 16
-; SI-NEXT:    v_readlane_b32 s54, v62, 50
+; SI-NEXT:    s_mov_b32 s76, s37
+; SI-NEXT:    s_mov_b32 s37, s97
+; SI-NEXT:    s_mov_b32 s97, s21
+; SI-NEXT:    s_mov_b32 s21, s18
+; SI-NEXT:    s_mov_b32 s18, s17
+; SI-NEXT:    s_mov_b32 s85, s87
+; SI-NEXT:    s_mov_b32 s87, s30
+; SI-NEXT:    s_mov_b32 s17, s24
 ; SI-NEXT:    s_lshr_b32 s43, s77, 16
-; SI-NEXT:    s_mov_b32 s76, s69
-; SI-NEXT:    s_mov_b32 s69, s95
-; SI-NEXT:    s_mov_b32 s77, s84
-; SI-NEXT:    s_mov_b32 s84, s99
-; SI-NEXT:    s_mov_b32 s99, s93
+; SI-NEXT:    s_mov_b32 s77, s35
+; SI-NEXT:    s_mov_b32 s35, s64
+; SI-NEXT:    s_mov_b32 s64, s55
+; SI-NEXT:    s_mov_b32 s55, s83
+; SI-NEXT:    s_mov_b32 s83, s99
+; SI-NEXT:    s_mov_b32 s99, s89
+; SI-NEXT:    s_mov_b32 s89, s39
+; SI-NEXT:    s_mov_b32 s39, s95
+; SI-NEXT:    s_mov_b32 s95, s6
 ; SI-NEXT:    s_lshr_b32 s41, s78, 16
-; SI-NEXT:    s_mov_b32 s78, s31
-; SI-NEXT:    s_mov_b32 s31, s50
-; SI-NEXT:    s_mov_b32 s50, s53
-; SI-NEXT:    s_mov_b32 s53, s82
-; SI-NEXT:    s_mov_b32 s82, s83
-; SI-NEXT:    s_mov_b32 s83, s85
-; SI-NEXT:    s_mov_b32 s85, s4
-; SI-NEXT:    s_mov_b64 s[4:5], 0
-; SI-NEXT:    v_mov_b32_e32 v6, v20
-; SI-NEXT:    v_mov_b32_e32 v13, v19
-; SI-NEXT:    v_mov_b32_e32 v14, v29
-; SI-NEXT:    s_branch .LBB97_3
-; SI-NEXT:  .LBB97_2:
-; SI-NEXT:    s_waitcnt expcnt(6)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(5)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT:    v_mov_b32_e32 v22, v46
-; SI-NEXT:    v_mov_b32_e32 v37, v32
-; SI-NEXT:    v_mov_b32_e32 v36, v35
-; SI-NEXT:    v_mov_b32_e32 v35, v45
-; SI-NEXT:    v_mov_b32_e32 v49, v30
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v33, v44
-; SI-NEXT:    v_mov_b32_e32 v18, v43
-; SI-NEXT:    v_mov_b32_e32 v23, v27
-; SI-NEXT:    v_mov_b32_e32 v21, v29
-; SI-NEXT:    s_mov_b64 s[4:5], -1
-; SI-NEXT:    ; implicit-def: $sgpr91
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $sgpr42
-; SI-NEXT:    ; implicit-def: $sgpr73
-; SI-NEXT:    ; implicit-def: $sgpr17
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $sgpr40
-; SI-NEXT:    ; implicit-def: $sgpr72
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $sgpr15
-; SI-NEXT:    ; implicit-def: $sgpr63
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $sgpr14
-; SI-NEXT:    ; implicit-def: $sgpr62
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $sgpr13
-; SI-NEXT:    ; implicit-def: $sgpr61
-; SI-NEXT:    ; implicit-def: $vgpr15
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $sgpr12
-; SI-NEXT:    ; implicit-def: $sgpr60
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $sgpr10
-; SI-NEXT:    ; implicit-def: $sgpr59
-; SI-NEXT:    ; implicit-def: $vgpr11
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $sgpr8
-; SI-NEXT:    ; implicit-def: $sgpr58
-; SI-NEXT:    ; implicit-def: $vgpr10
-; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $sgpr35
-; SI-NEXT:    ; implicit-def: $sgpr57
-; SI-NEXT:    ; implicit-def: $vgpr9
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $sgpr52
-; SI-NEXT:    ; implicit-def: $sgpr56
-; SI-NEXT:    ; implicit-def: $vgpr8
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $sgpr94
-; SI-NEXT:    ; implicit-def: $sgpr47
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $sgpr49
-; SI-NEXT:    ; implicit-def: $sgpr46
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $sgpr48
-; SI-NEXT:    ; implicit-def: $sgpr45
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $sgpr11
-; SI-NEXT:    ; implicit-def: $sgpr44
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    ; implicit-def: $sgpr9
-; SI-NEXT:    ; implicit-def: $sgpr43
-; SI-NEXT:    ; implicit-def: $vgpr3
-; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $sgpr16
-; SI-NEXT:    ; implicit-def: $sgpr41
-; SI-NEXT:  .LBB97_3: ; %Flow
-; SI-NEXT:    v_mov_b32_e32 v5, v39
-; SI-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT:    s_cbranch_vccnz .LBB97_5
-; SI-NEXT:  ; %bb.4: ; %cmp.true
-; SI-NEXT:    s_add_i32 s4, s79, 3
+; SI-NEXT:    s_mov_b32 s24, s96
+; SI-NEXT:    s_mov_b32 s96, s4
+; SI-NEXT:    s_cbranch_execnz .LBB97_3
+; SI-NEXT:  .LBB97_2: ; %cmp.true
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v36
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_mov_b32_e32 v6, v5
+; SI-NEXT:    v_mov_b32_e32 v5, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    s_add_i32 s4, s88, 3
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s7, 8
+; SI-NEXT:    s_lshl_b32 s5, s79, 8
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v34
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_or_b32_e32 v1, v22, v1
+; SI-NEXT:    v_or_b32_e32 v1, v25, v1
 ; SI-NEXT:    v_or_b32_e32 v1, s4, v1
-; SI-NEXT:    s_add_i32 s4, s82, 3
+; SI-NEXT:    v_readlane_b32 s4, v62, 11
+; SI-NEXT:    s_add_i32 s4, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 10
+; SI-NEXT:    v_readlane_b32 s6, v62, 9
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s53, 8
-; SI-NEXT:    s_add_i32 s8, s31, 3
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    s_add_i32 s8, s6, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    v_readlane_b32 s5, v62, 8
 ; SI-NEXT:    s_and_b32 s8, s8, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s50, 24
+; SI-NEXT:    s_lshl_b32 s5, s5, 24
 ; SI-NEXT:    s_lshl_b32 s8, s8, 16
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    s_add_i32 s71, s71, 3
+; SI-NEXT:    s_add_i32 s70, s70, 3
+; SI-NEXT:    v_readlane_b32 s6, v62, 30
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    s_and_b32 s5, s71, 0xff
-; SI-NEXT:    s_lshl_b32 s8, s36, 8
+; SI-NEXT:    s_and_b32 s5, s70, 0xff
+; SI-NEXT:    s_lshl_b32 s8, s6, 8
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v48
 ; SI-NEXT:    s_or_b32 s5, s8, s5
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_addk_i32 s5, 0x300
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_readlane_b32 s7, v62, 12
 ; SI-NEXT:    s_and_b32 s5, s5, 0xffff
-; SI-NEXT:    v_or_b32_e32 v2, v35, v2
-; SI-NEXT:    s_add_i32 s36, s86, 3
-; SI-NEXT:    s_lshl_b32 s8, s7, 8
-; SI-NEXT:    v_readlane_b32 s7, v62, 11
+; SI-NEXT:    v_or_b32_e32 v2, v39, v2
 ; SI-NEXT:    v_or_b32_e32 v2, s5, v2
-; SI-NEXT:    s_and_b32 s5, s36, 0xff
-; SI-NEXT:    s_add_i32 s9, s7, 3
+; SI-NEXT:    s_add_i32 s5, s35, 3
+; SI-NEXT:    v_readlane_b32 s6, v62, 13
+; SI-NEXT:    s_and_b32 s5, s5, 0xff
+; SI-NEXT:    s_lshl_b32 s8, s6, 8
+; SI-NEXT:    s_add_i32 s9, s77, 3
 ; SI-NEXT:    s_or_b32 s5, s8, s5
+; SI-NEXT:    v_readlane_b32 s6, v62, 12
 ; SI-NEXT:    s_and_b32 s9, s9, 0xff
-; SI-NEXT:    s_lshl_b32 s8, s78, 24
+; SI-NEXT:    s_lshl_b32 s8, s6, 24
 ; SI-NEXT:    s_lshl_b32 s9, s9, 16
 ; SI-NEXT:    s_addk_i32 s5, 0x300
 ; SI-NEXT:    s_or_b32 s8, s8, s9
 ; SI-NEXT:    s_and_b32 s5, s5, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s8, s5
-; SI-NEXT:    s_add_i32 s16, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 17
-; SI-NEXT:    s_add_i32 s9, s5, 0x3000000
-; SI-NEXT:    s_add_i32 s79, s4, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 16
+; SI-NEXT:    s_add_i32 s79, s92, 3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 0x3000000, v1
+; SI-NEXT:    s_add_i32 s16, s4, 0x3000000
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x3000000, v2
+; SI-NEXT:    s_add_i32 s9, s5, 0x3000000
 ; SI-NEXT:    s_and_b32 s4, s79, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s5, 8
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v5
+; SI-NEXT:    s_lshl_b32 s5, s7, 8
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v33
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_or_b32_e32 v2, v33, v2
+; SI-NEXT:    v_or_b32_e32 v2, v49, v2
 ; SI-NEXT:    v_or_b32_e32 v2, s4, v2
-; SI-NEXT:    s_add_i32 s4, s77, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 15
-; SI-NEXT:    v_readlane_b32 s6, v62, 14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v8, v9, v8
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; SI-NEXT:    s_add_i32 s4, s24, 3
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s5, 8
-; SI-NEXT:    s_add_i32 s8, s6, 3
+; SI-NEXT:    s_lshl_b32 s5, s55, 8
+; SI-NEXT:    s_add_i32 s8, s37, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 13
 ; SI-NEXT:    s_and_b32 s8, s8, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s5, 24
+; SI-NEXT:    s_lshl_b32 s5, s51, 24
 ; SI-NEXT:    s_lshl_b32 s8, s8, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    s_add_i32 s52, s98, 3
 ; SI-NEXT:    s_add_i32 s11, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 32
-; SI-NEXT:    s_add_i32 s53, s4, 3
-; SI-NEXT:    s_and_b32 s4, s53, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s68, 8
+; SI-NEXT:    s_and_b32 s4, s52, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s67, 8
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v38
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
-; SI-NEXT:    s_add_i32 s93, s98, 3
-; SI-NEXT:    v_readlane_b32 s6, v62, 18
+; SI-NEXT:    v_or_b32_e32 v4, v20, v4
+; SI-NEXT:    s_add_i32 s30, s87, 3
 ; SI-NEXT:    v_or_b32_e32 v4, s4, v4
-; SI-NEXT:    s_and_b32 s4, s93, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s75, 8
-; SI-NEXT:    s_add_i32 s8, s6, 3
+; SI-NEXT:    s_and_b32 s4, s30, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s21, 8
+; SI-NEXT:    s_add_i32 s8, s68, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_and_b32 s8, s8, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s81, 24
+; SI-NEXT:    s_lshl_b32 s5, s28, 24
 ; SI-NEXT:    s_lshl_b32 s8, s8, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    s_add_i32 s86, s27, 3
 ; SI-NEXT:    s_add_i32 s48, s4, 0x3000000
-; SI-NEXT:    s_and_b32 s4, s86, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s24, 8
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v37
+; SI-NEXT:    v_readlane_b32 s4, v62, 42
+; SI-NEXT:    v_mov_b32_e32 v22, v30
+; SI-NEXT:    s_add_i32 s87, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 39
+; SI-NEXT:    s_and_b32 s4, s87, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v22
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_or_b32_e32 v5, v23, v5
-; SI-NEXT:    v_or_b32_e32 v5, s4, v5
-; SI-NEXT:    v_readlane_b32 s4, v62, 38
-; SI-NEXT:    s_add_i32 s68, s4, 3
-; SI-NEXT:    s_and_b32 s4, s68, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s97, 8
-; SI-NEXT:    s_add_i32 s8, s67, 3
+; SI-NEXT:    v_or_b32_e32 v7, v35, v7
+; SI-NEXT:    v_or_b32_e32 v7, s4, v7
+; SI-NEXT:    v_readlane_b32 s4, v62, 32
+; SI-NEXT:    s_add_i32 s67, s4, 3
+; SI-NEXT:    s_and_b32 s4, s67, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s85, 8
+; SI-NEXT:    s_add_i32 s8, s69, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_and_b32 s8, s8, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s51, 24
+; SI-NEXT:    s_lshl_b32 s5, s74, 24
 ; SI-NEXT:    s_lshl_b32 s8, s8, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s50, s90, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 19
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 0x3000000, v5
+; SI-NEXT:    v_readlane_b32 s5, v62, 23
 ; SI-NEXT:    s_add_i32 s49, s4, 0x3000000
 ; SI-NEXT:    s_and_b32 s4, s50, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v36
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_or_b32_e32 v5, v25, v5
-; SI-NEXT:    s_add_i32 s94, s20, 3
-; SI-NEXT:    v_or_b32_e32 v5, s4, v5
+; SI-NEXT:    s_add_i32 s94, s86, 3
+; SI-NEXT:    v_or_b32_e32 v8, s4, v8
 ; SI-NEXT:    s_and_b32 s4, s94, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s85, 8
-; SI-NEXT:    s_add_i32 s8, s83, 3
+; SI-NEXT:    s_lshl_b32 s5, s38, 8
+; SI-NEXT:    s_add_i32 s8, s71, 3
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_and_b32 s8, s8, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s84, 24
+; SI-NEXT:    s_lshl_b32 s5, s81, 24
 ; SI-NEXT:    s_lshl_b32 s8, s8, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    s_add_i32 s20, s54, 3
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 0x3000000, v5
 ; SI-NEXT:    s_add_i32 s94, s4, 0x3000000
-; SI-NEXT:    s_and_b32 s4, s20, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s70, 8
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v49
+; SI-NEXT:    v_readlane_b32 s4, v62, 52
+; SI-NEXT:    s_add_i32 s18, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 51
+; SI-NEXT:    s_and_b32 s4, s18, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_or_b32_e32 v5, v24, v5
-; SI-NEXT:    v_or_b32_e32 v5, s4, v5
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 0x3000000, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT:    s_add_i32 s98, s38, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 20
+; SI-NEXT:    v_or_b32_e32 v9, v24, v9
+; SI-NEXT:    v_or_b32_e32 v9, s4, v9
+; SI-NEXT:    v_readlane_b32 s4, v62, 45
+; SI-NEXT:    s_add_i32 s98, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 43
+; SI-NEXT:    v_readlane_b32 s6, v62, 14
 ; SI-NEXT:    s_and_b32 s4, s98, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
-; SI-NEXT:    s_add_i32 s8, s92, 3
+; SI-NEXT:    s_add_i32 s8, s6, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_and_b32 s8, s8, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s23, 24
+; SI-NEXT:    s_lshl_b32 s5, s31, 24
 ; SI-NEXT:    s_lshl_b32 s8, s8, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    s_add_i32 s17, s37, 3
-; SI-NEXT:    s_add_i32 s52, s4, 0x3000000
-; SI-NEXT:    s_and_b32 s4, s17, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s19, 8
+; SI-NEXT:    s_add_i32 s53, s4, 0x3000000
+; SI-NEXT:    v_readlane_b32 s4, v62, 41
+; SI-NEXT:    s_add_i32 s86, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 38
+; SI-NEXT:    s_and_b32 s4, s86, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    s_addk_i32 s4, 0x300
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_readlane_b32 s5, v62, 34
-; SI-NEXT:    v_readlane_b32 s6, v62, 29
+; SI-NEXT:    v_or_b32_e32 v10, v59, v10
+; SI-NEXT:    v_or_b32_e32 v10, s4, v10
+; SI-NEXT:    v_readlane_b32 s4, v62, 31
+; SI-NEXT:    s_add_i32 s66, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 27
+; SI-NEXT:    s_and_b32 s4, s66, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
-; SI-NEXT:    s_add_i32 s38, s6, 3
-; SI-NEXT:    s_and_b32 s8, s38, 0xff
-; SI-NEXT:    s_lshl_b32 s8, s8, 16
-; SI-NEXT:    s_add_i32 s22, s22, 3
-; SI-NEXT:    s_add_i32 s85, s25, 3
-; SI-NEXT:    v_readlane_b32 s6, v62, 40
-; SI-NEXT:    s_add_i32 s70, s6, 3
-; SI-NEXT:    s_and_b32 s7, s70, 0xff
-; SI-NEXT:    s_lshl_b32 s7, s7, 16
-; SI-NEXT:    s_add_i32 s19, s69, 3
-; SI-NEXT:    s_add_i32 s51, s30, 3
-; SI-NEXT:    s_add_i32 s95, s89, 3
-; SI-NEXT:    v_mov_b32_e32 v30, s16
-; SI-NEXT:    v_mov_b32_e32 v39, s9
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x3000000, v2
-; SI-NEXT:    v_mov_b32_e32 v28, s11
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x3000000, v4
-; SI-NEXT:    v_mov_b32_e32 v27, s48
-; SI-NEXT:    v_mov_b32_e32 v26, s49
-; SI-NEXT:    v_mov_b32_e32 v25, s94
-; SI-NEXT:    v_mov_b32_e32 v24, s52
-; SI-NEXT:    v_alignbit_b32 v41, v24, v9, 16
-; SI-NEXT:    v_alignbit_b32 v40, v25, v8, 16
-; SI-NEXT:    v_alignbit_b32 v55, v26, v7, 16
-; SI-NEXT:    v_alignbit_b32 v54, v27, v4, 16
-; SI-NEXT:    v_alignbit_b32 v53, v28, v2, 16
-; SI-NEXT:    v_alignbit_b32 v52, v39, v1, 16
-; SI-NEXT:    v_alignbit_b32 v51, v30, v3, 16
-; SI-NEXT:    s_lshr_b32 s56, s52, 16
-; SI-NEXT:    s_lshr_b32 s47, s94, 16
-; SI-NEXT:    s_lshr_b32 s46, s49, 16
-; SI-NEXT:    s_lshr_b32 s45, s48, 16
-; SI-NEXT:    s_lshr_b32 s44, s11, 16
-; SI-NEXT:    s_lshr_b32 s43, s9, 16
-; SI-NEXT:    s_lshr_b32 s41, s16, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v31, v5
-; SI-NEXT:    v_or_b32_e32 v5, s4, v5
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 0x3000000, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT:    v_readlane_b32 s4, v62, 37
-; SI-NEXT:    s_add_i32 s67, s4, 3
-; SI-NEXT:    s_and_b32 s4, s67, 0xff
+; SI-NEXT:    s_add_i32 s37, s39, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 30
+; SI-NEXT:    s_and_b32 s8, s37, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s5, 24
+; SI-NEXT:    s_lshl_b32 s5, s95, 24
+; SI-NEXT:    s_lshl_b32 s8, s8, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 46
-; SI-NEXT:    s_add_i32 s35, s4, 0x3000000
-; SI-NEXT:    s_and_b32 s4, s22, 0xff
+; SI-NEXT:    s_add_i32 s36, s4, 0x3000000
+; SI-NEXT:    v_readlane_b32 s4, v62, 50
+; SI-NEXT:    s_add_i32 s21, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 49
+; SI-NEXT:    s_and_b32 s4, s21, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    s_lshl_b32 s5, s34, 8
-; SI-NEXT:    v_mov_b32_e32 v23, s35
-; SI-NEXT:    v_alignbit_b32 v42, v23, v10, 16
-; SI-NEXT:    s_lshr_b32 s57, s35, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v60, v5
-; SI-NEXT:    v_or_b32_e32 v5, s4, v5
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 0x3000000, v5
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; SI-NEXT:    v_readlane_b32 s5, v62, 37
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; SI-NEXT:    s_add_i32 s71, s22, 3
+; SI-NEXT:    s_and_b32 s8, s71, 0xff
+; SI-NEXT:    s_lshl_b32 s8, s8, 16
+; SI-NEXT:    s_add_i32 s35, s99, 3
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v5
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT:    v_or_b32_e32 v11, v32, v11
+; SI-NEXT:    v_or_b32_e32 v11, s4, v11
+; SI-NEXT:    v_readlane_b32 s4, v62, 40
+; SI-NEXT:    s_add_i32 s85, s4, 3
 ; SI-NEXT:    s_and_b32 s4, s85, 0xff
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 36
+; SI-NEXT:    v_readlane_b32 s5, v62, 33
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    s_lshl_b32 s5, s5, 24
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    s_or_b32 s5, s5, s7
+; SI-NEXT:    s_or_b32 s5, s5, s8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 47
 ; SI-NEXT:    s_add_i32 s8, s4, 0x3000000
-; SI-NEXT:    s_and_b32 s4, s19, 0xff
+; SI-NEXT:    v_readlane_b32 s4, v62, 54
+; SI-NEXT:    s_add_i32 s17, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 53
+; SI-NEXT:    s_and_b32 s4, s17, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_readlane_b32 s5, v62, 44
+; SI-NEXT:    v_readlane_b32 s5, v62, 47
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    s_and_b32 s6, s35, 0xff
+; SI-NEXT:    s_lshl_b32 s6, s6, 16
+; SI-NEXT:    v_mov_b32_e32 v30, s16
+; SI-NEXT:    v_mov_b32_e32 v39, s9
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x3000000, v2
+; SI-NEXT:    v_mov_b32_e32 v28, s11
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x3000000, v4
+; SI-NEXT:    v_mov_b32_e32 v27, s48
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 0x3000000, v7
+; SI-NEXT:    v_mov_b32_e32 v26, s49
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 0x3000000, v8
+; SI-NEXT:    v_mov_b32_e32 v25, s94
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 0x3000000, v9
+; SI-NEXT:    v_mov_b32_e32 v24, s53
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 0x3000000, v10
+; SI-NEXT:    v_mov_b32_e32 v23, s36
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 0x3000000, v11
 ; SI-NEXT:    v_mov_b32_e32 v22, s8
 ; SI-NEXT:    v_alignbit_b32 v43, v22, v11, 16
+; SI-NEXT:    v_alignbit_b32 v42, v23, v10, 16
+; SI-NEXT:    v_alignbit_b32 v41, v24, v9, 16
+; SI-NEXT:    v_alignbit_b32 v40, v25, v8, 16
+; SI-NEXT:    v_alignbit_b32 v55, v26, v7, 16
+; SI-NEXT:    v_alignbit_b32 v54, v27, v4, 16
+; SI-NEXT:    v_alignbit_b32 v53, v28, v2, 16
+; SI-NEXT:    v_alignbit_b32 v52, v39, v1, 16
+; SI-NEXT:    v_alignbit_b32 v51, v30, v3, 16
 ; SI-NEXT:    s_lshr_b32 s58, s8, 16
+; SI-NEXT:    s_lshr_b32 s57, s36, 16
+; SI-NEXT:    s_lshr_b32 s56, s53, 16
+; SI-NEXT:    s_lshr_b32 s47, s94, 16
+; SI-NEXT:    s_lshr_b32 s46, s49, 16
+; SI-NEXT:    s_lshr_b32 s45, s48, 16
+; SI-NEXT:    s_lshr_b32 s44, s11, 16
+; SI-NEXT:    s_lshr_b32 s43, s9, 16
+; SI-NEXT:    s_lshr_b32 s41, s16, 16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v61, v5
+; SI-NEXT:    v_or_b32_e32 v5, v12, v5
 ; SI-NEXT:    v_or_b32_e32 v5, s4, v5
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 0x3000000, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v14
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; SI-NEXT:    v_readlane_b32 s4, v62, 45
-; SI-NEXT:    s_add_i32 s6, s4, 3
-; SI-NEXT:    s_and_b32 s4, s6, 0xff
-; SI-NEXT:    v_readlane_b32 s6, v62, 27
-; SI-NEXT:    s_add_i32 s34, s6, 3
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v6
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; SI-NEXT:    v_readlane_b32 s4, v62, 48
+; SI-NEXT:    s_add_i32 s7, s4, 3
+; SI-NEXT:    s_and_b32 s4, s7, 0xff
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 28
-; SI-NEXT:    s_and_b32 s6, s34, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s5, 24
-; SI-NEXT:    s_lshl_b32 s6, s6, 16
+; SI-NEXT:    s_lshl_b32 s5, s89, 24
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s10, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 43
-; SI-NEXT:    s_add_i32 s97, s4, 3
-; SI-NEXT:    s_and_b32 s4, s97, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s28, 8
+; SI-NEXT:    v_readlane_b32 s4, v62, 46
+; SI-NEXT:    s_add_i32 s99, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 44
+; SI-NEXT:    s_and_b32 s4, s99, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_readlane_b32 s5, v62, 41
-; SI-NEXT:    v_readlane_b32 s6, v62, 33
-; SI-NEXT:    s_lshl_b32 s5, s5, 8
-; SI-NEXT:    s_add_i32 s54, s6, 3
-; SI-NEXT:    s_and_b32 s6, s54, 0xff
-; SI-NEXT:    s_lshl_b32 s6, s6, 16
-; SI-NEXT:    v_mov_b32_e32 v20, s10
-; SI-NEXT:    v_alignbit_b32 v44, v20, v12, 16
-; SI-NEXT:    s_lshr_b32 s59, s10, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v14, v5
+; SI-NEXT:    v_or_b32_e32 v5, v19, v5
 ; SI-NEXT:    v_or_b32_e32 v5, s4, v5
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x3000000, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v13
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    v_readlane_b32 s4, v62, 42
+; SI-NEXT:    v_readlane_b32 s4, v62, 36
 ; SI-NEXT:    s_add_i32 s81, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 35
+; SI-NEXT:    v_readlane_b32 s6, v62, 28
 ; SI-NEXT:    s_and_b32 s4, s81, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    s_add_i32 s55, s6, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 31
+; SI-NEXT:    v_readlane_b32 s5, v62, 26
+; SI-NEXT:    s_and_b32 s6, s55, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    s_lshl_b32 s5, s5, 24
+; SI-NEXT:    s_lshl_b32 s6, s6, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s12, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 39
+; SI-NEXT:    v_readlane_b32 s4, v62, 34
 ; SI-NEXT:    s_add_i32 s69, s4, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 35
+; SI-NEXT:    v_readlane_b32 s5, v62, 29
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x3000000, v5
 ; SI-NEXT:    s_and_b32 s4, s69, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v29
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_readlane_b32 s5, v62, 25
-; SI-NEXT:    v_readlane_b32 s6, v62, 24
+; SI-NEXT:    v_or_b32_e32 v5, v18, v5
+; SI-NEXT:    v_or_b32_e32 v5, s4, v5
+; SI-NEXT:    v_readlane_b32 s4, v62, 22
+; SI-NEXT:    s_add_i32 s34, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 21
+; SI-NEXT:    v_readlane_b32 s6, v62, 19
+; SI-NEXT:    s_and_b32 s4, s34, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_add_i32 s92, s6, 3
-; SI-NEXT:    s_and_b32 s6, s92, 0xff
-; SI-NEXT:    s_lshl_b32 s6, s6, 16
-; SI-NEXT:    v_mov_b32_e32 v19, s12
-; SI-NEXT:    v_alignbit_b32 v45, v19, v15, 16
-; SI-NEXT:    s_lshr_b32 s60, s12, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v13, v5
-; SI-NEXT:    v_or_b32_e32 v5, s4, v5
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 0x3000000, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v6
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_readlane_b32 s4, v62, 26
-; SI-NEXT:    s_add_i32 s31, s4, 3
-; SI-NEXT:    s_and_b32 s4, s31, 0xff
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    s_and_b32 s6, s92, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s87, 24
+; SI-NEXT:    s_lshl_b32 s5, s97, 24
+; SI-NEXT:    s_lshl_b32 s6, s6, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s13, s4, 0x3000000
+; SI-NEXT:    v_readlane_b32 s4, v62, 25
+; SI-NEXT:    s_add_i32 s51, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 24
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 0x3000000, v5
 ; SI-NEXT:    s_and_b32 s4, s51, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s88, 8
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v21
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    s_addk_i32 s4, 0x300
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    s_lshl_b32 s5, s99, 8
-; SI-NEXT:    s_add_i32 s6, s96, 3
-; SI-NEXT:    s_and_b32 s6, s6, 0xff
-; SI-NEXT:    s_lshl_b32 s6, s6, 16
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_mov_b32_e32 v18, s13
-; SI-NEXT:    v_alignbit_b32 v46, v18, v16, 16
-; SI-NEXT:    s_lshr_b32 s61, s13, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v5, v6, v5
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v5, v14, v5
 ; SI-NEXT:    v_or_b32_e32 v5, s4, v5
+; SI-NEXT:    v_readlane_b32 s4, v62, 20
+; SI-NEXT:    s_add_i32 s95, s4, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 18
 ; SI-NEXT:    s_and_b32 s4, s95, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    s_add_i32 s6, s96, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    s_and_b32 s6, s6, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s76, 24
+; SI-NEXT:    s_lshl_b32 s5, s80, 24
+; SI-NEXT:    s_lshl_b32 s6, s6, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s14, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 22
-; SI-NEXT:    s_add_i32 s4, s4, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 21
+; SI-NEXT:    s_add_i32 s4, s93, 3
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
-; SI-NEXT:    s_lshl_b32 s5, s5, 8
+; SI-NEXT:    s_lshl_b32 s5, s84, 8
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    s_addk_i32 s4, 0x300
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    s_lshl_b32 s5, s65, 8
-; SI-NEXT:    s_add_i32 s6, s55, 3
-; SI-NEXT:    s_and_b32 s6, s6, 0xff
-; SI-NEXT:    s_lshl_b32 s6, s6, 16
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 0x3000000, v5
-; SI-NEXT:    v_mov_b32_e32 v5, s14
-; SI-NEXT:    v_alignbit_b32 v47, v5, v17, 16
-; SI-NEXT:    s_lshr_b32 s62, s14, 16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v13
 ; SI-NEXT:    v_or_b32_e32 v6, s4, v6
-; SI-NEXT:    s_add_i32 s4, s80, 3
+; SI-NEXT:    s_add_i32 s4, s83, 3
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
+; SI-NEXT:    s_lshl_b32 s5, s25, 8
+; SI-NEXT:    s_add_i32 s6, s64, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    v_readlane_b32 s5, v62, 15
+; SI-NEXT:    s_and_b32 s6, s6, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s64, 24
+; SI-NEXT:    s_lshl_b32 s5, s5, 24
+; SI-NEXT:    s_lshl_b32 s6, s6, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s15, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 3
+; SI-NEXT:    v_readlane_b32 s4, v62, 2
 ; SI-NEXT:    s_add_i32 s4, s4, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 2
+; SI-NEXT:    v_readlane_b32 s5, v62, 1
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_add_i32 s6, s26, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 1
 ; SI-NEXT:    s_and_b32 s6, s6, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s5, 24
+; SI-NEXT:    s_lshl_b32 s5, s27, 24
 ; SI-NEXT:    s_lshl_b32 s6, s6, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
@@ -212263,42 +212199,41 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    s_add_i32 s4, s4, 3
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s29, 8
-; SI-NEXT:    s_add_i32 s6, s74, 3
+; SI-NEXT:    s_add_i32 s6, s76, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
+; SI-NEXT:    v_readlane_b32 s5, v62, 16
 ; SI-NEXT:    s_and_b32 s6, s6, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s21, 24
+; SI-NEXT:    s_lshl_b32 s5, s5, 24
 ; SI-NEXT:    s_lshl_b32 s6, s6, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s40, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 10
+; SI-NEXT:    v_readlane_b32 s4, v62, 7
 ; SI-NEXT:    s_add_i32 s4, s4, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 23
-; SI-NEXT:    v_readlane_b32 s6, v62, 9
+; SI-NEXT:    v_readlane_b32 s5, v62, 17
+; SI-NEXT:    v_readlane_b32 s6, v62, 6
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_add_i32 s6, s6, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 8
 ; SI-NEXT:    s_and_b32 s6, s6, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
-; SI-NEXT:    s_lshl_b32 s5, s5, 24
+; SI-NEXT:    s_lshl_b32 s5, s19, 24
 ; SI-NEXT:    s_lshl_b32 s6, s6, 16
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    s_add_i32 s91, s4, 0x3000000
-; SI-NEXT:    v_readlane_b32 s4, v62, 7
-; SI-NEXT:    s_add_i32 s4, s4, 3
-; SI-NEXT:    v_readlane_b32 s5, v62, 6
-; SI-NEXT:    v_readlane_b32 s6, v62, 5
+; SI-NEXT:    s_add_i32 s18, s4, 0x3000000
+; SI-NEXT:    s_add_i32 s4, s20, 3
+; SI-NEXT:    v_readlane_b32 s5, v62, 5
+; SI-NEXT:    v_readlane_b32 s6, v62, 4
 ; SI-NEXT:    s_and_b32 s4, s4, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s5, 8
 ; SI-NEXT:    s_add_i32 s6, s6, 3
 ; SI-NEXT:    s_or_b32 s4, s5, s4
-; SI-NEXT:    v_readlane_b32 s5, v62, 4
+; SI-NEXT:    v_readlane_b32 s5, v62, 3
 ; SI-NEXT:    s_and_b32 s6, s6, 0xff
 ; SI-NEXT:    s_addk_i32 s4, 0x300
 ; SI-NEXT:    s_lshl_b32 s5, s5, 24
@@ -212307,18 +212242,33 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    s_or_b32 s5, s5, s6
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    s_add_i32 s42, s4, 0x3000000
-; SI-NEXT:    v_mov_b32_e32 v13, s91
+; SI-NEXT:    v_mov_b32_e32 v13, s18
+; SI-NEXT:    v_mov_b32_e32 v20, s10
+; SI-NEXT:    v_mov_b32_e32 v19, s12
+; SI-NEXT:    v_mov_b32_e32 v18, s13
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 0x3000000, v5
+; SI-NEXT:    v_mov_b32_e32 v5, s14
 ; SI-NEXT:    v_add_i32_e32 v50, vcc, 0x3000000, v6
 ; SI-NEXT:    v_mov_b32_e32 v6, s15
 ; SI-NEXT:    v_alignbit_b32 v57, s42, v13, 16
 ; SI-NEXT:    v_mov_b32_e32 v13, s17
 ; SI-NEXT:    v_alignbit_b32 v58, s40, v13, 16
 ; SI-NEXT:    v_alignbit_b32 v56, v6, v50, 16
+; SI-NEXT:    v_alignbit_b32 v47, v5, v17, 16
+; SI-NEXT:    s_waitcnt expcnt(1)
+; SI-NEXT:    v_alignbit_b32 v46, v18, v16, 16
+; SI-NEXT:    v_alignbit_b32 v45, v19, v15, 16
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_alignbit_b32 v44, v20, v12, 16
 ; SI-NEXT:    s_lshr_b32 s73, s42, 16
 ; SI-NEXT:    s_lshr_b32 s72, s40, 16
 ; SI-NEXT:    s_lshr_b32 s63, s15, 16
-; SI-NEXT:  .LBB97_5: ; %end
-; SI-NEXT:    s_and_b32 s4, s91, 0xffff
+; SI-NEXT:    s_lshr_b32 s62, s14, 16
+; SI-NEXT:    s_lshr_b32 s61, s13, 16
+; SI-NEXT:    s_lshr_b32 s60, s12, 16
+; SI-NEXT:    s_lshr_b32 s59, s10, 16
+; SI-NEXT:  .LBB97_3: ; %end
+; SI-NEXT:    s_and_b32 s4, s18, 0xffff
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v57
 ; SI-NEXT:    v_or_b32_e32 v5, s4, v5
 ; SI-NEXT:    s_and_b32 s4, s42, 0xffff
@@ -212327,7 +212277,6 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 4, v0
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_mov_b32_e32 v6, s4
 ; SI-NEXT:    buffer_store_dword v6, v5, s[0:3], 0 offen
 ; SI-NEXT:    s_and_b32 s4, s17, 0xffff
@@ -212424,7 +212373,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v10
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v42
-; SI-NEXT:    s_and_b32 s4, s35, 0xffff
+; SI-NEXT:    s_and_b32 s4, s36, 0xffff
 ; SI-NEXT:    s_lshl_b32 s5, s57, 16
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v6
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 64, v0
@@ -212437,7 +212386,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v9
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v41
-; SI-NEXT:    s_and_b32 s4, s52, 0xffff
+; SI-NEXT:    s_and_b32 s4, s53, 0xffff
 ; SI-NEXT:    s_lshl_b32 s5, s56, 16
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v6
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 0x48, v0
@@ -212573,11 +212522,91 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; SI-NEXT:    v_readlane_b32 s35, v63, 1
 ; SI-NEXT:    v_readlane_b32 s34, v63, 0
 ; SI-NEXT:    s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[4:5]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
+; SI-NEXT:  .LBB97_4:
+; SI-NEXT:    v_mov_b32_e32 v5, v13
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; SI-NEXT:    v_mov_b32_e32 v25, v58
+; SI-NEXT:    v_mov_b32_e32 v48, v39
+; SI-NEXT:    v_mov_b32_e32 v39, v57
+; SI-NEXT:    v_mov_b32_e32 v49, v56
+; SI-NEXT:    v_mov_b32_e32 v20, v47
+; SI-NEXT:    v_mov_b32_e32 v30, v37
+; SI-NEXT:    v_mov_b32_e32 v36, v35
+; SI-NEXT:    v_mov_b32_e32 v35, v45
+; SI-NEXT:    v_mov_b32_e32 v27, v26
+; SI-NEXT:    s_waitcnt expcnt(2)
+; SI-NEXT:    v_mov_b32_e32 v32, v23
+; SI-NEXT:    ; implicit-def: $sgpr18
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $sgpr42
+; SI-NEXT:    ; implicit-def: $sgpr73
+; SI-NEXT:    ; implicit-def: $sgpr17
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $sgpr40
+; SI-NEXT:    ; implicit-def: $sgpr72
+; SI-NEXT:    ; implicit-def: $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $sgpr15
+; SI-NEXT:    ; implicit-def: $sgpr63
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $sgpr14
+; SI-NEXT:    ; implicit-def: $sgpr62
+; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $sgpr13
+; SI-NEXT:    ; implicit-def: $sgpr61
+; SI-NEXT:    ; implicit-def: $vgpr15
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $sgpr12
+; SI-NEXT:    ; implicit-def: $sgpr60
+; SI-NEXT:    ; implicit-def: $vgpr12
+; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $sgpr10
+; SI-NEXT:    ; implicit-def: $sgpr59
+; SI-NEXT:    ; implicit-def: $vgpr11
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $sgpr8
+; SI-NEXT:    ; implicit-def: $sgpr58
+; SI-NEXT:    ; implicit-def: $vgpr10
+; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $sgpr36
+; SI-NEXT:    ; implicit-def: $sgpr57
+; SI-NEXT:    ; implicit-def: $vgpr9
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $sgpr53
+; SI-NEXT:    ; implicit-def: $sgpr56
+; SI-NEXT:    ; implicit-def: $vgpr8
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $sgpr94
+; SI-NEXT:    ; implicit-def: $sgpr47
+; SI-NEXT:    ; implicit-def: $vgpr7
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $sgpr49
+; SI-NEXT:    ; implicit-def: $sgpr46
+; SI-NEXT:    ; implicit-def: $vgpr4
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $sgpr48
+; SI-NEXT:    ; implicit-def: $sgpr45
+; SI-NEXT:    ; implicit-def: $vgpr2
+; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $sgpr11
+; SI-NEXT:    ; implicit-def: $sgpr44
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $sgpr9
+; SI-NEXT:    ; implicit-def: $sgpr43
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $sgpr16
+; SI-NEXT:    ; implicit-def: $sgpr41
+; SI-NEXT:    s_branch .LBB97_2
 ;
 ; VI-LABEL: bitcast_v128i8_to_v64i16_scalar:
 ; VI:       ; %bb.0:
@@ -212968,17 +212997,17 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; VI-NEXT:    v_mov_b32_e32 v45, v62
 ; VI-NEXT:    v_or_b32_sdwa v16, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; VI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; VI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_or_b32_sdwa v3, v51, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v17, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_mov_b32_e32 v54, v22
-; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_mov_b32_e32 v32, v1
 ; VI-NEXT:    v_or_b32_sdwa v1, v41, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_mov_b32_e32 v54, v22
 ; VI-NEXT:    v_mov_b32_e32 v41, v24
 ; VI-NEXT:    v_or_b32_sdwa v18, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
@@ -214215,8 +214244,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    s_cbranch_vccnz .LBB97_5
 ; GFX9-NEXT:  ; %bb.4: ; %cmp.true
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:828 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_add_i32 s28, s28, 3
 ; GFX9-NEXT:    s_and_b32 s4, s28, 0xff
 ; GFX9-NEXT:    s_lshl_b32 s5, s29, 8
@@ -214249,7 +214278,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(13)
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -214299,9 +214328,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    s_or_b32 s7, s8, s7
 ; GFX9-NEXT:    s_and_b32 s8, s16, 0xff
 ; GFX9-NEXT:    s_or_b32 s8, s9, s8
+; GFX9-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
-; GFX9-NEXT:    s_and_b32 s9, s18, 0xff
 ; GFX9-NEXT:    s_or_b32 s9, s10, s9
 ; GFX9-NEXT:    s_addk_i32 s4, 0x300
 ; GFX9-NEXT:    s_addk_i32 s5, 0x300
@@ -214334,11 +214363,11 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    v_add_u32_e32 v1, 0x300, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v6, v1, 16, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -214349,11 +214378,11 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    v_add_u32_e32 v1, 0x300, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v7, v1, 16, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0x300, v0
@@ -214484,9 +214513,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    v_add_u32_e32 v2, 3, v2
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v2, 0x300, v2
 ; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
 ; GFX9-NEXT:    v_or_b32_sdwa v51, v34, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -214563,22 +214592,21 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v43, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v31, 0x300, v0
 ; GFX9-NEXT:    v_add_u32_e32 v52, 0x300, v43
 ; GFX9-NEXT:    v_and_b32_e32 v31, 0xffff, v31
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v44, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_add_u32_e32 v1, 3, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_add_u32_e32 v26, 3, v33
@@ -214715,8 +214743,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:324
 ; GFX11-TRUE16-NEXT:    ; meta instruction
 ; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:320
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v6 :: v_dual_mov_b32 v33, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:316
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v89, off, s32 offset:312
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v92, off, s32 offset:308
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v91, off, s32 offset:304
@@ -214766,72 +214795,56 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v162, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:120
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:112
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:64
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:44
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:24
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v86, v30 :: v_dual_mov_b32 v81, v29
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v84, v28 :: v_dual_mov_b32 v83, v27
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v85, v26 :: v_dual_mov_b32 v80, v24
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v69, v25 :: v_dual_mov_b32 v82, v22
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v71, v23 :: v_dual_mov_b32 v64, v21
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v67, v20 :: v_dual_mov_b32 v68, v19
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v70, v18 :: v_dual_mov_b32 v55, v16
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v17 :: v_dual_mov_b32 v65, v15
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v66, v14 :: v_dual_mov_b32 v49, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v12 :: v_dual_mov_b32 v54, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v11 :: v_dual_mov_b32 v36, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v8 :: v_dual_mov_b32 v37, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v38, v7 :: v_dual_mov_b32 v35, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, v5 :: v_dual_mov_b32 v39, v2
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v32, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB97_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -214889,20 +214902,20 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v86
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v85
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v83
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v98
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v86
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v98
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v84
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v85
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v103
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v87
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v102
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v101
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -214910,48 +214923,48 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v13, v14
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v99
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v96
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v116
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v97
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v112
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v3, 16, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v12, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v14, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v16, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v112
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v103
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v100
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v131
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v118
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v128
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v115
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v146
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v133
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v115
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v116
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v113
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v119
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v14, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v130
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v129
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v133
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v19
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v135
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v102
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v117
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v13
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v145
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v144
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v145
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v18, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v14
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v14, v3, 16, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v132
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v132
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xff, v117
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v114
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v135
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xff, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v129
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v15, v15, 16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v144
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v20, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v16
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v16, v18, 16, v19
@@ -215296,84 +215309,84 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v147
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 0x300, v0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v145
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v146
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v144
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v144
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v145
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v135
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v134
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v129
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v117
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v117, 0x300, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v119, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v133
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v131
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v131, 0x300, v2
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v132, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v129
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v146
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v114, 0x300, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v135
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v129, 0x300, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v130
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v129, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v128
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v128, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v119
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v102
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v117
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v131
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v102, 0x300, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v128
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v118
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v117, 0x300, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v116
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v115
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v113
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v116
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v114
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v113
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v112
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v112
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v103
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v103
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v102
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v101
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v99
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v100
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v98
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v86
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v87
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v96
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v55, 3, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v97
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v9
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v97
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v98
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v85
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v86
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v87
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v85
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v84
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 0x300, v5
@@ -215479,16 +215492,16 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v70, 16, v32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v13, 16, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v119
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v132
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v17
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v20, v20, 16, v34
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v16, v16, 16, v32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v129
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v18, v114, 16, v33
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v128
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v18, v129, 16, v33
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v21, v21, 16, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v176
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v17, v102, 16, v32
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v17, v117, 16, v32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v164
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v22
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
@@ -215507,7 +215520,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v15, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v19, v117, 16, v19
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v19, v131, 16, v19
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v24, v167, 16, v24
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v27, v45, 16, v32
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v28, v57, 16, v33
@@ -216468,13 +216481,15 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:60
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:76
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:124
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:132
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:128
-; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    ; kill: killed $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr27
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:128
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; SI-NEXT:    ; kill: killed $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr23
 ; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
@@ -216499,112 +216514,94 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; kill: killed $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr27
+; SI-NEXT:    ; kill: killed $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr23
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 16, v11
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v13
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v13
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:116
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:112
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:80
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v56
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:88
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:80
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v57
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:24
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:20
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:72
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v19, 16, v9
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:52
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v29, 16, v59
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:48
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:68
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v10
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v7
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v14
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v9
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v18
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v22
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v26
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v10
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v30
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v18
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v22
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v7
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:100
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v26
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v30
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
@@ -216718,6 +216715,8 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
@@ -216734,6 +216733,13 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:16
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
@@ -216764,10 +216770,6 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:64
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
@@ -216792,12 +216794,10 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v58
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v63
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v62
+; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v57
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v58
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v62
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr26
@@ -216809,37 +216809,54 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr6
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 16, v60
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v29, 16, v59
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 16, v63
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:100
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v61
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:64
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v60
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v12
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v16
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v20
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v24
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v47
@@ -216848,8 +216865,8 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB98_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v47
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v47
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -216860,7 +216877,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v45, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -216871,7 +216888,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v43, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -216882,7 +216899,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v41, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -216893,7 +216910,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v55, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -216904,7 +216921,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v53, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -216915,103 +216932,102 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v51, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v50, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v28
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v49, v2, v6
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_or_b32_e32 v49, v2, v27
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr27
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v48, v2, v6
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v39, v2, v23
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    v_or_b32_e32 v39, v2, v19
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v38, v2, v6
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v37, v2, v15
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr15
+; SI-NEXT:    v_or_b32_e32 v37, v2, v29
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v36, v2, v6
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v35, v2, v29
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    v_or_b32_e32 v35, v2, v25
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr25
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v34, v2, v6
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v33, v2, v21
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr21
+; SI-NEXT:    v_or_b32_e32 v33, v2, v11
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v31, v2, v19
+; SI-NEXT:    v_or_b32_e32 v31, v2, v6
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v32, v2, v17
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    v_or_b32_e32 v32, v2, v7
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr7
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v26, v2, v25
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr25
+; SI-NEXT:    v_or_b32_e32 v26, v2, v15
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v30, v2, v13
+; SI-NEXT:    v_or_b32_e32 v30, v2, v3
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr13
+; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v18, v2, v11
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr11
+; SI-NEXT:    v_or_b32_e32 v18, v2, v21
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v22, v2, v9
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr9
+; SI-NEXT:    v_or_b32_e32 v22, v2, v5
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v14, v2, v7
+; SI-NEXT:    v_or_b32_e32 v14, v2, v17
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr7
+; SI-NEXT:    ; implicit-def: $vgpr17
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v10, v2, v5
+; SI-NEXT:    v_or_b32_e32 v10, v2, v9
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr9
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v6, v2, v3
+; SI-NEXT:    v_or_b32_e32 v6, v2, v13
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr13
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v1
@@ -217039,17 +217055,17 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v60
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v58
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v59
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v59
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v63
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v57
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v63
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v58
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_bfe_u32 v1, v4, 8, 8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
@@ -217075,16 +217091,16 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_bfe_u32 v1, v60, 8, 8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_bfe_u32 v1, v58, 8, 8
+; SI-NEXT:    v_bfe_u32 v1, v59, 8, 8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_bfe_u32 v1, v59, 8, 8
+; SI-NEXT:    v_bfe_u32 v1, v63, 8, 8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_bfe_u32 v1, v57, 8, 8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_bfe_u32 v1, v63, 8, 8
+; SI-NEXT:    v_bfe_u32 v1, v58, 8, 8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v56
@@ -217094,22 +217110,22 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v61
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_bfe_u32 v1, v61, 8, 8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v62
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_bfe_u32 v1, v62, 8, 8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_bfe_u32 v1, v47, 8, 8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v45, v46, 24
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v45, v46, 16
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
@@ -217226,31 +217242,31 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v22, v18, 24
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v22, v18, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v22, v18, 8
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v10, v14, 24
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v10, v14, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v10, v14, 8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v2, v6, 24
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v2, v6, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v2, v6, 8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 8, v45
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
@@ -217402,59 +217418,66 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:  .LBB98_2: ; %Flow
 ; SI-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:700 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:708 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:720 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:728 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:736 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:740 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:744 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
 ; SI-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB98_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 0x30000, v2
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_or_b32_e32 v2, v13, v2
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
+; SI-NEXT:    v_or_b32_e32 v1, v1, v4
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    v_or_b32_e32 v8, v9, v6
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 0x30000, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v8
 ; SI-NEXT:    v_alignbit_b32 v12, v2, v6, 24
 ; SI-NEXT:    v_alignbit_b32 v20, v2, v6, 16
 ; SI-NEXT:    v_alignbit_b32 v47, v2, v6, 8
+; SI-NEXT:    v_lshrrev_b32_e32 v24, 24, v10
+; SI-NEXT:    v_lshrrev_b32_e32 v28, 16, v10
 ; SI-NEXT:    v_lshrrev_b32_e32 v8, 24, v2
-; SI-NEXT:    v_lshrrev_b32_e32 v27, 16, v2
+; SI-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v11, v1
+; SI-NEXT:    v_or_b32_e32 v1, v21, v1
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v9, v1
+; SI-NEXT:    v_or_b32_e32 v1, v5, v1
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
 ; SI-NEXT:    v_alignbit_b32 v62, v22, v18, 24
 ; SI-NEXT:    v_alignbit_b32 v63, v22, v18, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v59, 24, v22
@@ -217462,71 +217485,69 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v25, v1
+; SI-NEXT:    v_or_b32_e32 v1, v15, v1
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v7, v3
-; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v13, v1
+; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v30, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v19, v1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v31, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v17, v1
+; SI-NEXT:    v_or_b32_e32 v1, v7, v1
 ; SI-NEXT:    v_add_i32_e32 v32, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v34, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v21, v1
+; SI-NEXT:    v_or_b32_e32 v1, v11, v1
 ; SI-NEXT:    v_add_i32_e32 v33, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v58, 16, v33
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v36, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v29, v1
+; SI-NEXT:    v_or_b32_e32 v1, v25, v1
 ; SI-NEXT:    v_add_i32_e32 v35, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v38, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v15, v1
+; SI-NEXT:    v_or_b32_e32 v1, v29, v1
 ; SI-NEXT:    v_add_i32_e32 v37, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -217534,12 +217555,12 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v48, vcc, s6, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v23, v1
+; SI-NEXT:    v_or_b32_e32 v1, v19, v1
 ; SI-NEXT:    v_add_i32_e32 v39, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -217548,32 +217569,27 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v50, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v4
+; SI-NEXT:    v_or_b32_e32 v4, v17, v4
+; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v4
 ; SI-NEXT:    v_alignbit_b32 v56, v10, v14, 24
 ; SI-NEXT:    v_alignbit_b32 v57, v10, v14, 16
 ; SI-NEXT:    v_alignbit_b32 v61, v10, v14, 8
-; SI-NEXT:    v_lshrrev_b32_e32 v24, 24, v10
-; SI-NEXT:    v_lshrrev_b32_e32 v28, 16, v10
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v3, v1
+; SI-NEXT:    v_or_b32_e32 v1, v27, v1
 ; SI-NEXT:    v_add_i32_e32 v49, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v52, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217581,7 +217597,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v51, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217589,7 +217605,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v54, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217597,7 +217613,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v53, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217605,7 +217621,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v40, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217613,7 +217629,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v55, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217621,7 +217637,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v42, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217629,7 +217645,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v41, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217637,7 +217653,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v44, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217645,7 +217661,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v43, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -217653,7 +217669,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v46, vcc, s6, v1
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -218435,7 +218451,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v27
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
@@ -219706,8 +219722,8 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
 ; GFX9-NEXT:    ; kill: killed $vgpr50
@@ -219832,7 +219848,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
+; GFX9-NEXT:    s_waitcnt vmcnt(17)
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v33
 ; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:    ; kill: killed $vgpr33
@@ -219951,7 +219967,6 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 24, v2
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_waitcnt vmcnt(45)
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 16, v32
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v33, 8, v32
@@ -220089,7 +220104,6 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_lshrrev_b64 v[33:34], 24, v[13:14]
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
 ; GFX9-NEXT:    v_pk_add_u16 v32, v32, 3 op_sel_hi:[1,0]
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
 ; GFX9-NEXT:    v_pk_add_u16 v31, v31, 3 op_sel_hi:[1,0]
@@ -220505,13 +220519,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v22, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
@@ -220530,13 +220545,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v24, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
@@ -220555,13 +220571,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v26, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
@@ -220580,13 +220597,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v28, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
@@ -220605,13 +220623,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v30, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
@@ -222012,14 +222031,14 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
 ; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:68
 ; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; SI-NEXT:    s_mov_b32 s6, s18
@@ -222066,15 +222085,15 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
 ; SI-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v55
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v40
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v40
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v42
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v41
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v42
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v43
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v43
 ; SI-NEXT:    s_cbranch_scc0 .LBB99_4
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    s_and_b32 s4, s16, 0xffff
@@ -228001,12 +228020,6 @@ define <64 x half> @bitcast_v64bf16_to_v64f16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:84
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:92
 ; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:88
-; SI-NEXT:    s_waitcnt expcnt(6)
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:96
-; SI-NEXT:    s_waitcnt expcnt(4)
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -228043,8 +228056,12 @@ define <64 x half> @bitcast_v64bf16_to_v64f16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:116
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:120
 ; SI-NEXT:    v_mul_f32_e32 v32, 1.0, v32
 ; SI-NEXT:    v_mul_f32_e32 v33, 1.0, v33
@@ -228058,11 +228075,11 @@ define <64 x half> @bitcast_v64bf16_to_v64f16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_mul_f32_e32 v49, 1.0, v49
 ; SI-NEXT:    v_mul_f32_e32 v50, 1.0, v50
 ; SI-NEXT:    v_mul_f32_e32 v51, 1.0, v51
+; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_mul_f32_e32 v52, 1.0, v52
 ; SI-NEXT:    v_mul_f32_e32 v53, 1.0, v53
 ; SI-NEXT:    v_mul_f32_e32 v54, 1.0, v54
 ; SI-NEXT:    v_mul_f32_e32 v55, 1.0, v55
-; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_mul_f32_e32 v40, 1.0, v40
 ; SI-NEXT:    v_mul_f32_e32 v41, 1.0, v41
 ; SI-NEXT:    v_mul_f32_e32 v42, 1.0, v42
@@ -228075,21 +228092,22 @@ define <64 x half> @bitcast_v64bf16_to_v64f16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_mul_f32_e32 v46, 1.0, v46
 ; SI-NEXT:    v_mul_f32_e32 v47, 1.0, v47
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_mul_f32_e32 v56, 1.0, v56
-; SI-NEXT:    v_mul_f32_e32 v57, 1.0, v57
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_mul_f32_e32 v56, 1.0, v1
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_mul_f32_e32 v58, 1.0, v58
-; SI-NEXT:    v_mul_f32_e32 v59, 1.0, v59
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_mul_f32_e32 v60, 1.0, v1
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_mul_f32_e32 v61, 1.0, v31
+; SI-NEXT:    v_mul_f32_e32 v57, 1.0, v31
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:124
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:128
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_mul_f32_e32 v58, 1.0, v58
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_mul_f32_e32 v59, 1.0, v59
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_mul_f32_e32 v60, 1.0, v60
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v62
+; SI-NEXT:    v_mul_f32_e32 v61, 1.0, v61
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_mul_f32_e32 v62, 1.0, v31
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -235514,27 +235532,23 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:84
 ; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:92
 ; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:88
-; SI-NEXT:    s_waitcnt expcnt(6)
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:96
-; SI-NEXT:    s_waitcnt expcnt(4)
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v2
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
 ; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
@@ -235553,11 +235567,20 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; kill: killed $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:116
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:120
 ; SI-NEXT:    v_cvt_f16_f32_e32 v32, v32
 ; SI-NEXT:    v_cvt_f16_f32_e32 v33, v33
@@ -235572,35 +235595,36 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v50, v50
 ; SI-NEXT:    v_cvt_f16_f32_e32 v51, v51
 ; SI-NEXT:    v_cvt_f16_f32_e32 v52, v52
+; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v53, v53
 ; SI-NEXT:    v_cvt_f16_f32_e32 v54, v54
 ; SI-NEXT:    v_cvt_f16_f32_e32 v55, v55
-; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v40, v40
 ; SI-NEXT:    v_cvt_f16_f32_e32 v41, v41
 ; SI-NEXT:    v_cvt_f16_f32_e32 v42, v42
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v43, v43
-; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v44, v44
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v45, v45
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v46, v46
 ; SI-NEXT:    v_cvt_f16_f32_e32 v47, v47
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f16_f32_e32 v56, v56
-; SI-NEXT:    v_cvt_f16_f32_e32 v57, v57
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f16_f32_e32 v56, v1
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f16_f32_e32 v58, v58
-; SI-NEXT:    v_cvt_f16_f32_e32 v59, v59
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f16_f32_e32 v60, v1
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f16_f32_e32 v61, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v57, v31
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:124
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f16_f32_e32 v59, v59
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f16_f32_e32 v60, v60
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f16_f32_e32 v61, v61
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v62, v62
 ; SI-NEXT:    s_waitcnt vmcnt(2)
@@ -235731,217 +235755,211 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; kill: killed $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; kill: killed $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; kill: killed $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; kill: killed $vgpr2
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB102_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v63
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v31
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    ; kill: killed $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v8
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v9
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v10
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v11
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v12
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v13
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v14
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v15
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v16
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v17
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v18
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v19
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v20
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v21
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v22
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v23
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v24
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v25
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v26
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v27
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v28
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v29
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v30
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v32
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v33
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v34
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v35
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v36
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v37
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v38
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v39
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v48
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v49
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v50
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v51
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v52
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v53
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v54
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v55
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v40
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v41
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v42
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v43
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v44
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v45
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v46
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v47
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v56
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v57
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v58
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v59
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v60
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v61
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v62
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr5
 ; SI-NEXT:    ; implicit-def: $vgpr6
@@ -236009,18 +236027,11 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v31, v63
 ; SI-NEXT:    v_cvt_f32_f16_e32 v63, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v62
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
 ; SI-NEXT:    v_add_f32_e32 v62, 0x38000000, v63
 ; SI-NEXT:    v_add_f32_e32 v63, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v61
-; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
 ; SI-NEXT:    v_cvt_f32_f16_e32 v60, v60
 ; SI-NEXT:    v_cvt_f32_f16_e32 v61, v59
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v58
@@ -236028,14 +236039,6 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v57, v57
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v46
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; SI-NEXT:    v_add_f32_e32 v59, 0x38000000, v60
 ; SI-NEXT:    v_add_f32_e32 v60, 0x38000000, v61
 ; SI-NEXT:    v_add_f32_e32 v61, 0x38000000, v58
@@ -236044,16 +236047,6 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v46, 0x38000000, v57
 ; SI-NEXT:    v_add_f32_e32 v57, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v45
-; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v5
-; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v6
-; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v7
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v8
 ; SI-NEXT:    v_add_f32_e32 v47, 0x38000000, v56
 ; SI-NEXT:    v_add_f32_e32 v56, 0x38000000, v58
 ; SI-NEXT:    v_cvt_f32_f16_e32 v44, v44
@@ -236061,20 +236054,6 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v42
 ; SI-NEXT:    v_add_f32_e32 v42, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v41
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; SI-NEXT:    v_add_f32_e32 v43, 0x38000000, v44
 ; SI-NEXT:    v_add_f32_e32 v44, 0x38000000, v45
 ; SI-NEXT:    v_add_f32_e32 v45, 0x38000000, v58
@@ -236083,16 +236062,6 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v54
 ; SI-NEXT:    v_add_f32_e32 v54, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v53
-; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v9
-; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v10
-; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v11
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v12
 ; SI-NEXT:    v_add_f32_e32 v55, 0x38000000, v40
 ; SI-NEXT:    v_add_f32_e32 v40, 0x38000000, v41
 ; SI-NEXT:    v_add_f32_e32 v41, 0x38000000, v58
@@ -236101,20 +236070,6 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v50
 ; SI-NEXT:    v_add_f32_e32 v50, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v49
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; SI-NEXT:    v_add_f32_e32 v51, 0x38000000, v52
 ; SI-NEXT:    v_add_f32_e32 v52, 0x38000000, v53
 ; SI-NEXT:    v_add_f32_e32 v53, 0x38000000, v58
@@ -236123,16 +236078,6 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v38
 ; SI-NEXT:    v_add_f32_e32 v38, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v37
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v13
-; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v14
-; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v15
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v16
 ; SI-NEXT:    v_add_f32_e32 v39, 0x38000000, v48
 ; SI-NEXT:    v_add_f32_e32 v48, 0x38000000, v49
 ; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v58
@@ -236141,20 +236086,6 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v34
 ; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v33
-; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v36
 ; SI-NEXT:    v_add_f32_e32 v36, 0x38000000, v37
 ; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v58
@@ -236163,242 +236094,330 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v29
 ; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v28
-; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v17
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v18
-; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v19
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v20
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v30, 0x38000000, v32
 ; SI-NEXT:    v_add_f32_e32 v32, 0x38000000, v33
 ; SI-NEXT:    v_add_f32_e32 v33, 0x38000000, v58
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v25
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v24
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v26
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v27
+; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v28
+; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v58
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_cvt_f32_f16_e32 v23, v23
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v6
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v9
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v10
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v11
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v13
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v14
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v15
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v18
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
-; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v21
-; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v26
-; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v27
-; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v28
-; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v58
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v28
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v26
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v33
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v32
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v30
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v37
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v36
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v35
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v34
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v49
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v48
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v39
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v38
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v53
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v52
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v51
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v50
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v41
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v40
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v55
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v54
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v45
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v44
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v43
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v42
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v57
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v56
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v47
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v46
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v61
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v60
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v59
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v31
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_cvt_f32_f16_e32 v58, v58
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v63
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v62
+; SI-NEXT:    v_add_f32_e32 v58, 0x38000000, v58
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v58, v58
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v58, v58
-; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT:    v_add_f32_e32 v58, 0x38000000, v58
-; SI-NEXT:    v_cvt_f16_f32_e32 v58, v58
-; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v58
-; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v3, v24
+; SI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB102_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236406,11 +236425,9 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v2
 ; SI-NEXT:    v_alignbit_b32 v1, v1, v2, 16
 ; SI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v3
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v2
@@ -236418,8 +236435,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 4, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236429,8 +236446,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236440,8 +236457,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 12, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236451,8 +236468,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 16, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236462,8 +236479,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 20, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236473,8 +236490,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 24, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236484,8 +236501,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 28, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236495,8 +236512,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236506,8 +236523,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 36, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236517,8 +236534,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 40, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236528,8 +236545,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236539,8 +236556,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 48, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236550,8 +236567,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 52, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236561,8 +236578,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 56, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236572,8 +236589,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236583,8 +236600,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236594,8 +236611,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236605,8 +236622,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x48, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236616,8 +236633,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236627,8 +236644,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236638,8 +236655,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236649,8 +236666,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x58, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236660,8 +236677,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x5c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236671,8 +236688,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236682,8 +236699,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236693,8 +236710,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x68, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236704,8 +236721,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236715,8 +236732,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236726,8 +236743,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x74, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
@@ -236737,8 +236754,8 @@ define <64 x bfloat> @bitcast_v64f16_to_v64bf16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x78, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x7c, v0
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v1
@@ -238514,145 +238531,173 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72
 ; SI-NEXT:    s_waitcnt expcnt(3)
 ; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt expcnt(2)
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:80
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:84
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:104
-; SI-NEXT:    v_mul_f32_e32 v40, 1.0, v5
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v15
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v16
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v19
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v20
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v23
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v24
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:80
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v27
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v28
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:88
 ; SI-NEXT:    v_mul_f32_e32 v33, 1.0, v1
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v3
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v4
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v7
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v11
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v12
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT:    v_mul_f32_e32 v43, 1.0, v2
-; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v13
-; SI-NEXT:    v_mul_f32_e32 v13, 1.0, v22
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v14
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v15
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v16
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v19
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v20
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v23
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v24
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v27
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v28
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT:    v_mul_f32_e32 v54, 1.0, v13
+; SI-NEXT:    v_mul_f32_e32 v19, 1.0, v21
+; SI-NEXT:    v_mul_f32_e32 v12, 1.0, v22
+; SI-NEXT:    v_mul_f32_e32 v13, 1.0, v25
+; SI-NEXT:    v_mul_f32_e32 v15, 1.0, v17
+; SI-NEXT:    v_mul_f32_e32 v17, 1.0, v29
+; SI-NEXT:    v_mul_f32_e32 v43, 1.0, v2
+; SI-NEXT:    v_mul_f32_e32 v40, 1.0, v5
+; SI-NEXT:    v_mul_f32_e32 v32, 1.0, v6
+; SI-NEXT:    v_mul_f32_e32 v31, 1.0, v9
+; SI-NEXT:    v_mul_f32_e32 v55, 1.0, v10
+; SI-NEXT:    v_mul_f32_e32 v53, 1.0, v14
 ; SI-NEXT:    v_mul_f32_e32 v14, 1.0, v18
 ; SI-NEXT:    v_mul_f32_e32 v18, 1.0, v26
-; SI-NEXT:    v_mul_f32_e32 v15, 1.0, v21
-; SI-NEXT:    v_mul_f32_e32 v32, 1.0, v6
-; SI-NEXT:    v_mul_f32_e32 v4, 1.0, v9
-; SI-NEXT:    v_mul_f32_e32 v3, 1.0, v10
-; SI-NEXT:    v_mul_f32_e32 v16, 1.0, v17
-; SI-NEXT:    v_mul_f32_e32 v19, 1.0, v25
-; SI-NEXT:    v_mul_f32_e32 v17, 1.0, v29
 ; SI-NEXT:    v_mul_f32_e32 v11, 1.0, v30
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v34
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v35
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v36
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v39
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v35
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v48
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v36
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v51
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v39
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v52
-; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; kill: killed $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    v_mul_f32_e32 v34, 1.0, v47
-; SI-NEXT:    ; kill: killed $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; kill: killed $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    v_mul_f32_e32 v47, 1.0, v57
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v48
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mul_f32_e32 v10, 1.0, v37
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v52
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:120
+; SI-NEXT:    v_mul_f32_e32 v28, 1.0, v44
+; SI-NEXT:    v_mul_f32_e32 v44, 1.0, v46
 ; SI-NEXT:    v_mul_f32_e32 v9, 1.0, v38
-; SI-NEXT:    v_mul_f32_e32 v27, 1.0, v60
-; SI-NEXT:    v_mul_f32_e32 v57, 1.0, v61
 ; SI-NEXT:    v_mul_f32_e32 v8, 1.0, v49
 ; SI-NEXT:    v_mul_f32_e32 v7, 1.0, v50
+; SI-NEXT:    v_mul_f32_e32 v46, 1.0, v58
+; SI-NEXT:    v_mul_f32_e32 v29, 1.0, v59
+; SI-NEXT:    v_mul_f32_e32 v30, 1.0, v51
 ; SI-NEXT:    v_mul_f32_e32 v6, 1.0, v41
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mul_f32_e32 v5, 1.0, v42
-; SI-NEXT:    v_mul_f32_e32 v30, 1.0, v44
-; SI-NEXT:    v_mul_f32_e32 v21, 1.0, v55
-; SI-NEXT:    v_mul_f32_e32 v23, 1.0, v31
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    v_mul_f32_e32 v44, 1.0, v46
-; SI-NEXT:    v_mul_f32_e32 v29, 1.0, v56
-; SI-NEXT:    v_mul_f32_e32 v46, 1.0, v58
-; SI-NEXT:    v_mul_f32_e32 v28, 1.0, v59
-; SI-NEXT:    v_mul_f32_e32 v56, 1.0, v62
-; SI-NEXT:    v_mul_f32_e32 v24, 1.0, v45
-; SI-NEXT:    v_mul_f32_e32 v25, 1.0, v63
-; SI-NEXT:    v_mul_f32_e32 v59, 1.0, v54
-; SI-NEXT:    v_mul_f32_e32 v58, 1.0, v53
-; SI-NEXT:    ; kill: killed $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    v_mul_f32_e32 v27, 1.0, v45
+; SI-NEXT:    v_mul_f32_e32 v23, 1.0, v61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; kill: killed $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    v_mul_f32_e32 v4, 1.0, v47
+; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v56
+; SI-NEXT:    v_mul_f32_e32 v47, 1.0, v57
+; SI-NEXT:    v_mul_f32_e32 v60, 1.0, v60
+; SI-NEXT:    v_mul_f32_e32 v57, 1.0, v62
+; SI-NEXT:    v_mul_f32_e32 v56, 1.0, v63
+; SI-NEXT:    ; kill: killed $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr41
@@ -238665,720 +238710,672 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    ; kill: killed $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; kill: killed $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr53
-; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; kill: killed $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; kill: killed $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; kill: killed $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_mul_f32_e32 v59, 1.0, v1
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_mul_f32_e32 v58, 1.0, v3
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_mul_f32_e32 v16, 1.0, v16
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_mul_f32_e32 v24, 1.0, v20
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_mul_f32_e32 v61, 1.0, v12
+; SI-NEXT:    v_mul_f32_e32 v3, 1.0, v21
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_mul_f32_e32 v60, 1.0, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v20, 1.0, v22
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:124
+; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v22
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:124
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_mul_f32_e32 v20, 1.0, v25
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_mul_f32_e32 v26, 1.0, v12
+; SI-NEXT:    v_mul_f32_e32 v26, 1.0, v21
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_mul_f32_e32 v22, 1.0, v22
+; SI-NEXT:    v_mul_f32_e32 v25, 1.0, v22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mul_f32_e32 v12, 1.0, v31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    ; kill: killed $vgpr31
-; SI-NEXT:    ; implicit-def: $vgpr31
+; SI-NEXT:    v_mul_f32_e32 v21, 1.0, v34
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; kill: killed $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB104_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v33
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v33
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v43
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v43
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    v_lshrrev_b32_e32 v55, 16, v22
-; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    s_waitcnt expcnt(2)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshrrev_b32_e32 v62, 16, v31
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshrrev_b32_e32 v51, 16, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshrrev_b32_e32 v49, 16, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshrrev_b32_e32 v63, 16, v31
-; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v40
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v30
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v32
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr3
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshrrev_b32_e32 v50, 16, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v14
-; SI-NEXT:    ; implicit-def: $vgpr14
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v20
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshrrev_b32_e32 v41, 16, v31
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshrrev_b32_e32 v39, 16, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshrrev_b32_e32 v42, 16, v31
-; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v61
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v48, 16, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v15
-; SI-NEXT:    ; implicit-def: $vgpr15
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v26
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v13
-; SI-NEXT:    ; implicit-def: $vgpr13
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v25
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v37, 16, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v38, 16, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v19
-; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v21
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v18
-; SI-NEXT:    ; implicit-def: $vgpr18
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v47
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v36, 16, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v17
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v46
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v11
-; SI-NEXT:    ; implicit-def: $vgpr11
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v29
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v53, 16, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v54, 16, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v10
-; SI-NEXT:    ; implicit-def: $vgpr10
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v60
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v9
-; SI-NEXT:    ; implicit-def: $vgpr9
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v57
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v56
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v45, 16, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v8
-; SI-NEXT:    ; implicit-def: $vgpr8
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v23
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v7
-; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v27
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v59
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v58
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v6
-; SI-NEXT:    ; implicit-def: $vgpr6
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v28
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v5
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v16
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v30
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v44
+; SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v44
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v24
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v61, 16, v3
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr15
+; SI-NEXT:    ; implicit-def: $vgpr10
+; SI-NEXT:    ; implicit-def: $vgpr8
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr6
 ; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v34
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v29
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v47
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr4
+; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v46
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v28
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v27
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v57
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v56
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
 ; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v24
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr27
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr24
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v25
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr20
+; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v59
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; kill: killed $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr21
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_lshrrev_b32_e32 v62, 16, v22
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v39, 16, v14
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr7
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v58
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshrrev_b32_e32 v63, 16, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v40
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v23
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v32
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v21
-; SI-NEXT:    ; implicit-def: $vgpr21
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshrrev_b32_e32 v48, 16, v14
+; SI-NEXT:    v_lshrrev_b32_e32 v14, 16, v19
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr14
+; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v13
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v60
-; SI-NEXT:    ; implicit-def: $vgpr60
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v18
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v20
-; SI-NEXT:    ; implicit-def: $vgpr20
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshrrev_b32_e32 v45, 16, v9
+; SI-NEXT:    ; implicit-def: $vgpr13
+; SI-NEXT:    ; implicit-def: $vgpr18
+; SI-NEXT:    ; implicit-def: $vgpr9
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshrrev_b32_e32 v41, 16, v22
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshrrev_b32_e32 v42, 16, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v31
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v26
-; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v55
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v12
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr31
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v17
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; kill: killed $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; kill: killed $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; kill: killed $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; kill: killed $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; kill: killed $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; kill: killed $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshrrev_b32_e32 v51, 16, v22
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v52, 16, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v54
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v53
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v50, 16, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v11
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v34, 16, v11
+; SI-NEXT:    ; implicit-def: $vgpr11
 ; SI-NEXT:  .LBB104_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB104_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v33
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff0000, v33
 ; SI-NEXT:    v_and_b32_e32 v33, 0xffff0000, v43
-; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT:    v_add_f32_e32 v51, 0x40c00000, v1
 ; SI-NEXT:    v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT:    v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v51
-; SI-NEXT:    v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v33
-; SI-NEXT:    v_alignbit_b32 v1, v1, v2, 16
-; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v14
-; SI-NEXT:    v_alignbit_b32 v31, v35, v31, 16
+; SI-NEXT:    v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v34, 16, v33
+; SI-NEXT:    v_alignbit_b32 v22, v34, v22, 16
 ; SI-NEXT:    v_and_b32_e32 v32, 0xffff0000, v32
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT:    v_add_f32_e32 v14, 0x40c00000, v2
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v40
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff0000, v40
 ; SI-NEXT:    v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v14
+; SI-NEXT:    v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v34, 16, v32
+; SI-NEXT:    v_alignbit_b32 v22, v34, v22, 16
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff0000, v31
+; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v55
 ; SI-NEXT:    v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v32
-; SI-NEXT:    v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v13
-; SI-NEXT:    v_alignbit_b32 v31, v35, v31, 16
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v34, 16, v31
+; SI-NEXT:    v_alignbit_b32 v22, v34, v22, 16
+; SI-NEXT:    v_and_b32_e32 v34, 0xffff0000, v53
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v15
-; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v2
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff0000, v54
+; SI-NEXT:    v_add_f32_e32 v51, 0x40c00000, v34
+; SI-NEXT:    v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT:    v_lshrrev_b32_e32 v34, 16, v51
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT:    v_alignbit_b32 v22, v34, v22, 16
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT:    v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_add_f32_e32 v31, 0x40c00000, v3
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v13
-; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v31
-; SI-NEXT:    v_alignbit_b32 v1, v2, v1, 16
-; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v18
-; SI-NEXT:    v_alignbit_b32 v3, v3, v4, 16
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v14
+; SI-NEXT:    v_alignbit_b32 v15, v22, v15, 16
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v19
-; SI-NEXT:    v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v19
+; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
+; SI-NEXT:    v_alignbit_b32 v15, v19, v15, 16
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT:    v_alignbit_b32 v1, v3, v1, 16
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v11
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v18
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT:    v_lshrrev_b32_e32 v18, 16, v15
+; SI-NEXT:    v_alignbit_b32 v13, v18, v13, 16
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v17
-; SI-NEXT:    v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT:    v_alignbit_b32 v1, v4, v1, 16
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v9
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v17
+; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v11
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT:    v_alignbit_b32 v13, v17, v13, 16
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT:    v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
-; SI-NEXT:    v_alignbit_b32 v1, v9, v1, 16
+; SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v9
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v8
+; SI-NEXT:    v_alignbit_b32 v10, v13, v10, 16
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
 ; SI-NEXT:    v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT:    v_alignbit_b32 v1, v8, v1, 16
-; SI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v6
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
+; SI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT:    v_alignbit_b32 v8, v10, v8, 16
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
 ; SI-NEXT:    v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT:    v_alignbit_b32 v1, v6, v1, 16
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff0000, v29
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v8, 16, v5
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT:    v_alignbit_b32 v6, v8, v6, 16
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT:    v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; SI-NEXT:    v_alignbit_b32 v4, v6, v4, 16
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff0000, v60
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v34
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v29
 ; SI-NEXT:    v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
 ; SI-NEXT:    v_lshrrev_b32_e32 v8, 16, v6
-; SI-NEXT:    v_alignbit_b32 v1, v8, v1, 16
+; SI-NEXT:    v_alignbit_b32 v4, v8, v4, 16
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v27
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v28
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v23
 ; SI-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT:    v_alignbit_b32 v4, v10, v4, 16
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v24
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
 ; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT:    v_alignbit_b32 v1, v9, v1, 16
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff0000, v25
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v24
-; SI-NEXT:    v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT:    v_alignbit_b32 v1, v10, v1, 16
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v21
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v23
 ; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT:    v_alignbit_b32 v1, v11, v1, 16
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v26
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v10
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v61, v1, v3, 16
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v20
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
-; SI-NEXT:    v_alignbit_b32 v1, v15, v1, 16
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v10
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v58
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v59
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v22
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v18, v12, v1, 16
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v16
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v19, v10, v3, 16
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v60
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v61
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v16, v12, v11, 16
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v56
+; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v57
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_alignbit_b32 v4, v13, v4, 16
+; SI-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v58
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v59
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v19, v12, v11, 16
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v20
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v26
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v20, v10, v8, 16
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v56
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v57
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v20, v12, v11, 16
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v46
+; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v47
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v13
+; SI-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT:    v_alignbit_b32 v4, v16, v4, 16
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v21
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v21, v10, v8, 16
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v46
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v47
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v21, v12, v11, 16
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v44
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v28
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v23, v10, v8, 16
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v44
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v30
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v22, v12, v11, 16
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v30
+; SI-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v25
+; SI-NEXT:    v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_alignbit_b32 v18, v16, v4, 16
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v13
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT:    v_and_b32_e32 v7, 0xffff0000, v7
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff0000, v7
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT:    v_alignbit_b32 v5, v23, v5, 16
+; SI-NEXT:    v_alignbit_b32 v2, v21, v2, 16
+; SI-NEXT:    v_alignbit_b32 v1, v61, v1, 16
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT:    v_mov_b32_e32 v55, v18
-; SI-NEXT:    v_alignbit_b32 v1, v18, v1, 16
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v23, v12, v11, 16
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v24, v10, v8, 16
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
-; SI-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT:    v_alignbit_b32 v7, v24, v7, 16
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT:    v_lshrrev_b32_e32 v38, 16, v13
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT:    v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff0000, v17
 ; SI-NEXT:    v_add_f32_e32 v17, 0x40c00000, v17
 ; SI-NEXT:    v_lshrrev_b32_e32 v63, 16, v17
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_lshrrev_b32_e32 v45, 16, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_alignbit_b32 v24, v45, v11, 16
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_lshrrev_b32_e32 v54, 16, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_alignbit_b32 v53, v54, v11, 16
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_lshrrev_b32_e32 v45, 16, v10
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT:    v_alignbit_b32 v25, v45, v8, 16
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; SI-NEXT:    v_alignbit_b32 v8, v25, v8, 16
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT:    v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT:    v_alignbit_b32 v62, v63, v16, 16
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v33
+; SI-NEXT:    v_alignbit_b32 v16, v62, v16, 16
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_lshrrev_b32_e32 v34, 16, v10
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT:    v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT:    v_alignbit_b32 v22, v34, v9, 16
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_alignbit_b32 v9, v22, v9, 16
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
 ; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT:    v_alignbit_b32 v35, v36, v11, 16
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT:    v_alignbit_b32 v2, v35, v2, 16
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v2, v53, v3, 16
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT:    v_lshrrev_b32_e32 v38, 16, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
 ; SI-NEXT:    v_add_f32_e32 v11, 0x40c00000, v11
 ; SI-NEXT:    v_alignbit_b32 v37, v38, v11, 16
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v13
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
 ; SI-NEXT:    v_alignbit_b32 v11, v37, v11, 16
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
 ; SI-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
 ; SI-NEXT:    v_alignbit_b32 v39, v48, v12, 16
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff0000, v14
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT:    v_alignbit_b32 v35, v36, v10, 16
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff0000, v15
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
 ; SI-NEXT:    v_alignbit_b32 v12, v39, v12, 16
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_alignbit_b32 v10, v35, v10, 16
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
 ; SI-NEXT:    v_add_f32_e32 v14, 0x40c00000, v14
 ; SI-NEXT:    v_lshrrev_b32_e32 v50, 16, v14
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT:    v_lshrrev_b32_e32 v52, 16, v15
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
 ; SI-NEXT:    v_alignbit_b32 v49, v50, v13, 16
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v51
 ; SI-NEXT:    v_alignbit_b32 v13, v49, v13, 16
-; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
 ; SI-NEXT:    v_add_f32_e32 v14, 0x40c00000, v14
 ; SI-NEXT:    v_alignbit_b32 v51, v52, v14, 16
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff0000, v31
-; SI-NEXT:    v_mov_b32_e32 v31, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; SI-NEXT:    v_alignbit_b32 v14, v51, v14, 16
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT:    v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT:    v_lshrrev_b32_e32 v42, 16, v16
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
 ; SI-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
 ; SI-NEXT:    v_alignbit_b32 v41, v42, v15, 16
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v32
 ; SI-NEXT:    v_alignbit_b32 v15, v41, v15, 16
-; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT:    v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT:    v_alignbit_b32 v62, v63, v16, 16
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff0000, v33
-; SI-NEXT:    v_alignbit_b32 v16, v62, v16, 16
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v2, v24, v4, 16
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v2, v23, v7, 16
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v2, v22, v5, 16
+; SI-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v21, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v2, v21, v6, 16
+; SI-NEXT:    v_alignbit_b32 v2, v20, v6, 16
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v2, v20, v8, 16
+; SI-NEXT:    v_alignbit_b32 v2, v19, v3, 16
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_alignbit_b32 v2, v19, v9, 16
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v2, v31, v10, 16
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v1, v18, v4, 16
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB104_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239392,8 +239389,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 4, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239408,8 +239405,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 12, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239424,8 +239421,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 20, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239440,8 +239437,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 28, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239456,8 +239453,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 36, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239472,8 +239469,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239488,8 +239485,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 52, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239498,14 +239495,14 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 56, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v53
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v54
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v22
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v34
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239514,7 +239511,7 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v45
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -239522,8 +239519,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239532,8 +239529,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x48, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239542,8 +239539,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239552,8 +239549,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239562,8 +239559,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239572,8 +239569,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x58, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239582,8 +239579,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x5c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239592,8 +239589,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239602,8 +239599,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239612,8 +239609,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x68, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239622,8 +239619,8 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239631,17 +239628,17 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v31
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v61
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x74, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -239649,10 +239646,12 @@ define <64 x i16> @bitcast_v64bf16_to_v64i16(<64 x bfloat> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x78, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v55
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x7c, v0
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
@@ -245005,15 +245004,15 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:116
 ; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:124
 ; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:24
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:128
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:128
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
@@ -245032,14 +245031,14 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    v_lshlrev_b32_e32 v31, 16, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:56
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v3
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; kill: killed $vgpr3
@@ -245112,17 +245111,14 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB106_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v46
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v56
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v57
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:    ; kill: killed $vgpr1
@@ -245155,10 +245151,9 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v27, 16, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -245206,111 +245201,118 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v47
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v43
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v41
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v42
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v57
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v43
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v58
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v44
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v45
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v59
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v46
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v60
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v58
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v61
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v59
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v45
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v61
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v63
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v62
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v40
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v63
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v62
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v40
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v60
+; SI-NEXT:    ; implicit-def: $vgpr47
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:  .LBB106_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB106_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v62
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v60
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v55, v3
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v46
-; SI-NEXT:    v_add_i32_e32 v46, vcc, 0x30000, v3
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v40
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_or_b32_e32 v3, v55, v3
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v9, v52, v1
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x30000, v3
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v40
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v3, v51, v3
+; SI-NEXT:    v_add_i32_e32 v52, vcc, s6, v3
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v63
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v50, v3
-; SI-NEXT:    v_or_b32_e32 v1, v51, v1
 ; SI-NEXT:    v_add_i32_e32 v51, vcc, s6, v3
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v63
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v62
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v49, v3
 ; SI-NEXT:    v_add_i32_e32 v50, vcc, s6, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v45
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v48, v3
 ; SI-NEXT:    v_add_i32_e32 v49, vcc, s6, v3
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v61
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v59
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v39, v3
 ; SI-NEXT:    v_add_i32_e32 v48, vcc, s6, v3
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v60
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v58
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v38, v3
 ; SI-NEXT:    v_add_i32_e32 v39, vcc, s6, v3
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v59
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v46
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_add_i32_e32 v38, vcc, s6, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v44
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v36, v3
 ; SI-NEXT:    v_add_i32_e32 v37, vcc, s6, v3
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v58
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v44
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_add_i32_e32 v36, vcc, s6, v3
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v57
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v43
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v34, v3
 ; SI-NEXT:    v_add_i32_e32 v35, vcc, s6, v3
@@ -245318,11 +245320,11 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v33, v3
 ; SI-NEXT:    v_add_i32_e32 v34, vcc, s6, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v43
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v41
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v32, v3
 ; SI-NEXT:    v_add_i32_e32 v33, vcc, s6, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v41
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v47
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
@@ -245333,24 +245335,20 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v11, v18, v11
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v11
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v56
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v54, v5
 ; SI-NEXT:    v_add_i32_e32 v55, vcc, s6, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v47
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v56
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v53, v7
 ; SI-NEXT:    v_add_i32_e32 v54, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    v_or_b32_e32 v9, v52, v9
 ; SI-NEXT:    v_add_i32_e32 v53, vcc, s6, v9
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff0000, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_and_b32_e32 v52, 0xffff0000, v53
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
@@ -245362,8 +245360,8 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v11, v16, v11
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v30, 0xffff0000, v31
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
+; SI-NEXT:    v_and_b32_e32 v30, 0xffff0000, v31
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -245413,6 +245411,22 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v50
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v51
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v52
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v54
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
 ; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v32
 ; SI-NEXT:    v_and_b32_e32 v32, 0xffff0000, v33
 ; SI-NEXT:    v_and_b32_e32 v33, 0xffff0000, v34
@@ -245424,28 +245438,15 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v39, 0xffff0000, v48
 ; SI-NEXT:    v_and_b32_e32 v48, 0xffff0000, v49
 ; SI-NEXT:    v_and_b32_e32 v49, 0xffff0000, v50
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    v_and_b32_e32 v50, 0xffff0000, v51
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v51
-; SI-NEXT:    v_and_b32_e32 v51, 0xffff0000, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v53
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v54
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v51, 0xffff0000, v52
+; SI-NEXT:    v_and_b32_e32 v52, 0xffff0000, v53
 ; SI-NEXT:    v_and_b32_e32 v53, 0xffff0000, v54
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    v_and_b32_e32 v54, 0xffff0000, v55
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v55, 0xffff0000, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v55
-; SI-NEXT:    v_and_b32_e32 v55, 0xffff0000, v46
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v46
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
@@ -245524,7 +245525,7 @@ define <64 x bfloat> @bitcast_v64i16_to_v64bf16(<64 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:  .LBB106_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; SI-NEXT:    s_waitcnt vmcnt(7) expcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_mul_f32_e32 v1, 1.0, v2
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_mul_f32_e32 v2, 1.0, v27
@@ -247351,531 +247352,533 @@ define <64 x i16> @bitcast_v64f16_to_v64i16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:136
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:4
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:28
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt expcnt(6)
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:48
 ; SI-NEXT:    s_waitcnt expcnt(5)
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
 ; SI-NEXT:    s_waitcnt expcnt(4)
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:56
 ; SI-NEXT:    s_waitcnt expcnt(3)
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt expcnt(2)
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:80
 ; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:84
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:104
-; SI-NEXT:    v_cvt_f16_f32_e32 v55, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v17
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:88
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v2
+; SI-NEXT:    v_cvt_f16_f32_e32 v2, v9
 ; SI-NEXT:    v_cvt_f16_f32_e32 v43, v3
-; SI-NEXT:    v_cvt_f16_f32_e32 v44, v4
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v18
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v2
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v5
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v6
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v5
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v9
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v2, v13
+; SI-NEXT:    v_cvt_f16_f32_e32 v55, v7
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v30
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v12
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v6
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v22
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v40, v8
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v10
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v13
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v19
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v30
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v21
+; SI-NEXT:    v_cvt_f16_f32_e32 v53, v12
+; SI-NEXT:    v_cvt_f16_f32_e32 v9, v14
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v18
+; SI-NEXT:    v_cvt_f16_f32_e32 v18, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v23
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v40, v8
-; SI-NEXT:    v_cvt_f16_f32_e32 v8, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v11
-; SI-NEXT:    v_cvt_f16_f32_e32 v10, v14
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v44, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v52, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v15
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v11, v21
+; SI-NEXT:    v_cvt_f16_f32_e32 v48, v16
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v13, v20
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v24
-; SI-NEXT:    v_cvt_f16_f32_e32 v24, v26
+; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v39
 ; SI-NEXT:    v_cvt_f16_f32_e32 v31, v27
-; SI-NEXT:    v_cvt_f16_f32_e32 v27, v28
-; SI-NEXT:    v_cvt_f16_f32_e32 v28, v29
-; SI-NEXT:    v_cvt_f16_f32_e32 v26, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v29, v42
-; SI-NEXT:    v_cvt_f16_f32_e32 v42, v47
-; SI-NEXT:    v_cvt_f16_f32_e32 v21, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v50
-; SI-NEXT:    v_cvt_f16_f32_e32 v38, v54
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v56
-; SI-NEXT:    v_cvt_f16_f32_e32 v47, v41
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v50
+; SI-NEXT:    v_cvt_f16_f32_e32 v27, v29
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v42
+; SI-NEXT:    v_cvt_f16_f32_e32 v21, v47
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v38
+; SI-NEXT:    v_cvt_f16_f32_e32 v38, v49
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f16_f32_e32 v47, v54
+; SI-NEXT:    v_cvt_f16_f32_e32 v24, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v54, v46
+; SI-NEXT:    v_cvt_f16_f32_e32 v42, v56
+; SI-NEXT:    v_cvt_f16_f32_e32 v29, v51
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v41
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v59
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v45
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v59
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    v_cvt_f16_f32_e32 v18, v46
-; SI-NEXT:    v_cvt_f16_f32_e32 v46, v57
-; SI-NEXT:    v_cvt_f16_f32_e32 v41, v58
-; SI-NEXT:    v_cvt_f16_f32_e32 v50, v60
-; SI-NEXT:    v_cvt_f16_f32_e32 v30, v34
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v61
-; SI-NEXT:    v_cvt_f16_f32_e32 v34, v36
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:120
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cvt_f16_f32_e32 v39, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v57, v62
-; SI-NEXT:    v_cvt_f16_f32_e32 v13, v35
-; SI-NEXT:    v_cvt_f16_f32_e32 v58, v32
-; SI-NEXT:    v_cvt_f16_f32_e32 v61, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v11, v57
+; SI-NEXT:    v_cvt_f16_f32_e32 v58, v58
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(14) expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v62
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v32
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v60
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_cvt_f16_f32_e32 v50, v37
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:120
+; SI-NEXT:    v_cvt_f16_f32_e32 v61, v61
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v33
 ; SI-NEXT:    v_cvt_f16_f32_e32 v33, v63
-; SI-NEXT:    v_cvt_f16_f32_e32 v45, v48
-; SI-NEXT:    v_cvt_f16_f32_e32 v14, v49
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f16_f32_e32 v54, v53
-; SI-NEXT:    v_cvt_f16_f32_e32 v32, v52
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f16_f32_e32 v35, v7
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v9
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v59, v36
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:124
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:128
+; SI-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_cvt_f16_f32_e32 v45, v39
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f16_f32_e32 v14, v6
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v7
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f16_f32_e32 v41, v8
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f16_f32_e32 v30, v10
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:124
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:128
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:132
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v32
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f16_f32_e32 v8, v46
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f16_f32_e32 v46, v56
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f16_f32_e32 v56, v9
+; SI-NEXT:    v_cvt_f16_f32_e32 v32, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v37
+; SI-NEXT:    v_cvt_f16_f32_e32 v10, v37
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT:    v_mov_b32_e32 v60, v29
-; SI-NEXT:    v_mov_b32_e32 v29, v28
-; SI-NEXT:    v_mov_b32_e32 v62, v23
-; SI-NEXT:    v_mov_b32_e32 v63, v11
+; SI-NEXT:    v_mov_b32_e32 v59, v29
+; SI-NEXT:    v_mov_b32_e32 v29, v27
+; SI-NEXT:    v_mov_b32_e32 v57, v23
+; SI-NEXT:    v_mov_b32_e32 v60, v3
+; SI-NEXT:    v_mov_b32_e32 v62, v4
+; SI-NEXT:    v_mov_b32_e32 v63, v49
+; SI-NEXT:    v_mov_b32_e32 v49, v12
 ; SI-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB108_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.true
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; SI-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    v_cvt_f32_f16_e32 v34, v34
-; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; SI-NEXT:    v_cvt_f32_f16_e32 v36, v36
+; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f32_f16_e32 v33, v33
-; SI-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v36, v36
+; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
-; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v36, v36
 ; SI-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v33, v33
-; SI-NEXT:    v_or_b32_e32 v36, v36, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v7
-; SI-NEXT:    v_or_b32_e32 v35, v35, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v14
-; SI-NEXT:    v_or_b32_e32 v13, v13, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
+; SI-NEXT:    v_or_b32_e32 v6, v6, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v8
+; SI-NEXT:    v_or_b32_e32 v7, v7, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
+; SI-NEXT:    v_or_b32_e32 v14, v14, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v36
 ; SI-NEXT:    v_or_b32_e32 v33, v33, v37
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v58
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v57
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v41
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v58, v37
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v48
-; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
-; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v58
-; SI-NEXT:    v_or_b32_e32 v57, v37, v48
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v12
-; SI-NEXT:    v_or_b32_e32 v41, v11, v37
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v58
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
 ; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    v_cvt_f32_f16_e32 v26, v26
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; SI-NEXT:    v_cvt_f16_f32_e32 v58, v12
+; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
 ; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
-; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
-; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
-; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
 ; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v31, v31
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
-; SI-NEXT:    v_or_b32_e32 v17, v17, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v22
+; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT:    v_or_b32_e32 v21, v21, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v26
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v52
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT:    v_or_b32_e32 v25, v25, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v27
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, v3
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, v61
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v4
+; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT:    v_or_b32_e32 v61, v3, v37
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v58
+; SI-NEXT:    v_or_b32_e32 v11, v11, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v17
+; SI-NEXT:    v_or_b32_e32 v16, v16, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v22
+; SI-NEXT:    v_or_b32_e32 v21, v21, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v25
+; SI-NEXT:    v_or_b32_e32 v24, v24, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v28
 ; SI-NEXT:    v_or_b32_e32 v31, v31, v37
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_or_b32_e32 v16, v16, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v13
+; SI-NEXT:    v_or_b32_e32 v18, v18, v37
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v48
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; SI-NEXT:    v_cvt_f16_f32_e32 v48, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v48
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v6
-; SI-NEXT:    v_or_b32_e32 v5, v5, v37
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v53
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; SI-NEXT:    v_cvt_f16_f32_e32 v53, v37
+; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v39
+; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v53
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    v_or_b32_e32 v52, v37, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v37, v40
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v55
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v55
+; SI-NEXT:    v_cvt_f32_f16_e32 v56, v62
+; SI-NEXT:    v_cvt_f32_f16_e32 v26, v26
 ; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
 ; SI-NEXT:    v_cvt_f16_f32_e32 v40, v37
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v48
+; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v39
 ; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v40
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT:    v_or_b32_e32 v55, v37, v48
+; SI-NEXT:    v_add_f32_e32 v56, 0x38000000, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v40
+; SI-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; SI-NEXT:    v_or_b32_e32 v55, v37, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v37, v44
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v43
-; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v43
+; SI-NEXT:    v_cvt_f32_f16_e32 v57, v57
+; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
 ; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
 ; SI-NEXT:    v_cvt_f16_f32_e32 v44, v37
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v48
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v44
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v43, v37, v48
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v11
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v39
 ; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_cvt_f32_f16_e32 v52, v62
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v42
+; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v44
+; SI-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; SI-NEXT:    v_or_b32_e32 v43, v37, v39
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT:    v_cvt_f32_f16_e32 v47, v47
+; SI-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v50, v50
 ; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
-; SI-NEXT:    v_cvt_f32_f16_e32 v39, v39
-; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
-; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
-; SI-NEXT:    v_cvt_f32_f16_e32 v42, v45
-; SI-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
-; SI-NEXT:    v_cvt_f16_f32_e32 v30, v30
-; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
-; SI-NEXT:    v_cvt_f16_f32_e32 v39, v39
-; SI-NEXT:    v_add_f32_e32 v42, 0x38000000, v42
-; SI-NEXT:    v_cvt_f16_f32_e32 v42, v42
-; SI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
 ; SI-NEXT:    v_cvt_f32_f16_e32 v32, v32
-; SI-NEXT:    v_cvt_f32_f16_e32 v51, v51
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
 ; SI-NEXT:    v_cvt_f32_f16_e32 v38, v38
-; SI-NEXT:    v_cvt_f32_f16_e32 v50, v50
-; SI-NEXT:    v_or_b32_e32 v45, v42, v39
-; SI-NEXT:    v_cvt_f32_f16_e32 v42, v59
-; SI-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; SI-NEXT:    v_cvt_f32_f16_e32 v54, v54
 ; SI-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v32, v32
-; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
-; SI-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; SI-NEXT:    v_cvt_f32_f16_e32 v45, v45
+; SI-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; SI-NEXT:    v_cvt_f32_f16_e32 v46, v46
 ; SI-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; SI-NEXT:    v_add_f32_e32 v47, 0x38000000, v47
+; SI-NEXT:    v_add_f32_e32 v42, 0x38000000, v42
+; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
+; SI-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
+; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
+; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; SI-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
+; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; SI-NEXT:    v_cvt_f16_f32_e32 v47, v47
+; SI-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
+; SI-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; SI-NEXT:    v_add_f32_e32 v54, 0x38000000, v54
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v50, v50
+; SI-NEXT:    v_add_f32_e32 v45, 0x38000000, v45
+; SI-NEXT:    v_cvt_f16_f32_e32 v30, v30
+; SI-NEXT:    v_add_f32_e32 v41, 0x38000000, v41
+; SI-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; SI-NEXT:    v_add_f32_e32 v46, 0x38000000, v46
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v38, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v50, v50
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT:    v_cvt_f16_f32_e32 v54, v54
 ; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; SI-NEXT:    v_alignbit_b32 v39, v13, v39, 16
-; SI-NEXT:    v_or_b32_e32 v50, v50, v51
-; SI-NEXT:    v_alignbit_b32 v51, v57, v51, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v11
-; SI-NEXT:    v_or_b32_e32 v11, v37, v1
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    v_alignbit_b32 v1, v43, v1, 16
-; SI-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
-; SI-NEXT:    v_cvt_f16_f32_e32 v48, v48
+; SI-NEXT:    v_cvt_f16_f32_e32 v45, v45
+; SI-NEXT:    v_cvt_f16_f32_e32 v41, v41
+; SI-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; SI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT:    v_or_b32_e32 v38, v38, v47
+; SI-NEXT:    v_or_b32_e32 v54, v54, v42
+; SI-NEXT:    v_or_b32_e32 v45, v45, v50
+; SI-NEXT:    v_or_b32_e32 v41, v41, v30
+; SI-NEXT:    v_or_b32_e32 v46, v46, v32
+; SI-NEXT:    v_alignbit_b32 v47, v16, v47, 16
+; SI-NEXT:    v_alignbit_b32 v42, v11, v42, 16
+; SI-NEXT:    v_alignbit_b32 v50, v14, v50, 16
+; SI-NEXT:    v_alignbit_b32 v30, v7, v30, 16
+; SI-NEXT:    v_alignbit_b32 v32, v6, v32, 16
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v34
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v34, v43, v34, 16
+; SI-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; SI-NEXT:    v_cvt_f16_f32_e32 v39, v39
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_or_b32_e32 v11, v48, v4
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v3, v39, v1
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    v_alignbit_b32 v4, v55, v4, 16
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    v_alignbit_b32 v1, v55, v1, 16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v11
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
 ; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v11
-; SI-NEXT:    v_or_b32_e32 v11, v37, v8
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
-; SI-NEXT:    v_cvt_f16_f32_e32 v48, v48
-; SI-NEXT:    v_alignbit_b32 v8, v5, v8, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v11
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v5
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v49
+; SI-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; SI-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; SI-NEXT:    v_alignbit_b32 v5, v52, v5, 16
 ; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
 ; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v11
-; SI-NEXT:    v_or_b32_e32 v11, v48, v10
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT:    v_alignbit_b32 v10, v2, v10, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v11
-; SI-NEXT:    v_or_b32_e32 v11, v49, v37
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    v_or_b32_e32 v3, v39, v9
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v63
-; SI-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
-; SI-NEXT:    v_cvt_f16_f32_e32 v48, v48
-; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; SI-NEXT:    v_or_b32_e32 v63, v49, v48
-; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v52
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT:    v_cvt_f32_f16_e32 v52, v47
-; SI-NEXT:    v_or_b32_e32 v62, v49, v24
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v60
-; SI-NEXT:    v_add_f32_e32 v52, 0x38000000, v52
-; SI-NEXT:    v_cvt_f16_f32_e32 v52, v52
-; SI-NEXT:    v_alignbit_b32 v24, v31, v24, 16
-; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT:    v_or_b32_e32 v60, v49, v28
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v52
-; SI-NEXT:    v_cvt_f32_f16_e32 v52, v46
-; SI-NEXT:    v_or_b32_e32 v38, v38, v49
-; SI-NEXT:    v_add_f32_e32 v52, 0x38000000, v52
-; SI-NEXT:    v_cvt_f16_f32_e32 v52, v52
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; SI-NEXT:    v_alignbit_b32 v46, v41, v52, 16
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
+; SI-NEXT:    v_or_b32_e32 v62, v56, v37
+; SI-NEXT:    v_cvt_f32_f16_e32 v56, v60
+; SI-NEXT:    v_alignbit_b32 v9, v2, v9, 16
+; SI-NEXT:    v_add_f32_e32 v56, 0x38000000, v56
+; SI-NEXT:    v_cvt_f16_f32_e32 v56, v56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v11
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v39, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; SI-NEXT:    v_cvt_f16_f32_e32 v39, v39
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
+; SI-NEXT:    v_or_b32_e32 v60, v56, v39
+; SI-NEXT:    v_add_f32_e32 v56, 0x38000000, v57
+; SI-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; SI-NEXT:    v_or_b32_e32 v57, v56, v26
+; SI-NEXT:    v_cvt_f32_f16_e32 v56, v59
+; SI-NEXT:    v_alignbit_b32 v26, v31, v26, 16
+; SI-NEXT:    v_add_f32_e32 v56, 0x38000000, v56
+; SI-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v23, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v53, v11
-; SI-NEXT:    v_add_f32_e32 v53, 0x38000000, v53
-; SI-NEXT:    v_cvt_f16_f32_e32 v53, v53
-; SI-NEXT:    v_or_b32_e32 v11, v53, v52
-; SI-NEXT:    v_cvt_f32_f16_e32 v53, v61
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    v_cvt_f32_f16_e32 v27, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
+; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT:    v_or_b32_e32 v59, v56, v27
+; SI-NEXT:    v_cvt_f32_f16_e32 v56, v63
+; SI-NEXT:    v_add_f32_e32 v56, 0x38000000, v56
+; SI-NEXT:    v_cvt_f16_f32_e32 v56, v56
+; SI-NEXT:    v_or_b32_e32 v63, v56, v35
+; SI-NEXT:    v_alignbit_b32 v35, v33, v35, 16
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v49, v3
+; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
+; SI-NEXT:    v_or_b32_e32 v3, v49, v51
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v11, v16, v37, 16
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    v_add_f32_e32 v53, 0x38000000, v53
-; SI-NEXT:    v_cvt_f16_f32_e32 v53, v53
+; SI-NEXT:    v_alignbit_b32 v3, v19, v39, 16
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v11, v19, v48, 16
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_alignbit_b32 v3, v24, v23, 16
+; SI-NEXT:    v_alignbit_b32 v49, v18, v37, 16
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v11, v25, v23, 16
-; SI-NEXT:    v_or_b32_e32 v61, v53, v30
-; SI-NEXT:    v_cvt_f32_f16_e32 v53, v54
-; SI-NEXT:    v_cvt_f32_f16_e32 v54, v56
-; SI-NEXT:    v_alignbit_b32 v30, v33, v30, 16
-; SI-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    v_add_f32_e32 v53, 0x38000000, v53
-; SI-NEXT:    v_add_f32_e32 v54, 0x38000000, v54
-; SI-NEXT:    v_cvt_f16_f32_e32 v53, v53
-; SI-NEXT:    v_cvt_f16_f32_e32 v47, v54
-; SI-NEXT:    v_add_f32_e32 v54, 0x38000000, v42
-; SI-NEXT:    v_cvt_f16_f32_e32 v42, v54
-; SI-NEXT:    v_or_b32_e32 v54, v53, v32
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v47
-; SI-NEXT:    v_alignbit_b32 v47, v17, v49, 16
-; SI-NEXT:    v_or_b32_e32 v59, v42, v53
-; SI-NEXT:    v_alignbit_b32 v42, v21, v28, 16
-; SI-NEXT:    v_alignbit_b32 v32, v35, v32, 16
-; SI-NEXT:    v_alignbit_b32 v56, v36, v53, 16
+; SI-NEXT:    v_alignbit_b32 v3, v21, v27, 16
+; SI-NEXT:    v_alignbit_b32 v51, v61, v51, 16
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB108_2: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v37, 0xffff, v11
-; SI-NEXT:    v_or_b32_e32 v1, v37, v1
-; SI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v37, 0xffff, v3
+; SI-NEXT:    v_or_b32_e32 v34, v37, v34
+; SI-NEXT:    buffer_store_dword v34, v0, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v43
+; SI-NEXT:    v_and_b32_e32 v34, 0xffff, v43
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v44
-; SI-NEXT:    v_or_b32_e32 v1, v1, v37
+; SI-NEXT:    v_or_b32_e32 v34, v34, v37
 ; SI-NEXT:    v_add_i32_e32 v37, vcc, 4, v0
-; SI-NEXT:    buffer_store_dword v1, v37, s[0:3], 0 offen
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v1, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 8, v0
-; SI-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_dword v34, v37, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v34, 0xffff, v3
+; SI-NEXT:    v_or_b32_e32 v1, v34, v1
+; SI-NEXT:    v_add_i32_e32 v34, vcc, 8, v0
+; SI-NEXT:    buffer_store_dword v1, v34, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v55
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v40
-; SI-NEXT:    v_or_b32_e32 v1, v1, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 12, v0
-; SI-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v40
+; SI-NEXT:    v_or_b32_e32 v1, v1, v34
+; SI-NEXT:    v_add_i32_e32 v34, vcc, 12, v0
+; SI-NEXT:    buffer_store_dword v1, v34, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v8
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v1, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
-; SI-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen
+; SI-NEXT:    v_or_b32_e32 v1, v1, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 16, v0
+; SI-NEXT:    buffer_store_dword v1, v5, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
-; SI-NEXT:    v_or_b32_e32 v1, v1, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 20, v0
-; SI-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v53
+; SI-NEXT:    v_or_b32_e32 v1, v1, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 20, v0
+; SI-NEXT:    buffer_store_dword v1, v5, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v1, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 24, v0
-; SI-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen
+; SI-NEXT:    v_or_b32_e32 v1, v1, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 24, v0
+; SI-NEXT:    buffer_store_dword v1, v5, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v48
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 28, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v62
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v49
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v15
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 36, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v63
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v60
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
@@ -247888,14 +247891,14 @@ define <64 x i16> @bitcast_v64f16_to_v64i16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v62
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v24
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v57
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v26
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 48, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v31
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v28
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 52, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
@@ -247908,14 +247911,16 @@ define <64 x i16> @bitcast_v64f16_to_v64i16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 56, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v26
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v25
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v60
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v42
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v59
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
@@ -247932,82 +247937,84 @@ define <64 x i16> @bitcast_v64f16_to_v64i16(<64 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x48, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v18
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v17
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v46
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v42
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v41
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v12
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v58
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v50
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v51
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x58, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v57
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v58
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x5c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v30
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v63
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v35
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v33
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v36
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v45
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v39
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v50
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x68, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v14
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v15
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v32
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v30
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v35
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v7
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v8
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x74, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v59
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v56
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v46
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v32
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x78, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v36
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v9
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x7c, v0
 ; SI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index af3912108e53..6338eb3d4c43 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -7393,15 +7393,10 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v15, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v6 :: v_dual_mov_b32 v17, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v3 :: v_dual_mov_b32 v19, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v20, v2 :: v_dual_mov_b32 v21, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB27_4
@@ -7441,27 +7436,27 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v21
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v14, v23
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v25
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s29, 8
@@ -7470,14 +7465,14 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v3, v7
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v14, v23
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s6
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v3, s8
 ; GFX11-TRUE16-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
@@ -7535,22 +7530,22 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v20
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v16
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v17
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v22
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v9
@@ -14771,15 +14766,10 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v15, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v6 :: v_dual_mov_b32 v17, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v3 :: v_dual_mov_b32 v19, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v20, v2 :: v_dual_mov_b32 v21, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB51_4
@@ -14819,27 +14809,27 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v21
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v14, v23
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v25
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s29, 8
@@ -14848,14 +14838,14 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v3, v7
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v14, v23
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s6
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v3, s8
 ; GFX11-TRUE16-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
@@ -14913,22 +14903,22 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v20
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v16
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v17
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v22
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v9
@@ -21656,15 +21646,10 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v15, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v6 :: v_dual_mov_b32 v17, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v3 :: v_dual_mov_b32 v19, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v20, v2 :: v_dual_mov_b32 v21, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB71_4
@@ -21704,27 +21689,27 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v21
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v14, v23
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v25
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s29, 8
@@ -21733,14 +21718,14 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v3, v7
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v14, v23
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s6
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v3, s8
 ; GFX11-TRUE16-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
@@ -21798,22 +21783,22 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v20
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v16
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v17
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v22
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v9
@@ -28039,15 +28024,10 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v15, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v6 :: v_dual_mov_b32 v17, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v3 :: v_dual_mov_b32 v19, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v20, v2 :: v_dual_mov_b32 v21, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB87_4
@@ -28087,27 +28067,27 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v21
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xff, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 8, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v14, v23
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v25
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s29, 8
@@ -28116,14 +28096,14 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v3, v7
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v22, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v14, v23
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s6
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v3, s8
 ; GFX11-TRUE16-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
@@ -28181,22 +28161,22 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v20
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v19
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v8
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v16
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v17
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v22
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v9
@@ -34105,20 +34085,10 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v21, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v6 :: v_dual_mov_b32 v19, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v20, v4 :: v_dual_mov_b32 v15, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v22, v2 :: v_dual_mov_b32 v17, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB99_4
@@ -34149,25 +34119,25 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v16
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v9
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xff, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v18
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v11
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v13
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v5, v6
@@ -34231,42 +34201,42 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s10, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v22
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v10
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v11
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v20
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v22
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v15
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
@@ -34280,7 +34250,7 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v5, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v3, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v9, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v8, 16, v7
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -39305,20 +39275,10 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v21, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v6 :: v_dual_mov_b32 v19, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v20, v4 :: v_dual_mov_b32 v15, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v22, v2 :: v_dual_mov_b32 v17, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB107_4
@@ -39349,25 +39309,25 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v16
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v9
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xff, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v18
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v11
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v13
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v5, v6
@@ -39431,42 +39391,42 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s10, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v22
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v10
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v11
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v20
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v22
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v15
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
@@ -39480,7 +39440,7 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v5, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v3, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v9, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v8, 16, v7
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -43653,20 +43613,10 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v21, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v6 :: v_dual_mov_b32 v19, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v20, v4 :: v_dual_mov_b32 v15, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v22, v2 :: v_dual_mov_b32 v17, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB111_4
@@ -43697,25 +43647,25 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v16
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v9
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xff, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v18
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v11
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 8, v13
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v5, v6
@@ -43779,42 +43729,42 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s10, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v22
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v10
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v11
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v20
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v22
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v15
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
@@ -43828,7 +43778,7 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v5, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v3, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v9, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v8, 16, v7
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index 66ddd80c0e92..06525235b3a5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -4568,14 +4568,14 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
 ; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:4
@@ -4595,18 +4595,18 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v21, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v15, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 24, v0
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v2
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v19, 24, v4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 24, v4
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v11, 24, v8
-; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 8, v6
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 8, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v15, 24, v44
+; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
@@ -4862,13 +4862,13 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_mov_b32_e32 v33, v4
 ; VI-NEXT:    v_mov_b32_e32 v32, v2
 ; VI-NEXT:    v_mov_b32_e32 v31, v0
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; VI-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:16
 ; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
 ; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
@@ -4888,18 +4888,18 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
 ; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v11, 8, v4
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
-; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; VI-NEXT:    v_lshlrev_b16_e32 v13, 8, v6
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v44
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB14_2
@@ -5074,13 +5074,13 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_mov_b32_e32 v33, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v32, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; GFX9-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:16
 ; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
 ; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
@@ -5100,18 +5100,18 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v11, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
-; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v13, 8, v6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v44
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB14_2
@@ -6469,17 +6469,11 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v9 :: v_dual_mov_b32 v23, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v24, v8 :: v_dual_mov_b32 v25, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v26, v5 :: v_dual_mov_b32 v27, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v28, v4 :: v_dual_mov_b32 v29, v2
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v30, v1 :: v_dual_mov_b32 v31, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB15_4
@@ -6509,7 +6503,7 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s7, s7, s8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v30
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s7, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s8, 16
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s24, 0xff
@@ -6527,29 +6521,29 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v27
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v28
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v30
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 8, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v29
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v11
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v22
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v14
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v16
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xff, v18
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v19
@@ -6557,9 +6551,9 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v5, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v8, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v32, v33
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v22, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v34, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v36, v37
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v36, v37
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -6568,12 +6562,12 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v7, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v34, v8
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v9, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v9, v22
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s6
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v3, s8
 ; GFX11-TRUE16-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
@@ -6631,36 +6625,36 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v30
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v28
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v26
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v27
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v24
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v26
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v25
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v16
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v18
@@ -12053,14 +12047,14 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
 ; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:4
@@ -12080,18 +12074,18 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v21, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v15, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 24, v0
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v2
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v19, 24, v4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 24, v4
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v11, 24, v8
-; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 8, v6
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 8, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v15, 24, v44
+; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB34_2
@@ -12347,13 +12341,13 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_mov_b32_e32 v33, v4
 ; VI-NEXT:    v_mov_b32_e32 v32, v2
 ; VI-NEXT:    v_mov_b32_e32 v31, v0
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; VI-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:16
 ; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
 ; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
@@ -12373,18 +12367,18 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
 ; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v11, 8, v4
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
-; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; VI-NEXT:    v_lshlrev_b16_e32 v13, 8, v6
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v15, 8, v44
+; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB34_2
@@ -12559,13 +12553,13 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_mov_b32_e32 v33, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v32, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; GFX9-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:16
 ; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:20
 ; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:4
@@ -12585,18 +12579,18 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v2
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v11, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v11, 8, v8
-; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v13, 8, v6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v13, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v15, 8, v44
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB34_2
@@ -13954,17 +13948,11 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v9 :: v_dual_mov_b32 v23, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v24, v8 :: v_dual_mov_b32 v25, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v26, v5 :: v_dual_mov_b32 v27, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v28, v4 :: v_dual_mov_b32 v29, v2
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v30, v1 :: v_dual_mov_b32 v31, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB35_4
@@ -13994,7 +13982,7 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s7, s7, s8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v30
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s7, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s8, 16
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s24, 0xff
@@ -14012,29 +14000,29 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v27
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v28
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v30
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 8, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v29
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v11
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v22
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v14
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xff, v16
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v33, 8, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xff, v18
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 8, v19
@@ -14042,9 +14030,9 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v5, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v8, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v32, v33
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v22, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v34, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v36, v37
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v36, v37
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -14053,12 +14041,12 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v7, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v34, v8
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v9, v32
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v9, v22
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s6
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v3, s8
 ; GFX11-TRUE16-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
@@ -14116,36 +14104,36 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v30
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v28
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v26
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v27
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v24
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v26
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v25
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v16
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v18
@@ -19423,13 +19411,13 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_mov_b32_e32 v38, v4
 ; VI-NEXT:    v_mov_b32_e32 v32, v2
 ; VI-NEXT:    v_mov_b32_e32 v36, v0
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:20
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:4
@@ -19451,17 +19439,17 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v27
 ; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v2
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v4
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v4
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v8
+; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v6
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
+; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -19646,13 +19634,13 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_mov_b32_e32 v35, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v33, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v36, v0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:20
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:4
@@ -19674,17 +19662,17 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v2
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v47, 8, v6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v47, 8, v10
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v10
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -21021,28 +21009,14 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v30, v15 :: v_dual_mov_b32 v25, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, v14 :: v_dual_mov_b32 v27, v12
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v29, v11 :: v_dual_mov_b32 v36, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v10 :: v_dual_mov_b32 v24, v8
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v23, v7 :: v_dual_mov_b32 v26, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v28, v5 :: v_dual_mov_b32 v35, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v3 :: v_dual_mov_b32 v38, v2
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v1 :: v_dual_mov_b32 v37, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB51_4
@@ -21073,44 +21047,44 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v37
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v33
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v38
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s11, s12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v1, 0xffff, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v23
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v0, 16, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v20
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v32
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v30
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v27
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v6, v7
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v8, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v19
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v29
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v8, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v5
@@ -21170,61 +21144,61 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s1, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v20
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v18
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v35
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v32
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v34
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v31
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v21
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v29
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v23
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v38
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v28
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v27
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v25
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v8, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v35
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v37
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v28
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 0x300, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
@@ -25685,14 +25659,14 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v36, v4
 ; SI-NEXT:    v_mov_b32_e32 v31, v2
 ; SI-NEXT:    v_mov_b32_e32 v35, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:28
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 8, v1
@@ -25700,7 +25674,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v39, 8, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v49, 8, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 8, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 8, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v55, 8, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v17
@@ -25726,23 +25700,23 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr15
 ; SI-NEXT:    ; implicit-def: $vgpr17
 ; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 8, v0
+; SI-NEXT:    s_waitcnt vmcnt(9) expcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v0
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v2
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v32
-; SI-NEXT:    s_waitcnt vmcnt(5) expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v33
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; SI-NEXT:    s_waitcnt vmcnt(7) expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v4
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    ; implicit-def: $vgpr4
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v32
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v33
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 8, v34
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v34
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -25786,7 +25760,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v51
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v57
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v6
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v52
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v53
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v58
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v35
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v31
@@ -25794,13 +25768,13 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v8
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v10
 ; SI-NEXT:    v_cvt_f32_f16_e32 v34, v6
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v54
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v52
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v37
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v38
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v39
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v48
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v49
-; SI-NEXT:    v_or_b32_e32 v5, v5, v53
+; SI-NEXT:    v_or_b32_e32 v5, v5, v54
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v59
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
@@ -25827,14 +25801,14 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr52
-; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr41
@@ -25853,12 +25827,12 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB62_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v54
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v59, v0
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 0x300, v0
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v52
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v53
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    v_or_b32_e32 v0, v58, v0
@@ -25915,7 +25889,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v0
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v10
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v53, v0
+; SI-NEXT:    v_or_b32_e32 v0, v54, v0
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v0
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v8
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
@@ -25998,13 +25972,13 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_mov_b32_e32 v38, v4
 ; VI-NEXT:    v_mov_b32_e32 v32, v2
 ; VI-NEXT:    v_mov_b32_e32 v36, v0
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; VI-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:20
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:4
@@ -26026,17 +26000,17 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v27
 ; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v2
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v4
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v4
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v8
+; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v6
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
+; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -26221,13 +26195,13 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_mov_b32_e32 v35, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v33, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v36, v0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; GFX9-NEXT:    buffer_load_ushort v54, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:20
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:4
@@ -26249,17 +26223,17 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v2
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v47, 8, v6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v47, 8, v10
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v10
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -27569,28 +27543,14 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v30, v15 :: v_dual_mov_b32 v25, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, v14 :: v_dual_mov_b32 v27, v12
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v29, v11 :: v_dual_mov_b32 v36, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v10 :: v_dual_mov_b32 v24, v8
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v23, v7 :: v_dual_mov_b32 v26, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v28, v5 :: v_dual_mov_b32 v35, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v3 :: v_dual_mov_b32 v38, v2
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v1 :: v_dual_mov_b32 v37, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB63_4
@@ -27621,44 +27581,44 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v37
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v33
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v38
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s11, s12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v1, 0xffff, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v23
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v0, 16, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v24
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v20
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v32
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v30
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v27
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v6, v7
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v8, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v19
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v29
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v8, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v5
@@ -27718,61 +27678,61 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s1, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v20
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v18
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v35
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v32
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v34
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v31
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v21
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v29
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v23
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v38
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v28
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v27
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v25
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v8, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v35
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v37
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v28
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 0x300, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
@@ -30050,13 +30010,13 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:28
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:4
@@ -30079,17 +30039,17 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v27, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 24, v0
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 8, v2
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 24, v4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v4
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v6
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v23, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v10
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -30355,13 +30315,13 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_mov_b32_e32 v33, v4
 ; VI-NEXT:    v_mov_b32_e32 v32, v2
 ; VI-NEXT:    v_mov_b32_e32 v31, v0
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:20
 ; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:4
@@ -30383,17 +30343,17 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v27
 ; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v2
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v4
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v4
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
+; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v6
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v10
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v10
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -30578,13 +30538,13 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_mov_b32_e32 v33, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v32, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:20
 ; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:4
@@ -30606,17 +30566,17 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v2
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v10
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v10
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -31989,23 +31949,14 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v38, v15 :: v_dual_mov_b32 v23, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v24, v14 :: v_dual_mov_b32 v25, v12
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v26, v11 :: v_dual_mov_b32 v27, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v28, v10 :: v_dual_mov_b32 v29, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v30, v8 :: v_dual_mov_b32 v31, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v5 :: v_dual_mov_b32 v33, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, v4 :: v_dual_mov_b32 v35, v2
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v36, v1 :: v_dual_mov_b32 v37, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB73_4
@@ -32035,7 +31986,7 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s7, s7, s8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v36
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s7, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s8, 16
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s24, 0xff
@@ -32053,29 +32004,29 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v33
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v34
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v32
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v30
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v29
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v30
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v16
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v18
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v19
@@ -32157,49 +32108,49 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v36
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v35
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v34
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v33
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v32
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v28
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v31
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v32
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v16
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v18
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v6, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v24
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v17
@@ -34694,13 +34645,13 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:28
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:4
@@ -34723,17 +34674,17 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v51, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v27, 8, v29
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 24, v0
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 8, v2
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 24, v4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v4
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v6
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v23, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v10
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -34999,13 +34950,13 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_mov_b32_e32 v33, v4
 ; VI-NEXT:    v_mov_b32_e32 v32, v2
 ; VI-NEXT:    v_mov_b32_e32 v31, v0
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:20
 ; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:12
 ; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:4
@@ -35027,17 +34978,17 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v27
 ; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v0
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v2
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v4
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v4
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
+; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v6
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v10
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v10
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -35222,13 +35173,13 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_mov_b32_e32 v33, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v32, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:32
 ; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:16
 ; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:20
 ; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:4
@@ -35250,17 +35201,17 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v27
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v29
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v2
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v6
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v10
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v10
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -36633,23 +36584,14 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v0.l
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v38, v15 :: v_dual_mov_b32 v23, v13
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v24, v14 :: v_dual_mov_b32 v25, v12
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v26, v11 :: v_dual_mov_b32 v27, v9
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v28, v10 :: v_dual_mov_b32 v29, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v30, v8 :: v_dual_mov_b32 v31, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v5 :: v_dual_mov_b32 v33, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, v4 :: v_dual_mov_b32 v35, v2
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v36, v1 :: v_dual_mov_b32 v37, v0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB77_4
@@ -36679,7 +36621,7 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_or_b32 s7, s7, s8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s9, s10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v36
 ; GFX11-TRUE16-NEXT:    s_and_b32 s7, s7, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s8, s8, 16
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s24, 0xff
@@ -36697,29 +36639,29 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s10, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v33
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v34
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v36
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v32
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v30
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v29
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v30
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v16
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v18
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v19
@@ -36801,49 +36743,49 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v36
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s3, s4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v35
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v34
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v33
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v32
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s6, s29, 8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v33
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s6, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v28
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v31
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v32
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v16
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v18
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v6, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v7, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v24
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v17
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 20df26e30656..562bfb7304ac 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -11396,170 +11396,171 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v34, v6
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:132
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:128
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 8, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v28, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v30, 8, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v49, 8, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 24, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v17
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 24, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 8, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 8, v17
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v53
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 24, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 24, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 24, v12
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v39
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 24, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v50
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v54
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 24, v14
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v45
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v50
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 24, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v51
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 8, v58
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 8, v52
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 24, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v53
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v55
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v40
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v42
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 24, v58
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 8, v13
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB26_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v33
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v34
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v49
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v2, v2, v36
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
+; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v42
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v52
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v58
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v44
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -11567,18 +11568,18 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v48
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
@@ -11587,7 +11588,6 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v24, v6
-; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -11595,30 +11595,30 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v3, v16
+; SI-NEXT:    v_or_b32_e32 v3, v3, v38
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v17
+; SI-NEXT:    v_or_b32_e32 v4, v4, v18
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v22
@@ -11638,54 +11638,59 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v30
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v43
-; SI-NEXT:    v_or_b32_e32 v8, v8, v62
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v60
+; SI-NEXT:    v_or_b32_e32 v8, v8, v63
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v51
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v45
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v59
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr62
-; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v10, v47
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v55
+; SI-NEXT:    v_or_b32_e32 v10, v10, v46
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v58
-; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v11, v11, v41
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v12, v40
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v12, v12, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v41
-; SI-NEXT:    v_or_b32_e32 v13, v13, v50
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v40
+; SI-NEXT:    v_or_b32_e32 v13, v13, v19
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v23
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v25
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v19
-; SI-NEXT:    v_or_b32_e32 v15, v15, v21
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    ; implicit-def: $vgpr25
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_or_b32_e32 v15, v15, v16
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
@@ -11712,21 +11717,20 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    ; implicit-def: $vgpr21
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:  .LBB26_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB26_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v32
@@ -11734,55 +11738,54 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v34
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
+; SI-NEXT:    v_or_b32_e32 v1, v39, v1
 ; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v37, v2
+; SI-NEXT:    v_or_b32_e32 v2, v35, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v42
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v58
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v44
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v53
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -11791,20 +11794,20 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_or_b32_e32 v2, v48, v2
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -11820,35 +11823,34 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v38, v3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s7, v3
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v17, v4
+; SI-NEXT:    v_or_b32_e32 v4, v18, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s7, v4
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -11874,68 +11876,70 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v43
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v60
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v62, v8
+; SI-NEXT:    v_or_b32_e32 v8, v63, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v51
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v59, v9
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v55
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v47, v10
+; SI-NEXT:    v_or_b32_e32 v10, v46, v10
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v58
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v11, v41, v11
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v47
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v40, v12
+; SI-NEXT:    v_or_b32_e32 v12, v51, v12
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v41
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v40
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_or_b32_e32 v13, v50, v13
+; SI-NEXT:    v_or_b32_e32 v13, v19, v13
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v25, v14
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v19
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
+; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT:    v_or_b32_e32 v15, v21, v15
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x300, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x3000000, v15
 ; SI-NEXT:  .LBB26_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -11955,7 +11959,7 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i8_to_v16i32:
@@ -11993,38 +11997,31 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; VI-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; VI-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; VI-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; VI-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; VI-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -12035,38 +12032,52 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; VI-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
-; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
+; VI-NEXT:    s_waitcnt vmcnt(9)
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
+; VI-NEXT:    s_waitcnt vmcnt(8)
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; VI-NEXT:    s_waitcnt vmcnt(7)
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; VI-NEXT:    s_waitcnt vmcnt(6)
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; VI-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -12077,28 +12088,28 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr47
-; VI-NEXT:    ; implicit-def: $vgpr42
-; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr57
 ; VI-NEXT:    ; implicit-def: $vgpr46
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr27
-; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr21
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -12125,18 +12136,18 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr17
 ; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr38
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -12166,23 +12177,23 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
@@ -12216,24 +12227,24 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; kill: killed $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    ; implicit-def: $vgpr52
-; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr49
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr48
-; VI-NEXT:    ; implicit-def: $vgpr44
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr25
-; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr27
 ; VI-NEXT:  .LBB26_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB26_4
@@ -12244,27 +12255,27 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v15, 0x300
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v51
-; VI-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_add_u16_e32 v10, 3, v24
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
 ; VI-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v11, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v23
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v12, 3, v60
-; VI-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_add_u16_e32 v12, 3, v38
+; VI-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
-; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v58
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v14, 3, v42
-; VI-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v14, 3, v45
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
-; VI-NEXT:    v_or_b32_sdwa v17, v57, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v17, v60, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -12300,18 +12311,18 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
-; VI-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v53
-; VI-NEXT:    v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -12350,39 +12361,39 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v52
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
 ; VI-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
 ; VI-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
-; VI-NEXT:    v_add_u16_e32 v10, 3, v49
-; VI-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v10, 3, v29
+; VI-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v63
-; VI-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v48
-; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v12, 3, v61
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; VI-NEXT:    v_or_b32_e32 v12, v12, v13
-; VI-NEXT:    v_add_u16_e32 v13, 3, v44
-; VI-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_e32 v14, 3, v41
 ; VI-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v16
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v19
-; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    v_or_b32_e32 v15, v16, v15
 ; VI-NEXT:  .LBB26_4: ; %end
@@ -12441,39 +12452,32 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; GFX9-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; GFX9-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; GFX9-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; GFX9-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -12483,50 +12487,57 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(20)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(17)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
+; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
+; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
+; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; GFX9-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -12536,29 +12547,29 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr51
-; GFX9-NEXT:    ; implicit-def: $vgpr24
-; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr60
-; GFX9-NEXT:    ; implicit-def: $vgpr47
-; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr38
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
-; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr54
-; GFX9-NEXT:    ; implicit-def: $vgpr27
-; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -12585,18 +12596,18 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr20
-; GFX9-NEXT:    ; implicit-def: $vgpr26
-; GFX9-NEXT:    ; implicit-def: $vgpr38
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -12626,23 +12637,23 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
@@ -12676,24 +12687,24 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; kill: killed $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    ; implicit-def: $vgpr52
-; GFX9-NEXT:    ; implicit-def: $vgpr50
-; GFX9-NEXT:    ; implicit-def: $vgpr49
-; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr44
-; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
-; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr25
-; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr27
 ; GFX9-NEXT:  .LBB26_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB26_4
@@ -12703,28 +12714,28 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v51
-; GFX9-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v40
+; GFX9-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v24
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v30
-; GFX9-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v23
+; GFX9-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v60
-; GFX9-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
-; GFX9-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v58
+; GFX9-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v13, v13, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
-; GFX9-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v45
+; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v53
-; GFX9-NEXT:    v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
@@ -12761,18 +12772,18 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
-; GFX9-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v17
-; GFX9-NEXT:    v_or_b32_sdwa v16, v57, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v16, v60, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -12810,39 +12821,39 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v52
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v43
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v50
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v52
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
-; GFX9-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v29
+; GFX9-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v63
-; GFX9-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v26
+; GFX9-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v48
-; GFX9-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v61
+; GFX9-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_or_b32_e32 v12, v12, v13
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v44
-; GFX9-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v41
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v19
-; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
 ; GFX9-NEXT:  .LBB26_4: ; %end
@@ -15124,42 +15135,34 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v16i32_scalar:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v84, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v32, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v13 :: v_dual_mov_b32 v34, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v12 :: v_dual_mov_b32 v36, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v37, v9 :: v_dual_mov_b32 v38, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v8 :: v_dual_mov_b32 v48, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v4 :: v_dual_mov_b32 v52, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB27_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -15196,37 +15199,37 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_and_b32 s8, s9, 0xffff
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v54
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v49
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v52
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s11, s12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v37
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v34
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v70
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v39
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v7
@@ -15267,28 +15270,28 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v13, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v82
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v86
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v34
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v64
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v14, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v86, v87
@@ -15355,7 +15358,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s7, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s7, s6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v54
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s2, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
@@ -15367,14 +15370,14 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s2, s2, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v83
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v49
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
@@ -15387,31 +15390,31 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v71
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v68
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v66
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v35
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v64
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v55
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
@@ -15461,42 +15464,42 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v28
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v29
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v83
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v12, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v37
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v35
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v69
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v67
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v12, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v80
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v64
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v12, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v17, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v19, v15
@@ -26751,170 +26754,171 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v34, v6
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:132
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:128
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 8, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v28, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v30, 8, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v49, 8, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 24, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v17
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 24, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 8, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 8, v17
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v53
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 24, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 24, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 24, v12
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v39
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 24, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v50
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v54
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 24, v14
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v45
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v50
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 24, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v51
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 8, v58
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 8, v52
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 24, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v53
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v55
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v40
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v42
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 24, v58
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 8, v13
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v33
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v34
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v49
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v2, v2, v36
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
+; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v42
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v52
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v58
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v44
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -26922,18 +26926,18 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v48
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
@@ -26942,7 +26946,6 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v24, v6
-; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -26950,30 +26953,30 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v3, v16
+; SI-NEXT:    v_or_b32_e32 v3, v3, v38
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v17
+; SI-NEXT:    v_or_b32_e32 v4, v4, v18
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v22
@@ -26993,54 +26996,59 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v30
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v43
-; SI-NEXT:    v_or_b32_e32 v8, v8, v62
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v60
+; SI-NEXT:    v_or_b32_e32 v8, v8, v63
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v51
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v45
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v59
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr62
-; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v10, v47
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v55
+; SI-NEXT:    v_or_b32_e32 v10, v10, v46
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v58
-; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v11, v11, v41
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v12, v40
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v12, v12, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v41
-; SI-NEXT:    v_or_b32_e32 v13, v13, v50
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v40
+; SI-NEXT:    v_or_b32_e32 v13, v13, v19
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v23
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v25
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v19
-; SI-NEXT:    v_or_b32_e32 v15, v15, v21
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    ; implicit-def: $vgpr25
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_or_b32_e32 v15, v15, v16
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
@@ -27067,21 +27075,20 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    ; implicit-def: $vgpr21
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:  .LBB50_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v32
@@ -27089,55 +27096,54 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v34
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
+; SI-NEXT:    v_or_b32_e32 v1, v39, v1
 ; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v37, v2
+; SI-NEXT:    v_or_b32_e32 v2, v35, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v42
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v58
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v44
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v53
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -27146,20 +27152,20 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_or_b32_e32 v2, v48, v2
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -27175,35 +27181,34 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v38, v3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s7, v3
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v17, v4
+; SI-NEXT:    v_or_b32_e32 v4, v18, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s7, v4
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -27229,68 +27234,70 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v43
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v60
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v62, v8
+; SI-NEXT:    v_or_b32_e32 v8, v63, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v51
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v59, v9
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v55
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v47, v10
+; SI-NEXT:    v_or_b32_e32 v10, v46, v10
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v58
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v11, v41, v11
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v47
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v40, v12
+; SI-NEXT:    v_or_b32_e32 v12, v51, v12
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v41
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v40
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_or_b32_e32 v13, v50, v13
+; SI-NEXT:    v_or_b32_e32 v13, v19, v13
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v25, v14
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v19
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
+; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT:    v_or_b32_e32 v15, v21, v15
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x300, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x3000000, v15
 ; SI-NEXT:  .LBB50_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -27310,7 +27317,7 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i8_to_v16f32:
@@ -27348,38 +27355,31 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; VI-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; VI-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; VI-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; VI-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; VI-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -27390,38 +27390,52 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; VI-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
-; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
+; VI-NEXT:    s_waitcnt vmcnt(9)
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
+; VI-NEXT:    s_waitcnt vmcnt(8)
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; VI-NEXT:    s_waitcnt vmcnt(7)
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; VI-NEXT:    s_waitcnt vmcnt(6)
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; VI-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -27432,28 +27446,28 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr47
-; VI-NEXT:    ; implicit-def: $vgpr42
-; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr57
 ; VI-NEXT:    ; implicit-def: $vgpr46
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr27
-; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr21
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -27480,18 +27494,18 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr17
 ; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr38
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -27521,23 +27535,23 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
@@ -27571,24 +27585,24 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; kill: killed $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    ; implicit-def: $vgpr52
-; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr49
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr48
-; VI-NEXT:    ; implicit-def: $vgpr44
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr25
-; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr27
 ; VI-NEXT:  .LBB50_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB50_4
@@ -27599,27 +27613,27 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v15, 0x300
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v51
-; VI-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_add_u16_e32 v10, 3, v24
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
 ; VI-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v11, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v23
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v12, 3, v60
-; VI-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_add_u16_e32 v12, 3, v38
+; VI-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
-; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v58
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v14, 3, v42
-; VI-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v14, 3, v45
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
-; VI-NEXT:    v_or_b32_sdwa v17, v57, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v17, v60, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -27655,18 +27669,18 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
-; VI-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v53
-; VI-NEXT:    v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -27705,39 +27719,39 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v52
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
 ; VI-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
 ; VI-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
-; VI-NEXT:    v_add_u16_e32 v10, 3, v49
-; VI-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v10, 3, v29
+; VI-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v63
-; VI-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v48
-; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v12, 3, v61
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; VI-NEXT:    v_or_b32_e32 v12, v12, v13
-; VI-NEXT:    v_add_u16_e32 v13, 3, v44
-; VI-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_e32 v14, 3, v41
 ; VI-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v16
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v19
-; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    v_or_b32_e32 v15, v16, v15
 ; VI-NEXT:  .LBB50_4: ; %end
@@ -27796,39 +27810,32 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; GFX9-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; GFX9-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; GFX9-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; GFX9-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -27838,50 +27845,57 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(20)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(17)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
+; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
+; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
+; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; GFX9-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -27891,29 +27905,29 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr51
-; GFX9-NEXT:    ; implicit-def: $vgpr24
-; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr60
-; GFX9-NEXT:    ; implicit-def: $vgpr47
-; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr38
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
-; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr54
-; GFX9-NEXT:    ; implicit-def: $vgpr27
-; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -27940,18 +27954,18 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr20
-; GFX9-NEXT:    ; implicit-def: $vgpr26
-; GFX9-NEXT:    ; implicit-def: $vgpr38
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -27981,23 +27995,23 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
@@ -28031,24 +28045,24 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; kill: killed $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    ; implicit-def: $vgpr52
-; GFX9-NEXT:    ; implicit-def: $vgpr50
-; GFX9-NEXT:    ; implicit-def: $vgpr49
-; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr44
-; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
-; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr25
-; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr27
 ; GFX9-NEXT:  .LBB50_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB50_4
@@ -28058,28 +28072,28 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v51
-; GFX9-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v40
+; GFX9-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v24
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v30
-; GFX9-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v23
+; GFX9-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v60
-; GFX9-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
-; GFX9-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v58
+; GFX9-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v13, v13, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
-; GFX9-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v45
+; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v53
-; GFX9-NEXT:    v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
@@ -28116,18 +28130,18 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
-; GFX9-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v17
-; GFX9-NEXT:    v_or_b32_sdwa v16, v57, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v16, v60, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -28165,39 +28179,39 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v52
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v43
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v50
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v52
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
-; GFX9-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v29
+; GFX9-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v63
-; GFX9-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v26
+; GFX9-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v48
-; GFX9-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v61
+; GFX9-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_or_b32_e32 v12, v12, v13
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v44
-; GFX9-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v41
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v19
-; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
 ; GFX9-NEXT:  .LBB50_4: ; %end
@@ -30479,42 +30493,34 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
 ; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v16f32_scalar:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v84, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v32, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v13 :: v_dual_mov_b32 v34, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v12 :: v_dual_mov_b32 v36, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v37, v9 :: v_dual_mov_b32 v38, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v8 :: v_dual_mov_b32 v48, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v4 :: v_dual_mov_b32 v52, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB51_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -30551,37 +30557,37 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_and_b32 s8, s9, 0xffff
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v54
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v49
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v52
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s11, s12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v37
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v34
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v70
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v39
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v7
@@ -30622,28 +30628,28 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v13, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v82
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v86
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v34
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v64
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v14, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v86, v87
@@ -30710,7 +30716,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s7, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s7, s6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v54
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s2, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
@@ -30722,14 +30728,14 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s2, s2, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v83
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v49
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
@@ -30742,31 +30748,31 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v71
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v68
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v66
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v35
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v64
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v55
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
@@ -30816,42 +30822,42 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v28
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v29
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v83
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v12, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v37
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v35
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v69
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v67
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v12, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v80
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v64
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v12, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v17, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v19, v15
@@ -41377,170 +41383,171 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v34, v6
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:132
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:128
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 8, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v28, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v30, 8, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v49, 8, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 24, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v17
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 24, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 8, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 8, v17
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v53
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 24, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 24, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 24, v12
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v39
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 24, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v50
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v54
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 24, v14
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v45
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v50
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 24, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v51
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 8, v58
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 8, v52
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 24, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v53
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v55
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v40
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v42
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 24, v58
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 8, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB70_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v33
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v34
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v49
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v2, v2, v36
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
+; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v42
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v52
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v58
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v44
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -41548,18 +41555,18 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v48
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
@@ -41568,7 +41575,6 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v24, v6
-; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -41576,30 +41582,30 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v3, v16
+; SI-NEXT:    v_or_b32_e32 v3, v3, v38
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v17
+; SI-NEXT:    v_or_b32_e32 v4, v4, v18
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v22
@@ -41619,54 +41625,59 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v30
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v43
-; SI-NEXT:    v_or_b32_e32 v8, v8, v62
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v60
+; SI-NEXT:    v_or_b32_e32 v8, v8, v63
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v51
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v45
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v59
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr62
-; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v10, v47
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v55
+; SI-NEXT:    v_or_b32_e32 v10, v10, v46
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v58
-; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v11, v11, v41
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v12, v40
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v12, v12, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v41
-; SI-NEXT:    v_or_b32_e32 v13, v13, v50
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v40
+; SI-NEXT:    v_or_b32_e32 v13, v13, v19
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v23
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v25
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v19
-; SI-NEXT:    v_or_b32_e32 v15, v15, v21
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    ; implicit-def: $vgpr25
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_or_b32_e32 v15, v15, v16
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
@@ -41693,21 +41704,20 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    ; implicit-def: $vgpr21
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:  .LBB70_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB70_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v32
@@ -41715,55 +41725,54 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v34
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
+; SI-NEXT:    v_or_b32_e32 v1, v39, v1
 ; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v37, v2
+; SI-NEXT:    v_or_b32_e32 v2, v35, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v42
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v58
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v44
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v53
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -41772,20 +41781,20 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_or_b32_e32 v2, v48, v2
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -41801,35 +41810,34 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v38, v3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s7, v3
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v17, v4
+; SI-NEXT:    v_or_b32_e32 v4, v18, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s7, v4
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -41855,68 +41863,70 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v43
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v60
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v62, v8
+; SI-NEXT:    v_or_b32_e32 v8, v63, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v51
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v59, v9
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v55
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v47, v10
+; SI-NEXT:    v_or_b32_e32 v10, v46, v10
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v58
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v11, v41, v11
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v47
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v40, v12
+; SI-NEXT:    v_or_b32_e32 v12, v51, v12
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v41
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v40
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_or_b32_e32 v13, v50, v13
+; SI-NEXT:    v_or_b32_e32 v13, v19, v13
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v25, v14
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v19
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
+; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT:    v_or_b32_e32 v15, v21, v15
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x300, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x3000000, v15
 ; SI-NEXT:  .LBB70_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -41936,7 +41946,7 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i8_to_v8i64:
@@ -41974,38 +41984,31 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; VI-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; VI-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; VI-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; VI-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; VI-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -42016,38 +42019,52 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; VI-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
-; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
+; VI-NEXT:    s_waitcnt vmcnt(9)
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
+; VI-NEXT:    s_waitcnt vmcnt(8)
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; VI-NEXT:    s_waitcnt vmcnt(7)
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; VI-NEXT:    s_waitcnt vmcnt(6)
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; VI-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -42058,28 +42075,28 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr47
-; VI-NEXT:    ; implicit-def: $vgpr42
-; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr57
 ; VI-NEXT:    ; implicit-def: $vgpr46
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr27
-; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr21
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -42106,18 +42123,18 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr17
 ; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr38
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -42147,23 +42164,23 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
@@ -42197,24 +42214,24 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; kill: killed $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    ; implicit-def: $vgpr52
-; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr49
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr48
-; VI-NEXT:    ; implicit-def: $vgpr44
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr25
-; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr27
 ; VI-NEXT:  .LBB70_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB70_4
@@ -42225,27 +42242,27 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v15, 0x300
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v51
-; VI-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_add_u16_e32 v10, 3, v24
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
 ; VI-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v11, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v23
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v12, 3, v60
-; VI-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_add_u16_e32 v12, 3, v38
+; VI-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
-; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v58
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v14, 3, v42
-; VI-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v14, 3, v45
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
-; VI-NEXT:    v_or_b32_sdwa v17, v57, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v17, v60, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -42281,18 +42298,18 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
-; VI-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v53
-; VI-NEXT:    v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -42331,39 +42348,39 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v52
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
 ; VI-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
 ; VI-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
-; VI-NEXT:    v_add_u16_e32 v10, 3, v49
-; VI-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v10, 3, v29
+; VI-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v63
-; VI-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v48
-; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v12, 3, v61
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; VI-NEXT:    v_or_b32_e32 v12, v12, v13
-; VI-NEXT:    v_add_u16_e32 v13, 3, v44
-; VI-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_e32 v14, 3, v41
 ; VI-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v16
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v19
-; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    v_or_b32_e32 v15, v16, v15
 ; VI-NEXT:  .LBB70_4: ; %end
@@ -42422,39 +42439,32 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; GFX9-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; GFX9-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; GFX9-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; GFX9-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -42464,50 +42474,57 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(20)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(17)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
+; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
+; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
+; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; GFX9-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -42517,29 +42534,29 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr51
-; GFX9-NEXT:    ; implicit-def: $vgpr24
-; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr60
-; GFX9-NEXT:    ; implicit-def: $vgpr47
-; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr38
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
-; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr54
-; GFX9-NEXT:    ; implicit-def: $vgpr27
-; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -42566,18 +42583,18 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr20
-; GFX9-NEXT:    ; implicit-def: $vgpr26
-; GFX9-NEXT:    ; implicit-def: $vgpr38
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -42607,23 +42624,23 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
@@ -42657,24 +42674,24 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; kill: killed $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    ; implicit-def: $vgpr52
-; GFX9-NEXT:    ; implicit-def: $vgpr50
-; GFX9-NEXT:    ; implicit-def: $vgpr49
-; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr44
-; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
-; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr25
-; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr27
 ; GFX9-NEXT:  .LBB70_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB70_4
@@ -42684,28 +42701,28 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v51
-; GFX9-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v40
+; GFX9-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v24
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v30
-; GFX9-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v23
+; GFX9-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v60
-; GFX9-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
-; GFX9-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v58
+; GFX9-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v13, v13, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
-; GFX9-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v45
+; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v53
-; GFX9-NEXT:    v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
@@ -42742,18 +42759,18 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
-; GFX9-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v17
-; GFX9-NEXT:    v_or_b32_sdwa v16, v57, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v16, v60, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -42791,39 +42808,39 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v52
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v43
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v50
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v52
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
-; GFX9-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v29
+; GFX9-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v63
-; GFX9-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v26
+; GFX9-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v48
-; GFX9-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v61
+; GFX9-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_or_b32_e32 v12, v12, v13
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v44
-; GFX9-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v41
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v19
-; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
 ; GFX9-NEXT:  .LBB70_4: ; %end
@@ -45105,42 +45122,34 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v8i64_scalar:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v84, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v32, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v13 :: v_dual_mov_b32 v34, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v12 :: v_dual_mov_b32 v36, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v37, v9 :: v_dual_mov_b32 v38, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v8 :: v_dual_mov_b32 v48, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v4 :: v_dual_mov_b32 v52, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB71_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -45177,37 +45186,37 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_and_b32 s8, s9, 0xffff
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v54
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v49
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v52
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s11, s12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v37
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v34
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v70
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v39
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v7
@@ -45248,28 +45257,28 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v13, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v82
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v86
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v34
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v64
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v14, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v86, v87
@@ -45336,7 +45345,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s7, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s7, s6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v54
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s2, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
@@ -45348,14 +45357,14 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    s_or_b32 s2, s2, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v83
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v49
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
@@ -45368,31 +45377,31 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v71
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v68
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v66
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v35
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v64
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v55
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
@@ -45442,42 +45451,42 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v28
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v29
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v83
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v12, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v37
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v35
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v69
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v67
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v12, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v80
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v64
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v12, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v17, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v19, v15
@@ -55157,170 +55166,171 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v34, v6
 ; SI-NEXT:    v_mov_b32_e32 v33, v4
 ; SI-NEXT:    v_mov_b32_e32 v32, v2
 ; SI-NEXT:    v_mov_b32_e32 v31, v0
-; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:132
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:128
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 8, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v28, 24, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v30, 8, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v49, 8, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 8, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 24, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v17
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 24, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 8, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 24, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 8, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 24, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 8, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 8, v17
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v53
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 24, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 24, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 24, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 24, v12
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v39
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 24, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 24, v6
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v50
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 24, v52
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v10
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 8, v54
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v12
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 24, v14
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v45
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v50
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 24, v56
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 24, v51
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v21, 8, v58
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 8, v52
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 24, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 24, v53
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v19, 8, v55
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v27, 24, v40
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:68
+; SI-NEXT:    v_lshlrev_b32_e32 v25, 8, v42
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 24, v58
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 8, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v47
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:52
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB86_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v31
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v32
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v33
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v34
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v49
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v2, v2, v36
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
+; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v3
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v42
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v52
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v58
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v44
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v53
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr29
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr46
 ; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    ; implicit-def: $vgpr49
+; SI-NEXT:    ; implicit-def: $vgpr48
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -55328,18 +55338,18 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v2, v2, v48
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
@@ -55348,7 +55358,6 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v24, v6
-; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -55356,30 +55365,30 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v3, v16
+; SI-NEXT:    v_or_b32_e32 v3, v3, v38
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v4, v17
+; SI-NEXT:    v_or_b32_e32 v4, v4, v18
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v22
@@ -55399,54 +55408,59 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v30
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v8
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v43
-; SI-NEXT:    v_or_b32_e32 v8, v8, v62
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v60
+; SI-NEXT:    v_or_b32_e32 v8, v8, v63
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v9
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v51
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v45
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v59
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v10
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr62
-; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v10, v47
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v55
+; SI-NEXT:    v_or_b32_e32 v10, v10, v46
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v58
-; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v61
+; SI-NEXT:    v_or_b32_e32 v11, v11, v41
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v12, v40
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v12, v12, v51
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v13
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v41
-; SI-NEXT:    v_or_b32_e32 v13, v13, v50
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v40
+; SI-NEXT:    v_or_b32_e32 v13, v13, v19
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v14
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v23
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v25
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v15
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v19
-; SI-NEXT:    v_or_b32_e32 v15, v15, v21
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    ; implicit-def: $vgpr25
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_or_b32_e32 v15, v15, v16
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v16
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
@@ -55473,21 +55487,20 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr16
 ; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    ; kill: killed $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    ; implicit-def: $vgpr21
+; SI-NEXT:    ; kill: killed $vgpr50
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:  .LBB86_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB86_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v31
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v32
@@ -55495,55 +55508,54 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x300, v0
-; SI-NEXT:    v_or_b32_e32 v1, v35, v1
+; SI-NEXT:    v_or_b32_e32 v1, v48, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v33
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v34
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
+; SI-NEXT:    v_or_b32_e32 v1, v39, v1
 ; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v37, v2
+; SI-NEXT:    v_or_b32_e32 v2, v35, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v57
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v9, v60, v9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v39
+; SI-NEXT:    v_or_b32_e32 v9, v62, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v42
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
+; SI-NEXT:    v_or_b32_e32 v10, v56, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v52
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    v_or_b32_e32 v11, v46, v11
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v56
+; SI-NEXT:    v_or_b32_e32 v11, v43, v11
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v58
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v12, v42, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v45
+; SI-NEXT:    v_or_b32_e32 v12, v54, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v44
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_or_b32_e32 v13, v53, v13
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v54
+; SI-NEXT:    v_or_b32_e32 v13, v21, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v53
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v27, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v29
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT:    v_or_b32_e32 v15, v23, v15
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -55552,20 +55564,20 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_or_b32_e32 v2, v48, v2
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT:    v_or_b32_e32 v3, v38, v3
+; SI-NEXT:    v_or_b32_e32 v3, v37, v3
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v18, v4
+; SI-NEXT:    v_or_b32_e32 v4, v16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -55581,35 +55593,34 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v3, v16, v3
+; SI-NEXT:    v_or_b32_e32 v3, v38, v3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s7, v3
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v17, v8
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v4, v17, v4
+; SI-NEXT:    v_or_b32_e32 v4, v18, v4
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s7, v4
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v8, v63, v8
-; SI-NEXT:    v_or_b32_e32 v16, v52, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
@@ -55635,68 +55646,70 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v43
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v60
 ; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v8, v62, v8
+; SI-NEXT:    v_or_b32_e32 v8, v63, v8
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v51
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v59, v9
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v55
 ; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
-; SI-NEXT:    v_or_b32_e32 v10, v47, v10
+; SI-NEXT:    v_or_b32_e32 v10, v46, v10
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v58
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v11, v41, v11
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v12, v11
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
-; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
+; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v47
 ; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_or_b32_e32 v12, v40, v12
+; SI-NEXT:    v_or_b32_e32 v12, v51, v12
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v41
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v40
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_or_b32_e32 v13, v50, v13
+; SI-NEXT:    v_or_b32_e32 v13, v19, v13
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v13, v14, v13
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
-; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v25, v14
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v14, v15, v14
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v19
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s7, v7
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v8
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v9
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v10
+; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v11
+; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v12
+; SI-NEXT:    v_add_i32_e32 v13, vcc, s7, v13
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT:    v_or_b32_e32 v15, v21, v15
+; SI-NEXT:    v_or_b32_e32 v15, v16, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x300, v15
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT:    v_or_b32_e32 v16, v50, v16
 ; SI-NEXT:    v_or_b32_e32 v15, v16, v15
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 0x3000000, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 0x3000000, v15
 ; SI-NEXT:  .LBB86_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -55716,7 +55729,7 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: bitcast_v64i8_to_v8f64:
@@ -55754,38 +55767,31 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; VI-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; VI-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; VI-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; VI-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; VI-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; VI-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -55796,38 +55802,52 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; VI-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
-; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; VI-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; VI-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; VI-NEXT:    s_waitcnt vmcnt(7)
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; VI-NEXT:    s_waitcnt vmcnt(6)
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; VI-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
+; VI-NEXT:    s_waitcnt vmcnt(9)
+; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
+; VI-NEXT:    s_waitcnt vmcnt(8)
+; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; VI-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; VI-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; VI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; VI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -55838,28 +55858,28 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr47
-; VI-NEXT:    ; implicit-def: $vgpr42
-; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr49
+; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    ; implicit-def: $vgpr58
+; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    ; implicit-def: $vgpr53
+; VI-NEXT:    ; implicit-def: $vgpr57
 ; VI-NEXT:    ; implicit-def: $vgpr46
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr40
+; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr27
-; VI-NEXT:    ; implicit-def: $vgpr23
+; VI-NEXT:    ; implicit-def: $vgpr50
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr21
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -55886,18 +55906,18 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr17
 ; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr38
-; VI-NEXT:    ; implicit-def: $vgpr57
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr30
+; VI-NEXT:    ; implicit-def: $vgpr60
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr62
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -55927,23 +55947,23 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr28
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
@@ -55977,24 +55997,24 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; kill: killed $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    ; implicit-def: $vgpr52
-; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr49
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr48
-; VI-NEXT:    ; implicit-def: $vgpr44
-; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr47
+; VI-NEXT:    ; implicit-def: $vgpr41
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; implicit-def: $vgpr63
 ; VI-NEXT:    ; implicit-def: $vgpr59
 ; VI-NEXT:    ; implicit-def: $vgpr56
-; VI-NEXT:    ; implicit-def: $vgpr45
-; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr44
 ; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr48
 ; VI-NEXT:    ; implicit-def: $vgpr25
-; VI-NEXT:    ; implicit-def: $vgpr21
+; VI-NEXT:    ; implicit-def: $vgpr27
 ; VI-NEXT:  .LBB86_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB86_4
@@ -56005,27 +56025,27 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    v_mov_b32_e32 v15, 0x300
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_add_u16_e32 v9, 3, v51
-; VI-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v9, 3, v40
+; VI-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v9, v9, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_add_u16_e32 v10, 3, v24
+; VI-NEXT:    v_add_u16_e32 v10, 3, v49
 ; VI-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v10, v10, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v11, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v23
+; VI-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v11, v11, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v12, 3, v60
-; VI-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_add_u16_e32 v12, 3, v38
+; VI-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v12, v12, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v13, 3, v47
-; VI-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v58
+; VI-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v13, v13, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v14, 3, v42
-; VI-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v14, 3, v45
+; VI-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v14, v14, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v17, 3, v17
-; VI-NEXT:    v_or_b32_sdwa v17, v57, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v17, v60, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -56061,18 +56081,18 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_add_u16_sdwa v5, v5, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_add_u16_e32 v6, 3, v6
-; VI-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v6, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v53
-; VI-NEXT:    v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v16, v16, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_add_u16_e32 v8, 3, v8
-; VI-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v8, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -56111,39 +56131,39 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v7, 3, v7
-; VI-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; VI-NEXT:    v_or_b32_e32 v7, v7, v8
-; VI-NEXT:    v_add_u16_e32 v8, 3, v52
+; VI-NEXT:    v_add_u16_e32 v8, 3, v43
 ; VI-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; VI-NEXT:    v_or_b32_e32 v8, v8, v9
-; VI-NEXT:    v_add_u16_e32 v9, 3, v50
+; VI-NEXT:    v_add_u16_e32 v9, 3, v52
 ; VI-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; VI-NEXT:    v_or_b32_e32 v9, v9, v10
-; VI-NEXT:    v_add_u16_e32 v10, 3, v49
-; VI-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v10, 3, v29
+; VI-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; VI-NEXT:    v_or_b32_e32 v10, v10, v11
-; VI-NEXT:    v_add_u16_e32 v11, 3, v63
-; VI-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v11, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; VI-NEXT:    v_or_b32_e32 v11, v11, v12
-; VI-NEXT:    v_add_u16_e32 v12, 3, v48
-; VI-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v12, 3, v61
+; VI-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; VI-NEXT:    v_or_b32_e32 v12, v12, v13
-; VI-NEXT:    v_add_u16_e32 v13, 3, v44
-; VI-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v13, 3, v47
+; VI-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; VI-NEXT:    v_or_b32_e32 v13, v13, v14
-; VI-NEXT:    v_add_u16_e32 v14, 3, v39
+; VI-NEXT:    v_add_u16_e32 v14, 3, v41
 ; VI-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_or_b32_e32 v14, v14, v16
 ; VI-NEXT:    v_add_u16_e32 v16, 3, v19
-; VI-NEXT:    v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
 ; VI-NEXT:    v_or_b32_e32 v15, v16, v15
 ; VI-NEXT:  .LBB86_4: ; %end
@@ -56202,39 +56222,32 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:88
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:96
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    buffer_load_ushort v57, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v50, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:88
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:96
+; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v21
-; GFX9-NEXT:    v_lshlrev_b16_e32 v26, 8, v23
+; GFX9-NEXT:    v_lshlrev_b16_e32 v24, 8, v23
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v28, 8, v25
-; GFX9-NEXT:    v_lshlrev_b16_e32 v38, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v29
+; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v27
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v29
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v18, 8, v17
-; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v19
-; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; GFX9-NEXT:    buffer_load_ushort v17, off, s[0:3], s32 offset:124
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v31, 8, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v32, 8, v3
@@ -56244,50 +56257,57 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v11
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v37, 8, v13
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v16, 8, v15
-; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v54
-; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v2
-; GFX9-NEXT:    s_waitcnt vmcnt(22)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v4
-; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; GFX9-NEXT:    s_waitcnt vmcnt(20)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v8
-; GFX9-NEXT:    s_waitcnt vmcnt(19)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v12
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(17)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v14
+; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v24
+; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v30
+; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v53
+; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v42
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v44
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v47
+; GFX9-NEXT:    v_lshlrev_b16_e32 v42, 8, v14
 ; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v57
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v26
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v38
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v39
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v48
+; GFX9-NEXT:    s_waitcnt vmcnt(6)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v49
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v52
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v53
+; GFX9-NEXT:    s_waitcnt vmcnt(3)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v40
+; GFX9-NEXT:    buffer_load_ushort v23, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v40, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v19, off, s[0:3], s32 offset:116
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v60
+; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v41
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v63
+; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v45
 ; GFX9-NEXT:    buffer_load_ushort v53, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v60, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_ushort v45, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_ushort v58, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -56297,29 +56317,29 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v51, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v24, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v30, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v60, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v42, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr51
-; GFX9-NEXT:    ; implicit-def: $vgpr24
-; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr60
-; GFX9-NEXT:    ; implicit-def: $vgpr47
-; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_or_b32_sdwa v9, v40, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v23, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_or_b32_sdwa v12, v38, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v58, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v45, v39 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v53, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr38
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
+; GFX9-NEXT:    ; implicit-def: $vgpr45
+; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
-; GFX9-NEXT:    ; implicit-def: $vgpr43
-; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr42
 ; GFX9-NEXT:    ; implicit-def: $vgpr54
-; GFX9-NEXT:    ; implicit-def: $vgpr27
-; GFX9-NEXT:    ; implicit-def: $vgpr23
+; GFX9-NEXT:    ; implicit-def: $vgpr50
+; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
@@ -56346,18 +56366,18 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v26 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v30 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v16, v17, v60 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr20
-; GFX9-NEXT:    ; implicit-def: $vgpr26
-; GFX9-NEXT:    ; implicit-def: $vgpr38
-; GFX9-NEXT:    ; implicit-def: $vgpr57
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr30
+; GFX9-NEXT:    ; implicit-def: $vgpr60
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr62
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -56387,23 +56407,23 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr28
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v52, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v43, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v9, v50, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v52, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v10, v49, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v29, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v63, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v26, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v48, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v61, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v44, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v47, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v41, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v19, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
@@ -56437,24 +56457,24 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; kill: killed $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    ; implicit-def: $vgpr52
-; GFX9-NEXT:    ; implicit-def: $vgpr50
-; GFX9-NEXT:    ; implicit-def: $vgpr49
-; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr44
-; GFX9-NEXT:    ; implicit-def: $vgpr39
+; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr47
+; GFX9-NEXT:    ; implicit-def: $vgpr41
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
-; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
-; GFX9-NEXT:    ; implicit-def: $vgpr45
-; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr51
+; GFX9-NEXT:    ; implicit-def: $vgpr48
 ; GFX9-NEXT:    ; implicit-def: $vgpr25
-; GFX9-NEXT:    ; implicit-def: $vgpr21
+; GFX9-NEXT:    ; implicit-def: $vgpr27
 ; GFX9-NEXT:  .LBB86_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB86_4
@@ -56464,28 +56484,28 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_movk_i32 s6, 0x300
-; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v51
-; GFX9-NEXT:    v_or_b32_sdwa v9, v58, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(14)
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v40
+; GFX9-NEXT:    v_or_b32_sdwa v9, v57, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v9, v9, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v24
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
 ; GFX9-NEXT:    v_or_b32_sdwa v10, v46, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v10, v10, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v30
-; GFX9-NEXT:    v_or_b32_sdwa v11, v43, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v23
+; GFX9-NEXT:    v_or_b32_sdwa v11, v42, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v11, v11, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v60
-; GFX9-NEXT:    v_or_b32_sdwa v12, v40, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v12, v54, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v12, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
-; GFX9-NEXT:    v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v58
+; GFX9-NEXT:    v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v13, v13, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v42
-; GFX9-NEXT:    v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v45
+; GFX9-NEXT:    v_or_b32_sdwa v14, v39, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v14, v14, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v53
-; GFX9-NEXT:    v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v15, v15, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
@@ -56522,18 +56542,18 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_add_u16_sdwa v5, v5, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_add_u16_e32 v6, 3, v6
-; GFX9-NEXT:    v_or_b32_sdwa v6, v26, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v6, v24, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v6, v6, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v38, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v30, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v7, v7, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_add_u16_e32 v16, 3, v17
-; GFX9-NEXT:    v_or_b32_sdwa v16, v57, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v16, v60, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v16, v16, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v8, 3, v8
-; GFX9-NEXT:    v_or_b32_sdwa v8, v61, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v8, v62, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_sdwa v8, v8, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v2, 3, v2
@@ -56571,39 +56591,39 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v7, 3, v7
-; GFX9-NEXT:    v_or_b32_sdwa v7, v62, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v7, v63, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v7
 ; GFX9-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX9-NEXT:    v_add_u16_e32 v8, 3, v52
+; GFX9-NEXT:    v_add_u16_e32 v8, 3, v43
 ; GFX9-NEXT:    v_or_b32_sdwa v8, v59, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v8
 ; GFX9-NEXT:    v_or_b32_e32 v8, v8, v9
-; GFX9-NEXT:    v_add_u16_e32 v9, 3, v50
+; GFX9-NEXT:    v_add_u16_e32 v9, 3, v52
 ; GFX9-NEXT:    v_or_b32_sdwa v9, v56, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v9
 ; GFX9-NEXT:    v_or_b32_e32 v9, v9, v10
-; GFX9-NEXT:    v_add_u16_e32 v10, 3, v49
-; GFX9-NEXT:    v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v10, 3, v29
+; GFX9-NEXT:    v_or_b32_sdwa v10, v44, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v10
 ; GFX9-NEXT:    v_or_b32_e32 v10, v10, v11
-; GFX9-NEXT:    v_add_u16_e32 v11, 3, v63
-; GFX9-NEXT:    v_or_b32_sdwa v11, v41, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v11, 3, v26
+; GFX9-NEXT:    v_or_b32_sdwa v11, v55, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v11
 ; GFX9-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX9-NEXT:    v_add_u16_e32 v12, 3, v48
-; GFX9-NEXT:    v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v12, 3, v61
+; GFX9-NEXT:    v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v12
 ; GFX9-NEXT:    v_or_b32_e32 v12, v12, v13
-; GFX9-NEXT:    v_add_u16_e32 v13, 3, v44
-; GFX9-NEXT:    v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v13, 3, v47
+; GFX9-NEXT:    v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v13
 ; GFX9-NEXT:    v_or_b32_e32 v13, v13, v14
-; GFX9-NEXT:    v_add_u16_e32 v14, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v14, 3, v41
 ; GFX9-NEXT:    v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; GFX9-NEXT:    v_or_b32_e32 v14, v14, v15
 ; GFX9-NEXT:    v_add_u16_e32 v15, 3, v19
-; GFX9-NEXT:    v_or_b32_sdwa v15, v21, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v15
 ; GFX9-NEXT:    v_or_b32_e32 v15, v15, v16
 ; GFX9-NEXT:  .LBB86_4: ; %end
@@ -58885,42 +58905,34 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v8f64_scalar:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v84, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v32, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v13 :: v_dual_mov_b32 v34, v11
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v12 :: v_dual_mov_b32 v36, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v37, v9 :: v_dual_mov_b32 v38, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v39, v8 :: v_dual_mov_b32 v48, v6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v3
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v4 :: v_dual_mov_b32 v52, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB87_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -58957,37 +58969,37 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_and_b32 s8, s9, 0xffff
 ; GFX11-TRUE16-NEXT:    s_and_b32 s9, s26, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s10, s27, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v54
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s9, s10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v49
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v52
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    s_and_b32 s11, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s12, s29, 8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 s10, s11, s12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v37
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s10, 0xffff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v34
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s10, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v69
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v70
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v66
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v64
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v39
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xff, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v33
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v7
@@ -59028,28 +59040,28 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v29
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v13, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v50
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v82
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v86
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v37
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v34
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v31
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v80
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v64
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v98, 0xff, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v12
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v14, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v86, v87
@@ -59116,7 +59128,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s7, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s4, s5, s4
 ; GFX11-TRUE16-NEXT:    s_or_b32 s5, s7, s6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v54
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s2, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s3, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
@@ -59128,14 +59140,14 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s2, s2, s3
 ; GFX11-TRUE16-NEXT:    s_or_b32 s3, s4, s5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v84
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v83
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v80
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v49
 ; GFX11-TRUE16-NEXT:    s_add_i32 s28, s28, 3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v81
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s28, 0xff
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
@@ -59148,31 +59160,31 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    s_and_b32 s4, s4, 0xffff
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v71
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, s4, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v68
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v36
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v1, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v66
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v35
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v64
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v55
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
@@ -59222,42 +59234,42 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v28
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v29
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v1, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 3, v83
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v50
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v12, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v84
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v37
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v35
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 3, v69
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v67
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v12, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v48
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v38
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 3, v80
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 3, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 8, v68
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v11
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v71
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xff, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v34
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 8, v66
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v64
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v55
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v12, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v17, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v19, v15
@@ -69144,9 +69156,9 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:132
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:56
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:64
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
 ; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
@@ -69159,394 +69171,389 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:40
 ; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:72
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:92
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v29
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 24, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 24, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 24, v27
 ; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr17
 ; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr19
 ; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr27
-; SI-NEXT:    ; implicit-def: $vgpr29
 ; SI-NEXT:    ; implicit-def: $vgpr55
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:124
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:124
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 8, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v22, 24, v18
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v26
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v30
+; SI-NEXT:    v_lshlrev_b32_e32 v14, 24, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v30, 24, v22
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v26
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v31
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v32
-; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 8, v33
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_lshlrev_b32_e32 v12, 24, v34
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v35
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 8, v36
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v35
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 24, v37
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 8, v36
+; SI-NEXT:    v_lshlrev_b32_e32 v61, 24, v38
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 8, v25
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v30, 8, v25
-; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v39
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v48
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v49
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v50
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 8, v48
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr25
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v9
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    ; implicit-def: $vgpr1
-; SI-NEXT:    ; implicit-def: $vgpr9
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v5
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v13
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v7
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v15
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v3
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v11
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v13
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v9
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v15
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 24, v7
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v11
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 24, v23
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v23
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:28
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:12
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v17
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v29
+; SI-NEXT:    s_waitcnt expcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v49
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 24, v38
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v50
+; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr7
+; SI-NEXT:    ; kill: killed $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; kill: killed $vgpr7
+; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr7
+; SI-NEXT:    ; implicit-def: $vgpr9
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr11
 ; SI-NEXT:    ; implicit-def: $vgpr13
 ; SI-NEXT:    ; implicit-def: $vgpr15
-; SI-NEXT:    ; implicit-def: $vgpr19
+; SI-NEXT:    ; implicit-def: $vgpr17
 ; SI-NEXT:    ; implicit-def: $vgpr23
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr29
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB98_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v5, v5, v12
-; SI-NEXT:    v_and_b32_e32 v23, 0xff, v46
-; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
+; SI-NEXT:    v_and_b32_e32 v21, 0xff, v58
+; SI-NEXT:    v_or_b32_e32 v21, v21, v26
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v0, v44
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_or_b32_e32 v7, v3, v7
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xff, v12
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
+; SI-NEXT:    v_or_b32_e32 v26, v26, v27
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v3, v7, v3
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    v_or_b32_e32 v48, v1, v3
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v3, v9, v3
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v48, v7, v3
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    v_or_b32_e32 v11, v11, v9
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_or_b32_e32 v11, v9, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v37, v26, v11
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v9, v7, v9
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_or_b32_e32 v9, v9, v13
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v7, v7, v1
+; SI-NEXT:    v_or_b32_e32 v7, v13, v7
 ; SI-NEXT:    v_or_b32_e32 v49, v9, v7
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    v_alignbit_b32 v1, v48, v11, 16
+; SI-NEXT:    v_alignbit_b32 v13, v48, v11, 16
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_or_b32_e32 v15, v13, v9
 ; SI-NEXT:    v_alignbit_b32 v9, v49, v15, 16
 ; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT:    v_or_b32_e32 v11, v11, v26
+; SI-NEXT:    ; implicit-def: $vgpr26
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_or_b32_e32 v35, v11, v15
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_or_b32_e32 v8, v9, v8
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
+; SI-NEXT:    v_or_b32_e32 v9, v9, v40
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_or_b32_e32 v19, v17, v13
+; SI-NEXT:    v_or_b32_e32 v50, v9, v19
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_or_b32_e32 v11, v11, v15
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
+; SI-NEXT:    v_or_b32_e32 v6, v13, v6
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT:    v_or_b32_e32 v17, v17, v20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v19, v13, v9
-; SI-NEXT:    v_or_b32_e32 v50, v8, v19
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    v_alignbit_b32 v9, v50, v0, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_or_b32_e32 v6, v8, v6
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT:    v_or_b32_e32 v8, v16, v8
-; SI-NEXT:    v_or_b32_e32 v51, v6, v8
+; SI-NEXT:    v_or_b32_e32 v23, v63, v9
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_alignbit_b32 v9, v50, v23, 16
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    v_or_b32_e32 v33, v11, v23
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT:    v_or_b32_e32 v16, v16, v13
+; SI-NEXT:    v_or_b32_e32 v51, v6, v16
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v16, 0xff, v43
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v16, v22, v16
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v28
-; SI-NEXT:    v_or_b32_e32 v22, v22, v60
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT:    v_or_b32_e32 v14, v14, v20
+; SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v16
+; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    v_or_b32_e32 v52, v17, v14
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v46
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT:    v_or_b32_e32 v20, v30, v17
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    v_alignbit_b32 v17, v52, v20, 16
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT:    v_or_b32_e32 v11, v11, v22
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT:    v_or_b32_e32 v6, v14, v6
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v57
-; SI-NEXT:    v_or_b32_e32 v14, v14, v20
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    v_or_b32_e32 v52, v14, v16
-; SI-NEXT:    v_and_b32_e32 v14, 0xff, v45
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT:    v_or_b32_e32 v14, v18, v14
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v6, v24, v6
 ; SI-NEXT:    v_alignbit_b32 v13, v51, v6, 16
-; SI-NEXT:    v_alignbit_b32 v17, v52, v14, 16
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v20, 0xff, v20
-; SI-NEXT:    v_or_b32_e32 v18, v18, v26
-; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v10, v10, v20
-; SI-NEXT:    v_or_b32_e32 v53, v18, v10
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v20, 0xff, v40
-; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT:    v_or_b32_e32 v20, v42, v20
-; SI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT:    v_or_b32_e32 v2, v2, v18
-; SI-NEXT:    v_and_b32_e32 v18, 0xff, v24
-; SI-NEXT:    v_or_b32_e32 v24, v58, v23
-; SI-NEXT:    v_or_b32_e32 v55, v22, v24
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    v_or_b32_e32 v18, v18, v44
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v54, v18, v20
-; SI-NEXT:    v_and_b32_e32 v18, 0xff, v61
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT:    v_or_b32_e32 v18, v56, v18
+; SI-NEXT:    v_and_b32_e32 v24, 0xff, v47
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT:    v_or_b32_e32 v10, v10, v24
+; SI-NEXT:    v_or_b32_e32 v53, v21, v10
+; SI-NEXT:    v_and_b32_e32 v21, 0xff, v57
+; SI-NEXT:    v_and_b32_e32 v24, 0xff, v62
+; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT:    v_or_b32_e32 v2, v2, v21
+; SI-NEXT:    v_or_b32_e32 v12, v12, v24
 ; SI-NEXT:    v_alignbit_b32 v21, v53, v2, 16
-; SI-NEXT:    v_alignbit_b32 v25, v54, v18, 16
-; SI-NEXT:    v_alignbit_b32 v29, v55, v5, 16
+; SI-NEXT:    v_or_b32_e32 v54, v0, v12
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v28
+; SI-NEXT:    v_and_b32_e32 v24, 0xff, v43
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_or_b32_e32 v8, v8, v60
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT:    v_or_b32_e32 v0, v56, v0
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT:    v_or_b32_e32 v24, v59, v24
+; SI-NEXT:    v_alignbit_b32 v25, v54, v0, 16
+; SI-NEXT:    v_or_b32_e32 v55, v8, v24
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v41
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v8, v61, v8
+; SI-NEXT:    v_alignbit_b32 v29, v55, v8, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v23, 16, v10
+; SI-NEXT:    v_lshrrev_b32_e32 v27, 16, v12
 ; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
-; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr28
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr24
-; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr10
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v12, v12, v22
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT:    v_or_b32_e32 v37, v12, v11
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT:    v_or_b32_e32 v35, v11, v15
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v8
 ; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr8
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v11, v63
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT:    v_or_b32_e32 v33, v11, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    v_or_b32_e32 v32, v11, v6
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v11, 16, v19
-; SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
+; SI-NEXT:    ; implicit-def: $vgpr14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v30
-; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT:    v_or_b32_e32 v32, v0, v6
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT:    v_or_b32_e32 v4, v6, v4
 ; SI-NEXT:    ; implicit-def: $vgpr6
-; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_or_b32_e32 v34, v4, v20
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v4
-; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT:    v_or_b32_e32 v34, v0, v14
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    ; kill: killed $vgpr4
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    ; implicit-def: $vgpr14
-; SI-NEXT:    ; kill: killed $vgpr4
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_or_b32_e32 v4, v4, v45
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT:    v_or_b32_e32 v36, v4, v2
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v42
+; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v47
-; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT:    v_or_b32_e32 v36, v0, v2
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v2, v2, v5
+; SI-NEXT:    ; implicit-def: $vgpr5
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_or_b32_e32 v38, v2, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v18
+; SI-NEXT:    ; implicit-def: $vgpr18
 ; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v59
-; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT:    v_or_b32_e32 v38, v0, v18
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v41
-; SI-NEXT:    v_or_b32_e32 v0, v0, v62
+; SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT:    v_or_b32_e32 v39, v0, v5
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    v_or_b32_e32 v39, v0, v8
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
+; SI-NEXT:    ; implicit-def: $vgpr8
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    ; kill: killed $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
@@ -69568,25 +69575,21 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr18
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:  .LBB98_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB98_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v41
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v12
-; SI-NEXT:    v_or_b32_e32 v1, v62, v1
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v18
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_or_b32_e32 v1, v1, v3
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v41
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
-; SI-NEXT:    v_or_b32_e32 v3, v5, v3
+; SI-NEXT:    v_or_b32_e32 v3, v61, v3
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v28
@@ -69596,156 +69599,155 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
 ; SI-NEXT:    v_add_i32_e32 v39, vcc, s7, v1
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v46
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v43
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v3, v58, v3
+; SI-NEXT:    v_or_b32_e32 v3, v59, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
 ; SI-NEXT:    v_add_i32_e32 v55, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v61
-; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_or_b32_e32 v3, v56, v3
-; SI-NEXT:    v_alignbit_b32 v29, v55, v39, 16
-; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v55
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v59, v1
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v3, v1
-; SI-NEXT:    v_add_i32_e32 v38, vcc, s7, v1
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v42
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v40
-; SI-NEXT:    v_or_b32_e32 v1, v44, v1
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v8
+; SI-NEXT:    v_or_b32_e32 v1, v5, v1
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v3, v42, v3
+; SI-NEXT:    v_or_b32_e32 v3, v56, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
-; SI-NEXT:    v_add_i32_e32 v54, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT:    v_add_i32_e32 v38, vcc, s7, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v62
+; SI-NEXT:    v_or_b32_e32 v0, v44, v0
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v1, v12, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v54, vcc, s7, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v57
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    v_alignbit_b32 v25, v54, v38, 16
+; SI-NEXT:    v_alignbit_b32 v29, v55, v39, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v27, 16, v54
+; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v55
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v45, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v36, vcc, s7, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v58
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v47
+; SI-NEXT:    v_or_b32_e32 v0, v26, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_or_b32_e32 v1, v47, v1
-; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v36, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v1, v10, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v53, vcc, s7, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v46
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_or_b32_e32 v1, v26, v1
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v10, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v53, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v45
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v2, v18, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v1, v30, v1
 ; SI-NEXT:    v_alignbit_b32 v21, v53, v36, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v23, 16, v53
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v4, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v34, vcc, s7, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_or_b32_e32 v0, v20, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v4, v1
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v34, vcc, s7, v1
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v57
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v43
-; SI-NEXT:    v_or_b32_e32 v1, v20, v1
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v22, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v52, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v1, v14, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v52, vcc, s7, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; SI-NEXT:    v_alignbit_b32 v17, v52, v34, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v52
 ; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_or_b32_e32 v0, v22, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_or_b32_e32 v1, v30, v1
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v14, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v32, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v1, v24, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v32, vcc, s7, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_or_b32_e32 v0, v6, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_or_b32_e32 v1, v6, v1
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v2, v16, v2
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    v_add_i32_e32 v51, vcc, s7, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v1, v16, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v51, vcc, s7, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
 ; SI-NEXT:    v_alignbit_b32 v13, v51, v32, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v51
 ; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v63, v1
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v33, vcc, s7, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
-; SI-NEXT:    v_or_b32_e32 v0, v8, v0
+; SI-NEXT:    v_or_b32_e32 v0, v40, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
@@ -69774,8 +69776,8 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v35, vcc, s7, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
@@ -69792,8 +69794,8 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v49, vcc, s7, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v49
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
@@ -69811,7 +69813,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v37, vcc, s7, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
@@ -69828,12 +69830,15 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_add_i32_e32 v48, vcc, s7, v0
-; SI-NEXT:    v_alignbit_b32 v1, v48, v37, 16
+; SI-NEXT:    v_alignbit_b32 v0, v48, v37, 16
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v0, v49, v35, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v48
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB98_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
@@ -69861,6 +69866,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v12, v32
 ; SI-NEXT:    v_mov_b32_e32 v14, v51
 ; SI-NEXT:    v_mov_b32_e32 v16, v34
+; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_mov_b32_e32 v18, v52
 ; SI-NEXT:    v_mov_b32_e32 v20, v36
 ; SI-NEXT:    v_mov_b32_e32 v22, v53
@@ -69906,78 +69912,78 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:88
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:96
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
 ; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:4
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:116
-; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:124
-; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v7
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v9
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v11
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v29
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v27, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:116
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
+; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v17
 ; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
 ; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v0
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v2
-; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v4
-; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v8
-; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v10
-; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v12
-; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v14
-; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v16
-; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v18
-; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v20
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
+; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v10
+; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v12
+; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v14
+; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v16
+; VI-NEXT:    v_lshlrev_b16_e32 v61, 8, v18
+; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v20
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v22
+; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v22
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_lshlrev_b16_e32 v18, 8, v26
+; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v24
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshlrev_b16_e32 v26, 8, v28
+; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v26
+; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:124
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v30
+; VI-NEXT:    v_lshlrev_b16_e32 v36, 8, v30
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v34, 8, v31
+; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v31
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_ushort v34, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:84
 ; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:68
 ; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:52
+; VI-NEXT:    v_lshlrev_b16_e32 v28, 8, v28
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v32
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -69989,79 +69995,80 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT:    v_or_b32_sdwa v9, v39, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v10, v20, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v28, v59 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v22, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v30, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v35, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v31, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v16, v38, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr39
-; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_or_b32_sdwa v9, v29, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v46, v59 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_or_b32_sdwa v12, v22, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v30, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v34, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v31, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr22
 ; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
 ; VI-NEXT:    ; implicit-def: $vgpr31
-; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    ; implicit-def: $vgpr57
 ; VI-NEXT:    ; implicit-def: $vgpr59
-; VI-NEXT:    ; implicit-def: $vgpr62
-; VI-NEXT:    ; implicit-def: $vgpr18
-; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr36
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_or_b32_sdwa v0, v0, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v51 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr49
 ; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr52
-; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr23
-; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr40
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_or_b32_sdwa v8, v8, v45 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v4, v4, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr52
 ; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr40
-; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -70078,23 +70085,23 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr25
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v49, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v48, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v55, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v55, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v44, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v36, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v18, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v43, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v26, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v44, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v61, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
@@ -70128,98 +70135,92 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; kill: killed $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr49
-; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr27
 ; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr36
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr37
 ; VI-NEXT:    ; implicit-def: $vgpr44
-; VI-NEXT:    ; implicit-def: $vgpr61
-; VI-NEXT:    ; implicit-def: $vgpr29
-; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr45
 ; VI-NEXT:    ; implicit-def: $vgpr56
 ; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:  .LBB98_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB98_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v38
-; VI-NEXT:    v_add_u16_e32 v2, 3, v44
+; VI-NEXT:    v_add_u16_e32 v0, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v0, v32, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v14, v26, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_add_u16_e32 v2, 3, v35
 ; VI-NEXT:    v_mov_b32_e32 v3, 0x300
-; VI-NEXT:    v_or_b32_sdwa v2, v18, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v18, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v37
-; VI-NEXT:    v_or_b32_sdwa v24, v24, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v16, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v37
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v38, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v12, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v43
-; VI-NEXT:    v_or_b32_sdwa v16, v63, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v22
-; VI-NEXT:    v_or_b32_sdwa v0, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v11, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v36
-; VI-NEXT:    v_or_b32_sdwa v22, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v26, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v28
+; VI-NEXT:    v_add_u16_e32 v0, 3, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v11, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v18
+; VI-NEXT:    v_or_b32_sdwa v18, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v10, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v55
-; VI-NEXT:    v_or_b32_sdwa v28, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v20
+; VI-NEXT:    v_add_u16_e32 v0, 3, v44
+; VI-NEXT:    v_or_b32_sdwa v22, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v41
 ; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v2, 3, v35
 ; VI-NEXT:    v_add_u16_sdwa v9, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v48
-; VI-NEXT:    v_or_b32_sdwa v20, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v0, 3, v39
+; VI-NEXT:    v_add_u16_e32 v0, 3, v55
+; VI-NEXT:    v_or_b32_sdwa v14, v28, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v28, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v29
 ; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v49
-; VI-NEXT:    v_or_b32_sdwa v30, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v27
+; VI-NEXT:    v_or_b32_sdwa v27, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v2, 3, v34
+; VI-NEXT:    v_or_b32_sdwa v2, v24, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v13, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v1, 3, v61
-; VI-NEXT:    v_or_b32_sdwa v15, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 3, v63
+; VI-NEXT:    v_or_b32_sdwa v15, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v1, 3, v31
-; VI-NEXT:    v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v26, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
-; VI-NEXT:    v_or_b32_e32 v12, v16, v12
-; VI-NEXT:    v_add_u16_e32 v16, 0x300, v24
+; VI-NEXT:    v_or_b32_sdwa v1, v36, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v24, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
-; VI-NEXT:    v_or_b32_e32 v13, v16, v13
-; VI-NEXT:    v_or_b32_e32 v14, v14, v26
-; VI-NEXT:    v_or_b32_e32 v15, v15, v18
+; VI-NEXT:    v_or_b32_e32 v14, v14, v24
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v29, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v29, v42, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -70252,54 +70253,58 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v6, v17, v6
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v29
 ; VI-NEXT:    v_or_b32_e32 v7, v17, v7
-; VI-NEXT:    v_add_u16_e32 v17, 0x300, v30
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v27
 ; VI-NEXT:    v_or_b32_e32 v8, v17, v8
-; VI-NEXT:    v_add_u16_e32 v17, 0x300, v20
-; VI-NEXT:    v_or_b32_e32 v9, v17, v9
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v28
-; VI-NEXT:    v_or_b32_e32 v10, v17, v10
+; VI-NEXT:    v_or_b32_e32 v9, v17, v9
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v22
+; VI-NEXT:    v_or_b32_e32 v10, v17, v10
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v18
 ; VI-NEXT:    v_or_b32_e32 v11, v17, v11
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v26
+; VI-NEXT:    v_or_b32_e32 v12, v17, v12
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v20
+; VI-NEXT:    v_or_b32_e32 v13, v17, v13
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v42, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v19, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v23, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v23, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v2, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v27, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v30, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v1, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v31, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v31, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v3, v50, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v0, v3, v0
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v31
 ; VI-NEXT:    v_or_b32_e32 v1, v3, v1
-; VI-NEXT:    v_add_u16_e32 v3, 0x300, v27
+; VI-NEXT:    v_add_u16_e32 v3, 0x300, v30
 ; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v23
 ; VI-NEXT:    v_or_b32_e32 v3, v3, v19
@@ -70359,99 +70364,100 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:80
 ; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:88
 ; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:96
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v33, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v7
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v9
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v11
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v15
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v17
+; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v17
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v19
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v23
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
+; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
-; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v29
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v25, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v11
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v15
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(20)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(19)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v10
+; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v14
 ; GFX9-NEXT:    s_waitcnt vmcnt(17)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v12
+; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v14
+; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v16
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v18
+; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v20
+; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v24
-; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    v_lshlrev_b16_e32 v18, 8, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v24, 8, v28
+; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v31
+; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v31, 8, v32
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v34, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    v_lshlrev_b16_e32 v26, 8, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v33
+; GFX9-NEXT:    v_lshlrev_b16_e32 v28, 8, v31
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v32
+; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB98_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
@@ -70459,108 +70465,107 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v39, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v63, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v35, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v34, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v16, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v16, v38, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr39
-; GFX9-NEXT:    ; implicit-def: $vgpr62
-; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr35
-; GFX9-NEXT:    ; implicit-def: $vgpr34
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
+; GFX9-NEXT:    v_or_b32_sdwa v9, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v41, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_or_b32_sdwa v12, v26, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v37, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v20, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr31
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
-; GFX9-NEXT:    ; implicit-def: $vgpr36
-; GFX9-NEXT:    ; implicit-def: $vgpr26
-; GFX9-NEXT:    ; implicit-def: $vgpr20
-; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr30
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s6
 ; GFX9-NEXT:    v_perm_b32 v1, v3, v2, s6
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr49
 ; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr39
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
-; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr53
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr23
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v2, v3, v2, s6
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v3, v4, v3, s6
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr55
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v5, v6, v5, s6
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v6, v7, v6, s6
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr40
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v7, v8, v7, s6
-; GFX9-NEXT:    v_or_b32_sdwa v8, v51, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
-; GFX9-NEXT:    v_or_b32_sdwa v9, v52, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v42, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
-; GFX9-NEXT:    v_or_b32_sdwa v10, v42, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v47, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    v_or_b32_sdwa v11, v37, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v35, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    v_or_b32_sdwa v12, v44, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v16, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
-; GFX9-NEXT:    v_or_b32_sdwa v13, v28, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v24, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    v_or_b32_sdwa v14, v47, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v32, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    v_or_b32_sdwa v15, v18, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v38, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v16, v36, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
@@ -70594,111 +70599,110 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; kill: killed $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
-; GFX9-NEXT:    ; implicit-def: $vgpr51
-; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr29
 ; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr37
-; GFX9-NEXT:    ; implicit-def: $vgpr44
-; GFX9-NEXT:    ; implicit-def: $vgpr28
-; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr47
-; GFX9-NEXT:    ; implicit-def: $vgpr18
-; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; implicit-def: $vgpr22
-; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr24
-; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:  .LBB98_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB98_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
-; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v18
-; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v0, v28, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v28
-; GFX9-NEXT:    v_or_b32_sdwa v0, v30, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v24
+; GFX9-NEXT:    v_or_b32_sdwa v0, v22, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v20
+; GFX9-NEXT:    v_or_b32_sdwa v0, v18, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v16
-; GFX9-NEXT:    v_or_b32_sdwa v0, v26, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v44
-; GFX9-NEXT:    v_or_b32_sdwa v0, v22, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v63, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v34
-; GFX9-NEXT:    v_or_b32_sdwa v0, v36, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v37
+; GFX9-NEXT:    v_or_b32_sdwa v0, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v35
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v35
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v26
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v24, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
+; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v47
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v63
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v41
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v52
+; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v27
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v51
+; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v29
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v25
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v25, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_add_u16_e32 v3, 3, v32
-; GFX9-NEXT:    v_or_b32_sdwa v3, v20, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v3
+; GFX9-NEXT:    v_add_u16_e32 v3, 3, v31
+; GFX9-NEXT:    v_or_b32_sdwa v3, v30, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v3
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v32
+; GFX9-NEXT:    v_or_b32_sdwa v2, v34, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v2
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v38
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v36
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v1
+; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v1
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v8, v30, v8, s6
-; GFX9-NEXT:    v_perm_b32 v9, v28, v9, s6
-; GFX9-NEXT:    v_perm_b32 v10, v26, v10, s6
-; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s6
-; GFX9-NEXT:    v_perm_b32 v12, v22, v12, s6
-; GFX9-NEXT:    v_perm_b32 v13, v16, v13, s6
-; GFX9-NEXT:    v_perm_b32 v14, v20, v14, s6
-; GFX9-NEXT:    v_perm_b32 v15, v18, v15, s6
+; GFX9-NEXT:    v_perm_b32 v8, v25, v8, s6
+; GFX9-NEXT:    v_perm_b32 v9, v24, v9, s6
+; GFX9-NEXT:    v_perm_b32 v10, v22, v10, s6
+; GFX9-NEXT:    v_perm_b32 v11, v20, v11, s6
+; GFX9-NEXT:    v_perm_b32 v12, v16, v12, s6
+; GFX9-NEXT:    v_perm_b32 v13, v18, v13, s6
+; GFX9-NEXT:    v_perm_b32 v14, v30, v14, s6
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s6
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v31, 3, v31
-; GFX9-NEXT:    v_or_b32_sdwa v31, v48, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v31, v39, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v31, 0x300, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v27, 0x300, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_perm_b32 v7, v27, v7, s6
+; GFX9-NEXT:    v_perm_b32 v7, v26, v7, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v25, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -70720,7 +70724,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v5, v19, v5, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -70731,40 +70735,40 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v4, v17, v4, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v3, v21, v3, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v25, 0x300, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v27, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_perm_b32 v2, v25, v2, s6
+; GFX9-NEXT:    v_perm_b32 v2, v27, v2, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v1, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v29, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v1, v29, v1, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; GFX9-NEXT:    v_perm_b32 v0, v31, v0, s6
 ; GFX9-NEXT:  .LBB98_4: ; %end
@@ -71630,13 +71634,13 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt expcnt(4)
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt expcnt(2)
 ; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:52
 ; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:68
 ; SI-NEXT:    v_readfirstlane_b32 s15, v27
 ; SI-NEXT:    v_readfirstlane_b32 s40, v26
 ; SI-NEXT:    v_readfirstlane_b32 s12, v19
@@ -71668,14 +71672,14 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 24, v37
 ; SI-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 8, v38
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v38
 ; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v39
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_lshlrev_b32_e32 v47, 24, v48
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 8, v59
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 24, v61
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 24, v62
@@ -71768,10 +71772,9 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v43
 ; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
 ; SI-NEXT:    v_or_b32_e32 v33, v58, v32
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v32, 0xff, v46
+; SI-NEXT:    v_and_b32_e32 v32, 0xff, v45
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v34, 0xff, v45
+; SI-NEXT:    v_and_b32_e32 v34, 0xff, v46
 ; SI-NEXT:    s_or_b32 s4, s4, s56
 ; SI-NEXT:    v_or_b32_e32 v25, v54, v17
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
@@ -71789,13 +71792,13 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; SI-NEXT:    s_lshl_b32 s56, s15, 8
 ; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; SI-NEXT:    v_or_b32_e32 v48, v32, v63
-; SI-NEXT:    v_and_b32_e32 v32, 0xff, v56
+; SI-NEXT:    v_and_b32_e32 v32, 0xff, v57
 ; SI-NEXT:    s_or_b32 s4, s4, s56
 ; SI-NEXT:    v_or_b32_e32 v29, v44, v21
 ; SI-NEXT:    v_and_b32_e32 v26, 0xff, v40
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
 ; SI-NEXT:    s_and_b32 s4, s4, 0xffff
-; SI-NEXT:    v_or_b32_e32 v26, v26, v57
+; SI-NEXT:    v_or_b32_e32 v26, v26, v56
 ; SI-NEXT:    v_or_b32_e32 v34, v61, v32
 ; SI-NEXT:    v_or_b32_e32 v32, s4, v29
 ; SI-NEXT:    s_and_b32 s4, s43, 0xff
@@ -71833,7 +71836,8 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; SI-NEXT:    s_add_i32 s45, s45, 3
 ; SI-NEXT:    s_and_b32 s4, s45, 0xff
 ; SI-NEXT:    s_lshl_b32 s5, s44, 8
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v56
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v57
 ; SI-NEXT:    s_or_b32 s4, s5, s4
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_addk_i32 s4, 0x300
@@ -71842,11 +71846,10 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; SI-NEXT:    v_or_b32_e32 v1, v61, v1
 ; SI-NEXT:    v_or_b32_e32 v1, s4, v1
 ; SI-NEXT:    v_add_i32_e32 v34, vcc, 0x3000000, v1
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v46
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v45
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v46
 ; SI-NEXT:    v_or_b32_e32 v1, v60, v1
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
@@ -71870,7 +71873,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v40
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v55
-; SI-NEXT:    v_or_b32_e32 v1, v57, v1
+; SI-NEXT:    v_or_b32_e32 v1, v56, v1
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
@@ -72878,57 +72881,34 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32i16_scalar:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v1 :: v_dual_mov_b32 v52, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v86, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v15 :: v_dual_mov_b32 v34, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v37, v13 :: v_dual_mov_b32 v54, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v12 :: v_dual_mov_b32 v50, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v49, v11 :: v_dual_mov_b32 v48, v8
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v9 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v6 :: v_dual_mov_b32 v38, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v3 :: v_dual_mov_b32 v39, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v86
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB99_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -72952,8 +72932,8 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s10, s11
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v32
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s7, s8, s9
 ; GFX11-TRUE16-NEXT:    s_and_b32 s8, s24, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s25, 8
@@ -72961,14 +72941,14 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s10, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v31
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v52
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v18
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v53
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v50
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
@@ -72977,10 +72957,10 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v3, 0xffff, s9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v5, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v20
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v30
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v48
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v35
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v49
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v8
@@ -72988,63 +72968,63 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v34
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v17
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v6
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v0, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v19
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v39
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v23
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v37
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v49
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v66
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v3, 16, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v12, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v64
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v85
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v84
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v67
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v70
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v69
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v82
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v80
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v12, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v83
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v83
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v86, v87
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v50
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v80
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v68
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v67
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v14, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v11
@@ -73103,151 +73083,151 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s1, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s10, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v80
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v84
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v83
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v85
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v71
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v80
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v71
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v54
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v70
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v67
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v82
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v68
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v81
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v65
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v69
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v66
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v66
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v64
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v22
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v6, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v18
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v64
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v26
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v28
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v27
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v29
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v48
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v24
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v37
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v25
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v37, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v9, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v36
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v20
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v34
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v32, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v54
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v7, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 0x300, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 8, v49
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v8, v7
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v28, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v25
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v16, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 0x300, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v23, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v20, v4
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v18
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v32
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v6
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v22, v4
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v23, v8
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v23, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v18, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v8, 0xffff, s4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v5, 16, v8
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v6, 16, v16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v23, 16, v8
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v25, 16, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v6, 16, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v9
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v25, 16, v8
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v16, 16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v17, 16, v6
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v29, 16, v18
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v10, 16, v19
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v37, 16, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v23, 16, v6
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v20, 16, v19
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v10, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v18, 16, v15
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v12, v14, 16, v12
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v13, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v14, v2, 16, v16
@@ -82280,29 +82260,53 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(6)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:132
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:128
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:128
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v7
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v5
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v7
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v9
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v11
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v13
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v15
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v23
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:124
 ; SI-NEXT:    ; implicit-def: $vgpr7
 ; SI-NEXT:    ; kill: killed $vgpr7
 ; SI-NEXT:    ; implicit-def: $vgpr7
@@ -82314,21 +82318,19 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr7
 ; SI-NEXT:    ; kill: killed $vgpr7
 ; SI-NEXT:    ; implicit-def: $vgpr7
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 8, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 8, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 8, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v24, 8, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 8, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v22, 8, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 8, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 8, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 8, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 8, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v61, 8, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 8, v29
 ; SI-NEXT:    ; kill: killed $vgpr7
 ; SI-NEXT:    ; implicit-def: $vgpr7
 ; SI-NEXT:    ; kill: killed $vgpr7
 ; SI-NEXT:    ; implicit-def: $vgpr7
+; SI-NEXT:    ; implicit-def: $vgpr9
 ; SI-NEXT:    ; implicit-def: $vgpr11
+; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr13
 ; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr15
@@ -82336,205 +82338,104 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr19
 ; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    ; implicit-def: $vgpr23
+; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr25
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr27
 ; SI-NEXT:    ; implicit-def: $vgpr52
 ; SI-NEXT:    ; implicit-def: $vgpr29
 ; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:124
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
-; SI-NEXT:    v_lshlrev_b32_e32 v28, 8, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v20, 8, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 8, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 8, v6
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:116
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 8, v10
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 8, v31
-; SI-NEXT:    ; implicit-def: $vgpr31
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v28, 8, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v30, 8, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v20, 8, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 8, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v14, 8, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v12, 8, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 8, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v22
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 8, v26
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:52
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 8, v31
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v32
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v33
-; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v34
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 8, v35
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v55, 8, v36
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 8, v37
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 8, v35
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v36
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 8, v38
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 8, v39
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v48
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 8, v49
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:12
-; SI-NEXT:    ; implicit-def: $vgpr49
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:100
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v5
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v9
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v38
-; SI-NEXT:    ; implicit-def: $vgpr9
 ; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 8, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v32
-; SI-NEXT:    ; implicit-def: $vgpr32
+; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB106_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v6, v14
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v6
-; SI-NEXT:    ; implicit-def: $vgpr14
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_or_b32_e32 v7, v7, v9
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v6, v12
-; SI-NEXT:    v_cvt_f32_f16_e32 v36, v6
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v39
-; SI-NEXT:    v_or_b32_e32 v6, v6, v10
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v6
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v35
-; SI-NEXT:    v_or_b32_e32 v6, v6, v8
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v6
-; SI-NEXT:    v_and_b32_e32 v6, 0xff, v44
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr10
-; SI-NEXT:    ; implicit-def: $vgpr8
-; SI-NEXT:    v_or_b32_e32 v0, v6, v0
-; SI-NEXT:    ; implicit-def: $vgpr6
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v0
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v19, 0xff, v55
+; SI-NEXT:    v_or_b32_e32 v16, v19, v16
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v16
+; SI-NEXT:    v_and_b32_e32 v16, 0xff, v22
+; SI-NEXT:    v_or_b32_e32 v14, v16, v14
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v14
+; SI-NEXT:    v_and_b32_e32 v14, 0xff, v26
+; SI-NEXT:    v_or_b32_e32 v12, v14, v12
+; SI-NEXT:    v_cvt_f32_f16_e32 v36, v12
+; SI-NEXT:    v_and_b32_e32 v12, 0xff, v18
+; SI-NEXT:    v_or_b32_e32 v10, v12, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v10
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT:    v_or_b32_e32 v17, v17, v20
-; SI-NEXT:    v_or_b32_e32 v15, v15, v26
-; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    ; implicit-def: $vgpr20
-; SI-NEXT:    v_cvt_f32_f16_e32 v32, v17
-; SI-NEXT:    v_cvt_f32_f16_e32 v53, v15
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_or_b32_e32 v9, v9, v11
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v9
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v17, 0xff, v17
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
-; SI-NEXT:    v_or_b32_e32 v17, v17, v18
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_or_b32_e32 v0, v0, v2
-; SI-NEXT:    v_or_b32_e32 v15, v15, v28
-; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    ; implicit-def: $vgpr2
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v45
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    v_or_b32_e32 v0, v0, v41
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    v_cvt_f32_f16_e32 v25, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v3
-; SI-NEXT:    ; implicit-def: $vgpr3
-; SI-NEXT:    v_or_b32_e32 v0, v0, v55
-; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v41
+; SI-NEXT:    v_or_b32_e32 v8, v10, v8
+; SI-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v42
+; SI-NEXT:    v_or_b32_e32 v8, v8, v39
+; SI-NEXT:    v_cvt_f32_f16_e32 v23, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v43
+; SI-NEXT:    v_or_b32_e32 v8, v8, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v48, v8
+; SI-NEXT:    v_and_b32_e32 v8, 0xff, v44
+; SI-NEXT:    v_or_b32_e32 v4, v8, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v25, v4
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v45
+; SI-NEXT:    v_or_b32_e32 v0, v4, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v50, v0
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v5
-; SI-NEXT:    ; implicit-def: $vgpr5
-; SI-NEXT:    v_or_b32_e32 v0, v0, v43
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v30
-; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    ; implicit-def: $vgpr1
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v0, v0, v35
 ; SI-NEXT:    v_cvt_f32_f16_e32 v52, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v51
-; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    v_or_b32_e32 v0, v0, v46
-; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v56
+; SI-NEXT:    v_or_b32_e32 v0, v0, v3
 ; SI-NEXT:    v_cvt_f32_f16_e32 v29, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    v_or_b32_e32 v0, v0, v47
-; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v6
+; SI-NEXT:    v_or_b32_e32 v0, v0, v46
 ; SI-NEXT:    v_cvt_f32_f16_e32 v54, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v42
-; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    v_or_b32_e32 v0, v0, v56
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v1
+; SI-NEXT:    v_or_b32_e32 v0, v0, v51
 ; SI-NEXT:    v_cvt_f32_f16_e32 v31, v0
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
@@ -82577,332 +82478,389 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v60
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    v_or_b32_e32 v17, v17, v30
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    v_cvt_f32_f16_e32 v32, v17
+; SI-NEXT:    v_and_b32_e32 v17, 0xff, v62
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    v_or_b32_e32 v17, v17, v20
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr55
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr26
+; SI-NEXT:    ; implicit-def: $vgpr18
+; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr5
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr6
+; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:    ; kill: killed $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr20
+; SI-NEXT:    ; implicit-def: $vgpr16
+; SI-NEXT:    ; implicit-def: $vgpr14
+; SI-NEXT:    ; implicit-def: $vgpr12
+; SI-NEXT:    ; implicit-def: $vgpr10
+; SI-NEXT:    ; implicit-def: $vgpr8
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    ; implicit-def: $vgpr2
+; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr3
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_or_b32_e32 v7, v7, v9
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_or_b32_e32 v9, v9, v11
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v9
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v13
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v11
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v18, 0xff, v18
-; SI-NEXT:    v_or_b32_e32 v16, v18, v16
-; SI-NEXT:    ; implicit-def: $vgpr18
-; SI-NEXT:    v_cvt_f32_f16_e32 v34, v16
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_or_b32_e32 v13, v13, v58
+; SI-NEXT:    v_or_b32_e32 v13, v13, v15
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v13
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v11, v24
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v11
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_or_b32_e32 v15, v15, v24
 ; SI-NEXT:    ; implicit-def: $vgpr24
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v53, v15
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v9
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_or_b32_e32 v13, v13, v63
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v13
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v11, v60
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
-; SI-NEXT:    ; implicit-def: $vgpr60
-; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_or_b32_e32 v9, v9, v62
-; SI-NEXT:    v_cvt_f32_f16_e32 v33, v9
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v15, 0xff, v15
+; SI-NEXT:    v_or_b32_e32 v15, v15, v28
+; SI-NEXT:    ; implicit-def: $vgpr28
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v7, v57
+; SI-NEXT:    v_or_b32_e32 v7, v7, v9
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
-; SI-NEXT:    v_or_b32_e32 v13, v13, v22
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr62
-; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_or_b32_e32 v9, v9, v40
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v7, v59
+; SI-NEXT:    v_or_b32_e32 v7, v7, v9
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT:    v_or_b32_e32 v11, v11, v63
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v7, v61
+; SI-NEXT:    v_or_b32_e32 v7, v7, v9
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    ; implicit-def: $vgpr63
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    v_or_b32_e32 v9, v9, v57
+; SI-NEXT:    v_cvt_f32_f16_e32 v33, v9
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; SI-NEXT:    v_or_b32_e32 v11, v11, v13
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
+; SI-NEXT:    v_or_b32_e32 v13, v13, v58
+; SI-NEXT:    v_cvt_f32_f16_e32 v49, v13
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    v_or_b32_e32 v9, v9, v59
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v13, 0xff, v13
+; SI-NEXT:    v_or_b32_e32 v13, v13, v61
 ; SI-NEXT:    ; implicit-def: $vgpr61
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:  .LBB106_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB106_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v42
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v30
-; SI-NEXT:    v_or_b32_e32 v7, v56, v7
-; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v11, v1, v11
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v7
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v45
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    s_movk_i32 s6, 0x300
-; SI-NEXT:    v_or_b32_e32 v7, v41, v7
-; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v56
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; SI-NEXT:    v_or_b32_e32 v6, v14, v6
-; SI-NEXT:    v_add_i32_e32 v30, vcc, s6, v11
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v51
-; SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
-; SI-NEXT:    v_or_b32_e32 v9, v46, v9
-; SI-NEXT:    v_add_i32_e32 v29, vcc, s6, v9
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_or_b32_e32 v7, v3, v7
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v47
+; SI-NEXT:    v_or_b32_e32 v6, v46, v6
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v4, v47, v4
-; SI-NEXT:    v_or_b32_e32 v5, v43, v5
-; SI-NEXT:    v_or_b32_e32 v3, v55, v3
-; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
-; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
-; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    v_cvt_f32_f16_e32 v50, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v52, v30
-; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
-; SI-NEXT:    v_cvt_f32_f16_e32 v54, v4
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v2, v2, v7
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v44
+; SI-NEXT:    s_movk_i32 s6, 0x300
+; SI-NEXT:    v_or_b32_e32 v9, v35, v3
+; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v6
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v7
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v42
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v0, v0, v7
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v35
+; SI-NEXT:    v_or_b32_e32 v7, v39, v7
+; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v7
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v41
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v7
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v39
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v18
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v10, v7
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v8
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v2
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
-; SI-NEXT:    v_and_b32_e32 v27, 0xff, v27
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
-; SI-NEXT:    v_and_b32_e32 v23, 0xff, v23
-; SI-NEXT:    v_or_b32_e32 v23, v57, v23
-; SI-NEXT:    v_add_i32_e32 v23, vcc, s6, v23
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v23
-; SI-NEXT:    buffer_store_dword v23, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v0
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v26
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v12, v7
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v36, v12
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v22
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v16, v7
+; SI-NEXT:    v_or_b32_e32 v7, v14, v7
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v34, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v55
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v18, v7
+; SI-NEXT:    v_or_b32_e32 v7, v16, v7
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v7
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v62
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v20, v7
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v60
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT:    v_or_b32_e32 v7, v30, v7
+; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v29, vcc, s6, v9
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_or_b32_e32 v2, v2, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v45
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_or_b32_e32 v0, v0, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v44
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_or_b32_e32 v4, v4, v5
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v43
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_or_b32_e32 v1, v51, v1
+; SI-NEXT:    v_or_b32_e32 v5, v40, v5
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 0x300, v1
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
+; SI-NEXT:    v_add_i32_e32 v5, vcc, s6, v5
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v16
+; SI-NEXT:    v_cvt_f32_f16_e32 v36, v12
+; SI-NEXT:    v_cvt_f32_f16_e32 v38, v8
+; SI-NEXT:    v_cvt_f32_f16_e32 v23, v23
+; SI-NEXT:    v_cvt_f32_f16_e32 v48, v5
+; SI-NEXT:    v_cvt_f32_f16_e32 v50, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v52, v29
+; SI-NEXT:    v_cvt_f32_f16_e32 v29, v6
+; SI-NEXT:    v_cvt_f32_f16_e32 v54, v3
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v28, v7
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v27, v28, v27
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v26, v7
-; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v7
+; SI-NEXT:    v_or_b32_e32 v7, v24, v7
+; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
-; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    v_cvt_f32_f16_e32 v53, v18
-; SI-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v27, v5
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
-; SI-NEXT:    v_and_b32_e32 v28, 0xff, v28
-; SI-NEXT:    v_or_b32_e32 v28, v31, v28
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v28, vcc, s6, v28
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v53, v19
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v14
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v22, v7
+; SI-NEXT:    v_or_b32_e32 v7, v61, v7
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
-; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
-; SI-NEXT:    v_or_b32_e32 v31, v32, v31
-; SI-NEXT:    v_cvt_f32_f16_e32 v32, v17
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v16
-; SI-NEXT:    v_add_i32_e32 v31, vcc, s6, v31
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v63, v7
-; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v7
+; SI-NEXT:    v_or_b32_e32 v7, v58, v7
+; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; SI-NEXT:    v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT:    v_or_b32_e32 v22, v59, v22
-; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v22
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v19
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v6
-; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v1
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v49, v20
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v60, v7
+; SI-NEXT:    v_or_b32_e32 v7, v9, v7
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v24, v7
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s6, v7
+; SI-NEXT:    v_or_b32_e32 v7, v63, v7
+; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v20
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v37, v21
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v10
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v40, v7
+; SI-NEXT:    v_or_b32_e32 v7, v59, v7
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
-; SI-NEXT:    v_or_b32_e32 v24, v26, v24
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
 ; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v62, v7
-; SI-NEXT:    v_add_i32_e32 v21, vcc, s6, v7
+; SI-NEXT:    v_or_b32_e32 v7, v57, v7
+; SI-NEXT:    v_add_i32_e32 v22, vcc, s6, v7
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v33, v22
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
+; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; SI-NEXT:    v_or_b32_e32 v7, v24, v7
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    v_and_b32_e32 v24, 0xff, v24
+; SI-NEXT:    v_or_b32_e32 v24, v25, v24
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v24, vcc, s6, v24
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
+; SI-NEXT:    v_and_b32_e32 v25, 0xff, v25
+; SI-NEXT:    v_or_b32_e32 v25, v26, v25
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v25, vcc, s6, v25
+; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
+; SI-NEXT:    buffer_store_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v25, v4
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
 ; SI-NEXT:    v_and_b32_e32 v26, 0xff, v26
-; SI-NEXT:    v_or_b32_e32 v26, v58, v26
+; SI-NEXT:    v_or_b32_e32 v26, v27, v26
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, s6, v26
 ; SI-NEXT:    v_cvt_f32_f16_e32 v26, v26
-; SI-NEXT:    v_cvt_f32_f16_e32 v33, v21
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v10
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_and_b32_e32 v7, 0xff, v7
-; SI-NEXT:    v_or_b32_e32 v7, v61, v7
-; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
+; SI-NEXT:    v_and_b32_e32 v27, 0xff, v27
+; SI-NEXT:    v_or_b32_e32 v27, v28, v27
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v27, vcc, s6, v27
+; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
+; SI-NEXT:    buffer_store_dword v27, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v27, v2
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
+; SI-NEXT:    v_and_b32_e32 v28, 0xff, v28
+; SI-NEXT:    v_or_b32_e32 v28, v30, v28
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v28, vcc, s6, v28
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v30, vcc, 3, v30
+; SI-NEXT:    v_and_b32_e32 v30, 0xff, v30
+; SI-NEXT:    v_or_b32_e32 v30, v31, v30
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v30, vcc, s6, v30
+; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v31, vcc, 3, v31
+; SI-NEXT:    v_and_b32_e32 v31, 0xff, v31
+; SI-NEXT:    v_or_b32_e32 v31, v32, v31
+; SI-NEXT:    v_add_i32_e32 v31, vcc, s6, v31
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    v_cvt_f32_f16_e32 v32, v18
+; SI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v1
 ; SI-NEXT:  .LBB106_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
@@ -82927,11 +82885,9 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v16, v32
 ; SI-NEXT:    v_mov_b32_e32 v18, v34
 ; SI-NEXT:    v_mov_b32_e32 v20, v36
-; SI-NEXT:    s_waitcnt expcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_mov_b32_e32 v22, v38
-; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    v_mov_b32_e32 v24, v48
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v26, v50
 ; SI-NEXT:    v_mov_b32_e32 v28, v52
 ; SI-NEXT:    v_mov_b32_e32 v30, v54
@@ -82973,78 +82929,78 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:88
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:96
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
 ; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:4
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:116
-; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:124
-; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v7
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v9
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v11
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v29
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v27, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:116
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
+; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v17
 ; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
 ; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v0
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v2
-; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v4
-; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v8
-; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v10
-; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v12
-; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v14
-; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v16
-; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v18
-; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v20
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
+; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v10
+; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v12
+; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v14
+; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v16
+; VI-NEXT:    v_lshlrev_b16_e32 v61, 8, v18
+; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v20
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v22
+; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v22
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_lshlrev_b16_e32 v18, 8, v26
+; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v24
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshlrev_b16_e32 v26, 8, v28
+; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v26
+; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:124
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v30
+; VI-NEXT:    v_lshlrev_b16_e32 v36, 8, v30
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v34, 8, v31
+; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v31
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_ushort v34, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:84
 ; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:68
 ; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:52
+; VI-NEXT:    v_lshlrev_b16_e32 v28, 8, v28
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v32
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -83056,79 +83012,80 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT:    v_or_b32_sdwa v9, v39, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v10, v20, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v28, v59 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v22, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v30, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v35, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v31, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v16, v38, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr39
-; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_or_b32_sdwa v9, v29, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v46, v59 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_or_b32_sdwa v12, v22, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v30, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v34, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v31, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr22
 ; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
 ; VI-NEXT:    ; implicit-def: $vgpr31
-; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    ; implicit-def: $vgpr57
 ; VI-NEXT:    ; implicit-def: $vgpr59
-; VI-NEXT:    ; implicit-def: $vgpr62
-; VI-NEXT:    ; implicit-def: $vgpr18
-; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr36
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_or_b32_sdwa v0, v0, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v51 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr49
 ; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr52
-; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr23
-; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr40
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_or_b32_sdwa v8, v8, v45 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v4, v4, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr52
 ; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr40
-; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -83145,23 +83102,23 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr25
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v49, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v48, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v55, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v55, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v44, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v36, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v18, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v43, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v26, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v44, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v61, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
@@ -83195,98 +83152,92 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; kill: killed $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr49
-; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr27
 ; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr36
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr37
 ; VI-NEXT:    ; implicit-def: $vgpr44
-; VI-NEXT:    ; implicit-def: $vgpr61
-; VI-NEXT:    ; implicit-def: $vgpr29
-; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr45
 ; VI-NEXT:    ; implicit-def: $vgpr56
 ; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:  .LBB106_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB106_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v38
-; VI-NEXT:    v_add_u16_e32 v2, 3, v44
+; VI-NEXT:    v_add_u16_e32 v0, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v0, v32, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v14, v26, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_add_u16_e32 v2, 3, v35
 ; VI-NEXT:    v_mov_b32_e32 v3, 0x300
-; VI-NEXT:    v_or_b32_sdwa v2, v18, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v18, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v37
-; VI-NEXT:    v_or_b32_sdwa v24, v24, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v16, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v37
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v38, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v12, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v43
-; VI-NEXT:    v_or_b32_sdwa v16, v63, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v22
-; VI-NEXT:    v_or_b32_sdwa v0, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v11, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v36
-; VI-NEXT:    v_or_b32_sdwa v22, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v26, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v28
+; VI-NEXT:    v_add_u16_e32 v0, 3, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v11, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v18
+; VI-NEXT:    v_or_b32_sdwa v18, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v10, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v55
-; VI-NEXT:    v_or_b32_sdwa v28, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v20
+; VI-NEXT:    v_add_u16_e32 v0, 3, v44
+; VI-NEXT:    v_or_b32_sdwa v22, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v41
 ; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v2, 3, v35
 ; VI-NEXT:    v_add_u16_sdwa v9, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v48
-; VI-NEXT:    v_or_b32_sdwa v20, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v0, 3, v39
+; VI-NEXT:    v_add_u16_e32 v0, 3, v55
+; VI-NEXT:    v_or_b32_sdwa v14, v28, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v28, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v29
 ; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v49
-; VI-NEXT:    v_or_b32_sdwa v30, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v27
+; VI-NEXT:    v_or_b32_sdwa v27, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v2, 3, v34
+; VI-NEXT:    v_or_b32_sdwa v2, v24, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v13, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v1, 3, v61
-; VI-NEXT:    v_or_b32_sdwa v15, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 3, v63
+; VI-NEXT:    v_or_b32_sdwa v15, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v1, 3, v31
-; VI-NEXT:    v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v26, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
-; VI-NEXT:    v_or_b32_e32 v12, v16, v12
-; VI-NEXT:    v_add_u16_e32 v16, 0x300, v24
+; VI-NEXT:    v_or_b32_sdwa v1, v36, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v24, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
-; VI-NEXT:    v_or_b32_e32 v13, v16, v13
-; VI-NEXT:    v_or_b32_e32 v14, v14, v26
-; VI-NEXT:    v_or_b32_e32 v15, v15, v18
+; VI-NEXT:    v_or_b32_e32 v14, v14, v24
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v29, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v29, v42, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -83319,54 +83270,58 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v6, v17, v6
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v29
 ; VI-NEXT:    v_or_b32_e32 v7, v17, v7
-; VI-NEXT:    v_add_u16_e32 v17, 0x300, v30
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v27
 ; VI-NEXT:    v_or_b32_e32 v8, v17, v8
-; VI-NEXT:    v_add_u16_e32 v17, 0x300, v20
-; VI-NEXT:    v_or_b32_e32 v9, v17, v9
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v28
-; VI-NEXT:    v_or_b32_e32 v10, v17, v10
+; VI-NEXT:    v_or_b32_e32 v9, v17, v9
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v22
+; VI-NEXT:    v_or_b32_e32 v10, v17, v10
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v18
 ; VI-NEXT:    v_or_b32_e32 v11, v17, v11
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v26
+; VI-NEXT:    v_or_b32_e32 v12, v17, v12
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v20
+; VI-NEXT:    v_or_b32_e32 v13, v17, v13
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v42, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v19, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v23, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v23, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v2, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v27, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v30, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v1, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v31, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v31, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v3, v50, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v0, v3, v0
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v31
 ; VI-NEXT:    v_or_b32_e32 v1, v3, v1
-; VI-NEXT:    v_add_u16_e32 v3, 0x300, v27
+; VI-NEXT:    v_add_u16_e32 v3, 0x300, v30
 ; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v23
 ; VI-NEXT:    v_or_b32_e32 v3, v3, v19
@@ -83426,99 +83381,100 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:80
 ; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:88
 ; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:96
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v33, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v7
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v9
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v11
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v15
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v17
+; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v17
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v19
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v23
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
+; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
-; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v29
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v25, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v11
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v15
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(20)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(19)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v10
+; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v14
 ; GFX9-NEXT:    s_waitcnt vmcnt(17)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v12
+; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v14
+; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v16
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v18
+; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v20
+; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v24
-; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    v_lshlrev_b16_e32 v18, 8, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v24, 8, v28
+; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v31
+; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v31, 8, v32
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v34, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    v_lshlrev_b16_e32 v26, 8, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v33
+; GFX9-NEXT:    v_lshlrev_b16_e32 v28, 8, v31
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v32
+; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB106_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
@@ -83526,108 +83482,107 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v39, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v63, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v35, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v34, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v16, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v16, v38, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr39
-; GFX9-NEXT:    ; implicit-def: $vgpr62
-; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr35
-; GFX9-NEXT:    ; implicit-def: $vgpr34
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
+; GFX9-NEXT:    v_or_b32_sdwa v9, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v41, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_or_b32_sdwa v12, v26, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v37, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v20, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr31
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
-; GFX9-NEXT:    ; implicit-def: $vgpr36
-; GFX9-NEXT:    ; implicit-def: $vgpr26
-; GFX9-NEXT:    ; implicit-def: $vgpr20
-; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr30
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s6
 ; GFX9-NEXT:    v_perm_b32 v1, v3, v2, s6
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr49
 ; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr39
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
-; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr53
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr23
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v2, v3, v2, s6
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v3, v4, v3, s6
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr55
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v5, v6, v5, s6
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v6, v7, v6, s6
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr40
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v7, v8, v7, s6
-; GFX9-NEXT:    v_or_b32_sdwa v8, v51, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
-; GFX9-NEXT:    v_or_b32_sdwa v9, v52, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v42, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
-; GFX9-NEXT:    v_or_b32_sdwa v10, v42, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v47, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    v_or_b32_sdwa v11, v37, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v35, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    v_or_b32_sdwa v12, v44, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v16, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
-; GFX9-NEXT:    v_or_b32_sdwa v13, v28, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v24, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    v_or_b32_sdwa v14, v47, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v32, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    v_or_b32_sdwa v15, v18, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v38, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v16, v36, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
@@ -83661,111 +83616,110 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; kill: killed $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
-; GFX9-NEXT:    ; implicit-def: $vgpr51
-; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr29
 ; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr37
-; GFX9-NEXT:    ; implicit-def: $vgpr44
-; GFX9-NEXT:    ; implicit-def: $vgpr28
-; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr47
-; GFX9-NEXT:    ; implicit-def: $vgpr18
-; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; implicit-def: $vgpr22
-; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr24
-; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:  .LBB106_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB106_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
-; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v18
-; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v0, v28, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v28
-; GFX9-NEXT:    v_or_b32_sdwa v0, v30, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v24
+; GFX9-NEXT:    v_or_b32_sdwa v0, v22, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v20
+; GFX9-NEXT:    v_or_b32_sdwa v0, v18, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v16
-; GFX9-NEXT:    v_or_b32_sdwa v0, v26, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v44
-; GFX9-NEXT:    v_or_b32_sdwa v0, v22, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v63, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v34
-; GFX9-NEXT:    v_or_b32_sdwa v0, v36, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v37
+; GFX9-NEXT:    v_or_b32_sdwa v0, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v35
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v35
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v26
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v24, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
+; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v47
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v63
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v41
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v52
+; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v27
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v51
+; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v29
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v25
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v25, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_add_u16_e32 v3, 3, v32
-; GFX9-NEXT:    v_or_b32_sdwa v3, v20, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v3
+; GFX9-NEXT:    v_add_u16_e32 v3, 3, v31
+; GFX9-NEXT:    v_or_b32_sdwa v3, v30, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v3
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v32
+; GFX9-NEXT:    v_or_b32_sdwa v2, v34, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v2
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v38
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v36
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v1
+; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v1
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v8, v30, v8, s6
-; GFX9-NEXT:    v_perm_b32 v9, v28, v9, s6
-; GFX9-NEXT:    v_perm_b32 v10, v26, v10, s6
-; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s6
-; GFX9-NEXT:    v_perm_b32 v12, v22, v12, s6
-; GFX9-NEXT:    v_perm_b32 v13, v16, v13, s6
-; GFX9-NEXT:    v_perm_b32 v14, v20, v14, s6
-; GFX9-NEXT:    v_perm_b32 v15, v18, v15, s6
+; GFX9-NEXT:    v_perm_b32 v8, v25, v8, s6
+; GFX9-NEXT:    v_perm_b32 v9, v24, v9, s6
+; GFX9-NEXT:    v_perm_b32 v10, v22, v10, s6
+; GFX9-NEXT:    v_perm_b32 v11, v20, v11, s6
+; GFX9-NEXT:    v_perm_b32 v12, v16, v12, s6
+; GFX9-NEXT:    v_perm_b32 v13, v18, v13, s6
+; GFX9-NEXT:    v_perm_b32 v14, v30, v14, s6
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s6
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v31, 3, v31
-; GFX9-NEXT:    v_or_b32_sdwa v31, v48, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v31, v39, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v31, 0x300, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v27, 0x300, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_perm_b32 v7, v27, v7, s6
+; GFX9-NEXT:    v_perm_b32 v7, v26, v7, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v25, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -83787,7 +83741,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v5, v19, v5, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -83798,40 +83752,40 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v4, v17, v4, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v3, v21, v3, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v25, 0x300, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v27, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_perm_b32 v2, v25, v2, s6
+; GFX9-NEXT:    v_perm_b32 v2, v27, v2, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v1, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v29, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v1, v29, v1, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; GFX9-NEXT:    v_perm_b32 v0, v31, v0, s6
 ; GFX9-NEXT:  .LBB106_4: ; %end
@@ -85910,57 +85864,34 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32f16_scalar:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v1 :: v_dual_mov_b32 v52, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v86, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v15 :: v_dual_mov_b32 v34, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v37, v13 :: v_dual_mov_b32 v54, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v12 :: v_dual_mov_b32 v50, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v49, v11 :: v_dual_mov_b32 v48, v8
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v9 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v6 :: v_dual_mov_b32 v38, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v3 :: v_dual_mov_b32 v39, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v86
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB107_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -85984,8 +85915,8 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s10, s11
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v32
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s7, s8, s9
 ; GFX11-TRUE16-NEXT:    s_and_b32 s8, s24, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s25, 8
@@ -85993,14 +85924,14 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s10, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v31
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v52
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v18
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v53
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v50
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
@@ -86009,10 +85940,10 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v3, 0xffff, s9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v5, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v20
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v30
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v48
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v35
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v49
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v8
@@ -86020,63 +85951,63 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v34
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v17
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v6
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v0, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v19
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v39
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v23
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v37
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v49
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v66
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v3, 16, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v12, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v64
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v85
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v84
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v67
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v70
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v69
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v82
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v80
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v12, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v83
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v83
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v86, v87
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v50
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v80
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v68
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v67
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v14, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v11
@@ -86135,151 +86066,151 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s1, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s10, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v80
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v84
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v83
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v85
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v71
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v80
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v71
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v54
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v70
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v67
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v82
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v68
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v81
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v65
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v69
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v66
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v66
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v64
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v22
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v6, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v18
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v64
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v26
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v28
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v27
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v29
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v48
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v24
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v37
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v25
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v37, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v9, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v36
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v20
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v34
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v32, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v54
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v7, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 0x300, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 8, v49
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v8, v7
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v28, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v25
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v16, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 0x300, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v23, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v20, v4
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v18
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v32
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v6
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v22, v4
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v23, v8
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v23, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v18, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v8, 0xffff, s4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v5, 16, v8
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v6, 16, v16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v23, 16, v8
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v25, 16, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v6, 16, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v9
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v25, 16, v8
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v16, 16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v17, 16, v6
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v29, 16, v18
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v10, 16, v19
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v37, 16, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v23, 16, v6
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v20, 16, v19
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v10, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v18, 16, v15
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v12, v14, 16, v12
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v13, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v14, v2, 16, v16
@@ -93581,129 +93512,123 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(5)
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:132
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:128
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:128
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:28
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 8, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 8, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 24, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v14, 8, v29
 ; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr51
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr13
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr21
 ; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; implicit-def: $vgpr23
 ; SI-NEXT:    ; implicit-def: $vgpr52
 ; SI-NEXT:    ; implicit-def: $vgpr53
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr27
 ; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr29
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v22, 24, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v17
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_lshlrev_b32_e32 v14, 24, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v20
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 24, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 24, v24
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v12, 24, v28
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_lshlrev_b32_e32 v18, 24, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 8, v31
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_lshlrev_b32_e32 v30, 24, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 24, v32
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 8, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 24, v33
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 24, v31
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 8, v34
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v32
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 8, v33
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 24, v34
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 24, v35
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 8, v36
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 24, v37
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:12
+; SI-NEXT:    v_lshlrev_b32_e32 v61, 24, v36
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr36
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v3
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:116
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:124
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v5
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v7
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v11
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v15
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 24, v19
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:28
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:76
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:4
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:116
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:124
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v5
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v7
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v11
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v15
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v19
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76
+; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:40
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v23
 ; SI-NEXT:    ; kill: killed $vgpr3
 ; SI-NEXT:    ; implicit-def: $vgpr3
 ; SI-NEXT:    ; kill: killed $vgpr3
@@ -93713,7 +93638,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr11
 ; SI-NEXT:    ; implicit-def: $vgpr15
 ; SI-NEXT:    ; implicit-def: $vgpr19
-; SI-NEXT:    ; implicit-def: $vgpr29
+; SI-NEXT:    ; implicit-def: $vgpr23
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB110_2
@@ -93729,7 +93654,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 24, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
@@ -93740,7 +93665,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
@@ -93763,7 +93688,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 24, v7
 ; SI-NEXT:    v_or_b32_e32 v33, v7, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
@@ -93795,105 +93720,101 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT:    v_or_b32_e32 v1, v11, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v10, v11, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v11, v2, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT:    v_or_b32_e32 v11, v2, v10
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
+; SI-NEXT:    v_or_b32_e32 v32, v10, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr10
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_or_b32_e32 v13, v6, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT:    v_or_b32_e32 v2, v2, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v2
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    ; implicit-def: $vgpr14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_or_b32_e32 v15, v1, v2
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
-; SI-NEXT:    v_or_b32_e32 v32, v2, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v36, v2, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v13, v10, v1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    ; implicit-def: $vgpr10
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v15, v14, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT:    ; implicit-def: $vgpr14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v36, v1, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v62
+; SI-NEXT:    v_or_b32_e32 v37, v22, v1
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v62
+; SI-NEXT:    v_or_b32_e32 v1, v1, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v1
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr22
+; SI-NEXT:    ; implicit-def: $vgpr4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v37, v26, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v24
-; SI-NEXT:    v_or_b32_e32 v0, v0, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v19, v18, v0
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v19, v0, v1
 ; SI-NEXT:    v_and_b32_e32 v0, 0xff, v44
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v28
 ; SI-NEXT:    v_or_b32_e32 v48, v1, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v57
-; SI-NEXT:    ; implicit-def: $vgpr24
-; SI-NEXT:    ; implicit-def: $vgpr17
-; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    ; implicit-def: $vgpr8
-; SI-NEXT:    ; implicit-def: $vgpr18
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v24
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v21, v30, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v22
-; SI-NEXT:    v_or_b32_e32 v0, v0, v12
+; SI-NEXT:    v_or_b32_e32 v21, v18, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v47
+; SI-NEXT:    v_or_b32_e32 v0, v0, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v6
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v20
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v23, v25, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v58
+; SI-NEXT:    v_or_b32_e32 v23, v8, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v49
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v59
 ; SI-NEXT:    v_or_b32_e32 v52, v1, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v46
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v56
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v53, v16, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT:    v_or_b32_e32 v0, v0, v47
+; SI-NEXT:    v_or_b32_e32 v53, v12, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v60
+; SI-NEXT:    v_or_b32_e32 v0, v0, v57
 ; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v9
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v26
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v27, v45, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v35
+; SI-NEXT:    v_or_b32_e32 v27, v46, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v30
 ; SI-NEXT:    v_or_b32_e32 v40, v1, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v59
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v29, v56, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v20
-; SI-NEXT:    v_or_b32_e32 v0, v0, v60
+; SI-NEXT:    v_or_b32_e32 v29, v58, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v45
+; SI-NEXT:    v_or_b32_e32 v0, v0, v35
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xff, v28
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v16
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v31, v61, v0
 ; SI-NEXT:    ; implicit-def: $vgpr0
@@ -93955,177 +93876,183 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr0
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    ; implicit-def: $vgpr6
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr4
-; SI-NEXT:    ; implicit-def: $vgpr9
-; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr28
+; SI-NEXT:    ; implicit-def: $vgpr24
+; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr59
-; SI-NEXT:    ; implicit-def: $vgpr20
-; SI-NEXT:    ; implicit-def: $vgpr28
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr26
+; SI-NEXT:    ; implicit-def: $vgpr25
+; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr9
+; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr16
 ; SI-NEXT:    ; kill: killed $vgpr0
 ; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:    ; implicit-def: $vgpr0
-; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr18
+; SI-NEXT:    ; implicit-def: $vgpr17
+; SI-NEXT:    ; implicit-def: $vgpr8
 ; SI-NEXT:    ; implicit-def: $vgpr12
-; SI-NEXT:    ; implicit-def: $vgpr25
-; SI-NEXT:    ; implicit-def: $vgpr16
-; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr56
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr46
+; SI-NEXT:    ; implicit-def: $vgpr58
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:  .LBB110_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB110_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v20
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v45
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v28
-; SI-NEXT:    v_or_b32_e32 v3, v60, v3
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v16
+; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 0x300, v3
 ; SI-NEXT:    v_or_b32_e32 v5, v61, v5
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v35
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v25
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v49
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 8, v30
 ; SI-NEXT:    v_or_b32_e32 v5, v7, v5
 ; SI-NEXT:    s_movk_i32 s6, 0x300
 ; SI-NEXT:    s_mov_b32 s7, 0x3000000
-; SI-NEXT:    v_add_i32_e32 v20, vcc, s7, v3
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v5
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v59
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v9
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v5, v56, v5
+; SI-NEXT:    v_or_b32_e32 v5, v58, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v28, vcc, s7, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v4
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v3
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v60
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v9
-; SI-NEXT:    v_or_b32_e32 v3, v47, v3
-; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v26
+; SI-NEXT:    v_or_b32_e32 v3, v57, v3
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v4, v45, v4
-; SI-NEXT:    v_or_b32_e32 v3, v4, v3
-; SI-NEXT:    v_add_i32_e32 v4, vcc, s7, v3
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v58
+; SI-NEXT:    v_or_b32_e32 v5, v46, v5
+; SI-NEXT:    v_or_b32_e32 v3, v5, v3
+; SI-NEXT:    v_add_i32_e32 v25, vcc, s7, v3
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v49
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v57
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v59
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v46
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v56
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v5, v16, v5
+; SI-NEXT:    v_or_b32_e32 v5, v12, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s7, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v22
+; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v3
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v47
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v6
-; SI-NEXT:    v_or_b32_e32 v3, v12, v3
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v20
+; SI-NEXT:    v_or_b32_e32 v3, v17, v3
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_or_b32_e32 v5, v25, v5
+; SI-NEXT:    v_or_b32_e32 v5, v8, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v6, vcc, s7, v3
+; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v3
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v44
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v62
-; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff0000, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v6
-; SI-NEXT:    v_and_b32_e32 v53, 0xffff0000, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v9
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff0000, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v4
-; SI-NEXT:    v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v28
-; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v20
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v20
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
-; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v30, v5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v28
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v12, vcc, s7, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v24
-; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v17
-; SI-NEXT:    v_or_b32_e32 v3, v8, v3
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v24
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_or_b32_e32 v5, v18, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v8, vcc, s7, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff0000, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v8
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff0000, v12
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v12
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
+; SI-NEXT:    v_add_i32_e32 v17, vcc, s7, v3
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v62
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
-; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff0000, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v17
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff0000, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
+; SI-NEXT:    v_and_b32_e32 v53, 0xffff0000, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v12
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff0000, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v25
+; SI-NEXT:    v_and_b32_e32 v29, 0xffff0000, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v9
+; SI-NEXT:    v_and_b32_e32 v31, 0xffff0000, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v16
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v0, v0, v4
+; SI-NEXT:    v_or_b32_e32 v0, v0, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff0000, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v0
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v26, v5
-; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s7, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v1, v1, v5
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    v_and_b32_e32 v37, 0xffff0000, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v16
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v0, v0, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
-; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
+; SI-NEXT:    v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v4, v22, v4
+; SI-NEXT:    v_or_b32_e32 v3, v4, v3
+; SI-NEXT:    v_add_i32_e32 v4, vcc, s7, v3
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v37, 0xffff0000, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v14, v3
-; SI-NEXT:    v_or_b32_e32 v0, v3, v0
+; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT:    v_or_b32_e32 v1, v1, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v0
+; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff0000, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v1
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
@@ -94138,50 +94065,50 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT:    v_or_b32_e32 v5, v10, v5
+; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v3
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s7, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v10
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_or_b32_e32 v3, v10, v3
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT:    v_or_b32_e32 v2, v2, v5
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    v_or_b32_e32 v1, v2, v1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v1
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
-; SI-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
-; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v5, v3
-; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT:    v_or_b32_e32 v5, v7, v5
+; SI-NEXT:    v_or_b32_e32 v3, v5, v3
+; SI-NEXT:    v_add_i32_e32 v10, vcc, s7, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
-; SI-NEXT:    v_and_b32_e32 v55, 0xffff0000, v2
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v2
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v55, 0xffff0000, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v10
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -94196,7 +94123,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s7, v3
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v11
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
@@ -94253,21 +94180,21 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s7, v3
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v7
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff0000, v11
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v1
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v10
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff0000, v2
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff0000, v6
 ; SI-NEXT:  .LBB110_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mov_b32_e32 v2, v43
 ; SI-NEXT:    v_mov_b32_e32 v10, v41
-; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_mov_b32_e32 v28, v40
 ; SI-NEXT:    v_mov_b32_e32 v30, v42
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
@@ -94293,7 +94220,6 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v12, v32
 ; SI-NEXT:    v_mov_b32_e32 v14, v34
 ; SI-NEXT:    v_mov_b32_e32 v16, v36
-; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_mov_b32_e32 v17, v37
 ; SI-NEXT:    v_mov_b32_e32 v18, v38
 ; SI-NEXT:    v_mov_b32_e32 v20, v48
@@ -94339,78 +94265,78 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; VI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:132
-; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
+; VI-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; VI-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:88
 ; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:96
 ; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:104
 ; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:112
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
 ; VI-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_ushort v48, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_ushort v43, off, s[0:3], s32 offset:68
-; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:36
-; VI-NEXT:    buffer_load_ushort v49, off, s[0:3], s32 offset:4
-; VI-NEXT:    buffer_load_ushort v61, off, s[0:3], s32 offset:116
-; VI-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:124
-; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v1
-; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v3
-; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v5
-; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v7
-; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v9
-; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v11
-; VI-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
-; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v15
+; VI-NEXT:    v_lshlrev_b16_e32 v40, 8, v27
+; VI-NEXT:    v_lshlrev_b16_e32 v42, 8, v29
+; VI-NEXT:    buffer_load_ushort v46, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v55, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ushort v27, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:116
+; VI-NEXT:    v_lshlrev_b16_e32 v39, 8, v1
+; VI-NEXT:    v_lshlrev_b16_e32 v48, 8, v3
+; VI-NEXT:    v_lshlrev_b16_e32 v49, 8, v5
+; VI-NEXT:    v_lshlrev_b16_e32 v50, 8, v7
+; VI-NEXT:    v_lshlrev_b16_e32 v51, 8, v9
+; VI-NEXT:    v_lshlrev_b16_e32 v52, 8, v11
+; VI-NEXT:    v_lshlrev_b16_e32 v53, 8, v13
+; VI-NEXT:    v_lshlrev_b16_e32 v54, 8, v15
 ; VI-NEXT:    v_lshlrev_b16_e32 v17, 8, v17
 ; VI-NEXT:    v_lshlrev_b16_e32 v19, 8, v19
 ; VI-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
 ; VI-NEXT:    v_lshlrev_b16_e32 v23, 8, v23
 ; VI-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
-; VI-NEXT:    v_lshlrev_b16_e32 v27, 8, v27
-; VI-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
 ; VI-NEXT:    s_waitcnt vmcnt(14)
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v24
-; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v0
-; VI-NEXT:    v_lshlrev_b16_e32 v46, 8, v2
-; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v4
-; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v6
-; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v8
-; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v10
-; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v12
-; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v14
-; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v16
-; VI-NEXT:    v_lshlrev_b16_e32 v63, 8, v18
-; VI-NEXT:    v_lshlrev_b16_e32 v16, 8, v20
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_lshlrev_b16_e32 v43, 8, v2
+; VI-NEXT:    v_lshlrev_b16_e32 v45, 8, v4
+; VI-NEXT:    v_lshlrev_b16_e32 v47, 8, v6
+; VI-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
+; VI-NEXT:    v_lshlrev_b16_e32 v57, 8, v10
+; VI-NEXT:    v_lshlrev_b16_e32 v58, 8, v12
+; VI-NEXT:    v_lshlrev_b16_e32 v59, 8, v14
+; VI-NEXT:    v_lshlrev_b16_e32 v60, 8, v16
+; VI-NEXT:    v_lshlrev_b16_e32 v61, 8, v18
+; VI-NEXT:    v_lshlrev_b16_e32 v62, 8, v20
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v22
+; VI-NEXT:    v_lshlrev_b16_e32 v38, 8, v22
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_lshlrev_b16_e32 v18, 8, v26
+; VI-NEXT:    v_lshlrev_b16_e32 v20, 8, v24
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshlrev_b16_e32 v26, 8, v28
+; VI-NEXT:    v_lshlrev_b16_e32 v24, 8, v26
+; VI-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:124
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v30
+; VI-NEXT:    v_lshlrev_b16_e32 v36, 8, v30
 ; VI-NEXT:    s_waitcnt vmcnt(9)
-; VI-NEXT:    v_lshlrev_b16_e32 v34, 8, v31
+; VI-NEXT:    v_lshlrev_b16_e32 v33, 8, v31
 ; VI-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_ushort v34, off, s[0:3], s32 offset:92
 ; VI-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:84
 ; VI-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:68
 ; VI-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:52
+; VI-NEXT:    v_lshlrev_b16_e32 v28, 8, v28
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_lshlrev_b16_e32 v32, 8, v32
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -94422,79 +94348,80 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT:    v_or_b32_sdwa v9, v39, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v10, v20, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v28, v59 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v22, v62 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v30, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v35, v18 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v31, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v16, v38, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr39
-; VI-NEXT:    ; implicit-def: $vgpr20
-; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    s_waitcnt vmcnt(14)
+; VI-NEXT:    v_or_b32_sdwa v9, v29, v47 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v41, v57 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v46, v59 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_or_b32_sdwa v12, v22, v61 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v30, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v34, v24 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v31, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v16, v16, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr29
+; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr46
 ; VI-NEXT:    ; implicit-def: $vgpr22
 ; VI-NEXT:    ; implicit-def: $vgpr30
-; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr34
 ; VI-NEXT:    ; implicit-def: $vgpr31
-; VI-NEXT:    ; implicit-def: $vgpr38
 ; VI-NEXT:    ; implicit-def: $vgpr47
 ; VI-NEXT:    ; implicit-def: $vgpr57
 ; VI-NEXT:    ; implicit-def: $vgpr59
-; VI-NEXT:    ; implicit-def: $vgpr62
-; VI-NEXT:    ; implicit-def: $vgpr18
-; VI-NEXT:    ; implicit-def: $vgpr33
+; VI-NEXT:    ; implicit-def: $vgpr61
+; VI-NEXT:    ; implicit-def: $vgpr38
+; VI-NEXT:    ; implicit-def: $vgpr24
+; VI-NEXT:    ; implicit-def: $vgpr36
 ; VI-NEXT:    ; implicit-def: $vgpr32
 ; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(7)
-; VI-NEXT:    v_or_b32_sdwa v0, v0, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v51 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; VI-NEXT:    ; implicit-def: $vgpr39
+; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr49
 ; VI-NEXT:    ; implicit-def: $vgpr50
-; VI-NEXT:    ; implicit-def: $vgpr51
-; VI-NEXT:    ; implicit-def: $vgpr52
-; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(5)
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr19
 ; VI-NEXT:    ; implicit-def: $vgpr23
-; VI-NEXT:    ; implicit-def: $vgpr27
+; VI-NEXT:    ; implicit-def: $vgpr40
 ; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_or_b32_sdwa v8, v8, v45 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; VI-NEXT:    ; implicit-def: $vgpr45
+; VI-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr43
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_or_b32_sdwa v2, v2, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_or_b32_sdwa v4, v4, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    ; implicit-def: $vgpr51
+; VI-NEXT:    ; implicit-def: $vgpr52
 ; VI-NEXT:    ; implicit-def: $vgpr54
-; VI-NEXT:    ; implicit-def: $vgpr40
-; VI-NEXT:    ; implicit-def: $vgpr42
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v3, v3, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v3, v3, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; VI-NEXT:    ; implicit-def: $vgpr41
+; VI-NEXT:    ; implicit-def: $vgpr53
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -94511,23 +94438,23 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; VI-NEXT:    ; implicit-def: $vgpr25
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_or_b32_sdwa v7, v7, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v8, v49, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v8, v27, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v9, v48, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v55, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v10, v55, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v10, v44, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v11, v36, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v11, v18, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v12, v43, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v12, v26, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v13, v37, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v13, v37, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v14, v44, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v14, v35, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v15, v61, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v15, v63, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
@@ -94561,98 +94488,92 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    ; kill: killed $vgpr16
 ; VI-NEXT:    ; implicit-def: $vgpr16
 ; VI-NEXT:    ; kill: killed $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr49
-; VI-NEXT:    ; implicit-def: $vgpr48
+; VI-NEXT:    ; implicit-def: $vgpr27
 ; VI-NEXT:    ; implicit-def: $vgpr55
-; VI-NEXT:    ; implicit-def: $vgpr36
-; VI-NEXT:    ; implicit-def: $vgpr43
-; VI-NEXT:    ; implicit-def: $vgpr37
 ; VI-NEXT:    ; implicit-def: $vgpr44
-; VI-NEXT:    ; implicit-def: $vgpr61
-; VI-NEXT:    ; implicit-def: $vgpr29
-; VI-NEXT:    ; implicit-def: $vgpr46
+; VI-NEXT:    ; implicit-def: $vgpr18
+; VI-NEXT:    ; implicit-def: $vgpr26
+; VI-NEXT:    ; implicit-def: $vgpr37
+; VI-NEXT:    ; implicit-def: $vgpr35
+; VI-NEXT:    ; implicit-def: $vgpr63
+; VI-NEXT:    ; implicit-def: $vgpr16
+; VI-NEXT:    ; implicit-def: $vgpr42
+; VI-NEXT:    ; implicit-def: $vgpr45
 ; VI-NEXT:    ; implicit-def: $vgpr56
 ; VI-NEXT:    ; implicit-def: $vgpr58
 ; VI-NEXT:    ; implicit-def: $vgpr60
-; VI-NEXT:    ; implicit-def: $vgpr63
-; VI-NEXT:    ; implicit-def: $vgpr16
-; VI-NEXT:    ; implicit-def: $vgpr24
-; VI-NEXT:    ; implicit-def: $vgpr26
-; VI-NEXT:    ; implicit-def: $vgpr34
+; VI-NEXT:    ; implicit-def: $vgpr62
+; VI-NEXT:    ; implicit-def: $vgpr20
+; VI-NEXT:    ; implicit-def: $vgpr28
+; VI-NEXT:    ; implicit-def: $vgpr33
 ; VI-NEXT:  .LBB110_2: ; %Flow
 ; VI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; VI-NEXT:    s_cbranch_execz .LBB110_4
 ; VI-NEXT:  ; %bb.3: ; %cmp.true
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v38
-; VI-NEXT:    v_add_u16_e32 v2, 3, v44
+; VI-NEXT:    v_add_u16_e32 v0, 3, v16
 ; VI-NEXT:    v_or_b32_sdwa v0, v32, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v14, v26, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_add_u16_e32 v2, 3, v35
 ; VI-NEXT:    v_mov_b32_e32 v3, 0x300
-; VI-NEXT:    v_or_b32_sdwa v2, v18, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v18, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v37
-; VI-NEXT:    v_or_b32_sdwa v24, v24, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v16, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v37
+; VI-NEXT:    v_or_b32_sdwa v20, v20, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v30
-; VI-NEXT:    v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v38, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v12, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v43
-; VI-NEXT:    v_or_b32_sdwa v16, v63, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v22
-; VI-NEXT:    v_or_b32_sdwa v0, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v11, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v36
-; VI-NEXT:    v_or_b32_sdwa v22, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v26
+; VI-NEXT:    v_or_b32_sdwa v26, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v28
+; VI-NEXT:    v_add_u16_e32 v0, 3, v22
+; VI-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v11, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v0, 3, v18
+; VI-NEXT:    v_or_b32_sdwa v18, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v46
 ; VI-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v10, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v55
-; VI-NEXT:    v_or_b32_sdwa v28, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v0, 3, v20
+; VI-NEXT:    v_add_u16_e32 v0, 3, v44
+; VI-NEXT:    v_or_b32_sdwa v22, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v41
 ; VI-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v2, 3, v35
 ; VI-NEXT:    v_add_u16_sdwa v9, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v48
-; VI-NEXT:    v_or_b32_sdwa v20, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_e32 v0, 3, v39
+; VI-NEXT:    v_add_u16_e32 v0, 3, v55
+; VI-NEXT:    v_or_b32_sdwa v14, v28, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v28, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v29
 ; VI-NEXT:    v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v8, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v0, 3, v49
-; VI-NEXT:    v_or_b32_sdwa v30, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v0, 3, v27
+; VI-NEXT:    v_or_b32_sdwa v27, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; VI-NEXT:    v_add_u16_e32 v2, 3, v34
+; VI-NEXT:    v_or_b32_sdwa v2, v24, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v13, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v1, 3, v61
-; VI-NEXT:    v_or_b32_sdwa v15, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 3, v63
+; VI-NEXT:    v_or_b32_sdwa v15, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v1, 3, v31
-; VI-NEXT:    v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_add_u16_sdwa v26, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_add_u16_e32 v16, 0x300, v16
-; VI-NEXT:    v_or_b32_e32 v12, v16, v12
-; VI-NEXT:    v_add_u16_e32 v16, 0x300, v24
+; VI-NEXT:    v_or_b32_sdwa v1, v36, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_add_u16_sdwa v24, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    v_add_u16_e32 v14, 0x300, v14
 ; VI-NEXT:    v_add_u16_e32 v15, 0x300, v15
-; VI-NEXT:    v_or_b32_e32 v13, v16, v13
-; VI-NEXT:    v_or_b32_e32 v14, v14, v26
-; VI-NEXT:    v_or_b32_e32 v15, v15, v18
+; VI-NEXT:    v_or_b32_e32 v14, v14, v24
+; VI-NEXT:    v_or_b32_e32 v15, v15, v16
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v7, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v29, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v29, v42, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v6, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -94685,54 +94606,58 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; VI-NEXT:    v_or_b32_e32 v6, v17, v6
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v29
 ; VI-NEXT:    v_or_b32_e32 v7, v17, v7
-; VI-NEXT:    v_add_u16_e32 v17, 0x300, v30
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v27
 ; VI-NEXT:    v_or_b32_e32 v8, v17, v8
-; VI-NEXT:    v_add_u16_e32 v17, 0x300, v20
-; VI-NEXT:    v_or_b32_e32 v9, v17, v9
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v28
-; VI-NEXT:    v_or_b32_e32 v10, v17, v10
+; VI-NEXT:    v_or_b32_e32 v9, v17, v9
 ; VI-NEXT:    v_add_u16_e32 v17, 0x300, v22
+; VI-NEXT:    v_or_b32_e32 v10, v17, v10
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v18
 ; VI-NEXT:    v_or_b32_e32 v11, v17, v11
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v26
+; VI-NEXT:    v_or_b32_e32 v12, v17, v12
+; VI-NEXT:    v_add_u16_e32 v17, 0x300, v20
+; VI-NEXT:    v_or_b32_e32 v13, v17, v13
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v42, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v19, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v23, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v23, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v2, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v27, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v30, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v1, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v31, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v31, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v0, 3, v0
-; VI-NEXT:    v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_add_u16_e32 v3, 3, v3
-; VI-NEXT:    v_or_b32_sdwa v3, v50, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v3
 ; VI-NEXT:    v_or_b32_e32 v0, v3, v0
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v31
 ; VI-NEXT:    v_or_b32_e32 v1, v3, v1
-; VI-NEXT:    v_add_u16_e32 v3, 0x300, v27
+; VI-NEXT:    v_add_u16_e32 v3, 0x300, v30
 ; VI-NEXT:    v_or_b32_e32 v2, v3, v2
 ; VI-NEXT:    v_add_u16_e32 v3, 0x300, v23
 ; VI-NEXT:    v_or_b32_e32 v3, v3, v19
@@ -94792,99 +94717,100 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:132
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:132
 ; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_ushort v2, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_ushort v4, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ushort v6, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ushort v8, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ushort v10, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ushort v12, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_ushort v14, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_ushort v22, off, s[0:3], s32 offset:80
 ; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:88
 ; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:96
 ; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:104
-; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_ushort v33, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_ushort v52, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ushort v39, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_ushort v44, off, s[0:3], s32 offset:68
-; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:36
-; GFX9-NEXT:    buffer_load_ushort v51, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v1
-; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v5
-; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v7
-; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v9
-; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v11
-; GFX9-NEXT:    v_lshlrev_b16_e32 v41, 8, v13
-; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v15
-; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v17
+; GFX9-NEXT:    buffer_load_ushort v30, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:120
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    v_lshlrev_b16_e32 v55, 8, v17
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v17, 8, v19
-; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v19, 8, v23
-; GFX9-NEXT:    v_lshlrev_b16_e32 v25, 8, v25
+; GFX9-NEXT:    v_lshlrev_b16_e32 v40, 8, v25
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v23, 8, v27
-; GFX9-NEXT:    v_lshlrev_b16_e32 v29, 8, v29
-; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    v_lshlrev_b16_e32 v44, 8, v29
+; GFX9-NEXT:    buffer_load_ushort v41, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_ushort v47, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ushort v42, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ushort v25, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ushort v29, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ushort v38, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    buffer_load_ushort v36, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    v_lshlrev_b16_e32 v48, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v39, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v50, 8, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v49, 8, v7
+; GFX9-NEXT:    v_lshlrev_b16_e32 v52, 8, v9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v51, 8, v11
+; GFX9-NEXT:    v_lshlrev_b16_e32 v54, 8, v13
+; GFX9-NEXT:    v_lshlrev_b16_e32 v53, 8, v15
+; GFX9-NEXT:    v_lshlrev_b16_e32 v21, 8, v21
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_waitcnt vmcnt(24)
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v22
+; GFX9-NEXT:    v_lshlrev_b16_e32 v43, 8, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v27, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v4
 ; GFX9-NEXT:    s_waitcnt vmcnt(22)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v46, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v6
 ; GFX9-NEXT:    s_waitcnt vmcnt(21)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v45, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v8
 ; GFX9-NEXT:    s_waitcnt vmcnt(20)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v57, 8, v6
+; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v10
 ; GFX9-NEXT:    s_waitcnt vmcnt(19)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v56, 8, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(18)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v59, 8, v10
+; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v14
 ; GFX9-NEXT:    s_waitcnt vmcnt(17)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v58, 8, v12
+; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v61, 8, v14
+; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v18
 ; GFX9-NEXT:    s_waitcnt vmcnt(15)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v60, 8, v16
+; GFX9-NEXT:    v_lshlrev_b16_e32 v63, 8, v20
 ; GFX9-NEXT:    s_waitcnt vmcnt(14)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v18
+; GFX9-NEXT:    v_lshlrev_b16_e32 v62, 8, v22
 ; GFX9-NEXT:    s_waitcnt vmcnt(13)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v36, 8, v20
+; GFX9-NEXT:    v_lshlrev_b16_e32 v22, 8, v24
 ; GFX9-NEXT:    s_waitcnt vmcnt(12)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v24
-; GFX9-NEXT:    buffer_load_ushort v18, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    v_lshlrev_b16_e32 v18, 8, v26
 ; GFX9-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v24, 8, v28
+; GFX9-NEXT:    v_lshlrev_b16_e32 v34, 8, v28
 ; GFX9-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v20, 8, v31
+; GFX9-NEXT:    v_lshlrev_b16_e32 v30, 8, v30
 ; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v31, 8, v32
-; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_ushort v34, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_ushort v63, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_ushort v62, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    v_lshlrev_b16_e32 v26, 8, v26
-; GFX9-NEXT:    s_waitcnt vmcnt(16)
-; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v33
+; GFX9-NEXT:    v_lshlrev_b16_e32 v28, 8, v31
+; GFX9-NEXT:    s_waitcnt vmcnt(8)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v33, 8, v32
+; GFX9-NEXT:    buffer_load_ushort v31, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_ushort v32, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_ushort v20, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_ushort v24, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_ushort v37, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_ushort v16, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_ushort v26, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ushort v35, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB110_2
 ; GFX9-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
@@ -94892,108 +94818,107 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_waitcnt vmcnt(23)
-; GFX9-NEXT:    v_or_b32_sdwa v9, v39, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NEXT:    v_or_b32_sdwa v10, v62, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v11, v63, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v12, v35, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v13, v34, v36 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v14, v16, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v15, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v16, v38, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr39
-; GFX9-NEXT:    ; implicit-def: $vgpr62
-; GFX9-NEXT:    ; implicit-def: $vgpr63
-; GFX9-NEXT:    ; implicit-def: $vgpr35
-; GFX9-NEXT:    ; implicit-def: $vgpr34
-; GFX9-NEXT:    ; implicit-def: $vgpr32
-; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    s_waitcnt vmcnt(20)
+; GFX9-NEXT:    v_or_b32_sdwa v9, v25, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v27, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v41, v58 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(10)
+; GFX9-NEXT:    v_or_b32_sdwa v12, v26, v60 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v37, v62 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v20, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr26
+; GFX9-NEXT:    ; implicit-def: $vgpr37
+; GFX9-NEXT:    ; implicit-def: $vgpr20
+; GFX9-NEXT:    ; implicit-def: $vgpr31
 ; GFX9-NEXT:    ; implicit-def: $vgpr45
 ; GFX9-NEXT:    ; implicit-def: $vgpr56
 ; GFX9-NEXT:    ; implicit-def: $vgpr58
 ; GFX9-NEXT:    ; implicit-def: $vgpr60
-; GFX9-NEXT:    ; implicit-def: $vgpr36
-; GFX9-NEXT:    ; implicit-def: $vgpr26
-; GFX9-NEXT:    ; implicit-def: $vgpr20
-; GFX9-NEXT:    ; implicit-def: $vgpr33
+; GFX9-NEXT:    ; implicit-def: $vgpr62
+; GFX9-NEXT:    ; implicit-def: $vgpr18
+; GFX9-NEXT:    ; implicit-def: $vgpr30
 ; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s6
 ; GFX9-NEXT:    v_perm_b32 v1, v3, v2, s6
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(4)
 ; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(3)
 ; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr49
 ; GFX9-NEXT:    ; implicit-def: $vgpr48
-; GFX9-NEXT:    ; implicit-def: $vgpr53
+; GFX9-NEXT:    ; implicit-def: $vgpr39
 ; GFX9-NEXT:    ; implicit-def: $vgpr50
-; GFX9-NEXT:    ; implicit-def: $vgpr40
+; GFX9-NEXT:    ; implicit-def: $vgpr49
+; GFX9-NEXT:    ; implicit-def: $vgpr53
 ; GFX9-NEXT:    ; implicit-def: $vgpr17
 ; GFX9-NEXT:    ; implicit-def: $vgpr19
 ; GFX9-NEXT:    ; implicit-def: $vgpr23
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    ; implicit-def: $vgpr27
+; GFX9-NEXT:    v_or_b32_sdwa v8, v8, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    ; implicit-def: $vgpr43
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v51 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v2, v3, v2, s6
 ; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr55
-; GFX9-NEXT:    ; implicit-def: $vgpr54
+; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr51
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v3, v4, v3, s6
 ; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr41
+; GFX9-NEXT:    ; implicit-def: $vgpr54
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v43 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v4, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v4, v5, v4, s6
 ; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr43
+; GFX9-NEXT:    ; implicit-def: $vgpr55
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_or_b32_sdwa v5, v5, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v5, v6, v5, s6
 ; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; GFX9-NEXT:    ; implicit-def: $vgpr21
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v6, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v6, v7, v6, s6
 ; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT:    ; implicit-def: $vgpr25
+; GFX9-NEXT:    ; implicit-def: $vgpr40
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v7, v7, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v7, v8, v7, s6
-; GFX9-NEXT:    v_or_b32_sdwa v8, v51, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v29, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v8, v9, v8, s6
-; GFX9-NEXT:    v_or_b32_sdwa v9, v52, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v9, v42, v57 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v9, v10, v9, s6
-; GFX9-NEXT:    v_or_b32_sdwa v10, v42, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v10, v47, v59 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v10, v11, v10, s6
-; GFX9-NEXT:    v_or_b32_sdwa v11, v37, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v11, v35, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v11, v12, v11, s6
-; GFX9-NEXT:    v_or_b32_sdwa v12, v44, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v12, v16, v63 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v12, v13, v12, s6
-; GFX9-NEXT:    v_or_b32_sdwa v13, v28, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v13, v24, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v13, v14, v13, s6
-; GFX9-NEXT:    v_or_b32_sdwa v14, v47, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v14, v32, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v14, v15, v14, s6
-; GFX9-NEXT:    v_or_b32_sdwa v15, v18, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v15, v38, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v16, v36, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v15, v16, v15, s6
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
@@ -95027,111 +94952,110 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    ; kill: killed $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; kill: killed $vgpr16
-; GFX9-NEXT:    ; implicit-def: $vgpr51
-; GFX9-NEXT:    ; implicit-def: $vgpr52
+; GFX9-NEXT:    ; implicit-def: $vgpr29
 ; GFX9-NEXT:    ; implicit-def: $vgpr42
-; GFX9-NEXT:    ; implicit-def: $vgpr37
-; GFX9-NEXT:    ; implicit-def: $vgpr44
-; GFX9-NEXT:    ; implicit-def: $vgpr28
-; GFX9-NEXT:    ; implicit-def: $vgpr16
 ; GFX9-NEXT:    ; implicit-def: $vgpr47
-; GFX9-NEXT:    ; implicit-def: $vgpr18
-; GFX9-NEXT:    ; implicit-def: $vgpr29
+; GFX9-NEXT:    ; implicit-def: $vgpr35
+; GFX9-NEXT:    ; implicit-def: $vgpr16
+; GFX9-NEXT:    ; implicit-def: $vgpr24
+; GFX9-NEXT:    ; implicit-def: $vgpr32
+; GFX9-NEXT:    ; implicit-def: $vgpr38
+; GFX9-NEXT:    ; implicit-def: $vgpr36
+; GFX9-NEXT:    ; implicit-def: $vgpr44
 ; GFX9-NEXT:    ; implicit-def: $vgpr46
 ; GFX9-NEXT:    ; implicit-def: $vgpr57
 ; GFX9-NEXT:    ; implicit-def: $vgpr59
 ; GFX9-NEXT:    ; implicit-def: $vgpr61
+; GFX9-NEXT:    ; implicit-def: $vgpr63
 ; GFX9-NEXT:    ; implicit-def: $vgpr22
-; GFX9-NEXT:    ; implicit-def: $vgpr30
-; GFX9-NEXT:    ; implicit-def: $vgpr24
-; GFX9-NEXT:    ; implicit-def: $vgpr31
+; GFX9-NEXT:    ; implicit-def: $vgpr34
+; GFX9-NEXT:    ; implicit-def: $vgpr28
+; GFX9-NEXT:    ; implicit-def: $vgpr33
 ; GFX9-NEXT:  .LBB110_2: ; %Flow
 ; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB110_4
 ; GFX9-NEXT:  ; %bb.3: ; %cmp.true
-; GFX9-NEXT:    s_waitcnt vmcnt(8)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v18
-; GFX9-NEXT:    v_or_b32_sdwa v0, v31, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(9)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v38
+; GFX9-NEXT:    v_or_b32_sdwa v0, v28, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v15, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v28
-; GFX9-NEXT:    v_or_b32_sdwa v0, v30, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v24
+; GFX9-NEXT:    v_or_b32_sdwa v0, v22, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v13, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v20
+; GFX9-NEXT:    v_or_b32_sdwa v0, v18, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v16
-; GFX9-NEXT:    v_or_b32_sdwa v0, v26, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v44
-; GFX9-NEXT:    v_or_b32_sdwa v0, v22, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v63, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v12, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v34
-; GFX9-NEXT:    v_or_b32_sdwa v0, v36, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v37
+; GFX9-NEXT:    v_or_b32_sdwa v0, v62, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v16, 0x300, v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v35
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v61, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v11, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v35
-; GFX9-NEXT:    v_add_u16_e32 v2, 3, v47
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v26
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v60, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_or_b32_sdwa v2, v24, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
+; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v47
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v10, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v63
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v41
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v58, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v52
+; GFX9-NEXT:    v_add_u16_e32 v22, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v42
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v57, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v9, 0x300, v0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v62
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v27
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v51
+; GFX9-NEXT:    v_add_u16_e32 v24, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v29
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v46, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v8, 0x300, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, 3, v39
+; GFX9-NEXT:    v_add_u16_e32 v0, 3, v25
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v45, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v0
+; GFX9-NEXT:    v_add_u16_e32 v25, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_add_u16_e32 v3, 3, v32
-; GFX9-NEXT:    v_or_b32_sdwa v3, v20, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v20, 0x300, v3
+; GFX9-NEXT:    v_add_u16_e32 v3, 3, v31
+; GFX9-NEXT:    v_or_b32_sdwa v3, v30, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v30, 0x300, v3
+; GFX9-NEXT:    v_add_u16_e32 v2, 3, v32
+; GFX9-NEXT:    v_or_b32_sdwa v2, v34, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v14, 0x300, v2
-; GFX9-NEXT:    v_add_u16_e32 v1, 3, v38
+; GFX9-NEXT:    v_add_u16_e32 v1, 3, v36
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v18, 0x300, v1
+; GFX9-NEXT:    v_add_u16_e32 v28, 0x300, v1
 ; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v8, v30, v8, s6
-; GFX9-NEXT:    v_perm_b32 v9, v28, v9, s6
-; GFX9-NEXT:    v_perm_b32 v10, v26, v10, s6
-; GFX9-NEXT:    v_perm_b32 v11, v24, v11, s6
-; GFX9-NEXT:    v_perm_b32 v12, v22, v12, s6
-; GFX9-NEXT:    v_perm_b32 v13, v16, v13, s6
-; GFX9-NEXT:    v_perm_b32 v14, v20, v14, s6
-; GFX9-NEXT:    v_perm_b32 v15, v18, v15, s6
+; GFX9-NEXT:    v_perm_b32 v8, v25, v8, s6
+; GFX9-NEXT:    v_perm_b32 v9, v24, v9, s6
+; GFX9-NEXT:    v_perm_b32 v10, v22, v10, s6
+; GFX9-NEXT:    v_perm_b32 v11, v20, v11, s6
+; GFX9-NEXT:    v_perm_b32 v12, v16, v12, s6
+; GFX9-NEXT:    v_perm_b32 v13, v18, v13, s6
+; GFX9-NEXT:    v_perm_b32 v14, v30, v14, s6
+; GFX9-NEXT:    v_perm_b32 v15, v28, v15, s6
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v29, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v7, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_u16_e32 v31, 3, v31
-; GFX9-NEXT:    v_or_b32_sdwa v31, v48, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v31, v39, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v31, 0x300, v31
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v27, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v27, 0x300, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v26, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_perm_b32 v7, v27, v7, s6
+; GFX9-NEXT:    v_perm_b32 v7, v26, v7, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v25, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v6, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -95153,7 +95077,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v5, v19, v5, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v4, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -95164,40 +95088,40 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
 ; GFX9-NEXT:    v_perm_b32 v4, v17, v4, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v41, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v3, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v21, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v3, v21, v3, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v52, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v2, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v54, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT:    v_add_u16_e32 v25, 0x300, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_add_u16_e32 v27, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-NEXT:    v_perm_b32 v2, v25, v2, s6
+; GFX9-NEXT:    v_perm_b32 v2, v27, v2, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v1, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v50, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v29, 0x300, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; GFX9-NEXT:    v_perm_b32 v1, v29, v1, s6
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_add_u16_e32 v0, 3, v0
-; GFX9-NEXT:    v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v48, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; GFX9-NEXT:    v_add_u16_e32 v0, 0x300, v0
 ; GFX9-NEXT:    v_perm_b32 v0, v31, v0, s6
 ; GFX9-NEXT:  .LBB110_4: ; %end
@@ -97280,57 +97204,34 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
 ; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32bf16_scalar:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v31, v1 :: v_dual_mov_b32 v52, v0
 ; GFX11-TRUE16-NEXT:    s_clause 0xf
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v86, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v0, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:4
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v33, v15 :: v_dual_mov_b32 v34, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v37, v13 :: v_dual_mov_b32 v54, v10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v51, v12 :: v_dual_mov_b32 v50, v7
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v49, v11 :: v_dual_mov_b32 v48, v8
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v35, v9 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v53, v6 :: v_dual_mov_b32 v38, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, v3 :: v_dual_mov_b32 v39, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v86
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX11-TRUE16-NEXT:    s_and_b32 s5, vcc_lo, exec_lo
 ; GFX11-TRUE16-NEXT:    s_cbranch_scc0 .LBB111_4
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
@@ -97354,8 +97255,8 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s10, s11
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v32
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s7, s8, s9
 ; GFX11-TRUE16-NEXT:    s_and_b32 s8, s24, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s9, s25, 8
@@ -97363,14 +97264,14 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s27, 8
 ; GFX11-TRUE16-NEXT:    s_or_b32 s8, s8, s9
 ; GFX11-TRUE16-NEXT:    s_or_b32 s9, s10, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v31
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v52
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v18
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 8, v38
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v53
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v50
 ; GFX11-TRUE16-NEXT:    s_and_b32 s10, s28, 0xff
 ; GFX11-TRUE16-NEXT:    s_lshl_b32 s11, s29, 8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
@@ -97379,10 +97280,10 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v3, 0xffff, s9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v5, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v20
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v30
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v48
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v35
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v49
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v8
@@ -97390,63 +97291,63 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v34
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v33
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v17
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v6
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v0, 16, v1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v8
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xff, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v19
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v0, 16, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v39
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 8, v23
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v2, 16, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v35
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v21
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v48
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v37
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v52
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 8, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xff, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 8, v27
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v49
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v81
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v30
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v69
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v66
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v3, 16, v2
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v1, 16, v0
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v12, v13
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v66
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v64
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 8, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v85
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v84
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v82
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v67
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v70
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xff, v71
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 8, v69
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v65
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v82
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v80
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v3, v11
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v12, v13
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v83
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v71
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xff, v85
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 8, v83
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v86, v87
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v50
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v80
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v68
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xff, v70
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 8, v67
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v96, 0xff, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 8, v81
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v14, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v11
@@ -97505,151 +97406,151 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s1, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s10, 0x300
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v80
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 3, v84
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s1, s9, s10
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v83
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 3, v85
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s5, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s6, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s7, 0x300
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s8, 0x300
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v70
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v82
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s7, s8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v68
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 8, v81
 ; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s5, s6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v71
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v65
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 8, v83
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v80
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v85
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 3, v71
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v54
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 3, v70
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 8, v67
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v84
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v69
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v82
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 3, v68
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v11, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v81
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v30
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v67
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 8, v65
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v69
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v66
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v51
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v66
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v64
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 3, v22
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v6, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 3, v18
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v64
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v26
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v13, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v28
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v12, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v27
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v49
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v29
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v14, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v48
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v24
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v39
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v37
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 8, v25
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v15, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v37, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v18, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v10
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v9, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v36
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v35
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v20
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v9, 0x300, v5
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v34
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v10, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 8, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v31, 3, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v32, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 3, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v54
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v7, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v31
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 8, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v27, 3, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v29, 0x300, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 8, v49
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 3, v51
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v20, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v8, v7
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v7, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v28, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v25
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v26, 0x300, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v16, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v37
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v16, 0x300, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 3, v53
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 3, v48
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v8, 3, v39
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v50
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 8, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 3, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v23, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v20, v4
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 8, v18
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v19, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v17, 0x300, v4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 8, v32
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v24, 3, v36
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v25, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v6
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v22, v4
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v23, v8
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xff, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 8, v38
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v22, 0x300, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 3, v52
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v23, 0x300, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v4, 0x300, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v18, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 8, v31
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff, v5
 ; GFX11-TRUE16-NEXT:    s_addk_i32 s4, 0x300
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v6, 0x300, v6
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e64 v8, 0xffff, s4
-; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v4
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x300, v1
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x300, v2
 ; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v4, v5, 16, v8
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v6, 16, v16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v23, 16, v8
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v25, 16, v16
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v17
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v5, v6, 16, v21
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v9
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v7, v25, 16, v8
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v8, v16, 16, v17
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v17, 16, v6
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v29, 16, v18
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v10, 16, v19
-; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v37, 16, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v6, v23, 16, v6
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v9, v20, 16, v19
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v10, v10, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v11, v18, 16, v15
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v12, v14, 16, v12
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v13, v13, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v14, v2, 16, v16
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
index bf8fe832651f..7f291e359fd9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
@@ -3547,8 +3547,8 @@ define <18 x i32> @bitcast_v36i16_to_v18i32(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_mov_b32_e32 v36, v22
@@ -3580,9 +3580,9 @@ define <18 x i32> @bitcast_v36i16_to_v18i32(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v6
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -6864,6 +6864,7 @@ define <18 x i32> @bitcast_v36f16_to_v18i32(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v50
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v51
@@ -6887,13 +6888,12 @@ define <18 x i32> @bitcast_v36f16_to_v18i32(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
@@ -6905,7 +6905,6 @@ define <18 x i32> @bitcast_v36f16_to_v18i32(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v14, v16, v14
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
@@ -11192,8 +11191,8 @@ define <18 x float> @bitcast_v36i16_to_v18f32(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_mov_b32_e32 v36, v22
@@ -11225,9 +11224,9 @@ define <18 x float> @bitcast_v36i16_to_v18f32(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v6
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -14548,6 +14547,7 @@ define <18 x float> @bitcast_v36f16_to_v18f32(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v50
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v51
@@ -14571,13 +14571,12 @@ define <18 x float> @bitcast_v36f16_to_v18f32(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
@@ -14589,7 +14588,6 @@ define <18 x float> @bitcast_v36f16_to_v18f32(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v14, v16, v14
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
@@ -18055,8 +18053,8 @@ define <9 x i64> @bitcast_v36i16_to_v9i64(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_mov_b32_e32 v36, v22
@@ -18088,9 +18086,9 @@ define <9 x i64> @bitcast_v36i16_to_v9i64(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v6
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -21382,6 +21380,7 @@ define <9 x i64> @bitcast_v36f16_to_v9i64(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v50
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v51
@@ -21405,13 +21404,12 @@ define <9 x i64> @bitcast_v36f16_to_v9i64(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
@@ -21423,7 +21421,6 @@ define <9 x i64> @bitcast_v36f16_to_v9i64(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v14, v16, v14
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
@@ -24140,8 +24137,8 @@ define <9 x double> @bitcast_v36i16_to_v9f64(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12
 ; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_mov_b32_e32 v36, v22
@@ -24173,9 +24170,9 @@ define <9 x double> @bitcast_v36i16_to_v9f64(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v6
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -27415,6 +27412,7 @@ define <9 x double> @bitcast_v36f16_to_v9f64(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v50
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v10, v11, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v51
@@ -27438,13 +27436,12 @@ define <9 x double> @bitcast_v36f16_to_v9f64(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
@@ -27456,7 +27453,6 @@ define <9 x double> @bitcast_v36f16_to_v9f64(<36 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v14, v16, v14
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
@@ -28860,13 +28856,13 @@ define <36 x half> @bitcast_v36i16_to_v36f16(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:4
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; kill: killed $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr48
@@ -28911,7 +28907,7 @@ define <36 x half> @bitcast_v36i16_to_v36f16(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; kill: killed $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
 ; SI-NEXT:    ; implicit-def: $vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -28939,37 +28935,37 @@ define <36 x half> @bitcast_v36i16_to_v36f16(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v60, v10
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v45, v11
 ; SI-NEXT:    v_cvt_f32_f16_e32 v58, v12
 ; SI-NEXT:    v_cvt_f32_f16_e32 v43, v13
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v34
 ; SI-NEXT:    v_cvt_f32_f16_e32 v56, v14
 ; SI-NEXT:    v_cvt_f32_f16_e32 v41, v15
 ; SI-NEXT:    v_cvt_f32_f16_e32 v46, v16
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v35
 ; SI-NEXT:    v_cvt_f32_f16_e32 v55, v17
 ; SI-NEXT:    v_cvt_f32_f16_e32 v44, v18
 ; SI-NEXT:    v_cvt_f32_f16_e32 v53, v19
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v36
 ; SI-NEXT:    v_cvt_f32_f16_e32 v42, v20
 ; SI-NEXT:    v_cvt_f32_f16_e32 v51, v21
 ; SI-NEXT:    v_cvt_f32_f16_e32 v40, v22
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v37
 ; SI-NEXT:    v_cvt_f32_f16_e32 v49, v23
 ; SI-NEXT:    v_cvt_f32_f16_e32 v54, v24
 ; SI-NEXT:    v_cvt_f32_f16_e32 v52, v26
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v39
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
 ; SI-NEXT:    v_cvt_f32_f16_e32 v50, v28
 ; SI-NEXT:    v_cvt_f32_f16_e32 v48, v29
 ; SI-NEXT:    ; implicit-def: $vgpr2
@@ -29001,12 +28997,12 @@ define <36 x half> @bitcast_v36i16_to_v36f16(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr29
 ; SI-NEXT:    ; implicit-def: $vgpr30
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr1
 ; SI-NEXT:  .LBB56_2: ; %Flow
@@ -29024,36 +29020,36 @@ define <36 x half> @bitcast_v36i16_to_v36f16(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v27
 ; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_add_i32_e32 v39, vcc, 3, v39
 ; SI-NEXT:    v_add_i32_e32 v34, vcc, 3, v34
-; SI-NEXT:    v_add_i32_e32 v35, vcc, 3, v35
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v30
+; SI-NEXT:    v_add_i32_e32 v35, vcc, 3, v35
 ; SI-NEXT:    v_add_i32_e32 v36, vcc, 3, v36
-; SI-NEXT:    v_add_i32_e32 v37, vcc, 3, v37
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v39
+; SI-NEXT:    v_add_i32_e32 v37, vcc, 3, v37
 ; SI-NEXT:    v_add_i32_e32 v38, vcc, 3, v38
-; SI-NEXT:    v_add_i32_e32 v39, vcc, 3, v39
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v34
 ; SI-NEXT:    v_add_i32_e32 v29, vcc, 3, v29
 ; SI-NEXT:    v_add_i32_e32 v28, vcc, 3, v28
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v36
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v35
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v37
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v36
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v37
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
@@ -29102,7 +29098,7 @@ define <36 x half> @bitcast_v36i16_to_v36f16(<36 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v48, v29
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v39
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB56_4: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
index 0588d03f3b7a..903367fbaf16 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
@@ -3698,26 +3698,24 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v37, v20
 ; SI-NEXT:    v_mov_b32_e32 v38, v18
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v11
@@ -3732,23 +3730,25 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v29
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:20
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -3761,14 +3761,14 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
@@ -3795,8 +3795,8 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v42
-; SI-NEXT:    v_or_b32_e32 v1, v1, v36
-; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_or_b32_e32 v1, v1, v41
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v35
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v40
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v34
@@ -3822,8 +3822,8 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr34
@@ -3836,19 +3836,19 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v60
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v59
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v58
@@ -3870,14 +3870,14 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB14_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -3904,8 +3904,8 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_or_b32_e32 v0, v42, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
-; SI-NEXT:    v_or_b32_e32 v2, v41, v2
+; SI-NEXT:    v_or_b32_e32 v1, v41, v1
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v40, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v34, v5
@@ -3929,15 +3929,13 @@ define <20 x i32> @bitcast_v40i16_to_v20i32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
@@ -7274,6 +7272,8 @@ define <20 x i32> @bitcast_v40f16_to_v20i32(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v37
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v36
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
@@ -7362,58 +7362,55 @@ define <20 x i32> @bitcast_v40f16_to_v20i32(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v41
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
 ; SI-NEXT:    v_or_b32_e32 v11, v13, v11
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
-; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v13, v15, v13
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -11800,26 +11797,24 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v37, v20
 ; SI-NEXT:    v_mov_b32_e32 v38, v18
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v11
@@ -11834,23 +11829,25 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v29
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:20
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -11863,14 +11860,14 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB30_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
@@ -11897,8 +11894,8 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v42
-; SI-NEXT:    v_or_b32_e32 v1, v1, v36
-; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_or_b32_e32 v1, v1, v41
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v35
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v40
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v34
@@ -11924,8 +11921,8 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr34
@@ -11938,19 +11935,19 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v60
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v59
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v58
@@ -11972,14 +11969,14 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB30_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -12006,8 +12003,8 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_or_b32_e32 v0, v42, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
-; SI-NEXT:    v_or_b32_e32 v2, v41, v2
+; SI-NEXT:    v_or_b32_e32 v1, v41, v1
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v40, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v34, v5
@@ -12031,15 +12028,13 @@ define <20 x float> @bitcast_v40i16_to_v20f32(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
@@ -15364,6 +15359,8 @@ define <20 x float> @bitcast_v40f16_to_v20f32(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v37
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v36
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
@@ -15452,58 +15449,55 @@ define <20 x float> @bitcast_v40f16_to_v20f32(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v41
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
 ; SI-NEXT:    v_or_b32_e32 v11, v13, v11
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
-; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v13, v15, v13
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -19214,26 +19208,24 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v37, v20
 ; SI-NEXT:    v_mov_b32_e32 v38, v18
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v11
@@ -19248,23 +19240,25 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v29
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:20
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -19277,14 +19271,14 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB42_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
@@ -19311,8 +19305,8 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v42
-; SI-NEXT:    v_or_b32_e32 v1, v1, v36
-; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_or_b32_e32 v1, v1, v41
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v35
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v40
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v34
@@ -19338,8 +19332,8 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr34
@@ -19352,19 +19346,19 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v60
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v59
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v58
@@ -19386,14 +19380,14 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB42_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -19420,8 +19414,8 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_or_b32_e32 v0, v42, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
-; SI-NEXT:    v_or_b32_e32 v2, v41, v2
+; SI-NEXT:    v_or_b32_e32 v1, v41, v1
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v40, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v34, v5
@@ -19445,15 +19439,13 @@ define <10 x i64> @bitcast_v40i16_to_v10i64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
@@ -22800,6 +22792,8 @@ define <10 x i64> @bitcast_v40f16_to_v10i64(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v37
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v36
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
@@ -22888,58 +22882,55 @@ define <10 x i64> @bitcast_v40f16_to_v10i64(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v41
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
 ; SI-NEXT:    v_or_b32_e32 v11, v13, v11
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
-; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v13, v15, v13
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -25855,26 +25846,24 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v37, v20
 ; SI-NEXT:    v_mov_b32_e32 v38, v18
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v11
@@ -25889,23 +25878,25 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v29
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:20
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -25918,14 +25909,14 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
@@ -25952,8 +25943,8 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v42
-; SI-NEXT:    v_or_b32_e32 v1, v1, v36
-; SI-NEXT:    v_or_b32_e32 v2, v2, v41
+; SI-NEXT:    v_or_b32_e32 v1, v1, v41
+; SI-NEXT:    v_or_b32_e32 v2, v2, v36
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v35
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v40
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v34
@@ -25979,8 +25970,8 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr20
 ; SI-NEXT:    ; kill: killed $vgpr20
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr34
@@ -25993,19 +25984,19 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v60
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v59
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v58
@@ -26027,14 +26018,14 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -26061,8 +26052,8 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_or_b32_e32 v0, v42, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v36, v1
-; SI-NEXT:    v_or_b32_e32 v2, v41, v2
+; SI-NEXT:    v_or_b32_e32 v1, v41, v1
+; SI-NEXT:    v_or_b32_e32 v2, v36, v2
 ; SI-NEXT:    v_or_b32_e32 v3, v35, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v40, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v34, v5
@@ -26086,15 +26077,13 @@ define <10 x double> @bitcast_v40i16_to_v10f64(<40 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
+; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
@@ -29351,6 +29340,8 @@ define <10 x double> @bitcast_v40f16_to_v10f64(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v39
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v37
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v38
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v36
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
@@ -29439,58 +29430,55 @@ define <10 x double> @bitcast_v40f16_to_v10f64(<40 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
 ; SI-NEXT:    v_or_b32_e32 v9, v10, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v41
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
 ; SI-NEXT:    v_or_b32_e32 v11, v13, v11
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
-; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_or_b32_e32 v12, v13, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v13, v15, v13
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(1)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
index e5ec21928164..cab1c3408d69 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
@@ -3960,93 +3960,98 @@ define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:12
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v7
+; SI-NEXT:    v_mov_b32_e32 v48, v14
+; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v38, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v23
+; SI-NEXT:    s_waitcnt expcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v2
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:24
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v10
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
@@ -4075,20 +4080,19 @@ define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v48
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    v_or_b32_e32 v0, v0, v38
-; SI-NEXT:    v_or_b32_e32 v1, v1, v44
-; SI-NEXT:    v_or_b32_e32 v2, v2, v43
-; SI-NEXT:    v_or_b32_e32 v3, v3, v37
+; SI-NEXT:    v_or_b32_e32 v0, v0, v37
+; SI-NEXT:    v_or_b32_e32 v1, v1, v45
+; SI-NEXT:    v_or_b32_e32 v2, v2, v44
+; SI-NEXT:    v_or_b32_e32 v3, v3, v43
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v36
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v42
-; SI-NEXT:    v_or_b32_e32 v6, v6, v35
-; SI-NEXT:    v_or_b32_e32 v7, v7, v41
+; SI-NEXT:    v_or_b32_e32 v6, v6, v41
+; SI-NEXT:    v_or_b32_e32 v7, v7, v35
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v34
-; SI-NEXT:    v_or_b32_e32 v16, v16, v59
+; SI-NEXT:    v_or_b32_e32 v9, v9, v40
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -4098,63 +4102,63 @@ define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_or_b32_e32 v9, v9, v40
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v33
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v32
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v63
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v62
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v61
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v60
+; SI-NEXT:    v_or_b32_e32 v16, v16, v59
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v58
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v57
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v56
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v47
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v46
-; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr56
@@ -4164,18 +4168,19 @@ define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -4185,8 +4190,7 @@ define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -4196,18 +4200,18 @@ define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v38, v0
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_or_b32_e32 v0, v37, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v44, v1
-; SI-NEXT:    v_or_b32_e32 v2, v43, v2
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v45, v1
+; SI-NEXT:    v_or_b32_e32 v2, v44, v2
+; SI-NEXT:    v_or_b32_e32 v3, v43, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v36, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v42, v5
-; SI-NEXT:    v_or_b32_e32 v6, v35, v6
-; SI-NEXT:    v_or_b32_e32 v7, v41, v7
+; SI-NEXT:    v_or_b32_e32 v6, v41, v6
+; SI-NEXT:    v_or_b32_e32 v7, v35, v7
 ; SI-NEXT:    v_or_b32_e32 v8, v34, v8
-; SI-NEXT:    v_or_b32_e32 v16, v59, v16
+; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -4217,58 +4221,57 @@ define <22 x i32> @bitcast_v44i16_to_v22i32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v33, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v32, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v63, v12
 ; SI-NEXT:    v_or_b32_e32 v13, v62, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v61, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v60, v15
+; SI-NEXT:    v_or_b32_e32 v16, v59, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v58, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v57, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v56, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v47, v20
 ; SI-NEXT:    v_or_b32_e32 v21, v46, v21
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
@@ -7996,73 +7999,72 @@ define <22 x i32> @bitcast_v44f16_to_v22i32(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v63
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
 ; SI-NEXT:    v_cvt_f32_f16_e32 v23, v52
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v59
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v9, v11, v9
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
@@ -12848,93 +12850,98 @@ define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:12
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v7
+; SI-NEXT:    v_mov_b32_e32 v48, v14
+; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v38, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v23
+; SI-NEXT:    s_waitcnt expcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v2
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:4
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:24
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v10
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
@@ -12963,20 +12970,19 @@ define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v48
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    v_or_b32_e32 v0, v0, v38
-; SI-NEXT:    v_or_b32_e32 v1, v1, v44
-; SI-NEXT:    v_or_b32_e32 v2, v2, v43
-; SI-NEXT:    v_or_b32_e32 v3, v3, v37
+; SI-NEXT:    v_or_b32_e32 v0, v0, v37
+; SI-NEXT:    v_or_b32_e32 v1, v1, v45
+; SI-NEXT:    v_or_b32_e32 v2, v2, v44
+; SI-NEXT:    v_or_b32_e32 v3, v3, v43
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v36
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v42
-; SI-NEXT:    v_or_b32_e32 v6, v6, v35
-; SI-NEXT:    v_or_b32_e32 v7, v7, v41
+; SI-NEXT:    v_or_b32_e32 v6, v6, v41
+; SI-NEXT:    v_or_b32_e32 v7, v7, v35
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v34
-; SI-NEXT:    v_or_b32_e32 v16, v16, v59
+; SI-NEXT:    v_or_b32_e32 v9, v9, v40
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -12986,63 +12992,63 @@ define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_or_b32_e32 v9, v9, v40
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v33
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v32
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v63
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v62
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v61
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v60
+; SI-NEXT:    v_or_b32_e32 v16, v16, v59
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v58
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v57
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v56
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v47
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v46
-; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr56
@@ -13052,18 +13058,19 @@ define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -13073,8 +13080,7 @@ define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -13084,18 +13090,18 @@ define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v38, v0
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_or_b32_e32 v0, v37, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v44, v1
-; SI-NEXT:    v_or_b32_e32 v2, v43, v2
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v45, v1
+; SI-NEXT:    v_or_b32_e32 v2, v44, v2
+; SI-NEXT:    v_or_b32_e32 v3, v43, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v36, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v42, v5
-; SI-NEXT:    v_or_b32_e32 v6, v35, v6
-; SI-NEXT:    v_or_b32_e32 v7, v41, v7
+; SI-NEXT:    v_or_b32_e32 v6, v41, v6
+; SI-NEXT:    v_or_b32_e32 v7, v35, v7
 ; SI-NEXT:    v_or_b32_e32 v8, v34, v8
-; SI-NEXT:    v_or_b32_e32 v16, v59, v16
+; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -13105,58 +13111,57 @@ define <22 x float> @bitcast_v44i16_to_v22f32(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v33, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v32, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v63, v12
 ; SI-NEXT:    v_or_b32_e32 v13, v62, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v61, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v60, v15
+; SI-NEXT:    v_or_b32_e32 v16, v59, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v58, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v57, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v56, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v47, v20
 ; SI-NEXT:    v_or_b32_e32 v21, v46, v21
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
@@ -16884,73 +16889,72 @@ define <22 x float> @bitcast_v44f16_to_v22f32(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v63
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
 ; SI-NEXT:    v_cvt_f32_f16_e32 v23, v52
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v59
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v9, v11, v9
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
@@ -21026,93 +21030,98 @@ define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:12
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v7
+; SI-NEXT:    v_mov_b32_e32 v48, v14
+; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v38, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v23
+; SI-NEXT:    s_waitcnt expcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v2
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:4
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:24
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v10
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
@@ -21141,20 +21150,19 @@ define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v48
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    v_or_b32_e32 v0, v0, v38
-; SI-NEXT:    v_or_b32_e32 v1, v1, v44
-; SI-NEXT:    v_or_b32_e32 v2, v2, v43
-; SI-NEXT:    v_or_b32_e32 v3, v3, v37
+; SI-NEXT:    v_or_b32_e32 v0, v0, v37
+; SI-NEXT:    v_or_b32_e32 v1, v1, v45
+; SI-NEXT:    v_or_b32_e32 v2, v2, v44
+; SI-NEXT:    v_or_b32_e32 v3, v3, v43
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v36
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v42
-; SI-NEXT:    v_or_b32_e32 v6, v6, v35
-; SI-NEXT:    v_or_b32_e32 v7, v7, v41
+; SI-NEXT:    v_or_b32_e32 v6, v6, v41
+; SI-NEXT:    v_or_b32_e32 v7, v7, v35
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v34
-; SI-NEXT:    v_or_b32_e32 v16, v16, v59
+; SI-NEXT:    v_or_b32_e32 v9, v9, v40
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -21164,63 +21172,63 @@ define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_or_b32_e32 v9, v9, v40
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v33
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v32
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v63
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v62
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v61
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v60
+; SI-NEXT:    v_or_b32_e32 v16, v16, v59
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v58
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v57
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v56
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v47
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v46
-; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr56
@@ -21230,18 +21238,19 @@ define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -21251,8 +21260,7 @@ define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -21262,18 +21270,18 @@ define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v38, v0
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_or_b32_e32 v0, v37, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v44, v1
-; SI-NEXT:    v_or_b32_e32 v2, v43, v2
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v45, v1
+; SI-NEXT:    v_or_b32_e32 v2, v44, v2
+; SI-NEXT:    v_or_b32_e32 v3, v43, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v36, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v42, v5
-; SI-NEXT:    v_or_b32_e32 v6, v35, v6
-; SI-NEXT:    v_or_b32_e32 v7, v41, v7
+; SI-NEXT:    v_or_b32_e32 v6, v41, v6
+; SI-NEXT:    v_or_b32_e32 v7, v35, v7
 ; SI-NEXT:    v_or_b32_e32 v8, v34, v8
-; SI-NEXT:    v_or_b32_e32 v16, v59, v16
+; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -21283,58 +21291,57 @@ define <11 x i64> @bitcast_v44i16_to_v11i64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v33, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v32, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v63, v12
 ; SI-NEXT:    v_or_b32_e32 v13, v62, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v61, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v60, v15
+; SI-NEXT:    v_or_b32_e32 v16, v59, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v58, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v57, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v56, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v47, v20
 ; SI-NEXT:    v_or_b32_e32 v21, v46, v21
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
@@ -25074,73 +25081,72 @@ define <11 x i64> @bitcast_v44f16_to_v11i64(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v63
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
 ; SI-NEXT:    v_cvt_f32_f16_e32 v23, v52
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v59
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v9, v11, v9
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
@@ -28366,93 +28372,98 @@ define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:12
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v39, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v7
+; SI-NEXT:    v_mov_b32_e32 v48, v14
+; SI-NEXT:    v_mov_b32_e32 v49, v12
+; SI-NEXT:    v_mov_b32_e32 v38, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v7
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v9
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v23
+; SI-NEXT:    s_waitcnt expcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v2
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v4
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:4
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:24
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v10
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
@@ -28481,20 +28492,19 @@ define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v48
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v45
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
-; SI-NEXT:    v_or_b32_e32 v0, v0, v38
-; SI-NEXT:    v_or_b32_e32 v1, v1, v44
-; SI-NEXT:    v_or_b32_e32 v2, v2, v43
-; SI-NEXT:    v_or_b32_e32 v3, v3, v37
+; SI-NEXT:    v_or_b32_e32 v0, v0, v37
+; SI-NEXT:    v_or_b32_e32 v1, v1, v45
+; SI-NEXT:    v_or_b32_e32 v2, v2, v44
+; SI-NEXT:    v_or_b32_e32 v3, v3, v43
 ; SI-NEXT:    v_or_b32_e32 v4, v4, v36
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v42
-; SI-NEXT:    v_or_b32_e32 v6, v6, v35
-; SI-NEXT:    v_or_b32_e32 v7, v7, v41
+; SI-NEXT:    v_or_b32_e32 v6, v6, v41
+; SI-NEXT:    v_or_b32_e32 v7, v7, v35
 ; SI-NEXT:    v_or_b32_e32 v8, v8, v34
-; SI-NEXT:    v_or_b32_e32 v16, v16, v59
+; SI-NEXT:    v_or_b32_e32 v9, v9, v40
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -28504,63 +28514,63 @@ define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; kill: killed $vgpr22
 ; SI-NEXT:    ; implicit-def: $vgpr22
 ; SI-NEXT:    ; kill: killed $vgpr22
-; SI-NEXT:    ; implicit-def: $vgpr38
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr45
 ; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT:    v_or_b32_e32 v9, v9, v40
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v33
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v32
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v63
 ; SI-NEXT:    v_or_b32_e32 v13, v13, v62
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v61
 ; SI-NEXT:    v_or_b32_e32 v15, v15, v60
+; SI-NEXT:    v_or_b32_e32 v16, v16, v59
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v58
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v57
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v56
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v47
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v46
-; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr56
@@ -28570,18 +28580,19 @@ define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -28591,8 +28602,7 @@ define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v49
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v48
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v45
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v38
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -28602,18 +28612,18 @@ define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v38, v0
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_or_b32_e32 v0, v37, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v44, v1
-; SI-NEXT:    v_or_b32_e32 v2, v43, v2
-; SI-NEXT:    v_or_b32_e32 v3, v37, v3
+; SI-NEXT:    v_or_b32_e32 v1, v45, v1
+; SI-NEXT:    v_or_b32_e32 v2, v44, v2
+; SI-NEXT:    v_or_b32_e32 v3, v43, v3
 ; SI-NEXT:    v_or_b32_e32 v4, v36, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v42, v5
-; SI-NEXT:    v_or_b32_e32 v6, v35, v6
-; SI-NEXT:    v_or_b32_e32 v7, v41, v7
+; SI-NEXT:    v_or_b32_e32 v6, v41, v6
+; SI-NEXT:    v_or_b32_e32 v7, v35, v7
 ; SI-NEXT:    v_or_b32_e32 v8, v34, v8
-; SI-NEXT:    v_or_b32_e32 v16, v59, v16
+; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -28623,58 +28633,57 @@ define <11 x double> @bitcast_v44i16_to_v11f64(<44 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, s6, v8
-; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
+; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_or_b32_e32 v9, v40, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v33, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v32, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v63, v12
 ; SI-NEXT:    v_or_b32_e32 v13, v62, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v61, v14
 ; SI-NEXT:    v_or_b32_e32 v15, v60, v15
+; SI-NEXT:    v_or_b32_e32 v16, v59, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v58, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v57, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v56, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v47, v20
 ; SI-NEXT:    v_or_b32_e32 v21, v46, v21
-; SI-NEXT:    v_add_i32_e32 v9, vcc, s6, v9
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, s6, v10
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, s6, v11
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, s6, v12
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, s6, v13
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, s6, v14
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, s6, v15
+; SI-NEXT:    v_add_i32_e32 v16, vcc, s6, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, s6, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, s6, v19
@@ -32325,73 +32334,72 @@ define <11 x double> @bitcast_v44f16_to_v11f64(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v63
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_cvt_f32_f16_e32 v7, v61
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v53
 ; SI-NEXT:    v_cvt_f32_f16_e32 v23, v52
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v59
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
+; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
+; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    v_or_b32_e32 v8, v9, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_or_b32_e32 v9, v11, v9
 ; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v10, v12, v10
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
@@ -36331,162 +36339,157 @@ define <44 x i16> @bitcast_v44f16_to_v44i16(<44 x half> %a, i32 %b) {
 ; SI-LABEL: bitcast_v44f16_to_v44i16:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16
 ; SI-NEXT:    s_waitcnt expcnt(6)
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32
 ; SI-NEXT:    s_waitcnt expcnt(5)
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(4)
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(3)
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt expcnt(2)
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:16
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:40
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v1
-; SI-NEXT:    v_cvt_f16_f32_e32 v41, v2
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:24
 ; SI-NEXT:    v_cvt_f16_f32_e32 v48, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v49, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v55, v10
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v11
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v12
+; SI-NEXT:    v_cvt_f16_f32_e32 v33, v17
+; SI-NEXT:    v_cvt_f16_f32_e32 v53, v18
+; SI-NEXT:    v_cvt_f16_f32_e32 v17, v25
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v26
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v27
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:52
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v1
+; SI-NEXT:    v_cvt_f16_f32_e32 v41, v2
 ; SI-NEXT:    v_cvt_f16_f32_e32 v50, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v40, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v37, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v38, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v39, v9
-; SI-NEXT:    v_cvt_f16_f32_e32 v55, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v34, v11
-; SI-NEXT:    v_cvt_f16_f32_e32 v35, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v36, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v54, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v31, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v32, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v33, v17
-; SI-NEXT:    v_cvt_f16_f32_e32 v53, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v52, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v17, v24
-; SI-NEXT:    v_cvt_f16_f32_e32 v18, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v26
-; SI-NEXT:    v_cvt_f16_f32_e32 v11, v27
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v28
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v23
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v24
+; SI-NEXT:    v_cvt_f16_f32_e32 v13, v28
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v24, v30
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v42
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f16_f32_e32 v8, v43
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v43
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v44
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v44
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_cvt_f16_f32_e32 v10, v45
+; SI-NEXT:    v_cvt_f16_f32_e32 v8, v45
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v46
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v47
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v47
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v56
+; SI-NEXT:    v_cvt_f16_f32_e32 v2, v56
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v57
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v57
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f16_f32_e32 v13, v58
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v3
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v59
+; SI-NEXT:    v_cvt_f16_f32_e32 v9, v4
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v60
+; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v61
+; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v62
+; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v27
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB58_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.true
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
 ; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
-; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
-; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
+; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
+; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
-; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v2
-; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
-; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
-; SI-NEXT:    v_or_b32_e32 v1, v1, v26
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v6
-; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    v_or_b32_e32 v5, v5, v26
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v10
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; SI-NEXT:    v_or_b32_e32 v9, v9, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v2
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
-; SI-NEXT:    v_or_b32_e32 v3, v3, v26
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v9
+; SI-NEXT:    v_or_b32_e32 v1, v1, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    v_or_b32_e32 v8, v8, v26
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v12
-; SI-NEXT:    v_or_b32_e32 v11, v11, v26
-; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v17
-; SI-NEXT:    v_or_b32_e32 v16, v16, v26
+; SI-NEXT:    v_or_b32_e32 v6, v6, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v13
+; SI-NEXT:    v_or_b32_e32 v12, v12, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v16
+; SI-NEXT:    v_or_b32_e32 v15, v15, v26
 ; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v20
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v26
 ; SI-NEXT:    v_cvt_f32_f16_e32 v26, v32
@@ -36533,30 +36536,30 @@ define <44 x i16> @bitcast_v44f16_to_v44i16(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v49, v26
 ; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
-; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
 ; SI-NEXT:    v_lshlrev_b32_e32 v27, 16, v49
 ; SI-NEXT:    v_cvt_f32_f16_e32 v33, v33
 ; SI-NEXT:    v_or_b32_e32 v48, v26, v27
 ; SI-NEXT:    v_cvt_f32_f16_e32 v26, v41
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v51
 ; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
 ; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    v_or_b32_e32 v51, v27, v26
 ; SI-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
 ; SI-NEXT:    v_cvt_f32_f16_e32 v28, v55
 ; SI-NEXT:    v_or_b32_e32 v50, v29, v27
 ; SI-NEXT:    v_cvt_f32_f16_e32 v29, v54
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
 ; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
@@ -36570,29 +36573,29 @@ define <44 x i16> @bitcast_v44f16_to_v44i16(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
 ; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
+; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v30, v30
 ; SI-NEXT:    v_cvt_f16_f32_e32 v53, v36
 ; SI-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
 ; SI-NEXT:    v_cvt_f16_f32_e32 v52, v52
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
+; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
 ; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
-; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
-; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v33, v33
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
+; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
 ; SI-NEXT:    v_or_b32_e32 v36, v30, v29
 ; SI-NEXT:    v_lshlrev_b32_e32 v30, 16, v53
@@ -36601,25 +36604,25 @@ define <44 x i16> @bitcast_v44f16_to_v44i16(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; SI-NEXT:    v_or_b32_e32 v33, v33, v30
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v52
-; SI-NEXT:    v_or_b32_e32 v18, v18, v25
+; SI-NEXT:    v_or_b32_e32 v17, v17, v25
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v24
-; SI-NEXT:    v_or_b32_e32 v10, v10, v23
-; SI-NEXT:    v_or_b32_e32 v13, v13, v22
-; SI-NEXT:    v_or_b32_e32 v7, v7, v15
+; SI-NEXT:    v_or_b32_e32 v8, v8, v23
+; SI-NEXT:    v_or_b32_e32 v5, v5, v22
+; SI-NEXT:    v_or_b32_e32 v11, v11, v18
 ; SI-NEXT:    v_alignbit_b32 v41, v48, v26, 16
 ; SI-NEXT:    v_alignbit_b32 v40, v37, v27, 16
 ; SI-NEXT:    v_alignbit_b32 v55, v34, v28, 16
 ; SI-NEXT:    v_alignbit_b32 v54, v31, v29, 16
 ; SI-NEXT:    v_alignbit_b32 v53, v19, v30, 16
-; SI-NEXT:    v_alignbit_b32 v52, v16, v52, 16
-; SI-NEXT:    v_alignbit_b32 v25, v11, v25, 16
-; SI-NEXT:    v_alignbit_b32 v24, v8, v24, 16
-; SI-NEXT:    v_alignbit_b32 v23, v3, v23, 16
-; SI-NEXT:    v_alignbit_b32 v22, v5, v22, 16
-; SI-NEXT:    v_alignbit_b32 v15, v1, v15, 16
+; SI-NEXT:    v_alignbit_b32 v52, v15, v52, 16
+; SI-NEXT:    v_alignbit_b32 v25, v12, v25, 16
+; SI-NEXT:    v_alignbit_b32 v24, v6, v24, 16
+; SI-NEXT:    v_alignbit_b32 v23, v1, v23, 16
+; SI-NEXT:    v_alignbit_b32 v22, v9, v22, 16
+; SI-NEXT:    v_alignbit_b32 v18, v3, v18, 16
 ; SI-NEXT:  .LBB58_2: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v51
@@ -36682,85 +36685,81 @@ define <44 x i16> @bitcast_v44f16_to_v44i16(<44 x half> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v20, 16, v52
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v20
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 40, v0
-; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 44, v0
+; SI-NEXT:    v_or_b32_e32 v15, v15, v16
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 44, v0
 ; SI-NEXT:    buffer_store_dword v19, v20, s[0:3], 0 offen
-; SI-NEXT:    buffer_store_dword v16, v17, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_dword v15, v16, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v18
-; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v25
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT:    v_or_b32_e32 v16, v16, v17
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 48, v0
-; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 52, v0
-; SI-NEXT:    buffer_store_dword v16, v17, s[0:3], 0 offen
-; SI-NEXT:    buffer_store_dword v11, v12, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v25
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT:    v_or_b32_e32 v15, v15, v16
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 48, v0
+; SI-NEXT:    v_or_b32_e32 v12, v12, v13
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 52, v0
+; SI-NEXT:    buffer_store_dword v15, v16, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_dword v12, v13, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v14
-; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v24
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    v_or_b32_e32 v11, v11, v12
-; SI-NEXT:    v_add_i32_e32 v12, vcc, 56, v0
-; SI-NEXT:    v_or_b32_e32 v8, v8, v9
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 60, v0
-; SI-NEXT:    buffer_store_dword v11, v12, s[0:3], 0 offen
-; SI-NEXT:    buffer_store_dword v8, v9, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v24
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_or_b32_e32 v12, v12, v13
+; SI-NEXT:    v_add_i32_e32 v13, vcc, 56, v0
+; SI-NEXT:    v_or_b32_e32 v6, v6, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 60, v0
+; SI-NEXT:    buffer_store_dword v12, v13, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v10
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v23
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v8, v8, v9
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 64, v0
-; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x44, v0
-; SI-NEXT:    buffer_store_dword v8, v9, s[0:3], 0 offen
-; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v23
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_or_b32_e32 v6, v6, v7
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 64, v0
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v0
+; SI-NEXT:    buffer_store_dword v6, v7, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v22
-; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x48, v0
-; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v22
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x48, v0
+; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
-; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x4c, v0
-; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v10
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v0
+; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v15
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v3, v3, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 0x50, v0
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v18
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v0
+; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x54, v0
-; SI-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
 ; SI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
index 808f582a09e8..8d63879f3f2b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
@@ -4207,21 +4207,27 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v5
@@ -4230,25 +4236,15 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(5)
@@ -4276,22 +4272,26 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -4301,23 +4301,23 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    ; kill: killed $vgpr24
 ; SI-NEXT:    ; implicit-def: $vgpr24
@@ -4384,37 +4384,37 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_or_b32_e32 v8, v8, v35
-; SI-NEXT:    v_or_b32_e32 v9, v9, v43
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v8, v8, v43
+; SI-NEXT:    v_or_b32_e32 v9, v9, v35
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v42
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v34
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v41
@@ -4429,8 +4429,8 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v58
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v57
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v56
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
@@ -4449,23 +4449,23 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -4501,24 +4501,23 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -4540,8 +4539,8 @@ define <24 x i32> @bitcast_v48i16_to_v24i32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_or_b32_e32 v8, v35, v8
-; SI-NEXT:    v_or_b32_e32 v9, v43, v9
+; SI-NEXT:    v_or_b32_e32 v8, v43, v8
+; SI-NEXT:    v_or_b32_e32 v9, v35, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v42, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v34, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v41, v12
@@ -8770,15 +8769,15 @@ define <24 x i32> @bitcast_v48f16_to_v24i32(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
@@ -8805,7 +8804,7 @@ define <24 x i32> @bitcast_v48f16_to_v24i32(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
@@ -9733,8 +9732,8 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
 ; SI-NEXT:    s_cbranch_execnz .LBB19_3
 ; SI-NEXT:  .LBB19_2: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v54
@@ -9801,14 +9800,14 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
@@ -13996,21 +13995,27 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v5
@@ -14019,25 +14024,15 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(5)
@@ -14065,22 +14060,26 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -14090,23 +14089,23 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    ; kill: killed $vgpr24
 ; SI-NEXT:    ; implicit-def: $vgpr24
@@ -14173,37 +14172,37 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_or_b32_e32 v8, v8, v35
-; SI-NEXT:    v_or_b32_e32 v9, v9, v43
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v8, v8, v43
+; SI-NEXT:    v_or_b32_e32 v9, v9, v35
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v42
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v34
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v41
@@ -14218,8 +14217,8 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v58
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v57
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v56
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
@@ -14238,23 +14237,23 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -14290,24 +14289,23 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -14329,8 +14327,8 @@ define <24 x float> @bitcast_v48i16_to_v24f32(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_or_b32_e32 v8, v35, v8
-; SI-NEXT:    v_or_b32_e32 v9, v43, v9
+; SI-NEXT:    v_or_b32_e32 v8, v43, v8
+; SI-NEXT:    v_or_b32_e32 v9, v35, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v42, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v34, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v41, v12
@@ -18555,15 +18553,15 @@ define <24 x float> @bitcast_v48f16_to_v24f32(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
@@ -18590,7 +18588,7 @@ define <24 x float> @bitcast_v48f16_to_v24f32(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
@@ -19518,8 +19516,8 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
 ; SI-NEXT:    s_cbranch_execnz .LBB35_3
 ; SI-NEXT:  .LBB35_2: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v54
@@ -19586,14 +19584,14 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
@@ -23044,21 +23042,27 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v5
@@ -23067,25 +23071,15 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(5)
@@ -23113,22 +23107,26 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -23138,23 +23136,23 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    ; kill: killed $vgpr24
 ; SI-NEXT:    ; implicit-def: $vgpr24
@@ -23221,37 +23219,37 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_or_b32_e32 v8, v8, v35
-; SI-NEXT:    v_or_b32_e32 v9, v9, v43
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v8, v8, v43
+; SI-NEXT:    v_or_b32_e32 v9, v9, v35
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v42
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v34
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v41
@@ -23266,8 +23264,8 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v58
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v57
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v56
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
@@ -23286,23 +23284,23 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -23338,24 +23336,23 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -23377,8 +23374,8 @@ define <12 x i64> @bitcast_v48i16_to_v12i64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_or_b32_e32 v8, v35, v8
-; SI-NEXT:    v_or_b32_e32 v9, v43, v9
+; SI-NEXT:    v_or_b32_e32 v8, v43, v8
+; SI-NEXT:    v_or_b32_e32 v9, v35, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v42, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v34, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v41, v12
@@ -27619,15 +27616,15 @@ define <12 x i64> @bitcast_v48f16_to_v12i64(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
@@ -27654,7 +27651,7 @@ define <12 x i64> @bitcast_v48f16_to_v12i64(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
@@ -28582,8 +28579,8 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
 ; SI-NEXT:    s_cbranch_execnz .LBB47_3
 ; SI-NEXT:  .LBB47_2: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v54
@@ -28650,14 +28647,14 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
@@ -31213,21 +31210,27 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    v_mov_b32_e32 v54, v2
-; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v48, v14
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
 ; SI-NEXT:    v_mov_b32_e32 v50, v10
 ; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_mov_b32_e32 v52, v6
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
+; SI-NEXT:    v_mov_b32_e32 v54, v2
+; SI-NEXT:    v_mov_b32_e32 v55, v0
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v3
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v5
@@ -31236,25 +31239,15 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v19
 ; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v25
 ; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(5)
@@ -31282,22 +31275,26 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -31307,23 +31304,23 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr24
 ; SI-NEXT:    ; kill: killed $vgpr24
 ; SI-NEXT:    ; implicit-def: $vgpr24
@@ -31390,37 +31387,37 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
 ; SI-NEXT:    s_waitcnt vmcnt(8)
 ; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_or_b32_e32 v8, v8, v35
-; SI-NEXT:    v_or_b32_e32 v9, v9, v43
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v8, v8, v43
+; SI-NEXT:    v_or_b32_e32 v9, v9, v35
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v42
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v34
 ; SI-NEXT:    v_or_b32_e32 v12, v12, v41
@@ -31435,8 +31432,8 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v21, v21, v58
 ; SI-NEXT:    v_or_b32_e32 v22, v22, v57
 ; SI-NEXT:    v_or_b32_e32 v23, v23, v56
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
@@ -31455,23 +31452,23 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -31507,24 +31504,23 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, s6, v7
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
@@ -31546,8 +31542,8 @@ define <12 x double> @bitcast_v48i16_to_v12f64(<48 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_or_b32_e32 v8, v35, v8
-; SI-NEXT:    v_or_b32_e32 v9, v43, v9
+; SI-NEXT:    v_or_b32_e32 v8, v43, v8
+; SI-NEXT:    v_or_b32_e32 v9, v35, v9
 ; SI-NEXT:    v_or_b32_e32 v10, v42, v10
 ; SI-NEXT:    v_or_b32_e32 v11, v34, v11
 ; SI-NEXT:    v_or_b32_e32 v12, v41, v12
@@ -35690,15 +35686,15 @@ define <12 x double> @bitcast_v48f16_to_v12f64(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_or_b32_e32 v7, v8, v7
 ; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
@@ -35725,7 +35721,7 @@ define <12 x double> @bitcast_v48f16_to_v12f64(<48 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
 ; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
@@ -36653,8 +36649,8 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
 ; SI-NEXT:    s_cbranch_execnz .LBB55_3
 ; SI-NEXT:  .LBB55_2: ; %cmp.true
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v54
@@ -36721,14 +36717,14 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
 ; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
+; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
index f50698ce7544..35ed1d624f6d 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
@@ -4507,73 +4507,69 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
+; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:12
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v29
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v2
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v8
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v14
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -4584,60 +4580,64 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v55
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v53
@@ -4647,11 +4647,11 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v58
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v48
-; SI-NEXT:    v_or_b32_e32 v2, v2, v57
-; SI-NEXT:    v_or_b32_e32 v3, v3, v39
-; SI-NEXT:    v_or_b32_e32 v4, v4, v56
-; SI-NEXT:    v_or_b32_e32 v5, v5, v47
-; SI-NEXT:    v_or_b32_e32 v6, v6, v38
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
+; SI-NEXT:    v_or_b32_e32 v3, v3, v57
+; SI-NEXT:    v_or_b32_e32 v4, v4, v38
+; SI-NEXT:    v_or_b32_e32 v5, v5, v56
+; SI-NEXT:    v_or_b32_e32 v6, v6, v47
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -4661,27 +4661,25 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
@@ -4718,34 +4716,37 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v36
-; SI-NEXT:    v_or_b32_e32 v9, v9, v46
-; SI-NEXT:    v_or_b32_e32 v10, v10, v35
-; SI-NEXT:    v_or_b32_e32 v11, v11, v45
-; SI-NEXT:    v_or_b32_e32 v12, v12, v44
-; SI-NEXT:    v_or_b32_e32 v13, v13, v34
-; SI-NEXT:    v_or_b32_e32 v14, v14, v43
-; SI-NEXT:    v_or_b32_e32 v15, v15, v42
-; SI-NEXT:    v_or_b32_e32 v16, v16, v33
-; SI-NEXT:    v_or_b32_e32 v17, v17, v41
+; SI-NEXT:    v_or_b32_e32 v8, v8, v46
+; SI-NEXT:    v_or_b32_e32 v9, v9, v45
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v35
+; SI-NEXT:    v_or_b32_e32 v14, v14, v42
+; SI-NEXT:    v_or_b32_e32 v15, v15, v34
+; SI-NEXT:    v_or_b32_e32 v16, v16, v41
+; SI-NEXT:    v_or_b32_e32 v17, v17, v33
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v40
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v32
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v63
@@ -4755,16 +4756,16 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v59
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
@@ -4778,27 +4779,27 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -4816,11 +4817,11 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
 ; SI-NEXT:    v_or_b32_e32 v1, v48, v1
-; SI-NEXT:    v_or_b32_e32 v2, v57, v2
-; SI-NEXT:    v_or_b32_e32 v3, v39, v3
-; SI-NEXT:    v_or_b32_e32 v4, v56, v4
-; SI-NEXT:    v_or_b32_e32 v5, v47, v5
-; SI-NEXT:    v_or_b32_e32 v6, v38, v6
+; SI-NEXT:    v_or_b32_e32 v2, v39, v2
+; SI-NEXT:    v_or_b32_e32 v3, v57, v3
+; SI-NEXT:    v_or_b32_e32 v4, v38, v4
+; SI-NEXT:    v_or_b32_e32 v5, v56, v5
+; SI-NEXT:    v_or_b32_e32 v6, v47, v6
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -4830,30 +4831,33 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
@@ -4875,16 +4879,16 @@ define <26 x i32> @bitcast_v52i16_to_v26i32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v36, v8
-; SI-NEXT:    v_or_b32_e32 v9, v46, v9
-; SI-NEXT:    v_or_b32_e32 v10, v35, v10
-; SI-NEXT:    v_or_b32_e32 v11, v45, v11
-; SI-NEXT:    v_or_b32_e32 v12, v44, v12
-; SI-NEXT:    v_or_b32_e32 v13, v34, v13
-; SI-NEXT:    v_or_b32_e32 v14, v43, v14
-; SI-NEXT:    v_or_b32_e32 v15, v42, v15
-; SI-NEXT:    v_or_b32_e32 v16, v33, v16
-; SI-NEXT:    v_or_b32_e32 v17, v41, v17
+; SI-NEXT:    v_or_b32_e32 v8, v46, v8
+; SI-NEXT:    v_or_b32_e32 v9, v45, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v35, v13
+; SI-NEXT:    v_or_b32_e32 v14, v42, v14
+; SI-NEXT:    v_or_b32_e32 v15, v34, v15
+; SI-NEXT:    v_or_b32_e32 v16, v41, v16
+; SI-NEXT:    v_or_b32_e32 v17, v33, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v40, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v32, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v63, v20
@@ -9474,50 +9478,50 @@ define <26 x i32> @bitcast_v52f16_to_v26i32(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v4, v5, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
+; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v44
 ; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
 ; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v32
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
 ; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
@@ -9525,50 +9529,50 @@ define <26 x i32> @bitcast_v52f16_to_v26i32(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
 ; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
@@ -10544,7 +10548,9 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v53
@@ -10590,8 +10596,6 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
 ; SI-NEXT:    v_or_b32_e32 v18, v33, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v59, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v27, v20
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; SI-NEXT:    s_waitcnt vmcnt(4)
@@ -15201,73 +15205,69 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
+; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:12
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v29
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v2
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v8
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v14
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -15278,60 +15278,64 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v55
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v53
@@ -15341,11 +15345,11 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v58
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v48
-; SI-NEXT:    v_or_b32_e32 v2, v2, v57
-; SI-NEXT:    v_or_b32_e32 v3, v3, v39
-; SI-NEXT:    v_or_b32_e32 v4, v4, v56
-; SI-NEXT:    v_or_b32_e32 v5, v5, v47
-; SI-NEXT:    v_or_b32_e32 v6, v6, v38
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
+; SI-NEXT:    v_or_b32_e32 v3, v3, v57
+; SI-NEXT:    v_or_b32_e32 v4, v4, v38
+; SI-NEXT:    v_or_b32_e32 v5, v5, v56
+; SI-NEXT:    v_or_b32_e32 v6, v6, v47
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -15355,27 +15359,25 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
@@ -15412,34 +15414,37 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v36
-; SI-NEXT:    v_or_b32_e32 v9, v9, v46
-; SI-NEXT:    v_or_b32_e32 v10, v10, v35
-; SI-NEXT:    v_or_b32_e32 v11, v11, v45
-; SI-NEXT:    v_or_b32_e32 v12, v12, v44
-; SI-NEXT:    v_or_b32_e32 v13, v13, v34
-; SI-NEXT:    v_or_b32_e32 v14, v14, v43
-; SI-NEXT:    v_or_b32_e32 v15, v15, v42
-; SI-NEXT:    v_or_b32_e32 v16, v16, v33
-; SI-NEXT:    v_or_b32_e32 v17, v17, v41
+; SI-NEXT:    v_or_b32_e32 v8, v8, v46
+; SI-NEXT:    v_or_b32_e32 v9, v9, v45
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v35
+; SI-NEXT:    v_or_b32_e32 v14, v14, v42
+; SI-NEXT:    v_or_b32_e32 v15, v15, v34
+; SI-NEXT:    v_or_b32_e32 v16, v16, v41
+; SI-NEXT:    v_or_b32_e32 v17, v17, v33
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v40
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v32
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v63
@@ -15449,16 +15454,16 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v59
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
@@ -15472,27 +15477,27 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -15510,11 +15515,11 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
 ; SI-NEXT:    v_or_b32_e32 v1, v48, v1
-; SI-NEXT:    v_or_b32_e32 v2, v57, v2
-; SI-NEXT:    v_or_b32_e32 v3, v39, v3
-; SI-NEXT:    v_or_b32_e32 v4, v56, v4
-; SI-NEXT:    v_or_b32_e32 v5, v47, v5
-; SI-NEXT:    v_or_b32_e32 v6, v38, v6
+; SI-NEXT:    v_or_b32_e32 v2, v39, v2
+; SI-NEXT:    v_or_b32_e32 v3, v57, v3
+; SI-NEXT:    v_or_b32_e32 v4, v38, v4
+; SI-NEXT:    v_or_b32_e32 v5, v56, v5
+; SI-NEXT:    v_or_b32_e32 v6, v47, v6
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -15524,30 +15529,33 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
@@ -15569,16 +15577,16 @@ define <26 x float> @bitcast_v52i16_to_v26f32(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v36, v8
-; SI-NEXT:    v_or_b32_e32 v9, v46, v9
-; SI-NEXT:    v_or_b32_e32 v10, v35, v10
-; SI-NEXT:    v_or_b32_e32 v11, v45, v11
-; SI-NEXT:    v_or_b32_e32 v12, v44, v12
-; SI-NEXT:    v_or_b32_e32 v13, v34, v13
-; SI-NEXT:    v_or_b32_e32 v14, v43, v14
-; SI-NEXT:    v_or_b32_e32 v15, v42, v15
-; SI-NEXT:    v_or_b32_e32 v16, v33, v16
-; SI-NEXT:    v_or_b32_e32 v17, v41, v17
+; SI-NEXT:    v_or_b32_e32 v8, v46, v8
+; SI-NEXT:    v_or_b32_e32 v9, v45, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v35, v13
+; SI-NEXT:    v_or_b32_e32 v14, v42, v14
+; SI-NEXT:    v_or_b32_e32 v15, v34, v15
+; SI-NEXT:    v_or_b32_e32 v16, v41, v16
+; SI-NEXT:    v_or_b32_e32 v17, v33, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v40, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v32, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v63, v20
@@ -20169,50 +20177,50 @@ define <26 x float> @bitcast_v52f16_to_v26f32(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v4, v5, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
+; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v44
 ; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
 ; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v32
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
 ; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
@@ -20220,50 +20228,50 @@ define <26 x float> @bitcast_v52f16_to_v26f32(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
 ; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
@@ -21239,7 +21247,9 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v53
@@ -21285,8 +21295,6 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
 ; SI-NEXT:    v_or_b32_e32 v18, v33, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v59, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v27, v20
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; SI-NEXT:    s_waitcnt vmcnt(4)
@@ -25096,73 +25104,69 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
+; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:12
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v29
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v2
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v8
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v14
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -25173,60 +25177,64 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v55
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v53
@@ -25236,11 +25244,11 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v58
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v48
-; SI-NEXT:    v_or_b32_e32 v2, v2, v57
-; SI-NEXT:    v_or_b32_e32 v3, v3, v39
-; SI-NEXT:    v_or_b32_e32 v4, v4, v56
-; SI-NEXT:    v_or_b32_e32 v5, v5, v47
-; SI-NEXT:    v_or_b32_e32 v6, v6, v38
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
+; SI-NEXT:    v_or_b32_e32 v3, v3, v57
+; SI-NEXT:    v_or_b32_e32 v4, v4, v38
+; SI-NEXT:    v_or_b32_e32 v5, v5, v56
+; SI-NEXT:    v_or_b32_e32 v6, v6, v47
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -25250,27 +25258,25 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
@@ -25307,34 +25313,37 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v36
-; SI-NEXT:    v_or_b32_e32 v9, v9, v46
-; SI-NEXT:    v_or_b32_e32 v10, v10, v35
-; SI-NEXT:    v_or_b32_e32 v11, v11, v45
-; SI-NEXT:    v_or_b32_e32 v12, v12, v44
-; SI-NEXT:    v_or_b32_e32 v13, v13, v34
-; SI-NEXT:    v_or_b32_e32 v14, v14, v43
-; SI-NEXT:    v_or_b32_e32 v15, v15, v42
-; SI-NEXT:    v_or_b32_e32 v16, v16, v33
-; SI-NEXT:    v_or_b32_e32 v17, v17, v41
+; SI-NEXT:    v_or_b32_e32 v8, v8, v46
+; SI-NEXT:    v_or_b32_e32 v9, v9, v45
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v35
+; SI-NEXT:    v_or_b32_e32 v14, v14, v42
+; SI-NEXT:    v_or_b32_e32 v15, v15, v34
+; SI-NEXT:    v_or_b32_e32 v16, v16, v41
+; SI-NEXT:    v_or_b32_e32 v17, v17, v33
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v40
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v32
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v63
@@ -25344,16 +25353,16 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v59
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
@@ -25367,27 +25376,27 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -25405,11 +25414,11 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
 ; SI-NEXT:    v_or_b32_e32 v1, v48, v1
-; SI-NEXT:    v_or_b32_e32 v2, v57, v2
-; SI-NEXT:    v_or_b32_e32 v3, v39, v3
-; SI-NEXT:    v_or_b32_e32 v4, v56, v4
-; SI-NEXT:    v_or_b32_e32 v5, v47, v5
-; SI-NEXT:    v_or_b32_e32 v6, v38, v6
+; SI-NEXT:    v_or_b32_e32 v2, v39, v2
+; SI-NEXT:    v_or_b32_e32 v3, v57, v3
+; SI-NEXT:    v_or_b32_e32 v4, v38, v4
+; SI-NEXT:    v_or_b32_e32 v5, v56, v5
+; SI-NEXT:    v_or_b32_e32 v6, v47, v6
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -25419,30 +25428,33 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
@@ -25464,16 +25476,16 @@ define <13 x i64> @bitcast_v52i16_to_v13i64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v36, v8
-; SI-NEXT:    v_or_b32_e32 v9, v46, v9
-; SI-NEXT:    v_or_b32_e32 v10, v35, v10
-; SI-NEXT:    v_or_b32_e32 v11, v45, v11
-; SI-NEXT:    v_or_b32_e32 v12, v44, v12
-; SI-NEXT:    v_or_b32_e32 v13, v34, v13
-; SI-NEXT:    v_or_b32_e32 v14, v43, v14
-; SI-NEXT:    v_or_b32_e32 v15, v42, v15
-; SI-NEXT:    v_or_b32_e32 v16, v33, v16
-; SI-NEXT:    v_or_b32_e32 v17, v41, v17
+; SI-NEXT:    v_or_b32_e32 v8, v46, v8
+; SI-NEXT:    v_or_b32_e32 v9, v45, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v35, v13
+; SI-NEXT:    v_or_b32_e32 v14, v42, v14
+; SI-NEXT:    v_or_b32_e32 v15, v34, v15
+; SI-NEXT:    v_or_b32_e32 v16, v41, v16
+; SI-NEXT:    v_or_b32_e32 v17, v33, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v40, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v32, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v63, v20
@@ -30078,50 +30090,50 @@ define <13 x i64> @bitcast_v52f16_to_v13i64(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v4, v5, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
+; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v44
 ; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
 ; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v32
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
 ; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
@@ -30129,50 +30141,50 @@ define <13 x i64> @bitcast_v52f16_to_v13i64(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
 ; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
@@ -31148,7 +31160,9 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v53
@@ -31194,8 +31208,6 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
 ; SI-NEXT:    v_or_b32_e32 v18, v33, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v59, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v27, v20
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; SI-NEXT:    s_waitcnt vmcnt(4)
@@ -34085,73 +34097,69 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    v_mov_b32_e32 v50, v10
+; SI-NEXT:    v_mov_b32_e32 v51, v8
+; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    v_mov_b32_e32 v53, v4
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:44
-; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_mov_b32_e32 v52, v6
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:12
 ; SI-NEXT:    v_mov_b32_e32 v49, v12
-; SI-NEXT:    v_mov_b32_e32 v50, v10
-; SI-NEXT:    v_mov_b32_e32 v51, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v29
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v29
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v0
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v2
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:84
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v8
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v14
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v14
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -34162,60 +34170,64 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v55
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v53
@@ -34225,11 +34237,11 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v49
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v58
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v48
-; SI-NEXT:    v_or_b32_e32 v2, v2, v57
-; SI-NEXT:    v_or_b32_e32 v3, v3, v39
-; SI-NEXT:    v_or_b32_e32 v4, v4, v56
-; SI-NEXT:    v_or_b32_e32 v5, v5, v47
-; SI-NEXT:    v_or_b32_e32 v6, v6, v38
+; SI-NEXT:    v_or_b32_e32 v2, v2, v39
+; SI-NEXT:    v_or_b32_e32 v3, v3, v57
+; SI-NEXT:    v_or_b32_e32 v4, v4, v38
+; SI-NEXT:    v_or_b32_e32 v5, v5, v56
+; SI-NEXT:    v_or_b32_e32 v6, v6, v47
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
 ; SI-NEXT:    ; implicit-def: $vgpr53
@@ -34239,27 +34251,25 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr57
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    s_waitcnt vmcnt(13)
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(12)
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v25, v25, v26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
@@ -34296,34 +34306,37 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
 ; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr26
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v36
-; SI-NEXT:    v_or_b32_e32 v9, v9, v46
-; SI-NEXT:    v_or_b32_e32 v10, v10, v35
-; SI-NEXT:    v_or_b32_e32 v11, v11, v45
-; SI-NEXT:    v_or_b32_e32 v12, v12, v44
-; SI-NEXT:    v_or_b32_e32 v13, v13, v34
-; SI-NEXT:    v_or_b32_e32 v14, v14, v43
-; SI-NEXT:    v_or_b32_e32 v15, v15, v42
-; SI-NEXT:    v_or_b32_e32 v16, v16, v33
-; SI-NEXT:    v_or_b32_e32 v17, v17, v41
+; SI-NEXT:    v_or_b32_e32 v8, v8, v46
+; SI-NEXT:    v_or_b32_e32 v9, v9, v45
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v44
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v35
+; SI-NEXT:    v_or_b32_e32 v14, v14, v42
+; SI-NEXT:    v_or_b32_e32 v15, v15, v34
+; SI-NEXT:    v_or_b32_e32 v16, v16, v41
+; SI-NEXT:    v_or_b32_e32 v17, v17, v33
 ; SI-NEXT:    v_or_b32_e32 v18, v18, v40
 ; SI-NEXT:    v_or_b32_e32 v19, v19, v32
 ; SI-NEXT:    v_or_b32_e32 v20, v20, v63
@@ -34333,16 +34346,16 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v24, v24, v59
 ; SI-NEXT:    ; kill: killed $vgpr26
 ; SI-NEXT:    ; implicit-def: $vgpr37
-; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr35
 ; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr34
 ; SI-NEXT:    ; implicit-def: $vgpr41
+; SI-NEXT:    ; implicit-def: $vgpr33
 ; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
@@ -34356,27 +34369,27 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -34394,11 +34407,11 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
 ; SI-NEXT:    v_or_b32_e32 v1, v48, v1
-; SI-NEXT:    v_or_b32_e32 v2, v57, v2
-; SI-NEXT:    v_or_b32_e32 v3, v39, v3
-; SI-NEXT:    v_or_b32_e32 v4, v56, v4
-; SI-NEXT:    v_or_b32_e32 v5, v47, v5
-; SI-NEXT:    v_or_b32_e32 v6, v38, v6
+; SI-NEXT:    v_or_b32_e32 v2, v39, v2
+; SI-NEXT:    v_or_b32_e32 v3, v57, v3
+; SI-NEXT:    v_or_b32_e32 v4, v38, v4
+; SI-NEXT:    v_or_b32_e32 v5, v56, v5
+; SI-NEXT:    v_or_b32_e32 v6, v47, v6
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
@@ -34408,30 +34421,33 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
@@ -34453,16 +34469,16 @@ define <13 x double> @bitcast_v52i16_to_v13f64(<52 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v36, v8
-; SI-NEXT:    v_or_b32_e32 v9, v46, v9
-; SI-NEXT:    v_or_b32_e32 v10, v35, v10
-; SI-NEXT:    v_or_b32_e32 v11, v45, v11
-; SI-NEXT:    v_or_b32_e32 v12, v44, v12
-; SI-NEXT:    v_or_b32_e32 v13, v34, v13
-; SI-NEXT:    v_or_b32_e32 v14, v43, v14
-; SI-NEXT:    v_or_b32_e32 v15, v42, v15
-; SI-NEXT:    v_or_b32_e32 v16, v33, v16
-; SI-NEXT:    v_or_b32_e32 v17, v41, v17
+; SI-NEXT:    v_or_b32_e32 v8, v46, v8
+; SI-NEXT:    v_or_b32_e32 v9, v45, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v44, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v35, v13
+; SI-NEXT:    v_or_b32_e32 v14, v42, v14
+; SI-NEXT:    v_or_b32_e32 v15, v34, v15
+; SI-NEXT:    v_or_b32_e32 v16, v41, v16
+; SI-NEXT:    v_or_b32_e32 v17, v33, v17
 ; SI-NEXT:    v_or_b32_e32 v18, v40, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v32, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v63, v20
@@ -38962,50 +38978,50 @@ define <13 x double> @bitcast_v52f16_to_v13f64(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
-; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_or_b32_e32 v3, v4, v3
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v51
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_or_b32_e32 v4, v5, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v58
+; SI-NEXT:    v_cvt_f32_f16_e32 v23, v47
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v46
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v44
 ; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT:    v_or_b32_e32 v4, v5, v4
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v49
 ; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    v_cvt_f32_f16_e32 v26, v33
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v32
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
 ; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
@@ -39013,50 +39029,50 @@ define <13 x double> @bitcast_v52f16_to_v13f64(<52 x half> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v5, v6, v5
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
 ; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
 ; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
 ; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
 ; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
 ; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
 ; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
-; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
@@ -40032,7 +40048,9 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v53
@@ -40078,8 +40096,6 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
 ; SI-NEXT:    v_or_b32_e32 v18, v33, v18
 ; SI-NEXT:    v_or_b32_e32 v19, v59, v19
 ; SI-NEXT:    v_or_b32_e32 v20, v27, v20
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; SI-NEXT:    s_waitcnt vmcnt(4)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
index 4d6c274965cb..510deaa5787f 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
@@ -4831,152 +4831,152 @@ define <28 x i32> @bitcast_v56i16_to_v28i32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:52
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v22
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v24
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
@@ -5026,9 +5026,9 @@ define <28 x i32> @bitcast_v56i16_to_v28i32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v59
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    v_or_b32_e32 v0, v0, v52
-; SI-NEXT:    v_or_b32_e32 v1, v1, v58
-; SI-NEXT:    v_or_b32_e32 v2, v2, v51
+; SI-NEXT:    v_or_b32_e32 v0, v0, v58
+; SI-NEXT:    v_or_b32_e32 v1, v1, v52
+; SI-NEXT:    v_or_b32_e32 v2, v2, v57
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v49
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
@@ -5037,125 +5037,125 @@ define <28 x i32> @bitcast_v56i16_to_v28i32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
-; SI-NEXT:    ; implicit-def: $vgpr52
 ; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT:    v_or_b32_e32 v3, v3, v57
-; SI-NEXT:    v_or_b32_e32 v4, v4, v48
-; SI-NEXT:    v_or_b32_e32 v5, v5, v39
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_or_b32_e32 v3, v3, v51
+; SI-NEXT:    v_or_b32_e32 v4, v4, v56
+; SI-NEXT:    v_or_b32_e32 v5, v5, v50
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v46
-; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v45
-; SI-NEXT:    v_or_b32_e32 v9, v9, v44
-; SI-NEXT:    v_or_b32_e32 v10, v10, v34
-; SI-NEXT:    v_or_b32_e32 v11, v11, v33
-; SI-NEXT:    v_or_b32_e32 v12, v12, v32
-; SI-NEXT:    v_or_b32_e32 v13, v13, v63
-; SI-NEXT:    v_or_b32_e32 v14, v14, v61
-; SI-NEXT:    v_or_b32_e32 v15, v15, v50
-; SI-NEXT:    v_or_b32_e32 v17, v17, v56
-; SI-NEXT:    v_or_b32_e32 v18, v18, v47
-; SI-NEXT:    v_or_b32_e32 v19, v19, v38
-; SI-NEXT:    v_or_b32_e32 v20, v20, v36
-; SI-NEXT:    v_or_b32_e32 v21, v21, v35
-; SI-NEXT:    v_or_b32_e32 v22, v22, v43
-; SI-NEXT:    v_or_b32_e32 v23, v23, v42
-; SI-NEXT:    v_or_b32_e32 v24, v24, v41
-; SI-NEXT:    v_or_b32_e32 v25, v25, v40
-; SI-NEXT:    v_or_b32_e32 v26, v26, v62
+; SI-NEXT:    v_or_b32_e32 v7, v7, v45
+; SI-NEXT:    v_or_b32_e32 v8, v8, v39
+; SI-NEXT:    v_or_b32_e32 v9, v9, v37
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v34
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
+; SI-NEXT:    v_or_b32_e32 v14, v14, v40
+; SI-NEXT:    v_or_b32_e32 v15, v15, v47
+; SI-NEXT:    v_or_b32_e32 v17, v17, v48
+; SI-NEXT:    v_or_b32_e32 v18, v18, v38
+; SI-NEXT:    v_or_b32_e32 v19, v19, v44
+; SI-NEXT:    v_or_b32_e32 v20, v20, v35
+; SI-NEXT:    v_or_b32_e32 v21, v21, v33
+; SI-NEXT:    v_or_b32_e32 v22, v22, v32
+; SI-NEXT:    v_or_b32_e32 v23, v23, v41
+; SI-NEXT:    v_or_b32_e32 v24, v24, v63
+; SI-NEXT:    v_or_b32_e32 v25, v25, v62
+; SI-NEXT:    v_or_b32_e32 v26, v26, v61
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v60
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:  .LBB14_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -5164,10 +5164,10 @@ define <28 x i32> @bitcast_v56i16_to_v28i32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v52, v0
+; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v58, v1
-; SI-NEXT:    v_or_b32_e32 v2, v51, v2
+; SI-NEXT:    v_or_b32_e32 v1, v52, v1
+; SI-NEXT:    v_or_b32_e32 v2, v57, v2
 ; SI-NEXT:    v_or_b32_e32 v16, v49, v16
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
@@ -5177,35 +5177,37 @@ define <28 x i32> @bitcast_v56i16_to_v28i32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
@@ -5231,29 +5233,29 @@ define <28 x i32> @bitcast_v56i16_to_v28i32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; SI-NEXT:    v_or_b32_e32 v3, v57, v3
-; SI-NEXT:    v_or_b32_e32 v4, v48, v4
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v3, v51, v3
+; SI-NEXT:    v_or_b32_e32 v4, v56, v4
+; SI-NEXT:    v_or_b32_e32 v5, v50, v5
 ; SI-NEXT:    v_or_b32_e32 v6, v46, v6
-; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    v_or_b32_e32 v9, v44, v9
-; SI-NEXT:    v_or_b32_e32 v10, v34, v10
-; SI-NEXT:    v_or_b32_e32 v11, v33, v11
-; SI-NEXT:    v_or_b32_e32 v12, v32, v12
-; SI-NEXT:    v_or_b32_e32 v13, v63, v13
-; SI-NEXT:    v_or_b32_e32 v14, v61, v14
-; SI-NEXT:    v_or_b32_e32 v15, v50, v15
-; SI-NEXT:    v_or_b32_e32 v17, v56, v17
-; SI-NEXT:    v_or_b32_e32 v18, v47, v18
-; SI-NEXT:    v_or_b32_e32 v19, v38, v19
-; SI-NEXT:    v_or_b32_e32 v20, v36, v20
-; SI-NEXT:    v_or_b32_e32 v21, v35, v21
-; SI-NEXT:    v_or_b32_e32 v22, v43, v22
-; SI-NEXT:    v_or_b32_e32 v23, v42, v23
-; SI-NEXT:    v_or_b32_e32 v24, v41, v24
-; SI-NEXT:    v_or_b32_e32 v25, v40, v25
-; SI-NEXT:    v_or_b32_e32 v26, v62, v26
+; SI-NEXT:    v_or_b32_e32 v7, v45, v7
+; SI-NEXT:    v_or_b32_e32 v8, v39, v8
+; SI-NEXT:    v_or_b32_e32 v9, v37, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v34, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
+; SI-NEXT:    v_or_b32_e32 v14, v40, v14
+; SI-NEXT:    v_or_b32_e32 v15, v47, v15
+; SI-NEXT:    v_or_b32_e32 v17, v48, v17
+; SI-NEXT:    v_or_b32_e32 v18, v38, v18
+; SI-NEXT:    v_or_b32_e32 v19, v44, v19
+; SI-NEXT:    v_or_b32_e32 v20, v35, v20
+; SI-NEXT:    v_or_b32_e32 v21, v33, v21
+; SI-NEXT:    v_or_b32_e32 v22, v32, v22
+; SI-NEXT:    v_or_b32_e32 v23, v41, v23
+; SI-NEXT:    v_or_b32_e32 v24, v63, v24
+; SI-NEXT:    v_or_b32_e32 v25, v62, v25
+; SI-NEXT:    v_or_b32_e32 v26, v61, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v60, v27
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
@@ -11430,12 +11432,12 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v18, v16
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mov_b32_e32 v49, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -11485,7 +11487,7 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -11584,21 +11586,21 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
@@ -11611,37 +11613,37 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
-; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
@@ -16445,152 +16447,152 @@ define <28 x float> @bitcast_v56i16_to_v28f32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:52
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v22
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v24
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
@@ -16640,9 +16642,9 @@ define <28 x float> @bitcast_v56i16_to_v28f32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v59
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    v_or_b32_e32 v0, v0, v52
-; SI-NEXT:    v_or_b32_e32 v1, v1, v58
-; SI-NEXT:    v_or_b32_e32 v2, v2, v51
+; SI-NEXT:    v_or_b32_e32 v0, v0, v58
+; SI-NEXT:    v_or_b32_e32 v1, v1, v52
+; SI-NEXT:    v_or_b32_e32 v2, v2, v57
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v49
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
@@ -16651,125 +16653,125 @@ define <28 x float> @bitcast_v56i16_to_v28f32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
-; SI-NEXT:    ; implicit-def: $vgpr52
 ; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT:    v_or_b32_e32 v3, v3, v57
-; SI-NEXT:    v_or_b32_e32 v4, v4, v48
-; SI-NEXT:    v_or_b32_e32 v5, v5, v39
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_or_b32_e32 v3, v3, v51
+; SI-NEXT:    v_or_b32_e32 v4, v4, v56
+; SI-NEXT:    v_or_b32_e32 v5, v5, v50
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v46
-; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v45
-; SI-NEXT:    v_or_b32_e32 v9, v9, v44
-; SI-NEXT:    v_or_b32_e32 v10, v10, v34
-; SI-NEXT:    v_or_b32_e32 v11, v11, v33
-; SI-NEXT:    v_or_b32_e32 v12, v12, v32
-; SI-NEXT:    v_or_b32_e32 v13, v13, v63
-; SI-NEXT:    v_or_b32_e32 v14, v14, v61
-; SI-NEXT:    v_or_b32_e32 v15, v15, v50
-; SI-NEXT:    v_or_b32_e32 v17, v17, v56
-; SI-NEXT:    v_or_b32_e32 v18, v18, v47
-; SI-NEXT:    v_or_b32_e32 v19, v19, v38
-; SI-NEXT:    v_or_b32_e32 v20, v20, v36
-; SI-NEXT:    v_or_b32_e32 v21, v21, v35
-; SI-NEXT:    v_or_b32_e32 v22, v22, v43
-; SI-NEXT:    v_or_b32_e32 v23, v23, v42
-; SI-NEXT:    v_or_b32_e32 v24, v24, v41
-; SI-NEXT:    v_or_b32_e32 v25, v25, v40
-; SI-NEXT:    v_or_b32_e32 v26, v26, v62
+; SI-NEXT:    v_or_b32_e32 v7, v7, v45
+; SI-NEXT:    v_or_b32_e32 v8, v8, v39
+; SI-NEXT:    v_or_b32_e32 v9, v9, v37
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v34
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
+; SI-NEXT:    v_or_b32_e32 v14, v14, v40
+; SI-NEXT:    v_or_b32_e32 v15, v15, v47
+; SI-NEXT:    v_or_b32_e32 v17, v17, v48
+; SI-NEXT:    v_or_b32_e32 v18, v18, v38
+; SI-NEXT:    v_or_b32_e32 v19, v19, v44
+; SI-NEXT:    v_or_b32_e32 v20, v20, v35
+; SI-NEXT:    v_or_b32_e32 v21, v21, v33
+; SI-NEXT:    v_or_b32_e32 v22, v22, v32
+; SI-NEXT:    v_or_b32_e32 v23, v23, v41
+; SI-NEXT:    v_or_b32_e32 v24, v24, v63
+; SI-NEXT:    v_or_b32_e32 v25, v25, v62
+; SI-NEXT:    v_or_b32_e32 v26, v26, v61
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v60
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:  .LBB30_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -16778,10 +16780,10 @@ define <28 x float> @bitcast_v56i16_to_v28f32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v52, v0
+; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v58, v1
-; SI-NEXT:    v_or_b32_e32 v2, v51, v2
+; SI-NEXT:    v_or_b32_e32 v1, v52, v1
+; SI-NEXT:    v_or_b32_e32 v2, v57, v2
 ; SI-NEXT:    v_or_b32_e32 v16, v49, v16
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
@@ -16791,35 +16793,37 @@ define <28 x float> @bitcast_v56i16_to_v28f32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
@@ -16845,29 +16849,29 @@ define <28 x float> @bitcast_v56i16_to_v28f32(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; SI-NEXT:    v_or_b32_e32 v3, v57, v3
-; SI-NEXT:    v_or_b32_e32 v4, v48, v4
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v3, v51, v3
+; SI-NEXT:    v_or_b32_e32 v4, v56, v4
+; SI-NEXT:    v_or_b32_e32 v5, v50, v5
 ; SI-NEXT:    v_or_b32_e32 v6, v46, v6
-; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    v_or_b32_e32 v9, v44, v9
-; SI-NEXT:    v_or_b32_e32 v10, v34, v10
-; SI-NEXT:    v_or_b32_e32 v11, v33, v11
-; SI-NEXT:    v_or_b32_e32 v12, v32, v12
-; SI-NEXT:    v_or_b32_e32 v13, v63, v13
-; SI-NEXT:    v_or_b32_e32 v14, v61, v14
-; SI-NEXT:    v_or_b32_e32 v15, v50, v15
-; SI-NEXT:    v_or_b32_e32 v17, v56, v17
-; SI-NEXT:    v_or_b32_e32 v18, v47, v18
-; SI-NEXT:    v_or_b32_e32 v19, v38, v19
-; SI-NEXT:    v_or_b32_e32 v20, v36, v20
-; SI-NEXT:    v_or_b32_e32 v21, v35, v21
-; SI-NEXT:    v_or_b32_e32 v22, v43, v22
-; SI-NEXT:    v_or_b32_e32 v23, v42, v23
-; SI-NEXT:    v_or_b32_e32 v24, v41, v24
-; SI-NEXT:    v_or_b32_e32 v25, v40, v25
-; SI-NEXT:    v_or_b32_e32 v26, v62, v26
+; SI-NEXT:    v_or_b32_e32 v7, v45, v7
+; SI-NEXT:    v_or_b32_e32 v8, v39, v8
+; SI-NEXT:    v_or_b32_e32 v9, v37, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v34, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
+; SI-NEXT:    v_or_b32_e32 v14, v40, v14
+; SI-NEXT:    v_or_b32_e32 v15, v47, v15
+; SI-NEXT:    v_or_b32_e32 v17, v48, v17
+; SI-NEXT:    v_or_b32_e32 v18, v38, v18
+; SI-NEXT:    v_or_b32_e32 v19, v44, v19
+; SI-NEXT:    v_or_b32_e32 v20, v35, v20
+; SI-NEXT:    v_or_b32_e32 v21, v33, v21
+; SI-NEXT:    v_or_b32_e32 v22, v32, v22
+; SI-NEXT:    v_or_b32_e32 v23, v41, v23
+; SI-NEXT:    v_or_b32_e32 v24, v63, v24
+; SI-NEXT:    v_or_b32_e32 v25, v62, v25
+; SI-NEXT:    v_or_b32_e32 v26, v61, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v60, v27
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
@@ -23034,12 +23038,12 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v18, v16
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mov_b32_e32 v49, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -23089,7 +23093,7 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -23188,21 +23192,21 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
@@ -23215,37 +23219,37 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
-; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
@@ -27200,152 +27204,152 @@ define <14 x i64> @bitcast_v56i16_to_v14i64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:52
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v22
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v24
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
@@ -27395,9 +27399,9 @@ define <14 x i64> @bitcast_v56i16_to_v14i64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v59
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    v_or_b32_e32 v0, v0, v52
-; SI-NEXT:    v_or_b32_e32 v1, v1, v58
-; SI-NEXT:    v_or_b32_e32 v2, v2, v51
+; SI-NEXT:    v_or_b32_e32 v0, v0, v58
+; SI-NEXT:    v_or_b32_e32 v1, v1, v52
+; SI-NEXT:    v_or_b32_e32 v2, v2, v57
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v49
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
@@ -27406,125 +27410,125 @@ define <14 x i64> @bitcast_v56i16_to_v14i64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
-; SI-NEXT:    ; implicit-def: $vgpr52
 ; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT:    v_or_b32_e32 v3, v3, v57
-; SI-NEXT:    v_or_b32_e32 v4, v4, v48
-; SI-NEXT:    v_or_b32_e32 v5, v5, v39
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_or_b32_e32 v3, v3, v51
+; SI-NEXT:    v_or_b32_e32 v4, v4, v56
+; SI-NEXT:    v_or_b32_e32 v5, v5, v50
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v46
-; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v45
-; SI-NEXT:    v_or_b32_e32 v9, v9, v44
-; SI-NEXT:    v_or_b32_e32 v10, v10, v34
-; SI-NEXT:    v_or_b32_e32 v11, v11, v33
-; SI-NEXT:    v_or_b32_e32 v12, v12, v32
-; SI-NEXT:    v_or_b32_e32 v13, v13, v63
-; SI-NEXT:    v_or_b32_e32 v14, v14, v61
-; SI-NEXT:    v_or_b32_e32 v15, v15, v50
-; SI-NEXT:    v_or_b32_e32 v17, v17, v56
-; SI-NEXT:    v_or_b32_e32 v18, v18, v47
-; SI-NEXT:    v_or_b32_e32 v19, v19, v38
-; SI-NEXT:    v_or_b32_e32 v20, v20, v36
-; SI-NEXT:    v_or_b32_e32 v21, v21, v35
-; SI-NEXT:    v_or_b32_e32 v22, v22, v43
-; SI-NEXT:    v_or_b32_e32 v23, v23, v42
-; SI-NEXT:    v_or_b32_e32 v24, v24, v41
-; SI-NEXT:    v_or_b32_e32 v25, v25, v40
-; SI-NEXT:    v_or_b32_e32 v26, v26, v62
+; SI-NEXT:    v_or_b32_e32 v7, v7, v45
+; SI-NEXT:    v_or_b32_e32 v8, v8, v39
+; SI-NEXT:    v_or_b32_e32 v9, v9, v37
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v34
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
+; SI-NEXT:    v_or_b32_e32 v14, v14, v40
+; SI-NEXT:    v_or_b32_e32 v15, v15, v47
+; SI-NEXT:    v_or_b32_e32 v17, v17, v48
+; SI-NEXT:    v_or_b32_e32 v18, v18, v38
+; SI-NEXT:    v_or_b32_e32 v19, v19, v44
+; SI-NEXT:    v_or_b32_e32 v20, v20, v35
+; SI-NEXT:    v_or_b32_e32 v21, v21, v33
+; SI-NEXT:    v_or_b32_e32 v22, v22, v32
+; SI-NEXT:    v_or_b32_e32 v23, v23, v41
+; SI-NEXT:    v_or_b32_e32 v24, v24, v63
+; SI-NEXT:    v_or_b32_e32 v25, v25, v62
+; SI-NEXT:    v_or_b32_e32 v26, v26, v61
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v60
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:  .LBB42_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -27533,10 +27537,10 @@ define <14 x i64> @bitcast_v56i16_to_v14i64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v52, v0
+; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v58, v1
-; SI-NEXT:    v_or_b32_e32 v2, v51, v2
+; SI-NEXT:    v_or_b32_e32 v1, v52, v1
+; SI-NEXT:    v_or_b32_e32 v2, v57, v2
 ; SI-NEXT:    v_or_b32_e32 v16, v49, v16
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
@@ -27546,35 +27550,37 @@ define <14 x i64> @bitcast_v56i16_to_v14i64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
@@ -27600,29 +27606,29 @@ define <14 x i64> @bitcast_v56i16_to_v14i64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; SI-NEXT:    v_or_b32_e32 v3, v57, v3
-; SI-NEXT:    v_or_b32_e32 v4, v48, v4
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v3, v51, v3
+; SI-NEXT:    v_or_b32_e32 v4, v56, v4
+; SI-NEXT:    v_or_b32_e32 v5, v50, v5
 ; SI-NEXT:    v_or_b32_e32 v6, v46, v6
-; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    v_or_b32_e32 v9, v44, v9
-; SI-NEXT:    v_or_b32_e32 v10, v34, v10
-; SI-NEXT:    v_or_b32_e32 v11, v33, v11
-; SI-NEXT:    v_or_b32_e32 v12, v32, v12
-; SI-NEXT:    v_or_b32_e32 v13, v63, v13
-; SI-NEXT:    v_or_b32_e32 v14, v61, v14
-; SI-NEXT:    v_or_b32_e32 v15, v50, v15
-; SI-NEXT:    v_or_b32_e32 v17, v56, v17
-; SI-NEXT:    v_or_b32_e32 v18, v47, v18
-; SI-NEXT:    v_or_b32_e32 v19, v38, v19
-; SI-NEXT:    v_or_b32_e32 v20, v36, v20
-; SI-NEXT:    v_or_b32_e32 v21, v35, v21
-; SI-NEXT:    v_or_b32_e32 v22, v43, v22
-; SI-NEXT:    v_or_b32_e32 v23, v42, v23
-; SI-NEXT:    v_or_b32_e32 v24, v41, v24
-; SI-NEXT:    v_or_b32_e32 v25, v40, v25
-; SI-NEXT:    v_or_b32_e32 v26, v62, v26
+; SI-NEXT:    v_or_b32_e32 v7, v45, v7
+; SI-NEXT:    v_or_b32_e32 v8, v39, v8
+; SI-NEXT:    v_or_b32_e32 v9, v37, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v34, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
+; SI-NEXT:    v_or_b32_e32 v14, v40, v14
+; SI-NEXT:    v_or_b32_e32 v15, v47, v15
+; SI-NEXT:    v_or_b32_e32 v17, v48, v17
+; SI-NEXT:    v_or_b32_e32 v18, v38, v18
+; SI-NEXT:    v_or_b32_e32 v19, v44, v19
+; SI-NEXT:    v_or_b32_e32 v20, v35, v20
+; SI-NEXT:    v_or_b32_e32 v21, v33, v21
+; SI-NEXT:    v_or_b32_e32 v22, v32, v22
+; SI-NEXT:    v_or_b32_e32 v23, v41, v23
+; SI-NEXT:    v_or_b32_e32 v24, v63, v24
+; SI-NEXT:    v_or_b32_e32 v25, v62, v25
+; SI-NEXT:    v_or_b32_e32 v26, v61, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v60, v27
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
@@ -33813,12 +33819,12 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v18, v16
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mov_b32_e32 v49, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -33868,7 +33874,7 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -33967,21 +33973,21 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
@@ -33994,37 +34000,37 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
-; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
@@ -37005,152 +37011,152 @@ define <14 x double> @bitcast_v56i16_to_v14f64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v54, v2
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:92
 ; SI-NEXT:    v_mov_b32_e32 v53, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v27
-; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v29
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v29
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v61, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:76
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:68
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v6
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v8
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v16
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v18
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:20
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:52
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v20
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:84
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v22
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:44
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v24
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:28
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
@@ -37200,9 +37206,9 @@ define <14 x double> @bitcast_v56i16_to_v14f64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v59
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
-; SI-NEXT:    v_or_b32_e32 v0, v0, v52
-; SI-NEXT:    v_or_b32_e32 v1, v1, v58
-; SI-NEXT:    v_or_b32_e32 v2, v2, v51
+; SI-NEXT:    v_or_b32_e32 v0, v0, v58
+; SI-NEXT:    v_or_b32_e32 v1, v1, v52
+; SI-NEXT:    v_or_b32_e32 v2, v2, v57
 ; SI-NEXT:    v_or_b32_e32 v16, v16, v49
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr54
@@ -37211,125 +37217,125 @@ define <14 x double> @bitcast_v56i16_to_v14f64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    ; kill: killed $vgpr28
 ; SI-NEXT:    ; implicit-def: $vgpr28
 ; SI-NEXT:    ; kill: killed $vgpr28
-; SI-NEXT:    ; implicit-def: $vgpr52
 ; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT:    v_or_b32_e32 v3, v3, v57
-; SI-NEXT:    v_or_b32_e32 v4, v4, v48
-; SI-NEXT:    v_or_b32_e32 v5, v5, v39
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_or_b32_e32 v3, v3, v51
+; SI-NEXT:    v_or_b32_e32 v4, v4, v56
+; SI-NEXT:    v_or_b32_e32 v5, v5, v50
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v46
-; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_or_b32_e32 v8, v8, v45
-; SI-NEXT:    v_or_b32_e32 v9, v9, v44
-; SI-NEXT:    v_or_b32_e32 v10, v10, v34
-; SI-NEXT:    v_or_b32_e32 v11, v11, v33
-; SI-NEXT:    v_or_b32_e32 v12, v12, v32
-; SI-NEXT:    v_or_b32_e32 v13, v13, v63
-; SI-NEXT:    v_or_b32_e32 v14, v14, v61
-; SI-NEXT:    v_or_b32_e32 v15, v15, v50
-; SI-NEXT:    v_or_b32_e32 v17, v17, v56
-; SI-NEXT:    v_or_b32_e32 v18, v18, v47
-; SI-NEXT:    v_or_b32_e32 v19, v19, v38
-; SI-NEXT:    v_or_b32_e32 v20, v20, v36
-; SI-NEXT:    v_or_b32_e32 v21, v21, v35
-; SI-NEXT:    v_or_b32_e32 v22, v22, v43
-; SI-NEXT:    v_or_b32_e32 v23, v23, v42
-; SI-NEXT:    v_or_b32_e32 v24, v24, v41
-; SI-NEXT:    v_or_b32_e32 v25, v25, v40
-; SI-NEXT:    v_or_b32_e32 v26, v26, v62
+; SI-NEXT:    v_or_b32_e32 v7, v7, v45
+; SI-NEXT:    v_or_b32_e32 v8, v8, v39
+; SI-NEXT:    v_or_b32_e32 v9, v9, v37
+; SI-NEXT:    v_or_b32_e32 v10, v10, v36
+; SI-NEXT:    v_or_b32_e32 v11, v11, v34
+; SI-NEXT:    v_or_b32_e32 v12, v12, v43
+; SI-NEXT:    v_or_b32_e32 v13, v13, v42
+; SI-NEXT:    v_or_b32_e32 v14, v14, v40
+; SI-NEXT:    v_or_b32_e32 v15, v15, v47
+; SI-NEXT:    v_or_b32_e32 v17, v17, v48
+; SI-NEXT:    v_or_b32_e32 v18, v18, v38
+; SI-NEXT:    v_or_b32_e32 v19, v19, v44
+; SI-NEXT:    v_or_b32_e32 v20, v20, v35
+; SI-NEXT:    v_or_b32_e32 v21, v21, v33
+; SI-NEXT:    v_or_b32_e32 v22, v22, v32
+; SI-NEXT:    v_or_b32_e32 v23, v23, v41
+; SI-NEXT:    v_or_b32_e32 v24, v24, v63
+; SI-NEXT:    v_or_b32_e32 v25, v25, v62
+; SI-NEXT:    v_or_b32_e32 v26, v26, v61
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v60
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr51
+; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr45
-; SI-NEXT:    ; implicit-def: $vgpr44
+; SI-NEXT:    ; implicit-def: $vgpr39
+; SI-NEXT:    ; implicit-def: $vgpr37
+; SI-NEXT:    ; implicit-def: $vgpr36
 ; SI-NEXT:    ; implicit-def: $vgpr34
-; SI-NEXT:    ; implicit-def: $vgpr33
-; SI-NEXT:    ; implicit-def: $vgpr32
-; SI-NEXT:    ; implicit-def: $vgpr63
-; SI-NEXT:    ; implicit-def: $vgpr61
-; SI-NEXT:    ; implicit-def: $vgpr50
-; SI-NEXT:    ; implicit-def: $vgpr56
+; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr48
 ; SI-NEXT:    ; implicit-def: $vgpr38
-; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr44
 ; SI-NEXT:    ; implicit-def: $vgpr35
-; SI-NEXT:    ; implicit-def: $vgpr43
-; SI-NEXT:    ; implicit-def: $vgpr42
+; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
+; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
+; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; implicit-def: $vgpr60
 ; SI-NEXT:  .LBB50_2: ; %Flow
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v54
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v53
@@ -37338,10 +37344,10 @@ define <14 x double> @bitcast_v56i16_to_v14f64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT:    v_or_b32_e32 v0, v52, v0
+; SI-NEXT:    v_or_b32_e32 v0, v58, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v1, v58, v1
-; SI-NEXT:    v_or_b32_e32 v2, v51, v2
+; SI-NEXT:    v_or_b32_e32 v1, v52, v1
+; SI-NEXT:    v_or_b32_e32 v2, v57, v2
 ; SI-NEXT:    v_or_b32_e32 v16, v49, v16
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
@@ -37351,35 +37357,37 @@ define <14 x double> @bitcast_v56i16_to_v14f64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
+; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
 ; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
-; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v18
 ; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
-; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
@@ -37405,29 +37413,29 @@ define <14 x double> @bitcast_v56i16_to_v14f64(<56 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; SI-NEXT:    v_or_b32_e32 v3, v57, v3
-; SI-NEXT:    v_or_b32_e32 v4, v48, v4
-; SI-NEXT:    v_or_b32_e32 v5, v39, v5
+; SI-NEXT:    v_or_b32_e32 v3, v51, v3
+; SI-NEXT:    v_or_b32_e32 v4, v56, v4
+; SI-NEXT:    v_or_b32_e32 v5, v50, v5
 ; SI-NEXT:    v_or_b32_e32 v6, v46, v6
-; SI-NEXT:    v_or_b32_e32 v7, v37, v7
-; SI-NEXT:    v_or_b32_e32 v8, v45, v8
-; SI-NEXT:    v_or_b32_e32 v9, v44, v9
-; SI-NEXT:    v_or_b32_e32 v10, v34, v10
-; SI-NEXT:    v_or_b32_e32 v11, v33, v11
-; SI-NEXT:    v_or_b32_e32 v12, v32, v12
-; SI-NEXT:    v_or_b32_e32 v13, v63, v13
-; SI-NEXT:    v_or_b32_e32 v14, v61, v14
-; SI-NEXT:    v_or_b32_e32 v15, v50, v15
-; SI-NEXT:    v_or_b32_e32 v17, v56, v17
-; SI-NEXT:    v_or_b32_e32 v18, v47, v18
-; SI-NEXT:    v_or_b32_e32 v19, v38, v19
-; SI-NEXT:    v_or_b32_e32 v20, v36, v20
-; SI-NEXT:    v_or_b32_e32 v21, v35, v21
-; SI-NEXT:    v_or_b32_e32 v22, v43, v22
-; SI-NEXT:    v_or_b32_e32 v23, v42, v23
-; SI-NEXT:    v_or_b32_e32 v24, v41, v24
-; SI-NEXT:    v_or_b32_e32 v25, v40, v25
-; SI-NEXT:    v_or_b32_e32 v26, v62, v26
+; SI-NEXT:    v_or_b32_e32 v7, v45, v7
+; SI-NEXT:    v_or_b32_e32 v8, v39, v8
+; SI-NEXT:    v_or_b32_e32 v9, v37, v9
+; SI-NEXT:    v_or_b32_e32 v10, v36, v10
+; SI-NEXT:    v_or_b32_e32 v11, v34, v11
+; SI-NEXT:    v_or_b32_e32 v12, v43, v12
+; SI-NEXT:    v_or_b32_e32 v13, v42, v13
+; SI-NEXT:    v_or_b32_e32 v14, v40, v14
+; SI-NEXT:    v_or_b32_e32 v15, v47, v15
+; SI-NEXT:    v_or_b32_e32 v17, v48, v17
+; SI-NEXT:    v_or_b32_e32 v18, v38, v18
+; SI-NEXT:    v_or_b32_e32 v19, v44, v19
+; SI-NEXT:    v_or_b32_e32 v20, v35, v20
+; SI-NEXT:    v_or_b32_e32 v21, v33, v21
+; SI-NEXT:    v_or_b32_e32 v22, v32, v22
+; SI-NEXT:    v_or_b32_e32 v23, v41, v23
+; SI-NEXT:    v_or_b32_e32 v24, v63, v24
+; SI-NEXT:    v_or_b32_e32 v25, v62, v25
+; SI-NEXT:    v_or_b32_e32 v26, v61, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v60, v27
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
 ; SI-NEXT:    v_add_i32_e32 v4, vcc, s6, v4
@@ -43505,12 +43513,12 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; SI-NEXT:    v_or_b32_e32 v16, v18, v16
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    v_mov_b32_e32 v49, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -43560,7 +43568,7 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
 ; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(10)
 ; SI-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_or_b32_e32 v18, v19, v18
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -43659,21 +43667,21 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v22, v29
 ; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
@@ -43686,37 +43694,37 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
 ; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
 ; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
 ; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
-; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
-; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
 ; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
 ; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index 03295ba8b531..bc8f559d2e9c 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -5148,167 +5148,167 @@ define <30 x i32> @bitcast_v60i16_to_v30i32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v29
 ; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v8
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v14
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v16
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:52
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v28
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v30
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:88
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v30
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
@@ -5365,111 +5365,111 @@ define <30 x i32> @bitcast_v60i16_to_v30i32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    v_or_b32_e32 v0, v0, v54
-; SI-NEXT:    v_or_b32_e32 v18, v18, v43
+; SI-NEXT:    v_or_b32_e32 v0, v0, v60
+; SI-NEXT:    v_or_b32_e32 v18, v18, v37
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; SI-NEXT:    v_or_b32_e32 v1, v1, v60
-; SI-NEXT:    v_or_b32_e32 v2, v2, v53
-; SI-NEXT:    v_or_b32_e32 v3, v3, v52
-; SI-NEXT:    v_or_b32_e32 v4, v4, v51
-; SI-NEXT:    v_or_b32_e32 v5, v5, v59
-; SI-NEXT:    v_or_b32_e32 v6, v6, v50
-; SI-NEXT:    v_or_b32_e32 v7, v7, v49
-; SI-NEXT:    v_or_b32_e32 v8, v8, v48
-; SI-NEXT:    v_or_b32_e32 v9, v9, v58
-; SI-NEXT:    v_or_b32_e32 v10, v10, v57
-; SI-NEXT:    v_or_b32_e32 v11, v11, v39
-; SI-NEXT:    v_or_b32_e32 v12, v12, v56
-; SI-NEXT:    v_or_b32_e32 v13, v13, v47
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v1, v1, v54
+; SI-NEXT:    v_or_b32_e32 v2, v2, v59
+; SI-NEXT:    v_or_b32_e32 v3, v3, v58
+; SI-NEXT:    v_or_b32_e32 v4, v4, v53
+; SI-NEXT:    v_or_b32_e32 v5, v5, v57
+; SI-NEXT:    v_or_b32_e32 v6, v6, v52
+; SI-NEXT:    v_or_b32_e32 v7, v7, v56
+; SI-NEXT:    v_or_b32_e32 v8, v8, v51
+; SI-NEXT:    v_or_b32_e32 v9, v9, v50
+; SI-NEXT:    v_or_b32_e32 v10, v10, v49
+; SI-NEXT:    v_or_b32_e32 v11, v11, v48
+; SI-NEXT:    v_or_b32_e32 v12, v12, v47
+; SI-NEXT:    v_or_b32_e32 v13, v13, v39
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v46
-; SI-NEXT:    v_or_b32_e32 v15, v15, v38
-; SI-NEXT:    v_or_b32_e32 v16, v16, v45
+; SI-NEXT:    v_or_b32_e32 v15, v15, v45
+; SI-NEXT:    v_or_b32_e32 v16, v16, v38
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v44
-; SI-NEXT:    v_or_b32_e32 v19, v19, v42
-; SI-NEXT:    v_or_b32_e32 v20, v20, v41
-; SI-NEXT:    v_or_b32_e32 v21, v21, v40
-; SI-NEXT:    v_or_b32_e32 v22, v22, v37
-; SI-NEXT:    v_or_b32_e32 v23, v23, v36
-; SI-NEXT:    v_or_b32_e32 v24, v24, v35
-; SI-NEXT:    v_or_b32_e32 v25, v25, v34
-; SI-NEXT:    v_or_b32_e32 v26, v26, v33
+; SI-NEXT:    v_or_b32_e32 v19, v19, v36
+; SI-NEXT:    v_or_b32_e32 v20, v20, v43
+; SI-NEXT:    v_or_b32_e32 v21, v21, v35
+; SI-NEXT:    v_or_b32_e32 v22, v22, v42
+; SI-NEXT:    v_or_b32_e32 v23, v23, v34
+; SI-NEXT:    v_or_b32_e32 v24, v24, v41
+; SI-NEXT:    v_or_b32_e32 v25, v25, v33
+; SI-NEXT:    v_or_b32_e32 v26, v26, v40
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v32
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v63
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v62
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -5477,79 +5477,75 @@ define <30 x i32> @bitcast_v60i16_to_v30i32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB14_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v0, v54, v0
+; SI-NEXT:    v_or_b32_e32 v0, v60, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v18, v43, v18
+; SI-NEXT:    v_or_b32_e32 v18, v37, v18
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
@@ -5583,31 +5579,31 @@ define <30 x i32> @bitcast_v60i16_to_v30i32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; SI-NEXT:    v_or_b32_e32 v1, v60, v1
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
-; SI-NEXT:    v_or_b32_e32 v3, v52, v3
-; SI-NEXT:    v_or_b32_e32 v4, v51, v4
-; SI-NEXT:    v_or_b32_e32 v5, v59, v5
-; SI-NEXT:    v_or_b32_e32 v6, v50, v6
-; SI-NEXT:    v_or_b32_e32 v7, v49, v7
-; SI-NEXT:    v_or_b32_e32 v8, v48, v8
-; SI-NEXT:    v_or_b32_e32 v9, v58, v9
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_or_b32_e32 v11, v39, v11
-; SI-NEXT:    v_or_b32_e32 v12, v56, v12
-; SI-NEXT:    v_or_b32_e32 v13, v47, v13
+; SI-NEXT:    v_or_b32_e32 v1, v54, v1
+; SI-NEXT:    v_or_b32_e32 v2, v59, v2
+; SI-NEXT:    v_or_b32_e32 v3, v58, v3
+; SI-NEXT:    v_or_b32_e32 v4, v53, v4
+; SI-NEXT:    v_or_b32_e32 v5, v57, v5
+; SI-NEXT:    v_or_b32_e32 v6, v52, v6
+; SI-NEXT:    v_or_b32_e32 v7, v56, v7
+; SI-NEXT:    v_or_b32_e32 v8, v51, v8
+; SI-NEXT:    v_or_b32_e32 v9, v50, v9
+; SI-NEXT:    v_or_b32_e32 v10, v49, v10
+; SI-NEXT:    v_or_b32_e32 v11, v48, v11
+; SI-NEXT:    v_or_b32_e32 v12, v47, v12
+; SI-NEXT:    v_or_b32_e32 v13, v39, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v46, v14
-; SI-NEXT:    v_or_b32_e32 v15, v38, v15
-; SI-NEXT:    v_or_b32_e32 v16, v45, v16
+; SI-NEXT:    v_or_b32_e32 v15, v45, v15
+; SI-NEXT:    v_or_b32_e32 v16, v38, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v44, v17
-; SI-NEXT:    v_or_b32_e32 v19, v42, v19
-; SI-NEXT:    v_or_b32_e32 v20, v41, v20
-; SI-NEXT:    v_or_b32_e32 v21, v40, v21
-; SI-NEXT:    v_or_b32_e32 v22, v37, v22
-; SI-NEXT:    v_or_b32_e32 v23, v36, v23
-; SI-NEXT:    v_or_b32_e32 v24, v35, v24
-; SI-NEXT:    v_or_b32_e32 v25, v34, v25
-; SI-NEXT:    v_or_b32_e32 v26, v33, v26
+; SI-NEXT:    v_or_b32_e32 v19, v36, v19
+; SI-NEXT:    v_or_b32_e32 v20, v43, v20
+; SI-NEXT:    v_or_b32_e32 v21, v35, v21
+; SI-NEXT:    v_or_b32_e32 v22, v42, v22
+; SI-NEXT:    v_or_b32_e32 v23, v34, v23
+; SI-NEXT:    v_or_b32_e32 v24, v41, v24
+; SI-NEXT:    v_or_b32_e32 v25, v33, v25
+; SI-NEXT:    v_or_b32_e32 v26, v40, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v32, v27
 ; SI-NEXT:    v_or_b32_e32 v28, v63, v28
 ; SI-NEXT:    v_or_b32_e32 v29, v62, v29
@@ -10599,17 +10595,17 @@ define <30 x i32> @bitcast_v60f16_to_v30i32(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:48
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:60
 ; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:72
 ; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:68
@@ -10714,35 +10710,35 @@ define <30 x i32> @bitcast_v60f16_to_v30i32(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v61
-; SI-NEXT:    v_cvt_f16_f32_e32 v61, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
+; SI-NEXT:    v_cvt_f16_f32_e32 v61, v49
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v62
-; SI-NEXT:    v_cvt_f16_f32_e32 v62, v50
-; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
+; SI-NEXT:    v_cvt_f16_f32_e32 v62, v51
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v63
-; SI-NEXT:    v_cvt_f16_f32_e32 v63, v49
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
+; SI-NEXT:    v_cvt_f16_f32_e32 v63, v50
+; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v32
 ; SI-NEXT:    v_cvt_f16_f32_e32 v32, v47
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
-; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v34
 ; SI-NEXT:    v_cvt_f16_f32_e32 v34, v45
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v35
 ; SI-NEXT:    v_cvt_f16_f32_e32 v35, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v36
@@ -12312,21 +12308,26 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    v_mov_b32_e32 v33, v32
 ; SI-NEXT:    v_or_b32_e32 v10, v32, v10
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_mov_b32_e32 v44, v43
 ; SI-NEXT:    v_or_b32_e32 v13, v43, v13
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v8, v5
 ; SI-NEXT:    v_mov_b32_e32 v57, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v39
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v49
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -12366,11 +12367,6 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    v_or_b32_e32 v22, v30, v22
 ; SI-NEXT:    v_or_b32_e32 v23, v31, v23
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
@@ -12503,9 +12499,9 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v31
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
@@ -12514,8 +12510,8 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(13)
@@ -12530,38 +12526,38 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
-; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
 ; SI-NEXT:    v_cvt_f16_f32_e32 v31, v31
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -17703,167 +17699,167 @@ define <30 x float> @bitcast_v60i16_to_v30f32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v29
 ; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v14
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v18
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v16
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:52
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v28
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v30
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:88
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v30
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
@@ -17920,111 +17916,111 @@ define <30 x float> @bitcast_v60i16_to_v30f32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    v_or_b32_e32 v0, v0, v54
-; SI-NEXT:    v_or_b32_e32 v18, v18, v43
+; SI-NEXT:    v_or_b32_e32 v0, v0, v60
+; SI-NEXT:    v_or_b32_e32 v18, v18, v37
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; SI-NEXT:    v_or_b32_e32 v1, v1, v60
-; SI-NEXT:    v_or_b32_e32 v2, v2, v53
-; SI-NEXT:    v_or_b32_e32 v3, v3, v52
-; SI-NEXT:    v_or_b32_e32 v4, v4, v51
-; SI-NEXT:    v_or_b32_e32 v5, v5, v59
-; SI-NEXT:    v_or_b32_e32 v6, v6, v50
-; SI-NEXT:    v_or_b32_e32 v7, v7, v49
-; SI-NEXT:    v_or_b32_e32 v8, v8, v48
-; SI-NEXT:    v_or_b32_e32 v9, v9, v58
-; SI-NEXT:    v_or_b32_e32 v10, v10, v57
-; SI-NEXT:    v_or_b32_e32 v11, v11, v39
-; SI-NEXT:    v_or_b32_e32 v12, v12, v56
-; SI-NEXT:    v_or_b32_e32 v13, v13, v47
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v1, v1, v54
+; SI-NEXT:    v_or_b32_e32 v2, v2, v59
+; SI-NEXT:    v_or_b32_e32 v3, v3, v58
+; SI-NEXT:    v_or_b32_e32 v4, v4, v53
+; SI-NEXT:    v_or_b32_e32 v5, v5, v57
+; SI-NEXT:    v_or_b32_e32 v6, v6, v52
+; SI-NEXT:    v_or_b32_e32 v7, v7, v56
+; SI-NEXT:    v_or_b32_e32 v8, v8, v51
+; SI-NEXT:    v_or_b32_e32 v9, v9, v50
+; SI-NEXT:    v_or_b32_e32 v10, v10, v49
+; SI-NEXT:    v_or_b32_e32 v11, v11, v48
+; SI-NEXT:    v_or_b32_e32 v12, v12, v47
+; SI-NEXT:    v_or_b32_e32 v13, v13, v39
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v46
-; SI-NEXT:    v_or_b32_e32 v15, v15, v38
-; SI-NEXT:    v_or_b32_e32 v16, v16, v45
+; SI-NEXT:    v_or_b32_e32 v15, v15, v45
+; SI-NEXT:    v_or_b32_e32 v16, v16, v38
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v44
-; SI-NEXT:    v_or_b32_e32 v19, v19, v42
-; SI-NEXT:    v_or_b32_e32 v20, v20, v41
-; SI-NEXT:    v_or_b32_e32 v21, v21, v40
-; SI-NEXT:    v_or_b32_e32 v22, v22, v37
-; SI-NEXT:    v_or_b32_e32 v23, v23, v36
-; SI-NEXT:    v_or_b32_e32 v24, v24, v35
-; SI-NEXT:    v_or_b32_e32 v25, v25, v34
-; SI-NEXT:    v_or_b32_e32 v26, v26, v33
+; SI-NEXT:    v_or_b32_e32 v19, v19, v36
+; SI-NEXT:    v_or_b32_e32 v20, v20, v43
+; SI-NEXT:    v_or_b32_e32 v21, v21, v35
+; SI-NEXT:    v_or_b32_e32 v22, v22, v42
+; SI-NEXT:    v_or_b32_e32 v23, v23, v34
+; SI-NEXT:    v_or_b32_e32 v24, v24, v41
+; SI-NEXT:    v_or_b32_e32 v25, v25, v33
+; SI-NEXT:    v_or_b32_e32 v26, v26, v40
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v32
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v63
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v62
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -18032,79 +18028,75 @@ define <30 x float> @bitcast_v60i16_to_v30f32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB30_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v0, v54, v0
+; SI-NEXT:    v_or_b32_e32 v0, v60, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v18, v43, v18
+; SI-NEXT:    v_or_b32_e32 v18, v37, v18
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
@@ -18138,31 +18130,31 @@ define <30 x float> @bitcast_v60i16_to_v30f32(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; SI-NEXT:    v_or_b32_e32 v1, v60, v1
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
-; SI-NEXT:    v_or_b32_e32 v3, v52, v3
-; SI-NEXT:    v_or_b32_e32 v4, v51, v4
-; SI-NEXT:    v_or_b32_e32 v5, v59, v5
-; SI-NEXT:    v_or_b32_e32 v6, v50, v6
-; SI-NEXT:    v_or_b32_e32 v7, v49, v7
-; SI-NEXT:    v_or_b32_e32 v8, v48, v8
-; SI-NEXT:    v_or_b32_e32 v9, v58, v9
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_or_b32_e32 v11, v39, v11
-; SI-NEXT:    v_or_b32_e32 v12, v56, v12
-; SI-NEXT:    v_or_b32_e32 v13, v47, v13
+; SI-NEXT:    v_or_b32_e32 v1, v54, v1
+; SI-NEXT:    v_or_b32_e32 v2, v59, v2
+; SI-NEXT:    v_or_b32_e32 v3, v58, v3
+; SI-NEXT:    v_or_b32_e32 v4, v53, v4
+; SI-NEXT:    v_or_b32_e32 v5, v57, v5
+; SI-NEXT:    v_or_b32_e32 v6, v52, v6
+; SI-NEXT:    v_or_b32_e32 v7, v56, v7
+; SI-NEXT:    v_or_b32_e32 v8, v51, v8
+; SI-NEXT:    v_or_b32_e32 v9, v50, v9
+; SI-NEXT:    v_or_b32_e32 v10, v49, v10
+; SI-NEXT:    v_or_b32_e32 v11, v48, v11
+; SI-NEXT:    v_or_b32_e32 v12, v47, v12
+; SI-NEXT:    v_or_b32_e32 v13, v39, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v46, v14
-; SI-NEXT:    v_or_b32_e32 v15, v38, v15
-; SI-NEXT:    v_or_b32_e32 v16, v45, v16
+; SI-NEXT:    v_or_b32_e32 v15, v45, v15
+; SI-NEXT:    v_or_b32_e32 v16, v38, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v44, v17
-; SI-NEXT:    v_or_b32_e32 v19, v42, v19
-; SI-NEXT:    v_or_b32_e32 v20, v41, v20
-; SI-NEXT:    v_or_b32_e32 v21, v40, v21
-; SI-NEXT:    v_or_b32_e32 v22, v37, v22
-; SI-NEXT:    v_or_b32_e32 v23, v36, v23
-; SI-NEXT:    v_or_b32_e32 v24, v35, v24
-; SI-NEXT:    v_or_b32_e32 v25, v34, v25
-; SI-NEXT:    v_or_b32_e32 v26, v33, v26
+; SI-NEXT:    v_or_b32_e32 v19, v36, v19
+; SI-NEXT:    v_or_b32_e32 v20, v43, v20
+; SI-NEXT:    v_or_b32_e32 v21, v35, v21
+; SI-NEXT:    v_or_b32_e32 v22, v42, v22
+; SI-NEXT:    v_or_b32_e32 v23, v34, v23
+; SI-NEXT:    v_or_b32_e32 v24, v41, v24
+; SI-NEXT:    v_or_b32_e32 v25, v33, v25
+; SI-NEXT:    v_or_b32_e32 v26, v40, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v32, v27
 ; SI-NEXT:    v_or_b32_e32 v28, v63, v28
 ; SI-NEXT:    v_or_b32_e32 v29, v62, v29
@@ -23136,17 +23128,17 @@ define <30 x float> @bitcast_v60f16_to_v30f32(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:48
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:60
 ; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:72
 ; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:68
@@ -23251,35 +23243,35 @@ define <30 x float> @bitcast_v60f16_to_v30f32(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v61
-; SI-NEXT:    v_cvt_f16_f32_e32 v61, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
+; SI-NEXT:    v_cvt_f16_f32_e32 v61, v49
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v62
-; SI-NEXT:    v_cvt_f16_f32_e32 v62, v50
-; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
+; SI-NEXT:    v_cvt_f16_f32_e32 v62, v51
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v63
-; SI-NEXT:    v_cvt_f16_f32_e32 v63, v49
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
+; SI-NEXT:    v_cvt_f16_f32_e32 v63, v50
+; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v32
 ; SI-NEXT:    v_cvt_f16_f32_e32 v32, v47
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
-; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v34
 ; SI-NEXT:    v_cvt_f16_f32_e32 v34, v45
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v35
 ; SI-NEXT:    v_cvt_f16_f32_e32 v35, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v36
@@ -24849,21 +24841,26 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
 ; SI-NEXT:    v_mov_b32_e32 v33, v32
 ; SI-NEXT:    v_or_b32_e32 v10, v32, v10
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_mov_b32_e32 v44, v43
 ; SI-NEXT:    v_or_b32_e32 v13, v43, v13
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v8, v5
 ; SI-NEXT:    v_mov_b32_e32 v57, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v39
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v49
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -24903,11 +24900,6 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
 ; SI-NEXT:    v_or_b32_e32 v22, v30, v22
 ; SI-NEXT:    v_or_b32_e32 v23, v31, v23
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
@@ -25040,9 +25032,9 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v31
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
@@ -25051,8 +25043,8 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(13)
@@ -25067,38 +25059,38 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
-; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
 ; SI-NEXT:    v_cvt_f16_f32_e32 v31, v31
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -29360,167 +29352,167 @@ define <15 x i64> @bitcast_v60i16_to_v15i64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v29
 ; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v6
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v8
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:36
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v14
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v16
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:52
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v28
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v30
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:88
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v30
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
@@ -29577,111 +29569,111 @@ define <15 x i64> @bitcast_v60i16_to_v15i64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    v_or_b32_e32 v0, v0, v54
-; SI-NEXT:    v_or_b32_e32 v18, v18, v43
+; SI-NEXT:    v_or_b32_e32 v0, v0, v60
+; SI-NEXT:    v_or_b32_e32 v18, v18, v37
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; SI-NEXT:    v_or_b32_e32 v1, v1, v60
-; SI-NEXT:    v_or_b32_e32 v2, v2, v53
-; SI-NEXT:    v_or_b32_e32 v3, v3, v52
-; SI-NEXT:    v_or_b32_e32 v4, v4, v51
-; SI-NEXT:    v_or_b32_e32 v5, v5, v59
-; SI-NEXT:    v_or_b32_e32 v6, v6, v50
-; SI-NEXT:    v_or_b32_e32 v7, v7, v49
-; SI-NEXT:    v_or_b32_e32 v8, v8, v48
-; SI-NEXT:    v_or_b32_e32 v9, v9, v58
-; SI-NEXT:    v_or_b32_e32 v10, v10, v57
-; SI-NEXT:    v_or_b32_e32 v11, v11, v39
-; SI-NEXT:    v_or_b32_e32 v12, v12, v56
-; SI-NEXT:    v_or_b32_e32 v13, v13, v47
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v1, v1, v54
+; SI-NEXT:    v_or_b32_e32 v2, v2, v59
+; SI-NEXT:    v_or_b32_e32 v3, v3, v58
+; SI-NEXT:    v_or_b32_e32 v4, v4, v53
+; SI-NEXT:    v_or_b32_e32 v5, v5, v57
+; SI-NEXT:    v_or_b32_e32 v6, v6, v52
+; SI-NEXT:    v_or_b32_e32 v7, v7, v56
+; SI-NEXT:    v_or_b32_e32 v8, v8, v51
+; SI-NEXT:    v_or_b32_e32 v9, v9, v50
+; SI-NEXT:    v_or_b32_e32 v10, v10, v49
+; SI-NEXT:    v_or_b32_e32 v11, v11, v48
+; SI-NEXT:    v_or_b32_e32 v12, v12, v47
+; SI-NEXT:    v_or_b32_e32 v13, v13, v39
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v46
-; SI-NEXT:    v_or_b32_e32 v15, v15, v38
-; SI-NEXT:    v_or_b32_e32 v16, v16, v45
+; SI-NEXT:    v_or_b32_e32 v15, v15, v45
+; SI-NEXT:    v_or_b32_e32 v16, v16, v38
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v44
-; SI-NEXT:    v_or_b32_e32 v19, v19, v42
-; SI-NEXT:    v_or_b32_e32 v20, v20, v41
-; SI-NEXT:    v_or_b32_e32 v21, v21, v40
-; SI-NEXT:    v_or_b32_e32 v22, v22, v37
-; SI-NEXT:    v_or_b32_e32 v23, v23, v36
-; SI-NEXT:    v_or_b32_e32 v24, v24, v35
-; SI-NEXT:    v_or_b32_e32 v25, v25, v34
-; SI-NEXT:    v_or_b32_e32 v26, v26, v33
+; SI-NEXT:    v_or_b32_e32 v19, v19, v36
+; SI-NEXT:    v_or_b32_e32 v20, v20, v43
+; SI-NEXT:    v_or_b32_e32 v21, v21, v35
+; SI-NEXT:    v_or_b32_e32 v22, v22, v42
+; SI-NEXT:    v_or_b32_e32 v23, v23, v34
+; SI-NEXT:    v_or_b32_e32 v24, v24, v41
+; SI-NEXT:    v_or_b32_e32 v25, v25, v33
+; SI-NEXT:    v_or_b32_e32 v26, v26, v40
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v32
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v63
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v62
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -29689,79 +29681,75 @@ define <15 x i64> @bitcast_v60i16_to_v15i64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB42_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v0, v54, v0
+; SI-NEXT:    v_or_b32_e32 v0, v60, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v18, v43, v18
+; SI-NEXT:    v_or_b32_e32 v18, v37, v18
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
@@ -29795,31 +29783,31 @@ define <15 x i64> @bitcast_v60i16_to_v15i64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; SI-NEXT:    v_or_b32_e32 v1, v60, v1
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
-; SI-NEXT:    v_or_b32_e32 v3, v52, v3
-; SI-NEXT:    v_or_b32_e32 v4, v51, v4
-; SI-NEXT:    v_or_b32_e32 v5, v59, v5
-; SI-NEXT:    v_or_b32_e32 v6, v50, v6
-; SI-NEXT:    v_or_b32_e32 v7, v49, v7
-; SI-NEXT:    v_or_b32_e32 v8, v48, v8
-; SI-NEXT:    v_or_b32_e32 v9, v58, v9
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_or_b32_e32 v11, v39, v11
-; SI-NEXT:    v_or_b32_e32 v12, v56, v12
-; SI-NEXT:    v_or_b32_e32 v13, v47, v13
+; SI-NEXT:    v_or_b32_e32 v1, v54, v1
+; SI-NEXT:    v_or_b32_e32 v2, v59, v2
+; SI-NEXT:    v_or_b32_e32 v3, v58, v3
+; SI-NEXT:    v_or_b32_e32 v4, v53, v4
+; SI-NEXT:    v_or_b32_e32 v5, v57, v5
+; SI-NEXT:    v_or_b32_e32 v6, v52, v6
+; SI-NEXT:    v_or_b32_e32 v7, v56, v7
+; SI-NEXT:    v_or_b32_e32 v8, v51, v8
+; SI-NEXT:    v_or_b32_e32 v9, v50, v9
+; SI-NEXT:    v_or_b32_e32 v10, v49, v10
+; SI-NEXT:    v_or_b32_e32 v11, v48, v11
+; SI-NEXT:    v_or_b32_e32 v12, v47, v12
+; SI-NEXT:    v_or_b32_e32 v13, v39, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v46, v14
-; SI-NEXT:    v_or_b32_e32 v15, v38, v15
-; SI-NEXT:    v_or_b32_e32 v16, v45, v16
+; SI-NEXT:    v_or_b32_e32 v15, v45, v15
+; SI-NEXT:    v_or_b32_e32 v16, v38, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v44, v17
-; SI-NEXT:    v_or_b32_e32 v19, v42, v19
-; SI-NEXT:    v_or_b32_e32 v20, v41, v20
-; SI-NEXT:    v_or_b32_e32 v21, v40, v21
-; SI-NEXT:    v_or_b32_e32 v22, v37, v22
-; SI-NEXT:    v_or_b32_e32 v23, v36, v23
-; SI-NEXT:    v_or_b32_e32 v24, v35, v24
-; SI-NEXT:    v_or_b32_e32 v25, v34, v25
-; SI-NEXT:    v_or_b32_e32 v26, v33, v26
+; SI-NEXT:    v_or_b32_e32 v19, v36, v19
+; SI-NEXT:    v_or_b32_e32 v20, v43, v20
+; SI-NEXT:    v_or_b32_e32 v21, v35, v21
+; SI-NEXT:    v_or_b32_e32 v22, v42, v22
+; SI-NEXT:    v_or_b32_e32 v23, v34, v23
+; SI-NEXT:    v_or_b32_e32 v24, v41, v24
+; SI-NEXT:    v_or_b32_e32 v25, v33, v25
+; SI-NEXT:    v_or_b32_e32 v26, v40, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v32, v27
 ; SI-NEXT:    v_or_b32_e32 v28, v63, v28
 ; SI-NEXT:    v_or_b32_e32 v29, v62, v29
@@ -34828,17 +34816,17 @@ define <15 x i64> @bitcast_v60f16_to_v15i64(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:48
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:60
 ; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:72
 ; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:68
@@ -34943,35 +34931,35 @@ define <15 x i64> @bitcast_v60f16_to_v15i64(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v61
-; SI-NEXT:    v_cvt_f16_f32_e32 v61, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
+; SI-NEXT:    v_cvt_f16_f32_e32 v61, v49
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v62
-; SI-NEXT:    v_cvt_f16_f32_e32 v62, v50
-; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
+; SI-NEXT:    v_cvt_f16_f32_e32 v62, v51
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v63
-; SI-NEXT:    v_cvt_f16_f32_e32 v63, v49
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
+; SI-NEXT:    v_cvt_f16_f32_e32 v63, v50
+; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v32
 ; SI-NEXT:    v_cvt_f16_f32_e32 v32, v47
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
-; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v34
 ; SI-NEXT:    v_cvt_f16_f32_e32 v34, v45
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v35
 ; SI-NEXT:    v_cvt_f16_f32_e32 v35, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v36
@@ -36541,21 +36529,26 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    v_mov_b32_e32 v33, v32
 ; SI-NEXT:    v_or_b32_e32 v10, v32, v10
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_mov_b32_e32 v44, v43
 ; SI-NEXT:    v_or_b32_e32 v13, v43, v13
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v8, v5
 ; SI-NEXT:    v_mov_b32_e32 v57, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v39
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v49
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -36595,11 +36588,6 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    v_or_b32_e32 v22, v30, v22
 ; SI-NEXT:    v_or_b32_e32 v23, v31, v23
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
@@ -36732,9 +36720,9 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v31
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
@@ -36743,8 +36731,8 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(13)
@@ -36759,38 +36747,38 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
-; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
 ; SI-NEXT:    v_cvt_f16_f32_e32 v31, v31
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -40020,167 +40008,167 @@ define <15 x double> @bitcast_v60i16_to_v15f64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v24, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
 ; SI-NEXT:    v_mov_b32_e32 v55, v0
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
-; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v5
-; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v9
-; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v15
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v17
-; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v21
-; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v23
-; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v27
+; SI-NEXT:    v_lshlrev_b32_e32 v60, 16, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v54, 16, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v59, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v58, 16, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v57, 16, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
+; SI-NEXT:    v_lshlrev_b32_e32 v56, 16, v15
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v17
+; SI-NEXT:    v_lshlrev_b32_e32 v50, 16, v19
+; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v21
+; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v23
+; SI-NEXT:    v_lshlrev_b32_e32 v47, 16, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v39, 16, v27
 ; SI-NEXT:    v_lshlrev_b32_e32 v46, 16, v29
 ; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:20
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:84
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:60
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:8
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:4
 ; SI-NEXT:    v_lshlrev_b32_e32 v44, 16, v2
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v4
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v6
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v8
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:36
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:100
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v10
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:80
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:76
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v14
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:92
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v16
+; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v12
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v18, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:72
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:88
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:84
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v20, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:60
-; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:80
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:76
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v16
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshlrev_b32_e32 v36, 16, v20
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:108
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v22
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v62, 16, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v26, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:104
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:68
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v63, 16, v26
+; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v26
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v28, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:52
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v32, 16, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v28
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    buffer_store_dword v30, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v30
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:88
-; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:44
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:36
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshlrev_b32_e32 v33, 16, v30
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.false
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
@@ -40237,111 +40225,111 @@ define <15 x double> @bitcast_v60i16_to_v15f64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
-; SI-NEXT:    v_or_b32_e32 v0, v0, v54
-; SI-NEXT:    v_or_b32_e32 v18, v18, v43
+; SI-NEXT:    v_or_b32_e32 v0, v0, v60
+; SI-NEXT:    v_or_b32_e32 v18, v18, v37
 ; SI-NEXT:    ; implicit-def: $vgpr55
 ; SI-NEXT:    ; implicit-def: $vgpr61
 ; SI-NEXT:    ; kill: killed $vgpr30
 ; SI-NEXT:    ; implicit-def: $vgpr30
 ; SI-NEXT:    ; kill: killed $vgpr30
-; SI-NEXT:    ; implicit-def: $vgpr54
-; SI-NEXT:    ; implicit-def: $vgpr43
+; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
 ; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; SI-NEXT:    v_and_b32_e32 v6, 0xffff, v6
 ; SI-NEXT:    v_and_b32_e32 v16, 0xffff, v16
+; SI-NEXT:    v_and_b32_e32 v22, 0xffff, v22
+; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
+; SI-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
 ; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
 ; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
+; SI-NEXT:    v_and_b32_e32 v24, 0xffff, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT:    v_and_b32_e32 v10, 0xffff, v10
 ; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; SI-NEXT:    v_and_b32_e32 v26, 0xffff, v26
 ; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
 ; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_and_b32_e32 v12, 0xffff, v12
 ; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v17, 0xffff, v17
+; SI-NEXT:    v_and_b32_e32 v13, 0xffff, v13
 ; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT:    v_and_b32_e32 v14, 0xffff, v14
 ; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v20, 0xffff, v20
+; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; SI-NEXT:    s_waitcnt vmcnt(1)
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; SI-NEXT:    v_or_b32_e32 v1, v1, v60
-; SI-NEXT:    v_or_b32_e32 v2, v2, v53
-; SI-NEXT:    v_or_b32_e32 v3, v3, v52
-; SI-NEXT:    v_or_b32_e32 v4, v4, v51
-; SI-NEXT:    v_or_b32_e32 v5, v5, v59
-; SI-NEXT:    v_or_b32_e32 v6, v6, v50
-; SI-NEXT:    v_or_b32_e32 v7, v7, v49
-; SI-NEXT:    v_or_b32_e32 v8, v8, v48
-; SI-NEXT:    v_or_b32_e32 v9, v9, v58
-; SI-NEXT:    v_or_b32_e32 v10, v10, v57
-; SI-NEXT:    v_or_b32_e32 v11, v11, v39
-; SI-NEXT:    v_or_b32_e32 v12, v12, v56
-; SI-NEXT:    v_or_b32_e32 v13, v13, v47
+; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT:    v_or_b32_e32 v1, v1, v54
+; SI-NEXT:    v_or_b32_e32 v2, v2, v59
+; SI-NEXT:    v_or_b32_e32 v3, v3, v58
+; SI-NEXT:    v_or_b32_e32 v4, v4, v53
+; SI-NEXT:    v_or_b32_e32 v5, v5, v57
+; SI-NEXT:    v_or_b32_e32 v6, v6, v52
+; SI-NEXT:    v_or_b32_e32 v7, v7, v56
+; SI-NEXT:    v_or_b32_e32 v8, v8, v51
+; SI-NEXT:    v_or_b32_e32 v9, v9, v50
+; SI-NEXT:    v_or_b32_e32 v10, v10, v49
+; SI-NEXT:    v_or_b32_e32 v11, v11, v48
+; SI-NEXT:    v_or_b32_e32 v12, v12, v47
+; SI-NEXT:    v_or_b32_e32 v13, v13, v39
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v46
-; SI-NEXT:    v_or_b32_e32 v15, v15, v38
-; SI-NEXT:    v_or_b32_e32 v16, v16, v45
+; SI-NEXT:    v_or_b32_e32 v15, v15, v45
+; SI-NEXT:    v_or_b32_e32 v16, v16, v38
 ; SI-NEXT:    v_or_b32_e32 v17, v17, v44
-; SI-NEXT:    v_or_b32_e32 v19, v19, v42
-; SI-NEXT:    v_or_b32_e32 v20, v20, v41
-; SI-NEXT:    v_or_b32_e32 v21, v21, v40
-; SI-NEXT:    v_or_b32_e32 v22, v22, v37
-; SI-NEXT:    v_or_b32_e32 v23, v23, v36
-; SI-NEXT:    v_or_b32_e32 v24, v24, v35
-; SI-NEXT:    v_or_b32_e32 v25, v25, v34
-; SI-NEXT:    v_or_b32_e32 v26, v26, v33
+; SI-NEXT:    v_or_b32_e32 v19, v19, v36
+; SI-NEXT:    v_or_b32_e32 v20, v20, v43
+; SI-NEXT:    v_or_b32_e32 v21, v21, v35
+; SI-NEXT:    v_or_b32_e32 v22, v22, v42
+; SI-NEXT:    v_or_b32_e32 v23, v23, v34
+; SI-NEXT:    v_or_b32_e32 v24, v24, v41
+; SI-NEXT:    v_or_b32_e32 v25, v25, v33
+; SI-NEXT:    v_or_b32_e32 v26, v26, v40
 ; SI-NEXT:    v_or_b32_e32 v27, v27, v32
 ; SI-NEXT:    v_or_b32_e32 v28, v28, v63
 ; SI-NEXT:    v_or_b32_e32 v29, v29, v62
-; SI-NEXT:    ; implicit-def: $vgpr60
+; SI-NEXT:    ; implicit-def: $vgpr54
+; SI-NEXT:    ; implicit-def: $vgpr59
+; SI-NEXT:    ; implicit-def: $vgpr58
 ; SI-NEXT:    ; implicit-def: $vgpr53
+; SI-NEXT:    ; implicit-def: $vgpr57
 ; SI-NEXT:    ; implicit-def: $vgpr52
+; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr51
-; SI-NEXT:    ; implicit-def: $vgpr59
 ; SI-NEXT:    ; implicit-def: $vgpr50
 ; SI-NEXT:    ; implicit-def: $vgpr49
 ; SI-NEXT:    ; implicit-def: $vgpr48
-; SI-NEXT:    ; implicit-def: $vgpr58
-; SI-NEXT:    ; implicit-def: $vgpr57
-; SI-NEXT:    ; implicit-def: $vgpr39
-; SI-NEXT:    ; implicit-def: $vgpr56
 ; SI-NEXT:    ; implicit-def: $vgpr47
+; SI-NEXT:    ; implicit-def: $vgpr39
 ; SI-NEXT:    ; implicit-def: $vgpr46
-; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr45
+; SI-NEXT:    ; implicit-def: $vgpr38
 ; SI-NEXT:    ; implicit-def: $vgpr44
-; SI-NEXT:    ; implicit-def: $vgpr42
-; SI-NEXT:    ; implicit-def: $vgpr41
-; SI-NEXT:    ; implicit-def: $vgpr40
-; SI-NEXT:    ; implicit-def: $vgpr37
 ; SI-NEXT:    ; implicit-def: $vgpr36
+; SI-NEXT:    ; implicit-def: $vgpr43
 ; SI-NEXT:    ; implicit-def: $vgpr35
+; SI-NEXT:    ; implicit-def: $vgpr42
 ; SI-NEXT:    ; implicit-def: $vgpr34
+; SI-NEXT:    ; implicit-def: $vgpr41
 ; SI-NEXT:    ; implicit-def: $vgpr33
+; SI-NEXT:    ; implicit-def: $vgpr40
 ; SI-NEXT:    ; implicit-def: $vgpr32
 ; SI-NEXT:    ; implicit-def: $vgpr63
 ; SI-NEXT:    ; implicit-def: $vgpr62
@@ -40349,79 +40337,75 @@ define <15 x double> @bitcast_v60i16_to_v15f64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB50_4
 ; SI-NEXT:  ; %bb.3: ; %cmp.true
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v24, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 3, v55
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, 3, v61
 ; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; SI-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; SI-NEXT:    v_or_b32_e32 v0, v54, v0
+; SI-NEXT:    v_or_b32_e32 v0, v60, v0
 ; SI-NEXT:    s_mov_b32 s6, 0x30000
-; SI-NEXT:    v_or_b32_e32 v18, v43, v18
+; SI-NEXT:    v_or_b32_e32 v18, v37, v18
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x30000, v0
 ; SI-NEXT:    v_add_i32_e32 v18, vcc, s6, v18
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
 ; SI-NEXT:    v_add_i32_e32 v3, vcc, 3, v3
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v5, vcc, 3, v5
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, 3, v6
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 3, v4
 ; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v7
 ; SI-NEXT:    v_add_i32_e32 v8, vcc, 3, v8
+; SI-NEXT:    s_waitcnt vmcnt(13)
 ; SI-NEXT:    v_add_i32_e32 v9, vcc, 3, v9
-; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_add_i32_e32 v10, vcc, 3, v10
-; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    s_waitcnt vmcnt(7)
 ; SI-NEXT:    v_add_i32_e32 v11, vcc, 3, v11
-; SI-NEXT:    s_waitcnt vmcnt(9)
+; SI-NEXT:    s_waitcnt vmcnt(6)
 ; SI-NEXT:    v_add_i32_e32 v12, vcc, 3, v12
-; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v13, vcc, 3, v13
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(4)
 ; SI-NEXT:    v_add_i32_e32 v14, vcc, 3, v14
-; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v15, vcc, 3, v15
 ; SI-NEXT:    v_add_i32_e32 v16, vcc, 3, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_add_i32_e32 v17, vcc, 3, v17
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_add_i32_e32 v19, vcc, 3, v19
-; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_add_i32_e32 v20, vcc, 3, v20
-; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_add_i32_e32 v21, vcc, 3, v21
 ; SI-NEXT:    v_add_i32_e32 v22, vcc, 3, v22
 ; SI-NEXT:    v_add_i32_e32 v23, vcc, 3, v23
 ; SI-NEXT:    v_add_i32_e32 v24, vcc, 3, v24
-; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_add_i32_e32 v25, vcc, 3, v25
 ; SI-NEXT:    v_add_i32_e32 v26, vcc, 3, v26
 ; SI-NEXT:    v_add_i32_e32 v27, vcc, 3, v27
@@ -40455,31 +40439,31 @@ define <15 x double> @bitcast_v60i16_to_v15f64(<60 x i16> %a, i32 %b) {
 ; SI-NEXT:    v_and_b32_e32 v27, 0xffff, v27
 ; SI-NEXT:    v_and_b32_e32 v28, 0xffff, v28
 ; SI-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; SI-NEXT:    v_or_b32_e32 v1, v60, v1
-; SI-NEXT:    v_or_b32_e32 v2, v53, v2
-; SI-NEXT:    v_or_b32_e32 v3, v52, v3
-; SI-NEXT:    v_or_b32_e32 v4, v51, v4
-; SI-NEXT:    v_or_b32_e32 v5, v59, v5
-; SI-NEXT:    v_or_b32_e32 v6, v50, v6
-; SI-NEXT:    v_or_b32_e32 v7, v49, v7
-; SI-NEXT:    v_or_b32_e32 v8, v48, v8
-; SI-NEXT:    v_or_b32_e32 v9, v58, v9
-; SI-NEXT:    v_or_b32_e32 v10, v57, v10
-; SI-NEXT:    v_or_b32_e32 v11, v39, v11
-; SI-NEXT:    v_or_b32_e32 v12, v56, v12
-; SI-NEXT:    v_or_b32_e32 v13, v47, v13
+; SI-NEXT:    v_or_b32_e32 v1, v54, v1
+; SI-NEXT:    v_or_b32_e32 v2, v59, v2
+; SI-NEXT:    v_or_b32_e32 v3, v58, v3
+; SI-NEXT:    v_or_b32_e32 v4, v53, v4
+; SI-NEXT:    v_or_b32_e32 v5, v57, v5
+; SI-NEXT:    v_or_b32_e32 v6, v52, v6
+; SI-NEXT:    v_or_b32_e32 v7, v56, v7
+; SI-NEXT:    v_or_b32_e32 v8, v51, v8
+; SI-NEXT:    v_or_b32_e32 v9, v50, v9
+; SI-NEXT:    v_or_b32_e32 v10, v49, v10
+; SI-NEXT:    v_or_b32_e32 v11, v48, v11
+; SI-NEXT:    v_or_b32_e32 v12, v47, v12
+; SI-NEXT:    v_or_b32_e32 v13, v39, v13
 ; SI-NEXT:    v_or_b32_e32 v14, v46, v14
-; SI-NEXT:    v_or_b32_e32 v15, v38, v15
-; SI-NEXT:    v_or_b32_e32 v16, v45, v16
+; SI-NEXT:    v_or_b32_e32 v15, v45, v15
+; SI-NEXT:    v_or_b32_e32 v16, v38, v16
 ; SI-NEXT:    v_or_b32_e32 v17, v44, v17
-; SI-NEXT:    v_or_b32_e32 v19, v42, v19
-; SI-NEXT:    v_or_b32_e32 v20, v41, v20
-; SI-NEXT:    v_or_b32_e32 v21, v40, v21
-; SI-NEXT:    v_or_b32_e32 v22, v37, v22
-; SI-NEXT:    v_or_b32_e32 v23, v36, v23
-; SI-NEXT:    v_or_b32_e32 v24, v35, v24
-; SI-NEXT:    v_or_b32_e32 v25, v34, v25
-; SI-NEXT:    v_or_b32_e32 v26, v33, v26
+; SI-NEXT:    v_or_b32_e32 v19, v36, v19
+; SI-NEXT:    v_or_b32_e32 v20, v43, v20
+; SI-NEXT:    v_or_b32_e32 v21, v35, v21
+; SI-NEXT:    v_or_b32_e32 v22, v42, v22
+; SI-NEXT:    v_or_b32_e32 v23, v34, v23
+; SI-NEXT:    v_or_b32_e32 v24, v41, v24
+; SI-NEXT:    v_or_b32_e32 v25, v33, v25
+; SI-NEXT:    v_or_b32_e32 v26, v40, v26
 ; SI-NEXT:    v_or_b32_e32 v27, v32, v27
 ; SI-NEXT:    v_or_b32_e32 v28, v63, v28
 ; SI-NEXT:    v_or_b32_e32 v29, v62, v29
@@ -45357,17 +45341,17 @@ define <15 x double> @bitcast_v60f16_to_v15f64(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:16
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:20
 ; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:40
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:36
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:48
 ; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:52
 ; SI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:60
 ; SI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:72
 ; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:68
@@ -45472,35 +45456,35 @@ define <15 x double> @bitcast_v60f16_to_v15f64(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v61
-; SI-NEXT:    v_cvt_f16_f32_e32 v61, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
+; SI-NEXT:    v_cvt_f16_f32_e32 v61, v49
+; SI-NEXT:    s_waitcnt vmcnt(14)
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v62
-; SI-NEXT:    v_cvt_f16_f32_e32 v62, v50
-; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
+; SI-NEXT:    v_cvt_f16_f32_e32 v62, v51
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v53
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v63
-; SI-NEXT:    v_cvt_f16_f32_e32 v63, v49
-; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v55
+; SI-NEXT:    v_cvt_f16_f32_e32 v63, v50
+; SI-NEXT:    v_cvt_f16_f32_e32 v50, v54
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v32
 ; SI-NEXT:    v_cvt_f16_f32_e32 v32, v47
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
-; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v34
 ; SI-NEXT:    v_cvt_f16_f32_e32 v34, v45
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v35
 ; SI-NEXT:    v_cvt_f16_f32_e32 v35, v44
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v33, v46
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v36
@@ -47070,21 +47054,26 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
 ; SI-NEXT:    v_mov_b32_e32 v33, v32
 ; SI-NEXT:    v_or_b32_e32 v10, v32, v10
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_mov_b32_e32 v44, v43
 ; SI-NEXT:    v_or_b32_e32 v13, v43, v13
 ; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v5, v8, v5
 ; SI-NEXT:    v_mov_b32_e32 v57, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v39
 ; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_or_b32_e32 v6, v7, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v49
 ; SI-NEXT:    v_or_b32_e32 v7, v37, v7
 ; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
@@ -47124,11 +47113,6 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
 ; SI-NEXT:    v_or_b32_e32 v22, v30, v22
 ; SI-NEXT:    v_or_b32_e32 v23, v31, v23
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
-; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(11)
 ; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; SI-NEXT:    s_waitcnt vmcnt(10)
@@ -47261,9 +47245,9 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
 ; SI-NEXT:    v_cvt_f32_f16_e32 v25, v31
 ; SI-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
@@ -47272,8 +47256,8 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
 ; SI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    s_waitcnt vmcnt(13)
@@ -47288,38 +47272,38 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
 ; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt vmcnt(14)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    s_waitcnt vmcnt(12)
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    s_waitcnt vmcnt(11)
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    s_waitcnt vmcnt(9)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
 ; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT:    s_waitcnt vmcnt(10)
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v6
 ; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
 ; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
 ; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT:    s_waitcnt vmcnt(8)
-; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
 ; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; SI-NEXT:    s_waitcnt vmcnt(5)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
+; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f32_f16_e32 v21, v21
 ; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
-; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
 ; SI-NEXT:    v_cvt_f16_f32_e32 v31, v31
 ; SI-NEXT:    s_waitcnt vmcnt(1)
@@ -51934,502 +51918,496 @@ define <60 x i16> @bitcast_v60f16_to_v60i16(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    buffer_store_dword v61, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v62, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
 ; SI-NEXT:    buffer_store_dword v63, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:120
-; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:120
+; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32
+; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4
 ; SI-NEXT:    s_waitcnt expcnt(2)
-; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v61, off, s[0:3], s32 offset:8
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v62, off, s[0:3], s32 offset:12
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:24
-; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:28
-; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
-; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:36
-; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:40
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:48
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:52
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:56
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:60
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:64
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:68
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:72
+; SI-NEXT:    buffer_load_dword v63, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
+; SI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:28
+; SI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:32
+; SI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:36
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:40
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44
+; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:48
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:52
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:56
+; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:60
+; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:64
+; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:68
+; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:72
 ; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:76
 ; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:80
 ; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:84
-; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:92
-; SI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:88
-; SI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:100
-; SI-NEXT:    buffer_load_dword v60, off, s[0:3], s32 offset:96
-; SI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:108
-; SI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:104
-; SI-NEXT:    v_cvt_f16_f32_e32 v53, v8
-; SI-NEXT:    v_cvt_f16_f32_e32 v8, v9
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v22
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:88
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT:    v_cvt_f16_f32_e32 v52, v7
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v18
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v26
+; SI-NEXT:    v_cvt_f16_f32_e32 v55, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v22
+; SI-NEXT:    v_cvt_f16_f32_e32 v60, v2
 ; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v1, v2
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v5
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v30
-; SI-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v12
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT:    v_cvt_f16_f32_e32 v55, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v2, v18
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:    v_cvt_f16_f32_e32 v40, v4
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v6
-; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v11
-; SI-NEXT:    v_cvt_f16_f32_e32 v6, v13
-; SI-NEXT:    v_cvt_f16_f32_e32 v13, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v14, v15
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_cvt_f16_f32_e32 v2, v16
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v19
-; SI-NEXT:    v_cvt_f16_f32_e32 v18, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    v_cvt_f16_f32_e32 v1, v6
+; SI-NEXT:    v_cvt_f16_f32_e32 v53, v8
+; SI-NEXT:    v_cvt_f16_f32_e32 v8, v10
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v12
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v13
+; SI-NEXT:    v_cvt_f16_f32_e32 v37, v15
+; SI-NEXT:    v_cvt_f16_f32_e32 v52, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v7, v9
+; SI-NEXT:    v_cvt_f16_f32_e32 v48, v11
+; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT:    v_cvt_f16_f32_e32 v38, v16
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_cvt_f16_f32_e32 v2, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
+; SI-NEXT:    v_cvt_f16_f32_e32 v11, v23
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v24
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
+; SI-NEXT:    v_cvt_f16_f32_e32 v24, v26
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v27
 ; SI-NEXT:    v_cvt_f16_f32_e32 v27, v28
-; SI-NEXT:    v_cvt_f16_f32_e32 v28, v29
+; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; SI-NEXT:    v_cvt_f16_f32_e32 v28, v30
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    s_waitcnt vmcnt(14)
-; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v31
-; SI-NEXT:    v_cvt_f16_f32_e32 v31, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v22, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v23, v43
-; SI-NEXT:    v_cvt_f16_f32_e32 v29, v44
-; SI-NEXT:    v_cvt_f16_f32_e32 v44, v61
-; SI-NEXT:    v_cvt_f16_f32_e32 v19, v62
-; SI-NEXT:    v_cvt_f16_f32_e32 v20, v63
-; SI-NEXT:    v_cvt_f16_f32_e32 v61, v33
-; SI-NEXT:    v_cvt_f16_f32_e32 v43, v36
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v39
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v54
-; SI-NEXT:    v_cvt_f16_f32_e32 v16, v50
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v41
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v51
+; SI-NEXT:    v_cvt_f16_f32_e32 v22, v43
+; SI-NEXT:    v_cvt_f16_f32_e32 v23, v44
+; SI-NEXT:    v_cvt_f16_f32_e32 v61, v61
+; SI-NEXT:    v_cvt_f16_f32_e32 v44, v62
+; SI-NEXT:    v_cvt_f16_f32_e32 v18, v63
+; SI-NEXT:    v_cvt_f16_f32_e32 v19, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v33, v36
+; SI-NEXT:    v_cvt_f16_f32_e32 v43, v39
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v50
+; SI-NEXT:    v_cvt_f16_f32_e32 v16, v54
+; SI-NEXT:    v_cvt_f16_f32_e32 v54, v41
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v42
 ; SI-NEXT:    v_cvt_f16_f32_e32 v12, v45
-; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT:    v_cvt_f16_f32_e32 v30, v47
-; SI-NEXT:    v_cvt_f16_f32_e32 v11, v42
-; SI-NEXT:    v_cvt_f16_f32_e32 v50, v46
-; SI-NEXT:    v_cvt_f16_f32_e32 v54, v56
-; SI-NEXT:    v_cvt_f16_f32_e32 v45, v32
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v57
 ; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f16_f32_e32 v47, v34
-; SI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:112
-; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:116
-; SI-NEXT:    v_cvt_f16_f32_e32 v36, v58
-; SI-NEXT:    v_cvt_f16_f32_e32 v46, v59
-; SI-NEXT:    s_waitcnt vmcnt(13)
-; SI-NEXT:    v_cvt_f16_f32_e32 v39, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v42, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v13, v46
+; SI-NEXT:    s_waitcnt vmcnt(12)
+; SI-NEXT:    v_cvt_f16_f32_e32 v50, v47
 ; SI-NEXT:    s_waitcnt vmcnt(11)
-; SI-NEXT:    v_cvt_f16_f32_e32 v35, v60
-; SI-NEXT:    v_cvt_f16_f32_e32 v62, v38
+; SI-NEXT:    v_cvt_f16_f32_e32 v30, v56
+; SI-NEXT:    s_waitcnt vmcnt(10)
+; SI-NEXT:    v_cvt_f16_f32_e32 v63, v57
 ; SI-NEXT:    s_waitcnt vmcnt(9)
-; SI-NEXT:    v_cvt_f16_f32_e32 v33, v49
-; SI-NEXT:    v_cvt_f16_f32_e32 v41, v48
+; SI-NEXT:    v_cvt_f16_f32_e32 v62, v58
+; SI-NEXT:    s_waitcnt vmcnt(8)
+; SI-NEXT:    v_cvt_f16_f32_e32 v36, v31
+; SI-NEXT:    s_waitcnt vmcnt(7)
+; SI-NEXT:    v_cvt_f16_f32_e32 v45, v32
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f16_f32_e32 v31, v59
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f16_f32_e32 v32, v34
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v35
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:92
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:96
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:100
+; SI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:104
+; SI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:108
+; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:112
+; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:116
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f16_f32_e32 v42, v3
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f16_f32_e32 v9, v4
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_cvt_f16_f32_e32 v39, v34
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_cvt_f16_f32_e32 v41, v35
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_cvt_f16_f32_e32 v32, v32
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v46
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v47
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SI-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
-; SI-NEXT:    v_mov_b32_e32 v56, v29
-; SI-NEXT:    v_mov_b32_e32 v29, v28
-; SI-NEXT:    v_mov_b32_e32 v57, v25
-; SI-NEXT:    v_mov_b32_e32 v58, v21
-; SI-NEXT:    v_mov_b32_e32 v59, v17
-; SI-NEXT:    v_mov_b32_e32 v60, v6
-; SI-NEXT:    v_mov_b32_e32 v63, v8
+; SI-NEXT:    v_mov_b32_e32 v47, v21
+; SI-NEXT:    v_mov_b32_e32 v56, v17
+; SI-NEXT:    v_mov_b32_e32 v57, v6
+; SI-NEXT:    v_mov_b32_e32 v58, v7
+; SI-NEXT:    v_mov_b32_e32 v59, v33
 ; SI-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_cbranch_execz .LBB58_2
 ; SI-NEXT:  ; %bb.1: ; %cmp.true
-; SI-NEXT:    v_cvt_f32_f16_e32 v6, v62
-; SI-NEXT:    v_cvt_f32_f16_e32 v34, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
 ; SI-NEXT:    v_cvt_f32_f16_e32 v32, v32
-; SI-NEXT:    v_cvt_f32_f16_e32 v35, v35
-; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
-; SI-NEXT:    v_cvt_f16_f32_e32 v62, v6
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
+; SI-NEXT:    v_cvt_f32_f16_e32 v7, v62
+; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
+; SI-NEXT:    v_cvt_f32_f16_e32 v6, v63
+; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
+; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
 ; SI-NEXT:    v_add_f32_e32 v32, 0x38000000, v32
-; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
 ; SI-NEXT:    v_cvt_f16_f32_e32 v32, v32
-; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
-; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
-; SI-NEXT:    v_or_b32_e32 v32, v32, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v62
-; SI-NEXT:    v_or_b32_e32 v35, v35, v37
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v47
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v46
-; SI-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; SI-NEXT:    v_cvt_f32_f16_e32 v8, v54
+; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
+; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v31, v31
+; SI-NEXT:    v_cvt_f16_f32_e32 v62, v7
+; SI-NEXT:    v_add_f32_e32 v6, 0x38000000, v6
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v4
+; SI-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT:    v_or_b32_e32 v3, v3, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v10
+; SI-NEXT:    v_or_b32_e32 v9, v9, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v32
+; SI-NEXT:    v_or_b32_e32 v31, v31, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v62
+; SI-NEXT:    v_or_b32_e32 v63, v6, v34
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; SI-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
 ; SI-NEXT:    v_cvt_f32_f16_e32 v16, v16
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
 ; SI-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v47, v37
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v38
-; SI-NEXT:    v_add_f32_e32 v9, 0x38000000, v9
 ; SI-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
 ; SI-NEXT:    v_cvt_f32_f16_e32 v23, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v9, v9
-; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
-; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
+; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
 ; SI-NEXT:    v_cvt_f32_f16_e32 v22, v22
 ; SI-NEXT:    v_cvt_f32_f16_e32 v27, v27
-; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; SI-NEXT:    v_add_f32_e32 v12, 0x38000000, v12
 ; SI-NEXT:    v_add_f32_e32 v16, 0x38000000, v16
 ; SI-NEXT:    v_cvt_f32_f16_e32 v26, v26
-; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
-; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT:    v_cvt_f16_f32_e32 v12, v12
 ; SI-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; SI-NEXT:    v_add_f32_e32 v15, 0x38000000, v15
-; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
-; SI-NEXT:    v_cvt_f32_f16_e32 v31, v31
-; SI-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v47
-; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v19, 0x38000000, v19
+; SI-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; SI-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; SI-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
 ; SI-NEXT:    v_add_f32_e32 v23, 0x38000000, v23
-; SI-NEXT:    v_cvt_f32_f16_e32 v7, v7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT:    v_or_b32_e32 v46, v37, v38
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v9
-; SI-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
 ; SI-NEXT:    v_cvt_f16_f32_e32 v23, v23
 ; SI-NEXT:    v_add_f32_e32 v22, 0x38000000, v22
 ; SI-NEXT:    v_add_f32_e32 v27, 0x38000000, v27
-; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT:    v_or_b32_e32 v54, v8, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v13
 ; SI-NEXT:    v_cvt_f16_f32_e32 v22, v22
 ; SI-NEXT:    v_cvt_f16_f32_e32 v27, v27
 ; SI-NEXT:    v_add_f32_e32 v26, 0x38000000, v26
-; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
-; SI-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT:    v_or_b32_e32 v11, v11, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v16
+; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
+; SI-NEXT:    v_or_b32_e32 v12, v12, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v16
 ; SI-NEXT:    v_cvt_f16_f32_e32 v26, v26
-; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
-; SI-NEXT:    v_add_f32_e32 v31, 0x38000000, v31
-; SI-NEXT:    v_add_f32_e32 v18, 0x38000000, v18
-; SI-NEXT:    v_or_b32_e32 v15, v15, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v20
-; SI-NEXT:    v_cvt_f16_f32_e32 v31, v31
-; SI-NEXT:    v_cvt_f16_f32_e32 v18, v18
-; SI-NEXT:    v_add_f32_e32 v7, 0x38000000, v7
+; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT:    v_add_f32_e32 v11, 0x38000000, v11
+; SI-NEXT:    v_add_f32_e32 v20, 0x38000000, v20
+; SI-NEXT:    v_or_b32_e32 v15, v15, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v19
+; SI-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; SI-NEXT:    v_cvt_f16_f32_e32 v20, v20
 ; SI-NEXT:    v_add_f32_e32 v2, 0x38000000, v2
-; SI-NEXT:    v_or_b32_e32 v19, v19, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v23
-; SI-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; SI-NEXT:    v_or_b32_e32 v18, v18, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v23
 ; SI-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT:    v_add_f32_e32 v5, 0x38000000, v5
-; SI-NEXT:    v_or_b32_e32 v22, v22, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v27
-; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; SI-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT:    v_add_f32_e32 v4, 0x38000000, v4
-; SI-NEXT:    v_or_b32_e32 v26, v26, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v24
-; SI-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT:    v_or_b32_e32 v31, v31, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v18
-; SI-NEXT:    v_or_b32_e32 v7, v7, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v2
-; SI-NEXT:    v_or_b32_e32 v14, v14, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v5
-; SI-NEXT:    v_or_b32_e32 v4, v4, v37
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v53
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v52
+; SI-NEXT:    v_or_b32_e32 v22, v22, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v27
+; SI-NEXT:    v_or_b32_e32 v26, v26, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v5
+; SI-NEXT:    v_or_b32_e32 v11, v11, v34
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v20
+; SI-NEXT:    v_or_b32_e32 v2, v2, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v38
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v37
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v53, v37
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
+; SI-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v38, v34
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
 ; SI-NEXT:    v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v53
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v38
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT:    v_or_b32_e32 v52, v37, v38
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v40
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v55
+; SI-NEXT:    v_or_b32_e32 v37, v34, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v49
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v48
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_add_f32_e32 v3, 0x38000000, v3
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v40, v37
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v40
-; SI-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; SI-NEXT:    v_or_b32_e32 v55, v37, v38
+; SI-NEXT:    v_cvt_f32_f16_e32 v46, v58
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v49, v34
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_add_f32_e32 v8, 0x38000000, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v49
+; SI-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; SI-NEXT:    v_or_b32_e32 v48, v34, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v53
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v52
+; SI-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v53, v34
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_add_f32_e32 v14, 0x38000000, v14
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v53
+; SI-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; SI-NEXT:    v_or_b32_e32 v52, v34, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT:    v_cvt_f32_f16_e32 v42, v42
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v40, v34
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v24, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v40
+; SI-NEXT:    v_cvt_f32_f16_e32 v28, v28
+; SI-NEXT:    v_or_b32_e32 v55, v34, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v34, v60
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v6
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v63
-; SI-NEXT:    v_add_f32_e32 v10, 0x38000000, v10
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; SI-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; SI-NEXT:    v_cvt_f32_f16_e32 v48, v43
-; SI-NEXT:    v_or_b32_e32 v6, v37, v1
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v6
+; SI-NEXT:    v_cvt_f32_f16_e32 v43, v43
+; SI-NEXT:    v_cvt_f32_f16_e32 v51, v51
+; SI-NEXT:    v_add_f32_e32 v34, 0x38000000, v34
+; SI-NEXT:    v_cvt_f16_f32_e32 v34, v34
+; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; SI-NEXT:    v_cvt_f32_f16_e32 v41, v41
+; SI-NEXT:    v_or_b32_e32 v6, v35, v34
 ; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT:    v_add_f32_e32 v13, 0x38000000, v13
-; SI-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v61
-; SI-NEXT:    v_add_f32_e32 v48, 0x38000000, v48
-; SI-NEXT:    v_cvt_f16_f32_e32 v48, v48
-; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; SI-NEXT:    v_cvt_f32_f16_e32 v30, v30
-; SI-NEXT:    v_cvt_f32_f16_e32 v41, v41
-; SI-NEXT:    v_or_b32_e32 v61, v49, v48
+; SI-NEXT:    v_cvt_f32_f16_e32 v25, v25
 ; SI-NEXT:    v_cvt_f32_f16_e32 v29, v29
+; SI-NEXT:    v_cvt_f32_f16_e32 v54, v54
 ; SI-NEXT:    v_cvt_f32_f16_e32 v50, v50
 ; SI-NEXT:    v_cvt_f32_f16_e32 v36, v36
 ; SI-NEXT:    v_cvt_f32_f16_e32 v39, v39
-; SI-NEXT:    v_cvt_f32_f16_e32 v33, v33
+; SI-NEXT:    v_add_f32_e32 v42, 0x38000000, v42
+; SI-NEXT:    v_cvt_f16_f32_e32 v42, v42
+; SI-NEXT:    v_add_f32_e32 v24, 0x38000000, v24
+; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
+; SI-NEXT:    v_add_f32_e32 v43, 0x38000000, v43
+; SI-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
 ; SI-NEXT:    v_add_f32_e32 v30, 0x38000000, v30
 ; SI-NEXT:    v_add_f32_e32 v41, 0x38000000, v41
+; SI-NEXT:    v_cvt_f16_f32_e32 v24, v24
+; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
+; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
 ; SI-NEXT:    v_add_f32_e32 v29, 0x38000000, v29
+; SI-NEXT:    v_cvt_f16_f32_e32 v43, v43
+; SI-NEXT:    v_cvt_f16_f32_e32 v51, v51
+; SI-NEXT:    v_add_f32_e32 v54, 0x38000000, v54
 ; SI-NEXT:    v_cvt_f16_f32_e32 v30, v30
 ; SI-NEXT:    v_add_f32_e32 v50, 0x38000000, v50
 ; SI-NEXT:    v_add_f32_e32 v36, 0x38000000, v36
-; SI-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
 ; SI-NEXT:    v_cvt_f16_f32_e32 v41, v41
-; SI-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; SI-NEXT:    v_add_f32_e32 v39, 0x38000000, v39
+; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
 ; SI-NEXT:    v_cvt_f16_f32_e32 v29, v29
+; SI-NEXT:    v_cvt_f16_f32_e32 v54, v54
 ; SI-NEXT:    v_cvt_f16_f32_e32 v50, v50
 ; SI-NEXT:    v_cvt_f16_f32_e32 v36, v36
 ; SI-NEXT:    v_cvt_f16_f32_e32 v39, v39
-; SI-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v42
+; SI-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT:    v_lshlrev_b32_e32 v43, 16, v43
+; SI-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
 ; SI-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
 ; SI-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
+; SI-NEXT:    v_or_b32_e32 v25, v25, v24
+; SI-NEXT:    v_or_b32_e32 v29, v29, v28
+; SI-NEXT:    v_or_b32_e32 v54, v54, v51
 ; SI-NEXT:    v_or_b32_e32 v50, v50, v30
-; SI-NEXT:    v_or_b32_e32 v33, v33, v41
-; SI-NEXT:    v_alignbit_b32 v1, v55, v1, 16
-; SI-NEXT:    v_alignbit_b32 v43, v15, v48, 16
-; SI-NEXT:    v_alignbit_b32 v30, v54, v30, 16
-; SI-NEXT:    v_alignbit_b32 v41, v32, v41, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v6
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_or_b32_e32 v6, v37, v3
+; SI-NEXT:    v_or_b32_e32 v39, v39, v41
+; SI-NEXT:    v_alignbit_b32 v60, v55, v34, 16
+; SI-NEXT:    v_alignbit_b32 v24, v26, v24, 16
+; SI-NEXT:    v_alignbit_b32 v28, v22, v28, 16
+; SI-NEXT:    v_alignbit_b32 v51, v12, v51, 16
+; SI-NEXT:    v_alignbit_b32 v30, v63, v30, 16
+; SI-NEXT:    v_alignbit_b32 v41, v3, v41, 16
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v6
+; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_or_b32_e32 v6, v35, v1
 ; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v38
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v60
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_alignbit_b32 v3, v52, v3, 16
-; SI-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
-; SI-NEXT:    v_or_b32_e32 v63, v37, v10
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v59
-; SI-NEXT:    v_cvt_f16_f32_e32 v38, v38
-; SI-NEXT:    v_alignbit_b32 v10, v4, v10, 16
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
-; SI-NEXT:    v_or_b32_e32 v60, v38, v13
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v58
-; SI-NEXT:    v_alignbit_b32 v13, v14, v13, 16
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v46
+; SI-NEXT:    v_cvt_f32_f16_e32 v46, v57
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_alignbit_b32 v1, v52, v1, 16
+; SI-NEXT:    v_add_f32_e32 v46, 0x38000000, v46
+; SI-NEXT:    v_or_b32_e32 v58, v35, v8
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v56
+; SI-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; SI-NEXT:    v_alignbit_b32 v8, v48, v8, 16
+; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; SI-NEXT:    v_or_b32_e32 v57, v46, v14
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v46, v47
+; SI-NEXT:    v_alignbit_b32 v14, v37, v14, 16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v17, v6
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v17, 0x38000000, v17
 ; SI-NEXT:    v_cvt_f16_f32_e32 v17, v17
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; SI-NEXT:    v_or_b32_e32 v59, v37, v17
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v38
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v57
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v38, v38
+; SI-NEXT:    v_or_b32_e32 v56, v35, v17
+; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v46
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v46, v59
+; SI-NEXT:    v_add_f32_e32 v46, 0x38000000, v46
+; SI-NEXT:    v_cvt_f16_f32_e32 v46, v46
+; SI-NEXT:    v_or_b32_e32 v59, v46, v43
+; SI-NEXT:    v_alignbit_b32 v43, v15, v43, 16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v21, v6
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    v_add_f32_e32 v21, 0x38000000, v21
 ; SI-NEXT:    v_cvt_f16_f32_e32 v21, v21
 ; SI-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT:    v_or_b32_e32 v58, v37, v21
-; SI-NEXT:    v_cvt_f32_f16_e32 v37, v44
-; SI-NEXT:    v_add_f32_e32 v37, 0x38000000, v37
-; SI-NEXT:    v_cvt_f16_f32_e32 v37, v37
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; SI-NEXT:    v_alignbit_b32 v44, v19, v37, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v25, v6
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v25, 0x38000000, v25
-; SI-NEXT:    v_cvt_f16_f32_e32 v25, v25
-; SI-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; SI-NEXT:    v_or_b32_e32 v57, v38, v25
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v56
-; SI-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v38, v38
-; SI-NEXT:    v_or_b32_e32 v56, v38, v37
-; SI-NEXT:    v_cvt_f32_f16_e32 v38, v51
-; SI-NEXT:    v_cvt_f32_f16_e32 v51, v42
-; SI-NEXT:    v_add_f32_e32 v38, 0x38000000, v38
-; SI-NEXT:    v_cvt_f16_f32_e32 v38, v38
-; SI-NEXT:    v_add_f32_e32 v51, 0x38000000, v51
-; SI-NEXT:    v_cvt_f16_f32_e32 v51, v51
-; SI-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; SI-NEXT:    v_lshlrev_b32_e32 v42, 16, v51
-; SI-NEXT:    v_or_b32_e32 v39, v39, v42
-; SI-NEXT:    v_alignbit_b32 v51, v11, v38, 16
-; SI-NEXT:    v_alignbit_b32 v42, v35, v42, 16
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v28, v6
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT:    v_add_f32_e32 v28, 0x38000000, v28
-; SI-NEXT:    v_cvt_f16_f32_e32 v28, v28
-; SI-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT:    v_or_b32_e32 v29, v29, v28
+; SI-NEXT:    v_or_b32_e32 v47, v35, v21
+; SI-NEXT:    v_cvt_f32_f16_e32 v35, v44
+; SI-NEXT:    v_cvt_f32_f16_e32 v44, v61
+; SI-NEXT:    v_add_f32_e32 v35, 0x38000000, v35
+; SI-NEXT:    v_cvt_f16_f32_e32 v35, v35
+; SI-NEXT:    v_add_f32_e32 v44, 0x38000000, v44
+; SI-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
+; SI-NEXT:    v_or_b32_e32 v61, v44, v35
+; SI-NEXT:    v_cvt_f32_f16_e32 v44, v45
+; SI-NEXT:    v_add_f32_e32 v44, 0x38000000, v44
+; SI-NEXT:    v_cvt_f16_f32_e32 v44, v44
+; SI-NEXT:    v_lshlrev_b32_e32 v45, 16, v44
+; SI-NEXT:    v_or_b32_e32 v36, v36, v45
+; SI-NEXT:    v_alignbit_b32 v44, v18, v35, 16
+; SI-NEXT:    v_alignbit_b32 v45, v31, v45, 16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v6
-; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT:    v_or_b32_e32 v6, v49, v38
-; SI-NEXT:    v_cvt_f32_f16_e32 v49, v45
+; SI-NEXT:    v_cvt_f32_f16_e32 v33, v6
+; SI-NEXT:    v_add_f32_e32 v33, 0x38000000, v33
+; SI-NEXT:    v_cvt_f16_f32_e32 v33, v33
+; SI-NEXT:    v_or_b32_e32 v6, v33, v42
 ; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v6, v7, v17, 16
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT:    v_add_f32_e32 v49, 0x38000000, v49
-; SI-NEXT:    v_cvt_f16_f32_e32 v49, v49
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v6, v31, v21, 16
-; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v6, v26, v25, 16
-; SI-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; SI-NEXT:    v_or_b32_e32 v36, v36, v49
+; SI-NEXT:    v_alignbit_b32 v6, v2, v17, 16
 ; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_alignbit_b32 v6, v22, v28, 16
-; SI-NEXT:    v_alignbit_b32 v45, v46, v49, 16
+; SI-NEXT:    v_alignbit_b32 v6, v11, v21, 16
+; SI-NEXT:    v_alignbit_b32 v42, v9, v42, 16
 ; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
 ; SI-NEXT:  .LBB58_2: ; %end
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v60
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v37, 0xffff, v6
-; SI-NEXT:    v_or_b32_e32 v1, v37, v1
-; SI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v34, 0xffff, v6
+; SI-NEXT:    v_or_b32_e32 v34, v34, v35
+; SI-NEXT:    buffer_store_dword v34, v0, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v55
-; SI-NEXT:    v_lshlrev_b32_e32 v37, 16, v40
-; SI-NEXT:    v_or_b32_e32 v1, v1, v37
-; SI-NEXT:    v_add_i32_e32 v37, vcc, 4, v0
-; SI-NEXT:    buffer_store_dword v1, v37, s[0:3], 0 offen
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 8, v0
-; SI-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v34, 0xffff, v55
+; SI-NEXT:    v_lshlrev_b32_e32 v35, 16, v40
+; SI-NEXT:    v_or_b32_e32 v34, v34, v35
+; SI-NEXT:    v_add_i32_e32 v35, vcc, 4, v0
+; SI-NEXT:    buffer_store_dword v34, v35, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v34, 0xffff, v6
+; SI-NEXT:    v_or_b32_e32 v1, v34, v1
+; SI-NEXT:    v_add_i32_e32 v34, vcc, 8, v0
+; SI-NEXT:    buffer_store_dword v1, v34, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v52
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v53
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
-; SI-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v34, 16, v53
+; SI-NEXT:    v_or_b32_e32 v1, v1, v34
+; SI-NEXT:    v_add_i32_e32 v34, vcc, 12, v0
+; SI-NEXT:    buffer_store_dword v1, v34, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v63
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v10
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
-; SI-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 20, v0
-; SI-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v58
+; SI-NEXT:    v_or_b32_e32 v1, v1, v8
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 16, v0
+; SI-NEXT:    buffer_store_dword v1, v8, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v60
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v13
-; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_add_i32_e32 v3, vcc, 24, v0
-; SI-NEXT:    buffer_store_dword v1, v3, s[0:3], 0 offen
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v48
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v49
+; SI-NEXT:    v_or_b32_e32 v1, v1, v8
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 20, v0
+; SI-NEXT:    buffer_store_dword v1, v8, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v14
-; SI-NEXT:    v_or_b32_e32 v1, v1, v2
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 28, v0
-; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v57
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v14
+; SI-NEXT:    v_or_b32_e32 v1, v1, v8
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
+; SI-NEXT:    buffer_store_dword v1, v8, s[0:3], 0 offen
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v37
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v38
+; SI-NEXT:    v_or_b32_e32 v1, v1, v8
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 28, v0
+; SI-NEXT:    buffer_store_dword v1, v8, s[0:3], 0 offen
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v59
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v56
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_or_b32_e32 v1, v1, v2
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 32, v0
-; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
+; SI-NEXT:    v_or_b32_e32 v1, v1, v8
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; SI-NEXT:    buffer_store_dword v1, v8, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v18
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v20
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 36, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v58
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v47
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 40, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v31
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v24
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v57
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v25
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v24
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 48, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
@@ -52439,11 +52417,9 @@ define <60 x i16> @bitcast_v60f16_to_v60i16(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 52, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v29
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v28
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 56, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
@@ -52454,19 +52430,19 @@ define <60 x i16> @bitcast_v60f16_to_v60i16(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v56
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v61
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v44
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v19
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v20
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v18
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v19
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x44, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v61
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v59
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v43
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x48, v0
@@ -52478,16 +52454,14 @@ define <60 x i16> @bitcast_v60f16_to_v60i16(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x4c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v51
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x50, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v11
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v12
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v12
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v13
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x54, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
@@ -52498,8 +52472,8 @@ define <60 x i16> @bitcast_v60f16_to_v60i16(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x58, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v54
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v9
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v63
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v62
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x5c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
@@ -52510,32 +52484,34 @@ define <60 x i16> @bitcast_v60f16_to_v60i16(<60 x half> %a, i32 %b) {
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x60, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v46
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v47
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v31
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v32
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x64, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v39
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v42
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x68, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v35
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v62
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v9
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v10
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x6c, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v33
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v39
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v41
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v2, vcc, 0x70, v0
 ; SI-NEXT:    buffer_store_dword v1, v2, s[0:3], 0 offen
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v32
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v34
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v2
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x74, v0
 ; SI-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.ll
index c809f9504f4c..9db06a840c01 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.ll
@@ -2,7 +2,8 @@
 ; RUN: llc -mtriple=amdgcn -amdgpu-codegenprepare-break-large-phis-threshold=4096 < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-codegenprepare-break-large-phis-threshold=4096 < %s | FileCheck -check-prefixes=VI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-codegenprepare-break-large-phis-threshold=4096 < %s | FileCheck -check-prefixes=GFX9 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-codegenprepare-break-large-phis-threshold=4096 < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -amdgpu-codegenprepare-break-large-phis-threshold=4096 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-codegenprepare-break-large-phis-threshold=4096 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 ; This test just checks that the compiler doesn't crash.
 
@@ -8382,30 +8383,55 @@ define void @v_bitcast_v4i8_to_v2bf16(i32 %cond, ptr addrspace(1) %out, <4 x i8>
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_bitcast_v4i8_to_v2bf16:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-NEXT:    v_mov_b32_e32 v0, 0
-; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX11-NEXT:    s_cbranch_execz .LBB68_2
-; GFX11-NEXT:  ; %bb.1: ; %if
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v4
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:  .LBB68_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_bitcast_v4i8_to_v2bf16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-TRUE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB68_2
+; GFX11-TRUE16-NEXT:  ; %bb.1: ; %if
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v6.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT:  .LBB68_2: ; %end
+; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT:    global_store_b32 v[1:2], v0, off
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_bitcast_v4i8_to_v2bf16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_cbranch_execz .LBB68_2
+; GFX11-FAKE16-NEXT:  ; %bb.1: ; %if
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v4
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:  .LBB68_2: ; %end
+; GFX11-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT:    global_store_b32 v[1:2], v0, off
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %if, label %end
@@ -10318,49 +10344,98 @@ define void @v_bitcast_v12i8_to_v6bf16(i32 %cond, ptr addrspace(1) %out, <12 x i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_bitcast_v12i8_to_v6bf16:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v15, 0
-; GFX11-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_mov_b32_e32 v16, v15
-; GFX11-NEXT:    v_mov_b32_e32 v17, v15
-; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX11-NEXT:    s_cbranch_execz .LBB87_2
-; GFX11-NEXT:  ; %bb.1: ; %if
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v4
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v6
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v7
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v9
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v8
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v10
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v11
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v12
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v13
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v14
-; GFX11-NEXT:    v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v3, v6, v3
-; GFX11-NEXT:    v_or_b32_e32 v5, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v6, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v7, v11, v12
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    v_or_b32_e32 v15, v0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_or_b32_e32 v16, v3, v5
-; GFX11-NEXT:    v_or_b32_e32 v17, v6, v7
-; GFX11-NEXT:  .LBB87_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    global_store_b96 v[1:2], v[15:17], off
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_bitcast_v12i8_to_v6bf16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v15, 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v16, v15
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v17, v15
+; GFX11-TRUE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB87_2
+; GFX11-TRUE16-NEXT:  ; %bb.1: ; %if
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v8.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v7, v0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v5, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v3, v4
+; GFX11-TRUE16-NEXT:  .LBB87_2: ; %end
+; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT:    global_store_b96 v[1:2], v[15:17], off
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_bitcast_v12i8_to_v6bf16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v15, 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v16, v15
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v17, v15
+; GFX11-FAKE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-FAKE16-NEXT:    s_cbranch_execz .LBB87_2
+; GFX11-FAKE16-NEXT:  ; %bb.1: ; %if
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v4
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v7
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v10
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v12
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v6, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v11, v12
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v15, v0, v4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v16, v3, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v17, v6, v7
+; GFX11-FAKE16-NEXT:  .LBB87_2: ; %end
+; GFX11-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT:    global_store_b96 v[1:2], v[15:17], off
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %if, label %end
@@ -11464,59 +11539,113 @@ define void @v_bitcast_v16i8_to_v8bf16(i32 %cond, ptr addrspace(1) %out, <16 x i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_bitcast_v16i8_to_v8bf16:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v19, 0
-; GFX11-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_mov_b32_e32 v20, v19
-; GFX11-NEXT:    v_mov_b32_e32 v21, v19
-; GFX11-NEXT:    v_mov_b32_e32 v22, v19
-; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX11-NEXT:    s_cbranch_execz .LBB96_2
-; GFX11-NEXT:  ; %bb.1: ; %if
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v4
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v6
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v7
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v8
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v12
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_and_b32_e32 v5, 0xff, v9
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_lshlrev_b16 v6, 8, v10
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v11
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v13
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v14
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v15
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v16
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v17
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v18
-; GFX11-NEXT:    v_or_b32_e32 v5, v5, v6
-; GFX11-NEXT:    v_or_b32_e32 v6, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v7, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v8, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v9, v13, v14
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-NEXT:    v_or_b32_e32 v19, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v20, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v21, v6, v7
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_or_b32_e32 v22, v8, v9
-; GFX11-NEXT:  .LBB96_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    global_store_b128 v[1:2], v[19:22], off
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_bitcast_v16i8_to_v8bf16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v19, 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v20, v19
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v21, v19
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v22, v19
+; GFX11-TRUE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB96_2
+; GFX11-TRUE16-NEXT:  ; %bb.1: ; %if
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v4
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v7, v8
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v20, v0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v3, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v5, v6
+; GFX11-TRUE16-NEXT:  .LBB96_2: ; %end
+; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[19:22], off
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_bitcast_v16i8_to_v8bf16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v19, 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v20, v19
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v21, v19
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v22, v19
+; GFX11-FAKE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-FAKE16-NEXT:    s_cbranch_execz .LBB96_2
+; GFX11-FAKE16-NEXT:  ; %bb.1: ; %if
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v4
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v5, 0xff, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v6, 8, v10
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v14
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v16
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v18
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v5, v6
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v13, v14
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v19, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v20, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v21, v6, v7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v22, v8, v9
+; GFX11-FAKE16-NEXT:  .LBB96_2: ; %end
+; GFX11-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[19:22], off
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %if, label %end
@@ -14202,91 +14331,169 @@ define void @v_bitcast_v32i8_to_v16bf16(i32 %cond, ptr addrspace(1) %out, <32 x
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_bitcast_v32i8_to_v16bf16:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    scratch_load_u16 v39, off, s32 offset:12
-; GFX11-NEXT:    scratch_load_u16 v48, off, s32 offset:8
-; GFX11-NEXT:    scratch_load_u16 v49, off, s32 offset:4
-; GFX11-NEXT:    scratch_load_u16 v50, off, s32
-; GFX11-NEXT:    v_mov_b32_e32 v31, 0
-; GFX11-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_mov_b32_e32 v32, v31
-; GFX11-NEXT:    v_mov_b32_e32 v33, v31
-; GFX11-NEXT:    v_mov_b32_e32 v34, v31
-; GFX11-NEXT:    v_mov_b32_e32 v35, v31
-; GFX11-NEXT:    v_mov_b32_e32 v36, v31
-; GFX11-NEXT:    v_mov_b32_e32 v37, v31
-; GFX11-NEXT:    v_mov_b32_e32 v38, v31
-; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX11-NEXT:    s_cbranch_execz .LBB112_2
-; GFX11-NEXT:  ; %bb.1: ; %if
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v4
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v6
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v7
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v9
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v8
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v10
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v11
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v12
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v13
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v14
-; GFX11-NEXT:    v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v3, v6, v3
-; GFX11-NEXT:    v_or_b32_e32 v5, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v6, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v7, v11, v12
-; GFX11-NEXT:    v_perm_b32 v31, v4, v0, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v15
-; GFX11-NEXT:    v_perm_b32 v32, v5, v3, 0x5040100
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v16
-; GFX11-NEXT:    v_perm_b32 v33, v7, v6, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v17
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v18
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v19
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v20
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v21
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v22
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v23
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v24
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v25
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v26
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v27
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v28
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v29
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v30
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v50
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v49
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v48
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v39
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v34, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v35, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v36, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v37, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v38, v11, v10, 0x5040100
-; GFX11-NEXT:  .LBB112_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
-; GFX11-NEXT:    global_store_b128 v[1:2], v[31:34], off
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_bitcast_v32i8_to_v16bf16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x3
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v39, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v48, off, s32
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v31, 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v32, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v33, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v34, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v35, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v36, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v37, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v38, v31
+; GFX11-TRUE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB112_2
+; GFX11-TRUE16-NEXT:  ; %bb.1: ; %if
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v33.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v33.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v34.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v34.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v35.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v35.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v36.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v36.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v37.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v37.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v38.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v38.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:  .LBB112_2: ; %end
+; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[31:34], off
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_bitcast_v32i8_to_v16bf16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x3
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v39, off, s32 offset:12
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v48, off, s32 offset:8
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v49, off, s32 offset:4
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v50, off, s32
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v31, 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v32, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v33, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v34, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v35, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v36, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v37, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v38, v31
+; GFX11-FAKE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-FAKE16-NEXT:    s_cbranch_execz .LBB112_2
+; GFX11-FAKE16-NEXT:  ; %bb.1: ; %if
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v4
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v7
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v10
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v12
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v6, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v11, v12
+; GFX11-FAKE16-NEXT:    v_perm_b32 v31, v4, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v15
+; GFX11-FAKE16-NEXT:    v_perm_b32 v32, v5, v3, 0x5040100
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v33, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v18
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v19
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v20
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v21
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v22
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v23
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v24
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v26
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v27
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v28
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v30
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v50
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v49
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v48
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v39
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v34, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v35, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v36, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v37, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v38, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:  .LBB112_2: ; %end
+; GFX11-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[31:34], off
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %if, label %end
@@ -17297,190 +17504,359 @@ define void @v_bitcast_v64i8_to_v32bf16(i32 %cond, ptr addrspace(1) %out, <64 x
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_bitcast_v64i8_to_v32bf16:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_load_u16 v39, off, s32 offset:140
-; GFX11-NEXT:    scratch_load_u16 v64, off, s32 offset:136
-; GFX11-NEXT:    scratch_load_u16 v65, off, s32 offset:132
-; GFX11-NEXT:    scratch_load_u16 v66, off, s32 offset:128
-; GFX11-NEXT:    scratch_load_u16 v67, off, s32 offset:124
-; GFX11-NEXT:    scratch_load_u16 v68, off, s32 offset:120
-; GFX11-NEXT:    scratch_load_u16 v69, off, s32 offset:116
-; GFX11-NEXT:    scratch_load_u16 v70, off, s32 offset:112
-; GFX11-NEXT:    scratch_load_u16 v71, off, s32 offset:108
-; GFX11-NEXT:    scratch_load_u16 v80, off, s32 offset:104
-; GFX11-NEXT:    scratch_load_u16 v81, off, s32 offset:100
-; GFX11-NEXT:    scratch_load_u16 v82, off, s32 offset:96
-; GFX11-NEXT:    scratch_load_u16 v83, off, s32 offset:92
-; GFX11-NEXT:    scratch_load_u16 v84, off, s32 offset:88
-; GFX11-NEXT:    scratch_load_u16 v85, off, s32 offset:84
-; GFX11-NEXT:    scratch_load_u16 v86, off, s32 offset:80
-; GFX11-NEXT:    scratch_load_u16 v87, off, s32 offset:76
-; GFX11-NEXT:    scratch_load_u16 v96, off, s32 offset:72
-; GFX11-NEXT:    scratch_load_u16 v97, off, s32 offset:68
-; GFX11-NEXT:    scratch_load_u16 v98, off, s32 offset:64
-; GFX11-NEXT:    scratch_load_u16 v99, off, s32 offset:60
-; GFX11-NEXT:    scratch_load_u16 v100, off, s32 offset:56
-; GFX11-NEXT:    scratch_load_u16 v101, off, s32 offset:52
-; GFX11-NEXT:    scratch_load_u16 v102, off, s32 offset:48
-; GFX11-NEXT:    scratch_load_u16 v103, off, s32 offset:44
-; GFX11-NEXT:    scratch_load_u16 v112, off, s32 offset:40
-; GFX11-NEXT:    scratch_load_u16 v113, off, s32 offset:36
-; GFX11-NEXT:    scratch_load_u16 v114, off, s32 offset:32
-; GFX11-NEXT:    scratch_load_u16 v115, off, s32 offset:28
-; GFX11-NEXT:    scratch_load_u16 v116, off, s32 offset:24
-; GFX11-NEXT:    scratch_load_u16 v117, off, s32 offset:20
-; GFX11-NEXT:    scratch_load_u16 v118, off, s32 offset:16
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    scratch_load_u16 v119, off, s32 offset:12
-; GFX11-NEXT:    scratch_load_u16 v128, off, s32 offset:8
-; GFX11-NEXT:    scratch_load_u16 v129, off, s32 offset:4
-; GFX11-NEXT:    scratch_load_u16 v130, off, s32
-; GFX11-NEXT:    v_mov_b32_e32 v31, 0
-; GFX11-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_mov_b32_e32 v32, v31
-; GFX11-NEXT:    v_mov_b32_e32 v33, v31
-; GFX11-NEXT:    v_mov_b32_e32 v34, v31
-; GFX11-NEXT:    v_mov_b32_e32 v35, v31
-; GFX11-NEXT:    v_mov_b32_e32 v36, v31
-; GFX11-NEXT:    v_mov_b32_e32 v37, v31
-; GFX11-NEXT:    v_mov_b32_e32 v38, v31
-; GFX11-NEXT:    v_mov_b32_e32 v48, v31
-; GFX11-NEXT:    v_mov_b32_e32 v49, v31
-; GFX11-NEXT:    v_mov_b32_e32 v50, v31
-; GFX11-NEXT:    v_mov_b32_e32 v51, v31
-; GFX11-NEXT:    v_mov_b32_e32 v52, v31
-; GFX11-NEXT:    v_mov_b32_e32 v53, v31
-; GFX11-NEXT:    v_mov_b32_e32 v54, v31
-; GFX11-NEXT:    v_mov_b32_e32 v55, v31
-; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX11-NEXT:    s_cbranch_execz .LBB120_2
-; GFX11-NEXT:  ; %bb.1: ; %if
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v4
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v6
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v9
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v14
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v24
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v7
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v8
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v10
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v13
-; GFX11-NEXT:    v_perm_b32 v31, v3, v0, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v11
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v12
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v15
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v16
-; GFX11-NEXT:    v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v5, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v17
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v18
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v19
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v20
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v21
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v22
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v23
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v25
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v26
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v32, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v33, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v34, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v35, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v36, v11, v10, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v27
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v28
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v29
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v30
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v130
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v129
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v128
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v119
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v118
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v117
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v116
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v115
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v114
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v113
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v112
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v103
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v102
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v101
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v100
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v99
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v37, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v38, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v48, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v49, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v50, v11, v10, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v98
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v97
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v96
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v87
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v86
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v85
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v84
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v83
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v82
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v81
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v80
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v71
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v70
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v69
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v68
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v67
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v66
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v65
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v64
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v39
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v51, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v52, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v53, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v54, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v55, v11, v10, 0x5040100
-; GFX11-NEXT:  .LBB120_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    global_store_b128 v[1:2], v[52:55], off offset:48
-; GFX11-NEXT:    global_store_b128 v[1:2], v[48:51], off offset:32
-; GFX11-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
-; GFX11-NEXT:    global_store_b128 v[1:2], v[31:34], off
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_bitcast_v64i8_to_v32bf16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1f
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v39, off, s32 offset:136
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v83, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    s_clause 0x3
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v31, 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v32, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v33, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v34, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v35, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v36, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v37, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v38, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v48, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v49, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v50, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v51, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v52, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v53, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v54, v31
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v55, v31
+; GFX11-TRUE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB120_2
+; GFX11-TRUE16-NEXT:  ; %bb.1: ; %if
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v33.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v33.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v34.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v34.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v35.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v35.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v36.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v36.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v96.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v87.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v37.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v37.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v38.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v38.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v48.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v85.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v84.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v83.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v81.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v48.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v49.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v49.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v50.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v50.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v69.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v51.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v52.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v52.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v53.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v67.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v65.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v64.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v53.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v54.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v55.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v55.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:  .LBB120_2: ; %end
+; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT:    s_clause 0x3
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[52:55], off offset:48
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[48:51], off offset:32
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[31:34], off
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_bitcast_v64i8_to_v32bf16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1f
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v39, off, s32 offset:140
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v64, off, s32 offset:136
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v65, off, s32 offset:132
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v66, off, s32 offset:128
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v67, off, s32 offset:124
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v68, off, s32 offset:120
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v69, off, s32 offset:116
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v70, off, s32 offset:112
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v71, off, s32 offset:108
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v80, off, s32 offset:104
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v81, off, s32 offset:100
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v82, off, s32 offset:96
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v83, off, s32 offset:92
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v84, off, s32 offset:88
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v85, off, s32 offset:84
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v86, off, s32 offset:80
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v87, off, s32 offset:76
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v96, off, s32 offset:72
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v97, off, s32 offset:68
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v98, off, s32 offset:64
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v99, off, s32 offset:60
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v100, off, s32 offset:56
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v101, off, s32 offset:52
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v102, off, s32 offset:48
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v103, off, s32 offset:44
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v112, off, s32 offset:40
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v113, off, s32 offset:36
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v114, off, s32 offset:32
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v115, off, s32 offset:28
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v116, off, s32 offset:24
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v117, off, s32 offset:20
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v118, off, s32 offset:16
+; GFX11-FAKE16-NEXT:    s_clause 0x3
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v119, off, s32 offset:12
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v128, off, s32 offset:8
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v129, off, s32 offset:4
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v130, off, s32
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v31, 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v32, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v33, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v34, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v35, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v36, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v37, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v38, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v48, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v49, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v50, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v51, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v52, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v53, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v54, v31
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v55, v31
+; GFX11-FAKE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-FAKE16-NEXT:    s_cbranch_execz .LBB120_2
+; GFX11-FAKE16-NEXT:  ; %bb.1: ; %if
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v4
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v24
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v10
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v13
+; GFX11-FAKE16-NEXT:    v_perm_b32 v31, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v12
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v16
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v18
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v19
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v20
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v21
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v22
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v23
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v26
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v32, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v33, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v34, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v35, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v36, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v27
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v28
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v30
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v130
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v129
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v128
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v119
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v118
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v117
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v116
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v115
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v114
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v113
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v112
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v103
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v102
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v101
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v100
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v99
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v37, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v38, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v48, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v49, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v50, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v98
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v97
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v96
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v87
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v86
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v85
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v84
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v83
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v82
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v81
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v80
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v71
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v70
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v69
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v68
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v67
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v66
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v65
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v64
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v39
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v51, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v52, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v53, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v54, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v55, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:  .LBB120_2: ; %end
+; GFX11-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT:    s_clause 0x3
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[52:55], off offset:48
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[48:51], off offset:32
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[31:34], off
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %if, label %end
@@ -28844,544 +29220,942 @@ define void @v_bitcast_v128i8_to_v64bf16(i32 %cond, ptr addrspace(1) %out, <128
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: v_bitcast_v128i8_to_v64bf16:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_store_b32 off, v40, s32 offset:600
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v41, s32 offset:596
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v42, s32 offset:592
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v43, s32 offset:588
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v44, s32 offset:584
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v45, s32 offset:580
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v46, s32 offset:576
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v47, s32 offset:572
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v56, s32 offset:568
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v57, s32 offset:564
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v58, s32 offset:560
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v59, s32 offset:556
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v60, s32 offset:552
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v61, s32 offset:548
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v62, s32 offset:544
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v63, s32 offset:540
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v72, s32 offset:536
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v73, s32 offset:532
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v74, s32 offset:528
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v75, s32 offset:524
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v76, s32 offset:520
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v77, s32 offset:516
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v78, s32 offset:512
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v79, s32 offset:508
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v88, s32 offset:504
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v89, s32 offset:500
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v90, s32 offset:496
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v91, s32 offset:492
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v92, s32 offset:488
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v93, s32 offset:484
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v94, s32 offset:480
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v95, s32 offset:476
-; GFX11-NEXT:    s_clause 0x12
-; GFX11-NEXT:    scratch_store_b32 off, v104, s32 offset:472
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v105, s32 offset:468
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v106, s32 offset:464
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v107, s32 offset:460
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v108, s32 offset:456
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v109, s32 offset:452
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v110, s32 offset:448
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v111, s32 offset:444
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v120, s32 offset:440
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v121, s32 offset:436
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v122, s32 offset:432
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v123, s32 offset:428
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v124, s32 offset:424
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v125, s32 offset:420
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v126, s32 offset:416
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v127, s32 offset:412
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v136, s32 offset:408
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v137, s32 offset:404
-; GFX11-NEXT:    ; meta instruction
-; GFX11-NEXT:    scratch_store_b32 off, v138, s32 offset:400
-; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_load_u16 v64, off, s32 offset:396
-; GFX11-NEXT:    scratch_load_u16 v65, off, s32 offset:392
-; GFX11-NEXT:    scratch_load_u16 v66, off, s32 offset:388
-; GFX11-NEXT:    scratch_load_u16 v67, off, s32 offset:384
-; GFX11-NEXT:    scratch_load_u16 v68, off, s32 offset:380
-; GFX11-NEXT:    scratch_load_u16 v69, off, s32 offset:376
-; GFX11-NEXT:    scratch_load_u16 v70, off, s32 offset:372
-; GFX11-NEXT:    scratch_load_u16 v71, off, s32 offset:368
-; GFX11-NEXT:    scratch_load_u16 v80, off, s32 offset:364
-; GFX11-NEXT:    scratch_load_u16 v81, off, s32 offset:360
-; GFX11-NEXT:    scratch_load_u16 v82, off, s32 offset:356
-; GFX11-NEXT:    scratch_load_u16 v83, off, s32 offset:352
-; GFX11-NEXT:    scratch_load_u16 v84, off, s32 offset:348
-; GFX11-NEXT:    scratch_load_u16 v85, off, s32 offset:344
-; GFX11-NEXT:    scratch_load_u16 v86, off, s32 offset:340
-; GFX11-NEXT:    scratch_load_u16 v87, off, s32 offset:336
-; GFX11-NEXT:    scratch_load_u16 v96, off, s32 offset:332
-; GFX11-NEXT:    scratch_load_u16 v97, off, s32 offset:328
-; GFX11-NEXT:    scratch_load_u16 v98, off, s32 offset:324
-; GFX11-NEXT:    scratch_load_u16 v99, off, s32 offset:320
-; GFX11-NEXT:    scratch_load_u16 v100, off, s32 offset:316
-; GFX11-NEXT:    scratch_load_u16 v101, off, s32 offset:312
-; GFX11-NEXT:    scratch_load_u16 v102, off, s32 offset:308
-; GFX11-NEXT:    scratch_load_u16 v103, off, s32 offset:304
-; GFX11-NEXT:    scratch_load_u16 v112, off, s32 offset:300
-; GFX11-NEXT:    scratch_load_u16 v113, off, s32 offset:296
-; GFX11-NEXT:    scratch_load_u16 v114, off, s32 offset:292
-; GFX11-NEXT:    scratch_load_u16 v115, off, s32 offset:288
-; GFX11-NEXT:    scratch_load_u16 v116, off, s32 offset:284
-; GFX11-NEXT:    scratch_load_u16 v117, off, s32 offset:280
-; GFX11-NEXT:    scratch_load_u16 v118, off, s32 offset:276
-; GFX11-NEXT:    scratch_load_u16 v119, off, s32 offset:272
-; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_load_u16 v128, off, s32 offset:268
-; GFX11-NEXT:    scratch_load_u16 v129, off, s32 offset:264
-; GFX11-NEXT:    scratch_load_u16 v130, off, s32 offset:260
-; GFX11-NEXT:    scratch_load_u16 v131, off, s32 offset:256
-; GFX11-NEXT:    scratch_load_u16 v132, off, s32 offset:252
-; GFX11-NEXT:    scratch_load_u16 v133, off, s32 offset:248
-; GFX11-NEXT:    scratch_load_u16 v134, off, s32 offset:244
-; GFX11-NEXT:    scratch_load_u16 v135, off, s32 offset:240
-; GFX11-NEXT:    scratch_load_u16 v144, off, s32 offset:236
-; GFX11-NEXT:    scratch_load_u16 v145, off, s32 offset:232
-; GFX11-NEXT:    scratch_load_u16 v146, off, s32 offset:228
-; GFX11-NEXT:    scratch_load_u16 v147, off, s32 offset:224
-; GFX11-NEXT:    scratch_load_u16 v148, off, s32 offset:220
-; GFX11-NEXT:    scratch_load_u16 v149, off, s32 offset:216
-; GFX11-NEXT:    scratch_load_u16 v150, off, s32 offset:212
-; GFX11-NEXT:    scratch_load_u16 v151, off, s32 offset:208
-; GFX11-NEXT:    scratch_load_u16 v160, off, s32 offset:204
-; GFX11-NEXT:    scratch_load_u16 v161, off, s32 offset:200
-; GFX11-NEXT:    scratch_load_u16 v162, off, s32 offset:196
-; GFX11-NEXT:    scratch_load_u16 v163, off, s32 offset:192
-; GFX11-NEXT:    scratch_load_u16 v164, off, s32 offset:188
-; GFX11-NEXT:    scratch_load_u16 v165, off, s32 offset:184
-; GFX11-NEXT:    scratch_load_u16 v166, off, s32 offset:180
-; GFX11-NEXT:    scratch_load_u16 v167, off, s32 offset:176
-; GFX11-NEXT:    scratch_load_u16 v176, off, s32 offset:172
-; GFX11-NEXT:    scratch_load_u16 v177, off, s32 offset:168
-; GFX11-NEXT:    scratch_load_u16 v178, off, s32 offset:164
-; GFX11-NEXT:    scratch_load_u16 v179, off, s32 offset:160
-; GFX11-NEXT:    scratch_load_u16 v180, off, s32 offset:156
-; GFX11-NEXT:    scratch_load_u16 v181, off, s32 offset:152
-; GFX11-NEXT:    scratch_load_u16 v182, off, s32 offset:148
-; GFX11-NEXT:    scratch_load_u16 v183, off, s32 offset:144
-; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_load_u16 v63, off, s32 offset:140
-; GFX11-NEXT:    scratch_load_u16 v72, off, s32 offset:136
-; GFX11-NEXT:    scratch_load_u16 v73, off, s32 offset:132
-; GFX11-NEXT:    scratch_load_u16 v74, off, s32 offset:128
-; GFX11-NEXT:    scratch_load_u16 v75, off, s32 offset:124
-; GFX11-NEXT:    scratch_load_u16 v76, off, s32 offset:120
-; GFX11-NEXT:    scratch_load_u16 v77, off, s32 offset:116
-; GFX11-NEXT:    scratch_load_u16 v78, off, s32 offset:112
-; GFX11-NEXT:    scratch_load_u16 v79, off, s32 offset:108
-; GFX11-NEXT:    scratch_load_u16 v88, off, s32 offset:104
-; GFX11-NEXT:    scratch_load_u16 v89, off, s32 offset:100
-; GFX11-NEXT:    scratch_load_u16 v90, off, s32 offset:96
-; GFX11-NEXT:    scratch_load_u16 v91, off, s32 offset:92
-; GFX11-NEXT:    scratch_load_u16 v92, off, s32 offset:88
-; GFX11-NEXT:    scratch_load_u16 v93, off, s32 offset:84
-; GFX11-NEXT:    scratch_load_u16 v94, off, s32 offset:80
-; GFX11-NEXT:    scratch_load_u16 v95, off, s32 offset:76
-; GFX11-NEXT:    scratch_load_u16 v104, off, s32 offset:72
-; GFX11-NEXT:    scratch_load_u16 v105, off, s32 offset:68
-; GFX11-NEXT:    scratch_load_u16 v106, off, s32 offset:64
-; GFX11-NEXT:    scratch_load_u16 v107, off, s32 offset:60
-; GFX11-NEXT:    scratch_load_u16 v108, off, s32 offset:56
-; GFX11-NEXT:    scratch_load_u16 v109, off, s32 offset:52
-; GFX11-NEXT:    scratch_load_u16 v110, off, s32 offset:48
-; GFX11-NEXT:    scratch_load_u16 v111, off, s32 offset:44
-; GFX11-NEXT:    scratch_load_u16 v120, off, s32 offset:40
-; GFX11-NEXT:    scratch_load_u16 v121, off, s32 offset:36
-; GFX11-NEXT:    scratch_load_u16 v122, off, s32 offset:32
-; GFX11-NEXT:    scratch_load_u16 v123, off, s32 offset:28
-; GFX11-NEXT:    scratch_load_u16 v124, off, s32 offset:24
-; GFX11-NEXT:    scratch_load_u16 v125, off, s32 offset:20
-; GFX11-NEXT:    scratch_load_u16 v126, off, s32 offset:16
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    scratch_load_u16 v127, off, s32 offset:12
-; GFX11-NEXT:    scratch_load_u16 v136, off, s32 offset:8
-; GFX11-NEXT:    scratch_load_u16 v137, off, s32 offset:4
-; GFX11-NEXT:    scratch_load_u16 v138, off, s32
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_mov_b32 s15, s0
-; GFX11-NEXT:    s_mov_b32 s1, s0
-; GFX11-NEXT:    s_mov_b32 s2, s0
-; GFX11-NEXT:    s_mov_b32 s3, s0
-; GFX11-NEXT:    s_mov_b32 s4, s0
-; GFX11-NEXT:    s_mov_b32 s5, s0
-; GFX11-NEXT:    s_mov_b32 s6, s0
-; GFX11-NEXT:    s_mov_b32 s7, s0
-; GFX11-NEXT:    s_mov_b32 s8, s0
-; GFX11-NEXT:    s_mov_b32 s9, s0
-; GFX11-NEXT:    s_mov_b32 s10, s0
-; GFX11-NEXT:    s_mov_b32 s11, s0
-; GFX11-NEXT:    s_mov_b32 s12, s0
-; GFX11-NEXT:    s_mov_b32 s13, s0
-; GFX11-NEXT:    s_mov_b32 s14, s0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v46, s15 :: v_dual_mov_b32 v45, s14
-; GFX11-NEXT:    v_dual_mov_b32 v44, s13 :: v_dual_mov_b32 v43, s12
-; GFX11-NEXT:    v_dual_mov_b32 v42, s11 :: v_dual_mov_b32 v41, s10
-; GFX11-NEXT:    v_dual_mov_b32 v40, s9 :: v_dual_mov_b32 v39, s8
-; GFX11-NEXT:    v_dual_mov_b32 v38, s7 :: v_dual_mov_b32 v37, s6
-; GFX11-NEXT:    v_dual_mov_b32 v36, s5 :: v_dual_mov_b32 v35, s4
-; GFX11-NEXT:    v_dual_mov_b32 v34, s3 :: v_dual_mov_b32 v33, s2
-; GFX11-NEXT:    v_dual_mov_b32 v32, s1 :: v_dual_mov_b32 v31, s0
-; GFX11-NEXT:    v_dual_mov_b32 v62, v46 :: v_dual_mov_b32 v61, v45
-; GFX11-NEXT:    v_dual_mov_b32 v60, v44 :: v_dual_mov_b32 v59, v43
-; GFX11-NEXT:    v_dual_mov_b32 v58, v42 :: v_dual_mov_b32 v57, v41
-; GFX11-NEXT:    v_dual_mov_b32 v56, v40 :: v_dual_mov_b32 v55, v39
-; GFX11-NEXT:    v_dual_mov_b32 v54, v38 :: v_dual_mov_b32 v53, v37
-; GFX11-NEXT:    v_dual_mov_b32 v52, v36 :: v_dual_mov_b32 v51, v35
-; GFX11-NEXT:    v_dual_mov_b32 v50, v34 :: v_dual_mov_b32 v49, v33
-; GFX11-NEXT:    v_dual_mov_b32 v48, v32 :: v_dual_mov_b32 v47, v31
-; GFX11-NEXT:    s_mov_b32 s0, exec_lo
-; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX11-NEXT:    s_cbranch_execz .LBB139_2
-; GFX11-NEXT:  ; %bb.1: ; %if
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v3
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v4
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v6
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v7
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v8
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v9
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v10
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v15
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v16
-; GFX11-NEXT:    v_perm_b32 v31, v3, v0, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v11
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v12
-; GFX11-NEXT:    v_perm_b32 v32, v5, v4, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v13
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v14
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v17
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v18
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v19
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v20
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v21
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v22
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v23
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v24
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v25
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v26
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v27
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v28
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v29
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v30
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v33, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v34, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v35, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v36, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v37, v11, v10, 0x5040100
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v138
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v137
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v136
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v127
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v126
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v125
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v124
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v123
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v122
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v121
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v120
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v111
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v110
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v109
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v108
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v107
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v106
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v105
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v104
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v95
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v38, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v39, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v40, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v41, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v42, v11, v10, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v94
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v93
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v92
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v91
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v90
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v89
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v88
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v79
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v78
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v77
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v76
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v75
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v74
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v73
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v72
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v63
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v183
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v182
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v181
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v180
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v43, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v44, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v45, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v46, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v47, v11, v10, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v179
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v178
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v177
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v176
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v167
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v166
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v165
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v164
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v163
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v162
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v161
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v160
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v151
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v150
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v149
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v148
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v147
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v146
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v145
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v144
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v48, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v49, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v50, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v51, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v52, v11, v10, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v135
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v134
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v133
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v132
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v131
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v130
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v129
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v128
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v119
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v118
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v117
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v116
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v115
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v114
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v113
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v112
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v103
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v102
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v101
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v100
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v53, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v54, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v55, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v56, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v57, v11, v10, 0x5040100
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xff, v99
-; GFX11-NEXT:    v_lshlrev_b16 v3, 8, v98
-; GFX11-NEXT:    v_and_b32_e32 v4, 0xff, v97
-; GFX11-NEXT:    v_lshlrev_b16 v5, 8, v96
-; GFX11-NEXT:    v_and_b32_e32 v6, 0xff, v87
-; GFX11-NEXT:    v_lshlrev_b16 v7, 8, v86
-; GFX11-NEXT:    v_and_b32_e32 v8, 0xff, v85
-; GFX11-NEXT:    v_lshlrev_b16 v9, 8, v84
-; GFX11-NEXT:    v_and_b32_e32 v10, 0xff, v83
-; GFX11-NEXT:    v_lshlrev_b16 v11, 8, v82
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX11-NEXT:    v_or_b32_e32 v3, v4, v5
-; GFX11-NEXT:    v_or_b32_e32 v4, v6, v7
-; GFX11-NEXT:    v_or_b32_e32 v5, v8, v9
-; GFX11-NEXT:    v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT:    v_and_b32_e32 v7, 0xff, v81
-; GFX11-NEXT:    v_lshlrev_b16 v8, 8, v80
-; GFX11-NEXT:    v_and_b32_e32 v9, 0xff, v71
-; GFX11-NEXT:    v_lshlrev_b16 v10, 8, v70
-; GFX11-NEXT:    v_and_b32_e32 v11, 0xff, v69
-; GFX11-NEXT:    v_lshlrev_b16 v12, 8, v68
-; GFX11-NEXT:    v_and_b32_e32 v13, 0xff, v67
-; GFX11-NEXT:    v_lshlrev_b16 v14, 8, v66
-; GFX11-NEXT:    v_and_b32_e32 v15, 0xff, v65
-; GFX11-NEXT:    v_lshlrev_b16 v16, 8, v64
-; GFX11-NEXT:    v_or_b32_e32 v7, v7, v8
-; GFX11-NEXT:    v_or_b32_e32 v8, v9, v10
-; GFX11-NEXT:    v_or_b32_e32 v9, v11, v12
-; GFX11-NEXT:    v_or_b32_e32 v10, v13, v14
-; GFX11-NEXT:    v_or_b32_e32 v11, v15, v16
-; GFX11-NEXT:    v_perm_b32 v58, v3, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v59, v5, v4, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v60, v7, v6, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v61, v9, v8, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v62, v11, v10, 0x5040100
-; GFX11-NEXT:  .LBB139_2: ; %end
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT:    s_clause 0x7
-; GFX11-NEXT:    global_store_b128 v[1:2], v[43:46], off offset:48
-; GFX11-NEXT:    global_store_b128 v[1:2], v[39:42], off offset:32
-; GFX11-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
-; GFX11-NEXT:    global_store_b128 v[1:2], v[31:34], off
-; GFX11-NEXT:    global_store_b128 v[1:2], v[59:62], off offset:112
-; GFX11-NEXT:    global_store_b128 v[1:2], v[55:58], off offset:96
-; GFX11-NEXT:    global_store_b128 v[1:2], v[51:54], off offset:80
-; GFX11-NEXT:    global_store_b128 v[1:2], v[47:50], off offset:64
-; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    scratch_load_b32 v138, off, s32 offset:400
-; GFX11-NEXT:    scratch_load_b32 v137, off, s32 offset:404
-; GFX11-NEXT:    scratch_load_b32 v136, off, s32 offset:408
-; GFX11-NEXT:    scratch_load_b32 v127, off, s32 offset:412
-; GFX11-NEXT:    scratch_load_b32 v126, off, s32 offset:416
-; GFX11-NEXT:    scratch_load_b32 v125, off, s32 offset:420
-; GFX11-NEXT:    scratch_load_b32 v124, off, s32 offset:424
-; GFX11-NEXT:    scratch_load_b32 v123, off, s32 offset:428
-; GFX11-NEXT:    scratch_load_b32 v122, off, s32 offset:432
-; GFX11-NEXT:    scratch_load_b32 v121, off, s32 offset:436
-; GFX11-NEXT:    scratch_load_b32 v120, off, s32 offset:440
-; GFX11-NEXT:    scratch_load_b32 v111, off, s32 offset:444
-; GFX11-NEXT:    scratch_load_b32 v110, off, s32 offset:448
-; GFX11-NEXT:    scratch_load_b32 v109, off, s32 offset:452
-; GFX11-NEXT:    scratch_load_b32 v108, off, s32 offset:456
-; GFX11-NEXT:    scratch_load_b32 v107, off, s32 offset:460
-; GFX11-NEXT:    scratch_load_b32 v106, off, s32 offset:464
-; GFX11-NEXT:    scratch_load_b32 v105, off, s32 offset:468
-; GFX11-NEXT:    scratch_load_b32 v104, off, s32 offset:472
-; GFX11-NEXT:    scratch_load_b32 v95, off, s32 offset:476
-; GFX11-NEXT:    scratch_load_b32 v94, off, s32 offset:480
-; GFX11-NEXT:    scratch_load_b32 v93, off, s32 offset:484
-; GFX11-NEXT:    scratch_load_b32 v92, off, s32 offset:488
-; GFX11-NEXT:    scratch_load_b32 v91, off, s32 offset:492
-; GFX11-NEXT:    scratch_load_b32 v90, off, s32 offset:496
-; GFX11-NEXT:    scratch_load_b32 v89, off, s32 offset:500
-; GFX11-NEXT:    scratch_load_b32 v88, off, s32 offset:504
-; GFX11-NEXT:    scratch_load_b32 v79, off, s32 offset:508
-; GFX11-NEXT:    scratch_load_b32 v78, off, s32 offset:512
-; GFX11-NEXT:    scratch_load_b32 v77, off, s32 offset:516
-; GFX11-NEXT:    scratch_load_b32 v76, off, s32 offset:520
-; GFX11-NEXT:    scratch_load_b32 v75, off, s32 offset:524
-; GFX11-NEXT:    s_clause 0x12
-; GFX11-NEXT:    scratch_load_b32 v74, off, s32 offset:528
-; GFX11-NEXT:    scratch_load_b32 v73, off, s32 offset:532
-; GFX11-NEXT:    scratch_load_b32 v72, off, s32 offset:536
-; GFX11-NEXT:    scratch_load_b32 v63, off, s32 offset:540
-; GFX11-NEXT:    scratch_load_b32 v62, off, s32 offset:544
-; GFX11-NEXT:    scratch_load_b32 v61, off, s32 offset:548
-; GFX11-NEXT:    scratch_load_b32 v60, off, s32 offset:552
-; GFX11-NEXT:    scratch_load_b32 v59, off, s32 offset:556
-; GFX11-NEXT:    scratch_load_b32 v58, off, s32 offset:560
-; GFX11-NEXT:    scratch_load_b32 v57, off, s32 offset:564
-; GFX11-NEXT:    scratch_load_b32 v56, off, s32 offset:568
-; GFX11-NEXT:    scratch_load_b32 v47, off, s32 offset:572
-; GFX11-NEXT:    scratch_load_b32 v46, off, s32 offset:576
-; GFX11-NEXT:    scratch_load_b32 v45, off, s32 offset:580
-; GFX11-NEXT:    scratch_load_b32 v44, off, s32 offset:584
-; GFX11-NEXT:    scratch_load_b32 v43, off, s32 offset:588
-; GFX11-NEXT:    scratch_load_b32 v42, off, s32 offset:592
-; GFX11-NEXT:    scratch_load_b32 v41, off, s32 offset:596
-; GFX11-NEXT:    scratch_load_b32 v40, off, s32 offset:600
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_bitcast_v128i8_to_v64bf16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0xe
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v40, s32 offset:456
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v41, s32 offset:452
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v42, s32 offset:448
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v43, s32 offset:444
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v44, s32 offset:440
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v45, s32 offset:436
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v46, s32 offset:432
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v47, s32 offset:428
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v56, s32 offset:424
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v57, s32 offset:420
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v58, s32 offset:416
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v59, s32 offset:412
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v60, s32 offset:408
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v61, s32 offset:404
+; GFX11-TRUE16-NEXT:    ; meta instruction
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v62, s32 offset:400
+; GFX11-TRUE16-NEXT:    s_clause 0x1f
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:396
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:392
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:388
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:384
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:380
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:376
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:372
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:368
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:364
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:360
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:356
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:352
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:348
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:344
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:340
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:336
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:332
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32 offset:328
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:324
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32 offset:320
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:312
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:308
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v83, off, s32 offset:304
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:300
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:296
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    s_clause 0x1f
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:260
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:252
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:248
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:244
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:236
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v103, off, s32 offset:208
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v112, off, s32 offset:200
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v113, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v113, off, s32 offset:192
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v114, off, s32 offset:184
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v115, off, s32 offset:176
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v116, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v116, off, s32 offset:168
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v117, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v117, off, s32 offset:160
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v118, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v118, off, s32 offset:152
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v119, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v119, off, s32 offset:144
+; GFX11-TRUE16-NEXT:    s_clause 0x1f
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v128, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v128, off, s32 offset:136
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v129, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v129, off, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v130, off, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v130, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v131, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v131, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v132, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v132, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v133, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v133, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v134, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v134, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v135, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v135, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v144, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v144, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v145, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v145, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v146, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v146, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v147, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v147, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v148, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v148, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v149, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v149, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v150, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v150, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v151, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v151, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    s_clause 0x3
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v160, off, s32 offset:8
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v161, off, s32
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s15, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s1, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s2, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s3, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s4, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s5, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s12, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s13, s0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s14, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v46, s15 :: v_dual_mov_b32 v45, s14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v44, s13 :: v_dual_mov_b32 v43, s12
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v42, s11 :: v_dual_mov_b32 v41, s10
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v40, s9 :: v_dual_mov_b32 v39, s8
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v38, s7 :: v_dual_mov_b32 v37, s6
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v36, s5 :: v_dual_mov_b32 v35, s4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v34, s3 :: v_dual_mov_b32 v33, s2
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v32, s1 :: v_dual_mov_b32 v31, s0
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v62, v46 :: v_dual_mov_b32 v61, v45
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v60, v44 :: v_dual_mov_b32 v59, v43
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v58, v42 :: v_dual_mov_b32 v57, v41
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v56, v40 :: v_dual_mov_b32 v55, v39
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v54, v38 :: v_dual_mov_b32 v53, v37
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v52, v36 :: v_dual_mov_b32 v51, v35
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v34 :: v_dual_mov_b32 v49, v33
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v48, v32 :: v_dual_mov_b32 v47, v31
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB139_2
+; GFX11-TRUE16-NEXT:  ; %bb.1: ; %if
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v33.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v33.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v34.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v34.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v35.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v35.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v36.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v36.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v37.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v37.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v161.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v160.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v151.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v151.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v150.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v150.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v149.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v38.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v38.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v40.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v148.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v148.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v147.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v147.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v146.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v145.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v145.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v144.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v40.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v41.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v41.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v42.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v42.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v135.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v135.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v134.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v134.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v133.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v133.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v132.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v131.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v131.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v43.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v43.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v44.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v44.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v45.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v130.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v130.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v129.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v128.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v119.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v118.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v45.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v46.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v46.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v47.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v47.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v117.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v117.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v116.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v116.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v115.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v115.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v114.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v114.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v113.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v113.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v48.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v48.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v49.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v49.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v50.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v112.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v103.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v102.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v101.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v100.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v100.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v50.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v51.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v52.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v52.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v99.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v98.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v97.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v96.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v87.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v53.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v53.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v54.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v55.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v85.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v84.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v83.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v55.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v56.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v56.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v57.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v57.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v81.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v69.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v58.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v58.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v59.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v59.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v60.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v67.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v65.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v64.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v60.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v61.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v61.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v62.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v62.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT:  .LBB139_2: ; %end
+; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT:    s_clause 0x7
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[43:46], off offset:48
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[39:42], off offset:32
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[31:34], off
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[59:62], off offset:112
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[55:58], off offset:96
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[51:54], off offset:80
+; GFX11-TRUE16-NEXT:    global_store_b128 v[1:2], v[47:50], off offset:64
+; GFX11-TRUE16-NEXT:    s_clause 0xe
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v62, off, s32 offset:400
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v61, off, s32 offset:404
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v60, off, s32 offset:408
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v59, off, s32 offset:412
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v58, off, s32 offset:416
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v57, off, s32 offset:420
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v56, off, s32 offset:424
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v47, off, s32 offset:428
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v46, off, s32 offset:432
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v45, off, s32 offset:436
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v44, off, s32 offset:440
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v43, off, s32 offset:444
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v42, off, s32 offset:448
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v41, off, s32 offset:452
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v40, off, s32 offset:456
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_bitcast_v128i8_to_v64bf16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1f
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v40, s32 offset:600
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v41, s32 offset:596
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v42, s32 offset:592
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v43, s32 offset:588
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v44, s32 offset:584
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v45, s32 offset:580
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v46, s32 offset:576
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v47, s32 offset:572
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v56, s32 offset:568
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v57, s32 offset:564
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v58, s32 offset:560
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v59, s32 offset:556
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v60, s32 offset:552
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v61, s32 offset:548
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v62, s32 offset:544
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v63, s32 offset:540
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v72, s32 offset:536
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v73, s32 offset:532
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v74, s32 offset:528
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v75, s32 offset:524
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v76, s32 offset:520
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v77, s32 offset:516
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v78, s32 offset:512
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v79, s32 offset:508
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v88, s32 offset:504
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v89, s32 offset:500
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v90, s32 offset:496
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v91, s32 offset:492
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v92, s32 offset:488
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v93, s32 offset:484
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v94, s32 offset:480
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v95, s32 offset:476
+; GFX11-FAKE16-NEXT:    s_clause 0x12
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v104, s32 offset:472
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v105, s32 offset:468
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v106, s32 offset:464
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v107, s32 offset:460
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v108, s32 offset:456
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v109, s32 offset:452
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v110, s32 offset:448
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v111, s32 offset:444
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v120, s32 offset:440
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v121, s32 offset:436
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v122, s32 offset:432
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v123, s32 offset:428
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v124, s32 offset:424
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v125, s32 offset:420
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v126, s32 offset:416
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v127, s32 offset:412
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v136, s32 offset:408
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v137, s32 offset:404
+; GFX11-FAKE16-NEXT:    ; meta instruction
+; GFX11-FAKE16-NEXT:    scratch_store_b32 off, v138, s32 offset:400
+; GFX11-FAKE16-NEXT:    s_clause 0x1f
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v64, off, s32 offset:396
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v65, off, s32 offset:392
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v66, off, s32 offset:388
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v67, off, s32 offset:384
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v68, off, s32 offset:380
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v69, off, s32 offset:376
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v70, off, s32 offset:372
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v71, off, s32 offset:368
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v80, off, s32 offset:364
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v81, off, s32 offset:360
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v82, off, s32 offset:356
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v83, off, s32 offset:352
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v84, off, s32 offset:348
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v85, off, s32 offset:344
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v86, off, s32 offset:340
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v87, off, s32 offset:336
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v96, off, s32 offset:332
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v97, off, s32 offset:328
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v98, off, s32 offset:324
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v99, off, s32 offset:320
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v100, off, s32 offset:316
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v101, off, s32 offset:312
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v102, off, s32 offset:308
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v103, off, s32 offset:304
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v112, off, s32 offset:300
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v113, off, s32 offset:296
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v114, off, s32 offset:292
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v115, off, s32 offset:288
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v116, off, s32 offset:284
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v117, off, s32 offset:280
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v118, off, s32 offset:276
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v119, off, s32 offset:272
+; GFX11-FAKE16-NEXT:    s_clause 0x1f
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v128, off, s32 offset:268
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v129, off, s32 offset:264
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v130, off, s32 offset:260
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v131, off, s32 offset:256
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v132, off, s32 offset:252
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v133, off, s32 offset:248
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v134, off, s32 offset:244
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v135, off, s32 offset:240
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v144, off, s32 offset:236
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v145, off, s32 offset:232
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v146, off, s32 offset:228
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v147, off, s32 offset:224
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v148, off, s32 offset:220
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v149, off, s32 offset:216
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v150, off, s32 offset:212
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v151, off, s32 offset:208
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v160, off, s32 offset:204
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v161, off, s32 offset:200
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v162, off, s32 offset:196
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v163, off, s32 offset:192
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v164, off, s32 offset:188
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v165, off, s32 offset:184
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v166, off, s32 offset:180
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v167, off, s32 offset:176
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v176, off, s32 offset:172
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v177, off, s32 offset:168
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v178, off, s32 offset:164
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v179, off, s32 offset:160
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v180, off, s32 offset:156
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v181, off, s32 offset:152
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v182, off, s32 offset:148
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v183, off, s32 offset:144
+; GFX11-FAKE16-NEXT:    s_clause 0x1f
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v63, off, s32 offset:140
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v72, off, s32 offset:136
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v73, off, s32 offset:132
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v74, off, s32 offset:128
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v75, off, s32 offset:124
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v76, off, s32 offset:120
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v77, off, s32 offset:116
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v78, off, s32 offset:112
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v79, off, s32 offset:108
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v88, off, s32 offset:104
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v89, off, s32 offset:100
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v90, off, s32 offset:96
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v91, off, s32 offset:92
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v92, off, s32 offset:88
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v93, off, s32 offset:84
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v94, off, s32 offset:80
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v95, off, s32 offset:76
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v104, off, s32 offset:72
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v105, off, s32 offset:68
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v106, off, s32 offset:64
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v107, off, s32 offset:60
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v108, off, s32 offset:56
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v109, off, s32 offset:52
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v110, off, s32 offset:48
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v111, off, s32 offset:44
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v120, off, s32 offset:40
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v121, off, s32 offset:36
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v122, off, s32 offset:32
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v123, off, s32 offset:28
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v124, off, s32 offset:24
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v125, off, s32 offset:20
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v126, off, s32 offset:16
+; GFX11-FAKE16-NEXT:    s_clause 0x3
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v127, off, s32 offset:12
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v136, off, s32 offset:8
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v137, off, s32 offset:4
+; GFX11-FAKE16-NEXT:    scratch_load_u16 v138, off, s32
+; GFX11-FAKE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s15, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s1, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s2, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s3, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s4, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s5, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s12, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s13, s0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s14, s0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v46, s15 :: v_dual_mov_b32 v45, s14
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v44, s13 :: v_dual_mov_b32 v43, s12
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v42, s11 :: v_dual_mov_b32 v41, s10
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v40, s9 :: v_dual_mov_b32 v39, s8
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v38, s7 :: v_dual_mov_b32 v37, s6
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v36, s5 :: v_dual_mov_b32 v35, s4
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v34, s3 :: v_dual_mov_b32 v33, s2
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v32, s1 :: v_dual_mov_b32 v31, s0
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v62, v46 :: v_dual_mov_b32 v61, v45
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v60, v44 :: v_dual_mov_b32 v59, v43
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v58, v42 :: v_dual_mov_b32 v57, v41
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v56, v40 :: v_dual_mov_b32 v55, v39
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v54, v38 :: v_dual_mov_b32 v53, v37
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v52, v36 :: v_dual_mov_b32 v51, v35
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v50, v34 :: v_dual_mov_b32 v49, v33
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v48, v32 :: v_dual_mov_b32 v47, v31
+; GFX11-FAKE16-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-FAKE16-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-FAKE16-NEXT:    s_cbranch_execz .LBB139_2
+; GFX11-FAKE16-NEXT:  ; %bb.1: ; %if
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v4
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v8
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v15
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v31, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v12
+; GFX11-FAKE16-NEXT:    v_perm_b32 v32, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v14
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v18
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v19
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v20
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v21
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v22
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v23
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v26
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v27
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v28
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v30
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v33, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v34, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v35, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v36, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v37, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v138
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v137
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v136
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v127
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v126
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v125
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v124
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v123
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v122
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v121
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v120
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v111
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v110
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v109
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v108
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v107
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v106
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v105
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v104
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v95
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v38, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v39, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v40, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v41, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v42, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v94
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v93
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v92
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v91
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v90
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v89
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v88
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v79
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v78
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v77
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v76
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v75
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v74
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v73
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v72
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v63
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v183
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v182
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v181
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v180
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v43, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v44, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v45, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v46, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v47, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v179
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v178
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v177
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v176
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v167
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v166
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v165
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v164
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v163
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v162
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v161
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v160
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v151
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v150
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v149
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v148
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v147
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v146
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v145
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v144
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v48, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v49, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v50, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v51, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v52, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v135
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v134
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v133
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v132
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v131
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v130
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v129
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v128
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v119
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v118
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v117
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v116
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v115
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v114
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v113
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v112
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v103
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v102
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v101
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v100
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v53, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v54, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v55, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v56, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v57, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xff, v99
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v3, 8, v98
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v97
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v5, 8, v96
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v87
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v7, 8, v86
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff, v85
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v9, 8, v84
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xff, v83
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v11, 8, v82
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, v4, v5
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, v6, v7
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, v8, v9
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v6, v10, v11
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v81
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v8, 8, v80
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xff, v71
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v10, 8, v70
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xff, v69
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v12, 8, v68
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xff, v67
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v14, 8, v66
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xff, v65
+; GFX11-FAKE16-NEXT:    v_lshlrev_b16 v16, 8, v64
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v7, v7, v8
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, v9, v10
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, v11, v12
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, v13, v14
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, v15, v16
+; GFX11-FAKE16-NEXT:    v_perm_b32 v58, v3, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v59, v5, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v60, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v61, v9, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v62, v11, v10, 0x5040100
+; GFX11-FAKE16-NEXT:  .LBB139_2: ; %end
+; GFX11-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT:    s_clause 0x7
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[43:46], off offset:48
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[39:42], off offset:32
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[35:38], off offset:16
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[31:34], off
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[59:62], off offset:112
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[55:58], off offset:96
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[51:54], off offset:80
+; GFX11-FAKE16-NEXT:    global_store_b128 v[1:2], v[47:50], off offset:64
+; GFX11-FAKE16-NEXT:    s_clause 0x1f
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v138, off, s32 offset:400
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v137, off, s32 offset:404
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v136, off, s32 offset:408
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v127, off, s32 offset:412
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v126, off, s32 offset:416
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v125, off, s32 offset:420
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v124, off, s32 offset:424
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v123, off, s32 offset:428
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v122, off, s32 offset:432
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v121, off, s32 offset:436
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v120, off, s32 offset:440
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v111, off, s32 offset:444
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v110, off, s32 offset:448
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v109, off, s32 offset:452
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v108, off, s32 offset:456
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v107, off, s32 offset:460
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v106, off, s32 offset:464
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v105, off, s32 offset:468
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v104, off, s32 offset:472
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v95, off, s32 offset:476
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v94, off, s32 offset:480
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v93, off, s32 offset:484
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v92, off, s32 offset:488
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v91, off, s32 offset:492
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v90, off, s32 offset:496
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v89, off, s32 offset:500
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v88, off, s32 offset:504
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v79, off, s32 offset:508
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v78, off, s32 offset:512
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v77, off, s32 offset:516
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v76, off, s32 offset:520
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v75, off, s32 offset:524
+; GFX11-FAKE16-NEXT:    s_clause 0x12
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v74, off, s32 offset:528
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v73, off, s32 offset:532
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v72, off, s32 offset:536
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v63, off, s32 offset:540
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v62, off, s32 offset:544
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v61, off, s32 offset:548
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v60, off, s32 offset:552
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v59, off, s32 offset:556
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v58, off, s32 offset:560
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v57, off, s32 offset:564
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v56, off, s32 offset:568
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v47, off, s32 offset:572
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v46, off, s32 offset:576
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v45, off, s32 offset:580
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v44, off, s32 offset:584
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v43, off, s32 offset:588
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v42, off, s32 offset:592
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v41, off, s32 offset:596
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v40, off, s32 offset:600
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %if, label %end
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdpal.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdpal.ll
index 2e47b0163aa8..e99d5a3a7d65 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdpal.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdpal.ll
@@ -70,7 +70,7 @@ entry:
   store i32 %extra, ptr addrspace(5) %v
   store <2 x i32> %in, ptr addrspace(5) %v1
   %e = getelementptr [2 x i32], ptr addrspace(5) %v1, i32 0, i32 %idx
-  %x = load i32, ptr addrspace(5) %e
+  %x = load volatile i32, ptr addrspace(5) %e
   %xf = bitcast i32 %x to float
   call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %xf, ptr addrspace(8) poison, i32 0, i32 0, i32 0)
   ret void
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll
new file mode 100644
index 000000000000..0718a17dfd06
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll
@@ -0,0 +1,592 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+;; Test reduction of:
+;;
+;;   DST = ashr i64 X, Y
+;;
+;; where Y is in the range [63-32] to:
+;;
+;;   DST = [ashr i32 HI(X), (Y & 0x1F), ashr i32 HI(X), 31]
+
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range with metadata
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define i64 @ashr_metadata(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{}
+  %ashr = ashr i64 %arg0, %shift.amt
+  ret i64 %ashr
+}
+
+define amdgpu_ps i64 @ashr_metadata_sgpr_return(i64 inreg %arg0, ptr addrspace(1) inreg %arg1.ptr) {
+; CHECK-LABEL: ashr_metadata_sgpr_return:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_ashr_i64 s[0:1], s[0:1], s2
+; CHECK-NEXT:    ; return to shader part epilog
+  %shift.amt = load i64, ptr addrspace(1) %arg1.ptr, !range !0, !noundef !{}
+  %ashr = ashr i64 %arg0, %shift.amt
+  ret i64 %ashr
+}
+
+; Exact attribute does not inhibit reduction
+define i64 @ashr_exact_metadata(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_exact_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{}
+  %ashr = ashr exact i64 %arg0, %shift.amt
+  ret i64 %ashr
+}
+
+define i64 @ashr_metadata_two_ranges(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_metadata_two_ranges:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !1, !noundef !{}
+  %ashr = ashr i64 %arg0, %shift.amt
+  ret i64 %ashr
+}
+
+; Known minimum is too low.  Reduction must not be done.
+define i64 @ashr_metadata_out_of_range(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_metadata_out_of_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !2, !noundef !{}
+  %ashr = ashr i64 %arg0, %shift.amt
+  ret i64 %ashr
+}
+
+; Bounds cannot be truncated to i32 when load is narrowed to i32.
+; Reduction must not be done.
+; Bounds were chosen so that if bounds were truncated to i32 the
+; known minimum would be 32 and the ashr would be erroneously reduced.
+define i64 @ashr_metadata_cant_be_narrowed_to_i32(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_metadata_cant_be_narrowed_to_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !3, !noundef !{}
+  %ashr = ashr i64 %arg0, %shift.amt
+  ret i64 %ashr
+}
+
+define <2 x i64> @ashr_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_v2_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v6, v[2:3]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %ashr = ashr <2 x i64> %arg0, %shift.amt
+  ret <2 x i64> %ashr
+}
+
+; Exact attribute does not inhibit reduction
+define <2 x i64> @ashr_exact_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_exact_v2_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v6, v[2:3]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %ashr = ashr exact <2 x i64> %arg0, %shift.amt
+  ret <2 x i64> %ashr
+}
+
+define <3 x i64> @ashr_v3_metadata(<3 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_v3_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v12, v[6:7] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[6:7]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v12, v[4:5]
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v8, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v10, v[2:3]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <3 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %ashr = ashr <3 x i64> %arg0, %shift.amt
+  ret <3 x i64> %ashr
+}
+
+define <4 x i64> @ashr_v4_metadata(<4 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_v4_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[8:9]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[13:16], v[8:9] offset:16
+; CHECK-NEXT:    ; kill: killed $vgpr8 killed $vgpr9
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v10, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v12, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v13, v[4:5]
+; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v15, v[6:7]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <4 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %ashr = ashr <4 x i64> %arg0, %shift.amt
+  ret <4 x i64> %ashr
+}
+
+!0 = !{i64 32, i64 64}
+!1 = !{i64 32, i64 38, i64 42, i64 48}
+!2 = !{i64 31, i64 38, i64 42, i64 48}
+!3 = !{i64 32, i64 38, i64 2147483680, i64 2147483681}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range with an "or X, 16"
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; These cases must not be reduced because the known minimum, 16, is not in range.
+
+define i64 @ashr_or16(i64 %arg0, i64 %shift_amt) {
+; CHECK-LABEL: ashr_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v2, 16, v2
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 16
+  %ashr = ashr i64 %arg0, %or
+  ret i64 %ashr
+}
+
+define <2 x i64> @ashr_v2_or16(<2 x i64> %arg0, <2 x i64> %shift_amt) {
+; CHECK-LABEL: ashr_v2_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v5, 16, v6
+; CHECK-NEXT:    v_or_b32_e32 v4, 16, v4
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v5, v[2:3]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 16)
+  %ashr = ashr <2 x i64> %arg0, %or
+  ret <2 x i64> %ashr
+}
+
+define <3 x i64> @ashr_v3_or16(<3 x i64> %arg0, <3 x i64> %shift_amt) {
+; CHECK-LABEL: ashr_v3_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v7, 16, v10
+; CHECK-NEXT:    v_or_b32_e32 v8, 16, v8
+; CHECK-NEXT:    v_or_b32_e32 v6, 16, v6
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v6, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v8, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v7, v[4:5]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 16)
+  %ashr = ashr <3 x i64> %arg0, %or
+  ret <3 x i64> %ashr
+}
+
+define <4 x i64> @ashr_v4_or16(<4 x i64> %arg0, <4 x i64> %shift_amt) {
+; CHECK-LABEL: ashr_v4_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v9, 16, v14
+; CHECK-NEXT:    v_or_b32_e32 v11, 16, v12
+; CHECK-NEXT:    v_or_b32_e32 v10, 16, v10
+; CHECK-NEXT:    v_or_b32_e32 v8, 16, v8
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v8, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v10, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v11, v[4:5]
+; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v9, v[6:7]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 16)
+  %ashr = ashr <4 x i64> %arg0, %or
+  ret <4 x i64> %ashr
+}
+
+; test SGPR
+
+define i64 @ashr_or16_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: ashr_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s4, s18, 16
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 16
+  %ashr = ashr i64 %arg0, %or
+  ret i64 %ashr
+}
+
+define amdgpu_ps i64 @ashr_or16_sgpr_return(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: ashr_or16_sgpr_return:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_or_b32 s2, s2, 16
+; CHECK-NEXT:    s_ashr_i64 s[0:1], s[0:1], s2
+; CHECK-NEXT:    ; return to shader part epilog
+  %or = or i64 %shift_amt, 16
+  %ashr = ashr i64 %arg0, %or
+  ret i64 %ashr
+}
+
+define <2 x i64> @ashr_v2_or16_sgpr(<2 x i64> inreg %arg0, <2 x i64> inreg %shift_amt) {
+; CHECK-LABEL: ashr_v2_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s6, s22, 16
+; CHECK-NEXT:    s_or_b32 s4, s20, 16
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 16)
+  %ashr = ashr <2 x i64> %arg0, %or
+  ret <2 x i64> %ashr
+}
+
+define <3 x i64> @ashr_v3_or16_sgpr(<3 x i64> inreg %arg0, <3 x i64> inreg %shift_amt) {
+; CHECK-LABEL: ashr_v3_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s8, s26, 16
+; CHECK-NEXT:    s_or_b32 s6, s24, 16
+; CHECK-NEXT:    s_or_b32 s4, s22, 16
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    s_ashr_i64 s[8:9], s[20:21], s8
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_mov_b32_e32 v4, s8
+; CHECK-NEXT:    v_mov_b32_e32 v5, s9
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 16)
+  %ashr = ashr <3 x i64> %arg0, %or
+  ret <3 x i64> %ashr
+}
+
+define <4 x i64> @ashr_v4_or16_sgpr(<4 x i64> inreg %arg0, <4 x i64> inreg %shift_amt) {
+; CHECK-LABEL: ashr_v4_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v0, 16, v0
+; CHECK-NEXT:    s_or_b32 s8, s28, 16
+; CHECK-NEXT:    s_or_b32 s6, s26, 16
+; CHECK-NEXT:    s_or_b32 s4, s24, 16
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    s_ashr_i64 s[8:9], s[20:21], s8
+; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v0, s[22:23]
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_mov_b32_e32 v4, s8
+; CHECK-NEXT:    v_mov_b32_e32 v5, s9
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 16)
+  %ashr = ashr <4 x i64> %arg0, %or
+  ret <4 x i64> %ashr
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range with an "or X, 32"
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; These cases are reduced because computeKnownBits() can calculate a minimum of 32
+; based on the OR with 32.
+
+define i64 @ashr_or32(i64 %arg0, i64 %shift_amt) {
+; CHECK-LABEL: ashr_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v2, 32, v2
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 32
+  %ashr = ashr i64 %arg0, %or
+  ret i64 %ashr
+}
+
+define <2 x i64> @ashr_v2_or32(<2 x i64> %arg0, <2 x i64> %shift_amt) {
+; CHECK-LABEL: ashr_v2_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v5, 32, v6
+; CHECK-NEXT:    v_or_b32_e32 v4, 32, v4
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v5, v[2:3]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 32)
+  %ashr = ashr <2 x i64> %arg0, %or
+  ret <2 x i64> %ashr
+}
+
+define <3 x i64> @ashr_v3_or32(<3 x i64> %arg0, <3 x i64> %shift_amt) {
+; CHECK-LABEL: ashr_v3_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v7, 32, v10
+; CHECK-NEXT:    v_or_b32_e32 v8, 32, v8
+; CHECK-NEXT:    v_or_b32_e32 v6, 32, v6
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v6, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v8, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v7, v[4:5]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 32)
+  %ashr = ashr <3 x i64> %arg0, %or
+  ret <3 x i64> %ashr
+}
+
+define <4 x i64> @ashr_v4_or32(<4 x i64> %arg0, <4 x i64> %shift_amt) {
+; CHECK-LABEL: ashr_v4_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v9, 32, v14
+; CHECK-NEXT:    v_or_b32_e32 v11, 32, v12
+; CHECK-NEXT:    v_or_b32_e32 v10, 32, v10
+; CHECK-NEXT:    v_or_b32_e32 v8, 32, v8
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v8, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v10, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v11, v[4:5]
+; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v9, v[6:7]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 32)
+  %ashr = ashr <4 x i64> %arg0, %or
+  ret <4 x i64> %ashr
+}
+
+; test SGPR
+
+define i64 @ashr_or32_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: ashr_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s4, s18, 32
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 32
+  %ashr = ashr i64 %arg0, %or
+  ret i64 %ashr
+}
+
+define amdgpu_ps i64 @ashr_or32_sgpr_return(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: ashr_or32_sgpr_return:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_or_b32 s2, s2, 32
+; CHECK-NEXT:    s_ashr_i64 s[0:1], s[0:1], s2
+; CHECK-NEXT:    ; return to shader part epilog
+  %or = or i64 %shift_amt, 32
+  %ashr = ashr i64 %arg0, %or
+  ret i64 %ashr
+}
+
+define <2 x i64> @ashr_v2_or32_sgpr(<2 x i64> inreg %arg0, <2 x i64> inreg %shift_amt) {
+; CHECK-LABEL: ashr_v2_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s6, s22, 32
+; CHECK-NEXT:    s_or_b32 s4, s20, 32
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 32)
+  %ashr = ashr <2 x i64> %arg0, %or
+  ret <2 x i64> %ashr
+}
+
+define <3 x i64> @ashr_v3_or32_sgpr(<3 x i64> inreg %arg0, <3 x i64> inreg %shift_amt) {
+; CHECK-LABEL: ashr_v3_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s8, s26, 32
+; CHECK-NEXT:    s_or_b32 s6, s24, 32
+; CHECK-NEXT:    s_or_b32 s4, s22, 32
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    s_ashr_i64 s[8:9], s[20:21], s8
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_mov_b32_e32 v4, s8
+; CHECK-NEXT:    v_mov_b32_e32 v5, s9
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 32)
+  %ashr = ashr <3 x i64> %arg0, %or
+  ret <3 x i64> %ashr
+}
+
+define <4 x i64> @ashr_v4_or32_sgpr(<4 x i64> inreg %arg0, <4 x i64> inreg %shift_amt) {
+; CHECK-LABEL: ashr_v4_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v0, 32, v0
+; CHECK-NEXT:    s_or_b32 s8, s28, 32
+; CHECK-NEXT:    s_or_b32 s6, s26, 32
+; CHECK-NEXT:    s_or_b32 s4, s24, 32
+; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    s_ashr_i64 s[8:9], s[20:21], s8
+; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v0, s[22:23]
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_mov_b32_e32 v4, s8
+; CHECK-NEXT:    v_mov_b32_e32 v5, s9
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 32)
+  %ashr = ashr <4 x i64> %arg0, %or
+  ret <4 x i64> %ashr
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range from max/min
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; FIXME: This case should be reduced too, but computeKnownBits() cannot
+;        determine the range.  Match current results for now.
+
+define i64 @ashr_maxmin(i64 %arg0, i64 noundef %arg1) {
+; CHECK-LABEL: ashr_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, 32, v2, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, 63, v2, vcc
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call i64 @llvm.umax.i64(i64 %arg1, i64 32)
+  %min = call i64 @llvm.umin.i64(i64 %max,  i64 63)
+  %ashr = ashr i64 %arg0, %min
+  ret i64 %ashr
+}
+
+define <2 x i64> @ashr_v2_maxmin(<2 x i64> %arg0, <2 x i64> noundef %arg1) {
+; CHECK-LABEL: ashr_v2_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, 32, v4, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 32, v6, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 63, v6, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[4:5]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v6, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, 63, v4, vcc
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call <2 x i64> @llvm.umax.i64(<2 x i64> %arg1, <2 x i64> splat (i64 32))
+  %min = call <2 x i64> @llvm.umin.i64(<2 x i64> %max,  <2 x i64> splat (i64 63))
+  %ashr = ashr <2 x i64> %arg0, %min
+  ret <2 x i64> %ashr
+}
+
+define <3 x i64> @ashr_v3_maxmin(<3 x i64> %arg0, <3 x i64> noundef %arg1) {
+; CHECK-LABEL: ashr_v3_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 32, v6, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[8:9]
+; CHECK-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 32, v8, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[10:11]
+; CHECK-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 32, v10, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[10:11]
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 63, v10, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[8:9]
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v10, v[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 63, v8, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[6:7]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v8, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 63, v6, vcc
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v6, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call <3 x i64> @llvm.umax.i64(<3 x i64> %arg1, <3 x i64> splat (i64 32))
+  %min = call <3 x i64> @llvm.umin.i64(<3 x i64> %max,  <3 x i64> splat (i64 63))
+  %ashr = ashr <3 x i64> %arg0, %min
+  ret <3 x i64> %ashr
+}
+
+define <4 x i64> @ashr_v4_maxmin(<4 x i64> %arg0, <4 x i64> noundef %arg1) {
+; CHECK-LABEL: ashr_v4_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[8:9]
+; CHECK-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 32, v8, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[10:11]
+; CHECK-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 32, v10, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[12:13]
+; CHECK-NEXT:    v_cndmask_b32_e32 v13, 0, v13, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v12, 32, v12, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[14:15]
+; CHECK-NEXT:    v_cndmask_b32_e32 v15, 0, v15, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v14, 32, v14, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[14:15]
+; CHECK-NEXT:    v_cndmask_b32_e32 v14, 63, v14, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[12:13]
+; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v14, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v12, 63, v12, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[10:11]
+; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v12, v[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 63, v10, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[8:9]
+; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v10, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 63, v8, vcc
+; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v8, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call <4 x i64> @llvm.umax.i64(<4 x i64> %arg1, <4 x i64> splat (i64 32))
+  %min = call <4 x i64> @llvm.umin.i64(<4 x i64> %max,  <4 x i64> splat (i64 63))
+  %ashr = ashr <4 x i64> %arg0, %min
+  ret <4 x i64> %ashr
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
index 1669909e96eb..231f53d7f371 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
@@ -799,6 +799,31 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
 ; GFX90A-LABEL: optnone_atomicrmw_fadd_f64_expand:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    s_mov_b64 s[4:5], src_shared_base
+; GFX90A-NEXT:    s_mov_b32 s6, 32
+; GFX90A-NEXT:    s_lshr_b64 s[4:5], s[4:5], s6
+; GFX90A-NEXT:    s_getpc_b64 s[6:7]
+; GFX90A-NEXT:    s_add_u32 s6, s6, global@rel32@lo+4
+; GFX90A-NEXT:    s_addc_u32 s7, s7, global@rel32@hi+12
+; GFX90A-NEXT:    s_cmp_eq_u32 s7, s4
+; GFX90A-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX90A-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX90A-NEXT:    s_mov_b64 s[4:5], -1
+; GFX90A-NEXT:    s_mov_b32 s6, 1
+; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[6:7], v2, s6
+; GFX90A-NEXT:    s_and_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT:    s_cbranch_vccnz .LBB5_3
+; GFX90A-NEXT:  .LBB5_1: ; %Flow4
+; GFX90A-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX90A-NEXT:    s_mov_b32 s4, 1
+; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[4:5], v4, s4
+; GFX90A-NEXT:    s_and_b64 vcc, exec, s[4:5]
+; GFX90A-NEXT:    s_cbranch_vccnz .LBB5_10
+; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.shared
+; GFX90A-NEXT:    ds_add_rtn_f64 v[2:3], v0, v[0:1]
+; GFX90A-NEXT:    s_branch .LBB5_10
+; GFX90A-NEXT:  .LBB5_3: ; %atomicrmw.check.private
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], src_private_base
 ; GFX90A-NEXT:    s_mov_b32 s6, 32
 ; GFX90A-NEXT:    s_lshr_b64 s[4:5], s[4:5], s6
@@ -813,50 +838,54 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
 ; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[6:7], v2, s6
 ; GFX90A-NEXT:    s_and_b64 vcc, exec, s[6:7]
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT:    s_cbranch_vccnz .LBB5_2
-; GFX90A-NEXT:    s_branch .LBB5_3
-; GFX90A-NEXT:  .LBB5_1: ; %atomicrmw.private
+; GFX90A-NEXT:    s_cbranch_vccnz .LBB5_5
+; GFX90A-NEXT:    s_branch .LBB5_6
+; GFX90A-NEXT:  .LBB5_4: ; %atomicrmw.private
 ; GFX90A-NEXT:    buffer_load_dword v2, v0, s[0:3], 0 offen
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, v2
-; GFX90A-NEXT:    v_add_f64 v[0:1], v[2:3], v[0:1]
-; GFX90A-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
-; GFX90A-NEXT:    buffer_store_dword v0, v0, s[0:3], 0 offen
-; GFX90A-NEXT:    s_branch .LBB5_6
-; GFX90A-NEXT:  .LBB5_2: ; %atomicrmw.global
+; GFX90A-NEXT:    v_add_f64 v[4:5], v[2:3], v[0:1]
+; GFX90A-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen
+; GFX90A-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen
+; GFX90A-NEXT:    s_branch .LBB5_9
+; GFX90A-NEXT:  .LBB5_5: ; %atomicrmw.global
+; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX90A-NEXT:    s_getpc_b64 s[4:5]
 ; GFX90A-NEXT:    s_add_u32 s4, s4, global@rel32@lo+4
 ; GFX90A-NEXT:    s_addc_u32 s5, s5, global@rel32@hi+12
-; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
+; GFX90A-NEXT:    global_load_dwordx2 v[2:3], v2, s[4:5]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
-; GFX90A-NEXT:    s_branch .LBB5_4
-; GFX90A-NEXT:  .LBB5_3: ; %Flow
+; GFX90A-NEXT:    s_branch .LBB5_7
+; GFX90A-NEXT:  .LBB5_6: ; %Flow
 ; GFX90A-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_vccnz .LBB5_1
-; GFX90A-NEXT:    s_branch .LBB5_6
-; GFX90A-NEXT:  .LBB5_4: ; %atomicrmw.start
+; GFX90A-NEXT:    s_cbranch_vccnz .LBB5_4
+; GFX90A-NEXT:    s_branch .LBB5_9
+; GFX90A-NEXT:  .LBB5_7: ; %atomicrmw.start
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    v_add_f64 v[2:3], v[4:5], v[0:1]
 ; GFX90A-NEXT:    s_getpc_b64 s[6:7]
 ; GFX90A-NEXT:    s_add_u32 s6, s6, global@rel32@lo+4
 ; GFX90A-NEXT:    s_addc_u32 s7, s7, global@rel32@hi+12
-; GFX90A-NEXT:    v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v6, 0
+; GFX90A-NEXT:    global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[6:7] glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    v_cmp_eq_u64_e64 s[6:7], v[2:3], v[4:5]
 ; GFX90A-NEXT:    s_or_b64 s[4:5], s[6:7], s[4:5]
 ; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB5_4
-; GFX90A-NEXT:  ; %bb.5: ; %atomicrmw.end1
+; GFX90A-NEXT:    s_cbranch_execnz .LBB5_7
+; GFX90A-NEXT:  ; %bb.8: ; %atomicrmw.end1
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
-; GFX90A-NEXT:    s_branch .LBB5_3
-; GFX90A-NEXT:  .LBB5_6: ; %atomicrmw.phi
-; GFX90A-NEXT:  ; %bb.7: ; %atomicrmw.end
+; GFX90A-NEXT:    s_branch .LBB5_6
+; GFX90A-NEXT:  .LBB5_9: ; %Flow3
+; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
+; GFX90A-NEXT:    s_branch .LBB5_1
+; GFX90A-NEXT:  .LBB5_10: ; %atomicrmw.phi
+; GFX90A-NEXT:  ; %bb.11: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_mov_b32 s4, 32
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_lshrrev_b64 v[4:5], s4, v[2:3]
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
@@ -866,6 +895,31 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
 ; GFX942-LABEL: optnone_atomicrmw_fadd_f64_expand:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX942-NEXT:    s_mov_b32 s2, 32
+; GFX942-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX942-NEXT:    s_getpc_b64 s[2:3]
+; GFX942-NEXT:    s_add_u32 s2, s2, global@rel32@lo+4
+; GFX942-NEXT:    s_addc_u32 s3, s3, global@rel32@hi+12
+; GFX942-NEXT:    s_cmp_eq_u32 s3, s0
+; GFX942-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX942-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX942-NEXT:    s_mov_b64 s[0:1], -1
+; GFX942-NEXT:    s_mov_b32 s2, 1
+; GFX942-NEXT:    v_cmp_ne_u32_e64 s[2:3], v2, s2
+; GFX942-NEXT:    s_and_b64 vcc, exec, s[2:3]
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_cbranch_vccnz .LBB5_3
+; GFX942-NEXT:  .LBB5_1: ; %Flow4
+; GFX942-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
+; GFX942-NEXT:    s_mov_b32 s0, 1
+; GFX942-NEXT:    v_cmp_ne_u32_e64 s[0:1], v4, s0
+; GFX942-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GFX942-NEXT:    s_cbranch_vccnz .LBB5_10
+; GFX942-NEXT:  ; %bb.2: ; %atomicrmw.shared
+; GFX942-NEXT:    ds_add_rtn_f64 v[2:3], v0, v[0:1]
+; GFX942-NEXT:    s_branch .LBB5_10
+; GFX942-NEXT:  .LBB5_3: ; %atomicrmw.check.private
 ; GFX942-NEXT:    s_mov_b64 s[0:1], src_private_base
 ; GFX942-NEXT:    s_mov_b32 s2, 32
 ; GFX942-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
@@ -880,48 +934,52 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
 ; GFX942-NEXT:    v_cmp_ne_u32_e64 s[2:3], v2, s2
 ; GFX942-NEXT:    s_and_b64 vcc, exec, s[2:3]
 ; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX942-NEXT:    s_cbranch_vccnz .LBB5_2
-; GFX942-NEXT:    s_branch .LBB5_3
-; GFX942-NEXT:  .LBB5_1: ; %atomicrmw.private
+; GFX942-NEXT:    s_cbranch_vccnz .LBB5_5
+; GFX942-NEXT:    s_branch .LBB5_6
+; GFX942-NEXT:  .LBB5_4: ; %atomicrmw.private
 ; GFX942-NEXT:    scratch_load_dwordx2 v[2:3], off, s0
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
-; GFX942-NEXT:    v_add_f64 v[0:1], v[2:3], v[0:1]
-; GFX942-NEXT:    scratch_store_dwordx2 off, v[0:1], s0
-; GFX942-NEXT:    s_branch .LBB5_6
-; GFX942-NEXT:  .LBB5_2: ; %atomicrmw.global
+; GFX942-NEXT:    v_add_f64 v[4:5], v[2:3], v[0:1]
+; GFX942-NEXT:    scratch_store_dwordx2 off, v[4:5], s0
+; GFX942-NEXT:    s_branch .LBB5_9
+; GFX942-NEXT:  .LBB5_5: ; %atomicrmw.global
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX942-NEXT:    s_getpc_b64 s[0:1]
 ; GFX942-NEXT:    s_add_u32 s0, s0, global@rel32@lo+4
 ; GFX942-NEXT:    s_addc_u32 s1, s1, global@rel32@hi+12
-; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
+; GFX942-NEXT:    global_load_dwordx2 v[2:3], v2, s[0:1]
 ; GFX942-NEXT:    s_mov_b64 s[0:1], 0
-; GFX942-NEXT:    s_branch .LBB5_4
-; GFX942-NEXT:  .LBB5_3: ; %Flow
+; GFX942-NEXT:    s_branch .LBB5_7
+; GFX942-NEXT:  .LBB5_6: ; %Flow
 ; GFX942-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; GFX942-NEXT:    s_cbranch_vccnz .LBB5_1
-; GFX942-NEXT:    s_branch .LBB5_6
-; GFX942-NEXT:  .LBB5_4: ; %atomicrmw.start
+; GFX942-NEXT:    s_cbranch_vccnz .LBB5_4
+; GFX942-NEXT:    s_branch .LBB5_9
+; GFX942-NEXT:  .LBB5_7: ; %atomicrmw.start
 ; GFX942-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_mov_b64_e32 v[4:5], v[2:3]
 ; GFX942-NEXT:    v_add_f64 v[2:3], v[4:5], v[0:1]
 ; GFX942-NEXT:    s_getpc_b64 s[2:3]
 ; GFX942-NEXT:    s_add_u32 s2, s2, global@rel32@lo+4
 ; GFX942-NEXT:    s_addc_u32 s3, s3, global@rel32@hi+12
-; GFX942-NEXT:    v_mov_b64_e32 v[6:7], s[2:3]
-; GFX942-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] sc0 sc1
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v6, 0
+; GFX942-NEXT:    global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[2:3] sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_cmp_eq_u64_e64 s[2:3], v[2:3], v[4:5]
 ; GFX942-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
 ; GFX942-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT:    s_cbranch_execnz .LBB5_4
-; GFX942-NEXT:  ; %bb.5: ; %atomicrmw.end1
+; GFX942-NEXT:    s_cbranch_execnz .LBB5_7
+; GFX942-NEXT:  ; %bb.8: ; %atomicrmw.end1
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX942-NEXT:    s_mov_b64 s[0:1], 0
-; GFX942-NEXT:    s_branch .LBB5_3
-; GFX942-NEXT:  .LBB5_6: ; %atomicrmw.phi
-; GFX942-NEXT:  ; %bb.7: ; %atomicrmw.end
+; GFX942-NEXT:    s_branch .LBB5_6
+; GFX942-NEXT:  .LBB5_9: ; %Flow3
+; GFX942-NEXT:    s_mov_b64 s[0:1], 0
+; GFX942-NEXT:    s_branch .LBB5_1
+; GFX942-NEXT:  .LBB5_10: ; %atomicrmw.phi
+; GFX942-NEXT:  ; %bb.11: ; %atomicrmw.end
 ; GFX942-NEXT:    s_mov_b32 s0, 32
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    v_lshrrev_b64 v[4:5], s0, v[2:3]
 ; GFX942-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX942-NEXT:    v_mov_b32_e32 v1, v4
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/bf16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/bf16.ll
index e73096770aac..cb835b1f63c2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -5633,12 +5633,11 @@ define { <32 x i32>, bfloat } @test_overflow_stack(bfloat %a, <32 x i32> %b) {
 ; GFX9-NEXT:    buffer_store_dword v28, v0, s[0:3], 0 offen offset:104
 ; GFX9-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:100
 ; GFX9-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:96
-; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:92
-; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v25, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v27, off, s[0:3], s32
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v24, v0, s[0:3], 0 offen offset:88
 ; GFX9-NEXT:    buffer_store_dword v23, v0, s[0:3], 0 offen offset:84
@@ -5664,10 +5663,11 @@ define { <32 x i32>, bfloat } @test_overflow_stack(bfloat %a, <32 x i32> %b) {
 ; GFX9-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
 ; GFX9-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
 ; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:124
+; GFX9-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:124
+; GFX9-NEXT:    s_waitcnt vmcnt(25)
 ; GFX9-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:120
 ; GFX9-NEXT:    s_waitcnt vmcnt(25)
-; GFX9-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:116
+; GFX9-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:116
 ; GFX9-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen offset:128
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -8479,30 +8479,30 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
 ; GFX9-LABEL: global_extload_v32bf16_to_v32f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_ushort v8, v[1:2], off offset:62
-; GFX9-NEXT:    global_load_ushort v10, v[1:2], off offset:60
-; GFX9-NEXT:    global_load_ushort v11, v[1:2], off offset:58
-; GFX9-NEXT:    global_load_ushort v12, v[1:2], off offset:56
-; GFX9-NEXT:    global_load_ushort v13, v[1:2], off offset:54
-; GFX9-NEXT:    global_load_ushort v14, v[1:2], off offset:52
-; GFX9-NEXT:    global_load_ushort v15, v[1:2], off offset:50
-; GFX9-NEXT:    global_load_ushort v16, v[1:2], off offset:48
-; GFX9-NEXT:    global_load_ushort v17, v[1:2], off offset:46
-; GFX9-NEXT:    global_load_ushort v18, v[1:2], off offset:44
-; GFX9-NEXT:    global_load_ushort v19, v[1:2], off offset:42
-; GFX9-NEXT:    global_load_ushort v20, v[1:2], off offset:40
-; GFX9-NEXT:    global_load_ushort v21, v[1:2], off offset:38
-; GFX9-NEXT:    global_load_ushort v22, v[1:2], off offset:36
-; GFX9-NEXT:    global_load_ushort v23, v[1:2], off offset:34
-; GFX9-NEXT:    global_load_ushort v24, v[1:2], off offset:32
-; GFX9-NEXT:    global_load_ushort v25, v[1:2], off
-; GFX9-NEXT:    global_load_ushort v26, v[1:2], off offset:2
-; GFX9-NEXT:    global_load_ushort v27, v[1:2], off offset:30
+; GFX9-NEXT:    global_load_ushort v9, v[1:2], off offset:62
+; GFX9-NEXT:    global_load_ushort v11, v[1:2], off offset:60
+; GFX9-NEXT:    global_load_ushort v12, v[1:2], off offset:58
+; GFX9-NEXT:    global_load_ushort v13, v[1:2], off offset:56
+; GFX9-NEXT:    global_load_ushort v14, v[1:2], off offset:54
+; GFX9-NEXT:    global_load_ushort v15, v[1:2], off offset:52
+; GFX9-NEXT:    global_load_ushort v16, v[1:2], off offset:50
+; GFX9-NEXT:    global_load_ushort v17, v[1:2], off offset:48
+; GFX9-NEXT:    global_load_ushort v18, v[1:2], off offset:46
+; GFX9-NEXT:    global_load_ushort v19, v[1:2], off offset:44
+; GFX9-NEXT:    global_load_ushort v20, v[1:2], off offset:42
+; GFX9-NEXT:    global_load_ushort v21, v[1:2], off offset:40
+; GFX9-NEXT:    global_load_ushort v22, v[1:2], off offset:38
+; GFX9-NEXT:    global_load_ushort v23, v[1:2], off offset:36
+; GFX9-NEXT:    global_load_ushort v24, v[1:2], off offset:34
+; GFX9-NEXT:    global_load_ushort v25, v[1:2], off offset:32
+; GFX9-NEXT:    global_load_ushort v26, v[1:2], off
+; GFX9-NEXT:    global_load_ushort v27, v[1:2], off offset:2
 ; GFX9-NEXT:    global_load_ushort v3, v[1:2], off offset:16
 ; GFX9-NEXT:    global_load_ushort v4, v[1:2], off offset:18
 ; GFX9-NEXT:    global_load_ushort v5, v[1:2], off offset:20
 ; GFX9-NEXT:    global_load_ushort v6, v[1:2], off offset:22
-; GFX9-NEXT:    global_load_ushort v28, v[1:2], off offset:24
+; GFX9-NEXT:    global_load_ushort v8, v[1:2], off offset:24
+; GFX9-NEXT:    global_load_ushort v28, v[1:2], off offset:30
 ; GFX9-NEXT:    global_load_ushort v29, v[1:2], off offset:26
 ; GFX9-NEXT:    global_load_ushort v30, v[1:2], off offset:28
 ; GFX9-NEXT:    global_load_ushort v31, v[1:2], off offset:4
@@ -8513,122 +8513,122 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    global_load_ushort v1, v[1:2], off offset:14
 ; GFX9-NEXT:    s_waitcnt vmcnt(31)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v8
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v9
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(30)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v10
-; GFX9-NEXT:    s_waitcnt vmcnt(28)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:252
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:248
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v11
-; GFX9-NEXT:    s_waitcnt vmcnt(29)
+; GFX9-NEXT:    s_waitcnt vmcnt(28)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v11, 16, v13
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:244
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:240
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v2
-; GFX9-NEXT:    s_waitcnt vmcnt(30)
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:252
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:248
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v12
+; GFX9-NEXT:    s_waitcnt vmcnt(29)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v12, 16, v14
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:236
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:232
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v10
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[10:11], v11
-; GFX9-NEXT:    s_waitcnt vmcnt(31)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:244
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:240
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(30)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v14, 16, v16
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:228
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:224
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v12
+; GFX9-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:236
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:232
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v11
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[11:12], v12
 ; GFX9-NEXT:    s_waitcnt vmcnt(31)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v14, 16, v16
+; GFX9-NEXT:    s_waitcnt vmcnt(30)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v15, 16, v17
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[12:13], v13
-; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:220
-; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:216
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[10:11], v14
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[14:15], v15
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:228
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:224
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v13
+; GFX9-NEXT:    s_waitcnt vmcnt(31)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v16, 16, v18
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[13:14], v14
+; GFX9-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:220
+; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:216
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[11:12], v15
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[15:16], v16
 ; GFX9-NEXT:    s_waitcnt vmcnt(32)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v18
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v19
 ; GFX9-NEXT:    s_waitcnt vmcnt(30)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v18, 16, v20
+; GFX9-NEXT:    v_lshlrev_b32_e32 v19, 16, v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(28)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:212
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:208
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v20, 16, v22
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:212
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:208
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v16, 16, v19
-; GFX9-NEXT:    v_lshlrev_b32_e32 v19, 16, v21
-; GFX9-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:204
-; GFX9-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:200
-; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:196
-; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:192
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[10:11], v20
+; GFX9-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:204
+; GFX9-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:200
+; GFX9-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:196
+; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:192
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[11:12], v21
 ; GFX9-NEXT:    s_waitcnt vmcnt(33)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v23
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[16:17], v16
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[12:13], v18
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[18:19], v19
-; GFX9-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:188
-; GFX9-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:184
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:180
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:176
-; GFX9-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:172
-; GFX9-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:168
-; GFX9-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:164
-; GFX9-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:160
-; GFX9-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:156
-; GFX9-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:152
-; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:148
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v2
-; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:144
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v24
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[17:18], v17
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[13:14], v19
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[19:20], v20
+; GFX9-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:188
+; GFX9-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:184
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:180
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:176
+; GFX9-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:172
+; GFX9-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:168
+; GFX9-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:164
+; GFX9-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:160
+; GFX9-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:156
+; GFX9-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:152
+; GFX9-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:148
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v2
+; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:144
 ; GFX9-NEXT:    s_waitcnt vmcnt(44)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v10, 16, v24
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:140
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:136
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v10
-; GFX9-NEXT:    s_waitcnt vmcnt(43)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v12, 16, v27
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:132
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:128
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v12
+; GFX9-NEXT:    v_lshlrev_b32_e32 v11, 16, v25
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:140
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:136
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v11
+; GFX9-NEXT:    s_waitcnt vmcnt(38)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v13, 16, v28
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:132
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:128
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v13
 ; GFX9-NEXT:    s_waitcnt vmcnt(38)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v14, 16, v30
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:124
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:120
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v14
-; GFX9-NEXT:    v_lshlrev_b32_e32 v16, 16, v29
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:116
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:112
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v16
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v25
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[10:11], v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v15, 16, v30
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:124
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:120
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v15
+; GFX9-NEXT:    v_lshlrev_b32_e32 v17, 16, v29
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:116
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:112
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[9:10], v17
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v26
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[12:13], v2
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[11:12], v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v27
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[13:14], v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(41)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v31
-; GFX9-NEXT:    v_lshlrev_b32_e32 v18, 16, v28
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[14:15], v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[15:16], v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(40)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v32
-; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:108
-; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:104
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v18
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[16:17], v2
+; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:108
+; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:104
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v8
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[17:18], v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(41)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v33
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[18:19], v2
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[19:20], v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(40)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v34
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[20:21], v2
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[21:22], v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; GFX9-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:100
 ; GFX9-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:96
 ; GFX9-NEXT:    v_cvt_f64_f32_e32 v[8:9], v6
 ; GFX9-NEXT:    v_cvt_f64_f32_e32 v[5:6], v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(41)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v22, 16, v7
+; GFX9-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
 ; GFX9-NEXT:    s_waitcnt vmcnt(40)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
@@ -8642,25 +8642,25 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
 ; GFX9-NEXT:    v_cvt_f64_f32_e32 v[6:7], v3
 ; GFX9-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
-; GFX9-NEXT:    v_cvt_f64_f32_e32 v[1:2], v22
+; GFX9-NEXT:    v_cvt_f64_f32_e32 v[1:2], v10
 ; GFX9-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:68
 ; GFX9-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:64
 ; GFX9-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:60
 ; GFX9-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:56
 ; GFX9-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:52
 ; GFX9-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
-; GFX9-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen offset:44
-; GFX9-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:40
-; GFX9-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:36
-; GFX9-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:32
-; GFX9-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:28
-; GFX9-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:24
-; GFX9-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:20
-; GFX9-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:16
-; GFX9-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:12
-; GFX9-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:8
-; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:4
-; GFX9-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen
+; GFX9-NEXT:    buffer_store_dword v22, v0, s[0:3], 0 offen offset:44
+; GFX9-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen offset:40
+; GFX9-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:36
+; GFX9-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:32
+; GFX9-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:28
+; GFX9-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:24
+; GFX9-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:20
+; GFX9-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:16
+; GFX9-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:12
+; GFX9-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:8
+; GFX9-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8861,72 +8861,72 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1f
-; GFX11-NEXT:    global_load_u16 v3, v[1:2], off offset:12
-; GFX11-NEXT:    global_load_u16 v4, v[1:2], off offset:8
-; GFX11-NEXT:    global_load_u16 v5, v[1:2], off offset:4
-; GFX11-NEXT:    global_load_u16 v6, v[1:2], off offset:2
+; GFX11-NEXT:    global_load_u16 v3, v[1:2], off offset:2
+; GFX11-NEXT:    global_load_u16 v4, v[1:2], off offset:12
+; GFX11-NEXT:    global_load_u16 v5, v[1:2], off offset:8
+; GFX11-NEXT:    global_load_u16 v6, v[1:2], off offset:4
 ; GFX11-NEXT:    global_load_u16 v7, v[1:2], off
 ; GFX11-NEXT:    global_load_u16 v8, v[1:2], off offset:6
 ; GFX11-NEXT:    global_load_u16 v9, v[1:2], off offset:10
 ; GFX11-NEXT:    global_load_u16 v10, v[1:2], off offset:14
-; GFX11-NEXT:    global_load_u16 v11, v[1:2], off offset:28
-; GFX11-NEXT:    global_load_u16 v12, v[1:2], off offset:24
-; GFX11-NEXT:    global_load_u16 v13, v[1:2], off offset:20
-; GFX11-NEXT:    global_load_u16 v14, v[1:2], off offset:18
+; GFX11-NEXT:    global_load_u16 v11, v[1:2], off offset:18
+; GFX11-NEXT:    global_load_u16 v12, v[1:2], off offset:28
+; GFX11-NEXT:    global_load_u16 v13, v[1:2], off offset:24
+; GFX11-NEXT:    global_load_u16 v14, v[1:2], off offset:20
 ; GFX11-NEXT:    global_load_u16 v15, v[1:2], off offset:16
 ; GFX11-NEXT:    global_load_u16 v16, v[1:2], off offset:22
 ; GFX11-NEXT:    global_load_u16 v17, v[1:2], off offset:26
 ; GFX11-NEXT:    global_load_u16 v18, v[1:2], off offset:30
-; GFX11-NEXT:    global_load_u16 v19, v[1:2], off offset:44
-; GFX11-NEXT:    global_load_u16 v20, v[1:2], off offset:40
-; GFX11-NEXT:    global_load_u16 v21, v[1:2], off offset:36
-; GFX11-NEXT:    global_load_u16 v22, v[1:2], off offset:34
+; GFX11-NEXT:    global_load_u16 v19, v[1:2], off offset:34
+; GFX11-NEXT:    global_load_u16 v20, v[1:2], off offset:44
+; GFX11-NEXT:    global_load_u16 v21, v[1:2], off offset:40
+; GFX11-NEXT:    global_load_u16 v22, v[1:2], off offset:36
 ; GFX11-NEXT:    global_load_u16 v23, v[1:2], off offset:32
 ; GFX11-NEXT:    global_load_u16 v24, v[1:2], off offset:38
 ; GFX11-NEXT:    global_load_u16 v25, v[1:2], off offset:42
 ; GFX11-NEXT:    global_load_u16 v26, v[1:2], off offset:46
-; GFX11-NEXT:    global_load_u16 v27, v[1:2], off offset:60
-; GFX11-NEXT:    global_load_u16 v28, v[1:2], off offset:56
-; GFX11-NEXT:    global_load_u16 v29, v[1:2], off offset:52
-; GFX11-NEXT:    global_load_u16 v30, v[1:2], off offset:50
+; GFX11-NEXT:    global_load_u16 v27, v[1:2], off offset:50
+; GFX11-NEXT:    global_load_u16 v28, v[1:2], off offset:60
+; GFX11-NEXT:    global_load_u16 v29, v[1:2], off offset:56
+; GFX11-NEXT:    global_load_u16 v30, v[1:2], off offset:52
 ; GFX11-NEXT:    global_load_u16 v31, v[1:2], off offset:48
 ; GFX11-NEXT:    global_load_u16 v32, v[1:2], off offset:54
 ; GFX11-NEXT:    global_load_u16 v33, v[1:2], off offset:58
 ; GFX11-NEXT:    global_load_u16 v1, v[1:2], off offset:62
 ; GFX11-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v39, 16, v3
+; GFX11-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX11-NEXT:    s_waitcnt vmcnt(30)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v38, 16, v4
 ; GFX11-NEXT:    s_waitcnt vmcnt(29)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX11-NEXT:    s_waitcnt vmcnt(28)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-NEXT:    s_waitcnt vmcnt(27)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v37, 16, v7
 ; GFX11-NEXT:    s_waitcnt vmcnt(26)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 16, v8
+; GFX11-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX11-NEXT:    s_waitcnt vmcnt(25)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX11-NEXT:    s_waitcnt vmcnt(24)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX11-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v102, 16, v11
+; GFX11-NEXT:    v_lshlrev_b32_e32 v34, 16, v11
 ; GFX11-NEXT:    s_waitcnt vmcnt(22)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v101, 16, v12
+; GFX11-NEXT:    v_lshlrev_b32_e32 v100, 16, v12
 ; GFX11-NEXT:    s_waitcnt vmcnt(21)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX11-NEXT:    s_waitcnt vmcnt(20)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; GFX11-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v100, 16, v15
+; GFX11-NEXT:    v_lshlrev_b32_e32 v39, 16, v15
 ; GFX11-NEXT:    s_waitcnt vmcnt(18)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v34, 16, v16
+; GFX11-NEXT:    v_lshlrev_b32_e32 v35, 16, v16
 ; GFX11-NEXT:    s_waitcnt vmcnt(17)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX11-NEXT:    s_waitcnt vmcnt(16)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX11-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v52, 16, v19
+; GFX11-NEXT:    v_lshlrev_b32_e32 v36, 16, v19
 ; GFX11-NEXT:    s_waitcnt vmcnt(14)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v49, 16, v20
 ; GFX11-NEXT:    s_waitcnt vmcnt(13)
@@ -8934,7 +8934,7 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(12)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX11-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v103, 16, v23
+; GFX11-NEXT:    v_lshlrev_b32_e32 v101, 16, v23
 ; GFX11-NEXT:    s_waitcnt vmcnt(10)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v48, 16, v24
 ; GFX11-NEXT:    s_waitcnt vmcnt(9)
@@ -8942,7 +8942,7 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(8)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
 ; GFX11-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-NEXT:    v_lshlrev_b32_e32 v68, 16, v27
+; GFX11-NEXT:    v_lshlrev_b32_e32 v52, 16, v27
 ; GFX11-NEXT:    s_waitcnt vmcnt(6)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v65, 16, v28
 ; GFX11-NEXT:    s_waitcnt vmcnt(5)
@@ -8957,36 +8957,36 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[96:97], v68
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[84:85], v65
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[96:97], v65
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[84:85], v29
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[82:83], v64
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[86:87], v33
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[98:99], v1
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[80:81], v29
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[70:71], v30
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[80:81], v30
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[70:71], v52
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[68:69], v53
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[66:67], v26
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[64:65], v52
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[64:65], v49
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[54:55], v25
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[52:53], v49
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[52:53], v21
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[50:51], v48
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[48:49], v21
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[23:24], v34
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[35:36], v22
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[33:34], v103
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[48:49], v22
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[19:20], v34
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[23:24], v35
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[35:36], v36
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[33:34], v101
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[31:32], v18
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[29:30], v102
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[29:30], v100
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[27:28], v17
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[25:26], v101
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[21:22], v13
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[19:20], v14
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[17:18], v100
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[25:26], v13
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[21:22], v14
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[17:18], v39
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[15:16], v10
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[13:14], v39
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[13:14], v38
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[11:12], v9
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[9:10], v38
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[7:8], v6
-; GFX11-NEXT:    v_cvt_f64_f32_e32 v[5:6], v5
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[9:10], v5
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[7:8], v7
+; GFX11-NEXT:    v_cvt_f64_f32_e32 v[5:6], v6
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[3:4], v2
 ; GFX11-NEXT:    v_cvt_f64_f32_e32 v[1:2], v37
 ; GFX11-NEXT:    s_clause 0xf
@@ -30725,6 +30725,954 @@ define i1 @v_fcmp_true_bf16(bfloat %a, bfloat %b) {
   ret i1 %op
 }
 
+declare bfloat @llvm.copysign.bf16(bfloat, bfloat)
+
+define bfloat @v_copysign_bf16_bf16(bfloat %mag, bfloat %sign) {
+; GCN-LABEL: v_copysign_bf16_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_bf16_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_bf16_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_bf16_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_bf16_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_bf16_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  ret bfloat %op
+}
+
+define bfloat @v_copysign_bf16_s_bf16(bfloat %mag, bfloat inreg %sign) {
+; GCN-LABEL: v_copysign_bf16_s_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    s_and_b32 s4, s16, 0x80000000
+; GCN-NEXT:    s_lshr_b32 s4, s4, 16
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, s4, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_bf16_s_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    s_and_b32 s4, s16, 0x80000000
+; GFX7-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_bf16_s_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s16
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_bf16_s_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s16
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_bf16_s_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, s16
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_bf16_s_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  ret bfloat %op
+}
+
+define bfloat @v_copysign_s_bf16_bf16(bfloat inreg %mag, bfloat %sign) {
+; GCN-LABEL: v_copysign_s_bf16_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e64 v1, 1.0, s16
+; GCN-NEXT:    v_and_b32_e32 v0, 0x80000000, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_bfe_u32 v1, v1, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_s_bf16_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e64 v1, 1.0, s16
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x80000000, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    v_bfe_u32 v1, v1, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_s_bf16_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s16
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_s_bf16_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s16
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_s_bf16_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s16, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_s_bf16_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  ret bfloat %op
+}
+
+define bfloat @v_copysign_bf16_f32(bfloat %mag, float %sign.f32) {
+; GCN-LABEL: v_copysign_bf16_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_bf16_f32:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_bf16_f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_bf16_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_bf16_f32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_bf16_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %sign = fptrunc float %sign.f32 to bfloat
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  ret bfloat %op
+}
+
+define bfloat @v_copysign_bf16_f64(bfloat %mag, double %sign.f64) {
+; GCN-LABEL: v_copysign_bf16_f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0x80000000, v2
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_bf16_f64:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x80000000, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_bf16_f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_bf16_f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_bf16_f64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_bf16_f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %sign = fptrunc double %sign.f64 to bfloat
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  ret bfloat %op
+}
+
+define bfloat @v_copysign_bf16_f16(bfloat %mag, half %sign.f16) {
+; GCN-LABEL: v_copysign_bf16_f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GCN-NEXT:    v_and_b32_e32 v1, 0x8000, v1
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_bf16_f16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x8000, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_bf16_f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_bf16_f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_bf16_f16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_bf16_f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %sign = bitcast half %sign.f16 to bfloat
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  ret bfloat %op
+}
+
+define amdgpu_ps i32 @s_copysign_bf16_bf16(bfloat inreg %mag, bfloat inreg %sign) {
+; GCN-LABEL: s_copysign_bf16_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT:    s_and_b32 s0, s1, 0x80000000
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_copysign_bf16_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT:    s_and_b32 s0, s1, 0x80000000
+; GFX7-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_copysign_bf16_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_copysign_bf16_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_copysign_bf16_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_copysign_bf16_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_mov_b32_e32 v0, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  %cast = bitcast bfloat %op to i16
+  %zext = zext i16 %cast to i32
+  %readlane = call i32 @llvm.amdgcn.readfirstlane(i32 %zext)
+  ret i32 %readlane
+}
+
+define amdgpu_ps i32 @s_copysign_bf16_f32(bfloat inreg %mag, float inreg %sign.f32) {
+; GCN-LABEL: s_copysign_bf16_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT:    s_and_b32 s0, s1, 0x80000000
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_copysign_bf16_f32:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT:    s_and_b32 s0, s1, 0x80000000
+; GFX7-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_copysign_bf16_f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
+; GFX8-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_bfi_b32 v0, s1, v1, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_copysign_bf16_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_bfi_b32 v0, s1, v1, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_copysign_bf16_f32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_copysign_bf16_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %sign = fptrunc float %sign.f32 to bfloat
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  %cast = bitcast bfloat %op to i16
+  %zext = zext i16 %cast to i32
+  %readlane = call i32 @llvm.amdgcn.readfirstlane(i32 %zext)
+  ret i32 %readlane
+}
+
+define amdgpu_ps i32 @s_copysign_bf16_f64(bfloat inreg %mag, double inreg %sign.f64) {
+; GCN-LABEL: s_copysign_bf16_f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT:    s_and_b32 s0, s2, 0x80000000
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_copysign_bf16_f64:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT:    s_and_b32 s0, s2, 0x80000000
+; GFX7-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_copysign_bf16_f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
+; GFX8-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_bfi_b32 v0, s1, v1, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_copysign_bf16_f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_bfi_b32 v0, s1, v1, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_copysign_bf16_f64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_copysign_bf16_f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %sign = fptrunc double %sign.f64 to bfloat
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  %cast = bitcast bfloat %op to i16
+  %zext = zext i16 %cast to i32
+  %readlane = call i32 @llvm.amdgcn.readfirstlane(i32 %zext)
+  ret i32 %readlane
+}
+
+define amdgpu_ps i32 @s_copysign_bf16_f16(bfloat inreg %mag, half inreg %sign.f16) {
+; GCN-LABEL: s_copysign_bf16_f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, s1
+; GCN-NEXT:    v_and_b32_e32 v1, 0x8000, v1
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_copysign_bf16_f16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, s1
+; GFX7-NEXT:    v_mul_f32_e64 v1, 1.0, s0
+; GFX7-NEXT:    v_bfe_u32 v1, v1, 16, 15
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x8000, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_copysign_bf16_f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_copysign_bf16_f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_copysign_bf16_f16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_copysign_bf16_f16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_mov_b32_e32 v0, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %sign = bitcast half %sign.f16 to bfloat
+  %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
+  %cast = bitcast bfloat %op to i16
+  %zext = zext i16 %cast to i32
+  %readlane = call i32 @llvm.amdgcn.readfirstlane(i32 %zext)
+  ret i32 %readlane
+}
+
+declare float @llvm.copysign.f32(float, float)
+
+define float @v_copysign_f32_bf16(float %mag, bfloat %sign.bf16) {
+; GCN-LABEL: v_copysign_f32_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_brev_b32 s4, -2
+; GCN-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_f32_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_brev_b32 s4, -2
+; GFX7-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_f32_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_f32_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_f32_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_f32_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %sign = fpext bfloat %sign.bf16 to float
+  %op = call float @llvm.copysign.f32(float %mag, float %sign)
+  ret float %op
+}
+
+define amdgpu_ps i32 @s_copysign_f32_bf16(float inreg %mag, bfloat inreg %sign.bf16) {
+; GCN-LABEL: s_copysign_f32_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_brev_b32 s2, -2
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_copysign_f32_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_brev_b32 s2, -2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_copysign_f32_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshlrev_b32_e64 v0, 16, s1
+; GFX8-NEXT:    s_brev_b32 s1, -2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_bfi_b32 v0, s1, v1, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_copysign_f32_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_lshlrev_b32_e64 v0, 16, s1
+; GFX9-NEXT:    s_brev_b32 s1, -2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_bfi_b32 v0, s1, v1, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_copysign_f32_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshlrev_b32_e64 v0, 16, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fffffff, s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_copysign_f32_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_lshlrev_b32_e64 v0, 16, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fffffff, s0, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %sign = fpext bfloat %sign.bf16 to float
+  %op = call float @llvm.copysign.f32(float %mag, float %sign)
+  %cast = bitcast float %op to i32
+  %readlane = call i32 @llvm.amdgcn.readfirstlane(i32 %cast)
+  ret i32 %readlane
+}
+
+declare half @llvm.copysign.f16(half, half)
+
+define half @v_copysign_f16_bf16(half %mag, bfloat %sign.bf16) {
+; GCN-LABEL: v_copysign_f16_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    s_brev_b32 s4, -2
+; GCN-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_f16_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    s_brev_b32 s4, -2
+; GFX7-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_f16_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_f16_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_f16_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_f16_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %sign = bitcast bfloat %sign.bf16 to half
+  %op = call half @llvm.copysign.f16(half %mag, half %sign)
+  ret half %op
+}
+
+define amdgpu_ps i32 @s_copysign_f16_bf16(half inreg %mag, bfloat inreg %sign.bf16) {
+; GCN-LABEL: s_copysign_f16_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mul_f32_e64 v0, 1.0, s1
+; GCN-NEXT:    v_cvt_f16_f32_e32 v1, s0
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GCN-NEXT:    s_brev_b32 s0, -2
+; GCN-NEXT:    v_bfi_b32 v0, s0, v1, v0
+; GCN-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_copysign_f16_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, s0
+; GFX7-NEXT:    v_mul_f32_e64 v1, 1.0, s1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    s_brev_b32 s0, -2
+; GFX7-NEXT:    v_bfi_b32 v0, s0, v0, v1
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_copysign_f16_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_copysign_f16_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_bfi_b32 v0, s2, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_copysign_f16_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_copysign_f16_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_mov_b32_e32 v0, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %sign = bitcast bfloat %sign.bf16 to half
+  %op = call half @llvm.copysign.f16(half %mag, half %sign)
+  %cast = bitcast half %op to i16
+  %zext = zext i16 %cast to i32
+  %readlane = call i32 @llvm.amdgcn.readfirstlane(i32 %zext)
+  ret i32 %readlane
+}
+
+declare double @llvm.copysign.f64(double, double)
+
+define double @v_copysign_f64_bf16(double %mag, bfloat %sign.bf16) {
+; GCN-LABEL: v_copysign_f64_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_brev_b32 s4, -2
+; GCN-NEXT:    v_bfi_b32 v1, s4, v1, v2
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_copysign_f64_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_brev_b32 s4, -2
+; GFX7-NEXT:    v_bfi_b32 v1, s4, v1, v2
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_copysign_f64_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_bfi_b32 v1, s4, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_copysign_f64_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_bfi_b32 v1, s4, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_copysign_f64_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    v_bfi_b32 v1, 0x7fffffff, v1, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_copysign_f64_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v1, 0x7fffffff, v1, v2
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %sign = fpext bfloat %sign.bf16 to double
+  %op = call double @llvm.copysign.f64(double %mag, double %sign)
+  ret double %op
+}
+
+define amdgpu_ps <2 x i32> @s_copysign_f64_bf16(double inreg %mag, bfloat inreg %sign.bf16) {
+; GCN-LABEL: s_copysign_f64_bf16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_brev_b32 s3, -2
+; GCN-NEXT:    v_mov_b32_e32 v0, s1
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    v_bfi_b32 v0, s3, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s1, v0
+; GCN-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_copysign_f64_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_brev_b32 s3, -2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_bfi_b32 v0, s3, v0, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_copysign_f64_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshlrev_b32_e64 v0, 16, s2
+; GFX8-NEXT:    s_brev_b32 s2, -2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_bfi_b32 v0, s2, v1, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_copysign_f64_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_lshlrev_b32_e64 v0, 16, s2
+; GFX9-NEXT:    s_brev_b32 s2, -2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_bfi_b32 v0, s2, v1, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_copysign_f64_bf16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshlrev_b32_e64 v0, 16, s2
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fffffff, s1, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_copysign_f64_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_lshlrev_b32_e64 v0, 16, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fffffff, s1, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX11-NEXT:    ; return to shader part epilog
+  %sign = fpext bfloat %sign.bf16 to double
+  %op = call double @llvm.copysign.f64(double %mag, double %sign)
+  %cast = bitcast double %op to <2 x i32>
+  %cast.0 = extractelement <2 x i32> %cast, i32 0
+  %cast.1 = extractelement <2 x i32> %cast, i32 1
+  %readlane0 = call i32 @llvm.amdgcn.readfirstlane(i32 %cast.0)
+  %readlane1 = call i32 @llvm.amdgcn.readfirstlane(i32 %cast.1)
+  %ins.0 = insertelement <2 x i32> poison, i32 %readlane0, i32 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %readlane1, i32 1
+  ret <2 x i32> %ins.1
+}
+
 define i16 @v_fptosi_bf16_to_i16(bfloat %x) {
 ; GCN-LABEL: v_fptosi_bf16_to_i16:
 ; GCN:       ; %bb.0:
@@ -44505,15 +45453,14 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
 ; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11TRUE16-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11TRUE16-NEXT:    v_dual_cndmask_b32 v0, v9, v11 :: v_dual_and_b32 v1, 0xffff0000, v1
 ; GFX11TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v9, v11, vcc_lo
 ; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_dual_add_f32 v1, v1, v5 :: v_dual_add_f32 v2, v6, v8
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT:    v_dual_add_f32 v1, v1, v5 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT:    v_add_f32_e32 v2, v6, v8
 ; GFX11TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11TRUE16-NEXT:    v_bfe_u32 v5, v1, 16, 1
 ; GFX11TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v1
 ; GFX11TRUE16-NEXT:    v_dual_cndmask_b32 v3, v7, v6 :: v_dual_lshlrev_b32 v6, 16, v4
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/bfi_int.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/bfi_int.ll
index 201b97d479c6..b372dec38334 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/bfi_int.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/bfi_int.ll
@@ -16,9 +16,9 @@ define amdgpu_kernel void @s_bfi_def_i32(ptr addrspace(1) %out, i32 %x, i32 %y,
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_andn2_b32 s2, s2, s0
+; GFX7-NEXT:    s_xor_b32 s1, s1, s2
 ; GFX7-NEXT:    s_and_b32 s0, s1, s0
-; GFX7-NEXT:    s_or_b32 s0, s2, s0
+; GFX7-NEXT:    s_xor_b32 s0, s0, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX7-NEXT:    s_endpgm
@@ -28,9 +28,9 @@ define amdgpu_kernel void @s_bfi_def_i32(ptr addrspace(1) %out, i32 %x, i32 %y,
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_andn2_b32 s2, s2, s0
+; GFX8-NEXT:    s_xor_b32 s1, s1, s2
 ; GFX8-NEXT:    s_and_b32 s0, s1, s0
-; GFX8-NEXT:    s_or_b32 s0, s2, s0
+; GFX8-NEXT:    s_xor_b32 s0, s0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s0
@@ -44,9 +44,9 @@ define amdgpu_kernel void @s_bfi_def_i32(ptr addrspace(1) %out, i32 %x, i32 %y,
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_andn2_b32 s2, s2, s0
+; GFX10-NEXT:    s_xor_b32 s1, s1, s2
 ; GFX10-NEXT:    s_and_b32 s0, s1, s0
-; GFX10-NEXT:    s_or_b32 s0, s2, s0
+; GFX10-NEXT:    s_xor_b32 s0, s0, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-NEXT:    global_store_dword v0, v1, s[4:5]
 ; GFX10-NEXT:    s_endpgm
@@ -1407,9 +1407,9 @@ define amdgpu_kernel void @s_bitselect_i64_pat_0(i64 %a, i64 %b, i64 %mask) {
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s6, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b64 s[2:3], s[0:1], s[2:3]
-; GFX7-NEXT:    s_andn2_b64 s[0:1], s[4:5], s[0:1]
-; GFX7-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX7-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GFX7-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX7-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7-NEXT:    s_add_u32 s0, s0, 10
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
@@ -1422,9 +1422,9 @@ define amdgpu_kernel void @s_bitselect_i64_pat_0(i64 %a, i64 %b, i64 %mask) {
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b64 s[2:3], s[0:1], s[2:3]
-; GFX8-NEXT:    s_andn2_b64 s[0:1], s[4:5], s[0:1]
-; GFX8-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
 ; GFX8-NEXT:    s_add_u32 s0, s0, 10
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
@@ -1438,9 +1438,9 @@ define amdgpu_kernel void @s_bitselect_i64_pat_0(i64 %a, i64 %b, i64 %mask) {
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_and_b64 s[2:3], s[0:1], s[2:3]
-; GFX10-NEXT:    s_andn2_b64 s[0:1], s[4:5], s[0:1]
-; GFX10-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
 ; GFX10-NEXT:    s_add_u32 s0, s0, 10
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
new file mode 100644
index 000000000000..ad26dfa7f93e
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck %s
+
+; Check for situations where we could reduce the width of bitwise
+; operations.
+
+
+; Should be able to reduce this to a 32-bit or plus a copy
+; https://alive2.llvm.org/ce/z/9LddFX
+define i64 @v_xor_i64_known_hi_i32_from_arg_range(i64 range(i64 0, 4294967296) %arg0, i64 %arg1) {
+; CHECK-LABEL: v_xor_i64_known_hi_i32_from_arg_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_xor_b32_e32 v1, v1, v3
+; CHECK-NEXT:    v_xor_b32_e32 v0, v0, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %xor = xor i64 %arg0, %arg1
+  ret i64 %xor
+}
+
+; Should be able to reduce this to a 32-bit or plus a copy
+; https://alive2.llvm.org/ce/z/HaXnBJ
+define i64 @v_or_i64_known_hi_i32_from_arg_range(i64 range(i64 0, 4294967296) %arg0, i64 %arg1) {
+; CHECK-LABEL: v_or_i64_known_hi_i32_from_arg_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v1, v1, v3
+; CHECK-NEXT:    v_or_b32_e32 v0, v0, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %arg0, %arg1
+  ret i64 %or
+}
+
+; https://alive2.llvm.org/ce/z/M96Ror
+; Should be able to reduce this to a 32-bit plus a copy
+define i64 @v_and_i64_known_i32_from_arg_range(i64 range(i64 -4294967296, 0) %arg0, i64 %arg1) {
+; CHECK-LABEL: v_and_i64_known_i32_from_arg_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v1, v1, v3
+; CHECK-NEXT:    v_and_b32_e32 v0, v0, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %and = and i64 %arg0, %arg1
+  ret i64 %and
+}
+
+define i64 @s_xor_i64_known_i32_from_arg_range(i64 range(i64 0, 65) inreg %arg) {
+; CHECK-LABEL: s_xor_i64_known_i32_from_arg_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_not_b64 s[4:5], s[16:17]
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %xor = xor i64 %arg, -1
+  ret i64 %xor
+}
+
+define i64 @v_xor_i64_known_i32_from_call_range() {
+; CHECK-LABEL: v_xor_i64_known_i32_from_call_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def v[0:1]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_not_b32_e32 v1, v1
+; CHECK-NEXT:    v_not_b32_e32 v0, v0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %call = call range(i64 0, 65) i64 asm "; def $0", "=v"()
+  %xor = xor i64 %call, -1
+  ret i64 %xor
+}
+
+define i64 @s_xor_i64_known_i32_from_call_range() {
+; CHECK-LABEL: s_xor_i64_known_i32_from_call_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def s[4:5]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_not_b64 s[4:5], s[4:5]
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %call = call range(i64 0, 65) i64 asm "; def $0", "=s"()
+  %xor = xor i64 %call, -1
+  ret i64 %xor
+}
+
+; Reduced from -amdgpu-codegenprepare-expand-div64 output, produces a
+; not of 0 which ideally would fold out.
+; FIXME: Produces not of constant 0
+define i64 @v_xor_i64_known_i32_from_range_use_out_of_block(i64 %x) {
+; CHECK-LABEL: v_xor_i64_known_i32_from_range_use_out_of_block:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ffbh_u32_e32 v2, v0
+; CHECK-NEXT:    v_add_u32_e32 v2, 32, v2
+; CHECK-NEXT:    v_ffbh_u32_e32 v3, v1
+; CHECK-NEXT:    v_min_u32_e32 v4, v2, v3
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; CHECK-NEXT:  ; %bb.1: ; %inc
+; CHECK-NEXT:    v_not_b32_e32 v2, v4
+; CHECK-NEXT:    v_add_co_u32_e32 v2, vcc, v0, v2
+; CHECK-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v1, vcc
+; CHECK-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ctlz = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %cmp.entry.not = icmp eq i64 %ctlz, %x
+  br i1 %cmp.entry.not, label %inc, label %ret
+
+inc:                                              ; preds = %entry
+  %i1 = xor i64 %ctlz, -1
+  %i2 = add i64 %x, %i1
+  ret i64 %i2
+
+ret:                                              ; preds = %loop, %entry
+  ret i64 0
+}
+
+define i64 @s_xor_i64_known_i32_from_range_use_out_of_block(i64 inreg %x) {
+; CHECK-LABEL: s_xor_i64_known_i32_from_range_use_out_of_block:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_flbit_i32_b64 s4, s[16:17]
+; CHECK-NEXT:    s_mov_b32 s5, 0
+; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], s[16:17]
+; CHECK-NEXT:    s_cbranch_scc1 .LBB7_2
+; CHECK-NEXT:  ; %bb.1: ; %inc
+; CHECK-NEXT:    s_not_b64 s[4:5], s[4:5]
+; CHECK-NEXT:    s_add_u32 s4, s16, s4
+; CHECK-NEXT:    s_addc_u32 s5, s17, s5
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; CHECK-NEXT:  .LBB7_2: ; %ret
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %ctlz = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %cmp.entry.not = icmp eq i64 %ctlz, %x
+  br i1 %cmp.entry.not, label %inc, label %ret
+
+inc:                                              ; preds = %entry
+  %i1 = xor i64 %ctlz, -1
+  %i2 = add i64 %x, %i1
+  ret i64 %i2
+
+ret:                                              ; preds = %loop, %entry
+  ret i64 0
+}
+
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/br_cc.f16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/br_cc.f16.ll
index b27ad26cf97b..4cb5b7c43a46 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/br_cc.f16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/br_cc.f16.ll
@@ -77,18 +77,19 @@ define amdgpu_kernel void @br_cc_f16(
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    buffer_load_u16 v1, off, s[8:11], 0 glc dlc
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    s_mov_b32 s2, s6
-; GFX11-TRUE16-NEXT:    s_mov_b32 s3, s7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v2.l, v2.h
+; GFX11-TRUE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, v0.h, v1.h
 ; GFX11-TRUE16-NEXT:    s_cbranch_vccnz .LBB0_2
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %one
-; GFX11-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-TRUE16-NEXT:    s_endpgm
+; GFX11-TRUE16-NEXT:    s_branch .LBB0_3
 ; GFX11-TRUE16-NEXT:  .LBB0_2: ; %two
-; GFX11-TRUE16-NEXT:    buffer_store_b16 v1, off, s[0:3], 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT:  .LBB0_3: ; %one
+; GFX11-TRUE16-NEXT:    s_mov_b32 s2, s6
+; GFX11-TRUE16-NEXT:    s_mov_b32 s3, s7
+; GFX11-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
 ; GFX11-TRUE16-NEXT:    s_endpgm
 ;
 ; GFX11-FAKE16-LABEL: br_cc_f16:
@@ -192,13 +193,15 @@ define amdgpu_kernel void @br_cc_f16_imm_a(
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s5, s3
 ; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[4:7], 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, 0.5, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_nlt_f16_e32 vcc_lo, 0.5, v0.h
 ; GFX11-TRUE16-NEXT:    s_cbranch_vccnz .LBB1_2
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %one
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, 0x3800
+; GFX11-TRUE16-NEXT:    s_branch .LBB1_3
 ; GFX11-TRUE16-NEXT:  .LBB1_2: ; %two
+; GFX11-TRUE16-NEXT:  .LBB1_3: ; %one
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, s6
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s3, s7
 ; GFX11-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
@@ -298,13 +301,15 @@ define amdgpu_kernel void @br_cc_f16_imm_b(
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s5, s3
 ; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[4:7], 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, 0.5, v1.l
-; GFX11-TRUE16-NEXT:    s_cbranch_vccz .LBB2_2
-; GFX11-TRUE16-NEXT:  ; %bb.1: ; %two
+; GFX11-TRUE16-NEXT:    v_cmp_ngt_f16_e32 vcc_lo, 0.5, v0.h
+; GFX11-TRUE16-NEXT:    s_cbranch_vccnz .LBB2_2
+; GFX11-TRUE16-NEXT:  ; %bb.1: ; %one
+; GFX11-TRUE16-NEXT:    s_branch .LBB2_3
+; GFX11-TRUE16-NEXT:  .LBB2_2: ; %two
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, 0x3800
-; GFX11-TRUE16-NEXT:  .LBB2_2: ; %one
+; GFX11-TRUE16-NEXT:  .LBB2_3: ; %one
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, s6
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s3, s7
 ; GFX11-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 198bf839cb1c..3eba106b861c 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -9090,11 +9090,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-TRUE16-NEXT:    s_add_i32 s4, s16, 0x400
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
@@ -9146,11 +9145,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
 ; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-FAKE16-NEXT:    s_add_i32 s4, s16, 0x400
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB26_1: ; %atomicrmw.start
@@ -10698,11 +10696,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-TRUE16-NEXT:    s_add_i32 s4, s16, 0x400
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
@@ -10754,11 +10751,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
 ; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-FAKE16-NEXT:    s_add_i32 s4, s16, 0x400
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB29_1: ; %atomicrmw.start
@@ -11563,11 +11559,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-TRUE16-NEXT:    s_add_i32 s4, s16, 0x400
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
@@ -11619,11 +11614,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
 ; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-FAKE16-NEXT:    s_add_i32 s4, s16, 0x400
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB31_1: ; %atomicrmw.start
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index bee2813ca30f..c17225594164 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -7385,11 +7385,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX12-TRUE16-NEXT:    s_add_co_i32 s4, s16, 0x400
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX12-TRUE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX12-TRUE16-NEXT:  .LBB19_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7445,10 +7444,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
 ; GFX12-FAKE16-NEXT:    s_add_co_i32 s4, s16, 0x400
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s5, 0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX12-FAKE16-NEXT:  .LBB19_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
@@ -7540,11 +7539,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-TRUE16-NEXT:    s_add_i32 s4, s16, 0x400
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
@@ -7596,11 +7594,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
 ; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-FAKE16-NEXT:    s_add_i32 s4, s16, 0x400
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB19_1: ; %atomicrmw.start
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 1826743ed017..56719dccbd08 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -7385,11 +7385,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX12-TRUE16-NEXT:    s_add_co_i32 s4, s16, 0x400
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX12-TRUE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX12-TRUE16-NEXT:  .LBB19_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7445,10 +7444,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
 ; GFX12-FAKE16-NEXT:    s_add_co_i32 s4, s16, 0x400
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s5, 0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX12-FAKE16-NEXT:  .LBB19_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
@@ -7540,11 +7539,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-TRUE16-NEXT:    s_add_i32 s4, s16, 0x400
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s4, 0
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
@@ -7596,11 +7594,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
 ; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
 ; GFX11-FAKE16-NEXT:    s_add_i32 s4, s16, 0x400
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_and_b32 v3, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB19_1: ; %atomicrmw.start
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
index fe2b0bb1ff6a..e7177a5e7160 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
@@ -961,3 +961,25 @@ body:             |
     S_ENDPGM 0, implicit %2, implicit %3
 
 ...
+
+---
+name:            constant_v_or_b32_uses_subreg_or_0_regression
+tracksRegLiveness: true
+body:             |
+  bb.0:
+  liveins: $vgpr0, $vgpr1
+
+    ; GCN-LABEL: name: constant_v_or_b32_uses_subreg_or_0_regression
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+  %0:vgpr_32 = COPY $vgpr0
+  %1:vgpr_32 = COPY $vgpr1
+  %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  %3:vreg_64 = REG_SEQUENCE %2:vgpr_32, %subreg.sub0, %0:vgpr_32, %subreg.sub1
+  %4:vgpr_32 = V_OR_B32_e64 %3.sub0:vreg_64, %1, implicit $exec
+  S_ENDPGM 0, implicit %4
+
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index 9503ffbdb410..99b7c7737f4a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -910,38 +910,36 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s8, s6
 ; SI-NEXT:    s_mov_b32 s9, s7
-; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0 offset:5
-; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:7
-; SI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0
-; SI-NEXT:    buffer_load_ubyte v3, off, s[8:11], 0 offset:1
-; SI-NEXT:    buffer_load_ubyte v4, off, s[8:11], 0 offset:2
-; SI-NEXT:    buffer_load_ubyte v5, off, s[8:11], 0 offset:3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:1
+; SI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0 offset:2
+; SI-NEXT:    buffer_load_ubyte v3, off, s[8:11], 0 offset:3
+; SI-NEXT:    buffer_load_ubyte v4, off, s[8:11], 0 offset:5
+; SI-NEXT:    buffer_load_ubyte v5, off, s[8:11], 0 offset:7
 ; SI-NEXT:    buffer_load_ubyte v6, off, s[8:11], 0 offset:4
 ; SI-NEXT:    buffer_load_ubyte v7, off, s[8:11], 0 offset:6
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_mov_b32 s1, s5
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v6
+; SI-NEXT:    v_or_b32_e32 v4, v4, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v1, v7
-; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    v_or_b32_e32 v3, v5, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v5, v5, v7
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v2
-; SI-NEXT:    v_ffbh_u32_e32 v1, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v2, v2, v4
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_ffbh_u32_e32 v0, v0
-; SI-NEXT:    v_add_i32_e32 v1, vcc, 32, v1
-; SI-NEXT:    v_min_u32_e32 v0, v1, v0
+; SI-NEXT:    v_ffbh_u32_e32 v1, v2
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; SI-NEXT:    v_min_u32_e32 v0, v0, v1
 ; SI-NEXT:    v_min_u32_e32 v0, 64, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index 97bcd8b5ee68..73fddb53d1dc 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -874,38 +874,36 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s8, s6
 ; SI-NEXT:    s_mov_b32 s9, s7
-; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0 offset:5
-; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:7
-; SI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0
-; SI-NEXT:    buffer_load_ubyte v3, off, s[8:11], 0 offset:1
-; SI-NEXT:    buffer_load_ubyte v4, off, s[8:11], 0 offset:2
-; SI-NEXT:    buffer_load_ubyte v5, off, s[8:11], 0 offset:3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:1
+; SI-NEXT:    buffer_load_ubyte v2, off, s[8:11], 0 offset:2
+; SI-NEXT:    buffer_load_ubyte v3, off, s[8:11], 0 offset:3
+; SI-NEXT:    buffer_load_ubyte v4, off, s[8:11], 0 offset:5
+; SI-NEXT:    buffer_load_ubyte v5, off, s[8:11], 0 offset:7
 ; SI-NEXT:    buffer_load_ubyte v6, off, s[8:11], 0 offset:4
 ; SI-NEXT:    buffer_load_ubyte v7, off, s[8:11], 0 offset:6
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_mov_b32 s1, s5
-; SI-NEXT:    s_waitcnt vmcnt(7)
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
 ; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_or_b32_e32 v0, v0, v6
+; SI-NEXT:    v_or_b32_e32 v4, v4, v6
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_or_b32_e32 v1, v1, v7
-; SI-NEXT:    v_or_b32_e32 v2, v3, v2
-; SI-NEXT:    v_or_b32_e32 v3, v5, v4
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_or_b32_e32 v5, v5, v7
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v2
-; SI-NEXT:    v_ffbl_b32_e32 v1, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v2, v2, v4
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_ffbl_b32_e32 v0, v0
-; SI-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
-; SI-NEXT:    v_min_u32_e32 v0, v0, v1
+; SI-NEXT:    v_ffbl_b32_e32 v1, v2
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 32, v1
+; SI-NEXT:    v_min_u32_e32 v0, v1, v0
 ; SI-NEXT:    v_min_u32_e32 v0, 64, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index d1090738e24a..745e04734862 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1568,10 +1568,10 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned_multiuse(ptr addrspace(1
 ; GFX9-NEXT:    s_mov_b32 s0, 0x4000405
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_load_ubyte v1, v0, s[12:13] offset:2
-; GFX9-NEXT:    global_load_ubyte v2, v0, s[14:15] offset:3
 ; GFX9-NEXT:    global_load_ubyte v3, v0, s[12:13] offset:3
+; GFX9-NEXT:    global_load_ubyte v2, v0, s[14:15] offset:3
 ; GFX9-NEXT:    global_load_ubyte v4, v0, s[14:15] offset:2
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshl_or_b32 v6, v3, 8, v1
 ; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v1, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -1904,37 +1904,37 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 5, v0
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v10, v[2:3]
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 6, v0
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 6, v0
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v6, vcc, 2, v0
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 1, v0
 ; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; VI-NEXT:    v_add_u32_e32 v8, vcc, 3, v0
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 2, v0
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v6, v[6:7]
-; VI-NEXT:    flat_load_ubyte v7, v[8:9]
-; VI-NEXT:    flat_load_ubyte v8, v[2:3]
-; VI-NEXT:    flat_load_ubyte v2, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 3, v0
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_ubyte v12, v[2:3]
+; VI-NEXT:    flat_load_ubyte v2, v[8:9]
+; VI-NEXT:    flat_load_ubyte v3, v[10:11]
 ; VI-NEXT:    flat_load_ubyte v4, v[4:5]
+; VI-NEXT:    flat_load_ubyte v5, v[0:1]
+; VI-NEXT:    flat_load_ubyte v6, v[6:7]
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 4, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_ubyte v9, v[0:1]
+; VI-NEXT:    flat_load_ubyte v7, v[0:1]
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v5, v10
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v2
 ; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v3, v7
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v3, v3
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v2
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v6
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v5
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v4
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v6, v8
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v6
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v6, v4
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v5, v12
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v4, v9
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v4, v7
 ; VI-NEXT:    buffer_store_dwordx3 v[4:6], off, s[0:3], 0 offset:16
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds-alignment.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
index b1664c59a7e4..93422e259b82 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
@@ -209,27 +209,28 @@ define amdgpu_kernel void @ds8align1(ptr addrspace(3) %in, ptr addrspace(3) %out
 ; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-SDAG-NEXT:    ds_read_u8 v1, v0
-; ALIGNED-SDAG-NEXT:    ds_read_u8 v2, v0 offset:1
-; ALIGNED-SDAG-NEXT:    ds_read_u8 v3, v0 offset:2
-; ALIGNED-SDAG-NEXT:    ds_read_u8 v4, v0 offset:3
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v2, v0
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v3, v0 offset:1
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v4, v0 offset:2
 ; ALIGNED-SDAG-NEXT:    ds_read_u8 v5, v0 offset:4
 ; ALIGNED-SDAG-NEXT:    ds_read_u8 v6, v0 offset:5
+; ALIGNED-SDAG-NEXT:    ds_read_u8 v7, v0 offset:3
 ; ALIGNED-SDAG-NEXT:    ds_read_u8 v8, v0 offset:6
 ; ALIGNED-SDAG-NEXT:    ds_read_u8 v0, v0 offset:7
-; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v7, s1
-; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v5 offset:4
-; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v6 offset:5
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v1
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v2 offset:1
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v3 offset:2
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v4 offset:3
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v5 offset:4
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v6 offset:5
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v2
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v3 offset:1
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v4 offset:2
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v8 offset:6
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v7 offset:3
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-SDAG-NEXT:    ds_write_b8 v7, v0 offset:7
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v8 offset:6
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
+; ALIGNED-SDAG-NEXT:    ds_write_b8 v1, v0 offset:7
 ; ALIGNED-SDAG-NEXT:    s_endpgm
 ;
 ; ALIGNED-GISEL-LABEL: ds8align1:
@@ -492,23 +493,24 @@ define amdgpu_kernel void @ds12align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
 ; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-SDAG-NEXT:    ds_read_u16 v1, v0 offset:8
 ; ALIGNED-SDAG-NEXT:    ds_read_u16 v2, v0
-; ALIGNED-SDAG-NEXT:    ds_read_u16 v3, v0 offset:2
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v3, v0 offset:8
 ; ALIGNED-SDAG-NEXT:    ds_read_u16 v4, v0 offset:4
-; ALIGNED-SDAG-NEXT:    ds_read_u16 v5, v0 offset:6
-; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v6, s1
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v5, v0 offset:2
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v6, v0 offset:6
 ; ALIGNED-SDAG-NEXT:    ds_read_u16 v0, v0 offset:10
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v3 offset:8
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v4 offset:4
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v2
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v6, v1 offset:8
-; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v6, v4 offset:4
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v6, v2
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v6, v3 offset:2
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v5 offset:2
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v6, v5 offset:6
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v6 offset:6
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v6, v0 offset:10
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v0 offset:10
 ; ALIGNED-SDAG-NEXT:    s_endpgm
 ;
 ; ALIGNED-GISEL-LABEL: ds12align2:
@@ -808,29 +810,27 @@ define amdgpu_kernel void @ds16align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
 ; ALIGNED-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-SDAG-NEXT:    ds_read_u16 v1, v0 offset:12
 ; ALIGNED-SDAG-NEXT:    ds_read_u16 v2, v0
 ; ALIGNED-SDAG-NEXT:    ds_read_u16 v3, v0 offset:2
 ; ALIGNED-SDAG-NEXT:    ds_read_u16 v4, v0 offset:4
-; ALIGNED-SDAG-NEXT:    ds_read_u16 v5, v0 offset:6
-; ALIGNED-SDAG-NEXT:    ds_read_u16 v6, v0 offset:8
-; ALIGNED-SDAG-NEXT:    ds_read_u16 v7, v0 offset:10
-; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v8, s1
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v5, v0 offset:12
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v6, v0 offset:6
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v7, v0 offset:8
+; ALIGNED-SDAG-NEXT:    ds_read_u16 v8, v0 offset:10
 ; ALIGNED-SDAG-NEXT:    ds_read_u16 v0, v0 offset:14
-; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v1 offset:12
-; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v2
-; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(6)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v4 offset:4
+; ALIGNED-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v5 offset:12
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v2
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v4 offset:4
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(5)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v6 offset:8
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v3 offset:2
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v5 offset:6
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v7 offset:8
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v3 offset:2
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v6 offset:6
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v7 offset:10
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v8 offset:10
 ; ALIGNED-SDAG-NEXT:    s_waitcnt lgkmcnt(7)
-; ALIGNED-SDAG-NEXT:    ds_write_b16 v8, v0 offset:14
+; ALIGNED-SDAG-NEXT:    ds_write_b16 v1, v0 offset:14
 ; ALIGNED-SDAG-NEXT:    s_endpgm
 ;
 ; ALIGNED-GISEL-LABEL: ds16align2:
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds_read2.ll
index 06c30dfd3603..d95f528442ef 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ds_read2.ll
@@ -522,36 +522,39 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(ptr addrspace(1) %out) #0
 define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
 ; CI-LABEL: unaligned_read2_f32:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_load_dword s0, s[4:5], 0x2
+; CI-NEXT:    s_load_dword s2, s[4:5], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CI-NEXT:    s_mov_b32 m0, -1
 ; CI-NEXT:    s_mov_b32 s3, 0xf000
-; CI-NEXT:    s_mov_b32 s2, 0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s0, v0
-; CI-NEXT:    ds_read_u8 v2, v1 offset:34
-; CI-NEXT:    ds_read_u8 v3, v1 offset:32
-; CI-NEXT:    ds_read_u8 v4, v1 offset:3
+; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v0
+; CI-NEXT:    ds_read_u8 v2, v1 offset:1
+; CI-NEXT:    ds_read_u8 v3, v1 offset:34
+; CI-NEXT:    ds_read_u8 v4, v1 offset:32
 ; CI-NEXT:    ds_read_u8 v5, v1 offset:2
-; CI-NEXT:    ds_read_u8 v6, v1 offset:1
-; CI-NEXT:    ds_read_u8 v7, v1
+; CI-NEXT:    ds_read_u8 v6, v1
+; CI-NEXT:    ds_read_u8 v7, v1 offset:3
 ; CI-NEXT:    ds_read_u8 v8, v1 offset:33
 ; CI-NEXT:    ds_read_u8 v1, v1 offset:35
-; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT:    s_waitcnt lgkmcnt(7)
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; CI-NEXT:    s_waitcnt lgkmcnt(3)
+; CI-NEXT:    v_or_b32_e32 v2, v2, v6
+; CI-NEXT:    s_waitcnt lgkmcnt(2)
+; CI-NEXT:    v_lshlrev_b32_e32 v6, 8, v7
+; CI-NEXT:    v_or_b32_e32 v5, v6, v5
+; CI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; CI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; CI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; CI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; CI-NEXT:    v_or_b32_e32 v2, v5, v2
 ; CI-NEXT:    v_lshlrev_b32_e32 v5, 8, v8
-; CI-NEXT:    v_or_b32_e32 v1, v1, v2
-; CI-NEXT:    v_or_b32_e32 v6, v6, v7
-; CI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; CI-NEXT:    v_or_b32_e32 v3, v5, v3
-; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT:    v_or_b32_e32 v4, v4, v6
 ; CI-NEXT:    v_or_b32_e32 v1, v1, v3
-; CI-NEXT:    v_add_f32_e32 v2, v4, v1
+; CI-NEXT:    v_or_b32_e32 v4, v5, v4
+; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; CI-NEXT:    v_or_b32_e32 v1, v1, v4
+; CI-NEXT:    v_add_f32_e32 v2, v2, v1
+; CI-NEXT:    s_mov_b32 s2, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; CI-NEXT:    s_endpgm
@@ -612,36 +615,39 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
 define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
 ; CI-LABEL: unaligned_offset_read2_f32:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_load_dword s0, s[4:5], 0x2
+; CI-NEXT:    s_load_dword s2, s[4:5], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CI-NEXT:    s_mov_b32 m0, -1
 ; CI-NEXT:    s_mov_b32 s3, 0xf000
-; CI-NEXT:    s_mov_b32 s2, 0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_add_i32_e32 v1, vcc, s0, v0
-; CI-NEXT:    ds_read_u8 v2, v1 offset:11
-; CI-NEXT:    ds_read_u8 v3, v1 offset:9
-; CI-NEXT:    ds_read_u8 v4, v1 offset:8
+; CI-NEXT:    v_add_i32_e32 v1, vcc, s2, v0
+; CI-NEXT:    ds_read_u8 v2, v1 offset:6
+; CI-NEXT:    ds_read_u8 v3, v1 offset:11
+; CI-NEXT:    ds_read_u8 v4, v1 offset:9
 ; CI-NEXT:    ds_read_u8 v5, v1 offset:7
-; CI-NEXT:    ds_read_u8 v6, v1 offset:6
-; CI-NEXT:    ds_read_u8 v7, v1 offset:5
+; CI-NEXT:    ds_read_u8 v6, v1 offset:5
+; CI-NEXT:    ds_read_u8 v7, v1 offset:8
 ; CI-NEXT:    ds_read_u8 v8, v1 offset:10
 ; CI-NEXT:    ds_read_u8 v1, v1 offset:12
-; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT:    s_waitcnt lgkmcnt(7)
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; CI-NEXT:    s_waitcnt lgkmcnt(3)
+; CI-NEXT:    v_or_b32_e32 v2, v2, v6
+; CI-NEXT:    s_waitcnt lgkmcnt(2)
+; CI-NEXT:    v_lshlrev_b32_e32 v6, 8, v7
+; CI-NEXT:    v_or_b32_e32 v5, v6, v5
+; CI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; CI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; CI-NEXT:    v_or_b32_e32 v4, v4, v5
 ; CI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; CI-NEXT:    v_or_b32_e32 v2, v5, v2
 ; CI-NEXT:    v_lshlrev_b32_e32 v5, 8, v8
-; CI-NEXT:    v_or_b32_e32 v1, v1, v2
-; CI-NEXT:    v_or_b32_e32 v6, v6, v7
-; CI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; CI-NEXT:    v_or_b32_e32 v3, v5, v3
-; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT:    v_or_b32_e32 v4, v4, v6
 ; CI-NEXT:    v_or_b32_e32 v1, v1, v3
-; CI-NEXT:    v_add_f32_e32 v2, v4, v1
+; CI-NEXT:    v_or_b32_e32 v4, v5, v4
+; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; CI-NEXT:    v_or_b32_e32 v1, v1, v4
+; CI-NEXT:    v_add_f32_e32 v2, v2, v1
+; CI-NEXT:    s_mov_b32 s2, 0
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; CI-NEXT:    s_endpgm
@@ -709,17 +715,17 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out,
 ; CI-NEXT:    s_mov_b32 s2, 0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_add_i32_e32 v1, vcc, s0, v0
-; CI-NEXT:    ds_read_u16 v2, v1 offset:32
-; CI-NEXT:    ds_read_u16 v3, v1 offset:2
+; CI-NEXT:    ds_read_u16 v2, v1 offset:2
+; CI-NEXT:    ds_read_u16 v3, v1 offset:32
 ; CI-NEXT:    ds_read_u16 v4, v1
 ; CI-NEXT:    ds_read_u16 v1, v1 offset:34
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; CI-NEXT:    v_or_b32_e32 v3, v3, v4
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_or_b32_e32 v2, v2, v4
 ; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT:    v_or_b32_e32 v1, v1, v2
-; CI-NEXT:    v_add_f32_e32 v2, v3, v1
+; CI-NEXT:    v_or_b32_e32 v1, v1, v3
+; CI-NEXT:    v_add_f32_e32 v2, v2, v1
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; CI-NEXT:    s_endpgm
@@ -1460,17 +1466,17 @@ define amdgpu_kernel void @read2_v2i32_align1_odd_offset(ptr addrspace(1) %out)
 ; CI-NEXT:    v_or_b32_e32 v2, v2, v3
 ; CI-NEXT:    s_waitcnt lgkmcnt(1)
 ; CI-NEXT:    v_or_b32_e32 v1, v1, v4
-; CI-NEXT:    ds_read_u8 v4, v0 offset:67
-; CI-NEXT:    ds_read_u8 v6, v0 offset:66
+; CI-NEXT:    ds_read_u8 v4, v0 offset:66
+; CI-NEXT:    ds_read_u8 v6, v0 offset:67
 ; CI-NEXT:    ds_read_u8 v0, v0 offset:65
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; CI-NEXT:    v_or_b32_e32 v1, v2, v1
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_lshlrev_b32_e32 v2, 8, v6
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 8, v4
 ; CI-NEXT:    v_or_b32_e32 v0, v2, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 8, v5
-; CI-NEXT:    v_or_b32_e32 v2, v2, v4
+; CI-NEXT:    v_or_b32_e32 v2, v2, v6
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; CI-NEXT:    s_mov_b32 s3, 0xf000
 ; CI-NEXT:    s_mov_b32 s2, -1
@@ -1481,26 +1487,25 @@ define amdgpu_kernel void @read2_v2i32_align1_odd_offset(ptr addrspace(1) %out)
 ; GFX9-ALIGNED-LABEL: read2_v2i32_align1_odd_offset:
 ; GFX9-ALIGNED:       ; %bb.0: ; %entry
 ; GFX9-ALIGNED-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-ALIGNED-NEXT:    ds_read_u8 v0, v2 offset:70
-; GFX9-ALIGNED-NEXT:    ds_read_u8 v3, v2 offset:65
-; GFX9-ALIGNED-NEXT:    ds_read_u8 v4, v2 offset:66
-; GFX9-ALIGNED-NEXT:    ds_read_u8 v5, v2 offset:67
-; GFX9-ALIGNED-NEXT:    ds_read_u8 v6, v2 offset:68
-; GFX9-ALIGNED-NEXT:    ds_read_u8 v1, v2 offset:69
+; GFX9-ALIGNED-NEXT:    ds_read_u8 v0, v2 offset:65
+; GFX9-ALIGNED-NEXT:    ds_read_u8 v3, v2 offset:66
+; GFX9-ALIGNED-NEXT:    ds_read_u8 v4, v2 offset:67
+; GFX9-ALIGNED-NEXT:    ds_read_u8 v5, v2 offset:68
+; GFX9-ALIGNED-NEXT:    ds_read_u8 v1, v2 offset:70
+; GFX9-ALIGNED-NEXT:    ds_read_u8 v6, v2 offset:69
 ; GFX9-ALIGNED-NEXT:    ds_read_u8 v7, v2 offset:72
 ; GFX9-ALIGNED-NEXT:    ds_read_u8 v8, v2 offset:71
-; GFX9-ALIGNED-NEXT:    s_waitcnt lgkmcnt(7)
-; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX9-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; GFX9-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-ALIGNED-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 8, v7
-; GFX9-ALIGNED-NEXT:    v_or_b32_sdwa v1, v1, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-ALIGNED-NEXT:    v_or_b32_e32 v1, v1, v0
-; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 8, v4
-; GFX9-ALIGNED-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 8, v6
-; GFX9-ALIGNED-NEXT:    v_or_b32_sdwa v3, v3, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
+; GFX9-ALIGNED-NEXT:    v_or_b32_e32 v1, v1, v6
+; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v6, 8, v7
+; GFX9-ALIGNED-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX9-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 8, v5
+; GFX9-ALIGNED-NEXT:    v_or_b32_sdwa v6, v6, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-ALIGNED-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-ALIGNED-NEXT:    v_or_b32_e32 v1, v6, v1
 ; GFX9-ALIGNED-NEXT:    v_or_b32_e32 v0, v3, v0
 ; GFX9-ALIGNED-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX9-ALIGNED-NEXT:    s_endpgm
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
new file mode 100644
index 000000000000..71b1a16c79e6
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
@@ -0,0 +1,1201 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck -enable-var-scope -check-prefixes=CI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s
+
+; DAGCombiner will transform:
+; (fabs (bf16 bitcast (i16 a))) => (bf16 bitcast (and (i16 a), 0x7FFFFFFF))
+; unless isFabsFree returns true
+
+define amdgpu_kernel void @s_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
+; CI-LABEL: s_fabs_free_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fabs_free_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fabs_free_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: s_fabs_free_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: s_fabs_free_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %bc= bitcast i16 %in to bfloat
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %bc)
+  store bfloat %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
+; CI-LABEL: s_fabs_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fabs_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fabs_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: s_fabs_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: s_fabs_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %in)
+  store bfloat %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fabs_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) {
+; CI-LABEL: s_fabs_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fabs_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fabs_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fabs_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)
+  store <2 x bfloat> %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in) {
+; CI-LABEL: s_fabs_v4bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s4, s3, 0xffff0000
+; CI-NEXT:    s_lshl_b32 s3, s3, 16
+; CI-NEXT:    s_and_b32 s5, s2, 0xffff0000
+; CI-NEXT:    v_mul_f32_e64 v0, 1.0, |s4|
+; CI-NEXT:    v_mul_f32_e64 v1, 1.0, |s3|
+; CI-NEXT:    v_mul_f32_e64 v2, 1.0, |s5|
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_alignbit_b32 v1, v0, v1, 16
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
+; CI-NEXT:    v_mul_f32_e64 v2, 1.0, |s2|
+; CI-NEXT:    v_alignbit_b32 v0, v0, v2, 16
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fabs_v4bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s4, s3, 0x7fff
+; VI-NEXT:    s_lshr_b32 s3, s3, 16
+; VI-NEXT:    s_and_b32 s5, s2, 0x7fff
+; VI-NEXT:    s_lshr_b32 s2, s2, 16
+; VI-NEXT:    s_and_b32 s3, s3, 0x7fff
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; VI-NEXT:    s_and_b32 s4, 0xffff, s4
+; VI-NEXT:    s_and_b32 s5, 0xffff, s5
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    s_lshl_b32 s2, s2, 16
+; VI-NEXT:    s_or_b32 s3, s4, s3
+; VI-NEXT:    s_or_b32 s2, s5, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fabs_v4bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s4, s3, 0x7fff
+; GFX9-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX9-NEXT:    s_and_b32 s5, s2, 0x7fff
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX9-NEXT:    s_and_b32 s3, s3, 0x7fff
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: s_fabs_v4bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_and_b32 s4, s3, 0x7fff
+; GFX11-TRUE16-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT:    s_mov_b32 s5, s2
+; GFX11-TRUE16-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT:    s_and_b32 s5, s5, 0x7fff
+; GFX11-TRUE16-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX11-TRUE16-NEXT:    s_and_b32 s3, s3, 0x7fff
+; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
+; GFX11-TRUE16-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-TRUE16-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: s_fabs_v4bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s4, s3, 0x7fff
+; GFX11-FAKE16-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX11-FAKE16-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX11-FAKE16-NEXT:    s_and_b32 s5, s5, 0x7fff
+; GFX11-FAKE16-NEXT:    s_and_b32 s3, s3, 0x7fff
+; GFX11-FAKE16-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX11-FAKE16-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-FAKE16-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat> %in)
+  store <4 x bfloat> %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fabs_fold_bf16(ptr addrspace(1) %out, bfloat %in0, bfloat %in1) {
+; CI-LABEL: fabs_fold_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0x7fff
+; CI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; CI-NEXT:    s_lshl_b32 s3, s3, 16
+; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    v_mul_f32_e32 v0, s3, v0
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fabs_fold_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s3, s2, 0x7fff
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mul_f32_e32 v0, s3, v0
+; VI-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
+; VI-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fabs_fold_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s3, s2, 0x7fff
+; GFX9-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mul_f32_e32 v1, s3, v1
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    global_store_short_d16_hi v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: fabs_fold_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s1, s0
+; GFX11-TRUE16-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX11-TRUE16-NEXT:    s_and_b32 s1, s1, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT:    v_mul_f32_e64 v0, s1, s0
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_store_d16_hi_b16 v3, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fabs_fold_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s1, s0, 0x7fff
+; GFX11-FAKE16-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_mul_f32_e64 v0, s1, s0
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v1, v1, v0
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_store_d16_hi_b16 v3, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %in0)
+  %fmul = fmul bfloat %fabs, %in1
+  store bfloat %fmul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fabs_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_fabs_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v2, v[0:1]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v2, 0x7fff7fff, v2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_fabs_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v2, v[0:1]
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0x7fff7fff, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fabs_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_fabs_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %gep.out = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %val = load <2 x bfloat>, ptr addrspace(1) %gep.in, align 2
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)
+  store <2 x bfloat> %fabs, ptr addrspace(1) %gep.out
+  ret void
+}
+
+define amdgpu_kernel void @fabs_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {
+; CI-LABEL: fabs_free_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fabs_free_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fabs_free_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fabs_free_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %bc = bitcast i32 %in to <2 x bfloat>
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %bc)
+  store <2 x bfloat> %fabs, ptr addrspace(1) %out
+  ret void
+}
+
+; FIXME: Should do fabs after conversion to avoid converting multiple
+; times in this particular case.
+define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_fabs_fold_self_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v2, v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_mul_f32_e64 v4, 1.0, |v3|
+; CI-NEXT:    v_mul_f32_e64 v5, 1.0, |v2|
+; CI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
+; CI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
+; CI-NEXT:    v_mul_f32_e32 v3, v4, v3
+; CI-NEXT:    v_mul_f32_e32 v2, v5, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_alignbit_b32 v2, v3, v2, 16
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_fabs_fold_self_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v2, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, 0x7fff
+; VI-NEXT:    s_movk_i32 s2, 0x7fff
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
+; VI-NEXT:    v_and_b32_sdwa v5, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; VI-NEXT:    v_and_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_mul_f32_e32 v3, v5, v4
+; VI-NEXT:    v_mul_f32_e32 v2, v2, v6
+; VI-NEXT:    v_bfe_u32 v4, v3, 16, 1
+; VI-NEXT:    v_bfe_u32 v6, v2, 16, 1
+; VI-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
+; VI-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s2, v4
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 0x7fff, v6
+; VI-NEXT:    v_or_b32_e32 v5, 0x400000, v3
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; VI-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; VI-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v6, v7, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT:    v_alignbit_b32 v2, v2, v3, 16
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fabs_fold_self_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
+; GFX9-NEXT:    v_and_b32_sdwa v3, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
+; GFX9-NEXT:    v_and_b32_sdwa v0, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_mul_f32_e32 v0, v0, v4
+; GFX9-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT:    v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v2, s2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_or_b32_e32 v6, 0x400000, v0
+; GFX9-NEXT:    v_add3_u32 v5, v5, v0, s2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT:    s_mov_b32 s2, 0x7060302
+; GFX9-NEXT:    v_perm_b32 v0, v0, v2, s2
+; GFX9-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_fabs_fold_self_v2bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_b32 v2, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0x7fff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0x7fff, v2.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT:    v_dual_mul_f32 v0, v0, v3 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_fabs_fold_self_v2bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_b32 v0, v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0x7fff, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT:    v_dual_mul_f32 v2, v2, v3 :: v_dual_and_b32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v1, v2, 16, 1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-FAKE16-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11-FAKE16-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %val = load <2 x bfloat>, ptr addrspace(1) %gep
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)
+  %fmul = fmul <2 x bfloat> %fabs, %val
+  store <2 x bfloat> %fmul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fabs_fold_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %other.val) #0 {
+; CI-LABEL: v_fabs_fold_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v2, v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_and_b32 s1, s4, 0xffff0000
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_lshl_b32 s0, s4, 16
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v3, 0x7fff0000, v2
+; CI-NEXT:    v_and_b32_e32 v2, 0x7fff, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_mul_f32_e32 v3, s1, v3
+; CI-NEXT:    v_mul_f32_e32 v2, s0, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_alignbit_b32 v2, v3, v2, 16
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_fabs_fold_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v2, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, 0x7fff
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_lshl_b32 s0, s4, 16
+; VI-NEXT:    s_and_b32 s1, s4, 0xffff0000
+; VI-NEXT:    s_movk_i32 s2, 0x7fff
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_sdwa v4, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_and_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_mul_f32_e32 v3, s0, v4
+; VI-NEXT:    v_mul_f32_e32 v2, s1, v2
+; VI-NEXT:    v_bfe_u32 v4, v3, 16, 1
+; VI-NEXT:    v_bfe_u32 v6, v2, 16, 1
+; VI-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
+; VI-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s2, v4
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 0x7fff, v6
+; VI-NEXT:    v_or_b32_e32 v5, 0x400000, v3
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; VI-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; VI-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v6, v7, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT:    v_alignbit_b32 v2, v2, v3, 16
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fabs_fold_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    s_lshl_b32 s3, s4, 16
+; GFX9-NEXT:    s_and_b32 s4, s4, 0xffff0000
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_sdwa v2, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v0, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_mul_f32_e32 v2, s3, v2
+; GFX9-NEXT:    v_mul_f32_e32 v0, s4, v0
+; GFX9-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT:    v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v2, s2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_or_b32_e32 v6, 0x400000, v0
+; GFX9-NEXT:    v_add3_u32 v5, v5, v0, s2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT:    s_mov_b32 s2, 0x7060302
+; GFX9-NEXT:    v_perm_b32 v0, v0, v2, s2
+; GFX9-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_fabs_fold_v2bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x10
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_b32 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s2, s4, 16
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0x7fff, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT:    v_dual_mul_f32 v0, s2, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    s_and_b32 s2, s4, 0xffff0000
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mul_f32_e32 v1, s2, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_fabs_fold_v2bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_b32 v0, v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s2, s4, 16
+; GFX11-FAKE16-NEXT:    s_and_b32 s3, s4, 0xffff0000
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT:    v_dual_mul_f32 v0, s2, v0 :: v_dual_and_b32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v2, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_mul_f32 v1, s3, v1 :: v_dual_mov_b32 v2, 0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %val = load <2 x bfloat>, ptr addrspace(1) %gep
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)
+  %other.val.cvt = bitcast i32 %other.val to <2 x bfloat>
+  %fmul = fmul <2 x bfloat> %fabs, %other.val.cvt
+  store <2 x bfloat> %fmul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_extract_fabs_fold_v2bf16(ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_extract_fabs_fold_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v0, v[0:1]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; CI-NEXT:    v_mul_f32_e64 v1, 1.0, |v1|
+; CI-NEXT:    v_mul_f32_e64 v0, 1.0, |v0|
+; CI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; CI-NEXT:    v_mul_f32_e32 v1, 4.0, v1
+; CI-NEXT:    v_add_f32_e32 v0, 2.0, v0
+; CI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    flat_store_short v[0:1], v1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    flat_store_short v[0:1], v0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_extract_fabs_fold_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, 0x7fff
+; VI-NEXT:    s_movk_i32 s0, 0x7fff
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_sdwa v2, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT:    v_mul_f32_e32 v1, 4.0, v2
+; VI-NEXT:    v_add_f32_e32 v0, 2.0, v0
+; VI-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; VI-NEXT:    v_bfe_u32 v4, v0, 16, 1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v1
+; VI-NEXT:    v_add_u32_e32 v4, vcc, v4, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 0x7fff, v4
+; VI-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; VI-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; VI-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; VI-NEXT:    flat_store_short v[0:1], v1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    flat_store_short v[0:1], v0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_extract_fabs_fold_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_sdwa v1, s0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v0, s0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 4.0, v1
+; GFX9-NEXT:    v_add_f32_e32 v0, 2.0, v0
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v2, v2, v1, s0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX9-NEXT:    v_add3_u32 v4, v4, v0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT:    global_store_short_d16_hi v[0:1], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_short_d16_hi v[0:1], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_extract_fabs_fold_v2bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_b32 v0, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0x7fff, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT:    v_dual_mul_f32 v0, 4.0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v1, 2.0, v1
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_d16_hi_b16 v[0:1], v0, off dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT:    global_store_d16_hi_b16 v[0:1], v1, off dlc
+; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_extract_fabs_fold_v2bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_b32 v0, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT:    v_dual_mul_f32 v0, 4.0, v0 :: v_dual_and_b32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v2, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-FAKE16-NEXT:    v_add_f32_e32 v1, 2.0, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_d16_hi_b16 v[0:1], v0, off dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT:    global_store_d16_hi_b16 v[0:1], v1, off dlc
+; GFX11-FAKE16-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %val = load <2 x bfloat>, ptr addrspace(1) %gep.in
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)
+  %elt0 = extractelement <2 x bfloat> %fabs, i32 0
+  %elt1 = extractelement <2 x bfloat> %fabs, i32 1
+
+  %fmul0 = fmul bfloat %elt0, 4.0
+  %fadd1 = fadd bfloat %elt1, 2.0
+  store volatile bfloat %fmul0, ptr addrspace(1) poison
+  store volatile bfloat %fadd1, ptr addrspace(1) poison
+  ret void
+}
+
+define amdgpu_kernel void @v_extract_fabs_no_fold_v2bf16(ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_extract_fabs_no_fold_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v0, v[0:1]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_bfe_u32 v1, v0, 16, 15
+; CI-NEXT:    v_and_b32_e32 v0, 0x7fff7fff, v0
+; CI-NEXT:    flat_store_short v[0:1], v0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    flat_store_short v[0:1], v1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_extract_fabs_no_fold_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 0x7fff7fff, v0
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; VI-NEXT:    flat_store_short v[0:1], v0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    flat_store_short v[0:1], v1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_extract_fabs_no_fold_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX9-NEXT:    global_store_short v[0:1], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_short_d16_hi v[0:1], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_extract_fabs_no_fold_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v0, v0, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX11-NEXT:    global_store_b16 v[0:1], v0, off dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_d16_hi_b16 v[0:1], v0, off dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %val = load <2 x bfloat>, ptr addrspace(1) %gep.in
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)
+  %elt0 = extractelement <2 x bfloat> %fabs, i32 0
+  %elt1 = extractelement <2 x bfloat> %fabs, i32 1
+  store volatile bfloat %elt0, ptr addrspace(1) poison
+  store volatile bfloat %elt1, ptr addrspace(1) poison
+  ret void
+}
+
+declare bfloat @llvm.fabs.bf16(bfloat) #1
+declare <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat>) #1
+declare <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat>) #1
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
index 60334e46a445..52bcaed7ec75 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
@@ -153,14 +153,14 @@ define i32 @global_load_2xi16_align1(ptr addrspace(1) %p) #0 {
 ; GFX7-ALIGNED-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
 ; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v6, vcc, 3, v0
 ; GFX7-ALIGNED-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; GFX7-ALIGNED-NEXT:    flat_load_ubyte v6, v[6:7]
 ; GFX7-ALIGNED-NEXT:    flat_load_ubyte v4, v[4:5]
+; GFX7-ALIGNED-NEXT:    flat_load_ubyte v5, v[6:7]
 ; GFX7-ALIGNED-NEXT:    flat_load_ubyte v2, v[2:3]
 ; GFX7-ALIGNED-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
-; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(2)
 ; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 8, v4
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 24, v5
 ; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(1)
 ; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
index f9694dcd89ab..6f8da57e223e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
@@ -204,14 +204,14 @@ define i32 @private_load_2xi16_align1(ptr addrspace(5) %p) #0 {
 ; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v1, vcc, 2, v0
 ; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
 ; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v3, vcc, 3, v0
-; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v3, v3, s[0:3], 0 offen
 ; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen
+; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v3, v3, s[0:3], 0 offen
 ; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen
 ; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v0, v0, s[0:3], 0 offen
 ; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
-; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(2)
 ; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
 ; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(1)
 ; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
index fab45c9dc3bc..61f5b73033f5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
@@ -569,14 +569,13 @@ define amdgpu_kernel void @s_test_copysign_v3f32(ptr addrspace(1) %out, <3 x flo
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    s_load_b256 s[8:15], s[4:5], 0x34
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, s12
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s14 :: v_dual_mov_b32 v1, s13
-; GFX11-NEXT:    v_mov_b32_e32 v3, s12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_bfi_b32 v2, 0x7fffffff, s10, v0
 ; GFX11-NEXT:    v_bfi_b32 v1, 0x7fffffff, s9, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-NEXT:    v_bfi_b32 v0, 0x7fffffff, s8, v3
 ; GFX11-NEXT:    global_store_b96 v4, v[0:2], s[0:1]
 ; GFX11-NEXT:    s_endpgm
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
index f9db082a2e91..9b6a2f3a1aa1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
@@ -57,6 +57,57 @@ body:             |
     %4:vgpr_16 = V_CVT_F16_U16_t16_e64 0, %3:sreg_32, 0, 0, 0, implicit $mode, implicit $exec
 ...
 
+---
+name:            salu16_usedby_salu32
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: salu16_usedby_salu32
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_TRUNC_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_TRUNC_F16_t16_e64 0, [[DEF]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_TRUNC_F16_t16_e64_]], %subreg.lo16, [[DEF2]], %subreg.hi16
+    ; GCN-NEXT: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[REG_SEQUENCE]], [[DEF]], implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:sreg_32 = COPY %0:vgpr_32
+    %2:sreg_32 = S_TRUNC_F16 %1:sreg_32, implicit $mode
+    %3:sreg_32 = S_XOR_B32 %2:sreg_32, %1:sreg_32, implicit-def $scc
+...
+
+---
+name:            salu32_usedby_salu16
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: salu32_usedby_salu16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[DEF]], [[DEF]], implicit $exec
+    ; GCN-NEXT: [[V_TRUNC_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_TRUNC_F16_t16_e64 0, [[V_XOR_B32_e64_]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:sreg_32 = COPY %0:vgpr_32
+    %2:sreg_32 = S_XOR_B32 %1:sreg_32, %1:sreg_32, implicit-def $scc
+    %3:sreg_32 = S_TRUNC_F16 %2:sreg_32, implicit $mode
+...
+
+---
+name:            S_FMAC_F16
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: S_FMAC_F16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sgpr_lo16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[DEF]], %subreg.lo16, [[DEF2]], %subreg.hi16
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[DEF]], %subreg.lo16, [[DEF3]], %subreg.hi16
+    ; GCN-NEXT: [[V_FMAC_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_FMAC_F16_t16_e64 0, [[REG_SEQUENCE1]].lo16, 0, [[REG_SEQUENCE1]].lo16, 0, [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vgpr_16 = IMPLICIT_DEF
+    %1:sgpr_lo16 = COPY %0:vgpr_16
+    %2:sreg_32 = COPY %0:vgpr_16
+    %3:sreg_32 = COPY %1:sgpr_lo16
+    %4:sreg_32 = S_FMAC_F16 %3:sreg_32, %3:sreg_32, %2:sreg_32, implicit $mode
+...
+
 ---
 name:            vgpr16_to_spgr32
 body:             |
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index e13c895a1cc8..cfe4d24d427e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -5758,29 +5758,38 @@ define double @flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    v_mov_b32_e32 v5, v1
-; GFX942-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-NEXT:    s_mov_b64 s[0:1], src_shared_base
 ; GFX942-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s1, v5
 ; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX942-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX942-NEXT:    s_cbranch_execnz .LBB30_3
-; GFX942-NEXT:  ; %bb.1: ; %Flow
+; GFX942-NEXT:  ; %bb.1: ; %Flow2
 ; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execnz .LBB30_4
+; GFX942-NEXT:    s_cbranch_execnz .LBB30_8
 ; GFX942-NEXT:  .LBB30_2: ; %atomicrmw.phi
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-; GFX942-NEXT:  .LBB30_3: ; %atomicrmw.global
+; GFX942-NEXT:  .LBB30_3: ; %atomicrmw.check.private
+; GFX942-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s3, v5
+; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB30_5
+; GFX942-NEXT:  ; %bb.4: ; %atomicrmw.global
 ; GFX942-NEXT:    buffer_wbl2 sc1
-; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[4:5], v[2:3] sc0
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[4:5], v[2:3], off sc0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execz .LBB30_2
-; GFX942-NEXT:  .LBB30_4: ; %atomicrmw.private
+; GFX942-NEXT:  .LBB30_5: ; %Flow
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB30_7
+; GFX942-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX942-NEXT:    s_nop 1
 ; GFX942-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
@@ -5788,6 +5797,18 @@ define double @flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
 ; GFX942-NEXT:    scratch_store_dwordx2 v4, v[2:3], off
+; GFX942-NEXT:  .LBB30_7: ; %Flow1
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB30_2
+; GFX942-NEXT:  .LBB30_8: ; %atomicrmw.shared
+; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_cndmask_b32_e32 v0, -1, v4, vcc
+; GFX942-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
@@ -5894,28 +5915,37 @@ define double @flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
-; GFX90A-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX90A-NEXT:    s_mov_b64 s[4:5], src_shared_base
 ; GFX90A-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v5
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB30_3
-; GFX90A-NEXT:  ; %bb.1: ; %Flow
+; GFX90A-NEXT:  ; %bb.1: ; %Flow2
 ; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB30_4
+; GFX90A-NEXT:    s_cbranch_execnz .LBB30_8
 ; GFX90A-NEXT:  .LBB30_2: ; %atomicrmw.phi
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
-; GFX90A-NEXT:  .LBB30_3: ; %atomicrmw.global
-; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[4:5], v[2:3] glc
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:  .LBB30_3: ; %atomicrmw.check.private
+; GFX90A-NEXT:    s_mov_b64 s[6:7], src_private_base
+; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s7, v5
+; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB30_5
+; GFX90A-NEXT:  ; %bb.4: ; %atomicrmw.global
+; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[4:5], v[2:3], off glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB30_2
-; GFX90A-NEXT:  .LBB30_4: ; %atomicrmw.private
+; GFX90A-NEXT:  .LBB30_5: ; %Flow
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB30_7
+; GFX90A-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GFX90A-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -5924,6 +5954,17 @@ define double @flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX90A-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
 ; GFX90A-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
 ; GFX90A-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT:  .LBB30_7: ; %Flow1
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT:    s_cbranch_execz .LBB30_2
+; GFX90A-NEXT:  .LBB30_8: ; %atomicrmw.shared
+; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, -1, v4, vcc
+; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -6160,28 +6201,37 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    s_mov_b64 s[0:1], 0x7f8
 ; GFX942-NEXT:    v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-NEXT:    s_mov_b64 s[0:1], src_shared_base
 ; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s1, v5
 ; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX942-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX942-NEXT:    s_cbranch_execnz .LBB31_3
-; GFX942-NEXT:  ; %bb.1: ; %Flow
+; GFX942-NEXT:  ; %bb.1: ; %Flow2
 ; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execnz .LBB31_4
+; GFX942-NEXT:    s_cbranch_execnz .LBB31_8
 ; GFX942-NEXT:  .LBB31_2: ; %atomicrmw.phi
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-; GFX942-NEXT:  .LBB31_3: ; %atomicrmw.global
+; GFX942-NEXT:  .LBB31_3: ; %atomicrmw.check.private
+; GFX942-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s3, v5
+; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB31_5
+; GFX942-NEXT:  ; %bb.4: ; %atomicrmw.global
 ; GFX942-NEXT:    buffer_wbl2 sc1
-; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[4:5], v[2:3] sc0
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[4:5], v[2:3], off sc0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execz .LBB31_2
-; GFX942-NEXT:  .LBB31_4: ; %atomicrmw.private
+; GFX942-NEXT:  .LBB31_5: ; %Flow
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB31_7
+; GFX942-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX942-NEXT:    s_nop 1
 ; GFX942-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
@@ -6189,6 +6239,18 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
 ; GFX942-NEXT:    scratch_store_dwordx2 v4, v[2:3], off
+; GFX942-NEXT:  .LBB31_7: ; %Flow1
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB31_2
+; GFX942-NEXT:  .LBB31_8: ; %atomicrmw.shared
+; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_cndmask_b32_e32 v0, -1, v4, vcc
+; GFX942-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
@@ -6306,27 +6368,36 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0x7f8, v0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX90A-NEXT:    s_mov_b64 s[4:5], src_shared_base
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v5
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB31_3
-; GFX90A-NEXT:  ; %bb.1: ; %Flow
+; GFX90A-NEXT:  ; %bb.1: ; %Flow2
 ; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB31_4
+; GFX90A-NEXT:    s_cbranch_execnz .LBB31_8
 ; GFX90A-NEXT:  .LBB31_2: ; %atomicrmw.phi
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
-; GFX90A-NEXT:  .LBB31_3: ; %atomicrmw.global
-; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[4:5], v[2:3] glc
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:  .LBB31_3: ; %atomicrmw.check.private
+; GFX90A-NEXT:    s_mov_b64 s[6:7], src_private_base
+; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s7, v5
+; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB31_5
+; GFX90A-NEXT:  ; %bb.4: ; %atomicrmw.global
+; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[4:5], v[2:3], off glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB31_2
-; GFX90A-NEXT:  .LBB31_4: ; %atomicrmw.private
+; GFX90A-NEXT:  .LBB31_5: ; %Flow
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB31_7
+; GFX90A-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GFX90A-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -6335,6 +6406,17 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
 ; GFX90A-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
 ; GFX90A-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT:  .LBB31_7: ; %Flow1
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT:    s_cbranch_execz .LBB31_2
+; GFX90A-NEXT:  .LBB31_8: ; %atomicrmw.shared
+; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, -1, v4, vcc
+; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -6581,28 +6663,37 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_movk_i32 s0, 0xf800
 ; GFX942-NEXT:    s_mov_b32 s1, -1
 ; GFX942-NEXT:    v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-NEXT:    s_mov_b64 s[0:1], src_shared_base
 ; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s1, v5
 ; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX942-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX942-NEXT:    s_cbranch_execnz .LBB32_3
-; GFX942-NEXT:  ; %bb.1: ; %Flow
+; GFX942-NEXT:  ; %bb.1: ; %Flow2
 ; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execnz .LBB32_4
+; GFX942-NEXT:    s_cbranch_execnz .LBB32_8
 ; GFX942-NEXT:  .LBB32_2: ; %atomicrmw.phi
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-; GFX942-NEXT:  .LBB32_3: ; %atomicrmw.global
+; GFX942-NEXT:  .LBB32_3: ; %atomicrmw.check.private
+; GFX942-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s3, v5
+; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB32_5
+; GFX942-NEXT:  ; %bb.4: ; %atomicrmw.global
 ; GFX942-NEXT:    buffer_wbl2 sc1
-; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[4:5], v[2:3] sc0
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[4:5], v[2:3], off sc0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execz .LBB32_2
-; GFX942-NEXT:  .LBB32_4: ; %atomicrmw.private
+; GFX942-NEXT:  .LBB32_5: ; %Flow
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB32_7
+; GFX942-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX942-NEXT:    s_nop 1
 ; GFX942-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
@@ -6610,6 +6701,18 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
 ; GFX942-NEXT:    scratch_store_dwordx2 v4, v[2:3], off
+; GFX942-NEXT:  .LBB32_7: ; %Flow1
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB32_2
+; GFX942-NEXT:  .LBB32_8: ; %atomicrmw.shared
+; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_cndmask_b32_e32 v0, -1, v4, vcc
+; GFX942-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
@@ -6727,27 +6830,36 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX90A-NEXT:    s_mov_b64 s[4:5], src_shared_base
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v5
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB32_3
-; GFX90A-NEXT:  ; %bb.1: ; %Flow
+; GFX90A-NEXT:  ; %bb.1: ; %Flow2
 ; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB32_4
+; GFX90A-NEXT:    s_cbranch_execnz .LBB32_8
 ; GFX90A-NEXT:  .LBB32_2: ; %atomicrmw.phi
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
-; GFX90A-NEXT:  .LBB32_3: ; %atomicrmw.global
-; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[4:5], v[2:3] glc
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:  .LBB32_3: ; %atomicrmw.check.private
+; GFX90A-NEXT:    s_mov_b64 s[6:7], src_private_base
+; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s7, v5
+; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB32_5
+; GFX90A-NEXT:  ; %bb.4: ; %atomicrmw.global
+; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[4:5], v[2:3], off glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB32_2
-; GFX90A-NEXT:  .LBB32_4: ; %atomicrmw.private
+; GFX90A-NEXT:  .LBB32_5: ; %Flow
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB32_7
+; GFX90A-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GFX90A-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -6756,6 +6868,17 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
 ; GFX90A-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
 ; GFX90A-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT:  .LBB32_7: ; %Flow1
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT:    s_cbranch_execz .LBB32_2
+; GFX90A-NEXT:  .LBB32_8: ; %atomicrmw.shared
+; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, -1, v4, vcc
+; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -6994,27 +7117,35 @@ define void @flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX942-LABEL: flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-NEXT:    s_mov_b64 s[0:1], src_shared_base
 ; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s1, v1
 ; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX942-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX942-NEXT:    s_cbranch_execnz .LBB33_3
-; GFX942-NEXT:  ; %bb.1: ; %Flow
+; GFX942-NEXT:  ; %bb.1: ; %Flow2
 ; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execnz .LBB33_4
+; GFX942-NEXT:    s_cbranch_execnz .LBB33_8
 ; GFX942-NEXT:  .LBB33_2: ; %atomicrmw.phi
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-; GFX942-NEXT:  .LBB33_3: ; %atomicrmw.global
+; GFX942-NEXT:  .LBB33_3: ; %atomicrmw.check.private
+; GFX942-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s3, v1
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB33_5
+; GFX942-NEXT:  ; %bb.4: ; %atomicrmw.global
 ; GFX942-NEXT:    buffer_wbl2 sc1
-; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execz .LBB33_2
-; GFX942-NEXT:  .LBB33_4: ; %atomicrmw.private
+; GFX942-NEXT:  .LBB33_5: ; %Flow
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB33_7
+; GFX942-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GFX942-NEXT:    s_nop 1
 ; GFX942-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
@@ -7022,6 +7153,18 @@ define void @flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX942-NEXT:    scratch_store_dwordx2 v4, v[0:1], off
+; GFX942-NEXT:  .LBB33_7: ; %Flow1
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB33_2
+; GFX942-NEXT:  .LBB33_8: ; %atomicrmw.shared
+; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GFX942-NEXT:    ds_add_f64 v0, v[2:3]
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
@@ -7129,26 +7272,34 @@ define void @flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX90A-LABEL: flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX90A-NEXT:    s_mov_b64 s[4:5], src_shared_base
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v1
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB33_3
-; GFX90A-NEXT:  ; %bb.1: ; %Flow
+; GFX90A-NEXT:  ; %bb.1: ; %Flow2
 ; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB33_4
+; GFX90A-NEXT:    s_cbranch_execnz .LBB33_8
 ; GFX90A-NEXT:  .LBB33_2: ; %atomicrmw.phi
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
-; GFX90A-NEXT:  .LBB33_3: ; %atomicrmw.global
-; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:  .LBB33_3: ; %atomicrmw.check.private
+; GFX90A-NEXT:    s_mov_b64 s[6:7], src_private_base
+; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s7, v1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB33_5
+; GFX90A-NEXT:  ; %bb.4: ; %atomicrmw.global
+; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB33_2
-; GFX90A-NEXT:  .LBB33_4: ; %atomicrmw.private
+; GFX90A-NEXT:  .LBB33_5: ; %Flow
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB33_7
+; GFX90A-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GFX90A-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -7157,6 +7308,17 @@ define void @flat_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(ptr
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX90A-NEXT:    buffer_store_dword v0, v4, s[0:3], 0 offen
 ; GFX90A-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT:  .LBB33_7: ; %Flow1
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT:    s_cbranch_execz .LBB33_2
+; GFX90A-NEXT:  .LBB33_8: ; %atomicrmw.shared
+; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GFX90A-NEXT:    ds_add_f64 v0, v[2:3]
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -7390,27 +7552,35 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    s_mov_b64 s[0:1], 0x7f8
 ; GFX942-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-NEXT:    s_mov_b64 s[0:1], src_shared_base
 ; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s1, v1
 ; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX942-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX942-NEXT:    s_cbranch_execnz .LBB34_3
-; GFX942-NEXT:  ; %bb.1: ; %Flow
+; GFX942-NEXT:  ; %bb.1: ; %Flow2
 ; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execnz .LBB34_4
+; GFX942-NEXT:    s_cbranch_execnz .LBB34_8
 ; GFX942-NEXT:  .LBB34_2: ; %atomicrmw.phi
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-; GFX942-NEXT:  .LBB34_3: ; %atomicrmw.global
+; GFX942-NEXT:  .LBB34_3: ; %atomicrmw.check.private
+; GFX942-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s3, v1
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB34_5
+; GFX942-NEXT:  ; %bb.4: ; %atomicrmw.global
 ; GFX942-NEXT:    buffer_wbl2 sc1
-; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execz .LBB34_2
-; GFX942-NEXT:  .LBB34_4: ; %atomicrmw.private
+; GFX942-NEXT:  .LBB34_5: ; %Flow
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB34_7
+; GFX942-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GFX942-NEXT:    s_nop 1
 ; GFX942-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
@@ -7418,6 +7588,18 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX942-NEXT:    scratch_store_dwordx2 v4, v[0:1], off
+; GFX942-NEXT:  .LBB34_7: ; %Flow1
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB34_2
+; GFX942-NEXT:  .LBB34_8: ; %atomicrmw.shared
+; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GFX942-NEXT:    ds_add_f64 v0, v[2:3]
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
@@ -7532,26 +7714,34 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0x7f8, v0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX90A-NEXT:    s_mov_b64 s[4:5], src_shared_base
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v1
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB34_3
-; GFX90A-NEXT:  ; %bb.1: ; %Flow
+; GFX90A-NEXT:  ; %bb.1: ; %Flow2
 ; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB34_4
+; GFX90A-NEXT:    s_cbranch_execnz .LBB34_8
 ; GFX90A-NEXT:  .LBB34_2: ; %atomicrmw.phi
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
-; GFX90A-NEXT:  .LBB34_3: ; %atomicrmw.global
-; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:  .LBB34_3: ; %atomicrmw.check.private
+; GFX90A-NEXT:    s_mov_b64 s[6:7], src_private_base
+; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s7, v1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB34_5
+; GFX90A-NEXT:  ; %bb.4: ; %atomicrmw.global
+; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB34_2
-; GFX90A-NEXT:  .LBB34_4: ; %atomicrmw.private
+; GFX90A-NEXT:  .LBB34_5: ; %Flow
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB34_7
+; GFX90A-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GFX90A-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -7560,6 +7750,17 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX90A-NEXT:    buffer_store_dword v0, v4, s[0:3], 0 offen
 ; GFX90A-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT:  .LBB34_7: ; %Flow1
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT:    s_cbranch_execz .LBB34_2
+; GFX90A-NEXT:  .LBB34_8: ; %atomicrmw.shared
+; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GFX90A-NEXT:    ds_add_f64 v0, v[2:3]
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
@@ -7801,27 +8002,35 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_movk_i32 s0, 0xf800
 ; GFX942-NEXT:    s_mov_b32 s1, -1
 ; GFX942-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942-NEXT:    s_mov_b64 s[0:1], src_private_base
+; GFX942-NEXT:    s_mov_b64 s[0:1], src_shared_base
 ; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s1, v1
 ; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX942-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX942-NEXT:    s_cbranch_execnz .LBB35_3
-; GFX942-NEXT:  ; %bb.1: ; %Flow
+; GFX942-NEXT:  ; %bb.1: ; %Flow2
 ; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execnz .LBB35_4
+; GFX942-NEXT:    s_cbranch_execnz .LBB35_8
 ; GFX942-NEXT:  .LBB35_2: ; %atomicrmw.phi
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-; GFX942-NEXT:  .LBB35_3: ; %atomicrmw.global
+; GFX942-NEXT:  .LBB35_3: ; %atomicrmw.check.private
+; GFX942-NEXT:    s_mov_b64 s[2:3], src_private_base
+; GFX942-NEXT:    v_cmp_ne_u32_e32 vcc, s3, v1
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB35_5
+; GFX942-NEXT:  ; %bb.4: ; %atomicrmw.global
 ; GFX942-NEXT:    buffer_wbl2 sc1
-; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT:    s_cbranch_execz .LBB35_2
-; GFX942-NEXT:  .LBB35_4: ; %atomicrmw.private
+; GFX942-NEXT:  .LBB35_5: ; %Flow
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB35_7
+; GFX942-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GFX942-NEXT:    s_nop 1
 ; GFX942-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
@@ -7829,6 +8038,18 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX942-NEXT:    scratch_store_dwordx2 v4, v[0:1], off
+; GFX942-NEXT:  .LBB35_7: ; %Flow1
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB35_2
+; GFX942-NEXT:  .LBB35_8: ; %atomicrmw.shared
+; GFX942-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GFX942-NEXT:    ds_add_f64 v0, v[2:3]
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
@@ -7943,26 +8164,34 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT:    s_mov_b64 s[4:5], src_private_base
+; GFX90A-NEXT:    s_mov_b64 s[4:5], src_shared_base
 ; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v1
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB35_3
-; GFX90A-NEXT:  ; %bb.1: ; %Flow
+; GFX90A-NEXT:  ; %bb.1: ; %Flow2
 ; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execnz .LBB35_4
+; GFX90A-NEXT:    s_cbranch_execnz .LBB35_8
 ; GFX90A-NEXT:  .LBB35_2: ; %atomicrmw.phi
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
-; GFX90A-NEXT:  .LBB35_3: ; %atomicrmw.global
-; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:  .LBB35_3: ; %atomicrmw.check.private
+; GFX90A-NEXT:    s_mov_b64 s[6:7], src_private_base
+; GFX90A-NEXT:    v_cmp_ne_u32_e32 vcc, s7, v1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB35_5
+; GFX90A-NEXT:  ; %bb.4: ; %atomicrmw.global
+; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB35_2
-; GFX90A-NEXT:  .LBB35_4: ; %atomicrmw.private
+; GFX90A-NEXT:  .LBB35_5: ; %Flow
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX90A-NEXT:    s_cbranch_execz .LBB35_7
+; GFX90A-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GFX90A-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GFX90A-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -7971,6 +8200,17 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_gra
 ; GFX90A-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX90A-NEXT:    buffer_store_dword v0, v4, s[0:3], 0 offen
 ; GFX90A-NEXT:    buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT:  .LBB35_7: ; %Flow1
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX90A-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; GFX90A-NEXT:    s_cbranch_execz .LBB35_2
+; GFX90A-NEXT:  .LBB35_8: ; %atomicrmw.shared
+; GFX90A-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GFX90A-NEXT:    ds_add_f64 v0, v[2:3]
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat_atomics.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
index e674b57aae3e..3304dbf3eaa3 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
@@ -2,6 +2,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefixes=GCN1 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=GCN2 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN3 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 define amdgpu_kernel void @atomic_add_i32_offset(ptr %out, i32 %in) {
 ; GCN1-LABEL: atomic_add_i32_offset:
@@ -46,6 +48,21 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_add_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw add ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -95,6 +112,21 @@ define amdgpu_kernel void @atomic_add_i32_max_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_max_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_add_u32 v[0:1], v2 offset:4092
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1023
   %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -146,6 +178,23 @@ define amdgpu_kernel void @atomic_add_i32_max_offset_p1(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_max_offset_p1:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_add_co_u32 v0, s0, 0x1000, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, s1, s0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_add_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1024
   %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -204,6 +253,22 @@ define amdgpu_kernel void @atomic_add_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_add_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -266,6 +331,26 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_add_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -337,6 +422,27 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_add_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -384,6 +490,21 @@ define amdgpu_kernel void @atomic_add_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_add_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile add ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -437,6 +558,22 @@ define amdgpu_kernel void @atomic_add_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_add_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile add ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -494,6 +631,26 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_add_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile add ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -560,6 +717,27 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_add_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile add ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -610,6 +788,21 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_and_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -668,6 +861,22 @@ define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_and_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -730,6 +939,26 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_and_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -801,6 +1030,27 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_and_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -848,6 +1098,21 @@ define amdgpu_kernel void @atomic_and_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_and_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -901,6 +1166,22 @@ define amdgpu_kernel void @atomic_and_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_and_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -958,6 +1239,26 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_and_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -1024,6 +1325,27 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_and_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -1074,6 +1396,21 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_sub_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -1132,6 +1469,22 @@ define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_sub_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -1194,6 +1547,26 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_sub_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -1265,6 +1638,27 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_sub_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -1312,6 +1706,21 @@ define amdgpu_kernel void @atomic_sub_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_sub_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -1365,6 +1774,22 @@ define amdgpu_kernel void @atomic_sub_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_sub_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -1422,6 +1847,26 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_sub_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -1488,6 +1933,27 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_sub_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -1535,6 +2001,20 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_smax v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_max_i32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -1593,6 +2073,21 @@ define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_i32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -1652,6 +2147,25 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    flat_atomic_smax v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_i32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -1723,6 +2237,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_max_i32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -1767,6 +2301,20 @@ define amdgpu_kernel void @atomic_max_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_smax v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_max_i32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -1820,6 +2368,21 @@ define amdgpu_kernel void @atomic_max_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_i32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr %out2
@@ -1874,6 +2437,25 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    flat_atomic_smax v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_i32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -1940,6 +2522,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_max_i32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -1987,6 +2589,20 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_umax v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_max_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2045,6 +2661,21 @@ define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr %out, ptr %out2, i32 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2104,6 +2735,25 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i64
 ; GCN3-NEXT:    flat_atomic_umax v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -2175,6 +2825,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_max_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -2219,6 +2889,20 @@ define amdgpu_kernel void @atomic_umax_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_umax v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_max_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -2272,6 +2956,21 @@ define amdgpu_kernel void @atomic_umax_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr %out2
@@ -2326,6 +3025,25 @@ define amdgpu_kernel void @atomic_umax_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    flat_atomic_umax v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_max_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2392,6 +3110,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_max_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2439,6 +3177,20 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_smin v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_min_i32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2497,6 +3249,21 @@ define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_i32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2556,6 +3323,25 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    flat_atomic_smin v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_i32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -2627,6 +3413,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_min_i32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -2671,6 +3477,20 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_smin v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_min_i32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -2724,6 +3544,21 @@ define amdgpu_kernel void @atomic_min_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_i32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr %out2
@@ -2778,6 +3613,25 @@ define amdgpu_kernel void @atomic_min_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    flat_atomic_smin v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_i32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2844,6 +3698,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_min_i32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2891,6 +3765,20 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_umin v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_min_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2949,6 +3837,21 @@ define amdgpu_kernel void @atomic_umin_i32_ret_offset(ptr %out, ptr %out2, i32 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst
@@ -3008,6 +3911,25 @@ define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr %out, i32 %in, i64
 ; GCN3-NEXT:    flat_atomic_umin v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -3079,6 +4001,26 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64_offset(ptr %out, ptr %out2
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_min_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -3123,6 +4065,20 @@ define amdgpu_kernel void @atomic_umin_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    flat_atomic_umin v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_min_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -3176,6 +4132,21 @@ define amdgpu_kernel void @atomic_umin_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr %out2
@@ -3230,6 +4201,25 @@ define amdgpu_kernel void @atomic_umin_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    flat_atomic_umin v[0:1], v2
 ; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_min_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -3296,6 +4286,26 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64(ptr %out, ptr %out2, i32 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_min_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -3346,6 +4356,21 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_or_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -3404,6 +4429,22 @@ define amdgpu_kernel void @atomic_or_i32_ret_offset(ptr %out, ptr %out2, i32 %in
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_or_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -3466,6 +4507,26 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr %out, i32 %in, i64 %i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_or_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -3537,6 +4598,27 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_or_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -3584,6 +4666,21 @@ define amdgpu_kernel void @atomic_or_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_or_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -3637,6 +4734,22 @@ define amdgpu_kernel void @atomic_or_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_or_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -3694,6 +4807,26 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr %out, i32 %in, i64 %index) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_or_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -3760,6 +4893,27 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64(ptr %out, ptr %out2, i32 %in
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_or_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -3810,6 +4964,21 @@ define amdgpu_kernel void @atomic_xchg_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_swap_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile xchg ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -3859,6 +5028,21 @@ define amdgpu_kernel void @atomic_xchg_f32_offset(ptr %out, float %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_f32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_swap_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr float, ptr %out, i32 4
   %val = atomicrmw volatile xchg ptr %gep, float %in syncscope("agent") seq_cst
@@ -3917,6 +5101,22 @@ define amdgpu_kernel void @atomic_xchg_i32_ret_offset(ptr %out, ptr %out2, i32 %
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile xchg ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -3979,6 +5179,26 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr %out, i32 %in, i64
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_swap_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -4050,6 +5270,27 @@ define amdgpu_kernel void @atomic_xchg_i32_ret_addr64_offset(ptr %out, ptr %out2
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -4097,6 +5338,21 @@ define amdgpu_kernel void @atomic_xchg_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_swap_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -4150,6 +5406,22 @@ define amdgpu_kernel void @atomic_xchg_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -4207,6 +5479,26 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_swap_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile xchg ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -4273,6 +5565,27 @@ define amdgpu_kernel void @atomic_xchg_i32_ret_addr64(ptr %out, ptr %out2, i32 %
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile xchg ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -4325,6 +5638,19 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr %out, i32 %in, i32 %old
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4386,6 +5712,22 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr %out, ptr %out2, i3
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4455,6 +5797,27 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr %out, i32 %in, i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x3c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v[2:3], v[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -4532,6 +5895,28 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(ptr %out, ptr %o
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s9, s[4:5], 0x44
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v2, v[2:3], v[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -4580,6 +5965,19 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr %out, i32 %in, i32 %old) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3]
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = cmpxchg volatile ptr %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
   ret void
@@ -4636,6 +6034,22 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr %out, ptr %out2, i32 %in,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = cmpxchg volatile ptr %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
   %flag = extractvalue { i32, i1 } %val, 0
@@ -4700,6 +6114,27 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr %out, i32 %in, i64 %ind
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x3c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v[2:3], v[0:1]
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = cmpxchg volatile ptr %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4772,6 +6207,28 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(ptr %out, ptr %out2, i3
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s9, s[4:5], 0x44
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    flat_atomic_cmpswap_b32 v2, v[2:3], v[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = cmpxchg volatile ptr %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4823,6 +6280,21 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_xor_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -4881,6 +6353,22 @@ define amdgpu_kernel void @atomic_xor_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_xor_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -4943,6 +6431,26 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_xor_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -5014,6 +6522,27 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_xor_b32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -5061,6 +6590,21 @@ define amdgpu_kernel void @atomic_xor_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_xor_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -5114,6 +6658,22 @@ define amdgpu_kernel void @atomic_xor_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_xor_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -5171,6 +6731,26 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_xor_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -5237,6 +6817,27 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_xor_b32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -5290,6 +6891,19 @@ define amdgpu_kernel void @atomic_load_i32_offset(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %in, i32 4
   %val = load atomic i32, ptr %gep  seq_cst, align 4
@@ -5339,6 +6953,19 @@ define amdgpu_kernel void @atomic_load_i32(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = load atomic i32, ptr %in seq_cst, align 4
   store i32 %val, ptr %out
@@ -5403,6 +7030,25 @@ define amdgpu_kernel void @atomic_load_i32_addr64_offset(ptr %in, ptr %out, i64
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %in, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -5465,6 +7111,25 @@ define amdgpu_kernel void @atomic_load_i32_addr64(ptr %in, ptr %out, i64 %index)
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %in, i64 %index
   %val = load atomic i32, ptr %ptr seq_cst, align 4
@@ -5509,6 +7174,17 @@ define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   store atomic i32 %in, ptr %gep  seq_cst, align 4
@@ -5548,6 +7224,17 @@ define amdgpu_kernel void @atomic_store_i32(i32 %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   store atomic i32 %in, ptr %out seq_cst, align 4
   ret void
@@ -5599,6 +7286,21 @@ define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, ptr %out, i64
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -5648,6 +7350,21 @@ define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, ptr %out, i64 %index
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   store atomic i32 %in, ptr %ptr seq_cst, align 4
@@ -5700,6 +7417,19 @@ define amdgpu_kernel void @atomic_load_f32_offset(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr float, ptr %in, i32 4
   %val = load atomic float, ptr %gep  seq_cst, align 4
@@ -5749,6 +7479,19 @@ define amdgpu_kernel void @atomic_load_f32(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = load atomic float, ptr %in seq_cst, align 4
   store float %val, ptr %out
@@ -5813,6 +7556,25 @@ define amdgpu_kernel void @atomic_load_f32_addr64_offset(ptr %in, ptr %out, i64
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr float, ptr %in, i64 %index
   %gep = getelementptr float, ptr %ptr, i32 4
@@ -5875,6 +7637,25 @@ define amdgpu_kernel void @atomic_load_f32_addr64(ptr %in, ptr %out, i64 %index)
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr float, ptr %in, i64 %index
   %val = load atomic float, ptr %ptr seq_cst, align 4
@@ -5919,6 +7700,17 @@ define amdgpu_kernel void @atomic_store_f32_offset(float %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr float, ptr %out, i32 4
   store atomic float %in, ptr %gep  seq_cst, align 4
@@ -5958,6 +7750,17 @@ define amdgpu_kernel void @atomic_store_f32(float %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   store atomic float %in, ptr %out seq_cst, align 4
   ret void
@@ -6009,6 +7812,21 @@ define amdgpu_kernel void @atomic_store_f32_addr64_offset(float %in, ptr %out, i
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr float, ptr %out, i64 %index
   %gep = getelementptr float, ptr %ptr, i32 4
@@ -6058,6 +7876,21 @@ define amdgpu_kernel void @atomic_store_f32_addr64(float %in, ptr %out, i64 %ind
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr float, ptr %out, i64 %index
   store atomic float %in, ptr %ptr seq_cst, align 4
@@ -6110,6 +7943,33 @@ define amdgpu_kernel void @atomic_load_i8_offset(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_byte v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i8_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_u8 v0, v[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b8 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i8_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u8 v2, v[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b8 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr %in, i64 16
   %val = load atomic i8, ptr %gep  seq_cst, align 1
@@ -6159,6 +8019,33 @@ define amdgpu_kernel void @atomic_load_i8(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_byte v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i8:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_u8 v0, v[0:1] glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b8 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i8:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u8 v2, v[0:1] glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b8 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %val = load atomic i8, ptr %in seq_cst, align 1
   store i8 %val, ptr %out
@@ -6220,6 +8107,43 @@ define amdgpu_kernel void @atomic_load_i8_addr64_offset(ptr %in, ptr %out, i64 %
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_byte v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i8_addr64_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-TRUE16-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    flat_load_d16_u8 v0, v[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b8 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i8_addr64_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-FAKE16-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u8 v2, v[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b8 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i8, ptr %in, i64 %index
   %gep = getelementptr i8, ptr %ptr, i64 16
@@ -6265,6 +8189,28 @@ define amdgpu_kernel void @atomic_store_i8_offset(i8 %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_byte v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i8_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b8 v[1:2], v0 offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i8_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b8 v[0:1], v2 offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr %out, i64 16
   store atomic i8 %in, ptr %gep  seq_cst, align 1
@@ -6304,6 +8250,28 @@ define amdgpu_kernel void @atomic_store_i8(i8 %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_byte v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i8:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b8 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i8:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b8 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   store atomic i8 %in, ptr %out seq_cst, align 1
   ret void
@@ -6352,6 +8320,33 @@ define amdgpu_kernel void @atomic_store_i8_addr64_offset(i8 %in, ptr %out, i64 %
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN3-NEXT:    flat_store_byte v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i8_addr64_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-TRUE16-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s4
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    flat_store_b8 v[1:2], v0 offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i8_addr64_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-FAKE16-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT:    flat_store_b8 v[0:1], v2 offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i8, ptr %out, i64 %index
   %gep = getelementptr i8, ptr %ptr, i64 16
@@ -6405,6 +8400,33 @@ define amdgpu_kernel void @atomic_load_i16_offset(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i16_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_b16 v0, v[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i16_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u16 v2, v[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i16, ptr %in, i64 8
   %val = load atomic i16, ptr %gep  seq_cst, align 2
@@ -6454,6 +8476,33 @@ define amdgpu_kernel void @atomic_load_i16(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_b16 v0, v[0:1] glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u16 v2, v[0:1] glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %val = load atomic i16, ptr %in seq_cst, align 2
   store i16 %val, ptr %out
@@ -6518,6 +8567,45 @@ define amdgpu_kernel void @atomic_load_i16_addr64_offset(ptr %in, ptr %out, i64
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i16_addr64_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_lshl_b64 s[4:5], s[6:7], 1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-TRUE16-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    flat_load_d16_b16 v0, v[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i16_addr64_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_lshl_b64 s[4:5], s[6:7], 1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-FAKE16-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u16 v2, v[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i16, ptr %in, i64 %index
   %gep = getelementptr i16, ptr %ptr, i64 8
@@ -6563,6 +8651,28 @@ define amdgpu_kernel void @atomic_store_i16_offset(i16 %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_short v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i16_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0 offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i16_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2 offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i16, ptr %out, i64 8
   store atomic i16 %in, ptr %gep  seq_cst, align 2
@@ -6602,6 +8712,28 @@ define amdgpu_kernel void @atomic_store_i16(i16 %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   store atomic i16 %in, ptr %out seq_cst, align 2
   ret void
@@ -6653,6 +8785,36 @@ define amdgpu_kernel void @atomic_store_i16_addr64_offset(i16 %in, ptr %out, i64
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN3-NEXT:    flat_store_short v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i16_addr64_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_lshl_b64 s[2:3], s[2:3], 1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s4
+; GFX11-TRUE16-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-TRUE16-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0 offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i16_addr64_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_lshl_b64 s[2:3], s[2:3], 1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-FAKE16-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2 offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i16, ptr %out, i64 %index
   %gep = getelementptr i16, ptr %ptr, i64 8
@@ -6697,6 +8859,28 @@ define amdgpu_kernel void @atomic_store_f16_offset(half %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_short v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_f16_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0 offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_f16_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2 offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr half, ptr %out, i64 8
   store atomic half %in, ptr %gep  seq_cst, align 2
@@ -6736,6 +8920,28 @@ define amdgpu_kernel void @atomic_store_f16(half %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   store atomic half %in, ptr %out seq_cst, align 2
   ret void
@@ -6774,6 +8980,28 @@ define amdgpu_kernel void @atomic_store_bf16_offset(bfloat %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_bf16_offset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_bf16_offset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr bfloat, ptr %out, i64 8
   store atomic bfloat %in, ptr %out seq_cst, align 2
   ret void
@@ -6812,6 +9040,28 @@ define amdgpu_kernel void @atomic_store_bf16(bfloat %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
   store atomic bfloat %in, ptr %out seq_cst, align 2
   ret void
 }
@@ -6859,6 +9109,21 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -6908,6 +9173,21 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_max_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2 offset:4092
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1023
   %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -6959,6 +9239,23 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset_p1(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_max_offset_p1:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_add_co_u32 v0, s0, 0x1000, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, s1, s0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1024
   %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -7017,6 +9314,22 @@ define amdgpu_kernel void @atomic_inc_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -7079,6 +9392,26 @@ define amdgpu_kernel void @atomic_inc_i32_incr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_incr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -7150,6 +9483,27 @@ define amdgpu_kernel void @atomic_inc_i32_ret_incr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_ret_incr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -7197,6 +9551,21 @@ define amdgpu_kernel void @atomic_inc_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -7250,6 +9619,22 @@ define amdgpu_kernel void @atomic_inc_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_inc_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -7307,6 +9692,26 @@ define amdgpu_kernel void @atomic_inc_i32_incr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_incr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -7373,6 +9778,27 @@ define amdgpu_kernel void @atomic_inc_i32_ret_incr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_ret_incr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_inc_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -7423,6 +9849,21 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_dec_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -7472,6 +9913,21 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_max_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_dec_u32 v[0:1], v2 offset:4092
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1023
   %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -7523,6 +9979,23 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset_p1(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_max_offset_p1:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_add_co_u32 v0, s0, 0x1000, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, s1, s0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_dec_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1024
   %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -7581,6 +10054,22 @@ define amdgpu_kernel void @atomic_dec_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
   %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
@@ -7643,6 +10132,26 @@ define amdgpu_kernel void @atomic_dec_i32_decr64_offset(ptr %out, i32 %in, i64 %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_decr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_dec_u32 v[0:1], v2 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -7714,6 +10223,27 @@ define amdgpu_kernel void @atomic_dec_i32_ret_decr64_offset(ptr %out, ptr %out2,
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_ret_decr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
@@ -7761,6 +10291,21 @@ define amdgpu_kernel void @atomic_dec_i32(ptr %out, i32 %in) {
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_atomic_dec_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -7814,6 +10359,22 @@ define amdgpu_kernel void @atomic_dec_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_dec_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr %out2
@@ -7871,6 +10432,26 @@ define amdgpu_kernel void @atomic_dec_i32_decr64(ptr %out, i32 %in, i64 %index)
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_decr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-NEXT:    flat_atomic_dec_u32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -7937,6 +10518,27 @@ define amdgpu_kernel void @atomic_dec_i32_ret_decr64(ptr %out, ptr %out2, i32 %i
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_dword v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_ret_decr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s8
+; GFX11-NEXT:    flat_atomic_dec_u32 v2, v[0:1], v2 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
@@ -7990,6 +10592,33 @@ define amdgpu_kernel void @atomic_load_f16_offset(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_f16_offset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_b16 v0, v[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_f16_offset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u16 v2, v[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr half, ptr %in, i64 8
   %val = load atomic half, ptr %gep  seq_cst, align 2
   store half %val, ptr %out
@@ -8038,6 +10667,33 @@ define amdgpu_kernel void @atomic_load_f16(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_b16 v0, v[0:1] glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u16 v2, v[0:1] glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
   %val = load atomic half, ptr %in seq_cst, align 2
   store half %val, ptr %out
   ret void
@@ -8089,6 +10745,33 @@ define amdgpu_kernel void @atomic_load_bf16_offset(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_bf16_offset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_b16 v0, v[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_bf16_offset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u16 v2, v[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr bfloat, ptr %in, i64 8
   %val = load atomic bfloat, ptr %gep  seq_cst, align 2
   store bfloat %val, ptr %out
@@ -8137,6 +10820,33 @@ define amdgpu_kernel void @atomic_load_bf16(ptr %in, ptr %out) {
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN3-NEXT:    flat_store_short v[0:1], v2
 ; GCN3-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT:    flat_load_d16_b16 v0, v[0:1] glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT:    flat_store_b16 v[1:2], v0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-FAKE16-NEXT:    flat_load_u16 v2, v[0:1] glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-FAKE16-NEXT:    flat_store_b16 v[0:1], v2
+; GFX11-FAKE16-NEXT:    s_endpgm
   %val = load atomic bfloat, ptr %in seq_cst, align 2
   store bfloat %val, ptr %out
   ret void
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
index 46eb30080c37..8c75b5c7c027 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -1714,23 +1714,13 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -1740,22 +1730,14 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_sdwa v1, v2, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_mul_f32_e32 v1, 1.0, v1
-; GFX8-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-SDAG-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -1771,22 +1753,13 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v4
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-SDAG-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-SDAG-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -1797,21 +1770,13 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v1, v1, v1
-; GFX900-SDAG-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v3, v3, v1, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-SDAG-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -1830,9 +1795,6 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX950-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v4
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -1856,9 +1818,6 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX950-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v3
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v1, v1, v1
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v1, v1, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
@@ -1884,22 +1843,14 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX10-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX10-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
@@ -1909,21 +1860,13 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -1940,68 +1883,52 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v4, v5, s0
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2017,61 +1944,38 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v4
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v3, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2090,82 +1994,65 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v4, v5, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v3
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2187,72 +2074,47 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v3, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = call bfloat @llvm.maximumnum.bf16(bfloat %a, bfloat %b)
   %max3 = call bfloat @llvm.maximumnum.bf16(bfloat %tmp0, bfloat %c)
@@ -2324,24 +2186,14 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -2351,23 +2203,14 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
@@ -2378,23 +2221,14 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2404,23 +2238,14 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2438,23 +2263,14 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v6
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-SDAG-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-SDAG-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -2464,22 +2280,14 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v4
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-SDAG-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX900-SDAG-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
@@ -2490,22 +2298,14 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v5
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-SDAG-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2515,21 +2315,13 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX900-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v4
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-SDAG-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-SDAG-NEXT:    v_perm_b32 v0, v1, v0, s4
 ; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -2544,15 +2336,12 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_sdwa v4, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX950-SDAG-NEXT:    s_nop 0
+; GFX950-SDAG-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX950-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v6
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
@@ -2577,9 +2366,6 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX950-SDAG-NEXT:    s_nop 0
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
@@ -2603,9 +2389,6 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v5
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
@@ -2629,12 +2412,8 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v4
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-SDAG-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-SDAG-NEXT:    s_nop 0
+; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-SDAG-NEXT:    s_nop 1
@@ -2669,41 +2448,25 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v3, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX10-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v8, v5, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v10, 0x400000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v9, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v8, v8, v5, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v6, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v4, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
@@ -2720,37 +2483,21 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX10-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2764,137 +2511,108 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v8
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v7
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v8
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v6.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_dual_max_f32 v5, v5, v5 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.h, v3.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v8, v10, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v4.l, s1
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s2
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v7
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v5, v7
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-FAKE16-LABEL: v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2905,120 +2623,82 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v5, v4 :: v_dual_lshlrev_b32 v8, 16, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v3 :: v_dual_lshlrev_b32 v7, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v6
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v4, v3, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_and_b32 v6, 0xffff0000, v6
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v4, 16, v6
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v6, v0 :: v_dual_lshlrev_b32 v3, 16, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v6, 0xffff0000, v2
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v5
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v3
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v5, 0xffff0000, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3036,147 +2716,117 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v8
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v8
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v6.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_dual_max_num_f32 v5, v5, v5 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v6, v6, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.h, v3.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v8, v10, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v4.l, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v0.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v5, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v1.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-FAKE16-LABEL: v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0:
@@ -3192,144 +2842,102 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v5, v4 :: v_dual_lshlrev_b32 v8, 16, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v3 :: v_dual_lshlrev_b32 v7, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v4, v3, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v6, v6, v6
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_and_b32 v6, 0xffff0000, v6
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v4, 16, v6
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v6, v0 :: v_dual_lshlrev_b32 v3, 16, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v6, 0xffff0000, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v4, v4, v4
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
+; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v3
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v5, 0xffff0000, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = call <2 x bfloat> @llvm.maximumnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
index aebbbe2c176d..fd7c7006b361 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -1713,25 +1713,15 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
 ; GFX8-NEXT:    s_movk_i32 s4, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -1741,22 +1731,14 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_sdwa v1, v2, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_mul_f32_e32 v1, 1.0, v1
-; GFX8-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-SDAG-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -1771,24 +1753,15 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v4
+; GFX900-SDAG-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-SDAG-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-SDAG-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-SDAG-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -1799,21 +1772,13 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v1, v1, v1
-; GFX900-SDAG-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v3, v3, v1, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-SDAG-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -1822,7 +1787,7 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX950-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX950-SDAG-NEXT:    s_nop 0
+; GFX950-SDAG-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; GFX950-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
@@ -1832,10 +1797,6 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX950-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v4
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX950-SDAG-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -1859,9 +1820,6 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX950-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v1, v1, v1
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v1, v1, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
@@ -1887,22 +1845,14 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX10-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX10-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
@@ -1912,21 +1862,13 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -1943,68 +1885,52 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v4, v5, s0
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2020,61 +1946,38 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v4
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v3, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2093,82 +1996,65 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v4, v5, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v3
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.h, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2190,72 +2076,47 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v3, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v2, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = call bfloat @llvm.minimumnum.bf16(bfloat %a, bfloat %b)
   %min3 = call bfloat @llvm.minimumnum.bf16(bfloat %tmp0, bfloat %c)
@@ -2326,26 +2187,16 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v6
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -2355,23 +2206,14 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
@@ -2382,23 +2224,14 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2408,23 +2241,14 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2441,25 +2265,16 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v6
+; GFX900-SDAG-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-SDAG-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-SDAG-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-SDAG-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-SDAG-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -2469,22 +2284,14 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v4
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-SDAG-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX900-SDAG-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
@@ -2495,22 +2302,14 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v5
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-SDAG-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2520,21 +2319,13 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX900-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v4
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-SDAG-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-SDAG-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-SDAG-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-SDAG-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-SDAG-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-SDAG-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-SDAG-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-SDAG-NEXT:    v_perm_b32 v0, v1, v0, s4
 ; GFX900-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -2549,16 +2340,12 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_sdwa v4, v0, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-SDAG-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX950-SDAG-NEXT:    s_nop 0
+; GFX950-SDAG-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX950-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v6
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX950-SDAG-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v4
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
@@ -2583,9 +2370,6 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX950-SDAG-NEXT:    s_nop 0
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
@@ -2609,9 +2393,6 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v5
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
@@ -2635,9 +2416,6 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX950-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v4
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-SDAG-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-SDAG-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-SDAG-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-SDAG-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-SDAG-NEXT:    s_nop 1
 ; GFX950-SDAG-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -2675,41 +2453,25 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v3, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX10-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v8, v5, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v10, 0x400000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v9, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v8, v8, v5, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v6, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v4, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
@@ -2726,37 +2488,21 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX10-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2770,137 +2516,108 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v8
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v7
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v8
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v6.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_dual_max_f32 v5, v5, v5 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.h, v3.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v8, v10, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v4.l, s1
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s2
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v7
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v5, v7
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
 ; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
 ; GFX11-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-FAKE16-LABEL: v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2911,120 +2628,82 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v5, v4 :: v_dual_lshlrev_b32 v8, 16, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v3 :: v_dual_lshlrev_b32 v7, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v6
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v4, v3, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v1, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_and_b32 v6, 0xffff0000, v6
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v4, 16, v6
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v6, v0 :: v_dual_lshlrev_b32 v3, 16, v1
+; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v6, 0xffff0000, v2
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v5
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v3
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v5, 0xffff0000, v5
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3042,147 +2721,117 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v8
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v8
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v6.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_dual_max_num_f32 v5, v5, v5 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v6, v6, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.h, v3.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v8, v10, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v4.l, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v0.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v6, v6
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v5, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v1.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-FAKE16-LABEL: v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0:
@@ -3198,144 +2847,102 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v5, v4 :: v_dual_lshlrev_b32 v8, 16, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v3 :: v_dual_lshlrev_b32 v7, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v4, v3, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v1, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v6, v6, v6
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v8, v6, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_and_b32 v6, 0xffff0000, v6
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v4, 16, v6
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_lshlrev_b32 v3, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v6, v0 :: v_dual_lshlrev_b32 v3, 16, v1
+; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v6, 0xffff0000, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v4, v4, v4
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v3, 16, v1
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX12-SDAG-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-SDAG-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-SDAG-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v3, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v4, v1 :: v_dual_lshlrev_b32 v6, 16, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
+; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v3
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX12-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
 ; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v5, 0xffff0000, v5
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
-; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-SDAG-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX12-SDAG-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = call <2 x bfloat> @llvm.minimumnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
new file mode 100644
index 000000000000..d189b6d4c1e8
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
@@ -0,0 +1,1217 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck --check-prefixes=CIVI,CI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga < %s | FileCheck --check-prefixes=CIVI,VI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-FAKE16 %s
+
+define amdgpu_kernel void @fneg_fabs_fadd_bf16(ptr addrspace(1) %out, bfloat %x, bfloat %y) {
+; CI-LABEL: fneg_fabs_fadd_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0x7fff
+; CI-NEXT:    s_lshl_b32 s3, s3, 16
+; CI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; CI-NEXT:    v_mov_b32_e32 v0, s3
+; CI-NEXT:    v_sub_f32_e32 v0, s2, v0
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_fadd_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s3, s2, 0x7fff
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; VI-NEXT:    v_mov_b32_e32 v0, s3
+; VI-NEXT:    v_sub_f32_e32 v0, s2, v0
+; VI-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
+; VI-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fneg_fabs_fadd_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s3, s2, 0x7fff
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_sub_f32_e32 v1, s2, v1
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    global_store_short_d16_hi v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: fneg_fabs_fadd_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s1, s0
+; GFX11-TRUE16-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX11-TRUE16-NEXT:    s_and_b32 s1, s1, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT:    v_sub_f32_e64 v0, s0, s1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_store_d16_hi_b16 v3, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fneg_fabs_fadd_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s1, s0, 0x7fff
+; GFX11-FAKE16-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_sub_f32_e64 v0, s0, s1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v1, v1, v0
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_store_d16_hi_b16 v3, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %x)
+  %fsub = fsub bfloat -0.0, %fabs
+  %fadd = fadd bfloat %y, %fsub
+  store bfloat %fadd, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabs_fmul_bf16(ptr addrspace(1) %out, bfloat %x, bfloat %y) {
+; CI-LABEL: fneg_fabs_fmul_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0x7fff
+; CI-NEXT:    s_lshl_b32 s3, s3, 16
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s3
+; CI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; CI-NEXT:    v_mul_f32_e32 v0, s2, v0
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_fmul_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_or_b32 s3, s2, 0x8000
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; VI-NEXT:    v_mov_b32_e32 v0, s3
+; VI-NEXT:    v_mul_f32_e32 v0, s2, v0
+; VI-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
+; VI-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fneg_fabs_fmul_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_or_b32 s3, s2, 0x8000
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_mul_f32_e32 v1, s2, v1
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    global_store_short_d16_hi v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: fneg_fabs_fmul_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s1, s0
+; GFX11-TRUE16-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX11-TRUE16-NEXT:    s_bitset1_b32 s1, 15
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT:    v_mul_f32_e64 v0, s0, s1
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_store_d16_hi_b16 v3, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fneg_fabs_fmul_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_or_b32 s1, s0, 0x8000
+; GFX11-FAKE16-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_mul_f32_e64 v0, s0, s1
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v1, v1, v0
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_store_d16_hi_b16 v3, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %x)
+  %fsub = fsub bfloat -0.0, %fabs
+  %fmul = fmul bfloat %y, %fsub
+  store bfloat %fmul, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+; DAGCombiner will transform:
+; (fabs (bf16 bitcast (i16 a))) => (bf16 bitcast (and (i16 a), 0x7FFFFFFF))
+; unless isFabsFree returns true
+define amdgpu_kernel void @fneg_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
+; CI-LABEL: fneg_fabs_free_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_free_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset1_b32 s2, 15
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fneg_fabs_free_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_bitset1_b32 s2, 15
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: fneg_fabs_free_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_bitset1_b32 s2, 15
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fneg_fabs_free_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_bitset1_b32 s2, 15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %bc = bitcast i16 %in to bfloat
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %bc)
+  %fsub = fsub bfloat -0.0, %fabs
+  store bfloat %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
+; CI-LABEL: fneg_fabs_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_bitset1_b32 s2, 15
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fneg_fabs_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_bitset1_b32 s2, 15
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: fneg_fabs_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_bitset1_b32 s2, 15
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fneg_fabs_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_bitset1_b32 s2, 15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %in)
+  %fsub = fsub bfloat -0.0, %fabs
+  store bfloat %fsub, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+define amdgpu_kernel void @v_fneg_fabs_bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; CI-LABEL: v_fneg_fabs_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    flat_load_ushort v2, v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_mul_f32_e64 v2, 1.0, |v2|
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; CI-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_fneg_fabs_bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_ushort v2, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v2, 0x8000, v2
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fneg_fabs_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ushort v1, v0, s[2:3]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v1, 0x8000, v1
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_fneg_fabs_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, 0x8000, v0
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_fneg_fabs_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v1, 0x8000, v1
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %val = load bfloat, ptr addrspace(1) %in, align 2
+  %fabs = call bfloat @llvm.fabs.bf16(bfloat %val)
+  %fsub = fsub bfloat -0.0, %fabs
+  store bfloat %fsub, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out, <2 x bfloat> %in) {
+; CI-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0xffff0000
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_add_f32_e64 v0, s3, 2.0
+; CI-NEXT:    v_add_f32_e64 v1, s2, 1.0
+; CI-NEXT:    v_readfirstlane_b32 s2, v0
+; CI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; CI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; CI-NEXT:    s_bitset0_b32 s2, 31
+; CI-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v1
+; CI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; CI-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; CI-NEXT:    s_lshr_b32 s2, s2, 16
+; CI-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; CI-NEXT:    v_alignbit_b32 v2, s2, v0, 16
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshl_b32 s3, s2, 16
+; VI-NEXT:    v_add_f32_e64 v0, s3, 1.0
+; VI-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
+; VI-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; VI-NEXT:    v_add_f32_e64 v1, s2, 2.0
+; VI-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x7fff, v2
+; VI-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; VI-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT:    v_alignbit_b32 v0, v1, v0, 16
+; VI-NEXT:    v_or_b32_e32 v2, 0x80008000, v0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s3, s2, 0xffff0000
+; GFX9-NEXT:    v_add_f32_e64 v1, s3, 2.0
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_add_f32_e64 v2, s2, 1.0
+; GFX9-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b32 s1, s0, 16
+; GFX11-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX11-NEXT:    v_add_f32_e64 v0, s1, 1.0
+; GFX11-NEXT:    v_add_f32_e64 v1, s0, 2.0
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    v_dual_cndmask_b32 v1, v3, v5 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_or_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %add = fadd <2 x bfloat> %in, <bfloat 1.0, bfloat 2.0>
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %add)
+  %fneg.fabs = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs
+  store <2 x bfloat> %fneg.fabs, ptr addrspace(1) %out
+  ret void
+}
+
+; FIXME: single bit op
+; Combine turns this into integer op when bitcast source (from load)
+define amdgpu_kernel void @s_fneg_fabs_v2bf16_bc_src(ptr addrspace(1) %out, <2 x bfloat> %in) {
+; CI-LABEL: s_fneg_fabs_v2bf16_bc_src:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0x7fff
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff0000
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s2
+; CI-NEXT:    s_lshl_b32 s2, s3, 16
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    v_mul_f32_e64 v1, -1.0, s2
+; CI-NEXT:    v_alignbit_b32 v2, v0, v1, 16
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_fabs_v2bf16_bc_src:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_or_b32 s2, s2, 0x80008000
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_fabs_v2bf16_bc_src:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_or_b32 s2, s2, 0x80008000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_fabs_v2bf16_bc_src:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_or_b32 s2, s2, 0x80008000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)
+  %fneg.fabs = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs
+  store <2 x bfloat> %fneg.fabs, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in) {
+; CI-LABEL: fneg_fabs_v4bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshl_b32 s4, s2, 16
+; CI-NEXT:    s_and_b32 s2, s2, 0xffff0000
+; CI-NEXT:    v_mul_f32_e64 v2, 1.0, |s2|
+; CI-NEXT:    s_and_b32 s2, s3, 0xffff0000
+; CI-NEXT:    s_lshl_b32 s5, s3, 16
+; CI-NEXT:    v_mul_f32_e64 v3, 1.0, |s2|
+; CI-NEXT:    v_mul_f32_e64 v0, 1.0, |s4|
+; CI-NEXT:    v_mul_f32_e64 v1, 1.0, |s5|
+; CI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
+; CI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; CI-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
+; CI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; CI-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; CI-NEXT:    v_alignbit_b32 v1, v3, v1, 16
+; CI-NEXT:    v_alignbit_b32 v0, v2, v0, 16
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fneg_fabs_v4bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s4, s2, 16
+; VI-NEXT:    s_lshr_b32 s5, s3, 16
+; VI-NEXT:    s_bitset1_b32 s3, 15
+; VI-NEXT:    s_bitset1_b32 s2, 15
+; VI-NEXT:    s_bitset1_b32 s5, 15
+; VI-NEXT:    s_bitset1_b32 s4, 15
+; VI-NEXT:    s_and_b32 s3, 0xffff, s3
+; VI-NEXT:    s_lshl_b32 s5, s5, 16
+; VI-NEXT:    s_and_b32 s2, 0xffff, s2
+; VI-NEXT:    s_lshl_b32 s4, s4, 16
+; VI-NEXT:    s_or_b32 s3, s3, s5
+; VI-NEXT:    s_or_b32 s2, s2, s4
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fneg_fabs_v4bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_bitset1_b32 s3, 15
+; GFX9-NEXT:    s_bitset1_b32 s2, 15
+; GFX9-NEXT:    s_bitset1_b32 s5, 15
+; GFX9-NEXT:    s_bitset1_b32 s4, 15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fneg_fabs_v4bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX11-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX11-NEXT:    s_bitset1_b32 s3, 15
+; GFX11-NEXT:    s_bitset1_b32 s2, 15
+; GFX11-NEXT:    s_bitset1_b32 s4, 15
+; GFX11-NEXT:    s_bitset1_b32 s5, 15
+; GFX11-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %fabs = call <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat> %in)
+  %fsub = fsub <4 x bfloat> <bfloat -0.0, bfloat -0.0, bfloat -0.0, bfloat -0.0>, %fabs
+  store <4 x bfloat> %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fold_user_fneg_fabs_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) #0 {
+; CI-LABEL: fold_user_fneg_fabs_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0x7fff0000
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_mul_f32_e64 v0, s3, -4.0
+; CI-NEXT:    v_mul_f32_e64 v1, s2, -4.0
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    v_alignbit_b32 v2, v0, v1, 16
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: fold_user_fneg_fabs_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s3, s2, 0x7fff
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    v_mul_f32_e64 v0, s3, -4.0
+; VI-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; VI-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
+; VI-NEXT:    s_lshr_b32 s2, s2, 16
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; VI-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; VI-NEXT:    s_lshl_b32 s2, s2, 16
+; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; VI-NEXT:    v_mul_f32_e64 v1, s2, -4.0
+; VI-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 0x7fff, v2
+; VI-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; VI-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT:    v_alignbit_b32 v2, v1, v0, 16
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fold_user_fneg_fabs_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s3, s2, 0x7fff
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX9-NEXT:    v_mul_f32_e64 v1, s3, -4.0
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_mul_f32_e64 v2, s2, -4.0
+; GFX9-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: fold_user_fneg_fabs_v2bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s1, s0
+; GFX11-TRUE16-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT:    s_and_b32 s1, s1, 0x7fff
+; GFX11-TRUE16-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT:    v_mul_f32_e64 v0, s1, -4.0
+; GFX11-TRUE16-NEXT:    v_mul_f32_e64 v1, s0, -4.0
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v1, v3, v5 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: fold_user_fneg_fabs_v2bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s1, s0, 0x7fff
+; GFX11-FAKE16-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX11-FAKE16-NEXT:    v_mul_f32_e64 v0, s1, -4.0
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_mul_f32_e64 v1, s0, -4.0
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v3, v5 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)
+  %fneg.fabs = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs
+  %mul = fmul <2 x bfloat> %fneg.fabs, <bfloat 4.0, bfloat 4.0>
+  store <2 x bfloat> %mul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fneg_multi_use_fabs_v2bf16(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <2 x bfloat> %in) {
+; CI-LABEL: s_fneg_multi_use_fabs_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    s_and_b32 s1, s4, 0x7fff
+; CI-NEXT:    s_and_b32 s2, s4, 0x7fff0000
+; CI-NEXT:    v_mul_f32_e64 v4, -1.0, s2
+; CI-NEXT:    s_lshl_b32 s1, s1, 16
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_and_b32 s0, s4, 0x7fff7fff
+; CI-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; CI-NEXT:    v_mul_f32_e64 v5, -1.0, s1
+; CI-NEXT:    v_alignbit_b32 v4, v4, v5, 16
+; CI-NEXT:    v_mov_b32_e32 v5, s0
+; CI-NEXT:    v_mov_b32_e32 v3, s3
+; CI-NEXT:    flat_store_dword v[0:1], v5
+; CI-NEXT:    flat_store_dword v[2:3], v4
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_multi_use_fabs_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_and_b32 s0, s4, 0x7fff7fff
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_or_b32 s1, s4, 0x80008000
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    flat_store_dword v[0:1], v4
+; VI-NEXT:    v_mov_b32_e32 v0, s1
+; VI-NEXT:    flat_store_dword v[2:3], v0
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_multi_use_fabs_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s5, s4, 0x7fff7fff
+; GFX9-NEXT:    s_or_b32 s4, s4, 0x80008000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    global_store_dword v0, v2, s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_multi_use_fabs_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_and_b32 s4, s6, 0x7fff7fff
+; GFX11-NEXT:    s_or_b32 s5, s6, 0x80008000
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT:    v_mov_b32_e32 v2, s5
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    global_store_b32 v0, v2, s[2:3]
+; GFX11-NEXT:    s_endpgm
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)
+  %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs
+  store <2 x bfloat> %fabs, ptr addrspace(1) %out0
+  store <2 x bfloat> %fneg, ptr addrspace(1) %out1
+  ret void
+}
+
+define amdgpu_kernel void @s_fneg_multi_use_fabs_foldable_neg_v2bf16(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <2 x bfloat> %in) {
+; CI-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_load_dword s4, s[8:9], 0x4
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_and_b32 s1, s4, 0x7fff
+; CI-NEXT:    s_lshl_b32 s1, s1, 16
+; CI-NEXT:    v_mul_f32_e64 v4, s1, -4.0
+; CI-NEXT:    s_and_b32 s1, s4, 0x7fff0000
+; CI-NEXT:    v_mul_f32_e64 v5, s1, -4.0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_and_b32 s0, s4, 0x7fff7fff
+; CI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; CI-NEXT:    v_alignbit_b32 v4, v5, v4, 16
+; CI-NEXT:    v_mov_b32_e32 v5, s0
+; CI-NEXT:    v_mov_b32_e32 v2, s2
+; CI-NEXT:    v_mov_b32_e32 v3, s3
+; CI-NEXT:    flat_store_dword v[0:1], v5
+; CI-NEXT:    flat_store_dword v[2:3], v4
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_load_dword s4, s[8:9], 0x10
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_and_b32 s1, s4, 0x7fff
+; VI-NEXT:    s_lshl_b32 s1, s1, 16
+; VI-NEXT:    v_mul_f32_e64 v4, s1, -4.0
+; VI-NEXT:    v_bfe_u32 v5, v4, 16, 1
+; VI-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
+; VI-NEXT:    s_lshr_b32 s1, s4, 16
+; VI-NEXT:    v_add_u32_e32 v5, vcc, 0x7fff, v5
+; VI-NEXT:    s_and_b32 s1, s1, 0x7fff
+; VI-NEXT:    v_or_b32_e32 v6, 0x400000, v4
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
+; VI-NEXT:    s_lshl_b32 s1, s1, 16
+; VI-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
+; VI-NEXT:    v_mul_f32_e64 v5, s1, -4.0
+; VI-NEXT:    v_bfe_u32 v6, v5, 16, 1
+; VI-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 0x7fff, v6
+; VI-NEXT:    v_or_b32_e32 v7, 0x400000, v5
+; VI-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
+; VI-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_and_b32 s0, s4, 0x7fff7fff
+; VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; VI-NEXT:    v_alignbit_b32 v4, v5, v4, 16
+; VI-NEXT:    v_mov_b32_e32 v5, s0
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    flat_store_dword v[0:1], v5
+; VI-NEXT:    flat_store_dword v[2:3], v4
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s4, s[8:9], 0x10
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s6, s4, 0x7fff
+; GFX9-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX9-NEXT:    v_mul_f32_e64 v1, s6, -4.0
+; GFX9-NEXT:    s_and_b32 s5, s4, 0x7fff7fff
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    s_and_b32 s4, s4, 0x7fff
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_mul_f32_e64 v2, s4, -4.0
+; GFX9-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    global_store_dword v0, v2, s[0:1]
+; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, s6
+; GFX11-TRUE16-NEXT:    s_lshr_b32 s1, s6, 16
+; GFX11-TRUE16-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX11-TRUE16-NEXT:    s_and_b32 s1, s1, 0x7fff
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mul_f32_e64 v0, s0, -4.0
+; GFX11-TRUE16-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX11-TRUE16-NEXT:    v_mul_f32_e64 v1, s0, -4.0
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT:    s_and_b32 s4, s6, 0x7fff7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, s4 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_store_b32 v2, v3, s[0:1]
+; GFX11-TRUE16-NEXT:    global_store_b32 v2, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b32 s6, s[4:5], 0x10
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s0, s6, 0x7fff
+; GFX11-FAKE16-NEXT:    s_lshr_b32 s1, s6, 16
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT:    s_and_b32 s1, s1, 0x7fff
+; GFX11-FAKE16-NEXT:    v_mul_f32_e64 v0, s0, -4.0
+; GFX11-FAKE16-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_mul_f32_e64 v1, s0, -4.0
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-FAKE16-NEXT:    s_and_b32 s4, s6, 0x7fff7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v3, v3, v1
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v3, s4 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_store_b32 v2, v3, s[0:1]
+; GFX11-FAKE16-NEXT:    global_store_b32 v2, v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)
+  %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs
+  %mul = fmul <2 x bfloat> %fneg, <bfloat 4.0, bfloat 4.0>
+  store <2 x bfloat> %fabs, ptr addrspace(1) %out0
+  store <2 x bfloat> %mul, ptr addrspace(1) %out1
+  ret void
+}
+
+declare bfloat @llvm.fabs.bf16(bfloat) #1
+declare <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat>) #1
+declare <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat>) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CIVI: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
new file mode 100644
index 000000000000..d232693b46ad
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
@@ -0,0 +1,987 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=kaveri -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=CIVI,CI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=tonga -mtriple=amdgcn--amdhsa  < %s | FileCheck -enable-var-scope -check-prefixes=CIVI,GFX8 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=gfx900 -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=gfx1100 -mattr=+real-true16 -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=gfx1100 -mattr=-real-true16 -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s
+
+define amdgpu_kernel void @s_fneg_bf16(ptr addrspace(1) %out, bfloat %in) #0 {
+; CI-LABEL: s_fneg_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: s_fneg_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: s_fneg_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: s_fneg_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %fneg = fsub bfloat -0.0, %in
+  store bfloat %fneg, ptr addrspace(1) %out
+  ret void
+}
+
+; FIXME: Should be able to use bit operations when illegal type as
+; well.
+define amdgpu_kernel void @v_fneg_bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_fneg_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_ushort v2, v[0:1]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_mul_f32_e32 v2, -1.0, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: v_fneg_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x8000, v2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fneg_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_fneg_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_xor_b16 v0.l, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_fneg_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds bfloat, ptr addrspace(1) %in, i32 %tid
+  %gep.out = getelementptr inbounds bfloat, ptr addrspace(1) %in, i32 %tid
+  %val = load bfloat, ptr addrspace(1) %gep.in, align 2
+  %fneg = fsub bfloat -0.0, %val
+  store bfloat %fneg, ptr addrspace(1) %gep.out
+  ret void
+}
+
+define amdgpu_kernel void @s_fneg_free_bf16(ptr addrspace(1) %out, i16 %in) #0 {
+; CI-LABEL: s_fneg_free_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: s_fneg_free_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_free_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: s_fneg_free_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: s_fneg_free_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_xor_b32 s2, s2, 0x8000
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %bc = bitcast i16 %in to bfloat
+  %fsub = fsub bfloat -0.0, %bc
+  store bfloat %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fneg_fold_bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_fneg_fold_bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    flat_load_ushort v2, v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_mul_f32_e32 v3, -1.0, v2
+; CI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
+; CI-NEXT:    v_mul_f32_e32 v2, v3, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; CI-NEXT:    flat_store_short v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: v_fneg_fold_bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0x8000
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_xor_b32_sdwa v3, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX8-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v2
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fneg_fold_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ushort v1, v0, s[2:3]
+; GFX9-NEXT:    s_mov_b32 s2, 0x8000
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_sdwa v2, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    global_store_short_d16_hi v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: v_fneg_fold_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_xor_b32_e32 v2, 0x8000, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, v2, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_d16_hi_b16 v0, v1, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_fneg_fold_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v2, 0x8000, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v2, v2, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-FAKE16-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_d16_hi_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %val = load bfloat, ptr addrspace(1) %in
+  %fsub = fsub bfloat -0.0, %val
+  %fmul = fmul bfloat %fsub, %val
+  store bfloat %fmul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fneg_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) #0 {
+; CI-LABEL: s_fneg_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0xffff0000
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s3
+; CI-NEXT:    v_mul_f32_e64 v1, -1.0, s2
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    v_alignbit_b32 v2, v0, v1, 16
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: s_fneg_v2bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %in
+  store <2 x bfloat> %fneg, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_fneg_v2bf16_nonload(ptr addrspace(1) %out) #0 {
+; CI-LABEL: s_fneg_v2bf16_nonload:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    ;;#ASMSTART
+; CI-NEXT:    ; def s2
+; CI-NEXT:    ;;#ASMEND
+; CI-NEXT:    s_and_b32 s3, s2, 0xffff0000
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s3
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    v_mul_f32_e64 v1, -1.0, s2
+; CI-NEXT:    v_alignbit_b32 v2, v0, v1, 16
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: s_fneg_v2bf16_nonload:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    ;;#ASMSTART
+; GFX8-NEXT:    ; def s2
+; GFX8-NEXT:    ;;#ASMEND
+; GFX8-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_fneg_v2bf16_nonload:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    ;;#ASMSTART
+; GFX9-NEXT:    ; def s2
+; GFX9-NEXT:    ;;#ASMEND
+; GFX9-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_fneg_v2bf16_nonload:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    ;;#ASMSTART
+; GFX11-NEXT:    ; def s2
+; GFX11-NEXT:    ;;#ASMEND
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %in = call i32 asm sideeffect "; def $0", "=s"()
+  %in.bc = bitcast i32 %in to <2 x bfloat>
+  %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %in.bc
+  store <2 x bfloat> %fneg, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fneg_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_fneg_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x2
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT:    flat_load_dword v2, v[0:1]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_mul_f32_e32 v3, -1.0, v3
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_mul_f32_e32 v2, -1.0, v2
+; CI-NEXT:    v_alignbit_b32 v2, v3, v2, 16
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: v_fneg_v2bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fneg_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x8
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_fneg_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_xor_b32_e32 v1, 0x80008000, v1
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %gep.out = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid
+  %val = load <2 x bfloat>, ptr addrspace(1) %gep.in, align 2
+  %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val
+  store <2 x bfloat> %fneg, ptr addrspace(1) %gep.out
+  ret void
+}
+
+define amdgpu_kernel void @fneg_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {
+; CI-LABEL: fneg_free_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dword s2, s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_and_b32 s3, s2, 0xffff0000
+; CI-NEXT:    s_lshl_b32 s2, s2, 16
+; CI-NEXT:    v_mul_f32_e64 v0, -1.0, s3
+; CI-NEXT:    v_mul_f32_e64 v1, -1.0, s2
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    v_alignbit_b32 v2, v0, v1, 16
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: fneg_free_v2bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: fneg_free_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fneg_free_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80008000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %bc = bitcast i32 %in to <2 x bfloat>
+  %fsub = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %bc
+  store <2 x bfloat> %fsub, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fneg_fold_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_fneg_fold_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    flat_load_dword v2, v[0:1]
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
+; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT:    v_mul_f32_e32 v4, -1.0, v3
+; CI-NEXT:    v_mul_f32_e32 v5, -1.0, v2
+; CI-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
+; CI-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
+; CI-NEXT:    v_mul_f32_e32 v3, v4, v3
+; CI-NEXT:    v_mul_f32_e32 v2, v5, v2
+; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT:    v_alignbit_b32 v2, v3, v2, 16
+; CI-NEXT:    flat_store_dword v[0:1], v2
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: v_fneg_fold_v2bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0x8000
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT:    v_xor_b32_sdwa v5, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; GFX8-NEXT:    v_xor_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mul_f32_e32 v3, v5, v4
+; GFX8-NEXT:    v_mul_f32_e32 v2, v2, v6
+; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT:    v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_alignbit_b32 v2, v2, v3, 16
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fneg_fold_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX9-NEXT:    s_mov_b32 s2, 0x8000
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
+; GFX9-NEXT:    v_xor_b32_sdwa v4, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_e32 v5, 0xffff0000, v1
+; GFX9-NEXT:    v_xor_b32_sdwa v1, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT:    v_mul_f32_e32 v1, v1, v5
+; GFX9-NEXT:    v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT:    v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT:    v_add_u32_e32 v6, v6, v1
+; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT:    v_or_b32_e32 v7, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v6, 0x7fff, v6
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_fneg_fold_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-NEXT:    v_xor_b32_e32 v3, 0x8000, v1
+; GFX11-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_xor_b32_e32 v2, 0x8000, v2
+; GFX11-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_dual_mul_f32 v3, v3, v4 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_bfe_u32 v2, v3, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v3
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %val = load <2 x bfloat>, ptr addrspace(1) %in
+  %fsub = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val
+  %fmul = fmul <2 x bfloat> %fsub, %val
+  store <2 x bfloat> %fmul, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_extract_fneg_fold_v2bf16(ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_extract_fneg_fold_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_load_dword v0, v[0:1]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; CI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; CI-NEXT:    v_mul_f32_e32 v1, -4.0, v1
+; CI-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; CI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    flat_store_short v[0:1], v1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    flat_store_short v[0:1], v0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: v_extract_fneg_fold_v2bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT:    v_mul_f32_e32 v1, -4.0, v1
+; GFX8-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; GFX8-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX8-NEXT:    v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v1
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    flat_store_short v[0:1], v1
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    flat_store_short v[0:1], v0
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_extract_fneg_fold_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-NEXT:    v_mul_f32_e32 v1, -4.0, v1
+; GFX9-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT:    global_store_short_d16_hi v[0:1], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_short_d16_hi v[0:1], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_extract_fneg_fold_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v0, v0, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_dual_mul_f32 v1, -4.0, v1 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-NEXT:    v_sub_f32_e32 v0, 2.0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-NEXT:    global_store_d16_hi_b16 v[0:1], v1, off dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_d16_hi_b16 v[0:1], v0, off dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_endpgm
+  %val = load <2 x bfloat>, ptr addrspace(1) %in
+  %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val
+  %elt0 = extractelement <2 x bfloat> %fneg, i32 0
+  %elt1 = extractelement <2 x bfloat> %fneg, i32 1
+
+  %fmul0 = fmul bfloat %elt0, 4.0
+  %fadd1 = fadd bfloat %elt1, 2.0
+  store volatile bfloat %fmul0, ptr addrspace(1) poison
+  store volatile bfloat %fadd1, ptr addrspace(1) poison
+  ret void
+}
+
+define amdgpu_kernel void @v_extract_fneg_no_fold_v2bf16(ptr addrspace(1) %in) #0 {
+; CI-LABEL: v_extract_fneg_no_fold_v2bf16:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_load_dword v0, v[0:1]
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
+; CI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; CI-NEXT:    v_mul_f32_e32 v1, -1.0, v1
+; CI-NEXT:    v_mul_f32_e32 v0, -1.0, v0
+; CI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT:    flat_store_short v[0:1], v0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    flat_store_short v[0:1], v1
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    s_endpgm
+;
+; GFX8-LABEL: v_extract_fneg_no_fold_v2bf16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    flat_store_short v[0:1], v0
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    flat_store_short v[0:1], v1
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_extract_fneg_no_fold_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT:    global_store_short v[0:1], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_short_d16_hi v[0:1], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_extract_fneg_no_fold_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v0, v0, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT:    global_store_b16 v[0:1], v0, off dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_d16_hi_b16 v[0:1], v0, off dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_endpgm
+  %val = load <2 x bfloat>, ptr addrspace(1) %in
+  %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val
+  %elt0 = extractelement <2 x bfloat> %fneg, i32 0
+  %elt1 = extractelement <2 x bfloat> %fneg, i32 1
+  store volatile bfloat %elt0, ptr addrspace(1) poison
+  store volatile bfloat %elt1, ptr addrspace(1) poison
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CIVI: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-freeze-fmul-to-fma.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-freeze-fmul-to-fma.ll
new file mode 100644
index 000000000000..9a4ab9865369
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-freeze-fmul-to-fma.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck %s
+
+define float @fma_from_freeze_mul_add_left(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_add_left:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, 1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul contract float %x, %y
+  %mul.fr = freeze float %mul
+  %add = fadd contract float %mul.fr, 1.000000e+00
+  ret float %add
+}
+
+define float @fma_from_freeze_mul_add_left_with_nnan(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_add_left_with_nnan:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, 1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul nnan contract afn float %x, %y
+  %mul.fr = freeze float %mul
+  %add = fadd nnan contract float %mul.fr, 1.000000e+00
+  ret float %add
+}
+
+define float @fma_from_freeze_mul_add_right(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_add_right:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, 1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul contract float %x, %y
+  %mul.fr = freeze float %mul
+  %add = fadd contract float 1.000000e+00, %mul.fr
+  ret float %add
+}
+
+define float @fma_from_freeze_mul_add_right_with_nnan(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_add_right_with_nnan:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, 1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul nnan contract float %x, %y
+  %mul.fr = freeze float %mul
+  %add = fadd nnan contract float 1.000000e+00, %mul.fr
+  ret float %add
+}
+
+define float @fma_from_freeze_mul_sub_left(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_sub_left:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, -1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul contract float %x, %y
+  %mul.fr = freeze float %mul
+  %sub = fsub contract float %mul.fr, 1.000000e+00
+  ret float %sub
+}
+
+define float @fma_from_freeze_mul_sub_left_with_nnan(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_sub_left_with_nnan:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, -1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul nnan contract float %x, %y
+  %mul.fr = freeze float %mul
+  %sub = fsub nnan contract float %mul.fr, 1.000000e+00
+  ret float %sub
+}
+
+define float @fma_from_freeze_mul_sub_right(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_sub_right:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, -v0, v1, 1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul contract float %x, %y
+  %mul.fr = freeze float %mul
+  %sub = fsub contract float 1.000000e+00, %mul.fr
+  ret float %sub
+}
+
+define float @fma_from_freeze_mul_sub_right_with_nnan(float %x, float %y) {
+; CHECK-LABEL: fma_from_freeze_mul_sub_right_with_nnan:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fma_f32 v0, -v0, v1, 1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %mul = fmul nnan contract float %x, %y
+  %mul.fr = freeze float %mul
+  %sub = fsub nnan contract float 1.000000e+00, %mul.fr
+  ret float %sub
+}
+
+define float @fma_freeze_sink_multiple_maybe_poison_nnan_add(float %x, float %y) {
+; CHECK-LABEL: fma_freeze_sink_multiple_maybe_poison_nnan_add:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_dual_subrev_f32 v0, 1.0, v0 :: v_dual_add_f32 v1, 1.0, v1
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, 1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %fsub_x = fsub nnan contract float %x, 1.000000e+00
+  %fadd_y = fadd nnan contract float %y, 1.000000e+00
+  %mul = fmul nnan contract float %fsub_x, %fadd_y
+  %mul.fr = freeze float %mul
+  %add = fadd nnan contract float %mul.fr, 1.000000e+00
+  ret float %add
+}
+
+define float @fma_freeze_sink_multiple_maybe_poison_nnan_sub(float %x, float %y) {
+; CHECK-LABEL: fma_freeze_sink_multiple_maybe_poison_nnan_sub:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_dual_add_f32 v0, 1.0, v0 :: v_dual_add_f32 v1, -1.0, v1
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_fma_f32 v0, v0, v1, -1.0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %fadd_x = fadd nnan contract float %x, 1.000000e+00
+  %fsub_y = fsub nnan contract float %y, 1.000000e+00
+  %mul = fmul nnan contract float %fadd_x, %fsub_y
+  %mul.fr = freeze float %mul
+  %sub = fsub nnan contract float %mul.fr, 1.000000e+00
+  ret float %sub
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
index d00fd9b967f3..74c4a2da5022 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
@@ -43,8 +43,7 @@ body:             |
     ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[DEF]], %subreg.sub0, killed [[V_MOV_B32_e32_]], %subreg.sub1
-    ; GCN-NEXT: [[V_XOR_B32_e32_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 0, [[DEF1]], implicit $exec
-    ; GCN-NEXT: [[V_XOR_B32_e32_1:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 [[DEF2]], [[REG_SEQUENCE]].sub0, implicit $exec
+    ; GCN-NEXT: [[V_XOR_B32_e32_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 [[DEF2]], [[REG_SEQUENCE]].sub0, implicit $exec
     %0:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %2:vgpr_32 = IMPLICIT_DEF
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-redundant-sgpr-vgpr-rw-across-bb.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-redundant-sgpr-vgpr-rw-across-bb.ll
new file mode 100644
index 000000000000..2ad9c0e71c5f
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-redundant-sgpr-vgpr-rw-across-bb.ll
@@ -0,0 +1,79 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefix=CHECK1 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -amdgpu-si-fold-operands-preheader-threshold=10 < %s | FileCheck --check-prefix=CHECK2 %s
+
+define protected amdgpu_kernel void @main(ptr addrspace(1) noundef %args.coerce, ptr addrspace(1) noundef %args.coerce2, ptr addrspace(1) noundef %args.coerce4, i32 noundef %args12) {
+; CHECK1-LABEL: main:
+; check that non-redundant readfirstlanes are not removed
+; CHECK1:      v_readfirstlane_b32
+; check that all redundant readfirstlanes are removed
+; CHECK1-NOT:  v_readfirstlane_b32
+; CHECK1:      s_endpgm
+
+; CHECK2-LABEL: main:
+; CHECK2:      v_readfirstlane_b32
+; check that all redundant readfirstlanes across basic blocks persist
+; CHECK2:      v_readfirstlane_b32
+; CHECK2:      v_readfirstlane_b32
+; CHECK2:      s_endpgm
+entry:
+    %wid = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+    %div1 = lshr i32 %wid, 6
+    %rfl1 = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 %div1)
+    %sub1 = add nsw i32 %args12, 1023
+    %div2 = sdiv i32 %sub1, 1024
+    %rfl2 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 %div2)
+    %cmp24.i = icmp sgt i32 %rfl2, 0
+    br i1 %cmp24.i, label %for.body.lr.ph.i, label %add.exit
+
+for.body.lr.ph.i:                                 ; preds = %entry
+    %pti1 = ptrtoint ptr addrspace(1) %args.coerce4 to i64
+    %pti2 = ptrtoint ptr addrspace(1) %args.coerce2 to i64
+    %pti3 = ptrtoint ptr addrspace(1) %args.coerce to i64
+    %lshr1 = lshr i32 %rfl1, 2
+    %mbl = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+    %mbh = tail call noundef i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %mbl)
+    %lshr2 = lshr i32 %mbh, 6
+    %add8 = add i32 %lshr1, %lshr2
+    %sub3 = shl i32 %rfl1, 8
+    %mul2 = and i32 %sub3, 768
+    %add1 = or disjoint i32 %mbh, %mul2
+    %add3 = add nsw i32 %add1, %add8
+    %sext1 = add i64 4294967296, 4611686014132420608
+    %conv1 = lshr exact i64 64, 32
+    %add4 = add nuw nsw i64 %conv1, 1
+    %zext2 = zext i32 1 to i64
+    %tmp.sroa = add nuw nsw i64 %zext2, 4294967295
+    %sub5 = add i64 %tmp.sroa, 4294967296
+    %sext2 = mul i64 %sub5, 4294967296
+    %conv2 = lshr exact i64 %sext2, 32
+    %add5 = add nuw nsw i64 %add4, %conv2
+    %conv3 = trunc i64 %add5 to i32
+    %mul4 = shl i32 %conv3, 2
+    %bc1 = bitcast i64 %pti3 to <2 x i32>
+    %ee1 = extractelement <2 x i32> %bc1, i64 0
+    %ee2 = extractelement <2 x i32> %bc1, i64 1
+    br label %for.body.i
+
+for.body.i:                                       ; preds = %for.body.i, %for.body.lr.ph.i
+    %loopi = phi i32 [ 0, %for.body.lr.ph.i ], [ %inc.i, %for.body.i ]
+    %tmp1 = phi i32 [ %add3, %for.body.lr.ph.i ], [ %cnt, %for.body.i ]
+    %rfl3 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 %ee1)
+    %rfl4 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 %ee2)
+    %rfl5 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 %mul4)
+    %ie1 = insertelement <4 x i32> <i32 poison, i32 poison, i32 poison, i32 131072>, i32 %rfl3, i64 0
+    %ie2 = insertelement <4 x i32> %ie1, i32 %rfl4, i64 1
+    %ie3 = insertelement <4 x i32> %ie2, i32 %rfl5, i64 2
+    %mul5 = shl i32 %tmp1, 2
+    %buffload1 = tail call contract noundef <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> noundef %ie2, i32 noundef %mul5, i32 noundef 0, i32 noundef 0) #6
+    %add6 = add nsw i32 %tmp1, 1
+    %buffload3 = tail call contract noundef <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> noundef %ie3, i32 noundef %mul5, i32 noundef 0, i32 noundef 0) #6
+    %vec_add1 = fadd contract <4 x float> %buffload1, %buffload3
+    tail call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> noundef %vec_add1, <4 x i32> noundef %ie3, i32 noundef %mul5, i32 noundef 0, i32 noundef 0) #6
+    %cnt = add nsw i32 %tmp1, 1024
+    %inc.i = add nuw nsw i32 %loopi, 1
+    %exitcond.not.i = icmp eq i32 %inc.i, %rfl2
+    br i1 %exitcond.not.i, label %add.exit, label %for.body.i
+
+    add.exit: ; preds = %for.body.i, %entry
+    ret void
+}
\ No newline at end of file
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-skips-non-reg.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-skips-non-reg.mir
index b1aa88969c5b..dc03eb74cbf1 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-skips-non-reg.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fold-zero-high-bits-skips-non-reg.mir
@@ -8,8 +8,8 @@ body: |
     ; CHECK-LABEL: name: test_tryFoldZeroHighBits_skips_nonreg
     ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
-    ; CHECK-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 65535, 0, implicit $exec
-    ; CHECK-NEXT: S_NOP 0, implicit [[V_AND_B32_e64_]]
+    ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit [[V_MOV_B32_e32_1]]
   %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   %1:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %0, %subreg.sub1
   %2:vgpr_32 = V_AND_B32_e64 65535, %1.sub0, implicit $exec
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fp-classify.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fp-classify.ll
index 498df8a65fed..200f74beec38 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fp-classify.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fp-classify.ll
@@ -536,8 +536,8 @@ define amdgpu_kernel void @test_isfinite_pattern_4_commute_and(ptr addrspace(1)
 define amdgpu_kernel void @test_not_isfinite_pattern_4_wrong_ord_test(ptr addrspace(1) nocapture %out, float %x, [8 x i32], float %y) #0 {
 ; SI-LABEL: test_not_isfinite_pattern_4_wrong_ord_test:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s0, s[4:5], 0x14
 ; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s0, s[4:5], 0x14
 ; SI-NEXT:    s_load_dword s1, s[4:5], 0xb
 ; SI-NEXT:    s_mov_b32 s11, 0xf000
 ; SI-NEXT:    s_mov_b32 s10, -1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
index e5815e96fbe3..d8f21d285ddf 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
@@ -12,18 +12,272 @@ define <2 x half> @v_test_cvt_v2f32_v2f16(<2 x float> %src) {
   ret <2 x half> %res
 }
 
-define half @fptrunc_v2f32_v2f16_then_extract(<2 x float> %src) {
-; GFX950-LABEL: fptrunc_v2f32_v2f16_then_extract:
+define <3 x half> @v_test_cvt_v3f32_v3f16(<3 x float> %src) {
+; GFX950-LABEL: v_test_cvt_v3f32_v3f16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
-; GFX950-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX950-NEXT:    v_mov_b32_e32 v1, v2
+; GFX950-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <3 x float> %src to <3 x half>
+  ret <3 x half> %res
+}
+
+define <4 x half> @v_test_cvt_v4f32_v4f16(<4 x float> %src) {
+; GFX950-LABEL: v_test_cvt_v4f32_v4f16:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <4 x float> %src to <4 x half>
+  ret <4 x half> %res
+}
+
+define <8 x half> @v_test_cvt_v8f32_v2f16(<8 x float> %src) {
+; GFX950-LABEL: v_test_cvt_v8f32_v2f16:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v2, v4, v5
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v3, v6, v7
+; GFX950-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <8 x float> %src to <8 x half>
+  ret <8 x half> %res
+}
+
+define <16 x half> @v_test_cvt_v16f32_v16f16(<16 x float> %src) {
+; GFX950-LABEL: v_test_cvt_v16f32_v16f16:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v2, v4, v5
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v3, v6, v7
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v4, v8, v9
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v5, v10, v11
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v6, v12, v13
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v7, v14, v15
+; GFX950-NEXT:    s_setpc_b64 s[30:31]
+  %res = fptrunc <16 x float> %src to <16 x half>
+  ret <16 x half> %res
+}
+
+define half @fptrunc_v2f32_v2f16_extract_uses(<2 x float> %src) {
+; GFX950-LABEL: fptrunc_v2f32_v2f16_extract_uses:
+; GFX950:       ; %bb.0:
+; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
   %vec_half = fptrunc <2 x float> %src to <2 x half>
-  %first = extractelement <2 x half> %vec_half, i64 1
-  %second = extractelement <2 x half> %vec_half, i64 0
-  %res = fadd half %first, %second
-  ret half %res
+  %f0 = extractelement <2 x half> %vec_half, i64 0
+  %f1 = extractelement <2 x half> %vec_half, i64 1
+  %rslt = fadd half %f0, %f1
+  ret half %rslt
+}
+
+define half @fptrunc_v3f32_v3f16_extract_uses(<3 x float> %vec_float) {
+; GFX950-SDAG-LABEL: fptrunc_v3f32_v3f16_extract_uses:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX950-SDAG-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v2, v0
+; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v3f32_v3f16_extract_uses:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v0, v2, v0
+; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %vec_half = fptrunc <3 x float> %vec_float to <3 x half>
+  %f0 = extractelement <3 x half> %vec_half, i64 0
+  %f1 = extractelement <3 x half> %vec_half, i64 1
+  %f2 = extractelement <3 x half> %vec_half, i64 2
+  %sum0 = fadd half %f0, %f1
+  %rslt = fadd half %f2, %sum0
+  ret half %rslt
+}
+
+define half @fptrunc_v4f32_v4f16_extract_uses(<4 x float> %vec_float) {
+; GFX950-SDAG-LABEL: fptrunc_v4f32_v4f16_extract_uses:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v2, v2, v3
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v4f32_v4f16_extract_uses:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %vec_half = fptrunc <4 x float> %vec_float to <4 x half>
+  %f0 = extractelement <4 x half> %vec_half, i64 0
+  %f1 = extractelement <4 x half> %vec_half, i64 1
+  %f2 = extractelement <4 x half> %vec_half, i64 2
+  %f3 = extractelement <4 x half> %vec_half, i64 3
+  %sum0 = fadd half %f0, %f1
+  %sum1 = fadd half %f2, %f3
+  %rslt = fadd half %sum0, %sum1
+  ret half %rslt
+}
+
+define half @fptrunc_v8f32_v8f16_extract_uses(<8 x float> %vec_float) {
+; GFX950-SDAG-LABEL: fptrunc_v8f32_v8f16_extract_uses:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v6, v6, v7
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v4, v4, v5
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v2, v2, v3
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v1, v2, v3
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v8f32_v8f16_extract_uses:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v2, v4, v5
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v3, v6, v7
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v1, v2, v3
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %vec_half = fptrunc <8 x float> %vec_float to <8 x half>
+  %f0 = extractelement <8 x half> %vec_half, i64 0
+  %f1 = extractelement <8 x half> %vec_half, i64 1
+  %f2 = extractelement <8 x half> %vec_half, i64 2
+  %f3 = extractelement <8 x half> %vec_half, i64 3
+  %f4 = extractelement <8 x half> %vec_half, i64 4
+  %f5 = extractelement <8 x half> %vec_half, i64 5
+  %f6 = extractelement <8 x half> %vec_half, i64 6
+  %f7 = extractelement <8 x half> %vec_half, i64 7
+  %sum0 = fadd half %f0, %f1
+  %sum1 = fadd half %f2, %f3
+  %sum2 = fadd half %f4, %f5
+  %sum3 = fadd half %f6, %f7
+  %sum4 = fadd half %sum0, %sum1
+  %sum5 = fadd half %sum2, %sum3
+  %rslt = fadd half %sum4, %sum5
+  ret half %rslt
+}
+
+define half @fptrunc_v16f32_v16f16_extract_uses(<16 x float> %vec_float) {
+; GFX950-SDAG-LABEL: fptrunc_v16f32_v16f16_extract_uses:
+; GFX950-SDAG:       ; %bb.0:
+; GFX950-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v14, v14, v15
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v12, v12, v13
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v10, v10, v11
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v8, v8, v9
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v6, v6, v7
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v4, v4, v5
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v2, v2, v3
+; GFX950-SDAG-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v4, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v5, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v6, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_sdwa v7, v14, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v1, v2, v3
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v2, v4, v5
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v3, v6, v7
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v1, v2, v3
+; GFX950-SDAG-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v16f32_v16f16_extract_uses:
+; GFX950-GISEL:       ; %bb.0:
+; GFX950-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v2, v4, v5
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v3, v6, v7
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v4, v8, v9
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v5, v10, v11
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v6, v12, v13
+; GFX950-GISEL-NEXT:    v_cvt_pk_f16_f32 v7, v14, v15
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v1, v2, v3
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v2, v4, v5
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v3, v6, v7
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v1, v2, v3
+; GFX950-GISEL-NEXT:    v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %vec_half = fptrunc <16 x float> %vec_float to <16 x half>
+  %f0 = extractelement <16 x half> %vec_half, i64 0
+  %f1 = extractelement <16 x half> %vec_half, i64 1
+  %f2 = extractelement <16 x half> %vec_half, i64 2
+  %f3 = extractelement <16 x half> %vec_half, i64 3
+  %f4 = extractelement <16 x half> %vec_half, i64 4
+  %f5 = extractelement <16 x half> %vec_half, i64 5
+  %f6 = extractelement <16 x half> %vec_half, i64 6
+  %f7 = extractelement <16 x half> %vec_half, i64 7
+  %f8 = extractelement <16 x half> %vec_half, i64 8
+  %f9 = extractelement <16 x half> %vec_half, i64 9
+  %f10 = extractelement <16 x half> %vec_half, i64 10
+  %f11 = extractelement <16 x half> %vec_half, i64 11
+  %f12 = extractelement <16 x half> %vec_half, i64 12
+  %f13 = extractelement <16 x half> %vec_half, i64 13
+  %f14 = extractelement <16 x half> %vec_half, i64 14
+  %f15 = extractelement <16 x half> %vec_half, i64 15
+  %sum0 = fadd half %f0, %f1
+  %sum1 = fadd half %f2, %f3
+  %sum2 = fadd half %f4, %f5
+  %sum3 = fadd half %f6, %f7
+  %sum4 = fadd half %f8, %f9
+  %sum5 = fadd half %f10, %f11
+  %sum6 = fadd half %f12, %f13
+  %sum7 = fadd half %f14, %f15
+  %sum8 = fadd half %sum0, %sum1
+  %sum9 = fadd half %sum2, %sum3
+  %sum10 = fadd half %sum4, %sum5
+  %sum11 = fadd half %sum6, %sum7
+  %sum12 = fadd half %sum8, %sum9
+  %sum13 = fadd half %sum10, %sum11
+  %rslt = fadd half %sum12, %sum13
+  ret half %rslt
 }
 
 define <2 x half> @v_test_cvt_v2f64_v2f16(<2 x double> %src) {
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/freeze-binary.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/freeze-binary.ll
new file mode 100644
index 000000000000..b799d6e6b6e9
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/freeze-binary.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck %s
+
+define float @freeze_fneg(float %input) nounwind {
+; CHECK-LABEL: freeze_fneg:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fneg reassoc nsz arcp contract afn float %input
+  %y = freeze float %x
+  %z = fneg reassoc nsz arcp contract afn float %y
+  ret float %z
+}
+
+define float @freeze_fadd(float %input) nounwind {
+; CHECK-LABEL: freeze_fadd:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_add_f32_e32 v0, 2.0, v0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fadd reassoc nsz arcp contract afn float %input, 1.000000e+00
+  %y = freeze float %x
+  %z = fadd reassoc nsz arcp contract afn float %y, 1.000000e+00
+  ret float %z
+}
+
+define <4 x float> @freeze_fadd_vec(<4 x float> %input) nounwind {
+; CHECK-LABEL: freeze_fadd_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_dual_add_f32 v0, 0x40a00000, v0 :: v_dual_add_f32 v1, 0x40a00000, v1
+; CHECK-NEXT:    v_dual_add_f32 v2, 0x40a00000, v2 :: v_dual_add_f32 v3, 0x40a00000, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fadd reassoc nsz arcp contract afn <4 x float> %input, <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+  %y = freeze <4 x float> %x
+  %z = fadd reassoc nsz arcp contract afn <4 x float> %y, <float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00>
+  ret <4 x float> %z
+}
+
+define float @freeze_fsub(float %input) nounwind {
+; CHECK-LABEL: freeze_fsub:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_subrev_f32_e32 v0, 1.0, v0
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_subrev_f32_e32 v0, 1.0, v0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fsub reassoc nsz arcp contract afn float %input, 1.000000e+00
+  %y = freeze float %x
+  %z = fsub reassoc nsz arcp contract afn float %y, 1.000000e+00
+  ret float %z
+}
+
+define <4 x float> @freeze_fsub_vec(<4 x float> %input) nounwind {
+; CHECK-LABEL: freeze_fsub_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_dual_add_f32 v0, 0xc0a00000, v0 :: v_dual_add_f32 v1, 0xc0a00000, v1
+; CHECK-NEXT:    v_dual_add_f32 v2, 0xc0a00000, v2 :: v_dual_add_f32 v3, 0xc0a00000, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fsub reassoc nsz arcp contract afn <4 x float> %input, <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+  %y = freeze <4 x float> %x
+  %z = fsub reassoc nsz arcp contract afn <4 x float> %y, <float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00>
+  ret <4 x float> %z
+}
+
+define float @freeze_fmul(float %input) nounwind {
+; CHECK-LABEL: freeze_fmul:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mul_f32_e32 v0, 4.0, v0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fmul reassoc nsz arcp contract afn float %input, 2.000000e+00
+  %y = freeze float %x
+  %z = fmul reassoc nsz arcp contract afn float %y, 2.000000e+00
+  ret float %z
+}
+
+define <8 x float> @freeze_fmul_vec(<8 x float> %input) nounwind {
+; CHECK-LABEL: freeze_fmul_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_dual_mul_f32 v0, 4.0, v0 :: v_dual_mul_f32 v1, 0x40c00000, v1
+; CHECK-NEXT:    v_dual_mul_f32 v2, 0x40c00000, v2 :: v_dual_mul_f32 v3, 4.0, v3
+; CHECK-NEXT:    v_dual_mul_f32 v4, 4.0, v4 :: v_dual_mul_f32 v5, 0x40c00000, v5
+; CHECK-NEXT:    v_dual_mul_f32 v6, 0x40c00000, v6 :: v_dual_mul_f32 v7, 4.0, v7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fmul reassoc nsz arcp contract afn <8 x float> %input, <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00, float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00>
+  %y = freeze <8 x float> %x
+  %z = fmul reassoc nsz arcp contract afn <8 x float> %y, <float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+  ret <8 x float> %z
+}
+
+define float @freeze_fdiv(float %input) nounwind {
+; CHECK-LABEL: freeze_fdiv:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x3e800000, v0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fdiv reassoc nsz arcp contract afn float %input, 2.000000e+00
+  %y = freeze float %x
+  %z = fdiv reassoc nsz arcp contract afn float %y, 2.000000e+00
+  ret float %z
+}
+
+define <8 x float> @freeze_fdiv_vec(<8 x float> %input) nounwind {
+; CHECK-LABEL: freeze_fdiv_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_dual_mul_f32 v3, 0x3e800000, v3 :: v_dual_mul_f32 v4, 0x3e800000, v4
+; CHECK-NEXT:    v_dual_mul_f32 v0, 0x3e800000, v0 :: v_dual_mul_f32 v7, 0x3e800000, v7
+; CHECK-NEXT:    v_dual_mul_f32 v1, 0x3e2aaaab, v1 :: v_dual_mul_f32 v2, 0x3e2aaaab, v2
+; CHECK-NEXT:    v_dual_mul_f32 v5, 0x3e2aaaab, v5 :: v_dual_mul_f32 v6, 0x3e2aaaab, v6
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = fdiv reassoc nsz arcp contract afn <8 x float> %input, <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00, float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00>
+  %y = freeze <8 x float> %x
+  %z = fdiv reassoc nsz arcp contract afn <8 x float> %y, <float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+  ret <8 x float> %z
+}
+
+define float @freeze_frem(float %input) nounwind {
+; CHECK-LABEL: freeze_frem:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0.5, v0
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_trunc_f32_e32 v1, v1
+; CHECK-NEXT:    v_fmac_f32_e32 v0, -2.0, v1
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0.5, v0
+; CHECK-NEXT:    v_trunc_f32_e32 v1, v1
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT:    v_fmac_f32_e32 v0, -2.0, v1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = frem reassoc nsz arcp contract afn float %input, 2.000000e+00
+  %y = freeze float %x
+  %z = frem reassoc nsz arcp contract afn float %y, 2.000000e+00
+  ret float %z
+}
+
+define <8 x float> @freeze_frem_vec(<8 x float> %input) nounwind {
+; CHECK-LABEL: freeze_frem_vec:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_dual_mul_f32 v8, 0x3e800000, v4 :: v_dual_mul_f32 v9, 0x3e800000, v3
+; CHECK-NEXT:    v_trunc_f32_e32 v11, v0
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; CHECK-NEXT:    v_trunc_f32_e32 v8, v8
+; CHECK-NEXT:    v_trunc_f32_e32 v9, v9
+; CHECK-NEXT:    v_mul_f32_e32 v10, 0.5, v6
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; CHECK-NEXT:    v_dual_sub_f32 v0, v0, v11 :: v_dual_mul_f32 v11, 0x3eaaaaab, v5
+; CHECK-NEXT:    v_dual_fmac_f32 v4, -4.0, v8 :: v_dual_fmac_f32 v3, -4.0, v9
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; CHECK-NEXT:    v_trunc_f32_e32 v10, v10
+; CHECK-NEXT:    v_trunc_f32_e32 v9, v7
+; CHECK-NEXT:    v_dual_fmac_f32 v6, -2.0, v10 :: v_dual_sub_f32 v7, v7, v9
+; CHECK-NEXT:    v_mul_f32_e32 v8, 0.5, v1
+; CHECK-NEXT:    v_trunc_f32_e32 v9, v11
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; CHECK-NEXT:    v_mul_f32_e32 v11, 0x3e800000, v7
+; CHECK-NEXT:    v_trunc_f32_e32 v8, v8
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; CHECK-NEXT:    v_fmac_f32_e32 v1, -2.0, v8
+; CHECK-NEXT:    v_fmac_f32_e32 v5, 0xc0400000, v9
+; CHECK-NEXT:    v_mul_f32_e32 v10, 0x3eaaaaab, v2
+; CHECK-NEXT:    v_mul_f32_e32 v12, 0x3e800000, v0
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT:    v_trunc_f32_e32 v8, v10
+; CHECK-NEXT:    v_trunc_f32_e32 v10, v12
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; CHECK-NEXT:    v_fmac_f32_e32 v2, 0xc0400000, v8
+; CHECK-NEXT:    v_trunc_f32_e32 v8, v11
+; CHECK-NEXT:    v_mul_f32_e32 v12, 0x3eaaaaab, v1
+; CHECK-NEXT:    v_dual_fmac_f32 v0, -4.0, v10 :: v_dual_mul_f32 v11, 0.5, v5
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; CHECK-NEXT:    v_fmac_f32_e32 v7, -4.0, v8
+; CHECK-NEXT:    v_trunc_f32_e32 v9, v12
+; CHECK-NEXT:    v_mul_f32_e32 v12, 0x3eaaaaab, v6
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; CHECK-NEXT:    v_fmac_f32_e32 v1, 0xc0400000, v9
+; CHECK-NEXT:    v_trunc_f32_e32 v9, v11
+; CHECK-NEXT:    v_trunc_f32_e32 v11, v3
+; CHECK-NEXT:    v_dual_mul_f32 v10, 0.5, v2 :: v_dual_fmac_f32 v5, -2.0, v9
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT:    v_sub_f32_e32 v3, v3, v11
+; CHECK-NEXT:    v_trunc_f32_e32 v8, v10
+; CHECK-NEXT:    v_trunc_f32_e32 v10, v12
+; CHECK-NEXT:    v_trunc_f32_e32 v12, v4
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; CHECK-NEXT:    v_fmac_f32_e32 v2, -2.0, v8
+; CHECK-NEXT:    v_fmac_f32_e32 v6, 0xc0400000, v10
+; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; CHECK-NEXT:    v_sub_f32_e32 v4, v4, v12
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %x = frem reassoc nsz arcp contract afn <8 x float> %input, <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00, float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00>
+  %y = freeze <8 x float> %x
+  %z = frem reassoc nsz arcp contract afn <8 x float> %y, <float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+  ret <8 x float> %z
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/freeze.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/freeze.ll
index 2040aedc250e..ac438062ae20 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -10546,14 +10546,14 @@ define void @freeze_v16p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
 ; GFX6-SDAG-LABEL: freeze_v16p3:
 ; GFX6-SDAG:       ; %bb.0:
 ; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v6, vcc, 8, v0
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v2, vcc, 24, v0
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v4, vcc, 16, v0
-; GFX6-SDAG-NEXT:    s_mov_b32 m0, -1
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v14, vcc, 32, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v4, vcc, 24, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v6, vcc, 16, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v16, vcc, 56, v0
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v10, vcc, 48, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v14, vcc, 48, v0
+; GFX6-SDAG-NEXT:    s_mov_b32 m0, -1
 ; GFX6-SDAG-NEXT:    ds_read_b64 v[2:3], v2
 ; GFX6-SDAG-NEXT:    ds_read_b64 v[4:5], v4
 ; GFX6-SDAG-NEXT:    ds_read_b64 v[6:7], v6
@@ -10563,22 +10563,23 @@ define void @freeze_v16p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
 ; GFX6-SDAG-NEXT:    ds_read_b64 v[14:15], v14
 ; GFX6-SDAG-NEXT:    ds_read_b64 v[16:17], v16
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 48, v1
-; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(3)
-; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[10:11]
+; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(4)
+; GFX6-SDAG-NEXT:    ds_write_b64 v1, v[8:9]
+; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(2)
+; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[14:15]
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 56, v1
-; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[16:17]
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 32, v1
-; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[14:15]
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 40, v1
 ; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[12:13]
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 40, v1
+; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[10:11]
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 16, v1
-; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[4:5]
+; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[6:7]
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 24, v1
-; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[2:3]
+; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[4:5]
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 8, v1
-; GFX6-SDAG-NEXT:    ds_write_b64 v1, v[8:9]
-; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[6:7]
+; GFX6-SDAG-NEXT:    ds_write_b64 v0, v[2:3]
 ; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -11565,22 +11566,22 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v9, vcc, 44, v0
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v11, vcc, 36, v0
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v12, vcc, 28, v0
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v13, vcc, 24, v0
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v14, vcc, 20, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v13, vcc, 28, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v14, vcc, 24, v0
+; GFX6-SDAG-NEXT:    v_add_i32_e32 v15, vcc, 20, v0
 ; GFX6-SDAG-NEXT:    buffer_load_dword v2, v2, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_load_dword v3, v3, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_load_dword v4, v4, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_load_dword v9, v9, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_load_dword v10, v10, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_load_dword v11, v11, s[0:3], 0 offen
-; GFX6-SDAG-NEXT:    buffer_load_dword v15, v0, s[0:3], 0 offen
-; GFX6-SDAG-NEXT:    buffer_load_dword v12, v12, s[0:3], 0 offen
-; GFX6-SDAG-NEXT:    buffer_load_dword v13, v13, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    buffer_load_dword v16, v0, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    buffer_load_dword v15, v15, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_load_dword v14, v14, s[0:3], 0 offen
-; GFX6-SDAG-NEXT:    v_add_i32_e32 v16, vcc, 32, v0
+; GFX6-SDAG-NEXT:    buffer_load_dword v13, v13, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    buffer_load_dword v12, v12, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 60, v0
-; GFX6-SDAG-NEXT:    buffer_load_dword v16, v16, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v17, vcc, 4, v1
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v18, vcc, 8, v1
@@ -11603,13 +11604,15 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v19, vcc, 36, v1
 ; GFX6-SDAG-NEXT:    v_add_i32_e32 v8, vcc, 44, v1
 ; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(9)
-; GFX6-SDAG-NEXT:    buffer_store_dword v15, v1, s[0:3], 0 offen
-; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(7)
-; GFX6-SDAG-NEXT:    buffer_store_dword v14, v17, s[0:3], 0 offen
-; GFX6-SDAG-NEXT:    buffer_store_dword v13, v7, s[0:3], 0 offen
-; GFX6-SDAG-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    buffer_store_dword v16, v1, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(9)
-; GFX6-SDAG-NEXT:    buffer_store_dword v16, v6, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    buffer_store_dword v15, v17, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(9)
+; GFX6-SDAG-NEXT:    buffer_store_dword v14, v7, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(9)
+; GFX6-SDAG-NEXT:    buffer_store_dword v13, v18, s[0:3], 0 offen
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(9)
+; GFX6-SDAG-NEXT:    buffer_store_dword v12, v6, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_store_dword v11, v19, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_store_dword v10, v5, s[0:3], 0 offen
 ; GFX6-SDAG-NEXT:    buffer_store_dword v9, v8, s[0:3], 0 offen
@@ -11631,24 +11634,24 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX6-GISEL:       ; %bb.0:
 ; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 4, v0
-; GFX6-GISEL-NEXT:    buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v3, vcc, 8, v0
 ; GFX6-GISEL-NEXT:    buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 8, v0
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 12, v0
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v6, vcc, 16, v0
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v7, vcc, 20, v0
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v9, vcc, 28, v0
+; GFX6-GISEL-NEXT:    buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 12, v0
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 16, v0
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v7, vcc, 24, v0
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v8, vcc, 28, v0
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v10, vcc, 32, v0
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v11, vcc, 36, v0
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v13, vcc, 44, v0
 ; GFX6-GISEL-NEXT:    buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    buffer_load_dword v9, v0, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_load_dword v5, v5, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_load_dword v6, v6, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_load_dword v7, v7, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_load_dword v8, v8, s[0:3], 0 offen
-; GFX6-GISEL-NEXT:    buffer_load_dword v9, v9, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_load_dword v10, v10, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_load_dword v11, v11, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_load_dword v12, v12, s[0:3], 0 offen
@@ -11669,30 +11672,32 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX6-GISEL-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 16, v1
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v17, vcc, 20, v1
-; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(12)
-; GFX6-GISEL-NEXT:    buffer_store_dword v6, v2, s[0:3], 0 offen
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 40, v1
-; GFX6-GISEL-NEXT:    buffer_store_dword v4, v18, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    buffer_store_dword v3, v18, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    s_waitcnt expcnt(0)
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 24, v1
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v3, vcc, 24, v1
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v18, vcc, 28, v1
-; GFX6-GISEL-NEXT:    buffer_store_dword v5, v19, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(14)
+; GFX6-GISEL-NEXT:    buffer_store_dword v4, v19, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    s_waitcnt expcnt(0)
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 32, v1
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 32, v1
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(14)
+; GFX6-GISEL-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 40, v1
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v19, vcc, 36, v1
-; GFX6-GISEL-NEXT:    v_add_i32_e32 v6, vcc, 44, v1
-; GFX6-GISEL-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    s_waitcnt expcnt(0)
+; GFX6-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 44, v1
+; GFX6-GISEL-NEXT:    buffer_store_dword v9, v1, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX6-GISEL-NEXT:    buffer_store_dword v7, v17, s[0:3], 0 offen
-; GFX6-GISEL-NEXT:    buffer_store_dword v8, v4, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    buffer_store_dword v6, v17, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    buffer_store_dword v7, v3, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX6-GISEL-NEXT:    buffer_store_dword v9, v18, s[0:3], 0 offen
-; GFX6-GISEL-NEXT:    buffer_store_dword v10, v5, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    buffer_store_dword v8, v18, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(14)
 ; GFX6-GISEL-NEXT:    buffer_store_dword v11, v19, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    buffer_store_dword v12, v2, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX6-GISEL-NEXT:    buffer_store_dword v13, v6, s[0:3], 0 offen
+; GFX6-GISEL-NEXT:    buffer_store_dword v13, v5, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 48, v1
 ; GFX6-GISEL-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
 ; GFX6-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 52, v1
@@ -11723,22 +11728,22 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v9, vcc, 44, v0
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v10, vcc, 40, v0
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v11, vcc, 36, v0
-; GFX7-SDAG-NEXT:    v_add_i32_e32 v12, vcc, 28, v0
-; GFX7-SDAG-NEXT:    v_add_i32_e32 v13, vcc, 24, v0
-; GFX7-SDAG-NEXT:    v_add_i32_e32 v14, vcc, 20, v0
+; GFX7-SDAG-NEXT:    v_add_i32_e32 v12, vcc, 32, v0
+; GFX7-SDAG-NEXT:    v_add_i32_e32 v13, vcc, 28, v0
+; GFX7-SDAG-NEXT:    v_add_i32_e32 v14, vcc, 24, v0
+; GFX7-SDAG-NEXT:    v_add_i32_e32 v15, vcc, 20, v0
 ; GFX7-SDAG-NEXT:    buffer_load_dword v2, v2, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_load_dword v3, v3, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_load_dword v4, v4, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_load_dword v9, v9, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_load_dword v10, v10, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_load_dword v11, v11, s[0:3], 0 offen
-; GFX7-SDAG-NEXT:    buffer_load_dword v15, v0, s[0:3], 0 offen
-; GFX7-SDAG-NEXT:    buffer_load_dword v12, v12, s[0:3], 0 offen
-; GFX7-SDAG-NEXT:    buffer_load_dword v13, v13, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    buffer_load_dword v16, v0, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    buffer_load_dword v15, v15, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_load_dword v14, v14, s[0:3], 0 offen
-; GFX7-SDAG-NEXT:    v_add_i32_e32 v16, vcc, 32, v0
+; GFX7-SDAG-NEXT:    buffer_load_dword v13, v13, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    buffer_load_dword v12, v12, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v0, vcc, 60, v0
-; GFX7-SDAG-NEXT:    buffer_load_dword v16, v16, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v17, vcc, 4, v1
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v18, vcc, 8, v1
@@ -11759,13 +11764,15 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v19, vcc, 36, v1
 ; GFX7-SDAG-NEXT:    v_add_i32_e32 v8, vcc, 44, v1
 ; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(9)
-; GFX7-SDAG-NEXT:    buffer_store_dword v15, v1, s[0:3], 0 offen
-; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(7)
-; GFX7-SDAG-NEXT:    buffer_store_dword v14, v17, s[0:3], 0 offen
-; GFX7-SDAG-NEXT:    buffer_store_dword v13, v7, s[0:3], 0 offen
-; GFX7-SDAG-NEXT:    buffer_store_dword v12, v18, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    buffer_store_dword v16, v1, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(9)
+; GFX7-SDAG-NEXT:    buffer_store_dword v15, v17, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(9)
-; GFX7-SDAG-NEXT:    buffer_store_dword v16, v6, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    buffer_store_dword v14, v7, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(9)
+; GFX7-SDAG-NEXT:    buffer_store_dword v13, v18, s[0:3], 0 offen
+; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(9)
+; GFX7-SDAG-NEXT:    buffer_store_dword v12, v6, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_store_dword v11, v19, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_store_dword v10, v5, s[0:3], 0 offen
 ; GFX7-SDAG-NEXT:    buffer_store_dword v9, v8, s[0:3], 0 offen
@@ -11785,24 +11792,24 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX7-GISEL:       ; %bb.0:
 ; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 4, v0
-; GFX7-GISEL-NEXT:    buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v3, vcc, 8, v0
 ; GFX7-GISEL-NEXT:    buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 8, v0
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 12, v0
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v6, vcc, 16, v0
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v7, vcc, 20, v0
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v8, vcc, 24, v0
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v9, vcc, 28, v0
+; GFX7-GISEL-NEXT:    buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 12, v0
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 16, v0
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v6, vcc, 20, v0
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v7, vcc, 24, v0
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v8, vcc, 28, v0
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v10, vcc, 32, v0
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v11, vcc, 36, v0
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v12, vcc, 40, v0
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v13, vcc, 44, v0
 ; GFX7-GISEL-NEXT:    buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    buffer_load_dword v9, v0, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_load_dword v5, v5, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_load_dword v6, v6, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_load_dword v7, v7, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_load_dword v8, v8, s[0:3], 0 offen
-; GFX7-GISEL-NEXT:    buffer_load_dword v9, v9, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_load_dword v10, v10, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_load_dword v11, v11, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_load_dword v12, v12, s[0:3], 0 offen
@@ -11822,28 +11829,29 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX7-GISEL-NEXT:    buffer_store_dword v2, v17, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 16, v1
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v17, vcc, 20, v1
-; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(12)
-; GFX7-GISEL-NEXT:    buffer_store_dword v6, v2, s[0:3], 0 offen
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 40, v1
-; GFX7-GISEL-NEXT:    buffer_store_dword v4, v18, s[0:3], 0 offen
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 24, v1
+; GFX7-GISEL-NEXT:    buffer_store_dword v3, v18, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v3, vcc, 24, v1
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v18, vcc, 28, v1
-; GFX7-GISEL-NEXT:    buffer_store_dword v5, v19, s[0:3], 0 offen
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 32, v1
+; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(14)
+; GFX7-GISEL-NEXT:    buffer_store_dword v4, v19, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v4, vcc, 32, v1
+; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(14)
+; GFX7-GISEL-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 40, v1
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v19, vcc, 36, v1
-; GFX7-GISEL-NEXT:    v_add_i32_e32 v6, vcc, 44, v1
-; GFX7-GISEL-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    v_add_i32_e32 v5, vcc, 44, v1
+; GFX7-GISEL-NEXT:    buffer_store_dword v9, v1, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX7-GISEL-NEXT:    buffer_store_dword v7, v17, s[0:3], 0 offen
-; GFX7-GISEL-NEXT:    buffer_store_dword v8, v4, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    buffer_store_dword v6, v17, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    buffer_store_dword v7, v3, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX7-GISEL-NEXT:    buffer_store_dword v9, v18, s[0:3], 0 offen
-; GFX7-GISEL-NEXT:    buffer_store_dword v10, v5, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    buffer_store_dword v8, v18, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(14)
 ; GFX7-GISEL-NEXT:    buffer_store_dword v11, v19, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    buffer_store_dword v12, v2, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX7-GISEL-NEXT:    buffer_store_dword v13, v6, s[0:3], 0 offen
+; GFX7-GISEL-NEXT:    buffer_store_dword v13, v5, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 48, v1
 ; GFX7-GISEL-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
 ; GFX7-GISEL-NEXT:    v_add_i32_e32 v2, vcc, 52, v1
@@ -11861,24 +11869,24 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX8-GISEL:       ; %bb.0:
 ; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v2, vcc, 4, v0
-; GFX8-GISEL-NEXT:    buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v3, vcc, 8, v0
 ; GFX8-GISEL-NEXT:    buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v4, vcc, 8, v0
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v5, vcc, 12, v0
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v6, vcc, 16, v0
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v7, vcc, 20, v0
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v8, vcc, 24, v0
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v9, vcc, 28, v0
+; GFX8-GISEL-NEXT:    buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v4, vcc, 12, v0
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v5, vcc, 16, v0
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v6, vcc, 20, v0
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v7, vcc, 24, v0
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v8, vcc, 28, v0
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v10, vcc, 32, v0
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v11, vcc, 36, v0
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v12, vcc, 40, v0
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v13, vcc, 44, v0
 ; GFX8-GISEL-NEXT:    buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    buffer_load_dword v9, v0, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_load_dword v5, v5, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_load_dword v6, v6, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_load_dword v7, v7, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_load_dword v8, v8, s[0:3], 0 offen
-; GFX8-GISEL-NEXT:    buffer_load_dword v9, v9, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_load_dword v10, v10, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_load_dword v11, v11, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_load_dword v12, v12, s[0:3], 0 offen
@@ -11898,28 +11906,29 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
 ; GFX8-GISEL-NEXT:    buffer_store_dword v2, v17, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v2, vcc, 16, v1
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v17, vcc, 20, v1
-; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(12)
-; GFX8-GISEL-NEXT:    buffer_store_dword v6, v2, s[0:3], 0 offen
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v2, vcc, 40, v1
-; GFX8-GISEL-NEXT:    buffer_store_dword v4, v18, s[0:3], 0 offen
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v4, vcc, 24, v1
+; GFX8-GISEL-NEXT:    buffer_store_dword v3, v18, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v3, vcc, 24, v1
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v18, vcc, 28, v1
-; GFX8-GISEL-NEXT:    buffer_store_dword v5, v19, s[0:3], 0 offen
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v5, vcc, 32, v1
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(14)
+; GFX8-GISEL-NEXT:    buffer_store_dword v4, v19, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v4, vcc, 32, v1
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(14)
+; GFX8-GISEL-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v2, vcc, 40, v1
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v19, vcc, 36, v1
-; GFX8-GISEL-NEXT:    v_add_u32_e32 v6, vcc, 44, v1
-; GFX8-GISEL-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    v_add_u32_e32 v5, vcc, 44, v1
+; GFX8-GISEL-NEXT:    buffer_store_dword v9, v1, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX8-GISEL-NEXT:    buffer_store_dword v7, v17, s[0:3], 0 offen
-; GFX8-GISEL-NEXT:    buffer_store_dword v8, v4, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    buffer_store_dword v6, v17, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    buffer_store_dword v7, v3, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX8-GISEL-NEXT:    buffer_store_dword v9, v18, s[0:3], 0 offen
-; GFX8-GISEL-NEXT:    buffer_store_dword v10, v5, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    buffer_store_dword v8, v18, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    buffer_store_dword v10, v4, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(14)
 ; GFX8-GISEL-NEXT:    buffer_store_dword v11, v19, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    buffer_store_dword v12, v2, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(14)
-; GFX8-GISEL-NEXT:    buffer_store_dword v13, v6, s[0:3], 0 offen
+; GFX8-GISEL-NEXT:    buffer_store_dword v13, v5, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v2, vcc, 48, v1
 ; GFX8-GISEL-NEXT:    buffer_store_dword v14, v2, s[0:3], 0 offen
 ; GFX8-GISEL-NEXT:    v_add_u32_e32 v2, vcc, 52, v1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/frem.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/frem.ll
index 125d009429cb..d3432daedadf 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -6,7 +6,10 @@
 ; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11,GFX11-TRUE16 %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11,GFX11-FAKE16 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1150 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX1150 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1150 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX1150,GFX1150-TRUE16 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1150 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX1150,GFX1150-FAKE16 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX1200,GFX1200-TRUE16 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX1200,GFX1200-FAKE16 %s
 
 define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1,
 ; SI-LABEL: frem_f16:
@@ -255,42 +258,157 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
 ; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
 ; GFX11-FAKE16-NEXT:    s_endpgm
 ;
-; GFX1150-LABEL: frem_f16:
-; GFX1150:       ; %bb.0:
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1150-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1150-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1150-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    global_load_u16 v1, v0, s[2:3]
-; GFX1150-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
-; GFX1150-NEXT:    s_waitcnt vmcnt(1)
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v3, v1
-; GFX1150-NEXT:    s_waitcnt vmcnt(0)
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v4, v2
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1150-NEXT:    v_rcp_f32_e32 v4, v4
-; GFX1150-NEXT:    v_mul_f32_e32 v3, v3, v4
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v5, -v2, v3, v1 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_fmac_f32_e32 v3, v5, v4
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v5, -v2, v3, v1 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_mul_f32_e32 v4, v5, v4
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_and_b32_e32 v4, 0xff800000, v4
-; GFX1150-NEXT:    v_add_f32_e32 v3, v4, v3
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX1150-NEXT:    v_div_fixup_f16 v3, v3, v2, v1
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_trunc_f16_e32 v3, v3
-; GFX1150-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1150-NEXT:    v_fmac_f16_e32 v1, v3, v2
-; GFX1150-NEXT:    global_store_b16 v0, v1, s[0:1]
-; GFX1150-NEXT:    s_endpgm
+; GFX1150-TRUE16-LABEL: frem_f16:
+; GFX1150-TRUE16:       ; %bb.0:
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1150-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX1150-TRUE16-NEXT:    global_load_d16_b16 v1, v2, s[4:5] offset:8
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v3, v0.l
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v4, v1.l
+; GFX1150-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
+; GFX1150-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_rcp_f32_e32 v4, v4
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v3, v3, v4
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v7, -v5, v3, v6 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_fmac_f32_e32 v3, v7, v4
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v5, -v5, v3, v6 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v4, v5, v4
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff800000, v4
+; GFX1150-TRUE16-NEXT:    v_add_f32_e32 v3, v4, v3
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.h, v3
+; GFX1150-TRUE16-NEXT:    v_div_fixup_f16 v0.h, v0.h, v1.l, v0.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v3.l, v0.h
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fmac_f16_e32 v0.l, v3.l, v1.l
+; GFX1150-TRUE16-NEXT:    global_store_b16 v2, v0, s[0:1]
+; GFX1150-TRUE16-NEXT:    s_endpgm
+;
+; GFX1150-FAKE16-LABEL: frem_f16:
+; GFX1150-FAKE16:       ; %bb.0:
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1150-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX1150-FAKE16-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v3, v1
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v4, v2
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_rcp_f32_e32 v4, v4
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v3, v3, v4
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v5, -v2, v3, v1 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_fmac_f32_e32 v3, v5, v4
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v5, -v2, v3, v1 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v4, v5, v4
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff800000, v4
+; GFX1150-FAKE16-NEXT:    v_add_f32_e32 v3, v4, v3
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX1150-FAKE16-NEXT:    v_div_fixup_f16 v3, v3, v2, v1
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v3, v3
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v1, v3, v2
+; GFX1150-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX1150-FAKE16-NEXT:    s_endpgm
+;
+; GFX1200-TRUE16-LABEL: frem_f16:
+; GFX1200-TRUE16:       ; %bb.0:
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1200-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX1200-TRUE16-NEXT:    global_load_d16_b16 v1, v2, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x1
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v3, v0.l
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v4, v1.l
+; GFX1200-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
+; GFX1200-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_rcp_f32_e32 v4, v4
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v3, v3, v4
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v7, -v5, v3, v6 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_fmac_f32_e32 v3, v7, v4
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v5, -v5, v3, v6 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v4, v5, v4
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff800000, v4
+; GFX1200-TRUE16-NEXT:    v_add_f32_e32 v3, v4, v3
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.h, v3
+; GFX1200-TRUE16-NEXT:    v_div_fixup_f16 v0.h, v0.h, v1.l, v0.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v3.l, v0.h
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fmac_f16_e32 v0.l, v3.l, v1.l
+; GFX1200-TRUE16-NEXT:    global_store_b16 v2, v0, s[0:1]
+; GFX1200-TRUE16-NEXT:    s_endpgm
+;
+; GFX1200-FAKE16-LABEL: frem_f16:
+; GFX1200-FAKE16:       ; %bb.0:
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1200-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x1
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v3, v1
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v4, v2
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_rcp_f32_e32 v4, v4
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v3, v3, v4
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v5, -v2, v3, v1 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_fmac_f32_e32 v3, v5, v4
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v5, -v2, v3, v1 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v4, v5, v4
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff800000, v4
+; GFX1200-FAKE16-NEXT:    v_add_f32_e32 v3, v4, v3
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX1200-FAKE16-NEXT:    v_div_fixup_f16 v3, v3, v2, v1
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v3, v3
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v1, v3, v2
+; GFX1200-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX1200-FAKE16-NEXT:    s_endpgm
                       ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
    %r0 = load half, ptr addrspace(1) %in1, align 4
@@ -456,26 +574,89 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
 ; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
 ; GFX11-FAKE16-NEXT:    s_endpgm
 ;
-; GFX1150-LABEL: fast_frem_f16:
-; GFX1150:       ; %bb.0:
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1150-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1150-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1150-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    global_load_u16 v1, v0, s[2:3]
-; GFX1150-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
-; GFX1150-NEXT:    s_waitcnt vmcnt(0)
-; GFX1150-NEXT:    v_rcp_f16_e32 v3, v2
-; GFX1150-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f16_e32 v3, v1, v3
-; GFX1150-NEXT:    v_trunc_f16_e32 v3, v3
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
-; GFX1150-NEXT:    v_fmac_f16_e32 v1, v3, v2
-; GFX1150-NEXT:    global_store_b16 v0, v1, s[0:1]
-; GFX1150-NEXT:    s_endpgm
+; GFX1150-TRUE16-LABEL: fast_frem_f16:
+; GFX1150-TRUE16:       ; %bb.0:
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1150-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX1150-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5] offset:8
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-TRUE16-NEXT:    v_rcp_f16_e32 v1.l, v0.h
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f16_e32 v1.l, v0.l, v1.l
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v1.l, v1.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; GFX1150-TRUE16-NEXT:    v_fmac_f16_e32 v0.l, v1.l, v0.h
+; GFX1150-TRUE16-NEXT:    global_store_b16 v2, v0, s[0:1]
+; GFX1150-TRUE16-NEXT:    s_endpgm
+;
+; GFX1150-FAKE16-LABEL: fast_frem_f16:
+; GFX1150-FAKE16:       ; %bb.0:
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1150-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX1150-FAKE16-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-FAKE16-NEXT:    v_rcp_f16_e32 v3, v2
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f16_e32 v3, v1, v3
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v3, v3
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v1, v3, v2
+; GFX1150-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX1150-FAKE16-NEXT:    s_endpgm
+;
+; GFX1200-TRUE16-LABEL: fast_frem_f16:
+; GFX1200-TRUE16:       ; %bb.0:
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1200-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX1200-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-TRUE16-NEXT:    v_rcp_f16_e32 v1.l, v0.h
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f16_e32 v1.l, v0.l, v1.l
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v1.l, v1.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; GFX1200-TRUE16-NEXT:    v_fmac_f16_e32 v0.l, v1.l, v0.h
+; GFX1200-TRUE16-NEXT:    global_store_b16 v2, v0, s[0:1]
+; GFX1200-TRUE16-NEXT:    s_endpgm
+;
+; GFX1200-FAKE16-LABEL: fast_frem_f16:
+; GFX1200-FAKE16:       ; %bb.0:
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1200-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-FAKE16-NEXT:    v_rcp_f16_e32 v3, v2
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f16_e32 v3, v1, v3
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v3, v3
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v1, v3, v2
+; GFX1200-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX1200-FAKE16-NEXT:    s_endpgm
                       ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
    %r0 = load half, ptr addrspace(1) %in1, align 4
@@ -641,26 +822,89 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
 ; GFX11-FAKE16-NEXT:    s_endpgm
 ;
-; GFX1150-LABEL: unsafe_frem_f16:
-; GFX1150:       ; %bb.0:
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1150-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1150-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1150-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    global_load_u16 v1, v0, s[2:3]
-; GFX1150-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
-; GFX1150-NEXT:    s_waitcnt vmcnt(0)
-; GFX1150-NEXT:    v_rcp_f16_e32 v3, v2
-; GFX1150-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f16_e32 v3, v1, v3
-; GFX1150-NEXT:    v_trunc_f16_e32 v3, v3
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
-; GFX1150-NEXT:    v_fmac_f16_e32 v1, v3, v2
-; GFX1150-NEXT:    global_store_b16 v0, v1, s[0:1]
-; GFX1150-NEXT:    s_endpgm
+; GFX1150-TRUE16-LABEL: unsafe_frem_f16:
+; GFX1150-TRUE16:       ; %bb.0:
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1150-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX1150-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5] offset:8
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-TRUE16-NEXT:    v_rcp_f16_e32 v1.l, v0.h
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f16_e32 v1.l, v0.l, v1.l
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v1.l, v1.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; GFX1150-TRUE16-NEXT:    v_fmac_f16_e32 v0.l, v1.l, v0.h
+; GFX1150-TRUE16-NEXT:    global_store_b16 v2, v0, s[0:1]
+; GFX1150-TRUE16-NEXT:    s_endpgm
+;
+; GFX1150-FAKE16-LABEL: unsafe_frem_f16:
+; GFX1150-FAKE16:       ; %bb.0:
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1150-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX1150-FAKE16-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-FAKE16-NEXT:    v_rcp_f16_e32 v3, v2
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f16_e32 v3, v1, v3
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v3, v3
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v1, v3, v2
+; GFX1150-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX1150-FAKE16-NEXT:    s_endpgm
+;
+; GFX1200-TRUE16-LABEL: unsafe_frem_f16:
+; GFX1200-TRUE16:       ; %bb.0:
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1200-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX1200-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-TRUE16-NEXT:    v_rcp_f16_e32 v1.l, v0.h
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f16_e32 v1.l, v0.l, v1.l
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v1.l, v1.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
+; GFX1200-TRUE16-NEXT:    v_fmac_f16_e32 v0.l, v1.l, v0.h
+; GFX1200-TRUE16-NEXT:    global_store_b16 v2, v0, s[0:1]
+; GFX1200-TRUE16-NEXT:    s_endpgm
+;
+; GFX1200-FAKE16-LABEL: unsafe_frem_f16:
+; GFX1200-FAKE16:       ; %bb.0:
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1200-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT:    global_load_u16 v2, v0, s[4:5] offset:8
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-FAKE16-NEXT:    v_rcp_f16_e32 v3, v2
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f16_e32 v3, v1, v3
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v3, v3
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v3, 0x8000, v3
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v1, v3, v2
+; GFX1200-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX1200-FAKE16-NEXT:    s_endpgm
                              ptr addrspace(1) %in2) #1 {
    %gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
    %r0 = load half, ptr addrspace(1) %in1, align 4
@@ -903,6 +1147,42 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1
 ; GFX1150-NEXT:    v_fmac_f32_e32 v1, v3, v2
 ; GFX1150-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: frem_f32:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1200-NEXT:    global_load_b32 v2, v0, s[4:5] offset:16
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_div_scale_f32 v4, null, v2, v2, v1
+; GFX1200-NEXT:    v_div_scale_f32 v3, vcc_lo, v1, v2, v1
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX1200-NEXT:    s_denorm_mode 15
+; GFX1200-NEXT:    v_fma_f32 v6, -v4, v5, 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fmac_f32_e32 v5, v6, v5
+; GFX1200-NEXT:    v_mul_f32_e32 v6, v3, v5
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v7, -v4, v6, v3
+; GFX1200-NEXT:    v_fmac_f32_e32 v6, v7, v5
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v3, -v4, v6, v3
+; GFX1200-NEXT:    s_denorm_mode 12
+; GFX1200-NEXT:    v_div_fmas_f32 v3, v3, v5, v6
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f32 v3, v3, v2, v1
+; GFX1200-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-NEXT:    v_fmac_f32_e32 v1, v3, v2
+; GFX1200-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1200-NEXT:    s_endpgm
                       ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr float, ptr addrspace(1) %in2, i32 4
    %r0 = load float, ptr addrspace(1) %in1, align 4
@@ -1060,6 +1340,27 @@ define amdgpu_kernel void @fast_frem_f32(ptr addrspace(1) %out, ptr addrspace(1)
 ; GFX1150-NEXT:    v_fmac_f32_e32 v1, v3, v2
 ; GFX1150-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: fast_frem_f32:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1200-NEXT:    global_load_b32 v2, v0, s[4:5] offset:16
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX1200-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX1200-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-NEXT:    v_fmac_f32_e32 v1, v3, v2
+; GFX1200-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1200-NEXT:    s_endpgm
                       ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr float, ptr addrspace(1) %in2, i32 4
    %r0 = load float, ptr addrspace(1) %in1, align 4
@@ -1217,6 +1518,27 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1150-NEXT:    v_fmac_f32_e32 v1, v3, v2
 ; GFX1150-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: unsafe_frem_f32:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1200-NEXT:    global_load_b32 v2, v0, s[4:5] offset:16
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX1200-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX1200-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-NEXT:    v_fmac_f32_e32 v1, v3, v2
+; GFX1200-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1200-NEXT:    s_endpgm
                              ptr addrspace(1) %in2) #1 {
    %gep2 = getelementptr float, ptr addrspace(1) %in2, i32 4
    %r0 = load float, ptr addrspace(1) %in1, align 4
@@ -1469,6 +1791,39 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
 ; GFX1150-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; GFX1150-NEXT:    global_store_b64 v12, v[0:1], s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: frem_f64:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v12, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b64 v[0:1], v12, s[2:3]
+; GFX1200-NEXT:    global_load_b64 v[2:3], v12, s[4:5]
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_div_scale_f64 v[4:5], null, v[2:3], v[2:3], v[0:1]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX1200-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX1200-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX1200-NEXT:    v_div_scale_f64 v[8:9], vcc_lo, v[0:1], v[2:3], v[0:1]
+; GFX1200-NEXT:    v_mul_f64_e32 v[10:11], v[8:9], v[6:7]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
+; GFX1200-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
+; GFX1200-NEXT:    v_trunc_f64_e32 v[4:5], v[4:5]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; GFX1200-NEXT:    global_store_b64 v12, v[0:1], s[0:1]
+; GFX1200-NEXT:    s_endpgm
                       ptr addrspace(1) %in2) #0 {
    %r0 = load double, ptr addrspace(1) %in1, align 8
    %r1 = load double, ptr addrspace(1) %in2, align 8
@@ -1690,6 +2045,35 @@ define amdgpu_kernel void @fast_frem_f64(ptr addrspace(1) %out, ptr addrspace(1)
 ; GFX1150-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; GFX1150-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: fast_frem_f64:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b64 v[0:1], v10, s[2:3]
+; GFX1200-NEXT:    global_load_b64 v[2:3], v10, s[4:5]
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX1200-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX1200-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX1200-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_mul_f64_e32 v[6:7], v[0:1], v[4:5]
+; GFX1200-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
+; GFX1200-NEXT:    v_trunc_f64_e32 v[4:5], v[4:5]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; GFX1200-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX1200-NEXT:    s_endpgm
                       ptr addrspace(1) %in2) #0 {
    %r0 = load double, ptr addrspace(1) %in1, align 8
    %r1 = load double, ptr addrspace(1) %in2, align 8
@@ -1911,6 +2295,35 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1150-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
 ; GFX1150-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: unsafe_frem_f64:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b64 v[0:1], v10, s[2:3]
+; GFX1200-NEXT:    global_load_b64 v[2:3], v10, s[4:5]
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX1200-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX1200-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX1200-NEXT:    v_fma_f64 v[4:5], v[6:7], v[4:5], v[4:5]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_mul_f64_e32 v[6:7], v[0:1], v[4:5]
+; GFX1200-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
+; GFX1200-NEXT:    v_trunc_f64_e32 v[4:5], v[4:5]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; GFX1200-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX1200-NEXT:    s_endpgm
                              ptr addrspace(1) %in2) #1 {
    %r0 = load double, ptr addrspace(1) %in1, align 8
    %r1 = load double, ptr addrspace(1) %in2, align 8
@@ -2308,68 +2721,255 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-FAKE16-NEXT:    s_endpgm
 ;
-; GFX1150-LABEL: frem_v2f16:
-; GFX1150:       ; %bb.0:
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1150-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1150-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1150-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    global_load_b32 v1, v0, s[2:3]
-; GFX1150-NEXT:    global_load_b32 v2, v0, s[4:5] offset:16
-; GFX1150-NEXT:    s_waitcnt vmcnt(1)
-; GFX1150-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX1150-NEXT:    s_waitcnt vmcnt(0)
-; GFX1150-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v4, v3
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v6, v5
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1150-NEXT:    v_rcp_f32_e32 v6, v6
-; GFX1150-NEXT:    v_mul_f32_e32 v4, v4, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v7, -v2, v4, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_fmac_f32_e32 v4, v7, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v7, -v2, v4, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_mul_f32_e32 v6, v7, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
-; GFX1150-NEXT:    v_add_f32_e32 v4, v6, v4
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; GFX1150-NEXT:    v_div_fixup_f16 v4, v4, v5, v3
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_trunc_f16_e32 v4, v4
-; GFX1150-NEXT:    v_xor_b32_e32 v4, 0x8000, v4
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1150-NEXT:    v_fmac_f16_e32 v3, v4, v5
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v5, v2
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v4, v1
-; GFX1150-NEXT:    v_rcp_f32_e32 v5, v5
-; GFX1150-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f32_e32 v4, v4, v5
-; GFX1150-NEXT:    v_fma_mix_f32 v6, -v2, v4, v1 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fmac_f32_e32 v4, v6, v5
-; GFX1150-NEXT:    v_fma_mix_f32 v6, -v2, v4, v1 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f32_e32 v5, v6, v5
-; GFX1150-NEXT:    v_and_b32_e32 v5, 0xff800000, v5
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_add_f32_e32 v4, v5, v4
-; GFX1150-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_div_fixup_f16 v4, v4, v2, v1
-; GFX1150-NEXT:    v_trunc_f16_e32 v4, v4
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_xor_b32_e32 v4, 0x8000, v4
-; GFX1150-NEXT:    v_fmac_f16_e32 v1, v4, v2
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1150-NEXT:    v_pack_b32_f16 v1, v1, v3
-; GFX1150-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX1150-NEXT:    s_endpgm
+; GFX1150-TRUE16-LABEL: frem_v2f16:
+; GFX1150-TRUE16:       ; %bb.0:
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1150-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    global_load_b32 v2, v1, s[2:3]
+; GFX1150-TRUE16-NEXT:    global_load_b32 v3, v1, s[4:5] offset:16
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.h
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v4, v3.h
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_rcp_f32_e32 v4, v4
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v4
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v5, -v3, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_fmac_f32_e32 v0, v5, v4
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v5, -v3, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v4, v5, v4
+; GFX1150-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff800000, v4
+; GFX1150-TRUE16-NEXT:    v_add_f32_e32 v0, v4, v0
+; GFX1150-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1150-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v5.l, v4.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1150-TRUE16-NEXT:    v_fmac_f16_e32 v4.l, v0.l, v5.l
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v5, v3.l
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.l
+; GFX1150-TRUE16-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v5
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v6, -v3, v0, v2 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fmac_f32_e32 v0, v6, v5
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v6, -v3, v0, v2 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v5, v6, v5
+; GFX1150-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff800000, v5
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_add_f32_e32 v0, v5, v0
+; GFX1150-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1150-TRUE16-NEXT:    v_fmac_f16_e32 v2.l, v0.l, v3.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_pack_b32_f16 v0, v2.l, v4.l
+; GFX1150-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX1150-TRUE16-NEXT:    s_endpgm
+;
+; GFX1150-FAKE16-LABEL: frem_v2f16:
+; GFX1150-FAKE16:       ; %bb.0:
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1150-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1150-FAKE16-NEXT:    global_load_b32 v2, v0, s[4:5] offset:16
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX1150-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v4, v3
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v4, v4, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v7, -v2, v4, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_fmac_f32_e32 v4, v7, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v7, -v2, v4, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1150-FAKE16-NEXT:    v_add_f32_e32 v4, v6, v4
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX1150-FAKE16-NEXT:    v_div_fixup_f16 v4, v4, v5, v3
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v4, v4
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v4, 0x8000, v4
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v3, v4, v5
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v5, v2
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v4, v1
+; GFX1150-FAKE16-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v4, v4, v5
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v6, -v2, v4, v1 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fmac_f32_e32 v4, v6, v5
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v6, -v2, v4, v1 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v5, v6, v5
+; GFX1150-FAKE16-NEXT:    v_and_b32_e32 v5, 0xff800000, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_add_f32_e32 v4, v5, v4
+; GFX1150-FAKE16-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_div_fixup_f16 v4, v4, v2, v1
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v4, v4
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v4, 0x8000, v4
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v1, v4, v2
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_pack_b32_f16 v1, v1, v3
+; GFX1150-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1150-FAKE16-NEXT:    s_endpgm
+;
+; GFX1200-TRUE16-LABEL: frem_v2f16:
+; GFX1200-TRUE16:       ; %bb.0:
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    global_load_b32 v2, v1, s[2:3]
+; GFX1200-TRUE16-NEXT:    global_load_b32 v3, v1, s[4:5] offset:16
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x1
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.h
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v4, v3.h
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_rcp_f32_e32 v4, v4
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v4
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v5, -v3, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_fmac_f32_e32 v0, v5, v4
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v5, -v3, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v4, v5, v4
+; GFX1200-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_and_b32_e32 v4, 0xff800000, v4
+; GFX1200-TRUE16-NEXT:    v_add_f32_e32 v0, v4, v0
+; GFX1200-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1200-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v5.l, v4.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-TRUE16-NEXT:    v_fmac_f16_e32 v4.l, v0.l, v5.l
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v5, v3.l
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.l
+; GFX1200-TRUE16-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v5
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v6, -v3, v0, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fmac_f32_e32 v0, v6, v5
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v6, -v3, v0, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v5, v6, v5
+; GFX1200-TRUE16-NEXT:    v_and_b32_e32 v5, 0xff800000, v5
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_add_f32_e32 v0, v5, v0
+; GFX1200-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1200-TRUE16-NEXT:    v_fmac_f16_e32 v2.l, v0.l, v3.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_pack_b32_f16 v0, v2.l, v4.l
+; GFX1200-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX1200-TRUE16-NEXT:    s_endpgm
+;
+; GFX1200-FAKE16-LABEL: frem_v2f16:
+; GFX1200-FAKE16:       ; %bb.0:
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1200-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT:    global_load_b32 v2, v0, s[4:5] offset:16
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x1
+; GFX1200-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v4, v3
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v4, v4, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v7, -v2, v4, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_fmac_f32_e32 v4, v7, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v7, -v2, v4, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1200-FAKE16-NEXT:    v_add_f32_e32 v4, v6, v4
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX1200-FAKE16-NEXT:    v_div_fixup_f16 v4, v4, v5, v3
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v4, v4
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v4, 0x8000, v4
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v3, v4, v5
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v5, v2
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v4, v1
+; GFX1200-FAKE16-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v4, v4, v5
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v6, -v2, v4, v1 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fmac_f32_e32 v4, v6, v5
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v6, -v2, v4, v1 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v5, v6, v5
+; GFX1200-FAKE16-NEXT:    v_and_b32_e32 v5, 0xff800000, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_add_f32_e32 v4, v5, v4
+; GFX1200-FAKE16-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_div_fixup_f16 v4, v4, v2, v1
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v4, v4
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v4, 0x8000, v4
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v1, v4, v2
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_pack_b32_f16 v1, v1, v3
+; GFX1200-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1200-FAKE16-NEXT:    s_endpgm
                         ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr <2 x half>, ptr addrspace(1) %in2, i32 4
    %r0 = load <2 x half>, ptr addrspace(1) %in1, align 8
@@ -3034,115 +3634,447 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX11-FAKE16-NEXT:    global_store_b64 v4, v[0:1], s[0:1]
 ; GFX11-FAKE16-NEXT:    s_endpgm
 ;
-; GFX1150-LABEL: frem_v4f16:
-; GFX1150:       ; %bb.0:
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1150-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1150-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1150-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1150-NEXT:    s_clause 0x1
-; GFX1150-NEXT:    global_load_b64 v[0:1], v4, s[2:3]
-; GFX1150-NEXT:    global_load_b64 v[2:3], v4, s[4:5] offset:32
-; GFX1150-NEXT:    s_waitcnt vmcnt(1)
-; GFX1150-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX1150-NEXT:    s_waitcnt vmcnt(0)
-; GFX1150-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v6, v5
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v8, v7
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1150-NEXT:    v_rcp_f32_e32 v8, v8
-; GFX1150-NEXT:    v_mul_f32_e32 v6, v6, v8
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v9, -v2, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_fmac_f32_e32 v6, v9, v8
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v9, -v2, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_mul_f32_e32 v8, v9, v8
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_and_b32_e32 v8, 0xff800000, v8
-; GFX1150-NEXT:    v_add_f32_e32 v6, v8, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX1150-NEXT:    v_div_fixup_f16 v6, v6, v7, v5
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_trunc_f16_e32 v6, v6
-; GFX1150-NEXT:    v_xor_b32_e32 v6, 0x8000, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1150-NEXT:    v_fmac_f16_e32 v5, v6, v7
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v7, v2
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v6, v0
-; GFX1150-NEXT:    v_rcp_f32_e32 v7, v7
-; GFX1150-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f32_e32 v6, v6, v7
-; GFX1150-NEXT:    v_fma_mix_f32 v8, -v2, v6, v0 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fmac_f32_e32 v6, v8, v7
-; GFX1150-NEXT:    v_fma_mix_f32 v8, -v2, v6, v0 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f32_e32 v7, v8, v7
-; GFX1150-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_add_f32_e32 v6, v7, v6
-; GFX1150-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_div_fixup_f16 v6, v6, v2, v0
-; GFX1150-NEXT:    v_trunc_f16_e32 v6, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_xor_b32_e32 v6, 0x8000, v6
-; GFX1150-NEXT:    v_fma_f16 v0, v6, v2, v0
-; GFX1150-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
-; GFX1150-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1150-NEXT:    v_pack_b32_f16 v0, v0, v5
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v7, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v5, v2
-; GFX1150-NEXT:    v_rcp_f32_e32 v7, v7
-; GFX1150-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f32_e32 v5, v5, v7
-; GFX1150-NEXT:    v_fma_mix_f32 v8, -v3, v5, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fmac_f32_e32 v5, v8, v7
-; GFX1150-NEXT:    v_fma_mix_f32 v8, -v3, v5, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_mul_f32_e32 v7, v8, v7
-; GFX1150-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_add_f32_e32 v5, v7, v5
-; GFX1150-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_div_fixup_f16 v5, v5, v6, v2
-; GFX1150-NEXT:    v_trunc_f16_e32 v5, v5
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_xor_b32_e32 v5, 0x8000, v5
-; GFX1150-NEXT:    v_fmac_f16_e32 v2, v5, v6
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v6, v3
-; GFX1150-NEXT:    v_cvt_f32_f16_e32 v5, v1
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1150-NEXT:    v_rcp_f32_e32 v6, v6
-; GFX1150-NEXT:    v_mul_f32_e32 v5, v5, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v7, -v3, v5, v1 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_fmac_f32_e32 v5, v7, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fma_mix_f32 v7, -v3, v5, v1 op_sel_hi:[1,0,1]
-; GFX1150-NEXT:    v_mul_f32_e32 v6, v7, v6
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
-; GFX1150-NEXT:    v_add_f32_e32 v5, v6, v5
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX1150-NEXT:    v_div_fixup_f16 v5, v5, v3, v1
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_trunc_f16_e32 v5, v5
-; GFX1150-NEXT:    v_xor_b32_e32 v5, 0x8000, v5
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1150-NEXT:    v_fmac_f16_e32 v1, v5, v3
-; GFX1150-NEXT:    v_pack_b32_f16 v1, v1, v2
-; GFX1150-NEXT:    global_store_b64 v4, v[0:1], s[0:1]
-; GFX1150-NEXT:    s_endpgm
+; GFX1150-TRUE16-LABEL: frem_v4f16:
+; GFX1150-TRUE16:       ; %bb.0:
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-TRUE16-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1150-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-TRUE16-NEXT:    s_clause 0x1
+; GFX1150-TRUE16-NEXT:    global_load_b64 v[1:2], v5, s[2:3]
+; GFX1150-TRUE16-NEXT:    global_load_b64 v[3:4], v5, s[4:5] offset:32
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v1.h
+; GFX1150-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v6, v3.h
+; GFX1150-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v3.l
+; GFX1150-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v1.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v6
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v7, -v3, v0, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_fmac_f32_e32 v0, v7, v6
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v7, -v3, v0, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1150-TRUE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1150-TRUE16-NEXT:    v_add_f32_e32 v0, v6, v0
+; GFX1150-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1150-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v7.l, v6.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1150-TRUE16-NEXT:    v_fmac_f16_e32 v6.l, v0.l, v7.l
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v7, v3.l
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v1.l
+; GFX1150-TRUE16-NEXT:    v_rcp_f32_e32 v7, v7
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v7
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v10, -v8, v0, v9 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fmac_f32_e32 v0, v10, v7
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v8, -v8, v0, v9 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v7, v8, v7
+; GFX1150-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_add_f32_e32 v0, v7, v0
+; GFX1150-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v3.l, v1.l
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1150-TRUE16-NEXT:    v_fma_f16 v0.l, v0.l, v3.l, v1.l
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v3, v4.h
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-TRUE16-NEXT:    v_pack_b32_f16 v1, v0.l, v6.l
+; GFX1150-TRUE16-NEXT:    v_rcp_f32_e32 v3, v3
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.h
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v3
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v6, -v4, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_fmac_f32_e32 v0, v6, v3
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v6, -v4, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v3, v6, v3
+; GFX1150-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff800000, v3
+; GFX1150-TRUE16-NEXT:    v_add_f32_e32 v0, v3, v0
+; GFX1150-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1150-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v6.l, v3.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1150-TRUE16-NEXT:    v_fmac_f16_e32 v3.l, v0.l, v6.l
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v6, v4.l
+; GFX1150-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.l
+; GFX1150-TRUE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v6
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v7, -v4, v0, v2 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_fmac_f32_e32 v0, v7, v6
+; GFX1150-TRUE16-NEXT:    v_fma_mix_f32 v7, -v4, v0, v2 op_sel_hi:[1,0,1]
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1150-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_add_f32_e32 v0, v6, v0
+; GFX1150-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v4.l, v2.l
+; GFX1150-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1150-TRUE16-NEXT:    v_fma_f16 v0.l, v0.l, v4.l, v2.l
+; GFX1150-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1150-TRUE16-NEXT:    v_pack_b32_f16 v2, v0.l, v3.l
+; GFX1150-TRUE16-NEXT:    global_store_b64 v5, v[1:2], s[0:1]
+; GFX1150-TRUE16-NEXT:    s_endpgm
+;
+; GFX1150-FAKE16-LABEL: frem_v4f16:
+; GFX1150-FAKE16:       ; %bb.0:
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1150-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1150-FAKE16-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1150-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1150-FAKE16-NEXT:    s_clause 0x1
+; GFX1150-FAKE16-NEXT:    global_load_b64 v[0:1], v4, s[2:3]
+; GFX1150-FAKE16-NEXT:    global_load_b64 v[2:3], v4, s[4:5] offset:32
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX1150-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX1150-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX1150-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v5
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v8, v7
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_rcp_f32_e32 v8, v8
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v6, v6, v8
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v9, -v2, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_fmac_f32_e32 v6, v9, v8
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v9, -v2, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v8, v9, v8
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff800000, v8
+; GFX1150-FAKE16-NEXT:    v_add_f32_e32 v6, v8, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX1150-FAKE16-NEXT:    v_div_fixup_f16 v6, v6, v7, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v6, v6
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v6, 0x8000, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v5, v6, v7
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v7, v2
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v0
+; GFX1150-FAKE16-NEXT:    v_rcp_f32_e32 v7, v7
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v6, v6, v7
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v8, -v2, v6, v0 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fmac_f32_e32 v6, v8, v7
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v8, -v2, v6, v0 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v7, v8, v7
+; GFX1150-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_add_f32_e32 v6, v7, v6
+; GFX1150-FAKE16-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_div_fixup_f16 v6, v6, v2, v0
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v6, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v6, 0x8000, v6
+; GFX1150-FAKE16-NEXT:    v_fma_f16 v0, v6, v2, v0
+; GFX1150-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
+; GFX1150-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1150-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, v5
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v7, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v5, v2
+; GFX1150-FAKE16-NEXT:    v_rcp_f32_e32 v7, v7
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v5, v5, v7
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v8, -v3, v5, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fmac_f32_e32 v5, v8, v7
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v8, -v3, v5, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v7, v8, v7
+; GFX1150-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_add_f32_e32 v5, v7, v5
+; GFX1150-FAKE16-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_div_fixup_f16 v5, v5, v6, v2
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v5, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v5, 0x8000, v5
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v2, v5, v6
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GFX1150-FAKE16-NEXT:    v_cvt_f32_f16_e32 v5, v1
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v5, v5, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v7, -v3, v5, v1 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_fmac_f32_e32 v5, v7, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fma_mix_f32 v7, -v3, v5, v1 op_sel_hi:[1,0,1]
+; GFX1150-FAKE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1150-FAKE16-NEXT:    v_add_f32_e32 v5, v6, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GFX1150-FAKE16-NEXT:    v_div_fixup_f16 v5, v5, v3, v1
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_trunc_f16_e32 v5, v5
+; GFX1150-FAKE16-NEXT:    v_xor_b32_e32 v5, 0x8000, v5
+; GFX1150-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1150-FAKE16-NEXT:    v_fmac_f16_e32 v1, v5, v3
+; GFX1150-FAKE16-NEXT:    v_pack_b32_f16 v1, v1, v2
+; GFX1150-FAKE16-NEXT:    global_store_b64 v4, v[0:1], s[0:1]
+; GFX1150-FAKE16-NEXT:    s_endpgm
+;
+; GFX1200-TRUE16-LABEL: frem_v4f16:
+; GFX1200-TRUE16:       ; %bb.0:
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1200-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-TRUE16-NEXT:    s_clause 0x1
+; GFX1200-TRUE16-NEXT:    global_load_b64 v[1:2], v5, s[2:3]
+; GFX1200-TRUE16-NEXT:    global_load_b64 v[3:4], v5, s[4:5] offset:32
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x1
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v1.h
+; GFX1200-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v6, v3.h
+; GFX1200-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v3.l
+; GFX1200-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v1.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v6
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v7, -v3, v0, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_fmac_f32_e32 v0, v7, v6
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v7, -v3, v0, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1200-TRUE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1200-TRUE16-NEXT:    v_add_f32_e32 v0, v6, v0
+; GFX1200-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1200-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v7.l, v6.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-TRUE16-NEXT:    v_fmac_f16_e32 v6.l, v0.l, v7.l
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v7, v3.l
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v1.l
+; GFX1200-TRUE16-NEXT:    v_rcp_f32_e32 v7, v7
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v7
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v10, -v8, v0, v9 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fmac_f32_e32 v0, v10, v7
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v8, -v8, v0, v9 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v7, v8, v7
+; GFX1200-TRUE16-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_add_f32_e32 v0, v7, v0
+; GFX1200-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v3.l, v1.l
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1200-TRUE16-NEXT:    v_fma_f16 v0.l, v0.l, v3.l, v1.l
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v3, v4.h
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-TRUE16-NEXT:    v_pack_b32_f16 v1, v0.l, v6.l
+; GFX1200-TRUE16-NEXT:    v_rcp_f32_e32 v3, v3
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.h
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v3
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v6, -v4, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_fmac_f32_e32 v0, v6, v3
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v6, -v4, v0, v2 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v3, v6, v3
+; GFX1200-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_and_b32_e32 v3, 0xff800000, v3
+; GFX1200-TRUE16-NEXT:    v_add_f32_e32 v0, v3, v0
+; GFX1200-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1200-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v6.l, v3.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-TRUE16-NEXT:    v_fmac_f16_e32 v3.l, v0.l, v6.l
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v6, v4.l
+; GFX1200-TRUE16-NEXT:    v_cvt_f32_f16_e32 v0, v2.l
+; GFX1200-TRUE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v6
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v7, -v4, v0, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_fmac_f32_e32 v0, v7, v6
+; GFX1200-TRUE16-NEXT:    v_fma_mix_f32 v7, -v4, v0, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1200-TRUE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_add_f32_e32 v0, v6, v0
+; GFX1200-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v4.l, v2.l
+; GFX1200-TRUE16-NEXT:    v_trunc_f16_e32 v0.l, v0.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1200-TRUE16-NEXT:    v_fma_f16 v0.l, v0.l, v4.l, v2.l
+; GFX1200-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT:    v_pack_b32_f16 v2, v0.l, v3.l
+; GFX1200-TRUE16-NEXT:    global_store_b64 v5, v[1:2], s[0:1]
+; GFX1200-TRUE16-NEXT:    s_endpgm
+;
+; GFX1200-FAKE16-LABEL: frem_v4f16:
+; GFX1200-FAKE16:       ; %bb.0:
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1200-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-FAKE16-NEXT:    s_clause 0x1
+; GFX1200-FAKE16-NEXT:    global_load_b64 v[0:1], v4, s[2:3]
+; GFX1200-FAKE16-NEXT:    global_load_b64 v[2:3], v4, s[4:5] offset:32
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x1
+; GFX1200-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX1200-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v5
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v8, v7
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_rcp_f32_e32 v8, v8
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v6, v6, v8
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v9, -v2, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_fmac_f32_e32 v6, v9, v8
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v9, -v2, v6, v0 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v8, v9, v8
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_and_b32_e32 v8, 0xff800000, v8
+; GFX1200-FAKE16-NEXT:    v_add_f32_e32 v6, v8, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX1200-FAKE16-NEXT:    v_div_fixup_f16 v6, v6, v7, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v6, v6
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v6, 0x8000, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v5, v6, v7
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v7, v2
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v0
+; GFX1200-FAKE16-NEXT:    v_rcp_f32_e32 v7, v7
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v6, v6, v7
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v8, -v2, v6, v0 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fmac_f32_e32 v6, v8, v7
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v8, -v2, v6, v0 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v7, v8, v7
+; GFX1200-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_add_f32_e32 v6, v7, v6
+; GFX1200-FAKE16-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_div_fixup_f16 v6, v6, v2, v0
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v6, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v6, 0x8000, v6
+; GFX1200-FAKE16-NEXT:    v_fma_f16 v0, v6, v2, v0
+; GFX1200-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
+; GFX1200-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, v5
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v7, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v5, v2
+; GFX1200-FAKE16-NEXT:    v_rcp_f32_e32 v7, v7
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v5, v5, v7
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v8, -v3, v5, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fmac_f32_e32 v5, v8, v7
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v8, -v3, v5, v1 op_sel:[1,0,1] op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v7, v8, v7
+; GFX1200-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff800000, v7
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_add_f32_e32 v5, v7, v5
+; GFX1200-FAKE16-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_div_fixup_f16 v5, v5, v6, v2
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v5, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v5, 0x8000, v5
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v2, v5, v6
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v6, v3
+; GFX1200-FAKE16-NEXT:    v_cvt_f32_f16_e32 v5, v1
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_rcp_f32_e32 v6, v6
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v5, v5, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v7, -v3, v5, v1 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_fmac_f32_e32 v5, v7, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fma_mix_f32 v7, -v3, v5, v1 op_sel_hi:[1,0,1]
+; GFX1200-FAKE16-NEXT:    v_mul_f32_e32 v6, v7, v6
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff800000, v6
+; GFX1200-FAKE16-NEXT:    v_add_f32_e32 v5, v6, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_cvt_f16_f32_e32 v5, v5
+; GFX1200-FAKE16-NEXT:    v_div_fixup_f16 v5, v5, v3, v1
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_trunc_f16_e32 v5, v5
+; GFX1200-FAKE16-NEXT:    v_xor_b32_e32 v5, 0x8000, v5
+; GFX1200-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-FAKE16-NEXT:    v_fmac_f16_e32 v1, v5, v3
+; GFX1200-FAKE16-NEXT:    v_pack_b32_f16 v1, v1, v2
+; GFX1200-FAKE16-NEXT:    global_store_b64 v4, v[0:1], s[0:1]
+; GFX1200-FAKE16-NEXT:    s_endpgm
                         ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr <4 x half>, ptr addrspace(1) %in2, i32 4
    %r0 = load <4 x half>, ptr addrspace(1) %in1, align 16
@@ -3503,6 +4435,65 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX1150-NEXT:    v_fmac_f32_e32 v0, v3, v2
 ; GFX1150-NEXT:    global_store_b64 v4, v[0:1], s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: frem_v2f32:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b64 v[0:1], v4, s[2:3]
+; GFX1200-NEXT:    global_load_b64 v[2:3], v4, s[4:5] offset:32
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_div_scale_f32 v6, null, v3, v3, v1
+; GFX1200-NEXT:    v_div_scale_f32 v5, vcc_lo, v1, v3, v1
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f32_e32 v7, v6
+; GFX1200-NEXT:    s_denorm_mode 15
+; GFX1200-NEXT:    v_fma_f32 v8, -v6, v7, 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fmac_f32_e32 v7, v8, v7
+; GFX1200-NEXT:    v_mul_f32_e32 v8, v5, v7
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v9, -v6, v8, v5
+; GFX1200-NEXT:    v_fmac_f32_e32 v8, v9, v7
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v5, -v6, v8, v5
+; GFX1200-NEXT:    s_denorm_mode 12
+; GFX1200-NEXT:    v_div_fmas_f32 v5, v5, v7, v8
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f32 v5, v5, v3, v1
+; GFX1200-NEXT:    v_trunc_f32_e32 v5, v5
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v5, 0x80000000, v5
+; GFX1200-NEXT:    v_fma_f32 v1, v5, v3, v1
+; GFX1200-NEXT:    v_div_scale_f32 v5, null, v2, v2, v0
+; GFX1200-NEXT:    v_div_scale_f32 v3, vcc_lo, v0, v2, v0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f32_e32 v6, v5
+; GFX1200-NEXT:    s_denorm_mode 15
+; GFX1200-NEXT:    v_fma_f32 v7, -v5, v6, 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fmac_f32_e32 v6, v7, v6
+; GFX1200-NEXT:    v_mul_f32_e32 v7, v3, v6
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v8, -v5, v7, v3
+; GFX1200-NEXT:    v_fmac_f32_e32 v7, v8, v6
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v3, -v5, v7, v3
+; GFX1200-NEXT:    s_denorm_mode 12
+; GFX1200-NEXT:    s_wait_alu 0xfffd
+; GFX1200-NEXT:    v_div_fmas_f32 v3, v3, v6, v7
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f32 v3, v3, v2, v0
+; GFX1200-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-NEXT:    v_fmac_f32_e32 v0, v3, v2
+; GFX1200-NEXT:    global_store_b64 v4, v[0:1], s[0:1]
+; GFX1200-NEXT:    s_endpgm
                         ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr <2 x float>, ptr addrspace(1) %in2, i32 4
    %r0 = load <2 x float>, ptr addrspace(1) %in1, align 8
@@ -4099,6 +5090,111 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX1150-NEXT:    v_fmac_f32_e32 v0, v5, v4
 ; GFX1150-NEXT:    global_store_b128 v8, v[0:3], s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: frem_v4f32:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v8, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b128 v[0:3], v8, s[2:3]
+; GFX1200-NEXT:    global_load_b128 v[4:7], v8, s[4:5] offset:64
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_div_scale_f32 v10, null, v7, v7, v3
+; GFX1200-NEXT:    v_div_scale_f32 v9, vcc_lo, v3, v7, v3
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f32_e32 v11, v10
+; GFX1200-NEXT:    s_denorm_mode 15
+; GFX1200-NEXT:    v_fma_f32 v12, -v10, v11, 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fmac_f32_e32 v11, v12, v11
+; GFX1200-NEXT:    v_mul_f32_e32 v12, v9, v11
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v13, -v10, v12, v9
+; GFX1200-NEXT:    v_fmac_f32_e32 v12, v13, v11
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v9, -v10, v12, v9
+; GFX1200-NEXT:    s_denorm_mode 12
+; GFX1200-NEXT:    v_div_fmas_f32 v9, v9, v11, v12
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f32 v9, v9, v7, v3
+; GFX1200-NEXT:    v_trunc_f32_e32 v9, v9
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v9, 0x80000000, v9
+; GFX1200-NEXT:    v_fma_f32 v3, v9, v7, v3
+; GFX1200-NEXT:    v_div_scale_f32 v9, null, v6, v6, v2
+; GFX1200-NEXT:    v_div_scale_f32 v7, vcc_lo, v2, v6, v2
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f32_e32 v10, v9
+; GFX1200-NEXT:    s_denorm_mode 15
+; GFX1200-NEXT:    v_fma_f32 v11, -v9, v10, 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fmac_f32_e32 v10, v11, v10
+; GFX1200-NEXT:    v_mul_f32_e32 v11, v7, v10
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v12, -v9, v11, v7
+; GFX1200-NEXT:    v_fmac_f32_e32 v11, v12, v10
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v7, -v9, v11, v7
+; GFX1200-NEXT:    s_denorm_mode 12
+; GFX1200-NEXT:    s_wait_alu 0xfffd
+; GFX1200-NEXT:    v_div_fmas_f32 v7, v7, v10, v11
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f32 v7, v7, v6, v2
+; GFX1200-NEXT:    v_trunc_f32_e32 v7, v7
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v7, 0x80000000, v7
+; GFX1200-NEXT:    v_fma_f32 v2, v7, v6, v2
+; GFX1200-NEXT:    v_div_scale_f32 v7, null, v5, v5, v1
+; GFX1200-NEXT:    v_div_scale_f32 v6, vcc_lo, v1, v5, v1
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f32_e32 v9, v7
+; GFX1200-NEXT:    s_denorm_mode 15
+; GFX1200-NEXT:    v_fma_f32 v10, -v7, v9, 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fmac_f32_e32 v9, v10, v9
+; GFX1200-NEXT:    v_mul_f32_e32 v10, v6, v9
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v11, -v7, v10, v6
+; GFX1200-NEXT:    v_fmac_f32_e32 v10, v11, v9
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v6, -v7, v10, v6
+; GFX1200-NEXT:    s_denorm_mode 12
+; GFX1200-NEXT:    s_wait_alu 0xfffd
+; GFX1200-NEXT:    v_div_fmas_f32 v6, v6, v9, v10
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f32 v6, v6, v5, v1
+; GFX1200-NEXT:    v_trunc_f32_e32 v6, v6
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v6, 0x80000000, v6
+; GFX1200-NEXT:    v_fma_f32 v1, v6, v5, v1
+; GFX1200-NEXT:    v_div_scale_f32 v6, null, v4, v4, v0
+; GFX1200-NEXT:    v_div_scale_f32 v5, vcc_lo, v0, v4, v0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f32_e32 v7, v6
+; GFX1200-NEXT:    s_denorm_mode 15
+; GFX1200-NEXT:    v_fma_f32 v9, -v6, v7, 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fmac_f32_e32 v7, v9, v7
+; GFX1200-NEXT:    v_mul_f32_e32 v9, v5, v7
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v10, -v6, v9, v5
+; GFX1200-NEXT:    v_fmac_f32_e32 v9, v10, v7
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f32 v5, -v6, v9, v5
+; GFX1200-NEXT:    s_denorm_mode 12
+; GFX1200-NEXT:    s_wait_alu 0xfffd
+; GFX1200-NEXT:    v_div_fmas_f32 v5, v5, v7, v9
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f32 v5, v5, v4, v0
+; GFX1200-NEXT:    v_trunc_f32_e32 v5, v5
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_xor_b32_e32 v5, 0x80000000, v5
+; GFX1200-NEXT:    v_fmac_f32_e32 v0, v5, v4
+; GFX1200-NEXT:    global_store_b128 v8, v[0:3], s[0:1]
+; GFX1200-NEXT:    s_endpgm
                         ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr <4 x float>, ptr addrspace(1) %in2, i32 4
    %r0 = load <4 x float>, ptr addrspace(1) %in1, align 16
@@ -4479,6 +5575,58 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GFX1150-NEXT:    v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
 ; GFX1150-NEXT:    global_store_b128 v16, v[0:3], s[0:1]
 ; GFX1150-NEXT:    s_endpgm
+;
+; GFX1200-LABEL: frem_v2f64:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-NEXT:    v_mov_b32_e32 v16, 0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_clause 0x1
+; GFX1200-NEXT:    global_load_b128 v[0:3], v16, s[2:3]
+; GFX1200-NEXT:    global_load_b128 v[4:7], v16, s[4:5] offset:64
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    v_div_scale_f64 v[8:9], null, v[6:7], v[6:7], v[2:3]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-NEXT:    v_rcp_f64_e32 v[10:11], v[8:9]
+; GFX1200-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX1200-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX1200-NEXT:    v_div_scale_f64 v[12:13], vcc_lo, v[2:3], v[6:7], v[2:3]
+; GFX1200-NEXT:    v_mul_f64_e32 v[14:15], v[12:13], v[10:11]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[14:15], v[12:13]
+; GFX1200-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
+; GFX1200-NEXT:    v_trunc_f64_e32 v[8:9], v[8:9]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
+; GFX1200-NEXT:    v_div_scale_f64 v[6:7], null, v[4:5], v[4:5], v[0:1]
+; GFX1200-NEXT:    v_rcp_f64_e32 v[8:9], v[6:7]
+; GFX1200-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX1200-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
+; GFX1200-NEXT:    v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9]
+; GFX1200-NEXT:    v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_mul_f64_e32 v[12:13], v[10:11], v[8:9]
+; GFX1200-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11]
+; GFX1200-NEXT:    s_wait_alu 0xfffd
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13]
+; GFX1200-NEXT:    v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-NEXT:    v_trunc_f64_e32 v[6:7], v[6:7]
+; GFX1200-NEXT:    v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
+; GFX1200-NEXT:    global_store_b128 v16, v[0:3], s[0:1]
+; GFX1200-NEXT:    s_endpgm
                         ptr addrspace(1) %in2) #0 {
    %gep2 = getelementptr <2 x double>, ptr addrspace(1) %in2, i32 4
    %r0 = load <2 x double>, ptr addrspace(1) %in1, align 16
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-args.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-args.ll
index 81b8b3618074..a901d7f97eb3 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -3380,42 +3380,117 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i
 }
 
 define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2 x double> %arg2) #0 {
-; CIGFX89-LABEL: void_func_v32i32_v2i64_v2f64:
-; CIGFX89:       ; %bb.0:
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CIGFX89-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; CIGFX89-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:20
-; CIGFX89-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
-; CIGFX89-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:12
-; CIGFX89-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
-; CIGFX89-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
-; CIGFX89-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
-; CIGFX89-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:28
-; CIGFX89-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:24
-; CIGFX89-NEXT:    s_mov_b32 s7, 0xf000
-; CIGFX89-NEXT:    s_mov_b32 s6, -1
-; CIGFX89-NEXT:    s_waitcnt vmcnt(8)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[28:31], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[24:27], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[20:23], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[32:35], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    buffer_store_dwordx4 v[36:39], off, s[4:7], 0
-; CIGFX89-NEXT:    s_waitcnt vmcnt(0)
-; CIGFX89-NEXT:    s_setpc_b64 s[30:31]
+; CI-LABEL: void_func_v32i32_v2i64_v2f64:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; CI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:32
+; CI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:28
+; CI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; CI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:16
+; CI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:12
+; CI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:8
+; CI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; CI-NEXT:    s_mov_b32 s7, 0xf000
+; CI-NEXT:    s_mov_b32 s6, -1
+; CI-NEXT:    s_waitcnt vmcnt(7)
+; CI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; CI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[35:38], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[31:34], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: void_func_v32i32_v2i64_v2f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; VI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_waitcnt vmcnt(7)
+; VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[35:38], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[31:34], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: void_func_v32i32_v2i64_v2f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    s_waitcnt vmcnt(7)
+; GFX9-NEXT:    buffer_store_dwordx4 v[28:31], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[24:27], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[20:23], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[35:38], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[31:34], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: void_func_v32i32_v2i64_v2f64:
 ; GFX11:       ; %bb.0:
@@ -3552,13 +3627,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 ; CI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; CI-NEXT:    s_mov_b32 s7, 0xf000
 ; CI-NEXT:    s_mov_b32 s6, -1
-; CI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:64
-; CI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:60
-; CI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:56
-; CI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:52
-; CI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:16
-; CI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:12
-; CI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; CI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; CI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:12
+; CI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
+; CI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; CI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
+; CI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:28
+; CI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:24
 ; CI-NEXT:    s_waitcnt vmcnt(7)
 ; CI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -3570,29 +3645,29 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:4
-; CI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:32
-; CI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:28
-; CI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:24
-; CI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:20
-; CI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:48
-; CI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:44
-; CI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:40
-; CI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; CI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:20
+; CI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:48
+; CI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:44
+; CI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:40
+; CI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64
+; CI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:60
+; CI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:56
+; CI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:52
+; CI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:36
 ; CI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
-; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: void_func_v32i32_v8i32_v8f32:
@@ -3601,13 +3676,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
-; VI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:56
-; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:52
-; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:24
 ; VI-NEXT:    s_waitcnt vmcnt(7)
 ; VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -3619,29 +3694,29 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:4
-; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:44
-; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; VI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:44
+; VI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64
+; VI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:56
+; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:52
+; VI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:36
 ; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
-; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: void_func_v32i32_v8i32_v8f32:
@@ -3650,13 +3725,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:56
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:52
-; GFX9-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:24
 ; GFX9-NEXT:    s_waitcnt vmcnt(7)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[28:31], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -3668,15 +3743,15 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:48
-; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:40
-; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:36
+; GFX9-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:44
+; GFX9-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:40
+; GFX9-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:64
+; GFX9-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:52
+; GFX9-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:36
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -3684,14 +3759,14 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[36:39], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[32:35], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: void_func_v32i32_v8i32_v8f32:
@@ -3791,40 +3866,40 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
 ; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:4
-; CI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:96
-; CI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:92
-; CI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:88
-; CI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
-; CI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:112
-; CI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
-; CI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:104
+; CI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:112
+; CI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:108
+; CI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:104
+; CI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:100
+; CI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:128
+; CI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:124
+; CI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:120
 ; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:100
-; CI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:128
-; CI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:124
-; CI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:120
-; CI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; CI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80
-; CI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:76
-; CI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:72
-; CI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; CI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:116
+; CI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:80
+; CI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
+; CI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:72
+; CI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:96
+; CI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:92
+; CI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:88
+; CI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:84
+; CI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:68
 ; CI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: void_func_v32i32_v16i32_v16f32:
@@ -3864,40 +3939,40 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
 ; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:4
-; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:96
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:92
-; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:88
-; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
-; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:112
-; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
-; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:104
+; VI-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:112
+; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:108
+; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:104
+; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:100
+; VI-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:128
+; VI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:124
+; VI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:120
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:128
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:124
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:120
-; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80
-; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:76
-; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:72
-; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; VI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:116
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:80
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:72
+; VI-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:96
+; VI-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:92
+; VI-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:88
+; VI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:84
+; VI-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:68
 ; VI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: void_func_v32i32_v16i32_v16f32:
@@ -3938,27 +4013,27 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
 ; GFX9-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:96
-; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:92
-; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:88
-; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:84
-; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:112
-; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:108
-; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:112
+; GFX9-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:108
+; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:104
+; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:100
+; GFX9-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:128
+; GFX9-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:124
+; GFX9-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:120
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[32:35], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:100
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:128
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:124
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:120
-; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:80
-; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:76
-; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:72
-; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:68
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:116
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:80
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:76
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:72
+; GFX9-NEXT:    buffer_load_dword v23, off, s[0:3], s32 offset:96
+; GFX9-NEXT:    buffer_load_dword v22, off, s[0:3], s32 offset:92
+; GFX9-NEXT:    buffer_load_dword v21, off, s[0:3], s32 offset:88
+; GFX9-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:84
+; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:68
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dwordx4 v[36:39], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -3966,14 +4041,14 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[20:23], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: void_func_v32i32_v16i32_v16f32:
@@ -4259,9 +4334,9 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; CI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; CI-NEXT:    s_mov_b32 s7, 0xf000
 ; CI-NEXT:    s_mov_b32 s6, -1
-; CI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:60
-; CI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:64
-; CI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:48
+; CI-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:48
+; CI-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:60
+; CI-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:64
 ; CI-NEXT:    buffer_load_dword v35, off, s[0:3], s32 offset:52
 ; CI-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:56
 ; CI-NEXT:    buffer_load_dword v37, off, s[0:3], s32 offset:36
@@ -4275,16 +4350,16 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:28
-; CI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:32
-; CI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:20
-; CI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:24
 ; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:16
-; CI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:12
-; CI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:8
-; CI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:4
+; CI-NEXT:    buffer_load_dword v12, off, s[0:3], s32 offset:32
+; CI-NEXT:    buffer_load_dword v13, off, s[0:3], s32 offset:28
+; CI-NEXT:    buffer_load_dword v14, off, s[0:3], s32 offset:24
+; CI-NEXT:    buffer_load_dword v15, off, s[0:3], s32 offset:20
+; CI-NEXT:    buffer_load_dword v16, off, s[0:3], s32 offset:16
+; CI-NEXT:    buffer_load_dword v17, off, s[0:3], s32 offset:12
+; CI-NEXT:    buffer_load_dword v18, off, s[0:3], s32 offset:8
+; CI-NEXT:    buffer_load_dword v19, off, s[0:3], s32 offset:4
 ; CI-NEXT:    buffer_load_dword v20, off, s[0:3], s32 offset:44
 ; CI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -4292,15 +4367,15 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_byte v33, off, s[4:7], 0
+; CI-NEXT:    buffer_store_byte v34, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_byte v32, off, s[4:7], 0
+; CI-NEXT:    buffer_store_byte v33, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_byte v36, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_byte v35, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_byte v34, off, s[4:7], 0
+; CI-NEXT:    buffer_store_byte v32, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_byte v20, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
@@ -4308,14 +4383,6 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_byte v37, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_byte v17, off, s[4:7], 0
-; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_byte v16, off, s[4:7], 0
-; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_byte v19, off, s[4:7], 0
-; CI-NEXT:    s_waitcnt vmcnt(0)
-; CI-NEXT:    buffer_store_byte v18, off, s[4:7], 0
-; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_byte v12, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_byte v13, off, s[4:7], 0
@@ -4324,6 +4391,14 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_store_byte v15, off, s[4:7], 0
 ; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_byte v16, off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_byte v17, off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_byte v18, off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
+; CI-NEXT:    buffer_store_byte v19, off, s[4:7], 0
+; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: void_func_v32i32_v16i8:
@@ -4332,9 +4407,9 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
-; VI-NEXT:    buffer_load_ubyte v32, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_ubyte v33, off, s[0:3], s32 offset:64
-; VI-NEXT:    buffer_load_ubyte v34, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ubyte v32, off, s[0:3], s32 offset:48
+; VI-NEXT:    buffer_load_ubyte v33, off, s[0:3], s32 offset:60
+; VI-NEXT:    buffer_load_ubyte v34, off, s[0:3], s32 offset:64
 ; VI-NEXT:    buffer_load_ubyte v35, off, s[0:3], s32 offset:52
 ; VI-NEXT:    buffer_load_ubyte v36, off, s[0:3], s32 offset:56
 ; VI-NEXT:    buffer_load_ubyte v37, off, s[0:3], s32 offset:36
@@ -4348,16 +4423,16 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_load_ubyte v16, off, s[0:3], s32 offset:28
-; VI-NEXT:    buffer_load_ubyte v17, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_ubyte v18, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_ubyte v19, off, s[0:3], s32 offset:24
 ; VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_load_ubyte v12, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_ubyte v13, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_ubyte v14, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_ubyte v15, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_ubyte v12, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_ubyte v13, off, s[0:3], s32 offset:28
+; VI-NEXT:    buffer_load_ubyte v14, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_ubyte v15, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_ubyte v16, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_ubyte v17, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_ubyte v18, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_ubyte v19, off, s[0:3], s32 offset:4
 ; VI-NEXT:    buffer_load_ubyte v20, off, s[0:3], s32 offset:44
 ; VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -4365,15 +4440,15 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_byte v33, off, s[4:7], 0
+; VI-NEXT:    buffer_store_byte v34, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_byte v32, off, s[4:7], 0
+; VI-NEXT:    buffer_store_byte v33, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_byte v36, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_byte v35, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_byte v34, off, s[4:7], 0
+; VI-NEXT:    buffer_store_byte v32, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_byte v20, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
@@ -4381,14 +4456,6 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_byte v37, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_byte v17, off, s[4:7], 0
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_byte v16, off, s[4:7], 0
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_byte v19, off, s[4:7], 0
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_store_byte v18, off, s[4:7], 0
-; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_byte v12, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_byte v13, off, s[4:7], 0
@@ -4397,6 +4464,14 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_byte v15, off, s[4:7], 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v16, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v17, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v18, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v19, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: void_func_v32i32_v16i8:
@@ -4405,9 +4480,9 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    buffer_load_ubyte v32, off, s[0:3], s32 offset:60
-; GFX9-NEXT:    buffer_load_ubyte v33, off, s[0:3], s32 offset:64
-; GFX9-NEXT:    buffer_load_ubyte v34, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ubyte v32, off, s[0:3], s32 offset:48
+; GFX9-NEXT:    buffer_load_ubyte v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT:    buffer_load_ubyte v34, off, s[0:3], s32 offset:64
 ; GFX9-NEXT:    buffer_load_ubyte v35, off, s[0:3], s32 offset:52
 ; GFX9-NEXT:    buffer_load_ubyte v36, off, s[0:3], s32 offset:56
 ; GFX9-NEXT:    buffer_load_ubyte v37, off, s[0:3], s32 offset:36
@@ -4421,18 +4496,17 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[16:19], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_load_ubyte v16, off, s[0:3], s32 offset:28
-; GFX9-NEXT:    buffer_load_ubyte v17, off, s[0:3], s32 offset:32
-; GFX9-NEXT:    buffer_load_ubyte v18, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_ubyte v19, off, s[0:3], s32 offset:24
-; GFX9-NEXT:    buffer_load_ubyte v20, off, s[0:3], s32 offset:44
-; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_load_ubyte v12, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_ubyte v13, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_ubyte v14, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_ubyte v15, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ubyte v12, off, s[0:3], s32 offset:32
+; GFX9-NEXT:    buffer_load_ubyte v13, off, s[0:3], s32 offset:28
+; GFX9-NEXT:    buffer_load_ubyte v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    buffer_load_ubyte v15, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_ubyte v16, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_ubyte v17, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_ubyte v18, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_ubyte v19, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_ubyte v20, off, s[0:3], s32 offset:44
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -4440,15 +4514,15 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_byte v33, off, s[4:7], 0
+; GFX9-NEXT:    buffer_store_byte v34, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_byte v32, off, s[4:7], 0
+; GFX9-NEXT:    buffer_store_byte v33, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_byte v36, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_byte v35, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_byte v34, off, s[4:7], 0
+; GFX9-NEXT:    buffer_store_byte v32, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_byte v20, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -4456,14 +4530,6 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_byte v37, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_byte v17, off, s[4:7], 0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_byte v16, off, s[4:7], 0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_byte v19, off, s[4:7], 0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    buffer_store_byte v18, off, s[4:7], 0
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_byte v12, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_byte v13, off, s[4:7], 0
@@ -4472,6 +4538,14 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_byte v15, off, s[4:7], 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_byte v16, off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_byte v17, off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_byte v18, off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_store_byte v19, off, s[4:7], 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: void_func_v32i32_v16i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
index e152f2ddd525..59bcc5d8be9b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
@@ -495,17 +495,17 @@ define amdgpu_kernel void @usage_direct_recursion(i32 %n) #0 {
 ; GCN: NumVgprs: max(43, multi_stage_recurse1.num_vgpr)
 ; GCN: ScratchSize: 16+max(multi_stage_recurse1.private_seg_size)
 ; GCN-LABEL: {{^}}multi_stage_recurse1:
-; GCN: .set multi_stage_recurse1.num_vgpr, max(48, amdgpu.max_num_vgpr)
-; GCN: .set multi_stage_recurse1.num_agpr, max(0, amdgpu.max_num_agpr)
-; GCN: .set multi_stage_recurse1.numbered_sgpr, max(34, amdgpu.max_num_sgpr)
+; GCN: .set multi_stage_recurse1.num_vgpr, max(48, 43)
+; GCN: .set multi_stage_recurse1.num_agpr, max(0, 0)
+; GCN: .set multi_stage_recurse1.numbered_sgpr, max(34, 34)
 ; GCN: .set multi_stage_recurse1.private_seg_size, 16
 ; GCN: .set multi_stage_recurse1.uses_vcc, 1
 ; GCN: .set multi_stage_recurse1.uses_flat_scratch, 0
 ; GCN: .set multi_stage_recurse1.has_dyn_sized_stack, 0
 ; GCN: .set multi_stage_recurse1.has_recursion, 1
 ; GCN: .set multi_stage_recurse1.has_indirect_call, 0
-; GCN: TotalNumSgprs: multi_stage_recurse1.numbered_sgpr+4
-; GCN: NumVgprs: max(48, amdgpu.max_num_vgpr)
+; GCN: TotalNumSgprs: 38
+; GCN: NumVgprs: 48
 ; GCN: ScratchSize: 16
 define void @multi_stage_recurse1(i32 %val) #2 {
   call void @multi_stage_recurse2(i32 %val)
@@ -528,8 +528,8 @@ define void @multi_stage_recurse2(i32 %val) #2 {
 ; GCN: .set usage_multi_stage_recurse.has_dyn_sized_stack, or(0, multi_stage_recurse1.has_dyn_sized_stack)
 ; GCN: .set usage_multi_stage_recurse.has_recursion, or(1, multi_stage_recurse1.has_recursion)
 ; GCN: .set usage_multi_stage_recurse.has_indirect_call, or(0, multi_stage_recurse1.has_indirect_call)
-; GCN: TotalNumSgprs: usage_multi_stage_recurse.numbered_sgpr+6
-; GCN: NumVgprs: usage_multi_stage_recurse.num_vgpr
+; GCN: TotalNumSgprs: 40
+; GCN: NumVgprs: 48
 ; GCN: ScratchSize: 16
 define amdgpu_kernel void @usage_multi_stage_recurse(i32 %n) #0 {
   call void @multi_stage_recurse1(i32 %n)
@@ -550,17 +550,17 @@ define amdgpu_kernel void @usage_multi_stage_recurse(i32 %n) #0 {
 ; GCN: NumVgprs: max(41, multi_stage_recurse_noattr1.num_vgpr)
 ; GCN: ScratchSize: 16+max(multi_stage_recurse_noattr1.private_seg_size)
 ; GCN-LABEL: {{^}}multi_stage_recurse_noattr1:
-; GCN: .set multi_stage_recurse_noattr1.num_vgpr, max(41, amdgpu.max_num_vgpr)
-; GCN: .set multi_stage_recurse_noattr1.num_agpr, max(0, amdgpu.max_num_agpr)
-; GCN: .set multi_stage_recurse_noattr1.numbered_sgpr, max(57, amdgpu.max_num_sgpr)
+; GCN: .set multi_stage_recurse_noattr1.num_vgpr, max(41, 41)
+; GCN: .set multi_stage_recurse_noattr1.num_agpr, max(0, 0)
+; GCN: .set multi_stage_recurse_noattr1.numbered_sgpr, max(57, 54)
 ; GCN: .set multi_stage_recurse_noattr1.private_seg_size, 16
 ; GCN: .set multi_stage_recurse_noattr1.uses_vcc, 1
 ; GCN: .set multi_stage_recurse_noattr1.uses_flat_scratch, 0
 ; GCN: .set multi_stage_recurse_noattr1.has_dyn_sized_stack, 0
 ; GCN: .set multi_stage_recurse_noattr1.has_recursion, 0
 ; GCN: .set multi_stage_recurse_noattr1.has_indirect_call, 0
-; GCN: TotalNumSgprs: multi_stage_recurse_noattr1.numbered_sgpr+4
-; GCN: NumVgprs: max(41, amdgpu.max_num_vgpr)
+; GCN: TotalNumSgprs: 61
+; GCN: NumVgprs: 41
 ; GCN: ScratchSize: 16
 define void @multi_stage_recurse_noattr1(i32 %val) #0 {
   call void @multi_stage_recurse_noattr2(i32 %val)
@@ -583,8 +583,8 @@ define void @multi_stage_recurse_noattr2(i32 %val) #0 {
 ; GCN: .set usage_multi_stage_recurse_noattrs.has_dyn_sized_stack, or(0, multi_stage_recurse_noattr1.has_dyn_sized_stack)
 ; GCN: .set usage_multi_stage_recurse_noattrs.has_recursion, or(0, multi_stage_recurse_noattr1.has_recursion)
 ; GCN: .set usage_multi_stage_recurse_noattrs.has_indirect_call, or(0, multi_stage_recurse_noattr1.has_indirect_call)
-; GCN: TotalNumSgprs: usage_multi_stage_recurse_noattrs.numbered_sgpr+6
-; GCN: NumVgprs: usage_multi_stage_recurse_noattrs.num_vgpr
+; GCN: TotalNumSgprs: 63
+; GCN: NumVgprs: 41
 ; GCN: ScratchSize: 16
 define amdgpu_kernel void @usage_multi_stage_recurse_noattrs(i32 %n) #0 {
   call void @multi_stage_recurse_noattr1(i32 %n)
@@ -601,8 +601,8 @@ define amdgpu_kernel void @usage_multi_stage_recurse_noattrs(i32 %n) #0 {
 ; GCN:  .set multi_call_with_multi_stage_recurse.has_dyn_sized_stack, or(0, use_stack0.has_dyn_sized_stack, use_stack1.has_dyn_sized_stack, multi_stage_recurse1.has_dyn_sized_stack)
 ; GCN:  .set multi_call_with_multi_stage_recurse.has_recursion, or(1, use_stack0.has_recursion, use_stack1.has_recursion, multi_stage_recurse1.has_recursion)
 ; GCN:  .set multi_call_with_multi_stage_recurse.has_indirect_call, or(0, use_stack0.has_indirect_call, use_stack1.has_indirect_call, multi_stage_recurse1.has_indirect_call)
-; GCN: TotalNumSgprs: multi_call_with_multi_stage_recurse.numbered_sgpr+6
-; GCN: NumVgprs:  multi_call_with_multi_stage_recurse.num_vgpr
+; GCN: TotalNumSgprs: 59
+; GCN: NumVgprs:  48
 ; GCN: ScratchSize: 2052
 define amdgpu_kernel void @multi_call_with_multi_stage_recurse(i32 %n) #0 {
   call void @use_stack0()
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll
index f160cc96654f..b24ebbd9435c 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll
@@ -18,35 +18,54 @@ define protected amdgpu_kernel void @IllegalGEPConst(i32 %a, ptr addrspace(1) %b
 ; CHECK-NEXT:    s_add_u32 s0, s0, s6
 ; CHECK-NEXT:    s_addc_u32 s1, s1, s7
 ; CHECK-NEXT:    s_add_u32 s0, s0, -8
-; CHECK-NEXT:    s_mov_b64 s[4:5], src_private_base
+; CHECK-NEXT:    s_mov_b64 s[4:5], src_shared_base
 ; CHECK-NEXT:    s_addc_u32 s1, s1, -1
 ; CHECK-NEXT:    s_cmp_eq_u32 s1, s5
 ; CHECK-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
 ; CHECK-NEXT:    s_mov_b64 s[4:5], -1
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB0_3
-; CHECK-NEXT:  ; %bb.1: ; %Flow
+; CHECK-NEXT:  ; %bb.1: ; %Flow6
 ; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_4
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_8
 ; CHECK-NEXT:  .LBB0_2: ; %atomicrmw.phi
 ; CHECK-NEXT:    s_endpgm
-; CHECK-NEXT:  .LBB0_3: ; %atomicrmw.global
-; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; CHECK-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; CHECK-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:  .LBB0_3: ; %atomicrmw.check.private
+; CHECK-NEXT:    s_mov_b64 s[4:5], src_private_base
+; CHECK-NEXT:    s_cmp_eq_u32 s1, s5
+; CHECK-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; CHECK-NEXT:    s_mov_b64 s[4:5], -1
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_5
+; CHECK-NEXT:  ; %bb.4: ; %atomicrmw.global
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    buffer_wbinvl1_vol
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_2
-; CHECK-NEXT:  .LBB0_4: ; %atomicrmw.private
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:  .LBB0_5: ; %Flow
+; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_7
+; CHECK-NEXT:  ; %bb.6: ; %atomicrmw.private
 ; CHECK-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; CHECK-NEXT:    s_cselect_b32 s0, s0, -1
-; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    s_cselect_b32 s4, s0, -1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s4
 ; CHECK-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v1, v2, s[12:15], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_add_f64 v[0:1], v[0:1], s[2:3]
 ; CHECK-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
 ; CHECK-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen offset:4
+; CHECK-NEXT:  .LBB0_7: ; %Flow5
+; CHECK-NEXT:    s_cbranch_execnz .LBB0_2
+; CHECK-NEXT:  .LBB0_8: ; %atomicrmw.shared
+; CHECK-NEXT:    s_cmp_lg_u64 s[0:1], 0
+; CHECK-NEXT:    s_cselect_b32 s0, s0, -1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT:    ds_add_f64 v2, v[0:1]
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_endpgm
 entry:
   %i = add nsw i32 %a, -1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
index 2fb441877c63..2be47e8ac5a3 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
@@ -2482,22 +2482,21 @@ define amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> %val) #1 {
 ; GFX10-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:28
 ; GFX10-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; GFX10-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
-; GFX10-NEXT:    s_clause 0x3
-; GFX10-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:12
-; GFX10-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:16
-; GFX10-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:20
-; GFX10-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:24
 ; GFX10-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; GFX10-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
-; GFX10-NEXT:    s_clause 0x3
-; GFX10-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:4
-; GFX10-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:8
-; GFX10-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:160
-; GFX10-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:28
 ; GFX10-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
-; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32
+; GFX10-NEXT:    s_clause 0x8
+; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:28
+; GFX10-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:24
+; GFX10-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:20
+; GFX10-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:16
+; GFX10-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:12
+; GFX10-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; GFX10-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:4
+; GFX10-NEXT:    buffer_load_dword v10, off, s[0:3], s32
+; GFX10-NEXT:    buffer_load_dword v27, off, s[0:3], s32 offset:160
 ; GFX10-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
-; GFX10-NEXT:    s_waitcnt vmcnt(2)
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:284
 ; GFX10-NEXT:    buffer_store_dword v51, v0, s[0:3], 0 offen offset:280
 ; GFX10-NEXT:    buffer_store_dword v50, v0, s[0:3], 0 offen offset:276
@@ -2531,16 +2530,14 @@ define amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> %val) #1 {
 ; GFX10-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:164
 ; GFX10-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:160
 ; GFX10-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:156
-; GFX10-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:152
-; GFX10-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:148
-; GFX10-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:144
-; GFX10-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:140
-; GFX10-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:136
-; GFX10-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:132
-; GFX10-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:128
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:124
+; GFX10-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:152
+; GFX10-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:148
+; GFX10-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:144
+; GFX10-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:140
+; GFX10-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:136
+; GFX10-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:132
+; GFX10-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:128
+; GFX10-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:124
 ; GFX10-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/global_atomics.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/global_atomics.ll
index 3e15b135eeab..0512b9bc2b54 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/global_atomics.ll
@@ -2,6 +2,8 @@
 ; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=None -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=None -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=None -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-atomic-optimizer-strategy=None -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-atomic-optimizer-strategy=None -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s
 
 define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-LABEL: atomic_add_i32_offset:
@@ -41,6 +43,19 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -89,6 +104,19 @@ define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_max_neg_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:-4096
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 -1024
   %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -135,6 +163,19 @@ define amdgpu_kernel void @atomic_add_i32_soffset(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_soffset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:3232
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 9000
   %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -185,6 +226,21 @@ define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_huge_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    s_add_u32 s0, s0, 0xdeac
+; GFX11-NEXT:    s_addc_u32 s1, s1, 0xabcd
+; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 47224239175595
 
@@ -241,6 +297,20 @@ define amdgpu_kernel void @atomic_add_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -300,6 +370,24 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -370,6 +458,25 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -416,6 +523,19 @@ define amdgpu_kernel void @atomic_add_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile add ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -470,6 +590,20 @@ define amdgpu_kernel void @atomic_add_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile add ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -526,6 +660,24 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile add ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -593,6 +745,25 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_add_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile add ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -638,6 +809,19 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -693,6 +877,20 @@ define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -752,6 +950,24 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -822,6 +1038,25 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -868,6 +1103,19 @@ define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -922,6 +1170,20 @@ define amdgpu_kernel void @atomic_and_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -978,6 +1240,24 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -1045,6 +1325,25 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_and_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -1090,6 +1389,19 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -1145,6 +1457,20 @@ define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -1204,6 +1530,24 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -1274,6 +1618,25 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -1320,6 +1683,19 @@ define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -1374,6 +1750,20 @@ define amdgpu_kernel void @atomic_sub_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -1430,6 +1820,24 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -1497,6 +1905,25 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_sub_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -1542,6 +1969,19 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -1594,6 +2034,19 @@ define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
@@ -1647,6 +2100,23 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_smax v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -1714,6 +2184,24 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -1754,6 +2242,18 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_atomic_smax v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -1805,6 +2305,19 @@ define amdgpu_kernel void @atomic_max_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -1855,6 +2368,23 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_smax v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -1919,6 +2449,24 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_max_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -1958,6 +2506,18 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2010,6 +2570,19 @@ define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2063,6 +2636,23 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -2130,6 +2720,24 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -2170,6 +2778,18 @@ define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -2221,6 +2841,19 @@ define amdgpu_kernel void @atomic_umax_i32_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -2271,6 +2904,23 @@ define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2335,6 +2985,24 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umax_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2374,6 +3042,18 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2426,6 +3106,19 @@ define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2479,6 +3172,23 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -2546,6 +3256,24 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -2586,6 +3314,18 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -2637,6 +3377,19 @@ define amdgpu_kernel void @atomic_min_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -2687,6 +3440,23 @@ define amdgpu_kernel void @atomic_min_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2751,6 +3521,24 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_min_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -2790,6 +3578,18 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2842,6 +3642,19 @@ define amdgpu_kernel void @atomic_umin_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
@@ -2895,6 +3708,23 @@ define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -2962,6 +3792,24 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -3002,6 +3850,18 @@ define amdgpu_kernel void @atomic_umin_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   ret void
@@ -3053,6 +3913,19 @@ define amdgpu_kernel void @atomic_umin_i32_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -3103,6 +3976,23 @@ define amdgpu_kernel void @atomic_umin_i32_addr64(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -3167,6 +4057,24 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_umin_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
@@ -3212,6 +4120,19 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -3267,6 +4188,20 @@ define amdgpu_kernel void @atomic_or_i32_ret_offset(ptr addrspace(1) %out, ptr a
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -3326,6 +4261,24 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr addrspace(1) %out, i3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -3396,6 +4349,25 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64_offset(ptr addrspace(1) %out
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -3442,6 +4414,19 @@ define amdgpu_kernel void @atomic_or_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -3496,6 +4481,20 @@ define amdgpu_kernel void @atomic_or_i32_ret(ptr addrspace(1) %out, ptr addrspac
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -3552,6 +4551,24 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -3619,6 +4636,25 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64(ptr addrspace(1) %out, ptr a
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_or_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -3664,6 +4700,19 @@ define amdgpu_kernel void @atomic_xchg_i32_offset(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -3708,6 +4757,19 @@ define amdgpu_kernel void @atomic_xchg_f32_offset(ptr addrspace(1) %out, float %
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_f32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr float, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile xchg ptr addrspace(1) %gep, float %in syncscope("agent") seq_cst
@@ -3763,6 +4825,20 @@ define amdgpu_kernel void @atomic_xchg_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -3822,6 +4898,24 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -3892,6 +4986,25 @@ define amdgpu_kernel void @atomic_xchg_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -3938,6 +5051,19 @@ define amdgpu_kernel void @atomic_xchg_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -3992,6 +5118,20 @@ define amdgpu_kernel void @atomic_xchg_i32_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -4048,6 +5188,24 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile xchg ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -4115,6 +5273,25 @@ define amdgpu_kernel void @atomic_xchg_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xchg_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile xchg ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -4164,6 +5341,18 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr addrspace(1) %out, i32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4222,6 +5411,21 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4276,18 +5480,37 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dword s7, s[4:5], 0x3c
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x3c
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX9-NEXT:    s_add_u32 s0, s2, s0
 ; GFX9-NEXT:    s_addc_u32 s1, s3, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s7, s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -4364,6 +5587,26 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(ptr addrspace(1)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s9, s[4:5], 0x44
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -4415,6 +5658,18 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr addrspace(1) %out, i32 %in, i3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = cmpxchg volatile ptr addrspace(1) %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
   ret void
@@ -4472,6 +5727,21 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr addrspace(1) %out, ptr add
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = cmpxchg volatile ptr addrspace(1) %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
   %extract0 = extractvalue { i32, i1 } %val, 0
@@ -4523,18 +5793,37 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr addrspace(1) %out, i32
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dword s7, s[4:5], 0x3c
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x3c
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX9-NEXT:    s_add_u32 s0, s2, s0
 ; GFX9-NEXT:    s_addc_u32 s1, s3, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s7, s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = cmpxchg volatile ptr addrspace(1) %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4608,6 +5897,26 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(ptr addrspace(1) %out,
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s9, s[4:5], 0x44
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = cmpxchg volatile ptr addrspace(1) %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
@@ -4654,6 +5963,19 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -4709,6 +6031,20 @@ define amdgpu_kernel void @atomic_xor_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -4768,6 +6104,24 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -4838,6 +6192,25 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -4884,6 +6257,19 @@ define amdgpu_kernel void @atomic_xor_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   ret void
@@ -4938,6 +6324,20 @@ define amdgpu_kernel void @atomic_xor_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
   store i32 %val, ptr addrspace(1) %out2
@@ -4994,6 +6394,24 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -5061,6 +6479,25 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_xor_i32_ret_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
@@ -5113,6 +6550,18 @@ define amdgpu_kernel void @atomic_load_i32_offset(ptr addrspace(1) %in, ptr addr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %in, i64 4
   %val = load atomic i32, ptr addrspace(1) %gep  seq_cst, align 4
@@ -5167,6 +6616,18 @@ define amdgpu_kernel void @atomic_load_i32_negoffset(ptr addrspace(1) %in, ptr a
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32_negoffset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:-512 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %in, i64 -128
   %val = load atomic i32, ptr addrspace(1) %gep  seq_cst, align 4
@@ -5219,6 +6680,18 @@ define amdgpu_kernel void @atomic_load_f32_offset(ptr addrspace(1) %in, ptr addr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr float, ptr addrspace(1) %in, i64 4
   %val = load atomic float, ptr addrspace(1) %gep  seq_cst, align 4
@@ -5269,6 +6742,18 @@ define amdgpu_kernel void @atomic_load_i32(ptr addrspace(1) %in, ptr addrspace(1
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent") seq_cst, align 4
   store i32 %val, ptr addrspace(1) %out
@@ -5332,6 +6817,24 @@ define amdgpu_kernel void @atomic_load_i32_addr64_offset(ptr addrspace(1) %in, p
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %in, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -5395,6 +6898,24 @@ define amdgpu_kernel void @atomic_load_i32_addr64(ptr addrspace(1) %in, ptr addr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %in, i64 %index
   %val = load atomic i32, ptr addrspace(1) %ptr seq_cst, align 4
@@ -5459,6 +6980,24 @@ define amdgpu_kernel void @atomic_load_f32_addr64_offset(ptr addrspace(1) %in, p
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr float, ptr addrspace(1) %in, i64 %index
   %gep = getelementptr float, ptr addrspace(1) %ptr, i64 4
@@ -5501,6 +7040,16 @@ define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, ptr addrspace(1) %ou
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   store atomic i32 %in, ptr addrspace(1) %gep  seq_cst, align 4
@@ -5539,6 +7088,16 @@ define amdgpu_kernel void @atomic_store_i32(i32 %in, ptr addrspace(1) %out) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   store atomic i32 %in, ptr addrspace(1) %out seq_cst, align 4
   ret void
@@ -5576,6 +7135,16 @@ define amdgpu_kernel void @atomic_store_f32(float %in, ptr addrspace(1) %out) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   store atomic float %in, ptr addrspace(1) %out seq_cst, align 4
   ret void
@@ -5624,6 +7193,20 @@ define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, ptr addrspace
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -5674,6 +7257,20 @@ define amdgpu_kernel void @atomic_store_f32_addr64_offset(float %in, ptr addrspa
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr float, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr float, ptr addrspace(1) %ptr, i64 4
@@ -5723,6 +7320,20 @@ define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, ptr addrspace(1) %ou
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_i32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   store atomic i32 %in, ptr addrspace(1) %ptr seq_cst, align 4
@@ -5771,6 +7382,20 @@ define amdgpu_kernel void @atomic_store_f32_addr64(float %in, ptr addrspace(1) %
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32_addr64:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s4
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr float, ptr addrspace(1) %out, i64 %index
   store atomic float %in, ptr addrspace(1) %ptr seq_cst, align 4
@@ -5820,6 +7445,30 @@ define amdgpu_kernel void @atomic_load_i8_offset(ptr addrspace(1) %in, ptr addrs
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_byte v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i8_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_u8 v0, v1, s[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i8_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u8 v1, v0, s[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(1) %in, i64 16
   %val = load atomic i8, ptr addrspace(1) %gep  seq_cst, align 1
@@ -5874,6 +7523,30 @@ define amdgpu_kernel void @atomic_load_i8_negoffset(ptr addrspace(1) %in, ptr ad
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_byte v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i8_negoffset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_u8 v0, v1, s[0:1] offset:-512 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i8_negoffset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u8 v1, v0, s[0:1] offset:-512 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(1) %in, i64 -512
   %val = load atomic i8, ptr addrspace(1) %gep  seq_cst, align 1
@@ -5915,6 +7588,27 @@ define amdgpu_kernel void @atomic_store_i8_offset(i8 %in, ptr addrspace(1) %out)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_byte v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i8_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[0:1] offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i8_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[0:1] offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(1) %out, i64 16
   store atomic i8 %in, ptr addrspace(1) %gep  seq_cst, align 1
@@ -5953,6 +7647,27 @@ define amdgpu_kernel void @atomic_store_i8(i8 %in, ptr addrspace(1) %out) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i8:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i8:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   store atomic i8 %in, ptr addrspace(1) %out seq_cst, align 1
   ret void
@@ -6001,6 +7716,30 @@ define amdgpu_kernel void @atomic_load_i16_offset(ptr addrspace(1) %in, ptr addr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_short v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i16_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i16_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i16, ptr addrspace(1) %in, i64 8
   %val = load atomic i16, ptr addrspace(1) %gep  seq_cst, align 2
@@ -6055,6 +7794,30 @@ define amdgpu_kernel void @atomic_load_i16_negoffset(ptr addrspace(1) %in, ptr a
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_short v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_i16_negoffset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:-512 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_i16_negoffset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:-512 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i16, ptr addrspace(1) %in, i64 -256
   %val = load atomic i16, ptr addrspace(1) %gep  seq_cst, align 2
@@ -6096,6 +7859,27 @@ define amdgpu_kernel void @atomic_store_i16_offset(i16 %in, ptr addrspace(1) %ou
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_short v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i16_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1] offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i16_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1] offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i16, ptr addrspace(1) %out, i64 8
   store atomic i16 %in, ptr addrspace(1) %gep  seq_cst, align 2
@@ -6134,6 +7918,27 @@ define amdgpu_kernel void @atomic_store_i16(i16 %in, ptr addrspace(1) %out) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_i16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_i16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   store atomic i16 %in, ptr addrspace(1) %out seq_cst, align 2
   ret void
@@ -6173,6 +7978,27 @@ define amdgpu_kernel void @atomic_store_f16_offset(half %in, ptr addrspace(1) %o
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_short v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_f16_offset:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1] offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_f16_offset:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1] offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   %gep = getelementptr half, ptr addrspace(1) %out, i64 8
   store atomic half %in, ptr addrspace(1) %gep  seq_cst, align 2
@@ -6211,6 +8037,27 @@ define amdgpu_kernel void @atomic_store_f16(half %in, ptr addrspace(1) %out) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
 entry:
   store atomic half %in, ptr addrspace(1) %out seq_cst, align 2
   ret void
@@ -6250,6 +8097,27 @@ define amdgpu_kernel void @atomic_store_bf16_offset(bfloat %in, ptr addrspace(1)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_short v0, v1, s[0:1] offset:16
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_bf16_offset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1] offset:16
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_bf16_offset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1] offset:16
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr bfloat, ptr addrspace(1) %out, i64 8
   store atomic bfloat %in, ptr addrspace(1) %gep seq_cst, align 2
   ret void
@@ -6287,6 +8155,27 @@ define amdgpu_kernel void @atomic_store_bf16(bfloat %in, ptr addrspace(1) %out)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_store_bf16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_store_bf16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
   store atomic bfloat %in, ptr addrspace(1) %out seq_cst, align 2
   ret void
 }
@@ -6329,6 +8218,19 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6377,6 +8279,19 @@ define amdgpu_kernel void @atomic_inc_i32_max_neg_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_max_neg_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:-4096
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 -1024
   %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6423,6 +8338,19 @@ define amdgpu_kernel void @atomic_inc_i32_soffset(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_soffset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:3232
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 9000
   %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6473,6 +8401,21 @@ define amdgpu_kernel void @atomic_inc_i32_huge_offset(ptr addrspace(1) %out, i32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_huge_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    s_add_u32 s0, s0, 0xdeac
+; GFX11-NEXT:    s_addc_u32 s1, s1, 0xabcd
+; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 47224239175595
   %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6528,6 +8471,20 @@ define amdgpu_kernel void @atomic_inc_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_inc_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6587,6 +8544,24 @@ define amdgpu_kernel void @atomic_inc_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -6657,6 +8632,25 @@ define amdgpu_kernel void @atomic_inc_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_inc_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_inc_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -6703,6 +8697,19 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6751,6 +8758,19 @@ define amdgpu_kernel void @atomic_dec_i32_max_neg_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_max_neg_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:-4096
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 -1024
   %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6797,6 +8817,19 @@ define amdgpu_kernel void @atomic_dec_i32_soffset(ptr addrspace(1) %out, i32 %in
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_soffset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:3232
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 9000
   %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6847,6 +8880,21 @@ define amdgpu_kernel void @atomic_dec_i32_huge_offset(ptr addrspace(1) %out, i32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_huge_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    s_add_u32 s0, s0, 0xdeac
+; GFX11-NEXT:    s_addc_u32 s1, s1, 0xabcd
+; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 47224239175595
   %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6902,6 +8950,20 @@ define amdgpu_kernel void @atomic_dec_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_ret_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT:    global_atomic_dec_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
@@ -6961,6 +9023,24 @@ define amdgpu_kernel void @atomic_dec_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:    s_add_u32 s0, s2, s0
+; GFX11-NEXT:    s_addc_u32 s1, s3, s1
+; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:16
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -7031,6 +9111,25 @@ define amdgpu_kernel void @atomic_dec_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: atomic_dec_i32_ret_addr64_offset:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s8
+; GFX11-NEXT:    s_add_u32 s0, s0, s4
+; GFX11-NEXT:    s_addc_u32 s1, s1, s5
+; GFX11-NEXT:    global_atomic_dec_u32 v1, v0, v1, s[0:1] offset:16 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    buffer_gl1_inv
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
@@ -7082,6 +9181,30 @@ define amdgpu_kernel void @atomic_load_f16_offset(ptr addrspace(1) %in, ptr addr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_short v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_f16_offset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_f16_offset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr half, ptr addrspace(1) %in, i64 8
   %val = load atomic half, ptr addrspace(1) %gep  seq_cst, align 2
   store half %val, ptr addrspace(1) %out
@@ -7135,6 +9258,30 @@ define amdgpu_kernel void @atomic_load_f16_negoffset(ptr addrspace(1) %in, ptr a
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_short v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_f16_negoffset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:-512 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_f16_negoffset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:-512 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr half, ptr addrspace(1) %in, i64 -256
   %val = load atomic half, ptr addrspace(1) %gep  seq_cst, align 2
   store half %val, ptr addrspace(1) %out
@@ -7184,6 +9331,30 @@ define amdgpu_kernel void @atomic_load_bf16_offset(ptr addrspace(1) %in, ptr add
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_short v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_bf16_offset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:16 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_bf16_offset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:16 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr bfloat, ptr addrspace(1) %in, i64 8
   %val = load atomic bfloat, ptr addrspace(1) %gep  seq_cst, align 2
   store bfloat %val, ptr addrspace(1) %out
@@ -7237,6 +9408,30 @@ define amdgpu_kernel void @atomic_load_bf16_negoffset(ptr addrspace(1) %in, ptr
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:    global_store_short v0, v1, s[2:3]
 ; GFX9-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: atomic_load_bf16_negoffset:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:-512 glc
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    buffer_gl1_inv
+; GFX11-TRUE16-NEXT:    buffer_gl0_inv
+; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: atomic_load_bf16_negoffset:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:-512 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_gl1_inv
+; GFX11-FAKE16-NEXT:    buffer_gl0_inv
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_endpgm
   %gep = getelementptr bfloat, ptr addrspace(1) %in, i64 -256
   %val = load atomic bfloat, ptr addrspace(1) %gep  seq_cst, align 2
   store bfloat %val, ptr addrspace(1) %out
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/hard-clause-limit-attr.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/hard-clause-limit-attr.mir
new file mode 100644
index 000000000000..cfd4ea155316
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/hard-clause-limit-attr.mir
@@ -0,0 +1,178 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-insert-hard-clauses %s -o - | FileCheck %s -check-prefix=GFX11
+
+--- |
+  define void @long_clause_32() "amdgpu-hard-clause-length-limit"="32" {
+    ret void
+  }
+  define void @long_clause_7() "amdgpu-hard-clause-length-limit"="7" {
+    ret void
+  }
+
+---
+name: long_clause_32
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-LABEL: name: long_clause_32
+    ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT:   S_CLAUSE 31
+    ; GFX11-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX11-NEXT: }
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+...
+
+---
+name: long_clause_7
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-LABEL: name: long_clause_7
+    ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT:   S_CLAUSE 6
+    ; GFX11-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX11-NEXT: }
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT:   S_CLAUSE 6
+    ; GFX11-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX11-NEXT: }
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT:   S_CLAUSE 6
+    ; GFX11-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX11-NEXT: }
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT:   S_CLAUSE 6
+    ; GFX11-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX11-NEXT: }
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT:   S_CLAUSE 3
+    ; GFX11-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX11-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX11-NEXT: }
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/hard-clause-limit.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/hard-clause-limit.mir
new file mode 100644
index 000000000000..98221c21eee2
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/hard-clause-limit.mir
@@ -0,0 +1,270 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-insert-hard-clauses -amdgpu-hard-clause-length-limit=32 %s -o - | FileCheck %s -check-prefix=GFX11-32
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-insert-hard-clauses -amdgpu-hard-clause-length-limit=16 %s -o - | FileCheck %s -check-prefix=GFX11-16
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-insert-hard-clauses -amdgpu-hard-clause-length-limit=10 %s -o - | FileCheck %s -check-prefix=GFX11-10
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-insert-hard-clauses -amdgpu-hard-clause-length-limit=4 %s -o - | FileCheck %s -check-prefix=GFX11-4
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-insert-hard-clauses -amdgpu-hard-clause-length-limit=1 %s -o - | FileCheck %s -check-prefix=GFX11-1
+
+---
+name: long_clause
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-32-LABEL: name: long_clause
+    ; GFX11-32: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-32-NEXT: {{  $}}
+    ; GFX11-32-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-32-NEXT:   S_CLAUSE 31
+    ; GFX11-32-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX11-32-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX11-32-NEXT: }
+    ;
+    ; GFX11-16-LABEL: name: long_clause
+    ; GFX11-16: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-16-NEXT: {{  $}}
+    ; GFX11-16-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-16-NEXT:   S_CLAUSE 15
+    ; GFX11-16-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX11-16-NEXT: }
+    ; GFX11-16-NEXT: BUNDLE implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-16-NEXT:   S_CLAUSE 15
+    ; GFX11-16-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX11-16-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX11-16-NEXT: }
+    ;
+    ; GFX11-10-LABEL: name: long_clause
+    ; GFX11-10: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-10-NEXT: {{  $}}
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT:   S_CLAUSE 9
+    ; GFX11-10-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX11-10-NEXT: }
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT:   S_CLAUSE 9
+    ; GFX11-10-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX11-10-NEXT: }
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT:   S_CLAUSE 9
+    ; GFX11-10-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX11-10-NEXT: }
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT:   S_CLAUSE 1
+    ; GFX11-10-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX11-10-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX11-10-NEXT: }
+    ;
+    ; GFX11-4-LABEL: name: long_clause
+    ; GFX11-4: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-4-NEXT: {{  $}}
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT:   S_CLAUSE 3
+    ; GFX11-4-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX11-4-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX11-4-NEXT: }
+    ;
+    ; GFX11-1-LABEL: name: long_clause
+    ; GFX11-1: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX11-1-NEXT: {{  $}}
+    ; GFX11-1-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX11-1-NEXT: $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4s.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4s.ll
index d28f0a190e11..9e7968f1acb8 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -2903,13 +2903,13 @@ define amdgpu_kernel void @idot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,
 ; GFX9-DL-NEXT:    s_mov_b32 s1, 0xc0c0c01
 ; GFX9-DL-NEXT:    s_mov_b32 s2, 0xc020101
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    global_load_dword v1, v0, s[12:13]
-; GFX9-DL-NEXT:    global_load_dword v2, v0, s[8:9]
+; GFX9-DL-NEXT:    global_load_dword v1, v0, s[8:9]
+; GFX9-DL-NEXT:    global_load_dword v2, v0, s[12:13]
 ; GFX9-DL-NEXT:    global_load_dword v3, v0, s[10:11]
 ; GFX9-DL-NEXT:    s_load_dword s3, s[14:15], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT:    v_perm_b32 v1, v1, v2, s0
+; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v1, s0
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DL-NEXT:    v_perm_b32 v2, v3, v3, s1
 ; GFX9-DL-NEXT:    v_or_b32_e32 v1, v1, v2
@@ -2925,12 +2925,12 @@ define amdgpu_kernel void @idot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,
 ; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_clause 0x2
-; GFX10-DL-NEXT:    global_load_dword v1, v0, s[12:13]
-; GFX10-DL-NEXT:    global_load_dword v2, v0, s[8:9]
+; GFX10-DL-NEXT:    global_load_dword v1, v0, s[8:9]
+; GFX10-DL-NEXT:    global_load_dword v2, v0, s[12:13]
 ; GFX10-DL-NEXT:    global_load_dword v3, v0, s[10:11]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[14:15], 0x0
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT:    v_perm_b32 v0, v1, v2, 0xc06010c
+; GFX10-DL-NEXT:    v_perm_b32 v0, v2, v1, 0xc06010c
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-DL-NEXT:    v_perm_b32 v1, v3, v3, 0xc0c0c01
 ; GFX10-DL-NEXT:    v_perm_b32 v2, v3, v3, 0xc020101
@@ -2950,12 +2950,12 @@ define amdgpu_kernel void @idot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,
 ; GFX11-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-DL-NEXT:    s_clause 0x2
-; GFX11-DL-NEXT:    global_load_b32 v1, v0, s[4:5]
-; GFX11-DL-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-DL-NEXT:    global_load_b32 v1, v0, s[0:1]
+; GFX11-DL-NEXT:    global_load_b32 v2, v0, s[4:5]
 ; GFX11-DL-NEXT:    global_load_b32 v0, v0, s[2:3]
 ; GFX11-DL-NEXT:    s_load_b32 s0, s[6:7], 0x0
 ; GFX11-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-NEXT:    v_perm_b32 v1, v1, v2, 0xc06010c
+; GFX11-DL-NEXT:    v_perm_b32 v1, v2, v1, 0xc06010c
 ; GFX11-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-DL-NEXT:    v_perm_b32 v2, v0, v0, 0xc0c0c01
 ; GFX11-DL-NEXT:    v_perm_b32 v0, v0, v0, 0xc020101
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4u.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4u.ll
index df7775744339..f995f426c637 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -4454,13 +4454,13 @@ define amdgpu_kernel void @udot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,
 ; GFX9-DL-NEXT:    s_mov_b32 s1, 0xc0c0c01
 ; GFX9-DL-NEXT:    s_mov_b32 s2, 0xc020101
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    global_load_dword v1, v0, s[12:13]
-; GFX9-DL-NEXT:    global_load_dword v2, v0, s[8:9]
+; GFX9-DL-NEXT:    global_load_dword v1, v0, s[8:9]
+; GFX9-DL-NEXT:    global_load_dword v2, v0, s[12:13]
 ; GFX9-DL-NEXT:    global_load_dword v3, v0, s[10:11]
 ; GFX9-DL-NEXT:    s_load_dword s3, s[14:15], 0x0
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT:    v_perm_b32 v1, v1, v2, s0
+; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v1, s0
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DL-NEXT:    v_perm_b32 v2, v3, v3, s1
 ; GFX9-DL-NEXT:    v_or_b32_e32 v1, v1, v2
@@ -4476,12 +4476,12 @@ define amdgpu_kernel void @udot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,
 ; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_clause 0x2
-; GFX10-DL-NEXT:    global_load_dword v1, v0, s[12:13]
-; GFX10-DL-NEXT:    global_load_dword v2, v0, s[8:9]
+; GFX10-DL-NEXT:    global_load_dword v1, v0, s[8:9]
+; GFX10-DL-NEXT:    global_load_dword v2, v0, s[12:13]
 ; GFX10-DL-NEXT:    global_load_dword v3, v0, s[10:11]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[14:15], 0x0
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT:    v_perm_b32 v0, v1, v2, 0xc06010c
+; GFX10-DL-NEXT:    v_perm_b32 v0, v2, v1, 0xc06010c
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-DL-NEXT:    v_perm_b32 v1, v3, v3, 0xc0c0c01
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
@@ -4500,12 +4500,12 @@ define amdgpu_kernel void @udot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,
 ; GFX11-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-DL-NEXT:    s_clause 0x2
-; GFX11-DL-NEXT:    global_load_b32 v1, v0, s[4:5]
-; GFX11-DL-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-DL-NEXT:    global_load_b32 v1, v0, s[0:1]
+; GFX11-DL-NEXT:    global_load_b32 v2, v0, s[4:5]
 ; GFX11-DL-NEXT:    global_load_b32 v0, v0, s[2:3]
 ; GFX11-DL-NEXT:    s_load_b32 s0, s[6:7], 0x0
 ; GFX11-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-NEXT:    v_perm_b32 v1, v1, v2, 0xc06010c
+; GFX11-DL-NEXT:    v_perm_b32 v1, v2, v1, 0xc06010c
 ; GFX11-DL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-DL-NEXT:    v_perm_b32 v2, v0, v0, 0xc0c0c01
 ; GFX11-DL-NEXT:    v_perm_b32 v0, v0, v0, 0xc020101
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
index 6ce9f4c367f4..258aa9e299c3 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
@@ -122,25 +122,35 @@ define protected amdgpu_kernel void @InferPHI(i32 %a, ptr addrspace(1) %b, doubl
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[0:1]
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB3_1
 ; CHECK-NEXT:  ; %bb.2: ; %bb1
-; CHECK-NEXT:    s_mov_b64 s[0:1], src_private_base
+; CHECK-NEXT:    s_mov_b64 s[0:1], src_shared_base
 ; CHECK-NEXT:    s_cmp_eq_u32 s5, s1
 ; CHECK-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
 ; CHECK-NEXT:    s_mov_b64 s[0:1], -1
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB3_5
-; CHECK-NEXT:  ; %bb.3: ; %Flow
+; CHECK-NEXT:  ; %bb.3: ; %Flow6
 ; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
-; CHECK-NEXT:    s_cbranch_vccz .LBB3_6
+; CHECK-NEXT:    s_cbranch_vccz .LBB3_10
 ; CHECK-NEXT:  .LBB3_4: ; %atomicrmw.phi
 ; CHECK-NEXT:    s_endpgm
-; CHECK-NEXT:  .LBB3_5: ; %atomicrmw.global
-; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; CHECK-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; CHECK-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:  .LBB3_5: ; %atomicrmw.check.private
+; CHECK-NEXT:    s_mov_b64 s[0:1], src_private_base
+; CHECK-NEXT:    s_cmp_eq_u32 s5, s1
+; CHECK-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; CHECK-NEXT:    s_mov_b64 s[0:1], -1
+; CHECK-NEXT:    s_cbranch_vccz .LBB3_7
+; CHECK-NEXT:  ; %bb.6: ; %atomicrmw.global
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT:    global_atomic_add_f64 v2, v[0:1], s[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    buffer_wbinvl1_vol
-; CHECK-NEXT:    s_cbranch_execnz .LBB3_4
-; CHECK-NEXT:  .LBB3_6: ; %atomicrmw.private
+; CHECK-NEXT:    s_mov_b64 s[0:1], 0
+; CHECK-NEXT:  .LBB3_7: ; %Flow
+; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
+; CHECK-NEXT:    s_cbranch_vccnz .LBB3_9
+; CHECK-NEXT:  ; %bb.8: ; %atomicrmw.private
 ; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; CHECK-NEXT:    s_cselect_b32 s0, s4, -1
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s0
@@ -150,6 +160,15 @@ define protected amdgpu_kernel void @InferPHI(i32 %a, ptr addrspace(1) %b, doubl
 ; CHECK-NEXT:    v_add_f64 v[0:1], v[0:1], s[2:3]
 ; CHECK-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
 ; CHECK-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen offset:4
+; CHECK-NEXT:  .LBB3_9: ; %Flow5
+; CHECK-NEXT:    s_cbranch_execnz .LBB3_4
+; CHECK-NEXT:  .LBB3_10: ; %atomicrmw.shared
+; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; CHECK-NEXT:    s_cselect_b32 s0, s4, -1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT:    ds_add_f64 v2, v[0:1]
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_endpgm
 entry:
   %i = add nsw i32 %a, -1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index 6925a98f643b..e1b4cad370f9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -289,16 +289,16 @@ entry:
 define amdgpu_kernel void @half4_inselt(ptr addrspace(1) %out, <4 x half> %vec, i32 %sel) {
 ; GCN-LABEL: half4_inselt:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_mov_b32 s4, 0x3c003c00
 ; GCN-NEXT:    s_mov_b32 s5, s4
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_xor_b64 s[4:5], s[2:3], s[4:5]
 ; GCN-NEXT:    s_lshl_b32 s6, s6, 4
 ; GCN-NEXT:    s_lshl_b64 s[6:7], 0xffff, s6
-; GCN-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
-; GCN-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
-; GCN-NEXT:    s_or_b64 s[2:3], s[4:5], s[2:3]
+; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; GCN-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -317,10 +317,10 @@ define amdgpu_kernel void @half2_inselt(ptr addrspace(1) %out, <2 x half> %vec,
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_lshl_b32 s3, s3, 4
+; GCN-NEXT:    s_xor_b32 s4, s2, 0x3c003c00
 ; GCN-NEXT:    s_lshl_b32 s3, 0xffff, s3
-; GCN-NEXT:    s_andn2_b32 s2, s2, s3
-; GCN-NEXT:    s_and_b32 s3, s3, 0x3c003c00
-; GCN-NEXT:    s_or_b32 s2, s3, s2
+; GCN-NEXT:    s_and_b32 s3, s4, s3
+; GCN-NEXT:    s_xor_b32 s2, s3, s2
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
@@ -399,10 +399,10 @@ define amdgpu_kernel void @short2_inselt(ptr addrspace(1) %out, <2 x i16> %vec,
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_lshl_b32 s3, s3, 4
+; GCN-NEXT:    s_xor_b32 s4, s2, 0x10001
 ; GCN-NEXT:    s_lshl_b32 s3, 0xffff, s3
-; GCN-NEXT:    s_andn2_b32 s2, s2, s3
-; GCN-NEXT:    s_and_b32 s3, s3, 0x10001
-; GCN-NEXT:    s_or_b32 s2, s3, s2
+; GCN-NEXT:    s_and_b32 s3, s4, s3
+; GCN-NEXT:    s_xor_b32 s2, s3, s2
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
@@ -417,16 +417,16 @@ entry:
 define amdgpu_kernel void @short4_inselt(ptr addrspace(1) %out, <4 x i16> %vec, i32 %sel) {
 ; GCN-LABEL: short4_inselt:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_mov_b32 s4, 0x10001
 ; GCN-NEXT:    s_mov_b32 s5, s4
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_xor_b64 s[4:5], s[2:3], s[4:5]
 ; GCN-NEXT:    s_lshl_b32 s6, s6, 4
 ; GCN-NEXT:    s_lshl_b64 s[6:7], 0xffff, s6
-; GCN-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
-; GCN-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
-; GCN-NEXT:    s_or_b64 s[2:3], s[4:5], s[2:3]
+; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; GCN-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
@@ -442,15 +442,15 @@ entry:
 define amdgpu_kernel void @byte8_inselt(ptr addrspace(1) %out, <8 x i8> %vec, i32 %sel) {
 ; GCN-LABEL: byte8_inselt:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_lshl_b32 s4, s6, 3
-; GCN-NEXT:    s_lshl_b64 s[4:5], 0xff, s4
-; GCN-NEXT:    s_and_b32 s7, s5, 0x1010101
-; GCN-NEXT:    s_and_b32 s6, s4, 0x1010101
-; GCN-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GCN-NEXT:    s_or_b64 s[2:3], s[6:7], s[2:3]
+; GCN-NEXT:    s_xor_b32 s5, s3, 0x1010101
+; GCN-NEXT:    s_lshl_b32 s6, s6, 3
+; GCN-NEXT:    s_xor_b32 s4, s2, 0x1010101
+; GCN-NEXT:    s_lshl_b64 s[6:7], 0xff, s6
+; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; GCN-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index f6330f4eb821..44bd4090436e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -1101,14 +1101,14 @@ define amdgpu_kernel void @dynamic_insertelement_v3i32(ptr addrspace(1) %out, <3
 define amdgpu_kernel void @dynamic_insertelement_v4i32(ptr addrspace(1) %out, <4 x i32> %a, i32 %b, [8 x i32], i32 %val) nounwind {
 ; SI-LABEL: dynamic_insertelement_v4i32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
 ; SI-NEXT:    s_load_dword s10, s[8:9], 0x8
 ; SI-NEXT:    s_load_dword s11, s[8:9], 0x11
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; SI-NEXT:    s_mov_b32 s7, 0x100f000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_eq_u32 s10, 3
+; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_cselect_b32 s3, s11, s3
 ; SI-NEXT:    s_cmp_eq_u32 s10, 2
 ; SI-NEXT:    s_cselect_b32 s2, s11, s2
@@ -1125,14 +1125,14 @@ define amdgpu_kernel void @dynamic_insertelement_v4i32(ptr addrspace(1) %out, <4
 ;
 ; VI-LABEL: dynamic_insertelement_v4i32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
 ; VI-NEXT:    s_load_dword s10, s[8:9], 0x20
 ; VI-NEXT:    s_load_dword s11, s[8:9], 0x44
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; VI-NEXT:    s_mov_b32 s7, 0x1100f000
-; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_eq_u32 s10, 3
+; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_cselect_b32 s3, s11, s3
 ; VI-NEXT:    s_cmp_eq_u32 s10, 2
 ; VI-NEXT:    s_cselect_b32 s2, s11, s2
@@ -1511,13 +1511,13 @@ define amdgpu_kernel void @dynamic_insertelement_v2i16(ptr addrspace(1) %out, <2
 ; SI-NEXT:    s_mov_b32 s7, 0x100f000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_lshl_b32 s0, s3, 4
-; SI-NEXT:    s_lshl_b32 s0, 0xffff, s0
 ; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_andn2_b32 s1, s2, s0
-; SI-NEXT:    s_and_b32 s0, s0, 0x50005
-; SI-NEXT:    s_or_b32 s0, s0, s1
+; SI-NEXT:    s_lshl_b32 s1, s3, 4
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_xor_b32 s0, s2, 0x50005
+; SI-NEXT:    s_lshl_b32 s1, 0xffff, s1
+; SI-NEXT:    s_and_b32 s0, s0, s1
+; SI-NEXT:    s_xor_b32 s0, s0, s2
 ; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
@@ -1528,13 +1528,13 @@ define amdgpu_kernel void @dynamic_insertelement_v2i16(ptr addrspace(1) %out, <2
 ; VI-NEXT:    s_mov_b32 s7, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_lshl_b32 s0, s3, 4
-; VI-NEXT:    s_lshl_b32 s0, 0xffff, s0
 ; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    s_andn2_b32 s1, s2, s0
-; VI-NEXT:    s_and_b32 s0, s0, 0x50005
-; VI-NEXT:    s_or_b32 s0, s0, s1
+; VI-NEXT:    s_lshl_b32 s1, s3, 4
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_xor_b32 s0, s2, 0x50005
+; VI-NEXT:    s_lshl_b32 s1, 0xffff, s1
+; VI-NEXT:    s_and_b32 s0, s0, s1
+; VI-NEXT:    s_xor_b32 s0, s0, s2
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
@@ -1552,13 +1552,13 @@ define amdgpu_kernel void @dynamic_insertelement_v3i16(ptr addrspace(1) %out, <3
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_lshl_b32 s0, s8, 4
+; SI-NEXT:    s_lshl_b32 s8, s8, 4
 ; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_lshl_b64 s[0:1], 0xffff, s0
-; SI-NEXT:    s_and_b32 s9, s1, 0x50005
-; SI-NEXT:    s_and_b32 s8, s0, 0x50005
-; SI-NEXT:    s_andn2_b64 s[0:1], s[2:3], s[0:1]
-; SI-NEXT:    s_or_b64 s[0:1], s[8:9], s[0:1]
+; SI-NEXT:    s_xor_b32 s1, s3, 0x50005
+; SI-NEXT:    s_xor_b32 s0, s2, 0x50005
+; SI-NEXT:    s_lshl_b64 s[8:9], 0xffff, s8
+; SI-NEXT:    s_and_b64 s[0:1], s[0:1], s[8:9]
+; SI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; SI-NEXT:    v_mov_b32_e32 v0, s1
 ; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0 offset:4
 ; SI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1573,14 +1573,14 @@ define amdgpu_kernel void @dynamic_insertelement_v3i16(ptr addrspace(1) %out, <3
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_lshl_b32 s0, s8, 4
-; VI-NEXT:    s_mov_b32 s8, 0x50005
+; VI-NEXT:    s_mov_b32 s0, 0x50005
 ; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    s_lshl_b64 s[0:1], 0xffff, s0
-; VI-NEXT:    s_mov_b32 s9, s8
-; VI-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[0:1]
+; VI-NEXT:    s_mov_b32 s1, s0
+; VI-NEXT:    s_lshl_b32 s8, s8, 4
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
+; VI-NEXT:    s_lshl_b64 s[8:9], 0xffff, s8
 ; VI-NEXT:    s_and_b64 s[0:1], s[0:1], s[8:9]
-; VI-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; VI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v0, s1
 ; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0 offset:4
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
@@ -1594,35 +1594,34 @@ define amdgpu_kernel void @dynamic_insertelement_v3i16(ptr addrspace(1) %out, <3
 define amdgpu_kernel void @dynamic_insertelement_v2i8(ptr addrspace(1) %out, [8 x i32], <2 x i8> %a, [8 x i32], i32 %b) nounwind {
 ; SI-LABEL: dynamic_insertelement_v2i8:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s4, s[8:9], 0x13
+; SI-NEXT:    s_load_dword s4, s[8:9], 0xa
+; SI-NEXT:    s_load_dword s5, s[8:9], 0x13
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT:    s_load_dword s5, s[8:9], 0xa
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_lshl_b32 s4, s4, 3
-; SI-NEXT:    s_lshl_b32 s4, 0xff, s4
-; SI-NEXT:    s_andn2_b32 s5, s5, s4
-; SI-NEXT:    s_and_b32 s4, s4, 0x505
-; SI-NEXT:    s_or_b32 s4, s4, s5
+; SI-NEXT:    s_xor_b32 s6, s4, 0x505
+; SI-NEXT:    s_lshl_b32 s5, s5, 3
+; SI-NEXT:    s_lshl_b32 s5, 0xff, s5
+; SI-NEXT:    s_and_b32 s5, s6, s5
+; SI-NEXT:    s_xor_b32 s4, s5, s4
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v2i8:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dword s4, s[8:9], 0x4c
+; VI-NEXT:    s_load_dword s4, s[8:9], 0x28
+; VI-NEXT:    s_load_dword s5, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    s_load_dword s5, s[8:9], 0x28
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_lshl_b32 s4, s4, 3
-; VI-NEXT:    s_lshl_b32 s4, 0xff, s4
-; VI-NEXT:    s_and_b32 s6, s4, 0x505
-; VI-NEXT:    s_xor_b32 s4, s4, 0xffff
-; VI-NEXT:    s_and_b32 s4, s4, s5
-; VI-NEXT:    s_or_b32 s4, s6, s4
+; VI-NEXT:    s_xor_b32 s6, s4, 0x505
+; VI-NEXT:    s_lshl_b32 s5, s5, 3
+; VI-NEXT:    s_lshl_b32 s5, 0xff, s5
+; VI-NEXT:    s_and_b32 s5, s6, s5
+; VI-NEXT:    s_xor_b32 s4, s5, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
@@ -1636,17 +1635,17 @@ define amdgpu_kernel void @dynamic_insertelement_v2i8(ptr addrspace(1) %out, [8
 define amdgpu_kernel void @dynamic_insertelement_v3i8(ptr addrspace(1) %out, [8 x i32], <3 x i8> %a, [8 x i32], i32 %b) nounwind {
 ; SI-LABEL: dynamic_insertelement_v3i8:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s4, s[8:9], 0x13
+; SI-NEXT:    s_load_dword s4, s[8:9], 0xa
+; SI-NEXT:    s_load_dword s5, s[8:9], 0x13
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT:    s_load_dword s5, s[8:9], 0xa
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_lshl_b32 s4, s4, 3
-; SI-NEXT:    s_lshl_b32 s4, 0xff, s4
-; SI-NEXT:    s_andn2_b32 s5, s5, s4
-; SI-NEXT:    s_and_b32 s4, s4, 0x5050505
-; SI-NEXT:    s_or_b32 s4, s4, s5
+; SI-NEXT:    s_xor_b32 s6, s4, 0x5050505
+; SI-NEXT:    s_lshl_b32 s5, s5, 3
+; SI-NEXT:    s_lshl_b32 s5, 0xff, s5
+; SI-NEXT:    s_and_b32 s5, s6, s5
+; SI-NEXT:    s_xor_b32 s4, s5, s4
 ; SI-NEXT:    s_lshr_b32 s5, s4, 16
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
@@ -1656,17 +1655,17 @@ define amdgpu_kernel void @dynamic_insertelement_v3i8(ptr addrspace(1) %out, [8
 ;
 ; VI-LABEL: dynamic_insertelement_v3i8:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dword s4, s[8:9], 0x4c
+; VI-NEXT:    s_load_dword s4, s[8:9], 0x28
+; VI-NEXT:    s_load_dword s5, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    s_load_dword s5, s[8:9], 0x28
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_lshl_b32 s4, s4, 3
-; VI-NEXT:    s_lshl_b32 s4, 0xff, s4
-; VI-NEXT:    s_andn2_b32 s5, s5, s4
-; VI-NEXT:    s_and_b32 s4, s4, 0x5050505
-; VI-NEXT:    s_or_b32 s4, s4, s5
+; VI-NEXT:    s_xor_b32 s6, s4, 0x5050505
+; VI-NEXT:    s_lshl_b32 s5, s5, 3
+; VI-NEXT:    s_lshl_b32 s5, 0xff, s5
+; VI-NEXT:    s_and_b32 s5, s6, s5
+; VI-NEXT:    s_xor_b32 s4, s5, s4
 ; VI-NEXT:    s_lshr_b32 s5, s4, 16
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
@@ -1681,34 +1680,34 @@ define amdgpu_kernel void @dynamic_insertelement_v3i8(ptr addrspace(1) %out, [8
 define amdgpu_kernel void @dynamic_insertelement_v4i8(ptr addrspace(1) %out, [8 x i32], <4 x i8> %a, [8 x i32], i32 %b) nounwind {
 ; SI-LABEL: dynamic_insertelement_v4i8:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dword s4, s[8:9], 0x13
+; SI-NEXT:    s_load_dword s4, s[8:9], 0xa
+; SI-NEXT:    s_load_dword s5, s[8:9], 0x13
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT:    s_load_dword s5, s[8:9], 0xa
 ; SI-NEXT:    s_mov_b32 s3, 0x100f000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_lshl_b32 s4, s4, 3
-; SI-NEXT:    s_lshl_b32 s4, 0xff, s4
-; SI-NEXT:    s_andn2_b32 s5, s5, s4
-; SI-NEXT:    s_and_b32 s4, s4, 0x5050505
-; SI-NEXT:    s_or_b32 s4, s4, s5
+; SI-NEXT:    s_xor_b32 s6, s4, 0x5050505
+; SI-NEXT:    s_lshl_b32 s5, s5, 3
+; SI-NEXT:    s_lshl_b32 s5, 0xff, s5
+; SI-NEXT:    s_and_b32 s5, s6, s5
+; SI-NEXT:    s_xor_b32 s4, s5, s4
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: dynamic_insertelement_v4i8:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dword s4, s[8:9], 0x4c
+; VI-NEXT:    s_load_dword s4, s[8:9], 0x28
+; VI-NEXT:    s_load_dword s5, s[8:9], 0x4c
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT:    s_load_dword s5, s[8:9], 0x28
 ; VI-NEXT:    s_mov_b32 s3, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_lshl_b32 s4, s4, 3
-; VI-NEXT:    s_lshl_b32 s4, 0xff, s4
-; VI-NEXT:    s_andn2_b32 s5, s5, s4
-; VI-NEXT:    s_and_b32 s4, s4, 0x5050505
-; VI-NEXT:    s_or_b32 s4, s4, s5
+; VI-NEXT:    s_xor_b32 s6, s4, 0x5050505
+; VI-NEXT:    s_lshl_b32 s5, s5, 3
+; VI-NEXT:    s_lshl_b32 s5, 0xff, s5
+; VI-NEXT:    s_and_b32 s5, s6, s5
+; VI-NEXT:    s_xor_b32 s4, s5, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
@@ -1721,20 +1720,20 @@ define amdgpu_kernel void @s_dynamic_insertelement_v8i8(ptr addrspace(1) %out, p
 ; SI-LABEL: s_dynamic_insertelement_v8i8:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; SI-NEXT:    s_load_dword s8, s[8:9], 0x4
 ; SI-NEXT:    s_mov_b32 s7, 0x100f000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_load_dword s8, s[8:9], 0x4
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_lshl_b32 s0, s8, 3
 ; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_lshl_b64 s[0:1], 0xff, s0
-; SI-NEXT:    s_and_b32 s9, s1, 0x5050505
+; SI-NEXT:    s_lshl_b32 s8, s8, 3
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[0:1]
-; SI-NEXT:    s_and_b32 s8, s0, 0x5050505
-; SI-NEXT:    s_or_b64 s[0:1], s[8:9], s[2:3]
+; SI-NEXT:    s_xor_b32 s1, s3, 0x5050505
+; SI-NEXT:    s_xor_b32 s0, s2, 0x5050505
+; SI-NEXT:    s_lshl_b64 s[8:9], 0xff, s8
+; SI-NEXT:    s_and_b64 s[0:1], s[0:1], s[8:9]
+; SI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; SI-NEXT:    v_mov_b32_e32 v0, s0
 ; SI-NEXT:    v_mov_b32_e32 v1, s1
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -1743,20 +1742,20 @@ define amdgpu_kernel void @s_dynamic_insertelement_v8i8(ptr addrspace(1) %out, p
 ; VI-LABEL: s_dynamic_insertelement_v8i8:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; VI-NEXT:    s_load_dword s8, s[8:9], 0x10
 ; VI-NEXT:    s_mov_b32 s7, 0x1100f000
 ; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_load_dword s8, s[8:9], 0x10
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_lshl_b32 s0, s8, 3
 ; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    s_lshl_b64 s[0:1], 0xff, s0
-; VI-NEXT:    s_and_b32 s9, s1, 0x5050505
+; VI-NEXT:    s_lshl_b32 s8, s8, 3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[0:1]
-; VI-NEXT:    s_and_b32 s8, s0, 0x5050505
-; VI-NEXT:    s_or_b64 s[0:1], s[8:9], s[2:3]
+; VI-NEXT:    s_xor_b32 s1, s3, 0x5050505
+; VI-NEXT:    s_xor_b32 s0, s2, 0x5050505
+; VI-NEXT:    s_lshl_b64 s[8:9], 0xff, s8
+; VI-NEXT:    s_and_b64 s[0:1], s[0:1], s[8:9]
+; VI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index e0dacb7a59a4..a0ad6328b0c0 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -1534,11 +1534,11 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX9-NEXT:    s_load_dword s7, s[2:3], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s2, s6, 4
-; GFX9-NEXT:    s_lshl_b32 s2, 0xffff, s2
-; GFX9-NEXT:    s_andn2_b32 s3, s7, s2
-; GFX9-NEXT:    s_and_b32 s2, s2, 0x3e703e7
-; GFX9-NEXT:    s_or_b32 s2, s2, s3
+; GFX9-NEXT:    s_lshl_b32 s3, s6, 4
+; GFX9-NEXT:    s_xor_b32 s2, s7, 0x3e703e7
+; GFX9-NEXT:    s_lshl_b32 s3, 0xffff, s3
+; GFX9-NEXT:    s_and_b32 s2, s2, s3
+; GFX9-NEXT:    s_xor_b32 s2, s2, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
@@ -1553,14 +1553,14 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(ptr addrspace(1) %out,
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
-; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_lshl_b32 s0, s4, 4
-; VI-NEXT:    s_lshl_b32 s0, 0xffff, s0
-; VI-NEXT:    s_andn2_b32 s1, s2, s0
-; VI-NEXT:    s_and_b32 s0, s0, 0x3e703e7
-; VI-NEXT:    s_or_b32 s0, s0, s1
+; VI-NEXT:    s_lshl_b32 s1, s4, 4
+; VI-NEXT:    s_xor_b32 s0, s2, 0x3e703e7
+; VI-NEXT:    s_lshl_b32 s1, 0xffff, s1
+; VI-NEXT:    s_and_b32 s0, s0, s1
+; VI-NEXT:    s_xor_b32 s0, s0, s2
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -1575,14 +1575,14 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(ptr addrspace(1) %out,
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; CI-NEXT:    s_load_dword s2, s[2:3], 0x0
-; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    s_lshl_b32 s0, s4, 4
-; CI-NEXT:    s_lshl_b32 s0, 0xffff, s0
-; CI-NEXT:    s_andn2_b32 s1, s2, s0
-; CI-NEXT:    s_and_b32 s0, s0, 0x3e703e7
-; CI-NEXT:    s_or_b32 s0, s0, s1
+; CI-NEXT:    s_lshl_b32 s1, s4, 4
+; CI-NEXT:    s_xor_b32 s0, s2, 0x3e703e7
+; CI-NEXT:    s_lshl_b32 s1, 0xffff, s1
+; CI-NEXT:    s_and_b32 s0, s0, s1
+; CI-NEXT:    s_xor_b32 s0, s0, s2
 ; CI-NEXT:    v_mov_b32_e32 v2, s0
 ; CI-NEXT:    flat_store_dword v[0:1], v2
 ; CI-NEXT:    s_endpgm
@@ -1597,12 +1597,12 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(ptr addrspace(1) %out,
 ; GFX11-NEXT:    s_load_b32 s2, s[2:3], 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_lshl_b32 s3, s4, 4
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_xor_b32 s4, s2, 0x3e703e7
 ; GFX11-NEXT:    s_lshl_b32 s3, 0xffff, s3
-; GFX11-NEXT:    s_and_not1_b32 s2, s2, s3
-; GFX11-NEXT:    s_and_b32 s3, s3, 0x3e703e7
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_or_b32 s2, s3, s2
+; GFX11-NEXT:    s_and_b32 s3, s4, s3
+; GFX11-NEXT:    s_xor_b32 s2, s3, s2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    s_endpgm
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 9df995b5a706..a18b5b5396f6 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -1664,8 +1664,8 @@ entry:
 define amdgpu_kernel void @v5i16_arg(ptr addrspace(1) nocapture %out, <5 x i16> %in) nounwind {
 ; SI-LABEL: v5i16_arg:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dword s6, s[4:5], 0xf
 ; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s6, s[4:5], 0xf
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
@@ -5191,16 +5191,16 @@ define amdgpu_kernel void @array_3xi16(i8 %arg0, [3 x i16] %arg1) {
 ; VI-NEXT:    s_addc_u32 s1, s5, 0
 ; VI-NEXT:    s_add_u32 s2, s0, 2
 ; VI-NEXT:    s_addc_u32 s3, s1, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_add_u32 s0, s4, 42
-; VI-NEXT:    s_addc_u32 s1, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
-; VI-NEXT:    flat_load_ushort v4, v[0:1]
-; VI-NEXT:    flat_load_ushort v2, v[2:3]
+; VI-NEXT:    s_add_u32 s0, s4, 42
+; VI-NEXT:    s_addc_u32 s1, s5, 0
+; VI-NEXT:    v_mov_b32_e32 v5, s1
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    flat_load_ushort v4, v[4:5]
+; VI-NEXT:    flat_load_ushort v2, v[2:3]
 ; VI-NEXT:    flat_load_ushort v0, v[0:1]
 ; VI-NEXT:    s_load_dword s0, s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
@@ -5208,10 +5208,10 @@ define amdgpu_kernel void @array_3xi16(i8 %arg0, [3 x i16] %arg1) {
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    flat_store_byte v[0:1], v1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    flat_store_short v[0:1], v2
-; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    flat_store_short v[0:1], v4
 ; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    flat_store_short v[0:1], v0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_endpgm
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
index 1a46e6f6afcd..16d32b73b9b0 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9X,GFX942 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9X,GFX950 %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9X,GFX950 %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
 
 declare float @llvm.amdgcn.cvt.f32.bf8(i32, i32)
@@ -275,17 +276,29 @@ define i32 @test_cvt_pk_bf8_f32_word0(float %x, float %y, i32 %old) {
 ; GFX9X-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9X-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: test_cvt_pk_bf8_f32_word0:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_cvt_pk_bf8_f32 v2, v0, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0:
+; GFX12-TRUE16:       ; %bb.0:
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    v_cvt_pk_bf8_f32 v2.l, v0, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_cvt_pk_bf8_f32_word0:
+; GFX12-FAKE16:       ; %bb.0:
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_cvt_pk_bf8_f32 v2, v0, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %x, float %y, i32 %old, i1 false)
   ret i32 %ret
 }
@@ -299,17 +312,29 @@ define i32 @test_cvt_pk_bf8_f32_word1(float %x, float %y, i32 %old) {
 ; GFX9X-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9X-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: test_cvt_pk_bf8_f32_word1:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_cvt_pk_bf8_f32 v2, v0, v1 op_sel:[0,0,1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: test_cvt_pk_bf8_f32_word1:
+; GFX12-TRUE16:       ; %bb.0:
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    v_cvt_pk_bf8_f32 v2.h, v0, v1 op_sel:[0,0,1]
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_cvt_pk_bf8_f32_word1:
+; GFX12-FAKE16:       ; %bb.0:
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_cvt_pk_bf8_f32 v2, v0, v1 op_sel:[0,0,1]
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %x, float %y, i32 %old, i1 true)
   ret i32 %ret
 }
@@ -322,17 +347,29 @@ define i32 @test_cvt_pk_fp8_f32_word0(float %x, float %y, i32 %old) {
 ; GFX9X-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9X-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: test_cvt_pk_fp8_f32_word0:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_cvt_pk_fp8_f32 v2, v0, v1
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: test_cvt_pk_fp8_f32_word0:
+; GFX12-TRUE16:       ; %bb.0:
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    v_cvt_pk_fp8_f32 v2.l, v0, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_cvt_pk_fp8_f32_word0:
+; GFX12-FAKE16:       ; %bb.0:
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_cvt_pk_fp8_f32 v2, v0, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %x, float %y, i32 %old, i1 false)
   ret i32 %ret
 }
@@ -346,17 +383,29 @@ define i32 @test_cvt_pk_fp8_f32_word1(float %x, float %y, i32 %old) {
 ; GFX9X-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9X-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: test_cvt_pk_fp8_f32_word1:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_cvt_pk_fp8_f32 v2, v0, v1 op_sel:[0,0,1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_mov_b32_e32 v0, v2
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1:
+; GFX12-TRUE16:       ; %bb.0:
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    v_cvt_pk_fp8_f32 v2.h, v0, v1 op_sel:[0,0,1]
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1:
+; GFX12-FAKE16:       ; %bb.0:
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_cvt_pk_fp8_f32 v2, v0, v1 op_sel:[0,0,1]
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %x, float %y, i32 %old, i1 true)
   ret i32 %ret
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index e027dda957a6..04ee0bbd1767 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -2024,6 +2024,205 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
   ret void
 }
 
+define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, ptr addrspace(1) %ptr) #0 {
+; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x0
+; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x40
+; SDAG-NEXT:    s_movk_i32 s6, 0x41
+; SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x50
+; SDAG-NEXT:    v_mov_b32_e32 v16, 0
+; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-NEXT:    v_mov_b32_e32 v0, s8
+; SDAG-NEXT:    v_mov_b32_e32 v1, s9
+; SDAG-NEXT:    v_mov_b32_e32 v2, s10
+; SDAG-NEXT:    v_mov_b32_e32 v3, s11
+; SDAG-NEXT:    v_mov_b32_e32 v4, s12
+; SDAG-NEXT:    v_mov_b32_e32 v5, s13
+; SDAG-NEXT:    v_mov_b32_e32 v6, s14
+; SDAG-NEXT:    v_mov_b32_e32 v7, s15
+; SDAG-NEXT:    v_accvgpr_write_b32 a0, s0
+; SDAG-NEXT:    v_mov_b32_e32 v8, s16
+; SDAG-NEXT:    v_mov_b32_e32 v9, s17
+; SDAG-NEXT:    v_mov_b32_e32 v10, s18
+; SDAG-NEXT:    v_mov_b32_e32 v11, s19
+; SDAG-NEXT:    v_mov_b32_e32 v12, s20
+; SDAG-NEXT:    v_mov_b32_e32 v13, s21
+; SDAG-NEXT:    v_mov_b32_e32 v14, s22
+; SDAG-NEXT:    v_mov_b32_e32 v15, s23
+; SDAG-NEXT:    v_accvgpr_write_b32 a1, s1
+; SDAG-NEXT:    v_accvgpr_write_b32 a2, s2
+; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
+; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], s6, 1.0 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT:    s_nop 7
+; SDAG-NEXT:    s_nop 3
+; SDAG-NEXT:    global_store_dwordx4 v16, a[0:3], s[4:5]
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x0
+; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x40
+; GISEL-NEXT:    v_mov_b32_e32 v16, 0x41
+; GISEL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x50
+; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
+; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
+; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[16:17]
+; GISEL-NEXT:    v_accvgpr_write_b32 a0, s0
+; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
+; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT:    v_accvgpr_write_b32 a1, s1
+; GISEL-NEXT:    v_accvgpr_write_b32 a2, s2
+; GISEL-NEXT:    v_accvgpr_write_b32 a3, s3
+; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, 1.0 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 2
+; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[4:5]
+; GISEL-NEXT:    s_endpgm
+  %result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 65, i32 1, i32 1065353216)
+  store <4 x float> %result, ptr addrspace(1) %ptr, align 16
+  ret void
+}
+
+define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, ptr addrspace(1) %ptr) #0 {
+; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x0
+; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x40
+; SDAG-NEXT:    v_mov_b32_e32 v16, 0
+; SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x50
+; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-NEXT:    v_mov_b32_e32 v0, s8
+; SDAG-NEXT:    v_mov_b32_e32 v1, s9
+; SDAG-NEXT:    v_mov_b32_e32 v2, s10
+; SDAG-NEXT:    v_mov_b32_e32 v3, s11
+; SDAG-NEXT:    v_mov_b32_e32 v4, s12
+; SDAG-NEXT:    v_mov_b32_e32 v5, s13
+; SDAG-NEXT:    v_mov_b32_e32 v6, s14
+; SDAG-NEXT:    v_mov_b32_e32 v7, s15
+; SDAG-NEXT:    v_accvgpr_write_b32 a0, s0
+; SDAG-NEXT:    v_mov_b32_e32 v8, s16
+; SDAG-NEXT:    v_mov_b32_e32 v9, s17
+; SDAG-NEXT:    v_mov_b32_e32 v10, s18
+; SDAG-NEXT:    v_mov_b32_e32 v11, s19
+; SDAG-NEXT:    v_mov_b32_e32 v12, s20
+; SDAG-NEXT:    v_mov_b32_e32 v13, s21
+; SDAG-NEXT:    v_mov_b32_e32 v14, s22
+; SDAG-NEXT:    v_mov_b32_e32 v15, s23
+; SDAG-NEXT:    v_accvgpr_write_b32 a1, s1
+; SDAG-NEXT:    v_accvgpr_write_b32 a2, s2
+; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
+; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 1.0, -2 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT:    s_nop 7
+; SDAG-NEXT:    s_nop 3
+; SDAG-NEXT:    global_store_dwordx4 v16, a[0:3], s[4:5]
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x0
+; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x40
+; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
+; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
+; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[16:17]
+; GISEL-NEXT:    v_accvgpr_write_b32 a0, s0
+; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
+; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT:    v_accvgpr_write_b32 a1, s1
+; GISEL-NEXT:    v_accvgpr_write_b32 a2, s2
+; GISEL-NEXT:    v_accvgpr_write_b32 a3, s3
+; GISEL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x50
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 1.0, -2 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[4:5]
+; GISEL-NEXT:    s_endpgm
+  %result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 1065353216, i32 1, i32 -2)
+  store <4 x float> %result, ptr addrspace(1) %ptr, align 16
+  ret void
+}
+
+define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, ptr addrspace(1) %ptr) #0 {
+; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x0
+; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x40
+; SDAG-NEXT:    v_mov_b32_e32 v16, 0
+; SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x50
+; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-NEXT:    v_mov_b32_e32 v0, s8
+; SDAG-NEXT:    v_mov_b32_e32 v1, s9
+; SDAG-NEXT:    v_mov_b32_e32 v2, s10
+; SDAG-NEXT:    v_mov_b32_e32 v3, s11
+; SDAG-NEXT:    v_mov_b32_e32 v4, s12
+; SDAG-NEXT:    v_mov_b32_e32 v5, s13
+; SDAG-NEXT:    v_mov_b32_e32 v6, s14
+; SDAG-NEXT:    v_mov_b32_e32 v7, s15
+; SDAG-NEXT:    v_accvgpr_write_b32 a0, s0
+; SDAG-NEXT:    v_mov_b32_e32 v8, s16
+; SDAG-NEXT:    v_mov_b32_e32 v9, s17
+; SDAG-NEXT:    v_mov_b32_e32 v10, s18
+; SDAG-NEXT:    v_mov_b32_e32 v11, s19
+; SDAG-NEXT:    v_mov_b32_e32 v12, s20
+; SDAG-NEXT:    v_mov_b32_e32 v13, s21
+; SDAG-NEXT:    v_mov_b32_e32 v14, s22
+; SDAG-NEXT:    v_mov_b32_e32 v15, s23
+; SDAG-NEXT:    v_accvgpr_write_b32 a1, s1
+; SDAG-NEXT:    v_accvgpr_write_b32 a2, s2
+; SDAG-NEXT:    v_accvgpr_write_b32 a3, s3
+; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 1.0, 0.15915494 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT:    s_nop 7
+; SDAG-NEXT:    s_nop 3
+; SDAG-NEXT:    global_store_dwordx4 v16, a[0:3], s[4:5]
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x0
+; GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x40
+; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
+; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
+; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[16:17]
+; GISEL-NEXT:    v_accvgpr_write_b32 a0, s0
+; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
+; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT:    v_accvgpr_write_b32 a1, s1
+; GISEL-NEXT:    v_accvgpr_write_b32 a2, s2
+; GISEL-NEXT:    v_accvgpr_write_b32 a3, s3
+; GISEL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x50
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], 1.0, 0.15915494 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 1
+; GISEL-NEXT:    global_store_dwordx4 v0, a[0:3], s[4:5]
+; GISEL-NEXT:    s_endpgm
+  %result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 1065353216, i32 1, i32 1042479491)
+  store <4 x float> %result, ptr addrspace(1) %ptr, align 16
+  ret void
+}
+
 ; This should be optimized to avoid the scale
 define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_a(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %scale0, i32 %scale1) {
 ; GCN-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_a:
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 5574313f22a4..91197f915b65 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -4252,6 +4252,192 @@ define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scale
   ret <16 x float> %result
 }
 
+define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scaleB_FP_literal(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) {
+; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scaleB_FP_literal:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT:    scratch_load_dword a15, off, s32
+; SDAG-NEXT:    s_movk_i32 s0, 0x41
+; SDAG-NEXT:    v_accvgpr_write_b32 a0, v16
+; SDAG-NEXT:    v_accvgpr_write_b32 a1, v17
+; SDAG-NEXT:    v_accvgpr_write_b32 a2, v18
+; SDAG-NEXT:    v_accvgpr_write_b32 a3, v19
+; SDAG-NEXT:    v_accvgpr_write_b32 a4, v20
+; SDAG-NEXT:    v_accvgpr_write_b32 a5, v21
+; SDAG-NEXT:    v_accvgpr_write_b32 a6, v22
+; SDAG-NEXT:    v_accvgpr_write_b32 a7, v23
+; SDAG-NEXT:    v_accvgpr_write_b32 a8, v24
+; SDAG-NEXT:    v_accvgpr_write_b32 a9, v25
+; SDAG-NEXT:    v_accvgpr_write_b32 a10, v26
+; SDAG-NEXT:    v_accvgpr_write_b32 a11, v27
+; SDAG-NEXT:    v_accvgpr_write_b32 a12, v28
+; SDAG-NEXT:    v_accvgpr_write_b32 a13, v29
+; SDAG-NEXT:    v_accvgpr_write_b32 a14, v30
+; SDAG-NEXT:    s_waitcnt vmcnt(0)
+; SDAG-NEXT:    s_nop 0
+; SDAG-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, 1.0 op_sel_hi:[1,1,0]
+; SDAG-NEXT:    s_nop 7
+; SDAG-NEXT:    s_nop 7
+; SDAG-NEXT:    s_nop 3
+; SDAG-NEXT:    v_accvgpr_read_b32 v0, a0
+; SDAG-NEXT:    v_accvgpr_read_b32 v1, a1
+; SDAG-NEXT:    v_accvgpr_read_b32 v2, a2
+; SDAG-NEXT:    v_accvgpr_read_b32 v3, a3
+; SDAG-NEXT:    v_accvgpr_read_b32 v4, a4
+; SDAG-NEXT:    v_accvgpr_read_b32 v5, a5
+; SDAG-NEXT:    v_accvgpr_read_b32 v6, a6
+; SDAG-NEXT:    v_accvgpr_read_b32 v7, a7
+; SDAG-NEXT:    v_accvgpr_read_b32 v8, a8
+; SDAG-NEXT:    v_accvgpr_read_b32 v9, a9
+; SDAG-NEXT:    v_accvgpr_read_b32 v10, a10
+; SDAG-NEXT:    v_accvgpr_read_b32 v11, a11
+; SDAG-NEXT:    v_accvgpr_read_b32 v12, a12
+; SDAG-NEXT:    v_accvgpr_read_b32 v13, a13
+; SDAG-NEXT:    v_accvgpr_read_b32 v14, a14
+; SDAG-NEXT:    v_accvgpr_read_b32 v15, a15
+; SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scaleB_FP_literal:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    scratch_load_dword a15, off, s32
+; GISEL-NEXT:    v_mov_b32_e32 v31, 0x41
+; GISEL-NEXT:    v_accvgpr_write_b32 a0, v16
+; GISEL-NEXT:    v_accvgpr_write_b32 a1, v17
+; GISEL-NEXT:    v_accvgpr_write_b32 a2, v18
+; GISEL-NEXT:    v_accvgpr_write_b32 a3, v19
+; GISEL-NEXT:    v_accvgpr_write_b32 a4, v20
+; GISEL-NEXT:    v_accvgpr_write_b32 a5, v21
+; GISEL-NEXT:    v_accvgpr_write_b32 a6, v22
+; GISEL-NEXT:    v_accvgpr_write_b32 a7, v23
+; GISEL-NEXT:    v_accvgpr_write_b32 a8, v24
+; GISEL-NEXT:    v_accvgpr_write_b32 a9, v25
+; GISEL-NEXT:    v_accvgpr_write_b32 a10, v26
+; GISEL-NEXT:    v_accvgpr_write_b32 a11, v27
+; GISEL-NEXT:    v_accvgpr_write_b32 a12, v28
+; GISEL-NEXT:    v_accvgpr_write_b32 a13, v29
+; GISEL-NEXT:    v_accvgpr_write_b32 a14, v30
+; GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], v31, 1.0 op_sel_hi:[1,1,0]
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 3
+; GISEL-NEXT:    v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT:    v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT:    v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT:    v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT:    v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT:    v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT:    v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT:    v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT:    v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT:    v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT:    v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT:    v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT:    v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT:    v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT:    v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT:    v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 2, i32 65, i32 2, i32 1065353216)
+  ret <16 x float> %result
+}
+
+define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_FP_literal__scaleB_inlineimm(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) {
+; GCN-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_FP_literal__scaleB_inlineimm:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    scratch_load_dword a15, off, s32
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v16
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v17
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v18
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v19
+; GCN-NEXT:    v_accvgpr_write_b32 a4, v20
+; GCN-NEXT:    v_accvgpr_write_b32 a5, v21
+; GCN-NEXT:    v_accvgpr_write_b32 a6, v22
+; GCN-NEXT:    v_accvgpr_write_b32 a7, v23
+; GCN-NEXT:    v_accvgpr_write_b32 a8, v24
+; GCN-NEXT:    v_accvgpr_write_b32 a9, v25
+; GCN-NEXT:    v_accvgpr_write_b32 a10, v26
+; GCN-NEXT:    v_accvgpr_write_b32 a11, v27
+; GCN-NEXT:    v_accvgpr_write_b32 a12, v28
+; GCN-NEXT:    v_accvgpr_write_b32 a13, v29
+; GCN-NEXT:    v_accvgpr_write_b32 a14, v30
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], 1.0, -2 op_sel_hi:[1,1,0]
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    v_accvgpr_read_b32 v4, a4
+; GCN-NEXT:    v_accvgpr_read_b32 v5, a5
+; GCN-NEXT:    v_accvgpr_read_b32 v6, a6
+; GCN-NEXT:    v_accvgpr_read_b32 v7, a7
+; GCN-NEXT:    v_accvgpr_read_b32 v8, a8
+; GCN-NEXT:    v_accvgpr_read_b32 v9, a9
+; GCN-NEXT:    v_accvgpr_read_b32 v10, a10
+; GCN-NEXT:    v_accvgpr_read_b32 v11, a11
+; GCN-NEXT:    v_accvgpr_read_b32 v12, a12
+; GCN-NEXT:    v_accvgpr_read_b32 v13, a13
+; GCN-NEXT:    v_accvgpr_read_b32 v14, a14
+; GCN-NEXT:    v_accvgpr_read_b32 v15, a15
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 2, i32 1065353216, i32 2, i32 -2)
+  ret <16 x float> %result
+}
+
+define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_FP_literal__scaleB_FP_literal(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) {
+; GCN-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_FP_literal__scaleB_FP_literal:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    scratch_load_dword a15, off, s32
+; GCN-NEXT:    v_accvgpr_write_b32 a0, v16
+; GCN-NEXT:    v_accvgpr_write_b32 a1, v17
+; GCN-NEXT:    v_accvgpr_write_b32 a2, v18
+; GCN-NEXT:    v_accvgpr_write_b32 a3, v19
+; GCN-NEXT:    v_accvgpr_write_b32 a4, v20
+; GCN-NEXT:    v_accvgpr_write_b32 a5, v21
+; GCN-NEXT:    v_accvgpr_write_b32 a6, v22
+; GCN-NEXT:    v_accvgpr_write_b32 a7, v23
+; GCN-NEXT:    v_accvgpr_write_b32 a8, v24
+; GCN-NEXT:    v_accvgpr_write_b32 a9, v25
+; GCN-NEXT:    v_accvgpr_write_b32 a10, v26
+; GCN-NEXT:    v_accvgpr_write_b32 a11, v27
+; GCN-NEXT:    v_accvgpr_write_b32 a12, v28
+; GCN-NEXT:    v_accvgpr_write_b32 a13, v29
+; GCN-NEXT:    v_accvgpr_write_b32 a14, v30
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], 0.15915494, 1.0 op_sel_hi:[1,1,0]
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 7
+; GCN-NEXT:    s_nop 3
+; GCN-NEXT:    v_accvgpr_read_b32 v0, a0
+; GCN-NEXT:    v_accvgpr_read_b32 v1, a1
+; GCN-NEXT:    v_accvgpr_read_b32 v2, a2
+; GCN-NEXT:    v_accvgpr_read_b32 v3, a3
+; GCN-NEXT:    v_accvgpr_read_b32 v4, a4
+; GCN-NEXT:    v_accvgpr_read_b32 v5, a5
+; GCN-NEXT:    v_accvgpr_read_b32 v6, a6
+; GCN-NEXT:    v_accvgpr_read_b32 v7, a7
+; GCN-NEXT:    v_accvgpr_read_b32 v8, a8
+; GCN-NEXT:    v_accvgpr_read_b32 v9, a9
+; GCN-NEXT:    v_accvgpr_read_b32 v10, a10
+; GCN-NEXT:    v_accvgpr_read_b32 v11, a11
+; GCN-NEXT:    v_accvgpr_read_b32 v12, a12
+; GCN-NEXT:    v_accvgpr_read_b32 v13, a13
+; GCN-NEXT:    v_accvgpr_read_b32 v14, a14
+; GCN-NEXT:    v_accvgpr_read_b32 v15, a15
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 2, i32 1042479491, i32 2, i32 1065353216)
+  ret <16 x float> %result
+}
+
 define <16 x float> @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scaleB_kimm(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) {
 ; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__scaleA_kimm__scaleB_kimm:
 ; SDAG:       ; %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index 693e0ebd0280..d05501fb0c8b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -10410,3 +10410,71 @@ define void @v_permlanex16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32
   store <8 x double> %v, ptr addrspace(1) %out
   ret void
 }
+
+define amdgpu_kernel void @v_permlanex16_convergent(ptr addrspace(1) %out, i32 %src0, i32 %pattern_lo, i32 %pattern_hi) {
+; GFX10-LABEL: v_permlanex16_convergent:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x34
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-NEXT:    v_permlanex16_b32 v1, v1, s1, s2
+; GFX10-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB142_2
+; GFX10-NEXT:  ; %bb.1: ; %t
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT:  .LBB142_2: ; %f
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_permlanex16_convergent:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x34
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s0, exec_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_permlanex16_b32 v1, v1, s1, s2
+; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX11-NEXT:    s_cbranch_execz .LBB142_2
+; GFX11-NEXT:  ; %bb.1: ; %t
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:  .LBB142_2: ; %f
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_permlanex16_convergent:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x2c
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_mov_b32 s0, exec_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_permlanex16_b32 v1, v1, s1, s2
+; GFX12-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX12-NEXT:    s_cbranch_execz .LBB142_2
+; GFX12-NEXT:  ; %bb.1: ; %t
+; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:  .LBB142_2: ; %f
+; GFX12-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %src0, i32 %src0, i32 %pattern_lo, i32 %pattern_hi, i1 false, i1 false)
+  %select = icmp eq i32 %tidx, 0
+  br i1 %select, label %t, label %f
+t:
+  store i32 %v, ptr addrspace(1) %out
+  br label %f
+f:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
index f7c37caf41ea..393d8c1a1bf2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
@@ -69,14 +69,13 @@ define amdgpu_kernel void @test_v3p3(ptr addrspace(1) %out, <3 x ptr addrspace(3
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX11-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, s0
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s1
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v0
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v3
 ; GFX11-SDAG-NEXT:    global_store_b96 v4, v[0:2], s[4:5]
 ; GFX11-SDAG-NEXT:    s_endpgm
@@ -108,14 +107,13 @@ define amdgpu_kernel void @test_v3p5(ptr addrspace(1) %out, <3 x ptr addrspace(5
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX11-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, s0
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s1
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v0
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v3
 ; GFX11-SDAG-NEXT:    global_store_b96 v4, v[0:2], s[4:5]
 ; GFX11-SDAG-NEXT:    s_endpgm
@@ -147,14 +145,13 @@ define amdgpu_kernel void @test_v3p6(ptr addrspace(1) %out, <3 x ptr addrspace(6
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
 ; GFX11-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, s0
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s1
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v2, v0
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v1, v1
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-SDAG-NEXT:    v_permlane64_b32 v0, v3
 ; GFX11-SDAG-NEXT:    global_store_b96 v4, v[0:2], s[4:5]
 ; GFX11-SDAG-NEXT:    s_endpgm
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
index c6ea12dd6165..a2be749bb3c2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
@@ -1,17 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
 
 declare half @llvm.amdgcn.rcp.f16(half %a)
 
-; GCN-LABEL: {{^}}rcp_f16
-; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
-; VI:  v_rcp_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]]
-; GFX11-TRUE16:  v_rcp_f16_e32 v[[A_F16:[0-9]+]].l, v[[A_F16]].l
-; GFX11-FAKE16:  v_rcp_f16_e32 v[[A_F16:[0-9]+]], v[[A_F16]]
-; GCN: buffer_store_short v[[R_F16]]
-; GCN: s_endpgm
 define amdgpu_kernel void @rcp_f16(
+; GCN-LABEL: rcp_f16:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s10, s6
+; GCN-NEXT:    s_mov_b32 s11, s7
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s2
+; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_rcp_f16_e32 v0, v0
+; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: rcp_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, s6
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, s7
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s3
+; GFX11-TRUE16-NEXT:    s_mov_b32 s4, s0
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s5, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_rcp_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT:    buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: rcp_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, s6
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, s7
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s3
+; GFX11-FAKE16-NEXT:    s_mov_b32 s4, s0
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s5, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT:    buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-TRUE16-LABEL: rcp_f16:
+; GFX12-TRUE16:       ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-TRUE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-TRUE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-TRUE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-TRUE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-TRUE16-NEXT:    buffer_load_u16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT:    v_rcp_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-NEXT:    buffer_store_b16 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT:    s_endpgm
+;
+; GFX12-FAKE16-LABEL: rcp_f16:
+; GFX12-FAKE16:       ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-FAKE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-FAKE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-FAKE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-FAKE16-NEXT:    buffer_load_u16 v0, off, s[8:11], null
+; GFX12-FAKE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX12-FAKE16-NEXT:    buffer_store_b16 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a) {
 entry:
@@ -20,3 +105,5 @@ entry:
   store half %r.val, ptr addrspace(1) %r
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VI: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index ead68d89f7f3..56aa82d05329 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -1000,9 +1000,9 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s65, 13
 ; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s66, 14
 ; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s67, 15
-; CHECK-SDAG-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:4
-; CHECK-SDAG-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s61, v27
+; CHECK-SDAG-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:8
+; CHECK-SDAG-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4
 ; CHECK-SDAG-NEXT:    buffer_load_dword v27, off, s[0:3], s32
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s64, v30
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s55, v21
@@ -1033,9 +1033,9 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s41, v7
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s40, v6
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s66, v0
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s67, v0
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s67, v1
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s66, v1
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s65, v27
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
@@ -1445,9 +1445,9 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s65, 13
 ; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s66, 14
 ; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s67, 15
-; CHECK-SDAG-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:4
-; CHECK-SDAG-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s61, v27
+; CHECK-SDAG-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:8
+; CHECK-SDAG-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4
 ; CHECK-SDAG-NEXT:    buffer_load_dword v27, off, s[0:3], s32
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s64, v30
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s55, v21
@@ -1478,9 +1478,9 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s41, v7
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s40, v6
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s66, v0
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s67, v0
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s67, v1
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s66, v1
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s65, v27
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index 0924e9a5c231..bf371478cc1d 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -1,17 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
 
 declare half @llvm.amdgcn.rsq.f16(half %a)
 
-; GCN-LABEL: {{^}}rsq_f16
-; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
-; VI:  v_rsq_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]]
-; GFX11-TRUE16:  v_rsq_f16_e32 v[[A_F16:[0-9]+]].l, v[[A_F16]].l
-; GFX11-FAKE16:  v_rsq_f16_e32 v[[A_F16:[0-9]+]], v[[A_F16]]
-; GCN: buffer_store_short v[[R_F16]]
-; GCN: s_endpgm
 define amdgpu_kernel void @rsq_f16(
+; GCN-LABEL: rsq_f16:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s10, s6
+; GCN-NEXT:    s_mov_b32 s11, s7
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s2
+; GCN-NEXT:    s_mov_b32 s9, s3
+; GCN-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NEXT:    s_mov_b32 s4, s0
+; GCN-NEXT:    s_mov_b32 s5, s1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_rsq_f16_e32 v0, v0
+; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: rsq_f16:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX11-TRUE16-NEXT:    s_mov_b32 s10, s6
+; GFX11-TRUE16-NEXT:    s_mov_b32 s11, s7
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_mov_b32 s8, s2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s9, s3
+; GFX11-TRUE16-NEXT:    s_mov_b32 s4, s0
+; GFX11-TRUE16-NEXT:    buffer_load_u16 v0, off, s[8:11], 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s5, s1
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_rsq_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT:    buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: rsq_f16:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_mov_b32 s6, -1
+; GFX11-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX11-FAKE16-NEXT:    s_mov_b32 s10, s6
+; GFX11-FAKE16-NEXT:    s_mov_b32 s11, s7
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_mov_b32 s8, s2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s9, s3
+; GFX11-FAKE16-NEXT:    s_mov_b32 s4, s0
+; GFX11-FAKE16-NEXT:    buffer_load_u16 v0, off, s[8:11], 0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s5, s1
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_rsq_f16_e32 v0, v0
+; GFX11-FAKE16-NEXT:    buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-TRUE16-LABEL: rsq_f16:
+; GFX12-TRUE16:       ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-TRUE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-TRUE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-TRUE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-TRUE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-TRUE16-NEXT:    buffer_load_u16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT:    v_rsq_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-NEXT:    buffer_store_b16 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT:    s_endpgm
+;
+; GFX12-FAKE16-LABEL: rsq_f16:
+; GFX12-FAKE16:       ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-FAKE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-FAKE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-FAKE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-FAKE16-NEXT:    buffer_load_u16 v0, off, s[8:11], null
+; GFX12-FAKE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_rsq_f16_e32 v0, v0
+; GFX12-FAKE16-NEXT:    buffer_store_b16 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a) {
 entry:
@@ -20,3 +105,5 @@ entry:
   store half %r.val, ptr addrspace(1) %r
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VI: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
index 952841a31018..72c4397754ce 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
@@ -2522,9 +2522,9 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-LABEL: v_maximum_v16f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x19
-; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
+; GFX10-NEXT:    s_clause 0x18
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX10-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
 ; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
 ; GFX10-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
@@ -2548,22 +2548,21 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:80
 ; GFX10-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:76
 ; GFX10-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:72
-; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:104
-; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    s_waitcnt vmcnt(23)
 ; GFX10-NEXT:    v_max_f64 v[82:83], v[0:1], v[31:32]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[31:32]
-; GFX10-NEXT:    s_waitcnt vmcnt(22)
+; GFX10-NEXT:    s_waitcnt vmcnt(21)
 ; GFX10-NEXT:    v_max_f64 v[84:85], v[2:3], v[33:34]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[33:34]
-; GFX10-NEXT:    s_clause 0x3
-; GFX10-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:120
-; GFX10-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:112
-; GFX10-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108
-; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    s_waitcnt vmcnt(19)
 ; GFX10-NEXT:    v_max_f64 v[32:33], v[4:5], v[35:36]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[35:36]
-; GFX10-NEXT:    s_clause 0x2
+; GFX10-NEXT:    s_clause 0x7
+; GFX10-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112
+; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:104
+; GFX10-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
+; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:120
+; GFX10-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:116
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX10-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:128
 ; GFX10-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:124
@@ -2593,9 +2592,6 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-NEXT:    v_max_f64 v[50:51], v[18:19], v[80:81]
 ; GFX10-NEXT:    v_max_f64 v[70:71], v[22:23], v[68:69]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s14, v[22:23], v[68:69]
-; GFX10-NEXT:    s_waitcnt vmcnt(7)
-; GFX10-NEXT:    v_max_f64 v[68:69], v[24:25], v[66:67]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v6, v34, 0, s6
 ; GFX10-NEXT:    v_cndmask_b32_e64 v7, v35, 0x7ff80000, s6
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, v48, 0, s7
@@ -2614,27 +2610,30 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-NEXT:    v_cndmask_b32_e64 v21, v65, 0x7ff80000, s13
 ; GFX10-NEXT:    v_cndmask_b32_e64 v22, v70, 0, s14
 ; GFX10-NEXT:    v_cndmask_b32_e64 v23, v71, 0x7ff80000, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v68, 0, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15
+; GFX10-NEXT:    s_waitcnt vmcnt(6)
+; GFX10-NEXT:    v_max_f64 v[68:69], v[24:25], v[66:67]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
 ; GFX10-NEXT:    s_waitcnt vmcnt(5)
-; GFX10-NEXT:    v_max_f64 v[80:81], v[28:29], v[0:1]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s17, v[28:29], v[0:1]
+; GFX10-NEXT:    v_max_f64 v[66:67], v[26:27], v[0:1]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s16, v[26:27], v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(3)
-; GFX10-NEXT:    v_max_f64 v[66:67], v[26:27], v[2:3]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s16, v[26:27], v[2:3]
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v82, 0, vcc_lo
+; GFX10-NEXT:    v_max_f64 v[80:81], v[28:29], v[2:3]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s17, v[28:29], v[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_max_f64 v[86:87], v[30:31], v[4:5]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s18, v[30:31], v[4:5]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v82, 0, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v83, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v84, 0, s4
 ; GFX10-NEXT:    v_cndmask_b32_e64 v3, v85, 0x7ff80000, s4
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, v32, 0, s5
 ; GFX10-NEXT:    v_cndmask_b32_e64 v5, v33, 0x7ff80000, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v28, v80, 0, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v29, v81, 0x7ff80000, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v24, v68, 0, s15
+; GFX10-NEXT:    v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15
 ; GFX10-NEXT:    v_cndmask_b32_e64 v26, v66, 0, s16
 ; GFX10-NEXT:    v_cndmask_b32_e64 v27, v67, 0x7ff80000, s16
+; GFX10-NEXT:    v_cndmask_b32_e64 v28, v80, 0, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v29, v81, 0x7ff80000, s17
 ; GFX10-NEXT:    v_cndmask_b32_e64 v30, v86, 0, s18
 ; GFX10-NEXT:    v_cndmask_b32_e64 v31, v87, 0x7ff80000, s18
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
index 428d0fd7012b..526988d1f36a 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
@@ -2522,9 +2522,9 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-LABEL: v_minimum_v16f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x19
-; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
+; GFX10-NEXT:    s_clause 0x18
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
+; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX10-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:16
 ; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
 ; GFX10-NEXT:    buffer_load_dword v36, off, s[0:3], s32 offset:24
@@ -2548,22 +2548,21 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-NEXT:    buffer_load_dword v81, off, s[0:3], s32 offset:80
 ; GFX10-NEXT:    buffer_load_dword v80, off, s[0:3], s32 offset:76
 ; GFX10-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:72
-; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:104
-; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    s_waitcnt vmcnt(23)
 ; GFX10-NEXT:    v_min_f64 v[82:83], v[0:1], v[31:32]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[31:32]
-; GFX10-NEXT:    s_waitcnt vmcnt(22)
+; GFX10-NEXT:    s_waitcnt vmcnt(21)
 ; GFX10-NEXT:    v_min_f64 v[84:85], v[2:3], v[33:34]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s4, v[2:3], v[33:34]
-; GFX10-NEXT:    s_clause 0x3
-; GFX10-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:120
-; GFX10-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:116
-; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:112
-; GFX10-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:108
-; GFX10-NEXT:    s_waitcnt vmcnt(24)
+; GFX10-NEXT:    s_waitcnt vmcnt(19)
 ; GFX10-NEXT:    v_min_f64 v[32:33], v[4:5], v[35:36]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s5, v[4:5], v[35:36]
-; GFX10-NEXT:    s_clause 0x2
+; GFX10-NEXT:    s_clause 0x7
+; GFX10-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:112
+; GFX10-NEXT:    buffer_load_dword v67, off, s[0:3], s32 offset:104
+; GFX10-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:108
+; GFX10-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:120
+; GFX10-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:116
 ; GFX10-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX10-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:128
 ; GFX10-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:124
@@ -2593,9 +2592,6 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-NEXT:    v_min_f64 v[50:51], v[18:19], v[80:81]
 ; GFX10-NEXT:    v_min_f64 v[70:71], v[22:23], v[68:69]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s14, v[22:23], v[68:69]
-; GFX10-NEXT:    s_waitcnt vmcnt(7)
-; GFX10-NEXT:    v_min_f64 v[68:69], v[24:25], v[66:67]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v6, v34, 0, s6
 ; GFX10-NEXT:    v_cndmask_b32_e64 v7, v35, 0x7ff80000, s6
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, v48, 0, s7
@@ -2614,27 +2610,30 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX10-NEXT:    v_cndmask_b32_e64 v21, v65, 0x7ff80000, s13
 ; GFX10-NEXT:    v_cndmask_b32_e64 v22, v70, 0, s14
 ; GFX10-NEXT:    v_cndmask_b32_e64 v23, v71, 0x7ff80000, s14
-; GFX10-NEXT:    v_cndmask_b32_e64 v24, v68, 0, s15
-; GFX10-NEXT:    v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15
+; GFX10-NEXT:    s_waitcnt vmcnt(6)
+; GFX10-NEXT:    v_min_f64 v[68:69], v[24:25], v[66:67]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
 ; GFX10-NEXT:    s_waitcnt vmcnt(5)
-; GFX10-NEXT:    v_min_f64 v[80:81], v[28:29], v[0:1]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s17, v[28:29], v[0:1]
+; GFX10-NEXT:    v_min_f64 v[66:67], v[26:27], v[0:1]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s16, v[26:27], v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(3)
-; GFX10-NEXT:    v_min_f64 v[66:67], v[26:27], v[2:3]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s16, v[26:27], v[2:3]
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v82, 0, vcc_lo
+; GFX10-NEXT:    v_min_f64 v[80:81], v[28:29], v[2:3]
+; GFX10-NEXT:    v_cmp_u_f64_e64 s17, v[28:29], v[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_min_f64 v[86:87], v[30:31], v[4:5]
 ; GFX10-NEXT:    v_cmp_u_f64_e64 s18, v[30:31], v[4:5]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v82, 0, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v83, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v84, 0, s4
 ; GFX10-NEXT:    v_cndmask_b32_e64 v3, v85, 0x7ff80000, s4
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, v32, 0, s5
 ; GFX10-NEXT:    v_cndmask_b32_e64 v5, v33, 0x7ff80000, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v28, v80, 0, s17
-; GFX10-NEXT:    v_cndmask_b32_e64 v29, v81, 0x7ff80000, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v24, v68, 0, s15
+; GFX10-NEXT:    v_cndmask_b32_e64 v25, v69, 0x7ff80000, s15
 ; GFX10-NEXT:    v_cndmask_b32_e64 v26, v66, 0, s16
 ; GFX10-NEXT:    v_cndmask_b32_e64 v27, v67, 0x7ff80000, s16
+; GFX10-NEXT:    v_cndmask_b32_e64 v28, v80, 0, s17
+; GFX10-NEXT:    v_cndmask_b32_e64 v29, v81, 0x7ff80000, s17
 ; GFX10-NEXT:    v_cndmask_b32_e64 v30, v86, 0, s18
 ; GFX10-NEXT:    v_cndmask_b32_e64 v31, v87, 0x7ff80000, s18
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index 2996a4e22a3e..8604feb3b492 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=VI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11-TRUE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11-FAKE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12-TRUE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12-FAKE16 %s
 
 declare half @llvm.sqrt.f16(half %a)
 declare <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
@@ -81,6 +83,42 @@ define amdgpu_kernel void @sqrt_f16(
 ; GFX11-FAKE16-NEXT:    v_sqrt_f16_e32 v0, v0
 ; GFX11-FAKE16-NEXT:    buffer_store_b16 v0, off, s[4:7], 0
 ; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-TRUE16-LABEL: sqrt_f16:
+; GFX12-TRUE16:       ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-TRUE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-TRUE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-TRUE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-TRUE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-TRUE16-NEXT:    buffer_load_u16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT:    v_sqrt_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-NEXT:    buffer_store_b16 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT:    s_endpgm
+;
+; GFX12-FAKE16-LABEL: sqrt_f16:
+; GFX12-FAKE16:       ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-FAKE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-FAKE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-FAKE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-FAKE16-NEXT:    buffer_load_u16 v0, off, s[8:11], null
+; GFX12-FAKE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_sqrt_f16_e32 v0, v0
+; GFX12-FAKE16-NEXT:    buffer_store_b16 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a) {
 entry:
@@ -189,6 +227,50 @@ define amdgpu_kernel void @sqrt_v2f16(
 ; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, v1
 ; GFX11-FAKE16-NEXT:    buffer_store_b32 v0, off, s[4:7], 0
 ; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-TRUE16-LABEL: sqrt_v2f16:
+; GFX12-TRUE16:       ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-TRUE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-TRUE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-TRUE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-TRUE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-TRUE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-TRUE16-NEXT:    buffer_load_b32 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-TRUE16-NEXT:    v_sqrt_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-TRUE16-NEXT:    v_sqrt_f16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.l, v0.h
+; GFX12-TRUE16-NEXT:    buffer_store_b32 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT:    s_endpgm
+;
+; GFX12-FAKE16-LABEL: sqrt_v2f16:
+; GFX12-FAKE16:       ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    s_mov_b32 s6, -1
+; GFX12-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX12-FAKE16-NEXT:    s_mov_b32 s10, s6
+; GFX12-FAKE16-NEXT:    s_mov_b32 s11, s7
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s8, s2
+; GFX12-FAKE16-NEXT:    s_mov_b32 s9, s3
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, s0
+; GFX12-FAKE16-NEXT:    buffer_load_b32 v0, off, s[8:11], null
+; GFX12-FAKE16-NEXT:    s_mov_b32 s5, s1
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-FAKE16-NEXT:    v_sqrt_f16_e32 v0, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-FAKE16-NEXT:    v_sqrt_f16_e32 v1, v1
+; GFX12-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, v1
+; GFX12-FAKE16-NEXT:    buffer_store_b32 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a) {
 entry:
@@ -197,5 +279,3 @@ entry:
   store <2 x half> %r.val, ptr addrspace(1) %r
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index cce502221329..8e312a0e195f 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -760,34 +760,34 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_clause 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v3, v8, s[0:1] offset:28
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v2, v8, s[0:1] offset:24
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v1, v8, s[0:1] offset:20
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v0, v8, s[0:1] offset:16
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v7, v8, s[0:1] offset:12
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v6, v8, s[0:1] offset:8
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v5, v8, s[0:1] offset:4
-; GFX12-TRUE16-NEXT:    global_load_d16_b16 v4, v8, s[0:1]
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v3, v8, s[0:1] offset:12
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v7, v8, s[0:1] offset:28
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v6, v8, s[0:1] offset:24
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v5, v8, s[0:1] offset:20
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v4, v8, s[0:1] offset:16
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v2, v8, s[0:1] offset:8
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v1, v8, s[0:1] offset:4
+; GFX12-TRUE16-NEXT:    global_load_d16_b16 v0, v8, s[0:1]
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v3, v8, s[0:1] offset:30
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v2, v8, s[0:1] offset:26
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v1, v8, s[0:1] offset:22
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v8, s[0:1] offset:18
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v7, v8, s[0:1] offset:14
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v6, v8, s[0:1] offset:10
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v5, v8, s[0:1] offset:6
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v4, v8, s[0:1] offset:2
-; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x4
-; GFX12-TRUE16-NEXT:    global_store_b128 v[0:1], v[0:3], off
+; GFX12-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT:    s_clause 0x1
 ; GFX12-TRUE16-NEXT:    global_store_b128 v[0:1], v[4:7], off
+; GFX12-TRUE16-NEXT:    global_store_b128 v[0:1], v[0:3], off
 ; GFX12-TRUE16-NEXT:    s_endpgm
 ;
 ; GFX12-FAKE16-LABEL: constant_load_v16i16_align2:
@@ -796,34 +796,34 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
 ; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_clause 0x7
-; GFX12-FAKE16-NEXT:    global_load_u16 v3, v8, s[0:1] offset:28
-; GFX12-FAKE16-NEXT:    global_load_u16 v2, v8, s[0:1] offset:24
-; GFX12-FAKE16-NEXT:    global_load_u16 v1, v8, s[0:1] offset:20
-; GFX12-FAKE16-NEXT:    global_load_u16 v0, v8, s[0:1] offset:16
-; GFX12-FAKE16-NEXT:    global_load_u16 v7, v8, s[0:1] offset:12
-; GFX12-FAKE16-NEXT:    global_load_u16 v6, v8, s[0:1] offset:8
-; GFX12-FAKE16-NEXT:    global_load_u16 v5, v8, s[0:1] offset:4
-; GFX12-FAKE16-NEXT:    global_load_u16 v4, v8, s[0:1]
+; GFX12-FAKE16-NEXT:    global_load_u16 v3, v8, s[0:1] offset:12
+; GFX12-FAKE16-NEXT:    global_load_u16 v7, v8, s[0:1] offset:28
+; GFX12-FAKE16-NEXT:    global_load_u16 v6, v8, s[0:1] offset:24
+; GFX12-FAKE16-NEXT:    global_load_u16 v5, v8, s[0:1] offset:20
+; GFX12-FAKE16-NEXT:    global_load_u16 v4, v8, s[0:1] offset:16
+; GFX12-FAKE16-NEXT:    global_load_u16 v2, v8, s[0:1] offset:8
+; GFX12-FAKE16-NEXT:    global_load_u16 v1, v8, s[0:1] offset:4
+; GFX12-FAKE16-NEXT:    global_load_u16 v0, v8, s[0:1]
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v3, v8, s[0:1] offset:30
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v2, v8, s[0:1] offset:26
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v1, v8, s[0:1] offset:22
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v0, v8, s[0:1] offset:18
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v7, v8, s[0:1] offset:14
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v6, v8, s[0:1] offset:10
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v5, v8, s[0:1] offset:6
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v4, v8, s[0:1] offset:2
-; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x4
-; GFX12-FAKE16-NEXT:    global_store_b128 v[0:1], v[0:3], off
+; GFX12-FAKE16-NEXT:    global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
 ; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    s_clause 0x1
 ; GFX12-FAKE16-NEXT:    global_store_b128 v[0:1], v[4:7], off
+; GFX12-FAKE16-NEXT:    global_store_b128 v[0:1], v[0:3], off
 ; GFX12-FAKE16-NEXT:    s_endpgm
 entry:
   %ld =  load <16 x i16>, ptr addrspace(4) %ptr0, align 2
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
index a6ce512164b8..8a3cc57e0857 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
@@ -7,13 +7,13 @@
 define amdgpu_vs void @test(ptr addrspace(8) inreg %arg1, ptr addrspace(3) %arg2) {
 ; CHECK-LABEL: test:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 12, v0
-; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 8, v0
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 12, v0
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 8, v0
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
 ; CHECK-NEXT:    s_mov_b32 m0, -1
-; CHECK-NEXT:    ds_read_b32 v2, v1
+; CHECK-NEXT:    ds_read_b32 v3, v1
+; CHECK-NEXT:    ds_read_b32 v2, v2
 ; CHECK-NEXT:    ds_read_b32 v1, v4
-; CHECK-NEXT:    ds_read_b32 v3, v3
 ; CHECK-NEXT:    ds_read_b32 v0, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    exp mrt0 off, off, off, off
@@ -69,36 +69,36 @@ define amdgpu_vs void @test_3(i32 inreg %arg1, i32 inreg %arg2, ptr addrspace(8)
 ; CHECK-NEXT:    v_add_i32_e32 v0, vcc, 12, v1
 ; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 8, v1
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 4, v1
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 20, v1
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, 16, v1
-; CHECK-NEXT:    v_mov_b32_e32 v9, s0
-; CHECK-NEXT:    v_add_i32_e32 v10, vcc, 12, v2
-; CHECK-NEXT:    v_add_i32_e32 v11, vcc, 8, v2
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, 20, v1
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, 16, v1
+; CHECK-NEXT:    v_mov_b32_e32 v10, s0
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, 12, v2
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, 8, v2
 ; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_read_b32 v6, v0
 ; CHECK-NEXT:    ds_read_b32 v5, v3
 ; CHECK-NEXT:    ds_read_b32 v4, v4
-; CHECK-NEXT:    ds_read_b32 v8, v6
-; CHECK-NEXT:    ds_read_b32 v7, v7
-; CHECK-NEXT:    ds_read_b32 v6, v0
+; CHECK-NEXT:    ds_read_b32 v8, v7
+; CHECK-NEXT:    ds_read_b32 v7, v9
 ; CHECK-NEXT:    ds_read_b32 v3, v1
 ; CHECK-NEXT:    v_add_i32_e32 v0, vcc, 4, v2
 ; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 20, v2
-; CHECK-NEXT:    v_add_i32_e32 v12, vcc, 16, v2
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, 16, v2
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    tbuffer_store_format_xyzw v[3:6], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:264 glc slc
-; CHECK-NEXT:    tbuffer_store_format_xy v[7:8], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:280 glc slc
+; CHECK-NEXT:    tbuffer_store_format_xyzw v[3:6], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:264 glc slc
+; CHECK-NEXT:    tbuffer_store_format_xy v[7:8], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:280 glc slc
 ; CHECK-NEXT:    s_waitcnt expcnt(1)
-; CHECK-NEXT:    ds_read_b32 v4, v11
+; CHECK-NEXT:    ds_read_b32 v5, v11
+; CHECK-NEXT:    ds_read_b32 v4, v12
 ; CHECK-NEXT:    ds_read_b32 v3, v0
 ; CHECK-NEXT:    ds_read_b32 v1, v1
-; CHECK-NEXT:    ds_read_b32 v0, v12
-; CHECK-NEXT:    ds_read_b32 v5, v10
+; CHECK-NEXT:    ds_read_b32 v0, v9
 ; CHECK-NEXT:    ds_read_b32 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
 ; CHECK-NEXT:    exp mrt0 off, off, off, off
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    tbuffer_store_format_xyzw v[2:5], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:240 glc slc
-; CHECK-NEXT:    tbuffer_store_format_xy v[0:1], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:256 glc slc
+; CHECK-NEXT:    tbuffer_store_format_xyzw v[2:5], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:240 glc slc
+; CHECK-NEXT:    tbuffer_store_format_xy v[0:1], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:256 glc slc
 ; CHECK-NEXT:    s_endpgm
   %load1 = load <6 x float>, ptr addrspace(3) %arg5, align 4
   %vec11 = shufflevector <6 x float> %load1, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.128.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.128.ll
index 10dca76cc389..d634e40f1d79 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.128.ll
@@ -95,51 +95,51 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
-; GFX7-NEXT:    ds_read_u8 v1, v0 offset:6
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:4
-; GFX7-NEXT:    ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v2, v0 offset:6
+; GFX7-NEXT:    ds_read_u8 v3, v0 offset:4
+; GFX7-NEXT:    ds_read_u8 v4, v0 offset:2
 ; GFX7-NEXT:    ds_read_u8 v5, v0
 ; GFX7-NEXT:    ds_read_u8 v6, v0 offset:3
 ; GFX7-NEXT:    ds_read_u8 v7, v0 offset:5
 ; GFX7-NEXT:    ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(3)
-; GFX7-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v5
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v6
-; GFX7-NEXT:    v_or_b32_e32 v3, v5, v3
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_or_b32_e32 v4, v3, v4
+; GFX7-NEXT:    v_or_b32_e32 v4, v5, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT:    v_or_b32_e32 v4, v4, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v7
-; GFX7-NEXT:    v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v7
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v8
-; GFX7-NEXT:    ds_read_u8 v5, v0 offset:15
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:14
+; GFX7-NEXT:    ds_read_u8 v5, v0 offset:9
+; GFX7-NEXT:    ds_read_u8 v6, v0 offset:11
 ; GFX7-NEXT:    ds_read_u8 v7, v0 offset:13
-; GFX7-NEXT:    ds_read_u8 v8, v0 offset:12
-; GFX7-NEXT:    ds_read_u8 v9, v0 offset:11
-; GFX7-NEXT:    ds_read_u8 v10, v0 offset:10
-; GFX7-NEXT:    ds_read_u8 v11, v0 offset:9
+; GFX7-NEXT:    ds_read_u8 v8, v0 offset:15
+; GFX7-NEXT:    ds_read_u8 v9, v0 offset:14
+; GFX7-NEXT:    ds_read_u8 v10, v0 offset:12
+; GFX7-NEXT:    ds_read_u8 v11, v0 offset:10
 ; GFX7-NEXT:    ds_read_u8 v0, v0 offset:8
-; GFX7-NEXT:    v_or_b32_e32 v1, v3, v1
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v11
+; GFX7-NEXT:    v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v5
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v9
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v10
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v6
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v11
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v5
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v8
 ; GFX7-NEXT:    v_or_b32_e32 v2, v2, v0
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v7
-; GFX7-NEXT:    v_or_b32_e32 v3, v3, v6
-; GFX7-NEXT:    v_or_b32_e32 v0, v0, v8
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v9
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v10
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v4
@@ -331,21 +331,21 @@ define <4 x i32> @load_lds_v4i32_align2(ptr addrspace(3) %ptr) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    ds_read_u16 v1, v0 offset:2
 ; GFX7-NEXT:    ds_read_u16 v3, v0 offset:12
 ; GFX7-NEXT:    ds_read_u16 v2, v0 offset:8
-; GFX7-NEXT:    ds_read_u16 v1, v0 offset:4
-; GFX7-NEXT:    ds_read_u16 v4, v0 offset:2
+; GFX7-NEXT:    ds_read_u16 v4, v0 offset:4
 ; GFX7-NEXT:    ds_read_u16 v5, v0
 ; GFX7-NEXT:    ds_read_u16 v6, v0 offset:6
 ; GFX7-NEXT:    ds_read_u16 v7, v0 offset:10
 ; GFX7-NEXT:    ds_read_u16 v8, v0 offset:14
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v4
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(3)
 ; GFX7-NEXT:    v_or_b32_e32 v0, v0, v5
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
-; GFX7-NEXT:    v_or_b32_e32 v1, v4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v6
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v4
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
 ; GFX7-NEXT:    v_or_b32_e32 v2, v4, v2
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.96.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.96.ll
index 2da3fce72072..b917b48b90e6 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.96.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/load-local.96.ll
@@ -86,41 +86,41 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
-; GFX7-NEXT:    ds_read_u8 v1, v0 offset:6
-; GFX7-NEXT:    ds_read_u8 v2, v0 offset:4
+; GFX7-NEXT:    ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT:    ds_read_u8 v2, v0 offset:6
+; GFX7-NEXT:    ds_read_u8 v4, v0 offset:4
 ; GFX7-NEXT:    ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT:    ds_read_u8 v4, v0 offset:1
 ; GFX7-NEXT:    ds_read_u8 v5, v0
 ; GFX7-NEXT:    ds_read_u8 v6, v0 offset:3
 ; GFX7-NEXT:    ds_read_u8 v7, v0 offset:5
 ; GFX7-NEXT:    ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX7-NEXT:    s_waitcnt lgkmcnt(7)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(3)
-; GFX7-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v5
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v6
 ; GFX7-NEXT:    v_or_b32_e32 v3, v5, v3
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v7
-; GFX7-NEXT:    ds_read_u8 v5, v0 offset:11
-; GFX7-NEXT:    ds_read_u8 v6, v0 offset:10
-; GFX7-NEXT:    ds_read_u8 v7, v0 offset:9
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v7
+; GFX7-NEXT:    ds_read_u8 v5, v0 offset:9
+; GFX7-NEXT:    ds_read_u8 v6, v0 offset:11
+; GFX7-NEXT:    ds_read_u8 v7, v0 offset:10
 ; GFX7-NEXT:    ds_read_u8 v0, v0 offset:8
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v2
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v4
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(4)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v8
-; GFX7-NEXT:    v_or_b32_e32 v1, v4, v1
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v7
+; GFX7-NEXT:    v_or_b32_e32 v2, v4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(3)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v5
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_or_b32_e32 v0, v2, v0
-; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v5
-; GFX7-NEXT:    v_or_b32_e32 v2, v2, v6
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v6
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v7
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_or_b32_e32 v2, v2, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
@@ -274,19 +274,19 @@ define <3 x i32> @load_lds_v3i32_align2(ptr addrspace(3) %ptr) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    ds_read_u16 v1, v0 offset:2
 ; GFX7-NEXT:    ds_read_u16 v2, v0 offset:8
-; GFX7-NEXT:    ds_read_u16 v1, v0 offset:4
-; GFX7-NEXT:    ds_read_u16 v3, v0 offset:2
+; GFX7-NEXT:    ds_read_u16 v3, v0 offset:4
 ; GFX7-NEXT:    ds_read_u16 v4, v0
 ; GFX7-NEXT:    ds_read_u16 v5, v0 offset:6
 ; GFX7-NEXT:    ds_read_u16 v6, v0 offset:10
-; GFX7-NEXT:    s_waitcnt lgkmcnt(3)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(5)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(2)
 ; GFX7-NEXT:    v_or_b32_e32 v0, v0, v4
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX7-NEXT:    v_or_b32_e32 v1, v3, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX7-NEXT:    v_or_b32_e32 v2, v3, v2
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index bb642155cd0a..117f359be0c3 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -1945,16 +1945,14 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 {
 ; CI-LABEL: lshr_mad_i64_vec:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v6, v3
-; CI-NEXT:    v_mov_b32_e32 v3, v1
-; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    s_mov_b32 s4, 0xffff1c18
-; CI-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v3, s4, v[0:1]
-; CI-NEXT:    v_mov_b32_e32 v3, v1
+; CI-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[0:1]
 ; CI-NEXT:    s_mov_b32 s4, 0xffff1118
-; CI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s4, v[2:3]
+; CI-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v3, s4, v[2:3]
+; CI-NEXT:    v_sub_i32_e32 v1, vcc, v5, v1
+; CI-NEXT:    v_sub_i32_e32 v3, vcc, v7, v3
 ; CI-NEXT:    v_mov_b32_e32 v0, v4
-; CI-NEXT:    v_mov_b32_e32 v1, v5
+; CI-NEXT:    v_mov_b32_e32 v2, v6
 ; CI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SI-LABEL: lshr_mad_i64_vec:
@@ -1977,44 +1975,28 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 {
 ; GFX9-LABEL: lshr_mad_i64_vec:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v6, v3
-; GFX9-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_mov_b32 s4, 0xffff1c18
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v3, s4, v[0:1]
-; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[0:1]
 ; GFX9-NEXT:    s_mov_b32 s4, 0xffff1118
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s4, v[2:3]
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v3, s4, v[2:3]
+; GFX9-NEXT:    v_sub_u32_e32 v1, v5, v1
+; GFX9-NEXT:    v_sub_u32_e32 v3, v7, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
+; GFX9-NEXT:    v_mov_b32_e32 v2, v6
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX1100-LABEL: lshr_mad_i64_vec:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_mov_b32_e32 v8, v3
-; GFX1100-NEXT:    v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v1, 0
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_mad_u64_u32 v[4:5], null, 0xffff1c18, v6, v[0:1]
-; GFX1100-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, v4
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-NEXT:    v_mad_u64_u32 v[6:7], null, 0xffff1118, v8, v[2:3]
-; GFX1100-NEXT:    v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v2, v6
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-NEXT:    v_mov_b32_e32 v3, v7
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX1150-LABEL: lshr_mad_i64_vec:
-; GFX1150:       ; %bb.0:
-; GFX1150-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1150-NEXT:    v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v5, v1
-; GFX1150-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1150-NEXT:    v_mov_b32_e32 v3, v1
-; GFX1150-NEXT:    v_mad_u64_u32 v[0:1], null, 0xffff1c18, v5, v[0:1]
-; GFX1150-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1150-NEXT:    v_mad_u64_u32 v[2:3], null, 0xffff1118, v4, v[2:3]
-; GFX1150-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-LABEL: lshr_mad_i64_vec:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_mad_u64_u32 v[4:5], null, 0xffff1c18, v1, v[0:1]
+; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, 0xffff1118, v3, v[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_sub_nc_u32_e32 v1, v5, v1
+; GFX11-NEXT:    v_mov_b32_e32 v0, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_sub_nc_u32_e32 v3, v7, v3
+; GFX11-NEXT:    v_mov_b32_e32 v2, v6
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: lshr_mad_i64_vec:
 ; GFX12:       ; %bb.0:
@@ -2023,13 +2005,14 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 {
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v5, v1
-; GFX12-NEXT:    v_mov_b32_e32 v1, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_mov_b32_e32 v3, v1
-; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, 0xffff1c18, v5, v[0:1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT:    v_mad_co_u64_u32 v[2:3], null, 0xffff1118, v4, v[2:3]
+; GFX12-NEXT:    v_mad_co_u64_u32 v[4:5], null, 0xffff1c18, v1, v[0:1]
+; GFX12-NEXT:    v_mad_co_u64_u32 v[6:7], null, 0xffff1118, v3, v[2:3]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_sub_nc_u32_e32 v1, v5, v1
+; GFX12-NEXT:    v_mov_b32_e32 v0, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_sub_nc_u32_e32 v3, v7, v3
+; GFX12-NEXT:    v_mov_b32_e32 v2, v6
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %lsh = lshr <2 x i64> %arg0, <i64 32, i64 32>
   %mul = mul <2 x i64> %lsh, <i64 s0xffffffffffff1c18, i64 s0xffffffffffff1118>
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/max.i16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/max.i16.ll
index 1857eaba0a2a..1e246465ab1e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/max.i16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/max.i16.ll
@@ -139,17 +139,17 @@ define amdgpu_kernel void @v_test_imax_sge_v3i16(ptr addrspace(1) %out, ptr addr
 ;
 ; GFX9-LABEL: v_test_imax_sge_v3i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v3, v0, s[6:7]
-; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    global_load_short_d16 v2, v0, s[2:3] offset:4
 ; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    global_load_dword v3, v0, s[6:7]
 ; GFX9-NEXT:    global_load_dword v4, v0, s[2:3]
+; GFX9-NEXT:    ; kill: killed $sgpr2_sgpr3
 ; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    global_load_short_d16 v1, v0, s[6:7] offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index 850d1394f320..e58958bc4821 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -34,23 +34,13 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_mul_f32_e32 v2, 1.0, v2
-; GFX8-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v2
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
-; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_maximumnum_bf16:
@@ -66,22 +56,13 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX900-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX900-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v3, v3, v2, s4
-; GFX900-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_maximumnum_bf16:
@@ -100,9 +81,6 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v2, v2, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
@@ -128,21 +106,13 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_maximumnum_bf16:
@@ -151,39 +121,31 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_bf16:
@@ -202,26 +164,14 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_maximumnum_bf16:
@@ -234,45 +184,37 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_bf16:
@@ -298,29 +240,17 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call bfloat @llvm.maximumnum.bf16(bfloat %x, bfloat %y)
   ret bfloat %result
@@ -533,24 +463,14 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -560,23 +480,14 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -594,23 +505,14 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -620,21 +522,13 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v2, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -649,15 +543,12 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v3, v0, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
@@ -681,12 +572,8 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
@@ -721,37 +608,21 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v2, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v7, v4, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -765,66 +636,52 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v5, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16:
@@ -837,60 +694,39 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -908,71 +744,56 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v5, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16:
@@ -990,69 +811,48 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -1403,24 +1203,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -1430,23 +1220,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -1456,22 +1237,13 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -1490,23 +1262,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -1516,22 +1279,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -1541,21 +1296,13 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v4, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -1570,15 +1317,12 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v5, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -1602,9 +1346,6 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
@@ -1628,12 +1369,8 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
@@ -1648,180 +1385,125 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX10-LABEL: v_maximumnum_v3bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v5, v5
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_sdwa v0, v0, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_sdwa v10, v0, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v11
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX10-NEXT:    v_or_b32_e32 v11, 0x400000, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v10, v7, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v10, v10, v7, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v12, v8, 16, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v4, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_maximumnum_v3bf16:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v5.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v7, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v9, v11
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -1830,88 +1512,59 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v9
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX11-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_maximumnum_v3bf16:
@@ -1922,104 +1575,77 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v5.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v7, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v9, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v8, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -2032,98 +1658,67 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v9
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v7, v7, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX12-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
@@ -2131,7 +1726,7 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x bfloat> @llvm.maximumnum.v3bf16(<3 x bfloat> %x, <3 x bfloat> %y)
   ret <3 x bfloat> %result
@@ -2585,24 +2180,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -2614,23 +2199,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT:    v_mul_f32_e32 v7, 1.0, v7
-; GFX8-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v7
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s4, v8
-; GFX8-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -2640,23 +2216,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2666,22 +2233,13 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
@@ -2702,23 +2260,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -2730,22 +2279,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX900-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX900-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX900-NEXT:    v_add3_u32 v8, v8, v7, s4
-; GFX900-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -2755,22 +2296,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2780,21 +2313,13 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v5, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v4, v1, s4
@@ -2815,11 +2340,8 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -2843,9 +2365,6 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
@@ -2869,9 +2388,6 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
@@ -2885,7 +2401,7 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v1, v4, v1, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
@@ -2895,12 +2411,8 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    v_perm_b32 v1, v4, v1, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
@@ -2942,78 +2454,46 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v8, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX10-NEXT:    v_bfe_u32 v14, v8, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v7, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v13
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v12
-; GFX10-NEXT:    v_add3_u32 v12, v14, v8, 0x7fff
-; GFX10-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v11
-; GFX10-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v13, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_bfe_u32 v15, v9, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX10-NEXT:    v_add3_u32 v12, v12, v11, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_or_b32_e32 v16, 0x400000, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_bfe_u32 v12, v6, 16, 1
-; GFX10-NEXT:    v_add3_u32 v14, v15, v9, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_add3_u32 v12, v12, v6, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v12, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
@@ -3032,232 +2512,162 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v10, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v9, v13
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v9, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v11, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v4, v1, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3275,123 +2685,94 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v10, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v9, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v9, v13
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v11, v11, v11
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v11, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX12-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16:
@@ -3401,126 +2782,80 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v11
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v13
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
@@ -3531,10 +2866,10 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
@@ -4133,24 +3468,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v7
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v8, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v7, v6, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX8-NEXT:    v_mul_f32_e32 v8, 1.0, v8
-; GFX8-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v8
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, s4, v9
-; GFX8-NEXT:    v_or_b32_e32 v10, 0x400000, v8
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v10, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v8
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
-; GFX8-NEXT:    v_and_b32_e32 v7, 0xffff0000, v8
+; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX8-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
@@ -4162,23 +3487,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v8, v7, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX8-NEXT:    v_mul_f32_e32 v9, 1.0, v9
-; GFX8-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v9
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s4, v10
-; GFX8-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
-; GFX8-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
 ; GFX8-NEXT:    v_and_b32_e32 v8, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v3
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
@@ -4190,23 +3506,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX8-NEXT:    v_mul_f32_e32 v10, 1.0, v10
-; GFX8-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v10
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s4, v11
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
@@ -4216,23 +3523,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX8-NEXT:    v_mul_f32_e32 v9, 1.0, v9
-; GFX8-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v9
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s4, v10
-; GFX8-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v9
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
@@ -4242,23 +3540,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v9, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v5
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, s4, v9
-; GFX8-NEXT:    v_or_b32_e32 v10, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
@@ -4268,22 +3557,13 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v9, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v8
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
@@ -4307,23 +3587,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v7
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v8, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v7, v6, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX900-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX900-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v9, v9, v8, s4
-; GFX900-NEXT:    v_or_b32_e32 v10, 0x400000, v8
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v9, v10, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v8
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
-; GFX900-NEXT:    v_and_b32_e32 v7, 0xffff0000, v8
+; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX900-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
@@ -4335,22 +3606,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v8, v7, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX900-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX900-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX900-NEXT:    v_add3_u32 v10, v10, v9, s4
-; GFX900-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
-; GFX900-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
 ; GFX900-NEXT:    v_and_b32_e32 v8, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v3
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
@@ -4362,22 +3625,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX900-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX900-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX900-NEXT:    v_add3_u32 v11, v11, v10, s4
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
@@ -4387,22 +3642,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX900-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX900-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX900-NEXT:    v_add3_u32 v10, v10, v9, s4
-; GFX900-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v9
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
@@ -4412,22 +3659,14 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v9, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v9, v9, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v10, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
@@ -4437,21 +3676,13 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v9, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v8, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v7, v1, s4
@@ -4475,11 +3706,8 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v7
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v8, v9
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v7, v6, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX950-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v8, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
@@ -4504,9 +3732,6 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v8, v7, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX950-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
@@ -4530,9 +3755,6 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX950-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
@@ -4556,9 +3778,6 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX950-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
@@ -4572,7 +3791,7 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v2, v6, v2, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
@@ -4582,9 +3801,6 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v9, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
@@ -4598,7 +3814,7 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v1, v7, v1, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
@@ -4608,16 +3824,11 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v4
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    v_perm_b32 v1, v7, v1, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX950-NEXT:    v_perm_b32 v2, v6, v2, s0
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
@@ -4636,156 +3847,108 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v1
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
 ; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v0
+; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_sdwa v12, v2, v7, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v0
+; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v10, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v12, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v15, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v13
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v9
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v10, v7, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v14, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v16, v17
-; GFX10-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_bfe_u32 v16, v10, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v11, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v16, v16, v10, 0x7fff
-; GFX10-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v8
-; GFX10-NEXT:    v_bfe_u32 v17, v13, 16, 1
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v13
-; GFX10-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v8, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v16, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
-; GFX10-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v17, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v10
-; GFX10-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT:    v_bfe_u32 v18, v14, 16, 1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v10, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v14, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v12, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v9, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v11, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v8, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_add3_u32 v9, v18, v14, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v10, 0x400000, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v10, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v11, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v0
-; GFX10-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v9
-; GFX10-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v10
-; GFX10-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_bfe_u32 v16, v12, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v11, v14, v13, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v13
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_bfe_u32 v17, v10, 16, 1
-; GFX10-NEXT:    v_add3_u32 v13, v16, v12, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v14, vcc_lo
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_add3_u32 v16, v17, v10, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v17, 0x400000, v10
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v16, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v15, v8, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v4, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    v_perm_b32 v1, v6, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v2, v7, v2, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4793,165 +3956,126 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v3
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v1.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v12, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.h, v6.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v14, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v0
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s3
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v18, v19
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v8.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v9.l, v7.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v20, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.h, v10.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v14, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v14
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v14, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v16, v16, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v14, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s5, v13, v18
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s4
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v16, v20, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v19, v22, s6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s2
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v13, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v12, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v9.l, v8.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v6.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v13.l, v8.l, s3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v14, v16
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v12.h, v6.l, s0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v7.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v11.l, v10.l, s5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v9.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v12.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v7, v7, v7 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v7
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v15, v14
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v14, v6, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v9
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v16, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v9, v9, v12 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v14, 16, 1
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v1.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v8, v12, v14, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v14
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v10, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v12, v15, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v10, v14, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v10, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v7.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v14, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v4.l, v1.l, s1
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v0.h, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v1.h, v5.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v11
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v3.h, s2
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -4960,176 +4084,116 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_and_b32 v8, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v10 :: v_dual_lshlrev_b32 v11, 16, v7
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v6, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v7, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v10, v9 :: v_dual_lshlrev_b32 v13, 16, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v10, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v11 :: v_dual_max_f32 v9, v9, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v13, v9, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v13, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v14, v11 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v18, v12, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v16, v18, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v16, v15, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v15, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v15, v12 :: v_dual_lshlrev_b32 v14, 16, v9
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v14
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v10, v6 :: v_dual_lshlrev_b32 v13, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v15, v8 :: v_dual_lshlrev_b32 v17, 16, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v18, v6 :: v_dual_lshlrev_b32 v9, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v8 :: v_dual_lshlrev_b32 v15, 16, v7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v13, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v10, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v10, v13, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v15, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v8, v10, v13, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v16, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v11, 16, v2
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v5, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v8
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v4, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v3, v0 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v13, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v16, v10, 16, 1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v0 :: v_dual_lshlrev_b32 v12, 16, v1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v17, v9, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v16, v10, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_add3_u32 v16, v17, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v17, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v13, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v16, v17, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v15, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v4, v1 :: v_dual_lshlrev_b32 v11, 16, v3
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v9, v2 :: v_dual_lshlrev_b32 v13, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v3, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v13, v0 :: v_dual_and_b32 v5, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v4, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v4, 16, v10
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v3, 16, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v7, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v7, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v8, v1, 0x5040100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5141,191 +4205,143 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v5
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v3
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v1.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v12, v12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.h, v6.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v6.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v9.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v14, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v0
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v16
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s3
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v18, v19
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.h, v6.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v8.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v9.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v9.l, v7.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v20, v20
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v6.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s0
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.h, v10.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v14, v14, v14
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v14, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v14
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v12, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_add3_u32 v19, v19, v14, 0x7fff
-; GFX12-TRUE16-NEXT:    v_add3_u32 v16, v16, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v14, v14
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s5, v13, v18
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s4
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v16, v20, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v19, v22, s6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v9.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v12.h, v6.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v7.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v11.l, v10.l, s5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v15
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v9.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v16
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v12.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s2
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v13, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v9.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v8
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v12, v13
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v11.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v7, v7, v7 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v9.l, v8.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v6.l, s2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v13
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v15, v14
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v14, v6, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v9
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v16, v7, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v13.l, v8.l, s3
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v13.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v4.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v14, v16
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v9, v9, v12 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v14, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v1.h, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v8, v12, v14, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v14
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v7, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v10, v6, 16, 1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v10, v10, v6, 0x7fff
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v7, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v12, v15, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v1.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v7
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v10, v14, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v10, v9
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v7.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v14, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v4.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v11
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v0.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v1.h, v5.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v3.h, s2
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -5338,211 +4354,147 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v1
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v3
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_and_b32 v8, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v6, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v10 :: v_dual_lshlrev_b32 v11, 16, v7
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v10, v9 :: v_dual_lshlrev_b32 v13, 16, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v10, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v11 :: v_dual_max_num_f32 v9, v9, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v13, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v13, v9, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v17
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v15, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v14, v11 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v18, v12, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v16, v18, v12, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v16, v15, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v12
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v15, v12 :: v_dual_lshlrev_b32 v14, 16, v9
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v14
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v10, v6 :: v_dual_lshlrev_b32 v13, 16, v11
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v15, v8 :: v_dual_lshlrev_b32 v17, 16, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v18, v6 :: v_dual_lshlrev_b32 v9, 16, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v8 :: v_dual_lshlrev_b32 v15, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v10, v13, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v13
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v10, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v15, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v10, v13, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v16, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v16, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v5, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v0
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v11, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v8
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v4, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v3, v0 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v13, v12, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v16, v10, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v17, v9, 16, 1
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v16, v10, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_add3_u32 v16, v17, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v17, 0x400000, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v13, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v0 :: v_dual_lshlrev_b32 v12, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v16, v17, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v15, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v4, v1 :: v_dual_lshlrev_b32 v11, 16, v3
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v9, v2 :: v_dual_lshlrev_b32 v13, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v11
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v3, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v13, v0 :: v_dual_and_b32 v5, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v4, 0xffff0000, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v4, 16, v10
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v3, 16, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v7, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
+; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v7, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v8, v1, 0x5040100
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x bfloat> @llvm.maximumnum.v6bf16(<6 x bfloat> %x, <6 x bfloat> %y)
@@ -5618,24 +4570,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX8-NEXT:    v_mul_f32_e32 v10, 1.0, v10
-; GFX8-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v10
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s4, v11
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v6
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
@@ -5647,23 +4589,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v10
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v10, v9, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX8-NEXT:    v_mul_f32_e32 v11, 1.0, v11
-; GFX8-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v11
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, s4, v12
-; GFX8-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v10, vcc
-; GFX8-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX8-NEXT:    v_and_b32_e32 v10, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v5
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
@@ -5675,23 +4608,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v10, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX8-NEXT:    v_mul_f32_e32 v12, 1.0, v12
-; GFX8-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v12
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, s4, v13
-; GFX8-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
-; GFX8-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX8-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
@@ -5703,23 +4627,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v12, v11, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX8-NEXT:    v_mul_f32_e32 v13, 1.0, v13
-; GFX8-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, v14, v13
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, s4, v14
-; GFX8-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc
-; GFX8-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
@@ -5729,23 +4644,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX8-NEXT:    v_mul_f32_e32 v12, 1.0, v12
-; GFX8-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v12
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, s4, v13
-; GFX8-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
+; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
@@ -5755,23 +4661,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT:    v_mul_f32_e32 v7, 1.0, v7
-; GFX8-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v7
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, s4, v12
-; GFX8-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v12, v13, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
@@ -5781,23 +4678,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v12, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
@@ -5807,22 +4695,13 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v11
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
@@ -5848,23 +4727,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX900-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX900-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v11, v11, v10, s4
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v6
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
@@ -5876,22 +4746,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v10
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v10, v9, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX900-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX900-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX900-NEXT:    v_add3_u32 v12, v12, v11, s4
-; GFX900-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v9, v10, vcc
-; GFX900-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX900-NEXT:    v_and_b32_e32 v10, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v5
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
@@ -5903,22 +4765,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v11, v10, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX900-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX900-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX900-NEXT:    v_add3_u32 v13, v13, v12, s4
-; GFX900-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
-; GFX900-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX900-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
@@ -5930,22 +4784,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v12, v11, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX900-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX900-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX900-NEXT:    v_add3_u32 v14, v14, v13, s4
-; GFX900-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc
-; GFX900-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
@@ -5955,22 +4801,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX900-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX900-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX900-NEXT:    v_add3_u32 v13, v13, v12, s4
-; GFX900-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX900-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
+; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
@@ -5980,22 +4818,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX900-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX900-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX900-NEXT:    v_add3_u32 v12, v12, v7, s4
-; GFX900-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v12, v13, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
@@ -6005,22 +4835,14 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v5, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v12, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
@@ -6030,21 +4852,13 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v11, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v10, v1, s4
@@ -6071,11 +4885,9 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
 ; GFX950-NEXT:    v_and_b32_e32 v14, 0xffff0000, v4
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX950-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX950-NEXT:    s_nop 1
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX950-NEXT:    s_nop 1
@@ -6098,9 +4910,6 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v10, v9, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX950-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
@@ -6125,9 +4934,6 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v11, v10, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v12, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
@@ -6151,9 +4957,6 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v14
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v12, v11, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX950-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
@@ -6177,9 +4980,6 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v12
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v12, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
@@ -6193,7 +4993,7 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v3, v8, v3, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
@@ -6203,9 +5003,6 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
@@ -6219,7 +5016,7 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v2, v9, v2, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
@@ -6229,9 +5026,6 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v5, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
@@ -6245,7 +5039,7 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v1, v10, v1, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
@@ -6255,16 +5049,11 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    v_perm_b32 v1, v10, v1, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX950-NEXT:    v_perm_b32 v2, v9, v2, s0
-; GFX950-NEXT:    v_perm_b32 v3, v8, v3, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
@@ -6280,213 +5069,149 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
 ; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
 ; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v6
-; GFX10-NEXT:    v_and_b32_e32 v16, 0xffff0000, v1
-; GFX10-NEXT:    v_and_b32_e32 v17, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
 ; GFX10-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
+; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v14, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v1
-; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v5
-; GFX10-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v10, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v22, v11, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v13
-; GFX10-NEXT:    v_add3_u32 v22, v22, v11, 0x7fff
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v19
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v13, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v22, v19, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v12, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
+; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v9, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_max_f32_e32 v15, v16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v11
-; GFX10-NEXT:    v_bfe_u32 v19, v15, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v22, 0x400000, v15
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v21, v20, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX10-NEXT:    v_add3_u32 v19, v19, v15, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v20, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v19, v22, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
-; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v19, v20
-; GFX10-NEXT:    v_bfe_u32 v19, v9, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v18, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v15
+; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v11, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
-; GFX10-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX10-NEXT:    v_add3_u32 v13, v19, v9, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v13, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_bfe_u32 v13, v17, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v14, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_add3_u32 v13, v13, v17, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v11, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v17, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v14, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v16, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v15, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v12
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v13, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v7, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v12, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v19, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_max_f32_e32 v11, v15, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v14, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
-; GFX10-NEXT:    v_bfe_u32 v16, v11, 16, 1
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v11
-; GFX10-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_add3_u32 v15, v16, v11, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v6, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v4
-; GFX10-NEXT:    v_bfe_u32 v17, v14, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_add3_u32 v17, v17, v14, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v21, v20
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v14
-; GFX10-NEXT:    v_max_f32_e32 v16, v16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v19, v12, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_bfe_u32 v19, v16, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX10-NEXT:    v_or_b32_e32 v20, 0x400000, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v16
-; GFX10-NEXT:    v_bfe_u32 v17, v13, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v19, v16, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX10-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v18, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v17, v20, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v5, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v16
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    v_perm_b32 v2, v10, v2, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_perm_b32 v0, v12, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    v_perm_b32 v0, v11, v0, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX10-NEXT:    v_perm_b32 v1, v9, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v3, v8, v3, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6494,458 +5219,321 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v4
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v15, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v7.h, v8.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v14, v14
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v17
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v11.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v12, v13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v0.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v9.l, v8.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v5.h, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v9.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v15, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v5.h, v12.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v13, v18
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v11.l, v10.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v4.h, v13.l, s2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v21, v16, v16
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v17, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v10.l, v8.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v11.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v11.l, v9.l, s3
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v15.l, v12.l, s2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v21, v21
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v19, v19, v22, s2
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v18, v20
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v16, v16, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v19.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v19
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v13.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v9.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v20, v18, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v18, v16, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v11.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v17.h, v10.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v14.l, v13.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v19.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v18, v18, v16, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, 0x400000, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v18, v19, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v15.l, v9.l, s4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v8.h, v11.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v4.h, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v16, v16
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v11, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v15.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.h, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v14.l, v12.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v15.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v15.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v12, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v13, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v8.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v8.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v0
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v9, v18, s2
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v16, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v17.h, v8.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v10.h, v13.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v4
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v12, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v13, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v15, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v4.l, v0.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v9, 16, 1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v6.l, v2.l, s3
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v14, v13
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v0.h, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v16, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v15, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v14, v14, v13, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v12, v12, v12 :: v_dual_max_f32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v14, v16, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v9, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_add3_u32 v14, v14, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v0.h, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v14, v15, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v16, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v12.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v14
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v12.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v11.h, v1.h, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v6
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.l, v3.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v4.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v1.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v8bf16:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v8 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v14, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v12, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v14
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v13
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v19
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v13, v10 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v11, v11, v11 :: v_dual_and_b32 v16, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v22, v11, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v11
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v18
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v12 :: v_dual_and_b32 v15, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v22, v11, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v22, v19 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v15, v16, v16
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v19, v15, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v21, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX11-FAKE16-NEXT:    v_add3_u32 v19, v19, v15, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v12, v8 :: v_dual_and_b32 v13, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v12, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v20, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v19, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v18
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v17, v8 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v19, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v18, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v15, 16, v11
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v15
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v10 :: v_dual_and_b32 v15, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v17, 16, v17
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v19, v9, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v19, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v19, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v13, v17, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_lshlrev_b32 v13, 16, v12
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v13, v17, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v14 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v3
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v7, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v15
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v9 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_and_b32 v13, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v16, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v15 :: v_dual_lshlrev_b32 v16, 16, v13
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v19, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v12
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v16
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v12 :: v_dual_lshlrev_b32 v16, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v14, v18 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v12 :: v_dual_lshlrev_b32 v12, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v12
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v11, v15, v15 :: v_dual_lshlrev_b32 v18, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v5 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v4
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v16, v11, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v11
-; GFX11-FAKE16-NEXT:    v_add3_u32 v15, v16, v11, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v16
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v17, v14, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v5, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_add3_u32 v17, v17, v14, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v18 :: v_dual_lshlrev_b32 v16, 16, v16
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v21, v20
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v14
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v16, v16, v16 :: v_dual_and_b32 v13, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v4, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v19, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v6 :: v_dual_lshlrev_b32 v13, 16, v15
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v19, v16, 16, 1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v12, v3 :: v_dual_lshlrev_b32 v16, 16, v2
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v20, 0x400000, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v16
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v17, v13, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v7, v19, v16, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX11-FAKE16-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v18, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v7 :: v_dual_lshlrev_b32 v14, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v16, v14
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v7, 16, v12
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v17, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v13, 16, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v14, v2 :: v_dual_lshlrev_b32 v15, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v5, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v16
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_and_b32 v5, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_and_b32 v11, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v10, v2, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v13
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v11, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v12, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v9, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v8, v3, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -6958,260 +5546,200 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v7
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v6
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v5
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v15, v15
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v7.h, v8.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v14, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v17
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v11.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v12, v13
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v0.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v9.l, v8.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v5.h, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v9.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v15, v17
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v5.h, v12.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v13, v18
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v14.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v10.l, v8.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v11.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v11.l, v10.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v10.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v11.l, v9.l, s3
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v4.h, v13.l, s2
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v21, v16, v16
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v17, v18
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v21, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v15.l, v12.l, s2
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v17, v17, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v21, v21
-; GFX12-TRUE16-NEXT:    v_add3_u32 v19, v19, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v9.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v19, v19, v22, s2
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v18, v20
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v17
-; GFX12-TRUE16-NEXT:    v_add3_u32 v20, v23, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v16, v16, v16
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v19.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v19
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v13.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v15.l, v9.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v8.h, v11.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v4.h, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v16, v16
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v11, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v20, v18, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v18, v16, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v11.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v17.h, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v14.l, v13.l, s2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v19.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v18, v18, v16, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v19, 0x400000, v16
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v18, v19, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v15.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.h, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v14.l, v12.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v7
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v15.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v9.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v16
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v13, v15
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v12, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v8.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v14
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v8.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v0
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v9, v18, s2
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v16, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v17.h, v8.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v10.h, v13.l, s3
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v4
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v12, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v13, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v4.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v9, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v16, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v15, v14
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v13
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v15, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v14, v14, v13, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v4.l, v0.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v12, v12, v12 :: v_dual_max_num_f32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v15
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v14, v16, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v6.l, v2.l, s3
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v12, 16, 1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v7, v9, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_add3_u32 v14, v14, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v7, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v0.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v14, v13
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v14, v15, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v16, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v0.h, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v12.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v14
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.l, v3.h, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v12.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v11.h, v1.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v4.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v1.h, s3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v1, v3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7222,256 +5750,170 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v5
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v7
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v8 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v14, v13, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v12, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v14
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v13
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v19
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v13, v10 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v11, v11, v11 :: v_dual_and_b32 v16, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v22, v11, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v11
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v18
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v12 :: v_dual_and_b32 v15, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v22, v11, 0x7fff
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v22, v19 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v15, v16, v16
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v19, v15, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v21, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX12-FAKE16-NEXT:    v_add3_u32 v19, v19, v15, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v12, v8 :: v_dual_and_b32 v13, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v21
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v15, 16, v11
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v15
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v12, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v10 :: v_dual_and_b32 v15, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v20, v16, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_lshlrev_b32 v13, 16, v12
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v19, v22, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v18
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v17, v8 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v19, v20
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v18, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v17, 16, v17
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v17, v17, v17
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v13, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v19, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v19, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v19, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v13, v17, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v13, v17, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v9 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v7
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v14 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v3
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_and_b32 v13, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v6
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v16, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v7, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v15 :: v_dual_lshlrev_b32 v16, 16, v13
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v19, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v12
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v14, v18 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v11, v15, v15 :: v_dual_lshlrev_b32 v18, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v14
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v16
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v5
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v12 :: v_dual_lshlrev_b32 v16, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v5 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v14, v14, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v12 :: v_dual_lshlrev_b32 v12, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v4
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v16, v11, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v11
-; GFX12-FAKE16-NEXT:    v_add3_u32 v15, v16, v11, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v16
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v17, v14, 16, 1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v5, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    v_add3_u32 v17, v17, v14, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v18 :: v_dual_lshlrev_b32 v16, 16, v16
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v21, v20
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v14
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v16, v16, v16 :: v_dual_and_b32 v13, 0xffff0000, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v6 :: v_dual_lshlrev_b32 v13, 16, v15
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v4, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v12, v3 :: v_dual_lshlrev_b32 v16, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v19, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v19, v16, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v20, 0x400000, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v0
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v16
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v17, v13, 16, 1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v7, v19, v16, 0x7fff
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX12-FAKE16-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v7 :: v_dual_lshlrev_b32 v14, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v18, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v16, v14
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v7, 16, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v17, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v13, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v4
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v14, v2 :: v_dual_lshlrev_b32 v15, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v5, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v17, v16
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
@@ -7480,25 +5922,26 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_and_b32 v5, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_and_b32 v11, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v10, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v11, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v12, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v9, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v8, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x bfloat> @llvm.maximumnum.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y)
@@ -7624,24 +6067,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v18, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v18, v16, v17, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX8-NEXT:    v_mul_f32_e32 v18, 1.0, v18
-; GFX8-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v18
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, s4, v19
-; GFX8-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX8-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX8-NEXT:    v_and_b32_e32 v17, 0xffff0000, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v16, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
@@ -7653,23 +6086,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v19, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v17, v18, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX8-NEXT:    v_mul_f32_e32 v19, 1.0, v19
-; GFX8-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v19
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, s4, v20
-; GFX8-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
-; GFX8-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX8-NEXT:    v_and_b32_e32 v18, 0xffff0000, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v20, v17, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v5
@@ -7681,23 +6105,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v18
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v20, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v20, v18, v19, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX8-NEXT:    v_mul_f32_e32 v20, 1.0, v20
-; GFX8-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v20
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, s4, v21
-; GFX8-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX8-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX8-NEXT:    v_and_b32_e32 v19, 0xffff0000, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v21, v18, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
@@ -7709,23 +6124,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v21, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v21, v19, v20, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX8-NEXT:    v_mul_f32_e32 v21, 1.0, v21
-; GFX8-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v21
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, s4, v22
-; GFX8-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX8-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX8-NEXT:    v_and_b32_e32 v20, 0xffff0000, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v22, v19, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v11
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
@@ -7737,23 +6143,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v20
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v22, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v20, v21, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX8-NEXT:    v_mul_f32_e32 v22, 1.0, v22
-; GFX8-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v22
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, s4, v23
-; GFX8-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX8-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX8-NEXT:    v_and_b32_e32 v21, 0xffff0000, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v23, v20, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
@@ -7765,23 +6162,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v23, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v23, v21, v22, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX8-NEXT:    v_mul_f32_e32 v23, 1.0, v23
-; GFX8-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, v24, v23
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, s4, v24
-; GFX8-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX8-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX8-NEXT:    v_and_b32_e32 v22, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v24, v21, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v9
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
@@ -7793,23 +6181,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v24, v22, v23, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX8-NEXT:    v_mul_f32_e32 v24, 1.0, v24
-; GFX8-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, v25, v24
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, s4, v25
-; GFX8-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX8-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_and_b32_e32 v23, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v25, v22, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
@@ -7821,23 +6200,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v25, v23, v24, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX8-NEXT:    v_mul_f32_e32 v25, 1.0, v25
-; GFX8-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, v26, v25
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, s4, v26
-; GFX8-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v26, v24, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX8-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
@@ -7847,23 +6217,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v24, v15, v7, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX8-NEXT:    v_mul_f32_e32 v24, 1.0, v24
-; GFX8-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, v25, v24
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, s4, v25
-; GFX8-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX8-NEXT:    v_and_b32_e32 v15, 0xffff0000, v24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v15
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
@@ -7873,23 +6234,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX8-NEXT:    v_mul_f32_e32 v15, 1.0, v15
-; GFX8-NEXT:    v_bfe_u32 v24, v15, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, v24, v15
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, s4, v24
-; GFX8-NEXT:    v_or_b32_e32 v25, 0x400000, v15
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v24, v25, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX8-NEXT:    v_and_b32_e32 v14, 0xffff0000, v15
+; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v14
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
@@ -7899,23 +6251,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v5
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v15, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v13, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX8-NEXT:    v_mul_f32_e32 v14, 1.0, v14
-; GFX8-NEXT:    v_bfe_u32 v15, v14, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v15, vcc, v15, v14
-; GFX8-NEXT:    v_add_u32_e32 v15, vcc, s4, v15
-; GFX8-NEXT:    v_or_b32_e32 v24, 0x400000, v14
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v15, v24, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX8-NEXT:    v_and_b32_e32 v13, 0xffff0000, v14
+; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v13
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
@@ -7925,23 +6268,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v14, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v12, v4, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX8-NEXT:    v_mul_f32_e32 v13, 1.0, v13
-; GFX8-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, v14, v13
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, s4, v14
-; GFX8-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX8-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
@@ -7951,23 +6285,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX8-NEXT:    v_mul_f32_e32 v12, 1.0, v12
-; GFX8-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v12
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, s4, v13
-; GFX8-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
@@ -7977,23 +6302,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX8-NEXT:    v_mul_f32_e32 v11, 1.0, v11
-; GFX8-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v11
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, s4, v12
-; GFX8-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
@@ -8003,23 +6319,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v9, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX8-NEXT:    v_mul_f32_e32 v10, 1.0, v10
-; GFX8-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v10
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s4, v11
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
@@ -8029,22 +6336,13 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v8, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX8-NEXT:    v_mul_f32_e32 v9, 1.0, v9
-; GFX8-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v9
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s4, v10
-; GFX8-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v23
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v22
@@ -8078,23 +6376,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v18, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v18, v16, v17, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX900-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX900-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v19, v19, v18, s4
-; GFX900-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v17
-; GFX900-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX900-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_and_b32_e32 v17, 0xffff0000, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v16, v19, v16, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
@@ -8106,22 +6395,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v19, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v19, v17, v18, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX900-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX900-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX900-NEXT:    v_add3_u32 v20, v20, v19, s4
-; GFX900-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v18
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
-; GFX900-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX900-NEXT:    v_and_b32_e32 v18, 0xffff0000, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v17, v20, v17, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v5
@@ -8133,22 +6414,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v18
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v20, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v20, v18, v19, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX900-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX900-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX900-NEXT:    v_add3_u32 v21, v21, v20, s4
-; GFX900-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v19
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX900-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX900-NEXT:    v_and_b32_e32 v19, 0xffff0000, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v21, v18, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
@@ -8160,22 +6433,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v21, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v21, v19, v20, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX900-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX900-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX900-NEXT:    v_add3_u32 v22, v22, v21, s4
-; GFX900-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v20
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX900-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX900-NEXT:    v_and_b32_e32 v20, 0xffff0000, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v22, v19, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v11
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
@@ -8187,22 +6452,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v20
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v22, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v22, v20, v21, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX900-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX900-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX900-NEXT:    v_add3_u32 v23, v23, v22, s4
-; GFX900-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v21
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX900-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX900-NEXT:    v_and_b32_e32 v21, 0xffff0000, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v23, v20, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
@@ -8214,22 +6471,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v23, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v21, v22, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX900-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX900-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX900-NEXT:    v_add3_u32 v24, v24, v23, s4
-; GFX900-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v22
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX900-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX900-NEXT:    v_and_b32_e32 v22, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v24, v21, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v9
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
@@ -8241,22 +6490,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v22, v23, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX900-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX900-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX900-NEXT:    v_add3_u32 v25, v25, v24, s4
-; GFX900-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v23
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX900-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX900-NEXT:    v_and_b32_e32 v23, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v25, v22, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
@@ -8268,22 +6509,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v25, v23, v24, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX900-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX900-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX900-NEXT:    v_add3_u32 v26, v26, v25, s4
-; GFX900-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX900-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v26, v24, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX900-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
@@ -8293,22 +6526,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v15, v7, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX900-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX900-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX900-NEXT:    v_add3_u32 v25, v25, v24, s4
-; GFX900-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX900-NEXT:    v_and_b32_e32 v15, 0xffff0000, v24
+; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v15
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
@@ -8318,22 +6543,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX900-NEXT:    v_max_f32_e32 v15, v15, v15
-; GFX900-NEXT:    v_bfe_u32 v24, v15, 16, 1
-; GFX900-NEXT:    v_add3_u32 v24, v24, v15, s4
-; GFX900-NEXT:    v_or_b32_e32 v25, 0x400000, v15
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v24, v25, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX900-NEXT:    v_and_b32_e32 v14, 0xffff0000, v15
+; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v14
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
@@ -8343,22 +6560,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v5
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v15, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v13, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX900-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX900-NEXT:    v_bfe_u32 v15, v14, 16, 1
-; GFX900-NEXT:    v_add3_u32 v15, v15, v14, s4
-; GFX900-NEXT:    v_or_b32_e32 v24, 0x400000, v14
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
-; GFX900-NEXT:    v_cndmask_b32_e32 v14, v15, v24, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX900-NEXT:    v_and_b32_e32 v13, 0xffff0000, v14
+; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v13
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
@@ -8368,22 +6577,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v14, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v12, v4, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX900-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX900-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX900-NEXT:    v_add3_u32 v14, v14, v13, s4
-; GFX900-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX900-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
@@ -8393,22 +6594,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v11, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX900-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX900-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX900-NEXT:    v_add3_u32 v13, v13, v12, s4
-; GFX900-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX900-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
@@ -8418,22 +6611,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX900-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX900-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX900-NEXT:    v_add3_u32 v12, v12, v11, s4
-; GFX900-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX900-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
@@ -8443,22 +6628,14 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v9, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX900-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX900-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX900-NEXT:    v_add3_u32 v11, v11, v10, s4
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
@@ -8468,21 +6645,13 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v8, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX900-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX900-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX900-NEXT:    v_add3_u32 v10, v10, v9, s4
-; GFX900-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX900-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v23, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v22, v1, s4
@@ -8513,9 +6682,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
 ; GFX950-NEXT:    v_and_b32_e32 v22, 0xffff0000, v12
 ; GFX950-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX950-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v18, v18, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v16
 ; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v11
 ; GFX950-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
@@ -8527,7 +6693,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX950-NEXT:    v_and_b32_e32 v17, 0xffff0000, v6
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v18, 16, v14
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
@@ -8542,9 +6708,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v20, 16, v5
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v19, v18, v17, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX950-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v19, v19, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v17
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
@@ -8569,9 +6732,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v21, 16, v4
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v20, v19, v18, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX950-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v20, v20, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v18
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
@@ -8596,9 +6756,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v21, v20, v19, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX950-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v21, v21, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v19
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
@@ -8623,9 +6780,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v22, v21, v20, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX950-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v22, v22, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v20
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
@@ -8650,9 +6804,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v23, v22, v21, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v23, v23, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v21
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
@@ -8677,9 +6828,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v24, v23, v22, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX950-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v24, v24, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v22
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
@@ -8703,9 +6851,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v26
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v25, v24, v23, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX950-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v25, v25, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v23
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
@@ -8729,9 +6874,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v24
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v24, v15, v7, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX950-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v24, v24, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
@@ -8745,7 +6887,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v7, v16, v7, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
@@ -8755,9 +6897,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v15
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX950-NEXT:    v_max_f32_e32 v15, v15, v15
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
@@ -8771,7 +6910,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v14, 16, v13
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v6, v17, v6, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v5
@@ -8781,9 +6920,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v15, v14
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v13, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX950-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v14, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
@@ -8797,7 +6933,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v5, v18, v5, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
@@ -8807,9 +6943,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v14, v13
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v12, v4, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX950-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
@@ -8823,7 +6956,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v4, v19, v4, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
@@ -8833,9 +6966,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v13, v12
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v11, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v12, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
@@ -8849,7 +6979,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v3, v20, v3, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
@@ -8859,9 +6989,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v12, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX950-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
@@ -8875,7 +7002,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v9
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v2, v21, v2, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
@@ -8885,9 +7012,6 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v11, v10
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v9, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX950-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
@@ -8901,7 +7025,7 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v8
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v1, v22, v1, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
@@ -8911,25 +7035,17 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v10, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v8, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX950-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    v_perm_b32 v1, v22, v1, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX950-NEXT:    v_perm_b32 v2, v21, v2, s0
-; GFX950-NEXT:    v_perm_b32 v3, v20, v3, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX950-NEXT:    v_perm_b32 v4, v19, v4, s0
-; GFX950-NEXT:    v_perm_b32 v5, v18, v5, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX950-NEXT:    v_perm_b32 v0, v23, v0, s0
-; GFX950-NEXT:    v_perm_b32 v6, v17, v6, s0
-; GFX950-NEXT:    v_perm_b32 v7, v16, v7, s0
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximumnum_v16bf16:
@@ -8943,881 +7059,634 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v13
+; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v16, v18, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v16, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v8
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v19
 ; GFX10-NEXT:    v_and_b32_e32 v19, 0xffff0000, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_max_f32_e32 v18, v18, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v19, v21, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_bfe_u32 v23, v18, 16, 1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
-; GFX10-NEXT:    v_add3_u32 v23, v23, v18, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v17, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v21, v22
-; GFX10-NEXT:    v_or_b32_e32 v22, 0x400000, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v19, v20, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v18, 0xffff0000, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, v19, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v23, v22, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v13
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v18
-; GFX10-NEXT:    v_and_b32_e32 v18, 0xffff0000, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v25, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v22, v21, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v4
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v18, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v24, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v27
-; GFX10-NEXT:    v_add3_u32 v17, v26, v21, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX10-NEXT:    v_and_b32_e32 v27, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v22, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v29, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_and_b32_e32 v17, 0xffff0000, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v24, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v28, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v18, v20, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
 ; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v28
-; GFX10-NEXT:    v_add3_u32 v20, v26, v21, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v24, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_and_b32_e32 v27, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v30, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX10-NEXT:    v_max_f32_e32 v19, v21, v21
-; GFX10-NEXT:    v_and_b32_e32 v20, 0xffff0000, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v29, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_bfe_u32 v27, v19, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v18, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v24, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v21, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v26
+; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v21, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v29
-; GFX10-NEXT:    v_add3_u32 v23, v27, v19, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v27, 0x400000, v19
-; GFX10-NEXT:    v_and_b32_e32 v29, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v21, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v23, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v28
-; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v31, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_and_b32_e32 v19, 0xffff0000, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_max_f32_e32 v22, v23, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v30, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX10-NEXT:    v_bfe_u32 v28, v22, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v20, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v30
-; GFX10-NEXT:    v_add3_u32 v25, v28, v22, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v22
-; GFX10-NEXT:    v_and_b32_e32 v30, 0xffff0000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v23, v27, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v19, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
+; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v11
+; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v25, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v29
-; GFX10-NEXT:    v_and_b32_e32 v29, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v32, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v0
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v24, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
-; GFX10-NEXT:    v_max_f32_e32 v24, v25, v25
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v31, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26
-; GFX10-NEXT:    v_bfe_u32 v29, v24, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v33, 0x400000, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v20, v26, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_and_b32_e32 v25, 0xffff0000, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v22
+; GFX10-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v19, v20, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_and_b32_e32 v25, 0xffff0000, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v24, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v20, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_cndmask_b32_e32 v25, v28, v27, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v31
-; GFX10-NEXT:    v_and_b32_e32 v30, 0xffff0000, v0
-; GFX10-NEXT:    v_add3_u32 v26, v29, v24, 0x7fff
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v25, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v32, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v26, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_max_f32_e32 v26, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v33, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v31, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
-; GFX10-NEXT:    v_bfe_u32 v22, v26, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v34, 0x400000, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
-; GFX10-NEXT:    v_add3_u32 v22, v22, v26, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v21, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v31, v32
-; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v29, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v7, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v34, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v33
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v33, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v20, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v24, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v27, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v27, v23, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v26, v23, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v24, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v28
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v27, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX10-NEXT:    v_max_f32_e32 v27, v7, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v21, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v28
-; GFX10-NEXT:    v_bfe_u32 v23, v27, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v32, v31
-; GFX10-NEXT:    v_add3_u32 v23, v23, v27, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v15, v33, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v25, vcc_lo
-; GFX10-NEXT:    v_or_b32_e32 v25, 0x400000, v27
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v14
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v6, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_bfe_u32 v22, v24, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v26, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v5
-; GFX10-NEXT:    v_add3_u32 v22, v22, v24, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v21, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v5, v13, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
+; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v27, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v7
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v31, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v14, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v26, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX10-NEXT:    v_max_f32_e32 v23, v24, v24
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v21, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v13, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v21, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v26, v25, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v8
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v25, v25, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_bfe_u32 v21, v23, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v23
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_add3_u32 v21, v21, v23, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_bfe_u32 v23, v24, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v28, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v12
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_add3_u32 v23, v23, v24, 0x7fff
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
-; GFX10-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v26, v15, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v25, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v12, v4, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v15, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v22, v25, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v11
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX10-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_or_b32_e32 v30, 0x400000, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v25, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
-; GFX10-NEXT:    v_bfe_u32 v26, v24, 16, 1
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
-; GFX10-NEXT:    v_add3_u32 v26, v26, v24, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v25, v27, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v12, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v27, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v11, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v26, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v22, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v25, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_bfe_u32 v23, v21, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v11, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_add3_u32 v23, v23, v21, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v24
+; GFX10-NEXT:    v_perm_b32 v5, v18, v5, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
-; GFX10-NEXT:    v_and_b32_e32 v12, 0xffff0000, v24
-; GFX10-NEXT:    v_or_b32_e32 v24, 0x400000, v21
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
+; GFX10-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v10
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v10, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v23, v24, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v25
-; GFX10-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
-; GFX10-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v23
-; GFX10-NEXT:    v_max_f32_e32 v24, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_bfe_u32 v26, v24, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_bfe_u32 v28, v25, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v11, v23, v23
-; GFX10-NEXT:    v_add3_u32 v23, v26, v24, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_add3_u32 v24, v28, v25, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v29, v11, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v25
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_add3_u32 v28, v29, v11, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v29, 0x400000, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v24, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v28, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v9, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v8, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v9, 0xffff0000, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_perm_b32 v1, v6, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_perm_b32 v6, v17, v14, 0x5040100
-; GFX10-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_perm_b32 v5, v18, v13, 0x5040100
-; GFX10-NEXT:    v_perm_b32 v2, v7, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v7, v16, v15, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v4, v19, v4, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_maximumnum_v16bf16:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v7 :: v_dual_mov_b32 v17, v6
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v20, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v18, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v14
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v18
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v14
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v16
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v18.h, v15.h, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, s0
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v18
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v19, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v16.l, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v19.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v4.h, v12.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v23
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v23, v23
-; GFX11-TRUE16-NEXT:    v_add3_u32 v27, v27, v23, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v25, v26
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v7.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v26, v27, v28, s3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v22.l, v19.l, s2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
-; GFX11-TRUE16-NEXT:    v_add3_u32 v25, v25, v24, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v26.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v26
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v6.l, v5.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v22
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v24, v25, v27, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.h, v21.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v16.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v20.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v6.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v26.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.h, v16.l, s0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v23, v23
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-TRUE16-NEXT:    v_add3_u32 v16, v27, v23, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v23
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v25, v26
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v24.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v23, v16, v27, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v21.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v3.h, v11.h, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v23.h, v19.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v16.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v11.h, v16.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v23
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v28, v19, v19
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v2.h, v10.h, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v28, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v25
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v10.h, v19.l, s0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v28
-; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v28, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v7.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v22.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v24, v24, v25 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v23.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v28, v28
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v26, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v26, v21, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v22.l, v19.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v26, 16, 1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v9.h, v23.l, s0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v26
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v23.l
-; GFX11-TRUE16-NEXT:    v_add3_u32 v25, v25, v26, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v21.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v24.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.h, v5.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v26, v28, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v30
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v25, v25, v27, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v29
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v16.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v26, v26
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v28
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v25.h, v16.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v24, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v21.l, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v28, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v19.l, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v24.h, v19.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v19, v26, v26
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v19, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v18
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, 0x400000, v19
-; GFX11-TRUE16-NEXT:    v_add3_u32 v22, v22, v19, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v19, v19
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v27
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v22, v22, v30, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v23.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v18.l, v18.l, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v29
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v22.h, v23.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v18.h, v25.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v18.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.h, v21.l, s2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.h, v24.h, v16.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v14
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v27, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v24, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v17.l, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_add3_u32 v17, v27, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v22.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.l, v18.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v14.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v17, v23, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.l, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v24, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v22.l, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v20.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v19.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v11
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v20
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v3.h, v11.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v26, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v19.l, v22.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v11.h, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v20.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v5.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v5.l, s0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v23, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v13.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v14.l, v16.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v22, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v13.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v20, v22, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v17.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v17, v21, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v20.h, v18.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v21.l, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v20.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v22.l, v20.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v13.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_add3_u32 v15, v22, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v15, v15, v21, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v12.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16.l
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v17, v17, v17 :: v_dual_lshlrev_b32 v22, 16, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v15.h, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v12.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v21, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v12.l, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v17, v21, v22, s0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v9
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v9.h, v23.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.l, v21.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v27, v25
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v7.l, v6.l, s2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v21, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v5.l, v23.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v23.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v21.l, v23.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v15
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v20.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v23, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.l, v16.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v7.l, v19.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v16.l
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v13.l, s0
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v13, v21, v21 :: v_dual_lshlrev_b32 v22, 16, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v17.l, v17.l, v14.l, s1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT:    v_add3_u32 v23, v23, v13, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v13, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v9
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v20, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v23, v24, s3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v11.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v20.h, v4.l, s3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v19.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v24, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v17.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v16.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v21, v15
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v4.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v10.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v7.l, v16.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.l, v17.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v17.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v16.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v16.l, v17.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v14.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v14.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v17
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v18, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v12.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v16.l, v4.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v13.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v10.l, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v12.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v15
-; GFX11-TRUE16-NEXT:    v_add3_u32 v15, v23, v13, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v13
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v22, v21
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v21, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v8.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v17, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v15.h, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v9
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v17, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v12.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v22, v17
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v9, v12, s0
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v12, v13, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v13, v3, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v3
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v11, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.h, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v13, v3, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v11, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v13, v21, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v22, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v8.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v21
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v12
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v11.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v9.h, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v18.l, v15.h, v1.h, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v20.h, v0.h, s3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v21
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v17
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v2.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v8.l, v1.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v10.l, v0.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v13.l, v2.h, s3
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v14
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, v19 :: v_dual_mov_b32 v3, v18
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v16
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v19
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v16bf16:
@@ -9827,449 +7696,304 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v15
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v13
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v17, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v16
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v19
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v21 :: v_dual_and_b32 v19, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v21, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v17, 16, v18
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v21, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v19, v20 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v18, v18, v18 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v23, v18, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v18
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v13
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v19, v20 :: v_dual_and_b32 v18, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX11-FAKE16-NEXT:    v_add3_u32 v23, v23, v18, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v18
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v25, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v22, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v18, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v25, 16, v23
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v27, 16, v22
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v27
-; GFX11-FAKE16-NEXT:    v_add3_u32 v17, v26, v21, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v22, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v25, v29, v28 :: v_dual_and_b32 v18, 0xffff0000, v18
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v27, 16, v25
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v24 :: v_dual_lshlrev_b32 v25, 16, v21
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v28, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v24, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v18 :: v_dual_lshlrev_b32 v26, 16, v20
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v26
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v18, v21 :: v_dual_lshlrev_b32 v27, 16, v19
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v19, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v18, v20, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v19 :: v_dual_lshlrev_b32 v28, 16, v24
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v28
-; GFX11-FAKE16-NEXT:    v_add3_u32 v20, v26, v21, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v24, v25 :: v_dual_and_b32 v28, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v17
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v20, v26 :: v_dual_lshlrev_b32 v21, 16, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v20 :: v_dual_and_b32 v23, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v25 :: v_dual_and_b32 v22, 0xffff0000, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v24, v24, v23 :: v_dual_and_b32 v25, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v22
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v20, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v28, v27, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v20, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v29
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v23 :: v_dual_lshlrev_b32 v29, 16, v20
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v9
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v30, v29 :: v_dual_and_b32 v27, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v18, v19 :: v_dual_lshlrev_b32 v28, 16, v26
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v19, v21, v21 :: v_dual_and_b32 v20, 0xffff0000, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v29, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v18, v23, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v27, v25, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v29, 16, v21
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v27, v19, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v29
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v23, v27, v19, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v21, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v23, v27, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v28
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v31, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v22, v23, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v30, v27, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v22, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v20, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v22 :: v_dual_lshlrev_b32 v28, 16, v27
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v26, v23 :: v_dual_lshlrev_b32 v23, 16, v25
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v23
-; GFX11-FAKE16-NEXT:    v_add3_u32 v25, v28, v22, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v22
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v30
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v23, v27 :: v_dual_and_b32 v30, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v28 :: v_dual_lshlrev_b32 v25, 16, v29
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v19
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v32, v31, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v0
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v28
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v24 :: v_dual_max_f32 v24, v25, v25
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v33, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v31, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v29, v24, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v20, v26 :: v_dual_lshlrev_b32 v31, 16, v25
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v26, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v31
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    v_add3_u32 v26, v29, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v25, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v32, v31 :: v_dual_lshlrev_b32 v29, 16, v29
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v33, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v26, v29, v29 :: v_dual_lshlrev_b32 v33, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v31, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v22, v26, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v34, 0x400000, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v28
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v25 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v29
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v22, v26, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v21, v27 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v31, v32
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v31, v29, v30 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v27, v23, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v23 :: v_dual_and_b32 v22, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v28
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v31
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v28 :: v_dual_lshlrev_b32 v32, 16, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v32, v31
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v28, v15, v7 :: v_dual_lshlrev_b32 v31, 16, v6
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v22 :: v_dual_and_b32 v24, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v25 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v25, 0x400000, v26
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v27, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_and_b32 v26, 0xffff0000, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v25, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v15, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v27, 16, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v25, v28, v28 :: v_dual_lshlrev_b32 v32, 16, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v27, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v27, v25, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v27
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v25, v24 :: v_dual_lshlrev_b32 v23, 16, v14
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v15, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v27, v27, v25, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v23, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v27, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v32, v31
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v14, v6 :: v_dual_lshlrev_b32 v28, 16, v13
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v25
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v13
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v28, v7 :: v_dual_lshlrev_b32 v30, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v29
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v13, v5 :: v_dual_lshlrev_b32 v29, 16, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v24, v27, v27
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v15, v24, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v15, v15, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v26, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v27, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v14
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_lshlrev_b32 v28, 16, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v14, v6 :: v_dual_lshlrev_b32 v24, 16, v27
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v28, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v27
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v12, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v14 :: v_dual_lshlrev_b32 v29, 16, v3
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v14, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v11
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v24
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v27, v14, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v14
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v13 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v5 :: v_dual_lshlrev_b32 v14, 16, v12
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v5 :: v_dual_lshlrev_b32 v24, 16, v12
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v27, v27, v14, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v26, v5 :: v_dual_lshlrev_b32 v28, 16, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v12, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v11, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v27, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v0
-; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v10
-; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v26, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v14, v4 :: v_dual_lshlrev_b32 v13, 16, v15
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v11, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v14
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v5, v18, v5, 0x5040100
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v2 :: v_dual_lshlrev_b32 v25, 16, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v13, v13, v13 :: v_dual_lshlrev_b32 v26, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v13, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_add3_u32 v14, v24, v13, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v27, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v10, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v14, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v9, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v14
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v24, v26, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v8, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v26, v24, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v27, v3 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v11, v14, v14
-; GFX11-FAKE16-NEXT:    v_add3_u32 v14, v26, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v29, v11, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v25, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v25
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v28, v25, 0x7fff
-; GFX11-FAKE16-NEXT:    v_add3_u32 v28, v29, v11, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v29, 0x400000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v26, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v11 :: v_dual_lshlrev_b32 v12, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v10 :: v_dual_lshlrev_b32 v15, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v1 :: v_dual_lshlrev_b32 v12, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v0 :: v_dual_lshlrev_b32 v15, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v12
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v11, v2 :: v_dual_lshlrev_b32 v15, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v8, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v15
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v12, v1 :: v_dual_lshlrev_b32 v8, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v4, v19, v4, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -10281,534 +8005,406 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v18, v7 :: v_dual_mov_b32 v17, v6
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v20, v5
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v18, v5
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v15
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v14
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v18
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v14
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v16
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v13
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v18.h, v15.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, s0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.h, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v18
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v19, v21
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.h, v13.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v16.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v6.l, v5.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v22
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v19.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v20.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v4.h, v12.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v23, v23, v23
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v24, v23
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v24, v24, v24
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v23
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v23, v23
-; GFX12-TRUE16-NEXT:    v_add3_u32 v27, v27, v23, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v25, v26
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v7.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v26, v27, v28, s3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v22.l, v19.l, s2
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
-; GFX12-TRUE16-NEXT:    v_add3_u32 v25, v25, v24, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v26.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v26
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v19.l, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v19.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.l, v7.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v24, v25, v27, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.h, v21.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v16.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v23, v23, v23
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v7.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v24, v25
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v26.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.h, v16.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v23, v23
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX12-TRUE16-NEXT:    v_add3_u32 v16, v27, v23, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v23
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v25, v26
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v22.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v20.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v19.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v24.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v23, v16, v27, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v19.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v21.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v3.h, v11.h, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v23.h, v19.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v16.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v11.h, v16.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v23
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v28, v19, v19
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v11
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v22.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v20
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v2.h, v10.h, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v3.h, v11.h, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v28, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v25
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v26, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v19.l, v22.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v10.h, v19.l, s0
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v28
-; GFX12-TRUE16-NEXT:    v_add3_u32 v24, v24, v28, 0x7fff
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v19.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v7.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v22.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v24, v24, v25 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v23.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v28, v28
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v26, v27
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v24
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v11.h, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v20.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v5.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v21.l
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v26, v21, v21
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v22.l, v19.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v5.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v25, v26, 16, 1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v21.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v9.h, v23.l, s0
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v26
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v23.l
-; GFX12-TRUE16-NEXT:    v_add3_u32 v25, v25, v26, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v21.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v26, v26
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v21.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v20.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v24.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.h, v5.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v26, v28, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v30
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v25, v25, v27, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v29
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v16.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v26, v26
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v28
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v22.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v25.h, v16.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX12-TRUE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v21.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v21.l, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v28, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v19.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v9.h, v23.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.l, v21.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v27, v25
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v24.h, v19.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v19, v26, v26
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v7.l, v6.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v21, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v19, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v18
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v30, 0x400000, v19
-; GFX12-TRUE16-NEXT:    v_add3_u32 v22, v22, v19, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v19, v19
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v27
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v15
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v24
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v22, v22, v30, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v23.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v7.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v5.l, v23.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v23.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v18.l, v18.l, v15.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v29
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v22.h, v23.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v21.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v21.l, v23.l, s0
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v15
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v20.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v23, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v18.h, v25.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v18.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.h, v21.l, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v15.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.h, v24.h, v16.l, s1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.l, v16.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v7.l, v19.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v19.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v16.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v14
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v15.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v14
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v27, v21, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v24, v23
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v17.l, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    v_add3_u32 v17, v27, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v22.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.l, v18.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v17.l, v17.l, v14.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v19.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v14.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v24, v23
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v17.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v17, v23, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v7.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v17.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v14.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v16.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v21, v21, v21
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v23, v22
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v21, 16, 1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v13
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v14.l, v16.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v7.l, v16.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-TRUE16-NEXT:    v_add3_u32 v20, v22, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v13.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v15.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v20, v22, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v18.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.l, v17.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v17.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v17.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v17, v21, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v16.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v20.h, v18.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v15.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v13.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v16.l, v17.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v14.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    v_add3_u32 v15, v22, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v17
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v22
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v23
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v15, v15, v21, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v12.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16.l
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v17, v17, v17 :: v_dual_lshlrev_b32 v22, 16, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v14.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v15.h, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v12.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v21, v23, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v12.l, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v17
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v17, v21, v22, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v15
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v13.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v13.l, s0
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v13, v21, v21 :: v_dual_lshlrev_b32 v22, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v3.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v18, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT:    v_add3_u32 v23, v23, v13, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v13, v13
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v20, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v23, v24, s3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v4.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v10.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v11.l, v3.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v12.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v16.l, v4.h, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v20.h, v4.l, s3
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v13.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v10.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v9.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v21, v15
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v1.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v4.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v10.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v8.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v12.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v15
-; GFX12-TRUE16-NEXT:    v_add3_u32 v15, v23, v13, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v13
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v22, v21
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v21, v9, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v8.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v17, v21, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v15.h, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v9
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v17, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v12.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v22, v17
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v9, v12, s0
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v12, v13, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v13, v3, 16, 1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v3
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v11, v12, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v17
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.h, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_add3_u32 v13, v13, v3, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v11, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v22, v21
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v13.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v13, v21, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v22, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v8.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v21
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v12
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v21
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v17
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v11.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v9.h, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v18.l, v15.h, v1.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v20.h, v0.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v2.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v8.l, v1.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v10.l, v0.h, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v13.l, v2.h, s3
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v14
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v19 :: v_dual_mov_b32 v3, v18
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v16
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v19
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v16bf16:
@@ -10822,557 +8418,398 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v15
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v13
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v15
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v17, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v21 :: v_dual_and_b32 v19, 0xffff0000, v14
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v21, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v17, 16, v18
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v21, v22
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v19, v20 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v18, v18, v18 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v23, v18, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v18
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v19, v20 :: v_dual_and_b32 v18, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v21, v21, v21
-; GFX12-FAKE16-NEXT:    v_add3_u32 v23, v23, v18, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v22, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v18
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v22, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v18, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v24 :: v_dual_lshlrev_b32 v25, 16, v21
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v25, 16, v23
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v27, 16, v22
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v27
-; GFX12-FAKE16-NEXT:    v_add3_u32 v17, v26, v21, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v22, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v24, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v18 :: v_dual_lshlrev_b32 v26, 16, v20
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v25, v29, v28 :: v_dual_and_b32 v18, 0xffff0000, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v27, 16, v25
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v26
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v28, v25, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v18, v20, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v18, v21 :: v_dual_lshlrev_b32 v27, 16, v19
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v19 :: v_dual_lshlrev_b32 v28, 16, v24
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v28
-; GFX12-FAKE16-NEXT:    v_add3_u32 v20, v26, v21, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v19, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v24, v25 :: v_dual_and_b32 v28, 0xffff0000, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v17
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v20, v26 :: v_dual_lshlrev_b32 v21, 16, v27
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v20 :: v_dual_and_b32 v23, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v30, v29 :: v_dual_and_b32 v27, 0xffff0000, v11
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v25 :: v_dual_and_b32 v22, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v18, v19 :: v_dual_lshlrev_b32 v28, 16, v26
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v19, v21, v21 :: v_dual_and_b32 v20, 0xffff0000, v20
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v29, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v18, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v29, 16, v21
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v27, v19, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v29
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v23, v27, v19, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v19
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v24, v24, v23 :: v_dual_and_b32 v25, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v22
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v20, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v21, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v23, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v28
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v28, v27, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v31, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v20, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v23 :: v_dual_lshlrev_b32 v29, 16, v20
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v9
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v22, v23, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v30, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v22, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v27, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v22 :: v_dual_lshlrev_b32 v28, 16, v27
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v20, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v26, v23 :: v_dual_lshlrev_b32 v23, 16, v25
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v23
-; GFX12-FAKE16-NEXT:    v_add3_u32 v25, v28, v22, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v22
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v30
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v23, v27 :: v_dual_and_b32 v30, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v25 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v28 :: v_dual_lshlrev_b32 v25, 16, v29
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v19
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v32, v31, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v0
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v9
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v28
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v22 :: v_dual_and_b32 v24, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v24 :: v_dual_max_num_f32 v24, v25, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v33, 0x400000, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v31, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v29, v24, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v20, v26 :: v_dual_lshlrev_b32 v31, 16, v25
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v26, v21, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v31
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    v_add3_u32 v26, v29, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v25, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v32, v31 :: v_dual_lshlrev_b32 v29, 16, v29
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v27, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v7
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v33, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v26, v29, v29 :: v_dual_lshlrev_b32 v33, 16, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v31, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v22, v26, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v34, 0x400000, v26
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v29
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v22, v26, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v21, v27 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v31, v32
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v31, v29, v30 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_and_b32 v26, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v27, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v24, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v23 :: v_dual_and_b32 v22, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v28
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v31
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v28 :: v_dual_lshlrev_b32 v32, 16, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v26, v26, v26
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v32, v31
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v28, v15, v7 :: v_dual_lshlrev_b32 v31, 16, v6
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v25 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v25, 0x400000, v26
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v14
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v15, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v25, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v27, 16, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v25, v28, v28 :: v_dual_lshlrev_b32 v32, 16, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v27, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v27, v25, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v27, v27, v25, 0x7fff
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v23, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v25, v24 :: v_dual_lshlrev_b32 v23, 16, v14
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v15, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v27, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v32, v31
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v14, v6 :: v_dual_lshlrev_b32 v28, 16, v13
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v25
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v29
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v14
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v13
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_lshlrev_b32 v28, 16, v6
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v28, v7 :: v_dual_lshlrev_b32 v30, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v29
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v14, v6 :: v_dual_lshlrev_b32 v24, 16, v27
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v13, v5 :: v_dual_lshlrev_b32 v29, 16, v12
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v24, v27, v27
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v15, v24, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v15, v15, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v26, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v4
+; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v12
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v13 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v28, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v27
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v12, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v5 :: v_dual_lshlrev_b32 v14, 16, v12
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v14 :: v_dual_lshlrev_b32 v29, 16, v3
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v14, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v11
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v24
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v27, v14, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v14
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v5 :: v_dual_lshlrev_b32 v24, 16, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    v_add3_u32 v27, v27, v14, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v26, v5 :: v_dual_lshlrev_b32 v28, 16, v11
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v11, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v27, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v0
-; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v10
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v12, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v26, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v14, v4 :: v_dual_lshlrev_b32 v13, 16, v15
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v11, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v5, v18, v5, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v2 :: v_dual_lshlrev_b32 v25, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v13, v13, v13 :: v_dual_lshlrev_b32 v26, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v13, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    v_add3_u32 v14, v24, v13, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v27, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v10, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v11 :: v_dual_lshlrev_b32 v12, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v14, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v10 :: v_dual_lshlrev_b32 v15, 16, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v9, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v14
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v24, v26, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v8, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v26, v24, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v27, v3 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v11, v14, v14
-; GFX12-FAKE16-NEXT:    v_add3_u32 v14, v26, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v29, v11, 16, 1
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v25, v25, v25
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v25, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v25
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v28, v25, 0x7fff
-; GFX12-FAKE16-NEXT:    v_add3_u32 v28, v29, v11, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v29, 0x400000, v11
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v1 :: v_dual_lshlrev_b32 v12, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v0 :: v_dual_lshlrev_b32 v15, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v11, v2 :: v_dual_lshlrev_b32 v15, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v15
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v8, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v12, v1 :: v_dual_lshlrev_b32 v8, 16, v11
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v4, v19, v4, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x bfloat> @llvm.maximumnum.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y)
@@ -11646,98 +9083,80 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-LABEL: v_maximumnum_v32bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX8-NEXT:    buffer_load_dword v55, off, s[0:3], s32
 ; GFX8-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v34, 16, v30
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
-; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
+; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
+; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
+; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v31, v35, v34, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
+; GFX8-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
 ; GFX8-NEXT:    v_cndmask_b32_e32 v34, v34, v31, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v31
-; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v34
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v35, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v34, v31, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
-; GFX8-NEXT:    v_mul_f32_e32 v35, 1.0, v35
-; GFX8-NEXT:    v_bfe_u32 v36, v35, 16, 1
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, v36, v35
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, s4, v36
-; GFX8-NEXT:    v_or_b32_e32 v39, 0x400000, v35
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v36, v39, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v35
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
+; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
+; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v37, v39
+; GFX8-NEXT:    v_cndmask_b32_e32 v37, v34, v31, vcc
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v36, v48
+; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v31
-; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v36, v31, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
-; GFX8-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX8-NEXT:    v_cndmask_b32_e32 v31, v31, v34, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v35
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
+; GFX8-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX8-NEXT:    v_cndmask_b32_e32 v34, v35, v38, vcc
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v36, v31, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v32, v32
-; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
-; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
+; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
+; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
+; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
+; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
+; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
+; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
+; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
+; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
+; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
+; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
+; GFX8-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
+; GFX8-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v34, 16, v55
-; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v55
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v34, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v34, v32, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v55
+; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v35, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v35, v32, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v33, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v34, v32, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX8-NEXT:    v_mul_f32_e32 v33, 1.0, v33
-; GFX8-NEXT:    v_bfe_u32 v35, v33, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v35, vcc, v35, v33
-; GFX8-NEXT:    v_add_u32_e32 v35, vcc, s4, v35
-; GFX8-NEXT:    v_or_b32_e32 v36, 0x400000, v33
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v33, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v35, v36, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v33
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v33, v37
+; GFX8-NEXT:    v_cndmask_b32_e32 v33, v35, v32, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v32
-; GFX8-NEXT:    v_and_b32_e32 v33, 0xffff0000, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v35, v32, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v32, v34, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v35, v32, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v13
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v33, v38, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v38, v33, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v34, 16, v33
-; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v34, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v35, v33, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
-; GFX8-NEXT:    v_mul_f32_e32 v34, 1.0, v34
-; GFX8-NEXT:    v_bfe_u32 v36, v34, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, v36, v34
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, s4, v36
-; GFX8-NEXT:    v_or_b32_e32 v37, 0x400000, v34
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v36, v37, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v34
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
-; GFX8-NEXT:    v_and_b32_e32 v34, 0xffff0000, v34
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v33, v35, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v34
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
+; GFX8-NEXT:    v_cndmask_b32_e32 v33, v36, v34, vcc
 ; GFX8-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
@@ -11749,27 +9168,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v36, v37
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v35, v34, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX8-NEXT:    v_mul_f32_e32 v36, 1.0, v36
-; GFX8-NEXT:    v_bfe_u32 v37, v36, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v37, vcc, v37, v36
-; GFX8-NEXT:    v_add_u32_e32 v37, vcc, s4, v37
-; GFX8-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v37, v38, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v37, 16, v36
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
 ; GFX8-NEXT:    v_cndmask_b32_e32 v34, v34, v35, vcc
-; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v36
+; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v36
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v35
 ; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v27
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v36, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc
@@ -11777,27 +9186,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v38, 16, v36
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v37, v38
 ; GFX8-NEXT:    v_cndmask_b32_e32 v37, v36, v35, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX8-NEXT:    v_mul_f32_e32 v37, 1.0, v37
-; GFX8-NEXT:    v_bfe_u32 v38, v37, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v38, vcc, v38, v37
-; GFX8-NEXT:    v_add_u32_e32 v38, vcc, s4, v38
-; GFX8-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v38, v39, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v37
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v36
 ; GFX8-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc
-; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v37
+; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v37
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v36
 ; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v37, 16, v26
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v37, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
 ; GFX8-NEXT:    v_cndmask_b32_e32 v37, v37, v36, vcc
@@ -11805,23 +9204,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v38, v39
 ; GFX8-NEXT:    v_cndmask_b32_e32 v38, v37, v36, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX8-NEXT:    v_mul_f32_e32 v38, 1.0, v38
-; GFX8-NEXT:    v_bfe_u32 v39, v38, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v39, vcc, v39, v38
-; GFX8-NEXT:    v_add_u32_e32 v39, vcc, s4, v39
-; GFX8-NEXT:    v_or_b32_e32 v48, 0x400000, v38
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX8-NEXT:    v_cndmask_b32_e32 v38, v39, v48, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v38
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v37
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v36, v37, vcc
-; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v38
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v38
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
 ; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v25
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
@@ -11833,27 +9223,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v39, v48
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v38, v37, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX8-NEXT:    v_mul_f32_e32 v39, 1.0, v39
-; GFX8-NEXT:    v_bfe_u32 v48, v39, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v48, vcc, v48, v39
-; GFX8-NEXT:    v_add_u32_e32 v48, vcc, s4, v48
-; GFX8-NEXT:    v_or_b32_e32 v49, 0x400000, v39
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX8-NEXT:    v_cndmask_b32_e32 v39, v48, v49, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v48, 16, v39
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v37
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
 ; GFX8-NEXT:    v_cndmask_b32_e32 v37, v37, v38, vcc
-; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v39
+; GFX8-NEXT:    v_lshlrev_b32_e32 v38, 16, v39
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v38
 ; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v24
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v38, v48, v39, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v39, v38, vcc
@@ -11861,27 +9241,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v49, 16, v39
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v48, v49
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v39, v38, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX8-NEXT:    v_mul_f32_e32 v48, 1.0, v48
-; GFX8-NEXT:    v_bfe_u32 v49, v48, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v49, vcc, v49, v48
-; GFX8-NEXT:    v_add_u32_e32 v49, vcc, s4, v49
-; GFX8-NEXT:    v_or_b32_e32 v50, 0x400000, v48
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v49, v50, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v49, 16, v48
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
-; GFX8-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v39
 ; GFX8-NEXT:    v_cndmask_b32_e32 v38, v38, v39, vcc
-; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v48
+; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v48
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v49, v48, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc
@@ -11889,27 +9259,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v49, v50
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v48, v39, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX8-NEXT:    v_mul_f32_e32 v49, 1.0, v49
-; GFX8-NEXT:    v_bfe_u32 v50, v49, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v50, vcc, v50, v49
-; GFX8-NEXT:    v_add_u32_e32 v50, vcc, s4, v50
-; GFX8-NEXT:    v_or_b32_e32 v51, 0x400000, v49
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX8-NEXT:    v_cndmask_b32_e32 v49, v50, v51, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v50, 16, v49
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v39
-; GFX8-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v48
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc
-; GFX8-NEXT:    v_and_b32_e32 v48, 0xffff0000, v49
+; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v49
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
 ; GFX8-NEXT:    v_and_b32_e32 v48, 0xffff0000, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v50, v49, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc
@@ -11917,27 +9277,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v51, 16, v49
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v50, v51
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v49, v48, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX8-NEXT:    v_mul_f32_e32 v50, 1.0, v50
-; GFX8-NEXT:    v_bfe_u32 v51, v50, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v51, vcc, v51, v50
-; GFX8-NEXT:    v_add_u32_e32 v51, vcc, s4, v51
-; GFX8-NEXT:    v_or_b32_e32 v52, 0x400000, v50
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v51, v52, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v51, 16, v50
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v48
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v49
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc
-; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v50
+; GFX8-NEXT:    v_lshlrev_b32_e32 v49, 16, v50
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v49
 ; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v51, v50, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc
@@ -11945,27 +9295,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v51, v52
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v50, v49, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX8-NEXT:    v_mul_f32_e32 v51, 1.0, v51
-; GFX8-NEXT:    v_bfe_u32 v52, v51, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v52, vcc, v52, v51
-; GFX8-NEXT:    v_add_u32_e32 v52, vcc, s4, v52
-; GFX8-NEXT:    v_or_b32_e32 v53, 0x400000, v51
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX8-NEXT:    v_cndmask_b32_e32 v51, v52, v53, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v52, 16, v51
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v49
-; GFX8-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v50
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc
-; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v51
+; GFX8-NEXT:    v_lshlrev_b32_e32 v50, 16, v51
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
 ; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v52, v51, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v51, v50, vcc
@@ -11973,27 +9313,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v52, v53
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v51, v50, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX8-NEXT:    v_mul_f32_e32 v52, 1.0, v52
-; GFX8-NEXT:    v_bfe_u32 v53, v52, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v53, vcc, v53, v52
-; GFX8-NEXT:    v_add_u32_e32 v53, vcc, s4, v53
-; GFX8-NEXT:    v_or_b32_e32 v54, 0x400000, v52
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v53, v54, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v53, 16, v52
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v50
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v51
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v50, v51, vcc
-; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v52
+; GFX8-NEXT:    v_lshlrev_b32_e32 v51, 16, v52
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
 ; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v53, 16, v3
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v53, v52, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc
@@ -12001,27 +9331,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v53, v54
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v52, v51, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX8-NEXT:    v_mul_f32_e32 v53, 1.0, v53
-; GFX8-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v54, vcc, v54, v53
-; GFX8-NEXT:    v_add_u32_e32 v54, vcc, s4, v54
-; GFX8-NEXT:    v_or_b32_e32 v40, 0x400000, v53
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v54, v40, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v53
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v51
-; GFX8-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v52
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc
-; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v53
+; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v54, v53, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v53, v52, vcc
@@ -12029,27 +9349,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v53
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v54, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v53, v52, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX8-NEXT:    v_mul_f32_e32 v54, 1.0, v54
-; GFX8-NEXT:    v_bfe_u32 v40, v54, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, v40, v54
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, s4, v40
-; GFX8-NEXT:    v_or_b32_e32 v41, 0x400000, v54
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v40, v41, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v54
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v52
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v53
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v52, v53, vcc
-; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v54
+; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v54
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v53
 ; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX8-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v40, v54, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
@@ -12057,27 +9367,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v54
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v41
 ; GFX8-NEXT:    v_cndmask_b32_e32 v40, v54, v53, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX8-NEXT:    v_mul_f32_e32 v40, 1.0, v40
-; GFX8-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, v41, v40
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, s4, v41
-; GFX8-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v53
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v54
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc
-; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v40
+; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v41, v40, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
 ; GFX8-NEXT:    v_cndmask_b32_e32 v40, v40, v54, vcc
@@ -12085,23 +9385,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v42
 ; GFX8-NEXT:    v_cndmask_b32_e32 v41, v40, v54, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
-; GFX8-NEXT:    v_mul_f32_e32 v41, 1.0, v41
-; GFX8-NEXT:    v_bfe_u32 v42, v41, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v42, vcc, v42, v41
-; GFX8-NEXT:    v_add_u32_e32 v42, vcc, s4, v42
-; GFX8-NEXT:    v_or_b32_e32 v43, 0x400000, v41
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX8-NEXT:    v_cndmask_b32_e32 v41, v42, v43, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v42, 16, v41
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v54
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v40, vcc
-; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v41
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v41
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
@@ -12111,23 +9402,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v40, v55, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX8-NEXT:    v_mul_f32_e32 v40, 1.0, v40
-; GFX8-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, v41, v40
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, s4, v41
-; GFX8-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
-; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v40
+; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
@@ -12137,23 +9419,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v55, v30, v14, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX8-NEXT:    v_mul_f32_e32 v55, 1.0, v55
-; GFX8-NEXT:    v_bfe_u32 v40, v55, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, v40, v55
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, s4, v40
-; GFX8-NEXT:    v_or_b32_e32 v41, 0x400000, v55
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v40, v41, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT:    v_and_b32_e32 v30, 0xffff0000, v55
+; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v55
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
@@ -12163,23 +9436,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v55, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX8-NEXT:    v_mul_f32_e32 v30, 1.0, v30
-; GFX8-NEXT:    v_bfe_u32 v55, v30, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v55, vcc, v55, v30
-; GFX8-NEXT:    v_add_u32_e32 v55, vcc, s4, v55
-; GFX8-NEXT:    v_or_b32_e32 v40, 0x400000, v30
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX8-NEXT:    v_cndmask_b32_e32 v30, v55, v40, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v30
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX8-NEXT:    v_and_b32_e32 v29, 0xffff0000, v30
+; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v30
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v29
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
@@ -12189,23 +9453,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v12
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v30, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX8-NEXT:    v_mul_f32_e32 v29, 1.0, v29
-; GFX8-NEXT:    v_bfe_u32 v30, v29, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v30, vcc, v30, v29
-; GFX8-NEXT:    v_add_u32_e32 v30, vcc, s4, v30
-; GFX8-NEXT:    v_or_b32_e32 v55, 0x400000, v29
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v29, v30, v55, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v28
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX8-NEXT:    v_and_b32_e32 v28, 0xffff0000, v29
+; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v28
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
@@ -12215,23 +9470,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v29, v28
 ; GFX8-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX8-NEXT:    v_mul_f32_e32 v28, 1.0, v28
-; GFX8-NEXT:    v_bfe_u32 v29, v28, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v29, vcc, v29, v28
-; GFX8-NEXT:    v_add_u32_e32 v29, vcc, s4, v29
-; GFX8-NEXT:    v_or_b32_e32 v30, 0x400000, v28
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
-; GFX8-NEXT:    v_cndmask_b32_e32 v28, v29, v30, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v29, 16, v28
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX8-NEXT:    v_and_b32_e32 v27, 0xffff0000, v28
+; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v27
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
@@ -12241,23 +9487,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v10
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v28, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX8-NEXT:    v_mul_f32_e32 v27, 1.0, v27
-; GFX8-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v28, vcc, v28, v27
-; GFX8-NEXT:    v_add_u32_e32 v28, vcc, s4, v28
-; GFX8-NEXT:    v_or_b32_e32 v29, 0x400000, v27
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
-; GFX8-NEXT:    v_cndmask_b32_e32 v27, v28, v29, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX8-NEXT:    v_and_b32_e32 v26, 0xffff0000, v27
+; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v26
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
@@ -12267,23 +9504,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v27, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX8-NEXT:    v_mul_f32_e32 v26, 1.0, v26
-; GFX8-NEXT:    v_bfe_u32 v27, v26, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v27, vcc, v27, v26
-; GFX8-NEXT:    v_add_u32_e32 v27, vcc, s4, v27
-; GFX8-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v26, v27, v28, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX8-NEXT:    v_and_b32_e32 v25, 0xffff0000, v26
+; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v25
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
@@ -12293,23 +9521,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v26, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX8-NEXT:    v_mul_f32_e32 v25, 1.0, v25
-; GFX8-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, v26, v25
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, s4, v26
-; GFX8-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX8-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
@@ -12319,23 +9538,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX8-NEXT:    v_mul_f32_e32 v24, 1.0, v24
-; GFX8-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, v25, v24
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, s4, v25
-; GFX8-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX8-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
@@ -12345,23 +9558,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX8-NEXT:    v_mul_f32_e32 v23, 1.0, v23
-; GFX8-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, v24, v23
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, s4, v24
-; GFX8-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX8-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
@@ -12371,23 +9575,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v23, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX8-NEXT:    v_mul_f32_e32 v22, 1.0, v22
-; GFX8-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v22
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, s4, v23
-; GFX8-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX8-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
@@ -12397,23 +9592,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v4
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v22, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX8-NEXT:    v_mul_f32_e32 v21, 1.0, v21
-; GFX8-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v21
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, s4, v22
-; GFX8-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX8-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
@@ -12421,29 +9607,16 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX8-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v21, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX8-NEXT:    v_mul_f32_e32 v20, 1.0, v20
-; GFX8-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v20
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, s4, v21
-; GFX8-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX8-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
@@ -12453,23 +9626,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v20, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX8-NEXT:    v_mul_f32_e32 v19, 1.0, v19
-; GFX8-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v19
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, s4, v20
-; GFX8-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX8-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
@@ -12479,23 +9643,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v19, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX8-NEXT:    v_mul_f32_e32 v18, 1.0, v18
-; GFX8-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v18
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, s4, v19
-; GFX8-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX8-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
@@ -12505,22 +9660,13 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v18, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX8-NEXT:    v_mul_f32_e32 v17, 1.0, v17
-; GFX8-NEXT:    v_bfe_u32 v18, v17, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v18, v17
-; GFX8-NEXT:    v_add_u32_e32 v18, vcc, s4, v18
-; GFX8-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX8-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
+; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v17
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v54
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v53
@@ -12559,10 +9705,9 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-LABEL: v_maximumnum_v32bf16:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX900-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX900-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX900-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX900-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX900-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX900-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX900-NEXT:    buffer_load_dword v55, off, s[0:3], s32
 ; GFX900-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v34, 16, v30
@@ -12574,80 +9719,67 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v31, v35, v34, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
+; GFX900-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
 ; GFX900-NEXT:    v_cndmask_b32_e32 v34, v34, v31, vcc
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
+; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
+; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v37, v39
 ; GFX900-NEXT:    v_cndmask_b32_e32 v37, v34, v31, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX900-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_bfe_u32 v39, v37, 16, 1
-; GFX900-NEXT:    v_or_b32_e32 v49, 0x400000, v37
-; GFX900-NEXT:    v_add3_u32 v39, v39, v37, s4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v39, v49, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v37
+; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v36, v48
+; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v31
-; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v31, v39, v31, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
-; GFX900-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX900-NEXT:    v_cndmask_b32_e32 v31, v31, v34, vcc
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
+; GFX900-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX900-NEXT:    v_cndmask_b32_e32 v34, v35, v38, vcc
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v31, v39, v31, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v32, v32
-; GFX900-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
-; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
+; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
+; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
+; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
+; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
+; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
+; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
+; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
+; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
+; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
+; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
+; GFX900-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
+; GFX900-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
+; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    v_lshrrev_b32_e32 v34, 16, v55
+; GFX900-NEXT:    v_lshrrev_b32_e32 v35, 16, v55
 ; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v55
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v35, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v34, v32, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v35, v32, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
-; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
+; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v33, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v34, v32, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX900-NEXT:    v_max_f32_e32 v33, v33, v33
-; GFX900-NEXT:    v_bfe_u32 v37, v33, 16, 1
-; GFX900-NEXT:    v_or_b32_e32 v39, 0x400000, v33
-; GFX900-NEXT:    v_add3_u32 v37, v37, v33, s4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v33, v33
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v37, v39, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v33
+; GFX900-NEXT:    v_cndmask_b32_e32 v33, v35, v32, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v32
-; GFX900-NEXT:    v_and_b32_e32 v33, 0xffff0000, v33
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v37, v32, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v32, v34, vcc
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v33
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v37, v32, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v38, v35, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v34, 16, v33
-; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v36, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v33, v35, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
-; GFX900-NEXT:    v_max_f32_e32 v34, v34, v34
-; GFX900-NEXT:    v_bfe_u32 v36, v34, 16, 1
-; GFX900-NEXT:    v_add3_u32 v36, v36, v34, s4
-; GFX900-NEXT:    v_or_b32_e32 v37, 0x400000, v34
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v36, v37, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v36, 16, v34
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
-; GFX900-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v33
-; GFX900-NEXT:    v_and_b32_e32 v34, 0xffff0000, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v35, v33, vcc
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v34
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
+; GFX900-NEXT:    v_cndmask_b32_e32 v33, v36, v34, vcc
 ; GFX900-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
@@ -12659,26 +9791,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v36, v37
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v35, v34, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX900-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX900-NEXT:    v_bfe_u32 v37, v36, 16, 1
-; GFX900-NEXT:    v_add3_u32 v37, v37, v36, s4
-; GFX900-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX900-NEXT:    v_cndmask_b32_e32 v36, v37, v38, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v36
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
 ; GFX900-NEXT:    v_cndmask_b32_e32 v34, v34, v35, vcc
-; GFX900-NEXT:    v_and_b32_e32 v35, 0xffff0000, v36
+; GFX900-NEXT:    v_lshlrev_b32_e32 v35, 16, v36
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v35
 ; GFX900-NEXT:    v_and_b32_e32 v35, 0xffff0000, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v36, 16, v27
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v35, v37, v36, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc
@@ -12686,26 +9809,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v38, 16, v36
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v37, v38
 ; GFX900-NEXT:    v_cndmask_b32_e32 v37, v36, v35, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX900-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX900-NEXT:    v_bfe_u32 v38, v37, 16, 1
-; GFX900-NEXT:    v_add3_u32 v38, v38, v37, s4
-; GFX900-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v38, v39, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v38, 16, v37
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
-; GFX900-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v36
 ; GFX900-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc
-; GFX900-NEXT:    v_and_b32_e32 v36, 0xffff0000, v37
+; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v37
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v36
 ; GFX900-NEXT:    v_and_b32_e32 v36, 0xffff0000, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v26
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v37, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
 ; GFX900-NEXT:    v_cndmask_b32_e32 v37, v37, v36, vcc
@@ -12713,22 +9827,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v38, v39
 ; GFX900-NEXT:    v_cndmask_b32_e32 v38, v37, v36, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX900-NEXT:    v_max_f32_e32 v38, v38, v38
-; GFX900-NEXT:    v_bfe_u32 v39, v38, 16, 1
-; GFX900-NEXT:    v_add3_u32 v39, v39, v38, s4
-; GFX900-NEXT:    v_or_b32_e32 v48, 0x400000, v38
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX900-NEXT:    v_cndmask_b32_e32 v38, v39, v48, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v38
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v36
-; GFX900-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v37
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v36, v37, vcc
-; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v38
+; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v38
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
 ; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v38, 16, v25
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
@@ -12740,26 +9846,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v39, v48
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v38, v37, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX900-NEXT:    v_max_f32_e32 v39, v39, v39
-; GFX900-NEXT:    v_bfe_u32 v48, v39, 16, 1
-; GFX900-NEXT:    v_add3_u32 v48, v48, v39, s4
-; GFX900-NEXT:    v_or_b32_e32 v49, 0x400000, v39
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX900-NEXT:    v_cndmask_b32_e32 v39, v48, v49, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v48, 16, v39
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
 ; GFX900-NEXT:    v_cndmask_b32_e32 v37, v37, v38, vcc
-; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v39
+; GFX900-NEXT:    v_lshlrev_b32_e32 v38, 16, v39
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v38
 ; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v24
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v38, v48, v39, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v39, v38, vcc
@@ -12767,26 +9864,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v49, 16, v39
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v48, v49
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v39, v38, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX900-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX900-NEXT:    v_bfe_u32 v49, v48, 16, 1
-; GFX900-NEXT:    v_add3_u32 v49, v49, v48, s4
-; GFX900-NEXT:    v_or_b32_e32 v50, 0x400000, v48
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX900-NEXT:    v_cndmask_b32_e32 v48, v49, v50, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v49, 16, v48
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
-; GFX900-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v39
 ; GFX900-NEXT:    v_cndmask_b32_e32 v38, v38, v39, vcc
-; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v48
+; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v48
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v49, v48, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc
@@ -12794,26 +9882,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v49, v50
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v48, v39, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX900-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX900-NEXT:    v_bfe_u32 v50, v49, 16, 1
-; GFX900-NEXT:    v_add3_u32 v50, v50, v49, s4
-; GFX900-NEXT:    v_or_b32_e32 v51, 0x400000, v49
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX900-NEXT:    v_cndmask_b32_e32 v49, v50, v51, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v50, 16, v49
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v39
-; GFX900-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v48
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc
-; GFX900-NEXT:    v_and_b32_e32 v48, 0xffff0000, v49
+; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v49
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
 ; GFX900-NEXT:    v_and_b32_e32 v48, 0xffff0000, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v50, v49, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc
@@ -12821,26 +9900,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v51, 16, v49
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v50, v51
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v49, v48, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX900-NEXT:    v_max_f32_e32 v50, v50, v50
-; GFX900-NEXT:    v_bfe_u32 v51, v50, 16, 1
-; GFX900-NEXT:    v_add3_u32 v51, v51, v50, s4
-; GFX900-NEXT:    v_or_b32_e32 v52, 0x400000, v50
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX900-NEXT:    v_cndmask_b32_e32 v50, v51, v52, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v51, 16, v50
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v48
-; GFX900-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v49
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc
-; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v50
+; GFX900-NEXT:    v_lshlrev_b32_e32 v49, 16, v50
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v49
 ; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v51, v50, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc
@@ -12848,26 +9918,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v51, v52
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v50, v49, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX900-NEXT:    v_max_f32_e32 v51, v51, v51
-; GFX900-NEXT:    v_bfe_u32 v52, v51, 16, 1
-; GFX900-NEXT:    v_add3_u32 v52, v52, v51, s4
-; GFX900-NEXT:    v_or_b32_e32 v53, 0x400000, v51
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX900-NEXT:    v_cndmask_b32_e32 v51, v52, v53, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v52, 16, v51
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v49
-; GFX900-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v50
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc
-; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v51
+; GFX900-NEXT:    v_lshlrev_b32_e32 v50, 16, v51
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
 ; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v52, v51, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v51, v50, vcc
@@ -12875,26 +9936,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v52, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v51, v50, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX900-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX900-NEXT:    v_bfe_u32 v53, v52, 16, 1
-; GFX900-NEXT:    v_add3_u32 v53, v53, v52, s4
-; GFX900-NEXT:    v_or_b32_e32 v54, 0x400000, v52
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v53, v54, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v53, 16, v52
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v50
-; GFX900-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v51
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v50, v51, vcc
-; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v52
+; GFX900-NEXT:    v_lshlrev_b32_e32 v51, 16, v52
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
 ; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v53, 16, v3
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v53, v52, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc
@@ -12902,26 +9954,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v53, v54
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v52, v51, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX900-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX900-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX900-NEXT:    v_add3_u32 v54, v54, v53, s4
-; GFX900-NEXT:    v_or_b32_e32 v40, 0x400000, v53
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v54, v40, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v53
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v51
-; GFX900-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v52
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc
-; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v53
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v2
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v54, v53, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v53, v52, vcc
@@ -12929,26 +9972,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v53
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v54, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v53, v52, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX900-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX900-NEXT:    v_bfe_u32 v40, v54, 16, 1
-; GFX900-NEXT:    v_add3_u32 v40, v40, v54, s4
-; GFX900-NEXT:    v_or_b32_e32 v41, 0x400000, v54
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v40, v41, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v54
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v52
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v53, vcc
-; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v54
+; GFX900-NEXT:    v_lshlrev_b32_e32 v53, 16, v54
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v53
 ; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX900-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v40, v54, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
@@ -12956,26 +9990,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v54
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v41
 ; GFX900-NEXT:    v_cndmask_b32_e32 v40, v54, v53, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX900-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX900-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX900-NEXT:    v_add3_u32 v41, v41, v40, s4
-; GFX900-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v53
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v54
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc
-; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v40
+; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v41, v40, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
 ; GFX900-NEXT:    v_cndmask_b32_e32 v40, v40, v54, vcc
@@ -12983,22 +10008,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v42
 ; GFX900-NEXT:    v_cndmask_b32_e32 v41, v40, v54, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
-; GFX900-NEXT:    v_max_f32_e32 v41, v41, v41
-; GFX900-NEXT:    v_bfe_u32 v42, v41, 16, 1
-; GFX900-NEXT:    v_add3_u32 v42, v42, v41, s4
-; GFX900-NEXT:    v_or_b32_e32 v43, 0x400000, v41
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX900-NEXT:    v_cndmask_b32_e32 v41, v42, v43, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v42, 16, v41
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v54
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v40, vcc
-; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v41
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v41
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
@@ -13008,22 +10025,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v40, v55, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX900-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX900-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX900-NEXT:    v_add3_u32 v41, v41, v40, s4
-; GFX900-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
-; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v40
+; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v14
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
@@ -13033,22 +10042,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v55, v30, v14, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX900-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX900-NEXT:    v_bfe_u32 v40, v55, 16, 1
-; GFX900-NEXT:    v_add3_u32 v40, v40, v55, s4
-; GFX900-NEXT:    v_or_b32_e32 v41, 0x400000, v55
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v40, v41, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v14
-; GFX900-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX900-NEXT:    v_and_b32_e32 v30, 0xffff0000, v55
+; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v55
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
@@ -13058,22 +10059,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v55, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX900-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX900-NEXT:    v_bfe_u32 v55, v30, 16, 1
-; GFX900-NEXT:    v_add3_u32 v55, v55, v30, s4
-; GFX900-NEXT:    v_or_b32_e32 v40, 0x400000, v30
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX900-NEXT:    v_cndmask_b32_e32 v30, v55, v40, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v30
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX900-NEXT:    v_and_b32_e32 v29, 0xffff0000, v30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v30
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v29
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
@@ -13083,22 +10076,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v12
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v30, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX900-NEXT:    v_max_f32_e32 v29, v29, v29
-; GFX900-NEXT:    v_bfe_u32 v30, v29, 16, 1
-; GFX900-NEXT:    v_add3_u32 v30, v30, v29, s4
-; GFX900-NEXT:    v_or_b32_e32 v55, 0x400000, v29
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
-; GFX900-NEXT:    v_cndmask_b32_e32 v29, v30, v55, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v28
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX900-NEXT:    v_and_b32_e32 v28, 0xffff0000, v29
+; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v28
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
@@ -13108,22 +10093,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v29, v28
 ; GFX900-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX900-NEXT:    v_max_f32_e32 v28, v28, v28
-; GFX900-NEXT:    v_bfe_u32 v29, v28, 16, 1
-; GFX900-NEXT:    v_add3_u32 v29, v29, v28, s4
-; GFX900-NEXT:    v_or_b32_e32 v30, 0x400000, v28
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
-; GFX900-NEXT:    v_cndmask_b32_e32 v28, v29, v30, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v29, 16, v28
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX900-NEXT:    v_and_b32_e32 v27, 0xffff0000, v28
+; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v27
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
@@ -13133,22 +10110,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v10
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v28, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX900-NEXT:    v_max_f32_e32 v27, v27, v27
-; GFX900-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX900-NEXT:    v_add3_u32 v28, v28, v27, s4
-; GFX900-NEXT:    v_or_b32_e32 v29, 0x400000, v27
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
-; GFX900-NEXT:    v_cndmask_b32_e32 v27, v28, v29, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX900-NEXT:    v_and_b32_e32 v26, 0xffff0000, v27
+; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v26
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
@@ -13158,47 +10127,34 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v27, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX900-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX900-NEXT:    v_bfe_u32 v27, v26, 16, 1
-; GFX900-NEXT:    v_add3_u32 v27, v27, v26, s4
-; GFX900-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
-; GFX900-NEXT:    v_cndmask_b32_e32 v26, v27, v28, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX900-NEXT:    v_and_b32_e32 v25, 0xffff0000, v26
+; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v25
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc
+; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v26, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX900-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX900-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX900-NEXT:    v_add3_u32 v26, v26, v25, s4
-; GFX900-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX900-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX900-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
@@ -13208,22 +10164,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX900-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX900-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX900-NEXT:    v_add3_u32 v25, v25, v24, s4
-; GFX900-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX900-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
@@ -13233,22 +10181,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX900-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX900-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX900-NEXT:    v_add3_u32 v24, v24, v23, s4
-; GFX900-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX900-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
@@ -13258,26 +10198,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v23, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX900-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX900-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX900-NEXT:    v_add3_u32 v23, v23, v22, s4
-; GFX900-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX900-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX900-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
@@ -13287,22 +10215,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v4
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v22, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX900-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX900-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX900-NEXT:    v_add3_u32 v22, v22, v21, s4
-; GFX900-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX900-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
@@ -13312,22 +10232,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v21, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX900-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX900-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX900-NEXT:    v_add3_u32 v21, v21, v20, s4
-; GFX900-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX900-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
@@ -13337,22 +10249,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v20, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX900-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX900-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX900-NEXT:    v_add3_u32 v20, v20, v19, s4
-; GFX900-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX900-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
@@ -13362,22 +10266,14 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v19, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX900-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX900-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX900-NEXT:    v_add3_u32 v19, v19, v18, s4
-; GFX900-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX900-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
@@ -13387,22 +10283,13 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v18, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX900-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX900-NEXT:    v_bfe_u32 v18, v17, 16, 1
-; GFX900-NEXT:    v_add3_u32 v18, v18, v17, s4
-; GFX900-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
-; GFX900-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX900-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
+; GFX900-NEXT:    v_lshlrev_b32_e32 v16, 16, v17
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
-; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX900-NEXT:    v_perm_b32 v0, v54, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v53, v1, s4
 ; GFX900-NEXT:    v_perm_b32 v2, v52, v2, s4
@@ -13428,7 +10315,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_write_b32 a1, v41 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX950-NEXT:    scratch_load_dword v51, off, s32
+; GFX950-NEXT:    scratch_load_dword v50, off, s32
 ; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v34, 16, v30
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
@@ -13450,73 +10337,64 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
 ; GFX950-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v37, v39
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v37, v39
 ; GFX950-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
+; GFX950-NEXT:    v_and_b32_e32 v51, 0xffff0000, v23
 ; GFX950-NEXT:    v_cndmask_b32_e32 v37, v34, v31, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v36, v48
-; GFX950-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v37, v37, s0
+; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
+; GFX950-NEXT:    v_and_b32_e32 v52, 0xffff0000, v22
 ; GFX950-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v31
-; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
-; GFX950-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
+; GFX950-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
+; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
 ; GFX950-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
+; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
+; GFX950-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
-; GFX950-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v36, v36, s0
+; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
+; GFX950-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
 ; GFX950-NEXT:    v_cndmask_b32_e32 v31, v31, v34, vcc
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
+; GFX950-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX950-NEXT:    v_cndmask_b32_e32 v34, v35, v38, vcc
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
+; GFX950-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
 ; GFX950-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
-; GFX950-NEXT:    v_and_b32_e32 v48, 0xffff0000, v25
 ; GFX950-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v32, v32
-; GFX950-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
-; GFX950-NEXT:    v_and_b32_e32 v52, 0xffff0000, v22
-; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
-; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
-; GFX950-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
-; GFX950-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
-; GFX950-NEXT:    v_lshrrev_b32_e32 v34, 16, v51
-; GFX950-NEXT:    v_and_b32_e32 v37, 0xffff0000, v51
-; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v50
+; GFX950-NEXT:    v_and_b32_e32 v37, 0xffff0000, v50
+; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v35, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v34, v34, v32, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
+; GFX950-NEXT:    v_cndmask_b32_e32 v35, v35, v32, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v33, v37
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v33, v34, v32, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX950-NEXT:    v_max_f32_e32 v33, v33, v33
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v33, v33, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v33, v35, v32, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v32
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v32, v32, v34, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
-; GFX950-NEXT:    v_lshlrev_b32_e32 v34, 16, v36
+; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
 ; GFX950-NEXT:    v_and_b32_e32 v37, 0xffff0000, v28
 ; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
-; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v33, v36, v35, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
-; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
+; GFX950-NEXT:    v_and_b32_e32 v48, 0xffff0000, v25
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v33, v33, v38, vcc
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v34
+; GFX950-NEXT:    v_cndmask_b32_e32 v33, v36, v34, vcc
 ; GFX950-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX950-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
-; GFX950-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
 ; GFX950-NEXT:    s_nop 1
@@ -13530,9 +10408,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v36, v35, v34, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX950-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v36, v36, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v34
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
@@ -13557,9 +10432,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v37, v36, v35, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX950-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v37, v37, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v35
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
@@ -13584,9 +10456,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v38, v37, v36, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX950-NEXT:    v_max_f32_e32 v38, v38, v38
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v38, v38, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v36
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
@@ -13611,9 +10480,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v39, v38, v37, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX950-NEXT:    v_max_f32_e32 v39, v39, v39
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v39, v39, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v37
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
@@ -13638,9 +10504,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v48, v39, v38, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX950-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v48, v48, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v38
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
@@ -13656,18 +10519,15 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v39, v49, v48, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v39
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v49, v50
-; GFX950-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v48
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v49, v51
+; GFX950-NEXT:    v_lshrrev_b32_e32 v51, 16, v6
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v48, v39, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX950-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v49, v49, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v39
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
@@ -13682,84 +10542,75 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v48, v50, v49, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v48, v51, v49, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
+; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v48
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v49
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v50, v52
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v51, v52
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v52, 16, v5
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v49, v48, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX950-NEXT:    v_max_f32_e32 v50, v50, v50
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v50, v50, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v49, v48, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v48
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v49
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v51
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v49
 ; GFX950-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
-; GFX950-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
+; GFX950-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX950-NEXT:    v_lshrrev_b32_e32 v51, 16, v21
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v51, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v49
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v50
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v51, v49, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v52, v53
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v53, 16, v4
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v50, v49, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX950-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v52, v52, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v52, v51, v49, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v49
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v50
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v51
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
-; GFX950-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
+; GFX950-NEXT:    v_cndmask_b32_e32 v49, v49, v51, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v52
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
+; GFX950-NEXT:    v_and_b32_e32 v51, 0xffff0000, v4
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v52, 16, v20
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v53, v52, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v53, v52, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v53, v54
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v54, 16, v3
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v53, v52, v50, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX950-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v53, v53, s0
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v50
+; GFX950-NEXT:    v_cndmask_b32_e32 v53, v52, v51, vcc
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v51
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v52
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v50, v52, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX950-NEXT:    v_and_b32_e32 v52, 0xffff0000, v3
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v53, 16, v19
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
 ; GFX950-NEXT:    s_nop 1
@@ -13773,9 +10624,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v54, v53, v52, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX950-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v54, v54, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v52
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
@@ -13800,9 +10648,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v55, v54, v53, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX950-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v55, v55, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v53
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v53, v55, v53, vcc
@@ -13827,9 +10672,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v40, v55, v54, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX950-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v40, v40, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v54
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v54, v40, v54, vcc
@@ -13854,9 +10696,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v41, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
-; GFX950-NEXT:    v_max_f32_e32 v41, v41, v41
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v41, v41, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v55
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v55, v41, v55, vcc
@@ -13869,74 +10708,65 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v55, v41, v55, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v51
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v50
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v51, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v50, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v51, v51, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v51
+; GFX950-NEXT:    v_cndmask_b32_e32 v50, v50, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v50
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v40
 ; GFX950-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v40, v51, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX950-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v40, v40, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v40, v50, v15, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v15
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v51
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v50
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v51, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v40
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v14
+; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v50, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v40
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v14
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v30
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v30
+; GFX950-NEXT:    v_perm_b32 v15, v32, v15, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v30
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v51
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v30
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v50
 ; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v51, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX950-NEXT:    v_max_f32_e32 v51, v51, v51
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v51, v51, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v50, v30, v14, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v14
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v14, v51, v14, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v14, v50, v14, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v30
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v51
+; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v50
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v14, v51, v14, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v14, v50, v14, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v14, v31, v14, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v13
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v51, v30
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v50, v30
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX950-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v30, v30, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v13
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
@@ -13950,7 +10780,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v13, v33, v13, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v12
@@ -13960,9 +10790,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v30, v29
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX950-NEXT:    v_max_f32_e32 v29, v29, v29
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v29, v29, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v12
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
@@ -13976,7 +10803,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v12, v34, v12, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
@@ -13986,9 +10813,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v29, v28
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX950-NEXT:    v_max_f32_e32 v28, v28, v28
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v28, v28, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
@@ -14002,7 +10826,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v11, v35, v11, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v28, 16, v10
@@ -14012,9 +10836,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v28, v27
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX950-NEXT:    v_max_f32_e32 v27, v27, v27
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v27, v27, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v10
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
@@ -14028,7 +10849,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v10, v36, v10, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
@@ -14038,9 +10859,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v27, v26
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX950-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v26, v26, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
@@ -14054,7 +10872,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v9, v37, v9, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
@@ -14064,9 +10882,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v26, v25
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX950-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v25, v25, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
@@ -14080,7 +10895,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v8, v38, v8, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
@@ -14090,9 +10905,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v25, v24
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX950-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v24, v24, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
@@ -14106,7 +10918,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v7, v39, v7, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
@@ -14116,9 +10928,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v23
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v23, v23, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
@@ -14132,7 +10941,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v6, v48, v6, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
@@ -14142,9 +10951,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v23, v22
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX950-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v22, v22, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
@@ -14158,7 +10964,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v5, v49, v5, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v4
@@ -14168,9 +10974,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v22, v21
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX950-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v21, v21, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
@@ -14184,7 +10987,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v4, v51, v4, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
@@ -14194,9 +10997,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v21, v20
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX950-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v20, v20, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
@@ -14210,7 +11010,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v3, v52, v3, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
@@ -14220,9 +11020,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v20, v19
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX950-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v19, v19, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
@@ -14236,7 +11033,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v2, v53, v2, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v1
@@ -14246,9 +11043,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v19, v18
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX950-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v18, v18, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
@@ -14262,7 +11056,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v1, v54, v1, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
@@ -14272,2647 +11066,1887 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v18, v17
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX950-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v17, v17, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    v_perm_b32 v1, v54, v1, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v16
-; GFX950-NEXT:    v_perm_b32 v2, v53, v2, s0
-; GFX950-NEXT:    v_perm_b32 v3, v52, v3, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v16, 16, v17
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX950-NEXT:    v_perm_b32 v4, v50, v4, s0
-; GFX950-NEXT:    v_perm_b32 v5, v49, v5, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX950-NEXT:    v_perm_b32 v0, v55, v0, s0
-; GFX950-NEXT:    v_perm_b32 v6, v48, v6, s0
-; GFX950-NEXT:    v_perm_b32 v7, v39, v7, s0
-; GFX950-NEXT:    v_perm_b32 v8, v38, v8, s0
-; GFX950-NEXT:    v_perm_b32 v9, v37, v9, s0
-; GFX950-NEXT:    v_perm_b32 v10, v36, v10, s0
-; GFX950-NEXT:    v_perm_b32 v11, v35, v11, s0
-; GFX950-NEXT:    v_perm_b32 v12, v34, v12, s0
-; GFX950-NEXT:    v_perm_b32 v13, v33, v13, s0
-; GFX950-NEXT:    v_perm_b32 v14, v31, v14, s0
-; GFX950-NEXT:    v_perm_b32 v15, v32, v15, s0
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximumnum_v32bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    buffer_load_dword v53, off, s[0:3], s32
-; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v29
+; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GFX10-NEXT:    v_and_b32_e32 v33, 0xffff0000, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v28
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v53
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v32, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v31, v32, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v34, 16, v31
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v33, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v31, v32, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX10-NEXT:    v_max_f32_e32 v33, v33, v33
-; GFX10-NEXT:    v_bfe_u32 v34, v33, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v35, 0x400000, v33
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX10-NEXT:    v_add3_u32 v34, v34, v33, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v34, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v32
-; GFX10-NEXT:    v_and_b32_e32 v35, 0xffff0000, v30
-; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v34, v32, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v32, 0xffff0000, v33
-; GFX10-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v32
-; GFX10-NEXT:    v_and_b32_e32 v32, 0xffff0000, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v34, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v33, v32, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX10-NEXT:    v_lshlrev_b32_e32 v34, 16, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v32, v33, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v32
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v34, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v32, v33, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
-; GFX10-NEXT:    v_max_f32_e32 v34, v34, v34
-; GFX10-NEXT:    v_bfe_u32 v35, v34, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v36, 0x400000, v34
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_add3_u32 v35, v35, v34, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v35, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v33
+; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v11
 ; GFX10-NEXT:    v_and_b32_e32 v36, 0xffff0000, v29
-; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v35, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v32
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v33, 0xffff0000, v34
-; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v33
-; GFX10-NEXT:    v_and_b32_e32 v33, 0xffff0000, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v35, v32, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX10-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v34, v33, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v28
+; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v32
+; GFX10-NEXT:    v_cndmask_b32_e32 v34, v34, v38, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v26
+; GFX10-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v51, v51
+; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v33, v48, v39, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v36, 16, v33
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v35, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v33, v34, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
-; GFX10-NEXT:    v_max_f32_e32 v35, v35, v35
-; GFX10-NEXT:    v_bfe_u32 v36, v35, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v37, 0x400000, v35
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX10-NEXT:    v_add3_u32 v36, v36, v35, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v36, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v34
-; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v28
-; GFX10-NEXT:    v_lshrrev_b32_e32 v36, 16, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v27
+; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v23
+; GFX10-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX10-NEXT:    v_lshrrev_b32_e32 v66, 16, v22
+; GFX10-NEXT:    v_cndmask_b32_e32 v37, v35, v32, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v33
+; GFX10-NEXT:    v_lshrrev_b32_e32 v67, 16, v6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v70, 16, v4
+; GFX10-NEXT:    v_and_b32_e32 v71, 0xffff0000, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v36, v38, v34, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
+; GFX10-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX10-NEXT:    v_lshrrev_b32_e32 v85, 16, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
+; GFX10-NEXT:    v_cndmask_b32_e32 v35, v39, v33, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s5, v31, v38
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v26
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v53, v52, s6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v35
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s4, v39, v48
+; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v9
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v31, v31
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v25
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v49, v50
+; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v25
+; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v48, v52, v38, s6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v39, v39
+; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v38
+; GFX10-NEXT:    v_lshrrev_b32_e32 v53, 16, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v48
+; GFX10-NEXT:    v_cndmask_b32_e64 v39, v50, v49, s6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v31, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v39
+; GFX10-NEXT:    v_cndmask_b32_e64 v50, v49, v39, s6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v52, v52
+; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v24
+; GFX10-NEXT:    v_cndmask_b32_e64 v49, v54, v53, s6
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s6, v51, v55
+; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v52, v52
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v50
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v49
+; GFX10-NEXT:    v_cndmask_b32_e64 v52, v53, v49, s7
+; GFX10-NEXT:    v_and_b32_e32 v53, 0xffff0000, v23
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v55, v55
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s8, v31, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v55, v65, v64, s7
+; GFX10-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v53, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v68, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v53, v64, v55, s7
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v65, v65
+; GFX10-NEXT:    v_and_b32_e32 v64, 0xffff0000, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v69, 16, v53
+; GFX10-NEXT:    v_cndmask_b32_e64 v65, v67, v66, s7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v67, 16, v52
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v64, v64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v65
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s9, v54, v67
+; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v5
+; GFX10-NEXT:    v_cndmask_b32_e64 v64, v66, v65, s7
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s7, v68, v69
+; GFX10-NEXT:    v_lshrrev_b32_e32 v66, 16, v21
+; GFX10-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX10-NEXT:    v_and_b32_e32 v68, 0xffff0000, v4
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v54, v54
+; GFX10-NEXT:    v_lshrrev_b32_e32 v69, 16, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v64
+; GFX10-NEXT:    v_cndmask_b32_e64 v54, v67, v66, s10
+; GFX10-NEXT:    v_and_b32_e32 v67, 0xffff0000, v21
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v68, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v68, v70, v69, s10
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v67, v67
+; GFX10-NEXT:    v_lshlrev_b32_e32 v70, 16, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v82, 16, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v66, v66, v54, s10
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v71, v71
+; GFX10-NEXT:    v_lshrrev_b32_e32 v71, 16, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v81, 16, v66
+; GFX10-NEXT:    v_cndmask_b32_e64 v67, v69, v68, s10
+; GFX10-NEXT:    v_and_b32_e32 v69, 0xffff0000, v3
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s11, v70, v81
+; GFX10-NEXT:    v_lshlrev_b32_e32 v83, 16, v67
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v69, v69
+; GFX10-NEXT:    v_and_b32_e32 v70, 0xffff0000, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s12, v82, v83
+; GFX10-NEXT:    v_cndmask_b32_e64 v69, v80, v71, s10
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s10, v31, v51
+; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v19
+; GFX10-NEXT:    v_lshrrev_b32_e32 v80, 16, v18
+; GFX10-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v69
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v51, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v51, v71, v69, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v70, v70
+; GFX10-NEXT:    v_and_b32_e32 v71, 0xffff0000, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v83, 16, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v70, v81, v80, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v82, v82
+; GFX10-NEXT:    v_lshrrev_b32_e32 v81, 16, v17
+; GFX10-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v80, v80, v70, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v71, v71
+; GFX10-NEXT:    v_and_b32_e32 v71, 0xffff0000, v17
+; GFX10-NEXT:    v_cndmask_b32_e64 v82, v82, v81, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s14, v71, v71
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s13, v31, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v70
+; GFX10-NEXT:    v_lshlrev_b32_e32 v83, 16, v80
+; GFX10-NEXT:    v_cndmask_b32_e64 v71, v81, v82, s14
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s14, v31, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v82
+; GFX10-NEXT:    v_lshlrev_b32_e32 v81, 16, v71
+; GFX10-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s15, v31, v81
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v81, 16, v16
+; GFX10-NEXT:    v_cmp_u_f32_e64 s16, v31, v31
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v83, v83, v81, s16
+; GFX10-NEXT:    v_cmp_u_f32_e64 s16, v31, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v83
+; GFX10-NEXT:    v_cndmask_b32_e64 v81, v81, v83, s16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v84, 16, v81
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s16, v31, v84
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
+; GFX10-NEXT:    v_lshrrev_b32_e32 v84, 16, v30
+; GFX10-NEXT:    v_cmp_u_f32_e64 s17, v31, v31
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v85, v84, s17
+; GFX10-NEXT:    v_and_b32_e32 v85, 0xffff0000, v30
+; GFX10-NEXT:    v_cmp_u_f32_e64 s17, v85, v85
+; GFX10-NEXT:    v_lshlrev_b32_e32 v85, 16, v31
+; GFX10-NEXT:    v_cndmask_b32_e64 v84, v84, v31, s17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v86, 16, v84
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s17, v85, v86
+; GFX10-NEXT:    v_lshrrev_b32_e32 v86, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e64 v85, v84, v31, s17
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s17, 0, v31
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v85, v31, s17
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s17, 0, v84
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v31, v84, s17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v84, 16, v85
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s17, 0, v84
+; GFX10-NEXT:    v_cndmask_b32_e64 v84, v37, v32, s5
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0, v32
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v85, v31, s17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v85, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e64 v32, v84, v32, s5
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0, v37
+; GFX10-NEXT:    v_cndmask_b32_e64 v32, v32, v37, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v37, v36, v34, s4
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0, v34
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0, v39
+; GFX10-NEXT:    v_cndmask_b32_e64 v34, v37, v34, s4
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0, v36
+; GFX10-NEXT:    v_cndmask_b32_e64 v34, v34, v36, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v36, v35, v33, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v34, v33, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v35
-; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v34
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0, v38
 ; GFX10-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v35, v34, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX10-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v34, v35, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v36, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v34, v35, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX10-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX10-NEXT:    v_bfe_u32 v37, v36, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX10-NEXT:    v_add3_u32 v37, v37, v36, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v37, v38, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v35
-; GFX10-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
-; GFX10-NEXT:    v_lshrrev_b32_e32 v37, 16, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v35, v34, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v35, 0xffff0000, v36
-; GFX10-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v33, v33, v35, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v36
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v35
-; GFX10-NEXT:    v_and_b32_e32 v35, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v35
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v37, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v35, v36, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX10-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX10-NEXT:    v_bfe_u32 v38, v37, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX10-NEXT:    v_add3_u32 v38, v38, v37, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v38, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v36
-; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
-; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v36, 0xffff0000, v37
-; GFX10-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v36
-; GFX10-NEXT:    v_and_b32_e32 v36, 0xffff0000, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX10-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v37, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v36, v37, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v36
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v38, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v36, v37, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX10-NEXT:    v_max_f32_e32 v38, v38, v38
-; GFX10-NEXT:    v_bfe_u32 v39, v38, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v48, 0x400000, v38
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX10-NEXT:    v_add3_u32 v39, v39, v38, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v39, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v37
-; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v25
-; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v37, v36, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v38
-; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v35, v48, v38, s6
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s6, 0, v49
+; GFX10-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v35, v38, s4
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0, v48
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v38, v48, s4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v35
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s4, 0, v48
+; GFX10-NEXT:    v_cndmask_b32_e64 v48, v50, v39, s8
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0, v65
+; GFX10-NEXT:    v_cndmask_b32_e64 v39, v48, v39, s5
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0, v50
+; GFX10-NEXT:    v_cndmask_b32_e64 v39, v39, v50, s5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s5, 0, v50
+; GFX10-NEXT:    v_cndmask_b32_e64 v50, v52, v49, s9
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s9, 0, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v49, v50, v49, s6
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s6, 0, v52
+; GFX10-NEXT:    v_cndmask_b32_e64 v49, v49, v52, s6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s6, 0, v52
+; GFX10-NEXT:    v_cndmask_b32_e64 v52, v53, v55, s7
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s7, 0, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v55, v52, v55, s7
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s7, 0, v53
+; GFX10-NEXT:    v_cndmask_b32_e64 v53, v55, v53, s7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v52
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s7, 0, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v55, v64, v65, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v69
+; GFX10-NEXT:    v_cndmask_b32_e64 v36, v52, v53, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v65, v55, v65, s8
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0, v64
+; GFX10-NEXT:    v_cndmask_b32_e64 v64, v65, v64, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v65, v66, v54, s11
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0, v54
+; GFX10-NEXT:    v_cndmask_b32_e64 v54, v65, v54, s8
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0, v66
+; GFX10-NEXT:    v_cndmask_b32_e64 v54, v54, v66, s8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v65
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s8, 0, v66
+; GFX10-NEXT:    v_cndmask_b32_e64 v66, v67, v68, s12
+; GFX10-NEXT:    v_cndmask_b32_e64 v68, v66, v68, s9
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s9, 0, v67
+; GFX10-NEXT:    v_cndmask_b32_e64 v67, v68, v67, s9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v68, 16, v66
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s9, 0, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v68, v51, v69, s13
+; GFX10-NEXT:    v_cndmask_b32_e64 v69, v68, v69, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v51, v69, v51, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v69, v80, v70, s14
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v70
+; GFX10-NEXT:    v_cndmask_b32_e64 v70, v69, v70, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v80
+; GFX10-NEXT:    v_cndmask_b32_e64 v70, v70, v80, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v80, v71, v82, s15
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v82
+; GFX10-NEXT:    v_cndmask_b32_e64 v82, v80, v82, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v71
+; GFX10-NEXT:    v_cndmask_b32_e64 v71, v82, v71, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v82, v81, v83, s16
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v82
+; GFX10-NEXT:    v_cndmask_b32_e64 v83, v82, v83, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0, v81
+; GFX10-NEXT:    v_cndmask_b32_e64 v81, v83, v81, s10
+; GFX10-NEXT:    buffer_load_dword v83, off, s[0:3], s32
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v85, v85
+; GFX10-NEXT:    v_lshlrev_b32_e32 v85, 16, v14
+; GFX10-NEXT:    v_cmp_u_f32_e64 s11, v85, v85
+; GFX10-NEXT:    v_cndmask_b32_e64 v85, v14, v30, s11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v30
+; GFX10-NEXT:    v_cmp_u_f32_e64 s11, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v84
+; GFX10-NEXT:    v_cndmask_b32_e64 v87, v30, v85, s11
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s12, 0, v14
+; GFX10-NEXT:    v_cndmask_b32_e64 v30, v35, v38, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v35, v50, v49, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v65, v54, s8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v80
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v84, v32, s12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v37
+; GFX10-NEXT:    v_and_b32_e32 v84, 0xffff0000, v15
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s12, 0, v32
+; GFX10-NEXT:    v_cndmask_b32_e64 v32, v37, v34, s12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v34, v48, v39, s5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v68
+; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v69
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v37
-; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX10-NEXT:    v_lshrrev_b32_e32 v37, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v38, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v37, v38, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v39, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v37, v38, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX10-NEXT:    v_max_f32_e32 v39, v39, v39
-; GFX10-NEXT:    v_bfe_u32 v48, v39, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v49, 0x400000, v39
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX10-NEXT:    v_add3_u32 v48, v48, v39, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v48, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v38
-; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
-; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v38, v37, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v38, 0xffff0000, v39
-; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v38
-; GFX10-NEXT:    v_and_b32_e32 v38, 0xffff0000, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v39, v38, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v38, v39, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v38
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v48, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v38, v39, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX10-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX10-NEXT:    v_bfe_u32 v49, v48, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v50, 0x400000, v48
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX10-NEXT:    v_add3_u32 v49, v49, v48, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v49, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v39
-; GFX10-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
-; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v39, v38, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v48
-; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v37, v55, v64, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v84, v84
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v83
+; GFX10-NEXT:    v_and_b32_e32 v53, 0xffff0000, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v83
+; GFX10-NEXT:    v_cndmask_b32_e64 v64, v15, v83, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v66, v67, s9
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v86, v50, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v64
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v50, v54, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v55, v83, v64, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v39
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v49, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v39, v48, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX10-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX10-NEXT:    v_bfe_u32 v50, v49, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v51, 0x400000, v49
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX10-NEXT:    v_add3_u32 v50, v50, v49, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v50, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v48
-; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
-; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v48, v39, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v49
-; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e32 v39, v68, v51, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v53
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
-; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v48
+; GFX10-NEXT:    v_cndmask_b32_e32 v48, v69, v70, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v50, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v48, v49, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX10-NEXT:    v_max_f32_e32 v50, v50, v50
-; GFX10-NEXT:    v_bfe_u32 v51, v50, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v52, 0x400000, v50
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX10-NEXT:    v_add3_u32 v51, v51, v50, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v51, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v49
-; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v51, 16, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v49, v48, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v50
-; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v49
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v51, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v49, v50, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX10-NEXT:    v_max_f32_e32 v51, v51, v51
-; GFX10-NEXT:    v_bfe_u32 v52, v51, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v54, 0x400000, v51
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX10-NEXT:    v_add3_u32 v52, v52, v51, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v52, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v50
-; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
-; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v50, v49, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v50, 0xffff0000, v51
-; GFX10-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX10-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v51, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v50, v51, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v50
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v52, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v50, v51, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX10-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX10-NEXT:    v_bfe_u32 v54, v52, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v55, 0x400000, v52
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX10-NEXT:    v_add3_u32 v54, v54, v52, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v54, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v51
-; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v51, v50, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v52
-; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
-; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v54, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX10-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v51
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v54, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v51, v52, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX10-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX10-NEXT:    v_bfe_u32 v55, v54, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX10-NEXT:    v_add3_u32 v55, v55, v54, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v55, v64, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
-; GFX10-NEXT:    v_and_b32_e32 v64, 0xffff0000, v18
-; GFX10-NEXT:    v_lshrrev_b32_e32 v55, 16, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v55, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v52, v51, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v54
-; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v2
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v55, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v54, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v52, v54, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v52
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v52, v54, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX10-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX10-NEXT:    v_bfe_u32 v64, v55, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v65, 0x400000, v55
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX10-NEXT:    v_add3_u32 v64, v64, v55, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v64, v65, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
-; GFX10-NEXT:    v_and_b32_e32 v65, 0xffff0000, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v64, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v55
-; GFX10-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v64, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v54, v55, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v65
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v54, v55, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX10-NEXT:    v_max_f32_e32 v64, v64, v64
-; GFX10-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v65, v66, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v55
-; GFX10-NEXT:    v_and_b32_e32 v66, 0xffff0000, v16
-; GFX10-NEXT:    v_lshrrev_b32_e32 v65, 16, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v65, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v0
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v65, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX10-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v64, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v55, v64, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e32 v51, v53, v54, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v66
 ; GFX10-NEXT:    v_cndmask_b32_e32 v65, v55, v64, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX10-NEXT:    v_max_f32_e32 v65, v65, v65
-; GFX10-NEXT:    v_bfe_u32 v66, v65, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v67, 0x400000, v65
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX10-NEXT:    v_add3_u32 v66, v66, v65, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v65, v66, v67, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
+; GFX10-NEXT:    v_cndmask_b32_e32 v50, v80, v71, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v49, v51, v54, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v66, 16, v65
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v66, v64, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v65, v64, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v51
+; GFX10-NEXT:    v_cndmask_b32_e32 v49, v49, v53, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v64, v55, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v64, 0xffff0000, v65
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v65
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v54, v55, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v66, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v53, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v53, v15, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v53, v15, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX10-NEXT:    v_max_f32_e32 v64, v64, v64
-; GFX10-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v65, v66, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v65, 16, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v65, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v53, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v53, 0xffff0000, v64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
+; GFX10-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX10-NEXT:    v_cndmask_b32_e32 v52, v82, v81, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v65, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX10-NEXT:    v_perm_b32 v15, v31, v15, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v30, v14, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX10-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX10-NEXT:    v_bfe_u32 v64, v53, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v65, 0x400000, v53
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX10-NEXT:    v_add3_u32 v64, v64, v53, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v64, v65, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v64, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v30, 0xffff0000, v53
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v64, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX10-NEXT:    v_perm_b32 v14, v32, v14, 0x5040100
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v87
+; GFX10-NEXT:    v_cndmask_b32_e32 v51, v65, v54, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v85
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v13
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v54, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v28
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v87, v85, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
 ; GFX10-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX10-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX10-NEXT:    v_bfe_u32 v53, v30, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v64, 0x400000, v30
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_add3_u32 v53, v53, v30, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v53, v64, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v85
+; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v55, v53, v85, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v28, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v87
+; GFX10-NEXT:    v_cndmask_b32_e32 v28, v55, v87, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v55, v29, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
-; GFX10-NEXT:    v_lshrrev_b32_e32 v53, 16, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e32 v28, v53, v28, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v66, v65
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v54, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v29
 ; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v29, 0xffff0000, v30
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
-; GFX10-NEXT:    v_perm_b32 v13, v33, v13, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX10-NEXT:    v_max_f32_e32 v29, v29, v29
-; GFX10-NEXT:    v_bfe_u32 v30, v29, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v53, 0x400000, v29
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_add3_u32 v30, v30, v29, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v30, v53, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v29
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
-; GFX10-NEXT:    v_perm_b32 v12, v34, v12, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v53, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v53
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v54, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v27, v27, v11, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX10-NEXT:    v_max_f32_e32 v28, v28, v28
-; GFX10-NEXT:    v_bfe_u32 v29, v28, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v30, 0x400000, v28
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_add3_u32 v29, v29, v28, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v29, v30, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v26
+; GFX10-NEXT:    v_perm_b32 v13, v14, v13, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v53, v12, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v11
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v9
+; GFX10-NEXT:    v_perm_b32 v14, v31, v28, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v25
+; GFX10-NEXT:    v_cndmask_b32_e32 v29, v27, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v29, 16, v28
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX10-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v27, 0xffff0000, v28
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v29
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v26, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX10-NEXT:    v_perm_b32 v11, v35, v11, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX10-NEXT:    v_max_f32_e32 v27, v27, v27
-; GFX10-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v29, 0x400000, v27
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_add3_u32 v28, v28, v27, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v28, v29, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v53
+; GFX10-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v53, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v25, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26
 ; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v27
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
-; GFX10-NEXT:    v_perm_b32 v10, v36, v10, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX10-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX10-NEXT:    v_bfe_u32 v27, v26, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_add3_u32 v27, v27, v26, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v27, v28, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v53, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
+; GFX10-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v25, 0xffff0000, v26
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
-; GFX10-NEXT:    v_perm_b32 v9, v37, v9, 0x5040100
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v24
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX10-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX10-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v27, 0x400000, v25
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v8
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX10-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v24, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v23, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_add3_u32 v26, v26, v25, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
-; GFX10-NEXT:    v_perm_b32 v8, v38, v8, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX10-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_add3_u32 v25, v25, v24, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX10-NEXT:    v_perm_b32 v7, v39, v7, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v23
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
 ; GFX10-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX10-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX10-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_add3_u32 v24, v24, v23, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
+; GFX10-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
-; GFX10-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
+; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX10-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX10-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX10-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v24, 0x400000, v22
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v21, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_add3_u32 v23, v23, v22, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v20, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v22, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_add3_u32 v22, v22, v21, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v19, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
+; GFX10-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_perm_b32 v4, v50, v4, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v21, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX10-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX10-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_add3_u32 v21, v21, v20, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
-; GFX10-NEXT:    v_perm_b32 v3, v51, v3, 0x5040100
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX10-NEXT:    v_perm_b32 v3, v39, v3, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v16
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v20, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX10-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX10-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_add3_u32 v20, v20, v19, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v17, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v16, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_perm_b32 v2, v52, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v19, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX10-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX10-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_add3_u32 v19, v19, v18, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
-; GFX10-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v18, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX10-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX10-NEXT:    v_bfe_u32 v18, v17, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_add3_u32 v18, v18, v17, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX10-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v52, v0, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX10-NEXT:    v_perm_b32 v2, v48, v2, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v4, v15, v4, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v15, v49, v51, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_maximumnum_v32bf16:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v51, off, s32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v14
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v30
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v33, v33
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v34, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v13
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v11
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff0000, v9
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v67, 0xffff0000, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v16
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v55, v55
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s29, v85, v85
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v27
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v81, v81
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v52, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v82, v82
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v69, v69
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v80, 0xffff0000, v19
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s24
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v70, v70
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v15 :: v_dual_mov_b32 v49, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v82.l, v18.h, v52.l, s25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v52.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v80, v80
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v49
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v82.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v29
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v32, v32
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v38, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v134
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v33, v33
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s44, v112, v134
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v13.h, v29.h, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v10.h, v26.h, s8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v55, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v112.l, v82.l, v52.l, s44
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v66, v66
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v55.l, v29.h, v15.l, s3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v67, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v83, 0xffff0000, v1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v71, v71
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v33.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v39, v39
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v4.h, v20.h, s20
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v55.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v21
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v48, v48
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v66.l, v26.h, v33.l, s9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v80.l, v20.h, v39.l, s21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v39.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v67, v67
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v83, v83
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v80.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v3.h, v19.h, s22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v132
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v118
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v68, v68
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v5.h, v21.h, s18
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s42, v102, v132
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v81.l, v19.h, v48.l, s23
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s23, v85, v115
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v71.l, v21.h, v38.l, s19
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v102.l, v80.l, v39.l, s42
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v38.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v85.l, v55.l, v15.l, s23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v71.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v50
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v102, v102, v102 :: v_dual_lshlrev_b32 v101, 16, v101
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s26
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s26, v96, v118
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v131
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v85, v85, v85
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v30.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v96.l, v66.l, v33.l, s26
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v35, v35
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v37, v37
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v39, v39
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v49, v49
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v51, v51
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v53, v53
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v54, v54
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v65, v65
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v67, v67
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v69, v69
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v71, v71
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s40, v86, v86
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v32.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v84, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v54, v54
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s41, v101, v131
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v30
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v34, v34
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v84, v84
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v9.h, v25.h, s10
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v96, v96, v96
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v31, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v14.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v101.l, v71.l, v38.l, s41
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v54.l, v30.h, v14.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v35, v35
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v8.h, v24.h, s12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v67.l, v25.h, v34.l, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v54.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v101
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v27
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v64, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v114
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v65, v65
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v12.h, v28.h, s4
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v34.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v68.l, v24.h, v35.l, s13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v67.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s22, v84, v114
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v101, v101, v101
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v36, v36
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v7.h, v23.h, s14
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v37, v37
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v6.h, v22.h, s16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v64.l, v28.h, v31.l, s5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v11.h, v27.h, s6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v35.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v69.l, v23.h, v36.l, s15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v68.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v119
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v84.l, v54.l, v14.l, s22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v31.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v65.l, v27.h, v32.l, s7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v36.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v70.l, v22.h, v37.l, s17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v83.l, v17.h, v53.l, s27
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v69.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v128
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s27, v97, v119
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v37.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v36, v36
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v38, v38
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v48, v48
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v50, v50
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v52, v52
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v64, v64
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v66, v66
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v68, v68
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v70, v70
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v80, v80
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v81, v81
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v83, v83
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v9.h, v25.h, s11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v85.l, v16.h, v54.l, s40
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v65.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v14
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v82, v82
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s28, v84, v84
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v69.l, v24.h, v38.l, s14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v54.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s40, v86, v118
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v85.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v30
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v87, v87
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s41, v96, v96
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v36.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v39.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v50.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v51.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v64.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v65.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v67.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v68.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v69.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v71.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v80.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v81.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v145.l, v82.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v116
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v129
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s28, v98, v128
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v97.l, v67.l, v34.l, s27
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v84, v84, v84 :: v_dual_lshlrev_b32 v87, 16, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v117
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v130
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s24, v86, v116
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s29, v99, v129
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v98.l, v68.l, v35.l, s28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v114, v84, 16, 1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v48.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v81.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s25, v87, v117
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s40, v100, v130
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v86.l, v64.l, v31.l, s24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v99.l, v69.l, v36.l, s29
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v97, v97, v97 :: v_dual_lshlrev_b32 v98, 16, v98
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v84
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v130, v96, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v114, v114, v84, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v84, v84
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v133
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v87.l, v65.l, v32.l, s25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v100.l, v70.l, v37.l, s40
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v98, v98, v98 :: v_dual_lshlrev_b32 v99, 16, v99
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v131, 0x400000, v96
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v132, v97, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v114, v114, v115, s22
-; GFX11-TRUE16-NEXT:    v_add3_u32 v130, v130, v96, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v96, v96
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s43, v103, v133
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s42, v97, v97
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v32.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v37.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v52.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v53.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v86, v86, v86 :: v_dual_max_f32 v99, v99, v99
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v133, 0x400000, v97
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v134, v98, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v132, v132, v97, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v96, v130, v131, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v97, v97
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v87, v87, v87 :: v_dual_max_f32 v100, v100, v100
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v116, v85, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v118, v86, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v144, 0x400000, v98
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v145, v99, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v134, v134, v98, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v97, v132, v133, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v98, v98
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v117, 0x400000, v85
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v86
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v128, v87, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v146, 0x400000, v99
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v147, v100, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v116, v116, v85, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v85, v85
-; GFX11-TRUE16-NEXT:    v_add3_u32 v118, v118, v86, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v86, v86
-; GFX11-TRUE16-NEXT:    v_add3_u32 v145, v145, v99, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v98, v134, v144, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v99, v99
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v15.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v129, 0x400000, v87
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v84, 0x400000, v100
-; GFX11-TRUE16-NEXT:    v_add3_u32 v128, v128, v87, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v87, v87
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s23
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s24
-; GFX11-TRUE16-NEXT:    v_add3_u32 v147, v147, v100, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v99, v145, v146, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v100, v100
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v32.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v116.h, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v118.h, v31.l, s5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0, v34.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v128.h, v32.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v35.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0, v36.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v97.h, v34.l, s7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.h, v96.h, v33.l, s6
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v85, v101, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v98.h, v35.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v99.h, v36.l, s8
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v86, 0x400000, v101
-; GFX11-TRUE16-NEXT:    v_add3_u32 v85, v85, v101, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v84, v147, v84, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v101, v101
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v14.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0, v38.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0, v54.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0, v55.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v85, v85, v86, s22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v114.h, v14.l, s0
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v87, v102, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0, v64.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v102
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v102, v102
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v54.l, s11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v87, v87, v102, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0, v39.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v64.l, s13
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0, v65.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v128
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v86, v87, v115, s22
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v87, 0xffff0000, v114
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v53.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v83.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v103.l, v81.l, v48.l, s43
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v116
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v65.l, s14
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v102
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v101
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v113, 16, v113
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v83.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v147.l, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v129
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v135
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v144
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v144, 16, v145
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s63, v116, v86
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v86.l, v55.l, v32.l, s40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0, v55.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v147
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s42, v87, v118
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s43, v96, v119
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s45, v98, v129
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s56, v101, v132
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s59, v112, v135
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s60, v113, v144
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v86.l, v32.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v86.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v35.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0, v36.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0, v39.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0, v50.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0, v51.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s46, v99, v130
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s61, v114, v145
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s62, v115, v146
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v96.l, v65.l, v34.l, s43
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v98.l, v67.l, v36.l, s45
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v101.l, v70.l, v39.l, s56
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v112.l, v81.l, v50.l, s59
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v113.l, v82.l, v51.l, s60
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v55.l, s16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v118
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v33.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0, v37.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v48.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0, v52.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0, v53.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s23, 0, v70.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s26, 0, v81.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v87.l, v64.l, v33.l, s42
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v99.l, v68.l, v37.l, s46
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v114.l, v83.l, v52.l, s61
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v115.l, v84.l, v53.l, s62
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v96.l, v34.l, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v98.l, v36.l, s5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v101.l, v39.l, s8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v101.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v112.l, v50.l, s11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v112.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v113.l, v51.l, s12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v113.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v55
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v38.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v49.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s44, v97, v128
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v116.l, v85.l, v54.l, s63
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v87.l, v33.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v99.l, v37.l, s6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.h, v114.l, v52.l, s13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v114.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v115.l, v53.l, s14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v115.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v39
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v35.h, v81.l, s26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v35.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v101
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0, v67.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0, v68.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v131, 0xffff0000, v98
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v103, v103, v103
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0, v70.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v132, 0xffff0000, v84
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s17
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v131
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v117, v103, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v103
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v132
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v96
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0, v80.l
-; GFX11-TRUE16-NEXT:    v_add3_u32 v117, v117, v103, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v97
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0, v69.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v115
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v85
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v133
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s57, v102, v133
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v97.l, v66.l, v35.l, s44
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v116.l, v54.l, s15
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v116.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s12, 0, v51
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0, v48.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0, v65.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0, v66.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s47, v100, v131
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s58, v103, v134
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v102.l, v71.l, v48.l, s57
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v96.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v97.l, v35.l, s4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s13, 0, v52
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s14, 0, v53
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0, v38.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0, v49.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0, v64.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s24, 0, v71.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v100.l, v69.l, v38.l, s47
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v103.l, v80.l, v49.l, s58
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v87.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v102.l, v48.l, s9
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v102.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v30.h, v65.l, s18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v128
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v66.l, s19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v129
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s25, 0, v80.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s41, 0, v85.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v38.l, s7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v103.l, v49.l, s10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v103.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v14.h, v64.l, s17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v119
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.h, v34.h, v71.l, s24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v48
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v65
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v66
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v86.l, v13.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v49
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.l, v37.h, v85.l, s41
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v64
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v71
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v96.l, v30.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v97.l, v32.l, s4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0, v67.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0, v68.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v98.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v99.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v87.l, v38.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v102.l, v38.h, s9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v32.h, v67.l, s20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v130
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v131
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s29, 0, v84.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v70
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v67
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s27, 0, v82.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v68
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v37.l, v84.l, s29
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v101.l, v34.l, s8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s28, 0, v83.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s10, 0, v80
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v36.l, v82.l, s27
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.h, v115.l, v37.l, s14
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s22, 0, v69.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v36.h, v83.l, s28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.h, v103.l, v35.l, s10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.h, v113.l, v36.l, s12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v100.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v33.h, v69.l, s22
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s11, 0, v81
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.h, v114.l, v48.l, s13
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v132
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v69
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v33.h, s7
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v51
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v50.h, v51.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v118
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.h, v31.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v84.h, v37.l, s4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v85.h, v38.l, s9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v100
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v32.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v129
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v130
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v99
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v35, v36
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v14.h, v55.l, s12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v86.h, v39.l, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v86
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0, v31.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v32.l, v31.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v87
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.h, v118.h, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v39
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v36.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v114.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v32.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v31.h, v66.l, s15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.h, v128.h, v15.h, s2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v32.h, v67.l, s16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.h, v86.h, v35.l, s9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v96.h, v36.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v33.h, v69.l, s18
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v54, v38, v38
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v34.h, v71.l, s20
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v116.h, v35.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v98.h, v33.l, s5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.h, v84.h, v34.l, s7
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v55, v54, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v97.h, v37.l, s4
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v130
-; GFX11-TRUE16-NEXT:    v_add3_u32 v55, v55, v54, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v99.h, v37.h, s6
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v55, v64, s11
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v64, v112, v112
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v54.h, v31.l, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v54
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v32.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX11-TRUE16-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.h, v85.h, v38.l, s8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v54.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v135
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v103, v103
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v113, v54
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v55, v117, v119 :: v_dual_and_b32 v54, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v31
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v83.l, v53.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v55.h, v48.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v81.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v15.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v0.h, v16.h, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v50.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v15.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.h, v98.l, v32.h, s5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v51.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v99.l, v33.l, s6
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v65, v66, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v52.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v67
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v16.h, v15.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v81.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v54.h, v52.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v64, v64, v64
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v50.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v52, v53
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v31.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v82.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v65
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v67, v64, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v66
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v82.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX11-TRUE16-NEXT:    v_add3_u32 v66, v67, v64, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v67, 0x400000, v64
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v52, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.h, v55.h, v14.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v64, v66, v67, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v53.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v51
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v68
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v64.h, v53.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.l, v50.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v52
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v51.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v32.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v50.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v38, v53
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.h, v112.l, v39.l, s11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v51.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v29
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v50
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v30.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v50.l, v51.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v83.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.h, v54.h, v32.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v15.h, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v117, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v31.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v52, v52
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v51.l, v33.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v34.l, v83.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v64
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v66, v52, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v49.l, v30.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX11-TRUE16-NEXT:    v_add3_u32 v49, v66, v52, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v52
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v65, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v34.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v64.h, v32.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.l, v33.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v30.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v49, v49, v53, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v34.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v29
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v32.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v51, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v52
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v28
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v52
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v31.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v52, v55, v55 :: v_dual_and_b32 v53, 0xffff0000, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v30.l, v34.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v29.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v31.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v54, v52, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v12
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX11-TRUE16-NEXT:    v_add3_u32 v54, v54, v52, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v52, v52
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v29.l, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v28
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v52, v54, v65, s0
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v55, 0x400000, v53
-; GFX11-TRUE16-NEXT:    v_add3_u32 v54, v54, v53, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v15.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v52.h, v33.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v64
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v49, v54, v55, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v34.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v52
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v12.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v14.l, s0
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v34.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v64
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v65, v53, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v49
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v64, v65, v53, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v53
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v66
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v28.l, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v53, v64, v65, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v52, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v29.l, v13.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v27
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v30.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.h, v30.l, v13.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v51, v50
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v14.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.h, v53.h, v13.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v29.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v27.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v12.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.h, v11.h, v29.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v53
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v52.h, v12.h, s0
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v55, v49, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v29
-; GFX11-TRUE16-NEXT:    v_add3_u32 v27, v55, v49, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, 0x400000, v49
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v54, v52
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v53.h, v11.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v26.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v29, v27, v29, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v27.l, v13.l, v11.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v29.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v52, v49
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v28.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v27, v12, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v25.l, v25.l, v9.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v28.l, s1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v27
-; GFX11-TRUE16-NEXT:    v_add3_u32 v28, v28, v27, 0x7fff
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.h, v31.l, v12.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v26
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v28.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v31.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v27, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v29.h, v9.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v52, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v27, v28, v53, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v8
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v29, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v27.h, v11.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v24
-; GFX11-TRUE16-NEXT:    v_add3_u32 v29, v29, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v51, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.h, v10.h, v28.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v25
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v26.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v51, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v31.l, v10.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v25.l, v12.l, v10.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v9.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v25.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v27, v26
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v12.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v29, v49, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v10.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v8.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v24.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v12.h, v10.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v28, v28
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v11.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v13.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v26.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v10, 16, 1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v52, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v26
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v29, v29
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v52, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v26
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v10.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v23.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v24.h, v9.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v27.h, v8.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v25.l, s2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v12.h, v9.h, s1
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v27, v10, v10 :: v_dual_and_b32 v26, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v27, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v12, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT:    v_add3_u32 v23, v25, v27, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v27
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v22.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v24.h, v7.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v11.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v11.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v22.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v25, v8, v8 :: v_dual_lshlrev_b32 v24, 16, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v11.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v23, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v25, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v12.l, v6.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v26
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v25, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v22.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v10.l, v7.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v24, v25, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v10.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v22.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v21, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v6.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v11.h, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v10.l, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v10.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v12, v11
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v9.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v21
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v9.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v22, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v8, v9, v21 :: v_dual_lshlrev_b32 v9, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v8.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v9, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v11.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v21, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v6, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v7.l, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v19
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v12, v20, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_and_b32 v11, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v9.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v10.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v8.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v8.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v18.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v6, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v9.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v11, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v18.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v19, v20, s3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.h, v4.l, s3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v20
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v12, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v6.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v4.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v10, v4
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v16.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v18.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v7.l, v4.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v18.l, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v17
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v12, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v16, v12
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v19, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v18, v17
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v17, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v12.h, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v16, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s0
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v10, v3, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v16
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v5, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.h, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v5, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v10, v17, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v18, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v9.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.l, v12.h, v1.h, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v11.h, v0.h, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v5.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.l, v7.h, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.l, v2.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v8.l, v1.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v6.l, v0.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v9.l, v2.h, s3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, v29
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, v50 :: v_dual_mov_b32 v4, v48
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, v13 :: v_dual_mov_b32 v1, v30
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, v51
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, v39 :: v_dual_mov_b32 v6, v38
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v7, v37 :: v_dual_mov_b32 v8, v35
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v9, v33 :: v_dual_mov_b32 v10, v32
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v11, v31 :: v_dual_mov_b32 v12, v36
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v13, v34
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v49 :: v_dual_mov_b32 v2, v48
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, v39 :: v_dual_mov_b32 v4, v38
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, v36 :: v_dual_mov_b32 v6, v35
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v7, v34 :: v_dual_mov_b32 v8, v33
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v9, v32 :: v_dual_mov_b32 v10, v31
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v11, v30 :: v_dual_mov_b32 v12, v37
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v32bf16:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v12
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v51, 16, v28
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v30
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-FAKE16-NEXT:    scratch_load_b32 v50, off, s32
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v29
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v30
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v83, 16, v24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v52, v52, v51, s1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v13
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v84, 16, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v34, v35, v33 :: v_dual_and_b32 v49, 0xffff0000, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v38, v38
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v87, 16, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v52, v51, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v96, 16, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v99, 16, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v39, v64, v55 :: v_dual_and_b32 v70, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v48, v48, v39, s0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v100, 16, v6
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v115, 16, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v68, v67, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v30
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v80, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v82, v82
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v18
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v80, v71, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v82, v82
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v84, v83, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v84, v83, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v96, v87, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v26
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v100, v99, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v96, v87, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v25
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v146, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v100, v99, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v68, v112, v103 :: v_dual_and_b32 v81, 0xffff0000, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v114, v114
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v116, v115, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v118, v118
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v147, 16, v16
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v34, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v54, v54
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v112, v103, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v114, v114
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v128, v119, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v14
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v116, v115, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v118, v118
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v113, 0xffff0000, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v64, v64, v55, s2
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v128, v119, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v130, v130
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v82, v132, v131 :: v_dual_and_b32 v113, 0xffff0000, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v20
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v132, v131, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v134, v134
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v84, v144, v135 :: v_dual_and_b32 v117, 0xffff0000, v20
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v102, v144, v135 :: v_dual_and_b32 v133, 0xffff0000, v18
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v146, v146
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v96
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v34, v147, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v27
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v33
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v14, v30 :: v_dual_and_b32 v97, 0xffff0000, v23
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v37, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v97, 0xffff0000, v23
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v51
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v39, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v55, v64 :: v_dual_lshlrev_b32 v130, 16, v51
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v49, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v34
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s3, v66, v66
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v68, v68, v67, s3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v68, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v52 :: v_dual_lshlrev_b32 v118, 16, v67
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v68
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v80
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v67
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v54, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v80, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v97, v97
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v64 :: v_dual_lshlrev_b32 v128, 16, v83
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v84
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v82 :: v_dual_lshlrev_b32 v134, 16, v83
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v101, v101
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v97, v99, v66, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v16
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v99, v84, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v113, v113
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v103, v68, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v103, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v99
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v117, v117
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v39
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v101, v115, v70, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v36
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v115, v96, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v129, v129
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v34
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v38
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v49
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v87
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v103, v119, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v82
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v115, v119, v98 :: v_dual_lshlrev_b32 v146, 16, v113
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v133, v133
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v71
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v103
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v131, v82, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v86, v114
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v52
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v99
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v113
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v33, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v36, v115
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v48, v116
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v86, v86, v86 :: v_dual_lshlrev_b32 v85, 16, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v51, v37, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v114, v86, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v115, 0x400000, v86
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v55
-; GFX11-FAKE16-NEXT:    v_add3_u32 v114, v114, v86, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v117
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v48, v48
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v118
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v66
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v118, v48, 16, 1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v49, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v119
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v70
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v53, v53, v53 :: v_dual_lshlrev_b32 v102, 16, v80
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v119, 0x400000, v48
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v71, v52, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v81, v128
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v68
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v128, v53, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v118, v118, v48, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v69
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v85, v129
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v82
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v53
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v69, v69, v69
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v81
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v64, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v128, v128, v53, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v81, v81, v81
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s2
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v144, 0x400000, v81
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v97
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v85, v85, v85
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v96, v130
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v146, v85, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v147, 0x400000, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v97, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v98, v131
-; GFX11-FAKE16-NEXT:    v_add3_u32 v146, v146, v85, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v101
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v99, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v100, v132
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v132, v69, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v96, v96, v96
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v101, v70, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v102, v133
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v133, 0x400000, v69
-; GFX11-FAKE16-NEXT:    v_add3_u32 v132, v132, v69, 0x7fff
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v98, v98, v98
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v102, v103, v80, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v112, v134
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v134, v81, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v48, v98, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v98
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v82, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v86, v96, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v134, v134, v81, 0x7fff
-; GFX11-FAKE16-NEXT:    v_add3_u32 v48, v48, v98, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v114, v115, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v86, v86, v96, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v65, v65, v65
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v130, v65, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v131, 0x400000, v65
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_add3_u32 v130, v130, v65, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v130, v131, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v132, v133 :: v_dual_lshlrev_b32 v36, 16, v36
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v134, v144, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v116, v36, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v36
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v36, v36
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v130, 16, v81
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-FAKE16-NEXT:    v_add3_u32 v116, v116, v36, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v36, 0x400000, v96
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v146, v147, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v96, v96
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v100, v100, v100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v86, v36, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v115, v100, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v100
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v86, 16, v114
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v98, 16, v116
-; GFX11-FAKE16-NEXT:    v_add3_u32 v115, v115, v100, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v48, v53, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v36
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v116
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v128
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v36
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v115, v117, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v34
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v118
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v134, 16, v48
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v146, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v86, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v35
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v96, 0xffff0000, v114
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v114, 16, v118
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v118, 16, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v98, v35, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v37
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v128
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v69
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v144, 0xffff0000, v48
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v114, v37, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v39
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v116, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v49
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v118, v49, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v37, 16, v48
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v117, v131, v100, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v86
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v119, v135, v102, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v38, v147, v34 :: v_dual_lshlrev_b32 v49, 16, v52
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e64 s0, v49, v130
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v66, v30, v54 :: v_dual_lshlrev_b32 v53, 16, v64
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v35
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v30
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v70
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v117
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v130, v35, v36 :: v_dual_lshlrev_b32 v129, 16, v39
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v37, v129
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v129, v51, v52, s0
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v37, v39, v48 :: v_dual_lshlrev_b32 v118, 16, v102
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v55
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v131
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v55, v64 :: v_dual_lshlrev_b32 v50, 16, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v71
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v132
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v68, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v133
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v87
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v65
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v81, v134
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v85, v135
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v103
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v97, v144
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v97, v99, v84 :: v_dual_lshlrev_b32 v114, 16, v98
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v101, v145
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v147, 16, v115
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v101, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v97
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v112, v146
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v96, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v114, v147
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v119
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v38
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v115, v98, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v116, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v116, v117, v100, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v118, v30
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v118, v119, v102, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v128, v49
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v128, v38, v34, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v36
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v36, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v48
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v37, v48, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v128, v52, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v130, v54, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v129, v52, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v131, v64, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v147, v32, v52 :: v_dual_and_b32 v64, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v66
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v102, v102, v102
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v132, v66, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v129
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v53, v64, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v68
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v65
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v119, v102, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v102
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v134, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v69
-; GFX11-FAKE16-NEXT:    v_add3_u32 v119, v119, v102, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v147, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v65, v68, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v70
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v81
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v146, v70, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v33
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v34, v33, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v38
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v35, v35, v38 :: v_dual_lshlrev_b32 v34, 16, v147
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v69, v70, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v80
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v81, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v82
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v85, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v84
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v97, v84, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v86
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v101, v86, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v96
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v112, v96, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v98
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v98, v114, v98 :: v_dual_lshlrev_b32 v131, 16, v53
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v100, v116, v100 :: v_dual_lshlrev_b32 v133, 16, v69
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v35
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v35 :: v_dual_lshlrev_b32 v135, 16, v85
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v102
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v102, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v39
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v39 :: v_dual_lshlrev_b32 v145, 16, v101
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v34
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v128, v34, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v51
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v51 :: v_dual_and_b32 v85, 0xffff0000, v85
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v34, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v52, v147, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v49, v51, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v37, v55, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v128
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v64, v55, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v67
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v39, v67, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v67, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v71
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v48, v71, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v70, v71 :: v_dual_lshlrev_b32 v147, 16, v114
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v83
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v49, v83, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v80, v83, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v87
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v49, v54, v87 :: v_dual_lshlrev_b32 v34, 16, v34
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v97
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v31, v97, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v82, v87, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v99
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v34, v34, v34 :: v_dual_cndmask_b32 v55, v32, v99
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v96
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v54, v34, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v34
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v86, v33, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v100
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v54, v54, v34, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v98, v35, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v115
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v114, v36, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v117
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v116, v37, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v65
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v38, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v69
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v69, 16, v16
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v128, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v81
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v130, v48, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e64 vcc_lo, 0, v147
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v147, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v131, v49, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v52, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v84, v99, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v103
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v86, v103 :: v_dual_lshlrev_b32 v30, 16, v130
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v113
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v96, v113, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v115
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v98, v115, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v117
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v81
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v100, v117, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v119
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v35, v119, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v38
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v34, v38, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v37, v36, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v31
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v129, v39, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v131
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v53, v49, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v132
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v31
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v65, v55, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v133
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v54
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v48, v132, v51 :: v_dual_lshlrev_b32 v51, 16, v112
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v69, v64, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v134
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v81, v67, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v135
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v85, v68, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v144
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v134, v55, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v84
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v135, v84, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v119, v129, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v101
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v97, v70, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v145
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v101, v71, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v15, v31, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v31
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v148, 16, v116
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v55
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v112
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v33
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v146
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v112, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v118
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v31, v55 :: v_dual_lshlrev_b32 v64, 16, v52
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v147
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v114, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v148
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v116, v83, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v50, v64
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v52, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v67
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v64
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v53, v55, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v102
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v65
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v118, v84, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v33
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v55
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v65, v55, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v67, v51, v51
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v66, 16, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v64, v101, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v54, v84, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v80
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v66, v80, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v67, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v51, v146, v64 :: v_dual_lshlrev_b32 v80, 16, v15
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v103
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v53, v67, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v65, v103, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v65, 0x400000, v67
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v65 :: v_dual_and_b32 v68, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v70, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v16
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v52, v66, v64 :: v_dual_lshlrev_b32 v55, 16, v55
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v69, v67, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v82
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v67
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v68, v55, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v65, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v53, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v67
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v128, v86, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v68
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v65, v53, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_add3_u32 v66, v68, v55, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v68, 0x400000, v55
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v66, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v70
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v64, v67, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v15, v50, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v50
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v113
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v55
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v66, v66, v66
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v113 :: v_dual_lshlrev_b32 v80, 16, v68
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v68 :: v_dual_and_b32 v15, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v84
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v70, v84, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v65, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v80, v71
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v54, v66, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v71, 0x400000, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v30
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v50, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-FAKE16-NEXT:    v_add3_u32 v54, v54, v66, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v14, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v30, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v70, v53, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v53, 16, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v69
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v67
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v67, v53, v67 :: v_dual_and_b32 v54, 0xffff0000, v54
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v70, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v29
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v30, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v71, v71
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v65, v65, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v13, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v55, v65, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v80, 0x400000, v65
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v66
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v29 :: v_dual_lshlrev_b32 v64, 16, v54
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v66, v54, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v55, v55, v65, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v80, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v71
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v29, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v66, v13, v13 :: v_dual_cndmask_b32 v29, v67, v64
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v55
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v53, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v68
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v66, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v64, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v70, v67
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v50, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v50, v53, v66, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v53 :: v_dual_and_b32 v55, 0xffff0000, v55
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v53, v54, v54 :: v_dual_lshlrev_b32 v54, 16, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v28
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v50
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v53
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v64, v29 :: v_dual_and_b32 v50, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v64, v53, 16, 1
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v53, v54, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v69
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v64, v64, v53, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v55, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v27
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v64, v67, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v68
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v28, v12 :: v_dual_lshlrev_b32 v67, 16, v11
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v27, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v54, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v53
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v64
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v66
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v71
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v30, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v67, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v66, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v50, v54, 16, 1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v50, v50, v54, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v64, v64, v64 :: v_dual_cndmask_b32 v27, v55, v27
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v65, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v26
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v54, v64, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v67 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v65, v26, v10 :: v_dual_lshlrev_b32 v68, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v54, v64, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v30, v55, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v50
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v66
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v64
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v28
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v29, v13 :: v_dual_lshlrev_b32 v66, 16, v12
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v54 :: v_dual_lshlrev_b32 v64, 16, v55
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v66, v65
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v27
+; GFX11-FAKE16-NEXT:    v_perm_b32 v14, v14, v53, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v28, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v29
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v30, v12 :: v_dual_lshlrev_b32 v67, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v67, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v54, 16, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v28
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v9
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v27, v11 :: v_dual_lshlrev_b32 v28, 16, v54
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v26
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v26 :: v_dual_lshlrev_b32 v29, 16, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v54, v28
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v26
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v10
+; GFX11-FAKE16-NEXT:    v_perm_b32 v12, v34, v12, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v26, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v66
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v30, v54, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX11-FAKE16-NEXT:    v_add3_u32 v30, v30, v54, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v25, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v50, v50, v50 :: v_dual_lshlrev_b32 v55, 16, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v30, v64 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v50, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v24
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v30
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v8
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v53, v50, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v30
-; GFX11-FAKE16-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
+; GFX11-FAKE16-NEXT:    v_perm_b32 v11, v35, v11, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v24, v8 :: v_dual_lshlrev_b32 v55, 16, v23
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v53, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v65
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v23
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v65 :: v_dual_lshlrev_b32 v53, 16, v54
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v53, v53, v53 :: v_dual_cndmask_b32 v10, v28, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v25, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v53, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v54, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v23, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11-FAKE16-NEXT:    v_add3_u32 v25, v28, v53, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v28, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v22
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v26, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-FAKE16-NEXT:    v_perm_b32 v10, v36, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v27, v9 :: v_dual_lshlrev_b32 v28, 16, v23
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v8
+; GFX11-FAKE16-NEXT:    v_perm_b32 v9, v37, v9, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v28, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v54, v9 :: v_dual_lshlrev_b32 v64, 16, v6
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v53, v28, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v28
-; GFX11-FAKE16-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
-; GFX11-FAKE16-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v26
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v8 :: v_dual_lshlrev_b32 v25, 16, v22
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v23, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v50, v8 :: v_dual_lshlrev_b32 v55, 16, v22
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v22, v6 :: v_dual_lshlrev_b32 v54, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v24 :: v_dual_lshlrev_b32 v53, 16, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v28
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v24, v30, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v50, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v25, v7 :: v_dual_lshlrev_b32 v50, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v54, v53
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v21, v5 :: v_dual_lshlrev_b32 v53, 16, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v23, v24, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v50, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_add3_u32 v23, v23, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v28
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v30, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v50 :: v_dual_lshlrev_b32 v50, 16, v20
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v24, v30, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v50
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
-; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v20, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v28, 16, v50
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-FAKE16-NEXT:    v_perm_b32 v8, v38, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v22 :: v_dual_lshlrev_b32 v53, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v22, v28, v28
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v19, v3 :: v_dual_and_b32 v24, 0xffff0000, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v39, v7, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v21 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v5 :: v_dual_lshlrev_b32 v22, 16, v19
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v30, v22, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v28, v5 :: v_dual_lshlrev_b32 v50, 16, v19
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v20, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v30, v30, v22, 0x7fff
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v50
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v17
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v19, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v30, v54 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v21, v21, v21 :: v_dual_lshlrev_b32 v30, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v18
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v28, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v21, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v22, v4 :: v_dual_lshlrev_b32 v21, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v19, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v24, v21, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v16
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v17 :: v_dual_lshlrev_b32 v28, 16, v18
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v50, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v18, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v22, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v25
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v17, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v24, v28, v28 :: v_dual_lshlrev_b32 v25, 16, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v16, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v24, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v25, v25, v25 :: v_dual_lshlrev_b32 v22, 16, v22
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v50, v25, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v19, v22, v22
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v28, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v50, v25, 0x7fff
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v19, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v25
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v50, v53, v19, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v20, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v18 :: v_dual_lshlrev_b32 v23, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v50, v53, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v31, v3, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v17, v1 :: v_dual_lshlrev_b32 v20, 16, v16
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v0 :: v_dual_lshlrev_b32 v19, 16, v18
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_lshlrev_b32 v23, 16, v16
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v16, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v21, v1 :: v_dual_and_b32 v16, 0xffff0000, v22
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v20, v1 :: v_dual_lshlrev_b32 v16, 16, v19
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v14, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-FAKE16-NEXT:    v_perm_b32 v14, v31, v27, 0x5040100
-; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v13, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-FAKE16-NEXT:    v_perm_b32 v13, v32, v26, 0x5040100
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v15, v2, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_perm_b32 v15, v39, v29, 0x5040100
-; GFX11-FAKE16-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v52, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v32, v2, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v4, v15, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v15, v33, v51, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_maximumnum_v32bf16:
@@ -16922,990 +12956,751 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT:    scratch_load_b32 v51, off, s32
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v81, v81
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v52, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v82, v82
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v69, v69
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v80, 0xffff0000, v19
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s24
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v70, v70
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v50, v15 :: v_dual_mov_b32 v49, v14
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v13
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v82.l, v18.h, v52.l, s25
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v52.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v80, v80
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v49
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v82.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v29
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v32, v32
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v38, v38
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v134
+; GFX12-TRUE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v14
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v30
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v33, v33
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s44, v112, v134
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v33, v33
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v34, v34
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v13
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v13.h, v29.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v10.h, v26.h, s8
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v55, v55
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v112.l, v82.l, v52.l, s44
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v66, v66
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v55.l, v29.h, v15.l, s3
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v11
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v10
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v67, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v83, 0xffff0000, v1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v71, v71
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v33.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v39, v39
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v4.h, v20.h, s20
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v55.l
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v16
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v55, v55
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s29, v85, v85
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v29
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v28
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v27
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v26
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v21
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v48, v48
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v20
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v80, 0xffff0000, v19
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v83, 0xffff0000, v1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v35, v35
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v37, v37
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v39, v39
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v49, v49
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v51, v51
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v53, v53
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v54, v54
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v65, v65
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v67, v67
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v69, v69
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v71, v71
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s40, v86, v86
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v66.l, v26.h, v33.l, s9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v80.l, v20.h, v39.l, s21
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v39.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v67, v67
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v83, v83
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v66.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v80.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v3.h, v19.h, s22
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v132
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v32.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v55.l
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v84, 0xffff0000, v17
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v36, v36
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v38, v38
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v48, v48
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v50, v50
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v52, v52
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v64, v64
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v66, v66
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v68, v68
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v70, v70
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v80, v80
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v81, v81
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v83, v83
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v9.h, v25.h, s11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v85.l, v16.h, v54.l, s40
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v118
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v68, v68
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v5.h, v21.h, s18
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s42, v102, v132
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v81.l, v19.h, v48.l, s23
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s23, v85, v115
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v71.l, v21.h, v38.l, s19
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v15
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v14
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v82, v82
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s28, v84, v84
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v102.l, v80.l, v39.l, s42
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v38.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v85.l, v55.l, v15.l, s23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v71.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v50
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v102, v102, v102 :: v_dual_lshlrev_b32 v101, 16, v101
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v53, v53
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s26
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s26, v96, v118
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v131
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v85, v85, v85
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v30.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v69.l, v24.h, v38.l, s14
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v54.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s40, v86, v118
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v85.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v30
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v87, v87
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s41, v96, v96
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v33.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v34.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v36.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v39.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v50.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v51.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v64.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v65.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v66.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v67.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v68.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v69.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v70.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v71.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v80.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v81.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v145.l, v82.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v116
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s42, v97, v97
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v96.l, v66.l, v33.l, s26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v84, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v54, v54
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s41, v101, v131
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v32.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v37.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v52.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v53.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v87
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v30
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v34, v34
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v84, v84
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v9.h, v25.h, s10
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v96, v96, v96
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v31, v31
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v14.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v101.l, v71.l, v38.l, s41
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v54.l, v30.h, v14.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v35, v35
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v8.h, v24.h, s12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v67.l, v25.h, v34.l, s11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v54.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v101
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v27
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v64, v64
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v113, 16, v113
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v83.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v147.l, v84.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v119
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v128
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v129
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v130
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v131
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v132
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v133
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v134
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v135
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v144
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v144, 16, v145
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s63, v116, v86
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v86.l, v55.l, v32.l, s40
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0, v55.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v114
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v65, v65
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v12.h, v28.h, s4
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v34.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v68.l, v24.h, v35.l, s13
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v67.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s22, v84, v114
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v101, v101, v101
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v36, v36
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v7.h, v23.h, s14
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v37, v37
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v146
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v147
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s42, v87, v118
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s43, v96, v119
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s45, v98, v129
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s56, v101, v132
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s59, v112, v135
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s60, v113, v144
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v6.h, v22.h, s16
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v64.l, v28.h, v31.l, s5
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v11.h, v27.h, s6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v35.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v69.l, v23.h, v36.l, s15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v86.l, v32.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v86.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v34.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v35.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0, v36.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0, v39.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0, v50.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0, v51.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s46, v99, v130
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s61, v114, v145
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s62, v115, v146
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v96.l, v65.l, v34.l, s43
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v98.l, v67.l, v36.l, s45
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v101.l, v70.l, v39.l, s56
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v112.l, v81.l, v50.l, s59
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v113.l, v82.l, v51.l, s60
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v55.l, s16
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v118
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v33.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0, v37.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v48.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0, v52.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0, v53.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v68.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v119
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v84.l, v54.l, v14.l, s22
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v31.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v65.l, v27.h, v32.l, s7
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v36.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v70.l, v22.h, v37.l, s17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v83.l, v17.h, v53.l, s27
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v64.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v69.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v128
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s27, v97, v119
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v32.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v37.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v65.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v70.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v116
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v129
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s28, v98, v128
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s23, 0, v70.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s26, 0, v81.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v87.l, v64.l, v33.l, s42
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v97.l, v67.l, v34.l, s27
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v84, v84, v84 :: v_dual_lshlrev_b32 v87, 16, v87
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v99.l, v68.l, v37.l, s46
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v114.l, v83.l, v52.l, s61
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v115.l, v84.l, v53.l, s62
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v96.l, v34.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v98.l, v36.l, s5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v101.l, v39.l, s8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v101.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v112.l, v50.l, s11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v112.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v113.l, v51.l, s12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v113.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v55
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v38.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v49.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0, v54.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s44, v97, v128
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v116.l, v85.l, v54.l, s63
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v87.l, v33.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v99.l, v37.l, s6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.h, v114.l, v52.l, s13
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v114.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v115.l, v53.l, s14
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v115.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v39
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v35.h, v81.l, s26
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v35.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v117
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v130
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s24, v86, v116
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s29, v99, v129
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v98.l, v68.l, v35.l, s28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v114, v84, 16, 1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v48.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v81.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s25, v87, v117
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s40, v100, v130
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v86.l, v64.l, v31.l, s24
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v99.l, v69.l, v36.l, s29
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v97, v97, v97 :: v_dual_lshlrev_b32 v98, 16, v98
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v84
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v130, v96, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v114, v114, v84, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v84, v84
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v133
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v87.l, v65.l, v32.l, s25
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v100.l, v70.l, v37.l, s40
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v98, v98, v98 :: v_dual_lshlrev_b32 v99, 16, v99
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v131, 0x400000, v96
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v132, v97, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v114, v114, v115, s22
-; GFX12-TRUE16-NEXT:    v_add3_u32 v130, v130, v96, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v96, v96
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s43, v103, v133
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v87
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v86, v86, v86 :: v_dual_max_num_f32 v99, v99, v99
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v133, 0x400000, v97
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v134, v98, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v132, v132, v97, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v96, v130, v131, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v97, v97
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v87, v87, v87 :: v_dual_max_num_f32 v100, v100, v100
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v116, v85, 16, 1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v118, v86, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v144, 0x400000, v98
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v145, v99, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v134, v134, v98, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v97, v132, v133, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v98, v98
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v117, 0x400000, v85
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v86
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v128, v87, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v146, 0x400000, v99
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v147, v100, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v116, v116, v85, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v85, v85
-; GFX12-TRUE16-NEXT:    v_add3_u32 v118, v118, v86, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v86, v86
-; GFX12-TRUE16-NEXT:    v_add3_u32 v145, v145, v99, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v98, v134, v144, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v99, v99
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v15.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0, v31.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v129, 0x400000, v87
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v84, 0x400000, v100
-; GFX12-TRUE16-NEXT:    v_add3_u32 v128, v128, v87, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v87, v87
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s23
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s24
-; GFX12-TRUE16-NEXT:    v_add3_u32 v147, v147, v100, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s57, v102, v133
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v99, v145, v146, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v100, v100
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v32.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s25
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v116.h, v15.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v118.h, v31.l, s5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0, v34.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0, v33.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v97.l, v66.l, v35.l, s44
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v116.l, v54.l, s15
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v116.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s12, 0, v51
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0, v48.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0, v65.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0, v66.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s47, v100, v131
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s58, v103, v134
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v102.l, v71.l, v48.l, s57
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v96.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v97.l, v35.l, s4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v97.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s13, 0, v52
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s14, 0, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0, v38.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0, v49.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0, v64.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s24, 0, v71.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v128.h, v32.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v35.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0, v36.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v97.h, v34.l, s7
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.h, v96.h, v33.l, s6
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v85, v101, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v100.l, v69.l, v38.l, s47
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v103.l, v80.l, v49.l, s58
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v87.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v102.l, v48.l, s9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v102.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v30.h, v65.l, s18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v128
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v66.l, s19
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v129
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s25, 0, v80.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s41, 0, v85.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v38.l, s7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v103.l, v49.l, s10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v103.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v14.h, v64.l, s17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v119
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.h, v34.h, v71.l, s24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v48
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v65
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v66
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v86.l, v13.h, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v98.h, v35.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v99.h, v36.l, s8
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v37.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v86, 0x400000, v101
-; GFX12-TRUE16-NEXT:    v_add3_u32 v85, v85, v101, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v84, v147, v84, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v101, v101
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v14.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0, v38.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0, v54.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0, v55.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s25
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v49
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.l, v37.h, v85.l, s41
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v64
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v71
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v96.l, v30.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v97.l, v32.l, s4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0, v67.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0, v68.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v98.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v99.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v85, v85, v86, s22
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v114.h, v14.l, s0
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v87, v102, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0, v64.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v102
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v102, v102
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v54.l, s11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v87, v87, v102, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0, v39.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v87.l, v38.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v102.l, v38.h, s9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v32.h, v67.l, s20
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v130
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v131
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s29, 0, v84.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v70
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v67
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s27, 0, v82.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v68
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v64.l, s13
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0, v65.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v128
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v86, v87, v115, s22
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v87, 0xffff0000, v114
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v53.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0, v66.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v83.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v103.l, v81.l, v48.l, s43
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v116
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v37.l, v84.l, s29
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v101.l, v34.l, s8
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s28, 0, v83.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s10, 0, v80
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v36.l, v82.l, s27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.h, v115.l, v37.l, s14
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s22, 0, v69.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v65.l, s14
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v102
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0, v71.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v113, 16, v113
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v101
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0, v67.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0, v68.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v131, 0xffff0000, v98
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v103, v103, v103
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0, v70.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v132, 0xffff0000, v84
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v36.h, v83.l, s28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.h, v103.l, v35.l, s10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.h, v113.l, v36.l, s12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v100.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v33.h, v69.l, s22
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s11, 0, v81
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.h, v114.l, v48.l, s13
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v132
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v69
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s17
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v131
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v117, v103, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v103
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v132
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v96
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0, v80.l
-; GFX12-TRUE16-NEXT:    v_add3_u32 v117, v117, v103, 0x7fff
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v97
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0, v69.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v115
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v85
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v133
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v33.h, s7
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v51
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v50.h, v51.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v118
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v31
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.h, v31.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v84.h, v37.l, s4
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v85.h, v38.l, s9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v100
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v32.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v129
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s19
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v130
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v99
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v35, v36
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v14.h, v55.l, s12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v86.h, v39.l, s10
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v86
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0, v31.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v32.l, v31.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v87
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.h, v118.h, v15.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v39
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s21
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v36.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v114.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v32.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v31.h, v66.l, s15
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.h, v128.h, v15.h, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v32.h, v67.l, s16
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.h, v86.h, v35.l, s9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v96.h, v36.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v33.h, v69.l, s18
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v54, v38, v38
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v34.h, v71.l, s20
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v116.h, v35.h, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v98.h, v33.l, s5
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.h, v84.h, v34.l, s7
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v55, v54, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v54, v54
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v97.h, v37.l, s4
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v130
-; GFX12-TRUE16-NEXT:    v_add3_u32 v55, v55, v54, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v99.h, v37.h, s6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v55, v64, s11
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v64, v112, v112
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v54.h, v31.l, s10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v54
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v32.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX12-TRUE16-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.h, v85.h, v38.l, s8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v54.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v135
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v103, v103
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v113, v54
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v55, v117, v119 :: v_dual_and_b32 v54, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v48.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v83.l, v53.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v50.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v15.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v55.h, v48.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v81.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v15.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v0.h, v16.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.h, v98.l, v32.h, s5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v51.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v99.l, v33.l, s6
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v65, v66, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v52.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v67
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v55
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v16.h, v15.l, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v81.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v54.h, v52.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v64, v64, v64
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v50.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v52, v53
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v31.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v82.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v65
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v67, v64, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v66
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v82.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX12-TRUE16-NEXT:    v_add3_u32 v66, v67, v64, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v67, 0x400000, v64
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v52, v65
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v54
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.h, v55.h, v14.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v64, v66, v67, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v53.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v51
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v68
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.l, v50.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v52
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v51.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v32.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v50.l, s0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v38, v53
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.h, v112.l, v39.l, s11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v64.h, v53.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v51.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v14.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v29
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v50
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v30.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v33.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v50.l, v51.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v83.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.h, v54.h, v32.l, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v49
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v51.l, v33.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v33.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v34.l, v83.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v52, v52, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v64
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v66, v52, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v49.l, v30.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX12-TRUE16-NEXT:    v_add3_u32 v49, v66, v52, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v52
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v65, v55
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v15.h, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v117, v117
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v31.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v34.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v52, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v51, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v64.h, v32.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.l, v33.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v30.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v52
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v49, v49, v53, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v34.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v29
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v32.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v29.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v28
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v52
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v31.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v52, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s0
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v52, v55, v55 :: v_dual_and_b32 v53, 0xffff0000, v49
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v12.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v30.l, v34.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v29.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v31.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v15.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v54, v52, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v13.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v12
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX12-TRUE16-NEXT:    v_add3_u32 v54, v54, v52, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v52, v52
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v53, v53, v53
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v29.l, v13.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v28.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v13.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v29.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v27
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v30.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v52, v54, v65, s0
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v33.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v30.l, v13.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v51, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v55, 0x400000, v53
-; GFX12-TRUE16-NEXT:    v_add3_u32 v54, v54, v53, 0x7fff
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v15.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v52.h, v33.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v14.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v64
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v49, v54, v55, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v34.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v52
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v28.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v29.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v14.l, s0
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v53, v53, v53
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v34.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v55
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v64
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v65, v53, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v49
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v64, v65, v53, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v53
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v66
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v12.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v28.l, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v11.h, v29.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v53, v64, v65, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v13.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v15.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v14.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.h, v31.l, v12.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v27.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v11.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v26
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v28.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v31.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v53.h, v13.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v29.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v55
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v27.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v51, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.h, v10.h, v28.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v11.h, v29.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v53
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v49, v49, v49
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v52.h, v12.h, s0
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v55, v49, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v29
-; GFX12-TRUE16-NEXT:    v_add3_u32 v27, v55, v49, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v29, 0x400000, v49
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v54, v52
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v10.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v25
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v26.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v53.h, v11.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v10.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v26.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v29, v27, v29, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v27.l, v13.l, v11.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v29.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v51, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v52, v49
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v28.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v27, v12, v12
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v31.l, v10.h, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v28.l, s1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v25.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v9.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v27
-; GFX12-TRUE16-NEXT:    v_add3_u32 v28, v28, v27, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v27, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v25.l, v12.l, v10.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v9.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v29.h, v9.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v52, v49
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v27, v28, v53, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v11.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v8
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v29, v12, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v27.h, v11.l, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v24
-; GFX12-TRUE16-NEXT:    v_add3_u32 v29, v29, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v13.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v25.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v27, v26
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v12.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v29, v49, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v10.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v8.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v24.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v27
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v12.h, v10.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v28, v28
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v11.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v13.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v26.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v10, 16, 1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v52, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_add3_u32 v24, v24, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v26
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v29, v29
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v52, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v9.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v9.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v12.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v23.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v49
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v26
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v24.h, v9.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v25.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v27.h, v8.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v10.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v8.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v25.l, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v12.h, v9.h, s1
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v27, v10, v10 :: v_dual_and_b32 v26, 0xffff0000, v24
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v25, v27, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v23.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v11.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v12, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT:    v_add3_u32 v23, v25, v27, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v27
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v26, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v22.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v24.h, v7.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v11.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v11.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v12.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v22.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v25, v8, v8 :: v_dual_lshlrev_b32 v24, 16, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v11.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v11.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v23, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v25, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v26
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v25, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v12.l, v6.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v22.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v10.l, v7.h, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v24, v25, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v25
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v10.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v22.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v21, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v6.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v11.h, v7.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v10.l, v7.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v10.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v12, v11
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v9.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v21
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v8, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v9.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v22, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v8, v9, v21 :: v_dual_lshlrev_b32 v9, 16, v22
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v8.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v6, v9, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v11.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v21, v20
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v6, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v7.l, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v19
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v20
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v8.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v12, v20, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_and_b32 v11, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v8.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v11
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v8.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v9.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v6, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v20
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v12, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v18
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v6.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v4.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_add3_u32 v19, v19, v6, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v18.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v6, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v9.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v11, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v7.l, v4.h, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v18.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v19, v20, s3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v18.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v2.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.h, v4.l, s3
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v3.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v17
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v10, v4
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v17.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v12, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v16.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v16, v12
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v19, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v18, v17
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v17, v8, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v12, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v12.h, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v16, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v17, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s0
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v10, v3, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v16
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v5, v8, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.h, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_add3_u32 v10, v10, v3, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    v_add3_u32 v5, v5, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v2.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v10, v17, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v18, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v9.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v9.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.l, v12.h, v1.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v11.h, v0.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v5.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.l, v7.h, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v3, v50 :: v_dual_mov_b32 v4, v48
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v13 :: v_dual_mov_b32 v1, v30
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v51 :: v_dual_mov_b32 v5, v39
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v6, v38 :: v_dual_mov_b32 v7, v37
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v8, v35 :: v_dual_mov_b32 v9, v33
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v10, v32 :: v_dual_mov_b32 v11, v31
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v12, v36 :: v_dual_mov_b32 v13, v34
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.l, v2.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v8.l, v1.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v6.l, v0.h, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v9.l, v2.h, s3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v29 :: v_dual_mov_b32 v1, v49
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v48 :: v_dual_mov_b32 v3, v39
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v4, v38 :: v_dual_mov_b32 v5, v36
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v6, v35 :: v_dual_mov_b32 v7, v34
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v8, v33 :: v_dual_mov_b32 v9, v32
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v10, v31 :: v_dual_mov_b32 v11, v30
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v12, v37
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v32bf16:
@@ -17915,1099 +13710,792 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v12
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v51, 16, v28
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v30
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX12-FAKE16-NEXT:    scratch_load_b32 v50, off, s32
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v15
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v29
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v30
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v83, 16, v24
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v52, v52, v51, s1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v84, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v34, v35, v33 :: v_dual_and_b32 v49, 0xffff0000, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v38, v38
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v87, 16, v23
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v52, v51, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v96, 16, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v99, 16, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v39, v64, v55 :: v_dual_and_b32 v70, 0xffff0000, v9
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v48, v48, v39, s0
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v100, 16, v6
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v115, 16, v20
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v30
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v68, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v18
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v80, v71, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v80, v71, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v82, v82
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v18
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v84, v83, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v84, v83, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v26
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v96, v87, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v96, v87, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v26
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v25
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v146, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v100, v99, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v100, v99, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v147, 16, v16
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v34, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v54, v54
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v68, v112, v103 :: v_dual_and_b32 v81, 0xffff0000, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v112, v103, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v114, v114
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v22
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v14
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v116, v115, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v116, v115, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v118, v118
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v113, 0xffff0000, v21
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v64, v64, v55, s2
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
+; GFX12-FAKE16-NEXT:    scratch_load_b32 v31, off, s32
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v128, v119, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v128, v119, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v130, v130
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v20
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v82, v132, v131 :: v_dual_and_b32 v113, 0xffff0000, v21
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v132, v131, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v134, v134
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v84, v144, v135 :: v_dual_and_b32 v117, 0xffff0000, v20
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v102, v144, v135 :: v_dual_and_b32 v133, 0xffff0000, v18
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v146, v146
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v96
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v34, v147, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v27
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v33
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v14, v30 :: v_dual_and_b32 v97, 0xffff0000, v23
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v37, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v97, 0xffff0000, v23
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v51
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v39, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v55, v64 :: v_dual_lshlrev_b32 v130, 16, v51
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v34
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s3, v66, v66
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v68, v68, v67, s3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v49, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v68, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v68
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v80
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v52 :: v_dual_lshlrev_b32 v118, 16, v67
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v67
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v54, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v80, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v97, v97
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v84
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v64 :: v_dual_lshlrev_b32 v128, 16, v83
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v82 :: v_dual_lshlrev_b32 v134, 16, v83
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v101, v101
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v97, v99, v66, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v99, v84, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v113, v113
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v103, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v103, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v99
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v117, v117
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v39
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v36
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v101, v115, v70, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v115, v96, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v129, v129
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v34
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v38
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v49
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v87
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v82
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v103, v119, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v115, v119, v98 :: v_dual_lshlrev_b32 v146, 16, v113
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v133, v133
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v71
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v103
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v131, v82, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v86, v114
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v52
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v99
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v113
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v33, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v36, v115
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v48, v116
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v86, v86, v86 :: v_dual_lshlrev_b32 v85, 16, v64
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v51, v37, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v114, v86, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v115, 0x400000, v86
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v55
-; GFX12-FAKE16-NEXT:    v_add3_u32 v114, v114, v86, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v48, v48, v48
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v117
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v48, v48
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v118
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v66
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v118, v48, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v49, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v119
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v70
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v53, v53, v53 :: v_dual_lshlrev_b32 v102, 16, v80
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v119, 0x400000, v48
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v71, v52, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v81, v128
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v68
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v128, v53, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v118, v118, v48, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v85, v129
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v82
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v53
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v69, v69, v69
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v81
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v64, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v128, v128, v53, 0x7fff
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v53, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v81, v81, v81
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v144, 0x400000, v81
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v97
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v85, v85, v85
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v96, v130
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v146, v85, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v147, 0x400000, v85
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v97, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v98, v131
-; GFX12-FAKE16-NEXT:    v_add3_u32 v146, v146, v85, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v101
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v99, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v100, v132
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v132, v69, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v96, v96, v96
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v101, v70, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v102, v133
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v133, 0x400000, v69
-; GFX12-FAKE16-NEXT:    v_add3_u32 v132, v132, v69, 0x7fff
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v98, v98, v98
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v102, v103, v80, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v112, v134
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v134, v81, 16, 1
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v48, v98, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v98
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v82, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v86, v96, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v134, v134, v81, 0x7fff
-; GFX12-FAKE16-NEXT:    v_add3_u32 v48, v48, v98, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v114, v115, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v86, v86, v96, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v65, v65, v65
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v130, v65, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v131, 0x400000, v65
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v130, v130, v65, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v130, v131, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v37, 16, v48
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v132, v133 :: v_dual_lshlrev_b32 v36, 16, v36
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v36, v36, v36
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v117, v131, v100, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v86
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v134, v144, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v116, v36, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v36
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v36, v36
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v130, 16, v81
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-FAKE16-NEXT:    v_add3_u32 v116, v116, v36, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v36, 0x400000, v96
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v146, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v96, v96
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v100, v100, v100
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v85
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v119, v135, v102, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v86, v36, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v115, v100, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v100
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v86, 16, v114
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v98, 16, v116
-; GFX12-FAKE16-NEXT:    v_add3_u32 v115, v115, v100, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v48, v53, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v36
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v116
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v128
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v36
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v115, v117, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v34
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v118
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v134, 16, v48
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v146, 16, v53
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v38, v147, v34 :: v_dual_lshlrev_b32 v49, 16, v52
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e64 s0, v49, v130
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v86, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v35
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v96, 0xffff0000, v114
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v114, 16, v118
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v118, 16, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v98, v35, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v37
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v128
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v69
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v144, 0xffff0000, v48
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v114, v37, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v39
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v66, v30, v54 :: v_dual_lshlrev_b32 v53, 16, v64
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v35
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v14, v30
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v70
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v117
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v116, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v49
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v130, v35, v36 :: v_dual_lshlrev_b32 v129, 16, v39
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v37, v129
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v129, v51, v52, s0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v118, v49, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v37, v39, v48 :: v_dual_lshlrev_b32 v118, 16, v102
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v55
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v131
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v128, v52, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
-; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v50
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v55, v64 :: v_dual_lshlrev_b32 v50, 16, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v71
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v132
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v130, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v133
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v87
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v131, v64, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v65
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v81, v134
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v147, v32, v52 :: v_dual_and_b32 v64, 0xffff0000, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v66
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v102, v102, v102
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v85, v135
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v103
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v132, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v68
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v65
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v119, v102, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v102
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v82, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v97, v144
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v134, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v69
-; GFX12-FAKE16-NEXT:    v_add3_u32 v119, v119, v102, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v97, v99, v84 :: v_dual_lshlrev_b32 v114, 16, v98
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v101, v145
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v147, 16, v115
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v70
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v81
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v52
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v146, v70, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v33
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v0
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v34, v33, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v38
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v35, v35, v38 :: v_dual_lshlrev_b32 v34, 16, v147
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v51
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v101, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v97
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v112, v146
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v51 :: v_dual_and_b32 v85, 0xffff0000, v85
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v34, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v96, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v114, v147
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v119
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v38
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v52, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v115, v98, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v116, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v37, v55, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v67
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v116, v117, v100, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v118, v30
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v39, v67, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v71
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v118, v119, v102, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v128, v49
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v48, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v83
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v128, v38, v34, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v36
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v49, v83, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v87
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v36, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v48
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v49, v54, v87 :: v_dual_lshlrev_b32 v34, 16, v34
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v97
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v37, v48, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v31, v97, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v99
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v129, v52, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v64
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v129
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v34, v34, v34 :: v_dual_cndmask_b32 v55, v32, v99
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v96
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v54, v34, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v34
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v53, v64, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v68
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v86, v33, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v100
-; GFX12-FAKE16-NEXT:    v_add3_u32 v54, v54, v34, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v65, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v70
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v98, v35, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v115
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v69, v70, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v80
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v114, v36, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v81, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v82
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v117
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v85, v82, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v84
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v116, v37, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v65
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v97, v84, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v86
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v38, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v69
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v69, 16, v16
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v101, v86, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v96
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v128, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v81
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v112, v96, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v98
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v130, v48, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e64 vcc_lo, 0, v147
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v98, v114, v98 :: v_dual_lshlrev_b32 v131, 16, v53
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v85
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v100, v116, v100 :: v_dual_lshlrev_b32 v133, 16, v69
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v35
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v131, v49, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v35 :: v_dual_lshlrev_b32 v135, 16, v85
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v102
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v52, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v133
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v102, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v39
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v48, v132, v51 :: v_dual_lshlrev_b32 v51, 16, v112
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v144
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v39 :: v_dual_lshlrev_b32 v145, 16, v101
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v34
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v134, v55, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v84
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v128, v34, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v51
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v49, v51, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v55
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v128
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v135, v84, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v64, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v67
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v119, v129, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v101
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v67, v51, v51
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v66, 16, v52
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v71
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v64, v101, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v52
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v70, v71 :: v_dual_lshlrev_b32 v147, 16, v114
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v83
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v54, v84, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v80
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v80, v83, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v87
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v66, v80, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v67, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v82, v87, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v99
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v51, v146, v64 :: v_dual_lshlrev_b32 v80, 16, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v84, v99, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v103
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v53, v67, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v65, v103, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v65, 0x400000, v67
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v65 :: v_dual_and_b32 v68, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v86, v103 :: v_dual_lshlrev_b32 v30, 16, v130
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v113
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v70, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v16
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v96, v113, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v115
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v52, v66, v64 :: v_dual_lshlrev_b32 v55, 16, v55
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v98, v115, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v117
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v81
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v69, v67, vcc_lo
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v55, v55, v55
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v82
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v67
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v64
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v68, v55, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v100, v117, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v119
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v65, v82, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v66, v68, v55, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v68, 0x400000, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v35, v119, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v38
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v66, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v70
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v34, v38, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v64, v67, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v14, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v15, v50, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v113
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v55
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v66, v66, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v37, v36, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v31
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v113 :: v_dual_lshlrev_b32 v80, 16, v68
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v129, v39, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v131
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v68 :: v_dual_and_b32 v15, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v84
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v50
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v53, v49, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v132
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v31
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v70, v84, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v65, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v133
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v65, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v69, v64, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v134
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v80, v71
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v54, v66, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v71, 0x400000, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v30
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v81, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v135
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v50, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX12-FAKE16-NEXT:    v_add3_u32 v54, v54, v66, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v85, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v144
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v14, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v97, v70, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v145
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v13
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v101, v71, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v30, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v15, v31, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v31
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v148, 16, v116
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v55
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v112
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v33
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v146
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v70, v53, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v53, 16, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v69
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v67
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v112, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v67, v53, v67 :: v_dual_and_b32 v54, 0xffff0000, v54
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v70, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v29
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v118
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v30, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v71, v71
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v65, v65, v65
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v31, v55 :: v_dual_lshlrev_b32 v64, 16, v52
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v147
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v13, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v55, v65, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v80, 0x400000, v65
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v55, v55, v65, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v114, v82, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v148
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v80, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v71
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v116, v83, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v50, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v29, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v52, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v67
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v66, v13, v13 :: v_dual_cndmask_b32 v29, v67, v64
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v55
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v65
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v53, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v102
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v65
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v53, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v68
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v66, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v118, v84, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v33
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v64, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v70, v67
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v55
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v50
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v65, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v52
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v50, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v50, v53, v66, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v53 :: v_dual_and_b32 v55, 0xffff0000, v55
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v53, v54, v54 :: v_dual_lshlrev_b32 v54, 16, v28
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v53, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v67
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v50
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v64, v29 :: v_dual_and_b32 v50, 0xffff0000, v50
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v64, v53, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v128, v86, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v68
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v69
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v64, v64, v53, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v28
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v65, v53, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v66
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v55, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v27
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v29 :: v_dual_lshlrev_b32 v64, 16, v54
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v66, v54, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v64, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v28
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v69, v68
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v28, v12 :: v_dual_lshlrev_b32 v67, 16, v11
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v27, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v54
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v54, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v53
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v64
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v66
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v71
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v54, v54, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v53, v54, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v30, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v67, v64
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v66
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v66, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v50, v54, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX12-FAKE16-NEXT:    v_add3_u32 v50, v50, v54, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v65, v64
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v64, v64, v64 :: v_dual_cndmask_b32 v27, v55, v27
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v65
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v29, v13 :: v_dual_lshlrev_b32 v66, 16, v12
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v65, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v26
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v54, v64, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v67 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v65, v26, v10 :: v_dual_lshlrev_b32 v68, 16, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v54 :: v_dual_lshlrev_b32 v64, 16, v55
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v66, v65
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v27
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v54, v64, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v64
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v30, v55, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
+; GFX12-FAKE16-NEXT:    v_perm_b32 v14, v14, v53, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v30, v12 :: v_dual_lshlrev_b32 v67, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v28, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v67, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v54, 16, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v28
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v54, v54, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v27, v11 :: v_dual_lshlrev_b32 v28, 16, v54
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v66
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v30, v54, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v26
+; GFX12-FAKE16-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX12-FAKE16-NEXT:    v_add3_u32 v30, v30, v54, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v25, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v50, v50, v50 :: v_dual_lshlrev_b32 v55, 16, v8
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v26 :: v_dual_lshlrev_b32 v29, 16, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v54, v28
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v30, v64 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v50
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v50, 16, 1
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v26
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v24
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v30
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v8
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v53, v50, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v30
-; GFX12-FAKE16-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v10
+; GFX12-FAKE16-NEXT:    v_perm_b32 v12, v34, v12, 0x5040100
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v24, v8 :: v_dual_lshlrev_b32 v55, 16, v23
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v26, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v53, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v11, v35, v11, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v23
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v55, v54
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v65 :: v_dual_lshlrev_b32 v53, 16, v54
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v50
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v25, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v53, v53, v53 :: v_dual_cndmask_b32 v10, v28, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v53, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v54, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v23, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX12-FAKE16-NEXT:    v_add3_u32 v25, v28, v53, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v26, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v28, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v22
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX12-FAKE16-NEXT:    v_perm_b32 v10, v36, v10, 0x5040100
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v27, v9 :: v_dual_lshlrev_b32 v28, 16, v23
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v8
+; GFX12-FAKE16-NEXT:    v_perm_b32 v9, v37, v9, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v28, 16, 1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v27, v26
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v8 :: v_dual_lshlrev_b32 v25, 16, v22
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v29, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v54, v9 :: v_dual_lshlrev_b32 v64, 16, v6
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v23, v7, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v53, v28, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v28
-; GFX12-FAKE16-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
-; GFX12-FAKE16-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v50, v8 :: v_dual_lshlrev_b32 v55, 16, v22
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v64, v55
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v22, v6 :: v_dual_lshlrev_b32 v54, 16, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v24 :: v_dual_lshlrev_b32 v53, 16, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v28
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v24, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v50, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v28, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v25, v7 :: v_dual_lshlrev_b32 v50, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v54, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v21, v5 :: v_dual_lshlrev_b32 v53, 16, v20
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v23
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v23, v24, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v50, 0x400000, v24
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v30, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
+; GFX12-FAKE16-NEXT:    v_perm_b32 v8, v38, v8, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX12-FAKE16-NEXT:    v_add3_u32 v23, v23, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v39, v7, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v28
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v30, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v50 :: v_dual_lshlrev_b32 v50, 16, v20
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v24, v30, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v30
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v21 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v50
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
-; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v20, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX12-FAKE16-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v28, 16, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v22
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v22 :: v_dual_lshlrev_b32 v53, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v22, v28, v28
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v5 :: v_dual_lshlrev_b32 v22, 16, v19
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v19, v3 :: v_dual_and_b32 v24, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v30, v22, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v28, v5 :: v_dual_lshlrev_b32 v50, 16, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v19
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v20, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v21
-; GFX12-FAKE16-NEXT:    v_add3_u32 v30, v30, v22, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v53, v50
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v17
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v19, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v30, v54 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v21, v21, v21 :: v_dual_lshlrev_b32 v30, 16, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v18
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v28, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v21, 16, 1
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v22, v4 :: v_dual_lshlrev_b32 v21, 16, v23
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v19, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v20
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v2
+; GFX12-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v24, v21, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v16
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v17 :: v_dual_lshlrev_b32 v28, 16, v18
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v50, v28
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v20, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v18, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v22, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v30, v25
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v18 :: v_dual_lshlrev_b32 v23, 16, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v17, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v22
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v24, v28, v28 :: v_dual_lshlrev_b32 v25, 16, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v0
+; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v31, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v16, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v24, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v25, v25, v25 :: v_dual_lshlrev_b32 v22, 16, v22
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v19
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v50, v25, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v19, v22, v22
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v28, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v24
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v50, v25, 0x7fff
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v19, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v25
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_add3_u32 v50, v53, v19, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v17, v1 :: v_dual_lshlrev_b32 v20, 16, v16
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v0 :: v_dual_lshlrev_b32 v19, 16, v18
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v23, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v50, v53, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v24, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_lshlrev_b32 v23, 16, v16
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v25, v23
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v16, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v17
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v21, v1 :: v_dual_and_b32 v16, 0xffff0000, v22
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v20, v1 :: v_dual_lshlrev_b32 v16, 16, v19
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v14, v1, 0x5040100
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX12-FAKE16-NEXT:    v_perm_b32 v14, v31, v27, 0x5040100
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v13, v0, 0x5040100
+; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v52, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-FAKE16-NEXT:    v_perm_b32 v13, v32, v26, 0x5040100
-; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v15, v2, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v32, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_perm_b32 v15, v39, v29, 0x5040100
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_perm_b32 v4, v15, v4, 0x5040100
+; GFX12-FAKE16-NEXT:    v_perm_b32 v15, v33, v51, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <32 x bfloat> @llvm.maximumnum.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y)
   ret <32 x bfloat> %result
@@ -19038,23 +14526,13 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_mul_f32_e32 v2, 1.0, v2
-; GFX8-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v2
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
-; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_maximumnum_bf16_no_ieee:
@@ -19070,22 +14548,13 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX900-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX900-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v3, v3, v2, s4
-; GFX900-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_maximumnum_bf16_no_ieee:
@@ -19104,9 +14573,6 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v2, v2, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
@@ -19132,21 +14598,13 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee:
@@ -19155,39 +14613,31 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_bf16_no_ieee:
@@ -19206,26 +14656,14 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_maximumnum_bf16_no_ieee:
@@ -19238,45 +14676,37 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_bf16_no_ieee:
@@ -19302,29 +14732,17 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v2, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call bfloat @llvm.maximumnum.bf16(bfloat %x, bfloat %y)
   ret bfloat %result
@@ -19362,24 +14780,14 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -19389,23 +14797,14 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19423,23 +14822,14 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -19449,21 +14839,13 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v2, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -19478,15 +14860,12 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v3, v0, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
@@ -19510,12 +14889,8 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v4, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
@@ -19550,37 +14925,21 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v2, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v7, v4, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19594,66 +14953,52 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v5, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16_no_ieee:
@@ -19666,60 +15011,39 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19737,71 +15061,56 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v6, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v5, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16_no_ieee:
@@ -19819,69 +15128,48 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -19927,24 +15215,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -19954,23 +15232,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -19980,22 +15249,13 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -20014,23 +15274,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -20040,22 +15291,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -20065,21 +15308,13 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v4, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -20094,15 +15329,12 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v5, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -20126,9 +15358,6 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
@@ -20152,12 +15381,8 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v5, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
@@ -20172,180 +15397,125 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX10-LABEL: v_maximumnum_v3bf16_no_ieee:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v5, v5
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_sdwa v0, v0, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_sdwa v10, v0, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v11
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX10-NEXT:    v_or_b32_e32 v11, 0x400000, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v10, v7, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v10, v10, v7, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v12, v8, 16, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v4, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_maximumnum_v3bf16_no_ieee:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v5.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v7, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v9, v11
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -20354,88 +15524,59 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v9
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX11-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -20446,104 +15587,77 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v5.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v7, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v1.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s3, v9, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v8, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -20556,98 +15670,67 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v9
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v7, v7, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX12-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v7, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
@@ -20655,7 +15738,7 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x bfloat> @llvm.maximumnum.v3bf16(<3 x bfloat> %x, <3 x bfloat> %y)
   ret <3 x bfloat> %result
@@ -20705,24 +15788,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -20734,23 +15807,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT:    v_mul_f32_e32 v7, 1.0, v7
-; GFX8-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v7
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s4, v8
-; GFX8-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -20760,23 +15824,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -20786,22 +15841,13 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
@@ -20822,23 +15868,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -20850,22 +15887,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX900-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX900-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX900-NEXT:    v_add3_u32 v8, v8, v7, s4
-; GFX900-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -20875,22 +15904,14 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -20900,21 +15921,13 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v5, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v4, v1, s4
@@ -20935,11 +15948,8 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -20963,9 +15973,6 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
@@ -20989,9 +15996,6 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v7, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
@@ -21005,7 +16009,7 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_perm_b32 v1, v4, v1, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
@@ -21015,12 +16019,8 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v6, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
-; GFX950-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX950-NEXT:    v_perm_b32 v1, v4, v1, s0
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v2
 ; GFX950-NEXT:    s_nop 1
@@ -21062,78 +16062,46 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v8, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX10-NEXT:    v_bfe_u32 v14, v8, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v7, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v13
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v12
-; GFX10-NEXT:    v_add3_u32 v12, v14, v8, 0x7fff
-; GFX10-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v11
-; GFX10-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v13, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_bfe_u32 v15, v9, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX10-NEXT:    v_add3_u32 v12, v12, v11, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_or_b32_e32 v16, 0x400000, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_bfe_u32 v12, v6, 16, 1
-; GFX10-NEXT:    v_add3_u32 v14, v15, v9, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_add3_u32 v12, v12, v6, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v12, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
@@ -21152,232 +16120,162 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v10, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v9, v13
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v9, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v11, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v4, v1, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -21395,123 +16293,94 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v4.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v10, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s2, v9, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v11, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v9, v13
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v11, v11, v11
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v5.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v11, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX12-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
@@ -21521,126 +16390,80 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v13
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v13, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v6
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
@@ -21651,10 +16474,10 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
index 8d020b9e1a60..0003366f3a3e 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
@@ -173,54 +173,53 @@ define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %
 ; CHECK-NEXT:    v_mov_b32_e32 v26, s0
 ; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:124
 ; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
 ; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:116
 ; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:112
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
 ; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:104
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
 ; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen offset:96
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-NEXT:    buffer_load_dword v8, v26, s[20:23], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v9, v26, s[20:23], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v10, v26, s[20:23], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v11, v26, s[20:23], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v12, v26, s[20:23], 0 offen offset:32
-; CHECK-NEXT:    buffer_load_dword v13, v26, s[20:23], 0 offen offset:36
-; CHECK-NEXT:    buffer_load_dword v14, v26, s[20:23], 0 offen offset:40
-; CHECK-NEXT:    buffer_load_dword v15, v26, s[20:23], 0 offen offset:44
-; CHECK-NEXT:    buffer_load_dword v16, v26, s[20:23], 0 offen offset:48
-; CHECK-NEXT:    buffer_load_dword v17, v26, s[20:23], 0 offen offset:52
-; CHECK-NEXT:    buffer_load_dword v18, v26, s[20:23], 0 offen offset:56
-; CHECK-NEXT:    buffer_load_dword v19, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v8, v26, s[20:23], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v9, v26, s[20:23], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v10, v26, s[20:23], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v11, v26, s[20:23], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v12, v26, s[20:23], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v13, v26, s[20:23], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v14, v26, s[20:23], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v15, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v17, v26, s[20:23], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v19, v26, s[20:23], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v21, v26, s[20:23], 0 offen offset:84
 ; CHECK-NEXT:    buffer_load_dword v23, v26, s[20:23], 0 offen offset:92
 ; CHECK-NEXT:    buffer_load_dword v22, v26, s[20:23], 0 offen offset:88
-; CHECK-NEXT:    buffer_load_dword v21, v26, s[20:23], 0 offen offset:84
 ; CHECK-NEXT:    buffer_load_dword v20, v26, s[20:23], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v18, v26, s[20:23], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v16, v26, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v25, s1
 ; CHECK-NEXT:    v_mov_b32_e32 v24, s0
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    s_waitcnt vmcnt(18)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:112
-; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:76
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:72
-; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:68
-; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:12
 ; CHECK-NEXT:    s_nop 0
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[20:23] offset:80
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:64
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[16:19] offset:48
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[12:15] offset:32
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[8:11] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[16:19] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[12:15] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[8:11] offset:32
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7]
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3]
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
@@ -464,54 +463,53 @@ define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %
 ; CHECK-NEXT:    v_mov_b32_e32 v26, s0
 ; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:124
 ; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
 ; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:116
 ; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:112
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
 ; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:104
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
 ; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen offset:96
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-NEXT:    buffer_load_dword v8, v26, s[20:23], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v9, v26, s[20:23], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v10, v26, s[20:23], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v11, v26, s[20:23], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v12, v26, s[20:23], 0 offen offset:32
-; CHECK-NEXT:    buffer_load_dword v13, v26, s[20:23], 0 offen offset:36
-; CHECK-NEXT:    buffer_load_dword v14, v26, s[20:23], 0 offen offset:40
-; CHECK-NEXT:    buffer_load_dword v15, v26, s[20:23], 0 offen offset:44
-; CHECK-NEXT:    buffer_load_dword v16, v26, s[20:23], 0 offen offset:48
-; CHECK-NEXT:    buffer_load_dword v17, v26, s[20:23], 0 offen offset:52
-; CHECK-NEXT:    buffer_load_dword v18, v26, s[20:23], 0 offen offset:56
-; CHECK-NEXT:    buffer_load_dword v19, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v8, v26, s[20:23], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v9, v26, s[20:23], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v10, v26, s[20:23], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v11, v26, s[20:23], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v12, v26, s[20:23], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v13, v26, s[20:23], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v14, v26, s[20:23], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v15, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v17, v26, s[20:23], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v19, v26, s[20:23], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v21, v26, s[20:23], 0 offen offset:84
 ; CHECK-NEXT:    buffer_load_dword v23, v26, s[20:23], 0 offen offset:92
 ; CHECK-NEXT:    buffer_load_dword v22, v26, s[20:23], 0 offen offset:88
-; CHECK-NEXT:    buffer_load_dword v21, v26, s[20:23], 0 offen offset:84
 ; CHECK-NEXT:    buffer_load_dword v20, v26, s[20:23], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v18, v26, s[20:23], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v16, v26, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v25, s1
 ; CHECK-NEXT:    v_mov_b32_e32 v24, s0
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    s_waitcnt vmcnt(18)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:112
-; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:76
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:72
-; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:68
-; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[20:23], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[20:23], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[20:23], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[20:23], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[20:23], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[20:23], 0 offen offset:12
 ; CHECK-NEXT:    s_nop 0
 ; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[20:23] offset:80
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:64
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[16:19] offset:48
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[12:15] offset:32
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[8:11] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[16:19] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[12:15] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[8:11] offset:32
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7]
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3]
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
index cadc3dadb0a1..b43ccc551ca9 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
@@ -451,12 +451,12 @@ define void @memcpy_p0_p3_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
 ; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
 ; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
@@ -506,12 +506,12 @@ define void @memcpy_p0_p3_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
 ; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
 ; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
@@ -896,19 +896,18 @@ define void @memcpy_p0_p5_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:28
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:30
 ; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
@@ -924,18 +923,18 @@ define void @memcpy_p0_p5_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -966,19 +965,18 @@ define void @memcpy_p0_p5_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:28
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:30
 ; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
@@ -994,18 +992,18 @@ define void @memcpy_p0_p5_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index 267dc722234e..89d0d61b403f 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -3583,104 +3583,102 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; CHECK-NEXT:  .LBB4_1: ; %load-store-loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    s_clause 0x3e
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
-; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:36
-; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:40
-; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:44
-; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:48
-; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:52
-; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:56
-; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:60
-; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:124
-; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:120
-; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:116
-; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:64
 ; CHECK-NEXT:    buffer_load_dword v22, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:112
 ; CHECK-NEXT:    buffer_load_dword v21, v2, s[0:3], 0 offen offset:104
 ; CHECK-NEXT:    buffer_load_dword v20, v2, s[0:3], 0 offen offset:100
 ; CHECK-NEXT:    buffer_load_dword v19, v2, s[0:3], 0 offen offset:96
-; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:252
-; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:248
-; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:244
-; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:240
 ; CHECK-NEXT:    buffer_load_dword v30, v2, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:240
 ; CHECK-NEXT:    buffer_load_dword v29, v2, s[0:3], 0 offen offset:232
 ; CHECK-NEXT:    buffer_load_dword v28, v2, s[0:3], 0 offen offset:228
 ; CHECK-NEXT:    buffer_load_dword v27, v2, s[0:3], 0 offen offset:224
-; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:220
-; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:216
-; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:212
-; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:208
-; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:204
-; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:200
-; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:196
-; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:192
-; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:188
-; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:184
-; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:180
-; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:176
+; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:208
+; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:192
 ; CHECK-NEXT:    buffer_load_dword v55, v2, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:176
 ; CHECK-NEXT:    buffer_load_dword v54, v2, s[0:3], 0 offen offset:168
 ; CHECK-NEXT:    buffer_load_dword v53, v2, s[0:3], 0 offen offset:164
 ; CHECK-NEXT:    buffer_load_dword v52, v2, s[0:3], 0 offen offset:160
-; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:156
-; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:152
-; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:148
-; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:144
-; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:140
-; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:136
-; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:132
-; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:128
-; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:92
-; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:88
-; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:84
-; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:80
-; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:76
-; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:72
-; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:68
-; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen offset:64
-; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
 ; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v101, null, s5, v1, vcc_lo
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
 ; CHECK-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
 ; CHECK-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
-; CHECK-NEXT:    s_waitcnt vmcnt(41)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:240
-; CHECK-NEXT:    s_waitcnt vmcnt(37)
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[27:30] offset:224
-; CHECK-NEXT:    s_waitcnt vmcnt(33)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:208
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:192
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:176
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:192
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:160
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:144
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:128
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:128
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:112
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[19:22] offset:96
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:80
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:64
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:48
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:32
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:80
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:32
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87]
 ; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s6
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB4_1
 ; CHECK-NEXT:  ; %bb.2: ; %memcpy-split
@@ -3748,16 +3746,17 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:21
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:22
 ; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:23
-; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:24
 ; ALIGNED-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:25
 ; ALIGNED-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:26
-; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:30
-; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:31
 ; ALIGNED-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
 ; ALIGNED-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:33
 ; ALIGNED-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:34
-; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:29
-; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
 ; ALIGNED-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:27
 ; ALIGNED-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:35
 ; ALIGNED-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:36
@@ -3798,10 +3797,9 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:71
 ; ALIGNED-NEXT:    buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:76
 ; ALIGNED-NEXT:    buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:75
 ; ALIGNED-NEXT:    buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:78
 ; ALIGNED-NEXT:    buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT:    buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
-; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:75
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(57)
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(56)
@@ -3811,46 +3809,46 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(54)
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(53)
-; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(52)
 ; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(51)
 ; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(50)
-; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(49)
-; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(48)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(47)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(46)
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
 ; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
-; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
-; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(44)
-; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(43)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v7, 8, v5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(42)
 ; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v9, 8, v5
-; ALIGNED-NEXT:    s_waitcnt vmcnt(41)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v9, 8, v8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(40)
 ; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 8, v6
-; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v6
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v11, 8, v12
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v15, 8, v14
 ; ALIGNED-NEXT:    v_lshl_or_b32 v8, v19, 8, v17
-; ALIGNED-NEXT:    s_waitcnt vmcnt(40)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(39)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v9, v16, 8, v13
-; ALIGNED-NEXT:    s_waitcnt vmcnt(38)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(37)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v10, v20, 8, v18
-; ALIGNED-NEXT:    s_waitcnt vmcnt(36)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(35)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v11, v23, 8, v22
-; ALIGNED-NEXT:    s_waitcnt vmcnt(34)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(33)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v12, v28, 8, v25
-; ALIGNED-NEXT:    s_waitcnt vmcnt(32)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(31)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v13, v24, 8, v21
-; ALIGNED-NEXT:    s_waitcnt vmcnt(30)
-; ALIGNED-NEXT:    v_lshl_or_b32 v14, v27, 8, v26
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(29)
+; ALIGNED-NEXT:    v_lshl_or_b32 v14, v27, 8, v26
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v6, 16, v5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 16, v7
@@ -3858,27 +3856,27 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
 ; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(28)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(27)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v15, v31, 8, v30
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(26)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(25)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v34, 8, v33
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(24)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(23)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v37, 8, v32
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(22)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(21)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v36, 8, v35
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(17)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(16)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v50, 8, v38
 ; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(14)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v5, v49, 8, v39
 ; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v51, 8, v48
 ; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v53, 8, v52
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v0, 16, v15
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
@@ -3888,13 +3886,13 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v55, 8, v29
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v67, 8, v66
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v64, 8, v54
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v68, 8, v65
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    s_clause 0x1
@@ -3903,13 +3901,13 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v70, 8, v69
 ; ALIGNED-NEXT:    s_clause 0x1
 ; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:83
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:74
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v80, 8, v71
 ; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
@@ -3955,9 +3953,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
 ; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
 ; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:87
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
@@ -4240,7 +4236,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:148
 ; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:149
 ; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:150
-; ALIGNED-NEXT:    buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:151
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v126, 8, v125
@@ -4252,7 +4248,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v123, 8, v5
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v124, 8, v5
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 8, v3
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
@@ -4286,7 +4282,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:163
 ; ALIGNED-NEXT:    buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:164
 ; ALIGNED-NEXT:    buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:165
-; ALIGNED-NEXT:    buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:166
 ; ALIGNED-NEXT:    buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:167
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v78, 8, v89
@@ -4294,7 +4290,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v74, 8, v73
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v76
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v77
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v75, 8, v79
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
@@ -4303,20 +4299,20 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:172
 ; ALIGNED-NEXT:    buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:173
 ; ALIGNED-NEXT:    buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:174
-; ALIGNED-NEXT:    buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT:    buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:175
 ; ALIGNED-NEXT:    buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:171
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v61, 8, v63
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v60, 8, v62
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v59, 8, v62
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
-; ALIGNED-NEXT:    buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT:    buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:168
 ; ALIGNED-NEXT:    buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:169
 ; ALIGNED-NEXT:    buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:170
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v56, 8, v59
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v56, 8, v58
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v57, 8, v47
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
@@ -4326,7 +4322,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:177
 ; ALIGNED-NEXT:    buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:178
 ; ALIGNED-NEXT:    buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:179
-; ALIGNED-NEXT:    buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT:    buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:180
 ; ALIGNED-NEXT:    buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:181
 ; ALIGNED-NEXT:    buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:182
 ; ALIGNED-NEXT:    buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:183
@@ -4338,7 +4334,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v118, 8, v42
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v41, 8, v45
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v41, 8, v44
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x4
@@ -4373,15 +4369,16 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:198
 ; ALIGNED-NEXT:    buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:199
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v98, 8, v100
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v98, 8, v100
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v86, 8, v87
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v86, 8, v87
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v97, 8, v99
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v85, 8, v96
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v85, 8, v96
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v97, 8, v99
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x4
 ; ALIGNED-NEXT:    buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:204
@@ -4492,23 +4489,23 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:248
 ; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:249
 ; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:250
-; ALIGNED-NEXT:    v_lshl_or_b32 v124, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v123, v4, 16, v3
 ; ALIGNED-NEXT:    s_clause 0x5
 ; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen
 ; ALIGNED-NEXT:    buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:2
 ; ALIGNED-NEXT:    buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:4
 ; ALIGNED-NEXT:    buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:5
 ; ALIGNED-NEXT:    buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:6
-; ALIGNED-NEXT:    buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT:    buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:7
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(28)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v25, 8, v27
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(26)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v24, 8, v26
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(14)
-; ALIGNED-NEXT:    v_lshl_or_b32 v44, v12, 8, v16
+; ALIGNED-NEXT:    v_lshl_or_b32 v45, v12, 8, v16
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
-; ALIGNED-NEXT:    v_lshl_or_b32 v58, v8, 8, v10
-; ALIGNED-NEXT:    v_lshl_or_b32 v104, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v60, v8, 8, v10
+; ALIGNED-NEXT:    v_lshl_or_b32 v95, v4, 16, v3
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v21, 8, v22
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v23, 8, v20
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
@@ -4517,35 +4514,35 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_store_dword v90, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
 ; ALIGNED-NEXT:    buffer_store_dword v92, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v77, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v76, v4, 16, v3
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v17, 8, v19
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v14, 8, v13
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    buffer_store_dword v95, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v101, v4, 16, v3
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v15, 8, v18
-; ALIGNED-NEXT:    v_lshl_or_b32 v84, v44, 16, v4
-; ALIGNED-NEXT:    v_lshl_or_b32 v44, v9, 8, v11
-; ALIGNED-NEXT:    v_lshl_or_b32 v4, v58, 16, v44
-; ALIGNED-NEXT:    v_lshl_or_b32 v44, v5, 8, v6
-; ALIGNED-NEXT:    v_lshl_or_b32 v58, v7, 8, v1
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v58, 16, v44
+; ALIGNED-NEXT:    v_lshl_or_b32 v84, v45, 16, v4
+; ALIGNED-NEXT:    v_lshl_or_b32 v45, v9, 8, v11
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v60, 16, v45
+; ALIGNED-NEXT:    v_lshl_or_b32 v45, v5, 8, v6
+; ALIGNED-NEXT:    v_lshl_or_b32 v60, v7, 8, v1
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v60, 16, v45
 ; ALIGNED-NEXT:    s_clause 0x1
-; ALIGNED-NEXT:    buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:1
-; ALIGNED-NEXT:    buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:3
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v44, v44, 8, v0
-; ALIGNED-NEXT:    v_lshl_or_b32 v58, v58, 8, v94
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
-; ALIGNED-NEXT:    v_lshl_or_b32 v44, v90, 8, v88
-; ALIGNED-NEXT:    v_lshl_or_b32 v58, v95, 8, v92
+; ALIGNED-NEXT:    buffer_store_dword v60, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v45, v45, 8, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v60, v60, 8, v94
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v60, 16, v45
+; ALIGNED-NEXT:    v_lshl_or_b32 v45, v90, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v60, v104, 8, v92
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v60, 16, v45
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x4
 ; ALIGNED-NEXT:    buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:12
@@ -4554,34 +4551,34 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:15
 ; ALIGNED-NEXT:    buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:11
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; ALIGNED-NEXT:    v_lshl_or_b32 v44, v111, 8, v122
+; ALIGNED-NEXT:    v_lshl_or_b32 v45, v111, 8, v122
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v58, v110, 8, v120
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
+; ALIGNED-NEXT:    v_lshl_or_b32 v60, v110, 8, v120
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v60, 16, v45
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
-; ALIGNED-NEXT:    buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT:    buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:8
 ; ALIGNED-NEXT:    buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:9
 ; ALIGNED-NEXT:    buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:10
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v44, v92, 8, v95
+; ALIGNED-NEXT:    v_lshl_or_b32 v45, v92, 8, v104
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v58, v94, 8, v90
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v58, 16, v44
+; ALIGNED-NEXT:    v_lshl_or_b32 v60, v94, 8, v90
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v60, 16, v45
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:18
 ; ALIGNED-NEXT:    buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:16
-; ALIGNED-NEXT:    buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:18
-; ALIGNED-NEXT:    buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:17
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:232
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:236
 ; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:228
 ; ALIGNED-NEXT:    buffer_store_dword v101, off, s[0:3], s32 offset:224
 ; ALIGNED-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
-; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v127, 8, v44
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v127, 8, v60
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v127, v58, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v127, v45, 8, v88
 ; ALIGNED-NEXT:    v_lshl_or_b32 v127, v0, 16, v127
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
@@ -4606,9 +4603,9 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v16 offset:246
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v18 offset:244
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v19 offset:240
-; ALIGNED-NEXT:    buffer_store_dword v77, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT:    buffer_store_dword v124, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v76, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT:    buffer_store_dword v95, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:244
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_addc_u32 s5, s5, 0
 ; ALIGNED-NEXT:    v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
@@ -4713,7 +4710,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v118 offset:183
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v41 offset:181
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v42 offset:182
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v45 offset:180
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v44 offset:180
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v46 offset:176
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
@@ -4730,17 +4727,17 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v47 offset:170
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v57 offset:171
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v56 offset:169
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v60 offset:175
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v59 offset:175
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v61 offset:173
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v62 offset:174
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v63 offset:172
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v59 offset:168
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v58 offset:168
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v73 offset:162
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v74 offset:163
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v78 offset:161
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v72 offset:167
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v75 offset:165
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v76 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v77 offset:166
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v79 offset:164
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v89 offset:160
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
@@ -4768,7 +4765,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:145
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v123 offset:151
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v124 offset:151
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:149
@@ -5235,11 +5232,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:24
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v44 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v60 offset:18
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:19
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v58 offset:17
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v45 offset:17
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:23
@@ -5272,7 +5269,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v110 offset:15
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v120 offset:14
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v122 offset:12
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v95 offset:8
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v104 offset:8
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:2
@@ -12461,97 +12458,97 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; CHECK-NEXT:  .LBB9_1: ; %memmove_fwd_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    s_clause 0x3e
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
-; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:36
-; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:40
-; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:44
-; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:48
-; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:52
-; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:56
-; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:76
 ; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:92
 ; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:88
 ; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:84
 ; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:80
-; CHECK-NEXT:    buffer_load_dword v22, v2, s[0:3], 0 offen offset:124
-; CHECK-NEXT:    buffer_load_dword v21, v2, s[0:3], 0 offen offset:120
-; CHECK-NEXT:    buffer_load_dword v20, v2, s[0:3], 0 offen offset:116
-; CHECK-NEXT:    buffer_load_dword v19, v2, s[0:3], 0 offen offset:112
-; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:108
-; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:104
-; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:100
-; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:96
-; CHECK-NEXT:    buffer_load_dword v30, v2, s[0:3], 0 offen offset:156
-; CHECK-NEXT:    buffer_load_dword v29, v2, s[0:3], 0 offen offset:152
-; CHECK-NEXT:    buffer_load_dword v28, v2, s[0:3], 0 offen offset:148
-; CHECK-NEXT:    buffer_load_dword v27, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:64
+; CHECK-NEXT:    buffer_load_dword v22, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v21, v2, s[0:3], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v20, v2, s[0:3], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v19, v2, s[0:3], 0 offen offset:96
+; CHECK-NEXT:    buffer_load_dword v30, v2, s[0:3], 0 offen offset:172
 ; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:188
 ; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:184
 ; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:180
 ; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:176
-; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:172
-; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:168
-; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:164
-; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v29, v2, s[0:3], 0 offen offset:168
+; CHECK-NEXT:    buffer_load_dword v28, v2, s[0:3], 0 offen offset:164
+; CHECK-NEXT:    buffer_load_dword v27, v2, s[0:3], 0 offen offset:160
+; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:204
 ; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:220
 ; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:216
 ; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:212
 ; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:208
-; CHECK-NEXT:    buffer_load_dword v55, v2, s[0:3], 0 offen offset:252
-; CHECK-NEXT:    buffer_load_dword v54, v2, s[0:3], 0 offen offset:248
-; CHECK-NEXT:    buffer_load_dword v53, v2, s[0:3], 0 offen offset:244
-; CHECK-NEXT:    buffer_load_dword v52, v2, s[0:3], 0 offen offset:240
-; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:236
-; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:232
-; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:228
-; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:224
-; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:204
-; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:200
-; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:196
-; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:192
-; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:140
-; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:136
-; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:132
-; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:128
-; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:76
-; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:72
-; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:68
-; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen offset:64
-; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:200
+; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:196
+; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:192
+; CHECK-NEXT:    buffer_load_dword v55, v2, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:240
+; CHECK-NEXT:    buffer_load_dword v54, v2, s[0:3], 0 offen offset:232
+; CHECK-NEXT:    buffer_load_dword v53, v2, s[0:3], 0 offen offset:228
+; CHECK-NEXT:    buffer_load_dword v52, v2, s[0:3], 0 offen offset:224
+; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v101, null, s5, v1, vcc_lo
 ; CHECK-NEXT:    s_add_u32 s4, s4, 0x100
 ; CHECK-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:240
 ; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:224
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:224
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:208
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:192
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:192
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:176
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:160
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[27:30] offset:144
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[27:30] offset:160
+; CHECK-NEXT:    s_waitcnt vmcnt(11)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:144
 ; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:128
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[19:22] offset:112
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:96
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:128
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:112
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[19:22] offset:96
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:80
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:64
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:48
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:32
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87]
 ; CHECK-NEXT:    s_cmp_lg_u64 s[4:5], 0x800
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB9_1
 ; CHECK-NEXT:  .LBB9_2: ; %Flow10
@@ -12565,103 +12562,101 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; CHECK-NEXT:  .LBB9_4: ; %memmove_bwd_loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    s_clause 0x3e
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:32
-; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:36
-; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:40
-; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:44
-; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:48
-; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:52
-; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:56
-; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:60
-; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:124
-; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:120
-; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:116
-; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v14, v2, s[0:3], 0 offen offset:76
+; CHECK-NEXT:    buffer_load_dword v18, v2, s[0:3], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v17, v2, s[0:3], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v16, v2, s[0:3], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v15, v2, s[0:3], 0 offen offset:80
+; CHECK-NEXT:    buffer_load_dword v13, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v12, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v11, v2, s[0:3], 0 offen offset:64
 ; CHECK-NEXT:    buffer_load_dword v22, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:112
 ; CHECK-NEXT:    buffer_load_dword v21, v2, s[0:3], 0 offen offset:104
 ; CHECK-NEXT:    buffer_load_dword v20, v2, s[0:3], 0 offen offset:100
 ; CHECK-NEXT:    buffer_load_dword v19, v2, s[0:3], 0 offen offset:96
-; CHECK-NEXT:    buffer_load_dword v26, v2, s[0:3], 0 offen offset:252
-; CHECK-NEXT:    buffer_load_dword v25, v2, s[0:3], 0 offen offset:248
-; CHECK-NEXT:    buffer_load_dword v24, v2, s[0:3], 0 offen offset:244
-; CHECK-NEXT:    buffer_load_dword v23, v2, s[0:3], 0 offen offset:240
 ; CHECK-NEXT:    buffer_load_dword v30, v2, s[0:3], 0 offen offset:236
+; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:252
+; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:248
+; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:244
+; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:240
 ; CHECK-NEXT:    buffer_load_dword v29, v2, s[0:3], 0 offen offset:232
 ; CHECK-NEXT:    buffer_load_dword v28, v2, s[0:3], 0 offen offset:228
 ; CHECK-NEXT:    buffer_load_dword v27, v2, s[0:3], 0 offen offset:224
-; CHECK-NEXT:    buffer_load_dword v34, v2, s[0:3], 0 offen offset:220
-; CHECK-NEXT:    buffer_load_dword v33, v2, s[0:3], 0 offen offset:216
-; CHECK-NEXT:    buffer_load_dword v32, v2, s[0:3], 0 offen offset:212
-; CHECK-NEXT:    buffer_load_dword v31, v2, s[0:3], 0 offen offset:208
 ; CHECK-NEXT:    buffer_load_dword v38, v2, s[0:3], 0 offen offset:204
+; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:220
+; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:216
+; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:212
+; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:208
 ; CHECK-NEXT:    buffer_load_dword v37, v2, s[0:3], 0 offen offset:200
 ; CHECK-NEXT:    buffer_load_dword v36, v2, s[0:3], 0 offen offset:196
 ; CHECK-NEXT:    buffer_load_dword v35, v2, s[0:3], 0 offen offset:192
-; CHECK-NEXT:    buffer_load_dword v51, v2, s[0:3], 0 offen offset:188
-; CHECK-NEXT:    buffer_load_dword v50, v2, s[0:3], 0 offen offset:184
-; CHECK-NEXT:    buffer_load_dword v49, v2, s[0:3], 0 offen offset:180
-; CHECK-NEXT:    buffer_load_dword v48, v2, s[0:3], 0 offen offset:176
 ; CHECK-NEXT:    buffer_load_dword v55, v2, s[0:3], 0 offen offset:172
+; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:188
+; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:184
+; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:180
+; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:176
 ; CHECK-NEXT:    buffer_load_dword v54, v2, s[0:3], 0 offen offset:168
 ; CHECK-NEXT:    buffer_load_dword v53, v2, s[0:3], 0 offen offset:164
 ; CHECK-NEXT:    buffer_load_dword v52, v2, s[0:3], 0 offen offset:160
-; CHECK-NEXT:    buffer_load_dword v67, v2, s[0:3], 0 offen offset:156
-; CHECK-NEXT:    buffer_load_dword v66, v2, s[0:3], 0 offen offset:152
-; CHECK-NEXT:    buffer_load_dword v65, v2, s[0:3], 0 offen offset:148
-; CHECK-NEXT:    buffer_load_dword v64, v2, s[0:3], 0 offen offset:144
-; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:140
-; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:136
-; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:132
-; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:128
-; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:92
-; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:88
-; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:84
-; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:80
-; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:76
-; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:72
-; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:68
-; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen offset:64
-; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v71, v2, s[0:3], 0 offen offset:156
+; CHECK-NEXT:    buffer_load_dword v70, v2, s[0:3], 0 offen offset:152
+; CHECK-NEXT:    buffer_load_dword v69, v2, s[0:3], 0 offen offset:148
+; CHECK-NEXT:    buffer_load_dword v68, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT:    buffer_load_dword v83, v2, s[0:3], 0 offen offset:140
+; CHECK-NEXT:    buffer_load_dword v82, v2, s[0:3], 0 offen offset:136
+; CHECK-NEXT:    buffer_load_dword v81, v2, s[0:3], 0 offen offset:132
+; CHECK-NEXT:    buffer_load_dword v80, v2, s[0:3], 0 offen offset:128
+; CHECK-NEXT:    buffer_load_dword v84, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v85, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v86, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v96, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v97, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v98, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v99, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v87, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    v_add_co_u32 v100, vcc_lo, v0, s4
 ; CHECK-NEXT:    v_add_co_ci_u32_e64 v101, null, s5, v1, vcc_lo
 ; CHECK-NEXT:    v_add_nc_u32_e32 v2, 0xffffff00, v2
 ; CHECK-NEXT:    s_add_u32 s4, s4, 0xffffff00
 ; CHECK-NEXT:    s_addc_u32 s5, s5, -1
-; CHECK-NEXT:    s_waitcnt vmcnt(41)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:240
-; CHECK-NEXT:    s_waitcnt vmcnt(37)
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:240
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[27:30] offset:224
-; CHECK-NEXT:    s_waitcnt vmcnt(33)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[31:34] offset:208
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:208
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[35:38] offset:192
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[48:51] offset:176
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:176
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[52:55] offset:160
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[64:67] offset:144
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:128
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(12)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[68:71] offset:144
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:128
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[23:26] offset:112
 ; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[19:22] offset:96
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[80:83] offset:80
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87] offset:64
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:48
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:32
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[15:18] offset:80
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[11:14] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[7:10] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:32
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[3:6] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[96:99]
+; CHECK-NEXT:    flat_store_dwordx4 v[100:101], v[84:87]
 ; CHECK-NEXT:    s_cmp_eq_u64 s[4:5], s[6:7]
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB9_4
 ; CHECK-NEXT:  .LBB9_5: ; %Flow11
@@ -12736,16 +12731,17 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:21
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:22
 ; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:23
-; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:24
 ; ALIGNED-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:25
 ; ALIGNED-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:26
-; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:30
-; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:31
 ; ALIGNED-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
 ; ALIGNED-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:33
 ; ALIGNED-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:34
-; ALIGNED-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:29
-; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
 ; ALIGNED-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:27
 ; ALIGNED-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:35
 ; ALIGNED-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:36
@@ -12768,16 +12764,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:53
 ; ALIGNED-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:54
 ; ALIGNED-NEXT:    buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:55
-; ALIGNED-NEXT:    buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT:    buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:56
 ; ALIGNED-NEXT:    buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:57
 ; ALIGNED-NEXT:    buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:58
+; ALIGNED-NEXT:    buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT:    buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:61
 ; ALIGNED-NEXT:    buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:62
-; ALIGNED-NEXT:    buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT:    buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:63
 ; ALIGNED-NEXT:    buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:64
 ; ALIGNED-NEXT:    buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:65
 ; ALIGNED-NEXT:    buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:66
-; ALIGNED-NEXT:    buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:61
-; ALIGNED-NEXT:    buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:60
 ; ALIGNED-NEXT:    buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:59
 ; ALIGNED-NEXT:    buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:67
 ; ALIGNED-NEXT:    buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:68
@@ -12786,10 +12782,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:71
 ; ALIGNED-NEXT:    buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:76
 ; ALIGNED-NEXT:    buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:75
 ; ALIGNED-NEXT:    buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:78
 ; ALIGNED-NEXT:    buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT:    buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
-; ALIGNED-NEXT:    buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:75
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(57)
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(56)
@@ -12799,46 +12794,46 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(54)
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(53)
-; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(52)
 ; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(51)
 ; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(50)
-; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(49)
-; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(48)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(47)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(46)
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
 ; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 8, v3
-; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
-; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(44)
-; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(43)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v7, 8, v5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(42)
 ; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v9, 8, v5
-; ALIGNED-NEXT:    s_waitcnt vmcnt(41)
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v9, 8, v8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(40)
 ; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 8, v6
-; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v6
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v11, 8, v12
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v15, 8, v14
 ; ALIGNED-NEXT:    v_lshl_or_b32 v8, v19, 8, v17
-; ALIGNED-NEXT:    s_waitcnt vmcnt(40)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(39)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v9, v16, 8, v13
-; ALIGNED-NEXT:    s_waitcnt vmcnt(38)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(37)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v10, v20, 8, v18
-; ALIGNED-NEXT:    s_waitcnt vmcnt(36)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(35)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v11, v23, 8, v22
-; ALIGNED-NEXT:    s_waitcnt vmcnt(34)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(33)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v12, v27, 8, v25
-; ALIGNED-NEXT:    s_waitcnt vmcnt(32)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(31)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v13, v24, 8, v21
-; ALIGNED-NEXT:    s_waitcnt vmcnt(30)
-; ALIGNED-NEXT:    v_lshl_or_b32 v14, v28, 8, v26
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(29)
+; ALIGNED-NEXT:    v_lshl_or_b32 v14, v28, 8, v26
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v6, 16, v5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v8, 16, v7
@@ -12846,26 +12841,27 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
 ; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(28)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(27)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v15, v30, 8, v29
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(26)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(25)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v32, 8, v34
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(24)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(23)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v36, 8, v31
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(22)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(21)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v35, 8, v33
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(16)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v48, 8, v37
-; ALIGNED-NEXT:    v_lshl_or_b32 v5, v39, 8, v38
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(14)
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v49, 8, v38
 ; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v6, v50, 8, v49
+; ALIGNED-NEXT:    v_lshl_or_b32 v6, v50, 8, v39
 ; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v51, 8, v52
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v0, 16, v15
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
@@ -12875,13 +12871,13 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v54, 8, v53
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v55, 8, v65
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v66, 8, v64
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v68, 8, v67
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    s_clause 0x1
@@ -12890,13 +12886,13 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v70, 8, v69
 ; ALIGNED-NEXT:    s_clause 0x1
 ; ALIGNED-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:83
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:74
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(5)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v80, 8, v71
 ; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
@@ -12922,11 +12918,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v36, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v35, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v37, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v48, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v38, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v49, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v39, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v50, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v52, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v51, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
@@ -12942,9 +12938,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
 ; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v127, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:87
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
 ; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
@@ -13227,39 +13222,39 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v120, 8, v111
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
 ; ALIGNED-NEXT:    buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:149
 ; ALIGNED-NEXT:    buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:150
-; ALIGNED-NEXT:    buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:151
+; ALIGNED-NEXT:    buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:151
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v121, 8, v3
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v109, 8, v122
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v110, 8, v122
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x4
 ; ALIGNED-NEXT:    buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:156
 ; ALIGNED-NEXT:    buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:157
-; ALIGNED-NEXT:    buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:158
+; ALIGNED-NEXT:    buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:158
 ; ALIGNED-NEXT:    buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:159
-; ALIGNED-NEXT:    buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:155
+; ALIGNED-NEXT:    buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:155
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v105, 8, v108
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v104, 8, v107
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v104, 8, v106
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
-; ALIGNED-NEXT:    buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:152
+; ALIGNED-NEXT:    buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:152
 ; ALIGNED-NEXT:    buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:153
 ; ALIGNED-NEXT:    buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:154
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v92, 8, v93
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v92, 8, v95
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v95, 8, v90
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v94, 8, v90
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x7
@@ -13268,8 +13263,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:162
 ; ALIGNED-NEXT:    buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:163
 ; ALIGNED-NEXT:    buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:164
-; ALIGNED-NEXT:    buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:165
-; ALIGNED-NEXT:    buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:166
 ; ALIGNED-NEXT:    buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:167
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v79, 8, v89
@@ -13277,9 +13272,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v74, 8, v73
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v76
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v75
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v75, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v76, 8, v88
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x4
@@ -13356,15 +13351,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:198
 ; ALIGNED-NEXT:    buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:199
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v100, 8, v102
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v100, 8, v102
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v96, 8, v97
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v96, 8, v97
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
+; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v99, 8, v101
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v87, 8, v98
+; ALIGNED-NEXT:    v_lshl_or_b32 v4, v87, 8, v98
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v99, 8, v101
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x4
 ; ALIGNED-NEXT:    buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:204
@@ -13380,10 +13376,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
 ; ALIGNED-NEXT:    buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:200
-; ALIGNED-NEXT:    buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT:    buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:201
 ; ALIGNED-NEXT:    buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:202
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v70, 8, v80
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v71, 8, v80
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v81, 8, v69
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v4, 16, v3
@@ -13475,11 +13471,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:248
 ; ALIGNED-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:249
 ; ALIGNED-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:250
-; ALIGNED-NEXT:    v_lshl_or_b32 v110, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v109, v4, 16, v3
 ; ALIGNED-NEXT:    s_clause 0x4
 ; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen
 ; ALIGNED-NEXT:    buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:3
-; ALIGNED-NEXT:    buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:4
 ; ALIGNED-NEXT:    buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:5
 ; ALIGNED-NEXT:    buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:6
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(27)
@@ -13490,7 +13486,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v77, v13, 8, v16
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v91, v9, 8, v10
-; ALIGNED-NEXT:    v_lshl_or_b32 v94, v4, 16, v3
+; ALIGNED-NEXT:    v_lshl_or_b32 v93, v4, 16, v3
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v22, 8, v24
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v23, 8, v21
 ; ALIGNED-NEXT:    v_lshl_or_b32 v78, v4, 16, v3
@@ -13502,14 +13498,14 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; ALIGNED-NEXT:    buffer_store_dword v106, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v107, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
 ; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v86, v77, 16, v4
 ; ALIGNED-NEXT:    v_lshl_or_b32 v77, v11, 8, v12
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
 ; ALIGNED-NEXT:    buffer_store_dword v125, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v71, v91, 16, v77
+; ALIGNED-NEXT:    v_lshl_or_b32 v70, v91, 16, v77
 ; ALIGNED-NEXT:    v_lshl_or_b32 v77, v6, 8, v8
 ; ALIGNED-NEXT:    v_lshl_or_b32 v91, v7, 8, v5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v4, v91, 16, v77
@@ -13527,7 +13523,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v91, v0, 8, v91
 ; ALIGNED-NEXT:    buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:12
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v91, 16, v77
-; ALIGNED-NEXT:    v_lshl_or_b32 v77, v123, 8, v106
+; ALIGNED-NEXT:    v_lshl_or_b32 v77, v123, 8, v107
 ; ALIGNED-NEXT:    v_lshl_or_b32 v91, v3, 8, v125
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:13
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
@@ -13560,21 +13556,21 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v91, 16, v77
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
-; ALIGNED-NEXT:    buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:16
-; ALIGNED-NEXT:    buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:18
-; ALIGNED-NEXT:    buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT:    buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:17
 ; ALIGNED-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:236
 ; ALIGNED-NEXT:    buffer_store_dword v86, off, s[0:3], s32 offset:228
 ; ALIGNED-NEXT:    buffer_store_dword v103, off, s[0:3], s32 offset:224
 ; ALIGNED-NEXT:    s_clause 0x1
 ; ALIGNED-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:704
 ; ALIGNED-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:708
 ; ALIGNED-NEXT:    v_add_nc_u32_e32 v2, 0x100, v2
-; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v127, 8, v77
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v127, 8, v91
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; ALIGNED-NEXT:    v_lshl_or_b32 v127, v91, 8, v106
+; ALIGNED-NEXT:    v_lshl_or_b32 v127, v77, 8, v107
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
 ; ALIGNED-NEXT:    v_add_co_u32 v3, vcc_lo, v3, s4
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
@@ -13596,8 +13592,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v19 offset:244
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v20 offset:240
 ; ALIGNED-NEXT:    buffer_store_dword v78, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT:    buffer_store_dword v94, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT:    buffer_store_dword v110, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT:    buffer_store_dword v93, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT:    buffer_store_dword v109, off, s[0:3], s32 offset:244
 ; ALIGNED-NEXT:    v_lshl_or_b32 v127, v0, 16, v127
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_add_u32 s4, s4, 0x100
@@ -13663,7 +13659,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:208
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v69 offset:202
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v81 offset:203
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v70 offset:201
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v71 offset:201
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v82 offset:207
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v83 offset:205
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v84 offset:206
@@ -13729,8 +13725,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v74 offset:163
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v79 offset:161
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v72 offset:167
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v75 offset:165
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v76 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v76 offset:165
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v75 offset:166
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v88 offset:164
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v89 offset:160
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
@@ -13746,20 +13742,20 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:256
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v90 offset:154
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v95 offset:155
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v94 offset:155
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v92 offset:153
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v104 offset:159
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v105 offset:157
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v107 offset:158
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v106 offset:158
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v108 offset:156
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v93 offset:152
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v95 offset:152
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v111 offset:146
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v120 offset:147
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v124 offset:145
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v109 offset:151
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v110 offset:151
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v121 offset:149
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v122 offset:150
-; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:148
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
@@ -14219,11 +14215,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:24
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v77 offset:18
-; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v91 offset:18
+; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:19
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v91 offset:17
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v77 offset:17
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:23
@@ -14236,7 +14232,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[3:4], v0 offset:20
-; ALIGNED-NEXT:    flat_store_byte v[3:4], v106 offset:16
+; ALIGNED-NEXT:    flat_store_byte v[3:4], v107 offset:16
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:408
@@ -14305,16 +14301,17 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v1, v4, s[0:3], 0 offen offset:21
 ; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:22
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:23
-; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:24
+; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:24
 ; ALIGNED-NEXT:    buffer_load_ubyte v10, v4, s[0:3], 0 offen offset:25
 ; ALIGNED-NEXT:    buffer_load_ubyte v12, v4, s[0:3], 0 offen offset:26
-; ALIGNED-NEXT:    buffer_load_ubyte v6, v4, s[0:3], 0 offen offset:30
-; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:31
+; ALIGNED-NEXT:    buffer_load_ubyte v126, v4, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:28
+; ALIGNED-NEXT:    buffer_load_ubyte v7, v4, s[0:3], 0 offen offset:29
+; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:30
+; ALIGNED-NEXT:    buffer_load_ubyte v9, v4, s[0:3], 0 offen offset:31
 ; ALIGNED-NEXT:    buffer_load_ubyte v14, v4, s[0:3], 0 offen offset:32
 ; ALIGNED-NEXT:    buffer_load_ubyte v15, v4, s[0:3], 0 offen offset:33
 ; ALIGNED-NEXT:    buffer_load_ubyte v17, v4, s[0:3], 0 offen offset:34
-; ALIGNED-NEXT:    buffer_load_ubyte v9, v4, s[0:3], 0 offen offset:29
-; ALIGNED-NEXT:    buffer_load_ubyte v5, v4, s[0:3], 0 offen offset:28
 ; ALIGNED-NEXT:    buffer_load_ubyte v11, v4, s[0:3], 0 offen offset:27
 ; ALIGNED-NEXT:    buffer_load_ubyte v19, v4, s[0:3], 0 offen offset:35
 ; ALIGNED-NEXT:    buffer_load_ubyte v13, v4, s[0:3], 0 offen offset:36
@@ -14355,10 +14352,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_ubyte v68, v4, s[0:3], 0 offen offset:71
 ; ALIGNED-NEXT:    buffer_load_ubyte v69, v4, s[0:3], 0 offen offset:76
 ; ALIGNED-NEXT:    buffer_load_ubyte v70, v4, s[0:3], 0 offen offset:77
+; ALIGNED-NEXT:    buffer_load_ubyte v81, v4, s[0:3], 0 offen offset:75
 ; ALIGNED-NEXT:    buffer_load_ubyte v71, v4, s[0:3], 0 offen offset:78
 ; ALIGNED-NEXT:    buffer_load_ubyte v80, v4, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT:    buffer_load_ubyte v126, v4, s[0:3], 0 offen offset:19
-; ALIGNED-NEXT:    buffer_load_ubyte v81, v4, s[0:3], 0 offen offset:75
 ; ALIGNED-NEXT:    buffer_load_ubyte v125, v4, s[0:3], 0 offen offset:151
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(58)
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
@@ -14369,46 +14365,46 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(55)
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(54)
-; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(53)
 ; ALIGNED-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(52)
 ; ALIGNED-NEXT:    buffer_store_dword v12, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(51)
-; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(50)
-; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(49)
+; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(48)
+; ALIGNED-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(47)
+; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; ALIGNED-NEXT:    s_waitcnt vmcnt(46)
 ; ALIGNED-NEXT:    buffer_store_dword v14, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 8, v2
-; ALIGNED-NEXT:    s_waitcnt vmcnt(46)
-; ALIGNED-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(45)
-; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(44)
+; ALIGNED-NEXT:    v_lshl_or_b32 v2, v7, 8, v5
+; ALIGNED-NEXT:    s_waitcnt vmcnt(43)
 ; ALIGNED-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v2, v9, 8, v5
-; ALIGNED-NEXT:    s_waitcnt vmcnt(42)
+; ALIGNED-NEXT:    v_lshl_or_b32 v3, v9, 8, v8
+; ALIGNED-NEXT:    s_waitcnt vmcnt(41)
 ; ALIGNED-NEXT:    buffer_store_dword v13, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v3, v8, 8, v6
-; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v7
+; ALIGNED-NEXT:    v_lshl_or_b32 v5, v10, 8, v6
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v11, 8, v12
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v15, 8, v14
 ; ALIGNED-NEXT:    v_lshl_or_b32 v8, v19, 8, v17
-; ALIGNED-NEXT:    s_waitcnt vmcnt(41)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(40)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v9, v16, 8, v13
-; ALIGNED-NEXT:    s_waitcnt vmcnt(39)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(38)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v10, v20, 8, v18
-; ALIGNED-NEXT:    s_waitcnt vmcnt(37)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(36)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v11, v23, 8, v22
-; ALIGNED-NEXT:    s_waitcnt vmcnt(35)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(34)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v12, v28, 8, v25
-; ALIGNED-NEXT:    s_waitcnt vmcnt(33)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(32)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v13, v24, 8, v21
-; ALIGNED-NEXT:    s_waitcnt vmcnt(31)
-; ALIGNED-NEXT:    v_lshl_or_b32 v14, v27, 8, v26
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
+; ALIGNED-NEXT:    s_waitcnt vmcnt(30)
+; ALIGNED-NEXT:    v_lshl_or_b32 v14, v27, 8, v26
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
 ; ALIGNED-NEXT:    v_lshl_or_b32 v2, v6, 16, v5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
@@ -14416,27 +14412,27 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
 ; ALIGNED-NEXT:    buffer_store_dword v15, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(29)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(28)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v15, v31, 8, v30
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(27)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(26)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v34, 8, v33
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(25)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(24)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v37, 8, v32
 ; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(23)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(22)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v2, v36, 8, v35
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(18)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(17)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v50, 8, v38
 ; ALIGNED-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(16)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(15)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v5, v49, 8, v39
 ; ALIGNED-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v6, v51, 8, v48
 ; ALIGNED-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v7, v53, 8, v52
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v0, 16, v15
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
@@ -14446,13 +14442,13 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v55, 8, v29
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(12)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(11)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v67, 8, v66
 ; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(10)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v2, v64, 8, v54
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v68, 8, v65
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    s_clause 0x1
@@ -14461,13 +14457,13 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v17, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v70, 8, v69
 ; ALIGNED-NEXT:    s_clause 0x1
 ; ALIGNED-NEXT:    buffer_load_ubyte v3, v4, s[0:3], 0 offen offset:83
 ; ALIGNED-NEXT:    buffer_load_ubyte v2, v4, s[0:3], 0 offen offset:74
 ; ALIGNED-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
+; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v1, v80, 8, v71
 ; ALIGNED-NEXT:    buffer_store_dword v19, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v16, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
@@ -14513,9 +14509,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_store_dword v70, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v71, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_store_dword v80, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(9)
 ; ALIGNED-NEXT:    buffer_store_dword v126, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    s_waitcnt vmcnt(8)
 ; ALIGNED-NEXT:    buffer_store_dword v81, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    buffer_load_ubyte v8, v4, s[0:3], 0 offen offset:87
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(7)
@@ -14840,21 +14834,21 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    s_clause 0x7
 ; ALIGNED-NEXT:    buffer_load_ubyte v89, v4, s[0:3], 0 offen offset:160
 ; ALIGNED-NEXT:    buffer_load_ubyte v79, v4, s[0:3], 0 offen offset:161
-; ALIGNED-NEXT:    buffer_load_ubyte v73, v4, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT:    buffer_load_ubyte v75, v4, s[0:3], 0 offen offset:162
 ; ALIGNED-NEXT:    buffer_load_ubyte v74, v4, s[0:3], 0 offen offset:163
 ; ALIGNED-NEXT:    buffer_load_ubyte v88, v4, s[0:3], 0 offen offset:164
-; ALIGNED-NEXT:    buffer_load_ubyte v75, v4, s[0:3], 0 offen offset:165
-; ALIGNED-NEXT:    buffer_load_ubyte v77, v4, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT:    buffer_load_ubyte v77, v4, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT:    buffer_load_ubyte v76, v4, s[0:3], 0 offen offset:166
 ; ALIGNED-NEXT:    buffer_load_ubyte v72, v4, s[0:3], 0 offen offset:167
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(6)
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v79, 8, v89
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v74, 8, v73
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v74, 8, v75
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v77
+; ALIGNED-NEXT:    v_lshl_or_b32 v1, v72, 8, v76
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v75, 8, v88
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v77, 8, v88
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x4
@@ -15064,7 +15058,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v104, v3, 16, v2
 ; ALIGNED-NEXT:    v_lshl_or_b32 v2, v21, 8, v22
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v23, 8, v20
-; ALIGNED-NEXT:    v_lshl_or_b32 v76, v3, 16, v2
+; ALIGNED-NEXT:    v_lshl_or_b32 v73, v3, 16, v2
 ; ALIGNED-NEXT:    v_lshl_or_b32 v2, v17, 8, v19
 ; ALIGNED-NEXT:    v_lshl_or_b32 v3, v14, 8, v13
 ; ALIGNED-NEXT:    v_lshl_or_b32 v101, v3, 16, v2
@@ -15129,9 +15123,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    v_lshl_or_b32 v0, v57, 16, v43
 ; ALIGNED-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
 ; ALIGNED-NEXT:    s_clause 0x2
+; ALIGNED-NEXT:    buffer_load_ubyte v57, v4, s[0:3], 0 offen offset:18
 ; ALIGNED-NEXT:    buffer_load_ubyte v78, v4, s[0:3], 0 offen offset:16
-; ALIGNED-NEXT:    buffer_load_ubyte v43, v4, s[0:3], 0 offen offset:18
-; ALIGNED-NEXT:    buffer_load_ubyte v57, v4, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT:    buffer_load_ubyte v43, v4, s[0:3], 0 offen offset:17
 ; ALIGNED-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:488
 ; ALIGNED-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:492
 ; ALIGNED-NEXT:    buffer_store_dword v84, off, s[0:3], s32 offset:484
@@ -15140,10 +15134,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:704
 ; ALIGNED-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:708
 ; ALIGNED-NEXT:    v_add_nc_u32_e32 v4, 0xffffff00, v4
-; ALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; ALIGNED-NEXT:    v_lshl_or_b32 v0, v126, 8, v43
+; ALIGNED-NEXT:    s_waitcnt vmcnt(4)
+; ALIGNED-NEXT:    v_lshl_or_b32 v0, v126, 8, v57
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; ALIGNED-NEXT:    v_lshl_or_b32 v126, v57, 8, v78
+; ALIGNED-NEXT:    v_lshl_or_b32 v126, v43, 8, v78
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(1)
 ; ALIGNED-NEXT:    v_add_co_u32 v2, vcc_lo, v2, s4
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
@@ -15164,7 +15158,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v16 offset:246
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v18 offset:244
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v19 offset:240
-; ALIGNED-NEXT:    buffer_store_dword v76, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT:    buffer_store_dword v73, off, s[0:3], s32 offset:504
 ; ALIGNED-NEXT:    buffer_store_dword v104, off, s[0:3], s32 offset:508
 ; ALIGNED-NEXT:    buffer_store_dword v123, off, s[0:3], s32 offset:500
 ; ALIGNED-NEXT:    v_lshl_or_b32 v126, v0, 16, v126
@@ -15294,12 +15288,12 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v62 offset:174
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v63 offset:172
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v59 offset:168
-; ALIGNED-NEXT:    flat_store_byte v[2:3], v73 offset:162
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v75 offset:162
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v74 offset:163
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v79 offset:161
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v72 offset:167
-; ALIGNED-NEXT:    flat_store_byte v[2:3], v75 offset:165
-; ALIGNED-NEXT:    flat_store_byte v[2:3], v77 offset:166
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v77 offset:165
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v76 offset:166
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v88 offset:164
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v89 offset:160
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
@@ -15798,11 +15792,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:24
-; ALIGNED-NEXT:    flat_store_byte v[2:3], v43 offset:18
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v57 offset:18
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:19
-; ALIGNED-NEXT:    flat_store_byte v[2:3], v57 offset:17
+; ALIGNED-NEXT:    flat_store_byte v[2:3], v43 offset:17
 ; ALIGNED-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
 ; ALIGNED-NEXT:    s_waitcnt vmcnt(0)
 ; ALIGNED-NEXT:    flat_store_byte v[2:3], v0 offset:23
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
index 4e5688adcd6b..f08ea27040fb 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
@@ -485,12 +485,12 @@ define void @memmove_p0_p3_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
 ; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
 ; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
@@ -540,12 +540,12 @@ define void @memmove_p0_p3_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
 ; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
 ; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
 ; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
 ; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
@@ -939,19 +939,18 @@ define void @memmove_p0_p5_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:28
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:30
 ; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
@@ -967,18 +966,18 @@ define void @memmove_p0_p5_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1009,19 +1008,18 @@ define void @memmove_p0_p5_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:28
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:30
 ; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
@@ -1037,18 +1035,18 @@ define void @memmove_p0_p5_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1079,20 +1077,20 @@ define void @memmove_p0_p5_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
 ; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1149,20 +1147,20 @@ define void @memmove_p0_p5_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
 ; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2079,20 +2077,18 @@ define void @memmove_p1_p5_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    global_store_byte v[0:1], v11, off offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_short v[0:1], v11, off offset:28
+; CHECK-NEXT:    global_store_byte v[0:1], v10, off offset:30
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[7:9], off offset:16
@@ -2147,20 +2143,18 @@ define void @memmove_p1_p5_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    global_store_byte v[0:1], v11, off offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_short v[0:1], v11, off offset:28
+; CHECK-NEXT:    global_store_byte v[0:1], v10, off offset:30
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[7:9], off offset:16
@@ -2215,20 +2209,18 @@ define void @memmove_p1_p5_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    global_store_byte v[0:1], v11, off offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_short v[0:1], v11, off offset:28
+; CHECK-NEXT:    global_store_byte v[0:1], v10, off offset:30
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[7:9], off offset:16
@@ -2283,20 +2275,18 @@ define void @memmove_p1_p5_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    global_store_byte v[0:1], v11, off offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_short v[0:1], v11, off offset:28
+; CHECK-NEXT:    global_store_byte v[0:1], v10, off offset:30
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[7:9], off offset:16
@@ -3266,21 +3256,20 @@ define void @memmove_p3_p5_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
-; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    ds_write_b32 v0, v8 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    ds_write_b16 v0, v9 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    ds_write_b8 v0, v10 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    ds_write_b32 v0, v9 offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    ds_write_b16 v0, v10 offset:28
+; CHECK-NEXT:    ds_write_b8 v0, v8 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
 ; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
@@ -3339,21 +3328,20 @@ define void @memmove_p3_p5_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
-; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    ds_write_b32 v0, v8 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    ds_write_b16 v0, v9 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    ds_write_b8 v0, v10 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    ds_write_b32 v0, v9 offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    ds_write_b16 v0, v10 offset:28
+; CHECK-NEXT:    ds_write_b8 v0, v8 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
 ; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
@@ -3485,22 +3473,21 @@ define void @memmove_p3_p5_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x8
-; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    ds_write2_b32 v0, v7, v8 offset0:5 offset1:6
-; CHECK-NEXT:    ds_write_b32 v0, v6 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    ds_write_b16 v0, v9 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    ds_write_b8 v0, v10 offset:30
+; CHECK-NEXT:    ds_write2_b32 v0, v8, v9 offset0:5 offset1:6
+; CHECK-NEXT:    ds_write_b32 v0, v7 offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    ds_write_b16 v0, v10 offset:28
+; CHECK-NEXT:    ds_write_b8 v0, v6 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    ds_write_b128 v0, v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index 258ed848c9fa..da8b257bd69b 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -33,25 +33,15 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_mul_f32_e32 v2, 1.0, v2
-; GFX8-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v2
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
-; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
 ; GFX8-NEXT:    s_movk_i32 s4, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_minimumnum_bf16:
@@ -66,24 +56,15 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX900-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX900-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v3, v3, v2, s4
-; GFX900-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
 ; GFX900-NEXT:    s_movk_i32 s4, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_minimumnum_bf16:
@@ -92,7 +73,7 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
@@ -102,10 +83,6 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
@@ -131,21 +108,13 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_minimumnum_bf16:
@@ -154,39 +123,31 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_bf16:
@@ -205,26 +166,14 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_minimumnum_bf16:
@@ -237,45 +186,37 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_bf16:
@@ -301,29 +242,17 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call bfloat @llvm.minimumnum.bf16(bfloat %x, bfloat %y)
   ret bfloat %result
@@ -539,26 +468,16 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v5
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -568,23 +487,14 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -601,25 +511,16 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v5
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -629,21 +530,13 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v2, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -658,16 +551,12 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v3, v0, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
@@ -691,9 +580,6 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -731,37 +617,21 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v2, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v7, v4, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -775,66 +645,52 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v5, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16:
@@ -847,60 +703,39 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -918,71 +753,56 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v5, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16:
@@ -1000,69 +820,48 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -1415,26 +1214,16 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -1444,23 +1233,14 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -1470,22 +1250,13 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -1503,25 +1274,16 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -1531,22 +1293,14 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -1556,21 +1310,13 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v4, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -1585,16 +1331,12 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v5, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -1618,9 +1360,6 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
@@ -1644,9 +1383,6 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -1664,180 +1400,125 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX10-LABEL: v_minimumnum_v3bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v5, v5
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_sdwa v0, v0, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_sdwa v10, v0, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v11
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX10-NEXT:    v_or_b32_e32 v11, 0x400000, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v10, v7, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v10, v10, v7, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v12, v8, 16, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v4, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_minimumnum_v3bf16:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v5.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v7, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v9, v11
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -1846,88 +1527,59 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v9
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX11-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_minimumnum_v3bf16:
@@ -1938,104 +1590,77 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v5.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v7, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v9, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v8, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -2048,98 +1673,67 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v9
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v7, v7, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX12-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
@@ -2147,7 +1741,7 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x bfloat> @llvm.minimumnum.v3bf16(<3 x bfloat> %x, <3 x bfloat> %y)
   ret <3 x bfloat> %result
@@ -2603,26 +2197,16 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -2634,23 +2218,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT:    v_mul_f32_e32 v7, 1.0, v7
-; GFX8-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v7
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s4, v8
-; GFX8-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v6
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -2660,23 +2235,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2686,22 +2252,13 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
@@ -2721,25 +2278,16 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -2751,22 +2299,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX900-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX900-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX900-NEXT:    v_add3_u32 v8, v8, v7, s4
-; GFX900-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v6
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -2776,22 +2316,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2801,21 +2333,13 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v5, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v4, v1, s4
@@ -2831,17 +2355,13 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v5, v1, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
-; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v2
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v2
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -2865,9 +2385,6 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
@@ -2891,9 +2408,6 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
@@ -2917,9 +2431,6 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -2964,78 +2475,46 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v8, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX10-NEXT:    v_bfe_u32 v14, v8, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v7, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v13
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v12
-; GFX10-NEXT:    v_add3_u32 v12, v14, v8, 0x7fff
-; GFX10-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v11
-; GFX10-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v13, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX10-NEXT:    v_bfe_u32 v15, v9, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX10-NEXT:    v_add3_u32 v12, v12, v11, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_or_b32_e32 v16, 0x400000, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX10-NEXT:    v_bfe_u32 v12, v6, 16, 1
-; GFX10-NEXT:    v_add3_u32 v14, v15, v9, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX10-NEXT:    v_add3_u32 v12, v12, v6, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v12, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
@@ -3054,232 +2533,162 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v10, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v9, v13
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v9, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v11, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v4, v1, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3297,123 +2706,94 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v10, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v9, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v11, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v9, v13
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v11, v11, v11
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX12-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16:
@@ -3423,126 +2803,80 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v13
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
@@ -3553,10 +2887,10 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
@@ -4156,26 +3490,16 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v7
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v8, v9
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v7, v6, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX8-NEXT:    v_mul_f32_e32 v8, 1.0, v8
-; GFX8-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v8
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, s4, v9
-; GFX8-NEXT:    v_or_b32_e32 v10, 0x400000, v8
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v10, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v8
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v7
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
-; GFX8-NEXT:    v_and_b32_e32 v7, 0xffff0000, v8
+; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX8-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
@@ -4187,23 +3511,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v8, v7, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX8-NEXT:    v_mul_f32_e32 v9, 1.0, v9
-; GFX8-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v9
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s4, v10
-; GFX8-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v8
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
-; GFX8-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
 ; GFX8-NEXT:    v_and_b32_e32 v8, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v3
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
@@ -4215,23 +3530,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX8-NEXT:    v_mul_f32_e32 v10, 1.0, v10
-; GFX8-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v10
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s4, v11
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v9
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
@@ -4241,23 +3547,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX8-NEXT:    v_mul_f32_e32 v9, 1.0, v9
-; GFX8-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v9
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s4, v10
-; GFX8-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v9
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
@@ -4267,23 +3564,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v9, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v5
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, s4, v9
-; GFX8-NEXT:    v_or_b32_e32 v10, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v5
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
@@ -4293,22 +3581,13 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v9, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v8
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
@@ -4331,25 +3610,16 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v7
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v8, v9
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v7, v6, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX900-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX900-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v9, v9, v8, s4
-; GFX900-NEXT:    v_or_b32_e32 v10, 0x400000, v8
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v9, v10, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v8
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v7
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
-; GFX900-NEXT:    v_and_b32_e32 v7, 0xffff0000, v8
+; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX900-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
@@ -4361,22 +3631,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v8, v7, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX900-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX900-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX900-NEXT:    v_add3_u32 v10, v10, v9, s4
-; GFX900-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v8
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
-; GFX900-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
 ; GFX900-NEXT:    v_and_b32_e32 v8, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v3
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
@@ -4388,22 +3650,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX900-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX900-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX900-NEXT:    v_add3_u32 v11, v11, v10, s4
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v9
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
@@ -4413,22 +3667,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX900-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX900-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX900-NEXT:    v_add3_u32 v10, v10, v9, s4
-; GFX900-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v9
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
@@ -4438,22 +3684,14 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v9, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v9, v9, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v10, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v5
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
@@ -4463,21 +3701,13 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v9, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v8, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v7, v1, s4
@@ -4492,21 +3722,17 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX950-NEXT:    v_and_b32_e32 v9, 0xffff0000, v5
-; GFX950-NEXT:    v_and_b32_e32 v10, 0xffff0000, v4
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
-; GFX950-NEXT:    v_and_b32_e32 v11, 0xffff0000, v3
+; GFX950-NEXT:    v_and_b32_e32 v10, 0xffff0000, v4
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v7, v6, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v7
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v8, v9
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_and_b32_e32 v11, 0xffff0000, v3
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v7, v6, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX950-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v8, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
@@ -4531,9 +3757,6 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v8, v7, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX950-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
@@ -4557,9 +3780,6 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX950-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
@@ -4583,9 +3803,6 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX950-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc
@@ -4609,9 +3826,6 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v9, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v4, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
@@ -4635,9 +3849,6 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v4
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v3, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
@@ -4663,156 +3874,108 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v1
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
 ; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v0
+; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_sdwa v12, v2, v7, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v0
+; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v10, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v12, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v15, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v13
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v9
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v10, v7, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v14, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v16, v17
-; GFX10-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v9, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_bfe_u32 v16, v10, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v11, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v16, v16, v10, 0x7fff
-; GFX10-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v8
-; GFX10-NEXT:    v_bfe_u32 v17, v13, 16, 1
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v13
-; GFX10-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v8, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v16, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
-; GFX10-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v17, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v10
-; GFX10-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT:    v_bfe_u32 v18, v14, 16, 1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v10, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v14, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v12, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v9, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v11, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v8, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
-; GFX10-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_add3_u32 v9, v18, v14, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v10, 0x400000, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v10, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v11, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v0
-; GFX10-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v9
-; GFX10-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v10
-; GFX10-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX10-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX10-NEXT:    v_bfe_u32 v16, v12, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v11, v14, v13, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v13
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_bfe_u32 v17, v10, 16, 1
-; GFX10-NEXT:    v_add3_u32 v13, v16, v12, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v14, vcc_lo
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_add3_u32 v16, v17, v10, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v17, 0x400000, v10
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v16, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v15, v8, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v4, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    v_perm_b32 v1, v6, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v2, v7, v2, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4820,165 +3983,126 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v4
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v3
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v1.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v12, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.h, v6.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v14, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v0
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s3
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v18, v19
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v8.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v9.l, v7.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v20, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.h, v10.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v14, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v14
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v14, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v16, v16, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v14, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s5, v13, v18
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s4
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v16, v20, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v19, v22, s6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v12.h, v6.l, s0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v7.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v11.l, v10.l, s5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v9.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v12.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v7, v7, v7 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s2
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v13, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v12, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v9.l, v8.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v6.l, s2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v13.l, v8.l, s3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v14, v16
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v7
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v15, v14
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v14, v6, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v9
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v16, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v9, v9, v12 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v14, 16, 1
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v1.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v8, v12, v14, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v14
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v10, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v12, v15, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v10, v14, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v10, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v7.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v14, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v4.l, v1.l, s1
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v0.h, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v1.h, v5.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v11
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v3.h, s2
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -4987,176 +4111,116 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_and_b32 v8, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v10 :: v_dual_lshlrev_b32 v11, 16, v7
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v6, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v7, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v10, v9 :: v_dual_lshlrev_b32 v13, 16, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v10, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v11 :: v_dual_max_f32 v9, v9, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v13, v9, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v13, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v14, v11 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v18, v12, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v16, v18, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v16, v15, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v15, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v15, v12 :: v_dual_lshlrev_b32 v14, 16, v9
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v14
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v10, v6 :: v_dual_lshlrev_b32 v13, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v15, v8 :: v_dual_lshlrev_b32 v17, 16, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v18, v6 :: v_dual_lshlrev_b32 v9, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v8 :: v_dual_lshlrev_b32 v15, 16, v7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v13, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v10, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v10, v13, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v15, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v8, v10, v13, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v16, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v11, 16, v2
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v5, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v8
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v4, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v3, v0 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v13, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v16, v10, 16, 1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v0 :: v_dual_lshlrev_b32 v12, 16, v1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v17, v9, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v16, v10, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_add3_u32 v16, v17, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v17, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v13, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v16, v17, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v15, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v4, v1 :: v_dual_lshlrev_b32 v11, 16, v3
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v9, v2 :: v_dual_lshlrev_b32 v13, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v3, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v13, v0 :: v_dual_and_b32 v5, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v4, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v4, 16, v10
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v3, 16, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v7, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v7, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v8, v1, 0x5040100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5168,191 +4232,143 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v5
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v3
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v1.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v12, v12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.h, v6.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v6.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v9.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v14, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v0
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v16
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s3
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v18, v19
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.h, v6.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v8.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.h, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v9.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v9.l, v7.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v20, v20
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, s0
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.h, v10.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v14, v14, v14
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v14, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v14
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v12, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_add3_u32 v19, v19, v14, 0x7fff
-; GFX12-TRUE16-NEXT:    v_add3_u32 v16, v16, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v14, v14
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s5, v13, v18
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s4
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v16, v20, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v19, v22, s6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v9.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v12.h, v6.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v7.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v11.l, v10.l, s5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v15
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v9.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v16
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v12.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v3.h, s2
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v13, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v9.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v8
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v12, v13
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v11.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v7, v7, v7 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v9.l, v8.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v6.l, s2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v13
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v15, v14
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v14, v6, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v9
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v16, v7, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v13.l, v8.l, s3
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v13.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v4.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v15
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v14, v16
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v9, v9, v12 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v14, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.h, v1.h, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v8, v12, v14, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v14
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v7, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v10, v6, 16, 1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v10, v10, v6, 0x7fff
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v7, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v12, v15, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v1.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v7
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v10, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v10, v14, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v7.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v14, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v4.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v11
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v9.h, v0.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v1.h, v5.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v6.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v8.h, v2.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v3.h, s2
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -5365,211 +4381,147 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v1
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v3
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_and_b32 v8, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v6, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v10 :: v_dual_lshlrev_b32 v11, 16, v7
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v10, v9 :: v_dual_lshlrev_b32 v13, 16, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v10, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v11 :: v_dual_max_num_f32 v9, v9, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v13, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v13, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v17
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v14, v11 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v18, v12, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v16, v18, v12, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v16, v15, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v15, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v15, v8 :: v_dual_lshlrev_b32 v17, 16, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v18, v6 :: v_dual_lshlrev_b32 v9, 16, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v10, v13, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v15, v12 :: v_dual_lshlrev_b32 v14, 16, v9
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v14
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v15, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v10, v13, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v10, v6 :: v_dual_lshlrev_b32 v13, 16, v11
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v16, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v11, v8 :: v_dual_lshlrev_b32 v15, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v16, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v5, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v10, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v8
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v4, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v13, v12, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v3, v0 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v13, v12, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v16, v10, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v17, v9, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v11, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v16, v10, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_add3_u32 v16, v17, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v17, 0x400000, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v13, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v0 :: v_dual_lshlrev_b32 v12, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v16, v17, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v5, v2 :: v_dual_lshlrev_b32 v10, 16, v4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v15, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v4, v1 :: v_dual_lshlrev_b32 v11, 16, v3
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v9, v2 :: v_dual_lshlrev_b32 v13, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v11
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v3, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v13, v0 :: v_dual_and_b32 v5, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_and_b32 v4, 0xffff0000, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v4 :: v_dual_lshlrev_b32 v4, 16, v10
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v3 :: v_dual_lshlrev_b32 v3, 16, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v11
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v7, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
+; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v7, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v9, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v8, v1, 0x5040100
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x bfloat> @llvm.minimumnum.v6bf16(<6 x bfloat> %x, <6 x bfloat> %y)
@@ -5644,26 +4596,16 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX8-NEXT:    v_mul_f32_e32 v10, 1.0, v10
-; GFX8-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v10
-; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s5, v11
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
 ; GFX8-NEXT:    s_movk_i32 s4, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v6
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
@@ -5675,23 +4617,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v10
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v10, v9, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX8-NEXT:    v_mul_f32_e32 v11, 1.0, v11
-; GFX8-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v11
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, s5, v12
-; GFX8-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v10, vcc
-; GFX8-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX8-NEXT:    v_and_b32_e32 v10, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v5
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
@@ -5703,23 +4636,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v10, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX8-NEXT:    v_mul_f32_e32 v12, 1.0, v12
-; GFX8-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v12
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, s5, v13
-; GFX8-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
-; GFX8-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX8-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
@@ -5731,23 +4655,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v12, v11, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX8-NEXT:    v_mul_f32_e32 v13, 1.0, v13
-; GFX8-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, v14, v13
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, s5, v14
-; GFX8-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc
-; GFX8-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
@@ -5757,23 +4672,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX8-NEXT:    v_mul_f32_e32 v12, 1.0, v12
-; GFX8-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v12
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, s5, v13
-; GFX8-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
+; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
@@ -5783,23 +4689,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v7
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT:    v_mul_f32_e32 v7, 1.0, v7
-; GFX8-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v7
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, s5, v12
-; GFX8-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v12, v13, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
@@ -5809,23 +4706,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s5, v7
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v12, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
@@ -5835,22 +4723,13 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s5, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v11
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
@@ -5875,25 +4754,16 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v11
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX900-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX900-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v11, v11, v10, s4
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v9
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v11, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v6
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
@@ -5905,22 +4775,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v10
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v10, v9, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX900-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX900-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX900-NEXT:    v_add3_u32 v12, v12, v11, s4
-; GFX900-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v10
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v9, v10, vcc
-; GFX900-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX900-NEXT:    v_and_b32_e32 v10, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v5
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
@@ -5932,22 +4794,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v11, v10, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX900-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX900-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX900-NEXT:    v_add3_u32 v13, v13, v12, s4
-; GFX900-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v11
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
-; GFX900-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX900-NEXT:    v_and_b32_e32 v11, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
@@ -5959,22 +4813,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v12, v11, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX900-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX900-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX900-NEXT:    v_add3_u32 v14, v14, v13, s4
-; GFX900-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v12
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc
-; GFX900-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v14, v11, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
@@ -5984,22 +4830,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX900-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX900-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX900-NEXT:    v_add3_u32 v13, v13, v12, s4
-; GFX900-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v7
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX900-NEXT:    v_and_b32_e32 v7, 0xffff0000, v12
+; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v7
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
@@ -6009,22 +4847,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v7
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX900-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX900-NEXT:    v_bfe_u32 v12, v7, 16, 1
-; GFX900-NEXT:    v_add3_u32 v12, v12, v7, s4
-; GFX900-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v12, v13, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v7
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v6
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
@@ -6034,22 +4864,14 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v5, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v12, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
@@ -6059,21 +4881,13 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v4, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v11, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v10, v1, s4
@@ -6089,23 +4903,20 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v8, v8
 ; GFX950-NEXT:    v_and_b32_e32 v11, 0xffff0000, v7
-; GFX950-NEXT:    v_and_b32_e32 v12, 0xffff0000, v6
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
-; GFX950-NEXT:    v_and_b32_e32 v13, 0xffff0000, v5
+; GFX950-NEXT:    v_and_b32_e32 v12, 0xffff0000, v6
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v11
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
-; GFX950-NEXT:    v_and_b32_e32 v14, 0xffff0000, v4
+; GFX950-NEXT:    v_and_b32_e32 v13, 0xffff0000, v5
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX950-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v8
-; GFX950-NEXT:    s_nop 1
+; GFX950-NEXT:    v_and_b32_e32 v14, 0xffff0000, v4
+; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v9
 ; GFX950-NEXT:    s_nop 1
@@ -6128,9 +4939,6 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v10, v9, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX950-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
@@ -6155,9 +4963,6 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v11, v10, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v12, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v10
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
@@ -6181,9 +4986,6 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v14
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v12, v11, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX950-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v13, v11, vcc
@@ -6207,9 +5009,6 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v12
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v12, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
@@ -6233,9 +5032,6 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
@@ -6259,9 +5055,6 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v5, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
@@ -6285,9 +5078,6 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
@@ -6310,213 +5100,149 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
 ; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
 ; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v6
-; GFX10-NEXT:    v_and_b32_e32 v16, 0xffff0000, v1
-; GFX10-NEXT:    v_and_b32_e32 v17, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
 ; GFX10-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
+; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v14, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v1
-; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v5
-; GFX10-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v10, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v22, v11, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v13
-; GFX10-NEXT:    v_add3_u32 v22, v22, v11, 0x7fff
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v19
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v13, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v22, v19, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v12, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
+; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v9, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_max_f32_e32 v15, v16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v11
-; GFX10-NEXT:    v_bfe_u32 v19, v15, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v22, 0x400000, v15
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v21, v20, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX10-NEXT:    v_add3_u32 v19, v19, v15, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v20, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v19, v22, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
-; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v19, v20
-; GFX10-NEXT:    v_bfe_u32 v19, v9, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v18, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v15
+; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX10-NEXT:    v_and_b32_e32 v15, 0xffff0000, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v11, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
-; GFX10-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX10-NEXT:    v_add3_u32 v13, v19, v9, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v13, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_bfe_u32 v13, v17, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v14, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_add3_u32 v13, v13, v17, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v11, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v17, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v14, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v16, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v15, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v12
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v13, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v7, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v12, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v19, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
-; GFX10-NEXT:    v_max_f32_e32 v11, v15, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v14, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
-; GFX10-NEXT:    v_bfe_u32 v16, v11, 16, 1
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v11
-; GFX10-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX10-NEXT:    v_add3_u32 v15, v16, v11, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v6, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v4
-; GFX10-NEXT:    v_bfe_u32 v17, v14, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_add3_u32 v17, v17, v14, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v21, v20
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v14
-; GFX10-NEXT:    v_max_f32_e32 v16, v16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v19, v12, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX10-NEXT:    v_bfe_u32 v19, v16, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX10-NEXT:    v_or_b32_e32 v20, 0x400000, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX10-NEXT:    v_or_b32_e32 v18, 0x400000, v16
-; GFX10-NEXT:    v_bfe_u32 v17, v13, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v19, v16, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX10-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v18, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v17, v20, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v5, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v16
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    v_perm_b32 v2, v10, v2, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_perm_b32 v0, v12, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    v_perm_b32 v0, v11, v0, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX10-NEXT:    v_perm_b32 v1, v9, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v3, v8, v3, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6524,458 +5250,321 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v6
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v4
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v15, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v7.h, v8.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v14, v14
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v17
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v11.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v12, v13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v0.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v9.l, v8.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v5.h, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v9.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v15, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v5.h, v12.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v13, v18
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v11.l, v10.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v4.h, v13.l, s2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v21, v16, v16
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v17, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v10.l, v8.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v11.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v11.l, v9.l, s3
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v15.l, v12.l, s2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v21, v21
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v19, v19, v22, s2
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v18, v20
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v16, v16, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v19.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v19
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v13.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v9.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v20, v18, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v18, v16, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v11.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v17.h, v10.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v14.l, v13.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v19.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v18, v18, v16, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, 0x400000, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v18, v19, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v15.l, v9.l, s4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v8.h, v11.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v4.h, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v16, v16
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v11, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v15.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.h, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v14.l, v12.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v15.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v15.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v12, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v9, v18, s2
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v16, v12
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v17.h, v8.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v10.h, v13.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v9.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v13, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v9, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v13, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v8.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v10.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v16, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v15, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v8.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v12, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v13
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v14, v14, v13, 0x7fff
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v15, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v4.l, v0.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v12, v12, v12 :: v_dual_max_f32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v14, v16, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v6.l, v2.l, s3
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v9, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_add3_u32 v14, v14, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v0.h, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v14, v15, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v16, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v14, v13
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v0.h, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v12.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v14
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v12.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v11.h, v1.h, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v6
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.l, v3.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v4.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v1.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v8bf16:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v8 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v14, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v12, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v14
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v13
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v19
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v13, v10 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v11, v11, v11 :: v_dual_and_b32 v16, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v22, v11, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v11
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v18
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v12 :: v_dual_and_b32 v15, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v22, v11, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v22, v19 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v15, v16, v16
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v19, v15, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v21, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX11-FAKE16-NEXT:    v_add3_u32 v19, v19, v15, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v12, v8 :: v_dual_and_b32 v13, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v12, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v20, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v19, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v18
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v17, v8 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v19, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v18, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v15, 16, v11
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v15
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v10 :: v_dual_and_b32 v15, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v17, 16, v17
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v19, v9, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v19, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v19, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v13, v17, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_lshlrev_b32 v13, 16, v12
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_add3_u32 v13, v13, v17, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v14 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v3
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v7, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v15
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v9 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_and_b32 v13, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v16, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v15 :: v_dual_lshlrev_b32 v16, 16, v13
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v19, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v12
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v16
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v12 :: v_dual_lshlrev_b32 v16, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v14, v18 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v12 :: v_dual_lshlrev_b32 v12, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v12
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v11, v15, v15 :: v_dual_lshlrev_b32 v18, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v5 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v4
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v16, v11, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v11
-; GFX11-FAKE16-NEXT:    v_add3_u32 v15, v16, v11, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v16
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v17, v14, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v5, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_add3_u32 v17, v17, v14, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v18 :: v_dual_lshlrev_b32 v16, 16, v16
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v21, v20
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v14
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v16, v16, v16 :: v_dual_and_b32 v13, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v4, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v19, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v6 :: v_dual_lshlrev_b32 v13, 16, v15
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v19, v16, 16, 1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v12, v3 :: v_dual_lshlrev_b32 v16, 16, v2
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v20, 0x400000, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v16
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v17, v13, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v7, v19, v16, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX11-FAKE16-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v18, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v7 :: v_dual_lshlrev_b32 v14, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v16, v14
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v7, 16, v12
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v17, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v13, 16, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v14, v2 :: v_dual_lshlrev_b32 v15, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v5, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v16
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_and_b32 v5, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_and_b32 v11, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v10, v2, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v13
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v11, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v12, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v9, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v8, v3, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -6988,260 +5577,200 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v7
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v6
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v5
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v15, v15
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v7.h, v8.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v14, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v17
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v10.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v11.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v12, v13
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v0.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v9.l, v8.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v5.h, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v9.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v15, v17
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v5.h, v12.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v13, v18
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v14.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v10.l, v8.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v11.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v11.l, v10.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v10.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v4.h, v13.l, s2
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v21, v16, v16
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v17, v18
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v11.l, v9.l, s3
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v14.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v21, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v15.l, v12.l, s2
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v17, v17, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v21, v21
-; GFX12-TRUE16-NEXT:    v_add3_u32 v19, v19, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v9.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v19, v19, v22, s2
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v18, v20
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v17
-; GFX12-TRUE16-NEXT:    v_add3_u32 v20, v23, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v16, v16, v16
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v19.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v19
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v13.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v15.l, v9.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v8.h, v11.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v0.h, v4.h, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v16, v16
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v11, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v20, v18, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v18, v16, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v11.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v17.h, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v14.l, v13.l, s2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v19.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v18, v18, v16, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v19, 0x400000, v16
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v18, v19, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v15.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.h, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v14.l, v12.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v7
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v10, v10, v10 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v15.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v9.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v16
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v13, v15
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v12, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v8.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v14
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v11.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v8.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v0
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v9, v18, s2
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v16, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v17.h, v8.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v10.h, v13.l, s3
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v12, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v3.h, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v4
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v15, v15
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v13, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v4.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v9, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v16, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v15, v14
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v14, v14
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v11, v12, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v13
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v15, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v14, v14, v13, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v4.l, v0.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v12, v12, v12 :: v_dual_max_num_f32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v15
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v13, v14, v16, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v6.l, v2.l, s3
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v12, 16, 1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v7, v9, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v15, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_add3_u32 v14, v14, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v7, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v0.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v14, v13
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v14, v15, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v16, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v0.h, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v12.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v12
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v14
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.l, v3.h, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v12.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v11.h, v1.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v4.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v1.h, s3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.h, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v13.h, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v1, v3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7252,256 +5781,170 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v3
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v5
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v7
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v8 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v14, v13, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v12, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v14
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v13
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v19
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v13, v10 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v11, v11, v11 :: v_dual_and_b32 v16, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v22, v11, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v11
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v18
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v12 :: v_dual_and_b32 v15, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v22, v11, 0x7fff
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v22, v19 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v15, v16, v16
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v19, v15, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v21, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX12-FAKE16-NEXT:    v_add3_u32 v19, v19, v15, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v12, v8 :: v_dual_and_b32 v13, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v12, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v20, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v19, v22, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v18
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v9 :: v_dual_lshlrev_b32 v15, 16, v11
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v15
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v17, v8 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v19, v20
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v10 :: v_dual_and_b32 v15, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v18, v16, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v16, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v17, 16, v17
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v17, v17, v17
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_lshlrev_b32 v13, 16, v12
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v13, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v11, v10 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v19, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v19, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v13, v19, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v13, v17, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    v_add3_u32 v13, v13, v17, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v9 :: v_dual_lshlrev_b32 v18, 16, v14
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v7
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v11, v14 :: v_dual_lshlrev_b32 v15, 16, v7
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v3
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v14, v10 :: v_dual_and_b32 v13, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v6
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v16, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v7, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v15 :: v_dual_lshlrev_b32 v16, 16, v13
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v19, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v12
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v14, v18 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v11, v15, v15 :: v_dual_lshlrev_b32 v18, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v14
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v16
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v5
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v12 :: v_dual_lshlrev_b32 v16, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v5 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v14, v14, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v11, v9 :: v_dual_lshlrev_b32 v14, 16, v3
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v12 :: v_dual_lshlrev_b32 v12, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v17, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v4
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v16, v11, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v11
-; GFX12-FAKE16-NEXT:    v_add3_u32 v15, v16, v11, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v16
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v17, v14, 16, 1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v5, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    v_add3_u32 v17, v17, v14, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v7, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v15, v18 :: v_dual_lshlrev_b32 v16, 16, v16
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v21, v20
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v14
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v16, v16, v16 :: v_dual_and_b32 v13, 0xffff0000, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v6 :: v_dual_lshlrev_b32 v13, 16, v15
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v4, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v12, v3 :: v_dual_lshlrev_b32 v16, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v19, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v19, v16, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v20, 0x400000, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v2 :: v_dual_lshlrev_b32 v15, 16, v0
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v18, 0x400000, v16
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v17, v13, 16, 1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v7, v19, v16, 0x7fff
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX12-FAKE16-NEXT:    v_add3_u32 v17, v17, v13, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v7 :: v_dual_lshlrev_b32 v14, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v18, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v16, 16, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v16, v14
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v6, v2 :: v_dual_lshlrev_b32 v7, 16, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v17, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v0 :: v_dual_lshlrev_b32 v13, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v4
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v14, v2 :: v_dual_lshlrev_b32 v15, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v5, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v17, v16
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
@@ -7510,25 +5953,26 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_and_b32 v5, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_and_b32 v11, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v10, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v11, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v12, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v9, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v15, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v8, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x bfloat> @llvm.minimumnum.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y)
@@ -7653,26 +6097,16 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v18, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v16, v17, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX8-NEXT:    v_mul_f32_e32 v18, 1.0, v18
-; GFX8-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v18
-; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, s5, v19
-; GFX8-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
 ; GFX8-NEXT:    s_movk_i32 s4, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v16, v17, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX8-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX8-NEXT:    v_and_b32_e32 v17, 0xffff0000, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v16, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
@@ -7684,23 +6118,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v19, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v17, v18, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX8-NEXT:    v_mul_f32_e32 v19, 1.0, v19
-; GFX8-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v19
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, s5, v20
-; GFX8-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
-; GFX8-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX8-NEXT:    v_and_b32_e32 v18, 0xffff0000, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v20, v17, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v5
@@ -7712,23 +6137,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v18
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v20, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v20, v18, v19, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX8-NEXT:    v_mul_f32_e32 v20, 1.0, v20
-; GFX8-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v20
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, s5, v21
-; GFX8-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX8-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX8-NEXT:    v_and_b32_e32 v19, 0xffff0000, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v21, v18, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
@@ -7740,23 +6156,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v21, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v21, v19, v20, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX8-NEXT:    v_mul_f32_e32 v21, 1.0, v21
-; GFX8-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v21
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, s5, v22
-; GFX8-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX8-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX8-NEXT:    v_and_b32_e32 v20, 0xffff0000, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v22, v19, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v11
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
@@ -7768,23 +6175,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v20
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v22, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v20, v21, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX8-NEXT:    v_mul_f32_e32 v22, 1.0, v22
-; GFX8-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v22
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, s5, v23
-; GFX8-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX8-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX8-NEXT:    v_and_b32_e32 v21, 0xffff0000, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v23, v20, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
@@ -7796,23 +6194,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v23, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v23, v21, v22, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX8-NEXT:    v_mul_f32_e32 v23, 1.0, v23
-; GFX8-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, v24, v23
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, s5, v24
-; GFX8-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX8-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX8-NEXT:    v_and_b32_e32 v22, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v24, v21, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v9
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
@@ -7824,23 +6213,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v24, v22, v23, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX8-NEXT:    v_mul_f32_e32 v24, 1.0, v24
-; GFX8-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, v25, v24
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, s5, v25
-; GFX8-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX8-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_and_b32_e32 v23, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v25, v22, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
@@ -7852,23 +6232,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v25, v23, v24, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX8-NEXT:    v_mul_f32_e32 v25, 1.0, v25
-; GFX8-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, v26, v25
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, s5, v26
-; GFX8-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v26, v24, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX8-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
@@ -7878,23 +6249,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v24, v15, v7, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX8-NEXT:    v_mul_f32_e32 v24, 1.0, v24
-; GFX8-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, v25, v24
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, s5, v25
-; GFX8-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX8-NEXT:    v_and_b32_e32 v15, 0xffff0000, v24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v15
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
@@ -7904,23 +6266,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX8-NEXT:    v_mul_f32_e32 v15, 1.0, v15
-; GFX8-NEXT:    v_bfe_u32 v24, v15, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, v24, v15
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, s5, v24
-; GFX8-NEXT:    v_or_b32_e32 v25, 0x400000, v15
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v24, v25, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX8-NEXT:    v_and_b32_e32 v14, 0xffff0000, v15
+; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v14
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
@@ -7930,23 +6283,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v15, 16, v5
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v15, v14
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v13, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX8-NEXT:    v_mul_f32_e32 v14, 1.0, v14
-; GFX8-NEXT:    v_bfe_u32 v15, v14, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v15, vcc, v15, v14
-; GFX8-NEXT:    v_add_u32_e32 v15, vcc, s5, v15
-; GFX8-NEXT:    v_or_b32_e32 v24, 0x400000, v14
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v15, v24, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX8-NEXT:    v_and_b32_e32 v13, 0xffff0000, v14
+; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v13
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
@@ -7956,23 +6300,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v14, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v12, v4, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX8-NEXT:    v_mul_f32_e32 v13, 1.0, v13
-; GFX8-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, v14, v13
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, s5, v14
-; GFX8-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX8-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
@@ -7982,23 +6317,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v12
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX8-NEXT:    v_mul_f32_e32 v12, 1.0, v12
-; GFX8-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v12
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, s5, v13
-; GFX8-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
@@ -8008,23 +6334,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v11
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX8-NEXT:    v_mul_f32_e32 v11, 1.0, v11
-; GFX8-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v11
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, s5, v12
-; GFX8-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
@@ -8034,23 +6351,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v10
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v9, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX8-NEXT:    v_mul_f32_e32 v10, 1.0, v10
-; GFX8-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v10
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s5, v11
-; GFX8-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
@@ -8060,22 +6368,13 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v9
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v8, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX8-NEXT:    v_mul_f32_e32 v9, 1.0, v9
-; GFX8-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v9
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s5, v10
-; GFX8-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v23
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v22
@@ -8108,25 +6407,16 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v16
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v18, v19
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v16, v17, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX900-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX900-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX900-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v19, v19, v18, s5
-; GFX900-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
 ; GFX900-NEXT:    s_movk_i32 s4, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
+; GFX900-NEXT:    v_cndmask_b32_e32 v18, v16, v17, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v17
-; GFX900-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX900-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_and_b32_e32 v17, 0xffff0000, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v16, v19, v16, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
@@ -8138,22 +6428,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v19, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v19, v17, v18, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX900-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX900-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX900-NEXT:    v_add3_u32 v20, v20, v19, s5
-; GFX900-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v18
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v17, v18, v17, vcc
-; GFX900-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX900-NEXT:    v_and_b32_e32 v18, 0xffff0000, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v17, v20, v17, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v5
@@ -8165,22 +6447,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v18
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v20, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v20, v18, v19, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX900-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX900-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX900-NEXT:    v_add3_u32 v21, v21, v20, s5
-; GFX900-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v19
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX900-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX900-NEXT:    v_and_b32_e32 v19, 0xffff0000, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v21, v18, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v12
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
@@ -8192,22 +6466,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v21, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v21, v19, v20, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX900-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX900-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX900-NEXT:    v_add3_u32 v22, v22, v21, s5
-; GFX900-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v20
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX900-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX900-NEXT:    v_and_b32_e32 v20, 0xffff0000, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v22, v19, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v11
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v3
@@ -8219,22 +6485,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v20
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v22, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v22, v20, v21, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX900-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX900-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX900-NEXT:    v_add3_u32 v23, v23, v22, s5
-; GFX900-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v21
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX900-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX900-NEXT:    v_and_b32_e32 v21, 0xffff0000, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v23, v20, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v10
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v2
@@ -8246,22 +6504,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v23, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v21, v22, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX900-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX900-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX900-NEXT:    v_add3_u32 v24, v24, v23, s5
-; GFX900-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v22
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX900-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX900-NEXT:    v_and_b32_e32 v22, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v24, v21, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v9
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v1
@@ -8273,22 +6523,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v22, v23, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX900-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX900-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX900-NEXT:    v_add3_u32 v25, v25, v24, s5
-; GFX900-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v23
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX900-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX900-NEXT:    v_and_b32_e32 v23, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v25, v22, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
@@ -8300,22 +6542,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v25, v23, v24, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX900-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX900-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX900-NEXT:    v_add3_u32 v26, v26, v25, s5
-; GFX900-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX900-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v26, v24, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX900-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
@@ -8325,22 +6559,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v15, v7, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX900-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX900-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX900-NEXT:    v_add3_u32 v25, v25, v24, s5
-; GFX900-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX900-NEXT:    v_and_b32_e32 v15, 0xffff0000, v24
+; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v15
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
@@ -8350,22 +6576,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX900-NEXT:    v_max_f32_e32 v15, v15, v15
-; GFX900-NEXT:    v_bfe_u32 v24, v15, 16, 1
-; GFX900-NEXT:    v_add3_u32 v24, v24, v15, s5
-; GFX900-NEXT:    v_or_b32_e32 v25, 0x400000, v15
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v15, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v24, v25, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v15
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX900-NEXT:    v_and_b32_e32 v14, 0xffff0000, v15
+; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v14
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
@@ -8375,22 +6593,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v15, 16, v5
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v15, v14
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v13, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX900-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX900-NEXT:    v_bfe_u32 v15, v14, 16, 1
-; GFX900-NEXT:    v_add3_u32 v15, v15, v14, s5
-; GFX900-NEXT:    v_or_b32_e32 v24, 0x400000, v14
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v14, v14
-; GFX900-NEXT:    v_cndmask_b32_e32 v14, v15, v24, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX900-NEXT:    v_and_b32_e32 v13, 0xffff0000, v14
+; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v13
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
@@ -8400,22 +6610,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v14, 16, v4
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v14, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v12, v4, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX900-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX900-NEXT:    v_bfe_u32 v14, v13, 16, 1
-; GFX900-NEXT:    v_add3_u32 v14, v14, v13, s5
-; GFX900-NEXT:    v_or_b32_e32 v15, 0x400000, v13
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v13, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v14, v15, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v14, 16, v13
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX900-NEXT:    v_and_b32_e32 v12, 0xffff0000, v13
+; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
@@ -8425,22 +6627,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v12
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v11, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX900-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX900-NEXT:    v_bfe_u32 v13, v12, 16, 1
-; GFX900-NEXT:    v_add3_u32 v13, v13, v12, s5
-; GFX900-NEXT:    v_or_b32_e32 v14, 0x400000, v12
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v12, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX900-NEXT:    v_and_b32_e32 v11, 0xffff0000, v12
+; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v11
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
@@ -8450,22 +6644,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v11
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX900-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX900-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX900-NEXT:    v_add3_u32 v12, v12, v11, s5
-; GFX900-NEXT:    v_or_b32_e32 v13, 0x400000, v11
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v11, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v12, v13, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX900-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v11
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v10
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v12, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
@@ -8475,22 +6661,14 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v10
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v9, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX900-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX900-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX900-NEXT:    v_add3_u32 v11, v11, v10, s5
-; GFX900-NEXT:    v_or_b32_e32 v12, 0x400000, v10
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v10, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v11, v12, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX900-NEXT:    v_and_b32_e32 v9, 0xffff0000, v10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v10
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
@@ -8500,21 +6678,13 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v9
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v8, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX900-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX900-NEXT:    v_bfe_u32 v10, v9, 16, 1
-; GFX900-NEXT:    v_add3_u32 v10, v10, v9, s5
-; GFX900-NEXT:    v_or_b32_e32 v11, 0x400000, v9
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v9, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX900-NEXT:    v_and_b32_e32 v8, 0xffff0000, v9
+; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v23, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v22, v1, s4
@@ -8534,33 +6704,29 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v16, v16
 ; GFX950-NEXT:    v_and_b32_e32 v19, 0xffff0000, v15
-; GFX950-NEXT:    v_and_b32_e32 v20, 0xffff0000, v14
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v16, v18, v17, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v16
-; GFX950-NEXT:    v_and_b32_e32 v21, 0xffff0000, v13
+; GFX950-NEXT:    v_and_b32_e32 v20, 0xffff0000, v14
 ; GFX950-NEXT:    v_cndmask_b32_e32 v17, v17, v16, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v18, v19
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
-; GFX950-NEXT:    v_and_b32_e32 v22, 0xffff0000, v12
+; GFX950-NEXT:    v_and_b32_e32 v21, 0xffff0000, v13
 ; GFX950-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX950-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v18, v18, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v16
+; GFX950-NEXT:    v_and_b32_e32 v22, 0xffff0000, v12
 ; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v11
-; GFX950-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
 ; GFX950-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v17
+; GFX950-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
 ; GFX950-NEXT:    v_and_b32_e32 v25, 0xffff0000, v9
-; GFX950-NEXT:    v_and_b32_e32 v26, 0xffff0000, v8
 ; GFX950-NEXT:    v_cndmask_b32_e32 v16, v16, v17, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX950-NEXT:    v_and_b32_e32 v17, 0xffff0000, v6
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_and_b32_e32 v26, 0xffff0000, v8
 ; GFX950-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v18, 16, v14
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
@@ -8575,9 +6741,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v20, 16, v5
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v19, v18, v17, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX950-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v19, v19, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v17
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v17, v19, v17, vcc
@@ -8602,9 +6765,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v21, 16, v4
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v20, v19, v18, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX950-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v20, v20, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v18
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
@@ -8629,9 +6789,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v21, v20, v19, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX950-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v21, v21, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v19
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
@@ -8656,9 +6813,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v22, v21, v20, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX950-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v22, v22, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v20
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
@@ -8683,9 +6837,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v23, v22, v21, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v23, v23, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v21
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
@@ -8710,9 +6861,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v24, v23, v22, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX950-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v24, v24, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v22
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
@@ -8736,9 +6884,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v26
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v25, v24, v23, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX950-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v25, v25, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v23
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
@@ -8762,9 +6907,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v24
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v24, v15, v7, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX950-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v24, v24, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
@@ -8788,9 +6930,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v15
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX950-NEXT:    v_max_f32_e32 v15, v15, v15
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
@@ -8814,9 +6953,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v15, v14
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v13, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX950-NEXT:    v_max_f32_e32 v14, v14, v14
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v14, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
@@ -8840,9 +6976,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v14, v13
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v12, v4, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX950-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v4
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
@@ -8866,9 +6999,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v13, v12
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v11, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v12, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
@@ -8892,9 +7022,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v12, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX950-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
@@ -8918,9 +7045,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v11, v10
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v9, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX950-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
@@ -8944,9 +7068,6 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v10, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v8, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX950-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
@@ -8977,881 +7098,634 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v14
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v13
+; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v16, v18, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v16, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v8
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v19
 ; GFX10-NEXT:    v_and_b32_e32 v19, 0xffff0000, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_max_f32_e32 v18, v18, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v19, v21, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_bfe_u32 v23, v18, 16, 1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
-; GFX10-NEXT:    v_add3_u32 v23, v23, v18, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v17, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v21, v22
-; GFX10-NEXT:    v_or_b32_e32 v22, 0x400000, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v19, v20, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v18, 0xffff0000, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, v19, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v23, v22, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v13
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v18
-; GFX10-NEXT:    v_and_b32_e32 v18, 0xffff0000, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v25, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v22, v21, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v4
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v18, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v24, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v27
-; GFX10-NEXT:    v_add3_u32 v17, v26, v21, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX10-NEXT:    v_and_b32_e32 v27, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v22, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v29, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_and_b32_e32 v17, 0xffff0000, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v24, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v28, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v18, v20, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
 ; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v28
-; GFX10-NEXT:    v_add3_u32 v20, v26, v21, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v24, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_and_b32_e32 v27, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v30, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX10-NEXT:    v_max_f32_e32 v19, v21, v21
-; GFX10-NEXT:    v_and_b32_e32 v20, 0xffff0000, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v29, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
-; GFX10-NEXT:    v_bfe_u32 v27, v19, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v18, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v24, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v21, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v26
+; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v21, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v29
-; GFX10-NEXT:    v_add3_u32 v23, v27, v19, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v27, 0x400000, v19
-; GFX10-NEXT:    v_and_b32_e32 v29, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v21, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v23, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v28
-; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v31, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_and_b32_e32 v19, 0xffff0000, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_max_f32_e32 v22, v23, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v30, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX10-NEXT:    v_bfe_u32 v28, v22, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v20, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v30
-; GFX10-NEXT:    v_add3_u32 v25, v28, v22, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v22
-; GFX10-NEXT:    v_and_b32_e32 v30, 0xffff0000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v23, v27, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v19, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
+; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v11
+; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v25, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v29
-; GFX10-NEXT:    v_and_b32_e32 v29, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v32, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v0
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v24, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
-; GFX10-NEXT:    v_max_f32_e32 v24, v25, v25
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v31, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26
-; GFX10-NEXT:    v_bfe_u32 v29, v24, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v33, 0x400000, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v20, v26, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v25, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_and_b32_e32 v25, 0xffff0000, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v22
+; GFX10-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v19, v20, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_and_b32_e32 v25, 0xffff0000, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v24, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v20, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_cndmask_b32_e32 v25, v28, v27, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v31
-; GFX10-NEXT:    v_and_b32_e32 v30, 0xffff0000, v0
-; GFX10-NEXT:    v_add3_u32 v26, v29, v24, 0x7fff
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v25, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v32, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v26, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_max_f32_e32 v26, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v33, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v31, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
-; GFX10-NEXT:    v_bfe_u32 v22, v26, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v34, 0x400000, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
-; GFX10-NEXT:    v_add3_u32 v22, v22, v26, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v21, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v31, v32
-; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v29, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v7, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v34, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v33
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v33, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v20, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v24, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v27, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v27, v23, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v26, v23, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v24, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v28
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v27, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX10-NEXT:    v_max_f32_e32 v27, v7, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v21, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v28
-; GFX10-NEXT:    v_bfe_u32 v23, v27, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v32, v31
-; GFX10-NEXT:    v_add3_u32 v23, v23, v27, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v15, v33, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v25, vcc_lo
-; GFX10-NEXT:    v_or_b32_e32 v25, 0x400000, v27
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v14
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v6, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_bfe_u32 v22, v24, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v26, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v5
-; GFX10-NEXT:    v_add3_u32 v22, v22, v24, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v21, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v5, v13, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
+; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v27, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v7
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v31, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v14, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v26, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX10-NEXT:    v_max_f32_e32 v23, v24, v24
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v21, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v26, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v13, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v21, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v26, v25, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v8
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v25, v25, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_bfe_u32 v21, v23, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v23
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_add3_u32 v21, v21, v23, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_bfe_u32 v23, v24, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v28, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v12
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_add3_u32 v23, v23, v24, 0x7fff
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
-; GFX10-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v26, v15, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v25, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v12, v4, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v15, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v22, v25, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v14, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
+; GFX10-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v11
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX10-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
-; GFX10-NEXT:    v_or_b32_e32 v30, 0x400000, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v25, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
-; GFX10-NEXT:    v_bfe_u32 v26, v24, 16, 1
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
-; GFX10-NEXT:    v_add3_u32 v26, v26, v24, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v25, v27, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, v12, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v27, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v11, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v26, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v22, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v25, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_bfe_u32 v23, v21, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v11, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX10-NEXT:    v_add3_u32 v23, v23, v21, 0x7fff
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v24
+; GFX10-NEXT:    v_perm_b32 v5, v18, v5, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
-; GFX10-NEXT:    v_and_b32_e32 v12, 0xffff0000, v24
-; GFX10-NEXT:    v_or_b32_e32 v24, 0x400000, v21
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v9
+; GFX10-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v10
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v1
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v10, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v23, v24, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v25
-; GFX10-NEXT:    v_lshrrev_b32_e32 v27, 16, v21
-; GFX10-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v23
-; GFX10-NEXT:    v_max_f32_e32 v24, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v8, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX10-NEXT:    v_bfe_u32 v26, v24, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX10-NEXT:    v_bfe_u32 v28, v25, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v11, v23, v23
-; GFX10-NEXT:    v_add3_u32 v23, v26, v24, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_add3_u32 v24, v28, v25, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v29, v11, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v25
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_add3_u32 v28, v29, v11, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v29, 0x400000, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v24, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v28, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v15, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v9, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v8, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v9, 0xffff0000, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_perm_b32 v1, v6, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_perm_b32 v6, v17, v14, 0x5040100
-; GFX10-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_perm_b32 v5, v18, v13, 0x5040100
-; GFX10-NEXT:    v_perm_b32 v2, v7, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v7, v16, v15, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX10-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v4, v19, v4, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_minimumnum_v16bf16:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v18, v7 :: v_dual_mov_b32 v17, v6
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v20, v5
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v18, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v14
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v18
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v14
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v16
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v18.h, v15.h, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, s0
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v18
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v19, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v16.l, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v19.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v4.h, v12.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v23
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v23, v23
-; GFX11-TRUE16-NEXT:    v_add3_u32 v27, v27, v23, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v25, v26
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v26, v27, v28, s3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v22.l, v19.l, s2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
-; GFX11-TRUE16-NEXT:    v_add3_u32 v25, v25, v24, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v26.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v26
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v6.l, v5.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v22
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v24, v25, v27, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.h, v21.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v16.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v20.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v6.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v26.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.h, v16.l, s0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v23, v23
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-TRUE16-NEXT:    v_add3_u32 v16, v27, v23, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v23
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v25, v26
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v24.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v23, v16, v27, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v21.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v3.h, v11.h, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v23.h, v19.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v16.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v11.h, v16.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v23
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v28, v19, v19
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v2.h, v10.h, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v28, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v25
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v10.h, v19.l, s0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v28
-; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v28, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v7.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v22.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v24, v24, v25 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v23.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v28, v28
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v26, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v26, v21, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v22.l, v19.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v26, 16, 1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v9.h, v23.l, s0
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v26
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v23.l
-; GFX11-TRUE16-NEXT:    v_add3_u32 v25, v25, v26, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v21.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v24.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.h, v5.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v26, v28, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v30
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v25, v25, v27, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v29
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v16.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v26, v26
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v28
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v25.h, v16.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v24, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v21.l, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v28, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v19.l, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v24.h, v19.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v19, v26, v26
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v19, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v18
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, 0x400000, v19
-; GFX11-TRUE16-NEXT:    v_add3_u32 v22, v22, v19, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v19, v19
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v27
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v22, v22, v30, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v23.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v18.l, v18.l, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v29
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v22.h, v23.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v18.h, v25.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v18.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.h, v21.l, s2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.h, v24.h, v16.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v14
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v27, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v24, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v17.l, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_add3_u32 v17, v27, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v22.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.l, v18.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v14.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v17, v23, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.l, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v24, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v22.l, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v20.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v19.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v11
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v20
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v3.h, v11.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v26, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v19.l, v22.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v11.h, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v20.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v5.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v5.l, s0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v23, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v13.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v14.l, v16.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v22, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v13.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v20, v22, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v17.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v17, v21, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v20.h, v18.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v22.l, v21.l, v20.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v20.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v22.l, v20.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v13.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_add3_u32 v15, v22, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v15, v15, v21, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v12.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16.l
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v17, v17, v17 :: v_dual_lshlrev_b32 v22, 16, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v15.h, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v12.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v21, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v12.l, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v17, v21, v22, s0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v9
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v9.h, v23.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.l, v21.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v27, v25
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v7.l, v6.l, s2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v21, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v21.l, v5.l, v23.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v23.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v21.l, v23.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v15
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v20.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v23, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.l, v16.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v7.l, v19.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v16.l
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v13.l, s0
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v13, v21, v21 :: v_dual_lshlrev_b32 v22, 16, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v17.l, v17.l, v14.l, s1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT:    v_add3_u32 v23, v23, v13, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v13, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v9
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v20, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v23, v24, s3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v11.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v20.h, v4.l, s3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v19.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v24, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v17.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v16.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v21, v15
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v4.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v13, v13, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v10.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v7.l, v16.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.l, v17.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v17.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v16.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v16.l, v17.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v14.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v14.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v17
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v18, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v12.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v16.l, v4.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v13.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v10.l, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v12.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v15
-; GFX11-TRUE16-NEXT:    v_add3_u32 v15, v23, v13, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v13
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v22, v21
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v21, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v8.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v17, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v15.h, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v9
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v17, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v12.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v22, v17
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v9, v12, s0
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v12, v13, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v13, v3, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v3
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v11, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.h, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v13, v3, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v11, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v13, v21, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v22, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v8.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v21
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v12
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v11.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v9.h, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v18.l, v15.h, v1.h, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v20.h, v0.h, s3
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v21
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v17
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v2.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v8.l, v1.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v20.l, v10.l, v0.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v19.l, v13.l, v2.h, s3
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v14
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, v19 :: v_dual_mov_b32 v3, v18
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v16
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v19
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v16bf16:
@@ -9861,449 +7735,304 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v15
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v13
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v17, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v16
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v19
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v21 :: v_dual_and_b32 v19, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v21, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v17, 16, v18
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v21, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v19, v20 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v18, v18, v18 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v23, v18, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v18
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v13
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v19, v20 :: v_dual_and_b32 v18, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX11-FAKE16-NEXT:    v_add3_u32 v23, v23, v18, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v18
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v25, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v22, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v18, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v25, 16, v23
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v27, 16, v22
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v27
-; GFX11-FAKE16-NEXT:    v_add3_u32 v17, v26, v21, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v22, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v25, v29, v28 :: v_dual_and_b32 v18, 0xffff0000, v18
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v27, 16, v25
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v24 :: v_dual_lshlrev_b32 v25, 16, v21
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v28, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v24, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v18 :: v_dual_lshlrev_b32 v26, 16, v20
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v26
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v18, v21 :: v_dual_lshlrev_b32 v27, 16, v19
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v19, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v18, v20, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v19 :: v_dual_lshlrev_b32 v28, 16, v24
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v28
-; GFX11-FAKE16-NEXT:    v_add3_u32 v20, v26, v21, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v24, v25 :: v_dual_and_b32 v28, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v17
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v20, v26 :: v_dual_lshlrev_b32 v21, 16, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v20 :: v_dual_and_b32 v23, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v10
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v25 :: v_dual_and_b32 v22, 0xffff0000, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v24, v24, v23 :: v_dual_and_b32 v25, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v22
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v20, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v28, v27, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v20, v21, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v29
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v23 :: v_dual_lshlrev_b32 v29, 16, v20
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v9
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v30, v29 :: v_dual_and_b32 v27, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v18, v19 :: v_dual_lshlrev_b32 v28, 16, v26
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v19, v21, v21 :: v_dual_and_b32 v20, 0xffff0000, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v29, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v18, v23, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v27, v25, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v29, 16, v21
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v27, v19, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v29
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v23, v27, v19, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v21, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v23, v27, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v28
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v31, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v22, v23, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v30, v27, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v22, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v20, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v22 :: v_dual_lshlrev_b32 v28, 16, v27
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v26, v23 :: v_dual_lshlrev_b32 v23, 16, v25
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v23
-; GFX11-FAKE16-NEXT:    v_add3_u32 v25, v28, v22, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v22
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v30
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v23, v27 :: v_dual_and_b32 v30, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v28 :: v_dual_lshlrev_b32 v25, 16, v29
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v19
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v32, v31, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v0
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v28
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v24 :: v_dual_max_f32 v24, v25, v25
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v33, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v31, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v29, v24, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v20, v26 :: v_dual_lshlrev_b32 v31, 16, v25
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v26, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v31
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    v_add3_u32 v26, v29, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v25, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v32, v31 :: v_dual_lshlrev_b32 v29, 16, v29
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v33, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v26, v29, v29 :: v_dual_lshlrev_b32 v33, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v31, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v22, v26, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v34, 0x400000, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v28
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v25 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v29
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v22, v26, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v21, v27 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v31, v32
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v31, v29, v30 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v27, v23, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v23 :: v_dual_and_b32 v22, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v28
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v31
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v28 :: v_dual_lshlrev_b32 v32, 16, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v32, v31
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v28, v15, v7 :: v_dual_lshlrev_b32 v31, 16, v6
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v22 :: v_dual_and_b32 v24, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v25 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v25, 0x400000, v26
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v27, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_and_b32 v26, 0xffff0000, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v25, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v15, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v27, 16, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v25, v28, v28 :: v_dual_lshlrev_b32 v32, 16, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v27, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v27, v25, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v27
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v25, v24 :: v_dual_lshlrev_b32 v23, 16, v14
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v15, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v27, v27, v25, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v23, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v27, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v32, v31
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v14, v6 :: v_dual_lshlrev_b32 v28, 16, v13
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v25
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v13
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v28, v7 :: v_dual_lshlrev_b32 v30, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v29
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v13, v5 :: v_dual_lshlrev_b32 v29, 16, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v24, v27, v27
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v15, v24, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v15, v15, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v26, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v27, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v14
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_lshlrev_b32 v28, 16, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v14, v6 :: v_dual_lshlrev_b32 v24, 16, v27
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v28, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v27
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v12, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v14 :: v_dual_lshlrev_b32 v29, 16, v3
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v14, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v11
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v24
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v27, v14, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v14
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v13 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v5 :: v_dual_lshlrev_b32 v14, 16, v12
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v5 :: v_dual_lshlrev_b32 v24, 16, v12
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v27, v27, v14, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v26, v5 :: v_dual_lshlrev_b32 v28, 16, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v12, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v11, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v27, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v0
-; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v10
-; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v26, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v14, v4 :: v_dual_lshlrev_b32 v13, 16, v15
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v11, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v5, v18, v5, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    v_perm_b32 v5, v18, v5, 0x5040100
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v2 :: v_dual_lshlrev_b32 v25, 16, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v13, v13, v13 :: v_dual_lshlrev_b32 v26, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v13, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-FAKE16-NEXT:    v_add3_u32 v14, v24, v13, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v27, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v10, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v14, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v9, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v14
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v24, v26, v26
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v8, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v26, v24, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v27, v3 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v11, v14, v14
-; GFX11-FAKE16-NEXT:    v_add3_u32 v14, v26, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v29, v11, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v25, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v25
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v28, v25, 0x7fff
-; GFX11-FAKE16-NEXT:    v_add3_u32 v28, v29, v11, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v29, 0x400000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v26, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v11 :: v_dual_lshlrev_b32 v12, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v10 :: v_dual_lshlrev_b32 v15, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v1 :: v_dual_lshlrev_b32 v12, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v0 :: v_dual_lshlrev_b32 v15, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v11
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v12
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v11, v2 :: v_dual_lshlrev_b32 v15, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v8, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v15
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v12, v1 :: v_dual_lshlrev_b32 v8, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v4, v19, v4, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -10315,534 +8044,406 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v18, v7 :: v_dual_mov_b32 v17, v6
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v20, v5
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v18, v5
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v15
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v14
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v18
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v14
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v16
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v13
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v18.h, v15.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, s0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.h, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v18
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v19, v21
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.h, v13.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v16.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v6.l, v5.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v22
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v19.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v13.h, v20.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v4.h, v12.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v23, v23, v23
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v24, v23
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v24, v24, v24
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v23
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v23, v23
-; GFX12-TRUE16-NEXT:    v_add3_u32 v27, v27, v23, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v25, v26
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v7.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v26, v27, v28, s3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v22.l, v19.l, s2
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
-; GFX12-TRUE16-NEXT:    v_add3_u32 v25, v25, v24, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v26.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v26
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v19.l, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v19.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.l, v7.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v24, v25, v27, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v12.h, v21.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v16.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v23, v23, v23
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v7.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v24, v25
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v26.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v27, v23, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.h, v16.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v23, v23
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX12-TRUE16-NEXT:    v_add3_u32 v16, v27, v23, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v23
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v25, v26
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v22.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v20.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v19.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v24.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v23, v16, v27, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v19.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v21.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v3.h, v11.h, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v23.h, v19.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v16.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v11.h, v16.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v6.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v23
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v28, v19, v19
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v11
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v22.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v20
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v2.h, v10.h, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v3.h, v11.h, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v28, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v25
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v26, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v19.l, v22.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v10.h, v19.l, s0
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v28
-; GFX12-TRUE16-NEXT:    v_add3_u32 v24, v24, v28, 0x7fff
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v19.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v7.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v22.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v24, v24, v25 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v23.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v28, v28
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v26, v27
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v24
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v11.h, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v20.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v5.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v21.l
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v24.h, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v26, v21, v21
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v22.l, v19.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v5.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v19.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v25, v26, 16, 1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v21.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v9.h, v23.l, s0
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v27, 0x400000, v26
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v23.l
-; GFX12-TRUE16-NEXT:    v_add3_u32 v25, v25, v26, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v21.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v26, v26
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v22.l, v21.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v20.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v24.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v23.h, v5.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v26, v28, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v30
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v25, v25, v27, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v29
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v16.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v26, v26
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v28
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v22.l, v20.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v22.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v25.h, v16.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX12-TRUE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v23.l, v1.h, v9.h, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v21.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v21.l, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v28, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v19.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v9.h, v23.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v20.l, v21.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v27, v25
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v23.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v24.h, v19.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v19, v26, v26
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v7.l, v6.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v20.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v21, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v19, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v18
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v30, 0x400000, v19
-; GFX12-TRUE16-NEXT:    v_add3_u32 v22, v22, v19, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v19, v19
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v27
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v15
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v24
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v22, v22, v30, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v23.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v7.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v21.l, v5.l, v23.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v23.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v18.l, v18.l, v15.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v29
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v21.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v22.h, v23.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v21.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v21.l, v23.l, s0
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v19.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v15
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v16.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v20.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v23, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v18.h, v25.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v18.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.h, v21.l, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v15.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.h, v24.h, v16.l, s1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.l, v16.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v7.l, v19.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v19.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v16.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v14
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v15.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v14
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v27, v21, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v24, v23
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v17.l, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    v_add3_u32 v17, v27, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v22.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v15.l, v18.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v17.l, v17.l, v14.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v19.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v14.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v25
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v24, v23
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v17.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v17, v23, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v16.h, v7.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v17.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v14.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v6.l, v16.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v21, v21, v21
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v23, v22
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v21, 16, 1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v13
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v14.l, v16.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v7.l, v16.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-TRUE16-NEXT:    v_add3_u32 v20, v22, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v21
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v13.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v15.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v20, v22, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v18.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v14.l, v17.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v17.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v17.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v17, v21, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v16.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v20.h, v18.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v12
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v15.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v13.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v16.l, v17.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v14.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    v_add3_u32 v15, v22, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v17
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v22
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v23
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v15, v15, v21, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v12.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16.l
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v17, v17, v17 :: v_dual_lshlrev_b32 v22, 16, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v14.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v15.h, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v12.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v21, v23, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v12.l, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v17
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v17, v21, v22, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v13, v13
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v5.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v15
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v6.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v13.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v13.l, s0
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v13, v21, v21 :: v_dual_lshlrev_b32 v22, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v3.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v18, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v17.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT:    v_add3_u32 v23, v23, v13, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v13, v13
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v15.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v20, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v20, v23, v24, s3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v4.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v10.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v17, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v11.l, v3.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v12.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v16.l, v4.h, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v20.h, v4.l, s3
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v13.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v10.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v9.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v21, v15
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v1.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v4.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v13, v13, v13
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v10.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v8.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v18, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v12.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v15
-; GFX12-TRUE16-NEXT:    v_add3_u32 v15, v23, v13, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v13
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v22, v21
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v21, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v13, v13
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v8.l, v0.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v17, v21, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v15.h, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v9
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v17, v21, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v12.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v22, v17
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v0.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v9, v12, s0
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v12, v13, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v13, v3, 16, 1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v3
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v11, v12, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v17
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.h, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_add3_u32 v13, v13, v3, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v11, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v22, v21
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v13.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v13, v21, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v22, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v8.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v11.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v8.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v13
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v21
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v12
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v21
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v17
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v11.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v9.h, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v18.l, v15.h, v1.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v16.l, v20.h, v0.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v2.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v8.l, v1.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v20.l, v10.l, v0.h, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v19.l, v13.l, v2.h, s3
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v14
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v19 :: v_dual_mov_b32 v3, v18
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v16
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v19
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v16bf16:
@@ -10856,557 +8457,398 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v17, 16, v15
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v13
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v5
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v12
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v15
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v12
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v17, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v6
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v21 :: v_dual_and_b32 v19, 0xffff0000, v14
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v21, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v17, 16, v18
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v21, v22
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v22, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v19, v20 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v18, v18, v18 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v23, v18, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v22, 0x400000, v18
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v19, v20 :: v_dual_and_b32 v18, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v21, v21, v21
-; GFX12-FAKE16-NEXT:    v_add3_u32 v23, v23, v18, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v22, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v18
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v22, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v18, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v26, 0xffff0000, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v24 :: v_dual_lshlrev_b32 v25, 16, v21
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v25, 16, v23
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v17 :: v_dual_lshlrev_b32 v27, 16, v22
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v27
-; GFX12-FAKE16-NEXT:    v_add3_u32 v17, v26, v21, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v27, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v22, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v24, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v18 :: v_dual_lshlrev_b32 v26, 16, v20
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v25, v29, v28 :: v_dual_and_b32 v18, 0xffff0000, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v29, 16, v11
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v24, v16 :: v_dual_lshlrev_b32 v27, 16, v25
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v21, v21, v21 :: v_dual_and_b32 v26, 0xffff0000, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v26, v21, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v26
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v28, v25, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v18, v20, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v18, v18, v21 :: v_dual_lshlrev_b32 v27, 16, v19
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v19 :: v_dual_lshlrev_b32 v28, 16, v24
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v28
-; GFX12-FAKE16-NEXT:    v_add3_u32 v20, v26, v21, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v21
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v19, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v24, v25 :: v_dual_and_b32 v28, 0xffff0000, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v17
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v23, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v20, v26 :: v_dual_lshlrev_b32 v21, 16, v27
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v20, v22, v20 :: v_dual_and_b32 v23, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v24, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v30, v29 :: v_dual_and_b32 v27, 0xffff0000, v11
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v25 :: v_dual_and_b32 v22, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v18, v19 :: v_dual_lshlrev_b32 v28, 16, v26
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v19, v21, v21 :: v_dual_and_b32 v20, 0xffff0000, v20
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v25, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v29, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v18, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v29, 16, v21
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v27, v19, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v29
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v23, v27, v19, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v19
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v24, v24, v23 :: v_dual_and_b32 v25, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v22
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v20, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v21, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v22, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v23, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v28
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v28, v27, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v31, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v20, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v22, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v23 :: v_dual_lshlrev_b32 v29, 16, v20
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v9
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v22, v23, v23
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v30, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v22, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v20, v25, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v23
-; GFX12-FAKE16-NEXT:    v_add3_u32 v25, v28, v22, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v22
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v30
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v23, v27 :: v_dual_and_b32 v30, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v25, v28 :: v_dual_lshlrev_b32 v25, 16, v29
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v19
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v27, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v32, v31, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v0
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v9
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v28
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v22 :: v_dual_lshlrev_b32 v28, 16, v27
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v20, v24 :: v_dual_max_num_f32 v24, v25, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v33, 0x400000, v24
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v26, v23 :: v_dual_lshlrev_b32 v23, 16, v25
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v31, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v29, v24, 16, 1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v20, v26 :: v_dual_lshlrev_b32 v31, 16, v25
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v25 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v26, v21, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v31
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    v_add3_u32 v26, v29, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v25, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v32, v31 :: v_dual_lshlrev_b32 v29, 16, v29
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v33, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v26, v29, v29 :: v_dual_lshlrev_b32 v33, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v31, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v22, v26, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v34, 0x400000, v26
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v26, v22 :: v_dual_and_b32 v24, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v21, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v29
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v22, v26, 0x7fff
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v21, v27 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v31, v32
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v27, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v31, v29, v30 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v26, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_and_b32 v26, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v27, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v24, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v21, v21, v23 :: v_dual_and_b32 v22, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v28
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v31
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v27, v28 :: v_dual_lshlrev_b32 v32, 16, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v26, v26, v26
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v32, v31
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v28, v15, v7 :: v_dual_lshlrev_b32 v31, 16, v6
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v25 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v25, 0x400000, v26
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v14
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v15, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v25, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v23, v22 :: v_dual_lshlrev_b32 v27, 16, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v25, v28, v28 :: v_dual_lshlrev_b32 v32, 16, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v27, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v27, v25, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v27, v27, v25, 0x7fff
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v15
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v23, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v25, v24 :: v_dual_lshlrev_b32 v23, 16, v14
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v15, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v27, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v32, v31
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v14, v6 :: v_dual_lshlrev_b32 v28, 16, v13
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v25
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v25, 0xffff0000, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v29
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v14
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v13
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v15 :: v_dual_lshlrev_b32 v28, 16, v6
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v28, v7 :: v_dual_lshlrev_b32 v30, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v29
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v14, v6 :: v_dual_lshlrev_b32 v24, 16, v27
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v13, v5 :: v_dual_lshlrev_b32 v29, 16, v12
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v24, v27, v27
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v26, v23, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v15, v24, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v15, v15, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v26, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v27, 0x400000, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v27, v7 :: v_dual_lshlrev_b32 v24, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v4
+; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v26, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v15, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v12
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v24, v26, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v15
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v13 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v15
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v28, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v27
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v3
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v12, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v5 :: v_dual_lshlrev_b32 v14, 16, v12
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v14 :: v_dual_lshlrev_b32 v29, 16, v3
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v14, v26, v26 :: v_dual_lshlrev_b32 v27, 16, v11
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v26, 16, v24
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v27, v14, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v30, 0x400000, v14
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v15, v13, v5 :: v_dual_lshlrev_b32 v24, 16, v12
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v11
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX12-FAKE16-NEXT:    v_add3_u32 v27, v27, v14, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v26, v5 :: v_dual_lshlrev_b32 v28, 16, v11
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v11, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v27, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v0
-; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v10
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v17, v6, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v12, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v26, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v14, v4 :: v_dual_lshlrev_b32 v13, 16, v15
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v11, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v5, v18, v5, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v2 :: v_dual_lshlrev_b32 v25, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v13, v13, v13 :: v_dual_lshlrev_b32 v26, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v13, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-FAKE16-NEXT:    v_add3_u32 v14, v24, v13, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v13
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v27, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v9
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v10, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v11 :: v_dual_lshlrev_b32 v12, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v14, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v27, 16, v13
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v10 :: v_dual_lshlrev_b32 v15, 16, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v9, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v14
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v24, v26, v26
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v8, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v26, v24, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v27, v3 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v11, v14, v14
-; GFX12-FAKE16-NEXT:    v_add3_u32 v14, v26, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v29, v11, 16, 1
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v8 :: v_dual_lshlrev_b32 v11, 16, v9
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v14, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v14
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v25, v25, v25
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v25, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v26, 0x400000, v25
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v28, v25, 0x7fff
-; GFX12-FAKE16-NEXT:    v_add3_u32 v28, v29, v11, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v29, 0x400000, v11
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v1 :: v_dual_lshlrev_b32 v12, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v10
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v0 :: v_dual_lshlrev_b32 v15, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v9
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v11
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v10, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v27, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v9, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v11, v2 :: v_dual_lshlrev_b32 v15, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v15
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v8, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v12, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v12, v1 :: v_dual_lshlrev_b32 v8, 16, v11
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v22, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v15, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v23, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v30, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
-; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v20, v3, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v21, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v15, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v4, v19, v4, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x bfloat> @llvm.minimumnum.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y)
@@ -11680,99 +9122,81 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-LABEL: v_minimumnum_v32bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX8-NEXT:    buffer_load_dword v55, off, s[0:3], s32
 ; GFX8-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v34, 16, v30
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
-; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
+; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
+; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
+; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
 ; GFX8-NEXT:    v_cndmask_b32_e32 v31, v35, v34, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
+; GFX8-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
 ; GFX8-NEXT:    v_cndmask_b32_e32 v34, v34, v31, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v31
-; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v34
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v35, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v34, v31, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
-; GFX8-NEXT:    v_mul_f32_e32 v35, 1.0, v35
-; GFX8-NEXT:    v_bfe_u32 v36, v35, 16, 1
-; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, v36, v35
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, s5, v36
-; GFX8-NEXT:    v_or_b32_e32 v38, 0x400000, v35
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
+; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
+; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v37, v39
 ; GFX8-NEXT:    s_movk_i32 s4, 0x8000
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v36, v38, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v35
+; GFX8-NEXT:    v_cndmask_b32_e32 v37, v34, v31, vcc
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v36, v48
+; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v31
-; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v36, v31, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v34
-; GFX8-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX8-NEXT:    v_cndmask_b32_e32 v31, v31, v34, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v35
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v38
+; GFX8-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX8-NEXT:    v_cndmask_b32_e32 v34, v35, v38, vcc
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v36, v31, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v32, v32
-; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v34, 16, v55
-; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v55
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v34, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v34, v32, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
+; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
+; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
+; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
+; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
+; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
+; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
+; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
+; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
+; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
+; GFX8-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
+; GFX8-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v55
+; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v35, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v35, v32, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v33, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v34, v32, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX8-NEXT:    v_mul_f32_e32 v33, 1.0, v33
-; GFX8-NEXT:    v_bfe_u32 v35, v33, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v35, vcc, v35, v33
-; GFX8-NEXT:    v_add_u32_e32 v35, vcc, s5, v35
-; GFX8-NEXT:    v_or_b32_e32 v36, 0x400000, v33
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v33, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v35, v36, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v33
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v33, v37
+; GFX8-NEXT:    v_cndmask_b32_e32 v33, v35, v32, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v32
-; GFX8-NEXT:    v_and_b32_e32 v33, 0xffff0000, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v35, v32, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v34
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v32, v34, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v32, v35, v32, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
-; GFX8-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v34, v33, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v33
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v35, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v33, v34, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
-; GFX8-NEXT:    v_mul_f32_e32 v35, 1.0, v35
-; GFX8-NEXT:    v_bfe_u32 v36, v35, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, v36, v35
-; GFX8-NEXT:    v_add_u32_e32 v36, vcc, s5, v36
-; GFX8-NEXT:    v_or_b32_e32 v37, 0x400000, v35
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v36, v37, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v35
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v34
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v33
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v34, v33, vcc
-; GFX8-NEXT:    v_and_b32_e32 v34, 0xffff0000, v35
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v34
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
+; GFX8-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
+; GFX8-NEXT:    v_cndmask_b32_e32 v33, v36, v34, vcc
 ; GFX8-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
@@ -11784,27 +9208,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v36, v37
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v35, v34, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX8-NEXT:    v_mul_f32_e32 v36, 1.0, v36
-; GFX8-NEXT:    v_bfe_u32 v37, v36, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v37, vcc, v37, v36
-; GFX8-NEXT:    v_add_u32_e32 v37, vcc, s5, v37
-; GFX8-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v37, v38, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v37, 16, v36
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v34
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
 ; GFX8-NEXT:    v_cndmask_b32_e32 v34, v34, v35, vcc
-; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v36
+; GFX8-NEXT:    v_lshlrev_b32_e32 v35, 16, v36
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v35
 ; GFX8-NEXT:    v_and_b32_e32 v35, 0xffff0000, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v36, 16, v27
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v36, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc
@@ -11812,27 +9226,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v38, 16, v36
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v37, v38
 ; GFX8-NEXT:    v_cndmask_b32_e32 v37, v36, v35, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX8-NEXT:    v_mul_f32_e32 v37, 1.0, v37
-; GFX8-NEXT:    v_bfe_u32 v38, v37, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v38, vcc, v38, v37
-; GFX8-NEXT:    v_add_u32_e32 v38, vcc, s5, v38
-; GFX8-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v38, v39, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v37
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v36
 ; GFX8-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc
-; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v37
+; GFX8-NEXT:    v_lshlrev_b32_e32 v36, 16, v37
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v36
 ; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v37, 16, v26
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v37, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
 ; GFX8-NEXT:    v_cndmask_b32_e32 v37, v37, v36, vcc
@@ -11840,23 +9244,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v38, v39
 ; GFX8-NEXT:    v_cndmask_b32_e32 v38, v37, v36, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX8-NEXT:    v_mul_f32_e32 v38, 1.0, v38
-; GFX8-NEXT:    v_bfe_u32 v39, v38, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v39, vcc, v39, v38
-; GFX8-NEXT:    v_add_u32_e32 v39, vcc, s5, v39
-; GFX8-NEXT:    v_or_b32_e32 v48, 0x400000, v38
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX8-NEXT:    v_cndmask_b32_e32 v38, v39, v48, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v38
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v36
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v37
 ; GFX8-NEXT:    v_cndmask_b32_e32 v36, v36, v37, vcc
-; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v38
+; GFX8-NEXT:    v_lshlrev_b32_e32 v37, 16, v38
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
 ; GFX8-NEXT:    v_and_b32_e32 v37, 0xffff0000, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v38, 16, v25
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
@@ -11868,27 +9263,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v39, v48
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v38, v37, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX8-NEXT:    v_mul_f32_e32 v39, 1.0, v39
-; GFX8-NEXT:    v_bfe_u32 v48, v39, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v48, vcc, v48, v39
-; GFX8-NEXT:    v_add_u32_e32 v48, vcc, s5, v48
-; GFX8-NEXT:    v_or_b32_e32 v49, 0x400000, v39
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX8-NEXT:    v_cndmask_b32_e32 v39, v48, v49, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v48, 16, v39
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v37
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v38
 ; GFX8-NEXT:    v_cndmask_b32_e32 v37, v37, v38, vcc
-; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v39
+; GFX8-NEXT:    v_lshlrev_b32_e32 v38, 16, v39
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v38
 ; GFX8-NEXT:    v_and_b32_e32 v38, 0xffff0000, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v39, 16, v24
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v38, v48, v39, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v39, v38, vcc
@@ -11896,27 +9281,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v49, 16, v39
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v48, v49
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v39, v38, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX8-NEXT:    v_mul_f32_e32 v48, 1.0, v48
-; GFX8-NEXT:    v_bfe_u32 v49, v48, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v49, vcc, v49, v48
-; GFX8-NEXT:    v_add_u32_e32 v49, vcc, s5, v49
-; GFX8-NEXT:    v_or_b32_e32 v50, 0x400000, v48
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v49, v50, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v49, 16, v48
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v38
-; GFX8-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v39
 ; GFX8-NEXT:    v_cndmask_b32_e32 v38, v38, v39, vcc
-; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v48
+; GFX8-NEXT:    v_lshlrev_b32_e32 v39, 16, v48
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX8-NEXT:    v_and_b32_e32 v39, 0xffff0000, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v49, v48, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc
@@ -11924,27 +9299,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v49, v50
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v48, v39, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX8-NEXT:    v_mul_f32_e32 v49, 1.0, v49
-; GFX8-NEXT:    v_bfe_u32 v50, v49, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v50, vcc, v50, v49
-; GFX8-NEXT:    v_add_u32_e32 v50, vcc, s5, v50
-; GFX8-NEXT:    v_or_b32_e32 v51, 0x400000, v49
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX8-NEXT:    v_cndmask_b32_e32 v49, v50, v51, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v50, 16, v49
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v39
-; GFX8-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v48
 ; GFX8-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc
-; GFX8-NEXT:    v_and_b32_e32 v48, 0xffff0000, v49
+; GFX8-NEXT:    v_lshlrev_b32_e32 v48, 16, v49
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
 ; GFX8-NEXT:    v_and_b32_e32 v48, 0xffff0000, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v50, v49, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc
@@ -11952,27 +9317,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v51, 16, v49
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v50, v51
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v49, v48, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX8-NEXT:    v_mul_f32_e32 v50, 1.0, v50
-; GFX8-NEXT:    v_bfe_u32 v51, v50, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v51, vcc, v51, v50
-; GFX8-NEXT:    v_add_u32_e32 v51, vcc, s5, v51
-; GFX8-NEXT:    v_or_b32_e32 v52, 0x400000, v50
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v51, v52, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v51, 16, v50
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v48
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v49
 ; GFX8-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc
-; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v50
+; GFX8-NEXT:    v_lshlrev_b32_e32 v49, 16, v50
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v49
 ; GFX8-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v51, v50, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc
@@ -11980,27 +9335,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v51, v52
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v50, v49, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX8-NEXT:    v_mul_f32_e32 v51, 1.0, v51
-; GFX8-NEXT:    v_bfe_u32 v52, v51, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v52, vcc, v52, v51
-; GFX8-NEXT:    v_add_u32_e32 v52, vcc, s5, v52
-; GFX8-NEXT:    v_or_b32_e32 v53, 0x400000, v51
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX8-NEXT:    v_cndmask_b32_e32 v51, v52, v53, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v52, 16, v51
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v49
-; GFX8-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v50
 ; GFX8-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc
-; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v51
+; GFX8-NEXT:    v_lshlrev_b32_e32 v50, 16, v51
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
 ; GFX8-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v52, v51, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v51, v50, vcc
@@ -12008,27 +9353,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v53
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v51, v50, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX8-NEXT:    v_mul_f32_e32 v52, 1.0, v52
-; GFX8-NEXT:    v_bfe_u32 v53, v52, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v53, vcc, v53, v52
-; GFX8-NEXT:    v_add_u32_e32 v53, vcc, s5, v53
-; GFX8-NEXT:    v_or_b32_e32 v54, 0x400000, v52
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v53, v54, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v53, 16, v52
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v50
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v51
 ; GFX8-NEXT:    v_cndmask_b32_e32 v50, v50, v51, vcc
-; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v52
+; GFX8-NEXT:    v_lshlrev_b32_e32 v51, 16, v52
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
 ; GFX8-NEXT:    v_and_b32_e32 v51, 0xffff0000, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v53, 16, v3
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v53, v52, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc
@@ -12036,27 +9371,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v53, v54
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v52, v51, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX8-NEXT:    v_mul_f32_e32 v53, 1.0, v53
-; GFX8-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v54, vcc, v54, v53
-; GFX8-NEXT:    v_add_u32_e32 v54, vcc, s5, v54
-; GFX8-NEXT:    v_or_b32_e32 v40, 0x400000, v53
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v54, v40, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v53
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v51
-; GFX8-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v52
 ; GFX8-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc
-; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v53
+; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v54, v53, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v53, v52, vcc
@@ -12064,27 +9389,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v53
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v54, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v53, v52, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX8-NEXT:    v_mul_f32_e32 v54, 1.0, v54
-; GFX8-NEXT:    v_bfe_u32 v40, v54, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, v40, v54
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, s5, v40
-; GFX8-NEXT:    v_or_b32_e32 v41, 0x400000, v54
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v40, v41, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v54
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v52
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v53
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v52, v53, vcc
-; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v54
+; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v54
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v53
 ; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX8-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v40, v54, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
@@ -12092,27 +9407,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v54
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v41
 ; GFX8-NEXT:    v_cndmask_b32_e32 v40, v54, v53, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX8-NEXT:    v_mul_f32_e32 v40, 1.0, v40
-; GFX8-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, v41, v40
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, s5, v41
-; GFX8-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v53
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v54
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc
-; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v40
+; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v41, v40, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
 ; GFX8-NEXT:    v_cndmask_b32_e32 v40, v40, v54, vcc
@@ -12120,23 +9425,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v42
 ; GFX8-NEXT:    v_cndmask_b32_e32 v41, v40, v54, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
-; GFX8-NEXT:    v_mul_f32_e32 v41, 1.0, v41
-; GFX8-NEXT:    v_bfe_u32 v42, v41, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v42, vcc, v42, v41
-; GFX8-NEXT:    v_add_u32_e32 v42, vcc, s5, v42
-; GFX8-NEXT:    v_or_b32_e32 v43, 0x400000, v41
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX8-NEXT:    v_cndmask_b32_e32 v41, v42, v43, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v42, 16, v41
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v54
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v40, vcc
-; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v41
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v41
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
@@ -12146,23 +9442,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v40
 ; GFX8-NEXT:    v_cndmask_b32_e32 v40, v55, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX8-NEXT:    v_mul_f32_e32 v40, 1.0, v40
-; GFX8-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, v41, v40
-; GFX8-NEXT:    v_add_u32_e32 v41, vcc, s5, v41
-; GFX8-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
-; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v40
+; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v40
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
@@ -12172,23 +9459,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v55, v30, v14, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX8-NEXT:    v_mul_f32_e32 v55, 1.0, v55
-; GFX8-NEXT:    v_bfe_u32 v40, v55, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, v40, v55
-; GFX8-NEXT:    v_add_u32_e32 v40, vcc, s5, v40
-; GFX8-NEXT:    v_or_b32_e32 v41, 0x400000, v55
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v40, v41, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT:    v_and_b32_e32 v30, 0xffff0000, v55
+; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v55
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
@@ -12198,23 +9476,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v55, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX8-NEXT:    v_mul_f32_e32 v30, 1.0, v30
-; GFX8-NEXT:    v_bfe_u32 v55, v30, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v55, vcc, v55, v30
-; GFX8-NEXT:    v_add_u32_e32 v55, vcc, s5, v55
-; GFX8-NEXT:    v_or_b32_e32 v40, 0x400000, v30
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX8-NEXT:    v_cndmask_b32_e32 v30, v55, v40, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v30
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX8-NEXT:    v_and_b32_e32 v29, 0xffff0000, v30
+; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v30
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v29
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
@@ -12224,23 +9493,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v12
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v30, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX8-NEXT:    v_mul_f32_e32 v29, 1.0, v29
-; GFX8-NEXT:    v_bfe_u32 v30, v29, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v30, vcc, v30, v29
-; GFX8-NEXT:    v_add_u32_e32 v30, vcc, s5, v30
-; GFX8-NEXT:    v_or_b32_e32 v55, 0x400000, v29
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
-; GFX8-NEXT:    v_cndmask_b32_e32 v29, v30, v55, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v28
 ; GFX8-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX8-NEXT:    v_and_b32_e32 v28, 0xffff0000, v29
+; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v28
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
@@ -12250,23 +9510,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v29, v28
 ; GFX8-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX8-NEXT:    v_mul_f32_e32 v28, 1.0, v28
-; GFX8-NEXT:    v_bfe_u32 v29, v28, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v29, vcc, v29, v28
-; GFX8-NEXT:    v_add_u32_e32 v29, vcc, s5, v29
-; GFX8-NEXT:    v_or_b32_e32 v30, 0x400000, v28
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
-; GFX8-NEXT:    v_cndmask_b32_e32 v28, v29, v30, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v29, 16, v28
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX8-NEXT:    v_and_b32_e32 v27, 0xffff0000, v28
+; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v27
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
@@ -12276,23 +9527,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v28, 16, v10
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v28, v27
 ; GFX8-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX8-NEXT:    v_mul_f32_e32 v27, 1.0, v27
-; GFX8-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v28, vcc, v28, v27
-; GFX8-NEXT:    v_add_u32_e32 v28, vcc, s5, v28
-; GFX8-NEXT:    v_or_b32_e32 v29, 0x400000, v27
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
-; GFX8-NEXT:    v_cndmask_b32_e32 v27, v28, v29, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX8-NEXT:    v_and_b32_e32 v26, 0xffff0000, v27
+; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v26
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
@@ -12302,23 +9544,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v27, v26
 ; GFX8-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX8-NEXT:    v_mul_f32_e32 v26, 1.0, v26
-; GFX8-NEXT:    v_bfe_u32 v27, v26, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v27, vcc, v27, v26
-; GFX8-NEXT:    v_add_u32_e32 v27, vcc, s5, v27
-; GFX8-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v26, v27, v28, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX8-NEXT:    v_and_b32_e32 v25, 0xffff0000, v26
+; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v25
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
@@ -12328,23 +9561,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v26, v25
 ; GFX8-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX8-NEXT:    v_mul_f32_e32 v25, 1.0, v25
-; GFX8-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, v26, v25
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, s5, v26
-; GFX8-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX8-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
@@ -12354,23 +9578,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v24
 ; GFX8-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX8-NEXT:    v_mul_f32_e32 v24, 1.0, v24
-; GFX8-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, v25, v24
-; GFX8-NEXT:    v_add_u32_e32 v25, vcc, s5, v25
-; GFX8-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX8-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
@@ -12380,23 +9598,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX8-NEXT:    v_mul_f32_e32 v23, 1.0, v23
-; GFX8-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, v24, v23
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, s5, v24
-; GFX8-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX8-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
@@ -12406,23 +9615,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v23, v22
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX8-NEXT:    v_mul_f32_e32 v22, 1.0, v22
-; GFX8-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v22
-; GFX8-NEXT:    v_add_u32_e32 v23, vcc, s5, v23
-; GFX8-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX8-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
@@ -12432,23 +9632,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v22, 16, v4
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v22, v21
 ; GFX8-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX8-NEXT:    v_mul_f32_e32 v21, 1.0, v21
-; GFX8-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v21
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, s5, v22
-; GFX8-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX8-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
@@ -12456,29 +9647,16 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX8-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v21, v20
 ; GFX8-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX8-NEXT:    v_mul_f32_e32 v20, 1.0, v20
-; GFX8-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v20
-; GFX8-NEXT:    v_add_u32_e32 v21, vcc, s5, v21
-; GFX8-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX8-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
@@ -12488,23 +9666,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v20, v19
 ; GFX8-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX8-NEXT:    v_mul_f32_e32 v19, 1.0, v19
-; GFX8-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v19
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, s5, v20
-; GFX8-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX8-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
@@ -12514,23 +9683,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v19, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v19, v18
 ; GFX8-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX8-NEXT:    v_mul_f32_e32 v18, 1.0, v18
-; GFX8-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v18
-; GFX8-NEXT:    v_add_u32_e32 v19, vcc, s5, v19
-; GFX8-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX8-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
@@ -12540,22 +9700,13 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v18, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX8-NEXT:    v_mul_f32_e32 v17, 1.0, v17
-; GFX8-NEXT:    v_bfe_u32 v18, v17, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v18, v17
-; GFX8-NEXT:    v_add_u32_e32 v18, vcc, s5, v18
-; GFX8-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
-; GFX8-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v16
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX8-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
+; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v17
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v54
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v53
@@ -12594,10 +9745,9 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-LABEL: v_minimumnum_v32bf16:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX900-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX900-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX900-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX900-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX900-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX900-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX900-NEXT:    buffer_load_dword v55, off, s[0:3], s32
 ; GFX900-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v34, 16, v30
@@ -12609,81 +9759,67 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
 ; GFX900-NEXT:    v_cndmask_b32_e32 v31, v35, v34, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
+; GFX900-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
 ; GFX900-NEXT:    v_cndmask_b32_e32 v34, v34, v31, vcc
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
+; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
+; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v37, v39
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v34, v31, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX900-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX900-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX900-NEXT:    v_bfe_u32 v39, v37, 16, 1
-; GFX900-NEXT:    v_or_b32_e32 v48, 0x400000, v37
-; GFX900-NEXT:    v_add3_u32 v39, v39, v37, s5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
 ; GFX900-NEXT:    s_movk_i32 s4, 0x8000
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v39, v48, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v37
+; GFX900-NEXT:    v_cndmask_b32_e32 v37, v34, v31, vcc
+; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v36, v48
+; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v31
-; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v31, v39, v31, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v34
-; GFX900-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX900-NEXT:    v_cndmask_b32_e32 v31, v31, v34, vcc
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v38
+; GFX900-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX900-NEXT:    v_cndmask_b32_e32 v34, v35, v38, vcc
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v31, v39, v31, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v32, v32
-; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
+; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
+; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
+; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
+; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
+; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
+; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
+; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
+; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
+; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
+; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
+; GFX900-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
+; GFX900-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    v_lshrrev_b32_e32 v34, 16, v55
+; GFX900-NEXT:    v_lshrrev_b32_e32 v35, 16, v55
 ; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v55
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v35, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v34, v32, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v35, v32, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
-; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
+; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v33, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v34, v32, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX900-NEXT:    v_max_f32_e32 v33, v33, v33
-; GFX900-NEXT:    v_bfe_u32 v37, v33, 16, 1
-; GFX900-NEXT:    v_or_b32_e32 v39, 0x400000, v33
-; GFX900-NEXT:    v_add3_u32 v37, v37, v33, s5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v33, v33
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v37, v39, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v33
+; GFX900-NEXT:    v_cndmask_b32_e32 v33, v35, v32, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v32
-; GFX900-NEXT:    v_and_b32_e32 v33, 0xffff0000, v33
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v37, v32, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v32, v34, vcc
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v33
-; GFX900-NEXT:    v_and_b32_e32 v33, 0xffff0000, v29
-; GFX900-NEXT:    v_cndmask_b32_e32 v32, v37, v32, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v33, v33
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v38, v35, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v34, 16, v33
-; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v36, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v33, v35, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
-; GFX900-NEXT:    v_max_f32_e32 v34, v34, v34
-; GFX900-NEXT:    v_bfe_u32 v36, v34, 16, 1
-; GFX900-NEXT:    v_add3_u32 v36, v36, v34, s5
-; GFX900-NEXT:    v_or_b32_e32 v37, 0x400000, v34
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v36, v37, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v36, 16, v34
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
-; GFX900-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v33
-; GFX900-NEXT:    v_and_b32_e32 v34, 0xffff0000, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v35, v33, vcc
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v34
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
+; GFX900-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
+; GFX900-NEXT:    v_cndmask_b32_e32 v33, v36, v34, vcc
 ; GFX900-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
@@ -12695,26 +9831,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v36, v37
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v35, v34, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX900-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX900-NEXT:    v_bfe_u32 v37, v36, 16, 1
-; GFX900-NEXT:    v_add3_u32 v37, v37, v36, s5
-; GFX900-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX900-NEXT:    v_cndmask_b32_e32 v36, v37, v38, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v36
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v34
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
 ; GFX900-NEXT:    v_cndmask_b32_e32 v34, v34, v35, vcc
-; GFX900-NEXT:    v_and_b32_e32 v35, 0xffff0000, v36
+; GFX900-NEXT:    v_lshlrev_b32_e32 v35, 16, v36
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v35
 ; GFX900-NEXT:    v_and_b32_e32 v35, 0xffff0000, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v36, 16, v27
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v35, v35
-; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v35, v37, v36, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc
@@ -12722,26 +9849,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v38, 16, v36
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v37, v38
 ; GFX900-NEXT:    v_cndmask_b32_e32 v37, v36, v35, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX900-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX900-NEXT:    v_bfe_u32 v38, v37, 16, 1
-; GFX900-NEXT:    v_add3_u32 v38, v38, v37, s5
-; GFX900-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v38, v39, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v38, 16, v37
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v35
-; GFX900-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v36
 ; GFX900-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc
-; GFX900-NEXT:    v_and_b32_e32 v36, 0xffff0000, v37
+; GFX900-NEXT:    v_lshlrev_b32_e32 v36, 16, v37
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v36
 ; GFX900-NEXT:    v_and_b32_e32 v36, 0xffff0000, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v37, 16, v26
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v36, v36
-; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v37, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
 ; GFX900-NEXT:    v_cndmask_b32_e32 v37, v37, v36, vcc
@@ -12749,22 +9867,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v38, v39
 ; GFX900-NEXT:    v_cndmask_b32_e32 v38, v37, v36, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX900-NEXT:    v_max_f32_e32 v38, v38, v38
-; GFX900-NEXT:    v_bfe_u32 v39, v38, 16, 1
-; GFX900-NEXT:    v_add3_u32 v39, v39, v38, s5
-; GFX900-NEXT:    v_or_b32_e32 v48, 0x400000, v38
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX900-NEXT:    v_cndmask_b32_e32 v38, v39, v48, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v38
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v36
-; GFX900-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v37
 ; GFX900-NEXT:    v_cndmask_b32_e32 v36, v36, v37, vcc
-; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v38
+; GFX900-NEXT:    v_lshlrev_b32_e32 v37, 16, v38
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
 ; GFX900-NEXT:    v_and_b32_e32 v37, 0xffff0000, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v38, 16, v25
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
@@ -12776,26 +9886,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v39, v48
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v38, v37, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX900-NEXT:    v_max_f32_e32 v39, v39, v39
-; GFX900-NEXT:    v_bfe_u32 v48, v39, 16, 1
-; GFX900-NEXT:    v_add3_u32 v48, v48, v39, s5
-; GFX900-NEXT:    v_or_b32_e32 v49, 0x400000, v39
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX900-NEXT:    v_cndmask_b32_e32 v39, v48, v49, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v48, 16, v39
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v37
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v38
 ; GFX900-NEXT:    v_cndmask_b32_e32 v37, v37, v38, vcc
-; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v39
+; GFX900-NEXT:    v_lshlrev_b32_e32 v38, 16, v39
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v38
 ; GFX900-NEXT:    v_and_b32_e32 v38, 0xffff0000, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v39, 16, v24
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v38, v38
-; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v38, v48, v39, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v39, v38, vcc
@@ -12803,26 +9904,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v49, 16, v39
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v48, v49
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v39, v38, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX900-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX900-NEXT:    v_bfe_u32 v49, v48, 16, 1
-; GFX900-NEXT:    v_add3_u32 v49, v49, v48, s5
-; GFX900-NEXT:    v_or_b32_e32 v50, 0x400000, v48
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX900-NEXT:    v_cndmask_b32_e32 v48, v49, v50, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v49, 16, v48
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v38
-; GFX900-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v39
 ; GFX900-NEXT:    v_cndmask_b32_e32 v38, v38, v39, vcc
-; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v48
+; GFX900-NEXT:    v_lshlrev_b32_e32 v39, 16, v48
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
 ; GFX900-NEXT:    v_and_b32_e32 v39, 0xffff0000, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v48, 16, v23
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
-; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v49, v48, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc
@@ -12830,26 +9922,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v49, v50
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v48, v39, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX900-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX900-NEXT:    v_bfe_u32 v50, v49, 16, 1
-; GFX900-NEXT:    v_add3_u32 v50, v50, v49, s5
-; GFX900-NEXT:    v_or_b32_e32 v51, 0x400000, v49
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX900-NEXT:    v_cndmask_b32_e32 v49, v50, v51, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v50, 16, v49
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v39
-; GFX900-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v48
 ; GFX900-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc
-; GFX900-NEXT:    v_and_b32_e32 v48, 0xffff0000, v49
+; GFX900-NEXT:    v_lshlrev_b32_e32 v48, 16, v49
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
 ; GFX900-NEXT:    v_and_b32_e32 v48, 0xffff0000, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
-; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v50, v49, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc
@@ -12857,26 +9940,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v51, 16, v49
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v50, v51
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v49, v48, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX900-NEXT:    v_max_f32_e32 v50, v50, v50
-; GFX900-NEXT:    v_bfe_u32 v51, v50, 16, 1
-; GFX900-NEXT:    v_add3_u32 v51, v51, v50, s5
-; GFX900-NEXT:    v_or_b32_e32 v52, 0x400000, v50
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX900-NEXT:    v_cndmask_b32_e32 v50, v51, v52, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v51, 16, v50
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v48
-; GFX900-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v49
 ; GFX900-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc
-; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v50
+; GFX900-NEXT:    v_lshlrev_b32_e32 v49, 16, v50
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v49
 ; GFX900-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v51, 16, v5
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
-; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v51, v50, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc
@@ -12884,26 +9958,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v51, v52
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v50, v49, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX900-NEXT:    v_max_f32_e32 v51, v51, v51
-; GFX900-NEXT:    v_bfe_u32 v52, v51, 16, 1
-; GFX900-NEXT:    v_add3_u32 v52, v52, v51, s5
-; GFX900-NEXT:    v_or_b32_e32 v53, 0x400000, v51
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX900-NEXT:    v_cndmask_b32_e32 v51, v52, v53, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v52, 16, v51
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v49
-; GFX900-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v50
 ; GFX900-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc
-; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v51
+; GFX900-NEXT:    v_lshlrev_b32_e32 v50, 16, v51
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
 ; GFX900-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v51, 16, v20
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v52, 16, v4
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
-; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v52, v51, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v51, v50, vcc
@@ -12911,26 +9976,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v51, v50, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX900-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX900-NEXT:    v_bfe_u32 v53, v52, 16, 1
-; GFX900-NEXT:    v_add3_u32 v53, v53, v52, s5
-; GFX900-NEXT:    v_or_b32_e32 v54, 0x400000, v52
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v53, v54, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v53, 16, v52
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v50
-; GFX900-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v51
 ; GFX900-NEXT:    v_cndmask_b32_e32 v50, v50, v51, vcc
-; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v52
+; GFX900-NEXT:    v_lshlrev_b32_e32 v51, 16, v52
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
 ; GFX900-NEXT:    v_and_b32_e32 v51, 0xffff0000, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v53, 16, v3
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v53, v52, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc
@@ -12938,26 +9994,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v53, v54
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v52, v51, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX900-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX900-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX900-NEXT:    v_add3_u32 v54, v54, v53, s5
-; GFX900-NEXT:    v_or_b32_e32 v40, 0x400000, v53
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v54, v40, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v53
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v51
-; GFX900-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v52
 ; GFX900-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc
-; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v53
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX900-NEXT:    v_and_b32_e32 v52, 0xffff0000, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v2
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v54, v53, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v53, v52, vcc
@@ -12965,26 +10012,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v53
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v54, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v53, v52, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX900-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX900-NEXT:    v_bfe_u32 v40, v54, 16, 1
-; GFX900-NEXT:    v_add3_u32 v40, v40, v54, s5
-; GFX900-NEXT:    v_or_b32_e32 v41, 0x400000, v54
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v40, v41, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v54
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v52
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v53, vcc
-; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v54
+; GFX900-NEXT:    v_lshlrev_b32_e32 v53, 16, v54
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v53
 ; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v40, v52, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX900-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v40, v54, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
@@ -12992,26 +10030,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v54
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v41
 ; GFX900-NEXT:    v_cndmask_b32_e32 v40, v54, v53, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX900-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX900-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX900-NEXT:    v_add3_u32 v41, v41, v40, s5
-; GFX900-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v53
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v54
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc
-; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v40
+; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v41, v53, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v53, v40, v53, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v41, v40, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
 ; GFX900-NEXT:    v_cndmask_b32_e32 v40, v40, v54, vcc
@@ -13019,22 +10048,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v42
 ; GFX900-NEXT:    v_cndmask_b32_e32 v41, v40, v54, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
-; GFX900-NEXT:    v_max_f32_e32 v41, v41, v41
-; GFX900-NEXT:    v_bfe_u32 v42, v41, 16, 1
-; GFX900-NEXT:    v_add3_u32 v42, v42, v41, s5
-; GFX900-NEXT:    v_or_b32_e32 v43, 0x400000, v41
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX900-NEXT:    v_cndmask_b32_e32 v41, v42, v43, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v42, 16, v41
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v54
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v40, vcc
-; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v41
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v41
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v42, v54, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v41, v54, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
@@ -13044,22 +10065,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v40
 ; GFX900-NEXT:    v_cndmask_b32_e32 v40, v55, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX900-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX900-NEXT:    v_bfe_u32 v41, v40, 16, 1
-; GFX900-NEXT:    v_add3_u32 v41, v41, v40, s5
-; GFX900-NEXT:    v_or_b32_e32 v42, 0x400000, v40
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v41, v42, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v55, vcc
-; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v40
+; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v40
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v14
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v41, v15, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
@@ -13069,22 +10082,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v55, v30, v14, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX900-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX900-NEXT:    v_bfe_u32 v40, v55, 16, 1
-; GFX900-NEXT:    v_add3_u32 v40, v40, v55, s5
-; GFX900-NEXT:    v_or_b32_e32 v41, 0x400000, v55
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v40, v41, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v14
-; GFX900-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX900-NEXT:    v_and_b32_e32 v30, 0xffff0000, v55
+; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v55
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v14, v40, v14, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v14, v55, v14, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
@@ -13094,22 +10099,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v55, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX900-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX900-NEXT:    v_bfe_u32 v55, v30, 16, 1
-; GFX900-NEXT:    v_add3_u32 v55, v55, v30, s5
-; GFX900-NEXT:    v_or_b32_e32 v40, 0x400000, v30
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX900-NEXT:    v_cndmask_b32_e32 v30, v55, v40, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v30
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v13
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
-; GFX900-NEXT:    v_and_b32_e32 v29, 0xffff0000, v30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v30
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v29
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
@@ -13119,22 +10116,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v12
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v30, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX900-NEXT:    v_max_f32_e32 v29, v29, v29
-; GFX900-NEXT:    v_bfe_u32 v30, v29, 16, 1
-; GFX900-NEXT:    v_add3_u32 v30, v30, v29, s5
-; GFX900-NEXT:    v_or_b32_e32 v55, 0x400000, v29
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v29, v29
-; GFX900-NEXT:    v_cndmask_b32_e32 v29, v30, v55, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v12
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v28
 ; GFX900-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc
-; GFX900-NEXT:    v_and_b32_e32 v28, 0xffff0000, v29
+; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v28
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
@@ -13144,22 +10133,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v29, v28
 ; GFX900-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX900-NEXT:    v_max_f32_e32 v28, v28, v28
-; GFX900-NEXT:    v_bfe_u32 v29, v28, 16, 1
-; GFX900-NEXT:    v_add3_u32 v29, v29, v28, s5
-; GFX900-NEXT:    v_or_b32_e32 v30, 0x400000, v28
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v28, v28
-; GFX900-NEXT:    v_cndmask_b32_e32 v28, v29, v30, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v29, 16, v28
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v11
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc
-; GFX900-NEXT:    v_and_b32_e32 v27, 0xffff0000, v28
+; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v27
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
@@ -13169,22 +10150,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v28, 16, v10
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v28, v27
 ; GFX900-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX900-NEXT:    v_max_f32_e32 v27, v27, v27
-; GFX900-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX900-NEXT:    v_add3_u32 v28, v28, v27, s5
-; GFX900-NEXT:    v_or_b32_e32 v29, 0x400000, v27
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v27, v27
-; GFX900-NEXT:    v_cndmask_b32_e32 v27, v28, v29, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v10
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc
-; GFX900-NEXT:    v_and_b32_e32 v26, 0xffff0000, v27
+; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v26
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
@@ -13194,47 +10167,34 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v27, v26
 ; GFX900-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX900-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX900-NEXT:    v_bfe_u32 v27, v26, 16, 1
-; GFX900-NEXT:    v_add3_u32 v27, v27, v26, s5
-; GFX900-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v26, v26
-; GFX900-NEXT:    v_cndmask_b32_e32 v26, v27, v28, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v9
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc
-; GFX900-NEXT:    v_and_b32_e32 v25, 0xffff0000, v26
+; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v26
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v25
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc
+; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v26, v25
 ; GFX900-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX900-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX900-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX900-NEXT:    v_add3_u32 v26, v26, v25, s5
-; GFX900-NEXT:    v_or_b32_e32 v27, 0x400000, v25
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v25, v25
-; GFX900-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v8
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc
-; GFX900-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
+; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v25
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
@@ -13244,22 +10204,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX900-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX900-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX900-NEXT:    v_add3_u32 v25, v25, v24, s5
-; GFX900-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
-; GFX900-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
-; GFX900-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
+; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
@@ -13269,22 +10221,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v23
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX900-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX900-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX900-NEXT:    v_add3_u32 v24, v24, v23, s5
-; GFX900-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v23, v23
-; GFX900-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc
-; GFX900-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
+; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v22
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
@@ -13294,26 +10238,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v23, v22
 ; GFX900-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX900-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX900-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX900-NEXT:    v_add3_u32 v23, v23, v22, s5
-; GFX900-NEXT:    v_or_b32_e32 v24, 0x400000, v22
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v22, v22
-; GFX900-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc
-; GFX900-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX900-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc
-; GFX900-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
+; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v21
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
@@ -13323,22 +10255,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v22, 16, v4
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v22, v21
 ; GFX900-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX900-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX900-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX900-NEXT:    v_add3_u32 v22, v22, v21, s5
-; GFX900-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v21, v21
-; GFX900-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc
-; GFX900-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
+; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v20
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
@@ -13348,22 +10272,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v21, v20
 ; GFX900-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX900-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX900-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX900-NEXT:    v_add3_u32 v21, v21, v20, s5
-; GFX900-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v20, v20
-; GFX900-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX900-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
+; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v20
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v19
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
@@ -13373,22 +10289,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v20, v19
 ; GFX900-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX900-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX900-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX900-NEXT:    v_add3_u32 v20, v20, v19, s5
-; GFX900-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v19, v19
-; GFX900-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX900-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
+; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v19
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v18
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
@@ -13398,22 +10306,14 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v19, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v19, v18
 ; GFX900-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX900-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX900-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX900-NEXT:    v_add3_u32 v19, v19, v18, s5
-; GFX900-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v18, v18
-; GFX900-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX900-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
+; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v18
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
@@ -13423,21 +10323,13 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v18, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX900-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX900-NEXT:    v_bfe_u32 v18, v17, 16, 1
-; GFX900-NEXT:    v_add3_u32 v18, v18, v17, s5
-; GFX900-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v17, v17
-; GFX900-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v16
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX900-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
+; GFX900-NEXT:    v_lshlrev_b32_e32 v16, 16, v17
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v54, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v53, v1, s4
@@ -13464,7 +10356,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_write_b32 a1, v41 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX950-NEXT:    scratch_load_dword v51, off, s32
+; GFX950-NEXT:    scratch_load_dword v50, off, s32
 ; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v34, 16, v30
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
@@ -13486,76 +10378,65 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
 ; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
 ; GFX950-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v37, v39
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v48, 16, v38
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v37, v39
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX950-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
 ; GFX950-NEXT:    v_cndmask_b32_e32 v37, v34, v31, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v36, v48
-; GFX950-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v37, v37, s0
+; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
+; GFX950-NEXT:    v_and_b32_e32 v51, 0xffff0000, v23
 ; GFX950-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v31
-; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v37
-; GFX950-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX950-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v34
-; GFX950-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX950-NEXT:    v_and_b32_e32 v48, 0xffff0000, v25
-; GFX950-NEXT:    v_cndmask_b32_e32 v31, v31, v34, vcc
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
-; GFX950-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
-; GFX950-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
-; GFX950-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v32, v32
-; GFX950-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
+; GFX950-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
 ; GFX950-NEXT:    v_and_b32_e32 v52, 0xffff0000, v22
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v35
 ; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
 ; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
+; GFX950-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v34
 ; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
 ; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v31, v34, vcc
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v38
 ; GFX950-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
 ; GFX950-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
+; GFX950-NEXT:    v_cndmask_b32_e32 v34, v35, v38, vcc
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v39
+; GFX950-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
+; GFX950-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v37, v31, vcc
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v32, v32
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
-; GFX950-NEXT:    v_lshrrev_b32_e32 v34, 16, v51
-; GFX950-NEXT:    v_and_b32_e32 v37, 0xffff0000, v51
-; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v50
+; GFX950-NEXT:    v_and_b32_e32 v37, 0xffff0000, v50
+; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v35, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v37, v37
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v34, v34, v32, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
+; GFX950-NEXT:    v_cndmask_b32_e32 v35, v35, v32, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v35
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v33, v37
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v33, v34, v32, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX950-NEXT:    v_max_f32_e32 v33, v33, v33
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v33, v33, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v33, v35, v32, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v32
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v33
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v34
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v35
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v32, v32, v34, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v37
+; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
 ; GFX950-NEXT:    v_and_b32_e32 v37, 0xffff0000, v28
-; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v33, v36, s0
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v35
-; GFX950-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
-; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v34, v33, v35, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v38
-; GFX950-NEXT:    v_lshlrev_b32_e32 v35, 16, v33
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v48
+; GFX950-NEXT:    v_and_b32_e32 v48, 0xffff0000, v25
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v34, v34, v38, vcc
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v35
-; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v28
-; GFX950-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
-; GFX950-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v33, v36, v34, vcc
 ; GFX950-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
+; GFX950-NEXT:    v_lshrrev_b32_e32 v36, 16, v12
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v34, v34
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v34, v36, v35, vcc
@@ -13568,9 +10449,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v37, 16, v11
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v36, v35, v34, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX950-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v36, v36, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v34
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc
@@ -13595,9 +10473,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v38, 16, v10
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v37, v36, v35, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX950-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v37, v37, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v35
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc
@@ -13622,9 +10497,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v39, 16, v9
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v38, v37, v36, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX950-NEXT:    v_max_f32_e32 v38, v38, v38
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v38, v38, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v36
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc
@@ -13649,9 +10521,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v48, 16, v8
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v39, v38, v37, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX950-NEXT:    v_max_f32_e32 v39, v39, v39
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v39, v39, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v37
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
@@ -13676,9 +10545,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v49, 16, v7
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v48, v39, v38, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX950-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v48, v48, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v38
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc
@@ -13694,18 +10560,15 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v39, v39
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v39, v49, v48, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v39
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v49, v50
-; GFX950-NEXT:    v_lshrrev_b32_e32 v50, 16, v6
+; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v48
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v49, v51
+; GFX950-NEXT:    v_lshrrev_b32_e32 v51, 16, v6
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v48, v39, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX950-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v49, v49, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v39
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc
@@ -13720,84 +10583,75 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v49, 16, v22
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v48, v48
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v48, v50, v49, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v48, v51, v49, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
+; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v48
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v49
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v50, v52
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v51, v52
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v52, 16, v5
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v49, v48, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX950-NEXT:    v_max_f32_e32 v50, v50, v50
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v50, v50, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v49, v48, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v48
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v49
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v49, 16, v51
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v49
 ; GFX950-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc
-; GFX950-NEXT:    v_lshrrev_b32_e32 v50, 16, v21
+; GFX950-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc
+; GFX950-NEXT:    v_lshrrev_b32_e32 v51, 16, v21
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v49, v49
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v51, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v49
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v50
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v51, v49, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v53
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v53, 16, v4
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v50, v49, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX950-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v52, v52, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v52, v51, v49, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v49
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v50
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v51
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
-; GFX950-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
+; GFX950-NEXT:    v_cndmask_b32_e32 v49, v49, v51, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v52
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
+; GFX950-NEXT:    v_and_b32_e32 v51, 0xffff0000, v4
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v52, 16, v20
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v53, v52, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v53, v52, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v51
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v53, v54
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v54, 16, v3
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v53, v52, v50, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX950-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v53, v53, s0
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v50
+; GFX950-NEXT:    v_cndmask_b32_e32 v53, v52, v51, vcc
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v51
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v52
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v50, v52, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX950-NEXT:    v_and_b32_e32 v52, 0xffff0000, v3
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v50, v53, v50, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v53, 16, v19
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
 ; GFX950-NEXT:    s_nop 1
@@ -13811,9 +10665,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v54, v53, v52, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX950-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v54, v54, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v52
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
@@ -13838,9 +10689,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v55, v54, v53, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX950-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v55, v55, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v53
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v53, v55, v53, vcc
@@ -13865,9 +10713,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v40, v55, v54, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX950-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v40, v40, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v54
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v54, v40, v54, vcc
@@ -13892,9 +10737,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v41, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v41
-; GFX950-NEXT:    v_max_f32_e32 v41, v41, v41
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v41, v41, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v55
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v55, v41, v55, vcc
@@ -13907,74 +10749,65 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v55, v41, v55, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v51
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v50
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v51, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v50, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v51, v51, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v51
+; GFX950-NEXT:    v_cndmask_b32_e32 v50, v50, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v50
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v40
 ; GFX950-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v40, v51, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v40
-; GFX950-NEXT:    v_max_f32_e32 v40, v40, v40
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v40, v40, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v40, v50, v15, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v15
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
-; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v51
+; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v50
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v51, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v40
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v51
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v14
+; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v50, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v40
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v14
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v40, v15, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v30
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v30
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v51, v51
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v50, v50
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v30
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v51
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v30
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v50
 ; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v51, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX950-NEXT:    v_max_f32_e32 v51, v51, v51
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v51, v51, s0
+; GFX950-NEXT:    v_cndmask_b32_e32 v50, v30, v14, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v14
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v14, v51, v14, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v14, v50, v14, vcc
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v30
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v51
+; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v50
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v14, v51, v14, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v14, v50, v14, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX950-NEXT:    v_lshlrev_b32_e32 v51, 16, v13
+; GFX950-NEXT:    v_lshlrev_b32_e32 v50, 16, v13
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v51, v30
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v50, v30
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX950-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v30, v30, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v13
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v30, v13, vcc
@@ -13998,9 +10831,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v30, v29
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX950-NEXT:    v_max_f32_e32 v29, v29, v29
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v29, v29, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v12
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v12, v29, v12, vcc
@@ -14024,9 +10854,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v29, v28
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX950-NEXT:    v_max_f32_e32 v28, v28, v28
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v28, v28, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v11
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc
@@ -14050,9 +10877,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v28, v27
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX950-NEXT:    v_max_f32_e32 v27, v27, v27
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v27, v27, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v10
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v10, v27, v10, vcc
@@ -14076,9 +10900,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v27, v26
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX950-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v26, v26, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v9
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v9, v26, v9, vcc
@@ -14102,9 +10923,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v26, v25
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX950-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v25, v25, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v8, v25, v8, vcc
@@ -14128,9 +10946,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v25, v24
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX950-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v24, v24, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v24, v7, vcc
@@ -14154,9 +10969,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v23
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v23, v23, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc
@@ -14180,9 +10992,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v23, v22
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX950-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v22, v22, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v22, v5, vcc
@@ -14206,9 +11015,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v22, v21
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX950-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v21, v21, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v4
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v21, v4, vcc
@@ -14232,9 +11038,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v21, v20
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX950-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v20, v20, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v20, v3, vcc
@@ -14258,9 +11061,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v20, v19
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX950-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v19, v19, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v2
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc
@@ -14284,9 +11084,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v19, v18
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX950-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v18, v18, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
@@ -14310,9 +11107,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v18, v17
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX950-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v17, v17, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
@@ -14326,7 +11120,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX950-NEXT:    v_perm_b32 v3, v52, v3, s0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v17, v0, vcc
 ; GFX950-NEXT:    v_perm_b32 v0, v55, v0, s0
-; GFX950-NEXT:    v_perm_b32 v4, v50, v4, s0
+; GFX950-NEXT:    v_perm_b32 v4, v51, v4, s0
 ; GFX950-NEXT:    v_perm_b32 v5, v49, v5, s0
 ; GFX950-NEXT:    v_perm_b32 v6, v48, v6, s0
 ; GFX950-NEXT:    v_perm_b32 v7, v39, v7, s0
@@ -14343,2615 +11137,1871 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX10-LABEL: v_minimumnum_v32bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    buffer_load_dword v53, off, s[0:3], s32
-; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v29
+; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v13
+; GFX10-NEXT:    v_and_b32_e32 v33, 0xffff0000, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v28
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_lshrrev_b32_e32 v31, 16, v53
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v32, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_lshlrev_b32_e32 v33, 16, v32
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v31, v32, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v34, 16, v31
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v33, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v31, v32, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
-; GFX10-NEXT:    v_max_f32_e32 v33, v33, v33
-; GFX10-NEXT:    v_bfe_u32 v34, v33, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v35, 0x400000, v33
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX10-NEXT:    v_add3_u32 v34, v34, v33, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v34, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32
-; GFX10-NEXT:    v_and_b32_e32 v35, 0xffff0000, v30
-; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v34, v32, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v31
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v32, 0xffff0000, v33
-; GFX10-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v32
-; GFX10-NEXT:    v_and_b32_e32 v32, 0xffff0000, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v31, v34, v31, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v32, 16, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v33, v32, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX10-NEXT:    v_lshlrev_b32_e32 v34, 16, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v32, v33, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v32
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v34, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v32, v33, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
-; GFX10-NEXT:    v_max_f32_e32 v34, v34, v34
-; GFX10-NEXT:    v_bfe_u32 v35, v34, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v36, 0x400000, v34
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_add3_u32 v35, v35, v34, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v35, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v33
+; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v12
+; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v11
 ; GFX10-NEXT:    v_and_b32_e32 v36, 0xffff0000, v29
-; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v35, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v33, v32, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v33, 0xffff0000, v34
-; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v13
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v33
-; GFX10-NEXT:    v_and_b32_e32 v33, 0xffff0000, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v32, v35, v32, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v32, v32, v35, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX10-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v34, v33, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v11
+; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v28
+; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v32
+; GFX10-NEXT:    v_cndmask_b32_e32 v34, v34, v38, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v26
+; GFX10-NEXT:    v_lshrrev_b32_e32 v53, 16, v10
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v51, v51
+; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v33, v48, v39, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v36, 16, v33
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v35, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v33, v34, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
-; GFX10-NEXT:    v_max_f32_e32 v35, v35, v35
-; GFX10-NEXT:    v_bfe_u32 v36, v35, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v37, 0x400000, v35
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX10-NEXT:    v_add3_u32 v36, v36, v35, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v36, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v34
-; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v28
-; GFX10-NEXT:    v_lshrrev_b32_e32 v36, 16, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v36, v34, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v27
+; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v23
+; GFX10-NEXT:    v_lshrrev_b32_e32 v65, 16, v7
+; GFX10-NEXT:    v_lshrrev_b32_e32 v66, 16, v22
+; GFX10-NEXT:    v_cndmask_b32_e32 v37, v35, v32, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v33
+; GFX10-NEXT:    v_lshrrev_b32_e32 v67, 16, v6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v70, 16, v4
+; GFX10-NEXT:    v_and_b32_e32 v71, 0xffff0000, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v36, v38, v34, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
+; GFX10-NEXT:    v_lshrrev_b32_e32 v80, 16, v3
+; GFX10-NEXT:    v_lshrrev_b32_e32 v85, 16, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v36
+; GFX10-NEXT:    v_cndmask_b32_e32 v35, v39, v33, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v34
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, v31, v38
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v26
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v53, v52, s6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v35
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s4, v39, v48
+; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v9
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v31, v31
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v25
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v49, v50
+; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v25
+; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v48, v52, v38, s6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v39, v39
+; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v38
+; GFX10-NEXT:    v_lshrrev_b32_e32 v53, 16, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v48
+; GFX10-NEXT:    v_cndmask_b32_e64 v39, v50, v49, s6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v31, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v39
+; GFX10-NEXT:    v_cndmask_b32_e64 v50, v49, v39, s6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s6, v52, v52
+; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v24
+; GFX10-NEXT:    v_cndmask_b32_e64 v49, v54, v53, s6
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s6, v51, v55
+; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v52, v52
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v50
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v49
+; GFX10-NEXT:    v_cndmask_b32_e64 v52, v53, v49, s7
+; GFX10-NEXT:    v_and_b32_e32 v53, 0xffff0000, v23
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v55, v55
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s8, v31, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v55, v65, v64, s7
+; GFX10-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v53, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v68, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v53, v64, v55, s7
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v65, v65
+; GFX10-NEXT:    v_and_b32_e32 v64, 0xffff0000, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v69, 16, v53
+; GFX10-NEXT:    v_cndmask_b32_e64 v65, v67, v66, s7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v67, 16, v52
+; GFX10-NEXT:    v_cmp_u_f32_e64 s7, v64, v64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v65
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s9, v54, v67
+; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v5
+; GFX10-NEXT:    v_cndmask_b32_e64 v64, v66, v65, s7
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s7, v68, v69
+; GFX10-NEXT:    v_lshrrev_b32_e32 v66, 16, v21
+; GFX10-NEXT:    v_lshrrev_b32_e32 v67, 16, v5
+; GFX10-NEXT:    v_and_b32_e32 v68, 0xffff0000, v4
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v54, v54
+; GFX10-NEXT:    v_lshrrev_b32_e32 v69, 16, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v64
+; GFX10-NEXT:    v_cndmask_b32_e64 v54, v67, v66, s10
+; GFX10-NEXT:    v_and_b32_e32 v67, 0xffff0000, v21
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v68, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v68, v70, v69, s10
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v67, v67
+; GFX10-NEXT:    v_lshlrev_b32_e32 v70, 16, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v82, 16, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v66, v66, v54, s10
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v71, v71
+; GFX10-NEXT:    v_lshrrev_b32_e32 v71, 16, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v81, 16, v66
+; GFX10-NEXT:    v_cndmask_b32_e64 v67, v69, v68, s10
+; GFX10-NEXT:    v_and_b32_e32 v69, 0xffff0000, v3
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s11, v70, v81
+; GFX10-NEXT:    v_lshlrev_b32_e32 v83, 16, v67
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v69, v69
+; GFX10-NEXT:    v_and_b32_e32 v70, 0xffff0000, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v81, 16, v2
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s12, v82, v83
+; GFX10-NEXT:    v_cndmask_b32_e64 v69, v80, v71, s10
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s10, v31, v51
+; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v19
+; GFX10-NEXT:    v_lshrrev_b32_e32 v80, 16, v18
+; GFX10-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v69
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v51, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v51, v71, v69, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v70, v70
+; GFX10-NEXT:    v_and_b32_e32 v71, 0xffff0000, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v83, 16, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v70, v81, v80, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v82, v82
+; GFX10-NEXT:    v_lshrrev_b32_e32 v81, 16, v17
+; GFX10-NEXT:    v_lshrrev_b32_e32 v82, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v80, v80, v70, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s13, v71, v71
+; GFX10-NEXT:    v_and_b32_e32 v71, 0xffff0000, v17
+; GFX10-NEXT:    v_cndmask_b32_e64 v82, v82, v81, s13
+; GFX10-NEXT:    v_cmp_u_f32_e64 s14, v71, v71
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s13, v31, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v70
+; GFX10-NEXT:    v_lshlrev_b32_e32 v83, 16, v80
+; GFX10-NEXT:    v_cndmask_b32_e64 v71, v81, v82, s14
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s14, v31, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v82
+; GFX10-NEXT:    v_lshlrev_b32_e32 v81, 16, v71
+; GFX10-NEXT:    v_lshrrev_b32_e32 v83, 16, v0
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s15, v31, v81
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v81, 16, v16
+; GFX10-NEXT:    v_cmp_u_f32_e64 s16, v31, v31
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v83, v83, v81, s16
+; GFX10-NEXT:    v_cmp_u_f32_e64 s16, v31, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v31, 16, v83
+; GFX10-NEXT:    v_cndmask_b32_e64 v81, v81, v83, s16
+; GFX10-NEXT:    v_lshlrev_b32_e32 v84, 16, v81
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s16, v31, v84
+; GFX10-NEXT:    v_and_b32_e32 v31, 0xffff0000, v14
+; GFX10-NEXT:    v_lshrrev_b32_e32 v84, 16, v30
+; GFX10-NEXT:    v_cmp_u_f32_e64 s17, v31, v31
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v85, v84, s17
+; GFX10-NEXT:    v_and_b32_e32 v85, 0xffff0000, v30
+; GFX10-NEXT:    v_cmp_u_f32_e64 s17, v85, v85
+; GFX10-NEXT:    v_lshlrev_b32_e32 v85, 16, v31
+; GFX10-NEXT:    v_cndmask_b32_e64 v84, v84, v31, s17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v86, 16, v84
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s17, v85, v86
+; GFX10-NEXT:    v_lshrrev_b32_e32 v86, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e64 v85, v84, v31, s17
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s17, 0x8000, v31
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v85, v31, s17
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s17, 0x8000, v84
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v31, v84, s17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v84, 16, v85
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s17, 0, v84
+; GFX10-NEXT:    v_cndmask_b32_e64 v84, v37, v32, s5
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v32
+; GFX10-NEXT:    v_cndmask_b32_e64 v31, v85, v31, s17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v85, 16, v15
+; GFX10-NEXT:    v_cndmask_b32_e64 v32, v84, v32, s5
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v37
+; GFX10-NEXT:    v_cndmask_b32_e64 v32, v32, v37, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v37, v36, v34, s4
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v34
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v39
+; GFX10-NEXT:    v_cndmask_b32_e64 v34, v37, v34, s4
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v36
+; GFX10-NEXT:    v_cndmask_b32_e64 v34, v34, v36, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v36, v35, v33, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v33
-; GFX10-NEXT:    v_cndmask_b32_e32 v33, v34, v33, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v35
-; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v12
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v34
-; GFX10-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v38
 ; GFX10-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX10-NEXT:    v_lshrrev_b32_e32 v34, 16, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v35, v34, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX10-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v34, v35, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v34
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v36, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v34, v35, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v36, 16, v36
-; GFX10-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX10-NEXT:    v_bfe_u32 v37, v36, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX10-NEXT:    v_add3_u32 v37, v37, v36, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v37, v38, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v35
-; GFX10-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
-; GFX10-NEXT:    v_lshrrev_b32_e32 v37, 16, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v37, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v34
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v35, v34, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v35, 0xffff0000, v36
-; GFX10-NEXT:    v_lshrrev_b32_e32 v36, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v33, v33, v35, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v35, 16, v36
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v35
-; GFX10-NEXT:    v_and_b32_e32 v35, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v34, v37, v34, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX10-NEXT:    v_lshrrev_b32_e32 v35, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v35
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v37, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v35, v36, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v37
-; GFX10-NEXT:    v_max_f32_e32 v37, v37, v37
-; GFX10-NEXT:    v_bfe_u32 v38, v37, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX10-NEXT:    v_add3_u32 v38, v38, v37, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v38, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v36
-; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
-; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v38, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v35
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v36, v35, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v36, 0xffff0000, v37
-; GFX10-NEXT:    v_lshrrev_b32_e32 v37, 16, v10
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v36
-; GFX10-NEXT:    v_and_b32_e32 v36, 0xffff0000, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v35, v38, v35, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX10-NEXT:    v_lshrrev_b32_e32 v36, 16, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v37, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v36, v37, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v36
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v38, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v36, v37, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v38, 16, v38
-; GFX10-NEXT:    v_max_f32_e32 v38, v38, v38
-; GFX10-NEXT:    v_bfe_u32 v39, v38, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v48, 0x400000, v38
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX10-NEXT:    v_add3_u32 v39, v39, v38, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v39, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v37
-; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v25
-; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v36
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v37, v36, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v38
-; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v35, v48, v38, s6
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s6, 0x8000, v49
+; GFX10-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v35, v38, s4
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v48
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v38, v48, s4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v35
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s4, 0, v48
+; GFX10-NEXT:    v_cndmask_b32_e64 v48, v50, v39, s8
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v65
+; GFX10-NEXT:    v_cndmask_b32_e64 v39, v48, v39, s5
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v50
+; GFX10-NEXT:    v_cndmask_b32_e64 v39, v39, v50, s5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v48
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s5, 0, v50
+; GFX10-NEXT:    v_cndmask_b32_e64 v50, v52, v49, s9
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s9, 0x8000, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v49, v50, v49, s6
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s6, 0x8000, v52
+; GFX10-NEXT:    v_cndmask_b32_e64 v49, v49, v52, s6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s6, 0, v52
+; GFX10-NEXT:    v_cndmask_b32_e64 v52, v53, v55, s7
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s7, 0x8000, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v55, v52, v55, s7
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s7, 0x8000, v53
+; GFX10-NEXT:    v_cndmask_b32_e64 v53, v55, v53, s7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v52
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s7, 0, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v55, v64, v65, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v69
+; GFX10-NEXT:    v_cndmask_b32_e64 v36, v52, v53, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v65, v55, v65, s8
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v64
+; GFX10-NEXT:    v_cndmask_b32_e64 v64, v65, v64, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v65, v66, v54, s11
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v54
+; GFX10-NEXT:    v_cndmask_b32_e64 v54, v65, v54, s8
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v66
+; GFX10-NEXT:    v_cndmask_b32_e64 v54, v54, v66, s8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v65
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s8, 0, v66
+; GFX10-NEXT:    v_cndmask_b32_e64 v66, v67, v68, s12
+; GFX10-NEXT:    v_cndmask_b32_e64 v68, v66, v68, s9
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s9, 0x8000, v67
+; GFX10-NEXT:    v_cndmask_b32_e64 v67, v68, v67, s9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v68, 16, v66
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s9, 0, v68
+; GFX10-NEXT:    v_cndmask_b32_e64 v68, v51, v69, s13
+; GFX10-NEXT:    v_cndmask_b32_e64 v69, v68, v69, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v51
+; GFX10-NEXT:    v_cndmask_b32_e64 v51, v69, v51, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v69, v80, v70, s14
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v70
+; GFX10-NEXT:    v_cndmask_b32_e64 v70, v69, v70, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v80
+; GFX10-NEXT:    v_cndmask_b32_e64 v70, v70, v80, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v80, v71, v82, s15
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v82
+; GFX10-NEXT:    v_cndmask_b32_e64 v82, v80, v82, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v71
+; GFX10-NEXT:    v_cndmask_b32_e64 v71, v82, v71, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v82, v81, v83, s16
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v82
+; GFX10-NEXT:    v_cndmask_b32_e64 v83, v82, v83, s10
+; GFX10-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v81
+; GFX10-NEXT:    v_cndmask_b32_e64 v81, v83, v81, s10
+; GFX10-NEXT:    buffer_load_dword v83, off, s[0:3], s32
+; GFX10-NEXT:    v_cmp_u_f32_e64 s10, v85, v85
+; GFX10-NEXT:    v_lshlrev_b32_e32 v85, 16, v14
+; GFX10-NEXT:    v_cmp_u_f32_e64 s11, v85, v85
+; GFX10-NEXT:    v_cndmask_b32_e64 v85, v14, v30, s11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v30
+; GFX10-NEXT:    v_cmp_u_f32_e64 s11, v14, v14
+; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 16, v84
+; GFX10-NEXT:    v_cndmask_b32_e64 v87, v30, v85, s11
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s12, 0, v14
+; GFX10-NEXT:    v_cndmask_b32_e64 v30, v35, v38, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v35, v50, v49, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v38, v65, v54, s8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v80
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v84, v32, s12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v32, 16, v37
+; GFX10-NEXT:    v_and_b32_e32 v84, 0xffff0000, v15
+; GFX10-NEXT:    v_cmp_eq_f32_e64 s12, 0, v32
+; GFX10-NEXT:    v_cndmask_b32_e64 v32, v37, v34, s12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v37, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e64 v34, v48, v39, s5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v68
+; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v69
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v37
-; GFX10-NEXT:    v_and_b32_e32 v37, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v36, v39, v36, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX10-NEXT:    v_lshrrev_b32_e32 v37, 16, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v38, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v37, v38, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v39, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v37, v38, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v39, 16, v39
-; GFX10-NEXT:    v_max_f32_e32 v39, v39, v39
-; GFX10-NEXT:    v_bfe_u32 v48, v39, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v49, 0x400000, v39
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX10-NEXT:    v_add3_u32 v48, v48, v39, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v48, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v38
-; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v24
-; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v48, v38, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v37
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v38, v37, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v38, 0xffff0000, v39
-; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v8
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v38
-; GFX10-NEXT:    v_and_b32_e32 v38, 0xffff0000, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v37, v48, v37, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX10-NEXT:    v_lshrrev_b32_e32 v38, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v39, v38, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v38, v39, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v38
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v48, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v38, v39, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX10-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX10-NEXT:    v_bfe_u32 v49, v48, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v50, 0x400000, v48
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX10-NEXT:    v_add3_u32 v49, v49, v48, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v49, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v39
-; GFX10-NEXT:    v_and_b32_e32 v50, 0xffff0000, v23
-; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v49, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v38
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v39, v38, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v48
-; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v37, v55, v64, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v84, v84
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v83
+; GFX10-NEXT:    v_and_b32_e32 v53, 0xffff0000, v83
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v83
+; GFX10-NEXT:    v_cndmask_b32_e64 v64, v15, v83, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v66, v67, s9
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v86, v50, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v64
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v50, v54, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v55, v83, v64, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX10-NEXT:    v_and_b32_e32 v39, 0xffff0000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v38, v49, v38, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX10-NEXT:    v_lshrrev_b32_e32 v39, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v48, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v39
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v49, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v39, v48, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX10-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX10-NEXT:    v_bfe_u32 v50, v49, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v51, 0x400000, v49
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX10-NEXT:    v_add3_u32 v50, v50, v49, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v50, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48
-; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v22
-; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v50, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v39
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v48, v39, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v49
-; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e32 v39, v68, v51, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v53
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
-; GFX10-NEXT:    v_and_b32_e32 v48, 0xffff0000, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v39, v50, v39, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX10-NEXT:    v_lshrrev_b32_e32 v48, 16, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v49, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v48, v49, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v48
+; GFX10-NEXT:    v_cndmask_b32_e32 v48, v69, v70, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v50, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v48, v49, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX10-NEXT:    v_max_f32_e32 v50, v50, v50
-; GFX10-NEXT:    v_bfe_u32 v51, v50, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v52, 0x400000, v50
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX10-NEXT:    v_add3_u32 v51, v51, v50, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v51, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v49
-; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v51, 16, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v49, v48, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v50
-; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v5
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX10-NEXT:    v_and_b32_e32 v49, 0xffff0000, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v48, v51, v48, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX10-NEXT:    v_lshrrev_b32_e32 v49, 16, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v50, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v49, v50, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v49
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v51, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v49, v50, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
-; GFX10-NEXT:    v_max_f32_e32 v51, v51, v51
-; GFX10-NEXT:    v_bfe_u32 v52, v51, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v54, 0x400000, v51
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX10-NEXT:    v_add3_u32 v52, v52, v51, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v52, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v50
-; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
-; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v52, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v49
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v50, v49, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v50, 0xffff0000, v51
-; GFX10-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX10-NEXT:    v_and_b32_e32 v50, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v49, v52, v49, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX10-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v51, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v50, v51, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v50
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v52, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v50, v51, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX10-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX10-NEXT:    v_bfe_u32 v54, v52, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v55, 0x400000, v52
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX10-NEXT:    v_add3_u32 v54, v54, v52, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v54, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v51
-; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v54, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v50
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v51, v50, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v52
-; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v3
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
-; GFX10-NEXT:    v_and_b32_e32 v51, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v50, v54, v50, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX10-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v52, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v51
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v54, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v51, v52, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX10-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX10-NEXT:    v_bfe_u32 v55, v54, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX10-NEXT:    v_add3_u32 v55, v55, v54, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v55, v64, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
-; GFX10-NEXT:    v_and_b32_e32 v64, 0xffff0000, v18
-; GFX10-NEXT:    v_lshrrev_b32_e32 v55, 16, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v55, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v51
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v52, v51, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v54
-; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v2
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX10-NEXT:    v_and_b32_e32 v52, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v51, v55, v51, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX10-NEXT:    v_lshrrev_b32_e32 v52, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v54, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v52, v54, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v52
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v52, v54, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX10-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX10-NEXT:    v_bfe_u32 v64, v55, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v65, 0x400000, v55
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX10-NEXT:    v_add3_u32 v64, v64, v55, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v64, v65, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
-; GFX10-NEXT:    v_and_b32_e32 v65, 0xffff0000, v17
-; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v64, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v55
-; GFX10-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX10-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v52, v64, v52, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX10-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v54, v55, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v65
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v54, v55, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX10-NEXT:    v_max_f32_e32 v64, v64, v64
-; GFX10-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v65, v66, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v55
-; GFX10-NEXT:    v_and_b32_e32 v66, 0xffff0000, v16
-; GFX10-NEXT:    v_lshrrev_b32_e32 v65, 16, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v65, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v0
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX10-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v54, v65, v54, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX10-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v64, v55, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v55, v64, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e32 v51, v53, v54, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v66
 ; GFX10-NEXT:    v_cndmask_b32_e32 v65, v55, v64, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX10-NEXT:    v_max_f32_e32 v65, v65, v65
-; GFX10-NEXT:    v_bfe_u32 v66, v65, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v67, 0x400000, v65
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX10-NEXT:    v_add3_u32 v66, v66, v65, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v65, v66, v67, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
+; GFX10-NEXT:    v_cndmask_b32_e32 v50, v80, v71, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v49, v51, v54, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v66, 16, v65
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v66, v64, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v65, v64, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v51
+; GFX10-NEXT:    v_cndmask_b32_e32 v49, v49, v53, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v55
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v64, v55, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v64, 0xffff0000, v65
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v65
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v54, v55, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v55, v66, v55, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
+; GFX10-NEXT:    v_cndmask_b32_e32 v49, v51, v49, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX10-NEXT:    v_cndmask_b32_e32 v52, v82, v81, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v87
+; GFX10-NEXT:    v_cndmask_b32_e32 v51, v65, v54, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v85
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v54, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v28
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v13
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v87, v85, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v53, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v53, v15, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v53, v15, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX10-NEXT:    v_max_f32_e32 v64, v64, v64
-; GFX10-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX10-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v64, v65, v66, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v65, 16, v64
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v65, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v53, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v53, 0xffff0000, v64
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v65, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX10-NEXT:    v_perm_b32 v15, v31, v15, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v30, v14, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX10-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX10-NEXT:    v_bfe_u32 v64, v53, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v65, 0x400000, v53
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX10-NEXT:    v_add3_u32 v64, v64, v53, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v53, v64, v65, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v64, 16, v53
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v64, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v30, 0xffff0000, v53
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v64, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX10-NEXT:    v_perm_b32 v14, v32, v14, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v13
 ; GFX10-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v29, v13, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
-; GFX10-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX10-NEXT:    v_bfe_u32 v53, v30, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v64, 0x400000, v30
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX10-NEXT:    v_add3_u32 v53, v53, v30, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v30, v53, v64, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v85
+; GFX10-NEXT:    v_lshlrev_b32_e32 v66, 16, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v55, v53, v85, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v54, v28, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v87
+; GFX10-NEXT:    v_cndmask_b32_e32 v28, v55, v87, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v55, v29, v13, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
-; GFX10-NEXT:    v_lshrrev_b32_e32 v53, 16, v30
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v64, 16, v55
+; GFX10-NEXT:    v_cndmask_b32_e32 v28, v53, v28, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v66, v65
+; GFX10-NEXT:    v_lshlrev_b32_e32 v65, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v54, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v29
 ; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v29, 0xffff0000, v30
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v53, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
-; GFX10-NEXT:    v_perm_b32 v13, v33, v13, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_lshlrev_b32_e32 v30, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v28
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v28, v12, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX10-NEXT:    v_max_f32_e32 v29, v29, v29
-; GFX10-NEXT:    v_bfe_u32 v30, v29, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v53, 0x400000, v29
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX10-NEXT:    v_add3_u32 v30, v30, v29, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v29, v30, v53, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v30, 16, v29
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v28, 0xffff0000, v29
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
-; GFX10-NEXT:    v_perm_b32 v12, v34, v12, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v53, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v53
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v54, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v27, v27, v11, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX10-NEXT:    v_max_f32_e32 v28, v28, v28
-; GFX10-NEXT:    v_bfe_u32 v29, v28, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v30, 0x400000, v28
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX10-NEXT:    v_add3_u32 v29, v29, v28, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v28, v29, v30, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v27
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v26
+; GFX10-NEXT:    v_perm_b32 v13, v14, v13, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v53, v12, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v11
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v9
+; GFX10-NEXT:    v_perm_b32 v14, v31, v28, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v12, v32, v12, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v25
+; GFX10-NEXT:    v_cndmask_b32_e32 v29, v27, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v29, 16, v28
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX10-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v27, 0xffff0000, v28
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v29
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
+; GFX10-NEXT:    v_lshlrev_b32_e32 v54, 16, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v55, 16, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v53, v26, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
 ; GFX10-NEXT:    v_cndmask_b32_e32 v11, v29, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX10-NEXT:    v_perm_b32 v11, v35, v11, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_lshlrev_b32_e32 v28, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v26, v10, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX10-NEXT:    v_max_f32_e32 v27, v27, v27
-; GFX10-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v29, 0x400000, v27
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX10-NEXT:    v_add3_u32 v28, v28, v27, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v27, v28, v29, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX10-NEXT:    v_lshrrev_b32_e32 v28, 16, v27
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v53
+; GFX10-NEXT:    v_perm_b32 v11, v33, v11, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v53, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v25, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26
 ; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v26, 0xffff0000, v27
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
-; GFX10-NEXT:    v_perm_b32 v10, v36, v10, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v25
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v26
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v25, v9, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
-; GFX10-NEXT:    v_max_f32_e32 v26, v26, v26
-; GFX10-NEXT:    v_bfe_u32 v27, v26, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v28, 0x400000, v26
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX10-NEXT:    v_add3_u32 v27, v27, v26, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v26, v27, v28, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v27, 16, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v53, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
+; GFX10-NEXT:    v_perm_b32 v10, v30, v10, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v25, 0xffff0000, v26
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
-; GFX10-NEXT:    v_perm_b32 v9, v37, v9, 0x5040100
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v24
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v8
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX10-NEXT:    v_lshlrev_b32_e32 v53, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v24
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v25
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v24, v8, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX10-NEXT:    v_max_f32_e32 v25, v25, v25
-; GFX10-NEXT:    v_bfe_u32 v26, v25, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v27, 0x400000, v25
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v26, 16, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v27, 16, v8
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX10-NEXT:    v_perm_b32 v9, v34, v9, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v26
+; GFX10-NEXT:    v_cndmask_b32_e32 v26, v24, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v29
+; GFX10-NEXT:    v_lshlrev_b32_e32 v29, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, v23, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX10-NEXT:    v_add3_u32 v26, v26, v25, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v25, v26, v27, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v26, 16, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
-; GFX10-NEXT:    v_perm_b32 v8, v38, v8, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v23, v7, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX10-NEXT:    v_max_f32_e32 v24, v24, v24
-; GFX10-NEXT:    v_bfe_u32 v25, v24, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v26, 0x400000, v24
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX10-NEXT:    v_add3_u32 v25, v25, v24, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v24, v25, v26, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v25, 16, v24
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v26
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v23, 0xffff0000, v24
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX10-NEXT:    v_perm_b32 v7, v39, v7, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v23
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
 ; GFX10-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX10-NEXT:    v_max_f32_e32 v23, v23, v23
-; GFX10-NEXT:    v_bfe_u32 v24, v23, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v25, 0x400000, v23
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX10-NEXT:    v_add3_u32 v24, v24, v23, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v24, v25, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v24, 16, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v8, v35, v8, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
+; GFX10-NEXT:    v_perm_b32 v7, v36, v7, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v22, 0xffff0000, v23
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v24, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
-; GFX10-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
+; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX10-NEXT:    v_perm_b32 v6, v37, v6, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v21
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v21, v5, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX10-NEXT:    v_max_f32_e32 v22, v22, v22
-; GFX10-NEXT:    v_bfe_u32 v23, v22, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v24, 0x400000, v22
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v21, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX10-NEXT:    v_add3_u32 v23, v23, v22, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v24, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v20, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v21, 0xffff0000, v22
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_lshlrev_b32_e32 v22, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v22, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v20, v4, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX10-NEXT:    v_max_f32_e32 v21, v21, v21
-; GFX10-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v23, 0x400000, v21
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX10-NEXT:    v_add3_u32 v22, v22, v21, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v22, v23, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v21
+; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX10-NEXT:    v_cndmask_b32_e32 v24, v19, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
+; GFX10-NEXT:    v_perm_b32 v5, v38, v5, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v20, 0xffff0000, v21
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_perm_b32 v4, v50, v4, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_lshlrev_b32_e32 v21, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v21, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v19, v3, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
-; GFX10-NEXT:    v_max_f32_e32 v20, v20, v20
-; GFX10-NEXT:    v_bfe_u32 v21, v20, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX10-NEXT:    v_add3_u32 v21, v21, v20, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v21, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v19, 0xffff0000, v20
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v21, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
-; GFX10-NEXT:    v_perm_b32 v3, v51, v3, 0x5040100
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX10-NEXT:    v_perm_b32 v3, v39, v3, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v16
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v20, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX10-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v19
+; GFX10-NEXT:    v_lshlrev_b32_e32 v23, 16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
-; GFX10-NEXT:    v_max_f32_e32 v19, v19, v19
-; GFX10-NEXT:    v_bfe_u32 v20, v19, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v21, 0x400000, v19
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX10-NEXT:    v_add3_u32 v20, v20, v19, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, v20, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v17, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, v16, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v20, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_perm_b32 v2, v52, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_lshlrev_b32_e32 v19, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v19, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v17, v1, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX10-NEXT:    v_max_f32_e32 v18, v18, v18
-; GFX10-NEXT:    v_bfe_u32 v19, v18, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v20, 0x400000, v18
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX10-NEXT:    v_add3_u32 v19, v19, v18, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v19, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v18
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v17, 0xffff0000, v18
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
-; GFX10-NEXT:    v_perm_b32 v1, v54, v1, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_lshlrev_b32_e32 v18, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v16
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v18, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v16, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX10-NEXT:    v_max_f32_e32 v17, v17, v17
-; GFX10-NEXT:    v_bfe_u32 v18, v17, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v19, 0x400000, v17
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX10-NEXT:    v_add3_u32 v18, v18, v17, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v18, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_and_b32_e32 v16, 0xffff0000, v17
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX10-NEXT:    v_lshlrev_b32_e32 v16, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX10-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v18, v0, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v0, v55, v0, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v52, v0, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX10-NEXT:    v_perm_b32 v2, v48, v2, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v4, v15, v4, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v15, v49, v51, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_minimumnum_v32bf16:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v51, off, s32
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v14
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v30
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v33, v33
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v34, v34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v13
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v11
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v10
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff0000, v9
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v67, 0xffff0000, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v16
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v55, v55
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s29, v85, v85
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v29
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v28
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v27
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v81, v81
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v52, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v82, v82
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v69, v69
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v80, 0xffff0000, v19
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s24
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v70, v70
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v50, v15 :: v_dual_mov_b32 v49, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v82.l, v18.h, v52.l, s25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v52.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v80, v80
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v49
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v82.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v29
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v32, v32
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v38, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v134
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v33, v33
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s44, v112, v134
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v13.h, v29.h, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v10.h, v26.h, s8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v55, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v112.l, v82.l, v52.l, s44
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v66, v66
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v55.l, v29.h, v15.l, s3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v67, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v83, 0xffff0000, v1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v71, v71
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v33.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v39, v39
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v4.h, v20.h, s20
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v55.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v21
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v48, v48
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v66.l, v26.h, v33.l, s9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v80.l, v20.h, v39.l, s21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v39.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v67, v67
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v83, v83
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v80.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v3.h, v19.h, s22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v132
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v118
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v68, v68
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v5.h, v21.h, s18
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s42, v102, v132
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v81.l, v19.h, v48.l, s23
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s23, v85, v115
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v71.l, v21.h, v38.l, s19
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v102.l, v80.l, v39.l, s42
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v38.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v85.l, v55.l, v15.l, s23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v71.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v50
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v102, v102, v102 :: v_dual_lshlrev_b32 v101, 16, v101
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s26
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s26, v96, v118
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v131
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v85, v85, v85
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v30.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v96.l, v66.l, v33.l, s26
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v35, v35
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v37, v37
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v39, v39
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v49, v49
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v51, v51
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v53, v53
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v54, v54
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v65, v65
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v67, v67
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v69, v69
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v71, v71
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s40, v86, v86
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v32.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v84, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v54, v54
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s41, v101, v131
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v30
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v34, v34
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v84, v84
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v9.h, v25.h, s10
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v96, v96, v96
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v31, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v14.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v101.l, v71.l, v38.l, s41
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v54.l, v30.h, v14.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v35, v35
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v8.h, v24.h, s12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v67.l, v25.h, v34.l, s11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v54.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v101
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v27
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v64, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v114
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v65, v65
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v12.h, v28.h, s4
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v34.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v68.l, v24.h, v35.l, s13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v67.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s22, v84, v114
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v101, v101, v101
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v36, v36
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v7.h, v23.h, s14
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v37, v37
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v6.h, v22.h, s16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v64.l, v28.h, v31.l, s5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v11.h, v27.h, s6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v35.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v69.l, v23.h, v36.l, s15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v68.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v119
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v84.l, v54.l, v14.l, s22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v31.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v65.l, v27.h, v32.l, s7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v36.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v70.l, v22.h, v37.l, s17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v83.l, v17.h, v53.l, s27
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v69.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v128
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s27, v97, v119
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v37.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v36, v36
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v38, v38
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v48, v48
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v50, v50
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v52, v52
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v64, v64
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v66, v66
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v68, v68
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v70, v70
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v80, v80
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v81, v81
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v83, v83
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v9.h, v25.h, s11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v85.l, v16.h, v54.l, s40
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v65.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v118
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v14
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v82, v82
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s28, v84, v84
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v69.l, v24.h, v38.l, s14
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v54.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s40, v86, v118
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v85.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v30
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v87, v87
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s41, v96, v96
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v36.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v39.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v50.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v51.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v64.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v65.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v67.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v68.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v69.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v71.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v80.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v81.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v145.l, v82.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v116
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v129
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s28, v98, v128
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v97.l, v67.l, v34.l, s27
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v84, v84, v84 :: v_dual_lshlrev_b32 v87, 16, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v117
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v130
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s24, v86, v116
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s29, v99, v129
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v98.l, v68.l, v35.l, s28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v114, v84, 16, 1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v48.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v81.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s25, v87, v117
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s40, v100, v130
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v86.l, v64.l, v31.l, s24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v99.l, v69.l, v36.l, s29
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v97, v97, v97 :: v_dual_lshlrev_b32 v98, 16, v98
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v84
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v130, v96, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v114, v114, v84, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v84, v84
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v133
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v87.l, v65.l, v32.l, s25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v100.l, v70.l, v37.l, s40
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v98, v98, v98 :: v_dual_lshlrev_b32 v99, 16, v99
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v131, 0x400000, v96
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v132, v97, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v114, v114, v115, s22
-; GFX11-TRUE16-NEXT:    v_add3_u32 v130, v130, v96, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v96, v96
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s43, v103, v133
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s42, v97, v97
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v32.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v37.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v52.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v53.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v87
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v86, v86, v86 :: v_dual_max_f32 v99, v99, v99
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v133, 0x400000, v97
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v134, v98, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v132, v132, v97, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v96, v130, v131, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v97, v97
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v87, v87, v87 :: v_dual_max_f32 v100, v100, v100
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v116, v85, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v118, v86, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v144, 0x400000, v98
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v145, v99, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v134, v134, v98, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v97, v132, v133, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v98, v98
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v117, 0x400000, v85
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v86
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v128, v87, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v146, 0x400000, v99
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v147, v100, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v116, v116, v85, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v85, v85
-; GFX11-TRUE16-NEXT:    v_add3_u32 v118, v118, v86, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v86, v86
-; GFX11-TRUE16-NEXT:    v_add3_u32 v145, v145, v99, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v98, v134, v144, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v99, v99
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v15.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v129, 0x400000, v87
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v84, 0x400000, v100
-; GFX11-TRUE16-NEXT:    v_add3_u32 v128, v128, v87, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v87, v87
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s23
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s24
-; GFX11-TRUE16-NEXT:    v_add3_u32 v147, v147, v100, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v99, v145, v146, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v100, v100
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v32.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v116.h, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v118.h, v31.l, s5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0x8000, v34.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0x8000, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v128.h, v32.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v35.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v36.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v97.h, v34.l, s7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.h, v96.h, v33.l, s6
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v85, v101, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v98.h, v35.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v99.h, v36.l, s8
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v86, 0x400000, v101
-; GFX11-TRUE16-NEXT:    v_add3_u32 v85, v85, v101, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v84, v147, v84, s22
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v101, v101
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v14.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0x8000, v38.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0x8000, v54.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0x8000, v55.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v85, v85, v86, s22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v114.h, v14.l, s0
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v87, v102, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0x8000, v64.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v102
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v102, v102
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v54.l, s11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v87, v87, v102, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v39.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v64.l, s13
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0x8000, v65.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v128
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v86, v87, v115, s22
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v87, 0xffff0000, v114
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v53.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0x8000, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v83.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v103.l, v81.l, v48.l, s43
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v116
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v65.l, s14
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v102
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0x8000, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v101
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v113, 16, v113
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v83.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v147.l, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v119
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v128
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v129
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v130
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v131
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v132
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v133
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v134
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v135
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v144
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v144, 16, v145
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s63, v116, v86
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v86.l, v55.l, v32.l, s40
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0x8000, v55.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v114
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v146
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v147
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s42, v87, v118
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s43, v96, v119
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s45, v98, v129
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s56, v101, v132
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s59, v112, v135
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s60, v113, v144
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v86.l, v32.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v86.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v35.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v36.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v39.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0x8000, v50.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0x8000, v51.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s46, v99, v130
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s61, v114, v145
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s62, v115, v146
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v96.l, v65.l, v34.l, s43
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v98.l, v67.l, v36.l, s45
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v101.l, v70.l, v39.l, s56
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v112.l, v81.l, v50.l, s59
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v113.l, v82.l, v51.l, s60
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v55.l, s16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v118
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v33.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0x8000, v37.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v48.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0x8000, v52.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0x8000, v53.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s23, 0x8000, v70.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s26, 0x8000, v81.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v87.l, v64.l, v33.l, s42
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v99.l, v68.l, v37.l, s46
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v114.l, v83.l, v52.l, s61
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v115.l, v84.l, v53.l, s62
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v96.l, v34.l, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v98.l, v36.l, s5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v101.l, v39.l, s8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v101.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v112.l, v50.l, s11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v112.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v113.l, v51.l, s12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v113.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v55
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v38.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v49.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0x8000, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s44, v97, v128
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v116.l, v85.l, v54.l, s63
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v87.l, v33.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v99.l, v37.l, s6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.h, v114.l, v52.l, s13
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v114.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v115.l, v53.l, s14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v115.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v39
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v35.h, v81.l, s26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v35.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v101
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0x8000, v67.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0x8000, v68.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v131, 0xffff0000, v98
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v103, v103, v103
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0x8000, v70.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v132, 0xffff0000, v84
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s17
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v131
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v117, v103, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v103
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v132
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v96
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0x8000, v80.l
-; GFX11-TRUE16-NEXT:    v_add3_u32 v117, v117, v103, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v97
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0x8000, v69.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v115
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v85
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v133
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s57, v102, v133
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v97.l, v66.l, v35.l, s44
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v116.l, v54.l, s15
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v116.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s12, 0, v51
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0x8000, v48.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0x8000, v65.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0x8000, v66.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s47, v100, v131
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s58, v103, v134
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v102.l, v71.l, v48.l, s57
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v96.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v97.l, v35.l, s4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s13, 0, v52
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s14, 0, v53
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0x8000, v38.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v49.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0x8000, v64.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s24, 0x8000, v71.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v100.l, v69.l, v38.l, s47
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v103.l, v80.l, v49.l, s58
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v87.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v102.l, v48.l, s9
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v102.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v30.h, v65.l, s18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v128
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v66.l, s19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v129
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s25, 0x8000, v80.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s41, 0x8000, v85.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v38.l, s7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v103.l, v49.l, s10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v103.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v14.h, v64.l, s17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v119
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.h, v34.h, v71.l, s24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v48
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v65
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v66
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v86.l, v13.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v49
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.l, v37.h, v85.l, s41
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v64
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v71
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v96.l, v30.h, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v97.l, v32.l, s4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0x8000, v67.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0x8000, v68.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v98.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v99.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v87.l, v38.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v102.l, v38.h, s9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v32.h, v67.l, s20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v130
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s21
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v131
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s29, 0x8000, v84.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v70
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v67
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s27, 0x8000, v82.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v68
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v37.l, v84.l, s29
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v101.l, v34.l, s8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s28, 0x8000, v83.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s10, 0, v80
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v36.l, v82.l, s27
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.h, v115.l, v37.l, s14
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s22, 0x8000, v69.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v36.h, v83.l, s28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.h, v103.l, v35.l, s10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.h, v113.l, v36.l, s12
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v100.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v33.h, v69.l, s22
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s11, 0, v81
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.h, v114.l, v48.l, s13
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v132
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v69
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v33.h, s7
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v51
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v50.h, v51.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v118
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.h, v31.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v84.h, v37.l, s4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v85.h, v38.l, s9
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v100
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v32.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v129
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v130
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v99
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v35, v36
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v14.h, v55.l, s12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v86.h, v39.l, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v86
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v31.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v32.l, v31.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v87
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.h, v118.h, v15.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v39
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s21
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v36.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.h, v114.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v31.h, v66.l, s15
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.h, v128.h, v15.h, s2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v32.h, v67.l, s16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.h, v86.h, v35.l, s9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v96.h, v36.l, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v33.h, v69.l, s18
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v54, v38, v38
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v34.h, v71.l, s20
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.h, v116.h, v35.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.h, v98.h, v33.l, s5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.h, v84.h, v34.l, s7
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v55, v54, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.h, v97.h, v37.l, s4
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v130
-; GFX11-TRUE16-NEXT:    v_add3_u32 v55, v55, v54, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.h, v99.h, v37.h, s6
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v55, v64, s11
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v64, v112, v112
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v54.h, v31.l, s10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v54
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v32.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX11-TRUE16-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.h, v85.h, v38.l, s8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v54.h, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v135
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v103, v103
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v113, v54
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v55, v117, v119 :: v_dual_and_b32 v54, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v31
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v83.l, v53.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v55.h, v48.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v81.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v15.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v0.h, v16.h, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v50.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v15.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.h, v98.l, v32.h, s5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v51.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.h, v99.l, v33.l, s6
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v65, v66, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v52.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v67
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v16.h, v15.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v81.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v54.h, v52.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v64, v64, v64
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v50.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v52, v53
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v31.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v82.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v65
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v67, v64, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v66
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v82.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX11-TRUE16-NEXT:    v_add3_u32 v66, v67, v64, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v67, 0x400000, v64
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v52, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.h, v55.h, v14.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v64, v66, v67, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v53.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v51
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v68
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v64.h, v53.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.l, v50.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v52
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v51.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v32.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v50.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v38, v53
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.h, v112.l, v39.l, s11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v51.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v29
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v50
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v30.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v50.l, v51.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v83.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.h, v54.h, v32.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v15.h, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v117, v117
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v31.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v52, v52
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v51.l, v33.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v34.l, v83.l, s1
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v52, v52, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v64
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v66, v52, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v49.l, v30.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX11-TRUE16-NEXT:    v_add3_u32 v49, v66, v52, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v52
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v65, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v34.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.h, v64.h, v32.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.l, v33.l, s1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v30.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v49, v49, v53, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v34.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v29
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v32.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v51, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v52
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v28
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v52
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v31.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v52, v55, v55 :: v_dual_and_b32 v53, 0xffff0000, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v30.l, v34.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v29.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v31.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v54, v52, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v12
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX11-TRUE16-NEXT:    v_add3_u32 v54, v54, v52, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v52, v52
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v29.l, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v28
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v52, v54, v65, s0
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v33.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v55, 0x400000, v53
-; GFX11-TRUE16-NEXT:    v_add3_u32 v54, v54, v53, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v15.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v52.h, v33.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v64
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v49, v54, v55, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v34.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v52
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v12.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v14.l, s0
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v53, v53, v53
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v34.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v64
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v65, v53, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v49
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v64, v65, v53, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v53
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v66
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v28.l, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v53, v64, v65, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v52, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v29.l, v13.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v13.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v27
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v30.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.h, v30.l, v13.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v51, v50
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v14.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.h, v53.h, v13.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v29.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v55
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v27.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v12.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.h, v11.h, v29.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v53
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v49, v49, v49
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v15.l, v52.h, v12.h, s0
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v55, v49, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v29
-; GFX11-TRUE16-NEXT:    v_add3_u32 v27, v55, v49, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, 0x400000, v49
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v54, v52
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v53.h, v11.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v26.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v29, v27, v29, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v27.l, v13.l, v11.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v29.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v52, v49
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v28.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v27, v12, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v25.l, v25.l, v9.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v28.l, s1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v27
-; GFX11-TRUE16-NEXT:    v_add3_u32 v28, v28, v27, 0x7fff
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.h, v31.l, v12.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v11.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v26
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v28.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v31.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v27, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v29.h, v9.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v52, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v27, v28, v53, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v8
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v29, v12, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v27.h, v11.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v24
-; GFX11-TRUE16-NEXT:    v_add3_u32 v29, v29, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v51, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.h, v10.h, v28.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v25
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v26.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v51, v50
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v31.l, v10.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v25.l, v12.l, v10.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v9.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v25.l, v10.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v24
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v27, v26
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v12.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v29, v49, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v10.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v8.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v24.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v12.h, v10.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v28, v28
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v11.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v13.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v26.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v10, 16, 1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v52, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v26
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v29, v29
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v52, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v12.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v26
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v10.l, v9.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v12.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v23.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v49
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v24.h, v9.l, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v27.h, v8.h, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v25.l, s2
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v12.h, v9.h, s1
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v27, v10, v10 :: v_dual_and_b32 v26, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v27, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v12, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT:    v_add3_u32 v23, v25, v27, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v27
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v12.l, v22.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v24.h, v7.h, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v11.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v11.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v22.h, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v25, v8, v8 :: v_dual_lshlrev_b32 v24, 16, v21
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v11.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v23, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v25, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v12.l, v6.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v26
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v25, v24
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v22.l, v6.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v32.l, v10.l, v7.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v24, v25, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v10.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v22.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v21, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v6.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v11.h, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v10.l, v7.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v10.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v12, v11
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v9.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v21
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v9.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v22, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v8, v9, v21 :: v_dual_lshlrev_b32 v9, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v22
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.h, v8.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v9, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v37.l, v11.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v21, v20
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.h, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v6, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v7.l, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v19
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v12, v20, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_and_b32 v11, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v9.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v10.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v8.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v6
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v8.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v18.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v6, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v9.h, v4.h, s0
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v11, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v18.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v19, v20, s3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.h, v4.l, s3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v17
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v20
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v12, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v6.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v4.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v10, v4
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v0.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v16.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v18.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v4.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v35.l, v7.l, v4.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v18.l, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v17
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v12, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v11
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v16, v12
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v19, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v18, v17
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v17, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v16, v17, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v12.h, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v16, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v17
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s0
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v10, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v10, v3, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v3
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v16
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v5, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.h, v2.l, s0
-; GFX11-TRUE16-NEXT:    v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v5, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v10, v17, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v18, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v9.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v9.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v50.l, v12.h, v1.h, s2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v11.h, v0.h, s3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v13.l, v5.h, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v30.l, v3.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v51.l, v7.h, v2.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v16
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v49.l, v2.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v48.l, v8.l, v1.h, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v39.l, v6.l, v0.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v38.l, v9.l, v2.h, s3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, v29
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, v50 :: v_dual_mov_b32 v4, v48
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v0, v13 :: v_dual_mov_b32 v1, v30
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, v51
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, v39 :: v_dual_mov_b32 v6, v38
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v7, v37 :: v_dual_mov_b32 v8, v35
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v9, v33 :: v_dual_mov_b32 v10, v32
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v11, v31 :: v_dual_mov_b32 v12, v36
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v13, v34
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, v49 :: v_dual_mov_b32 v2, v48
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v3, v39 :: v_dual_mov_b32 v4, v38
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, v36 :: v_dual_mov_b32 v6, v35
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v7, v34 :: v_dual_mov_b32 v8, v33
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v9, v32 :: v_dual_mov_b32 v10, v31
+; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v11, v30 :: v_dual_mov_b32 v12, v37
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v32bf16:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v12
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v51, 16, v28
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v30
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-FAKE16-NEXT:    scratch_load_b32 v50, off, s32
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v15
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v29
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v30
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v83, 16, v24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v52, v52, v51, s1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v13
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v84, 16, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v34, v35, v33 :: v_dual_and_b32 v49, 0xffff0000, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v38, v38
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v87, 16, v23
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v52, v51, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v96, 16, v7
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v99, 16, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v39, v64, v55 :: v_dual_and_b32 v70, 0xffff0000, v9
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v48, v48, v39, s0
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v100, 16, v6
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v115, 16, v20
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v68, v67, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v30
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v4
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v80, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v82, v82
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v18
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v80, v71, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v82, v82
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v84, v83, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v84, v83, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v96, v87, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v26
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v100, v99, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v96, v87, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v25
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v146, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v100, v99, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v68, v112, v103 :: v_dual_and_b32 v81, 0xffff0000, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v114, v114
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v24
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v116, v115, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v118, v118
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v147, 16, v16
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v34, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v54, v54
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v112, v103, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v114, v114
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v128, v119, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v14
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v116, v115, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v118, v118
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v113, 0xffff0000, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v64, v64, v55, s2
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
+; GFX11-FAKE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v128, v119, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v130, v130
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v82, v132, v131 :: v_dual_and_b32 v113, 0xffff0000, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v20
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v132, v131, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v134, v134
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v84, v144, v135 :: v_dual_and_b32 v117, 0xffff0000, v20
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v102, v144, v135 :: v_dual_and_b32 v133, 0xffff0000, v18
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v146, v146
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v96
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v34, v147, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v27
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v33
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v14, v30 :: v_dual_and_b32 v97, 0xffff0000, v23
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v37, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v97, 0xffff0000, v23
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v51
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v39, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v55, v64 :: v_dual_lshlrev_b32 v130, 16, v51
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v49, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v34
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s3, v66, v66
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v68, v68, v67, s3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v68, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v52 :: v_dual_lshlrev_b32 v118, 16, v67
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v68
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v80
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v67
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v54, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v80, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v97, v97
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v64 :: v_dual_lshlrev_b32 v128, 16, v83
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v84
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v82 :: v_dual_lshlrev_b32 v134, 16, v83
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v101, v101
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v97, v99, v66, vcc_lo
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v16
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v99, v84, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v113, v113
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v103, v68, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v103, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v99
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v117, v117
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v39
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v101, v115, v70, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v36
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v115, v96, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v129, v129
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v34
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v38
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v49
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v87
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v103, v119, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v82
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v115, v119, v98 :: v_dual_lshlrev_b32 v146, 16, v113
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v133, v133
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v71
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v103
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v131, v82, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v86, v114
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v52
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v99
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v113
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v33, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v36, v115
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v48, v116
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v86, v86, v86 :: v_dual_lshlrev_b32 v85, 16, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v51, v37, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v114, v86, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v115, 0x400000, v86
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v55
-; GFX11-FAKE16-NEXT:    v_add3_u32 v114, v114, v86, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v48, v48, v48
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v117
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v48, v48
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v118
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v66
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v118, v48, 16, 1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v49, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v119
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v70
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v53, v53, v53 :: v_dual_lshlrev_b32 v102, 16, v80
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v119, 0x400000, v48
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v71, v52, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v81, v128
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v68
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v128, v53, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v118, v118, v48, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v69
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v85, v129
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v82
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v53
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v69, v69, v69
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v81
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v64, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v128, v128, v53, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v81, v81, v81
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s2
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v144, 0x400000, v81
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v97
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v85, v85, v85
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v96, v130
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v146, v85, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v147, 0x400000, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v97, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v98, v131
-; GFX11-FAKE16-NEXT:    v_add3_u32 v146, v146, v85, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v101
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v99, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v100, v132
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v132, v69, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v96, v96, v96
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v101, v70, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v102, v133
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v133, 0x400000, v69
-; GFX11-FAKE16-NEXT:    v_add3_u32 v132, v132, v69, 0x7fff
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v98, v98, v98
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v102, v103, v80, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v112, v134
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v134, v81, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v48, v98, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v98
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v82, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v86, v96, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v134, v134, v81, 0x7fff
-; GFX11-FAKE16-NEXT:    v_add3_u32 v48, v48, v98, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v114, v115, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v86, v86, v96, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v65, v65, v65
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v130, v65, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v131, 0x400000, v65
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_add3_u32 v130, v130, v65, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v130, v131, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v132, v133 :: v_dual_lshlrev_b32 v36, 16, v36
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v36, v36, v36
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v134, v144, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v116, v36, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v36
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v36, v36
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v130, 16, v81
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-FAKE16-NEXT:    v_add3_u32 v116, v116, v36, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v36, 0x400000, v96
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v146, v147, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v96, v96
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v100, v100, v100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v86, v36, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v115, v100, 16, 1
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v100
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v86, 16, v114
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v98, 16, v116
-; GFX11-FAKE16-NEXT:    v_add3_u32 v115, v115, v100, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v48, v53, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v36
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v116
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v128
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v36
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v115, v117, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v34
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v118
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v134, 16, v48
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v146, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v86, v34, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v35
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v96, 0xffff0000, v114
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v114, 16, v118
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v118, 16, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v98, v35, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v37
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v128
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v69
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v144, 0xffff0000, v48
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v114, v37, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v39
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v116, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v49
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v118, v49, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v37, 16, v48
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v117, v131, v100, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v86
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v119, v135, v102, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v38, v147, v34 :: v_dual_lshlrev_b32 v49, 16, v52
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e64 s0, v49, v130
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v66, v30, v54 :: v_dual_lshlrev_b32 v53, 16, v64
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v35
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v30
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v70
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v117
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v130, v35, v36 :: v_dual_lshlrev_b32 v129, 16, v39
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v37, v129
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v129, v51, v52, s0
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v37, v39, v48 :: v_dual_lshlrev_b32 v118, 16, v102
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v55
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v131
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v55, v64 :: v_dual_lshlrev_b32 v50, 16, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v71
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v132
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v68, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v133
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v87
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v65
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v81, v134
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v85, v135
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v103
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v97, v144
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v97, v99, v84 :: v_dual_lshlrev_b32 v114, 16, v98
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v101, v145
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v147, 16, v115
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v101, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v97
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v112, v146
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v96, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v114, v147
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v119
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v38
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v115, v98, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v116, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v116, v117, v100, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v118, v30
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v118, v119, v102, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v128, v49
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v128, v38, v34, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v36
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v36, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v37, v48, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v128, v52, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
-; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v130, v54, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v129, v52, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v131, v64, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v147, v32, v52 :: v_dual_and_b32 v64, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v66
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v102, v102, v102
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v132, v66, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v129
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v53, v64, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v68
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v65
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v119, v102, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v102
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v134, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v69
-; GFX11-FAKE16-NEXT:    v_add3_u32 v119, v119, v102, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v147, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v65, v68, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v70
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v81
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v146, v70, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v33
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v34, v33, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v38
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v35, v35, v38 :: v_dual_lshlrev_b32 v34, 16, v147
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v69, v70, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v80
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v81, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v82
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v85, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v84
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v97, v84, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v86
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v101, v86, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v96
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v112, v96, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v98
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v98, v114, v98 :: v_dual_lshlrev_b32 v131, 16, v53
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v100, v116, v100 :: v_dual_lshlrev_b32 v133, 16, v69
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v35
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v35 :: v_dual_lshlrev_b32 v135, 16, v85
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v102
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v102, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v39
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v39 :: v_dual_lshlrev_b32 v145, 16, v101
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v34
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v128, v34, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v51
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v51 :: v_dual_and_b32 v85, 0xffff0000, v85
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v34, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v52, v147, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v49, v51, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v37, v55, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v128
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v64, v55, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v67
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v39, v67, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v67, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v71
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v48, v71, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v70, v71 :: v_dual_lshlrev_b32 v147, 16, v114
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v83
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v49, v83, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v80, v83, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v87
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v49, v54, v87 :: v_dual_lshlrev_b32 v34, 16, v34
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v97
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v31, v97, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v82, v87, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v99
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v34, v34, v34 :: v_dual_cndmask_b32 v55, v32, v99
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v96
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v54, v34, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v34
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v86, v33, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v100
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v54, v54, v34, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v98, v35, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v115
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v114, v36, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v117
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v116, v37, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v65
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v38, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v69
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v69, 16, v16
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v128, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v81
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v130, v48, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e64 vcc_lo, 0x8000, v147
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v147, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v85
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v131, v49, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v52, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v84, v99, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v103
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v86, v103 :: v_dual_lshlrev_b32 v30, 16, v130
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v113
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v96, v113, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v115
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v98, v115, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v117
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v81
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v100, v117, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v119
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v35, v119, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v38
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v34, v38, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v37, v36, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v31
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v129, v39, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v131
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v53, v49, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v132
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v31
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v65, v55, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v133
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v54
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v48, v132, v51 :: v_dual_lshlrev_b32 v51, 16, v112
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v69, v64, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v134
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v81, v67, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v135
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v85, v68, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v144
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v134, v55, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v84
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v39, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v135, v84, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v119, v129, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v101
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v97, v70, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v145
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v101, v71, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v15, v31, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v31
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v148, 16, v116
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v55
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v112
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v33
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v146
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v112, v80, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v118
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v31, v55 :: v_dual_lshlrev_b32 v64, 16, v52
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v147
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v114, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v148
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v116, v83, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v50, v64
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v52, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v67
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v64
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v53, v55, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v102
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v65
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v118, v84, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v33
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v55
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v65, v55, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v67, v51, v51
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v66, 16, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v64, v101, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v52
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v54, v84, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v80
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v66, v80, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v67, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v51, v146, v64 :: v_dual_lshlrev_b32 v80, 16, v15
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v103
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v53, v67, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v65, v103, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v65, 0x400000, v67
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v65 :: v_dual_and_b32 v68, 0xffff0000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v70, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v16
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v52, v66, v64 :: v_dual_lshlrev_b32 v55, 16, v55
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v69, v67, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v55, v55, v55
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v82
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v67
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v68, v55, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v65, v82, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v53, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v67
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v128, v86, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v68
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v65, v53, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_add3_u32 v66, v68, v55, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v68, 0x400000, v55
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v66, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v70
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v64, v67, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v15, v50, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v50
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v113
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v55
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v66, v66, v66
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v113 :: v_dual_lshlrev_b32 v80, 16, v68
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v68 :: v_dual_and_b32 v15, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v84
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v70, v84, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v65, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v80, v71
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v54, v66, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v71, 0x400000, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v30
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v50, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-FAKE16-NEXT:    v_add3_u32 v54, v54, v66, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v14, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v13
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v30, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v70, v53, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v53, 16, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v69
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v67
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v67, v53, v67 :: v_dual_and_b32 v54, 0xffff0000, v54
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v70, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v29
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v30, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v71, v71
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v65, v65, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v13, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v55, v65, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v80, 0x400000, v65
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v66
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v12
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v29 :: v_dual_lshlrev_b32 v64, 16, v54
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v53
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v66, v54, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v55, v55, v65, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v80, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v71
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v29, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v66, v13, v13 :: v_dual_cndmask_b32 v29, v67, v64
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v55
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v53, v29, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v68
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v66, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v64, v68, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v70, v67
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v50, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add3_u32 v50, v53, v66, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v53 :: v_dual_and_b32 v55, 0xffff0000, v55
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v53, v54, v54 :: v_dual_lshlrev_b32 v54, 16, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v28
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v50
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v53
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v64, v29 :: v_dual_and_b32 v50, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v64, v53, 16, 1
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v53, v54, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v69
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v64, v64, v53, 0x7fff
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v55, v69, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v27
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v12
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v64, v67, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v68
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v28, v12 :: v_dual_lshlrev_b32 v67, 16, v11
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v27, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v54, v30, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v53
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v64
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v66
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v71
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v30, v71, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v67, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v54
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v66, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v50, v54, 16, 1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v50, v50, v54, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v64, v64, v64 :: v_dual_cndmask_b32 v27, v55, v27
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v65, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v26
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v54, v64, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v67 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v65, v26, v10 :: v_dual_lshlrev_b32 v68, 16, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v54, v64, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v30, v55, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v50
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v66
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v64
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v28
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v29, v13 :: v_dual_lshlrev_b32 v66, 16, v12
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v54 :: v_dual_lshlrev_b32 v64, 16, v55
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v66, v65
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v27
+; GFX11-FAKE16-NEXT:    v_perm_b32 v14, v14, v53, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v28, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v29
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v65
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v30, v12 :: v_dual_lshlrev_b32 v67, 16, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v67, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v54, 16, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v28
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v54, v54, v54
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v9
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v27, v11 :: v_dual_lshlrev_b32 v28, 16, v54
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v26
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v26 :: v_dual_lshlrev_b32 v29, 16, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v54, v28
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v25
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v26
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v10
+; GFX11-FAKE16-NEXT:    v_perm_b32 v12, v34, v12, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v26, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v66
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v30, v54, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX11-FAKE16-NEXT:    v_add3_u32 v30, v30, v54, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v24
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v25, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v50, v50, v50 :: v_dual_lshlrev_b32 v55, 16, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v30, v64 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v50, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v24
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v30
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v8
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v53, v50, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v30
-; GFX11-FAKE16-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
+; GFX11-FAKE16-NEXT:    v_perm_b32 v11, v35, v11, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v24, v8 :: v_dual_lshlrev_b32 v55, 16, v23
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v53, v66, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v65
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v23
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v65 :: v_dual_lshlrev_b32 v53, 16, v54
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v50
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v53, v53, v53 :: v_dual_cndmask_b32 v10, v28, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v25, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v53, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v54, v9, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v23, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11-FAKE16-NEXT:    v_add3_u32 v25, v28, v53, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v53
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v28, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v22
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v26, 16, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-FAKE16-NEXT:    v_perm_b32 v10, v36, v10, 0x5040100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v27, v9 :: v_dual_lshlrev_b32 v28, 16, v23
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v8
+; GFX11-FAKE16-NEXT:    v_perm_b32 v9, v37, v9, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v28, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v54, v9 :: v_dual_lshlrev_b32 v64, 16, v6
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_add3_u32 v53, v53, v28, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v28
-; GFX11-FAKE16-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
-; GFX11-FAKE16-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v26
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v8 :: v_dual_lshlrev_b32 v25, 16, v22
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v23, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v50, v8 :: v_dual_lshlrev_b32 v55, 16, v22
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v22, v6 :: v_dual_lshlrev_b32 v54, 16, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v24 :: v_dual_lshlrev_b32 v53, 16, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v28
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v24, v30, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v50, v8, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v25, v7 :: v_dual_lshlrev_b32 v50, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v54, v53
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v21, v5 :: v_dual_lshlrev_b32 v53, 16, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v23, v24, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v50, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v30, v30, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_add3_u32 v23, v23, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v28
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v30, 16, 1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v50 :: v_dual_lshlrev_b32 v50, 16, v20
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v24, v30, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v30
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v50
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
-; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v20, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX11-FAKE16-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v28, 16, v50
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-FAKE16-NEXT:    v_perm_b32 v8, v38, v8, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v22 :: v_dual_lshlrev_b32 v53, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v22, v28, v28
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v19, v3 :: v_dual_and_b32 v24, 0xffff0000, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v7, v39, v7, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v21 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v5 :: v_dual_lshlrev_b32 v22, 16, v19
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v30, v22, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v28, v5 :: v_dual_lshlrev_b32 v50, 16, v19
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v20, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v30, v30, v22, 0x7fff
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v50
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v17
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v19, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v30, v54 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v21, v21, v21 :: v_dual_lshlrev_b32 v30, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v18
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v28, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v24, v21, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v22, v4 :: v_dual_lshlrev_b32 v21, 16, v23
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v19, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v22
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v24, v21, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v16
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v17 :: v_dual_lshlrev_b32 v28, 16, v18
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v50, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v18, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v22, v24, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v25
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX11-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v17, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v22
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v24, v28, v28 :: v_dual_lshlrev_b32 v25, 16, v25
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v16, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v28, v24, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v25, v25, v25 :: v_dual_lshlrev_b32 v22, 16, v22
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v50, v25, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v19, v22, v22
-; GFX11-FAKE16-NEXT:    v_add3_u32 v22, v28, v24, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v24
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-FAKE16-NEXT:    v_add3_u32 v24, v50, v25, 0x7fff
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v53, v19, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v28, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v25
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-FAKE16-NEXT:    v_add3_u32 v50, v53, v19, 0x7fff
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v19
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v20, 16, v2
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v18 :: v_dual_lshlrev_b32 v23, 16, v24
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v50, v53, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v31, v3, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v17, v1 :: v_dual_lshlrev_b32 v20, 16, v16
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v0 :: v_dual_lshlrev_b32 v19, 16, v18
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v19
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v20
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_lshlrev_b32 v23, 16, v16
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v23
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v16, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v24
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v21, v1 :: v_dual_and_b32 v16, 0xffff0000, v22
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v20, v1 :: v_dual_lshlrev_b32 v16, 16, v19
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v14, v1, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX11-FAKE16-NEXT:    v_perm_b32 v14, v31, v27, 0x5040100
-; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v13, v0, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-FAKE16-NEXT:    v_perm_b32 v13, v32, v26, 0x5040100
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v15, v2, 0x5040100
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_perm_b32 v15, v39, v29, 0x5040100
-; GFX11-FAKE16-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v52, v0, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-FAKE16-NEXT:    v_perm_b32 v2, v32, v2, 0x5040100
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_perm_b32 v4, v15, v4, 0x5040100
+; GFX11-FAKE16-NEXT:    v_perm_b32 v15, v33, v51, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_minimumnum_v32bf16:
@@ -16961,990 +13011,751 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT:    scratch_load_b32 v51, off, s32
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v20
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v81, v81
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v52, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v82, v82
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v69, v69
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v80, 0xffff0000, v19
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s24
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v70, v70
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v50, v15 :: v_dual_mov_b32 v49, v14
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v13
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v82.l, v18.h, v52.l, s25
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v52.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v80, v80
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v49
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v82.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v29
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v32, v32
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v38, v38
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v134
+; GFX12-TRUE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v14
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v30
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v33, v33
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s44, v112, v134
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v33, v33
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v34, v34
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v13
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v13.h, v29.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v10.h, v26.h, s8
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v55, v55
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v112.l, v82.l, v52.l, s44
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v66, v66
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v55.l, v29.h, v15.l, s3
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v11
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v10
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff0000, v9
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v67, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v83, 0xffff0000, v1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v71, v71
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v85.l, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v33.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v39, v39
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v4.h, v20.h, s20
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v55.l
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v4
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v71, 0xffff0000, v3
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v16
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v55, v55
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s29, v85, v85
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v29
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v28
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v27
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v26
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v25
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v22
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v21
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v48, v48
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v20
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v80, 0xffff0000, v19
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v2
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v83, 0xffff0000, v1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v35, v35
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v37, v37
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v39, v39
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s9, v49, v49
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v51, v51
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v53, v53
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s14, v54, v54
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s17, v65, v65
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v67, v67
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s21, v69, v69
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v71, v71
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s40, v86, v86
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v66.l, v26.h, v33.l, s9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v80.l, v20.h, v39.l, s21
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v39.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v67, v67
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v83, v83
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v66.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v80.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v3.h, v19.h, s22
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v132
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v32.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v55.l
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v18
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v84, 0xffff0000, v17
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v36, v36
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v38, v38
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s8, v48, v48
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s10, v50, v50
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v52, v52
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v64, v64
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s18, v66, v66
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s20, v68, v68
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v70, v70
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v80, v80
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v81, v81
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v83, v83
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v9.h, v25.h, s11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v85.l, v16.h, v54.l, s40
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v118
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s19, v68, v68
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v5.h, v21.h, s18
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s42, v102, v132
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v81.l, v19.h, v48.l, s23
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s23, v85, v115
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v24
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v71.l, v21.h, v38.l, s19
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v15
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v14
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s26, v82, v82
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s28, v84, v84
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v102.l, v80.l, v39.l, s42
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v38.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v85.l, v55.l, v15.l, s23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v71.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff0000, v50
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v102, v102, v102 :: v_dual_lshlrev_b32 v101, 16, v101
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s12, v53, v53
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s26
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s26, v96, v118
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v131
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v85, v85, v85
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v30.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v69.l, v24.h, v38.l, s14
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v54.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s40, v86, v118
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v85.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v30
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v87, v87
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s41, v96, v96
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v33.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v96.l, v34.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v36.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v101.l, v39.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v112.l, v50.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v51.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v64.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v65.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v66.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v67.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v68.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v69.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v70.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v71.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v80.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v81.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v145.l, v82.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v116
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s42, v97, v97
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v96.l, v66.l, v33.l, s26
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v84, 0xffff0000, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s13, v54, v54
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s41, v101, v131
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v32.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v37.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v52.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v115.l, v53.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v87
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v30
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v34, v34
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s27, v84, v84
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v9.h, v25.h, s10
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v96, v96, v96
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v31, v31
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v84.l, v14.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v101.l, v71.l, v38.l, s41
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v64, 0xffff0000, v23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v54.l, v30.h, v14.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v35, v35
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v8.h, v24.h, s12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v67.l, v25.h, v34.l, s11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v114.l, v54.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v101
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v27
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s15, v64, v64
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v112
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v113, 16, v113
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v83.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v147.l, v84.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v118, 16, v119
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v128
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v129
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v130
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v131
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v132
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v133
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v134
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v135
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v144
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v144, 16, v145
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s63, v116, v86
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v86.l, v55.l, v32.l, s40
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0x8000, v55.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v114
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s16, v65, v65
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v12.h, v28.h, s4
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v34.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v68.l, v24.h, v35.l, s13
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v67.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s22, v84, v114
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v101, v101, v101
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s6, v36, v36
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v7.h, v23.h, s14
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s7, v37, v37
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v146
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v147
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s42, v87, v118
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s43, v96, v119
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s45, v98, v129
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s56, v101, v132
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s59, v112, v135
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s60, v113, v144
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v6.h, v22.h, s16
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v64.l, v28.h, v31.l, s5
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v11.h, v27.h, s6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v98.l, v35.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v69.l, v23.h, v36.l, s15
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v86.l, v32.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v118.l, v86.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v34.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v97.l, v35.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v36.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v39.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0x8000, v50.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0x8000, v51.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s46, v99, v130
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s61, v114, v145
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s62, v115, v146
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v96.l, v65.l, v34.l, s43
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v98.l, v67.l, v36.l, s45
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v101.l, v70.l, v39.l, s56
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v112.l, v81.l, v50.l, s59
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v113.l, v82.l, v51.l, s60
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v55.l, s16
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v118
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v33.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0x8000, v37.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v102.l, v48.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0x8000, v52.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0x8000, v53.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v68.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v119
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v84.l, v54.l, v14.l, s22
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v86.l, v31.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v65.l, v27.h, v32.l, s7
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v99.l, v36.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v70.l, v22.h, v37.l, s17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v83.l, v17.h, v53.l, s27
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v116.l, v64.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v69.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v128
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s27, v97, v119
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v87.l, v32.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v37.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v65.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v99, 16, v99
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v70.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v116
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v129
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s28, v98, v128
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s23, 0x8000, v70.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s26, 0x8000, v81.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v87.l, v64.l, v33.l, s42
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v97.l, v67.l, v34.l, s27
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v84, v84, v84 :: v_dual_lshlrev_b32 v87, 16, v87
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v99.l, v68.l, v37.l, s46
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v114.l, v83.l, v52.l, s61
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v115.l, v84.l, v53.l, s62
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v96.l, v34.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v98.l, v36.l, s5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v101.l, v39.l, s8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v101.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v112.l, v50.l, s11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v112.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v113.l, v51.l, s12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v113.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v55
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v100.l, v38.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v49.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0x8000, v54.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s44, v97, v128
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v116.l, v85.l, v54.l, s63
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v87.l, v33.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v99.l, v37.l, s6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.h, v114.l, v52.l, s13
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v114.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v115.l, v53.l, s14
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v115.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v39
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v35.h, v81.l, s26
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v35.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v117
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v130
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s24, v86, v116
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s29, v99, v129
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v98.l, v68.l, v35.l, s28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v114, v84, 16, 1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v103.l, v48.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v133.l, v81.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s25, v87, v117
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s40, v100, v130
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v86.l, v64.l, v31.l, s24
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v99.l, v69.l, v36.l, s29
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v97, v97, v97 :: v_dual_lshlrev_b32 v98, 16, v98
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v84
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v130, v96, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v114, v114, v84, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v84, v84
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v133
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v87.l, v65.l, v32.l, s25
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v100.l, v70.l, v37.l, s40
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v98, v98, v98 :: v_dual_lshlrev_b32 v99, 16, v99
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v131, 0x400000, v96
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v132, v97, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v114, v114, v115, s22
-; GFX12-TRUE16-NEXT:    v_add3_u32 v130, v130, v96, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v96, v96
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s43, v103, v133
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v87, 16, v87
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v86, v86, v86 :: v_dual_max_num_f32 v99, v99, v99
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v133, 0x400000, v97
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v134, v98, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v132, v132, v97, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v96, v130, v131, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v97, v97
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v87, v87, v87 :: v_dual_max_num_f32 v100, v100, v100
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v116, v85, 16, 1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v118, v86, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v144, 0x400000, v98
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v145, v99, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v134, v134, v98, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v97, v132, v133, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v98, v98
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v117, 0x400000, v85
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v86
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v128, v87, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v146, 0x400000, v99
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v147, v100, 16, 1
-; GFX12-TRUE16-NEXT:    v_add3_u32 v116, v116, v85, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s23, v85, v85
-; GFX12-TRUE16-NEXT:    v_add3_u32 v118, v118, v86, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s24, v86, v86
-; GFX12-TRUE16-NEXT:    v_add3_u32 v145, v145, v99, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s57, v102, v133
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v98, v134, v144, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v99, v99
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v15.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s5, 0x8000, v31.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v129, 0x400000, v87
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v84, 0x400000, v100
-; GFX12-TRUE16-NEXT:    v_add3_u32 v128, v128, v87, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s25, v87, v87
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s23
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s24
-; GFX12-TRUE16-NEXT:    v_add3_u32 v147, v147, v100, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v99, v145, v146, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v100, v100
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v32.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s25
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v116.h, v15.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v118.h, v31.l, s5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0x8000, v34.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s6, 0x8000, v33.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v97.l, v66.l, v35.l, s44
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v116.l, v54.l, s15
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v116.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s12, 0, v51
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0x8000, v48.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0x8000, v65.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0x8000, v66.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s47, v100, v131
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s58, v103, v134
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v102.l, v71.l, v48.l, s57
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v128.l, v96.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v97.l, v35.l, s4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v129.l, v97.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s13, 0, v52
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s14, 0, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s7, 0x8000, v38.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v49.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0x8000, v64.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s24, 0x8000, v71.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v128.h, v32.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v35.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s8, 0x8000, v36.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v97.h, v34.l, s7
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.h, v96.h, v33.l, s6
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v85, v101, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v100.l, v69.l, v38.l, s47
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v103.l, v80.l, v49.l, s58
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v87.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v102.l, v48.l, s9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v102.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v30.h, v65.l, s18
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v128
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v66.l, s19
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v129
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s25, 0x8000, v80.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s41, 0x8000, v85.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v38.l, s7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v103.l, v49.l, s10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v103.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v14.h, v64.l, s17
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v119
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.h, v34.h, v71.l, s24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v48
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v65
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v66
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v86.l, v13.h, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v98.h, v35.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v99.h, v36.l, s8
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v37.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v86, 0x400000, v101
-; GFX12-TRUE16-NEXT:    v_add3_u32 v85, v85, v101, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v84, v147, v84, s22
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v101, v101
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v14.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s9, 0x8000, v38.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s11, 0x8000, v54.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s12, 0x8000, v55.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s25
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v49
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.l, v37.h, v85.l, s41
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v64
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v71
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v96.l, v30.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v97.l, v32.l, s4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0x8000, v67.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0x8000, v68.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v98.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v99.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v85, v85, v86, s22
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v114.h, v14.l, s0
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v87, v102, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s13, 0x8000, v64.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v115, 0x400000, v102
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s22, v102, v102
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v54.l, s11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v87, v87, v102, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v39.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v87.l, v38.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v102.l, v38.h, s9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v32.h, v67.l, s20
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v130
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s21
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v131
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s29, 0x8000, v84.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v70
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v67
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s27, 0x8000, v82.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v68
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v64.l, s13
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s14, 0x8000, v65.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v128
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v86, v87, v115, s22
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v87, 0xffff0000, v114
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v113.l, v53.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s15, 0x8000, v66.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v135.l, v83.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v103.l, v81.l, v48.l, s43
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v116
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v37.l, v84.l, s29
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v101.l, v34.l, s8
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s28, 0x8000, v83.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s10, 0, v80
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v36.l, v82.l, s27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.h, v115.l, v37.l, s14
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s22, 0x8000, v69.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v65.l, s14
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v102
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s20, 0x8000, v71.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v113, 16, v113
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v103, 16, v103
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v101
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s16, 0x8000, v67.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s17, 0x8000, v68.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v131, 0xffff0000, v98
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v103, v103, v103
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s19, 0x8000, v70.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v132, 0xffff0000, v84
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v36.h, v83.l, s28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.h, v103.l, v35.l, s10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.h, v113.l, v36.l, s12
+; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v132.l, v100.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v33.h, v69.l, s22
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s11, 0, v81
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.h, v114.l, v48.l, s13
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v132
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v69
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v33.l, v68.l, s17
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s5, 0, v131
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v117, v103, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v119, 0x400000, v103
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v132
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v96
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s21, 0x8000, v80.l
-; GFX12-TRUE16-NEXT:    v_add3_u32 v117, v117, v103, 0x7fff
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v97
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s18, 0x8000, v69.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v115
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v85
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s8, 0, v133
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v33.h, s7
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v51
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v50.h, v51.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v118
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.h, v31.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v35, 16, v34
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v84.h, v37.l, s4
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v85.h, v38.l, s9
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v100
-; GFX12-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v32.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s4, 0, v129
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v34.l, v70.l, s19
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v130
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v99
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v35, v36
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v14.h, v55.l, s12
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v86.h, v39.l, s10
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v86
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s10, 0x8000, v31.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v32.l, v31.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v87
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.h, v118.h, v15.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s9, 0, v39
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v35.l, v80.l, s21
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v36.l
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v31
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.h, v114.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v31.h, v66.l, s15
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.h, v128.h, v15.h, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v37
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v32.h, v67.l, s16
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.h, v86.h, v35.l, s9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v96.h, v36.l, s3
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v33.h, v69.l, s18
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v54, v38, v38
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v34.h, v71.l, s20
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.h, v116.h, v35.h, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.h, v98.h, v33.l, s5
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.h, v84.h, v34.l, s7
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v55, v54, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s11, v54, v54
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v97.h, v37.l, s4
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s6, 0, v130
-; GFX12-TRUE16-NEXT:    v_add3_u32 v55, v55, v54, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.h, v99.h, v37.h, s6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v55, v64, s11
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v64, v112, v112
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v54.h, v31.l, s10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v54
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v65, v64, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v66, 0x400000, v64
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v32.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v39
-; GFX12-TRUE16-NEXT:    v_add3_u32 v65, v65, v64, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.h, v85.h, v38.l, s8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v54.h, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v135
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v103, v103
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v113, v54
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v55, v117, v119 :: v_dual_and_b32 v54, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v83.l, v53.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v50.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v15.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v55.h, v48.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v81.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v15.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v0.h, v16.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.h, v98.l, v32.h, s5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v51.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v99.l, v33.l, s6
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v54, v65, v66, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v52.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v67
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v55
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v16.h, v15.l, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v81.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v54.h, v52.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v64, v64, v64
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v50.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v52, v53
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v31.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v82.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v65
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v67, v64, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v66
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v32.l, v82.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v64, v64
-; GFX12-TRUE16-NEXT:    v_add3_u32 v66, v67, v64, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v67, 0x400000, v64
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v52, v65
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v50
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff0000, v54
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.h, v55.h, v14.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v64, v66, v67, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v53.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v51
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v68
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.l, v50.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v52
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v51.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v32.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v50.l, s0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v38, v53
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.h, v112.l, v39.l, s11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v64.h, v53.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v15.h, v51.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v14.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v29
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v50
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v30.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v33.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v50.l, v51.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v83.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.h, v54.h, v32.l, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v49
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v51.l, v33.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v33.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v34.l, v83.l, s1
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v52, v52, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v14.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v64
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v66, v52, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.h, v32.l, v15.h, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v117, v117
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v31.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v49.l, v30.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX12-TRUE16-NEXT:    v_add3_u32 v49, v66, v52, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v52
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v65, v55
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v30.l, v34.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v13
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v52, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v51, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v15.l, v31.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.h, v64.h, v32.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v14.l, v33.l, s1
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v30.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v52
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v49, v49, v53, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v34.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v29
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v32.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v29.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v28
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v13.l, v29.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v52
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v31.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v52, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.l, v29.l, v13.l, s0
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v52, v55, v55 :: v_dual_and_b32 v53, 0xffff0000, v49
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v30.l, v34.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v13.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v29.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v13.h, v31.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v15.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v55
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v54, v52, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v12.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.h, v49.h, v13.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v12
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX12-TRUE16-NEXT:    v_add3_u32 v54, v54, v52, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v52, v52
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v53, v53, v53
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v29.l, v13.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v28.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v33.l, v15.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v13.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v29.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v27
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v30.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v52, v54, v65, s0
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v54, v53, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v33.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v12.l, v28.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v55, 0x400000, v53
-; GFX12-TRUE16-NEXT:    v_add3_u32 v54, v54, v53, 0x7fff
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v15.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v30.l, v13.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v51, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v28.l, v28.l, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v52.h, v33.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v14.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v64
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v49, v54, v55, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v34.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v52
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v28.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v53
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v29.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.h, v12.h, v14.l, s0
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v53, v53, v53
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v34.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v55
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v64
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v65, v53, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v12.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v14.l, v30.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v49
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v53, v53
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v64, v65, v53, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v65, 0x400000, v53
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v66
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v28.l, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v11.h, v29.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v53, v64, v65, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v13.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v15.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v14.l, v49.h, v14.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.h, v31.l, v12.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v27.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v11.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v26
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v28.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v31.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v53.h, v13.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v29.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v55
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v52
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v27.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v10
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v51, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.h, v10.h, v28.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.h, v11.h, v29.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff0000, v53
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v49, v49, v49
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v13.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v26
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v15.l, v52.h, v12.h, s0
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v55, v49, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v29
-; GFX12-TRUE16-NEXT:    v_add3_u32 v27, v55, v49, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v29, 0x400000, v49
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v54, v52
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v10.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v25
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v26.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v51
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v53.h, v11.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v10.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v26.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v29, v27, v29, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v27.l, v13.l, v11.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v28, v28
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v29.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v27
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v9.l, v25.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v53, v53
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v52, v49
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v28.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v27, v12, v12
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v52
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v51, v50
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v31.l, v10.h, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v26.l, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v28.l, s1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v28, v27, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v49
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v25.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v9.l
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v53, 0x400000, v27
-; GFX12-TRUE16-NEXT:    v_add3_u32 v28, v28, v27, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v49
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v52
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v27, v27
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v25.l, v12.l, v10.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v11.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v9.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v29.h, v9.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v52, v49
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v27, v28, v53, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v11.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v8
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v29, v12, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v27
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v27.h, v11.l, s0
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v25.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v24
-; GFX12-TRUE16-NEXT:    v_add3_u32 v29, v29, v12, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v13.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v25.l, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v24
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v27, v26
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v12.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v29, v49, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v10.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v8.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v24.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v27
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v12.h, v10.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v28, v28
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v11.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v13.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.h, v9.h, v26.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v10, 16, 1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v28
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v52, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_add3_u32 v24, v24, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v26
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v29, v29
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v23
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v24, v24, v52, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v9.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v9.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.h, v8.h, v12.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v11.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v23.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v49
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v26
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v24.h, v9.l, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v25.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v27.h, v8.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v10.l, v9.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v8.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v12.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v10.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v25.l, s2
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v12.h, v9.h, s1
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v27, v10, v10 :: v_dual_and_b32 v26, 0xffff0000, v24
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v25, v27, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v7.l, v23.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v24, v24
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v11.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v27
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v26
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v12, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT:    v_add3_u32 v23, v25, v27, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v27
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v26, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v12.l, v22.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v24.h, v7.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v11.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v23.l, v7.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v6
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.h, v7.h, v11.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v22, v23, v25, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v12.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v22.h, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v25, v8, v8 :: v_dual_lshlrev_b32 v24, 16, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v11.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v11.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v23, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v6.l, v22.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v23, v23
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v24, v25, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v26
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v25, v24
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v12.l, v6.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v25, v25
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v22.l, v6.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v10.l, v7.h, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v24, v25, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v25
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v5.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v10.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v22.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v11, v21, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v6.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v4
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v11.h, v7.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v10.l, v7.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v21
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v10.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v12, v11
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v9.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v21
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v20
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v22, v8, 16, 1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v9.l, s1
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v10.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v22, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v7.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v20.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_dual_cndmask_b32 v8, v9, v21 :: v_dual_lshlrev_b32 v9, 16, v22
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v20
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v21
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v8.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v6, v9, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v22
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v37.l, v11.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v21, v20
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v8.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.h, v5.h, v9.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v5.h, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v6, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v7.l, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v19
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v20
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v8.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v4.l, v20.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v21
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v9, v12, v20, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_and_b32 v11, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v20.l, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v8.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v11
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v19
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v8.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v9.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff0000, v9
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v6, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v20
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v12, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.h, v4.h, v10.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v19.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v3.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v18
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v18
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v6.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v4.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_add3_u32 v19, v19, v6, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v20, 0x400000, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v18.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v6, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v9.h, v4.h, s0
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v11, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.h, v9.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v35.l, v7.l, v4.h, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v18.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v11, v19, v20, s3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v18.l, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v8.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v2.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v3.h, v6.l, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v11.h, v4.l, s3
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v17
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v16
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v3.l, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v17
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v16
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v10, v4
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v11, v11
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v17.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v17.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v0.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v12, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v6.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v16.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v4.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v9.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v2.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v19, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v16, v12
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v19, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v18, v17
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v17, v8, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v10, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v2.h, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v10.l, v9.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v16, v17, 16, 1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v12, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v12.h, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v16, v17, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v17
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v2.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v17, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v3, v3, v3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s0
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v10, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v10, v3, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v3
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v16
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v5, v8, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v7.h, v2.l, s0
-; GFX12-TRUE16-NEXT:    v_add3_u32 v10, v10, v3, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v18, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    v_add3_u32 v5, v5, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v1.h, v4.l, s1
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v2.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v3, v10, v17, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v18, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v9.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v9.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v17
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v50.l, v12.h, v1.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v11.h, v0.h, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v16
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v11
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s3, 0, v10
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v13.l, v5.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v30.l, v3.h, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v51.l, v7.h, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v3, v50 :: v_dual_mov_b32 v4, v48
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v13 :: v_dual_mov_b32 v1, v30
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v51 :: v_dual_mov_b32 v5, v39
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v6, v38 :: v_dual_mov_b32 v7, v37
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v8, v35 :: v_dual_mov_b32 v9, v33
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v10, v32 :: v_dual_mov_b32 v11, v31
-; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v12, v36 :: v_dual_mov_b32 v13, v34
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v49.l, v2.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v48.l, v8.l, v1.h, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v39.l, v6.l, v0.h, s2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v38.l, v9.l, v2.h, s3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v0, v29 :: v_dual_mov_b32 v1, v49
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, v48 :: v_dual_mov_b32 v3, v39
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v4, v38 :: v_dual_mov_b32 v5, v36
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v6, v35 :: v_dual_mov_b32 v7, v34
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v8, v33 :: v_dual_mov_b32 v9, v32
+; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v10, v31 :: v_dual_mov_b32 v11, v30
+; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v12, v37
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v32bf16:
@@ -17954,1099 +13765,792 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v12
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v51, 16, v28
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v12
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v30
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v14
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v71, 16, v25
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v13
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX12-FAKE16-NEXT:    scratch_load_b32 v50, off, s32
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v31, 0xffff0000, v15
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v38, 16, v29
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v39, 16, v29
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v48, 16, v13
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v35, 16, v30
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v36, 16, v14
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v83, 16, v24
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v70, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v52, v52, v51, s1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v13
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v80, 16, v9
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v84, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v34, v35, v33 :: v_dual_and_b32 v49, 0xffff0000, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v48, 0xffff0000, v29
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v39, v38, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v38, v38
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v87, 16, v23
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v52, v51, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v82, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v96, 16, v7
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v99, 16, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v39, v64, v55 :: v_dual_and_b32 v70, 0xffff0000, v9
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v48, v48, v39, s0
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v100, 16, v6
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v103, 16, v21
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v115, 16, v20
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v30
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v112, 16, v5
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v68, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v36, v35, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v86, 0xffff0000, v7
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v29
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v4
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v119, 16, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v18
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v80, v71, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v80, v71, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v82, v82
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v98, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v18
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v3
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v84, v83, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v84, v83, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v28
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v102, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v32, 16, v15
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v26
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v135, 16, v17
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v96, v87, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v96, v87, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v26
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v118, 0xffff0000, v3
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v25
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v144, 16, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v146, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v100, v99, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v114, 0xffff0000, v4
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v100, v99, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v147, 16, v16
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v34, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v54, v54
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v68, v112, v103 :: v_dual_and_b32 v81, 0xffff0000, v25
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v112, v103, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v114, v114
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v85, 0xffff0000, v24
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v130, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v22
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v14
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v11
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v67, 16, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v116, v115, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v116, v115, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v118, v118
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v101, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v134, 0xffff0000, v1
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v113, 0xffff0000, v21
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v64, v64, v55, s2
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v68, 16, v10
+; GFX12-FAKE16-NEXT:    scratch_load_b32 v31, off, s32
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v128, v119, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v128, v119, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v130, v130
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v20
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v82, v132, v131 :: v_dual_and_b32 v113, 0xffff0000, v21
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v132, v131, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v134, v134
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v84, v144, v135 :: v_dual_and_b32 v117, 0xffff0000, v20
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v102, v144, v135 :: v_dual_and_b32 v133, 0xffff0000, v18
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v146, v146
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v96
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v34, v147, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v27
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v18
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v114, 16, v33
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v38, v35, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v14, v30 :: v_dual_and_b32 v97, 0xffff0000, v23
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v35, v36, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v48, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v37, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v51, v52, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v97, 0xffff0000, v23
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v51
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v116, 16, v100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v39, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v55, v64 :: v_dual_lshlrev_b32 v130, 16, v51
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v66, 0xffff0000, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v34
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s3, v66, v66
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v68, v68, v67, s3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v49, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v68, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v68
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v80
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v52 :: v_dual_lshlrev_b32 v118, 16, v67
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v67
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v54, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v83, v80, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v97, v97
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v97, 16, v84
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v64 :: v_dual_lshlrev_b32 v128, 16, v83
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v87, v87, v82 :: v_dual_lshlrev_b32 v134, 16, v83
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v101, v101
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v97, v99, v66, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v99, v84, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v113, v113
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v129, 0xffff0000, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v99, v103, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v103, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v99
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v117, v117
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v39
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v36
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v101, v115, v70, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v115, v96, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v129, v129
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v34
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v115, 16, v38
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v49
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v129, 16, v87
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v82
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v103, v119, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v115, v119, v98 :: v_dual_lshlrev_b32 v146, 16, v113
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v133, v133
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v35
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v119, 16, v71
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v103
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v113, v131, v82, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v86, v114
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v52
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v99
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v113
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v33, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v36, v115
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v145, 0xffff0000, v17
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v38, v35, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v48, v116
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v86, v86, v86 :: v_dual_lshlrev_b32 v85, 16, v64
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v51, v37, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v114, v86, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v115, 0x400000, v86
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v48
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v117, 16, v55
-; GFX12-FAKE16-NEXT:    v_add3_u32 v114, v114, v86, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v48, v48, v48
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v117
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s1, v48, v48
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v118
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v66
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v118, v48, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v49, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v119
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v70
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v53, v53, v53 :: v_dual_lshlrev_b32 v102, 16, v80
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v119, 0x400000, v48
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v71, v52, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v81, v128
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v68
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v128, v53, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v118, v118, v48, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v85, v129
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v112, 16, v82
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v53
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v69, v69, v69
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v81, 16, v81
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v64, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v128, v128, v53, 0x7fff
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s2, v53, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v118, v118, v119, s1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v81, v81, v81
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v128, v128, v129, s2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v144, 0x400000, v81
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v130, 16, v97
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v85, v85, v85
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v96, v130
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v146, v85, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v147, 0x400000, v85
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v97, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v98, v131
-; GFX12-FAKE16-NEXT:    v_add3_u32 v146, v146, v85, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v132, 16, v101
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v98, v99, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v100, v132
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v132, v69, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v96, v96, v96
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v98, 16, v98
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v100, v101, v70, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v102, v133
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v133, 0x400000, v69
-; GFX12-FAKE16-NEXT:    v_add3_u32 v132, v132, v69, 0x7fff
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v98, v98, v98
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v102, v103, v80, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v112, v134
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v134, v81, 16, 1
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v48, v98, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v98
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v82, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v86, v96, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v134, v134, v81, 0x7fff
-; GFX12-FAKE16-NEXT:    v_add3_u32 v48, v48, v98, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v114, v115, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v86, v86, v96, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v65, v65, v65
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v130, v65, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v131, 0x400000, v65
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v130, v130, v65, 0x7fff
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v37, 16, v48
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v130, v131, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v132, v133 :: v_dual_lshlrev_b32 v36, 16, v36
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v36, v36, v36
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v117, v131, v100, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v101, 16, v86
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v134, v144, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v116, v36, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v85, v85
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v36
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v36, v36
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v130, 16, v81
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-FAKE16-NEXT:    v_add3_u32 v116, v116, v36, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v36, 0x400000, v96
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v146, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v96, v96
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v100, v100, v100
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v116, v116, v117, s0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v131, 16, v85
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v119, v135, v102, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v38, v38
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v86, v36, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v115, v100, 16, 1
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v98, v98
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v117, 0x400000, v100
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v86, 16, v114
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v98, 16, v116
-; GFX12-FAKE16-NEXT:    v_add3_u32 v115, v115, v100, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v48, v53, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v100, v100
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v132, 16, v36
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v100, 0xffff0000, v116
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v116, 16, v128
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v133, 0xffff0000, v36
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v115, v117, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v34
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v115, 0xffff0000, v118
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v134, 16, v48
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v146, 16, v53
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v38, v147, v34 :: v_dual_lshlrev_b32 v49, 16, v52
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e64 s0, v49, v130
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v86, v34, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v35
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v96, 0xffff0000, v114
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v114, 16, v118
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v118, 16, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v98, v35, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v37
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v117, 0xffff0000, v128
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v128, 16, v69
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v144, 0xffff0000, v48
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v114, v37, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v39
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v66, v30, v54 :: v_dual_lshlrev_b32 v53, 16, v64
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v35
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v14, v30
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v70
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v117
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v116, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v49
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v130, v35, v36 :: v_dual_lshlrev_b32 v129, 16, v39
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v37, v129
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v129, v51, v52, s0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v118, v49, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v37, v39, v48 :: v_dual_lshlrev_b32 v118, 16, v102
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v131, 16, v55
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v131
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v128, v52, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
-; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v50
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v55, v64 :: v_dual_lshlrev_b32 v50, 16, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v133, 16, v71
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v132
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v130, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v67, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v133
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v135, 16, v87
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v131, v64, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v70 :: v_dual_lshlrev_b32 v132, 16, v65
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v81, v134
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v147, v32, v52 :: v_dual_and_b32 v64, 0xffff0000, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v66
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v102, v102, v102
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v81, v83, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v85, v135
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v145, 16, v103
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v132, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v68
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v65
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v119, v102, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v129, 0x400000, v102
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v85, v87, v82, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v97, v144
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v134, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v69, 0xffff0000, v69
-; GFX12-FAKE16-NEXT:    v_add3_u32 v119, v119, v102, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v97, v99, v84 :: v_dual_lshlrev_b32 v114, 16, v98
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v101, v145
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v147, 16, v115
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v70
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v81, 0xffff0000, v81
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v52
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v101, v103, v86 :: v_dual_lshlrev_b32 v144, 16, v97
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v112, v146
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v146, v70, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v33
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v0
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v34, v33, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v38
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v35, v35, v38 :: v_dual_lshlrev_b32 v34, 16, v147
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v51
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v51 :: v_dual_and_b32 v85, 0xffff0000, v85
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v34, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v112, v113, v96, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v114, v147
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v119
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v49, 16, v38
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v52, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v114, v115, v98, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v116, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v37, v55, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v67
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v116, v117, v100, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v118, v30
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v39, v67, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v71
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v118, v119, v102, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v128, v49
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v48, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v83
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v128, v38, v34, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v36
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v49, v83, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v87
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v36, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v49, v54, v87 :: v_dual_lshlrev_b32 v34, 16, v34
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v97
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v37, v48, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v31, v97, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v99
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v129, v52, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v64
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v52, 16, v129
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v34, v34, v34 :: v_dual_cndmask_b32 v55, v32, v99
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v96
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v54, v34, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v34
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v53, v64, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v68
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v86, v33, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v100
-; GFX12-FAKE16-NEXT:    v_add3_u32 v54, v54, v34, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v65, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v70
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v98, v35, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v115
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v69, v70, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v80
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v114, v36, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v81, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v82
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v117
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v85, v82, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v84
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v116, v37, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v65
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v97, v84, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v86
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v38, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v69
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v69, 16, v16
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v101, v86, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v96
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v128, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v81
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v96, v112, v96, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v98
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v130, v48, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e64 vcc_lo, 0x8000, v147
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v98, v114, v98 :: v_dual_lshlrev_b32 v131, 16, v53
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v147, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v85
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v100, v116, v100 :: v_dual_lshlrev_b32 v133, 16, v69
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v35
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v131, v49, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v14, v14, v35 :: v_dual_lshlrev_b32 v135, 16, v85
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v102
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v39, v52, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v133
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v54, 0xffff0000, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v118, v102, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v39
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v48, v132, v51 :: v_dual_lshlrev_b32 v51, 16, v112
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v144
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v36, v36, v39 :: v_dual_lshlrev_b32 v145, 16, v101
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v34
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v134, v55, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v84
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v128, v34, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v51
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v65, v39, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v145, v145
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v49, v51, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v55
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v51, 16, v128
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v135, v84, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v102, v102
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v64, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v67
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v119, v129, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v101
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v54
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v67, v51, v51
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v66, 16, v52
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v71
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v64, v101, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v52, 0xffff0000, v52
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v70, v71 :: v_dual_lshlrev_b32 v147, 16, v114
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v83
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v54, v84, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v80
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v80, v83, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v87
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v66, v80, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v67, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v82, v87, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v99
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v51, v146, v64 :: v_dual_lshlrev_b32 v80, 16, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v70, v84, v99, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v103
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v53, v67, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v65, v103, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v65, 0x400000, v67
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v65 :: v_dual_and_b32 v68, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v71, v86, v103 :: v_dual_lshlrev_b32 v30, 16, v130
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v113
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v48, 16, v37
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v70, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v65, 0xffff0000, v16
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v80, v96, v113, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v115
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v52, v66, v64 :: v_dual_lshlrev_b32 v55, 16, v55
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v65, 16, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v82, v98, v115, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v117
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v134, 16, v81
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v69, v67, vcc_lo
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v55, v55, v55
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v82
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v67
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v64
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v68, v55, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v83, v100, v117, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v119
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v65, v82, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v66, v68, v55, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v68, 0x400000, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v84, v35, v119, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v38
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v66, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v70
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v86, v34, v38, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v64, v67, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v130, v14, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v48
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v15, v50, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v113
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v70, 16, v55
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v66, v66, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v37, v36, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v52
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v52, 16, v31
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v69, v71, v113 :: v_dual_lshlrev_b32 v80, 16, v68
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v34, v129, v39, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v131
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v68 :: v_dual_and_b32 v15, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v84
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v50
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v35, v53, v49, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v132
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v53, 0xffff0000, v31
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v70, v84, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v36, v65, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v133
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v65, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v14
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v37, v69, v64, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v134
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v80, v71
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v54, v66, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v71, 0x400000, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v80, 16, v30
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v38, v81, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v135
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v50, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX12-FAKE16-NEXT:    v_add3_u32 v54, v54, v66, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v39, v85, v68, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v144
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v69, v14, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v48, v97, v70, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v145
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v71, 16, v13
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v49, v101, v71, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v30, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v15, v31, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v31
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v148, 16, v116
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v55
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v146, 16, v112
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v33
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v30
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v146
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v14, v70, v53, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v53, 16, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v69
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v67
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v15, v112, v80, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v67, v53, v67 :: v_dual_and_b32 v54, 0xffff0000, v54
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v70, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v29
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v65
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v52, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v102, 16, v118
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v30, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v71, v71
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v65, v65, v65
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v31, v55 :: v_dual_lshlrev_b32 v64, 16, v52
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v147
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v71, v13, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v55, v65, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v80, 0x400000, v65
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v55, v55, v65, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v31, v114, v82, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v148
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v55, v80, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v70, 16, v71
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v66
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v32, v116, v83, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v50, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v29, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v52, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v67
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v66, v13, v13 :: v_dual_cndmask_b32 v29, v67, v64
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v54
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v64, 16, v55
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v65
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v65, v53, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v102
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v65
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v53, v29, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v68
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v66, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v118, v84, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v33
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v64, v68, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v70, v67
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v55
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v50
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v65, v55, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v52
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v50, vcc_lo
-; GFX12-FAKE16-NEXT:    v_add3_u32 v50, v53, v66, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v67, 16, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v11
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v33, v52, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v53 :: v_dual_and_b32 v55, 0xffff0000, v55
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v53, v54, v54 :: v_dual_lshlrev_b32 v54, 16, v28
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v55, v53, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v67
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v55
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v55, 16, v50
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v33, v64, v33, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v51
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v29, v64, v29 :: v_dual_and_b32 v50, 0xffff0000, v50
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v64, v53, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v52, v128, v86, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v68
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v69
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v64, v64, v53, 0x7fff
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v28
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v51, v65, v53, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v66
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v55, v69, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v27
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v13, v13, v29 :: v_dual_lshlrev_b32 v64, 16, v54
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v53
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v12
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v66, v54, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v64, v67, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v28
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v69, v68
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v64, v28, v12 :: v_dual_lshlrev_b32 v67, 16, v11
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v66, v27, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v54, v30, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v53
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v64
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v66
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v71
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v54, v54, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v53, v54, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v68, v30, v71, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v67, v64
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v67, 0x400000, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v28, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v66
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v64, v66, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v50, v54, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v64
-; GFX12-FAKE16-NEXT:    v_add3_u32 v50, v50, v54, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v54, v66, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v65, v64
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v53
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v64, v64, v64 :: v_dual_cndmask_b32 v27, v55, v27
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v65
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v55, v29, v13 :: v_dual_lshlrev_b32 v66, 16, v12
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v55, v68, v65, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v68, 16, v26
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v54, v64, 16, 1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v50, v50, v67 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v65, v26, v10 :: v_dual_lshlrev_b32 v68, 16, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v53, v54 :: v_dual_lshlrev_b32 v64, 16, v55
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v66, v65
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v27
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v54, v64, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v64
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v30, v55, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v65
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
+; GFX12-FAKE16-NEXT:    v_perm_b32 v14, v14, v53, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v30, v12 :: v_dual_lshlrev_b32 v67, 16, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v28, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v67, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v11
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v54, v65, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v68, v68
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v54, 16, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v28
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v54, v54, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v12, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v53
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v25
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v65, v65
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v27, v27, v11 :: v_dual_lshlrev_b32 v28, 16, v54
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v64
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v30, v12, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v66
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v30, v54, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v12, v33, v12, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v13, v55, v13, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v26
+; GFX12-FAKE16-NEXT:    v_perm_b32 v13, v30, v13, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX12-FAKE16-NEXT:    v_add3_u32 v30, v30, v54, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v64, 0x400000, v54
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v66, 16, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v12, v54, v12, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v25, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v50
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v50, v50, v50 :: v_dual_lshlrev_b32 v55, 16, v8
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v26 :: v_dual_lshlrev_b32 v29, 16, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v27
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v54, v28
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v27, v11, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v26, v26, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v25
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v9, v25, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v26
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v30, v64 :: v_dual_and_b32 v53, 0xffff0000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v66, 0x400000, v50
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v53
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v50, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v11, v27, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v28
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v10
+; GFX12-FAKE16-NEXT:    v_perm_b32 v12, v34, v12, 0x5040100
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v54, 16, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v11, v28, v11 :: v_dual_lshlrev_b32 v54, 16, v24
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v29, v26, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v11, v28, v11, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v8
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v53, v50, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v30, 0xffff0000, v30
-; GFX12-FAKE16-NEXT:    v_perm_b32 v11, v34, v11, 0x5040100
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v29
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v11, v35, v11, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v28, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v55, v54
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v54, v24, v8 :: v_dual_lshlrev_b32 v55, 16, v23
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v64, v64
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v25, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v64, 16, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v10, v26, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v53, v66, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v27, v9, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v65
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v23
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v29, v10, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v10, v65 :: v_dual_lshlrev_b32 v53, 16, v54
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v30
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v50
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v26, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v7
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v53, v53, v53 :: v_dual_cndmask_b32 v10, v28, v10
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v53, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
+; GFX12-FAKE16-NEXT:    v_perm_b32 v10, v36, v10, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v54, v9, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v55, 16, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v50, 0xffff0000, v50
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v30, v23, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v9, v25 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX12-FAKE16-NEXT:    v_add3_u32 v25, v28, v53, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v53
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v7 :: v_dual_lshlrev_b32 v26, 16, v24
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v25, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v28, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v22
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v27, v9 :: v_dual_lshlrev_b32 v28, 16, v23
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v8
+; GFX12-FAKE16-NEXT:    v_perm_b32 v9, v37, v9, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v50
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v28, 16, 1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v27, v26
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v26, v24, v8 :: v_dual_lshlrev_b32 v25, 16, v22
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v29, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v9, v54, v9 :: v_dual_lshlrev_b32 v64, 16, v6
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v27, v23, v7, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v53, v53, v28, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v28
-; GFX12-FAKE16-NEXT:    v_perm_b32 v10, v35, v10, 0x5040100
-; GFX12-FAKE16-NEXT:    v_perm_b32 v9, v36, v9, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v5
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v50, v8 :: v_dual_lshlrev_b32 v55, 16, v22
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v64, v55
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v53, v22, v6 :: v_dual_lshlrev_b32 v54, 16, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v8, v8, v24 :: v_dual_lshlrev_b32 v53, 16, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v24, 0xffff0000, v25
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v28
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v24, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v50, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v24, 16, v26
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v22
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v6
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v28, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v27
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v25, v7 :: v_dual_lshlrev_b32 v50, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v54, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v22, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v30, v21, v5 :: v_dual_lshlrev_b32 v53, 16, v20
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v26, v8, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v23 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v23, v24, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v50, 0x400000, v24
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v30, v30, v30
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v21
+; GFX12-FAKE16-NEXT:    v_perm_b32 v8, v38, v8, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX12-FAKE16-NEXT:    v_add3_u32 v23, v23, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v4
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v27, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v39, v7, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v28, 0xffff0000, v28
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v22, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v30, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v23, v50 :: v_dual_lshlrev_b32 v50, 16, v20
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v28
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v24, v30, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v30
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v21 :: v_dual_lshlrev_b32 v24, 16, v4
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v23
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v50
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v23
-; GFX12-FAKE16-NEXT:    v_perm_b32 v7, v38, v7, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v23, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v50, v20, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX12-FAKE16-NEXT:    v_perm_b32 v8, v37, v8, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v21, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v21
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v20, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v28, 16, v50
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v22
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v20
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v4
+; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v6, v22 :: v_dual_lshlrev_b32 v53, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v22, v28, v28
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v28, 16, v24
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v23, v21, v5 :: v_dual_lshlrev_b32 v22, 16, v19
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v19, v19, v3 :: v_dual_and_b32 v24, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v19, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v30, v22, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v54, 0x400000, v22
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v28, v5 :: v_dual_lshlrev_b32 v50, 16, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v19
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v20, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v21
-; GFX12-FAKE16-NEXT:    v_add3_u32 v30, v30, v22, 0x7fff
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v21, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v53, v50
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v17
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v19, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v22, v30, v54 :: v_dual_lshlrev_b32 v21, 16, v21
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v21, v21, v21 :: v_dual_lshlrev_b32 v30, 16, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v23, 16, v22
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v25, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v18
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v28, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v24, v21, 16, 1
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v22, v4 :: v_dual_lshlrev_b32 v21, 16, v23
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v24
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v19, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v22
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v23, v5, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v20
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v50, 16, v2
+; GFX12-FAKE16-NEXT:    v_perm_b32 v5, v49, v5, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v20, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v1
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v22
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v24, v21, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v24, 0x400000, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v16
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v17 :: v_dual_lshlrev_b32 v28, 16, v18
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v17, v17, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v50, v28
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v19 :: v_dual_lshlrev_b32 v20, 16, v2
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v18
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v28, v18, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v17
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v16, v16, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v21, v22, v24, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v16
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v30, v25
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v30, 16, v21
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v21
-; GFX12-FAKE16-NEXT:    v_perm_b32 v6, v48, v6, 0x5040100
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v18 :: v_dual_lshlrev_b32 v23, 16, v24
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v25, v17, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v22
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v24, v28, v28 :: v_dual_lshlrev_b32 v25, 16, v25
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v0
+; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v31, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v16, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v28, v24, 16, 1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v25, v25, v25 :: v_dual_lshlrev_b32 v22, 16, v22
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v18, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v1
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v19
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v50, v25, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc_lo
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v19, v22, v22
-; GFX12-FAKE16-NEXT:    v_add3_u32 v22, v28, v24, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v24
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-FAKE16-NEXT:    v_add3_u32 v24, v50, v25, 0x7fff
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v53, v19, 16, 1
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v22, v22, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v28, 0x400000, v25
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-FAKE16-NEXT:    v_add3_u32 v50, v53, v19, 0x7fff
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v53, 0x400000, v19
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v54, 16, v22
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v17, v17, v1 :: v_dual_lshlrev_b32 v20, 16, v16
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v2
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v24, v24, v28, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v16, v16, v0 :: v_dual_lshlrev_b32 v19, 16, v18
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v17
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v23, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v50, v53, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v21, 16, v24
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v25, 16, v19
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v19, v18, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v24, v20
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v30, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v20, v17, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX12-FAKE16-NEXT:    v_perm_b32 v3, v52, v3, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v19, v2 :: v_dual_lshlrev_b32 v23, 16, v16
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v25, v23
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v23, v16, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v21, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v20, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v18
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v18, 0xffff0000, v19
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v17
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v17, 0xffff0000, v24
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v20
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v16
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v23
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v21, v1 :: v_dual_and_b32 v16, 0xffff0000, v22
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v20, v1 :: v_dual_lshlrev_b32 v16, 16, v19
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v14, v1, 0x5040100
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v1, v50, v1, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v25, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v23, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v16
-; GFX12-FAKE16-NEXT:    v_perm_b32 v14, v31, v27, 0x5040100
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v13, v0, 0x5040100
+; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v52, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v54, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-FAKE16-NEXT:    v_perm_b32 v13, v32, v26, 0x5040100
-; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v15, v2, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v19, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_perm_b32 v2, v32, v2, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v23, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_perm_b32 v15, v39, v29, 0x5040100
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_perm_b32 v4, v51, v4, 0x5040100
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v22, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_perm_b32 v4, v15, v4, 0x5040100
+; GFX12-FAKE16-NEXT:    v_perm_b32 v15, v33, v51, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <32 x bfloat> @llvm.minimumnum.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y)
   ret <32 x bfloat> %result
@@ -19076,25 +14580,15 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v3, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_mul_f32_e32 v2, 1.0, v2
-; GFX8-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v2
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
-; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
 ; GFX8-NEXT:    s_movk_i32 s4, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_minimumnum_bf16_no_ieee:
@@ -19109,24 +14603,15 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX900-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX900-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v3, v3, v2, s4
-; GFX900-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
 ; GFX900-NEXT:    s_movk_i32 s4, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_minimumnum_bf16_no_ieee:
@@ -19135,7 +14620,7 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
@@ -19145,10 +14630,6 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
@@ -19174,21 +14655,13 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX10-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee:
@@ -19197,39 +14670,31 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_bf16_no_ieee:
@@ -19248,26 +14713,14 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v2, v2, v2
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_minimumnum_bf16_no_ieee:
@@ -19280,45 +14733,37 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v1, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_bf16_no_ieee:
@@ -19344,29 +14789,17 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v2, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v2, v2, v2
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v2
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call bfloat @llvm.minimumnum.bf16(bfloat %x, bfloat %y)
   ret bfloat %result
@@ -19403,26 +14836,16 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v5
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT:    v_mul_f32_e32 v4, 1.0, v4
-; GFX8-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v4
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -19432,23 +14855,14 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19465,25 +14879,16 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v5
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX900-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX900-NEXT:    v_bfe_u32 v5, v4, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v5, v5, v4, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v4
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v4, v4
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -19493,21 +14898,13 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v2, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -19522,16 +14919,12 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v3, v0, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v2, v3, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
@@ -19555,9 +14948,6 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v4, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v1, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -19595,37 +14985,21 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v2, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc_lo
-; GFX10-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v4, v4, v4
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v7, v4, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v7, v7, v4, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v3, v6, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
 ; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19639,66 +15013,52 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v7
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v5, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16_no_ieee:
@@ -19711,60 +15071,39 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19782,71 +15121,56 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, s2
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v6, v7
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v5, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s1
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v7, v9, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v3.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.h, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v5.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16_no_ieee:
@@ -19864,69 +15188,48 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v3 :: v_dual_lshlrev_b32 v5, 16, v0
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v0 :: v_dual_lshlrev_b32 v4, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v4, v4, v4 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v3, v2 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v6, v4, 16, 1
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v1, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v8 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v5, v5, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_lshlrev_b32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v7, v0 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -19971,26 +15274,16 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -20000,23 +15293,14 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT:    v_mul_f32_e32 v5, 1.0, v5
-; GFX8-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -20026,22 +15310,13 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -20059,25 +15334,16 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -20087,22 +15353,14 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX900-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX900-NEXT:    v_bfe_u32 v6, v5, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v5, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v5
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -20112,21 +15370,13 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v4, v0, s4
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -20141,16 +15391,12 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v5, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -20174,9 +15420,6 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v5, v5, v5
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
@@ -20200,9 +15443,6 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v5, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -20220,180 +15460,125 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX10-LABEL: v_minimumnum_v3bf16_no_ieee:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v1
 ; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v5, v5
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cmp_u_f32_e64 s4, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_sdwa v0, v0, v6, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
-; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_sdwa v10, v0, v5, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v0, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v4
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v11
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v4, v0, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX10-NEXT:    v_or_b32_e32 v11, 0x400000, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v10, v7, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v10, v10, v7, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v12, v8, 16, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX10-NEXT:    v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v4, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v11
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
-; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-TRUE16-LABEL: v_minimumnum_v3bf16_no_ieee:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v5.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v7, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v9, v11
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_max_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -20402,88 +15587,59 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v9
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX11-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v7
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-TRUE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -20494,104 +15650,77 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v2
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v5, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.h, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v5.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s3, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s4, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v7, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v1.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s3, v9, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v8, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v6, v6
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v2.l, v0.l, s3
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v6, v9, v10, s5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v14, 0x400000, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_dual_max_num_f32 v7, v7, v7 :: v_dual_and_b32 v4, 0xffff0000, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s2
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v11, v11, v7, 0x7fff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s1
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v7, v11, v13, s5
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s5, v8, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v4
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v8, v12, v14, s5
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v6.l, v0.h, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v8.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v7.l, v1.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -20604,98 +15733,67 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v5, vcc_lo
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v2, v2, v0 :: v_dual_lshlrev_b32 v7, 16, v5
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v4 :: v_dual_lshlrev_b32 v6, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v8, 16, v0
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v7
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v9, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v1
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v6, v5, v4 :: v_dual_lshlrev_b32 v9, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v2, v0 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v9
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v6, v6, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v7, v7, v7
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v7, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v10, vcc_lo
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX12-FAKE16-NEXT:    v_add3_u32 v10, v11, v7, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v10, v9 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v12, v12, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v10
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v12, v11, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v10, 16, v6
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v7, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v7, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v2, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v9, v0 :: v_dual_and_b32 v5, 0xffff0000, v7
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v6
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v7
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v10, v1 :: v_dual_and_b32 v2, 0xffff0000, v8
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v5
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v6, v1 :: v_dual_lshlrev_b32 v2, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
@@ -20703,7 +15801,7 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x bfloat> @llvm.minimumnum.v3bf16(<3 x bfloat> %x, <3 x bfloat> %y)
   ret <3 x bfloat> %result
@@ -20752,26 +15850,16 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX8-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    s_movk_i32 s5, 0x8000
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -20783,23 +15871,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v8
 ; GFX8-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT:    v_mul_f32_e32 v7, 1.0, v7
-; GFX8-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v7
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s4, v8
-; GFX8-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v6
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -20809,23 +15888,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT:    v_mul_f32_e32 v6, 1.0, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -20835,22 +15905,13 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT:    v_mul_f32_e32 v3, 1.0, v3
-; GFX8-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
@@ -20870,25 +15931,16 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
+; GFX900-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    s_movk_i32 s5, 0x8000
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
 ; GFX900-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v5
 ; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v5, v5
@@ -20900,22 +15952,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v6
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v8
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX900-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX900-NEXT:    v_bfe_u32 v8, v7, 16, 1
-; GFX900-NEXT:    v_add3_u32 v8, v8, v7, s4
-; GFX900-NEXT:    v_or_b32_e32 v9, 0x400000, v7
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
-; GFX900-NEXT:    v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v5
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v6
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v5
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v6, 0xffff0000, v7
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
@@ -20925,22 +15969,14 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX900-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v6
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -20950,21 +15986,13 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v3
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX900-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX900-NEXT:    v_bfe_u32 v6, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v6, v6, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX900-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v2
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX900-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX900-NEXT:    v_perm_b32 v0, v5, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v4, v1, s4
@@ -20980,17 +16008,13 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cndmask_b32_sdwa v5, v1, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v7, v7
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
-; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v2
+; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v7
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v2
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v4, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
-; GFX950-NEXT:    s_movk_i32 s0, 0x8000
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -21014,9 +16038,6 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v8
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v7, v6, v5, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_max_f32_e32 v7, v7, v7
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v5
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
@@ -21040,9 +16061,6 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v7, v6
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, v3, v1, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v1
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
@@ -21066,9 +16084,6 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v6, v3
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v2, v0, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v3, v3, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
 ; GFX950-NEXT:    v_cmp_eq_u16_e32 vcc, s0, v0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
@@ -21113,78 +16128,46 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v8, v5, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX10-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v8, v10, vcc_lo
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX10-NEXT:    v_bfe_u32 v14, v8, 16, 1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v8
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v12
-; GFX10-NEXT:    v_lshlrev_b32_e32 v12, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v7, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v13
-; GFX10-NEXT:    v_or_b32_e32 v13, 0x400000, v8
-; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v6, v12
-; GFX10-NEXT:    v_add3_u32 v12, v14, v8, 0x7fff
-; GFX10-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX10-NEXT:    v_or_b32_e32 v14, 0x400000, v11
-; GFX10-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v12, v13, vcc_lo
-; GFX10-NEXT:    v_bfe_u32 v12, v11, 16, 1
-; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v10
-; GFX10-NEXT:    v_bfe_u32 v15, v9, 16, 1
-; GFX10-NEXT:    v_max_f32_e32 v6, v6, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX10-NEXT:    v_add3_u32 v12, v12, v11, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT:    v_or_b32_e32 v16, 0x400000, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX10-NEXT:    v_bfe_u32 v12, v6, 16, 1
-; GFX10-NEXT:    v_add3_u32 v14, v15, v9, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v10, v5, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX10-NEXT:    v_add3_u32 v12, v12, v6, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v11, 0xffff0000, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v12, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v13, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v6
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v7
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
@@ -21203,232 +16186,162 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v10, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v12
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v9, v13
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v11, v11, v11
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v9, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX11-TRUE16-NEXT:    v_max_f32_e32 v9, v9, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v11, v12
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v7.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
 ; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v11
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v10
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v8, v8, v8
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v13
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT:    v_dual_max_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v12, v12, v12
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_max_f32_e32 v10, v10, v10
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX11-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX11-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX11-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX11-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
-; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v8
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
-; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v0, v5, v0, 0x5040100
 ; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_perm_b32 v1, v4, v1, 0x5040100
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -21446,123 +16359,94 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v8, v8
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s4, v10, v10
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s2
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v11, v11
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v4.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v3.l, v3.l, v1.l, s3
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v10, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v2.l
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v2.l
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s2, v9, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v12
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v8.l, v4.l, s1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v13
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v11, v12
-; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
-; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v9, v13
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v15
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v8.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v9.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v10, 16, 1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v13, 0x400000, v10
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v11, v11, v11
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT:    v_add3_u32 v12, v12, v10, 0x7fff
-; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v14, v11, 16, 1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v7.l, v6.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v5.l
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v10, v12, v13, s1
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v11
-; GFX12-TRUE16-NEXT:    v_add3_u32 v13, v14, v11, 0x7fff
-; GFX12-TRUE16-NEXT:    v_max_num_f32_e32 v9, v9, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v8.l
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v11, v12
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v7.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v10.h, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v10
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v13
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v14
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v5.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v3.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v13, v12, vcc_lo
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v12, v8, 16, 1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    v_bfe_u32 v4, v9, 16, 1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v11.h, v6.l, s2
-; GFX12-TRUE16-NEXT:    v_add3_u32 v5, v12, v8, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v8
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT:    v_add3_u32 v4, v4, v9, 0x7fff
-; GFX12-TRUE16-NEXT:    v_or_b32_e32 v12, 0x400000, v9
-; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s0
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v10
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v11
-; GFX12-TRUE16-NEXT:    v_cndmask_b32_e64 v4, v4, v12, s2
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v2.h, v1.h, v7.l, s2
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s2
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v6
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s0
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s2, 0, v7
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s1
-; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v3.l, s1
+; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s1, 0, v10
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v8.l, v0.h, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.h, s2
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.h, v10.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v11.h, v2.h, s2
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v4.h, v0.l, s0
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.h, v1.l, s1
+; GFX12-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v1.l, s1
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
@@ -21572,126 +16456,80 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX12-FAKE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v1
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v6, v5 :: v_dual_and_b32 v7, 0xffff0000, v3
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v9, v8, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v11
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v8, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v10
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v10, v5, v4 :: v_dual_and_b32 v9, 0xffff0000, v2
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v7
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_lshlrev_b32 v10, 16, v6
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v8, v8, v8
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v10, v13
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v1
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v11, 0x400000, v8
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v13, 16, v3
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v9, v8, 16, 1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_add3_u32 v9, v9, v8, 0x7fff
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v7, v7, v6 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v3, v3, v1 :: v_dual_lshlrev_b32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v12, 16, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v4, v5 :: v_dual_lshlrev_b32 v9, 16, v7
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v15, 16, v0
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v2
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v9, v11, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v13, v12
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v0
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v3, v1, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v8
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v12, v2, v0 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4
-; GFX12-FAKE16-NEXT:    v_dual_max_num_f32 v9, v9, v9 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v4, v10, v4 :: v_dual_lshlrev_b32 v5, 16, v6
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v13, v13
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v15, 0x400000, v9
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v12, v12, v12
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v16, 0x400000, v12
-; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-FAKE16-NEXT:    v_max_num_f32_e32 v10, v10, v10
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v10, 16, 1
-; GFX12-FAKE16-NEXT:    v_or_b32_e32 v14, 0x400000, v10
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v10, 0x7fff
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v11, v9, 16, 1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6
-; GFX12-FAKE16-NEXT:    v_bfe_u32 v5, v12, 16, 1
-; GFX12-FAKE16-NEXT:    v_add3_u32 v11, v11, v9, 0x7fff
-; GFX12-FAKE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v14, v6, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-FAKE16-NEXT:    v_add3_u32 v5, v5, v12, 0x7fff
-; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v9, v11, v15, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v5, v6, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v9, v8
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc_lo
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v8, v3, v1, vcc_lo
+; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v2
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc_lo
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v7, v2, v0, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1
-; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0
@@ -21702,10 +16540,10 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v7
+; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v9
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v14, v6 :: v_dual_and_b32 v2, 0xffff0000, v9
+; GFX12-FAKE16-NEXT:    v_dual_cndmask_b32 v5, v5, v6 :: v_dual_lshlrev_b32 v2, 16, v8
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-fake16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-fake16.ll
new file mode 100644
index 000000000000..a6819359561b
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-fake16.ll
@@ -0,0 +1,113 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck %s
+
+define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: exp_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[V_EXP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_EXP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_EXP_F16_fake16_e64_]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.exp2.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: log_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[V_LOG_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_LOG_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_LOG_F16_fake16_e64_]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.log.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: rcp_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[V_RCP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_RCP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_RCP_F16_fake16_e64_]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.rcp.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: rsq_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[V_RSQ_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_RSQ_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_RSQ_F16_fake16_e64_]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.rsq.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: sqrt_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[V_SQRT_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_SQRT_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_SQRT_F16_fake16_e64_]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.sqrt.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+declare half @llvm.amdgcn.exp2.f16(half)
+declare half @llvm.amdgcn.log.f16(half)
+declare half @llvm.amdgcn.rcp.f16(half)
+declare half @llvm.amdgcn.rsq.f16(half)
+declare half @llvm.amdgcn.sqrt.f16(half)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
new file mode 100644
index 000000000000..b1b5b6b62296
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck %s
+
+define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: exp_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+  ; CHECK-NEXT:   [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_EXP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.exp2.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: log_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+  ; CHECK-NEXT:   [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_LOG_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.log.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: rcp_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+  ; CHECK-NEXT:   [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_RCP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.rcp.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: rsq_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+  ; CHECK-NEXT:   [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_RSQ_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.rsq.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
+  ; CHECK-LABEL: name: sqrt_f16
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+  ; CHECK-NEXT:   [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_SQRT_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %val = load volatile half, ptr addrspace(1) %ptr
+  %res = call half @llvm.amdgcn.sqrt.f16(half %val)
+  store half %res, ptr addrspace(1) %ptr
+  ret void
+}
+
+declare half @llvm.amdgcn.exp2.f16(half)
+declare half @llvm.amdgcn.log.f16(half)
+declare half @llvm.amdgcn.rcp.f16(half)
+declare half @llvm.amdgcn.rsq.f16(half)
+declare half @llvm.amdgcn.sqrt.f16(half)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans.ll
index 9407c8a9ee43..56848eabf860 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck %s
 
 define amdgpu_kernel void @exp_f32(ptr addrspace(1) %ptr) {
@@ -21,27 +21,6 @@ define amdgpu_kernel void @exp_f32(ptr addrspace(1) %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
-  ; CHECK-LABEL: name: exp_f16
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
-  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
-  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_EXP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_EXP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_EXP_F16_fake16_e64_]]
-  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   S_ENDPGM 0
-  %val = load volatile half, ptr addrspace(1) %ptr
-  %res = call half @llvm.amdgcn.exp2.f16(half %val)
-  store half %res, ptr addrspace(1) %ptr
-  ret void
-}
-
 define amdgpu_kernel void @log_f32(ptr addrspace(1) %ptr) {
   ; CHECK-LABEL: name: log_f32
   ; CHECK: bb.0 (%ir-block.0):
@@ -62,27 +41,6 @@ define amdgpu_kernel void @log_f32(ptr addrspace(1) %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
-  ; CHECK-LABEL: name: log_f16
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
-  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
-  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_LOG_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_LOG_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_LOG_F16_fake16_e64_]]
-  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   S_ENDPGM 0
-  %val = load volatile half, ptr addrspace(1) %ptr
-  %res = call half @llvm.amdgcn.log.f16(half %val)
-  store half %res, ptr addrspace(1) %ptr
-  ret void
-}
-
 define amdgpu_kernel void @rcp_f32(ptr addrspace(1) %ptr) {
   ; CHECK-LABEL: name: rcp_f32
   ; CHECK: bb.0 (%ir-block.0):
@@ -103,27 +61,6 @@ define amdgpu_kernel void @rcp_f32(ptr addrspace(1) %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
-  ; CHECK-LABEL: name: rcp_f16
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
-  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
-  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_RCP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_RCP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_RCP_F16_fake16_e64_]]
-  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   S_ENDPGM 0
-  %val = load volatile half, ptr addrspace(1) %ptr
-  %res = call half @llvm.amdgcn.rcp.f16(half %val)
-  store half %res, ptr addrspace(1) %ptr
-  ret void
-}
-
 define amdgpu_kernel void @rsq_f32(ptr addrspace(1) %ptr) {
   ; CHECK-LABEL: name: rsq_f32
   ; CHECK: bb.0 (%ir-block.0):
@@ -144,27 +81,6 @@ define amdgpu_kernel void @rsq_f32(ptr addrspace(1) %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
-  ; CHECK-LABEL: name: rsq_f16
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
-  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
-  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_RSQ_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_RSQ_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_RSQ_F16_fake16_e64_]]
-  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   S_ENDPGM 0
-  %val = load volatile half, ptr addrspace(1) %ptr
-  %res = call half @llvm.amdgcn.rsq.f16(half %val)
-  store half %res, ptr addrspace(1) %ptr
-  ret void
-}
-
 define amdgpu_kernel void @sqrt_f32(ptr addrspace(1) %ptr) {
   ; CHECK-LABEL: name: sqrt_f32
   ; CHECK: bb.0 (%ir-block.0):
@@ -185,34 +101,8 @@ define amdgpu_kernel void @sqrt_f32(ptr addrspace(1) %ptr) {
   ret void
 }
 
-define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
-  ; CHECK-LABEL: name: sqrt_f16
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
-  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
-  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_SQRT_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_SQRT_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_SQRT_F16_fake16_e64_]]
-  ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
-  ; CHECK-NEXT:   S_ENDPGM 0
-  %val = load volatile half, ptr addrspace(1) %ptr
-  %res = call half @llvm.amdgcn.sqrt.f16(half %val)
-  store half %res, ptr addrspace(1) %ptr
-  ret void
-}
-
 declare float @llvm.amdgcn.exp2.f32(float)
-declare half @llvm.amdgcn.exp2.f16(half)
 declare float @llvm.amdgcn.log.f32(float)
-declare half @llvm.amdgcn.log.f16(half)
 declare float @llvm.amdgcn.rcp.f32(float)
-declare half @llvm.amdgcn.rcp.f16(half)
 declare float @llvm.amdgcn.rsq.f32(float)
-declare half @llvm.amdgcn.rsq.f16(half)
 declare float @llvm.amdgcn.sqrt.f32(float)
-declare half @llvm.amdgcn.sqrt.f16(half)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/mul.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/mul.ll
index 0f47a31f52dc..b5e7589cbd13 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -2689,45 +2689,45 @@ define amdgpu_kernel void @s_mul_i128(ptr addrspace(1) %out, [8 x i32], i128 %a,
 ;
 ; GFX9-LABEL: s_mul_i128:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x4c
-; GFX9-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x7c
+; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x7c
+; GFX9-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x4c
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mul_i32 s4, s12, s11
-; GFX9-NEXT:    s_mul_hi_u32 s5, s12, s10
-; GFX9-NEXT:    s_mul_i32 s6, s14, s9
-; GFX9-NEXT:    s_mul_hi_u32 s7, s14, s8
+; GFX9-NEXT:    s_mul_i32 s4, s8, s15
+; GFX9-NEXT:    s_mul_hi_u32 s5, s8, s14
+; GFX9-NEXT:    s_mul_i32 s6, s10, s13
+; GFX9-NEXT:    s_mul_hi_u32 s7, s10, s12
 ; GFX9-NEXT:    s_add_i32 s4, s5, s4
-; GFX9-NEXT:    s_mul_i32 s5, s13, s10
+; GFX9-NEXT:    s_mul_i32 s5, s9, s14
 ; GFX9-NEXT:    s_add_i32 s6, s7, s6
-; GFX9-NEXT:    s_mul_i32 s7, s15, s8
+; GFX9-NEXT:    s_mul_i32 s7, s11, s12
 ; GFX9-NEXT:    s_add_i32 s4, s4, s5
-; GFX9-NEXT:    s_mul_i32 s5, s12, s10
+; GFX9-NEXT:    s_mul_i32 s5, s8, s14
 ; GFX9-NEXT:    s_add_i32 s6, s6, s7
-; GFX9-NEXT:    s_mul_i32 s7, s14, s8
+; GFX9-NEXT:    s_mul_i32 s7, s10, s12
 ; GFX9-NEXT:    s_add_u32 s7, s7, s5
 ; GFX9-NEXT:    s_addc_u32 s6, s6, s4
-; GFX9-NEXT:    s_mul_i32 s14, s9, s12
-; GFX9-NEXT:    s_mul_hi_u32 s15, s8, s12
-; GFX9-NEXT:    s_mul_hi_u32 s11, s9, s12
+; GFX9-NEXT:    s_mul_i32 s14, s13, s8
+; GFX9-NEXT:    s_mul_hi_u32 s15, s12, s8
+; GFX9-NEXT:    s_mul_hi_u32 s11, s13, s8
 ; GFX9-NEXT:    s_add_u32 s14, s14, s15
-; GFX9-NEXT:    s_mul_i32 s5, s8, s13
+; GFX9-NEXT:    s_mul_i32 s5, s12, s9
 ; GFX9-NEXT:    s_addc_u32 s11, s11, 0
-; GFX9-NEXT:    s_mul_hi_u32 s10, s8, s13
+; GFX9-NEXT:    s_mul_hi_u32 s10, s12, s9
 ; GFX9-NEXT:    s_add_u32 s5, s5, s14
 ; GFX9-NEXT:    s_addc_u32 s10, s10, 0
 ; GFX9-NEXT:    s_add_u32 s10, s11, s10
 ; GFX9-NEXT:    s_addc_u32 s11, 0, 0
-; GFX9-NEXT:    s_mul_hi_u32 s14, s9, s13
-; GFX9-NEXT:    s_mul_i32 s9, s9, s13
+; GFX9-NEXT:    s_mul_hi_u32 s14, s13, s9
+; GFX9-NEXT:    s_mul_i32 s9, s13, s9
 ; GFX9-NEXT:    s_add_u32 s9, s9, s10
 ; GFX9-NEXT:    s_addc_u32 s10, s14, s11
 ; GFX9-NEXT:    s_mov_b32 s4, 0
 ; GFX9-NEXT:    s_add_u32 s9, s9, s7
 ; GFX9-NEXT:    s_addc_u32 s10, s10, s6
-; GFX9-NEXT:    s_mul_i32 s6, s8, s12
+; GFX9-NEXT:    s_mul_i32 s6, s12, s8
 ; GFX9-NEXT:    s_mov_b32 s7, s4
 ; GFX9-NEXT:    s_or_b64 s[4:5], s[6:7], s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/or.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/or.ll
index cc9650b9a730..1abd2e6b60f2 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/or.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/or.ll
@@ -355,15 +355,15 @@ define amdgpu_kernel void @scalar_or_literal_multi_use_i64(ptr addrspace(1) %out
 ;
 ; GFX8-LABEL: scalar_or_literal_multi_use_i64:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x74
 ; GFX8-NEXT:    s_movk_i32 s8, 0x3039
 ; GFX8-NEXT:    s_mov_b32 s9, 0xf237b
-; GFX8-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s2, -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index b4e5fa088b53..a4ddfee115fa 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -3656,14 +3656,14 @@ define hidden void @extract_v6i16(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
 ; GFX9-LABEL: extract_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_ushort v2, v[0:1], off offset:6
-; GFX9-NEXT:    global_load_ushort v3, v[0:1], off
-; GFX9-NEXT:    global_load_ushort v8, v[0:1], off offset:4
+; GFX9-NEXT:    global_load_ushort v2, v[0:1], off offset:4
+; GFX9-NEXT:    global_load_ushort v3, v[0:1], off offset:6
+; GFX9-NEXT:    global_load_ushort v8, v[0:1], off
 ; GFX9-NEXT:    global_load_ushort v9, v[0:1], off offset:2
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v8
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_lshl_or_b32 v0, v3, 16, v2
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_lshl_or_b32 v1, v9, 16, v3
+; GFX9-NEXT:    v_lshl_or_b32 v1, v9, 16, v8
 ; GFX9-NEXT:    global_store_dword v[4:5], v1, off
 ; GFX9-NEXT:    global_store_dword v[6:7], v0, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/pr51516.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/pr51516.mir
index cc299789af8d..69983faf2b15 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/pr51516.mir
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/pr51516.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -amdgpu-disable-unclustered-high-rp-reschedule -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN %s
 # RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -amdgpu-disable-unclustered-high-rp-reschedule -amdgpu-use-amdgpu-trackers=1 -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
 
@@ -116,3 +117,6 @@ body:             |
     S_ENDPGM 0
 
 ...
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# GCN: {{.*}}
+# GCN-GCNTRACKER: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info-multi-entry.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info-multi-entry.ll
index f9fb1e639b24..dc0eb0f58bb4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info-multi-entry.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info-multi-entry.ll
@@ -1,4 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; XFAIL: *
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX942 %s
 
 define amdgpu_kernel void @preload_block_count_x(ptr addrspace(1) inreg noundef %dst.coerce, ptr addrspace(1) inreg noundef %src.coerce, i64 inreg noundef %nElts, i64 inreg noundef %redOpArg, i1 inreg noundef %redOpArgIsPtr) #0 !dbg !4 {
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info.ll
index 1055abe6d349..0f9a08a85a8c 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs-debug-info.ll
@@ -28,6 +28,6 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 1, type: !5, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
-!5 = !DISubroutineType(cc: DW_CC_LLVM_OpenCLKernel, types: !6)
+!5 = !DISubroutineType(cc: DW_CC_LLVM_DeviceKernel, types: !6)
 !6 = !{null}
 !7 = !{i32 1024, i32 1, i32 1}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
index b1fc76f457ec..93f43b274e28 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
@@ -2,16 +2,22 @@
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto<O1>" -print-pipeline-passes %s -o - | FileCheck %s
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto<O2>" -print-pipeline-passes %s -o - | FileCheck %s
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto<O3>" -print-pipeline-passes %s -o - | FileCheck %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O0 -print-pipeline-passes %s -o - | FileCheck --check-prefix=O0 %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O1 -print-pipeline-passes %s -o - | FileCheck %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O2 -print-pipeline-passes %s -o - | FileCheck %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -print-pipeline-passes %s -o - | FileCheck %s
 
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto-pre-link<O0>" -print-pipeline-passes -amdgpu-internalize-symbols %s -o - | FileCheck --check-prefix=PRE %s
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto-pre-link<O1>" -print-pipeline-passes -amdgpu-internalize-symbols %s -o - | FileCheck --check-prefix=PRE %s
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto-pre-link<O2>" -print-pipeline-passes -amdgpu-internalize-symbols %s -o - | FileCheck --check-prefix=PRE %s
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto-pre-link<O3>" -print-pipeline-passes -amdgpu-internalize-symbols %s -o - | FileCheck --check-prefix=PRE %s
 
-
+; CHECK: amdgpu-expand-feature-predicates
 ; CHECK: amdgpu-attributor
+; O0: amdgpu-expand-feature-predicates
 ; O0-NOT: amdgpu-attributor
 
+; PRE: amdgpu-expand-feature-predicates
 ; PRE-NOT: internalize
 ; PRE-NOT: amdgpu-attributor
 ; PRE-NOT: printfToRuntime
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-gep-of-gep.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-gep-of-gep.ll
new file mode 100644
index 000000000000..a24f041a1785
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-gep-of-gep.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca < %s | FileCheck %s
+define amdgpu_ps void @scalar_alloca_ptr_with_vector_gep_of_gep(i32 %idx, ptr addrspace(1) %output) #0 {
+; CHECK-LABEL: define amdgpu_ps void @scalar_alloca_ptr_with_vector_gep_of_gep(
+; CHECK-SAME: i32 [[IDX:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[BUF:%.*]] = freeze <20 x i32> poison
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[IDX]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <20 x i32> [[BUF]], i32 1, i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <20 x i32> [[TMP1]], i32 2, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[IDX]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 1, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <20 x i32> [[TMP3]], i32 [[TMP5]]
+; CHECK-NEXT:    store i32 [[TMP6]], ptr addrspace(1) [[OUTPUT]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %alloca = alloca [10 x <2 x i32>], align 8, addrspace(5)
+  %row = getelementptr [10 x <2 x i32>], ptr addrspace(5) %alloca, i32 0, i32 %idx
+  store <2 x i32> <i32 1, i32 2>, ptr addrspace(5) %row, align 8
+  %elt1 = getelementptr i8, ptr addrspace(5) %row, i32 4
+  %val = load i32, ptr addrspace(5) %elt1, align 4
+  store i32 %val, ptr addrspace(1) %output
+  ret void
+}
+
+define amdgpu_ps void @scalar_alloca_ptr_with_vector_gep_of_gep3(i32 %idx, ptr addrspace(1) %output) #0 {
+; CHECK-LABEL: define amdgpu_ps void @scalar_alloca_ptr_with_vector_gep_of_gep3(
+; CHECK-SAME: i32 [[IDX:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[ALLOCA:%.*]] = freeze <16 x i32> poison
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[IDX]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 8, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <16 x i32> [[ALLOCA]], i32 10, i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP2]], i32 20, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[IDX]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 9, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP4]], i32 [[TMP6]]
+; CHECK-NEXT:    store i32 [[TMP7]], ptr addrspace(1) [[OUTPUT]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %alloca = alloca [2 x [4 x <2 x i32>]], align 8, addrspace(5)
+  %lvl1 = getelementptr inbounds [2 x [4 x <2 x i32>]], ptr addrspace(5) %alloca, i32 0, i32 1
+  %lvl2 = getelementptr inbounds [4 x <2 x i32>], ptr addrspace(5) %lvl1, i32 0, i32 %idx
+  store <2 x i32> <i32 10, i32 20>, ptr addrspace(5) %lvl2, align 8
+  %byte = getelementptr inbounds i8, ptr addrspace(5) %lvl2, i32 4
+  %val = load i32, ptr addrspace(5) %byte, align 4
+  store i32 %val, ptr addrspace(1) %output
+  ret void
+}
+
+define amdgpu_ps void @scalar_alloca_ptr_with_vector_gep_twice_idx(i32 %idx, ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_ps void @scalar_alloca_ptr_with_vector_gep_twice_idx(
+; CHECK-SAME: i32 [[IDX:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[BUF:%.*]] = freeze <20 x i32> poison
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[IDX]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <20 x i32> [[BUF]], i32 1, i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <20 x i32> [[TMP4]], i32 2, i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <20 x i32> [[TMP3]], i32 [[TMP1]]
+; CHECK-NEXT:    store i32 [[TMP2]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %alloca = alloca [10 x [2 x i32]], align 8, addrspace(5)
+  %row = getelementptr inbounds [10 x [2 x i32]], ptr addrspace(5) %alloca, i32 0, i32 %idx
+  store <2 x i32> <i32 1, i32 2>, ptr addrspace(5) %row, align 8
+  %elt = getelementptr inbounds [2 x i32], ptr addrspace(5) %row, i32 0, i32 %idx
+  %val = load i32, ptr addrspace(5) %elt, align 4
+  store i32 %val, ptr addrspace(1) %out
+  ret void
+}
+
+attributes #0 = { "amdgpu-promote-alloca-to-vector-max-regs"="32" }
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index b1a618ae4d5c..c4842c1f4f52 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -1193,9 +1193,9 @@ define amdgpu_kernel void @Address32(ptr addrspace(1) %buffer) {
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 2
-; GFX10-NEXT:    v_and_b32_e32 v8, 0xffff8000, v1
+; GFX10-NEXT:    v_and_b32_e32 v10, 0xffff8000, v1
 ; GFX10-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT:    v_add_co_u32 v1, s0, s34, v8
+; GFX10-NEXT:    v_add_co_u32 v1, s0, s34, v10
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v2, s0, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v1, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
@@ -1203,38 +1203,38 @@ define amdgpu_kernel void @Address32(ptr addrspace(1) %buffer) {
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 0x1000
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    s_clause 0x3
+; GFX10-NEXT:    global_load_dword v11, v[0:1], off
+; GFX10-NEXT:    global_load_dword v12, v[0:1], off offset:1024
+; GFX10-NEXT:    global_load_dword v13, v[4:5], off offset:-2048
+; GFX10-NEXT:    global_load_dword v14, v[2:3], off offset:1024
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, 0x1000, v0
-; GFX10-NEXT:    s_clause 0x4
-; GFX10-NEXT:    global_load_dword v9, v[0:1], off
-; GFX10-NEXT:    global_load_dword v10, v[0:1], off offset:1024
-; GFX10-NEXT:    global_load_dword v11, v[2:3], off offset:1024
-; GFX10-NEXT:    global_load_dword v12, v[4:5], off offset:-2048
-; GFX10-NEXT:    global_load_dword v13, v[4:5], off
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, 0x1800, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 0x2000
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    global_load_dword v14, v[6:7], off offset:1024
-; GFX10-NEXT:    global_load_dword v15, v[2:3], off offset:1024
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v0, 0x2000
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    s_clause 0x2
+; GFX10-NEXT:    global_load_dword v15, v[4:5], off
+; GFX10-NEXT:    global_load_dword v16, v[6:7], off offset:1024
+; GFX10-NEXT:    global_load_dword v17, v[2:3], off offset:1024
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, 0x2000, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    s_clause 0x2
-; GFX10-NEXT:    global_load_dword v2, v[4:5], off offset:-2048
-; GFX10-NEXT:    global_load_dword v3, v[4:5], off
-; GFX10-NEXT:    global_load_dword v6, v[0:1], off offset:1024
+; GFX10-NEXT:    global_load_dword v2, v[8:9], off offset:-2048
+; GFX10-NEXT:    global_load_dword v3, v[8:9], off
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off offset:1024
 ; GFX10-NEXT:    s_waitcnt vmcnt(8)
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v10, v9
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v12, v11
 ; GFX10-NEXT:    s_waitcnt vmcnt(6)
-; GFX10-NEXT:    v_add3_u32 v0, v12, v0, v11
-; GFX10-NEXT:    s_waitcnt vmcnt(4)
 ; GFX10-NEXT:    v_add3_u32 v0, v13, v0, v14
+; GFX10-NEXT:    s_waitcnt vmcnt(4)
+; GFX10-NEXT:    v_add3_u32 v0, v15, v0, v16
 ; GFX10-NEXT:    s_waitcnt vmcnt(2)
-; GFX10-NEXT:    v_add3_u32 v0, v2, v0, v15
+; GFX10-NEXT:    v_add3_u32 v0, v2, v0, v17
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add3_u32 v0, v3, v0, v6
-; GFX10-NEXT:    global_store_dword v8, v0, s[34:35]
+; GFX10-NEXT:    v_add3_u32 v0, v3, v0, v4
+; GFX10-NEXT:    global_store_dword v10, v0, s[34:35]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: Address32:
@@ -1375,19 +1375,19 @@ define amdgpu_kernel void @Offset64(ptr addrspace(1)  %buffer) {
 ; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v1, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, 0, v2, vcc
 ; GFX8-NEXT:    s_movk_i32 s0, 0xf000
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s0, v3
-; GFX8-NEXT:    v_addc_u32_e32 v8, vcc, 0, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s0, v3
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v4, vcc
 ; GFX8-NEXT:    s_movk_i32 s0, 0xf800
-; GFX8-NEXT:    flat_load_dwordx2 v[5:6], v[3:4]
-; GFX8-NEXT:    flat_load_dwordx2 v[7:8], v[7:8]
+; GFX8-NEXT:    flat_load_dwordx2 v[7:8], v[3:4]
+; GFX8-NEXT:    flat_load_dwordx2 v[5:6], v[5:6]
 ; GFX8-NEXT:    v_add_u32_e32 v9, vcc, s0, v3
 ; GFX8-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
 ; GFX8-NEXT:    flat_load_dwordx2 v[9:10], v[9:10]
 ; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v4
 ; GFX8-NEXT:    flat_load_dwordx2 v[3:4], v[3:4]
 ; GFX8-NEXT:    s_waitcnt vmcnt(2)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v7, v5
-; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v8, v6, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v5, v7
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v6, v8, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(1)
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v9, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v10, v5, vcc
@@ -1429,14 +1429,14 @@ define amdgpu_kernel void @Offset64(ptr addrspace(1)  %buffer) {
 ; GFX9-NEXT:    s_movk_i32 s0, 0xf000
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
-; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[2:3], off
 ; GFX9-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off offset:2048
 ; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
 ; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v4, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v7, v5, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v8, v2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v9, v3, vcc
@@ -1521,15 +1521,15 @@ define amdgpu_kernel void @Offset64(ptr addrspace(1)  %buffer) {
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, 0xfffff000, v0
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
 ; GFX11-NEXT:    s_clause 0x2
-; GFX11-NEXT:    global_load_b64 v[4:5], v[2:3], off
-; GFX11-NEXT:    global_load_b64 v[6:7], v[0:1], off
+; GFX11-NEXT:    global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT:    global_load_b64 v[6:7], v[2:3], off
 ; GFX11-NEXT:    global_load_b64 v[2:3], v[2:3], off offset:2048
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 1, v1
 ; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off
 ; GFX11-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v6
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v6, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, v5, v7, vcc_lo
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, v7, v5, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v3, v5, vcc_lo
@@ -1686,26 +1686,26 @@ define amdgpu_kernel void @p32Offset64(ptr addrspace(1)  %buffer) {
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 7, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 2
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff8000, v1
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xffff8000, v1
 ; GFX10-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT:    v_add_co_u32 v1, s0, s34, v4
+; GFX10-NEXT:    v_add_co_u32 v1, s0, s34, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v2, s0, s35, 0, s0
 ; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v1, v0
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 0x80000000
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT:    global_load_dword v5, v[0:1], off
-; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, 0x7ffff800, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT:    s_clause 0x2
-; GFX10-NEXT:    global_load_dword v6, v[2:3], off offset:-2048
-; GFX10-NEXT:    global_load_dword v7, v[2:3], off
-; GFX10-NEXT:    global_load_dword v8, v[0:1], off offset:1024
+; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, 0x7ffff800, v0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    s_clause 0x3
+; GFX10-NEXT:    global_load_dword v7, v[0:1], off
+; GFX10-NEXT:    global_load_dword v8, v[2:3], off offset:-2048
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    global_load_dword v10, v[4:5], off offset:1024
 ; GFX10-NEXT:    s_waitcnt vmcnt(2)
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v6, v5
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v8, v7
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add3_u32 v0, v8, v0, v7
-; GFX10-NEXT:    global_store_dword v4, v0, s[34:35]
+; GFX10-NEXT:    v_add3_u32 v0, v10, v0, v9
+; GFX10-NEXT:    global_store_dword v6, v0, s[34:35]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: p32Offset64:
@@ -2160,25 +2160,25 @@ define amdgpu_kernel void @ReverseOrder(ptr addrspace(1) %buffer) {
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v2, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    s_movk_i32 s0, 0x3000
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v0
-; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[0:1], off
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[4:5], off offset:2048
-; GFX9-NEXT:    global_load_dwordx2 v[8:9], v[4:5], off
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[2:3], off offset:2048
+; GFX9-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off
 ; GFX9-NEXT:    s_movk_i32 s0, 0x2000
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v0
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX9-NEXT:    global_load_dwordx2 v[10:11], v[4:5], off offset:2048
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dwordx2 v[10:11], v[2:3], off offset:2048
 ; GFX9-NEXT:    s_movk_i32 s0, 0x1000
 ; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, s0, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v1, vcc
 ; GFX9-NEXT:    global_load_dwordx2 v[14:15], v[12:13], off
-; GFX9-NEXT:    global_load_dwordx2 v[16:17], v[4:5], off
+; GFX9-NEXT:    global_load_dwordx2 v[16:17], v[2:3], off
 ; GFX9-NEXT:    global_load_dwordx2 v[18:19], v[12:13], off offset:2048
 ; GFX9-NEXT:    global_load_dwordx2 v[20:21], v[0:1], off offset:2048
 ; GFX9-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v7, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v7, v5, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(5)
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v8, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v9, v1, vcc
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll
new file mode 100644
index 000000000000..653d4b85a9a5
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll
@@ -0,0 +1,537 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -amdgpu-use-sdag-ptradd=1 < %s | FileCheck %s -check-prefixes=GFX8,GFX8_PTRADD
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -amdgpu-use-sdag-ptradd=0 < %s | FileCheck %s -check-prefixes=GFX8,GFX8_LEGACY
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -amdgpu-use-sdag-ptradd=1 < %s | FileCheck %s -check-prefixes=GFX942,GFX942_PTRADD
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -amdgpu-use-sdag-ptradd=0 < %s | FileCheck %s -check-prefixes=GFX942,GFX942_LEGACY
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -amdgpu-use-sdag-ptradd=1 < %s | FileCheck %s -check-prefixes=GFX10,GFX10_PTRADD
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -amdgpu-use-sdag-ptradd=0 < %s | FileCheck %s -check-prefixes=GFX10,GFX10_LEGACY
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -amdgpu-use-sdag-ptradd=1 < %s | FileCheck %s -check-prefixes=GFX11,GFX11_PTRADD
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -amdgpu-use-sdag-ptradd=0 < %s | FileCheck %s -check-prefixes=GFX11,GFX11_LEGACY
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -amdgpu-use-sdag-ptradd=1 < %s | FileCheck %s -check-prefixes=GFX12,GFX12_PTRADD
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -amdgpu-use-sdag-ptradd=0 < %s | FileCheck %s -check-prefixes=GFX12,GFX12_LEGACY
+
+; Tests for the ISD::PTRADD SelectionDAG opcode. This only tests 64-bit address
+; spaces since PTRADD is currently only used for these.
+
+; Check that basic pointer arithmetic can be lowered.
+define ptr @gep_as0(ptr %p, i64 %offset) {
+; GFX8-LABEL: gep_as0:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 5, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_PTRADD-LABEL: gep_as0:
+; GFX942_PTRADD:       ; %bb.0: ; %entry
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 5
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: gep_as0:
+; GFX942_LEGACY:       ; %bb.0: ; %entry
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 5
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: gep_as0:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: gep_as0:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: gep_as0:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %gep1 = getelementptr inbounds i32, ptr %p, i64 %offset
+  %gep2 = getelementptr inbounds i8, ptr %gep1, i64 5
+  ret ptr %gep2
+}
+
+define amdgpu_kernel void @gep_as0_uniform(ptr %p, i64 %offset, ptr %ret) {
+; GFX8-LABEL: gep_as0_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX8-NEXT:    s_add_u32 s0, s0, s2
+; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    s_add_u32 s0, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    s_addc_u32 s1, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    flat_store_dword v[2:3], v5
+; GFX8-NEXT:    flat_store_dword v[0:1], v4
+; GFX8-NEXT:    s_endpgm
+;
+; GFX942-LABEL: gep_as0_uniform:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX942-NEXT:    s_add_u32 s0, s0, s2
+; GFX942-NEXT:    s_addc_u32 s1, s1, s3
+; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
+; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
+; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX942-NEXT:    s_endpgm
+;
+; GFX10-LABEL: gep_as0_uniform:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-NEXT:    s_add_u32 s0, s0, s2
+; GFX10-NEXT:    s_addc_u32 s1, s1, s3
+; GFX10-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-NEXT:    v_mov_b32_e32 v3, s1
+; GFX10-NEXT:    v_mov_b32_e32 v2, s0
+; GFX10-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: gep_as0_uniform:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: gep_as0_uniform:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX12-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr inbounds i32, ptr %p, i64 %offset
+  store ptr %gep, ptr %ret
+  ret void
+}
+
+; Check that pointer arithmetic with multiple indexing steps can be lowered.
+define ptr @multi_gep_as0(ptr %p, i64 %offset) {
+; GFX8-LABEL: multi_gep_as0:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 5, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_PTRADD-LABEL: multi_gep_as0:
+; GFX942_PTRADD:       ; %bb.0: ; %entry
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 5
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: multi_gep_as0:
+; GFX942_LEGACY:       ; %bb.0: ; %entry
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 5
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: multi_gep_as0:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: multi_gep_as0:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: multi_gep_as0:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %gep1 = getelementptr inbounds i32, ptr %p, i64 %offset
+  %gep2 = getelementptr inbounds i8, ptr %gep1, i64 5
+  ret ptr %gep2
+}
+
+define amdgpu_kernel void @multi_gep_as0_uniform(ptr %p, i64 %offset, ptr %ret) {
+; GFX8-LABEL: multi_gep_as0_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX8-NEXT:    s_add_i32 s12, s12, s17
+; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX8-NEXT:    s_add_u32 s0, s0, s2
+; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_add_u32 s0, s0, 5
+; GFX8-NEXT:    s_addc_u32 s1, s1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    s_add_u32 s0, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    s_addc_u32 s1, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    flat_store_dword v[2:3], v5
+; GFX8-NEXT:    flat_store_dword v[0:1], v4
+; GFX8-NEXT:    s_endpgm
+;
+; GFX942-LABEL: multi_gep_as0_uniform:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX942-NEXT:    s_add_u32 s0, s0, s2
+; GFX942-NEXT:    s_addc_u32 s1, s1, s3
+; GFX942-NEXT:    s_add_u32 s0, s0, 5
+; GFX942-NEXT:    s_addc_u32 s1, s1, 0
+; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
+; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
+; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX942-NEXT:    s_endpgm
+;
+; GFX10-LABEL: multi_gep_as0_uniform:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_add_u32 s12, s12, s17
+; GFX10-NEXT:    s_addc_u32 s13, s13, 0
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-NEXT:    s_add_u32 s0, s0, s2
+; GFX10-NEXT:    s_addc_u32 s1, s1, s3
+; GFX10-NEXT:    s_add_u32 s0, s0, 5
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-NEXT:    v_mov_b32_e32 v3, s1
+; GFX10-NEXT:    v_mov_b32_e32 v2, s0
+; GFX10-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: multi_gep_as0_uniform:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT:    s_add_u32 s0, s0, s2
+; GFX11-NEXT:    s_addc_u32 s1, s1, s3
+; GFX11-NEXT:    s_add_u32 s0, s0, 5
+; GFX11-NEXT:    s_addc_u32 s1, s1, 0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: multi_gep_as0_uniform:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 5
+; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX12-NEXT:    s_endpgm
+entry:
+  %gep1 = getelementptr inbounds i32, ptr %p, i64 %offset
+  %gep2 = getelementptr inbounds i8, ptr %gep1, i64 5
+  store ptr %gep2, ptr %ret
+  ret void
+}
+
+; Check that constant offsets are folded into memory instructions.
+
+define void @fold_as0(ptr %from, ptr %to) {
+; GFX8-LABEL: fold_as0:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 8, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 8, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_store_dword v[0:1], v4
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: fold_as0:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    flat_load_dword v0, v[0:1] offset:8
+; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    flat_store_dword v[2:3], v0 offset:8
+; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fold_as0:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_load_dword v0, v[0:1] offset:8
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    flat_store_dword v[2:3], v0 offset:8
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fold_as0:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    flat_load_b32 v0, v[0:1] offset:8
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    flat_store_b32 v[2:3], v0 offset:8
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: fold_as0:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    flat_load_b32 v0, v[0:1] offset:8
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    flat_store_b32 v[2:3], v0 offset:8
+; GFX12-NEXT:    s_wait_dscnt 0x0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %gep.from = getelementptr inbounds i8, ptr %from, i64 8
+  %val = load i32, ptr %gep.from, align 4
+  %gep.to = getelementptr inbounds i8, ptr %to, i64 8
+  store i32 %val, ptr %gep.to, align 4
+  ret void
+}
+
+define void @fold_as1(ptr addrspace(1) %from, ptr addrspace(1) %to) {
+; GFX8-LABEL: fold_as1:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 8, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 8, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    flat_store_dword v[0:1], v4
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: fold_as1:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_load_dword v0, v[0:1], off offset:8
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dword v[2:3], v0, off offset:8
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fold_as1:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off offset:8
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off offset:8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fold_as1:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v0, v[0:1], off offset:8
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_store_b32 v[2:3], v0, off offset:8
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: fold_as1:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:8
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b32 v[2:3], v0, off offset:8
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %gep.from = getelementptr inbounds i8, ptr addrspace(1) %from, i64 8
+  %val = load i32, ptr addrspace(1) %gep.from, align 4
+  %gep.to = getelementptr inbounds i8, ptr addrspace(1) %to, i64 8
+  store i32 %val, ptr addrspace(1) %gep.to, align 4
+  ret void
+}
+
+define void @fold_as4(ptr addrspace(4) %from, ptr addrspace(1) %to) {
+; GFX8-LABEL: fold_as4:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 8, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 8, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    flat_store_dword v[0:1], v4
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: fold_as4:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_load_dword v0, v[0:1], off offset:8
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dword v[2:3], v0, off offset:8
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fold_as4:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off offset:8
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off offset:8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fold_as4:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v0, v[0:1], off offset:8
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_store_b32 v[2:3], v0, off offset:8
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: fold_as4:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:8
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b32 v[2:3], v0, off offset:8
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %gep.from = getelementptr inbounds i8, ptr addrspace(4) %from, i64 8
+  %val = load i32, ptr addrspace(4) %gep.from, align 4
+  %gep.to = getelementptr inbounds i8, ptr addrspace(1) %to, i64 8
+  store i32 %val, ptr addrspace(1) %gep.to, align 4
+  ret void
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10_LEGACY: {{.*}}
+; GFX10_PTRADD: {{.*}}
+; GFX11_LEGACY: {{.*}}
+; GFX11_PTRADD: {{.*}}
+; GFX12_LEGACY: {{.*}}
+; GFX12_PTRADD: {{.*}}
+; GFX8_LEGACY: {{.*}}
+; GFX8_PTRADD: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
index 3093349bff37..a41a06592f62 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s
 
+; Recursion: foo -> bar -> baz -> qux -> foo
+
 ; CHECK-LABEL: {{^}}qux
 ; CHECK: .set qux.num_vgpr, max(71, foo.num_vgpr)
 ; CHECK: .set qux.num_agpr, max(0, foo.num_agpr)
@@ -34,9 +36,9 @@
 ; CHECK: .set bar.has_indirect_call, or(0, baz.has_indirect_call)
 
 ; CHECK-LABEL: {{^}}foo
-; CHECK: .set foo.num_vgpr, max(46, amdgpu.max_num_vgpr)
-; CHECK: .set foo.num_agpr, max(0, amdgpu.max_num_agpr)
-; CHECK: .set foo.numbered_sgpr, max(71, amdgpu.max_num_sgpr)
+; CHECK: .set foo.num_vgpr, max(46, 71)
+; CHECK: .set foo.num_agpr, max(0, 0)
+; CHECK: .set foo.numbered_sgpr, max(71, 61)
 ; CHECK: .set foo.private_seg_size, 16
 ; CHECK: .set foo.uses_vcc, 1
 ; CHECK: .set foo.uses_flat_scratch, 0
@@ -91,3 +93,77 @@ define amdgpu_kernel void @usefoo() {
   ret void
 }
 
+; Recursion: A -> B -> C -> A && C -> D -> C
+
+; CHECK-LABEL: {{^}}D
+; CHECK: .set D.num_vgpr, max(71, C.num_vgpr)
+; CHECK: .set D.num_agpr, max(0, C.num_agpr)
+; CHECK: .set D.numbered_sgpr, max(71, C.numbered_sgpr)
+; CHECK: .set D.private_seg_size, 16+max(C.private_seg_size)
+; CHECK: .set D.uses_vcc, or(1, C.uses_vcc)
+; CHECK: .set D.uses_flat_scratch, or(0, C.uses_flat_scratch)
+; CHECK: .set D.has_dyn_sized_stack, or(0, C.has_dyn_sized_stack)
+; CHECK: .set D.has_recursion, or(1, C.has_recursion)
+; CHECK: .set D.has_indirect_call, or(0, C.has_indirect_call)
+
+; CHECK-LABEL: {{^}}C
+; CHECK: .set C.num_vgpr, max(42, A.num_vgpr, 71)
+; CHECK: .set C.num_agpr, max(0, A.num_agpr, 0)
+; CHECK: .set C.numbered_sgpr, max(71, A.numbered_sgpr, 71)
+; CHECK: .set C.private_seg_size, 16+max(A.private_seg_size)
+; CHECK: .set C.uses_vcc, or(1, A.uses_vcc)
+; CHECK: .set C.uses_flat_scratch, or(0, A.uses_flat_scratch)
+; CHECK: .set C.has_dyn_sized_stack, or(0, A.has_dyn_sized_stack)
+; CHECK: .set C.has_recursion, or(1, A.has_recursion)
+; CHECK: .set C.has_indirect_call, or(0, A.has_indirect_call)
+
+; CHECK-LABEL: {{^}}B
+; CHECK: .set B.num_vgpr, max(42, C.num_vgpr)
+; CHECK: .set B.num_agpr, max(0, C.num_agpr)
+; CHECK: .set B.numbered_sgpr, max(71, C.numbered_sgpr)
+; CHECK: .set B.private_seg_size, 16+max(C.private_seg_size)
+; CHECK: .set B.uses_vcc, or(1, C.uses_vcc)
+; CHECK: .set B.uses_flat_scratch, or(0, C.uses_flat_scratch)
+; CHECK: .set B.has_dyn_sized_stack, or(0, C.has_dyn_sized_stack)
+; CHECK: .set B.has_recursion, or(1, C.has_recursion)
+; CHECK: .set B.has_indirect_call, or(0, C.has_indirect_call)
+
+; CHECK-LABEL: {{^}}A
+; CHECK: .set A.num_vgpr, max(42, 71)
+; CHECK: .set A.num_agpr, max(0, 0)
+; CHECK: .set A.numbered_sgpr, max(71, 71)
+; CHECK: .set A.private_seg_size, 16
+; CHECK: .set A.uses_vcc, 1
+; CHECK: .set A.uses_flat_scratch, 0
+; CHECK: .set A.has_dyn_sized_stack, 0
+; CHECK: .set A.has_recursion, 1
+; CHECK: .set A.has_indirect_call, 0
+
+define void @A() {
+  call void @B()
+  call void asm sideeffect "", "~{v10}"()
+  call void asm sideeffect "", "~{s50}"()
+  ret void
+}
+
+define void @B() {
+  call void @C()
+  call void asm sideeffect "", "~{v20}"()
+  call void asm sideeffect "", "~{s30}"()
+  ret void
+}
+
+define void @C() {
+  call void @A()
+  call void @D()
+  call void asm sideeffect "", "~{v30}"()
+  call void asm sideeffect "", "~{s40}"()
+  ret void
+}
+
+define void @D() {
+  call void @C()
+  call void asm sideeffect "", "~{v70}"()
+  call void asm sideeffect "", "~{s70}"()
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sched-image-sample-post-RA.mir b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sched-image-sample-post-RA.mir
new file mode 100644
index 000000000000..a2a0794ac59f
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sched-image-sample-post-RA.mir
@@ -0,0 +1,161 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=postmisched -verify-misched -o - %s | FileCheck -check-prefix=TOPDOWN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=postmisched -misched-postra-direction=bottomup -verify-misched -o - %s | FileCheck -check-prefix=BOTTOMUP %s
+
+# This test demonstrates how bottom-up scheduling moves IMAGE_SAMPLE instructions
+# earlier compared to top-down scheduling.
+
+---
+name: sched_image_sample_post_ra
+tracksRegLiveness: true
+tracksDebugUserValues: true
+body: |
+  ; TOPDOWN-LABEL: name: sched_image_sample_post_ra
+  ; TOPDOWN: bb.0.entry:
+  ; TOPDOWN-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; TOPDOWN-NEXT:   liveins: $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; TOPDOWN-NEXT: {{  $}}
+  ; TOPDOWN-NEXT:   $sgpr12_sgpr13 = S_MOV_B64 $exec
+  ; TOPDOWN-NEXT:   $exec = S_WQM_B64 $exec, implicit-def $scc
+  ; TOPDOWN-NEXT:   $sgpr14 = S_MOV_B32 killed $sgpr5
+  ; TOPDOWN-NEXT:   $m0 = S_MOV_B32 killed $sgpr6
+  ; TOPDOWN-NEXT:   $sgpr16_sgpr17 = S_MOV_B64 $exec
+  ; TOPDOWN-NEXT:   $exec = S_WQM_B64 $exec, implicit-def $scc
+  ; TOPDOWN-NEXT:   $vgpr2 = LDS_PARAM_LOAD 0, 0, 0, implicit $m0, implicit $exec
+  ; TOPDOWN-NEXT:   $vgpr3 = LDS_PARAM_LOAD 0, 1, 0, implicit $m0, implicit $exec
+  ; TOPDOWN-NEXT:   $sgpr8 = S_MOV_B32 killed $sgpr1
+  ; TOPDOWN-NEXT:   renamable $sgpr0_sgpr1 = S_GETPC_B64
+  ; TOPDOWN-NEXT:   $sgpr10 = S_MOV_B32 killed $sgpr3
+  ; TOPDOWN-NEXT:   $sgpr9 = S_MOV_B32 killed $sgpr2
+  ; TOPDOWN-NEXT:   $sgpr15 = S_MOV_B32 killed $sgpr1
+  ; TOPDOWN-NEXT:   $exec = S_MOV_B64 killed renamable $sgpr16_sgpr17
+  ; TOPDOWN-NEXT:   BUNDLE implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit-def $sgpr16, implicit-def $sgpr16_lo16, implicit-def $sgpr16_hi16, implicit-def $sgpr17, implicit-def $sgpr17_lo16, implicit-def $sgpr17_hi16, implicit-def $sgpr18, implicit-def $sgpr18_lo16, implicit-def $sgpr18_hi16, implicit-def $sgpr19, implicit-def $sgpr19_lo16, implicit-def $sgpr19_hi16, implicit-def $sgpr20, implicit-def $sgpr20_lo16, implicit-def $sgpr20_hi16, implicit-def $sgpr21, implicit-def $sgpr21_lo16, implicit-def $sgpr21_hi16, implicit-def $sgpr22, implicit-def $sgpr22_lo16, implicit-def $sgpr22_hi16, implicit-def $sgpr23, implicit-def $sgpr23_lo16, implicit-def $sgpr23_hi16, implicit-def $sgpr16_sgpr17, implicit-def $sgpr16_sgpr17_sgpr18, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21, implicit-def $sgpr18_sgpr19, implicit-def $sgpr20_sgpr21, implicit-def $sgpr20_sgpr21_sgpr22, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23, implicit-def $sgpr22_sgpr23, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3, implicit-def $sgpr0, implicit-def $sgpr0_lo16, implicit-def $sgpr0_hi16, implicit-def $sgpr1, implicit-def $sgpr1_lo16, implicit-def $sgpr1_hi16, implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit-def $sgpr0_sgpr1, implicit-def $sgpr0_sgpr1_sgpr2, implicit-def $sgpr2_sgpr3, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr24, implicit-def $sgpr24_lo16, implicit-def $sgpr24_hi16, implicit-def $sgpr25, implicit-def $sgpr25_lo16, implicit-def $sgpr25_hi16, implicit-def $sgpr26, implicit-def $sgpr26_lo16, implicit-def $sgpr26_hi16, implicit-def $sgpr27, implicit-def $sgpr27_lo16, implicit-def $sgpr27_hi16, implicit-def $sgpr28, implicit-def $sgpr28_lo16, implicit-def $sgpr28_hi16, implicit-def $sgpr29, implicit-def $sgpr29_lo16, implicit-def $sgpr29_hi16, implicit-def $sgpr30, implicit-def $sgpr30_lo16, implicit-def $sgpr30_hi16, implicit-def $sgpr31, implicit-def $sgpr31_lo16, implicit-def $sgpr31_hi16, implicit-def $sgpr24_sgpr25, implicit-def $sgpr24_sgpr25_sgpr26, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29, implicit-def $sgpr26_sgpr27, implicit-def $sgpr28_sgpr29, implicit-def $sgpr28_sgpr29_sgpr30, implicit-def $sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr30_sgpr31, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30, implicit-def $sgpr36_sgpr37_sgpr38_sgpr39, implicit-def $sgpr36, implicit-def $sgpr36_lo16, implicit-def $sgpr36_hi16, implicit-def $sgpr37, implicit-def $sgpr37_lo16, implicit-def $sgpr37_hi16, implicit-def $sgpr38, implicit-def $sgpr38_lo16, implicit-def $sgpr38_hi16, implicit-def $sgpr39, implicit-def $sgpr39_lo16, implicit-def $sgpr39_hi16, implicit-def $sgpr36_sgpr37, implicit-def $sgpr36_sgpr37_sgpr38, implicit-def $sgpr38_sgpr39, implicit killed $sgpr14_sgpr15 {
+  ; TOPDOWN-NEXT:     renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23 = S_LOAD_DWORDX8_IMM renamable $sgpr14_sgpr15, 0, 0 :: (invariant load (s256))
+  ; TOPDOWN-NEXT:     renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr14_sgpr15, 128, 0 :: (invariant load (s128))
+  ; TOPDOWN-NEXT:     renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX8_IMM renamable $sgpr14_sgpr15, 96, 0 :: (invariant load (s256))
+  ; TOPDOWN-NEXT:     renamable $sgpr36_sgpr37_sgpr38_sgpr39 = S_LOAD_DWORDX4_IMM killed renamable $sgpr14_sgpr15, 32, 0 :: (invariant load (s128))
+  ; TOPDOWN-NEXT:   }
+  ; TOPDOWN-NEXT:   renamable $vgpr5 = V_INTERP_P10_F32_inreg 0, $vgpr2, 0, $vgpr0, 0, $vgpr2, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; TOPDOWN-NEXT:   renamable $vgpr6 = V_INTERP_P10_F32_inreg 0, $vgpr3, 0, killed $vgpr0, 0, $vgpr3, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; TOPDOWN-NEXT:   renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
+  ; TOPDOWN-NEXT:   renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
+  ; TOPDOWN-NEXT:   renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
+  ; TOPDOWN-NEXT:   renamable $vgpr9 = V_INTERP_P2_F32_inreg 0, killed $vgpr2, 0, $vgpr1, 0, killed $vgpr5, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; TOPDOWN-NEXT:   renamable $vgpr10 = V_INTERP_P2_F32_inreg 0, killed $vgpr3, 0, killed $vgpr1, 0, killed $vgpr6, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; TOPDOWN-NEXT:   BUNDLE implicit-def $vgpr11, implicit killed $vgpr9_vgpr10, implicit killed $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec, implicit killed $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39 {
+  ; TOPDOWN-NEXT:     renamable $vgpr11 = IMAGE_SAMPLE_V1_V2_gfx11 $vgpr9_vgpr10, killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; TOPDOWN-NEXT:     renamable $vgpr5_vgpr6_vgpr7_vgpr8 = IMAGE_SAMPLE_V4_V2_gfx11 killed $vgpr9_vgpr10, killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, 15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 8)
+  ; TOPDOWN-NEXT:   }
+  ; TOPDOWN-NEXT:   nofpexcept V_CMP_GT_F32_e32 1065353216, killed $vgpr11, implicit-def $vcc_lo, implicit $mode, implicit $exec
+  ; TOPDOWN-NEXT:   renamable $sgpr0_sgpr1 = COPY $vcc
+  ; TOPDOWN-NEXT:   nofpexcept V_CMP_GT_F32_e32 1065353216, killed $vgpr8, implicit-def $vcc_lo, implicit $mode, implicit $exec
+  ; TOPDOWN-NEXT:   renamable $sgpr2_sgpr3 = S_AND_B64 killed renamable $sgpr0_sgpr1, killed renamable $vcc, implicit-def dead $scc
+  ; TOPDOWN-NEXT:   $sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 killed $sgpr2_sgpr3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; TOPDOWN-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; TOPDOWN-NEXT: {{  $}}
+  ; TOPDOWN-NEXT: bb.1:
+  ; TOPDOWN-NEXT:   successors: %bb.2(0x80000000)
+  ; TOPDOWN-NEXT: {{  $}}
+  ; TOPDOWN-NEXT:   S_BRANCH %bb.2
+  ; TOPDOWN-NEXT: {{  $}}
+  ; TOPDOWN-NEXT: bb.2:
+  ; TOPDOWN-NEXT:   S_ENDPGM 0
+  ;
+  ; BOTTOMUP-LABEL: name: sched_image_sample_post_ra
+  ; BOTTOMUP: bb.0.entry:
+  ; BOTTOMUP-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; BOTTOMUP-NEXT:   liveins: $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; BOTTOMUP-NEXT: {{  $}}
+  ; BOTTOMUP-NEXT:   $sgpr12_sgpr13 = S_MOV_B64 $exec
+  ; BOTTOMUP-NEXT:   $exec = S_WQM_B64 $exec, implicit-def $scc
+  ; BOTTOMUP-NEXT:   $sgpr16_sgpr17 = S_MOV_B64 $exec
+  ; BOTTOMUP-NEXT:   $m0 = S_MOV_B32 killed $sgpr6
+  ; BOTTOMUP-NEXT:   $sgpr14 = S_MOV_B32 killed $sgpr5
+  ; BOTTOMUP-NEXT:   $exec = S_WQM_B64 $exec, implicit-def $scc
+  ; BOTTOMUP-NEXT:   $vgpr2 = LDS_PARAM_LOAD 0, 0, 0, implicit $m0, implicit $exec
+  ; BOTTOMUP-NEXT:   $vgpr3 = LDS_PARAM_LOAD 0, 1, 0, implicit $m0, implicit $exec
+  ; BOTTOMUP-NEXT:   $sgpr8 = S_MOV_B32 killed $sgpr1
+  ; BOTTOMUP-NEXT:   renamable $sgpr0_sgpr1 = S_GETPC_B64
+  ; BOTTOMUP-NEXT:   $sgpr9 = S_MOV_B32 killed $sgpr2
+  ; BOTTOMUP-NEXT:   $sgpr10 = S_MOV_B32 killed $sgpr3
+  ; BOTTOMUP-NEXT:   $sgpr15 = S_MOV_B32 killed $sgpr1
+  ; BOTTOMUP-NEXT:   $exec = S_MOV_B64 killed renamable $sgpr16_sgpr17
+  ; BOTTOMUP-NEXT:   BUNDLE implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit-def $sgpr16, implicit-def $sgpr16_lo16, implicit-def $sgpr16_hi16, implicit-def $sgpr17, implicit-def $sgpr17_lo16, implicit-def $sgpr17_hi16, implicit-def $sgpr18, implicit-def $sgpr18_lo16, implicit-def $sgpr18_hi16, implicit-def $sgpr19, implicit-def $sgpr19_lo16, implicit-def $sgpr19_hi16, implicit-def $sgpr20, implicit-def $sgpr20_lo16, implicit-def $sgpr20_hi16, implicit-def $sgpr21, implicit-def $sgpr21_lo16, implicit-def $sgpr21_hi16, implicit-def $sgpr22, implicit-def $sgpr22_lo16, implicit-def $sgpr22_hi16, implicit-def $sgpr23, implicit-def $sgpr23_lo16, implicit-def $sgpr23_hi16, implicit-def $sgpr16_sgpr17, implicit-def $sgpr16_sgpr17_sgpr18, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21, implicit-def $sgpr18_sgpr19, implicit-def $sgpr20_sgpr21, implicit-def $sgpr20_sgpr21_sgpr22, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23, implicit-def $sgpr22_sgpr23, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3, implicit-def $sgpr0, implicit-def $sgpr0_lo16, implicit-def $sgpr0_hi16, implicit-def $sgpr1, implicit-def $sgpr1_lo16, implicit-def $sgpr1_hi16, implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit-def $sgpr0_sgpr1, implicit-def $sgpr0_sgpr1_sgpr2, implicit-def $sgpr2_sgpr3, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr24, implicit-def $sgpr24_lo16, implicit-def $sgpr24_hi16, implicit-def $sgpr25, implicit-def $sgpr25_lo16, implicit-def $sgpr25_hi16, implicit-def $sgpr26, implicit-def $sgpr26_lo16, implicit-def $sgpr26_hi16, implicit-def $sgpr27, implicit-def $sgpr27_lo16, implicit-def $sgpr27_hi16, implicit-def $sgpr28, implicit-def $sgpr28_lo16, implicit-def $sgpr28_hi16, implicit-def $sgpr29, implicit-def $sgpr29_lo16, implicit-def $sgpr29_hi16, implicit-def $sgpr30, implicit-def $sgpr30_lo16, implicit-def $sgpr30_hi16, implicit-def $sgpr31, implicit-def $sgpr31_lo16, implicit-def $sgpr31_hi16, implicit-def $sgpr24_sgpr25, implicit-def $sgpr24_sgpr25_sgpr26, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29, implicit-def $sgpr26_sgpr27, implicit-def $sgpr28_sgpr29, implicit-def $sgpr28_sgpr29_sgpr30, implicit-def $sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr30_sgpr31, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30, implicit-def $sgpr36_sgpr37_sgpr38_sgpr39, implicit-def $sgpr36, implicit-def $sgpr36_lo16, implicit-def $sgpr36_hi16, implicit-def $sgpr37, implicit-def $sgpr37_lo16, implicit-def $sgpr37_hi16, implicit-def $sgpr38, implicit-def $sgpr38_lo16, implicit-def $sgpr38_hi16, implicit-def $sgpr39, implicit-def $sgpr39_lo16, implicit-def $sgpr39_hi16, implicit-def $sgpr36_sgpr37, implicit-def $sgpr36_sgpr37_sgpr38, implicit-def $sgpr38_sgpr39, implicit killed $sgpr14_sgpr15 {
+  ; BOTTOMUP-NEXT:     renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23 = S_LOAD_DWORDX8_IMM renamable $sgpr14_sgpr15, 0, 0 :: (invariant load (s256))
+  ; BOTTOMUP-NEXT:     renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr14_sgpr15, 128, 0 :: (invariant load (s128))
+  ; BOTTOMUP-NEXT:     renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX8_IMM renamable $sgpr14_sgpr15, 96, 0 :: (invariant load (s256))
+  ; BOTTOMUP-NEXT:     renamable $sgpr36_sgpr37_sgpr38_sgpr39 = S_LOAD_DWORDX4_IMM killed renamable $sgpr14_sgpr15, 32, 0 :: (invariant load (s128))
+  ; BOTTOMUP-NEXT:   }
+  ; BOTTOMUP-NEXT:   renamable $vgpr6 = V_INTERP_P10_F32_inreg 0, $vgpr3, 0, $vgpr0, 0, $vgpr3, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; BOTTOMUP-NEXT:   renamable $vgpr5 = V_INTERP_P10_F32_inreg 0, $vgpr2, 0, killed $vgpr0, 0, $vgpr2, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; BOTTOMUP-NEXT:   renamable $vgpr10 = V_INTERP_P2_F32_inreg 0, killed $vgpr3, 0, $vgpr1, 0, killed $vgpr6, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; BOTTOMUP-NEXT:   renamable $vgpr9 = V_INTERP_P2_F32_inreg 0, killed $vgpr2, 0, killed $vgpr1, 0, killed $vgpr5, 0, 7, implicit $m0, implicit $exec, implicit $mode
+  ; BOTTOMUP-NEXT:   BUNDLE implicit-def $vgpr11, implicit killed $vgpr9_vgpr10, implicit killed $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec, implicit killed $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39 {
+  ; BOTTOMUP-NEXT:     renamable $vgpr11 = IMAGE_SAMPLE_V1_V2_gfx11 $vgpr9_vgpr10, killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; BOTTOMUP-NEXT:     renamable $vgpr5_vgpr6_vgpr7_vgpr8 = IMAGE_SAMPLE_V4_V2_gfx11 killed $vgpr9_vgpr10, killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, 15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 8)
+  ; BOTTOMUP-NEXT:   }
+  ; BOTTOMUP-NEXT:   renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
+  ; BOTTOMUP-NEXT:   nofpexcept V_CMP_GT_F32_e32 1065353216, killed $vgpr11, implicit-def $vcc_lo, implicit $mode, implicit $exec
+  ; BOTTOMUP-NEXT:   renamable $sgpr0_sgpr1 = COPY $vcc
+  ; BOTTOMUP-NEXT:   nofpexcept V_CMP_GT_F32_e32 1065353216, killed $vgpr8, implicit-def $vcc_lo, implicit $mode, implicit $exec
+  ; BOTTOMUP-NEXT:   renamable $sgpr2_sgpr3 = S_AND_B64 killed renamable $sgpr0_sgpr1, killed renamable $vcc, implicit-def dead $scc
+  ; BOTTOMUP-NEXT:   renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
+  ; BOTTOMUP-NEXT:   renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
+  ; BOTTOMUP-NEXT:   $sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 killed $sgpr2_sgpr3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; BOTTOMUP-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; BOTTOMUP-NEXT: {{  $}}
+  ; BOTTOMUP-NEXT: bb.1:
+  ; BOTTOMUP-NEXT:   successors: %bb.2(0x80000000)
+  ; BOTTOMUP-NEXT: {{  $}}
+  ; BOTTOMUP-NEXT:   S_BRANCH %bb.2
+  ; BOTTOMUP-NEXT: {{  $}}
+  ; BOTTOMUP-NEXT: bb.2:
+  ; BOTTOMUP-NEXT:   S_ENDPGM 0
+  bb.0.entry:
+    successors: %bb.1(0x40000000), %bb.34(0x40000000); %bb.1(50.00%), %bb.34(50.00%)
+    liveins: $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+    $sgpr12_sgpr13 = S_MOV_B64 $exec
+    $exec = S_WQM_B64 $exec, implicit-def $scc
+    $sgpr14 = S_MOV_B32 $sgpr5
+    $m0 = S_MOV_B32 $sgpr6
+    $sgpr16_sgpr17 = S_MOV_B64 $exec
+    $exec = S_WQM_B64 $exec, implicit-def $scc
+    $sgpr10 = S_MOV_B32 $sgpr3
+    $sgpr9  = S_MOV_B32 $sgpr2
+    $sgpr8  = S_MOV_B32 $sgpr1
+    $vgpr2 = LDS_PARAM_LOAD 0, 0, 0, implicit $m0, implicit $exec
+    $vgpr3 = LDS_PARAM_LOAD 0, 1, 0, implicit $m0, implicit $exec
+    renamable $sgpr0_sgpr1 = S_GETPC_B64
+    $sgpr15 = S_MOV_B32 killed $sgpr1
+    $exec = S_MOV_B64 killed renamable $sgpr16_sgpr17
+    BUNDLE implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit-def $sgpr16, implicit-def $sgpr16_lo16, implicit-def $sgpr16_hi16, implicit-def $sgpr17, implicit-def $sgpr17_lo16, implicit-def $sgpr17_hi16, implicit-def $sgpr18, implicit-def $sgpr18_lo16, implicit-def $sgpr18_hi16, implicit-def $sgpr19, implicit-def $sgpr19_lo16, implicit-def $sgpr19_hi16, implicit-def $sgpr20, implicit-def $sgpr20_lo16, implicit-def $sgpr20_hi16, implicit-def $sgpr21, implicit-def $sgpr21_lo16, implicit-def $sgpr21_hi16, implicit-def $sgpr22, implicit-def $sgpr22_lo16, implicit-def $sgpr22_hi16, implicit-def $sgpr23, implicit-def $sgpr23_lo16, implicit-def $sgpr23_hi16, implicit-def $sgpr16_sgpr17, implicit-def $sgpr16_sgpr17_sgpr18, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21, implicit-def $sgpr18_sgpr19, implicit-def $sgpr20_sgpr21, implicit-def $sgpr20_sgpr21_sgpr22, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23, implicit-def $sgpr22_sgpr23, implicit-def $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3, implicit-def $sgpr0, implicit-def $sgpr0_lo16, implicit-def $sgpr0_hi16, implicit-def $sgpr1, implicit-def $sgpr1_lo16, implicit-def $sgpr1_hi16, implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit-def $sgpr0_sgpr1, implicit-def $sgpr0_sgpr1_sgpr2, implicit-def $sgpr2_sgpr3, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr24, implicit-def $sgpr24_lo16, implicit-def $sgpr24_hi16, implicit-def $sgpr25, implicit-def $sgpr25_lo16, implicit-def $sgpr25_hi16, implicit-def $sgpr26, implicit-def $sgpr26_lo16, implicit-def $sgpr26_hi16, implicit-def $sgpr27, implicit-def $sgpr27_lo16, implicit-def $sgpr27_hi16, implicit-def $sgpr28, implicit-def $sgpr28_lo16, implicit-def $sgpr28_hi16, implicit-def $sgpr29, implicit-def $sgpr29_lo16, implicit-def $sgpr29_hi16, implicit-def $sgpr30, implicit-def $sgpr30_lo16, implicit-def $sgpr30_hi16, implicit-def $sgpr31, implicit-def $sgpr31_lo16, implicit-def $sgpr31_hi16, implicit-def $sgpr24_sgpr25, implicit-def $sgpr24_sgpr25_sgpr26, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29, implicit-def $sgpr26_sgpr27, implicit-def $sgpr28_sgpr29, implicit-def $sgpr28_sgpr29_sgpr30, implicit-def $sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr30_sgpr31, implicit-def $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30, implicit-def $sgpr36_sgpr37_sgpr38_sgpr39, implicit-def $sgpr36, implicit-def $sgpr36_lo16, implicit-def $sgpr36_hi16, implicit-def $sgpr37, implicit-def $sgpr37_lo16, implicit-def $sgpr37_hi16, implicit-def $sgpr38, implicit-def $sgpr38_lo16, implicit-def $sgpr38_hi16, implicit-def $sgpr39, implicit-def $sgpr39_lo16, implicit-def $sgpr39_hi16, implicit-def $sgpr36_sgpr37, implicit-def $sgpr36_sgpr37_sgpr38, implicit-def $sgpr38_sgpr39, implicit $sgpr14_sgpr15 {
+      renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23 = S_LOAD_DWORDX8_IMM renamable $sgpr14_sgpr15, 0, 0 :: (invariant load (s256))
+      renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr14_sgpr15, 128, 0 :: (invariant load (s128))
+      renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX8_IMM renamable $sgpr14_sgpr15, 96, 0 :: (invariant load (s256))
+      renamable $sgpr36_sgpr37_sgpr38_sgpr39 = S_LOAD_DWORDX4_IMM renamable $sgpr14_sgpr15, 32, 0 :: (invariant load (s128))
+    }
+    renamable $vgpr5  = V_INTERP_P10_F32_inreg 0, $vgpr2, 0, $vgpr0, 0, $vgpr2, 0, 7, implicit $m0, implicit $exec, implicit $mode
+    renamable $vgpr6  = V_INTERP_P10_F32_inreg 0, $vgpr3, 0, $vgpr0, 0, $vgpr3, 0, 7, implicit $m0, implicit $exec, implicit $mode
+    renamable $vgpr9  = V_INTERP_P2_F32_inreg 0, $vgpr2, 0, $vgpr1, 0, killed $vgpr5, 0, 7, implicit $m0, implicit $exec, implicit $mode
+    renamable $vgpr10 = V_INTERP_P2_F32_inreg 0, $vgpr3, 0, $vgpr1, 0, killed $vgpr6, 0, 7, implicit $m0, implicit $exec, implicit $mode
+    BUNDLE implicit-def $vgpr11, implicit $vgpr9_vgpr10, implicit killed $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec, implicit killed $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39 {
+      renamable $vgpr11 = IMAGE_SAMPLE_V1_V2_gfx11 $vgpr9_vgpr10, killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+      renamable $vgpr5_vgpr6_vgpr7_vgpr8 = IMAGE_SAMPLE_V4_V2_gfx11 $vgpr9_vgpr10, killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, 15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 8)
+    }
+    nofpexcept V_CMP_GT_F32_e32 1065353216, $vgpr11, implicit-def $vcc, implicit $mode, implicit $exec
+    renamable $sgpr0_sgpr1 = COPY $vcc
+    nofpexcept V_CMP_GT_F32_e32 1065353216, $vgpr8,  implicit-def $vcc, implicit $mode, implicit $exec
+    renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
+    renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
+    renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr2_sgpr3 = S_AND_B64 killed renamable $sgpr0_sgpr1, killed renamable $vcc, implicit-def dead $scc
+    $sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 $sgpr2_sgpr3, implicit-def $exec, implicit-def $scc, implicit $exec
+    S_CBRANCH_EXECZ %bb.34, implicit $exec
+
+  bb.1:
+    successors: %bb.34(0x80000000)
+    S_BRANCH %bb.34
+
+  bb.34:
+    S_ENDPGM 0
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 15eb41a1a5b6..df496258a250 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -404,12 +404,11 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v16, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v17, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v5, v10
+; GCN-IR-NEXT:    v_not_b32_e32 v4, v10
 ; GCN-IR-NEXT:    v_lshr_b64 v[8:9], v[6:7], v8
-; GCN-IR-NEXT:    v_not_b32_e32 v4, 0
-; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, v5, v11
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, v4, v11
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
-; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v4, vcc
+; GCN-IR-NEXT:    v_addc_u32_e64 v7, s[4:5], -1, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
index f7bd5f8d5bfb..39428dc44801 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
@@ -355,17 +355,15 @@ define <2 x float> @v_test_fmin_legacy_ule_v2f32_nnan_nsz_flag(<2 x float> %a, <
 ; GFX7-LABEL: v_test_fmin_legacy_ule_v2f32_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_legacy_f32_e32 v0, v2, v0
-; GFX7-NEXT:    v_min_legacy_f32_e32 v1, v3, v1
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_test_fmin_legacy_ule_v2f32_nnan_nsz_flag:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_ngt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_ngt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_test_fmin_legacy_ule_v2f32_nnan_nsz_flag:
@@ -375,12 +373,7 @@ define <2 x float> @v_test_fmin_legacy_ule_v2f32_nnan_nsz_flag(<2 x float> %a, <
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX12-NEXT:    v_dual_min_num_f32 v0, v0, v2 :: v_dual_min_num_f32 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp ule <2 x float> %a, %b
   %val = select nnan nsz <2 x i1> %cmp, <2 x float> %a, <2 x float> %b
@@ -499,17 +492,15 @@ define <2 x float> @v_test_fmax_legacy_uge_v2f32_nnan_nsz_flag(<2 x float> %a, <
 ; GFX7-LABEL: v_test_fmax_legacy_uge_v2f32_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_legacy_f32_e32 v0, v2, v0
-; GFX7-NEXT:    v_max_legacy_f32_e32 v1, v3, v1
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_test_fmax_legacy_uge_v2f32_nnan_nsz_flag:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_nlt_f32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_nlt_f32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_test_fmax_legacy_uge_v2f32_nnan_nsz_flag:
@@ -519,12 +510,7 @@ define <2 x float> @v_test_fmax_legacy_uge_v2f32_nnan_nsz_flag(<2 x float> %a, <
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc_lo, v1, v3
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX12-NEXT:    v_dual_max_num_f32 v0, v0, v2 :: v_dual_max_num_f32 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp uge <2 x float> %a, %b
   %val = select nnan nsz <2 x i1> %cmp, <2 x float> %a, <2 x float> %b
@@ -673,10 +659,10 @@ define half @v_test_fmin_legacy_ule_f16_nnan_nsz_flag(half %a, half %b) {
 ; GFX7-LABEL: v_test_fmin_legacy_ule_f16_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_min_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -852,10 +838,10 @@ define half @v_test_fmax_legacy_uge_f16_nnan_nsz_flag(half %a, half %b) {
 ; GFX7-LABEL: v_test_fmax_legacy_uge_f16_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1097,16 +1083,16 @@ define <2 x half> @v_test_fmin_legacy_ule_v2f16_nnan_nsz_flag(<2 x half> %a, <2
 ; GFX7-LABEL: v_test_fmin_legacy_ule_v2f16_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_min_legacy_f32_e32 v0, v2, v0
-; GFX7-NEXT:    v_min_legacy_f32_e32 v1, v3, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_test_fmin_legacy_ule_v2f16_nnan_nsz_flag:
@@ -1337,16 +1323,16 @@ define <2 x half> @v_test_fmax_legacy_uge_v2f16_nnan_nsz_flag(<2 x half> %a, <2
 ; GFX7-LABEL: v_test_fmax_legacy_uge_v2f16_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_max_legacy_f32_e32 v0, v2, v0
-; GFX7-NEXT:    v_max_legacy_f32_e32 v1, v3, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_test_fmax_legacy_uge_v2f16_nnan_nsz_flag:
@@ -1667,26 +1653,26 @@ define <4 x half> @v_test_fmin_legacy_ule_v4f16_nnan_nsz_flag(<4 x half> %a, <4
 ; GFX7-LABEL: v_test_fmin_legacy_ule_v4f16_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_min_legacy_f32_e32 v0, v4, v0
-; GFX7-NEXT:    v_min_legacy_f32_e32 v1, v5, v1
-; GFX7-NEXT:    v_min_legacy_f32_e32 v2, v6, v2
-; GFX7-NEXT:    v_min_legacy_f32_e32 v3, v7, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_min_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_min_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_min_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_test_fmin_legacy_ule_v4f16_nnan_nsz_flag:
@@ -2009,26 +1995,26 @@ define <4 x half> @v_test_fmax_legacy_uge_v4f16_nnan_nsz_flag(<4 x half> %a, <4
 ; GFX7-LABEL: v_test_fmax_legacy_uge_v4f16_nnan_nsz_flag:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_max_legacy_f32_e32 v0, v4, v0
-; GFX7-NEXT:    v_max_legacy_f32_e32 v1, v5, v1
-; GFX7-NEXT:    v_max_legacy_f32_e32 v2, v6, v2
-; GFX7-NEXT:    v_max_legacy_f32_e32 v3, v7, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v4
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v5
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v6
+; GFX7-NEXT:    v_max_f32_e32 v3, v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_test_fmax_legacy_uge_v4f16_nnan_nsz_flag:
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/select.f16.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/select.f16.ll
index 2863fccac9fb..51de691e0ecc 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/select.f16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/select.f16.ll
@@ -3214,11 +3214,11 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
 ; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
 ; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v18
 ; VI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:108
-; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:92
+; VI-NEXT:    s_waitcnt vmcnt(6)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v49
 ; VI-NEXT:    v_cndmask_b32_e32 v49, v43, v55, vcc
 ; VI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:100
-; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:92
 ; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v1
 ; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v17
 ; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:84
@@ -3249,10 +3249,9 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
 ; VI-NEXT:    s_waitcnt vmcnt(13)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v44
 ; VI-NEXT:    v_cndmask_b32_e32 v13, v29, v13, vcc
-; VI-NEXT:    s_waitcnt vmcnt(12)
+; VI-NEXT:    s_waitcnt vmcnt(11)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v55
 ; VI-NEXT:    v_cndmask_b32_e32 v12, v28, v12, vcc
-; VI-NEXT:    s_waitcnt vmcnt(11)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v45
 ; VI-NEXT:    v_cndmask_b32_e32 v11, v27, v11, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(10)
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/shl.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/shl.ll
index 593cff712004..a82a6a8a4c36 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/shl.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/shl.ll
@@ -878,19 +878,19 @@ define amdgpu_kernel void @shl_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s8, s2
 ; SI-NEXT:    s_mov_b32 s9, s3
-; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
-; SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:48
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:48
+; SI-NEXT:    buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
 ; SI-NEXT:    buffer_load_dwordx4 v[7:10], off, s[8:11], 0
 ; SI-NEXT:    buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
 ; SI-NEXT:    s_mov_b32 s4, s0
 ; SI-NEXT:    s_mov_b32 s5, s1
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshl_b64 v[2:3], v[2:3], v6
-; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], v4
+; SI-NEXT:    v_lshl_b64 v[5:6], v[5:6], v2
+; SI-NEXT:    v_lshl_b64 v[3:4], v[3:4], v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshl_b64 v[9:10], v[9:10], v13
 ; SI-NEXT:    v_lshl_b64 v[7:8], v[7:8], v11
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[3:6], off, s[4:7], 0 offset:16
 ; SI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sra.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sra.ll
index ce89b2a962ee..0b49b9c815da 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -563,19 +563,19 @@ define amdgpu_kernel void @ashr_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s8, s2
 ; SI-NEXT:    s_mov_b32 s9, s3
-; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
-; SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:48
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:48
+; SI-NEXT:    buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
 ; SI-NEXT:    buffer_load_dwordx4 v[7:10], off, s[8:11], 0
 ; SI-NEXT:    buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
 ; SI-NEXT:    s_mov_b32 s4, s0
 ; SI-NEXT:    s_mov_b32 s5, s1
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_ashr_i64 v[2:3], v[2:3], v6
-; SI-NEXT:    v_ashr_i64 v[0:1], v[0:1], v4
+; SI-NEXT:    v_ashr_i64 v[5:6], v[5:6], v2
+; SI-NEXT:    v_ashr_i64 v[3:4], v[3:4], v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_ashr_i64 v[9:10], v[9:10], v13
 ; SI-NEXT:    v_ashr_i64 v[7:8], v[7:8], v11
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[3:6], off, s[4:7], 0 offset:16
 ; SI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -589,19 +589,19 @@ define amdgpu_kernel void @ashr_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s8, s6
 ; VI-NEXT:    s_mov_b32 s9, s7
-; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
-; VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:48
+; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:48
+; VI-NEXT:    buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
 ; VI-NEXT:    buffer_load_dwordx4 v[7:10], off, s[8:11], 0
 ; VI-NEXT:    buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
 ; VI-NEXT:    s_mov_b32 s0, s4
 ; VI-NEXT:    s_mov_b32 s1, s5
 ; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_ashrrev_i64 v[2:3], v6, v[2:3]
-; VI-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
+; VI-NEXT:    v_ashrrev_i64 v[5:6], v2, v[5:6]
+; VI-NEXT:    v_ashrrev_i64 v[3:4], v0, v[3:4]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_ashrrev_i64 v[9:10], v13, v[9:10]
 ; VI-NEXT:    v_ashrrev_i64 v[7:8], v11, v[7:8]
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; VI-NEXT:    buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:16
 ; VI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll
index c729c3fb8a4e..47dfa9f4fc2d 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -380,12 +380,11 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v16, vcc, -1, v2
 ; GCN-IR-NEXT:    v_addc_u32_e32 v17, vcc, -1, v3, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v7, v12
+; GCN-IR-NEXT:    v_not_b32_e32 v6, v12
 ; GCN-IR-NEXT:    v_lshr_b64 v[10:11], v[0:1], v8
-; GCN-IR-NEXT:    v_not_b32_e32 v6, 0
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v7, v13
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v6, v13
 ; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v6, vcc
+; GCN-IR-NEXT:    v_addc_u32_e64 v9, s[4:5], -1, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v13, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/srl.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srl.ll
index badb1f6fe984..239de43baa45 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/srl.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srl.ll
@@ -266,19 +266,19 @@ define amdgpu_kernel void @lshr_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s8, s2
 ; SI-NEXT:    s_mov_b32 s9, s3
-; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
-; SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:48
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:48
+; SI-NEXT:    buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
 ; SI-NEXT:    buffer_load_dwordx4 v[7:10], off, s[8:11], 0
 ; SI-NEXT:    buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
 ; SI-NEXT:    s_mov_b32 s4, s0
 ; SI-NEXT:    s_mov_b32 s5, s1
 ; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], v6
-; SI-NEXT:    v_lshr_b64 v[0:1], v[0:1], v4
+; SI-NEXT:    v_lshr_b64 v[5:6], v[5:6], v2
+; SI-NEXT:    v_lshr_b64 v[3:4], v[3:4], v0
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshr_b64 v[9:10], v[9:10], v13
 ; SI-NEXT:    v_lshr_b64 v[7:8], v[7:8], v11
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[3:6], off, s[4:7], 0 offset:16
 ; SI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/srl64_reduce.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srl64_reduce.ll
new file mode 100644
index 000000000000..3567bafe5b1c
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srl64_reduce.ll
@@ -0,0 +1,598 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+;; Test reduction of:
+;;
+;;   DST = lshr i64 X, Y
+;;
+;; where Y is in the range [63-32] to:
+;;
+;;   DST = [srl i32 X, (Y & 0x1F), 0]
+
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range with metadata
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define i64 @srl_metadata(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v0, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{}
+  %srl = lshr i64 %arg0, %shift.amt
+  ret i64 %srl
+}
+
+define amdgpu_ps i64 @srl_metadata_sgpr_return(i64 inreg %arg0, ptr addrspace(1) inreg %arg1.ptr) {
+; CHECK-LABEL: srl_metadata_sgpr_return:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s0, s[2:3], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_lshr_b32 s0, s1, s0
+; CHECK-NEXT:    s_mov_b32 s1, 0
+; CHECK-NEXT:    ; return to shader part epilog
+  %shift.amt = load i64, ptr addrspace(1) %arg1.ptr, !range !0, !noundef !{}
+  %srl = lshr i64 %arg0, %shift.amt
+  ret i64 %srl
+}
+
+; Exact attribute does not inhibit reduction
+define i64 @srl_exact_metadata(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_exact_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v0, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{}
+  %srl = lshr exact i64 %arg0, %shift.amt
+  ret i64 %srl
+}
+
+define i64 @srl_metadata_two_ranges(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_metadata_two_ranges:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v0, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !1, !noundef !{}
+  %srl = lshr i64 %arg0, %shift.amt
+  ret i64 %srl
+}
+
+; Known minimum is too low.  Reduction must not be done.
+define i64 @srl_metadata_out_of_range(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_metadata_out_of_range:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !2, !noundef !{}
+  %srl = lshr i64 %arg0, %shift.amt
+  ret i64 %srl
+}
+
+; Bounds cannot be truncated to i32 when load is narrowed to i32.
+; Reduction must not be done.
+; Bounds were chosen so that if bounds were truncated to i32 the
+; known minimum would be 32 and the srl would be erroneously reduced.
+define i64 @srl_metadata_cant_be_narrowed_to_i32(i64 %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_metadata_cant_be_narrowed_to_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load i64, ptr %arg1.ptr, !range !3, !noundef !{}
+  %srl = lshr i64 %arg0, %shift.amt
+  ret i64 %srl
+}
+
+define <2 x i64> @srl_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_v2_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v4, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, v6, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %srl = lshr <2 x i64> %arg0, %shift.amt
+  ret <2 x i64> %srl
+}
+
+; Exact attribute does not inhibit reduction
+define <2 x i64> @srl_exact_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_exact_v2_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v4, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, v6, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %srl = lshr exact <2 x i64> %arg0, %shift.amt
+  ret <2 x i64> %srl
+}
+
+define <3 x i64> @srl_v3_metadata(<3 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_v3_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dword v0, v[6:7] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[6:7]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v4, v0, v5
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v8, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, v10, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <3 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %srl = lshr <3 x i64> %arg0, %shift.amt
+  ret <3 x i64> %srl
+}
+
+define <4 x i64> @srl_v4_metadata(<4 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: srl_v4_metadata:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[8:9]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_load_dwordx4 v[13:16], v[8:9] offset:16
+; CHECK-NEXT:    ; kill: killed $vgpr8 killed $vgpr9
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v10, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, v12, v3
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v4, v13, v5
+; CHECK-NEXT:    v_lshrrev_b32_e32 v6, v15, v7
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    v_mov_b32_e32 v7, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <4 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
+  %srl = lshr <4 x i64> %arg0, %shift.amt
+  ret <4 x i64> %srl
+}
+
+!0 = !{i64 32, i64 64}
+!1 = !{i64 32, i64 38, i64 42, i64 48}
+!2 = !{i64 31, i64 38, i64 42, i64 48}
+!3 = !{i64 32, i64 38, i64 2147483680, i64 2147483681}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range with an "or X, 16"
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; These cases must not be reduced because the known minimum, 16, is not in range.
+
+define i64 @srl_or16(i64 %arg0, i64 %shift_amt) {
+; CHECK-LABEL: srl_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v2, 16, v2
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 16
+  %srl = lshr i64 %arg0, %or
+  ret i64 %srl
+}
+
+define <2 x i64> @srl_v2_or16(<2 x i64> %arg0, <2 x i64> %shift_amt) {
+; CHECK-LABEL: srl_v2_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v5, 16, v6
+; CHECK-NEXT:    v_or_b32_e32 v4, 16, v4
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v4, v[0:1]
+; CHECK-NEXT:    v_lshrrev_b64 v[2:3], v5, v[2:3]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 16)
+  %srl = lshr <2 x i64> %arg0, %or
+  ret <2 x i64> %srl
+}
+
+define <3 x i64> @srl_v3_or16(<3 x i64> %arg0, <3 x i64> %shift_amt) {
+; CHECK-LABEL: srl_v3_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v7, 16, v10
+; CHECK-NEXT:    v_or_b32_e32 v8, 16, v8
+; CHECK-NEXT:    v_or_b32_e32 v6, 16, v6
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v6, v[0:1]
+; CHECK-NEXT:    v_lshrrev_b64 v[2:3], v8, v[2:3]
+; CHECK-NEXT:    v_lshrrev_b64 v[4:5], v7, v[4:5]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 16)
+  %srl = lshr <3 x i64> %arg0, %or
+  ret <3 x i64> %srl
+}
+
+define <4 x i64> @srl_v4_or16(<4 x i64> %arg0, <4 x i64> %shift_amt) {
+; CHECK-LABEL: srl_v4_or16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v9, 16, v14
+; CHECK-NEXT:    v_or_b32_e32 v11, 16, v12
+; CHECK-NEXT:    v_or_b32_e32 v10, 16, v10
+; CHECK-NEXT:    v_or_b32_e32 v8, 16, v8
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v8, v[0:1]
+; CHECK-NEXT:    v_lshrrev_b64 v[2:3], v10, v[2:3]
+; CHECK-NEXT:    v_lshrrev_b64 v[4:5], v11, v[4:5]
+; CHECK-NEXT:    v_lshrrev_b64 v[6:7], v9, v[6:7]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 16)
+  %srl = lshr <4 x i64> %arg0, %or
+  ret <4 x i64> %srl
+}
+
+; test SGPR
+
+define i64 @srl_or16_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: srl_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s4, s18, 16
+; CHECK-NEXT:    s_lshr_b64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 16
+  %srl = lshr i64 %arg0, %or
+  ret i64 %srl
+}
+
+define amdgpu_ps i64 @srl_or16_sgpr_return(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: srl_or16_sgpr_return:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_or_b32 s2, s2, 16
+; CHECK-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; CHECK-NEXT:    ; return to shader part epilog
+  %or = or i64 %shift_amt, 16
+  %srl = lshr i64 %arg0, %or
+  ret i64 %srl
+}
+
+define <2 x i64> @srl_v2_or16_sgpr(<2 x i64> inreg %arg0, <2 x i64> inreg %shift_amt) {
+; CHECK-LABEL: srl_v2_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s6, s22, 16
+; CHECK-NEXT:    s_or_b32 s4, s20, 16
+; CHECK-NEXT:    s_lshr_b64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_lshr_b64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 16)
+  %srl = lshr <2 x i64> %arg0, %or
+  ret <2 x i64> %srl
+}
+
+define <3 x i64> @srl_v3_or16_sgpr(<3 x i64> inreg %arg0, <3 x i64> inreg %shift_amt) {
+; CHECK-LABEL: srl_v3_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_or_b32 s8, s26, 16
+; CHECK-NEXT:    s_or_b32 s6, s24, 16
+; CHECK-NEXT:    s_or_b32 s4, s22, 16
+; CHECK-NEXT:    s_lshr_b64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_lshr_b64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    s_lshr_b64 s[8:9], s[20:21], s8
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_mov_b32_e32 v4, s8
+; CHECK-NEXT:    v_mov_b32_e32 v5, s9
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 16)
+  %srl = lshr <3 x i64> %arg0, %or
+  ret <3 x i64> %srl
+}
+
+define <4 x i64> @srl_v4_or16_sgpr(<4 x i64> inreg %arg0, <4 x i64> inreg %shift_amt) {
+; CHECK-LABEL: srl_v4_or16_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v0, 16, v0
+; CHECK-NEXT:    s_or_b32 s8, s28, 16
+; CHECK-NEXT:    s_or_b32 s6, s26, 16
+; CHECK-NEXT:    s_or_b32 s4, s24, 16
+; CHECK-NEXT:    s_lshr_b64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_lshr_b64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    s_lshr_b64 s[8:9], s[20:21], s8
+; CHECK-NEXT:    v_lshrrev_b64 v[6:7], v0, s[22:23]
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s6
+; CHECK-NEXT:    v_mov_b32_e32 v3, s7
+; CHECK-NEXT:    v_mov_b32_e32 v4, s8
+; CHECK-NEXT:    v_mov_b32_e32 v5, s9
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 16)
+  %srl = lshr <4 x i64> %arg0, %or
+  ret <4 x i64> %srl
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range with an "or X, 32"
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; These cases are reduced because computeKnownBits() can calculate a minimum of 32
+; based on the OR with 32.
+
+define i64 @srl_or32(i64 %arg0, i64 %shift_amt) {
+; CHECK-LABEL: srl_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v2, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 32
+  %srl = lshr i64 %arg0, %or
+  ret i64 %srl
+}
+
+define <2 x i64> @srl_v2_or32(<2 x i64> %arg0, <2 x i64> %shift_amt) {
+; CHECK-LABEL: srl_v2_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v4, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, v6, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 32)
+  %srl = lshr <2 x i64> %arg0, %or
+  ret <2 x i64> %srl
+}
+
+define <3 x i64> @srl_v3_or32(<3 x i64> %arg0, <3 x i64> %shift_amt) {
+; CHECK-LABEL: srl_v3_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v6, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, v8, v3
+; CHECK-NEXT:    v_lshrrev_b32_e32 v4, v10, v5
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 32)
+  %srl = lshr <3 x i64> %arg0, %or
+  ret <3 x i64> %srl
+}
+
+define <4 x i64> @srl_v4_or32(<4 x i64> %arg0, <4 x i64> %shift_amt) {
+; CHECK-LABEL: srl_v4_or32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_lshrrev_b32_e32 v0, v8, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, v10, v3
+; CHECK-NEXT:    v_lshrrev_b32_e32 v4, v12, v5
+; CHECK-NEXT:    v_lshrrev_b32_e32 v6, v14, v7
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    v_mov_b32_e32 v7, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 32)
+  %srl = lshr <4 x i64> %arg0, %or
+  ret <4 x i64> %srl
+}
+
+; test SGPR
+
+define i64 @srl_or32_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: srl_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_lshr_b32 s4, s17, s18
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or i64 %shift_amt, 32
+  %srl = lshr i64 %arg0, %or
+  ret i64 %srl
+}
+
+define amdgpu_ps i64 @srl_or32_sgpr_return(i64 inreg %arg0, i64 inreg %shift_amt) {
+; CHECK-LABEL: srl_or32_sgpr_return:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_lshr_b32 s0, s1, s2
+; CHECK-NEXT:    s_mov_b32 s1, 0
+; CHECK-NEXT:    ; return to shader part epilog
+  %or = or i64 %shift_amt, 32
+  %srl = lshr i64 %arg0, %or
+  ret i64 %srl
+}
+
+define <2 x i64> @srl_v2_or32_sgpr(<2 x i64> inreg %arg0, <2 x i64> inreg %shift_amt) {
+; CHECK-LABEL: srl_v2_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_lshr_b32 s4, s17, s20
+; CHECK-NEXT:    s_lshr_b32 s5, s19, s22
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, s5
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <2 x i64> %shift_amt, splat (i64 32)
+  %srl = lshr <2 x i64> %arg0, %or
+  ret <2 x i64> %srl
+}
+
+define <3 x i64> @srl_v3_or32_sgpr(<3 x i64> inreg %arg0, <3 x i64> inreg %shift_amt) {
+; CHECK-LABEL: srl_v3_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_lshr_b32 s4, s17, s22
+; CHECK-NEXT:    s_lshr_b32 s5, s19, s24
+; CHECK-NEXT:    s_lshr_b32 s6, s21, s26
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, s5
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v4, s6
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <3 x i64> %shift_amt, splat (i64 32)
+  %srl = lshr <3 x i64> %arg0, %or
+  ret <3 x i64> %srl
+}
+
+define <4 x i64> @srl_v4_or32_sgpr(<4 x i64> inreg %arg0, <4 x i64> inreg %shift_amt) {
+; CHECK-LABEL: srl_v4_or32_sgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_lshr_b32 s4, s17, s24
+; CHECK-NEXT:    s_lshr_b32 s5, s19, s26
+; CHECK-NEXT:    s_lshr_b32 s6, s21, s28
+; CHECK-NEXT:    v_lshrrev_b32_e64 v6, v0, s23
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, s5
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v4, s6
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:    v_mov_b32_e32 v7, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %or = or <4 x i64> %shift_amt, splat (i64 32)
+  %srl = lshr <4 x i64> %arg0, %or
+  ret <4 x i64> %srl
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Test range from max/min
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; FIXME: This case should be reduced too, but computeKnownBits() cannot
+;        determine the range.  Match current results for now.
+
+define i64 @srl_maxmin(i64 %arg0, i64 noundef %arg1) {
+; CHECK-LABEL: srl_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, 32, v2, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, 63, v2, vcc
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call i64 @llvm.umax.i64(i64 %arg1, i64 32)
+  %min = call i64 @llvm.umin.i64(i64 %max,  i64 63)
+  %srl = lshr i64 %arg0, %min
+  ret i64 %srl
+}
+
+define <2 x i64> @srl_v2_maxmin(<2 x i64> %arg0, <2 x i64> noundef %arg1) {
+; CHECK-LABEL: srl_v2_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, 32, v4, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 32, v6, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 63, v6, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[4:5]
+; CHECK-NEXT:    v_lshrrev_b64 v[2:3], v6, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, 63, v4, vcc
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v4, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call <2 x i64> @llvm.umax.i64(<2 x i64> %arg1, <2 x i64> splat (i64 32))
+  %min = call <2 x i64> @llvm.umin.i64(<2 x i64> %max,  <2 x i64> splat (i64 63))
+  %srl = lshr <2 x i64> %arg0, %min
+  ret <2 x i64> %srl
+}
+
+define <3 x i64> @srl_v3_maxmin(<3 x i64> %arg0, <3 x i64> noundef %arg1) {
+; CHECK-LABEL: srl_v3_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v7, 0, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 32, v6, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[8:9]
+; CHECK-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 32, v8, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[10:11]
+; CHECK-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 32, v10, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[10:11]
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 63, v10, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[8:9]
+; CHECK-NEXT:    v_lshrrev_b64 v[4:5], v10, v[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 63, v8, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[6:7]
+; CHECK-NEXT:    v_lshrrev_b64 v[2:3], v8, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, 63, v6, vcc
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v6, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call <3 x i64> @llvm.umax.i64(<3 x i64> %arg1, <3 x i64> splat (i64 32))
+  %min = call <3 x i64> @llvm.umin.i64(<3 x i64> %max,  <3 x i64> splat (i64 63))
+  %srl = lshr <3 x i64> %arg0, %min
+  ret <3 x i64> %srl
+}
+
+define <4 x i64> @srl_v4_maxmin(<4 x i64> %arg0, <4 x i64> noundef %arg1) {
+; CHECK-LABEL: srl_v4_maxmin:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[8:9]
+; CHECK-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 32, v8, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[10:11]
+; CHECK-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 32, v10, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[12:13]
+; CHECK-NEXT:    v_cndmask_b32_e32 v13, 0, v13, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v12, 32, v12, vcc
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, 32, v[14:15]
+; CHECK-NEXT:    v_cndmask_b32_e32 v15, 0, v15, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v14, 32, v14, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[14:15]
+; CHECK-NEXT:    v_cndmask_b32_e32 v14, 63, v14, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[12:13]
+; CHECK-NEXT:    v_lshrrev_b64 v[6:7], v14, v[6:7]
+; CHECK-NEXT:    v_cndmask_b32_e32 v12, 63, v12, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[10:11]
+; CHECK-NEXT:    v_lshrrev_b64 v[4:5], v12, v[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v10, 63, v10, vcc
+; CHECK-NEXT:    v_cmp_gt_u64_e32 vcc, 63, v[8:9]
+; CHECK-NEXT:    v_lshrrev_b64 v[2:3], v10, v[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e32 v8, 63, v8, vcc
+; CHECK-NEXT:    v_lshrrev_b64 v[0:1], v8, v[0:1]
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %max = call <4 x i64> @llvm.umax.i64(<4 x i64> %arg1, <4 x i64> splat (i64 32))
+  %min = call <4 x i64> @llvm.umin.i64(<4 x i64> %max,  <4 x i64> splat (i64 63))
+  %srl = lshr <4 x i64> %arg0, %min
+  ret <4 x i64> %srl
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.128.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.128.ll
index dcf3643756cb..2efa022efd70 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.128.ll
@@ -279,36 +279,37 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
 ; GFX11-NEXT:    s_lshr_b32 s4, s3, 8
-; GFX11-NEXT:    s_lshr_b32 s3, s3, 24
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s4
 ; GFX11-NEXT:    s_lshr_b32 s5, s2, 8
-; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
 ; GFX11-NEXT:    s_lshr_b32 s2, s2, 24
 ; GFX11-NEXT:    s_lshr_b32 s6, s1, 8
-; GFX11-NEXT:    v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s5
-; GFX11-NEXT:    v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s6
-; GFX11-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s4
+; GFX11-NEXT:    s_lshr_b32 s3, s3, 24
 ; GFX11-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX11-NEXT:    s_lshr_b32 s7, s0, 8
 ; GFX11-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX11-NEXT:    v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s6
+; GFX11-NEXT:    v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s5
+; GFX11-NEXT:    v_mov_b32_e32 v10, s1
 ; GFX11-NEXT:    ds_store_b8 v0, v2 offset:8
 ; GFX11-NEXT:    ds_store_b8_d16_hi v0, v2 offset:10
 ; GFX11-NEXT:    ds_store_b8 v0, v1 offset:12
-; GFX11-NEXT:    ds_store_b8 v0, v4
-; GFX11-NEXT:    ds_store_b8_d16_hi v0, v4 offset:2
-; GFX11-NEXT:    ds_store_b8 v0, v3 offset:4
-; GFX11-NEXT:    ds_store_b8 v0, v5 offset:13
 ; GFX11-NEXT:    ds_store_b8_d16_hi v0, v1 offset:14
+; GFX11-NEXT:    ds_store_b8 v0, v5 offset:13
 ; GFX11-NEXT:    ds_store_b8 v0, v6 offset:15
-; GFX11-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v10, s7
-; GFX11-NEXT:    v_mov_b32_e32 v11, s0
 ; GFX11-NEXT:    ds_store_b8 v0, v7 offset:9
 ; GFX11-NEXT:    ds_store_b8 v0, v8 offset:11
-; GFX11-NEXT:    ds_store_b8 v0, v9 offset:5
+; GFX11-NEXT:    v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    ds_store_b8 v0, v4
+; GFX11-NEXT:    ds_store_b8_d16_hi v0, v4 offset:2
+; GFX11-NEXT:    ds_store_b8 v0, v3 offset:4
 ; GFX11-NEXT:    ds_store_b8_d16_hi v0, v3 offset:6
-; GFX11-NEXT:    ds_store_b8 v0, v1 offset:7
-; GFX11-NEXT:    ds_store_b8 v0, v10 offset:1
-; GFX11-NEXT:    ds_store_b8 v0, v11 offset:3
+; GFX11-NEXT:    ds_store_b8 v0, v9 offset:5
+; GFX11-NEXT:    ds_store_b8 v0, v10 offset:7
+; GFX11-NEXT:    ds_store_b8 v0, v1 offset:1
+; GFX11-NEXT:    ds_store_b8 v0, v2 offset:3
 ; GFX11-NEXT:    s_endpgm
   store <4 x i32> %x, ptr addrspace(3) %out, align 1
   ret void
@@ -420,17 +421,17 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
 ; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x0
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s3
-; GFX11-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT:    v_mov_b32_e32 v4, s2
-; GFX11-NEXT:    ds_store_b16_d16_hi v0, v1 offset:14
-; GFX11-NEXT:    ds_store_b16 v0, v2
-; GFX11-NEXT:    ds_store_b16 v0, v3 offset:4
-; GFX11-NEXT:    ds_store_b16 v0, v4 offset:8
-; GFX11-NEXT:    ds_store_b16 v0, v1 offset:12
-; GFX11-NEXT:    ds_store_b16_d16_hi v0, v4 offset:10
-; GFX11-NEXT:    ds_store_b16_d16_hi v0, v3 offset:6
-; GFX11-NEXT:    ds_store_b16_d16_hi v0, v2 offset:2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT:    v_mov_b32_e32 v4, s3
+; GFX11-NEXT:    ds_store_b16 v0, v1
+; GFX11-NEXT:    ds_store_b16 v0, v2 offset:4
+; GFX11-NEXT:    ds_store_b16 v0, v3 offset:8
+; GFX11-NEXT:    ds_store_b16 v0, v4 offset:12
+; GFX11-NEXT:    ds_store_b16_d16_hi v0, v4 offset:14
+; GFX11-NEXT:    ds_store_b16_d16_hi v0, v3 offset:10
+; GFX11-NEXT:    ds_store_b16_d16_hi v0, v2 offset:6
+; GFX11-NEXT:    ds_store_b16_d16_hi v0, v1 offset:2
 ; GFX11-NEXT:    s_endpgm
   store <4 x i32> %x, ptr addrspace(3) %out, align 2
   ret void
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.96.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.96.ll
index 70906d8474aa..03a7ec4883ff 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.96.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/store-local.96.ll
@@ -239,25 +239,26 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
 ; GFX11-NEXT:    s_lshr_b32 s3, s2, 8
 ; GFX11-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s2
+; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
 ; GFX11-NEXT:    s_lshr_b32 s4, s1, 8
 ; GFX11-NEXT:    s_lshr_b32 s1, s1, 24
 ; GFX11-NEXT:    s_lshr_b32 s5, s0, 8
 ; GFX11-NEXT:    s_lshr_b32 s0, s0, 24
-; GFX11-NEXT:    v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s2
 ; GFX11-NEXT:    v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s1
 ; GFX11-NEXT:    v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v9, s0
 ; GFX11-NEXT:    ds_store_b8 v0, v1 offset:8
+; GFX11-NEXT:    ds_store_b8_d16_hi v0, v1 offset:10
+; GFX11-NEXT:    ds_store_b8 v0, v4 offset:9
+; GFX11-NEXT:    ds_store_b8 v0, v5 offset:11
 ; GFX11-NEXT:    ds_store_b8 v0, v3
 ; GFX11-NEXT:    ds_store_b8_d16_hi v0, v3 offset:2
 ; GFX11-NEXT:    ds_store_b8 v0, v2 offset:4
-; GFX11-NEXT:    ds_store_b8 v0, v4 offset:9
-; GFX11-NEXT:    ds_store_b8_d16_hi v0, v1 offset:10
-; GFX11-NEXT:    ds_store_b8 v0, v5 offset:11
-; GFX11-NEXT:    ds_store_b8 v0, v6 offset:5
 ; GFX11-NEXT:    ds_store_b8_d16_hi v0, v2 offset:6
+; GFX11-NEXT:    ds_store_b8 v0, v6 offset:5
 ; GFX11-NEXT:    ds_store_b8 v0, v7 offset:7
 ; GFX11-NEXT:    ds_store_b8 v0, v8 offset:1
 ; GFX11-NEXT:    ds_store_b8 v0, v9 offset:3
@@ -356,14 +357,14 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
 ; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x0
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT:    ds_store_b16_d16_hi v0, v1 offset:10
-; GFX11-NEXT:    ds_store_b16 v0, v2
-; GFX11-NEXT:    ds_store_b16 v0, v3 offset:4
-; GFX11-NEXT:    ds_store_b16 v0, v1 offset:8
-; GFX11-NEXT:    ds_store_b16_d16_hi v0, v3 offset:6
-; GFX11-NEXT:    ds_store_b16_d16_hi v0, v2 offset:2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT:    ds_store_b16 v0, v1
+; GFX11-NEXT:    ds_store_b16 v0, v2 offset:4
+; GFX11-NEXT:    ds_store_b16 v0, v3 offset:8
+; GFX11-NEXT:    ds_store_b16_d16_hi v0, v3 offset:10
+; GFX11-NEXT:    ds_store_b16_d16_hi v0, v2 offset:6
+; GFX11-NEXT:    ds_store_b16_d16_hi v0, v1 offset:2
 ; GFX11-NEXT:    s_endpgm
   store <3 x i32> %x, ptr addrspace(3) %out, align 2
   ret void
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sub.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sub.ll
index a3aeea8a145c..ec065b4daa37 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sub.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sub.ll
@@ -967,20 +967,20 @@ define amdgpu_kernel void @v_test_sub_v4i64(ptr addrspace(1) %out, ptr addrspace
 ;
 ; GFX9-LABEL: v_test_sub_v4i64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v16, 5, v0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx4 v[0:3], v16, s[2:3]
-; GFX9-NEXT:    global_load_dwordx4 v[4:7], v16, s[6:7]
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v16, s[6:7]
+; GFX9-NEXT:    global_load_dwordx4 v[4:7], v16, s[2:3]
 ; GFX9-NEXT:    global_load_dwordx4 v[8:11], v16, s[2:3] offset:16
 ; GFX9-NEXT:    global_load_dwordx4 v[12:15], v16, s[6:7] offset:16
 ; GFX9-NEXT:    v_mov_b32_e32 v16, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v2, v6
-; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v7, vcc
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v7, v3, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v4, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v5, v1, vcc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, v10, v14
 ; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v11, v15, vcc
@@ -993,22 +993,22 @@ define amdgpu_kernel void @v_test_sub_v4i64(ptr addrspace(1) %out, ptr addrspace
 ; GFX12-LABEL: v_test_sub_v4i64:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
 ; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
 ; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-NEXT:    v_mov_b32_e32 v16, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_lshlrev_b32_e32 v12, 5, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_clause 0x3
-; GFX12-NEXT:    global_load_b128 v[0:3], v12, s[2:3]
-; GFX12-NEXT:    global_load_b128 v[4:7], v12, s[4:5]
+; GFX12-NEXT:    global_load_b128 v[0:3], v12, s[6:7]
+; GFX12-NEXT:    global_load_b128 v[4:7], v12, s[2:3]
 ; GFX12-NEXT:    global_load_b128 v[8:11], v12, s[2:3] offset:16
-; GFX12-NEXT:    global_load_b128 v[12:15], v12, s[4:5] offset:16
+; GFX12-NEXT:    global_load_b128 v[12:15], v12, s[6:7] offset:16
 ; GFX12-NEXT:    s_wait_loadcnt 0x2
-; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v2, v6
+; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v6, v2
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_sub_co_ci_u32_e64 v3, null, v3, v7, vcc_lo
+; GFX12-NEXT:    v_sub_co_ci_u32_e64 v3, null, v7, v3, vcc_lo
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    v_sub_co_u32 v10, vcc_lo, v10, v14
 ; GFX12-NEXT:    s_wait_alu 0xfffd
@@ -1016,9 +1016,9 @@ define amdgpu_kernel void @v_test_sub_v4i64(ptr addrspace(1) %out, ptr addrspace
 ; GFX12-NEXT:    v_sub_co_u32 v8, vcc_lo, v8, v12
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_sub_co_ci_u32_e64 v9, null, v9, v13, vcc_lo
-; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v4
+; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v4, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_sub_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-NEXT:    v_sub_co_ci_u32_e64 v1, null, v5, v1, vcc_lo
 ; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    global_store_b128 v16, v[8:11], s[0:1] offset:16
 ; GFX12-NEXT:    global_store_b128 v16, v[0:3], s[0:1]
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 5acbb044c105..e9017939f8a4 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -348,10 +348,9 @@ define i64 @v_test_udiv_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    v_lshr_b64 v[8:9], v[0:1], v10
 ; GCN-IR-NEXT:    v_addc_u32_e32 v13, vcc, -1, v3, vcc
 ; GCN-IR-NEXT:    v_not_b32_e32 v0, v14
-; GCN-IR-NEXT:    v_not_b32_e32 v1, 0
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v15
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
-; GCN-IR-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[4:5], -1, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem.ll
index a56346f3bb45..74e536f81371 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem.ll
@@ -37,22 +37,23 @@ define amdgpu_kernel void @test_udivrem(ptr addrspace(1) %out0, [8 x i32], ptr a
 ; GFX6-LABEL: test_udivrem:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dword s8, s[4:5], 0x26
-; GFX6-NEXT:    s_load_dword s9, s[4:5], 0x1d
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s8
 ; GFX6-NEXT:    s_sub_i32 s0, 0, s8
-; GFX6-NEXT:    s_mov_b32 s7, s3
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_lo_u32 v1, s0, v0
+; GFX6-NEXT:    s_load_dword s9, s[4:5], 0x1d
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x13
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_mul_hi_u32 v0, s9, v0
 ; GFX6-NEXT:    v_readfirstlane_b32 s10, v0
 ; GFX6-NEXT:    s_mul_i32 s10, s10, s8
@@ -69,7 +70,6 @@ define amdgpu_kernel void @test_udivrem(ptr addrspace(1) %out0, [8 x i32], ptr a
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; GFX6-NEXT:    s_cselect_b32 s8, s10, s9
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    v_mov_b32_e32 v0, s8
@@ -79,7 +79,6 @@ define amdgpu_kernel void @test_udivrem(ptr addrspace(1) %out0, [8 x i32], ptr a
 ; GFX8-LABEL: test_udivrem:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dword s6, s[4:5], 0x98
-; GFX8-NEXT:    s_load_dword s7, s[4:5], 0x74
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s6
 ; GFX8-NEXT:    s_sub_i32 s0, 0, s6
@@ -87,6 +86,7 @@ define amdgpu_kernel void @test_udivrem(ptr addrspace(1) %out0, [8 x i32], ptr a
 ; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX8-NEXT:    v_mul_lo_u32 v1, s0, v0
+; GFX8-NEXT:    s_load_dword s7, s[4:5], 0x74
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x4c
 ; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/uncalled-local-functions.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/uncalled-local-functions.ll
index a0023b70f28b..6cefcedaf02f 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/uncalled-local-functions.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/uncalled-local-functions.ll
@@ -22,7 +22,7 @@ define internal i32 @aliased_internal_func() {
 ; CHECK-LABEL: take_alias_addr
 ; CHECK:      Function info:
 ; CHECK-NEXT: codeLenInByte = 60
-; CHECK-NEXT: NumSgprs: 37
+; CHECK-NEXT: TotalNumSgprs: 37
 ; CHECK-NEXT: NumVgprs: 1
 ; CHECK-NEXT: NumAgprs: 0
 ; CHECK-NEXT: TotalNumVgprs: 1
@@ -37,7 +37,7 @@ define void @take_alias_addr() {
 ; CHECK: aliased_taken_func
 ; CHECK:      Function info:
 ; CHECK-NEXT: codeLenInByte = 12
-; CHECK-NEXT: NumSgprs: 36
+; CHECK-NEXT: TotalNumSgprs: 36
 ; CHECK-NEXT: NumVgprs: 1
 ; CHECK-NEXT: NumAgprs: 0
 ; CHECK-NEXT: TotalNumVgprs: 1
@@ -50,7 +50,7 @@ define internal i32 @aliased_taken_func() {
 ; CHECK-LABEL: addr_taken
 ; CHECK:      Function info:
 ; CHECK-NEXT: codeLenInByte = 12
-; CHECK-NEXT: NumSgprs: 36
+; CHECK-NEXT: TotalNumSgprs: 36
 ; CHECK-NEXT: NumVgprs: 1
 ; CHECK-NEXT: NumAgprs: 0
 ; CHECK-NEXT: TotalNumVgprs: 1
@@ -63,7 +63,7 @@ define internal i32 @addr_taken() {
 ; CHECK-LABEL: non_local
 ; CHECK:      Function info:
 ; CHECK-NEXT: codeLenInByte = 12
-; CHECK-NEXT: NumSgprs: 36
+; CHECK-NEXT: TotalNumSgprs: 36
 ; CHECK-NEXT: NumVgprs: 1
 ; CHECK-NEXT: NumAgprs: 0
 ; CHECK-NEXT: TotalNumVgprs: 1
@@ -76,7 +76,7 @@ define i32 @non_local() {
 ; CHECK-LABEL: take_addr
 ; CHECK:      Function info:
 ; CHECK-NEXT: codeLenInByte = 60
-; CHECK-NEXT: NumSgprs: 37
+; CHECK-NEXT: TotalNumSgprs: 37
 ; CHECK-NEXT: NumVgprs: 1
 ; CHECK-NEXT: NumAgprs: 0
 ; CHECK-NEXT: TotalNumVgprs: 1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/unfold-masked-merge-scalar-variablemask.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/unfold-masked-merge-scalar-variablemask.ll
index eef8d1d99bb6..c871293de743 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/unfold-masked-merge-scalar-variablemask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/unfold-masked-merge-scalar-variablemask.ll
@@ -5,10 +5,11 @@ define i32 @s_out32(i32 inreg %x, i32 inreg %y, i32 inreg %mask) {
 ; GCN-LABEL: s_out32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_and_b32 s0, s0, s2
-; GCN-NEXT:    s_and_not1_b32 s1, s1, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
 ; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s0, s0, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %mx = and i32 %x, %mask
@@ -22,10 +23,11 @@ define i64 @s_out64(i64 inreg %x, i64 inreg %y, i64 inreg %mask) {
 ; GCN-LABEL: s_out64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], s[16:17]
-; GCN-NEXT:    s_and_not1_b64 s[2:3], s[2:3], s[16:17]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], s[16:17]
+; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GCN-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %mx = and i64 %x, %mask
@@ -427,10 +429,11 @@ define i32 @s_out_constant_varx_42(i32 inreg %x, i32 inreg %y, i32 inreg %mask)
 ; GCN-LABEL: s_out_constant_varx_42:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_and_b32 s0, s2, s0
-; GCN-NEXT:    s_and_not1_b32 s1, 42, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, 42
 ; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s0, s0, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, 42
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %notmask = xor i32 %mask, -1
@@ -462,10 +465,11 @@ define i32 @s_out_constant_varx_42_invmask(i32 inreg %x, i32 inreg %y, i32 inreg
 ; GCN-LABEL: s_out_constant_varx_42_invmask:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_and_not1_b32 s0, s0, s2
-; GCN-NEXT:    s_and_b32 s1, s2, 42
+; GCN-NEXT:    s_xor_b32 s1, s0, 42
 ; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s1, s2
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %notmask = xor i32 %mask, -1
@@ -560,10 +564,11 @@ define i32 @s_out_constant_42_vary(i32 inreg %x, i32 inreg %y, i32 inreg %mask)
 ; GCN-LABEL: s_out_constant_42_vary:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_and_b32 s0, s2, 42
-; GCN-NEXT:    s_and_not1_b32 s1, s1, s2
+; GCN-NEXT:    s_xor_b32 s0, s1, 42
 ; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s0, s0, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %notmask = xor i32 %mask, -1
@@ -595,10 +600,11 @@ define i32 @s_out_constant_42_vary_invmask(i32 inreg %x, i32 inreg %y, i32 inreg
 ; GCN-LABEL: s_out_constant_42_vary_invmask:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_and_not1_b32 s0, 42, s2
-; GCN-NEXT:    s_and_b32 s1, s2, s1
+; GCN-NEXT:    s_xor_b32 s0, s1, 42
 ; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s0, s0, s2
+; GCN-NEXT:    s_xor_b32 s0, s0, 42
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %notmask = xor i32 %mask, -1
diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll
index 94f1b83ea276..6480a88d40f5 100644
--- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -355,12 +355,11 @@ define i64 @v_test_urem_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v14, vcc, -1, v2
 ; GCN-IR-NEXT:    v_addc_u32_e32 v15, vcc, -1, v3, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v7, v12
+; GCN-IR-NEXT:    v_not_b32_e32 v6, v12
 ; GCN-IR-NEXT:    v_lshr_b64 v[10:11], v[0:1], v8
-; GCN-IR-NEXT:    v_not_b32_e32 v6, 0
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v7, v13
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v6, v13
 ; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v6, vcc
+; GCN-IR-NEXT:    v_addc_u32_e64 v9, s[4:5], -1, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v13, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/alias_store.ll b/external/llvm-project/llvm/test/CodeGen/ARM/alias_store.ll
index c6612334eaf1..60aa58d37499 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/alias_store.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/alias_store.ll
@@ -13,4 +13,4 @@ entry:
 ; CHECK: ldr r{{.*}}, [[L:.*]]
 ; CHECK: [[L]]:
 ; CHECK-NEXT: .long XA
-; CHECK: .set XA, X+1
+; CHECK: XA = X+1
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/aliases.ll b/external/llvm-project/llvm/test/CodeGen/ARM/aliases.ll
index 6075ad813e99..8d9f938155d1 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/aliases.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/aliases.ll
@@ -6,30 +6,30 @@
 ; CHECK: .size .Lstructvar, 8
 
 ; CHECK: .globl	foo1
-; CHECK: .set foo1, bar
+; CHECK: foo1 = bar
 ; CHECK-NOT: .size foo1
 
 ; CHECK: .globl	foo2
-; CHECK: .set foo2, bar
+; CHECK: foo2 = bar
 ; CHECK-NOT: .size foo2
 
 ; CHECK: .weak	bar_f
-; CHECK: .set bar_f, foo_f
+; CHECK: bar_f = foo_f
 ; CHECK-NOT: .size bar_f
 
-; CHECK: .set bar_i, bar
+; CHECK: bar_i = bar
 ; CHECK-NOT: .size bar_i
 
 ; CHECK: .globl	A
-; CHECK: .set A, bar
+; CHECK: A = bar
 ; CHECK-NOT: .size A
 
 ; CHECK: .globl elem0
-; CHECK: .set elem0, .Lstructvar
+; CHECK: elem0 = .Lstructvar
 ; CHECK: .size elem0, 4
 
 ; CHECK: .globl elem1
-; CHECK: .set elem1, .Lstructvar+4
+; CHECK: elem1 = .Lstructvar+4
 ; CHECK: .size elem1, 4
 
 @bar = global i32 42
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/float-helpers.s b/external/llvm-project/llvm/test/CodeGen/ARM/float-helpers.ll
similarity index 100%
rename from external/llvm-project/llvm/test/CodeGen/ARM/float-helpers.s
rename to external/llvm-project/llvm/test/CodeGen/ARM/float-helpers.ll
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll b/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll
index 996b46c51ab3..52fe5ce1a8a5 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll
@@ -517,9 +517,9 @@ define half @fp16_vminmaxnm_e_0(half %a) {
 ; CHECK-NEXT:    .short 0x0000 @ half 0
 entry:
   %cmp1 = fcmp nsz ole half 0., %a
-  %cond1 = select i1 %cmp1, half 0., half %a
+  %cond1 = select nsz i1 %cmp1, half 0., half %a
   %cmp2 = fcmp nsz uge half 0., %cond1
-  %cond2 = select i1 %cmp2, half 0., half %cond1
+  %cond2 = select nsz i1 %cmp2, half 0., half %cond1
   ret half %cond2
 }
 
@@ -540,7 +540,7 @@ define half @fp16_vminmaxnm_e_neg0(half %a) {
 ; CHECK-NEXT:    .short 0x8000 @ half -0
 entry:
   %cmp1 = fcmp nsz ule half -0., %a
-  %cond1 = select i1 %cmp1, half -0., half %a
+  %cond1 = select nsz i1 %cmp1, half -0., half %a
   %cmp2 = fcmp nsz oge half -0., %cond1
   %cond2 = select i1 %cmp2, half -0., half %cond1
   ret half %cond2
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-vector.ll b/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-vector.ll
index 6a61bb594b43..0b41c738080a 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-vector.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm-vector.ll
@@ -10,7 +10,7 @@ define <4 x half> @test1(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ogt <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -19,7 +19,7 @@ define <4 x half> @test2(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ogt <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -28,7 +28,7 @@ define <4 x half> @test3(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast oge <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -37,7 +37,7 @@ define <4 x half> @test4(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast oge <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -46,7 +46,7 @@ define <4 x half> @test5(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast olt <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -55,7 +55,7 @@ define <4 x half> @test6(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast olt <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -64,7 +64,7 @@ define <4 x half> @test7(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ole <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -73,7 +73,7 @@ define <4 x half> @test8(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ole <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -84,7 +84,7 @@ define <4 x half> @test11(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ugt <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -93,7 +93,7 @@ define <4 x half> @test12(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ugt <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -102,7 +102,7 @@ define <4 x half> @test13(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast uge <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -111,7 +111,7 @@ define <4 x half> @test14(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast uge <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -120,7 +120,7 @@ define <4 x half> @test15(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ult <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -129,7 +129,7 @@ define <4 x half> @test16(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ult <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -138,7 +138,7 @@ define <4 x half> @test17(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vminnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ule <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %A, <4 x half> %B
   ret <4 x half> %tmp4
 }
 
@@ -147,7 +147,7 @@ define <4 x half> @test18(<4 x half> %A, <4 x half> %B) {
 ; CHECK:         vmaxnm.f16 d0, d0, d1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ule <4 x half> %A, %B
-  %tmp4 = select <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
+  %tmp4 = select nsz <4 x i1> %tmp3, <4 x half> %B, <4 x half> %A
   ret <4 x half> %tmp4
 }
 
@@ -160,7 +160,7 @@ define <8 x half> @test201(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ogt <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -169,7 +169,7 @@ define <8 x half> @test202(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ogt <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
 
@@ -178,7 +178,7 @@ define <8 x half> @test203(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast oge <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -187,7 +187,7 @@ define <8 x half> @test204(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast oge <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
 
@@ -196,7 +196,7 @@ define <8 x half> @test205(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast olt <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -205,7 +205,7 @@ define <8 x half> @test206(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast olt <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
 
@@ -214,7 +214,7 @@ define <8 x half> @test207(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ole <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -223,7 +223,7 @@ define <8 x half> @test208(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ole <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
 
@@ -234,7 +234,7 @@ define <8 x half> @test209(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ugt <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -243,7 +243,7 @@ define <8 x half> @test210(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ugt <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
 
@@ -252,7 +252,7 @@ define <8 x half> @test211(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast uge <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -261,7 +261,7 @@ define <8 x half> @test214(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast uge <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
 
@@ -270,7 +270,7 @@ define <8 x half> @test215(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ult <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -279,7 +279,7 @@ define <8 x half> @test216(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ult <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
 
@@ -288,7 +288,7 @@ define <8 x half> @test217(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vminnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ule <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %A, <8 x half> %B
   ret <8 x half> %tmp4
 }
 
@@ -297,6 +297,6 @@ define <8 x half> @test218(<8 x half> %A, <8 x half> %B) {
 ; CHECK:         vmaxnm.f16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
   %tmp3 = fcmp fast ule <8 x half> %A, %B
-  %tmp4 = select <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
+  %tmp4 = select nsz <8 x i1> %tmp3, <8 x half> %B, <8 x half> %A
   ret <8 x half> %tmp4
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm.ll b/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm.ll
index 525c27be4f97..33ff71e8c473 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/fp16-vminmaxnm.ll
@@ -16,7 +16,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast olt half %0, %1
-  %cond = select i1 %cmp, half %0, half %1
+  %cond = select nsz i1 %cmp, half %0, half %1
   ret half %cond
 }
 
@@ -29,7 +29,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ogt half %0, %1
-  %cond = select i1 %cmp, half %1, half %0
+  %cond = select nsz i1 %cmp, half %1, half %0
   ret half %cond
 }
 
@@ -42,7 +42,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ult half %0, %1
-  %cond = select i1 %cmp, half %0, half %1
+  %cond = select nsz i1 %cmp, half %0, half %1
   ret half %cond
 }
 
@@ -55,7 +55,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ule half %0, %1
-  %cond = select i1 %cmp, half %0, half %1
+  %cond = select nsz i1 %cmp, half %0, half %1
   ret half %cond
 }
 
@@ -68,7 +68,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ugt half %0, %1
-  %cond = select i1 %cmp, half %1, half %0
+  %cond = select nsz i1 %cmp, half %1, half %0
   ret half %cond
 }
 
@@ -81,7 +81,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ogt half %0, %1
-  %cond = select i1 %cmp, half %0, half %1
+  %cond = select nsz i1 %cmp, half %0, half %1
   ret half %cond
 }
 
@@ -94,7 +94,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast oge half %0, %1
-  %cond = select i1 %cmp, half %0, half %1
+  %cond = select nsz i1 %cmp, half %0, half %1
   ret half %cond
 }
 
@@ -107,7 +107,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast olt half %0, %1
-  %cond = select i1 %cmp, half %1, half %0
+  %cond = select nsz i1 %cmp, half %1, half %0
   ret half %cond
 }
 
@@ -120,7 +120,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ole half %0, %1
-  %cond = select i1 %cmp, half %1, half %0
+  %cond = select nsz i1 %cmp, half %1, half %0
   ret half %cond
 }
 
@@ -133,7 +133,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ugt half %0, %1
-  %cond = select i1 %cmp, half %0, half %1
+  %cond = select nsz i1 %cmp, half %0, half %1
   ret half %cond
 }
 
@@ -146,7 +146,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast uge half %0, %1
-  %cond = select i1 %cmp, half %0, half %1
+  %cond = select nsz i1 %cmp, half %0, half %1
   ret half %cond
 }
 
@@ -159,7 +159,7 @@ entry:
   %0 = bitcast i16 %a to half
   %1 = bitcast i16 %b to half
   %cmp = fcmp fast ult half %0, %1
-  %cond = select i1 %cmp, half %1, half %0
+  %cond = select nsz i1 %cmp, half %1, half %0
   ret half %cond
 }
 
@@ -175,9 +175,9 @@ define half @fp16_vminnm_NNNo(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast olt half %0, 12.
-  %cond1 = select i1 %cmp1, half %0, half 12.
+  %cond1 = select nsz i1 %cmp1, half %0, half 12.
   %cmp2 = fcmp fast olt half 34., %cond1
-  %cond2 = select i1 %cmp2, half 34., half %cond1
+  %cond2 = select nsz i1 %cmp2, half 34., half %cond1
   ret half %cond2
 }
 
@@ -191,9 +191,9 @@ define half @fp16_vminnm_NNNo_rev(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast ogt half %0, 56.
-  %cond1 = select i1 %cmp1, half 56., half %0
+  %cond1 = select nsz i1 %cmp1, half 56., half %0
   %cmp2 = fcmp fast ogt half 78., %cond1
-  %cond2 = select i1 %cmp2, half %cond1, half 78.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 78.
   ret half %cond2
 }
 
@@ -207,9 +207,9 @@ define half @fp16_vminnm_NNNu(i16 signext %b) {
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp fast ult half 12., %0
-  %cond1 = select i1 %cmp1, half 12., half %0
+  %cond1 = select nsz i1 %cmp1, half 12., half %0
   %cmp2 = fcmp fast ult half %cond1, 34.
-  %cond2 = select i1 %cmp2, half %cond1, half 34.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 34.
   ret half %cond2
 }
 
@@ -223,9 +223,9 @@ define half @fp16_vminnm_NNNule(i16 signext %b) {
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp fast ule half 34., %0
-  %cond1 = select i1 %cmp1, half 34., half %0
+  %cond1 = select nsz i1 %cmp1, half 34., half %0
   %cmp2 = fcmp fast ule half %cond1, 56.
-  %cond2 = select i1 %cmp2, half %cond1, half 56.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 56.
   ret half %cond2
 }
 
@@ -239,9 +239,9 @@ define half @fp16_vminnm_NNNu_rev(i16 signext %b) {
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp fast ugt half 56., %0
-  %cond1 = select i1 %cmp1, half %0, half 56.
+  %cond1 = select nsz i1 %cmp1, half %0, half 56.
   %cmp2 = fcmp fast ugt half %cond1, 78.
-  %cond2 = select i1 %cmp2, half 78., half %cond1
+  %cond2 = select nsz i1 %cmp2, half 78., half %cond1
   ret half %cond2
 }
 
@@ -255,9 +255,9 @@ define half @fp16_vmaxnm_NNNo(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast ogt half %0, 12.
-  %cond1 = select i1 %cmp1, half %0, half 12.
+  %cond1 = select nsz i1 %cmp1, half %0, half 12.
   %cmp2 = fcmp fast ogt half 34., %cond1
-  %cond2 = select i1 %cmp2, half 34., half %cond1
+  %cond2 = select nsz i1 %cmp2, half 34., half %cond1
   ret half %cond2
 }
 
@@ -271,9 +271,9 @@ define half @fp16_vmaxnm_NNNoge(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast oge half %0, 34.
-  %cond1 = select i1 %cmp1, half %0, half 34.
+  %cond1 = select nsz i1 %cmp1, half %0, half 34.
   %cmp2 = fcmp fast oge half 56., %cond1
-  %cond2 = select i1 %cmp2, half 56., half %cond1
+  %cond2 = select nsz i1 %cmp2, half 56., half %cond1
   ret half %cond2
 }
 
@@ -287,9 +287,9 @@ define half @fp16_vmaxnm_NNNo_rev(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast olt half %0, 56.
-  %cond1 = select i1 %cmp1, half 56., half %0
+  %cond1 = select nsz i1 %cmp1, half 56., half %0
   %cmp2 = fcmp fast olt half 78., %cond1
-  %cond2 = select i1 %cmp2, half %cond1, half 78.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 78.
   ret half %cond2
 }
 
@@ -303,9 +303,9 @@ define half @fp16_vmaxnm_NNNole_rev(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast ole half %0, 78.
-  %cond1 = select i1 %cmp1, half 78., half %0
+  %cond1 = select nsz i1 %cmp1, half 78., half %0
   %cmp2 = fcmp fast ole half 90., %cond1
-  %cond2 = select i1 %cmp2, half %cond1, half 90.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 90.
   ret half %cond2
 }
 
@@ -319,9 +319,9 @@ define half @fp16_vmaxnm_NNNu(i16 signext %b) {
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp fast ugt half 12., %0
-  %cond1 = select i1 %cmp1, half 12., half %0
+  %cond1 = select nsz i1 %cmp1, half 12., half %0
   %cmp2 = fcmp fast ugt half %cond1, 34.
-  %cond2 = select i1 %cmp2, half %cond1, half 34.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 34.
   ret half %cond2
 }
 
@@ -335,9 +335,9 @@ define half @fp16_vmaxnm_NNNuge(i16 signext %b) {
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp fast uge half 34., %0
-  %cond1 = select i1 %cmp1, half 34., half %0
+  %cond1 = select nsz i1 %cmp1, half 34., half %0
   %cmp2 = fcmp fast uge half %cond1, 56.
-  %cond2 = select i1 %cmp2, half %cond1, half 56.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 56.
   ret half %cond2
 }
 
@@ -351,9 +351,9 @@ define half @fp16_vmaxnm_NNNu_rev(i16 signext %b) {
 entry:
   %0 = bitcast i16 %b to half
   %cmp1 = fcmp fast ult half 56., %0
-  %cond1 = select i1 %cmp1, half %0, half 56.
+  %cond1 = select nsz i1 %cmp1, half %0, half 56.
   %cmp2 = fcmp fast ult half %cond1, 78.
-  %cond2 = select i1 %cmp2, half 78., half %cond1
+  %cond2 = select nsz i1 %cmp2, half 78., half %cond1
   ret half %cond2
 }
 
@@ -366,9 +366,9 @@ define half @fp16_vminmaxnm_0(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast olt half %0, 0.
-  %cond1 = select i1 %cmp1, half %0, half 0.
+  %cond1 = select nsz i1 %cmp1, half %0, half 0.
   %cmp2 = fcmp fast ogt half %cond1, 0.
-  %cond2 = select i1 %cmp2, half %cond1, half 0.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half 0.
   ret half %cond2
 }
 
@@ -381,9 +381,9 @@ define half @fp16_vminmaxnm_neg0(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast olt half %0, -0.
-  %cond1 = select i1 %cmp1, half %0, half -0.
+  %cond1 = select nsz i1 %cmp1, half %0, half -0.
   %cmp2 = fcmp fast ugt half %cond1, -0.
-  %cond2 = select i1 %cmp2, half %cond1, half -0.
+  %cond2 = select nsz i1 %cmp2, half %cond1, half -0.
   ret half %cond2
 }
 
@@ -396,9 +396,9 @@ define half @fp16_vminmaxnm_e_0(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast ule half 0., %0
-  %cond1 = select i1 %cmp1, half 0., half %0
+  %cond1 = select nsz i1 %cmp1, half 0., half %0
   %cmp2 = fcmp fast uge half 0., %cond1
-  %cond2 = select i1 %cmp2, half 0., half %cond1
+  %cond2 = select nsz i1 %cmp2, half 0., half %cond1
   ret half %cond2
 }
 
@@ -411,8 +411,8 @@ define half @fp16_vminmaxnm_e_neg0(i16 signext %a) {
 entry:
   %0 = bitcast i16 %a to half
   %cmp1 = fcmp fast ule half -0., %0
-  %cond1 = select i1 %cmp1, half -0., half %0
+  %cond1 = select nsz i1 %cmp1, half -0., half %0
   %cmp2 = fcmp fast oge half -0., %cond1
-  %cond2 = select i1 %cmp2, half -0., half %cond1
+  %cond2 = select nsz i1 %cmp2, half -0., half %cond1
   ret half %cond2
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/fpcmp_ueq.ll b/external/llvm-project/llvm/test/CodeGen/ARM/fpcmp_ueq.ll
index 698c7506cc59..f77720fd935f 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/fpcmp_ueq.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/fpcmp_ueq.ll
@@ -9,12 +9,13 @@ entry:
 }
 
 ; CHECK-ARMv4-LABEL: f7:
-; CHECK-ARMv4-DAG: bl ___eqsf2
-; CHECK-ARMv4-DAG: bl ___unordsf2
-; CHECK-ARMv4: cmp r0, #0
-; CHECK-ARMv4: movne r0, #1
-; CHECK-ARMv4: orrs r0, r0,
-; CHECK-ARMv4: moveq r0, #42
+; CHECK-ARMv4: bl ___eqsf2
+; CHECK-ARMv4-NEXT: rsbs r1, r0, #0
+; CHECK-ARMv4-NEXT: adc	r6, r0, r1
+
+; CHECK-ARMv4: bl ___unordsf2
+; CHECK-ARMv4-NEXT: orrs r0, r0, r6
+; CHECK-ARMv4-NEXT: mov r0, #154
 
 ; CHECK-ARMv7-LABEL: f7:
 ; CHECK-ARMv7: vcmp.f32
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-dllexport.ll b/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-dllexport.ll
index 89e8a859b939..f5961d7f79e3 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-dllexport.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-dllexport.ll
@@ -16,6 +16,6 @@ define void @f1(i32 %a1, i32 %a2) {
 ; CHECK: .section .drectve,"yni"
 ; CHECK: .ascii " /EXPORT:y,DATA"
 ; CHECK: .globl x
-; CHECK: .set x, .L_MergedGlobals
+; CHECK: x = .L_MergedGlobals
 ; CHECK: .globl y
-; CHECK: .set y, .L_MergedGlobals+4
+; CHECK: y = .L_MergedGlobals+4
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external-2.ll b/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external-2.ll
index 602533e045e0..c9e92d98e484 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external-2.ll
@@ -50,16 +50,16 @@ define dso_local void @g1(i32 %a1, i32 %a2) {
 ;CHECK-WIN32:   .lcomm  .L_MergedGlobals,8,4
 
 ;CHECK-MERGE:   .globl  x
-;CHECK-MERGE: .set x, .L_MergedGlobals
+;CHECK-MERGE: x = .L_MergedGlobals
 ;CHECK-MERGE: .size x, 4
 ;CHECK-MERGE:   .globl  y
-;CHECK-MERGE: .set y, .L_MergedGlobals+4
+;CHECK-MERGE: y = .L_MergedGlobals+4
 ;CHECK-MERGE: .size y, 4
-;CHECK-MERGE-NOT: .set z, .L_MergedGlobals+8
+;CHECK-MERGE-NOT: z = .L_MergedGlobals+8
 
 
 ;CHECK-WIN32:   .globl  x
-;CHECK-WIN32: .set x, .L_MergedGlobals
+;CHECK-WIN32: x = .L_MergedGlobals
 ;CHECK-WIN32:   .globl  y
-;CHECK-WIN32: .set y, .L_MergedGlobals+4
-;CHECK-WIN32-NOT: .set z, .L_MergedGlobals+8
+;CHECK-WIN32: y = .L_MergedGlobals+4
+;CHECK-WIN32-NOT: z = .L_MergedGlobals+8
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external.ll b/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external.ll
index 364659b36bb9..4fe1914aae35 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/global-merge-external.ll
@@ -45,18 +45,18 @@ define dso_local void @g1(i32 %a1, i32 %a2) {
 ;CHECK-WIN32:	.lcomm	.L_MergedGlobals,12,4
 
 ;CHECK-MERGE:	.globl	x
-;CHECK-MERGE: .set x, .L_MergedGlobals
+;CHECK-MERGE: x = .L_MergedGlobals
 ;CHECK-MERGE: .size x, 4
 ;CHECK-MERGE:	.globl	y
-;CHECK-MERGE: .set y, .L_MergedGlobals+4
+;CHECK-MERGE: y = .L_MergedGlobals+4
 ;CHECK-MERGE: .size y, 4
 ;CHECK-MERGE:	.globl	z
-;CHECK-MERGE: .set z, .L_MergedGlobals+8
+;CHECK-MERGE: z = .L_MergedGlobals+8
 ;CHECK-MERGE: .size z, 4
 
 ;CHECK-WIN32:	.globl	x
-;CHECK-WIN32: .set x, .L_MergedGlobals
+;CHECK-WIN32: x = .L_MergedGlobals
 ;CHECK-WIN32:	.globl	y
-;CHECK-WIN32: .set y, .L_MergedGlobals+4
+;CHECK-WIN32: y = .L_MergedGlobals+4
 ;CHECK-WIN32:	.globl	z
-;CHECK-WIN32: .set z, .L_MergedGlobals+8
+;CHECK-WIN32: z = .L_MergedGlobals+8
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll b/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll
index feb23ea1f398..5577ab49bb83 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll
@@ -44,7 +44,7 @@ define float @fp-armv8_vminnm_o(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_o":
 ; CHECK-NOT: vminnm.f32
   %cmp = fcmp olt float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -52,7 +52,7 @@ define double @fp-armv8_vminnm_ole(double %a, double %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_ole":
 ; CHECK-NOT: vminnm.f64
   %cmp = fcmp ole double %a, %b
-  %cond = select i1 %cmp, double %a, double %b
+  %cond = select nsz i1 %cmp, double %a, double %b
   ret double %cond
 }
 
@@ -60,7 +60,7 @@ define float @fp-armv8_vminnm_o_rev(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_o_rev":
 ; CHECK-NOT: vminnm.f32
   %cmp = fcmp ogt float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -68,7 +68,7 @@ define double @fp-armv8_vminnm_oge_rev(double %a, double %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_oge_rev":
 ; CHECK-NOT: vminnm.f64
   %cmp = fcmp oge double %a, %b
-  %cond = select i1 %cmp, double %b, double %a
+  %cond = select nsz i1 %cmp, double %b, double %a
   ret double %cond
 }
 
@@ -76,7 +76,7 @@ define float @fp-armv8_vminnm_u(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_u":
 ; CHECK-NOT: vminnm.f32
   %cmp = fcmp ult float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -84,7 +84,7 @@ define float @fp-armv8_vminnm_ule(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_ule":
 ; CHECK-NOT: vminnm.f32
   %cmp = fcmp ule float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -92,7 +92,7 @@ define float @fp-armv8_vminnm_u_rev(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_u_rev":
 ; CHECK-NOT: vminnm.f32
   %cmp = fcmp ugt float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -100,7 +100,7 @@ define double @fp-armv8_vminnm_uge_rev(double %a, double %b) {
 ; CHECK-LABEL: "fp-armv8_vminnm_uge_rev":
 ; CHECK-NOT: vminnm.f64
   %cmp = fcmp uge double %a, %b
-  %cond = select i1 %cmp, double %b, double %a
+  %cond = select nsz i1 %cmp, double %b, double %a
   ret double %cond
 }
 
@@ -108,7 +108,7 @@ define float @fp-armv8_vmaxnm_o(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_o":
 ; CHECK-NOT: vmaxnm.f32
   %cmp = fcmp ogt float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -116,7 +116,7 @@ define float @fp-armv8_vmaxnm_oge(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_oge":
 ; CHECK-NOT: vmaxnm.f32
   %cmp = fcmp oge float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -124,7 +124,7 @@ define float @fp-armv8_vmaxnm_o_rev(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_o_rev":
 ; CHECK-NOT: vmaxnm.f32
   %cmp = fcmp olt float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -132,7 +132,7 @@ define float @fp-armv8_vmaxnm_ole_rev(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_ole_rev":
 ; CHECK-NOT: vmaxnm.f32
   %cmp = fcmp ole float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -140,7 +140,7 @@ define float @fp-armv8_vmaxnm_u(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_u":
 ; CHECK-NOT: vmaxnm.f32
   %cmp = fcmp ugt float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -148,7 +148,7 @@ define float @fp-armv8_vmaxnm_uge(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_uge":
 ; CHECK-NOT: vmaxnm.f32
   %cmp = fcmp uge float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -156,7 +156,7 @@ define float @fp-armv8_vmaxnm_u_rev(float %a, float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_u_rev":
 ; CHECK-NOT: vmaxnm.f32
   %cmp = fcmp ult float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -164,7 +164,7 @@ define double @fp-armv8_vmaxnm_ule_rev(double %a, double %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_ule_rev":
 ; CHECK-NOT: vmaxnm.f64
   %cmp = fcmp ule double %a, %b
-  %cond = select i1 %cmp, double %b, double %a
+  %cond = select nsz i1 %cmp, double %b, double %a
   ret double %cond
 }
 
@@ -175,7 +175,7 @@ define float @fp-armv8_vminnm_NNNo(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK-NOT: vminnm.f32
   %cmp1 = fcmp olt float %a, 12.
-  %cond1 = select i1 %cmp1, float %a, float 12.
+  %cond1 = select nsz i1 %cmp1, float %a, float 12.
   %cmp2 = fcmp olt float 34., %cond1
   %cond2 = select i1 %cmp2, float 34., float %cond1
   ret float %cond2
@@ -186,7 +186,7 @@ define double @fp-armv8_vminnm_NNNole(double %a) {
 ; CHECK: vminnm.f64
 ; CHECK-NOT: vminnm.f64
   %cmp1 = fcmp ole double %a, 34.
-  %cond1 = select i1 %cmp1, double %a, double 34.
+  %cond1 = select nsz i1 %cmp1, double %a, double 34.
   %cmp2 = fcmp ole double 56., %cond1
   %cond2 = select i1 %cmp2, double 56., double %cond1
   ret double %cond2
@@ -197,9 +197,9 @@ define float @fp-armv8_vminnm_NNNo_rev(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK-NOT: vminnm.f32
   %cmp1 = fcmp ogt float %a, 56.
-  %cond1 = select i1 %cmp1, float 56., float %a
+  %cond1 = select nsz i1 %cmp1, float 56., float %a
   %cmp2 = fcmp ogt float 78., %cond1
-  %cond2 = select i1 %cmp2, float %cond1, float 78.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 78.
   ret float %cond2
 }
 
@@ -208,9 +208,9 @@ define double @fp-armv8_vminnm_NNNoge_rev(double %a) {
 ; CHECK: vminnm.f64
 ; CHECK-NOT: vminnm.f64
   %cmp1 = fcmp oge double %a, 78.
-  %cond1 = select i1 %cmp1, double 78., double %a
+  %cond1 = select nsz i1 %cmp1, double 78., double %a
   %cmp2 = fcmp oge double 90., %cond1
-  %cond2 = select i1 %cmp2, double %cond1, double 90.
+  %cond2 = select nsz i1 %cmp2, double %cond1, double 90.
   ret double %cond2
 }
 
@@ -219,7 +219,7 @@ define float @fp-armv8_vminnm_NNNu(float %b) {
 ; CHECK: vminnm.f32
 ; CHECK-NOT: vminnm.f32
   %cmp1 = fcmp ult float 12., %b
-  %cond1 = select i1 %cmp1, float 12., float %b
+  %cond1 = select nsz i1 %cmp1, float 12., float %b
   %cmp2 = fcmp ult float %cond1, 34.
   %cond2 = select i1 %cmp2, float %cond1, float 34.
   ret float %cond2
@@ -230,7 +230,7 @@ define float @fp-armv8_vminnm_NNNule(float %b) {
 ; CHECK: vminnm.f32
 ; CHECK-NOT: vminnm.f32
   %cmp1 = fcmp ule float 34., %b
-  %cond1 = select i1 %cmp1, float 34., float %b
+  %cond1 = select nsz i1 %cmp1, float 34., float %b
   %cmp2 = fcmp ule float %cond1, 56.
   %cond2 = select i1 %cmp2, float %cond1, float 56.
   ret float %cond2
@@ -241,9 +241,9 @@ define float @fp-armv8_vminnm_NNNu_rev(float %b) {
 ; CHECK: vminnm.f32
 ; CHECK-NOT: vminnm.f32
   %cmp1 = fcmp ugt float 56., %b
-  %cond1 = select i1 %cmp1, float %b, float 56.
+  %cond1 = select nsz i1 %cmp1, float %b, float 56.
   %cmp2 = fcmp ugt float %cond1, 78.
-  %cond2 = select i1 %cmp2, float 78., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 78., float %cond1
   ret float %cond2
 }
 
@@ -252,9 +252,9 @@ define double @fp-armv8_vminnm_NNNuge_rev(double %b) {
 ; CHECK: vminnm.f64
 ; CHECK-NOT: vminnm.f64
   %cmp1 = fcmp uge double 78., %b
-  %cond1 = select i1 %cmp1, double %b, double 78.
+  %cond1 = select nsz i1 %cmp1, double %b, double 78.
   %cmp2 = fcmp uge double %cond1, 90.
-  %cond2 = select i1 %cmp2, double 90., double %cond1
+  %cond2 = select nsz i1 %cmp2, double 90., double %cond1
   ret double %cond2
 }
 
@@ -263,7 +263,7 @@ define float @fp-armv8_vmaxnm_NNNo(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp ogt float %a, 12.
-  %cond1 = select i1 %cmp1, float %a, float 12.
+  %cond1 = select nsz i1 %cmp1, float %a, float 12.
   %cmp2 = fcmp ogt float 34., %cond1
   %cond2 = select i1 %cmp2, float 34., float %cond1
   ret float %cond2
@@ -274,7 +274,7 @@ define float @fp-armv8_vmaxnm_NNNoge(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp oge float %a, 34.
-  %cond1 = select i1 %cmp1, float %a, float 34.
+  %cond1 = select nsz i1 %cmp1, float %a, float 34.
   %cmp2 = fcmp oge float 56., %cond1
   %cond2 = select i1 %cmp2, float 56., float %cond1
   ret float %cond2
@@ -285,9 +285,9 @@ define float @fp-armv8_vmaxnm_NNNo_rev(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp olt float %a, 56.
-  %cond1 = select i1 %cmp1, float 56., float %a
+  %cond1 = select nsz i1 %cmp1, float 56., float %a
   %cmp2 = fcmp olt float 78., %cond1
-  %cond2 = select i1 %cmp2, float %cond1, float 78.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 78.
   ret float %cond2
 }
 
@@ -296,9 +296,9 @@ define float @fp-armv8_vmaxnm_NNNole_rev(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp ole float %a, 78.
-  %cond1 = select i1 %cmp1, float 78., float %a
+  %cond1 = select nsz i1 %cmp1, float 78., float %a
   %cmp2 = fcmp ole float 90., %cond1
-  %cond2 = select i1 %cmp2, float %cond1, float 90.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 90.
   ret float %cond2
 }
 
@@ -307,7 +307,7 @@ define float @fp-armv8_vmaxnm_NNNu(float %b) {
 ; CHECK: vmaxnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp ugt float 12., %b
-  %cond1 = select i1 %cmp1, float 12., float %b
+  %cond1 = select nsz i1 %cmp1, float 12., float %b
   %cmp2 = fcmp ugt float %cond1, 34.
   %cond2 = select i1 %cmp2, float %cond1, float 34.
   ret float %cond2
@@ -318,7 +318,7 @@ define float @fp-armv8_vmaxnm_NNNuge(float %b) {
 ; CHECK: vmaxnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp uge float 34., %b
-  %cond1 = select i1 %cmp1, float 34., float %b
+  %cond1 = select nsz i1 %cmp1, float 34., float %b
   %cmp2 = fcmp uge float %cond1, 56.
   %cond2 = select i1 %cmp2, float %cond1, float 56.
   ret float %cond2
@@ -329,9 +329,9 @@ define float @fp-armv8_vmaxnm_NNNu_rev(float %b) {
 ; CHECK: vmaxnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp ult float 56., %b
-  %cond1 = select i1 %cmp1, float %b, float 56.
+  %cond1 = select nsz i1 %cmp1, float %b, float 56.
   %cmp2 = fcmp ult float %cond1, 78.
-  %cond2 = select i1 %cmp2, float 78., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 78., float %cond1
   ret float %cond2
 }
 
@@ -340,9 +340,9 @@ define double @fp-armv8_vmaxnm_NNNule_rev( double %b) {
 ; CHECK: vmaxnm.f64
 ; CHECK-NOT: vmaxnm.f64
   %cmp1 = fcmp ule double 78., %b
-  %cond1 = select i1 %cmp1, double %b, double 78.
+  %cond1 = select nsz i1 %cmp1, double %b, double 78.
   %cmp2 = fcmp ule double %cond1, 90.
-  %cond2 = select i1 %cmp2, double 90., double %cond1
+  %cond2 = select nsz i1 %cmp2, double 90., double %cond1
   ret double %cond2
 }
 
@@ -351,9 +351,9 @@ define float @fp-armv8_vminmaxnm_0(float %a) {
 ; CHECK-NOT: vminnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp ult float %a, 0.
-  %cond1 = select i1 %cmp1, float %a, float 0.
+  %cond1 = select nsz i1 %cmp1, float %a, float 0.
   %cmp2 = fcmp ogt float %cond1, 0.
-  %cond2 = select i1 %cmp2, float %cond1, float 0.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 0.
   ret float %cond2
 }
 
@@ -362,7 +362,7 @@ define float @fp-armv8_vminmaxnm_neg0(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp olt float %a, -0.
-  %cond1 = select i1 %cmp1, float %a, float -0.
+  %cond1 = select nsz i1 %cmp1, float %a, float -0.
   %cmp2 = fcmp ugt float %cond1, -0.
   %cond2 = select i1 %cmp2, float %cond1, float -0.
   ret float %cond2
@@ -373,9 +373,9 @@ define float @fp-armv8_vminmaxnm_e_0(float %a) {
 ; CHECK-NOT: vminnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp nsz ole float 0., %a
-  %cond1 = select i1 %cmp1, float 0., float %a
+  %cond1 = select nsz i1 %cmp1, float 0., float %a
   %cmp2 = fcmp nsz uge float 0., %cond1
-  %cond2 = select i1 %cmp2, float 0., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 0., float %cond1
   ret float %cond2
 }
 
@@ -384,7 +384,7 @@ define float @fp-armv8_vminmaxnm_e_neg0(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp nsz ule float -0., %a
-  %cond1 = select i1 %cmp1, float -0., float %a
+  %cond1 = select nsz i1 %cmp1, float -0., float %a
   %cmp2 = fcmp nsz oge float -0., %cond1
   %cond2 = select i1 %cmp2, float -0., float %cond1
   ret float %cond2
diff --git a/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm.ll b/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm.ll
index a6803fc78d8c..bb3ea3067541 100644
--- a/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/ARM/vminmaxnm.ll
@@ -7,7 +7,7 @@ define float @fp-armv8_vminnm_o(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f32
   %cmp = fcmp fast olt float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -16,7 +16,7 @@ define double @fp-armv8_vminnm_ole(double %a, double %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f64
   %cmp = fcmp fast ole double %a, %b
-  %cond = select i1 %cmp, double %a, double %b
+  %cond = select nsz i1 %cmp, double %a, double %b
   ret double %cond
 }
 
@@ -25,7 +25,7 @@ define float @fp-armv8_vminnm_o_rev(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f32
   %cmp = fcmp fast ogt float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -34,7 +34,7 @@ define double @fp-armv8_vminnm_oge_rev(double %a, double %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f64
   %cmp = fcmp fast oge double %a, %b
-  %cond = select i1 %cmp, double %b, double %a
+  %cond = select nsz i1 %cmp, double %b, double %a
   ret double %cond
 }
 
@@ -43,7 +43,7 @@ define float @fp-armv8_vminnm_u(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f32
   %cmp = fcmp fast ult float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -52,7 +52,7 @@ define float @fp-armv8_vminnm_ule(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f32
   %cmp = fcmp fast ule float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -61,7 +61,7 @@ define float @fp-armv8_vminnm_u_rev(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f32
   %cmp = fcmp fast ugt float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -70,7 +70,7 @@ define double @fp-armv8_vminnm_uge_rev(double %a, double %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vminnm.f64
   %cmp = fcmp fast uge double %a, %b
-  %cond = select i1 %cmp, double %b, double %a
+  %cond = select nsz i1 %cmp, double %b, double %a
   ret double %cond
 }
 
@@ -79,7 +79,7 @@ define float @fp-armv8_vmaxnm_o(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f32
   %cmp = fcmp fast ogt float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -88,7 +88,7 @@ define float @fp-armv8_vmaxnm_oge(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f32
   %cmp = fcmp fast oge float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -97,7 +97,7 @@ define float @fp-armv8_vmaxnm_o_rev(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f32
   %cmp = fcmp fast olt float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -106,7 +106,7 @@ define float @fp-armv8_vmaxnm_ole_rev(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f32
   %cmp = fcmp fast ole float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -115,7 +115,7 @@ define float @fp-armv8_vmaxnm_u(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f32
   %cmp = fcmp fast ugt float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -124,7 +124,7 @@ define float @fp-armv8_vmaxnm_uge(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f32
   %cmp = fcmp fast uge float %a, %b
-  %cond = select i1 %cmp, float %a, float %b
+  %cond = select nsz i1 %cmp, float %a, float %b
   ret float %cond
 }
 
@@ -133,7 +133,7 @@ define float @fp-armv8_vmaxnm_u_rev(float %a, float %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f32
   %cmp = fcmp fast ult float %a, %b
-  %cond = select i1 %cmp, float %b, float %a
+  %cond = select nsz i1 %cmp, float %b, float %a
   ret float %cond
 }
 
@@ -142,7 +142,7 @@ define double @fp-armv8_vmaxnm_ule_rev(double %a, double %b) {
 ; CHECK-NOT: vcmp
 ; CHECK: vmaxnm.f64
   %cmp = fcmp fast ule double %a, %b
-  %cond = select i1 %cmp, double %b, double %a
+  %cond = select nsz i1 %cmp, double %b, double %a
   ret double %cond
 }
 
@@ -153,9 +153,9 @@ define float @fp-armv8_vminnm_NNNo(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK: vminnm.f32
   %cmp1 = fcmp fast olt float %a, 12.
-  %cond1 = select i1 %cmp1, float %a, float 12.
+  %cond1 = select nsz i1 %cmp1, float %a, float 12.
   %cmp2 = fcmp fast olt float 34., %cond1
-  %cond2 = select i1 %cmp2, float 34., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 34., float %cond1
   ret float %cond2
 }
 
@@ -164,9 +164,9 @@ define double @fp-armv8_vminnm_NNNole(double %a) {
 ; CHECK: vminnm.f64
 ; CHECK: vminnm.f64
   %cmp1 = fcmp fast ole double %a, 34.
-  %cond1 = select i1 %cmp1, double %a, double 34.
+  %cond1 = select nsz i1 %cmp1, double %a, double 34.
   %cmp2 = fcmp fast ole double 56., %cond1
-  %cond2 = select i1 %cmp2, double 56., double %cond1
+  %cond2 = select nsz i1 %cmp2, double 56., double %cond1
   ret double %cond2
 }
 
@@ -175,9 +175,9 @@ define float @fp-armv8_vminnm_NNNo_rev(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK: vminnm.f32
   %cmp1 = fcmp fast ogt float %a, 56.
-  %cond1 = select i1 %cmp1, float 56., float %a
+  %cond1 = select nsz i1 %cmp1, float 56., float %a
   %cmp2 = fcmp fast ogt float 78., %cond1
-  %cond2 = select i1 %cmp2, float %cond1, float 78.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 78.
   ret float %cond2
 }
 
@@ -186,9 +186,9 @@ define double @fp-armv8_vminnm_NNNoge_rev(double %a) {
 ; CHECK: vminnm.f64
 ; CHECK: vminnm.f64
   %cmp1 = fcmp fast oge double %a, 78.
-  %cond1 = select i1 %cmp1, double 78., double %a
+  %cond1 = select nsz i1 %cmp1, double 78., double %a
   %cmp2 = fcmp fast oge double 90., %cond1
-  %cond2 = select i1 %cmp2, double %cond1, double 90.
+  %cond2 = select nsz i1 %cmp2, double %cond1, double 90.
   ret double %cond2
 }
 
@@ -197,9 +197,9 @@ define float @fp-armv8_vminnm_NNNu(float %b) {
 ; CHECK: vminnm.f32
 ; CHECK: vminnm.f32
   %cmp1 = fcmp fast ult float 12., %b
-  %cond1 = select i1 %cmp1, float 12., float %b
+  %cond1 = select nsz i1 %cmp1, float 12., float %b
   %cmp2 = fcmp fast ult float %cond1, 34.
-  %cond2 = select i1 %cmp2, float %cond1, float 34.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 34.
   ret float %cond2
 }
 
@@ -208,9 +208,9 @@ define float @fp-armv8_vminnm_NNNule(float %b) {
 ; CHECK: vminnm.f32
 ; CHECK: vminnm.f32
   %cmp1 = fcmp fast ule float 34., %b
-  %cond1 = select i1 %cmp1, float 34., float %b
+  %cond1 = select nsz i1 %cmp1, float 34., float %b
   %cmp2 = fcmp fast ule float %cond1, 56.
-  %cond2 = select i1 %cmp2, float %cond1, float 56.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 56.
   ret float %cond2
 }
 
@@ -219,9 +219,9 @@ define float @fp-armv8_vminnm_NNNu_rev(float %b) {
 ; CHECK: vminnm.f32
 ; CHECK: vminnm.f32
   %cmp1 = fcmp fast ugt float 56., %b
-  %cond1 = select i1 %cmp1, float %b, float 56.
+  %cond1 = select nsz i1 %cmp1, float %b, float 56.
   %cmp2 = fcmp fast ugt float %cond1, 78.
-  %cond2 = select i1 %cmp2, float 78., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 78., float %cond1
   ret float %cond2
 }
 
@@ -230,9 +230,9 @@ define double @fp-armv8_vminnm_NNNuge_rev(double %b) {
 ; CHECK: vminnm.f64
 ; CHECK: vminnm.f64
   %cmp1 = fcmp fast uge double 78., %b
-  %cond1 = select i1 %cmp1, double %b, double 78.
+  %cond1 = select nsz i1 %cmp1, double %b, double 78.
   %cmp2 = fcmp fast uge double %cond1, 90.
-  %cond2 = select i1 %cmp2, double 90., double %cond1
+  %cond2 = select nsz i1 %cmp2, double 90., double %cond1
   ret double %cond2
 }
 
@@ -241,9 +241,9 @@ define float @fp-armv8_vmaxnm_NNNo(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast ogt float %a, 12.
-  %cond1 = select i1 %cmp1, float %a, float 12.
+  %cond1 = select nsz i1 %cmp1, float %a, float 12.
   %cmp2 = fcmp fast ogt float 34., %cond1
-  %cond2 = select i1 %cmp2, float 34., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 34., float %cond1
   ret float %cond2
 }
 
@@ -252,9 +252,9 @@ define float @fp-armv8_vmaxnm_NNNoge(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast oge float %a, 34.
-  %cond1 = select i1 %cmp1, float %a, float 34.
+  %cond1 = select nsz i1 %cmp1, float %a, float 34.
   %cmp2 = fcmp fast oge float 56., %cond1
-  %cond2 = select i1 %cmp2, float 56., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 56., float %cond1
   ret float %cond2
 }
 
@@ -263,9 +263,9 @@ define float @fp-armv8_vmaxnm_NNNo_rev(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast olt float %a, 56.
-  %cond1 = select i1 %cmp1, float 56., float %a
+  %cond1 = select nsz i1 %cmp1, float 56., float %a
   %cmp2 = fcmp fast olt float 78., %cond1
-  %cond2 = select i1 %cmp2, float %cond1, float 78.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 78.
   ret float %cond2
 }
 
@@ -274,9 +274,9 @@ define float @fp-armv8_vmaxnm_NNNole_rev(float %a) {
 ; CHECK: vmaxnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast ole float %a, 78.
-  %cond1 = select i1 %cmp1, float 78., float %a
+  %cond1 = select nsz i1 %cmp1, float 78., float %a
   %cmp2 = fcmp fast ole float 90., %cond1
-  %cond2 = select i1 %cmp2, float %cond1, float 90.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 90.
   ret float %cond2
 }
 
@@ -285,9 +285,9 @@ define float @fp-armv8_vmaxnm_NNNu(float %b) {
 ; CHECK: vmaxnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast ugt float 12., %b
-  %cond1 = select i1 %cmp1, float 12., float %b
+  %cond1 = select nsz i1 %cmp1, float 12., float %b
   %cmp2 = fcmp fast ugt float %cond1, 34.
-  %cond2 = select i1 %cmp2, float %cond1, float 34.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 34.
   ret float %cond2
 }
 
@@ -296,9 +296,9 @@ define float @fp-armv8_vmaxnm_NNNuge(float %b) {
 ; CHECK: vmaxnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast uge float 34., %b
-  %cond1 = select i1 %cmp1, float 34., float %b
+  %cond1 = select nsz i1 %cmp1, float 34., float %b
   %cmp2 = fcmp fast uge float %cond1, 56.
-  %cond2 = select i1 %cmp2, float %cond1, float 56.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 56.
   ret float %cond2
 }
 
@@ -307,9 +307,9 @@ define float @fp-armv8_vmaxnm_NNNu_rev(float %b) {
 ; CHECK: vmaxnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast ult float 56., %b
-  %cond1 = select i1 %cmp1, float %b, float 56.
+  %cond1 = select nsz i1 %cmp1, float %b, float 56.
   %cmp2 = fcmp fast ult float %cond1, 78.
-  %cond2 = select i1 %cmp2, float 78., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 78., float %cond1
   ret float %cond2
 }
 
@@ -318,9 +318,9 @@ define double @fp-armv8_vmaxnm_NNNule_rev( double %b) {
 ; CHECK: vmaxnm.f64
 ; CHECK: vmaxnm.f64
   %cmp1 = fcmp fast ule double 78., %b
-  %cond1 = select i1 %cmp1, double %b, double 78.
+  %cond1 = select nsz i1 %cmp1, double %b, double 78.
   %cmp2 = fcmp fast ule double %cond1, 90.
-  %cond2 = select i1 %cmp2, double 90., double %cond1
+  %cond2 = select nsz i1 %cmp2, double 90., double %cond1
   ret double %cond2
 }
 
@@ -330,9 +330,9 @@ define float @fp-armv8_vminmaxnm_0(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast olt float %a, 0.
-  %cond1 = select i1 %cmp1, float %a, float 0.
+  %cond1 = select nsz i1 %cmp1, float %a, float 0.
   %cmp2 = fcmp fast ogt float %cond1, 0.
-  %cond2 = select i1 %cmp2, float %cond1, float 0.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float 0.
   ret float %cond2
 }
 
@@ -342,9 +342,9 @@ define float @fp-armv8_vminmaxnm_neg0(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast olt float %a, -0.
-  %cond1 = select i1 %cmp1, float %a, float -0.
+  %cond1 = select nsz i1 %cmp1, float %a, float -0.
   %cmp2 = fcmp fast ugt float %cond1, -0.
-  %cond2 = select i1 %cmp2, float %cond1, float -0.
+  %cond2 = select nsz i1 %cmp2, float %cond1, float -0.
   ret float %cond2
 }
 
@@ -354,9 +354,9 @@ define float @fp-armv8_vminmaxnm_e_0(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast ule float 0., %a
-  %cond1 = select i1 %cmp1, float 0., float %a
+  %cond1 = select nsz i1 %cmp1, float 0., float %a
   %cmp2 = fcmp fast uge float 0., %cond1
-  %cond2 = select i1 %cmp2, float 0., float %cond1
+  %cond2 = select nsz i1 %cmp2, float 0., float %cond1
   ret float %cond2
 }
 
@@ -366,9 +366,9 @@ define float @fp-armv8_vminmaxnm_e_neg0(float %a) {
 ; CHECK: vminnm.f32
 ; CHECK: vmaxnm.f32
   %cmp1 = fcmp fast ule float -0., %a
-  %cond1 = select i1 %cmp1, float -0., float %a
+  %cond1 = select nsz i1 %cmp1, float -0., float %a
   %cmp2 = fcmp fast oge float -0., %cond1
-  %cond2 = select i1 %cmp2, float -0., float %cond1
+  %cond2 = select nsz i1 %cmp2, float -0., float %cond1
   ret float %cond2
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/AVR/global-aliases.ll b/external/llvm-project/llvm/test/CodeGen/AVR/global-aliases.ll
index 91bcedc7e0db..b948003e8b88 100644
--- a/external/llvm-project/llvm/test/CodeGen/AVR/global-aliases.ll
+++ b/external/llvm-project/llvm/test/CodeGen/AVR/global-aliases.ll
@@ -1,18 +1,18 @@
 ; RUN: llc < %s -mtriple=avr -mcpu=atxmega384c3 | FileCheck %s --check-prefixes=MEGA
 ; RUN: llc < %s -mtriple=avr -mcpu=attiny40 | FileCheck %s --check-prefixes=TINY
 
-; MEGA: .set __tmp_reg__, 0
-; MEGA: .set __zero_reg__, 1
-; MEGA: .set __SREG__, 63
-; MEGA: .set __SP_H__, 62
-; MEGA: .set __SP_L__, 61
-; MEGA: .set __EIND__, 60
-; MEGA: .set __RAMPZ__, 59
+; MEGA: __tmp_reg__ = 0
+; MEGA: __zero_reg__ = 1
+; MEGA: __SREG__ = 63
+; MEGA: __SP_H__ = 62
+; MEGA: __SP_L__ = 61
+; MEGA: __EIND__ = 60
+; MEGA: __RAMPZ__ = 59
 
-; TINY:     .set __tmp_reg__, 16
-; TINY:     .set __zero_reg__, 17
-; TINY:     .set __SREG__, 63
-; TINY-NOT: .set __SP_H__, 62
-; TINY:     .set __SP_L__, 61
-; TINY-NOT: .set __EIND__, 60
-; TINY-NOT: .set __RAMPZ__, 59
+; TINY:     __tmp_reg__ = 16
+; TINY:     __zero_reg__ = 17
+; TINY:     __SREG__ = 63
+; TINY-NOT: __SP_H__ = 62
+; TINY:     __SP_L__ = 61
+; TINY-NOT: __EIND__ = 60
+; TINY-NOT: __RAMPZ__ = 59
diff --git a/external/llvm-project/llvm/test/CodeGen/BPF/warn-cmp.ll b/external/llvm-project/llvm/test/CodeGen/BPF/warn-cmp.ll
new file mode 100644
index 000000000000..109d177b0fb4
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/BPF/warn-cmp.ll
@@ -0,0 +1,15 @@
+; RUN: llc -mtriple=bpfel -filetype=obj < %s 2>&1 >/dev/null | FileCheck %s
+
+; CHECK: warning: immediate out of range, shall fit in 32 bits
+define dso_local void @test_1() naked {
+  tail call void asm sideeffect
+    "r1 = 40; if r1 == 0x1deadbeef goto +0; r0 = 0; exit;", "~{r0},~{r1}"()
+  unreachable
+}
+
+; CHECK: warning: immediate out of range, shall fit in 32 bits
+define dso_local void @test_2() naked {
+  tail call void asm sideeffect
+    "r1 = 40; if r1 == 0xffffffff00000000 goto +0; r0 = 0; exit;", "~{r0},~{r1}"()
+  unreachable
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Invalid-ParameterIsNotString.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Invalid-ParameterIsNotString.ll
new file mode 100644
index 000000000000..04edd00eee64
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Invalid-ParameterIsNotString.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+; CHECK: error: Invalid format for Root Element
+; CHECK-NOT: Root Signature Definitions
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+
+define void @main() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+!dx.rootsignatures = !{!0}
+!0 = !{ ptr @main, !1 }
+!1 = !{ !2 }
+!2 = !{ i32 0 }
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
index b55d1283df0c..714c76213e1b 100644
--- a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
@@ -12,19 +12,25 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
 !2 = !{ ptr @main, !3 } ; function, root signature
-!3 = !{ !4, !5 } ; list of root signature elements
+!3 = !{ !4, !5, !6 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
 !5 = !{ !"RootConstants", i32 0, i32 1, i32 2, i32 3 }
+!6 = !{ !"RootSRV", i32 1, i32 4, i32 5, i32 6 }
 
 ;CHECK-LABEL: Definition for 'main':
 ;CHECK-NEXT:  Flags: 0x000001
 ;CHECK-NEXT:  Version: 2
 ;CHECK-NEXT:  RootParametersOffset: 24
-;CHECK-NEXT:  NumParameters: 1
+;CHECK-NEXT:  NumParameters: 2
 ;CHECK-NEXT:   - Parameter Type: 1
 ;CHECK-NEXT:     Shader Visibility: 0
 ;CHECK-NEXT:     Register Space: 2
 ;CHECK-NEXT:     Shader Register: 1
 ;CHECK-NEXT:     Num 32 Bit Values: 3
+;CHECK-NEXT:   - Parameter Type: 3
+;CHECK-NEXT:     Shader Visibility: 1
+;CHECK-NEXT:     Register Space: 5
+;CHECK-NEXT:     Shader Register: 4
+;CHECK-NEXT:     Flags: 6
 ;CHECK-NEXT:  NumStaticSamplers: 0
 ;CHECK-NEXT:  StaticSamplersOffset: 0
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
new file mode 100644
index 000000000000..422998124091
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
@@ -0,0 +1,18 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+
+; CHECK: error: Invalid value for DescriptorFlag: 3
+; CHECK-NOT: Root Signature Definitions
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"RootCBV", i32 0, i32 1, i32 2, i32 3  }
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterKind.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterKind.ll
new file mode 100644
index 000000000000..4aed84efbe2b
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterKind.ll
@@ -0,0 +1,18 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+
+; CHECK: error:  Invalid Root Signature Element: Invalid 
+; CHECK-NOT: Root Signature Definitions
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"Invalid", i32 0, i32 1, i32 2, i32 3  }
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterSpace.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterSpace.ll
new file mode 100644
index 000000000000..020d117ba45d
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterSpace.ll
@@ -0,0 +1,18 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+
+; CHECK: error:  Invalid value for RegisterSpace: 4294967280
+; CHECK-NOT: Root Signature Definitions
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"RootCBV", i32 0, i32 1, i32 4294967280, i32 0  }
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterValue.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterValue.ll
new file mode 100644
index 000000000000..edb8b943c6e3
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterValue.ll
@@ -0,0 +1,18 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+
+; CHECK: error: Invalid value for ShaderRegister: 4294967295 
+; CHECK-NOT: Root Signature Definitions
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"RootCBV", i32 0, i32 4294967295, i32 2, i32 3  }
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor.ll
new file mode 100644
index 000000000000..9217945855cd
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor.ll
@@ -0,0 +1,34 @@
+; RUN: opt %s -dxil-embed -dxil-globals -S -o - | FileCheck %s
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s --check-prefix=DXC
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: @dx.rts0 = private constant [48 x i8]  c"{{.*}}", section "RTS0", align 4
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"RootCBV", i32 0, i32 1, i32 2, i32 8  }
+
+; DXC:  - Name:            RTS0
+; DXC-NEXT:    Size:            48
+; DXC-NEXT:    RootSignature:
+; DXC-NEXT:      Version:         2
+; DXC-NEXT:      NumRootParameters: 1 
+; DXC-NEXT:      RootParametersOffset: 24 
+; DXC-NEXT:      NumStaticSamplers: 0
+; DXC-NEXT:      StaticSamplersOffset: 0
+; DXC-NEXT:      Parameters:
+; DXC-NEXT:        - ParameterType:   2
+; DXC-NEXT:          ShaderVisibility: 0
+; DXC-NEXT:          Descriptor:
+; DXC-NEXT:            RegisterSpace: 2
+; DXC-NEXT:            ShaderRegister: 1
+; DXC-NEXT:            DATA_STATIC: true
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-array.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-array.ll
index 754d5a25ca90..5c761014d471 100644
--- a/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-array.ll
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-array.ll
@@ -31,7 +31,7 @@ define void @alloca_4d_test ()  {
 ; CHECK-LABEL: gep_2d_test
 define void @gep_2d_test ()  {
     ; CHECK: [[a:%.*]] = alloca [9 x i32], align 4
-    ; CHECK-COUNT-9: getelementptr inbounds [9 x i32], ptr [[a]], i32 {{[0-8]}}
+    ; CHECK-COUNT-9: getelementptr inbounds [9 x i32], ptr [[a]], i32 0, i32 {{[0-8]}}
     ; CHECK-NEXT:    ret void
     %1 = alloca [3 x [3 x i32]], align 4
     %g2d0 = getelementptr inbounds [3 x [3 x i32]], [3 x [3 x i32]]* %1, i32 0, i32 0
@@ -53,7 +53,7 @@ define void @gep_2d_test ()  {
 ; CHECK-LABEL: gep_3d_test
 define void @gep_3d_test ()  {
     ; CHECK: [[a:%.*]] = alloca [8 x i32], align 4
-    ; CHECK-COUNT-8: getelementptr inbounds [8 x i32], ptr [[a]], i32 {{[0-7]}}
+    ; CHECK-COUNT-8: getelementptr inbounds [8 x i32], ptr [[a]], i32 0, i32 {{[0-7]}}
     ; CHECK-NEXT:    ret void
     %1 = alloca [2 x[2 x [2 x i32]]], align 4
     %g3d0 = getelementptr inbounds [2 x[2 x [2 x i32]]], [2 x[2 x [2 x i32]]]* %1, i32 0, i32 0
@@ -76,7 +76,7 @@ define void @gep_3d_test ()  {
 ; CHECK-LABEL: gep_4d_test
 define void @gep_4d_test ()  {
     ; CHECK: [[a:%.*]] = alloca [16 x i32], align 4
-    ; CHECK-COUNT-16: getelementptr inbounds [16 x i32], ptr [[a]], i32 {{[0-9]|1[0-5]}}
+    ; CHECK-COUNT-16: getelementptr inbounds [16 x i32], ptr [[a]], i32 0, i32 {{[0-9]|1[0-5]}}
     ; CHECK-NEXT:    ret void
     %1 = alloca [2x[2 x[2 x [2 x i32]]]], align 4
     %g4d0 = getelementptr inbounds [2x[2 x[2 x [2 x i32]]]], [2x[2 x[2 x [2 x i32]]]]* %1, i32 0, i32 0
@@ -123,8 +123,8 @@ define void @gep_4d_test ()  {
 @b = internal global [2 x [3 x [4 x i32]]] zeroinitializer, align 16
 
 define void @global_gep_load() {
-  ; CHECK: [[GEP_PTR:%.*]] = getelementptr inbounds [24 x i32], ptr @a.1dim, i32 6
-  ; CHECK: load i32, ptr [[GEP_PTR]], align 4
+  ; CHECK: [[GEP_PTR:%.*]] = getelementptr inbounds [24 x i32], ptr @a.1dim, i32 0, i32 6
+  ; CHECK-NEXT: load i32, ptr [[GEP_PTR]], align 4
   ; CHECK-NEXT:    ret void
   %1 = getelementptr inbounds [2 x [3 x [4 x i32]]], [2 x [3 x [4 x i32]]]* @a, i32 0, i32 0
   %2 = getelementptr inbounds [3 x [4 x i32]], [3 x [4 x i32]]* %1, i32 0, i32 1
@@ -142,7 +142,7 @@ define void @global_gep_load_index(i32 %row, i32 %col, i32 %timeIndex) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[ROW]], 12
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[DOTFLAT:%.*]] = getelementptr inbounds [24 x i32], ptr @a.1dim, i32 [[TMP6]]
+; CHECK-NEXT:    [[DOTFLAT:%.*]] = getelementptr inbounds [24 x i32], ptr @a.1dim, i32 0, i32 [[TMP6]]
 ; CHECK-NOT: getelementptr inbounds [2 x [3 x [4 x i32]]]{{.*}}
 ; CHECK-NOT: getelementptr inbounds [3 x [4 x i32]]{{.*}}
 ; CHECK-NOT: getelementptr inbounds [4 x i32]{{.*}}
@@ -163,7 +163,7 @@ define void @global_incomplete_gep_chain(i32 %row, i32 %col) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i32 0, [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[ROW]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[DOTFLAT:%.*]] = getelementptr inbounds [24 x i32], ptr @a.1dim, i32 [[TMP4]]
+; CHECK-NEXT:    [[DOTFLAT:%.*]] = getelementptr inbounds [24 x i32], ptr @a.1dim, i32 0, i32 [[TMP4]]
 ; CHECK-NOT: getelementptr inbounds [2 x [3 x [4 x i32]]]{{.*}}
 ; CHECK-NOT: getelementptr inbounds [3 x [4 x i32]]{{.*}}
 ; CHECK-NOT: getelementptr inbounds [4 x i32]{{.*}}
@@ -177,8 +177,8 @@ define void @global_incomplete_gep_chain(i32 %row, i32 %col) {
 }
 
 define void @global_gep_store() {
-  ; CHECK: [[GEP_PTR:%.*]] = getelementptr inbounds [24 x i32], ptr @b.1dim, i32 13
-  ; CHECK:  store i32 1, ptr [[GEP_PTR]], align 4
+  ; CHECK: [[GEP_PTR:%.*]] = getelementptr inbounds [24 x i32], ptr @b.1dim, i32 0, i32 13
+  ; CHECK-NEXT: store i32 1, ptr [[GEP_PTR]], align 4
   ; CHECK-NEXT:    ret void
   %1 = getelementptr inbounds [2 x [3 x [4 x i32]]], [2 x [3 x [4 x i32]]]* @b, i32 0, i32 1
   %2 = getelementptr inbounds [3 x [4 x i32]], [3 x [4 x i32]]* %1, i32 0, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-bug-117273.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-bug-117273.ll
index 3ae5832ce832..c73e5017348d 100644
--- a/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-bug-117273.ll
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/flatten-bug-117273.ll
@@ -8,9 +8,9 @@
 define internal void @main() {
 ; CHECK-LABEL: define internal void @main() {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr [24 x float], ptr @ZerroInitArr.1dim, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr [24 x float], ptr @ZerroInitArr.1dim, i32 0, i32 1
 ; CHECK-NEXT:    [[DOTI0:%.*]] = load float, ptr [[TMP0]], align 16
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [24 x float], ptr @ZerroInitArr.1dim, i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [24 x float], ptr @ZerroInitArr.1dim, i32 0, i32 2
 ; CHECK-NEXT:    [[DOTI03:%.*]] = load float, ptr [[TMP1]], align 16
 ; CHECK-NEXT:    ret void
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/legalize-i64-high-low-vec-split.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/legalize-i64-high-low-vec-split.ll
new file mode 100644
index 000000000000..17fd3bf54acd
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/legalize-i64-high-low-vec-split.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes='dxil-legalize' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+
+define void @split_via_extract(i64 noundef %a) {
+; CHECK-LABEL: define void @split_via_extract(
+; CHECK-SAME: i64 noundef [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[A]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[A]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret void
+;
+entry:
+  %vecA = bitcast i64 %a to <2 x i32>
+  %low = extractelement <2 x i32> %vecA, i32 0 ; low 32 bits
+  %high = extractelement <2 x i32> %vecA, i32 1 ; high 32 bits
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/legalize-i8.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/legalize-i8.ll
index 2602be778cd8..f8aa2c5ecd93 100644
--- a/external/llvm-project/llvm/test/CodeGen/DirectX/legalize-i8.ll
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/legalize-i8.ll
@@ -106,3 +106,75 @@ define i32 @all_imm() {
   %2 = sext i8 %1 to i32
   ret i32 %2
 }
+
+define i32 @scalar_i8_geps() {
+  ; CHECK-LABEL: define i32 @scalar_i8_geps(
+  ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4
+  ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [1 x i32], ptr [[ALLOCA]], i32 0, i32 0
+  ; CHECK:         [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
+  ; CHECK-NEXT:    ret i32 [[LOAD]]
+    %1 = alloca i8, align 4
+    %2 = getelementptr inbounds nuw i8, ptr %1, i32 0
+    %3 = load i8, ptr %2
+    %4 = sext i8 %3 to i32
+    ret i32 %4
+}
+
+define i32 @i8_geps_index0() {
+  ; CHECK-LABEL: define i32 @i8_geps_index0(
+  ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [2 x i32], align 8
+  ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [2 x i32], ptr [[ALLOCA]], i32 0, i32 0
+  ; CHECK:         [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
+  ; CHECK-NEXT:    ret i32 [[LOAD]]
+  %1 = alloca [2 x i32], align 8
+  %2 = getelementptr inbounds nuw i8, ptr %1, i32 0
+  %3 = load i8, ptr %2
+  %4 = sext i8 %3 to i32
+  ret i32 %4
+}
+
+define i32 @i8_geps_index1() {
+  ; CHECK-LABEL: define i32 @i8_geps_index1(
+  ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [2 x i32], align 8
+  ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [2 x i32], ptr [[ALLOCA]], i32 0, i32 1
+  ; CHECK:         [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
+  ; CHECK-NEXT:    ret i32 [[LOAD]]
+  %1 = alloca [2 x i32], align 8
+  %2 = getelementptr inbounds nuw i8, ptr %1, i32 4
+  %3 = load i8, ptr %2
+  %4 = sext i8 %3 to i32
+  ret i32 %4
+}
+
+define i32 @i8_gep_store() {
+  ; CHECK-LABEL: define i32 @i8_gep_store(
+  ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [2 x i32], align 8
+  ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [2 x i32], ptr [[ALLOCA]], i32 0, i32 1
+  ; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+  ; CHECK:         [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
+  ; CHECK-NEXT:    ret i32 [[LOAD]]
+  %1 = alloca [2 x i32], align 8
+  %2 = getelementptr inbounds nuw i8, ptr %1, i32 4
+  store i8 1, ptr %2
+  %3 = load i8, ptr %2
+  %4 = sext i8 %3 to i32
+  ret i32 %4
+}
+
+@g = local_unnamed_addr addrspace(3) global [2 x float] zeroinitializer, align 4
+define float @i8_gep_global_index() {
+  ; CHECK-LABEL: define float @i8_gep_global_index(
+  ; CHECK-NEXT: [[LOAD:%.*]] = load float, ptr addrspace(3) getelementptr inbounds nuw ([2 x float], ptr addrspace(3) @g, i32 0, i32 1), align 4
+  ; CHECK-NEXT:    ret float [[LOAD]]
+  %1 = getelementptr inbounds nuw i8, ptr addrspace(3) @g, i32 4
+  %2 = load float, ptr addrspace(3) %1, align 4
+  ret float %2
+}
+
+define float @i8_gep_global_constexpr() {
+  ; CHECK-LABEL: define float @i8_gep_global_constexpr(
+  ; CHECK-NEXT: [[LOAD:%.*]] = load float, ptr addrspace(3) getelementptr inbounds nuw ([2 x float], ptr addrspace(3) @g, i32 0, i32 1), align 4
+  ; CHECK-NEXT: ret float [[LOAD]]
+  %1 = load float, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g, i32 4), align 4
+  ret float %1
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/llc-vector-load-scalarize.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/llc-vector-load-scalarize.ll
index 7e5a92e1311f..c960aad3d262 100644
--- a/external/llvm-project/llvm/test/CodeGen/DirectX/llc-vector-load-scalarize.ll
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/llc-vector-load-scalarize.ll
@@ -32,23 +32,23 @@ define <4 x i32> @load_array_vec_test() #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(3) [[TMP5]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 3) to ptr addrspace(3)
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr addrspace(3) [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 1) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 0, i32 1) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr addrspace(3) [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 0, i32 1), i32 1) to ptr addrspace(3)
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr addrspace(3) [[TMP11]], align 4
-; CHECK-NEXT:    [[DOTI12:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 1), i32 1) to ptr addrspace(3)
-; CHECK-NEXT:    [[DOTI13:%.*]] = load i32, ptr addrspace(3) [[DOTI12]], align 4
-; CHECK-NEXT:    [[DOTI24:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 1), i32 2) to ptr addrspace(3)
-; CHECK-NEXT:    [[DOTI25:%.*]] = load i32, ptr addrspace(3) [[DOTI24]], align 4
-; CHECK-NEXT:    [[DOTI36:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 1), i32 3) to ptr addrspace(3)
-; CHECK-NEXT:    [[DOTI37:%.*]] = load i32, ptr addrspace(3) [[DOTI36]], align 4
-; CHECK-NEXT:    [[DOTI08:%.*]] = add i32 [[TMP2]], [[TMP12]]
-; CHECK-NEXT:    [[DOTI19:%.*]] = add i32 [[TMP4]], [[DOTI13]]
-; CHECK-NEXT:    [[DOTI210:%.*]] = add i32 [[TMP6]], [[DOTI25]]
-; CHECK-NEXT:    [[DOTI311:%.*]] = add i32 [[TMP8]], [[DOTI37]]
-; CHECK-NEXT:    [[DOTUPTO015:%.*]] = insertelement <4 x i32> poison, i32 [[DOTI08]], i32 0
-; CHECK-NEXT:    [[DOTUPTO116:%.*]] = insertelement <4 x i32> [[DOTUPTO015]], i32 [[DOTI19]], i32 1
-; CHECK-NEXT:    [[DOTUPTO217:%.*]] = insertelement <4 x i32> [[DOTUPTO116]], i32 [[DOTI210]], i32 2
-; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> [[DOTUPTO217]], i32 [[DOTI311]], i32 3
-; CHECK-NEXT:    ret <4 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 0, i32 1), i32 2) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr addrspace(3) [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([6 x float], ptr addrspace(3) @arrayofVecData.scalarized.1dim, i32 0, i32 1), i32 3) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr addrspace(3) [[TMP15]], align 4
+; CHECK-NEXT:    [[DOTI05:%.*]] = add i32 [[TMP2]], [[TMP10]]
+; CHECK-NEXT:    [[DOTI16:%.*]] = add i32 [[TMP4]], [[TMP12]]
+; CHECK-NEXT:    [[DOTI27:%.*]] = add i32 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[DOTI38:%.*]] = add i32 [[TMP8]], [[TMP16]]
+; CHECK-NEXT:    [[DOTUPTO01215:%.*]] = insertelement <4 x i32> poison, i32 [[DOTI05]], i32 0
+; CHECK-NEXT:    [[DOTUPTO11316:%.*]] = insertelement <4 x i32> [[DOTUPTO01215]], i32 [[DOTI16]], i32 1
+; CHECK-NEXT:    [[DOTUPTO21417:%.*]] = insertelement <4 x i32> [[DOTUPTO11316]], i32 [[DOTI27]], i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[DOTUPTO21417]], i32 [[DOTI38]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[TMP17]]
 ;
   %1 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([2 x <4 x i32>], [2 x <4 x i32>] addrspace(3)* @"arrayofVecData", i32 0, i32 0), align 4
   %2 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([2 x <4 x i32>], [2 x <4 x i32>] addrspace(3)* @"arrayofVecData", i32 0, i32 1), align 4
@@ -81,23 +81,19 @@ define <4 x i32> @load_vec_test() #0 {
 define <4 x i32> @load_static_array_of_vec_test(i32 %index) #0 {
 ; CHECK-LABEL: define <4 x i32> @load_static_array_of_vec_test(
 ; CHECK-SAME: i32 [[INDEX:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[DOTFLAT:%.*]] = getelementptr inbounds [12 x i32], ptr @staticArrayOfVecData.scalarized.1dim, i32 [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast ptr [[DOTFLAT]] to ptr
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast ptr [[DOTFLAT]] to ptr
-; CHECK-NEXT:    [[DOTFLAT_I1:%.*]] = getelementptr i32, ptr [[TMP3]], i32 1
+; CHECK-NEXT:    [[DOTFLAT:%.*]] = getelementptr inbounds [12 x i32], ptr @staticArrayOfVecData.scalarized.1dim, i32 0, i32 [[INDEX]]
+; CHECK-NEXT:    [[DOTI0:%.*]] = load i32, ptr [[DOTFLAT]], align 4
+; CHECK-NEXT:    [[DOTFLAT_I1:%.*]] = getelementptr i32, ptr [[DOTFLAT]], i32 1
 ; CHECK-NEXT:    [[DOTI1:%.*]] = load i32, ptr [[DOTFLAT_I1]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast ptr [[DOTFLAT]] to ptr
-; CHECK-NEXT:    [[DOTFLAT_I2:%.*]] = getelementptr i32, ptr [[TMP4]], i32 2
+; CHECK-NEXT:    [[DOTFLAT_I2:%.*]] = getelementptr i32, ptr [[DOTFLAT]], i32 2
 ; CHECK-NEXT:    [[DOTI2:%.*]] = load i32, ptr [[DOTFLAT_I2]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast ptr [[DOTFLAT]] to ptr
-; CHECK-NEXT:    [[DOTFLAT_I3:%.*]] = getelementptr i32, ptr [[TMP5]], i32 3
+; CHECK-NEXT:    [[DOTFLAT_I3:%.*]] = getelementptr i32, ptr [[DOTFLAT]], i32 3
 ; CHECK-NEXT:    [[DOTI3:%.*]] = load i32, ptr [[DOTFLAT_I3]], align 4
-; CHECK-NEXT:    [[DOTUPTO0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i32 0
-; CHECK-NEXT:    [[DOTUPTO1:%.*]] = insertelement <4 x i32> [[DOTUPTO0]], i32 [[DOTI1]], i32 1
-; CHECK-NEXT:    [[DOTUPTO2:%.*]] = insertelement <4 x i32> [[DOTUPTO1]], i32 [[DOTI2]], i32 2
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[DOTUPTO2]], i32 [[DOTI3]], i32 3
-; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[DOTUPTO01:%.*]] = insertelement <4 x i32> poison, i32 [[DOTI0]], i32 0
+; CHECK-NEXT:    [[DOTUPTO12:%.*]] = insertelement <4 x i32> [[DOTUPTO01]], i32 [[DOTI1]], i32 1
+; CHECK-NEXT:    [[DOTUPTO23:%.*]] = insertelement <4 x i32> [[DOTUPTO12]], i32 [[DOTI2]], i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[DOTUPTO23]], i32 [[DOTI3]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %3 = getelementptr inbounds [3 x <4 x i32>], [3 x <4 x i32>]* @staticArrayOfVecData, i32 0, i32 %index
   %4 = load <4 x i32>, <4 x i32>* %3, align 4
@@ -115,23 +111,23 @@ define <4 x i32> @multid_load_test() #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(3) [[TMP5]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 3) to ptr addrspace(3)
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr addrspace(3) [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 1) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 0, i32 1) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr addrspace(3) [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 0, i32 1), i32 1) to ptr addrspace(3)
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr addrspace(3) [[TMP11]], align 4
-; CHECK-NEXT:    [[DOTI12:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 1), i32 1) to ptr addrspace(3)
-; CHECK-NEXT:    [[DOTI13:%.*]] = load i32, ptr addrspace(3) [[DOTI12]], align 4
-; CHECK-NEXT:    [[DOTI24:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 1), i32 2) to ptr addrspace(3)
-; CHECK-NEXT:    [[DOTI25:%.*]] = load i32, ptr addrspace(3) [[DOTI24]], align 4
-; CHECK-NEXT:    [[DOTI36:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 1), i32 3) to ptr addrspace(3)
-; CHECK-NEXT:    [[DOTI37:%.*]] = load i32, ptr addrspace(3) [[DOTI36]], align 4
-; CHECK-NEXT:    [[DOTI08:%.*]] = add i32 [[TMP2]], [[TMP12]]
-; CHECK-NEXT:    [[DOTI19:%.*]] = add i32 [[TMP4]], [[DOTI13]]
-; CHECK-NEXT:    [[DOTI210:%.*]] = add i32 [[TMP6]], [[DOTI25]]
-; CHECK-NEXT:    [[DOTI311:%.*]] = add i32 [[TMP8]], [[DOTI37]]
-; CHECK-NEXT:    [[DOTUPTO015:%.*]] = insertelement <4 x i32> poison, i32 [[DOTI08]], i32 0
-; CHECK-NEXT:    [[DOTUPTO116:%.*]] = insertelement <4 x i32> [[DOTUPTO015]], i32 [[DOTI19]], i32 1
-; CHECK-NEXT:    [[DOTUPTO217:%.*]] = insertelement <4 x i32> [[DOTUPTO116]], i32 [[DOTI210]], i32 2
-; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> [[DOTUPTO217]], i32 [[DOTI311]], i32 3
-; CHECK-NEXT:    ret <4 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 0, i32 1), i32 2) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr addrspace(3) [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast ptr addrspace(3) getelementptr (i32, ptr addrspace(3) getelementptr inbounds ([36 x i32], ptr addrspace(3) @groushared2dArrayofVectors.scalarized.1dim, i32 0, i32 1), i32 3) to ptr addrspace(3)
+; CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr addrspace(3) [[TMP15]], align 4
+; CHECK-NEXT:    [[DOTI05:%.*]] = add i32 [[TMP2]], [[TMP10]]
+; CHECK-NEXT:    [[DOTI16:%.*]] = add i32 [[TMP4]], [[TMP12]]
+; CHECK-NEXT:    [[DOTI27:%.*]] = add i32 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[DOTI38:%.*]] = add i32 [[TMP8]], [[TMP16]]
+; CHECK-NEXT:    [[DOTUPTO01215:%.*]] = insertelement <4 x i32> poison, i32 [[DOTI05]], i32 0
+; CHECK-NEXT:    [[DOTUPTO11316:%.*]] = insertelement <4 x i32> [[DOTUPTO01215]], i32 [[DOTI16]], i32 1
+; CHECK-NEXT:    [[DOTUPTO21417:%.*]] = insertelement <4 x i32> [[DOTUPTO11316]], i32 [[DOTI27]], i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[DOTUPTO21417]], i32 [[DOTI38]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[TMP17]]
 ;
   %1 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([3 x [3 x <4 x i32>]], [3 x [3 x <4 x i32>]] addrspace(3)* @"groushared2dArrayofVectors", i32 0, i32 0, i32 0), align 4
   %2 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([3 x [3 x <4 x i32>]], [3 x [3 x <4 x i32>]] addrspace(3)* @"groushared2dArrayofVectors", i32 0, i32 1, i32 1), align 4
diff --git a/external/llvm-project/llvm/test/CodeGen/DirectX/scalar-bug-117273.ll b/external/llvm-project/llvm/test/CodeGen/DirectX/scalar-bug-117273.ll
index 2676abec1d8a..a07ce2c24f7a 100644
--- a/external/llvm-project/llvm/test/CodeGen/DirectX/scalar-bug-117273.ll
+++ b/external/llvm-project/llvm/test/CodeGen/DirectX/scalar-bug-117273.ll
@@ -8,13 +8,13 @@
 define internal void @main() #1 {
 ; CHECK-LABEL: define internal void @main() {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [24 x float], ptr @StaticArr.scalarized.1dim, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [24 x float], ptr @StaticArr.scalarized.1dim, i32 0, i32 1
 ; CHECK-NEXT:    [[DOTI0:%.*]] = load float, ptr [[TMP0]], align 16
 ; CHECK-NEXT:    [[DOTI1:%.*]] = getelementptr float, ptr [[TMP0]], i32 1
 ; CHECK-NEXT:    [[DOTI11:%.*]] = load float, ptr [[DOTI1]], align 4
 ; CHECK-NEXT:    [[DOTI2:%.*]] = getelementptr float, ptr [[TMP0]], i32 2
 ; CHECK-NEXT:    [[DOTI22:%.*]] = load float, ptr [[DOTI2]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [24 x float], ptr @StaticArr.scalarized.1dim, i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [24 x float], ptr @StaticArr.scalarized.1dim, i32 0, i32 2
 ; CHECK-NEXT:    [[DOTI03:%.*]] = load float, ptr [[TMP1]], align 16
 ; CHECK-NEXT:    [[DOTI14:%.*]] = getelementptr float, ptr [[TMP1]], i32 1
 ; CHECK-NEXT:    [[DOTI15:%.*]] = load float, ptr [[DOTI14]], align 4
diff --git a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir
index 40ea01189f2c..eaa627966347 100644
--- a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir
+++ b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir
@@ -1,6 +1,5 @@
 # REQUIRES: x86-registered-target
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-check-debugify -o - %s 2>&1 | FileCheck %s
-# RUN: llc --experimental-debuginfo-iterators=false -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-check-debugify -o - %s 2>&1 | FileCheck %s
 --- |
   ; ModuleID = 'check-line-and-variables.mir'
   source_filename = "check-line-and-variables.c"
diff --git a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll
index 56c7cf45705a..9033fd2f147c 100644
--- a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -debugify-check-and-strip-all-safe -o - %s 2>&1 | FileCheck %s
-; RUN: llc --experimental-debuginfo-iterators=false -debugify-check-and-strip-all-safe -o - %s 2>&1 | FileCheck %s
 
 ; ModuleID = 'main.c'
 source_filename = "main.c"
diff --git a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir
index 0805a7f4cfc6..9eb722258b70 100644
--- a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir
+++ b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir
@@ -1,8 +1,6 @@
 # REQUIRES: x86-registered-target
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,dead-mi-elimination,mir-check-debugify -o - %s 2>&1 | FileCheck %s
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s --check-prefix=CHECK-PASS
-# RUN: llc --experimental-debuginfo-iterators=false -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,dead-mi-elimination,mir-check-debugify -o - %s 2>&1 | FileCheck %s
-# RUN: llc --experimental-debuginfo-iterators=false -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s --check-prefix=CHECK-PASS
 --- |
   ; ModuleID = 'check-line-and-variables.mir'
   source_filename = "check-line-and-variables.ll"
diff --git a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir
index 8079db926e1b..27f942468a8d 100644
--- a/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir
+++ b/external/llvm-project/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir
@@ -1,5 +1,4 @@
 # RUN: llc -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s
-# RUN: llc --experimental-debuginfo-iterators=false -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s
 
 # CHECK: Machine IR debug info check: PASS
 # CHECK-NOT: Assertion `Var <= NumVars && "Unexpected name for DILocalVariable"'
diff --git a/external/llvm-project/llvm/test/CodeGen/Generic/selection-dag-determinism.ll b/external/llvm-project/llvm/test/CodeGen/Generic/selection-dag-determinism.ll
index 1adff3d61ba2..522894236807 100644
--- a/external/llvm-project/llvm/test/CodeGen/Generic/selection-dag-determinism.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Generic/selection-dag-determinism.ll
@@ -1,8 +1,8 @@
-; RUN: llc -O2 -o %t1.o < %s
-; RUN: llc -O2 -o %t2.o < %s
-; RUN: llc -O2 -o %t3.o < %s
-; RUN: llc -O2 -o %t4.o < %s
-; RUN: llc -O2 -o %t5.o < %s
+; RUN: llc -O2 < %s > %t1.o
+; RUN: llc -O2 < %s > %t2.o
+; RUN: llc -O2 < %s > %t3.o
+; RUN: llc -O2 < %s > %t4.o
+; RUN: llc -O2 < %s > %t5.o
 ; RUN: cmp %t1.o %t2.o
 ; RUN: cmp %t1.o %t3.o
 ; RUN: cmp %t1.o %t4.o
diff --git a/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep1.mir b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep1.mir
new file mode 100644
index 000000000000..17ee07f49324
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep1.mir
@@ -0,0 +1,110 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
+# REQUIRES: asserts
+
+# Test that loop-carried memory dependencies are added correctly.
+# The original code is as follows.
+#
+# ```
+# void f(int *a, int n) {
+#   for (int i = 0; i < n-1; i++) {
+#     a[i] += a[i];
+#     a[i+1] += i;
+#   }
+# }
+# ```
+# 
+# Loop-carried dependencies exist from store for a[i+1] to load/store for a[i], but not vice versa.
+# FIXME: Currently the following dependencies are missed.
+#
+# Loop carried edges from SU(6)
+#   Order
+#     SU(4)
+# Loop carried edges from SU(8)
+#   Order
+#     SU(4)
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  define dso_local void @f(ptr nocapture noundef %a, i32 noundef %n) local_unnamed_addr {
+  entry:
+    %cmp12 = icmp sgt i32 %n, 1
+    br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %.pre = load i32, ptr %a, align 4, !tbaa !5
+    %0 = add i32 %n, -1
+    %cgep = getelementptr i8, ptr %a, i32 4
+    br label %for.body
+
+  for.cond.cleanup:
+    ret void
+
+  for.body:
+    %lsr.iv14 = phi ptr [ %cgep, %for.body.preheader ], [ %cgep18, %for.body ]
+    %lsr.iv = phi i32 [ %0, %for.body.preheader ], [ %lsr.iv.next, %for.body ]
+    %1 = phi i32 [ %add4, %for.body ], [ %.pre, %for.body.preheader ]
+    %i.013 = phi i32 [ %add2, %for.body ], [ 0, %for.body.preheader ]
+    %add = shl nsw i32 %1, 1
+    %cgep17 = getelementptr i8, ptr %lsr.iv14, i32 -4
+    store i32 %add, ptr %cgep17, align 4, !tbaa !5
+    %add2 = add nuw nsw i32 %i.013, 1
+    %2 = load i32, ptr %lsr.iv14, align 4, !tbaa !5
+    %add4 = add nsw i32 %2, %i.013
+    %3 = add i32 %i.013, %2
+    store i32 %3, ptr %lsr.iv14, align 4, !tbaa !5
+    %lsr.iv.next = add i32 %lsr.iv, -1
+    %exitcond.not = icmp eq i32 %lsr.iv.next, 0
+    %cgep18 = getelementptr i8, ptr %lsr.iv14, i32 4
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  !5 = !{!6, !6, i64 0}
+  !6 = !{!"int", !7, i64 0}
+  !7 = !{!"omnipotent char", !8, i64 0}
+  !8 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    liveins: $r0, $r1
+  
+    %12:intregs = COPY $r1
+    %11:intregs = COPY $r0
+    %13:predregs = C2_cmpgti %12, 1
+    J2_jumpf %13, %bb.2, implicit-def dead $pc
+    J2_jump %bb.1, implicit-def dead $pc
+  
+  bb.1.for.body.preheader:
+    %0:intregs, %2:intregs = L2_loadri_pi %11, 4 :: (load (s32) from %ir.a, !tbaa !5)
+    %1:intregs = A2_addi %12, -1
+    %15:intregs = A2_tfrsi 0
+    %19:intregs = COPY %1
+    J2_loop0r %bb.3, %19, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+    J2_jump %bb.3, implicit-def dead $pc
+  
+  bb.2.for.cond.cleanup:
+    PS_jmpret $r31, implicit-def dead $pc
+  
+  bb.3.for.body:
+    successors: %bb.2, %bb.3
+  
+    %3:intregs = PHI %2, %bb.1, %10, %bb.3
+    %5:intregs = PHI %0, %bb.1, %8, %bb.3
+    %6:intregs = PHI %15, %bb.1, %7, %bb.3
+    %16:intregs = nsw S2_asl_i_r %5, 1
+    S2_storeri_io %3, -4, killed %16 :: (store (s32) into %ir.cgep17, !tbaa !5)
+    %7:intregs = nuw nsw A2_addi %6, 1
+    %17:intregs = L2_loadri_io %3, 0 :: (load (s32) from %ir.lsr.iv14, !tbaa !5)
+    %8:intregs = A2_add killed %17, %6
+    S2_storeri_io %3, 0, %8 :: (store (s32) into %ir.lsr.iv14, !tbaa !5)
+    %10:intregs = A2_addi %3, 4
+    ENDLOOP0 %bb.3, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def $pc
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep2.mir b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep2.mir
new file mode 100644
index 000000000000..850e602c9146
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep2.mir
@@ -0,0 +1,105 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
+# REQUIRES: asserts
+
+# Test that loop-carried memory dependencies are added correctly.
+# The original code is as follows.
+#
+# ```
+# void f(int *a, int n) {
+#   for (int i = 1; i < n; i++) {
+#     a[i] += a[i];
+#     a[i-1] += i;
+#   }
+# }
+# ```
+# 
+# Loop-carried dependencies exist from load/store for a[i] to store for a[i-1], but not vice versa.
+# FIXME: Currently the following dependencies are missed.
+#
+#  Loop carried edges from SU(5)
+#    Order
+#      SU(7)
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT:   Loop carried edges from SU(3)
+# CHECK-NEXT:     Order
+# CHECK-NEXT:       SU(7)
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  define dso_local void @f(ptr nocapture noundef %a, i32 noundef %n) local_unnamed_addr {
+  entry:
+    %cmp11 = icmp sgt i32 %n, 1
+    br i1 %cmp11, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %load_initial = load i32, ptr %a, align 4
+    %cgep = getelementptr i8, ptr %a, i32 4
+    br label %for.body
+
+  for.cond.cleanup:
+    ret void
+
+  for.body:
+    %lsr.iv = phi ptr [ %cgep, %for.body.preheader ], [ %cgep16, %for.body ]
+    %store_forwarded = phi i32 [ %load_initial, %for.body.preheader ], [ %add, %for.body ]
+    %i.012 = phi i32 [ 1, %for.body.preheader ], [ %inc, %for.body ]
+    %0 = load i32, ptr %lsr.iv, align 4, !tbaa !5
+    %add = shl nsw i32 %0, 1
+    store i32 %add, ptr %lsr.iv, align 4, !tbaa !5
+    %1 = add i32 %store_forwarded, %i.012
+    %cgep15 = getelementptr i8, ptr %lsr.iv, i32 -4
+    store i32 %1, ptr %cgep15, align 4, !tbaa !5
+    %inc = add nuw nsw i32 %i.012, 1
+    %exitcond.not = icmp eq i32 %n, %inc
+    %cgep16 = getelementptr i8, ptr %lsr.iv, i32 4
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  !5 = !{!6, !6, i64 0}
+  !6 = !{!"int", !7, i64 0}
+  !7 = !{!"omnipotent char", !8, i64 0}
+  !8 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    liveins: $r0, $r1
+  
+    %9:intregs = COPY $r1
+    %8:intregs = COPY $r0
+    %10:predregs = C2_cmpgti %9, 1
+    J2_jumpf %10, %bb.2, implicit-def dead $pc
+    J2_jump %bb.1, implicit-def dead $pc
+  
+  bb.1.for.body.preheader:
+    %0:intregs, %1:intregs = L2_loadri_pi %8, 4 :: (load (s32) from %ir.a)
+    %12:intregs = A2_tfrsi 1
+    %16:intregs = A2_addi %9, -1
+    %17:intregs = COPY %16
+    J2_loop0r %bb.3, %17, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+    J2_jump %bb.3, implicit-def dead $pc
+  
+  bb.2.for.cond.cleanup:
+    PS_jmpret $r31, implicit-def dead $pc
+  
+  bb.3.for.body (machine-block-address-taken):
+    successors: %bb.2(0x04000000), %bb.3(0x7c000000)
+  
+    %2:intregs = PHI %1, %bb.1, %7, %bb.3
+    %3:intregs = PHI %0, %bb.1, %5, %bb.3
+    %4:intregs = PHI %12, %bb.1, %6, %bb.3
+    %13:intregs = L2_loadri_io %2, 0 :: (load (s32) from %ir.lsr.iv, !tbaa !5)
+    %5:intregs = nsw S2_asl_i_r killed %13, 1
+    S2_storeri_io %2, 0, %5 :: (store (s32) into %ir.lsr.iv, !tbaa !5)
+    %14:intregs = A2_add %3, %4
+    S2_storeri_io %2, -4, killed %14 :: (store (s32) into %ir.cgep15, !tbaa !5)
+    %6:intregs = nuw nsw A2_addi %4, 1
+    %7:intregs = A2_addi %2, 4
+    ENDLOOP0 %bb.3, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def $pc
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep3.mir b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep3.mir
new file mode 100644
index 000000000000..ca59b97dd11e
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep3.mir
@@ -0,0 +1,109 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
+# REQUIRES: asserts
+
+# Test that loop-carried memory dependencies are added correctly.
+# The original code is as follows.
+#
+# ```
+# void f(int * restrict a, int * restrict b, int n) {
+#   for (int i = 0; i < n; i++) {
+#     a[i] += i;
+#     b[i] += a[i+1];
+#   }
+# }
+# ```
+# 
+# Loop-carried dependencies exist from load for a[i+1] to store for a[i].
+# FIXME: Currently the following dependencies are missed.
+#
+# Loop carried edges from SU(7)
+#   Order
+#     SU(5)
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  define dso_local void @f(ptr noalias nocapture noundef %a, ptr noalias nocapture noundef %b, i32 noundef %n) local_unnamed_addr {
+  entry:
+    %cmp11 = icmp sgt i32 %n, 0
+    br i1 %cmp11, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %.pre = load i32, ptr %a, align 4, !tbaa !5
+    %cgep = getelementptr i8, ptr %a, i32 4
+    br label %for.body
+
+  for.cond.cleanup:
+    ret void
+
+  for.body:
+    %lsr.iv15 = phi ptr [ %cgep, %for.body.preheader ], [ %cgep20, %for.body ]
+    %lsr.iv13 = phi i32 [ %n, %for.body.preheader ], [ %lsr.iv.next, %for.body ]
+    %lsr.iv = phi ptr [ %b, %for.body.preheader ], [ %cgep19, %for.body ]
+    %0 = phi i32 [ %2, %for.body ], [ %.pre, %for.body.preheader ]
+    %i.012 = phi i32 [ %add1, %for.body ], [ 0, %for.body.preheader ]
+    %1 = add i32 %0, %i.012
+    %cgep18 = getelementptr i8, ptr %lsr.iv15, i32 -4
+    store i32 %1, ptr %cgep18, align 4, !tbaa !5
+    %add1 = add nuw nsw i32 %i.012, 1
+    %2 = load i32, ptr %lsr.iv15, align 4, !tbaa !5
+    %3 = load i32, ptr %lsr.iv, align 4, !tbaa !5
+    %add4 = add nsw i32 %3, %2
+    store i32 %add4, ptr %lsr.iv, align 4, !tbaa !5
+    %lsr.iv.next = add i32 %lsr.iv13, -1
+    %exitcond.not = icmp eq i32 %lsr.iv.next, 0
+    %cgep19 = getelementptr i8, ptr %lsr.iv, i32 4
+    %cgep20 = getelementptr i8, ptr %lsr.iv15, i32 4
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  !5 = !{!6, !6, i64 0}
+  !6 = !{!"int", !7, i64 0}
+  !7 = !{!"omnipotent char", !8, i64 0}
+  !8 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    liveins: $r0, $r1, $r2
+  
+    %14:intregs = COPY $r2
+    %13:intregs = COPY $r1
+    %12:intregs = COPY $r0
+    %15:predregs = C2_cmpgti %14, 0
+    J2_jumpf %15, %bb.2, implicit-def dead $pc
+    J2_jump %bb.1, implicit-def dead $pc
+  
+  bb.1.for.body.preheader:
+    %0:intregs, %1:intregs = L2_loadri_pi %12, 4 :: (load (s32) from %ir.a, !tbaa !5)
+    %17:intregs = A2_tfrsi 0
+    %22:intregs = COPY %14
+    J2_loop0r %bb.3, %22, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+    J2_jump %bb.3, implicit-def dead $pc
+  
+  bb.2.for.cond.cleanup:
+    PS_jmpret $r31, implicit-def dead $pc
+  
+  bb.3.for.body:
+    successors: %bb.2, %bb.3
+  
+    %2:intregs = PHI %1, %bb.1, %11, %bb.3
+    %4:intregs = PHI %13, %bb.1, %10, %bb.3
+    %5:intregs = PHI %0, %bb.1, %8, %bb.3
+    %6:intregs = PHI %17, %bb.1, %7, %bb.3
+    %18:intregs = A2_add %5, %6
+    S2_storeri_io %2, -4, killed %18 :: (store (s32) into %ir.cgep18, !tbaa !5)
+    %7:intregs = nuw nsw A2_addi %6, 1
+    %8:intregs = L2_loadri_io %2, 0 :: (load (s32) from %ir.lsr.iv15, !tbaa !5)
+    %19:intregs = L2_loadri_io %4, 0 :: (load (s32) from %ir.lsr.iv, !tbaa !5)
+    %20:intregs = nsw A2_add killed %19, %8
+    %10:intregs = S2_storeri_pi %4, 4, killed %20 :: (store (s32) into %ir.lsr.iv, !tbaa !5)
+    %11:intregs = A2_addi %2, 4
+    ENDLOOP0 %bb.3, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def $pc
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep4.mir b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep4.mir
new file mode 100644
index 000000000000..4bc4b4873594
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep4.mir
@@ -0,0 +1,109 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
+# REQUIRES: asserts
+
+# Test that loop carried memory dependences are computed correctly.
+# The original code is as follows.
+#
+# ```
+# void f(int *a, int n) {
+#   for (int i = 0; i < n-2; i++) {
+#     a[i] += a[i+10];
+#     a[i+2] += i;
+#   }
+# }
+# ```
+#
+# Here is what each instruction does.
+# SU(2): Load a[i+10]
+# SU(3): Store it to a[i]
+# SU(4): Load a[i+2], add i, then store it
+#
+# FIXME: Currently the following dependencies are missed.
+#
+# Loop carried edges from SU(4)
+#   Order
+#     SU(3)
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT:   Loop carried edges from SU(2)
+# CHECK-NEXT:     Order
+# CHECK-NEXT:       SU(3)
+# CHECK-NEXT:       SU(4)
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  define dso_local void @f(ptr nocapture noundef %a, i32 noundef %n) {
+  entry:
+    %cmp13 = icmp sgt i32 %n, 2
+    br i1 %cmp13, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %0 = add i32 %n, -2
+    br label %for.body
+
+  for.cond.cleanup:
+    ret void
+
+  for.body:
+    %lsr.iv15 = phi ptr [ %a, %for.body.preheader ], [ %cgep19, %for.body ]
+    %lsr.iv = phi i32 [ %0, %for.body.preheader ], [ %lsr.iv.next, %for.body ]
+    %i.014 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+    %cgep = getelementptr i8, ptr %lsr.iv15, i32 40
+    %1 = load i32, ptr %cgep, align 4, !tbaa !5
+    %2 = load i32, ptr %lsr.iv15, align 4, !tbaa !5
+    %add2 = add nsw i32 %2, %1
+    store i32 %add2, ptr %lsr.iv15, align 4, !tbaa !5
+    %cgep18 = getelementptr i8, ptr %lsr.iv15, i32 8
+    %3 = load i32, ptr %cgep18, align 4, !tbaa !5
+    %4 = add i32 %i.014, %3
+    store i32 %4, ptr %cgep18, align 4, !tbaa !5
+    %inc = add nuw nsw i32 %i.014, 1
+    %lsr.iv.next = add i32 %lsr.iv, -1
+    %exitcond.not = icmp eq i32 %lsr.iv.next, 0
+    %cgep19 = getelementptr i8, ptr %lsr.iv15, i32 4
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  !5 = !{!6, !6, i64 0}
+  !6 = !{!"int", !7, i64 0}
+  !7 = !{!"omnipotent char", !8, i64 0}
+  !8 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    liveins: $r0, $r1
+  
+    %8:intregs = COPY $r1
+    %7:intregs = COPY $r0
+    %9:predregs = C2_cmpgti %8, 2
+    J2_jumpf %9, %bb.2, implicit-def dead $pc
+    J2_jump %bb.1, implicit-def dead $pc
+  
+  bb.1.for.body.preheader:
+    %0:intregs = A2_addi %8, -2
+    %11:intregs = A2_tfrsi 0
+    %14:intregs = COPY %0
+    J2_loop0r %bb.3, %14, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+    J2_jump %bb.3, implicit-def dead $pc
+  
+  bb.2.for.cond.cleanup:
+    PS_jmpret $r31, implicit-def dead $pc
+  
+  bb.3.for.body:
+    successors: %bb.2, %bb.3
+  
+    %1:intregs = PHI %7, %bb.1, %6, %bb.3
+    %3:intregs = PHI %11, %bb.1, %4, %bb.3
+    %12:intregs = L2_loadri_io %1, 40 :: (load (s32) from %ir.cgep, !tbaa !5)
+    L4_add_memopw_io %1, 0, killed %12 :: (store (s32) into %ir.lsr.iv15, !tbaa !5), (load (s32) from %ir.lsr.iv15, !tbaa !5)
+    L4_add_memopw_io %1, 8, %3 :: (store (s32) into %ir.cgep18, !tbaa !5), (load (s32) from %ir.cgep18, !tbaa !5)
+    %4:intregs = nuw nsw A2_addi %3, 1
+    %6:intregs = A2_addi %1, 4
+    ENDLOOP0 %bb.3, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def $pc
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep5.mir b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep5.mir
new file mode 100644
index 000000000000..77c3d569db18
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep5.mir
@@ -0,0 +1,111 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
+# REQUIRES: asserts
+
+# Test that loop carried memory dependencies are correctly added when two
+# arrays may point to the same memory location.
+#
+# ```
+# void f(int *a, int *b, int n) {
+#   for (int i = 0; i < n; i++) {
+#     a[i] += b[i];
+#     b[i] += a[i];
+#   }
+# }
+# ```
+#
+# Here is what each instruction does.
+# SU(2): Load b[i]
+# SU(3): Load a[i]
+# SU(5): Store a[i]
+# SU(6): Load b[i]
+# SU(8): Store b[i]
+#
+# Note that if there is already a dependency between two instructions, we don't
+# add loop-carried on between them since non-loop-carried one imposes stronger
+# constraint than loop-carried one.
+#
+# FIXME: Currently the following dependencies are missed.
+#  Loop carried edges from SU(5)
+#    Order
+#      SU(2)
+#  Loop carried edges from SU(6)
+#    Order
+#      SU(5)
+#  Loop carried edges from SU(8)
+#    Order
+#      SU(3)
+#      SU(5)
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  define dso_local void @f(ptr nocapture noundef %a, ptr nocapture noundef %b, i32 noundef %n) local_unnamed_addr {
+  entry:
+    %cmp12 = icmp sgt i32 %n, 0
+    br i1 %cmp12, label %for.body, label %for.cond.cleanup
+
+  for.cond.cleanup:
+    ret void
+
+  for.body:
+    %lsr.iv15 = phi ptr [ %cgep17, %for.body ], [ %b, %entry ]
+    %lsr.iv14 = phi ptr [ %cgep, %for.body ], [ %a, %entry ]
+    %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ %n, %entry ]
+    %0 = load i32, ptr %lsr.iv15, align 4, !tbaa !5
+    %1 = load i32, ptr %lsr.iv14, align 4, !tbaa !5
+    %add = add nsw i32 %1, %0
+    store i32 %add, ptr %lsr.iv14, align 4, !tbaa !5
+    %2 = load i32, ptr %lsr.iv15, align 4, !tbaa !5
+    %add4 = add nsw i32 %2, %add
+    store i32 %add4, ptr %lsr.iv15, align 4, !tbaa !5
+    %lsr.iv.next = add i32 %lsr.iv, -1
+    %exitcond.not = icmp eq i32 %lsr.iv.next, 0
+    %cgep = getelementptr i8, ptr %lsr.iv14, i32 4
+    %cgep17 = getelementptr i8, ptr %lsr.iv15, i32 4
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  !5 = !{!6, !6, i64 0}
+  !6 = !{!"int", !7, i64 0}
+  !7 = !{!"omnipotent char", !8, i64 0}
+  !8 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.3, %bb.1
+    liveins: $r0, $r1, $r2
+  
+    %8:intregs = COPY $r2
+    %7:intregs = COPY $r1
+    %6:intregs = COPY $r0
+    %9:predregs = C2_cmpgti %8, 0
+    J2_jumpf %9, %bb.1, implicit-def $pc
+  
+  bb.3:
+    %16:intregs = COPY %8
+    J2_loop0r %bb.2, %16, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+    J2_jump %bb.2, implicit-def $pc
+  
+  bb.1.for.cond.cleanup:
+    PS_jmpret $r31, implicit-def dead $pc
+  
+  bb.2.for.body:
+    successors: %bb.1, %bb.2
+  
+    %0:intregs = PHI %7, %bb.3, %5, %bb.2
+    %1:intregs = PHI %6, %bb.3, %4, %bb.2
+    %10:intregs = L2_loadri_io %0, 0 :: (load (s32) from %ir.lsr.iv15, !tbaa !5)
+    %11:intregs = L2_loadri_io %1, 0 :: (load (s32) from %ir.lsr.iv14, !tbaa !5)
+    %12:intregs = nsw A2_add killed %11, killed %10
+    %4:intregs = S2_storeri_pi %1, 4, %12 :: (store (s32) into %ir.lsr.iv14, !tbaa !5)
+    %13:intregs = L2_loadri_io %0, 0 :: (load (s32) from %ir.lsr.iv15, !tbaa !5)
+    %14:intregs = nsw A2_add killed %13, %12
+    %5:intregs = S2_storeri_pi %0, 4, killed %14 :: (store (s32) into %ir.lsr.iv15, !tbaa !5)
+    ENDLOOP0 %bb.2, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.1, implicit-def $pc
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir
new file mode 100644
index 000000000000..4281d1537714
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir
@@ -0,0 +1,154 @@
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
+# REQUIRES: asserts
+
+# Test that loop carried memory dependencies are computed correctly
+# when barrier instructions exist in the loop.
+# The original code is as follows.
+# 
+# ```
+# volatile int x = 0;
+# void f(int * restrict a, int * restrict b, int * restrict c, int n) {
+#   for (int i = 0; i < n; i++) {
+#     a[i] *= c[i];
+#     b[i] *= c[i];
+#     x += i;
+#     a[i + 1] *= i;
+#     x += i;
+#     b[i + 1] *= i;
+#   }
+# }
+# ```
+#
+# FIXME: Currently the following dependencies are missed.
+# Loop carried edges from SU(16)
+#   Order
+#     SU(6)
+#     SU(8)
+#     SU(10)
+#     SU(11)
+# Loop carried edges from SU(17)
+#   Order
+#     SU(10)
+#     SU(11)
+# Loop carried edges from SU(19)
+#   Order
+#     SU(10)
+#     SU(11)
+
+# CHECK:      ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: ===== Loop Carried Edges End =====
+
+--- |
+  @x = dso_local global i32 0, align 4
+
+  define dso_local void @f(ptr noalias nocapture noundef %a, ptr noalias nocapture noundef %b, ptr noalias nocapture noundef readonly %c, i32 noundef %n) {
+  entry:
+    %cmp26 = icmp sgt i32 %n, 0
+    br i1 %cmp26, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %.pre = load i32, ptr %a, align 4, !tbaa !5
+    %.pre28 = load i32, ptr %b, align 4, !tbaa !5
+    %cgep = getelementptr i8, ptr %b, i32 4
+    %cgep37 = getelementptr i8, ptr %a, i32 4
+    br label %for.body
+
+  for.cond.cleanup:
+    ret void
+
+  for.body:
+    %lsr.iv35 = phi ptr [ %c, %for.body.preheader ], [ %cgep42, %for.body ]
+    %lsr.iv31 = phi ptr [ %cgep37, %for.body.preheader ], [ %cgep41, %for.body ]
+    %lsr.iv = phi ptr [ %cgep, %for.body.preheader ], [ %cgep40, %for.body ]
+    %0 = phi i32 [ %mul11, %for.body ], [ %.pre28, %for.body.preheader ]
+    %1 = phi i32 [ %mul7, %for.body ], [ %.pre, %for.body.preheader ]
+    %i.027 = phi i32 [ %add5, %for.body ], [ 0, %for.body.preheader ]
+    %2 = load i32, ptr %lsr.iv35, align 4, !tbaa !5
+    %mul = mul nsw i32 %1, %2
+    %cgep38 = getelementptr i8, ptr %lsr.iv31, i32 -4
+    store i32 %mul, ptr %cgep38, align 4, !tbaa !5
+    %mul4 = mul nsw i32 %0, %2
+    %cgep39 = getelementptr i8, ptr %lsr.iv, i32 -4
+    store i32 %mul4, ptr %cgep39, align 4, !tbaa !5
+    %3 = load volatile i32, ptr @x, align 4, !tbaa !5
+    %4 = add i32 %i.027, %3
+    store volatile i32 %4, ptr @x, align 4, !tbaa !5
+    %add5 = add nuw nsw i32 %i.027, 1
+    %5 = load i32, ptr %lsr.iv31, align 4, !tbaa !5
+    %mul7 = mul nsw i32 %5, %i.027
+    store i32 %mul7, ptr %lsr.iv31, align 4, !tbaa !5
+    %6 = load volatile i32, ptr @x, align 4, !tbaa !5
+    %7 = add i32 %i.027, %6
+    store volatile i32 %7, ptr @x, align 4, !tbaa !5
+    %8 = load i32, ptr %lsr.iv, align 4, !tbaa !5
+    %mul11 = mul nsw i32 %8, %i.027
+    store i32 %mul11, ptr %lsr.iv, align 4, !tbaa !5
+    %exitcond.not = icmp eq i32 %n, %add5
+    %cgep40 = getelementptr i8, ptr %lsr.iv, i32 4
+    %cgep41 = getelementptr i8, ptr %lsr.iv31, i32 4
+    %cgep42 = getelementptr i8, ptr %lsr.iv35, i32 4
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+  !5 = !{!6, !6, i64 0}
+  !6 = !{!"int", !7, i64 0}
+  !7 = !{!"omnipotent char", !8, i64 0}
+  !8 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            f
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    liveins: $r0, $r1, $r2, $r3
+  
+    %19:intregs = COPY $r3
+    %18:intregs = COPY $r2
+    %17:intregs = COPY $r1
+    %16:intregs = COPY $r0
+    %20:predregs = C2_cmpgti %19, 0
+    J2_jumpf %20, %bb.2, implicit-def dead $pc
+    J2_jump %bb.1, implicit-def dead $pc
+  
+  bb.1.for.body.preheader:
+    %0:intregs, %3:intregs = L2_loadri_pi %16, 4 :: (load (s32) from %ir.a, !tbaa !5)
+    %1:intregs, %2:intregs = L2_loadri_pi %17, 4 :: (load (s32) from %ir.b, !tbaa !5)
+    %22:intregs = A2_tfrsi 0
+    %26:intregs = C4_addipc target-flags(hexagon-pcrel) @x
+    %30:intregs = COPY %19
+    J2_loop0r %bb.3, %30, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+    J2_jump %bb.3, implicit-def dead $pc
+  
+  bb.2.for.cond.cleanup:
+    PS_jmpret $r31, implicit-def dead $pc
+  
+  bb.3.for.body:
+    successors: %bb.2, %bb.3
+  
+    %4:intregs = PHI %18, %bb.1, %15, %bb.3
+    %5:intregs = PHI %3, %bb.1, %14, %bb.3
+    %6:intregs = PHI %2, %bb.1, %13, %bb.3
+    %7:intregs = PHI %1, %bb.1, %12, %bb.3
+    %8:intregs = PHI %0, %bb.1, %11, %bb.3
+    %9:intregs = PHI %22, %bb.1, %10, %bb.3
+    %23:intregs, %15:intregs = L2_loadri_pi %4, 4 :: (load (s32) from %ir.lsr.iv35, !tbaa !5)
+    %24:intregs = nsw M2_mpyi %8, %23
+    S2_storeri_io %5, -4, killed %24 :: (store (s32) into %ir.cgep38, !tbaa !5)
+    %25:intregs = nsw M2_mpyi %7, %23
+    S2_storeri_io %6, -4, killed %25 :: (store (s32) into %ir.cgep39, !tbaa !5)
+    L4_add_memopw_io %26, 0, %9 :: (volatile store (s32) into @x, !tbaa !5), (volatile dereferenceable load (s32) from @x, !tbaa !5)
+    %10:intregs = nuw nsw A2_addi %9, 1
+    %27:intregs = L2_loadri_io %5, 0 :: (load (s32) from %ir.lsr.iv31, !tbaa !5)
+    %11:intregs = nsw M2_mpyi killed %27, %9
+    S2_storeri_io %5, 0, %11 :: (store (s32) into %ir.lsr.iv31, !tbaa !5)
+    L4_add_memopw_io %26, 0, %9 :: (volatile store (s32) into @x, !tbaa !5), (volatile dereferenceable load (s32) from @x, !tbaa !5)
+    %28:intregs = L2_loadri_io %6, 0 :: (load (s32) from %ir.lsr.iv, !tbaa !5)
+    %12:intregs = nsw M2_mpyi killed %28, %9
+    S2_storeri_io %6, 0, %12 :: (store (s32) into %ir.lsr.iv, !tbaa !5)
+    %13:intregs = A2_addi %6, 4
+    %14:intregs = A2_addi %5, 4
+    ENDLOOP0 %bb.3, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def $pc
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/O0-pipeline.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
index d16cb1c15870..9006b5c8d6fe 100644
--- a/external/llvm-project/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
@@ -34,15 +34,7 @@
 ; CHECK-NEXT:       Safe Stack instrumentation pass
 ; CHECK-NEXT:       Insert stack protectors
 ; CHECK-NEXT:       Module Verifier
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Natural Loop Information
-; CHECK-NEXT:       Post-Dominator Tree Construction
-; CHECK-NEXT:       Branch Probability Analysis
 ; CHECK-NEXT:       Assignment Tracking Analysis
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       LoongArch DAG->DAG Pattern Instruction Selection
 ; CHECK-NEXT:       Finalize ISel and expand pseudo-instructions
 ; CHECK-NEXT:       Local Stack Slot Allocation
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/bf16-promote.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/bf16-promote.ll
new file mode 100644
index 000000000000..42651eb53ace
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/bf16-promote.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=loongarch64 -mattr=+d -target-abi=lp64d < %s | FileCheck --check-prefixes=CHECK,LA64 %s
+; RUN: llc -mtriple=loongarch32 -mattr=+d -target-abi=ilp32d < %s | FileCheck --check-prefixes=CHECK,LA32 %s
+
+define void @test_load_store(ptr %p, ptr %q) nounwind {
+; CHECK-LABEL: test_load_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    st.h $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %a = load bfloat, ptr %p
+  store bfloat %a, ptr %q
+  ret void
+}
+
+define float @test_fpextend_float(ptr %p) nounwind {
+; LA64-LABEL: test_fpextend_float:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ret
+;
+; LA32-LABEL: test_fpextend_float:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    slli.w $a0, $a0, 16
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    ret
+  %a = load bfloat, ptr %p
+  %r = fpext bfloat %a to float
+  ret float %r
+}
+
+define double @test_fpextend_double(ptr %p) nounwind {
+; LA64-LABEL: test_fpextend_double:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    fcvt.d.s $fa0, $fa0
+; LA64-NEXT:    ret
+;
+; LA32-LABEL: test_fpextend_double:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    slli.w $a0, $a0, 16
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    fcvt.d.s $fa0, $fa0
+; LA32-NEXT:    ret
+  %a = load bfloat, ptr %p
+  %r = fpext bfloat %a to double
+  ret double %r
+}
+
+define void @test_fptrunc_float(float %f, ptr %p) nounwind {
+; LA64-LABEL: test_fptrunc_float:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    st.h $a0, $fp, 0
+; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32-LABEL: test_fptrunc_float:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    bl __truncsfbf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    st.h $a0, $fp, 0
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+  %a = fptrunc float %f to bfloat
+  store bfloat %a, ptr %p
+  ret void
+}
+
+define void @test_fptrunc_double(double %d, ptr %p) nounwind {
+; LA64-LABEL: test_fptrunc_double:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncdfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    st.h $a0, $fp, 0
+; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32-LABEL: test_fptrunc_double:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    bl __truncdfbf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    st.h $a0, $fp, 0
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+  %a = fptrunc double %d to bfloat
+  store bfloat %a, ptr %p
+  ret void
+}
+
+define void @test_fadd(ptr %p, ptr %q) nounwind {
+; LA64-LABEL: test_fadd:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    ld.hu $a1, $a1, 0
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    slli.d $a1, $a1, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a1
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa1, $a0
+; LA64-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    st.h $a0, $fp, 0
+; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32-LABEL: test_fadd:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    ld.hu $a1, $a1, 0
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    movgr2fr.w $fa0, $a1
+; LA32-NEXT:    slli.w $a0, $a0, 16
+; LA32-NEXT:    movgr2fr.w $fa1, $a0
+; LA32-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32-NEXT:    bl __truncsfbf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    st.h $a0, $fp, 0
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+  %a = load bfloat, ptr %p
+  %b = load bfloat, ptr %q
+  %r = fadd bfloat %a, %b
+  store bfloat %r, ptr %p
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/bf16.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/bf16.ll
new file mode 100644
index 000000000000..e580bcc69f52
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/bf16.ll
@@ -0,0 +1,1048 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+;; For `double` parameters and return values, compiling on loongarch32 with `-mattr=+d` and
+;; `-target-abi=ilp32s` is incompatible, resulting in the error 'Passing f64 with GPR on LA32 is undefined'.
+;; Therefore, such case is currently skipped in testing.
+; RUN: llc -mtriple=loongarch32 -verify-machineinstrs < %s | FileCheck %s -check-prefix=LA32
+; RUN: llc -mtriple=loongarch64 -verify-machineinstrs < %s | FileCheck %s -check-prefix=LA64
+; RUN: llc -mtriple=loongarch32 -mattr=+f -target-abi=ilp32s -verify-machineinstrs < %s | FileCheck %s -check-prefix=LA32F-ILP32S
+; RUN: llc -mtriple=loongarch32 -mattr=+f -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s -check-prefix=LA32F-ILP32D
+; RUN: llc -mtriple=loongarch32 -mattr=+d -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s -check-prefix=LA32D-ILP32D
+; RUN: llc -mtriple=loongarch64 -mattr=+f -target-abi=lp64s -verify-machineinstrs < %s | FileCheck %s -check-prefix=LA64F-LP64S
+; RUN: llc -mtriple=loongarch64 -mattr=+f -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s -check-prefix=LA64F-LP64D
+; RUN: llc -mtriple=loongarch64 -mattr=+d -target-abi=lp64s -verify-machineinstrs < %s  | FileCheck %s -check-prefix=LA64D-LP64S
+; RUN: llc -mtriple=loongarch64 -mattr=+d -target-abi=lp64d -verify-machineinstrs < %s  | FileCheck %s -check-prefix=LA64D-LP64D
+
+define bfloat @float_to_bfloat(float %a) nounwind {
+; LA32-LABEL: float_to_bfloat:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl __truncsfbf2
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: float_to_bfloat:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: float_to_bfloat:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    bl __truncsfbf2
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: float_to_bfloat:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    bl __truncsfbf2
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: float_to_bfloat:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    bl __truncsfbf2
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: float_to_bfloat:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: float_to_bfloat:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: float_to_bfloat:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: float_to_bfloat:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = fptrunc float %a to bfloat
+  ret bfloat %1
+}
+
+define bfloat @double_to_bfloat(double %a) nounwind {
+; LA32-LABEL: double_to_bfloat:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl __truncdfbf2
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: double_to_bfloat:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncdfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: double_to_bfloat:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    bl __truncdfbf2
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: double_to_bfloat:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    bl __truncdfbf2
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: double_to_bfloat:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    bl __truncdfbf2
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: double_to_bfloat:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncdfbf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: double_to_bfloat:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncdfbf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: double_to_bfloat:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncdfbf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: double_to_bfloat:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncdfbf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = fptrunc double %a to bfloat
+  ret bfloat %1
+}
+
+define float @bfloat_to_float(bfloat %a) nounwind {
+; LA32-LABEL: bfloat_to_float:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a0, $a0, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bfloat_to_float:
+; LA64:       # %bb.0:
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: bfloat_to_float:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: bfloat_to_float:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: bfloat_to_float:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: bfloat_to_float:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: bfloat_to_float:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: bfloat_to_float:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: bfloat_to_float:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    ret
+  %1 = fpext bfloat %a to float
+  ret float %1
+}
+
+define double @bfloat_to_double(bfloat %a) nounwind {
+; LA32-LABEL: bfloat_to_double:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    slli.w $a0, $a0, 16
+; LA32-NEXT:    bl __extendsfdf2
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bfloat_to_double:
+; LA64:       # %bb.0:
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    fcvt.d.s $fa0, $fa0
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: bfloat_to_double:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32S-NEXT:    bl __extendsfdf2
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: bfloat_to_double:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    bl __extendsfdf2
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: bfloat_to_double:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    fcvt.d.s $fa0, $fa0
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: bfloat_to_double:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    fcvt.d.s $fa0, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: bfloat_to_double:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    fcvt.d.s $fa0, $fa0
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: bfloat_to_double:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    fcvt.d.s $fa0, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: bfloat_to_double:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    fcvt.d.s $fa0, $fa0
+; LA64D-LP64D-NEXT:    ret
+  %1 = fpext bfloat %a to double
+  ret double %1
+}
+
+define bfloat @i16_to_bfloat(i16 %a) nounwind {
+; LA32-LABEL: i16_to_bfloat:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: i16_to_bfloat:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: i16_to_bfloat:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: i16_to_bfloat:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: i16_to_bfloat:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: i16_to_bfloat:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: i16_to_bfloat:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: i16_to_bfloat:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: i16_to_bfloat:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    ret
+  %1 = bitcast i16 %a to bfloat
+  ret bfloat %1
+}
+
+define i16 @bfloat_to_i16(bfloat %a) nounwind {
+; LA32-LABEL: bfloat_to_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bfloat_to_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: bfloat_to_i16:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: bfloat_to_i16:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: bfloat_to_i16:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: bfloat_to_i16:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: bfloat_to_i16:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: bfloat_to_i16:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: bfloat_to_i16:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    ret
+  %1 = bitcast bfloat %a to i16
+  ret i16 %1
+}
+
+define bfloat @bfloat_add(bfloat %a, bfloat %b) nounwind {
+; LA32-LABEL: bfloat_add:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    slli.w $a0, $a0, 16
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    bl __addsf3
+; LA32-NEXT:    bl __truncsfbf2
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bfloat_add:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    movfr2gr.s $a1, $fa1
+; LA64-NEXT:    slli.d $a1, $a1, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a1
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa1, $a0
+; LA64-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: bfloat_add:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    slli.w $a1, $a1, 16
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a1
+; LA32F-ILP32S-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa1, $a0
+; LA32F-ILP32S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    bl __truncsfbf2
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: bfloat_add:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a1, $fa1
+; LA32F-ILP32D-NEXT:    slli.w $a1, $a1, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a1
+; LA32F-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa1, $a0
+; LA32F-ILP32D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32F-ILP32D-NEXT:    bl __truncsfbf2
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: bfloat_add:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a1, $fa1
+; LA32D-ILP32D-NEXT:    slli.w $a1, $a1, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a1
+; LA32D-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa1, $a0
+; LA32D-ILP32D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32D-ILP32D-NEXT:    bl __truncsfbf2
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: bfloat_add:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    slli.d $a1, $a1, 16
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a1
+; LA64F-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa1, $a0
+; LA64F-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: bfloat_add:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a1, $fa1
+; LA64F-LP64D-NEXT:    slli.d $a1, $a1, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a1
+; LA64F-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa1, $a0
+; LA64F-LP64D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: bfloat_add:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    slli.d $a1, $a1, 16
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a1
+; LA64D-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa1, $a0
+; LA64D-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: bfloat_add:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a1, $fa1
+; LA64D-LP64D-NEXT:    slli.d $a1, $a1, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a1
+; LA64D-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa1, $a0
+; LA64D-LP64D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = fadd bfloat %a, %b
+  ret bfloat %1
+}
+
+define bfloat @bfloat_load(ptr %a) nounwind {
+; LA32-LABEL: bfloat_load:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ld.h $a1, $a0, 0
+; LA32-NEXT:    ld.h $a2, $a0, 6
+; LA32-NEXT:    slli.w $a0, $a1, 16
+; LA32-NEXT:    slli.w $a1, $a2, 16
+; LA32-NEXT:    bl __addsf3
+; LA32-NEXT:    bl __truncsfbf2
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bfloat_load:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    ld.hu $a1, $a0, 6
+; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    slli.d $a1, $a1, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a1
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa1, $a0
+; LA64-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: bfloat_load:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    ld.hu $a1, $a0, 6
+; LA32F-ILP32S-NEXT:    ld.hu $a0, $a0, 0
+; LA32F-ILP32S-NEXT:    slli.w $a1, $a1, 16
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a1
+; LA32F-ILP32S-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa1, $a0
+; LA32F-ILP32S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    bl __truncsfbf2
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: bfloat_load:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    ld.hu $a1, $a0, 6
+; LA32F-ILP32D-NEXT:    ld.hu $a0, $a0, 0
+; LA32F-ILP32D-NEXT:    slli.w $a1, $a1, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a1
+; LA32F-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa1, $a0
+; LA32F-ILP32D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32F-ILP32D-NEXT:    bl __truncsfbf2
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: bfloat_load:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    ld.hu $a1, $a0, 6
+; LA32D-ILP32D-NEXT:    ld.hu $a0, $a0, 0
+; LA32D-ILP32D-NEXT:    slli.w $a1, $a1, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a1
+; LA32D-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa1, $a0
+; LA32D-ILP32D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32D-ILP32D-NEXT:    bl __truncsfbf2
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: bfloat_load:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    ld.hu $a1, $a0, 6
+; LA64F-LP64S-NEXT:    ld.hu $a0, $a0, 0
+; LA64F-LP64S-NEXT:    slli.d $a1, $a1, 16
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a1
+; LA64F-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa1, $a0
+; LA64F-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: bfloat_load:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    ld.hu $a1, $a0, 6
+; LA64F-LP64D-NEXT:    ld.hu $a0, $a0, 0
+; LA64F-LP64D-NEXT:    slli.d $a1, $a1, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a1
+; LA64F-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa1, $a0
+; LA64F-LP64D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: bfloat_load:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    ld.hu $a1, $a0, 6
+; LA64D-LP64S-NEXT:    ld.hu $a0, $a0, 0
+; LA64D-LP64S-NEXT:    slli.d $a1, $a1, 16
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a1
+; LA64D-LP64S-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa1, $a0
+; LA64D-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: bfloat_load:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    ld.hu $a1, $a0, 6
+; LA64D-LP64D-NEXT:    ld.hu $a0, $a0, 0
+; LA64D-LP64D-NEXT:    slli.d $a1, $a1, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a1
+; LA64D-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa1, $a0
+; LA64D-LP64D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = load bfloat, ptr %a
+  %2 = getelementptr bfloat, ptr %a, i32 3
+  %3 = load bfloat, ptr %2
+  %4 = fadd bfloat %1, %3
+  ret bfloat %4
+}
+
+define void @bfloat_store(ptr %a, bfloat %b, bfloat %c) nounwind {
+; LA32-LABEL: bfloat_store:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    slli.w $a0, $a1, 16
+; LA32-NEXT:    slli.w $a1, $a2, 16
+; LA32-NEXT:    bl __addsf3
+; LA32-NEXT:    bl __truncsfbf2
+; LA32-NEXT:    st.h $a0, $fp, 0
+; LA32-NEXT:    st.h $a0, $fp, 16
+; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bfloat_store:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    movfr2gr.s $a1, $fa1
+; LA64-NEXT:    slli.d $a1, $a1, 16
+; LA64-NEXT:    movgr2fr.w $fa0, $a1
+; LA64-NEXT:    slli.d $a0, $a0, 16
+; LA64-NEXT:    movgr2fr.w $fa1, $a0
+; LA64-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    st.h $a0, $fp, 0
+; LA64-NEXT:    st.h $a0, $fp, 16
+; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: bfloat_store:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    move $fp, $a0
+; LA32F-ILP32S-NEXT:    slli.w $a0, $a2, 16
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    slli.w $a0, $a1, 16
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa1, $a0
+; LA32F-ILP32S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    bl __truncsfbf2
+; LA32F-ILP32S-NEXT:    st.h $a0, $fp, 0
+; LA32F-ILP32S-NEXT:    st.h $a0, $fp, 16
+; LA32F-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: bfloat_store:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    move $fp, $a0
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a1, $fa1
+; LA32F-ILP32D-NEXT:    slli.w $a1, $a1, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a1
+; LA32F-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa1, $a0
+; LA32F-ILP32D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32F-ILP32D-NEXT:    bl __truncsfbf2
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    st.h $a0, $fp, 0
+; LA32F-ILP32D-NEXT:    st.h $a0, $fp, 16
+; LA32F-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: bfloat_store:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    move $fp, $a0
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a1, $fa1
+; LA32D-ILP32D-NEXT:    slli.w $a1, $a1, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a1
+; LA32D-ILP32D-NEXT:    slli.w $a0, $a0, 16
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa1, $a0
+; LA32D-ILP32D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32D-ILP32D-NEXT:    bl __truncsfbf2
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    st.h $a0, $fp, 0
+; LA32D-ILP32D-NEXT:    st.h $a0, $fp, 16
+; LA32D-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: bfloat_store:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    move $fp, $a0
+; LA64F-LP64S-NEXT:    slli.d $a0, $a2, 16
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    slli.d $a0, $a1, 16
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa1, $a0
+; LA64F-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    st.h $a0, $fp, 0
+; LA64F-LP64S-NEXT:    st.h $a0, $fp, 16
+; LA64F-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: bfloat_store:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    move $fp, $a0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a1, $fa1
+; LA64F-LP64D-NEXT:    slli.d $a1, $a1, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a1
+; LA64F-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa1, $a0
+; LA64F-LP64D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    st.h $a0, $fp, 0
+; LA64F-LP64D-NEXT:    st.h $a0, $fp, 16
+; LA64F-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: bfloat_store:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    move $fp, $a0
+; LA64D-LP64S-NEXT:    slli.d $a0, $a2, 16
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    slli.d $a0, $a1, 16
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa1, $a0
+; LA64D-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    st.h $a0, $fp, 0
+; LA64D-LP64S-NEXT:    st.h $a0, $fp, 16
+; LA64D-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: bfloat_store:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    move $fp, $a0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a1, $fa1
+; LA64D-LP64D-NEXT:    slli.d $a1, $a1, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a1
+; LA64D-LP64D-NEXT:    slli.d $a0, $a0, 16
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa1, $a0
+; LA64D-LP64D-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfbf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    st.h $a0, $fp, 0
+; LA64D-LP64D-NEXT:    st.h $a0, $fp, 16
+; LA64D-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = fadd bfloat %b, %c
+  store bfloat %1, ptr %a
+  %2 = getelementptr bfloat, ptr %a, i32 8
+  store bfloat %1, ptr %2
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/calling-conv-half.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
new file mode 100644
index 000000000000..c88b67f13d1e
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
@@ -0,0 +1,1628 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32S
+; RUN: llc --mtriple=loongarch32 --mattr=+f -target-abi=ilp32s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32F-ILP32S
+; RUN: llc --mtriple=loongarch32 --mattr=+f -target-abi=ilp32d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32F-ILP32D
+; RUN: llc --mtriple=loongarch32 --mattr=+d -target-abi=ilp32s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32D-ILP32S
+; RUN: llc --mtriple=loongarch32 --mattr=+d -target-abi=ilp32d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32D-ILP32D
+; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64S
+; RUN: llc --mtriple=loongarch64 --mattr=+f -target-abi=lp64s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64F-LP64S
+; RUN: llc --mtriple=loongarch64 --mattr=+f -target-abi=lp64d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64F-LP64D
+; RUN: llc --mtriple=loongarch64 --mattr=+d -target-abi=lp64s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64D-LP64S
+; RUN: llc --mtriple=loongarch64 --mattr=+d -target-abi=lp64d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64D-LP64D
+
+define i32 @callee_half_in_fregs(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, half %i) nounwind {
+; LA32S-LABEL: callee_half_in_fregs:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -16
+; LA32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32S-NEXT:    ld.hu $a1, $sp, 16
+; LA32S-NEXT:    move $fp, $a0
+; LA32S-NEXT:    move $a0, $a1
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    bl __fixsfsi
+; LA32S-NEXT:    add.w $a0, $fp, $a0
+; LA32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 16
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: callee_half_in_fregs:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    ld.hu $a1, $sp, 16
+; LA32F-ILP32S-NEXT:    move $fp, $a0
+; LA32F-ILP32S-NEXT:    move $a0, $a1
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    add.w $a0, $fp, $a0
+; LA32F-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: callee_half_in_fregs:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    move $fp, $a0
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    add.w $a0, $fp, $a0
+; LA32F-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: callee_half_in_fregs:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    ld.hu $a1, $sp, 16
+; LA32D-ILP32S-NEXT:    move $fp, $a0
+; LA32D-ILP32S-NEXT:    move $a0, $a1
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32S-NEXT:    add.w $a0, $fp, $a0
+; LA32D-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: callee_half_in_fregs:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    move $fp, $a0
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    add.w $a0, $fp, $a0
+; LA32D-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: callee_half_in_fregs:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -16
+; LA64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64S-NEXT:    move $fp, $a0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64S-NEXT:    add.w $a0, $fp, $a0
+; LA64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 16
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: callee_half_in_fregs:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    ld.hu $a1, $sp, 16
+; LA64F-LP64S-NEXT:    move $fp, $a0
+; LA64F-LP64S-NEXT:    move $a0, $a1
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64S-NEXT:    add.w $a0, $fp, $a0
+; LA64F-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: callee_half_in_fregs:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    move $fp, $a0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64D-NEXT:    add.w $a0, $fp, $a0
+; LA64F-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: callee_half_in_fregs:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    ld.hu $a1, $sp, 16
+; LA64D-LP64S-NEXT:    move $fp, $a0
+; LA64D-LP64S-NEXT:    move $a0, $a1
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64S-NEXT:    add.w $a0, $fp, $a0
+; LA64D-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: callee_half_in_fregs:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    move $fp, $a0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64D-NEXT:    add.w $a0, $fp, $a0
+; LA64D-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = fptosi half %i to i32
+  %2 = add i32 %a, %1
+  ret i32 %2
+}
+
+define i32 @caller_half_in_fregs() nounwind {
+; LA32S-LABEL: caller_half_in_fregs:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -16
+; LA32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32S-NEXT:    lu12i.w $t0, 4
+; LA32S-NEXT:    ori $a0, $zero, 1
+; LA32S-NEXT:    ori $a1, $zero, 2
+; LA32S-NEXT:    ori $a2, $zero, 3
+; LA32S-NEXT:    ori $a3, $zero, 4
+; LA32S-NEXT:    ori $a4, $zero, 5
+; LA32S-NEXT:    ori $a5, $zero, 6
+; LA32S-NEXT:    ori $a6, $zero, 7
+; LA32S-NEXT:    ori $a7, $zero, 8
+; LA32S-NEXT:    st.w $t0, $sp, 0
+; LA32S-NEXT:    bl callee_half_in_fregs
+; LA32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 16
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: caller_half_in_fregs:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    lu12i.w $t0, -12
+; LA32F-ILP32S-NEXT:    ori $a0, $zero, 1
+; LA32F-ILP32S-NEXT:    ori $a1, $zero, 2
+; LA32F-ILP32S-NEXT:    ori $a2, $zero, 3
+; LA32F-ILP32S-NEXT:    ori $a3, $zero, 4
+; LA32F-ILP32S-NEXT:    ori $a4, $zero, 5
+; LA32F-ILP32S-NEXT:    ori $a5, $zero, 6
+; LA32F-ILP32S-NEXT:    ori $a6, $zero, 7
+; LA32F-ILP32S-NEXT:    ori $a7, $zero, 8
+; LA32F-ILP32S-NEXT:    st.w $t0, $sp, 0
+; LA32F-ILP32S-NEXT:    bl callee_half_in_fregs
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: caller_half_in_fregs:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
+; LA32F-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI1_0)
+; LA32F-ILP32D-NEXT:    ori $a0, $zero, 1
+; LA32F-ILP32D-NEXT:    ori $a1, $zero, 2
+; LA32F-ILP32D-NEXT:    ori $a2, $zero, 3
+; LA32F-ILP32D-NEXT:    ori $a3, $zero, 4
+; LA32F-ILP32D-NEXT:    ori $a4, $zero, 5
+; LA32F-ILP32D-NEXT:    ori $a5, $zero, 6
+; LA32F-ILP32D-NEXT:    ori $a6, $zero, 7
+; LA32F-ILP32D-NEXT:    ori $a7, $zero, 8
+; LA32F-ILP32D-NEXT:    bl callee_half_in_fregs
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: caller_half_in_fregs:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    lu12i.w $t0, -12
+; LA32D-ILP32S-NEXT:    ori $a0, $zero, 1
+; LA32D-ILP32S-NEXT:    ori $a1, $zero, 2
+; LA32D-ILP32S-NEXT:    ori $a2, $zero, 3
+; LA32D-ILP32S-NEXT:    ori $a3, $zero, 4
+; LA32D-ILP32S-NEXT:    ori $a4, $zero, 5
+; LA32D-ILP32S-NEXT:    ori $a5, $zero, 6
+; LA32D-ILP32S-NEXT:    ori $a6, $zero, 7
+; LA32D-ILP32S-NEXT:    ori $a7, $zero, 8
+; LA32D-ILP32S-NEXT:    st.w $t0, $sp, 0
+; LA32D-ILP32S-NEXT:    bl callee_half_in_fregs
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: caller_half_in_fregs:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
+; LA32D-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI1_0)
+; LA32D-ILP32D-NEXT:    ori $a0, $zero, 1
+; LA32D-ILP32D-NEXT:    ori $a1, $zero, 2
+; LA32D-ILP32D-NEXT:    ori $a2, $zero, 3
+; LA32D-ILP32D-NEXT:    ori $a3, $zero, 4
+; LA32D-ILP32D-NEXT:    ori $a4, $zero, 5
+; LA32D-ILP32D-NEXT:    ori $a5, $zero, 6
+; LA32D-ILP32D-NEXT:    ori $a6, $zero, 7
+; LA32D-ILP32D-NEXT:    ori $a7, $zero, 8
+; LA32D-ILP32D-NEXT:    bl callee_half_in_fregs
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: caller_half_in_fregs:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -16
+; LA64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
+; LA64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI1_0)
+; LA64S-NEXT:    ori $a0, $zero, 1
+; LA64S-NEXT:    ori $a1, $zero, 2
+; LA64S-NEXT:    ori $a2, $zero, 3
+; LA64S-NEXT:    ori $a3, $zero, 4
+; LA64S-NEXT:    ori $a4, $zero, 5
+; LA64S-NEXT:    ori $a5, $zero, 6
+; LA64S-NEXT:    ori $a6, $zero, 7
+; LA64S-NEXT:    ori $a7, $zero, 8
+; LA64S-NEXT:    pcaddu18i $ra, %call36(callee_half_in_fregs)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 16
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: caller_half_in_fregs:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    lu12i.w $t0, -12
+; LA64F-LP64S-NEXT:    lu32i.d $t0, 0
+; LA64F-LP64S-NEXT:    ori $a0, $zero, 1
+; LA64F-LP64S-NEXT:    ori $a1, $zero, 2
+; LA64F-LP64S-NEXT:    ori $a2, $zero, 3
+; LA64F-LP64S-NEXT:    ori $a3, $zero, 4
+; LA64F-LP64S-NEXT:    ori $a4, $zero, 5
+; LA64F-LP64S-NEXT:    ori $a5, $zero, 6
+; LA64F-LP64S-NEXT:    ori $a6, $zero, 7
+; LA64F-LP64S-NEXT:    ori $a7, $zero, 8
+; LA64F-LP64S-NEXT:    st.w $t0, $sp, 0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_in_fregs)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: caller_half_in_fregs:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
+; LA64F-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI1_0)
+; LA64F-LP64D-NEXT:    ori $a0, $zero, 1
+; LA64F-LP64D-NEXT:    ori $a1, $zero, 2
+; LA64F-LP64D-NEXT:    ori $a2, $zero, 3
+; LA64F-LP64D-NEXT:    ori $a3, $zero, 4
+; LA64F-LP64D-NEXT:    ori $a4, $zero, 5
+; LA64F-LP64D-NEXT:    ori $a5, $zero, 6
+; LA64F-LP64D-NEXT:    ori $a6, $zero, 7
+; LA64F-LP64D-NEXT:    ori $a7, $zero, 8
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_in_fregs)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: caller_half_in_fregs:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    lu12i.w $t0, -12
+; LA64D-LP64S-NEXT:    lu32i.d $t0, 0
+; LA64D-LP64S-NEXT:    ori $a0, $zero, 1
+; LA64D-LP64S-NEXT:    ori $a1, $zero, 2
+; LA64D-LP64S-NEXT:    ori $a2, $zero, 3
+; LA64D-LP64S-NEXT:    ori $a3, $zero, 4
+; LA64D-LP64S-NEXT:    ori $a4, $zero, 5
+; LA64D-LP64S-NEXT:    ori $a5, $zero, 6
+; LA64D-LP64S-NEXT:    ori $a6, $zero, 7
+; LA64D-LP64S-NEXT:    ori $a7, $zero, 8
+; LA64D-LP64S-NEXT:    st.w $t0, $sp, 0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_in_fregs)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: caller_half_in_fregs:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
+; LA64D-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI1_0)
+; LA64D-LP64D-NEXT:    ori $a0, $zero, 1
+; LA64D-LP64D-NEXT:    ori $a1, $zero, 2
+; LA64D-LP64D-NEXT:    ori $a2, $zero, 3
+; LA64D-LP64D-NEXT:    ori $a3, $zero, 4
+; LA64D-LP64D-NEXT:    ori $a4, $zero, 5
+; LA64D-LP64D-NEXT:    ori $a5, $zero, 6
+; LA64D-LP64D-NEXT:    ori $a6, $zero, 7
+; LA64D-LP64D-NEXT:    ori $a7, $zero, 8
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_in_fregs)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = call i32 @callee_half_in_fregs(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, half 2.0)
+  ret i32 %1
+}
+
+define i32 @callee_half_in_gregs(half %a, half %b, half %c, half %d, half %e, half %f, half %g, half %h, half %i, i32 %j) nounwind {
+; LA32S-LABEL: callee_half_in_gregs:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -16
+; LA32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32S-NEXT:    ld.w $fp, $sp, 20
+; LA32S-NEXT:    ld.hu $a0, $sp, 16
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    bl __fixsfsi
+; LA32S-NEXT:    add.w $a0, $fp, $a0
+; LA32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 16
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: callee_half_in_gregs:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    ld.w $fp, $sp, 20
+; LA32F-ILP32S-NEXT:    ld.hu $a0, $sp, 16
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    add.w $a0, $fp, $a0
+; LA32F-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: callee_half_in_gregs:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    move $fp, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    add.w $a0, $fp, $a0
+; LA32F-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: callee_half_in_gregs:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    ld.w $fp, $sp, 20
+; LA32D-ILP32S-NEXT:    ld.hu $a0, $sp, 16
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32S-NEXT:    add.w $a0, $fp, $a0
+; LA32D-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: callee_half_in_gregs:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    move $fp, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    add.w $a0, $fp, $a0
+; LA32D-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: callee_half_in_gregs:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -16
+; LA64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64S-NEXT:    move $fp, $a1
+; LA64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64S-NEXT:    add.w $a0, $fp, $a0
+; LA64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 16
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: callee_half_in_gregs:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    ld.w $fp, $sp, 24
+; LA64F-LP64S-NEXT:    ld.hu $a0, $sp, 16
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64S-NEXT:    add.w $a0, $fp, $a0
+; LA64F-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: callee_half_in_gregs:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    move $fp, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64D-NEXT:    add.w $a0, $fp, $a0
+; LA64F-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: callee_half_in_gregs:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    ld.w $fp, $sp, 24
+; LA64D-LP64S-NEXT:    ld.hu $a0, $sp, 16
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64S-NEXT:    add.w $a0, $fp, $a0
+; LA64D-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: callee_half_in_gregs:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    move $fp, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64D-NEXT:    add.w $a0, $fp, $a0
+; LA64D-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = fptosi half %i to i32
+  %2 = add i32 %j, %1
+  ret i32 %2
+}
+
+define i32 @caller_half_in_gregs() nounwind {
+; LA32S-LABEL: caller_half_in_gregs:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -16
+; LA32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32S-NEXT:    ori $a0, $zero, 10
+; LA32S-NEXT:    st.w $a0, $sp, 4
+; LA32S-NEXT:    lu12i.w $a1, 4
+; LA32S-NEXT:    ori $t0, $a1, 2176
+; LA32S-NEXT:    lu12i.w $a0, 3
+; LA32S-NEXT:    ori $a0, $a0, 3072
+; LA32S-NEXT:    ori $a2, $a1, 512
+; LA32S-NEXT:    ori $a3, $a1, 1024
+; LA32S-NEXT:    ori $a4, $a1, 1280
+; LA32S-NEXT:    ori $a5, $a1, 1536
+; LA32S-NEXT:    ori $a6, $a1, 1792
+; LA32S-NEXT:    ori $a7, $a1, 2048
+; LA32S-NEXT:    st.w $t0, $sp, 0
+; LA32S-NEXT:    bl callee_half_in_gregs
+; LA32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 16
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: caller_half_in_gregs:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    ori $a0, $zero, 10
+; LA32F-ILP32S-NEXT:    st.w $a0, $sp, 4
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -12
+; LA32F-ILP32S-NEXT:    ori $t0, $a1, 2176
+; LA32F-ILP32S-NEXT:    lu12i.w $a0, -13
+; LA32F-ILP32S-NEXT:    ori $a0, $a0, 3072
+; LA32F-ILP32S-NEXT:    ori $a2, $a1, 512
+; LA32F-ILP32S-NEXT:    ori $a3, $a1, 1024
+; LA32F-ILP32S-NEXT:    ori $a4, $a1, 1280
+; LA32F-ILP32S-NEXT:    ori $a5, $a1, 1536
+; LA32F-ILP32S-NEXT:    ori $a6, $a1, 1792
+; LA32F-ILP32S-NEXT:    ori $a7, $a1, 2048
+; LA32F-ILP32S-NEXT:    st.w $t0, $sp, 0
+; LA32F-ILP32S-NEXT:    bl callee_half_in_gregs
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: caller_half_in_gregs:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; LA32F-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI3_0)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; LA32F-ILP32D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI3_1)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_2)
+; LA32F-ILP32D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI3_2)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_3)
+; LA32F-ILP32D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI3_3)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_4)
+; LA32F-ILP32D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI3_4)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_5)
+; LA32F-ILP32D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI3_5)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_6)
+; LA32F-ILP32D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI3_6)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_7)
+; LA32F-ILP32D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI3_7)
+; LA32F-ILP32D-NEXT:    lu12i.w $a0, -12
+; LA32F-ILP32D-NEXT:    ori $a0, $a0, 2176
+; LA32F-ILP32D-NEXT:    ori $a1, $zero, 10
+; LA32F-ILP32D-NEXT:    bl callee_half_in_gregs
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: caller_half_in_gregs:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    ori $a0, $zero, 10
+; LA32D-ILP32S-NEXT:    st.w $a0, $sp, 4
+; LA32D-ILP32S-NEXT:    lu12i.w $a1, -12
+; LA32D-ILP32S-NEXT:    ori $t0, $a1, 2176
+; LA32D-ILP32S-NEXT:    lu12i.w $a0, -13
+; LA32D-ILP32S-NEXT:    ori $a0, $a0, 3072
+; LA32D-ILP32S-NEXT:    ori $a2, $a1, 512
+; LA32D-ILP32S-NEXT:    ori $a3, $a1, 1024
+; LA32D-ILP32S-NEXT:    ori $a4, $a1, 1280
+; LA32D-ILP32S-NEXT:    ori $a5, $a1, 1536
+; LA32D-ILP32S-NEXT:    ori $a6, $a1, 1792
+; LA32D-ILP32S-NEXT:    ori $a7, $a1, 2048
+; LA32D-ILP32S-NEXT:    st.w $t0, $sp, 0
+; LA32D-ILP32S-NEXT:    bl callee_half_in_gregs
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: caller_half_in_gregs:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; LA32D-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI3_0)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; LA32D-ILP32D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI3_1)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_2)
+; LA32D-ILP32D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI3_2)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_3)
+; LA32D-ILP32D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI3_3)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_4)
+; LA32D-ILP32D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI3_4)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_5)
+; LA32D-ILP32D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI3_5)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_6)
+; LA32D-ILP32D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI3_6)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_7)
+; LA32D-ILP32D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI3_7)
+; LA32D-ILP32D-NEXT:    lu12i.w $a0, -12
+; LA32D-ILP32D-NEXT:    ori $a0, $a0, 2176
+; LA32D-ILP32D-NEXT:    ori $a1, $zero, 10
+; LA32D-ILP32D-NEXT:    bl callee_half_in_gregs
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: caller_half_in_gregs:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -16
+; LA64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; LA64S-NEXT:    fld.s $ft0, $a0, %pc_lo12(.LCPI3_0)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; LA64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI3_1)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_2)
+; LA64S-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI3_2)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_3)
+; LA64S-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI3_3)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_4)
+; LA64S-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI3_4)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_5)
+; LA64S-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI3_5)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_6)
+; LA64S-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI3_6)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_7)
+; LA64S-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI3_7)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_8)
+; LA64S-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI3_8)
+; LA64S-NEXT:    movfr2gr.s $a0, $ft0
+; LA64S-NEXT:    ori $a1, $zero, 10
+; LA64S-NEXT:    pcaddu18i $ra, %call36(callee_half_in_gregs)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 16
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: caller_half_in_gregs:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -32
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; LA64F-LP64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI3_0)
+; LA64F-LP64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; LA64F-LP64S-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI3_1)
+; LA64F-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64S-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI3_2)
+; LA64F-LP64S-NEXT:    fld.s $fa0, $a1, %pc_lo12(.LCPI3_2)
+; LA64F-LP64S-NEXT:    movfr2gr.s $a1, $fa1
+; LA64F-LP64S-NEXT:    pcalau12i $a2, %pc_hi20(.LCPI3_3)
+; LA64F-LP64S-NEXT:    fld.s $fa1, $a2, %pc_lo12(.LCPI3_3)
+; LA64F-LP64S-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-LP64S-NEXT:    pcalau12i $a3, %pc_hi20(.LCPI3_4)
+; LA64F-LP64S-NEXT:    fld.s $fa0, $a3, %pc_lo12(.LCPI3_4)
+; LA64F-LP64S-NEXT:    movfr2gr.s $a3, $fa1
+; LA64F-LP64S-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI3_5)
+; LA64F-LP64S-NEXT:    fld.s $fa1, $a4, %pc_lo12(.LCPI3_5)
+; LA64F-LP64S-NEXT:    movfr2gr.s $a4, $fa0
+; LA64F-LP64S-NEXT:    pcalau12i $a5, %pc_hi20(.LCPI3_6)
+; LA64F-LP64S-NEXT:    fld.s $fa0, $a5, %pc_lo12(.LCPI3_6)
+; LA64F-LP64S-NEXT:    movfr2gr.s $a5, $fa1
+; LA64F-LP64S-NEXT:    ori $a6, $zero, 10
+; LA64F-LP64S-NEXT:    st.d $a6, $sp, 8
+; LA64F-LP64S-NEXT:    movfr2gr.s $a6, $fa0
+; LA64F-LP64S-NEXT:    pcalau12i $a7, %pc_hi20(.LCPI3_7)
+; LA64F-LP64S-NEXT:    fld.s $fa0, $a7, %pc_lo12(.LCPI3_7)
+; LA64F-LP64S-NEXT:    lu12i.w $a7, -12
+; LA64F-LP64S-NEXT:    ori $t0, $a7, 2176
+; LA64F-LP64S-NEXT:    lu32i.d $t0, 0
+; LA64F-LP64S-NEXT:    movfr2gr.s $a7, $fa0
+; LA64F-LP64S-NEXT:    st.w $t0, $sp, 0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_in_gregs)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 32
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: caller_half_in_gregs:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; LA64F-LP64D-NEXT:    fld.s $ft0, $a0, %pc_lo12(.LCPI3_0)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; LA64F-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI3_1)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_2)
+; LA64F-LP64D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI3_2)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_3)
+; LA64F-LP64D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI3_3)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_4)
+; LA64F-LP64D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI3_4)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_5)
+; LA64F-LP64D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI3_5)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_6)
+; LA64F-LP64D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI3_6)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_7)
+; LA64F-LP64D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI3_7)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_8)
+; LA64F-LP64D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI3_8)
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $ft0
+; LA64F-LP64D-NEXT:    ori $a1, $zero, 10
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_in_gregs)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: caller_half_in_gregs:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -32
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; LA64D-LP64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI3_0)
+; LA64D-LP64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; LA64D-LP64S-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI3_1)
+; LA64D-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64S-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI3_2)
+; LA64D-LP64S-NEXT:    fld.s $fa0, $a1, %pc_lo12(.LCPI3_2)
+; LA64D-LP64S-NEXT:    movfr2gr.s $a1, $fa1
+; LA64D-LP64S-NEXT:    pcalau12i $a2, %pc_hi20(.LCPI3_3)
+; LA64D-LP64S-NEXT:    fld.s $fa1, $a2, %pc_lo12(.LCPI3_3)
+; LA64D-LP64S-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-LP64S-NEXT:    pcalau12i $a3, %pc_hi20(.LCPI3_4)
+; LA64D-LP64S-NEXT:    fld.s $fa0, $a3, %pc_lo12(.LCPI3_4)
+; LA64D-LP64S-NEXT:    movfr2gr.s $a3, $fa1
+; LA64D-LP64S-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI3_5)
+; LA64D-LP64S-NEXT:    fld.s $fa1, $a4, %pc_lo12(.LCPI3_5)
+; LA64D-LP64S-NEXT:    movfr2gr.s $a4, $fa0
+; LA64D-LP64S-NEXT:    pcalau12i $a5, %pc_hi20(.LCPI3_6)
+; LA64D-LP64S-NEXT:    fld.s $fa0, $a5, %pc_lo12(.LCPI3_6)
+; LA64D-LP64S-NEXT:    movfr2gr.s $a5, $fa1
+; LA64D-LP64S-NEXT:    ori $a6, $zero, 10
+; LA64D-LP64S-NEXT:    st.d $a6, $sp, 8
+; LA64D-LP64S-NEXT:    movfr2gr.s $a6, $fa0
+; LA64D-LP64S-NEXT:    pcalau12i $a7, %pc_hi20(.LCPI3_7)
+; LA64D-LP64S-NEXT:    fld.s $fa0, $a7, %pc_lo12(.LCPI3_7)
+; LA64D-LP64S-NEXT:    lu12i.w $a7, -12
+; LA64D-LP64S-NEXT:    ori $t0, $a7, 2176
+; LA64D-LP64S-NEXT:    lu32i.d $t0, 0
+; LA64D-LP64S-NEXT:    movfr2gr.s $a7, $fa0
+; LA64D-LP64S-NEXT:    st.w $t0, $sp, 0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_in_gregs)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 32
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: caller_half_in_gregs:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; LA64D-LP64D-NEXT:    fld.s $ft0, $a0, %pc_lo12(.LCPI3_0)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; LA64D-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI3_1)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_2)
+; LA64D-LP64D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI3_2)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_3)
+; LA64D-LP64D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI3_3)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_4)
+; LA64D-LP64D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI3_4)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_5)
+; LA64D-LP64D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI3_5)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_6)
+; LA64D-LP64D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI3_6)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_7)
+; LA64D-LP64D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI3_7)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_8)
+; LA64D-LP64D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI3_8)
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $ft0
+; LA64D-LP64D-NEXT:    ori $a1, $zero, 10
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_in_gregs)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = call i32 @callee_half_in_gregs(half 1.0, half 2.0, half 3.0, half 4.0, half 5.0, half 6.0, half 7.0, half 8.0, half 9.0, i32 10)
+  ret i32 %1
+}
+
+define i32 @callee_half_on_stack(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, half %i, half %j, half %k, half %l, half %m, half %n, half %o, half %p, half %q) nounwind {
+; LA32S-LABEL: callee_half_on_stack:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -16
+; LA32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32S-NEXT:    ld.hu $a0, $sp, 48
+; LA32S-NEXT:    move $fp, $a7
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    bl __fixsfsi
+; LA32S-NEXT:    add.w $a0, $fp, $a0
+; LA32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 16
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: callee_half_on_stack:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    ld.hu $a0, $sp, 48
+; LA32F-ILP32S-NEXT:    move $fp, $a7
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    add.w $a0, $fp, $a0
+; LA32F-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: callee_half_on_stack:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    ld.hu $a0, $sp, 16
+; LA32F-ILP32D-NEXT:    move $fp, $a7
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    add.w $a0, $fp, $a0
+; LA32F-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: callee_half_on_stack:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    ld.hu $a0, $sp, 48
+; LA32D-ILP32S-NEXT:    move $fp, $a7
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32S-NEXT:    add.w $a0, $fp, $a0
+; LA32D-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: callee_half_on_stack:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    ld.hu $a0, $sp, 16
+; LA32D-ILP32D-NEXT:    move $fp, $a7
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    add.w $a0, $fp, $a0
+; LA32D-ILP32D-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: callee_half_on_stack:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -16
+; LA64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64S-NEXT:    ld.hu $a0, $sp, 16
+; LA64S-NEXT:    move $fp, $a7
+; LA64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64S-NEXT:    add.w $a0, $fp, $a0
+; LA64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 16
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: callee_half_on_stack:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    ld.hu $a0, $sp, 80
+; LA64F-LP64S-NEXT:    move $fp, $a7
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64S-NEXT:    add.w $a0, $fp, $a0
+; LA64F-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: callee_half_on_stack:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    ld.hu $a0, $sp, 16
+; LA64F-LP64D-NEXT:    move $fp, $a7
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64D-NEXT:    add.w $a0, $fp, $a0
+; LA64F-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: callee_half_on_stack:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    ld.hu $a0, $sp, 80
+; LA64D-LP64S-NEXT:    move $fp, $a7
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64S-NEXT:    add.w $a0, $fp, $a0
+; LA64D-LP64S-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: callee_half_on_stack:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    ld.hu $a0, $sp, 16
+; LA64D-LP64D-NEXT:    move $fp, $a7
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64D-NEXT:    add.w $a0, $fp, $a0
+; LA64D-LP64D-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = fptosi half %q to i32
+  %2 = add i32 %h, %1
+  ret i32 %2
+}
+
+define i32 @caller_half_on_stack() nounwind {
+; LA32S-LABEL: caller_half_on_stack:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -48
+; LA32S-NEXT:    st.w $ra, $sp, 44 # 4-byte Folded Spill
+; LA32S-NEXT:    lu12i.w $a0, 4
+; LA32S-NEXT:    ori $a1, $a0, 3200
+; LA32S-NEXT:    st.w $a1, $sp, 32
+; LA32S-NEXT:    ori $a1, $a0, 3136
+; LA32S-NEXT:    st.w $a1, $sp, 28
+; LA32S-NEXT:    ori $a1, $a0, 3072
+; LA32S-NEXT:    st.w $a1, $sp, 24
+; LA32S-NEXT:    ori $a1, $a0, 2944
+; LA32S-NEXT:    st.w $a1, $sp, 20
+; LA32S-NEXT:    ori $a1, $a0, 2816
+; LA32S-NEXT:    st.w $a1, $sp, 16
+; LA32S-NEXT:    ori $a1, $a0, 2688
+; LA32S-NEXT:    st.w $a1, $sp, 12
+; LA32S-NEXT:    ori $a1, $a0, 2560
+; LA32S-NEXT:    st.w $a1, $sp, 8
+; LA32S-NEXT:    ori $a1, $a0, 2432
+; LA32S-NEXT:    st.w $a1, $sp, 4
+; LA32S-NEXT:    ori $t0, $a0, 2304
+; LA32S-NEXT:    ori $a0, $zero, 1
+; LA32S-NEXT:    ori $a1, $zero, 2
+; LA32S-NEXT:    ori $a2, $zero, 3
+; LA32S-NEXT:    ori $a3, $zero, 4
+; LA32S-NEXT:    ori $a4, $zero, 5
+; LA32S-NEXT:    ori $a5, $zero, 6
+; LA32S-NEXT:    ori $a6, $zero, 7
+; LA32S-NEXT:    ori $a7, $zero, 8
+; LA32S-NEXT:    st.w $t0, $sp, 0
+; LA32S-NEXT:    bl callee_half_on_stack
+; LA32S-NEXT:    ld.w $ra, $sp, 44 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 48
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: caller_half_on_stack:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -48
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 44 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    lu12i.w $a0, -12
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 3200
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 32
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 3136
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 28
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 3072
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 24
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 2944
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 20
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 2816
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 16
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 2688
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 12
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 2560
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 8
+; LA32F-ILP32S-NEXT:    ori $a1, $a0, 2432
+; LA32F-ILP32S-NEXT:    st.w $a1, $sp, 4
+; LA32F-ILP32S-NEXT:    ori $t0, $a0, 2304
+; LA32F-ILP32S-NEXT:    ori $a0, $zero, 1
+; LA32F-ILP32S-NEXT:    ori $a1, $zero, 2
+; LA32F-ILP32S-NEXT:    ori $a2, $zero, 3
+; LA32F-ILP32S-NEXT:    ori $a3, $zero, 4
+; LA32F-ILP32S-NEXT:    ori $a4, $zero, 5
+; LA32F-ILP32S-NEXT:    ori $a5, $zero, 6
+; LA32F-ILP32S-NEXT:    ori $a6, $zero, 7
+; LA32F-ILP32S-NEXT:    ori $a7, $zero, 8
+; LA32F-ILP32S-NEXT:    st.w $t0, $sp, 0
+; LA32F-ILP32S-NEXT:    bl callee_half_on_stack
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 44 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 48
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: caller_half_on_stack:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    lu12i.w $a0, -12
+; LA32F-ILP32D-NEXT:    ori $t0, $a0, 3200
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; LA32F-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI5_0)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
+; LA32F-ILP32D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI5_1)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_2)
+; LA32F-ILP32D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI5_2)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_3)
+; LA32F-ILP32D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI5_3)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_4)
+; LA32F-ILP32D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI5_4)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_5)
+; LA32F-ILP32D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI5_5)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_6)
+; LA32F-ILP32D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI5_6)
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_7)
+; LA32F-ILP32D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI5_7)
+; LA32F-ILP32D-NEXT:    ori $a0, $zero, 1
+; LA32F-ILP32D-NEXT:    ori $a1, $zero, 2
+; LA32F-ILP32D-NEXT:    ori $a2, $zero, 3
+; LA32F-ILP32D-NEXT:    ori $a3, $zero, 4
+; LA32F-ILP32D-NEXT:    ori $a4, $zero, 5
+; LA32F-ILP32D-NEXT:    ori $a5, $zero, 6
+; LA32F-ILP32D-NEXT:    ori $a6, $zero, 7
+; LA32F-ILP32D-NEXT:    ori $a7, $zero, 8
+; LA32F-ILP32D-NEXT:    st.w $t0, $sp, 0
+; LA32F-ILP32D-NEXT:    bl callee_half_on_stack
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: caller_half_on_stack:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -48
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 44 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    lu12i.w $a0, -12
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 3200
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 32
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 3136
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 28
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 3072
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 24
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 2944
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 20
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 2816
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 16
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 2688
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 12
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 2560
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 8
+; LA32D-ILP32S-NEXT:    ori $a1, $a0, 2432
+; LA32D-ILP32S-NEXT:    st.w $a1, $sp, 4
+; LA32D-ILP32S-NEXT:    ori $t0, $a0, 2304
+; LA32D-ILP32S-NEXT:    ori $a0, $zero, 1
+; LA32D-ILP32S-NEXT:    ori $a1, $zero, 2
+; LA32D-ILP32S-NEXT:    ori $a2, $zero, 3
+; LA32D-ILP32S-NEXT:    ori $a3, $zero, 4
+; LA32D-ILP32S-NEXT:    ori $a4, $zero, 5
+; LA32D-ILP32S-NEXT:    ori $a5, $zero, 6
+; LA32D-ILP32S-NEXT:    ori $a6, $zero, 7
+; LA32D-ILP32S-NEXT:    ori $a7, $zero, 8
+; LA32D-ILP32S-NEXT:    st.w $t0, $sp, 0
+; LA32D-ILP32S-NEXT:    bl callee_half_on_stack
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 44 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 48
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: caller_half_on_stack:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    lu12i.w $a0, -12
+; LA32D-ILP32D-NEXT:    ori $t0, $a0, 3200
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; LA32D-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI5_0)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
+; LA32D-ILP32D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI5_1)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_2)
+; LA32D-ILP32D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI5_2)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_3)
+; LA32D-ILP32D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI5_3)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_4)
+; LA32D-ILP32D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI5_4)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_5)
+; LA32D-ILP32D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI5_5)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_6)
+; LA32D-ILP32D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI5_6)
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_7)
+; LA32D-ILP32D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI5_7)
+; LA32D-ILP32D-NEXT:    ori $a0, $zero, 1
+; LA32D-ILP32D-NEXT:    ori $a1, $zero, 2
+; LA32D-ILP32D-NEXT:    ori $a2, $zero, 3
+; LA32D-ILP32D-NEXT:    ori $a3, $zero, 4
+; LA32D-ILP32D-NEXT:    ori $a4, $zero, 5
+; LA32D-ILP32D-NEXT:    ori $a5, $zero, 6
+; LA32D-ILP32D-NEXT:    ori $a6, $zero, 7
+; LA32D-ILP32D-NEXT:    ori $a7, $zero, 8
+; LA32D-ILP32D-NEXT:    st.w $t0, $sp, 0
+; LA32D-ILP32D-NEXT:    bl callee_half_on_stack
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: caller_half_on_stack:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -16
+; LA64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    lu12i.w $a0, -12
+; LA64S-NEXT:    ori $t0, $a0, 3200
+; LA64S-NEXT:    lu32i.d $t0, 0
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; LA64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI5_0)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
+; LA64S-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI5_1)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_2)
+; LA64S-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI5_2)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_3)
+; LA64S-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI5_3)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_4)
+; LA64S-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI5_4)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_5)
+; LA64S-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI5_5)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_6)
+; LA64S-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI5_6)
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_7)
+; LA64S-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI5_7)
+; LA64S-NEXT:    ori $a0, $zero, 1
+; LA64S-NEXT:    ori $a1, $zero, 2
+; LA64S-NEXT:    ori $a2, $zero, 3
+; LA64S-NEXT:    ori $a3, $zero, 4
+; LA64S-NEXT:    ori $a4, $zero, 5
+; LA64S-NEXT:    ori $a5, $zero, 6
+; LA64S-NEXT:    ori $a6, $zero, 7
+; LA64S-NEXT:    ori $a7, $zero, 8
+; LA64S-NEXT:    st.w $t0, $sp, 0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 16
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: caller_half_on_stack:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -80
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    lu12i.w $a0, -12
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 3200
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 64
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 3136
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 56
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 3072
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 48
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 2944
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 40
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 2816
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 32
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 2688
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 24
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 2560
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 16
+; LA64F-LP64S-NEXT:    ori $a1, $a0, 2432
+; LA64F-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64F-LP64S-NEXT:    st.w $a1, $sp, 8
+; LA64F-LP64S-NEXT:    ori $t0, $a0, 2304
+; LA64F-LP64S-NEXT:    lu32i.d $t0, 0
+; LA64F-LP64S-NEXT:    ori $a0, $zero, 1
+; LA64F-LP64S-NEXT:    ori $a1, $zero, 2
+; LA64F-LP64S-NEXT:    ori $a2, $zero, 3
+; LA64F-LP64S-NEXT:    ori $a3, $zero, 4
+; LA64F-LP64S-NEXT:    ori $a4, $zero, 5
+; LA64F-LP64S-NEXT:    ori $a5, $zero, 6
+; LA64F-LP64S-NEXT:    ori $a6, $zero, 7
+; LA64F-LP64S-NEXT:    ori $a7, $zero, 8
+; LA64F-LP64S-NEXT:    st.w $t0, $sp, 0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: caller_half_on_stack:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    lu12i.w $a0, -12
+; LA64F-LP64D-NEXT:    ori $t0, $a0, 3200
+; LA64F-LP64D-NEXT:    lu32i.d $t0, 0
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; LA64F-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI5_0)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
+; LA64F-LP64D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI5_1)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_2)
+; LA64F-LP64D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI5_2)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_3)
+; LA64F-LP64D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI5_3)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_4)
+; LA64F-LP64D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI5_4)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_5)
+; LA64F-LP64D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI5_5)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_6)
+; LA64F-LP64D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI5_6)
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_7)
+; LA64F-LP64D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI5_7)
+; LA64F-LP64D-NEXT:    ori $a0, $zero, 1
+; LA64F-LP64D-NEXT:    ori $a1, $zero, 2
+; LA64F-LP64D-NEXT:    ori $a2, $zero, 3
+; LA64F-LP64D-NEXT:    ori $a3, $zero, 4
+; LA64F-LP64D-NEXT:    ori $a4, $zero, 5
+; LA64F-LP64D-NEXT:    ori $a5, $zero, 6
+; LA64F-LP64D-NEXT:    ori $a6, $zero, 7
+; LA64F-LP64D-NEXT:    ori $a7, $zero, 8
+; LA64F-LP64D-NEXT:    st.w $t0, $sp, 0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: caller_half_on_stack:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -80
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    lu12i.w $a0, -12
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 3200
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 64
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 3136
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 56
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 3072
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 48
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 2944
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 40
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 2816
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 32
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 2688
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 24
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 2560
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 16
+; LA64D-LP64S-NEXT:    ori $a1, $a0, 2432
+; LA64D-LP64S-NEXT:    lu32i.d $a1, 0
+; LA64D-LP64S-NEXT:    st.w $a1, $sp, 8
+; LA64D-LP64S-NEXT:    ori $t0, $a0, 2304
+; LA64D-LP64S-NEXT:    lu32i.d $t0, 0
+; LA64D-LP64S-NEXT:    ori $a0, $zero, 1
+; LA64D-LP64S-NEXT:    ori $a1, $zero, 2
+; LA64D-LP64S-NEXT:    ori $a2, $zero, 3
+; LA64D-LP64S-NEXT:    ori $a3, $zero, 4
+; LA64D-LP64S-NEXT:    ori $a4, $zero, 5
+; LA64D-LP64S-NEXT:    ori $a5, $zero, 6
+; LA64D-LP64S-NEXT:    ori $a6, $zero, 7
+; LA64D-LP64S-NEXT:    ori $a7, $zero, 8
+; LA64D-LP64S-NEXT:    st.w $t0, $sp, 0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 80
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: caller_half_on_stack:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    lu12i.w $a0, -12
+; LA64D-LP64D-NEXT:    ori $t0, $a0, 3200
+; LA64D-LP64D-NEXT:    lu32i.d $t0, 0
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; LA64D-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI5_0)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
+; LA64D-LP64D-NEXT:    fld.s $fa1, $a0, %pc_lo12(.LCPI5_1)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_2)
+; LA64D-LP64D-NEXT:    fld.s $fa2, $a0, %pc_lo12(.LCPI5_2)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_3)
+; LA64D-LP64D-NEXT:    fld.s $fa3, $a0, %pc_lo12(.LCPI5_3)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_4)
+; LA64D-LP64D-NEXT:    fld.s $fa4, $a0, %pc_lo12(.LCPI5_4)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_5)
+; LA64D-LP64D-NEXT:    fld.s $fa5, $a0, %pc_lo12(.LCPI5_5)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_6)
+; LA64D-LP64D-NEXT:    fld.s $fa6, $a0, %pc_lo12(.LCPI5_6)
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_7)
+; LA64D-LP64D-NEXT:    fld.s $fa7, $a0, %pc_lo12(.LCPI5_7)
+; LA64D-LP64D-NEXT:    ori $a0, $zero, 1
+; LA64D-LP64D-NEXT:    ori $a1, $zero, 2
+; LA64D-LP64D-NEXT:    ori $a2, $zero, 3
+; LA64D-LP64D-NEXT:    ori $a3, $zero, 4
+; LA64D-LP64D-NEXT:    ori $a4, $zero, 5
+; LA64D-LP64D-NEXT:    ori $a5, $zero, 6
+; LA64D-LP64D-NEXT:    ori $a6, $zero, 7
+; LA64D-LP64D-NEXT:    ori $a7, $zero, 8
+; LA64D-LP64D-NEXT:    st.w $t0, $sp, 0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_on_stack)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = call i32 @callee_half_on_stack(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, half 10.0, half 11.0, half 12.0, half 13.0, half 14.0, half 15.0, half 16.0, half 17.0, half 18.0)
+  ret i32 %1
+}
+
+define half @callee_half_ret() nounwind {
+; LA32S-LABEL: callee_half_ret:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    lu12i.w $a0, 3
+; LA32S-NEXT:    ori $a0, $a0, 3072
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: callee_half_ret:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    lu12i.w $a0, -13
+; LA32F-ILP32S-NEXT:    ori $a0, $a0, 3072
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: callee_half_ret:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; LA32F-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI6_0)
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: callee_half_ret:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    lu12i.w $a0, -13
+; LA32D-ILP32S-NEXT:    ori $a0, $a0, 3072
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: callee_half_ret:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; LA32D-ILP32D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI6_0)
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: callee_half_ret:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; LA64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI6_0)
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: callee_half_ret:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; LA64F-LP64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI6_0)
+; LA64F-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: callee_half_ret:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; LA64F-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI6_0)
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: callee_half_ret:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; LA64D-LP64S-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI6_0)
+; LA64D-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: callee_half_ret:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; LA64D-LP64D-NEXT:    fld.s $fa0, $a0, %pc_lo12(.LCPI6_0)
+; LA64D-LP64D-NEXT:    ret
+  ret half 1.0
+}
+
+define i32 @caller_half_ret() nounwind {
+; LA32S-LABEL: caller_half_ret:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -16
+; LA32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32S-NEXT:    bl callee_half_ret
+; LA32S-NEXT:    lu12i.w $a1, 15
+; LA32S-NEXT:    ori $a1, $a1, 4095
+; LA32S-NEXT:    and $a0, $a0, $a1
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    bl __fixsfsi
+; LA32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 16
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: caller_half_ret:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    bl callee_half_ret
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: caller_half_ret:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    bl callee_half_ret
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: caller_half_ret:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    bl callee_half_ret
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: caller_half_ret:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    bl callee_half_ret
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    ftintrz.w.s $fa0, $fa0
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: caller_half_ret:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -16
+; LA64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    pcaddu18i $ra, %call36(callee_half_ret)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 16
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: caller_half_ret:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_ret)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: caller_half_ret:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_ret)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64F-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: caller_half_ret:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(callee_half_ret)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: caller_half_ret:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -16
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(callee_half_ret)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64D-LP64D-NEXT:    movfr2gr.d $a0, $fa0
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 16
+; LA64D-LP64D-NEXT:    ret
+  %1 = call half @callee_half_ret()
+  %2 = fptosi half %1 to i32
+  ret i32 %2
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll
index e564d7bddea6..27be02c50f1c 100644
--- a/external/llvm-project/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll
@@ -885,9 +885,8 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 define i8 @test_not_cttz_i8(i8 %a) nounwind {
 ; LA32R-LABEL: test_not_cttz_i8:
 ; LA32R:       # %bb.0:
-; LA32R-NEXT:    nor $a1, $a0, $zero
-; LA32R-NEXT:    addi.w $a1, $a1, -1
-; LA32R-NEXT:    and $a0, $a0, $a1
+; LA32R-NEXT:    addi.w $a1, $a0, 1
+; LA32R-NEXT:    andn $a0, $a0, $a1
 ; LA32R-NEXT:    srli.w $a1, $a0, 1
 ; LA32R-NEXT:    andi $a1, $a1, 85
 ; LA32R-NEXT:    sub.w $a0, $a0, $a1
@@ -921,9 +920,8 @@ define i8 @test_not_cttz_i8(i8 %a) nounwind {
 define i16 @test_not_cttz_i16(i16 %a) nounwind {
 ; LA32R-LABEL: test_not_cttz_i16:
 ; LA32R:       # %bb.0:
-; LA32R-NEXT:    nor $a1, $a0, $zero
-; LA32R-NEXT:    addi.w $a1, $a1, -1
-; LA32R-NEXT:    and $a0, $a0, $a1
+; LA32R-NEXT:    addi.w $a1, $a0, 1
+; LA32R-NEXT:    andn $a0, $a0, $a1
 ; LA32R-NEXT:    srli.w $a1, $a0, 1
 ; LA32R-NEXT:    lu12i.w $a2, 5
 ; LA32R-NEXT:    ori $a2, $a2, 1365
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/fp16-promote.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/fp16-promote.ll
index 6a1610c27937..c49e9ba99e0f 100644
--- a/external/llvm-project/llvm/test/CodeGen/LoongArch/fp16-promote.ll
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/fp16-promote.ll
@@ -22,14 +22,26 @@ define void @test_load_store(ptr %p, ptr %q) nounwind {
 define float @test_fpextend_float(ptr %p) nounwind {
 ; LA32-LABEL: test_fpextend_float:
 ; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
 ; LA32-NEXT:    ld.hu $a0, $a0, 0
-; LA32-NEXT:    b __extendhfsf2
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: test_fpextend_float:
 ; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; LA64-NEXT:    ld.hu $a0, $a0, 0
-; LA64-NEXT:    pcaddu18i $t8, %call36(__extendhfsf2)
-; LA64-NEXT:    jr $t8
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
   %a = load half, ptr %p
   %r = fpext half %a to float
   ret float %r
@@ -41,6 +53,7 @@ define double @test_fpextend_double(ptr %p) nounwind {
 ; LA32-NEXT:    addi.w $sp, $sp, -16
 ; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
 ; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
 ; LA32-NEXT:    bl __extendhfsf2
 ; LA32-NEXT:    fcvt.d.s $fa0, $fa0
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
@@ -52,6 +65,7 @@ define double @test_fpextend_double(ptr %p) nounwind {
 ; LA64-NEXT:    addi.d $sp, $sp, -16
 ; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; LA64-NEXT:    ld.hu $a0, $a0, 0
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    fcvt.d.s $fa0, $fa0
@@ -71,6 +85,7 @@ define void @test_fptrunc_float(float %f, ptr %p) nounwind {
 ; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
 ; LA32-NEXT:    move $fp, $a0
 ; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
 ; LA32-NEXT:    st.h $a0, $fp, 0
 ; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
@@ -85,6 +100,7 @@ define void @test_fptrunc_float(float %f, ptr %p) nounwind {
 ; LA64-NEXT:    move $fp, $a0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
 ; LA64-NEXT:    st.h $a0, $fp, 0
 ; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
@@ -103,6 +119,7 @@ define void @test_fptrunc_double(double %d, ptr %p) nounwind {
 ; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
 ; LA32-NEXT:    move $fp, $a0
 ; LA32-NEXT:    bl __truncdfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
 ; LA32-NEXT:    st.h $a0, $fp, 0
 ; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
@@ -117,6 +134,7 @@ define void @test_fptrunc_double(double %d, ptr %p) nounwind {
 ; LA64-NEXT:    move $fp, $a0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncdfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
 ; LA64-NEXT:    st.h $a0, $fp, 0
 ; LA64-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
@@ -130,43 +148,51 @@ define void @test_fptrunc_double(double %d, ptr %p) nounwind {
 define half @test_fadd_reg(half %a, half %b) nounwind {
 ; LA32-LABEL: test_fadd_reg:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    addi.w $sp, $sp, -16
-; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
-; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
-; LA32-NEXT:    fst.d $fs0, $sp, 0 # 8-byte Folded Spill
-; LA32-NEXT:    move $fp, $a0
-; LA32-NEXT:    move $a0, $a1
-; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
 ; LA32-NEXT:    fmov.s $fs0, $fa0
-; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    fmov.s $fa0, $fa1
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
 ; LA32-NEXT:    bl __extendhfsf2
-; LA32-NEXT:    fadd.s $fa0, $fa0, $fs0
+; LA32-NEXT:    fadd.s $fa0, $fa0, $fs1
 ; LA32-NEXT:    bl __truncsfhf2
-; LA32-NEXT:    fld.d $fs0, $sp, 0 # 8-byte Folded Reload
-; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
-; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
-; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    lu12i.w $a1, -16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: test_fadd_reg:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.d $sp, $sp, -32
 ; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-NEXT:    fst.d $fs0, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    move $fp, $a0
-; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fa1
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fmov.s $fs0, $fa0
-; LA64-NEXT:    move $a0, $fp
+; LA64-NEXT:    fmov.s $fs1, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fadd.s $fa0, $fa0, $fs0
+; LA64-NEXT:    fadd.s $fa0, $fa0, $fs1
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fld.d $fs0, $sp, 8 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
 ; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 32
 ; LA64-NEXT:    ret
@@ -180,20 +206,23 @@ define void @test_fadd_mem(ptr %p, ptr %q) nounwind {
 ; LA32-NEXT:    addi.w $sp, $sp, -32
 ; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
 ; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
-; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
-; LA32-NEXT:    fst.d $fs0, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
 ; LA32-NEXT:    move $fp, $a0
-; LA32-NEXT:    ld.hu $s0, $a0, 0
-; LA32-NEXT:    ld.hu $a0, $a1, 0
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    ld.hu $a1, $a1, 0
+; LA32-NEXT:    movgr2fr.w $fs0, $a0
+; LA32-NEXT:    movgr2fr.w $fa0, $a1
 ; LA32-NEXT:    bl __extendhfsf2
-; LA32-NEXT:    fmov.s $fs0, $fa0
-; LA32-NEXT:    move $a0, $s0
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
 ; LA32-NEXT:    bl __extendhfsf2
-; LA32-NEXT:    fadd.s $fa0, $fa0, $fs0
+; LA32-NEXT:    fadd.s $fa0, $fa0, $fs1
 ; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
 ; LA32-NEXT:    st.h $a0, $fp, 0
-; LA32-NEXT:    fld.d $fs0, $sp, 8 # 8-byte Folded Reload
-; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
 ; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
 ; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 32
@@ -209,15 +238,17 @@ define void @test_fadd_mem(ptr %p, ptr %q) nounwind {
 ; LA64-NEXT:    move $fp, $a0
 ; LA64-NEXT:    ld.hu $s0, $a0, 0
 ; LA64-NEXT:    ld.hu $a0, $a1, 0
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    fmov.s $fs0, $fa0
-; LA64-NEXT:    move $a0, $s0
+; LA64-NEXT:    movgr2fr.w $fa0, $s0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    fadd.s $fa0, $fa0, $fs0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
 ; LA64-NEXT:    st.h $a0, $fp, 0
 ; LA64-NEXT:    fld.d $fs0, $sp, 0 # 8-byte Folded Reload
 ; LA64-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
@@ -235,43 +266,51 @@ define void @test_fadd_mem(ptr %p, ptr %q) nounwind {
 define half @test_fmul_reg(half %a, half %b) nounwind {
 ; LA32-LABEL: test_fmul_reg:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    addi.w $sp, $sp, -16
-; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
-; LA32-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
-; LA32-NEXT:    fst.d $fs0, $sp, 0 # 8-byte Folded Spill
-; LA32-NEXT:    move $fp, $a0
-; LA32-NEXT:    move $a0, $a1
-; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
 ; LA32-NEXT:    fmov.s $fs0, $fa0
-; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    fmov.s $fa0, $fa1
 ; LA32-NEXT:    bl __extendhfsf2
-; LA32-NEXT:    fmul.s $fa0, $fa0, $fs0
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    fmul.s $fa0, $fa0, $fs1
 ; LA32-NEXT:    bl __truncsfhf2
-; LA32-NEXT:    fld.d $fs0, $sp, 0 # 8-byte Folded Reload
-; LA32-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
-; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
-; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    lu12i.w $a1, -16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: test_fmul_reg:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.d $sp, $sp, -32
 ; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-NEXT:    fst.d $fs0, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    move $fp, $a0
-; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fa1
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fmov.s $fs0, $fa0
-; LA64-NEXT:    move $a0, $fp
+; LA64-NEXT:    fmov.s $fs1, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fmul.s $fa0, $fa0, $fs0
+; LA64-NEXT:    fmul.s $fa0, $fa0, $fs1
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    fld.d $fs0, $sp, 8 # 8-byte Folded Reload
-; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
 ; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 32
 ; LA64-NEXT:    ret
@@ -285,20 +324,23 @@ define void @test_fmul_mem(ptr %p, ptr %q) nounwind {
 ; LA32-NEXT:    addi.w $sp, $sp, -32
 ; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
 ; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
-; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
-; LA32-NEXT:    fst.d $fs0, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
 ; LA32-NEXT:    move $fp, $a0
-; LA32-NEXT:    ld.hu $s0, $a0, 0
-; LA32-NEXT:    ld.hu $a0, $a1, 0
+; LA32-NEXT:    ld.hu $a0, $a0, 0
+; LA32-NEXT:    ld.hu $a1, $a1, 0
+; LA32-NEXT:    movgr2fr.w $fs0, $a0
+; LA32-NEXT:    movgr2fr.w $fa0, $a1
 ; LA32-NEXT:    bl __extendhfsf2
-; LA32-NEXT:    fmov.s $fs0, $fa0
-; LA32-NEXT:    move $a0, $s0
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
 ; LA32-NEXT:    bl __extendhfsf2
-; LA32-NEXT:    fmul.s $fa0, $fa0, $fs0
+; LA32-NEXT:    fmul.s $fa0, $fa0, $fs1
 ; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
 ; LA32-NEXT:    st.h $a0, $fp, 0
-; LA32-NEXT:    fld.d $fs0, $sp, 8 # 8-byte Folded Reload
-; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
 ; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
 ; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 32
@@ -314,15 +356,17 @@ define void @test_fmul_mem(ptr %p, ptr %q) nounwind {
 ; LA64-NEXT:    move $fp, $a0
 ; LA64-NEXT:    ld.hu $s0, $a0, 0
 ; LA64-NEXT:    ld.hu $a0, $a1, 0
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    fmov.s $fs0, $fa0
-; LA64-NEXT:    move $a0, $s0
+; LA64-NEXT:    movgr2fr.w $fa0, $s0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
 ; LA64-NEXT:    fmul.s $fa0, $fa0, $fs0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
 ; LA64-NEXT:    st.h $a0, $fp, 0
 ; LA64-NEXT:    fld.d $fs0, $sp, 0 # 8-byte Folded Reload
 ; LA64-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
@@ -347,6 +391,10 @@ define half @freeze_half_undef() nounwind {
 ; LA32-NEXT:    bl __extendhfsf2
 ; LA32-NEXT:    fadd.s $fa0, $fa0, $fa0
 ; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    lu12i.w $a1, -16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 16
 ; LA32-NEXT:    ret
@@ -363,6 +411,10 @@ define half @freeze_half_undef() nounwind {
 ; LA64-NEXT:    fadd.s $fa0, $fa0, $fa0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 16
 ; LA64-NEXT:    ret
@@ -379,6 +431,10 @@ define half @freeze_half_poison(half %maybe.poison) nounwind {
 ; LA32-NEXT:    bl __extendhfsf2
 ; LA32-NEXT:    fadd.s $fa0, $fa0, $fa0
 ; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    lu12i.w $a1, -16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 16
 ; LA32-NEXT:    ret
@@ -392,6 +448,10 @@ define half @freeze_half_poison(half %maybe.poison) nounwind {
 ; LA64-NEXT:    fadd.s $fa0, $fa0, $fa0
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 16
 ; LA64-NEXT:    ret
@@ -418,8 +478,8 @@ define signext i32 @test_half_to_s32(half %a) nounwind {
 ; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    ftintrz.w.s $fa0, $fa0
-; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 16
 ; LA64-NEXT:    ret
@@ -446,8 +506,8 @@ define zeroext i32 @test_half_to_s32_u32(half %a) nounwind {
 ; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
 ; LA64-NEXT:    jirl $ra, $ra, 0
-; LA64-NEXT:    ftintrz.w.s $fa0, $fa0
-; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    ftintrz.l.s $fa0, $fa0
+; LA64-NEXT:    movfr2gr.d $a0, $fa0
 ; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 16
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
index 096c2242661c..305b55c1ebb4 100644
--- a/external/llvm-project/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
@@ -1,10 +1,35 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | \
+; RUN:   FileCheck %s --check-prefix=LA32
 ; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | \
 ; RUN:   FileCheck %s --check-prefix=LA64
 
-;; TODO: Testing for LA32 architecture will be added later
-
 define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i8_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a6, $a1, .LBB0_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB0_3: # in Loop: Header=BB0_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB0_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -33,6 +58,32 @@ define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i16_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a6, $a1, .LBB1_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB1_3: # in Loop: Header=BB1_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB1_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -62,6 +113,21 @@ define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umax_i32_acquire(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i32_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a2, $a1, .LBB2_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB2_3: # in Loop: Header=BB2_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB2_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
@@ -72,6 +138,65 @@ define i32 @atomicrmw_umax_i32_acquire(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umax_i64_acquire(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i64_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB3_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB3_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB3_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 2
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB3_7
+; LA32-NEXT:  .LBB3_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB3_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB3_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB3_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB3_4: # in Loop: Header=BB3_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB3_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB3_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB3_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB3_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB3_1
+; LA32-NEXT:  .LBB3_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i64_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
@@ -82,6 +207,31 @@ define i64 @atomicrmw_umax_i64_acquire(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i8_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a1, $a6, .LBB4_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB4_3: # in Loop: Header=BB4_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB4_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -110,6 +260,32 @@ define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i16_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a1, $a6, .LBB5_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB5_3: # in Loop: Header=BB5_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB5_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -139,6 +315,21 @@ define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umin_i32_acquire(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i32_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a1, $a2, .LBB6_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB6_3: # in Loop: Header=BB6_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB6_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
@@ -149,6 +340,66 @@ define i32 @atomicrmw_umin_i32_acquire(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umin_i64_acquire(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i64_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB7_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB7_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB7_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 2
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB7_7
+; LA32-NEXT:  .LBB7_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB7_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB7_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB7_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB7_4: # in Loop: Header=BB7_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB7_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB7_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB7_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB7_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB7_1
+; LA32-NEXT:  .LBB7_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i64_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
@@ -159,6 +410,36 @@ define i64 @atomicrmw_umin_i64_acquire(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i8_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB8_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB8_3: # in Loop: Header=BB8_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB8_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -191,6 +472,38 @@ define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i16_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB9_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB9_3: # in Loop: Header=BB9_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB9_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -225,6 +538,21 @@ define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_max_i32_acquire(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i32_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a2, $a1, .LBB10_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB10_3: # in Loop: Header=BB10_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB10_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
@@ -235,6 +563,65 @@ define i32 @atomicrmw_max_i32_acquire(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_max_i64_acquire(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i64_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB11_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB11_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB11_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 2
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB11_7
+; LA32-NEXT:  .LBB11_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB11_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB11_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB11_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB11_4: # in Loop: Header=BB11_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB11_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB11_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB11_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB11_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB11_1
+; LA32-NEXT:  .LBB11_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i64_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
@@ -245,6 +632,36 @@ define i64 @atomicrmw_max_i64_acquire(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i8_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB12_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB12_3: # in Loop: Header=BB12_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB12_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -277,6 +694,38 @@ define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i16_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB13_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB13_3: # in Loop: Header=BB13_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB13_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -311,6 +760,21 @@ define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_min_i32_acquire(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i32_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a1, $a2, .LBB14_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB14_3: # in Loop: Header=BB14_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB14_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
@@ -321,6 +785,66 @@ define i32 @atomicrmw_min_i32_acquire(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_min_i64_acquire(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i64_acquire:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB15_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB15_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB15_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 2
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB15_7
+; LA32-NEXT:  .LBB15_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB15_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB15_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB15_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB15_4: # in Loop: Header=BB15_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB15_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB15_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB15_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB15_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB15_1
+; LA32-NEXT:  .LBB15_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i64_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
@@ -331,6 +855,31 @@ define i64 @atomicrmw_min_i64_acquire(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a6, $a1, .LBB16_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB16_3: # in Loop: Header=BB16_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB16_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i8_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -359,6 +908,32 @@ define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a6, $a1, .LBB17_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB17_3: # in Loop: Header=BB17_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB17_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i16_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -388,6 +963,21 @@ define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umax_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a2, $a1, .LBB18_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB18_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB18_3: # in Loop: Header=BB18_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB18_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i32_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
@@ -398,6 +988,65 @@ define i32 @atomicrmw_umax_i32_release(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umax_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB19_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB19_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 3
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB19_7
+; LA32-NEXT:  .LBB19_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB19_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB19_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB19_4: # in Loop: Header=BB19_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB19_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB19_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB19_1
+; LA32-NEXT:  .LBB19_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i64_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
@@ -408,6 +1057,31 @@ define i64 @atomicrmw_umax_i64_release(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a1, $a6, .LBB20_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB20_3: # in Loop: Header=BB20_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB20_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i8_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -436,6 +1110,32 @@ define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a1, $a6, .LBB21_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB21_3: # in Loop: Header=BB21_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB21_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i16_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -465,6 +1165,21 @@ define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umin_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a1, $a2, .LBB22_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB22_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB22_3: # in Loop: Header=BB22_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB22_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i32_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
@@ -475,6 +1190,66 @@ define i32 @atomicrmw_umin_i32_release(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umin_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB23_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB23_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 3
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB23_7
+; LA32-NEXT:  .LBB23_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB23_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB23_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB23_4: # in Loop: Header=BB23_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB23_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB23_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB23_1
+; LA32-NEXT:  .LBB23_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i64_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
@@ -485,6 +1260,36 @@ define i64 @atomicrmw_umin_i64_release(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_max_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB24_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB24_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB24_3: # in Loop: Header=BB24_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB24_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i8_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -517,6 +1322,38 @@ define i8 @atomicrmw_max_i8_release(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_max_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB25_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB25_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB25_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB25_3: # in Loop: Header=BB25_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB25_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i16_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -551,6 +1388,21 @@ define i16 @atomicrmw_max_i16_release(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_max_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB26_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a2, $a1, .LBB26_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB26_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB26_3: # in Loop: Header=BB26_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB26_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i32_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
@@ -561,6 +1413,65 @@ define i32 @atomicrmw_max_i32_release(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_max_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB27_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB27_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 3
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB27_7
+; LA32-NEXT:  .LBB27_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB27_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB27_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB27_4: # in Loop: Header=BB27_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB27_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB27_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB27_1
+; LA32-NEXT:  .LBB27_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i64_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
@@ -571,6 +1482,36 @@ define i64 @atomicrmw_max_i64_release(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_min_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB28_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB28_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB28_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB28_3: # in Loop: Header=BB28_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB28_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i8_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -603,6 +1544,38 @@ define i8 @atomicrmw_min_i8_release(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_min_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB29_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB29_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB29_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB29_3: # in Loop: Header=BB29_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB29_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i16_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -637,6 +1610,21 @@ define i16 @atomicrmw_min_i16_release(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_min_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB30_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a1, $a2, .LBB30_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB30_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB30_3: # in Loop: Header=BB30_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB30_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i32_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
@@ -647,6 +1635,66 @@ define i32 @atomicrmw_min_i32_release(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_min_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB31_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB31_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 3
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB31_7
+; LA32-NEXT:  .LBB31_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB31_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB31_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB31_4: # in Loop: Header=BB31_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB31_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB31_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB31_1
+; LA32-NEXT:  .LBB31_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i64_release:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
@@ -657,6 +1705,31 @@ define i64 @atomicrmw_min_i64_release(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB32_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a6, $a1, .LBB32_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB32_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB32_3: # in Loop: Header=BB32_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB32_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i8_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -685,6 +1758,32 @@ define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB33_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a6, $a1, .LBB33_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB33_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB33_3: # in Loop: Header=BB33_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB33_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i16_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -714,6 +1813,21 @@ define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umax_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB34_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a2, $a1, .LBB34_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB34_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB34_3: # in Loop: Header=BB34_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB34_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i32_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
@@ -724,6 +1838,65 @@ define i32 @atomicrmw_umax_i32_acq_rel(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umax_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB35_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB35_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 4
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB35_7
+; LA32-NEXT:  .LBB35_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB35_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB35_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB35_4: # in Loop: Header=BB35_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB35_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB35_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB35_1
+; LA32-NEXT:  .LBB35_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i64_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
@@ -734,6 +1907,31 @@ define i64 @atomicrmw_umax_i64_acq_rel(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a1, $a6, .LBB36_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB36_3: # in Loop: Header=BB36_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB36_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i8_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -762,6 +1960,32 @@ define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a1, $a6, .LBB37_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB37_3: # in Loop: Header=BB37_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB37_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i16_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -791,6 +2015,21 @@ define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umin_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a1, $a2, .LBB38_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB38_3: # in Loop: Header=BB38_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB38_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i32_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
@@ -801,6 +2040,66 @@ define i32 @atomicrmw_umin_i32_acq_rel(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umin_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB39_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB39_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 4
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB39_7
+; LA32-NEXT:  .LBB39_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB39_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB39_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB39_4: # in Loop: Header=BB39_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB39_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB39_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB39_1
+; LA32-NEXT:  .LBB39_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i64_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
@@ -811,6 +2110,36 @@ define i64 @atomicrmw_umin_i64_acq_rel(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_max_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB40_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB40_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i8_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -843,6 +2172,38 @@ define i8 @atomicrmw_max_i8_acq_rel(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_max_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB41_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB41_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i16_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -877,6 +2238,21 @@ define i16 @atomicrmw_max_i16_acq_rel(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_max_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a2, $a1, .LBB42_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB42_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB42_3: # in Loop: Header=BB42_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB42_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i32_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
@@ -887,6 +2263,65 @@ define i32 @atomicrmw_max_i32_acq_rel(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_max_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB43_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB43_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 4
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB43_7
+; LA32-NEXT:  .LBB43_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB43_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB43_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB43_4: # in Loop: Header=BB43_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB43_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB43_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB43_1
+; LA32-NEXT:  .LBB43_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i64_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
@@ -897,6 +2332,36 @@ define i64 @atomicrmw_max_i64_acq_rel(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_min_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB44_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB44_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i8_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -929,6 +2394,38 @@ define i8 @atomicrmw_min_i8_acq_rel(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_min_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB45_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB45_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i16_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -963,6 +2460,21 @@ define i16 @atomicrmw_min_i16_acq_rel(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_min_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a1, $a2, .LBB46_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB46_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB46_3: # in Loop: Header=BB46_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB46_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i32_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
@@ -973,6 +2485,66 @@ define i32 @atomicrmw_min_i32_acq_rel(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_min_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB47_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB47_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 4
+; LA32-NEXT:    ori $a5, $zero, 2
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB47_7
+; LA32-NEXT:  .LBB47_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB47_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB47_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB47_4: # in Loop: Header=BB47_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB47_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB47_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB47_1
+; LA32-NEXT:  .LBB47_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i64_acq_rel:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
@@ -983,6 +2555,31 @@ define i64 @atomicrmw_min_i64_acq_rel(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a6, $a1, .LBB48_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB48_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB48_3: # in Loop: Header=BB48_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB48_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i8_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1011,6 +2608,32 @@ define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a6, $a1, .LBB49_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB49_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB49_3: # in Loop: Header=BB49_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB49_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i16_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1040,6 +2663,21 @@ define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umax_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a2, $a1, .LBB50_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB50_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB50_3: # in Loop: Header=BB50_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB50_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i32_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
@@ -1050,6 +2688,65 @@ define i32 @atomicrmw_umax_i32_seq_cst(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umax_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB51_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB51_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 5
+; LA32-NEXT:    ori $a5, $zero, 5
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB51_7
+; LA32-NEXT:  .LBB51_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB51_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB51_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB51_4: # in Loop: Header=BB51_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB51_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB51_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB51_1
+; LA32-NEXT:  .LBB51_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i64_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
@@ -1060,6 +2757,31 @@ define i64 @atomicrmw_umax_i64_seq_cst(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a1, $a6, .LBB52_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB52_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB52_3: # in Loop: Header=BB52_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB52_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i8_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1088,6 +2810,32 @@ define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a1, $a6, .LBB53_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB53_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB53_3: # in Loop: Header=BB53_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB53_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i16_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1117,6 +2865,21 @@ define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umin_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a1, $a2, .LBB54_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB54_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB54_3: # in Loop: Header=BB54_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB54_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i32_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
@@ -1127,6 +2890,66 @@ define i32 @atomicrmw_umin_i32_seq_cst(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umin_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB55_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB55_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB55_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 5
+; LA32-NEXT:    ori $a5, $zero, 5
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB55_7
+; LA32-NEXT:  .LBB55_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB55_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB55_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB55_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB55_4: # in Loop: Header=BB55_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB55_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB55_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB55_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB55_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB55_1
+; LA32-NEXT:  .LBB55_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i64_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
@@ -1137,6 +2960,36 @@ define i64 @atomicrmw_umin_i64_seq_cst(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_max_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB56_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB56_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB56_3: # in Loop: Header=BB56_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB56_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i8_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1169,6 +3022,38 @@ define i8 @atomicrmw_max_i8_seq_cst(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_max_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB57_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB57_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB57_3: # in Loop: Header=BB57_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB57_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i16_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1203,6 +3088,21 @@ define i16 @atomicrmw_max_i16_seq_cst(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_max_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a2, $a1, .LBB58_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB58_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB58_3: # in Loop: Header=BB58_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB58_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i32_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
@@ -1213,6 +3113,65 @@ define i32 @atomicrmw_max_i32_seq_cst(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_max_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB59_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB59_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB59_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 5
+; LA32-NEXT:    ori $a5, $zero, 5
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB59_7
+; LA32-NEXT:  .LBB59_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB59_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB59_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB59_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB59_4: # in Loop: Header=BB59_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB59_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB59_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB59_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB59_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB59_1
+; LA32-NEXT:  .LBB59_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i64_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
@@ -1223,6 +3182,36 @@ define i64 @atomicrmw_max_i64_seq_cst(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_min_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB60_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB60_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB60_3: # in Loop: Header=BB60_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB60_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i8_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1255,6 +3244,38 @@ define i8 @atomicrmw_min_i8_seq_cst(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_min_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB61_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB61_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB61_3: # in Loop: Header=BB61_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB61_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i16_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1289,6 +3310,21 @@ define i16 @atomicrmw_min_i16_seq_cst(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_min_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a1, $a2, .LBB62_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB62_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB62_3: # in Loop: Header=BB62_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB62_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i32_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
@@ -1299,6 +3335,66 @@ define i32 @atomicrmw_min_i32_seq_cst(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_min_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB63_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB63_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB63_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    ori $a4, $zero, 5
+; LA32-NEXT:    ori $a5, $zero, 5
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB63_7
+; LA32-NEXT:  .LBB63_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB63_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB63_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB63_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB63_4: # in Loop: Header=BB63_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB63_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB63_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB63_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB63_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB63_1
+; LA32-NEXT:  .LBB63_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i64_seq_cst:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
@@ -1309,6 +3405,31 @@ define i64 @atomicrmw_min_i64_seq_cst(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i8_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a6, $a1, .LBB64_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB64_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB64_3: # in Loop: Header=BB64_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB64_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i8_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1337,6 +3458,32 @@ define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i16_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a6, $a1, .LBB65_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB65_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB65_3: # in Loop: Header=BB65_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB65_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i16_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1366,6 +3513,21 @@ define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umax_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i32_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a2, $a1, .LBB66_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB66_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB66_3: # in Loop: Header=BB66_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB66_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i32_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax.wu $a2, $a1, $a0
@@ -1376,6 +3538,65 @@ define i32 @atomicrmw_umax_i32_monotonic(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umax_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umax_i64_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB67_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB67_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB67_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a4, $zero
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB67_7
+; LA32-NEXT:  .LBB67_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB67_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB67_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB67_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB67_4: # in Loop: Header=BB67_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB67_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB67_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB67_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB67_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB67_1
+; LA32-NEXT:  .LBB67_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umax_i64_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax.du $a2, $a1, $a0
@@ -1386,6 +3607,31 @@ define i64 @atomicrmw_umax_i64_monotonic(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i8_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a0
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a2, 0
+; LA32-NEXT:    and $a6, $a4, $a3
+; LA32-NEXT:    move $a5, $a4
+; LA32-NEXT:    bgeu $a1, $a6, .LBB68_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB68_1 Depth=1
+; LA32-NEXT:    xor $a5, $a4, $a1
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:  .LBB68_3: # in Loop: Header=BB68_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB68_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a4, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i8_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1414,6 +3660,32 @@ define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i16_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    lu12i.w $a3, 15
+; LA32-NEXT:    ori $a3, $a3, 4095
+; LA32-NEXT:    sll.w $a4, $a3, $a0
+; LA32-NEXT:    and $a1, $a1, $a3
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a2, 0
+; LA32-NEXT:    and $a6, $a3, $a4
+; LA32-NEXT:    move $a5, $a3
+; LA32-NEXT:    bgeu $a1, $a6, .LBB69_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB69_1 Depth=1
+; LA32-NEXT:    xor $a5, $a3, $a1
+; LA32-NEXT:    and $a5, $a5, $a4
+; LA32-NEXT:    xor $a5, $a3, $a5
+; LA32-NEXT:  .LBB69_3: # in Loop: Header=BB69_1 Depth=1
+; LA32-NEXT:    sc.w $a5, $a2, 0
+; LA32-NEXT:    beq $a5, $zero, .LBB69_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a3, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i16_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1443,6 +3715,21 @@ define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_umin_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i32_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB70_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bgeu $a1, $a2, .LBB70_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB70_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB70_3: # in Loop: Header=BB70_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB70_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i32_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin.wu $a2, $a1, $a0
@@ -1453,6 +3740,66 @@ define i32 @atomicrmw_umin_i32_monotonic(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_umin_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_umin_i64_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB71_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB71_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB71_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a4, $zero
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB71_7
+; LA32-NEXT:  .LBB71_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB71_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB71_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s0, $a5
+; LA32-NEXT:    b .LBB71_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB71_4: # in Loop: Header=BB71_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB71_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB71_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB71_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB71_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB71_1
+; LA32-NEXT:  .LBB71_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_umin_i64_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin.du $a2, $a1, $a0
@@ -1463,6 +3810,36 @@ define i64 @atomicrmw_umin_i64_monotonic(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_max_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i8_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB72_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB72_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB72_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB72_3: # in Loop: Header=BB72_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB72_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i8_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1495,6 +3872,38 @@ define i8 @atomicrmw_max_i8_monotonic(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_max_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i16_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB73_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a7, $a1, .LBB73_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB73_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB73_3: # in Loop: Header=BB73_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB73_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i16_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1529,6 +3938,21 @@ define i16 @atomicrmw_max_i16_monotonic(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_max_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i32_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB74_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a2, $a1, .LBB74_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB74_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB74_3: # in Loop: Header=BB74_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB74_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i32_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax.w $a2, $a1, $a0
@@ -1539,6 +3963,65 @@ define i32 @atomicrmw_max_i32_monotonic(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_max_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_max_i64_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB75_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB75_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB75_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a4, $zero
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB75_7
+; LA32-NEXT:  .LBB75_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB75_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB75_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB75_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB75_4: # in Loop: Header=BB75_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB75_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB75_2 Depth=1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB75_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB75_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB75_1
+; LA32-NEXT:  .LBB75_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_max_i64_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammax.d $a2, $a1, $a0
@@ -1549,6 +4032,36 @@ define i64 @atomicrmw_max_i64_monotonic(ptr %a, i64 %b) nounwind {
 }
 
 define i8 @atomicrmw_min_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i8_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    ori $a4, $zero, 255
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 24
+; LA32-NEXT:    srai.w $a1, $a1, 24
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    xori $a3, $a3, 24
+; LA32-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB76_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB76_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB76_3: # in Loop: Header=BB76_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB76_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i8_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1581,6 +4094,38 @@ define i8 @atomicrmw_min_i8_monotonic(ptr %a, i8 %b) nounwind {
 }
 
 define i16 @atomicrmw_min_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i16_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $a2, $zero, -4
+; LA32-NEXT:    and $a2, $a0, $a2
+; LA32-NEXT:    slli.w $a0, $a0, 3
+; LA32-NEXT:    andi $a3, $a0, 24
+; LA32-NEXT:    lu12i.w $a4, 15
+; LA32-NEXT:    ori $a4, $a4, 4095
+; LA32-NEXT:    sll.w $a4, $a4, $a0
+; LA32-NEXT:    slli.w $a1, $a1, 16
+; LA32-NEXT:    srai.w $a1, $a1, 16
+; LA32-NEXT:    sll.w $a1, $a1, $a0
+; LA32-NEXT:    ori $a5, $zero, 16
+; LA32-NEXT:    sub.w $a3, $a5, $a3
+; LA32-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a5, $a2, 0
+; LA32-NEXT:    and $a7, $a5, $a4
+; LA32-NEXT:    move $a6, $a5
+; LA32-NEXT:    sll.w $a7, $a7, $a3
+; LA32-NEXT:    sra.w $a7, $a7, $a3
+; LA32-NEXT:    bge $a1, $a7, .LBB77_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB77_1 Depth=1
+; LA32-NEXT:    xor $a6, $a5, $a1
+; LA32-NEXT:    and $a6, $a6, $a4
+; LA32-NEXT:    xor $a6, $a5, $a6
+; LA32-NEXT:  .LBB77_3: # in Loop: Header=BB77_1 Depth=1
+; LA32-NEXT:    sc.w $a6, $a2, 0
+; LA32-NEXT:    beq $a6, $zero, .LBB77_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    srl.w $a0, $a5, $a0
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i16_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
@@ -1615,6 +4160,21 @@ define i16 @atomicrmw_min_i16_monotonic(ptr %a, i16 %b) nounwind {
 }
 
 define i32 @atomicrmw_min_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i32_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB78_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a2
+; LA32-NEXT:    bge $a1, $a2, .LBB78_3
+; LA32-NEXT:  # %bb.2: # in Loop: Header=BB78_1 Depth=1
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:  .LBB78_3: # in Loop: Header=BB78_1 Depth=1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beq $a3, $zero, .LBB78_1
+; LA32-NEXT:  # %bb.4:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i32_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin.w $a2, $a1, $a0
@@ -1625,6 +4185,66 @@ define i32 @atomicrmw_min_i32_monotonic(ptr %a, i32 %b) nounwind {
 }
 
 define i64 @atomicrmw_min_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_min_i64_monotonic:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    ld.w $a5, $a0, 4
+; LA32-NEXT:    ld.w $a4, $a0, 0
+; LA32-NEXT:    move $s0, $a2
+; LA32-NEXT:    move $s1, $a1
+; LA32-NEXT:    addi.w $s2, $sp, 0
+; LA32-NEXT:    b .LBB79_2
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB79_1: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB79_2 Depth=1
+; LA32-NEXT:    st.w $a4, $sp, 0
+; LA32-NEXT:    st.w $a5, $sp, 4
+; LA32-NEXT:    move $a0, $fp
+; LA32-NEXT:    move $a1, $s2
+; LA32-NEXT:    move $a4, $zero
+; LA32-NEXT:    move $a5, $zero
+; LA32-NEXT:    bl __atomic_compare_exchange_8
+; LA32-NEXT:    ld.w $a5, $sp, 4
+; LA32-NEXT:    ld.w $a4, $sp, 0
+; LA32-NEXT:    bne $a0, $zero, .LBB79_7
+; LA32-NEXT:  .LBB79_2: # %atomicrmw.start
+; LA32-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    beq $a5, $s0, .LBB79_4
+; LA32-NEXT:  # %bb.3: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB79_2 Depth=1
+; LA32-NEXT:    slt $a0, $s0, $a5
+; LA32-NEXT:    b .LBB79_5
+; LA32-NEXT:    .p2align 4, , 16
+; LA32-NEXT:  .LBB79_4: # in Loop: Header=BB79_2 Depth=1
+; LA32-NEXT:    sltu $a0, $s1, $a4
+; LA32-NEXT:  .LBB79_5: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB79_2 Depth=1
+; LA32-NEXT:    xori $a0, $a0, 1
+; LA32-NEXT:    move $a2, $a4
+; LA32-NEXT:    move $a3, $a5
+; LA32-NEXT:    bne $a0, $zero, .LBB79_1
+; LA32-NEXT:  # %bb.6: # %atomicrmw.start
+; LA32-NEXT:    # in Loop: Header=BB79_2 Depth=1
+; LA32-NEXT:    move $a2, $s1
+; LA32-NEXT:    move $a3, $s0
+; LA32-NEXT:    b .LBB79_1
+; LA32-NEXT:  .LBB79_7: # %atomicrmw.end
+; LA32-NEXT:    move $a0, $a4
+; LA32-NEXT:    move $a1, $a5
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
 ; LA64-LABEL: atomicrmw_min_i64_monotonic:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ammin.d $a2, $a1, $a0
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/isel-optnone.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/isel-optnone.ll
new file mode 100644
index 000000000000..4d2528a3148a
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/isel-optnone.ll
@@ -0,0 +1,10 @@
+; REQUIRES: asserts
+; RUN: llc %s -O0 -mtriple=loongarch64 -o /dev/null -debug-only=isel 2>&1 | FileCheck %s
+
+define void @fooOptnone() #0 {
+; CHECK-NOT: Changing optimization level for Function fooOptnone
+; CHECK-NOT: Restoring optimization level for Function fooOptnone
+  ret void
+}
+
+attributes #0 = { nounwind optnone noinline }
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/issue97975.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/issue97975.ll
new file mode 100644
index 000000000000..98db8f0a0829
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/issue97975.ll
@@ -0,0 +1,444 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32S
+; RUN: llc --mtriple=loongarch32 --mattr=+f -target-abi=ilp32s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32F-ILP32S
+; RUN: llc --mtriple=loongarch32 --mattr=+f -target-abi=ilp32d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32F-ILP32D
+; RUN: llc --mtriple=loongarch32 --mattr=+d -target-abi=ilp32s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32D-ILP32S
+; RUN: llc --mtriple=loongarch32 --mattr=+d -target-abi=ilp32d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32D-ILP32D
+; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64S
+; RUN: llc --mtriple=loongarch64 --mattr=+f -target-abi=lp64s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64F-LP64S
+; RUN: llc --mtriple=loongarch64 --mattr=+f -target-abi=lp64d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64F-LP64D
+; RUN: llc --mtriple=loongarch64 --mattr=+d -target-abi=lp64s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64D-LP64S
+; RUN: llc --mtriple=loongarch64 --mattr=+d -target-abi=lp64d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64D-LP64D
+
+define half @f(half %a, half %b, half %c) {
+; LA32S-LABEL: f:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    addi.w $sp, $sp, -32
+; LA32S-NEXT:    .cfi_def_cfa_offset 32
+; LA32S-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32S-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32S-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32S-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32S-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32S-NEXT:    .cfi_offset 1, -4
+; LA32S-NEXT:    .cfi_offset 22, -8
+; LA32S-NEXT:    .cfi_offset 23, -12
+; LA32S-NEXT:    .cfi_offset 24, -16
+; LA32S-NEXT:    .cfi_offset 25, -20
+; LA32S-NEXT:    move $fp, $a2
+; LA32S-NEXT:    move $s0, $a1
+; LA32S-NEXT:    lu12i.w $a1, 15
+; LA32S-NEXT:    ori $s2, $a1, 4095
+; LA32S-NEXT:    and $a0, $a0, $s2
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    move $s1, $a0
+; LA32S-NEXT:    and $a0, $s0, $s2
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    move $a1, $a0
+; LA32S-NEXT:    move $a0, $s1
+; LA32S-NEXT:    bl __addsf3
+; LA32S-NEXT:    bl __truncsfhf2
+; LA32S-NEXT:    move $s0, $a0
+; LA32S-NEXT:    and $a0, $fp, $s2
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    move $fp, $a0
+; LA32S-NEXT:    and $a0, $s0, $s2
+; LA32S-NEXT:    bl __extendhfsf2
+; LA32S-NEXT:    move $a1, $fp
+; LA32S-NEXT:    bl __addsf3
+; LA32S-NEXT:    bl __truncsfhf2
+; LA32S-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32S-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32S-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32S-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32S-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32S-NEXT:    addi.w $sp, $sp, 32
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: f:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32S-NEXT:    .cfi_def_cfa_offset 16
+; LA32F-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    st.w $s0, $sp, 4 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    .cfi_offset 1, -4
+; LA32F-ILP32S-NEXT:    .cfi_offset 22, -8
+; LA32F-ILP32S-NEXT:    .cfi_offset 23, -12
+; LA32F-ILP32S-NEXT:    move $fp, $a2
+; LA32F-ILP32S-NEXT:    move $s0, $a0
+; LA32F-ILP32S-NEXT:    move $a0, $a1
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    fst.s $fa0, $sp, 0 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    move $a0, $s0
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    fld.s $fa1, $sp, 0 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    fadd.s $fa0, $fa0, $fa1
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    bl __truncsfhf2
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    fst.s $fa0, $sp, 0 # 4-byte Folded Spill
+; LA32F-ILP32S-NEXT:    move $a0, $fp
+; LA32F-ILP32S-NEXT:    bl __extendhfsf2
+; LA32F-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32S-NEXT:    fld.s $fa1, $sp, 0 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32F-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32S-NEXT:    bl __truncsfhf2
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32S-NEXT:    ld.w $s0, $sp, 4 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: f:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, -16
+; LA32F-ILP32D-NEXT:    .cfi_def_cfa_offset 16
+; LA32F-ILP32D-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    fst.s $fs0, $sp, 8 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    fst.s $fs1, $sp, 4 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    fst.s $fs2, $sp, 0 # 4-byte Folded Spill
+; LA32F-ILP32D-NEXT:    .cfi_offset 1, -4
+; LA32F-ILP32D-NEXT:    .cfi_offset 56, -8
+; LA32F-ILP32D-NEXT:    .cfi_offset 57, -12
+; LA32F-ILP32D-NEXT:    .cfi_offset 58, -16
+; LA32F-ILP32D-NEXT:    fmov.s $fs0, $fa2
+; LA32F-ILP32D-NEXT:    fmov.s $fs1, $fa0
+; LA32F-ILP32D-NEXT:    fmov.s $fa0, $fa1
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    fmov.s $fs2, $fa0
+; LA32F-ILP32D-NEXT:    fmov.s $fa0, $fs1
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    fadd.s $fa0, $fa0, $fs2
+; LA32F-ILP32D-NEXT:    bl __truncsfhf2
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    fmov.s $fs1, $fa0
+; LA32F-ILP32D-NEXT:    fmov.s $fa0, $fs0
+; LA32F-ILP32D-NEXT:    bl __extendhfsf2
+; LA32F-ILP32D-NEXT:    fadd.s $fa0, $fs1, $fa0
+; LA32F-ILP32D-NEXT:    bl __truncsfhf2
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    fld.s $fs2, $sp, 0 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    fld.s $fs1, $sp, 4 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    fld.s $fs0, $sp, 8 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32F-ILP32D-NEXT:    addi.w $sp, $sp, 16
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: f:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, -16
+; LA32D-ILP32S-NEXT:    .cfi_def_cfa_offset 16
+; LA32D-ILP32S-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    st.w $fp, $sp, 8 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    st.w $s0, $sp, 4 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    .cfi_offset 1, -4
+; LA32D-ILP32S-NEXT:    .cfi_offset 22, -8
+; LA32D-ILP32S-NEXT:    .cfi_offset 23, -12
+; LA32D-ILP32S-NEXT:    move $fp, $a2
+; LA32D-ILP32S-NEXT:    move $s0, $a0
+; LA32D-ILP32S-NEXT:    move $a0, $a1
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    fst.s $fa0, $sp, 0 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    move $a0, $s0
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    fld.s $fa1, $sp, 0 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    fadd.s $fa0, $fa0, $fa1
+; LA32D-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32S-NEXT:    bl __truncsfhf2
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    fst.s $fa0, $sp, 0 # 4-byte Folded Spill
+; LA32D-ILP32S-NEXT:    move $a0, $fp
+; LA32D-ILP32S-NEXT:    bl __extendhfsf2
+; LA32D-ILP32S-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32S-NEXT:    fld.s $fa1, $sp, 0 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA32D-ILP32S-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32S-NEXT:    bl __truncsfhf2
+; LA32D-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32S-NEXT:    ld.w $s0, $sp, 4 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    ld.w $fp, $sp, 8 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32D-ILP32S-NEXT:    addi.w $sp, $sp, 16
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: f:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, -32
+; LA32D-ILP32D-NEXT:    .cfi_def_cfa_offset 32
+; LA32D-ILP32D-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32D-ILP32D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32D-ILP32D-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA32D-ILP32D-NEXT:    fst.d $fs2, $sp, 0 # 8-byte Folded Spill
+; LA32D-ILP32D-NEXT:    .cfi_offset 1, -4
+; LA32D-ILP32D-NEXT:    .cfi_offset 56, -16
+; LA32D-ILP32D-NEXT:    .cfi_offset 57, -24
+; LA32D-ILP32D-NEXT:    .cfi_offset 58, -32
+; LA32D-ILP32D-NEXT:    fmov.s $fs0, $fa2
+; LA32D-ILP32D-NEXT:    fmov.s $fs1, $fa0
+; LA32D-ILP32D-NEXT:    fmov.s $fa0, $fa1
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    fmov.s $fs2, $fa0
+; LA32D-ILP32D-NEXT:    fmov.s $fa0, $fs1
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    fadd.s $fa0, $fa0, $fs2
+; LA32D-ILP32D-NEXT:    bl __truncsfhf2
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    fmov.s $fs1, $fa0
+; LA32D-ILP32D-NEXT:    fmov.s $fa0, $fs0
+; LA32D-ILP32D-NEXT:    bl __extendhfsf2
+; LA32D-ILP32D-NEXT:    fadd.s $fa0, $fs1, $fa0
+; LA32D-ILP32D-NEXT:    bl __truncsfhf2
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    fld.d $fs2, $sp, 0 # 8-byte Folded Reload
+; LA32D-ILP32D-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32D-ILP32D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32D-ILP32D-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32D-ILP32D-NEXT:    addi.w $sp, $sp, 32
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: f:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    addi.d $sp, $sp, -32
+; LA64S-NEXT:    .cfi_def_cfa_offset 32
+; LA64S-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64S-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64S-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64S-NEXT:    fst.d $fs2, $sp, 0 # 8-byte Folded Spill
+; LA64S-NEXT:    .cfi_offset 1, -8
+; LA64S-NEXT:    .cfi_offset 56, -16
+; LA64S-NEXT:    .cfi_offset 57, -24
+; LA64S-NEXT:    .cfi_offset 58, -32
+; LA64S-NEXT:    fmov.s $fs0, $fa2
+; LA64S-NEXT:    fmov.s $fs1, $fa0
+; LA64S-NEXT:    fmov.s $fa0, $fa1
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    fmov.s $fs2, $fa0
+; LA64S-NEXT:    fmov.s $fa0, $fs1
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    fadd.s $fa0, $fa0, $fs2
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    fmov.s $fs1, $fa0
+; LA64S-NEXT:    fmov.s $fa0, $fs0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    fadd.s $fa0, $fs1, $fa0
+; LA64S-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64S-NEXT:    jirl $ra, $ra, 0
+; LA64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64S-NEXT:    lu12i.w $a1, -16
+; LA64S-NEXT:    or $a0, $a0, $a1
+; LA64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64S-NEXT:    fld.d $fs2, $sp, 0 # 8-byte Folded Reload
+; LA64S-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64S-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64S-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64S-NEXT:    addi.d $sp, $sp, 32
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: f:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, -32
+; LA64F-LP64S-NEXT:    .cfi_def_cfa_offset 32
+; LA64F-LP64S-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT:    .cfi_offset 1, -8
+; LA64F-LP64S-NEXT:    .cfi_offset 22, -16
+; LA64F-LP64S-NEXT:    .cfi_offset 23, -24
+; LA64F-LP64S-NEXT:    move $fp, $a2
+; LA64F-LP64S-NEXT:    move $s0, $a0
+; LA64F-LP64S-NEXT:    move $a0, $a1
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    fst.s $fa0, $sp, 4 # 4-byte Folded Spill
+; LA64F-LP64S-NEXT:    move $a0, $s0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    fld.s $fa1, $sp, 4 # 4-byte Folded Reload
+; LA64F-LP64S-NEXT:    fadd.s $fa0, $fa0, $fa1
+; LA64F-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    fst.s $fa0, $sp, 4 # 4-byte Folded Spill
+; LA64F-LP64S-NEXT:    move $a0, $fp
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64S-NEXT:    fld.s $fa1, $sp, 4 # 4-byte Folded Reload
+; LA64F-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64F-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64F-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64S-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT:    addi.d $sp, $sp, 32
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: f:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, -32
+; LA64F-LP64D-NEXT:    .cfi_def_cfa_offset 32
+; LA64F-LP64D-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    fst.d $fs2, $sp, 0 # 8-byte Folded Spill
+; LA64F-LP64D-NEXT:    .cfi_offset 1, -8
+; LA64F-LP64D-NEXT:    .cfi_offset 56, -16
+; LA64F-LP64D-NEXT:    .cfi_offset 57, -24
+; LA64F-LP64D-NEXT:    .cfi_offset 58, -32
+; LA64F-LP64D-NEXT:    fmov.s $fs0, $fa2
+; LA64F-LP64D-NEXT:    fmov.s $fs1, $fa0
+; LA64F-LP64D-NEXT:    fmov.s $fa0, $fa1
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    fmov.s $fs2, $fa0
+; LA64F-LP64D-NEXT:    fmov.s $fa0, $fs1
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    fadd.s $fa0, $fa0, $fs2
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    fmov.s $fs1, $fa0
+; LA64F-LP64D-NEXT:    fmov.s $fa0, $fs0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    fadd.s $fa0, $fs1, $fa0
+; LA64F-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64F-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    fld.d $fs2, $sp, 0 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64F-LP64D-NEXT:    addi.d $sp, $sp, 32
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: f:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, -32
+; LA64D-LP64S-NEXT:    .cfi_def_cfa_offset 32
+; LA64D-LP64S-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    st.d $s0, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT:    .cfi_offset 1, -8
+; LA64D-LP64S-NEXT:    .cfi_offset 22, -16
+; LA64D-LP64S-NEXT:    .cfi_offset 23, -24
+; LA64D-LP64S-NEXT:    move $fp, $a2
+; LA64D-LP64S-NEXT:    move $s0, $a0
+; LA64D-LP64S-NEXT:    move $a0, $a1
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    fst.s $fa0, $sp, 4 # 4-byte Folded Spill
+; LA64D-LP64S-NEXT:    move $a0, $s0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    fld.s $fa1, $sp, 4 # 4-byte Folded Reload
+; LA64D-LP64S-NEXT:    fadd.s $fa0, $fa0, $fa1
+; LA64D-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    fst.s $fa0, $sp, 4 # 4-byte Folded Spill
+; LA64D-LP64S-NEXT:    move $a0, $fp
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64S-NEXT:    fld.s $fa1, $sp, 4 # 4-byte Folded Reload
+; LA64D-LP64S-NEXT:    fadd.s $fa0, $fa1, $fa0
+; LA64D-LP64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64S-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64D-LP64S-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64S-NEXT:    ld.d $s0, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT:    addi.d $sp, $sp, 32
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: f:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, -32
+; LA64D-LP64D-NEXT:    .cfi_def_cfa_offset 32
+; LA64D-LP64D-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    fst.d $fs2, $sp, 0 # 8-byte Folded Spill
+; LA64D-LP64D-NEXT:    .cfi_offset 1, -8
+; LA64D-LP64D-NEXT:    .cfi_offset 56, -16
+; LA64D-LP64D-NEXT:    .cfi_offset 57, -24
+; LA64D-LP64D-NEXT:    .cfi_offset 58, -32
+; LA64D-LP64D-NEXT:    fmov.s $fs0, $fa2
+; LA64D-LP64D-NEXT:    fmov.s $fs1, $fa0
+; LA64D-LP64D-NEXT:    fmov.s $fa0, $fa1
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    fmov.s $fs2, $fa0
+; LA64D-LP64D-NEXT:    fmov.s $fa0, $fs1
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    fadd.s $fa0, $fa0, $fs2
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    fmov.s $fs1, $fa0
+; LA64D-LP64D-NEXT:    fmov.s $fa0, $fs0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    fadd.s $fa0, $fs1, $fa0
+; LA64D-LP64D-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64D-LP64D-NEXT:    jirl $ra, $ra, 0
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    fld.d $fs2, $sp, 0 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64D-LP64D-NEXT:    addi.d $sp, $sp, 32
+; LA64D-LP64D-NEXT:    ret
+    %d = fadd half %a, %b
+    %e = fadd half %d, %c
+    ret half %e
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/issue97981.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/issue97981.ll
new file mode 100644
index 000000000000..856cd46de8d3
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/issue97981.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32S
+; RUN: llc --mtriple=loongarch32 --mattr=+f -target-abi=ilp32s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32F-ILP32S
+; RUN: llc --mtriple=loongarch32 --mattr=+f -target-abi=ilp32d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32F-ILP32D
+; RUN: llc --mtriple=loongarch32 --mattr=+d -target-abi=ilp32s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32D-ILP32S
+; RUN: llc --mtriple=loongarch32 --mattr=+d -target-abi=ilp32d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32D-ILP32D
+; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64S
+; RUN: llc --mtriple=loongarch64 --mattr=+f -target-abi=lp64s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64F-LP64S
+; RUN: llc --mtriple=loongarch64 --mattr=+f -target-abi=lp64d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64F-LP64D
+; RUN: llc --mtriple=loongarch64 --mattr=+d -target-abi=lp64s --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64D-LP64S
+; RUN: llc --mtriple=loongarch64 --mattr=+d -target-abi=lp64d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64D-LP64D
+
+define half @to_half(i16 %bits) {
+; LA32S-LABEL: to_half:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: to_half:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: to_half:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32F-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32F-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: to_half:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32S-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: to_half:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    lu12i.w $a1, -16
+; LA32D-ILP32D-NEXT:    or $a0, $a0, $a1
+; LA32D-ILP32D-NEXT:    movgr2fr.w $fa0, $a0
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: to_half:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    lu12i.w $a1, -16
+; LA64S-NEXT:    or $a0, $a0, $a1
+; LA64S-NEXT:    movgr2fr.w $fa0, $a0
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: to_half:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: to_half:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64F-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64F-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: to_half:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64S-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: to_half:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    lu12i.w $a1, -16
+; LA64D-LP64D-NEXT:    or $a0, $a0, $a1
+; LA64D-LP64D-NEXT:    movgr2fr.w $fa0, $a0
+; LA64D-LP64D-NEXT:    ret
+    %f = bitcast i16 %bits to half
+    ret half %f
+}
+
+define i16 @from_half(half %f) {
+; LA32S-LABEL: from_half:
+; LA32S:       # %bb.0:
+; LA32S-NEXT:    ret
+;
+; LA32F-ILP32S-LABEL: from_half:
+; LA32F-ILP32S:       # %bb.0:
+; LA32F-ILP32S-NEXT:    ret
+;
+; LA32F-ILP32D-LABEL: from_half:
+; LA32F-ILP32D:       # %bb.0:
+; LA32F-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32F-ILP32D-NEXT:    ret
+;
+; LA32D-ILP32S-LABEL: from_half:
+; LA32D-ILP32S:       # %bb.0:
+; LA32D-ILP32S-NEXT:    ret
+;
+; LA32D-ILP32D-LABEL: from_half:
+; LA32D-ILP32D:       # %bb.0:
+; LA32D-ILP32D-NEXT:    movfr2gr.s $a0, $fa0
+; LA32D-ILP32D-NEXT:    ret
+;
+; LA64S-LABEL: from_half:
+; LA64S:       # %bb.0:
+; LA64S-NEXT:    movfr2gr.s $a0, $fa0
+; LA64S-NEXT:    ret
+;
+; LA64F-LP64S-LABEL: from_half:
+; LA64F-LP64S:       # %bb.0:
+; LA64F-LP64S-NEXT:    ret
+;
+; LA64F-LP64D-LABEL: from_half:
+; LA64F-LP64D:       # %bb.0:
+; LA64F-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64F-LP64D-NEXT:    ret
+;
+; LA64D-LP64S-LABEL: from_half:
+; LA64D-LP64S:       # %bb.0:
+; LA64D-LP64S-NEXT:    ret
+;
+; LA64D-LP64D-LABEL: from_half:
+; LA64D-LP64D:       # %bb.0:
+; LA64D-LP64D-NEXT:    movfr2gr.s $a0, $fa0
+; LA64D-LP64D-NEXT:    ret
+    %bits = bitcast half %f to i16
+    ret i16 %bits
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll
index 95fc9479a587..0e172950340e 100644
--- a/external/llvm-project/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll
@@ -4,76 +4,14 @@
 define <32 x i8> @shuffle_v32i8(<32 x i8> %a) {
 ; CHECK-LABEL: shuffle_v32i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi.d $sp, $sp, -64
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset 1, -8
-; CHECK-NEXT:    .cfi_offset 22, -16
-; CHECK-NEXT:    addi.d $fp, $sp, 64
-; CHECK-NEXT:    .cfi_def_cfa 22, 0
-; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvst $xr0, $sp, 0
-; CHECK-NEXT:    ld.h $a0, $sp, 16
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
-; CHECK-NEXT:    vpickve2gr.h $a1, $vr1, 0
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 1
-; CHECK-NEXT:    vpickve2gr.h $a1, $vr1, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 2
-; CHECK-NEXT:    vpickve2gr.h $a1, $vr1, 2
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 3
-; CHECK-NEXT:    vpickve2gr.h $a1, $vr1, 3
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 4
-; CHECK-NEXT:    vpickve2gr.h $a1, $vr1, 4
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 5
-; CHECK-NEXT:    vpickve2gr.h $a1, $vr1, 5
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 6
-; CHECK-NEXT:    vpickve2gr.h $a1, $vr1, 6
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 7
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 0
-; CHECK-NEXT:    ld.h $a0, $sp, 18
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 1
-; CHECK-NEXT:    ld.h $a0, $sp, 20
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 2
-; CHECK-NEXT:    ld.h $a0, $sp, 22
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 3
-; CHECK-NEXT:    ld.h $a0, $sp, 24
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 4
-; CHECK-NEXT:    ld.h $a0, $sp, 26
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 5
-; CHECK-NEXT:    ld.h $a0, $sp, 28
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 6
-; CHECK-NEXT:    ld.h $a0, $sp, 30
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
-; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 7
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    addi.d $sp, $fp, -64
-; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI0_1)
+; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
+; CHECK-NEXT:    xvshuf.h $xr1, $xr2, $xr0
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 16, i32 17, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
   ret <32 x i8> %shuffle
@@ -83,21 +21,13 @@ define <32 x i8> @shuffle_v32i8(<32 x i8> %a) {
 define <16 x i16> @shuffle_v16i16(<16 x i16> %a) {
 ; CHECK-LABEL: shuffle_v16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 4
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a0, 0
-; CHECK-NEXT:    xvpickve2gr.w $a1, $xr0, 0
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a1, 1
-; CHECK-NEXT:    xvpickve2gr.w $a1, $xr0, 1
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a1, 2
-; CHECK-NEXT:    xvpickve2gr.w $a1, $xr0, 2
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a1, 3
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a0, 4
-; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 5
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a0, 5
-; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 6
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a0, 6
-; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
-; CHECK-NEXT:    xvinsgr2vr.w $xr1, $a0, 7
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI1_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI1_1)
+; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
+; CHECK-NEXT:    xvshuf.w $xr1, $xr2, $xr0
 ; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 8, i32 9, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -107,13 +37,13 @@ define <16 x i16> @shuffle_v16i16(<16 x i16> %a) {
 define <8 x i32> @shuffle_v8i32(<8 x i32> %a) {
 ; CHECK-LABEL: shuffle_v8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
-; CHECK-NEXT:    xvpickve2gr.d $a1, $xr0, 0
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a1, 1
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI2_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI2_1)
+; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
+; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
 ; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 4, i32 5, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
@@ -123,14 +53,13 @@ define <8 x i32> @shuffle_v8i32(<8 x i32> %a) {
 define <4 x i64> @shuffle_v4i64(<4 x i64> %a) {
 ; CHECK-LABEL: shuffle_v4i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 1
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_1)
+; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
+; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
 ; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 3, i32 1, i32 2, i32 0>
@@ -140,19 +69,13 @@ define <4 x i64> @shuffle_v4i64(<4 x i64> %a) {
 define <8 x float> @shuffle_v8f32(<8 x float> %a) {
 ; CHECK-LABEL: shuffle_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
-; CHECK-NEXT:    movgr2fr.d $fa1, $a0
-; CHECK-NEXT:    movfr2gr.d $a0, $fa1
-; CHECK-NEXT:    xvpickve2gr.d $a1, $xr0, 0
-; CHECK-NEXT:    movgr2fr.d $fa2, $a1
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
-; CHECK-NEXT:    movfr2gr.d $a1, $fa2
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a1, 1
-; CHECK-NEXT:    xvpickve2gr.d $a1, $xr0, 3
-; CHECK-NEXT:    movgr2fr.d $fa0, $a1
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
-; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI4_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI4_1)
+; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
+; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
 ; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 4, i32 5, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
@@ -162,22 +85,13 @@ define <8 x float> @shuffle_v8f32(<8 x float> %a) {
 define <4 x double> @shuffle_v4f64(<4 x double> %a) {
 ; CHECK-LABEL: shuffle_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
-; CHECK-NEXT:    movgr2fr.d $fa1, $a0
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
-; CHECK-NEXT:    movgr2fr.d $fa2, $a0
-; CHECK-NEXT:    movfr2gr.d $a0, $fa1
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 0
-; CHECK-NEXT:    movfr2gr.d $a0, $fa2
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 1
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 2
-; CHECK-NEXT:    movgr2fr.d $fa2, $a0
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
-; CHECK-NEXT:    movgr2fr.d $fa0, $a0
-; CHECK-NEXT:    movfr2gr.d $a0, $fa2
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 2
-; CHECK-NEXT:    movfr2gr.d $a0, $fa0
-; CHECK-NEXT:    xvinsgr2vr.d $xr1, $a0, 3
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI5_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI5_1)
+; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
+; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
 ; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
 ; CHECK-NEXT:    ret
   %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 1, i32 2, i32 0>
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
new file mode 100644
index 000000000000..5a861be95977
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
@@ -0,0 +1,649 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx --verify-machineinstrs < %s | FileCheck %s
+
+define i32 @xmsk_eq_allzeros_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_eq_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmsknz.b $xr0, $xr0
+; CHECK-NEXT:    xvnor.v $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp eq <32 x i8> %a, splat (i8 0)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i32 @xmsk_sgt_allzeros_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_sgt_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvrepli.b $xr1, 0
+; CHECK-NEXT:    xvslt.b $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sgt <32 x i8> %a, splat (i8 0)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i32 @xmsk_sgt_allones_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_sgt_allones_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskgez.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sgt <32 x i8> %a, splat (i8 -1)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i32 @xmsk_sge_allzeros_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_sge_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskgez.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sge <32 x i8> %a, splat (i8 0)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i32 @xmsk_slt_allzeros_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_slt_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <32 x i8> %a, splat (i8 0)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i16 @xmsk_slt_allzeros_i16(<16 x i16 > %a) {
+; CHECK-LABEL: xmsk_slt_allzeros_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.h $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 15, 8
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <16 x i16> %a, splat (i16 0)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i8 @xmsk_slt_allzeros_i32(<8 x i32 > %a) {
+; CHECK-LABEL: xmsk_slt_allzeros_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <8 x i32> %a, splat (i32 0)
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+define i4 @xmsk_slt_allzeros_i64(<4 x i64 > %a) {
+; CHECK-LABEL: xmsk_slt_allzeros_i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.d $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 3, 2
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <4 x i64> %a, splat (i64 0)
+  %2 = bitcast <4 x i1> %1 to i4
+  ret i4 %2
+}
+
+define i32 @xmsk_sle_allzeros_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_sle_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvslei.b $xr0, $xr0, 0
+; CHECK-NEXT:    xvmskltz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <32 x i8> %a, splat (i8 0)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i32 @xmsk_sle_allones_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_sle_allones_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <32 x i8> %a, splat (i8 -1)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i16 @xmsk_sle_allones_i32(<16 x i16 > %a) {
+; CHECK-LABEL: xmsk_sle_allones_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.h $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 15, 8
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <16 x i16> %a, splat (i16 -1)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i8 @xmsk_sle_allones_i16(<8 x i32 > %a) {
+; CHECK-LABEL: xmsk_sle_allones_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <8 x i32> %a, splat (i32 -1)
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+define i4 @xmsk_sle_allones_i64(<4 x i64 > %a) {
+; CHECK-LABEL: xmsk_sle_allones_i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmskltz.d $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 3, 2
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <4 x i64> %a, splat (i64 -1)
+  %2 = bitcast <4 x i1> %1 to i4
+  ret i4 %2
+}
+
+define i32 @xmsk_ne_allzeros_i8(<32 x i8 > %a) {
+; CHECK-LABEL: xmsk_ne_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvmsknz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp ne <32 x i8> %a, splat (i8 0)
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+define i4 @xvmsk_sgt_v4i64(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: xvmsk_sgt_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.d $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 3, 2
+; CHECK-NEXT:    ret
+  %x = icmp sgt <4 x i64> %a, %b
+  %res = bitcast <4 x i1> %x to i4
+  ret i4 %res
+}
+
+define i4 @xvmsk_ogt_v4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: xvmsk_ogt_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvfcmp.clt.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.d $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 3, 2
+; CHECK-NEXT:    ret
+  %x = fcmp ogt <4 x double> %a, %b
+  %res = bitcast <4 x i1> %x to i4
+  ret i4 %res
+}
+
+define i8 @xvmsk_sgt_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; CHECK-LABEL: xvmsk_sgt_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.w $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x = icmp sgt <8 x i32> %a, %b
+  %res = bitcast <8 x i1> %x to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_ogt_v8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: xvmsk_ogt_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvfcmp.clt.s $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x = fcmp ogt <8 x float> %a, %b
+  %res = bitcast <8 x i1> %x to i8
+  ret i8 %res
+}
+
+define i16 @xvmsk_sgt_v16i16(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: xvmsk_sgt_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.h $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.h $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 15, 8
+; CHECK-NEXT:    ret
+  %x = icmp sgt <16 x i16> %a, %b
+  %res = bitcast <16 x i1> %x to i16
+  ret i16 %res
+}
+
+define i32 @xvmsk_sgt_v32i8(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: xvmsk_sgt_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.b $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+  %x = icmp sgt <32 x i8> %a, %b
+  %res = bitcast <32 x i1> %x to i32
+  ret i32 %res
+}
+
+define i4 @xvmsk_sgt_and_sgt_v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
+; CHECK-LABEL: xvmsk_sgt_and_sgt_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.d $xr2, $xr3, $xr2
+; CHECK-NEXT:    xvslt.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.d $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 3, 2
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <4 x i64> %a, %b
+  %x1 = icmp sgt <4 x i64> %c, %d
+  %y = and <4 x i1> %x0, %x1
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
+
+define i4 @xvmsk_ogt_and_ogt_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double> %d) {
+; CHECK-LABEL: xvmsk_ogt_and_ogt_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvfcmp.clt.d $xr2, $xr3, $xr2
+; CHECK-NEXT:    xvfcmp.clt.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.d $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 3, 2
+; CHECK-NEXT:    ret
+  %x0 = fcmp ogt <4 x double> %a, %b
+  %x1 = fcmp ogt <4 x double> %c, %d
+  %y = and <4 x i1> %x0, %x1
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
+
+define i8 @xvmsk_sgt_and_sgt_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
+; CHECK-LABEL: xvmsk_sgt_and_sgt_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.w $xr2, $xr3, $xr2
+; CHECK-NEXT:    xvslt.w $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <8 x i32> %a, %b
+  %x1 = icmp sgt <8 x i32> %c, %d
+  %y = and <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_sgt_or_sgt_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
+; CHECK-LABEL: xvmsk_sgt_or_sgt_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.w $xr2, $xr3, $xr2
+; CHECK-NEXT:    xvslt.w $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <8 x i32> %a, %b
+  %x1 = icmp sgt <8 x i32> %c, %d
+  %y = or <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_sgt_or_slt_and_eq_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d, <8 x i32> %e, <8 x i32> %f) {
+; CHECK-LABEL: xvmsk_sgt_or_slt_and_eq_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.w $xr2, $xr2, $xr3
+; CHECK-NEXT:    xvslt.w $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvseq.w $xr1, $xr4, $xr5
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <8 x i32> %a, %b
+  %x1 = icmp slt <8 x i32> %c, %d
+  %x2 = icmp eq <8 x i32> %e, %f
+  %y = or <8 x i1> %x0, %x1
+  %z = and <8 x i1> %y, %x2
+  %res = bitcast <8 x i1> %z to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_eq_vsel_slt_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2) {
+; CHECK-LABEL: xvmsk_eq_vsel_slt_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvseq.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvrepli.b $xr1, -1
+; CHECK-NEXT:    xvbitsel.v $xr0, $xr2, $xr1, $xr0
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <8 x i32> %a0, %a1
+  %slt = icmp slt <8 x i32> %a2, zeroinitializer
+  %sel = select <8 x i1> %cmp, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i1> %slt
+  %res = bitcast <8 x i1> %sel to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_sel_eq_or_eq_or_slt_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2, <8 x i32> %a3, i1 %a4) {
+; CHECK-LABEL: xvmsk_sel_eq_or_eq_or_slt_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andi $a0, $a0, 1
+; CHECK-NEXT:    xvseq.w $xr2, $xr0, $xr2
+; CHECK-NEXT:    addi.d $a1, $zero, -1
+; CHECK-NEXT:    maskeqz $a0, $a1, $a0
+; CHECK-NEXT:    xvreplgr2vr.w $xr4, $a0
+; CHECK-NEXT:    xvand.v $xr2, $xr2, $xr4
+; CHECK-NEXT:    xvseq.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvor.v $xr0, $xr3, $xr0
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %cmp0 = icmp eq <8 x i32> %a0, %a1
+  %cmp1 = icmp eq <8 x i32> %a0, %a2
+  %cmp2 = icmp slt <8 x i32> %a3, zeroinitializer
+  %sel = select i1 %a4, <8 x i1> %cmp1, <8 x i1> zeroinitializer
+  %or0 = or <8 x i1> %cmp2, %cmp0
+  %or1 = or <8 x i1> %or0, %sel
+  %res = bitcast <8 x i1> %or1 to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_ogt_and_ogt_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
+; CHECK-LABEL: xvmsk_ogt_and_ogt_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvfcmp.clt.s $xr2, $xr3, $xr2
+; CHECK-NEXT:    xvfcmp.clt.s $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x0 = fcmp ogt <8 x float> %a, %b
+  %x1 = fcmp ogt <8 x float> %c, %d
+  %y = and <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_sgt_xor_sgt_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
+; CHECK-LABEL: xvmsk_sgt_xor_sgt_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvfcmp.clt.s $xr2, $xr3, $xr2
+; CHECK-NEXT:    xvfcmp.clt.s $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x0 = fcmp ogt <8 x float> %a, %b
+  %x1 = fcmp ogt <8 x float> %c, %d
+  %y = xor <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_ugt_xor_ueq_and_ogt_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d, <8 x float> %e, <8 x float> %f) {
+; CHECK-LABEL: xvmsk_ugt_xor_ueq_and_ogt_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvfcmp.cueq.s $xr2, $xr2, $xr3
+; CHECK-NEXT:    xvfcmp.cult.s $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvfcmp.clt.s $xr1, $xr5, $xr4
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %x0 = fcmp ugt <8 x float> %a, %b
+  %x1 = fcmp ueq <8 x float> %c, %d
+  %x2 = fcmp ogt <8 x float> %e, %f
+  %y = xor <8 x i1> %x0, %x1
+  %z = and <8 x i1> %y, %x2
+  %res = bitcast <8 x i1> %z to i8
+  ret i8 %res
+}
+
+define i16 @xvmsk_sgt_and_sgt_v16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c, <16 x i16> %d) {
+; CHECK-LABEL: xvmsk_sgt_and_sgt_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.h $xr2, $xr3, $xr2
+; CHECK-NEXT:    xvslt.h $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr2
+; CHECK-NEXT:    xvmskltz.h $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 15, 8
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <16 x i16> %a, %b
+  %x1 = icmp sgt <16 x i16> %c, %d
+  %y = and <16 x i1> %x0, %x1
+  %res = bitcast <16 x i1> %y to i16
+  ret i16 %res
+}
+
+define i32 @xvmsk_sgt_and_sgt_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
+; CHECK-LABEL: xvmsk_sgt_and_sgt_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslt.b $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvslt.b $xr1, $xr3, $xr2
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvmskltz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <32 x i8> %a, %b
+  %x1 = icmp sgt <32 x i8> %c, %d
+  %y = and <32 x i1> %x0, %x1
+  %res = bitcast <32 x i1> %y to i32
+  ret i32 %res
+}
+
+define i8 @xvmsk_eq_v2i64_concat_poison(<2 x i64> %vec) {
+; CHECK-LABEL: xvmsk_eq_v2i64_concat_poison:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vseqi.d $vr0, $vr0, 0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr1, $a0, 0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 1
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vpackev.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vslli.h $vr0, $vr0, 15
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %tobool = icmp eq <2 x i64> %vec, zeroinitializer
+  %insertvec = shufflevector <2 x i1> %tobool, <2 x i1> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %res = bitcast <8 x i1> %insertvec to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_ne_v4i32_concat_poison(<4 x i32> %vec) {
+; CHECK-LABEL: xvmsk_ne_v4i32_concat_poison:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    vseqi.w $vr0, $vr0, 0
+; CHECK-NEXT:    vrepli.b $vr1, -1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 3
+; CHECK-NEXT:    st.h $a0, $sp, 6
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 2
+; CHECK-NEXT:    st.h $a0, $sp, 4
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 1
+; CHECK-NEXT:    st.h $a0, $sp, 2
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 0
+; CHECK-NEXT:    st.h $a0, $sp, 0
+; CHECK-NEXT:    vld $vr0, $sp, 0
+; CHECK-NEXT:    vslli.h $vr0, $vr0, 15
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    addi.d $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %tobool = icmp ne <4 x i32> %vec, zeroinitializer
+  %insertvec = shufflevector <4 x i1> %tobool, <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+  %res = bitcast <8 x i1> %insertvec to i8
+  ret i8 %res
+}
+
+define i8 @xvmsk_ogt_v4f64_concat_poison(<4 x double> %vec) {
+; CHECK-LABEL: xvmsk_ogt_v4f64_concat_poison:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    xvrepli.b $xr1, 0
+; CHECK-NEXT:    xvfcmp.clt.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.d $a1, $xr0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a2, $xr0, 2
+; CHECK-NEXT:    xvpickve2gr.d $a3, $xr0, 3
+; CHECK-NEXT:    st.h $a3, $sp, 6
+; CHECK-NEXT:    st.h $a2, $sp, 4
+; CHECK-NEXT:    st.h $a1, $sp, 2
+; CHECK-NEXT:    st.h $a0, $sp, 0
+; CHECK-NEXT:    vld $vr0, $sp, 0
+; CHECK-NEXT:    vslli.h $vr0, $vr0, 15
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    addi.d $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %tobool = fcmp ogt <4 x double> %vec, zeroinitializer
+  %insertvec = shufflevector <4 x i1> %tobool, <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+  %res = bitcast <8 x i1> %insertvec to i8
+  ret i8 %res
+}
+
+define i32 @xvmsk_trunc_i8(<32 x i8> %a) {
+; CHECK-LABEL: xvmsk_trunc_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslli.b $xr0, $xr0, 7
+; CHECK-NEXT:    xvmskltz.b $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 31, 16
+; CHECK-NEXT:    ret
+  %y = trunc <32 x i8> %a to <32 x i1>
+  %res = bitcast <32 x i1> %y to i32
+  ret i32 %res
+}
+
+define i16 @xvmsk_trunc_i16(<16 x i16> %a) {
+; CHECK-LABEL: xvmsk_trunc_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslli.h $xr0, $xr0, 15
+; CHECK-NEXT:    xvmskltz.h $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 15, 8
+; CHECK-NEXT:    ret
+  %y = trunc <16 x i16> %a to <16 x i1>
+  %res = bitcast <16 x i1> %y to i16
+  ret i16 %res
+}
+
+define i8 @xvmsk_trunc_i32(<8 x i32> %a) {
+; CHECK-LABEL: xvmsk_trunc_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslli.w $xr0, $xr0, 31
+; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 7, 4
+; CHECK-NEXT:    ret
+  %y = trunc <8 x i32> %a to <8 x i1>
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i4 @xvmsk_trunc_i64(<4 x i64> %a) {
+; CHECK-LABEL: xvmsk_trunc_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvslli.d $xr0, $xr0, 63
+; CHECK-NEXT:    xvmskltz.d $xr0, $xr0
+; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
+; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
+; CHECK-NEXT:    bstrins.d $a0, $a1, 3, 2
+; CHECK-NEXT:    ret
+  %y = trunc <4 x i64> %a to <4 x i1>
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/lsx/vmskcond.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/lsx/vmskcond.ll
new file mode 100644
index 000000000000..0ee30120f77a
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/lsx/vmskcond.ll
@@ -0,0 +1,590 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx --verify-machineinstrs < %s | FileCheck %s
+
+define i16 @vmsk_eq_allzeros_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_eq_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmsknz.b $vr0, $vr0
+; CHECK-NEXT:    vnor.v $vr0, $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp eq <16 x i8> %a, splat (i8 0)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i16 @vmsk_sgt_allzeros_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_sgt_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vrepli.b $vr1, 0
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vmskltz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sgt <16 x i8> %a, splat (i8 0)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i16 @vmsk_sgt_allones_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_sgt_allones_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskgez.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sgt <16 x i8> %a, splat (i8 -1)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i16 @vmsk_sge_allzeros_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_sge_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskgez.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sge <16 x i8> %a, splat (i8 0)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i16 @vmsk_slt_allzeros_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_slt_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <16 x i8> %a, splat (i8 0)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i8 @vmsk_slt_allzeros_i16(<8 x i16 > %a) {
+; CHECK-LABEL: vmsk_slt_allzeros_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <8 x i16> %a, splat (i16 0)
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+define i4 @vmsk_slt_allzeros_i32(<4 x i32 > %a) {
+; CHECK-LABEL: vmsk_slt_allzeros_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <4 x i32> %a, splat (i32 0)
+  %2 = bitcast <4 x i1> %1 to i4
+  ret i4 %2
+}
+
+define i2 @vmsk_slt_allzeros_i64(<2 x i64 > %a) {
+; CHECK-LABEL: vmsk_slt_allzeros_i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp slt <2 x i64> %a, splat (i64 0)
+  %2 = bitcast <2 x i1> %1 to i2
+  ret i2 %2
+}
+
+define i16 @vmsk_sle_allzeros_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_sle_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vslei.b $vr0, $vr0, 0
+; CHECK-NEXT:    vmskltz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <16 x i8> %a, splat (i8 0)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i16 @vmsk_sle_allones_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_sle_allones_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <16 x i8> %a, splat (i8 -1)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i8 @vmsk_sle_allones_i16(<8 x i16 > %a) {
+; CHECK-LABEL: vmsk_sle_allones_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <8 x i16> %a, splat (i16 -1)
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+define i4 @vmsk_sle_allones_i32(<4 x i32 > %a) {
+; CHECK-LABEL: vmsk_sle_allones_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <4 x i32> %a, splat (i32 -1)
+  %2 = bitcast <4 x i1> %1 to i4
+  ret i4 %2
+}
+
+define i2 @vmsk_sle_allones_i64(<2 x i64 > %a) {
+; CHECK-LABEL: vmsk_sle_allones_i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp sle <2 x i64> %a, splat (i64 -1)
+  %2 = bitcast <2 x i1> %1 to i2
+  ret i2 %2
+}
+
+define i16 @vmsk_ne_allzeros_i8(<16 x i8 > %a) {
+; CHECK-LABEL: vmsk_ne_allzeros_i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmsknz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+entry:
+  %1 = icmp ne <16 x i8> %a, splat (i8 0)
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+define i2 @vmsk_sgt_v2i8(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: vmsk_sgt_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vilvl.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.d $vr0, $vr0, 56
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <2 x i8> %a, %b
+  %res = bitcast <2 x i1> %x to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_sgt_v2i16(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: vmsk_sgt_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.h $vr0, $vr1, $vr0
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.d $vr0, $vr0, 48
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <2 x i16> %a, %b
+  %res = bitcast <2 x i1> %x to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_sgt_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: vmsk_sgt_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.w $vr0, $vr1, $vr0
+; CHECK-NEXT:    vshuf4i.w $vr0, $vr0, 16
+; CHECK-NEXT:    vslli.d $vr0, $vr0, 32
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <2 x i32> %a, %b
+  %res = bitcast <2 x i1> %x to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_sgt_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: vmsk_sgt_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.d $vr0, $vr1, $vr0
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <2 x i64> %a, %b
+  %res = bitcast <2 x i1> %x to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_ogt_v2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: vmsk_ogt_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfcmp.clt.d $vr0, $vr1, $vr0
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = fcmp ogt <2 x double> %a, %b
+  %res = bitcast <2 x i1> %x to i2
+  ret i2 %res
+}
+
+define i4 @vmsk_sgt_v4i8(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: vmsk_sgt_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vilvl.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.w $vr0, $vr0, 24
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <4 x i8> %a, %b
+  %res = bitcast <4 x i1> %x to i4
+  ret i4 %res
+}
+
+define i4 @vmsk_sgt_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: vmsk_sgt_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.h $vr0, $vr1, $vr0
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.w $vr0, $vr0, 16
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <4 x i16> %a, %b
+  %res = bitcast <4 x i1> %x to i4
+  ret i4 %res
+}
+
+define i4 @vmsk_sgt_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: vmsk_sgt_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.w $vr0, $vr1, $vr0
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <4 x i32> %a, %b
+  %res = bitcast <4 x i1> %x to i4
+  ret i4 %res
+}
+
+define i4 @vmsk_ogt_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: vmsk_ogt_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfcmp.clt.s $vr0, $vr1, $vr0
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = fcmp ogt <4 x float> %a, %b
+  %res = bitcast <4 x i1> %x to i4
+  ret i4 %res
+}
+
+define i8 @vmsk_sgt_v8i8(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: vmsk_sgt_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vilvl.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.h $vr0, $vr0, 8
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <8 x i8> %a, %b
+  %res = bitcast <8 x i1> %x to i8
+  ret i8 %res
+}
+
+define i8 @vmsk_sgt_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: vmsk_sgt_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.h $vr0, $vr1, $vr0
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <8 x i16> %a, %b
+  %res = bitcast <8 x i1> %x to i8
+  ret i8 %res
+}
+
+define i16 @vmsk_sgt_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: vmsk_sgt_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vmskltz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x = icmp sgt <16 x i8> %a, %b
+  %res = bitcast <16 x i1> %x to i16
+  ret i16 %res
+}
+
+define i2 @vmsk_sgt_and_sgt_v2i8(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.b $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vilvl.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.d $vr0, $vr0, 56
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <2 x i8> %a, %b
+  %x1 = icmp sgt <2 x i8> %c, %d
+  %y = and <2 x i1> %x0, %x1
+  %res = bitcast <2 x i1> %y to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_sgt_and_sgt_v2i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.h $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.h $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.d $vr0, $vr0, 48
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <2 x i16> %a, %b
+  %x1 = icmp sgt <2 x i16> %c, %d
+  %y = and <2 x i1> %x0, %x1
+  %res = bitcast <2 x i1> %y to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_sgt_and_sgt_v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.w $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.w $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.w $vr0, $vr0, 16
+; CHECK-NEXT:    vslli.d $vr0, $vr0, 32
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <2 x i32> %a, %b
+  %x1 = icmp sgt <2 x i32> %c, %d
+  %y = and <2 x i1> %x0, %x1
+  %res = bitcast <2 x i1> %y to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_sgt_and_sgt_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c, <2 x i64> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.d $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.d $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <2 x i64> %a, %b
+  %x1 = icmp sgt <2 x i64> %c, %d
+  %y = and <2 x i1> %x0, %x1
+  %res = bitcast <2 x i1> %y to i2
+  ret i2 %res
+}
+
+define i2 @vmsk_ogt_and_ogt_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c, <2 x double> %d) {
+; CHECK-LABEL: vmsk_ogt_and_ogt_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfcmp.clt.d $vr0, $vr1, $vr0
+; CHECK-NEXT:    vfcmp.clt.d $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = fcmp ogt <2 x double> %a, %b
+  %x1 = fcmp ogt <2 x double> %c, %d
+  %y = and <2 x i1> %x0, %x1
+  %res = bitcast <2 x i1> %y to i2
+  ret i2 %res
+}
+
+define i4 @vmsk_sgt_and_sgt_v4i8(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.b $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vilvl.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.w $vr0, $vr0, 24
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <4 x i8> %a, %b
+  %x1 = icmp sgt <4 x i8> %c, %d
+  %y = and <4 x i1> %x0, %x1
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
+
+define i4 @vmsk_sgt_and_sgt_v4i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c, <4 x i16> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.h $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.h $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vilvl.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.w $vr0, $vr0, 16
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <4 x i16> %a, %b
+  %x1 = icmp sgt <4 x i16> %c, %d
+  %y = and <4 x i1> %x0, %x1
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
+
+define i4 @vmsk_sgt_and_sgt_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.w $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.w $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <4 x i32> %a, %b
+  %x1 = icmp sgt <4 x i32> %c, %d
+  %y = and <4 x i1> %x0, %x1
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
+
+define i4 @vmsk_ogt_and_ogt_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d) {
+; CHECK-LABEL: vmsk_ogt_and_ogt_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfcmp.clt.s $vr0, $vr1, $vr0
+; CHECK-NEXT:    vfcmp.clt.s $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = fcmp ogt <4 x float> %a, %b
+  %x1 = fcmp ogt <4 x float> %c, %d
+  %y = and <4 x i1> %x0, %x1
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
+
+define i8 @vmsk_sgt_and_sgt_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.b $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vilvl.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vslli.h $vr0, $vr0, 8
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <8 x i8> %a, %b
+  %x1 = icmp sgt <8 x i8> %c, %d
+  %y = and <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i8 @vmsk_sgt_and_sgt_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c, <8 x i16> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.h $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.h $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <8 x i16> %a, %b
+  %x1 = icmp sgt <8 x i16> %c, %d
+  %y = and <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i16 @vmsk_sgt_and_sgt_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
+; CHECK-LABEL: vmsk_sgt_and_sgt_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslt.b $vr0, $vr1, $vr0
+; CHECK-NEXT:    vslt.b $vr1, $vr3, $vr2
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vmskltz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %x0 = icmp sgt <16 x i8> %a, %b
+  %x1 = icmp sgt <16 x i8> %c, %d
+  %y = and <16 x i1> %x0, %x1
+  %res = bitcast <16 x i1> %y to i16
+  ret i16 %res
+}
+
+define i16 @vmsk_trunc_i8(<16 x i8> %a) {
+; CHECK-LABEL: vmsk_trunc_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslli.b $vr0, $vr0, 7
+; CHECK-NEXT:    vmskltz.b $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %y = trunc <16 x i8> %a to <16 x i1>
+  %res = bitcast <16 x i1> %y to i16
+  ret i16 %res
+}
+
+define i8 @vmsk_trunc_i16(<8 x i16> %a) {
+; CHECK-LABEL: vmsk_trunc_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslli.h $vr0, $vr0, 15
+; CHECK-NEXT:    vmskltz.h $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %y = trunc <8 x i16> %a to <8 x i1>
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+define i4 @vmsk_trunc_i32(<4 x i32> %a) {
+; CHECK-LABEL: vmsk_trunc_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslli.w $vr0, $vr0, 31
+; CHECK-NEXT:    vmskltz.w $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %y = trunc <4 x i32> %a to <4 x i1>
+  %res = bitcast <4 x i1> %y to i4
+  ret i4 %res
+}
+
+define i2 @vmsk_trunc_i64(<2 x i64> %a) {
+; CHECK-LABEL: vmsk_trunc_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vslli.d $vr0, $vr0, 63
+; CHECK-NEXT:    vmskltz.d $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.hu $a0, $vr0, 0
+; CHECK-NEXT:    ret
+  %y = trunc <2 x i64> %a to <2 x i1>
+  %res = bitcast <2 x i1> %y to i2
+  ret i2 %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll b/external/llvm-project/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll
index 08534e307e4e..c1b1c1f7568b 100644
--- a/external/llvm-project/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll
+++ b/external/llvm-project/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll
@@ -39,6 +39,7 @@ define dso_local ptr @f(i32 noundef signext %i) "frame-pointer"="all" {
 ; CHECK-NEXT:    b .LBB0_3
 ; CHECK-NEXT:  .LBB0_3: # %if.end
 ; CHECK-NEXT:    ld.d $a0, $fp, -48 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    ori $a1, $zero, 1
 ; CHECK-NEXT:    bne $a0, $a1, .LBB0_6
 ; CHECK-NEXT:    b .LBB0_4
diff --git a/external/llvm-project/llvm/test/CodeGen/MIR/AMDGPU/virtreg-uses-unallocatable-class.mir b/external/llvm-project/llvm/test/CodeGen/MIR/AMDGPU/virtreg-uses-unallocatable-class.mir
index e5daee29a52d..8a0e7a3d1e2b 100644
--- a/external/llvm-project/llvm/test/CodeGen/MIR/AMDGPU/virtreg-uses-unallocatable-class.mir
+++ b/external/llvm-project/llvm/test/CodeGen/MIR/AMDGPU/virtreg-uses-unallocatable-class.mir
@@ -1,14 +1,13 @@
-# UNSUPPORTED: reverse_iteration
 # RUN: not llc -mtriple=amdgcn-- -mcpu=gfx900 -run-pass=none -o - %s 2>&1 | FileCheck %s
 
 # Check a diagnostic is emitted if non-allocatable classes are used
 # with virtual registers, and there's no assert.
 
-# CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register named_use in function 'virtreg_unallocatable'
-# CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register named_def in function 'virtreg_unallocatable'
 # CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register 0 in function 'virtreg_unallocatable'
-# CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register 2 in function 'virtreg_unallocatable'
 # CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register 1 in function 'virtreg_unallocatable'
+# CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register 2 in function 'virtreg_unallocatable'
+# CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register named_def in function 'virtreg_unallocatable'
+# CHECK: error: {{.*}}: Cannot use non-allocatable class 'TTMP_32' for virtual register named_use in function 'virtreg_unallocatable'
 
 ---
 name: virtreg_unallocatable
diff --git a/external/llvm-project/llvm/test/CodeGen/MSP430/fake_use_float.ll b/external/llvm-project/llvm/test/CodeGen/MSP430/fake_use_float.ll
new file mode 100644
index 000000000000..558d0d393bc4
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/MSP430/fake_use_float.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s | FileCheck %s
+target triple = "msp430"
+
+; CHECK:       mov	#19923, r12
+; CHECK:       mov	#4194, r12
+; CHECK:       mov	#25688, r12
+; CHECK:       mov	#-16245, r12
+; CHECK:       ret
+define void @test-double() {
+entry:
+  call void (...) @llvm.fake.use(double -8.765430e+02)
+  ret void
+}
+
+; CHECK:       call	#__mspabi_addd
+; CHECK:       ret
+define void @test-double2(double %0) {
+entry:
+  %1 = fadd double %0, %0
+  call void (...) @llvm.fake.use(double %1)
+  ret void
+}
+
+; CHECK:       call	#__mspabi_addf
+; CHECK:       ret
+define void @test-float(float %0) {
+entry:
+  %1 = fadd float %0, %0
+  call void (...) @llvm.fake.use(float %1)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/MSP430/fcmp.ll b/external/llvm-project/llvm/test/CodeGen/MSP430/fcmp.ll
new file mode 100644
index 000000000000..df1edc61b337
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/MSP430/fcmp.ll
@@ -0,0 +1,761 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=msp430-unknown-unknown < %s | FileCheck %s
+
+define i1 @fcmp_false_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_false_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    clr.b r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp false double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_oeq_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_oeq_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ogt_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_ogt_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jge .LBB2_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp ogt double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_oge_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_oge_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jge .LBB3_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB3_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp oge double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_olt_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_olt_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jl .LBB4_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB4_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp olt double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ole_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_ole_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jl .LBB5_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB5_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp ole double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_one_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_one_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r4
+; CHECK-NEXT:    push r5
+; CHECK-NEXT:    push r6
+; CHECK-NEXT:    push r7
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    sub #8, r1
+; CHECK-NEXT:    mov r15, r7
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 24(r1), r12
+; CHECK-NEXT:    mov 26(r1), r5
+; CHECK-NEXT:    mov 28(r1), r4
+; CHECK-NEXT:    mov 30(r1), r6
+; CHECK-NEXT:    mov r7, r11
+; CHECK-NEXT:    mov r5, r13
+; CHECK-NEXT:    mov r4, r14
+; CHECK-NEXT:    mov r6, r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r6, 6(r1)
+; CHECK-NEXT:    mov r4, 4(r1)
+; CHECK-NEXT:    mov r5, 2(r1)
+; CHECK-NEXT:    mov 24(r1), r13
+; CHECK-NEXT:    mov r13, 0(r1)
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r8, r12
+; CHECK-NEXT:    mov r9, r13
+; CHECK-NEXT:    mov r10, r14
+; CHECK-NEXT:    mov r7, r15
+; CHECK-NEXT:    call #__unorddf2
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r6
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    bic r6, r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    add #8, r1
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    pop r7
+; CHECK-NEXT:    pop r6
+; CHECK-NEXT:    pop r5
+; CHECK-NEXT:    pop r4
+; CHECK-NEXT:    ret
+  %cmp = fcmp one double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ord_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_ord_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub #8, r1
+; CHECK-NEXT:    mov 16(r1), 6(r1)
+; CHECK-NEXT:    mov 14(r1), 4(r1)
+; CHECK-NEXT:    mov 12(r1), 2(r1)
+; CHECK-NEXT:    mov 10(r1), 0(r1)
+; CHECK-NEXT:    call #__unorddf2
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    add #8, r1
+; CHECK-NEXT:    ret
+  %cmp = fcmp ord double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_uno_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_uno_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub #8, r1
+; CHECK-NEXT:    mov 16(r1), 6(r1)
+; CHECK-NEXT:    mov 14(r1), 4(r1)
+; CHECK-NEXT:    mov 12(r1), 2(r1)
+; CHECK-NEXT:    mov 10(r1), 0(r1)
+; CHECK-NEXT:    call #__unorddf2
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r13
+; CHECK-NEXT:    rra r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    bic r13, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    add #8, r1
+; CHECK-NEXT:    ret
+  %cmp = fcmp uno double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ueq_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_ueq_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r4
+; CHECK-NEXT:    push r5
+; CHECK-NEXT:    push r6
+; CHECK-NEXT:    push r7
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    sub #8, r1
+; CHECK-NEXT:    mov r15, r7
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 24(r1), r12
+; CHECK-NEXT:    mov 26(r1), r5
+; CHECK-NEXT:    mov 28(r1), r4
+; CHECK-NEXT:    mov 30(r1), r6
+; CHECK-NEXT:    mov r7, r11
+; CHECK-NEXT:    mov r5, r13
+; CHECK-NEXT:    mov r4, r14
+; CHECK-NEXT:    mov r6, r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r6, 6(r1)
+; CHECK-NEXT:    mov r4, 4(r1)
+; CHECK-NEXT:    mov r5, 2(r1)
+; CHECK-NEXT:    mov 24(r1), r13
+; CHECK-NEXT:    mov r13, 0(r1)
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    rra r6
+; CHECK-NEXT:    and #1, r6
+; CHECK-NEXT:    mov r8, r12
+; CHECK-NEXT:    mov r9, r13
+; CHECK-NEXT:    mov r10, r14
+; CHECK-NEXT:    mov r7, r15
+; CHECK-NEXT:    call #__unorddf2
+; CHECK-NEXT:    bis r6, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    add #8, r1
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    pop r7
+; CHECK-NEXT:    pop r6
+; CHECK-NEXT:    pop r5
+; CHECK-NEXT:    pop r4
+; CHECK-NEXT:    ret
+  %cmp = fcmp ueq double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ugt_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_ugt_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jge .LBB10_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB10_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp ugt double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_uge_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_uge_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jge .LBB11_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB11_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp uge double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ult_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_ult_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jl .LBB12_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB12_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp ult double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ule_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_ule_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jl .LBB13_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB13_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp ule double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_une_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_une_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r11
+; CHECK-NEXT:    mov r14, r10
+; CHECK-NEXT:    mov r13, r9
+; CHECK-NEXT:    mov r12, r8
+; CHECK-NEXT:    mov 8(r1), r12
+; CHECK-NEXT:    mov 10(r1), r13
+; CHECK-NEXT:    mov 12(r1), r14
+; CHECK-NEXT:    mov 14(r1), r15
+; CHECK-NEXT:    call #__mspabi_cmpd
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r13
+; CHECK-NEXT:    rra r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    bic r13, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    ret
+  %cmp = fcmp une double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_true_f64(double %a, double %b) #0 {
+; CHECK-LABEL: fcmp_true_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov.b #1, r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp true double %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_false_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_false_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    clr.b r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp false float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_oeq_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_oeq_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp oeq float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ogt_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_ogt_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jge .LBB18_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB18_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp ogt float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_oge_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_oge_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jge .LBB19_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB19_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp oge float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_olt_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_olt_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jl .LBB20_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB20_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp olt float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ole_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_ole_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jl .LBB21_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB21_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp ole float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_one_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_one_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r6
+; CHECK-NEXT:    push r7
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r10
+; CHECK-NEXT:    mov r14, r9
+; CHECK-NEXT:    mov r13, r8
+; CHECK-NEXT:    mov r12, r7
+; CHECK-NEXT:    call #__unordsf2
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r7, r12
+; CHECK-NEXT:    mov r8, r13
+; CHECK-NEXT:    mov r9, r14
+; CHECK-NEXT:    mov r10, r15
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    rra r6
+; CHECK-NEXT:    bic r12, r6
+; CHECK-NEXT:    and #1, r6
+; CHECK-NEXT:    mov.b r6, r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    pop r7
+; CHECK-NEXT:    pop r6
+; CHECK-NEXT:    ret
+  %cmp = fcmp one float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ord_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_ord_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__unordsf2
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp ord float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_uno_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_uno_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__unordsf2
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r13
+; CHECK-NEXT:    rra r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    bic r13, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp uno float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ueq_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_ueq_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r6
+; CHECK-NEXT:    push r7
+; CHECK-NEXT:    push r8
+; CHECK-NEXT:    push r9
+; CHECK-NEXT:    push r10
+; CHECK-NEXT:    mov r15, r10
+; CHECK-NEXT:    mov r14, r9
+; CHECK-NEXT:    mov r13, r8
+; CHECK-NEXT:    mov r12, r7
+; CHECK-NEXT:    call #__unordsf2
+; CHECK-NEXT:    mov r12, r6
+; CHECK-NEXT:    mov r7, r12
+; CHECK-NEXT:    mov r8, r13
+; CHECK-NEXT:    mov r9, r14
+; CHECK-NEXT:    mov r10, r15
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r12
+; CHECK-NEXT:    rra r12
+; CHECK-NEXT:    and #1, r12
+; CHECK-NEXT:    bis r6, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    pop r10
+; CHECK-NEXT:    pop r9
+; CHECK-NEXT:    pop r8
+; CHECK-NEXT:    pop r7
+; CHECK-NEXT:    pop r6
+; CHECK-NEXT:    ret
+  %cmp = fcmp ueq float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ugt_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_ugt_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jge .LBB26_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB26_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp ugt float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_uge_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_uge_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jge .LBB27_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB27_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp uge float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ult_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_ult_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    tst r13
+; CHECK-NEXT:    jl .LBB28_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB28_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp ult float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_ule_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_ule_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    mov r12, r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    cmp #1, r13
+; CHECK-NEXT:    jl .LBB29_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    clr r12
+; CHECK-NEXT:  .LBB29_2:
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp ule float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_une_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_une_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    call #__mspabi_cmpf
+; CHECK-NEXT:    tst r12
+; CHECK-NEXT:    mov r2, r13
+; CHECK-NEXT:    rra r13
+; CHECK-NEXT:    mov #1, r12
+; CHECK-NEXT:    bic r13, r12
+; CHECK-NEXT:    ; kill: def $r12b killed $r12b killed $r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp une float %a, %b
+  ret i1 %cmp
+}
+
+define i1 @fcmp_true_f32(float %a, float %b) #0 {
+; CHECK-LABEL: fcmp_true_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov.b #1, r12
+; CHECK-NEXT:    ret
+  %cmp = fcmp true float %a, %b
+  ret i1 %cmp
+}
+
+attributes #0 = { nounwind }
diff --git a/external/llvm-project/llvm/test/CodeGen/Mips/fp-maximumnum-minimumnum.ll b/external/llvm-project/llvm/test/CodeGen/Mips/fp-maximumnum-minimumnum.ll
index 3f8ff789d593..7aaf00f87113 100644
--- a/external/llvm-project/llvm/test/CodeGen/Mips/fp-maximumnum-minimumnum.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Mips/fp-maximumnum-minimumnum.ll
@@ -24,15 +24,12 @@ define float @maximumnum_float(float %x, float %y) {
 ; MIPS64R2-NEXT:    c.ule.s $f12, $f13
 ; MIPS64R2-NEXT:    mov.s $f0, $f13
 ; MIPS64R2-NEXT:    movf.s $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.s $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.s $f0, $f0
-; MIPS64R2-NEXT:    movt.s $f0, $f1, $fcc0
 ; MIPS64R2-NEXT:    mfc1 $1, $f12
 ; MIPS64R2-NEXT:    mov.s $f1, $f0
 ; MIPS64R2-NEXT:    movz.s $f1, $f12, $1
 ; MIPS64R2-NEXT:    mfc1 $1, $f13
-; MIPS64R2-NEXT:    mtc1 $zero, $f2
 ; MIPS64R2-NEXT:    movz.s $f1, $f13, $1
+; MIPS64R2-NEXT:    mtc1 $zero, $f2
 ; MIPS64R2-NEXT:    c.eq.s $f0, $f2
 ; MIPS64R2-NEXT:    jr $ra
 ; MIPS64R2-NEXT:    movt.s $f0, $f1, $fcc0
@@ -56,11 +53,8 @@ define float @maximumnum_float_nsz(float %x, float %y) {
 ; MIPS64R2-NEXT:    c.un.s $f13, $f13
 ; MIPS64R2-NEXT:    movt.s $f0, $f12, $fcc0
 ; MIPS64R2-NEXT:    c.ule.s $f12, $f0
-; MIPS64R2-NEXT:    movf.s $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.s $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.s $f0, $f0
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    movt.s $f0, $f1, $fcc0
+; MIPS64R2-NEXT:    movf.s $f0, $f12, $fcc0
   %z = call nsz float @llvm.maximumnum.f32(float %x, float %y)
   ret float %z
 }
@@ -107,9 +101,6 @@ define double @maximumnum_double(double %x, double %y) {
 ; MIPS64R2-NEXT:    c.ule.d $f12, $f13
 ; MIPS64R2-NEXT:    mov.d $f0, $f13
 ; MIPS64R2-NEXT:    movf.d $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.d $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.d $f0, $f0
-; MIPS64R2-NEXT:    movt.d $f0, $f1, $fcc0
 ; MIPS64R2-NEXT:    dmfc1 $1, $f12
 ; MIPS64R2-NEXT:    mov.d $f1, $f0
 ; MIPS64R2-NEXT:    movz.d $f1, $f12, $1
@@ -139,11 +130,8 @@ define double @maximumnum_double_nsz(double %x, double %y) {
 ; MIPS64R2-NEXT:    c.un.d $f13, $f13
 ; MIPS64R2-NEXT:    movt.d $f0, $f12, $fcc0
 ; MIPS64R2-NEXT:    c.ule.d $f12, $f0
-; MIPS64R2-NEXT:    movf.d $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.d $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.d $f0, $f0
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    movt.d $f0, $f1, $fcc0
+; MIPS64R2-NEXT:    movf.d $f0, $f12, $fcc0
   %z = call nsz double @llvm.maximumnum.f64(double %x, double %y)
   ret double %z
 }
@@ -189,9 +177,6 @@ define float @minimumnum_float(float %x, float %y) {
 ; MIPS64R2-NEXT:    c.olt.s $f12, $f13
 ; MIPS64R2-NEXT:    mov.s $f0, $f13
 ; MIPS64R2-NEXT:    movt.s $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.s $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.s $f0, $f0
-; MIPS64R2-NEXT:    movt.s $f0, $f1, $fcc0
 ; MIPS64R2-NEXT:    mfc1 $1, $f12
 ; MIPS64R2-NEXT:    lui $2, 32768
 ; MIPS64R2-NEXT:    xor $1, $1, $2
@@ -199,8 +184,8 @@ define float @minimumnum_float(float %x, float %y) {
 ; MIPS64R2-NEXT:    movz.s $f1, $f12, $1
 ; MIPS64R2-NEXT:    mfc1 $1, $f13
 ; MIPS64R2-NEXT:    xor $1, $1, $2
-; MIPS64R2-NEXT:    mtc1 $zero, $f2
 ; MIPS64R2-NEXT:    movz.s $f1, $f13, $1
+; MIPS64R2-NEXT:    mtc1 $zero, $f2
 ; MIPS64R2-NEXT:    c.eq.s $f0, $f2
 ; MIPS64R2-NEXT:    jr $ra
 ; MIPS64R2-NEXT:    movt.s $f0, $f1, $fcc0
@@ -224,11 +209,8 @@ define float @minimumnum_float_nsz(float %x, float %y) {
 ; MIPS64R2-NEXT:    c.un.s $f13, $f13
 ; MIPS64R2-NEXT:    movt.s $f0, $f12, $fcc0
 ; MIPS64R2-NEXT:    c.olt.s $f12, $f0
-; MIPS64R2-NEXT:    movt.s $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.s $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.s $f0, $f0
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    movt.s $f0, $f1, $fcc0
+; MIPS64R2-NEXT:    movt.s $f0, $f12, $fcc0
   %z = call nsz float @llvm.minimumnum.f32(float %x, float %y)
   ret float %z
 }
@@ -277,9 +259,6 @@ define double @minimumnum_double(double %x, double %y) {
 ; MIPS64R2-NEXT:    c.olt.d $f12, $f13
 ; MIPS64R2-NEXT:    mov.d $f0, $f13
 ; MIPS64R2-NEXT:    movt.d $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.d $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.d $f0, $f0
-; MIPS64R2-NEXT:    movt.d $f0, $f1, $fcc0
 ; MIPS64R2-NEXT:    dmfc1 $1, $f12
 ; MIPS64R2-NEXT:    daddiu $2, $zero, 1
 ; MIPS64R2-NEXT:    dsll $2, $2, 63
@@ -313,11 +292,8 @@ define double @minimumnum_double_nsz(double %x, double %y) {
 ; MIPS64R2-NEXT:    c.un.d $f13, $f13
 ; MIPS64R2-NEXT:    movt.d $f0, $f12, $fcc0
 ; MIPS64R2-NEXT:    c.olt.d $f12, $f0
-; MIPS64R2-NEXT:    movt.d $f0, $f12, $fcc0
-; MIPS64R2-NEXT:    add.d $f1, $f0, $f0
-; MIPS64R2-NEXT:    c.un.d $f0, $f0
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    movt.d $f0, $f1, $fcc0
+; MIPS64R2-NEXT:    movt.d $f0, $f12, $fcc0
   %z = call nsz double @llvm.minimumnum.f64(double %x, double %y)
   ret double %z
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/Mips/hf16call32_body.ll b/external/llvm-project/llvm/test/CodeGen/Mips/hf16call32_body.ll
index ea83f776bd40..3bcb6f6bc015 100644
--- a/external/llvm-project/llvm/test/CodeGen/Mips/hf16call32_body.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Mips/hf16call32_body.ll
@@ -24,7 +24,7 @@ entry:
 ; stel: addiu $25, $25, %lo(v_sf)
 ; stel: mfc1 $4, $f12
 ; stel: jr $25
-; stel: .set $__fn_local_v_sf, v_sf
+; stel: $__fn_local_v_sf = v_sf
 ; stel: .end __fn_stub_v_sf
 
 declare i32 @printf(ptr, ...) #1
@@ -46,7 +46,7 @@ entry:
 ; stel: mfc1 $4, $f12
 ; stel: mfc1 $5, $f13
 ; stel: jr $25
-; stel: .set $__fn_local_v_df, v_df
+; stel: $__fn_local_v_df = v_df
 ; stel: .end __fn_stub_v_df
 
 ; Function Attrs: nounwind
@@ -70,7 +70,7 @@ entry:
 ; stel: mfc1 $4, $f12
 ; stel: mfc1 $5, $f14
 ; stel: jr $25
-; stel: .set $__fn_local_v_sf_sf, v_sf_sf
+; stel: $__fn_local_v_sf_sf = v_sf_sf
 ; stel: .end __fn_stub_v_sf_sf
 
 ; Function Attrs: nounwind
@@ -95,7 +95,7 @@ entry:
 ; stel: mfc1 $6, $f14
 ; stel: mfc1 $7, $f15
 ; stel: jr $25
-; stel: .set $__fn_local_v_sf_df, v_sf_df
+; stel: $__fn_local_v_sf_df = v_sf_df
 ; stel: .end __fn_stub_v_sf_df
 
 ; Function Attrs: nounwind
@@ -120,7 +120,7 @@ entry:
 ; stel: mfc1 $5, $f13
 ; stel: mfc1 $6, $f14
 ; stel: jr $25
-; stel: .set $__fn_local_v_df_sf, v_df_sf
+; stel: $__fn_local_v_df_sf = v_df_sf
 ; stel: .end __fn_stub_v_df_sf
 
 ; Function Attrs: nounwind
@@ -146,7 +146,7 @@ entry:
 ; stel: mfc1 $6, $f14
 ; stel: mfc1 $7, $f15
 ; stel: jr $25
-; stel: .set $__fn_local_v_df_df, v_df_df
+; stel: $__fn_local_v_df_df = v_df_df
 ; stel: .end __fn_stub_v_df_df
 
 ; Function Attrs: nounwind
@@ -174,7 +174,7 @@ entry:
 ; stel: addiu $25, $25, %lo(sf_sf)
 ; stel: mfc1 $4, $f12
 ; stel: jr $25
-; stel: .set $__fn_local_sf_sf, sf_sf
+; stel: $__fn_local_sf_sf = sf_sf
 ; stel: .end __fn_stub_sf_sf
 
 
@@ -196,7 +196,7 @@ entry:
 ; stel: mfc1 $4, $f12
 ; stel: mfc1 $5, $f13
 ; stel: jr $25
-; stel: .set $__fn_local_sf_df, sf_df
+; stel: $__fn_local_sf_df = sf_df
 ; stel: .end __fn_stub_sf_df
 
 ; Function Attrs: nounwind
@@ -221,7 +221,7 @@ entry:
 ; stel: mfc1 $4, $f12
 ; stel: mfc1 $5, $f14
 ; stel: jr $25
-; stel: .set $__fn_local_sf_sf_sf, sf_sf_sf
+; stel: $__fn_local_sf_sf_sf = sf_sf_sf
 ; stel: .end __fn_stub_sf_sf_sf
 
 ; Function Attrs: nounwind
@@ -247,7 +247,7 @@ entry:
 ; stel: mfc1 $6, $f14
 ; stel: mfc1 $7, $f15
 ; stel: jr $25
-; stel: .set $__fn_local_sf_sf_df, sf_sf_df
+; stel: $__fn_local_sf_sf_df = sf_sf_df
 ; stel: .end __fn_stub_sf_sf_df
 
 ; Function Attrs: nounwind
@@ -273,7 +273,7 @@ entry:
 ; stel: mfc1 $5, $f13
 ; stel: mfc1 $6, $f14
 ; stel: jr $25
-; stel: .set $__fn_local_sf_df_sf, sf_df_sf
+; stel: $__fn_local_sf_df_sf = sf_df_sf
 ; stel: .end __fn_stub_sf_df_sf
 
 ; Function Attrs: nounwind
@@ -300,7 +300,7 @@ entry:
 ; stel: mfc1 $6, $f14
 ; stel: mfc1 $7, $f15
 ; stel: jr $25
-; stel: .set $__fn_local_sf_df_df, sf_df_df
+; stel: $__fn_local_sf_df_df = sf_df_df
 ; stel: .end __fn_stub_sf_df_df
 
 attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/external/llvm-project/llvm/test/CodeGen/Mips/jumptable_labels.ll b/external/llvm-project/llvm/test/CodeGen/Mips/jumptable_labels.ll
index 075b57e08d35..c12deaf03b87 100644
--- a/external/llvm-project/llvm/test/CodeGen/Mips/jumptable_labels.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Mips/jumptable_labels.ll
@@ -1,6 +1,7 @@
 ; RUN: llc -mtriple=mips-elf < %s | FileCheck %s -check-prefix=O32
 ; RUN: llc -mtriple=mips64-elf -target-abi=n32 < %s | FileCheck %s -check-prefix=N32
 ; RUN: llc -mtriple=mips64-elf < %s | FileCheck %s -check-prefix=N64
+; RUN: llc -mtriple=mipsel-windows-gnu < %s | FileCheck %s -check-prefix=MIPSEL
 
 ; We only use the '$' prefix on O32. The others use the ELF convention.
 ; O32: $JTI0_0
@@ -12,6 +13,63 @@
 ; N32: .LBB0_2:
 ; N64: .LBB0_2:
 
+; MIPSEL-LABEL: _Z3fooi:
+; MIPSEL:       # %bb.0: # %entry
+; MIPSEL-NEXT:    addiu $sp, $sp, -16
+; MIPSEL-NEXT:    sltiu	$1, $4, 7
+; MIPSEL-NEXT:    beqz $1, .LBB0_6
+; MIPSEL-NEXT:    sw $4, 4($sp)
+; MIPSEL-NEXT:  # %bb.1: # %entry
+; MIPSEL-NEXT:    sll $1, $4, 2
+; MIPSEL-NEXT:    lui $2, %hi($JTI0_0)
+; MIPSEL-NEXT:    addu $1, $1, $2
+; MIPSEL-NEXT:    lw $1, %lo($JTI0_0)($1)
+; MIPSEL-NEXT:    jr $1
+; MIPSEL-NEXT:    nop
+; MIPSEL-NEXT:  .LBB0_2: # %sw.bb
+; MIPSEL-NEXT:    lui $1, %hi($.str)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str)
+; MIPSEL-NEXT:    j .LBB0_10
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_3: # %sw.bb4
+; MIPSEL-NEXT:    lui $1, %hi($.str.4)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str.4)
+; MIPSEL-NEXT:    j .LBB0_10
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_4: # %sw.bb2
+; MIPSEL-NEXT:    lui $1, %hi($.str.2)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str.2)
+; MIPSEL-NEXT:    j .LBB0_10
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_5: # %sw.bb3
+; MIPSEL-NEXT:    lui $1, %hi($.str.3)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str.3)
+; MIPSEL-NEXT:    j .LBB0_10
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_6: # %sw.epilog
+; MIPSEL-NEXT:    lui $1, %hi($.str.7)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str.7)
+; MIPSEL-NEXT:    j .LBB0_10
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_7: # %sw.bb1
+; MIPSEL-NEXT:    lui $1, %hi($.str.1)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str.1)
+; MIPSEL-NEXT:    j .LBB0_10
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_8: # %sw.bb5
+; MIPSEL-NEXT:    lui $1, %hi($.str.5)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str.5)
+; MIPSEL-NEXT:    j .LBB0_10
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_9: # %sw.bb6
+; MIPSEL-NEXT:    lui $1, %hi($.str.6)
+; MIPSEL-NEXT:    addiu	$1, $1, %lo($.str.6)
+; MIPSEL-NEXT:    sw $1, 8($sp)
+; MIPSEL-NEXT:  .LBB0_10: # %return
+; MIPSEL-NEXT:    lw $2, 8($sp)
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    addiu	$sp, $sp, 16
+
 @.str = private unnamed_addr constant [2 x i8] c"A\00", align 1
 @.str.1 = private unnamed_addr constant [2 x i8] c"B\00", align 1
 @.str.2 = private unnamed_addr constant [2 x i8] c"C\00", align 1
diff --git a/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/fptosi.ll b/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/fptosi.ll
index 3bf17abc7965..79fe2fd26a6e 100644
--- a/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/fptosi.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/fptosi.ll
@@ -38,189 +38,189 @@
 define i32 @test1(float %t) {
 ; M32-LABEL: test1:
 ; M32:       # %bb.0: # %entry
-; M32-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST1:]] TRUNC_W_S
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; M32-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
+; M32-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3:]] MFC1
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M32R2-FP64-LABEL: test1:
 ; M32R2-FP64:       # %bb.0: # %entry
-; M32R2-FP64-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R2-FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32R2-FP64-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST1:]] TRUNC_W_S
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; M32R2-FP64-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
+; M32R2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3:]] MFC1
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M32R2-SF-LABEL: test1:
 ; M32R2-SF:       # %bb.0: # %entry
-; M32R2-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #{{[0-9]+}} ADDiu
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #[[#MCINST4:]] ADDiu
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:-24>>
 ; M32R2-SF-NEXT:    .cfi_def_cfa_offset 24
 ; M32R2-SF-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
-; M32R2-SF-NEXT:    # <MCInst #{{[0-9]+}} SW
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    # <MCInst #[[#MCINST5:]] SW
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:20>>
 ; M32R2-SF-NEXT:    .cfi_offset 31, -4
-; M32R2-SF-NEXT:    jal __fixsfsi # <MCInst #{{[0-9]+}} JAL
-; M32R2-SF-NEXT:    # <MCOperand Expr:(__fixsfsi)>>
-; M32R2-SF-NEXT:    nop # <MCInst #{{[0-9]+}} SLL
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    jal __fixsfsi # <MCInst #[[#MCINST6:]] JAL
+; M32R2-SF-NEXT:    # <MCOperand Expr:__fixsfsi>>
+; M32R2-SF-NEXT:    nop # <MCInst #[[#MCINST7:]] SLL
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:0>>
 ; M32R2-SF-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
-; M32R2-SF-NEXT:    # <MCInst #{{[0-9]+}} LW
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    # <MCInst #[[#MCINST8:]] LW
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:20>>
-; M32R2-SF-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R2-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #{{[0-9]+}} ADDiu
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; M32R2-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #[[#MCINST4]] ADDiu
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:24>>
 ;
 ; M32R3R5-LABEL: test1:
 ; M32R3R5:       # %bb.0: # %entry
-; M32R3R5-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R3R5-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R3R5-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32R3R5-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST1:]] TRUNC_W_S
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; M32R3R5-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
+; M32R3R5-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3:]] MFC1
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M32R6-LABEL: test1:
 ; M32R6:       # %bb.0: # %entry
-; M32R6-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R6-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32R6-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST1:]] TRUNC_W_S
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; M32R6-NEXT:    jr $ra # <MCInst #[[#MCINST9:]] JALR
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
+; M32R6-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3:]] MFC1
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M64-LABEL: test1:
 ; M64:       # %bb.0: # %entry
-; M64-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M64-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST1:]] TRUNC_W_S
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; M64-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>>
+; M64-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3:]] MFC1
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M64R6-LABEL: test1:
 ; M64R6:       # %bb.0: # %entry
-; M64R6-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64R6-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M64R6-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST1:]] TRUNC_W_S
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; M64R6-NEXT:    jr $ra # <MCInst #[[#MCINST10:]] JALR64
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>>
+; M64R6-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3:]] MFC1
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; MMR2-FP32-LABEL: test1:
 ; MMR2-FP32:       # %bb.0: # %entry
-; MMR2-FP32-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S_MM
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP32-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1_MM
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR2-FP32-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST11:]] TRUNC_W_S_MM
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MMR2-FP32-NEXT:    jr $ra # <MCInst #[[#MCINST12:]] JR_MM
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
+; MMR2-FP32-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST13:]] MFC1_MM
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; MMR2-FP64-LABEL: test1:
 ; MMR2-FP64:       # %bb.0: # %entry
-; MMR2-FP64-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S_MM
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1_MM
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR2-FP64-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST11:]] TRUNC_W_S_MM
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MMR2-FP64-NEXT:    jr $ra # <MCInst #[[#MCINST12:]] JR_MM
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
+; MMR2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST13:]] MFC1_MM
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; MMR2-SF-LABEL: test1:
 ; MMR2-SF:       # %bb.0: # %entry
-; MMR2-SF-NEXT:    addiusp -24 # <MCInst #{{[0-9]+}} ADDIUSP_MM
+; MMR2-SF-NEXT:    addiusp -24 # <MCInst #[[#MCINST14:]] ADDIUSP_MM
 ; MMR2-SF-NEXT:    # <MCOperand Imm:-24>>
 ; MMR2-SF-NEXT:    .cfi_def_cfa_offset 24
 ; MMR2-SF-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
-; MMR2-SF-NEXT:    # <MCInst #{{[0-9]+}} SWSP_MM
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR2-SF-NEXT:    # <MCInst #[[#MCINST15:]] SWSP_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
 ; MMR2-SF-NEXT:    # <MCOperand Imm:20>>
 ; MMR2-SF-NEXT:    .cfi_offset 31, -4
-; MMR2-SF-NEXT:    jal __fixsfsi # <MCInst #{{[0-9]+}} JAL_MM
-; MMR2-SF-NEXT:    # <MCOperand Expr:(__fixsfsi)>>
-; MMR2-SF-NEXT:    nop # <MCInst #{{[0-9]+}} SLL
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR2-SF-NEXT:    jal __fixsfsi # <MCInst #[[#MCINST16:]] JAL_MM
+; MMR2-SF-NEXT:    # <MCOperand Expr:__fixsfsi>>
+; MMR2-SF-NEXT:    nop # <MCInst #[[#MCINST17:]] SLL_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
 ; MMR2-SF-NEXT:    # <MCOperand Imm:0>>
 ; MMR2-SF-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
-; MMR2-SF-NEXT:    # <MCInst #{{[0-9]+}} LWSP_MM
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR2-SF-NEXT:    # <MCInst #[[#MCINST18:]] LWSP_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR2-SF-NEXT:    # <MCOperand Imm:20>>
-; MMR2-SF-NEXT:    addiusp 24 # <MCInst #{{[0-9]+}} ADDIUSP_MM
+; MMR2-SF-NEXT:    addiusp 24 # <MCInst #[[#MCINST14]] ADDIUSP_MM
 ; MMR2-SF-NEXT:    # <MCOperand Imm:24>>
-; MMR2-SF-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR2-SF-NEXT:    jrc $ra # <MCInst #[[#MCINST19:]] JRC16_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 ;
 ; MMR6-LABEL: test1:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    trunc.w.s $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_S_MMR6
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR6-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    trunc.w.s $f0, $f12 # <MCInst #[[#MCINST20:]] TRUNC_W_S_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MMR6-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST13:]] MFC1_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST19:]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
 ;
 ; MMR6-SF-LABEL: test1:
 ; MMR6-SF:       # %bb.0: # %entry
-; MMR6-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #{{[0-9]+}} ADDiu
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #[[#MCINST4:]] ADDiu
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:-24>>
 ; MMR6-SF-NEXT:    .cfi_def_cfa_offset 24
 ; MMR6-SF-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
-; MMR6-SF-NEXT:    # <MCInst #{{[0-9]+}} SW
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    # <MCInst #[[#MCINST5:]] SW
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:20>>
 ; MMR6-SF-NEXT:    .cfi_offset 31, -4
-; MMR6-SF-NEXT:    balc __fixsfsi # <MCInst #{{[0-9]+}} BALC_MMR6
-; MMR6-SF-NEXT:    # <MCOperand Expr:(__fixsfsi)>>
+; MMR6-SF-NEXT:    balc __fixsfsi # <MCInst #[[#MCINST21:]] BALC_MMR6
+; MMR6-SF-NEXT:    # <MCOperand Expr:__fixsfsi>>
 ; MMR6-SF-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
-; MMR6-SF-NEXT:    # <MCInst #{{[0-9]+}} LW
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    # <MCInst #[[#MCINST8:]] LW
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:20>>
-; MMR6-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #{{[0-9]+}} ADDiu
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #[[#MCINST4]] ADDiu
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:24>>
-; MMR6-SF-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-SF-NEXT:    jrc $ra # <MCInst #[[#MCINST19:]] JRC16_MM
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %conv = fptosi float %t to i32
   ret i32 %conv
@@ -229,189 +229,189 @@ entry:
 define i32 @test2(double %t) {
 ; M32-LABEL: test2:
 ; M32:       # %bb.0: # %entry
-; M32-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_D32
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST22:]] TRUNC_W_D32
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>>
+; M32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; M32-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3]] MFC1
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; M32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M32R2-FP64-LABEL: test2:
 ; M32R2-FP64:       # %bb.0: # %entry
-; M32R2-FP64-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_D64
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R2-FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32R2-FP64-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST23:]] TRUNC_W_D64
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>>
+; M32R2-FP64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; M32R2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3]] MFC1
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; M32R2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M32R2-SF-LABEL: test2:
 ; M32R2-SF:       # %bb.0: # %entry
-; M32R2-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #{{[0-9]+}} ADDiu
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #[[#MCINST4]] ADDiu
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:-24>>
 ; M32R2-SF-NEXT:    .cfi_def_cfa_offset 24
 ; M32R2-SF-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
-; M32R2-SF-NEXT:    # <MCInst #{{[0-9]+}} SW
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    # <MCInst #[[#MCINST5]] SW
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:20>>
 ; M32R2-SF-NEXT:    .cfi_offset 31, -4
-; M32R2-SF-NEXT:    jal __fixdfsi # <MCInst #{{[0-9]+}} JAL
-; M32R2-SF-NEXT:    # <MCOperand Expr:(__fixdfsi)>>
-; M32R2-SF-NEXT:    nop # <MCInst #{{[0-9]+}} SLL
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    jal __fixdfsi # <MCInst #[[#MCINST6]] JAL
+; M32R2-SF-NEXT:    # <MCOperand Expr:__fixdfsi>>
+; M32R2-SF-NEXT:    nop # <MCInst #[[#MCINST7]] SLL
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:0>>
 ; M32R2-SF-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
-; M32R2-SF-NEXT:    # <MCInst #{{[0-9]+}} LW
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    # <MCInst #[[#MCINST8]] LW
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:20>>
-; M32R2-SF-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R2-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #{{[0-9]+}} ADDiu
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; M32R2-SF-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; M32R2-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #[[#MCINST4]] ADDiu
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; M32R2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; M32R2-SF-NEXT:    # <MCOperand Imm:24>>
 ;
 ; M32R3R5-LABEL: test2:
 ; M32R3R5:       # %bb.0: # %entry
-; M32R3R5-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_D32
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R3R5-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R3R5-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R3R5-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32R3R5-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST22:]] TRUNC_W_D32
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>>
+; M32R3R5-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; M32R3R5-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3]] MFC1
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; M32R3R5-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M32R6-LABEL: test2:
 ; M32R6:       # %bb.0: # %entry
-; M32R6-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_D64
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M32R6-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M32R6-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST23:]] TRUNC_W_D64
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>>
+; M32R6-NEXT:    jr $ra # <MCInst #[[#MCINST9]] JALR
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; M32R6-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3]] MFC1
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; M32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M64-LABEL: test2:
 ; M64:       # %bb.0: # %entry
-; M64-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_D64
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M64-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST23:]] TRUNC_W_D64
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>>
+; M64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG7]]>>
+; M64-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3]] MFC1
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; M64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; M64R6-LABEL: test2:
 ; M64R6:       # %bb.0: # %entry
-; M64R6-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_D64
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; M64R6-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; M64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; M64R6-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST23:]] TRUNC_W_D64
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>>
+; M64R6-NEXT:    jr $ra # <MCInst #[[#MCINST10]] JALR64
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>>
+; M64R6-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST3]] MFC1
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; M64R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; MMR2-FP32-LABEL: test2:
 ; MMR2-FP32:       # %bb.0: # %entry
-; MMR2-FP32-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_MM
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP32-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1_MM
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR2-FP32-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST24:]] TRUNC_W_MM
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>>
+; MMR2-FP32-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JR_MM
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; MMR2-FP32-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST13]] MFC1_MM
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MMR2-FP32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; MMR2-FP64-LABEL: test2:
 ; MMR2-FP64:       # %bb.0: # %entry
-; MMR2-FP64-NEXT:    cvt.w.d $f0, $f12 # <MCInst #{{[0-9]+}} CVT_W_D64_MM
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1_MM
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR2-FP64-NEXT:    cvt.w.d $f0, $f12 # <MCInst #[[#MCINST25:]] CVT_W_D64_MM
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>>
+; MMR2-FP64-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JR_MM
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
+; MMR2-FP64-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST13]] MFC1_MM
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MMR2-FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
 ;
 ; MMR2-SF-LABEL: test2:
 ; MMR2-SF:       # %bb.0: # %entry
-; MMR2-SF-NEXT:    addiusp -24 # <MCInst #{{[0-9]+}} ADDIUSP_MM
+; MMR2-SF-NEXT:    addiusp -24 # <MCInst #[[#MCINST14]] ADDIUSP_MM
 ; MMR2-SF-NEXT:    # <MCOperand Imm:-24>>
 ; MMR2-SF-NEXT:    .cfi_def_cfa_offset 24
 ; MMR2-SF-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
-; MMR2-SF-NEXT:    # <MCInst #{{[0-9]+}} SWSP_MM
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR2-SF-NEXT:    # <MCInst #[[#MCINST15]] SWSP_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR2-SF-NEXT:    # <MCOperand Imm:20>>
 ; MMR2-SF-NEXT:    .cfi_offset 31, -4
-; MMR2-SF-NEXT:    jal __fixdfsi # <MCInst #{{[0-9]+}} JAL_MM
-; MMR2-SF-NEXT:    # <MCOperand Expr:(__fixdfsi)>>
-; MMR2-SF-NEXT:    nop # <MCInst #{{[0-9]+}} SLL
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR2-SF-NEXT:    jal __fixdfsi # <MCInst #[[#MCINST16]] JAL_MM
+; MMR2-SF-NEXT:    # <MCOperand Expr:__fixdfsi>>
+; MMR2-SF-NEXT:    nop # <MCInst #[[#MCINST17]] SLL_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
 ; MMR2-SF-NEXT:    # <MCOperand Imm:0>>
 ; MMR2-SF-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
-; MMR2-SF-NEXT:    # <MCInst #{{[0-9]+}} LWSP_MM
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR2-SF-NEXT:    # <MCInst #[[#MCINST18]] LWSP_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR2-SF-NEXT:    # <MCOperand Imm:20>>
-; MMR2-SF-NEXT:    addiusp 24 # <MCInst #{{[0-9]+}} ADDIUSP_MM
+; MMR2-SF-NEXT:    addiusp 24 # <MCInst #[[#MCINST14]] ADDIUSP_MM
 ; MMR2-SF-NEXT:    # <MCOperand Imm:24>>
-; MMR2-SF-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR2-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR2-SF-NEXT:    jrc $ra # <MCInst #[[#MCINST19]] JRC16_MM
+; MMR2-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 ;
 ; MMR6-LABEL: test2:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    trunc.w.d $f0, $f12 # <MCInst #{{[0-9]+}} TRUNC_W_D_MMR6
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR6-NEXT:    mfc1 $2, $f0 # <MCInst #{{[0-9]+}} MFC1_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    trunc.w.d $f0, $f12 # <MCInst #[[#MCINST26:]] TRUNC_W_D_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>>
+; MMR6-NEXT:    mfc1 $2, $f0 # <MCInst #[[#MCINST13]] MFC1_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST19]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 ;
 ; MMR6-SF-LABEL: test2:
 ; MMR6-SF:       # %bb.0: # %entry
-; MMR6-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #{{[0-9]+}} ADDiu
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    addiu $sp, $sp, -24 # <MCInst #[[#MCINST4]] ADDiu
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:-24>>
 ; MMR6-SF-NEXT:    .cfi_def_cfa_offset 24
 ; MMR6-SF-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
-; MMR6-SF-NEXT:    # <MCInst #{{[0-9]+}} SW
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    # <MCInst #[[#MCINST5]] SW
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:20>>
 ; MMR6-SF-NEXT:    .cfi_offset 31, -4
-; MMR6-SF-NEXT:    balc __fixdfsi # <MCInst #{{[0-9]+}} BALC_MMR6
-; MMR6-SF-NEXT:    # <MCOperand Expr:(__fixdfsi)>>
+; MMR6-SF-NEXT:    balc __fixdfsi # <MCInst #[[#MCINST21]] BALC_MMR6
+; MMR6-SF-NEXT:    # <MCOperand Expr:__fixdfsi>>
 ; MMR6-SF-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
-; MMR6-SF-NEXT:    # <MCInst #{{[0-9]+}} LW
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    # <MCInst #[[#MCINST8]] LW
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:20>>
-; MMR6-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #{{[0-9]+}} ADDiu
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-SF-NEXT:    addiu $sp, $sp, 24 # <MCInst #[[#MCINST4]] ADDiu
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MMR6-SF-NEXT:    # <MCOperand Imm:24>>
-; MMR6-SF-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-SF-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-SF-NEXT:    jrc $ra # <MCInst #[[#MCINST19]] JRC16_MM
+; MMR6-SF-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %conv = fptosi double %t to i32
   ret i32 %conv
diff --git a/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/load.ll b/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/load.ll
index b96bdff227ca..ee858ac94aed 100644
--- a/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/load.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/load.ll
@@ -25,161 +25,161 @@
 define i8 @f1() {
 ; MIPS32-LABEL: f1:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MIPS32-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR3-LABEL: f1:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MMR3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS32R6-LABEL: f1:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7:]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MIPS32R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR6-LABEL: f1:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(a))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ;
 ; MIPS3-LABEL: f1:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>>
+; MIPS3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS64-LABEL: f1:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>>
+; MIPS64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS64R6-LABEL: f1:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12:]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>>
+; MIPS64R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR5FP64-LABEL: f1:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MMR5FP64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f1:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    lbu $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LBu
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MIPS32R5FP643-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 entry:
   %0 = load i8, ptr @a
   ret i8 %0
@@ -188,161 +188,161 @@ entry:
 define i32 @f2() {
 ; MIPS32-LABEL: f2:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR3-LABEL: f2:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS32R6-LABEL: f2:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR6-LABEL: f2:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR6-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(a))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f2:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS64-LABEL: f2:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS64R6-LABEL: f2:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR5FP64-LABEL: f2:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f2:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    lb $2, %lo(a)($1) # <MCInst #{{[0-9]+}} LB
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 entry:
   %0 = load i8, ptr @a
   %1 = sext i8 %0 to i32
@@ -352,161 +352,161 @@ entry:
 define i16 @f3() {
 ; MIPS32-LABEL: f3:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR3-LABEL: f3:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS32R6-LABEL: f3:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR6-LABEL: f3:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(b))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f3:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS64-LABEL: f3:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS64R6-LABEL: f3:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR5FP64-LABEL: f3:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f3:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    lhu $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LHu
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 entry:
   %0 = load i16, ptr @b
   ret i16 %0
@@ -515,161 +515,161 @@ entry:
 define i32 @f4() {
 ; MIPS32-LABEL: f4:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR3-LABEL: f4:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS32R6-LABEL: f4:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR6-LABEL: f4:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR6-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(b))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f4:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS64-LABEL: f4:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS64R6-LABEL: f4:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR5FP64-LABEL: f4:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f4:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    lh $2, %lo(b)($1) # <MCInst #{{[0-9]+}} LH
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 entry:
   %0 = load i16, ptr @b
   %1 = sext i16 %0 to i32
@@ -679,161 +679,161 @@ entry:
 define i32 @f5() {
 ; MIPS32-LABEL: f5:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR3-LABEL: f5:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS32R6-LABEL: f5:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR6-LABEL: f5:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR6-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f5:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS64-LABEL: f5:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS64R6-LABEL: f5:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR5FP64-LABEL: f5:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f5:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 entry:
   %0 = load i32, ptr @c
   ret i32 %0
@@ -842,181 +842,181 @@ entry:
 define i64 @f6() {
 ; MIPS32-LABEL: f6:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    addiu $2, $zero, 0 # <MCInst #{{[0-9]+}} ADDiu
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    addiu $2, $zero, 0 # <MCInst #[[#MCINST21:]] ADDiu
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
 ; MIPS32-NEXT:    # <MCOperand Imm:0>>
 ;
 ; MMR3-LABEL: f6:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR3-NEXT:    li16 $2, 0 # <MCInst #{{[0-9]+}} LI16_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Imm:0>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS32R6-LABEL: f6:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    addiu $2, $zero, 0 # <MCInst #{{[0-9]+}} ADDiu
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    addiu $2, $zero, 0 # <MCInst #[[#MCINST21:]] ADDiu
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Imm:0>>
 ;
 ; MMR6-LABEL: f6:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MMR6-NEXT:    li16 $2, 0 # <MCInst #{{[0-9]+}} LI16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Imm:0>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f6:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lwu $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LWu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS64-LABEL: f6:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lwu $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LWu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS64R6-LABEL: f6:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lwu $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LWu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR5FP64-LABEL: f6:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR5FP64-NEXT:    li16 $2, 0 # <MCInst #{{[0-9]+}} LI16_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Imm:0>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f6:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    addiu $2, $zero, 0 # <MCInst #{{[0-9]+}} ADDiu
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    addiu $2, $zero, 0 # <MCInst #[[#MCINST21:]] ADDiu
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Imm:0>>
 entry:
   %0 = load i32, ptr @c
@@ -1027,184 +1027,184 @@ entry:
 define i64 @f7() {
 ; MIPS32-LABEL: f7:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    sra $2, $3, 31 # <MCInst #{{[0-9]+}} SRA
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST24:]] SRA
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32-NEXT:    # <MCOperand Imm:31>>
 ;
 ; MMR3-LABEL: f7:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    sra $2, $3, 31 # <MCInst #{{[0-9]+}} SRA_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR3-NEXT:    # <MCOperand Imm:31>>
 ;
 ; MIPS32R6-LABEL: f7:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    sra $2, $3, 31 # <MCInst #{{[0-9]+}} SRA
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST24:]] SRA
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32R6-NEXT:    # <MCOperand Imm:31>>
 ;
 ; MMR6-LABEL: f7:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MMR6-NEXT:    sra $2, $3, 31 # <MCInst #{{[0-9]+}} SRA_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR6-NEXT:    # <MCOperand Imm:31>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f7:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS64-LABEL: f7:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS64R6-LABEL: f7:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #{{[0-9]+}} LW64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR5FP64-LABEL: f7:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    sra $2, $3, 31 # <MCInst #{{[0-9]+}} SRA_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR5FP64-NEXT:    # <MCOperand Imm:31>>
 ;
 ; MIPS32R5FP643-LABEL: f7:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #{{[0-9]+}} LW
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    sra $2, $3, 31 # <MCInst #{{[0-9]+}} SRA
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST24:]] SRA
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Imm:31>>
 entry:
   %0 = load i32, ptr @c
@@ -1215,161 +1215,161 @@ entry:
 define float @f8() {
 ; MIPS32-LABEL: f8:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MMR3-LABEL: f8:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MIPS32R6-LABEL: f8:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MMR6-LABEL: f8:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MMR6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(e))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f8:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(e) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(e))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(e))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MIPS64-LABEL: f8:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(e) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(e))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(e))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MIPS64R6-LABEL: f8:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(e))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(e))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MMR5FP64-LABEL: f8:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MIPS32R5FP643-LABEL: f8:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #{{[0-9]+}} LWC1
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 entry:
   %0 = load float, ptr @e
   ret float %0
@@ -1378,161 +1378,161 @@ entry:
 define double @f9() {
 ; MIPS32-LABEL: f9:
 ; MIPS32:       # %bb.0: # %entry
-; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC1
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST29:]] LDC1
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MMR3-LABEL: f9:
 ; MMR3:       # %bb.0: # %entry
-; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC1_MM_D32
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST30:]] LDC1_MM_D32
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MIPS32R6-LABEL: f9:
 ; MIPS32R6:       # %bb.0: # %entry
-; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC164
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MMR6-LABEL: f9:
 ; MMR6:       # %bb.0: # %entry
-; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MMR6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC1_D64_MMR6
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(f))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST32:]] LDC1_D64_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS3-LABEL: f9:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    lui $1, %highest(f) # <MCInst #{{[0-9]+}} LUi64
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%highest(f))>>
-; MIPS3-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%higher(f))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS3-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS3-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Imm:16>>
-; MIPS3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC164
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS3-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS3-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MIPS64-LABEL: f9:
 ; MIPS64:       # %bb.0: # %entry
-; MIPS64-NEXT:    lui $1, %highest(f) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%highest(f))>>
-; MIPS64-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%higher(f))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Imm:16>>
-; MIPS64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC164
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS64-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MIPS64R6-LABEL: f9:
 ; MIPS64R6:       # %bb.0: # %entry
-; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(f))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(f))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC164
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST12]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MMR5FP64-LABEL: f9:
 ; MMR5FP64:       # %bb.0: # %entry
-; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC1_MM_D64
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST33:]] LDC1_MM_D64
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MIPS32R5FP643-LABEL: f9:
 ; MIPS32R5FP643:       # %bb.0: # %entry
-; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #{{[0-9]+}} LDC164
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 entry:
   %0 = load double, ptr @f
   ret double %0
diff --git a/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/store.ll b/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/store.ll
index 3922db72f2a7..880a0f522574 100644
--- a/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/store.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Mips/llvm-ir/store.ll
@@ -24,133 +24,133 @@
 define void @f1(i8 %a) {
 ; MIPS32-LABEL: f1:
 ; MIPS32:       # %bb.0:
-; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MIPS32-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR3-LABEL: f1:
 ; MMR3:       # %bb.0:
-; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MMR3-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS32R6-LABEL: f1:
 ; MIPS32R6:       # %bb.0:
-; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7:]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MIPS32R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR6-LABEL: f1:
 ; MMR6:       # %bb.0:
-; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR6-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(a))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ;
 ; MIPS4-LABEL: f1:
 ; MIPS4:       # %bb.0:
-; MIPS4-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS4-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS4-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS4-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS4-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>>
+; MIPS4-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST12:]] SB64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS64R6-LABEL: f1:
 ; MIPS64R6:       # %bb.0:
-; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(a))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(a))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST13:]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>>
+; MIPS64R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST12:]] SB64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MMR5FP64-LABEL: f1:
 ; MMR5FP64:       # %bb.0:
-; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MMR5FP64-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f1:
 ; MIPS32R5FP643:       # %bb.0:
-; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(a))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    sb $4, %lo(a)($1) # <MCInst #{{[0-9]+}} SB
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(a))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
+; MIPS32R5FP643-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
   store i8 %a, ptr @a
   ret void
 }
@@ -158,133 +158,133 @@ define void @f1(i8 %a) {
 define void @f2(i16 %a) {
 ; MIPS32-LABEL: f2:
 ; MIPS32:       # %bb.0:
-; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR3-LABEL: f2:
 ; MMR3:       # %bb.0:
-; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS32R6-LABEL: f2:
 ; MIPS32R6:       # %bb.0:
-; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR6-LABEL: f2:
 ; MMR6:       # %bb.0:
-; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR6-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(b))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS4-LABEL: f2:
 ; MIPS4:       # %bb.0:
-; MIPS4-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS4-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS4-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS4-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS4-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS4-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST16:]] SH64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS64R6-LABEL: f2:
 ; MIPS64R6:       # %bb.0:
-; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(b))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(b))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST13]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST16:]] SH64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MMR5FP64-LABEL: f2:
 ; MMR5FP64:       # %bb.0:
-; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f2:
 ; MIPS32R5FP643:       # %bb.0:
-; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(b))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    sh $4, %lo(b)($1) # <MCInst #{{[0-9]+}} SH
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(b))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
   store i16 %a, ptr @b
   ret void
 }
@@ -292,133 +292,133 @@ define void @f2(i16 %a) {
 define void @f3(i32 %a) {
 ; MIPS32-LABEL: f3:
 ; MIPS32:       # %bb.0:
-; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR3-LABEL: f3:
 ; MMR3:       # %bb.0:
-; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS32R6-LABEL: f3:
 ; MIPS32R6:       # %bb.0:
-; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR6-LABEL: f3:
 ; MMR6:       # %bb.0:
-; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR6-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(c))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS4-LABEL: f3:
 ; MIPS4:       # %bb.0:
-; MIPS4-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS4-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS4-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS4-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS4-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS4-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST19:]] SW64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS64R6-LABEL: f3:
 ; MIPS64R6:       # %bb.0:
-; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(c))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST13]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST19:]] SW64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MMR5FP64-LABEL: f3:
 ; MMR5FP64:       # %bb.0:
-; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f3:
 ; MIPS32R5FP643:       # %bb.0:
-; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(c))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    sw $4, %lo(c)($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(c))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
   store i32 %a, ptr @c
   ret void
 }
@@ -426,180 +426,180 @@ define void @f3(i32 %a) {
 define void @f4(i64 %a) {
 ; MIPS32-LABEL: f4:
 ; MIPS32:       # %bb.0:
-; MIPS32-NEXT:    lui $1, %hi(d) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MIPS32-NEXT:    sw $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MIPS32-NEXT:    addiu $1, $1, %lo(d) # <MCInst #{{[0-9]+}} ADDiu
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    sw $5, 4($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    sw $5, 4($1) # <MCInst #[[#MCINST17]] SW
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32-NEXT:    # <MCOperand Imm:4>>
 ;
 ; MMR3-LABEL: f4:
 ; MMR3:       # %bb.0:
-; MMR3-NEXT:    lui $1, %hi(d) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MMR3-NEXT:    sw $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SW_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MMR3-NEXT:    addiu $2, $1, %lo(d) # <MCInst #{{[0-9]+}} ADDiu_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MMR3-NEXT:    sw16 $5, 4($2) # <MCInst #{{[0-9]+}} SW16_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR3-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR3-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR3-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR3-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
 ; MMR3-NEXT:    # <MCOperand Imm:4>>
-; MMR3-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR3-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS32R6-LABEL: f4:
 ; MIPS32R6:       # %bb.0:
-; MIPS32R6-NEXT:    lui $1, %hi(d) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MIPS32R6-NEXT:    sw $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MIPS32R6-NEXT:    addiu $1, $1, %lo(d) # <MCInst #{{[0-9]+}} ADDiu
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    sw $5, 4($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32R6-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32R6-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R6-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    sw $5, 4($1) # <MCInst #[[#MCINST17]] SW
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32R6-NEXT:    # <MCOperand Imm:4>>
 ;
 ; MMR6-LABEL: f4:
 ; MMR6:       # %bb.0:
-; MMR6-NEXT:    lui $1, %hi(d) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MMR6-NEXT:    sw $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SW_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MMR6-NEXT:    addiu $2, $1, %lo(d) # <MCInst #{{[0-9]+}} ADDiu_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MMR6-NEXT:    sw16 $5, 4($2) # <MCInst #{{[0-9]+}} SW16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR6-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR6-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR6-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR6-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
 ; MMR6-NEXT:    # <MCOperand Imm:4>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS4-LABEL: f4:
 ; MIPS4:       # %bb.0:
-; MIPS4-NEXT:    lui $1, %highest(d) # <MCInst #{{[0-9]+}} LUi64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%highest(d))>>
-; MIPS4-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%higher(d))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    lui $1, %highest(d) # <MCInst #[[#MCINST9]] LUi64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,d)>>
+; MIPS4-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,d)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS4-NEXT:    sd $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SD
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%lo(d))>>
+; MIPS4-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS4-NEXT:    sd $4, %lo(d)($1) # <MCInst #[[#MCINST23:]] SD
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
 ;
 ; MIPS64R6-LABEL: f4:
 ; MIPS64R6:       # %bb.0:
-; MIPS64R6-NEXT:    lui $1, %highest(d) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(d))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(d))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(d) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,d)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,d)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    sd $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SD
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(d))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST13]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    sd $4, %lo(d)($1) # <MCInst #[[#MCINST23:]] SD
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
 ;
 ; MMR5FP64-LABEL: f4:
 ; MMR5FP64:       # %bb.0:
-; MMR5FP64-NEXT:    lui $1, %hi(d) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MMR5FP64-NEXT:    sw $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SW_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MMR5FP64-NEXT:    addiu $2, $1, %lo(d) # <MCInst #{{[0-9]+}} ADDiu_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MMR5FP64-NEXT:    sw16 $5, 4($2) # <MCInst #{{[0-9]+}} SW16_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MMR5FP64-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR5FP64-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR5FP64-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR5FP64-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
 ; MMR5FP64-NEXT:    # <MCOperand Imm:4>>
-; MMR5FP64-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR5FP64-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS32R5FP643-LABEL: f4:
 ; MIPS32R5FP643:       # %bb.0:
-; MIPS32R5FP643-NEXT:    lui $1, %hi(d) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(d))>>
-; MIPS32R5FP643-NEXT:    sw $4, %lo(d)($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MIPS32R5FP643-NEXT:    addiu $1, $1, %lo(d) # <MCInst #{{[0-9]+}} ADDiu
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(d))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    sw $5, 4($1) # <MCInst #{{[0-9]+}} SW
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32R5FP643-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R5FP643-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    sw $5, 4($1) # <MCInst #[[#MCINST17]] SW
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Imm:4>>
   store i64 %a, ptr @d
   ret void
@@ -608,133 +608,133 @@ define void @f4(i64 %a) {
 define void @f5(float %e) {
 ; MIPS32-LABEL: f5:
 ; MIPS32:       # %bb.0:
-; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MMR3-LABEL: f5:
 ; MMR3:       # %bb.0:
-; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MIPS32R6-LABEL: f5:
 ; MIPS32R6:       # %bb.0:
-; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MMR6-LABEL: f5:
 ; MMR6:       # %bb.0:
-; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MMR6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(e))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS4-LABEL: f5:
 ; MIPS4:       # %bb.0:
-; MIPS4-NEXT:    lui $1, %highest(e) # <MCInst #{{[0-9]+}} LUi64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%highest(e))>>
-; MIPS4-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%higher(e))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS4-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS4-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS4-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS4-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MIPS64R6-LABEL: f5:
 ; MIPS64R6:       # %bb.0:
-; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(e))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(e))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST13]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MMR5FP64-LABEL: f5:
 ; MMR5FP64:       # %bb.0:
-; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
 ;
 ; MIPS32R5FP643-LABEL: f5:
 ; MIPS32R5FP643:       # %bb.0:
-; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(e))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #{{[0-9]+}} SWC1
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(e))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
   store float %e, ptr @e
   ret void
 }
@@ -742,133 +742,133 @@ define void @f5(float %e) {
 define void @f6(double %f) {
 ; MIPS32-LABEL: f6:
 ; MIPS32:       # %bb.0:
-; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS32-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC1
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST26:]] SDC1
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MMR3-LABEL: f6:
 ; MMR3:       # %bb.0:
-; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MMR3-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR3-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC1_MM_D32
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR3-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR3-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST27:]] SDC1_MM_D32
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
+; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MIPS32R6-LABEL: f6:
 ; MIPS32R6:       # %bb.0:
-; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS32R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC164
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R6-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
+; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MMR6-LABEL: f6:
 ; MMR6:       # %bb.0:
-; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MMR6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC1_D64_MMR6
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR6-NEXT:    # <MCOperand Expr:(%lo(f))>>
-; MMR6-NEXT:    jrc $ra # <MCInst #{{[0-9]+}} JRC16_MM
-; MMR6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
+; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST29:]] SDC1_D64_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
 ; MIPS4-LABEL: f6:
 ; MIPS4:       # %bb.0:
-; MIPS4-NEXT:    lui $1, %highest(f) # <MCInst #{{[0-9]+}} LUi64
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%highest(f))>>
-; MIPS4-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%higher(f))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS4-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS4-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Imm:16>>
-; MIPS4-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS4-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC164
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS4-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS4-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS4-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
+; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MIPS64R6-LABEL: f6:
 ; MIPS64R6:       # %bb.0:
-; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #{{[0-9]+}} LUi64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%highest(f))>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%higher(f))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #{{[0-9]+}} DADDiu
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #{{[0-9]+}} DSLL
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
+; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Imm:16>>
-; MIPS64R6-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JALR64
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS64R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC164
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS64R6-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS64R6-NEXT:    jr $ra # <MCInst #[[#MCINST13]] JALR64
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>>
+; MIPS64R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
+; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MMR5FP64-LABEL: f6:
 ; MMR5FP64:       # %bb.0:
-; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MMR5FP64-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR_MM
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MMR5FP64-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC1_MM_D64
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MMR5FP64-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR5FP64-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST30:]] SDC1_MM_D64
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
+; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
 ;
 ; MIPS32R5FP643-LABEL: f6:
 ; MIPS32R5FP643:       # %bb.0:
-; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #{{[0-9]+}} LUi
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%hi(f))>>
-; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #{{[0-9]+}} JR
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>>
-; MIPS32R5FP643-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #{{[0-9]+}} SDC164
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Reg:{{[0-9]+}}>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:(%lo(f))>>
+; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MIPS32R5FP643-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
   store double %f, ptr @f
   ret void
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/Mips/mips16ex.ll b/external/llvm-project/llvm/test/CodeGen/Mips/mips16ex.ll
index fb9a44e76751..f4d1125718a9 100644
--- a/external/llvm-project/llvm/test/CodeGen/Mips/mips16ex.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Mips/mips16ex.ll
@@ -2,7 +2,7 @@
 
 ;16: main:
 ;16-NEXT: [[TMP:.*]]:
-;16-NEXT: .set $func_begin0, [[TMP]]
+;16-NEXT: $func_begin0 = [[TMP]]
 ;16-NEXT: .cfi_startproc
 ;16-NEXT: .cfi_personality
 @.str = private unnamed_addr constant [7 x i8] c"hello\0A\00", align 1
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-1cta.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-1cta.ll
new file mode 100644
index 000000000000..5cfa25dfe55f
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-1cta.ll
@@ -0,0 +1,435 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86| FileCheck --check-prefixes=CHECK-PTX64 %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr| FileCheck --check-prefixes=CHECK-PTX-SHARED32 %s
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86| %ptxas-verify -arch=sm_100a %}
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr| %ptxas-verify -arch=sm_100a %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_1d
+define void @test_cp_async_bulk_tensor_g2s_tile_1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_1d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<2>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_1d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_1d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_1d_param_5];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_1d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<4>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_1d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_1d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_1d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_2d
+define void @test_cp_async_bulk_tensor_g2s_tile_2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_2d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<3>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_2d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_2d_param_6];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1, %r2}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1, %r2}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_2d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<5>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_2d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_2d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3, %r4}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3, %r4}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_3d
+define void @test_cp_async_bulk_tensor_g2s_tile_3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_3d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<4>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_3d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_3d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_3d_param_7];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_3d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_3d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_3d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_3d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_4d
+define void @test_cp_async_bulk_tensor_g2s_tile_4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_4d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<5>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_4d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_4d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_4d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_4d_param_8];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_4d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<7>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_4d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_4d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_4d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_tile_4d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_5d
+define void @test_cp_async_bulk_tensor_g2s_tile_5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_5d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_5d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_5d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_5d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_5d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_8];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_5d_param_9];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_5d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<8>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_5d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_5d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_5d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_tile_5d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r7, [test_cp_async_bulk_tensor_g2s_tile_5d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_9];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_im2col_3d
+define void @test_cp_async_bulk_tensor_g2s_im2col_3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_im2col_3d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<3>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<4>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_8];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1}, %rs2, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1}, %rs2;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1}, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1};
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_im2col_3d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<3>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rs2, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rs2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1};
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_im2col_4d
+define void @test_cp_async_bulk_tensor_g2s_im2col_4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_im2col_4d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<4>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<5>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_8];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_9];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_10];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2}, %rs3, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2}, %rs3;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2}, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2};
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_im2col_4d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<4>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<7>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_9];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_10];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rs3, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rs3;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_im2col_5d
+define void @test_cp_async_bulk_tensor_g2s_im2col_5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_im2col_5d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<5>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_8];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_9];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_10];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_11];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_12];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3}, %rs4, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3}, %rs4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3}, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::1 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3};
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_im2col_5d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<5>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<8>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r7, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_9];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_10];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_11];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_12];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rs4, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rs4;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::1 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3};
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 1, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 1, i1 0, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 0, i1 1, i32 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 0, i1 0, i32 1)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-2cta.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-2cta.ll
new file mode 100644
index 000000000000..a7e6bec6aef1
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-2cta.ll
@@ -0,0 +1,435 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86| FileCheck --check-prefixes=CHECK-PTX64 %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr| FileCheck --check-prefixes=CHECK-PTX-SHARED32 %s
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86| %ptxas-verify -arch=sm_100a %}
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx86 --nvptx-short-ptr| %ptxas-verify -arch=sm_100a %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 %f1, i1 %f2, i32 %f3);
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_1d
+define void @test_cp_async_bulk_tensor_g2s_tile_1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_1d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<2>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_1d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_1d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_1d_param_5];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_1d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<4>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_1d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_1d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_1d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_1d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_2d
+define void @test_cp_async_bulk_tensor_g2s_tile_2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_2d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<3>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_2d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_2d_param_6];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1, %r2}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1, %r2}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_2d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<5>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_2d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_2d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_2d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_2d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3, %r4}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3, %r4}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_3d
+define void @test_cp_async_bulk_tensor_g2s_tile_3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_3d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<4>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_3d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_3d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_3d_param_7];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_3d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_3d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_3d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_3d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_3d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_3d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_4d
+define void @test_cp_async_bulk_tensor_g2s_tile_4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_4d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<5>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_4d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_4d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_4d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_4d_param_8];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_4d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<7>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_4d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_4d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_4d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_tile_4d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_4d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_4d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_tile_5d
+define void @test_cp_async_bulk_tensor_g2s_tile_5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_tile_5d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_tile_5d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_5d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_5d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_5d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_8];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_tile_5d_param_9];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], %rs1, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], %rs1;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2];
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_tile_5d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<8>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_tile_5d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_tile_5d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_tile_5d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_tile_5d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r7, [test_cp_async_bulk_tensor_g2s_tile_5d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_tile_5d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_tile_5d_param_9];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rs1, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rs1;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2];
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_im2col_3d
+define void @test_cp_async_bulk_tensor_g2s_im2col_3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_im2col_3d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<3>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<4>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_8];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1}, %rs2, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1}, %rs2;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1}, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1};
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_im2col_3d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<3>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_3d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rs2, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rs2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1};
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_im2col_4d
+define void @test_cp_async_bulk_tensor_g2s_im2col_4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_im2col_4d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<4>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<5>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_8];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_9];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_10];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2}, %rs3, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2}, %rs3;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2}, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs1, %rs2};
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_im2col_4d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<4>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<7>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_9];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_4d_param_10];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rs3, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rs3;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
+
+; CHECK-LABEL: test_cp_async_bulk_tensor_g2s_im2col_5d
+define void @test_cp_async_bulk_tensor_g2s_im2col_5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch) {
+; CHECK-PTX64-LABEL: test_cp_async_bulk_tensor_g2s_im2col_5d(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b16 %rs<5>;
+; CHECK-PTX64-NEXT:    .reg .b32 %r<6>;
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_0];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_1];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_2];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_3];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_4];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_5];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_6];
+; CHECK-PTX64-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_7];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_8];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_9];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_10];
+; CHECK-PTX64-NEXT:    ld.param.b16 %rs4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_11];
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_12];
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3}, %rs4, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3}, %rs4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3}, %rd4;
+; CHECK-PTX64-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::2 [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2], {%rs1, %rs2, %rs3};
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: test_cp_async_bulk_tensor_g2s_im2col_5d(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b16 %rs<5>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<8>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_1];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_2];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_3];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_4];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r5, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_5];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r6, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_6];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b32 %r7, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_7];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs1, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_8];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_9];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs3, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_10];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b16 %rs4, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_11];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [test_cp_async_bulk_tensor_g2s_im2col_5d_param_12];
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rs4, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rs4;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.L2::cache_hint.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rd2;
+; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.cta_group::2 [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3};
+; CHECK-PTX-SHARED32-NEXT:    ret;
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 0, i1 1, i32 2)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 0, i1 0, i32 2)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-invalid.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-invalid.ll
new file mode 100644
index 000000000000..1c35fbead389
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-invalid.ll
@@ -0,0 +1,15 @@
+; RUN: not llc < %s -mtriple=nvptx64 -mcpu=sm_100a -o /dev/null 2>&1 | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) writeonly, ptr addrspace(3), ptr readonly, i32, i16, i64, i1 immarg, i1 immarg, i32 immarg range(i32 0, 3))
+
+define void @test_cp_async_bulk_tensor_g2s_tile_1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch) {
+  ; CHECK: immarg value 3 out of range [0, 3)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 0, i32 3)
+
+  ; CHECK: immarg value -1 out of range [0, 3)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 0, i32 -1)
+
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/fast-math.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/fast-math.ll
index 4cb6a35e796f..bc48d242f88f 100644
--- a/external/llvm-project/llvm/test/CodeGen/NVPTX/fast-math.ll
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/fast-math.ll
@@ -1,58 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
 declare float @llvm.sqrt.f32(float)
 declare double @llvm.sqrt.f64(double)
 
-; CHECK-LABEL: sqrt_div(
-; CHECK: sqrt.rn.f32
-; CHECK: div.rn.f32
 define float @sqrt_div(float %a, float %b) {
+; CHECK-LABEL: sqrt_div(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_param_0];
+; CHECK-NEXT:    sqrt.rn.f32 %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [sqrt_div_param_1];
+; CHECK-NEXT:    div.rn.f32 %r4, %r2, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
   %t1 = tail call float @llvm.sqrt.f32(float %a)
   %t2 = fdiv float %t1, %b
   ret float %t2
 }
 
-; CHECK-LABEL: sqrt_div_fast(
-; CHECK: sqrt.rn.f32
-; CHECK: div.approx.f32
 define float @sqrt_div_fast(float %a, float %b) #0 {
+; CHECK-LABEL: sqrt_div_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_fast_param_0];
+; CHECK-NEXT:    sqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [sqrt_div_fast_param_1];
+; CHECK-NEXT:    div.approx.f32 %r4, %r2, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
   %t1 = tail call float @llvm.sqrt.f32(float %a)
   %t2 = fdiv float %t1, %b
   ret float %t2
 }
 
-; CHECK-LABEL: sqrt_div_fast_ninf(
-; CHECK: sqrt.approx.f32
-; CHECK: div.approx.f32
 define float @sqrt_div_fast_ninf(float %a, float %b) #0 {
+; CHECK-LABEL: sqrt_div_fast_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_fast_ninf_param_0];
+; CHECK-NEXT:    sqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    abs.f32 %r3, %r1;
+; CHECK-NEXT:    setp.lt.f32 %p1, %r3, 0f00800000;
+; CHECK-NEXT:    selp.f32 %r4, 0f00000000, %r2, %p1;
+; CHECK-NEXT:    ld.param.b32 %r5, [sqrt_div_fast_ninf_param_1];
+; CHECK-NEXT:    div.approx.f32 %r6, %r4, %r5;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
   %t1 = tail call ninf afn float @llvm.sqrt.f32(float %a)
   %t2 = fdiv float %t1, %b
   ret float %t2
 }
 
-; CHECK-LABEL: sqrt_div_ftz(
-; CHECK: sqrt.rn.ftz.f32
-; CHECK: div.rn.ftz.f32
 define float @sqrt_div_ftz(float %a, float %b) #1 {
+; CHECK-LABEL: sqrt_div_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_ftz_param_0];
+; CHECK-NEXT:    sqrt.rn.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [sqrt_div_ftz_param_1];
+; CHECK-NEXT:    div.rn.ftz.f32 %r4, %r2, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
   %t1 = tail call float @llvm.sqrt.f32(float %a)
   %t2 = fdiv float %t1, %b
   ret float %t2
 }
 
-; CHECK-LABEL: sqrt_div_fast_ftz(
-; CHECK: sqrt.rn.ftz.f32
-; CHECK: div.approx.ftz.f32
 define float @sqrt_div_fast_ftz(float %a, float %b) #0 #1 {
+; CHECK-LABEL: sqrt_div_fast_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_fast_ftz_param_0];
+; CHECK-NEXT:    sqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [sqrt_div_fast_ftz_param_1];
+; CHECK-NEXT:    div.approx.ftz.f32 %r4, %r2, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
   %t1 = tail call float @llvm.sqrt.f32(float %a)
   %t2 = fdiv float %t1, %b
   ret float %t2
 }
 
-; CHECK-LABEL: sqrt_div_fast_ftz_ninf(
-; CHECK: sqrt.approx.ftz.f32
-; CHECK: div.approx.ftz.f32
 define float @sqrt_div_fast_ftz_ninf(float %a, float %b) #0 #1 {
+; CHECK-LABEL: sqrt_div_fast_ftz_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_fast_ftz_ninf_param_0];
+; CHECK-NEXT:    setp.eq.ftz.f32 %p1, %r1, 0f00000000;
+; CHECK-NEXT:    sqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    selp.f32 %r3, 0f00000000, %r2, %p1;
+; CHECK-NEXT:    ld.param.b32 %r4, [sqrt_div_fast_ftz_ninf_param_1];
+; CHECK-NEXT:    div.approx.ftz.f32 %r5, %r3, %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ret;
   %t1 = tail call ninf afn float @llvm.sqrt.f32(float %a)
   %t2 = fdiv float %t1, %b
   ret float %t2
@@ -61,69 +117,117 @@ define float @sqrt_div_fast_ftz_ninf(float %a, float %b) #0 #1 {
 ; There are no fast-math or ftz versions of sqrt and div for f64.  We use
 ; reciprocal(rsqrt(x)) for sqrt(x), and emit a vanilla divide.
 
-; CHECK-LABEL: sqrt_div_fast_ftz_f64(
-; CHECK: sqrt.rn.f64
-; CHECK: div.rn.f64
 define double @sqrt_div_fast_ftz_f64(double %a, double %b) #0 #1 {
+; CHECK-LABEL: sqrt_div_fast_ftz_f64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [sqrt_div_fast_ftz_f64_param_0];
+; CHECK-NEXT:    sqrt.rn.f64 %rd2, %rd1;
+; CHECK-NEXT:    ld.param.b64 %rd3, [sqrt_div_fast_ftz_f64_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd4, %rd2, %rd3;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd4;
+; CHECK-NEXT:    ret;
   %t1 = tail call double @llvm.sqrt.f64(double %a)
   %t2 = fdiv double %t1, %b
   ret double %t2
 }
 
-; CHECK-LABEL: sqrt_div_fast_ftz_f64_ninf(
-; CHECK: rsqrt.approx.f64
-; CHECK: rcp.approx.ftz.f64
-; CHECK: div.rn.f64
 define double @sqrt_div_fast_ftz_f64_ninf(double %a, double %b) #0 #1 {
+; CHECK-LABEL: sqrt_div_fast_ftz_f64_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [sqrt_div_fast_ftz_f64_ninf_param_0];
+; CHECK-NEXT:    abs.f64 %rd2, %rd1;
+; CHECK-NEXT:    setp.lt.f64 %p1, %rd2, 0d0010000000000000;
+; CHECK-NEXT:    rsqrt.approx.f64 %rd3, %rd1;
+; CHECK-NEXT:    rcp.approx.ftz.f64 %rd4, %rd3;
+; CHECK-NEXT:    selp.f64 %rd5, 0d0000000000000000, %rd4, %p1;
+; CHECK-NEXT:    ld.param.b64 %rd6, [sqrt_div_fast_ftz_f64_ninf_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd7, %rd5, %rd6;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd7;
+; CHECK-NEXT:    ret;
   %t1 = tail call ninf afn double @llvm.sqrt.f64(double %a)
   %t2 = fdiv double %t1, %b
   ret double %t2
 }
 
-; CHECK-LABEL: rsqrt(
-; CHECK-NOT: rsqrt.approx
-; CHECK: sqrt.rn.f32
-; CHECK-NOT: rsqrt.approx
 define float @rsqrt(float %a) {
+; CHECK-LABEL: rsqrt(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [rsqrt_param_0];
+; CHECK-NEXT:    sqrt.rn.f32 %r2, %r1;
+; CHECK-NEXT:    rcp.rn.f32 %r3, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
   %b = tail call float @llvm.sqrt.f32(float %a)
   %ret = fdiv float 1.0, %b
   ret float %ret
 }
 
-; CHECK-LABEL: rsqrt_fast(
-; CHECK-NOT: div.
-; CHECK-NOT: sqrt.
-; CHECK: rsqrt.approx.f32
-; CHECK-NOT: div.
-; CHECK-NOT: sqrt.
 define float @rsqrt_fast(float %a) #0 {
+; CHECK-LABEL: rsqrt_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [rsqrt_fast_param_0];
+; CHECK-NEXT:    rsqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %b = tail call float @llvm.sqrt.f32(float %a)
   %ret = fdiv float 1.0, %b
   ret float %ret
 }
 
-; CHECK-LABEL: rsqrt_fast_ftz(
-; CHECK-NOT: div.
-; CHECK-NOT: sqrt.
-; CHECK: rsqrt.approx.ftz.f32
-; CHECK-NOT: div.
-; CHECK-NOT: sqrt.
 define float @rsqrt_fast_ftz(float %a) #0 #1 {
+; CHECK-LABEL: rsqrt_fast_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [rsqrt_fast_ftz_param_0];
+; CHECK-NEXT:    rsqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %b = tail call float @llvm.sqrt.f32(float %a)
   %ret = fdiv float 1.0, %b
   ret float %ret
 }
 
-; CHECK-LABEL: fadd
-; CHECK: add.rn.f32
 define float @fadd(float %a, float %b) {
+; CHECK-LABEL: fadd(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fadd_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [fadd_param_1];
+; CHECK-NEXT:    add.rn.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
   %t1 = fadd float %a, %b
   ret float %t1
 }
 
-; CHECK-LABEL: fadd_ftz
-; CHECK: add.rn.ftz.f32
 define float @fadd_ftz(float %a, float %b) #1 {
+; CHECK-LABEL: fadd_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fadd_ftz_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [fadd_ftz_param_1];
+; CHECK-NEXT:    add.rn.ftz.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
   %t1 = fadd float %a, %b
   ret float %t1
 }
@@ -131,41 +235,83 @@ define float @fadd_ftz(float %a, float %b) #1 {
 declare float @llvm.sin.f32(float)
 declare float @llvm.cos.f32(float)
 
-; CHECK-LABEL: fsin_approx_afn
-; CHECK:       sin.approx.f32
 define float @fsin_approx_afn(float %a) {
+; CHECK-LABEL: fsin_approx_afn(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fsin_approx_afn_param_0];
+; CHECK-NEXT:    sin.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %r = tail call afn float @llvm.sin.f32(float %a)
   ret float %r
 }
 
-; CHECK-LABEL: fcos_approx_afn
-; CHECK:       cos.approx.f32
 define float @fcos_approx_afn(float %a) {
+; CHECK-LABEL: fcos_approx_afn(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fcos_approx_afn_param_0];
+; CHECK-NEXT:    cos.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %r = tail call afn float @llvm.cos.f32(float %a)
   ret float %r
 }
 
-; CHECK-LABEL: fsin_approx
-; CHECK:       sin.approx.f32
 define float @fsin_approx(float %a) #0 {
+; CHECK-LABEL: fsin_approx(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fsin_approx_param_0];
+; CHECK-NEXT:    sin.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %r = tail call float @llvm.sin.f32(float %a)
   ret float %r
 }
 
-; CHECK-LABEL: fcos_approx
-; CHECK:       cos.approx.f32
 define float @fcos_approx(float %a) #0 {
+; CHECK-LABEL: fcos_approx(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [fcos_approx_param_0];
+; CHECK-NEXT:    cos.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %r = tail call float @llvm.cos.f32(float %a)
   ret float %r
 }
 
-; CHECK-LABEL: repeated_div_recip_allowed
 define float @repeated_div_recip_allowed(i1 %pred, float %a, float %b, float %divisor) {
-; CHECK: rcp.rn.f32
-; CHECK: mul.rn.f32
-; CHECK: mul.rn.f32
-; CHECK: mul.rn.f32
-; CHECK: selp.f32
+; CHECK-LABEL: repeated_div_recip_allowed(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_recip_allowed_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_recip_allowed_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_recip_allowed_param_3];
+; CHECK-NEXT:    rcp.rn.f32 %r3, %r2;
+; CHECK-NEXT:    mul.rn.f32 %r4, %r1, %r3;
+; CHECK-NEXT:    ld.param.b32 %r5, [repeated_div_recip_allowed_param_2];
+; CHECK-NEXT:    mul.rn.f32 %r6, %r5, %r3;
+; CHECK-NEXT:    mul.rn.f32 %r7, %r4, %r6;
+; CHECK-NEXT:    selp.f32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %x = fdiv arcp float %a, %divisor
   %y = fdiv arcp float %b, %divisor
   %z = fmul float %x, %y
@@ -173,23 +319,51 @@ define float @repeated_div_recip_allowed(i1 %pred, float %a, float %b, float %di
   ret float %w
 }
 
-; CHECK-LABEL: repeated_div_recip_allowed_sel
 define float @repeated_div_recip_allowed_sel(i1 %pred, float %a, float %b, float %divisor) {
-; CHECK: selp.f32
-; CHECK: div.rn.f32
+; CHECK-LABEL: repeated_div_recip_allowed_sel(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_recip_allowed_sel_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_recip_allowed_sel_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_recip_allowed_sel_param_2];
+; CHECK-NEXT:    selp.f32 %r3, %r1, %r2, %p1;
+; CHECK-NEXT:    ld.param.b32 %r4, [repeated_div_recip_allowed_sel_param_3];
+; CHECK-NEXT:    div.rn.f32 %r5, %r3, %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ret;
   %x = fdiv arcp float %a, %divisor
   %y = fdiv arcp float %b, %divisor
   %w = select i1 %pred, float %x, float %y
   ret float %w
 }
 
-; CHECK-LABEL: repeated_div_recip_allowed_ftz
 define float @repeated_div_recip_allowed_ftz(i1 %pred, float %a, float %b, float %divisor) #1 {
-; CHECK: rcp.rn.ftz.f32
-; CHECK: mul.rn.ftz.f32
-; CHECK: mul.rn.ftz.f32
-; CHECK: mul.rn.ftz.f32
-; CHECK: selp.f32
+; CHECK-LABEL: repeated_div_recip_allowed_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_recip_allowed_ftz_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_recip_allowed_ftz_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_recip_allowed_ftz_param_3];
+; CHECK-NEXT:    rcp.rn.ftz.f32 %r3, %r2;
+; CHECK-NEXT:    mul.rn.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT:    ld.param.b32 %r5, [repeated_div_recip_allowed_ftz_param_2];
+; CHECK-NEXT:    mul.rn.ftz.f32 %r6, %r5, %r3;
+; CHECK-NEXT:    mul.rn.ftz.f32 %r7, %r4, %r6;
+; CHECK-NEXT:    selp.f32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %x = fdiv arcp float %a, %divisor
   %y = fdiv arcp float %b, %divisor
   %z = fmul float %x, %y
@@ -197,23 +371,51 @@ define float @repeated_div_recip_allowed_ftz(i1 %pred, float %a, float %b, float
   ret float %w
 }
 
-; CHECK-LABEL: repeated_div_recip_allowed_ftz_sel
 define float @repeated_div_recip_allowed_ftz_sel(i1 %pred, float %a, float %b, float %divisor) #1 {
-; CHECK: selp.f32
-; CHECK: div.rn.ftz.f32
+; CHECK-LABEL: repeated_div_recip_allowed_ftz_sel(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_recip_allowed_ftz_sel_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_recip_allowed_ftz_sel_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_recip_allowed_ftz_sel_param_2];
+; CHECK-NEXT:    selp.f32 %r3, %r1, %r2, %p1;
+; CHECK-NEXT:    ld.param.b32 %r4, [repeated_div_recip_allowed_ftz_sel_param_3];
+; CHECK-NEXT:    div.rn.ftz.f32 %r5, %r3, %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ret;
   %x = fdiv arcp float %a, %divisor
   %y = fdiv arcp float %b, %divisor
   %w = select i1 %pred, float %x, float %y
   ret float %w
 }
 
-; CHECK-LABEL: repeated_div_fast
 define float @repeated_div_fast(i1 %pred, float %a, float %b, float %divisor) #0 {
-; CHECK: rcp.approx.f32
-; CHECK: mul.f32
-; CHECK: mul.f32
-; CHECK: mul.f32
-; CHECK: selp.f32
+; CHECK-LABEL: repeated_div_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_fast_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_fast_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_fast_param_3];
+; CHECK-NEXT:    rcp.approx.f32 %r3, %r2;
+; CHECK-NEXT:    mul.f32 %r4, %r1, %r3;
+; CHECK-NEXT:    ld.param.b32 %r5, [repeated_div_fast_param_2];
+; CHECK-NEXT:    mul.f32 %r6, %r5, %r3;
+; CHECK-NEXT:    mul.f32 %r7, %r4, %r6;
+; CHECK-NEXT:    selp.f32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %x = fdiv float %a, %divisor
   %y = fdiv float %b, %divisor
   %z = fmul float %x, %y
@@ -221,23 +423,51 @@ define float @repeated_div_fast(i1 %pred, float %a, float %b, float %divisor) #0
   ret float %w
 }
 
-; CHECK-LABEL: repeated_div_fast_sel
 define float @repeated_div_fast_sel(i1 %pred, float %a, float %b, float %divisor) #0 {
-; CHECK: selp.f32
-; CHECK: div.approx.f32
+; CHECK-LABEL: repeated_div_fast_sel(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_fast_sel_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_fast_sel_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_fast_sel_param_2];
+; CHECK-NEXT:    selp.f32 %r3, %r1, %r2, %p1;
+; CHECK-NEXT:    ld.param.b32 %r4, [repeated_div_fast_sel_param_3];
+; CHECK-NEXT:    div.approx.f32 %r5, %r3, %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ret;
   %x = fdiv float %a, %divisor
   %y = fdiv float %b, %divisor
   %w = select i1 %pred, float %x, float %y
   ret float %w
 }
 
-; CHECK-LABEL: repeated_div_fast_ftz
 define float @repeated_div_fast_ftz(i1 %pred, float %a, float %b, float %divisor) #0 #1 {
-; CHECK: rcp.approx.ftz.f32
-; CHECK: mul.ftz.f32
-; CHECK: mul.ftz.f32
-; CHECK: mul.ftz.f32
-; CHECK: selp.f32
+; CHECK-LABEL: repeated_div_fast_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_fast_ftz_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_fast_ftz_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_fast_ftz_param_3];
+; CHECK-NEXT:    rcp.approx.ftz.f32 %r3, %r2;
+; CHECK-NEXT:    mul.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT:    ld.param.b32 %r5, [repeated_div_fast_ftz_param_2];
+; CHECK-NEXT:    mul.ftz.f32 %r6, %r5, %r3;
+; CHECK-NEXT:    mul.ftz.f32 %r7, %r4, %r6;
+; CHECK-NEXT:    selp.f32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %x = fdiv float %a, %divisor
   %y = fdiv float %b, %divisor
   %z = fmul float %x, %y
@@ -245,33 +475,80 @@ define float @repeated_div_fast_ftz(i1 %pred, float %a, float %b, float %divisor
   ret float %w
 }
 
-; CHECK-LABEL: repeated_div_fast_ftz_sel
 define float @repeated_div_fast_ftz_sel(i1 %pred, float %a, float %b, float %divisor) #0 #1 {
-; CHECK: selp.f32
-; CHECK: div.approx.ftz.f32
+; CHECK-LABEL: repeated_div_fast_ftz_sel(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [repeated_div_fast_ftz_sel_param_0];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [repeated_div_fast_ftz_sel_param_1];
+; CHECK-NEXT:    ld.param.b32 %r2, [repeated_div_fast_ftz_sel_param_2];
+; CHECK-NEXT:    selp.f32 %r3, %r1, %r2, %p1;
+; CHECK-NEXT:    ld.param.b32 %r4, [repeated_div_fast_ftz_sel_param_3];
+; CHECK-NEXT:    div.approx.ftz.f32 %r5, %r3, %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ret;
   %x = fdiv float %a, %divisor
   %y = fdiv float %b, %divisor
   %w = select i1 %pred, float %x, float %y
   ret float %w
 }
 
-; CHECK-LABEL: frem
 define float @frem(float %a, float %b) #0 {
-  ; CHECK-NOT: testp.infinite
+; CHECK-LABEL: frem(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [frem_param_1];
+; CHECK-NEXT:    div.approx.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
   %rem = frem float %a, %b
   ret float %rem
 }
 
-; CHECK-LABEL: frem_ftz
 define float @frem_ftz(float %a, float %b) #0 #1 {
-  ; CHECK-NOT: testp.infinite
+; CHECK-LABEL: frem_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_ftz_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [frem_ftz_param_1];
+; CHECK-NEXT:    div.approx.ftz.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rzi.ftz.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.ftz.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.ftz.f32 %r6, %r5, %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
   %rem = frem float %a, %b
   ret float %rem
 }
 
-; CHECK-LABEL: frem_f64
 define double @frem_f64(double %a, double %b) #0 {
-  ; CHECK-NOT: testp.infinite
+; CHECK-LABEL: frem_f64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [frem_f64_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [frem_f64_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
+; CHECK-NEXT:    neg.f64 %rd5, %rd4;
+; CHECK-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
+; CHECK-NEXT:    ret;
   %rem = frem double %a, %b
   ret double %rem
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/pm-event.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/pm-event.ll
new file mode 100644
index 000000000000..871da6d41497
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/pm-event.ll
@@ -0,0 +1,15 @@
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
+
+declare void @llvm.nvvm.pm.event.mask(i16 %mask)
+
+; CHECK-LABEL: test_pm_event
+define void @test_pm_event() {
+  ; CHECK: pmevent.mask 255;
+  call void @llvm.nvvm.pm.event.mask(i16 u0xff)
+
+  ; CHECK: pmevent.mask 4096;
+  call void @llvm.nvvm.pm.event.mask(i16 u0x1000)
+
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/prmt.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/prmt.ll
index 271e4c86cd23..48b9eefb9fb3 100644
--- a/external/llvm-project/llvm/test/CodeGen/NVPTX/prmt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/prmt.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -verify-machineinstrs | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -verify-machineinstrs | %ptxas-verify %}
+; RUN: llc < %s -verify-machineinstrs -mcpu=sm_50 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -verify-machineinstrs -mcpu=sm_50 | %ptxas-verify %}
 
 target triple = "nvptx64-nvidia-cuda"
 
@@ -111,3 +111,20 @@ define i32 @test_prmt_rc16(i32 %lo, i32 %selector) {
   %val = call i32 @llvm.nvvm.prmt.rc16(i32 %lo, i32 %selector)
   ret i32 %val
 }
+
+define i32 @test_prmt_f4e_folding(i32 %lo, i32 %hi, i32 %ptr) {
+; CHECK-LABEL: test_prmt_f4e_folding(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_prmt_f4e_folding_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [test_prmt_f4e_folding_param_1];
+; CHECK-NEXT:    ld.param.b32 %r3, [test_prmt_f4e_folding_param_2];
+; CHECK-NEXT:    prmt.b32.f4e %r4, %r1, %r2, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+  %sh_amt = shl i32 %ptr, 3
+  %val = call i32 @llvm.fshr.i32(i32 %hi, i32 %lo, i32 %sh_amt)
+  ret i32 %val
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/sqrt-approx.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
index e2a3f1cf0d2d..a28d264cd8ec 100644
--- a/external/llvm-project/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -nvptx-prec-divf32=0 -nvptx-prec-sqrtf32=0 \
 ; RUN:   | FileCheck %s
 ; RUN: %if ptxas %{                                                                   \
@@ -12,34 +13,62 @@ declare double @llvm.sqrt.f64(double)
 
 ; -- reciprocal sqrt --
 
-; CHECK-LABEL: test_rsqrt32
 define float @test_rsqrt32(float %a) #0 {
-; CHECK: rsqrt.approx.f32
+; CHECK-LABEL: test_rsqrt32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_rsqrt32_param_0];
+; CHECK-NEXT:    rsqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
   %ret = fdiv float 1.0, %val
   ret float %ret
 }
 
-; CHECK-LABEL: test_rsqrt_ftz
 define float @test_rsqrt_ftz(float %a) #0 #1 {
-; CHECK: rsqrt.approx.ftz.f32
+; CHECK-LABEL: test_rsqrt_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_rsqrt_ftz_param_0];
+; CHECK-NEXT:    rsqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
   %ret = fdiv float 1.0, %val
   ret float %ret
 }
 
-; CHECK-LABEL: test_rsqrt64
 define double @test_rsqrt64(double %a) #0 {
-; CHECK: rsqrt.approx.f64
+; CHECK-LABEL: test_rsqrt64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_rsqrt64_param_0];
+; CHECK-NEXT:    rsqrt.approx.f64 %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
   %ret = fdiv double 1.0, %val
   ret double %ret
 }
 
-; CHECK-LABEL: test_rsqrt64_ftz
-define double @test_rsqrt64_ftz(double %a) #0 #1 {
 ; There's no rsqrt.approx.ftz.f64 instruction; we just use the non-ftz version.
-; CHECK: rsqrt.approx.f64
+define double @test_rsqrt64_ftz(double %a) #0 #1 {
+; CHECK-LABEL: test_rsqrt64_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_rsqrt64_ftz_param_0];
+; CHECK-NEXT:    rsqrt.approx.f64 %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
   %ret = fdiv double 1.0, %val
   ret double %ret
@@ -47,64 +76,135 @@ define double @test_rsqrt64_ftz(double %a) #0 #1 {
 
 ; -- sqrt --
 
-; CHECK-LABEL: test_sqrt32
 define float @test_sqrt32(float %a) #0 {
-; CHECK: sqrt.rn.f32
+; CHECK-LABEL: test_sqrt32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt32_param_0];
+; CHECK-NEXT:    sqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt32_ninf
 define float @test_sqrt32_ninf(float %a) #0 {
-; CHECK: sqrt.approx.f32
+; CHECK-LABEL: test_sqrt32_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt32_ninf_param_0];
+; CHECK-NEXT:    sqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    abs.f32 %r3, %r1;
+; CHECK-NEXT:    setp.lt.f32 %p1, %r3, 0f00800000;
+; CHECK-NEXT:    selp.f32 %r4, 0f00000000, %r2, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt_ftz
 define float @test_sqrt_ftz(float %a) #0 #1 {
-; CHECK: sqrt.rn.ftz.f32
+; CHECK-LABEL: test_sqrt_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt_ftz_param_0];
+; CHECK-NEXT:    sqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt_ftz_ninf
 define float @test_sqrt_ftz_ninf(float %a) #0 #1 {
-; CHECK: sqrt.approx.ftz.f32
+; CHECK-LABEL: test_sqrt_ftz_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt_ftz_ninf_param_0];
+; CHECK-NEXT:    setp.eq.ftz.f32 %p1, %r1, 0f00000000;
+; CHECK-NEXT:    sqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    selp.f32 %r3, 0f00000000, %r2, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt64
 define double @test_sqrt64(double %a) #0 {
-; CHECK: sqrt.rn.f64
+; CHECK-LABEL: test_sqrt64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_param_0];
+; CHECK-NEXT:    sqrt.rn.f64 %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
   %ret = tail call double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
-; CHECK-LABEL: test_sqrt64_ninf
-define double @test_sqrt64_ninf(double %a) #0 {
 ; There's no sqrt.approx.f64 instruction; we emit
 ; reciprocal(rsqrt.approx.f64(x)).  There's no non-ftz approximate reciprocal,
 ; so we just use the ftz version.
-; CHECK: rsqrt.approx.f64
-; CHECK: rcp.approx.ftz.f64
+define double @test_sqrt64_ninf(double %a) #0 {
+; CHECK-LABEL: test_sqrt64_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_ninf_param_0];
+; CHECK-NEXT:    abs.f64 %rd2, %rd1;
+; CHECK-NEXT:    setp.lt.f64 %p1, %rd2, 0d0010000000000000;
+; CHECK-NEXT:    rsqrt.approx.f64 %rd3, %rd1;
+; CHECK-NEXT:    rcp.approx.ftz.f64 %rd4, %rd3;
+; CHECK-NEXT:    selp.f64 %rd5, 0d0000000000000000, %rd4, %p1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd5;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
-; CHECK-LABEL: test_sqrt64_ftz
 define double @test_sqrt64_ftz(double %a) #0 #1 {
-; CHECK: sqrt.rn.f64
+; CHECK-LABEL: test_sqrt64_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_ftz_param_0];
+; CHECK-NEXT:    sqrt.rn.f64 %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
   %ret = tail call double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
-; CHECK-LABEL: test_sqrt64_ftz_ninf
-define double @test_sqrt64_ftz_ninf(double %a) #0 #1 {
 ; There's no sqrt.approx.ftz.f64 instruction; we just use the non-ftz version.
-; CHECK: rsqrt.approx.f64
-; CHECK: rcp.approx.ftz.f64
+define double @test_sqrt64_ftz_ninf(double %a) #0 #1 {
+; CHECK-LABEL: test_sqrt64_ftz_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_ftz_ninf_param_0];
+; CHECK-NEXT:    abs.f64 %rd2, %rd1;
+; CHECK-NEXT:    setp.lt.f64 %p1, %rd2, 0d0010000000000000;
+; CHECK-NEXT:    rsqrt.approx.f64 %rd3, %rd1;
+; CHECK-NEXT:    rcp.approx.ftz.f64 %rd4, %rd3;
+; CHECK-NEXT:    selp.f64 %rd5, 0d0000000000000000, %rd4, %p1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd5;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
@@ -114,93 +214,224 @@ define double @test_sqrt64_ftz_ninf(double %a) #0 #1 {
 ; The sqrt and rsqrt refinement algorithms both emit an rsqrt.approx, followed
 ; by some math.
 
-; CHECK-LABEL: test_rsqrt32_refined
 define float @test_rsqrt32_refined(float %a) #0 #2 {
-; CHECK: rsqrt.approx.f32
+; CHECK-LABEL: test_rsqrt32_refined(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_rsqrt32_refined_param_0];
+; CHECK-NEXT:    rsqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    mul.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    fma.rn.f32 %r4, %r3, %r2, 0fC0400000;
+; CHECK-NEXT:    mul.f32 %r5, %r2, 0fBF000000;
+; CHECK-NEXT:    mul.f32 %r6, %r5, %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
   %ret = fdiv float 1.0, %val
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt32_refined
 define float @test_sqrt32_refined(float %a) #0 #2 {
-; CHECK: sqrt.rn.f32
+; CHECK-LABEL: test_sqrt32_refined(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt32_refined_param_0];
+; CHECK-NEXT:    sqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt32_refined_ninf
 define float @test_sqrt32_refined_ninf(float %a) #0 #2 {
-; CHECK: rsqrt.approx.f32
+; CHECK-LABEL: test_sqrt32_refined_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt32_refined_ninf_param_0];
+; CHECK-NEXT:    rsqrt.approx.f32 %r2, %r1;
+; CHECK-NEXT:    mul.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    fma.rn.f32 %r4, %r3, %r2, 0fC0400000;
+; CHECK-NEXT:    mul.f32 %r5, %r3, 0fBF000000;
+; CHECK-NEXT:    mul.f32 %r6, %r5, %r4;
+; CHECK-NEXT:    abs.f32 %r7, %r1;
+; CHECK-NEXT:    setp.lt.f32 %p1, %r7, 0f00800000;
+; CHECK-NEXT:    selp.f32 %r8, 0f00000000, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_rsqrt64_refined
 define double @test_rsqrt64_refined(double %a) #0 #2 {
-; CHECK: rsqrt.approx.f64
+; CHECK-LABEL: test_rsqrt64_refined(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_rsqrt64_refined_param_0];
+; CHECK-NEXT:    rsqrt.approx.f64 %rd2, %rd1;
+; CHECK-NEXT:    mul.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    fma.rn.f64 %rd4, %rd3, %rd2, 0dC008000000000000;
+; CHECK-NEXT:    mul.f64 %rd5, %rd2, 0dBFE0000000000000;
+; CHECK-NEXT:    mul.f64 %rd6, %rd5, %rd4;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
+; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
   %ret = fdiv double 1.0, %val
   ret double %ret
 }
 
-; CHECK-LABEL: test_sqrt64_refined
 define double @test_sqrt64_refined(double %a) #0 #2 {
-; CHECK: sqrt.rn.f64
+; CHECK-LABEL: test_sqrt64_refined(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_refined_param_0];
+; CHECK-NEXT:    sqrt.rn.f64 %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
   %ret = tail call double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
-; CHECK-LABEL: test_sqrt64_refined_ninf
 define double @test_sqrt64_refined_ninf(double %a) #0 #2 {
-; CHECK: rsqrt.approx.f64
+; CHECK-LABEL: test_sqrt64_refined_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_refined_ninf_param_0];
+; CHECK-NEXT:    rsqrt.approx.f64 %rd2, %rd1;
+; CHECK-NEXT:    mul.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    fma.rn.f64 %rd4, %rd3, %rd2, 0dC008000000000000;
+; CHECK-NEXT:    mul.f64 %rd5, %rd3, 0dBFE0000000000000;
+; CHECK-NEXT:    mul.f64 %rd6, %rd5, %rd4;
+; CHECK-NEXT:    abs.f64 %rd7, %rd1;
+; CHECK-NEXT:    setp.lt.f64 %p1, %rd7, 0d0010000000000000;
+; CHECK-NEXT:    selp.f64 %rd8, 0d0000000000000000, %rd6, %p1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd8;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
 ; -- refined sqrt and rsqrt with ftz enabled --
 
-; CHECK-LABEL: test_rsqrt32_refined_ftz
 define float @test_rsqrt32_refined_ftz(float %a) #0 #1 #2 {
-; CHECK: rsqrt.approx.ftz.f32
+; CHECK-LABEL: test_rsqrt32_refined_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_rsqrt32_refined_ftz_param_0];
+; CHECK-NEXT:    rsqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    mul.ftz.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    fma.rn.ftz.f32 %r4, %r3, %r2, 0fC0400000;
+; CHECK-NEXT:    mul.ftz.f32 %r5, %r2, 0fBF000000;
+; CHECK-NEXT:    mul.ftz.f32 %r6, %r5, %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
   %ret = fdiv float 1.0, %val
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt32_refined_ftz
 define float @test_sqrt32_refined_ftz(float %a) #0 #1 #2 {
-; CHECK: sqrt.rn.ftz.f32
+; CHECK-LABEL: test_sqrt32_refined_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt32_refined_ftz_param_0];
+; CHECK-NEXT:    sqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_sqrt32_refined_ftz_ninf
 define float @test_sqrt32_refined_ftz_ninf(float %a) #0 #1 #2 {
-; CHECK: rsqrt.approx.ftz.f32
+; CHECK-LABEL: test_sqrt32_refined_ftz_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt32_refined_ftz_ninf_param_0];
+; CHECK-NEXT:    rsqrt.approx.ftz.f32 %r2, %r1;
+; CHECK-NEXT:    mul.ftz.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    fma.rn.ftz.f32 %r4, %r3, %r2, 0fC0400000;
+; CHECK-NEXT:    mul.ftz.f32 %r5, %r3, 0fBF000000;
+; CHECK-NEXT:    mul.ftz.f32 %r6, %r5, %r4;
+; CHECK-NEXT:    setp.eq.ftz.f32 %p1, %r1, 0f00000000;
+; CHECK-NEXT:    selp.f32 %r7, 0f00000000, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-; CHECK-LABEL: test_rsqrt64_refined_ftz
-define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
 ; There's no rsqrt.approx.ftz.f64, so we just use the non-ftz version.
-; CHECK: rsqrt.approx.f64
+define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
+; CHECK-LABEL: test_rsqrt64_refined_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_rsqrt64_refined_ftz_param_0];
+; CHECK-NEXT:    rsqrt.approx.f64 %rd2, %rd1;
+; CHECK-NEXT:    mul.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    fma.rn.f64 %rd4, %rd3, %rd2, 0dC008000000000000;
+; CHECK-NEXT:    mul.f64 %rd5, %rd2, 0dBFE0000000000000;
+; CHECK-NEXT:    mul.f64 %rd6, %rd5, %rd4;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
+; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
   %ret = fdiv double 1.0, %val
   ret double %ret
 }
 
-; CHECK-LABEL: test_sqrt64_refined_ftz
 define double @test_sqrt64_refined_ftz(double %a) #0 #1 #2 {
-; CHECK: sqrt.rn.f64
+; CHECK-LABEL: test_sqrt64_refined_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_refined_ftz_param_0];
+; CHECK-NEXT:    sqrt.rn.f64 %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
   %ret = tail call double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
-; CHECK-LABEL: test_sqrt64_refined_ftz_ninf
 define double @test_sqrt64_refined_ftz_ninf(double %a) #0 #1 #2 {
-; CHECK: rsqrt.approx.f64
+; CHECK-LABEL: test_sqrt64_refined_ftz_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt64_refined_ftz_ninf_param_0];
+; CHECK-NEXT:    rsqrt.approx.f64 %rd2, %rd1;
+; CHECK-NEXT:    mul.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    fma.rn.f64 %rd4, %rd3, %rd2, 0dC008000000000000;
+; CHECK-NEXT:    mul.f64 %rd5, %rd3, 0dBFE0000000000000;
+; CHECK-NEXT:    mul.f64 %rd6, %rd5, %rd4;
+; CHECK-NEXT:    abs.f64 %rd7, %rd1;
+; CHECK-NEXT:    setp.lt.f64 %p1, %rd7, 0d0010000000000000;
+; CHECK-NEXT:    selp.f64 %rd8, 0d0000000000000000, %rd6, %p1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd8;
+; CHECK-NEXT:    ret;
   %ret = tail call ninf afn double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll b/external/llvm-project/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
index 8e4c77e76029..bac8bbbf0b4d 100644
--- a/external/llvm-project/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
@@ -8,17 +8,16 @@
 define <1 x i8> @out_v1i8(<1 x i8> %x, <1 x i8> %y, <1 x i8> %mask) nounwind {
 ; CHECK-LABEL: out_v1i8(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<8>;
+; CHECK-NEXT:    .reg .b16 %rs<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b8 %rs1, [out_v1i8_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [out_v1i8_param_2];
-; CHECK-NEXT:    and.b16 %rs3, %rs1, %rs2;
-; CHECK-NEXT:    ld.param.b8 %rs4, [out_v1i8_param_1];
-; CHECK-NEXT:    not.b16 %rs5, %rs2;
-; CHECK-NEXT:    and.b16 %rs6, %rs4, %rs5;
-; CHECK-NEXT:    or.b16 %rs7, %rs3, %rs6;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs7;
+; CHECK-NEXT:    ld.param.b8 %rs2, [out_v1i8_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs3, [out_v1i8_param_2];
+; CHECK-NEXT:    xor.b16 %rs4, %rs1, %rs2;
+; CHECK-NEXT:    and.b16 %rs5, %rs4, %rs3;
+; CHECK-NEXT:    xor.b16 %rs6, %rs5, %rs2;
+; CHECK-NEXT:    st.param.b8 [func_retval0], %rs6;
 ; CHECK-NEXT:    ret;
   %mx = and <1 x i8> %x, %mask
   %notmask = xor <1 x i8> %mask, <i8 -1>
@@ -34,17 +33,16 @@ define <1 x i8> @out_v1i8(<1 x i8> %x, <1 x i8> %y, <1 x i8> %mask) nounwind {
 define <1 x i16> @out_v1i16(<1 x i16> %x, <1 x i16> %y, <1 x i16> %mask) nounwind {
 ; CHECK-LABEL: out_v1i16(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<8>;
+; CHECK-NEXT:    .reg .b16 %rs<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b16 %rs1, [out_v1i16_param_0];
-; CHECK-NEXT:    ld.param.b16 %rs2, [out_v1i16_param_2];
-; CHECK-NEXT:    and.b16 %rs3, %rs1, %rs2;
-; CHECK-NEXT:    ld.param.b16 %rs4, [out_v1i16_param_1];
-; CHECK-NEXT:    not.b16 %rs5, %rs2;
-; CHECK-NEXT:    and.b16 %rs6, %rs4, %rs5;
-; CHECK-NEXT:    or.b16 %rs7, %rs3, %rs6;
-; CHECK-NEXT:    st.param.b16 [func_retval0], %rs7;
+; CHECK-NEXT:    ld.param.b16 %rs2, [out_v1i16_param_1];
+; CHECK-NEXT:    ld.param.b16 %rs3, [out_v1i16_param_2];
+; CHECK-NEXT:    xor.b16 %rs4, %rs1, %rs2;
+; CHECK-NEXT:    and.b16 %rs5, %rs4, %rs3;
+; CHECK-NEXT:    xor.b16 %rs6, %rs5, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs6;
 ; CHECK-NEXT:    ret;
   %mx = and <1 x i16> %x, %mask
   %notmask = xor <1 x i16> %mask, <i16 -1>
@@ -126,17 +124,16 @@ define <2 x i16> @out_v2i16(<2 x i16> %x, <2 x i16> %y, <2 x i16> %mask) nounwin
 define <1 x i32> @out_v1i32(<1 x i32> %x, <1 x i32> %y, <1 x i32> %mask) nounwind {
 ; CHECK-LABEL: out_v1i32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [out_v1i32_param_0];
-; CHECK-NEXT:    ld.param.b32 %r2, [out_v1i32_param_2];
-; CHECK-NEXT:    and.b32 %r3, %r1, %r2;
-; CHECK-NEXT:    ld.param.b32 %r4, [out_v1i32_param_1];
-; CHECK-NEXT:    not.b32 %r5, %r2;
-; CHECK-NEXT:    and.b32 %r6, %r4, %r5;
-; CHECK-NEXT:    or.b32 %r7, %r3, %r6;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT:    ld.param.b32 %r2, [out_v1i32_param_1];
+; CHECK-NEXT:    ld.param.b32 %r3, [out_v1i32_param_2];
+; CHECK-NEXT:    xor.b32 %r4, %r1, %r2;
+; CHECK-NEXT:    and.b32 %r5, %r4, %r3;
+; CHECK-NEXT:    xor.b32 %r6, %r5, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %mx = and <1 x i32> %x, %mask
   %notmask = xor <1 x i32> %mask, <i32 -1>
@@ -230,21 +227,19 @@ define <4 x i16> @out_v4i16_undef(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) n
 define <2 x i32> @out_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %mask) nounwind {
 ; CHECK-LABEL: out_v2i32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<15>;
+; CHECK-NEXT:    .reg .b32 %r<13>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [out_v2i32_param_0];
-; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [out_v2i32_param_2];
-; CHECK-NEXT:    and.b32 %r5, %r1, %r3;
-; CHECK-NEXT:    and.b32 %r6, %r2, %r4;
-; CHECK-NEXT:    ld.param.v2.b32 {%r7, %r8}, [out_v2i32_param_1];
-; CHECK-NEXT:    not.b32 %r9, %r4;
-; CHECK-NEXT:    not.b32 %r10, %r3;
-; CHECK-NEXT:    and.b32 %r11, %r7, %r10;
-; CHECK-NEXT:    and.b32 %r12, %r8, %r9;
-; CHECK-NEXT:    or.b32 %r13, %r6, %r12;
-; CHECK-NEXT:    or.b32 %r14, %r5, %r11;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r14, %r13};
+; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [out_v2i32_param_1];
+; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [out_v2i32_param_2];
+; CHECK-NEXT:    xor.b32 %r7, %r2, %r4;
+; CHECK-NEXT:    and.b32 %r8, %r7, %r6;
+; CHECK-NEXT:    xor.b32 %r9, %r8, %r4;
+; CHECK-NEXT:    xor.b32 %r10, %r1, %r3;
+; CHECK-NEXT:    and.b32 %r11, %r10, %r5;
+; CHECK-NEXT:    xor.b32 %r12, %r11, %r3;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r12, %r9};
 ; CHECK-NEXT:    ret;
   %mx = and <2 x i32> %x, %mask
   %notmask = xor <2 x i32> %mask, <i32 -1, i32 -1>
@@ -256,17 +251,16 @@ define <2 x i32> @out_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %mask) nounwin
 define <1 x i64> @out_v1i64(<1 x i64> %x, <1 x i64> %y, <1 x i64> %mask) nounwind {
 ; CHECK-LABEL: out_v1i64(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<8>;
+; CHECK-NEXT:    .reg .b64 %rd<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [out_v1i64_param_0];
-; CHECK-NEXT:    ld.param.b64 %rd2, [out_v1i64_param_2];
-; CHECK-NEXT:    and.b64 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    ld.param.b64 %rd4, [out_v1i64_param_1];
-; CHECK-NEXT:    not.b64 %rd5, %rd2;
-; CHECK-NEXT:    and.b64 %rd6, %rd4, %rd5;
-; CHECK-NEXT:    or.b64 %rd7, %rd3, %rd6;
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd7;
+; CHECK-NEXT:    ld.param.b64 %rd2, [out_v1i64_param_1];
+; CHECK-NEXT:    ld.param.b64 %rd3, [out_v1i64_param_2];
+; CHECK-NEXT:    xor.b64 %rd4, %rd1, %rd2;
+; CHECK-NEXT:    and.b64 %rd5, %rd4, %rd3;
+; CHECK-NEXT:    xor.b64 %rd6, %rd5, %rd2;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
   %mx = and <1 x i64> %x, %mask
   %notmask = xor <1 x i64> %mask, <i64 -1>
@@ -350,29 +344,25 @@ define <8 x i16> @out_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %mask) nounwin
 define <4 x i32> @out_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %mask) nounwind {
 ; CHECK-LABEL: out_v4i32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<29>;
+; CHECK-NEXT:    .reg .b32 %r<25>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [out_v4i32_param_0];
-; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [out_v4i32_param_2];
-; CHECK-NEXT:    and.b32 %r9, %r1, %r5;
-; CHECK-NEXT:    and.b32 %r10, %r2, %r6;
-; CHECK-NEXT:    and.b32 %r11, %r3, %r7;
-; CHECK-NEXT:    and.b32 %r12, %r4, %r8;
-; CHECK-NEXT:    ld.param.v4.b32 {%r13, %r14, %r15, %r16}, [out_v4i32_param_1];
-; CHECK-NEXT:    not.b32 %r17, %r8;
-; CHECK-NEXT:    not.b32 %r18, %r7;
-; CHECK-NEXT:    not.b32 %r19, %r6;
-; CHECK-NEXT:    not.b32 %r20, %r5;
-; CHECK-NEXT:    and.b32 %r21, %r13, %r20;
-; CHECK-NEXT:    and.b32 %r22, %r14, %r19;
-; CHECK-NEXT:    and.b32 %r23, %r15, %r18;
-; CHECK-NEXT:    and.b32 %r24, %r16, %r17;
-; CHECK-NEXT:    or.b32 %r25, %r12, %r24;
-; CHECK-NEXT:    or.b32 %r26, %r11, %r23;
-; CHECK-NEXT:    or.b32 %r27, %r10, %r22;
-; CHECK-NEXT:    or.b32 %r28, %r9, %r21;
-; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r28, %r27, %r26, %r25};
+; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [out_v4i32_param_1];
+; CHECK-NEXT:    ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [out_v4i32_param_2];
+; CHECK-NEXT:    xor.b32 %r13, %r4, %r8;
+; CHECK-NEXT:    and.b32 %r14, %r13, %r12;
+; CHECK-NEXT:    xor.b32 %r15, %r14, %r8;
+; CHECK-NEXT:    xor.b32 %r16, %r3, %r7;
+; CHECK-NEXT:    and.b32 %r17, %r16, %r11;
+; CHECK-NEXT:    xor.b32 %r18, %r17, %r7;
+; CHECK-NEXT:    xor.b32 %r19, %r2, %r6;
+; CHECK-NEXT:    and.b32 %r20, %r19, %r10;
+; CHECK-NEXT:    xor.b32 %r21, %r20, %r6;
+; CHECK-NEXT:    xor.b32 %r22, %r1, %r5;
+; CHECK-NEXT:    and.b32 %r23, %r22, %r9;
+; CHECK-NEXT:    xor.b32 %r24, %r23, %r5;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r24, %r21, %r18, %r15};
 ; CHECK-NEXT:    ret;
   %mx = and <4 x i32> %x, %mask
   %notmask = xor <4 x i32> %mask, <i32 -1, i32 -1, i32 -1, i32 -1>
@@ -384,26 +374,23 @@ define <4 x i32> @out_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %mask) nounwin
 define <4 x i32> @out_v4i32_undef(<4 x i32> %x, <4 x i32> %y, <4 x i32> %mask) nounwind {
 ; CHECK-LABEL: out_v4i32_undef(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<26>;
+; CHECK-NEXT:    .reg .b32 %r<23>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [out_v4i32_undef_param_0];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [out_v4i32_undef_param_2];
 ; CHECK-NEXT:    and.b32 %r9, %r3, %r7;
-; CHECK-NEXT:    and.b32 %r10, %r1, %r5;
-; CHECK-NEXT:    and.b32 %r11, %r2, %r6;
-; CHECK-NEXT:    and.b32 %r12, %r4, %r8;
-; CHECK-NEXT:    ld.param.v4.b32 {%r13, %r14, %r15, %r16}, [out_v4i32_undef_param_1];
-; CHECK-NEXT:    not.b32 %r17, %r8;
-; CHECK-NEXT:    not.b32 %r18, %r6;
-; CHECK-NEXT:    not.b32 %r19, %r5;
-; CHECK-NEXT:    and.b32 %r20, %r13, %r19;
-; CHECK-NEXT:    and.b32 %r21, %r14, %r18;
-; CHECK-NEXT:    and.b32 %r22, %r16, %r17;
-; CHECK-NEXT:    or.b32 %r23, %r12, %r22;
-; CHECK-NEXT:    or.b32 %r24, %r11, %r21;
-; CHECK-NEXT:    or.b32 %r25, %r10, %r20;
-; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r25, %r24, %r9, %r23};
+; CHECK-NEXT:    ld.param.v4.b32 {%r10, %r11, %r12, %r13}, [out_v4i32_undef_param_1];
+; CHECK-NEXT:    xor.b32 %r14, %r4, %r13;
+; CHECK-NEXT:    and.b32 %r15, %r14, %r8;
+; CHECK-NEXT:    xor.b32 %r16, %r15, %r13;
+; CHECK-NEXT:    xor.b32 %r17, %r2, %r11;
+; CHECK-NEXT:    and.b32 %r18, %r17, %r6;
+; CHECK-NEXT:    xor.b32 %r19, %r18, %r11;
+; CHECK-NEXT:    xor.b32 %r20, %r1, %r10;
+; CHECK-NEXT:    and.b32 %r21, %r20, %r5;
+; CHECK-NEXT:    xor.b32 %r22, %r21, %r10;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r22, %r19, %r9, %r16};
 ; CHECK-NEXT:    ret;
   %mx = and <4 x i32> %x, %mask
   %notmask = xor <4 x i32> %mask, <i32 -1, i32 -1, i32 undef, i32 -1>
@@ -415,21 +402,19 @@ define <4 x i32> @out_v4i32_undef(<4 x i32> %x, <4 x i32> %y, <4 x i32> %mask) n
 define <2 x i64> @out_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %mask) nounwind {
 ; CHECK-LABEL: out_v2i64(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<15>;
+; CHECK-NEXT:    .reg .b64 %rd<13>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [out_v2i64_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [out_v2i64_param_2];
-; CHECK-NEXT:    and.b64 %rd5, %rd1, %rd3;
-; CHECK-NEXT:    and.b64 %rd6, %rd2, %rd4;
-; CHECK-NEXT:    ld.param.v2.b64 {%rd7, %rd8}, [out_v2i64_param_1];
-; CHECK-NEXT:    not.b64 %rd9, %rd4;
-; CHECK-NEXT:    not.b64 %rd10, %rd3;
-; CHECK-NEXT:    and.b64 %rd11, %rd7, %rd10;
-; CHECK-NEXT:    and.b64 %rd12, %rd8, %rd9;
-; CHECK-NEXT:    or.b64 %rd13, %rd6, %rd12;
-; CHECK-NEXT:    or.b64 %rd14, %rd5, %rd11;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd14, %rd13};
+; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [out_v2i64_param_1];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [out_v2i64_param_2];
+; CHECK-NEXT:    xor.b64 %rd7, %rd2, %rd4;
+; CHECK-NEXT:    and.b64 %rd8, %rd7, %rd6;
+; CHECK-NEXT:    xor.b64 %rd9, %rd8, %rd4;
+; CHECK-NEXT:    xor.b64 %rd10, %rd1, %rd3;
+; CHECK-NEXT:    and.b64 %rd11, %rd10, %rd5;
+; CHECK-NEXT:    xor.b64 %rd12, %rd11, %rd3;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd12, %rd9};
 ; CHECK-NEXT:    ret;
   %mx = and <2 x i64> %x, %mask
   %notmask = xor <2 x i64> %mask, <i64 -1, i64 -1>
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
index 8517783e3ebd..1a8dabc5ad71 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
@@ -15,50 +15,57 @@ define signext i32 @main() nounwind {
 ; CHECK-NEXT:    stdu 1, -48(1)
 ; CHECK-NEXT:    li 3, -32477
 ; CHECK-NEXT:    std 0, 64(1)
-; CHECK-NEXT:    li 4, 234
-; CHECK-NEXT:    addi 6, 1, 46
 ; CHECK-NEXT:    sth 3, 46(1)
-; CHECK-NEXT:    lis 3, 0
+; CHECK-NEXT:    addi 3, 1, 46
+; CHECK-NEXT:    lharx 4, 0, 3
+; CHECK-NEXT:    clrlwi  4, 4, 16
+; CHECK-NEXT:    cmplwi  4, 33059
+; CHECK-NEXT:    bne     0, .LBB0_4
+; CHECK-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    ori 3, 3, 33059
-; CHECK-NEXT:  .LBB0_1: # %L.entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lharx 5, 0, 6
-; CHECK-NEXT:    cmpw 5, 3
-; CHECK-NEXT:    bne 0, .LBB0_3
-; CHECK-NEXT:  # %bb.2: # %L.entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    sthcx. 4, 0, 6
-; CHECK-NEXT:    bne 0, .LBB0_1
-; CHECK-NEXT:  .LBB0_3: # %L.entry
-; CHECK-NEXT:    cmplwi 5, 33059
+; CHECK-NEXT:    li 4, 234
+; CHECK-NEXT:    .p2align        5
+; CHECK-NEXT:  .LBB0_2:                                # %cmpxchg.trystore
+; CHECK-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    sthcx. 4, 0, 3
+; CHECK-NEXT:    beq     0, .LBB0_7
+; CHECK-NEXT:  # %bb.3:                                # %cmpxchg.releasedload
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    lharx 5, 0, 3
+; CHECK-NEXT:    clrlwi  5, 5, 16
+; CHECK-NEXT:    cmplwi  5, 33059
+; CHECK-NEXT:    beq     0, .LBB0_2
+; CHECK-NEXT:  .LBB0_4:                                # %cmpxchg.nostore
 ; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    bne 0, .LBB0_6
-; CHECK-NEXT:  # %bb.4: # %L.B0000
+; CHECK-NEXT:    b .LBB0_8
+; CHECK-NEXT:  .LBB0_5:                                # %L.B0000
 ; CHECK-NEXT:    lhz 3, 46(1)
-; CHECK-NEXT:    cmplwi 3, 234
-; CHECK-NEXT:    bne 0, .LBB0_7
-; CHECK-NEXT:  # %bb.5: # %L.B0001
+; CHECK-NEXT:    cmplwi  3, 234
+; CHECK-NEXT:    bne     0, .LBB0_9
+; CHECK-NEXT:  # %bb.6:                                # %L.B0001
 ; CHECK-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-NEXT:    bl puts
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    li 3, 0
-; CHECK-NEXT:    b .LBB0_9
-; CHECK-NEXT:  .LBB0_6: # %L.B0003
+; CHECK-NEXT:    b .LBB0_11
+; CHECK-NEXT:  .LBB0_7:                                # %cmpxchg.success
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    b .LBB0_5
+; CHECK-NEXT:  .LBB0_8:                                # %L.B0003
 ; CHECK-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-NEXT:    addi 3, 3, 16
-; CHECK-NEXT:    b .LBB0_8
-; CHECK-NEXT:  .LBB0_7: # %L.B0005
+; CHECK-NEXT:    b .LBB0_10
+; CHECK-NEXT:  .LBB0_9:                                # %L.B0005
 ; CHECK-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-NEXT:    addi 3, 3, 64
-; CHECK-NEXT:  .LBB0_8: # %L.B0003
+; CHECK-NEXT:  .LBB0_10:                               # %L.B0003
 ; CHECK-NEXT:    bl puts
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    li 3, 1
-; CHECK-NEXT:  .LBB0_9: # %L.B0003
+; CHECK-NEXT:  .LBB0_11:                               # %L.B0003
 ; CHECK-NEXT:    addi 1, 1, 48
 ; CHECK-NEXT:    ld 0, 16(1)
 ; CHECK-NEXT:    mtlr 0
@@ -69,64 +76,69 @@ define signext i32 @main() nounwind {
 ; CHECK-P7-NEXT:    mflr 0
 ; CHECK-P7-NEXT:    stdu 1, -48(1)
 ; CHECK-P7-NEXT:    li 3, -32477
-; CHECK-P7-NEXT:    std 0, 64(1)
 ; CHECK-P7-NEXT:    addi 4, 1, 46
-; CHECK-P7-NEXT:    li 6, 234
+; CHECK-P7-NEXT:    std 0, 64(1)
 ; CHECK-P7-NEXT:    sth 3, 46(1)
-; CHECK-P7-NEXT:    lis 3, 0
+; CHECK-P7-NEXT:    rldicr 3, 4, 0, 61
+; CHECK-P7-NEXT:    rlwinm 4, 4, 3, 27, 27
+; CHECK-P7-NEXT:    lwarx 5, 0, 3
+; CHECK-P7-NEXT:    srw 6, 5, 4
+; CHECK-P7-NEXT:    clrlwi  6, 6, 16
+; CHECK-P7-NEXT:    cmplwi  6, 33059
+; CHECK-P7-NEXT:    bne     0, .LBB0_4
+; CHECK-P7-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
+; CHECK-P7-NEXT:    lis 6, 0
+; CHECK-P7-NEXT:    li 7, 234
 ; CHECK-P7-NEXT:    sync
-; CHECK-P7-NEXT:    ori 5, 3, 33059
-; CHECK-P7-NEXT:    rlwinm 3, 4, 3, 27, 27
-; CHECK-P7-NEXT:    rldicr 4, 4, 0, 61
-; CHECK-P7-NEXT:    slw 7, 5, 3
-; CHECK-P7-NEXT:    li 5, 0
-; CHECK-P7-NEXT:    slw 6, 6, 3
-; CHECK-P7-NEXT:    ori 5, 5, 65535
-; CHECK-P7-NEXT:    slw 5, 5, 3
-; CHECK-P7-NEXT:    and 6, 6, 5
-; CHECK-P7-NEXT:    and 7, 7, 5
-; CHECK-P7-NEXT:  .LBB0_1: # %L.entry
-; CHECK-P7-NEXT:    #
-; CHECK-P7-NEXT:    lwarx 9, 0, 4
-; CHECK-P7-NEXT:    and 8, 9, 5
-; CHECK-P7-NEXT:    cmpw 8, 7
-; CHECK-P7-NEXT:    bne 0, .LBB0_3
-; CHECK-P7-NEXT:  # %bb.2: # %L.entry
-; CHECK-P7-NEXT:    #
-; CHECK-P7-NEXT:    andc 9, 9, 5
-; CHECK-P7-NEXT:    or 9, 9, 6
-; CHECK-P7-NEXT:    stwcx. 9, 0, 4
-; CHECK-P7-NEXT:    bne 0, .LBB0_1
-; CHECK-P7-NEXT:  .LBB0_3: # %L.entry
-; CHECK-P7-NEXT:    srw 3, 8, 3
+; CHECK-P7-NEXT:    ori 6, 6, 65535
+; CHECK-P7-NEXT:    slw 7, 7, 4
+; CHECK-P7-NEXT:    slw 6, 6, 4
+; CHECK-P7-NEXT:    not     6, 6
+; CHECK-P7-NEXT:    .p2align        4
+; CHECK-P7-NEXT:  .LBB0_2:                                # %cmpxchg.trystore
+; CHECK-P7-NEXT:                                        # =>This Inner Loop Header: Depth=1
+; CHECK-P7-NEXT:    and 5, 5, 6
+; CHECK-P7-NEXT:    or 5, 5, 7
+; CHECK-P7-NEXT:    stwcx. 5, 0, 3
+; CHECK-P7-NEXT:    beq     0, .LBB0_7
+; CHECK-P7-NEXT:  # %bb.3:                                # %cmpxchg.releasedload
+; CHECK-P7-NEXT:                                        #   in Loop: Header=BB0_2 Depth=1
+; CHECK-P7-NEXT:    lwarx 5, 0, 3
+; CHECK-P7-NEXT:    srw 8, 5, 4
+; CHECK-P7-NEXT:    clrlwi  8, 8, 16
+; CHECK-P7-NEXT:    cmplwi  8, 33059
+; CHECK-P7-NEXT:    beq     0, .LBB0_2
+; CHECK-P7-NEXT:  .LBB0_4:                                # %cmpxchg.nostore
 ; CHECK-P7-NEXT:    lwsync
-; CHECK-P7-NEXT:    cmplwi 3, 33059
-; CHECK-P7-NEXT:    bne 0, .LBB0_6
-; CHECK-P7-NEXT:  # %bb.4: # %L.B0000
+; CHECK-P7-NEXT:    b .LBB0_8
+; CHECK-P7-NEXT:  .LBB0_5:                                # %L.B0000
 ; CHECK-P7-NEXT:    lhz 3, 46(1)
-; CHECK-P7-NEXT:    cmplwi 3, 234
-; CHECK-P7-NEXT:    bne 0, .LBB0_7
-; CHECK-P7-NEXT:  # %bb.5: # %L.B0001
+; CHECK-P7-NEXT:    cmplwi  3, 234
+; CHECK-P7-NEXT:    bne     0, .LBB0_9
+; CHECK-P7-NEXT:  # %bb.6:                                # %L.B0001
 ; CHECK-P7-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-P7-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-P7-NEXT:    bl puts
 ; CHECK-P7-NEXT:    nop
 ; CHECK-P7-NEXT:    li 3, 0
-; CHECK-P7-NEXT:    b .LBB0_9
-; CHECK-P7-NEXT:  .LBB0_6: # %L.B0003
+; CHECK-P7-NEXT:    b .LBB0_11
+; CHECK-P7-NEXT:  .LBB0_7:                                # %cmpxchg.success
+; CHECK-P7-NEXT:    lwsync
+; CHECK-P7-NEXT:    b .LBB0_5
+; CHECK-P7-NEXT:  .LBB0_8:                                # %L.B0003
 ; CHECK-P7-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-P7-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-P7-NEXT:    addi 3, 3, 16
-; CHECK-P7-NEXT:    b .LBB0_8
-; CHECK-P7-NEXT:  .LBB0_7: # %L.B0005
+; CHECK-P7-NEXT:    b .LBB0_10
+; CHECK-P7-NEXT:  .LBB0_9:                                # %L.B0005
 ; CHECK-P7-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-P7-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-P7-NEXT:    addi 3, 3, 64
-; CHECK-P7-NEXT:  .LBB0_8: # %L.B0003
+; CHECK-P7-NEXT:  .LBB0_10:                               # %L.B0003
 ; CHECK-P7-NEXT:    bl puts
 ; CHECK-P7-NEXT:    nop
 ; CHECK-P7-NEXT:    li 3, 1
-; CHECK-P7-NEXT:  .LBB0_9: # %L.B0003
+; CHECK-P7-NEXT:  .LBB0_11:                               # %L.B0003
 ; CHECK-P7-NEXT:    addi 1, 1, 48
 ; CHECK-P7-NEXT:    ld 0, 16(1)
 ; CHECK-P7-NEXT:    mtlr 0
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/all-atomics.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/all-atomics.ll
index 531e559ea730..67cee358882f 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/all-atomics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/all-atomics.ll
@@ -4336,704 +4336,959 @@ entry:
 define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-LABEL: test_compare_and_swap:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, uc@toc@ha
-; CHECK-NEXT:    addis 4, 2, sc@toc@ha
-; CHECK-NEXT:    std 27, -40(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 28, -32(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 29, -24(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 30, -16(1) # 8-byte Folded Spill
-; CHECK-NEXT:    lbz 5, uc@toc@l(3)
-; CHECK-NEXT:    lbz 8, sc@toc@l(4)
-; CHECK-NEXT:    addi 6, 3, uc@toc@l
-; CHECK-NEXT:    addi 0, 4, sc@toc@l
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_1: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lbarx 7, 0, 0
-; CHECK-NEXT:    cmpw 7, 5
-; CHECK-NEXT:    bne 0, .LBB3_3
-; CHECK-NEXT:  # %bb.2: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stbcx. 8, 0, 0
-; CHECK-NEXT:    bne 0, .LBB3_1
-; CHECK-NEXT:  .LBB3_3: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    stb 7, sc@toc@l(4)
-; CHECK-NEXT:    lbz 8, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_4: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lbarx 5, 0, 6
-; CHECK-NEXT:    cmpw 5, 8
-; CHECK-NEXT:    bne 0, .LBB3_6
-; CHECK-NEXT:  # %bb.5: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stbcx. 7, 0, 6
-; CHECK-NEXT:    bne 0, .LBB3_4
-; CHECK-NEXT:  .LBB3_6: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    stb 5, uc@toc@l(3)
-; CHECK-NEXT:    lbz 7, sc@toc@l(4)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:    extsb 8, 7
-; CHECK-NEXT:    addis 7, 2, ss@toc@ha
-; CHECK-NEXT:    addi 12, 7, ss@toc@l
-; CHECK-NEXT:  .LBB3_7: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lharx 9, 0, 12
-; CHECK-NEXT:    cmpw 9, 5
-; CHECK-NEXT:    bne 0, .LBB3_9
-; CHECK-NEXT:  # %bb.8: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    sthcx. 8, 0, 12
-; CHECK-NEXT:    bne 0, .LBB3_7
-; CHECK-NEXT:  .LBB3_9: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    sth 9, ss@toc@l(7)
-; CHECK-NEXT:    lbz 7, sc@toc@l(4)
-; CHECK-NEXT:    lbz 5, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:    extsb 8, 7
-; CHECK-NEXT:    addis 7, 2, us@toc@ha
-; CHECK-NEXT:    addi 11, 7, us@toc@l
-; CHECK-NEXT:  .LBB3_10: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lharx 9, 0, 11
-; CHECK-NEXT:    cmpw 9, 5
-; CHECK-NEXT:    bne 0, .LBB3_12
-; CHECK-NEXT:  # %bb.11: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    sthcx. 8, 0, 11
-; CHECK-NEXT:    bne 0, .LBB3_10
-; CHECK-NEXT:  .LBB3_12: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    sth 9, us@toc@l(7)
-; CHECK-NEXT:    lbz 7, sc@toc@l(4)
-; CHECK-NEXT:    lbz 5, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:    extsb 8, 7
-; CHECK-NEXT:    addis 7, 2, si@toc@ha
-; CHECK-NEXT:    addi 10, 7, si@toc@l
-; CHECK-NEXT:  .LBB3_13: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lwarx 9, 0, 10
-; CHECK-NEXT:    cmpw 9, 5
-; CHECK-NEXT:    bne 0, .LBB3_15
-; CHECK-NEXT:  # %bb.14: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stwcx. 8, 0, 10
-; CHECK-NEXT:    bne 0, .LBB3_13
-; CHECK-NEXT:  .LBB3_15: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    stw 9, si@toc@l(7)
-; CHECK-NEXT:    lbz 5, sc@toc@l(4)
-; CHECK-NEXT:    lbz 7, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:    extsb 8, 5
-; CHECK-NEXT:    addis 5, 2, ui@toc@ha
-; CHECK-NEXT:    addi 9, 5, ui@toc@l
-; CHECK-NEXT:  .LBB3_16: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lwarx 30, 0, 9
-; CHECK-NEXT:    cmpw 30, 7
-; CHECK-NEXT:    bne 0, .LBB3_18
-; CHECK-NEXT:  # %bb.17: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stwcx. 8, 0, 9
-; CHECK-NEXT:    bne 0, .LBB3_16
-; CHECK-NEXT:  .LBB3_18: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    stw 30, ui@toc@l(5)
-; CHECK-NEXT:    addis 30, 2, sll@toc@ha
-; CHECK-NEXT:    lbz 8, sc@toc@l(4)
-; CHECK-NEXT:    lbz 7, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:    extsb 29, 8
-; CHECK-NEXT:    addi 8, 30, sll@toc@l
-; CHECK-NEXT:  .LBB3_19: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    ldarx 28, 0, 8
-; CHECK-NEXT:    cmpd 28, 7
-; CHECK-NEXT:    bne 0, .LBB3_21
-; CHECK-NEXT:  # %bb.20: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stdcx. 29, 0, 8
-; CHECK-NEXT:    bne 0, .LBB3_19
-; CHECK-NEXT:  .LBB3_21: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    addis 29, 2, ull@toc@ha
-; CHECK-NEXT:    std 28, sll@toc@l(30)
-; CHECK-NEXT:    lbz 7, sc@toc@l(4)
-; CHECK-NEXT:    lbz 30, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:    extsb 28, 7
-; CHECK-NEXT:    addi 7, 29, ull@toc@l
-; CHECK-NEXT:  .LBB3_22: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    ldarx 27, 0, 7
-; CHECK-NEXT:    cmpd 27, 30
-; CHECK-NEXT:    bne 0, .LBB3_24
-; CHECK-NEXT:  # %bb.23: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stdcx. 28, 0, 7
-; CHECK-NEXT:    bne 0, .LBB3_22
-; CHECK-NEXT:  .LBB3_24: # %entry
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    std 27, ull@toc@l(29)
-; CHECK-NEXT:    lbz 30, uc@toc@l(3)
-; CHECK-NEXT:    lbz 29, sc@toc@l(4)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_25: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lbarx 28, 0, 0
-; CHECK-NEXT:    cmpw 28, 30
-; CHECK-NEXT:    bne 0, .LBB3_27
-; CHECK-NEXT:  # %bb.26: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stbcx. 29, 0, 0
-; CHECK-NEXT:    bne 0, .LBB3_25
-; CHECK-NEXT:  .LBB3_27: # %entry
-; CHECK-NEXT:    xor 0, 28, 30
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    lbz 30, sc@toc@l(4)
-; CHECK-NEXT:    cntlzw 0, 0
-; CHECK-NEXT:    srwi 0, 0, 5
-; CHECK-NEXT:    stw 0, ui@toc@l(5)
-; CHECK-NEXT:    lbz 0, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_28: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lbarx 29, 0, 6
-; CHECK-NEXT:    cmpw 29, 0
-; CHECK-NEXT:    bne 0, .LBB3_30
-; CHECK-NEXT:  # %bb.29: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stbcx. 30, 0, 6
-; CHECK-NEXT:    bne 0, .LBB3_28
-; CHECK-NEXT:  .LBB3_30: # %entry
-; CHECK-NEXT:    xor 6, 29, 0
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    lbz 0, sc@toc@l(4)
-; CHECK-NEXT:    cntlzw 6, 6
-; CHECK-NEXT:    extsb 0, 0
-; CHECK-NEXT:    srwi 6, 6, 5
-; CHECK-NEXT:    stw 6, ui@toc@l(5)
-; CHECK-NEXT:    lbz 6, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_31: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lharx 30, 0, 12
-; CHECK-NEXT:    cmpw 30, 6
-; CHECK-NEXT:    bne 0, .LBB3_33
-; CHECK-NEXT:  # %bb.32: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    sthcx. 0, 0, 12
-; CHECK-NEXT:    bne 0, .LBB3_31
-; CHECK-NEXT:  .LBB3_33: # %entry
-; CHECK-NEXT:    xor 6, 30, 6
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    lbz 12, sc@toc@l(4)
-; CHECK-NEXT:    cntlzw 6, 6
-; CHECK-NEXT:    extsb 12, 12
-; CHECK-NEXT:    srwi 6, 6, 5
-; CHECK-NEXT:    stw 6, ui@toc@l(5)
-; CHECK-NEXT:    lbz 6, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_34: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lharx 0, 0, 11
-; CHECK-NEXT:    cmpw 0, 6
-; CHECK-NEXT:    bne 0, .LBB3_36
-; CHECK-NEXT:  # %bb.35: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    sthcx. 12, 0, 11
-; CHECK-NEXT:    bne 0, .LBB3_34
-; CHECK-NEXT:  .LBB3_36: # %entry
-; CHECK-NEXT:    xor 6, 0, 6
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    lbz 11, sc@toc@l(4)
-; CHECK-NEXT:    cntlzw 6, 6
-; CHECK-NEXT:    extsb 11, 11
-; CHECK-NEXT:    srwi 6, 6, 5
-; CHECK-NEXT:    stw 6, ui@toc@l(5)
-; CHECK-NEXT:    lbz 6, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_37: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lwarx 12, 0, 10
-; CHECK-NEXT:    cmpw 12, 6
-; CHECK-NEXT:    bne 0, .LBB3_39
-; CHECK-NEXT:  # %bb.38: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stwcx. 11, 0, 10
-; CHECK-NEXT:    bne 0, .LBB3_37
-; CHECK-NEXT:  .LBB3_39: # %entry
-; CHECK-NEXT:    xor 6, 12, 6
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    lbz 10, sc@toc@l(4)
-; CHECK-NEXT:    cntlzw 6, 6
-; CHECK-NEXT:    extsb 10, 10
-; CHECK-NEXT:    srwi 6, 6, 5
-; CHECK-NEXT:    stw 6, ui@toc@l(5)
-; CHECK-NEXT:    lbz 6, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_40: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    lwarx 11, 0, 9
-; CHECK-NEXT:    cmpw 11, 6
-; CHECK-NEXT:    bne 0, .LBB3_42
-; CHECK-NEXT:  # %bb.41: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stwcx. 10, 0, 9
-; CHECK-NEXT:    bne 0, .LBB3_40
-; CHECK-NEXT:  .LBB3_42: # %entry
-; CHECK-NEXT:    xor 6, 11, 6
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    lbz 9, sc@toc@l(4)
-; CHECK-NEXT:    cntlzw 6, 6
-; CHECK-NEXT:    extsb 9, 9
-; CHECK-NEXT:    srwi 6, 6, 5
-; CHECK-NEXT:    stw 6, ui@toc@l(5)
-; CHECK-NEXT:    lbz 6, uc@toc@l(3)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_43: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    ldarx 10, 0, 8
-; CHECK-NEXT:    cmpd 10, 6
-; CHECK-NEXT:    bne 0, .LBB3_45
-; CHECK-NEXT:  # %bb.44: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stdcx. 9, 0, 8
-; CHECK-NEXT:    bne 0, .LBB3_43
-; CHECK-NEXT:  .LBB3_45: # %entry
-; CHECK-NEXT:    xor 6, 10, 6
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    lbz 4, sc@toc@l(4)
-; CHECK-NEXT:    lbz 3, uc@toc@l(3)
-; CHECK-NEXT:    cntlzd 6, 6
-; CHECK-NEXT:    extsb 4, 4
-; CHECK-NEXT:    rldicl 6, 6, 58, 63
-; CHECK-NEXT:    stw 6, ui@toc@l(5)
-; CHECK-NEXT:    sync
-; CHECK-NEXT:  .LBB3_46: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    ldarx 6, 0, 7
-; CHECK-NEXT:    cmpd 6, 3
-; CHECK-NEXT:    bne 0, .LBB3_48
-; CHECK-NEXT:  # %bb.47: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stdcx. 4, 0, 7
-; CHECK-NEXT:    bne 0, .LBB3_46
-; CHECK-NEXT:  .LBB3_48: # %entry
-; CHECK-NEXT:    xor 3, 6, 3
-; CHECK-NEXT:    lwsync
-; CHECK-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 29, -24(1) # 8-byte Folded Reload
-; CHECK-NEXT:    cntlzd 3, 3
-; CHECK-NEXT:    ld 28, -32(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 27, -40(1) # 8-byte Folded Reload
-; CHECK-NEXT:    rldicl 3, 3, 58, 63
-; CHECK-NEXT:    stw 3, ui@toc@l(5)
-; CHECK-NEXT:    blr
+; CHECK-NEXT:   addis 4, 2, sc@toc@ha
+; CHECK-NEXT:   addis 3, 2, uc@toc@ha
+; CHECK-NEXT:   std 27, -40(1)                          # 8-byte Folded Spill
+; CHECK-NEXT:   std 28, -32(1)                          # 8-byte Folded Spill
+; CHECK-NEXT:   std 29, -24(1)                          # 8-byte Folded Spill
+; CHECK-NEXT:   std 30, -16(1)                          # 8-byte Folded Spill
+; CHECK-NEXT:   addi 6, 4, sc@toc@l
+; CHECK-NEXT:   lbz 7, uc@toc@l(3)
+; CHECK-NEXT:   lbz 8, sc@toc@l(4)
+; CHECK-NEXT:   lbarx 5, 0, 6
+; CHECK-NEXT:   clrlwi  9, 5, 24
+; CHECK-NEXT:   cmplw   9, 7
+; CHECK-NEXT:   bne     0, .LBB3_4
+; CHECK-NEXT: # %bb.1:                                # %cmpxchg.fencedstore276
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_2:                                # %cmpxchg.trystore275
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stbcx. 8, 0, 6
+; CHECK-NEXT:   beq     0, .LBB3_4
+; CHECK-NEXT: # %bb.3:                                # %cmpxchg.releasedload274
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_2 Depth=1
+; CHECK-NEXT:   lbarx 5, 0, 6
+; CHECK-NEXT:   clrlwi  9, 5, 24
+; CHECK-NEXT:   cmplw   9, 7
+; CHECK-NEXT:   beq     0, .LBB3_2
+; CHECK-NEXT: .LBB3_4:                                # %cmpxchg.nostore272
+; CHECK-NEXT:   addi 7, 3, uc@toc@l
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   stb 5, sc@toc@l(4)
+; CHECK-NEXT:   lbz 9, uc@toc@l(3)
+; CHECK-NEXT:   lbarx 8, 0, 7
+; CHECK-NEXT:   clrlwi  10, 8, 24
+; CHECK-NEXT:   cmplw   10, 9
+; CHECK-NEXT:   bne     0, .LBB3_8
+; CHECK-NEXT: # %bb.5:                                # %cmpxchg.fencedstore257
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   clrlwi  5, 5, 24
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_6:                                # %cmpxchg.trystore256
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stbcx. 5, 0, 7
+; CHECK-NEXT:   beq     0, .LBB3_8
+; CHECK-NEXT: # %bb.7:                                # %cmpxchg.releasedload255
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_6 Depth=1
+; CHECK-NEXT:   lbarx 8, 0, 7
+; CHECK-NEXT:   clrlwi  10, 8, 24
+; CHECK-NEXT:   cmplw   10, 9
+; CHECK-NEXT:   beq     0, .LBB3_6
+; CHECK-NEXT: .LBB3_8:                                # %cmpxchg.nostore253
+; CHECK-NEXT:   addis 5, 2, ss@toc@ha
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   stb 8, uc@toc@l(3)
+; CHECK-NEXT:   clrlwi  10, 8, 24
+; CHECK-NEXT:   lbz 11, sc@toc@l(4)
+; CHECK-NEXT:   addi 8, 5, ss@toc@l
+; CHECK-NEXT:   lharx 9, 0, 8
+; CHECK-NEXT:   clrlwi  12, 9, 16
+; CHECK-NEXT:   cmplw   12, 10
+; CHECK-NEXT:   bne     0, .LBB3_12
+; CHECK-NEXT: # %bb.9:                                # %cmpxchg.fencedstore238
+; CHECK-NEXT:   extsb 11, 11
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   clrlwi  11, 11, 16
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_10:                               # %cmpxchg.trystore237
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   sthcx. 11, 0, 8
+; CHECK-NEXT:   beq     0, .LBB3_12
+; CHECK-NEXT: # %bb.11:                               # %cmpxchg.releasedload236
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_10 Depth=1
+; CHECK-NEXT:   lharx 9, 0, 8
+; CHECK-NEXT:   clrlwi  12, 9, 16
+; CHECK-NEXT:   cmplw   12, 10
+; CHECK-NEXT:   beq     0, .LBB3_10
+; CHECK-NEXT: .LBB3_12:                               # %cmpxchg.nostore234
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   sth 9, ss@toc@l(5)
+; CHECK-NEXT:   addis 5, 2, us@toc@ha
+; CHECK-NEXT:   lbz 11, uc@toc@l(3)
+; CHECK-NEXT:   lbz 12, sc@toc@l(4)
+; CHECK-NEXT:   addi 9, 5, us@toc@l
+; CHECK-NEXT:   lharx 10, 0, 9
+; CHECK-NEXT:   clrlwi  0, 10, 16
+; CHECK-NEXT:   cmplw   0, 11
+; CHECK-NEXT:   bne     0, .LBB3_16
+; CHECK-NEXT: # %bb.13:                               # %cmpxchg.fencedstore219
+; CHECK-NEXT:   extsb 12, 12
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   clrlwi  12, 12, 16
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_14:                               # %cmpxchg.trystore218
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   sthcx. 12, 0, 9
+; CHECK-NEXT:   beq     0, .LBB3_16
+; CHECK-NEXT: # %bb.15:                               # %cmpxchg.releasedload217
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_14 Depth=1
+; CHECK-NEXT:   lharx 10, 0, 9
+; CHECK-NEXT:   clrlwi  0, 10, 16
+; CHECK-NEXT:   cmplw   0, 11
+; CHECK-NEXT:   beq     0, .LBB3_14
+; CHECK-NEXT: .LBB3_16:                               # %cmpxchg.nostore215
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   sth 10, us@toc@l(5)
+; CHECK-NEXT:   addis 5, 2, si@toc@ha
+; CHECK-NEXT:   lbz 12, uc@toc@l(3)
+; CHECK-NEXT:   lbz 0, sc@toc@l(4)
+; CHECK-NEXT:   addi 10, 5, si@toc@l
+; CHECK-NEXT:   lwarx 11, 0, 10
+; CHECK-NEXT:   cmplw   11, 12
+; CHECK-NEXT:   bne     0, .LBB3_20
+; CHECK-NEXT: # %bb.17:                               # %cmpxchg.fencedstore200
+; CHECK-NEXT:   extsb 0, 0
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_18:                               # %cmpxchg.trystore199
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stwcx. 0, 0, 10
+; CHECK-NEXT:   beq     0, .LBB3_20
+; CHECK-NEXT: # %bb.19:                               # %cmpxchg.releasedload198
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_18 Depth=1
+; CHECK-NEXT:   lwarx 11, 0, 10
+; CHECK-NEXT:   cmplw   11, 12
+; CHECK-NEXT:   beq     0, .LBB3_18
+; CHECK-NEXT: .LBB3_20:                               # %cmpxchg.nostore196
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   stw 11, si@toc@l(5)
+; CHECK-NEXT:   addis 5, 2, ui@toc@ha
+; CHECK-NEXT:   lbz 0, uc@toc@l(3)
+; CHECK-NEXT:   lbz 30, sc@toc@l(4)
+; CHECK-NEXT:   addi 11, 5, ui@toc@l
+; CHECK-NEXT:   lwarx 12, 0, 11
+; CHECK-NEXT:   cmplw   12, 0
+; CHECK-NEXT:   bne     0, .LBB3_24
+; CHECK-NEXT: # %bb.21:                               # %cmpxchg.fencedstore181
+; CHECK-NEXT:   extsb 30, 30
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_22:                               # %cmpxchg.trystore180
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stwcx. 30, 0, 11
+; CHECK-NEXT:   beq     0, .LBB3_24
+; CHECK-NEXT: # %bb.23:                               # %cmpxchg.releasedload179
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_22 Depth=1
+; CHECK-NEXT:   lwarx 12, 0, 11
+; CHECK-NEXT:   cmplw   12, 0
+; CHECK-NEXT:   beq     0, .LBB3_22
+; CHECK-NEXT: .LBB3_24:                               # %cmpxchg.nostore177
+; CHECK-NEXT:   addis 30, 2, sll@toc@ha
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   stw 12, ui@toc@l(5)
+; CHECK-NEXT:   lbz 29, uc@toc@l(3)
+; CHECK-NEXT:   lbz 28, sc@toc@l(4)
+; CHECK-NEXT:   addi 12, 30, sll@toc@l
+; CHECK-NEXT:   ldarx 0, 0, 12
+; CHECK-NEXT:   cmpld   0, 29
+; CHECK-NEXT:   bne     0, .LBB3_28
+; CHECK-NEXT: # %bb.25:                               # %cmpxchg.fencedstore162
+; CHECK-NEXT:   extsb 28, 28
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_26:                               # %cmpxchg.trystore161
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stdcx. 28, 0, 12
+; CHECK-NEXT:   beq     0, .LBB3_28
+; CHECK-NEXT: # %bb.27:                               # %cmpxchg.releasedload160
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_26 Depth=1
+; CHECK-NEXT:   ldarx 0, 0, 12
+; CHECK-NEXT:   cmpld   0, 29
+; CHECK-NEXT:   beq     0, .LBB3_26
+; CHECK-NEXT: .LBB3_28:                               # %cmpxchg.nostore158
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   std 0, sll@toc@l(30)
+; CHECK-NEXT:   addis 30, 2, ull@toc@ha
+; CHECK-NEXT:   lbz 28, uc@toc@l(3)
+; CHECK-NEXT:   lbz 27, sc@toc@l(4)
+; CHECK-NEXT:   addi 0, 30, ull@toc@l
+; CHECK-NEXT:   ldarx 29, 0, 0
+; CHECK-NEXT:   cmpld   29, 28
+; CHECK-NEXT:   bne     0, .LBB3_32
+; CHECK-NEXT: # %bb.29:                               # %cmpxchg.fencedstore143
+; CHECK-NEXT:   extsb 27, 27
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_30:                               # %cmpxchg.trystore142
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stdcx. 27, 0, 0
+; CHECK-NEXT:   beq     0, .LBB3_32
+; CHECK-NEXT: # %bb.31:                               # %cmpxchg.releasedload141
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_30 Depth=1
+; CHECK-NEXT:   ldarx 29, 0, 0
+; CHECK-NEXT:   cmpld   29, 28
+; CHECK-NEXT:   beq     0, .LBB3_30
+; CHECK-NEXT: .LBB3_32:                               # %cmpxchg.nostore139
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   std 29, ull@toc@l(30)
+; CHECK-NEXT:   lbz 30, uc@toc@l(3)
+; CHECK-NEXT:   lbz 29, sc@toc@l(4)
+; CHECK-NEXT:   lbarx 28, 0, 6
+; CHECK-NEXT:   clrlwi  28, 28, 24
+; CHECK-NEXT:   cmplw   28, 30
+; CHECK-NEXT:   bne     0, .LBB3_36
+; CHECK-NEXT: # %bb.33:                               # %cmpxchg.fencedstore124
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_34:                               # %cmpxchg.trystore123
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stbcx. 29, 0, 6
+; CHECK-NEXT:   beq     0, .LBB3_37
+; CHECK-NEXT: # %bb.35:                               # %cmpxchg.releasedload122
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_34 Depth=1
+; CHECK-NEXT:   lbarx 28, 0, 6
+; CHECK-NEXT:   clrlwi  28, 28, 24
+; CHECK-NEXT:   cmplw   28, 30
+; CHECK-NEXT:   beq     0, .LBB3_34
+; CHECK-NEXT: .LBB3_36:                               # %cmpxchg.nostore120
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_38
+; CHECK-NEXT: .LBB3_37:                               # %cmpxchg.success121
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_38:                               # %cmpxchg.end118
+; CHECK-NEXT:   li 6, 0
+; CHECK-NEXT:   li 30, 1
+; CHECK-NEXT:   isel 6, 30, 6, 20
+; CHECK-NEXT:   lbz 30, sc@toc@l(4)
+; CHECK-NEXT:   stw 6, ui@toc@l(5)
+; CHECK-NEXT:   lbz 6, uc@toc@l(3)
+; CHECK-NEXT:   lbarx 29, 0, 7
+; CHECK-NEXT:   clrlwi  29, 29, 24
+; CHECK-NEXT:   cmplw   29, 6
+; CHECK-NEXT:   bne     0, .LBB3_42
+; CHECK-NEXT: # %bb.39:                               # %cmpxchg.fencedstore105
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_40:                               # %cmpxchg.trystore104
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stbcx. 30, 0, 7
+; CHECK-NEXT:   beq     0, .LBB3_43
+; CHECK-NEXT: # %bb.41:                               # %cmpxchg.releasedload103
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_40 Depth=1
+; CHECK-NEXT:   lbarx 29, 0, 7
+; CHECK-NEXT:   clrlwi  29, 29, 24
+; CHECK-NEXT:   cmplw   29, 6
+; CHECK-NEXT:   beq     0, .LBB3_40
+; CHECK-NEXT: .LBB3_42:                               # %cmpxchg.nostore101
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_44
+; CHECK-NEXT: .LBB3_43:                               # %cmpxchg.success102
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_44:                               # %cmpxchg.end99
+; CHECK-NEXT:   li 6, 0
+; CHECK-NEXT:   li 7, 1
+; CHECK-NEXT:   isel 6, 7, 6, 20
+; CHECK-NEXT:   lbz 7, sc@toc@l(4)
+; CHECK-NEXT:   stw 6, ui@toc@l(5)
+; CHECK-NEXT:   lbz 6, uc@toc@l(3)
+; CHECK-NEXT:   lharx 30, 0, 8
+; CHECK-NEXT:   clrlwi  30, 30, 16
+; CHECK-NEXT:   cmplw   30, 6
+; CHECK-NEXT:   bne     0, .LBB3_48
+; CHECK-NEXT: # %bb.45:                               # %cmpxchg.fencedstore86
+; CHECK-NEXT:   extsb 7, 7
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   clrlwi  7, 7, 16
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_46:                               # %cmpxchg.trystore85
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   sthcx. 7, 0, 8
+; CHECK-NEXT:   beq     0, .LBB3_49
+; CHECK-NEXT: # %bb.47:                               # %cmpxchg.releasedload84
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_46 Depth=1
+; CHECK-NEXT:   lharx 30, 0, 8
+; CHECK-NEXT:   clrlwi  30, 30, 16
+; CHECK-NEXT:   cmplw   30, 6
+; CHECK-NEXT:   beq     0, .LBB3_46
+; CHECK-NEXT: .LBB3_48:                               # %cmpxchg.nostore82
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_50
+; CHECK-NEXT: .LBB3_49:                               # %cmpxchg.success83
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_50:                               # %cmpxchg.end80
+; CHECK-NEXT:   li 6, 0
+; CHECK-NEXT:   li 7, 1
+; CHECK-NEXT:   isel 6, 7, 6, 20
+; CHECK-NEXT:   lbz 7, sc@toc@l(4)
+; CHECK-NEXT:   stw 6, ui@toc@l(5)
+; CHECK-NEXT:   lbz 6, uc@toc@l(3)
+; CHECK-NEXT:   lharx 8, 0, 9
+; CHECK-NEXT:   clrlwi  8, 8, 16
+; CHECK-NEXT:   cmplw   8, 6
+; CHECK-NEXT:   bne     0, .LBB3_54
+; CHECK-NEXT: # %bb.51:                               # %cmpxchg.fencedstore67
+; CHECK-NEXT:   extsb 7, 7
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   clrlwi  7, 7, 16
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_52:                               # %cmpxchg.trystore66
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   sthcx. 7, 0, 9
+; CHECK-NEXT:   beq     0, .LBB3_55
+; CHECK-NEXT: # %bb.53:                               # %cmpxchg.releasedload65
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_52 Depth=1
+; CHECK-NEXT:   lharx 8, 0, 9
+; CHECK-NEXT:   clrlwi  8, 8, 16
+; CHECK-NEXT:   cmplw   8, 6
+; CHECK-NEXT:   beq     0, .LBB3_52
+; CHECK-NEXT: .LBB3_54:                               # %cmpxchg.nostore63
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_56
+; CHECK-NEXT: .LBB3_55:                               # %cmpxchg.success64
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_56:                               # %cmpxchg.end61
+; CHECK-NEXT:   li 6, 0
+; CHECK-NEXT:   li 7, 1
+; CHECK-NEXT:   isel 6, 7, 6, 20
+; CHECK-NEXT:   lbz 7, sc@toc@l(4)
+; CHECK-NEXT:   stw 6, ui@toc@l(5)
+; CHECK-NEXT:   lbz 6, uc@toc@l(3)
+; CHECK-NEXT:   lwarx 8, 0, 10
+; CHECK-NEXT:   cmplw   8, 6
+; CHECK-NEXT:   bne     0, .LBB3_60
+; CHECK-NEXT: # %bb.57:                               # %cmpxchg.fencedstore48
+; CHECK-NEXT:   extsb 7, 7
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_58:                               # %cmpxchg.trystore47
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stwcx. 7, 0, 10
+; CHECK-NEXT:   beq     0, .LBB3_61
+; CHECK-NEXT: # %bb.59:                               # %cmpxchg.releasedload46
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_58 Depth=1
+; CHECK-NEXT:   lwarx 8, 0, 10
+; CHECK-NEXT:   cmplw   8, 6
+; CHECK-NEXT:   beq     0, .LBB3_58
+; CHECK-NEXT: .LBB3_60:                               # %cmpxchg.nostore44
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_62
+; CHECK-NEXT: .LBB3_61:                               # %cmpxchg.success45
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_62:                               # %cmpxchg.end42
+; CHECK-NEXT:   li 6, 0
+; CHECK-NEXT:   li 7, 1
+; CHECK-NEXT:   isel 6, 7, 6, 20
+; CHECK-NEXT:   lbz 7, sc@toc@l(4)
+; CHECK-NEXT:   stw 6, ui@toc@l(5)
+; CHECK-NEXT:   lbz 6, uc@toc@l(3)
+; CHECK-NEXT:   lwarx 8, 0, 11
+; CHECK-NEXT:   cmplw   8, 6
+; CHECK-NEXT:   bne     0, .LBB3_66
+; CHECK-NEXT: # %bb.63:                               # %cmpxchg.fencedstore29
+; CHECK-NEXT:   extsb 7, 7
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_64:                               # %cmpxchg.trystore28
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stwcx. 7, 0, 11
+; CHECK-NEXT:   beq     0, .LBB3_67
+; CHECK-NEXT: # %bb.65:                               # %cmpxchg.releasedload27
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_64 Depth=1
+; CHECK-NEXT:   lwarx 8, 0, 11
+; CHECK-NEXT:   cmplw   8, 6
+; CHECK-NEXT:   beq     0, .LBB3_64
+; CHECK-NEXT: .LBB3_66:                               # %cmpxchg.nostore25
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_68
+; CHECK-NEXT: .LBB3_67:                               # %cmpxchg.success26
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_68:                               # %cmpxchg.end23
+; CHECK-NEXT:   li 6, 0
+; CHECK-NEXT:   li 7, 1
+; CHECK-NEXT:   isel 6, 7, 6, 20
+; CHECK-NEXT:   lbz 7, sc@toc@l(4)
+; CHECK-NEXT:   stw 6, ui@toc@l(5)
+; CHECK-NEXT:   lbz 6, uc@toc@l(3)
+; CHECK-NEXT:   ldarx 8, 0, 12
+; CHECK-NEXT:   cmpld   8, 6
+; CHECK-NEXT:   bne     0, .LBB3_72
+; CHECK-NEXT: # %bb.69:                               # %cmpxchg.fencedstore10
+; CHECK-NEXT:   extsb 7, 7
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_70:                               # %cmpxchg.trystore9
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stdcx. 7, 0, 12
+; CHECK-NEXT:   beq     0, .LBB3_73
+; CHECK-NEXT: # %bb.71:                               # %cmpxchg.releasedload8
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_70 Depth=1
+; CHECK-NEXT:   ldarx 8, 0, 12
+; CHECK-NEXT:   cmpld   8, 6
+; CHECK-NEXT:   beq     0, .LBB3_70
+; CHECK-NEXT: .LBB3_72:                               # %cmpxchg.nostore6
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_74
+; CHECK-NEXT: .LBB3_73:                               # %cmpxchg.success7
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_74:                               # %cmpxchg.end4
+; CHECK-NEXT:   li 6, 0
+; CHECK-NEXT:   li 7, 1
+; CHECK-NEXT:   lbz 3, uc@toc@l(3)
+; CHECK-NEXT:   lbz 4, sc@toc@l(4)
+; CHECK-NEXT:   isel 6, 7, 6, 20
+; CHECK-NEXT:   stw 6, ui@toc@l(5)
+; CHECK-NEXT:   ldarx 6, 0, 0
+; CHECK-NEXT:   cmpld   6, 3
+; CHECK-NEXT:   bne     0, .LBB3_78
+; CHECK-NEXT: # %bb.75:                               # %cmpxchg.fencedstore
+; CHECK-NEXT:   extsb 4, 4
+; CHECK-NEXT:   sync
+; CHECK-NEXT:   .p2align        5
+; CHECK-NEXT: .LBB3_76:                               # %cmpxchg.trystore
+; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:   stdcx. 4, 0, 0
+; CHECK-NEXT:   beq     0, .LBB3_79
+; CHECK-NEXT: # %bb.77:                               # %cmpxchg.releasedload
+; CHECK-NEXT:                                         #   in Loop: Header=BB3_76 Depth=1
+; CHECK-NEXT:   ldarx 6, 0, 0
+; CHECK-NEXT:   cmpld   6, 3
+; CHECK-NEXT:   beq     0, .LBB3_76
+; CHECK-NEXT: .LBB3_78:                               # %cmpxchg.nostore
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT:   b .LBB3_80
+; CHECK-NEXT: .LBB3_79:                               # %cmpxchg.success
+; CHECK-NEXT:   lwsync
+; CHECK-NEXT:   creqv 20, 20, 20
+; CHECK-NEXT: .LBB3_80:                               # %cmpxchg.end
+; CHECK-NEXT:   li 3, 0
+; CHECK-NEXT:   li 4, 1
+; CHECK-NEXT:   ld 30, -16(1)                           # 8-byte Folded Reload
+; CHECK-NEXT:   ld 29, -24(1)                           # 8-byte Folded Reload
+; CHECK-NEXT:   ld 28, -32(1)                           # 8-byte Folded Reload
+; CHECK-NEXT:   ld 27, -40(1)                           # 8-byte Folded Reload
+; CHECK-NEXT:   isel 3, 4, 3, 20
+; CHECK-NEXT:   stw 3, ui@toc@l(5)
+; CHECK-NEXT:   blr
 ;
 ; AIX32-LABEL: test_compare_and_swap:
 ; AIX32:       # %bb.0: # %entry
 ; AIX32-NEXT:    mflr 0
-; AIX32-NEXT:    stwu 1, -128(1)
-; AIX32-NEXT:    stw 0, 136(1)
-; AIX32-NEXT:    stw 28, 112(1) # 4-byte Folded Spill
-; AIX32-NEXT:    lwz 28, L..C0(2) # @sc
-; AIX32-NEXT:    stw 29, 116(1) # 4-byte Folded Spill
-; AIX32-NEXT:    lwz 29, L..C1(2) # @uc
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    rlwinm 5, 28, 3, 27, 28
-; AIX32-NEXT:    stw 21, 84(1) # 4-byte Folded Spill
-; AIX32-NEXT:    lbz 4, 0(28)
-; AIX32-NEXT:    stw 17, 68(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 18, 72(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 19, 76(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 20, 80(1) # 4-byte Folded Spill
-; AIX32-NEXT:    xori 21, 5, 24
-; AIX32-NEXT:    stw 22, 88(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 23, 92(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 24, 96(1) # 4-byte Folded Spill
-; AIX32-NEXT:    slw 5, 3, 21
-; AIX32-NEXT:    li 3, 255
-; AIX32-NEXT:    slw 4, 4, 21
-; AIX32-NEXT:    stw 25, 100(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 26, 104(1) # 4-byte Folded Spill
-; AIX32-NEXT:    slw 3, 3, 21
-; AIX32-NEXT:    stw 27, 108(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 30, 120(1) # 4-byte Folded Spill
-; AIX32-NEXT:    stw 31, 124(1) # 4-byte Folded Spill
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    rlwinm 18, 28, 0, 0, 29
-; AIX32-NEXT:    and 4, 4, 3
-; AIX32-NEXT:    and 5, 5, 3
-; AIX32-NEXT:  L..BB3_1: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 7, 0, 18
-; AIX32-NEXT:    and 6, 7, 3
-; AIX32-NEXT:    cmpw 6, 5
-; AIX32-NEXT:    bne 0, L..BB3_3
-; AIX32-NEXT:  # %bb.2: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 7, 7, 3
-; AIX32-NEXT:    or 7, 7, 4
-; AIX32-NEXT:    stwcx. 7, 0, 18
-; AIX32-NEXT:    bne 0, L..BB3_1
-; AIX32-NEXT:  L..BB3_3: # %entry
-; AIX32-NEXT:    rlwinm 5, 29, 3, 27, 28
-; AIX32-NEXT:    srw 3, 6, 21
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    lbz 4, 0(29)
-; AIX32-NEXT:    rlwinm 20, 29, 0, 0, 29
-; AIX32-NEXT:    xori 25, 5, 24
-; AIX32-NEXT:    slw 5, 3, 25
-; AIX32-NEXT:    stb 3, 0(28)
-; AIX32-NEXT:    li 3, 255
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    slw 6, 4, 25
-; AIX32-NEXT:    slw 3, 3, 25
-; AIX32-NEXT:    and 4, 5, 3
-; AIX32-NEXT:    and 5, 6, 3
-; AIX32-NEXT:  L..BB3_4: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 7, 0, 20
-; AIX32-NEXT:    and 6, 7, 3
-; AIX32-NEXT:    cmpw 6, 5
-; AIX32-NEXT:    bne 0, L..BB3_6
-; AIX32-NEXT:  # %bb.5: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 7, 7, 3
-; AIX32-NEXT:    or 7, 7, 4
-; AIX32-NEXT:    stwcx. 7, 0, 20
-; AIX32-NEXT:    bne 0, L..BB3_4
-; AIX32-NEXT:  L..BB3_6: # %entry
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    srw 4, 6, 25
-; AIX32-NEXT:    lbz 3, 0(28)
-; AIX32-NEXT:    extsb 5, 3
-; AIX32-NEXT:    lwz 3, L..C2(2) # @ss
-; AIX32-NEXT:    stb 4, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    rlwinm 6, 3, 3, 27, 27
-; AIX32-NEXT:    rlwinm 22, 3, 0, 0, 29
-; AIX32-NEXT:    xori 26, 6, 16
-; AIX32-NEXT:    slw 6, 4, 26
-; AIX32-NEXT:    li 4, 0
-; AIX32-NEXT:    slw 5, 5, 26
-; AIX32-NEXT:    ori 4, 4, 65535
-; AIX32-NEXT:    slw 4, 4, 26
-; AIX32-NEXT:    and 5, 5, 4
-; AIX32-NEXT:    and 6, 6, 4
-; AIX32-NEXT:  L..BB3_7: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 8, 0, 22
-; AIX32-NEXT:    and 7, 8, 4
-; AIX32-NEXT:    cmpw 7, 6
-; AIX32-NEXT:    bne 0, L..BB3_9
-; AIX32-NEXT:  # %bb.8: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 8, 8, 4
-; AIX32-NEXT:    or 8, 8, 5
-; AIX32-NEXT:    stwcx. 8, 0, 22
-; AIX32-NEXT:    bne 0, L..BB3_7
-; AIX32-NEXT:  L..BB3_9: # %entry
-; AIX32-NEXT:    srw 4, 7, 26
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    sth 4, 0(3)
-; AIX32-NEXT:    lbz 3, 0(28)
-; AIX32-NEXT:    lbz 4, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    extsb 5, 3
-; AIX32-NEXT:    lwz 3, L..C3(2) # @us
-; AIX32-NEXT:    rlwinm 6, 3, 3, 27, 27
-; AIX32-NEXT:    rlwinm 19, 3, 0, 0, 29
-; AIX32-NEXT:    xori 24, 6, 16
-; AIX32-NEXT:    slw 6, 4, 24
-; AIX32-NEXT:    li 4, 0
-; AIX32-NEXT:    slw 5, 5, 24
-; AIX32-NEXT:    ori 4, 4, 65535
-; AIX32-NEXT:    slw 4, 4, 24
-; AIX32-NEXT:    and 5, 5, 4
-; AIX32-NEXT:    and 6, 6, 4
-; AIX32-NEXT:  L..BB3_10: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 8, 0, 19
-; AIX32-NEXT:    and 7, 8, 4
-; AIX32-NEXT:    cmpw 7, 6
-; AIX32-NEXT:    bne 0, L..BB3_12
-; AIX32-NEXT:  # %bb.11: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 8, 8, 4
-; AIX32-NEXT:    or 8, 8, 5
-; AIX32-NEXT:    stwcx. 8, 0, 19
-; AIX32-NEXT:    bne 0, L..BB3_10
-; AIX32-NEXT:  L..BB3_12: # %entry
-; AIX32-NEXT:    srw 4, 7, 24
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    lwz 17, L..C4(2) # @si
-; AIX32-NEXT:    sth 4, 0(3)
-; AIX32-NEXT:    lbz 4, 0(28)
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    extsb 4, 4
-; AIX32-NEXT:  L..BB3_13: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 5, 0, 17
-; AIX32-NEXT:    cmpw 5, 3
-; AIX32-NEXT:    bne 0, L..BB3_15
-; AIX32-NEXT:  # %bb.14: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    stwcx. 4, 0, 17
-; AIX32-NEXT:    bne 0, L..BB3_13
-; AIX32-NEXT:  L..BB3_15: # %entry
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    stw 5, 0(17)
-; AIX32-NEXT:    lwz 27, L..C5(2) # @ui
-; AIX32-NEXT:    lbz 4, 0(28)
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    extsb 4, 4
-; AIX32-NEXT:  L..BB3_16: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 5, 0, 27
-; AIX32-NEXT:    cmpw 5, 3
-; AIX32-NEXT:    bne 0, L..BB3_18
-; AIX32-NEXT:  # %bb.17: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    stwcx. 4, 0, 27
-; AIX32-NEXT:    bne 0, L..BB3_16
-; AIX32-NEXT:  L..BB3_18: # %entry
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    lwz 31, L..C6(2) # @sll
-; AIX32-NEXT:    stw 5, 0(27)
-; AIX32-NEXT:    lbz 3, 0(28)
-; AIX32-NEXT:    li 23, 0
-; AIX32-NEXT:    addi 4, 1, 56
-; AIX32-NEXT:    li 7, 5
-; AIX32-NEXT:    li 8, 5
-; AIX32-NEXT:    stw 23, 56(1)
-; AIX32-NEXT:    extsb 6, 3
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    srawi 5, 6, 31
-; AIX32-NEXT:    stw 3, 60(1)
-; AIX32-NEXT:    mr 3, 31
-; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:    nop
-; AIX32-NEXT:    lwz 3, 60(1)
-; AIX32-NEXT:    lbz 4, 0(28)
-; AIX32-NEXT:    lwz 30, L..C7(2) # @ull
-; AIX32-NEXT:    li 7, 5
-; AIX32-NEXT:    li 8, 5
-; AIX32-NEXT:    stw 3, 4(31)
-; AIX32-NEXT:    lwz 3, 56(1)
-; AIX32-NEXT:    extsb 6, 4
-; AIX32-NEXT:    addi 4, 1, 56
-; AIX32-NEXT:    srawi 5, 6, 31
-; AIX32-NEXT:    stw 23, 56(1)
-; AIX32-NEXT:    stw 3, 0(31)
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    stw 3, 60(1)
-; AIX32-NEXT:    mr 3, 30
-; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:    nop
-; AIX32-NEXT:    lwz 4, 60(1)
-; AIX32-NEXT:    lwz 3, 56(1)
-; AIX32-NEXT:    stw 4, 4(30)
-; AIX32-NEXT:    lbz 4, 0(28)
-; AIX32-NEXT:    stw 3, 0(30)
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    slw 5, 4, 21
-; AIX32-NEXT:    li 4, 255
-; AIX32-NEXT:    slw 6, 3, 21
-; AIX32-NEXT:    slw 4, 4, 21
-; AIX32-NEXT:    and 5, 5, 4
-; AIX32-NEXT:    and 6, 6, 4
-; AIX32-NEXT:  L..BB3_19: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 8, 0, 18
-; AIX32-NEXT:    and 7, 8, 4
-; AIX32-NEXT:    cmpw 7, 6
-; AIX32-NEXT:    bne 0, L..BB3_21
-; AIX32-NEXT:  # %bb.20: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 8, 8, 4
-; AIX32-NEXT:    or 8, 8, 5
-; AIX32-NEXT:    stwcx. 8, 0, 18
-; AIX32-NEXT:    bne 0, L..BB3_19
-; AIX32-NEXT:  L..BB3_21: # %entry
-; AIX32-NEXT:    srw 4, 7, 21
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    lbz 5, 0(28)
-; AIX32-NEXT:    cmpw 4, 3
-; AIX32-NEXT:    li 3, 1
-; AIX32-NEXT:    iseleq 4, 3, 23
-; AIX32-NEXT:    slw 6, 5, 25
-; AIX32-NEXT:    li 5, 255
-; AIX32-NEXT:    stw 4, 0(27)
-; AIX32-NEXT:    lbz 4, 0(29)
-; AIX32-NEXT:    slw 5, 5, 25
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    slw 7, 4, 25
-; AIX32-NEXT:    and 6, 6, 5
-; AIX32-NEXT:    and 7, 7, 5
-; AIX32-NEXT:  L..BB3_22: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 9, 0, 20
-; AIX32-NEXT:    and 8, 9, 5
-; AIX32-NEXT:    cmpw 8, 7
-; AIX32-NEXT:    bne 0, L..BB3_24
-; AIX32-NEXT:  # %bb.23: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 9, 9, 5
-; AIX32-NEXT:    or 9, 9, 6
-; AIX32-NEXT:    stwcx. 9, 0, 20
-; AIX32-NEXT:    bne 0, L..BB3_22
-; AIX32-NEXT:  L..BB3_24: # %entry
-; AIX32-NEXT:    srw 5, 8, 25
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    cmpw 5, 4
-; AIX32-NEXT:    lbz 5, 0(28)
-; AIX32-NEXT:    iseleq 4, 3, 23
-; AIX32-NEXT:    extsb 5, 5
-; AIX32-NEXT:    stw 4, 0(27)
-; AIX32-NEXT:    lbz 4, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    slw 6, 5, 26
-; AIX32-NEXT:    li 5, 0
-; AIX32-NEXT:    slw 7, 4, 26
-; AIX32-NEXT:    ori 5, 5, 65535
-; AIX32-NEXT:    slw 5, 5, 26
-; AIX32-NEXT:    and 6, 6, 5
-; AIX32-NEXT:    and 7, 7, 5
-; AIX32-NEXT:  L..BB3_25: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 9, 0, 22
-; AIX32-NEXT:    and 8, 9, 5
-; AIX32-NEXT:    cmpw 8, 7
-; AIX32-NEXT:    bne 0, L..BB3_27
-; AIX32-NEXT:  # %bb.26: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 9, 9, 5
-; AIX32-NEXT:    or 9, 9, 6
-; AIX32-NEXT:    stwcx. 9, 0, 22
-; AIX32-NEXT:    bne 0, L..BB3_25
-; AIX32-NEXT:  L..BB3_27: # %entry
-; AIX32-NEXT:    srw 5, 8, 26
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    cmpw 5, 4
-; AIX32-NEXT:    lbz 5, 0(28)
-; AIX32-NEXT:    iseleq 4, 3, 23
-; AIX32-NEXT:    extsb 5, 5
-; AIX32-NEXT:    stw 4, 0(27)
-; AIX32-NEXT:    lbz 4, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    slw 6, 5, 24
-; AIX32-NEXT:    li 5, 0
-; AIX32-NEXT:    slw 7, 4, 24
-; AIX32-NEXT:    ori 5, 5, 65535
-; AIX32-NEXT:    slw 5, 5, 24
-; AIX32-NEXT:    and 6, 6, 5
-; AIX32-NEXT:    and 7, 7, 5
-; AIX32-NEXT:  L..BB3_28: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 9, 0, 19
-; AIX32-NEXT:    and 8, 9, 5
-; AIX32-NEXT:    cmpw 8, 7
-; AIX32-NEXT:    bne 0, L..BB3_30
-; AIX32-NEXT:  # %bb.29: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    andc 9, 9, 5
-; AIX32-NEXT:    or 9, 9, 6
-; AIX32-NEXT:    stwcx. 9, 0, 19
-; AIX32-NEXT:    bne 0, L..BB3_28
-; AIX32-NEXT:  L..BB3_30: # %entry
-; AIX32-NEXT:    srw 5, 8, 24
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    cmpw 5, 4
-; AIX32-NEXT:    lbz 5, 0(28)
-; AIX32-NEXT:    iseleq 4, 3, 23
-; AIX32-NEXT:    stw 4, 0(27)
-; AIX32-NEXT:    lbz 4, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    extsb 5, 5
-; AIX32-NEXT:  L..BB3_31: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 6, 0, 17
-; AIX32-NEXT:    cmpw 1, 6, 4
-; AIX32-NEXT:    bne 1, L..BB3_33
-; AIX32-NEXT:  # %bb.32: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    stwcx. 5, 0, 17
-; AIX32-NEXT:    bne 0, L..BB3_31
-; AIX32-NEXT:  L..BB3_33: # %entry
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    isel 4, 3, 23, 6
-; AIX32-NEXT:    lbz 5, 0(28)
-; AIX32-NEXT:    stw 4, 0(27)
-; AIX32-NEXT:    lbz 4, 0(29)
-; AIX32-NEXT:    sync
-; AIX32-NEXT:    extsb 5, 5
-; AIX32-NEXT:  L..BB3_34: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    lwarx 6, 0, 27
-; AIX32-NEXT:    cmpw 1, 6, 4
-; AIX32-NEXT:    bne 1, L..BB3_36
-; AIX32-NEXT:  # %bb.35: # %entry
-; AIX32-NEXT:    #
-; AIX32-NEXT:    stwcx. 5, 0, 27
-; AIX32-NEXT:    bne 0, L..BB3_34
-; AIX32-NEXT:  L..BB3_36: # %entry
-; AIX32-NEXT:    lwsync
-; AIX32-NEXT:    isel 3, 3, 23, 6
-; AIX32-NEXT:    li 7, 5
-; AIX32-NEXT:    li 8, 5
-; AIX32-NEXT:    lbz 4, 0(28)
-; AIX32-NEXT:    stw 3, 0(27)
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    stw 23, 56(1)
-; AIX32-NEXT:    extsb 6, 4
-; AIX32-NEXT:    addi 4, 1, 56
-; AIX32-NEXT:    stw 3, 60(1)
-; AIX32-NEXT:    mr 3, 31
-; AIX32-NEXT:    srawi 5, 6, 31
-; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:    nop
-; AIX32-NEXT:    lbz 4, 0(28)
-; AIX32-NEXT:    stw 3, 0(27)
-; AIX32-NEXT:    lbz 3, 0(29)
-; AIX32-NEXT:    li 7, 5
-; AIX32-NEXT:    li 8, 5
-; AIX32-NEXT:    extsb 6, 4
-; AIX32-NEXT:    addi 4, 1, 56
-; AIX32-NEXT:    stw 3, 60(1)
-; AIX32-NEXT:    mr 3, 30
-; AIX32-NEXT:    stw 23, 56(1)
-; AIX32-NEXT:    srawi 5, 6, 31
-; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:    nop
-; AIX32-NEXT:    stw 3, 0(27)
-; AIX32-NEXT:    lwz 31, 124(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 30, 120(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 29, 116(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 28, 112(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 27, 108(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 26, 104(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 25, 100(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 24, 96(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 23, 92(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 22, 88(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 21, 84(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 20, 80(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 19, 76(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 18, 72(1) # 4-byte Folded Reload
-; AIX32-NEXT:    lwz 17, 68(1) # 4-byte Folded Reload
-; AIX32-NEXT:    addi 1, 1, 128
-; AIX32-NEXT:    lwz 0, 8(1)
-; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:   stwu 1, -144(1)
+; AIX32-NEXT:   stw 0, 152(1)
+; AIX32-NEXT:   stw 29, 132(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   lwz 29, L..C0(2)                        # @sc
+; AIX32-NEXT:   stw 26, 120(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   not     3, 29
+; AIX32-NEXT:   stw 30, 136(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   lwz 30, L..C1(2)                        # @uc
+; AIX32-NEXT:   lbz 4, 0(30)
+; AIX32-NEXT:   lbz 5, 0(29)
+; AIX32-NEXT:   stw 27, 124(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   rlwinm 27, 29, 0, 0, 29
+; AIX32-NEXT:   stw 14, 72(1)                           # 4-byte Folded Spill
+; AIX32-NEXT:   stw 15, 76(1)                           # 4-byte Folded Spill
+; AIX32-NEXT:   rlwinm 26, 3, 3, 27, 28
+; AIX32-NEXT:   li 3, 255
+; AIX32-NEXT:   slw 3, 3, 26
+; AIX32-NEXT:   stw 16, 80(1)                           # 4-byte Folded Spill
+; AIX32-NEXT:   stw 17, 84(1)                           # 4-byte Folded Spill
+; AIX32-NEXT:   stw 18, 88(1)                           # 4-byte Folded Spill
+; AIX32-NEXT:   stw 19, 92(1)                           # 4-byte Folded Spill
+; AIX32-NEXT:   stw 20, 96(1)                           # 4-byte Folded Spill
+; AIX32-NEXT:   stw 21, 100(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   stw 22, 104(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   stw 23, 108(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   stw 24, 112(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   stw 25, 116(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   stw 28, 128(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   stw 31, 140(1)                          # 4-byte Folded Spill
+; AIX32-NEXT:   not     25, 3
+; AIX32-NEXT:   lwarx 3, 0, 27
+; AIX32-NEXT:   srw 6, 3, 26
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 4
+; AIX32-NEXT:   bne     0, L..BB3_4
+; AIX32-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore289
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   slw 5, 5, 26
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_2:                               # %cmpxchg.trystore288
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 6, 3, 25
+; AIX32-NEXT:   or 6, 6, 5
+; AIX32-NEXT:   stwcx. 6, 0, 27
+; AIX32-NEXT:   beq     0, L..BB3_4
+; AIX32-NEXT:  # %bb.3:                                # %cmpxchg.releasedload287
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_2 Depth=1
+; AIX32-NEXT:   lwarx 3, 0, 27
+; AIX32-NEXT:   srw 6, 3, 26
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 4
+; AIX32-NEXT:   beq     0, L..BB3_2
+; AIX32-NEXT:  L..BB3_4:                               # %cmpxchg.nostore285
+; AIX32-NEXT:   not     4, 30
+; AIX32-NEXT:   srw 5, 3, 26
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   rlwinm 24, 30, 0, 0, 29
+; AIX32-NEXT:   rlwinm 23, 4, 3, 27, 28
+; AIX32-NEXT:   li 4, 255
+; AIX32-NEXT:   stb 5, 0(29)
+; AIX32-NEXT:   slw 4, 4, 23
+; AIX32-NEXT:   not     22, 4
+; AIX32-NEXT:   lwarx 4, 0, 24
+; AIX32-NEXT:   srw 6, 4, 23
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   bne     0, L..BB3_8
+; AIX32-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore256
+; AIX32-NEXT:   clrlwi  5, 5, 24
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   slw 5, 5, 23
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_6:                               # %cmpxchg.trystore255
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 6, 4, 22
+; AIX32-NEXT:   or 6, 6, 5
+; AIX32-NEXT:   stwcx. 6, 0, 24
+; AIX32-NEXT:   beq     0, L..BB3_8
+; AIX32-NEXT:  # %bb.7:                                # %cmpxchg.releasedload254
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_6 Depth=1
+; AIX32-NEXT:   lwarx 4, 0, 24
+; AIX32-NEXT:   srw 6, 4, 23
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   beq     0, L..BB3_6
+; AIX32-NEXT:  L..BB3_8:                               # %cmpxchg.nostore252
+; AIX32-NEXT:   srw 4, 4, 23
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   lis 3, 0
+; AIX32-NEXT:   lbz 7, 0(29)
+; AIX32-NEXT:   stb 4, 0(30)
+; AIX32-NEXT:   clrlwi  6, 4, 24
+; AIX32-NEXT:   lwz 4, L..C2(2)                         # @ss
+; AIX32-NEXT:   ori 3, 3, 65535
+; AIX32-NEXT:   clrlwi  5, 4, 30
+; AIX32-NEXT:   rlwinm 21, 4, 0, 0, 29
+; AIX32-NEXT:   xori 5, 5, 2
+; AIX32-NEXT:   slwi 20, 5, 3
+; AIX32-NEXT:   slw 5, 3, 20
+; AIX32-NEXT:   not     19, 5
+; AIX32-NEXT:   lwarx 5, 0, 21
+; AIX32-NEXT:   srw 8, 5, 20
+; AIX32-NEXT:   clrlwi  8, 8, 16
+; AIX32-NEXT:   cmplw   8, 6
+; AIX32-NEXT:   bne     0, L..BB3_12
+; AIX32-NEXT:  # %bb.9:                                # %cmpxchg.fencedstore223
+; AIX32-NEXT:   extsb 7, 7
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   clrlwi  7, 7, 16
+; AIX32-NEXT:   slw 7, 7, 20
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_10:                              # %cmpxchg.trystore222
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 8, 5, 19
+; AIX32-NEXT:   or 8, 8, 7
+; AIX32-NEXT:   stwcx. 8, 0, 21
+; AIX32-NEXT:   beq     0, L..BB3_12
+; AIX32-NEXT:  # %bb.11:                               # %cmpxchg.releasedload221
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_10 Depth=1
+; AIX32-NEXT:   lwarx 5, 0, 21
+; AIX32-NEXT:   srw 8, 5, 20
+; AIX32-NEXT:   clrlwi  8, 8, 16
+; AIX32-NEXT:   cmplw   8, 6
+; AIX32-NEXT:   beq     0, L..BB3_10
+; AIX32-NEXT:  L..BB3_12:                              # %cmpxchg.nostore219
+; AIX32-NEXT:   srw 5, 5, 20
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   lbz 6, 0(29)
+; AIX32-NEXT:   sth 5, 0(4)
+; AIX32-NEXT:   lwz 4, L..C3(2)                         # @us
+; AIX32-NEXT:   lbz 5, 0(30)
+; AIX32-NEXT:   clrlwi  7, 4, 30
+; AIX32-NEXT:   rlwinm 18, 4, 0, 0, 29
+; AIX32-NEXT:   xori 7, 7, 2
+; AIX32-NEXT:   slwi 17, 7, 3
+; AIX32-NEXT:   slw 3, 3, 17
+; AIX32-NEXT:   not     16, 3
+; AIX32-NEXT:   lwarx 3, 0, 18
+; AIX32-NEXT:   srw 7, 3, 17
+; AIX32-NEXT:   clrlwi  7, 7, 16
+; AIX32-NEXT:   cmplw   7, 5
+; AIX32-NEXT:   bne     0, L..BB3_16
+; AIX32-NEXT:  # %bb.13:                               # %cmpxchg.fencedstore190
+; AIX32-NEXT:   extsb 6, 6
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   clrlwi  6, 6, 16
+; AIX32-NEXT:   slw 6, 6, 17
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_14:                              # %cmpxchg.trystore189
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 7, 3, 16
+; AIX32-NEXT:   or 7, 7, 6
+; AIX32-NEXT:   stwcx. 7, 0, 18
+; AIX32-NEXT:   beq     0, L..BB3_16
+; AIX32-NEXT:  # %bb.15:                               # %cmpxchg.releasedload188
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_14 Depth=1
+; AIX32-NEXT:   lwarx 3, 0, 18
+; AIX32-NEXT:   srw 7, 3, 17
+; AIX32-NEXT:   clrlwi  7, 7, 16
+; AIX32-NEXT:   cmplw   7, 5
+; AIX32-NEXT:   beq     0, L..BB3_14
+; AIX32-NEXT:  L..BB3_16:                              # %cmpxchg.nostore186
+; AIX32-NEXT:   srw 3, 3, 17
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   lwz 15, L..C4(2)                        # @si
+; AIX32-NEXT:   lbz 5, 0(29)
+; AIX32-NEXT:   sth 3, 0(4)
+; AIX32-NEXT:   lbz 4, 0(30)
+; AIX32-NEXT:   lwarx 3, 0, 15
+; AIX32-NEXT:   cmplw   3, 4
+; AIX32-NEXT:   bne     0, L..BB3_20
+; AIX32-NEXT:  # %bb.17:                               # %cmpxchg.fencedstore171
+; AIX32-NEXT:   extsb 5, 5
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   .align  5
+; AIX32-NEXT:  L..BB3_18:                              # %cmpxchg.trystore170
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   stwcx. 5, 0, 15
+; AIX32-NEXT:   beq     0, L..BB3_20
+; AIX32-NEXT:  # %bb.19:                               # %cmpxchg.releasedload169
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_18 Depth=1
+; AIX32-NEXT:   lwarx 3, 0, 15
+; AIX32-NEXT:   cmplw   3, 4
+; AIX32-NEXT:   beq     0, L..BB3_18
+; AIX32-NEXT:  L..BB3_20:                              # %cmpxchg.nostore167
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   lwz 28, L..C5(2)                        # @ui
+; AIX32-NEXT:   stw 3, 0(15)
+; AIX32-NEXT:   lbz 4, 0(30)
+; AIX32-NEXT:   lbz 5, 0(29)
+; AIX32-NEXT:   lwarx 3, 0, 28
+; AIX32-NEXT:   cmplw   3, 4
+; AIX32-NEXT:   bne     0, L..BB3_24
+; AIX32-NEXT:  # %bb.21:                               # %cmpxchg.fencedstore152
+; AIX32-NEXT:   extsb 5, 5
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   .align  5
+; AIX32-NEXT:  L..BB3_22:                              # %cmpxchg.trystore151
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   stwcx. 5, 0, 28
+; AIX32-NEXT:   beq     0, L..BB3_24
+; AIX32-NEXT:  # %bb.23:                               # %cmpxchg.releasedload150
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_22 Depth=1
+; AIX32-NEXT:   lwarx 3, 0, 28
+; AIX32-NEXT:   cmplw   3, 4
+; AIX32-NEXT:   beq     0, L..BB3_22
+; AIX32-NEXT:  L..BB3_24:                              # %cmpxchg.nostore148
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lwz 31, L..C6(2)                        # @sll
+; AIX32-NEXT:   lbz 3, 0(29)
+; AIX32-NEXT:   li 14, 0
+; AIX32-NEXT:   addi 4, 1, 64
+; AIX32-NEXT:   li 7, 5
+; AIX32-NEXT:   li 8, 5
+; AIX32-NEXT:   stw 14, 64(1)
+; AIX32-NEXT:   extsb 6, 3
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   srawi 5, 6, 31
+; AIX32-NEXT:   stw 3, 68(1)
+; AIX32-NEXT:   mr      3, 31
+; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:   nop
+; AIX32-NEXT:   lwz 3, 68(1)
+; AIX32-NEXT:   lbz 4, 0(29)
+; AIX32-NEXT:   li 7, 5
+; AIX32-NEXT:   li 8, 5
+; AIX32-NEXT:   stw 3, 4(31)
+; AIX32-NEXT:   lwz 3, 64(1)
+; AIX32-NEXT:   extsb 6, 4
+; AIX32-NEXT:   addi 4, 1, 64
+; AIX32-NEXT:   stw 14, 64(1)
+; AIX32-NEXT:   srawi 5, 6, 31
+; AIX32-NEXT:   stw 3, 0(31)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   lwz 31, L..C7(2)                        # @ull
+; AIX32-NEXT:   stw 3, 68(1)
+; AIX32-NEXT:   mr      3, 31
+; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:   nop
+; AIX32-NEXT:   lwz 3, 64(1)
+; AIX32-NEXT:   lwz 4, 68(1)
+; AIX32-NEXT:   lbz 5, 0(29)
+; AIX32-NEXT:   stw 4, 4(31)
+; AIX32-NEXT:   stw 3, 0(31)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   lwarx 4, 0, 27
+; AIX32-NEXT:   srw 6, 4, 26
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   bne     0, L..BB3_28
+; AIX32-NEXT:  # %bb.25:                               # %cmpxchg.fencedstore119
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   slw 5, 5, 26
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_26:                              # %cmpxchg.trystore118
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 4, 4, 25
+; AIX32-NEXT:   or 4, 4, 5
+; AIX32-NEXT:   stwcx. 4, 0, 27
+; AIX32-NEXT:   beq     0, L..BB3_29
+; AIX32-NEXT:  # %bb.27:                               # %cmpxchg.releasedload117
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_26 Depth=1
+; AIX32-NEXT:   lwarx 4, 0, 27
+; AIX32-NEXT:   srw 6, 4, 26
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   beq     0, L..BB3_26
+; AIX32-NEXT:  L..BB3_28:                              # %cmpxchg.nostore115
+; AIX32-NEXT:   crxor 20, 20, 20
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   b L..BB3_30
+; AIX32-NEXT:  L..BB3_29:                              # %cmpxchg.success116
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_30:                              # %cmpxchg.end113
+; AIX32-NEXT:   li 3, 0
+; AIX32-NEXT:   li 4, 1
+; AIX32-NEXT:   lbz 5, 0(29)
+; AIX32-NEXT:   isel 3, 4, 3, 20
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   lwarx 4, 0, 24
+; AIX32-NEXT:   srw 6, 4, 23
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   bne     0, L..BB3_34
+; AIX32-NEXT:  # %bb.31:                               # %cmpxchg.fencedstore86
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   slw 5, 5, 23
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_32:                              # %cmpxchg.trystore85
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 4, 4, 22
+; AIX32-NEXT:   or 4, 4, 5
+; AIX32-NEXT:   stwcx. 4, 0, 24
+; AIX32-NEXT:   beq     0, L..BB3_35
+; AIX32-NEXT:  # %bb.33:                               # %cmpxchg.releasedload84
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_32 Depth=1
+; AIX32-NEXT:   lwarx 4, 0, 24
+; AIX32-NEXT:   srw 6, 4, 23
+; AIX32-NEXT:   clrlwi  6, 6, 24
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   beq     0, L..BB3_32
+; AIX32-NEXT:  L..BB3_34:                              # %cmpxchg.nostore82
+; AIX32-NEXT:   crxor 20, 20, 20
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   b L..BB3_36
+; AIX32-NEXT:  L..BB3_35:                              # %cmpxchg.success83
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_36:                              # %cmpxchg.end80
+; AIX32-NEXT:   li 3, 0
+; AIX32-NEXT:   li 4, 1
+; AIX32-NEXT:   lbz 5, 0(29)
+; AIX32-NEXT:   isel 3, 4, 3, 20
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   lwarx 4, 0, 21
+; AIX32-NEXT:   srw 6, 4, 20
+; AIX32-NEXT:   clrlwi  6, 6, 16
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   bne     0, L..BB3_40
+; AIX32-NEXT:  # %bb.37:                               # %cmpxchg.fencedstore53
+; AIX32-NEXT:   extsb 5, 5
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   clrlwi  5, 5, 16
+; AIX32-NEXT:   slw 5, 5, 20
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_38:                              # %cmpxchg.trystore52
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 4, 4, 19
+; AIX32-NEXT:   or 4, 4, 5
+; AIX32-NEXT:   stwcx. 4, 0, 21
+; AIX32-NEXT:   beq     0, L..BB3_41
+; AIX32-NEXT:  # %bb.39:                               # %cmpxchg.releasedload51
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_38 Depth=1
+; AIX32-NEXT:   lwarx 4, 0, 21
+; AIX32-NEXT:   srw 6, 4, 20
+; AIX32-NEXT:   clrlwi  6, 6, 16
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   beq     0, L..BB3_38
+; AIX32-NEXT:  L..BB3_40:                              # %cmpxchg.nostore49
+; AIX32-NEXT:   crxor 20, 20, 20
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   b L..BB3_42
+; AIX32-NEXT:  L..BB3_41:                              # %cmpxchg.success50
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_42:                              # %cmpxchg.end47
+; AIX32-NEXT:   li 3, 0
+; AIX32-NEXT:   li 4, 1
+; AIX32-NEXT:   lbz 5, 0(29)
+; AIX32-NEXT:   isel 3, 4, 3, 20
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   lwarx 4, 0, 18
+; AIX32-NEXT:   srw 6, 4, 17
+; AIX32-NEXT:   clrlwi  6, 6, 16
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   bne     0, L..BB3_46
+; AIX32-NEXT:  # %bb.43:                               # %cmpxchg.fencedstore29
+; AIX32-NEXT:   extsb 5, 5
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   clrlwi  5, 5, 16
+; AIX32-NEXT:   slw 5, 5, 17
+; AIX32-NEXT:   .align  4
+; AIX32-NEXT:  L..BB3_44:                              # %cmpxchg.trystore28
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   and 4, 4, 16
+; AIX32-NEXT:   or 4, 4, 5
+; AIX32-NEXT:   stwcx. 4, 0, 18
+; AIX32-NEXT:   beq     0, L..BB3_47
+; AIX32-NEXT:  # %bb.45:                               # %cmpxchg.releasedload27
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_44 Depth=1
+; AIX32-NEXT:   lwarx 4, 0, 18
+; AIX32-NEXT:   srw 6, 4, 17
+; AIX32-NEXT:   clrlwi  6, 6, 16
+; AIX32-NEXT:   cmplw   6, 3
+; AIX32-NEXT:   beq     0, L..BB3_44
+; AIX32-NEXT:  L..BB3_46:                              # %cmpxchg.nostore25
+; AIX32-NEXT:   crxor 20, 20, 20
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   b L..BB3_48
+; AIX32-NEXT:  L..BB3_47:                              # %cmpxchg.success26
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_48:                              # %cmpxchg.end23
+; AIX32-NEXT:   li 3, 0
+; AIX32-NEXT:   li 4, 1
+; AIX32-NEXT:   isel 3, 4, 3, 20
+; AIX32-NEXT:   lbz 4, 0(29)
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   lwarx 5, 0, 15
+; AIX32-NEXT:   cmplw   5, 3
+; AIX32-NEXT:   bne     0, L..BB3_52
+; AIX32-NEXT:  # %bb.49:                               # %cmpxchg.fencedstore10
+; AIX32-NEXT:   extsb 4, 4
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   .align  5
+; AIX32-NEXT:  L..BB3_50:                              # %cmpxchg.trystore9
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   stwcx. 4, 0, 15
+; AIX32-NEXT:   beq     0, L..BB3_53
+; AIX32-NEXT:  # %bb.51:                               # %cmpxchg.releasedload8
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_50 Depth=1
+; AIX32-NEXT:   lwarx 5, 0, 15
+; AIX32-NEXT:   cmplw   5, 3
+; AIX32-NEXT:   beq     0, L..BB3_50
+; AIX32-NEXT:  L..BB3_52:                              # %cmpxchg.nostore6
+; AIX32-NEXT:   crxor 20, 20, 20
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   b L..BB3_54
+; AIX32-NEXT:  L..BB3_53:                              # %cmpxchg.success7
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_54:                              # %cmpxchg.end4
+; AIX32-NEXT:   li 3, 0
+; AIX32-NEXT:   li 4, 1
+; AIX32-NEXT:   isel 3, 4, 3, 20
+; AIX32-NEXT:   lbz 4, 0(29)
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   lwarx 5, 0, 28
+; AIX32-NEXT:   cmplw   5, 3
+; AIX32-NEXT:   bne     0, L..BB3_58
+; AIX32-NEXT:  # %bb.55:                               # %cmpxchg.fencedstore
+; AIX32-NEXT:   extsb 4, 4
+; AIX32-NEXT:   sync
+; AIX32-NEXT:   .align  5
+; AIX32-NEXT:  L..BB3_56:                              # %cmpxchg.trystore
+; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; AIX32-NEXT:   stwcx. 4, 0, 28
+; AIX32-NEXT:   beq     0, L..BB3_59
+; AIX32-NEXT:  # %bb.57:                               # %cmpxchg.releasedload
+; AIX32-NEXT:                                          #   in Loop: Header=BB3_56 Depth=1
+; AIX32-NEXT:   lwarx 5, 0, 28
+; AIX32-NEXT:   cmplw   5, 3
+; AIX32-NEXT:   beq     0, L..BB3_56
+; AIX32-NEXT:  L..BB3_58:                              # %cmpxchg.nostore
+; AIX32-NEXT:   crxor 20, 20, 20
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   b L..BB3_60
+; AIX32-NEXT:  L..BB3_59:                              # %cmpxchg.success
+; AIX32-NEXT:   lwsync
+; AIX32-NEXT:   creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_60:                              # %cmpxchg.end
+; AIX32-NEXT:   li 3, 1
+; AIX32-NEXT:   li 31, 0
+; AIX32-NEXT:   lbz 4, 0(29)
+; AIX32-NEXT:   isel 3, 3, 31, 20
+; AIX32-NEXT:   li 7, 5
+; AIX32-NEXT:   li 8, 5
+; AIX32-NEXT:   extsb 6, 4
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   addi 4, 1, 64
+; AIX32-NEXT:   stw 31, 64(1)
+; AIX32-NEXT:   srawi 5, 6, 31
+; AIX32-NEXT:   stw 3, 68(1)
+; AIX32-NEXT:   lwz 3, L..C6(2)                         # @sll
+; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:   nop
+; AIX32-NEXT:   lbz 4, 0(29)
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lbz 3, 0(30)
+; AIX32-NEXT:   li 7, 5
+; AIX32-NEXT:   li 8, 5
+; AIX32-NEXT:   extsb 6, 4
+; AIX32-NEXT:   stw 3, 68(1)
+; AIX32-NEXT:   lwz 3, L..C7(2)                         # @ull
+; AIX32-NEXT:   addi 4, 1, 64
+; AIX32-NEXT:   stw 31, 64(1)
+; AIX32-NEXT:   srawi 5, 6, 31
+; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:   nop
+; AIX32-NEXT:   stw 3, 0(28)
+; AIX32-NEXT:   lwz 31, 140(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 30, 136(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 29, 132(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 28, 128(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 27, 124(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 26, 120(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 25, 116(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 24, 112(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 23, 108(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 22, 104(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 21, 100(1)                          # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 20, 96(1)                           # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 19, 92(1)                           # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 18, 88(1)                           # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 17, 84(1)                           # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 16, 80(1)                           # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 15, 76(1)                           # 4-byte Folded Reload
+; AIX32-NEXT:   lwz 14, 72(1)                           # 4-byte Folded Reload
+; AIX32-NEXT:   addi 1, 1, 144
+; AIX32-NEXT:   lwz 0, 8(1)
+; AIX32-NEXT:   mtlr 0
 ; AIX32-NEXT:    blr
 entry:
   %0 = load i8, ptr @uc, align 1
@@ -5597,21 +5852,20 @@ entry:
 define dso_local i64 @cmpswplp(ptr noundef %ptr, ptr nocapture noundef readnone %oldval, i64 noundef %newval) local_unnamed_addr #0 {
 ; CHECK-LABEL: cmpswplp:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi 4, 5, 1
-; CHECK-NEXT:  .LBB6_1: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    ldarx 6, 0, 3
-; CHECK-NEXT:    cmpd 1, 6, 5
-; CHECK-NEXT:    bne 1, .LBB6_3
-; CHECK-NEXT:  # %bb.2: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stdcx. 4, 0, 3
-; CHECK-NEXT:    bne 0, .LBB6_1
-; CHECK-NEXT:  .LBB6_3: # %entry
-; CHECK-NEXT:    li 3, 66
-; CHECK-NEXT:    li 4, 55
-; CHECK-NEXT:    isel 3, 4, 3, 6
-; CHECK-NEXT:    blr
+; CHECK-NEXT:   ldarx 4, 0, 3
+; CHECK-NEXT:   cmpld   4, 5
+; CHECK-NEXT:   bne     0, .LBB6_2
+; CHECK-NEXT: # %bb.1:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:   addi 4, 5, 1
+; CHECK-NEXT:   stdcx. 4, 0, 3
+; CHECK-NEXT:   beq     0, .LBB6_4
+; CHECK-NEXT: .LBB6_2:                                # %cmpxchg.failure
+; CHECK-NEXT:   crxor 20, 20, 20
+; CHECK-NEXT: .LBB6_3:                                # %cmpxchg.end
+; CHECK-NEXT:   li 3, 66
+; CHECK-NEXT:   li 4, 55
+; CHECK-NEXT:   isel 3, 4, 3, 20
+; CHECK-NEXT:   blr
 ;
 ; AIX32-LABEL: cmpswplp:
 ; AIX32:       # %bb.0: # %entry
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/asm-printer-topological-order.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/asm-printer-topological-order.ll
index 6299b4e393d9..3218c77f08c8 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/asm-printer-topological-order.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/asm-printer-topological-order.ll
@@ -10,6 +10,6 @@ entry:
 }
 
 ; CHECK-LABEL: TestD:
-; CHECK: .set TestC, TestD
-; CHECK-DAG: .set TestB, TestC
-; CHECK-DAG: .set TestA, TestC
+; CHECK: TestC = TestD
+; CHECK-DAG: TestB = TestC
+; CHECK-DAG: TestA = TestC
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-2.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-2.ll
index 10476541870f..8fa0d767b329 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-2.ll
@@ -42,8 +42,8 @@ define i64 @exchange_and_cmp(ptr %mem) nounwind {
 
 define i8 @exchange_and_cmp8(ptr %mem) nounwind {
 ; CHECK-LABEL: exchange_and_cmp8:
-; CHECK-BE: xori
-; CHECK-LE-NOT: xori
+; CHECK-BE: or r{{.*}} r{{.*}} r{{.*}}
+; CHECK-LE-NOT: or r{{.*}} r{{.*}} r{{.*}}
 ; CHECK-P8U: lbarx
   %tmppair = cmpxchg ptr %mem, i8 0, i8 1 monotonic monotonic
   %tmp = extractvalue { i8, i1 } %tmppair, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-compare-exchange-weak.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-compare-exchange-weak.ll
index 399645f671f7..65a12a6222f2 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-compare-exchange-weak.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-compare-exchange-weak.ll
@@ -12,62 +12,60 @@
 define i32 @foo(ptr noundef %cp, ptr noundef %old, i32 noundef %c)  {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lwz r7, 0(r4)
 ; CHECK-NEXT:    stw r3, -4(r1)
 ; CHECK-NEXT:    stw r4, -8(r1)
+; CHECK-NEXT:    lwz r7, 0(r4)
 ; CHECK-NEXT:    stw r5, -12(r1)
 ; CHECK-NEXT:    stw r5, -16(r1)
-; CHECK-NEXT:  L..BB0_1: # %entry
-; CHECK-NEXT:    #
 ; CHECK-NEXT:    lwarx r6, 0, r3
-; CHECK-NEXT:    cmpw cr1, r6, r7
-; CHECK-NEXT:    bne cr1, L..BB0_3
-; CHECK-NEXT:  # %bb.2: # %entry
-; CHECK-NEXT:    #
-; CHECK-NEXT:    stwcx. r5, 0, r3
-; CHECK-NEXT:    bne cr0, L..BB0_1
-; CHECK-NEXT:  L..BB0_3: # %entry
 ; CHECK-NEXT:    cmplw r6, r7
+; CHECK-NEXT:    bne cr0, L..BB0_2
+; CHECK-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; CHECK-NEXT:    stwcx. r5, 0, r3
 ; CHECK-NEXT:    beq cr0, L..BB0_5
-; CHECK-NEXT:  # %bb.4: # %cmpxchg.store_expected
+; CHECK-NEXT:  L..BB0_2: # %cmpxchg.failure
+; CHECK-NEXT:    crxor 4*cr5+lt, 4*cr5+lt, 4*cr5+lt
+; CHECK-NEXT:  # %bb.3: # %cmpxchg.store_expected
 ; CHECK-NEXT:    stw r6, 0(r4)
-; CHECK-NEXT:  L..BB0_5: # %cmpxchg.continue
+; CHECK-NEXT:  L..BB0_4: # %cmpxchg.continue
 ; CHECK-NEXT:    li r3, 0
 ; CHECK-NEXT:    li r4, 1
-; CHECK-NEXT:    isel r3, r4, r3, 4*cr1+eq
+; CHECK-NEXT:    isel r3, r4, r3, 4*cr5+lt
 ; CHECK-NEXT:    stb r3, -17(r1)
 ; CHECK-NEXT:    blr
+; CHECK-NEXT:  L..BB0_5:
+; CHECK-NEXT:    creqv 4*cr5+lt, 4*cr5+lt, 4*cr5+lt
+; CHECK-NEXT:    b L..BB0_4
 ;
 ; CHECK64-LABEL: foo:
 ; CHECK64:       # %bb.0: # %entry
-; CHECK64-NEXT:    lwz r7, 0(r4)
 ; CHECK64-NEXT:    std r3, -8(r1)
 ; CHECK64-NEXT:    std r4, -16(r1)
+; CHECK64-NEXT:    lwz r7, 0(r4)
 ; CHECK64-NEXT:    stw r5, -20(r1)
 ; CHECK64-NEXT:    stw r5, -24(r1)
-; CHECK64-NEXT:  L..BB0_1: # %entry
-; CHECK64-NEXT:    #
 ; CHECK64-NEXT:    lwarx r6, 0, r3
-; CHECK64-NEXT:    cmpw cr1, r6, r7
-; CHECK64-NEXT:    bne cr1, L..BB0_3
-; CHECK64-NEXT:  # %bb.2: # %entry
-; CHECK64-NEXT:    #
-; CHECK64-NEXT:    stwcx. r5, 0, r3
-; CHECK64-NEXT:    bne cr0, L..BB0_1
-; CHECK64-NEXT:  L..BB0_3: # %entry
 ; CHECK64-NEXT:    cmplw r6, r7
+; CHECK64-NEXT:    bne cr0, L..BB0_2
+; CHECK64-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; CHECK64-NEXT:    stwcx. r5, 0, r3
 ; CHECK64-NEXT:    beq cr0, L..BB0_5
-; CHECK64-NEXT:  # %bb.4: # %cmpxchg.store_expected
+; CHECK64-NEXT:  L..BB0_2: # %cmpxchg.failure
+; CHECK64-NEXT:    crxor 4*cr5+lt, 4*cr5+lt, 4*cr5+lt
+; CHECK64-NEXT:  # %bb.3: # %cmpxchg.store_expected
 ; CHECK64-NEXT:    stw r6, 0(r4)
-; CHECK64-NEXT:  L..BB0_5: # %cmpxchg.continue
+; CHECK64-NEXT:  L..BB0_4: # %cmpxchg.continue
 ; CHECK64-NEXT:    li r3, 0
 ; CHECK64-NEXT:    li r4, 1
-; CHECK64-NEXT:    isel r3, r4, r3, 4*cr1+eq
+; CHECK64-NEXT:    isel r3, r4, r3, 4*cr5+lt
 ; CHECK64-NEXT:    li r4, 1
 ; CHECK64-NEXT:    stb r3, -25(r1)
 ; CHECK64-NEXT:    li r3, 0
-; CHECK64-NEXT:    isel r3, r4, r3, 4*cr1+eq
+; CHECK64-NEXT:    isel r3, r4, r3, 4*cr5+lt
 ; CHECK64-NEXT:    blr
+; CHECK64-NEXT:  L..BB0_5:
+; CHECK64-NEXT:    creqv 4*cr5+lt, 4*cr5+lt, 4*cr5+lt
+; CHECK64-NEXT:    b L..BB0_4
 entry:
   %cp.addr = alloca ptr, align 4
   %old.addr = alloca ptr, align 4
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-float.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-float.ll
index e2a46130ab79..600d28936c16 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-float.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomic-float.ll
@@ -9,33 +9,37 @@ define float @test_add(ptr %ptr, float %incr) {
 ; CHECK-64:       # %bb.0: # %entry
 ; CHECK-64-NEXT:    sync
 ; CHECK-64-NEXT:    lfs 0, 0(3)
-; CHECK-64-NEXT:    b .LBB0_2
-; CHECK-64-NEXT:  .LBB0_1: # %atomicrmw.start
-; CHECK-64-NEXT:    #
-; CHECK-64-NEXT:    stw 6, -4(1)
-; CHECK-64-NEXT:    cmplw 6, 4
-; CHECK-64-NEXT:    lfs 0, -4(1)
-; CHECK-64-NEXT:    beq 0, .LBB0_5
-; CHECK-64-NEXT:  .LBB0_2: # %atomicrmw.start
-; CHECK-64-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-64-NEXT:    # Child Loop BB0_3 Depth 2
+; CHECK-64-NEXT:    b .LBB0_3
+; CHECK-64-NEXT:  .LBB0_1:                                # %cmpxchg.nostore
+; CHECK-64-NEXT:                                          #   in Loop: Header=BB0_3 Depth=1
+; CHECK-64-NEXT:    crxor 20, 20, 20
+; CHECK-64-NEXT:  .LBB0_2:                                # %cmpxchg.end
+; CHECK-64-NEXT:                                          #   in Loop: Header=BB0_3 Depth=1
+; CHECK-64-NEXT:    stw 4, -12(1)
+; CHECK-64-NEXT:    lfs 0, -12(1)
+; CHECK-64-NEXT:    bc 12, 20, .LBB0_7
+; CHECK-64-NEXT:  .LBB0_3:                                # %atomicrmw.start
+; CHECK-64-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-64-NEXT:                                          #     Child Loop BB0_4 Depth 2
 ; CHECK-64-NEXT:    fadds 2, 0, 1
-; CHECK-64-NEXT:    stfs 2, -8(1)
-; CHECK-64-NEXT:    stfs 0, -12(1)
-; CHECK-64-NEXT:    lwz 5, -8(1)
-; CHECK-64-NEXT:    lwz 4, -12(1)
-; CHECK-64-NEXT:  .LBB0_3: # %atomicrmw.start
-; CHECK-64-NEXT:    # Parent Loop BB0_2 Depth=1
-; CHECK-64-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-64-NEXT:    lwarx 6, 0, 3
-; CHECK-64-NEXT:    cmpw 6, 4
-; CHECK-64-NEXT:    bne 0, .LBB0_1
-; CHECK-64-NEXT:  # %bb.4: # %atomicrmw.start
-; CHECK-64-NEXT:    #
+; CHECK-64-NEXT:    stfs 2, -4(1)
+; CHECK-64-NEXT:    stfs 0, -8(1)
+; CHECK-64-NEXT:    lwz 5, -4(1)
+; CHECK-64-NEXT:    lwz 6, -8(1)
+; CHECK-64-NEXT:  .LBB0_4:                                # %cmpxchg.start
+; CHECK-64-NEXT:                                          #   Parent Loop BB0_3 Depth=1
+; CHECK-64-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-64-NEXT:    lwarx 4, 0, 3
+; CHECK-64-NEXT:    cmplw   4, 6
+; CHECK-64-NEXT:    bne     0, .LBB0_1
+; CHECK-64-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore
+; CHECK-64-NEXT:                                          #   in Loop: Header=BB0_4 Depth=2
 ; CHECK-64-NEXT:    stwcx. 5, 0, 3
-; CHECK-64-NEXT:    bne 0, .LBB0_3
-; CHECK-64-NEXT:    b .LBB0_1
-; CHECK-64-NEXT:  .LBB0_5: # %atomicrmw.end
+; CHECK-64-NEXT:    bne     0, .LBB0_4
+; CHECK-64-NEXT:  # %bb.6:                                #   in Loop: Header=BB0_3 Depth=1
+; CHECK-64-NEXT:    creqv 20, 20, 20
+; CHECK-64-NEXT:    b .LBB0_2
+; CHECK-64-NEXT:  .LBB0_7:                                # %atomicrmw.end
 ; CHECK-64-NEXT:    fmr 1, 0
 ; CHECK-64-NEXT:    lwsync
 ; CHECK-64-NEXT:    blr
@@ -46,33 +50,37 @@ define float @test_add(ptr %ptr, float %incr) {
 ; CHECK-32-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-32-NEXT:    sync
 ; CHECK-32-NEXT:    lfs 0, 0(3)
-; CHECK-32-NEXT:    b .LBB0_2
-; CHECK-32-NEXT:  .LBB0_1: # %atomicrmw.start
-; CHECK-32-NEXT:    #
-; CHECK-32-NEXT:    stw 6, 28(1)
-; CHECK-32-NEXT:    cmplw 6, 4
-; CHECK-32-NEXT:    lfs 0, 28(1)
-; CHECK-32-NEXT:    beq 0, .LBB0_5
-; CHECK-32-NEXT:  .LBB0_2: # %atomicrmw.start
-; CHECK-32-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-32-NEXT:    # Child Loop BB0_3 Depth 2
+; CHECK-32-NEXT:    b .LBB0_3
+; CHECK-32-NEXT:  .LBB0_1:                                # %cmpxchg.nostore
+; CHECK-32-NEXT:                                          #   in Loop: Header=BB0_3 Depth=1
+; CHECK-32-NEXT:    crxor 20, 20, 20
+; CHECK-32-NEXT:  .LBB0_2:                                # %cmpxchg.end
+; CHECK-32-NEXT:                                          #   in Loop: Header=BB0_3 Depth=1
+; CHECK-32-NEXT:    stw 4, 20(1)
+; CHECK-32-NEXT:    lfs 0, 20(1)
+; CHECK-32-NEXT:    bc 12, 20, .LBB0_7
+; CHECK-32-NEXT:  .LBB0_3:                                # %atomicrmw.start
+; CHECK-32-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-32-NEXT:                                          #     Child Loop BB0_4 Depth 2
 ; CHECK-32-NEXT:    fadds 2, 0, 1
-; CHECK-32-NEXT:    stfs 2, 24(1)
-; CHECK-32-NEXT:    stfs 0, 20(1)
-; CHECK-32-NEXT:    lwz 5, 24(1)
-; CHECK-32-NEXT:    lwz 4, 20(1)
-; CHECK-32-NEXT:  .LBB0_3: # %atomicrmw.start
-; CHECK-32-NEXT:    # Parent Loop BB0_2 Depth=1
-; CHECK-32-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-32-NEXT:    lwarx 6, 0, 3
-; CHECK-32-NEXT:    cmpw 6, 4
-; CHECK-32-NEXT:    bne 0, .LBB0_1
-; CHECK-32-NEXT:  # %bb.4: # %atomicrmw.start
-; CHECK-32-NEXT:    #
+; CHECK-32-NEXT:    stfs 2, 28(1)
+; CHECK-32-NEXT:    stfs 0, 24(1)
+; CHECK-32-NEXT:    lwz 5, 28(1)
+; CHECK-32-NEXT:    lwz 6, 24(1)
+; CHECK-32-NEXT:  .LBB0_4:                                # %cmpxchg.start
+; CHECK-32-NEXT:                                          #   Parent Loop BB0_3 Depth=1
+; CHECK-32-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-32-NEXT:    lwarx 4, 0, 3
+; CHECK-32-NEXT:    cmplw   4, 6
+; CHECK-32-NEXT:    bne     0, .LBB0_1
+; CHECK-32-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore
+; CHECK-32-NEXT:                                          #   in Loop: Header=BB0_4 Depth=2
 ; CHECK-32-NEXT:    stwcx. 5, 0, 3
-; CHECK-32-NEXT:    bne 0, .LBB0_3
-; CHECK-32-NEXT:    b .LBB0_1
-; CHECK-32-NEXT:  .LBB0_5: # %atomicrmw.end
+; CHECK-32-NEXT:    bne     0, .LBB0_4
+; CHECK-32-NEXT:  # %bb.6:                                #   in Loop: Header=BB0_3 Depth=1
+; CHECK-32-NEXT:    creqv 20, 20, 20
+; CHECK-32-NEXT:    b .LBB0_2
+; CHECK-32-NEXT:  .LBB0_7:                                # %atomicrmw.end
 ; CHECK-32-NEXT:    fmr 1, 0
 ; CHECK-32-NEXT:    lwsync
 ; CHECK-32-NEXT:    addi 1, 1, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll
index 4f00cff83942..27a26aaca8b2 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll
@@ -5,49 +5,47 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
 ; CHECK-LABEL: atomicrmw_usub_cond_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    rlwinm 7, 5, 3, 27, 28
-; CHECK-NEXT:    lbz 3, 0(3)
-; CHECK-NEXT:    xori 7, 7, 24
-; CHECK-NEXT:    li 8, 255
-; CHECK-NEXT:    clrlwi 6, 4, 24
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 8, 8, 7
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    not     3, 3
+; CHECK-NEXT:    li 6, 255
+; CHECK-NEXT:    lwz 8, 0(5)
+; CHECK-NEXT:    rlwinm 3, 3, 3, 27, 28
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  7, 4, 24
 ; CHECK-NEXT:    b .LBB0_2
-; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 11, 7
-; CHECK-NEXT:    cmplw 3, 9
-; CHECK-NEXT:    beq 0, .LBB0_7
-; CHECK-NEXT:  .LBB0_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB0_5 Depth 2
-; CHECK-NEXT:    clrlwi 9, 3, 24
-; CHECK-NEXT:    cmplw 9, 6
-; CHECK-NEXT:    blt 0, .LBB0_4
-; CHECK-NEXT:  # %bb.3:
-; CHECK-NEXT:    sub 3, 3, 4
-; CHECK-NEXT:  .LBB0_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 3, 7
-; CHECK-NEXT:    slw 10, 9, 7
-; CHECK-NEXT:    and 3, 3, 8
-; CHECK-NEXT:    and 10, 10, 8
-; CHECK-NEXT:  .LBB0_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB0_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 12, 0, 5
-; CHECK-NEXT:    and 11, 12, 8
-; CHECK-NEXT:    cmpw 11, 10
-; CHECK-NEXT:    bne 0, .LBB0_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 12, 12, 8
-; CHECK-NEXT:    or 12, 12, 3
-; CHECK-NEXT:    stwcx. 12, 0, 5
-; CHECK-NEXT:    bne 0, .LBB0_5
-; CHECK-NEXT:    b .LBB0_1
-; CHECK-NEXT:  .LBB0_7: # %atomicrmw.end
+; CHECK-NEXT:  .LBB0_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  .LBB0_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB0_5 Depth 2
+; CHECK-NEXT:    srw 9, 8, 3
+; CHECK-NEXT:    clrlwi  10, 9, 24
+; CHECK-NEXT:    cmplw   10, 7
+; CHECK-NEXT:    blt     0, .LBB0_4
+; CHECK-NEXT:  # %bb.3:                                #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    sub     9, 9, 4
+; CHECK-NEXT:  .LBB0_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    clrlwi  9, 9, 24
+; CHECK-NEXT:    slw 9, 9, 3
+; CHECK-NEXT:    and 10, 8, 6
+; CHECK-NEXT:    or 10, 10, 9
+; CHECK-NEXT:  .LBB0_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 9, 0, 5
+; CHECK-NEXT:    cmplw   9, 8
+; CHECK-NEXT:    bne     0, .LBB0_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_5 Depth=2
+; CHECK-NEXT:    stwcx. 10, 0, 5
+; CHECK-NEXT:    bne     0, .LBB0_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 9, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
@@ -58,50 +56,49 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
 ; CHECK-LABEL: atomicrmw_usub_cond_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    li 8, 0
-; CHECK-NEXT:    lhz 3, 0(3)
-; CHECK-NEXT:    rlwinm 7, 5, 3, 27, 27
-; CHECK-NEXT:    xori 7, 7, 16
-; CHECK-NEXT:    ori 8, 8, 65535
-; CHECK-NEXT:    clrlwi 6, 4, 16
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 8, 8, 7
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    clrlwi  3, 3, 30
+; CHECK-NEXT:    lis 6, 0
+; CHECK-NEXT:    xori 3, 3, 2
+; CHECK-NEXT:    lwz 8, 0(5)
+; CHECK-NEXT:    ori 6, 6, 65535
+; CHECK-NEXT:    slwi 3, 3, 3
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  7, 4, 16
 ; CHECK-NEXT:    b .LBB1_2
-; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 11, 7
-; CHECK-NEXT:    cmplw 3, 9
-; CHECK-NEXT:    beq 0, .LBB1_7
-; CHECK-NEXT:  .LBB1_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB1_5 Depth 2
-; CHECK-NEXT:    clrlwi 9, 3, 16
-; CHECK-NEXT:    cmplw 9, 6
-; CHECK-NEXT:    blt 0, .LBB1_4
-; CHECK-NEXT:  # %bb.3:
-; CHECK-NEXT:    sub 3, 3, 4
-; CHECK-NEXT:  .LBB1_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 3, 7
-; CHECK-NEXT:    slw 10, 9, 7
-; CHECK-NEXT:    and 3, 3, 8
-; CHECK-NEXT:    and 10, 10, 8
-; CHECK-NEXT:  .LBB1_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB1_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 12, 0, 5
-; CHECK-NEXT:    and 11, 12, 8
-; CHECK-NEXT:    cmpw 11, 10
-; CHECK-NEXT:    bne 0, .LBB1_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 12, 12, 8
-; CHECK-NEXT:    or 12, 12, 3
-; CHECK-NEXT:    stwcx. 12, 0, 5
-; CHECK-NEXT:    bne 0, .LBB1_5
-; CHECK-NEXT:    b .LBB1_1
-; CHECK-NEXT:  .LBB1_7: # %atomicrmw.end
+; CHECK-NEXT:  .LBB1_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  .LBB1_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB1_5 Depth 2
+; CHECK-NEXT:    srw 9, 8, 3
+; CHECK-NEXT:    clrlwi  10, 9, 16
+; CHECK-NEXT:    cmplw   10, 7
+; CHECK-NEXT:    blt     0, .LBB1_4
+; CHECK-NEXT:  # %bb.3:                                #   in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    sub     9, 9, 4
+; CHECK-NEXT:  .LBB1_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    clrlwi  9, 9, 16
+; CHECK-NEXT:    slw 9, 9, 3
+; CHECK-NEXT:    and 10, 8, 6
+; CHECK-NEXT:    or 10, 10, 9
+; CHECK-NEXT:  .LBB1_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB1_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 9, 0, 5
+; CHECK-NEXT:    cmplw   9, 8
+; CHECK-NEXT:    bne     0, .LBB1_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB1_5 Depth=2
+; CHECK-NEXT:    stwcx. 10, 0, 5
+; CHECK-NEXT:    bne     0, .LBB1_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 9, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
@@ -114,34 +111,33 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    lwz 6, 0(3)
 ; CHECK-NEXT:    b .LBB2_2
-; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmplw 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB2_7
-; CHECK-NEXT:  .LBB2_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB2_5 Depth 2
-; CHECK-NEXT:    cmplw 6, 4
+; CHECK-NEXT:  .LBB2_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB2_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB2_5 Depth 2
+; CHECK-NEXT:    cmplw   6, 4
 ; CHECK-NEXT:    bge 0, .LBB2_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    mr 7, 6
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    mr      7, 6
 ; CHECK-NEXT:    b .LBB2_5
-; CHECK-NEXT:  .LBB2_4:
-; CHECK-NEXT:    sub 7, 6, 4
-; CHECK-NEXT:  .LBB2_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB2_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:  .LBB2_4:                                #   in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    sub     7, 6, 4
+; CHECK-NEXT:  .LBB2_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB2_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    lwarx 5, 0, 3
-; CHECK-NEXT:    cmpw 5, 6
-; CHECK-NEXT:    bne 0, .LBB2_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmplw   5, 6
+; CHECK-NEXT:    bne     0, .LBB2_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB2_5 Depth=2
 ; CHECK-NEXT:    stwcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB2_5
-; CHECK-NEXT:    b .LBB2_1
-; CHECK-NEXT:  .LBB2_7: # %atomicrmw.end
+; CHECK-NEXT:    bne     0, .LBB2_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
 ; CHECK-NEXT:    mr 3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
@@ -155,34 +151,33 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    ld 6, 0(3)
 ; CHECK-NEXT:    b .LBB3_2
-; CHECK-NEXT:  .LBB3_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmpld 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB3_7
-; CHECK-NEXT:  .LBB3_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB3_5 Depth 2
-; CHECK-NEXT:    cmpld 6, 4
+; CHECK-NEXT:  .LBB3_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB3_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB3_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB3_5 Depth 2
+; CHECK-NEXT:    cmpld   6, 4
 ; CHECK-NEXT:    bge 0, .LBB3_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    mr 7, 6
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB3_2 Depth=1
+; CHECK-NEXT:    mr      7, 6
 ; CHECK-NEXT:    b .LBB3_5
-; CHECK-NEXT:  .LBB3_4:
-; CHECK-NEXT:    sub 7, 6, 4
-; CHECK-NEXT:  .LBB3_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB3_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:  .LBB3_4:                                #   in Loop: Header=BB3_2 Depth=1
+; CHECK-NEXT:    sub     7, 6, 4
+; CHECK-NEXT:  .LBB3_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB3_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldarx 5, 0, 3
-; CHECK-NEXT:    cmpd 5, 6
-; CHECK-NEXT:    bne 0, .LBB3_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmpld   5, 6
+; CHECK-NEXT:    bne     0, .LBB3_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB3_5 Depth=2
 ; CHECK-NEXT:    stdcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB3_5
-; CHECK-NEXT:    b .LBB3_1
-; CHECK-NEXT:  .LBB3_7: # %atomicrmw.end
+; CHECK-NEXT:    bne     0, .LBB3_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
 ; CHECK-NEXT:    mr 3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
@@ -194,52 +189,49 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
 ; CHECK-LABEL: atomicrmw_usub_sat_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    rlwinm 6, 5, 3, 27, 28
-; CHECK-NEXT:    lbz 3, 0(3)
-; CHECK-NEXT:    xori 6, 6, 24
-; CHECK-NEXT:    li 7, 255
-; CHECK-NEXT:    clrlwi 4, 4, 24
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 7, 7, 6
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    not     3, 3
+; CHECK-NEXT:    li 6, 255
+; CHECK-NEXT:    lwz 7, 0(5)
+; CHECK-NEXT:    rlwinm 3, 3, 3, 27, 28
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  4, 4, 24
 ; CHECK-NEXT:    b .LBB4_2
-; CHECK-NEXT:  .LBB4_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 10, 6
-; CHECK-NEXT:    cmplw 3, 8
-; CHECK-NEXT:    beq 0, .LBB4_7
-; CHECK-NEXT:  .LBB4_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB4_5 Depth 2
-; CHECK-NEXT:    clrlwi 8, 3, 24
-; CHECK-NEXT:    sub 3, 8, 4
-; CHECK-NEXT:    cmplw 3, 8
+; CHECK-NEXT:  .LBB4_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  .LBB4_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB4_5 Depth 2
+; CHECK-NEXT:    srw 8, 7, 3
+; CHECK-NEXT:    clrlwi  9, 8, 24
+; CHECK-NEXT:    sub     8, 9, 4
+; CHECK-NEXT:    cmplw   8, 9
 ; CHECK-NEXT:    li 9, 0
-; CHECK-NEXT:    bgt 0, .LBB4_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    mr 9, 3
-; CHECK-NEXT:  .LBB4_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 9, 6
-; CHECK-NEXT:    slw 9, 8, 6
-; CHECK-NEXT:    and 3, 3, 7
-; CHECK-NEXT:    and 9, 9, 7
-; CHECK-NEXT:  .LBB4_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB4_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 11, 0, 5
-; CHECK-NEXT:    and 10, 11, 7
-; CHECK-NEXT:    cmpw 10, 9
-; CHECK-NEXT:    bne 0, .LBB4_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 11, 11, 7
-; CHECK-NEXT:    or 11, 11, 3
-; CHECK-NEXT:    stwcx. 11, 0, 5
-; CHECK-NEXT:    bne 0, .LBB4_5
-; CHECK-NEXT:    b .LBB4_1
-; CHECK-NEXT:  .LBB4_7: # %atomicrmw.end
+; CHECK-NEXT:    bgt     0, .LBB4_4
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    mr      9, 8
+; CHECK-NEXT:  .LBB4_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    slw 8, 9, 3
+; CHECK-NEXT:    and 9, 7, 6
+; CHECK-NEXT:    or 9, 9, 8
+; CHECK-NEXT:  .LBB4_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB4_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 8, 0, 5
+; CHECK-NEXT:    cmplw   8, 7
+; CHECK-NEXT:    bne     0, .LBB4_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_5 Depth=2
+; CHECK-NEXT:    stwcx. 9, 0, 5
+; CHECK-NEXT:    bne     0, .LBB4_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 8, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
@@ -250,53 +242,51 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
 ; CHECK-LABEL: atomicrmw_usub_sat_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    li 7, 0
-; CHECK-NEXT:    lhz 3, 0(3)
-; CHECK-NEXT:    rlwinm 6, 5, 3, 27, 27
-; CHECK-NEXT:    xori 6, 6, 16
-; CHECK-NEXT:    ori 7, 7, 65535
-; CHECK-NEXT:    clrlwi 4, 4, 16
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 7, 7, 6
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    clrlwi  3, 3, 30
+; CHECK-NEXT:    lis 6, 0
+; CHECK-NEXT:    xori 3, 3, 2
+; CHECK-NEXT:    lwz 7, 0(5)
+; CHECK-NEXT:    ori 6, 6, 65535
+; CHECK-NEXT:    slwi 3, 3, 3
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  4, 4, 16
 ; CHECK-NEXT:    b .LBB5_2
-; CHECK-NEXT:  .LBB5_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 10, 6
-; CHECK-NEXT:    cmplw 3, 8
-; CHECK-NEXT:    beq 0, .LBB5_7
-; CHECK-NEXT:  .LBB5_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB5_5 Depth 2
-; CHECK-NEXT:    clrlwi 8, 3, 16
-; CHECK-NEXT:    sub 3, 8, 4
-; CHECK-NEXT:    cmplw 3, 8
+; CHECK-NEXT:  .LBB5_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  .LBB5_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB5_5 Depth 2
+; CHECK-NEXT:    srw 8, 7, 3
+; CHECK-NEXT:    clrlwi  9, 8, 16
+; CHECK-NEXT:    sub     8, 9, 4
+; CHECK-NEXT:    cmplw   8, 9
 ; CHECK-NEXT:    li 9, 0
-; CHECK-NEXT:    bgt 0, .LBB5_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    mr 9, 3
-; CHECK-NEXT:  .LBB5_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 9, 6
-; CHECK-NEXT:    slw 9, 8, 6
-; CHECK-NEXT:    and 3, 3, 7
-; CHECK-NEXT:    and 9, 9, 7
-; CHECK-NEXT:  .LBB5_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB5_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 11, 0, 5
-; CHECK-NEXT:    and 10, 11, 7
-; CHECK-NEXT:    cmpw 10, 9
-; CHECK-NEXT:    bne 0, .LBB5_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 11, 11, 7
-; CHECK-NEXT:    or 11, 11, 3
-; CHECK-NEXT:    stwcx. 11, 0, 5
-; CHECK-NEXT:    bne 0, .LBB5_5
-; CHECK-NEXT:    b .LBB5_1
-; CHECK-NEXT:  .LBB5_7: # %atomicrmw.end
+; CHECK-NEXT:    bgt     0, .LBB5_4
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    mr      9, 8
+; CHECK-NEXT:  .LBB5_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    slw 8, 9, 3
+; CHECK-NEXT:    and 9, 7, 6
+; CHECK-NEXT:    or 9, 9, 8
+; CHECK-NEXT:  .LBB5_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB5_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 8, 0, 5
+; CHECK-NEXT:    cmplw   8, 7
+; CHECK-NEXT:    bne     0, .LBB5_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_5 Depth=2
+; CHECK-NEXT:    stwcx. 9, 0, 5
+; CHECK-NEXT:    bne     0, .LBB5_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 8, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
@@ -309,33 +299,32 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    lwz 6, 0(3)
 ; CHECK-NEXT:    b .LBB6_2
-; CHECK-NEXT:  .LBB6_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmplw 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB6_6
-; CHECK-NEXT:  .LBB6_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB6_4 Depth 2
-; CHECK-NEXT:    sub 5, 6, 4
-; CHECK-NEXT:    cmplw 5, 6
+; CHECK-NEXT:  .LBB6_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB6_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB6_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB6_4 Depth 2
+; CHECK-NEXT:    sub     5, 6, 4
+; CHECK-NEXT:    cmplw   5, 6
 ; CHECK-NEXT:    li 7, 0
-; CHECK-NEXT:    bgt 0, .LBB6_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    mr 7, 5
-; CHECK-NEXT:  .LBB6_4: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB6_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    bgt     0, .LBB6_4
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB6_2 Depth=1
+; CHECK-NEXT:    mr      7, 5
+; CHECK-NEXT:  .LBB6_4:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB6_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    lwarx 5, 0, 3
-; CHECK-NEXT:    cmpw 5, 6
-; CHECK-NEXT:    bne 0, .LBB6_1
-; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmplw   5, 6
+; CHECK-NEXT:    bne     0, .LBB6_1
+; CHECK-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB6_4 Depth=2
 ; CHECK-NEXT:    stwcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB6_4
-; CHECK-NEXT:    b .LBB6_1
-; CHECK-NEXT:  .LBB6_6: # %atomicrmw.end
+; CHECK-NEXT:    bne     0, .LBB6_4
+; CHECK-NEXT:  # %bb.6:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.7:                                # %atomicrmw.end
 ; CHECK-NEXT:    mr 3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
@@ -349,33 +338,32 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    ld 6, 0(3)
 ; CHECK-NEXT:    b .LBB7_2
-; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmpld 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB7_6
-; CHECK-NEXT:  .LBB7_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB7_4 Depth 2
-; CHECK-NEXT:    subc 5, 6, 4
+; CHECK-NEXT:  .LBB7_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB7_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB7_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB7_4 Depth 2
+; CHECK-NEXT:    subc    5, 6, 4
 ; CHECK-NEXT:    li 7, 0
 ; CHECK-NEXT:    addze. 8, 7
-; CHECK-NEXT:    beq 0, .LBB7_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    mr 7, 5
-; CHECK-NEXT:  .LBB7_4: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB7_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    beq     0, .LBB7_4
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB7_2 Depth=1
+; CHECK-NEXT:    mr      7, 5
+; CHECK-NEXT:  .LBB7_4:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB7_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldarx 5, 0, 3
-; CHECK-NEXT:    cmpd 5, 6
-; CHECK-NEXT:    bne 0, .LBB7_1
-; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmpld   5, 6
+; CHECK-NEXT:    bne     0, .LBB7_1
+; CHECK-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB7_4 Depth=2
 ; CHECK-NEXT:    stdcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB7_4
-; CHECK-NEXT:    b .LBB7_1
-; CHECK-NEXT:  .LBB7_6: # %atomicrmw.end
+; CHECK-NEXT:    bne     0, .LBB7_4
+; CHECK-NEXT:  # %bb.6:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.7:                                # %atomicrmw.end
 ; CHECK-NEXT:    mr 3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll
index 2882dc420b60..6ced47bd6bcb 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll
@@ -5,51 +5,49 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; CHECK-LABEL: atomicrmw_uinc_wrap_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    rlwinm 6, 5, 3, 27, 28
-; CHECK-NEXT:    lbz 3, 0(3)
-; CHECK-NEXT:    xori 6, 6, 24
-; CHECK-NEXT:    li 7, 255
-; CHECK-NEXT:    clrlwi 4, 4, 24
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 7, 7, 6
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    not     3, 3
+; CHECK-NEXT:    li 6, 255
+; CHECK-NEXT:    lwz 7, 0(5)
+; CHECK-NEXT:    rlwinm 3, 3, 3, 27, 28
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  4, 4, 24
 ; CHECK-NEXT:    b .LBB0_2
-; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 10, 6
-; CHECK-NEXT:    cmplw 3, 8
-; CHECK-NEXT:    beq 0, .LBB0_7
-; CHECK-NEXT:  .LBB0_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB0_5 Depth 2
-; CHECK-NEXT:    clrlwi 8, 3, 24
-; CHECK-NEXT:    cmplw 8, 4
+; CHECK-NEXT:  .LBB0_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  .LBB0_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB0_5 Depth 2
+; CHECK-NEXT:    srw 8, 7, 3
+; CHECK-NEXT:    clrlwi  9, 8, 24
+; CHECK-NEXT:    cmplw   9, 4
 ; CHECK-NEXT:    li 9, 0
 ; CHECK-NEXT:    bge 0, .LBB0_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    addi 9, 3, 1
-; CHECK-NEXT:  .LBB0_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 9, 6
-; CHECK-NEXT:    slw 9, 8, 6
-; CHECK-NEXT:    and 3, 3, 7
-; CHECK-NEXT:    and 9, 9, 7
-; CHECK-NEXT:  .LBB0_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB0_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 11, 0, 5
-; CHECK-NEXT:    and 10, 11, 7
-; CHECK-NEXT:    cmpw 10, 9
-; CHECK-NEXT:    bne 0, .LBB0_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 11, 11, 7
-; CHECK-NEXT:    or 11, 11, 3
-; CHECK-NEXT:    stwcx. 11, 0, 5
-; CHECK-NEXT:    bne 0, .LBB0_5
-; CHECK-NEXT:    b .LBB0_1
-; CHECK-NEXT:  .LBB0_7: # %atomicrmw.end
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    addi 9, 8, 1
+; CHECK-NEXT:  .LBB0_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    clrlwi  8, 9, 24
+; CHECK-NEXT:    slw 8, 8, 3
+; CHECK-NEXT:    and 9, 7, 6
+; CHECK-NEXT:    or 9, 9, 8
+; CHECK-NEXT:  .LBB0_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 8, 0, 5
+; CHECK-NEXT:    cmplw   8, 7
+; CHECK-NEXT:    bne     0, .LBB0_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB0_5 Depth=2
+; CHECK-NEXT:    stwcx. 9, 0, 5
+; CHECK-NEXT:    bne     0, .LBB0_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 8, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw uinc_wrap ptr %ptr, i8 %val seq_cst
@@ -60,52 +58,51 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; CHECK-LABEL: atomicrmw_uinc_wrap_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    li 7, 0
-; CHECK-NEXT:    lhz 3, 0(3)
-; CHECK-NEXT:    rlwinm 6, 5, 3, 27, 27
-; CHECK-NEXT:    xori 6, 6, 16
-; CHECK-NEXT:    ori 7, 7, 65535
-; CHECK-NEXT:    clrlwi 4, 4, 16
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 7, 7, 6
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    clrlwi  3, 3, 30
+; CHECK-NEXT:    lis 6, 0
+; CHECK-NEXT:    xori 3, 3, 2
+; CHECK-NEXT:    lwz 7, 0(5)
+; CHECK-NEXT:    ori 6, 6, 65535
+; CHECK-NEXT:    slwi 3, 3, 3
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  4, 4, 16
 ; CHECK-NEXT:    b .LBB1_2
-; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 10, 6
-; CHECK-NEXT:    cmplw 3, 8
-; CHECK-NEXT:    beq 0, .LBB1_7
-; CHECK-NEXT:  .LBB1_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB1_5 Depth 2
-; CHECK-NEXT:    clrlwi 8, 3, 16
-; CHECK-NEXT:    cmplw 8, 4
+; CHECK-NEXT:  .LBB1_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  .LBB1_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB1_5 Depth 2
+; CHECK-NEXT:    srw 8, 7, 3
+; CHECK-NEXT:    clrlwi  9, 8, 16
+; CHECK-NEXT:    cmplw   9, 4
 ; CHECK-NEXT:    li 9, 0
 ; CHECK-NEXT:    bge 0, .LBB1_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    addi 9, 3, 1
-; CHECK-NEXT:  .LBB1_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 9, 6
-; CHECK-NEXT:    slw 9, 8, 6
-; CHECK-NEXT:    and 3, 3, 7
-; CHECK-NEXT:    and 9, 9, 7
-; CHECK-NEXT:  .LBB1_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB1_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 11, 0, 5
-; CHECK-NEXT:    and 10, 11, 7
-; CHECK-NEXT:    cmpw 10, 9
-; CHECK-NEXT:    bne 0, .LBB1_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 11, 11, 7
-; CHECK-NEXT:    or 11, 11, 3
-; CHECK-NEXT:    stwcx. 11, 0, 5
-; CHECK-NEXT:    bne 0, .LBB1_5
-; CHECK-NEXT:    b .LBB1_1
-; CHECK-NEXT:  .LBB1_7: # %atomicrmw.end
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    addi 9, 8, 1
+; CHECK-NEXT:  .LBB1_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    clrlwi  8, 9, 16
+; CHECK-NEXT:    slw 8, 8, 3
+; CHECK-NEXT:    and 9, 7, 6
+; CHECK-NEXT:    or 9, 9, 8
+; CHECK-NEXT:  .LBB1_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB1_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 8, 0, 5
+; CHECK-NEXT:    cmplw   8, 7
+; CHECK-NEXT:    bne     0, .LBB1_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB1_5 Depth=2
+; CHECK-NEXT:    stwcx. 9, 0, 5
+; CHECK-NEXT:    bne     0, .LBB1_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      7, 8
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 8, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw uinc_wrap ptr %ptr, i16 %val seq_cst
@@ -118,32 +115,31 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    lwz 6, 0(3)
 ; CHECK-NEXT:    b .LBB2_2
-; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmplw 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB2_6
-; CHECK-NEXT:  .LBB2_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB2_4 Depth 2
-; CHECK-NEXT:    cmplw 6, 4
+; CHECK-NEXT:  .LBB2_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB2_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB2_4 Depth 2
+; CHECK-NEXT:    cmplw   6, 4
 ; CHECK-NEXT:    li 7, 0
 ; CHECK-NEXT:    bge 0, .LBB2_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB2_2 Depth=1
 ; CHECK-NEXT:    addi 7, 6, 1
-; CHECK-NEXT:  .LBB2_4: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB2_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:  .LBB2_4:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB2_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    lwarx 5, 0, 3
-; CHECK-NEXT:    cmpw 5, 6
-; CHECK-NEXT:    bne 0, .LBB2_1
-; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmplw   5, 6
+; CHECK-NEXT:    bne     0, .LBB2_1
+; CHECK-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB2_4 Depth=2
 ; CHECK-NEXT:    stwcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB2_4
-; CHECK-NEXT:    b .LBB2_1
-; CHECK-NEXT:  .LBB2_6: # %atomicrmw.end
+; CHECK-NEXT:    bne     0, .LBB2_4
+; CHECK-NEXT:  # %bb.6:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.7:                                # %atomicrmw.end
 ; CHECK-NEXT:    mr 3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
@@ -157,32 +153,31 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    ld 6, 0(3)
 ; CHECK-NEXT:    b .LBB3_2
-; CHECK-NEXT:  .LBB3_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmpld 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB3_6
-; CHECK-NEXT:  .LBB3_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB3_4 Depth 2
-; CHECK-NEXT:    cmpld 6, 4
+; CHECK-NEXT:  .LBB3_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB3_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB3_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB3_4 Depth 2
+; CHECK-NEXT:    cmpld   6, 4
 ; CHECK-NEXT:    li 7, 0
 ; CHECK-NEXT:    bge 0, .LBB3_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB3_2 Depth=1
 ; CHECK-NEXT:    addi 7, 6, 1
-; CHECK-NEXT:  .LBB3_4: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB3_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:  .LBB3_4:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB3_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldarx 5, 0, 3
-; CHECK-NEXT:    cmpd 5, 6
-; CHECK-NEXT:    bne 0, .LBB3_1
-; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmpld   5, 6
+; CHECK-NEXT:    bne     0, .LBB3_1
+; CHECK-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB3_4 Depth=2
 ; CHECK-NEXT:    stdcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB3_4
-; CHECK-NEXT:    b .LBB3_1
-; CHECK-NEXT:  .LBB3_6: # %atomicrmw.end
+; CHECK-NEXT:    bne     0, .LBB3_4
+; CHECK-NEXT:  # %bb.6:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.7:                                # %atomicrmw.end
 ; CHECK-NEXT:    mr 3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
@@ -194,52 +189,50 @@ define i8 @atomicrmw_udec_wrap_i8(ptr %ptr, i8 %val) {
 ; CHECK-LABEL: atomicrmw_udec_wrap_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    rlwinm 7, 5, 3, 27, 28
-; CHECK-NEXT:    lbz 3, 0(3)
-; CHECK-NEXT:    xori 7, 7, 24
-; CHECK-NEXT:    li 8, 255
-; CHECK-NEXT:    clrlwi 6, 4, 24
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 8, 8, 7
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    not     3, 3
+; CHECK-NEXT:    li 6, 255
+; CHECK-NEXT:    lwz 8, 0(5)
+; CHECK-NEXT:    rlwinm 3, 3, 3, 27, 28
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  7, 4, 24
 ; CHECK-NEXT:    b .LBB4_2
-; CHECK-NEXT:  .LBB4_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 11, 7
-; CHECK-NEXT:    cmplw 3, 9
-; CHECK-NEXT:    beq 0, .LBB4_7
-; CHECK-NEXT:  .LBB4_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB4_5 Depth 2
-; CHECK-NEXT:    andi. 9, 3, 255
-; CHECK-NEXT:    cmplw 1, 9, 6
+; CHECK-NEXT:  .LBB4_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  .LBB4_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB4_5 Depth 2
+; CHECK-NEXT:    srw 9, 8, 3
+; CHECK-NEXT:    andi. 10, 9, 255
+; CHECK-NEXT:    cmplw 1, 10, 7
 ; CHECK-NEXT:    cror 20, 2, 5
-; CHECK-NEXT:    mr 10, 4
+; CHECK-NEXT:    mr      10, 4
 ; CHECK-NEXT:    bc 12, 20, .LBB4_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    addi 10, 3, -1
-; CHECK-NEXT:  .LBB4_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 10, 7
-; CHECK-NEXT:    slw 10, 9, 7
-; CHECK-NEXT:    and 3, 3, 8
-; CHECK-NEXT:    and 10, 10, 8
-; CHECK-NEXT:  .LBB4_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB4_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 12, 0, 5
-; CHECK-NEXT:    and 11, 12, 8
-; CHECK-NEXT:    cmpw 11, 10
-; CHECK-NEXT:    bne 0, .LBB4_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 12, 12, 8
-; CHECK-NEXT:    or 12, 12, 3
-; CHECK-NEXT:    stwcx. 12, 0, 5
-; CHECK-NEXT:    bne 0, .LBB4_5
-; CHECK-NEXT:    b .LBB4_1
-; CHECK-NEXT:  .LBB4_7: # %atomicrmw.end
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    addi 10, 9, -1
+; CHECK-NEXT:  .LBB4_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    clrlwi  9, 10, 24
+; CHECK-NEXT:    slw 9, 9, 3
+; CHECK-NEXT:    and 10, 8, 6
+; CHECK-NEXT:    or 10, 10, 9
+; CHECK-NEXT:  .LBB4_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB4_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 9, 0, 5
+; CHECK-NEXT:    cmplw   9, 8
+; CHECK-NEXT:    bne     0, .LBB4_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB4_5 Depth=2
+; CHECK-NEXT:    stwcx. 10, 0, 5
+; CHECK-NEXT:    bne     0, .LBB4_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 9, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw udec_wrap ptr %ptr, i8 %val seq_cst
@@ -250,53 +243,52 @@ define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) {
 ; CHECK-LABEL: atomicrmw_udec_wrap_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    mr 5, 3
-; CHECK-NEXT:    li 8, 0
-; CHECK-NEXT:    lhz 3, 0(3)
-; CHECK-NEXT:    rlwinm 7, 5, 3, 27, 27
-; CHECK-NEXT:    xori 7, 7, 16
-; CHECK-NEXT:    ori 8, 8, 65535
-; CHECK-NEXT:    clrlwi 6, 4, 16
-; CHECK-NEXT:    rldicr 5, 5, 0, 61
-; CHECK-NEXT:    slw 8, 8, 7
+; CHECK-NEXT:    rldicr 5, 3, 0, 61
+; CHECK-NEXT:    clrlwi  3, 3, 30
+; CHECK-NEXT:    lis 6, 0
+; CHECK-NEXT:    xori 3, 3, 2
+; CHECK-NEXT:    lwz 8, 0(5)
+; CHECK-NEXT:    ori 6, 6, 65535
+; CHECK-NEXT:    slwi 3, 3, 3
+; CHECK-NEXT:    slw 6, 6, 3
+; CHECK-NEXT:    not     6, 6
+; CHECK-NEXT:    clrlwi  7, 4, 16
 ; CHECK-NEXT:    b .LBB5_2
-; CHECK-NEXT:  .LBB5_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    srw 3, 11, 7
-; CHECK-NEXT:    cmplw 3, 9
-; CHECK-NEXT:    beq 0, .LBB5_7
-; CHECK-NEXT:  .LBB5_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB5_5 Depth 2
-; CHECK-NEXT:    andi. 9, 3, 65535
-; CHECK-NEXT:    cmplw 1, 9, 6
+; CHECK-NEXT:  .LBB5_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  .LBB5_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB5_5 Depth 2
+; CHECK-NEXT:    srw 9, 8, 3
+; CHECK-NEXT:    andi. 10, 9, 65535
+; CHECK-NEXT:    cmplw 1, 10, 7
 ; CHECK-NEXT:    cror 20, 2, 5
-; CHECK-NEXT:    mr 10, 4
+; CHECK-NEXT:    mr      10, 4
 ; CHECK-NEXT:    bc 12, 20, .LBB5_4
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    addi 10, 3, -1
-; CHECK-NEXT:  .LBB5_4: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    slw 3, 10, 7
-; CHECK-NEXT:    slw 10, 9, 7
-; CHECK-NEXT:    and 3, 3, 8
-; CHECK-NEXT:    and 10, 10, 8
-; CHECK-NEXT:  .LBB5_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB5_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lwarx 12, 0, 5
-; CHECK-NEXT:    and 11, 12, 8
-; CHECK-NEXT:    cmpw 11, 10
-; CHECK-NEXT:    bne 0, .LBB5_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    andc 12, 12, 8
-; CHECK-NEXT:    or 12, 12, 3
-; CHECK-NEXT:    stwcx. 12, 0, 5
-; CHECK-NEXT:    bne 0, .LBB5_5
-; CHECK-NEXT:    b .LBB5_1
-; CHECK-NEXT:  .LBB5_7: # %atomicrmw.end
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    addi 10, 9, -1
+; CHECK-NEXT:  .LBB5_4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    clrlwi  9, 10, 16
+; CHECK-NEXT:    slw 9, 9, 3
+; CHECK-NEXT:    and 10, 8, 6
+; CHECK-NEXT:    or 10, 10, 9
+; CHECK-NEXT:  .LBB5_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB5_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lwarx 9, 0, 5
+; CHECK-NEXT:    cmplw   9, 8
+; CHECK-NEXT:    bne     0, .LBB5_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB5_5 Depth=2
+; CHECK-NEXT:    stwcx. 10, 0, 5
+; CHECK-NEXT:    bne     0, .LBB5_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      8, 9
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    srw 3, 9, 3
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw udec_wrap ptr %ptr, i16 %val seq_cst
@@ -309,37 +301,36 @@ define i32 @atomicrmw_udec_wrap_i32(ptr %ptr, i32 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    lwz 6, 0(3)
 ; CHECK-NEXT:    b .LBB6_2
-; CHECK-NEXT:  .LBB6_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmplw 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB6_7
-; CHECK-NEXT:  .LBB6_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB6_5 Depth 2
-; CHECK-NEXT:    cmpwi 6, 0
-; CHECK-NEXT:    mr 7, 4
+; CHECK-NEXT:  .LBB6_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB6_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB6_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB6_5 Depth 2
+; CHECK-NEXT:    cmpwi   6, 0
+; CHECK-NEXT:    mr      7, 4
 ; CHECK-NEXT:    bc 12, 2, .LBB6_5
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmplw 6, 4
-; CHECK-NEXT:    mr 7, 4
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB6_2 Depth=1
+; CHECK-NEXT:    cmplw   6, 4
+; CHECK-NEXT:    mr      7, 4
 ; CHECK-NEXT:    bc 12, 1, .LBB6_5
-; CHECK-NEXT:  # %bb.4: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:  # %bb.4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB6_2 Depth=1
 ; CHECK-NEXT:    addi 7, 6, -1
-; CHECK-NEXT:  .LBB6_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB6_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:  .LBB6_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB6_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    lwarx 5, 0, 3
-; CHECK-NEXT:    cmpw 5, 6
-; CHECK-NEXT:    bne 0, .LBB6_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmplw   5, 6
+; CHECK-NEXT:    bne     0, .LBB6_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB6_5 Depth=2
 ; CHECK-NEXT:    stwcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB6_5
-; CHECK-NEXT:    b .LBB6_1
-; CHECK-NEXT:  .LBB6_7: # %atomicrmw.end
+; CHECK-NEXT:    bne     0, .LBB6_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
 ; CHECK-NEXT:    mr 3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
@@ -353,38 +344,37 @@ define i64 @atomicrmw_udec_wrap_i64(ptr %ptr, i64 %val) {
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    ld 6, 0(3)
 ; CHECK-NEXT:    b .LBB7_2
-; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmpld 5, 6
-; CHECK-NEXT:    mr 6, 5
-; CHECK-NEXT:    beq 0, .LBB7_7
-; CHECK-NEXT:  .LBB7_2: # %atomicrmw.start
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB7_5 Depth 2
-; CHECK-NEXT:    cmpdi 6, 0
-; CHECK-NEXT:    mr 7, 4
+; CHECK-NEXT:  .LBB7_1:                                # %cmpxchg.nostore
+; CHECK-NEXT:                                          #   in Loop: Header=BB7_2 Depth=1
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  .LBB7_2:                                # %atomicrmw.start
+; CHECK-NEXT:                                          # =>This Loop Header: Depth=1
+; CHECK-NEXT:                                          #     Child Loop BB7_5 Depth 2
+; CHECK-NEXT:    cmpdi   6, 0
+; CHECK-NEXT:    mr      7, 4
 ; CHECK-NEXT:    bc 12, 2, .LBB7_5
-; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
-; CHECK-NEXT:    #
-; CHECK-NEXT:    cmpld 6, 4
-; CHECK-NEXT:    mr 7, 4
+; CHECK-NEXT:  # %bb.3:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB7_2 Depth=1
+; CHECK-NEXT:    cmpld   6, 4
+; CHECK-NEXT:    mr      7, 4
 ; CHECK-NEXT:    bc 12, 1, .LBB7_5
-; CHECK-NEXT:  # %bb.4: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:  # %bb.4:                                # %atomicrmw.start
+; CHECK-NEXT:                                          #   in Loop: Header=BB7_2 Depth=1
 ; CHECK-NEXT:    addi 7, 6, -1
-; CHECK-NEXT:  .LBB7_5: # %atomicrmw.start
-; CHECK-NEXT:    # Parent Loop BB7_2 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:  .LBB7_5:                                # %cmpxchg.start
+; CHECK-NEXT:                                          #   Parent Loop BB7_2 Depth=1
+; CHECK-NEXT:                                          # =>  This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldarx 5, 0, 3
-; CHECK-NEXT:    cmpd 5, 6
-; CHECK-NEXT:    bne 0, .LBB7_1
-; CHECK-NEXT:  # %bb.6: # %atomicrmw.start
-; CHECK-NEXT:    #
+; CHECK-NEXT:    cmpld   5, 6
+; CHECK-NEXT:    bne     0, .LBB7_1
+; CHECK-NEXT:  # %bb.6:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:                                          #   in Loop: Header=BB7_5 Depth=2
 ; CHECK-NEXT:    stdcx. 7, 0, 3
-; CHECK-NEXT:    bne 0, .LBB7_5
-; CHECK-NEXT:    b .LBB7_1
-; CHECK-NEXT:  .LBB7_7: # %atomicrmw.end
-; CHECK-NEXT:    mr 3, 5
+; CHECK-NEXT:    bne     0, .LBB7_5
+; CHECK-NEXT:  # %bb.7:
+; CHECK-NEXT:    mr      6, 5
+; CHECK-NEXT:  # %bb.8:                                # %atomicrmw.end
+; CHECK-NEXT:    mr      3, 5
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %result = atomicrmw udec_wrap ptr %ptr, i64 %val seq_cst
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics-regression.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics-regression.ll
index b31be701454d..280c4299c30b 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics-regression.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics-regression.ll
@@ -400,15 +400,20 @@ define void @test39() {
 define void @test40(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test40:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB40_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB40_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB40_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
   ret void
@@ -417,15 +422,20 @@ define void @test40(ptr %ptr, i8 %cmp, i8 %val) {
 define void @test41(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test41:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB41_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB41_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB41_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB41_1
-; PPC64LE-NEXT:  .LBB41_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire monotonic
@@ -435,15 +445,20 @@ define void @test41(ptr %ptr, i8 %cmp, i8 %val) {
 define void @test42(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test42:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB42_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB42_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB42_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB42_1
-; PPC64LE-NEXT:  .LBB42_3:
+; PPC64LE-NEXT:  .LBB42_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire acquire
@@ -452,17 +467,26 @@ define void @test42(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test43(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test43:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB43_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB43_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB43_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB43_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val release monotonic
   ret void
@@ -470,17 +494,27 @@ define void @test43(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test44(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test44:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB44_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB44_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB44_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB44_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB44_1
-; PPC64LE-NEXT:  .LBB44_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB44_2
+; PPC64LE-NEXT:  .LBB44_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val release acquire
@@ -489,17 +523,29 @@ define void @test44(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test45(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test45:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB45_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB45_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB45_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB45_1
-; PPC64LE-NEXT:  .LBB45_3:
+; PPC64LE-NEXT:    beq 0, .LBB45_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB45_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB45_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acq_rel monotonic
@@ -508,17 +554,27 @@ define void @test45(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test46(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test46:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB46_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB46_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB46_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB46_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB46_1
-; PPC64LE-NEXT:  .LBB46_3:
+; PPC64LE-NEXT:    beq 0, .LBB46_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB46_2
+; PPC64LE-NEXT:  .LBB46_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acq_rel acquire
@@ -527,17 +583,29 @@ define void @test46(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test47(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test47:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB47_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB47_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB47_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB47_1
-; PPC64LE-NEXT:  .LBB47_3:
+; PPC64LE-NEXT:    beq 0, .LBB47_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB47_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB47_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val seq_cst monotonic
@@ -546,17 +614,27 @@ define void @test47(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test48(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test48:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB48_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB48_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB48_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB48_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB48_1
-; PPC64LE-NEXT:  .LBB48_3:
+; PPC64LE-NEXT:    beq 0, .LBB48_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB48_2
+; PPC64LE-NEXT:  .LBB48_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val seq_cst acquire
@@ -565,17 +643,27 @@ define void @test48(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test49(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test49:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB49_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB49_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB49_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB49_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB49_1
-; PPC64LE-NEXT:  .LBB49_3:
+; PPC64LE-NEXT:    beq 0, .LBB49_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB49_2
+; PPC64LE-NEXT:  .LBB49_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val seq_cst seq_cst
@@ -585,15 +673,20 @@ define void @test49(ptr %ptr, i8 %cmp, i8 %val) {
 define void @test50(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test50:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB50_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB50_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB50_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic
   ret void
@@ -602,15 +695,20 @@ define void @test50(ptr %ptr, i16 %cmp, i16 %val) {
 define void @test51(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test51:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB51_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB51_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB51_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB51_1
-; PPC64LE-NEXT:  .LBB51_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire monotonic
@@ -620,15 +718,20 @@ define void @test51(ptr %ptr, i16 %cmp, i16 %val) {
 define void @test52(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test52:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB52_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB52_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB52_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB52_1
-; PPC64LE-NEXT:  .LBB52_3:
+; PPC64LE-NEXT:  .LBB52_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire
@@ -637,17 +740,26 @@ define void @test52(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test53(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test53:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB53_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB53_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB53_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB53_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val release monotonic
   ret void
@@ -655,17 +767,27 @@ define void @test53(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test54(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test54:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB54_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB54_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB54_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB54_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB54_1
-; PPC64LE-NEXT:  .LBB54_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB54_2
+; PPC64LE-NEXT:  .LBB54_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val release acquire
@@ -674,17 +796,29 @@ define void @test54(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test55(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test55:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB55_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB55_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB55_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB55_1
-; PPC64LE-NEXT:  .LBB55_3:
+; PPC64LE-NEXT:    beq 0, .LBB55_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB55_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB55_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acq_rel monotonic
@@ -693,17 +827,27 @@ define void @test55(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test56(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test56:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB56_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB56_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB56_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB56_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB56_1
-; PPC64LE-NEXT:  .LBB56_3:
+; PPC64LE-NEXT:    beq 0, .LBB56_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB56_2
+; PPC64LE-NEXT:  .LBB56_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acq_rel acquire
@@ -712,17 +856,29 @@ define void @test56(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test57(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test57:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB57_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB57_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB57_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB57_1
-; PPC64LE-NEXT:  .LBB57_3:
+; PPC64LE-NEXT:    beq 0, .LBB57_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB57_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB57_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val seq_cst monotonic
@@ -731,17 +887,27 @@ define void @test57(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test58(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test58:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB58_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB58_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB58_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB58_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB58_1
-; PPC64LE-NEXT:  .LBB58_3:
+; PPC64LE-NEXT:    beq 0, .LBB58_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB58_2
+; PPC64LE-NEXT:  .LBB58_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val seq_cst acquire
@@ -750,17 +916,27 @@ define void @test58(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test59(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test59:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB59_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB59_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB59_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB59_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB59_1
-; PPC64LE-NEXT:  .LBB59_3:
+; PPC64LE-NEXT:    beq 0, .LBB59_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB59_2
+; PPC64LE-NEXT:  .LBB59_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val seq_cst seq_cst
@@ -770,14 +946,17 @@ define void @test59(ptr %ptr, i16 %cmp, i16 %val) {
 define void @test60(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test60:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB60_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB60_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB60_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
   ret void
@@ -786,14 +965,17 @@ define void @test60(ptr %ptr, i32 %cmp, i32 %val) {
 define void @test61(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test61:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB61_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB61_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB61_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB61_1
-; PPC64LE-NEXT:  .LBB61_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire monotonic
@@ -803,14 +985,17 @@ define void @test61(ptr %ptr, i32 %cmp, i32 %val) {
 define void @test62(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test62:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB62_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB62_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB62_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB62_1
-; PPC64LE-NEXT:  .LBB62_3:
+; PPC64LE-NEXT:  .LBB62_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire acquire
@@ -819,16 +1004,22 @@ define void @test62(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test63(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test63:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB63_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB63_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB63_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB63_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val release monotonic
   ret void
@@ -836,16 +1027,23 @@ define void @test63(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test64(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test64:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB64_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB64_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB64_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB64_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB64_1
-; PPC64LE-NEXT:  .LBB64_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB64_2
+; PPC64LE-NEXT:  .LBB64_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val release acquire
@@ -854,16 +1052,25 @@ define void @test64(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test65(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test65:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB65_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB65_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB65_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB65_1
-; PPC64LE-NEXT:  .LBB65_3:
+; PPC64LE-NEXT:    beq 0, .LBB65_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB65_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB65_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acq_rel monotonic
@@ -872,16 +1079,23 @@ define void @test65(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test66(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test66:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB66_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB66_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB66_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB66_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB66_1
-; PPC64LE-NEXT:  .LBB66_3:
+; PPC64LE-NEXT:    beq 0, .LBB66_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB66_2
+; PPC64LE-NEXT:  .LBB66_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acq_rel acquire
@@ -890,16 +1104,25 @@ define void @test66(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test67(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test67:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB67_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB67_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB67_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB67_1
-; PPC64LE-NEXT:  .LBB67_3:
+; PPC64LE-NEXT:    beq 0, .LBB67_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB67_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB67_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst monotonic
@@ -908,16 +1131,23 @@ define void @test67(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test68(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test68:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB68_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB68_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB68_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB68_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB68_1
-; PPC64LE-NEXT:  .LBB68_3:
+; PPC64LE-NEXT:    beq 0, .LBB68_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB68_2
+; PPC64LE-NEXT:  .LBB68_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst acquire
@@ -926,16 +1156,23 @@ define void @test68(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test69(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test69:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB69_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB69_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB69_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB69_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB69_1
-; PPC64LE-NEXT:  .LBB69_3:
+; PPC64LE-NEXT:    beq 0, .LBB69_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB69_2
+; PPC64LE-NEXT:  .LBB69_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst seq_cst
@@ -945,14 +1182,17 @@ define void @test69(ptr %ptr, i32 %cmp, i32 %val) {
 define void @test70(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test70:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB70_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB70_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
+; PPC64LE-NEXT:    cmpld 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB70_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val monotonic monotonic
   ret void
@@ -961,14 +1201,17 @@ define void @test70(ptr %ptr, i64 %cmp, i64 %val) {
 define void @test71(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test71:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB71_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB71_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB71_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB71_1
-; PPC64LE-NEXT:  .LBB71_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire monotonic
@@ -978,14 +1221,17 @@ define void @test71(ptr %ptr, i64 %cmp, i64 %val) {
 define void @test72(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test72:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB72_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB72_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
+; PPC64LE-NEXT:    cmpld 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB72_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB72_1
-; PPC64LE-NEXT:  .LBB72_3:
+; PPC64LE-NEXT:  .LBB72_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire acquire
@@ -994,16 +1240,22 @@ define void @test72(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test73(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test73:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB73_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
+; PPC64LE-NEXT:    cmpld 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB73_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB73_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB73_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val release monotonic
   ret void
@@ -1011,16 +1263,23 @@ define void @test73(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test74(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test74:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB74_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB74_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB74_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB74_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB74_1
-; PPC64LE-NEXT:  .LBB74_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB74_2
+; PPC64LE-NEXT:  .LBB74_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val release acquire
@@ -1029,16 +1288,25 @@ define void @test74(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test75(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test75:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB75_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB75_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB75_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB75_1
-; PPC64LE-NEXT:  .LBB75_3:
+; PPC64LE-NEXT:    beq 0, .LBB75_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB75_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB75_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acq_rel monotonic
@@ -1047,16 +1315,23 @@ define void @test75(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test76(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test76:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB76_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB76_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB76_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB76_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB76_1
-; PPC64LE-NEXT:  .LBB76_3:
+; PPC64LE-NEXT:    beq 0, .LBB76_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB76_2
+; PPC64LE-NEXT:  .LBB76_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acq_rel acquire
@@ -1065,16 +1340,25 @@ define void @test76(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test77(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test77:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB77_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB77_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB77_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB77_1
-; PPC64LE-NEXT:  .LBB77_3:
+; PPC64LE-NEXT:    beq 0, .LBB77_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB77_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB77_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val seq_cst monotonic
@@ -1083,16 +1367,23 @@ define void @test77(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test78(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test78:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB78_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB78_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB78_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB78_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB78_1
-; PPC64LE-NEXT:  .LBB78_3:
+; PPC64LE-NEXT:    beq 0, .LBB78_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB78_2
+; PPC64LE-NEXT:  .LBB78_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val seq_cst acquire
@@ -1101,16 +1392,23 @@ define void @test78(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test79(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test79:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB79_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB79_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB79_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB79_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB79_1
-; PPC64LE-NEXT:  .LBB79_3:
+; PPC64LE-NEXT:    beq 0, .LBB79_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB79_2
+; PPC64LE-NEXT:  .LBB79_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val seq_cst seq_cst
@@ -1120,15 +1418,20 @@ define void @test79(ptr %ptr, i64 %cmp, i64 %val) {
 define void @test80(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test80:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB80_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB80_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB80_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") monotonic monotonic
   ret void
@@ -1137,15 +1440,20 @@ define void @test80(ptr %ptr, i8 %cmp, i8 %val) {
 define void @test81(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test81:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB81_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB81_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB81_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB81_1
-; PPC64LE-NEXT:  .LBB81_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") acquire monotonic
@@ -1155,15 +1463,20 @@ define void @test81(ptr %ptr, i8 %cmp, i8 %val) {
 define void @test82(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test82:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB82_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB82_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB82_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB82_1
-; PPC64LE-NEXT:  .LBB82_3:
+; PPC64LE-NEXT:  .LBB82_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") acquire acquire
@@ -1172,17 +1485,26 @@ define void @test82(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test83(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test83:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB83_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB83_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB83_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB83_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") release monotonic
   ret void
@@ -1190,17 +1512,27 @@ define void @test83(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test84(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test84:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB84_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB84_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB84_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB84_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB84_1
-; PPC64LE-NEXT:  .LBB84_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB84_2
+; PPC64LE-NEXT:  .LBB84_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") release acquire
@@ -1209,17 +1541,29 @@ define void @test84(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test85(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test85:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB85_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB85_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB85_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB85_1
-; PPC64LE-NEXT:  .LBB85_3:
+; PPC64LE-NEXT:    beq 0, .LBB85_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB85_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB85_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") acq_rel monotonic
@@ -1228,17 +1572,27 @@ define void @test85(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test86(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test86:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB86_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB86_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB86_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB86_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB86_1
-; PPC64LE-NEXT:  .LBB86_3:
+; PPC64LE-NEXT:    beq 0, .LBB86_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB86_2
+; PPC64LE-NEXT:  .LBB86_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") acq_rel acquire
@@ -1247,17 +1601,29 @@ define void @test86(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test87(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test87:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB87_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB87_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB87_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB87_1
-; PPC64LE-NEXT:  .LBB87_3:
+; PPC64LE-NEXT:    beq 0, .LBB87_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB87_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB87_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") seq_cst monotonic
@@ -1266,17 +1632,27 @@ define void @test87(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test88(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test88:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB88_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB88_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB88_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB88_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB88_1
-; PPC64LE-NEXT:  .LBB88_3:
+; PPC64LE-NEXT:    beq 0, .LBB88_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB88_2
+; PPC64LE-NEXT:  .LBB88_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") seq_cst acquire
@@ -1285,17 +1661,27 @@ define void @test88(ptr %ptr, i8 %cmp, i8 %val) {
 
 define void @test89(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test89:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB89_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB89_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB89_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB89_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB89_1
-; PPC64LE-NEXT:  .LBB89_3:
+; PPC64LE-NEXT:    beq 0, .LBB89_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB89_2
+; PPC64LE-NEXT:  .LBB89_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val syncscope("singlethread") seq_cst seq_cst
@@ -1305,15 +1691,20 @@ define void @test89(ptr %ptr, i8 %cmp, i8 %val) {
 define void @test90(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test90:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB90_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB90_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB90_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") monotonic monotonic
   ret void
@@ -1322,15 +1713,20 @@ define void @test90(ptr %ptr, i16 %cmp, i16 %val) {
 define void @test91(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test91:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB91_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB91_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB91_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB91_1
-; PPC64LE-NEXT:  .LBB91_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") acquire monotonic
@@ -1340,15 +1736,20 @@ define void @test91(ptr %ptr, i16 %cmp, i16 %val) {
 define void @test92(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test92:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB92_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB92_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB92_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB92_1
-; PPC64LE-NEXT:  .LBB92_3:
+; PPC64LE-NEXT:  .LBB92_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") acquire acquire
@@ -1357,17 +1758,26 @@ define void @test92(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test93(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test93:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB93_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB93_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB93_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB93_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") release monotonic
   ret void
@@ -1375,17 +1785,27 @@ define void @test93(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test94(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test94:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB94_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB94_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB94_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB94_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB94_1
-; PPC64LE-NEXT:  .LBB94_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB94_2
+; PPC64LE-NEXT:  .LBB94_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") release acquire
@@ -1394,17 +1814,29 @@ define void @test94(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test95(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test95:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB95_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB95_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB95_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB95_1
-; PPC64LE-NEXT:  .LBB95_3:
+; PPC64LE-NEXT:    beq 0, .LBB95_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB95_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB95_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") acq_rel monotonic
@@ -1413,17 +1845,27 @@ define void @test95(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test96(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test96:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB96_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB96_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB96_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB96_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB96_1
-; PPC64LE-NEXT:  .LBB96_3:
+; PPC64LE-NEXT:    beq 0, .LBB96_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB96_2
+; PPC64LE-NEXT:  .LBB96_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") acq_rel acquire
@@ -1432,17 +1874,29 @@ define void @test96(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test97(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test97:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB97_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB97_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB97_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB97_1
-; PPC64LE-NEXT:  .LBB97_3:
+; PPC64LE-NEXT:    beq 0, .LBB97_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB97_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB97_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") seq_cst monotonic
@@ -1451,17 +1905,27 @@ define void @test97(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test98(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test98:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB98_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB98_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB98_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB98_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB98_1
-; PPC64LE-NEXT:  .LBB98_3:
+; PPC64LE-NEXT:    beq 0, .LBB98_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB98_2
+; PPC64LE-NEXT:  .LBB98_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") seq_cst acquire
@@ -1470,17 +1934,27 @@ define void @test98(ptr %ptr, i16 %cmp, i16 %val) {
 
 define void @test99(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test99:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:  .LBB99_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB99_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    clrlwi 4, 4, 16
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB99_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    clrlwi 5, 5, 16
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB99_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB99_1
-; PPC64LE-NEXT:  .LBB99_3:
+; PPC64LE-NEXT:    beq 0, .LBB99_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lharx 6, 0, 3
+; PPC64LE-NEXT:    clrlwi 6, 6, 16
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB99_2
+; PPC64LE-NEXT:  .LBB99_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val syncscope("singlethread") seq_cst seq_cst
@@ -1490,14 +1964,17 @@ define void @test99(ptr %ptr, i16 %cmp, i16 %val) {
 define void @test100(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test100:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB100_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB100_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB100_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") monotonic monotonic
   ret void
@@ -1506,14 +1983,17 @@ define void @test100(ptr %ptr, i32 %cmp, i32 %val) {
 define void @test101(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test101:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB101_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB101_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB101_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB101_1
-; PPC64LE-NEXT:  .LBB101_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") acquire monotonic
@@ -1523,14 +2003,17 @@ define void @test101(ptr %ptr, i32 %cmp, i32 %val) {
 define void @test102(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test102:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB102_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB102_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB102_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB102_1
-; PPC64LE-NEXT:  .LBB102_3:
+; PPC64LE-NEXT:  .LBB102_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") acquire acquire
@@ -1539,16 +2022,22 @@ define void @test102(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test103(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test103:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB103_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB103_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB103_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB103_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") release monotonic
   ret void
@@ -1556,16 +2045,23 @@ define void @test103(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test104(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test104:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB104_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB104_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB104_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB104_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB104_1
-; PPC64LE-NEXT:  .LBB104_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB104_2
+; PPC64LE-NEXT:  .LBB104_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") release acquire
@@ -1574,16 +2070,25 @@ define void @test104(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test105(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test105:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB105_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB105_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB105_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB105_1
-; PPC64LE-NEXT:  .LBB105_3:
+; PPC64LE-NEXT:    beq 0, .LBB105_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB105_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB105_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") acq_rel monotonic
@@ -1592,16 +2097,23 @@ define void @test105(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test106(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test106:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB106_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB106_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB106_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB106_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB106_1
-; PPC64LE-NEXT:  .LBB106_3:
+; PPC64LE-NEXT:    beq 0, .LBB106_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB106_2
+; PPC64LE-NEXT:  .LBB106_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") acq_rel acquire
@@ -1610,16 +2122,25 @@ define void @test106(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test107(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test107:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB107_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB107_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB107_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB107_1
-; PPC64LE-NEXT:  .LBB107_3:
+; PPC64LE-NEXT:    beq 0, .LBB107_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB107_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB107_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") seq_cst monotonic
@@ -1628,16 +2149,23 @@ define void @test107(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test108(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test108:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB108_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB108_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB108_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB108_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB108_1
-; PPC64LE-NEXT:  .LBB108_3:
+; PPC64LE-NEXT:    beq 0, .LBB108_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB108_2
+; PPC64LE-NEXT:  .LBB108_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") seq_cst acquire
@@ -1646,16 +2174,23 @@ define void @test108(ptr %ptr, i32 %cmp, i32 %val) {
 
 define void @test109(ptr %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test109:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB109_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB109_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB109_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB109_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB109_1
-; PPC64LE-NEXT:  .LBB109_3:
+; PPC64LE-NEXT:    beq 0, .LBB109_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    lwarx 6, 0, 3
+; PPC64LE-NEXT:    cmplw 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB109_2
+; PPC64LE-NEXT:  .LBB109_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val syncscope("singlethread") seq_cst seq_cst
@@ -1665,14 +2200,17 @@ define void @test109(ptr %ptr, i32 %cmp, i32 %val) {
 define void @test110(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test110:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB110_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB110_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
+; PPC64LE-NEXT:    cmpld 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB110_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") monotonic monotonic
   ret void
@@ -1681,14 +2219,17 @@ define void @test110(ptr %ptr, i64 %cmp, i64 %val) {
 define void @test111(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test111:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB111_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB111_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB111_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB111_1
-; PPC64LE-NEXT:  .LBB111_3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") acquire monotonic
@@ -1698,14 +2239,17 @@ define void @test111(ptr %ptr, i64 %cmp, i64 %val) {
 define void @test112(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test112:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB112_1:
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB112_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
+; PPC64LE-NEXT:    cmpld 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB112_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB112_1
-; PPC64LE-NEXT:  .LBB112_3:
+; PPC64LE-NEXT:  .LBB112_3: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") acquire acquire
@@ -1714,16 +2258,22 @@ define void @test112(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test113(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test113:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB113_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
+; PPC64LE-NEXT:    cmpld 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB113_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB113_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB113_2
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") release monotonic
   ret void
@@ -1731,16 +2281,23 @@ define void @test113(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test114(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test114:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB114_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB114_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB114_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB114_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB114_1
-; PPC64LE-NEXT:  .LBB114_3:
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB114_2
+; PPC64LE-NEXT:  .LBB114_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") release acquire
@@ -1749,16 +2306,25 @@ define void @test114(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test115(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test115:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB115_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB115_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB115_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB115_1
-; PPC64LE-NEXT:  .LBB115_3:
+; PPC64LE-NEXT:    beq 0, .LBB115_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB115_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB115_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") acq_rel monotonic
@@ -1767,16 +2333,23 @@ define void @test115(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test116(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test116:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB116_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB116_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB116_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    lwsync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB116_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB116_1
-; PPC64LE-NEXT:  .LBB116_3:
+; PPC64LE-NEXT:    beq 0, .LBB116_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB116_2
+; PPC64LE-NEXT:  .LBB116_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") acq_rel acquire
@@ -1785,16 +2358,25 @@ define void @test116(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test117(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test117:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB117_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB117_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bnelr 0
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB117_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB117_1
-; PPC64LE-NEXT:  .LBB117_3:
+; PPC64LE-NEXT:    beq 0, .LBB117_5
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB117_2
+; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
+; PPC64LE-NEXT:    blr
+; PPC64LE-NEXT:  .LBB117_5: # %cmpxchg.success
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") seq_cst monotonic
@@ -1803,16 +2385,23 @@ define void @test117(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test118(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test118:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB118_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB118_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB118_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB118_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB118_1
-; PPC64LE-NEXT:  .LBB118_3:
+; PPC64LE-NEXT:    beq 0, .LBB118_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB118_2
+; PPC64LE-NEXT:  .LBB118_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") seq_cst acquire
@@ -1821,16 +2410,23 @@ define void @test118(ptr %ptr, i64 %cmp, i64 %val) {
 
 define void @test119(ptr %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test119:
-; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB119_1:
+; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
-; PPC64LE-NEXT:    cmpd 6, 4
-; PPC64LE-NEXT:    bne 0, .LBB119_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    bne 0, .LBB119_4
+; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    sync
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB119_2: # %cmpxchg.trystore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
-; PPC64LE-NEXT:    bne 0, .LBB119_1
-; PPC64LE-NEXT:  .LBB119_3:
+; PPC64LE-NEXT:    beq 0, .LBB119_4
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; PPC64LE-NEXT:    #
+; PPC64LE-NEXT:    ldarx 6, 0, 3
+; PPC64LE-NEXT:    cmpld 6, 4
+; PPC64LE-NEXT:    beq 0, .LBB119_2
+; PPC64LE-NEXT:  .LBB119_4: # %cmpxchg.nostore
 ; PPC64LE-NEXT:    lwsync
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val syncscope("singlethread") seq_cst seq_cst
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics.ll
index 24e71c87414e..40786057ead5 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/atomics.ll
@@ -139,59 +139,67 @@ define void @store_i64_seq_cst(ptr %mem) {
 define i8 @cas_strong_i8_sc_sc(ptr %mem) {
 ; PPC32-LABEL: cas_strong_i8_sc_sc:
 ; PPC32:       # %bb.0:
-; PPC32-NEXT:    rlwinm r8, r3, 3, 27, 28
-; PPC32-NEXT:    li r5, 1
-; PPC32-NEXT:    li r6, 0
-; PPC32-NEXT:    li r7, 255
-; PPC32-NEXT:    rlwinm r4, r3, 0, 0, 29
-; PPC32-NEXT:    xori r3, r8, 24
-; PPC32-NEXT:    slw r8, r5, r3
-; PPC32-NEXT:    slw r9, r6, r3
-; PPC32-NEXT:    slw r5, r7, r3
-; PPC32-NEXT:    and r6, r8, r5
-; PPC32-NEXT:    and r7, r9, r5
+; PPC32-NEXT:    rlwinm r5, r3, 0, 0, 29
+; PPC32-NEXT:    lwarx r4, 0, r5
+; PPC32-NEXT:    not     r3, r3
+; PPC32-NEXT:    rlwinm r3, r3, 3, 27, 28
+; PPC32-NEXT:    srw r6, r4, r3
+; PPC32-NEXT:    andi. r6, r6, 255
+; PPC32-NEXT:    bne     cr0, .LBB8_4
+; PPC32-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
+; PPC32-NEXT:    li r6, 255
+; PPC32-NEXT:    li r7, 1
+; PPC32-NEXT:    slw r6, r6, r3
+; PPC32-NEXT:    not     r6, r6
+; PPC32-NEXT:    slw r7, r7, r3
 ; PPC32-NEXT:    sync
-; PPC32-NEXT:  .LBB8_1:
-; PPC32-NEXT:    lwarx r9, 0, r4
-; PPC32-NEXT:    and r8, r9, r5
-; PPC32-NEXT:    cmpw r8, r7
-; PPC32-NEXT:    bne cr0, .LBB8_3
-; PPC32-NEXT:  # %bb.2:
-; PPC32-NEXT:    andc r9, r9, r5
-; PPC32-NEXT:    or r9, r9, r6
-; PPC32-NEXT:    stwcx. r9, 0, r4
-; PPC32-NEXT:    bne cr0, .LBB8_1
-; PPC32-NEXT:  .LBB8_3:
-; PPC32-NEXT:    srw r3, r8, r3
+; PPC32-NEXT:  .LBB8_2:                                # %cmpxchg.trystore
+; PPC32-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; PPC32-NEXT:    and r8, r4, r6
+; PPC32-NEXT:    or r8, r8, r7
+; PPC32-NEXT:    stwcx. r8, 0, r5
+; PPC32-NEXT:    beq     cr0, .LBB8_4
+; PPC32-NEXT:  # %bb.3:                                # %cmpxchg.releasedload
+; PPC32-NEXT:                                          #   in Loop: Header=BB8_2 Depth=1
+; PPC32-NEXT:    lwarx r4, 0, r5
+; PPC32-NEXT:    srw r8, r4, r3
+; PPC32-NEXT:    andi. r8, r8, 255
+; PPC32-NEXT:    beq     cr0, .LBB8_2
+; PPC32-NEXT:  .LBB8_4:                                # %cmpxchg.nostore
+; PPC32-NEXT:    srw r3, r4, r3
 ; PPC32-NEXT:    lwsync
 ; PPC32-NEXT:    blr
 ;
 ; PPC64-LABEL: cas_strong_i8_sc_sc:
 ; PPC64:       # %bb.0:
-; PPC64-NEXT:    rlwinm r8, r3, 3, 27, 28
-; PPC64-NEXT:    li r5, 1
-; PPC64-NEXT:    li r6, 0
-; PPC64-NEXT:    li r7, 255
-; PPC64-NEXT:    rldicr r4, r3, 0, 61
-; PPC64-NEXT:    xori r3, r8, 24
-; PPC64-NEXT:    slw r8, r5, r3
-; PPC64-NEXT:    slw r9, r6, r3
-; PPC64-NEXT:    slw r5, r7, r3
-; PPC64-NEXT:    and r6, r8, r5
-; PPC64-NEXT:    and r7, r9, r5
+; PPC64-NEXT:    rldicr r5, r3, 0, 61
+; PPC64-NEXT:    not     r3, r3
+; PPC64-NEXT:    lwarx r4, 0, r5
+; PPC64-NEXT:    rlwinm r3, r3, 3, 27, 28
+; PPC64-NEXT:    srw r6, r4, r3
+; PPC64-NEXT:    andi. r6, r6, 255
+; PPC64-NEXT:    bne     cr0, .LBB8_4
+; PPC64-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
+; PPC64-NEXT:    li r6, 255
+; PPC64-NEXT:    li r7, 1
+; PPC64-NEXT:    slw r6, r6, r3
+; PPC64-NEXT:    not     r6, r6
+; PPC64-NEXT:    slw r7, r7, r3
 ; PPC64-NEXT:    sync
-; PPC64-NEXT:  .LBB8_1:
-; PPC64-NEXT:    lwarx r9, 0, r4
-; PPC64-NEXT:    and r8, r9, r5
-; PPC64-NEXT:    cmpw r8, r7
-; PPC64-NEXT:    bne cr0, .LBB8_3
-; PPC64-NEXT:  # %bb.2:
-; PPC64-NEXT:    andc r9, r9, r5
-; PPC64-NEXT:    or r9, r9, r6
-; PPC64-NEXT:    stwcx. r9, 0, r4
-; PPC64-NEXT:    bne cr0, .LBB8_1
-; PPC64-NEXT:  .LBB8_3:
-; PPC64-NEXT:    srw r3, r8, r3
+; PPC64-NEXT:  .LBB8_2:                                # %cmpxchg.trystore
+; PPC64-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; PPC64-NEXT:    and r8, r4, r6
+; PPC64-NEXT:    or r8, r8, r7
+; PPC64-NEXT:    stwcx. r8, 0, r5
+; PPC64-NEXT:    beq     cr0, .LBB8_4
+; PPC64-NEXT:  # %bb.3:                                # %cmpxchg.releasedload
+; PPC64-NEXT:                                          #   in Loop: Header=BB8_2 Depth=1
+; PPC64-NEXT:    lwarx r4, 0, r5
+; PPC64-NEXT:    srw r8, r4, r3
+; PPC64-NEXT:    andi. r8, r8, 255
+; PPC64-NEXT:    beq     cr0, .LBB8_2
+; PPC64-NEXT:  .LBB8_4:                                # %cmpxchg.nostore
+; PPC64-NEXT:    srw r3, r4, r3
 ; PPC64-NEXT:    lwsync
 ; PPC64-NEXT:    blr
   %val = cmpxchg ptr %mem, i8 0, i8 1 seq_cst seq_cst
@@ -201,57 +209,53 @@ define i8 @cas_strong_i8_sc_sc(ptr %mem) {
 define i16 @cas_weak_i16_acquire_acquire(ptr %mem) {
 ; PPC32-LABEL: cas_weak_i16_acquire_acquire:
 ; PPC32:       # %bb.0:
-; PPC32-NEXT:    li r6, 0
-; PPC32-NEXT:    rlwinm r4, r3, 3, 27, 27
-; PPC32-NEXT:    li r5, 1
-; PPC32-NEXT:    ori r7, r6, 65535
-; PPC32-NEXT:    xori r4, r4, 16
-; PPC32-NEXT:    slw r8, r5, r4
-; PPC32-NEXT:    slw r9, r6, r4
-; PPC32-NEXT:    slw r5, r7, r4
-; PPC32-NEXT:    rlwinm r3, r3, 0, 0, 29
-; PPC32-NEXT:    and r6, r8, r5
-; PPC32-NEXT:    and r7, r9, r5
-; PPC32-NEXT:  .LBB9_1:
-; PPC32-NEXT:    lwarx r9, 0, r3
-; PPC32-NEXT:    and r8, r9, r5
-; PPC32-NEXT:    cmpw r8, r7
-; PPC32-NEXT:    bne cr0, .LBB9_3
-; PPC32-NEXT:  # %bb.2:
-; PPC32-NEXT:    andc r9, r9, r5
-; PPC32-NEXT:    or r9, r9, r6
-; PPC32-NEXT:    stwcx. r9, 0, r3
-; PPC32-NEXT:    bne cr0, .LBB9_1
-; PPC32-NEXT:  .LBB9_3:
-; PPC32-NEXT:    srw r3, r8, r4
+; PPC32-NEXT:    rlwinm r4, r3, 0, 0, 29
+; PPC32-NEXT:    lwarx r5, 0, r4
+; PPC32-NEXT:    clrlwi  r3, r3, 30
+; PPC32-NEXT:    xori r3, r3, 2
+; PPC32-NEXT:    slwi r6, r3, 3
+; PPC32-NEXT:    srw r3, r5, r6
+; PPC32-NEXT:    andi. r7, r3, 65535
+; PPC32-NEXT:    beq     cr0, .LBB9_2
+; PPC32-NEXT:  # %bb.1:                                # %cmpxchg.failure
+; PPC32-NEXT:    lwsync
+; PPC32-NEXT:    blr
+; PPC32-NEXT:  .LBB9_2:                                # %cmpxchg.fencedstore
+; PPC32-NEXT:    lis r7, 0
+; PPC32-NEXT:    ori r7, r7, 65535
+; PPC32-NEXT:    slw r7, r7, r6
+; PPC32-NEXT:    li r8, 1
+; PPC32-NEXT:    not     r7, r7
+; PPC32-NEXT:    slw r6, r8, r6
+; PPC32-NEXT:    and r5, r5, r7
+; PPC32-NEXT:    or r5, r5, r6
+; PPC32-NEXT:    stwcx. r5, 0, r4
 ; PPC32-NEXT:    lwsync
 ; PPC32-NEXT:    blr
 ;
 ; PPC64-LABEL: cas_weak_i16_acquire_acquire:
 ; PPC64:       # %bb.0:
-; PPC64-NEXT:    li r6, 0
-; PPC64-NEXT:    rlwinm r4, r3, 3, 27, 27
-; PPC64-NEXT:    li r5, 1
-; PPC64-NEXT:    ori r7, r6, 65535
-; PPC64-NEXT:    xori r4, r4, 16
-; PPC64-NEXT:    slw r8, r5, r4
-; PPC64-NEXT:    slw r9, r6, r4
-; PPC64-NEXT:    slw r5, r7, r4
-; PPC64-NEXT:    rldicr r3, r3, 0, 61
-; PPC64-NEXT:    and r6, r8, r5
-; PPC64-NEXT:    and r7, r9, r5
-; PPC64-NEXT:  .LBB9_1:
-; PPC64-NEXT:    lwarx r9, 0, r3
-; PPC64-NEXT:    and r8, r9, r5
-; PPC64-NEXT:    cmpw r8, r7
-; PPC64-NEXT:    bne cr0, .LBB9_3
-; PPC64-NEXT:  # %bb.2:
-; PPC64-NEXT:    andc r9, r9, r5
-; PPC64-NEXT:    or r9, r9, r6
-; PPC64-NEXT:    stwcx. r9, 0, r3
-; PPC64-NEXT:    bne cr0, .LBB9_1
-; PPC64-NEXT:  .LBB9_3:
-; PPC64-NEXT:    srw r3, r8, r4
+; PPC64-NEXT:   rldicr r4, r3, 0, 61
+; PPC64-NEXT:    clrlwi  r3, r3, 30
+; PPC64-NEXT:    lwarx r5, 0, r4
+; PPC64-NEXT:    xori r3, r3, 2
+; PPC64-NEXT:    slwi r6, r3, 3
+; PPC64-NEXT:    srw r3, r5, r6
+; PPC64-NEXT:    andi. r7, r3, 65535
+; PPC64-NEXT:    beq     cr0, .LBB9_2
+; PPC64-NEXT:  # %bb.1:                                # %cmpxchg.failure
+; PPC64-NEXT:    lwsync
+; PPC64-NEXT:    blr
+; PPC64-NEXT:  .LBB9_2:                                # %cmpxchg.fencedstore
+; PPC64-NEXT:    lis r7, 0
+; PPC64-NEXT:    ori r7, r7, 65535
+; PPC64-NEXT:    slw r7, r7, r6
+; PPC64-NEXT:    li r8, 1
+; PPC64-NEXT:    not     r7, r7
+; PPC64-NEXT:    slw r6, r8, r6
+; PPC64-NEXT:    and r5, r5, r7
+; PPC64-NEXT:    or r5, r5, r6
+; PPC64-NEXT:    stwcx. r5, 0, r4
 ; PPC64-NEXT:    lwsync
 ; PPC64-NEXT:    blr
   %val = cmpxchg weak ptr %mem, i16 0, i16 1 acquire acquire
@@ -261,17 +265,23 @@ define i16 @cas_weak_i16_acquire_acquire(ptr %mem) {
 define i32 @cas_strong_i32_acqrel_acquire(ptr %mem) {
 ; CHECK-LABEL: cas_strong_i32_acqrel_acquire:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    mr      r4, r3
+; CHECK-NEXT:    lwarx r3, 0, r3
+; CHECK-NEXT:    cmplwi  r3, 0
+; CHECK-NEXT:    bne     cr0, .LBB10_4
+; CHECK-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
 ; CHECK-NEXT:    li r5, 1
 ; CHECK-NEXT:    lwsync
-; CHECK-NEXT:  .LBB10_1:
-; CHECK-NEXT:    lwarx r4, 0, r3
-; CHECK-NEXT:    cmpwi r4, 0
-; CHECK-NEXT:    bne cr0, .LBB10_3
-; CHECK-NEXT:  # %bb.2:
-; CHECK-NEXT:    stwcx. r5, 0, r3
-; CHECK-NEXT:    bne cr0, .LBB10_1
-; CHECK-NEXT:  .LBB10_3:
-; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:  .LBB10_2:                               # %cmpxchg.trystore
+; CHECK-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    stwcx. r5, 0, r4
+; CHECK-NEXT:    beq     cr0, .LBB10_4
+; CHECK-NEXT:  # %bb.3:                                # %cmpxchg.releasedload
+; CHECK-NEXT:                                          #   in Loop: Header=BB10_2 Depth=1
+; CHECK-NEXT:    lwarx r3, 0, r4
+; CHECK-NEXT:    cmplwi  r3, 0
+; CHECK-NEXT:    beq     cr0, .LBB10_2
+; CHECK-NEXT:  .LBB10_4:                               # %cmpxchg.nostore
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    blr
   %val = cmpxchg ptr %mem, i32 0, i32 1 acq_rel acquire
@@ -304,17 +314,14 @@ define i64 @cas_weak_i64_release_monotonic(ptr %mem) {
 ;
 ; PPC64-LABEL: cas_weak_i64_release_monotonic:
 ; PPC64:       # %bb.0:
+; PPC64-NEXT:    mr      r4, r3
+; PPC64-NEXT:    ldarx r3, 0, r3
+; PPC64-NEXT:    cmpldi  r3, 0
+; PPC64-NEXT:    bnelr   cr0
+; PPC64-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
 ; PPC64-NEXT:    li r5, 1
 ; PPC64-NEXT:    lwsync
-; PPC64-NEXT:  .LBB11_1:
-; PPC64-NEXT:    ldarx r4, 0, r3
-; PPC64-NEXT:    cmpdi r4, 0
-; PPC64-NEXT:    bne cr0, .LBB11_3
-; PPC64-NEXT:  # %bb.2:
-; PPC64-NEXT:    stdcx. r5, 0, r3
-; PPC64-NEXT:    bne cr0, .LBB11_1
-; PPC64-NEXT:  .LBB11_3:
-; PPC64-NEXT:    mr r3, r4
+; PPC64-NEXT:    stdcx. r5, 0, r4
 ; PPC64-NEXT:    blr
   %val = cmpxchg weak ptr %mem, i64 0, i64 1 release monotonic
   %loaded = extractvalue { i64, i1} %val, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/build-vector-tests.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
index 91431ed15f6a..9dd0fbe4474b 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
@@ -3713,30 +3713,26 @@ entry:
 define <2 x i64> @spltConst1ll() {
 ; P9BE-LABEL: spltConst1ll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
-; P9BE-NEXT:    lxv v2, 0(r3)
+; P9BE-NEXT:    vspltisw v2, 1
+; P9BE-NEXT:    vupklsw v2, v2
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst1ll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
-; P9LE-NEXT:    lxv v2, 0(r3)
+; P9LE-NEXT:    vspltisw v2, 1
+; P9LE-NEXT:    vupklsw v2, v2
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst1ll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
-; P8BE-NEXT:    lxvd2x v2, 0, r3
+; P8BE-NEXT:    vspltisw v2, 1
+; P8BE-NEXT:    vupklsw v2, v2
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst1ll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI65_0@toc@l
-; P8LE-NEXT:    lxvd2x v2, 0, r3
+; P8LE-NEXT:    vspltisw v2, 1
+; P8LE-NEXT:    vupklsw v2, v2
 ; P8LE-NEXT:    blr
 entry:
   ret <2 x i64> <i64 1, i64 1>
@@ -4173,30 +4169,26 @@ entry:
 define <2 x i64> @spltCnstConvftoll() {
 ; P9BE-LABEL: spltCnstConvftoll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
-; P9BE-NEXT:    lxv v2, 0(r3)
+; P9BE-NEXT:    vspltisw v2, 4
+; P9BE-NEXT:    vupklsw v2, v2
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvftoll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
-; P9LE-NEXT:    lxv v2, 0(r3)
+; P9LE-NEXT:    vspltisw v2, 4
+; P9LE-NEXT:    vupklsw v2, v2
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvftoll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
-; P8BE-NEXT:    lxvd2x v2, 0, r3
+; P8BE-NEXT:    vspltisw v2, 4
+; P8BE-NEXT:    vupklsw v2, v2
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvftoll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI78_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI78_0@toc@l
-; P8LE-NEXT:    lxvd2x v2, 0, r3
+; P8LE-NEXT:    vspltisw v2, 4
+; P8LE-NEXT:    vupklsw v2, v2
 ; P8LE-NEXT:    blr
 entry:
   ret <2 x i64> <i64 4, i64 4>
@@ -4526,30 +4518,26 @@ entry:
 define <2 x i64> @spltCnstConvdtoll() {
 ; P9BE-LABEL: spltCnstConvdtoll:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
-; P9BE-NEXT:    lxv v2, 0(r3)
+; P9BE-NEXT:    vspltisw v2, 4
+; P9BE-NEXT:    vupklsw v2, v2
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvdtoll:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
-; P9LE-NEXT:    lxv v2, 0(r3)
+; P9LE-NEXT:    vspltisw v2, 4
+; P9LE-NEXT:    vupklsw v2, v2
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvdtoll:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
-; P8BE-NEXT:    lxvd2x v2, 0, r3
+; P8BE-NEXT:    vspltisw v2, 4
+; P8BE-NEXT:    vupklsw v2, v2
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvdtoll:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI87_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI87_0@toc@l
-; P8LE-NEXT:    lxvd2x v2, 0, r3
+; P8LE-NEXT:    vspltisw v2, 4
+; P8LE-NEXT:    vupklsw v2, v2
 ; P8LE-NEXT:    blr
 entry:
   ret <2 x i64> <i64 4, i64 4>
@@ -4879,30 +4867,26 @@ entry:
 define <2 x i64> @spltConst1ull() {
 ; P9BE-LABEL: spltConst1ull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
-; P9BE-NEXT:    lxv v2, 0(r3)
+; P9BE-NEXT:    vspltisw v2, 1
+; P9BE-NEXT:    vupklsw v2, v2
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltConst1ull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
-; P9LE-NEXT:    lxv v2, 0(r3)
+; P9LE-NEXT:    vspltisw v2, 1
+; P9LE-NEXT:    vupklsw v2, v2
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltConst1ull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
-; P8BE-NEXT:    lxvd2x v2, 0, r3
+; P8BE-NEXT:    vspltisw v2, 1
+; P8BE-NEXT:    vupklsw v2, v2
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltConst1ull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI97_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI97_0@toc@l
-; P8LE-NEXT:    lxvd2x v2, 0, r3
+; P8LE-NEXT:    vspltisw v2, 1
+; P8LE-NEXT:    vupklsw v2, v2
 ; P8LE-NEXT:    blr
 entry:
   ret <2 x i64> <i64 1, i64 1>
@@ -5339,30 +5323,26 @@ entry:
 define <2 x i64> @spltCnstConvftoull() {
 ; P9BE-LABEL: spltCnstConvftoull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
-; P9BE-NEXT:    lxv v2, 0(r3)
+; P9BE-NEXT:    vspltisw v2, 4
+; P9BE-NEXT:    vupklsw v2, v2
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvftoull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
-; P9LE-NEXT:    lxv v2, 0(r3)
+; P9LE-NEXT:    vspltisw v2, 4
+; P9LE-NEXT:    vupklsw v2, v2
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvftoull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
-; P8BE-NEXT:    lxvd2x v2, 0, r3
+; P8BE-NEXT:    vspltisw v2, 4
+; P8BE-NEXT:    vupklsw v2, v2
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvftoull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI110_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI110_0@toc@l
-; P8LE-NEXT:    lxvd2x v2, 0, r3
+; P8LE-NEXT:    vspltisw v2, 4
+; P8LE-NEXT:    vupklsw v2, v2
 ; P8LE-NEXT:    blr
 entry:
   ret <2 x i64> <i64 4, i64 4>
@@ -5692,30 +5672,26 @@ entry:
 define <2 x i64> @spltCnstConvdtoull() {
 ; P9BE-LABEL: spltCnstConvdtoull:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
-; P9BE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
-; P9BE-NEXT:    lxv v2, 0(r3)
+; P9BE-NEXT:    vspltisw v2, 4
+; P9BE-NEXT:    vupklsw v2, v2
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltCnstConvdtoull:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
-; P9LE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
-; P9LE-NEXT:    lxv v2, 0(r3)
+; P9LE-NEXT:    vspltisw v2, 4
+; P9LE-NEXT:    vupklsw v2, v2
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltCnstConvdtoull:
 ; P8BE:       # %bb.0: # %entry
-; P8BE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
-; P8BE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
-; P8BE-NEXT:    lxvd2x v2, 0, r3
+; P8BE-NEXT:    vspltisw v2, 4
+; P8BE-NEXT:    vupklsw v2, v2
 ; P8BE-NEXT:    blr
 ;
 ; P8LE-LABEL: spltCnstConvdtoull:
 ; P8LE:       # %bb.0: # %entry
-; P8LE-NEXT:    addis r3, r2, .LCPI119_0@toc@ha
-; P8LE-NEXT:    addi r3, r3, .LCPI119_0@toc@l
-; P8LE-NEXT:    lxvd2x v2, 0, r3
+; P8LE-NEXT:    vspltisw v2, 4
+; P8LE-NEXT:    vupklsw v2, v2
 ; P8LE-NEXT:    blr
 entry:
   ret <2 x i64> <i64 4, i64 4>
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/callbr-asm-outputs-indirect-isel.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/callbr-asm-outputs-indirect-isel.ll
index 987b8da75ccf..d52a9dbc577a 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/callbr-asm-outputs-indirect-isel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/callbr-asm-outputs-indirect-isel.ll
@@ -22,7 +22,7 @@ define void @strncpy_from_kernel_nofault_count() {
   ; CHECK-NEXT: bb.2.Efault:
   ; CHECK-NEXT:   BLR8 implicit $lr8, implicit $rm
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.3.Efault.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.3.Efault.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   STB %1, 0, $zero8 :: (store (s8) into `ptr null`)
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/data-align.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/data-align.ll
index bfedec139369..42dee13d152a 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/data-align.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/data-align.ll
@@ -2,23 +2,23 @@
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux | FileCheck %s
 
-; CHECK:      .set .Li8,
+; CHECK:      .Li8 =
 ; CHECK-NEXT:  .size	.Li8, 1
 @i8 = private constant i8 42
 
-; CHECK:      .set .Li16,
+; CHECK:      .Li16 =
 ; CHECK-NEXT: .size	.Li16, 2
 @i16 = private constant i16 42
 
-; CHECK:      .set .Li32,
+; CHECK:      .Li32 =
 ; CHECK-NEXT: .size	.Li32, 4
 @i32 = private constant i32 42
 
-; CHECK:      .set .Li64,
+; CHECK:      .Li64 =
 ; CHECK-NEXT: .size	.Li64, 8
 @i64 = private constant i64 42
 
-; CHECK:        .set .Li128,
+; CHECK:        .Li128 =
 ; CHECK-NEXT:	.size	.Li128, 16
 @i128 = private constant i128 42
 
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/dmr-spill.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/dmr-spill.ll
new file mode 100644
index 000000000000..c1b01cd2d3fd
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/dmr-spill.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mcpu=future < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-aix \
+; RUN:   -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mcpu=future < %s | FileCheck %s --check-prefix=AIX
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-unknown-aix \
+; RUN:   -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mcpu=future < %s | FileCheck %s --check-prefix=AIX32
+
+declare <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2pp(<1024 x i1>, <256 x i1>, <16 x i8>)
+declare void @dummy_func()
+
+define void @spillDMRreg(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) nounwind {
+; CHECK-LABEL: spillDMRreg:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -176(r1)
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    mr r30, r6
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    lxv v2, 16(r4)
+; CHECK-NEXT:    lxv v3, 0(r4)
+; CHECK-NEXT:    lxv vs0, 0(r5)
+; CHECK-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; CHECK-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
+; CHECK-NEXT:    dmxxextfdmr512 vsp38, vsp32, wacc0, 0
+; CHECK-NEXT:    stxvp vsp38, 128(r1)
+; CHECK-NEXT:    stxvp vsp32, 96(r1)
+; CHECK-NEXT:    stxvp vsp36, 64(r1)
+; CHECK-NEXT:    stxvp vsp34, 32(r1)
+; CHECK-NEXT:    bl dummy_func@notoc
+; CHECK-NEXT:    lxvp vsp34, 128(r1)
+; CHECK-NEXT:    lxvp vsp36, 96(r1)
+; CHECK-NEXT:    lxvp vsp32, 64(r1)
+; CHECK-NEXT:    lxvp vsp38, 32(r1)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp32, vsp38, 1
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r30)
+; CHECK-NEXT:    stxvp vsp36, 64(r30)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r30)
+; CHECK-NEXT:    stxvp vsp36, 0(r30)
+; CHECK-NEXT:    addi r1, r1, 176
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; AIX-LABEL: spillDMRreg:
+; AIX:       # %bb.0:
+; AIX-NEXT:    mflr r0
+; AIX-NEXT:    std r0, 16(r1)
+; AIX-NEXT:    stdu r1, -256(r1)
+; AIX-NEXT:    std r31, 248(r1) # 8-byte Folded Spill
+; AIX-NEXT:    lxvp vsp34, 96(r3)
+; AIX-NEXT:    lxvp vsp36, 64(r3)
+; AIX-NEXT:    mr r31, r6
+; AIX-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; AIX-NEXT:    lxvp vsp34, 32(r3)
+; AIX-NEXT:    lxvp vsp36, 0(r3)
+; AIX-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; AIX-NEXT:    lxv v2, 0(r4)
+; AIX-NEXT:    lxv v3, 16(r4)
+; AIX-NEXT:    lxv vs0, 0(r5)
+; AIX-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; AIX-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
+; AIX-NEXT:    dmxxextfdmr512 vsp38, vsp32, wacc0, 0
+; AIX-NEXT:    stxvp vsp38, 112(r1)
+; AIX-NEXT:    stxvp vsp32, 144(r1)
+; AIX-NEXT:    stxvp vsp36, 176(r1)
+; AIX-NEXT:    stxvp vsp34, 208(r1)
+; AIX-NEXT:    bl .dummy_func[PR]
+; AIX-NEXT:    nop
+; AIX-NEXT:    lxvp vsp34, 112(r1)
+; AIX-NEXT:    lxvp vsp36, 144(r1)
+; AIX-NEXT:    lxvp vsp32, 176(r1)
+; AIX-NEXT:    lxvp vsp38, 208(r1)
+; AIX-NEXT:    dmxxinstdmr512 wacc_hi0, vsp32, vsp38, 1
+; AIX-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; AIX-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; AIX-NEXT:    stxvp vsp36, 96(r31)
+; AIX-NEXT:    stxvp vsp34, 64(r31)
+; AIX-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; AIX-NEXT:    stxvp vsp36, 32(r31)
+; AIX-NEXT:    stxvp vsp34, 0(r31)
+; AIX-NEXT:    ld r31, 248(r1) # 8-byte Folded Reload
+; AIX-NEXT:    addi r1, r1, 256
+; AIX-NEXT:    ld r0, 16(r1)
+; AIX-NEXT:    mtlr r0
+; AIX-NEXT:    blr
+;
+; AIX32-LABEL: spillDMRreg:
+; AIX32:       # %bb.0:
+; AIX32-NEXT:    mflr r0
+; AIX32-NEXT:    stw r0, 8(r1)
+; AIX32-NEXT:    stwu r1, -208(r1)
+; AIX32-NEXT:    stw r31, 204(r1) # 4-byte Folded Spill
+; AIX32-NEXT:    lxvp vsp34, 96(r3)
+; AIX32-NEXT:    lxvp vsp36, 64(r3)
+; AIX32-NEXT:    mr r31, r6
+; AIX32-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; AIX32-NEXT:    lxvp vsp34, 32(r3)
+; AIX32-NEXT:    lxvp vsp36, 0(r3)
+; AIX32-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; AIX32-NEXT:    lxv v2, 0(r4)
+; AIX32-NEXT:    lxv v3, 16(r4)
+; AIX32-NEXT:    lxv vs0, 0(r5)
+; AIX32-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; AIX32-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
+; AIX32-NEXT:    dmxxextfdmr512 vsp38, vsp32, wacc0, 0
+; AIX32-NEXT:    stxvp vsp38, 64(r1)
+; AIX32-NEXT:    stxvp vsp32, 96(r1)
+; AIX32-NEXT:    stxvp vsp36, 128(r1)
+; AIX32-NEXT:    stxvp vsp34, 160(r1)
+; AIX32-NEXT:    bl .dummy_func[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lxvp vsp34, 64(r1)
+; AIX32-NEXT:    lxvp vsp36, 96(r1)
+; AIX32-NEXT:    lxvp vsp32, 128(r1)
+; AIX32-NEXT:    lxvp vsp38, 160(r1)
+; AIX32-NEXT:    dmxxinstdmr512 wacc_hi0, vsp32, vsp38, 1
+; AIX32-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; AIX32-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; AIX32-NEXT:    stxvp vsp36, 96(r31)
+; AIX32-NEXT:    stxvp vsp34, 64(r31)
+; AIX32-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; AIX32-NEXT:    stxvp vsp36, 32(r31)
+; AIX32-NEXT:    stxvp vsp34, 0(r31)
+; AIX32-NEXT:    lwz r31, 204(r1) # 4-byte Folded Reload
+; AIX32-NEXT:    addi r1, r1, 208
+; AIX32-NEXT:    lwz r0, 8(r1)
+; AIX32-NEXT:    mtlr r0
+; AIX32-NEXT:    blr
+  %v.dmr = load <1024 x i1>, ptr %vop, align 64
+  %v1 = load <256 x i1>, ptr %vpp, align 32
+  %v2 = load <16 x i8>, ptr %vcp, align 32
+  %call = tail call <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2pp(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2)
+  tail call void @dummy_func()
+  %call2 = tail call <1024 x i1> @llvm.ppc.mma.dmxvbf16gerx2pp(<1024 x i1> %v.dmr, <256 x i1> %v1, <16 x i8> %v2)
+  store <1024 x i1> %call, ptr %resp, align 64
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/git_revision.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/git_revision.ll
index 86dcc5048425..c4003a776375 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/git_revision.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/git_revision.ll
@@ -1,6 +1,7 @@
 ; Check that the git revision is contained in the assembly/object files
 
 ; REQUIRES: vc-rev-enabled 
+; REQUIRES: vanilla-revision
 
 ; RUN: llc < %s | FileCheck %s -DREVISION=git-revision
 ; RUN: llc -filetype=obj < %s | FileCheck %s -DREVISION=git-revision
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/loop-comment.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/loop-comment.ll
index 14f6791fc779..1fa9dda51ef9 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/loop-comment.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/loop-comment.ll
@@ -4,12 +4,17 @@
 define void @test(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test:
 ; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:  .LBB0_1:
+; PPC64LE-NEXT:    .p2align        5
+; PPC64LE-NEXT:  .LBB0_1:                                # %cmpxchg.start
+; PPC64LE-NEXT:                                          # =>This Inner Loop Header: Depth=1
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    cmpw 6, 4
+; PPC64LE-NEXT:    clrlwi  6, 6, 24
+; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:                                          #   in Loop: Header=BB0_1 Depth=1
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB0_1
 ; PPC64LE-NEXT:  # %bb.3:
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/mmaplus-acc-spill.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/mmaplus-acc-spill.ll
index c2c8a42c402a..8dd17abb2634 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/mmaplus-acc-spill.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/mmaplus-acc-spill.ll
@@ -1,11 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; This test is a copy of mma-acc-spill.ll except that it uses mcpu=future.
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -disable-auto-paired-vec-st=false \
 ; RUN:   -mcpu=future -ppc-asm-full-reg-names \
 ; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
-; RUN:   -disable-auto-paired-vec-st=false \
 ; RUN:   -mcpu=future -ppc-asm-full-reg-names \
 ; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s --check-prefix=CHECK-BE
 
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
new file mode 100644
index 000000000000..5486dc02faf9
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Verify whether the generated assembly for the following function includes the mtvsrbmi instruction.
+; vector unsigned char v00FF()
+; {
+; vector unsigned char x = { 0xFF, 0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
+; return x;
+; }
+
+; RUN: llc < %s -ppc-asm-full-reg-names  -mtriple=powerpc-ibm-aix -mcpu=pwr10  -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefix=CHECK
+
+define dso_local noundef range(i8 -1, 1) <16 x i8> @_Z5v00FFv() {
+; CHECK:      L..CPI0_0:
+; CHECK-NEXT:   .byte   255                             # 0xff
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NEXT:   .byte   0                               # 0x0
+
+; CHECK-LABEL: _Z5v00FFv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lwz r3, L..C0(r2) # %const.0
+; CHECK-NEXT:    lxv vs34, 0(r3)
+; CHECK-NEXT:    blr
+entry:
+  ret <16 x i8> <i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/mul-const-vector.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/mul-const-vector.ll
index e3d231adf734..2d67de0d4477 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/mul-const-vector.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/mul-const-vector.ll
@@ -271,8 +271,7 @@ define <2 x i64> @test1_v2i64(<2 x i64> %a) {
         ret <2 x i64> %tmp.1
 }
 ; CHECK-LABEL: test1_v2i64:
-; CHECK-P8: lxvd2x v[[REG1:[0-9]+]], 0, r{{[0-9]+}}
-; CHECK-P9: lxv v[[REG2:[0-9]+]], 0(r{{[0-9]+}})
+; CHECK: vupklsw v[[REG1:[0-9]+]], v{{[0-9]+}}
 ; CHECK-NOT: vmul
 ; CHECK-NEXT: vsld v{{[0-9]+}}, v2, v[[REG2]]
 
@@ -282,8 +281,7 @@ define <2 x i64> @test2_v2i64(<2 x i64> %a) {
 }
 
 ; CHECK-LABEL: test2_v2i64:
-; CHECK-P8: lxvd2x v[[REG1:[0-9]+]], 0, r{{[0-9]+}}
-; CHECK-P9: lxv v[[REG2:[0-9]+]], 0(r{{[0-9]+}})
+; CHECK: vupklsw v[[REG1:[0-9]+]], v{{[0-9]+}}
 ; CHECK-NOT: vmul
 ; CHECK-NEXT: vsld v[[REG3:[0-9]+]], v2, v[[REG2]]
 ; CHECK-NEXT: vaddudm v{{[0-9]+}}, v2, v[[REG3]]
@@ -294,8 +292,7 @@ define <2 x i64> @test3_v2i64(<2 x i64> %a) {
 }
 
 ; CHECK-LABEL: test3_v2i64:
-; CHECK-P8: lxvd2x v[[REG1:[0-9]+]], 0, r{{[0-9]+}}
-; CHECK-P9: lxv v[[REG2:[0-9]+]], 0(r{{[0-9]+}})
+; CHECK: vupklsw v[[REG1:[0-9]+]], v{{[0-9]+}}
 ; CHECK-NOT: vmul
 ; CHECK-NEXT: vsld v[[REG3:[0-9]+]], v2, v[[REG2]]
 ; CHECK-NEXT: vsubudm v{{[0-9]+}}, v[[REG3]], v2
@@ -308,8 +305,7 @@ define <2 x i64> @test4_v2i64(<2 x i64> %a) {
 }
 
 ; CHECK-LABEL: test4_v2i64:
-; CHECK-P8: lxvd2x v[[REG1:[0-9]+]], 0, r{{[0-9]+}}
-; CHECK-P9: lxv v[[REG2:[0-9]+]], 0(r{{[0-9]+}})
+; CHECK: vupklsw v[[REG1:[0-9]+]], v{{[0-9]+}}
 ; CHECK-NOT: vmul
 ; CHECK-NEXT: vsld v[[REG3:[0-9]+]], v2, v[[REG2]]
 ; CHECK-P8-NEXT: xxlxor v[[REG4:[0-9]+]],
@@ -322,8 +318,7 @@ define <2 x i64> @test5_v2i64(<2 x i64> %a) {
 }
 
 ; CHECK-LABEL: test5_v2i64:
-; CHECK-P8: lxvd2x v[[REG1:[0-9]+]], 0, r{{[0-9]+}}
-; CHECK-P9: lxv v[[REG2:[0-9]+]], 0(r{{[0-9]+}})
+; CHECK: vupklsw v[[REG1:[0-9]+]], v{{[0-9]+}}
 ; CHECK-NOT: vmul
 ; CHECK-NEXT: vsld v[[REG3:[0-9]+]], v2, v[[REG2]]
 ; CHECK-NEXT: vaddudm v[[REG4:[0-9]+]], v2, v[[REG3]]
@@ -337,8 +332,7 @@ define <2 x i64> @test6_v2i64(<2 x i64> %a) {
 }
 
 ; CHECK-LABEL: test6_v2i64:
-; CHECK-P8: lxvd2x v[[REG1:[0-9]+]], 0, r{{[0-9]+}}
-; CHECK-P9: lxv v[[REG2:[0-9]+]], 0(r{{[0-9]+}})
+; CHECK: vupklsw v[[REG1:[0-9]+]], v{{[0-9]+}}
 ; CHECK-NOT: vmul
 ; CHECK-NEXT: vsld v[[REG3:[0-9]+]], v2, v[[REG2]]
 ; CHECK-NEXT: vsubudm v{{[0-9]+}}, v2, v[[REG3]]
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
index 842cb929541c..1ab74e6cb1ce 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
@@ -105,9 +105,8 @@ define dso_local <2 x double> @testDoubleToDoubleNaNFail() local_unnamed_addr {
 ;
 ; CHECK-NOPREFIX-LABEL: testDoubleToDoubleNaNFail:
 ; CHECK-NOPREFIX:       # %bb.0: # %entry
-; CHECK-NOPREFIX-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
-; CHECK-NOPREFIX-NEXT:    addi r3, r3, .LCPI2_0@toc@l
-; CHECK-NOPREFIX-NEXT:    lxv vs34, 0(r3)
+; CHECK-NOPREFIX-NEXT:    vspltisw v2, -16
+; CHECK-NOPREFIX-NEXT:    vupklsw v2, v2
 ; CHECK-NOPREFIX-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testDoubleToDoubleNaNFail:
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/ppc64-inlineasm-clobber.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/ppc64-inlineasm-clobber.ll
index 57bc882f6046..3316f1b0b87a 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/ppc64-inlineasm-clobber.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/ppc64-inlineasm-clobber.ll
@@ -86,7 +86,7 @@ define dso_local signext i32 @ClobberLR_BR(i32 signext %in) #0 {
 ; PPC64LE-NEXT:    ld r0, 16(r1)
 ; PPC64LE-NEXT:    mtlr r0
 ; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB3_2: # Block address taken
+; PPC64LE-NEXT:  .LBB3_2: # Inline asm indirect target
 ; PPC64LE-NEXT:    # %return_early
 ; PPC64LE-NEXT:    # Label of block must be emitted
 ; PPC64LE-NEXT:    li r3, 0
@@ -105,7 +105,7 @@ define dso_local signext i32 @ClobberLR_BR(i32 signext %in) #0 {
 ; PPC64BE-NEXT:    ld r0, 16(r1)
 ; PPC64BE-NEXT:    mtlr r0
 ; PPC64BE-NEXT:    blr
-; PPC64BE-NEXT:  .LBB3_2: # Block address taken
+; PPC64BE-NEXT:  .LBB3_2: # Inline asm indirect target
 ; PPC64BE-NEXT:    # %return_early
 ; PPC64BE-NEXT:    # Label of block must be emitted
 ; PPC64BE-NEXT:    li r3, 0
@@ -130,7 +130,7 @@ define dso_local signext i32 @ClobberR5_BR(i32 signext %in) #0 {
 ; PPC64LE-NEXT:    #NO_APP
 ; PPC64LE-NEXT:  # %bb.1: # %return
 ; PPC64LE-NEXT:    blr
-; PPC64LE-NEXT:  .LBB4_2: # Block address taken
+; PPC64LE-NEXT:  .LBB4_2: # Inline asm indirect target
 ; PPC64LE-NEXT:    # %return_early
 ; PPC64LE-NEXT:    # Label of block must be emitted
 ; PPC64LE-NEXT:    li r3, 0
@@ -143,7 +143,7 @@ define dso_local signext i32 @ClobberR5_BR(i32 signext %in) #0 {
 ; PPC64BE-NEXT:    #NO_APP
 ; PPC64BE-NEXT:  # %bb.1: # %return
 ; PPC64BE-NEXT:    blr
-; PPC64BE-NEXT:  .LBB4_2: # Block address taken
+; PPC64BE-NEXT:  .LBB4_2: # Inline asm indirect target
 ; PPC64BE-NEXT:    # %return_early
 ; PPC64BE-NEXT:    # Label of block must be emitted
 ; PPC64BE-NEXT:    li r3, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll
index 4435484ae0b9..6b29c780de60 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/pre-inc-disable.ll
@@ -22,10 +22,10 @@ define void @test64(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9LE-NEXT:    lfdx 0, 3, 4
 ; P9LE-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
 ; P9LE-NEXT:    xxlxor 2, 2, 2
-; P9LE-NEXT:    vspltisw 4, 8
+; P9LE-NEXT:    xxspltib 4, 16
 ; P9LE-NEXT:    lxsd 3, 4(5)
 ; P9LE-NEXT:    addi 3, 3, .LCPI0_0@toc@l
-; P9LE-NEXT:    vadduwm 4, 4, 4
+; P9LE-NEXT:    vextsb2w 4, 4
 ; P9LE-NEXT:    lxv 1, 0(3)
 ; P9LE-NEXT:    addis 3, 2, .LCPI0_1@toc@ha
 ; P9LE-NEXT:    addi 3, 3, .LCPI0_1@toc@l
@@ -45,10 +45,10 @@ define void @test64(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9BE-NEXT:    lxsdx 2, 3, 4
 ; P9BE-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
 ; P9BE-NEXT:    xxlxor 1, 1, 1
-; P9BE-NEXT:    vspltisw 4, 8
+; P9BE-NEXT:    xxspltib 4, 16
 ; P9BE-NEXT:    lxsd 3, 4(5)
 ; P9BE-NEXT:    addi 3, 3, .LCPI0_0@toc@l
-; P9BE-NEXT:    vadduwm 4, 4, 4
+; P9BE-NEXT:    vextsb2w 4, 4
 ; P9BE-NEXT:    lxv 0, 0(3)
 ; P9BE-NEXT:    addis 3, 2, .LCPI0_1@toc@ha
 ; P9BE-NEXT:    addi 3, 3, .LCPI0_1@toc@l
@@ -68,11 +68,11 @@ define void @test64(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9BE-AIX-NEXT:    lxsdx 2, 3, 4
 ; P9BE-AIX-NEXT:    ld 3, L..C0(2) # %const.0
 ; P9BE-AIX-NEXT:    xxlxor 1, 1, 1
-; P9BE-AIX-NEXT:    vspltisw 4, 8
+; P9BE-AIX-NEXT:    xxspltib 4, 16
 ; P9BE-AIX-NEXT:    lxsd 3, 4(5)
 ; P9BE-AIX-NEXT:    lxv 0, 0(3)
 ; P9BE-AIX-NEXT:    ld 3, L..C1(2) # %const.1
-; P9BE-AIX-NEXT:    vadduwm 4, 4, 4
+; P9BE-AIX-NEXT:    vextsb2w 4, 4
 ; P9BE-AIX-NEXT:    xxperm 2, 1, 0
 ; P9BE-AIX-NEXT:    lxv 0, 0(3)
 ; P9BE-AIX-NEXT:    xxperm 3, 3, 0
@@ -89,10 +89,10 @@ define void @test64(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9BE-AIX32-NEXT:    lxvwsx 0, 3, 4
 ; P9BE-AIX32-NEXT:    li 3, 4
 ; P9BE-AIX32-NEXT:    xxlxor 2, 2, 2
-; P9BE-AIX32-NEXT:    vspltisw 4, 8
+; P9BE-AIX32-NEXT:    xxspltib 4, 16
 ; P9BE-AIX32-NEXT:    lxvwsx 1, 5, 3
 ; P9BE-AIX32-NEXT:    lwz 3, L..C0(2) # %const.0
-; P9BE-AIX32-NEXT:    vadduwm 4, 4, 4
+; P9BE-AIX32-NEXT:    vextsb2w 4, 4
 ; P9BE-AIX32-NEXT:    xxmrghw 2, 0, 1
 ; P9BE-AIX32-NEXT:    lxv 0, 0(3)
 ; P9BE-AIX32-NEXT:    li 3, 8
@@ -137,11 +137,11 @@ define void @test32(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9LE-NEXT:    lxsiwzx 2, 3, 4
 ; P9LE-NEXT:    addis 3, 2, .LCPI1_0@toc@ha
 ; P9LE-NEXT:    xxlxor 0, 0, 0
-; P9LE-NEXT:    vspltisw 4, 8
+; P9LE-NEXT:    xxspltib 4, 16
 ; P9LE-NEXT:    addi 3, 3, .LCPI1_0@toc@l
 ; P9LE-NEXT:    lxv 1, 0(3)
 ; P9LE-NEXT:    li 3, 4
-; P9LE-NEXT:    vadduwm 4, 4, 4
+; P9LE-NEXT:    vextsb2w 4, 4
 ; P9LE-NEXT:    lxsiwzx 3, 5, 3
 ; P9LE-NEXT:    xxperm 2, 0, 1
 ; P9LE-NEXT:    xxperm 3, 0, 1
@@ -158,11 +158,11 @@ define void @test32(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9BE-NEXT:    lxsiwzx 2, 3, 4
 ; P9BE-NEXT:    addis 3, 2, .LCPI1_0@toc@ha
 ; P9BE-NEXT:    xxlxor 0, 0, 0
-; P9BE-NEXT:    vspltisw 4, 8
+; P9BE-NEXT:    xxspltib 4, 16
 ; P9BE-NEXT:    addi 3, 3, .LCPI1_0@toc@l
 ; P9BE-NEXT:    lxv 1, 0(3)
 ; P9BE-NEXT:    li 3, 4
-; P9BE-NEXT:    vadduwm 4, 4, 4
+; P9BE-NEXT:    vextsb2w 4, 4
 ; P9BE-NEXT:    lxsiwzx 3, 5, 3
 ; P9BE-NEXT:    xxperm 2, 0, 1
 ; P9BE-NEXT:    xxperm 3, 0, 1
@@ -179,10 +179,10 @@ define void @test32(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9BE-AIX-NEXT:    lxsiwzx 2, 3, 4
 ; P9BE-AIX-NEXT:    ld 3, L..C2(2) # %const.0
 ; P9BE-AIX-NEXT:    xxlxor 0, 0, 0
-; P9BE-AIX-NEXT:    vspltisw 4, 8
+; P9BE-AIX-NEXT:    xxspltib 4, 16
 ; P9BE-AIX-NEXT:    lxv 1, 0(3)
 ; P9BE-AIX-NEXT:    li 3, 4
-; P9BE-AIX-NEXT:    vadduwm 4, 4, 4
+; P9BE-AIX-NEXT:    vextsb2w 4, 4
 ; P9BE-AIX-NEXT:    lxsiwzx 3, 5, 3
 ; P9BE-AIX-NEXT:    xxperm 2, 0, 1
 ; P9BE-AIX-NEXT:    xxperm 3, 0, 1
@@ -199,10 +199,10 @@ define void @test32(ptr nocapture readonly %pix2, i32 signext %i_pix2) {
 ; P9BE-AIX32-NEXT:    lxsiwzx 2, 3, 4
 ; P9BE-AIX32-NEXT:    lwz 3, L..C2(2) # %const.0
 ; P9BE-AIX32-NEXT:    xxlxor 0, 0, 0
-; P9BE-AIX32-NEXT:    vspltisw 4, 8
+; P9BE-AIX32-NEXT:    xxspltib 4, 16
 ; P9BE-AIX32-NEXT:    lxv 1, 0(3)
 ; P9BE-AIX32-NEXT:    li 3, 4
-; P9BE-AIX32-NEXT:    vadduwm 4, 4, 4
+; P9BE-AIX32-NEXT:    vextsb2w 4, 4
 ; P9BE-AIX32-NEXT:    lxsiwzx 3, 5, 3
 ; P9BE-AIX32-NEXT:    xxperm 2, 0, 1
 ; P9BE-AIX32-NEXT:    xxperm 3, 0, 1
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/splat-extend.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/splat-extend.ll
new file mode 100644
index 000000000000..5621f523cc28
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/splat-extend.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs  -mtriple=powerpc64-aix-xcoff \
+; RUN:   -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs  -mtriple=powerpc-aix-xcoff \
+; RUN:   -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+
+define dso_local noundef <8 x i16> @v103s() local_unnamed_addr {
+; CHECK-LABEL: v103s:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v2, 103
+; CHECK-NEXT:    vupklsb v2, v2
+; CHECK-NEXT:    blr
+entry:
+  ret <8 x i16> splat (i16 103)
+}
+
+define dso_local noundef <2 x i64> @v103l() local_unnamed_addr {
+; CHECK-LABEL: v103l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v2, 103
+; CHECK-NEXT:    vextsb2d v2, v2
+; CHECK-NEXT:    blr
+entry:
+  ret <2 x i64> splat (i64 103)
+}
+
+define dso_local noundef <4 x i32> @v103i() local_unnamed_addr {
+; CHECK-LABEL: v103i:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v2, 103
+; CHECK-NEXT:    vextsb2w v2, v2
+; CHECK-NEXT:    blr
+entry:
+  ret <4 x i32> splat (i32 103)
+}
+
+define dso_local noundef <2 x i64> @v11l() local_unnamed_addr {
+; CHECK-LABEL: v11l:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vspltisw v2, -11
+; CHECK-NEXT:    vupklsw v2, v2
+; CHECK-NEXT:    blr
+entry:
+  ret <2 x i64> splat (i64 -11)
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/vec-min-max.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/vec-min-max.ll
index f204827005e2..8124fde2667d 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/vec-min-max.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/vec-min-max.ll
@@ -79,7 +79,7 @@ define <4 x float> @getsmaxf32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    blr
 entry:
   %0 = fcmp nnan nsz oge <4 x float> %a, %b
-  %1 = select <4 x i1> %0, <4 x float> %a, <4 x float> %b
+  %1 = select nsz <4 x i1> %0, <4 x float> %a, <4 x float> %b
   ret <4 x float> %1
 }
 
@@ -90,7 +90,7 @@ define <2 x double> @getsmaxf64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-NEXT:    blr
 entry:
   %0 = fcmp nnan nsz oge <2 x double> %a, %b
-  %1 = select <2 x i1> %0, <2 x double> %a, <2 x double> %b
+  %1 = select nsz <2 x i1> %0, <2 x double> %a, <2 x double> %b
   ret <2 x double> %1
 }
 
@@ -171,7 +171,7 @@ define <4 x float> @getsminf32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    blr
 entry:
   %0 = fcmp nnan nsz ole <4 x float> %a, %b
-  %1 = select <4 x i1> %0, <4 x float> %a, <4 x float> %b
+  %1 = select nsz <4 x i1> %0, <4 x float> %a, <4 x float> %b
   ret <4 x float> %1
 }
 
@@ -182,7 +182,7 @@ define <2 x double> @getsminf64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-NEXT:    blr
 entry:
   %0 = fcmp nnan nsz ole <2 x double> %a, %b
-  %1 = select <2 x i1> %0, <2 x double> %a, <2 x double> %b
+  %1 = select nsz <2 x i1> %0, <2 x double> %a, <2 x double> %b
   ret <2 x double> %1
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll
index 3f7e0b694ce4..210aee13486c 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/vec_add_sub_doubleword.ll
@@ -16,9 +16,8 @@ define <2 x i64> @test_add(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <2 x i64> @increment_by_one(<2 x i64> %x) nounwind {
 ; VSX-LABEL: increment_by_one:
 ; VSX:       # %bb.0:
-; VSX-NEXT:    addis 3, 2, .LCPI1_0@toc@ha
-; VSX-NEXT:    addi 3, 3, .LCPI1_0@toc@l
-; VSX-NEXT:    lxvd2x 35, 0, 3
+; VSX-NEXT:    vspltisw 3, 1
+; VSX-NEXT:    vupklsw 3, 3
 ; VSX-NEXT:    vaddudm 2, 2, 3
 ; VSX-NEXT:    blr
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-extend-sign.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-extend-sign.ll
index 540a00fa84c5..ef3988e20e1f 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-extend-sign.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-extend-sign.ll
@@ -144,9 +144,8 @@ entry:
 define <2 x i64> @test_none(<2 x i64> %m) {
 ; CHECK-P9-LABEL: test_none:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI5_0@toc@ha
-; CHECK-P9-NEXT:    addi 3, 3, .LCPI5_0@toc@l
-; CHECK-P9-NEXT:    lxv 35, 0(3)
+; CHECK-P9-NEXT:    xxspltib 35, 16
+; CHECK-P9-NEXT:    vextsb2d 3, 3
 ; CHECK-P9-NEXT:    vsld 2, 2, 3
 ; CHECK-P9-NEXT:    vsrad 2, 2, 3
 ; CHECK-P9-NEXT:    blr
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-popcnt-128-ult-ugt.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-popcnt-128-ult-ugt.ll
index 04351346745b..43cbc62e0bb1 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-popcnt-128-ult-ugt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/vector-popcnt-128-ult-ugt.ll
@@ -7838,9 +7838,9 @@ define <4 x i32> @ult_16_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_16_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 8
+; PWR9-NEXT:    xxspltib 35, 16
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -7974,9 +7974,9 @@ define <4 x i32> @ugt_16_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_16_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 8
+; PWR9-NEXT:    xxspltib 35, 16
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -8108,10 +8108,9 @@ define <4 x i32> @ult_17_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_17_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 1
+; PWR9-NEXT:    xxspltib 35, 17
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -8243,10 +8242,9 @@ define <4 x i32> @ugt_17_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_17_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 1
+; PWR9-NEXT:    xxspltib 35, 17
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -8380,9 +8378,9 @@ define <4 x i32> @ult_18_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_18_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 9
+; PWR9-NEXT:    xxspltib 35, 18
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -8516,9 +8514,9 @@ define <4 x i32> @ugt_18_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_18_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 9
+; PWR9-NEXT:    xxspltib 35, 18
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -8653,10 +8651,9 @@ define <4 x i32> @ult_19_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_19_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 3
+; PWR9-NEXT:    xxspltib 35, 19
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -8791,10 +8788,9 @@ define <4 x i32> @ugt_19_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_19_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 3
+; PWR9-NEXT:    xxspltib 35, 19
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -8928,9 +8924,9 @@ define <4 x i32> @ult_20_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_20_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 10
+; PWR9-NEXT:    xxspltib 35, 20
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -9064,9 +9060,9 @@ define <4 x i32> @ugt_20_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_20_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 10
+; PWR9-NEXT:    xxspltib 35, 20
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -9201,10 +9197,9 @@ define <4 x i32> @ult_21_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_21_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 5
+; PWR9-NEXT:    xxspltib 35, 21
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -9339,10 +9334,9 @@ define <4 x i32> @ugt_21_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_21_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 5
+; PWR9-NEXT:    xxspltib 35, 21
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -9476,9 +9470,9 @@ define <4 x i32> @ult_22_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_22_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 11
+; PWR9-NEXT:    xxspltib 35, 22
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -9612,9 +9606,9 @@ define <4 x i32> @ugt_22_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_22_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 11
+; PWR9-NEXT:    xxspltib 35, 22
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -9749,10 +9743,9 @@ define <4 x i32> @ult_23_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_23_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 7
+; PWR9-NEXT:    xxspltib 35, 23
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -9887,10 +9880,9 @@ define <4 x i32> @ugt_23_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_23_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 7
+; PWR9-NEXT:    xxspltib 35, 23
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10018,9 +10010,9 @@ define <4 x i32> @ult_24_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_24_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 12
+; PWR9-NEXT:    xxspltib 35, 24
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10148,9 +10140,9 @@ define <4 x i32> @ugt_24_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_24_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 12
+; PWR9-NEXT:    xxspltib 35, 24
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10285,10 +10277,9 @@ define <4 x i32> @ult_25_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_25_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 9
+; PWR9-NEXT:    xxspltib 35, 25
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10423,10 +10414,9 @@ define <4 x i32> @ugt_25_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_25_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 9
+; PWR9-NEXT:    xxspltib 35, 25
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10560,9 +10550,9 @@ define <4 x i32> @ult_26_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_26_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 13
+; PWR9-NEXT:    xxspltib 35, 26
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10696,9 +10686,9 @@ define <4 x i32> @ugt_26_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_26_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 13
+; PWR9-NEXT:    xxspltib 35, 26
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10833,10 +10823,9 @@ define <4 x i32> @ult_27_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_27_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 11
+; PWR9-NEXT:    xxspltib 35, 27
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -10971,10 +10960,9 @@ define <4 x i32> @ugt_27_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_27_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 11
+; PWR9-NEXT:    xxspltib 35, 27
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11108,9 +11096,9 @@ define <4 x i32> @ult_28_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_28_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 14
+; PWR9-NEXT:    xxspltib 35, 28
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11244,9 +11232,9 @@ define <4 x i32> @ugt_28_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_28_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 14
+; PWR9-NEXT:    xxspltib 35, 28
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11381,10 +11369,9 @@ define <4 x i32> @ult_29_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_29_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 13
+; PWR9-NEXT:    xxspltib 35, 29
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11519,10 +11506,9 @@ define <4 x i32> @ugt_29_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_29_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 13
+; PWR9-NEXT:    xxspltib 35, 29
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11656,9 +11642,9 @@ define <4 x i32> @ult_30_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_30_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 15
+; PWR9-NEXT:    xxspltib 35, 30
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11792,9 +11778,9 @@ define <4 x i32> @ugt_30_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ugt_30_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, 15
+; PWR9-NEXT:    xxspltib 35, 30
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vadduwm 3, 3, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11929,10 +11915,9 @@ define <4 x i32> @ult_31_v4i32(<4 x i32> %0) {
 ;
 ; PWR9-LABEL: ult_31_v4i32:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    vspltisw 3, -16
-; PWR9-NEXT:    vspltisw 4, 15
+; PWR9-NEXT:    xxspltib 35, 31
 ; PWR9-NEXT:    vpopcntw 2, 2
-; PWR9-NEXT:    vsubuwm 3, 4, 3
+; PWR9-NEXT:    vextsb2w 3, 3
 ; PWR9-NEXT:    vcmpgtuw 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
@@ -11991,19 +11976,17 @@ define <2 x i64> @ugt_1_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_1_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI100_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 1
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI100_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_1_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI100_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 1
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI100_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12061,19 +12044,17 @@ define <2 x i64> @ult_2_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_2_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI101_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 2
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI101_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_2_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI101_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 2
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI101_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12195,19 +12176,17 @@ define <2 x i64> @ugt_2_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_2_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI102_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 2
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI102_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_2_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI102_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 2
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI102_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12329,19 +12308,17 @@ define <2 x i64> @ult_3_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_3_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI103_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 3
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI103_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_3_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI103_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 3
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI103_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12463,19 +12440,17 @@ define <2 x i64> @ugt_3_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_3_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI104_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 3
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI104_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_3_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI104_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 3
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI104_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12597,19 +12572,17 @@ define <2 x i64> @ult_4_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_4_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI105_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 4
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI105_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_4_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI105_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 4
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI105_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12731,19 +12704,17 @@ define <2 x i64> @ugt_4_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_4_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI106_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 4
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI106_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_4_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI106_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 4
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI106_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12865,19 +12836,17 @@ define <2 x i64> @ult_5_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_5_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI107_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 5
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI107_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_5_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI107_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 5
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI107_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -12999,19 +12968,17 @@ define <2 x i64> @ugt_5_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_5_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI108_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 5
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI108_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_5_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI108_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 5
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI108_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -13133,19 +13100,17 @@ define <2 x i64> @ult_6_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_6_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI109_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 6
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI109_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_6_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI109_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 6
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI109_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -13267,19 +13232,17 @@ define <2 x i64> @ugt_6_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_6_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI110_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 6
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI110_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_6_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI110_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 6
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI110_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -13401,19 +13364,17 @@ define <2 x i64> @ult_7_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_7_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI111_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 7
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI111_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_7_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI111_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 7
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI111_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -13535,19 +13496,17 @@ define <2 x i64> @ugt_7_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_7_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI112_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 7
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI112_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_7_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI112_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 7
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI112_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -13669,19 +13628,17 @@ define <2 x i64> @ult_8_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_8_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI113_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 8
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI113_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_8_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI113_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 8
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI113_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -13803,19 +13760,17 @@ define <2 x i64> @ugt_8_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_8_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI114_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 8
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI114_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_8_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI114_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 8
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI114_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -13937,19 +13892,17 @@ define <2 x i64> @ult_9_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_9_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI115_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 9
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI115_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_9_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI115_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 9
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI115_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -14071,19 +14024,17 @@ define <2 x i64> @ugt_9_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_9_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI116_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 9
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI116_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_9_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI116_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 9
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI116_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -14205,19 +14156,17 @@ define <2 x i64> @ult_10_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_10_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI117_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 10
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI117_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_10_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI117_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 10
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI117_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -14339,19 +14288,17 @@ define <2 x i64> @ugt_10_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_10_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI118_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 10
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI118_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_10_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI118_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 10
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI118_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -14473,19 +14420,17 @@ define <2 x i64> @ult_11_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_11_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI119_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 11
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI119_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_11_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI119_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 11
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI119_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -14607,19 +14552,17 @@ define <2 x i64> @ugt_11_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_11_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI120_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 11
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI120_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_11_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI120_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 11
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI120_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -14741,19 +14684,17 @@ define <2 x i64> @ult_12_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_12_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI121_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 12
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI121_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_12_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI121_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 12
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI121_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -14875,19 +14816,17 @@ define <2 x i64> @ugt_12_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_12_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI122_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 12
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI122_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_12_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI122_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 12
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI122_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15009,19 +14948,17 @@ define <2 x i64> @ult_13_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_13_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI123_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 13
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI123_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_13_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI123_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 13
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI123_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15143,19 +15080,17 @@ define <2 x i64> @ugt_13_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_13_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI124_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 13
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI124_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_13_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI124_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 13
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI124_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15277,19 +15212,17 @@ define <2 x i64> @ult_14_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_14_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI125_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 14
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI125_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_14_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI125_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 14
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI125_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15411,19 +15344,17 @@ define <2 x i64> @ugt_14_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_14_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI126_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 14
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI126_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_14_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI126_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 14
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI126_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15545,19 +15476,17 @@ define <2 x i64> @ult_15_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ult_15_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI127_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 15
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI127_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 3, 2
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ult_15_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI127_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 15
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI127_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15679,19 +15608,17 @@ define <2 x i64> @ugt_15_v2i64(<2 x i64> %0) {
 ;
 ; PWR8-LABEL: ugt_15_v2i64:
 ; PWR8:       # %bb.0:
-; PWR8-NEXT:    addis 3, 2, .LCPI128_0@toc@ha
+; PWR8-NEXT:    vspltisw 3, 15
 ; PWR8-NEXT:    vpopcntd 2, 2
-; PWR8-NEXT:    addi 3, 3, .LCPI128_0@toc@l
-; PWR8-NEXT:    lxvd2x 35, 0, 3
+; PWR8-NEXT:    vupklsw 3, 3
 ; PWR8-NEXT:    vcmpgtud 2, 2, 3
 ; PWR8-NEXT:    blr
 ;
 ; PWR9-LABEL: ugt_15_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI128_0@toc@ha
+; PWR9-NEXT:    vspltisw 3, 15
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI128_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vupklsw 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15822,10 +15749,9 @@ define <2 x i64> @ult_16_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_16_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI129_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 16
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI129_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -15956,10 +15882,9 @@ define <2 x i64> @ugt_16_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_16_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI130_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 16
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI130_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -16090,10 +16015,9 @@ define <2 x i64> @ult_17_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_17_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI131_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 17
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI131_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -16224,10 +16148,9 @@ define <2 x i64> @ugt_17_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_17_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI132_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 17
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI132_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -16358,10 +16281,9 @@ define <2 x i64> @ult_18_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_18_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI133_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 18
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI133_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -16492,10 +16414,9 @@ define <2 x i64> @ugt_18_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_18_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI134_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 18
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI134_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -16626,10 +16547,9 @@ define <2 x i64> @ult_19_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_19_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI135_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 19
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI135_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -16760,10 +16680,9 @@ define <2 x i64> @ugt_19_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_19_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI136_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 19
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI136_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -16894,10 +16813,9 @@ define <2 x i64> @ult_20_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_20_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI137_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 20
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI137_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17028,10 +16946,9 @@ define <2 x i64> @ugt_20_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_20_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI138_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 20
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI138_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17162,10 +17079,9 @@ define <2 x i64> @ult_21_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_21_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI139_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 21
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI139_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17296,10 +17212,9 @@ define <2 x i64> @ugt_21_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_21_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI140_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 21
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI140_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17430,10 +17345,9 @@ define <2 x i64> @ult_22_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_22_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI141_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 22
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI141_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17564,10 +17478,9 @@ define <2 x i64> @ugt_22_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_22_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI142_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 22
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI142_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17698,10 +17611,9 @@ define <2 x i64> @ult_23_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_23_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI143_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 23
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI143_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17832,10 +17744,9 @@ define <2 x i64> @ugt_23_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_23_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI144_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 23
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI144_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -17966,10 +17877,9 @@ define <2 x i64> @ult_24_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_24_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI145_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 24
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI145_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -18100,10 +18010,9 @@ define <2 x i64> @ugt_24_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_24_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI146_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 24
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI146_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -18234,10 +18143,9 @@ define <2 x i64> @ult_25_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_25_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI147_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 25
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI147_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -18368,10 +18276,9 @@ define <2 x i64> @ugt_25_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_25_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI148_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 25
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI148_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -18502,10 +18409,9 @@ define <2 x i64> @ult_26_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_26_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI149_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 26
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI149_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -18636,10 +18542,9 @@ define <2 x i64> @ugt_26_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_26_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI150_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 26
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI150_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -18770,10 +18675,9 @@ define <2 x i64> @ult_27_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_27_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI151_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 27
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI151_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -18904,10 +18808,9 @@ define <2 x i64> @ugt_27_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_27_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI152_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 27
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI152_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19038,10 +18941,9 @@ define <2 x i64> @ult_28_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_28_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI153_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 28
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI153_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19172,10 +19074,9 @@ define <2 x i64> @ugt_28_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_28_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI154_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 28
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI154_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19306,10 +19207,9 @@ define <2 x i64> @ult_29_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_29_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI155_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 29
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI155_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19440,10 +19340,9 @@ define <2 x i64> @ugt_29_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_29_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI156_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 29
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI156_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19574,10 +19473,9 @@ define <2 x i64> @ult_30_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_30_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI157_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 30
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI157_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19708,10 +19606,9 @@ define <2 x i64> @ugt_30_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_30_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI158_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 30
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI158_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19842,10 +19739,9 @@ define <2 x i64> @ult_31_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_31_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI159_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 31
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI159_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -19976,10 +19872,9 @@ define <2 x i64> @ugt_31_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_31_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI160_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 31
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI160_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -20110,10 +20005,9 @@ define <2 x i64> @ult_32_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_32_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI161_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 32
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI161_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -20244,10 +20138,9 @@ define <2 x i64> @ugt_32_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_32_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI162_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 32
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI162_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -20378,10 +20271,9 @@ define <2 x i64> @ult_33_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_33_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI163_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 33
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI163_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -20512,10 +20404,9 @@ define <2 x i64> @ugt_33_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_33_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI164_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 33
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI164_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -20646,10 +20537,9 @@ define <2 x i64> @ult_34_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_34_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI165_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 34
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI165_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -20780,10 +20670,9 @@ define <2 x i64> @ugt_34_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_34_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI166_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 34
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI166_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -20914,10 +20803,9 @@ define <2 x i64> @ult_35_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_35_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI167_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 35
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI167_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21048,10 +20936,9 @@ define <2 x i64> @ugt_35_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_35_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI168_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 35
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI168_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21182,10 +21069,9 @@ define <2 x i64> @ult_36_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_36_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI169_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 36
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI169_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21316,10 +21202,9 @@ define <2 x i64> @ugt_36_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_36_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI170_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 36
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI170_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21450,10 +21335,9 @@ define <2 x i64> @ult_37_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_37_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI171_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 37
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI171_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21584,10 +21468,9 @@ define <2 x i64> @ugt_37_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_37_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI172_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 37
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI172_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21718,10 +21601,9 @@ define <2 x i64> @ult_38_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_38_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI173_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 38
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI173_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21852,10 +21734,9 @@ define <2 x i64> @ugt_38_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_38_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI174_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 38
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI174_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -21986,10 +21867,9 @@ define <2 x i64> @ult_39_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_39_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI175_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 39
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI175_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -22120,10 +22000,9 @@ define <2 x i64> @ugt_39_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_39_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI176_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 39
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI176_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -22254,10 +22133,9 @@ define <2 x i64> @ult_40_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_40_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI177_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 40
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI177_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -22388,10 +22266,9 @@ define <2 x i64> @ugt_40_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_40_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI178_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 40
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI178_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -22522,10 +22399,9 @@ define <2 x i64> @ult_41_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_41_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI179_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 41
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI179_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -22656,10 +22532,9 @@ define <2 x i64> @ugt_41_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_41_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI180_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 41
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI180_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -22790,10 +22665,9 @@ define <2 x i64> @ult_42_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_42_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI181_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 42
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI181_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -22924,10 +22798,9 @@ define <2 x i64> @ugt_42_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_42_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI182_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 42
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI182_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23058,10 +22931,9 @@ define <2 x i64> @ult_43_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_43_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI183_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 43
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI183_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23192,10 +23064,9 @@ define <2 x i64> @ugt_43_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_43_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI184_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 43
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI184_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23326,10 +23197,9 @@ define <2 x i64> @ult_44_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_44_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI185_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 44
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI185_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23460,10 +23330,9 @@ define <2 x i64> @ugt_44_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_44_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI186_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 44
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI186_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23594,10 +23463,9 @@ define <2 x i64> @ult_45_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_45_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI187_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 45
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI187_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23728,10 +23596,9 @@ define <2 x i64> @ugt_45_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_45_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI188_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 45
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI188_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23862,10 +23729,9 @@ define <2 x i64> @ult_46_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_46_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI189_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 46
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI189_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -23996,10 +23862,9 @@ define <2 x i64> @ugt_46_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_46_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI190_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 46
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI190_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -24130,10 +23995,9 @@ define <2 x i64> @ult_47_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_47_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI191_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 47
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI191_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -24264,10 +24128,9 @@ define <2 x i64> @ugt_47_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_47_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI192_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 47
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI192_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -24398,10 +24261,9 @@ define <2 x i64> @ult_48_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_48_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI193_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 48
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI193_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -24532,10 +24394,9 @@ define <2 x i64> @ugt_48_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_48_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI194_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 48
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI194_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -24666,10 +24527,9 @@ define <2 x i64> @ult_49_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_49_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI195_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 49
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI195_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -24800,10 +24660,9 @@ define <2 x i64> @ugt_49_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_49_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI196_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 49
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI196_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -24934,10 +24793,9 @@ define <2 x i64> @ult_50_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_50_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI197_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 50
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI197_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -25068,10 +24926,9 @@ define <2 x i64> @ugt_50_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_50_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI198_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 50
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI198_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -25202,10 +25059,9 @@ define <2 x i64> @ult_51_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_51_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI199_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 51
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI199_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -25336,10 +25192,9 @@ define <2 x i64> @ugt_51_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_51_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI200_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 51
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI200_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -25470,10 +25325,9 @@ define <2 x i64> @ult_52_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_52_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI201_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 52
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI201_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -25604,10 +25458,9 @@ define <2 x i64> @ugt_52_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_52_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI202_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 52
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI202_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -25738,10 +25591,9 @@ define <2 x i64> @ult_53_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_53_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI203_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 53
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI203_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -25872,10 +25724,9 @@ define <2 x i64> @ugt_53_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_53_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI204_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 53
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI204_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26006,10 +25857,9 @@ define <2 x i64> @ult_54_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_54_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI205_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 54
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI205_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26140,10 +25990,9 @@ define <2 x i64> @ugt_54_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_54_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI206_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 54
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI206_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26274,10 +26123,9 @@ define <2 x i64> @ult_55_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_55_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI207_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 55
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI207_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26408,10 +26256,9 @@ define <2 x i64> @ugt_55_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_55_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI208_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 55
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI208_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26542,10 +26389,9 @@ define <2 x i64> @ult_56_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_56_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI209_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 56
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI209_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26676,10 +26522,9 @@ define <2 x i64> @ugt_56_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_56_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI210_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 56
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI210_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26810,10 +26655,9 @@ define <2 x i64> @ult_57_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_57_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI211_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 57
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI211_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -26944,10 +26788,9 @@ define <2 x i64> @ugt_57_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_57_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI212_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 57
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI212_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -27078,10 +26921,9 @@ define <2 x i64> @ult_58_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_58_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI213_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 58
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI213_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -27212,10 +27054,9 @@ define <2 x i64> @ugt_58_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_58_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI214_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 58
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI214_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -27346,10 +27187,9 @@ define <2 x i64> @ult_59_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_59_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI215_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 59
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI215_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -27480,10 +27320,9 @@ define <2 x i64> @ugt_59_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_59_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI216_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 59
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI216_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -27614,10 +27453,9 @@ define <2 x i64> @ult_60_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_60_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI217_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 60
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI217_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -27748,10 +27586,9 @@ define <2 x i64> @ugt_60_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_60_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI218_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 60
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI218_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -27882,10 +27719,9 @@ define <2 x i64> @ult_61_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_61_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI219_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 61
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI219_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -28016,10 +27852,9 @@ define <2 x i64> @ugt_61_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_61_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI220_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 61
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI220_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -28150,10 +27985,9 @@ define <2 x i64> @ult_62_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_62_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI221_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 62
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI221_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -28284,10 +28118,9 @@ define <2 x i64> @ugt_62_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ugt_62_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI222_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 62
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI222_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 2, 3
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
@@ -28418,10 +28251,9 @@ define <2 x i64> @ult_63_v2i64(<2 x i64> %0) {
 ;
 ; PWR9-LABEL: ult_63_v2i64:
 ; PWR9:       # %bb.0:
-; PWR9-NEXT:    addis 3, 2, .LCPI223_0@toc@ha
+; PWR9-NEXT:    xxspltib 35, 63
 ; PWR9-NEXT:    vpopcntd 2, 2
-; PWR9-NEXT:    addi 3, 3, .LCPI223_0@toc@l
-; PWR9-NEXT:    lxv 35, 0(3)
+; PWR9-NEXT:    vextsb2d 3, 3
 ; PWR9-NEXT:    vcmpgtud 2, 3, 2
 ; PWR9-NEXT:    blr
   %2 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/vsx-fma-m-early.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/vsx-fma-m-early.ll
index 96f64f5d0cab..9cb2d4444b97 100644
--- a/external/llvm-project/llvm/test/CodeGen/PowerPC/vsx-fma-m-early.ll
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/vsx-fma-m-early.ll
@@ -69,14 +69,14 @@ declare <4 x i32> @llvm.ppc.vsx.xvcmpgtsp(<4 x float>, <4 x float>)
 ; CHECK64-NEXT:         bltlr   cr0
 ; CHECK64-NEXT: # %bb.1:                                # %for.body.preheader
 ; CHECK64-NEXT:         xxspltiw vs0, 1069066811
+; CHECK64-NEXT:         xxspltiw vs1, 1170469888
 ; CHECK64-NEXT:         mtctr r5
 ; CHECK64-NEXT:         li r5, 0
 ; CHECK64-NEXT:         {{.*}}align  5
 ; CHECK64-NEXT: [[L2_bar:.*]]:                               # %for.body
 ; CHECK64-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK64-NEXT:         lxvx vs1, r4, r5
-; CHECK64-NEXT:         xxspltiw vs2, 1170469888
-; CHECK64-NEXT:         xvmaddasp vs2, vs1, vs0
+; CHECK64-NEXT:         lxvx vs2, r4, r5
+; CHECK64-NEXT:         xvmaddmsp vs2, vs0, vs1
 ; CHECK64-NEXT:         stxvx vs2, r3, r5
 ; CHECK64-NEXT:         addi r5, r5, 16
 ; CHECK64-NEXT:         bdnz [[L2_bar]]
@@ -139,17 +139,17 @@ declare <4 x i32> @llvm.ppc.vsx.xvcmpgtsp(<4 x float>, <4 x float>)
 ; CHECK32-NEXT:       blelr cr0
 ; CHECK32-NEXT: # %bb.1:                                # %for.body.preheader
 ; CHECK32-NEXT:       xxspltiw vs0, 1069066811
+; CHECK32-NEXT:       xxspltiw vs1, 1170469888
 ; CHECK32-NEXT:       li r6, 0
 ; CHECK32-NEXT:       li r7, 0
 ; CHECK32-NEXT:       .align  4
 ; CHECK32-NEXT: [[L2_foo:.*]]:                               # %for.body
 ; CHECK32-NEXT:                                         # =>This Inner Loop Header: Depth=1
 ; CHECK32-NEXT:       slwi r8, r7, 4
-; CHECK32-NEXT:       xxspltiw vs2, 1170469888
 ; CHECK32-NEXT:       addic r7, r7, 1
 ; CHECK32-NEXT:       addze r6, r6
-; CHECK32-NEXT:       lxvx vs1, r4, r8
-; CHECK32-NEXT:       xvmaddasp vs2, vs1, vs0
+; CHECK32-NEXT:       lxvx vs2, r4, r8
+; CHECK32-NEXT:       xvmaddmsp vs2, vs0, vs1
 ; CHECK32-NEXT:       stxvx vs2, r3, r8
 ; CHECK32-NEXT:       xor r8, r7, r5
 ; CHECK32-NEXT:       or. r8, r8, r6
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
new file mode 100644
index 000000000000..2868669c52ce
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; Function to test ternary(A, xor(B, C), and(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_xor_BC_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_xor_BC_and_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <4 x i32> %B, %C
+  %and = and <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %xor, <4 x i32> %and
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, xor(B, C), and(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_xor_BC_and_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_xor_BC_and_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <2 x i64> %B, %C
+  %and = and <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %xor, <2 x i64> %and
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, nor(B, C), and(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_nor_BC_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_and_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <4 x i32> %B, %C
+  %nor = xor <4 x i32> %or, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector NOR operation
+  %and = and <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %nor, <4 x i32> %and
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, nor(B, C), and(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_nor_BC_and_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_and_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <2 x i64> %B, %C
+  %nor = xor <2 x i64> %or, <i64 -1, i64 -1>  ; Vector NOR operation
+  %and = and <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %nor, <2 x i64> %and
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, eqv(B, C), and(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_eqv_BC_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_and_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <4 x i32> %B, %C
+  %eqv = xor <4 x i32> %xor, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector eqv operation
+  %and = and <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %eqv, <4 x i32> %and
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, eqv(B, C), and(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_eqv_BC_and_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_and_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <2 x i64> %B, %C
+  %eqv = xor <2 x i64> %xor, <i64 -1, i64 -1>  ; Vector eqv operation
+  %and = and <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %eqv, <2 x i64> %and
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, not(C), and(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_not_C_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_not_C_and_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <4 x i32> %C, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector not operation
+  %and = and <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %not, <4 x i32> %and
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, not(C), and(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_not_C_and_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_not_C_and_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <2 x i64> %C, <i64 -1, i64 -1>  ; Vector not operation
+  %and = and <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %and
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, not(B), and(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_not_B_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_not_B_and_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <4 x i32> %B, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector not operation
+  %and = and <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %not, <4 x i32> %and
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, not(B), and(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_not_B_and_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_not_B_and_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <2 x i64> %B, <i64 -1, i64 -1>  ; Vector not operation
+  %and = and <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %and
+  ret <2 x i64> %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
new file mode 100644
index 000000000000..37a0edb14b78
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; Function to test ternary(A, and(B, C), B) for <4 x i32>
+define <4 x i32> @ternary_A_and_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_and_BC_B_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %and, <4 x i32> %B
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, and(B, C), B) for <2 x i64>
+define <2 x i64> @ternary_A_and_BC_B_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_and_BC_B_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %and, <2 x i64> %B
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, nor(B, C), B) for <4 x i32>
+define <4 x i32> @ternary_A_nor_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_B_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <4 x i32> %B, %C
+  %nor = xor <4 x i32> %or, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector NOR operation
+  %res = select <4 x i1> %A, <4 x i32> %nor, <4 x i32> %B
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, nor(B, C), B) for <2 x i64>
+define <2 x i64> @ternary_A_nor_BC_B_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_B_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <2 x i64> %B, %C
+  %nor = xor <2 x i64> %or, <i64 -1, i64 -1>  ; Vector NOR operation
+  %res = select <2 x i1> %A, <2 x i64> %nor, <2 x i64> %B
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, eqv(B, C), B) for <4 x i32>
+define <4 x i32> @ternary_A_eqv_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_B_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <4 x i32> %B, %C
+  %eqv = xor <4 x i32> %xor, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector eqv operation
+  %res = select <4 x i1> %A, <4 x i32> %eqv, <4 x i32> %B
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, eqv(B, C), B) for <2 x i64>
+define <2 x i64> @ternary_A_eqv_BC_B_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_B_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <2 x i64> %B, %C
+  %eqv = xor <2 x i64> %xor, <i64 -1, i64 -1>  ; Vector eqv operation
+  %res = select <2 x i1> %A, <2 x i64> %eqv, <2 x i64> %B
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, not(C), B) for <4 x i32>
+define <4 x i32> @ternary_A_not_C_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_not_C_B_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <4 x i32> %C, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector not operation
+  %res = select <4 x i1> %A, <4 x i32> %not, <4 x i32> %B
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, not(C), B) for <2 x i64>
+define <2 x i64> @ternary_A_not_C_B_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_not_C_B_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <2 x i64> %C, <i64 -1, i64 -1>  ; Vector not operation
+  %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %B
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, nand(B, C), B) for <4 x i32>
+define <4 x i32> @ternary_A_nand_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_B_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <4 x i32> %B, %C
+  %nand = xor <4 x i32> %and, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector nand operation
+  %res = select <4 x i1> %A, <4 x i32> %nand, <4 x i32> %B
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, nand(B, C), B) for <2 x i64>
+define <2 x i64> @ternary_A_nand_BC_B_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_B_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <2 x i64> %B, %C
+  %nand = xor <2 x i64> %and, <i64 -1, i64 -1>  ; Vector nand operation
+  %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %B
+  ret <2 x i64> %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
new file mode 100644
index 000000000000..411aa27a6186
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; Function to test ternary(A, and(B, C), C) for <4 x i32>
+define <4 x i32> @ternary_A_and_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_and_BC_C_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %and, <4 x i32> %C
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, and(B, C), C) for <2 x i64>
+define <2 x i64> @ternary_A_and_BC_C_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_and_BC_C_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %and, <2 x i64> %C
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, nor(B, C), C) for <4 x i32>
+define <4 x i32> @ternary_A_nor_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_C_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <4 x i32> %B, %C
+  %nor = xor <4 x i32> %or, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector NOR operation
+  %res = select <4 x i1> %A, <4 x i32> %nor, <4 x i32> %C
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, nor(B, C), C) for <2 x i64>
+define <2 x i64> @ternary_A_nor_BC_C_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_C_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <2 x i64> %B, %C
+  %nor = xor <2 x i64> %or, <i64 -1, i64 -1>  ; Vector NOR operation
+  %res = select <2 x i1> %A, <2 x i64> %nor, <2 x i64> %C
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, eqv(B, C), C) for <4 x i32>
+define <4 x i32> @ternary_A_eqv_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_C_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <4 x i32> %B, %C
+  %eqv = xor <4 x i32> %xor, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector eqv operation
+  %res = select <4 x i1> %A, <4 x i32> %eqv, <4 x i32> %C
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, eqv(B, C), C) for <2 x i64>
+define <2 x i64> @ternary_A_eqv_BC_C_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_C_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <2 x i64> %B, %C
+  %eqv = xor <2 x i64> %xor, <i64 -1, i64 -1>  ; Vector eqv operation
+  %res = select <2 x i1> %A, <2 x i64> %eqv, <2 x i64> %C
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, nand(B, C), C) for <4 x i32>
+define <4 x i32> @ternary_A_nand_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_C_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <4 x i32> %B, %C
+  %nand = xor <4 x i32> %and, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector nand operation
+  %res = select <4 x i1> %A, <4 x i32> %nand, <4 x i32> %C
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, nand(B, C), C) for <2 x i64>
+define <2 x i64> @ternary_A_nand_BC_C_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_C_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <2 x i64> %B, %C
+  %nand = xor <2 x i64> %and, <i64 -1, i64 -1>  ; Vector nand operation
+  %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %C
+  ret <2 x i64> %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
new file mode 100644
index 000000000000..1ad7e95e3682
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
@@ -0,0 +1,268 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; Function to test ternary(A, and(B, C), or(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_and_BC_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_and_BC_or_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <4 x i32> %B, %C
+  %or = or <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %and, <4 x i32> %or
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, and(B, C), or(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_and_BC_or_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_and_BC_or_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <2 x i64> %B, %C
+  %or = or <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %and, <2 x i64> %or
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, B, or(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_B_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_B_or_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %B, <4 x i32> %or
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, B, or(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_B_or_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_B_or_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %B, <2 x i64> %or
+  ret <2 x i64> %res
+}
+
+
+; Function to test ternary(A, C, or(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_C_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_C_or_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %C, <4 x i32> %or
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, C, or(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_C_or_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_C_or_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %C, <2 x i64> %or
+  ret <2 x i64> %res
+}
+
+
+; Function to test ternary(A, eqv(B,C), or(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_eqv_BC_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_or_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <4 x i32> %B, %C
+  %eqv = xor <4 x i32> %xor, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector eqv operation
+  %or = or <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %eqv, <4 x i32> %or
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, eqv(B,C), or(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_eqv_BC_or_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_or_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <2 x i64> %B, %C
+  %eqv = xor <2 x i64> %xor, <i64 -1, i64 -1>  ; Vector eqv operation
+  %or = or <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %eqv, <2 x i64> %or
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, not(C), or(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_not_C_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_not_C_or_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <4 x i32> %C, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector not operation
+  %or = or <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %not, <4 x i32> %or
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, not(C), or(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_not_C_or_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_not_C_or_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <2 x i64> %C, <i64 -1, i64 -1>  ; Vector not operation
+  %or = or <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %or
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, not(B), or(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_not_B_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_not_B_or_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <4 x i32> %B, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector not operation
+  %or = or <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %not, <4 x i32> %or
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, not(B), or(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_not_B_or_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_not_B_or_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <2 x i64> %B, <i64 -1, i64 -1>  ; Vector not operation
+  %or = or <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %or
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, nand(B,C), or(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_nand_BC_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_or_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <4 x i32> %B, %C
+  %nand = xor <4 x i32> %and, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector nand operation
+  %or = or <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %nand, <4 x i32> %or
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, nand(B,C), or(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_nand_BC_or_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_or_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <2 x i64> %B, %C
+  %nand = xor <2 x i64> %and, <i64 -1, i64 -1>  ; Vector nand operation
+  %or = or <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %or
+  ret <2 x i64> %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
new file mode 100644
index 000000000000..f8a8b6e9a048
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
@@ -0,0 +1,190 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; Function to test ternary(A, and(B, C), xor(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_and_BC_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_and_BC_xor_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <4 x i32> %B, %C
+  %xor = xor <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %and, <4 x i32> %xor
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, and(B, C), xor(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_and_BC_xor_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_and_BC_xor_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <2 x i64> %B, %C
+  %xor = xor <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %and, <2 x i64> %xor
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, B, xor(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_B_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_B_xor_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %B, <4 x i32> %xor
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, B, xor(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_B_xor_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_B_xor_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %B, <2 x i64> %xor
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, C, xor(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_C_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_C_xor_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %C, <4 x i32> %xor
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, C, xor(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_C_xor_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_C_xor_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %C, <2 x i64> %xor
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, or(B, C), xor(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_or_BC_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_or_BC_xor_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <4 x i32> %B, %C
+  %xor = xor <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %or, <4 x i32> %xor
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, or(B, C), xor(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_or_BC_xor_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_or_BC_xor_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <2 x i64> %B, %C
+  %xor = xor <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %or, <2 x i64> %xor
+  ret <2 x i64> %res
+}
+
+; Function to test ternary(A, nor(B, C), xor(B, C)) for <4 x i32>
+define <4 x i32> @ternary_A_nor_BC_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_xor_BC_4x32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxleqv v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslw v2, v2, v5
+; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <4 x i32> %B, %C
+  %nor = xor <4 x i32> %or, <i32 -1, i32 -1, i32 -1, i32 -1>    ; vector nor operation
+  %xor = xor <4 x i32> %B, %C
+  %res = select <4 x i1> %A, <4 x i32> %nor, <4 x i32> %xor
+  ret <4 x i32> %res
+}
+
+; Function to test ternary(A, nor(B, C), xor(B, C)) for <2 x i64>
+define <2 x i64> @ternary_A_nor_BC_xor_BC_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_xor_BC_2x64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxlxor v5, v5, v5
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    xxsplti32dx v5, 1, 63
+; CHECK-NEXT:    vsld v2, v2, v5
+; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <2 x i64> %B, %C
+  %nor = xor <2 x i64> %or, <i64 -1, i64 -1>    ; vector nor operation
+  %xor = xor <2 x i64> %B, %C
+  %res = select <2 x i1> %A, <2 x i64> %nor, <2 x i64> %xor
+  ret <2 x i64> %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll
index 1632f92e96b5..487cb5768dca 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll
@@ -468,7 +468,7 @@ define i64 @subi_i64(i64 %a) {
 ; RV64IM-LABEL: subi_i64:
 ; RV64IM:       # %bb.0: # %entry
 ; RV64IM-NEXT:    lui a1, 1048275
-; RV64IM-NEXT:    addiw a1, a1, -1548
+; RV64IM-NEXT:    addi a1, a1, -1548
 ; RV64IM-NEXT:    add a0, a0, a1
 ; RV64IM-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/bitmanip.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/bitmanip.ll
index bce6dfacf8e8..68bc1e5db609 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/bitmanip.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/bitmanip.ll
@@ -174,8 +174,8 @@ define i24 @bitreverse_i24(i24 %x) {
 ; RV64-NEXT:    slli a1, a0, 16
 ; RV64-NEXT:    lui a2, 4096
 ; RV64-NEXT:    lui a3, 1048335
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    addiw a3, a3, 240
+; RV64-NEXT:    addi a2, a2, -1
+; RV64-NEXT:    addi a3, a3, 240
 ; RV64-NEXT:    and a0, a0, a2
 ; RV64-NEXT:    srli a0, a0, 16
 ; RV64-NEXT:    or a0, a0, a1
@@ -184,7 +184,7 @@ define i24 @bitreverse_i24(i24 %x) {
 ; RV64-NEXT:    slli a0, a0, 4
 ; RV64-NEXT:    and a0, a0, a3
 ; RV64-NEXT:    lui a3, 1047757
-; RV64-NEXT:    addiw a3, a3, -820
+; RV64-NEXT:    addi a3, a3, -820
 ; RV64-NEXT:    srli a1, a1, 4
 ; RV64-NEXT:    or a0, a1, a0
 ; RV64-NEXT:    and a1, a3, a2
@@ -192,7 +192,7 @@ define i24 @bitreverse_i24(i24 %x) {
 ; RV64-NEXT:    slli a0, a0, 2
 ; RV64-NEXT:    and a0, a0, a3
 ; RV64-NEXT:    lui a3, 1047211
-; RV64-NEXT:    addiw a3, a3, -1366
+; RV64-NEXT:    addi a3, a3, -1366
 ; RV64-NEXT:    and a2, a3, a2
 ; RV64-NEXT:    srli a1, a1, 2
 ; RV64-NEXT:    or a0, a1, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll
index 309ebf71127c..b24ea9ec1561 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll
@@ -27,24 +27,14 @@ define void @constant_fold_barrier_i128(ptr %p) {
 ; RV32-NEXT:    lw a5, 12(a0)
 ; RV32-NEXT:    slli a1, a1, 11
 ; RV32-NEXT:    and a2, a2, a1
-; RV32-NEXT:    and a3, a3, zero
-; RV32-NEXT:    and a4, a4, zero
-; RV32-NEXT:    and a5, a5, zero
 ; RV32-NEXT:    add a2, a2, a1
-; RV32-NEXT:    add a6, a3, zero
 ; RV32-NEXT:    sltu a1, a2, a1
-; RV32-NEXT:    sltu a3, a3, a3
-; RV32-NEXT:    add a6, a6, a1
-; RV32-NEXT:    seqz a7, a6
+; RV32-NEXT:    mv a6, a1
+; RV32-NEXT:    seqz a7, a1
 ; RV32-NEXT:    and a1, a7, a1
-; RV32-NEXT:    add a7, a4, zero
-; RV32-NEXT:    sltu a4, a4, a4
-; RV32-NEXT:    or a1, a3, a1
-; RV32-NEXT:    add a7, a7, a1
-; RV32-NEXT:    seqz a3, a7
+; RV32-NEXT:    mv a7, a1
+; RV32-NEXT:    seqz a3, a1
 ; RV32-NEXT:    and a1, a3, a1
-; RV32-NEXT:    or a1, a4, a1
-; RV32-NEXT:    add a1, a5, a1
 ; RV32-NEXT:    sw a2, 0(a0)
 ; RV32-NEXT:    sw a6, 4(a0)
 ; RV32-NEXT:    sw a7, 8(a0)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll
index 8e4c0376ad0a..ec6b6568efa2 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll
@@ -21,12 +21,11 @@ define i128 @constant_fold_barrier_i128(i128 %x) {
 ; RV64-LABEL: constant_fold_barrier_i128:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    li a2, 1
-; RV64-NEXT:    and a1, a1, zero
 ; RV64-NEXT:    slli a2, a2, 11
 ; RV64-NEXT:    and a0, a0, a2
 ; RV64-NEXT:    add a0, a0, a2
 ; RV64-NEXT:    sltu a2, a0, a2
-; RV64-NEXT:    add a1, a1, a2
+; RV64-NEXT:    mv a1, a2
 ; RV64-NEXT:    ret
 entry:
   %and = and i128 %x, 2048
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/div-by-constant.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/div-by-constant.ll
index 9c46e6792e8d..389d59298505 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/div-by-constant.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/div-by-constant.ll
@@ -25,7 +25,7 @@ define i32 @udiv_constant_no_add(i32 %a) nounwind {
 ; RV64IM-NEXT:    slli a0, a0, 32
 ; RV64IM-NEXT:    lui a1, 205
 ; RV64IM-NEXT:    srli a0, a0, 32
-; RV64IM-NEXT:    addiw a1, a1, -819
+; RV64IM-NEXT:    addi a1, a1, -819
 ; RV64IM-NEXT:    slli a1, a1, 12
 ; RV64IM-NEXT:    addi a1, a1, -819
 ; RV64IM-NEXT:    mul a0, a0, a1
@@ -62,7 +62,7 @@ define i32 @udiv_constant_add(i32 %a) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    lui a1, 149797
 ; RV64IM-NEXT:    slli a2, a0, 32
-; RV64IM-NEXT:    addiw a1, a1, -1755
+; RV64IM-NEXT:    addi a1, a1, -1755
 ; RV64IM-NEXT:    srli a2, a2, 32
 ; RV64IM-NEXT:    mul a1, a2, a1
 ; RV64IM-NEXT:    srli a1, a1, 32
@@ -75,7 +75,7 @@ define i32 @udiv_constant_add(i32 %a) nounwind {
 ; RV64IMZB-LABEL: udiv_constant_add:
 ; RV64IMZB:       # %bb.0:
 ; RV64IMZB-NEXT:    lui a1, 149797
-; RV64IMZB-NEXT:    addiw a1, a1, -1755
+; RV64IMZB-NEXT:    addi a1, a1, -1755
 ; RV64IMZB-NEXT:    zext.w a2, a0
 ; RV64IMZB-NEXT:    mul a1, a2, a1
 ; RV64IMZB-NEXT:    srli a1, a1, 32
@@ -92,34 +92,26 @@ define i64 @udiv64_constant_no_add(i64 %a) nounwind {
 ; RV32-LABEL: udiv64_constant_no_add:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    lui a2, 838861
-; RV32-NEXT:    mulhu a3, a0, zero
 ; RV32-NEXT:    addi a4, a2, -819
 ; RV32-NEXT:    addi a2, a2, -820
 ; RV32-NEXT:    mul a5, a1, a4
 ; RV32-NEXT:    mul a6, a0, a2
 ; RV32-NEXT:    mulhu a7, a0, a4
-; RV32-NEXT:    mul t0, zero, a4
 ; RV32-NEXT:    mul t1, a1, a2
 ; RV32-NEXT:    mulhu t2, a1, a4
 ; RV32-NEXT:    mulhu a0, a0, a2
 ; RV32-NEXT:    mulhu a1, a1, a2
-; RV32-NEXT:    mul a2, zero, a2
-; RV32-NEXT:    mulhu a4, zero, a4
 ; RV32-NEXT:    add a5, a5, a6
-; RV32-NEXT:    add a2, t0, a2
-; RV32-NEXT:    add t0, t0, t1
-; RV32-NEXT:    add a1, a4, a1
+; RV32-NEXT:    mv t0, t1
 ; RV32-NEXT:    sltu a4, a5, a6
 ; RV32-NEXT:    add a5, a5, a7
-; RV32-NEXT:    sltu a6, t0, t1
-; RV32-NEXT:    sltiu t1, t0, 0
+; RV32-NEXT:    sltu a6, t1, t1
+; RV32-NEXT:    sltiu t1, t1, 0
 ; RV32-NEXT:    add t0, t0, t2
-; RV32-NEXT:    add a1, a2, a1
 ; RV32-NEXT:    sltu a2, a5, a7
 ; RV32-NEXT:    add a6, a6, t1
 ; RV32-NEXT:    sltu a5, t0, t2
 ; RV32-NEXT:    add t0, t0, a0
-; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    add a2, a4, a2
 ; RV32-NEXT:    add a5, a6, a5
 ; RV32-NEXT:    sltu a0, t0, a0
@@ -156,34 +148,27 @@ define i64 @udiv64_constant_add(i64 %a) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    lui a2, 599186
 ; RV32-NEXT:    lui a3, 149797
-; RV32-NEXT:    mulhu a4, a0, zero
 ; RV32-NEXT:    addi a2, a2, 1171
 ; RV32-NEXT:    addi a3, a3, -1756
 ; RV32-NEXT:    mul a5, a1, a2
 ; RV32-NEXT:    mul a6, a0, a3
 ; RV32-NEXT:    mulhu a7, a0, a2
-; RV32-NEXT:    mul t0, zero, a2
-; RV32-NEXT:    mulhu t1, zero, a2
 ; RV32-NEXT:    mulhu t2, a1, a3
-; RV32-NEXT:    add t1, t1, t2
-; RV32-NEXT:    mul t2, zero, a3
-; RV32-NEXT:    add t2, t0, t2
-; RV32-NEXT:    add t1, t2, t1
+; RV32-NEXT:    mv t1, t2
 ; RV32-NEXT:    mul t2, a1, a3
 ; RV32-NEXT:    mulhu a2, a1, a2
 ; RV32-NEXT:    mulhu a3, a0, a3
 ; RV32-NEXT:    add a5, a5, a6
-; RV32-NEXT:    add t0, t0, t2
+; RV32-NEXT:    mv t0, t2
 ; RV32-NEXT:    sltu a6, a5, a6
 ; RV32-NEXT:    add a5, a5, a7
-; RV32-NEXT:    sltu t2, t0, t2
+; RV32-NEXT:    sltu t2, t2, t2
 ; RV32-NEXT:    sltu a5, a5, a7
 ; RV32-NEXT:    sltiu a7, t0, 0
 ; RV32-NEXT:    add t0, t0, a2
 ; RV32-NEXT:    add a7, t2, a7
 ; RV32-NEXT:    sltu a2, t0, a2
 ; RV32-NEXT:    add t0, t0, a3
-; RV32-NEXT:    add a4, t1, a4
 ; RV32-NEXT:    add a5, a6, a5
 ; RV32-NEXT:    add a2, a7, a2
 ; RV32-NEXT:    sltu a3, t0, a3
@@ -195,7 +180,7 @@ define i64 @udiv64_constant_add(i64 %a) nounwind {
 ; RV32-NEXT:    add a2, a2, a3
 ; RV32-NEXT:    sub a1, a1, a0
 ; RV32-NEXT:    srli a5, a5, 1
-; RV32-NEXT:    add a2, a4, a2
+; RV32-NEXT:    add a2, t1, a2
 ; RV32-NEXT:    sub a1, a1, a2
 ; RV32-NEXT:    slli a0, a1, 31
 ; RV32-NEXT:    srli a1, a1, 1
@@ -301,7 +286,7 @@ define i16 @udiv16_constant_no_add(i16 %a) nounwind {
 ; RV64IM-NEXT:    slli a0, a0, 48
 ; RV64IM-NEXT:    lui a1, 13
 ; RV64IM-NEXT:    srli a0, a0, 48
-; RV64IM-NEXT:    addiw a1, a1, -819
+; RV64IM-NEXT:    addi a1, a1, -819
 ; RV64IM-NEXT:    mul a0, a0, a1
 ; RV64IM-NEXT:    srli a0, a0, 18
 ; RV64IM-NEXT:    ret
@@ -310,7 +295,7 @@ define i16 @udiv16_constant_no_add(i16 %a) nounwind {
 ; RV64IMZB:       # %bb.0:
 ; RV64IMZB-NEXT:    zext.h a0, a0
 ; RV64IMZB-NEXT:    lui a1, 13
-; RV64IMZB-NEXT:    addiw a1, a1, -819
+; RV64IMZB-NEXT:    addi a1, a1, -819
 ; RV64IMZB-NEXT:    mul a0, a0, a1
 ; RV64IMZB-NEXT:    srli a0, a0, 18
 ; RV64IMZB-NEXT:    ret
@@ -355,8 +340,8 @@ define i16 @udiv16_constant_add(i16 %a) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    lui a1, 2
 ; RV64IM-NEXT:    lui a2, 16
-; RV64IM-NEXT:    addiw a1, a1, 1171
-; RV64IM-NEXT:    addiw a2, a2, -1
+; RV64IM-NEXT:    addi a1, a1, 1171
+; RV64IM-NEXT:    addi a2, a2, -1
 ; RV64IM-NEXT:    and a3, a0, a2
 ; RV64IM-NEXT:    mul a1, a3, a1
 ; RV64IM-NEXT:    srli a1, a1, 16
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
index 05730a710b4d..88413291c26c 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
@@ -1002,7 +1002,7 @@ define i1 @fpclass(float %x) {
 ; RV64I-NEXT:    lui a4, 2048
 ; RV64I-NEXT:    lui a5, 520192
 ; RV64I-NEXT:    srli a2, a2, 33
-; RV64I-NEXT:    addiw a6, a4, -1
+; RV64I-NEXT:    addi a6, a4, -1
 ; RV64I-NEXT:    xor a0, a0, a2
 ; RV64I-NEXT:    subw a3, a2, a3
 ; RV64I-NEXT:    sltu a3, a3, a6
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/constant64.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/constant64.mir
index 646152e2e4ed..0f00bd0ced26 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/constant64.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/constant64.mir
@@ -159,8 +159,8 @@ body:            |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[LUI:%[0-9]+]]:gpr = LUI 524288
-    ; CHECK-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LUI]], 648
-    ; CHECK-NEXT: $x10 = COPY [[ADDIW]]
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], 648
+    ; CHECK-NEXT: $x10 = COPY [[ADDI]]
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:gprb(s64) = G_CONSTANT i64 -2147483000
     $x10 = COPY %0(s64)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant-f16.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant-f16.mir
index 8951e373ba7a..3028b6476e20 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant-f16.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant-f16.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
 # RUN: llc -mtriple=riscv32 -mattr=+zfh -run-pass=instruction-select \
-# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,RV32
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
 # RUN: llc -mtriple=riscv64 -mattr=+zfh -run-pass=instruction-select \
-# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,RV64
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 name:            half_imm
@@ -10,19 +10,12 @@ legalized:       true
 regBankSelected: true
 body:             |
   bb.1:
-    ; RV32-LABEL: name: half_imm
-    ; RV32: [[LUI:%[0-9]+]]:gpr = LUI 4
-    ; RV32-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], 584
-    ; RV32-NEXT: [[FMV_H_X:%[0-9]+]]:fpr16 = FMV_H_X [[ADDI]]
-    ; RV32-NEXT: $f10_h = COPY [[FMV_H_X]]
-    ; RV32-NEXT: PseudoRET implicit $f10_h
-    ;
-    ; RV64-LABEL: name: half_imm
-    ; RV64: [[LUI:%[0-9]+]]:gpr = LUI 4
-    ; RV64-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LUI]], 584
-    ; RV64-NEXT: [[FMV_H_X:%[0-9]+]]:fpr16 = FMV_H_X [[ADDIW]]
-    ; RV64-NEXT: $f10_h = COPY [[FMV_H_X]]
-    ; RV64-NEXT: PseudoRET implicit $f10_h
+    ; CHECK-LABEL: name: half_imm
+    ; CHECK: [[LUI:%[0-9]+]]:gpr = LUI 4
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], 584
+    ; CHECK-NEXT: [[FMV_H_X:%[0-9]+]]:fpr16 = FMV_H_X [[ADDI]]
+    ; CHECK-NEXT: $f10_h = COPY [[FMV_H_X]]
+    ; CHECK-NEXT: PseudoRET implicit $f10_h
     %0:fprb(s16) = G_FCONSTANT half 0xH4248
     $f10_h = COPY %0(s16)
     PseudoRET implicit $f10_h
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir
index 43f5ec1f5790..e82d4bcec48b 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir
@@ -10,19 +10,12 @@ legalized:       true
 regBankSelected: true
 body:             |
   bb.1:
-    ; RV32-LABEL: name: float_imm
-    ; RV32: [[LUI:%[0-9]+]]:gpr = LUI 263313
-    ; RV32-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], -37
-    ; RV32-NEXT: [[FMV_W_X:%[0-9]+]]:fpr32 = FMV_W_X [[ADDI]]
-    ; RV32-NEXT: $f10_f = COPY [[FMV_W_X]]
-    ; RV32-NEXT: PseudoRET implicit $f10_f
-    ;
-    ; RV64-LABEL: name: float_imm
-    ; RV64: [[LUI:%[0-9]+]]:gpr = LUI 263313
-    ; RV64-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LUI]], -37
-    ; RV64-NEXT: [[FMV_W_X:%[0-9]+]]:fpr32 = FMV_W_X [[ADDIW]]
-    ; RV64-NEXT: $f10_f = COPY [[FMV_W_X]]
-    ; RV64-NEXT: PseudoRET implicit $f10_f
+    ; CHECK-LABEL: name: float_imm
+    ; CHECK: [[LUI:%[0-9]+]]:gpr = LUI 263313
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], -37
+    ; CHECK-NEXT: [[FMV_W_X:%[0-9]+]]:fpr32 = FMV_W_X [[ADDI]]
+    ; CHECK-NEXT: $f10_f = COPY [[FMV_W_X]]
+    ; CHECK-NEXT: PseudoRET implicit $f10_f
     %0:fprb(s32) = G_FCONSTANT float 0x400921FB60000000
     $f10_f = COPY %0(s32)
     PseudoRET implicit $f10_f
@@ -109,14 +102,14 @@ body:             |
     ;
     ; RV64-LABEL: name: double_imm
     ; RV64: [[LUI:%[0-9]+]]:gpr = LUI 512
-    ; RV64-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LUI]], 1169
-    ; RV64-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[ADDIW]], 15
-    ; RV64-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[SLLI]], -299
-    ; RV64-NEXT: [[SLLI1:%[0-9]+]]:gpr = SLLI [[ADDI]], 14
-    ; RV64-NEXT: [[ADDI1:%[0-9]+]]:gpr = ADDI [[SLLI1]], 1091
-    ; RV64-NEXT: [[SLLI2:%[0-9]+]]:gpr = SLLI [[ADDI1]], 12
-    ; RV64-NEXT: [[ADDI2:%[0-9]+]]:gpr = ADDI [[SLLI2]], -744
-    ; RV64-NEXT: [[FMV_D_X:%[0-9]+]]:fpr64 = FMV_D_X [[ADDI2]]
+    ; RV64-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], 1169
+    ; RV64-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[ADDI]], 15
+    ; RV64-NEXT: [[ADDI1:%[0-9]+]]:gpr = ADDI [[SLLI]], -299
+    ; RV64-NEXT: [[SLLI1:%[0-9]+]]:gpr = SLLI [[ADDI1]], 14
+    ; RV64-NEXT: [[ADDI2:%[0-9]+]]:gpr = ADDI [[SLLI1]], 1091
+    ; RV64-NEXT: [[SLLI2:%[0-9]+]]:gpr = SLLI [[ADDI2]], 12
+    ; RV64-NEXT: [[ADDI3:%[0-9]+]]:gpr = ADDI [[SLLI2]], -744
+    ; RV64-NEXT: [[FMV_D_X:%[0-9]+]]:fpr64 = FMV_D_X [[ADDI3]]
     ; RV64-NEXT: $f10_d = COPY [[FMV_D_X]]
     ; RV64-NEXT: PseudoRET implicit $f10_d
     %0:fprb(s64) = G_FCONSTANT double 0x400921FB54442D18
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll
index 0632caecf890..0b376dd77988 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv32zbb.ll
@@ -385,7 +385,7 @@ define i1 @ctpop_i64_ugt_two(i64 %a) nounwind {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    j .LBB6_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    sltiu a0, zero, 0
+; RV32ZBB-NEXT:    li a0, 0
 ; RV32ZBB-NEXT:    ret
 ; RV32ZBB-NEXT:  .LBB6_2:
 ; RV32ZBB-NEXT:    cpop a0, a0
@@ -415,7 +415,7 @@ define i1 @ctpop_i64_ugt_one(i64 %a) nounwind {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    j .LBB7_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    snez a0, zero
+; RV32ZBB-NEXT:    li a0, 0
 ; RV32ZBB-NEXT:    ret
 ; RV32ZBB-NEXT:  .LBB7_2:
 ; RV32ZBB-NEXT:    cpop a0, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
index 8549a7c526e4..969030255209 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbb.ll
@@ -40,9 +40,9 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    li a1, 32
@@ -97,9 +97,9 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    li a1, 32
@@ -162,9 +162,9 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    li a1, 32
@@ -221,9 +221,9 @@ define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    beqz s0, .LBB3_2
 ; RV64I-NEXT:  # %bb.1:
@@ -292,9 +292,9 @@ define i32 @ctlz_lshr_i32(i32 signext %a) {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    li a1, 32
@@ -421,9 +421,9 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -464,9 +464,9 @@ define signext i32 @cttz_zero_undef_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -506,9 +506,9 @@ define signext i32 @findFirstSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    beqz s0, .LBB8_2
 ; RV64I-NEXT:  # %bb.1:
@@ -562,9 +562,9 @@ define signext i32 @ffs_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    mv a1, a0
 ; RV64I-NEXT:    li a0, 0
@@ -681,9 +681,9 @@ define signext i32 @ctpop_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -737,9 +737,9 @@ define signext i32 @ctpop_i32_load(ptr %p) nounwind {
 ; RV64I-NEXT:    sraiw a1, a0, 4
 ; RV64I-NEXT:    addw a0, a1, a0
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -1127,7 +1127,7 @@ define signext i32 @bswap_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    lui a3, 16
 ; RV64I-NEXT:    or a1, a2, a1
 ; RV64I-NEXT:    srliw a2, a0, 8
-; RV64I-NEXT:    addiw a3, a3, -256
+; RV64I-NEXT:    addi a3, a3, -256
 ; RV64I-NEXT:    and a0, a0, a3
 ; RV64I-NEXT:    and a2, a2, a3
 ; RV64I-NEXT:    slliw a0, a0, 8
@@ -1185,7 +1185,7 @@ define i64 @bswap_i64(i64 %a) {
 ; RV64I-NEXT:    srli a4, a0, 40
 ; RV64I-NEXT:    or a1, a2, a1
 ; RV64I-NEXT:    lui a2, 4080
-; RV64I-NEXT:    addiw a3, a3, -256
+; RV64I-NEXT:    addi a3, a3, -256
 ; RV64I-NEXT:    and a4, a4, a3
 ; RV64I-NEXT:    or a1, a1, a4
 ; RV64I-NEXT:    srli a4, a0, 24
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
index f413abffcdcc..cd59c9e01806 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/rv64zbkb.ll
@@ -141,7 +141,7 @@ define signext i32 @packh_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    zext.b a0, a0
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    slli a1, a1, 8
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a0, a1, a0
@@ -151,7 +151,7 @@ define signext i32 @packh_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64ZBKB:       # %bb.0:
 ; RV64ZBKB-NEXT:    lui a2, 16
 ; RV64ZBKB-NEXT:    zext.b a0, a0
-; RV64ZBKB-NEXT:    addiw a2, a2, -256
+; RV64ZBKB-NEXT:    addi a2, a2, -256
 ; RV64ZBKB-NEXT:    slli a1, a1, 8
 ; RV64ZBKB-NEXT:    and a1, a1, a2
 ; RV64ZBKB-NEXT:    or a0, a1, a0
@@ -189,7 +189,7 @@ define i64 @packh_i64(i64 %a, i64 %b) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    zext.b a0, a0
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    slli a1, a1, 8
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a0, a1, a0
@@ -199,7 +199,7 @@ define i64 @packh_i64(i64 %a, i64 %b) nounwind {
 ; RV64ZBKB:       # %bb.0:
 ; RV64ZBKB-NEXT:    lui a2, 16
 ; RV64ZBKB-NEXT:    zext.b a0, a0
-; RV64ZBKB-NEXT:    addiw a2, a2, -256
+; RV64ZBKB-NEXT:    addi a2, a2, -256
 ; RV64ZBKB-NEXT:    slli a1, a1, 8
 ; RV64ZBKB-NEXT:    and a1, a1, a2
 ; RV64ZBKB-NEXT:    or a0, a1, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
index fc9be9498845..afef96db5e29 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
@@ -1171,7 +1171,7 @@ define void @va3_caller() nounwind {
 ; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    lui a1, 5
 ; RV64-NEXT:    li a0, 2
-; RV64-NEXT:    addiw a2, a1, -480
+; RV64-NEXT:    addi a2, a1, -480
 ; RV64-NEXT:    li a1, 1111
 ; RV64-NEXT:    call va3
 ; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -1203,7 +1203,7 @@ define void @va3_caller() nounwind {
 ; RV64-WITHFP-NEXT:    addi s0, sp, 16
 ; RV64-WITHFP-NEXT:    lui a1, 5
 ; RV64-WITHFP-NEXT:    li a0, 2
-; RV64-WITHFP-NEXT:    addiw a2, a1, -480
+; RV64-WITHFP-NEXT:    addi a2, a1, -480
 ; RV64-WITHFP-NEXT:    li a1, 1111
 ; RV64-WITHFP-NEXT:    call va3
 ; RV64-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -1618,7 +1618,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-LABEL: va_large_stack:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a0, 24414
-; RV64-NEXT:    addiw a0, a0, 336
+; RV64-NEXT:    addi a0, a0, 336
 ; RV64-NEXT:    sub sp, sp, a0
 ; RV64-NEXT:    .cfi_def_cfa_offset 100000080
 ; RV64-NEXT:    lui a0, 24414
@@ -1635,7 +1635,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-NEXT:    sd a4, 304(a0)
 ; RV64-NEXT:    addi a0, sp, 8
 ; RV64-NEXT:    lui a1, 24414
-; RV64-NEXT:    addiw a1, a1, 280
+; RV64-NEXT:    addi a1, a1, 280
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    sd a1, 8(sp)
 ; RV64-NEXT:    lw a0, 4(a0)
@@ -1657,7 +1657,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-NEXT:    sw a2, 12(sp)
 ; RV64-NEXT:    lw a0, 0(a0)
 ; RV64-NEXT:    lui a1, 24414
-; RV64-NEXT:    addiw a1, a1, 336
+; RV64-NEXT:    addi a1, a1, 336
 ; RV64-NEXT:    add sp, sp, a1
 ; RV64-NEXT:    .cfi_def_cfa_offset 0
 ; RV64-NEXT:    ret
@@ -1714,10 +1714,10 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-WITHFP-NEXT:    addi s0, sp, 1968
 ; RV64-WITHFP-NEXT:    .cfi_def_cfa s0, 64
 ; RV64-WITHFP-NEXT:    lui a0, 24414
-; RV64-WITHFP-NEXT:    addiw a0, a0, -1680
+; RV64-WITHFP-NEXT:    addi a0, a0, -1680
 ; RV64-WITHFP-NEXT:    sub sp, sp, a0
 ; RV64-WITHFP-NEXT:    lui a0, 24414
-; RV64-WITHFP-NEXT:    addiw a0, a0, 288
+; RV64-WITHFP-NEXT:    addi a0, a0, 288
 ; RV64-WITHFP-NEXT:    sub a0, s0, a0
 ; RV64-WITHFP-NEXT:    sd a1, 8(s0)
 ; RV64-WITHFP-NEXT:    sd a2, 16(s0)
@@ -1738,7 +1738,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-WITHFP-NEXT:    sw a3, 4(a0)
 ; RV64-WITHFP-NEXT:    lw a0, 0(a1)
 ; RV64-WITHFP-NEXT:    lui a1, 24414
-; RV64-WITHFP-NEXT:    addiw a1, a1, -1680
+; RV64-WITHFP-NEXT:    addi a1, a1, -1680
 ; RV64-WITHFP-NEXT:    add sp, sp, a1
 ; RV64-WITHFP-NEXT:    .cfi_def_cfa sp, 2032
 ; RV64-WITHFP-NEXT:    ld ra, 1960(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/wide-scalar-shift-by-byte-multiple-legalization.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/wide-scalar-shift-by-byte-multiple-legalization.ll
index 5278c951ecdd..69519c00f88e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/GlobalISel/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -1481,38 +1481,38 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ;
 ; RV32I-LABEL: shl_16bytes:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
 ; RV32I-NEXT:    lbu a5, 2(a0)
 ; RV32I-NEXT:    lbu a6, 3(a0)
-; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 4(a0)
-; RV32I-NEXT:    lbu a7, 5(a0)
-; RV32I-NEXT:    lbu t0, 6(a0)
-; RV32I-NEXT:    lbu t1, 7(a0)
+; RV32I-NEXT:    lbu a7, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a3, a4, a3
 ; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    lbu a6, 0(a1)
-; RV32I-NEXT:    lbu a7, 1(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a7, t2, t1
+; RV32I-NEXT:    lbu a6, 1(a1)
+; RV32I-NEXT:    lbu t0, 0(a1)
 ; RV32I-NEXT:    lbu t1, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a7, a7, a6
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    or t2, a6, t0
 ; RV32I-NEXT:    li a6, 64
 ; RV32I-NEXT:    slli a1, a1, 8
 ; RV32I-NEXT:    or a1, a1, t1
 ; RV32I-NEXT:    li t1, 32
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli t2, t0, 16
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a7, a7, 16
 ; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    or t0, a5, a3
-; RV32I-NEXT:    or a4, t2, a4
-; RV32I-NEXT:    or a5, a1, a7
+; RV32I-NEXT:    or t0, a4, a3
+; RV32I-NEXT:    or a4, a7, a5
+; RV32I-NEXT:    or a5, a1, t2
 ; RV32I-NEXT:    slli a5, a5, 3
 ; RV32I-NEXT:    neg t3, a5
 ; RV32I-NEXT:    srl t4, t0, t3
@@ -1810,38 +1810,38 @@ define void @shl_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ;
 ; RV32I-LABEL: shl_16bytes_wordOff:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    lbu a4, 1(a0)
 ; RV32I-NEXT:    lbu a5, 2(a0)
 ; RV32I-NEXT:    lbu a6, 3(a0)
-; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 4(a0)
-; RV32I-NEXT:    lbu a7, 5(a0)
-; RV32I-NEXT:    lbu t0, 6(a0)
-; RV32I-NEXT:    lbu t1, 7(a0)
+; RV32I-NEXT:    lbu a7, 4(a0)
+; RV32I-NEXT:    lbu t0, 5(a0)
+; RV32I-NEXT:    lbu t1, 6(a0)
+; RV32I-NEXT:    lbu t2, 7(a0)
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a3, a4, a3
 ; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    lbu a6, 0(a1)
-; RV32I-NEXT:    lbu a7, 1(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a7, t2, t1
+; RV32I-NEXT:    lbu a6, 1(a1)
+; RV32I-NEXT:    lbu t0, 0(a1)
 ; RV32I-NEXT:    lbu t1, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a7, a7, a6
+; RV32I-NEXT:    slli a6, a6, 8
+; RV32I-NEXT:    or t2, a6, t0
 ; RV32I-NEXT:    li a6, 64
 ; RV32I-NEXT:    slli a1, a1, 8
 ; RV32I-NEXT:    or a1, a1, t1
 ; RV32I-NEXT:    li t1, 32
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli t2, t0, 16
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a7, a7, 16
 ; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    or t0, a5, a3
-; RV32I-NEXT:    or a4, t2, a4
-; RV32I-NEXT:    or a5, a1, a7
+; RV32I-NEXT:    or t0, a4, a3
+; RV32I-NEXT:    or a4, a7, a5
+; RV32I-NEXT:    or a5, a1, t2
 ; RV32I-NEXT:    slli a5, a5, 5
 ; RV32I-NEXT:    neg t3, a5
 ; RV32I-NEXT:    srl t4, t0, t3
@@ -5781,26 +5781,26 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu a3, 15(a0)
 ; RV32I-NEXT:    slli a7, a7, 8
 ; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    slli t3, t3, 8
 ; RV32I-NEXT:    or a5, a7, a5
 ; RV32I-NEXT:    or a7, t1, t0
-; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    or t0, t3, t2
 ; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    slli t3, t3, 8
-; RV32I-NEXT:    or t2, t3, t2
+; RV32I-NEXT:    lbu t2, 0(a1)
 ; RV32I-NEXT:    lbu t3, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
 ; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    or t1, t1, t2
 ; RV32I-NEXT:    li s9, 64
 ; RV32I-NEXT:    slli a1, a1, 8
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    li t4, 32
 ; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli t0, t0, 16
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    or t3, a5, a4
-; RV32I-NEXT:    or a5, t2, a7
-; RV32I-NEXT:    or a4, a1, t0
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a4, a1, t1
 ; RV32I-NEXT:    slli a4, a4, 3
 ; RV32I-NEXT:    neg s10, a4
 ; RV32I-NEXT:    srl t5, t3, s10
@@ -6695,26 +6695,26 @@ define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV32I-NEXT:    lbu a3, 15(a0)
 ; RV32I-NEXT:    slli a7, a7, 8
 ; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    slli t3, t3, 8
 ; RV32I-NEXT:    or a5, a7, a5
 ; RV32I-NEXT:    or a7, t1, t0
-; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    or t0, t3, t2
 ; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    slli t3, t3, 8
-; RV32I-NEXT:    or t2, t3, t2
+; RV32I-NEXT:    lbu t2, 0(a1)
 ; RV32I-NEXT:    lbu t3, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
 ; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    or t1, t1, t2
 ; RV32I-NEXT:    li s9, 64
 ; RV32I-NEXT:    slli a1, a1, 8
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    li t4, 32
 ; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli t0, t0, 16
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    or t3, a5, a4
-; RV32I-NEXT:    or a5, t2, a7
-; RV32I-NEXT:    or a4, a1, t0
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a4, a1, t1
 ; RV32I-NEXT:    slli a4, a4, 5
 ; RV32I-NEXT:    neg s10, a4
 ; RV32I-NEXT:    srl t5, t3, s10
@@ -7609,26 +7609,26 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
 ; RV32I-NEXT:    lbu a3, 15(a0)
 ; RV32I-NEXT:    slli a7, a7, 8
 ; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    slli t3, t3, 8
 ; RV32I-NEXT:    or a5, a7, a5
 ; RV32I-NEXT:    or a7, t1, t0
-; RV32I-NEXT:    lbu t0, 0(a1)
+; RV32I-NEXT:    or t0, t3, t2
 ; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    slli t3, t3, 8
-; RV32I-NEXT:    or t2, t3, t2
+; RV32I-NEXT:    lbu t2, 0(a1)
 ; RV32I-NEXT:    lbu t3, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
 ; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    or t1, t1, t2
 ; RV32I-NEXT:    li s9, 64
 ; RV32I-NEXT:    slli a1, a1, 8
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    li t4, 32
 ; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli t0, t0, 16
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    or t3, a5, a4
-; RV32I-NEXT:    or a5, t2, a7
-; RV32I-NEXT:    or a4, a1, t0
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a4, a1, t1
 ; RV32I-NEXT:    slli a4, a4, 6
 ; RV32I-NEXT:    neg s10, a4
 ; RV32I-NEXT:    srl t5, t3, s10
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/abds-neg.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/abds-neg.ll
index c9a48acb8d14..3fb0f2c53bdf 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/abds-neg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/abds-neg.ll
@@ -625,9 +625,9 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a4, 4(a1)
 ; RV32I-NEXT:    lw a6, 8(a1)
 ; RV32I-NEXT:    lw t1, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
 ; RV32I-NEXT:    lw t0, 8(a2)
 ; RV32I-NEXT:    lw t2, 12(a2)
-; RV32I-NEXT:    lw a1, 0(a2)
 ; RV32I-NEXT:    lw a2, 4(a2)
 ; RV32I-NEXT:    sltu t3, t0, a6
 ; RV32I-NEXT:    mv t4, t3
@@ -744,9 +744,9 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a4, 4(a1)
 ; RV32ZBB-NEXT:    lw a6, 8(a1)
 ; RV32ZBB-NEXT:    lw t1, 12(a1)
+; RV32ZBB-NEXT:    lw a1, 0(a2)
 ; RV32ZBB-NEXT:    lw t0, 8(a2)
 ; RV32ZBB-NEXT:    lw t2, 12(a2)
-; RV32ZBB-NEXT:    lw a1, 0(a2)
 ; RV32ZBB-NEXT:    lw a2, 4(a2)
 ; RV32ZBB-NEXT:    sltu t3, t0, a6
 ; RV32ZBB-NEXT:    mv t4, t3
@@ -872,9 +872,9 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a4, 4(a1)
 ; RV32I-NEXT:    lw a6, 8(a1)
 ; RV32I-NEXT:    lw t1, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
 ; RV32I-NEXT:    lw t0, 8(a2)
 ; RV32I-NEXT:    lw t2, 12(a2)
-; RV32I-NEXT:    lw a1, 0(a2)
 ; RV32I-NEXT:    lw a2, 4(a2)
 ; RV32I-NEXT:    sltu t3, t0, a6
 ; RV32I-NEXT:    mv t4, t3
@@ -991,9 +991,9 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a4, 4(a1)
 ; RV32ZBB-NEXT:    lw a6, 8(a1)
 ; RV32ZBB-NEXT:    lw t1, 12(a1)
+; RV32ZBB-NEXT:    lw a1, 0(a2)
 ; RV32ZBB-NEXT:    lw t0, 8(a2)
 ; RV32ZBB-NEXT:    lw t2, 12(a2)
-; RV32ZBB-NEXT:    lw a1, 0(a2)
 ; RV32ZBB-NEXT:    lw a2, 4(a2)
 ; RV32ZBB-NEXT:    sltu t3, t0, a6
 ; RV32ZBB-NEXT:    mv t4, t3
@@ -1385,8 +1385,8 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a6, 4(a2)
 ; RV32I-NEXT:    lw a7, 8(a2)
 ; RV32I-NEXT:    lw t0, 12(a2)
-; RV32I-NEXT:    lw a5, 12(a1)
 ; RV32I-NEXT:    lw a3, 4(a1)
+; RV32I-NEXT:    lw a5, 12(a1)
 ; RV32I-NEXT:    lw a4, 8(a1)
 ; RV32I-NEXT:    beq a5, t0, .LBB17_2
 ; RV32I-NEXT:  # %bb.1:
@@ -1512,8 +1512,8 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a6, 4(a2)
 ; RV32ZBB-NEXT:    lw a7, 8(a2)
 ; RV32ZBB-NEXT:    lw t0, 12(a2)
-; RV32ZBB-NEXT:    lw a5, 12(a1)
 ; RV32ZBB-NEXT:    lw a3, 4(a1)
+; RV32ZBB-NEXT:    lw a5, 12(a1)
 ; RV32ZBB-NEXT:    lw a4, 8(a1)
 ; RV32ZBB-NEXT:    beq a5, t0, .LBB17_2
 ; RV32ZBB-NEXT:  # %bb.1:
@@ -1864,15 +1864,15 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a4, 4(a2)
 ; RV32I-NEXT:    lw a5, 8(a2)
 ; RV32I-NEXT:    lw a7, 12(a2)
-; RV32I-NEXT:    lw a6, 8(a1)
-; RV32I-NEXT:    lw t0, 12(a1)
 ; RV32I-NEXT:    lw a2, 0(a1)
+; RV32I-NEXT:    lw a6, 8(a1)
+; RV32I-NEXT:    lw t1, 12(a1)
 ; RV32I-NEXT:    lw a1, 4(a1)
-; RV32I-NEXT:    sltu t1, a6, a5
-; RV32I-NEXT:    mv t4, t1
-; RV32I-NEXT:    beq t0, a7, .LBB22_2
+; RV32I-NEXT:    sltu t0, a6, a5
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beq t1, a7, .LBB22_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slt t4, t0, a7
+; RV32I-NEXT:    slt t4, t1, a7
 ; RV32I-NEXT:  .LBB22_2:
 ; RV32I-NEXT:    sltu t2, a2, a3
 ; RV32I-NEXT:    mv t3, t2
@@ -1880,7 +1880,7 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    sltu t3, a1, a4
 ; RV32I-NEXT:  .LBB22_4:
-; RV32I-NEXT:    xor t5, t0, a7
+; RV32I-NEXT:    xor t5, t1, a7
 ; RV32I-NEXT:    xor t6, a6, a5
 ; RV32I-NEXT:    or t5, t6, t5
 ; RV32I-NEXT:    mv t6, t3
@@ -1896,11 +1896,11 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:  .LBB22_8:
 ; RV32I-NEXT:    bnez t6, .LBB22_10
 ; RV32I-NEXT:  # %bb.9:
-; RV32I-NEXT:    sltu t1, a5, a6
-; RV32I-NEXT:    sub a7, a7, t0
+; RV32I-NEXT:    sltu t0, a5, a6
+; RV32I-NEXT:    sub a7, a7, t1
 ; RV32I-NEXT:    sub a5, a5, a6
 ; RV32I-NEXT:    sub a4, a4, a1
-; RV32I-NEXT:    sub a6, a7, t1
+; RV32I-NEXT:    sub a6, a7, t0
 ; RV32I-NEXT:    sltu a7, a5, t5
 ; RV32I-NEXT:    sub a1, a5, t5
 ; RV32I-NEXT:    sub a5, a4, t4
@@ -1908,10 +1908,10 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a2, a3, a2
 ; RV32I-NEXT:    j .LBB22_11
 ; RV32I-NEXT:  .LBB22_10:
-; RV32I-NEXT:    sub a7, t0, a7
+; RV32I-NEXT:    sub a7, t1, a7
 ; RV32I-NEXT:    sub a5, a6, a5
 ; RV32I-NEXT:    sub a4, a1, a4
-; RV32I-NEXT:    sub a6, a7, t1
+; RV32I-NEXT:    sub a6, a7, t0
 ; RV32I-NEXT:    sltu a7, a5, t3
 ; RV32I-NEXT:    sub a1, a5, t3
 ; RV32I-NEXT:    sub a5, a4, t2
@@ -1951,15 +1951,15 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a4, 4(a2)
 ; RV32ZBB-NEXT:    lw a5, 8(a2)
 ; RV32ZBB-NEXT:    lw a7, 12(a2)
-; RV32ZBB-NEXT:    lw a6, 8(a1)
-; RV32ZBB-NEXT:    lw t0, 12(a1)
 ; RV32ZBB-NEXT:    lw a2, 0(a1)
+; RV32ZBB-NEXT:    lw a6, 8(a1)
+; RV32ZBB-NEXT:    lw t1, 12(a1)
 ; RV32ZBB-NEXT:    lw a1, 4(a1)
-; RV32ZBB-NEXT:    sltu t1, a6, a5
-; RV32ZBB-NEXT:    mv t4, t1
-; RV32ZBB-NEXT:    beq t0, a7, .LBB22_2
+; RV32ZBB-NEXT:    sltu t0, a6, a5
+; RV32ZBB-NEXT:    mv t4, t0
+; RV32ZBB-NEXT:    beq t1, a7, .LBB22_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    slt t4, t0, a7
+; RV32ZBB-NEXT:    slt t4, t1, a7
 ; RV32ZBB-NEXT:  .LBB22_2:
 ; RV32ZBB-NEXT:    sltu t2, a2, a3
 ; RV32ZBB-NEXT:    mv t3, t2
@@ -1967,7 +1967,7 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:  # %bb.3:
 ; RV32ZBB-NEXT:    sltu t3, a1, a4
 ; RV32ZBB-NEXT:  .LBB22_4:
-; RV32ZBB-NEXT:    xor t5, t0, a7
+; RV32ZBB-NEXT:    xor t5, t1, a7
 ; RV32ZBB-NEXT:    xor t6, a6, a5
 ; RV32ZBB-NEXT:    or t5, t6, t5
 ; RV32ZBB-NEXT:    mv t6, t3
@@ -1983,11 +1983,11 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:  .LBB22_8:
 ; RV32ZBB-NEXT:    bnez t6, .LBB22_10
 ; RV32ZBB-NEXT:  # %bb.9:
-; RV32ZBB-NEXT:    sltu t1, a5, a6
-; RV32ZBB-NEXT:    sub a7, a7, t0
+; RV32ZBB-NEXT:    sltu t0, a5, a6
+; RV32ZBB-NEXT:    sub a7, a7, t1
 ; RV32ZBB-NEXT:    sub a5, a5, a6
 ; RV32ZBB-NEXT:    sub a4, a4, a1
-; RV32ZBB-NEXT:    sub a6, a7, t1
+; RV32ZBB-NEXT:    sub a6, a7, t0
 ; RV32ZBB-NEXT:    sltu a7, a5, t5
 ; RV32ZBB-NEXT:    sub a1, a5, t5
 ; RV32ZBB-NEXT:    sub a5, a4, t4
@@ -1995,10 +1995,10 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a2, a3, a2
 ; RV32ZBB-NEXT:    j .LBB22_11
 ; RV32ZBB-NEXT:  .LBB22_10:
-; RV32ZBB-NEXT:    sub a7, t0, a7
+; RV32ZBB-NEXT:    sub a7, t1, a7
 ; RV32ZBB-NEXT:    sub a5, a6, a5
 ; RV32ZBB-NEXT:    sub a4, a1, a4
-; RV32ZBB-NEXT:    sub a6, a7, t1
+; RV32ZBB-NEXT:    sub a6, a7, t0
 ; RV32ZBB-NEXT:    sltu a7, a5, t3
 ; RV32ZBB-NEXT:    sub a1, a5, t3
 ; RV32ZBB-NEXT:    sub a5, a4, t2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/abds.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/abds.ll
index 56e6dacff974..efb4e1a6f15d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/abds.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/abds.ll
@@ -536,73 +536,73 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    lw a6, 8(a1)
-; RV32I-NEXT:    lw t0, 12(a1)
-; RV32I-NEXT:    lw a7, 8(a2)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a7, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
+; RV32I-NEXT:    lw a6, 8(a2)
 ; RV32I-NEXT:    lw t1, 12(a2)
-; RV32I-NEXT:    lw a5, 0(a2)
-; RV32I-NEXT:    lw a1, 4(a2)
-; RV32I-NEXT:    sltu a2, a7, a6
-; RV32I-NEXT:    mv t4, a2
-; RV32I-NEXT:    beq t0, t1, .LBB11_2
+; RV32I-NEXT:    lw a2, 4(a2)
+; RV32I-NEXT:    sltu t0, a6, a5
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beq a7, t1, .LBB11_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slt t4, t1, t0
+; RV32I-NEXT:    slt t4, t1, a7
 ; RV32I-NEXT:  .LBB11_2:
-; RV32I-NEXT:    sltu t2, a5, a3
-; RV32I-NEXT:    sltu t5, a1, a4
+; RV32I-NEXT:    sltu t2, a1, a3
+; RV32I-NEXT:    sltu t5, a2, a4
 ; RV32I-NEXT:    mv t3, t2
-; RV32I-NEXT:    beq a4, a1, .LBB11_4
+; RV32I-NEXT:    beq a4, a2, .LBB11_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    mv t3, t5
 ; RV32I-NEXT:  .LBB11_4:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    xor t6, t0, t1
-; RV32I-NEXT:    xor s0, a6, a7
+; RV32I-NEXT:    xor t6, a7, t1
+; RV32I-NEXT:    xor s0, a5, a6
 ; RV32I-NEXT:    or t6, s0, t6
 ; RV32I-NEXT:    beqz t6, .LBB11_6
 ; RV32I-NEXT:  # %bb.5:
 ; RV32I-NEXT:    mv t3, t4
 ; RV32I-NEXT:  .LBB11_6:
 ; RV32I-NEXT:    mv t4, t2
-; RV32I-NEXT:    beq a1, a4, .LBB11_8
+; RV32I-NEXT:    beq a2, a4, .LBB11_8
 ; RV32I-NEXT:  # %bb.7:
 ; RV32I-NEXT:    mv t4, t5
 ; RV32I-NEXT:  .LBB11_8:
-; RV32I-NEXT:    sltu t5, a3, a5
+; RV32I-NEXT:    sltu t5, a3, a1
 ; RV32I-NEXT:    mv t6, t5
-; RV32I-NEXT:    beq a4, a1, .LBB11_10
+; RV32I-NEXT:    beq a4, a2, .LBB11_10
 ; RV32I-NEXT:  # %bb.9:
-; RV32I-NEXT:    sltu t6, a4, a1
+; RV32I-NEXT:    sltu t6, a4, a2
 ; RV32I-NEXT:  .LBB11_10:
 ; RV32I-NEXT:    bnez t3, .LBB11_12
 ; RV32I-NEXT:  # %bb.11:
-; RV32I-NEXT:    sub t0, t1, t0
-; RV32I-NEXT:    sub a6, a7, a6
-; RV32I-NEXT:    sub a3, a5, a3
-; RV32I-NEXT:    sub a1, a1, a4
-; RV32I-NEXT:    sub a4, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t4
-; RV32I-NEXT:    sub a2, a1, t2
-; RV32I-NEXT:    sub a1, a4, a5
-; RV32I-NEXT:    sub a4, a6, t4
+; RV32I-NEXT:    sub a7, t1, a7
+; RV32I-NEXT:    sub a5, a6, a5
+; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a2, a2, a4
+; RV32I-NEXT:    sub a4, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t4
+; RV32I-NEXT:    sub a3, a2, t2
+; RV32I-NEXT:    sub a2, a4, a6
+; RV32I-NEXT:    sub a4, a5, t4
 ; RV32I-NEXT:    j .LBB11_13
 ; RV32I-NEXT:  .LBB11_12:
-; RV32I-NEXT:    sltu a2, a6, a7
-; RV32I-NEXT:    sub t0, t0, t1
-; RV32I-NEXT:    sub a6, a6, a7
-; RV32I-NEXT:    sub a3, a3, a5
-; RV32I-NEXT:    sub a4, a4, a1
-; RV32I-NEXT:    sub a1, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t6
-; RV32I-NEXT:    sub a2, a4, t5
-; RV32I-NEXT:    sub a1, a1, a5
-; RV32I-NEXT:    sub a4, a6, t6
+; RV32I-NEXT:    sltu t0, a5, a6
+; RV32I-NEXT:    sub a7, a7, t1
+; RV32I-NEXT:    sub a5, a5, a6
+; RV32I-NEXT:    sub a1, a3, a1
+; RV32I-NEXT:    sub a4, a4, a2
+; RV32I-NEXT:    sub a2, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t6
+; RV32I-NEXT:    sub a3, a4, t5
+; RV32I-NEXT:    sub a2, a2, a6
+; RV32I-NEXT:    sub a4, a5, t6
 ; RV32I-NEXT:  .LBB11_13:
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    sw a2, 4(a0)
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 12(a0)
+; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -632,73 +632,73 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    lw a3, 0(a1)
 ; RV32ZBB-NEXT:    lw a4, 4(a1)
-; RV32ZBB-NEXT:    lw a6, 8(a1)
-; RV32ZBB-NEXT:    lw t0, 12(a1)
-; RV32ZBB-NEXT:    lw a7, 8(a2)
+; RV32ZBB-NEXT:    lw a5, 8(a1)
+; RV32ZBB-NEXT:    lw a7, 12(a1)
+; RV32ZBB-NEXT:    lw a1, 0(a2)
+; RV32ZBB-NEXT:    lw a6, 8(a2)
 ; RV32ZBB-NEXT:    lw t1, 12(a2)
-; RV32ZBB-NEXT:    lw a5, 0(a2)
-; RV32ZBB-NEXT:    lw a1, 4(a2)
-; RV32ZBB-NEXT:    sltu a2, a7, a6
-; RV32ZBB-NEXT:    mv t4, a2
-; RV32ZBB-NEXT:    beq t0, t1, .LBB11_2
+; RV32ZBB-NEXT:    lw a2, 4(a2)
+; RV32ZBB-NEXT:    sltu t0, a6, a5
+; RV32ZBB-NEXT:    mv t4, t0
+; RV32ZBB-NEXT:    beq a7, t1, .LBB11_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    slt t4, t1, t0
+; RV32ZBB-NEXT:    slt t4, t1, a7
 ; RV32ZBB-NEXT:  .LBB11_2:
-; RV32ZBB-NEXT:    sltu t2, a5, a3
-; RV32ZBB-NEXT:    sltu t5, a1, a4
+; RV32ZBB-NEXT:    sltu t2, a1, a3
+; RV32ZBB-NEXT:    sltu t5, a2, a4
 ; RV32ZBB-NEXT:    mv t3, t2
-; RV32ZBB-NEXT:    beq a4, a1, .LBB11_4
+; RV32ZBB-NEXT:    beq a4, a2, .LBB11_4
 ; RV32ZBB-NEXT:  # %bb.3:
 ; RV32ZBB-NEXT:    mv t3, t5
 ; RV32ZBB-NEXT:  .LBB11_4:
 ; RV32ZBB-NEXT:    addi sp, sp, -16
 ; RV32ZBB-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32ZBB-NEXT:    xor t6, t0, t1
-; RV32ZBB-NEXT:    xor s0, a6, a7
+; RV32ZBB-NEXT:    xor t6, a7, t1
+; RV32ZBB-NEXT:    xor s0, a5, a6
 ; RV32ZBB-NEXT:    or t6, s0, t6
 ; RV32ZBB-NEXT:    beqz t6, .LBB11_6
 ; RV32ZBB-NEXT:  # %bb.5:
 ; RV32ZBB-NEXT:    mv t3, t4
 ; RV32ZBB-NEXT:  .LBB11_6:
 ; RV32ZBB-NEXT:    mv t4, t2
-; RV32ZBB-NEXT:    beq a1, a4, .LBB11_8
+; RV32ZBB-NEXT:    beq a2, a4, .LBB11_8
 ; RV32ZBB-NEXT:  # %bb.7:
 ; RV32ZBB-NEXT:    mv t4, t5
 ; RV32ZBB-NEXT:  .LBB11_8:
-; RV32ZBB-NEXT:    sltu t5, a3, a5
+; RV32ZBB-NEXT:    sltu t5, a3, a1
 ; RV32ZBB-NEXT:    mv t6, t5
-; RV32ZBB-NEXT:    beq a4, a1, .LBB11_10
+; RV32ZBB-NEXT:    beq a4, a2, .LBB11_10
 ; RV32ZBB-NEXT:  # %bb.9:
-; RV32ZBB-NEXT:    sltu t6, a4, a1
+; RV32ZBB-NEXT:    sltu t6, a4, a2
 ; RV32ZBB-NEXT:  .LBB11_10:
 ; RV32ZBB-NEXT:    bnez t3, .LBB11_12
 ; RV32ZBB-NEXT:  # %bb.11:
-; RV32ZBB-NEXT:    sub t0, t1, t0
-; RV32ZBB-NEXT:    sub a6, a7, a6
-; RV32ZBB-NEXT:    sub a3, a5, a3
-; RV32ZBB-NEXT:    sub a1, a1, a4
-; RV32ZBB-NEXT:    sub a4, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t4
-; RV32ZBB-NEXT:    sub a2, a1, t2
-; RV32ZBB-NEXT:    sub a1, a4, a5
-; RV32ZBB-NEXT:    sub a4, a6, t4
+; RV32ZBB-NEXT:    sub a7, t1, a7
+; RV32ZBB-NEXT:    sub a5, a6, a5
+; RV32ZBB-NEXT:    sub a1, a1, a3
+; RV32ZBB-NEXT:    sub a2, a2, a4
+; RV32ZBB-NEXT:    sub a4, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t4
+; RV32ZBB-NEXT:    sub a3, a2, t2
+; RV32ZBB-NEXT:    sub a2, a4, a6
+; RV32ZBB-NEXT:    sub a4, a5, t4
 ; RV32ZBB-NEXT:    j .LBB11_13
 ; RV32ZBB-NEXT:  .LBB11_12:
-; RV32ZBB-NEXT:    sltu a2, a6, a7
-; RV32ZBB-NEXT:    sub t0, t0, t1
-; RV32ZBB-NEXT:    sub a6, a6, a7
-; RV32ZBB-NEXT:    sub a3, a3, a5
-; RV32ZBB-NEXT:    sub a4, a4, a1
-; RV32ZBB-NEXT:    sub a1, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t6
-; RV32ZBB-NEXT:    sub a2, a4, t5
-; RV32ZBB-NEXT:    sub a1, a1, a5
-; RV32ZBB-NEXT:    sub a4, a6, t6
+; RV32ZBB-NEXT:    sltu t0, a5, a6
+; RV32ZBB-NEXT:    sub a7, a7, t1
+; RV32ZBB-NEXT:    sub a5, a5, a6
+; RV32ZBB-NEXT:    sub a1, a3, a1
+; RV32ZBB-NEXT:    sub a4, a4, a2
+; RV32ZBB-NEXT:    sub a2, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t6
+; RV32ZBB-NEXT:    sub a3, a4, t5
+; RV32ZBB-NEXT:    sub a2, a2, a6
+; RV32ZBB-NEXT:    sub a4, a5, t6
 ; RV32ZBB-NEXT:  .LBB11_13:
-; RV32ZBB-NEXT:    sw a3, 0(a0)
-; RV32ZBB-NEXT:    sw a2, 4(a0)
+; RV32ZBB-NEXT:    sw a1, 0(a0)
+; RV32ZBB-NEXT:    sw a3, 4(a0)
 ; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 12(a0)
+; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
 ; RV32ZBB-NEXT:    ret
@@ -736,73 +736,73 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    lw a6, 8(a1)
-; RV32I-NEXT:    lw t0, 12(a1)
-; RV32I-NEXT:    lw a7, 8(a2)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a7, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
+; RV32I-NEXT:    lw a6, 8(a2)
 ; RV32I-NEXT:    lw t1, 12(a2)
-; RV32I-NEXT:    lw a5, 0(a2)
-; RV32I-NEXT:    lw a1, 4(a2)
-; RV32I-NEXT:    sltu a2, a7, a6
-; RV32I-NEXT:    mv t4, a2
-; RV32I-NEXT:    beq t0, t1, .LBB12_2
+; RV32I-NEXT:    lw a2, 4(a2)
+; RV32I-NEXT:    sltu t0, a6, a5
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beq a7, t1, .LBB12_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slt t4, t1, t0
+; RV32I-NEXT:    slt t4, t1, a7
 ; RV32I-NEXT:  .LBB12_2:
-; RV32I-NEXT:    sltu t2, a5, a3
-; RV32I-NEXT:    sltu t5, a1, a4
+; RV32I-NEXT:    sltu t2, a1, a3
+; RV32I-NEXT:    sltu t5, a2, a4
 ; RV32I-NEXT:    mv t3, t2
-; RV32I-NEXT:    beq a4, a1, .LBB12_4
+; RV32I-NEXT:    beq a4, a2, .LBB12_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    mv t3, t5
 ; RV32I-NEXT:  .LBB12_4:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    xor t6, t0, t1
-; RV32I-NEXT:    xor s0, a6, a7
+; RV32I-NEXT:    xor t6, a7, t1
+; RV32I-NEXT:    xor s0, a5, a6
 ; RV32I-NEXT:    or t6, s0, t6
 ; RV32I-NEXT:    beqz t6, .LBB12_6
 ; RV32I-NEXT:  # %bb.5:
 ; RV32I-NEXT:    mv t3, t4
 ; RV32I-NEXT:  .LBB12_6:
 ; RV32I-NEXT:    mv t4, t2
-; RV32I-NEXT:    beq a1, a4, .LBB12_8
+; RV32I-NEXT:    beq a2, a4, .LBB12_8
 ; RV32I-NEXT:  # %bb.7:
 ; RV32I-NEXT:    mv t4, t5
 ; RV32I-NEXT:  .LBB12_8:
-; RV32I-NEXT:    sltu t5, a3, a5
+; RV32I-NEXT:    sltu t5, a3, a1
 ; RV32I-NEXT:    mv t6, t5
-; RV32I-NEXT:    beq a4, a1, .LBB12_10
+; RV32I-NEXT:    beq a4, a2, .LBB12_10
 ; RV32I-NEXT:  # %bb.9:
-; RV32I-NEXT:    sltu t6, a4, a1
+; RV32I-NEXT:    sltu t6, a4, a2
 ; RV32I-NEXT:  .LBB12_10:
 ; RV32I-NEXT:    bnez t3, .LBB12_12
 ; RV32I-NEXT:  # %bb.11:
-; RV32I-NEXT:    sub t0, t1, t0
-; RV32I-NEXT:    sub a6, a7, a6
-; RV32I-NEXT:    sub a3, a5, a3
-; RV32I-NEXT:    sub a1, a1, a4
-; RV32I-NEXT:    sub a4, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t4
-; RV32I-NEXT:    sub a2, a1, t2
-; RV32I-NEXT:    sub a1, a4, a5
-; RV32I-NEXT:    sub a4, a6, t4
+; RV32I-NEXT:    sub a7, t1, a7
+; RV32I-NEXT:    sub a5, a6, a5
+; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a2, a2, a4
+; RV32I-NEXT:    sub a4, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t4
+; RV32I-NEXT:    sub a3, a2, t2
+; RV32I-NEXT:    sub a2, a4, a6
+; RV32I-NEXT:    sub a4, a5, t4
 ; RV32I-NEXT:    j .LBB12_13
 ; RV32I-NEXT:  .LBB12_12:
-; RV32I-NEXT:    sltu a2, a6, a7
-; RV32I-NEXT:    sub t0, t0, t1
-; RV32I-NEXT:    sub a6, a6, a7
-; RV32I-NEXT:    sub a3, a3, a5
-; RV32I-NEXT:    sub a4, a4, a1
-; RV32I-NEXT:    sub a1, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t6
-; RV32I-NEXT:    sub a2, a4, t5
-; RV32I-NEXT:    sub a1, a1, a5
-; RV32I-NEXT:    sub a4, a6, t6
+; RV32I-NEXT:    sltu t0, a5, a6
+; RV32I-NEXT:    sub a7, a7, t1
+; RV32I-NEXT:    sub a5, a5, a6
+; RV32I-NEXT:    sub a1, a3, a1
+; RV32I-NEXT:    sub a4, a4, a2
+; RV32I-NEXT:    sub a2, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t6
+; RV32I-NEXT:    sub a3, a4, t5
+; RV32I-NEXT:    sub a2, a2, a6
+; RV32I-NEXT:    sub a4, a5, t6
 ; RV32I-NEXT:  .LBB12_13:
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    sw a2, 4(a0)
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 12(a0)
+; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -832,73 +832,73 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    lw a3, 0(a1)
 ; RV32ZBB-NEXT:    lw a4, 4(a1)
-; RV32ZBB-NEXT:    lw a6, 8(a1)
-; RV32ZBB-NEXT:    lw t0, 12(a1)
-; RV32ZBB-NEXT:    lw a7, 8(a2)
+; RV32ZBB-NEXT:    lw a5, 8(a1)
+; RV32ZBB-NEXT:    lw a7, 12(a1)
+; RV32ZBB-NEXT:    lw a1, 0(a2)
+; RV32ZBB-NEXT:    lw a6, 8(a2)
 ; RV32ZBB-NEXT:    lw t1, 12(a2)
-; RV32ZBB-NEXT:    lw a5, 0(a2)
-; RV32ZBB-NEXT:    lw a1, 4(a2)
-; RV32ZBB-NEXT:    sltu a2, a7, a6
-; RV32ZBB-NEXT:    mv t4, a2
-; RV32ZBB-NEXT:    beq t0, t1, .LBB12_2
+; RV32ZBB-NEXT:    lw a2, 4(a2)
+; RV32ZBB-NEXT:    sltu t0, a6, a5
+; RV32ZBB-NEXT:    mv t4, t0
+; RV32ZBB-NEXT:    beq a7, t1, .LBB12_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    slt t4, t1, t0
+; RV32ZBB-NEXT:    slt t4, t1, a7
 ; RV32ZBB-NEXT:  .LBB12_2:
-; RV32ZBB-NEXT:    sltu t2, a5, a3
-; RV32ZBB-NEXT:    sltu t5, a1, a4
+; RV32ZBB-NEXT:    sltu t2, a1, a3
+; RV32ZBB-NEXT:    sltu t5, a2, a4
 ; RV32ZBB-NEXT:    mv t3, t2
-; RV32ZBB-NEXT:    beq a4, a1, .LBB12_4
+; RV32ZBB-NEXT:    beq a4, a2, .LBB12_4
 ; RV32ZBB-NEXT:  # %bb.3:
 ; RV32ZBB-NEXT:    mv t3, t5
 ; RV32ZBB-NEXT:  .LBB12_4:
 ; RV32ZBB-NEXT:    addi sp, sp, -16
 ; RV32ZBB-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32ZBB-NEXT:    xor t6, t0, t1
-; RV32ZBB-NEXT:    xor s0, a6, a7
+; RV32ZBB-NEXT:    xor t6, a7, t1
+; RV32ZBB-NEXT:    xor s0, a5, a6
 ; RV32ZBB-NEXT:    or t6, s0, t6
 ; RV32ZBB-NEXT:    beqz t6, .LBB12_6
 ; RV32ZBB-NEXT:  # %bb.5:
 ; RV32ZBB-NEXT:    mv t3, t4
 ; RV32ZBB-NEXT:  .LBB12_6:
 ; RV32ZBB-NEXT:    mv t4, t2
-; RV32ZBB-NEXT:    beq a1, a4, .LBB12_8
+; RV32ZBB-NEXT:    beq a2, a4, .LBB12_8
 ; RV32ZBB-NEXT:  # %bb.7:
 ; RV32ZBB-NEXT:    mv t4, t5
 ; RV32ZBB-NEXT:  .LBB12_8:
-; RV32ZBB-NEXT:    sltu t5, a3, a5
+; RV32ZBB-NEXT:    sltu t5, a3, a1
 ; RV32ZBB-NEXT:    mv t6, t5
-; RV32ZBB-NEXT:    beq a4, a1, .LBB12_10
+; RV32ZBB-NEXT:    beq a4, a2, .LBB12_10
 ; RV32ZBB-NEXT:  # %bb.9:
-; RV32ZBB-NEXT:    sltu t6, a4, a1
+; RV32ZBB-NEXT:    sltu t6, a4, a2
 ; RV32ZBB-NEXT:  .LBB12_10:
 ; RV32ZBB-NEXT:    bnez t3, .LBB12_12
 ; RV32ZBB-NEXT:  # %bb.11:
-; RV32ZBB-NEXT:    sub t0, t1, t0
-; RV32ZBB-NEXT:    sub a6, a7, a6
-; RV32ZBB-NEXT:    sub a3, a5, a3
-; RV32ZBB-NEXT:    sub a1, a1, a4
-; RV32ZBB-NEXT:    sub a4, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t4
-; RV32ZBB-NEXT:    sub a2, a1, t2
-; RV32ZBB-NEXT:    sub a1, a4, a5
-; RV32ZBB-NEXT:    sub a4, a6, t4
+; RV32ZBB-NEXT:    sub a7, t1, a7
+; RV32ZBB-NEXT:    sub a5, a6, a5
+; RV32ZBB-NEXT:    sub a1, a1, a3
+; RV32ZBB-NEXT:    sub a2, a2, a4
+; RV32ZBB-NEXT:    sub a4, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t4
+; RV32ZBB-NEXT:    sub a3, a2, t2
+; RV32ZBB-NEXT:    sub a2, a4, a6
+; RV32ZBB-NEXT:    sub a4, a5, t4
 ; RV32ZBB-NEXT:    j .LBB12_13
 ; RV32ZBB-NEXT:  .LBB12_12:
-; RV32ZBB-NEXT:    sltu a2, a6, a7
-; RV32ZBB-NEXT:    sub t0, t0, t1
-; RV32ZBB-NEXT:    sub a6, a6, a7
-; RV32ZBB-NEXT:    sub a3, a3, a5
-; RV32ZBB-NEXT:    sub a4, a4, a1
-; RV32ZBB-NEXT:    sub a1, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t6
-; RV32ZBB-NEXT:    sub a2, a4, t5
-; RV32ZBB-NEXT:    sub a1, a1, a5
-; RV32ZBB-NEXT:    sub a4, a6, t6
+; RV32ZBB-NEXT:    sltu t0, a5, a6
+; RV32ZBB-NEXT:    sub a7, a7, t1
+; RV32ZBB-NEXT:    sub a5, a5, a6
+; RV32ZBB-NEXT:    sub a1, a3, a1
+; RV32ZBB-NEXT:    sub a4, a4, a2
+; RV32ZBB-NEXT:    sub a2, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t6
+; RV32ZBB-NEXT:    sub a3, a4, t5
+; RV32ZBB-NEXT:    sub a2, a2, a6
+; RV32ZBB-NEXT:    sub a4, a5, t6
 ; RV32ZBB-NEXT:  .LBB12_13:
-; RV32ZBB-NEXT:    sw a3, 0(a0)
-; RV32ZBB-NEXT:    sw a2, 4(a0)
+; RV32ZBB-NEXT:    sw a1, 0(a0)
+; RV32ZBB-NEXT:    sw a3, 4(a0)
 ; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 12(a0)
+; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
 ; RV32ZBB-NEXT:    ret
@@ -1125,73 +1125,73 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    lw a6, 8(a1)
-; RV32I-NEXT:    lw t0, 12(a1)
-; RV32I-NEXT:    lw a7, 8(a2)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a7, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
+; RV32I-NEXT:    lw a6, 8(a2)
 ; RV32I-NEXT:    lw t1, 12(a2)
-; RV32I-NEXT:    lw a5, 0(a2)
-; RV32I-NEXT:    lw a1, 4(a2)
-; RV32I-NEXT:    sltu a2, a7, a6
-; RV32I-NEXT:    mv t4, a2
-; RV32I-NEXT:    beq t0, t1, .LBB17_2
+; RV32I-NEXT:    lw a2, 4(a2)
+; RV32I-NEXT:    sltu t0, a6, a5
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beq a7, t1, .LBB17_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slt t4, t1, t0
+; RV32I-NEXT:    slt t4, t1, a7
 ; RV32I-NEXT:  .LBB17_2:
-; RV32I-NEXT:    sltu t2, a5, a3
-; RV32I-NEXT:    sltu t5, a1, a4
+; RV32I-NEXT:    sltu t2, a1, a3
+; RV32I-NEXT:    sltu t5, a2, a4
 ; RV32I-NEXT:    mv t3, t2
-; RV32I-NEXT:    beq a4, a1, .LBB17_4
+; RV32I-NEXT:    beq a4, a2, .LBB17_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    mv t3, t5
 ; RV32I-NEXT:  .LBB17_4:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    xor t6, t0, t1
-; RV32I-NEXT:    xor s0, a6, a7
+; RV32I-NEXT:    xor t6, a7, t1
+; RV32I-NEXT:    xor s0, a5, a6
 ; RV32I-NEXT:    or t6, s0, t6
 ; RV32I-NEXT:    beqz t6, .LBB17_6
 ; RV32I-NEXT:  # %bb.5:
 ; RV32I-NEXT:    mv t3, t4
 ; RV32I-NEXT:  .LBB17_6:
 ; RV32I-NEXT:    mv t4, t2
-; RV32I-NEXT:    beq a1, a4, .LBB17_8
+; RV32I-NEXT:    beq a2, a4, .LBB17_8
 ; RV32I-NEXT:  # %bb.7:
 ; RV32I-NEXT:    mv t4, t5
 ; RV32I-NEXT:  .LBB17_8:
-; RV32I-NEXT:    sltu t5, a3, a5
+; RV32I-NEXT:    sltu t5, a3, a1
 ; RV32I-NEXT:    mv t6, t5
-; RV32I-NEXT:    beq a4, a1, .LBB17_10
+; RV32I-NEXT:    beq a4, a2, .LBB17_10
 ; RV32I-NEXT:  # %bb.9:
-; RV32I-NEXT:    sltu t6, a4, a1
+; RV32I-NEXT:    sltu t6, a4, a2
 ; RV32I-NEXT:  .LBB17_10:
 ; RV32I-NEXT:    bnez t3, .LBB17_12
 ; RV32I-NEXT:  # %bb.11:
-; RV32I-NEXT:    sub t0, t1, t0
-; RV32I-NEXT:    sub a6, a7, a6
-; RV32I-NEXT:    sub a3, a5, a3
-; RV32I-NEXT:    sub a1, a1, a4
-; RV32I-NEXT:    sub a4, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t4
-; RV32I-NEXT:    sub a2, a1, t2
-; RV32I-NEXT:    sub a1, a4, a5
-; RV32I-NEXT:    sub a4, a6, t4
+; RV32I-NEXT:    sub a7, t1, a7
+; RV32I-NEXT:    sub a5, a6, a5
+; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a2, a2, a4
+; RV32I-NEXT:    sub a4, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t4
+; RV32I-NEXT:    sub a3, a2, t2
+; RV32I-NEXT:    sub a2, a4, a6
+; RV32I-NEXT:    sub a4, a5, t4
 ; RV32I-NEXT:    j .LBB17_13
 ; RV32I-NEXT:  .LBB17_12:
-; RV32I-NEXT:    sltu a2, a6, a7
-; RV32I-NEXT:    sub t0, t0, t1
-; RV32I-NEXT:    sub a6, a6, a7
-; RV32I-NEXT:    sub a3, a3, a5
-; RV32I-NEXT:    sub a4, a4, a1
-; RV32I-NEXT:    sub a1, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t6
-; RV32I-NEXT:    sub a2, a4, t5
-; RV32I-NEXT:    sub a1, a1, a5
-; RV32I-NEXT:    sub a4, a6, t6
+; RV32I-NEXT:    sltu t0, a5, a6
+; RV32I-NEXT:    sub a7, a7, t1
+; RV32I-NEXT:    sub a5, a5, a6
+; RV32I-NEXT:    sub a1, a3, a1
+; RV32I-NEXT:    sub a4, a4, a2
+; RV32I-NEXT:    sub a2, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t6
+; RV32I-NEXT:    sub a3, a4, t5
+; RV32I-NEXT:    sub a2, a2, a6
+; RV32I-NEXT:    sub a4, a5, t6
 ; RV32I-NEXT:  .LBB17_13:
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    sw a2, 4(a0)
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 12(a0)
+; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -1221,73 +1221,73 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    lw a3, 0(a1)
 ; RV32ZBB-NEXT:    lw a4, 4(a1)
-; RV32ZBB-NEXT:    lw a6, 8(a1)
-; RV32ZBB-NEXT:    lw t0, 12(a1)
-; RV32ZBB-NEXT:    lw a7, 8(a2)
+; RV32ZBB-NEXT:    lw a5, 8(a1)
+; RV32ZBB-NEXT:    lw a7, 12(a1)
+; RV32ZBB-NEXT:    lw a1, 0(a2)
+; RV32ZBB-NEXT:    lw a6, 8(a2)
 ; RV32ZBB-NEXT:    lw t1, 12(a2)
-; RV32ZBB-NEXT:    lw a5, 0(a2)
-; RV32ZBB-NEXT:    lw a1, 4(a2)
-; RV32ZBB-NEXT:    sltu a2, a7, a6
-; RV32ZBB-NEXT:    mv t4, a2
-; RV32ZBB-NEXT:    beq t0, t1, .LBB17_2
+; RV32ZBB-NEXT:    lw a2, 4(a2)
+; RV32ZBB-NEXT:    sltu t0, a6, a5
+; RV32ZBB-NEXT:    mv t4, t0
+; RV32ZBB-NEXT:    beq a7, t1, .LBB17_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    slt t4, t1, t0
+; RV32ZBB-NEXT:    slt t4, t1, a7
 ; RV32ZBB-NEXT:  .LBB17_2:
-; RV32ZBB-NEXT:    sltu t2, a5, a3
-; RV32ZBB-NEXT:    sltu t5, a1, a4
+; RV32ZBB-NEXT:    sltu t2, a1, a3
+; RV32ZBB-NEXT:    sltu t5, a2, a4
 ; RV32ZBB-NEXT:    mv t3, t2
-; RV32ZBB-NEXT:    beq a4, a1, .LBB17_4
+; RV32ZBB-NEXT:    beq a4, a2, .LBB17_4
 ; RV32ZBB-NEXT:  # %bb.3:
 ; RV32ZBB-NEXT:    mv t3, t5
 ; RV32ZBB-NEXT:  .LBB17_4:
 ; RV32ZBB-NEXT:    addi sp, sp, -16
 ; RV32ZBB-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32ZBB-NEXT:    xor t6, t0, t1
-; RV32ZBB-NEXT:    xor s0, a6, a7
+; RV32ZBB-NEXT:    xor t6, a7, t1
+; RV32ZBB-NEXT:    xor s0, a5, a6
 ; RV32ZBB-NEXT:    or t6, s0, t6
 ; RV32ZBB-NEXT:    beqz t6, .LBB17_6
 ; RV32ZBB-NEXT:  # %bb.5:
 ; RV32ZBB-NEXT:    mv t3, t4
 ; RV32ZBB-NEXT:  .LBB17_6:
 ; RV32ZBB-NEXT:    mv t4, t2
-; RV32ZBB-NEXT:    beq a1, a4, .LBB17_8
+; RV32ZBB-NEXT:    beq a2, a4, .LBB17_8
 ; RV32ZBB-NEXT:  # %bb.7:
 ; RV32ZBB-NEXT:    mv t4, t5
 ; RV32ZBB-NEXT:  .LBB17_8:
-; RV32ZBB-NEXT:    sltu t5, a3, a5
+; RV32ZBB-NEXT:    sltu t5, a3, a1
 ; RV32ZBB-NEXT:    mv t6, t5
-; RV32ZBB-NEXT:    beq a4, a1, .LBB17_10
+; RV32ZBB-NEXT:    beq a4, a2, .LBB17_10
 ; RV32ZBB-NEXT:  # %bb.9:
-; RV32ZBB-NEXT:    sltu t6, a4, a1
+; RV32ZBB-NEXT:    sltu t6, a4, a2
 ; RV32ZBB-NEXT:  .LBB17_10:
 ; RV32ZBB-NEXT:    bnez t3, .LBB17_12
 ; RV32ZBB-NEXT:  # %bb.11:
-; RV32ZBB-NEXT:    sub t0, t1, t0
-; RV32ZBB-NEXT:    sub a6, a7, a6
-; RV32ZBB-NEXT:    sub a3, a5, a3
-; RV32ZBB-NEXT:    sub a1, a1, a4
-; RV32ZBB-NEXT:    sub a4, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t4
-; RV32ZBB-NEXT:    sub a2, a1, t2
-; RV32ZBB-NEXT:    sub a1, a4, a5
-; RV32ZBB-NEXT:    sub a4, a6, t4
+; RV32ZBB-NEXT:    sub a7, t1, a7
+; RV32ZBB-NEXT:    sub a5, a6, a5
+; RV32ZBB-NEXT:    sub a1, a1, a3
+; RV32ZBB-NEXT:    sub a2, a2, a4
+; RV32ZBB-NEXT:    sub a4, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t4
+; RV32ZBB-NEXT:    sub a3, a2, t2
+; RV32ZBB-NEXT:    sub a2, a4, a6
+; RV32ZBB-NEXT:    sub a4, a5, t4
 ; RV32ZBB-NEXT:    j .LBB17_13
 ; RV32ZBB-NEXT:  .LBB17_12:
-; RV32ZBB-NEXT:    sltu a2, a6, a7
-; RV32ZBB-NEXT:    sub t0, t0, t1
-; RV32ZBB-NEXT:    sub a6, a6, a7
-; RV32ZBB-NEXT:    sub a3, a3, a5
-; RV32ZBB-NEXT:    sub a4, a4, a1
-; RV32ZBB-NEXT:    sub a1, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t6
-; RV32ZBB-NEXT:    sub a2, a4, t5
-; RV32ZBB-NEXT:    sub a1, a1, a5
-; RV32ZBB-NEXT:    sub a4, a6, t6
+; RV32ZBB-NEXT:    sltu t0, a5, a6
+; RV32ZBB-NEXT:    sub a7, a7, t1
+; RV32ZBB-NEXT:    sub a5, a5, a6
+; RV32ZBB-NEXT:    sub a1, a3, a1
+; RV32ZBB-NEXT:    sub a4, a4, a2
+; RV32ZBB-NEXT:    sub a2, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t6
+; RV32ZBB-NEXT:    sub a3, a4, t5
+; RV32ZBB-NEXT:    sub a2, a2, a6
+; RV32ZBB-NEXT:    sub a4, a5, t6
 ; RV32ZBB-NEXT:  .LBB17_13:
-; RV32ZBB-NEXT:    sw a3, 0(a0)
-; RV32ZBB-NEXT:    sw a2, 4(a0)
+; RV32ZBB-NEXT:    sw a1, 0(a0)
+; RV32ZBB-NEXT:    sw a3, 4(a0)
 ; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 12(a0)
+; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
 ; RV32ZBB-NEXT:    ret
@@ -1516,73 +1516,73 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    lw a6, 8(a1)
-; RV32I-NEXT:    lw t0, 12(a1)
-; RV32I-NEXT:    lw a7, 8(a2)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a7, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
+; RV32I-NEXT:    lw a6, 8(a2)
 ; RV32I-NEXT:    lw t1, 12(a2)
-; RV32I-NEXT:    lw a5, 0(a2)
-; RV32I-NEXT:    lw a1, 4(a2)
-; RV32I-NEXT:    sltu a2, a7, a6
-; RV32I-NEXT:    mv t4, a2
-; RV32I-NEXT:    beq t0, t1, .LBB22_2
+; RV32I-NEXT:    lw a2, 4(a2)
+; RV32I-NEXT:    sltu t0, a6, a5
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beq a7, t1, .LBB22_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slt t4, t1, t0
+; RV32I-NEXT:    slt t4, t1, a7
 ; RV32I-NEXT:  .LBB22_2:
-; RV32I-NEXT:    sltu t2, a5, a3
-; RV32I-NEXT:    sltu t5, a1, a4
+; RV32I-NEXT:    sltu t2, a1, a3
+; RV32I-NEXT:    sltu t5, a2, a4
 ; RV32I-NEXT:    mv t3, t2
-; RV32I-NEXT:    beq a4, a1, .LBB22_4
+; RV32I-NEXT:    beq a4, a2, .LBB22_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    mv t3, t5
 ; RV32I-NEXT:  .LBB22_4:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    xor t6, t0, t1
-; RV32I-NEXT:    xor s0, a6, a7
+; RV32I-NEXT:    xor t6, a7, t1
+; RV32I-NEXT:    xor s0, a5, a6
 ; RV32I-NEXT:    or t6, s0, t6
 ; RV32I-NEXT:    beqz t6, .LBB22_6
 ; RV32I-NEXT:  # %bb.5:
 ; RV32I-NEXT:    mv t3, t4
 ; RV32I-NEXT:  .LBB22_6:
 ; RV32I-NEXT:    mv t4, t2
-; RV32I-NEXT:    beq a1, a4, .LBB22_8
+; RV32I-NEXT:    beq a2, a4, .LBB22_8
 ; RV32I-NEXT:  # %bb.7:
 ; RV32I-NEXT:    mv t4, t5
 ; RV32I-NEXT:  .LBB22_8:
-; RV32I-NEXT:    sltu t5, a3, a5
+; RV32I-NEXT:    sltu t5, a3, a1
 ; RV32I-NEXT:    mv t6, t5
-; RV32I-NEXT:    beq a4, a1, .LBB22_10
+; RV32I-NEXT:    beq a4, a2, .LBB22_10
 ; RV32I-NEXT:  # %bb.9:
-; RV32I-NEXT:    sltu t6, a4, a1
+; RV32I-NEXT:    sltu t6, a4, a2
 ; RV32I-NEXT:  .LBB22_10:
 ; RV32I-NEXT:    bnez t3, .LBB22_12
 ; RV32I-NEXT:  # %bb.11:
-; RV32I-NEXT:    sub t0, t1, t0
-; RV32I-NEXT:    sub a6, a7, a6
-; RV32I-NEXT:    sub a3, a5, a3
-; RV32I-NEXT:    sub a1, a1, a4
-; RV32I-NEXT:    sub a4, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t4
-; RV32I-NEXT:    sub a2, a1, t2
-; RV32I-NEXT:    sub a1, a4, a5
-; RV32I-NEXT:    sub a4, a6, t4
+; RV32I-NEXT:    sub a7, t1, a7
+; RV32I-NEXT:    sub a5, a6, a5
+; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a2, a2, a4
+; RV32I-NEXT:    sub a4, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t4
+; RV32I-NEXT:    sub a3, a2, t2
+; RV32I-NEXT:    sub a2, a4, a6
+; RV32I-NEXT:    sub a4, a5, t4
 ; RV32I-NEXT:    j .LBB22_13
 ; RV32I-NEXT:  .LBB22_12:
-; RV32I-NEXT:    sltu a2, a6, a7
-; RV32I-NEXT:    sub t0, t0, t1
-; RV32I-NEXT:    sub a6, a6, a7
-; RV32I-NEXT:    sub a3, a3, a5
-; RV32I-NEXT:    sub a4, a4, a1
-; RV32I-NEXT:    sub a1, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t6
-; RV32I-NEXT:    sub a2, a4, t5
-; RV32I-NEXT:    sub a1, a1, a5
-; RV32I-NEXT:    sub a4, a6, t6
+; RV32I-NEXT:    sltu t0, a5, a6
+; RV32I-NEXT:    sub a7, a7, t1
+; RV32I-NEXT:    sub a5, a5, a6
+; RV32I-NEXT:    sub a1, a3, a1
+; RV32I-NEXT:    sub a4, a4, a2
+; RV32I-NEXT:    sub a2, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t6
+; RV32I-NEXT:    sub a3, a4, t5
+; RV32I-NEXT:    sub a2, a2, a6
+; RV32I-NEXT:    sub a4, a5, t6
 ; RV32I-NEXT:  .LBB22_13:
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    sw a2, 4(a0)
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 12(a0)
+; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -1612,73 +1612,73 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    lw a3, 0(a1)
 ; RV32ZBB-NEXT:    lw a4, 4(a1)
-; RV32ZBB-NEXT:    lw a6, 8(a1)
-; RV32ZBB-NEXT:    lw t0, 12(a1)
-; RV32ZBB-NEXT:    lw a7, 8(a2)
+; RV32ZBB-NEXT:    lw a5, 8(a1)
+; RV32ZBB-NEXT:    lw a7, 12(a1)
+; RV32ZBB-NEXT:    lw a1, 0(a2)
+; RV32ZBB-NEXT:    lw a6, 8(a2)
 ; RV32ZBB-NEXT:    lw t1, 12(a2)
-; RV32ZBB-NEXT:    lw a5, 0(a2)
-; RV32ZBB-NEXT:    lw a1, 4(a2)
-; RV32ZBB-NEXT:    sltu a2, a7, a6
-; RV32ZBB-NEXT:    mv t4, a2
-; RV32ZBB-NEXT:    beq t0, t1, .LBB22_2
+; RV32ZBB-NEXT:    lw a2, 4(a2)
+; RV32ZBB-NEXT:    sltu t0, a6, a5
+; RV32ZBB-NEXT:    mv t4, t0
+; RV32ZBB-NEXT:    beq a7, t1, .LBB22_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    slt t4, t1, t0
+; RV32ZBB-NEXT:    slt t4, t1, a7
 ; RV32ZBB-NEXT:  .LBB22_2:
-; RV32ZBB-NEXT:    sltu t2, a5, a3
-; RV32ZBB-NEXT:    sltu t5, a1, a4
+; RV32ZBB-NEXT:    sltu t2, a1, a3
+; RV32ZBB-NEXT:    sltu t5, a2, a4
 ; RV32ZBB-NEXT:    mv t3, t2
-; RV32ZBB-NEXT:    beq a4, a1, .LBB22_4
+; RV32ZBB-NEXT:    beq a4, a2, .LBB22_4
 ; RV32ZBB-NEXT:  # %bb.3:
 ; RV32ZBB-NEXT:    mv t3, t5
 ; RV32ZBB-NEXT:  .LBB22_4:
 ; RV32ZBB-NEXT:    addi sp, sp, -16
 ; RV32ZBB-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32ZBB-NEXT:    xor t6, t0, t1
-; RV32ZBB-NEXT:    xor s0, a6, a7
+; RV32ZBB-NEXT:    xor t6, a7, t1
+; RV32ZBB-NEXT:    xor s0, a5, a6
 ; RV32ZBB-NEXT:    or t6, s0, t6
 ; RV32ZBB-NEXT:    beqz t6, .LBB22_6
 ; RV32ZBB-NEXT:  # %bb.5:
 ; RV32ZBB-NEXT:    mv t3, t4
 ; RV32ZBB-NEXT:  .LBB22_6:
 ; RV32ZBB-NEXT:    mv t4, t2
-; RV32ZBB-NEXT:    beq a1, a4, .LBB22_8
+; RV32ZBB-NEXT:    beq a2, a4, .LBB22_8
 ; RV32ZBB-NEXT:  # %bb.7:
 ; RV32ZBB-NEXT:    mv t4, t5
 ; RV32ZBB-NEXT:  .LBB22_8:
-; RV32ZBB-NEXT:    sltu t5, a3, a5
+; RV32ZBB-NEXT:    sltu t5, a3, a1
 ; RV32ZBB-NEXT:    mv t6, t5
-; RV32ZBB-NEXT:    beq a4, a1, .LBB22_10
+; RV32ZBB-NEXT:    beq a4, a2, .LBB22_10
 ; RV32ZBB-NEXT:  # %bb.9:
-; RV32ZBB-NEXT:    sltu t6, a4, a1
+; RV32ZBB-NEXT:    sltu t6, a4, a2
 ; RV32ZBB-NEXT:  .LBB22_10:
 ; RV32ZBB-NEXT:    bnez t3, .LBB22_12
 ; RV32ZBB-NEXT:  # %bb.11:
-; RV32ZBB-NEXT:    sub t0, t1, t0
-; RV32ZBB-NEXT:    sub a6, a7, a6
-; RV32ZBB-NEXT:    sub a3, a5, a3
-; RV32ZBB-NEXT:    sub a1, a1, a4
-; RV32ZBB-NEXT:    sub a4, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t4
-; RV32ZBB-NEXT:    sub a2, a1, t2
-; RV32ZBB-NEXT:    sub a1, a4, a5
-; RV32ZBB-NEXT:    sub a4, a6, t4
+; RV32ZBB-NEXT:    sub a7, t1, a7
+; RV32ZBB-NEXT:    sub a5, a6, a5
+; RV32ZBB-NEXT:    sub a1, a1, a3
+; RV32ZBB-NEXT:    sub a2, a2, a4
+; RV32ZBB-NEXT:    sub a4, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t4
+; RV32ZBB-NEXT:    sub a3, a2, t2
+; RV32ZBB-NEXT:    sub a2, a4, a6
+; RV32ZBB-NEXT:    sub a4, a5, t4
 ; RV32ZBB-NEXT:    j .LBB22_13
 ; RV32ZBB-NEXT:  .LBB22_12:
-; RV32ZBB-NEXT:    sltu a2, a6, a7
-; RV32ZBB-NEXT:    sub t0, t0, t1
-; RV32ZBB-NEXT:    sub a6, a6, a7
-; RV32ZBB-NEXT:    sub a3, a3, a5
-; RV32ZBB-NEXT:    sub a4, a4, a1
-; RV32ZBB-NEXT:    sub a1, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t6
-; RV32ZBB-NEXT:    sub a2, a4, t5
-; RV32ZBB-NEXT:    sub a1, a1, a5
-; RV32ZBB-NEXT:    sub a4, a6, t6
+; RV32ZBB-NEXT:    sltu t0, a5, a6
+; RV32ZBB-NEXT:    sub a7, a7, t1
+; RV32ZBB-NEXT:    sub a5, a5, a6
+; RV32ZBB-NEXT:    sub a1, a3, a1
+; RV32ZBB-NEXT:    sub a4, a4, a2
+; RV32ZBB-NEXT:    sub a2, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t6
+; RV32ZBB-NEXT:    sub a3, a4, t5
+; RV32ZBB-NEXT:    sub a2, a2, a6
+; RV32ZBB-NEXT:    sub a4, a5, t6
 ; RV32ZBB-NEXT:  .LBB22_13:
-; RV32ZBB-NEXT:    sw a3, 0(a0)
-; RV32ZBB-NEXT:    sw a2, 4(a0)
+; RV32ZBB-NEXT:    sw a1, 0(a0)
+; RV32ZBB-NEXT:    sw a3, 4(a0)
 ; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 12(a0)
+; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
 ; RV32ZBB-NEXT:    ret
@@ -2539,73 +2539,73 @@ define i128 @abd_select_i128(i128 %a, i128 %b) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    lw a6, 8(a1)
-; RV32I-NEXT:    lw t0, 12(a1)
-; RV32I-NEXT:    lw a7, 8(a2)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a7, 12(a1)
+; RV32I-NEXT:    lw a1, 0(a2)
+; RV32I-NEXT:    lw a6, 8(a2)
 ; RV32I-NEXT:    lw t1, 12(a2)
-; RV32I-NEXT:    lw a5, 0(a2)
-; RV32I-NEXT:    lw a1, 4(a2)
-; RV32I-NEXT:    sltu a2, a7, a6
-; RV32I-NEXT:    mv t4, a2
-; RV32I-NEXT:    beq t0, t1, .LBB38_2
+; RV32I-NEXT:    lw a2, 4(a2)
+; RV32I-NEXT:    sltu t0, a6, a5
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beq a7, t1, .LBB38_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slt t4, t1, t0
+; RV32I-NEXT:    slt t4, t1, a7
 ; RV32I-NEXT:  .LBB38_2:
-; RV32I-NEXT:    sltu t2, a5, a3
-; RV32I-NEXT:    sltu t5, a1, a4
+; RV32I-NEXT:    sltu t2, a1, a3
+; RV32I-NEXT:    sltu t5, a2, a4
 ; RV32I-NEXT:    mv t3, t2
-; RV32I-NEXT:    beq a4, a1, .LBB38_4
+; RV32I-NEXT:    beq a4, a2, .LBB38_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    mv t3, t5
 ; RV32I-NEXT:  .LBB38_4:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    xor t6, t0, t1
-; RV32I-NEXT:    xor s0, a6, a7
+; RV32I-NEXT:    xor t6, a7, t1
+; RV32I-NEXT:    xor s0, a5, a6
 ; RV32I-NEXT:    or t6, s0, t6
 ; RV32I-NEXT:    beqz t6, .LBB38_6
 ; RV32I-NEXT:  # %bb.5:
 ; RV32I-NEXT:    mv t3, t4
 ; RV32I-NEXT:  .LBB38_6:
 ; RV32I-NEXT:    mv t4, t2
-; RV32I-NEXT:    beq a1, a4, .LBB38_8
+; RV32I-NEXT:    beq a2, a4, .LBB38_8
 ; RV32I-NEXT:  # %bb.7:
 ; RV32I-NEXT:    mv t4, t5
 ; RV32I-NEXT:  .LBB38_8:
-; RV32I-NEXT:    sltu t5, a3, a5
+; RV32I-NEXT:    sltu t5, a3, a1
 ; RV32I-NEXT:    mv t6, t5
-; RV32I-NEXT:    beq a4, a1, .LBB38_10
+; RV32I-NEXT:    beq a4, a2, .LBB38_10
 ; RV32I-NEXT:  # %bb.9:
-; RV32I-NEXT:    sltu t6, a4, a1
+; RV32I-NEXT:    sltu t6, a4, a2
 ; RV32I-NEXT:  .LBB38_10:
 ; RV32I-NEXT:    bnez t3, .LBB38_12
 ; RV32I-NEXT:  # %bb.11:
-; RV32I-NEXT:    sub t0, t1, t0
-; RV32I-NEXT:    sub a6, a7, a6
-; RV32I-NEXT:    sub a3, a5, a3
-; RV32I-NEXT:    sub a1, a1, a4
-; RV32I-NEXT:    sub a4, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t4
-; RV32I-NEXT:    sub a2, a1, t2
-; RV32I-NEXT:    sub a1, a4, a5
-; RV32I-NEXT:    sub a4, a6, t4
+; RV32I-NEXT:    sub a7, t1, a7
+; RV32I-NEXT:    sub a5, a6, a5
+; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a2, a2, a4
+; RV32I-NEXT:    sub a4, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t4
+; RV32I-NEXT:    sub a3, a2, t2
+; RV32I-NEXT:    sub a2, a4, a6
+; RV32I-NEXT:    sub a4, a5, t4
 ; RV32I-NEXT:    j .LBB38_13
 ; RV32I-NEXT:  .LBB38_12:
-; RV32I-NEXT:    sltu a2, a6, a7
-; RV32I-NEXT:    sub t0, t0, t1
-; RV32I-NEXT:    sub a6, a6, a7
-; RV32I-NEXT:    sub a3, a3, a5
-; RV32I-NEXT:    sub a4, a4, a1
-; RV32I-NEXT:    sub a1, t0, a2
-; RV32I-NEXT:    sltu a5, a6, t6
-; RV32I-NEXT:    sub a2, a4, t5
-; RV32I-NEXT:    sub a1, a1, a5
-; RV32I-NEXT:    sub a4, a6, t6
+; RV32I-NEXT:    sltu t0, a5, a6
+; RV32I-NEXT:    sub a7, a7, t1
+; RV32I-NEXT:    sub a5, a5, a6
+; RV32I-NEXT:    sub a1, a3, a1
+; RV32I-NEXT:    sub a4, a4, a2
+; RV32I-NEXT:    sub a2, a7, t0
+; RV32I-NEXT:    sltu a6, a5, t6
+; RV32I-NEXT:    sub a3, a4, t5
+; RV32I-NEXT:    sub a2, a2, a6
+; RV32I-NEXT:    sub a4, a5, t6
 ; RV32I-NEXT:  .LBB38_13:
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    sw a2, 4(a0)
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 12(a0)
+; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -2635,73 +2635,73 @@ define i128 @abd_select_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    lw a3, 0(a1)
 ; RV32ZBB-NEXT:    lw a4, 4(a1)
-; RV32ZBB-NEXT:    lw a6, 8(a1)
-; RV32ZBB-NEXT:    lw t0, 12(a1)
-; RV32ZBB-NEXT:    lw a7, 8(a2)
+; RV32ZBB-NEXT:    lw a5, 8(a1)
+; RV32ZBB-NEXT:    lw a7, 12(a1)
+; RV32ZBB-NEXT:    lw a1, 0(a2)
+; RV32ZBB-NEXT:    lw a6, 8(a2)
 ; RV32ZBB-NEXT:    lw t1, 12(a2)
-; RV32ZBB-NEXT:    lw a5, 0(a2)
-; RV32ZBB-NEXT:    lw a1, 4(a2)
-; RV32ZBB-NEXT:    sltu a2, a7, a6
-; RV32ZBB-NEXT:    mv t4, a2
-; RV32ZBB-NEXT:    beq t0, t1, .LBB38_2
+; RV32ZBB-NEXT:    lw a2, 4(a2)
+; RV32ZBB-NEXT:    sltu t0, a6, a5
+; RV32ZBB-NEXT:    mv t4, t0
+; RV32ZBB-NEXT:    beq a7, t1, .LBB38_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    slt t4, t1, t0
+; RV32ZBB-NEXT:    slt t4, t1, a7
 ; RV32ZBB-NEXT:  .LBB38_2:
-; RV32ZBB-NEXT:    sltu t2, a5, a3
-; RV32ZBB-NEXT:    sltu t5, a1, a4
+; RV32ZBB-NEXT:    sltu t2, a1, a3
+; RV32ZBB-NEXT:    sltu t5, a2, a4
 ; RV32ZBB-NEXT:    mv t3, t2
-; RV32ZBB-NEXT:    beq a4, a1, .LBB38_4
+; RV32ZBB-NEXT:    beq a4, a2, .LBB38_4
 ; RV32ZBB-NEXT:  # %bb.3:
 ; RV32ZBB-NEXT:    mv t3, t5
 ; RV32ZBB-NEXT:  .LBB38_4:
 ; RV32ZBB-NEXT:    addi sp, sp, -16
 ; RV32ZBB-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
-; RV32ZBB-NEXT:    xor t6, t0, t1
-; RV32ZBB-NEXT:    xor s0, a6, a7
+; RV32ZBB-NEXT:    xor t6, a7, t1
+; RV32ZBB-NEXT:    xor s0, a5, a6
 ; RV32ZBB-NEXT:    or t6, s0, t6
 ; RV32ZBB-NEXT:    beqz t6, .LBB38_6
 ; RV32ZBB-NEXT:  # %bb.5:
 ; RV32ZBB-NEXT:    mv t3, t4
 ; RV32ZBB-NEXT:  .LBB38_6:
 ; RV32ZBB-NEXT:    mv t4, t2
-; RV32ZBB-NEXT:    beq a1, a4, .LBB38_8
+; RV32ZBB-NEXT:    beq a2, a4, .LBB38_8
 ; RV32ZBB-NEXT:  # %bb.7:
 ; RV32ZBB-NEXT:    mv t4, t5
 ; RV32ZBB-NEXT:  .LBB38_8:
-; RV32ZBB-NEXT:    sltu t5, a3, a5
+; RV32ZBB-NEXT:    sltu t5, a3, a1
 ; RV32ZBB-NEXT:    mv t6, t5
-; RV32ZBB-NEXT:    beq a4, a1, .LBB38_10
+; RV32ZBB-NEXT:    beq a4, a2, .LBB38_10
 ; RV32ZBB-NEXT:  # %bb.9:
-; RV32ZBB-NEXT:    sltu t6, a4, a1
+; RV32ZBB-NEXT:    sltu t6, a4, a2
 ; RV32ZBB-NEXT:  .LBB38_10:
 ; RV32ZBB-NEXT:    bnez t3, .LBB38_12
 ; RV32ZBB-NEXT:  # %bb.11:
-; RV32ZBB-NEXT:    sub t0, t1, t0
-; RV32ZBB-NEXT:    sub a6, a7, a6
-; RV32ZBB-NEXT:    sub a3, a5, a3
-; RV32ZBB-NEXT:    sub a1, a1, a4
-; RV32ZBB-NEXT:    sub a4, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t4
-; RV32ZBB-NEXT:    sub a2, a1, t2
-; RV32ZBB-NEXT:    sub a1, a4, a5
-; RV32ZBB-NEXT:    sub a4, a6, t4
+; RV32ZBB-NEXT:    sub a7, t1, a7
+; RV32ZBB-NEXT:    sub a5, a6, a5
+; RV32ZBB-NEXT:    sub a1, a1, a3
+; RV32ZBB-NEXT:    sub a2, a2, a4
+; RV32ZBB-NEXT:    sub a4, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t4
+; RV32ZBB-NEXT:    sub a3, a2, t2
+; RV32ZBB-NEXT:    sub a2, a4, a6
+; RV32ZBB-NEXT:    sub a4, a5, t4
 ; RV32ZBB-NEXT:    j .LBB38_13
 ; RV32ZBB-NEXT:  .LBB38_12:
-; RV32ZBB-NEXT:    sltu a2, a6, a7
-; RV32ZBB-NEXT:    sub t0, t0, t1
-; RV32ZBB-NEXT:    sub a6, a6, a7
-; RV32ZBB-NEXT:    sub a3, a3, a5
-; RV32ZBB-NEXT:    sub a4, a4, a1
-; RV32ZBB-NEXT:    sub a1, t0, a2
-; RV32ZBB-NEXT:    sltu a5, a6, t6
-; RV32ZBB-NEXT:    sub a2, a4, t5
-; RV32ZBB-NEXT:    sub a1, a1, a5
-; RV32ZBB-NEXT:    sub a4, a6, t6
+; RV32ZBB-NEXT:    sltu t0, a5, a6
+; RV32ZBB-NEXT:    sub a7, a7, t1
+; RV32ZBB-NEXT:    sub a5, a5, a6
+; RV32ZBB-NEXT:    sub a1, a3, a1
+; RV32ZBB-NEXT:    sub a4, a4, a2
+; RV32ZBB-NEXT:    sub a2, a7, t0
+; RV32ZBB-NEXT:    sltu a6, a5, t6
+; RV32ZBB-NEXT:    sub a3, a4, t5
+; RV32ZBB-NEXT:    sub a2, a2, a6
+; RV32ZBB-NEXT:    sub a4, a5, t6
 ; RV32ZBB-NEXT:  .LBB38_13:
-; RV32ZBB-NEXT:    sw a3, 0(a0)
-; RV32ZBB-NEXT:    sw a2, 4(a0)
+; RV32ZBB-NEXT:    sw a1, 0(a0)
+; RV32ZBB-NEXT:    sw a3, 4(a0)
 ; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 12(a0)
+; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
 ; RV32ZBB-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/abdu-neg.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/abdu-neg.ll
index 9fa142ee2aa1..713b52f53e3d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/abdu-neg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/abdu-neg.ll
@@ -167,7 +167,7 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: abd_ext_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    sub a0, a0, a1
@@ -271,7 +271,7 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: abd_ext_i16_undef:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    sub a0, a0, a1
@@ -1125,45 +1125,25 @@ define i8 @abd_minmax_i8(i8 %a, i8 %b) nounwind {
 }
 
 define i16 @abd_minmax_i16(i16 %a, i16 %b) nounwind {
-; RV32I-LABEL: abd_minmax_i16:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 16
-; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu a0, a1, .LBB14_3
-; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    bgeu a1, a0, .LBB14_4
-; RV32I-NEXT:  .LBB14_2:
-; RV32I-NEXT:    sub a0, a2, a0
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB14_3:
-; RV32I-NEXT:    mv a2, a1
-; RV32I-NEXT:    bltu a1, a0, .LBB14_2
-; RV32I-NEXT:  .LBB14_4:
-; RV32I-NEXT:    sub a0, a2, a1
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: abd_minmax_i16:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu a0, a1, .LBB14_3
-; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    bgeu a1, a0, .LBB14_4
-; RV64I-NEXT:  .LBB14_2:
-; RV64I-NEXT:    sub a0, a2, a0
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB14_3:
-; RV64I-NEXT:    mv a2, a1
-; RV64I-NEXT:    bltu a1, a0, .LBB14_2
-; RV64I-NEXT:  .LBB14_4:
-; RV64I-NEXT:    sub a0, a2, a1
-; RV64I-NEXT:    ret
+; NOZBB-LABEL: abd_minmax_i16:
+; NOZBB:       # %bb.0:
+; NOZBB-NEXT:    lui a2, 16
+; NOZBB-NEXT:    addi a2, a2, -1
+; NOZBB-NEXT:    and a1, a1, a2
+; NOZBB-NEXT:    and a0, a0, a2
+; NOZBB-NEXT:    mv a2, a0
+; NOZBB-NEXT:    bgeu a0, a1, .LBB14_3
+; NOZBB-NEXT:  # %bb.1:
+; NOZBB-NEXT:    bgeu a1, a0, .LBB14_4
+; NOZBB-NEXT:  .LBB14_2:
+; NOZBB-NEXT:    sub a0, a2, a0
+; NOZBB-NEXT:    ret
+; NOZBB-NEXT:  .LBB14_3:
+; NOZBB-NEXT:    mv a2, a1
+; NOZBB-NEXT:    bltu a1, a0, .LBB14_2
+; NOZBB-NEXT:  .LBB14_4:
+; NOZBB-NEXT:    sub a0, a2, a1
+; NOZBB-NEXT:    ret
 ;
 ; ZBB-LABEL: abd_minmax_i16:
 ; ZBB:       # %bb.0:
@@ -1338,8 +1318,8 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a6, 4(a2)
 ; RV32I-NEXT:    lw a7, 8(a2)
 ; RV32I-NEXT:    lw t0, 12(a2)
-; RV32I-NEXT:    lw a5, 12(a1)
 ; RV32I-NEXT:    lw a3, 4(a1)
+; RV32I-NEXT:    lw a5, 12(a1)
 ; RV32I-NEXT:    lw a4, 8(a1)
 ; RV32I-NEXT:    beq a5, t0, .LBB17_2
 ; RV32I-NEXT:  # %bb.1:
@@ -1465,8 +1445,8 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a6, 4(a2)
 ; RV32ZBB-NEXT:    lw a7, 8(a2)
 ; RV32ZBB-NEXT:    lw t0, 12(a2)
-; RV32ZBB-NEXT:    lw a5, 12(a1)
 ; RV32ZBB-NEXT:    lw a3, 4(a1)
+; RV32ZBB-NEXT:    lw a5, 12(a1)
 ; RV32ZBB-NEXT:    lw a4, 8(a1)
 ; RV32ZBB-NEXT:    beq a5, t0, .LBB17_2
 ; RV32ZBB-NEXT:  # %bb.1:
@@ -1628,33 +1608,19 @@ define i8 @abd_cmp_i8(i8 %a, i8 %b) nounwind {
 }
 
 define i16 @abd_cmp_i16(i16 %a, i16 %b) nounwind {
-; RV32I-LABEL: abd_cmp_i16:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 16
-; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    and a3, a1, a2
-; RV32I-NEXT:    and a2, a0, a2
-; RV32I-NEXT:    bltu a2, a3, .LBB19_2
-; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    sub a0, a1, a0
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB19_2:
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: abd_cmp_i16:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
-; RV64I-NEXT:    and a3, a1, a2
-; RV64I-NEXT:    and a2, a0, a2
-; RV64I-NEXT:    bltu a2, a3, .LBB19_2
-; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    sub a0, a1, a0
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB19_2:
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    ret
+; NOZBB-LABEL: abd_cmp_i16:
+; NOZBB:       # %bb.0:
+; NOZBB-NEXT:    lui a2, 16
+; NOZBB-NEXT:    addi a2, a2, -1
+; NOZBB-NEXT:    and a3, a1, a2
+; NOZBB-NEXT:    and a2, a0, a2
+; NOZBB-NEXT:    bltu a2, a3, .LBB19_2
+; NOZBB-NEXT:  # %bb.1:
+; NOZBB-NEXT:    sub a0, a1, a0
+; NOZBB-NEXT:    ret
+; NOZBB-NEXT:  .LBB19_2:
+; NOZBB-NEXT:    sub a0, a0, a1
+; NOZBB-NEXT:    ret
 ;
 ; ZBB-LABEL: abd_cmp_i16:
 ; ZBB:       # %bb.0:
@@ -1801,15 +1767,15 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a4, 4(a2)
 ; RV32I-NEXT:    lw a5, 8(a2)
 ; RV32I-NEXT:    lw a7, 12(a2)
-; RV32I-NEXT:    lw a6, 8(a1)
-; RV32I-NEXT:    lw t0, 12(a1)
 ; RV32I-NEXT:    lw a2, 0(a1)
+; RV32I-NEXT:    lw a6, 8(a1)
+; RV32I-NEXT:    lw t1, 12(a1)
 ; RV32I-NEXT:    lw a1, 4(a1)
-; RV32I-NEXT:    sltu t1, a6, a5
-; RV32I-NEXT:    mv t4, t1
-; RV32I-NEXT:    beq t0, a7, .LBB22_2
+; RV32I-NEXT:    sltu t0, a6, a5
+; RV32I-NEXT:    mv t4, t0
+; RV32I-NEXT:    beq t1, a7, .LBB22_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    sltu t4, t0, a7
+; RV32I-NEXT:    sltu t4, t1, a7
 ; RV32I-NEXT:  .LBB22_2:
 ; RV32I-NEXT:    sltu t2, a2, a3
 ; RV32I-NEXT:    mv t3, t2
@@ -1817,7 +1783,7 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    sltu t3, a1, a4
 ; RV32I-NEXT:  .LBB22_4:
-; RV32I-NEXT:    xor t5, t0, a7
+; RV32I-NEXT:    xor t5, t1, a7
 ; RV32I-NEXT:    xor t6, a6, a5
 ; RV32I-NEXT:    or t5, t6, t5
 ; RV32I-NEXT:    mv t6, t3
@@ -1833,11 +1799,11 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:  .LBB22_8:
 ; RV32I-NEXT:    bnez t6, .LBB22_10
 ; RV32I-NEXT:  # %bb.9:
-; RV32I-NEXT:    sltu t1, a5, a6
-; RV32I-NEXT:    sub a7, a7, t0
+; RV32I-NEXT:    sltu t0, a5, a6
+; RV32I-NEXT:    sub a7, a7, t1
 ; RV32I-NEXT:    sub a5, a5, a6
 ; RV32I-NEXT:    sub a4, a4, a1
-; RV32I-NEXT:    sub a6, a7, t1
+; RV32I-NEXT:    sub a6, a7, t0
 ; RV32I-NEXT:    sltu a7, a5, t5
 ; RV32I-NEXT:    sub a1, a5, t5
 ; RV32I-NEXT:    sub a5, a4, t4
@@ -1845,10 +1811,10 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a2, a3, a2
 ; RV32I-NEXT:    j .LBB22_11
 ; RV32I-NEXT:  .LBB22_10:
-; RV32I-NEXT:    sub a7, t0, a7
+; RV32I-NEXT:    sub a7, t1, a7
 ; RV32I-NEXT:    sub a5, a6, a5
 ; RV32I-NEXT:    sub a4, a1, a4
-; RV32I-NEXT:    sub a6, a7, t1
+; RV32I-NEXT:    sub a6, a7, t0
 ; RV32I-NEXT:    sltu a7, a5, t3
 ; RV32I-NEXT:    sub a1, a5, t3
 ; RV32I-NEXT:    sub a5, a4, t2
@@ -1888,15 +1854,15 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a4, 4(a2)
 ; RV32ZBB-NEXT:    lw a5, 8(a2)
 ; RV32ZBB-NEXT:    lw a7, 12(a2)
-; RV32ZBB-NEXT:    lw a6, 8(a1)
-; RV32ZBB-NEXT:    lw t0, 12(a1)
 ; RV32ZBB-NEXT:    lw a2, 0(a1)
+; RV32ZBB-NEXT:    lw a6, 8(a1)
+; RV32ZBB-NEXT:    lw t1, 12(a1)
 ; RV32ZBB-NEXT:    lw a1, 4(a1)
-; RV32ZBB-NEXT:    sltu t1, a6, a5
-; RV32ZBB-NEXT:    mv t4, t1
-; RV32ZBB-NEXT:    beq t0, a7, .LBB22_2
+; RV32ZBB-NEXT:    sltu t0, a6, a5
+; RV32ZBB-NEXT:    mv t4, t0
+; RV32ZBB-NEXT:    beq t1, a7, .LBB22_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    sltu t4, t0, a7
+; RV32ZBB-NEXT:    sltu t4, t1, a7
 ; RV32ZBB-NEXT:  .LBB22_2:
 ; RV32ZBB-NEXT:    sltu t2, a2, a3
 ; RV32ZBB-NEXT:    mv t3, t2
@@ -1904,7 +1870,7 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:  # %bb.3:
 ; RV32ZBB-NEXT:    sltu t3, a1, a4
 ; RV32ZBB-NEXT:  .LBB22_4:
-; RV32ZBB-NEXT:    xor t5, t0, a7
+; RV32ZBB-NEXT:    xor t5, t1, a7
 ; RV32ZBB-NEXT:    xor t6, a6, a5
 ; RV32ZBB-NEXT:    or t5, t6, t5
 ; RV32ZBB-NEXT:    mv t6, t3
@@ -1920,11 +1886,11 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:  .LBB22_8:
 ; RV32ZBB-NEXT:    bnez t6, .LBB22_10
 ; RV32ZBB-NEXT:  # %bb.9:
-; RV32ZBB-NEXT:    sltu t1, a5, a6
-; RV32ZBB-NEXT:    sub a7, a7, t0
+; RV32ZBB-NEXT:    sltu t0, a5, a6
+; RV32ZBB-NEXT:    sub a7, a7, t1
 ; RV32ZBB-NEXT:    sub a5, a5, a6
 ; RV32ZBB-NEXT:    sub a4, a4, a1
-; RV32ZBB-NEXT:    sub a6, a7, t1
+; RV32ZBB-NEXT:    sub a6, a7, t0
 ; RV32ZBB-NEXT:    sltu a7, a5, t5
 ; RV32ZBB-NEXT:    sub a1, a5, t5
 ; RV32ZBB-NEXT:    sub a5, a4, t4
@@ -1932,10 +1898,10 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a2, a3, a2
 ; RV32ZBB-NEXT:    j .LBB22_11
 ; RV32ZBB-NEXT:  .LBB22_10:
-; RV32ZBB-NEXT:    sub a7, t0, a7
+; RV32ZBB-NEXT:    sub a7, t1, a7
 ; RV32ZBB-NEXT:    sub a5, a6, a5
 ; RV32ZBB-NEXT:    sub a4, a1, a4
-; RV32ZBB-NEXT:    sub a6, a7, t1
+; RV32ZBB-NEXT:    sub a6, a7, t0
 ; RV32ZBB-NEXT:    sltu a7, a5, t3
 ; RV32ZBB-NEXT:    sub a1, a5, t3
 ; RV32ZBB-NEXT:    sub a5, a4, t2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/abdu.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/abdu.ll
index 614d9c20ac57..6ef172a6cd61 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/abdu.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/abdu.ll
@@ -137,7 +137,7 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: abd_ext_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sub a0, a0, a1
@@ -228,7 +228,7 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: abd_ext_i16_undef:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sub a0, a0, a1
@@ -992,7 +992,7 @@ define i16 @abd_minmax_i16(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: abd_minmax_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sub a0, a0, a1
@@ -1382,7 +1382,7 @@ define i16 @abd_cmp_i16(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: abd_cmp_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sub a0, a0, a1
@@ -1776,7 +1776,7 @@ define i16 @abd_select_i16(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: abd_select_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sub a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/add-before-shl.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/add-before-shl.ll
index b6ff3c9060af..35a39b89a2cb 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/add-before-shl.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/add-before-shl.ll
@@ -200,26 +200,26 @@ define i128 @add_wide_operand(i128 %a) nounwind {
 ;
 ; RV32C-LABEL: add_wide_operand:
 ; RV32C:       # %bb.0:
+; RV32C-NEXT:    c.lw a2, 0(a1)
 ; RV32C-NEXT:    c.lw a4, 12(a1)
-; RV32C-NEXT:    c.lw a3, 0(a1)
-; RV32C-NEXT:    c.lw a2, 4(a1)
+; RV32C-NEXT:    c.lw a3, 4(a1)
 ; RV32C-NEXT:    c.lw a1, 8(a1)
 ; RV32C-NEXT:    c.lui a5, 16
 ; RV32C-NEXT:    add a6, a4, a5
-; RV32C-NEXT:    srli a5, a3, 29
-; RV32C-NEXT:    slli a4, a2, 3
+; RV32C-NEXT:    srli a5, a2, 29
+; RV32C-NEXT:    slli a4, a3, 3
 ; RV32C-NEXT:    c.or a4, a5
 ; RV32C-NEXT:    srli a5, a1, 29
-; RV32C-NEXT:    c.srli a2, 29
+; RV32C-NEXT:    c.srli a3, 29
 ; RV32C-NEXT:    c.slli a1, 3
-; RV32C-NEXT:    c.slli a3, 3
+; RV32C-NEXT:    c.slli a2, 3
 ; RV32C-NEXT:    c.slli a6, 3
-; RV32C-NEXT:    c.or a1, a2
-; RV32C-NEXT:    or a2, a6, a5
-; RV32C-NEXT:    c.sw a3, 0(a0)
+; RV32C-NEXT:    c.or a1, a3
+; RV32C-NEXT:    or a3, a6, a5
+; RV32C-NEXT:    c.sw a2, 0(a0)
 ; RV32C-NEXT:    c.sw a4, 4(a0)
 ; RV32C-NEXT:    c.sw a1, 8(a0)
-; RV32C-NEXT:    c.sw a2, 12(a0)
+; RV32C-NEXT:    c.sw a3, 12(a0)
 ; RV32C-NEXT:    c.jr ra
 ;
 ; RV64C-LABEL: add_wide_operand:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/add_sext_shl_constant.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/add_sext_shl_constant.ll
index d7f62ae83434..9cb90cc16a51 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/add_sext_shl_constant.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/add_sext_shl_constant.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
 ; RUN: llc -mtriple=riscv64 < %s | FileCheck -check-prefixes=RV64,NO-ZBA %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zba < %s | FileCheck -check-prefixes=RV64,ZBA %s
+; RUN: llc -mtriple=riscv64 -mattr=+xandesperf < %s | FileCheck -check-prefixes=RV64,XANDESPERF %s
 
 define void @add_sext_shl_moreOneUse_add(ptr %array1, i32 %a, i32 %b) {
 ; NO-ZBA-LABEL: add_sext_shl_moreOneUse_add:
@@ -23,6 +24,16 @@ define void @add_sext_shl_moreOneUse_add(ptr %array1, i32 %a, i32 %b) {
 ; ZBA-NEXT:    sw a2, 24(a0)
 ; ZBA-NEXT:    sw a3, 140(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_sext_shl_moreOneUse_add:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a3, a1, 5
+; XANDESPERF-NEXT:    sext.w a1, a1
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    sw a2, 20(a0)
+; XANDESPERF-NEXT:    sw a2, 24(a0)
+; XANDESPERF-NEXT:    sw a3, 140(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 5
   %idxprom = sext i32 %add to i64
@@ -66,6 +77,19 @@ define void @add_sext_shl_moreOneUse_addexceedsign12(ptr %array1, i32 %a, i32 %b
 ; ZBA-NEXT:    sw a3, 4(a0)
 ; ZBA-NEXT:    sw a2, 120(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_sext_shl_moreOneUse_addexceedsign12:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a3, a1, 2047
+; XANDESPERF-NEXT:    lui a4, 2
+; XANDESPERF-NEXT:    sext.w a1, a1
+; XANDESPERF-NEXT:    addi a3, a3, 1
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    add a0, a0, a4
+; XANDESPERF-NEXT:    sw a2, 0(a0)
+; XANDESPERF-NEXT:    sw a3, 4(a0)
+; XANDESPERF-NEXT:    sw a2, 120(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 2048
   %idxprom = sext i32 %add to i64
@@ -101,6 +125,16 @@ define void @add_sext_shl_moreOneUse_sext(ptr %array1, i32 %a, i32 %b) {
 ; ZBA-NEXT:    sw a2, 24(a0)
 ; ZBA-NEXT:    sd a3, 140(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_sext_shl_moreOneUse_sext:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    sext.w a1, a1
+; XANDESPERF-NEXT:    addi a3, a1, 5
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    sw a2, 20(a0)
+; XANDESPERF-NEXT:    sw a2, 24(a0)
+; XANDESPERF-NEXT:    sd a3, 140(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 5
   %idxprom = sext i32 %add to i64
@@ -147,6 +181,20 @@ define void @add_sext_shl_moreOneUse_add_inSelect(ptr %array1, i32 signext  %a,
 ; ZBA-NEXT:    sw a5, 24(a0)
 ; ZBA-NEXT:    sw a4, 140(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_sext_shl_moreOneUse_add_inSelect:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a4, a1, 5
+; XANDESPERF-NEXT:    mv a5, a4
+; XANDESPERF-NEXT:    bgtz a3, .LBB3_2
+; XANDESPERF-NEXT:  # %bb.1: # %entry
+; XANDESPERF-NEXT:    mv a5, a2
+; XANDESPERF-NEXT:  .LBB3_2: # %entry
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    sw a5, 20(a0)
+; XANDESPERF-NEXT:    sw a5, 24(a0)
+; XANDESPERF-NEXT:    sw a4, 140(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 5
   %cmp = icmp sgt i32 %x, 0
@@ -200,6 +248,23 @@ define void @add_sext_shl_moreOneUse_add_inSelect_addexceedsign12(ptr %array1, i
 ; ZBA-NEXT:    sw a1, 4(a0)
 ; ZBA-NEXT:    sw a4, 120(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_sext_shl_moreOneUse_add_inSelect_addexceedsign12:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a4, a1, 2047
+; XANDESPERF-NEXT:    lui a5, 2
+; XANDESPERF-NEXT:    addi a4, a4, 1
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    add a0, a0, a5
+; XANDESPERF-NEXT:    mv a1, a4
+; XANDESPERF-NEXT:    bgtz a3, .LBB4_2
+; XANDESPERF-NEXT:  # %bb.1: # %entry
+; XANDESPERF-NEXT:    mv a1, a2
+; XANDESPERF-NEXT:  .LBB4_2: # %entry
+; XANDESPERF-NEXT:    sw a1, 0(a0)
+; XANDESPERF-NEXT:    sw a1, 4(a0)
+; XANDESPERF-NEXT:    sw a4, 120(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 2048
   %cmp = icmp sgt i32 %x, 0
@@ -245,6 +310,20 @@ define void @add_shl_moreOneUse_inSelect(ptr %array1, i64 %a, i64 %b, i64 %x) {
 ; ZBA-NEXT:    sd a5, 48(a0)
 ; ZBA-NEXT:    sd a4, 280(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_inSelect:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a4, a1, 5
+; XANDESPERF-NEXT:    mv a5, a4
+; XANDESPERF-NEXT:    bgtz a3, .LBB5_2
+; XANDESPERF-NEXT:  # %bb.1: # %entry
+; XANDESPERF-NEXT:    mv a5, a2
+; XANDESPERF-NEXT:  .LBB5_2: # %entry
+; XANDESPERF-NEXT:    nds.lea.d a0, a0, a1
+; XANDESPERF-NEXT:    sd a5, 40(a0)
+; XANDESPERF-NEXT:    sd a5, 48(a0)
+; XANDESPERF-NEXT:    sd a4, 280(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i64 %a, 5
   %cmp = icmp sgt i64 %x, 0
@@ -274,6 +353,12 @@ define i64 @add_shl_moreOneUse_sh1add(i64 %x) {
 ; ZBA-NEXT:    ori a0, a0, 1
 ; ZBA-NEXT:    sh1add a0, a0, a0
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_sh1add:
+; XANDESPERF:       # %bb.0:
+; XANDESPERF-NEXT:    ori a0, a0, 1
+; XANDESPERF-NEXT:    nds.lea.h a0, a0, a0
+; XANDESPERF-NEXT:    ret
   %or = or i64 %x, 1
   %mul = shl i64 %or, 1
   %add = add i64 %mul, %or
@@ -294,6 +379,12 @@ define i64 @add_shl_moreOneUse_sh2add(i64 %x) {
 ; ZBA-NEXT:    ori a0, a0, 1
 ; ZBA-NEXT:    sh2add a0, a0, a0
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_sh2add:
+; XANDESPERF:       # %bb.0:
+; XANDESPERF-NEXT:    ori a0, a0, 1
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a0
+; XANDESPERF-NEXT:    ret
   %or = or i64 %x, 1
   %mul = shl i64 %or, 2
   %add = add i64 %mul, %or
@@ -314,6 +405,12 @@ define i64 @add_shl_moreOneUse_sh3add(i64 %x) {
 ; ZBA-NEXT:    ori a0, a0, 1
 ; ZBA-NEXT:    sh3add a0, a0, a0
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_sh3add:
+; XANDESPERF:       # %bb.0:
+; XANDESPERF-NEXT:    ori a0, a0, 1
+; XANDESPERF-NEXT:    nds.lea.d a0, a0, a0
+; XANDESPERF-NEXT:    ret
   %or = or i64 %x, 1
   %mul = shl i64 %or, 3
   %add = add i64 %mul, %or
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/add_shl_constant.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/add_shl_constant.ll
index a4da9e268364..bb0456f7fe1a 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/add_shl_constant.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/add_shl_constant.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=riscv32  < %s | FileCheck -check-prefixes=RV32,NO-ZBA %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zba  < %s | FileCheck -check-prefixes=RV32,ZBA %s
+; RUN: llc -mtriple=riscv32 -mattr=+xandesperf  < %s | FileCheck -check-prefixes=RV32,XANDESPERF %s
 
 define i32 @add_shl_oneUse(i32 %x, i32 %y) nounwind {
 ; NO-ZBA-LABEL: add_shl_oneUse:
@@ -15,6 +16,12 @@ define i32 @add_shl_oneUse(i32 %x, i32 %y) nounwind {
 ; ZBA-NEXT:    addi a0, a0, 123
 ; ZBA-NEXT:    sh3add a0, a0, a1
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_oneUse:
+; XANDESPERF:       # %bb.0:
+; XANDESPERF-NEXT:    addi a0, a0, 123
+; XANDESPERF-NEXT:    nds.lea.d a0, a1, a0
+; XANDESPERF-NEXT:    ret
   %add.0 = add i32 %x, 123
   %shl = shl i32 %add.0, 3
   %add.1 = add i32 %shl, %y
@@ -40,6 +47,15 @@ define void @add_shl_moreOneUse_inStore(ptr %array1, i32 %a, i32 %b)  {
 ; ZBA-NEXT:    sw a2, 24(a0)
 ; ZBA-NEXT:    sw a3, 140(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_inStore:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a3, a1, 5
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    sw a2, 20(a0)
+; XANDESPERF-NEXT:    sw a2, 24(a0)
+; XANDESPERF-NEXT:    sw a3, 140(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 5
   %arrayidx = getelementptr inbounds i32, ptr %array1, i32 %add
@@ -77,6 +93,18 @@ define void @add_shl_moreOneUse_inStore_addexceedsign12(ptr %array1, i32 %a, i32
 ; ZBA-NEXT:    sw a3, 4(a0)
 ; ZBA-NEXT:    sw a2, 120(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_inStore_addexceedsign12:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a3, a1, 2047
+; XANDESPERF-NEXT:    lui a4, 2
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    addi a3, a3, 1
+; XANDESPERF-NEXT:    add a0, a0, a4
+; XANDESPERF-NEXT:    sw a2, 0(a0)
+; XANDESPERF-NEXT:    sw a3, 4(a0)
+; XANDESPERF-NEXT:    sw a2, 120(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 2048
   %arrayidx = getelementptr inbounds i32, ptr %array1, i32 %add
@@ -118,6 +146,20 @@ define void @add_shl_moreOneUse_inSelect(ptr %array1, i32 %a, i32 %b, i32 %x) {
 ; ZBA-NEXT:    sw a5, 24(a0)
 ; ZBA-NEXT:    sw a4, 140(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_inSelect:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a4, a1, 5
+; XANDESPERF-NEXT:    mv a5, a4
+; XANDESPERF-NEXT:    bgtz a3, .LBB3_2
+; XANDESPERF-NEXT:  # %bb.1: # %entry
+; XANDESPERF-NEXT:    mv a5, a2
+; XANDESPERF-NEXT:  .LBB3_2: # %entry
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    sw a5, 20(a0)
+; XANDESPERF-NEXT:    sw a5, 24(a0)
+; XANDESPERF-NEXT:    sw a4, 140(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 5
   %cmp = icmp sgt i32 %x, 0
@@ -167,6 +209,23 @@ define void @add_shl_moreOneUse_inSelect_addexceedsign12(ptr %array1, i32 %a, i3
 ; ZBA-NEXT:    sw a5, 4(a0)
 ; ZBA-NEXT:    sw a4, 120(a0)
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_inSelect_addexceedsign12:
+; XANDESPERF:       # %bb.0: # %entry
+; XANDESPERF-NEXT:    addi a4, a1, 2047
+; XANDESPERF-NEXT:    addi a4, a4, 1
+; XANDESPERF-NEXT:    mv a5, a4
+; XANDESPERF-NEXT:    bgtz a3, .LBB4_2
+; XANDESPERF-NEXT:  # %bb.1: # %entry
+; XANDESPERF-NEXT:    mv a5, a2
+; XANDESPERF-NEXT:  .LBB4_2: # %entry
+; XANDESPERF-NEXT:    lui a2, 2
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; XANDESPERF-NEXT:    add a0, a0, a2
+; XANDESPERF-NEXT:    sw a5, 0(a0)
+; XANDESPERF-NEXT:    sw a5, 4(a0)
+; XANDESPERF-NEXT:    sw a4, 120(a0)
+; XANDESPERF-NEXT:    ret
 entry:
   %add = add nsw i32 %a, 2048
   %cmp = icmp sgt i32 %x, 0
@@ -195,6 +254,12 @@ define i32 @add_shl_moreOneUse_sh1add(i32 %x) {
 ; ZBA-NEXT:    ori a0, a0, 1
 ; ZBA-NEXT:    sh1add a0, a0, a0
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_sh1add:
+; XANDESPERF:       # %bb.0:
+; XANDESPERF-NEXT:    ori a0, a0, 1
+; XANDESPERF-NEXT:    nds.lea.h a0, a0, a0
+; XANDESPERF-NEXT:    ret
   %or = or i32 %x, 1
   %mul = shl i32 %or, 1
   %add = add i32 %mul, %or
@@ -215,6 +280,12 @@ define i32 @add_shl_moreOneUse_sh2add(i32 %x) {
 ; ZBA-NEXT:    ori a0, a0, 1
 ; ZBA-NEXT:    sh2add a0, a0, a0
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_sh2add:
+; XANDESPERF:       # %bb.0:
+; XANDESPERF-NEXT:    ori a0, a0, 1
+; XANDESPERF-NEXT:    nds.lea.w a0, a0, a0
+; XANDESPERF-NEXT:    ret
   %or = or i32 %x, 1
   %mul = shl i32 %or, 2
   %add = add i32 %mul, %or
@@ -235,6 +306,12 @@ define i32 @add_shl_moreOneUse_sh3add(i32 %x) {
 ; ZBA-NEXT:    ori a0, a0, 1
 ; ZBA-NEXT:    sh3add a0, a0, a0
 ; ZBA-NEXT:    ret
+;
+; XANDESPERF-LABEL: add_shl_moreOneUse_sh3add:
+; XANDESPERF:       # %bb.0:
+; XANDESPERF-NEXT:    ori a0, a0, 1
+; XANDESPERF-NEXT:    nds.lea.d a0, a0, a0
+; XANDESPERF-NEXT:    ret
   %or = or i32 %x, 1
   %mul = shl i32 %or, 3
   %add = add i32 %mul, %or
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/addimm-mulimm.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
index 8e445511b611..aac355e3f055 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
@@ -153,7 +153,7 @@ define i64 @add_mul_combine_accept_b3(i64 %x) {
 ; RV64IMB-NEXT:    slli a0, a0, 5
 ; RV64IMB-NEXT:    sub a0, a0, a1
 ; RV64IMB-NEXT:    lui a1, 50
-; RV64IMB-NEXT:    addiw a1, a1, 1119
+; RV64IMB-NEXT:    addi a1, a1, 1119
 ; RV64IMB-NEXT:    add a0, a0, a1
 ; RV64IMB-NEXT:    ret
   %tmp0 = add i64 %x, 8953
@@ -685,7 +685,7 @@ define i64 @mul3000_add8990_c(i64 %x) {
 ; RV64IMB:       # %bb.0:
 ; RV64IMB-NEXT:    addi a0, a0, 3
 ; RV64IMB-NEXT:    lui a1, 1
-; RV64IMB-NEXT:    addiw a1, a1, -1096
+; RV64IMB-NEXT:    addi a1, a1, -1096
 ; RV64IMB-NEXT:    mul a0, a0, a1
 ; RV64IMB-NEXT:    addi a0, a0, -10
 ; RV64IMB-NEXT:    ret
@@ -761,7 +761,7 @@ define i64 @mul3000_sub8990_c(i64 %x) {
 ; RV64IMB:       # %bb.0:
 ; RV64IMB-NEXT:    addi a0, a0, -3
 ; RV64IMB-NEXT:    lui a1, 1
-; RV64IMB-NEXT:    addiw a1, a1, -1096
+; RV64IMB-NEXT:    addi a1, a1, -1096
 ; RV64IMB-NEXT:    mul a0, a0, a1
 ; RV64IMB-NEXT:    addi a0, a0, 10
 ; RV64IMB-NEXT:    ret
@@ -837,7 +837,7 @@ define i64 @mulneg3000_add8990_c(i64 %x) {
 ; RV64IMB:       # %bb.0:
 ; RV64IMB-NEXT:    addi a0, a0, -3
 ; RV64IMB-NEXT:    lui a1, 1048575
-; RV64IMB-NEXT:    addiw a1, a1, 1096
+; RV64IMB-NEXT:    addi a1, a1, 1096
 ; RV64IMB-NEXT:    mul a0, a0, a1
 ; RV64IMB-NEXT:    addi a0, a0, -10
 ; RV64IMB-NEXT:    ret
@@ -914,7 +914,7 @@ define i64 @mulneg3000_sub8990_c(i64 %x) {
 ; RV64IMB:       # %bb.0:
 ; RV64IMB-NEXT:    addi a0, a0, 3
 ; RV64IMB-NEXT:    lui a1, 1048575
-; RV64IMB-NEXT:    addiw a1, a1, 1096
+; RV64IMB-NEXT:    addi a1, a1, 1096
 ; RV64IMB-NEXT:    mul a0, a0, a1
 ; RV64IMB-NEXT:    addi a0, a0, 10
 ; RV64IMB-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/alu16.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/alu16.ll
index 41f26526ef03..b5a2524410c0 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/alu16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/alu16.ll
@@ -286,7 +286,7 @@ define i16 @sltu(i16 %a, i16 %b) nounwind {
 ; RV64I-LABEL: sltu:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sltu a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/and-shl.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/and-shl.ll
index c3cb5d8e2e37..26b4a90d88fb 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/and-shl.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/and-shl.ll
@@ -77,3 +77,21 @@ define i32 @and_0xfff_shl_2_multi_use(i32 %x) {
   %r = add i32 %a, %s
   ret i32 %r
 }
+
+define i64 @and_0xfff_shl_33(i64 %x) {
+; RV32I-LABEL: and_0xfff_shl_33:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 20
+; RV32I-NEXT:    srli a1, a0, 19
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: and_0xfff_shl_33:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 52
+; RV64I-NEXT:    srli a0, a0, 19
+; RV64I-NEXT:    ret
+  %a = and i64 %x, 4095
+  %s = shl i64 %a, 33
+  ret i64 %s
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-rmw.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-rmw.ll
index 1e5acd2575b8..b0510f85c49a 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-rmw.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-rmw.ll
@@ -11018,7 +11018,7 @@ define i16 @atomicrmw_xchg_minus_1_i16_monotonic(ptr %a) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    li a2, 0
 ; RV64I-NEXT:    call __atomic_exchange_2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -11102,7 +11102,7 @@ define i16 @atomicrmw_xchg_minus_1_i16_acquire(ptr %a) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    li a2, 2
 ; RV64I-NEXT:    call __atomic_exchange_2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -11208,7 +11208,7 @@ define i16 @atomicrmw_xchg_minus_1_i16_release(ptr %a) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    li a2, 3
 ; RV64I-NEXT:    call __atomic_exchange_2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -11314,7 +11314,7 @@ define i16 @atomicrmw_xchg_minus_1_i16_acq_rel(ptr %a) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    li a2, 4
 ; RV64I-NEXT:    call __atomic_exchange_2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -11420,7 +11420,7 @@ define i16 @atomicrmw_xchg_minus_1_i16_seq_cst(ptr %a) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    li a2, 5
 ; RV64I-NEXT:    call __atomic_exchange_2
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -17923,7 +17923,7 @@ define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB120_2
 ; RV64I-NEXT:  .LBB120_1: # %atomicrmw.start
@@ -18125,7 +18125,7 @@ define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB121_2
 ; RV64I-NEXT:  .LBB121_1: # %atomicrmw.start
@@ -18377,7 +18377,7 @@ define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB122_2
 ; RV64I-NEXT:  .LBB122_1: # %atomicrmw.start
@@ -18629,7 +18629,7 @@ define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB123_2
 ; RV64I-NEXT:  .LBB123_1: # %atomicrmw.start
@@ -18856,7 +18856,7 @@ define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB124_2
 ; RV64I-NEXT:  .LBB124_1: # %atomicrmw.start
@@ -19033,7 +19033,7 @@ define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB125_2
 ; RV64I-NEXT:  .LBB125_1: # %atomicrmw.start
@@ -19235,7 +19235,7 @@ define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB126_2
 ; RV64I-NEXT:  .LBB126_1: # %atomicrmw.start
@@ -19487,7 +19487,7 @@ define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB127_2
 ; RV64I-NEXT:  .LBB127_1: # %atomicrmw.start
@@ -19739,7 +19739,7 @@ define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB128_2
 ; RV64I-NEXT:  .LBB128_1: # %atomicrmw.start
@@ -19966,7 +19966,7 @@ define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB129_2
 ; RV64I-NEXT:  .LBB129_1: # %atomicrmw.start
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-signext.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-signext.ll
index b9702e9fe0fc..bebc097deb19 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-signext.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/atomic-signext.ll
@@ -2023,7 +2023,7 @@ define signext i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB23_2
 ; RV64I-NEXT:  .LBB23_1: # %atomicrmw.start
@@ -2171,7 +2171,7 @@ define signext i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB24_2
 ; RV64I-NEXT:  .LBB24_1: # %atomicrmw.start
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
index 2db6f80f4fd6..27704d107f93 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
@@ -292,7 +292,7 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:  .LBB1_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -331,7 +331,7 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
 ; RV64IA-NEXT:    slli a5, a0, 3
 ; RV64IA-NEXT:    lui a3, 16
 ; RV64IA-NEXT:    andi a0, a5, 24
-; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    addi a3, a3, -1
 ; RV64IA-NEXT:    lw a4, 0(a2)
 ; RV64IA-NEXT:    sllw a5, a3, a5
 ; RV64IA-NEXT:    not a5, a5
@@ -955,7 +955,7 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    lui s1, 16
-; RV64I-NEXT:    addiw s1, s1, -1
+; RV64I-NEXT:    addi s1, s1, -1
 ; RV64I-NEXT:    and s2, a1, s1
 ; RV64I-NEXT:  .LBB5_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -992,7 +992,7 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
 ; RV64IA-NEXT:    slli a5, a0, 3
 ; RV64IA-NEXT:    lui a3, 16
 ; RV64IA-NEXT:    andi a0, a5, 24
-; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    addi a3, a3, -1
 ; RV64IA-NEXT:    lw a4, 0(a2)
 ; RV64IA-NEXT:    sllw a5, a3, a5
 ; RV64IA-NEXT:    not a5, a5
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
index ae1db4f1d62d..ada1933d91d6 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
@@ -274,7 +274,7 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    lui s1, 16
-; RV64I-NEXT:    addiw s1, s1, -1
+; RV64I-NEXT:    addi s1, s1, -1
 ; RV64I-NEXT:    and s2, a1, s1
 ; RV64I-NEXT:  .LBB1_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -311,7 +311,7 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; RV64IA-NEXT:    slli a5, a0, 3
 ; RV64IA-NEXT:    lui a3, 16
 ; RV64IA-NEXT:    andi a0, a5, 24
-; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    addi a3, a3, -1
 ; RV64IA-NEXT:    lw a4, 0(a2)
 ; RV64IA-NEXT:    sllw a5, a3, a5
 ; RV64IA-NEXT:    not a5, a5
@@ -1001,7 +1001,7 @@ define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
 ; RV64I-NEXT:    lui s2, 16
-; RV64I-NEXT:    addiw s2, s2, -1
+; RV64I-NEXT:    addi s2, s2, -1
 ; RV64I-NEXT:    and s3, s0, s2
 ; RV64I-NEXT:    j .LBB5_2
 ; RV64I-NEXT:  .LBB5_1: # %atomicrmw.start
@@ -1048,7 +1048,7 @@ define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) {
 ; RV64IA-NEXT:    slli a5, a0, 3
 ; RV64IA-NEXT:    lui a3, 16
 ; RV64IA-NEXT:    andi a0, a5, 24
-; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    addi a3, a3, -1
 ; RV64IA-NEXT:    lw a4, 0(a2)
 ; RV64IA-NEXT:    sllw a5, a3, a5
 ; RV64IA-NEXT:    not a5, a5
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/avgceilu.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/avgceilu.ll
index 735bd19909d5..1c1d1cbfd12c 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/avgceilu.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/avgceilu.ll
@@ -75,7 +75,7 @@ define i16 @test_fixed_i16(i16 %a0, i16 %a1) nounwind {
 ; RV64I-LABEL: test_fixed_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    add a0, a0, a1
@@ -104,7 +104,7 @@ define i16 @test_ext_i16(i16 %a0, i16 %a1) nounwind {
 ; RV64I-LABEL: test_ext_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    add a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/avgflooru.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/avgflooru.ll
index 8a69c9393c87..2e56f3359434 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/avgflooru.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/avgflooru.ll
@@ -69,7 +69,7 @@ define i16 @test_fixed_i16(i16 %a0, i16 %a1) nounwind {
 ; RV64I-LABEL: test_fixed_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    add a0, a0, a1
@@ -96,7 +96,7 @@ define i16 @test_ext_i16(i16 %a0, i16 %a1) nounwind {
 ; RV64I-LABEL: test_ext_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    add a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/bittest.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/bittest.ll
index d69ab0550a03..fa6892be44d9 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/bittest.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/bittest.ll
@@ -271,19 +271,12 @@ define i1 @bittest_constant_by_var_shr_i32(i32 signext %b) nounwind {
 ; RV64I-NEXT:    andi a0, a0, 1
 ; RV64I-NEXT:    ret
 ;
-; RV32ZBS-LABEL: bittest_constant_by_var_shr_i32:
-; RV32ZBS:       # %bb.0:
-; RV32ZBS-NEXT:    lui a1, 301408
-; RV32ZBS-NEXT:    addi a1, a1, 722
-; RV32ZBS-NEXT:    bext a0, a1, a0
-; RV32ZBS-NEXT:    ret
-;
-; RV64ZBS-LABEL: bittest_constant_by_var_shr_i32:
-; RV64ZBS:       # %bb.0:
-; RV64ZBS-NEXT:    lui a1, 301408
-; RV64ZBS-NEXT:    addiw a1, a1, 722
-; RV64ZBS-NEXT:    bext a0, a1, a0
-; RV64ZBS-NEXT:    ret
+; ZBS-LABEL: bittest_constant_by_var_shr_i32:
+; ZBS:       # %bb.0:
+; ZBS-NEXT:    lui a1, 301408
+; ZBS-NEXT:    addi a1, a1, 722
+; ZBS-NEXT:    bext a0, a1, a0
+; ZBS-NEXT:    ret
 ;
 ; RV32XTHEADBS-LABEL: bittest_constant_by_var_shr_i32:
 ; RV32XTHEADBS:       # %bb.0:
@@ -324,19 +317,12 @@ define i1 @bittest_constant_by_var_shl_i32(i32 signext %b) nounwind {
 ; RV64I-NEXT:    andi a0, a0, 1
 ; RV64I-NEXT:    ret
 ;
-; RV32ZBS-LABEL: bittest_constant_by_var_shl_i32:
-; RV32ZBS:       # %bb.0:
-; RV32ZBS-NEXT:    lui a1, 301408
-; RV32ZBS-NEXT:    addi a1, a1, 722
-; RV32ZBS-NEXT:    bext a0, a1, a0
-; RV32ZBS-NEXT:    ret
-;
-; RV64ZBS-LABEL: bittest_constant_by_var_shl_i32:
-; RV64ZBS:       # %bb.0:
-; RV64ZBS-NEXT:    lui a1, 301408
-; RV64ZBS-NEXT:    addiw a1, a1, 722
-; RV64ZBS-NEXT:    bext a0, a1, a0
-; RV64ZBS-NEXT:    ret
+; ZBS-LABEL: bittest_constant_by_var_shl_i32:
+; ZBS:       # %bb.0:
+; ZBS-NEXT:    lui a1, 301408
+; ZBS-NEXT:    addi a1, a1, 722
+; ZBS-NEXT:    bext a0, a1, a0
+; ZBS-NEXT:    ret
 ;
 ; RV32XTHEADBS-LABEL: bittest_constant_by_var_shl_i32:
 ; RV32XTHEADBS:       # %bb.0:
@@ -374,7 +360,7 @@ define i1 @bittest_constant_by_var_shr_i64(i64 %b) nounwind {
 ; RV64I-LABEL: bittest_constant_by_var_shr_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 301408
-; RV64I-NEXT:    addiw a1, a1, 722
+; RV64I-NEXT:    addi a1, a1, 722
 ; RV64I-NEXT:    srl a0, a1, a0
 ; RV64I-NEXT:    andi a0, a0, 1
 ; RV64I-NEXT:    ret
@@ -382,14 +368,14 @@ define i1 @bittest_constant_by_var_shr_i64(i64 %b) nounwind {
 ; RV64ZBS-LABEL: bittest_constant_by_var_shr_i64:
 ; RV64ZBS:       # %bb.0:
 ; RV64ZBS-NEXT:    lui a1, 301408
-; RV64ZBS-NEXT:    addiw a1, a1, 722
+; RV64ZBS-NEXT:    addi a1, a1, 722
 ; RV64ZBS-NEXT:    bext a0, a1, a0
 ; RV64ZBS-NEXT:    ret
 ;
 ; RV64XTHEADBS-LABEL: bittest_constant_by_var_shr_i64:
 ; RV64XTHEADBS:       # %bb.0:
 ; RV64XTHEADBS-NEXT:    lui a1, 301408
-; RV64XTHEADBS-NEXT:    addiw a1, a1, 722
+; RV64XTHEADBS-NEXT:    addi a1, a1, 722
 ; RV64XTHEADBS-NEXT:    srl a0, a1, a0
 ; RV64XTHEADBS-NEXT:    andi a0, a0, 1
 ; RV64XTHEADBS-NEXT:    ret
@@ -414,7 +400,7 @@ define i1 @bittest_constant_by_var_shl_i64(i64 %b) nounwind {
 ; RV64I-LABEL: bittest_constant_by_var_shl_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 301408
-; RV64I-NEXT:    addiw a1, a1, 722
+; RV64I-NEXT:    addi a1, a1, 722
 ; RV64I-NEXT:    srl a0, a1, a0
 ; RV64I-NEXT:    andi a0, a0, 1
 ; RV64I-NEXT:    ret
@@ -422,14 +408,14 @@ define i1 @bittest_constant_by_var_shl_i64(i64 %b) nounwind {
 ; RV64ZBS-LABEL: bittest_constant_by_var_shl_i64:
 ; RV64ZBS:       # %bb.0:
 ; RV64ZBS-NEXT:    lui a1, 301408
-; RV64ZBS-NEXT:    addiw a1, a1, 722
+; RV64ZBS-NEXT:    addi a1, a1, 722
 ; RV64ZBS-NEXT:    bext a0, a1, a0
 ; RV64ZBS-NEXT:    ret
 ;
 ; RV64XTHEADBS-LABEL: bittest_constant_by_var_shl_i64:
 ; RV64XTHEADBS:       # %bb.0:
 ; RV64XTHEADBS-NEXT:    lui a1, 301408
-; RV64XTHEADBS-NEXT:    addiw a1, a1, 722
+; RV64XTHEADBS-NEXT:    addi a1, a1, 722
 ; RV64XTHEADBS-NEXT:    srl a0, a1, a0
 ; RV64XTHEADBS-NEXT:    andi a0, a0, 1
 ; RV64XTHEADBS-NEXT:    ret
@@ -462,7 +448,7 @@ define void @bittest_switch(i32 signext %0) {
 ; RV64I-NEXT:    bltu a1, a0, .LBB14_3
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    lui a1, 2048
-; RV64I-NEXT:    addiw a1, a1, 51
+; RV64I-NEXT:    addi a1, a1, 51
 ; RV64I-NEXT:    slli a1, a1, 8
 ; RV64I-NEXT:    srl a0, a1, a0
 ; RV64I-NEXT:    andi a0, a0, 1
@@ -492,7 +478,7 @@ define void @bittest_switch(i32 signext %0) {
 ; RV64ZBS-NEXT:    bltu a1, a0, .LBB14_3
 ; RV64ZBS-NEXT:  # %bb.1:
 ; RV64ZBS-NEXT:    lui a1, 2048
-; RV64ZBS-NEXT:    addiw a1, a1, 51
+; RV64ZBS-NEXT:    addi a1, a1, 51
 ; RV64ZBS-NEXT:    slli a1, a1, 8
 ; RV64ZBS-NEXT:    bext a0, a1, a0
 ; RV64ZBS-NEXT:    beqz a0, .LBB14_3
@@ -522,7 +508,7 @@ define void @bittest_switch(i32 signext %0) {
 ; RV64XTHEADBS-NEXT:    bltu a1, a0, .LBB14_3
 ; RV64XTHEADBS-NEXT:  # %bb.1:
 ; RV64XTHEADBS-NEXT:    lui a1, 2048
-; RV64XTHEADBS-NEXT:    addiw a1, a1, 51
+; RV64XTHEADBS-NEXT:    addi a1, a1, 51
 ; RV64XTHEADBS-NEXT:    slli a1, a1, 8
 ; RV64XTHEADBS-NEXT:    srl a0, a1, a0
 ; RV64XTHEADBS-NEXT:    andi a0, a0, 1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/branch-relaxation-rv64.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/branch-relaxation-rv64.ll
index 23a862ec580e..9c794a73dc63 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/branch-relaxation-rv64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/branch-relaxation-rv64.ll
@@ -189,7 +189,7 @@ define void @relax_jal_spill_64() {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    beq t5, t6, .LBB2_1
 ; CHECK-NEXT:  # %bb.3:
-; CHECK-NEXT:    sd s11, 0(sp)
+; CHECK-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    jump .LBB2_4, s11
 ; CHECK-NEXT:  .LBB2_1: # %branch_1
 ; CHECK-NEXT:    #APP
@@ -197,7 +197,7 @@ define void @relax_jal_spill_64() {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    j .LBB2_2
 ; CHECK-NEXT:  .LBB2_4: # %branch_2
-; CHECK-NEXT:    ld s11, 0(sp)
+; CHECK-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:  .LBB2_2: # %branch_2
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    # reg use ra
@@ -419,7 +419,7 @@ define void @relax_jal_spill_64_adjust_spill_slot() {
 ; CHECK-NEXT:    addi s0, sp, 2032
 ; CHECK-NEXT:    .cfi_def_cfa s0, 0
 ; CHECK-NEXT:    lui a0, 2
-; CHECK-NEXT:    addiw a0, a0, -2032
+; CHECK-NEXT:    addi a0, a0, -2032
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    srli a0, sp, 12
 ; CHECK-NEXT:    slli sp, a0, 12
@@ -509,7 +509,7 @@ define void @relax_jal_spill_64_adjust_spill_slot() {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    beq t5, t6, .LBB3_1
 ; CHECK-NEXT:  # %bb.3:
-; CHECK-NEXT:    sd s11, 0(sp)
+; CHECK-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    jump .LBB3_4, s11
 ; CHECK-NEXT:  .LBB3_1: # %branch_1
 ; CHECK-NEXT:    #APP
@@ -517,7 +517,7 @@ define void @relax_jal_spill_64_adjust_spill_slot() {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    j .LBB3_2
 ; CHECK-NEXT:  .LBB3_4: # %branch_2
-; CHECK-NEXT:    ld s11, 0(sp)
+; CHECK-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:  .LBB3_2: # %branch_2
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    # reg use ra
@@ -825,7 +825,7 @@ define void @relax_jal_spill_64_restore_block_correspondence() {
 ; CHECK-NEXT:    bne t5, t6, .LBB4_2
 ; CHECK-NEXT:    j .LBB4_1
 ; CHECK-NEXT:  .LBB4_8: # %dest_1
-; CHECK-NEXT:    ld s11, 0(sp)
+; CHECK-NEXT:    ld s11, 0(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:  .LBB4_1: # %dest_1
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    # dest 1
@@ -962,7 +962,7 @@ define void @relax_jal_spill_64_restore_block_correspondence() {
 ; CHECK-NEXT:    .zero 1048576
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  # %bb.7: # %space
-; CHECK-NEXT:    sd s11, 0(sp)
+; CHECK-NEXT:    sd s11, 0(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    jump .LBB4_8, s11
 entry:
   %ra = call i64 asm sideeffect "addi ra, x0, 1", "={ra}"()
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/bswap-bitreverse.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/bswap-bitreverse.ll
index 35ab6c8d0cab..1605e686e917 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/bswap-bitreverse.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/bswap-bitreverse.ll
@@ -73,7 +73,7 @@ define i32 @test_bswap_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 8
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    srliw a3, a0, 24
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a2, a0, a2
 ; RV64I-NEXT:    or a1, a1, a3
@@ -129,7 +129,7 @@ define i64 @test_bswap_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 56
 ; RV64I-NEXT:    srli a4, a0, 24
 ; RV64I-NEXT:    lui a5, 4080
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srli a3, a0, 8
@@ -213,7 +213,7 @@ define i7 @test_bitreverse_i7(i7 %a) nounwind {
 ; RV64I-NEXT:    lui a5, 4080
 ; RV64I-NEXT:    srli a6, a0, 8
 ; RV64I-NEXT:    srliw a7, a0, 24
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    lui a3, 61681
@@ -224,8 +224,8 @@ define i7 @test_bitreverse_i7(i7 %a) nounwind {
 ; RV64I-NEXT:    lui a6, 209715
 ; RV64I-NEXT:    and a5, a0, a5
 ; RV64I-NEXT:    slli a7, a7, 32
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a6, a6, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a6, a6, 819
 ; RV64I-NEXT:    slli a5, a5, 24
 ; RV64I-NEXT:    or a5, a5, a7
 ; RV64I-NEXT:    slli a7, a3, 32
@@ -293,8 +293,8 @@ define i7 @test_bitreverse_i7(i7 %a) nounwind {
 ; RV64ZBB-NEXT:    rev8 a0, a0
 ; RV64ZBB-NEXT:    lui a1, 61681
 ; RV64ZBB-NEXT:    lui a2, 209715
-; RV64ZBB-NEXT:    addiw a1, a1, -241
-; RV64ZBB-NEXT:    addiw a2, a2, 819
+; RV64ZBB-NEXT:    addi a1, a1, -241
+; RV64ZBB-NEXT:    addi a2, a2, 819
 ; RV64ZBB-NEXT:    slli a3, a1, 32
 ; RV64ZBB-NEXT:    add a1, a1, a3
 ; RV64ZBB-NEXT:    slli a3, a2, 32
@@ -464,20 +464,20 @@ define i16 @test_bitreverse_i16(i16 %a) nounwind {
 ; RV64I-NEXT:    slli a0, a0, 48
 ; RV64I-NEXT:    lui a2, 1
 ; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 4
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 3
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    slli a0, a0, 4
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 5
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    slli a0, a0, 2
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 1
@@ -518,18 +518,18 @@ define i16 @test_bitreverse_i16(i16 %a) nounwind {
 ; RV64ZBB-NEXT:    rev8 a0, a0
 ; RV64ZBB-NEXT:    lui a1, 15
 ; RV64ZBB-NEXT:    srli a2, a0, 44
-; RV64ZBB-NEXT:    addiw a1, a1, 240
+; RV64ZBB-NEXT:    addi a1, a1, 240
 ; RV64ZBB-NEXT:    and a1, a2, a1
 ; RV64ZBB-NEXT:    lui a2, 3
 ; RV64ZBB-NEXT:    srli a0, a0, 52
-; RV64ZBB-NEXT:    addiw a2, a2, 819
+; RV64ZBB-NEXT:    addi a2, a2, 819
 ; RV64ZBB-NEXT:    andi a0, a0, -241
 ; RV64ZBB-NEXT:    or a0, a0, a1
 ; RV64ZBB-NEXT:    srli a1, a0, 2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    lui a2, 5
-; RV64ZBB-NEXT:    addiw a2, a2, 1365
+; RV64ZBB-NEXT:    addi a2, a2, 1365
 ; RV64ZBB-NEXT:    slli a0, a0, 2
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 1
@@ -598,7 +598,7 @@ define i32 @test_bitreverse_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 8
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    srliw a3, a0, 24
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a2, a0, a2
 ; RV64I-NEXT:    slliw a0, a0, 24
@@ -607,14 +607,14 @@ define i32 @test_bitreverse_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    slli a2, a2, 8
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    srli a1, a0, 4
 ; RV64I-NEXT:    and a0, a0, a3
 ; RV64I-NEXT:    and a1, a1, a3
 ; RV64I-NEXT:    lui a3, 349525
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, 1365
 ; RV64I-NEXT:    slliw a0, a0, 4
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 2
@@ -660,21 +660,21 @@ define i32 @test_bitreverse_i32(i32 %a) nounwind {
 ; RV64ZBB-NEXT:    rev8 a0, a0
 ; RV64ZBB-NEXT:    lui a1, 61681
 ; RV64ZBB-NEXT:    srli a2, a0, 36
-; RV64ZBB-NEXT:    addiw a1, a1, -241
+; RV64ZBB-NEXT:    addi a1, a1, -241
 ; RV64ZBB-NEXT:    and a1, a2, a1
 ; RV64ZBB-NEXT:    lui a2, 986895
 ; RV64ZBB-NEXT:    srli a0, a0, 28
 ; RV64ZBB-NEXT:    addi a2, a2, 240
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    lui a2, 209715
-; RV64ZBB-NEXT:    addiw a2, a2, 819
+; RV64ZBB-NEXT:    addi a2, a2, 819
 ; RV64ZBB-NEXT:    sext.w a0, a0
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    lui a2, 349525
-; RV64ZBB-NEXT:    addiw a2, a2, 1365
+; RV64ZBB-NEXT:    addi a2, a2, 1365
 ; RV64ZBB-NEXT:    slliw a0, a0, 2
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 1
@@ -771,7 +771,7 @@ define i64 @test_bitreverse_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a6, a0, 8
 ; RV64I-NEXT:    srliw a7, a0, 24
 ; RV64I-NEXT:    lui t0, 61681
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    lui a3, 209715
@@ -782,9 +782,9 @@ define i64 @test_bitreverse_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    lui a6, 349525
 ; RV64I-NEXT:    and a5, a0, a5
 ; RV64I-NEXT:    slli a7, a7, 32
-; RV64I-NEXT:    addiw t0, t0, -241
-; RV64I-NEXT:    addiw a3, a3, 819
-; RV64I-NEXT:    addiw a6, a6, 1365
+; RV64I-NEXT:    addi t0, t0, -241
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    addi a6, a6, 1365
 ; RV64I-NEXT:    slli a5, a5, 24
 ; RV64I-NEXT:    or a5, a5, a7
 ; RV64I-NEXT:    slli a7, t0, 32
@@ -865,9 +865,9 @@ define i64 @test_bitreverse_i64(i64 %a) nounwind {
 ; RV64ZBB-NEXT:    lui a1, 61681
 ; RV64ZBB-NEXT:    lui a2, 209715
 ; RV64ZBB-NEXT:    lui a3, 349525
-; RV64ZBB-NEXT:    addiw a1, a1, -241
-; RV64ZBB-NEXT:    addiw a2, a2, 819
-; RV64ZBB-NEXT:    addiw a3, a3, 1365
+; RV64ZBB-NEXT:    addi a1, a1, -241
+; RV64ZBB-NEXT:    addi a2, a2, 819
+; RV64ZBB-NEXT:    addi a3, a3, 1365
 ; RV64ZBB-NEXT:    slli a4, a1, 32
 ; RV64ZBB-NEXT:    add a1, a1, a4
 ; RV64ZBB-NEXT:    slli a4, a2, 32
@@ -938,18 +938,18 @@ define i16 @test_bswap_bitreverse_i16(i16 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srli a1, a0, 4
 ; RV64I-NEXT:    lui a2, 1
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    lui a2, 3
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    slli a0, a0, 4
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 5
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    slli a0, a0, 2
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 1
@@ -988,18 +988,18 @@ define i16 @test_bswap_bitreverse_i16(i16 %a) nounwind {
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    srli a1, a0, 4
 ; RV64ZBB-NEXT:    lui a2, 1
-; RV64ZBB-NEXT:    addiw a2, a2, -241
+; RV64ZBB-NEXT:    addi a2, a2, -241
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    lui a2, 3
-; RV64ZBB-NEXT:    addiw a2, a2, 819
+; RV64ZBB-NEXT:    addi a2, a2, 819
 ; RV64ZBB-NEXT:    slli a0, a0, 4
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    lui a2, 5
-; RV64ZBB-NEXT:    addiw a2, a2, 1365
+; RV64ZBB-NEXT:    addi a2, a2, 1365
 ; RV64ZBB-NEXT:    slli a0, a0, 2
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 1
@@ -1053,18 +1053,18 @@ define i32 @test_bswap_bitreverse_i32(i32 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srli a1, a0, 4
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    slliw a0, a0, 4
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    slliw a0, a0, 2
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 1
@@ -1103,18 +1103,18 @@ define i32 @test_bswap_bitreverse_i32(i32 %a) nounwind {
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    srli a1, a0, 4
 ; RV64ZBB-NEXT:    lui a2, 61681
-; RV64ZBB-NEXT:    addiw a2, a2, -241
+; RV64ZBB-NEXT:    addi a2, a2, -241
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    lui a2, 209715
-; RV64ZBB-NEXT:    addiw a2, a2, 819
+; RV64ZBB-NEXT:    addi a2, a2, 819
 ; RV64ZBB-NEXT:    slliw a0, a0, 4
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    lui a2, 349525
-; RV64ZBB-NEXT:    addiw a2, a2, 1365
+; RV64ZBB-NEXT:    addi a2, a2, 1365
 ; RV64ZBB-NEXT:    slliw a0, a0, 2
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 1
@@ -1184,9 +1184,9 @@ define i64 @test_bswap_bitreverse_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    lui a1, 61681
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 349525
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, 1365
 ; RV64I-NEXT:    slli a4, a1, 32
 ; RV64I-NEXT:    add a1, a1, a4
 ; RV64I-NEXT:    slli a4, a2, 32
@@ -1255,9 +1255,9 @@ define i64 @test_bswap_bitreverse_i64(i64 %a) nounwind {
 ; RV64ZBB-NEXT:    lui a1, 61681
 ; RV64ZBB-NEXT:    lui a2, 209715
 ; RV64ZBB-NEXT:    lui a3, 349525
-; RV64ZBB-NEXT:    addiw a1, a1, -241
-; RV64ZBB-NEXT:    addiw a2, a2, 819
-; RV64ZBB-NEXT:    addiw a3, a3, 1365
+; RV64ZBB-NEXT:    addi a1, a1, -241
+; RV64ZBB-NEXT:    addi a2, a2, 819
+; RV64ZBB-NEXT:    addi a3, a3, 1365
 ; RV64ZBB-NEXT:    slli a4, a1, 32
 ; RV64ZBB-NEXT:    add a1, a1, a4
 ; RV64ZBB-NEXT:    slli a4, a2, 32
@@ -1326,18 +1326,18 @@ define i16 @test_bitreverse_bswap_i16(i16 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srli a1, a0, 4
 ; RV64I-NEXT:    lui a2, 1
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    lui a2, 3
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    slli a0, a0, 4
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 5
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    slli a0, a0, 2
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 1
@@ -1376,18 +1376,18 @@ define i16 @test_bitreverse_bswap_i16(i16 %a) nounwind {
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    srli a1, a0, 4
 ; RV64ZBB-NEXT:    lui a2, 1
-; RV64ZBB-NEXT:    addiw a2, a2, -241
+; RV64ZBB-NEXT:    addi a2, a2, -241
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    lui a2, 3
-; RV64ZBB-NEXT:    addiw a2, a2, 819
+; RV64ZBB-NEXT:    addi a2, a2, 819
 ; RV64ZBB-NEXT:    slli a0, a0, 4
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    lui a2, 5
-; RV64ZBB-NEXT:    addiw a2, a2, 1365
+; RV64ZBB-NEXT:    addi a2, a2, 1365
 ; RV64ZBB-NEXT:    slli a0, a0, 2
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 1
@@ -1441,18 +1441,18 @@ define i32 @test_bitreverse_bswap_i32(i32 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srli a1, a0, 4
 ; RV64I-NEXT:    lui a2, 61681
-; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addi a2, a2, -241
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    slliw a0, a0, 4
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    slliw a0, a0, 2
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    srli a1, a0, 1
@@ -1491,18 +1491,18 @@ define i32 @test_bitreverse_bswap_i32(i32 %a) nounwind {
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    srli a1, a0, 4
 ; RV64ZBB-NEXT:    lui a2, 61681
-; RV64ZBB-NEXT:    addiw a2, a2, -241
+; RV64ZBB-NEXT:    addi a2, a2, -241
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    lui a2, 209715
-; RV64ZBB-NEXT:    addiw a2, a2, 819
+; RV64ZBB-NEXT:    addi a2, a2, 819
 ; RV64ZBB-NEXT:    slliw a0, a0, 4
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 2
 ; RV64ZBB-NEXT:    and a0, a0, a2
 ; RV64ZBB-NEXT:    and a1, a1, a2
 ; RV64ZBB-NEXT:    lui a2, 349525
-; RV64ZBB-NEXT:    addiw a2, a2, 1365
+; RV64ZBB-NEXT:    addi a2, a2, 1365
 ; RV64ZBB-NEXT:    slliw a0, a0, 2
 ; RV64ZBB-NEXT:    or a0, a1, a0
 ; RV64ZBB-NEXT:    srli a1, a0, 1
@@ -1572,9 +1572,9 @@ define i64 @test_bitreverse_bswap_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    lui a1, 61681
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 349525
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, 1365
 ; RV64I-NEXT:    slli a4, a1, 32
 ; RV64I-NEXT:    add a1, a1, a4
 ; RV64I-NEXT:    slli a4, a2, 32
@@ -1643,9 +1643,9 @@ define i64 @test_bitreverse_bswap_i64(i64 %a) nounwind {
 ; RV64ZBB-NEXT:    lui a1, 61681
 ; RV64ZBB-NEXT:    lui a2, 209715
 ; RV64ZBB-NEXT:    lui a3, 349525
-; RV64ZBB-NEXT:    addiw a1, a1, -241
-; RV64ZBB-NEXT:    addiw a2, a2, 819
-; RV64ZBB-NEXT:    addiw a3, a3, 1365
+; RV64ZBB-NEXT:    addi a1, a1, -241
+; RV64ZBB-NEXT:    addi a2, a2, 819
+; RV64ZBB-NEXT:    addi a3, a3, 1365
 ; RV64ZBB-NEXT:    slli a4, a1, 32
 ; RV64ZBB-NEXT:    add a1, a1, a4
 ; RV64ZBB-NEXT:    slli a4, a2, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-half.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-half.ll
index 541c9b4d40c7..008036b2d2e2 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-half.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-half.ll
@@ -361,7 +361,7 @@ define i32 @caller_half_on_stack() nounwind {
 ; RV64I-NEXT:    li a4, 5
 ; RV64I-NEXT:    li a5, 6
 ; RV64I-NEXT:    li a6, 7
-; RV64I-NEXT:    addiw t0, a7, -1792
+; RV64I-NEXT:    addi t0, a7, -1792
 ; RV64I-NEXT:    li a7, 8
 ; RV64I-NEXT:    sd t0, 0(sp)
 ; RV64I-NEXT:    call callee_half_on_stack
@@ -510,7 +510,7 @@ define half @callee_half_ret() nounwind {
 ; RV64IF-LABEL: callee_half_ret:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    lui a0, 1048564
-; RV64IF-NEXT:    addiw a0, a0, -1024
+; RV64IF-NEXT:    addi a0, a0, -1024
 ; RV64IF-NEXT:    ret
 ;
 ; RV32-ILP32F-LABEL: callee_half_ret:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
index e0f46e748451..a63dc0ef3a3a 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -450,7 +450,7 @@ define i256 @callee_large_scalar_ret() nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a1, -1
 ; RV64I-NEXT:    lui a2, 1018435
-; RV64I-NEXT:    addiw a2, a2, 747
+; RV64I-NEXT:    addi a2, a2, 747
 ; RV64I-NEXT:    sd a2, 0(a0)
 ; RV64I-NEXT:    sd a1, 8(a0)
 ; RV64I-NEXT:    sd a1, 16(a0)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/codemodel-lowering.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
index 4831f0b24c7f..086c3ac18152 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
@@ -303,21 +303,21 @@ define float @lower_constantpool(float %a) nounwind {
 ; RV64FINX-SMALL-LABEL: lower_constantpool:
 ; RV64FINX-SMALL:       # %bb.0:
 ; RV64FINX-SMALL-NEXT:    lui a1, 260097
-; RV64FINX-SMALL-NEXT:    addiw a1, a1, -2048
+; RV64FINX-SMALL-NEXT:    addi a1, a1, -2048
 ; RV64FINX-SMALL-NEXT:    fadd.s a0, a0, a1
 ; RV64FINX-SMALL-NEXT:    ret
 ;
 ; RV64FINX-MEDIUM-LABEL: lower_constantpool:
 ; RV64FINX-MEDIUM:       # %bb.0:
 ; RV64FINX-MEDIUM-NEXT:    lui a1, 260097
-; RV64FINX-MEDIUM-NEXT:    addiw a1, a1, -2048
+; RV64FINX-MEDIUM-NEXT:    addi a1, a1, -2048
 ; RV64FINX-MEDIUM-NEXT:    fadd.s a0, a0, a1
 ; RV64FINX-MEDIUM-NEXT:    ret
 ;
 ; RV64FINX-LARGE-LABEL: lower_constantpool:
 ; RV64FINX-LARGE:       # %bb.0:
 ; RV64FINX-LARGE-NEXT:    lui a1, 260097
-; RV64FINX-LARGE-NEXT:    addiw a1, a1, -2048
+; RV64FINX-LARGE-NEXT:    addi a1, a1, -2048
 ; RV64FINX-LARGE-NEXT:    fadd.s a0, a0, a1
 ; RV64FINX-LARGE-NEXT:    ret
   %1 = fadd float %a, 1.000244140625
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
index 8b9d602dcde8..724891853630 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
@@ -163,11 +163,11 @@ define i16 @test_cttz_i16(i16 %a) nounwind {
 ; RV64NOZBB-NEXT:    not a0, a0
 ; RV64NOZBB-NEXT:    lui a2, 5
 ; RV64NOZBB-NEXT:    and a0, a0, a1
-; RV64NOZBB-NEXT:    addiw a1, a2, 1365
+; RV64NOZBB-NEXT:    addi a1, a2, 1365
 ; RV64NOZBB-NEXT:    srli a2, a0, 1
 ; RV64NOZBB-NEXT:    and a1, a2, a1
 ; RV64NOZBB-NEXT:    lui a2, 3
-; RV64NOZBB-NEXT:    addiw a2, a2, 819
+; RV64NOZBB-NEXT:    addi a2, a2, 819
 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    and a1, a0, a2
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
@@ -671,11 +671,11 @@ define i16 @test_cttz_i16_zero_undef(i16 %a) nounwind {
 ; RV64NOZBB-NEXT:    not a0, a0
 ; RV64NOZBB-NEXT:    lui a2, 5
 ; RV64NOZBB-NEXT:    and a0, a0, a1
-; RV64NOZBB-NEXT:    addiw a1, a2, 1365
+; RV64NOZBB-NEXT:    addi a1, a2, 1365
 ; RV64NOZBB-NEXT:    srli a2, a0, 1
 ; RV64NOZBB-NEXT:    and a1, a2, a1
 ; RV64NOZBB-NEXT:    lui a2, 3
-; RV64NOZBB-NEXT:    addiw a2, a2, 819
+; RV64NOZBB-NEXT:    addi a2, a2, 819
 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    and a1, a0, a2
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
@@ -1131,7 +1131,7 @@ define i16 @test_ctlz_i16(i16 %a) nounwind {
 ; RV64NOZBB-NEXT:    srli a1, a1, 49
 ; RV64NOZBB-NEXT:    lui a2, 5
 ; RV64NOZBB-NEXT:    or a0, a0, a1
-; RV64NOZBB-NEXT:    addiw a1, a2, 1365
+; RV64NOZBB-NEXT:    addi a1, a2, 1365
 ; RV64NOZBB-NEXT:    slli a2, a0, 48
 ; RV64NOZBB-NEXT:    srli a2, a2, 50
 ; RV64NOZBB-NEXT:    or a0, a0, a2
@@ -1145,7 +1145,7 @@ define i16 @test_ctlz_i16(i16 %a) nounwind {
 ; RV64NOZBB-NEXT:    srli a2, a0, 1
 ; RV64NOZBB-NEXT:    and a1, a2, a1
 ; RV64NOZBB-NEXT:    lui a2, 3
-; RV64NOZBB-NEXT:    addiw a2, a2, 819
+; RV64NOZBB-NEXT:    addi a2, a2, 819
 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    and a1, a0, a2
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
@@ -1243,7 +1243,7 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -1256,7 +1256,7 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -1325,7 +1325,7 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV64M-NEXT:    srliw a1, a0, 1
 ; RV64M-NEXT:    lui a2, 349525
 ; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    addiw a1, a2, 1365
+; RV64M-NEXT:    addi a1, a2, 1365
 ; RV64M-NEXT:    srliw a2, a0, 2
 ; RV64M-NEXT:    or a0, a0, a2
 ; RV64M-NEXT:    srliw a2, a0, 4
@@ -1338,7 +1338,7 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV64M-NEXT:    srli a2, a0, 1
 ; RV64M-NEXT:    and a1, a2, a1
 ; RV64M-NEXT:    lui a2, 209715
-; RV64M-NEXT:    addiw a2, a2, 819
+; RV64M-NEXT:    addi a2, a2, 819
 ; RV64M-NEXT:    sub a0, a0, a1
 ; RV64M-NEXT:    and a1, a0, a2
 ; RV64M-NEXT:    srli a0, a0, 2
@@ -1468,8 +1468,8 @@ define i64 @test_ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    lui a3, 209715
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
-; RV64I-NEXT:    addiw a2, a3, 819
+; RV64I-NEXT:    addi a1, a2, 1365
+; RV64I-NEXT:    addi a2, a3, 819
 ; RV64I-NEXT:    srli a3, a0, 2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a3, a1, 32
@@ -1488,7 +1488,7 @@ define i64 @test_ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 1
 ; RV64I-NEXT:    and a1, a3, a1
 ; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -1592,9 +1592,9 @@ define i64 @test_ctlz_i64(i64 %a) nounwind {
 ; RV64M-NEXT:    lui a3, 209715
 ; RV64M-NEXT:    lui a4, 61681
 ; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    addiw a1, a2, 1365
-; RV64M-NEXT:    addiw a2, a3, 819
-; RV64M-NEXT:    addiw a3, a4, -241
+; RV64M-NEXT:    addi a1, a2, 1365
+; RV64M-NEXT:    addi a2, a3, 819
+; RV64M-NEXT:    addi a3, a4, -241
 ; RV64M-NEXT:    srli a4, a0, 2
 ; RV64M-NEXT:    or a0, a0, a4
 ; RV64M-NEXT:    slli a4, a1, 32
@@ -1619,7 +1619,7 @@ define i64 @test_ctlz_i64(i64 %a) nounwind {
 ; RV64M-NEXT:    srli a0, a0, 2
 ; RV64M-NEXT:    and a0, a0, a2
 ; RV64M-NEXT:    lui a2, 4112
-; RV64M-NEXT:    addiw a2, a2, 257
+; RV64M-NEXT:    addi a2, a2, 257
 ; RV64M-NEXT:    add a0, a1, a0
 ; RV64M-NEXT:    srli a1, a0, 4
 ; RV64M-NEXT:    add a0, a0, a1
@@ -1788,7 +1788,7 @@ define i16 @test_ctlz_i16_zero_undef(i16 %a) nounwind {
 ; RV64NOZBB-NEXT:    slli a1, a0, 48
 ; RV64NOZBB-NEXT:    lui a2, 5
 ; RV64NOZBB-NEXT:    srli a1, a1, 49
-; RV64NOZBB-NEXT:    addiw a2, a2, 1365
+; RV64NOZBB-NEXT:    addi a2, a2, 1365
 ; RV64NOZBB-NEXT:    or a0, a0, a1
 ; RV64NOZBB-NEXT:    slli a1, a0, 48
 ; RV64NOZBB-NEXT:    srli a1, a1, 50
@@ -1803,7 +1803,7 @@ define i16 @test_ctlz_i16_zero_undef(i16 %a) nounwind {
 ; RV64NOZBB-NEXT:    srli a1, a0, 1
 ; RV64NOZBB-NEXT:    and a1, a1, a2
 ; RV64NOZBB-NEXT:    lui a2, 3
-; RV64NOZBB-NEXT:    addiw a2, a2, 819
+; RV64NOZBB-NEXT:    addi a2, a2, 819
 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    and a1, a0, a2
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
@@ -1886,7 +1886,7 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -1899,7 +1899,7 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -1957,7 +1957,7 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
 ; RV64M-NEXT:    srliw a1, a0, 1
 ; RV64M-NEXT:    lui a2, 349525
 ; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    addiw a1, a2, 1365
+; RV64M-NEXT:    addi a1, a2, 1365
 ; RV64M-NEXT:    srliw a2, a0, 2
 ; RV64M-NEXT:    or a0, a0, a2
 ; RV64M-NEXT:    srliw a2, a0, 4
@@ -1970,7 +1970,7 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
 ; RV64M-NEXT:    srli a2, a0, 1
 ; RV64M-NEXT:    and a1, a2, a1
 ; RV64M-NEXT:    lui a2, 209715
-; RV64M-NEXT:    addiw a2, a2, 819
+; RV64M-NEXT:    addi a2, a2, 819
 ; RV64M-NEXT:    sub a0, a0, a1
 ; RV64M-NEXT:    and a1, a0, a2
 ; RV64M-NEXT:    srli a0, a0, 2
@@ -2088,8 +2088,8 @@ define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    lui a3, 209715
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
-; RV64I-NEXT:    addiw a2, a3, 819
+; RV64I-NEXT:    addi a1, a2, 1365
+; RV64I-NEXT:    addi a2, a3, 819
 ; RV64I-NEXT:    srli a3, a0, 2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a3, a1, 32
@@ -2108,7 +2108,7 @@ define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 1
 ; RV64I-NEXT:    and a1, a3, a1
 ; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -2200,9 +2200,9 @@ define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
 ; RV64M-NEXT:    lui a3, 209715
 ; RV64M-NEXT:    lui a4, 61681
 ; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    addiw a1, a2, 1365
-; RV64M-NEXT:    addiw a2, a3, 819
-; RV64M-NEXT:    addiw a3, a4, -241
+; RV64M-NEXT:    addi a1, a2, 1365
+; RV64M-NEXT:    addi a2, a3, 819
+; RV64M-NEXT:    addi a3, a4, -241
 ; RV64M-NEXT:    srli a4, a0, 2
 ; RV64M-NEXT:    or a0, a0, a4
 ; RV64M-NEXT:    slli a4, a1, 32
@@ -2227,7 +2227,7 @@ define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
 ; RV64M-NEXT:    srli a0, a0, 2
 ; RV64M-NEXT:    and a0, a0, a2
 ; RV64M-NEXT:    lui a2, 4112
-; RV64M-NEXT:    addiw a2, a2, 257
+; RV64M-NEXT:    addi a2, a2, 257
 ; RV64M-NEXT:    add a0, a1, a0
 ; RV64M-NEXT:    srli a1, a0, 4
 ; RV64M-NEXT:    add a0, a0, a1
@@ -2375,10 +2375,10 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
 ; RV64NOZBB:       # %bb.0:
 ; RV64NOZBB-NEXT:    srli a1, a0, 1
 ; RV64NOZBB-NEXT:    lui a2, 5
-; RV64NOZBB-NEXT:    addiw a2, a2, 1365
+; RV64NOZBB-NEXT:    addi a2, a2, 1365
 ; RV64NOZBB-NEXT:    and a1, a1, a2
 ; RV64NOZBB-NEXT:    lui a2, 3
-; RV64NOZBB-NEXT:    addiw a2, a2, 819
+; RV64NOZBB-NEXT:    addi a2, a2, 819
 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    and a1, a0, a2
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
@@ -2428,10 +2428,10 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
 ; RV64XTHEADBB:       # %bb.0:
 ; RV64XTHEADBB-NEXT:    srli a1, a0, 1
 ; RV64XTHEADBB-NEXT:    lui a2, 5
-; RV64XTHEADBB-NEXT:    addiw a2, a2, 1365
+; RV64XTHEADBB-NEXT:    addi a2, a2, 1365
 ; RV64XTHEADBB-NEXT:    and a1, a1, a2
 ; RV64XTHEADBB-NEXT:    lui a2, 3
-; RV64XTHEADBB-NEXT:    addiw a2, a2, 819
+; RV64XTHEADBB-NEXT:    addi a2, a2, 819
 ; RV64XTHEADBB-NEXT:    sub a0, a0, a1
 ; RV64XTHEADBB-NEXT:    and a1, a0, a2
 ; RV64XTHEADBB-NEXT:    srli a0, a0, 2
@@ -2477,10 +2477,10 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -2526,10 +2526,10 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV64M:       # %bb.0:
 ; RV64M-NEXT:    srli a1, a0, 1
 ; RV64M-NEXT:    lui a2, 349525
-; RV64M-NEXT:    addiw a2, a2, 1365
+; RV64M-NEXT:    addi a2, a2, 1365
 ; RV64M-NEXT:    and a1, a1, a2
 ; RV64M-NEXT:    lui a2, 209715
-; RV64M-NEXT:    addiw a2, a2, 819
+; RV64M-NEXT:    addi a2, a2, 819
 ; RV64M-NEXT:    sub a0, a0, a1
 ; RV64M-NEXT:    and a1, a0, a2
 ; RV64M-NEXT:    srli a0, a0, 2
@@ -2585,10 +2585,10 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV64XTHEADBB:       # %bb.0:
 ; RV64XTHEADBB-NEXT:    srli a1, a0, 1
 ; RV64XTHEADBB-NEXT:    lui a2, 349525
-; RV64XTHEADBB-NEXT:    addiw a2, a2, 1365
+; RV64XTHEADBB-NEXT:    addi a2, a2, 1365
 ; RV64XTHEADBB-NEXT:    and a1, a1, a2
 ; RV64XTHEADBB-NEXT:    lui a2, 209715
-; RV64XTHEADBB-NEXT:    addiw a2, a2, 819
+; RV64XTHEADBB-NEXT:    addi a2, a2, 819
 ; RV64XTHEADBB-NEXT:    sub a0, a0, a1
 ; RV64XTHEADBB-NEXT:    and a1, a0, a2
 ; RV64XTHEADBB-NEXT:    srli a0, a0, 2
@@ -2656,8 +2656,8 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 349525
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    slli a3, a1, 32
 ; RV64I-NEXT:    add a1, a1, a3
 ; RV64I-NEXT:    slli a3, a2, 32
@@ -2665,7 +2665,7 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 1
 ; RV64I-NEXT:    and a1, a3, a1
 ; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -2728,9 +2728,9 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV64M-NEXT:    lui a1, 349525
 ; RV64M-NEXT:    lui a2, 209715
 ; RV64M-NEXT:    lui a3, 61681
-; RV64M-NEXT:    addiw a1, a1, 1365
-; RV64M-NEXT:    addiw a2, a2, 819
-; RV64M-NEXT:    addiw a3, a3, -241
+; RV64M-NEXT:    addi a1, a1, 1365
+; RV64M-NEXT:    addi a2, a2, 819
+; RV64M-NEXT:    addi a3, a3, -241
 ; RV64M-NEXT:    slli a4, a1, 32
 ; RV64M-NEXT:    add a1, a1, a4
 ; RV64M-NEXT:    slli a4, a2, 32
@@ -2744,7 +2744,7 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV64M-NEXT:    srli a0, a0, 2
 ; RV64M-NEXT:    and a0, a0, a2
 ; RV64M-NEXT:    lui a2, 4112
-; RV64M-NEXT:    addiw a2, a2, 257
+; RV64M-NEXT:    addi a2, a2, 257
 ; RV64M-NEXT:    add a0, a1, a0
 ; RV64M-NEXT:    srli a1, a0, 4
 ; RV64M-NEXT:    add a0, a0, a1
@@ -2814,8 +2814,8 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV64XTHEADBB:       # %bb.0:
 ; RV64XTHEADBB-NEXT:    lui a1, 349525
 ; RV64XTHEADBB-NEXT:    lui a2, 209715
-; RV64XTHEADBB-NEXT:    addiw a1, a1, 1365
-; RV64XTHEADBB-NEXT:    addiw a2, a2, 819
+; RV64XTHEADBB-NEXT:    addi a1, a1, 1365
+; RV64XTHEADBB-NEXT:    addi a2, a2, 819
 ; RV64XTHEADBB-NEXT:    slli a3, a1, 32
 ; RV64XTHEADBB-NEXT:    add a1, a1, a3
 ; RV64XTHEADBB-NEXT:    slli a3, a2, 32
@@ -2823,7 +2823,7 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV64XTHEADBB-NEXT:    srli a3, a0, 1
 ; RV64XTHEADBB-NEXT:    and a1, a3, a1
 ; RV64XTHEADBB-NEXT:    lui a3, 61681
-; RV64XTHEADBB-NEXT:    addiw a3, a3, -241
+; RV64XTHEADBB-NEXT:    addi a3, a3, -241
 ; RV64XTHEADBB-NEXT:    sub a0, a0, a1
 ; RV64XTHEADBB-NEXT:    and a1, a0, a2
 ; RV64XTHEADBB-NEXT:    srli a0, a0, 2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
index 33907e10730a..637fb314e953 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
@@ -732,8 +732,8 @@ define signext i32 @ctlz(i64 %b) nounwind {
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    lui a3, 209715
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
-; RV64I-NEXT:    addiw a2, a3, 819
+; RV64I-NEXT:    addi a1, a2, 1365
+; RV64I-NEXT:    addi a2, a3, 819
 ; RV64I-NEXT:    srli a3, a0, 2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a3, a1, 32
@@ -752,7 +752,7 @@ define signext i32 @ctlz(i64 %b) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 1
 ; RV64I-NEXT:    and a1, a3, a1
 ; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/div-by-constant.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/div-by-constant.ll
index e14a894bf187..ea8b04d727ac 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/div-by-constant.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/div-by-constant.ll
@@ -64,7 +64,7 @@ define i32 @udiv_constant_add(i32 %a) nounwind {
 ; RV64IMZB:       # %bb.0:
 ; RV64IMZB-NEXT:    zext.w a1, a0
 ; RV64IMZB-NEXT:    lui a2, 149797
-; RV64IMZB-NEXT:    addiw a2, a2, -1755
+; RV64IMZB-NEXT:    addi a2, a2, -1755
 ; RV64IMZB-NEXT:    mul a1, a1, a2
 ; RV64IMZB-NEXT:    srli a1, a1, 32
 ; RV64IMZB-NEXT:    subw a0, a0, a1
@@ -104,7 +104,7 @@ define i64 @udiv64_constant_no_add(i64 %a) nounwind {
 ; RV64-LABEL: udiv64_constant_no_add:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a1, 838861
-; RV64-NEXT:    addiw a1, a1, -819
+; RV64-NEXT:    addi a1, a1, -819
 ; RV64-NEXT:    slli a2, a1, 32
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    mulhu a0, a0, a1
@@ -282,7 +282,7 @@ define i32 @sdiv_constant_no_srai(i32 %a) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    sext.w a0, a0
 ; RV64-NEXT:    lui a1, 349525
-; RV64-NEXT:    addiw a1, a1, 1366
+; RV64-NEXT:    addi a1, a1, 1366
 ; RV64-NEXT:    mul a0, a0, a1
 ; RV64-NEXT:    srli a1, a0, 63
 ; RV64-NEXT:    srli a0, a0, 32
@@ -308,7 +308,7 @@ define i32 @sdiv_constant_srai(i32 %a) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    sext.w a0, a0
 ; RV64-NEXT:    lui a1, 419430
-; RV64-NEXT:    addiw a1, a1, 1639
+; RV64-NEXT:    addi a1, a1, 1639
 ; RV64-NEXT:    mul a0, a0, a1
 ; RV64-NEXT:    srli a1, a0, 63
 ; RV64-NEXT:    srai a0, a0, 33
@@ -335,7 +335,7 @@ define i32 @sdiv_constant_add_srai(i32 %a) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    sext.w a1, a0
 ; RV64-NEXT:    lui a2, 599186
-; RV64-NEXT:    addiw a2, a2, 1171
+; RV64-NEXT:    addi a2, a2, 1171
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    add a0, a1, a0
@@ -364,7 +364,7 @@ define i32 @sdiv_constant_sub_srai(i32 %a) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    sext.w a1, a0
 ; RV64-NEXT:    lui a2, 449390
-; RV64-NEXT:    addiw a2, a2, -1171
+; RV64-NEXT:    addi a2, a2, -1171
 ; RV64-NEXT:    mul a1, a1, a2
 ; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    subw a1, a1, a0
@@ -440,7 +440,7 @@ define i64 @sdiv64_constant_add_srai(i64 %a) nounwind {
 ; RV64-LABEL: sdiv64_constant_add_srai:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a1, 559241
-; RV64-NEXT:    addiw a1, a1, -1911
+; RV64-NEXT:    addi a1, a1, -1911
 ; RV64-NEXT:    slli a2, a1, 32
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    mulh a1, a0, a1
@@ -468,7 +468,7 @@ define i64 @sdiv64_constant_sub_srai(i64 %a) nounwind {
 ; RV64-LABEL: sdiv64_constant_sub_srai:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a1, 349525
-; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addi a1, a1, 1365
 ; RV64-NEXT:    slli a2, a1, 32
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    mulh a1, a0, a1
@@ -718,7 +718,7 @@ define i16 @sdiv16_constant_no_srai(i16 %a) nounwind {
 ; RV64IM-NEXT:    slli a0, a0, 48
 ; RV64IM-NEXT:    lui a1, 5
 ; RV64IM-NEXT:    srai a0, a0, 48
-; RV64IM-NEXT:    addiw a1, a1, 1366
+; RV64IM-NEXT:    addi a1, a1, 1366
 ; RV64IM-NEXT:    mul a0, a0, a1
 ; RV64IM-NEXT:    srli a1, a0, 63
 ; RV64IM-NEXT:    srli a0, a0, 16
@@ -729,7 +729,7 @@ define i16 @sdiv16_constant_no_srai(i16 %a) nounwind {
 ; RV64IMZB:       # %bb.0:
 ; RV64IMZB-NEXT:    sext.h a0, a0
 ; RV64IMZB-NEXT:    lui a1, 5
-; RV64IMZB-NEXT:    addiw a1, a1, 1366
+; RV64IMZB-NEXT:    addi a1, a1, 1366
 ; RV64IMZB-NEXT:    mul a0, a0, a1
 ; RV64IMZB-NEXT:    srli a1, a0, 63
 ; RV64IMZB-NEXT:    srli a0, a0, 16
@@ -768,7 +768,7 @@ define i16 @sdiv16_constant_srai(i16 %a) nounwind {
 ; RV64IM-NEXT:    slli a0, a0, 48
 ; RV64IM-NEXT:    lui a1, 6
 ; RV64IM-NEXT:    srai a0, a0, 48
-; RV64IM-NEXT:    addiw a1, a1, 1639
+; RV64IM-NEXT:    addi a1, a1, 1639
 ; RV64IM-NEXT:    mul a0, a0, a1
 ; RV64IM-NEXT:    srli a1, a0, 63
 ; RV64IM-NEXT:    srai a0, a0, 17
@@ -779,7 +779,7 @@ define i16 @sdiv16_constant_srai(i16 %a) nounwind {
 ; RV64IMZB:       # %bb.0:
 ; RV64IMZB-NEXT:    sext.h a0, a0
 ; RV64IMZB-NEXT:    lui a1, 6
-; RV64IMZB-NEXT:    addiw a1, a1, 1639
+; RV64IMZB-NEXT:    addi a1, a1, 1639
 ; RV64IMZB-NEXT:    mul a0, a0, a1
 ; RV64IMZB-NEXT:    srli a1, a0, 63
 ; RV64IMZB-NEXT:    srai a0, a0, 17
@@ -824,7 +824,7 @@ define i16 @sdiv16_constant_add_srai(i16 %a) nounwind {
 ; RV64IM-NEXT:    slli a1, a0, 48
 ; RV64IM-NEXT:    lui a2, 1048569
 ; RV64IM-NEXT:    srai a1, a1, 48
-; RV64IM-NEXT:    addiw a2, a2, -1911
+; RV64IM-NEXT:    addi a2, a2, -1911
 ; RV64IM-NEXT:    mul a1, a1, a2
 ; RV64IM-NEXT:    srli a1, a1, 16
 ; RV64IM-NEXT:    add a0, a1, a0
@@ -838,7 +838,7 @@ define i16 @sdiv16_constant_add_srai(i16 %a) nounwind {
 ; RV64IMZB:       # %bb.0:
 ; RV64IMZB-NEXT:    sext.h a1, a0
 ; RV64IMZB-NEXT:    lui a2, 1048569
-; RV64IMZB-NEXT:    addiw a2, a2, -1911
+; RV64IMZB-NEXT:    addi a2, a2, -1911
 ; RV64IMZB-NEXT:    mul a1, a1, a2
 ; RV64IMZB-NEXT:    srli a1, a1, 16
 ; RV64IMZB-NEXT:    add a0, a1, a0
@@ -886,7 +886,7 @@ define i16 @sdiv16_constant_sub_srai(i16 %a) nounwind {
 ; RV64IM-NEXT:    slli a1, a0, 48
 ; RV64IM-NEXT:    lui a2, 7
 ; RV64IM-NEXT:    srai a1, a1, 48
-; RV64IM-NEXT:    addiw a2, a2, 1911
+; RV64IM-NEXT:    addi a2, a2, 1911
 ; RV64IM-NEXT:    mul a1, a1, a2
 ; RV64IM-NEXT:    srli a1, a1, 16
 ; RV64IM-NEXT:    subw a1, a1, a0
@@ -900,7 +900,7 @@ define i16 @sdiv16_constant_sub_srai(i16 %a) nounwind {
 ; RV64IMZB:       # %bb.0:
 ; RV64IMZB-NEXT:    sext.h a1, a0
 ; RV64IMZB-NEXT:    lui a2, 7
-; RV64IMZB-NEXT:    addiw a2, a2, 1911
+; RV64IMZB-NEXT:    addi a2, a2, 1911
 ; RV64IMZB-NEXT:    mul a1, a1, a2
 ; RV64IMZB-NEXT:    srli a1, a1, 16
 ; RV64IMZB-NEXT:    subw a1, a1, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/div.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/div.ll
index 415848449e13..b160a532770d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/div.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/div.ll
@@ -211,7 +211,7 @@ define i64 @udiv64_constant(i64 %a) nounwind {
 ; RV64IM-LABEL: udiv64_constant:
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    lui a1, 838861
-; RV64IM-NEXT:    addiw a1, a1, -819
+; RV64IM-NEXT:    addi a1, a1, -819
 ; RV64IM-NEXT:    slli a2, a1, 32
 ; RV64IM-NEXT:    add a1, a1, a2
 ; RV64IM-NEXT:    mulhu a0, a0, a1
@@ -441,7 +441,7 @@ define i16 @udiv16(i16 %a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    call __udivdi3
@@ -638,7 +638,7 @@ define i32 @sdiv_constant(i32 %a) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    sext.w a0, a0
 ; RV64IM-NEXT:    lui a1, 419430
-; RV64IM-NEXT:    addiw a1, a1, 1639
+; RV64IM-NEXT:    addi a1, a1, 1639
 ; RV64IM-NEXT:    mul a0, a0, a1
 ; RV64IM-NEXT:    srli a1, a0, 63
 ; RV64IM-NEXT:    srai a0, a0, 33
@@ -1189,7 +1189,7 @@ define i16 @sdiv16_constant(i16 %a) nounwind {
 ; RV64IM-NEXT:    slli a0, a0, 48
 ; RV64IM-NEXT:    lui a1, 6
 ; RV64IM-NEXT:    srai a0, a0, 48
-; RV64IM-NEXT:    addiw a1, a1, 1639
+; RV64IM-NEXT:    addi a1, a1, 1639
 ; RV64IM-NEXT:    mul a0, a0, a1
 ; RV64IM-NEXT:    srli a1, a0, 63
 ; RV64IM-NEXT:    srai a0, a0, 17
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/double-convert.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/double-convert.ll
index c41239f93cf2..fad9e21fee72 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -1937,7 +1937,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(double %a) nounwind {
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtdf2
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    blez a0, .LBB28_2
 ; RV64I-NEXT:  # %bb.1: # %start
 ; RV64I-NEXT:    mv a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
index 949668f640db..7c5332f71986 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
@@ -471,15 +471,15 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
-; RV32I-NEXT:    call __eqdf2
-; RV32I-NEXT:    seqz s4, a0
+; RV32I-NEXT:    call __unorddf2
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    mv a3, s0
-; RV32I-NEXT:    call __unorddf2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    or a0, a0, s4
+; RV32I-NEXT:    call __eqdf2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, s4, a0
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -498,13 +498,13 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    call __eqdf2
-; RV64I-NEXT:    seqz s2, a0
+; RV64I-NEXT:    call __unorddf2
+; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __unorddf2
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    or a0, a0, s2
+; RV64I-NEXT:    call __eqdf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, s2, a0
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
@@ -1199,15 +1199,15 @@ define i32 @fcmps_ueq(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
-; RV32I-NEXT:    call __eqdf2
-; RV32I-NEXT:    seqz s4, a0
+; RV32I-NEXT:    call __unorddf2
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    mv a3, s0
-; RV32I-NEXT:    call __unorddf2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    or a0, a0, s4
+; RV32I-NEXT:    call __eqdf2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, s4, a0
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -1226,13 +1226,13 @@ define i32 @fcmps_ueq(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    call __eqdf2
-; RV64I-NEXT:    seqz s2, a0
+; RV64I-NEXT:    call __unorddf2
+; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __unorddf2
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    or a0, a0, s2
+; RV64I-NEXT:    call __eqdf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, s2, a0
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp.ll
index 1e609f8081eb..f73e6865cf47 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/double-fcmp.ll
@@ -403,15 +403,15 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind {
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
-; RV32I-NEXT:    call __eqdf2
-; RV32I-NEXT:    seqz s4, a0
+; RV32I-NEXT:    call __unorddf2
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    mv a3, s0
-; RV32I-NEXT:    call __unorddf2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    or a0, a0, s4
+; RV32I-NEXT:    call __eqdf2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, s4, a0
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -430,13 +430,13 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    call __eqdf2
-; RV64I-NEXT:    seqz s2, a0
+; RV64I-NEXT:    call __unorddf2
+; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __unorddf2
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    or a0, a0, s2
+; RV64I-NEXT:    call __eqdf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, s2, a0
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/features-info.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/features-info.ll
index b7b27cd579fb..fab2e9495930 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/features-info.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/features-info.ll
@@ -6,6 +6,7 @@
 ; CHECK-NEXT:   32bit                            - Implements RV32.
 ; CHECK-NEXT:   64bit                            - Implements RV64.
 ; CHECK-NEXT:   a                                - 'A' (Atomic Instructions).
+; CHECK-NEXT:   andes45                          - Andes 45-Series processors.
 ; CHECK-NEXT:   auipc-addi-fusion                - Enable AUIPC+ADDI macrofusion.
 ; CHECK-NEXT:   b                                - 'B' (the collection of the Zba, Zbb, Zbs extensions).
 ; CHECK-NEXT:   c                                - 'C' (Compressed Instructions).
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/float-convert.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/float-convert.ll
index 3fc2b6598b69..797d59dfcdc8 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -126,7 +126,7 @@ define i32 @fcvt_w_s_sat(float %a) nounwind {
 ; RV64I-NEXT:    lui s1, 524288
 ; RV64I-NEXT:  .LBB1_2: # %start
 ; RV64I-NEXT:    lui a1, 323584
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB1_4
@@ -311,7 +311,7 @@ define i32 @fcvt_wu_s_sat(float %a) nounwind {
 ; RV64I-NEXT:    call __fixunssfdi
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a1, 325632
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB4_2
@@ -788,7 +788,7 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV64I-NEXT:    slli s1, s3, 63
 ; RV64I-NEXT:  .LBB12_2: # %start
 ; RV64I-NEXT:    lui a1, 389120
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB12_4
@@ -976,7 +976,7 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a1, 391168
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    sgtz a0, a0
 ; RV64I-NEXT:    neg s1, a0
@@ -1462,7 +1462,7 @@ define signext i16 @fcvt_w_s_sat_i16(float %a) nounwind {
 ; RV64IZFINX-NEXT:    fmax.s a0, a0, a2
 ; RV64IZFINX-NEXT:    lui a2, 290816
 ; RV64IZFINX-NEXT:    neg a1, a1
-; RV64IZFINX-NEXT:    addiw a2, a2, -512
+; RV64IZFINX-NEXT:    addi a2, a2, -512
 ; RV64IZFINX-NEXT:    fmin.s a0, a0, a2
 ; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
 ; RV64IZFINX-NEXT:    and a0, a1, a0
@@ -1529,7 +1529,7 @@ define signext i16 @fcvt_w_s_sat_i16(float %a) nounwind {
 ; RV64I-NEXT:    lui s1, 1048568
 ; RV64I-NEXT:  .LBB24_2: # %start
 ; RV64I-NEXT:    lui a0, 290816
-; RV64I-NEXT:    addiw a1, a0, -512
+; RV64I-NEXT:    addi a1, a0, -512
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB24_4
@@ -1633,7 +1633,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(float %a) nounwind {
 ; RV64IZFINX:       # %bb.0: # %start
 ; RV64IZFINX-NEXT:    fmax.s a0, a0, zero
 ; RV64IZFINX-NEXT:    lui a1, 292864
-; RV64IZFINX-NEXT:    addiw a1, a1, -256
+; RV64IZFINX-NEXT:    addi a1, a1, -256
 ; RV64IZFINX-NEXT:    fmin.s a0, a0, a1
 ; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
 ; RV64IZFINX-NEXT:    ret
@@ -1690,11 +1690,11 @@ define zeroext i16 @fcvt_wu_s_sat_i16(float %a) nounwind {
 ; RV64I-NEXT:    call __fixunssfdi
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 292864
-; RV64I-NEXT:    addiw a1, a0, -256
+; RV64I-NEXT:    addi a1, a0, -256
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    blez a0, .LBB26_2
 ; RV64I-NEXT:  # %bb.1: # %start
 ; RV64I-NEXT:    mv a0, a1
@@ -2132,7 +2132,7 @@ define zeroext i32 @fcvt_wu_s_sat_zext(float %a) nounwind {
 ; RV64I-NEXT:    call __fixunssfdi
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a1, 325632
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB31_2
@@ -2240,7 +2240,7 @@ define signext i32 @fcvt_w_s_sat_sext(float %a) nounwind {
 ; RV64I-NEXT:    lui s1, 524288
 ; RV64I-NEXT:  .LBB32_2: # %start
 ; RV64I-NEXT:    lui a1, 323584
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB32_4
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
index 0cbfc96bf485..fd3baa057525 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
@@ -382,13 +382,13 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    call __eqsf2
-; RV32I-NEXT:    seqz s2, a0
+; RV32I-NEXT:    call __unordsf2
+; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __unordsf2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    or a0, a0, s2
+; RV32I-NEXT:    call __eqsf2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, s2, a0
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -405,13 +405,13 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    call __eqsf2
-; RV64I-NEXT:    seqz s2, a0
+; RV64I-NEXT:    call __unordsf2
+; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __unordsf2
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    or a0, a0, s2
+; RV64I-NEXT:    call __eqsf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, s2, a0
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
@@ -991,13 +991,13 @@ define i32 @fcmps_ueq(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    call __eqsf2
-; RV32I-NEXT:    seqz s2, a0
+; RV32I-NEXT:    call __unordsf2
+; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __unordsf2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    or a0, a0, s2
+; RV32I-NEXT:    call __eqsf2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, s2, a0
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -1014,13 +1014,13 @@ define i32 @fcmps_ueq(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    call __eqsf2
-; RV64I-NEXT:    seqz s2, a0
+; RV64I-NEXT:    call __unordsf2
+; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __unordsf2
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    or a0, a0, s2
+; RV64I-NEXT:    call __eqsf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, s2, a0
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp.ll
index 265d553a3e5d..2e9c39f331bb 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/float-fcmp.ll
@@ -344,13 +344,13 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind {
 ; RV32I-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    call __eqsf2
-; RV32I-NEXT:    seqz s2, a0
+; RV32I-NEXT:    call __unordsf2
+; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __unordsf2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    or a0, a0, s2
+; RV32I-NEXT:    call __eqsf2
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    or a0, s2, a0
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -367,13 +367,13 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    call __eqsf2
-; RV64I-NEXT:    seqz s2, a0
+; RV64I-NEXT:    call __unordsf2
+; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call __unordsf2
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    or a0, a0, s2
+; RV64I-NEXT:    call __eqsf2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    or a0, s2, a0
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/float-imm.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/float-imm.ll
index 58cbc72e2197..a010ab49b282 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/float-imm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/float-imm.ll
@@ -16,19 +16,12 @@ define float @float_imm() nounwind {
 ; CHECK-NEXT:    flw fa0, %lo(.LCPI0_0)(a0)
 ; CHECK-NEXT:    ret
 ;
-; RV32ZFINX-LABEL: float_imm:
-; RV32ZFINX:       # %bb.0:
-; RV32ZFINX-NEXT:    lui a0, 263313
-; RV32ZFINX-NEXT:    addi a0, a0, -37
-; RV32ZFINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
-; RV32ZFINX-NEXT:    ret
-;
-; RV64ZFINX-LABEL: float_imm:
-; RV64ZFINX:       # %bb.0:
-; RV64ZFINX-NEXT:    lui a0, 263313
-; RV64ZFINX-NEXT:    addiw a0, a0, -37
-; RV64ZFINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
-; RV64ZFINX-NEXT:    ret
+; CHECKZFINX-LABEL: float_imm:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    lui a0, 263313
+; CHECKZFINX-NEXT:    addi a0, a0, -37
+; CHECKZFINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
+; CHECKZFINX-NEXT:    ret
   ret float 3.14159274101257324218750
 }
 
@@ -75,3 +68,6 @@ define float @float_negative_zero(ptr %pf) nounwind {
 ; CHECKZFINX-NEXT:    ret
   ret float -0.0
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32ZFINX: {{.*}}
+; RV64ZFINX: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index eb81e5b0eb80..be9ddc68ce66 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -1655,7 +1655,7 @@ define i1 @fpclass(float %x) {
 ; RV64I-NEXT:    lui a3, 522240
 ; RV64I-NEXT:    lui a4, 1046528
 ; RV64I-NEXT:    srli a0, a0, 33
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    slti a1, a1, 0
 ; RV64I-NEXT:    addi a5, a0, -1
 ; RV64I-NEXT:    sltu a2, a5, a2
@@ -1764,7 +1764,7 @@ define i1 @isqnan_fpclass(float %x) {
 ; RV64I-NEXT:    slli a0, a0, 33
 ; RV64I-NEXT:    lui a1, 523264
 ; RV64I-NEXT:    srli a0, a0, 33
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    slt a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 2)  ; qnan
@@ -2153,7 +2153,7 @@ define i1 @isnotfinite_fpclass(float %x) {
 ; RV64I-NEXT:    slli a0, a0, 33
 ; RV64I-NEXT:    lui a1, 522240
 ; RV64I-NEXT:    srli a0, a0, 33
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    slt a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 519)  ; ox207 = "inf|nan"
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
index 291c8c3d7c90..477a7d1ce7b6 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
@@ -1263,7 +1263,7 @@ define i1 @pr134525() nounwind {
 ; RV64I-NEXT:    lui a0, %hi(ki_end+2145386496)
 ; RV64I-NEXT:    addi a0, a0, %lo(ki_end+2145386496)
 ; RV64I-NEXT:    lui a1, 32
-; RV64I-NEXT:    addiw a1, a1, 1
+; RV64I-NEXT:    addi a1, a1, 1
 ; RV64I-NEXT:    sltu a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -1275,7 +1275,7 @@ define i1 @pr134525() nounwind {
 ; RV64I-MEDIUM-NEXT:    addi a0, a0, %pcrel_lo(.Lpcrel_hi15)
 ; RV64I-MEDIUM-NEXT:    add a0, a0, a1
 ; RV64I-MEDIUM-NEXT:    lui a1, 32
-; RV64I-MEDIUM-NEXT:    addiw a1, a1, 1
+; RV64I-MEDIUM-NEXT:    addi a1, a1, 1
 ; RV64I-MEDIUM-NEXT:    sltu a0, a0, a1
 ; RV64I-MEDIUM-NEXT:    ret
 ;
@@ -1287,7 +1287,7 @@ define i1 @pr134525() nounwind {
 ; RV64I-LARGE-NEXT:    lui a1, 523776
 ; RV64I-LARGE-NEXT:    add a0, a0, a1
 ; RV64I-LARGE-NEXT:    lui a1, 32
-; RV64I-LARGE-NEXT:    addiw a1, a1, 1
+; RV64I-LARGE-NEXT:    addi a1, a1, 1
 ; RV64I-LARGE-NEXT:    sltu a0, a0, a1
 ; RV64I-LARGE-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/fold-masked-merge.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/fold-masked-merge.ll
new file mode 100644
index 000000000000..631b7109281e
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/fold-masked-merge.ll
@@ -0,0 +1,302 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,CHECK-I,RV32,RV32I
+; RUN: llc -mtriple=riscv64 < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,CHECK-I,RV64,RV64I
+; RUN: llc -mtriple=riscv32 -mattr=+zbb < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,CHECK-ZBB,RV32,RV32ZBB
+; RUN: llc -mtriple=riscv64 -mattr=+zbb < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,CHECK-ZBB,RV64,RV64ZBB
+;
+; test that masked-merge code is generated as "xor;and;xor" sequence or
+; "andn ; and; or" if and-not is available.
+
+define i32 @masked_merge0(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-I-LABEL: masked_merge0:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    xor a1, a1, a2
+; CHECK-I-NEXT:    and a0, a1, a0
+; CHECK-I-NEXT:    xor a0, a0, a2
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: masked_merge0:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    and a1, a0, a1
+; CHECK-ZBB-NEXT:    andn a0, a2, a0
+; CHECK-ZBB-NEXT:    or a0, a1, a0
+; CHECK-ZBB-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i16 @masked_merge1(i16 %a0, i16 %a1, i16 %a2) {
+; CHECK-I-LABEL: masked_merge1:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    xor a1, a1, a2
+; CHECK-I-NEXT:    and a0, a1, a0
+; CHECK-I-NEXT:    xor a0, a0, a2
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: masked_merge1:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    and a1, a0, a1
+; CHECK-ZBB-NEXT:    andn a0, a2, a0
+; CHECK-ZBB-NEXT:    or a0, a1, a0
+; CHECK-ZBB-NEXT:    ret
+  %and0 = and i16 %a0, %a1
+  %not = xor i16 %a0, -1
+  %and1 = and i16 %a2, %not
+  %or = or i16 %and0, %and1
+  ret i16 %or
+}
+
+define i8 @masked_merge2(i8 %a0, i8 %a1, i8 %a2) {
+; CHECK-I-LABEL: masked_merge2:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    mv a0, a1
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: masked_merge2:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    andn a2, a1, a0
+; CHECK-ZBB-NEXT:    and a0, a1, a0
+; CHECK-ZBB-NEXT:    or a0, a2, a0
+; CHECK-ZBB-NEXT:    ret
+  %not = xor i8 %a0, -1
+  %and0 = and i8 %not, %a1
+  %and1 = and i8 %a1, %a0
+  %or = or i8 %and0, %and1
+  ret i8 %or
+}
+
+define i64 @masked_merge3(i64 %a0, i64 %a1, i64 %a2) {
+; RV32I-LABEL: masked_merge3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    not a5, a5
+; RV32I-NEXT:    not a4, a4
+; RV32I-NEXT:    xor a3, a3, a5
+; RV32I-NEXT:    xor a2, a2, a4
+; RV32I-NEXT:    not a2, a2
+; RV32I-NEXT:    not a3, a3
+; RV32I-NEXT:    and a0, a2, a0
+; RV32I-NEXT:    and a1, a3, a1
+; RV32I-NEXT:    xor a0, a0, a4
+; RV32I-NEXT:    xor a1, a1, a5
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: masked_merge3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    not a2, a2
+; RV64I-NEXT:    xor a1, a1, a2
+; RV64I-NEXT:    not a1, a1
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    xor a0, a0, a2
+; RV64I-NEXT:    ret
+;
+; RV32ZBB-LABEL: masked_merge3:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    not a6, a0
+; RV32ZBB-NEXT:    not a7, a1
+; RV32ZBB-NEXT:    andn a1, a1, a3
+; RV32ZBB-NEXT:    andn a0, a0, a2
+; RV32ZBB-NEXT:    andn a2, a7, a5
+; RV32ZBB-NEXT:    andn a3, a6, a4
+; RV32ZBB-NEXT:    or a0, a3, a0
+; RV32ZBB-NEXT:    or a1, a2, a1
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: masked_merge3:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    not a3, a0
+; RV64ZBB-NEXT:    andn a2, a3, a2
+; RV64ZBB-NEXT:    andn a0, a0, a1
+; RV64ZBB-NEXT:    or a0, a2, a0
+; RV64ZBB-NEXT:    ret
+  %v0 = xor i64 %a1, -1
+  %v1 = xor i64 %a2, -1
+  %not = xor i64 %a0, -1
+  %and0 = and i64 %not, %v1
+  %and1 = and i64 %v0, %a0
+  %or = or i64 %and0, %and1
+  ret i64 %or
+}
+
+define i32 @not_a_masked_merge0(i32 %a0, i32 %a1, i32 %a2) {
+; RV32-LABEL: not_a_masked_merge0:
+; RV32:       # %bb.0:
+; RV32-NEXT:    and a1, a0, a1
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    or a0, a1, a0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: not_a_masked_merge0:
+; RV64:       # %bb.0:
+; RV64-NEXT:    and a1, a0, a1
+; RV64-NEXT:    negw a0, a0
+; RV64-NEXT:    and a0, a0, a2
+; RV64-NEXT:    or a0, a1, a0
+; RV64-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not_a_not = sub i32 0, %a0
+  %and1 = and i32 %not_a_not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge1(i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
+; CHECK-I-LABEL: not_a_masked_merge1:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    and a0, a0, a1
+; CHECK-I-NEXT:    not a1, a3
+; CHECK-I-NEXT:    and a1, a1, a2
+; CHECK-I-NEXT:    or a0, a0, a1
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: not_a_masked_merge1:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    and a0, a0, a1
+; CHECK-ZBB-NEXT:    andn a1, a2, a3
+; CHECK-ZBB-NEXT:    or a0, a0, a1
+; CHECK-ZBB-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a3, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge2(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-I-LABEL: not_a_masked_merge2:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    or a1, a0, a1
+; CHECK-I-NEXT:    not a0, a0
+; CHECK-I-NEXT:    and a0, a0, a2
+; CHECK-I-NEXT:    or a0, a1, a0
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: not_a_masked_merge2:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    or a1, a0, a1
+; CHECK-ZBB-NEXT:    andn a0, a2, a0
+; CHECK-ZBB-NEXT:    or a0, a1, a0
+; CHECK-ZBB-NEXT:    ret
+  %not_an_and0 = or i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %not_an_and0, %and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge3(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-I-LABEL: not_a_masked_merge3:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    and a1, a0, a1
+; CHECK-I-NEXT:    xor a0, a0, a2
+; CHECK-I-NEXT:    not a0, a0
+; CHECK-I-NEXT:    or a0, a1, a0
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: not_a_masked_merge3:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    and a1, a0, a1
+; CHECK-ZBB-NEXT:    xor a0, a0, a2
+; CHECK-ZBB-NEXT:    orn a0, a1, a0
+; CHECK-ZBB-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %not_an_and1 = xor i32 %not, %a2
+  %or = or i32 %and0, %not_an_and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge4(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-LABEL: not_a_masked_merge4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a2, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i32 @masked_merge_no_transform0(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; CHECK-I-LABEL: masked_merge_no_transform0:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    and a1, a0, a1
+; CHECK-I-NEXT:    not a0, a0
+; CHECK-I-NEXT:    and a0, a0, a2
+; CHECK-I-NEXT:    or a0, a1, a0
+; CHECK-I-NEXT:    sw a1, 0(a3)
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: masked_merge_no_transform0:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    and a1, a0, a1
+; CHECK-ZBB-NEXT:    andn a0, a2, a0
+; CHECK-ZBB-NEXT:    or a0, a1, a0
+; CHECK-ZBB-NEXT:    sw a1, 0(a3)
+; CHECK-ZBB-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  store i32 %and0, ptr %p1
+  ret i32 %or
+}
+
+define i32 @masked_merge_no_transform1(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; CHECK-I-LABEL: masked_merge_no_transform1:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    and a1, a0, a1
+; CHECK-I-NEXT:    not a4, a0
+; CHECK-I-NEXT:    and a0, a4, a2
+; CHECK-I-NEXT:    or a0, a1, a0
+; CHECK-I-NEXT:    sw a4, 0(a3)
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: masked_merge_no_transform1:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    and a1, a0, a1
+; CHECK-ZBB-NEXT:    not a4, a0
+; CHECK-ZBB-NEXT:    andn a0, a2, a0
+; CHECK-ZBB-NEXT:    or a0, a1, a0
+; CHECK-ZBB-NEXT:    sw a4, 0(a3)
+; CHECK-ZBB-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  store i32 %not, ptr %p1
+  ret i32 %or
+}
+
+define i32 @masked_merge_no_transform2(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; CHECK-I-LABEL: masked_merge_no_transform2:
+; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    and a1, a0, a1
+; CHECK-I-NEXT:    not a0, a0
+; CHECK-I-NEXT:    and a2, a0, a2
+; CHECK-I-NEXT:    or a0, a1, a2
+; CHECK-I-NEXT:    sw a2, 0(a3)
+; CHECK-I-NEXT:    ret
+;
+; CHECK-ZBB-LABEL: masked_merge_no_transform2:
+; CHECK-ZBB:       # %bb.0:
+; CHECK-ZBB-NEXT:    and a1, a0, a1
+; CHECK-ZBB-NEXT:    andn a2, a2, a0
+; CHECK-ZBB-NEXT:    or a0, a1, a2
+; CHECK-ZBB-NEXT:    sw a2, 0(a3)
+; CHECK-ZBB-NEXT:    ret
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  store i32 %and1, ptr %p1
+  ret i32 %or
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/fold-mem-offset.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/fold-mem-offset.ll
index 7d8b8d29aa3c..e6f1a08cc187 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/fold-mem-offset.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/fold-mem-offset.ll
@@ -213,12 +213,12 @@ define i64 @test_sh3add_uw(ptr %p, i32 signext %x, i32 signext %y) {
 ; RV32I-NEXT:    slli a2, a2, 3
 ; RV32I-NEXT:    add a1, a0, a1
 ; RV32I-NEXT:    add a0, a0, a2
-; RV32I-NEXT:    lw a2, 404(a0)
-; RV32I-NEXT:    lw a3, 400(a1)
+; RV32I-NEXT:    lw a2, 400(a1)
 ; RV32I-NEXT:    lw a1, 404(a1)
+; RV32I-NEXT:    lw a3, 404(a0)
 ; RV32I-NEXT:    lw a4, 400(a0)
-; RV32I-NEXT:    add a1, a2, a1
-; RV32I-NEXT:    add a0, a4, a3
+; RV32I-NEXT:    add a1, a3, a1
+; RV32I-NEXT:    add a0, a4, a2
 ; RV32I-NEXT:    sltu a2, a0, a4
 ; RV32I-NEXT:    add a1, a1, a2
 ; RV32I-NEXT:    ret
@@ -240,12 +240,12 @@ define i64 @test_sh3add_uw(ptr %p, i32 signext %x, i32 signext %y) {
 ; RV32ZBA:       # %bb.0: # %entry
 ; RV32ZBA-NEXT:    sh3add a1, a1, a0
 ; RV32ZBA-NEXT:    sh3add a0, a2, a0
-; RV32ZBA-NEXT:    lw a2, 404(a0)
-; RV32ZBA-NEXT:    lw a3, 400(a1)
+; RV32ZBA-NEXT:    lw a2, 400(a1)
 ; RV32ZBA-NEXT:    lw a1, 404(a1)
+; RV32ZBA-NEXT:    lw a3, 404(a0)
 ; RV32ZBA-NEXT:    lw a4, 400(a0)
-; RV32ZBA-NEXT:    add a1, a2, a1
-; RV32ZBA-NEXT:    add a0, a4, a3
+; RV32ZBA-NEXT:    add a1, a3, a1
+; RV32ZBA-NEXT:    add a0, a4, a2
 ; RV32ZBA-NEXT:    sltu a2, a0, a4
 ; RV32ZBA-NEXT:    add a1, a1, a2
 ; RV32ZBA-NEXT:    ret
@@ -575,12 +575,11 @@ define signext i32 @test_large_offset(ptr %p, iXLen %x, iXLen %y) {
 ; RV64I-NEXT:    lui a3, 2
 ; RV64I-NEXT:    slli a1, a1, 2
 ; RV64I-NEXT:    slli a2, a2, 2
-; RV64I-NEXT:    addiw a3, a3, -1392
 ; RV64I-NEXT:    add a0, a0, a3
 ; RV64I-NEXT:    add a1, a0, a1
 ; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    lw a1, 0(a1)
-; RV64I-NEXT:    lw a0, 40(a0)
+; RV64I-NEXT:    lw a1, -1392(a1)
+; RV64I-NEXT:    lw a0, -1352(a0)
 ; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/fpclamptosat.ll
index c5c3b199447a..117e3e4aac45 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/fpclamptosat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/fpclamptosat.ll
@@ -585,7 +585,7 @@ define i16 @stest_f64i16(double %x) {
 ; RV64IF-NEXT:    .cfi_offset ra, -8
 ; RV64IF-NEXT:    call __fixdfsi
 ; RV64IF-NEXT:    lui a1, 8
-; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    blt a0, a1, .LBB9_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
 ; RV64IF-NEXT:    mv a0, a1
@@ -624,7 +624,7 @@ define i16 @stest_f64i16(double %x) {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; RV64IFD-NEXT:    lui a1, 8
-; RV64IFD-NEXT:    addiw a1, a1, -1
+; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    bge a0, a1, .LBB9_3
 ; RV64IFD-NEXT:  # %bb.1: # %entry
 ; RV64IFD-NEXT:    lui a1, 1048568
@@ -676,7 +676,7 @@ define i16 @utest_f64i16(double %x) {
 ; RV64IF-NEXT:    .cfi_offset ra, -8
 ; RV64IF-NEXT:    call __fixunsdfsi
 ; RV64IF-NEXT:    lui a1, 16
-; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    bltu a0, a1, .LBB10_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
 ; RV64IF-NEXT:    mv a0, a1
@@ -702,7 +702,7 @@ define i16 @utest_f64i16(double %x) {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    lui a1, 16
-; RV64IFD-NEXT:    addiw a1, a1, -1
+; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    bltu a0, a1, .LBB10_2
 ; RV64IFD-NEXT:  # %bb.1: # %entry
 ; RV64IFD-NEXT:    mv a0, a1
@@ -747,7 +747,7 @@ define i16 @ustest_f64i16(double %x) {
 ; RV64IF-NEXT:    .cfi_offset ra, -8
 ; RV64IF-NEXT:    call __fixdfsi
 ; RV64IF-NEXT:    lui a1, 16
-; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    blt a0, a1, .LBB11_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
 ; RV64IF-NEXT:    mv a0, a1
@@ -779,7 +779,7 @@ define i16 @ustest_f64i16(double %x) {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; RV64IFD-NEXT:    lui a1, 16
-; RV64IFD-NEXT:    addiw a1, a1, -1
+; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    blt a0, a1, .LBB11_2
 ; RV64IFD-NEXT:  # %bb.1: # %entry
 ; RV64IFD-NEXT:    mv a0, a1
@@ -822,7 +822,7 @@ define i16 @stest_f32i16(float %x) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 8
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    bge a0, a1, .LBB12_3
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    lui a1, 1048568
@@ -862,7 +862,7 @@ define i16 @utest_f32i16(float %x) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    bltu a0, a1, .LBB13_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -895,7 +895,7 @@ define i16 @ustest_f32i16(float %x) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    blt a0, a1, .LBB14_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -949,7 +949,7 @@ define i16 @stest_f16i16(half %x) {
 ; RV64-NEXT:    call __extendhfsf2
 ; RV64-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 8
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    blt a0, a1, .LBB15_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -1004,7 +1004,7 @@ define i16 @utesth_f16i16(half %x) {
 ; RV64-NEXT:    call __extendhfsf2
 ; RV64-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    bltu a0, a1, .LBB16_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -1055,7 +1055,7 @@ define i16 @ustest_f16i16(half %x) {
 ; RV64-NEXT:    call __extendhfsf2
 ; RV64-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    blt a0, a1, .LBB17_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -2536,7 +2536,7 @@ define i16 @stest_f64i16_mm(double %x) {
 ; RV64IF-NEXT:    .cfi_offset ra, -8
 ; RV64IF-NEXT:    call __fixdfsi
 ; RV64IF-NEXT:    lui a1, 8
-; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    blt a0, a1, .LBB36_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
 ; RV64IF-NEXT:    mv a0, a1
@@ -2575,7 +2575,7 @@ define i16 @stest_f64i16_mm(double %x) {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; RV64IFD-NEXT:    lui a1, 8
-; RV64IFD-NEXT:    addiw a1, a1, -1
+; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    bge a0, a1, .LBB36_3
 ; RV64IFD-NEXT:  # %bb.1: # %entry
 ; RV64IFD-NEXT:    lui a1, 1048568
@@ -2625,7 +2625,7 @@ define i16 @utest_f64i16_mm(double %x) {
 ; RV64IF-NEXT:    .cfi_offset ra, -8
 ; RV64IF-NEXT:    call __fixunsdfsi
 ; RV64IF-NEXT:    lui a1, 16
-; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    bltu a0, a1, .LBB37_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
 ; RV64IF-NEXT:    mv a0, a1
@@ -2651,7 +2651,7 @@ define i16 @utest_f64i16_mm(double %x) {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    lui a1, 16
-; RV64IFD-NEXT:    addiw a1, a1, -1
+; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    bltu a0, a1, .LBB37_2
 ; RV64IFD-NEXT:  # %bb.1: # %entry
 ; RV64IFD-NEXT:    mv a0, a1
@@ -2695,7 +2695,7 @@ define i16 @ustest_f64i16_mm(double %x) {
 ; RV64IF-NEXT:    .cfi_offset ra, -8
 ; RV64IF-NEXT:    call __fixdfsi
 ; RV64IF-NEXT:    lui a1, 16
-; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    blt a0, a1, .LBB38_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
 ; RV64IF-NEXT:    mv a0, a1
@@ -2727,7 +2727,7 @@ define i16 @ustest_f64i16_mm(double %x) {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; RV64IFD-NEXT:    lui a1, 16
-; RV64IFD-NEXT:    addiw a1, a1, -1
+; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    blt a0, a1, .LBB38_2
 ; RV64IFD-NEXT:  # %bb.1: # %entry
 ; RV64IFD-NEXT:    mv a0, a1
@@ -2768,7 +2768,7 @@ define i16 @stest_f32i16_mm(float %x) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 8
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    bge a0, a1, .LBB39_3
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    lui a1, 1048568
@@ -2806,7 +2806,7 @@ define i16 @utest_f32i16_mm(float %x) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    bltu a0, a1, .LBB40_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -2838,7 +2838,7 @@ define i16 @ustest_f32i16_mm(float %x) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    blt a0, a1, .LBB41_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -2890,7 +2890,7 @@ define i16 @stest_f16i16_mm(half %x) {
 ; RV64-NEXT:    call __extendhfsf2
 ; RV64-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 8
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    blt a0, a1, .LBB42_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -2943,7 +2943,7 @@ define i16 @utesth_f16i16_mm(half %x) {
 ; RV64-NEXT:    call __extendhfsf2
 ; RV64-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    bltu a0, a1, .LBB43_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -2993,7 +2993,7 @@ define i16 @ustest_f16i16_mm(half %x) {
 ; RV64-NEXT:    call __extendhfsf2
 ; RV64-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    blt a0, a1, .LBB44_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/fpenv-xlen.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/fpenv-xlen.ll
new file mode 100644
index 000000000000..148186b21c12
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/fpenv-xlen.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs -O0 | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs -O0 | FileCheck %s
+
+define iXLen @func_get_fpenv() {
+; CHECK-LABEL: func_get_fpenv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    frcsr a0
+; CHECK-NEXT:    ret
+entry:
+  %fpenv = call iXLen @llvm.get.fpenv.iXLen()
+  ret iXLen %fpenv
+}
+
+define void @func_set_fpenv(iXLen %fpenv) {
+; CHECK-LABEL: func_set_fpenv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fscsr a0
+; CHECK-NEXT:    ret
+entry:
+  call void @llvm.set.fpenv.iXLen(iXLen %fpenv)
+  ret void
+}
+
+define void @func_reset_fpenv() {
+; CHECK-LABEL: func_reset_fpenv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fscsr zero
+; CHECK-NEXT:    ret
+entry:
+  call void @llvm.reset.fpenv()
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/fpenv.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/fpenv.ll
index 11e104a290e1..b4a1400dbd54 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/fpenv.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/fpenv.ll
@@ -18,7 +18,7 @@ define i32 @func_01() {
 ; RV64IF-NEXT:    frrm a0
 ; RV64IF-NEXT:    lui a1, 66
 ; RV64IF-NEXT:    slli a0, a0, 2
-; RV64IF-NEXT:    addiw a1, a1, 769
+; RV64IF-NEXT:    addi a1, a1, 769
 ; RV64IF-NEXT:    srl a0, a1, a0
 ; RV64IF-NEXT:    andi a0, a0, 7
 ; RV64IF-NEXT:    ret
@@ -77,7 +77,7 @@ define i1 @test_get_rounding_sideeffect() #0 {
 ; RV64IF-NEXT:    frrm a0
 ; RV64IF-NEXT:    lui a1, 66
 ; RV64IF-NEXT:    slli a0, a0, 2
-; RV64IF-NEXT:    addiw s0, a1, 769
+; RV64IF-NEXT:    addi s0, a1, 769
 ; RV64IF-NEXT:    srl a0, s0, a0
 ; RV64IF-NEXT:    andi a0, a0, 7
 ; RV64IF-NEXT:    beqz a0, .LBB1_2
@@ -133,7 +133,7 @@ define void @func_02(i32 %rm) {
 ; RV64IF-NEXT:    slli a0, a0, 32
 ; RV64IF-NEXT:    lui a1, 66
 ; RV64IF-NEXT:    srli a0, a0, 30
-; RV64IF-NEXT:    addiw a1, a1, 769
+; RV64IF-NEXT:    addi a1, a1, 769
 ; RV64IF-NEXT:    srl a0, a1, a0
 ; RV64IF-NEXT:    andi a0, a0, 7
 ; RV64IF-NEXT:    fsrm a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/frm-write-in-loop.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/frm-write-in-loop.ll
index 4f435067343b..72c595117827 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/frm-write-in-loop.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/frm-write-in-loop.ll
@@ -90,3 +90,34 @@ loop:
 exit:
     ret double %f2
 }
+
+define double @foo2(double %0, double %1, i64 %n, i64 %fcsr) strictfp {
+; CHECK-LABEL: foo2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fmv.d.x fa5, zero
+; CHECK-NEXT:  .LBB2_1: # %loop
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    csrwi fcsr, 0
+; CHECK-NEXT:    fadd.d fa5, fa5, fa0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    fscsr a1
+; CHECK-NEXT:    fadd.d fa5, fa5, fa1
+; CHECK-NEXT:    beqz a0, .LBB2_1
+; CHECK-NEXT:  # %bb.2: # %exit
+; CHECK-NEXT:    fmv.d fa0, fa5
+; CHECK-NEXT:    ret
+entry:
+    br label %loop
+loop:
+    %cnt = phi i64 [0, %entry], [%cnt_inc, %loop]
+    %acc = phi double [0.0, %entry], [%f2, %loop]
+    call void @llvm.set.fpenv(i64 0) strictfp
+    %f1 = call double @llvm.experimental.constrained.fadd.f64(double %acc, double %0, metadata !"round.dynamic", metadata !"fpexcept.ignore") strictfp
+    call void @llvm.set.fpenv(i64 %fcsr) strictfp
+    %f2 = call double @llvm.experimental.constrained.fadd.f64(double %f1, double %1, metadata !"round.dynamic", metadata !"fpexcept.ignore") strictfp
+    %cnt_inc = add i64 %cnt, 1
+    %cond = icmp eq i64 %cnt_inc, %n
+    br i1 %cond, label %loop, label %exit
+exit:
+    ret double %f2
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/half-arith.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/half-arith.ll
index a218e89948d4..84163b52bb98 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/half-arith.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/half-arith.ll
@@ -71,7 +71,7 @@ define half @fadd_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -153,7 +153,7 @@ define half @fsub_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -235,7 +235,7 @@ define half @fmul_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -317,7 +317,7 @@ define half @fdiv_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -546,7 +546,7 @@ define i32 @fneg_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    addi s1, a1, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv a1, a0
@@ -667,7 +667,7 @@ define half @fsgnjn_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 16
-; RV64I-NEXT:    addiw s3, a0, -1
+; RV64I-NEXT:    addi s3, a0, -1
 ; RV64I-NEXT:    and a0, s1, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -831,7 +831,7 @@ define half @fabs_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -972,7 +972,7 @@ define half @fmin_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -1056,7 +1056,7 @@ define half @fmax_h(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -1149,7 +1149,7 @@ define half @fmadd_h(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s3, a1, -1
+; RV64I-NEXT:    addi s3, a1, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -1261,7 +1261,7 @@ define half @fmsub_h(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 16
-; RV64I-NEXT:    addiw s3, a0, -1
+; RV64I-NEXT:    addi s3, a0, -1
 ; RV64I-NEXT:    and a0, a2, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -1416,7 +1416,7 @@ define half @fnmadd_h(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s1, a2
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui s3, 16
-; RV64I-NEXT:    addiw s3, s3, -1
+; RV64I-NEXT:    addi s3, s3, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -1596,7 +1596,7 @@ define half @fnmadd_h_2(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s1, a2
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui s3, 16
-; RV64I-NEXT:    addiw s3, s3, -1
+; RV64I-NEXT:    addi s3, s3, -1
 ; RV64I-NEXT:    and a0, a1, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -1761,7 +1761,7 @@ define half @fnmadd_h_3(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s3, a1, -1
+; RV64I-NEXT:    addi s3, a1, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -1882,7 +1882,7 @@ define half @fnmadd_nsz(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s3, a1, -1
+; RV64I-NEXT:    addi s3, a1, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -2003,7 +2003,7 @@ define half @fnmsub_h(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s3, a1, -1
+; RV64I-NEXT:    addi s3, a1, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -2140,7 +2140,7 @@ define half @fnmsub_h_2(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 16
-; RV64I-NEXT:    addiw s3, a0, -1
+; RV64I-NEXT:    addi s3, a0, -1
 ; RV64I-NEXT:    and a0, a1, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -2269,7 +2269,7 @@ define half @fmadd_h_contract(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s3, a1, -1
+; RV64I-NEXT:    addi s3, a1, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -2393,7 +2393,7 @@ define half @fmsub_h_contract(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 16
-; RV64I-NEXT:    addiw s3, a0, -1
+; RV64I-NEXT:    addi s3, a0, -1
 ; RV64I-NEXT:    and a0, a2, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -2552,7 +2552,7 @@ define half @fnmadd_h_contract(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui s3, 16
-; RV64I-NEXT:    addiw s3, s3, -1
+; RV64I-NEXT:    addi s3, s3, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -2737,7 +2737,7 @@ define half @fnmsub_h_contract(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui s3, 16
-; RV64I-NEXT:    addiw s3, s3, -1
+; RV64I-NEXT:    addi s3, s3, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    li a1, 0
@@ -2871,7 +2871,7 @@ define half @fsgnjx_f16(half %x, half %y) nounwind {
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a0, 12
-; RV64I-NEXT:    addiw a0, a0, -1024
+; RV64I-NEXT:    addi a0, a0, -1024
 ; RV64I-NEXT:    and a0, s1, a0
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv a1, s0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/half-convert.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/half-convert.ll
index f59f86ded76b..c53237ed6aef 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/half-convert.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -272,7 +272,7 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64IZHINX-NEXT:    lui a2, 290816
 ; RV64IZHINX-NEXT:    fmax.s a1, a0, a1
 ; RV64IZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZHINX-NEXT:    addiw a2, a2, -512
+; RV64IZHINX-NEXT:    addi a2, a2, -512
 ; RV64IZHINX-NEXT:    neg a0, a0
 ; RV64IZHINX-NEXT:    fmin.s a1, a1, a2
 ; RV64IZHINX-NEXT:    fcvt.l.s a1, a1, rtz
@@ -300,7 +300,7 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64IZDINXZHINX-NEXT:    lui a2, 290816
 ; RV64IZDINXZHINX-NEXT:    fmax.s a1, a0, a1
 ; RV64IZDINXZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZDINXZHINX-NEXT:    addiw a2, a2, -512
+; RV64IZDINXZHINX-NEXT:    addi a2, a2, -512
 ; RV64IZDINXZHINX-NEXT:    neg a0, a0
 ; RV64IZDINXZHINX-NEXT:    fmin.s a1, a1, a2
 ; RV64IZDINXZHINX-NEXT:    fcvt.l.s a1, a1, rtz
@@ -372,13 +372,13 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    lui s1, 1048568
 ; RV64I-NEXT:  .LBB1_2: # %start
 ; RV64I-NEXT:    lui a0, 290816
-; RV64I-NEXT:    addiw a1, a0, -512
+; RV64I-NEXT:    addi a1, a0, -512
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB1_4
 ; RV64I-NEXT:  # %bb.3: # %start
 ; RV64I-NEXT:    lui s1, 8
-; RV64I-NEXT:    addiw s1, s1, -1
+; RV64I-NEXT:    addi s1, s1, -1
 ; RV64I-NEXT:  .LBB1_4: # %start
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s0
@@ -522,7 +522,7 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; CHECK64-IZHINXMIN-NEXT:    lui a2, 290816
 ; CHECK64-IZHINXMIN-NEXT:    fmax.s a1, a0, a1
 ; CHECK64-IZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZHINXMIN-NEXT:    addiw a2, a2, -512
+; CHECK64-IZHINXMIN-NEXT:    addi a2, a2, -512
 ; CHECK64-IZHINXMIN-NEXT:    neg a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    fmin.s a1, a1, a2
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
@@ -550,7 +550,7 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lui a2, 290816
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a1, a0, a1
 ; CHECK64-IZDINXZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    addiw a2, a2, -512
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi a2, a2, -512
 ; CHECK64-IZDINXZHINXMIN-NEXT:    neg a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a1, a1, a2
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
@@ -768,7 +768,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
 ; RV64IZHINX-NEXT:    lui a1, 292864
 ; RV64IZHINX-NEXT:    fmax.s a0, a0, zero
-; RV64IZHINX-NEXT:    addiw a1, a1, -256
+; RV64IZHINX-NEXT:    addi a1, a1, -256
 ; RV64IZHINX-NEXT:    fmin.s a0, a0, a1
 ; RV64IZHINX-NEXT:    fcvt.lu.s a0, a0, rtz
 ; RV64IZHINX-NEXT:    ret
@@ -788,7 +788,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
 ; RV64IZDINXZHINX-NEXT:    lui a1, 292864
 ; RV64IZDINXZHINX-NEXT:    fmax.s a0, a0, zero
-; RV64IZDINXZHINX-NEXT:    addiw a1, a1, -256
+; RV64IZDINXZHINX-NEXT:    addi a1, a1, -256
 ; RV64IZDINXZHINX-NEXT:    fmin.s a0, a0, a1
 ; RV64IZDINXZHINX-NEXT:    fcvt.lu.s a0, a0, rtz
 ; RV64IZDINXZHINX-NEXT:    ret
@@ -840,7 +840,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui s0, 16
-; RV64I-NEXT:    addiw s0, s0, -1
+; RV64I-NEXT:    addi s0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s0
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s3, a0
@@ -851,7 +851,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    call __gesf2
 ; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    lui a0, 292864
-; RV64I-NEXT:    addiw a1, a0, -256
+; RV64I-NEXT:    addi a1, a0, -256
 ; RV64I-NEXT:    mv a0, s3
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    bgtz a0, .LBB3_2
@@ -968,7 +968,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    lui a1, 292864
 ; CHECK64-IZHINXMIN-NEXT:    fmax.s a0, a0, zero
-; CHECK64-IZHINXMIN-NEXT:    addiw a1, a1, -256
+; CHECK64-IZHINXMIN-NEXT:    addi a1, a1, -256
 ; CHECK64-IZHINXMIN-NEXT:    fmin.s a0, a0, a1
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.lu.s a0, a0, rtz
 ; CHECK64-IZHINXMIN-NEXT:    ret
@@ -988,7 +988,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lui a1, 292864
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a0, a0, zero
-; CHECK64-IZDINXZHINXMIN-NEXT:    addiw a1, a1, -256
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi a1, a1, -256
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a0, a0, a1
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.lu.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
@@ -1244,7 +1244,7 @@ define i32 @fcvt_w_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    lui s1, 524288
 ; RV64I-NEXT:  .LBB5_2: # %start
 ; RV64I-NEXT:    lui a1, 323584
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB5_4
@@ -1819,7 +1819,7 @@ define i32 @fcvt_wu_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    call __fixunssfdi
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a1, 325632
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB8_2
@@ -2411,7 +2411,7 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    slli s1, s3, 63
 ; RV64I-NEXT:  .LBB10_2: # %start
 ; RV64I-NEXT:    lui a1, 389120
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB10_4
@@ -3076,7 +3076,7 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a1, 391168
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    sgtz a0, a0
 ; RV64I-NEXT:    neg s1, a0
@@ -6374,7 +6374,7 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64IZHINX-NEXT:    lui a2, 290816
 ; RV64IZHINX-NEXT:    fmax.s a1, a0, a1
 ; RV64IZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZHINX-NEXT:    addiw a2, a2, -512
+; RV64IZHINX-NEXT:    addi a2, a2, -512
 ; RV64IZHINX-NEXT:    neg a0, a0
 ; RV64IZHINX-NEXT:    fmin.s a1, a1, a2
 ; RV64IZHINX-NEXT:    fcvt.l.s a1, a1, rtz
@@ -6402,7 +6402,7 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64IZDINXZHINX-NEXT:    lui a2, 290816
 ; RV64IZDINXZHINX-NEXT:    fmax.s a1, a0, a1
 ; RV64IZDINXZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZDINXZHINX-NEXT:    addiw a2, a2, -512
+; RV64IZDINXZHINX-NEXT:    addi a2, a2, -512
 ; RV64IZDINXZHINX-NEXT:    neg a0, a0
 ; RV64IZDINXZHINX-NEXT:    fmin.s a1, a1, a2
 ; RV64IZDINXZHINX-NEXT:    fcvt.l.s a1, a1, rtz
@@ -6476,7 +6476,7 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64I-NEXT:    lui s1, 1048568
 ; RV64I-NEXT:  .LBB32_2: # %start
 ; RV64I-NEXT:    lui a0, 290816
-; RV64I-NEXT:    addiw a1, a0, -512
+; RV64I-NEXT:    addi a1, a0, -512
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB32_4
@@ -6628,7 +6628,7 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; CHECK64-IZHINXMIN-NEXT:    lui a2, 290816
 ; CHECK64-IZHINXMIN-NEXT:    fmax.s a1, a0, a1
 ; CHECK64-IZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZHINXMIN-NEXT:    addiw a2, a2, -512
+; CHECK64-IZHINXMIN-NEXT:    addi a2, a2, -512
 ; CHECK64-IZHINXMIN-NEXT:    neg a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    fmin.s a1, a1, a2
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
@@ -6656,7 +6656,7 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lui a2, 290816
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a1, a0, a1
 ; CHECK64-IZDINXZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    addiw a2, a2, -512
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi a2, a2, -512
 ; CHECK64-IZDINXZHINXMIN-NEXT:    neg a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a1, a1, a2
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
@@ -6873,7 +6873,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
 ; RV64IZHINX-NEXT:    lui a1, 292864
 ; RV64IZHINX-NEXT:    fmax.s a0, a0, zero
-; RV64IZHINX-NEXT:    addiw a1, a1, -256
+; RV64IZHINX-NEXT:    addi a1, a1, -256
 ; RV64IZHINX-NEXT:    fmin.s a0, a0, a1
 ; RV64IZHINX-NEXT:    fcvt.lu.s a0, a0, rtz
 ; RV64IZHINX-NEXT:    ret
@@ -6893,7 +6893,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
 ; RV64IZDINXZHINX-NEXT:    lui a1, 292864
 ; RV64IZDINXZHINX-NEXT:    fmax.s a0, a0, zero
-; RV64IZDINXZHINX-NEXT:    addiw a1, a1, -256
+; RV64IZDINXZHINX-NEXT:    addi a1, a1, -256
 ; RV64IZDINXZHINX-NEXT:    fmin.s a0, a0, a1
 ; RV64IZDINXZHINX-NEXT:    fcvt.lu.s a0, a0, rtz
 ; RV64IZDINXZHINX-NEXT:    ret
@@ -6948,7 +6948,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui s3, 16
-; RV64I-NEXT:    addiw s3, s3, -1
+; RV64I-NEXT:    addi s3, s3, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -6959,7 +6959,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; RV64I-NEXT:    call __gesf2
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 292864
-; RV64I-NEXT:    addiw a1, a0, -256
+; RV64I-NEXT:    addi a1, a0, -256
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB34_2
@@ -7079,7 +7079,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    lui a1, 292864
 ; CHECK64-IZHINXMIN-NEXT:    fmax.s a0, a0, zero
-; CHECK64-IZHINXMIN-NEXT:    addiw a1, a1, -256
+; CHECK64-IZHINXMIN-NEXT:    addi a1, a1, -256
 ; CHECK64-IZHINXMIN-NEXT:    fmin.s a0, a0, a1
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.lu.s a0, a0, rtz
 ; CHECK64-IZHINXMIN-NEXT:    ret
@@ -7099,7 +7099,7 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lui a1, 292864
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a0, a0, zero
-; CHECK64-IZDINXZHINXMIN-NEXT:    addiw a1, a1, -256
+; CHECK64-IZDINXZHINXMIN-NEXT:    addi a1, a1, -256
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a0, a0, a1
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.lu.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
@@ -8175,7 +8175,7 @@ define zeroext i32 @fcvt_wu_h_sat_zext(half %a) nounwind {
 ; RV64I-NEXT:    call __fixunssfdi
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a1, 325632
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB39_2
@@ -8444,7 +8444,7 @@ define signext i32 @fcvt_w_h_sat_sext(half %a) nounwind {
 ; RV64I-NEXT:    lui s1, 524288
 ; RV64I-NEXT:  .LBB40_2: # %start
 ; RV64I-NEXT:    lui a1, 323584
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    blez a0, .LBB40_4
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/half-imm.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/half-imm.ll
index 1045df1c3e76..d68e19d15b4b 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/half-imm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/half-imm.ll
@@ -38,7 +38,7 @@ define half @half_imm() nounwind {
 ; RV64IZHINX-LABEL: half_imm:
 ; RV64IZHINX:       # %bb.0:
 ; RV64IZHINX-NEXT:    lui a0, 4
-; RV64IZHINX-NEXT:    addiw a0, a0, 512
+; RV64IZHINX-NEXT:    addi a0, a0, 512
 ; RV64IZHINX-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINX-NEXT:    ret
 ;
@@ -48,19 +48,12 @@ define half @half_imm() nounwind {
 ; CHECKIZFHMIN-NEXT:    flh fa0, %lo(.LCPI0_0)(a0)
 ; CHECKIZFHMIN-NEXT:    ret
 ;
-; RV32IZHINXMIN-LABEL: half_imm:
-; RV32IZHINXMIN:       # %bb.0:
-; RV32IZHINXMIN-NEXT:    lui a0, 4
-; RV32IZHINXMIN-NEXT:    addi a0, a0, 512
-; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
-; RV32IZHINXMIN-NEXT:    ret
-;
-; RV64IZHINXMIN-LABEL: half_imm:
-; RV64IZHINXMIN:       # %bb.0:
-; RV64IZHINXMIN-NEXT:    lui a0, 4
-; RV64IZHINXMIN-NEXT:    addiw a0, a0, 512
-; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
-; RV64IZHINXMIN-NEXT:    ret
+; CHECKIZHINXMIN-LABEL: half_imm:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    lui a0, 4
+; CHECKIZHINXMIN-NEXT:    addi a0, a0, 512
+; CHECKIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
+; CHECKIZHINXMIN-NEXT:    ret
   ret half 3.0
 }
 
@@ -163,3 +156,6 @@ define half @half_negative_zero(ptr %pf) nounwind {
 ; CHECKIZHINXMIN-NEXT:    ret
   ret half -0.0
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32IZHINXMIN: {{.*}}
+; RV64IZHINXMIN: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/half-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/half-intrinsics.ll
index 458bd0dc8d17..4f0026175e7c 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/half-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/half-intrinsics.ll
@@ -649,7 +649,7 @@ define half @sincos_f16(half %a) nounwind {
 ; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s0, a0
@@ -905,7 +905,7 @@ define half @pow_f16(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -1748,7 +1748,7 @@ define half @fma_f16(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s3, a1, -1
+; RV64I-NEXT:    addi s3, a1, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -1853,7 +1853,7 @@ define half @fmuladd_f16(half %a, half %b, half %c) nounwind {
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s3, a1, -1
+; RV64I-NEXT:    addi s3, a1, -1
 ; RV64I-NEXT:    and a0, a0, s3
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s2, a0
@@ -2015,7 +2015,7 @@ define half @minnum_f16(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -2099,7 +2099,7 @@ define half @maxnum_f16(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -3143,7 +3143,7 @@ define half @maximumnum_half(half %x, half %y) {
 ; RV64I-NEXT:    .cfi_offset s2, -32
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -3247,7 +3247,7 @@ define half @minimumnum_half(half %x, half %y) {
 ; RV64I-NEXT:    .cfi_offset s2, -32
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
@@ -3990,7 +3990,7 @@ define half @atan2_f16(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/i64-icmp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/i64-icmp.ll
index e7efe76968cc..49103231a075 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/i64-icmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/i64-icmp.ll
@@ -28,7 +28,7 @@ define i64 @icmp_eq_constant_2049(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_eq_constant_2049:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, -2047
+; RV64I-NEXT:    addi a1, a1, -2047
 ; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    seqz a0, a0
 ; RV64I-NEXT:    ret
@@ -106,7 +106,7 @@ define i64 @icmp_ne_constant_2049(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_ne_constant_2049:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, -2047
+; RV64I-NEXT:    addi a1, a1, -2047
 ; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    snez a0, a0
 ; RV64I-NEXT:    ret
@@ -226,7 +226,7 @@ define i64 @icmp_ugt_constant_neg_2050(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_ugt_constant_neg_2050:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2046
+; RV64I-NEXT:    addi a1, a1, 2046
 ; RV64I-NEXT:    sltu a0, a1, a0
 ; RV64I-NEXT:    ret
 ; 18446744073709549566 signed extend is -2050
@@ -294,7 +294,7 @@ define i64 @icmp_uge_constant_neg_2049(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_uge_constant_neg_2049:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2046
+; RV64I-NEXT:    addi a1, a1, 2046
 ; RV64I-NEXT:    sltu a0, a1, a0
 ; RV64I-NEXT:    ret
 ; 18446744073709549567 signed extend is -2049
@@ -359,7 +359,7 @@ define i64 @icmp_ult_constant_neg_2049(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_ult_constant_neg_2049:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    addi a1, a1, 2047
 ; RV64I-NEXT:    sltu a0, a0, a1
 ; RV64I-NEXT:    ret
 ; 18446744073709549567 signed extend is -2049
@@ -425,7 +425,7 @@ define i64 @icmp_ule_constant_neg_2050(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_ule_constant_neg_2050:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    addi a1, a1, 2047
 ; RV64I-NEXT:    sltu a0, a0, a1
 ; RV64I-NEXT:    ret
 ; 18446744073709549566 signed extend is -2050
@@ -491,7 +491,7 @@ define i64 @icmp_sgt_constant_neg_2050(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_sgt_constant_neg_2050:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2046
+; RV64I-NEXT:    addi a1, a1, 2046
 ; RV64I-NEXT:    slt a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = icmp sgt i64 %a, -2050
@@ -621,7 +621,7 @@ define i64 @icmp_slt_constant_neg_2049(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_slt_constant_neg_2049:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    addi a1, a1, 2047
 ; RV64I-NEXT:    slt a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = icmp slt i64 %a, -2049
@@ -686,7 +686,7 @@ define i64 @icmp_sle_constant_neg_2050(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_sle_constant_neg_2050:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    addi a1, a1, 2047
 ; RV64I-NEXT:    slt a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = icmp sle i64 %a, -2050
@@ -712,7 +712,7 @@ define i64 @icmp_eq_zext_inreg_large_constant(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    lui a1, 563901
-; RV64I-NEXT:    addiw a1, a1, -529
+; RV64I-NEXT:    addi a1, a1, -529
 ; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    seqz a0, a0
 ; RV64I-NEXT:    ret
@@ -753,7 +753,7 @@ define i64 @icmp_ne_zext_inreg_umin(i64 %a) nounwind {
 ; RV64I-LABEL: icmp_ne_zext_inreg_umin:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 30141
-; RV64I-NEXT:    addiw a1, a1, -747
+; RV64I-NEXT:    addi a1, a1, -747
 ; RV64I-NEXT:    bltu a0, a1, .LBB67_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/imm.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/imm.ll
index f324a9bc120e..8c9a9b43952b 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/imm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/imm.ll
@@ -186,31 +186,31 @@ define signext i32 @pos_i32() nounwind {
 ; RV64I-LABEL: pos_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 423811
-; RV64I-NEXT:    addiw a0, a0, -1297
+; RV64I-NEXT:    addi a0, a0, -1297
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: pos_i32:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 423811
-; RV64IZBA-NEXT:    addiw a0, a0, -1297
+; RV64IZBA-NEXT:    addi a0, a0, -1297
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: pos_i32:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 423811
-; RV64IZBB-NEXT:    addiw a0, a0, -1297
+; RV64IZBB-NEXT:    addi a0, a0, -1297
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: pos_i32:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 423811
-; RV64IZBS-NEXT:    addiw a0, a0, -1297
+; RV64IZBS-NEXT:    addi a0, a0, -1297
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: pos_i32:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 423811
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1297
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1297
 ; RV64IXTHEADBB-NEXT:    ret
 ;
 ; RV32-REMAT-LABEL: pos_i32:
@@ -222,7 +222,7 @@ define signext i32 @pos_i32() nounwind {
 ; RV64-REMAT-LABEL: pos_i32:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 423811
-; RV64-REMAT-NEXT:    addiw a0, a0, -1297
+; RV64-REMAT-NEXT:    addi a0, a0, -1297
 ; RV64-REMAT-NEXT:    ret
   ret i32 1735928559
 }
@@ -242,31 +242,31 @@ define signext i32 @neg_i32() nounwind {
 ; RV64I-LABEL: neg_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 912092
-; RV64I-NEXT:    addiw a0, a0, -273
+; RV64I-NEXT:    addi a0, a0, -273
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: neg_i32:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 912092
-; RV64IZBA-NEXT:    addiw a0, a0, -273
+; RV64IZBA-NEXT:    addi a0, a0, -273
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: neg_i32:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 912092
-; RV64IZBB-NEXT:    addiw a0, a0, -273
+; RV64IZBB-NEXT:    addi a0, a0, -273
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: neg_i32:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 912092
-; RV64IZBS-NEXT:    addiw a0, a0, -273
+; RV64IZBS-NEXT:    addi a0, a0, -273
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: neg_i32:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 912092
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -273
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -273
 ; RV64IXTHEADBB-NEXT:    ret
 ;
 ; RV32-REMAT-LABEL: neg_i32:
@@ -278,7 +278,7 @@ define signext i32 @neg_i32() nounwind {
 ; RV64-REMAT-LABEL: neg_i32:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 912092
-; RV64-REMAT-NEXT:    addiw a0, a0, -273
+; RV64-REMAT-NEXT:    addi a0, a0, -273
 ; RV64-REMAT-NEXT:    ret
   ret i32 -559038737
 }
@@ -396,31 +396,31 @@ define signext i32 @imm_left_shifted_addi() nounwind {
 ; RV64I-LABEL: imm_left_shifted_addi:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 32
-; RV64I-NEXT:    addiw a0, a0, -64
+; RV64I-NEXT:    addi a0, a0, -64
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_left_shifted_addi:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 32
-; RV64IZBA-NEXT:    addiw a0, a0, -64
+; RV64IZBA-NEXT:    addi a0, a0, -64
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_left_shifted_addi:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 32
-; RV64IZBB-NEXT:    addiw a0, a0, -64
+; RV64IZBB-NEXT:    addi a0, a0, -64
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: imm_left_shifted_addi:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 32
-; RV64IZBS-NEXT:    addiw a0, a0, -64
+; RV64IZBS-NEXT:    addi a0, a0, -64
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_left_shifted_addi:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 32
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -64
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -64
 ; RV64IXTHEADBB-NEXT:    ret
 ;
 ; RV32-REMAT-LABEL: imm_left_shifted_addi:
@@ -432,7 +432,7 @@ define signext i32 @imm_left_shifted_addi() nounwind {
 ; RV64-REMAT-LABEL: imm_left_shifted_addi:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 32
-; RV64-REMAT-NEXT:    addiw a0, a0, -64
+; RV64-REMAT-NEXT:    addi a0, a0, -64
 ; RV64-REMAT-NEXT:    ret
   ret i32 131008 ; 0x1FFC0
 }
@@ -512,31 +512,31 @@ define signext i32 @imm_right_shifted_lui() nounwind {
 ; RV64I-LABEL: imm_right_shifted_lui:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 56
-; RV64I-NEXT:    addiw a0, a0, 580
+; RV64I-NEXT:    addi a0, a0, 580
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_right_shifted_lui:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 56
-; RV64IZBA-NEXT:    addiw a0, a0, 580
+; RV64IZBA-NEXT:    addi a0, a0, 580
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_right_shifted_lui:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 56
-; RV64IZBB-NEXT:    addiw a0, a0, 580
+; RV64IZBB-NEXT:    addi a0, a0, 580
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: imm_right_shifted_lui:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 56
-; RV64IZBS-NEXT:    addiw a0, a0, 580
+; RV64IZBS-NEXT:    addi a0, a0, 580
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_right_shifted_lui:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 56
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 580
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 580
 ; RV64IXTHEADBB-NEXT:    ret
 ;
 ; RV32-REMAT-LABEL: imm_right_shifted_lui:
@@ -548,7 +548,7 @@ define signext i32 @imm_right_shifted_lui() nounwind {
 ; RV64-REMAT-LABEL: imm_right_shifted_lui:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 56
-; RV64-REMAT-NEXT:    addiw a0, a0, 580
+; RV64-REMAT-NEXT:    addi a0, a0, 580
 ; RV64-REMAT-NEXT:    ret
   ret i32 229956 ; 0x38244
 }
@@ -1664,7 +1664,7 @@ define i64 @imm_2reg_1() nounwind {
 ; RV64I-LABEL: imm_2reg_1:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 74565
-; RV64I-NEXT:    addiw a0, a0, 1656
+; RV64I-NEXT:    addi a0, a0, 1656
 ; RV64I-NEXT:    slli a1, a0, 57
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
@@ -1672,7 +1672,7 @@ define i64 @imm_2reg_1() nounwind {
 ; RV64IZBA-LABEL: imm_2reg_1:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 74565
-; RV64IZBA-NEXT:    addiw a0, a0, 1656
+; RV64IZBA-NEXT:    addi a0, a0, 1656
 ; RV64IZBA-NEXT:    slli a1, a0, 57
 ; RV64IZBA-NEXT:    add a0, a0, a1
 ; RV64IZBA-NEXT:    ret
@@ -1680,7 +1680,7 @@ define i64 @imm_2reg_1() nounwind {
 ; RV64IZBB-LABEL: imm_2reg_1:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 74565
-; RV64IZBB-NEXT:    addiw a0, a0, 1656
+; RV64IZBB-NEXT:    addi a0, a0, 1656
 ; RV64IZBB-NEXT:    slli a1, a0, 57
 ; RV64IZBB-NEXT:    add a0, a0, a1
 ; RV64IZBB-NEXT:    ret
@@ -1688,7 +1688,7 @@ define i64 @imm_2reg_1() nounwind {
 ; RV64IZBS-LABEL: imm_2reg_1:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 74565
-; RV64IZBS-NEXT:    addiw a0, a0, 1656
+; RV64IZBS-NEXT:    addi a0, a0, 1656
 ; RV64IZBS-NEXT:    slli a1, a0, 57
 ; RV64IZBS-NEXT:    add a0, a0, a1
 ; RV64IZBS-NEXT:    ret
@@ -1696,7 +1696,7 @@ define i64 @imm_2reg_1() nounwind {
 ; RV64IXTHEADBB-LABEL: imm_2reg_1:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 74565
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1656
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1656
 ; RV64IXTHEADBB-NEXT:    slli a1, a0, 57
 ; RV64IXTHEADBB-NEXT:    add a0, a0, a1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -1711,7 +1711,7 @@ define i64 @imm_2reg_1() nounwind {
 ; RV64-REMAT-LABEL: imm_2reg_1:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 74565
-; RV64-REMAT-NEXT:    addiw a0, a0, 1656
+; RV64-REMAT-NEXT:    addi a0, a0, 1656
 ; RV64-REMAT-NEXT:    slli a1, a0, 57
 ; RV64-REMAT-NEXT:    add a0, a0, a1
 ; RV64-REMAT-NEXT:    ret
@@ -1909,7 +1909,7 @@ define i64 @imm_5372288229() {
 ; RV64I-LABEL: imm_5372288229:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 160
-; RV64I-NEXT:    addiw a0, a0, 437
+; RV64I-NEXT:    addi a0, a0, 437
 ; RV64I-NEXT:    slli a0, a0, 13
 ; RV64I-NEXT:    addi a0, a0, -795
 ; RV64I-NEXT:    ret
@@ -1924,7 +1924,7 @@ define i64 @imm_5372288229() {
 ; RV64IZBB-LABEL: imm_5372288229:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 160
-; RV64IZBB-NEXT:    addiw a0, a0, 437
+; RV64IZBB-NEXT:    addi a0, a0, 437
 ; RV64IZBB-NEXT:    slli a0, a0, 13
 ; RV64IZBB-NEXT:    addi a0, a0, -795
 ; RV64IZBB-NEXT:    ret
@@ -1932,14 +1932,14 @@ define i64 @imm_5372288229() {
 ; RV64IZBS-LABEL: imm_5372288229:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 263018
-; RV64IZBS-NEXT:    addiw a0, a0, -795
+; RV64IZBS-NEXT:    addi a0, a0, -795
 ; RV64IZBS-NEXT:    bseti a0, a0, 32
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_5372288229:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 160
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 437
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 437
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 13
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -795
 ; RV64IXTHEADBB-NEXT:    ret
@@ -1954,7 +1954,7 @@ define i64 @imm_5372288229() {
 ; RV64-REMAT-LABEL: imm_5372288229:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 160
-; RV64-REMAT-NEXT:    addiw a0, a0, 437
+; RV64-REMAT-NEXT:    addi a0, a0, 437
 ; RV64-REMAT-NEXT:    slli a0, a0, 13
 ; RV64-REMAT-NEXT:    addi a0, a0, -795
 ; RV64-REMAT-NEXT:    ret
@@ -1978,7 +1978,7 @@ define i64 @imm_neg_5372288229() {
 ; RV64I-LABEL: imm_neg_5372288229:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1048416
-; RV64I-NEXT:    addiw a0, a0, -437
+; RV64I-NEXT:    addi a0, a0, -437
 ; RV64I-NEXT:    slli a0, a0, 13
 ; RV64I-NEXT:    addi a0, a0, 795
 ; RV64I-NEXT:    ret
@@ -1986,14 +1986,14 @@ define i64 @imm_neg_5372288229() {
 ; RV64IZBA-LABEL: imm_neg_5372288229:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 611378
-; RV64IZBA-NEXT:    addiw a0, a0, 265
+; RV64IZBA-NEXT:    addi a0, a0, 265
 ; RV64IZBA-NEXT:    sh1add a0, a0, a0
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_neg_5372288229:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1048416
-; RV64IZBB-NEXT:    addiw a0, a0, -437
+; RV64IZBB-NEXT:    addi a0, a0, -437
 ; RV64IZBB-NEXT:    slli a0, a0, 13
 ; RV64IZBB-NEXT:    addi a0, a0, 795
 ; RV64IZBB-NEXT:    ret
@@ -2001,14 +2001,14 @@ define i64 @imm_neg_5372288229() {
 ; RV64IZBS-LABEL: imm_neg_5372288229:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 785558
-; RV64IZBS-NEXT:    addiw a0, a0, 795
+; RV64IZBS-NEXT:    addi a0, a0, 795
 ; RV64IZBS-NEXT:    bclri a0, a0, 32
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_5372288229:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1048416
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -437
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -437
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 13
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 795
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2023,7 +2023,7 @@ define i64 @imm_neg_5372288229() {
 ; RV64-REMAT-LABEL: imm_neg_5372288229:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1048416
-; RV64-REMAT-NEXT:    addiw a0, a0, -437
+; RV64-REMAT-NEXT:    addi a0, a0, -437
 ; RV64-REMAT-NEXT:    slli a0, a0, 13
 ; RV64-REMAT-NEXT:    addi a0, a0, 795
 ; RV64-REMAT-NEXT:    ret
@@ -2047,7 +2047,7 @@ define i64 @imm_8953813715() {
 ; RV64I-LABEL: imm_8953813715:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 267
-; RV64I-NEXT:    addiw a0, a0, -637
+; RV64I-NEXT:    addi a0, a0, -637
 ; RV64I-NEXT:    slli a0, a0, 13
 ; RV64I-NEXT:    addi a0, a0, -1325
 ; RV64I-NEXT:    ret
@@ -2055,14 +2055,14 @@ define i64 @imm_8953813715() {
 ; RV64IZBA-LABEL: imm_8953813715:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 437198
-; RV64IZBA-NEXT:    addiw a0, a0, -265
+; RV64IZBA-NEXT:    addi a0, a0, -265
 ; RV64IZBA-NEXT:    sh2add a0, a0, a0
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_8953813715:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 267
-; RV64IZBB-NEXT:    addiw a0, a0, -637
+; RV64IZBB-NEXT:    addi a0, a0, -637
 ; RV64IZBB-NEXT:    slli a0, a0, 13
 ; RV64IZBB-NEXT:    addi a0, a0, -1325
 ; RV64IZBB-NEXT:    ret
@@ -2070,14 +2070,14 @@ define i64 @imm_8953813715() {
 ; RV64IZBS-LABEL: imm_8953813715:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 88838
-; RV64IZBS-NEXT:    addiw a0, a0, -1325
+; RV64IZBS-NEXT:    addi a0, a0, -1325
 ; RV64IZBS-NEXT:    bseti a0, a0, 33
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_8953813715:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 267
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -637
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -637
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 13
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -1325
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2092,7 +2092,7 @@ define i64 @imm_8953813715() {
 ; RV64-REMAT-LABEL: imm_8953813715:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 267
-; RV64-REMAT-NEXT:    addiw a0, a0, -637
+; RV64-REMAT-NEXT:    addi a0, a0, -637
 ; RV64-REMAT-NEXT:    slli a0, a0, 13
 ; RV64-REMAT-NEXT:    addi a0, a0, -1325
 ; RV64-REMAT-NEXT:    ret
@@ -2116,7 +2116,7 @@ define i64 @imm_neg_8953813715() {
 ; RV64I-LABEL: imm_neg_8953813715:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1048309
-; RV64I-NEXT:    addiw a0, a0, 637
+; RV64I-NEXT:    addi a0, a0, 637
 ; RV64I-NEXT:    slli a0, a0, 13
 ; RV64I-NEXT:    addi a0, a0, 1325
 ; RV64I-NEXT:    ret
@@ -2124,14 +2124,14 @@ define i64 @imm_neg_8953813715() {
 ; RV64IZBA-LABEL: imm_neg_8953813715:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 611378
-; RV64IZBA-NEXT:    addiw a0, a0, 265
+; RV64IZBA-NEXT:    addi a0, a0, 265
 ; RV64IZBA-NEXT:    sh2add a0, a0, a0
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_neg_8953813715:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1048309
-; RV64IZBB-NEXT:    addiw a0, a0, 637
+; RV64IZBB-NEXT:    addi a0, a0, 637
 ; RV64IZBB-NEXT:    slli a0, a0, 13
 ; RV64IZBB-NEXT:    addi a0, a0, 1325
 ; RV64IZBB-NEXT:    ret
@@ -2139,14 +2139,14 @@ define i64 @imm_neg_8953813715() {
 ; RV64IZBS-LABEL: imm_neg_8953813715:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 959738
-; RV64IZBS-NEXT:    addiw a0, a0, 1325
+; RV64IZBS-NEXT:    addi a0, a0, 1325
 ; RV64IZBS-NEXT:    bclri a0, a0, 33
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_8953813715:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1048309
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 637
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 637
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 13
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1325
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2161,7 +2161,7 @@ define i64 @imm_neg_8953813715() {
 ; RV64-REMAT-LABEL: imm_neg_8953813715:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1048309
-; RV64-REMAT-NEXT:    addiw a0, a0, 637
+; RV64-REMAT-NEXT:    addi a0, a0, 637
 ; RV64-REMAT-NEXT:    slli a0, a0, 13
 ; RV64-REMAT-NEXT:    addi a0, a0, 1325
 ; RV64-REMAT-NEXT:    ret
@@ -2185,7 +2185,7 @@ define i64 @imm_16116864687() {
 ; RV64I-LABEL: imm_16116864687:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 961
-; RV64I-NEXT:    addiw a0, a0, -1475
+; RV64I-NEXT:    addi a0, a0, -1475
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, 1711
 ; RV64I-NEXT:    ret
@@ -2193,14 +2193,14 @@ define i64 @imm_16116864687() {
 ; RV64IZBA-LABEL: imm_16116864687:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 437198
-; RV64IZBA-NEXT:    addiw a0, a0, -265
+; RV64IZBA-NEXT:    addi a0, a0, -265
 ; RV64IZBA-NEXT:    sh3add a0, a0, a0
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_16116864687:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 961
-; RV64IZBB-NEXT:    addiw a0, a0, -1475
+; RV64IZBB-NEXT:    addi a0, a0, -1475
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, 1711
 ; RV64IZBB-NEXT:    ret
@@ -2208,7 +2208,7 @@ define i64 @imm_16116864687() {
 ; RV64IZBS-LABEL: imm_16116864687:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 961
-; RV64IZBS-NEXT:    addiw a0, a0, -1475
+; RV64IZBS-NEXT:    addi a0, a0, -1475
 ; RV64IZBS-NEXT:    slli a0, a0, 12
 ; RV64IZBS-NEXT:    addi a0, a0, 1711
 ; RV64IZBS-NEXT:    ret
@@ -2216,7 +2216,7 @@ define i64 @imm_16116864687() {
 ; RV64IXTHEADBB-LABEL: imm_16116864687:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 961
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1475
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1475
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1711
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2231,7 +2231,7 @@ define i64 @imm_16116864687() {
 ; RV64-REMAT-LABEL: imm_16116864687:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 961
-; RV64-REMAT-NEXT:    addiw a0, a0, -1475
+; RV64-REMAT-NEXT:    addi a0, a0, -1475
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, 1711
 ; RV64-REMAT-NEXT:    ret
@@ -2255,7 +2255,7 @@ define i64 @imm_neg_16116864687() {
 ; RV64I-LABEL: imm_neg_16116864687:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1047615
-; RV64I-NEXT:    addiw a0, a0, 1475
+; RV64I-NEXT:    addi a0, a0, 1475
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, -1711
 ; RV64I-NEXT:    ret
@@ -2263,14 +2263,14 @@ define i64 @imm_neg_16116864687() {
 ; RV64IZBA-LABEL: imm_neg_16116864687:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 611378
-; RV64IZBA-NEXT:    addiw a0, a0, 265
+; RV64IZBA-NEXT:    addi a0, a0, 265
 ; RV64IZBA-NEXT:    sh3add a0, a0, a0
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_neg_16116864687:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1047615
-; RV64IZBB-NEXT:    addiw a0, a0, 1475
+; RV64IZBB-NEXT:    addi a0, a0, 1475
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, -1711
 ; RV64IZBB-NEXT:    ret
@@ -2278,7 +2278,7 @@ define i64 @imm_neg_16116864687() {
 ; RV64IZBS-LABEL: imm_neg_16116864687:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 1047615
-; RV64IZBS-NEXT:    addiw a0, a0, 1475
+; RV64IZBS-NEXT:    addi a0, a0, 1475
 ; RV64IZBS-NEXT:    slli a0, a0, 12
 ; RV64IZBS-NEXT:    addi a0, a0, -1711
 ; RV64IZBS-NEXT:    ret
@@ -2286,7 +2286,7 @@ define i64 @imm_neg_16116864687() {
 ; RV64IXTHEADBB-LABEL: imm_neg_16116864687:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1047615
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1475
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1475
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -1711
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2301,7 +2301,7 @@ define i64 @imm_neg_16116864687() {
 ; RV64-REMAT-LABEL: imm_neg_16116864687:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1047615
-; RV64-REMAT-NEXT:    addiw a0, a0, 1475
+; RV64-REMAT-NEXT:    addi a0, a0, 1475
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, -1711
 ; RV64-REMAT-NEXT:    ret
@@ -2390,7 +2390,7 @@ define i64 @imm_70370820078523() {
 ; RV64-NOPOOL-LABEL: imm_70370820078523:
 ; RV64-NOPOOL:       # %bb.0:
 ; RV64-NOPOOL-NEXT:    lui a0, 256
-; RV64-NOPOOL-NEXT:    addiw a0, a0, 31
+; RV64-NOPOOL-NEXT:    addi a0, a0, 31
 ; RV64-NOPOOL-NEXT:    slli a0, a0, 12
 ; RV64-NOPOOL-NEXT:    addi a0, a0, -273
 ; RV64-NOPOOL-NEXT:    slli a0, a0, 14
@@ -2406,7 +2406,7 @@ define i64 @imm_70370820078523() {
 ; RV64IZBA-LABEL: imm_70370820078523:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 256
-; RV64IZBA-NEXT:    addiw a0, a0, 31
+; RV64IZBA-NEXT:    addi a0, a0, 31
 ; RV64IZBA-NEXT:    slli a0, a0, 12
 ; RV64IZBA-NEXT:    addi a0, a0, -273
 ; RV64IZBA-NEXT:    slli a0, a0, 14
@@ -2416,7 +2416,7 @@ define i64 @imm_70370820078523() {
 ; RV64IZBB-LABEL: imm_70370820078523:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 256
-; RV64IZBB-NEXT:    addiw a0, a0, 31
+; RV64IZBB-NEXT:    addi a0, a0, 31
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, -273
 ; RV64IZBB-NEXT:    slli a0, a0, 14
@@ -2426,14 +2426,14 @@ define i64 @imm_70370820078523() {
 ; RV64IZBS-LABEL: imm_70370820078523:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 506812
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    bseti a0, a0, 46
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_70370820078523:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 256
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 31
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 31
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -273
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 14
@@ -2450,7 +2450,7 @@ define i64 @imm_70370820078523() {
 ; RV64-REMAT-LABEL: imm_70370820078523:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 256
-; RV64-REMAT-NEXT:    addiw a0, a0, 31
+; RV64-REMAT-NEXT:    addi a0, a0, 31
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, -273
 ; RV64-REMAT-NEXT:    slli a0, a0, 14
@@ -2476,7 +2476,7 @@ define i64 @imm_neg_9223372034778874949() {
 ; RV64I-LABEL: imm_neg_9223372034778874949:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 506812
-; RV64I-NEXT:    addiw a0, a0, -1093
+; RV64I-NEXT:    addi a0, a0, -1093
 ; RV64I-NEXT:    slli a1, a0, 63
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
@@ -2484,7 +2484,7 @@ define i64 @imm_neg_9223372034778874949() {
 ; RV64IZBA-LABEL: imm_neg_9223372034778874949:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 506812
-; RV64IZBA-NEXT:    addiw a0, a0, -1093
+; RV64IZBA-NEXT:    addi a0, a0, -1093
 ; RV64IZBA-NEXT:    slli a1, a0, 63
 ; RV64IZBA-NEXT:    add a0, a0, a1
 ; RV64IZBA-NEXT:    ret
@@ -2492,7 +2492,7 @@ define i64 @imm_neg_9223372034778874949() {
 ; RV64IZBB-LABEL: imm_neg_9223372034778874949:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 506812
-; RV64IZBB-NEXT:    addiw a0, a0, -1093
+; RV64IZBB-NEXT:    addi a0, a0, -1093
 ; RV64IZBB-NEXT:    slli a1, a0, 63
 ; RV64IZBB-NEXT:    add a0, a0, a1
 ; RV64IZBB-NEXT:    ret
@@ -2500,14 +2500,14 @@ define i64 @imm_neg_9223372034778874949() {
 ; RV64IZBS-LABEL: imm_neg_9223372034778874949:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 506812
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    bseti a0, a0, 63
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_9223372034778874949:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 506812
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1093
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1093
 ; RV64IXTHEADBB-NEXT:    slli a1, a0, 63
 ; RV64IXTHEADBB-NEXT:    add a0, a0, a1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2522,7 +2522,7 @@ define i64 @imm_neg_9223372034778874949() {
 ; RV64-REMAT-LABEL: imm_neg_9223372034778874949:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 506812
-; RV64-REMAT-NEXT:    addiw a0, a0, -1093
+; RV64-REMAT-NEXT:    addi a0, a0, -1093
 ; RV64-REMAT-NEXT:    slli a1, a0, 63
 ; RV64-REMAT-NEXT:    add a0, a0, a1
 ; RV64-REMAT-NEXT:    ret
@@ -2585,7 +2585,7 @@ define i64 @imm_neg_9223301666034697285() {
 ; RV64IZBS-LABEL: imm_neg_9223301666034697285:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 506812
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    bseti a0, a0, 46
 ; RV64IZBS-NEXT:    bseti a0, a0, 63
 ; RV64IZBS-NEXT:    ret
@@ -2704,7 +2704,7 @@ define i64 @imm_neg_8798043653189() {
 ; RV64I-LABEL: imm_neg_8798043653189:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 917475
-; RV64I-NEXT:    addiw a0, a0, -273
+; RV64I-NEXT:    addi a0, a0, -273
 ; RV64I-NEXT:    slli a0, a0, 14
 ; RV64I-NEXT:    addi a0, a0, -1093
 ; RV64I-NEXT:    ret
@@ -2712,7 +2712,7 @@ define i64 @imm_neg_8798043653189() {
 ; RV64IZBA-LABEL: imm_neg_8798043653189:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 917475
-; RV64IZBA-NEXT:    addiw a0, a0, -273
+; RV64IZBA-NEXT:    addi a0, a0, -273
 ; RV64IZBA-NEXT:    slli a0, a0, 14
 ; RV64IZBA-NEXT:    addi a0, a0, -1093
 ; RV64IZBA-NEXT:    ret
@@ -2720,7 +2720,7 @@ define i64 @imm_neg_8798043653189() {
 ; RV64IZBB-LABEL: imm_neg_8798043653189:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 917475
-; RV64IZBB-NEXT:    addiw a0, a0, -273
+; RV64IZBB-NEXT:    addi a0, a0, -273
 ; RV64IZBB-NEXT:    slli a0, a0, 14
 ; RV64IZBB-NEXT:    addi a0, a0, -1093
 ; RV64IZBB-NEXT:    ret
@@ -2728,14 +2728,14 @@ define i64 @imm_neg_8798043653189() {
 ; RV64IZBS-LABEL: imm_neg_8798043653189:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 572348
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    bclri a0, a0, 43
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_8798043653189:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 917475
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -273
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -273
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 14
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -1093
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2751,7 +2751,7 @@ define i64 @imm_neg_8798043653189() {
 ; RV64-REMAT-LABEL: imm_neg_8798043653189:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 917475
-; RV64-REMAT-NEXT:    addiw a0, a0, -273
+; RV64-REMAT-NEXT:    addi a0, a0, -273
 ; RV64-REMAT-NEXT:    slli a0, a0, 14
 ; RV64-REMAT-NEXT:    addi a0, a0, -1093
 ; RV64-REMAT-NEXT:    ret
@@ -2776,7 +2776,7 @@ define i64 @imm_9223372034904144827() {
 ; RV64I-LABEL: imm_9223372034904144827:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 572348
-; RV64I-NEXT:    addiw a0, a0, -1093
+; RV64I-NEXT:    addi a0, a0, -1093
 ; RV64I-NEXT:    slli a1, a0, 63
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
@@ -2784,7 +2784,7 @@ define i64 @imm_9223372034904144827() {
 ; RV64IZBA-LABEL: imm_9223372034904144827:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 572348
-; RV64IZBA-NEXT:    addiw a0, a0, -1093
+; RV64IZBA-NEXT:    addi a0, a0, -1093
 ; RV64IZBA-NEXT:    slli a1, a0, 63
 ; RV64IZBA-NEXT:    add a0, a0, a1
 ; RV64IZBA-NEXT:    ret
@@ -2792,7 +2792,7 @@ define i64 @imm_9223372034904144827() {
 ; RV64IZBB-LABEL: imm_9223372034904144827:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 572348
-; RV64IZBB-NEXT:    addiw a0, a0, -1093
+; RV64IZBB-NEXT:    addi a0, a0, -1093
 ; RV64IZBB-NEXT:    slli a1, a0, 63
 ; RV64IZBB-NEXT:    add a0, a0, a1
 ; RV64IZBB-NEXT:    ret
@@ -2800,14 +2800,14 @@ define i64 @imm_9223372034904144827() {
 ; RV64IZBS-LABEL: imm_9223372034904144827:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 572348
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    bclri a0, a0, 63
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_9223372034904144827:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 572348
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1093
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1093
 ; RV64IXTHEADBB-NEXT:    slli a1, a0, 63
 ; RV64IXTHEADBB-NEXT:    add a0, a0, a1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -2823,7 +2823,7 @@ define i64 @imm_9223372034904144827() {
 ; RV64-REMAT-LABEL: imm_9223372034904144827:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 572348
-; RV64-REMAT-NEXT:    addiw a0, a0, -1093
+; RV64-REMAT-NEXT:    addi a0, a0, -1093
 ; RV64-REMAT-NEXT:    slli a1, a0, 63
 ; RV64-REMAT-NEXT:    add a0, a0, a1
 ; RV64-REMAT-NEXT:    ret
@@ -2887,7 +2887,7 @@ define i64 @imm_neg_9223354442718100411() {
 ; RV64IZBS-LABEL: imm_neg_9223354442718100411:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 572348
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    bclri a0, a0, 44
 ; RV64IZBS-NEXT:    bclri a0, a0, 63
 ; RV64IZBS-NEXT:    ret
@@ -2941,35 +2941,35 @@ define i64 @imm_2863311530() {
 ; RV64I-LABEL: imm_2863311530:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 349525
-; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addi a0, a0, 1365
 ; RV64I-NEXT:    slli a0, a0, 1
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_2863311530:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 349525
-; RV64IZBA-NEXT:    addiw a0, a0, 1365
+; RV64IZBA-NEXT:    addi a0, a0, 1365
 ; RV64IZBA-NEXT:    slli a0, a0, 1
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_2863311530:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 349525
-; RV64IZBB-NEXT:    addiw a0, a0, 1365
+; RV64IZBB-NEXT:    addi a0, a0, 1365
 ; RV64IZBB-NEXT:    slli a0, a0, 1
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: imm_2863311530:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 349525
-; RV64IZBS-NEXT:    addiw a0, a0, 1365
+; RV64IZBS-NEXT:    addi a0, a0, 1365
 ; RV64IZBS-NEXT:    slli a0, a0, 1
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_2863311530:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 349525
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1365
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1365
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    ret
 ;
@@ -2983,7 +2983,7 @@ define i64 @imm_2863311530() {
 ; RV64-REMAT-LABEL: imm_2863311530:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 349525
-; RV64-REMAT-NEXT:    addiw a0, a0, 1365
+; RV64-REMAT-NEXT:    addi a0, a0, 1365
 ; RV64-REMAT-NEXT:    slli a0, a0, 1
 ; RV64-REMAT-NEXT:    ret
 	ret i64 2863311530 ; #0xaaaaaaaa
@@ -3006,35 +3006,35 @@ define i64 @imm_neg_2863311530() {
 ; RV64I-LABEL: imm_neg_2863311530:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 699051
-; RV64I-NEXT:    addiw a0, a0, -1365
+; RV64I-NEXT:    addi a0, a0, -1365
 ; RV64I-NEXT:    slli a0, a0, 1
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_neg_2863311530:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 699051
-; RV64IZBA-NEXT:    addiw a0, a0, -1365
+; RV64IZBA-NEXT:    addi a0, a0, -1365
 ; RV64IZBA-NEXT:    slli a0, a0, 1
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_neg_2863311530:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 699051
-; RV64IZBB-NEXT:    addiw a0, a0, -1365
+; RV64IZBB-NEXT:    addi a0, a0, -1365
 ; RV64IZBB-NEXT:    slli a0, a0, 1
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: imm_neg_2863311530:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 699051
-; RV64IZBS-NEXT:    addiw a0, a0, -1365
+; RV64IZBS-NEXT:    addi a0, a0, -1365
 ; RV64IZBS-NEXT:    slli a0, a0, 1
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_2863311530:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 699051
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1365
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1365
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    ret
 ;
@@ -3048,7 +3048,7 @@ define i64 @imm_neg_2863311530() {
 ; RV64-REMAT-LABEL: imm_neg_2863311530:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 699051
-; RV64-REMAT-NEXT:    addiw a0, a0, -1365
+; RV64-REMAT-NEXT:    addi a0, a0, -1365
 ; RV64-REMAT-NEXT:    slli a0, a0, 1
 ; RV64-REMAT-NEXT:    ret
 	ret i64 -2863311530 ; #0xffffffff55555556
@@ -3195,7 +3195,7 @@ define i64 @imm_12900924131259() {
 ; RV64I-LABEL: imm_12900924131259:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 188
-; RV64I-NEXT:    addiw a0, a0, -1093
+; RV64I-NEXT:    addi a0, a0, -1093
 ; RV64I-NEXT:    slli a0, a0, 24
 ; RV64I-NEXT:    addi a0, a0, 1979
 ; RV64I-NEXT:    ret
@@ -3210,7 +3210,7 @@ define i64 @imm_12900924131259() {
 ; RV64IZBB-LABEL: imm_12900924131259:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 188
-; RV64IZBB-NEXT:    addiw a0, a0, -1093
+; RV64IZBB-NEXT:    addi a0, a0, -1093
 ; RV64IZBB-NEXT:    slli a0, a0, 24
 ; RV64IZBB-NEXT:    addi a0, a0, 1979
 ; RV64IZBB-NEXT:    ret
@@ -3218,7 +3218,7 @@ define i64 @imm_12900924131259() {
 ; RV64IZBS-LABEL: imm_12900924131259:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 188
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    slli a0, a0, 24
 ; RV64IZBS-NEXT:    addi a0, a0, 1979
 ; RV64IZBS-NEXT:    ret
@@ -3226,7 +3226,7 @@ define i64 @imm_12900924131259() {
 ; RV64IXTHEADBB-LABEL: imm_12900924131259:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 188
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1093
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1093
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 24
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1979
 ; RV64IXTHEADBB-NEXT:    ret
@@ -3242,7 +3242,7 @@ define i64 @imm_12900924131259() {
 ; RV64-REMAT-LABEL: imm_12900924131259:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 188
-; RV64-REMAT-NEXT:    addiw a0, a0, -1093
+; RV64-REMAT-NEXT:    addi a0, a0, -1093
 ; RV64-REMAT-NEXT:    slli a0, a0, 24
 ; RV64-REMAT-NEXT:    addi a0, a0, 1979
 ; RV64-REMAT-NEXT:    ret
@@ -3265,7 +3265,7 @@ define i64 @imm_50394234880() {
 ; RV64I-LABEL: imm_50394234880:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 188
-; RV64I-NEXT:    addiw a0, a0, -1093
+; RV64I-NEXT:    addi a0, a0, -1093
 ; RV64I-NEXT:    slli a0, a0, 16
 ; RV64I-NEXT:    ret
 ;
@@ -3278,21 +3278,21 @@ define i64 @imm_50394234880() {
 ; RV64IZBB-LABEL: imm_50394234880:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 188
-; RV64IZBB-NEXT:    addiw a0, a0, -1093
+; RV64IZBB-NEXT:    addi a0, a0, -1093
 ; RV64IZBB-NEXT:    slli a0, a0, 16
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: imm_50394234880:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 188
-; RV64IZBS-NEXT:    addiw a0, a0, -1093
+; RV64IZBS-NEXT:    addi a0, a0, -1093
 ; RV64IZBS-NEXT:    slli a0, a0, 16
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_50394234880:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 188
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1093
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1093
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 16
 ; RV64IXTHEADBB-NEXT:    ret
 ;
@@ -3305,7 +3305,7 @@ define i64 @imm_50394234880() {
 ; RV64-REMAT-LABEL: imm_50394234880:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 188
-; RV64-REMAT-NEXT:    addiw a0, a0, -1093
+; RV64-REMAT-NEXT:    addi a0, a0, -1093
 ; RV64-REMAT-NEXT:    slli a0, a0, 16
 ; RV64-REMAT-NEXT:    ret
   ret i64 50394234880
@@ -3407,7 +3407,7 @@ define i64 @imm_12900918536874() {
 ; RV64I-LABEL: imm_12900918536874:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 384477
-; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addi a0, a0, 1365
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, 1365
 ; RV64I-NEXT:    slli a0, a0, 1
@@ -3424,7 +3424,7 @@ define i64 @imm_12900918536874() {
 ; RV64IZBB-LABEL: imm_12900918536874:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 384477
-; RV64IZBB-NEXT:    addiw a0, a0, 1365
+; RV64IZBB-NEXT:    addi a0, a0, 1365
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, 1365
 ; RV64IZBB-NEXT:    slli a0, a0, 1
@@ -3433,7 +3433,7 @@ define i64 @imm_12900918536874() {
 ; RV64IZBS-LABEL: imm_12900918536874:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 384477
-; RV64IZBS-NEXT:    addiw a0, a0, 1365
+; RV64IZBS-NEXT:    addi a0, a0, 1365
 ; RV64IZBS-NEXT:    slli a0, a0, 12
 ; RV64IZBS-NEXT:    addi a0, a0, 1365
 ; RV64IZBS-NEXT:    slli a0, a0, 1
@@ -3442,7 +3442,7 @@ define i64 @imm_12900918536874() {
 ; RV64IXTHEADBB-LABEL: imm_12900918536874:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 384477
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1365
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1365
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1365
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 1
@@ -3459,7 +3459,7 @@ define i64 @imm_12900918536874() {
 ; RV64-REMAT-LABEL: imm_12900918536874:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 384477
-; RV64-REMAT-NEXT:    addiw a0, a0, 1365
+; RV64-REMAT-NEXT:    addi a0, a0, 1365
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, 1365
 ; RV64-REMAT-NEXT:    slli a0, a0, 1
@@ -3485,7 +3485,7 @@ define i64 @imm_12900925247761() {
 ; RV64I-LABEL: imm_12900925247761:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 384478
-; RV64I-NEXT:    addiw a0, a0, -1911
+; RV64I-NEXT:    addi a0, a0, -1911
 ; RV64I-NEXT:    slli a0, a0, 13
 ; RV64I-NEXT:    addi a0, a0, -2048
 ; RV64I-NEXT:    addi a0, a0, -1775
@@ -3502,7 +3502,7 @@ define i64 @imm_12900925247761() {
 ; RV64IZBB-LABEL: imm_12900925247761:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 384478
-; RV64IZBB-NEXT:    addiw a0, a0, -1911
+; RV64IZBB-NEXT:    addi a0, a0, -1911
 ; RV64IZBB-NEXT:    slli a0, a0, 13
 ; RV64IZBB-NEXT:    addi a0, a0, -2048
 ; RV64IZBB-NEXT:    addi a0, a0, -1775
@@ -3511,7 +3511,7 @@ define i64 @imm_12900925247761() {
 ; RV64IZBS-LABEL: imm_12900925247761:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 384478
-; RV64IZBS-NEXT:    addiw a0, a0, -1911
+; RV64IZBS-NEXT:    addi a0, a0, -1911
 ; RV64IZBS-NEXT:    slli a0, a0, 13
 ; RV64IZBS-NEXT:    addi a0, a0, -2048
 ; RV64IZBS-NEXT:    addi a0, a0, -1775
@@ -3520,7 +3520,7 @@ define i64 @imm_12900925247761() {
 ; RV64IXTHEADBB-LABEL: imm_12900925247761:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 384478
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1911
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1911
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 13
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -2048
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -1775
@@ -3537,7 +3537,7 @@ define i64 @imm_12900925247761() {
 ; RV64-REMAT-LABEL: imm_12900925247761:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 384478
-; RV64-REMAT-NEXT:    addiw a0, a0, -1911
+; RV64-REMAT-NEXT:    addi a0, a0, -1911
 ; RV64-REMAT-NEXT:    slli a0, a0, 13
 ; RV64-REMAT-NEXT:    addi a0, a0, -2048
 ; RV64-REMAT-NEXT:    addi a0, a0, -1775
@@ -3562,7 +3562,7 @@ define i64 @imm_7158272001() {
 ; RV64I-LABEL: imm_7158272001:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 427
-; RV64I-NEXT:    addiw a0, a0, -1367
+; RV64I-NEXT:    addi a0, a0, -1367
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, 1
 ; RV64I-NEXT:    ret
@@ -3577,7 +3577,7 @@ define i64 @imm_7158272001() {
 ; RV64IZBB-LABEL: imm_7158272001:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 427
-; RV64IZBB-NEXT:    addiw a0, a0, -1367
+; RV64IZBB-NEXT:    addi a0, a0, -1367
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, 1
 ; RV64IZBB-NEXT:    ret
@@ -3585,7 +3585,7 @@ define i64 @imm_7158272001() {
 ; RV64IZBS-LABEL: imm_7158272001:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 427
-; RV64IZBS-NEXT:    addiw a0, a0, -1367
+; RV64IZBS-NEXT:    addi a0, a0, -1367
 ; RV64IZBS-NEXT:    slli a0, a0, 12
 ; RV64IZBS-NEXT:    addi a0, a0, 1
 ; RV64IZBS-NEXT:    ret
@@ -3593,7 +3593,7 @@ define i64 @imm_7158272001() {
 ; RV64IXTHEADBB-LABEL: imm_7158272001:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 427
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1367
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1367
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -3608,7 +3608,7 @@ define i64 @imm_7158272001() {
 ; RV64-REMAT-LABEL: imm_7158272001:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 427
-; RV64-REMAT-NEXT:    addiw a0, a0, -1367
+; RV64-REMAT-NEXT:    addi a0, a0, -1367
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, 1
 ; RV64-REMAT-NEXT:    ret
@@ -3632,7 +3632,7 @@ define i64 @imm_12884889601() {
 ; RV64I-LABEL: imm_12884889601:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 768
-; RV64I-NEXT:    addiw a0, a0, -3
+; RV64I-NEXT:    addi a0, a0, -3
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, 1
 ; RV64I-NEXT:    ret
@@ -3647,7 +3647,7 @@ define i64 @imm_12884889601() {
 ; RV64IZBB-LABEL: imm_12884889601:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 768
-; RV64IZBB-NEXT:    addiw a0, a0, -3
+; RV64IZBB-NEXT:    addi a0, a0, -3
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, 1
 ; RV64IZBB-NEXT:    ret
@@ -3655,7 +3655,7 @@ define i64 @imm_12884889601() {
 ; RV64IZBS-LABEL: imm_12884889601:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 768
-; RV64IZBS-NEXT:    addiw a0, a0, -3
+; RV64IZBS-NEXT:    addi a0, a0, -3
 ; RV64IZBS-NEXT:    slli a0, a0, 12
 ; RV64IZBS-NEXT:    addi a0, a0, 1
 ; RV64IZBS-NEXT:    ret
@@ -3663,7 +3663,7 @@ define i64 @imm_12884889601() {
 ; RV64IXTHEADBB-LABEL: imm_12884889601:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 768
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -3
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -3
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -3678,7 +3678,7 @@ define i64 @imm_12884889601() {
 ; RV64-REMAT-LABEL: imm_12884889601:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 768
-; RV64-REMAT-NEXT:    addiw a0, a0, -3
+; RV64-REMAT-NEXT:    addi a0, a0, -3
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, 1
 ; RV64-REMAT-NEXT:    ret
@@ -3702,7 +3702,7 @@ define i64 @imm_neg_3435982847() {
 ; RV64I-LABEL: imm_neg_3435982847:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1048371
-; RV64I-NEXT:    addiw a0, a0, 817
+; RV64I-NEXT:    addi a0, a0, 817
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, 1
 ; RV64I-NEXT:    ret
@@ -3717,7 +3717,7 @@ define i64 @imm_neg_3435982847() {
 ; RV64IZBB-LABEL: imm_neg_3435982847:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1048371
-; RV64IZBB-NEXT:    addiw a0, a0, 817
+; RV64IZBB-NEXT:    addi a0, a0, 817
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, 1
 ; RV64IZBB-NEXT:    ret
@@ -3725,14 +3725,14 @@ define i64 @imm_neg_3435982847() {
 ; RV64IZBS-LABEL: imm_neg_3435982847:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 734001
-; RV64IZBS-NEXT:    addiw a0, a0, 1
+; RV64IZBS-NEXT:    addi a0, a0, 1
 ; RV64IZBS-NEXT:    bclri a0, a0, 31
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_3435982847:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1048371
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 817
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 817
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -3747,7 +3747,7 @@ define i64 @imm_neg_3435982847() {
 ; RV64-REMAT-LABEL: imm_neg_3435982847:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1048371
-; RV64-REMAT-NEXT:    addiw a0, a0, 817
+; RV64-REMAT-NEXT:    addi a0, a0, 817
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, 1
 ; RV64-REMAT-NEXT:    ret
@@ -3771,7 +3771,7 @@ define i64 @imm_neg_5726842879() {
 ; RV64I-LABEL: imm_neg_5726842879:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1048235
-; RV64I-NEXT:    addiw a0, a0, -1419
+; RV64I-NEXT:    addi a0, a0, -1419
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, 1
 ; RV64I-NEXT:    ret
@@ -3786,7 +3786,7 @@ define i64 @imm_neg_5726842879() {
 ; RV64IZBB-LABEL: imm_neg_5726842879:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1048235
-; RV64IZBB-NEXT:    addiw a0, a0, -1419
+; RV64IZBB-NEXT:    addi a0, a0, -1419
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, 1
 ; RV64IZBB-NEXT:    ret
@@ -3794,14 +3794,14 @@ define i64 @imm_neg_5726842879() {
 ; RV64IZBS-LABEL: imm_neg_5726842879:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 698997
-; RV64IZBS-NEXT:    addiw a0, a0, 1
+; RV64IZBS-NEXT:    addi a0, a0, 1
 ; RV64IZBS-NEXT:    bclri a0, a0, 32
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_5726842879:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1048235
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1419
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1419
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -3816,7 +3816,7 @@ define i64 @imm_neg_5726842879() {
 ; RV64-REMAT-LABEL: imm_neg_5726842879:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1048235
-; RV64-REMAT-NEXT:    addiw a0, a0, -1419
+; RV64-REMAT-NEXT:    addi a0, a0, -1419
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, 1
 ; RV64-REMAT-NEXT:    ret
@@ -3840,7 +3840,7 @@ define i64 @imm_neg_10307948543() {
 ; RV64I-LABEL: imm_neg_10307948543:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1047962
-; RV64I-NEXT:    addiw a0, a0, -1645
+; RV64I-NEXT:    addi a0, a0, -1645
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    addi a0, a0, 1
 ; RV64I-NEXT:    ret
@@ -3855,7 +3855,7 @@ define i64 @imm_neg_10307948543() {
 ; RV64IZBB-LABEL: imm_neg_10307948543:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1047962
-; RV64IZBB-NEXT:    addiw a0, a0, -1645
+; RV64IZBB-NEXT:    addi a0, a0, -1645
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    addi a0, a0, 1
 ; RV64IZBB-NEXT:    ret
@@ -3863,14 +3863,14 @@ define i64 @imm_neg_10307948543() {
 ; RV64IZBS-LABEL: imm_neg_10307948543:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 629139
-; RV64IZBS-NEXT:    addiw a0, a0, 1
+; RV64IZBS-NEXT:    addi a0, a0, 1
 ; RV64IZBS-NEXT:    bclri a0, a0, 33
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_neg_10307948543:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1047962
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1645
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1645
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -3885,7 +3885,7 @@ define i64 @imm_neg_10307948543() {
 ; RV64-REMAT-LABEL: imm_neg_10307948543:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1047962
-; RV64-REMAT-NEXT:    addiw a0, a0, -1645
+; RV64-REMAT-NEXT:    addi a0, a0, -1645
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    addi a0, a0, 1
 ; RV64-REMAT-NEXT:    ret
@@ -4098,7 +4098,7 @@ define i64 @PR54812() {
 ; RV64I-LABEL: PR54812:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1048447
-; RV64I-NEXT:    addiw a0, a0, 1407
+; RV64I-NEXT:    addi a0, a0, 1407
 ; RV64I-NEXT:    slli a0, a0, 12
 ; RV64I-NEXT:    ret
 ;
@@ -4111,7 +4111,7 @@ define i64 @PR54812() {
 ; RV64IZBB-LABEL: PR54812:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1048447
-; RV64IZBB-NEXT:    addiw a0, a0, 1407
+; RV64IZBB-NEXT:    addi a0, a0, 1407
 ; RV64IZBB-NEXT:    slli a0, a0, 12
 ; RV64IZBB-NEXT:    ret
 ;
@@ -4124,7 +4124,7 @@ define i64 @PR54812() {
 ; RV64IXTHEADBB-LABEL: PR54812:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1048447
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1407
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1407
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 12
 ; RV64IXTHEADBB-NEXT:    ret
 ;
@@ -4137,7 +4137,7 @@ define i64 @PR54812() {
 ; RV64-REMAT-LABEL: PR54812:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1048447
-; RV64-REMAT-NEXT:    addiw a0, a0, 1407
+; RV64-REMAT-NEXT:    addi a0, a0, 1407
 ; RV64-REMAT-NEXT:    slli a0, a0, 12
 ; RV64-REMAT-NEXT:    ret
   ret i64 -2158497792;
@@ -4215,7 +4215,7 @@ define i64 @imm64_same_lo_hi() nounwind {
 ; RV64I-LABEL: imm64_same_lo_hi:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 65793
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    slli a1, a0, 32
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
@@ -4223,7 +4223,7 @@ define i64 @imm64_same_lo_hi() nounwind {
 ; RV64IZBA-LABEL: imm64_same_lo_hi:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 65793
-; RV64IZBA-NEXT:    addiw a0, a0, 16
+; RV64IZBA-NEXT:    addi a0, a0, 16
 ; RV64IZBA-NEXT:    slli a1, a0, 32
 ; RV64IZBA-NEXT:    add a0, a0, a1
 ; RV64IZBA-NEXT:    ret
@@ -4231,7 +4231,7 @@ define i64 @imm64_same_lo_hi() nounwind {
 ; RV64IZBB-LABEL: imm64_same_lo_hi:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 65793
-; RV64IZBB-NEXT:    addiw a0, a0, 16
+; RV64IZBB-NEXT:    addi a0, a0, 16
 ; RV64IZBB-NEXT:    slli a1, a0, 32
 ; RV64IZBB-NEXT:    add a0, a0, a1
 ; RV64IZBB-NEXT:    ret
@@ -4239,7 +4239,7 @@ define i64 @imm64_same_lo_hi() nounwind {
 ; RV64IZBS-LABEL: imm64_same_lo_hi:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 65793
-; RV64IZBS-NEXT:    addiw a0, a0, 16
+; RV64IZBS-NEXT:    addi a0, a0, 16
 ; RV64IZBS-NEXT:    slli a1, a0, 32
 ; RV64IZBS-NEXT:    add a0, a0, a1
 ; RV64IZBS-NEXT:    ret
@@ -4247,7 +4247,7 @@ define i64 @imm64_same_lo_hi() nounwind {
 ; RV64IXTHEADBB-LABEL: imm64_same_lo_hi:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 65793
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 16
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 16
 ; RV64IXTHEADBB-NEXT:    slli a1, a0, 32
 ; RV64IXTHEADBB-NEXT:    add a0, a0, a1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -4262,7 +4262,7 @@ define i64 @imm64_same_lo_hi() nounwind {
 ; RV64-REMAT-LABEL: imm64_same_lo_hi:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 65793
-; RV64-REMAT-NEXT:    addiw a0, a0, 16
+; RV64-REMAT-NEXT:    addi a0, a0, 16
 ; RV64-REMAT-NEXT:    slli a1, a0, 32
 ; RV64-REMAT-NEXT:    add a0, a0, a1
 ; RV64-REMAT-NEXT:    ret
@@ -4287,7 +4287,7 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ; RV64-NOPOOL-LABEL: imm64_same_lo_hi_optsize:
 ; RV64-NOPOOL:       # %bb.0:
 ; RV64-NOPOOL-NEXT:    lui a0, 65793
-; RV64-NOPOOL-NEXT:    addiw a0, a0, 16
+; RV64-NOPOOL-NEXT:    addi a0, a0, 16
 ; RV64-NOPOOL-NEXT:    slli a1, a0, 32
 ; RV64-NOPOOL-NEXT:    add a0, a0, a1
 ; RV64-NOPOOL-NEXT:    ret
@@ -4301,7 +4301,7 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ; RV64IZBA-LABEL: imm64_same_lo_hi_optsize:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 65793
-; RV64IZBA-NEXT:    addiw a0, a0, 16
+; RV64IZBA-NEXT:    addi a0, a0, 16
 ; RV64IZBA-NEXT:    slli a1, a0, 32
 ; RV64IZBA-NEXT:    add a0, a0, a1
 ; RV64IZBA-NEXT:    ret
@@ -4309,7 +4309,7 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ; RV64IZBB-LABEL: imm64_same_lo_hi_optsize:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 65793
-; RV64IZBB-NEXT:    addiw a0, a0, 16
+; RV64IZBB-NEXT:    addi a0, a0, 16
 ; RV64IZBB-NEXT:    slli a1, a0, 32
 ; RV64IZBB-NEXT:    add a0, a0, a1
 ; RV64IZBB-NEXT:    ret
@@ -4317,7 +4317,7 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ; RV64IZBS-LABEL: imm64_same_lo_hi_optsize:
 ; RV64IZBS:       # %bb.0:
 ; RV64IZBS-NEXT:    lui a0, 65793
-; RV64IZBS-NEXT:    addiw a0, a0, 16
+; RV64IZBS-NEXT:    addi a0, a0, 16
 ; RV64IZBS-NEXT:    slli a1, a0, 32
 ; RV64IZBS-NEXT:    add a0, a0, a1
 ; RV64IZBS-NEXT:    ret
@@ -4325,7 +4325,7 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ; RV64IXTHEADBB-LABEL: imm64_same_lo_hi_optsize:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 65793
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 16
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 16
 ; RV64IXTHEADBB-NEXT:    slli a1, a0, 32
 ; RV64IXTHEADBB-NEXT:    add a0, a0, a1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -4340,7 +4340,7 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ; RV64-REMAT-LABEL: imm64_same_lo_hi_optsize:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 65793
-; RV64-REMAT-NEXT:    addiw a0, a0, 16
+; RV64-REMAT-NEXT:    addi a0, a0, 16
 ; RV64-REMAT-NEXT:    slli a1, a0, 32
 ; RV64-REMAT-NEXT:    add a0, a0, a1
 ; RV64-REMAT-NEXT:    ret
@@ -4456,21 +4456,21 @@ define i64 @imm64_0x8000080000000() {
 ; RV64I-LABEL: imm64_0x8000080000000:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 256
-; RV64I-NEXT:    addiw a0, a0, 1
+; RV64I-NEXT:    addi a0, a0, 1
 ; RV64I-NEXT:    slli a0, a0, 31
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm64_0x8000080000000:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 256
-; RV64IZBA-NEXT:    addiw a0, a0, 1
+; RV64IZBA-NEXT:    addi a0, a0, 1
 ; RV64IZBA-NEXT:    slli a0, a0, 31
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm64_0x8000080000000:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 256
-; RV64IZBB-NEXT:    addiw a0, a0, 1
+; RV64IZBB-NEXT:    addi a0, a0, 1
 ; RV64IZBB-NEXT:    slli a0, a0, 31
 ; RV64IZBB-NEXT:    ret
 ;
@@ -4483,7 +4483,7 @@ define i64 @imm64_0x8000080000000() {
 ; RV64IXTHEADBB-LABEL: imm64_0x8000080000000:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 256
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 31
 ; RV64IXTHEADBB-NEXT:    ret
 ;
@@ -4496,7 +4496,7 @@ define i64 @imm64_0x8000080000000() {
 ; RV64-REMAT-LABEL: imm64_0x8000080000000:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 256
-; RV64-REMAT-NEXT:    addiw a0, a0, 1
+; RV64-REMAT-NEXT:    addi a0, a0, 1
 ; RV64-REMAT-NEXT:    slli a0, a0, 31
 ; RV64-REMAT-NEXT:    ret
   ret i64 2251801961168896 ; 0x8000080000000
@@ -4584,7 +4584,7 @@ define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
 ; RV64I-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1044480
-; RV64I-NEXT:    addiw a0, a0, -1
+; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    slli a0, a0, 31
 ; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    ret
@@ -4592,7 +4592,7 @@ define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
 ; RV64IZBA-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
 ; RV64IZBA:       # %bb.0:
 ; RV64IZBA-NEXT:    lui a0, 1044480
-; RV64IZBA-NEXT:    addiw a0, a0, -1
+; RV64IZBA-NEXT:    addi a0, a0, -1
 ; RV64IZBA-NEXT:    slli a0, a0, 31
 ; RV64IZBA-NEXT:    addi a0, a0, -1
 ; RV64IZBA-NEXT:    ret
@@ -4600,7 +4600,7 @@ define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
 ; RV64IZBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
 ; RV64IZBB:       # %bb.0:
 ; RV64IZBB-NEXT:    lui a0, 1044480
-; RV64IZBB-NEXT:    addiw a0, a0, -1
+; RV64IZBB-NEXT:    addi a0, a0, -1
 ; RV64IZBB-NEXT:    slli a0, a0, 31
 ; RV64IZBB-NEXT:    addi a0, a0, -1
 ; RV64IZBB-NEXT:    ret
@@ -4615,7 +4615,7 @@ define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
 ; RV64IXTHEADBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
 ; RV64IXTHEADBB:       # %bb.0:
 ; RV64IXTHEADBB-NEXT:    lui a0, 1044480
-; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1
 ; RV64IXTHEADBB-NEXT:    slli a0, a0, 31
 ; RV64IXTHEADBB-NEXT:    addi a0, a0, -1
 ; RV64IXTHEADBB-NEXT:    ret
@@ -4631,9 +4631,133 @@ define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
 ; RV64-REMAT-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
 ; RV64-REMAT:       # %bb.0:
 ; RV64-REMAT-NEXT:    lui a0, 1044480
-; RV64-REMAT-NEXT:    addiw a0, a0, -1
+; RV64-REMAT-NEXT:    addi a0, a0, -1
 ; RV64-REMAT-NEXT:    slli a0, a0, 31
 ; RV64-REMAT-NEXT:    addi a0, a0, -1
 ; RV64-REMAT-NEXT:    ret
   ret i64 -36028799166447617 ; 0xFF7FFFFF7FFFFFFE
 }
+
+define i64 @imm64_0xFFFFFFFF0() {
+; RV32I-LABEL: imm64_0xFFFFFFFF0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a0, -16
+; RV32I-NEXT:    li a1, 15
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILI-LABEL: imm64_0xFFFFFFFF0:
+; RV32IXQCILI:       # %bb.0:
+; RV32IXQCILI-NEXT:    li a0, -16
+; RV32IXQCILI-NEXT:    li a1, 15
+; RV32IXQCILI-NEXT:    ret
+;
+; RV64I-LABEL: imm64_0xFFFFFFFF0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    slli a0, a0, 36
+; RV64I-NEXT:    addi a0, a0, -16
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0xFFFFFFFF0:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    li a0, 1
+; RV64IZBA-NEXT:    slli a0, a0, 36
+; RV64IZBA-NEXT:    addi a0, a0, -16
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0xFFFFFFFF0:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    li a0, 1
+; RV64IZBB-NEXT:    slli a0, a0, 36
+; RV64IZBB-NEXT:    addi a0, a0, -16
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0xFFFFFFFF0:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    bseti a0, zero, 36
+; RV64IZBS-NEXT:    addi a0, a0, -16
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0xFFFFFFFF0:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    li a0, 1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 36
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -16
+; RV64IXTHEADBB-NEXT:    ret
+;
+; RV32-REMAT-LABEL: imm64_0xFFFFFFFF0:
+; RV32-REMAT:       # %bb.0:
+; RV32-REMAT-NEXT:    li a0, -16
+; RV32-REMAT-NEXT:    li a1, 15
+; RV32-REMAT-NEXT:    ret
+;
+; RV64-REMAT-LABEL: imm64_0xFFFFFFFF0:
+; RV64-REMAT:       # %bb.0:
+; RV64-REMAT-NEXT:    li a0, 1
+; RV64-REMAT-NEXT:    slli a0, a0, 36
+; RV64-REMAT-NEXT:    addi a0, a0, -16
+; RV64-REMAT-NEXT:    ret
+  ret i64 68719476720 ; 0xFFFFFFFF0
+}
+
+define i64 @imm64_0x1FFFFFF08() {
+; RV32I-LABEL: imm64_0x1FFFFFF08:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a0, -248
+; RV32I-NEXT:    li a1, 1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCILI-LABEL: imm64_0x1FFFFFF08:
+; RV32IXQCILI:       # %bb.0:
+; RV32IXQCILI-NEXT:    li a0, -248
+; RV32IXQCILI-NEXT:    li a1, 1
+; RV32IXQCILI-NEXT:    ret
+;
+; RV64I-LABEL: imm64_0x1FFFFFF08:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a0, 1
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    addi a0, a0, -248
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0x1FFFFFF08:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    li a0, 1
+; RV64IZBA-NEXT:    slli a0, a0, 33
+; RV64IZBA-NEXT:    addi a0, a0, -248
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0x1FFFFFF08:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    li a0, 1
+; RV64IZBB-NEXT:    slli a0, a0, 33
+; RV64IZBB-NEXT:    addi a0, a0, -248
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0x1FFFFFF08:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    bseti a0, zero, 33
+; RV64IZBS-NEXT:    addi a0, a0, -248
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0x1FFFFFF08:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    li a0, 1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 33
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -248
+; RV64IXTHEADBB-NEXT:    ret
+;
+; RV32-REMAT-LABEL: imm64_0x1FFFFFF08:
+; RV32-REMAT:       # %bb.0:
+; RV32-REMAT-NEXT:    li a0, -248
+; RV32-REMAT-NEXT:    li a1, 1
+; RV32-REMAT-NEXT:    ret
+;
+; RV64-REMAT-LABEL: imm64_0x1FFFFFF08:
+; RV64-REMAT:       # %bb.0:
+; RV64-REMAT-NEXT:    li a0, 1
+; RV64-REMAT-NEXT:    slli a0, a0, 33
+; RV64-REMAT-NEXT:    addi a0, a0, -248
+; RV64-REMAT-NEXT:    ret
+  ret i64 8589934344 ; 0x1FFFFFF08
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/inline-asm-mem-constraint.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/inline-asm-mem-constraint.ll
index f0837d55bc53..f5e52446b418 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/inline-asm-mem-constraint.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/inline-asm-mem-constraint.ll
@@ -273,7 +273,7 @@ define void @constraint_m_with_global_3() nounwind {
 ; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI5_0)
 ; RV64I-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi2)(a0)
 ; RV64I-LARGE-NEXT:    lui a1, 2
-; RV64I-LARGE-NEXT:    addiw a1, a1, -192
+; RV64I-LARGE-NEXT:    addi a1, a1, -192
 ; RV64I-LARGE-NEXT:    add a0, a0, a1
 ; RV64I-LARGE-NEXT:    #APP
 ; RV64I-LARGE-NEXT:    sw zero, 0(a0)
@@ -419,7 +419,7 @@ define void @constraint_m_with_extern_weak_global_3() nounwind {
 ; RV64I-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(ewg)
 ; RV64I-MEDIUM-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi5)(a0)
 ; RV64I-MEDIUM-NEXT:    lui a1, 2
-; RV64I-MEDIUM-NEXT:    addiw a1, a1, -192
+; RV64I-MEDIUM-NEXT:    addi a1, a1, -192
 ; RV64I-MEDIUM-NEXT:    add a0, a0, a1
 ; RV64I-MEDIUM-NEXT:    #APP
 ; RV64I-MEDIUM-NEXT:    sw zero, 0(a0)
@@ -432,7 +432,7 @@ define void @constraint_m_with_extern_weak_global_3() nounwind {
 ; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI8_0)
 ; RV64I-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi5)(a0)
 ; RV64I-LARGE-NEXT:    lui a1, 2
-; RV64I-LARGE-NEXT:    addiw a1, a1, -192
+; RV64I-LARGE-NEXT:    addi a1, a1, -192
 ; RV64I-LARGE-NEXT:    add a0, a0, a1
 ; RV64I-LARGE-NEXT:    #APP
 ; RV64I-LARGE-NEXT:    sw zero, 0(a0)
@@ -796,7 +796,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV32I-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-NO-INTEGRATED-NEXT:    ret
-; RV32I-NO-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV32I-NO-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV32I-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -811,7 +811,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV64I-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-NO-INTEGRATED-NEXT:    ret
-; RV64I-NO-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV64I-NO-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV64I-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -827,7 +827,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -843,7 +843,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -860,7 +860,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 1
@@ -877,7 +877,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV32I-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-INTEGRATED-NEXT:    ret
-; RV32I-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV32I-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV32I-INTEGRATED-NEXT:    # %fail
 ; RV32I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-INTEGRATED-NEXT:    li a0, 1
@@ -894,7 +894,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV64I-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-INTEGRATED-NEXT:    ret
-; RV64I-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV64I-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV64I-INTEGRATED-NEXT:    # %fail
 ; RV64I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-INTEGRATED-NEXT:    li a0, 1
@@ -912,7 +912,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV32I-MEDIUM-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -930,7 +930,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV64I-MEDIUM-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -949,7 +949,7 @@ define i32 @constraint_m_with_callbr_multi_operands(i32 %a) {
 ; RV64I-LARGE-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-INTEGRATED-NEXT:  .LBB14_2: # Block address taken
+; RV64I-LARGE-INTEGRATED-NEXT:  .LBB14_2: # Inline asm indirect target
 ; RV64I-LARGE-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 1
@@ -978,7 +978,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV32I-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-NO-INTEGRATED-NEXT:    ret
-; RV32I-NO-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV32I-NO-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV32I-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -997,7 +997,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV64I-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-NO-INTEGRATED-NEXT:    ret
-; RV64I-NO-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV64I-NO-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV64I-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1017,7 +1017,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1037,7 +1037,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1058,7 +1058,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1079,7 +1079,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV32I-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-INTEGRATED-NEXT:    ret
-; RV32I-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV32I-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV32I-INTEGRATED-NEXT:    # %fail
 ; RV32I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-INTEGRATED-NEXT:    li a0, 1
@@ -1100,7 +1100,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV64I-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-INTEGRATED-NEXT:    ret
-; RV64I-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV64I-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV64I-INTEGRATED-NEXT:    # %fail
 ; RV64I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-INTEGRATED-NEXT:    li a0, 1
@@ -1122,7 +1122,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV32I-MEDIUM-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -1144,7 +1144,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV64I-MEDIUM-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -1167,7 +1167,7 @@ define i32 @constraint_m_with_multi_callbr_asm(i32 %a) {
 ; RV64I-LARGE-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-INTEGRATED-NEXT:  .LBB15_3: # Block address taken
+; RV64I-LARGE-INTEGRATED-NEXT:  .LBB15_3: # Inline asm indirect target
 ; RV64I-LARGE-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 1
@@ -1435,7 +1435,7 @@ define void @constraint_o_with_global_3() nounwind {
 ; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI21_0)
 ; RV64I-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi15)(a0)
 ; RV64I-LARGE-NEXT:    lui a1, 2
-; RV64I-LARGE-NEXT:    addiw a1, a1, -192
+; RV64I-LARGE-NEXT:    addi a1, a1, -192
 ; RV64I-LARGE-NEXT:    add a0, a0, a1
 ; RV64I-LARGE-NEXT:    #APP
 ; RV64I-LARGE-NEXT:    sw zero, 0(a0)
@@ -1581,7 +1581,7 @@ define void @constraint_o_with_extern_weak_global_3() nounwind {
 ; RV64I-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(ewg)
 ; RV64I-MEDIUM-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi18)(a0)
 ; RV64I-MEDIUM-NEXT:    lui a1, 2
-; RV64I-MEDIUM-NEXT:    addiw a1, a1, -192
+; RV64I-MEDIUM-NEXT:    addi a1, a1, -192
 ; RV64I-MEDIUM-NEXT:    add a0, a0, a1
 ; RV64I-MEDIUM-NEXT:    #APP
 ; RV64I-MEDIUM-NEXT:    sw zero, 0(a0)
@@ -1594,7 +1594,7 @@ define void @constraint_o_with_extern_weak_global_3() nounwind {
 ; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI24_0)
 ; RV64I-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi18)(a0)
 ; RV64I-LARGE-NEXT:    lui a1, 2
-; RV64I-LARGE-NEXT:    addiw a1, a1, -192
+; RV64I-LARGE-NEXT:    addi a1, a1, -192
 ; RV64I-LARGE-NEXT:    add a0, a0, a1
 ; RV64I-LARGE-NEXT:    #APP
 ; RV64I-LARGE-NEXT:    sw zero, 0(a0)
@@ -1678,7 +1678,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV32I-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-NO-INTEGRATED-NEXT:    ret
-; RV32I-NO-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV32I-NO-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV32I-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1693,7 +1693,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV64I-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-NO-INTEGRATED-NEXT:    ret
-; RV64I-NO-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV64I-NO-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV64I-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1709,7 +1709,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1725,7 +1725,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1742,7 +1742,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1759,7 +1759,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV32I-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-INTEGRATED-NEXT:    ret
-; RV32I-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV32I-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV32I-INTEGRATED-NEXT:    # %fail
 ; RV32I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-INTEGRATED-NEXT:    li a0, 1
@@ -1776,7 +1776,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV64I-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-INTEGRATED-NEXT:    ret
-; RV64I-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV64I-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV64I-INTEGRATED-NEXT:    # %fail
 ; RV64I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-INTEGRATED-NEXT:    li a0, 1
@@ -1794,7 +1794,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV32I-MEDIUM-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -1812,7 +1812,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV64I-MEDIUM-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -1831,7 +1831,7 @@ define i32 @constraint_o_with_callbr_multi_operands(i32 %a) {
 ; RV64I-LARGE-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-INTEGRATED-NEXT:  .LBB26_2: # Block address taken
+; RV64I-LARGE-INTEGRATED-NEXT:  .LBB26_2: # Inline asm indirect target
 ; RV64I-LARGE-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 1
@@ -1860,7 +1860,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV32I-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-NO-INTEGRATED-NEXT:    ret
-; RV32I-NO-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV32I-NO-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV32I-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1879,7 +1879,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV64I-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-NO-INTEGRATED-NEXT:    ret
-; RV64I-NO-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV64I-NO-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV64I-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1899,7 +1899,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1919,7 +1919,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1940,7 +1940,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 1
@@ -1961,7 +1961,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV32I-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-INTEGRATED-NEXT:    ret
-; RV32I-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV32I-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV32I-INTEGRATED-NEXT:    # %fail
 ; RV32I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-INTEGRATED-NEXT:    li a0, 1
@@ -1982,7 +1982,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV64I-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-INTEGRATED-NEXT:    ret
-; RV64I-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV64I-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV64I-INTEGRATED-NEXT:    # %fail
 ; RV64I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-INTEGRATED-NEXT:    li a0, 1
@@ -2004,7 +2004,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV32I-MEDIUM-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -2026,7 +2026,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV64I-MEDIUM-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -2049,7 +2049,7 @@ define i32 @constraint_o_with_multi_callbr_asm(i32 %a) {
 ; RV64I-LARGE-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-INTEGRATED-NEXT:  .LBB27_3: # Block address taken
+; RV64I-LARGE-INTEGRATED-NEXT:  .LBB27_3: # Inline asm indirect target
 ; RV64I-LARGE-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 1
@@ -2500,7 +2500,7 @@ define void @constraint_A_with_global_3() nounwind {
 ; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI35_0)
 ; RV64I-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi27)(a0)
 ; RV64I-LARGE-NEXT:    lui a1, 2
-; RV64I-LARGE-NEXT:    addiw a1, a1, -192
+; RV64I-LARGE-NEXT:    addi a1, a1, -192
 ; RV64I-LARGE-NEXT:    add a0, a0, a1
 ; RV64I-LARGE-NEXT:    #APP
 ; RV64I-LARGE-NEXT:    sw zero, 0(a0)
@@ -2655,7 +2655,7 @@ define void @constraint_A_with_extern_weak_global_3() nounwind {
 ; RV64I-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(ewg)
 ; RV64I-MEDIUM-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi30)(a0)
 ; RV64I-MEDIUM-NEXT:    lui a1, 2
-; RV64I-MEDIUM-NEXT:    addiw a1, a1, -192
+; RV64I-MEDIUM-NEXT:    addi a1, a1, -192
 ; RV64I-MEDIUM-NEXT:    add a0, a0, a1
 ; RV64I-MEDIUM-NEXT:    #APP
 ; RV64I-MEDIUM-NEXT:    sw zero, 0(a0)
@@ -2668,7 +2668,7 @@ define void @constraint_A_with_extern_weak_global_3() nounwind {
 ; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI38_0)
 ; RV64I-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi30)(a0)
 ; RV64I-LARGE-NEXT:    lui a1, 2
-; RV64I-LARGE-NEXT:    addiw a1, a1, -192
+; RV64I-LARGE-NEXT:    addi a1, a1, -192
 ; RV64I-LARGE-NEXT:    add a0, a0, a1
 ; RV64I-LARGE-NEXT:    #APP
 ; RV64I-LARGE-NEXT:    sw zero, 0(a0)
@@ -2757,7 +2757,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV32I-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-NO-INTEGRATED-NEXT:    ret
-; RV32I-NO-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV32I-NO-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV32I-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -2773,7 +2773,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV64I-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-NO-INTEGRATED-NEXT:    ret
-; RV64I-NO-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV64I-NO-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV64I-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -2790,7 +2790,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -2807,7 +2807,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -2824,7 +2824,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 1
@@ -2842,7 +2842,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV32I-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-INTEGRATED-NEXT:    ret
-; RV32I-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV32I-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV32I-INTEGRATED-NEXT:    # %fail
 ; RV32I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-INTEGRATED-NEXT:    li a0, 1
@@ -2860,7 +2860,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV64I-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-INTEGRATED-NEXT:    ret
-; RV64I-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV64I-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV64I-INTEGRATED-NEXT:    # %fail
 ; RV64I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-INTEGRATED-NEXT:    li a0, 1
@@ -2879,7 +2879,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV32I-MEDIUM-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -2898,7 +2898,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV64I-MEDIUM-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -2917,7 +2917,7 @@ define i32 @constraint_A_with_callbr_multi_operands(i32 %a) {
 ; RV64I-LARGE-INTEGRATED-NEXT:  # %bb.1: # %normal
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-INTEGRATED-NEXT:  .LBB40_2: # Block address taken
+; RV64I-LARGE-INTEGRATED-NEXT:  .LBB40_2: # Inline asm indirect target
 ; RV64I-LARGE-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 1
@@ -2947,7 +2947,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV32I-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-NO-INTEGRATED-NEXT:    ret
-; RV32I-NO-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV32I-NO-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV32I-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -2967,7 +2967,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV64I-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-NO-INTEGRATED-NEXT:    ret
-; RV64I-NO-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV64I-NO-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV64I-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-NO-INTEGRATED-NEXT:    li a0, 1
@@ -2988,7 +2988,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV32I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -3009,7 +3009,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV64I-MEDIUM-NO-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-NO-INTEGRATED-NEXT:    li a0, 1
@@ -3030,7 +3030,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV64I-LARGE-NO-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-NO-INTEGRATED-NEXT:    li a0, 1
@@ -3052,7 +3052,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV32I-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-INTEGRATED-NEXT:    ret
-; RV32I-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV32I-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV32I-INTEGRATED-NEXT:    # %fail
 ; RV32I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-INTEGRATED-NEXT:    li a0, 1
@@ -3074,7 +3074,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV64I-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-INTEGRATED-NEXT:    ret
-; RV64I-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV64I-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV64I-INTEGRATED-NEXT:    # %fail
 ; RV64I-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-INTEGRATED-NEXT:    li a0, 1
@@ -3097,7 +3097,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV32I-MEDIUM-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV32I-MEDIUM-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -3120,7 +3120,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV64I-MEDIUM-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    ret
-; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV64I-MEDIUM-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # %fail
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-MEDIUM-INTEGRATED-NEXT:    li a0, 1
@@ -3143,7 +3143,7 @@ define i32 @constraint_A_with_multi_callbr_asm(i32 %a) {
 ; RV64I-LARGE-INTEGRATED-NEXT:  # %bb.2: # %normal1
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 0
 ; RV64I-LARGE-INTEGRATED-NEXT:    ret
-; RV64I-LARGE-INTEGRATED-NEXT:  .LBB41_3: # Block address taken
+; RV64I-LARGE-INTEGRATED-NEXT:  .LBB41_3: # Inline asm indirect target
 ; RV64I-LARGE-INTEGRATED-NEXT:    # %fail
 ; RV64I-LARGE-INTEGRATED-NEXT:    # Label of block must be emitted
 ; RV64I-LARGE-INTEGRATED-NEXT:    li a0, 1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/lack-of-signed-truncation-check.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/lack-of-signed-truncation-check.ll
index 4a338ce5bd1f..64a257c1d578 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/lack-of-signed-truncation-check.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/lack-of-signed-truncation-check.ll
@@ -766,7 +766,7 @@ define i1 @add_ugecmp_bad_i16_i8_cmp(i16 %x, i16 %y) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    addi a0, a0, 128
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sltu a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/legalize-fneg.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/legalize-fneg.ll
index f60b77b92c09..9e66eb7a2ae6 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/legalize-fneg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/legalize-fneg.ll
@@ -56,16 +56,16 @@ entry:
 define void @test3(ptr %a, ptr %b) nounwind {
 ; RV32-LABEL: test3:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    lw a2, 12(a1)
-; RV32-NEXT:    lw a3, 0(a1)
+; RV32-NEXT:    lw a2, 0(a1)
+; RV32-NEXT:    lw a3, 12(a1)
 ; RV32-NEXT:    lw a4, 4(a1)
 ; RV32-NEXT:    lw a1, 8(a1)
 ; RV32-NEXT:    lui a5, 524288
-; RV32-NEXT:    xor a2, a2, a5
-; RV32-NEXT:    sw a3, 0(a0)
+; RV32-NEXT:    xor a3, a3, a5
+; RV32-NEXT:    sw a2, 0(a0)
 ; RV32-NEXT:    sw a4, 4(a0)
 ; RV32-NEXT:    sw a1, 8(a0)
-; RV32-NEXT:    sw a2, 12(a0)
+; RV32-NEXT:    sw a3, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: test3:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
index 3c83f5f91b37..a691fc885704 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
@@ -26,16 +26,16 @@ define void @use_frame_base_reg() {
 ; RV64I-LABEL: use_frame_base_reg:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 24
-; RV64I-NEXT:    addiw a0, a0, 1712
+; RV64I-NEXT:    addi a0, a0, 1712
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 100016
 ; RV64I-NEXT:    lui a0, 24
-; RV64I-NEXT:    addiw a0, a0, 1704
+; RV64I-NEXT:    addi a0, a0, 1704
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    lbu zero, 4(a0)
 ; RV64I-NEXT:    lbu zero, 0(a0)
 ; RV64I-NEXT:    lui a0, 24
-; RV64I-NEXT:    addiw a0, a0, 1712
+; RV64I-NEXT:    addi a0, a0, 1712
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/loop-strength-reduce-add-cheaper-than-mul.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/loop-strength-reduce-add-cheaper-than-mul.ll
index fa8ca071d218..2ec8b46cf3ee 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/loop-strength-reduce-add-cheaper-than-mul.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/loop-strength-reduce-add-cheaper-than-mul.ll
@@ -57,7 +57,7 @@ define void @test(i32 signext %i) nounwind {
 ; RV64-NEXT:  # %bb.1: # %bb.preheader
 ; RV64-NEXT:    lui a2, %hi(flags2)
 ; RV64-NEXT:    addi a2, a2, %lo(flags2)
-; RV64-NEXT:    addiw a3, a3, 1
+; RV64-NEXT:    addi a3, a3, 1
 ; RV64-NEXT:  .LBB0_2: # %bb
 ; RV64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV64-NEXT:    slli a4, a1, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/machine-copyprop-simplifyinstruction.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/machine-copyprop-simplifyinstruction.mir
index e329028ac0b8..31c1649189a2 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/machine-copyprop-simplifyinstruction.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/machine-copyprop-simplifyinstruction.mir
@@ -743,6 +743,18 @@ body: |
     PseudoRET implicit $x10
 ...
 ---
+name: multipass
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: multipass
+    ; CHECK: renamable $x9 = ADDI $x0, 0
+    ; CHECK-NEXT: PseudoRET implicit $x9
+    renamable $x11 = COPY $x0
+    renamable $x10 = SLLI renamable $x11, 13
+    renamable $x9 = SRLI renamable $x10, 13
+    PseudoRET implicit $x9
+...
+---
 name: beq
 body: |
   ; CHECK-LABEL: name: beq
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
index d1b10af16063..ed99e6030ebd 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
@@ -77,19 +77,19 @@ define i32 @test_matint() {
 ; FUSION-LABEL: test_matint:
 ; FUSION:       # %bb.0:
 ; FUSION-NEXT:    lui a0, 1
-; FUSION-NEXT:    addiw a0, a0, -2048
+; FUSION-NEXT:    addi a0, a0, -2048
 ; FUSION-NEXT:    ret
 ;
 ; FUSION-POSTRA-LABEL: test_matint:
 ; FUSION-POSTRA:       # %bb.0:
 ; FUSION-POSTRA-NEXT:    lui a0, 1
-; FUSION-POSTRA-NEXT:    addiw a0, a0, -2048
+; FUSION-POSTRA-NEXT:    addi a0, a0, -2048
 ; FUSION-POSTRA-NEXT:    ret
 ;
 ; FUSION-GENERIC-LABEL: test_matint:
 ; FUSION-GENERIC:       # %bb.0:
 ; FUSION-GENERIC-NEXT:    lui a0, 1
-; FUSION-GENERIC-NEXT:    addiw a0, a0, -2048
+; FUSION-GENERIC-NEXT:    addi a0, a0, -2048
 ; FUSION-GENERIC-NEXT:    ret
   ret i32 2048
 }
@@ -99,27 +99,27 @@ define void @test_regalloc_hint(i32 noundef signext %0, i32 noundef signext %1)
 ; NOFUSION:       # %bb.0:
 ; NOFUSION-NEXT:    mv a0, a1
 ; NOFUSION-NEXT:    lui a1, 3014
-; NOFUSION-NEXT:    addiw a1, a1, 334
+; NOFUSION-NEXT:    addi a1, a1, 334
 ; NOFUSION-NEXT:    tail bar
 ;
 ; FUSION-LABEL: test_regalloc_hint:
 ; FUSION:       # %bb.0:
 ; FUSION-NEXT:    mv a0, a1
 ; FUSION-NEXT:    lui a1, 3014
-; FUSION-NEXT:    addiw a1, a1, 334
+; FUSION-NEXT:    addi a1, a1, 334
 ; FUSION-NEXT:    tail bar
 ;
 ; FUSION-POSTRA-LABEL: test_regalloc_hint:
 ; FUSION-POSTRA:       # %bb.0:
 ; FUSION-POSTRA-NEXT:    mv a0, a1
 ; FUSION-POSTRA-NEXT:    lui a1, 3014
-; FUSION-POSTRA-NEXT:    addiw a1, a1, 334
+; FUSION-POSTRA-NEXT:    addi a1, a1, 334
 ; FUSION-POSTRA-NEXT:    tail bar
 ;
 ; FUSION-GENERIC-LABEL: test_regalloc_hint:
 ; FUSION-GENERIC:       # %bb.0:
 ; FUSION-GENERIC-NEXT:    lui a2, 3014
-; FUSION-GENERIC-NEXT:    addiw a2, a2, 334
+; FUSION-GENERIC-NEXT:    addi a2, a2, 334
 ; FUSION-GENERIC-NEXT:    mv a0, a1
 ; FUSION-GENERIC-NEXT:    mv a1, a2
 ; FUSION-GENERIC-NEXT:    tail bar
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp-optsize.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
index fc714e3faef4..3742383675b9 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
@@ -870,13 +870,11 @@ define i32 @bcmp_size_8(ptr %s1, ptr %s2) nounwind optsize {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_8:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
@@ -1073,18 +1071,18 @@ define i32 @bcmp_size_15(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 7(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 11(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 0(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 7(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 11(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a5
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a6
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, a7
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a3, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a4
 ; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
@@ -1284,33 +1282,21 @@ define i32 @bcmp_size_16(ptr %s1, ptr %s2) nounwind optsize {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_16:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a4, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_16:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -1319,15 +1305,15 @@ entry:
 }
 
 define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
-; CHECK-RV32-LABEL: bcmp_size_31:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 31
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_31:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 31
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV64-LABEL: bcmp_size_31:
 ; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
@@ -1339,6 +1325,16 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_31:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 31
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_31:
 ; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
@@ -1349,6 +1345,16 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_31:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 31
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_31:
 ; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
@@ -1359,6 +1365,16 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_31:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 31
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_31:
 ; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
@@ -1369,6 +1385,16 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_31:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 31
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_31:
 ; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-NEXT:    ld a2, 0(a0)
@@ -1389,6 +1415,16 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_31:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 31
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_31:
 ; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a2, 0(a0)
@@ -1409,6 +1445,16 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_31:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 31
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_31:
 ; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a2, 0(a0)
@@ -1429,260 +1475,1278 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
 ;
-; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_31:
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_31:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 12(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 15(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 19(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 23(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 27(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t1
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 15(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 19(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 23(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 27(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a6
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t0
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, t3, t1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, t2, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, t4, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a2, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a5, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a3, a6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a1, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_31:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 15(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 23(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 8(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 15(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 23(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a3, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a4
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
+entry:
+  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 31)
+  ret i32 %bcmp
+}
+
+define i32 @bcmp_size_32(ptr %s1, ptr %s2) nounwind optsize {
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 32
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a3, 8(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a4, 16(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a0, 24(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a5, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a6, 8(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a7, 16(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a1, 24(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV64-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV64-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 32
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a3, 8(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a4, 16(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a0, 24(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a5, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a6, 8(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a7, 16(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a1, 24(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 32
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a3, 8(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a4, 16(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a0, 24(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a5, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a6, 8(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a7, 16(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a1, 24(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 32
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v12
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 32
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v12
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
+entry:
+  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 32)
+  ret i32 %bcmp
+}
+
+define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind optsize {
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV64-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV64-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -48
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 44(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 40(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 36(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 32(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 28(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 20(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 16(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 8(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 4(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 16(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 20(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 24(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 28(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 20(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 24(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 28(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 0(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 4(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 8(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 12(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 47(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 51(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 55(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 59(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 31(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 35(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 39(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 43(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, t1, s2
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, a7, s0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 31(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 35(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 39(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 43(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, t3
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, t0, s1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 47(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 51(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 55(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 59(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, a6, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, s10
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, s6, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, s8, s0
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, s4, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, s9, s2
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s5, s1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, s7, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, s3, t3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, t3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, a6, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, s0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t0, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, a7, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a7, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t0, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a6, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 44(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 40(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 36(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 28(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 24(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 20(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 16(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 4(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 48
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 24(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 31(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 39(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 47(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 55(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a7
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t1
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 31(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 39(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 47(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 55(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a6
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t0
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, t3, t1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, t2, a7
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, t4, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, a7
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a2, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a5, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a3, a6
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a1, a4
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
+entry:
+  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 63)
+  ret i32 %bcmp
+}
+
+define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind optsize {
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV64-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV64-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v16
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v16
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
+entry:
+  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 64)
+  ret i32 %bcmp
+}
+
+define i32 @bcmp_size_127(ptr %s1, ptr %s2) nounwind optsize {
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -96
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 32(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 36(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 40(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 44(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 0(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 4(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 8(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 12(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 48(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 52(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 56(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 60(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 16(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 20(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 24(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 28(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 12(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 60(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 16(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 20(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 24(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 28(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 36(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 40(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 44(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 40(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a3, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a4, s4
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 32(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 56(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 48(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a6, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 28(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 4(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a7, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 24(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t0, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 20(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 0(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t1, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 16(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 107(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t5, s10
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 75(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, t6, s8
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 123(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s2, s2, s4
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s0, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, t4, s11
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 83(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 87(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 91(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s1, s1, s6
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 107(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, t3, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 91(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, t2, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 123(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, a5, s3
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 75(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s5, s11, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s7, s8, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 87(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 83(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s3, s10, s3
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 115(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s6, ra, s6
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 115(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s4, s4, s11
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 119(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 119(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s10, s10, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 71(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 67(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 67(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 71(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 99(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 99(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, a5, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 103(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 103(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, a4, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, s9, s8
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, s11, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, ra, a6
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a5, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 95(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 63(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 111(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 79(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 79(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 111(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 63(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 95(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s11
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s9, s9, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, s8, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, a5, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t2, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t3, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t4, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, s1, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t5, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, s0, a6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, s2, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, t6, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t2, t0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t1, t2, t1
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 20(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t2, s10
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t3, t3, s4
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 28(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t4, s6
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 32(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t5, t5, s3
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 36(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, t6, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or s0, s0, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, s0, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t5, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t3, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t1, t0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a6, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t4, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t0, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 96
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_127:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 23(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 23(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, -96
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 32(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 40(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 48(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 56(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 32(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 40(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 48(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 56(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t6, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 8(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 16(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 24(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 95(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 103(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 111(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 119(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 63(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 71(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 79(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 87(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t1, t1, s2
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, a7, s0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 63(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 71(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 79(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 87(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, t3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t0, t0, s1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t4
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 95(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 103(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 111(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 119(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, a6, t6
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, t2
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, s10
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, s6, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t2, s8, s0
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t4, s4, t4
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t6, s9, s2
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor s0, s5, s1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t5, s7, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t3, s3, t3
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, t3
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a6, a6, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, s0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or t0, t0, t6
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, a3, t4
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a7, a7, t2
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a5, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, t1, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a7, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, t0, a4
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a6, a2
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a4, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, 96
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
-  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 31)
+  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 127)
   ret i32 %bcmp
 }
 
-define i32 @bcmp_size_32(ptr %s1, ptr %s2) nounwind optsize {
-; CHECK-RV32-LABEL: bcmp_size_32:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 32
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
+define i32 @bcmp_size_128(ptr %s1, ptr %s2) nounwind optsize {
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
 ;
-; CHECK-ALIGNED-RV64-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV64-LABEL: bcmp_size_128:
 ; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 128
 ; CHECK-ALIGNED-RV64-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
 ;
-; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_128:
 ; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 128
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
 ;
-; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_128:
 ; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 128
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
 ;
-; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_32:
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_128:
 ; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 32
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 128
 ; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_128:
 ; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-NEXT:    ld a0, 24(a0)
-; CHECK-UNALIGNED-RV64-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-NEXT:    ld a7, 16(a1)
-; CHECK-UNALIGNED-RV64-NEXT:    ld a1, 24(a1)
-; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-NEXT:    or a0, a4, a0
-; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-UNALIGNED-RV64-NEXT:    ret
 ;
-; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_128:
 ; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a0, 24(a0)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a7, 16(a1)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a1, 24(a1)
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a4, a0
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
 ;
-; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_128:
 ; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a0, 24(a0)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a7, 16(a1)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a1, 24(a1)
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a4, a0
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
 ;
-; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_32:
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v24
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_128:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 24(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a4, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v24
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
-entry:
-  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 32)
-  ret i32 %bcmp
-}
-
-define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind optsize {
-; CHECK-RV32-LABEL: bcmp_size_63:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 63
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
-;
-; CHECK-RV64-LABEL: bcmp_size_63:
-; CHECK-RV64:       # %bb.0: # %entry
-; CHECK-RV64-NEXT:    addi sp, sp, -16
-; CHECK-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-RV64-NEXT:    li a2, 63
-; CHECK-RV64-NEXT:    call bcmp
-; CHECK-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-RV64-NEXT:    addi sp, sp, 16
-; CHECK-RV64-NEXT:    ret
-entry:
-  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 63)
-  ret i32 %bcmp
-}
-
-define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind optsize {
-; CHECK-RV32-LABEL: bcmp_size_64:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 64
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
-;
-; CHECK-RV64-LABEL: bcmp_size_64:
-; CHECK-RV64:       # %bb.0: # %entry
-; CHECK-RV64-NEXT:    addi sp, sp, -16
-; CHECK-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-RV64-NEXT:    li a2, 64
-; CHECK-RV64-NEXT:    call bcmp
-; CHECK-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-RV64-NEXT:    addi sp, sp, 16
-; CHECK-RV64-NEXT:    ret
-entry:
-  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 64)
-  ret i32 %bcmp
-}
-
-define i32 @bcmp_size_127(ptr %s1, ptr %s2) nounwind optsize {
-; CHECK-RV32-LABEL: bcmp_size_127:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 127
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
-;
-; CHECK-RV64-LABEL: bcmp_size_127:
-; CHECK-RV64:       # %bb.0: # %entry
-; CHECK-RV64-NEXT:    addi sp, sp, -16
-; CHECK-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-RV64-NEXT:    li a2, 127
-; CHECK-RV64-NEXT:    call bcmp
-; CHECK-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-RV64-NEXT:    addi sp, sp, 16
-; CHECK-RV64-NEXT:    ret
-entry:
-  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 127)
-  ret i32 %bcmp
-}
-
-define i32 @bcmp_size_128(ptr %s1, ptr %s2) nounwind optsize {
-; CHECK-RV32-LABEL: bcmp_size_128:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 128
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
-;
-; CHECK-RV64-LABEL: bcmp_size_128:
-; CHECK-RV64:       # %bb.0: # %entry
-; CHECK-RV64-NEXT:    addi sp, sp, -16
-; CHECK-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-RV64-NEXT:    li a2, 128
-; CHECK-RV64-NEXT:    call bcmp
-; CHECK-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-RV64-NEXT:    addi sp, sp, 16
-; CHECK-RV64-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 128)
   ret i32 %bcmp
@@ -4411,25 +5475,25 @@ define i1 @memcmp_eq_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV32-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV32-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a1, 3(a1)
+; CHECK-ALIGNED-RV32-NEXT:    lbu a3, 2(a1)
+; CHECK-ALIGNED-RV32-NEXT:    lbu a4, 3(a1)
+; CHECK-ALIGNED-RV32-NEXT:    lbu a1, 0(a1)
 ; CHECK-ALIGNED-RV32-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV32-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV32-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV32-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV32-NEXT:    lbu a4, 2(a0)
+; CHECK-ALIGNED-RV32-NEXT:    slli a3, a3, 16
+; CHECK-ALIGNED-RV32-NEXT:    slli a4, a4, 24
+; CHECK-ALIGNED-RV32-NEXT:    or a1, a2, a1
+; CHECK-ALIGNED-RV32-NEXT:    or a3, a4, a3
+; CHECK-ALIGNED-RV32-NEXT:    lbu a2, 1(a0)
+; CHECK-ALIGNED-RV32-NEXT:    lbu a4, 0(a0)
+; CHECK-ALIGNED-RV32-NEXT:    lbu a5, 2(a0)
 ; CHECK-ALIGNED-RV32-NEXT:    lbu a0, 3(a0)
-; CHECK-ALIGNED-RV32-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV32-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV32-NEXT:    slli a4, a4, 16
+; CHECK-ALIGNED-RV32-NEXT:    slli a2, a2, 8
+; CHECK-ALIGNED-RV32-NEXT:    or a2, a2, a4
+; CHECK-ALIGNED-RV32-NEXT:    slli a5, a5, 16
 ; CHECK-ALIGNED-RV32-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV32-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV32-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV32-NEXT:    or a0, a0, a3
+; CHECK-ALIGNED-RV32-NEXT:    or a0, a0, a5
+; CHECK-ALIGNED-RV32-NEXT:    or a1, a3, a1
+; CHECK-ALIGNED-RV32-NEXT:    or a0, a0, a2
 ; CHECK-ALIGNED-RV32-NEXT:    xor a0, a0, a1
 ; CHECK-ALIGNED-RV32-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV32-NEXT:    ret
@@ -4437,25 +5501,25 @@ define i1 @memcmp_eq_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV64-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV64-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV64-NEXT:    lb a1, 3(a1)
+; CHECK-ALIGNED-RV64-NEXT:    lbu a3, 2(a1)
+; CHECK-ALIGNED-RV64-NEXT:    lb a4, 3(a1)
+; CHECK-ALIGNED-RV64-NEXT:    lbu a1, 0(a1)
 ; CHECK-ALIGNED-RV64-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV64-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV64-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV64-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV64-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV64-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV64-NEXT:    lbu a4, 2(a0)
+; CHECK-ALIGNED-RV64-NEXT:    slli a3, a3, 16
+; CHECK-ALIGNED-RV64-NEXT:    slli a4, a4, 24
+; CHECK-ALIGNED-RV64-NEXT:    or a1, a2, a1
+; CHECK-ALIGNED-RV64-NEXT:    or a3, a4, a3
+; CHECK-ALIGNED-RV64-NEXT:    lbu a2, 1(a0)
+; CHECK-ALIGNED-RV64-NEXT:    lbu a4, 0(a0)
+; CHECK-ALIGNED-RV64-NEXT:    lbu a5, 2(a0)
 ; CHECK-ALIGNED-RV64-NEXT:    lb a0, 3(a0)
-; CHECK-ALIGNED-RV64-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV64-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV64-NEXT:    slli a4, a4, 16
+; CHECK-ALIGNED-RV64-NEXT:    slli a2, a2, 8
+; CHECK-ALIGNED-RV64-NEXT:    or a2, a2, a4
+; CHECK-ALIGNED-RV64-NEXT:    slli a5, a5, 16
 ; CHECK-ALIGNED-RV64-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV64-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV64-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV64-NEXT:    or a0, a0, a3
+; CHECK-ALIGNED-RV64-NEXT:    or a0, a0, a5
+; CHECK-ALIGNED-RV64-NEXT:    or a1, a3, a1
+; CHECK-ALIGNED-RV64-NEXT:    or a0, a0, a2
 ; CHECK-ALIGNED-RV64-NEXT:    xor a0, a0, a1
 ; CHECK-ALIGNED-RV64-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV64-NEXT:    ret
@@ -4463,25 +5527,25 @@ define i1 @memcmp_eq_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV32-ZBB-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a1, 3(a1)
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a3, 2(a1)
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a4, 3(a1)
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a1, 0(a1)
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a4, 2(a0)
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a3, a3, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a4, a4, 24
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a1, a2, a1
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a3, a4, a3
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a2, 1(a0)
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a4, 0(a0)
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a5, 2(a0)
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a0, 3(a0)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a4, a4, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a2, a2, 8
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a2, a2, a4
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a5, a5, 16
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a0, a0, a3
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a0, a0, a5
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a1, a3, a1
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a0, a0, a2
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
@@ -4489,25 +5553,25 @@ define i1 @memcmp_eq_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-ZBB-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lb a1, 3(a1)
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a3, 2(a1)
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    lb a4, 3(a1)
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a1, 0(a1)
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a4, 2(a0)
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a3, a3, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a4, a4, 24
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a1, a2, a1
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a3, a4, a3
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a2, 1(a0)
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a4, 0(a0)
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a5, 2(a0)
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    lb a0, 3(a0)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a4, a4, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a2, a2, 8
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a2, a2, a4
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a5, a5, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a0, a0, a3
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a0, a0, a5
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a1, a3, a1
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a0, a0, a2
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
@@ -4559,25 +5623,25 @@ define i1 @memcmp_eq_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV32-V-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV32-V-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a1, 3(a1)
+; CHECK-ALIGNED-RV32-V-NEXT:    lbu a3, 2(a1)
+; CHECK-ALIGNED-RV32-V-NEXT:    lbu a4, 3(a1)
+; CHECK-ALIGNED-RV32-V-NEXT:    lbu a1, 0(a1)
 ; CHECK-ALIGNED-RV32-V-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV32-V-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a4, 2(a0)
+; CHECK-ALIGNED-RV32-V-NEXT:    slli a3, a3, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    slli a4, a4, 24
+; CHECK-ALIGNED-RV32-V-NEXT:    or a1, a2, a1
+; CHECK-ALIGNED-RV32-V-NEXT:    or a3, a4, a3
+; CHECK-ALIGNED-RV32-V-NEXT:    lbu a2, 1(a0)
+; CHECK-ALIGNED-RV32-V-NEXT:    lbu a4, 0(a0)
+; CHECK-ALIGNED-RV32-V-NEXT:    lbu a5, 2(a0)
 ; CHECK-ALIGNED-RV32-V-NEXT:    lbu a0, 3(a0)
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV32-V-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a4, a4, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    slli a2, a2, 8
+; CHECK-ALIGNED-RV32-V-NEXT:    or a2, a2, a4
+; CHECK-ALIGNED-RV32-V-NEXT:    slli a5, a5, 16
 ; CHECK-ALIGNED-RV32-V-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV32-V-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV32-V-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV32-V-NEXT:    or a0, a0, a3
+; CHECK-ALIGNED-RV32-V-NEXT:    or a0, a0, a5
+; CHECK-ALIGNED-RV32-V-NEXT:    or a1, a3, a1
+; CHECK-ALIGNED-RV32-V-NEXT:    or a0, a0, a2
 ; CHECK-ALIGNED-RV32-V-NEXT:    xor a0, a0, a1
 ; CHECK-ALIGNED-RV32-V-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV32-V-NEXT:    ret
@@ -4585,25 +5649,25 @@ define i1 @memcmp_eq_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-V-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-V-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV64-V-NEXT:    lb a1, 3(a1)
+; CHECK-ALIGNED-RV64-V-NEXT:    lbu a3, 2(a1)
+; CHECK-ALIGNED-RV64-V-NEXT:    lb a4, 3(a1)
+; CHECK-ALIGNED-RV64-V-NEXT:    lbu a1, 0(a1)
 ; CHECK-ALIGNED-RV64-V-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV64-V-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a4, 2(a0)
+; CHECK-ALIGNED-RV64-V-NEXT:    slli a3, a3, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    slli a4, a4, 24
+; CHECK-ALIGNED-RV64-V-NEXT:    or a1, a2, a1
+; CHECK-ALIGNED-RV64-V-NEXT:    or a3, a4, a3
+; CHECK-ALIGNED-RV64-V-NEXT:    lbu a2, 1(a0)
+; CHECK-ALIGNED-RV64-V-NEXT:    lbu a4, 0(a0)
+; CHECK-ALIGNED-RV64-V-NEXT:    lbu a5, 2(a0)
 ; CHECK-ALIGNED-RV64-V-NEXT:    lb a0, 3(a0)
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV64-V-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a4, a4, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    slli a2, a2, 8
+; CHECK-ALIGNED-RV64-V-NEXT:    or a2, a2, a4
+; CHECK-ALIGNED-RV64-V-NEXT:    slli a5, a5, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV64-V-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV64-V-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV64-V-NEXT:    or a0, a0, a3
+; CHECK-ALIGNED-RV64-V-NEXT:    or a0, a0, a5
+; CHECK-ALIGNED-RV64-V-NEXT:    or a1, a3, a1
+; CHECK-ALIGNED-RV64-V-NEXT:    or a0, a0, a2
 ; CHECK-ALIGNED-RV64-V-NEXT:    xor a0, a0, a1
 ; CHECK-ALIGNED-RV64-V-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp.ll
index ddfbd649a43b..f9a6dbba04fc 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/memcmp.ll
@@ -870,13 +870,11 @@ define i32 @bcmp_size_8(ptr %s1, ptr %s2) nounwind {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_8:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
@@ -1073,18 +1071,18 @@ define i32 @bcmp_size_15(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 7(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 11(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 0(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 7(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 11(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a5
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a6
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, a7
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a3, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a4
 ; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
@@ -1284,33 +1282,21 @@ define i32 @bcmp_size_16(ptr %s1, ptr %s2) nounwind {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_16:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a4, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_16:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -1577,29 +1563,29 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 16(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 20(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 24(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 15(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 19(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 23(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 27(a0)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 16(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 20(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 24(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 15(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 19(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 23(a1)
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 27(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, a6, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, t2, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, t3, t1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a6
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t0
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, t3, t1
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a6, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t0, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, t2, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, t4, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a2, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a5, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a3, a6
 ; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a1, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
@@ -1607,18 +1593,18 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 15(a0)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 23(a0)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 0(a1)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 16(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 15(a1)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 23(a1)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a5
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a6
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, a7
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a3, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a4
 ; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
@@ -1878,57 +1864,23 @@ define i32 @bcmp_size_32(ptr %s1, ptr %s2) nounwind {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_32:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 16(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 20(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 24(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 28(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 16(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 20(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 24(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 28(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, a6, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, t2, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, t3, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a6, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t0, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 32
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v12
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_32:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 24(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a4, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 32
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v12
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -1937,15 +1889,15 @@ entry:
 }
 
 define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
-; CHECK-RV32-LABEL: bcmp_size_63:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 63
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV64-LABEL: bcmp_size_63:
 ; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
@@ -1957,6 +1909,16 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_63:
 ; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
@@ -1967,6 +1929,16 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_63:
 ; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
@@ -1977,6 +1949,16 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_63:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 63
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_63:
 ; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
@@ -1987,6 +1969,16 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-NEXT:    ld a2, 0(a0)
@@ -2023,6 +2015,16 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a2, 0(a0)
@@ -2059,6 +2061,16 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a2, 0(a0)
@@ -2095,6 +2107,98 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_63:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -48
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 44(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 40(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 36(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 32(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 28(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 20(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 16(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 8(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 4(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 16(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 20(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 24(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 28(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 20(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 24(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 28(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 0(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 4(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 8(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 12(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 47(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 51(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 55(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 59(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 31(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 35(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 39(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 43(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, t1, s2
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, a7, s0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 31(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 35(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 39(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 43(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, t3
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, t0, s1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 47(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 51(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 55(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 59(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, a6, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, s10
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, s6, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, s8, s0
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, s4, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, s9, s2
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s5, s1
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, s7, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, s3, t3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, t3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, a6, t5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, s0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t0, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, a7, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a7, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t0, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a6, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 44(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 40(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 36(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 28(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 24(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 20(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 16(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 4(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 48
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
@@ -2105,29 +2209,29 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a1)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a1)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 32(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 40(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 48(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 31(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 39(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 47(a0)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 55(a0)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 32(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 40(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 48(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 31(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 39(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 47(a1)
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 55(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, a6, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, t2, t0
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t0, t3, t1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a6
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t0
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, t3, t1
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, a5
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a6, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, t0, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, t2, a7
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, t4, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, a7
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a2, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a5, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a3, a6
 ; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a1, a4
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -2136,15 +2240,15 @@ entry:
 }
 
 define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
-; CHECK-RV32-LABEL: bcmp_size_64:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 64
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV64-LABEL: bcmp_size_64:
 ; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
@@ -2156,6 +2260,16 @@ define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_64:
 ; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
@@ -2166,6 +2280,16 @@ define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_64:
 ; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
@@ -2176,6 +2300,16 @@ define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
 ;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_64:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 64
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
 ; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_64:
 ; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
@@ -2186,6 +2320,16 @@ define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_64:
 ; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-NEXT:    ld a2, 0(a0)
@@ -2222,6 +2366,16 @@ define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_64:
 ; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a2, 0(a0)
@@ -2258,6 +2412,16 @@ define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_64:
 ; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a2, 0(a0)
@@ -2294,39 +2458,25 @@ define i32 @bcmp_size_64(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
 ;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_64:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v16
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_64:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 24(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 32(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 40(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 48(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 56(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t0
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 32(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 40(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 48(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 56(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, a6, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, t2, t0
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t0, t3, t1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, a5
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a6, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, t0, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 64
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v16
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -2335,50 +2485,580 @@ entry:
 }
 
 define i32 @bcmp_size_127(ptr %s1, ptr %s2) nounwind {
-; CHECK-RV32-LABEL: bcmp_size_127:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 127
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
 ;
-; CHECK-RV64-LABEL: bcmp_size_127:
-; CHECK-RV64:       # %bb.0: # %entry
-; CHECK-RV64-NEXT:    addi sp, sp, -16
-; CHECK-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-RV64-NEXT:    li a2, 127
-; CHECK-RV64-NEXT:    call bcmp
-; CHECK-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-RV64-NEXT:    addi sp, sp, 16
-; CHECK-RV64-NEXT:    ret
+; CHECK-ALIGNED-RV64-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_127:
+; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 127
+; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -96
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 32(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 36(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 40(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 44(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 0(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 4(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 8(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 12(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 48(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 52(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 56(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 60(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 16(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 20(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 24(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 28(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 12(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 60(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 16(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 20(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 24(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 28(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 36(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 40(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 44(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 40(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a3, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a4, s4
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 32(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 56(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 48(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a6, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 28(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 4(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a7, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 24(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t0, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 20(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 0(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t1, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 16(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 107(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t5, s10
+; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 75(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, t6, s8
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 123(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s2, s2, s4
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s0, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, t4, s11
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 83(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 87(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 91(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s1, s1, s6
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 107(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, t3, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 91(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, t2, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 123(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, a5, s3
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 75(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s5, s11, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s7, s8, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 87(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 83(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s3, s10, s3
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 115(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s6, ra, s6
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 115(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s4, s4, s11
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 119(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 119(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s10, s10, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 71(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 67(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 67(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 71(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 99(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 99(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, a5, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 103(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 103(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, a4, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, s9, s8
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, s11, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, ra, a6
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a5, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 95(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 63(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 111(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 79(a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 79(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 111(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 63(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 95(a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s11
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor s9, s9, ra
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, s8, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, a5, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t2, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t3, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t4, s9
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, s1, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t5, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, s0, a6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, s2, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, t6, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t2, t0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t1, t2, t1
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 20(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t2, s10
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t3, t3, s4
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 28(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t4, s6
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 32(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t5, t5, s3
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 36(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, t6, s7
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    or s0, s0, s5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, s0, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t5, t4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t3, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t1, t0
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a6, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a5
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t4, t6
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t0, t2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, a4
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a2
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a3
+; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 96
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_127:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, -96
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 32(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 40(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 48(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 56(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 32(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 40(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 48(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 56(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t6, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 8(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 16(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 24(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 95(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 103(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 111(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 119(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 63(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 71(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 79(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 87(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t1, t1, s2
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, a7, s0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 63(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 71(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 79(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 87(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, t3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t0, t0, s1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t4
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 95(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 103(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 111(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 119(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, a6, t6
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, t2
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, s10
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, s6, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t2, s8, s0
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t4, s4, t4
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t6, s9, s2
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor s0, s5, s1
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t5, s7, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor t3, s3, t3
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, t3
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a6, a6, t5
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, s0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or t0, t0, t6
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, a3, t4
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a7, a7, t2
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a5, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, t1, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a7, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, t0, a4
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a6, a2
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, 96
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 127)
   ret i32 %bcmp
 }
 
 define i32 @bcmp_size_128(ptr %s1, ptr %s2) nounwind {
-; CHECK-RV32-LABEL: bcmp_size_128:
-; CHECK-RV32:       # %bb.0: # %entry
-; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-RV32-NEXT:    li a2, 128
-; CHECK-RV32-NEXT:    call bcmp
-; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    ret
+; CHECK-ALIGNED-RV32-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-NEXT:    ret
 ;
-; CHECK-RV64-LABEL: bcmp_size_128:
-; CHECK-RV64:       # %bb.0: # %entry
-; CHECK-RV64-NEXT:    addi sp, sp, -16
-; CHECK-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-RV64-NEXT:    li a2, 128
-; CHECK-RV64-NEXT:    call bcmp
-; CHECK-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-RV64-NEXT:    addi sp, sp, 16
-; CHECK-RV64-NEXT:    ret
+; CHECK-ALIGNED-RV64-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV64-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBB-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBB-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-ZBKB-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-ZBKB-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-ALIGNED-RV32-V-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-ALIGNED-RV64-V-LABEL: bcmp_size_128:
+; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 128
+; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
+; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    call bcmp
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v24
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_128:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 128
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v24
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 128)
   ret i32 %bcmp
@@ -2412,7 +3092,7 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-ALIGNED-RV32-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV32-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV32-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2423,7 +3103,7 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV64-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -2434,7 +3114,7 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2445,7 +3125,7 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -2456,7 +3136,7 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2467,7 +3147,7 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -2478,7 +3158,7 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV32-V-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2489,22 +3169,130 @@ define i1 @bcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 4
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
 ; CHECK-ALIGNED-RV64-V-NEXT:    seqz a0, a0
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: bcmp_eq_zero:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a0, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lw a1, 0(a1)
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    seqz a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 4(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a0, 12(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a5, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a6, 4(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV32-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV32-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a0, 8(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a0, 12(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a5, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a6, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a1, 12(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a0, 8(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a1, 8(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a0, 12(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a5, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a6, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a1, 12(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a0, 8(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a1, 8(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_eq_zero:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV64-V-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
-  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 4)
+  %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 16)
   %ret = icmp eq i32 %bcmp, 0
   ret i1 %ret
 }
@@ -5980,213 +6768,209 @@ entry:
 define i1 @memcmp_eq_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV32:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV32-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a1, 3(a1)
-; CHECK-ALIGNED-RV32-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV32-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV32-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV32-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV32-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV32-NEXT:    lbu a4, 2(a0)
-; CHECK-ALIGNED-RV32-NEXT:    lbu a0, 3(a0)
-; CHECK-ALIGNED-RV32-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV32-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV32-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV32-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV32-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV32-NEXT:    or a0, a0, a3
-; CHECK-ALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV32-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV32-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV32-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV64-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV64:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV64-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV64-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV64-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV64-NEXT:    lb a1, 3(a1)
-; CHECK-ALIGNED-RV64-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV64-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV64-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV64-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV64-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV64-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV64-NEXT:    lbu a4, 2(a0)
-; CHECK-ALIGNED-RV64-NEXT:    lb a0, 3(a0)
-; CHECK-ALIGNED-RV64-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV64-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV64-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV64-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV64-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV64-NEXT:    or a0, a0, a3
-; CHECK-ALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV64-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV64-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV32-ZBB-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV32-ZBB:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a1, 3(a1)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a4, 2(a0)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    lbu a0, 3(a0)
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    or a0, a0, a3
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV64-ZBB-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV64-ZBB:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lb a1, 3(a1)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lbu a4, 2(a0)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    lb a0, 3(a0)
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    or a0, a0, a3
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV32-ZBKB-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV32-ZBKB:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a2, 0(a1)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a3, 1(a1)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a1, 3(a1)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a6, 2(a0)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a7, 3(a0)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lbu a0, 0(a0)
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    packh a1, a4, a1
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    packh a2, a2, a3
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    packh a3, a6, a7
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    packh a0, a0, a5
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    pack a1, a2, a1
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    pack a0, a0, a3
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV64-ZBKB-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV64-ZBKB:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lbu a2, 0(a1)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lbu a3, 1(a1)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lb a1, 3(a1)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lbu a5, 0(a0)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lbu a6, 1(a0)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    packh a2, a2, a3
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lbu a3, 2(a0)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    lb a0, 3(a0)
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    packh a5, a5, a6
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slli a3, a3, 16
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    or a0, a0, a3
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    or a0, a0, a5
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV32-V-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a1, 3(a1)
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV32-V-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a4, 2(a0)
-; CHECK-ALIGNED-RV32-V-NEXT:    lbu a0, 3(a0)
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV32-V-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV32-V-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV32-V-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV32-V-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV32-V-NEXT:    or a0, a0, a3
-; CHECK-ALIGNED-RV32-V-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV32-V-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV32-V-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-ALIGNED-RV64-V-LABEL: memcmp_eq_zero:
 ; CHECK-ALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a2, 1(a1)
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a3, 0(a1)
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a4, 2(a1)
-; CHECK-ALIGNED-RV64-V-NEXT:    lb a1, 3(a1)
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a2, a2, 8
-; CHECK-ALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a3, 0(a0)
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a5, 1(a0)
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a1, a1, 24
-; CHECK-ALIGNED-RV64-V-NEXT:    or a1, a1, a4
-; CHECK-ALIGNED-RV64-V-NEXT:    lbu a4, 2(a0)
-; CHECK-ALIGNED-RV64-V-NEXT:    lb a0, 3(a0)
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a5, a5, 8
-; CHECK-ALIGNED-RV64-V-NEXT:    or a3, a5, a3
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a4, a4, 16
-; CHECK-ALIGNED-RV64-V-NEXT:    slli a0, a0, 24
-; CHECK-ALIGNED-RV64-V-NEXT:    or a0, a0, a4
-; CHECK-ALIGNED-RV64-V-NEXT:    or a1, a1, a2
-; CHECK-ALIGNED-RV64-V-NEXT:    or a0, a0, a3
-; CHECK-ALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, -16
+; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 16
+; CHECK-ALIGNED-RV64-V-NEXT:    call memcmp
 ; CHECK-ALIGNED-RV64-V-NEXT:    seqz a0, a0
+; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: memcmp_eq_zero:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a0, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lw a1, 0(a1)
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    seqz a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 4(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a0, 12(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a5, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a6, 4(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV32-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV32-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a0, 8(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a0, 12(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a5, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a6, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a1, 12(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a0, 8(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ld a1, 8(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a4, 8(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a0, 12(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a5, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a6, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a7, 8(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a1, 12(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a5
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a3, a3, a6
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a4, a4, a7
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a4, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a0, 8(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ld a1, 8(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: memcmp_eq_zero:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV64-V-NEXT:    seqz a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
-  %memcmp = call signext i32 @memcmp(ptr %s1, ptr %s2, iXLen 4)
+  %memcmp = call signext i32 @memcmp(ptr %s1, ptr %s2, iXLen 16)
   %ret = icmp eq i32 %memcmp, 0
   ret i1 %ret
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/memset-inline.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/memset-inline.ll
index aa397504d909..12638927c907 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/memset-inline.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/memset-inline.ll
@@ -138,7 +138,7 @@ define void @memset_8(ptr %a, i8 %value) nounwind {
 ; RV64-FAST:       # %bb.0:
 ; RV64-FAST-NEXT:    zext.b a1, a1
 ; RV64-FAST-NEXT:    lui a2, 4112
-; RV64-FAST-NEXT:    addiw a2, a2, 257
+; RV64-FAST-NEXT:    addi a2, a2, 257
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
@@ -205,7 +205,7 @@ define void @memset_16(ptr %a, i8 %value) nounwind {
 ; RV64-FAST:       # %bb.0:
 ; RV64-FAST-NEXT:    zext.b a1, a1
 ; RV64-FAST-NEXT:    lui a2, 4112
-; RV64-FAST-NEXT:    addiw a2, a2, 257
+; RV64-FAST-NEXT:    addi a2, a2, 257
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
@@ -309,7 +309,7 @@ define void @memset_32(ptr %a, i8 %value) nounwind {
 ; RV64-FAST:       # %bb.0:
 ; RV64-FAST-NEXT:    zext.b a1, a1
 ; RV64-FAST-NEXT:    lui a2, 4112
-; RV64-FAST-NEXT:    addiw a2, a2, 257
+; RV64-FAST-NEXT:    addi a2, a2, 257
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
@@ -487,7 +487,7 @@ define void @memset_64(ptr %a, i8 %value) nounwind {
 ; RV64-FAST:       # %bb.0:
 ; RV64-FAST-NEXT:    zext.b a1, a1
 ; RV64-FAST-NEXT:    lui a2, 4112
-; RV64-FAST-NEXT:    addiw a2, a2, 257
+; RV64-FAST-NEXT:    addi a2, a2, 257
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
@@ -564,7 +564,7 @@ define void @aligned_memset_8(ptr align 8 %a, i8 %value) nounwind {
 ; RV64-BOTH:       # %bb.0:
 ; RV64-BOTH-NEXT:    zext.b a1, a1
 ; RV64-BOTH-NEXT:    lui a2, 4112
-; RV64-BOTH-NEXT:    addiw a2, a2, 257
+; RV64-BOTH-NEXT:    addi a2, a2, 257
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
@@ -591,7 +591,7 @@ define void @aligned_memset_16(ptr align 16 %a, i8 %value) nounwind {
 ; RV64-BOTH:       # %bb.0:
 ; RV64-BOTH-NEXT:    zext.b a1, a1
 ; RV64-BOTH-NEXT:    lui a2, 4112
-; RV64-BOTH-NEXT:    addiw a2, a2, 257
+; RV64-BOTH-NEXT:    addi a2, a2, 257
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
@@ -623,7 +623,7 @@ define void @aligned_memset_32(ptr align 32 %a, i8 %value) nounwind {
 ; RV64-BOTH:       # %bb.0:
 ; RV64-BOTH-NEXT:    zext.b a1, a1
 ; RV64-BOTH-NEXT:    lui a2, 4112
-; RV64-BOTH-NEXT:    addiw a2, a2, 257
+; RV64-BOTH-NEXT:    addi a2, a2, 257
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
@@ -665,7 +665,7 @@ define void @aligned_memset_64(ptr align 64 %a, i8 %value) nounwind {
 ; RV64-BOTH:       # %bb.0:
 ; RV64-BOTH-NEXT:    zext.b a1, a1
 ; RV64-BOTH-NEXT:    lui a2, 4112
-; RV64-BOTH-NEXT:    addiw a2, a2, 257
+; RV64-BOTH-NEXT:    addi a2, a2, 257
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/narrow-shl-cst.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/narrow-shl-cst.ll
index 64d2caabede0..f709c8d932f5 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/narrow-shl-cst.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/narrow-shl-cst.ll
@@ -246,7 +246,7 @@ define signext i32 @test14(i32 signext %x) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    slliw a0, a0, 10
 ; RV64-NEXT:    lui a1, 8
-; RV64-NEXT:    addiw a1, a1, -1027
+; RV64-NEXT:    addi a1, a1, -1027
 ; RV64-NEXT:    or a0, a0, a1
 ; RV64-NEXT:    ret
   %or = shl i32 %x, 10
@@ -269,7 +269,7 @@ define signext i32 @test15(i32 signext %x) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    slliw a0, a0, 10
 ; RV64-NEXT:    lui a1, 8
-; RV64-NEXT:    addiw a1, a1, -515
+; RV64-NEXT:    addi a1, a1, -515
 ; RV64-NEXT:    xor a0, a0, a1
 ; RV64-NEXT:    ret
   %xor = shl i32 %x, 10
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/nest-register.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/nest-register.ll
index e222beee4578..6e892e05c429 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/nest-register.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/nest-register.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zicfilp -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I-ZICFILP %s
+; RUN: not llc -mtriple=riscv64 -target-abi=lp64e -mattr=+experimental-zicfilp \
+; RUN:   -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=LP64E-ZICFILP %s
 
 ; Tests that the 'nest' parameter attribute causes the relevant parameter to be
 ; passed in the right register.
@@ -17,6 +21,12 @@ define ptr @nest_receiver(ptr nest %arg) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    mv a0, t2
 ; RV64I-NEXT:    ret
+;
+; RV64I-ZICFILP-LABEL: nest_receiver:
+; RV64I-ZICFILP:       # %bb.0:
+; RV64I-ZICFILP-NEXT:    lpad 0
+; RV64I-ZICFILP-NEXT:    mv a0, t3
+; RV64I-ZICFILP-NEXT:    ret
   ret ptr %arg
 }
 
@@ -40,6 +50,22 @@ define ptr @nest_caller(ptr %arg) nounwind {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; RV64I-ZICFILP-LABEL: nest_caller:
+; RV64I-ZICFILP:       # %bb.0:
+; RV64I-ZICFILP-NEXT:    lpad 0
+; RV64I-ZICFILP-NEXT:    addi sp, sp, -16
+; RV64I-ZICFILP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-ZICFILP-NEXT:    mv t3, a0
+; RV64I-ZICFILP-NEXT:    call nest_receiver
+; RV64I-ZICFILP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-ZICFILP-NEXT:    addi sp, sp, 16
+; RV64I-ZICFILP-NEXT:    ret
   %result = call ptr @nest_receiver(ptr nest %arg)
   ret ptr %result
 }
+
+; LP64E-ZICFILP: LLVM ERROR: Nested functions with control flow protection are not usable with ILP32E or LP64E ABI.
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 8, !"cf-protection-branch", i32 1}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/opt-w-instrs.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
index 7d0e483c17ae..b4ea9260297f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
@@ -169,3 +169,101 @@ body:             |
     PseudoRET implicit $x10
 
 ...
+---
+name:            lui_addi_1
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: lui_addi_1
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x10
+  ; CHECK-NEXT:   [[ADDIW:%[0-9]+]]:gpr = ADDIW [[COPY]], 0
+  ; CHECK-NEXT:   [[LUI:%[0-9]+]]:gpr = LUI 11
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI killed [[LUI]], -1756
+  ; CHECK-NEXT:   BEQ killed [[ADDIW]], [[ADDI]], %bb.2
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LW:%[0-9]+]]:gpr = LW $x0, 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[ADDI]], %bb.0, [[LW]], %bb.1
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gprtc = COPY $x0
+  ; CHECK-NEXT:   $x10 = COPY [[COPY1]]
+  ; CHECK-NEXT:   $x11 = COPY [[PHI]]
+  ; CHECK-NEXT:   PseudoTAILIndirect [[COPY1]], csr_ilp32_lp64, implicit $x2, implicit $x10, implicit $x11
+  bb.0:
+    liveins: $x10
+
+    %2:gpr = COPY $x10
+    %4:gpr = ADDIW %2, 0
+    %5:gpr = LUI 11
+    %3:gpr = ADDI killed %5, -1756
+    BEQ killed %4, %3, %bb.2
+    PseudoBR %bb.1
+
+  bb.1:
+    %0:gpr = LW $x0, 0
+
+  bb.2:
+    %1:gpr = PHI %3, %bb.0, %0, %bb.1
+    %6:gpr = ADDIW %1, 0
+    %7:gprtc = COPY $x0
+    $x10 = COPY %7
+    $x11 = COPY %6
+    PseudoTAILIndirect %7, csr_ilp32_lp64, implicit $x2, implicit $x10, implicit $x11
+...
+---
+# The sext.w was erroneously removed in an earlier broken version of the patch
+# for RISCVOptWInstrs that added lui/addi handling.
+name:            lui_addi_2
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: lui_addi_2
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LUI:%[0-9]+]]:gpr = LUI 524289
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI [[LUI]], -2048
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   [[LWU:%[0-9]+]]:gpr = LWU $x0, 0 :: (load (s32) from `ptr null`)
+  ; CHECK-NEXT:   [[AND:%[0-9]+]]:gpr = AND [[LWU]], [[ADDI]]
+  ; CHECK-NEXT:   [[ADDIW:%[0-9]+]]:gpr = ADDIW killed [[AND]], 0
+  ; CHECK-NEXT:   $x10 = COPY [[ADDIW]]
+  ; CHECK-NEXT:   PseudoRET implicit $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; CHECK-NEXT:   $x10 = COPY [[ADDIW]]
+  ; CHECK-NEXT:   %call_target_dummy:gprjalr = COPY $x0
+  ; CHECK-NEXT:   PseudoCALLIndirect %call_target_dummy, csr_ilp32_lp64, implicit-def $x1, implicit $x10
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+  ; CHECK-NEXT:   $x10 = COPY [[LWU]]
+  ; CHECK-NEXT:   PseudoRET implicit $x10
+  bb.0.entry:
+    %13:gpr = LUI 524289
+    %14:gpr = ADDI %13, -2048
+
+  bb.1:
+    %1:gpr = LWU $x0, 0 :: (load (s32) from `ptr null`)
+    %15:gpr = AND %1, %14
+    %16:gpr = ADDIW killed %15, 0
+
+    $x10 = COPY %16
+    PseudoRET implicit $x10
+
+  bb.2:
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %16
+    %call_target_dummy:gprjalr = COPY $x0
+    PseudoCALLIndirect %call_target_dummy, csr_ilp32_lp64, implicit-def $x1, implicit $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir
index a3a1818993f0..5f6f3bf818d6 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir
@@ -28,18 +28,18 @@
   ; CHECK-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
   ; CHECK-NEXT:    sd s0, 2016(sp) # 8-byte Folded Spill
   ; CHECK-NEXT:    addi s0, sp, 2032
-  ; CHECK-NEXT:    sd a0, 0(sp)
+  ; CHECK-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
   ; CHECK-NEXT:    lui a0, 2
-  ; CHECK-NEXT:    addiw a0, a0, -2032
+  ; CHECK-NEXT:    addi a0, a0, -2032
   ; CHECK-NEXT:    sub sp, sp, a0
   ; CHECK-NEXT:    srli a0, sp, 12
   ; CHECK-NEXT:    slli sp, a0, 12
-  ; CHECK-NEXT:    ld a0, 0(sp)
-  ; CHECK-NEXT:    sd a1, 0(sp)
+  ; CHECK-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
+  ; CHECK-NEXT:    sd a1, 0(sp) # 8-byte Folded Spill
   ; CHECK-NEXT:    lui a1, 1
   ; CHECK-NEXT:    add a1, sp, a1
   ; CHECK-NEXT:    sd a0, -8(a1)
-  ; CHECK-NEXT:    ld a1, 0(sp)
+  ; CHECK-NEXT:    ld a1, 0(sp) # 8-byte Folded Reload
   ; CHECK-NEXT:    call foo
   ; CHECK-NEXT:    addi sp, s0, -2032
   ; CHECK-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
index a5426e560bd6..98c897084ab4 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
@@ -1330,7 +1330,7 @@ define i16 @overflow_not_used(i16 %a, i16 %b, ptr %res) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a3, 16
 ; RV64-NEXT:    add a0, a1, a0
-; RV64-NEXT:    addiw a3, a3, -1
+; RV64-NEXT:    addi a3, a3, -1
 ; RV64-NEXT:    and a4, a1, a3
 ; RV64-NEXT:    and a3, a0, a3
 ; RV64-NEXT:    bltu a3, a4, .LBB38_2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr135206.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr135206.ll
index 196e78d8ed8b..75b11c373895 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr135206.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr135206.ll
@@ -42,10 +42,10 @@ define i1 @foo() nounwind "probe-stack"="inline-asm" "target-features"="+v" {
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.x v8, a0
 ; CHECK-NEXT:    lui a0, 8
-; CHECK-NEXT:    addiw a0, a0, 32
+; CHECK-NEXT:    addi a0, a0, 32
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; CHECK-NEXT:    addiw a0, a1, 1622
+; CHECK-NEXT:    addi a0, a1, 1622
 ; CHECK-NEXT:    vse8.v v8, (s0)
 ; CHECK-NEXT:    vse8.v v8, (s1)
 ; CHECK-NEXT:    vse8.v v8, (s2)
@@ -54,7 +54,7 @@ define i1 @foo() nounwind "probe-stack"="inline-asm" "target-features"="+v" {
 ; CHECK-NEXT:    sd s3, 64(sp)
 ; CHECK-NEXT:    call bar
 ; CHECK-NEXT:    lui a0, 8
-; CHECK-NEXT:    addiw a0, a0, 32
+; CHECK-NEXT:    addi a0, a0, 32
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
@@ -66,7 +66,7 @@ define i1 @foo() nounwind "probe-stack"="inline-asm" "target-features"="+v" {
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    add sp, sp, a1
 ; CHECK-NEXT:    lui a1, 8
-; CHECK-NEXT:    addiw a1, a1, -1952
+; CHECK-NEXT:    addi a1, a1, -1952
 ; CHECK-NEXT:    add sp, sp, a1
 ; CHECK-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr56457.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr56457.ll
index cf518b31a190..0dca85808916 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr56457.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr56457.ll
@@ -14,9 +14,9 @@ define i15 @foo(i15 %x) nounwind {
 ; CHECK-NEXT:    lui a3, 209715
 ; CHECK-NEXT:    lui a4, 61681
 ; CHECK-NEXT:    or a0, a0, a1
-; CHECK-NEXT:    addiw a1, a2, 1365
-; CHECK-NEXT:    addiw a2, a3, 819
-; CHECK-NEXT:    addiw a3, a4, -241
+; CHECK-NEXT:    addi a1, a2, 1365
+; CHECK-NEXT:    addi a2, a3, 819
+; CHECK-NEXT:    addi a3, a4, -241
 ; CHECK-NEXT:    slli a4, a2, 32
 ; CHECK-NEXT:    add a2, a2, a4
 ; CHECK-NEXT:    slli a4, a3, 32
@@ -43,7 +43,7 @@ define i15 @foo(i15 %x) nounwind {
 ; CHECK-NEXT:    srli a1, a0, 4
 ; CHECK-NEXT:    add a0, a0, a1
 ; CHECK-NEXT:    lui a1, 4112
-; CHECK-NEXT:    addiw a1, a1, 257
+; CHECK-NEXT:    addi a1, a1, 257
 ; CHECK-NEXT:    and a0, a0, a3
 ; CHECK-NEXT:    slli a2, a1, 32
 ; CHECK-NEXT:    add a1, a1, a2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr58286.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr58286.ll
index 5e146b4b12ca..ec53646aca62 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr58286.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr58286.ll
@@ -7,7 +7,7 @@ define void @func() {
 ; RV64I-LABEL: func:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 4112
 ; RV64I-NEXT:    lui a0, %hi(var)
@@ -25,11 +25,11 @@ define void @func() {
 ; RV64I-NEXT:    lw t4, %lo(var)(a0)
 ; RV64I-NEXT:    lw t5, %lo(var)(a0)
 ; RV64I-NEXT:    lw t6, %lo(var)(a0)
-; RV64I-NEXT:    sd s0, 0(sp)
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui s0, 1
 ; RV64I-NEXT:    add s0, sp, s0
 ; RV64I-NEXT:    sw a1, 12(s0)
-; RV64I-NEXT:    ld s0, 0(sp)
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a1, %lo(var)(a0)
 ; RV64I-NEXT:    sw a2, %lo(var)(a0)
 ; RV64I-NEXT:    sw a3, %lo(var)(a0)
@@ -45,7 +45,7 @@ define void @func() {
 ; RV64I-NEXT:    sw t5, %lo(var)(a0)
 ; RV64I-NEXT:    sw t6, %lo(var)(a0)
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
@@ -71,11 +71,11 @@ define void @func() {
 ; RV32I-NEXT:    lw t4, %lo(var)(a0)
 ; RV32I-NEXT:    lw t5, %lo(var)(a0)
 ; RV32I-NEXT:    lw t6, %lo(var)(a0)
-; RV32I-NEXT:    sw s0, 0(sp)
+; RV32I-NEXT:    sw s0, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui s0, 1
 ; RV32I-NEXT:    add s0, sp, s0
 ; RV32I-NEXT:    sw a1, 12(s0)
-; RV32I-NEXT:    lw s0, 0(sp)
+; RV32I-NEXT:    lw s0, 0(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a1, %lo(var)(a0)
 ; RV32I-NEXT:    sw a2, %lo(var)(a0)
 ; RV32I-NEXT:    sw a3, %lo(var)(a0)
@@ -142,7 +142,7 @@ define void @shrink_wrap(i1 %c) {
 ; RV64I-NEXT:    bnez a0, .LBB1_2
 ; RV64I-NEXT:  # %bb.1: # %bar
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 4112
 ; RV64I-NEXT:    lui a0, %hi(var)
@@ -160,11 +160,11 @@ define void @shrink_wrap(i1 %c) {
 ; RV64I-NEXT:    lw t4, %lo(var)(a0)
 ; RV64I-NEXT:    lw t5, %lo(var)(a0)
 ; RV64I-NEXT:    lw t6, %lo(var)(a0)
-; RV64I-NEXT:    sd s0, 0(sp)
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui s0, 1
 ; RV64I-NEXT:    add s0, sp, s0
 ; RV64I-NEXT:    sw a1, 12(s0)
-; RV64I-NEXT:    ld s0, 0(sp)
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a1, %lo(var)(a0)
 ; RV64I-NEXT:    sw a2, %lo(var)(a0)
 ; RV64I-NEXT:    sw a3, %lo(var)(a0)
@@ -180,7 +180,7 @@ define void @shrink_wrap(i1 %c) {
 ; RV64I-NEXT:    sw t5, %lo(var)(a0)
 ; RV64I-NEXT:    sw t6, %lo(var)(a0)
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:  .LBB1_2: # %foo
@@ -210,11 +210,11 @@ define void @shrink_wrap(i1 %c) {
 ; RV32I-NEXT:    lw t4, %lo(var)(a0)
 ; RV32I-NEXT:    lw t5, %lo(var)(a0)
 ; RV32I-NEXT:    lw t6, %lo(var)(a0)
-; RV32I-NEXT:    sw s0, 0(sp)
+; RV32I-NEXT:    sw s0, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui s0, 1
 ; RV32I-NEXT:    add s0, sp, s0
 ; RV32I-NEXT:    sw a1, 12(s0)
-; RV32I-NEXT:    lw s0, 0(sp)
+; RV32I-NEXT:    lw s0, 0(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a1, %lo(var)(a0)
 ; RV32I-NEXT:    sw a2, %lo(var)(a0)
 ; RV32I-NEXT:    sw a3, %lo(var)(a0)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr58511.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr58511.ll
index e5cba679729f..a93fef20f5fe 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr58511.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr58511.ll
@@ -7,7 +7,7 @@ define i32 @f(i1 %0, i32 %1, ptr %2) {
 ; CHECK-NEXT:    slli a0, a0, 63
 ; CHECK-NEXT:    lui a3, 4097
 ; CHECK-NEXT:    srai a0, a0, 63
-; CHECK-NEXT:    addiw a3, a3, -2047
+; CHECK-NEXT:    addi a3, a3, -2047
 ; CHECK-NEXT:    or a0, a0, a3
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sw a1, 0(a2)
@@ -26,7 +26,7 @@ define i32 @g(i1 %0, i32 %1, ptr %2) {
 ; CHECK-NEXT:    andi a0, a0, 1
 ; CHECK-NEXT:    lui a3, 4097
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    addiw a3, a3, -2047
+; CHECK-NEXT:    addi a3, a3, -2047
 ; CHECK-NEXT:    or a0, a0, a3
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sw a1, 0(a2)
@@ -44,7 +44,7 @@ define i32 @h(i1 %0, i32 %1, ptr %2) {
 ; CHECK:       # %bb.0: # %BB
 ; CHECK-NEXT:    lui a3, 4097
 ; CHECK-NEXT:    slli a0, a0, 63
-; CHECK-NEXT:    addiw a3, a3, -2047
+; CHECK-NEXT:    addi a3, a3, -2047
 ; CHECK-NEXT:    srai a0, a0, 63
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    and a0, a0, a3
@@ -63,7 +63,7 @@ define i32 @i(i1 %0, i32 %1, ptr %2) {
 ; CHECK:       # %bb.0: # %BB
 ; CHECK-NEXT:    andi a0, a0, 1
 ; CHECK-NEXT:    lui a3, 4097
-; CHECK-NEXT:    addiw a3, a3, -2047
+; CHECK-NEXT:    addi a3, a3, -2047
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    and a0, a0, a3
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr68855.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr68855.ll
index 8031bf4f3041..90d4be4d9a0e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr68855.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr68855.ll
@@ -7,7 +7,7 @@ define i16 @narrow_load(ptr %p1, ptr %p2) {
 ; CHECK-NEXT:    lhu a2, 0(a0)
 ; CHECK-NEXT:    lui a3, 2
 ; CHECK-NEXT:    lui a4, 16
-; CHECK-NEXT:    addiw a3, a3, -1
+; CHECK-NEXT:    addi a3, a3, -1
 ; CHECK-NEXT:    addi a4, a4, -1
 ; CHECK-NEXT:    xor a2, a2, a3
 ; CHECK-NEXT:    xor a4, a3, a4
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr69586.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr69586.ll
index 4ab48930ae78..e761d3a9a75c 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr69586.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr69586.ll
@@ -253,7 +253,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v4, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v26, v8
 ; NOREMAT-NEXT:    lui a4, 4
-; NOREMAT-NEXT:    addiw a0, a4, 512
+; NOREMAT-NEXT:    addi a0, a4, 512
 ; NOREMAT-NEXT:    sd a0, 496(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a0, a7, a0
 ; NOREMAT-NEXT:    vle32.v v8, (a0)
@@ -265,7 +265,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v28, (a2)
 ; NOREMAT-NEXT:    vle32.v v6, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v30, v12
-; NOREMAT-NEXT:    addiw a2, a4, 1536
+; NOREMAT-NEXT:    addi a2, a4, 1536
 ; NOREMAT-NEXT:    sd a2, 480(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v12, (a2)
@@ -278,7 +278,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v4, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v22, v8
 ; NOREMAT-NEXT:    lui a5, 5
-; NOREMAT-NEXT:    addiw a2, a5, -1536
+; NOREMAT-NEXT:    addi a2, a5, -1536
 ; NOREMAT-NEXT:    sd a2, 464(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v8, (a2)
@@ -291,13 +291,13 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v26, (a2)
 ; NOREMAT-NEXT:    vle32.v v28, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v6, v12
-; NOREMAT-NEXT:    addiw a2, a5, -512
+; NOREMAT-NEXT:    addi a2, a5, -512
 ; NOREMAT-NEXT:    sd a2, 448(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v12, (a2)
 ; NOREMAT-NEXT:    vle32.v v6, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v30, v24
-; NOREMAT-NEXT:    addiw a2, a5, 512
+; NOREMAT-NEXT:    addi a2, a5, 512
 ; NOREMAT-NEXT:    sd a2, 440(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v24, (a2)
@@ -309,7 +309,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v8, (a2)
 ; NOREMAT-NEXT:    vle32.v v4, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v22, v26
-; NOREMAT-NEXT:    addiw a2, a5, 1536
+; NOREMAT-NEXT:    addi a2, a5, 1536
 ; NOREMAT-NEXT:    sd a2, 424(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v22, (a2)
@@ -322,7 +322,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v28, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v6, v18
 ; NOREMAT-NEXT:    lui a6, 6
-; NOREMAT-NEXT:    addiw a2, a6, -1536
+; NOREMAT-NEXT:    addi a2, a6, -1536
 ; NOREMAT-NEXT:    sd a2, 408(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v18, (a2)
@@ -334,13 +334,13 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v16, (a2)
 ; NOREMAT-NEXT:    vle32.v v24, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v30, v8
-; NOREMAT-NEXT:    addiw a2, a6, -512
+; NOREMAT-NEXT:    addi a2, a6, -512
 ; NOREMAT-NEXT:    sd a2, 392(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v8, (a2)
 ; NOREMAT-NEXT:    vle32.v v30, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v4, v22
-; NOREMAT-NEXT:    addiw a2, a6, 512
+; NOREMAT-NEXT:    addi a2, a6, 512
 ; NOREMAT-NEXT:    sd a2, 384(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v22, (a2)
@@ -352,7 +352,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v26, (a2)
 ; NOREMAT-NEXT:    vle32.v v2, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v28, v18
-; NOREMAT-NEXT:    addiw a2, a6, 1536
+; NOREMAT-NEXT:    addi a2, a6, 1536
 ; NOREMAT-NEXT:    sd a2, 368(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v18, (a2)
@@ -365,7 +365,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v6, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v24, v8
 ; NOREMAT-NEXT:    lui s0, 7
-; NOREMAT-NEXT:    addiw a2, s0, -1536
+; NOREMAT-NEXT:    addi a2, s0, -1536
 ; NOREMAT-NEXT:    sd a2, 352(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v8, (a2)
@@ -379,13 +379,13 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    addi a0, sp, 640
 ; NOREMAT-NEXT:    vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v12, v22
-; NOREMAT-NEXT:    addiw a2, s0, -512
+; NOREMAT-NEXT:    addi a2, s0, -512
 ; NOREMAT-NEXT:    sd a2, 336(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v22, (a2)
 ; NOREMAT-NEXT:    vle32.v v12, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v4, v26
-; NOREMAT-NEXT:    addiw a2, s0, 512
+; NOREMAT-NEXT:    addi a2, s0, 512
 ; NOREMAT-NEXT:    sd a2, 328(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    lui t3, 7
 ; NOREMAT-NEXT:    add a2, a7, a2
@@ -398,7 +398,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v18, (a2)
 ; NOREMAT-NEXT:    vle32.v v2, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v28, v16
-; NOREMAT-NEXT:    addiw a2, t3, 1536
+; NOREMAT-NEXT:    addi a2, t3, 1536
 ; NOREMAT-NEXT:    sd a2, 312(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v16, (a2)
@@ -410,7 +410,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    vle32.v v8, (a2)
 ; NOREMAT-NEXT:    vle32.v v6, (a2)
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v24, v14
-; NOREMAT-NEXT:    addiw a2, t4, -1536
+; NOREMAT-NEXT:    addi a2, t4, -1536
 ; NOREMAT-NEXT:    sd a2, 296(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v14, (a2)
@@ -421,7 +421,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    add a2, a7, a2
 ; NOREMAT-NEXT:    vle32.v v22, (a2)
 ; NOREMAT-NEXT:    vle32.v v30, (a2)
-; NOREMAT-NEXT:    addiw a0, t4, -512
+; NOREMAT-NEXT:    addi a0, t4, -512
 ; NOREMAT-NEXT:    sd a0, 280(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a0, a7, a0
 ; NOREMAT-NEXT:    sf.vc.vv 3, 0, v12, v0
@@ -456,34 +456,34 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; NOREMAT-NEXT:    sd t3, 224(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a0, a1, t4
 ; NOREMAT-NEXT:    sd a0, 216(sp) # 8-byte Folded Spill
-; NOREMAT-NEXT:    addiw a0, t4, 512
+; NOREMAT-NEXT:    addi a0, t4, 512
 ; NOREMAT-NEXT:    sd a0, 192(sp) # 8-byte Folded Spill
-; NOREMAT-NEXT:    addiw a0, t4, 1024
+; NOREMAT-NEXT:    addi a0, t4, 1024
 ; NOREMAT-NEXT:    sd a0, 176(sp) # 8-byte Folded Spill
-; NOREMAT-NEXT:    addiw a0, t4, 1536
+; NOREMAT-NEXT:    addi a0, t4, 1536
 ; NOREMAT-NEXT:    sd a0, 160(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    slli s1, s1, 11
 ; NOREMAT-NEXT:    sd s1, 128(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    lui a0, 9
-; NOREMAT-NEXT:    addiw a2, a0, -1536
+; NOREMAT-NEXT:    addi a2, a0, -1536
 ; NOREMAT-NEXT:    sd a2, 88(sp) # 8-byte Folded Spill
-; NOREMAT-NEXT:    addiw a2, a0, -1024
+; NOREMAT-NEXT:    addi a2, a0, -1024
 ; NOREMAT-NEXT:    sd a2, 72(sp) # 8-byte Folded Spill
-; NOREMAT-NEXT:    addiw a2, a0, -512
+; NOREMAT-NEXT:    addi a2, a0, -512
 ; NOREMAT-NEXT:    sd a2, 40(sp) # 8-byte Folded Spill
 ; NOREMAT-NEXT:    add a2, a1, a0
 ; NOREMAT-NEXT:    sd a2, 208(sp) # 8-byte Folded Spill
-; NOREMAT-NEXT:    addiw s11, a0, 512
-; NOREMAT-NEXT:    addiw s7, a0, 1024
-; NOREMAT-NEXT:    addiw s3, a0, 1536
+; NOREMAT-NEXT:    addi s11, a0, 512
+; NOREMAT-NEXT:    addi s7, a0, 1024
+; NOREMAT-NEXT:    addi s3, a0, 1536
 ; NOREMAT-NEXT:    slli s1, t2, 11
 ; NOREMAT-NEXT:    lui a0, 10
-; NOREMAT-NEXT:    addiw t2, a0, -1536
-; NOREMAT-NEXT:    addiw a7, a0, -1024
-; NOREMAT-NEXT:    addiw a4, a0, -512
+; NOREMAT-NEXT:    addi t2, a0, -1536
+; NOREMAT-NEXT:    addi a7, a0, -1024
+; NOREMAT-NEXT:    addi a4, a0, -512
 ; NOREMAT-NEXT:    add a2, a1, a0
 ; NOREMAT-NEXT:    sd a2, 200(sp) # 8-byte Folded Spill
-; NOREMAT-NEXT:    addiw a0, a0, 512
+; NOREMAT-NEXT:    addi a0, a0, 512
 ; NOREMAT-NEXT:    ld a2, 512(sp) # 8-byte Folded Reload
 ; NOREMAT-NEXT:    add a2, a1, a2
 ; NOREMAT-NEXT:    ld a3, 504(sp) # 8-byte Folded Reload
@@ -1195,7 +1195,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    addi a2, a2, 432
 ; REMAT-NEXT:    vs2r.v v8, (a2) # vscale x 16-byte Folded Spill
 ; REMAT-NEXT:    lui a2, 4
-; REMAT-NEXT:    addiw a2, a2, 512
+; REMAT-NEXT:    addi a2, a2, 512
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v2, (a2)
 ; REMAT-NEXT:    csrr a3, vlenb
@@ -1217,7 +1217,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v8, v14
 ; REMAT-NEXT:    vle32.v v22, (a2)
 ; REMAT-NEXT:    lui a2, 4
-; REMAT-NEXT:    addiw a2, a2, 1536
+; REMAT-NEXT:    addi a2, a2, 1536
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v24, (a2)
 ; REMAT-NEXT:    csrr a3, vlenb
@@ -1242,7 +1242,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v10, v18
 ; REMAT-NEXT:    vle32.v v10, (a2)
 ; REMAT-NEXT:    lui a2, 5
-; REMAT-NEXT:    addiw a2, a2, -1536
+; REMAT-NEXT:    addi a2, a2, -1536
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v28, (a2)
 ; REMAT-NEXT:    csrr a3, vlenb
@@ -1266,7 +1266,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v14, v6
 ; REMAT-NEXT:    vle32.v v14, (a2)
 ; REMAT-NEXT:    lui a2, 5
-; REMAT-NEXT:    addiw a2, a2, -512
+; REMAT-NEXT:    addi a2, a2, -512
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v6, (a2)
 ; REMAT-NEXT:    csrr a3, vlenb
@@ -1288,7 +1288,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v18, v2
 ; REMAT-NEXT:    vle32.v v18, (a2)
 ; REMAT-NEXT:    lui a2, 5
-; REMAT-NEXT:    addiw a2, a2, 512
+; REMAT-NEXT:    addi a2, a2, 512
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v2, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v20, v0
@@ -1300,7 +1300,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v22, v24
 ; REMAT-NEXT:    vle32.v v22, (a2)
 ; REMAT-NEXT:    lui s4, 5
-; REMAT-NEXT:    addiw s4, s4, 1536
+; REMAT-NEXT:    addi s4, s4, 1536
 ; REMAT-NEXT:    add a2, a0, s4
 ; REMAT-NEXT:    vle32.v v24, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v8, v26
@@ -1312,7 +1312,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v10, v28
 ; REMAT-NEXT:    vle32.v v10, (a2)
 ; REMAT-NEXT:    lui s3, 6
-; REMAT-NEXT:    addiw s3, s3, -1536
+; REMAT-NEXT:    addi s3, s3, -1536
 ; REMAT-NEXT:    add a2, a0, s3
 ; REMAT-NEXT:    vle32.v v28, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v12, v30
@@ -1324,7 +1324,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v14, v6
 ; REMAT-NEXT:    vle32.v v14, (a2)
 ; REMAT-NEXT:    lui a2, 6
-; REMAT-NEXT:    addiw a2, a2, -512
+; REMAT-NEXT:    addi a2, a2, -512
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v6, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v16, v4
@@ -1336,7 +1336,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v18, v2
 ; REMAT-NEXT:    vle32.v v18, (a2)
 ; REMAT-NEXT:    lui s0, 6
-; REMAT-NEXT:    addiw s0, s0, 512
+; REMAT-NEXT:    addi s0, s0, 512
 ; REMAT-NEXT:    add a2, a0, s0
 ; REMAT-NEXT:    vle32.v v2, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v20, v0
@@ -1348,7 +1348,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v22, v24
 ; REMAT-NEXT:    vle32.v v22, (a2)
 ; REMAT-NEXT:    lui t6, 6
-; REMAT-NEXT:    addiw t6, t6, 1536
+; REMAT-NEXT:    addi t6, t6, 1536
 ; REMAT-NEXT:    add a2, a0, t6
 ; REMAT-NEXT:    vle32.v v24, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v8, v26
@@ -1360,7 +1360,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v10, v28
 ; REMAT-NEXT:    vle32.v v10, (a2)
 ; REMAT-NEXT:    lui a2, 7
-; REMAT-NEXT:    addiw a2, a2, -1536
+; REMAT-NEXT:    addi a2, a2, -1536
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v28, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v12, v30
@@ -1372,7 +1372,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v14, v6
 ; REMAT-NEXT:    vle32.v v14, (a2)
 ; REMAT-NEXT:    lui a2, 7
-; REMAT-NEXT:    addiw a2, a2, -512
+; REMAT-NEXT:    addi a2, a2, -512
 ; REMAT-NEXT:    add a2, a0, a2
 ; REMAT-NEXT:    vle32.v v6, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v16, v4
@@ -1384,7 +1384,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v18, v2
 ; REMAT-NEXT:    vle32.v v18, (a2)
 ; REMAT-NEXT:    lui t2, 7
-; REMAT-NEXT:    addiw t2, t2, 512
+; REMAT-NEXT:    addi t2, t2, 512
 ; REMAT-NEXT:    add a2, a0, t2
 ; REMAT-NEXT:    vle32.v v2, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v20, v0
@@ -1396,7 +1396,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v22, v24
 ; REMAT-NEXT:    vle32.v v22, (a2)
 ; REMAT-NEXT:    lui t0, 7
-; REMAT-NEXT:    addiw t0, t0, 1536
+; REMAT-NEXT:    addi t0, t0, 1536
 ; REMAT-NEXT:    add a2, a0, t0
 ; REMAT-NEXT:    vle32.v v24, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v8, v26
@@ -1408,7 +1408,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v10, v28
 ; REMAT-NEXT:    vle32.v v10, (a2)
 ; REMAT-NEXT:    lui a6, 8
-; REMAT-NEXT:    addiw a6, a6, -1536
+; REMAT-NEXT:    addi a6, a6, -1536
 ; REMAT-NEXT:    add a2, a0, a6
 ; REMAT-NEXT:    vle32.v v28, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v12, v30
@@ -1420,7 +1420,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v14, v6
 ; REMAT-NEXT:    vle32.v v14, (a2)
 ; REMAT-NEXT:    lui a3, 8
-; REMAT-NEXT:    addiw a3, a3, -512
+; REMAT-NEXT:    addi a3, a3, -512
 ; REMAT-NEXT:    add a2, a0, a3
 ; REMAT-NEXT:    vle32.v v6, (a2)
 ; REMAT-NEXT:    sf.vc.vv 3, 0, v16, v4
@@ -1537,7 +1537,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 192(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 4
-; REMAT-NEXT:    addiw a0, a0, 512
+; REMAT-NEXT:    addi a0, a0, 512
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 184(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    li a0, 17
@@ -1545,7 +1545,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 176(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 4
-; REMAT-NEXT:    addiw a0, a0, 1536
+; REMAT-NEXT:    addi a0, a0, 1536
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 168(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    li a0, 9
@@ -1553,7 +1553,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 160(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 5
-; REMAT-NEXT:    addiw a0, a0, -1536
+; REMAT-NEXT:    addi a0, a0, -1536
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 152(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    li a0, 19
@@ -1561,14 +1561,14 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 144(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 5
-; REMAT-NEXT:    addiw a0, a0, -512
+; REMAT-NEXT:    addi a0, a0, -512
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 136(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 5
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 128(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 5
-; REMAT-NEXT:    addiw a0, a0, 512
+; REMAT-NEXT:    addi a0, a0, 512
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 120(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    add s7, a1, s7
@@ -1584,7 +1584,7 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    add s2, a1, s2
 ; REMAT-NEXT:    sd s2, 80(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 6
-; REMAT-NEXT:    addiw a0, a0, -512
+; REMAT-NEXT:    addi a0, a0, -512
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 72(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    add s1, a1, s1
@@ -1600,13 +1600,13 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    add t5, a1, t5
 ; REMAT-NEXT:    sd t5, 32(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui a0, 7
-; REMAT-NEXT:    addiw a0, a0, -1536
+; REMAT-NEXT:    addi a0, a0, -1536
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    add t4, a1, t4
 ; REMAT-NEXT:    sd t4, 16(sp) # 8-byte Folded Spill
 ; REMAT-NEXT:    lui ra, 7
-; REMAT-NEXT:    addiw ra, ra, -512
+; REMAT-NEXT:    addi ra, ra, -512
 ; REMAT-NEXT:    add ra, a1, ra
 ; REMAT-NEXT:    add s11, a1, t3
 ; REMAT-NEXT:    add s10, a1, t2
@@ -1618,53 +1618,53 @@ define void @test(ptr %0, ptr %1, i64 %2) {
 ; REMAT-NEXT:    add s4, a1, a3
 ; REMAT-NEXT:    add s3, a1, a2
 ; REMAT-NEXT:    lui s2, 8
-; REMAT-NEXT:    addiw s2, s2, 512
+; REMAT-NEXT:    addi s2, s2, 512
 ; REMAT-NEXT:    add s2, a1, s2
 ; REMAT-NEXT:    lui s1, 8
-; REMAT-NEXT:    addiw s1, s1, 1024
+; REMAT-NEXT:    addi s1, s1, 1024
 ; REMAT-NEXT:    add s1, a1, s1
 ; REMAT-NEXT:    lui s0, 8
-; REMAT-NEXT:    addiw s0, s0, 1536
+; REMAT-NEXT:    addi s0, s0, 1536
 ; REMAT-NEXT:    add s0, a1, s0
 ; REMAT-NEXT:    li t6, 17
 ; REMAT-NEXT:    slli t6, t6, 11
 ; REMAT-NEXT:    add t6, a1, t6
 ; REMAT-NEXT:    lui t5, 9
-; REMAT-NEXT:    addiw t5, t5, -1536
+; REMAT-NEXT:    addi t5, t5, -1536
 ; REMAT-NEXT:    add t5, a1, t5
 ; REMAT-NEXT:    lui t4, 9
-; REMAT-NEXT:    addiw t4, t4, -1024
+; REMAT-NEXT:    addi t4, t4, -1024
 ; REMAT-NEXT:    add t4, a1, t4
 ; REMAT-NEXT:    lui t3, 9
-; REMAT-NEXT:    addiw t3, t3, -512
+; REMAT-NEXT:    addi t3, t3, -512
 ; REMAT-NEXT:    add t3, a1, t3
 ; REMAT-NEXT:    lui t2, 9
 ; REMAT-NEXT:    add t2, a1, t2
 ; REMAT-NEXT:    lui t1, 9
-; REMAT-NEXT:    addiw t1, t1, 512
+; REMAT-NEXT:    addi t1, t1, 512
 ; REMAT-NEXT:    add t1, a1, t1
 ; REMAT-NEXT:    lui t0, 9
-; REMAT-NEXT:    addiw t0, t0, 1024
+; REMAT-NEXT:    addi t0, t0, 1024
 ; REMAT-NEXT:    add t0, a1, t0
 ; REMAT-NEXT:    lui a7, 9
-; REMAT-NEXT:    addiw a7, a7, 1536
+; REMAT-NEXT:    addi a7, a7, 1536
 ; REMAT-NEXT:    add a7, a1, a7
 ; REMAT-NEXT:    li a6, 19
 ; REMAT-NEXT:    slli a6, a6, 11
 ; REMAT-NEXT:    add a6, a1, a6
 ; REMAT-NEXT:    lui a5, 10
-; REMAT-NEXT:    addiw a5, a5, -1536
+; REMAT-NEXT:    addi a5, a5, -1536
 ; REMAT-NEXT:    add a5, a1, a5
 ; REMAT-NEXT:    lui a4, 10
-; REMAT-NEXT:    addiw a4, a4, -1024
+; REMAT-NEXT:    addi a4, a4, -1024
 ; REMAT-NEXT:    add a4, a1, a4
 ; REMAT-NEXT:    lui a3, 10
-; REMAT-NEXT:    addiw a3, a3, -512
+; REMAT-NEXT:    addi a3, a3, -512
 ; REMAT-NEXT:    add a3, a1, a3
 ; REMAT-NEXT:    lui a2, 10
 ; REMAT-NEXT:    add a2, a1, a2
 ; REMAT-NEXT:    lui a0, 10
-; REMAT-NEXT:    addiw a0, a0, 512
+; REMAT-NEXT:    addi a0, a0, 512
 ; REMAT-NEXT:    add a0, a1, a0
 ; REMAT-NEXT:    addi a1, a1, 1536
 ; REMAT-NEXT:    sf.vc.v.i 2, 0, v8, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr90730.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr90730.ll
index 7c3f4b43089c..904761845c4d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr90730.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr90730.ll
@@ -5,7 +5,7 @@ define i32 @pr90730(i32 %x, i1 %y, ptr %p) {
 ; CHECK-LABEL: pr90730:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lui a1, 8
-; CHECK-NEXT:    addiw a1, a1, -960
+; CHECK-NEXT:    addi a1, a1, -960
 ; CHECK-NEXT:    andn a0, a1, a0
 ; CHECK-NEXT:    sw zero, 0(a2)
 ; CHECK-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/pr95271.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/pr95271.ll
index aa941cb80362..fd69e85d0b76 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/pr95271.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/pr95271.ll
@@ -34,12 +34,12 @@ define i32 @PR95271(ptr %p) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lw a0, 0(a0)
 ; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    addiw a1, a1, 1365
+; RV64I-NEXT:    addi a1, a1, 1365
 ; RV64I-NEXT:    addi a2, a0, 1
 ; RV64I-NEXT:    srli a2, a2, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    addiw a0, a0, 1
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/prefer-w-inst.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/prefer-w-inst.ll
index 34ab74d78a76..dcc4f0434005 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/prefer-w-inst.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/prefer-w-inst.ll
@@ -17,7 +17,7 @@ define i32 @addiw(i32 %a) {
 ; NO-STRIP-LABEL: addiw:
 ; NO-STRIP:       # %bb.0:
 ; NO-STRIP-NEXT:    lui a1, 1
-; NO-STRIP-NEXT:    addiw a1, a1, -1
+; NO-STRIP-NEXT:    addi a1, a1, -1
 ; NO-STRIP-NEXT:    addw a0, a0, a1
 ; NO-STRIP-NEXT:    ret
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/prefetch.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/prefetch.ll
index e4c9b91b0dbb..bc46c60c053f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/prefetch.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/prefetch.ll
@@ -715,10 +715,10 @@ define void @test_prefetch_frameindex_1() nounwind {
 ; RV64I-LABEL: test_prefetch_frameindex_1:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    ret
 ;
@@ -737,25 +737,25 @@ define void @test_prefetch_frameindex_1() nounwind {
 ; RV64ZICBOP-LABEL: test_prefetch_frameindex_1:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    lui a0, 1
-; RV64ZICBOP-NEXT:    addiw a0, a0, 16
+; RV64ZICBOP-NEXT:    addi a0, a0, 16
 ; RV64ZICBOP-NEXT:    sub sp, sp, a0
 ; RV64ZICBOP-NEXT:    addi a0, sp, 16
 ; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOP-NEXT:    lui a0, 1
-; RV64ZICBOP-NEXT:    addiw a0, a0, 16
+; RV64ZICBOP-NEXT:    addi a0, a0, 16
 ; RV64ZICBOP-NEXT:    add sp, sp, a0
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_1:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
 ; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1
-; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 16
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, 16
 ; RV64ZICBOPZIHINTNTL-NEXT:    sub sp, sp, a0
 ; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, sp, 16
 ; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1
-; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 16
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, 16
 ; RV64ZICBOPZIHINTNTL-NEXT:    add sp, sp, a0
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   %data = alloca [1024 x i32], align 4
@@ -1158,14 +1158,14 @@ define void @test_prefetch_constant_address_1() nounwind {
 ; RV64ZICBOP-LABEL: test_prefetch_constant_address_1:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    lui a0, 1
-; RV64ZICBOP-NEXT:    addiw a0, a0, 31
+; RV64ZICBOP-NEXT:    addi a0, a0, 31
 ; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_constant_address_1:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
 ; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1
-; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 31
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, 31
 ; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
@@ -1225,14 +1225,14 @@ define void @test_prefetch_constant_address_3() nounwind {
 ; RV64ZICBOP-LABEL: test_prefetch_constant_address_3:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    lui a0, 1048561
-; RV64ZICBOP-NEXT:    addiw a0, a0, 31
+; RV64ZICBOP-NEXT:    addi a0, a0, 31
 ; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_constant_address_3:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
 ; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1048561
-; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 31
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, 31
 ; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/prolog-epilogue.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/prolog-epilogue.ll
index 18cfa7233e4f..e86b4125d327 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/prolog-epilogue.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/prolog-epilogue.ll
@@ -257,13 +257,13 @@ define void @frame_4kb_offset_128() {
 ; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 128
+; RV64I-NEXT:    addi a0, a0, 128
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 6256
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 128
+; RV64I-NEXT:    addi a0, a0, 128
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
@@ -393,13 +393,13 @@ define void @frame_8kb_offset_128() {
 ; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    addiw a0, a0, 128
+; RV64I-NEXT:    addi a0, a0, 128
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 10352
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    addiw a0, a0, 128
+; RV64I-NEXT:    addi a0, a0, 128
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
@@ -482,13 +482,13 @@ define void @frame_16kb_minus_80() {
 ; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    lui a0, 4
-; RV64I-NEXT:    addiw a0, a0, -80
+; RV64I-NEXT:    addi a0, a0, -80
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 18336
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    lui a0, 4
-; RV64I-NEXT:    addiw a0, a0, -80
+; RV64I-NEXT:    addi a0, a0, -80
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/reassoc-shl-addi-add.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/reassoc-shl-addi-add.ll
index 88ab1c0c3eae..64d542766ce4 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/reassoc-shl-addi-add.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/reassoc-shl-addi-add.ll
@@ -1,88 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=riscv32-unknown-elf -mattr=+zba %s -o - | FileCheck %s
 
-declare i32 @callee1(i32 noundef)
-declare i32 @callee2(i32 noundef, i32 noundef)
-declare i32 @callee(i32 noundef, i32 noundef, i32 noundef, i32 noundef)
+declare i32 @callee2(i32, i32)
+declare i32 @callee3(i32, i32, i32)
 
-define void @t1(i32 noundef %a, i32 noundef %b, i32 noundef %c, i32 noundef %d) #0 {
+; Positive test
+; The users of addi aren't more than 2
+define void @t1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: t1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    sh2add a2, a0, a2
-; CHECK-NEXT:    sh2add a1, a0, a1
-; CHECK-NEXT:    addi a1, a1, 45
-; CHECK-NEXT:    addi a2, a2, 45
-; CHECK-NEXT:    sh2add a3, a0, a3
-; CHECK-NEXT:    mv a0, a1
-; CHECK-NEXT:    tail callee
-entry:
-  %shl = shl i32 %a, 2
-  %add = add nsw i32 %shl, 45
-  %add1 = add nsw i32 %add, %b
-  %add3 = add nsw i32 %add, %c
-  %add5 = add nsw i32 %shl, %d
-  %call = tail call i32 @callee(i32 noundef %add1, i32 noundef %add1, i32 noundef %add3, i32 noundef %add5)
-  ret void
-}
-
-define void @t2(i32 noundef %a, i32 noundef %b, i32 noundef %c) #0 {
-; CHECK-LABEL: t2:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    slli a0, a0, 2
-; CHECK-NEXT:    addi a5, a0, 42
-; CHECK-NEXT:    add a4, a5, a1
-; CHECK-NEXT:    add a3, a5, a2
-; CHECK-NEXT:    mv a1, a5
-; CHECK-NEXT:    mv a2, a4
-; CHECK-NEXT:    tail callee
-entry:
-  %shl = shl i32 %a, 2
-  %add = add nsw i32 %shl, 42
-  %add4 = add nsw i32 %add, %b
-  %add7 = add nsw i32 %add, %c
-  %call = tail call i32 @callee(i32 noundef %shl, i32 noundef %add, i32 noundef %add4, i32 noundef %add7)
-  ret void
-}
-
-define void @t3(i32 noundef %a, i32 noundef %b, i32 noundef %c, i32 noundef %d, i32 noundef %e) #0 {
-; CHECK-LABEL: t3:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    slli a0, a0, 2
-; CHECK-NEXT:    addi a5, a0, 42
-; CHECK-NEXT:    add a0, a5, a1
-; CHECK-NEXT:    add a1, a5, a2
-; CHECK-NEXT:    add a2, a5, a3
-; CHECK-NEXT:    add a3, a5, a4
-; CHECK-NEXT:    tail callee
-entry:
-  %shl = shl i32 %a, 2
-  %add = add nsw i32 %shl, 42
-  %add1 = add nsw i32 %add, %b
-  %add2 = add nsw i32 %add, %c
-  %add3 = add nsw i32 %add, %d
-  %add4 = add nsw i32 %add, %e
-  %call = tail call i32 @callee(i32 noundef %add1, i32 noundef %add2, i32 noundef %add3, i32 noundef %add4)
-  ret void
-}
-
-define void @t4(i32 noundef %a, i32 noundef %b) #0 {
-; CHECK-LABEL: t4:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sh2add a0, a0, a1
-; CHECK-NEXT:    addi a0, a0, 42
-; CHECK-NEXT:    tail callee1
-entry:
-  %shl = shl i32 %a, 2
-  %add = add nsw i32 %shl, 42
-  %add1 = add nsw i32 %add, %b
-  %call = tail call i32 @callee1(i32 noundef %add1)
-  ret void
-}
-
-define void @t5(i32 noundef %a, i32 noundef %b, i32 noundef %c) #0 {
-; CHECK-LABEL: t5:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sh2add a2, a0, a2
 ; CHECK-NEXT:    sh2add a0, a0, a1
 ; CHECK-NEXT:    addi a0, a0, 42
 ; CHECK-NEXT:    addi a1, a2, 42
@@ -92,98 +19,68 @@ entry:
   %add = add nsw i32 %shl, 42
   %add1 = add nsw i32 %add, %b
   %add2 = add nsw i32 %add, %c
-  %call = tail call i32 @callee2(i32 noundef %add1, i32 noundef %add2)
+  %call = tail call i32 @callee2(i32 %add1, i32 %add2)
   ret void
 }
 
-define void @t6(i32 noundef %a, i32 noundef %b) #0 {
-; CHECK-LABEL: t6:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    slli a2, a0, 2
-; CHECK-NEXT:    sh2add a0, a0, a1
-; CHECK-NEXT:    addi a0, a0, 42
-; CHECK-NEXT:    mv a1, a2
-; CHECK-NEXT:    mv a3, a2
-; CHECK-NEXT:    tail callee
-entry:
-  %shl = shl i32 %a, 2
-  %add = add nsw i32 %shl, 42
-  %add1 = add nsw i32 %add, %b
-  %call = tail call i32 @callee(i32 noundef %add1, i32 noundef %shl, i32 noundef %shl, i32 noundef %shl)
-  ret void
-}
-
-define void @t7(i32 noundef %a, i32 noundef %b) #0 {
-; CHECK-LABEL: t7:
+; Negative test
+; The users of addi are more than 2.
+define void @t2(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
+; CHECK-LABEL: t2:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    slli a0, a0, 2
-; CHECK-NEXT:    addi a2, a0, 42
-; CHECK-NEXT:    add a0, a2, a1
-; CHECK-NEXT:    mv a1, a2
-; CHECK-NEXT:    mv a3, a2
-; CHECK-NEXT:    tail callee
+; CHECK-NEXT:    addi a4, a0, 42
+; CHECK-NEXT:    add a0, a4, a1
+; CHECK-NEXT:    add a1, a4, a2
+; CHECK-NEXT:    add a2, a4, a3
+; CHECK-NEXT:    tail callee3
 entry:
   %shl = shl i32 %a, 2
   %add = add nsw i32 %shl, 42
   %add1 = add nsw i32 %add, %b
-  %call = tail call i32 @callee(i32 noundef %add1, i32 noundef %add, i32 noundef %add, i32 noundef %add)
-  ret void
-}
-
-define void @t8(i32 noundef %a, i32 noundef %b, i32 noundef %c, i32 noundef %d) #0 {
-; CHECK-LABEL: t8:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sh3add a2, a0, a2
-; CHECK-NEXT:    sh3add a1, a0, a1
-; CHECK-NEXT:    lui a4, 1
-; CHECK-NEXT:    addi a4, a4, 1307
-; CHECK-NEXT:    add a1, a1, a4
-; CHECK-NEXT:    add a2, a2, a4
-; CHECK-NEXT:    sh3add a3, a0, a3
-; CHECK-NEXT:    mv a0, a1
-; CHECK-NEXT:    tail callee
-entry:
-  %shl = shl i32 %a, 3
-  %add = add nsw i32 %shl, 5403
-  %add1 = add nsw i32 %add, %b
-  %add3 = add nsw i32 %add, %c
-  %add5 = add nsw i32 %shl, %d
-  %call = tail call i32 @callee(i32 noundef %add1, i32 noundef %add1, i32 noundef %add3, i32 noundef %add5)
+  %add2 = add nsw i32 %add, %c
+  %add3 = add nsw i32 %add, %d
+  %call = tail call i32 @callee3(i32 %add1, i32 %add2, i32 %add3)
   ret void
 }
 
-define void @t9(i32 noundef %a, i32 noundef %b, i32 noundef %c, i32 noundef %d) #0 {
-; CHECK-LABEL: t9:
+; Negative test
+; shl doesn't have one use.
+define void @t3(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: t3:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sh2add a2, a0, a2
-; CHECK-NEXT:    sh2add a1, a0, a1
-; CHECK-NEXT:    addi a1, a1, -42
-; CHECK-NEXT:    addi a2, a2, -42
-; CHECK-NEXT:    sh2add a3, a0, a3
-; CHECK-NEXT:    mv a0, a1
-; CHECK-NEXT:    tail callee
+; CHECK-NEXT:    slli a4, a0, 2
+; CHECK-NEXT:    addi a5, a4, 45
+; CHECK-NEXT:    add a4, a5, a1
+; CHECK-NEXT:    add a1, a5, a2
+; CHECK-NEXT:    sh2add a2, a0, a3
+; CHECK-NEXT:    mv a0, a4
+; CHECK-NEXT:    tail callee3
 entry:
   %shl = shl i32 %a, 2
-  %add = add nsw i32 %shl, -42
+  %add = add nsw i32 %shl, 45
   %add1 = add nsw i32 %add, %b
-  %add3 = add nsw i32 %add, %c
-  %add5 = add nsw i32 %shl, %d
-  %call = tail call i32 @callee(i32 noundef %add1, i32 noundef %add1, i32 noundef %add3, i32 noundef %add5)
+  %add2 = add nsw i32 %add, %c
+  %add3 = add nsw i32 %shl, %d
+  %call = tail call i32 @callee3(i32 %add1, i32 %add2, i32 %add3)
   ret void
 }
 
-define void @t10(i32 noundef %a, i32 noundef %b, i32 noundef %c, i32 noundef %d) #0 {
-; CHECK-LABEL: t10:
+; Negative test
+; shift amount isn't 1, 2, or 3.
+define void @t4(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: t4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    tail callee
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    addi a3, a0, 45
+; CHECK-NEXT:    add a0, a3, a1
+; CHECK-NEXT:    add a1, a3, a2
+; CHECK-NEXT:    tail callee2
 entry:
-  %shl = shl i32 %a, -2
-  %add = add nsw i32 %shl, 42
+  %shl = shl i32 %a, 4
+  %add = add nsw i32 %shl, 45
   %add1 = add nsw i32 %add, %b
-  %add3 = add nsw i32 %add, %c
-  %add5 = add nsw i32 %shl, %d
-  %call = tail call i32 @callee(i32 noundef %add1, i32 noundef %add1, i32 noundef %add3, i32 noundef %add5)
+  %add2 = add nsw i32 %add, %c
+  %call = tail call i32 @callee2(i32 %add1, i32 %add2)
   ret void
 }
-
-attributes #0 = { nounwind optsize }
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rem.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rem.ll
index 612aaafc53bc..12697e86a91d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rem.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rem.ll
@@ -567,7 +567,7 @@ define i16 @urem16(i16 %a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    call __umoddi3
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv32i-rv64i-half.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv32i-rv64i-half.ll
index 99b111b10f66..8a05c867bb07 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv32i-rv64i-half.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv32i-rv64i-half.ll
@@ -51,7 +51,7 @@ define half @half_test(half %a, half %b) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s2, a1, -1
+; RV64I-NEXT:    addi s2, a1, -1
 ; RV64I-NEXT:    and a0, a0, s2
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s1, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv32xandesperf.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
index 72dddddf9f38..3e7f09f3d6c2 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
@@ -2,6 +2,10 @@
 ; RUN: llc -O0 -mtriple=riscv32 -mattr=+xandesperf -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
+; NDS.BFOZ
+
+; MSB >= LSB
+
 define i32 @bfoz_from_and_i32(i32 %x) {
 ; CHECK-LABEL: bfoz_from_and_i32:
 ; CHECK:       # %bb.0:
@@ -70,6 +74,112 @@ define i64 @bfoz_from_lshr_and_i64(i64 %x) {
   ret i64 %shifted
 }
 
+; MSB = 0
+
+define i32 @bfoz_from_and_shl_with_msb_zero_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_and_shl_with_msb_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 0, 15
+; CHECK-NEXT:    ret
+  %shifted = shl i32 %x, 15
+  %masked = and i32 %shifted, 32768
+  ret i32 %masked
+}
+
+define i32 @bfoz_from_lshr_shl_with_msb_zero_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_lshr_shl_with_msb_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 0, 18
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 31
+  %lshr = lshr i32 %shl, 13
+  ret i32 %lshr
+}
+
+; MSB < LSB
+
+define i32 @bfoz_from_and_shl_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_and_shl_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 12, 23
+; CHECK-NEXT:    ret
+  %shifted = shl i32 %x, 12
+  %masked = and i32 %shifted, 16773120
+  ret i32 %masked
+}
+
+define i32 @bfoz_from_lshr_shl_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_lshr_shl_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 19, 24
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 26
+  %lshr = lshr i32 %shl, 7
+  ret i32 %lshr
+}
+
+; NDS.BFOS
+
+; MSB >= LSB
+
+define i32 @bfos_from_ashr_shl_i32(i32 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 23, 16
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 8
+  %ashr = ashr i32 %shl, 24
+  ret i32 %ashr
+}
+
+define i32 @bfos_from_ashr_sexti8_i32(i8 %x) {
+; CHECK-LABEL: bfos_from_ashr_sexti8_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $x11 killed $x10
+; CHECK-NEXT:    nds.bfos a0, a0, 7, 5
+; CHECK-NEXT:    ret
+  %sext = sext i8 %x to i32
+  %ashr = ashr i32 %sext, 5
+  ret i32 %ashr
+}
+
+define i32 @bfos_from_ashr_sexti16_i32(i16 %x) {
+; CHECK-LABEL: bfos_from_ashr_sexti16_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $x11 killed $x10
+; CHECK-NEXT:    nds.bfos a0, a0, 15, 11
+; CHECK-NEXT:    ret
+  %sext = sext i16 %x to i32
+  %ashr = ashr i32 %sext, 11
+  ret i32 %ashr
+}
+
+; MSB = 0
+
+define i32 @bfos_from_ashr_shl_with_msb_zero_insert_i32(i32 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_with_msb_zero_insert_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 0, 14
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 31
+  %lshr = ashr i32 %shl, 17
+  ret i32 %lshr
+}
+
+; MSB < LSB
+
+define i32 @bfos_from_ashr_shl_insert_i32(i32 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_insert_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 18, 20
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 29
+  %lshr = ashr i32 %shl, 11
+  ret i32 %lshr
+}
+
+; sext
+
 define i32 @sexti1_i32(i32 %a) {
 ; CHECK-LABEL: sexti1_i32:
 ; CHECK:       # %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv32zbb.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv32zbb.ll
index 62bc7b3336a5..8dd63015971d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv32zbb.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv32zbb.ll
@@ -1590,8 +1590,8 @@ define i128 @sub_if_uge_i128(i128 %x, i128 %y) {
 ; CHECK-NEXT:    lw a7, 4(a2)
 ; CHECK-NEXT:    lw a6, 8(a2)
 ; CHECK-NEXT:    lw t0, 12(a2)
-; CHECK-NEXT:    lw a4, 12(a1)
 ; CHECK-NEXT:    lw a3, 4(a1)
+; CHECK-NEXT:    lw a4, 12(a1)
 ; CHECK-NEXT:    lw a5, 8(a1)
 ; CHECK-NEXT:    beq a4, t0, .LBB53_2
 ; CHECK-NEXT:  # %bb.1:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-float-convert.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
index e387586cd46c..6aae8984546e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
@@ -83,7 +83,7 @@ define i128 @fptosi_sat_f32_to_i128(float %a) nounwind {
 ; RV64I-NEXT:    slli s3, s5, 63
 ; RV64I-NEXT:  .LBB4_2:
 ; RV64I-NEXT:    lui a1, 520192
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    mv s4, a0
@@ -168,7 +168,7 @@ define i128 @fptosi_sat_f32_to_i128(float %a) nounwind {
 ; RV64IZFINX-NEXT:    slli a1, a2, 63
 ; RV64IZFINX-NEXT:  .LBB4_2:
 ; RV64IZFINX-NEXT:    lui a3, 520192
-; RV64IZFINX-NEXT:    addiw a3, a3, -1
+; RV64IZFINX-NEXT:    addi a3, a3, -1
 ; RV64IZFINX-NEXT:    flt.s a3, a3, s0
 ; RV64IZFINX-NEXT:    beqz a3, .LBB4_4
 ; RV64IZFINX-NEXT:  # %bb.3:
@@ -202,7 +202,7 @@ define i128 @fptoui_sat_f32_to_i128(float %a) nounwind {
 ; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a1, 522240
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    sgtz a0, a0
 ; RV64I-NEXT:    neg s1, a0
@@ -263,7 +263,7 @@ define i128 @fptoui_sat_f32_to_i128(float %a) nounwind {
 ; RV64IZFINX-NEXT:    and a0, s1, a0
 ; RV64IZFINX-NEXT:    lui a2, 522240
 ; RV64IZFINX-NEXT:    and a1, s1, a1
-; RV64IZFINX-NEXT:    addiw a2, a2, -1
+; RV64IZFINX-NEXT:    addi a2, a2, -1
 ; RV64IZFINX-NEXT:    flt.s a2, a2, s0
 ; RV64IZFINX-NEXT:    neg a2, a2
 ; RV64IZFINX-NEXT:    or a0, a2, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-half-convert.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
index ea582ac258b7..57061e1bde83 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
@@ -160,7 +160,7 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    slli s3, s5, 63
 ; RV64I-NEXT:  .LBB4_2:
 ; RV64I-NEXT:    lui a1, 520192
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    mv s4, a0
@@ -246,7 +246,7 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind {
 ; RV64IZHINX-NEXT:    slli a1, a2, 63
 ; RV64IZHINX-NEXT:  .LBB4_2:
 ; RV64IZHINX-NEXT:    lui a3, 520192
-; RV64IZHINX-NEXT:    addiw a3, a3, -1
+; RV64IZHINX-NEXT:    addi a3, a3, -1
 ; RV64IZHINX-NEXT:    flt.s a3, a3, s0
 ; RV64IZHINX-NEXT:    beqz a3, .LBB4_4
 ; RV64IZHINX-NEXT:  # %bb.3:
@@ -281,7 +281,7 @@ define i128 @fptoui_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    call __extendhfsf2
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a1, 522240
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    sgtz a0, a0
 ; RV64I-NEXT:    neg s1, a0
@@ -336,7 +336,7 @@ define i128 @fptoui_sat_f16_to_i128(half %a) nounwind {
 ; RV64IZHINX-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
 ; RV64IZHINX-NEXT:    lui a1, 522240
-; RV64IZHINX-NEXT:    addiw a1, a1, -1
+; RV64IZHINX-NEXT:    addi a1, a1, -1
 ; RV64IZHINX-NEXT:    fle.s a2, zero, a0
 ; RV64IZHINX-NEXT:    flt.s a1, a1, a0
 ; RV64IZHINX-NEXT:    neg s0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-patchpoint.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-patchpoint.ll
index beccd39e819d..0850cc65d81e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-patchpoint.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-patchpoint.ll
@@ -15,7 +15,7 @@ define i64 @trivial_patchpoint_codegen(i64 %p1, i64 %p2, i64 %p3, i64 %p4) {
 ; CHECK-NEXT:    mv s0, a0
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    lui ra, 3563
-; CHECK-NEXT:    addiw ra, ra, -577
+; CHECK-NEXT:    addi ra, ra, -577
 ; CHECK-NEXT:    slli ra, ra, 12
 ; CHECK-NEXT:    addi ra, ra, -259
 ; CHECK-NEXT:    slli ra, ra, 12
@@ -26,7 +26,7 @@ define i64 @trivial_patchpoint_codegen(i64 %p1, i64 %p2, i64 %p3, i64 %p4) {
 ; CHECK-NEXT:    mv a1, s1
 ; CHECK-NEXT:  .Ltmp1:
 ; CHECK-NEXT:    lui ra, 3563
-; CHECK-NEXT:    addiw ra, ra, -577
+; CHECK-NEXT:    addi ra, ra, -577
 ; CHECK-NEXT:    slli ra, ra, 12
 ; CHECK-NEXT:    addi ra, ra, -259
 ; CHECK-NEXT:    slli ra, ra, 12
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-trampoline-cfi.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-trampoline-cfi.ll
new file mode 100644
index 000000000000..8a338a855c86
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-trampoline-cfi.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O0 -mtriple=riscv64 -mattr=+experimental-zicfilp -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64 %s
+; RUN: llc -O0 -mtriple=riscv64-unknown-linux-gnu -mattr=+experimental-zicfilp -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64-LINUX %s
+
+declare void @llvm.init.trampoline(ptr, ptr, ptr)
+declare ptr @llvm.adjust.trampoline(ptr)
+declare i64 @f(ptr nest, i64)
+
+define i64 @test0(i64 %n, ptr %p) nounwind {
+; RV64-LABEL: test0:
+; RV64:       # %bb.0:
+; RV64-NEXT:    lpad 0
+; RV64-NEXT:    addi sp, sp, -64
+; RV64-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT:    lui a0, %hi(f)
+; RV64-NEXT:    addi a0, a0, %lo(f)
+; RV64-NEXT:    sw a0, 44(sp)
+; RV64-NEXT:    srli a0, a0, 32
+; RV64-NEXT:    sw a0, 48(sp)
+; RV64-NEXT:    sw a1, 36(sp)
+; RV64-NEXT:    srli a0, a1, 32
+; RV64-NEXT:    sw a0, 40(sp)
+; RV64-NEXT:    li a0, 23
+; RV64-NEXT:    sw a0, 16(sp)
+; RV64-NEXT:    lui a0, 56
+; RV64-NEXT:    addi a0, a0, 103
+; RV64-NEXT:    sw a0, 32(sp)
+; RV64-NEXT:    lui a0, 4324
+; RV64-NEXT:    addi a0, a0, -509
+; RV64-NEXT:    sw a0, 28(sp)
+; RV64-NEXT:    lui a0, 6371
+; RV64-NEXT:    addi a0, a0, 899
+; RV64-NEXT:    sw a0, 24(sp)
+; RV64-NEXT:    lui a0, 1
+; RV64-NEXT:    addi a0, a0, -489
+; RV64-NEXT:    sw a0, 20(sp)
+; RV64-NEXT:    addi a1, sp, 36
+; RV64-NEXT:    addi a0, sp, 16
+; RV64-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    call __clear_cache
+; RV64-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld a1, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    jalr a1
+; RV64-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 64
+; RV64-NEXT:    ret
+;
+; RV64-LINUX-LABEL: test0:
+; RV64-LINUX:       # %bb.0:
+; RV64-LINUX-NEXT:    lpad 0
+; RV64-LINUX-NEXT:    addi sp, sp, -64
+; RV64-LINUX-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64-LINUX-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
+; RV64-LINUX-NEXT:    lui a0, %hi(f)
+; RV64-LINUX-NEXT:    addi a0, a0, %lo(f)
+; RV64-LINUX-NEXT:    sw a0, 44(sp)
+; RV64-LINUX-NEXT:    srli a0, a0, 32
+; RV64-LINUX-NEXT:    sw a0, 48(sp)
+; RV64-LINUX-NEXT:    sw a1, 36(sp)
+; RV64-LINUX-NEXT:    srli a0, a1, 32
+; RV64-LINUX-NEXT:    sw a0, 40(sp)
+; RV64-LINUX-NEXT:    li a0, 23
+; RV64-LINUX-NEXT:    sw a0, 16(sp)
+; RV64-LINUX-NEXT:    lui a0, 56
+; RV64-LINUX-NEXT:    addi a0, a0, 103
+; RV64-LINUX-NEXT:    sw a0, 32(sp)
+; RV64-LINUX-NEXT:    lui a0, 4324
+; RV64-LINUX-NEXT:    addi a0, a0, -509
+; RV64-LINUX-NEXT:    sw a0, 28(sp)
+; RV64-LINUX-NEXT:    lui a0, 6371
+; RV64-LINUX-NEXT:    addi a0, a0, 899
+; RV64-LINUX-NEXT:    sw a0, 24(sp)
+; RV64-LINUX-NEXT:    lui a0, 1
+; RV64-LINUX-NEXT:    addi a0, a0, -489
+; RV64-LINUX-NEXT:    sw a0, 20(sp)
+; RV64-LINUX-NEXT:    addi a1, sp, 36
+; RV64-LINUX-NEXT:    addi a0, sp, 16
+; RV64-LINUX-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
+; RV64-LINUX-NEXT:    li a2, 0
+; RV64-LINUX-NEXT:    call __riscv_flush_icache
+; RV64-LINUX-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
+; RV64-LINUX-NEXT:    ld a1, 8(sp) # 8-byte Folded Reload
+; RV64-LINUX-NEXT:    jalr a1
+; RV64-LINUX-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; RV64-LINUX-NEXT:    addi sp, sp, 64
+; RV64-LINUX-NEXT:    ret
+  %alloca = alloca [36 x i8], align 8
+  call void @llvm.init.trampoline(ptr %alloca, ptr @f, ptr %p)
+  %tramp = call ptr @llvm.adjust.trampoline(ptr %alloca)
+  %ret = call i64 %tramp(i64 %n)
+  ret i64 %ret
+}
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 8, !"cf-protection-branch", i32 1}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-trampoline.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-trampoline.ll
index 1ec4d8ddd1d8..34d46579518e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-trampoline.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64-trampoline.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN: llc -O0 -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64 %s
-; RUN: llc -mtriple=riscv64-unknown-linux-gnu -verify-machineinstrs < %s \
+; RUN: llc -O0 -mtriple=riscv64-unknown-linux-gnu -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64-LINUX %s
 
 declare void @llvm.init.trampoline(ptr, ptr, ptr)
@@ -13,30 +13,30 @@ define i64 @test0(i64 %n, ptr %p) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi sp, sp, -64
 ; RV64-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
-; RV64-NEXT:    mv s0, a0
+; RV64-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    lui a0, %hi(f)
 ; RV64-NEXT:    addi a0, a0, %lo(f)
-; RV64-NEXT:    li a2, 919
-; RV64-NEXT:    lui a3, %hi(.LCPI0_0)
-; RV64-NEXT:    sd a0, 32(sp)
+; RV64-NEXT:    sd a0, 48(sp)
+; RV64-NEXT:    sd a1, 40(sp)
+; RV64-NEXT:    li a0, 919
+; RV64-NEXT:    sw a0, 24(sp)
+; RV64-NEXT:    lui a0, 40
+; RV64-NEXT:    addi a0, a0, 103
+; RV64-NEXT:    sw a0, 36(sp)
+; RV64-NEXT:    lui a0, 4155
+; RV64-NEXT:    addi a0, a0, 899
+; RV64-NEXT:    sw a0, 32(sp)
 ; RV64-NEXT:    lui a0, 6203
-; RV64-NEXT:    ld a3, %lo(.LCPI0_0)(a3)
 ; RV64-NEXT:    addi a0, a0, 643
-; RV64-NEXT:    sw a2, 8(sp)
-; RV64-NEXT:    sw a0, 12(sp)
-; RV64-NEXT:    sd a3, 16(sp)
-; RV64-NEXT:    sd a1, 24(sp)
-; RV64-NEXT:    addi a1, sp, 24
-; RV64-NEXT:    addi a0, sp, 8
-; RV64-NEXT:    addi s1, sp, 8
+; RV64-NEXT:    sw a0, 28(sp)
+; RV64-NEXT:    addi a1, sp, 40
+; RV64-NEXT:    addi a0, sp, 24
+; RV64-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    call __clear_cache
-; RV64-NEXT:    mv a0, s0
-; RV64-NEXT:    jalr s1
+; RV64-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld a1, 16(sp) # 8-byte Folded Reload
+; RV64-NEXT:    jalr a1
 ; RV64-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 64
 ; RV64-NEXT:    ret
 ;
@@ -44,31 +44,31 @@ define i64 @test0(i64 %n, ptr %p) nounwind {
 ; RV64-LINUX:       # %bb.0:
 ; RV64-LINUX-NEXT:    addi sp, sp, -64
 ; RV64-LINUX-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-LINUX-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-LINUX-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
-; RV64-LINUX-NEXT:    mv s0, a0
+; RV64-LINUX-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64-LINUX-NEXT:    lui a0, %hi(f)
 ; RV64-LINUX-NEXT:    addi a0, a0, %lo(f)
-; RV64-LINUX-NEXT:    li a2, 919
-; RV64-LINUX-NEXT:    lui a3, %hi(.LCPI0_0)
-; RV64-LINUX-NEXT:    sd a0, 32(sp)
+; RV64-LINUX-NEXT:    sd a0, 48(sp)
+; RV64-LINUX-NEXT:    sd a1, 40(sp)
+; RV64-LINUX-NEXT:    li a0, 919
+; RV64-LINUX-NEXT:    sw a0, 24(sp)
+; RV64-LINUX-NEXT:    lui a0, 40
+; RV64-LINUX-NEXT:    addi a0, a0, 103
+; RV64-LINUX-NEXT:    sw a0, 36(sp)
+; RV64-LINUX-NEXT:    lui a0, 4155
+; RV64-LINUX-NEXT:    addi a0, a0, 899
+; RV64-LINUX-NEXT:    sw a0, 32(sp)
 ; RV64-LINUX-NEXT:    lui a0, 6203
-; RV64-LINUX-NEXT:    ld a3, %lo(.LCPI0_0)(a3)
 ; RV64-LINUX-NEXT:    addi a0, a0, 643
-; RV64-LINUX-NEXT:    sw a2, 8(sp)
-; RV64-LINUX-NEXT:    sw a0, 12(sp)
-; RV64-LINUX-NEXT:    sd a3, 16(sp)
-; RV64-LINUX-NEXT:    sd a1, 24(sp)
-; RV64-LINUX-NEXT:    addi a1, sp, 24
-; RV64-LINUX-NEXT:    addi a0, sp, 8
-; RV64-LINUX-NEXT:    addi s1, sp, 8
+; RV64-LINUX-NEXT:    sw a0, 28(sp)
+; RV64-LINUX-NEXT:    addi a1, sp, 40
+; RV64-LINUX-NEXT:    addi a0, sp, 24
+; RV64-LINUX-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64-LINUX-NEXT:    li a2, 0
 ; RV64-LINUX-NEXT:    call __riscv_flush_icache
-; RV64-LINUX-NEXT:    mv a0, s0
-; RV64-LINUX-NEXT:    jalr s1
+; RV64-LINUX-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; RV64-LINUX-NEXT:    ld a1, 16(sp) # 8-byte Folded Reload
+; RV64-LINUX-NEXT:    jalr a1
 ; RV64-LINUX-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-LINUX-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-LINUX-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
 ; RV64-LINUX-NEXT:    addi sp, sp, 64
 ; RV64-LINUX-NEXT:    ret
   %alloca = alloca [32 x i8], align 8
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xandesperf.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xandesperf.ll
index 13c2234071eb..98cda4266516 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xandesperf.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xandesperf.ll
@@ -2,6 +2,10 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesperf -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
+; NDS.BFOZ
+
+; MSB >= LSB
+
 define i32 @bfoz_from_and_i32(i32 %x) {
 ; CHECK-LABEL: bfoz_from_and_i32:
 ; CHECK:       # %bb.0:
@@ -60,6 +64,200 @@ define i64 @bfoz_from_lshr_and_i64(i64 %x) {
   ret i64 %shifted
 }
 
+; MSB = 0
+
+define i32 @bfoz_from_and_shl_with_msb_zero_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_and_shl_with_msb_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 0, 15
+; CHECK-NEXT:    ret
+  %shifted = shl i32 %x, 15
+  %masked = and i32 %shifted, 32768
+  ret i32 %masked
+}
+
+define i64 @bfoz_from_and_shl_with_msb_zero_i64(i64 %x) {
+; CHECK-LABEL: bfoz_from_and_shl_with_msb_zero_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 0, 48
+; CHECK-NEXT:    ret
+  %shifted = shl i64 %x, 48
+  %masked = and i64 %shifted, 281474976710656
+  ret i64 %masked
+}
+
+define i32 @bfoz_from_lshr_shl_with_msb_zero_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_lshr_shl_with_msb_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 0, 18
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 31
+  %lshr = lshr i32 %shl, 13
+  ret i32 %lshr
+}
+
+define i64 @bfoz_from_lshr_shl_with_msb_zero_i64(i64 %x) {
+; CHECK-LABEL: bfoz_from_lshr_shl_with_msb_zero_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 0, 44
+; CHECK-NEXT:    ret
+  %shl = shl i64 %x, 63
+  %lshr = lshr i64 %shl, 19
+  ret i64 %lshr
+}
+
+; MSB < LSB
+
+define i32 @bfoz_from_and_shl_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_and_shl_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 12, 23
+; CHECK-NEXT:    ret
+  %shifted = shl i32 %x, 12
+  %masked = and i32 %shifted, 16773120
+  ret i32 %masked
+}
+
+define i64 @bfoz_from_and_shl_i64(i64 %x) {
+; CHECK-LABEL: bfoz_from_and_shl_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 24, 35
+; CHECK-NEXT:    ret
+  %shifted = shl i64 %x, 24
+  %masked = and i64 %shifted, 68702699520
+  ret i64 %masked
+}
+
+define i32 @bfoz_from_lshr_shl_i32(i32 %x) {
+; CHECK-LABEL: bfoz_from_lshr_shl_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 19, 24
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 26
+  %lshr = lshr i32 %shl, 7
+  ret i32 %lshr
+}
+
+define i64 @bfoz_from_lshr_shl_i64(i64 %x) {
+; CHECK-LABEL: bfoz_from_lshr_shl_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfoz a0, a0, 25, 48
+; CHECK-NEXT:    ret
+  %shl = shl i64 %x, 40
+  %lshr = lshr i64 %shl, 15
+  ret i64 %lshr
+}
+
+; NDS.BFOS
+
+; MSB >= LSB
+
+define i32 @bfos_from_ashr_shl_i32(i32 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 23, 16
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 8
+  %ashr = ashr i32 %shl, 24
+  ret i32 %ashr
+}
+
+define i64 @bfos_from_ashr_shl_i64(i64 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 55, 16
+; CHECK-NEXT:    ret
+  %shl = shl i64 %x, 8
+  %ashr = ashr i64 %shl, 24
+  ret i64 %ashr
+}
+
+define i32 @bfos_from_ashr_sexti8_i32(i8 %x) {
+; CHECK-LABEL: bfos_from_ashr_sexti8_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 7, 5
+; CHECK-NEXT:    ret
+  %sext = sext i8 %x to i32
+  %ashr = ashr i32 %sext, 5
+  ret i32 %ashr
+}
+
+define i64 @bfos_from_ashr_sexti8_i64(i8 %x) {
+; CHECK-LABEL: bfos_from_ashr_sexti8_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 7, 5
+; CHECK-NEXT:    ret
+  %sext = sext i8 %x to i64
+  %ashr = ashr i64 %sext, 5
+  ret i64 %ashr
+}
+
+define i32 @bfos_from_ashr_sexti16_i32(i16 %x) {
+; CHECK-LABEL: bfos_from_ashr_sexti16_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 15, 11
+; CHECK-NEXT:    ret
+  %sext = sext i16 %x to i32
+  %ashr = ashr i32 %sext, 11
+  ret i32 %ashr
+}
+
+define i64 @bfos_from_ashr_sexti16_i64(i16 %x) {
+; CHECK-LABEL: bfos_from_ashr_sexti16_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 15, 11
+; CHECK-NEXT:    ret
+  %sext = sext i16 %x to i64
+  %ashr = ashr i64 %sext, 11
+  ret i64 %ashr
+}
+
+; MSB = 0
+
+define i32 @bfos_from_ashr_shl_with_msb_zero_insert_i32(i32 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_with_msb_zero_insert_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 0, 14
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 31
+  %lshr = ashr i32 %shl, 17
+  ret i32 %lshr
+}
+
+define i64 @bfos_from_ashr_shl_with_msb_zero_insert_i64(i64 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_with_msb_zero_insert_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 0, 46
+; CHECK-NEXT:    ret
+  %shl = shl i64 %x, 63
+  %lshr = ashr i64 %shl, 17
+  ret i64 %lshr
+}
+
+; MSB < LSB
+
+define i32 @bfos_from_ashr_shl_insert_i32(i32 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_insert_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 18, 20
+; CHECK-NEXT:    ret
+  %shl = shl i32 %x, 29
+  %lshr = ashr i32 %shl, 11
+  ret i32 %lshr
+}
+
+define i64 @bfos_from_ashr_shl_insert_i64(i64 %x) {
+; CHECK-LABEL: bfos_from_ashr_shl_insert_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bfos a0, a0, 18, 52
+; CHECK-NEXT:    ret
+  %shl = shl i64 %x, 29
+  %lshr = ashr i64 %shl, 11
+  ret i64 %lshr
+}
+
+; sext
+
 define signext i32 @sexti1_i32(i32 signext %a) {
 ; CHECK-LABEL: sexti1_i32:
 ; CHECK:       # %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadba.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadba.ll
index 05396e3355ff..d20fb66dbbee 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadba.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadba.ll
@@ -507,7 +507,7 @@ define i64 @addmul4230(i64 %a, i64 %b) {
 ; CHECK-LABEL: addmul4230:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a2, 1
-; CHECK-NEXT:    addiw a2, a2, 134
+; CHECK-NEXT:    addi a2, a2, 134
 ; CHECK-NEXT:    mul a0, a0, a2
 ; CHECK-NEXT:    add a0, a0, a1
 ; CHECK-NEXT:    ret
@@ -1034,7 +1034,7 @@ define i64 @add4104(i64 %a) {
 ; RV64I-LABEL: add4104:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 8
+; RV64I-NEXT:    addi a1, a1, 8
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -1051,7 +1051,7 @@ define i64 @add4104_2(i64 %a) {
 ; RV64I-LABEL: add4104_2:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 8
+; RV64I-NEXT:    addi a1, a1, 8
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -1068,7 +1068,7 @@ define i64 @add8208(i64 %a) {
 ; RV64I-LABEL: add8208:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 2
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
index 10ef3357d478..00f7b462f68d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
@@ -14,7 +14,7 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -27,7 +27,7 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -66,7 +66,7 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -79,7 +79,7 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -127,7 +127,7 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a2, a1, 1
 ; RV64I-NEXT:    lui a3, 349525
 ; RV64I-NEXT:    or a1, a1, a2
-; RV64I-NEXT:    addiw a2, a3, 1365
+; RV64I-NEXT:    addi a2, a3, 1365
 ; RV64I-NEXT:    srliw a3, a1, 2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srliw a3, a1, 4
@@ -140,7 +140,7 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a3, a1, 1
 ; RV64I-NEXT:    and a2, a3, a2
 ; RV64I-NEXT:    lui a3, 209715
-; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    addi a3, a3, 819
 ; RV64I-NEXT:    sub a1, a1, a2
 ; RV64I-NEXT:    and a2, a1, a3
 ; RV64I-NEXT:    srli a1, a1, 2
@@ -181,7 +181,7 @@ define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a1, a0, a1
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    srliw a3, a1, 2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srliw a3, a1, 4
@@ -194,7 +194,7 @@ define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a3, a1, 1
 ; RV64I-NEXT:    and a2, a3, a2
 ; RV64I-NEXT:    lui a3, 209715
-; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    addi a3, a3, 819
 ; RV64I-NEXT:    sub a1, a1, a2
 ; RV64I-NEXT:    and a2, a1, a3
 ; RV64I-NEXT:    srli a1, a1, 2
@@ -242,7 +242,7 @@ define i32 @ctlz_lshr_i32(i32 signext %a) {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -255,7 +255,7 @@ define i32 @ctlz_lshr_i32(i32 signext %a) {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -299,8 +299,8 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    lui a3, 209715
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
-; RV64I-NEXT:    addiw a2, a3, 819
+; RV64I-NEXT:    addi a1, a2, 1365
+; RV64I-NEXT:    addi a2, a3, 819
 ; RV64I-NEXT:    srli a3, a0, 2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a3, a1, 32
@@ -319,7 +319,7 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 1
 ; RV64I-NEXT:    and a1, a3, a1
 ; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -849,7 +849,7 @@ define signext i32 @bswap_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 8
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    srliw a3, a0, 24
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a2, a0, a2
 ; RV64I-NEXT:    or a1, a1, a3
@@ -905,7 +905,7 @@ define i64 @bswap_i64(i64 %a) {
 ; RV64I-NEXT:    srli a3, a0, 56
 ; RV64I-NEXT:    srli a4, a0, 24
 ; RV64I-NEXT:    lui a5, 4080
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srli a3, a0, 8
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zba.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zba.ll
index 622b302da4c5..fdff4a39932b 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -24,8 +24,7 @@ define i64 @slliuw(i64 %a) nounwind {
 ;
 ; RV64XANDESPERF-LABEL: slliuw:
 ; RV64XANDESPERF:       # %bb.0:
-; RV64XANDESPERF-NEXT:    slli a0, a0, 32
-; RV64XANDESPERF-NEXT:    srli a0, a0, 31
+; RV64XANDESPERF-NEXT:    nds.bfoz a0, a0, 1, 32
 ; RV64XANDESPERF-NEXT:    ret
   %conv1 = shl i64 %a, 1
   %shl = and i64 %conv1, 8589934590
@@ -1049,7 +1048,7 @@ define i64 @addmul4230(i64 %a, i64 %b) {
 ; CHECK-LABEL: addmul4230:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a2, 1
-; CHECK-NEXT:    addiw a2, a2, 134
+; CHECK-NEXT:    addi a2, a2, 134
 ; CHECK-NEXT:    mul a0, a0, a2
 ; CHECK-NEXT:    add a0, a0, a1
 ; CHECK-NEXT:    ret
@@ -2061,7 +2060,7 @@ define i64 @add4104(i64 %a) {
 ; RV64I-LABEL: add4104:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 8
+; RV64I-NEXT:    addi a1, a1, 8
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -2084,7 +2083,7 @@ define i64 @add4104_2(i64 %a) {
 ; RV64I-LABEL: add4104_2:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 8
+; RV64I-NEXT:    addi a1, a1, 8
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -2107,7 +2106,7 @@ define i64 @add8208(i64 %a) {
 ; RV64I-LABEL: add8208:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 2
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -3688,15 +3687,14 @@ define ptr @test_gep_gep_dont_crash(ptr %p, i64 %a1, i64 %a2) {
 ; RV64ZBA-LABEL: test_gep_gep_dont_crash:
 ; RV64ZBA:       # %bb.0:
 ; RV64ZBA-NEXT:    srliw a2, a2, 6
-; RV64ZBA-NEXT:    add a1, a2, a1
+; RV64ZBA-NEXT:    sh3add a0, a2, a0
 ; RV64ZBA-NEXT:    sh3add a0, a1, a0
 ; RV64ZBA-NEXT:    ret
 ;
 ; RV64XANDESPERF-LABEL: test_gep_gep_dont_crash:
 ; RV64XANDESPERF:       # %bb.0:
 ; RV64XANDESPERF-NEXT:    srliw a2, a2, 6
-; RV64XANDESPERF-NEXT:    slli a2, a2, 3
-; RV64XANDESPERF-NEXT:    add a0, a0, a2
+; RV64XANDESPERF-NEXT:    nds.lea.d a0, a0, a2
 ; RV64XANDESPERF-NEXT:    nds.lea.d a0, a0, a1
 ; RV64XANDESPERF-NEXT:    ret
   %lshr = lshr i64 %a2, 6
@@ -4099,7 +4097,7 @@ define i64 @srli_slli_i16(i64 %1) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    slli a0, a0, 2
 ; CHECK-NEXT:    lui a1, 256
-; CHECK-NEXT:    addiw a1, a1, -16
+; CHECK-NEXT:    addi a1, a1, -16
 ; CHECK-NEXT:    and a0, a0, a1
 ; CHECK-NEXT:    ret
 entry:
@@ -4262,10 +4260,11 @@ define i64 @add_u32simm32_zextw(i64 %x) nounwind {
 ;
 ; RV64XANDESPERF-LABEL: add_u32simm32_zextw:
 ; RV64XANDESPERF:       # %bb.0: # %entry
+; RV64XANDESPERF-NEXT:    li a1, -2
+; RV64XANDESPERF-NEXT:    nds.lea.b.ze a0, a0, a1
 ; RV64XANDESPERF-NEXT:    li a1, 1
 ; RV64XANDESPERF-NEXT:    slli a1, a1, 32
 ; RV64XANDESPERF-NEXT:    addi a1, a1, -2
-; RV64XANDESPERF-NEXT:    add a0, a0, a1
 ; RV64XANDESPERF-NEXT:    addi a1, a1, 1
 ; RV64XANDESPERF-NEXT:    and a0, a0, a1
 ; RV64XANDESPERF-NEXT:    ret
@@ -4274,3 +4273,306 @@ entry:
   %and = and i64 %add, 4294967295
   ret i64 %and
 }
+
+define ptr @shl_and_gep(ptr %p, i64 %i) {
+; RV64I-LABEL: shl_and_gep:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a1, a1, 2
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: shl_and_gep:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    srliw a1, a1, 2
+; RV64ZBA-NEXT:    sh3add a0, a1, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: shl_and_gep:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    srliw a1, a1, 2
+; RV64XANDESPERF-NEXT:    nds.lea.d a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %shl = shl i64 %i, 1
+  %and = and i64 %shl, 8589934584
+  %gep = getelementptr i8, ptr %p, i64 %and
+  ret ptr %gep
+}
+
+define ptr @shr_and_gep(ptr %p, i64 %i) {
+; RV64I-LABEL: shr_and_gep:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a1, a1, 6
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: shr_and_gep:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    srliw a1, a1, 6
+; RV64ZBA-NEXT:    sh1add a0, a1, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: shr_and_gep:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    srliw a1, a1, 6
+; RV64XANDESPERF-NEXT:    nds.lea.h a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %lshr = lshr i64 %i, 6
+  %and = and i64 %lshr, 67108863
+  %gep = getelementptr i16, ptr %p, i64 %and
+  ret ptr %gep
+}
+
+define ptr @slt_select_gep(ptr %p, i32 %y) {
+; RV64I-LABEL: slt_select_gep:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srli a1, a1, 28
+; RV64I-NEXT:    andi a1, a1, 8
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    addi a0, a0, 16
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: slt_select_gep:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    srliw a1, a1, 31
+; RV64ZBA-NEXT:    sh3add a0, a1, a0
+; RV64ZBA-NEXT:    addi a0, a0, 16
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: slt_select_gep:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    srliw a1, a1, 31
+; RV64XANDESPERF-NEXT:    nds.lea.d a0, a0, a1
+; RV64XANDESPERF-NEXT:    addi a0, a0, 16
+; RV64XANDESPERF-NEXT:    ret
+  %cmp = icmp slt i32 %y, 0
+  %select = select i1 %cmp, i64 24, i64 16
+  %gep = getelementptr i8, ptr %p, i64 %select
+  ret ptr %gep
+}
+
+define i32 @shr_and_add(i32 %x, i32 %y) {
+; RV64I-LABEL: shr_and_add:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a1, a1, 9
+; RV64I-NEXT:    slli a1, a1, 2
+; RV64I-NEXT:    addw a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: shr_and_add:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    srliw a1, a1, 9
+; RV64ZBA-NEXT:    sh2add a0, a1, a0
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: shr_and_add:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    srliw a1, a1, 9
+; RV64XANDESPERF-NEXT:    nds.lea.w a0, a0, a1
+; RV64XANDESPERF-NEXT:    sext.w a0, a0
+; RV64XANDESPERF-NEXT:    ret
+  %lshr = lshr i32 %y, 7
+  %and = and i32 %lshr, 33554428
+  %add = add i32 %x, %and
+  ret i32 %add
+}
+
+define ptr @udiv1280_gep(ptr %p, i16 zeroext %i) {
+; RV64I-LABEL: udiv1280_gep:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a2, 13
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    mul a1, a1, a2
+; RV64I-NEXT:    srliw a1, a1, 26
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: udiv1280_gep:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    lui a2, 13
+; RV64ZBA-NEXT:    addi a2, a2, -819
+; RV64ZBA-NEXT:    mul a1, a1, a2
+; RV64ZBA-NEXT:    srliw a1, a1, 26
+; RV64ZBA-NEXT:    sh3add a0, a1, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: udiv1280_gep:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    lui a2, 13
+; RV64XANDESPERF-NEXT:    addi a2, a2, -819
+; RV64XANDESPERF-NEXT:    mul a1, a1, a2
+; RV64XANDESPERF-NEXT:    srliw a1, a1, 26
+; RV64XANDESPERF-NEXT:    nds.lea.d a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %udiv = udiv i16 %i, 1280
+  %idx.ext = zext nneg i16 %udiv to i64
+  %add.ptr = getelementptr i64, ptr %p, i64 %idx.ext
+  ret ptr %add.ptr
+}
+
+define i64 @adduw_m1(i64 %x) {
+; RV64I-LABEL: adduw_m1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, -1
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: adduw_m1:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    li a1, -1
+; RV64ZBA-NEXT:    add.uw a0, a1, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: adduw_m1:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    li a1, -1
+; RV64XANDESPERF-NEXT:    nds.lea.b.ze a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %a = add i64 %x, 4294967295
+  ret i64 %a
+}
+
+define i64 @adduw_m3(i64 %x) {
+; RV64I-LABEL: adduw_m3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    addi a1, a1, -3
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: adduw_m3:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    li a1, -3
+; RV64ZBA-NEXT:    add.uw a0, a1, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: adduw_m3:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    li a1, -3
+; RV64XANDESPERF-NEXT:    nds.lea.b.ze a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %a = add i64 %x, 4294967293
+  ret i64 %a
+}
+
+define i64 @adduw_3shl30(i64 %x) {
+; RV64I-LABEL: adduw_3shl30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 3
+; RV64I-NEXT:    slli a1, a1, 30
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: adduw_3shl30:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    lui a1, 786432
+; RV64ZBA-NEXT:    add.uw a0, a1, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: adduw_3shl30:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    lui a1, 786432
+; RV64XANDESPERF-NEXT:    nds.lea.b.ze a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %a = add i64 %x, 3221225472
+  ret i64 %a
+}
+
+define i64 @adduw_m3_multiuse(i64 %x, i64 %y) {
+; RV64I-LABEL: adduw_m3_multiuse:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a2, 1
+; RV64I-NEXT:    slli a2, a2, 32
+; RV64I-NEXT:    addi a2, a2, -3
+; RV64I-NEXT:    add a0, a0, a2
+; RV64I-NEXT:    add a1, a1, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: adduw_m3_multiuse:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    li a2, -3
+; RV64ZBA-NEXT:    add.uw a0, a2, a0
+; RV64ZBA-NEXT:    add.uw a1, a2, a1
+; RV64ZBA-NEXT:    or a0, a0, a1
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: adduw_m3_multiuse:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    li a2, -3
+; RV64XANDESPERF-NEXT:    nds.lea.b.ze a0, a0, a2
+; RV64XANDESPERF-NEXT:    nds.lea.b.ze a1, a1, a2
+; RV64XANDESPERF-NEXT:    or a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %a = add i64 %x, 4294967293
+  %b = add i64 %y, 4294967293
+  %c = or i64 %a, %b
+  ret i64 %c
+}
+
+define i64 @add_or_m3(i64 %x) {
+; RV64I-LABEL: add_or_m3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    addi a1, a1, -3
+; RV64I-NEXT:    or a2, a0, a1
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    add a0, a0, a2
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: add_or_m3:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    li a1, -3
+; RV64ZBA-NEXT:    zext.w a1, a1
+; RV64ZBA-NEXT:    or a2, a0, a1
+; RV64ZBA-NEXT:    add a0, a0, a1
+; RV64ZBA-NEXT:    add a0, a0, a2
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: add_or_m3:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    li a1, 1
+; RV64XANDESPERF-NEXT:    slli a1, a1, 32
+; RV64XANDESPERF-NEXT:    addi a1, a1, -3
+; RV64XANDESPERF-NEXT:    or a2, a0, a1
+; RV64XANDESPERF-NEXT:    add a0, a0, a1
+; RV64XANDESPERF-NEXT:    add a0, a0, a2
+; RV64XANDESPERF-NEXT:    ret
+  %a = add i64 %x, 4294967293
+  %o = or i64 %x, 4294967293
+  %c = add i64 %a, %o
+  ret i64 %c
+}
+
+define i64 @append_32ones(i64 %x) {
+; RV64I-LABEL: append_32ones:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    li a1, -1
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: append_32ones:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    slli a0, a0, 32
+; RV64ZBA-NEXT:    li a1, -1
+; RV64ZBA-NEXT:    add.uw a0, a1, a0
+; RV64ZBA-NEXT:    ret
+;
+; RV64XANDESPERF-LABEL: append_32ones:
+; RV64XANDESPERF:       # %bb.0:
+; RV64XANDESPERF-NEXT:    slli a0, a0, 32
+; RV64XANDESPERF-NEXT:    li a1, -1
+; RV64XANDESPERF-NEXT:    nds.lea.b.ze a0, a0, a1
+; RV64XANDESPERF-NEXT:    ret
+  %s = shl i64 %x, 32
+  %o = or i64 %s, 4294967295
+  ret i64 %o
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb-intrinsic.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb-intrinsic.ll
index 08972fe20d16..500d51be80a6 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb-intrinsic.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb-intrinsic.ll
@@ -62,7 +62,7 @@ define i64 @orcb64_knownbits(i64 %a) nounwind {
 ; RV64ZBB-NEXT:    lui a1, 65535
 ; RV64ZBB-NEXT:    lui a2, 4080
 ; RV64ZBB-NEXT:    slli a1, a1, 12
-; RV64ZBB-NEXT:    addiw a2, a2, 255
+; RV64ZBB-NEXT:    addi a2, a2, 255
 ; RV64ZBB-NEXT:    and a0, a0, a1
 ; RV64ZBB-NEXT:    slli a1, a2, 40
 ; RV64ZBB-NEXT:    orc.b a0, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb.ll
index 97b8b2aa5e52..e6407279870d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -14,7 +14,7 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -27,7 +27,7 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -64,7 +64,7 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -77,7 +77,7 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -123,7 +123,7 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a2, a1, 1
 ; RV64I-NEXT:    lui a3, 349525
 ; RV64I-NEXT:    or a1, a1, a2
-; RV64I-NEXT:    addiw a2, a3, 1365
+; RV64I-NEXT:    addi a2, a3, 1365
 ; RV64I-NEXT:    srliw a3, a1, 2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srliw a3, a1, 4
@@ -136,7 +136,7 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a3, a1, 1
 ; RV64I-NEXT:    and a2, a3, a2
 ; RV64I-NEXT:    lui a3, 209715
-; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    addi a3, a3, 819
 ; RV64I-NEXT:    sub a1, a1, a2
 ; RV64I-NEXT:    and a2, a1, a3
 ; RV64I-NEXT:    srli a1, a1, 2
@@ -175,7 +175,7 @@ define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a1, a0, a1
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    srliw a3, a1, 2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srliw a3, a1, 4
@@ -188,7 +188,7 @@ define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a3, a1, 1
 ; RV64I-NEXT:    and a2, a3, a2
 ; RV64I-NEXT:    lui a3, 209715
-; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    addi a3, a3, 819
 ; RV64I-NEXT:    sub a1, a1, a2
 ; RV64I-NEXT:    and a2, a1, a3
 ; RV64I-NEXT:    srli a1, a1, 2
@@ -234,7 +234,7 @@ define i32 @ctlz_lshr_i32(i32 signext %a) {
 ; RV64I-NEXT:    srliw a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srliw a2, a0, 2
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    srliw a2, a0, 4
@@ -247,7 +247,7 @@ define i32 @ctlz_lshr_i32(i32 signext %a) {
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -289,8 +289,8 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    lui a2, 349525
 ; RV64I-NEXT:    lui a3, 209715
 ; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
-; RV64I-NEXT:    addiw a2, a3, 819
+; RV64I-NEXT:    addi a1, a2, 1365
+; RV64I-NEXT:    addi a2, a3, 819
 ; RV64I-NEXT:    srli a3, a0, 2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a3, a1, 32
@@ -309,7 +309,7 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 1
 ; RV64I-NEXT:    and a1, a3, a1
 ; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -567,10 +567,10 @@ define signext i32 @ctpop_i32(i32 signext %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    addi a2, a2, 1365
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -677,11 +677,11 @@ define signext i32 @ctpop_i32_load(ptr %p) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lw a0, 0(a0)
 ; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    addiw a1, a1, 1365
+; RV64I-NEXT:    addi a1, a1, 1365
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -718,11 +718,11 @@ define <2 x i32> @ctpop_v2i32(<2 x i32> %a) nounwind {
 ; RV64I-NEXT:    lui a3, 349525
 ; RV64I-NEXT:    lui a4, 209715
 ; RV64I-NEXT:    srli a5, a1, 1
-; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    addi a3, a3, 1365
 ; RV64I-NEXT:    and a2, a2, a3
 ; RV64I-NEXT:    and a3, a5, a3
 ; RV64I-NEXT:    lui a5, 61681
-; RV64I-NEXT:    addiw a4, a4, 819
+; RV64I-NEXT:    addi a4, a4, 819
 ; RV64I-NEXT:    addi a5, a5, -241
 ; RV64I-NEXT:    sub a0, a0, a2
 ; RV64I-NEXT:    sub a1, a1, a3
@@ -876,8 +876,8 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 349525
 ; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    slli a3, a1, 32
 ; RV64I-NEXT:    add a1, a1, a3
 ; RV64I-NEXT:    slli a3, a2, 32
@@ -885,7 +885,7 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a3, a0, 1
 ; RV64I-NEXT:    and a1, a3, a1
 ; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
@@ -998,9 +998,9 @@ define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
 ; RV64I-NEXT:    lui a3, 349525
 ; RV64I-NEXT:    lui a4, 209715
 ; RV64I-NEXT:    lui a5, 61681
-; RV64I-NEXT:    addiw a3, a3, 1365
-; RV64I-NEXT:    addiw a4, a4, 819
-; RV64I-NEXT:    addiw a5, a5, -241
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    addi a4, a4, 819
+; RV64I-NEXT:    addi a5, a5, -241
 ; RV64I-NEXT:    slli a6, a3, 32
 ; RV64I-NEXT:    add a3, a3, a6
 ; RV64I-NEXT:    slli a6, a4, 32
@@ -1453,7 +1453,7 @@ define signext i32 @bswap_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 8
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    srliw a3, a0, 24
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a2, a0, a2
 ; RV64I-NEXT:    or a1, a1, a3
@@ -1511,7 +1511,7 @@ define i64 @bswap_i64(i64 %a) {
 ; RV64I-NEXT:    srli a3, a0, 56
 ; RV64I-NEXT:    srli a4, a0, 24
 ; RV64I-NEXT:    lui a5, 4080
-; RV64I-NEXT:    addiw a2, a2, -256
+; RV64I-NEXT:    addi a2, a2, -256
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srli a3, a0, 8
@@ -1572,7 +1572,7 @@ define i32 @orc_b_i32(i32 %a) {
 ; RV64ZBB-LABEL: orc_b_i32:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    lui a1, 4112
-; RV64ZBB-NEXT:    addiw a1, a1, 257
+; RV64ZBB-NEXT:    addi a1, a1, 257
 ; RV64ZBB-NEXT:    and a0, a0, a1
 ; RV64ZBB-NEXT:    orc.b a0, a0
 ; RV64ZBB-NEXT:    ret
@@ -1585,7 +1585,7 @@ define i64 @orc_b_i64(i64 %a) {
 ; RV64I-LABEL: orc_b_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    slli a2, a1, 32
 ; RV64I-NEXT:    add a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a1
@@ -1596,7 +1596,7 @@ define i64 @orc_b_i64(i64 %a) {
 ; RV64ZBB-LABEL: orc_b_i64:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    lui a1, 4112
-; RV64ZBB-NEXT:    addiw a1, a1, 257
+; RV64ZBB-NEXT:    addi a1, a1, 257
 ; RV64ZBB-NEXT:    slli a2, a1, 32
 ; RV64ZBB-NEXT:    add a1, a1, a2
 ; RV64ZBB-NEXT:    and a0, a0, a1
@@ -1755,7 +1755,7 @@ define i16 @sub_if_uge_i16(i16 %x, i16 %y) {
 ; RV64I-LABEL: sub_if_uge_i16:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a3, a1, a2
 ; RV64I-NEXT:    and a2, a0, a2
 ; RV64I-NEXT:    sltu a2, a2, a3
@@ -1978,7 +1978,7 @@ define i32 @sub_if_uge_C_i32(i32 signext %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 16
 ; RV64I-NEXT:    lui a2, 1048560
-; RV64I-NEXT:    addiw a1, a1, -16
+; RV64I-NEXT:    addi a1, a1, -16
 ; RV64I-NEXT:    sltu a1, a1, a0
 ; RV64I-NEXT:    negw a1, a1
 ; RV64I-NEXT:    addi a2, a2, 15
@@ -2004,8 +2004,8 @@ define i64 @sub_if_uge_C_i64(i64 %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 298
 ; RV64I-NEXT:    lui a2, 1046192
-; RV64I-NEXT:    addiw a1, a1, 95
-; RV64I-NEXT:    addiw a2, a2, -761
+; RV64I-NEXT:    addi a1, a1, 95
+; RV64I-NEXT:    addi a2, a2, -761
 ; RV64I-NEXT:    slli a1, a1, 12
 ; RV64I-NEXT:    addi a1, a1, 511
 ; RV64I-NEXT:    sltu a1, a1, a0
@@ -2018,7 +2018,7 @@ define i64 @sub_if_uge_C_i64(i64 %x) {
 ; RV64ZBB-LABEL: sub_if_uge_C_i64:
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    lui a1, 1046192
-; RV64ZBB-NEXT:    addiw a1, a1, -761
+; RV64ZBB-NEXT:    addi a1, a1, -761
 ; RV64ZBB-NEXT:    slli a1, a1, 9
 ; RV64ZBB-NEXT:    add a1, a0, a1
 ; RV64ZBB-NEXT:    minu a0, a1, a0
@@ -2034,7 +2034,7 @@ define i32 @sub_if_uge_C_multiuse_cmp_i32(i32 signext %x, ptr %z) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    lui a3, 1048560
-; RV64I-NEXT:    addiw a2, a2, -16
+; RV64I-NEXT:    addi a2, a2, -16
 ; RV64I-NEXT:    sltu a2, a2, a0
 ; RV64I-NEXT:    negw a4, a2
 ; RV64I-NEXT:    addi a3, a3, 15
@@ -2047,7 +2047,7 @@ define i32 @sub_if_uge_C_multiuse_cmp_i32(i32 signext %x, ptr %z) {
 ; RV64ZBB:       # %bb.0:
 ; RV64ZBB-NEXT:    lui a2, 16
 ; RV64ZBB-NEXT:    lui a3, 1048560
-; RV64ZBB-NEXT:    addiw a2, a2, -16
+; RV64ZBB-NEXT:    addi a2, a2, -16
 ; RV64ZBB-NEXT:    addi a3, a3, 15
 ; RV64ZBB-NEXT:    sltu a2, a2, a0
 ; RV64ZBB-NEXT:    addw a3, a0, a3
@@ -2069,7 +2069,7 @@ define i32 @sub_if_uge_C_multiuse_sub_i32(i32 signext %x, ptr %z) {
 ; RV64I-NEXT:    lui a3, 16
 ; RV64I-NEXT:    addi a2, a2, 15
 ; RV64I-NEXT:    addw a2, a0, a2
-; RV64I-NEXT:    addiw a3, a3, -16
+; RV64I-NEXT:    addi a3, a3, -16
 ; RV64I-NEXT:    sw a2, 0(a1)
 ; RV64I-NEXT:    bltu a3, a0, .LBB75_2
 ; RV64I-NEXT:  # %bb.1:
@@ -2098,7 +2098,7 @@ define i32 @sub_if_uge_C_swapped_i32(i32 signext %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 16
 ; RV64I-NEXT:    lui a2, 1048560
-; RV64I-NEXT:    addiw a1, a1, -15
+; RV64I-NEXT:    addi a1, a1, -15
 ; RV64I-NEXT:    sltu a1, a0, a1
 ; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    addi a2, a2, 15
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbkb.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbkb.ll
index 734bffee96f0..696c2a5e0f80 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbkb.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbkb.ll
@@ -319,7 +319,7 @@ define i64 @pack_i64_imm() {
 ; RV64I-LABEL: pack_i64_imm:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a0, 65793
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    slli a1, a0, 32
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbs.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbs.ll
index c1e1e16d0d4a..a8b06b2af576 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rv64zbs.ll
@@ -513,7 +513,7 @@ define signext i32 @bclri_i32_11(i32 signext %a) nounwind {
 ; RV64I-LABEL: bclri_i32_11:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    addi a1, a1, 2047
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -529,7 +529,7 @@ define signext i32 @bclri_i32_30(i32 signext %a) nounwind {
 ; RV64I-LABEL: bclri_i32_30:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 786432
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -564,7 +564,7 @@ define i64 @bclri_i64_11(i64 %a) nounwind {
 ; RV64I-LABEL: bclri_i64_11:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1048575
-; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    addi a1, a1, 2047
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -580,7 +580,7 @@ define i64 @bclri_i64_30(i64 %a) nounwind {
 ; RV64I-LABEL: bclri_i64_30:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 786432
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -644,7 +644,7 @@ define i64 @bclri_i64_large0(i64 %a) nounwind {
 ; RV64I-LABEL: bclri_i64_large0:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1044480
-; RV64I-NEXT:    addiw a1, a1, -256
+; RV64I-NEXT:    addi a1, a1, -256
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -661,7 +661,7 @@ define i64 @bclri_i64_large1(i64 %a) nounwind {
 ; RV64I-LABEL: bclri_i64_large1:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1044464
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -972,7 +972,7 @@ define i64 @xor_i64_4099(i64 %a) nounwind {
 ; RV64I-LABEL: xor_i64_4099:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 3
+; RV64I-NEXT:    addi a1, a1, 3
 ; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -1016,7 +1016,7 @@ define i64 @xor_i64_66901(i64 %a) nounwind {
 ; RV64I-LABEL: xor_i64_66901:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 1365
+; RV64I-NEXT:    addi a1, a1, 1365
 ; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -1033,7 +1033,7 @@ define i64 @or_i64_4099(i64 %a) nounwind {
 ; RV64I-LABEL: or_i64_4099:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 3
+; RV64I-NEXT:    addi a1, a1, 3
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -1059,7 +1059,7 @@ define i64 @or_i64_66901(i64 %a) nounwind {
 ; RV64I-LABEL: or_i64_66901:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 1365
+; RV64I-NEXT:    addi a1, a1, 1365
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
@@ -1203,7 +1203,7 @@ define i1 @icmp_eq_nonpow2(i32 signext %x) nounwind {
 ; CHECK-LABEL: icmp_eq_nonpow2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a1, 8
-; CHECK-NEXT:    addiw a1, a1, -1
+; CHECK-NEXT:    addi a1, a1, -1
 ; CHECK-NEXT:    xor a0, a0, a1
 ; CHECK-NEXT:    seqz a0, a0
 ; CHECK-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
index 4d955d46f2fd..a75c159339be 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
@@ -782,7 +782,7 @@ define <vscale x 1 x i64> @bitreverse_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v10, v8, a1
 ; RV64-NEXT:    vsrl.vx v11, v8, a0
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v11, v11, a2
 ; RV64-NEXT:    vor.vv v10, v11, v10
 ; RV64-NEXT:    vsrl.vi v11, v8, 8
@@ -801,9 +801,9 @@ define <vscale x 1 x i64> @bitreverse_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64-NEXT:    vor.vv v10, v11, v10
 ; RV64-NEXT:    vsll.vx v11, v8, a1
 ; RV64-NEXT:    lui a1, 349525
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 819
+; RV64-NEXT:    addi a1, a1, 1365
 ; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    slli a2, a3, 32
 ; RV64-NEXT:    vsll.vx v8, v8, a0
@@ -923,7 +923,7 @@ define <vscale x 2 x i64> @bitreverse_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v12, v8, a1
 ; RV64-NEXT:    vsrl.vx v14, v8, a0
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v14, v14, a2
 ; RV64-NEXT:    vor.vv v12, v14, v12
 ; RV64-NEXT:    vsrl.vi v14, v8, 8
@@ -942,9 +942,9 @@ define <vscale x 2 x i64> @bitreverse_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64-NEXT:    vor.vv v12, v14, v12
 ; RV64-NEXT:    vsll.vx v14, v8, a1
 ; RV64-NEXT:    lui a1, 349525
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 819
+; RV64-NEXT:    addi a1, a1, 1365
 ; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    slli a2, a3, 32
 ; RV64-NEXT:    vsll.vx v8, v8, a0
@@ -1064,7 +1064,7 @@ define <vscale x 4 x i64> @bitreverse_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v12, v8, a1
 ; RV64-NEXT:    vsrl.vx v20, v8, a0
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v20, v20, a2
 ; RV64-NEXT:    vor.vv v12, v20, v12
 ; RV64-NEXT:    vsrl.vi v20, v8, 8
@@ -1083,9 +1083,9 @@ define <vscale x 4 x i64> @bitreverse_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64-NEXT:    vor.vv v16, v16, v20
 ; RV64-NEXT:    vsll.vx v20, v8, a1
 ; RV64-NEXT:    lui a1, 349525
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 819
+; RV64-NEXT:    addi a1, a1, 1365
 ; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    slli a2, a3, 32
 ; RV64-NEXT:    vsll.vx v8, v8, a0
@@ -1227,7 +1227,7 @@ define <vscale x 8 x i64> @bitreverse_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v16, v8, a1
 ; RV64-NEXT:    vsrl.vx v0, v8, a0
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v0, v0, a2
 ; RV64-NEXT:    vor.vv v16, v0, v16
 ; RV64-NEXT:    vsrl.vi v0, v8, 8
@@ -1246,9 +1246,9 @@ define <vscale x 8 x i64> @bitreverse_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64-NEXT:    vor.vv v24, v24, v0
 ; RV64-NEXT:    vsll.vx v0, v8, a1
 ; RV64-NEXT:    lui a1, 349525
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 819
+; RV64-NEXT:    addi a1, a1, 1365
 ; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    slli a2, a3, 32
 ; RV64-NEXT:    vsll.vx v8, v8, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
index a68efe992921..f704a8ca875b 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
@@ -1512,9 +1512,9 @@ define <vscale x 1 x i64> @vp_bitreverse_nxv1i64(<vscale x 1 x i64> %va, <vscale
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 209715
 ; RV64-NEXT:    lui a7, 349525
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 819
-; RV64-NEXT:    addiw a7, a7, 1365
+; RV64-NEXT:    addi a5, a5, -241
+; RV64-NEXT:    addi a6, a6, 819
+; RV64-NEXT:    addi a7, a7, 1365
 ; RV64-NEXT:    slli t0, a5, 32
 ; RV64-NEXT:    add t0, a5, t0
 ; RV64-NEXT:    slli a5, a6, 32
@@ -1525,7 +1525,7 @@ define <vscale x 1 x i64> @vp_bitreverse_nxv1i64(<vscale x 1 x i64> %va, <vscale
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
 ; RV64-NEXT:    slli a3, a3, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v9, v9, 24, v0.t
 ; RV64-NEXT:    vand.vx v10, v8, a3, v0.t
 ; RV64-NEXT:    vsll.vi v10, v10, 8, v0.t
@@ -1653,7 +1653,7 @@ define <vscale x 1 x i64> @vp_bitreverse_nxv1i64_unmasked(<vscale x 1 x i64> %va
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vsrl.vi v9, v8, 24
 ; RV64-NEXT:    vsrl.vi v10, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v11, v8, a3
 ; RV64-NEXT:    vsrl.vx v12, v8, a5
 ; RV64-NEXT:    vand.vx v12, v12, a0
@@ -1674,9 +1674,9 @@ define <vscale x 1 x i64> @vp_bitreverse_nxv1i64_unmasked(<vscale x 1 x i64> %va
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -1794,9 +1794,9 @@ define <vscale x 2 x i64> @vp_bitreverse_nxv2i64(<vscale x 2 x i64> %va, <vscale
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 209715
 ; RV64-NEXT:    lui a7, 349525
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 819
-; RV64-NEXT:    addiw a7, a7, 1365
+; RV64-NEXT:    addi a5, a5, -241
+; RV64-NEXT:    addi a6, a6, 819
+; RV64-NEXT:    addi a7, a7, 1365
 ; RV64-NEXT:    slli t0, a5, 32
 ; RV64-NEXT:    add t0, a5, t0
 ; RV64-NEXT:    slli a5, a6, 32
@@ -1807,7 +1807,7 @@ define <vscale x 2 x i64> @vp_bitreverse_nxv2i64(<vscale x 2 x i64> %va, <vscale
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vand.vx v10, v8, a1, v0.t
 ; RV64-NEXT:    slli a3, a3, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v10, v10, 24, v0.t
 ; RV64-NEXT:    vand.vx v12, v8, a3, v0.t
 ; RV64-NEXT:    vsll.vi v12, v12, 8, v0.t
@@ -1935,7 +1935,7 @@ define <vscale x 2 x i64> @vp_bitreverse_nxv2i64_unmasked(<vscale x 2 x i64> %va
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vsrl.vi v12, v8, 24
 ; RV64-NEXT:    vsrl.vi v14, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v10, v8, a3
 ; RV64-NEXT:    vsrl.vx v16, v8, a5
 ; RV64-NEXT:    vand.vx v16, v16, a0
@@ -1956,9 +1956,9 @@ define <vscale x 2 x i64> @vp_bitreverse_nxv2i64_unmasked(<vscale x 2 x i64> %va
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -2076,9 +2076,9 @@ define <vscale x 4 x i64> @vp_bitreverse_nxv4i64(<vscale x 4 x i64> %va, <vscale
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 209715
 ; RV64-NEXT:    lui a7, 349525
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 819
-; RV64-NEXT:    addiw a7, a7, 1365
+; RV64-NEXT:    addi a5, a5, -241
+; RV64-NEXT:    addi a6, a6, 819
+; RV64-NEXT:    addi a7, a7, 1365
 ; RV64-NEXT:    slli t0, a5, 32
 ; RV64-NEXT:    add t0, a5, t0
 ; RV64-NEXT:    slli a5, a6, 32
@@ -2089,7 +2089,7 @@ define <vscale x 4 x i64> @vp_bitreverse_nxv4i64(<vscale x 4 x i64> %va, <vscale
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vand.vx v12, v8, a1, v0.t
 ; RV64-NEXT:    slli a3, a3, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v12, v12, 24, v0.t
 ; RV64-NEXT:    vand.vx v16, v8, a3, v0.t
 ; RV64-NEXT:    vsll.vi v16, v16, 8, v0.t
@@ -2217,7 +2217,7 @@ define <vscale x 4 x i64> @vp_bitreverse_nxv4i64_unmasked(<vscale x 4 x i64> %va
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vsrl.vi v16, v8, 24
 ; RV64-NEXT:    vsrl.vi v20, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v12, v8, a3
 ; RV64-NEXT:    vsrl.vx v24, v8, a5
 ; RV64-NEXT:    vand.vx v24, v24, a0
@@ -2238,9 +2238,9 @@ define <vscale x 4 x i64> @vp_bitreverse_nxv4i64_unmasked(<vscale x 4 x i64> %va
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -2414,7 +2414,7 @@ define <vscale x 7 x i64> @vp_bitreverse_nxv7i64(<vscale x 7 x i64> %va, <vscale
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -2441,9 +2441,9 @@ define <vscale x 7 x i64> @vp_bitreverse_nxv7i64(<vscale x 7 x i64> %va, <vscale
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -2593,7 +2593,7 @@ define <vscale x 7 x i64> @vp_bitreverse_nxv7i64_unmasked(<vscale x 7 x i64> %va
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
@@ -2617,9 +2617,9 @@ define <vscale x 7 x i64> @vp_bitreverse_nxv7i64_unmasked(<vscale x 7 x i64> %va
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -2801,7 +2801,7 @@ define <vscale x 8 x i64> @vp_bitreverse_nxv8i64(<vscale x 8 x i64> %va, <vscale
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -2828,9 +2828,9 @@ define <vscale x 8 x i64> @vp_bitreverse_nxv8i64(<vscale x 8 x i64> %va, <vscale
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -2980,7 +2980,7 @@ define <vscale x 8 x i64> @vp_bitreverse_nxv8i64_unmasked(<vscale x 8 x i64> %va
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
@@ -3004,9 +3004,9 @@ define <vscale x 8 x i64> @vp_bitreverse_nxv8i64_unmasked(<vscale x 8 x i64> %va
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll
index 5bd15df031da..b8521c37e490 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll
@@ -304,7 +304,7 @@ define <vscale x 1 x i64> @bswap_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v10, v8, a0
 ; RV64-NEXT:    vsrl.vx v11, v8, a1
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v11, v11, a2
 ; RV64-NEXT:    vor.vv v10, v11, v10
 ; RV64-NEXT:    vsrl.vi v11, v8, 8
@@ -388,7 +388,7 @@ define <vscale x 2 x i64> @bswap_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v12, v8, a0
 ; RV64-NEXT:    vsrl.vx v14, v8, a1
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v14, v14, a2
 ; RV64-NEXT:    vor.vv v12, v14, v12
 ; RV64-NEXT:    vsrl.vi v14, v8, 8
@@ -472,7 +472,7 @@ define <vscale x 4 x i64> @bswap_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v16, v8, a0
 ; RV64-NEXT:    vsrl.vx v20, v8, a1
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v20, v20, a2
 ; RV64-NEXT:    vor.vv v16, v20, v16
 ; RV64-NEXT:    vsrl.vi v20, v8, 8
@@ -578,7 +578,7 @@ define <vscale x 8 x i64> @bswap_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64-NEXT:    lui a3, 4080
 ; RV64-NEXT:    vsrl.vx v16, v8, a0
 ; RV64-NEXT:    vsrl.vx v0, v8, a1
-; RV64-NEXT:    addiw a2, a2, -256
+; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vand.vx v0, v0, a2
 ; RV64-NEXT:    vor.vv v16, v0, v16
 ; RV64-NEXT:    vsrl.vi v0, v8, 8
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
index ca637432cc0c..3d31cf80cdd3 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
@@ -559,7 +559,7 @@ define <vscale x 1 x i64> @vp_bswap_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v9, v9, 24, v0.t
 ; RV64-NEXT:    vand.vx v10, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v10, v10, 8, v0.t
@@ -642,7 +642,7 @@ define <vscale x 1 x i64> @vp_bswap_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vsrl.vi v9, v8, 24
 ; RV64-NEXT:    vsrl.vi v10, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v11, v8, a3
 ; RV64-NEXT:    vsrl.vx v12, v8, a5
 ; RV64-NEXT:    vand.vx v12, v12, a0
@@ -727,7 +727,7 @@ define <vscale x 2 x i64> @vp_bswap_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vand.vx v10, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v10, v10, 24, v0.t
 ; RV64-NEXT:    vand.vx v12, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v12, v12, 8, v0.t
@@ -810,7 +810,7 @@ define <vscale x 2 x i64> @vp_bswap_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vsrl.vi v10, v8, 24
 ; RV64-NEXT:    vsrl.vi v12, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v14, v8, a3
 ; RV64-NEXT:    vsrl.vx v16, v8, a5
 ; RV64-NEXT:    vand.vx v16, v16, a0
@@ -895,7 +895,7 @@ define <vscale x 4 x i64> @vp_bswap_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vand.vx v12, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v12, v12, 24, v0.t
 ; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v16, v16, 8, v0.t
@@ -978,7 +978,7 @@ define <vscale x 4 x i64> @vp_bswap_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vsrl.vi v12, v8, 24
 ; RV64-NEXT:    vsrl.vi v16, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v20, v8, a3
 ; RV64-NEXT:    vsrl.vx v24, v8, a5
 ; RV64-NEXT:    vand.vx v24, v24, a0
@@ -1118,7 +1118,7 @@ define <vscale x 7 x i64> @vp_bswap_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -1239,7 +1239,7 @@ define <vscale x 7 x i64> @vp_bswap_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
@@ -1390,7 +1390,7 @@ define <vscale x 8 x i64> @vp_bswap_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -1511,7 +1511,7 @@ define <vscale x 8 x i64> @vp_bswap_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
@@ -1709,7 +1709,7 @@ define <vscale x 1 x i48> @vp_bswap_nxv1i48(<vscale x 1 x i48> %va, <vscale x 1
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v9, v9, 24, v0.t
 ; RV64-NEXT:    vand.vx v10, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v10, v10, 8, v0.t
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
index 88894f887cc2..5dc532273b77 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
@@ -313,12 +313,12 @@ define i32 @test_nxv128i1(<vscale x 128 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v0, v6, a0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v7, a1
+; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v7, a0
 ; CHECK-NEXT:    vslidedown.vx v5, v6, a0
-; CHECK-NEXT:    vslidedown.vx v4, v7, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vmv1r.v v0, v4
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v5
 ; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
@@ -364,9 +364,9 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vmv1r.v v7, v9
 ; CHECK-NEXT:    vmv1r.v v5, v8
 ; CHECK-NEXT:    vmv1r.v v4, v0
-; CHECK-NEXT:    vmv.v.i v16, 0
+; CHECK-NEXT:    vmv.v.i v24, 0
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    mv a2, a0
@@ -376,7 +376,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v0, v5
-; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
@@ -388,9 +388,8 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v3, v4, a0
 ; CHECK-NEXT:    vslidedown.vx v2, v5, a0
 ; CHECK-NEXT:    vmv.v.v v0, v3
-; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    mv a3, a2
@@ -398,42 +397,43 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    add a2, a2, a3
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v16, (a2) # vscale x 64-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v0, v2
-; CHECK-NEXT:    vmerge.vim v16, v8, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 4
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v16, (a2) # vscale x 64-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v3, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v16, v8, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v16, (a2) # vscale x 64-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v2, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v24, v8, 1, v0
+; CHECK-NEXT:    vmerge.vim v16, v24, 1, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v4, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v16, v8, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v5, a1
+; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmerge.vim v24, v24, 1, v0
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v6, a1
 ; CHECK-NEXT:    vslidedown.vx v5, v7, a1
-; CHECK-NEXT:    vslidedown.vx v4, v6, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vmv1r.v v0, v4
-; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
+; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v5
-; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
-; CHECK-NEXT:    vadd.vv v8, v16, v8
+; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
+; CHECK-NEXT:    vadd.vv v8, v8, v24
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
@@ -443,7 +443,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v0, v4, a1
 ; CHECK-NEXT:    vslidedown.vx v3, v5, a1
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
+; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v3
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -451,7 +451,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vadd.vv v8, v8, v24
+; CHECK-NEXT:    vadd.vv v8, v8, v16
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
@@ -492,16 +492,16 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
-; CHECK-NEXT:    vadd.vv v24, v24, v8
+; CHECK-NEXT:    vadd.vv v0, v24, v8
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v0, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vadd.vv v8, v8, v0
-; CHECK-NEXT:    vadd.vv v16, v24, v16
+; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    vadd.vv v8, v8, v24
+; CHECK-NEXT:    vadd.vv v16, v0, v16
 ; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    vmv.s.x v16, zero
 ; CHECK-NEXT:    vredsum.vs v8, v8, v16
@@ -537,18 +537,17 @@ entry:
 define i16 @test_narrow_nxv64i1(<vscale x 64 x i1> %x) {
 ; CHECK-LABEL: test_narrow_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
-; CHECK-NEXT:    vmv.v.i v16, 0
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v0, a0
+; CHECK-NEXT:    vslidedown.vx v0, v0, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, mu
-; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
-; CHECK-NEXT:    vmv1r.v v0, v8
-; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
-; CHECK-NEXT:    vmv.s.x v8, zero
-; CHECK-NEXT:    vredsum.vs v8, v16, v8
+; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
+; CHECK-NEXT:    vmv.s.x v16, zero
+; CHECK-NEXT:    vredsum.vs v8, v8, v16
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll
index 97e1a7f41b92..319d82f724ca 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll
@@ -1189,10 +1189,10 @@ define <vscale x 1 x i64> @ctlz_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1329,10 +1329,10 @@ define <vscale x 2 x i64> @ctlz_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1469,10 +1469,10 @@ define <vscale x 4 x i64> @ctlz_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1609,10 +1609,10 @@ define <vscale x 8 x i64> @ctlz_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2791,10 +2791,10 @@ define <vscale x 1 x i64> @ctlz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2925,10 +2925,10 @@ define <vscale x 2 x i64> @ctlz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -3059,10 +3059,10 @@ define <vscale x 4 x i64> @ctlz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -3193,10 +3193,10 @@ define <vscale x 8 x i64> @ctlz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll
index fa8e332e5076..1018130b472d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-sdnode.ll
@@ -717,10 +717,10 @@ define <vscale x 1 x i64> @ctpop_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -800,10 +800,10 @@ define <vscale x 2 x i64> @ctpop_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -883,10 +883,10 @@ define <vscale x 4 x i64> @ctpop_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -966,10 +966,10 @@ define <vscale x 8 x i64> @ctpop_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll
index c70d5b9954f9..fba27e3d548c 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/ctpop-vp.ll
@@ -1227,10 +1227,10 @@ define <vscale x 1 x i64> @vp_ctpop_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1309,10 +1309,10 @@ define <vscale x 1 x i64> @vp_ctpop_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1393,10 +1393,10 @@ define <vscale x 2 x i64> @vp_ctpop_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1475,10 +1475,10 @@ define <vscale x 2 x i64> @vp_ctpop_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1559,10 +1559,10 @@ define <vscale x 4 x i64> @vp_ctpop_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1641,10 +1641,10 @@ define <vscale x 4 x i64> @vp_ctpop_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1725,10 +1725,10 @@ define <vscale x 7 x i64> @vp_ctpop_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1807,10 +1807,10 @@ define <vscale x 7 x i64> @vp_ctpop_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1891,10 +1891,10 @@ define <vscale x 8 x i64> @vp_ctpop_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1973,10 +1973,10 @@ define <vscale x 8 x i64> @vp_ctpop_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -2165,10 +2165,10 @@ define <vscale x 16 x i64> @vp_ctpop_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV64-NEXT:    lui a3, 209715
 ; RV64-NEXT:    lui a4, 61681
 ; RV64-NEXT:    lui a5, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a4, a4, -241
-; RV64-NEXT:    addiw a5, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a4, a4, -241
+; RV64-NEXT:    addi a5, a5, 257
 ; RV64-NEXT:    slli a6, a2, 32
 ; RV64-NEXT:    add a2, a2, a6
 ; RV64-NEXT:    slli a6, a3, 32
@@ -2356,10 +2356,10 @@ define <vscale x 16 x i64> @vp_ctpop_nxv16i64_unmasked(<vscale x 16 x i64> %va,
 ; RV64-NEXT:    lui a4, 209715
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    addiw a3, a3, 1365
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 257
+; RV64-NEXT:    addi a3, a3, 1365
+; RV64-NEXT:    addi a4, a4, 819
+; RV64-NEXT:    addi a5, a5, -241
+; RV64-NEXT:    addi a6, a6, 257
 ; RV64-NEXT:    slli a7, a3, 32
 ; RV64-NEXT:    add a3, a3, a7
 ; RV64-NEXT:    slli a7, a4, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
index 09aa487cb085..faa3c48c49e5 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
@@ -1162,10 +1162,10 @@ define <vscale x 1 x i64> @cttz_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1286,10 +1286,10 @@ define <vscale x 2 x i64> @cttz_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1410,10 +1410,10 @@ define <vscale x 4 x i64> @cttz_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1534,10 +1534,10 @@ define <vscale x 8 x i64> @cttz_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2666,10 +2666,10 @@ define <vscale x 1 x i64> @cttz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2782,10 +2782,10 @@ define <vscale x 2 x i64> @cttz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2898,10 +2898,10 @@ define <vscale x 4 x i64> @cttz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -3014,10 +3014,10 @@ define <vscale x 8 x i64> @cttz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64I-NEXT:    lui a1, 209715
 ; RV64I-NEXT:    lui a2, 61681
 ; RV64I-NEXT:    lui a3, 4112
-; RV64I-NEXT:    addiw a0, a0, 1365
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    addiw a2, a2, -241
-; RV64I-NEXT:    addiw a3, a3, 257
+; RV64I-NEXT:    addi a0, a0, 1365
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    addi a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
index 230a32754879..6bf882fe47fe 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
@@ -1338,10 +1338,10 @@ define <vscale x 1 x i64> @vp_cttz_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1426,10 +1426,10 @@ define <vscale x 1 x i64> @vp_cttz_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1516,10 +1516,10 @@ define <vscale x 2 x i64> @vp_cttz_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1604,10 +1604,10 @@ define <vscale x 2 x i64> @vp_cttz_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1694,10 +1694,10 @@ define <vscale x 4 x i64> @vp_cttz_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4 x
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1782,10 +1782,10 @@ define <vscale x 4 x i64> @vp_cttz_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1872,10 +1872,10 @@ define <vscale x 7 x i64> @vp_cttz_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7 x
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1960,10 +1960,10 @@ define <vscale x 7 x i64> @vp_cttz_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -2050,10 +2050,10 @@ define <vscale x 8 x i64> @vp_cttz_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8 x
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -2138,10 +2138,10 @@ define <vscale x 8 x i64> @vp_cttz_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -2334,10 +2334,10 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV64-NEXT:    lui a5, 4112
 ; RV64-NEXT:    srli a6, a1, 3
 ; RV64-NEXT:    sub a7, a0, a1
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a4, a4, -241
-; RV64-NEXT:    addiw t0, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a4, a4, -241
+; RV64-NEXT:    addi t0, a5, 257
 ; RV64-NEXT:    vslidedown.vx v0, v0, a6
 ; RV64-NEXT:    slli a6, a2, 32
 ; RV64-NEXT:    add a6, a2, a6
@@ -2537,10 +2537,10 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64_unmasked(<vscale x 16 x i64> %va, i
 ; RV64-NEXT:    lui a4, 61681
 ; RV64-NEXT:    lui a5, 4112
 ; RV64-NEXT:    sub a6, a0, a1
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a7, a4, -241
-; RV64-NEXT:    addiw t0, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a7, a4, -241
+; RV64-NEXT:    addi t0, a5, 257
 ; RV64-NEXT:    slli a5, a2, 32
 ; RV64-NEXT:    add a5, a2, a5
 ; RV64-NEXT:    slli a4, a3, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/extractelt-int-rv64.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/extractelt-int-rv64.ll
index a9e129ef11a2..22bb7bf31724 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/extractelt-int-rv64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/extractelt-int-rv64.ll
@@ -797,7 +797,7 @@ define i32 @extractelt_sdiv_nxv4i32_splat(<vscale x 4 x i32> %x) {
 ; RV64M-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64M-NEXT:    vmv.x.s a0, v8
 ; RV64M-NEXT:    lui a1, 349525
-; RV64M-NEXT:    addiw a1, a1, 1366
+; RV64M-NEXT:    addi a1, a1, 1366
 ; RV64M-NEXT:    mul a0, a0, a1
 ; RV64M-NEXT:    srli a1, a0, 63
 ; RV64M-NEXT:    srli a0, a0, 32
@@ -825,7 +825,7 @@ define i32 @extractelt_udiv_nxv4i32_splat(<vscale x 4 x i32> %x) {
 ; RV64M-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64M-NEXT:    vmv.x.s a0, v8
 ; RV64M-NEXT:    lui a1, 349525
-; RV64M-NEXT:    addiw a1, a1, 1366
+; RV64M-NEXT:    addi a1, a1, 1366
 ; RV64M-NEXT:    mul a0, a0, a1
 ; RV64M-NEXT:    srli a1, a0, 63
 ; RV64M-NEXT:    srli a0, a0, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
index 6ab688ebbf66..3d83065009f2 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
@@ -911,9 +911,9 @@ define <2 x i64> @vp_bitreverse_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %e
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 209715
 ; RV64-NEXT:    lui a7, 349525
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 819
-; RV64-NEXT:    addiw a7, a7, 1365
+; RV64-NEXT:    addi a5, a5, -241
+; RV64-NEXT:    addi a6, a6, 819
+; RV64-NEXT:    addi a7, a7, 1365
 ; RV64-NEXT:    slli t0, a5, 32
 ; RV64-NEXT:    add t0, a5, t0
 ; RV64-NEXT:    slli a5, a6, 32
@@ -924,7 +924,7 @@ define <2 x i64> @vp_bitreverse_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %e
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
 ; RV64-NEXT:    slli a3, a3, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v9, v9, 24, v0.t
 ; RV64-NEXT:    vand.vx v10, v8, a3, v0.t
 ; RV64-NEXT:    vsll.vi v10, v10, 8, v0.t
@@ -1048,7 +1048,7 @@ define <2 x i64> @vp_bitreverse_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl)
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vsrl.vi v9, v8, 24
 ; RV64-NEXT:    vsrl.vi v10, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v11, v8, a3
 ; RV64-NEXT:    vsrl.vx v12, v8, a5
 ; RV64-NEXT:    vand.vx v12, v12, a0
@@ -1069,9 +1069,9 @@ define <2 x i64> @vp_bitreverse_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl)
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -1184,9 +1184,9 @@ define <4 x i64> @vp_bitreverse_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %e
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 209715
 ; RV64-NEXT:    lui a7, 349525
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 819
-; RV64-NEXT:    addiw a7, a7, 1365
+; RV64-NEXT:    addi a5, a5, -241
+; RV64-NEXT:    addi a6, a6, 819
+; RV64-NEXT:    addi a7, a7, 1365
 ; RV64-NEXT:    slli t0, a5, 32
 ; RV64-NEXT:    add t0, a5, t0
 ; RV64-NEXT:    slli a5, a6, 32
@@ -1197,7 +1197,7 @@ define <4 x i64> @vp_bitreverse_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %e
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vand.vx v10, v8, a1, v0.t
 ; RV64-NEXT:    slli a3, a3, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v10, v10, 24, v0.t
 ; RV64-NEXT:    vand.vx v12, v8, a3, v0.t
 ; RV64-NEXT:    vsll.vi v12, v12, 8, v0.t
@@ -1321,7 +1321,7 @@ define <4 x i64> @vp_bitreverse_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl)
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vsrl.vi v12, v8, 24
 ; RV64-NEXT:    vsrl.vi v14, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v10, v8, a3
 ; RV64-NEXT:    vsrl.vx v16, v8, a5
 ; RV64-NEXT:    vand.vx v16, v16, a0
@@ -1342,9 +1342,9 @@ define <4 x i64> @vp_bitreverse_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl)
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -1457,9 +1457,9 @@ define <8 x i64> @vp_bitreverse_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %e
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 209715
 ; RV64-NEXT:    lui a7, 349525
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 819
-; RV64-NEXT:    addiw a7, a7, 1365
+; RV64-NEXT:    addi a5, a5, -241
+; RV64-NEXT:    addi a6, a6, 819
+; RV64-NEXT:    addi a7, a7, 1365
 ; RV64-NEXT:    slli t0, a5, 32
 ; RV64-NEXT:    add t0, a5, t0
 ; RV64-NEXT:    slli a5, a6, 32
@@ -1470,7 +1470,7 @@ define <8 x i64> @vp_bitreverse_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %e
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vand.vx v12, v8, a1, v0.t
 ; RV64-NEXT:    slli a3, a3, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v12, v12, 24, v0.t
 ; RV64-NEXT:    vand.vx v16, v8, a3, v0.t
 ; RV64-NEXT:    vsll.vi v16, v16, 8, v0.t
@@ -1594,7 +1594,7 @@ define <8 x i64> @vp_bitreverse_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl)
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vsrl.vi v16, v8, 24
 ; RV64-NEXT:    vsrl.vi v20, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v12, v8, a3
 ; RV64-NEXT:    vsrl.vx v24, v8, a5
 ; RV64-NEXT:    vand.vx v24, v24, a0
@@ -1615,9 +1615,9 @@ define <8 x i64> @vp_bitreverse_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl)
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -1787,7 +1787,7 @@ define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroex
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -1814,9 +1814,9 @@ define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroex
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -1962,7 +1962,7 @@ define <15 x i64> @vp_bitreverse_v15i64_unmasked(<15 x i64> %va, i32 zeroext %ev
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
@@ -1986,9 +1986,9 @@ define <15 x i64> @vp_bitreverse_v15i64_unmasked(<15 x i64> %va, i32 zeroext %ev
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -2166,7 +2166,7 @@ define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroex
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -2193,9 +2193,9 @@ define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroex
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
@@ -2341,7 +2341,7 @@ define <16 x i64> @vp_bitreverse_v16i64_unmasked(<16 x i64> %va, i32 zeroext %ev
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
@@ -2365,9 +2365,9 @@ define <16 x i64> @vp_bitreverse_v16i64_unmasked(<16 x i64> %va, i32 zeroext %ev
 ; RV64-NEXT:    lui a0, 61681
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    addiw a0, a0, -241
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addi a0, a0, -241
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, 1365
 ; RV64-NEXT:    slli a3, a0, 32
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    add a0, a0, a3
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
index 946ca4d1ab90..6d9793c12153 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
@@ -188,7 +188,7 @@ define void @bitreverse_v2i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a3, 16
 ; RV64-NEXT:    lui a4, 4080
 ; RV64-NEXT:    li a5, 255
-; RV64-NEXT:    addiw a3, a3, -256
+; RV64-NEXT:    addi a3, a3, -256
 ; RV64-NEXT:    slli a5, a5, 24
 ; RV64-NEXT:    vsrl.vx v9, v8, a1
 ; RV64-NEXT:    vsrl.vx v10, v8, a2
@@ -211,9 +211,9 @@ define void @bitreverse_v2i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a1, 61681
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 349525
-; RV64-NEXT:    addiw a1, a1, -241
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, 1365
+; RV64-NEXT:    addi a1, a1, -241
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, 1365
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    slli a5, a2, 32
 ; RV64-NEXT:    add a1, a1, a4
@@ -440,7 +440,7 @@ define void @bitreverse_v4i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a3, 16
 ; RV64-NEXT:    lui a4, 4080
 ; RV64-NEXT:    li a5, 255
-; RV64-NEXT:    addiw a3, a3, -256
+; RV64-NEXT:    addi a3, a3, -256
 ; RV64-NEXT:    slli a5, a5, 24
 ; RV64-NEXT:    vsrl.vx v8, v14, a1
 ; RV64-NEXT:    vsrl.vx v10, v14, a2
@@ -463,9 +463,9 @@ define void @bitreverse_v4i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a1, 61681
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 349525
-; RV64-NEXT:    addiw a1, a1, -241
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, 1365
+; RV64-NEXT:    addi a1, a1, -241
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, 1365
 ; RV64-NEXT:    slli a4, a1, 32
 ; RV64-NEXT:    slli a5, a2, 32
 ; RV64-NEXT:    add a1, a1, a4
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll
index 4eb189d13278..b7ca932bb1c4 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll
@@ -331,7 +331,7 @@ define <2 x i64> @vp_bswap_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v9, v9, 24, v0.t
 ; RV64-NEXT:    vand.vx v10, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v10, v10, 8, v0.t
@@ -410,7 +410,7 @@ define <2 x i64> @vp_bswap_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV64-NEXT:    vsrl.vi v9, v8, 24
 ; RV64-NEXT:    vsrl.vi v10, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v11, v8, a3
 ; RV64-NEXT:    vsrl.vx v12, v8, a5
 ; RV64-NEXT:    vand.vx v12, v12, a0
@@ -491,7 +491,7 @@ define <4 x i64> @vp_bswap_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vand.vx v10, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v10, v10, 24, v0.t
 ; RV64-NEXT:    vand.vx v12, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v12, v12, 8, v0.t
@@ -570,7 +570,7 @@ define <4 x i64> @vp_bswap_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV64-NEXT:    vsrl.vi v10, v8, 24
 ; RV64-NEXT:    vsrl.vi v12, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v14, v8, a3
 ; RV64-NEXT:    vsrl.vx v16, v8, a5
 ; RV64-NEXT:    vand.vx v16, v16, a0
@@ -651,7 +651,7 @@ define <8 x i64> @vp_bswap_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vand.vx v12, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v12, v12, 24, v0.t
 ; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v16, v16, 8, v0.t
@@ -730,7 +730,7 @@ define <8 x i64> @vp_bswap_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV64-NEXT:    vsrl.vi v12, v8, 24
 ; RV64-NEXT:    vsrl.vi v16, v8, 8
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v20, v8, a3
 ; RV64-NEXT:    vsrl.vx v24, v8, a5
 ; RV64-NEXT:    vand.vx v24, v24, a0
@@ -866,7 +866,7 @@ define <15 x i64> @vp_bswap_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %ev
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -984,7 +984,7 @@ define <15 x i64> @vp_bswap_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
@@ -1131,7 +1131,7 @@ define <16 x i64> @vp_bswap_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %ev
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    slli a2, a2, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsll.vi v16, v16, 24, v0.t
 ; RV64-NEXT:    vand.vx v24, v8, a2, v0.t
 ; RV64-NEXT:    vsll.vi v24, v24, 8, v0.t
@@ -1249,7 +1249,7 @@ define <16 x i64> @vp_bswap_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    li a5, 40
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 24
-; RV64-NEXT:    addiw a0, a4, -256
+; RV64-NEXT:    addi a0, a4, -256
 ; RV64-NEXT:    vsrl.vx v16, v8, a3
 ; RV64-NEXT:    vsrl.vx v0, v8, a5
 ; RV64-NEXT:    vand.vx v0, v0, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
index 5e491f21e621..5b823442c8b0 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
@@ -116,7 +116,7 @@ define void @bswap_v2i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a3, 16
 ; RV64-NEXT:    lui a4, 4080
 ; RV64-NEXT:    li a5, 255
-; RV64-NEXT:    addiw a3, a3, -256
+; RV64-NEXT:    addi a3, a3, -256
 ; RV64-NEXT:    slli a5, a5, 24
 ; RV64-NEXT:    vsrl.vx v9, v8, a1
 ; RV64-NEXT:    vsrl.vx v10, v8, a2
@@ -269,7 +269,7 @@ define void @bswap_v4i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a3, 16
 ; RV64-NEXT:    lui a4, 4080
 ; RV64-NEXT:    li a5, 255
-; RV64-NEXT:    addiw a3, a3, -256
+; RV64-NEXT:    addi a3, a3, -256
 ; RV64-NEXT:    slli a5, a5, 24
 ; RV64-NEXT:    vsrl.vx v10, v8, a1
 ; RV64-NEXT:    vsrl.vx v12, v8, a2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
index 8adf87bb27d0..2250ab5bd0bb 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
@@ -931,10 +931,10 @@ define <2 x i64> @vp_ctlz_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1035,10 +1035,10 @@ define <2 x i64> @vp_ctlz_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1141,10 +1141,10 @@ define <4 x i64> @vp_ctlz_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1245,10 +1245,10 @@ define <4 x i64> @vp_ctlz_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1351,10 +1351,10 @@ define <8 x i64> @vp_ctlz_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1455,10 +1455,10 @@ define <8 x i64> @vp_ctlz_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1561,10 +1561,10 @@ define <15 x i64> @vp_ctlz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1665,10 +1665,10 @@ define <15 x i64> @vp_ctlz_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1771,10 +1771,10 @@ define <16 x i64> @vp_ctlz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -1875,10 +1875,10 @@ define <16 x i64> @vp_ctlz_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -2101,10 +2101,10 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV64-NEXT:    lui a3, 209715
 ; RV64-NEXT:    lui a4, 61681
 ; RV64-NEXT:    lui a5, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a6, a4, -241
-; RV64-NEXT:    addiw a7, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a6, a4, -241
+; RV64-NEXT:    addi a7, a5, 257
 ; RV64-NEXT:    slli a5, a2, 32
 ; RV64-NEXT:    add a5, a2, a5
 ; RV64-NEXT:    slli a4, a3, 32
@@ -2321,10 +2321,10 @@ define <32 x i64> @vp_ctlz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a4, 209715
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    addiw a7, a3, 1365
-; RV64-NEXT:    addiw a3, a4, 819
-; RV64-NEXT:    addiw a4, a5, -241
-; RV64-NEXT:    addiw a6, a6, 257
+; RV64-NEXT:    addi a7, a3, 1365
+; RV64-NEXT:    addi a3, a4, 819
+; RV64-NEXT:    addi a4, a5, -241
+; RV64-NEXT:    addi a6, a6, 257
 ; RV64-NEXT:    slli a5, a7, 32
 ; RV64-NEXT:    add a7, a7, a5
 ; RV64-NEXT:    slli a5, a3, 32
@@ -3316,10 +3316,10 @@ define <2 x i64> @vp_ctlz_zero_undef_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroe
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -3420,10 +3420,10 @@ define <2 x i64> @vp_ctlz_zero_undef_v2i64_unmasked(<2 x i64> %va, i32 zeroext %
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -3524,10 +3524,10 @@ define <4 x i64> @vp_ctlz_zero_undef_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroe
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -3628,10 +3628,10 @@ define <4 x i64> @vp_ctlz_zero_undef_v4i64_unmasked(<4 x i64> %va, i32 zeroext %
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -3732,10 +3732,10 @@ define <8 x i64> @vp_ctlz_zero_undef_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroe
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -3836,10 +3836,10 @@ define <8 x i64> @vp_ctlz_zero_undef_v8i64_unmasked(<8 x i64> %va, i32 zeroext %
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -3940,10 +3940,10 @@ define <15 x i64> @vp_ctlz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 z
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -4044,10 +4044,10 @@ define <15 x i64> @vp_ctlz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroex
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -4148,10 +4148,10 @@ define <16 x i64> @vp_ctlz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 z
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -4252,10 +4252,10 @@ define <16 x i64> @vp_ctlz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroex
 ; RV64-NEXT:    lui a1, 209715
 ; RV64-NEXT:    lui a2, 61681
 ; RV64-NEXT:    lui a3, 4112
-; RV64-NEXT:    addiw a0, a0, 1365
-; RV64-NEXT:    addiw a1, a1, 819
-; RV64-NEXT:    addiw a2, a2, -241
-; RV64-NEXT:    addiw a3, a3, 257
+; RV64-NEXT:    addi a0, a0, 1365
+; RV64-NEXT:    addi a1, a1, 819
+; RV64-NEXT:    addi a2, a2, -241
+; RV64-NEXT:    addi a3, a3, 257
 ; RV64-NEXT:    slli a4, a0, 32
 ; RV64-NEXT:    add a0, a0, a4
 ; RV64-NEXT:    slli a4, a1, 32
@@ -4476,10 +4476,10 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV64-NEXT:    lui a3, 209715
 ; RV64-NEXT:    lui a4, 61681
 ; RV64-NEXT:    lui a5, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a6, a4, -241
-; RV64-NEXT:    addiw a7, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a6, a4, -241
+; RV64-NEXT:    addi a7, a5, 257
 ; RV64-NEXT:    slli a5, a2, 32
 ; RV64-NEXT:    add a5, a2, a5
 ; RV64-NEXT:    slli a4, a3, 32
@@ -4696,10 +4696,10 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroex
 ; RV64-NEXT:    lui a4, 209715
 ; RV64-NEXT:    lui a5, 61681
 ; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    addiw a7, a3, 1365
-; RV64-NEXT:    addiw a3, a4, 819
-; RV64-NEXT:    addiw a4, a5, -241
-; RV64-NEXT:    addiw a6, a6, 257
+; RV64-NEXT:    addi a7, a3, 1365
+; RV64-NEXT:    addi a3, a4, 819
+; RV64-NEXT:    addi a4, a5, -241
+; RV64-NEXT:    addi a6, a6, 257
 ; RV64-NEXT:    slli a5, a7, 32
 ; RV64-NEXT:    add a7, a7, a5
 ; RV64-NEXT:    slli a5, a3, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
index 829cc9bac7ed..61730b87c551 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
@@ -305,10 +305,10 @@ define void @ctlz_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
@@ -696,10 +696,10 @@ define void @ctlz_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
@@ -1068,10 +1068,10 @@ define void @ctlz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
@@ -1438,10 +1438,10 @@ define void @ctlz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
index 7a0897c9da41..94fecbdfde18 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
@@ -699,10 +699,10 @@ define <2 x i64> @vp_ctpop_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -775,10 +775,10 @@ define <2 x i64> @vp_ctpop_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -853,10 +853,10 @@ define <4 x i64> @vp_ctpop_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -929,10 +929,10 @@ define <4 x i64> @vp_ctpop_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1007,10 +1007,10 @@ define <8 x i64> @vp_ctpop_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1083,10 +1083,10 @@ define <8 x i64> @vp_ctpop_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1161,10 +1161,10 @@ define <15 x i64> @vp_ctpop_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %ev
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1237,10 +1237,10 @@ define <15 x i64> @vp_ctpop_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1315,10 +1315,10 @@ define <16 x i64> @vp_ctpop_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %ev
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1391,10 +1391,10 @@ define <16 x i64> @vp_ctpop_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1577,10 +1577,10 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1708,10 +1708,10 @@ define <32 x i64> @vp_ctpop_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a3, 209715
 ; RV64-NEXT:    lui a4, 61681
 ; RV64-NEXT:    lui a5, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a4, a4, -241
-; RV64-NEXT:    addiw a5, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a4, a4, -241
+; RV64-NEXT:    addi a5, a5, 257
 ; RV64-NEXT:    slli a6, a2, 32
 ; RV64-NEXT:    add a2, a2, a6
 ; RV64-NEXT:    slli a6, a3, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll
index 4fbe67cfcd64..44b9331fd2ca 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll
@@ -173,10 +173,10 @@ define void @ctpop_v2i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -479,10 +479,10 @@ define void @ctpop_v4i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
index 6cee5a6d24ad..bdce00b10e5a 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
@@ -774,10 +774,10 @@ define <2 x i64> @vp_cttz_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -856,10 +856,10 @@ define <2 x i64> @vp_cttz_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -940,10 +940,10 @@ define <4 x i64> @vp_cttz_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1022,10 +1022,10 @@ define <4 x i64> @vp_cttz_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1106,10 +1106,10 @@ define <8 x i64> @vp_cttz_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1188,10 +1188,10 @@ define <8 x i64> @vp_cttz_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1272,10 +1272,10 @@ define <15 x i64> @vp_cttz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1354,10 +1354,10 @@ define <15 x i64> @vp_cttz_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1438,10 +1438,10 @@ define <16 x i64> @vp_cttz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1520,10 +1520,10 @@ define <16 x i64> @vp_cttz_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -1715,10 +1715,10 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a5, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a5, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a3, a1, 32
 ; RV64-NEXT:    add a6, a1, a3
 ; RV64-NEXT:    slli a3, a2, 32
@@ -1884,10 +1884,10 @@ define <32 x i64> @vp_cttz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:    lui a3, 209715
 ; RV64-NEXT:    lui a4, 61681
 ; RV64-NEXT:    lui a5, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a4, a4, -241
-; RV64-NEXT:    addiw a5, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a4, a4, -241
+; RV64-NEXT:    addi a5, a5, 257
 ; RV64-NEXT:    slli a6, a2, 32
 ; RV64-NEXT:    add a2, a2, a6
 ; RV64-NEXT:    slli a6, a3, 32
@@ -2689,10 +2689,10 @@ define <2 x i64> @vp_cttz_zero_undef_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroe
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -2771,10 +2771,10 @@ define <2 x i64> @vp_cttz_zero_undef_v2i64_unmasked(<2 x i64> %va, i32 zeroext %
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -2853,10 +2853,10 @@ define <4 x i64> @vp_cttz_zero_undef_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroe
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -2935,10 +2935,10 @@ define <4 x i64> @vp_cttz_zero_undef_v4i64_unmasked(<4 x i64> %va, i32 zeroext %
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -3017,10 +3017,10 @@ define <8 x i64> @vp_cttz_zero_undef_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroe
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -3099,10 +3099,10 @@ define <8 x i64> @vp_cttz_zero_undef_v8i64_unmasked(<8 x i64> %va, i32 zeroext %
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -3181,10 +3181,10 @@ define <15 x i64> @vp_cttz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 z
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -3263,10 +3263,10 @@ define <15 x i64> @vp_cttz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroex
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -3345,10 +3345,10 @@ define <16 x i64> @vp_cttz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 z
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -3427,10 +3427,10 @@ define <16 x i64> @vp_cttz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroex
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a3, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a3, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a5, a1, 32
 ; RV64-NEXT:    add a1, a1, a5
 ; RV64-NEXT:    slli a5, a2, 32
@@ -3620,10 +3620,10 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV64-NEXT:    lui a2, 209715
 ; RV64-NEXT:    lui a3, 61681
 ; RV64-NEXT:    lui a4, 4112
-; RV64-NEXT:    addiw a1, a1, 1365
-; RV64-NEXT:    addiw a2, a2, 819
-; RV64-NEXT:    addiw a5, a3, -241
-; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    addi a2, a2, 819
+; RV64-NEXT:    addi a5, a3, -241
+; RV64-NEXT:    addi a4, a4, 257
 ; RV64-NEXT:    slli a3, a1, 32
 ; RV64-NEXT:    add a6, a1, a3
 ; RV64-NEXT:    slli a3, a2, 32
@@ -3789,10 +3789,10 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroex
 ; RV64-NEXT:    lui a3, 209715
 ; RV64-NEXT:    lui a4, 61681
 ; RV64-NEXT:    lui a5, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a4, a4, -241
-; RV64-NEXT:    addiw a5, a5, 257
+; RV64-NEXT:    addi a2, a2, 1365
+; RV64-NEXT:    addi a3, a3, 819
+; RV64-NEXT:    addi a4, a4, -241
+; RV64-NEXT:    addi a5, a5, 257
 ; RV64-NEXT:    slli a6, a2, 32
 ; RV64-NEXT:    add a2, a2, a6
 ; RV64-NEXT:    slli a6, a3, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
index d884cece8950..307b143f4449 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
@@ -294,10 +294,10 @@ define void @cttz_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
@@ -671,10 +671,10 @@ define void @cttz_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
@@ -1025,10 +1025,10 @@ define void @cttz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
@@ -1373,10 +1373,10 @@ define void @cttz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw a1, a1, 1365
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    addiw a3, a3, -241
-; RV64I-NEXT:    addiw a4, a4, 257
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    addi a4, a4, 257
 ; RV64I-NEXT:    slli a5, a1, 32
 ; RV64I-NEXT:    add a1, a1, a5
 ; RV64I-NEXT:    slli a5, a2, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll
index e13f4f4b50b0..651894a1bb66 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll
@@ -26,26 +26,26 @@ define void @add_v4i32(ptr %x, ptr %y) {
 define void @add_v2i64(ptr %x, ptr %y) {
 ; RV32-LABEL: add_v2i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lw a2, 0(a1)
-; RV32-NEXT:    lw a3, 4(a1)
-; RV32-NEXT:    lw a4, 0(a0)
-; RV32-NEXT:    lw a5, 4(a0)
-; RV32-NEXT:    lw a6, 8(a0)
-; RV32-NEXT:    lw a7, 12(a0)
+; RV32-NEXT:    lw a2, 0(a0)
+; RV32-NEXT:    lw a3, 4(a0)
+; RV32-NEXT:    lw a4, 8(a0)
+; RV32-NEXT:    lw a5, 12(a0)
+; RV32-NEXT:    lw a6, 4(a1)
+; RV32-NEXT:    lw a7, 0(a1)
 ; RV32-NEXT:    lw t0, 12(a1)
 ; RV32-NEXT:    lw a1, 8(a1)
-; RV32-NEXT:    add a3, a5, a3
-; RV32-NEXT:    add a2, a4, a2
-; RV32-NEXT:    add a7, a7, t0
-; RV32-NEXT:    add a1, a6, a1
-; RV32-NEXT:    sltu a4, a2, a4
-; RV32-NEXT:    sltu a5, a1, a6
-; RV32-NEXT:    add a3, a3, a4
-; RV32-NEXT:    add a5, a7, a5
-; RV32-NEXT:    sw a2, 0(a0)
-; RV32-NEXT:    sw a3, 4(a0)
+; RV32-NEXT:    add a3, a3, a6
+; RV32-NEXT:    add a7, a2, a7
+; RV32-NEXT:    add a5, a5, t0
+; RV32-NEXT:    add a1, a4, a1
+; RV32-NEXT:    sltu a2, a7, a2
+; RV32-NEXT:    sltu a4, a1, a4
+; RV32-NEXT:    add a2, a3, a2
+; RV32-NEXT:    add a4, a5, a4
+; RV32-NEXT:    sw a7, 0(a0)
+; RV32-NEXT:    sw a2, 4(a0)
 ; RV32-NEXT:    sw a1, 8(a0)
-; RV32-NEXT:    sw a5, 12(a0)
+; RV32-NEXT:    sw a4, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: add_v2i64:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
index 75732fe2f7e6..dba5d26c216f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
@@ -1368,7 +1368,7 @@ define i32 @extractelt_sdiv_v4i32(<4 x i32> %x) {
 ; RV64M-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64M-NEXT:    lui a0, 322639
 ; RV64M-NEXT:    vmv.x.s a1, v8
-; RV64M-NEXT:    addiw a0, a0, -945
+; RV64M-NEXT:    addi a0, a0, -945
 ; RV64M-NEXT:    mul a0, a1, a0
 ; RV64M-NEXT:    srli a1, a0, 63
 ; RV64M-NEXT:    srai a0, a0, 34
@@ -1381,7 +1381,7 @@ define i32 @extractelt_sdiv_v4i32(<4 x i32> %x) {
 ; VISNI-NEXT:    ri.vextract.x.v a0, v8, 2
 ; VISNI-NEXT:    lui a1, 322639
 ; VISNI-NEXT:    sext.w a0, a0
-; VISNI-NEXT:    addiw a1, a1, -945
+; VISNI-NEXT:    addi a1, a1, -945
 ; VISNI-NEXT:    mul a0, a0, a1
 ; VISNI-NEXT:    srli a1, a0, 63
 ; VISNI-NEXT:    srai a0, a0, 34
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index c12e322f4780..1fa96d3c07ca 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -668,7 +668,7 @@ define void @buildvec_seq2_v16i8_v2i64(ptr %x) {
 ; RV64V-LABEL: buildvec_seq2_v16i8_v2i64:
 ; RV64V:       # %bb.0:
 ; RV64V-NEXT:    lui a1, 528432
-; RV64V-NEXT:    addiw a1, a1, 513
+; RV64V-NEXT:    addi a1, a1, 513
 ; RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64V-NEXT:    vmv.v.x v8, a1
 ; RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
@@ -1406,28 +1406,28 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; RV32VB-NEXT:    slli a7, a7, 16
 ; RV32VB-NEXT:    slli t0, t0, 24
 ; RV32VB-NEXT:    slli a5, a5, 8
-; RV32VB-NEXT:    or a7, t0, a7
-; RV32VB-NEXT:    or a4, a4, a5
-; RV32VB-NEXT:    lbu a5, 12(a0)
-; RV32VB-NEXT:    lbu t0, 13(a0)
 ; RV32VB-NEXT:    slli a6, a6, 16
 ; RV32VB-NEXT:    slli t1, t1, 24
-; RV32VB-NEXT:    or a6, t1, a6
+; RV32VB-NEXT:    or a7, t0, a7
+; RV32VB-NEXT:    or a4, a4, a5
+; RV32VB-NEXT:    or a5, t1, a6
+; RV32VB-NEXT:    lbu a6, 13(a0)
+; RV32VB-NEXT:    lbu t0, 12(a0)
 ; RV32VB-NEXT:    lbu t1, 14(a0)
 ; RV32VB-NEXT:    lbu a0, 15(a0)
-; RV32VB-NEXT:    slli t0, t0, 8
-; RV32VB-NEXT:    or a5, a5, t0
+; RV32VB-NEXT:    slli a6, a6, 8
+; RV32VB-NEXT:    or a6, t0, a6
 ; RV32VB-NEXT:    slli t1, t1, 16
 ; RV32VB-NEXT:    slli a0, a0, 24
 ; RV32VB-NEXT:    or a0, a0, t1
 ; RV32VB-NEXT:    or a1, a1, a3
 ; RV32VB-NEXT:    or a2, a2, a7
-; RV32VB-NEXT:    or a3, a4, a6
-; RV32VB-NEXT:    or a0, a5, a0
+; RV32VB-NEXT:    or a4, a4, a5
+; RV32VB-NEXT:    or a0, a6, a0
 ; RV32VB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32VB-NEXT:    vmv.v.x v8, a1
 ; RV32VB-NEXT:    vslide1down.vx v8, v8, a2
-; RV32VB-NEXT:    vslide1down.vx v8, v8, a3
+; RV32VB-NEXT:    vslide1down.vx v8, v8, a4
 ; RV32VB-NEXT:    vslide1down.vx v8, v8, a0
 ; RV32VB-NEXT:    ret
 ;
@@ -1767,36 +1767,36 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; RV32VB-NEXT:    lbu a7, 55(a0)
 ; RV32VB-NEXT:    lbu t0, 75(a0)
 ; RV32VB-NEXT:    lbu t1, 82(a0)
-; RV32VB-NEXT:    lbu t2, 154(a0)
-; RV32VB-NEXT:    lbu t3, 161(a0)
 ; RV32VB-NEXT:    slli a2, a2, 8
 ; RV32VB-NEXT:    slli a3, a3, 16
 ; RV32VB-NEXT:    slli a4, a4, 24
 ; RV32VB-NEXT:    slli a7, a7, 8
-; RV32VB-NEXT:    slli a5, a5, 16
-; RV32VB-NEXT:    slli t0, t0, 24
 ; RV32VB-NEXT:    or a1, a1, a2
 ; RV32VB-NEXT:    or a3, a4, a3
 ; RV32VB-NEXT:    or a2, a6, a7
-; RV32VB-NEXT:    or a4, t0, a5
-; RV32VB-NEXT:    lbu a5, 93(a0)
+; RV32VB-NEXT:    lbu a4, 93(a0)
 ; RV32VB-NEXT:    lbu a6, 105(a0)
 ; RV32VB-NEXT:    lbu a7, 124(a0)
-; RV32VB-NEXT:    lbu t0, 144(a0)
-; RV32VB-NEXT:    slli a5, a5, 8
-; RV32VB-NEXT:    or a5, t1, a5
+; RV32VB-NEXT:    lbu t2, 144(a0)
+; RV32VB-NEXT:    slli a5, a5, 16
+; RV32VB-NEXT:    slli t0, t0, 24
+; RV32VB-NEXT:    slli a4, a4, 8
+; RV32VB-NEXT:    or a5, t0, a5
+; RV32VB-NEXT:    or a4, t1, a4
+; RV32VB-NEXT:    lbu t0, 161(a0)
+; RV32VB-NEXT:    lbu t1, 154(a0)
 ; RV32VB-NEXT:    lbu a0, 163(a0)
 ; RV32VB-NEXT:    slli a6, a6, 16
-; RV32VB-NEXT:    slli t3, t3, 24
-; RV32VB-NEXT:    or a6, t3, a6
+; RV32VB-NEXT:    slli t0, t0, 24
+; RV32VB-NEXT:    or a6, t0, a6
 ; RV32VB-NEXT:    slli a0, a0, 8
 ; RV32VB-NEXT:    or a0, a7, a0
-; RV32VB-NEXT:    slli t0, t0, 16
-; RV32VB-NEXT:    slli t2, t2, 24
-; RV32VB-NEXT:    or a7, t2, t0
+; RV32VB-NEXT:    slli t2, t2, 16
+; RV32VB-NEXT:    slli t1, t1, 24
+; RV32VB-NEXT:    or a7, t1, t2
 ; RV32VB-NEXT:    or a1, a1, a3
-; RV32VB-NEXT:    or a2, a2, a4
-; RV32VB-NEXT:    or a3, a5, a6
+; RV32VB-NEXT:    or a2, a2, a5
+; RV32VB-NEXT:    or a3, a4, a6
 ; RV32VB-NEXT:    or a0, a0, a7
 ; RV32VB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32VB-NEXT:    vmv.v.x v8, a1
@@ -1893,50 +1893,50 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ;
 ; RVA22U64-LABEL: buildvec_v16i8_loads_gather:
 ; RVA22U64:       # %bb.0:
-; RVA22U64-NEXT:    lbu a6, 0(a0)
-; RVA22U64-NEXT:    lbu a7, 1(a0)
-; RVA22U64-NEXT:    lbu t0, 22(a0)
-; RVA22U64-NEXT:    lbu t1, 31(a0)
-; RVA22U64-NEXT:    lbu t2, 623(a0)
-; RVA22U64-NEXT:    lbu a1, 44(a0)
-; RVA22U64-NEXT:    lbu a2, 55(a0)
-; RVA22U64-NEXT:    lbu a3, 75(a0)
-; RVA22U64-NEXT:    lbu t3, 82(a0)
-; RVA22U64-NEXT:    lbu t4, 154(a0)
-; RVA22U64-NEXT:    lbu t5, 161(a0)
-; RVA22U64-NEXT:    slli a7, a7, 8
-; RVA22U64-NEXT:    slli t0, t0, 16
-; RVA22U64-NEXT:    slli t1, t1, 24
-; RVA22U64-NEXT:    slli a1, a1, 32
-; RVA22U64-NEXT:    slli a2, a2, 40
-; RVA22U64-NEXT:    slli t2, t2, 48
-; RVA22U64-NEXT:    slli a3, a3, 56
-; RVA22U64-NEXT:    or a6, a6, a7
-; RVA22U64-NEXT:    or t0, t1, t0
-; RVA22U64-NEXT:    or a7, a2, a1
-; RVA22U64-NEXT:    or a2, a3, t2
-; RVA22U64-NEXT:    lbu a3, 93(a0)
-; RVA22U64-NEXT:    lbu a5, 105(a0)
-; RVA22U64-NEXT:    lbu a1, 124(a0)
-; RVA22U64-NEXT:    lbu a4, 144(a0)
-; RVA22U64-NEXT:    slli a3, a3, 8
-; RVA22U64-NEXT:    or a3, t3, a3
+; RVA22U64-NEXT:    lbu a7, 0(a0)
+; RVA22U64-NEXT:    lbu a2, 1(a0)
+; RVA22U64-NEXT:    lbu a3, 22(a0)
+; RVA22U64-NEXT:    lbu a4, 31(a0)
+; RVA22U64-NEXT:    lbu a6, 623(a0)
+; RVA22U64-NEXT:    lbu a5, 44(a0)
+; RVA22U64-NEXT:    lbu a1, 55(a0)
+; RVA22U64-NEXT:    lbu t0, 75(a0)
+; RVA22U64-NEXT:    lbu t1, 82(a0)
+; RVA22U64-NEXT:    slli a2, a2, 8
+; RVA22U64-NEXT:    slli a3, a3, 16
+; RVA22U64-NEXT:    slli a4, a4, 24
+; RVA22U64-NEXT:    slli a5, a5, 32
+; RVA22U64-NEXT:    slli a1, a1, 40
+; RVA22U64-NEXT:    or a7, a7, a2
+; RVA22U64-NEXT:    or t3, a4, a3
+; RVA22U64-NEXT:    or t2, a1, a5
+; RVA22U64-NEXT:    lbu a4, 93(a0)
+; RVA22U64-NEXT:    lbu t4, 105(a0)
+; RVA22U64-NEXT:    lbu a2, 124(a0)
+; RVA22U64-NEXT:    lbu t5, 144(a0)
+; RVA22U64-NEXT:    slli a6, a6, 48
+; RVA22U64-NEXT:    slli t0, t0, 56
+; RVA22U64-NEXT:    slli a4, a4, 8
+; RVA22U64-NEXT:    or a3, t0, a6
+; RVA22U64-NEXT:    or a4, t1, a4
+; RVA22U64-NEXT:    lbu a5, 161(a0)
+; RVA22U64-NEXT:    lbu a1, 154(a0)
 ; RVA22U64-NEXT:    lbu a0, 163(a0)
-; RVA22U64-NEXT:    slli a5, a5, 16
-; RVA22U64-NEXT:    slli t5, t5, 24
-; RVA22U64-NEXT:    or a5, t5, a5
-; RVA22U64-NEXT:    slli a1, a1, 32
+; RVA22U64-NEXT:    slli t4, t4, 16
+; RVA22U64-NEXT:    slli a5, a5, 24
+; RVA22U64-NEXT:    or a5, a5, t4
+; RVA22U64-NEXT:    slli a2, a2, 32
 ; RVA22U64-NEXT:    slli a0, a0, 40
+; RVA22U64-NEXT:    or a0, a0, a2
+; RVA22U64-NEXT:    slli t5, t5, 48
+; RVA22U64-NEXT:    slli a1, a1, 56
+; RVA22U64-NEXT:    or a1, a1, t5
+; RVA22U64-NEXT:    or a2, a7, t3
+; RVA22U64-NEXT:    or a3, a3, t2
+; RVA22U64-NEXT:    or a4, a4, a5
 ; RVA22U64-NEXT:    or a0, a0, a1
-; RVA22U64-NEXT:    slli a4, a4, 48
-; RVA22U64-NEXT:    slli t4, t4, 56
-; RVA22U64-NEXT:    or a1, t4, a4
-; RVA22U64-NEXT:    or a4, a6, t0
-; RVA22U64-NEXT:    or a2, a2, a7
-; RVA22U64-NEXT:    or a3, a3, a5
-; RVA22U64-NEXT:    or a0, a0, a1
-; RVA22U64-NEXT:    or a2, a2, a4
-; RVA22U64-NEXT:    or a0, a0, a3
+; RVA22U64-NEXT:    or a2, a2, a3
+; RVA22U64-NEXT:    or a0, a0, a4
 ; RVA22U64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RVA22U64-NEXT:    vmv.v.x v8, a2
 ; RVA22U64-NEXT:    vslide1down.vx v8, v8, a0
@@ -2111,24 +2111,24 @@ define <16 x i8> @buildvec_v16i8_undef_low_half(ptr %p) {
 ;
 ; RV32VB-LABEL: buildvec_v16i8_undef_low_half:
 ; RV32VB:       # %bb.0:
-; RV32VB-NEXT:    lbu a1, 82(a0)
-; RV32VB-NEXT:    lbu a2, 93(a0)
-; RV32VB-NEXT:    lbu a3, 144(a0)
-; RV32VB-NEXT:    lbu a4, 154(a0)
-; RV32VB-NEXT:    lbu a5, 161(a0)
-; RV32VB-NEXT:    lbu a6, 105(a0)
-; RV32VB-NEXT:    lbu a7, 124(a0)
-; RV32VB-NEXT:    slli a2, a2, 8
-; RV32VB-NEXT:    or a1, a1, a2
+; RV32VB-NEXT:    lbu a1, 93(a0)
+; RV32VB-NEXT:    lbu a2, 82(a0)
+; RV32VB-NEXT:    lbu a3, 105(a0)
+; RV32VB-NEXT:    lbu a4, 124(a0)
+; RV32VB-NEXT:    slli a1, a1, 8
+; RV32VB-NEXT:    or a1, a2, a1
+; RV32VB-NEXT:    lbu a2, 161(a0)
+; RV32VB-NEXT:    lbu a5, 144(a0)
+; RV32VB-NEXT:    lbu a6, 154(a0)
 ; RV32VB-NEXT:    lbu a0, 163(a0)
-; RV32VB-NEXT:    slli a6, a6, 16
-; RV32VB-NEXT:    slli a5, a5, 24
-; RV32VB-NEXT:    or a2, a5, a6
-; RV32VB-NEXT:    slli a0, a0, 8
-; RV32VB-NEXT:    or a0, a7, a0
 ; RV32VB-NEXT:    slli a3, a3, 16
-; RV32VB-NEXT:    slli a4, a4, 24
-; RV32VB-NEXT:    or a3, a4, a3
+; RV32VB-NEXT:    slli a2, a2, 24
+; RV32VB-NEXT:    or a2, a2, a3
+; RV32VB-NEXT:    slli a0, a0, 8
+; RV32VB-NEXT:    or a0, a4, a0
+; RV32VB-NEXT:    slli a5, a5, 16
+; RV32VB-NEXT:    slli a6, a6, 24
+; RV32VB-NEXT:    or a3, a6, a5
 ; RV32VB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32VB-NEXT:    vmv.v.i v8, 0
 ; RV32VB-NEXT:    or a1, a1, a2
@@ -2186,28 +2186,28 @@ define <16 x i8> @buildvec_v16i8_undef_low_half(ptr %p) {
 ;
 ; RVA22U64-LABEL: buildvec_v16i8_undef_low_half:
 ; RVA22U64:       # %bb.0:
-; RVA22U64-NEXT:    lbu a6, 82(a0)
-; RVA22U64-NEXT:    lbu a7, 93(a0)
-; RVA22U64-NEXT:    lbu t0, 144(a0)
-; RVA22U64-NEXT:    lbu a4, 154(a0)
-; RVA22U64-NEXT:    lbu a5, 161(a0)
-; RVA22U64-NEXT:    lbu a1, 105(a0)
-; RVA22U64-NEXT:    lbu a2, 124(a0)
-; RVA22U64-NEXT:    slli a7, a7, 8
-; RVA22U64-NEXT:    or a3, a6, a7
+; RVA22U64-NEXT:    lbu a1, 93(a0)
+; RVA22U64-NEXT:    lbu a2, 82(a0)
+; RVA22U64-NEXT:    lbu a3, 105(a0)
+; RVA22U64-NEXT:    lbu a4, 124(a0)
+; RVA22U64-NEXT:    slli a1, a1, 8
+; RVA22U64-NEXT:    or a6, a2, a1
+; RVA22U64-NEXT:    lbu a2, 161(a0)
+; RVA22U64-NEXT:    lbu a5, 144(a0)
+; RVA22U64-NEXT:    lbu a1, 154(a0)
 ; RVA22U64-NEXT:    lbu a0, 163(a0)
-; RVA22U64-NEXT:    slli a1, a1, 16
-; RVA22U64-NEXT:    slli a5, a5, 24
-; RVA22U64-NEXT:    or a1, a1, a5
-; RVA22U64-NEXT:    slli a2, a2, 32
+; RVA22U64-NEXT:    slli a3, a3, 16
+; RVA22U64-NEXT:    slli a2, a2, 24
+; RVA22U64-NEXT:    or a2, a2, a3
+; RVA22U64-NEXT:    slli a4, a4, 32
 ; RVA22U64-NEXT:    slli a0, a0, 40
-; RVA22U64-NEXT:    or a0, a0, a2
-; RVA22U64-NEXT:    slli t0, t0, 48
-; RVA22U64-NEXT:    slli a4, a4, 56
-; RVA22U64-NEXT:    or a2, a4, t0
-; RVA22U64-NEXT:    or a1, a1, a3
-; RVA22U64-NEXT:    or a0, a0, a2
+; RVA22U64-NEXT:    or a0, a0, a4
+; RVA22U64-NEXT:    slli a5, a5, 48
+; RVA22U64-NEXT:    slli a1, a1, 56
+; RVA22U64-NEXT:    or a1, a1, a5
+; RVA22U64-NEXT:    or a2, a6, a2
 ; RVA22U64-NEXT:    or a0, a0, a1
+; RVA22U64-NEXT:    or a0, a0, a2
 ; RVA22U64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RVA22U64-NEXT:    vmv.v.i v8, 0
 ; RVA22U64-NEXT:    vslide1down.vx v8, v8, a0
@@ -2520,30 +2520,30 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ;
 ; RV32VB-LABEL: buildvec_v16i8_undef_edges:
 ; RV32VB:       # %bb.0:
-; RV32VB-NEXT:    lbu a1, 55(a0)
-; RV32VB-NEXT:    lbu a2, 31(a0)
-; RV32VB-NEXT:    lbu a3, 44(a0)
-; RV32VB-NEXT:    lbu a4, 623(a0)
-; RV32VB-NEXT:    lbu a5, 75(a0)
-; RV32VB-NEXT:    slli a1, a1, 8
-; RV32VB-NEXT:    or a1, a3, a1
-; RV32VB-NEXT:    lbu a3, 82(a0)
-; RV32VB-NEXT:    lbu a6, 93(a0)
-; RV32VB-NEXT:    slli a4, a4, 16
-; RV32VB-NEXT:    slli a5, a5, 24
-; RV32VB-NEXT:    or a4, a5, a4
-; RV32VB-NEXT:    lbu a5, 105(a0)
+; RV32VB-NEXT:    lbu a1, 623(a0)
+; RV32VB-NEXT:    lbu a2, 55(a0)
+; RV32VB-NEXT:    lbu a3, 31(a0)
+; RV32VB-NEXT:    lbu a4, 75(a0)
+; RV32VB-NEXT:    lbu a5, 44(a0)
+; RV32VB-NEXT:    slli a2, a2, 8
+; RV32VB-NEXT:    slli a1, a1, 16
+; RV32VB-NEXT:    slli a4, a4, 24
+; RV32VB-NEXT:    or a2, a5, a2
+; RV32VB-NEXT:    or a1, a4, a1
+; RV32VB-NEXT:    lbu a4, 93(a0)
+; RV32VB-NEXT:    lbu a5, 82(a0)
+; RV32VB-NEXT:    lbu a6, 105(a0)
 ; RV32VB-NEXT:    lbu a0, 161(a0)
-; RV32VB-NEXT:    slli a6, a6, 8
-; RV32VB-NEXT:    or a3, a3, a6
-; RV32VB-NEXT:    slli a5, a5, 16
+; RV32VB-NEXT:    slli a4, a4, 8
+; RV32VB-NEXT:    or a4, a5, a4
+; RV32VB-NEXT:    slli a6, a6, 16
 ; RV32VB-NEXT:    slli a0, a0, 24
-; RV32VB-NEXT:    or a0, a0, a5
-; RV32VB-NEXT:    slli a2, a2, 24
-; RV32VB-NEXT:    or a1, a1, a4
-; RV32VB-NEXT:    or a0, a3, a0
+; RV32VB-NEXT:    or a0, a0, a6
+; RV32VB-NEXT:    slli a3, a3, 24
+; RV32VB-NEXT:    or a1, a2, a1
+; RV32VB-NEXT:    or a0, a4, a0
 ; RV32VB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32VB-NEXT:    vmv.v.x v8, a2
+; RV32VB-NEXT:    vmv.v.x v8, a3
 ; RV32VB-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32VB-NEXT:    vslide1down.vx v8, v8, a0
 ; RV32VB-NEXT:    vslide1down.vx v8, v8, zero
@@ -2607,32 +2607,32 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ;
 ; RVA22U64-LABEL: buildvec_v16i8_undef_edges:
 ; RVA22U64:       # %bb.0:
-; RVA22U64-NEXT:    lbu a6, 31(a0)
+; RVA22U64-NEXT:    lbu a1, 623(a0)
 ; RVA22U64-NEXT:    lbu a2, 44(a0)
 ; RVA22U64-NEXT:    lbu a3, 55(a0)
-; RVA22U64-NEXT:    lbu a4, 623(a0)
+; RVA22U64-NEXT:    lbu a6, 31(a0)
 ; RVA22U64-NEXT:    lbu a5, 75(a0)
 ; RVA22U64-NEXT:    slli a2, a2, 32
 ; RVA22U64-NEXT:    slli a3, a3, 40
-; RVA22U64-NEXT:    or a2, a2, a3
-; RVA22U64-NEXT:    lbu a3, 82(a0)
-; RVA22U64-NEXT:    lbu a1, 93(a0)
-; RVA22U64-NEXT:    slli a4, a4, 48
+; RVA22U64-NEXT:    slli a1, a1, 48
 ; RVA22U64-NEXT:    slli a5, a5, 56
-; RVA22U64-NEXT:    or a4, a4, a5
-; RVA22U64-NEXT:    lbu a5, 105(a0)
+; RVA22U64-NEXT:    or a2, a2, a3
+; RVA22U64-NEXT:    or a1, a1, a5
+; RVA22U64-NEXT:    lbu a3, 93(a0)
+; RVA22U64-NEXT:    lbu a5, 82(a0)
+; RVA22U64-NEXT:    lbu a4, 105(a0)
 ; RVA22U64-NEXT:    lbu a0, 161(a0)
-; RVA22U64-NEXT:    slli a1, a1, 8
-; RVA22U64-NEXT:    or a1, a1, a3
-; RVA22U64-NEXT:    slli a5, a5, 16
+; RVA22U64-NEXT:    slli a3, a3, 8
+; RVA22U64-NEXT:    or a3, a3, a5
+; RVA22U64-NEXT:    slli a4, a4, 16
 ; RVA22U64-NEXT:    slli a0, a0, 24
-; RVA22U64-NEXT:    or a0, a0, a5
+; RVA22U64-NEXT:    or a0, a0, a4
 ; RVA22U64-NEXT:    slli a6, a6, 24
-; RVA22U64-NEXT:    or a2, a2, a4
-; RVA22U64-NEXT:    add.uw a2, a6, a2
-; RVA22U64-NEXT:    or a0, a0, a1
+; RVA22U64-NEXT:    or a1, a1, a2
+; RVA22U64-NEXT:    add.uw a1, a6, a1
+; RVA22U64-NEXT:    or a0, a0, a3
 ; RVA22U64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RVA22U64-NEXT:    vmv.v.x v8, a2
+; RVA22U64-NEXT:    vmv.v.x v8, a1
 ; RVA22U64-NEXT:    vslide1down.vx v8, v8, a0
 ; RVA22U64-NEXT:    ret
 ;
@@ -2794,26 +2794,26 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RV32VB-PACK-NEXT:    lbu a2, 1(a0)
 ; RV32VB-PACK-NEXT:    lbu a3, 44(a0)
 ; RV32VB-PACK-NEXT:    lbu a4, 55(a0)
-; RV32VB-PACK-NEXT:    lbu a5, 75(a0)
-; RV32VB-PACK-NEXT:    lbu a6, 82(a0)
-; RV32VB-PACK-NEXT:    lbu a7, 93(a0)
+; RV32VB-PACK-NEXT:    lbu a5, 82(a0)
+; RV32VB-PACK-NEXT:    lbu a6, 93(a0)
 ; RV32VB-PACK-NEXT:    packh a1, a1, a2
 ; RV32VB-PACK-NEXT:    lbu a2, 144(a0)
-; RV32VB-PACK-NEXT:    lbu t0, 154(a0)
+; RV32VB-PACK-NEXT:    lbu a7, 154(a0)
 ; RV32VB-PACK-NEXT:    packh a3, a3, a4
+; RV32VB-PACK-NEXT:    lbu a4, 75(a0)
 ; RV32VB-PACK-NEXT:    lbu a0, 124(a0)
-; RV32VB-PACK-NEXT:    packh a4, a6, a7
-; RV32VB-PACK-NEXT:    packh a2, a2, t0
-; RV32VB-PACK-NEXT:    packh a5, a0, a5
-; RV32VB-PACK-NEXT:    pack a3, a3, a5
-; RV32VB-PACK-NEXT:    packh a5, a0, a0
+; RV32VB-PACK-NEXT:    packh a5, a5, a6
+; RV32VB-PACK-NEXT:    packh a2, a2, a7
+; RV32VB-PACK-NEXT:    packh a4, a0, a4
+; RV32VB-PACK-NEXT:    pack a3, a3, a4
+; RV32VB-PACK-NEXT:    packh a4, a0, a0
 ; RV32VB-PACK-NEXT:    packh a0, a0, a0
 ; RV32VB-PACK-NEXT:    pack a0, a0, a2
-; RV32VB-PACK-NEXT:    pack a1, a1, a5
+; RV32VB-PACK-NEXT:    pack a1, a1, a4
 ; RV32VB-PACK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32VB-PACK-NEXT:    vmv.v.x v8, a1
 ; RV32VB-PACK-NEXT:    vslide1down.vx v8, v8, a3
-; RV32VB-PACK-NEXT:    pack a1, a4, a5
+; RV32VB-PACK-NEXT:    pack a1, a5, a4
 ; RV32VB-PACK-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32VB-PACK-NEXT:    vslide1down.vx v8, v8, a0
 ; RV32VB-PACK-NEXT:    ret
@@ -2888,23 +2888,23 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RVA22U64-PACK:       # %bb.0:
 ; RVA22U64-PACK-NEXT:    lbu a1, 0(a0)
 ; RVA22U64-PACK-NEXT:    lbu a2, 1(a0)
-; RVA22U64-PACK-NEXT:    lbu a7, 44(a0)
-; RVA22U64-PACK-NEXT:    lbu t0, 55(a0)
-; RVA22U64-PACK-NEXT:    lbu a6, 75(a0)
-; RVA22U64-PACK-NEXT:    lbu a5, 82(a0)
+; RVA22U64-PACK-NEXT:    lbu a6, 44(a0)
+; RVA22U64-PACK-NEXT:    lbu a7, 55(a0)
+; RVA22U64-PACK-NEXT:    lbu t1, 82(a0)
 ; RVA22U64-PACK-NEXT:    lbu a3, 93(a0)
-; RVA22U64-PACK-NEXT:    packh t1, a1, a2
+; RVA22U64-PACK-NEXT:    packh t0, a1, a2
 ; RVA22U64-PACK-NEXT:    lbu a2, 144(a0)
 ; RVA22U64-PACK-NEXT:    lbu a4, 154(a0)
-; RVA22U64-PACK-NEXT:    packh a1, a7, t0
+; RVA22U64-PACK-NEXT:    packh a1, a6, a7
+; RVA22U64-PACK-NEXT:    lbu a5, 75(a0)
 ; RVA22U64-PACK-NEXT:    lbu a0, 124(a0)
-; RVA22U64-PACK-NEXT:    packh a3, a5, a3
+; RVA22U64-PACK-NEXT:    packh a3, t1, a3
 ; RVA22U64-PACK-NEXT:    packh a2, a2, a4
-; RVA22U64-PACK-NEXT:    packh a4, a0, a6
+; RVA22U64-PACK-NEXT:    packh a4, a0, a5
 ; RVA22U64-PACK-NEXT:    packw a1, a1, a4
 ; RVA22U64-PACK-NEXT:    packh a4, a0, a0
 ; RVA22U64-PACK-NEXT:    packh a0, a0, a0
-; RVA22U64-PACK-NEXT:    packw a5, t1, a4
+; RVA22U64-PACK-NEXT:    packw a5, t0, a4
 ; RVA22U64-PACK-NEXT:    packw a0, a0, a2
 ; RVA22U64-PACK-NEXT:    packw a2, a3, a4
 ; RVA22U64-PACK-NEXT:    pack a1, a5, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
index a9adc87d29c8..0c30cbe4a42e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
@@ -1207,8 +1207,8 @@ define void @mulhu_v2i64(ptr %x) {
 ; RV64-NEXT:    vle64.v v8, (a0)
 ; RV64-NEXT:    lui a1, 838861
 ; RV64-NEXT:    lui a2, 699051
-; RV64-NEXT:    addiw a1, a1, -819
-; RV64-NEXT:    addiw a2, a2, -1365
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    addi a2, a2, -1365
 ; RV64-NEXT:    slli a3, a1, 32
 ; RV64-NEXT:    add a1, a1, a3
 ; RV64-NEXT:    slli a3, a2, 32
@@ -1368,7 +1368,7 @@ define void @mulhs_v2i64(ptr %x) {
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
 ; RV64-NEXT:    lui a1, 349525
-; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addi a1, a1, 1365
 ; RV64-NEXT:    slli a2, a1, 32
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    lui a2, %hi(.LCPI74_0)
@@ -3501,7 +3501,7 @@ define void @mulhs_v4i64(ptr %x) {
 ; RV64-NEXT:    vle64.v v8, (a0)
 ; RV64-NEXT:    lui a1, 349525
 ; RV64-NEXT:    lui a2, 1044496
-; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addi a1, a1, 1365
 ; RV64-NEXT:    addi a2, a2, -256
 ; RV64-NEXT:    vmv.s.x v14, a2
 ; RV64-NEXT:    slli a2, a1, 32
@@ -5527,7 +5527,7 @@ define void @mulhu_vx_v2i64(ptr %x) {
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
 ; RV64-NEXT:    lui a1, 699051
-; RV64-NEXT:    addiw a1, a1, -1365
+; RV64-NEXT:    addi a1, a1, -1365
 ; RV64-NEXT:    slli a2, a1, 32
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    vmulhu.vx v8, v8, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 71760a672378..533b8b6864eb 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -1128,7 +1128,7 @@ define <2 x i64> @mgather_v2i16_zextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    lui a1, 16
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32F-NEXT:    addiw a1, a1, -1
+; RV64ZVE32F-NEXT:    addi a1, a1, -1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    and a0, a0, a1
 ; RV64ZVE32F-NEXT:    and a1, a2, a1
@@ -14191,37 +14191,37 @@ define <8 x i16> @mgather_strided_unaligned(ptr %base) {
 ; RV64ZVE32F-NEXT:    lbu t0, 13(a0)
 ; RV64ZVE32F-NEXT:    slli a2, a2, 8
 ; RV64ZVE32F-NEXT:    slli a4, a4, 8
-; RV64ZVE32F-NEXT:    slli a6, a6, 8
 ; RV64ZVE32F-NEXT:    or a1, a2, a1
 ; RV64ZVE32F-NEXT:    or a3, a4, a3
-; RV64ZVE32F-NEXT:    or a2, a6, a5
-; RV64ZVE32F-NEXT:    lbu a4, 16(a0)
-; RV64ZVE32F-NEXT:    lbu a5, 17(a0)
-; RV64ZVE32F-NEXT:    lbu a6, 20(a0)
-; RV64ZVE32F-NEXT:    lbu t1, 21(a0)
+; RV64ZVE32F-NEXT:    lbu a2, 16(a0)
+; RV64ZVE32F-NEXT:    lbu a4, 17(a0)
+; RV64ZVE32F-NEXT:    lbu t1, 20(a0)
+; RV64ZVE32F-NEXT:    lbu t2, 21(a0)
+; RV64ZVE32F-NEXT:    slli a6, a6, 8
+; RV64ZVE32F-NEXT:    or a5, a6, a5
 ; RV64ZVE32F-NEXT:    slli t0, t0, 8
-; RV64ZVE32F-NEXT:    slli a5, a5, 8
-; RV64ZVE32F-NEXT:    or a7, t0, a7
-; RV64ZVE32F-NEXT:    or a4, a5, a4
-; RV64ZVE32F-NEXT:    lbu a5, 24(a0)
-; RV64ZVE32F-NEXT:    lbu t0, 25(a0)
-; RV64ZVE32F-NEXT:    slli t1, t1, 8
-; RV64ZVE32F-NEXT:    or a6, t1, a6
+; RV64ZVE32F-NEXT:    slli a4, a4, 8
+; RV64ZVE32F-NEXT:    slli t2, t2, 8
+; RV64ZVE32F-NEXT:    or a6, t0, a7
+; RV64ZVE32F-NEXT:    or a2, a4, a2
+; RV64ZVE32F-NEXT:    or a4, t2, t1
+; RV64ZVE32F-NEXT:    lbu a7, 25(a0)
+; RV64ZVE32F-NEXT:    lbu t0, 24(a0)
 ; RV64ZVE32F-NEXT:    lbu t1, 28(a0)
 ; RV64ZVE32F-NEXT:    lbu a0, 29(a0)
-; RV64ZVE32F-NEXT:    slli t0, t0, 8
-; RV64ZVE32F-NEXT:    or a5, t0, a5
+; RV64ZVE32F-NEXT:    slli a7, a7, 8
+; RV64ZVE32F-NEXT:    or a7, a7, t0
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    slli a0, a0, 8
 ; RV64ZVE32F-NEXT:    or a0, a0, t1
 ; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a4
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a2
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a6
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
index 545c89495e62..ed60d9130849 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
@@ -341,3 +341,16 @@ define <7 x i8> @masked_load_v7i8(ptr %a, <7 x i1> %mask) {
   ret <7 x i8> %load
 }
 
+define <7 x i8> @masked_load_passthru_v7i8(ptr %a, <7 x i1> %mask) {
+; CHECK-LABEL: masked_load_passthru_v7i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 127
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
+; CHECK-NEXT:    vmv.s.x v8, a1
+; CHECK-NEXT:    vmand.mm v0, v0, v8
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vle8.v v8, (a0), v0.t
+; CHECK-NEXT:    ret
+  %load = call <7 x i8> @llvm.masked.load.v7i8(ptr %a, i32 8, <7 x i1> %mask, <7 x i8> zeroinitializer)
+  ret <7 x i8> %load
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
index c65d7c36a219..b6267bf481c8 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
@@ -347,9 +347,8 @@ define void @vnsrl_0_i64(ptr %in, ptr %out) {
 ; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
 ; ZIP-NEXT:    vle64.v v8, (a0)
 ; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; ZIP-NEXT:    vslidedown.vi v9, v8, 2
-; ZIP-NEXT:    vslideup.vi v8, v9, 1
-; ZIP-NEXT:    vse64.v v8, (a1)
+; ZIP-NEXT:    ri.vunzip2a.vv v10, v8, v9
+; ZIP-NEXT:    vse64.v v10, (a1)
 ; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x i64>, ptr %in, align 8
@@ -383,8 +382,7 @@ define void @vnsrl_64_i64(ptr %in, ptr %out) {
 ; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
 ; ZIP-NEXT:    vle64.v v8, (a0)
 ; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; ZIP-NEXT:    vslidedown.vi v9, v8, 2
-; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
+; ZIP-NEXT:    ri.vunzip2b.vv v10, v8, v9
 ; ZIP-NEXT:    vse64.v v10, (a1)
 ; ZIP-NEXT:    ret
 entry:
@@ -417,10 +415,9 @@ define void @vnsrl_0_double(ptr %in, ptr %out) {
 ; ZIP:       # %bb.0: # %entry
 ; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
 ; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    ri.vunzip2a.vv v10, v8, v9
 ; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; ZIP-NEXT:    vslidedown.vi v9, v8, 2
-; ZIP-NEXT:    vslideup.vi v8, v9, 1
-; ZIP-NEXT:    vse64.v v8, (a1)
+; ZIP-NEXT:    vse64.v v10, (a1)
 ; ZIP-NEXT:    ret
 entry:
   %0 = load <4 x double>, ptr %in, align 8
@@ -453,9 +450,8 @@ define void @vnsrl_64_double(ptr %in, ptr %out) {
 ; ZIP:       # %bb.0: # %entry
 ; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
 ; ZIP-NEXT:    vle64.v v8, (a0)
+; ZIP-NEXT:    ri.vunzip2b.vv v10, v8, v9
 ; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; ZIP-NEXT:    vslidedown.vi v9, v8, 2
-; ZIP-NEXT:    ri.vzipodd.vv v10, v8, v9
 ; ZIP-NEXT:    vse64.v v10, (a1)
 ; ZIP-NEXT:    ret
 entry:
@@ -1306,3 +1302,535 @@ entry:
   store <2 x double> %shuffle.i5, ptr %out, align 8
   ret void
 }
+
+define <2 x i64> @unzip2a_dual_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; V-LABEL: unzip2a_dual_v2i64:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; V-NEXT:    vslideup.vi v8, v9, 1
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2a_dual_v2i64:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    mv a1, a2
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2a_dual_v2i64:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v9, 1
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 2>
+  ret <2 x i64> %c
+}
+
+define <4 x i64> @unzip2a_dual_v4i64(<4 x i64> %a, <4 x i64> %b) {
+; V-LABEL: unzip2a_dual_v4i64:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vmv.v.i v0, 8
+; V-NEXT:    vslideup.vi v10, v9, 2
+; V-NEXT:    vslideup.vi v10, v9, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 2
+; V-NEXT:    vslidedown.vi v8, v8, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 12
+; V-NEXT:    vmerge.vvm v8, v8, v10, v0
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2a_dual_v4i64:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a3, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
+; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
+; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a6, a3, 32
+; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli t0, a4, 32
+; ZVE32F-NEXT:    vmv.v.x v8, a4
+; ZVE32F-NEXT:    vmv.v.x v9, a3
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2a_dual_v4i64:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v11, v9, v10
+; ZIP-NEXT:    ri.vunzip2a.vv v9, v8, v10
+; ZIP-NEXT:    vslideup.vi v9, v11, 2
+; ZIP-NEXT:    vmv.v.v v8, v9
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  ret <4 x i64> %c
+}
+
+define <16 x i64> @unzip2a_dual_v16i64(<16 x i64> %a, <16 x i64> %b) {
+; V-LABEL: unzip2a_dual_v16i64:
+; V:       # %bb.0: # %entry
+; V-NEXT:    lui a0, 5
+; V-NEXT:    vsetivli zero, 16, e16, m1, ta, ma
+; V-NEXT:    vid.v v16
+; V-NEXT:    addi a0, a0, 1365
+; V-NEXT:    vmv.s.x v20, a0
+; V-NEXT:    li a0, -256
+; V-NEXT:    vadd.vv v21, v16, v16
+; V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; V-NEXT:    vcompress.vm v16, v8, v20
+; V-NEXT:    vmv.s.x v0, a0
+; V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; V-NEXT:    vadd.vi v8, v21, -16
+; V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; V-NEXT:    vrgatherei16.vv v16, v12, v8, v0.t
+; V-NEXT:    vmv.v.v v8, v16
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2a_dual_v16i64:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    addi sp, sp, -256
+; ZVE32F-NEXT:    .cfi_def_cfa_offset 256
+; ZVE32F-NEXT:    sd ra, 248(sp) # 8-byte Folded Spill
+; ZVE32F-NEXT:    sd s0, 240(sp) # 8-byte Folded Spill
+; ZVE32F-NEXT:    sd s2, 232(sp) # 8-byte Folded Spill
+; ZVE32F-NEXT:    sd s3, 224(sp) # 8-byte Folded Spill
+; ZVE32F-NEXT:    sd s4, 216(sp) # 8-byte Folded Spill
+; ZVE32F-NEXT:    .cfi_offset ra, -8
+; ZVE32F-NEXT:    .cfi_offset s0, -16
+; ZVE32F-NEXT:    .cfi_offset s2, -24
+; ZVE32F-NEXT:    .cfi_offset s3, -32
+; ZVE32F-NEXT:    .cfi_offset s4, -40
+; ZVE32F-NEXT:    addi s0, sp, 256
+; ZVE32F-NEXT:    .cfi_def_cfa s0, 0
+; ZVE32F-NEXT:    andi sp, sp, -128
+; ZVE32F-NEXT:    ld t5, 0(a1)
+; ZVE32F-NEXT:    ld t2, 16(a1)
+; ZVE32F-NEXT:    ld a4, 32(a1)
+; ZVE32F-NEXT:    ld a3, 48(a1)
+; ZVE32F-NEXT:    ld a6, 64(a1)
+; ZVE32F-NEXT:    ld a5, 80(a1)
+; ZVE32F-NEXT:    ld a7, 96(a1)
+; ZVE32F-NEXT:    ld a1, 112(a1)
+; ZVE32F-NEXT:    ld t1, 0(a2)
+; ZVE32F-NEXT:    ld t0, 16(a2)
+; ZVE32F-NEXT:    ld t4, 32(a2)
+; ZVE32F-NEXT:    ld t3, 48(a2)
+; ZVE32F-NEXT:    ld t6, 64(a2)
+; ZVE32F-NEXT:    ld s2, 80(a2)
+; ZVE32F-NEXT:    ld s3, 96(a2)
+; ZVE32F-NEXT:    ld a2, 112(a2)
+; ZVE32F-NEXT:    srli s4, t5, 32
+; ZVE32F-NEXT:    sw t5, 0(sp)
+; ZVE32F-NEXT:    sw s4, 4(sp)
+; ZVE32F-NEXT:    srli t5, t2, 32
+; ZVE32F-NEXT:    sw t2, 8(sp)
+; ZVE32F-NEXT:    srli t2, s3, 32
+; ZVE32F-NEXT:    sw s3, 112(sp)
+; ZVE32F-NEXT:    sw t2, 116(sp)
+; ZVE32F-NEXT:    srli t2, a2, 32
+; ZVE32F-NEXT:    sw a2, 120(sp)
+; ZVE32F-NEXT:    sw t2, 124(sp)
+; ZVE32F-NEXT:    srli a2, t6, 32
+; ZVE32F-NEXT:    sw t6, 96(sp)
+; ZVE32F-NEXT:    sw a2, 100(sp)
+; ZVE32F-NEXT:    srli a2, s2, 32
+; ZVE32F-NEXT:    sw s2, 104(sp)
+; ZVE32F-NEXT:    sw a2, 108(sp)
+; ZVE32F-NEXT:    srli a2, t4, 32
+; ZVE32F-NEXT:    sw t4, 80(sp)
+; ZVE32F-NEXT:    sw a2, 84(sp)
+; ZVE32F-NEXT:    srli a2, t3, 32
+; ZVE32F-NEXT:    sw t3, 88(sp)
+; ZVE32F-NEXT:    sw a2, 92(sp)
+; ZVE32F-NEXT:    srli a2, t1, 32
+; ZVE32F-NEXT:    sw t1, 64(sp)
+; ZVE32F-NEXT:    sw a2, 68(sp)
+; ZVE32F-NEXT:    srli a2, t0, 32
+; ZVE32F-NEXT:    sw t0, 72(sp)
+; ZVE32F-NEXT:    sw a2, 76(sp)
+; ZVE32F-NEXT:    srli a2, a7, 32
+; ZVE32F-NEXT:    sw a7, 48(sp)
+; ZVE32F-NEXT:    sw a2, 52(sp)
+; ZVE32F-NEXT:    srli a2, a1, 32
+; ZVE32F-NEXT:    sw a1, 56(sp)
+; ZVE32F-NEXT:    sw a2, 60(sp)
+; ZVE32F-NEXT:    srli a1, a6, 32
+; ZVE32F-NEXT:    sw a6, 32(sp)
+; ZVE32F-NEXT:    sw a1, 36(sp)
+; ZVE32F-NEXT:    srli a1, a5, 32
+; ZVE32F-NEXT:    sw a5, 40(sp)
+; ZVE32F-NEXT:    sw a1, 44(sp)
+; ZVE32F-NEXT:    srli a1, a4, 32
+; ZVE32F-NEXT:    sw a4, 16(sp)
+; ZVE32F-NEXT:    sw a1, 20(sp)
+; ZVE32F-NEXT:    srli a1, a3, 32
+; ZVE32F-NEXT:    sw a3, 24(sp)
+; ZVE32F-NEXT:    sw a1, 28(sp)
+; ZVE32F-NEXT:    li a1, 32
+; ZVE32F-NEXT:    sw t5, 12(sp)
+; ZVE32F-NEXT:    mv a2, sp
+; ZVE32F-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; ZVE32F-NEXT:    vle32.v v8, (a2)
+; ZVE32F-NEXT:    vse32.v v8, (a0)
+; ZVE32F-NEXT:    addi sp, s0, -256
+; ZVE32F-NEXT:    .cfi_def_cfa sp, 256
+; ZVE32F-NEXT:    ld ra, 248(sp) # 8-byte Folded Reload
+; ZVE32F-NEXT:    ld s0, 240(sp) # 8-byte Folded Reload
+; ZVE32F-NEXT:    ld s2, 232(sp) # 8-byte Folded Reload
+; ZVE32F-NEXT:    ld s3, 224(sp) # 8-byte Folded Reload
+; ZVE32F-NEXT:    ld s4, 216(sp) # 8-byte Folded Reload
+; ZVE32F-NEXT:    .cfi_restore ra
+; ZVE32F-NEXT:    .cfi_restore s0
+; ZVE32F-NEXT:    .cfi_restore s2
+; ZVE32F-NEXT:    .cfi_restore s3
+; ZVE32F-NEXT:    .cfi_restore s4
+; ZVE32F-NEXT:    addi sp, sp, 256
+; ZVE32F-NEXT:    .cfi_def_cfa_offset 0
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2a_dual_v16i64:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e64, m2, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v16, v12, v14
+; ZIP-NEXT:    ri.vunzip2a.vv v12, v8, v10
+; ZIP-NEXT:    vsetivli zero, 16, e64, m4, ta, ma
+; ZIP-NEXT:    vslideup.vi v12, v16, 8
+; ZIP-NEXT:    vmv.v.v v8, v12
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <16 x i64> %a, <16 x i64> %b, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  ret <16 x i64> %c
+}
+
+define <4 x i64> @unzip2a_dual_v4i64_exact(<4 x i64> %a, <4 x i64> %b) vscale_range(4,4) {
+; V-LABEL: unzip2a_dual_v4i64_exact:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vmv.v.i v0, 8
+; V-NEXT:    vslideup.vi v10, v9, 2
+; V-NEXT:    vslideup.vi v10, v9, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 2
+; V-NEXT:    vslidedown.vi v8, v8, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 12
+; V-NEXT:    vmerge.vvm v8, v8, v10, v0
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2a_dual_v4i64_exact:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a3, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
+; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
+; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a6, a3, 32
+; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli t0, a4, 32
+; ZVE32F-NEXT:    vmv.v.x v8, a4
+; ZVE32F-NEXT:    vmv.v.x v9, a3
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vs1r.v v9, (a0)
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2a_dual_v4i64_exact:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v10, v8, v9
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @unzip2a_dual_v4i64_exact_nf2(<4 x i64> %a, <4 x i64> %b) vscale_range(8,8) {
+; V-LABEL: unzip2a_dual_v4i64_exact_nf2:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vmv.v.i v0, 8
+; V-NEXT:    vslideup.vi v10, v9, 2
+; V-NEXT:    vslideup.vi v10, v9, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 2
+; V-NEXT:    vslidedown.vi v8, v8, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 12
+; V-NEXT:    vmerge.vvm v8, v8, v10, v0
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2a_dual_v4i64_exact_nf2:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a3, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
+; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
+; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a6, a3, 32
+; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli t0, a4, 32
+; ZVE32F-NEXT:    vmv.v.x v8, a4
+; ZVE32F-NEXT:    vmv.v.x v9, a3
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2a_dual_v4i64_exact_nf2:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 8, e64, m1, ta, ma
+; ZIP-NEXT:    vslideup.vi v8, v9, 4
+; ZIP-NEXT:    ri.vunzip2a.vv v9, v8, v10
+; ZIP-NEXT:    vmv.v.v v8, v9
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  ret <4 x i64> %c
+}
+
+define <16 x i64> @unzip2a_dual_v16i64_exact(<16 x i64> %a, <16 x i64> %b) vscale_range(4,4) {
+; V-LABEL: unzip2a_dual_v16i64_exact:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vslideup.vi v19, v15, 2
+; V-NEXT:    vmv.v.i v16, 8
+; V-NEXT:    vmv.v.i v17, 2
+; V-NEXT:    vmv.v.i v18, 12
+; V-NEXT:    vmv.v.v v0, v16
+; V-NEXT:    vslideup.vi v19, v15, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v17
+; V-NEXT:    vslidedown.vi v14, v14, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v18
+; V-NEXT:    vmerge.vvm v15, v14, v19, v0
+; V-NEXT:    vslideup.vi v14, v13, 2
+; V-NEXT:    vmv.v.v v0, v16
+; V-NEXT:    vslideup.vi v14, v13, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v17
+; V-NEXT:    vslidedown.vi v12, v12, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v18
+; V-NEXT:    vmerge.vvm v14, v12, v14, v0
+; V-NEXT:    vslideup.vi v12, v11, 2
+; V-NEXT:    li a0, -256
+; V-NEXT:    vmv.v.v v0, v16
+; V-NEXT:    vslideup.vi v12, v11, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v17
+; V-NEXT:    vslidedown.vi v10, v10, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v18
+; V-NEXT:    vmerge.vvm v13, v10, v12, v0
+; V-NEXT:    vslideup.vi v10, v9, 2
+; V-NEXT:    vmv.v.v v0, v16
+; V-NEXT:    vslideup.vi v10, v9, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v17
+; V-NEXT:    vslidedown.vi v8, v8, 1, v0.t
+; V-NEXT:    vmv.v.v v0, v18
+; V-NEXT:    vmerge.vvm v12, v8, v10, v0
+; V-NEXT:    vmv.s.x v0, a0
+; V-NEXT:    vsetivli zero, 16, e64, m4, ta, ma
+; V-NEXT:    vmerge.vvm v8, v12, v12, v0
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2a_dual_v16i64_exact:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a6, 0(a1)
+; ZVE32F-NEXT:    ld a4, 16(a1)
+; ZVE32F-NEXT:    ld a7, 32(a1)
+; ZVE32F-NEXT:    ld a3, 48(a1)
+; ZVE32F-NEXT:    ld a5, 80(a1)
+; ZVE32F-NEXT:    ld t0, 96(a1)
+; ZVE32F-NEXT:    ld t1, 64(a1)
+; ZVE32F-NEXT:    ld a1, 112(a1)
+; ZVE32F-NEXT:    srli t2, a6, 32
+; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
+; ZVE32F-NEXT:    vmv.v.x v8, t0
+; ZVE32F-NEXT:    srli t0, t0, 32
+; ZVE32F-NEXT:    vmv.v.x v9, t1
+; ZVE32F-NEXT:    srli t1, t1, 32
+; ZVE32F-NEXT:    vmv.v.x v10, a6
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, t1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, t2
+; ZVE32F-NEXT:    ld t1, 32(a2)
+; ZVE32F-NEXT:    ld t0, 16(a2)
+; ZVE32F-NEXT:    ld t2, 0(a2)
+; ZVE32F-NEXT:    ld a6, 48(a2)
+; ZVE32F-NEXT:    vmv.v.x v11, t1
+; ZVE32F-NEXT:    srli t1, t1, 32
+; ZVE32F-NEXT:    vmv.v.x v12, t2
+; ZVE32F-NEXT:    srli t2, t2, 32
+; ZVE32F-NEXT:    vmv.v.x v13, a7
+; ZVE32F-NEXT:    srli a7, a7, 32
+; ZVE32F-NEXT:    vslide1down.vx v13, v13, a7
+; ZVE32F-NEXT:    vslide1down.vx v12, v12, t2
+; ZVE32F-NEXT:    vslide1down.vx v11, v11, t1
+; ZVE32F-NEXT:    ld a7, 64(a2)
+; ZVE32F-NEXT:    ld t1, 80(a2)
+; ZVE32F-NEXT:    ld t2, 96(a2)
+; ZVE32F-NEXT:    ld a2, 112(a2)
+; ZVE32F-NEXT:    vmv.v.x v14, a7
+; ZVE32F-NEXT:    srli a7, a7, 32
+; ZVE32F-NEXT:    vslide1down.vx v14, v14, a7
+; ZVE32F-NEXT:    vmv.v.x v15, t2
+; ZVE32F-NEXT:    srli a7, t2, 32
+; ZVE32F-NEXT:    vslide1down.vx v15, v15, a7
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    srli a5, a5, 32
+; ZVE32F-NEXT:    vslide1down.vx v16, v9, a5
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    srli a1, a1, 32
+; ZVE32F-NEXT:    vslide1down.vx v9, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v10, a4
+; ZVE32F-NEXT:    srli a4, a4, 32
+; ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
+; ZVE32F-NEXT:    vslide1down.vx v8, v12, t0
+; ZVE32F-NEXT:    srli a1, t0, 32
+; ZVE32F-NEXT:    vslide1down.vx v12, v8, a1
+; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    vslide1down.vx v8, v14, t1
+; ZVE32F-NEXT:    srli a1, t1, 32
+; ZVE32F-NEXT:    vslide1down.vx v14, v8, a1
+; ZVE32F-NEXT:    vslidedown.vi v9, v16, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v8, v13, a3
+; ZVE32F-NEXT:    srli a3, a3, 32
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v10, v11, a6
+; ZVE32F-NEXT:    srli a1, a6, 32
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, a1
+; ZVE32F-NEXT:    vslidedown.vi v10, v12, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v11, v15, a2
+; ZVE32F-NEXT:    srli a2, a2, 32
+; ZVE32F-NEXT:    vslide1down.vx v11, v11, a2
+; ZVE32F-NEXT:    vslidedown.vi v11, v14, 4, v0.t
+; ZVE32F-NEXT:    vs4r.v v8, (a0)
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2a_dual_v16i64_exact:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 16, e64, m4, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v16, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v16
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <16 x i64> %a, <16 x i64> %b, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  ret <16 x i64> %c
+}
+
+define <4 x i64> @unzip2b_dual_v4i64(<4 x i64> %a, <4 x i64> %b) {
+; V-LABEL: unzip2b_dual_v4i64:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vmv.v.i v0, 2
+; V-NEXT:    vslidedown.vi v10, v8, 1
+; V-NEXT:    vslidedown.vi v10, v8, 2, v0.t
+; V-NEXT:    vmv.v.i v0, 4
+; V-NEXT:    vmv1r.v v8, v9
+; V-NEXT:    vslideup.vi v8, v9, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 12
+; V-NEXT:    vmerge.vvm v8, v10, v8, v0
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2b_dual_v4i64:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a3, 8(a2)
+; ZVE32F-NEXT:    ld a2, 24(a2)
+; ZVE32F-NEXT:    ld a4, 8(a1)
+; ZVE32F-NEXT:    ld a1, 24(a1)
+; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
+; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a6, a3, 32
+; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli t0, a4, 32
+; ZVE32F-NEXT:    vmv.v.x v8, a4
+; ZVE32F-NEXT:    vmv.v.x v9, a3
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2b_dual_v4i64:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    ri.vunzip2b.vv v11, v9, v10
+; ZIP-NEXT:    ri.vunzip2b.vv v9, v8, v10
+; ZIP-NEXT:    vslideup.vi v9, v11, 2
+; ZIP-NEXT:    vmv.v.v v8, v9
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @unzip2b_dual_v4i64_exact(<4 x i64> %a, <4 x i64> %b) vscale_range(4,4) {
+; V-LABEL: unzip2b_dual_v4i64_exact:
+; V:       # %bb.0: # %entry
+; V-NEXT:    vsetivli zero, 4, e64, m1, ta, mu
+; V-NEXT:    vmv.v.i v0, 2
+; V-NEXT:    vslidedown.vi v10, v8, 1
+; V-NEXT:    vslidedown.vi v10, v8, 2, v0.t
+; V-NEXT:    vmv.v.i v0, 4
+; V-NEXT:    vmv1r.v v8, v9
+; V-NEXT:    vslideup.vi v8, v9, 1, v0.t
+; V-NEXT:    vmv.v.i v0, 12
+; V-NEXT:    vmerge.vvm v8, v10, v8, v0
+; V-NEXT:    ret
+;
+; ZVE32F-LABEL: unzip2b_dual_v4i64_exact:
+; ZVE32F:       # %bb.0: # %entry
+; ZVE32F-NEXT:    ld a3, 8(a2)
+; ZVE32F-NEXT:    ld a2, 24(a2)
+; ZVE32F-NEXT:    ld a4, 8(a1)
+; ZVE32F-NEXT:    ld a1, 24(a1)
+; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
+; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a6, a3, 32
+; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli t0, a4, 32
+; ZVE32F-NEXT:    vmv.v.x v8, a4
+; ZVE32F-NEXT:    vmv.v.x v9, a3
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vs1r.v v9, (a0)
+; ZVE32F-NEXT:    ret
+;
+; ZIP-LABEL: unzip2b_dual_v4i64_exact:
+; ZIP:       # %bb.0: # %entry
+; ZIP-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
+; ZIP-NEXT:    ri.vunzip2b.vv v10, v8, v9
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    ret
+entry:
+  %c = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x i64> %c
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
index 4e367bb0d70c..3b1dc298c12c 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
@@ -268,7 +268,7 @@ define void @trunc_sat_i32i64_notopt(ptr %x, ptr %y) {
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
 ; CHECK-NEXT:    lui a0, 524288
-; CHECK-NEXT:    addiw a0, a0, 1
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vmax.vx v8, v8, a0
 ; CHECK-NEXT:    li a0, 1
 ; CHECK-NEXT:    slli a0, a0, 31
@@ -627,7 +627,7 @@ define void @trunc_sat_u16u64_notopt(ptr %x, ptr %y) {
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
 ; CHECK-NEXT:    lui a0, 8
-; CHECK-NEXT:    addiw a0, a0, -1
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vminu.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnsrl.wi v10, v8, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvqdotq.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvqdotq.ll
index 0237faea9efb..a189711d1147 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvqdotq.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvqdotq.ll
@@ -230,29 +230,17 @@ define i32 @reduce_of_sext(<16 x i8> %a) {
 ; NODOT-NEXT:    vmv.x.s a0, v8
 ; NODOT-NEXT:    ret
 ;
-; DOT32-LABEL: reduce_of_sext:
-; DOT32:       # %bb.0: # %entry
-; DOT32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; DOT32-NEXT:    vmv.v.i v9, 0
-; DOT32-NEXT:    lui a0, 4112
-; DOT32-NEXT:    addi a0, a0, 257
-; DOT32-NEXT:    vqdot.vx v9, v8, a0
-; DOT32-NEXT:    vmv.s.x v8, zero
-; DOT32-NEXT:    vredsum.vs v8, v9, v8
-; DOT32-NEXT:    vmv.x.s a0, v8
-; DOT32-NEXT:    ret
-;
-; DOT64-LABEL: reduce_of_sext:
-; DOT64:       # %bb.0: # %entry
-; DOT64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; DOT64-NEXT:    vmv.v.i v9, 0
-; DOT64-NEXT:    lui a0, 4112
-; DOT64-NEXT:    addiw a0, a0, 257
-; DOT64-NEXT:    vqdot.vx v9, v8, a0
-; DOT64-NEXT:    vmv.s.x v8, zero
-; DOT64-NEXT:    vredsum.vs v8, v9, v8
-; DOT64-NEXT:    vmv.x.s a0, v8
-; DOT64-NEXT:    ret
+; DOT-LABEL: reduce_of_sext:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; DOT-NEXT:    vmv.v.i v9, 1
+; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT:    vmv.v.i v10, 0
+; DOT-NEXT:    vqdot.vv v10, v8, v9
+; DOT-NEXT:    vmv.s.x v8, zero
+; DOT-NEXT:    vredsum.vs v8, v10, v8
+; DOT-NEXT:    vmv.x.s a0, v8
+; DOT-NEXT:    ret
 entry:
   %a.ext = sext <16 x i8> %a to <16 x i32>
   %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a.ext)
@@ -269,29 +257,17 @@ define i32 @reduce_of_zext(<16 x i8> %a) {
 ; NODOT-NEXT:    vmv.x.s a0, v8
 ; NODOT-NEXT:    ret
 ;
-; DOT32-LABEL: reduce_of_zext:
-; DOT32:       # %bb.0: # %entry
-; DOT32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; DOT32-NEXT:    vmv.v.i v9, 0
-; DOT32-NEXT:    lui a0, 4112
-; DOT32-NEXT:    addi a0, a0, 257
-; DOT32-NEXT:    vqdotu.vx v9, v8, a0
-; DOT32-NEXT:    vmv.s.x v8, zero
-; DOT32-NEXT:    vredsum.vs v8, v9, v8
-; DOT32-NEXT:    vmv.x.s a0, v8
-; DOT32-NEXT:    ret
-;
-; DOT64-LABEL: reduce_of_zext:
-; DOT64:       # %bb.0: # %entry
-; DOT64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; DOT64-NEXT:    vmv.v.i v9, 0
-; DOT64-NEXT:    lui a0, 4112
-; DOT64-NEXT:    addiw a0, a0, 257
-; DOT64-NEXT:    vqdotu.vx v9, v8, a0
-; DOT64-NEXT:    vmv.s.x v8, zero
-; DOT64-NEXT:    vredsum.vs v8, v9, v8
-; DOT64-NEXT:    vmv.x.s a0, v8
-; DOT64-NEXT:    ret
+; DOT-LABEL: reduce_of_zext:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; DOT-NEXT:    vmv.v.i v9, 1
+; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT:    vmv.v.i v10, 0
+; DOT-NEXT:    vqdotu.vv v10, v8, v9
+; DOT-NEXT:    vmv.s.x v8, zero
+; DOT-NEXT:    vredsum.vs v8, v10, v8
+; DOT-NEXT:    vmv.x.s a0, v8
+; DOT-NEXT:    ret
 entry:
   %a.ext = zext <16 x i8> %a to <16 x i32>
   %res = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a.ext)
@@ -670,23 +646,31 @@ entry:
 }
 
 define <1 x i32> @vqdotsu_vv_partial_reduce_v1i32_v4i8(<4 x i8> %a, <4 x i8> %b) {
-; CHECK-LABEL: vqdotsu_vv_partial_reduce_v1i32_v4i8:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vsext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v8, v9
-; CHECK-NEXT:    vwmulsu.vv v9, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v9, 3
-; CHECK-NEXT:    vslidedown.vi v10, v9, 2
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 1
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vadd.vv v9, v9, v10
-; CHECK-NEXT:    vadd.vv v8, v9, v8
-; CHECK-NEXT:    ret
+; NODOT-LABEL: vqdotsu_vv_partial_reduce_v1i32_v4i8:
+; NODOT:       # %bb.0: # %entry
+; NODOT-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; NODOT-NEXT:    vsext.vf2 v10, v8
+; NODOT-NEXT:    vzext.vf2 v8, v9
+; NODOT-NEXT:    vwmulsu.vv v9, v10, v8
+; NODOT-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; NODOT-NEXT:    vslidedown.vi v8, v9, 3
+; NODOT-NEXT:    vslidedown.vi v10, v9, 2
+; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; NODOT-NEXT:    vadd.vv v8, v8, v9
+; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; NODOT-NEXT:    vslidedown.vi v9, v9, 1
+; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; NODOT-NEXT:    vadd.vv v9, v9, v10
+; NODOT-NEXT:    vadd.vv v8, v9, v8
+; NODOT-NEXT:    ret
+;
+; DOT-LABEL: vqdotsu_vv_partial_reduce_v1i32_v4i8:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; DOT-NEXT:    vmv.s.x v10, zero
+; DOT-NEXT:    vqdotsu.vv v10, v8, v9
+; DOT-NEXT:    vmv1r.v v8, v10
+; DOT-NEXT:    ret
 entry:
   %a.sext = sext <4 x i8> %a to <4 x i32>
   %b.sext = zext <4 x i8> %b to <4 x i32>
@@ -696,23 +680,31 @@ entry:
 }
 
 define <1 x i32> @vqdotsu_vv_partial_reduce_swapped(<4 x i8> %a, <4 x i8> %b) {
-; CHECK-LABEL: vqdotsu_vv_partial_reduce_swapped:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vsext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v8, v9
-; CHECK-NEXT:    vwmulsu.vv v9, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v9, 3
-; CHECK-NEXT:    vslidedown.vi v10, v9, 2
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 1
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vadd.vv v9, v9, v10
-; CHECK-NEXT:    vadd.vv v8, v9, v8
-; CHECK-NEXT:    ret
+; NODOT-LABEL: vqdotsu_vv_partial_reduce_swapped:
+; NODOT:       # %bb.0: # %entry
+; NODOT-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; NODOT-NEXT:    vsext.vf2 v10, v8
+; NODOT-NEXT:    vzext.vf2 v8, v9
+; NODOT-NEXT:    vwmulsu.vv v9, v10, v8
+; NODOT-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; NODOT-NEXT:    vslidedown.vi v8, v9, 3
+; NODOT-NEXT:    vslidedown.vi v10, v9, 2
+; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; NODOT-NEXT:    vadd.vv v8, v8, v9
+; NODOT-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; NODOT-NEXT:    vslidedown.vi v9, v9, 1
+; NODOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; NODOT-NEXT:    vadd.vv v9, v9, v10
+; NODOT-NEXT:    vadd.vv v8, v9, v8
+; NODOT-NEXT:    ret
+;
+; DOT-LABEL: vqdotsu_vv_partial_reduce_swapped:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; DOT-NEXT:    vmv.s.x v10, zero
+; DOT-NEXT:    vqdotsu.vv v10, v8, v9
+; DOT-NEXT:    vmv1r.v v8, v10
+; DOT-NEXT:    ret
 entry:
   %a.ext = sext <4 x i8> %a to <4 x i32>
   %b.ext = zext <4 x i8> %b to <4 x i32>
@@ -1089,222 +1081,291 @@ entry:
 
 ; Test legalization - type split
 define <64 x i32> @vqdotsu_vv_partial_v64i32_v256i8(<256 x i8> %a, <256 x i8> %b) {
-; CHECK-LABEL: vqdotsu_vv_partial_v64i32_v256i8:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    mv a2, a1
-; CHECK-NEXT:    slli a1, a1, 2
-; CHECK-NEXT:    add a1, a1, a2
-; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 4
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 5
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    addi a1, a0, 128
-; CHECK-NEXT:    li a2, 128
-; CHECK-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
-; CHECK-NEXT:    vle8.v v0, (a0)
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    mv a3, a0
-; CHECK-NEXT:    slli a0, a0, 1
-; CHECK-NEXT:    add a0, a0, a3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v0, (a0) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v24, v8, a0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vsext.vf2 v8, v24
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v12, v0, a0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v4, v12
-; CHECK-NEXT:    vwmulsu.vv v24, v8, v4
-; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    slli a3, a3, 5
-; CHECK-NEXT:    add a3, sp, a3
-; CHECK-NEXT:    addi a3, a3, 16
-; CHECK-NEXT:    vl8r.v v8, (a3) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsext.vf2 v4, v8
-; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    slli a3, a3, 3
-; CHECK-NEXT:    mv a4, a3
-; CHECK-NEXT:    slli a3, a3, 1
-; CHECK-NEXT:    add a3, a3, a4
-; CHECK-NEXT:    add a3, sp, a3
-; CHECK-NEXT:    addi a3, a3, 16
-; CHECK-NEXT:    vl8r.v v8, (a3) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vzext.vf2 v0, v8
-; CHECK-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
-; CHECK-NEXT:    addi a1, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vwmaccsu.vv v24, v4, v0
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vsext.vf2 v12, v4
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v4, v8, a0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v16, v4
-; CHECK-NEXT:    vwmulsu.vv v0, v12, v16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 4
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsext.vf2 v12, v16
-; CHECK-NEXT:    vzext.vf2 v20, v8
-; CHECK-NEXT:    vwmaccsu.vv v0, v12, v20
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 5
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v16, a1
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 5
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
-; CHECK-NEXT:    mv a3, a2
-; CHECK-NEXT:    slli a2, a2, 1
-; CHECK-NEXT:    add a2, a2, a3
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vslidedown.vx v8, v16, a1
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 5
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vsext.vf2 v16, v8
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vzext.vf2 v20, v8
-; CHECK-NEXT:    vwmaccsu.vv v24, v16, v20
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 4
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
-; CHECK-NEXT:    vslidedown.vx v16, v16, a1
-; CHECK-NEXT:    addi a2, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vslidedown.vx v8, v8, a1
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    mv a2, a1
-; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    add a1, a1, a2
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vsext.vf2 v8, v16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 4
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vs4r.v v8, (a1) # vscale x 32-byte Folded Spill
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    mv a2, a1
-; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    add a1, a1, a2
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vzext.vf2 v20, v8
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 4
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vl4r.v v8, (a1) # vscale x 32-byte Folded Reload
-; CHECK-NEXT:    vwmaccsu.vv v0, v8, v20
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 5
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v20, v8, a0
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vsext.vf2 v12, v20
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 5
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vs4r.v v12, (a1) # vscale x 32-byte Folded Spill
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 5
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vl4r.v v8, (a1) # vscale x 32-byte Folded Reload
-; CHECK-NEXT:    vwmaccsu.vv v24, v8, v12
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v12, v16, a0
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    mv a2, a1
-; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    add a1, a1, a2
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
-; CHECK-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vslidedown.vx v8, v16, a0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vsext.vf2 v16, v12
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    vwmaccsu.vv v0, v16, v12
-; CHECK-NEXT:    vmv8r.v v8, v24
-; CHECK-NEXT:    vmv8r.v v16, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    mv a1, a0
-; CHECK-NEXT:    slli a0, a0, 2
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    add sp, sp, a0
-; CHECK-NEXT:    .cfi_def_cfa sp, 16
-; CHECK-NEXT:    addi sp, sp, 16
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    ret
+; NODOT-LABEL: vqdotsu_vv_partial_v64i32_v256i8:
+; NODOT:       # %bb.0: # %entry
+; NODOT-NEXT:    addi sp, sp, -16
+; NODOT-NEXT:    .cfi_def_cfa_offset 16
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 3
+; NODOT-NEXT:    mv a2, a1
+; NODOT-NEXT:    slli a1, a1, 2
+; NODOT-NEXT:    add a1, a1, a2
+; NODOT-NEXT:    sub sp, sp, a1
+; NODOT-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 4
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 5
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; NODOT-NEXT:    addi a1, a0, 128
+; NODOT-NEXT:    li a2, 128
+; NODOT-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; NODOT-NEXT:    vle8.v v0, (a0)
+; NODOT-NEXT:    csrr a0, vlenb
+; NODOT-NEXT:    slli a0, a0, 3
+; NODOT-NEXT:    mv a3, a0
+; NODOT-NEXT:    slli a0, a0, 1
+; NODOT-NEXT:    add a0, a0, a3
+; NODOT-NEXT:    add a0, sp, a0
+; NODOT-NEXT:    addi a0, a0, 16
+; NODOT-NEXT:    vs8r.v v0, (a0) # vscale x 64-byte Folded Spill
+; NODOT-NEXT:    li a0, 32
+; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vx v24, v8, a0
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vsext.vf2 v8, v24
+; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vx v12, v0, a0
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vzext.vf2 v4, v12
+; NODOT-NEXT:    vwmulsu.vv v24, v8, v4
+; NODOT-NEXT:    csrr a3, vlenb
+; NODOT-NEXT:    slli a3, a3, 5
+; NODOT-NEXT:    add a3, sp, a3
+; NODOT-NEXT:    addi a3, a3, 16
+; NODOT-NEXT:    vl8r.v v8, (a3) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vsext.vf2 v4, v8
+; NODOT-NEXT:    csrr a3, vlenb
+; NODOT-NEXT:    slli a3, a3, 3
+; NODOT-NEXT:    mv a4, a3
+; NODOT-NEXT:    slli a3, a3, 1
+; NODOT-NEXT:    add a3, a3, a4
+; NODOT-NEXT:    add a3, sp, a3
+; NODOT-NEXT:    addi a3, a3, 16
+; NODOT-NEXT:    vl8r.v v8, (a3) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vzext.vf2 v0, v8
+; NODOT-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; NODOT-NEXT:    vle8.v v8, (a1)
+; NODOT-NEXT:    addi a1, sp, 16
+; NODOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vwmaccsu.vv v24, v4, v0
+; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vx v4, v16, a0
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vsext.vf2 v12, v4
+; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vx v4, v8, a0
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vzext.vf2 v16, v4
+; NODOT-NEXT:    vwmulsu.vv v0, v12, v16
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 4
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vsext.vf2 v12, v16
+; NODOT-NEXT:    vzext.vf2 v20, v8
+; NODOT-NEXT:    vwmaccsu.vv v0, v12, v20
+; NODOT-NEXT:    li a1, 64
+; NODOT-NEXT:    csrr a2, vlenb
+; NODOT-NEXT:    slli a2, a2, 5
+; NODOT-NEXT:    add a2, sp, a2
+; NODOT-NEXT:    addi a2, a2, 16
+; NODOT-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; NODOT-NEXT:    vslidedown.vx v8, v16, a1
+; NODOT-NEXT:    csrr a2, vlenb
+; NODOT-NEXT:    slli a2, a2, 5
+; NODOT-NEXT:    add a2, sp, a2
+; NODOT-NEXT:    addi a2, a2, 16
+; NODOT-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; NODOT-NEXT:    csrr a2, vlenb
+; NODOT-NEXT:    slli a2, a2, 3
+; NODOT-NEXT:    mv a3, a2
+; NODOT-NEXT:    slli a2, a2, 1
+; NODOT-NEXT:    add a2, a2, a3
+; NODOT-NEXT:    add a2, sp, a2
+; NODOT-NEXT:    addi a2, a2, 16
+; NODOT-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vslidedown.vx v8, v16, a1
+; NODOT-NEXT:    csrr a2, vlenb
+; NODOT-NEXT:    slli a2, a2, 3
+; NODOT-NEXT:    add a2, sp, a2
+; NODOT-NEXT:    addi a2, a2, 16
+; NODOT-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; NODOT-NEXT:    csrr a2, vlenb
+; NODOT-NEXT:    slli a2, a2, 5
+; NODOT-NEXT:    add a2, sp, a2
+; NODOT-NEXT:    addi a2, a2, 16
+; NODOT-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vsext.vf2 v16, v8
+; NODOT-NEXT:    csrr a2, vlenb
+; NODOT-NEXT:    slli a2, a2, 3
+; NODOT-NEXT:    add a2, sp, a2
+; NODOT-NEXT:    addi a2, a2, 16
+; NODOT-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vzext.vf2 v20, v8
+; NODOT-NEXT:    vwmaccsu.vv v24, v16, v20
+; NODOT-NEXT:    csrr a2, vlenb
+; NODOT-NEXT:    slli a2, a2, 4
+; NODOT-NEXT:    add a2, sp, a2
+; NODOT-NEXT:    addi a2, a2, 16
+; NODOT-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; NODOT-NEXT:    vslidedown.vx v16, v16, a1
+; NODOT-NEXT:    addi a2, sp, 16
+; NODOT-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vslidedown.vx v8, v8, a1
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 3
+; NODOT-NEXT:    mv a2, a1
+; NODOT-NEXT:    slli a1, a1, 1
+; NODOT-NEXT:    add a1, a1, a2
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vsext.vf2 v8, v16
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 4
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vs4r.v v8, (a1) # vscale x 32-byte Folded Spill
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 3
+; NODOT-NEXT:    mv a2, a1
+; NODOT-NEXT:    slli a1, a1, 1
+; NODOT-NEXT:    add a1, a1, a2
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vzext.vf2 v20, v8
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 4
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vl4r.v v8, (a1) # vscale x 32-byte Folded Reload
+; NODOT-NEXT:    vwmaccsu.vv v0, v8, v20
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 5
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vx v20, v8, a0
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 3
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vslidedown.vx v8, v8, a0
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vsext.vf2 v12, v20
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 5
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vs4r.v v12, (a1) # vscale x 32-byte Folded Spill
+; NODOT-NEXT:    vzext.vf2 v12, v8
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 5
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vl4r.v v8, (a1) # vscale x 32-byte Folded Reload
+; NODOT-NEXT:    vwmaccsu.vv v24, v8, v12
+; NODOT-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vx v12, v16, a0
+; NODOT-NEXT:    csrr a1, vlenb
+; NODOT-NEXT:    slli a1, a1, 3
+; NODOT-NEXT:    mv a2, a1
+; NODOT-NEXT:    slli a1, a1, 1
+; NODOT-NEXT:    add a1, a1, a2
+; NODOT-NEXT:    add a1, sp, a1
+; NODOT-NEXT:    addi a1, a1, 16
+; NODOT-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
+; NODOT-NEXT:    vslidedown.vx v8, v16, a0
+; NODOT-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; NODOT-NEXT:    vsext.vf2 v16, v12
+; NODOT-NEXT:    vzext.vf2 v12, v8
+; NODOT-NEXT:    vwmaccsu.vv v0, v16, v12
+; NODOT-NEXT:    vmv8r.v v8, v24
+; NODOT-NEXT:    vmv8r.v v16, v0
+; NODOT-NEXT:    csrr a0, vlenb
+; NODOT-NEXT:    slli a0, a0, 3
+; NODOT-NEXT:    mv a1, a0
+; NODOT-NEXT:    slli a0, a0, 2
+; NODOT-NEXT:    add a0, a0, a1
+; NODOT-NEXT:    add sp, sp, a0
+; NODOT-NEXT:    .cfi_def_cfa sp, 16
+; NODOT-NEXT:    addi sp, sp, 16
+; NODOT-NEXT:    .cfi_def_cfa_offset 0
+; NODOT-NEXT:    ret
+;
+; DOT-LABEL: vqdotsu_vv_partial_v64i32_v256i8:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    addi sp, sp, -16
+; DOT-NEXT:    .cfi_def_cfa_offset 16
+; DOT-NEXT:    csrr a1, vlenb
+; DOT-NEXT:    slli a1, a1, 5
+; DOT-NEXT:    sub sp, sp, a1
+; DOT-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; DOT-NEXT:    csrr a1, vlenb
+; DOT-NEXT:    slli a1, a1, 3
+; DOT-NEXT:    mv a2, a1
+; DOT-NEXT:    slli a1, a1, 1
+; DOT-NEXT:    add a1, a1, a2
+; DOT-NEXT:    add a1, sp, a1
+; DOT-NEXT:    addi a1, a1, 16
+; DOT-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
+; DOT-NEXT:    csrr a1, vlenb
+; DOT-NEXT:    slli a1, a1, 4
+; DOT-NEXT:    add a1, sp, a1
+; DOT-NEXT:    addi a1, a1, 16
+; DOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; DOT-NEXT:    addi a1, a0, 128
+; DOT-NEXT:    li a2, 128
+; DOT-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; DOT-NEXT:    vle8.v v8, (a0)
+; DOT-NEXT:    csrr a0, vlenb
+; DOT-NEXT:    slli a0, a0, 3
+; DOT-NEXT:    add a0, sp, a0
+; DOT-NEXT:    addi a0, a0, 16
+; DOT-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; DOT-NEXT:    li a0, 32
+; DOT-NEXT:    vle8.v v8, (a1)
+; DOT-NEXT:    addi a1, sp, 16
+; DOT-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; DOT-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; DOT-NEXT:    vmv.v.i v24, 0
+; DOT-NEXT:    vmv.v.i v0, 0
+; DOT-NEXT:    csrr a0, vlenb
+; DOT-NEXT:    slli a0, a0, 4
+; DOT-NEXT:    add a0, sp, a0
+; DOT-NEXT:    addi a0, a0, 16
+; DOT-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
+; DOT-NEXT:    csrr a0, vlenb
+; DOT-NEXT:    slli a0, a0, 3
+; DOT-NEXT:    add a0, sp, a0
+; DOT-NEXT:    addi a0, a0, 16
+; DOT-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; DOT-NEXT:    vqdotsu.vv v0, v16, v8
+; DOT-NEXT:    csrr a0, vlenb
+; DOT-NEXT:    slli a0, a0, 3
+; DOT-NEXT:    mv a1, a0
+; DOT-NEXT:    slli a0, a0, 1
+; DOT-NEXT:    add a0, a0, a1
+; DOT-NEXT:    add a0, sp, a0
+; DOT-NEXT:    addi a0, a0, 16
+; DOT-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
+; DOT-NEXT:    addi a0, sp, 16
+; DOT-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; DOT-NEXT:    vqdotsu.vv v24, v16, v8
+; DOT-NEXT:    vmv.v.v v8, v0
+; DOT-NEXT:    vmv.v.v v16, v24
+; DOT-NEXT:    csrr a0, vlenb
+; DOT-NEXT:    slli a0, a0, 5
+; DOT-NEXT:    add sp, sp, a0
+; DOT-NEXT:    .cfi_def_cfa sp, 16
+; DOT-NEXT:    addi sp, sp, 16
+; DOT-NEXT:    .cfi_def_cfa_offset 0
+; DOT-NEXT:    ret
 entry:
   %a.ext = sext <256 x i8> %a to <256 x i32>
   %b.ext = zext <256 x i8> %b to <256 x i32>
@@ -1313,6 +1374,56 @@ entry:
   ret <64 x i32> %res
 }
 
+; Test legalization - integer promote
+define <4 x i31> @vqdotsu_vv_partial_v4i31_v16i7(<16 x i7> %a, <16 x i7> %b) {
+; NODOT-LABEL: vqdotsu_vv_partial_v4i31_v16i7:
+; NODOT:       # %bb.0: # %entry
+; NODOT-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; NODOT-NEXT:    vzext.vf4 v12, v8
+; NODOT-NEXT:    li a0, 127
+; NODOT-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; NODOT-NEXT:    vand.vx v16, v9, a0
+; NODOT-NEXT:    lui a0, 524288
+; NODOT-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; NODOT-NEXT:    vsll.vi v8, v12, 25
+; NODOT-NEXT:    addi a0, a0, -1
+; NODOT-NEXT:    vsra.vi v8, v8, 25
+; NODOT-NEXT:    vzext.vf4 v12, v16
+; NODOT-NEXT:    vmul.vv v8, v12, v8
+; NODOT-NEXT:    vand.vx v8, v8, a0
+; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vi v12, v8, 12
+; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; NODOT-NEXT:    vadd.vv v16, v12, v8
+; NODOT-NEXT:    vsetivli zero, 4, e32, m4, ta, ma
+; NODOT-NEXT:    vslidedown.vi v12, v8, 8
+; NODOT-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
+; NODOT-NEXT:    vslidedown.vi v8, v8, 4
+; NODOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; NODOT-NEXT:    vadd.vv v8, v8, v12
+; NODOT-NEXT:    vadd.vv v8, v8, v16
+; NODOT-NEXT:    ret
+;
+; DOT-LABEL: vqdotsu_vv_partial_v4i31_v16i7:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    li a0, 127
+; DOT-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; DOT-NEXT:    vadd.vv v8, v8, v8
+; DOT-NEXT:    vand.vx v9, v9, a0
+; DOT-NEXT:    vsra.vi v10, v8, 1
+; DOT-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT:    vmv.v.i v8, 0
+; DOT-NEXT:    vqdotsu.vv v8, v10, v9
+; DOT-NEXT:    ret
+entry:
+  %a.ext = sext <16 x i7> %a to <16 x i31>
+  %b.ext = zext <16 x i7> %b to <16 x i31>
+  %mul = mul <16 x i31> %b.ext, %a.ext
+  %res = call <4 x i31> @llvm.experimental.vector.partial.reduce.add(<4 x i31> zeroinitializer, <16 x i31> %mul)
+  ret <4 x i31> %res
+}
+
+
 ; Test legalization - expand
 define <1 x i32> @vqdotsu_vv_partial_v1i32_v2i8(<2 x i8> %a, <2 x i8> %b) {
 ; CHECK-LABEL: vqdotsu_vv_partial_v1i32_v2i8:
@@ -1441,3 +1552,6 @@ entry:
   %res = call <4 x i32> @llvm.experimental.vector.partial.reduce.add(<4 x i32> zeroinitializer, <16 x i32> %a.ext)
   ret <4 x i32> %res
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; DOT32: {{.*}}
+; DOT64: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 0640a6f3af25..f9ac53b76eba 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -897,7 +897,7 @@ define <2 x i16> @stest_f64i16(<2 x double> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.d a1, fa1, rtz
 ; CHECK-NOV-NEXT:    lui a2, 8
-; CHECK-NOV-NEXT:    addiw a2, a2, -1
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECK-NOV-NEXT:    bge a1, a2, .LBB9_5
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -945,7 +945,7 @@ define <2 x i16> @utest_f64i16(<2 x double> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a2, 16
-; CHECK-NOV-NEXT:    addiw a2, a2, -1
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    fcvt.wu.d a1, fa1, rtz
 ; CHECK-NOV-NEXT:    bgeu a0, a2, .LBB10_3
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -979,7 +979,7 @@ define <2 x i16> @ustest_f64i16(<2 x double> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.d a1, fa1, rtz
 ; CHECK-NOV-NEXT:    lui a2, 16
-; CHECK-NOV-NEXT:    addiw a2, a2, -1
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECK-NOV-NEXT:    blt a1, a2, .LBB11_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -1020,7 +1020,7 @@ define <4 x i16> @stest_f32i16(<4 x float> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.s a1, fa3, rtz
 ; CHECK-NOV-NEXT:    lui a5, 8
-; CHECK-NOV-NEXT:    addiw a5, a5, -1
+; CHECK-NOV-NEXT:    addi a5, a5, -1
 ; CHECK-NOV-NEXT:    fcvt.w.s a2, fa2, rtz
 ; CHECK-NOV-NEXT:    bge a1, a5, .LBB12_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -1096,7 +1096,7 @@ define <4 x i16> @utest_f32i16(<4 x float> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.wu.s a1, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a3, 16
-; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    addi a3, a3, -1
 ; CHECK-NOV-NEXT:    fcvt.wu.s a2, fa1, rtz
 ; CHECK-NOV-NEXT:    bgeu a1, a3, .LBB13_6
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -1148,7 +1148,7 @@ define <4 x i16> @ustest_f32i16(<4 x float> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.s a1, fa3, rtz
 ; CHECK-NOV-NEXT:    lui a4, 16
-; CHECK-NOV-NEXT:    addiw a4, a4, -1
+; CHECK-NOV-NEXT:    addi a4, a4, -1
 ; CHECK-NOV-NEXT:    fcvt.w.s a2, fa2, rtz
 ; CHECK-NOV-NEXT:    bge a1, a4, .LBB14_6
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -1283,7 +1283,7 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a7, 8
-; CHECK-NOV-NEXT:    addiw a7, a7, -1
+; CHECK-NOV-NEXT:    addi a7, a7, -1
 ; CHECK-NOV-NEXT:    bge a0, a7, .LBB15_18
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a1, fs5, rtz
@@ -1667,7 +1667,7 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a3, 16
-; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    addi a3, a3, -1
 ; CHECK-NOV-NEXT:    bgeu a0, a3, .LBB16_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a1, fs5, rtz
@@ -2007,7 +2007,7 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a4, 16
-; CHECK-NOV-NEXT:    addiw a4, a4, -1
+; CHECK-NOV-NEXT:    addi a4, a4, -1
 ; CHECK-NOV-NEXT:    bge a0, a4, .LBB17_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a1, fs5, rtz
@@ -4449,7 +4449,7 @@ define <2 x i16> @stest_f64i16_mm(<2 x double> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.d a1, fa1, rtz
 ; CHECK-NOV-NEXT:    lui a2, 8
-; CHECK-NOV-NEXT:    addiw a2, a2, -1
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECK-NOV-NEXT:    bge a1, a2, .LBB36_5
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -4495,7 +4495,7 @@ define <2 x i16> @utest_f64i16_mm(<2 x double> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a2, 16
-; CHECK-NOV-NEXT:    addiw a2, a2, -1
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    fcvt.wu.d a1, fa1, rtz
 ; CHECK-NOV-NEXT:    bgeu a0, a2, .LBB37_3
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -4528,7 +4528,7 @@ define <2 x i16> @ustest_f64i16_mm(<2 x double> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.d a1, fa1, rtz
 ; CHECK-NOV-NEXT:    lui a2, 16
-; CHECK-NOV-NEXT:    addiw a2, a2, -1
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECK-NOV-NEXT:    blt a1, a2, .LBB38_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -4567,7 +4567,7 @@ define <4 x i16> @stest_f32i16_mm(<4 x float> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.s a1, fa3, rtz
 ; CHECK-NOV-NEXT:    lui a5, 8
-; CHECK-NOV-NEXT:    addiw a5, a5, -1
+; CHECK-NOV-NEXT:    addi a5, a5, -1
 ; CHECK-NOV-NEXT:    fcvt.w.s a2, fa2, rtz
 ; CHECK-NOV-NEXT:    bge a1, a5, .LBB39_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -4641,7 +4641,7 @@ define <4 x i16> @utest_f32i16_mm(<4 x float> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.wu.s a1, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a3, 16
-; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    addi a3, a3, -1
 ; CHECK-NOV-NEXT:    fcvt.wu.s a2, fa1, rtz
 ; CHECK-NOV-NEXT:    bgeu a1, a3, .LBB40_6
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -4692,7 +4692,7 @@ define <4 x i16> @ustest_f32i16_mm(<4 x float> %x) {
 ; CHECK-NOV:       # %bb.0: # %entry
 ; CHECK-NOV-NEXT:    fcvt.w.s a1, fa3, rtz
 ; CHECK-NOV-NEXT:    lui a3, 16
-; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    addi a3, a3, -1
 ; CHECK-NOV-NEXT:    fcvt.w.s a2, fa2, rtz
 ; CHECK-NOV-NEXT:    bge a1, a3, .LBB41_6
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
@@ -4825,7 +4825,7 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a7, 8
-; CHECK-NOV-NEXT:    addiw a7, a7, -1
+; CHECK-NOV-NEXT:    addi a7, a7, -1
 ; CHECK-NOV-NEXT:    bge a0, a7, .LBB42_18
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a1, fs5, rtz
@@ -5207,7 +5207,7 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a3, 16
-; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    addi a3, a3, -1
 ; CHECK-NOV-NEXT:    bgeu a0, a3, .LBB43_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a1, fs5, rtz
@@ -5546,7 +5546,7 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-NOV-NEXT:    lui a3, 16
-; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    addi a3, a3, -1
 ; CHECK-NOV-NEXT:    bge a0, a3, .LBB44_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a1, fs5, rtz
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
index cafc36bd1ef6..a68f171a7bde 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
@@ -461,7 +461,7 @@ define <vscale x 1 x float> @test5(<vscale x 1 x float> %0, <vscale x 1 x float>
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
 ; CHECK-NEXT:    lui a0, 66
 ; CHECK-NEXT:    fsrm a2
-; CHECK-NEXT:    addiw a0, a0, 769
+; CHECK-NEXT:    addi a0, a0, 769
 ; CHECK-NEXT:    frrm a2
 ; CHECK-NEXT:    slli a2, a2, 2
 ; CHECK-NEXT:    srl a0, a0, a2
@@ -477,7 +477,7 @@ define <vscale x 1 x float> @test5(<vscale x 1 x float> %0, <vscale x 1 x float>
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
 ; UNOPT-NEXT:    lui a0, 66
 ; UNOPT-NEXT:    fsrm a2
-; UNOPT-NEXT:    addiw a0, a0, 769
+; UNOPT-NEXT:    addi a0, a0, 769
 ; UNOPT-NEXT:    frrm a2
 ; UNOPT-NEXT:    slli a2, a2, 2
 ; UNOPT-NEXT:    srl a0, a0, a2
@@ -590,7 +590,7 @@ define <vscale x 1 x float> @after_fsrm4(<vscale x 1 x float> %0, <vscale x 1 x
 ; CHECK-NEXT:    slli a0, a0, 32
 ; CHECK-NEXT:    lui a2, 66
 ; CHECK-NEXT:    srli a0, a0, 30
-; CHECK-NEXT:    addiw a2, a2, 769
+; CHECK-NEXT:    addi a2, a2, 769
 ; CHECK-NEXT:    srl a0, a2, a0
 ; CHECK-NEXT:    andi a0, a0, 7
 ; CHECK-NEXT:    fsrm a0
@@ -603,7 +603,7 @@ define <vscale x 1 x float> @after_fsrm4(<vscale x 1 x float> %0, <vscale x 1 x
 ; UNOPT-NEXT:    slli a0, a0, 32
 ; UNOPT-NEXT:    lui a2, 66
 ; UNOPT-NEXT:    srli a0, a0, 30
-; UNOPT-NEXT:    addiw a2, a2, 769
+; UNOPT-NEXT:    addi a2, a2, 769
 ; UNOPT-NEXT:    srl a0, a2, a0
 ; UNOPT-NEXT:    andi a0, a0, 7
 ; UNOPT-NEXT:    fsrm a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
index d992669306fb..75537406f351 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
@@ -21,7 +21,27 @@ define <vscale x 1 x i8> @masked_load_nxv1i8(ptr %a, <vscale x 1 x i1> %mask) no
   %load = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8(ptr %a, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> undef)
   ret <vscale x 1 x i8> %load
 }
-declare <vscale x 1 x i8> @llvm.masked.load.nxv1i8(ptr, i32, <vscale x 1 x i1>, <vscale x 1 x i8>)
+
+define <vscale x 1 x i8> @masked_load_passthru_nxv1i8(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; V-LABEL: masked_load_passthru_nxv1i8:
+; V:       # %bb.0:
+; V-NEXT:    vsetvli a1, zero, e8, mf8, ta, mu
+; V-NEXT:    vmv.v.i v8, 0
+; V-NEXT:    vle8.v v8, (a0), v0.t
+; V-NEXT:    ret
+;
+; ZVE32-LABEL: masked_load_passthru_nxv1i8:
+; ZVE32:       # %bb.0:
+; ZVE32-NEXT:    csrr a1, vlenb
+; ZVE32-NEXT:    srli a1, a1, 3
+; ZVE32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; ZVE32-NEXT:    vmv.v.i v8, 0
+; ZVE32-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
+; ZVE32-NEXT:    vle8.v v8, (a0), v0.t
+; ZVE32-NEXT:    ret
+  %load = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8(ptr %a, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> zeroinitializer)
+  ret <vscale x 1 x i8> %load
+}
 
 define <vscale x 1 x i16> @masked_load_nxv1i16(ptr %a, <vscale x 1 x i1> %mask) nounwind {
 ; V-LABEL: masked_load_nxv1i16:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll
index 6bed05dcda15..896394017b6f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll
@@ -138,7 +138,7 @@ define void @memset_8(ptr %a, i8 %value) nounwind {
 ; RV64-FAST:       # %bb.0:
 ; RV64-FAST-NEXT:    zext.b a1, a1
 ; RV64-FAST-NEXT:    lui a2, 4112
-; RV64-FAST-NEXT:    addiw a2, a2, 257
+; RV64-FAST-NEXT:    addi a2, a2, 257
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
@@ -278,7 +278,7 @@ define void @aligned_memset_8(ptr align 8 %a, i8 %value) nounwind {
 ; RV64-BOTH:       # %bb.0:
 ; RV64-BOTH-NEXT:    zext.b a1, a1
 ; RV64-BOTH-NEXT:    lui a2, 4112
-; RV64-BOTH-NEXT:    addiw a2, a2, 257
+; RV64-BOTH-NEXT:    addi a2, a2, 257
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/pr141907.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/pr141907.ll
new file mode 100644
index 000000000000..648b47dc440c
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/pr141907.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -mcpu=sifive-p670 | FileCheck %s
+
+define void @pr141907(ptr %0) nounwind {
+; CHECK-LABEL: pr141907:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 2
+; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v9, 0
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    li a1, 0
+; CHECK-NEXT:    vsetvli a3, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v12, 0
+; CHECK-NEXT:    addi a2, sp, 16
+; CHECK-NEXT:  .LBB0_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vs4r.v v8, (a2)
+; CHECK-NEXT:    vsetvli a1, a1, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v11, v9, 0, v0.t
+; CHECK-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vlseg3e32.v v8, (a2)
+; CHECK-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsseg2e16.v v11, (zero)
+; CHECK-NEXT:    bnez a1, .LBB0_1
+; CHECK-NEXT:  .LBB0_2: # %while.body5
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vse16.v v9, (a0)
+; CHECK-NEXT:    j .LBB0_2
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %evl.based.iv = phi i64 [ 0, %entry ], [ %2, %vector.body ]
+  %vector.recur = phi <vscale x 2 x i32> [ zeroinitializer, %entry ], [ %3, %vector.body ]
+  %1 = call i32 @llvm.experimental.get.vector.length.i64(i64 %evl.based.iv, i32 1, i1 true)
+  %2 = zext i32 %1 to i64
+  %wide.masked.load = call <vscale x 6 x i32> @llvm.vp.load.nxv6i32.p0(ptr null, <vscale x 6 x i1> zeroinitializer, i32 0)
+  %deinterleaved.results = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave3.nxv6i32(<vscale x 6 x i32> %wide.masked.load)
+  %3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %deinterleaved.results, 1
+  %vp.cast65 = call <vscale x 2 x i16> @llvm.vp.trunc.nxv2i16.nxv2i32(<vscale x 2 x i32> %vector.recur, <vscale x 2 x i1> zeroinitializer, i32 0)
+  %interleaved.vec = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %vp.cast65, <vscale x 2 x i16> zeroinitializer)
+  call void @llvm.vp.store.nxv4i16.p0(<vscale x 4 x i16> %interleaved.vec, ptr null, <vscale x 4 x i1> splat (i1 true), i32 0)
+  %4 = icmp eq i32 %1, 0
+  br i1 %4, label %while.body5, label %vector.body
+
+while.body5:                                      ; preds = %while.body5, %vector.body
+  %5 = bitcast <vscale x 2 x i32> %3 to <vscale x 4 x i16>
+  %cond52 = extractelement <vscale x 4 x i16> %5, i64 0
+  store i16 %cond52, ptr %0, align 2
+  br label %while.body5
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/pr88799.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/pr88799.ll
index 7212a789f9e7..dbbec85a85c3 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/pr88799.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/pr88799.ll
@@ -5,7 +5,7 @@ define i32 @main() vscale_range(2,2) {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %vector.body
 ; CHECK-NEXT:    lui a0, 1040368
-; CHECK-NEXT:    addiw a0, a0, -144
+; CHECK-NEXT:    addi a0, a0, -144
 ; CHECK-NEXT:    vl2re16.v v8, (a0)
 ; CHECK-NEXT:    vs2r.v v8, (zero)
 ; CHECK-NEXT:    li a0, 0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-dynamic.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-dynamic.ll
index 604271702eba..d666832cf6e0 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-dynamic.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-dynamic.ll
@@ -467,7 +467,7 @@ define void @reserved_call_frame(i64 %n) #0 {
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    call callee_stack_args
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 48
+; RV64I-NEXT:    addi a0, a0, 48
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-rvv.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-rvv.ll
index d7f9ae73eaea..02f03602e84a 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-rvv.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stack-probing-rvv.ll
@@ -347,7 +347,7 @@ define void @f1_vector_4096_arr(ptr %out) #0 {
 ; RV64IV-NEXT:    add sp, sp, a0
 ; RV64IV-NEXT:    .cfi_def_cfa sp, 12304
 ; RV64IV-NEXT:    lui a0, 3
-; RV64IV-NEXT:    addiw a0, a0, 16
+; RV64IV-NEXT:    addi a0, a0, 16
 ; RV64IV-NEXT:    add sp, sp, a0
 ; RV64IV-NEXT:    .cfi_def_cfa_offset 0
 ; RV64IV-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stepvector.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stepvector.ll
index 62339130678d..d4e2c08d70d3 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stepvector.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/stepvector.ll
@@ -501,7 +501,7 @@ define <vscale x 8 x i64> @mul_bigimm_stepvector_nxv8i64() {
 ; RV64-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vid.v v8
 ; RV64-NEXT:    lui a0, 1987
-; RV64-NEXT:    addiw a0, a0, -731
+; RV64-NEXT:    addi a0, a0, -731
 ; RV64-NEXT:    slli a0, a0, 12
 ; RV64-NEXT:    addi a0, a0, -683
 ; RV64-NEXT:    vmul.vx v8, v8, a0
@@ -670,7 +670,7 @@ define <vscale x 16 x i64> @mul_bigimm_stepvector_nxv16i64() {
 ; RV64-NEXT:    lui a1, 1987
 ; RV64-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vid.v v8
-; RV64-NEXT:    addiw a1, a1, -731
+; RV64-NEXT:    addi a1, a1, -731
 ; RV64-NEXT:    slli a1, a1, 12
 ; RV64-NEXT:    addi a1, a1, -683
 ; RV64-NEXT:    mul a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
index 01a90d8a33b6..3ed437eeed2f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
@@ -267,7 +267,7 @@ define void @trunc_sat_i32i64_notopt(ptr %x, ptr %y) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
 ; CHECK-NEXT:    lui a0, 524288
-; CHECK-NEXT:    addiw a0, a0, 1
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a2, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmax.vx v8, v8, a0
 ; CHECK-NEXT:    li a0, 1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index aab2f0827783..ca7f2563e4fc 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -85,11 +85,11 @@ define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) {
 ;
 ; ZIP-LABEL: vector_deinterleave_v2i64_v4i64:
 ; ZIP:       # %bb.0:
-; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; ZIP-NEXT:    vslidedown.vi v10, v8, 2
 ; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; ZIP-NEXT:    ri.vzipodd.vv v9, v8, v10
-; ZIP-NEXT:    vslideup.vi v8, v10, 1
+; ZIP-NEXT:    ri.vunzip2a.vv v10, v8, v9
+; ZIP-NEXT:    ri.vunzip2b.vv v11, v8, v9
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    vmv.v.v v9, v11
 ; ZIP-NEXT:    ret
 %retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec)
 ret {<2 x i64>, <2 x i64>} %retval
@@ -129,62 +129,51 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
 ; ZIP-LABEL: vector_deinterleave_v4i64_v8i64:
 ; ZIP:       # %bb.0:
 ; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; ZIP-NEXT:    vslidedown.vi v12, v8, 1
-; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; ZIP-NEXT:    vmv.v.i v0, 2
-; ZIP-NEXT:    vmv.v.i v14, 12
-; ZIP-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
-; ZIP-NEXT:    vslidedown.vi v16, v8, 4
-; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; ZIP-NEXT:    vslidedown.vi v10, v8, 2
-; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; ZIP-NEXT:    vslidedown.vi v12, v8, 2, v0.t
-; ZIP-NEXT:    ri.vzip2a.vv v18, v8, v10
-; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; ZIP-NEXT:    vslidedown.vi v8, v16, 2
-; ZIP-NEXT:    vmv1r.v v0, v14
-; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; ZIP-NEXT:    ri.vzip2a.vv v12, v16, v8, v0.t
-; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; ZIP-NEXT:    vmv.v.i v0, 8
-; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; ZIP-NEXT:    vslideup.vi v8, v16, 2
-; ZIP-NEXT:    vslideup.vi v8, v16, 1, v0.t
-; ZIP-NEXT:    vmv1r.v v0, v14
-; ZIP-NEXT:    vmerge.vvm v8, v18, v8, v0
-; ZIP-NEXT:    vmv2r.v v10, v12
+; ZIP-NEXT:    ri.vunzip2a.vv v12, v8, v10
+; ZIP-NEXT:    ri.vunzip2b.vv v14, v8, v10
+; ZIP-NEXT:    vmv.v.v v8, v12
+; ZIP-NEXT:    vmv.v.v v10, v14
 ; ZIP-NEXT:    ret
   %retval = call {<4 x i64>, <4 x i64>} @llvm.vector.deinterleave2.v8i64(<8 x i64> %vec)
   ret {<4 x i64>, <4 x i64>} %retval
 }
 
 define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_v8i64_v16i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.i v0, -16
-; CHECK-NEXT:    vid.v v16
-; CHECK-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v24, v8, 8
-; CHECK-NEXT:    vmv.s.x v12, a0
-; CHECK-NEXT:    li a0, 170
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v20, v16, v16
-; CHECK-NEXT:    vmv.s.x v21, a0
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vcompress.vm v16, v8, v12
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vadd.vi v22, v20, -8
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vcompress.vm v12, v8, v21
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vadd.vi v8, v20, -7
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v16, v24, v22, v0.t
-; CHECK-NEXT:    vrgatherei16.vv v12, v24, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v16
-; CHECK-NEXT:    ret
+; V-LABEL: vector_deinterleave_v8i64_v16i64:
+; V:       # %bb.0:
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; V-NEXT:    vmv.v.i v0, -16
+; V-NEXT:    vid.v v16
+; V-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
+; V-NEXT:    vslidedown.vi v24, v8, 8
+; V-NEXT:    vmv.s.x v12, a0
+; V-NEXT:    li a0, 170
+; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; V-NEXT:    vadd.vv v20, v16, v16
+; V-NEXT:    vmv.s.x v21, a0
+; V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; V-NEXT:    vcompress.vm v16, v8, v12
+; V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; V-NEXT:    vadd.vi v22, v20, -8
+; V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; V-NEXT:    vcompress.vm v12, v8, v21
+; V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; V-NEXT:    vadd.vi v8, v20, -7
+; V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; V-NEXT:    vrgatherei16.vv v16, v24, v22, v0.t
+; V-NEXT:    vrgatherei16.vv v12, v24, v8, v0.t
+; V-NEXT:    vmv.v.v v8, v16
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_v8i64_v16i64:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v16, v8, v12
+; ZIP-NEXT:    ri.vunzip2b.vv v20, v8, v12
+; ZIP-NEXT:    vmv.v.v v8, v16
+; ZIP-NEXT:    vmv.v.v v12, v20
+; ZIP-NEXT:    ret
   %retval = call {<8 x i64>, <8 x i64>} @llvm.vector.deinterleave2.v16i64(<16 x i64> %vec)
   ret {<8 x i64>, <8 x i64>} %retval
 }
@@ -498,11 +487,11 @@ define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double
 ;
 ; ZIP-LABEL: vector_deinterleave_v2f64_v4f64:
 ; ZIP:       # %bb.0:
-; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; ZIP-NEXT:    vslidedown.vi v10, v8, 2
 ; ZIP-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; ZIP-NEXT:    ri.vzipodd.vv v9, v8, v10
-; ZIP-NEXT:    vslideup.vi v8, v10, 1
+; ZIP-NEXT:    ri.vunzip2a.vv v10, v8, v9
+; ZIP-NEXT:    ri.vunzip2b.vv v12, v8, v9
+; ZIP-NEXT:    vmv.v.v v8, v10
+; ZIP-NEXT:    vmv.v.v v9, v12
 ; ZIP-NEXT:    ret
 %retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec)
 ret {<2 x double>, <2 x double>} %retval
@@ -541,31 +530,11 @@ define {<4 x double>, <4 x double>} @vector_deinterleave_v4f64_v8f64(<8 x double
 ;
 ; ZIP-LABEL: vector_deinterleave_v4f64_v8f64:
 ; ZIP:       # %bb.0:
-; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; ZIP-NEXT:    vmv.v.i v0, 8
-; ZIP-NEXT:    vsetivli zero, 4, e64, m4, ta, ma
-; ZIP-NEXT:    vslidedown.vi v16, v8, 4
-; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; ZIP-NEXT:    vslidedown.vi v12, v8, 2
-; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; ZIP-NEXT:    vmv.v.i v10, 12
-; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; ZIP-NEXT:    vslideup.vi v14, v16, 2
-; ZIP-NEXT:    vslideup.vi v14, v16, 1, v0.t
-; ZIP-NEXT:    ri.vzip2a.vv v18, v8, v12
-; ZIP-NEXT:    vmv1r.v v0, v10
-; ZIP-NEXT:    vmerge.vvm v12, v18, v14, v0
-; ZIP-NEXT:    vsetivli zero, 2, e64, m2, ta, ma
-; ZIP-NEXT:    vslidedown.vi v14, v16, 2
-; ZIP-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; ZIP-NEXT:    vmv.v.i v0, 2
-; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; ZIP-NEXT:    ri.vzip2a.vv v18, v16, v14
-; ZIP-NEXT:    vslidedown.vi v14, v8, 1
-; ZIP-NEXT:    vslidedown.vi v14, v8, 2, v0.t
-; ZIP-NEXT:    vmv1r.v v0, v10
-; ZIP-NEXT:    vmerge.vvm v10, v14, v18, v0
-; ZIP-NEXT:    vmv2r.v v8, v12
+; ZIP-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v12, v8, v10
+; ZIP-NEXT:    ri.vunzip2b.vv v16, v8, v10
+; ZIP-NEXT:    vmv.v.v v8, v12
+; ZIP-NEXT:    vmv.v.v v10, v16
 ; ZIP-NEXT:    ret
 %retval = call {<4 x double>, <4 x double>} @llvm.vector.deinterleave2.v8f64(<8 x double> %vec)
 ret {<4 x double>, <4 x double>} %retval
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
index 77723609a60c..e297e88c71f1 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
@@ -260,18 +260,18 @@ define <vscale x 128 x i1> @vector_interleave_nxv128i1_nxv64i1(<vscale x 64 x i1
 ; ZIP-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
 ; ZIP-NEXT:    vmv1r.v v9, v0
 ; ZIP-NEXT:    vmv1r.v v0, v8
-; ZIP-NEXT:    vmv.v.i v16, 0
-; ZIP-NEXT:    vmerge.vim v24, v16, 1, v0
+; ZIP-NEXT:    vmv.v.i v24, 0
+; ZIP-NEXT:    vmerge.vim v16, v24, 1, v0
 ; ZIP-NEXT:    vmv1r.v v0, v9
-; ZIP-NEXT:    vmerge.vim v8, v16, 1, v0
+; ZIP-NEXT:    vmerge.vim v8, v24, 1, v0
 ; ZIP-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
-; ZIP-NEXT:    ri.vzip2b.vv v4, v8, v24
-; ZIP-NEXT:    ri.vzip2b.vv v20, v12, v28
-; ZIP-NEXT:    ri.vzip2a.vv v0, v8, v24
-; ZIP-NEXT:    ri.vzip2a.vv v16, v12, v28
+; ZIP-NEXT:    ri.vzip2b.vv v4, v8, v16
+; ZIP-NEXT:    ri.vzip2b.vv v28, v12, v20
+; ZIP-NEXT:    ri.vzip2a.vv v0, v8, v16
+; ZIP-NEXT:    ri.vzip2a.vv v24, v12, v20
 ; ZIP-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
 ; ZIP-NEXT:    vmsne.vi v9, v0, 0
-; ZIP-NEXT:    vmsne.vi v8, v16, 0
+; ZIP-NEXT:    vmsne.vi v8, v24, 0
 ; ZIP-NEXT:    vmv1r.v v0, v9
 ; ZIP-NEXT:    ret
   %res = call <vscale x 128 x i1> @llvm.vector.interleave2.nxv128i1(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll
index e4235d03cda3..db31866b5637 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll
@@ -14,7 +14,7 @@ define i64 @test_vleff_nxv8i8(ptr %p, i64 %vl) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[PseudoVLE8FF_V_M1_:%[0-9]+]]:vr, [[PseudoVLE8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLE8FF_V_M1 $noreg, [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */, implicit-def dead $vl :: (load unknown-size from %ir.p, align 1)
+  ; CHECK-NEXT:   [[PseudoVLE8FF_V_M1_:%[0-9]+]]:vr, [[PseudoVLE8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLE8FF_V_M1 $noreg, [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 1)
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVLE8FF_V_M1_1]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
 entry:
@@ -31,7 +31,7 @@ define i64 @test_vleff_nxv8i8_tu(<vscale x 8 x i8> %passthru, ptr %p, i64 %vl) {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr = COPY $v8
-  ; CHECK-NEXT:   [[PseudoVLE8FF_V_M1_:%[0-9]+]]:vr, [[PseudoVLE8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLE8FF_V_M1 [[COPY2]], [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */, implicit-def dead $vl :: (load unknown-size from %ir.p, align 1)
+  ; CHECK-NEXT:   [[PseudoVLE8FF_V_M1_:%[0-9]+]]:vr, [[PseudoVLE8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLE8FF_V_M1 [[COPY2]], [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 1)
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVLE8FF_V_M1_1]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
 entry:
@@ -50,7 +50,7 @@ define i64 @test_vleff_nxv8i8_mask(<vscale x 8 x i8> %maskedoff, ptr %p, <vscale
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vmv0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLE8FF_V_M1_MASK:%[0-9]+]]:vrnov0, [[PseudoVLE8FF_V_M1_MASK1:%[0-9]+]]:gpr = PseudoVLE8FF_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 3 /* e8 */, 0 /* tu, mu */, implicit-def dead $vl :: (load unknown-size from %ir.p, align 1)
+  ; CHECK-NEXT:   [[PseudoVLE8FF_V_M1_MASK:%[0-9]+]]:vrnov0, [[PseudoVLE8FF_V_M1_MASK1:%[0-9]+]]:gpr = PseudoVLE8FF_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 3 /* e8 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 1)
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVLE8FF_V_M1_MASK1]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
 entry:
@@ -66,7 +66,7 @@ define i64 @test_vlseg2ff_nxv8i8(ptr %base, i64 %vl, ptr %outvl) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_:%[0-9]+]]:vrn2m1, [[PseudoVLSEG2E8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1 $noreg, [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */, implicit-def dead $vl :: (load unknown-size from %ir.base, align 1)
+  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_:%[0-9]+]]:vrn2m1, [[PseudoVLSEG2E8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1 $noreg, [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */ :: (load unknown-size from %ir.base, align 1)
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVLSEG2E8FF_V_M1_1]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
 entry:
@@ -83,7 +83,7 @@ define i64 @test_vlseg2ff_nxv8i8_tu(target("riscv.vector.tuple", <vscale x 8 x i
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY $v8_v9
-  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_:%[0-9]+]]:vrn2m1, [[PseudoVLSEG2E8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1 [[COPY2]], [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */, implicit-def dead $vl :: (load unknown-size from %ir.base, align 1)
+  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_:%[0-9]+]]:vrn2m1, [[PseudoVLSEG2E8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1 [[COPY2]], [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */ :: (load unknown-size from %ir.base, align 1)
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVLSEG2E8FF_V_M1_1]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
 entry:
@@ -102,7 +102,7 @@ define i64 @test_vlseg2ff_nxv8i8_mask(target("riscv.vector.tuple", <vscale x 8 x
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrn2m1nov0 = COPY $v8_v9
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vmv0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_MASK:%[0-9]+]]:vrn2m1nov0, [[PseudoVLSEG2E8FF_V_M1_MASK1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 3 /* e8 */, 0 /* tu, mu */, implicit-def dead $vl :: (load unknown-size from %ir.base, align 1)
+  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_MASK:%[0-9]+]]:vrn2m1nov0, [[PseudoVLSEG2E8FF_V_M1_MASK1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 3 /* e8 */, 0 /* tu, mu */ :: (load unknown-size from %ir.base, align 1)
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVLSEG2E8FF_V_M1_MASK1]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
 entry:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff.ll
index 1f3959c1eac8..4c989ce87290 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vleff.ll
@@ -3016,12 +3016,9 @@ entry:
   ret void
 }
 
-; Test with both outputs dead. Make sure the vleff isn't deleted.
 define void @intrinsic_vleff_dead_all(ptr %0, iXLen %1, ptr %2) nounwind {
 ; CHECK-LABEL: intrinsic_vleff_dead_all:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
-; CHECK-NEXT:    vle64ff.v v8, (a0)
 ; CHECK-NEXT:    ret
 entry:
   %a = call { <vscale x 1 x double>, iXLen } @llvm.riscv.vleff.nxv1f64(
@@ -3034,8 +3031,6 @@ entry:
 define void @intrinsic_vleff_mask_dead_all(<vscale x 1 x double> %0, ptr %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vleff_mask_dead_all:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
-; CHECK-NEXT:    vle64ff.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
 entry:
   %a = call { <vscale x 1 x double>, iXLen } @llvm.riscv.vleff.mask.nxv1f64(
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll
index d8bff08ea551..333ba83f69ee 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv32-dead.ll
@@ -66,8 +66,6 @@ entry:
 define void @test_vlseg2ff_dead_all(ptr %base, i32 %vl) {
 ; CHECK-LABEL: test_vlseg2ff_dead_all:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
-; CHECK-NEXT:    vlseg2e16ff.v v8, (a0)
 ; CHECK-NEXT:    ret
 entry:
   tail call {target("riscv.vector.tuple", <vscale x 32 x i8>, 2), i32} @llvm.riscv.vlseg2ff.triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) undef, ptr %base, i32 %vl, i32 4)
@@ -77,8 +75,6 @@ entry:
 define void @test_vlseg2ff_mask_dead_all(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i32 %vl, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: test_vlseg2ff_mask_dead_all:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
-; CHECK-NEXT:    vlseg2e16ff.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
 entry:
   tail call {target("riscv.vector.tuple", <vscale x 32 x i8>, 2), i32} @llvm.riscv.vlseg2ff.mask.triscv.vector.tuple_nxv32i8_2t.nxv16i1(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, <vscale x 16 x i1> %mask, i32 %vl, i32 1, i32 4)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll
index 05a5be295cc7..b9e45cc190a6 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vlsegff-rv64-dead.ll
@@ -66,8 +66,6 @@ entry:
 define void @test_vlseg2ff_dead_all(ptr %base, i64 %vl) {
 ; CHECK-LABEL: test_vlseg2ff_dead_all:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
-; CHECK-NEXT:    vlseg2e16ff.v v8, (a0)
 ; CHECK-NEXT:    ret
 entry:
   tail call {target("riscv.vector.tuple", <vscale x 32 x i8>, 2), i64} @llvm.riscv.vlseg2ff.triscv.vector.tuple_nxv32i8_2t(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) undef, ptr %base, i64 %vl, i64 4)
@@ -77,8 +75,6 @@ entry:
 define void @test_vlseg2ff_mask_dead_all(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, i64 %vl, <vscale x 16 x i1> %mask) {
 ; CHECK-LABEL: test_vlseg2ff_mask_dead_all:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
-; CHECK-NEXT:    vlseg2e16ff.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
 entry:
   tail call {target("riscv.vector.tuple", <vscale x 32 x i8>, 2), i64} @llvm.riscv.vlseg2ff.mask.triscv.vector.tuple_nxv32i8_2t.nxv16i1(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %val, ptr %base, <vscale x 16 x i1> %mask, i64 %vl, i64 1, i64 4)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll
index 6345b90db23b..1e2e7795f654 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll
@@ -206,3 +206,19 @@ define <vscale x 1 x i64> @undef_passthru(<vscale x 1 x i64> %passthru, <vscale
   %b = call <vscale x 1 x i64> @llvm.riscv.vmv.v.v.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i64> %a, iXLen %avl)
   ret <vscale x 1 x i64> %b
 }
+
+; Check that we can fold into vle64ff.v even if we need to move it past the
+; passthru and it's safe.
+define <vscale x 1 x i64> @vleff_move_past_passthru(ptr %p, ptr %q, iXLen %avl) {
+; CHECK-LABEL: vleff_move_past_passthru:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl1re64.v v8, (a1)
+; CHECK-NEXT:    vsetvli zero, a2, e64, m1, tu, ma
+; CHECK-NEXT:    vle64ff.v v8, (a0)
+; CHECK-NEXT:    ret
+  %a = call { <vscale x 1 x i64>, iXLen } @llvm.riscv.vleff(<vscale x 1 x i64> poison, ptr %p, iXLen %avl)
+  %vec = extractvalue { <vscale x 1 x i64>, iXLen } %a, 0
+  %passthru = load <vscale x 1 x i64>, ptr %q
+  %b = call <vscale x 1 x i64> @llvm.riscv.vmv.v.v.nxv1i64(<vscale x 1 x i64> %passthru, <vscale x 1 x i64> %vec, iXLen %avl)
+  ret <vscale x 1 x i64> %b
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir
index f545ecc5e53d..6e106e50634f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir
@@ -135,3 +135,20 @@ body: |
     %3:vrnov0 = PseudoVMV_V_V_MF2 $noreg, %2, 0, 5 /* e32 */, 0 /* tu, mu */
     %7:vmv0 = COPY $v8
     %6:vrnov0 = PseudoVLSE32_V_MF2_MASK %3, $noreg, $noreg, %7, 0, 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size, align 4)
+...
+---
+name: move_vleff
+body: |
+  bb.0:
+    liveins: $v8
+    ; CHECK-LABEL: name: move_vleff
+    ; CHECK: liveins: $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %passthru:vr = COPY $v8
+    ; CHECK-NEXT: %x:vr, %vl:gpr = PseudoVLE32FF_V_M1 %passthru, $noreg, 4, 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size, align 1)
+    ; CHECK-NEXT: %y:gpr = ADDI $x0, 1
+    %x:vr, %vl:gpr = PseudoVLE32FF_V_M1 $noreg, $noreg, 4, 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size)
+    %passthru:vr = COPY $v8
+    %y:gpr = ADDI $x0, 1
+    %z:vr = PseudoVMV_V_V_M1 %passthru, %x, 4, 5 /* e32 */, 0 /* tu, mu */
+...
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
index f386fd9cd3ae..75c00e406b4f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vrgatherei16.vv.nxv1i8(
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvl-cross-inline-asm.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvl-cross-inline-asm.ll
index f2b566e83616..e5ba9a0a9bc5 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvl-cross-inline-asm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvl-cross-inline-asm.ll
@@ -7,7 +7,7 @@ define void @foo(<vscale x 8 x half> %0) {
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    lui a0, 1
-; CHECK-NEXT:    addiw a0, a0, -1096
+; CHECK-NEXT:    addi a0, a0, -1096
 ; CHECK-NEXT:    vmv.v.i v11, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    #APP
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
index 266f35f6dd62..6bd03eb0c222 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
@@ -96,6 +96,10 @@
     ret void
   }
 
+  define void @coalesce_vl_avl_same_reg() {
+    ret void
+  }
+
   declare <vscale x 1 x i64> @llvm.riscv.vadd.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, i64) #1
 
   declare <vscale x 1 x i64> @llvm.riscv.vle.nxv1i64.i64(<vscale x 1 x i64>, ptr nocapture, i64) #4
@@ -408,7 +412,7 @@ body:             |
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10
     ; CHECK-NEXT: dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: [[PseudoVLE64_V_M1_:%[0-9]+]]:vr = PseudoVLE64_V_M1 undef $noreg, [[COPY2]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-    ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */
+    ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 undef $noreg, [[PseudoVLE64_V_M1_]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: $v8 = COPY [[PseudoVADD_VV_M1_]], implicit $vtype
@@ -601,3 +605,20 @@ body: |
     %x:gpr = COPY $x10
     renamable $v8 = PseudoVMV_S_X undef renamable $v8, killed renamable %x, 1, 5
     PseudoRET implicit $v8
+...
+---
+name: coalesce_vl_avl_same_reg
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x8, $v8
+    ; CHECK-LABEL: name: coalesce_vl_avl_same_reg
+    ; CHECK: liveins: $x8, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %x:gprnox0 = COPY $x8
+    ; CHECK-NEXT: dead %x:gprnox0 = PseudoVSETVLI %x, 208 /* e32, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
+    ; CHECK-NEXT: dead %v:vr = COPY $v8, implicit $vtype
+    %x:gprnox0 = COPY $x8
+    dead $x0 = PseudoVSETIVLI 1, 208, implicit-def $vl, implicit-def $vtype
+    %v:vr = COPY $v8, implicit $vtype
+    %x = PseudoVSETVLI %x, 208, implicit-def $vl, implicit-def $vtype
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
index 2be187c50af2..f0f78c211c4a 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslidedown.nxv1i8(
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslideup.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
index 1e3ede7fee9c..8e3c05611bc7 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfh \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfh \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslideup.nxv1i8(
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll
index 3a747449d0a6..1e1bf5e5d508 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll
@@ -115,21 +115,13 @@ define <vscale x 8 x i64> @vadd_vx_nxv8i64_8(<vscale x 8 x i64> %v) {
 }
 
 define <vscale x 8 x i64> @vadd_vx_nxv8i64_9(<vscale x 8 x i64> %v) {
-; RV32V-LABEL: vadd_vx_nxv8i64_9:
-; RV32V:       # %bb.0:
-; RV32V-NEXT:    lui a0, 503808
-; RV32V-NEXT:    addi a0, a0, -1281
-; RV32V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32V-NEXT:    vadd.vx v8, v8, a0
-; RV32V-NEXT:    ret
-;
-; RV64V-LABEL: vadd_vx_nxv8i64_9:
-; RV64V:       # %bb.0:
-; RV64V-NEXT:    lui a0, 503808
-; RV64V-NEXT:    addiw a0, a0, -1281
-; RV64V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV64V-NEXT:    vadd.vx v8, v8, a0
-; RV64V-NEXT:    ret
+; CHECK-LABEL: vadd_vx_nxv8i64_9:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a0, 503808
+; CHECK-NEXT:    addi a0, a0, -1281
+; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vadd.vx v8, v8, a0
+; CHECK-NEXT:    ret
   %vret = add <vscale x 8 x i64> %v, splat (i64 2063596287)
   ret <vscale x 8 x i64> %vret
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/zvqdotq-sdnode.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/zvqdotq-sdnode.ll
index d0fc915a0d07..87a984bda1fe 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/zvqdotq-sdnode.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/rvv/zvqdotq-sdnode.ll
@@ -230,29 +230,17 @@ define i32 @reduce_of_sext(<vscale x 16 x i8> %a) {
 ; NODOT-NEXT:    vmv.x.s a0, v8
 ; NODOT-NEXT:    ret
 ;
-; DOT32-LABEL: reduce_of_sext:
-; DOT32:       # %bb.0: # %entry
-; DOT32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; DOT32-NEXT:    vmv.v.i v10, 0
-; DOT32-NEXT:    lui a0, 4112
-; DOT32-NEXT:    addi a0, a0, 257
-; DOT32-NEXT:    vqdot.vx v10, v8, a0
-; DOT32-NEXT:    vmv.s.x v8, zero
-; DOT32-NEXT:    vredsum.vs v8, v10, v8
-; DOT32-NEXT:    vmv.x.s a0, v8
-; DOT32-NEXT:    ret
-;
-; DOT64-LABEL: reduce_of_sext:
-; DOT64:       # %bb.0: # %entry
-; DOT64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; DOT64-NEXT:    vmv.v.i v10, 0
-; DOT64-NEXT:    lui a0, 4112
-; DOT64-NEXT:    addiw a0, a0, 257
-; DOT64-NEXT:    vqdot.vx v10, v8, a0
-; DOT64-NEXT:    vmv.s.x v8, zero
-; DOT64-NEXT:    vredsum.vs v8, v10, v8
-; DOT64-NEXT:    vmv.x.s a0, v8
-; DOT64-NEXT:    ret
+; DOT-LABEL: reduce_of_sext:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; DOT-NEXT:    vmv.v.i v10, 1
+; DOT-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; DOT-NEXT:    vmv.v.i v12, 0
+; DOT-NEXT:    vqdot.vv v12, v8, v10
+; DOT-NEXT:    vmv.s.x v8, zero
+; DOT-NEXT:    vredsum.vs v8, v12, v8
+; DOT-NEXT:    vmv.x.s a0, v8
+; DOT-NEXT:    ret
 entry:
   %a.ext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>
   %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %a.ext)
@@ -269,29 +257,17 @@ define i32 @reduce_of_zext(<vscale x 16 x i8> %a) {
 ; NODOT-NEXT:    vmv.x.s a0, v8
 ; NODOT-NEXT:    ret
 ;
-; DOT32-LABEL: reduce_of_zext:
-; DOT32:       # %bb.0: # %entry
-; DOT32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; DOT32-NEXT:    vmv.v.i v10, 0
-; DOT32-NEXT:    lui a0, 4112
-; DOT32-NEXT:    addi a0, a0, 257
-; DOT32-NEXT:    vqdotu.vx v10, v8, a0
-; DOT32-NEXT:    vmv.s.x v8, zero
-; DOT32-NEXT:    vredsum.vs v8, v10, v8
-; DOT32-NEXT:    vmv.x.s a0, v8
-; DOT32-NEXT:    ret
-;
-; DOT64-LABEL: reduce_of_zext:
-; DOT64:       # %bb.0: # %entry
-; DOT64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; DOT64-NEXT:    vmv.v.i v10, 0
-; DOT64-NEXT:    lui a0, 4112
-; DOT64-NEXT:    addiw a0, a0, 257
-; DOT64-NEXT:    vqdotu.vx v10, v8, a0
-; DOT64-NEXT:    vmv.s.x v8, zero
-; DOT64-NEXT:    vredsum.vs v8, v10, v8
-; DOT64-NEXT:    vmv.x.s a0, v8
-; DOT64-NEXT:    ret
+; DOT-LABEL: reduce_of_zext:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; DOT-NEXT:    vmv.v.i v10, 1
+; DOT-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; DOT-NEXT:    vmv.v.i v12, 0
+; DOT-NEXT:    vqdotu.vv v12, v8, v10
+; DOT-NEXT:    vmv.s.x v8, zero
+; DOT-NEXT:    vredsum.vs v8, v12, v8
+; DOT-NEXT:    vmv.x.s a0, v8
+; DOT-NEXT:    ret
 entry:
   %a.ext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>
   %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %a.ext)
@@ -934,22 +910,30 @@ entry:
 }
 
 define <vscale x 1 x i32> @partial_reduce_vqdotsu(<vscale x 4 x i8> %a, <vscale x 4 x i8> %b) {
-; CHECK-LABEL: partial_reduce_vqdotsu:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vsext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vwmulsu.vv v8, v10, v11
-; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v10, v9, a0
-; CHECK-NEXT:    vslidedown.vx v11, v8, a0
-; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v10, v8
-; CHECK-NEXT:    vadd.vv v9, v11, v9
-; CHECK-NEXT:    vadd.vv v8, v9, v8
-; CHECK-NEXT:    ret
+; NODOT-LABEL: partial_reduce_vqdotsu:
+; NODOT:       # %bb.0: # %entry
+; NODOT-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; NODOT-NEXT:    vsext.vf2 v10, v8
+; NODOT-NEXT:    vzext.vf2 v11, v9
+; NODOT-NEXT:    csrr a0, vlenb
+; NODOT-NEXT:    vwmulsu.vv v8, v10, v11
+; NODOT-NEXT:    srli a0, a0, 3
+; NODOT-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; NODOT-NEXT:    vslidedown.vx v10, v9, a0
+; NODOT-NEXT:    vslidedown.vx v11, v8, a0
+; NODOT-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; NODOT-NEXT:    vadd.vv v8, v10, v8
+; NODOT-NEXT:    vadd.vv v9, v11, v9
+; NODOT-NEXT:    vadd.vv v8, v9, v8
+; NODOT-NEXT:    ret
+;
+; DOT-LABEL: partial_reduce_vqdotsu:
+; DOT:       # %bb.0: # %entry
+; DOT-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; DOT-NEXT:    vmv.v.i v10, 0
+; DOT-NEXT:    vqdotsu.vv v10, v8, v9
+; DOT-NEXT:    vmv1r.v v8, v10
+; DOT-NEXT:    ret
 entry:
   %a.sext = sext <vscale x 4 x i8> %a to <vscale x 4 x i32>
   %b.sext = zext <vscale x 4 x i8> %b to <vscale x 4 x i32>
@@ -1010,3 +994,6 @@ entry:
   %res = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i32> %a.ext)
   ret <vscale x 4 x i32> %res
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; DOT32: {{.*}}
+; DOT64: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat.ll
index ab03ccc4ba59..04f2436201e9 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat.ll
@@ -133,7 +133,7 @@ define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    lui a1, 8
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    bge a0, a1, .LBB2_3
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    lui a1, 1048568
@@ -162,7 +162,7 @@ define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
 ; RV64IZbb:       # %bb.0:
 ; RV64IZbb-NEXT:    add a0, a0, a1
 ; RV64IZbb-NEXT:    lui a1, 8
-; RV64IZbb-NEXT:    addiw a1, a1, -1
+; RV64IZbb-NEXT:    addi a1, a1, -1
 ; RV64IZbb-NEXT:    min a0, a0, a1
 ; RV64IZbb-NEXT:    lui a1, 1048568
 ; RV64IZbb-NEXT:    max a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
index abcf3379d0a6..857026cce0d4 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
@@ -148,7 +148,7 @@ define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
 ; RV64I-NEXT:    slli a1, a1, 48
 ; RV64I-NEXT:    srai a1, a1, 48
 ; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, -1
+; RV64I-NEXT:    addi a1, a2, -1
 ; RV64I-NEXT:    bge a0, a1, .LBB2_3
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    lui a1, 1048568
@@ -182,7 +182,7 @@ define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
 ; RV64IZbb-NEXT:    mul a1, a1, a2
 ; RV64IZbb-NEXT:    lui a2, 8
 ; RV64IZbb-NEXT:    sext.h a1, a1
-; RV64IZbb-NEXT:    addiw a2, a2, -1
+; RV64IZbb-NEXT:    addi a2, a2, -1
 ; RV64IZbb-NEXT:    add a0, a0, a1
 ; RV64IZbb-NEXT:    min a0, a0, a2
 ; RV64IZbb-NEXT:    lui a1, 1048568
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/scmp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/scmp.ll
index a212714db53e..56c876a2409d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/scmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/scmp.ll
@@ -89,8 +89,8 @@ define i8 @scmp.8.128(i128 %x, i128 %y) nounwind {
 ; RV32I-NEXT:    lw a2, 4(a1)
 ; RV32I-NEXT:    lw a4, 8(a1)
 ; RV32I-NEXT:    lw a5, 12(a1)
-; RV32I-NEXT:    lw a6, 12(a0)
 ; RV32I-NEXT:    lw a3, 4(a0)
+; RV32I-NEXT:    lw a6, 12(a0)
 ; RV32I-NEXT:    lw a7, 8(a0)
 ; RV32I-NEXT:    beq a6, a5, .LBB4_2
 ; RV32I-NEXT:  # %bb.1:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/select-cc.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/select-cc.ll
index e69dc303d85d..ec1f8aeddcaa 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/select-cc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/select-cc.ll
@@ -358,7 +358,7 @@ define i32 @select_sge_int16min(i32 signext %x, i32 signext %y, i32 signext %z)
 ; RV64I-LABEL: select_sge_int16min:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a3, 1048560
-; RV64I-NEXT:    addiw a3, a3, -1
+; RV64I-NEXT:    addi a3, a3, -1
 ; RV64I-NEXT:    blt a3, a0, .LBB2_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a1, a2
@@ -369,7 +369,7 @@ define i32 @select_sge_int16min(i32 signext %x, i32 signext %y, i32 signext %z)
 ; RV64I-CCMOV-LABEL: select_sge_int16min:
 ; RV64I-CCMOV:       # %bb.0:
 ; RV64I-CCMOV-NEXT:    lui a3, 1048560
-; RV64I-CCMOV-NEXT:    addiw a3, a3, -1
+; RV64I-CCMOV-NEXT:    addi a3, a3, -1
 ; RV64I-CCMOV-NEXT:    slt a0, a3, a0
 ; RV64I-CCMOV-NEXT:    mips.ccmov a0, a0, a1, a2
 ; RV64I-CCMOV-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/select-cond.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/select-cond.ll
new file mode 100644
index 000000000000..a5f4677f73f1
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/select-cond.ll
@@ -0,0 +1,1018 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV32
+; RUN: llc -mtriple=riscv32 -mattr=+xtheadcondmov -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV32-THEAD
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV64
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscmov -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV64-MIPS
+
+define signext i32 @select_i32_trunc(i32 signext %cond, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_trunc:
+; RV32:       # %bb.0:
+; RV32-NEXT:    andi a3, a0, 1
+; RV32-NEXT:    mv a0, a1
+; RV32-NEXT:    bnez a3, .LBB0_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:  .LBB0_2:
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_trunc:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    andi a0, a0, 1
+; RV32-THEAD-NEXT:    th.mveqz a1, a2, a0
+; RV32-THEAD-NEXT:    mv a0, a1
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_trunc:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a3, a0, 1
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:    bnez a3, .LBB0_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:  .LBB0_2:
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_trunc:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    andi a0, a0, 1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64-MIPS-NEXT:    ret
+  %cond_trunc = trunc i32 %cond to i1
+  %res = select i1 %cond_trunc, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_param(i1 signext %cond, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_param:
+; RV32:       # %bb.0:
+; RV32-NEXT:    andi a3, a0, 1
+; RV32-NEXT:    mv a0, a1
+; RV32-NEXT:    bnez a3, .LBB1_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:  .LBB1_2:
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_param:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    andi a0, a0, 1
+; RV32-THEAD-NEXT:    th.mveqz a1, a2, a0
+; RV32-THEAD-NEXT:    mv a0, a1
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_param:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a3, a0, 1
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:    bnez a3, .LBB1_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:  .LBB1_2:
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_param:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    andi a0, a0, 1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64-MIPS-NEXT:    ret
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_eq(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_eq:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a0, a1, .LBB2_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB2_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_eq:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor a0, a0, a1
+; RV32-THEAD-NEXT:    th.mvnez a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_eq:
+; RV64:       # %bb.0:
+; RV64-NEXT:    beq a0, a1, .LBB2_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB2_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_eq:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    xor a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp eq i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_ne(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_ne:
+; RV32:       # %bb.0:
+; RV32-NEXT:    bne a0, a1, .LBB3_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB3_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_ne:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor a0, a0, a1
+; RV32-THEAD-NEXT:    th.mveqz a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_ne:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bne a0, a1, .LBB3_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB3_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_ne:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    xor a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ne i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_ugt(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_ugt:
+; RV32:       # %bb.0:
+; RV32-NEXT:    bltu a1, a0, .LBB4_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB4_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_ugt:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    sltu a0, a1, a0
+; RV32-THEAD-NEXT:    th.mveqz a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_ugt:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bltu a1, a0, .LBB4_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB4_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_ugt:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ugt i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_uge(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_uge:
+; RV32:       # %bb.0:
+; RV32-NEXT:    bgeu a0, a1, .LBB5_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB5_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_uge:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    sltu a0, a0, a1
+; RV32-THEAD-NEXT:    th.mvnez a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_uge:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bgeu a0, a1, .LBB5_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB5_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_uge:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp uge i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_ult(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_ult:
+; RV32:       # %bb.0:
+; RV32-NEXT:    bltu a0, a1, .LBB6_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB6_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_ult:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    sltu a0, a0, a1
+; RV32-THEAD-NEXT:    th.mveqz a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_ult:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bltu a0, a1, .LBB6_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB6_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_ult:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ult i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_ule(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_ule:
+; RV32:       # %bb.0:
+; RV32-NEXT:    bgeu a1, a0, .LBB7_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB7_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_ule:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    sltu a0, a1, a0
+; RV32-THEAD-NEXT:    th.mvnez a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_ule:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bgeu a1, a0, .LBB7_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB7_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_ule:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ule i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_sgt(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_sgt:
+; RV32:       # %bb.0:
+; RV32-NEXT:    blt a1, a0, .LBB8_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB8_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_sgt:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    slt a0, a1, a0
+; RV32-THEAD-NEXT:    th.mveqz a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_sgt:
+; RV64:       # %bb.0:
+; RV64-NEXT:    blt a1, a0, .LBB8_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB8_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_sgt:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp sgt i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_sge(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_sge:
+; RV32:       # %bb.0:
+; RV32-NEXT:    bge a0, a1, .LBB9_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB9_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_sge:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    slt a0, a0, a1
+; RV32-THEAD-NEXT:    th.mvnez a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_sge:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bge a0, a1, .LBB9_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB9_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_sge:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp sge i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_slt(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_slt:
+; RV32:       # %bb.0:
+; RV32-NEXT:    blt a0, a1, .LBB10_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB10_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_slt:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    slt a0, a0, a1
+; RV32-THEAD-NEXT:    th.mveqz a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_slt:
+; RV64:       # %bb.0:
+; RV64-NEXT:    blt a0, a1, .LBB10_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB10_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_slt:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp slt i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define signext i32 @select_i32_sle(i32 signext %a, i32 signext %b, i32 signext %x, i32 signext %y) nounwind {
+; RV32-LABEL: select_i32_sle:
+; RV32:       # %bb.0:
+; RV32-NEXT:    bge a1, a0, .LBB11_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a2, a3
+; RV32-NEXT:  .LBB11_2:
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i32_sle:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    slt a0, a1, a0
+; RV32-THEAD-NEXT:    th.mvnez a2, a3, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i32_sle:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bge a1, a0, .LBB11_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB11_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i32_sle:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp sle i32 %a, %b
+  %res = select i1 %cond, i32 %x, i32 %y
+  ret i32 %res
+}
+
+define i64 @select_i64_trunc(i64 %cond, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_trunc:
+; RV32:       # %bb.0:
+; RV32-NEXT:    mv a1, a3
+; RV32-NEXT:    andi a3, a0, 1
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    bnez a3, .LBB12_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:  .LBB12_2:
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_trunc:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    mv a1, a3
+; RV32-THEAD-NEXT:    andi a0, a0, 1
+; RV32-THEAD-NEXT:    th.mveqz a2, a4, a0
+; RV32-THEAD-NEXT:    th.mveqz a1, a5, a0
+; RV32-THEAD-NEXT:    mv a0, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_trunc:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a3, a0, 1
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:    bnez a3, .LBB12_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:  .LBB12_2:
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_trunc:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    andi a0, a0, 1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64-MIPS-NEXT:    ret
+  %cond_trunc = trunc i64 %cond to i1
+  %res = select i1 %cond_trunc, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_param(i1 %cond, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_param:
+; RV32:       # %bb.0:
+; RV32-NEXT:    andi a5, a0, 1
+; RV32-NEXT:    mv a0, a1
+; RV32-NEXT:    bnez a5, .LBB13_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a0, a3
+; RV32-NEXT:    mv a2, a4
+; RV32-NEXT:  .LBB13_2:
+; RV32-NEXT:    mv a1, a2
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_param:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    andi a0, a0, 1
+; RV32-THEAD-NEXT:    th.mveqz a1, a3, a0
+; RV32-THEAD-NEXT:    th.mveqz a2, a4, a0
+; RV32-THEAD-NEXT:    mv a0, a1
+; RV32-THEAD-NEXT:    mv a1, a2
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_param:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a3, a0, 1
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:    bnez a3, .LBB13_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:  .LBB13_2:
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_param:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    andi a0, a0, 1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a1, a2
+; RV64-MIPS-NEXT:    ret
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_eq(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_eq:
+; RV32:       # %bb.0:
+; RV32-NEXT:    xor a1, a1, a3
+; RV32-NEXT:    xor a0, a0, a2
+; RV32-NEXT:    or a1, a0, a1
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    beqz a1, .LBB14_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a0, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB14_2:
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_eq:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor a1, a1, a3
+; RV32-THEAD-NEXT:    xor a0, a0, a2
+; RV32-THEAD-NEXT:    or a0, a0, a1
+; RV32-THEAD-NEXT:    th.mvnez a4, a6, a0
+; RV32-THEAD-NEXT:    th.mvnez a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_eq:
+; RV64:       # %bb.0:
+; RV64-NEXT:    beq a0, a1, .LBB14_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB14_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_eq:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    xor a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp eq i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_ne(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_ne:
+; RV32:       # %bb.0:
+; RV32-NEXT:    xor a1, a1, a3
+; RV32-NEXT:    xor a0, a0, a2
+; RV32-NEXT:    or a1, a0, a1
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    bnez a1, .LBB15_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    mv a0, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB15_2:
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_ne:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor a1, a1, a3
+; RV32-THEAD-NEXT:    xor a0, a0, a2
+; RV32-THEAD-NEXT:    or a0, a0, a1
+; RV32-THEAD-NEXT:    th.mveqz a4, a6, a0
+; RV32-THEAD-NEXT:    th.mveqz a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_ne:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bne a0, a1, .LBB15_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB15_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_ne:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    xor a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ne i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_ugt(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_ugt:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB16_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    sltu a0, a3, a1
+; RV32-NEXT:    beqz a0, .LBB16_3
+; RV32-NEXT:    j .LBB16_4
+; RV32-NEXT:  .LBB16_2:
+; RV32-NEXT:    sltu a0, a2, a0
+; RV32-NEXT:    bnez a0, .LBB16_4
+; RV32-NEXT:  .LBB16_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB16_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_ugt:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    sltu a1, a3, a1
+; RV32-THEAD-NEXT:    sltu a0, a2, a0
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mveqz a4, a6, a0
+; RV32-THEAD-NEXT:    th.mveqz a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_ugt:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bltu a1, a0, .LBB16_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB16_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_ugt:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ugt i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_uge(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_uge:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB17_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    sltu a0, a1, a3
+; RV32-NEXT:    bnez a0, .LBB17_3
+; RV32-NEXT:    j .LBB17_4
+; RV32-NEXT:  .LBB17_2:
+; RV32-NEXT:    sltu a0, a0, a2
+; RV32-NEXT:    beqz a0, .LBB17_4
+; RV32-NEXT:  .LBB17_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB17_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_uge:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    sltu a1, a1, a3
+; RV32-THEAD-NEXT:    sltu a0, a0, a2
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mvnez a4, a6, a0
+; RV32-THEAD-NEXT:    th.mvnez a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_uge:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bgeu a0, a1, .LBB17_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB17_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_uge:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp uge i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_ult(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_ult:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB18_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    sltu a0, a1, a3
+; RV32-NEXT:    beqz a0, .LBB18_3
+; RV32-NEXT:    j .LBB18_4
+; RV32-NEXT:  .LBB18_2:
+; RV32-NEXT:    sltu a0, a0, a2
+; RV32-NEXT:    bnez a0, .LBB18_4
+; RV32-NEXT:  .LBB18_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB18_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_ult:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    sltu a1, a1, a3
+; RV32-THEAD-NEXT:    sltu a0, a0, a2
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mveqz a4, a6, a0
+; RV32-THEAD-NEXT:    th.mveqz a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_ult:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bltu a0, a1, .LBB18_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB18_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_ult:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ult i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_ule(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_ule:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB19_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    sltu a0, a3, a1
+; RV32-NEXT:    bnez a0, .LBB19_3
+; RV32-NEXT:    j .LBB19_4
+; RV32-NEXT:  .LBB19_2:
+; RV32-NEXT:    sltu a0, a2, a0
+; RV32-NEXT:    beqz a0, .LBB19_4
+; RV32-NEXT:  .LBB19_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB19_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_ule:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    sltu a1, a3, a1
+; RV32-THEAD-NEXT:    sltu a0, a2, a0
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mvnez a4, a6, a0
+; RV32-THEAD-NEXT:    th.mvnez a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_ule:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bgeu a1, a0, .LBB19_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB19_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_ule:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    sltu a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp ule i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_sgt(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_sgt:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB20_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    slt a0, a3, a1
+; RV32-NEXT:    beqz a0, .LBB20_3
+; RV32-NEXT:    j .LBB20_4
+; RV32-NEXT:  .LBB20_2:
+; RV32-NEXT:    sltu a0, a2, a0
+; RV32-NEXT:    bnez a0, .LBB20_4
+; RV32-NEXT:  .LBB20_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB20_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_sgt:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    slt a1, a3, a1
+; RV32-THEAD-NEXT:    sltu a0, a2, a0
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mveqz a4, a6, a0
+; RV32-THEAD-NEXT:    th.mveqz a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_sgt:
+; RV64:       # %bb.0:
+; RV64-NEXT:    blt a1, a0, .LBB20_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB20_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_sgt:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp sgt i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_sge(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_sge:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB21_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    slt a0, a1, a3
+; RV32-NEXT:    bnez a0, .LBB21_3
+; RV32-NEXT:    j .LBB21_4
+; RV32-NEXT:  .LBB21_2:
+; RV32-NEXT:    sltu a0, a0, a2
+; RV32-NEXT:    beqz a0, .LBB21_4
+; RV32-NEXT:  .LBB21_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB21_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_sge:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    slt a1, a1, a3
+; RV32-THEAD-NEXT:    sltu a0, a0, a2
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mvnez a4, a6, a0
+; RV32-THEAD-NEXT:    th.mvnez a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_sge:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bge a0, a1, .LBB21_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB21_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_sge:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp sge i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_slt(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_slt:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB22_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    slt a0, a1, a3
+; RV32-NEXT:    beqz a0, .LBB22_3
+; RV32-NEXT:    j .LBB22_4
+; RV32-NEXT:  .LBB22_2:
+; RV32-NEXT:    sltu a0, a0, a2
+; RV32-NEXT:    bnez a0, .LBB22_4
+; RV32-NEXT:  .LBB22_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB22_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_slt:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    slt a1, a1, a3
+; RV32-THEAD-NEXT:    sltu a0, a0, a2
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mveqz a4, a6, a0
+; RV32-THEAD-NEXT:    th.mveqz a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_slt:
+; RV64:       # %bb.0:
+; RV64-NEXT:    blt a0, a1, .LBB22_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB22_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_slt:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a0, a1
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a2, a3
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp slt i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
+define i64 @select_i64_sle(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
+; RV32-LABEL: select_i64_sle:
+; RV32:       # %bb.0:
+; RV32-NEXT:    beq a1, a3, .LBB23_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    slt a0, a3, a1
+; RV32-NEXT:    bnez a0, .LBB23_3
+; RV32-NEXT:    j .LBB23_4
+; RV32-NEXT:  .LBB23_2:
+; RV32-NEXT:    sltu a0, a2, a0
+; RV32-NEXT:    beqz a0, .LBB23_4
+; RV32-NEXT:  .LBB23_3:
+; RV32-NEXT:    mv a4, a6
+; RV32-NEXT:    mv a5, a7
+; RV32-NEXT:  .LBB23_4:
+; RV32-NEXT:    mv a0, a4
+; RV32-NEXT:    mv a1, a5
+; RV32-NEXT:    ret
+;
+; RV32-THEAD-LABEL: select_i64_sle:
+; RV32-THEAD:       # %bb.0:
+; RV32-THEAD-NEXT:    xor t0, a1, a3
+; RV32-THEAD-NEXT:    slt a1, a3, a1
+; RV32-THEAD-NEXT:    sltu a0, a2, a0
+; RV32-THEAD-NEXT:    th.mvnez a0, a1, t0
+; RV32-THEAD-NEXT:    th.mvnez a4, a6, a0
+; RV32-THEAD-NEXT:    th.mvnez a5, a7, a0
+; RV32-THEAD-NEXT:    mv a0, a4
+; RV32-THEAD-NEXT:    mv a1, a5
+; RV32-THEAD-NEXT:    ret
+;
+; RV64-LABEL: select_i64_sle:
+; RV64:       # %bb.0:
+; RV64-NEXT:    bge a1, a0, .LBB23_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a2, a3
+; RV64-NEXT:  .LBB23_2:
+; RV64-NEXT:    mv a0, a2
+; RV64-NEXT:    ret
+;
+; RV64-MIPS-LABEL: select_i64_sle:
+; RV64-MIPS:       # %bb.0:
+; RV64-MIPS-NEXT:    slt a0, a1, a0
+; RV64-MIPS-NEXT:    mips.ccmov a0, a0, a3, a2
+; RV64-MIPS-NEXT:    ret
+  %cond = icmp sle i64 %a, %b
+  %res = select i1 %cond, i64 %x, i64 %y
+  ret i64 %res
+}
+
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/select-const.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/select-const.ll
index 4538572e90ca..bc56408c0ca0 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/select-const.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/select-const.ll
@@ -413,7 +413,7 @@ define i32 @select_eq_10000_10001(i32 signext %a, i32 signext %b) {
 ; RV64-NEXT:    xor a0, a0, a1
 ; RV64-NEXT:    lui a1, 2
 ; RV64-NEXT:    seqz a0, a0
-; RV64-NEXT:    addiw a1, a1, 1810
+; RV64-NEXT:    addi a1, a1, 1810
 ; RV64-NEXT:    sub a0, a1, a0
 ; RV64-NEXT:    ret
   %1 = icmp eq i32 %a, %b
@@ -436,7 +436,7 @@ define i32 @select_ne_10001_10002(i32 signext %a, i32 signext %b) {
 ; RV64-NEXT:    xor a0, a0, a1
 ; RV64-NEXT:    lui a1, 2
 ; RV64-NEXT:    snez a0, a0
-; RV64-NEXT:    addiw a1, a1, 1810
+; RV64-NEXT:    addi a1, a1, 1810
 ; RV64-NEXT:    sub a0, a1, a0
 ; RV64-NEXT:    ret
   %1 = icmp ne i32 %a, %b
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/select.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/select.ll
index 303c4ac23b31..0ea80bf59299 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/select.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/select.ll
@@ -1111,7 +1111,7 @@ define i32 @select_udiv_3(i1 zeroext %cond, i32 %a) {
 ; RV64IM-NEXT:  # %bb.1: # %entry
 ; RV64IM-NEXT:    srliw a0, a1, 1
 ; RV64IM-NEXT:    lui a1, 199729
-; RV64IM-NEXT:    addiw a1, a1, -975
+; RV64IM-NEXT:    addi a1, a1, -975
 ; RV64IM-NEXT:    mul a1, a0, a1
 ; RV64IM-NEXT:    srli a1, a1, 34
 ; RV64IM-NEXT:  .LBB27_2: # %entry
@@ -1122,7 +1122,7 @@ define i32 @select_udiv_3(i1 zeroext %cond, i32 %a) {
 ; RV64IMXVTCONDOPS:       # %bb.0: # %entry
 ; RV64IMXVTCONDOPS-NEXT:    srliw a2, a1, 1
 ; RV64IMXVTCONDOPS-NEXT:    lui a3, 199729
-; RV64IMXVTCONDOPS-NEXT:    addiw a3, a3, -975
+; RV64IMXVTCONDOPS-NEXT:    addi a3, a3, -975
 ; RV64IMXVTCONDOPS-NEXT:    mul a2, a2, a3
 ; RV64IMXVTCONDOPS-NEXT:    srli a2, a2, 34
 ; RV64IMXVTCONDOPS-NEXT:    vt.maskc a1, a1, a0
@@ -1146,7 +1146,7 @@ define i32 @select_udiv_3(i1 zeroext %cond, i32 %a) {
 ; RV64IMZICOND:       # %bb.0: # %entry
 ; RV64IMZICOND-NEXT:    srliw a2, a1, 1
 ; RV64IMZICOND-NEXT:    lui a3, 199729
-; RV64IMZICOND-NEXT:    addiw a3, a3, -975
+; RV64IMZICOND-NEXT:    addi a3, a3, -975
 ; RV64IMZICOND-NEXT:    mul a2, a2, a3
 ; RV64IMZICOND-NEXT:    srli a2, a2, 34
 ; RV64IMZICOND-NEXT:    czero.eqz a1, a1, a0
@@ -1533,50 +1533,14 @@ define i32 @select_cst_not4(i32 signext %a, i32 signext %b) {
 }
 
 define i32 @select_cst_not5(i32 signext %a, i32 signext %b) {
-; RV32IM-LABEL: select_cst_not5:
-; RV32IM:       # %bb.0:
-; RV32IM-NEXT:    slt a0, a0, a1
-; RV32IM-NEXT:    lui a1, 16
-; RV32IM-NEXT:    neg a0, a0
-; RV32IM-NEXT:    addi a1, a1, -5
-; RV32IM-NEXT:    xor a0, a0, a1
-; RV32IM-NEXT:    ret
-;
-; RV64IM-LABEL: select_cst_not5:
-; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    slt a0, a0, a1
-; RV64IM-NEXT:    lui a1, 16
-; RV64IM-NEXT:    neg a0, a0
-; RV64IM-NEXT:    addiw a1, a1, -5
-; RV64IM-NEXT:    xor a0, a0, a1
-; RV64IM-NEXT:    ret
-;
-; RV64IMXVTCONDOPS-LABEL: select_cst_not5:
-; RV64IMXVTCONDOPS:       # %bb.0:
-; RV64IMXVTCONDOPS-NEXT:    slt a0, a0, a1
-; RV64IMXVTCONDOPS-NEXT:    lui a1, 16
-; RV64IMXVTCONDOPS-NEXT:    neg a0, a0
-; RV64IMXVTCONDOPS-NEXT:    addiw a1, a1, -5
-; RV64IMXVTCONDOPS-NEXT:    xor a0, a0, a1
-; RV64IMXVTCONDOPS-NEXT:    ret
-;
-; RV32IMZICOND-LABEL: select_cst_not5:
-; RV32IMZICOND:       # %bb.0:
-; RV32IMZICOND-NEXT:    slt a0, a0, a1
-; RV32IMZICOND-NEXT:    lui a1, 16
-; RV32IMZICOND-NEXT:    neg a0, a0
-; RV32IMZICOND-NEXT:    addi a1, a1, -5
-; RV32IMZICOND-NEXT:    xor a0, a0, a1
-; RV32IMZICOND-NEXT:    ret
-;
-; RV64IMZICOND-LABEL: select_cst_not5:
-; RV64IMZICOND:       # %bb.0:
-; RV64IMZICOND-NEXT:    slt a0, a0, a1
-; RV64IMZICOND-NEXT:    lui a1, 16
-; RV64IMZICOND-NEXT:    neg a0, a0
-; RV64IMZICOND-NEXT:    addiw a1, a1, -5
-; RV64IMZICOND-NEXT:    xor a0, a0, a1
-; RV64IMZICOND-NEXT:    ret
+; CHECK-LABEL: select_cst_not5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    lui a1, 16
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    addi a1, a1, -5
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    ret
   %cond = icmp slt i32 %a, %b
   %ret = select i1 %cond, i32 -65532, i32 65531
   ret i32 %ret
@@ -1678,7 +1642,7 @@ define i32 @select_cst2(i1 zeroext %cond) {
 ; RV64IM-NEXT:    bnez a0, .LBB44_2
 ; RV64IM-NEXT:  # %bb.1:
 ; RV64IM-NEXT:    lui a0, 5
-; RV64IM-NEXT:    addiw a0, a0, -480
+; RV64IM-NEXT:    addi a0, a0, -480
 ; RV64IM-NEXT:    ret
 ; RV64IM-NEXT:  .LBB44_2:
 ; RV64IM-NEXT:    li a0, 10
@@ -1687,26 +1651,18 @@ define i32 @select_cst2(i1 zeroext %cond) {
 ; RV64IMXVTCONDOPS-LABEL: select_cst2:
 ; RV64IMXVTCONDOPS:       # %bb.0:
 ; RV64IMXVTCONDOPS-NEXT:    lui a1, 5
-; RV64IMXVTCONDOPS-NEXT:    addiw a1, a1, -490
+; RV64IMXVTCONDOPS-NEXT:    addi a1, a1, -490
 ; RV64IMXVTCONDOPS-NEXT:    vt.maskcn a0, a1, a0
 ; RV64IMXVTCONDOPS-NEXT:    addi a0, a0, 10
 ; RV64IMXVTCONDOPS-NEXT:    ret
 ;
-; RV32IMZICOND-LABEL: select_cst2:
-; RV32IMZICOND:       # %bb.0:
-; RV32IMZICOND-NEXT:    lui a1, 5
-; RV32IMZICOND-NEXT:    addi a1, a1, -490
-; RV32IMZICOND-NEXT:    czero.nez a0, a1, a0
-; RV32IMZICOND-NEXT:    addi a0, a0, 10
-; RV32IMZICOND-NEXT:    ret
-;
-; RV64IMZICOND-LABEL: select_cst2:
-; RV64IMZICOND:       # %bb.0:
-; RV64IMZICOND-NEXT:    lui a1, 5
-; RV64IMZICOND-NEXT:    addiw a1, a1, -490
-; RV64IMZICOND-NEXT:    czero.nez a0, a1, a0
-; RV64IMZICOND-NEXT:    addi a0, a0, 10
-; RV64IMZICOND-NEXT:    ret
+; CHECKZICOND-LABEL: select_cst2:
+; CHECKZICOND:       # %bb.0:
+; CHECKZICOND-NEXT:    lui a1, 5
+; CHECKZICOND-NEXT:    addi a1, a1, -490
+; CHECKZICOND-NEXT:    czero.nez a0, a1, a0
+; CHECKZICOND-NEXT:    addi a0, a0, 10
+; CHECKZICOND-NEXT:    ret
   %ret = select i1 %cond, i32 10, i32 20000
   ret i32 %ret
 }
@@ -1729,42 +1685,32 @@ define i32 @select_cst3(i1 zeroext %cond) {
 ; RV64IM-NEXT:    bnez a0, .LBB45_2
 ; RV64IM-NEXT:  # %bb.1:
 ; RV64IM-NEXT:    lui a0, 5
-; RV64IM-NEXT:    addiw a0, a0, -480
+; RV64IM-NEXT:    addi a0, a0, -480
 ; RV64IM-NEXT:    ret
 ; RV64IM-NEXT:  .LBB45_2:
 ; RV64IM-NEXT:    lui a0, 7
-; RV64IM-NEXT:    addiw a0, a0, 1328
+; RV64IM-NEXT:    addi a0, a0, 1328
 ; RV64IM-NEXT:    ret
 ;
 ; RV64IMXVTCONDOPS-LABEL: select_cst3:
 ; RV64IMXVTCONDOPS:       # %bb.0:
 ; RV64IMXVTCONDOPS-NEXT:    lui a1, 1048574
-; RV64IMXVTCONDOPS-NEXT:    addiw a1, a1, -1808
+; RV64IMXVTCONDOPS-NEXT:    addi a1, a1, -1808
 ; RV64IMXVTCONDOPS-NEXT:    vt.maskcn a0, a1, a0
 ; RV64IMXVTCONDOPS-NEXT:    lui a1, 7
-; RV64IMXVTCONDOPS-NEXT:    addiw a1, a1, 1328
+; RV64IMXVTCONDOPS-NEXT:    addi a1, a1, 1328
 ; RV64IMXVTCONDOPS-NEXT:    add a0, a0, a1
 ; RV64IMXVTCONDOPS-NEXT:    ret
 ;
-; RV32IMZICOND-LABEL: select_cst3:
-; RV32IMZICOND:       # %bb.0:
-; RV32IMZICOND-NEXT:    lui a1, 1048574
-; RV32IMZICOND-NEXT:    addi a1, a1, -1808
-; RV32IMZICOND-NEXT:    czero.nez a0, a1, a0
-; RV32IMZICOND-NEXT:    lui a1, 7
-; RV32IMZICOND-NEXT:    addi a1, a1, 1328
-; RV32IMZICOND-NEXT:    add a0, a0, a1
-; RV32IMZICOND-NEXT:    ret
-;
-; RV64IMZICOND-LABEL: select_cst3:
-; RV64IMZICOND:       # %bb.0:
-; RV64IMZICOND-NEXT:    lui a1, 1048574
-; RV64IMZICOND-NEXT:    addiw a1, a1, -1808
-; RV64IMZICOND-NEXT:    czero.nez a0, a1, a0
-; RV64IMZICOND-NEXT:    lui a1, 7
-; RV64IMZICOND-NEXT:    addiw a1, a1, 1328
-; RV64IMZICOND-NEXT:    add a0, a0, a1
-; RV64IMZICOND-NEXT:    ret
+; CHECKZICOND-LABEL: select_cst3:
+; CHECKZICOND:       # %bb.0:
+; CHECKZICOND-NEXT:    lui a1, 1048574
+; CHECKZICOND-NEXT:    addi a1, a1, -1808
+; CHECKZICOND-NEXT:    czero.nez a0, a1, a0
+; CHECKZICOND-NEXT:    lui a1, 7
+; CHECKZICOND-NEXT:    addi a1, a1, 1328
+; CHECKZICOND-NEXT:    add a0, a0, a1
+; CHECKZICOND-NEXT:    ret
   %ret = select i1 %cond, i32 30000, i32 20000
   ret i32 %ret
 }
@@ -1796,7 +1742,7 @@ define i32 @select_cst5(i1 zeroext %cond) {
 ; RV64IM-NEXT:    bnez a0, .LBB47_2
 ; RV64IM-NEXT:  # %bb.1:
 ; RV64IM-NEXT:    lui a0, 1
-; RV64IM-NEXT:    addiw a0, a0, -2047
+; RV64IM-NEXT:    addi a0, a0, -2047
 ; RV64IM-NEXT:    ret
 ; RV64IM-NEXT:  .LBB47_2:
 ; RV64IM-NEXT:    li a0, 2047
@@ -1839,7 +1785,7 @@ define i32 @select_cst5_invert(i1 zeroext %cond) {
 ; RV64IM-NEXT:    ret
 ; RV64IM-NEXT:  .LBB48_2:
 ; RV64IM-NEXT:    lui a0, 1
-; RV64IM-NEXT:    addiw a0, a0, -2047
+; RV64IM-NEXT:    addi a0, a0, -2047
 ; RV64IM-NEXT:    ret
 ;
 ; RV64IMXVTCONDOPS-LABEL: select_cst5_invert:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/sextw-removal.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/sextw-removal.ll
index 49494608eee4..1a978d1a0fca 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/sextw-removal.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/sextw-removal.ll
@@ -180,8 +180,8 @@ define void @test5(i32 signext %arg, i32 signext %arg1) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw s0, a1, 1365
-; RV64I-NEXT:    addiw s1, a2, 819
+; RV64I-NEXT:    addi s0, a1, 1365
+; RV64I-NEXT:    addi s1, a2, 819
 ; RV64I-NEXT:    addi s2, a3, -241
 ; RV64I-NEXT:    addi s3, a4, 257
 ; RV64I-NEXT:  .LBB4_1: # %bb2
@@ -327,10 +327,10 @@ define void @test7(i32 signext %arg, i32 signext %arg1) nounwind {
 ; RV64I-NEXT:    lui a2, 209715
 ; RV64I-NEXT:    lui a3, 61681
 ; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    addiw s0, a1, 1365
-; RV64I-NEXT:    addiw s1, a2, 819
-; RV64I-NEXT:    addiw s2, a3, -241
-; RV64I-NEXT:    addiw s3, a4, 257
+; RV64I-NEXT:    addi s0, a1, 1365
+; RV64I-NEXT:    addi s1, a2, 819
+; RV64I-NEXT:    addi s2, a3, -241
+; RV64I-NEXT:    addi s3, a4, 257
 ; RV64I-NEXT:    slli a1, s0, 32
 ; RV64I-NEXT:    add s0, s0, a1
 ; RV64I-NEXT:    slli a1, s1, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/shl-cttz.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/shl-cttz.ll
index 500673cc2919..99dc4f816d66 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/shl-cttz.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/shl-cttz.ll
@@ -158,11 +158,11 @@ define i16 @shl_cttz_i16(i16 %x, i16 %y) {
 ; RV64I-NEXT:    not a1, a1
 ; RV64I-NEXT:    lui a3, 5
 ; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    addiw a2, a3, 1365
+; RV64I-NEXT:    addi a2, a3, 1365
 ; RV64I-NEXT:    srli a3, a1, 1
 ; RV64I-NEXT:    and a2, a3, a2
 ; RV64I-NEXT:    lui a3, 3
-; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    addi a3, a3, 819
 ; RV64I-NEXT:    sub a1, a1, a2
 ; RV64I-NEXT:    and a2, a1, a3
 ; RV64I-NEXT:    srli a1, a1, 2
@@ -228,11 +228,11 @@ define i16 @shl_cttz_constant_i16(i16 %y) {
 ; RV64I-NEXT:    not a0, a0
 ; RV64I-NEXT:    lui a2, 5
 ; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, 1365
+; RV64I-NEXT:    addi a1, a2, 1365
 ; RV64I-NEXT:    srli a2, a0, 1
 ; RV64I-NEXT:    and a1, a2, a1
 ; RV64I-NEXT:    lui a2, 3
-; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    addi a2, a2, 819
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    and a1, a0, a2
 ; RV64I-NEXT:    srli a0, a0, 2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/shlimm-addimm.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/shlimm-addimm.ll
index c842ba5da520..da7f62cd3ff0 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/shlimm-addimm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/shlimm-addimm.ll
@@ -125,7 +125,7 @@ define i64 @shl5_add101024_c(i64 %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 5
 ; RV64I-NEXT:    lui a1, 25
-; RV64I-NEXT:    addiw a1, a1, -1376
+; RV64I-NEXT:    addi a1, a1, -1376
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
   %tmp0 = shl i64 %x, 5
@@ -193,7 +193,7 @@ define i64 @shl5_add47968_c(i64 %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 5
 ; RV64I-NEXT:    lui a1, 12
-; RV64I-NEXT:    addiw a1, a1, -1184
+; RV64I-NEXT:    addi a1, a1, -1184
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
   %tmp0 = shl i64 %x, 5
@@ -261,7 +261,7 @@ define i64 @shl5_add47969_c(i64 %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 5
 ; RV64I-NEXT:    lui a1, 12
-; RV64I-NEXT:    addiw a1, a1, -1183
+; RV64I-NEXT:    addi a1, a1, -1183
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
   %tmp0 = shl i64 %x, 5
@@ -330,7 +330,7 @@ define i64 @shl5_sub47968_c(i64 %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 5
 ; RV64I-NEXT:    lui a1, 1048564
-; RV64I-NEXT:    addiw a1, a1, 1184
+; RV64I-NEXT:    addi a1, a1, 1184
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
   %tmp0 = shl i64 %x, 5
@@ -399,7 +399,7 @@ define i64 @shl5_sub47969_c(i64 %x) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 5
 ; RV64I-NEXT:    lui a1, 1048564
-; RV64I-NEXT:    addiw a1, a1, 1183
+; RV64I-NEXT:    addi a1, a1, 1183
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ret
   %tmp0 = shl i64 %x, 5
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
index b7b88584f3bd..13c43a3875a0 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
@@ -173,14 +173,21 @@ define signext i32 @test6(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: test6:
-; SFB:       # %bb.0:
-; SFB-NEXT:    li a2, -1
-; SFB-NEXT:    beqz a1, .LBB5_2
-; SFB-NEXT:  # %bb.1:
-; SFB-NEXT:    mv a0, a2
-; SFB-NEXT:  .LBB5_2:
-; SFB-NEXT:    ret
+; NOZICOND-LABEL: test6:
+; NOZICOND:       # %bb.0:
+; NOZICOND-NEXT:    li a2, -1
+; NOZICOND-NEXT:    beqz a1, .LBB5_2
+; NOZICOND-NEXT:  # %bb.1:
+; NOZICOND-NEXT:    mv a0, a2
+; NOZICOND-NEXT:  .LBB5_2:
+; NOZICOND-NEXT:    ret
+;
+; ZICOND-LABEL: test6:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    addi a0, a0, 1
+; ZICOND-NEXT:    czero.nez a0, a0, a1
+; ZICOND-NEXT:    addi a0, a0, -1
+; ZICOND-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %x, i32 -1
   ret i32 %b
@@ -195,14 +202,21 @@ define signext i32 @test7(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: test7:
-; SFB:       # %bb.0:
-; SFB-NEXT:    li a2, -1
-; SFB-NEXT:    bnez a1, .LBB6_2
-; SFB-NEXT:  # %bb.1:
-; SFB-NEXT:    mv a0, a2
-; SFB-NEXT:  .LBB6_2:
-; SFB-NEXT:    ret
+; NOZICOND-LABEL: test7:
+; NOZICOND:       # %bb.0:
+; NOZICOND-NEXT:    li a2, -1
+; NOZICOND-NEXT:    bnez a1, .LBB6_2
+; NOZICOND-NEXT:  # %bb.1:
+; NOZICOND-NEXT:    mv a0, a2
+; NOZICOND-NEXT:  .LBB6_2:
+; NOZICOND-NEXT:    ret
+;
+; ZICOND-LABEL: test7:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    addi a0, a0, 1
+; ZICOND-NEXT:    czero.eqz a0, a0, a1
+; ZICOND-NEXT:    addi a0, a0, -1
+; ZICOND-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 -1, i32 %x
   ret i32 %b
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/signed-truncation-check.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
index d43dfd46d62f..e70c8e354469 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
@@ -349,7 +349,7 @@ define i1 @add_ugecmp_i32_i16(i32 %x) nounwind {
 ; RV64I-NEXT:    lui a1, 1048568
 ; RV64I-NEXT:    addw a0, a0, a1
 ; RV64I-NEXT:    lui a1, 1048560
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    sltu a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
@@ -429,7 +429,7 @@ define i1 @add_ugecmp_i64_i16(i64 %x) nounwind {
 ; RV64I-NEXT:    lui a1, 1048568
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    lui a1, 1048560
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    sltu a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
@@ -805,7 +805,7 @@ define i1 @add_ultcmp_bad_i16_i8_cmp(i16 %x, i16 %y) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    addi a0, a0, 128
-; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    sltu a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/split-offsets.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/split-offsets.ll
index cecd34956df8..bf08270e0bb9 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/split-offsets.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/split-offsets.ll
@@ -25,17 +25,16 @@ define void @test1(ptr %sp, ptr %t, i32 %n) {
 ;
 ; RV64I-LABEL: test1:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    lui a2, 20
 ; RV64I-NEXT:    ld a0, 0(a0)
+; RV64I-NEXT:    lui a2, 20
 ; RV64I-NEXT:    li a3, 2
-; RV64I-NEXT:    addiw a2, a2, -1920
 ; RV64I-NEXT:    add a1, a1, a2
 ; RV64I-NEXT:    add a0, a0, a2
 ; RV64I-NEXT:    li a2, 1
-; RV64I-NEXT:    sw a3, 0(a0)
-; RV64I-NEXT:    sw a2, 4(a0)
-; RV64I-NEXT:    sw a2, 0(a1)
-; RV64I-NEXT:    sw a3, 4(a1)
+; RV64I-NEXT:    sw a3, -1920(a0)
+; RV64I-NEXT:    sw a2, -1916(a0)
+; RV64I-NEXT:    sw a2, -1920(a1)
+; RV64I-NEXT:    sw a3, -1916(a1)
 ; RV64I-NEXT:    ret
 entry:
   %s = load ptr, ptr %sp
@@ -77,7 +76,6 @@ define void @test2(ptr %sp, ptr %t, i32 %n) {
 ; RV64I-NEXT:    li a3, 0
 ; RV64I-NEXT:    ld a0, 0(a0)
 ; RV64I-NEXT:    lui a4, 20
-; RV64I-NEXT:    addiw a4, a4, -1920
 ; RV64I-NEXT:    add a1, a1, a4
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    sext.w a2, a2
@@ -85,10 +83,10 @@ define void @test2(ptr %sp, ptr %t, i32 %n) {
 ; RV64I-NEXT:  .LBB1_1: # %while_body
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV64I-NEXT:    addiw a4, a3, 1
-; RV64I-NEXT:    sw a4, 0(a0)
-; RV64I-NEXT:    sw a3, 4(a0)
-; RV64I-NEXT:    sw a4, 0(a1)
-; RV64I-NEXT:    sw a3, 4(a1)
+; RV64I-NEXT:    sw a4, -1920(a0)
+; RV64I-NEXT:    sw a3, -1916(a0)
+; RV64I-NEXT:    sw a4, -1920(a1)
+; RV64I-NEXT:    sw a3, -1916(a1)
 ; RV64I-NEXT:    mv a3, a4
 ; RV64I-NEXT:    blt a4, a2, .LBB1_1
 ; RV64I-NEXT:  .LBB1_2: # %while_end
@@ -134,11 +132,10 @@ define void @test3(ptr %t) {
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    lui a1, 20
 ; RV64I-NEXT:    li a2, 2
-; RV64I-NEXT:    addiw a1, a1, -1920
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    li a1, 3
-; RV64I-NEXT:    sw a2, 4(a0)
-; RV64I-NEXT:    sw a1, 8(a0)
+; RV64I-NEXT:    sw a2, -1916(a0)
+; RV64I-NEXT:    sw a1, -1912(a0)
 ; RV64I-NEXT:    ret
 entry:
   %splitgep = getelementptr i8, ptr %t, i64 80000
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/split-udiv-by-constant.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/split-udiv-by-constant.ll
index 83ae03452db5..eb70d7f43c0e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/split-udiv-by-constant.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/split-udiv-by-constant.ll
@@ -35,7 +35,7 @@ define iXLen2 @test_udiv_3(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a3, 699051
 ; RV64-NEXT:    lui a4, %hi(.LCPI0_0)
 ; RV64-NEXT:    sltu a5, a2, a0
-; RV64-NEXT:    addiw a3, a3, -1365
+; RV64-NEXT:    addi a3, a3, -1365
 ; RV64-NEXT:    ld a4, %lo(.LCPI0_0)(a4)
 ; RV64-NEXT:    add a2, a2, a5
 ; RV64-NEXT:    slli a5, a3, 32
@@ -90,7 +90,7 @@ define iXLen2 @test_udiv_5(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a3, 838861
 ; RV64-NEXT:    lui a4, %hi(.LCPI1_0)
 ; RV64-NEXT:    sltu a5, a2, a0
-; RV64-NEXT:    addiw a3, a3, -819
+; RV64-NEXT:    addi a3, a3, -819
 ; RV64-NEXT:    ld a4, %lo(.LCPI1_0)(a4)
 ; RV64-NEXT:    add a2, a2, a5
 ; RV64-NEXT:    slli a5, a3, 32
@@ -200,9 +200,9 @@ define iXLen2 @test_udiv_15(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a4, %hi(.LCPI4_0)
 ; RV64-NEXT:    lui a5, 978671
 ; RV64-NEXT:    sltu a6, a2, a0
-; RV64-NEXT:    addiw a3, a3, -1911
+; RV64-NEXT:    addi a3, a3, -1911
 ; RV64-NEXT:    ld a4, %lo(.LCPI4_0)(a4)
-; RV64-NEXT:    addiw a5, a5, -273
+; RV64-NEXT:    addi a5, a5, -273
 ; RV64-NEXT:    add a2, a2, a6
 ; RV64-NEXT:    slli a6, a3, 32
 ; RV64-NEXT:    add a3, a3, a6
@@ -258,7 +258,7 @@ define iXLen2 @test_udiv_17(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a3, 986895
 ; RV64-NEXT:    lui a4, %hi(.LCPI5_0)
 ; RV64-NEXT:    sltu a5, a2, a0
-; RV64-NEXT:    addiw a3, a3, 241
+; RV64-NEXT:    addi a3, a3, 241
 ; RV64-NEXT:    ld a4, %lo(.LCPI5_0)(a4)
 ; RV64-NEXT:    add a2, a2, a5
 ; RV64-NEXT:    slli a5, a3, 32
@@ -316,9 +316,9 @@ define iXLen2 @test_udiv_255(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a4, %hi(.LCPI6_0)
 ; RV64-NEXT:    lui a5, 1044464
 ; RV64-NEXT:    sltu a6, a2, a0
-; RV64-NEXT:    addiw a3, a3, 129
+; RV64-NEXT:    addi a3, a3, 129
 ; RV64-NEXT:    ld a4, %lo(.LCPI6_0)(a4)
-; RV64-NEXT:    addiw a5, a5, -257
+; RV64-NEXT:    addi a5, a5, -257
 ; RV64-NEXT:    add a2, a2, a6
 ; RV64-NEXT:    slli a6, a3, 32
 ; RV64-NEXT:    add a3, a3, a6
@@ -374,7 +374,7 @@ define iXLen2 @test_udiv_257(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a3, 1044496
 ; RV64-NEXT:    lui a4, %hi(.LCPI7_0)
 ; RV64-NEXT:    sltu a5, a2, a0
-; RV64-NEXT:    addiw a3, a3, -255
+; RV64-NEXT:    addi a3, a3, -255
 ; RV64-NEXT:    ld a4, %lo(.LCPI7_0)(a4)
 ; RV64-NEXT:    add a2, a2, a5
 ; RV64-NEXT:    slli a5, a3, 32
@@ -435,9 +435,9 @@ define iXLen2 @test_udiv_65535(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a4, 983039
 ; RV64-NEXT:    lui a5, 1048560
 ; RV64-NEXT:    sltu a6, a2, a0
-; RV64-NEXT:    addiw a3, a3, 1
+; RV64-NEXT:    addi a3, a3, 1
 ; RV64-NEXT:    slli a4, a4, 4
-; RV64-NEXT:    addiw a5, a5, -1
+; RV64-NEXT:    addi a5, a5, -1
 ; RV64-NEXT:    add a2, a2, a6
 ; RV64-NEXT:    slli a6, a3, 32
 ; RV64-NEXT:    addi a4, a4, -1
@@ -496,7 +496,7 @@ define iXLen2 @test_udiv_65537(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a3, 1048560
 ; RV64-NEXT:    lui a4, 983041
 ; RV64-NEXT:    sltu a5, a2, a0
-; RV64-NEXT:    addiw a6, a3, 1
+; RV64-NEXT:    addi a6, a3, 1
 ; RV64-NEXT:    slli a4, a4, 4
 ; RV64-NEXT:    add a2, a2, a5
 ; RV64-NEXT:    slli a5, a6, 32
@@ -559,7 +559,7 @@ define iXLen2 @test_udiv_12(iXLen2 %x) nounwind {
 ; RV64-NEXT:    lui a3, 699051
 ; RV64-NEXT:    lui a4, %hi(.LCPI10_0)
 ; RV64-NEXT:    or a0, a0, a2
-; RV64-NEXT:    addiw a2, a3, -1365
+; RV64-NEXT:    addi a2, a3, -1365
 ; RV64-NEXT:    ld a3, %lo(.LCPI10_0)(a4)
 ; RV64-NEXT:    add a4, a0, a1
 ; RV64-NEXT:    slli a5, a2, 32
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/split-urem-by-constant.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/split-urem-by-constant.ll
index ae8117c3ce0b..bc4a99a00ac6 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/split-urem-by-constant.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/split-urem-by-constant.ll
@@ -25,7 +25,7 @@ define iXLen2 @test_urem_3(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 699051
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a2, a2, -1365
+; RV64-NEXT:    addi a2, a2, -1365
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a2, 32
 ; RV64-NEXT:    add a1, a2, a1
@@ -61,7 +61,7 @@ define iXLen2 @test_urem_5(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 838861
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a2, a2, -819
+; RV64-NEXT:    addi a2, a2, -819
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a2, 32
 ; RV64-NEXT:    add a1, a2, a1
@@ -149,7 +149,7 @@ define iXLen2 @test_urem_15(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 559241
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a2, a2, -1911
+; RV64-NEXT:    addi a2, a2, -1911
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a2, 32
 ; RV64-NEXT:    add a1, a2, a1
@@ -185,7 +185,7 @@ define iXLen2 @test_urem_17(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 986895
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a2, a2, 241
+; RV64-NEXT:    addi a2, a2, 241
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a2, 32
 ; RV64-NEXT:    add a1, a2, a1
@@ -221,7 +221,7 @@ define iXLen2 @test_urem_255(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 526344
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a2, a2, 129
+; RV64-NEXT:    addi a2, a2, 129
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a2, 32
 ; RV64-NEXT:    add a1, a2, a1
@@ -257,7 +257,7 @@ define iXLen2 @test_urem_257(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 1044496
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a2, a2, -255
+; RV64-NEXT:    addi a2, a2, -255
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a2, 32
 ; RV64-NEXT:    add a1, a2, a1
@@ -293,7 +293,7 @@ define iXLen2 @test_urem_65535(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 524296
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a2, a2, 1
+; RV64-NEXT:    addi a2, a2, 1
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a2, 32
 ; RV64-NEXT:    add a1, a2, a1
@@ -329,7 +329,7 @@ define iXLen2 @test_urem_65537(iXLen2 %x) nounwind {
 ; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    lui a2, 1048560
 ; RV64-NEXT:    sltu a0, a1, a0
-; RV64-NEXT:    addiw a3, a2, 1
+; RV64-NEXT:    addi a3, a2, 1
 ; RV64-NEXT:    add a0, a1, a0
 ; RV64-NEXT:    slli a1, a3, 32
 ; RV64-NEXT:    add a1, a3, a1
@@ -373,7 +373,7 @@ define iXLen2 @test_urem_12(iXLen2 %x) nounwind {
 ; RV64-NEXT:    srli a3, a0, 2
 ; RV64-NEXT:    lui a4, 699051
 ; RV64-NEXT:    or a2, a3, a2
-; RV64-NEXT:    addiw a3, a4, -1365
+; RV64-NEXT:    addi a3, a4, -1365
 ; RV64-NEXT:    slli a4, a3, 32
 ; RV64-NEXT:    add a3, a3, a4
 ; RV64-NEXT:    srli a1, a1, 2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/srem-lkk.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/srem-lkk.ll
index 7c291bbceedc..54a8f8625bbe 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/srem-lkk.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/srem-lkk.ll
@@ -43,7 +43,7 @@ define i32 @fold_srem_positive_odd(i32 %x) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    sext.w a1, a0
 ; RV64IM-NEXT:    lui a2, 706409
-; RV64IM-NEXT:    addiw a2, a2, 389
+; RV64IM-NEXT:    addi a2, a2, 389
 ; RV64IM-NEXT:    mul a1, a1, a2
 ; RV64IM-NEXT:    srli a1, a1, 32
 ; RV64IM-NEXT:    add a1, a1, a0
@@ -93,7 +93,7 @@ define i32 @fold_srem_positive_even(i32 %x) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    sext.w a1, a0
 ; RV64IM-NEXT:    lui a2, 253241
-; RV64IM-NEXT:    addiw a2, a2, -15
+; RV64IM-NEXT:    addi a2, a2, -15
 ; RV64IM-NEXT:    mul a1, a1, a2
 ; RV64IM-NEXT:    srli a2, a1, 63
 ; RV64IM-NEXT:    srai a1, a1, 40
@@ -141,7 +141,7 @@ define i32 @fold_srem_negative_odd(i32 %x) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    sext.w a1, a0
 ; RV64IM-NEXT:    lui a2, 677296
-; RV64IM-NEXT:    addiw a2, a2, -91
+; RV64IM-NEXT:    addi a2, a2, -91
 ; RV64IM-NEXT:    mul a1, a1, a2
 ; RV64IM-NEXT:    srli a2, a1, 63
 ; RV64IM-NEXT:    srai a1, a1, 40
@@ -182,7 +182,7 @@ define i32 @fold_srem_negative_even(i32 %x) nounwind {
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    lui a1, 1048570
-; RV64I-NEXT:    addiw a1, a1, 1595
+; RV64I-NEXT:    addi a1, a1, 1595
 ; RV64I-NEXT:    call __moddi3
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -192,7 +192,7 @@ define i32 @fold_srem_negative_even(i32 %x) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    sext.w a1, a0
 ; RV64IM-NEXT:    lui a2, 1036895
-; RV64IM-NEXT:    addiw a2, a2, 999
+; RV64IM-NEXT:    addi a2, a2, 999
 ; RV64IM-NEXT:    mul a1, a1, a2
 ; RV64IM-NEXT:    srli a2, a1, 63
 ; RV64IM-NEXT:    srai a1, a1, 40
@@ -269,7 +269,7 @@ define i32 @combine_srem_sdiv(i32 %x) nounwind {
 ; RV64IM:       # %bb.0:
 ; RV64IM-NEXT:    sext.w a1, a0
 ; RV64IM-NEXT:    lui a2, 706409
-; RV64IM-NEXT:    addiw a2, a2, 389
+; RV64IM-NEXT:    addi a2, a2, 389
 ; RV64IM-NEXT:    mul a1, a1, a2
 ; RV64IM-NEXT:    srli a1, a1, 32
 ; RV64IM-NEXT:    add a1, a1, a0
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
index 17a09bf7dbe6..93fb230f51ce 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
@@ -63,7 +63,7 @@ define i1 @test_srem_odd(i29 %X) nounwind {
 ; RV64-NEXT:    subw a1, a1, a0
 ; RV64-NEXT:    slli a1, a1, 35
 ; RV64-NEXT:    srli a1, a1, 35
-; RV64-NEXT:    addiw a0, a3, -165
+; RV64-NEXT:    addi a0, a3, -165
 ; RV64-NEXT:    sltu a0, a1, a0
 ; RV64-NEXT:    ret
 ;
@@ -93,7 +93,7 @@ define i1 @test_srem_odd(i29 %X) nounwind {
 ; RV64M-NEXT:    lui a1, 1324
 ; RV64M-NEXT:    slli a0, a0, 35
 ; RV64M-NEXT:    srli a0, a0, 35
-; RV64M-NEXT:    addiw a1, a1, -165
+; RV64M-NEXT:    addi a1, a1, -165
 ; RV64M-NEXT:    sltu a0, a0, a1
 ; RV64M-NEXT:    ret
 ;
@@ -123,7 +123,7 @@ define i1 @test_srem_odd(i29 %X) nounwind {
 ; RV64MV-NEXT:    lui a1, 1324
 ; RV64MV-NEXT:    slli a0, a0, 35
 ; RV64MV-NEXT:    srli a0, a0, 35
-; RV64MV-NEXT:    addiw a1, a1, -165
+; RV64MV-NEXT:    addi a1, a1, -165
 ; RV64MV-NEXT:    sltu a0, a0, a1
 ; RV64MV-NEXT:    ret
   %srem = srem i29 %X, 99
@@ -612,7 +612,7 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV64M-NEXT:    lbu a3, 12(a0)
 ; RV64M-NEXT:    lui a4, %hi(.LCPI3_0)
 ; RV64M-NEXT:    lui a5, 699051
-; RV64M-NEXT:    addiw a5, a5, -1365
+; RV64M-NEXT:    addi a5, a5, -1365
 ; RV64M-NEXT:    slli a6, a5, 32
 ; RV64M-NEXT:    add a5, a5, a6
 ; RV64M-NEXT:    srli a6, a1, 2
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
index cf65d4e0cf80..30ffaf6c7cec 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
@@ -562,49 +562,49 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ;
 ; RV64IM-LABEL: combine_srem_sdiv:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    lh a2, 16(a1)
-; RV64IM-NEXT:    lh a3, 24(a1)
-; RV64IM-NEXT:    lui a4, %hi(.LCPI2_0)
-; RV64IM-NEXT:    ld a4, %lo(.LCPI2_0)(a4)
+; RV64IM-NEXT:    lui a2, %hi(.LCPI2_0)
+; RV64IM-NEXT:    ld a2, %lo(.LCPI2_0)(a2)
+; RV64IM-NEXT:    lh a3, 16(a1)
+; RV64IM-NEXT:    lh a4, 24(a1)
 ; RV64IM-NEXT:    lh a5, 0(a1)
 ; RV64IM-NEXT:    lh a1, 8(a1)
 ; RV64IM-NEXT:    li a6, 95
-; RV64IM-NEXT:    mulh a7, a3, a4
-; RV64IM-NEXT:    mulh t0, a2, a4
-; RV64IM-NEXT:    mulh t1, a1, a4
-; RV64IM-NEXT:    mulh a4, a5, a4
-; RV64IM-NEXT:    add a7, a7, a3
-; RV64IM-NEXT:    add t0, t0, a2
+; RV64IM-NEXT:    mulh a7, a4, a2
+; RV64IM-NEXT:    mulh t0, a3, a2
+; RV64IM-NEXT:    mulh t1, a1, a2
+; RV64IM-NEXT:    mulh a2, a5, a2
+; RV64IM-NEXT:    add a7, a7, a4
+; RV64IM-NEXT:    add t0, t0, a3
 ; RV64IM-NEXT:    add t1, t1, a1
-; RV64IM-NEXT:    add a4, a4, a5
+; RV64IM-NEXT:    add a2, a2, a5
 ; RV64IM-NEXT:    srli t2, a7, 63
 ; RV64IM-NEXT:    srai a7, a7, 6
 ; RV64IM-NEXT:    srli t3, t0, 63
 ; RV64IM-NEXT:    srai t0, t0, 6
 ; RV64IM-NEXT:    srli t4, t1, 63
 ; RV64IM-NEXT:    srai t1, t1, 6
-; RV64IM-NEXT:    srli t5, a4, 63
-; RV64IM-NEXT:    srai a4, a4, 6
+; RV64IM-NEXT:    srli t5, a2, 63
+; RV64IM-NEXT:    srai a2, a2, 6
 ; RV64IM-NEXT:    add a7, a7, t2
 ; RV64IM-NEXT:    add t0, t0, t3
 ; RV64IM-NEXT:    add t1, t1, t4
-; RV64IM-NEXT:    add a4, a4, t5
+; RV64IM-NEXT:    add a2, a2, t5
 ; RV64IM-NEXT:    mul t2, a7, a6
 ; RV64IM-NEXT:    mul t3, t0, a6
 ; RV64IM-NEXT:    mul t4, t1, a6
-; RV64IM-NEXT:    mul a6, a4, a6
-; RV64IM-NEXT:    add a4, a5, a4
+; RV64IM-NEXT:    mul a6, a2, a6
+; RV64IM-NEXT:    add a2, a5, a2
 ; RV64IM-NEXT:    add a1, a1, t1
-; RV64IM-NEXT:    add a2, a2, t0
-; RV64IM-NEXT:    add a3, a3, a7
-; RV64IM-NEXT:    subw a4, a4, a6
+; RV64IM-NEXT:    add a3, a3, t0
+; RV64IM-NEXT:    add a4, a4, a7
+; RV64IM-NEXT:    subw a2, a2, a6
 ; RV64IM-NEXT:    subw a1, a1, t4
-; RV64IM-NEXT:    subw a2, a2, t3
-; RV64IM-NEXT:    subw a3, a3, t2
-; RV64IM-NEXT:    sh a4, 0(a0)
+; RV64IM-NEXT:    subw a3, a3, t3
+; RV64IM-NEXT:    subw a4, a4, t2
+; RV64IM-NEXT:    sh a2, 0(a0)
 ; RV64IM-NEXT:    sh a1, 2(a0)
-; RV64IM-NEXT:    sh a2, 4(a0)
-; RV64IM-NEXT:    sh a3, 6(a0)
+; RV64IM-NEXT:    sh a3, 4(a0)
+; RV64IM-NEXT:    sh a4, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
   %2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
@@ -863,7 +863,7 @@ define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    call __moddi3
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    addi a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __moddi3
 ; RV64I-NEXT:    sh zero, 0(s2)
@@ -1016,7 +1016,7 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    call __moddi3
 ; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    addi a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __moddi3
 ; RV64I-NEXT:    sh zero, 0(s0)
@@ -1233,7 +1233,7 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV64I-NEXT:    call __moddi3
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    addi a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __moddi3
 ; RV64I-NEXT:    sd zero, 0(s2)
@@ -1275,7 +1275,7 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV64IM-NEXT:    li a7, 654
 ; RV64IM-NEXT:    mul a5, a5, a7
 ; RV64IM-NEXT:    lui a7, 1
-; RV64IM-NEXT:    addiw a7, a7, 1327
+; RV64IM-NEXT:    addi a7, a7, 1327
 ; RV64IM-NEXT:    mul a6, a6, a7
 ; RV64IM-NEXT:    li a7, 23
 ; RV64IM-NEXT:    mul a4, a4, a7
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat.ll
index cc5cd76e913c..ba4d170c719f 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat.ll
@@ -113,7 +113,7 @@ define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    sub a0, a0, a1
 ; RV64I-NEXT:    lui a1, 8
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    bge a0, a1, .LBB2_3
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    lui a1, 1048568
@@ -142,7 +142,7 @@ define signext i16 @func16(i16 signext %x, i16 signext %y) nounwind {
 ; RV64IZbb:       # %bb.0:
 ; RV64IZbb-NEXT:    sub a0, a0, a1
 ; RV64IZbb-NEXT:    lui a1, 8
-; RV64IZbb-NEXT:    addiw a1, a1, -1
+; RV64IZbb-NEXT:    addi a1, a1, -1
 ; RV64IZbb-NEXT:    min a0, a0, a1
 ; RV64IZbb-NEXT:    lui a1, 1048568
 ; RV64IZbb-NEXT:    max a0, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
index 0499992b7177..437c1e2a2e48 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
@@ -128,7 +128,7 @@ define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
 ; RV64I-NEXT:    slli a1, a1, 48
 ; RV64I-NEXT:    srai a1, a1, 48
 ; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    addiw a1, a2, -1
+; RV64I-NEXT:    addi a1, a2, -1
 ; RV64I-NEXT:    bge a0, a1, .LBB2_3
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    lui a1, 1048568
@@ -162,7 +162,7 @@ define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
 ; RV64IZbb-NEXT:    mul a1, a1, a2
 ; RV64IZbb-NEXT:    lui a2, 8
 ; RV64IZbb-NEXT:    sext.h a1, a1
-; RV64IZbb-NEXT:    addiw a2, a2, -1
+; RV64IZbb-NEXT:    addi a2, a2, -1
 ; RV64IZbb-NEXT:    sub a0, a0, a1
 ; RV64IZbb-NEXT:    min a0, a0, a2
 ; RV64IZbb-NEXT:    lui a1, 1048568
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue-nounwind.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue-nounwind.ll
index 882d0b814063..92b2afe2e72c 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue-nounwind.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue-nounwind.ll
@@ -44,7 +44,7 @@ define i8 @f1() #0 nounwind {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    ret
 ;
@@ -86,7 +86,7 @@ define i8 @f2() #0 nounwind {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    ret
 ;
@@ -131,7 +131,7 @@ define i8 @f3() #0 "stack-probe-size"="32768" nounwind {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    ret
 ;
@@ -164,13 +164,13 @@ define i8 @f4() nounwind {
 ; RV64I-LABEL: f4:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    lui a0, 16
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    li a0, 3
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    ret
 ;
@@ -211,7 +211,7 @@ define i8 @f5() #0 "stack-probe-size"="65536" nounwind {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 256
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    ret
 ;
@@ -259,7 +259,7 @@ define i8 @f6() #0 nounwind {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 262144
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    ret
 ;
@@ -303,13 +303,13 @@ define i8 @f7() #0 "stack-probe-size"="65536" nounwind {
 ; RV64I-NEXT:    bne sp, t1, .LBB7_1
 ; RV64I-NEXT:  # %bb.2: # %entry
 ; RV64I-NEXT:    lui a0, 13
-; RV64I-NEXT:    addiw a0, a0, -1520
+; RV64I-NEXT:    addi a0, a0, -1520
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    li a0, 3
 ; RV64I-NEXT:    sb a0, 9(sp)
 ; RV64I-NEXT:    lbu a0, 9(sp)
 ; RV64I-NEXT:    lui a1, 244141
-; RV64I-NEXT:    addiw a1, a1, -1520
+; RV64I-NEXT:    addi a1, a1, -1520
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    ret
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue.ll
index b1c0755c36ec..b76f2b246470 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-clash-prologue.ll
@@ -50,7 +50,7 @@ define i8 @f1() #0 {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
@@ -99,7 +99,7 @@ define i8 @f2() #0 {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
@@ -152,7 +152,7 @@ define i8 @f3() #0 "stack-probe-size"="32768" {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
@@ -190,14 +190,14 @@ define i8 @f4() {
 ; RV64I-LABEL: f4:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    lui a0, 16
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 65552
 ; RV64I-NEXT:    li a0, 3
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
@@ -244,7 +244,7 @@ define i8 @f5() #0 "stack-probe-size"="65536" {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 256
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
@@ -300,7 +300,7 @@ define i8 @f6() #0 {
 ; RV64I-NEXT:    sb a0, 16(sp)
 ; RV64I-NEXT:    lbu a0, 16(sp)
 ; RV64I-NEXT:    lui a1, 262144
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
@@ -351,14 +351,14 @@ define i8 @f7() #0 "stack-probe-size"="65536" {
 ; RV64I-NEXT:  # %bb.2: # %entry
 ; RV64I-NEXT:    .cfi_def_cfa_register sp
 ; RV64I-NEXT:    lui a0, 13
-; RV64I-NEXT:    addiw a0, a0, -1520
+; RV64I-NEXT:    addi a0, a0, -1520
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 1000000016
 ; RV64I-NEXT:    li a0, 3
 ; RV64I-NEXT:    sb a0, 9(sp)
 ; RV64I-NEXT:    lbu a0, 9(sp)
 ; RV64I-NEXT:    lui a1, 244141
-; RV64I-NEXT:    addiw a1, a1, -1520
+; RV64I-NEXT:    addi a1, a1, -1520
 ; RV64I-NEXT:    add sp, sp, a1
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-inst-compress.mir b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-inst-compress.mir
index fe84d2996335..bd31662ae2da 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-inst-compress.mir
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-inst-compress.mir
@@ -300,13 +300,13 @@ body:             |
     ; CHECK-RV64-NO-COM-NEXT: frame-setup SD killed $x1, $x2, 2024 :: (store (s64) into %stack.1)
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-setup LUI 2
-    ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-setup ADDIW killed $x10, -2016
+    ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-setup ADDI killed $x10, -2016
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-setup SUB $x2, killed $x10
     ; CHECK-RV64-NO-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8208
     ; CHECK-RV64-NO-COM-NEXT: renamable $x10 = ADDI $x2, 8
     ; CHECK-RV64-NO-COM-NEXT: PseudoCALL target-flags(riscv-call) @_Z6calleePi, csr_ilp32_lp64, implicit-def dead $x1, implicit killed $x10, implicit-def $x2
     ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-destroy LUI 2
-    ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-destroy ADDIW killed $x10, -2016
+    ; CHECK-RV64-NO-COM-NEXT: $x10 = frame-destroy ADDI killed $x10, -2016
     ; CHECK-RV64-NO-COM-NEXT: $x2 = frame-destroy ADD $x2, killed $x10
     ; CHECK-RV64-NO-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 2032
     ; CHECK-RV64-NO-COM-NEXT: $x1 = frame-destroy LD $x2, 2024 :: (load (s64) from %stack.1)
@@ -323,13 +323,13 @@ body:             |
     ; CHECK-RV64-COM-NEXT: frame-setup SD killed $x1, $x2, 488 :: (store (s64) into %stack.1)
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
     ; CHECK-RV64-COM-NEXT: $x10 = frame-setup LUI 2
-    ; CHECK-RV64-COM-NEXT: $x10 = frame-setup ADDIW killed $x10, -480
+    ; CHECK-RV64-COM-NEXT: $x10 = frame-setup ADDI killed $x10, -480
     ; CHECK-RV64-COM-NEXT: $x2 = frame-setup SUB $x2, killed $x10
     ; CHECK-RV64-COM-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8208
     ; CHECK-RV64-COM-NEXT: renamable $x10 = ADDI $x2, 8
     ; CHECK-RV64-COM-NEXT: PseudoCALL target-flags(riscv-call) @_Z6calleePi, csr_ilp32_lp64, implicit-def dead $x1, implicit killed $x10, implicit-def $x2
     ; CHECK-RV64-COM-NEXT: $x10 = frame-destroy LUI 2
-    ; CHECK-RV64-COM-NEXT: $x10 = frame-destroy ADDIW killed $x10, -480
+    ; CHECK-RV64-COM-NEXT: $x10 = frame-destroy ADDI killed $x10, -480
     ; CHECK-RV64-COM-NEXT: $x2 = frame-destroy ADD $x2, killed $x10
     ; CHECK-RV64-COM-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 496
     ; CHECK-RV64-COM-NEXT: $x1 = frame-destroy LD $x2, 488 :: (load (s64) from %stack.1)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-offset.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-offset.ll
index 3dc4fcfe26a8..79b937a1a292 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-offset.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-offset.ll
@@ -101,10 +101,10 @@ define void @test() {
 ; RV64I-NEXT:    addi a1, sp, 2047
 ; RV64I-NEXT:    addi a1, a1, 9
 ; RV64I-NEXT:    lui a2, 1
-; RV64I-NEXT:    addiw a2, a2, 8
+; RV64I-NEXT:    addi a2, a2, 8
 ; RV64I-NEXT:    add a2, sp, a2
 ; RV64I-NEXT:    lui a3, 1
-; RV64I-NEXT:    addiw a3, a3, 1032
+; RV64I-NEXT:    addi a3, a3, 1032
 ; RV64I-NEXT:    add a3, sp, a3
 ; RV64I-NEXT:    call inspect
 ; RV64I-NEXT:    addi sp, sp, 2032
@@ -225,7 +225,7 @@ define void @align_8() {
 ; RV64I-NEXT:    .cfi_def_cfa_offset 4128
 ; RV64I-NEXT:    addi a0, sp, 15
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 16
 ; RV64I-NEXT:    add a1, sp, a1
 ; RV64I-NEXT:    call inspect
 ; RV64I-NEXT:    addi sp, sp, 2032
@@ -340,7 +340,7 @@ define void @align_4() {
 ; RV64I-NEXT:    .cfi_def_cfa_offset 4128
 ; RV64I-NEXT:    addi a0, sp, 19
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 20
+; RV64I-NEXT:    addi a1, a1, 20
 ; RV64I-NEXT:    add a1, sp, a1
 ; RV64I-NEXT:    call inspect
 ; RV64I-NEXT:    addi sp, sp, 2032
@@ -433,7 +433,7 @@ define void @align_2() {
 ; RV64-NEXT:    .cfi_def_cfa_offset 4128
 ; RV64-NEXT:    addi a0, sp, 21
 ; RV64-NEXT:    lui a1, 1
-; RV64-NEXT:    addiw a1, a1, 22
+; RV64-NEXT:    addi a1, a1, 22
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    call inspect
 ; RV64-NEXT:    addi sp, sp, 2032
@@ -505,7 +505,7 @@ define void @align_1() {
 ; RV64-NEXT:    .cfi_def_cfa_offset 4128
 ; RV64-NEXT:    addi a0, sp, 22
 ; RV64-NEXT:    lui a1, 1
-; RV64-NEXT:    addiw a1, a1, 23
+; RV64-NEXT:    addi a1, a1, 23
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    call inspect
 ; RV64-NEXT:    addi sp, sp, 2032
@@ -573,16 +573,16 @@ define void @align_1_lui() {
 ; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 6144
 ; RV64I-NEXT:    addi a0, sp, 20
 ; RV64I-NEXT:    lui a1, 1
-; RV64I-NEXT:    addiw a1, a1, 2039
+; RV64I-NEXT:    addi a1, a1, 2039
 ; RV64I-NEXT:    add a1, sp, a1
 ; RV64I-NEXT:    call inspect
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    addi a0, a0, 16
 ; RV64I-NEXT:    add sp, sp, a0
 ; RV64I-NEXT:    .cfi_def_cfa_offset 2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
@@ -602,7 +602,7 @@ define void @align_1_lui() {
 ; RV64ZBA-NEXT:    .cfi_def_cfa_offset 6144
 ; RV64ZBA-NEXT:    addi a0, sp, 20
 ; RV64ZBA-NEXT:    lui a1, 1
-; RV64ZBA-NEXT:    addiw a1, a1, 2039
+; RV64ZBA-NEXT:    addi a1, a1, 2039
 ; RV64ZBA-NEXT:    add a1, sp, a1
 ; RV64ZBA-NEXT:    call inspect
 ; RV64ZBA-NEXT:    li a0, 514
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-realignment.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-realignment.ll
index 368840933f04..55e1c62c6670 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/stack-realignment.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/stack-realignment.ll
@@ -1340,7 +1340,7 @@ define void @caller4096() {
 ; RV64I-NEXT:    addi s0, sp, 2032
 ; RV64I-NEXT:    .cfi_def_cfa s0, 0
 ; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    addiw a0, a0, -2032
+; RV64I-NEXT:    addi a0, a0, -2032
 ; RV64I-NEXT:    sub sp, sp, a0
 ; RV64I-NEXT:    srli a0, sp, 12
 ; RV64I-NEXT:    slli sp, a0, 12
@@ -1368,7 +1368,7 @@ define void @caller4096() {
 ; RV64I-LP64E-NEXT:    addi s0, sp, 2040
 ; RV64I-LP64E-NEXT:    .cfi_def_cfa s0, 0
 ; RV64I-LP64E-NEXT:    lui a0, 2
-; RV64I-LP64E-NEXT:    addiw a0, a0, -2040
+; RV64I-LP64E-NEXT:    addi a0, a0, -2040
 ; RV64I-LP64E-NEXT:    sub sp, sp, a0
 ; RV64I-LP64E-NEXT:    srli a0, sp, 12
 ; RV64I-LP64E-NEXT:    slli sp, a0, 12
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/switch-width.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/switch-width.ll
index d902bd3276a3..c570e737780d 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/switch-width.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/switch-width.ll
@@ -119,7 +119,7 @@ define i32 @trunc_i17(i64 %a)  {
 ; CHECK-LABEL: trunc_i17:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lui a1, 32
-; CHECK-NEXT:    addiw a1, a1, -1
+; CHECK-NEXT:    addi a1, a1, -1
 ; CHECK-NEXT:    and a0, a0, a1
 ; CHECK-NEXT:    beq a0, a1, .LBB3_3
 ; CHECK-NEXT:  # %bb.1: # %entry
@@ -159,7 +159,7 @@ define i32 @trunc_i16(i64 %a)  {
 ; CHECK-LABEL: trunc_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lui a1, 16
-; CHECK-NEXT:    addiw a1, a1, -1
+; CHECK-NEXT:    addi a1, a1, -1
 ; CHECK-NEXT:    and a0, a0, a1
 ; CHECK-NEXT:    beq a0, a1, .LBB4_3
 ; CHECK-NEXT:  # %bb.1: # %entry
@@ -200,7 +200,7 @@ define i32 @trunc_i12(i64 %a)  {
 ; CHECK-LABEL: trunc_i12:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lui a1, 1
-; CHECK-NEXT:    addiw a1, a1, -1
+; CHECK-NEXT:    addi a1, a1, -1
 ; CHECK-NEXT:    and a0, a0, a1
 ; CHECK-NEXT:    beq a0, a1, .LBB5_3
 ; CHECK-NEXT:  # %bb.1: # %entry
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/trunc-nsw-nuw.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/trunc-nsw-nuw.ll
index 9f81ff8c8d31..8fb69693adb7 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/trunc-nsw-nuw.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/trunc-nsw-nuw.ll
@@ -17,7 +17,7 @@ define signext i32 @trunc_nuw_nsw_urem(i64 %x) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lui a1, 210
 ; CHECK-NEXT:    lui a2, 2
-; CHECK-NEXT:    addiw a1, a1, -1167
+; CHECK-NEXT:    addi a1, a1, -1167
 ; CHECK-NEXT:    slli a1, a1, 12
 ; CHECK-NEXT:    addi a1, a1, 1881
 ; CHECK-NEXT:    mul a1, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat.ll
index dbcb68eb3c9e..ee591a178463 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat.ll
@@ -109,7 +109,7 @@ define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    addi a1, a1, -1
 ; RV64I-NEXT:    bltu a0, a1, .LBB2_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
@@ -128,7 +128,7 @@ define zeroext i16 @func16(i16 zeroext %x, i16 zeroext %y) nounwind {
 ; RV64IZbb:       # %bb.0:
 ; RV64IZbb-NEXT:    add a0, a0, a1
 ; RV64IZbb-NEXT:    lui a1, 16
-; RV64IZbb-NEXT:    addiw a1, a1, -1
+; RV64IZbb-NEXT:    addi a1, a1, -1
 ; RV64IZbb-NEXT:    minu a0, a0, a1
 ; RV64IZbb-NEXT:    ret
   %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %y);
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
index 65b3d763fd72..da29d26b7147 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
@@ -120,7 +120,7 @@ define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a3, 16
 ; RV64I-NEXT:    mul a2, a1, a2
-; RV64I-NEXT:    addiw a1, a3, -1
+; RV64I-NEXT:    addi a1, a3, -1
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    and a2, a2, a1
 ; RV64I-NEXT:    add a0, a0, a2
@@ -148,7 +148,7 @@ define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
 ; RV64IZbb-NEXT:    lui a2, 16
 ; RV64IZbb-NEXT:    zext.h a1, a1
 ; RV64IZbb-NEXT:    add a0, a0, a1
-; RV64IZbb-NEXT:    addiw a2, a2, -1
+; RV64IZbb-NEXT:    addi a2, a2, -1
 ; RV64IZbb-NEXT:    minu a0, a0, a2
 ; RV64IZbb-NEXT:    ret
   %a = mul i16 %y, %z
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/ucmp.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/ucmp.ll
index 50da56fbc595..0a400b1c04a3 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/ucmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/ucmp.ll
@@ -89,8 +89,8 @@ define i8 @ucmp.8.128(i128 %x, i128 %y) nounwind {
 ; RV32I-NEXT:    lw a2, 4(a1)
 ; RV32I-NEXT:    lw a4, 8(a1)
 ; RV32I-NEXT:    lw a5, 12(a1)
-; RV32I-NEXT:    lw a6, 12(a0)
 ; RV32I-NEXT:    lw a3, 4(a0)
+; RV32I-NEXT:    lw a6, 12(a0)
 ; RV32I-NEXT:    lw a7, 8(a0)
 ; RV32I-NEXT:    beq a6, a5, .LBB4_2
 ; RV32I-NEXT:  # %bb.1:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/unaligned-load-store.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
index 068b9f762002..c9c49e8f7f53 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
@@ -133,49 +133,49 @@ define i64 @load_i64(ptr %p) {
 ; RV32I-LABEL: load_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a1, 1(a0)
-; RV32I-NEXT:    lbu a2, 0(a0)
-; RV32I-NEXT:    lbu a3, 2(a0)
-; RV32I-NEXT:    lbu a4, 3(a0)
+; RV32I-NEXT:    lbu a2, 2(a0)
+; RV32I-NEXT:    lbu a3, 3(a0)
+; RV32I-NEXT:    lbu a4, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 8
-; RV32I-NEXT:    or a1, a1, a2
-; RV32I-NEXT:    lbu a2, 4(a0)
-; RV32I-NEXT:    lbu a5, 5(a0)
-; RV32I-NEXT:    slli a3, a3, 16
-; RV32I-NEXT:    slli a4, a4, 24
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    lbu a4, 6(a0)
+; RV32I-NEXT:    slli a2, a2, 16
+; RV32I-NEXT:    slli a3, a3, 24
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a2, a3, a2
+; RV32I-NEXT:    lbu a3, 5(a0)
+; RV32I-NEXT:    lbu a4, 4(a0)
+; RV32I-NEXT:    lbu a5, 6(a0)
 ; RV32I-NEXT:    lbu a0, 7(a0)
-; RV32I-NEXT:    slli a5, a5, 8
-; RV32I-NEXT:    or a2, a5, a2
-; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    slli a5, a5, 16
 ; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or a4, a0, a4
-; RV32I-NEXT:    or a0, a3, a1
-; RV32I-NEXT:    or a1, a4, a2
+; RV32I-NEXT:    or a5, a0, a5
+; RV32I-NEXT:    or a0, a2, a1
+; RV32I-NEXT:    or a1, a5, a3
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: load_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lbu a1, 1(a0)
-; RV64I-NEXT:    lbu a2, 0(a0)
-; RV64I-NEXT:    lbu a3, 2(a0)
-; RV64I-NEXT:    lbu a4, 3(a0)
+; RV64I-NEXT:    lbu a2, 2(a0)
+; RV64I-NEXT:    lbu a3, 3(a0)
+; RV64I-NEXT:    lbu a4, 0(a0)
 ; RV64I-NEXT:    slli a1, a1, 8
-; RV64I-NEXT:    or a1, a1, a2
-; RV64I-NEXT:    lbu a2, 4(a0)
-; RV64I-NEXT:    lbu a5, 5(a0)
-; RV64I-NEXT:    slli a3, a3, 16
-; RV64I-NEXT:    slli a4, a4, 24
-; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    lbu a4, 6(a0)
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    slli a3, a3, 24
+; RV64I-NEXT:    or a1, a1, a4
+; RV64I-NEXT:    or a2, a3, a2
+; RV64I-NEXT:    lbu a3, 5(a0)
+; RV64I-NEXT:    lbu a4, 4(a0)
+; RV64I-NEXT:    lbu a5, 6(a0)
 ; RV64I-NEXT:    lbu a0, 7(a0)
-; RV64I-NEXT:    slli a5, a5, 8
-; RV64I-NEXT:    or a2, a5, a2
-; RV64I-NEXT:    slli a4, a4, 16
+; RV64I-NEXT:    slli a3, a3, 8
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    slli a5, a5, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, a4
-; RV64I-NEXT:    or a1, a3, a1
-; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/unfold-masked-merge-scalar-variablemask.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/unfold-masked-merge-scalar-variablemask.ll
index 1517e524a7f7..efc8243df71e 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/unfold-masked-merge-scalar-variablemask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/unfold-masked-merge-scalar-variablemask.ll
@@ -8,16 +8,13 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+zbb < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK,CHECK-ZBB,RV64,RV64ZBB
 
-; TODO: Should we convert these to X ^ ((X ^ Y) & M) form when Zbb isn't
-; present?
 
 define i8 @out8(i8 %x, i8 %y, i8 %mask) {
 ; CHECK-I-LABEL: out8:
 ; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    xor a0, a0, a1
 ; CHECK-I-NEXT:    and a0, a0, a2
-; CHECK-I-NEXT:    not a2, a2
-; CHECK-I-NEXT:    and a1, a1, a2
-; CHECK-I-NEXT:    or a0, a0, a1
+; CHECK-I-NEXT:    xor a0, a0, a1
 ; CHECK-I-NEXT:    ret
 ;
 ; CHECK-ZBB-LABEL: out8:
@@ -36,10 +33,9 @@ define i8 @out8(i8 %x, i8 %y, i8 %mask) {
 define i16 @out16(i16 %x, i16 %y, i16 %mask) {
 ; CHECK-I-LABEL: out16:
 ; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    xor a0, a0, a1
 ; CHECK-I-NEXT:    and a0, a0, a2
-; CHECK-I-NEXT:    not a2, a2
-; CHECK-I-NEXT:    and a1, a1, a2
-; CHECK-I-NEXT:    or a0, a0, a1
+; CHECK-I-NEXT:    xor a0, a0, a1
 ; CHECK-I-NEXT:    ret
 ;
 ; CHECK-ZBB-LABEL: out16:
@@ -58,10 +54,9 @@ define i16 @out16(i16 %x, i16 %y, i16 %mask) {
 define i32 @out32(i32 %x, i32 %y, i32 %mask) {
 ; CHECK-I-LABEL: out32:
 ; CHECK-I:       # %bb.0:
+; CHECK-I-NEXT:    xor a0, a0, a1
 ; CHECK-I-NEXT:    and a0, a0, a2
-; CHECK-I-NEXT:    not a2, a2
-; CHECK-I-NEXT:    and a1, a1, a2
-; CHECK-I-NEXT:    or a0, a0, a1
+; CHECK-I-NEXT:    xor a0, a0, a1
 ; CHECK-I-NEXT:    ret
 ;
 ; CHECK-ZBB-LABEL: out32:
@@ -80,22 +75,19 @@ define i32 @out32(i32 %x, i32 %y, i32 %mask) {
 define i64 @out64(i64 %x, i64 %y, i64 %mask) {
 ; RV32I-LABEL: out64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    and a1, a1, a5
+; RV32I-NEXT:    xor a0, a0, a2
+; RV32I-NEXT:    xor a1, a1, a3
 ; RV32I-NEXT:    and a0, a0, a4
-; RV32I-NEXT:    not a4, a4
-; RV32I-NEXT:    not a5, a5
-; RV32I-NEXT:    and a3, a3, a5
-; RV32I-NEXT:    and a2, a2, a4
-; RV32I-NEXT:    or a0, a0, a2
-; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    and a1, a1, a5
+; RV32I-NEXT:    xor a0, a0, a2
+; RV32I-NEXT:    xor a1, a1, a3
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: out64:
 ; RV64I:       # %bb.0:
+; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    not a2, a2
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: out64:
@@ -660,10 +652,9 @@ define i32 @in_constant_varx_mone_invmask(i32 %x, i32 %y, i32 %mask) {
 define i32 @out_constant_varx_42(i32 %x, i32 %y, i32 %mask) {
 ; CHECK-I-LABEL: out_constant_varx_42:
 ; CHECK-I:       # %bb.0:
-; CHECK-I-NEXT:    not a1, a2
-; CHECK-I-NEXT:    and a0, a2, a0
-; CHECK-I-NEXT:    andi a1, a1, 42
-; CHECK-I-NEXT:    or a0, a0, a1
+; CHECK-I-NEXT:    xori a0, a0, 42
+; CHECK-I-NEXT:    and a0, a0, a2
+; CHECK-I-NEXT:    xori a0, a0, 42
 ; CHECK-I-NEXT:    ret
 ;
 ; CHECK-ZBB-LABEL: out_constant_varx_42:
@@ -704,10 +695,9 @@ define i32 @in_constant_varx_42(i32 %x, i32 %y, i32 %mask) {
 define i32 @out_constant_varx_42_invmask(i32 %x, i32 %y, i32 %mask) {
 ; CHECK-I-LABEL: out_constant_varx_42_invmask:
 ; CHECK-I:       # %bb.0:
-; CHECK-I-NEXT:    not a1, a2
-; CHECK-I-NEXT:    and a0, a1, a0
-; CHECK-I-NEXT:    andi a1, a2, 42
-; CHECK-I-NEXT:    or a0, a0, a1
+; CHECK-I-NEXT:    xori a1, a0, 42
+; CHECK-I-NEXT:    and a1, a1, a2
+; CHECK-I-NEXT:    xor a0, a1, a0
 ; CHECK-I-NEXT:    ret
 ;
 ; CHECK-ZBB-LABEL: out_constant_varx_42_invmask:
@@ -812,10 +802,9 @@ define i32 @in_constant_mone_vary_invmask(i32 %x, i32 %y, i32 %mask) {
 define i32 @out_constant_42_vary(i32 %x, i32 %y, i32 %mask) {
 ; CHECK-I-LABEL: out_constant_42_vary:
 ; CHECK-I:       # %bb.0:
-; CHECK-I-NEXT:    not a0, a2
-; CHECK-I-NEXT:    andi a2, a2, 42
-; CHECK-I-NEXT:    and a0, a0, a1
-; CHECK-I-NEXT:    or a0, a2, a0
+; CHECK-I-NEXT:    xori a0, a1, 42
+; CHECK-I-NEXT:    and a0, a0, a2
+; CHECK-I-NEXT:    xor a0, a0, a1
 ; CHECK-I-NEXT:    ret
 ;
 ; CHECK-ZBB-LABEL: out_constant_42_vary:
@@ -855,10 +844,9 @@ define i32 @in_constant_42_vary(i32 %x, i32 %y, i32 %mask) {
 define i32 @out_constant_42_vary_invmask(i32 %x, i32 %y, i32 %mask) {
 ; CHECK-I-LABEL: out_constant_42_vary_invmask:
 ; CHECK-I:       # %bb.0:
-; CHECK-I-NEXT:    not a0, a2
-; CHECK-I-NEXT:    andi a0, a0, 42
-; CHECK-I-NEXT:    and a1, a2, a1
-; CHECK-I-NEXT:    or a0, a0, a1
+; CHECK-I-NEXT:    xori a0, a1, 42
+; CHECK-I-NEXT:    and a0, a0, a2
+; CHECK-I-NEXT:    xori a0, a0, 42
 ; CHECK-I-NEXT:    ret
 ;
 ; CHECK-ZBB-LABEL: out_constant_42_vary_invmask:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
index 46e250710f9c..d33c6662ceb5 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
@@ -148,7 +148,7 @@ define i1 @test_urem_even(i27 %X) nounwind {
 ; RV64-NEXT:    or a0, a0, a1
 ; RV64-NEXT:    slli a0, a0, 37
 ; RV64-NEXT:    srli a0, a0, 37
-; RV64-NEXT:    addiw a1, a2, -1755
+; RV64-NEXT:    addi a1, a2, -1755
 ; RV64-NEXT:    sltu a0, a0, a1
 ; RV64-NEXT:    ret
 ;
@@ -180,7 +180,7 @@ define i1 @test_urem_even(i27 %X) nounwind {
 ; RV64M-NEXT:    lui a1, 2341
 ; RV64M-NEXT:    slli a0, a0, 37
 ; RV64M-NEXT:    srli a0, a0, 37
-; RV64M-NEXT:    addiw a1, a1, -1755
+; RV64M-NEXT:    addi a1, a1, -1755
 ; RV64M-NEXT:    sltu a0, a0, a1
 ; RV64M-NEXT:    ret
 ;
@@ -212,7 +212,7 @@ define i1 @test_urem_even(i27 %X) nounwind {
 ; RV64MV-NEXT:    lui a1, 2341
 ; RV64MV-NEXT:    slli a0, a0, 37
 ; RV64MV-NEXT:    srli a0, a0, 37
-; RV64MV-NEXT:    addiw a1, a1, -1755
+; RV64MV-NEXT:    addi a1, a1, -1755
 ; RV64MV-NEXT:    sltu a0, a0, a1
 ; RV64MV-NEXT:    ret
   %urem = urem i27 %X, 14
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
index 988856ca7092..3ef9f3f94510 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
@@ -489,33 +489,33 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ;
 ; RV64IM-LABEL: combine_urem_udiv:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    lhu a2, 16(a1)
-; RV64IM-NEXT:    lhu a3, 24(a1)
-; RV64IM-NEXT:    lui a4, %hi(.LCPI2_0)
-; RV64IM-NEXT:    ld a4, %lo(.LCPI2_0)(a4)
+; RV64IM-NEXT:    lui a2, %hi(.LCPI2_0)
+; RV64IM-NEXT:    ld a2, %lo(.LCPI2_0)(a2)
+; RV64IM-NEXT:    lhu a3, 16(a1)
+; RV64IM-NEXT:    lhu a4, 24(a1)
 ; RV64IM-NEXT:    lhu a5, 0(a1)
 ; RV64IM-NEXT:    lhu a1, 8(a1)
 ; RV64IM-NEXT:    li a6, 95
-; RV64IM-NEXT:    mulhu a7, a3, a4
-; RV64IM-NEXT:    mulhu t0, a2, a4
-; RV64IM-NEXT:    mulhu t1, a1, a4
-; RV64IM-NEXT:    mulhu a4, a5, a4
+; RV64IM-NEXT:    mulhu a7, a4, a2
+; RV64IM-NEXT:    mulhu t0, a3, a2
+; RV64IM-NEXT:    mulhu t1, a1, a2
+; RV64IM-NEXT:    mulhu a2, a5, a2
 ; RV64IM-NEXT:    mul t2, a7, a6
 ; RV64IM-NEXT:    mul t3, t0, a6
 ; RV64IM-NEXT:    mul t4, t1, a6
-; RV64IM-NEXT:    mul a6, a4, a6
-; RV64IM-NEXT:    add a4, a5, a4
+; RV64IM-NEXT:    mul a6, a2, a6
+; RV64IM-NEXT:    add a2, a5, a2
 ; RV64IM-NEXT:    add a1, a1, t1
-; RV64IM-NEXT:    add a2, a2, t0
-; RV64IM-NEXT:    add a3, a3, a7
-; RV64IM-NEXT:    subw a4, a4, a6
+; RV64IM-NEXT:    add a3, a3, t0
+; RV64IM-NEXT:    add a4, a4, a7
+; RV64IM-NEXT:    subw a2, a2, a6
 ; RV64IM-NEXT:    subw a1, a1, t4
-; RV64IM-NEXT:    subw a2, a2, t3
-; RV64IM-NEXT:    subw a3, a3, t2
-; RV64IM-NEXT:    sh a4, 0(a0)
+; RV64IM-NEXT:    subw a3, a3, t3
+; RV64IM-NEXT:    subw a4, a4, t2
+; RV64IM-NEXT:    sh a2, 0(a0)
 ; RV64IM-NEXT:    sh a1, 2(a0)
-; RV64IM-NEXT:    sh a2, 4(a0)
-; RV64IM-NEXT:    sh a3, 6(a0)
+; RV64IM-NEXT:    sh a3, 4(a0)
+; RV64IM-NEXT:    sh a4, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
   %2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
@@ -721,7 +721,7 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    call __umoddi3
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    addi a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __umoddi3
 ; RV64I-NEXT:    sh zero, 0(s2)
@@ -941,7 +941,7 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV64I-NEXT:    call __umoddi3
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    addiw a1, a0, 1327
+; RV64I-NEXT:    addi a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __umoddi3
 ; RV64I-NEXT:    sd zero, 0(s2)
@@ -973,7 +973,7 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV64IM-NEXT:    srli a4, a4, 7
 ; RV64IM-NEXT:    mul a4, a4, a6
 ; RV64IM-NEXT:    lui a6, 1
-; RV64IM-NEXT:    addiw a6, a6, 1327
+; RV64IM-NEXT:    addi a6, a6, 1327
 ; RV64IM-NEXT:    mulhu a5, a3, a5
 ; RV64IM-NEXT:    mulhu a7, a1, a7
 ; RV64IM-NEXT:    srli a7, a7, 12
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/usub_sat_plus.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/usub_sat_plus.ll
index aa42568c539b..3285349ea068 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/usub_sat_plus.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/usub_sat_plus.ll
@@ -122,7 +122,7 @@ define i16 @func16(i16 %x, i16 %y, i16 %z) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a3, 16
 ; RV64I-NEXT:    mul a1, a1, a2
-; RV64I-NEXT:    addiw a3, a3, -1
+; RV64I-NEXT:    addi a3, a3, -1
 ; RV64I-NEXT:    and a0, a0, a3
 ; RV64I-NEXT:    and a1, a1, a3
 ; RV64I-NEXT:    sub a1, a0, a1
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/vararg.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/vararg.ll
index de7b25640184..f600c2ed2643 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/vararg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/vararg.ll
@@ -2520,7 +2520,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld t4, %lo(.LCPI11_0)(a2)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a2, %lo(.LCPI11_1)(a3)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a3, %lo(.LCPI11_2)(a6)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a6, t3, 761
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a6, t3, 761
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a6, a6, 11
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t4, 0(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t2, 8(sp)
@@ -2552,7 +2552,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld t4, %lo(.LCPI11_0)(a2)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a2, %lo(.LCPI11_1)(a3)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a3, %lo(.LCPI11_2)(a6)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a6, t3, 761
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a6, t3, 761
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a6, a6, 11
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t4, 0(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t2, 8(sp)
@@ -2583,7 +2583,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64E-FPELIM-NEXT:    sd a2, 40(sp)
 ; LP64E-FPELIM-NEXT:    li a5, 13
 ; LP64E-FPELIM-NEXT:    ld a7, %lo(.LCPI11_0)(a7)
-; LP64E-FPELIM-NEXT:    addiw t1, t1, 761
+; LP64E-FPELIM-NEXT:    addi t1, t1, 761
 ; LP64E-FPELIM-NEXT:    ld a2, %lo(.LCPI11_1)(t2)
 ; LP64E-FPELIM-NEXT:    ld a3, %lo(.LCPI11_2)(t3)
 ; LP64E-FPELIM-NEXT:    slli t1, t1, 11
@@ -2617,7 +2617,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64E-WITHFP-NEXT:    sd a2, 40(sp)
 ; LP64E-WITHFP-NEXT:    li a5, 13
 ; LP64E-WITHFP-NEXT:    ld a7, %lo(.LCPI11_0)(a7)
-; LP64E-WITHFP-NEXT:    addiw t1, t1, 761
+; LP64E-WITHFP-NEXT:    addi t1, t1, 761
 ; LP64E-WITHFP-NEXT:    ld a2, %lo(.LCPI11_1)(t2)
 ; LP64E-WITHFP-NEXT:    ld a3, %lo(.LCPI11_2)(t3)
 ; LP64E-WITHFP-NEXT:    slli t1, t1, 11
@@ -2989,14 +2989,14 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va_large_stack:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 336
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 336
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sub sp, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    .cfi_def_cfa_offset 100000080
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 280(a0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 284
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 284
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
@@ -3021,7 +3021,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, sp, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 304(a1)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a1, a1, 336
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 336
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add sp, sp, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    .cfi_def_cfa_offset 0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
@@ -3037,7 +3037,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 1968
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa s0, 64
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 24414
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, -1680
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, -1680
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sub sp, sp, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 12
@@ -3052,7 +3052,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 24414
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a1, a1, -1680
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, -1680
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add sp, sp, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa sp, 2032
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 1960(sp) # 8-byte Folded Reload
@@ -3066,14 +3066,14 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64E-FPELIM-LABEL: va_large_stack:
 ; LP64E-FPELIM:       # %bb.0:
 ; LP64E-FPELIM-NEXT:    lui a0, 24414
-; LP64E-FPELIM-NEXT:    addiw a0, a0, 320
+; LP64E-FPELIM-NEXT:    addi a0, a0, 320
 ; LP64E-FPELIM-NEXT:    sub sp, sp, a0
 ; LP64E-FPELIM-NEXT:    .cfi_def_cfa_offset 100000064
 ; LP64E-FPELIM-NEXT:    lui a0, 24414
 ; LP64E-FPELIM-NEXT:    add a0, sp, a0
 ; LP64E-FPELIM-NEXT:    sd a1, 280(a0)
 ; LP64E-FPELIM-NEXT:    lui a0, 24414
-; LP64E-FPELIM-NEXT:    addiw a0, a0, 284
+; LP64E-FPELIM-NEXT:    addi a0, a0, 284
 ; LP64E-FPELIM-NEXT:    add a0, sp, a0
 ; LP64E-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64E-FPELIM-NEXT:    lui a0, 24414
@@ -3092,7 +3092,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64E-FPELIM-NEXT:    add a1, sp, a1
 ; LP64E-FPELIM-NEXT:    sd a4, 304(a1)
 ; LP64E-FPELIM-NEXT:    lui a1, 24414
-; LP64E-FPELIM-NEXT:    addiw a1, a1, 320
+; LP64E-FPELIM-NEXT:    addi a1, a1, 320
 ; LP64E-FPELIM-NEXT:    add sp, sp, a1
 ; LP64E-FPELIM-NEXT:    .cfi_def_cfa_offset 0
 ; LP64E-FPELIM-NEXT:    ret
@@ -3108,7 +3108,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 1992
 ; LP64E-WITHFP-NEXT:    .cfi_def_cfa s0, 48
 ; LP64E-WITHFP-NEXT:    lui a0, 24414
-; LP64E-WITHFP-NEXT:    addiw a0, a0, -1704
+; LP64E-WITHFP-NEXT:    addi a0, a0, -1704
 ; LP64E-WITHFP-NEXT:    sub sp, sp, a0
 ; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64E-WITHFP-NEXT:    addi a0, s0, 12
@@ -3121,7 +3121,7 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    lui a1, 24414
-; LP64E-WITHFP-NEXT:    addiw a1, a1, -1704
+; LP64E-WITHFP-NEXT:    addi a1, a1, -1704
 ; LP64E-WITHFP-NEXT:    add sp, sp, a1
 ; LP64E-WITHFP-NEXT:    .cfi_def_cfa sp, 2040
 ; LP64E-WITHFP-NEXT:    ld ra, 1984(sp) # 8-byte Folded Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/varargs-with-fp-and-second-adj.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/varargs-with-fp-and-second-adj.ll
index 73d3827b8475..c8c364208da9 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/varargs-with-fp-and-second-adj.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/varargs-with-fp-and-second-adj.ll
@@ -16,7 +16,7 @@ define dso_local void @_Z3fooPKcz(ptr noundef %0, ...) "frame-pointer"="all" {
 ; RV64V-NEXT:    addi s0, sp, 432
 ; RV64V-NEXT:    .cfi_def_cfa s0, 64
 ; RV64V-NEXT:    lui t0, 2
-; RV64V-NEXT:    addiw t0, t0, -576
+; RV64V-NEXT:    addi t0, t0, -576
 ; RV64V-NEXT:    sub sp, sp, t0
 ; RV64V-NEXT:    sd a5, 40(s0)
 ; RV64V-NEXT:    sd a6, 48(s0)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll
index 2a0228f95f1c..09b2eeb19a69 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -30,25 +30,25 @@ define void @lshr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: lshr_4bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a6, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or a0, a0, a5
-; RV32I-NEXT:    lbu a5, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a3, 1(a1)
+; RV32I-NEXT:    lbu a5, 0(a1)
+; RV32I-NEXT:    lbu a6, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a5
-; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    srl a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
@@ -94,25 +94,25 @@ define void @shl_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: shl_4bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a6, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or a0, a0, a5
-; RV32I-NEXT:    lbu a5, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a3, 1(a1)
+; RV32I-NEXT:    lbu a5, 0(a1)
+; RV32I-NEXT:    lbu a6, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a5
-; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    sll a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
@@ -158,25 +158,25 @@ define void @ashr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: ashr_4bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a6, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or a0, a0, a5
-; RV32I-NEXT:    lbu a5, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a3, 1(a1)
+; RV32I-NEXT:    lbu a5, 0(a1)
+; RV32I-NEXT:    lbu a6, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a5
-; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    sra a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
@@ -220,24 +220,24 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t1, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or t0, t2, t0
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t2, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t1, 0(a1)
 ; RV64I-NEXT:    lbu t2, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
 ; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t2
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a0, a0, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    slli a4, a4, 35
@@ -264,26 +264,26 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: lshr_8bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 5(a0)
-; RV32I-NEXT:    lbu a4, 4(a0)
-; RV32I-NEXT:    lbu a5, 6(a0)
-; RV32I-NEXT:    lbu a6, 7(a0)
+; RV32I-NEXT:    lbu a4, 6(a0)
+; RV32I-NEXT:    lbu a5, 7(a0)
+; RV32I-NEXT:    lbu a6, 4(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a7, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a3, a3, a6
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a5, 1(a1)
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    slli a6, a6, 16
+; RV32I-NEXT:    slli a5, a5, 8
+; RV32I-NEXT:    or a6, a5, a6
+; RV32I-NEXT:    slli a7, a7, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    or a5, a5, a3
-; RV32I-NEXT:    or a1, a1, a4
-; RV32I-NEXT:    slli a4, a1, 3
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    or a5, a4, a3
+; RV32I-NEXT:    or a4, a1, a6
+; RV32I-NEXT:    slli a4, a4, 3
 ; RV32I-NEXT:    addi a3, a4, -32
 ; RV32I-NEXT:    srl a1, a5, a4
 ; RV32I-NEXT:    bltz a3, .LBB3_2
@@ -356,24 +356,24 @@ define void @shl_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t1, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or t0, t2, t0
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t2, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t1, 0(a1)
 ; RV64I-NEXT:    lbu t2, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
 ; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t2
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a0, a0, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    slli a4, a4, 35
@@ -400,26 +400,26 @@ define void @shl_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: shl_8bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a6, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a6, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a7, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a3, a3, a6
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a5, 1(a1)
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    slli a6, a6, 16
+; RV32I-NEXT:    slli a5, a5, 8
+; RV32I-NEXT:    or a6, a5, a6
+; RV32I-NEXT:    slli a7, a7, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    or a5, a5, a3
-; RV32I-NEXT:    or a1, a1, a4
-; RV32I-NEXT:    slli a4, a1, 3
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    or a5, a4, a3
+; RV32I-NEXT:    or a4, a1, a6
+; RV32I-NEXT:    slli a4, a4, 3
 ; RV32I-NEXT:    addi a3, a4, -32
 ; RV32I-NEXT:    sll a1, a5, a4
 ; RV32I-NEXT:    bltz a3, .LBB4_2
@@ -492,24 +492,24 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t1, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or t0, t2, t0
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t2, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t1, 0(a1)
 ; RV64I-NEXT:    lbu t2, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
 ; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t2
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a0, a0, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    slli a4, a4, 35
@@ -535,21 +535,21 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ;
 ; RV32I-LABEL: ashr_8bytes:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lbu a3, 4(a0)
-; RV32I-NEXT:    lbu a4, 5(a0)
+; RV32I-NEXT:    lbu a3, 5(a0)
+; RV32I-NEXT:    lbu a4, 4(a0)
 ; RV32I-NEXT:    lbu a5, 6(a0)
 ; RV32I-NEXT:    lbu a6, 7(a0)
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    lbu a4, 1(a1)
 ; RV32I-NEXT:    lbu a7, 0(a1)
-; RV32I-NEXT:    lbu t0, 1(a1)
-; RV32I-NEXT:    slli a4, a4, 8
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    lbu a4, 2(a1)
+; RV32I-NEXT:    lbu t0, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    or a7, t0, a7
-; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a7, a4, a7
+; RV32I-NEXT:    slli t0, t0, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, t0
 ; RV32I-NEXT:    slli a4, a5, 16
 ; RV32I-NEXT:    slli a5, a6, 24
 ; RV32I-NEXT:    or a4, a5, a4
@@ -629,24 +629,24 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t2, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t2, 0(a1)
 ; RV64I-NEXT:    lbu t3, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, t1, a5
-; RV64I-NEXT:    or a5, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a5, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    slli a6, a5, 35
@@ -660,31 +660,31 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    j .LBB6_3
 ; RV64I-NEXT:  .LBB6_2:
 ; RV64I-NEXT:    lbu a6, 1(a0)
-; RV64I-NEXT:    lbu a7, 0(a0)
-; RV64I-NEXT:    lbu t0, 2(a0)
-; RV64I-NEXT:    lbu t1, 3(a0)
+; RV64I-NEXT:    lbu a7, 2(a0)
+; RV64I-NEXT:    lbu t0, 3(a0)
+; RV64I-NEXT:    lbu t1, 0(a0)
 ; RV64I-NEXT:    slli a6, a6, 8
-; RV64I-NEXT:    or a6, a6, a7
-; RV64I-NEXT:    lbu a7, 4(a0)
-; RV64I-NEXT:    lbu t2, 5(a0)
-; RV64I-NEXT:    slli t0, t0, 16
-; RV64I-NEXT:    slli t1, t1, 24
-; RV64I-NEXT:    or t0, t1, t0
-; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t0, t0, 24
+; RV64I-NEXT:    or a6, a6, t1
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 4(a0)
+; RV64I-NEXT:    lbu t2, 6(a0)
 ; RV64I-NEXT:    lbu a0, 7(a0)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
-; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    not t0, a4
+; RV64I-NEXT:    or a0, a0, t2
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    not a7, a4
 ; RV64I-NEXT:    slli a5, a5, 1
-; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a0, a0, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    srl a0, a0, a4
-; RV64I-NEXT:    sll a4, a5, t0
+; RV64I-NEXT:    sll a4, a5, a7
 ; RV64I-NEXT:    or a0, a0, a4
 ; RV64I-NEXT:  .LBB6_3:
 ; RV64I-NEXT:    srai a3, a3, 63
@@ -787,27 +787,27 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    andi a1, a1, 12
 ; RV32I-NEXT:    add a1, t2, a1
 ; RV32I-NEXT:    andi a3, a0, 24
-; RV32I-NEXT:    lw a4, 0(a1)
-; RV32I-NEXT:    lw a5, 4(a1)
-; RV32I-NEXT:    lw a6, 8(a1)
 ; RV32I-NEXT:    xori a3, a3, 31
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a6, 0(a1)
 ; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    srl a7, a5, a0
-; RV32I-NEXT:    slli t0, a6, 1
-; RV32I-NEXT:    srl a4, a4, a0
-; RV32I-NEXT:    slli a5, a5, 1
+; RV32I-NEXT:    srl a7, a4, a0
+; RV32I-NEXT:    slli t0, a5, 1
 ; RV32I-NEXT:    srl a6, a6, a0
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl a5, a5, a0
 ; RV32I-NEXT:    slli t1, a1, 1
 ; RV32I-NEXT:    srl a0, a1, a0
 ; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a5, a5, a3
+; RV32I-NEXT:    sll a4, a4, a3
 ; RV32I-NEXT:    sll a3, t1, a3
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
 ; RV32I-NEXT:    or a1, a7, a1
-; RV32I-NEXT:    or a5, a4, a5
-; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    or a4, a6, a4
+; RV32I-NEXT:    or a3, a5, a3
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
@@ -815,18 +815,18 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srli a0, a3, 16
 ; RV32I-NEXT:    srli t0, a3, 24
 ; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    srli t2, a5, 24
-; RV32I-NEXT:    srli a5, a5, 8
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 24
+; RV32I-NEXT:    srli a4, a4, 8
 ; RV32I-NEXT:    srli t3, a1, 16
 ; RV32I-NEXT:    srli t4, a1, 24
 ; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a6, 8(a2)
+; RV32I-NEXT:    sb a5, 8(a2)
 ; RV32I-NEXT:    sb a3, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
 ; RV32I-NEXT:    sb t0, 11(a2)
-; RV32I-NEXT:    sb a4, 0(a2)
-; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a6, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb t1, 2(a2)
 ; RV32I-NEXT:    sb t2, 3(a2)
 ; RV32I-NEXT:    sb a7, 4(a2)
@@ -868,24 +868,24 @@ define void @lshr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t2, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t2, 0(a1)
 ; RV64I-NEXT:    lbu t3, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, t1, a5
-; RV64I-NEXT:    or a5, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a5, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a1, a1, 5
 ; RV64I-NEXT:    slli a6, a5, 37
@@ -899,31 +899,31 @@ define void @lshr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    j .LBB7_3
 ; RV64I-NEXT:  .LBB7_2:
 ; RV64I-NEXT:    lbu a6, 1(a0)
-; RV64I-NEXT:    lbu a7, 0(a0)
-; RV64I-NEXT:    lbu t0, 2(a0)
-; RV64I-NEXT:    lbu t1, 3(a0)
+; RV64I-NEXT:    lbu a7, 2(a0)
+; RV64I-NEXT:    lbu t0, 3(a0)
+; RV64I-NEXT:    lbu t1, 0(a0)
 ; RV64I-NEXT:    slli a6, a6, 8
-; RV64I-NEXT:    or a6, a6, a7
-; RV64I-NEXT:    lbu a7, 4(a0)
-; RV64I-NEXT:    lbu t2, 5(a0)
-; RV64I-NEXT:    slli t0, t0, 16
-; RV64I-NEXT:    slli t1, t1, 24
-; RV64I-NEXT:    or t0, t1, t0
-; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t0, t0, 24
+; RV64I-NEXT:    or a6, a6, t1
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 4(a0)
+; RV64I-NEXT:    lbu t2, 6(a0)
 ; RV64I-NEXT:    lbu a0, 7(a0)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
-; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    not t0, a4
+; RV64I-NEXT:    or a0, a0, t2
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    not a7, a4
 ; RV64I-NEXT:    slli a5, a5, 1
-; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a0, a0, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    srl a0, a0, a4
-; RV64I-NEXT:    sll a4, a5, t0
+; RV64I-NEXT:    sll a4, a5, a7
 ; RV64I-NEXT:    or a0, a0, a4
 ; RV64I-NEXT:  .LBB7_3:
 ; RV64I-NEXT:    srai a3, a3, 63
@@ -1083,24 +1083,24 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t2, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t2, 0(a1)
 ; RV64I-NEXT:    lbu t3, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, t1, a5
-; RV64I-NEXT:    or a5, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a5, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    slli a6, a5, 35
@@ -1114,31 +1114,31 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    j .LBB8_3
 ; RV64I-NEXT:  .LBB8_2:
 ; RV64I-NEXT:    lbu a6, 9(a0)
-; RV64I-NEXT:    lbu a7, 8(a0)
-; RV64I-NEXT:    lbu t0, 10(a0)
-; RV64I-NEXT:    lbu t1, 11(a0)
+; RV64I-NEXT:    lbu a7, 10(a0)
+; RV64I-NEXT:    lbu t0, 11(a0)
+; RV64I-NEXT:    lbu t1, 8(a0)
 ; RV64I-NEXT:    slli a6, a6, 8
-; RV64I-NEXT:    or a6, a6, a7
-; RV64I-NEXT:    lbu a7, 12(a0)
-; RV64I-NEXT:    lbu t2, 13(a0)
-; RV64I-NEXT:    slli t0, t0, 16
-; RV64I-NEXT:    slli t1, t1, 24
-; RV64I-NEXT:    or t0, t1, t0
-; RV64I-NEXT:    lbu t1, 14(a0)
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t0, t0, 24
+; RV64I-NEXT:    or a6, a6, t1
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu t0, 13(a0)
+; RV64I-NEXT:    lbu t1, 12(a0)
+; RV64I-NEXT:    lbu t2, 14(a0)
 ; RV64I-NEXT:    lbu a0, 15(a0)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
-; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    not t0, a4
+; RV64I-NEXT:    or a0, a0, t2
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    not a7, a4
 ; RV64I-NEXT:    srli a5, a5, 1
-; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a0, a0, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    sll a0, a0, a4
-; RV64I-NEXT:    srl a4, a5, t0
+; RV64I-NEXT:    srl a4, a5, a7
 ; RV64I-NEXT:    or a0, a0, a4
 ; RV64I-NEXT:  .LBB8_3:
 ; RV64I-NEXT:    srai a3, a3, 63
@@ -1322,24 +1322,24 @@ define void @shl_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t2, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t2, 0(a1)
 ; RV64I-NEXT:    lbu t3, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, t1, a5
-; RV64I-NEXT:    or a5, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a5, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a1, a1, 5
 ; RV64I-NEXT:    slli a6, a5, 37
@@ -1353,31 +1353,31 @@ define void @shl_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV64I-NEXT:    j .LBB9_3
 ; RV64I-NEXT:  .LBB9_2:
 ; RV64I-NEXT:    lbu a6, 9(a0)
-; RV64I-NEXT:    lbu a7, 8(a0)
-; RV64I-NEXT:    lbu t0, 10(a0)
-; RV64I-NEXT:    lbu t1, 11(a0)
+; RV64I-NEXT:    lbu a7, 10(a0)
+; RV64I-NEXT:    lbu t0, 11(a0)
+; RV64I-NEXT:    lbu t1, 8(a0)
 ; RV64I-NEXT:    slli a6, a6, 8
-; RV64I-NEXT:    or a6, a6, a7
-; RV64I-NEXT:    lbu a7, 12(a0)
-; RV64I-NEXT:    lbu t2, 13(a0)
-; RV64I-NEXT:    slli t0, t0, 16
-; RV64I-NEXT:    slli t1, t1, 24
-; RV64I-NEXT:    or t0, t1, t0
-; RV64I-NEXT:    lbu t1, 14(a0)
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t0, t0, 24
+; RV64I-NEXT:    or a6, a6, t1
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu t0, 13(a0)
+; RV64I-NEXT:    lbu t1, 12(a0)
+; RV64I-NEXT:    lbu t2, 14(a0)
 ; RV64I-NEXT:    lbu a0, 15(a0)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
-; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    not t0, a4
+; RV64I-NEXT:    or a0, a0, t2
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    not a7, a4
 ; RV64I-NEXT:    srli a5, a5, 1
-; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a0, a0, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    sll a0, a0, a4
-; RV64I-NEXT:    srl a4, a5, t0
+; RV64I-NEXT:    srl a4, a5, a7
 ; RV64I-NEXT:    or a0, a0, a4
 ; RV64I-NEXT:  .LBB9_3:
 ; RV64I-NEXT:    srai a3, a3, 63
@@ -1538,24 +1538,24 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t2, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t2, 0(a1)
 ; RV64I-NEXT:    lbu t3, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a5, t1, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a6, a5, 32
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    slli a7, a4, 35
@@ -1571,31 +1571,31 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    j .LBB10_3
 ; RV64I-NEXT:  .LBB10_2:
 ; RV64I-NEXT:    lbu a5, 1(a0)
-; RV64I-NEXT:    lbu a6, 0(a0)
-; RV64I-NEXT:    lbu a7, 2(a0)
-; RV64I-NEXT:    lbu t0, 3(a0)
+; RV64I-NEXT:    lbu a6, 2(a0)
+; RV64I-NEXT:    lbu a7, 3(a0)
+; RV64I-NEXT:    lbu t0, 0(a0)
 ; RV64I-NEXT:    slli a5, a5, 8
-; RV64I-NEXT:    or a5, a5, a6
-; RV64I-NEXT:    lbu a6, 4(a0)
-; RV64I-NEXT:    lbu t1, 5(a0)
-; RV64I-NEXT:    slli a7, a7, 16
-; RV64I-NEXT:    slli t0, t0, 24
-; RV64I-NEXT:    or a7, t0, a7
-; RV64I-NEXT:    lbu t0, 6(a0)
+; RV64I-NEXT:    slli a6, a6, 16
+; RV64I-NEXT:    slli a7, a7, 24
+; RV64I-NEXT:    or a5, a5, t0
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    lbu a7, 5(a0)
+; RV64I-NEXT:    lbu t0, 4(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
 ; RV64I-NEXT:    lbu a0, 7(a0)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a6, t1, a6
-; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    slli a7, a7, 8
+; RV64I-NEXT:    or a7, a7, t0
+; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    or a5, a7, a5
-; RV64I-NEXT:    not a7, a3
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    not a6, a3
 ; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    or a0, a0, a6
+; RV64I-NEXT:    or a0, a0, a7
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a5
 ; RV64I-NEXT:    srl a0, a0, a3
-; RV64I-NEXT:    sll a3, a4, a7
+; RV64I-NEXT:    sll a3, a4, a6
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:  .LBB10_3:
 ; RV64I-NEXT:    srli a3, a1, 56
@@ -1662,20 +1662,20 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli t3, t3, 8
 ; RV32I-NEXT:    slli t4, t4, 16
 ; RV32I-NEXT:    slli t5, t5, 24
+; RV32I-NEXT:    slli t1, t1, 8
 ; RV32I-NEXT:    or a4, t3, a4
 ; RV32I-NEXT:    or t3, t5, t4
-; RV32I-NEXT:    lbu t4, 0(a1)
-; RV32I-NEXT:    lbu t5, 1(a1)
-; RV32I-NEXT:    slli t1, t1, 8
 ; RV32I-NEXT:    or t0, t1, t0
-; RV32I-NEXT:    lbu t1, 2(a1)
+; RV32I-NEXT:    lbu t1, 1(a1)
+; RV32I-NEXT:    lbu t4, 0(a1)
+; RV32I-NEXT:    lbu t5, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t5, t5, 8
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t1, t1, t4
+; RV32I-NEXT:    slli t5, t5, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t1
-; RV32I-NEXT:    mv t1, sp
+; RV32I-NEXT:    or a1, a1, t5
+; RV32I-NEXT:    mv t4, sp
 ; RV32I-NEXT:    slli t2, t2, 16
 ; RV32I-NEXT:    slli a0, a0, 24
 ; RV32I-NEXT:    or t2, a0, t2
@@ -1684,7 +1684,7 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a5, a7, a6
 ; RV32I-NEXT:    or a4, t3, a4
 ; RV32I-NEXT:    or a6, t2, t0
-; RV32I-NEXT:    or a1, a1, t4
+; RV32I-NEXT:    or a1, a1, t1
 ; RV32I-NEXT:    sw a0, 16(sp)
 ; RV32I-NEXT:    sw a0, 20(sp)
 ; RV32I-NEXT:    sw a0, 24(sp)
@@ -1695,29 +1695,29 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    slli a0, a1, 3
 ; RV32I-NEXT:    andi a1, a1, 12
-; RV32I-NEXT:    add a1, t1, a1
+; RV32I-NEXT:    add a1, t4, a1
 ; RV32I-NEXT:    andi a3, a0, 24
-; RV32I-NEXT:    lw a4, 0(a1)
-; RV32I-NEXT:    lw a5, 4(a1)
-; RV32I-NEXT:    lw a6, 8(a1)
 ; RV32I-NEXT:    xori a3, a3, 31
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a6, 0(a1)
 ; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    srl a7, a5, a0
-; RV32I-NEXT:    slli t0, a6, 1
-; RV32I-NEXT:    srl a4, a4, a0
-; RV32I-NEXT:    slli a5, a5, 1
+; RV32I-NEXT:    srl a7, a4, a0
+; RV32I-NEXT:    slli t0, a5, 1
 ; RV32I-NEXT:    srl a6, a6, a0
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl a5, a5, a0
 ; RV32I-NEXT:    slli t1, a1, 1
 ; RV32I-NEXT:    sra a0, a1, a0
 ; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a5, a5, a3
+; RV32I-NEXT:    sll a4, a4, a3
 ; RV32I-NEXT:    sll a3, t1, a3
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
 ; RV32I-NEXT:    or a1, a7, a1
-; RV32I-NEXT:    or a5, a4, a5
-; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    or a4, a6, a4
+; RV32I-NEXT:    or a3, a5, a3
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
@@ -1725,18 +1725,18 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srli a0, a3, 16
 ; RV32I-NEXT:    srli t0, a3, 24
 ; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    srli t2, a5, 24
-; RV32I-NEXT:    srli a5, a5, 8
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 24
+; RV32I-NEXT:    srli a4, a4, 8
 ; RV32I-NEXT:    srli t3, a1, 16
 ; RV32I-NEXT:    srli t4, a1, 24
 ; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a6, 8(a2)
+; RV32I-NEXT:    sb a5, 8(a2)
 ; RV32I-NEXT:    sb a3, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
 ; RV32I-NEXT:    sb t0, 11(a2)
-; RV32I-NEXT:    sb a4, 0(a2)
-; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a6, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb t1, 2(a2)
 ; RV32I-NEXT:    sb t2, 3(a2)
 ; RV32I-NEXT:    sb a7, 4(a2)
@@ -1778,24 +1778,24 @@ define void @ashr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 0(a1)
-; RV64I-NEXT:    lbu t2, 1(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 1(a1)
+; RV64I-NEXT:    lbu t2, 0(a1)
 ; RV64I-NEXT:    lbu t3, 2(a1)
 ; RV64I-NEXT:    lbu a1, 3(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a5, t1, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a6, a5, 32
 ; RV64I-NEXT:    slli a1, a1, 5
 ; RV64I-NEXT:    slli a7, a4, 37
@@ -1811,31 +1811,31 @@ define void @ashr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    j .LBB11_3
 ; RV64I-NEXT:  .LBB11_2:
 ; RV64I-NEXT:    lbu a5, 1(a0)
-; RV64I-NEXT:    lbu a6, 0(a0)
-; RV64I-NEXT:    lbu a7, 2(a0)
-; RV64I-NEXT:    lbu t0, 3(a0)
+; RV64I-NEXT:    lbu a6, 2(a0)
+; RV64I-NEXT:    lbu a7, 3(a0)
+; RV64I-NEXT:    lbu t0, 0(a0)
 ; RV64I-NEXT:    slli a5, a5, 8
-; RV64I-NEXT:    or a5, a5, a6
-; RV64I-NEXT:    lbu a6, 4(a0)
-; RV64I-NEXT:    lbu t1, 5(a0)
-; RV64I-NEXT:    slli a7, a7, 16
-; RV64I-NEXT:    slli t0, t0, 24
-; RV64I-NEXT:    or a7, t0, a7
-; RV64I-NEXT:    lbu t0, 6(a0)
+; RV64I-NEXT:    slli a6, a6, 16
+; RV64I-NEXT:    slli a7, a7, 24
+; RV64I-NEXT:    or a5, a5, t0
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    lbu a7, 5(a0)
+; RV64I-NEXT:    lbu t0, 4(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
 ; RV64I-NEXT:    lbu a0, 7(a0)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a6, t1, a6
-; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    slli a7, a7, 8
+; RV64I-NEXT:    or a7, a7, t0
+; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    or a5, a7, a5
-; RV64I-NEXT:    not a7, a3
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    not a6, a3
 ; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    or a0, a0, a6
+; RV64I-NEXT:    or a0, a0, a7
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a5
 ; RV64I-NEXT:    srl a0, a0, a3
-; RV64I-NEXT:    sll a3, a4, a7
+; RV64I-NEXT:    sll a3, a4, a6
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:  .LBB11_3:
 ; RV64I-NEXT:    srli a3, a1, 56
@@ -2061,17 +2061,17 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    sd zero, 32(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 48(sp)
@@ -2088,8 +2088,8 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t0, s0, t6
 ; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t2, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -2108,22 +2108,22 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    andi a1, a1, 24
 ; RV64I-NEXT:    add a1, s6, a1
 ; RV64I-NEXT:    andi a0, a4, 56
-; RV64I-NEXT:    ld a3, 0(a1)
-; RV64I-NEXT:    ld a5, 8(a1)
+; RV64I-NEXT:    xori a5, a0, 63
+; RV64I-NEXT:    ld a3, 8(a1)
 ; RV64I-NEXT:    ld a6, 16(a1)
-; RV64I-NEXT:    xori a7, a0, 63
+; RV64I-NEXT:    ld a7, 0(a1)
 ; RV64I-NEXT:    ld t0, 24(a1)
-; RV64I-NEXT:    srl a0, a5, a4
+; RV64I-NEXT:    srl a0, a3, a4
 ; RV64I-NEXT:    slli t1, a6, 1
-; RV64I-NEXT:    srl a1, a3, a4
-; RV64I-NEXT:    slli a5, a5, 1
+; RV64I-NEXT:    srl a1, a7, a4
+; RV64I-NEXT:    slli a7, a3, 1
 ; RV64I-NEXT:    srl a3, a6, a4
 ; RV64I-NEXT:    slli a6, t0, 1
 ; RV64I-NEXT:    srl t0, t0, a4
-; RV64I-NEXT:    sll a4, t1, a7
-; RV64I-NEXT:    sll a5, a5, a7
-; RV64I-NEXT:    sll a6, a6, a7
-; RV64I-NEXT:    srli a7, t0, 56
+; RV64I-NEXT:    sll a4, t1, a5
+; RV64I-NEXT:    sll a7, a7, a5
+; RV64I-NEXT:    sll a5, a6, a5
+; RV64I-NEXT:    srli a6, t0, 56
 ; RV64I-NEXT:    srli t1, t0, 48
 ; RV64I-NEXT:    srli t2, t0, 40
 ; RV64I-NEXT:    srli t3, t0, 32
@@ -2131,40 +2131,40 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli t5, t0, 16
 ; RV64I-NEXT:    srli t6, t0, 8
 ; RV64I-NEXT:    or a4, a0, a4
-; RV64I-NEXT:    or a5, a1, a5
-; RV64I-NEXT:    or a6, a3, a6
+; RV64I-NEXT:    or a7, a1, a7
+; RV64I-NEXT:    or a5, a3, a5
 ; RV64I-NEXT:    sb t3, 28(a2)
 ; RV64I-NEXT:    sb t2, 29(a2)
 ; RV64I-NEXT:    sb t1, 30(a2)
-; RV64I-NEXT:    sb a7, 31(a2)
+; RV64I-NEXT:    sb a6, 31(a2)
 ; RV64I-NEXT:    sb t0, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
 ; RV64I-NEXT:    sb t4, 27(a2)
-; RV64I-NEXT:    srli a7, a6, 56
-; RV64I-NEXT:    srli t0, a6, 48
-; RV64I-NEXT:    srli t1, a6, 40
-; RV64I-NEXT:    srli t2, a6, 32
-; RV64I-NEXT:    srli t3, a6, 24
-; RV64I-NEXT:    srli t4, a6, 16
-; RV64I-NEXT:    srli a6, a6, 8
-; RV64I-NEXT:    srli t5, a5, 56
-; RV64I-NEXT:    srli t6, a5, 48
-; RV64I-NEXT:    srli s0, a5, 40
-; RV64I-NEXT:    srli s1, a5, 32
-; RV64I-NEXT:    srli s2, a5, 24
-; RV64I-NEXT:    srli s3, a5, 16
+; RV64I-NEXT:    srli a6, a5, 56
+; RV64I-NEXT:    srli t0, a5, 48
+; RV64I-NEXT:    srli t1, a5, 40
+; RV64I-NEXT:    srli t2, a5, 32
+; RV64I-NEXT:    srli t3, a5, 24
+; RV64I-NEXT:    srli t4, a5, 16
 ; RV64I-NEXT:    srli a5, a5, 8
+; RV64I-NEXT:    srli t5, a7, 56
+; RV64I-NEXT:    srli t6, a7, 48
+; RV64I-NEXT:    srli s0, a7, 40
+; RV64I-NEXT:    srli s1, a7, 32
+; RV64I-NEXT:    srli s2, a7, 24
+; RV64I-NEXT:    srli s3, a7, 16
+; RV64I-NEXT:    srli a7, a7, 8
 ; RV64I-NEXT:    srli s4, a4, 56
 ; RV64I-NEXT:    srli s5, a4, 48
 ; RV64I-NEXT:    srli s6, a4, 40
 ; RV64I-NEXT:    sb t2, 20(a2)
 ; RV64I-NEXT:    sb t1, 21(a2)
 ; RV64I-NEXT:    sb t0, 22(a2)
-; RV64I-NEXT:    sb a7, 23(a2)
-; RV64I-NEXT:    srli a7, a4, 32
+; RV64I-NEXT:    sb a6, 23(a2)
+; RV64I-NEXT:    srli a6, a4, 32
 ; RV64I-NEXT:    sb a3, 16(a2)
-; RV64I-NEXT:    sb a6, 17(a2)
+; RV64I-NEXT:    sb a5, 17(a2)
 ; RV64I-NEXT:    sb t4, 18(a2)
 ; RV64I-NEXT:    sb t3, 19(a2)
 ; RV64I-NEXT:    srli a3, a4, 24
@@ -2172,19 +2172,19 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sb s0, 5(a2)
 ; RV64I-NEXT:    sb t6, 6(a2)
 ; RV64I-NEXT:    sb t5, 7(a2)
-; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a5, a4, 16
 ; RV64I-NEXT:    srli a4, a4, 8
 ; RV64I-NEXT:    sb a1, 0(a2)
-; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a7, 1(a2)
 ; RV64I-NEXT:    sb s3, 2(a2)
 ; RV64I-NEXT:    sb s2, 3(a2)
-; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a6, 12(a2)
 ; RV64I-NEXT:    sb s6, 13(a2)
 ; RV64I-NEXT:    sb s5, 14(a2)
 ; RV64I-NEXT:    sb s4, 15(a2)
 ; RV64I-NEXT:    sb a0, 8(a2)
 ; RV64I-NEXT:    sb a4, 9(a2)
-; RV64I-NEXT:    sb a6, 10(a2)
+; RV64I-NEXT:    sb a5, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
 ; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
@@ -2539,17 +2539,17 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    sd zero, 32(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 48(sp)
@@ -2566,8 +2566,8 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t0, s0, t6
 ; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t2, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -2587,24 +2587,24 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    andi a1, a1, 24
 ; RV64I-NEXT:    andi a0, a3, 32
 ; RV64I-NEXT:    add a1, s6, a1
-; RV64I-NEXT:    ld a4, 0(a1)
+; RV64I-NEXT:    xori a4, a0, 63
 ; RV64I-NEXT:    ld a5, 8(a1)
 ; RV64I-NEXT:    ld a6, 16(a1)
-; RV64I-NEXT:    xori a7, a0, 63
+; RV64I-NEXT:    ld a7, 0(a1)
 ; RV64I-NEXT:    ld t0, 24(a1)
 ; RV64I-NEXT:    srl a0, a5, a3
 ; RV64I-NEXT:    slli t1, a6, 1
-; RV64I-NEXT:    srl a1, a4, a3
+; RV64I-NEXT:    srl a1, a7, a3
 ; RV64I-NEXT:    slli a5, a5, 1
-; RV64I-NEXT:    srl a4, a6, a3
-; RV64I-NEXT:    slli a6, t0, 1
+; RV64I-NEXT:    srl a6, a6, a3
+; RV64I-NEXT:    slli a7, t0, 1
 ; RV64I-NEXT:    srl a3, t0, a3
-; RV64I-NEXT:    sll t0, t1, a7
-; RV64I-NEXT:    sll a5, a5, a7
-; RV64I-NEXT:    sll a6, a6, a7
-; RV64I-NEXT:    srli a7, a4, 24
-; RV64I-NEXT:    srli t1, a4, 16
-; RV64I-NEXT:    srli t2, a4, 8
+; RV64I-NEXT:    sll t0, t1, a4
+; RV64I-NEXT:    sll a5, a5, a4
+; RV64I-NEXT:    sll a4, a7, a4
+; RV64I-NEXT:    srli a7, a6, 24
+; RV64I-NEXT:    srli t1, a6, 16
+; RV64I-NEXT:    srli t2, a6, 8
 ; RV64I-NEXT:    srli t3, a3, 56
 ; RV64I-NEXT:    srli t4, a3, 48
 ; RV64I-NEXT:    srli t5, a3, 40
@@ -2616,12 +2616,12 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    srli s4, a1, 16
 ; RV64I-NEXT:    srli s5, a1, 8
 ; RV64I-NEXT:    srli s6, a0, 24
-; RV64I-NEXT:    or a6, a4, a6
-; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    or a4, a6, a4
+; RV64I-NEXT:    sb a6, 16(a2)
 ; RV64I-NEXT:    sb t2, 17(a2)
 ; RV64I-NEXT:    sb t1, 18(a2)
 ; RV64I-NEXT:    sb a7, 19(a2)
-; RV64I-NEXT:    srli a4, a0, 16
+; RV64I-NEXT:    srli a6, a0, 16
 ; RV64I-NEXT:    sb t6, 28(a2)
 ; RV64I-NEXT:    sb t5, 29(a2)
 ; RV64I-NEXT:    sb t4, 30(a2)
@@ -2639,12 +2639,12 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    sb s3, 3(a2)
 ; RV64I-NEXT:    sb a0, 8(a2)
 ; RV64I-NEXT:    sb a7, 9(a2)
-; RV64I-NEXT:    sb a4, 10(a2)
+; RV64I-NEXT:    sb a6, 10(a2)
 ; RV64I-NEXT:    sb s6, 11(a2)
-; RV64I-NEXT:    srli a0, a6, 56
-; RV64I-NEXT:    srli a1, a6, 48
-; RV64I-NEXT:    srli a3, a6, 40
-; RV64I-NEXT:    srli a4, a6, 32
+; RV64I-NEXT:    srli a0, a4, 56
+; RV64I-NEXT:    srli a1, a4, 48
+; RV64I-NEXT:    srli a3, a4, 40
+; RV64I-NEXT:    srli a4, a4, 32
 ; RV64I-NEXT:    srli a6, a5, 56
 ; RV64I-NEXT:    srli a7, a5, 48
 ; RV64I-NEXT:    srli t1, a5, 40
@@ -2797,13 +2797,13 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV32I-NEXT:    sw t0, 12(sp)
 ; RV32I-NEXT:    sw t1, 16(sp)
 ; RV32I-NEXT:    sw a5, 20(sp)
-; RV32I-NEXT:    lw a6, 16(t6)
-; RV32I-NEXT:    lw a5, 20(t6)
-; RV32I-NEXT:    lw a7, 24(t6)
 ; RV32I-NEXT:    lw a1, 0(t6)
 ; RV32I-NEXT:    lw a0, 4(t6)
 ; RV32I-NEXT:    lw a4, 8(t6)
 ; RV32I-NEXT:    lw a3, 12(t6)
+; RV32I-NEXT:    lw a7, 24(t6)
+; RV32I-NEXT:    lw a5, 20(t6)
+; RV32I-NEXT:    lw a6, 16(t6)
 ; RV32I-NEXT:    lw t0, 28(t6)
 ; RV32I-NEXT:    srli t1, a7, 24
 ; RV32I-NEXT:    srli t2, a7, 16
@@ -3197,13 +3197,13 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
 ; RV32I-NEXT:    sw t0, 12(sp)
 ; RV32I-NEXT:    sw t1, 16(sp)
 ; RV32I-NEXT:    sw a5, 20(sp)
-; RV32I-NEXT:    lw a6, 16(t6)
-; RV32I-NEXT:    lw a5, 20(t6)
-; RV32I-NEXT:    lw a7, 24(t6)
 ; RV32I-NEXT:    lw a1, 0(t6)
 ; RV32I-NEXT:    lw a0, 4(t6)
 ; RV32I-NEXT:    lw a4, 8(t6)
 ; RV32I-NEXT:    lw a3, 12(t6)
+; RV32I-NEXT:    lw a7, 24(t6)
+; RV32I-NEXT:    lw a5, 20(t6)
+; RV32I-NEXT:    lw a6, 16(t6)
 ; RV32I-NEXT:    lw t0, 28(t6)
 ; RV32I-NEXT:    srli t1, a7, 24
 ; RV32I-NEXT:    srli t2, a7, 16
@@ -3376,17 +3376,17 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    sd zero, 0(sp)
 ; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    sd zero, 16(sp)
@@ -3403,8 +3403,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t0, s0, t6
 ; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t2, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -3854,17 +3854,17 @@ define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    sd zero, 0(sp)
 ; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    sd zero, 16(sp)
@@ -3881,8 +3881,8 @@ define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t0, s0, t6
 ; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t2, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -4112,13 +4112,13 @@ define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV32I-NEXT:    sw t0, 44(sp)
 ; RV32I-NEXT:    sw t1, 48(sp)
 ; RV32I-NEXT:    sw a5, 52(sp)
-; RV32I-NEXT:    lw a6, 16(t2)
-; RV32I-NEXT:    lw a5, 20(t2)
-; RV32I-NEXT:    lw a7, 24(t2)
 ; RV32I-NEXT:    lw a1, 0(t2)
 ; RV32I-NEXT:    lw a0, 4(t2)
 ; RV32I-NEXT:    lw a4, 8(t2)
 ; RV32I-NEXT:    lw a3, 12(t2)
+; RV32I-NEXT:    lw a7, 24(t2)
+; RV32I-NEXT:    lw a5, 20(t2)
+; RV32I-NEXT:    lw a6, 16(t2)
 ; RV32I-NEXT:    lw t0, 28(t2)
 ; RV32I-NEXT:    srli t1, a7, 24
 ; RV32I-NEXT:    srli t2, a7, 16
@@ -4512,13 +4512,13 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
 ; RV32I-NEXT:    sw t0, 44(sp)
 ; RV32I-NEXT:    sw t1, 48(sp)
 ; RV32I-NEXT:    sw a5, 52(sp)
-; RV32I-NEXT:    lw a6, 16(t2)
-; RV32I-NEXT:    lw a5, 20(t2)
-; RV32I-NEXT:    lw a7, 24(t2)
 ; RV32I-NEXT:    lw a1, 0(t2)
 ; RV32I-NEXT:    lw a0, 4(t2)
 ; RV32I-NEXT:    lw a4, 8(t2)
 ; RV32I-NEXT:    lw a3, 12(t2)
+; RV32I-NEXT:    lw a7, 24(t2)
+; RV32I-NEXT:    lw a5, 20(t2)
+; RV32I-NEXT:    lw a6, 16(t2)
 ; RV32I-NEXT:    lw t0, 28(t2)
 ; RV32I-NEXT:    srli t1, a7, 24
 ; RV32I-NEXT:    srli t2, a7, 16
@@ -4691,17 +4691,17 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    slli s7, s7, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, s7
@@ -4714,8 +4714,8 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t0, s0, t6
 ; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t2, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -4739,22 +4739,22 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    andi a1, a1, 24
 ; RV64I-NEXT:    add a1, s6, a1
 ; RV64I-NEXT:    andi a0, a4, 56
-; RV64I-NEXT:    ld a3, 0(a1)
-; RV64I-NEXT:    ld a5, 8(a1)
+; RV64I-NEXT:    xori a5, a0, 63
+; RV64I-NEXT:    ld a3, 8(a1)
 ; RV64I-NEXT:    ld a6, 16(a1)
-; RV64I-NEXT:    xori a7, a0, 63
+; RV64I-NEXT:    ld a7, 0(a1)
 ; RV64I-NEXT:    ld t0, 24(a1)
-; RV64I-NEXT:    srl a0, a5, a4
+; RV64I-NEXT:    srl a0, a3, a4
 ; RV64I-NEXT:    slli t1, a6, 1
-; RV64I-NEXT:    srl a1, a3, a4
-; RV64I-NEXT:    slli a5, a5, 1
+; RV64I-NEXT:    srl a1, a7, a4
+; RV64I-NEXT:    slli a7, a3, 1
 ; RV64I-NEXT:    srl a3, a6, a4
 ; RV64I-NEXT:    slli a6, t0, 1
 ; RV64I-NEXT:    sra t0, t0, a4
-; RV64I-NEXT:    sll a4, t1, a7
-; RV64I-NEXT:    sll a5, a5, a7
-; RV64I-NEXT:    sll a6, a6, a7
-; RV64I-NEXT:    srli a7, t0, 56
+; RV64I-NEXT:    sll a4, t1, a5
+; RV64I-NEXT:    sll a7, a7, a5
+; RV64I-NEXT:    sll a5, a6, a5
+; RV64I-NEXT:    srli a6, t0, 56
 ; RV64I-NEXT:    srli t1, t0, 48
 ; RV64I-NEXT:    srli t2, t0, 40
 ; RV64I-NEXT:    srli t3, t0, 32
@@ -4762,40 +4762,40 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli t5, t0, 16
 ; RV64I-NEXT:    srli t6, t0, 8
 ; RV64I-NEXT:    or a4, a0, a4
-; RV64I-NEXT:    or a5, a1, a5
-; RV64I-NEXT:    or a6, a3, a6
+; RV64I-NEXT:    or a7, a1, a7
+; RV64I-NEXT:    or a5, a3, a5
 ; RV64I-NEXT:    sb t3, 28(a2)
 ; RV64I-NEXT:    sb t2, 29(a2)
 ; RV64I-NEXT:    sb t1, 30(a2)
-; RV64I-NEXT:    sb a7, 31(a2)
+; RV64I-NEXT:    sb a6, 31(a2)
 ; RV64I-NEXT:    sb t0, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
 ; RV64I-NEXT:    sb t4, 27(a2)
-; RV64I-NEXT:    srli a7, a6, 56
-; RV64I-NEXT:    srli t0, a6, 48
-; RV64I-NEXT:    srli t1, a6, 40
-; RV64I-NEXT:    srli t2, a6, 32
-; RV64I-NEXT:    srli t3, a6, 24
-; RV64I-NEXT:    srli t4, a6, 16
-; RV64I-NEXT:    srli a6, a6, 8
-; RV64I-NEXT:    srli t5, a5, 56
-; RV64I-NEXT:    srli t6, a5, 48
-; RV64I-NEXT:    srli s0, a5, 40
-; RV64I-NEXT:    srli s1, a5, 32
-; RV64I-NEXT:    srli s2, a5, 24
-; RV64I-NEXT:    srli s3, a5, 16
+; RV64I-NEXT:    srli a6, a5, 56
+; RV64I-NEXT:    srli t0, a5, 48
+; RV64I-NEXT:    srli t1, a5, 40
+; RV64I-NEXT:    srli t2, a5, 32
+; RV64I-NEXT:    srli t3, a5, 24
+; RV64I-NEXT:    srli t4, a5, 16
 ; RV64I-NEXT:    srli a5, a5, 8
+; RV64I-NEXT:    srli t5, a7, 56
+; RV64I-NEXT:    srli t6, a7, 48
+; RV64I-NEXT:    srli s0, a7, 40
+; RV64I-NEXT:    srli s1, a7, 32
+; RV64I-NEXT:    srli s2, a7, 24
+; RV64I-NEXT:    srli s3, a7, 16
+; RV64I-NEXT:    srli a7, a7, 8
 ; RV64I-NEXT:    srli s4, a4, 56
 ; RV64I-NEXT:    srli s5, a4, 48
 ; RV64I-NEXT:    srli s6, a4, 40
 ; RV64I-NEXT:    sb t2, 20(a2)
 ; RV64I-NEXT:    sb t1, 21(a2)
 ; RV64I-NEXT:    sb t0, 22(a2)
-; RV64I-NEXT:    sb a7, 23(a2)
-; RV64I-NEXT:    srli a7, a4, 32
+; RV64I-NEXT:    sb a6, 23(a2)
+; RV64I-NEXT:    srli a6, a4, 32
 ; RV64I-NEXT:    sb a3, 16(a2)
-; RV64I-NEXT:    sb a6, 17(a2)
+; RV64I-NEXT:    sb a5, 17(a2)
 ; RV64I-NEXT:    sb t4, 18(a2)
 ; RV64I-NEXT:    sb t3, 19(a2)
 ; RV64I-NEXT:    srli a3, a4, 24
@@ -4803,19 +4803,19 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sb s0, 5(a2)
 ; RV64I-NEXT:    sb t6, 6(a2)
 ; RV64I-NEXT:    sb t5, 7(a2)
-; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a5, a4, 16
 ; RV64I-NEXT:    srli a4, a4, 8
 ; RV64I-NEXT:    sb a1, 0(a2)
-; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a7, 1(a2)
 ; RV64I-NEXT:    sb s3, 2(a2)
 ; RV64I-NEXT:    sb s2, 3(a2)
-; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a6, 12(a2)
 ; RV64I-NEXT:    sb s6, 13(a2)
 ; RV64I-NEXT:    sb s5, 14(a2)
 ; RV64I-NEXT:    sb s4, 15(a2)
 ; RV64I-NEXT:    sb a0, 8(a2)
 ; RV64I-NEXT:    sb a4, 9(a2)
-; RV64I-NEXT:    sb a6, 10(a2)
+; RV64I-NEXT:    sb a5, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
 ; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
@@ -5171,17 +5171,17 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    slli s7, s7, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, s7
@@ -5194,8 +5194,8 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t0, s0, t6
 ; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t2, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -5220,24 +5220,24 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    andi a1, a1, 24
 ; RV64I-NEXT:    andi a0, a3, 32
 ; RV64I-NEXT:    add a1, s6, a1
-; RV64I-NEXT:    ld a4, 0(a1)
+; RV64I-NEXT:    xori a4, a0, 63
 ; RV64I-NEXT:    ld a5, 8(a1)
 ; RV64I-NEXT:    ld a6, 16(a1)
-; RV64I-NEXT:    xori a7, a0, 63
+; RV64I-NEXT:    ld a7, 0(a1)
 ; RV64I-NEXT:    ld t0, 24(a1)
 ; RV64I-NEXT:    srl a0, a5, a3
 ; RV64I-NEXT:    slli t1, a6, 1
-; RV64I-NEXT:    srl a1, a4, a3
+; RV64I-NEXT:    srl a1, a7, a3
 ; RV64I-NEXT:    slli a5, a5, 1
-; RV64I-NEXT:    srl a4, a6, a3
-; RV64I-NEXT:    slli a6, t0, 1
+; RV64I-NEXT:    srl a6, a6, a3
+; RV64I-NEXT:    slli a7, t0, 1
 ; RV64I-NEXT:    sra a3, t0, a3
-; RV64I-NEXT:    sll t0, t1, a7
-; RV64I-NEXT:    sll a5, a5, a7
-; RV64I-NEXT:    sll a6, a6, a7
-; RV64I-NEXT:    srli a7, a4, 24
-; RV64I-NEXT:    srli t1, a4, 16
-; RV64I-NEXT:    srli t2, a4, 8
+; RV64I-NEXT:    sll t0, t1, a4
+; RV64I-NEXT:    sll a5, a5, a4
+; RV64I-NEXT:    sll a4, a7, a4
+; RV64I-NEXT:    srli a7, a6, 24
+; RV64I-NEXT:    srli t1, a6, 16
+; RV64I-NEXT:    srli t2, a6, 8
 ; RV64I-NEXT:    srli t3, a3, 56
 ; RV64I-NEXT:    srli t4, a3, 48
 ; RV64I-NEXT:    srli t5, a3, 40
@@ -5249,12 +5249,12 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    srli s4, a1, 16
 ; RV64I-NEXT:    srli s5, a1, 8
 ; RV64I-NEXT:    srli s6, a0, 24
-; RV64I-NEXT:    or a6, a4, a6
-; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    or a4, a6, a4
+; RV64I-NEXT:    sb a6, 16(a2)
 ; RV64I-NEXT:    sb t2, 17(a2)
 ; RV64I-NEXT:    sb t1, 18(a2)
 ; RV64I-NEXT:    sb a7, 19(a2)
-; RV64I-NEXT:    srli a4, a0, 16
+; RV64I-NEXT:    srli a6, a0, 16
 ; RV64I-NEXT:    sb t6, 28(a2)
 ; RV64I-NEXT:    sb t5, 29(a2)
 ; RV64I-NEXT:    sb t4, 30(a2)
@@ -5272,12 +5272,12 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    sb s3, 3(a2)
 ; RV64I-NEXT:    sb a0, 8(a2)
 ; RV64I-NEXT:    sb a7, 9(a2)
-; RV64I-NEXT:    sb a4, 10(a2)
+; RV64I-NEXT:    sb a6, 10(a2)
 ; RV64I-NEXT:    sb s6, 11(a2)
-; RV64I-NEXT:    srli a0, a6, 56
-; RV64I-NEXT:    srli a1, a6, 48
-; RV64I-NEXT:    srli a3, a6, 40
-; RV64I-NEXT:    srli a4, a6, 32
+; RV64I-NEXT:    srli a0, a4, 56
+; RV64I-NEXT:    srli a1, a4, 48
+; RV64I-NEXT:    srli a3, a4, 40
+; RV64I-NEXT:    srli a4, a4, 32
 ; RV64I-NEXT:    srli a6, a5, 56
 ; RV64I-NEXT:    srli a7, a5, 48
 ; RV64I-NEXT:    srli t1, a5, 40
@@ -5431,13 +5431,13 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV32I-NEXT:    sw t0, 12(sp)
 ; RV32I-NEXT:    sw t1, 16(sp)
 ; RV32I-NEXT:    sw a5, 20(sp)
-; RV32I-NEXT:    lw a6, 16(s6)
-; RV32I-NEXT:    lw a5, 20(s6)
-; RV32I-NEXT:    lw a7, 24(s6)
 ; RV32I-NEXT:    lw a1, 0(s6)
 ; RV32I-NEXT:    lw a0, 4(s6)
 ; RV32I-NEXT:    lw a4, 8(s6)
 ; RV32I-NEXT:    lw a3, 12(s6)
+; RV32I-NEXT:    lw a7, 24(s6)
+; RV32I-NEXT:    lw a5, 20(s6)
+; RV32I-NEXT:    lw a6, 16(s6)
 ; RV32I-NEXT:    lw t0, 28(s6)
 ; RV32I-NEXT:    srli t1, a7, 24
 ; RV32I-NEXT:    srli t2, a7, 16
@@ -5833,13 +5833,13 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
 ; RV32I-NEXT:    sw t0, 12(sp)
 ; RV32I-NEXT:    sw t1, 16(sp)
 ; RV32I-NEXT:    sw a5, 20(sp)
-; RV32I-NEXT:    lw a6, 16(s6)
-; RV32I-NEXT:    lw a5, 20(s6)
-; RV32I-NEXT:    lw a7, 24(s6)
 ; RV32I-NEXT:    lw a1, 0(s6)
 ; RV32I-NEXT:    lw a0, 4(s6)
 ; RV32I-NEXT:    lw a4, 8(s6)
 ; RV32I-NEXT:    lw a3, 12(s6)
+; RV32I-NEXT:    lw a7, 24(s6)
+; RV32I-NEXT:    lw a5, 20(s6)
+; RV32I-NEXT:    lw a6, 16(s6)
 ; RV32I-NEXT:    lw t0, 28(s6)
 ; RV32I-NEXT:    srli t1, a7, 24
 ; RV32I-NEXT:    srli t2, a7, 16
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
index 78f63c72d046..cd7f30d8f589 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
@@ -29,25 +29,25 @@ define void @lshr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: lshr_4bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a6, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or a0, a0, a5
-; RV32I-NEXT:    lbu a5, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a3, 1(a1)
+; RV32I-NEXT:    lbu a5, 0(a1)
+; RV32I-NEXT:    lbu a6, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a5
-; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    srl a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    srli a3, a0, 24
@@ -90,25 +90,25 @@ define void @shl_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: shl_4bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a6, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or a0, a0, a5
-; RV32I-NEXT:    lbu a5, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a3, 1(a1)
+; RV32I-NEXT:    lbu a5, 0(a1)
+; RV32I-NEXT:    lbu a6, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a5
-; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    sll a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    srli a3, a0, 24
@@ -151,25 +151,25 @@ define void @ashr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: ashr_4bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a0, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a6, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or a0, a0, a5
-; RV32I-NEXT:    lbu a5, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a3, 1(a1)
+; RV32I-NEXT:    lbu a5, 0(a1)
+; RV32I-NEXT:    lbu a6, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    slli a5, a5, 16
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a5
-; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    sra a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    srli a3, a0, 24
@@ -211,24 +211,24 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 4(a1)
-; RV64I-NEXT:    lbu t1, 5(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or t0, t2, t0
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t2, t0
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t1, 4(a1)
 ; RV64I-NEXT:    lbu t2, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
 ; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t2
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a0, a0, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a0, a0, a3
@@ -254,25 +254,25 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: lshr_8bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 5(a0)
-; RV32I-NEXT:    lbu a4, 4(a0)
-; RV32I-NEXT:    lbu a5, 6(a0)
-; RV32I-NEXT:    lbu a6, 7(a0)
+; RV32I-NEXT:    lbu a4, 6(a0)
+; RV32I-NEXT:    lbu a5, 7(a0)
+; RV32I-NEXT:    lbu a6, 4(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a7, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a3, a3, a6
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a5, 1(a1)
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    slli a6, a6, 16
+; RV32I-NEXT:    slli a5, a5, 8
+; RV32I-NEXT:    or a6, a5, a6
+; RV32I-NEXT:    slli a7, a7, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    or a5, a5, a3
-; RV32I-NEXT:    or a4, a1, a4
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    or a5, a4, a3
+; RV32I-NEXT:    or a4, a1, a6
 ; RV32I-NEXT:    addi a3, a4, -32
 ; RV32I-NEXT:    srl a1, a5, a4
 ; RV32I-NEXT:    bltz a3, .LBB3_2
@@ -344,24 +344,24 @@ define void @shl_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 4(a1)
-; RV64I-NEXT:    lbu t1, 5(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or t0, t2, t0
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t2, t0
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t1, 4(a1)
 ; RV64I-NEXT:    lbu t2, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
 ; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t2
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a0, a0, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a0, a0, a3
@@ -387,25 +387,25 @@ define void @shl_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-LABEL: shl_8bytes:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lbu a3, 1(a0)
-; RV32I-NEXT:    lbu a4, 0(a0)
-; RV32I-NEXT:    lbu a5, 2(a0)
-; RV32I-NEXT:    lbu a6, 3(a0)
+; RV32I-NEXT:    lbu a4, 2(a0)
+; RV32I-NEXT:    lbu a5, 3(a0)
+; RV32I-NEXT:    lbu a6, 0(a0)
 ; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, a4
-; RV32I-NEXT:    lbu a4, 0(a1)
-; RV32I-NEXT:    lbu a7, 1(a1)
-; RV32I-NEXT:    slli a5, a5, 16
-; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 2(a1)
+; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a5, a5, 24
+; RV32I-NEXT:    or a3, a3, a6
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    lbu a5, 1(a1)
+; RV32I-NEXT:    lbu a6, 0(a1)
+; RV32I-NEXT:    lbu a7, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a7, a7, 8
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    slli a6, a6, 16
+; RV32I-NEXT:    slli a5, a5, 8
+; RV32I-NEXT:    or a6, a5, a6
+; RV32I-NEXT:    slli a7, a7, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    or a5, a5, a3
-; RV32I-NEXT:    or a4, a1, a4
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    or a5, a4, a3
+; RV32I-NEXT:    or a4, a1, a6
 ; RV32I-NEXT:    addi a3, a4, -32
 ; RV32I-NEXT:    sll a1, a5, a4
 ; RV32I-NEXT:    bltz a3, .LBB4_2
@@ -477,24 +477,24 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 4(a1)
-; RV64I-NEXT:    lbu t1, 5(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or t0, t2, t0
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t2, t0
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t1, 4(a1)
 ; RV64I-NEXT:    lbu t2, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a7, t1, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
 ; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t2
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a0, a0, a5
-; RV64I-NEXT:    or a4, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a4, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a0, a0, a3
@@ -519,21 +519,21 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ;
 ; RV32I-LABEL: ashr_8bytes:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lbu a3, 4(a0)
-; RV32I-NEXT:    lbu a4, 5(a0)
+; RV32I-NEXT:    lbu a3, 5(a0)
+; RV32I-NEXT:    lbu a4, 4(a0)
 ; RV32I-NEXT:    lbu a5, 6(a0)
 ; RV32I-NEXT:    lbu a6, 7(a0)
+; RV32I-NEXT:    slli a3, a3, 8
+; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    lbu a4, 1(a1)
 ; RV32I-NEXT:    lbu a7, 0(a1)
-; RV32I-NEXT:    lbu t0, 1(a1)
-; RV32I-NEXT:    slli a4, a4, 8
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    lbu a4, 2(a1)
+; RV32I-NEXT:    lbu t0, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    or a7, t0, a7
-; RV32I-NEXT:    slli a4, a4, 16
+; RV32I-NEXT:    slli a4, a4, 8
+; RV32I-NEXT:    or a7, a4, a7
+; RV32I-NEXT:    slli t0, t0, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    or a1, a1, t0
 ; RV32I-NEXT:    slli a4, a5, 16
 ; RV32I-NEXT:    slli a5, a6, 24
 ; RV32I-NEXT:    or a4, a5, a4
@@ -611,24 +611,24 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 4(a1)
-; RV64I-NEXT:    lbu t2, 5(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t2, 4(a1)
 ; RV64I-NEXT:    lbu t3, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, t1, a5
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a5, a4, a3
@@ -641,31 +641,31 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    j .LBB6_3
 ; RV64I-NEXT:  .LBB6_2:
 ; RV64I-NEXT:    lbu a6, 1(a0)
-; RV64I-NEXT:    lbu a7, 0(a0)
-; RV64I-NEXT:    lbu t0, 2(a0)
-; RV64I-NEXT:    lbu t1, 3(a0)
+; RV64I-NEXT:    lbu a7, 2(a0)
+; RV64I-NEXT:    lbu t0, 3(a0)
+; RV64I-NEXT:    lbu t1, 0(a0)
 ; RV64I-NEXT:    slli a6, a6, 8
-; RV64I-NEXT:    or a6, a6, a7
-; RV64I-NEXT:    lbu a7, 4(a0)
-; RV64I-NEXT:    lbu t2, 5(a0)
-; RV64I-NEXT:    slli t0, t0, 16
-; RV64I-NEXT:    slli t1, t1, 24
-; RV64I-NEXT:    or t0, t1, t0
-; RV64I-NEXT:    lbu t1, 6(a0)
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t0, t0, 24
+; RV64I-NEXT:    or a6, a6, t1
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu t0, 5(a0)
+; RV64I-NEXT:    lbu t1, 4(a0)
+; RV64I-NEXT:    lbu t2, 6(a0)
 ; RV64I-NEXT:    lbu a0, 7(a0)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
-; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    not t0, a4
+; RV64I-NEXT:    or a0, a0, t2
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    not a7, a4
 ; RV64I-NEXT:    slli a5, a5, 1
-; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a0, a0, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    srl a0, a0, a4
-; RV64I-NEXT:    sll a4, a5, t0
+; RV64I-NEXT:    sll a4, a5, a7
 ; RV64I-NEXT:    or a0, a0, a4
 ; RV64I-NEXT:  .LBB6_3:
 ; RV64I-NEXT:    srai a3, a3, 63
@@ -736,24 +736,24 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli t3, t3, 16
 ; RV32I-NEXT:    slli t4, t4, 24
 ; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t3, t4, t3
-; RV32I-NEXT:    or a6, t1, a6
-; RV32I-NEXT:    lbu t1, 0(a1)
-; RV32I-NEXT:    lbu t4, 1(a1)
 ; RV32I-NEXT:    slli t2, t2, 16
 ; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    or t3, t4, t3
+; RV32I-NEXT:    or a6, t1, a6
 ; RV32I-NEXT:    or a0, a0, t2
-; RV32I-NEXT:    lbu t2, 2(a1)
+; RV32I-NEXT:    lbu t1, 1(a1)
+; RV32I-NEXT:    lbu t2, 0(a1)
+; RV32I-NEXT:    lbu t4, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    or t1, t4, t1
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t1, t1, t2
 ; RV32I-NEXT:    sw zero, 16(sp)
 ; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 24(sp)
 ; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli t4, t4, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t2
+; RV32I-NEXT:    or a1, a1, t4
 ; RV32I-NEXT:    mv t2, sp
 ; RV32I-NEXT:    or a3, a4, a3
 ; RV32I-NEXT:    or a4, t0, a7
@@ -767,28 +767,28 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srli a0, a1, 3
 ; RV32I-NEXT:    andi a3, a1, 31
 ; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    add a0, t2, a0
-; RV32I-NEXT:    lw a4, 0(a0)
-; RV32I-NEXT:    lw a5, 4(a0)
-; RV32I-NEXT:    lw a6, 8(a0)
 ; RV32I-NEXT:    xori a3, a3, 31
+; RV32I-NEXT:    add a0, t2, a0
+; RV32I-NEXT:    lw a4, 4(a0)
+; RV32I-NEXT:    lw a5, 8(a0)
+; RV32I-NEXT:    lw a6, 0(a0)
 ; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    srl a7, a5, a1
-; RV32I-NEXT:    slli t0, a6, 1
-; RV32I-NEXT:    srl a4, a4, a1
-; RV32I-NEXT:    slli a5, a5, 1
+; RV32I-NEXT:    srl a7, a4, a1
+; RV32I-NEXT:    slli t0, a5, 1
 ; RV32I-NEXT:    srl a6, a6, a1
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl a5, a5, a1
 ; RV32I-NEXT:    slli t1, a0, 1
 ; RV32I-NEXT:    srl a0, a0, a1
 ; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a5, a5, a3
+; RV32I-NEXT:    sll a4, a4, a3
 ; RV32I-NEXT:    sll a3, t1, a3
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
 ; RV32I-NEXT:    or a1, a7, a1
-; RV32I-NEXT:    or a4, a4, a5
-; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    or a4, a6, a4
+; RV32I-NEXT:    or a3, a5, a3
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
@@ -847,24 +847,24 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 4(a1)
-; RV64I-NEXT:    lbu t2, 5(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t2, 4(a1)
 ; RV64I-NEXT:    lbu t3, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, t1, a5
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a5, a4, a3
@@ -877,31 +877,31 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    j .LBB7_3
 ; RV64I-NEXT:  .LBB7_2:
 ; RV64I-NEXT:    lbu a6, 9(a0)
-; RV64I-NEXT:    lbu a7, 8(a0)
-; RV64I-NEXT:    lbu t0, 10(a0)
-; RV64I-NEXT:    lbu t1, 11(a0)
+; RV64I-NEXT:    lbu a7, 10(a0)
+; RV64I-NEXT:    lbu t0, 11(a0)
+; RV64I-NEXT:    lbu t1, 8(a0)
 ; RV64I-NEXT:    slli a6, a6, 8
-; RV64I-NEXT:    or a6, a6, a7
-; RV64I-NEXT:    lbu a7, 12(a0)
-; RV64I-NEXT:    lbu t2, 13(a0)
-; RV64I-NEXT:    slli t0, t0, 16
-; RV64I-NEXT:    slli t1, t1, 24
-; RV64I-NEXT:    or t0, t1, t0
-; RV64I-NEXT:    lbu t1, 14(a0)
+; RV64I-NEXT:    slli a7, a7, 16
+; RV64I-NEXT:    slli t0, t0, 24
+; RV64I-NEXT:    or a6, a6, t1
+; RV64I-NEXT:    or a7, t0, a7
+; RV64I-NEXT:    lbu t0, 13(a0)
+; RV64I-NEXT:    lbu t1, 12(a0)
+; RV64I-NEXT:    lbu t2, 14(a0)
 ; RV64I-NEXT:    lbu a0, 15(a0)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
-; RV64I-NEXT:    slli t1, t1, 16
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t1
+; RV64I-NEXT:    slli t2, t2, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t1
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    not t0, a4
+; RV64I-NEXT:    or a0, a0, t2
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    not a7, a4
 ; RV64I-NEXT:    srli a5, a5, 1
-; RV64I-NEXT:    or a0, a0, a7
+; RV64I-NEXT:    or a0, a0, t0
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    sll a0, a0, a4
-; RV64I-NEXT:    srl a4, a5, t0
+; RV64I-NEXT:    srl a4, a5, a7
 ; RV64I-NEXT:    or a0, a0, a4
 ; RV64I-NEXT:  .LBB7_3:
 ; RV64I-NEXT:    srai a3, a3, 63
@@ -972,24 +972,24 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli t3, t3, 16
 ; RV32I-NEXT:    slli t4, t4, 24
 ; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t3, t4, t3
-; RV32I-NEXT:    or a6, t1, a6
-; RV32I-NEXT:    lbu t1, 0(a1)
-; RV32I-NEXT:    lbu t4, 1(a1)
 ; RV32I-NEXT:    slli t2, t2, 16
 ; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    or t3, t4, t3
+; RV32I-NEXT:    or a6, t1, a6
 ; RV32I-NEXT:    or a0, a0, t2
-; RV32I-NEXT:    lbu t2, 2(a1)
+; RV32I-NEXT:    lbu t1, 1(a1)
+; RV32I-NEXT:    lbu t2, 0(a1)
+; RV32I-NEXT:    lbu t4, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    or t1, t4, t1
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t1, t1, t2
 ; RV32I-NEXT:    sw zero, 0(sp)
 ; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    sw zero, 8(sp)
 ; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    slli t2, t2, 16
+; RV32I-NEXT:    slli t4, t4, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t2
+; RV32I-NEXT:    or a1, a1, t4
 ; RV32I-NEXT:    addi t2, sp, 16
 ; RV32I-NEXT:    or a3, a4, a3
 ; RV32I-NEXT:    or a4, t0, a7
@@ -1083,24 +1083,24 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
 ; RV64I-NEXT:    slli a7, a7, 8
-; RV64I-NEXT:    or t1, t2, t1
-; RV64I-NEXT:    or a6, a7, a6
-; RV64I-NEXT:    lbu a7, 4(a1)
-; RV64I-NEXT:    lbu t2, 5(a1)
 ; RV64I-NEXT:    slli t0, t0, 16
 ; RV64I-NEXT:    slli t3, t3, 24
-; RV64I-NEXT:    or t0, t3, t0
+; RV64I-NEXT:    or t1, t2, t1
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a7, t3, t0
+; RV64I-NEXT:    lbu t0, 5(a1)
+; RV64I-NEXT:    lbu t2, 4(a1)
 ; RV64I-NEXT:    lbu t3, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli t2, t2, 8
-; RV64I-NEXT:    or a7, t2, a7
+; RV64I-NEXT:    slli t0, t0, 8
+; RV64I-NEXT:    or t0, t0, t2
 ; RV64I-NEXT:    slli t3, t3, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, t3
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a5, t1, a5
-; RV64I-NEXT:    or a6, t0, a6
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a4, a5, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a4, a4, a3
@@ -1115,31 +1115,31 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    j .LBB8_3
 ; RV64I-NEXT:  .LBB8_2:
 ; RV64I-NEXT:    lbu a5, 1(a0)
-; RV64I-NEXT:    lbu a6, 0(a0)
-; RV64I-NEXT:    lbu a7, 2(a0)
-; RV64I-NEXT:    lbu t0, 3(a0)
+; RV64I-NEXT:    lbu a6, 2(a0)
+; RV64I-NEXT:    lbu a7, 3(a0)
+; RV64I-NEXT:    lbu t0, 0(a0)
 ; RV64I-NEXT:    slli a5, a5, 8
-; RV64I-NEXT:    or a5, a5, a6
-; RV64I-NEXT:    lbu a6, 4(a0)
-; RV64I-NEXT:    lbu t1, 5(a0)
-; RV64I-NEXT:    slli a7, a7, 16
-; RV64I-NEXT:    slli t0, t0, 24
-; RV64I-NEXT:    or a7, t0, a7
-; RV64I-NEXT:    lbu t0, 6(a0)
+; RV64I-NEXT:    slli a6, a6, 16
+; RV64I-NEXT:    slli a7, a7, 24
+; RV64I-NEXT:    or a5, a5, t0
+; RV64I-NEXT:    or a6, a7, a6
+; RV64I-NEXT:    lbu a7, 5(a0)
+; RV64I-NEXT:    lbu t0, 4(a0)
+; RV64I-NEXT:    lbu t1, 6(a0)
 ; RV64I-NEXT:    lbu a0, 7(a0)
-; RV64I-NEXT:    slli t1, t1, 8
-; RV64I-NEXT:    or a6, t1, a6
-; RV64I-NEXT:    slli t0, t0, 16
+; RV64I-NEXT:    slli a7, a7, 8
+; RV64I-NEXT:    or a7, a7, t0
+; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli a0, a0, 24
-; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    or a5, a7, a5
-; RV64I-NEXT:    not a7, a3
+; RV64I-NEXT:    or a0, a0, t1
+; RV64I-NEXT:    or a5, a6, a5
+; RV64I-NEXT:    not a6, a3
 ; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    or a0, a0, a6
+; RV64I-NEXT:    or a0, a0, a7
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a5
 ; RV64I-NEXT:    srl a0, a0, a3
-; RV64I-NEXT:    sll a3, a4, a7
+; RV64I-NEXT:    sll a3, a4, a6
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:  .LBB8_3:
 ; RV64I-NEXT:    srli a3, a1, 56
@@ -1206,29 +1206,29 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli t3, t3, 8
 ; RV32I-NEXT:    slli t4, t4, 16
 ; RV32I-NEXT:    slli t5, t5, 24
+; RV32I-NEXT:    slli t1, t1, 8
 ; RV32I-NEXT:    or a4, t3, a4
 ; RV32I-NEXT:    or t3, t5, t4
-; RV32I-NEXT:    lbu t4, 0(a1)
-; RV32I-NEXT:    lbu t5, 1(a1)
-; RV32I-NEXT:    slli t1, t1, 8
 ; RV32I-NEXT:    or t0, t1, t0
-; RV32I-NEXT:    lbu t1, 2(a1)
+; RV32I-NEXT:    lbu t1, 1(a1)
+; RV32I-NEXT:    lbu t4, 0(a1)
+; RV32I-NEXT:    lbu t5, 2(a1)
 ; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t5, t5, 8
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    slli t1, t1, 16
+; RV32I-NEXT:    slli t1, t1, 8
+; RV32I-NEXT:    or t1, t1, t4
+; RV32I-NEXT:    slli t5, t5, 16
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t1
+; RV32I-NEXT:    or a1, a1, t5
 ; RV32I-NEXT:    or a3, a5, a3
 ; RV32I-NEXT:    mv a5, sp
 ; RV32I-NEXT:    slli t2, t2, 16
 ; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or t1, a0, t2
+; RV32I-NEXT:    or t2, a0, t2
 ; RV32I-NEXT:    srai a0, a0, 31
 ; RV32I-NEXT:    or a6, a7, a6
 ; RV32I-NEXT:    or a4, t3, a4
-; RV32I-NEXT:    or a7, t1, t0
-; RV32I-NEXT:    or a1, a1, t4
+; RV32I-NEXT:    or a7, t2, t0
+; RV32I-NEXT:    or a1, a1, t1
 ; RV32I-NEXT:    sw a0, 16(sp)
 ; RV32I-NEXT:    sw a0, 20(sp)
 ; RV32I-NEXT:    sw a0, 24(sp)
@@ -1240,28 +1240,28 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srli a0, a1, 3
 ; RV32I-NEXT:    andi a3, a1, 31
 ; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    add a0, a5, a0
-; RV32I-NEXT:    lw a4, 0(a0)
-; RV32I-NEXT:    lw a5, 4(a0)
-; RV32I-NEXT:    lw a6, 8(a0)
 ; RV32I-NEXT:    xori a3, a3, 31
+; RV32I-NEXT:    add a0, a5, a0
+; RV32I-NEXT:    lw a4, 4(a0)
+; RV32I-NEXT:    lw a5, 8(a0)
+; RV32I-NEXT:    lw a6, 0(a0)
 ; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    srl a7, a5, a1
-; RV32I-NEXT:    slli t0, a6, 1
-; RV32I-NEXT:    srl a4, a4, a1
-; RV32I-NEXT:    slli a5, a5, 1
+; RV32I-NEXT:    srl a7, a4, a1
+; RV32I-NEXT:    slli t0, a5, 1
 ; RV32I-NEXT:    srl a6, a6, a1
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl a5, a5, a1
 ; RV32I-NEXT:    slli t1, a0, 1
 ; RV32I-NEXT:    sra a0, a0, a1
 ; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a5, a5, a3
+; RV32I-NEXT:    sll a4, a4, a3
 ; RV32I-NEXT:    sll a3, t1, a3
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
 ; RV32I-NEXT:    or a1, a7, a1
-; RV32I-NEXT:    or a4, a4, a5
-; RV32I-NEXT:    or a3, a6, a3
+; RV32I-NEXT:    or a4, a6, a4
+; RV32I-NEXT:    or a3, a5, a3
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
@@ -1388,17 +1388,17 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    slli s7, s7, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, s7
@@ -1415,8 +1415,8 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t1, s0, t6
 ; RV64I-NEXT:    or t2, s5, s1
-; RV64I-NEXT:    or t3, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t3, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a3, a3, 32
 ; RV64I-NEXT:    slli a7, a7, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -1434,23 +1434,23 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli a0, a1, 3
 ; RV64I-NEXT:    andi a3, a1, 63
 ; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    add a0, a6, a0
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a5, 8(a0)
-; RV64I-NEXT:    ld a6, 16(a0)
 ; RV64I-NEXT:    xori a3, a3, 63
+; RV64I-NEXT:    add a0, a6, a0
+; RV64I-NEXT:    ld a4, 8(a0)
+; RV64I-NEXT:    ld a5, 16(a0)
+; RV64I-NEXT:    ld a6, 0(a0)
 ; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    srl a7, a5, a1
-; RV64I-NEXT:    slli t0, a6, 1
-; RV64I-NEXT:    srl a4, a4, a1
-; RV64I-NEXT:    slli a5, a5, 1
+; RV64I-NEXT:    srl a7, a4, a1
+; RV64I-NEXT:    slli t0, a5, 1
 ; RV64I-NEXT:    srl a6, a6, a1
+; RV64I-NEXT:    slli a4, a4, 1
+; RV64I-NEXT:    srl a5, a5, a1
 ; RV64I-NEXT:    slli t1, a0, 1
 ; RV64I-NEXT:    srl t2, a0, a1
 ; RV64I-NEXT:    sll a0, t0, a3
-; RV64I-NEXT:    sll a1, a5, a3
+; RV64I-NEXT:    sll a1, a4, a3
 ; RV64I-NEXT:    sll a3, t1, a3
-; RV64I-NEXT:    srli a5, t2, 56
+; RV64I-NEXT:    srli a4, t2, 56
 ; RV64I-NEXT:    srli t0, t2, 48
 ; RV64I-NEXT:    srli t1, t2, 40
 ; RV64I-NEXT:    srli t3, t2, 32
@@ -1458,12 +1458,12 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli t5, t2, 16
 ; RV64I-NEXT:    srli t6, t2, 8
 ; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    or a1, a4, a1
-; RV64I-NEXT:    or a3, a6, a3
+; RV64I-NEXT:    or a1, a6, a1
+; RV64I-NEXT:    or a3, a5, a3
 ; RV64I-NEXT:    sb t3, 28(a2)
 ; RV64I-NEXT:    sb t1, 29(a2)
 ; RV64I-NEXT:    sb t0, 30(a2)
-; RV64I-NEXT:    sb a5, 31(a2)
+; RV64I-NEXT:    sb a4, 31(a2)
 ; RV64I-NEXT:    sb t2, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
@@ -1864,17 +1864,17 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    slli s7, s7, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, s7
@@ -1891,8 +1891,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t1, s0, t6
 ; RV64I-NEXT:    or t2, s5, s1
-; RV64I-NEXT:    or t3, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t3, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a3, a3, 32
 ; RV64I-NEXT:    slli a7, a7, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -2340,17 +2340,17 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
 ; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    lbu s3, 4(a1)
-; RV64I-NEXT:    lbu s6, 5(a1)
 ; RV64I-NEXT:    slli s4, s4, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s4, s7, s4
+; RV64I-NEXT:    or s5, s6, s5
+; RV64I-NEXT:    or s2, s3, s2
+; RV64I-NEXT:    or s3, s7, s4
+; RV64I-NEXT:    lbu s4, 5(a1)
+; RV64I-NEXT:    lbu s6, 4(a1)
 ; RV64I-NEXT:    lbu s7, 6(a1)
 ; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s6, s6, 8
-; RV64I-NEXT:    or s3, s6, s3
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    or s4, s4, s6
 ; RV64I-NEXT:    slli s7, s7, 16
 ; RV64I-NEXT:    slli a1, a1, 24
 ; RV64I-NEXT:    or a1, a1, s7
@@ -2363,8 +2363,8 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, t5
 ; RV64I-NEXT:    or t0, s0, t6
 ; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s4, s2
-; RV64I-NEXT:    or a1, a1, s3
+; RV64I-NEXT:    or t2, s3, s2
+; RV64I-NEXT:    or a1, a1, s4
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
 ; RV64I-NEXT:    slli a0, a0, 32
@@ -2387,23 +2387,23 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli a0, a1, 3
 ; RV64I-NEXT:    andi a3, a1, 63
 ; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    add a0, s6, a0
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a5, 8(a0)
-; RV64I-NEXT:    ld a6, 16(a0)
 ; RV64I-NEXT:    xori a3, a3, 63
+; RV64I-NEXT:    add a0, s6, a0
+; RV64I-NEXT:    ld a4, 8(a0)
+; RV64I-NEXT:    ld a5, 16(a0)
+; RV64I-NEXT:    ld a6, 0(a0)
 ; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    srl a7, a5, a1
-; RV64I-NEXT:    slli t0, a6, 1
-; RV64I-NEXT:    srl a4, a4, a1
-; RV64I-NEXT:    slli a5, a5, 1
+; RV64I-NEXT:    srl a7, a4, a1
+; RV64I-NEXT:    slli t0, a5, 1
 ; RV64I-NEXT:    srl a6, a6, a1
+; RV64I-NEXT:    slli a4, a4, 1
+; RV64I-NEXT:    srl a5, a5, a1
 ; RV64I-NEXT:    slli t1, a0, 1
 ; RV64I-NEXT:    sra t2, a0, a1
 ; RV64I-NEXT:    sll a0, t0, a3
-; RV64I-NEXT:    sll a1, a5, a3
+; RV64I-NEXT:    sll a1, a4, a3
 ; RV64I-NEXT:    sll a3, t1, a3
-; RV64I-NEXT:    srli a5, t2, 56
+; RV64I-NEXT:    srli a4, t2, 56
 ; RV64I-NEXT:    srli t0, t2, 48
 ; RV64I-NEXT:    srli t1, t2, 40
 ; RV64I-NEXT:    srli t3, t2, 32
@@ -2411,12 +2411,12 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli t5, t2, 16
 ; RV64I-NEXT:    srli t6, t2, 8
 ; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    or a1, a4, a1
-; RV64I-NEXT:    or a3, a6, a3
+; RV64I-NEXT:    or a1, a6, a1
+; RV64I-NEXT:    or a3, a5, a3
 ; RV64I-NEXT:    sb t3, 28(a2)
 ; RV64I-NEXT:    sb t1, 29(a2)
 ; RV64I-NEXT:    sb t0, 30(a2)
-; RV64I-NEXT:    sb a5, 31(a2)
+; RV64I-NEXT:    sb a4, 31(a2)
 ; RV64I-NEXT:    sb t2, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/xqcibm-extract.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
index 3f5b949585fa..edf6e9a2d501 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
@@ -231,3 +231,193 @@ define i64 @sexti32_i64_2(i32 %a) {
   %1 = sext i32 %a to i64
   ret i64 %1
 }
+
+define i32 @extu_from_and_i32(i32 %x) {
+; RV32I-LABEL: extu_from_and_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 20
+; RV32I-NEXT:    srli a0, a0, 20
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_and_i32:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 0
+; RV32XQCIBM-NEXT:    ret
+  %a = and i32 %x, 4095
+  ret i32 %a
+}
+
+define i32 @no_extu_from_and_i32(i32 %x) {
+; RV32I-LABEL: no_extu_from_and_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, 31
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: no_extu_from_and_i32:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    andi a0, a0, 31
+; RV32XQCIBM-NEXT:    ret
+  %a = and i32 %x, 31
+  ret i32 %a
+}
+
+define i32 @extu_from_and_i32_simm12_lb(i32 %x) {
+; RV32I-LABEL: extu_from_and_i32_simm12_lb:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, 63
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_and_i32_simm12_lb:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 6, 0
+; RV32XQCIBM-NEXT:    ret
+  %a = and i32 %x, 63
+  ret i32 %a
+}
+
+define i32 @extu_from_and_i32_simm12_ub(i32 %x) {
+; RV32I-LABEL: extu_from_and_i32_simm12_ub:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, 2047
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_and_i32_simm12_ub:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 11, 0
+; RV32XQCIBM-NEXT:    ret
+  %a = and i32 %x, 2047
+  ret i32 %a
+}
+
+define i64 @extu_from_and_i64(i64 %x) {
+; RV32I-LABEL: extu_from_and_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 20
+; RV32I-NEXT:    srli a0, a0, 20
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_and_i64:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 0
+; RV32XQCIBM-NEXT:    li a1, 0
+; RV32XQCIBM-NEXT:    ret
+  %a = and i64 %x, 4095
+  ret i64 %a
+}
+
+define i32 @extu_from_and_lshr_i32(i32 %x) {
+; RV32I-LABEL: extu_from_and_lshr_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 6
+; RV32I-NEXT:    srli a0, a0, 29
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_and_lshr_i32:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 3, 23
+; RV32XQCIBM-NEXT:    ret
+  %shifted = lshr i32 %x, 23
+  %masked = and i32 %shifted, 7
+  ret i32 %masked
+}
+
+define i64 @extu_from_and_lshr_i64(i64 %x) {
+; RV32I-LABEL: extu_from_and_lshr_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a1, 6
+; RV32I-NEXT:    srli a0, a0, 20
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_and_lshr_i64:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a1, 12, 14
+; RV32XQCIBM-NEXT:    li a1, 0
+; RV32XQCIBM-NEXT:    ret
+  %shifted = lshr i64 %x, 46
+  %masked = and i64 %shifted, 4095
+  ret i64 %masked
+}
+
+define i32 @extu_from_lshr_and_i32(i32 %x) {
+; RV32I-LABEL: extu_from_lshr_and_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    srli a0, a0, 20
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_lshr_and_i32:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 12
+; RV32XQCIBM-NEXT:    ret
+  %masked = and i32 %x, 16773120
+  %shifted = lshr i32 %masked, 12
+  ret i32 %shifted
+}
+
+define i64 @extu_from_lshr_and_i64(i64 %x) {
+; RV32I-LABEL: extu_from_lshr_and_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    srli a0, a0, 20
+; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: extu_from_lshr_and_i64:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 12
+; RV32XQCIBM-NEXT:    li a1, 0
+; RV32XQCIBM-NEXT:    ret
+  %masked = and i64 %x, 16773120
+  %shifted = lshr i64 %masked, 12
+  ret i64 %shifted
+}
+
+define i32 @ext_from_ashr_shl_i32(i32 %x) {
+; RV32I-LABEL: ext_from_ashr_shl_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: ext_from_ashr_shl_i32:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.ext a0, a0, 8, 16
+; RV32XQCIBM-NEXT:    ret
+  %shl = shl i32 %x, 8
+  %ashr = ashr i32 %shl, 24
+  ret i32 %ashr
+}
+
+define i32 @ext_from_ashr_sexti8_i32(i8 %x) {
+; RV32I-LABEL: ext_from_ashr_sexti8_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    srai a0, a0, 29
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: ext_from_ashr_sexti8_i32:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.ext a0, a0, 3, 5
+; RV32XQCIBM-NEXT:    ret
+  %sext = sext i8 %x to i32
+  %ashr = ashr i32 %sext, 5
+  ret i32 %ashr
+}
+
+define i32 @ext_from_ashr_sexti16_i32(i16 %x) {
+; RV32I-LABEL: ext_from_ashr_sexti16_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: ext_from_ashr_sexti16_i32:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.ext a0, a0, 1, 15
+; RV32XQCIBM-NEXT:    ret
+  %sext = sext i16 %x to i32
+  %ashr = ashr i32 %sext, 24
+  ret i32 %ashr
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/xtheadmempair.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/xtheadmempair.ll
index 3525c4002606..4df61dad7d03 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/xtheadmempair.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/xtheadmempair.ll
@@ -57,14 +57,14 @@ define i64 @lwud(ptr %a) {
 define i64 @ldd(ptr %a) {
 ; RV32XTHEADMEMPAIR-LABEL: ldd:
 ; RV32XTHEADMEMPAIR:       # %bb.0:
-; RV32XTHEADMEMPAIR-NEXT:    lw a1, 44(a0)
-; RV32XTHEADMEMPAIR-NEXT:    lw a2, 32(a0)
-; RV32XTHEADMEMPAIR-NEXT:    lw a3, 36(a0)
+; RV32XTHEADMEMPAIR-NEXT:    lw a1, 32(a0)
+; RV32XTHEADMEMPAIR-NEXT:    lw a2, 36(a0)
+; RV32XTHEADMEMPAIR-NEXT:    lw a3, 44(a0)
 ; RV32XTHEADMEMPAIR-NEXT:    lw a0, 40(a0)
-; RV32XTHEADMEMPAIR-NEXT:    add a1, a3, a1
-; RV32XTHEADMEMPAIR-NEXT:    add a0, a2, a0
-; RV32XTHEADMEMPAIR-NEXT:    sltu a2, a0, a2
-; RV32XTHEADMEMPAIR-NEXT:    add a1, a1, a2
+; RV32XTHEADMEMPAIR-NEXT:    add a2, a2, a3
+; RV32XTHEADMEMPAIR-NEXT:    add a0, a1, a0
+; RV32XTHEADMEMPAIR-NEXT:    sltu a1, a0, a1
+; RV32XTHEADMEMPAIR-NEXT:    add a1, a2, a1
 ; RV32XTHEADMEMPAIR-NEXT:    ret
 ;
 ; RV64XTHEADMEMPAIR-LABEL: ldd:
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/zbb-logic-neg-imm.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/zbb-logic-neg-imm.ll
index 449e983fb6b5..751b05ebdba2 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/zbb-logic-neg-imm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/zbb-logic-neg-imm.ll
@@ -97,7 +97,7 @@ define i32 @addorlow16(i32 %x) {
 ; RV64-LABEL: addorlow16:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
+; RV64-NEXT:    addi a1, a1, -1
 ; RV64-NEXT:    addw a0, a0, a1
 ; RV64-NEXT:    or a0, a0, a1
 ; RV64-NEXT:    ret
@@ -107,19 +107,12 @@ define i32 @addorlow16(i32 %x) {
 }
 
 define i32 @andxorlow16(i32 %x) {
-; RV32-LABEL: andxorlow16:
-; RV32:       # %bb.0:
-; RV32-NEXT:    lui a1, 16
-; RV32-NEXT:    addi a1, a1, -1
-; RV32-NEXT:    andn a0, a1, a0
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: andxorlow16:
-; RV64:       # %bb.0:
-; RV64-NEXT:    lui a1, 16
-; RV64-NEXT:    addiw a1, a1, -1
-; RV64-NEXT:    andn a0, a1, a0
-; RV64-NEXT:    ret
+; CHECK-LABEL: andxorlow16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a1, 16
+; CHECK-NEXT:    addi a1, a1, -1
+; CHECK-NEXT:    andn a0, a1, a0
+; CHECK-NEXT:    ret
   %and = and i32 %x, 65535
   %xor = xor i32 %and, 65535
   ret i32 %xor
@@ -385,3 +378,41 @@ define i64 @xornofff(i64 %x) {
   %xor = xor i64 %x, -1152921504606846721
   ret i64 %xor
 }
+
+define i64 @and_or_or(i64 %x, i64 %y) {
+; RV32-LABEL: and_or_or:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ori a1, a1, -2
+; RV32-NEXT:    ori a0, a0, 1
+; RV32-NEXT:    ori a3, a3, 1
+; RV32-NEXT:    ori a2, a2, -2
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    ret
+;
+; NOZBS64-LABEL: and_or_or:
+; NOZBS64:       # %bb.0:
+; NOZBS64-NEXT:    li a2, -1
+; NOZBS64-NEXT:    slli a2, a2, 33
+; NOZBS64-NEXT:    addi a2, a2, 1
+; NOZBS64-NEXT:    or a0, a0, a2
+; NOZBS64-NEXT:    li a2, 1
+; NOZBS64-NEXT:    slli a2, a2, 33
+; NOZBS64-NEXT:    addi a2, a2, -2
+; NOZBS64-NEXT:    or a1, a1, a2
+; NOZBS64-NEXT:    and a0, a0, a1
+; NOZBS64-NEXT:    ret
+;
+; ZBS64-LABEL: and_or_or:
+; ZBS64:       # %bb.0:
+; ZBS64-NEXT:    bseti a2, zero, 33
+; ZBS64-NEXT:    addi a2, a2, -2
+; ZBS64-NEXT:    orn a0, a0, a2
+; ZBS64-NEXT:    or a1, a1, a2
+; ZBS64-NEXT:    and a0, a0, a1
+; ZBS64-NEXT:    ret
+  %a = or i64 %x, -8589934591
+  %b = or i64 %y, 8589934590
+  %c = and i64 %a, %b
+  ret i64 %c
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/zicond-opts.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/zicond-opts.ll
new file mode 100644
index 000000000000..2512ba803cf4
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/zicond-opts.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -O2 -verify-machineinstrs -mattr=+b,+zicond < %s | FileCheck %s -check-prefix=RV32ZICOND
+; RUN: llc -mtriple=riscv64 -O2 -verify-machineinstrs -mattr=+b,+zicond < %s | FileCheck %s -check-prefix=RV64ZICOND
+
+; (and (icmp x. 0, ne), (icmp y, 0, ne)) -> (czero.eqz (icmp x, 0, ne), y)
+define i32 @icmp_and(i64 %x, i64 %y) {
+; RV32ZICOND-LABEL: icmp_and:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    or a2, a2, a3
+; RV32ZICOND-NEXT:    or a0, a0, a1
+; RV32ZICOND-NEXT:    snez a1, a2
+; RV32ZICOND-NEXT:    snez a0, a0
+; RV32ZICOND-NEXT:    and a0, a0, a1
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: icmp_and:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    snez a1, a1
+; RV64ZICOND-NEXT:    snez a0, a0
+; RV64ZICOND-NEXT:    and a0, a0, a1
+; RV64ZICOND-NEXT:    ret
+  %3 = icmp ne i64 %y, 0
+  %4 = icmp ne i64 %x, 0
+  %5 = and i1 %4, %3
+  %6 = zext i1 %5 to i32
+  ret i32 %6
+}
+
+; (and (and (icmp x, 0, ne), (icmp y, 0, ne)), (icmp z, 0, ne)) -> (czero.eqz (czero.eqz (icmp x, 0, ne), y), z)
+define i32 @icmp_and_and(i64 %x, i64 %y, i64 %z) {
+; RV32ZICOND-LABEL: icmp_and_and:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    or a2, a2, a3
+; RV32ZICOND-NEXT:    or a0, a0, a1
+; RV32ZICOND-NEXT:    or a4, a4, a5
+; RV32ZICOND-NEXT:    snez a1, a2
+; RV32ZICOND-NEXT:    snez a0, a0
+; RV32ZICOND-NEXT:    and a0, a1, a0
+; RV32ZICOND-NEXT:    snez a1, a4
+; RV32ZICOND-NEXT:    and a0, a1, a0
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: icmp_and_and:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    snez a1, a1
+; RV64ZICOND-NEXT:    snez a0, a0
+; RV64ZICOND-NEXT:    and a0, a1, a0
+; RV64ZICOND-NEXT:    snez a1, a2
+; RV64ZICOND-NEXT:    and a0, a1, a0
+; RV64ZICOND-NEXT:    ret
+  %4 = icmp ne i64 %y, 0
+  %5 = icmp ne i64 %x, 0
+  %6 = and i1 %4, %5
+  %7 = icmp ne i64 %z, 0
+  %8 = and i1 %7, %6
+  %9 = zext i1 %8 to i32
+  ret i32 %9
+}
+
+; (select cond, x, rotl(x, rot.amt)) -> (rotl x, (czero_nez rot.amt, cond))
+define i64 @rotate_l_nez(i64 %x, i64 %rot.amt, i1 %cond) {
+; RV32ZICOND-LABEL: rotate_l_nez:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    andi a4, a4, 1
+; RV32ZICOND-NEXT:    bexti a3, a2, 5
+; RV32ZICOND-NEXT:    not a5, a2
+; RV32ZICOND-NEXT:    czero.nez a6, a1, a3
+; RV32ZICOND-NEXT:    czero.eqz a7, a0, a3
+; RV32ZICOND-NEXT:    czero.nez t0, a0, a3
+; RV32ZICOND-NEXT:    czero.eqz a3, a1, a3
+; RV32ZICOND-NEXT:    czero.eqz a0, a0, a4
+; RV32ZICOND-NEXT:    czero.eqz a1, a1, a4
+; RV32ZICOND-NEXT:    or a6, a7, a6
+; RV32ZICOND-NEXT:    or a3, a3, t0
+; RV32ZICOND-NEXT:    sll a7, a6, a2
+; RV32ZICOND-NEXT:    srli t0, a3, 1
+; RV32ZICOND-NEXT:    sll a2, a3, a2
+; RV32ZICOND-NEXT:    srli a3, a6, 1
+; RV32ZICOND-NEXT:    srl a6, t0, a5
+; RV32ZICOND-NEXT:    srl a3, a3, a5
+; RV32ZICOND-NEXT:    or a5, a7, a6
+; RV32ZICOND-NEXT:    or a2, a2, a3
+; RV32ZICOND-NEXT:    czero.nez a2, a2, a4
+; RV32ZICOND-NEXT:    czero.nez a3, a5, a4
+; RV32ZICOND-NEXT:    or a0, a0, a2
+; RV32ZICOND-NEXT:    or a1, a1, a3
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: rotate_l_nez:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a2, a2, 1
+; RV64ZICOND-NEXT:    rol a1, a0, a1
+; RV64ZICOND-NEXT:    czero.nez a1, a1, a2
+; RV64ZICOND-NEXT:    czero.eqz a0, a0, a2
+; RV64ZICOND-NEXT:    or a0, a0, a1
+; RV64ZICOND-NEXT:    ret
+  %6 = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %rot.amt)
+  %7 = select i1 %cond, i64 %x, i64 %6
+  ret i64 %7
+}
+
+; (select cond, rotl(x, rot.amt), x) -> (rotl x, (czero_eqz rot.amt, cond))
+define i64 @rotate_l_eqz(i64 %x, i64 %rot.amt, i1 %cond) {
+; RV32ZICOND-LABEL: rotate_l_eqz:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    andi a4, a4, 1
+; RV32ZICOND-NEXT:    bexti a3, a2, 5
+; RV32ZICOND-NEXT:    not a5, a2
+; RV32ZICOND-NEXT:    czero.nez a6, a1, a3
+; RV32ZICOND-NEXT:    czero.eqz a7, a0, a3
+; RV32ZICOND-NEXT:    czero.nez t0, a0, a3
+; RV32ZICOND-NEXT:    czero.eqz a3, a1, a3
+; RV32ZICOND-NEXT:    czero.nez a0, a0, a4
+; RV32ZICOND-NEXT:    czero.nez a1, a1, a4
+; RV32ZICOND-NEXT:    or a6, a7, a6
+; RV32ZICOND-NEXT:    or a3, a3, t0
+; RV32ZICOND-NEXT:    sll a7, a6, a2
+; RV32ZICOND-NEXT:    srli t0, a3, 1
+; RV32ZICOND-NEXT:    sll a2, a3, a2
+; RV32ZICOND-NEXT:    srli a3, a6, 1
+; RV32ZICOND-NEXT:    srl a6, t0, a5
+; RV32ZICOND-NEXT:    srl a3, a3, a5
+; RV32ZICOND-NEXT:    or a5, a7, a6
+; RV32ZICOND-NEXT:    or a2, a2, a3
+; RV32ZICOND-NEXT:    czero.eqz a2, a2, a4
+; RV32ZICOND-NEXT:    czero.eqz a3, a5, a4
+; RV32ZICOND-NEXT:    or a0, a2, a0
+; RV32ZICOND-NEXT:    or a1, a3, a1
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: rotate_l_eqz:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a2, a2, 1
+; RV64ZICOND-NEXT:    rol a1, a0, a1
+; RV64ZICOND-NEXT:    czero.nez a0, a0, a2
+; RV64ZICOND-NEXT:    czero.eqz a1, a1, a2
+; RV64ZICOND-NEXT:    or a0, a1, a0
+; RV64ZICOND-NEXT:    ret
+  %6 = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %rot.amt)
+  %7 = select i1 %cond, i64 %6, i64 %x
+  ret i64 %7
+}
+
+; (select cond, const, t) -> (add (czero_nez t - const, cond), const)
+define i64 @select_imm_reg(i64 %t, i1 %cond) {
+; RV32ZICOND-LABEL: select_imm_reg:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    andi a2, a2, 1
+; RV32ZICOND-NEXT:    addi a0, a0, -3
+; RV32ZICOND-NEXT:    czero.nez a1, a1, a2
+; RV32ZICOND-NEXT:    czero.nez a0, a0, a2
+; RV32ZICOND-NEXT:    addi a0, a0, 3
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: select_imm_reg:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a1, a1, 1
+; RV64ZICOND-NEXT:    addi a0, a0, -3
+; RV64ZICOND-NEXT:    czero.nez a0, a0, a1
+; RV64ZICOND-NEXT:    addi a0, a0, 3
+; RV64ZICOND-NEXT:    ret
+  %4 = select i1 %cond, i64 3, i64 %t
+  ret i64 %4
+}
+
+; (select cond, t, const) -> (add (czero_eqz t - const, cond), const)
+define i64 @select_reg_imm(i64 %t, i1 %cond) {
+; RV32ZICOND-LABEL: select_reg_imm:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    andi a2, a2, 1
+; RV32ZICOND-NEXT:    addi a0, a0, -3
+; RV32ZICOND-NEXT:    czero.eqz a1, a1, a2
+; RV32ZICOND-NEXT:    czero.eqz a0, a0, a2
+; RV32ZICOND-NEXT:    addi a0, a0, 3
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: select_reg_imm:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a1, a1, 1
+; RV64ZICOND-NEXT:    addi a0, a0, -3
+; RV64ZICOND-NEXT:    czero.eqz a0, a0, a1
+; RV64ZICOND-NEXT:    addi a0, a0, 3
+; RV64ZICOND-NEXT:    ret
+  %4 = select i1 %cond, i64 %t, i64 3
+  ret i64 %4
+}
+
+; (select cond, -2048, t) -> (xor (czero_nez (xor t, -2048), cond), -2048)
+define i64 @select_imm_reg_neg_2048(i64 %t, i1 %cond) {
+; RV32ZICOND-LABEL: select_imm_reg_neg_2048:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    andi a2, a2, 1
+; RV32ZICOND-NEXT:    xori a0, a0, -2048
+; RV32ZICOND-NEXT:    neg a3, a2
+; RV32ZICOND-NEXT:    czero.nez a0, a0, a2
+; RV32ZICOND-NEXT:    or a1, a3, a1
+; RV32ZICOND-NEXT:    xori a0, a0, -2048
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: select_imm_reg_neg_2048:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a1, a1, 1
+; RV64ZICOND-NEXT:    xori a0, a0, -2048
+; RV64ZICOND-NEXT:    czero.nez a0, a0, a1
+; RV64ZICOND-NEXT:    xori a0, a0, -2048
+; RV64ZICOND-NEXT:    ret
+  %4 = select i1 %cond, i64 -2048, i64 %t
+  ret i64 %4
+}
+
+; (select cond, 2048, t) -> no transform
+define i64 @select_imm_reg_2048(i64 %t, i1 %cond) {
+; RV32ZICOND-LABEL: select_imm_reg_2048:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    andi a2, a2, 1
+; RV32ZICOND-NEXT:    bseti a3, zero, 11
+; RV32ZICOND-NEXT:    czero.nez a0, a0, a2
+; RV32ZICOND-NEXT:    czero.eqz a3, a3, a2
+; RV32ZICOND-NEXT:    or a0, a3, a0
+; RV32ZICOND-NEXT:    czero.nez a1, a1, a2
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: select_imm_reg_2048:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a1, a1, 1
+; RV64ZICOND-NEXT:    bseti a2, zero, 11
+; RV64ZICOND-NEXT:    czero.nez a0, a0, a1
+; RV64ZICOND-NEXT:    czero.eqz a1, a2, a1
+; RV64ZICOND-NEXT:    or a0, a1, a0
+; RV64ZICOND-NEXT:    ret
+  %4 = select i1 %cond, i64 2048, i64 %t
+  ret i64 %4
+}
+
+; (select cond, (and f, ~x), f) -> (andn f, (czero_eqz x, cond))
+define i64 @test_inv_and_nez(i64 %f, i64 %x, i1 %cond) {
+; RV32ZICOND-LABEL: test_inv_and_nez:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    andi a4, a4, 1
+; RV32ZICOND-NEXT:    addi a4, a4, -1
+; RV32ZICOND-NEXT:    orn a3, a4, a3
+; RV32ZICOND-NEXT:    orn a2, a4, a2
+; RV32ZICOND-NEXT:    and a0, a2, a0
+; RV32ZICOND-NEXT:    and a1, a3, a1
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: test_inv_and_nez:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a2, a2, 1
+; RV64ZICOND-NEXT:    andn a1, a0, a1
+; RV64ZICOND-NEXT:    czero.nez a0, a0, a2
+; RV64ZICOND-NEXT:    or a0, a1, a0
+; RV64ZICOND-NEXT:    ret
+  %5 = xor i64 %x, -1
+  %6 = select i1 %cond, i64 %5, i64 -1
+  %7 = and i64 %6, %f
+  ret i64 %7
+}
+
+; (select cond, f, (and f, ~x)) -> (andn f, (czero_nez x, cond))
+define i64 @test_inv_and_eqz(i64 %f, i64 %x, i1 %cond) {
+; RV32ZICOND-LABEL: test_inv_and_eqz:
+; RV32ZICOND:       # %bb.0:
+; RV32ZICOND-NEXT:    slli a4, a4, 31
+; RV32ZICOND-NEXT:    srai a4, a4, 31
+; RV32ZICOND-NEXT:    orn a3, a4, a3
+; RV32ZICOND-NEXT:    orn a2, a4, a2
+; RV32ZICOND-NEXT:    and a0, a2, a0
+; RV32ZICOND-NEXT:    and a1, a3, a1
+; RV32ZICOND-NEXT:    ret
+;
+; RV64ZICOND-LABEL: test_inv_and_eqz:
+; RV64ZICOND:       # %bb.0:
+; RV64ZICOND-NEXT:    andi a2, a2, 1
+; RV64ZICOND-NEXT:    andn a1, a0, a1
+; RV64ZICOND-NEXT:    czero.eqz a0, a0, a2
+; RV64ZICOND-NEXT:    or a0, a1, a0
+; RV64ZICOND-NEXT:    ret
+  %5 = xor i64 %x, -1
+  %6 = select i1 %cond, i64 -1, i64 %5
+  %7 = and i64 %6, %f
+  ret i64 %7
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/RISCV/zilsd.ll b/external/llvm-project/llvm/test/CodeGen/RISCV/zilsd.ll
index 6a4578bb02d8..09b065a8e9dd 100644
--- a/external/llvm-project/llvm/test/CodeGen/RISCV/zilsd.ll
+++ b/external/llvm-project/llvm/test/CodeGen/RISCV/zilsd.ll
@@ -36,25 +36,25 @@ define i64 @load_unaligned(ptr %p) {
 ; SLOW-LABEL: load_unaligned:
 ; SLOW:       # %bb.0:
 ; SLOW-NEXT:    lbu a1, 1(a0)
-; SLOW-NEXT:    lbu a2, 0(a0)
-; SLOW-NEXT:    lbu a3, 2(a0)
-; SLOW-NEXT:    lbu a4, 3(a0)
+; SLOW-NEXT:    lbu a2, 2(a0)
+; SLOW-NEXT:    lbu a3, 3(a0)
+; SLOW-NEXT:    lbu a4, 0(a0)
 ; SLOW-NEXT:    slli a1, a1, 8
-; SLOW-NEXT:    or a1, a1, a2
-; SLOW-NEXT:    lbu a2, 4(a0)
-; SLOW-NEXT:    lbu a5, 5(a0)
-; SLOW-NEXT:    slli a3, a3, 16
-; SLOW-NEXT:    slli a4, a4, 24
-; SLOW-NEXT:    or a3, a4, a3
-; SLOW-NEXT:    lbu a4, 6(a0)
+; SLOW-NEXT:    slli a2, a2, 16
+; SLOW-NEXT:    slli a3, a3, 24
+; SLOW-NEXT:    or a1, a1, a4
+; SLOW-NEXT:    or a2, a3, a2
+; SLOW-NEXT:    lbu a3, 5(a0)
+; SLOW-NEXT:    lbu a4, 4(a0)
+; SLOW-NEXT:    lbu a5, 6(a0)
 ; SLOW-NEXT:    lbu a0, 7(a0)
-; SLOW-NEXT:    slli a5, a5, 8
-; SLOW-NEXT:    or a2, a5, a2
-; SLOW-NEXT:    slli a4, a4, 16
+; SLOW-NEXT:    slli a3, a3, 8
+; SLOW-NEXT:    or a3, a3, a4
+; SLOW-NEXT:    slli a5, a5, 16
 ; SLOW-NEXT:    slli a0, a0, 24
-; SLOW-NEXT:    or a4, a0, a4
-; SLOW-NEXT:    or a0, a3, a1
-; SLOW-NEXT:    or a1, a4, a2
+; SLOW-NEXT:    or a5, a0, a5
+; SLOW-NEXT:    or a0, a2, a1
+; SLOW-NEXT:    or a1, a5, a3
 ; SLOW-NEXT:    ret
 ;
 ; FAST-LABEL: load_unaligned:
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/ExecutionMode_Fragment.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/ExecutionMode_Fragment.ll
new file mode 100644
index 000000000000..aab0ae05753f
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/ExecutionMode_Fragment.ll
@@ -0,0 +1,19 @@
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3-pixel %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-pixel %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK-DAG: OpEntryPoint Fragment %[[#entry:]] "main" {{.*}}
+; CHECK-DAG: OpExecutionMode %[[#entry]] OriginUpperLeft
+
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
+define void @main() #0 {
+entry:
+  %0 = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_1t(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+  %1 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i32_12_1t(target("spirv.VulkanBuffer", [0 x i32], 12, 1) %0, i32 0)
+  store i32 1, ptr addrspace(11) %1, align 4
+
+  ret void
+}
+
+attributes #0 = { "hlsl.shader"="pixel" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(none) }
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/ExecutionMode_Vertex.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/ExecutionMode_Vertex.ll
new file mode 100644
index 000000000000..e93e7a9c302e
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/ExecutionMode_Vertex.ll
@@ -0,0 +1,12 @@
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3-vertex %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-vertex %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK: OpCapability Shader
+; CHECK: OpEntryPoint Vertex %[[#entry:]] "main"
+
+define void @main() #1 {
+entry:
+  ret void
+}
+
+attributes #1 = { "hlsl.shader"="vertex" }
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/OpVariable_order.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/OpVariable_order.ll
index c68250697c4a..1e94be088630 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/OpVariable_order.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/OpVariable_order.ll
@@ -10,12 +10,10 @@
 ; CHECK-SPIRV: OpReturn
 ; CHECK-SPIRV: OpFunctionEnd
 
-define void @main() #1 {
+define void @main() {
 entry:
   %0 = alloca <2 x i32>, align 4
   %1 = getelementptr <2 x i32>, ptr %0, i32 0, i32 0
   %2 = alloca float, align 4
   ret void
 }
-
-attributes #1 = { "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" }
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/const-nested-vecs.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/const-nested-vecs.ll
index 9234106e5fcd..266b46e65f31 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/const-nested-vecs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/const-nested-vecs.ll
@@ -25,8 +25,8 @@
 ; CHECK-SPIRV-DAG: %[[#IntZero:]] = OpConstantNull %[[#IntTy]]
 ; CHECK-SPIRV-DAG: %[[#LongZero:]] = OpConstantNull %[[#LongTy]]
 ; CHECK-SPIRV64-DAG: %[[#ConstLong2:]] = OpConstant %[[#LongTy]] 2
-; CHECK-SPIRV64-DAG: %[[#PvarInit:]] = OpSpecConstantOp %[[#PtrCharTy]] 70 %[[#VarV2Char:]] %[[#IntZero]] %[[#ConstLong2]]
-; CHECK-SPIRV32-DAG: %[[#PvarInit:]] = OpSpecConstantOp %[[#PtrCharTy]] 70 %[[#VarV2Char:]] %[[#IntZero]] %[[#Const2]]
+; CHECK-SPIRV64-DAG: %[[#PvarInit:]] = OpSpecConstantOp %[[#PtrCharTy]] InBoundsPtrAccessChain %[[#VarV2Char:]] %[[#IntZero]] %[[#ConstLong2]]
+; CHECK-SPIRV32-DAG: %[[#PvarInit:]] = OpSpecConstantOp %[[#PtrCharTy]] InBoundsPtrAccessChain %[[#VarV2Char:]] %[[#IntZero]] %[[#Const2]]
 ; CHECK-SPIRV-DAG: %[[#PtrPtrCharTy:]] = OpTypePointer CrossWorkgroup %[[#PtrCharTy]]
 ; CHECK-SPIRV-DAG: %[[#AVar]] = OpVariable %[[#PtrArr2V2CharTy]] CrossWorkgroup %[[#Arr2V2Char]]
 ; CHECK-SPIRV-DAG: %[[#PVar]] = OpVariable %[[#PtrPtrCharTy]] CrossWorkgroup %[[#PvarInit]]
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-logical.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-logical.ll
index 1c6604006e2c..eeacd31953f4 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-logical.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-logical.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ;; Ensure the required Capabilities are listed.
 ; CHECK-DAG: OpCapability Shader
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-module.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-module.ll
index b7cea042ee5d..30a8a24e832e 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-module.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/empty-module.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOOCL
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOOCL
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG:        OpCapability Linkage
 ; CHECK-NOOCL-DAG:  OpCapability Shader
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fun-ptr-addrcast.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fun-ptr-addrcast.ll
index 8edecc1329d0..e5736b88b63a 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fun-ptr-addrcast.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fun-ptr-addrcast.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - --spirv-ext=+SPV_INTEL_function_pointers | FileCheck %s
 ; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; CHECK-COUNT-3: %[[#]] = OpSpecConstantOp %[[#]] 121 %[[#]]
+; CHECK-COUNT-3: %[[#]] = OpSpecConstantOp %[[#]] PtrCastToGeneric %[[#]]
 ; CHECK-COUNT-3: OpPtrCastToGeneric
 
 @G1 = addrspace(1) constant { [3 x ptr addrspace(4)] } { [3 x ptr addrspace(4)] [ptr addrspace(4) null, ptr addrspace(4) addrspacecast (ptr @foo to ptr addrspace(4)), ptr addrspace(4) addrspacecast (ptr @bar to ptr addrspace(4))] }
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/global-var-name-align.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/global-var-name-align.ll
index d73c98e55b87..173c3dc608e5 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/global-var-name-align.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/global-var-name-align.ll
@@ -6,8 +6,8 @@
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s -check-prefixes=CHECK,OCL
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s -check-prefixes=CHECK,VK
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s -check-prefixes=CHECK,VK
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: OpName %[[#id18:]] "G1"
 ; CHECK-DAG: OpName %[[#id22:]] "g1"
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/AddUint64.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/AddUint64.ll
index 1e7243e37b4a..a97492b8453e 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/AddUint64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/AddUint64.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Code here is an excerpt of clang/test/CodeGenHLSL/builtins/AddUint64.hlsl compiled for spirv using the following command
 ; clang -cc1 -finclude-default-header -triple spirv-unknown-vulkan-compute clang/test/CodeGenHLSL/builtins/AddUint64.hlsl -emit-llvm -disable-llvm-passes -o llvm/test/CodeGen/SPIRV/hlsl-intrinsics/uadd_with_overflow.ll
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll
index 3b4ff4f293c6..4a15fa8b1453 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/abs.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll
index 9726bfbf1ada..8b74135f89a2 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/acos.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll
index 0980a28f4e8e..1850ff97b138 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/all.ll
@@ -1,6 +1,6 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; Make sure spirv operation function calls for all are generated.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
index 5d805202be5a..7ba0f9a632fe 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
@@ -1,6 +1,6 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 ; Make sure spirv operation function calls for any are generated.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll
index 73b29b6a264b..29848aa471d4 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/asin.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll
index f0ee310d3020..1a2af12affe1 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
index 922e017b0d5a..e13b3a8296ac 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll
index 29ac39ef9b41..14ad4e6eef3f 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ceil.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
 define noundef float @ceil_float(float noundef %a) {
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/clamp.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/clamp.ll
index 7ad786f78974..124aba0746c2 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/clamp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/clamp.ll
@@ -1,5 +1,5 @@
-; RUN: llc  -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc  -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext:]] = OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll
index 09f1cf07267f..239b668644e1 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cos.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll
index bc6dd75bce94..9dad03aa6f91 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cosh.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll
index 4fe0a6fff50d..69f01dacc74a 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/countbits.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpMemoryModel Logical GLSL450
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cross.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cross.ll
index b1625c07111e..bcb8cd32b2a9 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cross.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cross.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for cross are lowered correctly.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll
index 5945bc390744..d2d929b152cd 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/degrees.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/discard.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/discard.ll
index 831592610d10..60fd1c8a8d17 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/discard.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/discard.ll
@@ -1,9 +1,9 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.5-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV15
-; RUN: llc -verify-machineinstrs -spirv-ext=+SPV_EXT_demote_to_helper_invocation -O0 -mtriple=spirv1.5-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV16,WITH-EXTENSION,WITH-CAPABILITY
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.6-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV16,WITH-CAPABILITY
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -O0 -spirv-ext=+SPV_EXT_demote_to_helper_invocation -mtriple=spirv1.5-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.5-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV15
+; RUN: llc -verify-machineinstrs -spirv-ext=+SPV_EXT_demote_to_helper_invocation -O0 -mtriple=spirv1.5-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV16,WITH-EXTENSION,WITH-CAPABILITY
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.6-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV16,WITH-CAPABILITY
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -spirv-ext=+SPV_EXT_demote_to_helper_invocation -mtriple=spirv1.5-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 
 ; Make sure lowering is correctly generating spirv code.
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/distance.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/distance.ll
index cb92f775eef3..53171780b90f 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/distance.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/distance.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for distance are lowered correctly.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_i8packed.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_i8packed.ll
index 222aca53ab6a..9ff4babd8a91 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_i8packed.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_i8packed.ll
@@ -1,9 +1,9 @@
-; RUN: llc -O0 -mtriple=spirv1.5-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXP
-; RUN: llc -O0 -mtriple=spirv1.6-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown -spirv-ext=+SPV_KHR_integer_dot_product %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-EXT
-; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.5-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.6-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown -spirv-ext=+SPV_KHR_integer_dot_product %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv1.5-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXP
+; RUN: llc -O0 -mtriple=spirv1.6-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan -spirv-ext=+SPV_KHR_integer_dot_product %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-EXT
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.5-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.6-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan -spirv-ext=+SPV_KHR_integer_dot_product %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DOT: OpCapability DotProduct
 ; CHECK-DOT: OpCapability DotProductInput4x8BitPacked
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_u8packed.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_u8packed.ll
index 616f4cc01978..5e41c9dc79dd 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_u8packed.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/dot4add_u8packed.ll
@@ -1,9 +1,9 @@
-; RUN: llc -O0 -mtriple=spirv1.5-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXP
-; RUN: llc -O0 -mtriple=spirv1.6-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown -spirv-ext=+SPV_KHR_integer_dot_product %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-EXT
-; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.5-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.6-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown -spirv-ext=+SPV_KHR_integer_dot_product %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv1.5-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXP
+; RUN: llc -O0 -mtriple=spirv1.6-unknown-vulkan %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan -spirv-ext=+SPV_KHR_integer_dot_product %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-EXT
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.5-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv1.6-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan -spirv-ext=+SPV_KHR_integer_dot_product %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DOT: OpCapability DotProduct
 ; CHECK-DOT: OpCapability DotProductInput4x8BitPacked
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll
index 949f750db2dc..afd8b96db26e 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll
index 9e24c46cfcde..3f7c47f391be 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/exp2.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/faceforward.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/faceforward.ll
index 22742169506f..dbd992ef4ff5 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/faceforward.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/faceforward.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val --target-env spv1.4 %}
 
 ; FIXME(#136344): Change --target-env to vulkan1.3 and update this test accordingly once the issue is resolved.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbithigh.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbithigh.ll
index a4dd09d84d99..f99e699ce223 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbithigh.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbithigh.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: [[glsl_450_ext:%.+]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: OpMemoryModel Logical GLSL450
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbitlow.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbitlow.ll
index 6de6cdc60ea9..af94815faf7e 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbitlow.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/firstbitlow.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: [[glsl_450_ext:%.+]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: OpMemoryModel Logical GLSL450
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll
index 1f79cc951656..261f63445004 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/floor.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll
index 0bb969021de3..4a51f0b528f9 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmad.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll
index 073a5c4f8b8a..c62c3da59981 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmax.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
 define noundef half @test_fmax_half(half noundef %a, half noundef %b) {
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll
index d4bd1a6044e5..c3ad1b5609e6 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/fmin.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 ; CHECK: OpMemoryModel Logical GLSL450
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll
index d660abc1b623..5920abf8d8a4 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/frac.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
index 10310cd30088..adfd62d810c7 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/group_memory_barrier_with_group_sync.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpMemoryModel Logical GLSL450
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/length.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/length.ll
index 1ac862b79a3f..1dcea29b36d6 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/length.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/length.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for length are lowered correctly.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll
index 1d7f3a6ccc53..83ba0d2a8081 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/lerp.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for lerp are generated as FMix
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll
index 20339b98411a..5c0d8546bacc 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll
index 10264d3ef11d..7583066c01cf 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log10.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: %[[#extinst:]] = OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll
index 7bbd4596e99f..62f98b3b640d 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/log2.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll
index 71f69438a5ae..798a8dbbae8a 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/normalize.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for normalize are lowered correctly.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll
index 945414391311..f3cb9ce090ee 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/pow.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
index 26e90bd7ef70..ce521c0a05ea 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reflect.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reflect.ll
index 18962807f84f..0058e34d393e 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reflect.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reflect.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for reflect are lowered correctly.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll
index 9dc43cb25f7d..6571b2992fab 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/reversebits.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpMemoryModel Logical GLSL450
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll
index 58e54b0fbf26..f32a0fa57901 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/round.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll
index cd9374bb180f..fc43017cea24 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rsqrt.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/saturate.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/saturate.ll
index 0b05b615c4ad..d1cd6f9030a1 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/saturate.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/saturate.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for saturate are lowered correctly.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll
index 19aec458172f..d7f8be628878 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sin.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll
index 99da3ac81b43..bb594e97657d 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sinh.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll
index c6f17b677b47..a77142b96ef7 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smax.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll
index 3c7befe1078b..7c2b44dcdc5b 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/smin.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll
index 3230f56511eb..f793eba4273d 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/sqrt.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll
index f833c3256a81..20380688bff1 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/step.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; Make sure SPIRV operation function calls for step are lowered correctly.
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
index f0969667f98f..8301e0565177 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll
index 712c10d019b5..3bc908c0c8b9 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tanh.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
 ; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll
index b0f2a32d001e..d72543ae6c6d 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/trunc.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll
index 3b3dbc997817..81e64feddd9b 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umax.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll
index 1a3bcdbbcc31..3d91c58216ce 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/umin.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: OpExtInstImport "GLSL.std.450"
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/vk-ext-builtin-input.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/vk-ext-builtin-input.ll
new file mode 100644
index 000000000000..2253b9761d5b
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/vk-ext-builtin-input.ll
@@ -0,0 +1,26 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
+
+; FIXME(138268): Alignment decoration is emitted.
+; FIXME: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG:        OpDecorate %[[#WorkgroupId:]] BuiltIn WorkgroupId
+
+; CHECK-DAG:             %[[#uint:]] = OpTypeInt 32 0
+; CHECK-DAG:           %[[#uint_0:]] = OpConstant %[[#uint]] 0
+; CHECK-DAG:           %[[#v3uint:]] = OpTypeVector %[[#uint]] 3
+; CHECK-DAG:   %[[#ptr_Input_uint:]] = OpTypePointer Input %[[#uint]]
+; CHECK-DAG: %[[#ptr_Input_v3uint:]] = OpTypePointer Input %[[#v3uint]]
+; CHECK-DAG:      %[[#WorkgroupId:]] = OpVariable %[[#ptr_Input_v3uint]] Input
+@var = external local_unnamed_addr addrspace(7) externally_initialized constant <3 x i32>, align 16, !spirv.Decorations !0
+
+define i32 @foo() {
+entry:
+; CHECK: %[[#ptr:]] = OpAccessChain %[[#ptr_Input_uint]] %[[#WorkgroupId]] %[[#uint_0]]
+; CHECK: %[[#res:]] = OpLoad %[[#uint]] %[[#ptr]] Aligned 16
+; CHECK:              OpReturnValue %[[#res]]
+  %0 = load i32, ptr addrspace(7) @var, align 16
+  ret i32 %0
+}
+
+!0 = !{!1}
+!1 = !{i32 11, i32 26}
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoad.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoad.ll
index 58252fe297f3..b14b6af156ca 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoad.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoad.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
 ; CHECK-NOT: OpCapability StorageImageReadWithoutFormat
 
 ; CHECK-DAG: OpDecorate [[IntBufferVar:%[0-9]+]] DescriptorSet 16
@@ -20,7 +22,7 @@ define void @RWBufferLoad_Vec4_I32() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_24(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
 
 ; CHECK: OpImageRead [[v4_int]] [[buffer]] [[zero]]
   %data0 = call <4 x i32> @llvm.spv.resource.load.typedbuffer(
@@ -35,7 +37,7 @@ define void @RWBufferLoad_I32() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_24(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
 
 ; CHECK: [[V:%[0-9]+]] = OpImageRead [[v4_int]] [[buffer]] [[zero]]
 ; CHECK: OpCompositeExtract [[int]] [[V]] 0
@@ -51,7 +53,7 @@ define void @RWBufferLoad_Vec2_I32() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_24(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
 
 ; CHECK: [[V:%[0-9]+]] = OpImageRead [[v4_int]] [[buffer]] [[zero]]
 ; CHECK: [[e0:%[0-9]+]] = OpCompositeExtract [[int]] [[V]] 0
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll
index d810ef9ccecc..22fb4c3e78dc 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferLoadStore.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
 ; CHECK-DAG: [[float:%[0-9]+]] = OpTypeFloat 32
 ; CHECK-DAG: [[v2float:%[0-9]+]] = OpTypeVector [[float]] 2
 ; CHECK-DAG: [[v4float:%[0-9]+]] = OpTypeVector [[float]] 4
@@ -18,7 +20,7 @@
 define void @main_scalar() local_unnamed_addr #0 {
 entry:
 ; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
-  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false)
+  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
 
 ; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4float]] [[H]] [[one]]
 ; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[float]] [[R]] 0
@@ -57,7 +59,7 @@ bb_both:
 define void @main_vector2() local_unnamed_addr #0 {
 entry:
 ; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
-  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false)
+  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
 
 ; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4float]] [[H]] [[one]]
 ; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[float]] [[R]] 0
@@ -100,7 +102,7 @@ bb_both:
 define void @main_vector4() local_unnamed_addr #0 {
 entry:
 ; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
-  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false)
+  %s_h.i = tail call target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
 
 ; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4float]] [[H]] [[one]]
   %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1) %s_h.i, i32 1)
@@ -132,11 +134,5 @@ bb_both:
   ret void
 }
 
-; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(none)
-declare ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 1), i32) #1
-
-; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(none)
-declare target("spirv.Image", float, 5, 2, 0, 0, 2, 1) @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0t(i32, i32, i32, i32, i1) #1
-
 attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none) "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(none) }
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferStore.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferStore.ll
index 812e20e45565..ee976f1a4110 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferStore.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/BufferStore.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -O3 -verify-machineinstrs -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
+@.str.b = private unnamed_addr constant [2 x i8] c"B\00", align 1
+
 ; CHECK-NOT: OpCapability StorageImageReadWithoutFormat
 
 ; CHECK-DAG: OpDecorate [[IntBufferVar:%[0-9]+]] DescriptorSet 16
@@ -22,7 +24,7 @@ declare <4 x i32> @get_data() #1
 define void @RWBufferStore_Vec4_I32() #0 {
   %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_24(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.b)
 
 ; CHECK: [[data:%[0-9]+]] = OpFunctionCall
   %data = call <4 x i32> @get_data()
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/Packed.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/Packed.ll
index d5f654518014..5e9d88fd9af0 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/Packed.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/Packed.ll
@@ -3,6 +3,10 @@
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
 
+
+@.str.unpacked = private unnamed_addr constant [12 x i8] c"UnpackedRes\00", align 1
+@.str.packed = private unnamed_addr constant [10 x i8] c"PackedRes\00", align 1
+
 ; CHECK-DAG: OpName [[unpacked:%[0-9]+]] "unpacked"
 ; CHECK-DAG: OpName [[packed:%[0-9]+]] "packed"
 
@@ -22,7 +26,7 @@ target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:
 
 define external i32 @unpacked_vulkan_buffer_load() {
 entry:
-  %handle = tail call target("spirv.VulkanBuffer", [0 x %unpacked], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+  %handle = tail call target("spirv.VulkanBuffer", [0 x %unpacked], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str.unpacked)
   %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer(target("spirv.VulkanBuffer", [0 x %unpacked], 12, 0) %handle, i32 1)
   %1 = load i32, ptr addrspace(11) %0, align 4
   ret i32 %1
@@ -30,7 +34,7 @@ entry:
 
 define external i32 @packed_vulkan_buffer_load() {
 entry:
-  %handle = tail call target("spirv.VulkanBuffer", [0 x %packed], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, i1 false)
+  %handle = tail call target("spirv.VulkanBuffer", [0 x %packed], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.packed)
   %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer(target("spirv.VulkanBuffer", [0 x %packed], 12, 0) %handle, i32 1)
   %1 = load i32, ptr addrspace(11) %0, align 4
   ret i32 %1
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/ScalarResourceType.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/ScalarResourceType.ll
index f52fd44bf380..03b41ae0df31 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/ScalarResourceType.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/ScalarResourceType.ll
@@ -1,6 +1,9 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
+@.str.int_buf = private unnamed_addr constant [7 x i8] c"IntBuf\00", align 1
+@.str.float_buf = private unnamed_addr constant [9 x i8] c"FloatBuf\00", align 1
+
 ; CHECK-DAG: OpDecorate [[IntBufferVar:%[0-9]+]] DescriptorSet 16
 ; CHECK-DAG: OpDecorate [[IntBufferVar]] Binding 7
 ; CHECK-DAG: OpDecorate [[FloatBufferVar:%[0-9]+]] DescriptorSet 16
@@ -21,7 +24,7 @@ define void @RWBufferLoad() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_24(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.int_buf)
   %ptr0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 24) %buffer0, i32 0)
   store i32 0, ptr %ptr0, align 4
 
@@ -29,7 +32,7 @@ define void @RWBufferLoad() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_24(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.int_buf)
   %ptr1 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 24) %buffer1, i32 0)
   store i32 0, ptr %ptr1, align 4
   ret void
@@ -43,7 +46,7 @@ define void @UseDifferentGlobalVar() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeFloat]] [[FloatBufferVar]]
   %buffer0 = call target("spirv.Image", float, 5, 2, 0, 0, 2, 3)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_3(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.float_buf )
   %ptr0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", float, 5, 2, 0, 0, 2, 3) %buffer0, i32 0)
   store float 0.0, ptr %ptr0, align 4
   ret void
@@ -57,7 +60,7 @@ define void @ReuseGlobalVarFromFirstFunction() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer1 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_24(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.int_buf)
   %ptr1 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 24) %buffer1, i32 0)
   store i32 0, ptr %ptr1, align 4
   ret void
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll
index 6a6d810e6bab..236c5e4ea56a 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageDynIdx.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
 ; CHECK-DAG: OpCapability Shader
 ; CHECK-DAG: OpCapability StorageImageArrayDynamicIndexing
 ; CHECK-DAG: OpCapability Image1D
@@ -26,7 +28,7 @@ define void @main() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
   %buffer0 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_0_2_0_0_2_24(
-          i32 3, i32 4, i32 3, i32 0, i1 false)
+          i32 3, i32 4, i32 3, i32 0, i1 false, ptr nonnull @.str.b0)
   %ptr0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", i32, 0, 2, 0, 0, 2, 24) %buffer0, i32 0)
   store i32 0, ptr %ptr0, align 4
 
@@ -34,7 +36,7 @@ define void @main() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[BufferType]] [[ac]]
   %buffer1 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_0_2_0_0_2_24(
-          i32 3, i32 4, i32 3, i32 1, i1 false)
+          i32 3, i32 4, i32 3, i32 1, i1 false, ptr nonnull @.str.b0)
   %ptr1 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", i32, 0, 2, 0, 0, 2, 24) %buffer1, i32 0)
   store i32 0, ptr %ptr1, align 4
   ret void
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll
index 16f3724d5d10..5693f797c798 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StorageImageNonUniformIdx.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.5-vulkan-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.5-vulkan-library %s -o - -filetype=obj | spirv-val %}
 
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
 ; CHECK-DAG: OpCapability Shader
 ; CHECK-DAG: OpCapability ShaderNonUniformEXT
 ; CHECK-DAG: OpCapability StorageImageArrayNonUniformIndexing
@@ -33,7 +35,7 @@ define void @main() #0 {
 ; CHECK: [[ld0]] = OpLoad [[BufferType]] [[ac0]]
   %buffer0 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_0_2_0_0_2_24(
-          i32 3, i32 4, i32 3, i32 0, i1 true)
+          i32 3, i32 4, i32 3, i32 0, i1 true, ptr nonnull @.str.b0)
   %ptr0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", i32, 0, 2, 0, 0, 2, 24) %buffer0, i32 0)
   store i32 0, ptr %ptr0, align 4
 
@@ -41,7 +43,7 @@ define void @main() #0 {
 ; CHECK: [[ld1]] = OpLoad [[BufferType]] [[ac1]]
   %buffer1 = call target("spirv.Image", i32, 0, 2, 0, 0, 2, 24)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_0_2_0_0_2_24(
-          i32 3, i32 4, i32 3, i32 1, i1 true)
+          i32 3, i32 4, i32 3, i32 1, i1 true, ptr nonnull @.str.b0)
   %ptr1 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_f32_5_2_0_0_2_0t(target("spirv.Image", i32, 0, 2, 0, 0, 2, 24) %buffer1, i32 0)
   store i32 0, ptr %ptr1, align 4
   ret void
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StructuredBuffer.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StructuredBuffer.ll
index f539fdefa3fa..e47685cd38a2 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StructuredBuffer.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/StructuredBuffer.ll
@@ -3,11 +3,8 @@
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
 
-; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(none)
-declare target("spirv.VulkanBuffer", [0 x i32], 12, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_0t(i32, i32, i32, i32, i1) #0
-
-; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(none)
-declare target("spirv.VulkanBuffer", [0 x i32], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_1t(i32, i32, i32, i32, i1) #0
+@.str.b = private unnamed_addr constant [2 x i8] c"B\00", align 1
+@.str.rwb = private unnamed_addr constant [4 x i8] c"RWB\00", align 1
 
 ; CHECK: OpDecorate [[BufferVar:%.+]] DescriptorSet 0
 ; CHECK: OpDecorate [[BufferVar]] Binding 0
@@ -40,9 +37,9 @@ entry:
 ; CHECK-DAG: [[BufferHandle:%.+]] = OpCopyObject [[BufferPtrType]] [[BufferVar]]
 ; CHECK-DAG: [[BufferHandle2:%.+]] = OpCopyObject [[BufferPtrType]] [[BufferVar]]
 ; CHECK-DAG: [[RWBufferHandle:%.+]] = OpCopyObject [[RWBufferPtrType]] [[RWBufferVar]]
-  %BufferHandle = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_0t(i32 0, i32 0, i32 1, i32 0, i1 false)
-  %BufferHandle2 = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_0t(i32 0, i32 0, i32 1, i32 0, i1 false)
-  %RWBufferHandle = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_1t(i32 0, i32 1, i32 1, i32 0, i1 false)
+  %BufferHandle = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str.b)
+  %BufferHandle2 = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str.b)
+  %RWBufferHandle = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0i32_12_1t(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.rwb)
 
 ; CHECK: [[AC:%.+]] = OpAccessChain {{.*}} [[BufferHandle]] [[zero]] [[one]]
   %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0i32_12_0t(target("spirv.VulkanBuffer", [0 x i32], 12, 0) %BufferHandle,  i32 1)
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferLoad.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferLoad.ll
index 4ec8605f6813..704665d7e52e 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferLoad.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferLoad.ll
@@ -1,8 +1,11 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val %}
 
+@.str = private unnamed_addr constant [4 x i8] c"Buf\00", align 1
+
 ; CHECK: OpCapability StorageImageReadWithoutFormat
-; CHECK-DAG: OpDecorate [[IntBufferVar:%[0-9]+]] DescriptorSet 16
+; CHECK: OpName [[IntBufferVar:%[0-9]+]] "Buf"
+; CHECK-DAG: OpDecorate [[IntBufferVar]] DescriptorSet 16
 ; CHECK-DAG: OpDecorate [[IntBufferVar]] Binding 7
 
 ; CHECK-DAG: [[int:%[0-9]+]] = OpTypeInt 32 0
@@ -18,7 +21,7 @@ define void @RWBufferLoad_Vec4_I32() #0 {
 ; CHECK: [[buffer:%[0-9]+]] = OpLoad [[RWBufferTypeInt]] [[IntBufferVar]]
   %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str)
 
 ; CHECK: OpImageRead [[v4_int]] [[buffer]] [[zero]]
   %data0 = call <4 x i32> @llvm.spv.resource.load.typedbuffer(
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferStore.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferStore.ll
index a4123c36a448..27ae6a03797c 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferStore.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/UnknownBufferStore.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val %}
 
+@.str.b = private unnamed_addr constant [2 x i8] c"B\00", align 1
+
 ; CHECK: OpCapability StorageImageWriteWithoutFormat
 ; CHECK-DAG: OpDecorate [[IntBufferVar:%[0-9]+]] DescriptorSet 16
 ; CHECK-DAG: OpDecorate [[IntBufferVar]] Binding 7
@@ -20,7 +22,7 @@ declare <4 x i32> @get_data() #1
 define void @RWBufferLoad_Vec4_I32() #0 {
   %buffer0 = call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0)
       @llvm.spv.resource.handlefrombinding.tspirv.Image_f32_5_2_0_0_2_0(
-          i32 16, i32 7, i32 1, i32 0, i1 false)
+          i32 16, i32 7, i32 1, i32 0, i1 false, ptr nonnull @.str.b)
 
 ; CHECK: [[data:%[0-9]+]] = OpFunctionCall
   %data = call <4 x i32> @get_data()
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/spirv.layout.type.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/spirv.layout.type.ll
index 14c98b2fd55a..064251a57dfc 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/spirv.layout.type.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/hlsl-resources/spirv.layout.type.ll
@@ -3,6 +3,12 @@
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G10"
 
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+@.str.b1 = private unnamed_addr constant [3 x i8] c"B1\00", align 1
+@.str.b2 = private unnamed_addr constant [3 x i8] c"B2\00", align 1
+@.str.b3 = private unnamed_addr constant [3 x i8] c"B3\00", align 1
+@.str.b4 = private unnamed_addr constant [3 x i8] c"B4\00", align 1
+
 ; CHECK-DAG: OpName [[standard_layout:%[0-9]+]] "standard_layout"
 ; CHECK-DAG: OpMemberDecorate [[standard_layout]] 0 Offset 0
 ; CHECK-DAG: OpMemberDecorate [[standard_layout]] 1 Offset 4
@@ -33,11 +39,11 @@ target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:
 
 define void @main() local_unnamed_addr #1 {
 entry:
-  %standard_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %standard_layout, 8, 0, 4), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false)
-  %standard_handle_with_different_offset = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %standard_layout, 12, 0, 8), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false)
-  %backwards_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %backwards_layout, 8, 4, 0), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false)
-  %large_gap_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %large_gap, 1024, 0, 64, 1020, 4), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false)
-  %mixed_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %mixed_layout, 16, 0, 8, 4, 12), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false)
+  %standard_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %standard_layout, 8, 0, 4), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+  %standard_handle_with_different_offset = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %standard_layout, 12, 0, 8), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.b1)
+  %backwards_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %backwards_layout, 8, 4, 0), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.b2)
+  %large_gap_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %large_gap, 1024, 0, 64, 1020, 4), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.b3)
+  %mixed_handle = tail call target("spirv.VulkanBuffer", target("spirv.Layout", %mixed_layout, 16, 0, 8, 4, 12), 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_tspirv.Layout_s___cblayout_Bs_8_0_4t_2_0t(i32 0, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.b4)
   ret void
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.coop-matrix.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.coop-matrix.ll
index 12effdfd464f..3bd55058e51d 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.coop-matrix.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.coop-matrix.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
 
 ; TODO: enable spirv-val once we can add cooperative matrix capability and extension
 
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.ll
index 0607563381b1..be29ddfa21ba 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - | spirv-as - -o - | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | spirv-as - -o - | spirv-val %}
 
 ; CHECK-DAG: [[float_t:%[0-9]+]] = OpTypeFloat 32
 ; CHECK-DAG: [[uint32_t:%[0-9]+]] = OpTypeInt 32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.undef.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.undef.ll
index 8cb90007c9ab..e84e6262d887 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.undef.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/inline/type.undef.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - | spirv-as - -o - | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | spirv-as - -o - | spirv-val %}
 
 %literal_32 = type target("spirv.Literal", 32)
 %literal_true = type target("spirv.Literal", 1)
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/opencl/basic/progvar_prog_scope_init.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/opencl/basic/progvar_prog_scope_init.ll
index 9d759a1cf47d..fbc83c7a1e04 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/opencl/basic/progvar_prog_scope_init.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/opencl/basic/progvar_prog_scope_init.ll
@@ -10,7 +10,7 @@
 ; CHECK-DAG: %[[#pt2:]] = OpTypePointer CrossWorkgroup %[[#arr2]]
 ; CHECK-DAG: %[[#pt3:]] = OpTypePointer CrossWorkgroup %[[#pt1]]
 ; CHECK-DAG: %[[#a_var]] = OpVariable %[[#pt2]] CrossWorkgroup
-; CHECK-DAG: %[[#const:]] = OpSpecConstantOp %[[#pt1]] 70 %[[#a_var]]
+; CHECK-DAG: %[[#const:]] = OpSpecConstantOp %[[#pt1]] InBoundsPtrAccessChain %[[#a_var]]
 ; CHECK-DAG: %[[#p_var]] = OpVariable %[[#pt3]] CrossWorkgroup %[[#const]]
 @var = addrspace(1) global i8 0, align 1
 @g_var = addrspace(1) global i8 1, align 1
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/opt-gepoperator-of-gvar.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/opt-gepoperator-of-gvar.ll
index 5f9229f5a5bd..447dfa701b65 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/opt-gepoperator-of-gvar.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/opt-gepoperator-of-gvar.ll
@@ -14,7 +14,7 @@
 ; CHECK-DAG: %[[#PtrStruct:]] = OpTypePointer CrossWorkgroup %[[#Struct]]
 ; CHECK-DAG: %[[#Var:]] = OpVariable %[[#PtrStruct]] CrossWorkgroup %[[#VarInit]]
 ; CHECK-DAG: %[[#Bytes:]] = OpVariable %[[#PtrChar]] CrossWorkgroup %[[#]]
-; CHECK-DAG: %[[#BytesGEP:]] = OpSpecConstantOp %[[#PtrChar]] 70 %[[#Bytes]] %[[#C648]]
+; CHECK-DAG: %[[#BytesGEP:]] = OpSpecConstantOp %[[#PtrChar]] InBoundsPtrAccessChain %[[#Bytes]] %[[#C648]]
 
 ; CHECK: OpFunction
 ; CHECK: %[[#]] = OpFunctionParameter %[[#]]
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll
index 55d638f80cc5..ca7ca06fbdc8 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/PtrCast-in-OpSpecConstantOp.ll
@@ -23,20 +23,20 @@
 ; CHECK-DAG: %[[WPtr:.*]] = OpTypePointer Workgroup %[[Int]]
 
 ; CHECK-DAG: %[[F]] = OpVariable %[[CWPtr]] CrossWorkgroup %[[#]]
-; CHECK-DAG: %[[GenF:.*]] = OpSpecConstantOp %[[GenPtrChar]] 121 %[[F]]
+; CHECK-DAG: %[[GenF:.*]] = OpSpecConstantOp %[[GenPtrChar]] PtrCastToGeneric %[[F]]
 ; CHECK-DAG: %[[B]] = OpVariable %[[CWPtr]] CrossWorkgroup %[[#]]
-; CHECK-DAG: %[[GenB:.*]] = OpSpecConstantOp %[[GenPtrChar]] 121 %[[B]]
+; CHECK-DAG: %[[GenB:.*]] = OpSpecConstantOp %[[GenPtrChar]] PtrCastToGeneric %[[B]]
 ; CHECK-DAG: %[[GenFB:.*]] = OpConstantComposite %[[Arr2]] %[[GenF]] %[[GenB]]
 ; CHECK-DAG: %[[GenBF:.*]] = OpConstantComposite %[[Arr2]] %[[GenB]] %[[GenF]]
 ; CHECK-DAG: %[[CG1:.*]] = OpConstantComposite %[[Struct2]] %[[GenFB]]
 ; CHECK-DAG: %[[CG2:.*]] = OpConstantComposite %[[Struct2]] %[[GenBF]]
 
 ; CHECK-DAG: %[[X]] = OpVariable %[[WPtr]] Workgroup %[[#]]
-; CHECK-DAG: %[[GenX:.*]] = OpSpecConstantOp %[[GenPtr]] 121 %[[X]]
-; CHECK-DAG: %[[CWX:.*]] = OpSpecConstantOp %[[CWPtrChar]] 122 %[[GenX]]
+; CHECK-DAG: %[[GenX:.*]] = OpSpecConstantOp %[[GenPtr]] PtrCastToGeneric %[[X]]
+; CHECK-DAG: %[[CWX:.*]] = OpSpecConstantOp %[[CWPtrChar]] GenericCastToPtr %[[GenX]]
 ; CHECK-DAG: %[[Y]] = OpVariable %[[WPtr]] Workgroup %[[#]]
-; CHECK-DAG: %[[GenY:.*]] = OpSpecConstantOp %[[GenPtr]] 121 %[[Y]]
-; CHECK-DAG: %[[CWY:.*]] = OpSpecConstantOp %[[CWPtrChar]] 122 %[[GenY]]
+; CHECK-DAG: %[[GenY:.*]] = OpSpecConstantOp %[[GenPtr]] PtrCastToGeneric %[[Y]]
+; CHECK-DAG: %[[CWY:.*]] = OpSpecConstantOp %[[CWPtrChar]] GenericCastToPtr %[[GenY]]
 ; CHECK-DAG: %[[CWXY:.*]] = OpConstantComposite %[[Arr1]] %[[CWX]] %[[CWY]]
 ; CHECK-DAG: %[[CWYX:.*]] = OpConstantComposite %[[Arr1]] %[[CWY]] %[[CWX]]
 ; CHECK-DAG: %[[CG3:.*]] = OpConstantComposite %[[Struct1]] %[[CWXY]]
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll
index 16c20f9067e6..0fd2f622dc84 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/global-ptrtoint.ll
@@ -11,9 +11,9 @@
 ; CHECK-DAG: %[[TyStruct:.*]] = OpTypeStruct %[[TyI64]] %[[TyI64]]
 ; CHECK-DAG: %[[Const128:.*]] = OpConstant %[[TyI64]] 128
 ; CHECK-DAG: %[[GlobalValue]] = OpVariable
-; CHECK-DAG: %[[PtrToInt:.*]] = OpSpecConstantOp %[[TyI64]] 117 %[[GlobalValue]]
+; CHECK-DAG: %[[PtrToInt:.*]] = OpSpecConstantOp %[[TyI64]] ConvertPtrToU %[[GlobalValue]]
 ; TODO: The following bitcast line looks unneeded and we may expect it to be removed in future
-; CHECK-DAG: %[[UseGlobalValue:.*]] = OpSpecConstantOp %[[TyI64]] 124 %[[PtrToInt]]
+; CHECK-DAG: %[[UseGlobalValue:.*]] = OpSpecConstantOp %[[TyI64]] Bitcast %[[PtrToInt]]
 ; CHECK-DAG: %[[ConstComposite:.*]] = OpConstantComposite %[[TyStruct]] %[[Const128]] %[[UseGlobalValue]]
 ; CHECK-DAG: %[[TyPtrStruct:.*]] = OpTypePointer CrossWorkgroup %[[TyStruct]]
 ; CHECK: OpVariable %[[TyPtrStruct]] CrossWorkgroup %[[ConstComposite]]
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/irtrans-added-int-const-32-64.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/irtrans-added-int-const-32-64.ll
index c2738229aa4d..f5abcd38d040 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/irtrans-added-int-const-32-64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/irtrans-added-int-const-32-64.ll
@@ -12,7 +12,7 @@
 ; CHECK-SPIRV64-DAG: %[[#IntTy:]] = OpTypeInt 64 0
 ; CHECK-SPIRV32-DAG: %[[#IntTy:]] = OpTypeInt 32 0
 ; CHECK-SPIRV-DAG: %[[#Const2:]] = OpConstant %[[#IntTy]] 2
-; CHECK-SPIRV-DAG: %[[#]] = OpSpecConstantOp %[[#]] 70 %[[#]] %[[#]] %[[#Const2]]
+; CHECK-SPIRV-DAG: %[[#]] = OpSpecConstantOp %[[#]] InBoundsPtrAccessChain %[[#]] %[[#]] %[[#Const2]]
 ; CHECK-SPIRV: OpFunction
 
 @a_var = addrspace(1) global [2 x i8] [i8 1, i8 1]
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast-2.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast-2.ll
index d608529b421c..d87c175c3691 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast-2.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -verify-machineinstrs -O3 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - | FileCheck %s --match-full-lines
 ; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
 
+@.str = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
 %S2 = type { { [10 x { i32, i32 } ] }, i32 }
 
 ; CHECK-DAG:                     %[[#uint:]] = OpTypeInt 32 0
@@ -21,11 +23,9 @@
 ; CHECK-DAG:              %[[#rarr_struct:]] = OpTypeStruct %[[#rarr]]
 ; CHECK-DAG:       %[[#spirv_VulkanBuffer:]] = OpTypePointer StorageBuffer %[[#rarr_struct]]
 
-declare target("spirv.VulkanBuffer", [0 x %S2], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0s_Ss_12_1t(i32, i32, i32, i32, i1)
-
 define void @main() "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" {
 entry:
-  %handle = tail call target("spirv.VulkanBuffer", [0 x %S2], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0s_Ss_12_1t(i32 0, i32 0, i32 1, i32 0, i1 false)
+  %handle = tail call target("spirv.VulkanBuffer", [0 x %S2], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0s_Ss_12_1t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str)
 ; CHECK:      %[[#resource:]] = OpVariable %[[#spirv_VulkanBuffer]] StorageBuffer
 
   %ptr = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0s_Ss_12_1t(target("spirv.VulkanBuffer", [0 x %S2], 12, 1) %handle, i32 0)
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast.ll
index b1446b7529ea..5a469a4515b7 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/pointers/resource-addrspacecast.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -verify-machineinstrs -O3 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
 
+@.str = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
 %struct.S = type { i32 }
 
 ; CHECK-DAG:                     %[[#uint:]] = OpTypeInt 32 0
@@ -13,11 +15,9 @@
 ; CHECK-DAG:              %[[#rarr_struct:]] = OpTypeStruct %[[#rarr]]
 ; CHECK-DAG:       %[[#spirv_VulkanBuffer:]] = OpTypePointer StorageBuffer %[[#rarr_struct]]
 
-declare target("spirv.VulkanBuffer", [0 x %struct.S], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0s_struct.Ss_12_1t(i32, i32, i32, i32, i1)
-
 define void @main() "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" {
 entry:
-  %handle = tail call target("spirv.VulkanBuffer", [0 x %struct.S], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0s_struct.Ss_12_1t(i32 0, i32 0, i32 1, i32 0, i1 false)
+  %handle = tail call target("spirv.VulkanBuffer", [0 x %struct.S], 12, 1) @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0s_struct.Ss_12_1t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str)
 ; CHECK:      %[[#resource:]] = OpVariable %[[#spirv_VulkanBuffer]] StorageBuffer
 
   %ptr = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0s_struct.Ss_12_1t(target("spirv.VulkanBuffer", [0 x %struct.S], 12, 1) %handle, i32 0)
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/spirv-explicit-layout.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/spirv-explicit-layout.ll
index 7303471c9929..4cc07c249be9 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/spirv-explicit-layout.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/spirv-explicit-layout.ll
@@ -3,9 +3,14 @@
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
 
-; CHECK-DAG: OpName [[ScalarBlock_var:%[0-9]+]] "__resource_p_12_{_u32[0]}_0_0"
-; CHECK-DAG: OpName [[buffer_var:%[0-9]+]] "__resource_p_12_{_{_{_u32_f32[3]}[10]}[0]}_0_0"
-; CHECK-DAG: OpName [[array_buffer_var:%[0-9]+]] "__resource_p_12_{_{_{_u32_f32[3]}[10]}[0]}[10]_0_0"
+@.str.scalarblock = private unnamed_addr constant [12 x i8] c"ScalarBlock\00", align 1
+@.str.buffervar = private unnamed_addr constant [10 x i8] c"BufferVar\00", align 1
+@.str.arraybuffervar = private unnamed_addr constant [15 x i8] c"ArrayBufferVar\00", align 1
+
+
+; CHECK-DAG: OpName [[ScalarBlock_var:%[0-9]+]] "ScalarBlock"
+; CHECK-DAG: OpName [[buffer_var:%[0-9]+]] "BufferVar"
+; CHECK-DAG: OpName [[array_buffer_var:%[0-9]+]] "ArrayBufferVar"
 
 ; CHECK-DAG: OpMemberDecorate [[ScalarBlock:%[0-9]+]] 0 Offset 0
 ; CHECK-DAG: OpDecorate [[ScalarBlock]] Block
@@ -63,8 +68,8 @@ target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:
 define external i32 @scalar_vulkan_buffer_load() {
 ; CHECK-NEXT: OpLabel
 entry:
-; CHECK-NEXT: [[handle:%[0-9]+]] = OpCopyObject [[ScalarBlock_ptr]] [[ScalarBlock_var]]
-  %handle = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+; CHECK: [[handle:%[0-9]+]] = OpCopyObject [[ScalarBlock_ptr]] [[ScalarBlock_var]]
+  %handle = tail call target("spirv.VulkanBuffer", [0 x i32], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str.scalarblock)
 
 ; CHECK-NEXT: [[ptr:%[0-9]+]] = OpAccessChain [[storagebuffer_int_ptr]] [[handle]] [[zero]] [[one]]
   %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer(target("spirv.VulkanBuffer", [0 x i32], 12, 0) %handle, i32 1)
@@ -83,7 +88,7 @@ define external %struct.S @private_load() {
 ; CHECK-NEXT: OpLabel
 entry:
 
-; CHECK-NEXT: [[ld:%[0-9]+]] = OpLoad [[S]] [[private_var]] Aligned 4
+; CHECK: [[ld:%[0-9]+]] = OpLoad [[S]] [[private_var]] Aligned 4
   %1 = load %struct.S, ptr addrspace(10) @private, align 4
 
 ; CHECK-NEXT: OpReturnValue [[ld]]
@@ -97,7 +102,7 @@ define external %struct.S @storage_buffer_load() {
 ; CHECK-NEXT: OpLabel
 entry:
 
-; CHECK-NEXT: [[ld:%[0-9]+]] = OpLoad [[S_explicit]] [[storage_buffer]] Aligned 4
+; CHECK: [[ld:%[0-9]+]] = OpLoad [[S_explicit]] [[storage_buffer]] Aligned 4
 ; CHECK-NEXT: [[copy:%[0-9]+]] = OpCopyLogical [[S]] [[ld]]
   %1 = load %struct.S, ptr addrspace(11) @storage_buffer, align 4
 
@@ -111,8 +116,8 @@ entry:
 define external %struct.S @vulkan_buffer_load() {
 ; CHECK-NEXT: OpLabel
 entry:
-; CHECK-NEXT: [[handle:%[0-9]+]] = OpCopyObject [[buffer_ptr]] [[buffer_var]]
-  %handle = tail call target("spirv.VulkanBuffer", [0 x %struct.S], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+; CHECK: [[handle:%[0-9]+]] = OpCopyObject [[buffer_ptr]] [[buffer_var]]
+  %handle = tail call target("spirv.VulkanBuffer", [0 x %struct.S], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str.buffervar)
 
 ; CHECK-NEXT: [[ptr:%[0-9]+]] = OpAccessChain [[storagebuffer_S_ptr]] [[handle]] [[zero]] [[one]]
   %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer(target("spirv.VulkanBuffer", [0 x %struct.S], 12, 0) %handle, i32 1)
@@ -131,9 +136,9 @@ entry:
 define external %struct.S @array_of_vulkan_buffers_load() {
 ; CHECK-NEXT: OpLabel
 entry:
-; CHECK-NEXT: [[h:%[0-9]+]] = OpAccessChain [[buffer_ptr]] [[array_buffer_var]] [[one]]
-; CHECK-NEXT: [[handle:%[0-9]+]] = OpCopyObject [[buffer_ptr]] [[h]]
-  %handle = tail call target("spirv.VulkanBuffer", [0 x %struct.S], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 10, i32 1, i1 false)
+; CHECK: [[h:%[0-9]+]] = OpAccessChain [[buffer_ptr]] [[array_buffer_var]] [[one]]
+; CHECK: [[handle:%[0-9]+]] = OpCopyObject [[buffer_ptr]] [[h]]
+  %handle = tail call target("spirv.VulkanBuffer", [0 x %struct.S], 12, 0) @llvm.spv.resource.handlefrombinding(i32 0, i32 0, i32 10, i32 1, i1 false, ptr nonnull @.str.arraybuffervar)
 
 ; CHECK-NEXT: [[ptr:%[0-9]+]] = OpAccessChain [[storagebuffer_S_ptr]] [[handle]] [[zero]] [[one]]
   %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer(target("spirv.VulkanBuffer", [0 x %struct.S], 12, 0) %handle, i32 1)
diff --git a/external/llvm-project/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll b/external/llvm-project/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll
index 9c6f977dc9b3..5e72b66310db 100644
--- a/external/llvm-project/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll
+++ b/external/llvm-project/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv-unknown-vulkan %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %}
 
 
 define spir_func noundef i32 @test_branch(i32 noundef %X) {
diff --git a/external/llvm-project/llvm/test/CodeGen/SystemZ/fold-masked-merge.ll b/external/llvm-project/llvm/test/CodeGen/SystemZ/fold-masked-merge.ll
new file mode 100644
index 000000000000..c014345507f6
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/SystemZ/fold-masked-merge.ll
@@ -0,0 +1,277 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s --check-prefix=NO-MISC3
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s --check-prefix=MISC3
+
+; test that masked-merge code is generated as "xor;and;xor" sequence or
+; "andn ; and; or" if and-not is available.
+
+define i32 @masked_merge0(i32 %a0, i32 %a1, i32 %a2) {
+; NO-MISC3-LABEL: masked_merge0:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    xr %r3, %r4
+; NO-MISC3-NEXT:    nr %r2, %r3
+; NO-MISC3-NEXT:    xr %r2, %r4
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: masked_merge0:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    nr %r3, %r2
+; MISC3-NEXT:    ncrk %r2, %r4, %r2
+; MISC3-NEXT:    or %r2, %r3
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i16 @masked_merge1(i16 %a0, i16 %a1, i16 %a2) {
+; NO-MISC3-LABEL: masked_merge1:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    xr %r3, %r4
+; NO-MISC3-NEXT:    nr %r2, %r3
+; NO-MISC3-NEXT:    xr %r2, %r4
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: masked_merge1:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    ncrk %r0, %r4, %r2
+; MISC3-NEXT:    nr %r2, %r3
+; MISC3-NEXT:    or %r2, %r0
+; MISC3-NEXT:    br %r14
+  %and0 = and i16 %a0, %a1
+  %not = xor i16 %a0, -1
+  %and1 = and i16 %a2, %not
+  %or = or i16 %and0, %and1
+  ret i16 %or
+}
+
+define i8 @masked_merge2(i8 %a0, i8 %a1, i8 %a2) {
+; NO-MISC3-LABEL: masked_merge2:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    lr %r2, %r3
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: masked_merge2:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    lr %r2, %r3
+; MISC3-NEXT:    br %r14
+  %not = xor i8 %a0, -1
+  %and0 = and i8 %not, %a1
+  %and1 = and i8 %a1, %a0
+  %or = or i8 %and0, %and1
+  ret i8 %or
+}
+
+define i64 @masked_merge3(i64 %a0, i64 %a1, i64 %a2) {
+; NO-MISC3-LABEL: masked_merge3:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    lcgr %r0, %r4
+; NO-MISC3-NEXT:    aghi %r0, -1
+; NO-MISC3-NEXT:    xgr %r3, %r0
+; NO-MISC3-NEXT:    ngr %r3, %r2
+; NO-MISC3-NEXT:    xgr %r3, %r2
+; NO-MISC3-NEXT:    xgrk %r2, %r3, %r0
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: masked_merge3:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    lcgr %r0, %r2
+; MISC3-NEXT:    aghi %r0, -1
+; MISC3-NEXT:    ncgrk %r0, %r0, %r4
+; MISC3-NEXT:    ncgrk %r2, %r2, %r3
+; MISC3-NEXT:    ogr %r2, %r0
+; MISC3-NEXT:    br %r14
+  %v0 = xor i64 %a1, -1
+  %v1 = xor i64 %a2, -1
+  %not = xor i64 %a0, -1
+  %and0 = and i64 %not, %v1
+  %and1 = and i64 %v0, %a0
+  %or = or i64 %and0, %and1
+  ret i64 %or
+}
+
+define i32 @not_a_masked_merge0(i32 %a0, i32 %a1, i32 %a2) {
+; NO-MISC3-LABEL: not_a_masked_merge0:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    lcr %r0, %r2
+; NO-MISC3-NEXT:    nr %r3, %r2
+; NO-MISC3-NEXT:    nr %r0, %r4
+; NO-MISC3-NEXT:    ork %r2, %r3, %r0
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: not_a_masked_merge0:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    lcr %r0, %r2
+; MISC3-NEXT:    nr %r3, %r2
+; MISC3-NEXT:    nr %r0, %r4
+; MISC3-NEXT:    ork %r2, %r3, %r0
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not_a_not = sub i32 0, %a0
+  %and1 = and i32 %not_a_not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge1(i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
+; NO-MISC3-LABEL: not_a_masked_merge1:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    xilf %r5, 4294967295
+; NO-MISC3-NEXT:    nr %r2, %r3
+; NO-MISC3-NEXT:    nr %r4, %r5
+; NO-MISC3-NEXT:    or %r2, %r4
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: not_a_masked_merge1:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    nr %r2, %r3
+; MISC3-NEXT:    ncrk %r0, %r4, %r5
+; MISC3-NEXT:    or %r2, %r0
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a3, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge2(i32 %a0, i32 %a1, i32 %a2) {
+; NO-MISC3-LABEL: not_a_masked_merge2:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    or %r3, %r2
+; NO-MISC3-NEXT:    xilf %r2, 4294967295
+; NO-MISC3-NEXT:    nr %r2, %r4
+; NO-MISC3-NEXT:    or %r2, %r3
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: not_a_masked_merge2:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    or %r3, %r2
+; MISC3-NEXT:    ncrk %r2, %r4, %r2
+; MISC3-NEXT:    or %r2, %r3
+; MISC3-NEXT:    br %r14
+  %not_an_and0 = or i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %not_an_and0, %and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge3(i32 %a0, i32 %a1, i32 %a2) {
+; NO-MISC3-LABEL: not_a_masked_merge3:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    nr %r3, %r2
+; NO-MISC3-NEXT:    xr %r2, %r4
+; NO-MISC3-NEXT:    xilf %r2, 4294967295
+; NO-MISC3-NEXT:    or %r2, %r3
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: not_a_masked_merge3:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    nr %r3, %r2
+; MISC3-NEXT:    xr %r2, %r4
+; MISC3-NEXT:    ocrk %r2, %r3, %r2
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %not_an_and1 = xor i32 %not, %a2
+  %or = or i32 %and0, %not_an_and1
+  ret i32 %or
+}
+
+define i32 @not_a_masked_merge4(i32 %a0, i32 %a1, i32 %a2) {
+; NO-MISC3-LABEL: not_a_masked_merge4:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    nr %r2, %r3
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: not_a_masked_merge4:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    nr %r2, %r3
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a2, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  ret i32 %or
+}
+
+define i32 @masked_merge_no_transform0(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; NO-MISC3-LABEL: masked_merge_no_transform0:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    nr %r3, %r2
+; NO-MISC3-NEXT:    xilf %r2, 4294967295
+; NO-MISC3-NEXT:    nr %r2, %r4
+; NO-MISC3-NEXT:    or %r2, %r3
+; NO-MISC3-NEXT:    st %r3, 0(%r5)
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: masked_merge_no_transform0:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    nr %r3, %r2
+; MISC3-NEXT:    ncrk %r2, %r4, %r2
+; MISC3-NEXT:    or %r2, %r3
+; MISC3-NEXT:    st %r3, 0(%r5)
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  store i32 %and0, ptr %p1
+  ret i32 %or
+}
+
+define i32 @masked_merge_no_transform1(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; NO-MISC3-LABEL: masked_merge_no_transform1:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    nrk %r0, %r2, %r3
+; NO-MISC3-NEXT:    xilf %r2, 4294967295
+; NO-MISC3-NEXT:    nr %r4, %r2
+; NO-MISC3-NEXT:    or %r0, %r4
+; NO-MISC3-NEXT:    st %r2, 0(%r5)
+; NO-MISC3-NEXT:    lr %r2, %r0
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: masked_merge_no_transform1:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    nrk %r0, %r2, %r3
+; MISC3-NEXT:    ncrk %r1, %r4, %r2
+; MISC3-NEXT:    xilf %r2, 4294967295
+; MISC3-NEXT:    or %r0, %r1
+; MISC3-NEXT:    st %r2, 0(%r5)
+; MISC3-NEXT:    lr %r2, %r0
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  store i32 %not, ptr %p1
+  ret i32 %or
+}
+
+define i32 @masked_merge_no_transform2(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; NO-MISC3-LABEL: masked_merge_no_transform2:
+; NO-MISC3:       # %bb.0:
+; NO-MISC3-NEXT:    nr %r3, %r2
+; NO-MISC3-NEXT:    xilf %r2, 4294967295
+; NO-MISC3-NEXT:    nr %r4, %r2
+; NO-MISC3-NEXT:    ork %r2, %r3, %r4
+; NO-MISC3-NEXT:    st %r4, 0(%r5)
+; NO-MISC3-NEXT:    br %r14
+;
+; MISC3-LABEL: masked_merge_no_transform2:
+; MISC3:       # %bb.0:
+; MISC3-NEXT:    nr %r3, %r2
+; MISC3-NEXT:    ncrk %r0, %r4, %r2
+; MISC3-NEXT:    ork %r2, %r3, %r0
+; MISC3-NEXT:    st %r0, 0(%r5)
+; MISC3-NEXT:    br %r14
+  %and0 = and i32 %a0, %a1
+  %not = xor i32 %a0, -1
+  %and1 = and i32 %not, %a2
+  %or = or i32 %and0, %and1
+  store i32 %and1, ptr %p1
+  ret i32 %or
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/Thumb2/float-cmp.ll b/external/llvm-project/llvm/test/CodeGen/Thumb2/float-cmp.ll
index 73e0063a9278..ed8054437720 100644
--- a/external/llvm-project/llvm/test/CodeGen/Thumb2/float-cmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Thumb2/float-cmp.ll
@@ -200,8 +200,13 @@ define i1 @cmp_d_one(double %a, double %b) {
 ; CHECK-LABEL: cmp_d_one:
 ; NONE: bl __aeabi_dcmpeq
 ; NONE: bl __aeabi_dcmpun
-; SP: bl __aeabi_dcmpeq
 ; SP: bl __aeabi_dcmpun
+; SP: eor r8, r0, #1
+; SP: bl __aeabi_dcmpeq
+; SP-NEXT: clz r0, r0
+; SP-NEXT: lsrs r0, r0, #5
+; SP-NEXT: ands.w r0, r0, r8
+
 ; DP: vcmp.f64
 ; DP: movmi r0, #1
 ; DP: movgt r0, #1
diff --git a/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-frint.ll b/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-frint.ll
index 1d7dcc8bf844..6946ec37ddf3 100644
--- a/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-frint.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-frint.ll
@@ -424,21 +424,74 @@ entry:
   ret <2 x double> %0
 }
 
-declare <4 x float> @llvm.ceil.v4f32(<4 x float>)
-declare <4 x float> @llvm.trunc.v4f32(<4 x float>)
-declare <4 x float> @llvm.rint.v4f32(<4 x float>)
-declare <4 x float> @llvm.nearbyint.v4f32(<4 x float>)
-declare <4 x float> @llvm.floor.v4f32(<4 x float>)
-declare <4 x float> @llvm.round.v4f32(<4 x float>)
-declare <8 x half> @llvm.ceil.v8f16(<8 x half>)
-declare <8 x half> @llvm.trunc.v8f16(<8 x half>)
-declare <8 x half> @llvm.rint.v8f16(<8 x half>)
-declare <8 x half> @llvm.nearbyint.v8f16(<8 x half>)
-declare <8 x half> @llvm.floor.v8f16(<8 x half>)
-declare <8 x half> @llvm.round.v8f16(<8 x half>)
-declare <2 x double> @llvm.ceil.v2f64(<2 x double>)
-declare <2 x double> @llvm.trunc.v2f64(<2 x double>)
-declare <2 x double> @llvm.rint.v2f64(<2 x double>)
-declare <2 x double> @llvm.nearbyint.v2f64(<2 x double>)
-declare <2 x double> @llvm.floor.v2f64(<2 x double>)
-declare <2 x double> @llvm.round.v2f64(<2 x double>)
+define arm_aapcs_vfpcc <4 x float> @froundeven_float32_t(<4 x float> %src) {
+; CHECK-MVE-LABEL: froundeven_float32_t:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vrintn.f32 s3, s3
+; CHECK-MVE-NEXT:    vrintn.f32 s2, s2
+; CHECK-MVE-NEXT:    vrintn.f32 s1, s1
+; CHECK-MVE-NEXT:    vrintn.f32 s0, s0
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: froundeven_float32_t:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vrintn.f32 q0, q0
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %0 = call fast <4 x float> @llvm.roundeven.v4f32(<4 x float> %src)
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @froundeven_float16_t(<8 x half> %src) {
+; CHECK-MVE-LABEL: froundeven_float16_t:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s0
+; CHECK-MVE-NEXT:    vrintn.f16 s0, s0
+; CHECK-MVE-NEXT:    vrintn.f16 s4, s4
+; CHECK-MVE-NEXT:    vins.f16 s0, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s1
+; CHECK-MVE-NEXT:    vrintn.f16 s4, s4
+; CHECK-MVE-NEXT:    vrintn.f16 s1, s1
+; CHECK-MVE-NEXT:    vins.f16 s1, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s2
+; CHECK-MVE-NEXT:    vrintn.f16 s4, s4
+; CHECK-MVE-NEXT:    vrintn.f16 s2, s2
+; CHECK-MVE-NEXT:    vins.f16 s2, s4
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s3
+; CHECK-MVE-NEXT:    vrintn.f16 s4, s4
+; CHECK-MVE-NEXT:    vrintn.f16 s3, s3
+; CHECK-MVE-NEXT:    vins.f16 s3, s4
+; CHECK-MVE-NEXT:    bx lr
+;
+; CHECK-MVEFP-LABEL: froundeven_float16_t:
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:    vrintn.f16 q0, q0
+; CHECK-MVEFP-NEXT:    bx lr
+entry:
+  %0 = call fast <8 x half> @llvm.roundeven.v8f16(<8 x half> %src)
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <2 x double> @froundeven_float64_t(<2 x double> %src) {
+; CHECK-LABEL: froundeven_float64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, r1, d9
+; CHECK-NEXT:    bl roundeven
+; CHECK-NEXT:    vmov r2, r3, d8
+; CHECK-NEXT:    vmov d9, r0, r1
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    bl roundeven
+; CHECK-NEXT:    vmov d8, r0, r1
+; CHECK-NEXT:    vmov q0, q4
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %0 = call fast <2 x double> @llvm.roundeven.v2f64(<2 x double> %src)
+  ret <2 x double> %0
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrintn.ll b/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrintn.ll
index a70975e1e731..b30bb2e3ad3f 100644
--- a/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrintn.ll
+++ b/external/llvm-project/llvm/test/CodeGen/Thumb2/mve-intrinsics/vrintn.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
 
+; The llvm.arm.mve.vrintn should auto-upgrade to llvm.roundeven, which are selected to vrintn.
+
 define arm_aapcs_vfpcc <8 x half> @test_vrndnq_f16(<8 x half> %a) {
 ; CHECK-LABEL: test_vrndnq_f16:
 ; CHECK:       @ %bb.0: @ %entry
diff --git a/external/llvm-project/llvm/test/CodeGen/WebAssembly/aliases.ll b/external/llvm-project/llvm/test/CodeGen/WebAssembly/aliases.ll
index 91b57b90df1d..87b292f53c62 100644
--- a/external/llvm-project/llvm/test/CodeGen/WebAssembly/aliases.ll
+++ b/external/llvm-project/llvm/test/CodeGen/WebAssembly/aliases.ll
@@ -4,11 +4,11 @@
 @bar = global i32 42
 
 ; CHECK-DAG: .globl	foo1
-; CHECK-DAG: .set foo1, bar
+; CHECK-DAG: foo1 = bar
 @foo1 = alias i32, ptr @bar
 
 ; CHECK-DAG: .globl	foo2
-; CHECK-DAG: .set foo2, bar
+; CHECK-DAG: foo2 = bar
 @foo2 = alias i32, ptr @bar
 
 %FunTy = type i32()
@@ -19,14 +19,14 @@ define i32 @foo_f() {
 
 ; CHECK-DAG: .weak	bar_f
 ; CHECK-DAG: .type	bar_f,@function
-; CHECK-DAG: .set bar_f, foo_f
+; CHECK-DAG: bar_f = foo_f
 @bar_f = weak alias %FunTy, ptr @foo_f
 
 ; CHECK-DAG: .weak	bar_l
-; CHECK-DAG: .set bar_l, bar
+; CHECK-DAG: bar_l = bar
 @bar_l = linkonce_odr alias i32, ptr @bar
 
-; CHECK-DAG: .set bar_i, bar
+; CHECK-DAG: bar_i = bar
 @bar_i = internal alias i32, ptr @bar
 
 ; CHECK-DAG: .globl	A
@@ -34,24 +34,24 @@ define i32 @foo_f() {
 
 ; CHECK-DAG: .globl	bar_h
 ; CHECK-DAG: .hidden	bar_h
-; CHECK-DAG: .set bar_h, bar
+; CHECK-DAG: bar_h = bar
 @bar_h = hidden alias i32, ptr @bar
 
 ; CHECK-DAG: .globl	bar_p
 ; CHECK-DAG: .protected	bar_p
-; CHECK-DAG: .set bar_p, bar
+; CHECK-DAG: bar_p = bar
 @bar_p = protected alias i32, ptr @bar
 
-; CHECK-DAG: .set test2, bar+4
+; CHECK-DAG: test2 = bar+4
 @test2 = alias i32, getelementptr(i32, ptr @bar, i32 1)
 
-; CHECK-DAG: .set test3, 42
+; CHECK-DAG: test3 = 42
 @test3 = alias i32, inttoptr(i32 42 to ptr)
 
-; CHECK-DAG: .set test4, bar
+; CHECK-DAG: test4 = bar
 @test4 = alias i32, inttoptr(i64 ptrtoint (ptr @bar to i64) to ptr)
 
-; CHECK-DAG: .set test5, test2-bar
+; CHECK-DAG: test5 = test2-bar
 @test5 = alias i32, inttoptr(i32 sub (i32 ptrtoint (ptr @test2 to i32),
                                  i32 ptrtoint (ptr @bar to i32)) to ptr)
 
diff --git a/external/llvm-project/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/external/llvm-project/llvm/test/CodeGen/WebAssembly/simd-arith.ll
index 185c46aa5681..e3607e12bf53 100644
--- a/external/llvm-project/llvm/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/external/llvm-project/llvm/test/CodeGen/WebAssembly/simd-arith.ll
@@ -4465,203 +4465,139 @@ define <16 x i8> @bitselect_v16i8(<16 x i8> %c, <16 x i8> %v1, <16 x i8> %v2) {
 ; NO-SIMD128-LABEL: bitselect_v16i8:
 ; NO-SIMD128:         .functype bitselect_v16i8 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    i32.and $push0=, $16, $32
-; NO-SIMD128-NEXT:    i32.const $push1=, -1
-; NO-SIMD128-NEXT:    i32.xor $push2=, $16, $pop1
-; NO-SIMD128-NEXT:    i32.and $push3=, $pop2, $48
-; NO-SIMD128-NEXT:    i32.or $push4=, $pop0, $pop3
-; NO-SIMD128-NEXT:    i32.store8 15($0), $pop4
-; NO-SIMD128-NEXT:    i32.and $push5=, $15, $31
-; NO-SIMD128-NEXT:    i32.const $push79=, -1
-; NO-SIMD128-NEXT:    i32.xor $push6=, $15, $pop79
-; NO-SIMD128-NEXT:    i32.and $push7=, $pop6, $47
-; NO-SIMD128-NEXT:    i32.or $push8=, $pop5, $pop7
-; NO-SIMD128-NEXT:    i32.store8 14($0), $pop8
-; NO-SIMD128-NEXT:    i32.and $push9=, $14, $30
-; NO-SIMD128-NEXT:    i32.const $push78=, -1
-; NO-SIMD128-NEXT:    i32.xor $push10=, $14, $pop78
-; NO-SIMD128-NEXT:    i32.and $push11=, $pop10, $46
-; NO-SIMD128-NEXT:    i32.or $push12=, $pop9, $pop11
-; NO-SIMD128-NEXT:    i32.store8 13($0), $pop12
-; NO-SIMD128-NEXT:    i32.and $push13=, $13, $29
-; NO-SIMD128-NEXT:    i32.const $push77=, -1
-; NO-SIMD128-NEXT:    i32.xor $push14=, $13, $pop77
-; NO-SIMD128-NEXT:    i32.and $push15=, $pop14, $45
-; NO-SIMD128-NEXT:    i32.or $push16=, $pop13, $pop15
-; NO-SIMD128-NEXT:    i32.store8 12($0), $pop16
-; NO-SIMD128-NEXT:    i32.and $push17=, $12, $28
-; NO-SIMD128-NEXT:    i32.const $push76=, -1
-; NO-SIMD128-NEXT:    i32.xor $push18=, $12, $pop76
-; NO-SIMD128-NEXT:    i32.and $push19=, $pop18, $44
-; NO-SIMD128-NEXT:    i32.or $push20=, $pop17, $pop19
-; NO-SIMD128-NEXT:    i32.store8 11($0), $pop20
-; NO-SIMD128-NEXT:    i32.and $push21=, $11, $27
-; NO-SIMD128-NEXT:    i32.const $push75=, -1
-; NO-SIMD128-NEXT:    i32.xor $push22=, $11, $pop75
-; NO-SIMD128-NEXT:    i32.and $push23=, $pop22, $43
-; NO-SIMD128-NEXT:    i32.or $push24=, $pop21, $pop23
-; NO-SIMD128-NEXT:    i32.store8 10($0), $pop24
-; NO-SIMD128-NEXT:    i32.and $push25=, $10, $26
-; NO-SIMD128-NEXT:    i32.const $push74=, -1
-; NO-SIMD128-NEXT:    i32.xor $push26=, $10, $pop74
-; NO-SIMD128-NEXT:    i32.and $push27=, $pop26, $42
-; NO-SIMD128-NEXT:    i32.or $push28=, $pop25, $pop27
-; NO-SIMD128-NEXT:    i32.store8 9($0), $pop28
-; NO-SIMD128-NEXT:    i32.and $push29=, $9, $25
-; NO-SIMD128-NEXT:    i32.const $push73=, -1
-; NO-SIMD128-NEXT:    i32.xor $push30=, $9, $pop73
-; NO-SIMD128-NEXT:    i32.and $push31=, $pop30, $41
-; NO-SIMD128-NEXT:    i32.or $push32=, $pop29, $pop31
-; NO-SIMD128-NEXT:    i32.store8 8($0), $pop32
-; NO-SIMD128-NEXT:    i32.and $push33=, $8, $24
-; NO-SIMD128-NEXT:    i32.const $push72=, -1
-; NO-SIMD128-NEXT:    i32.xor $push34=, $8, $pop72
-; NO-SIMD128-NEXT:    i32.and $push35=, $pop34, $40
-; NO-SIMD128-NEXT:    i32.or $push36=, $pop33, $pop35
-; NO-SIMD128-NEXT:    i32.store8 7($0), $pop36
-; NO-SIMD128-NEXT:    i32.and $push37=, $7, $23
-; NO-SIMD128-NEXT:    i32.const $push71=, -1
-; NO-SIMD128-NEXT:    i32.xor $push38=, $7, $pop71
-; NO-SIMD128-NEXT:    i32.and $push39=, $pop38, $39
-; NO-SIMD128-NEXT:    i32.or $push40=, $pop37, $pop39
-; NO-SIMD128-NEXT:    i32.store8 6($0), $pop40
-; NO-SIMD128-NEXT:    i32.and $push41=, $6, $22
-; NO-SIMD128-NEXT:    i32.const $push70=, -1
-; NO-SIMD128-NEXT:    i32.xor $push42=, $6, $pop70
-; NO-SIMD128-NEXT:    i32.and $push43=, $pop42, $38
-; NO-SIMD128-NEXT:    i32.or $push44=, $pop41, $pop43
-; NO-SIMD128-NEXT:    i32.store8 5($0), $pop44
-; NO-SIMD128-NEXT:    i32.and $push45=, $5, $21
-; NO-SIMD128-NEXT:    i32.const $push69=, -1
-; NO-SIMD128-NEXT:    i32.xor $push46=, $5, $pop69
-; NO-SIMD128-NEXT:    i32.and $push47=, $pop46, $37
-; NO-SIMD128-NEXT:    i32.or $push48=, $pop45, $pop47
-; NO-SIMD128-NEXT:    i32.store8 4($0), $pop48
-; NO-SIMD128-NEXT:    i32.and $push49=, $4, $20
-; NO-SIMD128-NEXT:    i32.const $push68=, -1
-; NO-SIMD128-NEXT:    i32.xor $push50=, $4, $pop68
-; NO-SIMD128-NEXT:    i32.and $push51=, $pop50, $36
-; NO-SIMD128-NEXT:    i32.or $push52=, $pop49, $pop51
-; NO-SIMD128-NEXT:    i32.store8 3($0), $pop52
-; NO-SIMD128-NEXT:    i32.and $push53=, $3, $19
-; NO-SIMD128-NEXT:    i32.const $push67=, -1
-; NO-SIMD128-NEXT:    i32.xor $push54=, $3, $pop67
-; NO-SIMD128-NEXT:    i32.and $push55=, $pop54, $35
-; NO-SIMD128-NEXT:    i32.or $push56=, $pop53, $pop55
-; NO-SIMD128-NEXT:    i32.store8 2($0), $pop56
-; NO-SIMD128-NEXT:    i32.and $push57=, $2, $18
-; NO-SIMD128-NEXT:    i32.const $push66=, -1
-; NO-SIMD128-NEXT:    i32.xor $push58=, $2, $pop66
-; NO-SIMD128-NEXT:    i32.and $push59=, $pop58, $34
-; NO-SIMD128-NEXT:    i32.or $push60=, $pop57, $pop59
-; NO-SIMD128-NEXT:    i32.store8 1($0), $pop60
-; NO-SIMD128-NEXT:    i32.and $push61=, $1, $17
-; NO-SIMD128-NEXT:    i32.const $push65=, -1
-; NO-SIMD128-NEXT:    i32.xor $push62=, $1, $pop65
-; NO-SIMD128-NEXT:    i32.and $push63=, $pop62, $33
-; NO-SIMD128-NEXT:    i32.or $push64=, $pop61, $pop63
-; NO-SIMD128-NEXT:    i32.store8 0($0), $pop64
+; NO-SIMD128-NEXT:    i32.xor $push0=, $32, $48
+; NO-SIMD128-NEXT:    i32.and $push1=, $pop0, $16
+; NO-SIMD128-NEXT:    i32.xor $push2=, $pop1, $48
+; NO-SIMD128-NEXT:    i32.store8 15($0), $pop2
+; NO-SIMD128-NEXT:    i32.xor $push3=, $31, $47
+; NO-SIMD128-NEXT:    i32.and $push4=, $pop3, $15
+; NO-SIMD128-NEXT:    i32.xor $push5=, $pop4, $47
+; NO-SIMD128-NEXT:    i32.store8 14($0), $pop5
+; NO-SIMD128-NEXT:    i32.xor $push6=, $30, $46
+; NO-SIMD128-NEXT:    i32.and $push7=, $pop6, $14
+; NO-SIMD128-NEXT:    i32.xor $push8=, $pop7, $46
+; NO-SIMD128-NEXT:    i32.store8 13($0), $pop8
+; NO-SIMD128-NEXT:    i32.xor $push9=, $29, $45
+; NO-SIMD128-NEXT:    i32.and $push10=, $pop9, $13
+; NO-SIMD128-NEXT:    i32.xor $push11=, $pop10, $45
+; NO-SIMD128-NEXT:    i32.store8 12($0), $pop11
+; NO-SIMD128-NEXT:    i32.xor $push12=, $28, $44
+; NO-SIMD128-NEXT:    i32.and $push13=, $pop12, $12
+; NO-SIMD128-NEXT:    i32.xor $push14=, $pop13, $44
+; NO-SIMD128-NEXT:    i32.store8 11($0), $pop14
+; NO-SIMD128-NEXT:    i32.xor $push15=, $27, $43
+; NO-SIMD128-NEXT:    i32.and $push16=, $pop15, $11
+; NO-SIMD128-NEXT:    i32.xor $push17=, $pop16, $43
+; NO-SIMD128-NEXT:    i32.store8 10($0), $pop17
+; NO-SIMD128-NEXT:    i32.xor $push18=, $26, $42
+; NO-SIMD128-NEXT:    i32.and $push19=, $pop18, $10
+; NO-SIMD128-NEXT:    i32.xor $push20=, $pop19, $42
+; NO-SIMD128-NEXT:    i32.store8 9($0), $pop20
+; NO-SIMD128-NEXT:    i32.xor $push21=, $25, $41
+; NO-SIMD128-NEXT:    i32.and $push22=, $pop21, $9
+; NO-SIMD128-NEXT:    i32.xor $push23=, $pop22, $41
+; NO-SIMD128-NEXT:    i32.store8 8($0), $pop23
+; NO-SIMD128-NEXT:    i32.xor $push24=, $24, $40
+; NO-SIMD128-NEXT:    i32.and $push25=, $pop24, $8
+; NO-SIMD128-NEXT:    i32.xor $push26=, $pop25, $40
+; NO-SIMD128-NEXT:    i32.store8 7($0), $pop26
+; NO-SIMD128-NEXT:    i32.xor $push27=, $23, $39
+; NO-SIMD128-NEXT:    i32.and $push28=, $pop27, $7
+; NO-SIMD128-NEXT:    i32.xor $push29=, $pop28, $39
+; NO-SIMD128-NEXT:    i32.store8 6($0), $pop29
+; NO-SIMD128-NEXT:    i32.xor $push30=, $22, $38
+; NO-SIMD128-NEXT:    i32.and $push31=, $pop30, $6
+; NO-SIMD128-NEXT:    i32.xor $push32=, $pop31, $38
+; NO-SIMD128-NEXT:    i32.store8 5($0), $pop32
+; NO-SIMD128-NEXT:    i32.xor $push33=, $21, $37
+; NO-SIMD128-NEXT:    i32.and $push34=, $pop33, $5
+; NO-SIMD128-NEXT:    i32.xor $push35=, $pop34, $37
+; NO-SIMD128-NEXT:    i32.store8 4($0), $pop35
+; NO-SIMD128-NEXT:    i32.xor $push36=, $20, $36
+; NO-SIMD128-NEXT:    i32.and $push37=, $pop36, $4
+; NO-SIMD128-NEXT:    i32.xor $push38=, $pop37, $36
+; NO-SIMD128-NEXT:    i32.store8 3($0), $pop38
+; NO-SIMD128-NEXT:    i32.xor $push39=, $19, $35
+; NO-SIMD128-NEXT:    i32.and $push40=, $pop39, $3
+; NO-SIMD128-NEXT:    i32.xor $push41=, $pop40, $35
+; NO-SIMD128-NEXT:    i32.store8 2($0), $pop41
+; NO-SIMD128-NEXT:    i32.xor $push42=, $18, $34
+; NO-SIMD128-NEXT:    i32.and $push43=, $pop42, $2
+; NO-SIMD128-NEXT:    i32.xor $push44=, $pop43, $34
+; NO-SIMD128-NEXT:    i32.store8 1($0), $pop44
+; NO-SIMD128-NEXT:    i32.xor $push45=, $17, $33
+; NO-SIMD128-NEXT:    i32.and $push46=, $pop45, $1
+; NO-SIMD128-NEXT:    i32.xor $push47=, $pop46, $33
+; NO-SIMD128-NEXT:    i32.store8 0($0), $pop47
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: bitselect_v16i8:
 ; NO-SIMD128-FAST:         .functype bitselect_v16i8 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    i32.and $push0=, $1, $17
-; NO-SIMD128-FAST-NEXT:    i32.const $push1=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push2=, $1, $pop1
-; NO-SIMD128-FAST-NEXT:    i32.and $push3=, $pop2, $33
-; NO-SIMD128-FAST-NEXT:    i32.or $push4=, $pop0, $pop3
-; NO-SIMD128-FAST-NEXT:    i32.store8 0($0), $pop4
-; NO-SIMD128-FAST-NEXT:    i32.and $push5=, $2, $18
-; NO-SIMD128-FAST-NEXT:    i32.const $push79=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push6=, $2, $pop79
-; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $pop6, $34
-; NO-SIMD128-FAST-NEXT:    i32.or $push8=, $pop5, $pop7
-; NO-SIMD128-FAST-NEXT:    i32.store8 1($0), $pop8
-; NO-SIMD128-FAST-NEXT:    i32.and $push9=, $3, $19
-; NO-SIMD128-FAST-NEXT:    i32.const $push78=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push10=, $3, $pop78
-; NO-SIMD128-FAST-NEXT:    i32.and $push11=, $pop10, $35
-; NO-SIMD128-FAST-NEXT:    i32.or $push12=, $pop9, $pop11
-; NO-SIMD128-FAST-NEXT:    i32.store8 2($0), $pop12
-; NO-SIMD128-FAST-NEXT:    i32.and $push13=, $4, $20
-; NO-SIMD128-FAST-NEXT:    i32.const $push77=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push14=, $4, $pop77
-; NO-SIMD128-FAST-NEXT:    i32.and $push15=, $pop14, $36
-; NO-SIMD128-FAST-NEXT:    i32.or $push16=, $pop13, $pop15
-; NO-SIMD128-FAST-NEXT:    i32.store8 3($0), $pop16
-; NO-SIMD128-FAST-NEXT:    i32.and $push17=, $5, $21
-; NO-SIMD128-FAST-NEXT:    i32.const $push76=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push18=, $5, $pop76
-; NO-SIMD128-FAST-NEXT:    i32.and $push19=, $pop18, $37
-; NO-SIMD128-FAST-NEXT:    i32.or $push20=, $pop17, $pop19
-; NO-SIMD128-FAST-NEXT:    i32.store8 4($0), $pop20
-; NO-SIMD128-FAST-NEXT:    i32.and $push21=, $6, $22
-; NO-SIMD128-FAST-NEXT:    i32.const $push75=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push22=, $6, $pop75
-; NO-SIMD128-FAST-NEXT:    i32.and $push23=, $pop22, $38
-; NO-SIMD128-FAST-NEXT:    i32.or $push24=, $pop21, $pop23
-; NO-SIMD128-FAST-NEXT:    i32.store8 5($0), $pop24
-; NO-SIMD128-FAST-NEXT:    i32.and $push25=, $7, $23
-; NO-SIMD128-FAST-NEXT:    i32.const $push74=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push26=, $7, $pop74
-; NO-SIMD128-FAST-NEXT:    i32.and $push27=, $pop26, $39
-; NO-SIMD128-FAST-NEXT:    i32.or $push28=, $pop25, $pop27
-; NO-SIMD128-FAST-NEXT:    i32.store8 6($0), $pop28
-; NO-SIMD128-FAST-NEXT:    i32.and $push29=, $8, $24
-; NO-SIMD128-FAST-NEXT:    i32.const $push73=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push30=, $8, $pop73
-; NO-SIMD128-FAST-NEXT:    i32.and $push31=, $pop30, $40
-; NO-SIMD128-FAST-NEXT:    i32.or $push32=, $pop29, $pop31
-; NO-SIMD128-FAST-NEXT:    i32.store8 7($0), $pop32
-; NO-SIMD128-FAST-NEXT:    i32.and $push33=, $9, $25
-; NO-SIMD128-FAST-NEXT:    i32.const $push72=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push34=, $9, $pop72
-; NO-SIMD128-FAST-NEXT:    i32.and $push35=, $pop34, $41
-; NO-SIMD128-FAST-NEXT:    i32.or $push36=, $pop33, $pop35
-; NO-SIMD128-FAST-NEXT:    i32.store8 8($0), $pop36
-; NO-SIMD128-FAST-NEXT:    i32.and $push37=, $10, $26
-; NO-SIMD128-FAST-NEXT:    i32.const $push71=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push38=, $10, $pop71
-; NO-SIMD128-FAST-NEXT:    i32.and $push39=, $pop38, $42
-; NO-SIMD128-FAST-NEXT:    i32.or $push40=, $pop37, $pop39
-; NO-SIMD128-FAST-NEXT:    i32.store8 9($0), $pop40
-; NO-SIMD128-FAST-NEXT:    i32.and $push41=, $11, $27
-; NO-SIMD128-FAST-NEXT:    i32.const $push70=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push42=, $11, $pop70
-; NO-SIMD128-FAST-NEXT:    i32.and $push43=, $pop42, $43
-; NO-SIMD128-FAST-NEXT:    i32.or $push44=, $pop41, $pop43
-; NO-SIMD128-FAST-NEXT:    i32.store8 10($0), $pop44
-; NO-SIMD128-FAST-NEXT:    i32.and $push45=, $12, $28
-; NO-SIMD128-FAST-NEXT:    i32.const $push69=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push46=, $12, $pop69
-; NO-SIMD128-FAST-NEXT:    i32.and $push47=, $pop46, $44
-; NO-SIMD128-FAST-NEXT:    i32.or $push48=, $pop45, $pop47
-; NO-SIMD128-FAST-NEXT:    i32.store8 11($0), $pop48
-; NO-SIMD128-FAST-NEXT:    i32.and $push49=, $13, $29
-; NO-SIMD128-FAST-NEXT:    i32.const $push68=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push50=, $13, $pop68
-; NO-SIMD128-FAST-NEXT:    i32.and $push51=, $pop50, $45
-; NO-SIMD128-FAST-NEXT:    i32.or $push52=, $pop49, $pop51
-; NO-SIMD128-FAST-NEXT:    i32.store8 12($0), $pop52
-; NO-SIMD128-FAST-NEXT:    i32.and $push53=, $14, $30
-; NO-SIMD128-FAST-NEXT:    i32.const $push67=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push54=, $14, $pop67
-; NO-SIMD128-FAST-NEXT:    i32.and $push55=, $pop54, $46
-; NO-SIMD128-FAST-NEXT:    i32.or $push56=, $pop53, $pop55
-; NO-SIMD128-FAST-NEXT:    i32.store8 13($0), $pop56
-; NO-SIMD128-FAST-NEXT:    i32.and $push57=, $15, $31
-; NO-SIMD128-FAST-NEXT:    i32.const $push66=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push58=, $15, $pop66
-; NO-SIMD128-FAST-NEXT:    i32.and $push59=, $pop58, $47
-; NO-SIMD128-FAST-NEXT:    i32.or $push60=, $pop57, $pop59
-; NO-SIMD128-FAST-NEXT:    i32.store8 14($0), $pop60
-; NO-SIMD128-FAST-NEXT:    i32.and $push61=, $16, $32
-; NO-SIMD128-FAST-NEXT:    i32.const $push65=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push62=, $16, $pop65
-; NO-SIMD128-FAST-NEXT:    i32.and $push63=, $pop62, $48
-; NO-SIMD128-FAST-NEXT:    i32.or $push64=, $pop61, $pop63
-; NO-SIMD128-FAST-NEXT:    i32.store8 15($0), $pop64
+; NO-SIMD128-FAST-NEXT:    i32.xor $push0=, $17, $33
+; NO-SIMD128-FAST-NEXT:    i32.and $push1=, $pop0, $1
+; NO-SIMD128-FAST-NEXT:    i32.xor $push2=, $pop1, $33
+; NO-SIMD128-FAST-NEXT:    i32.store8 0($0), $pop2
+; NO-SIMD128-FAST-NEXT:    i32.xor $push3=, $18, $34
+; NO-SIMD128-FAST-NEXT:    i32.and $push4=, $pop3, $2
+; NO-SIMD128-FAST-NEXT:    i32.xor $push5=, $pop4, $34
+; NO-SIMD128-FAST-NEXT:    i32.store8 1($0), $pop5
+; NO-SIMD128-FAST-NEXT:    i32.xor $push6=, $19, $35
+; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $pop6, $3
+; NO-SIMD128-FAST-NEXT:    i32.xor $push8=, $pop7, $35
+; NO-SIMD128-FAST-NEXT:    i32.store8 2($0), $pop8
+; NO-SIMD128-FAST-NEXT:    i32.xor $push9=, $20, $36
+; NO-SIMD128-FAST-NEXT:    i32.and $push10=, $pop9, $4
+; NO-SIMD128-FAST-NEXT:    i32.xor $push11=, $pop10, $36
+; NO-SIMD128-FAST-NEXT:    i32.store8 3($0), $pop11
+; NO-SIMD128-FAST-NEXT:    i32.xor $push12=, $21, $37
+; NO-SIMD128-FAST-NEXT:    i32.and $push13=, $pop12, $5
+; NO-SIMD128-FAST-NEXT:    i32.xor $push14=, $pop13, $37
+; NO-SIMD128-FAST-NEXT:    i32.store8 4($0), $pop14
+; NO-SIMD128-FAST-NEXT:    i32.xor $push15=, $22, $38
+; NO-SIMD128-FAST-NEXT:    i32.and $push16=, $pop15, $6
+; NO-SIMD128-FAST-NEXT:    i32.xor $push17=, $pop16, $38
+; NO-SIMD128-FAST-NEXT:    i32.store8 5($0), $pop17
+; NO-SIMD128-FAST-NEXT:    i32.xor $push18=, $23, $39
+; NO-SIMD128-FAST-NEXT:    i32.and $push19=, $pop18, $7
+; NO-SIMD128-FAST-NEXT:    i32.xor $push20=, $pop19, $39
+; NO-SIMD128-FAST-NEXT:    i32.store8 6($0), $pop20
+; NO-SIMD128-FAST-NEXT:    i32.xor $push21=, $24, $40
+; NO-SIMD128-FAST-NEXT:    i32.and $push22=, $pop21, $8
+; NO-SIMD128-FAST-NEXT:    i32.xor $push23=, $pop22, $40
+; NO-SIMD128-FAST-NEXT:    i32.store8 7($0), $pop23
+; NO-SIMD128-FAST-NEXT:    i32.xor $push24=, $25, $41
+; NO-SIMD128-FAST-NEXT:    i32.and $push25=, $pop24, $9
+; NO-SIMD128-FAST-NEXT:    i32.xor $push26=, $pop25, $41
+; NO-SIMD128-FAST-NEXT:    i32.store8 8($0), $pop26
+; NO-SIMD128-FAST-NEXT:    i32.xor $push27=, $26, $42
+; NO-SIMD128-FAST-NEXT:    i32.and $push28=, $pop27, $10
+; NO-SIMD128-FAST-NEXT:    i32.xor $push29=, $pop28, $42
+; NO-SIMD128-FAST-NEXT:    i32.store8 9($0), $pop29
+; NO-SIMD128-FAST-NEXT:    i32.xor $push30=, $27, $43
+; NO-SIMD128-FAST-NEXT:    i32.and $push31=, $pop30, $11
+; NO-SIMD128-FAST-NEXT:    i32.xor $push32=, $pop31, $43
+; NO-SIMD128-FAST-NEXT:    i32.store8 10($0), $pop32
+; NO-SIMD128-FAST-NEXT:    i32.xor $push33=, $28, $44
+; NO-SIMD128-FAST-NEXT:    i32.and $push34=, $pop33, $12
+; NO-SIMD128-FAST-NEXT:    i32.xor $push35=, $pop34, $44
+; NO-SIMD128-FAST-NEXT:    i32.store8 11($0), $pop35
+; NO-SIMD128-FAST-NEXT:    i32.xor $push36=, $29, $45
+; NO-SIMD128-FAST-NEXT:    i32.and $push37=, $pop36, $13
+; NO-SIMD128-FAST-NEXT:    i32.xor $push38=, $pop37, $45
+; NO-SIMD128-FAST-NEXT:    i32.store8 12($0), $pop38
+; NO-SIMD128-FAST-NEXT:    i32.xor $push39=, $30, $46
+; NO-SIMD128-FAST-NEXT:    i32.and $push40=, $pop39, $14
+; NO-SIMD128-FAST-NEXT:    i32.xor $push41=, $pop40, $46
+; NO-SIMD128-FAST-NEXT:    i32.store8 13($0), $pop41
+; NO-SIMD128-FAST-NEXT:    i32.xor $push42=, $31, $47
+; NO-SIMD128-FAST-NEXT:    i32.and $push43=, $pop42, $15
+; NO-SIMD128-FAST-NEXT:    i32.xor $push44=, $pop43, $47
+; NO-SIMD128-FAST-NEXT:    i32.store8 14($0), $pop44
+; NO-SIMD128-FAST-NEXT:    i32.xor $push45=, $32, $48
+; NO-SIMD128-FAST-NEXT:    i32.and $push46=, $pop45, $16
+; NO-SIMD128-FAST-NEXT:    i32.xor $push47=, $pop46, $48
+; NO-SIMD128-FAST-NEXT:    i32.store8 15($0), $pop47
 ; NO-SIMD128-FAST-NEXT:    return
   %masked_v1 = and <16 x i8> %c, %v1
   %inv_mask = xor <16 x i8> %c,
@@ -7546,107 +7482,75 @@ define <8 x i16> @bitselect_v8i16(<8 x i16> %c, <8 x i16> %v1, <8 x i16> %v2) {
 ; NO-SIMD128-LABEL: bitselect_v8i16:
 ; NO-SIMD128:         .functype bitselect_v8i16 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    i32.and $push0=, $16, $8
-; NO-SIMD128-NEXT:    i32.const $push1=, -1
-; NO-SIMD128-NEXT:    i32.xor $push2=, $8, $pop1
-; NO-SIMD128-NEXT:    i32.and $push3=, $24, $pop2
-; NO-SIMD128-NEXT:    i32.or $push4=, $pop0, $pop3
-; NO-SIMD128-NEXT:    i32.store16 14($0), $pop4
-; NO-SIMD128-NEXT:    i32.and $push5=, $15, $7
-; NO-SIMD128-NEXT:    i32.const $push39=, -1
-; NO-SIMD128-NEXT:    i32.xor $push6=, $7, $pop39
-; NO-SIMD128-NEXT:    i32.and $push7=, $23, $pop6
-; NO-SIMD128-NEXT:    i32.or $push8=, $pop5, $pop7
-; NO-SIMD128-NEXT:    i32.store16 12($0), $pop8
-; NO-SIMD128-NEXT:    i32.and $push9=, $14, $6
-; NO-SIMD128-NEXT:    i32.const $push38=, -1
-; NO-SIMD128-NEXT:    i32.xor $push10=, $6, $pop38
-; NO-SIMD128-NEXT:    i32.and $push11=, $22, $pop10
-; NO-SIMD128-NEXT:    i32.or $push12=, $pop9, $pop11
-; NO-SIMD128-NEXT:    i32.store16 10($0), $pop12
-; NO-SIMD128-NEXT:    i32.and $push13=, $13, $5
-; NO-SIMD128-NEXT:    i32.const $push37=, -1
-; NO-SIMD128-NEXT:    i32.xor $push14=, $5, $pop37
-; NO-SIMD128-NEXT:    i32.and $push15=, $21, $pop14
-; NO-SIMD128-NEXT:    i32.or $push16=, $pop13, $pop15
-; NO-SIMD128-NEXT:    i32.store16 8($0), $pop16
-; NO-SIMD128-NEXT:    i32.and $push17=, $12, $4
-; NO-SIMD128-NEXT:    i32.const $push36=, -1
-; NO-SIMD128-NEXT:    i32.xor $push18=, $4, $pop36
-; NO-SIMD128-NEXT:    i32.and $push19=, $20, $pop18
-; NO-SIMD128-NEXT:    i32.or $push20=, $pop17, $pop19
-; NO-SIMD128-NEXT:    i32.store16 6($0), $pop20
-; NO-SIMD128-NEXT:    i32.and $push21=, $11, $3
-; NO-SIMD128-NEXT:    i32.const $push35=, -1
-; NO-SIMD128-NEXT:    i32.xor $push22=, $3, $pop35
-; NO-SIMD128-NEXT:    i32.and $push23=, $19, $pop22
-; NO-SIMD128-NEXT:    i32.or $push24=, $pop21, $pop23
-; NO-SIMD128-NEXT:    i32.store16 4($0), $pop24
-; NO-SIMD128-NEXT:    i32.and $push25=, $10, $2
-; NO-SIMD128-NEXT:    i32.const $push34=, -1
-; NO-SIMD128-NEXT:    i32.xor $push26=, $2, $pop34
-; NO-SIMD128-NEXT:    i32.and $push27=, $18, $pop26
-; NO-SIMD128-NEXT:    i32.or $push28=, $pop25, $pop27
-; NO-SIMD128-NEXT:    i32.store16 2($0), $pop28
-; NO-SIMD128-NEXT:    i32.and $push29=, $9, $1
-; NO-SIMD128-NEXT:    i32.const $push33=, -1
-; NO-SIMD128-NEXT:    i32.xor $push30=, $1, $pop33
-; NO-SIMD128-NEXT:    i32.and $push31=, $17, $pop30
-; NO-SIMD128-NEXT:    i32.or $push32=, $pop29, $pop31
-; NO-SIMD128-NEXT:    i32.store16 0($0), $pop32
+; NO-SIMD128-NEXT:    i32.xor $push0=, $16, $24
+; NO-SIMD128-NEXT:    i32.and $push1=, $pop0, $8
+; NO-SIMD128-NEXT:    i32.xor $push2=, $pop1, $24
+; NO-SIMD128-NEXT:    i32.store16 14($0), $pop2
+; NO-SIMD128-NEXT:    i32.xor $push3=, $15, $23
+; NO-SIMD128-NEXT:    i32.and $push4=, $pop3, $7
+; NO-SIMD128-NEXT:    i32.xor $push5=, $pop4, $23
+; NO-SIMD128-NEXT:    i32.store16 12($0), $pop5
+; NO-SIMD128-NEXT:    i32.xor $push6=, $14, $22
+; NO-SIMD128-NEXT:    i32.and $push7=, $pop6, $6
+; NO-SIMD128-NEXT:    i32.xor $push8=, $pop7, $22
+; NO-SIMD128-NEXT:    i32.store16 10($0), $pop8
+; NO-SIMD128-NEXT:    i32.xor $push9=, $13, $21
+; NO-SIMD128-NEXT:    i32.and $push10=, $pop9, $5
+; NO-SIMD128-NEXT:    i32.xor $push11=, $pop10, $21
+; NO-SIMD128-NEXT:    i32.store16 8($0), $pop11
+; NO-SIMD128-NEXT:    i32.xor $push12=, $12, $20
+; NO-SIMD128-NEXT:    i32.and $push13=, $pop12, $4
+; NO-SIMD128-NEXT:    i32.xor $push14=, $pop13, $20
+; NO-SIMD128-NEXT:    i32.store16 6($0), $pop14
+; NO-SIMD128-NEXT:    i32.xor $push15=, $11, $19
+; NO-SIMD128-NEXT:    i32.and $push16=, $pop15, $3
+; NO-SIMD128-NEXT:    i32.xor $push17=, $pop16, $19
+; NO-SIMD128-NEXT:    i32.store16 4($0), $pop17
+; NO-SIMD128-NEXT:    i32.xor $push18=, $10, $18
+; NO-SIMD128-NEXT:    i32.and $push19=, $pop18, $2
+; NO-SIMD128-NEXT:    i32.xor $push20=, $pop19, $18
+; NO-SIMD128-NEXT:    i32.store16 2($0), $pop20
+; NO-SIMD128-NEXT:    i32.xor $push21=, $9, $17
+; NO-SIMD128-NEXT:    i32.and $push22=, $pop21, $1
+; NO-SIMD128-NEXT:    i32.xor $push23=, $pop22, $17
+; NO-SIMD128-NEXT:    i32.store16 0($0), $pop23
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: bitselect_v8i16:
 ; NO-SIMD128-FAST:         .functype bitselect_v8i16 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    i32.and $push0=, $9, $1
-; NO-SIMD128-FAST-NEXT:    i32.const $push1=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push2=, $1, $pop1
-; NO-SIMD128-FAST-NEXT:    i32.and $push3=, $17, $pop2
-; NO-SIMD128-FAST-NEXT:    i32.or $push4=, $pop0, $pop3
-; NO-SIMD128-FAST-NEXT:    i32.store16 0($0), $pop4
-; NO-SIMD128-FAST-NEXT:    i32.and $push5=, $10, $2
-; NO-SIMD128-FAST-NEXT:    i32.const $push39=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push6=, $2, $pop39
-; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $18, $pop6
-; NO-SIMD128-FAST-NEXT:    i32.or $push8=, $pop5, $pop7
-; NO-SIMD128-FAST-NEXT:    i32.store16 2($0), $pop8
-; NO-SIMD128-FAST-NEXT:    i32.and $push9=, $11, $3
-; NO-SIMD128-FAST-NEXT:    i32.const $push38=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push10=, $3, $pop38
-; NO-SIMD128-FAST-NEXT:    i32.and $push11=, $19, $pop10
-; NO-SIMD128-FAST-NEXT:    i32.or $push12=, $pop9, $pop11
-; NO-SIMD128-FAST-NEXT:    i32.store16 4($0), $pop12
-; NO-SIMD128-FAST-NEXT:    i32.and $push13=, $12, $4
-; NO-SIMD128-FAST-NEXT:    i32.const $push37=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push14=, $4, $pop37
-; NO-SIMD128-FAST-NEXT:    i32.and $push15=, $20, $pop14
-; NO-SIMD128-FAST-NEXT:    i32.or $push16=, $pop13, $pop15
-; NO-SIMD128-FAST-NEXT:    i32.store16 6($0), $pop16
-; NO-SIMD128-FAST-NEXT:    i32.and $push17=, $13, $5
-; NO-SIMD128-FAST-NEXT:    i32.const $push36=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push18=, $5, $pop36
-; NO-SIMD128-FAST-NEXT:    i32.and $push19=, $21, $pop18
-; NO-SIMD128-FAST-NEXT:    i32.or $push20=, $pop17, $pop19
-; NO-SIMD128-FAST-NEXT:    i32.store16 8($0), $pop20
-; NO-SIMD128-FAST-NEXT:    i32.and $push21=, $14, $6
-; NO-SIMD128-FAST-NEXT:    i32.const $push35=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push22=, $6, $pop35
-; NO-SIMD128-FAST-NEXT:    i32.and $push23=, $22, $pop22
-; NO-SIMD128-FAST-NEXT:    i32.or $push24=, $pop21, $pop23
-; NO-SIMD128-FAST-NEXT:    i32.store16 10($0), $pop24
-; NO-SIMD128-FAST-NEXT:    i32.and $push25=, $15, $7
-; NO-SIMD128-FAST-NEXT:    i32.const $push34=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push26=, $7, $pop34
-; NO-SIMD128-FAST-NEXT:    i32.and $push27=, $23, $pop26
-; NO-SIMD128-FAST-NEXT:    i32.or $push28=, $pop25, $pop27
-; NO-SIMD128-FAST-NEXT:    i32.store16 12($0), $pop28
-; NO-SIMD128-FAST-NEXT:    i32.and $push29=, $16, $8
-; NO-SIMD128-FAST-NEXT:    i32.const $push33=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push30=, $8, $pop33
-; NO-SIMD128-FAST-NEXT:    i32.and $push31=, $24, $pop30
-; NO-SIMD128-FAST-NEXT:    i32.or $push32=, $pop29, $pop31
-; NO-SIMD128-FAST-NEXT:    i32.store16 14($0), $pop32
+; NO-SIMD128-FAST-NEXT:    i32.xor $push0=, $9, $17
+; NO-SIMD128-FAST-NEXT:    i32.and $push1=, $pop0, $1
+; NO-SIMD128-FAST-NEXT:    i32.xor $push2=, $pop1, $17
+; NO-SIMD128-FAST-NEXT:    i32.store16 0($0), $pop2
+; NO-SIMD128-FAST-NEXT:    i32.xor $push3=, $10, $18
+; NO-SIMD128-FAST-NEXT:    i32.and $push4=, $pop3, $2
+; NO-SIMD128-FAST-NEXT:    i32.xor $push5=, $pop4, $18
+; NO-SIMD128-FAST-NEXT:    i32.store16 2($0), $pop5
+; NO-SIMD128-FAST-NEXT:    i32.xor $push6=, $11, $19
+; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $pop6, $3
+; NO-SIMD128-FAST-NEXT:    i32.xor $push8=, $pop7, $19
+; NO-SIMD128-FAST-NEXT:    i32.store16 4($0), $pop8
+; NO-SIMD128-FAST-NEXT:    i32.xor $push9=, $12, $20
+; NO-SIMD128-FAST-NEXT:    i32.and $push10=, $pop9, $4
+; NO-SIMD128-FAST-NEXT:    i32.xor $push11=, $pop10, $20
+; NO-SIMD128-FAST-NEXT:    i32.store16 6($0), $pop11
+; NO-SIMD128-FAST-NEXT:    i32.xor $push12=, $13, $21
+; NO-SIMD128-FAST-NEXT:    i32.and $push13=, $pop12, $5
+; NO-SIMD128-FAST-NEXT:    i32.xor $push14=, $pop13, $21
+; NO-SIMD128-FAST-NEXT:    i32.store16 8($0), $pop14
+; NO-SIMD128-FAST-NEXT:    i32.xor $push15=, $14, $22
+; NO-SIMD128-FAST-NEXT:    i32.and $push16=, $pop15, $6
+; NO-SIMD128-FAST-NEXT:    i32.xor $push17=, $pop16, $22
+; NO-SIMD128-FAST-NEXT:    i32.store16 10($0), $pop17
+; NO-SIMD128-FAST-NEXT:    i32.xor $push18=, $15, $23
+; NO-SIMD128-FAST-NEXT:    i32.and $push19=, $pop18, $7
+; NO-SIMD128-FAST-NEXT:    i32.xor $push20=, $pop19, $23
+; NO-SIMD128-FAST-NEXT:    i32.store16 12($0), $pop20
+; NO-SIMD128-FAST-NEXT:    i32.xor $push21=, $16, $24
+; NO-SIMD128-FAST-NEXT:    i32.and $push22=, $pop21, $8
+; NO-SIMD128-FAST-NEXT:    i32.xor $push23=, $pop22, $24
+; NO-SIMD128-FAST-NEXT:    i32.store16 14($0), $pop23
 ; NO-SIMD128-FAST-NEXT:    return
   %masked_v1 = and <8 x i16> %v1, %c
   %inv_mask = xor <8 x i16>
@@ -9453,59 +9357,43 @@ define <4 x i32> @bitselect_v4i32(<4 x i32> %c, <4 x i32> %v1, <4 x i32> %v2) {
 ; NO-SIMD128-LABEL: bitselect_v4i32:
 ; NO-SIMD128:         .functype bitselect_v4i32 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    i32.const $push1=, -1
-; NO-SIMD128-NEXT:    i32.xor $push2=, $4, $pop1
-; NO-SIMD128-NEXT:    i32.and $push3=, $pop2, $12
-; NO-SIMD128-NEXT:    i32.and $push0=, $4, $8
-; NO-SIMD128-NEXT:    i32.or $push4=, $pop3, $pop0
-; NO-SIMD128-NEXT:    i32.store 12($0), $pop4
-; NO-SIMD128-NEXT:    i32.const $push19=, -1
-; NO-SIMD128-NEXT:    i32.xor $push6=, $3, $pop19
-; NO-SIMD128-NEXT:    i32.and $push7=, $pop6, $11
-; NO-SIMD128-NEXT:    i32.and $push5=, $3, $7
-; NO-SIMD128-NEXT:    i32.or $push8=, $pop7, $pop5
-; NO-SIMD128-NEXT:    i32.store 8($0), $pop8
-; NO-SIMD128-NEXT:    i32.const $push18=, -1
-; NO-SIMD128-NEXT:    i32.xor $push10=, $2, $pop18
-; NO-SIMD128-NEXT:    i32.and $push11=, $pop10, $10
-; NO-SIMD128-NEXT:    i32.and $push9=, $2, $6
-; NO-SIMD128-NEXT:    i32.or $push12=, $pop11, $pop9
-; NO-SIMD128-NEXT:    i32.store 4($0), $pop12
-; NO-SIMD128-NEXT:    i32.const $push17=, -1
-; NO-SIMD128-NEXT:    i32.xor $push14=, $1, $pop17
-; NO-SIMD128-NEXT:    i32.and $push15=, $pop14, $9
-; NO-SIMD128-NEXT:    i32.and $push13=, $1, $5
-; NO-SIMD128-NEXT:    i32.or $push16=, $pop15, $pop13
-; NO-SIMD128-NEXT:    i32.store 0($0), $pop16
+; NO-SIMD128-NEXT:    i32.xor $push0=, $8, $12
+; NO-SIMD128-NEXT:    i32.and $push1=, $pop0, $4
+; NO-SIMD128-NEXT:    i32.xor $push2=, $pop1, $12
+; NO-SIMD128-NEXT:    i32.store 12($0), $pop2
+; NO-SIMD128-NEXT:    i32.xor $push3=, $7, $11
+; NO-SIMD128-NEXT:    i32.and $push4=, $pop3, $3
+; NO-SIMD128-NEXT:    i32.xor $push5=, $pop4, $11
+; NO-SIMD128-NEXT:    i32.store 8($0), $pop5
+; NO-SIMD128-NEXT:    i32.xor $push6=, $6, $10
+; NO-SIMD128-NEXT:    i32.and $push7=, $pop6, $2
+; NO-SIMD128-NEXT:    i32.xor $push8=, $pop7, $10
+; NO-SIMD128-NEXT:    i32.store 4($0), $pop8
+; NO-SIMD128-NEXT:    i32.xor $push9=, $5, $9
+; NO-SIMD128-NEXT:    i32.and $push10=, $pop9, $1
+; NO-SIMD128-NEXT:    i32.xor $push11=, $pop10, $9
+; NO-SIMD128-NEXT:    i32.store 0($0), $pop11
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: bitselect_v4i32:
 ; NO-SIMD128-FAST:         .functype bitselect_v4i32 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    i32.const $push1=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push2=, $1, $pop1
-; NO-SIMD128-FAST-NEXT:    i32.and $push3=, $pop2, $9
-; NO-SIMD128-FAST-NEXT:    i32.and $push0=, $1, $5
-; NO-SIMD128-FAST-NEXT:    i32.or $push4=, $pop3, $pop0
-; NO-SIMD128-FAST-NEXT:    i32.store 0($0), $pop4
-; NO-SIMD128-FAST-NEXT:    i32.const $push19=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push6=, $2, $pop19
-; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $pop6, $10
-; NO-SIMD128-FAST-NEXT:    i32.and $push5=, $2, $6
-; NO-SIMD128-FAST-NEXT:    i32.or $push8=, $pop7, $pop5
-; NO-SIMD128-FAST-NEXT:    i32.store 4($0), $pop8
-; NO-SIMD128-FAST-NEXT:    i32.const $push18=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push10=, $3, $pop18
-; NO-SIMD128-FAST-NEXT:    i32.and $push11=, $pop10, $11
-; NO-SIMD128-FAST-NEXT:    i32.and $push9=, $3, $7
-; NO-SIMD128-FAST-NEXT:    i32.or $push12=, $pop11, $pop9
-; NO-SIMD128-FAST-NEXT:    i32.store 8($0), $pop12
-; NO-SIMD128-FAST-NEXT:    i32.const $push17=, -1
-; NO-SIMD128-FAST-NEXT:    i32.xor $push14=, $4, $pop17
-; NO-SIMD128-FAST-NEXT:    i32.and $push15=, $pop14, $12
-; NO-SIMD128-FAST-NEXT:    i32.and $push13=, $4, $8
-; NO-SIMD128-FAST-NEXT:    i32.or $push16=, $pop15, $pop13
-; NO-SIMD128-FAST-NEXT:    i32.store 12($0), $pop16
+; NO-SIMD128-FAST-NEXT:    i32.xor $push0=, $5, $9
+; NO-SIMD128-FAST-NEXT:    i32.and $push1=, $pop0, $1
+; NO-SIMD128-FAST-NEXT:    i32.xor $push2=, $pop1, $9
+; NO-SIMD128-FAST-NEXT:    i32.store 0($0), $pop2
+; NO-SIMD128-FAST-NEXT:    i32.xor $push3=, $6, $10
+; NO-SIMD128-FAST-NEXT:    i32.and $push4=, $pop3, $2
+; NO-SIMD128-FAST-NEXT:    i32.xor $push5=, $pop4, $10
+; NO-SIMD128-FAST-NEXT:    i32.store 4($0), $pop5
+; NO-SIMD128-FAST-NEXT:    i32.xor $push6=, $7, $11
+; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $pop6, $3
+; NO-SIMD128-FAST-NEXT:    i32.xor $push8=, $pop7, $11
+; NO-SIMD128-FAST-NEXT:    i32.store 8($0), $pop8
+; NO-SIMD128-FAST-NEXT:    i32.xor $push9=, $8, $12
+; NO-SIMD128-FAST-NEXT:    i32.and $push10=, $pop9, $4
+; NO-SIMD128-FAST-NEXT:    i32.xor $push11=, $pop10, $12
+; NO-SIMD128-FAST-NEXT:    i32.store 12($0), $pop11
 ; NO-SIMD128-FAST-NEXT:    return
   %masked_v1 = and <4 x i32> %c, %v1
   %inv_mask = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %c
@@ -10974,35 +10862,27 @@ define <2 x i64> @bitselect_v2i64(<2 x i64> %c, <2 x i64> %v1, <2 x i64> %v2) {
 ; NO-SIMD128-LABEL: bitselect_v2i64:
 ; NO-SIMD128:         .functype bitselect_v2i64 (i32, i64, i64, i64, i64, i64, i64) -> ()
 ; NO-SIMD128-NEXT:  # %bb.0:
-; NO-SIMD128-NEXT:    i64.const $push1=, -1
-; NO-SIMD128-NEXT:    i64.xor $push2=, $2, $pop1
-; NO-SIMD128-NEXT:    i64.and $push3=, $6, $pop2
-; NO-SIMD128-NEXT:    i64.and $push0=, $4, $2
-; NO-SIMD128-NEXT:    i64.or $push4=, $pop3, $pop0
-; NO-SIMD128-NEXT:    i64.store 8($0), $pop4
-; NO-SIMD128-NEXT:    i64.const $push9=, -1
-; NO-SIMD128-NEXT:    i64.xor $push6=, $1, $pop9
-; NO-SIMD128-NEXT:    i64.and $push7=, $5, $pop6
-; NO-SIMD128-NEXT:    i64.and $push5=, $3, $1
-; NO-SIMD128-NEXT:    i64.or $push8=, $pop7, $pop5
-; NO-SIMD128-NEXT:    i64.store 0($0), $pop8
+; NO-SIMD128-NEXT:    i64.xor $push0=, $4, $6
+; NO-SIMD128-NEXT:    i64.and $push1=, $pop0, $2
+; NO-SIMD128-NEXT:    i64.xor $push2=, $pop1, $6
+; NO-SIMD128-NEXT:    i64.store 8($0), $pop2
+; NO-SIMD128-NEXT:    i64.xor $push3=, $3, $5
+; NO-SIMD128-NEXT:    i64.and $push4=, $pop3, $1
+; NO-SIMD128-NEXT:    i64.xor $push5=, $pop4, $5
+; NO-SIMD128-NEXT:    i64.store 0($0), $pop5
 ; NO-SIMD128-NEXT:    return
 ;
 ; NO-SIMD128-FAST-LABEL: bitselect_v2i64:
 ; NO-SIMD128-FAST:         .functype bitselect_v2i64 (i32, i64, i64, i64, i64, i64, i64) -> ()
 ; NO-SIMD128-FAST-NEXT:  # %bb.0:
-; NO-SIMD128-FAST-NEXT:    i64.const $push1=, -1
-; NO-SIMD128-FAST-NEXT:    i64.xor $push2=, $1, $pop1
-; NO-SIMD128-FAST-NEXT:    i64.and $push3=, $5, $pop2
-; NO-SIMD128-FAST-NEXT:    i64.and $push0=, $3, $1
-; NO-SIMD128-FAST-NEXT:    i64.or $push4=, $pop3, $pop0
-; NO-SIMD128-FAST-NEXT:    i64.store 0($0), $pop4
-; NO-SIMD128-FAST-NEXT:    i64.const $push9=, -1
-; NO-SIMD128-FAST-NEXT:    i64.xor $push6=, $2, $pop9
-; NO-SIMD128-FAST-NEXT:    i64.and $push7=, $6, $pop6
-; NO-SIMD128-FAST-NEXT:    i64.and $push5=, $4, $2
-; NO-SIMD128-FAST-NEXT:    i64.or $push8=, $pop7, $pop5
-; NO-SIMD128-FAST-NEXT:    i64.store 8($0), $pop8
+; NO-SIMD128-FAST-NEXT:    i64.xor $push0=, $3, $5
+; NO-SIMD128-FAST-NEXT:    i64.and $push1=, $pop0, $1
+; NO-SIMD128-FAST-NEXT:    i64.xor $push2=, $pop1, $5
+; NO-SIMD128-FAST-NEXT:    i64.store 0($0), $pop2
+; NO-SIMD128-FAST-NEXT:    i64.xor $push3=, $4, $6
+; NO-SIMD128-FAST-NEXT:    i64.and $push4=, $pop3, $2
+; NO-SIMD128-FAST-NEXT:    i64.xor $push5=, $pop4, $6
+; NO-SIMD128-FAST-NEXT:    i64.store 8($0), $pop5
 ; NO-SIMD128-FAST-NEXT:    return
   %masked_v1 = and <2 x i64> %v1, %c
   %inv_mask = xor <2 x i64> <i64 -1, i64 -1>, %c
diff --git a/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll b/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll
index 7a5baa09f95e..10985de88bf2 100644
--- a/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll
+++ b/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard-mingw.ll
@@ -35,7 +35,7 @@
 ; }
 ;-------------------------------------------------------------------------------
 
-; CHECK: .set @feat.00, 2048
+; CHECK: @feat.00 = 2048
 
 ; CHECK: .section .gfids$y
 ; CHECK: .symidx _ZNK7Derived4calcEv
diff --git a/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard.ll b/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard.ll
index 2ec2e573f716..a77d5490ef87 100644
--- a/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard.ll
+++ b/external/llvm-project/llvm/test/CodeGen/WinCFGuard/cfguard.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s
 ; Control Flow Guard is currently only available on Windows
 
-; CHECK: .set @feat.00, 2048
+; CHECK: @feat.00 = 2048
 
 ; CHECK: .section .gfids$y
 ; CHECK: .symidx "?address_taken@@YAXXZ"
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll b/external/llvm-project/llvm/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll
index d59953fb4e37..cc80f87fda31 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll
@@ -10,4 +10,4 @@ define weak i32 @pthread_once(ptr, ptr) {
 ; CHECK: pthread_once:
 
 ; CHECK: .weak   __gthrw_pthread_once
-; CHECK: .set __gthrw_pthread_once, pthread_once
+; CHECK: __gthrw_pthread_once = pthread_once
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll b/external/llvm-project/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
index 7050889d7102..527684f5a27d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
@@ -125,31 +125,31 @@ define internal fastcc i32 @foo(i64 %bar) nounwind ssp {
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:    .data_region jt32
-; CHECK-NEXT:  .set L0_0_set_3, LBB0_3-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_4, LBB0_4-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_5, LBB0_5-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_6, LBB0_6-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_7, LBB0_7-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_8, LBB0_8-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_9, LBB0_9-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_10, LBB0_10-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_11, LBB0_11-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_12, LBB0_12-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_13, LBB0_13-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_14, LBB0_14-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_15, LBB0_15-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_16, LBB0_16-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_17, LBB0_17-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_18, LBB0_18-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_19, LBB0_19-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_20, LBB0_20-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_21, LBB0_21-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_22, LBB0_22-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_23, LBB0_23-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_24, LBB0_24-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_25, LBB0_25-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_26, LBB0_26-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_27, LBB0_27-LJTI0_0
+; CHECK-NEXT:  L0_0_set_3 = LBB0_3-LJTI0_0
+; CHECK-NEXT:  L0_0_set_4 = LBB0_4-LJTI0_0
+; CHECK-NEXT:  L0_0_set_5 = LBB0_5-LJTI0_0
+; CHECK-NEXT:  L0_0_set_6 = LBB0_6-LJTI0_0
+; CHECK-NEXT:  L0_0_set_7 = LBB0_7-LJTI0_0
+; CHECK-NEXT:  L0_0_set_8 = LBB0_8-LJTI0_0
+; CHECK-NEXT:  L0_0_set_9 = LBB0_9-LJTI0_0
+; CHECK-NEXT:  L0_0_set_10 = LBB0_10-LJTI0_0
+; CHECK-NEXT:  L0_0_set_11 = LBB0_11-LJTI0_0
+; CHECK-NEXT:  L0_0_set_12 = LBB0_12-LJTI0_0
+; CHECK-NEXT:  L0_0_set_13 = LBB0_13-LJTI0_0
+; CHECK-NEXT:  L0_0_set_14 = LBB0_14-LJTI0_0
+; CHECK-NEXT:  L0_0_set_15 = LBB0_15-LJTI0_0
+; CHECK-NEXT:  L0_0_set_16 = LBB0_16-LJTI0_0
+; CHECK-NEXT:  L0_0_set_17 = LBB0_17-LJTI0_0
+; CHECK-NEXT:  L0_0_set_18 = LBB0_18-LJTI0_0
+; CHECK-NEXT:  L0_0_set_19 = LBB0_19-LJTI0_0
+; CHECK-NEXT:  L0_0_set_20 = LBB0_20-LJTI0_0
+; CHECK-NEXT:  L0_0_set_21 = LBB0_21-LJTI0_0
+; CHECK-NEXT:  L0_0_set_22 = LBB0_22-LJTI0_0
+; CHECK-NEXT:  L0_0_set_23 = LBB0_23-LJTI0_0
+; CHECK-NEXT:  L0_0_set_24 = LBB0_24-LJTI0_0
+; CHECK-NEXT:  L0_0_set_25 = LBB0_25-LJTI0_0
+; CHECK-NEXT:  L0_0_set_26 = LBB0_26-LJTI0_0
+; CHECK-NEXT:  L0_0_set_27 = LBB0_27-LJTI0_0
 ; CHECK-NEXT:  LJTI0_0:
 ; CHECK-NEXT:    .long L0_0_set_3
 ; CHECK-NEXT:    .long L0_0_set_3
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll b/external/llvm-project/llvm/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll
index cf20cfaced5d..17df3e10fd3d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll
@@ -64,15 +64,15 @@ attributes #1 = { nounwind readnone }
 ; CHECK-NEXT: [[CLOBBER:Ltmp[0-9]*]]
 
 ; CHECK: Ldebug_loc0:
-; CHECK-NEXT: .set [[SET1:.*]], Lfunc_begin0-Lfunc_begin0
+; CHECK-NEXT: [[SET1:.*]] = Lfunc_begin0-Lfunc_begin0
 ; CHECK-NEXT: .quad   [[SET1]]
-; CHECK-NEXT: .set [[SET2:.*]], [[LABEL]]-Lfunc_begin0
+; CHECK-NEXT: [[SET2:.*]] = [[LABEL]]-Lfunc_begin0
 ; CHECK-NEXT: .quad   [[SET2]]
 ; CHECK-NEXT: .short  1     ## Loc expr size
 ; CHECK-NEXT: .byte   85
-; CHECK-NEXT: .set [[SET3:.*]], [[LABEL]]-Lfunc_begin0
+; CHECK-NEXT: [[SET3:.*]] = [[LABEL]]-Lfunc_begin0
 ; CHECK-NEXT: .quad   [[SET3]]
-; CHECK-NEXT: .set [[SET4:.*]], [[CLOBBER]]-Lfunc_begin0
+; CHECK-NEXT: [[SET4:.*]] = [[CLOBBER]]-Lfunc_begin0
 ; CHECK-NEXT: .quad   [[SET4]]
 ; CHECK-NEXT: .short  1     ## Loc expr size
 ; CHECK-NEXT: .byte   83
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll b/external/llvm-project/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
index 254a53fcac4d..65273870c3df 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
@@ -11,7 +11,7 @@ define void @endless_loop() {
 ; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
 ; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX1-NEXT:    vmovaps %ymm0, (%eax)
 ; AVX1-NEXT:    vmovaps %ymm1, (%eax)
 ; AVX1-NEXT:    vzeroupper
@@ -21,7 +21,7 @@ define void @endless_loop() {
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vbroadcastss (%eax), %xmm0
 ; AVX2-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vblendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vbroadcastss %xmm0, %ymm0
 ; AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/GlobalISel/isel-fp64-to-sint-x86.mir b/external/llvm-project/llvm/test/CodeGen/X86/GlobalISel/isel-fp64-to-sint-x86.mir
new file mode 100644
index 000000000000..3e1af9d836ae
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/GlobalISel/isel-fp64-to-sint-x86.mir
@@ -0,0 +1,136 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# NOTE: This MIR test is required because the support for 64 bit memory ops is missing in X86 mode, Due to distinction between float/int types, support is expected in near future and there is this RFC in place https://discourse.llvm.org/t/rfc-globalisel-adding-fp-type-information-to-llt/83349. Once this support is introduced this test must be dropped and integrated into the LLVM IR tests.
+# RUN: llc -O2 -mtriple=i686-linux-gnu -mattr=+x87,-sse,-sse2 -run-pass=regbankselect,instruction-select -disable-gisel-legality-check -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes GISEL-X86
+
+---
+name:            test_double_to_int16
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 8, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 2, alignment: 2,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_double_to_int16
+    ; GISEL-X86: [[DEF:%[0-9]+]]:rfp64 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: FP64_TO_INT16_IN_MEM %stack.0, 1, $noreg, 0, $noreg, [[DEF]], implicit-def dead $eflags :: (store (s16) into %stack.0)
+    ; GISEL-X86-NEXT: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %stack.0, 1, $noreg, 0, $noreg :: (load (s16) from %stack.0)
+    ; GISEL-X86-NEXT: $ax = COPY [[MOV16rm]]
+    ; GISEL-X86-NEXT: RET 0, implicit $ax
+    %0:_(s64) = IMPLICIT_DEF
+    %3:_(p0) = G_FRAME_INDEX %stack.0
+    G_FIST %0(s64), %3(p0) :: (store (s16) into %stack.0)
+    %2:_(s16) = G_LOAD %3(p0) :: (load (s16) from %stack.0)
+    $ax = COPY %2(s16)
+    RET 0, implicit $ax
+...
+---
+name:            test_double_to_int32
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 8, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_double_to_int32
+    ; GISEL-X86: [[DEF:%[0-9]+]]:rfp64 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: FP64_TO_INT32_IN_MEM %stack.0, 1, $noreg, 0, $noreg, [[DEF]], implicit-def dead $eflags :: (store (s32) into %stack.0)
+    ; GISEL-X86-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %stack.0)
+    ; GISEL-X86-NEXT: $eax = COPY [[MOV32rm]]
+    ; GISEL-X86-NEXT: RET 0, implicit $eax
+    %0:_(s64) = IMPLICIT_DEF
+    %3:_(p0) = G_FRAME_INDEX %stack.0
+    G_FIST %0(s64), %3(p0) :: (store (s32) into %stack.0)
+    %2:_(s32) = G_LOAD %3(p0) :: (load (s32) from %stack.0)
+    $eax = COPY %2(s32)
+    RET 0, implicit $eax
+...
+---
+name:            test_double_to_int64
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 8, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_double_to_int64
+    ; GISEL-X86: [[DEF:%[0-9]+]]:rfp64 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r %stack.0, 1, $noreg, 0, $noreg
+    ; GISEL-X86-NEXT: FP64_TO_INT64_IN_MEM %stack.0, 1, $noreg, 0, $noreg, [[DEF]], implicit-def dead $eflags :: (store (s64) into %stack.0)
+    ; GISEL-X86-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %stack.0, align 8)
+    ; GISEL-X86-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[LEA32r]], 1, $noreg, 4, $noreg :: (load (s32) from %stack.0 + 4, basealign 8)
+    ; GISEL-X86-NEXT: $eax = COPY [[MOV32rm]]
+    ; GISEL-X86-NEXT: $edx = COPY [[MOV32rm1]]
+    ; GISEL-X86-NEXT: RET 0, implicit $eax, implicit $edx
+    %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %10:_(s32) = G_LOAD %1(p0) :: (invariant load (s32) from %fixed-stack.0, align 16)
+    %8:_(s32) = G_CONSTANT i32 4
+    %11:_(p0) = G_PTR_ADD %1, %8(s32)
+    %12:_(s32) = G_LOAD %11(p0) :: (invariant load (s32) from %fixed-stack.0 + 4, basealign 16)
+    %0:_(s64) = IMPLICIT_DEF
+    %5:_(p0) = G_FRAME_INDEX %stack.0
+    G_FIST %0(s64), %5(p0) :: (store (s64) into %stack.0)
+    %6:_(s32) = G_LOAD %5(p0) :: (load (s32) from %stack.0, align 8)
+    %7:_(p0) = G_PTR_ADD %5, %8(s32)
+    %9:_(s32) = G_LOAD %7(p0) :: (load (s32) from %stack.0 + 4, basealign 8)
+    $eax = COPY %6(s32)
+    $edx = COPY %9(s32)
+    RET 0, implicit $eax, implicit $edx
+...
+---
+name:            test_double_to_int8
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 8, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 2, alignment: 2,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_double_to_int8
+    ; GISEL-X86: [[DEF:%[0-9]+]]:rfp64 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: FP64_TO_INT16_IN_MEM %stack.0, 1, $noreg, 0, $noreg, [[DEF]], implicit-def dead $eflags :: (store (s16) into %stack.0)
+    ; GISEL-X86-NEXT: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %stack.0, 1, $noreg, 0, $noreg :: (load (s16) from %stack.0)
+    ; GISEL-X86-NEXT: [[COPY:%[0-9]+]]:gr16_abcd = COPY [[MOV16rm]]
+    ; GISEL-X86-NEXT: [[COPY1:%[0-9]+]]:gr8_abcd_l = COPY [[COPY]].sub_8bit
+    ; GISEL-X86-NEXT: $al = COPY [[COPY1]]
+    ; GISEL-X86-NEXT: RET 0, implicit $al
+    %0:_(s64) = IMPLICIT_DEF
+    %4:_(p0) = G_FRAME_INDEX %stack.0
+    G_FIST %0(s64), %4(p0) :: (store (s16) into %stack.0)
+    %3:_(s16) = G_LOAD %4(p0) :: (load (s16) from %stack.0)
+    %2:_(s8) = G_TRUNC %3(s16)
+    $al = COPY %2(s8)
+    RET 0, implicit $al
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/GlobalISel/isel-sint-to-fp64-x86.mir b/external/llvm-project/llvm/test/CodeGen/X86/GlobalISel/isel-sint-to-fp64-x86.mir
new file mode 100644
index 000000000000..b6b2bd610fa3
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/GlobalISel/isel-sint-to-fp64-x86.mir
@@ -0,0 +1,152 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# NOTE: This MIR test is required because the support for 64 bit memory ops is missing in X86 mode, Due to distinction between float/int types, support is expected in near future and there is this RFC in place https://discourse.llvm.org/t/rfc-globalisel-adding-fp-type-information-to-llt/83349. Once this support is introduced this test must be dropped and integrated into the LLVM IR tests.
+# RUN: llc -O2 -mtriple=i686-linux-gnu -mattr=+x87,-sse,-sse2 -run-pass=regbankselect,instruction-select -disable-gisel-legality-check -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes GISEL-X86
+
+---
+name:            test_si8tofp64
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, type: default, offset: 0, size: 1, alignment: 4, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 2, alignment: 2,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_si8tofp64
+    ; GISEL-X86: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (p0) from %fixed-stack.1)
+    ; GISEL-X86-NEXT: [[DEF:%[0-9]+]]:gr16 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: MOV16mr %stack.0, 1, $noreg, 0, $noreg, [[DEF]] :: (store (s16) into %stack.0)
+    ; GISEL-X86-NEXT: [[ILD_Fp16m64_:%[0-9]+]]:rfp64 = ILD_Fp16m64 %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s16) from %stack.0)
+    ; GISEL-X86-NEXT: nofpexcept ST_Fp64m [[MOV32rm]], 1, $noreg, 0, $noreg, [[ILD_Fp16m64_]], implicit-def dead $fpsw, implicit $fpcw :: (store (s64))
+    ; GISEL-X86-NEXT: RET 0
+    %4:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %1:_(p0) = G_LOAD %4(p0) :: (invariant load (p0) from %fixed-stack.0)
+    %6:_(s16) = IMPLICIT_DEF
+    %7:_(p0) = G_FRAME_INDEX %stack.0
+    G_STORE %6(s16), %7(p0) :: (store (s16) into %stack.0)
+    %5:_(s64) = G_FILD %7(p0) :: (load (s16) from %stack.0)
+    G_STORE %5(s64), %1(p0) :: (store (s64))
+    RET 0
+...
+---
+name:            test_si16tofp64
+alignment:       16
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, type: default, offset: 0, size: 2, alignment: 4, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 2, alignment: 2,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_si16tofp64
+    ; GISEL-X86: [[DEF:%[0-9]+]]:gr16 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (p0) from %fixed-stack.1)
+    ; GISEL-X86-NEXT: MOV16mr %stack.0, 1, $noreg, 0, $noreg, [[DEF]] :: (store (s16) into %stack.0)
+    ; GISEL-X86-NEXT: [[ILD_Fp16m64_:%[0-9]+]]:rfp64 = ILD_Fp16m64 %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s16) from %stack.0)
+    ; GISEL-X86-NEXT: nofpexcept ST_Fp64m [[MOV32rm]], 1, $noreg, 0, $noreg, [[ILD_Fp16m64_]], implicit-def dead $fpsw, implicit $fpcw :: (store (s64))
+    ; GISEL-X86-NEXT: RET 0
+    %0:_(s16) = IMPLICIT_DEF
+    %4:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %1:_(p0) = G_LOAD %4(p0) :: (invariant load (p0) from %fixed-stack.0)
+    %6:_(p0) = G_FRAME_INDEX %stack.0
+    G_STORE %0(s16), %6(p0) :: (store (s16) into %stack.0)
+    %5:_(s64) = G_FILD %6(p0):: (load (s16) from %stack.0)
+    G_STORE %5(s64), %1(p0) :: (store (s64))
+    RET 0
+...
+---
+name:            test_si32tofp64
+alignment:       4
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, type: default, offset: 0, size: 4, alignment: 4, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_si32tofp64
+    ; GISEL-X86: [[DEF:%[0-9]+]]:gr32 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (p0) from %fixed-stack.1)
+    ; GISEL-X86-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, [[DEF]] :: (store (s32) into %stack.0)
+    ; GISEL-X86-NEXT: [[ILD_Fp32m64_:%[0-9]+]]:rfp64 = ILD_Fp32m64 %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s32) from %stack.0)
+    ; GISEL-X86-NEXT: nofpexcept ST_Fp64m [[MOV32rm]], 1, $noreg, 0, $noreg, [[ILD_Fp32m64_]], implicit-def dead $fpsw, implicit $fpcw :: (store (s64))
+    ; GISEL-X86-NEXT: RET 0
+    %0:_(s32) = IMPLICIT_DEF
+    %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %1:_(p0) = G_LOAD %3(p0) :: (invariant load (p0) from %fixed-stack.0)
+    %5:_(p0) = G_FRAME_INDEX %stack.0
+    G_STORE %0(s32), %5(p0) :: (store (s32) into %stack.0)
+    %4:_(s64) = G_FILD %5(p0) :: (load (s32) from %stack.0)
+    G_STORE %4(s64), %1(p0) :: (store (s64))
+    RET 0
+...
+---
+name:            test_si64tofp64
+alignment:       16
+exposesReturnsTwice: false
+legalized:       true
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 8, size: 4, alignment: 8, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, type: default, offset: 0, size: 4, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; GISEL-X86-LABEL: name: test_si64tofp64
+    ; GISEL-X86: [[DEF:%[0-9]+]]:gr32 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: [[DEF1:%[0-9]+]]:gr32 = IMPLICIT_DEF
+    ; GISEL-X86-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.2, 1, $noreg, 0, $noreg :: (invariant load (p0) from %fixed-stack.2, align 8)
+    ; GISEL-X86-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r %stack.0, 1, $noreg, 0, $noreg
+    ; GISEL-X86-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, [[DEF]] :: (store (s32) into %stack.0, align 8)
+    ; GISEL-X86-NEXT: MOV32mr [[LEA32r]], 1, $noreg, 4, $noreg, [[DEF1]] :: (store (s32) into %stack.0 + 4, basealign 8)
+    ; GISEL-X86-NEXT: [[ILD_Fp64m64_:%[0-9]+]]:rfp64 = ILD_Fp64m64 %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s64) from %stack.0)
+    ; GISEL-X86-NEXT: nofpexcept ST_Fp64m [[MOV32rm]], 1, $noreg, 0, $noreg, [[ILD_Fp64m64_]], implicit-def dead $fpsw, implicit $fpcw :: (store (s64))
+    ; GISEL-X86-NEXT: RET 0
+    %2:_(s32) = IMPLICIT_DEF
+    %3:_(s32) = IMPLICIT_DEF
+    %6:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %1:_(p0) = G_LOAD %6(p0) :: (invariant load (p0) from %fixed-stack.0, align 8)
+    %8:_(p0) = G_FRAME_INDEX %stack.0
+    G_STORE %2(s32), %8(p0) :: (store (s32) into %stack.0, align 8)
+    %12:_(s32) = G_CONSTANT i32 4
+    %11:_(p0) = G_PTR_ADD %8, %12(s32)
+    G_STORE %3(s32), %11(p0) :: (store (s32) into %stack.0 + 4, basealign 8)
+    %7:_(s64) = G_FILD %8(p0) :: (load (s64) from %stack.0)
+    G_STORE %7(s64), %1(p0) :: (store (s64))
+    RET 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/addr-mode-matcher-3.ll b/external/llvm-project/llvm/test/CodeGen/X86/addr-mode-matcher-3.ll
index 0c7275ec2867..beea6d36fe87 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/addr-mode-matcher-3.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/addr-mode-matcher-3.ll
@@ -98,3 +98,46 @@ define i64 @add_shl_zext(ptr %ptr, i8 %arg) nounwind {
   %sum = add i64 %val, %shl
   ret i64 %sum
 }
+
+define i32 @PR55714_i32(i32 %n, i32 %q) {
+; X86-LABEL: PR55714_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal (,%ecx,8), %eax
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: PR55714_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rsi,8), %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %mul = mul i32 %q, 7
+  %add = add i32 %mul, %n
+  ret i32 %add
+}
+
+define i64 @PR55714_i64(i64 %n, i64 %q) {
+; X86-LABEL: PR55714_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $7, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    leal (%edx,%ecx,8), %edx
+; X86-NEXT:    subl %ecx, %edx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    retl
+;
+; X64-LABEL: PR55714_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    leaq (%rdi,%rsi,8), %rax
+; X64-NEXT:    subq %rsi, %rax
+; X64-NEXT:    retq
+  %mul = mul i64 %q, 7
+  %add = add i64 %mul, %n
+  ret i64 %add
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/alias-gep.ll b/external/llvm-project/llvm/test/CodeGen/X86/alias-gep.ll
index 904a611f61d1..65d2ced6df5b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/alias-gep.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/alias-gep.ll
@@ -3,17 +3,17 @@
 
 ;MACHO: .globl _offsetSym0
 ;MACHO-NOT: .alt_entry
-;MACHO: .set _offsetSym0, _s
+;MACHO: _offsetSym0 = _s
 ;MACHO: .globl _offsetSym1
 ;MACHO: .alt_entry _offsetSym1
-;MACHO: .set _offsetSym1, _s+8
+;MACHO: _offsetSym1 = _s+8
 
 ;ELF: .globl offsetSym0
 ;ELF-NOT: .alt_entry
-;ELF: .set offsetSym0, s
+;ELF: offsetSym0 = s
 ;ELF: .globl offsetSym1
 ;ELF-NOT: .alt_entry
-;ELF: .set offsetSym1, s+8
+;ELF: offsetSym1 = s+8
 
 %struct.S1 = type { i32, i32, i32 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/aliases.ll b/external/llvm-project/llvm/test/CodeGen/X86/aliases.ll
index 03ea2579d0f8..d36798820fe8 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/aliases.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/aliases.ll
@@ -48,16 +48,16 @@ define i32 @foo_f() {
 ; CHECK-DAG: .protected	bar_p
 @bar_p = protected alias i32, ptr @bar
 
-; CHECK-DAG: .set test2, bar+4
+; CHECK-DAG: test2 = bar+4
 @test2 = alias i32, getelementptr(i32, ptr @bar, i32 1)
 
-; CHECK-DAG: .set test3, 42
+; CHECK-DAG: test3 = 42
 @test3 = alias i32, inttoptr(i32 42 to ptr)
 
-; CHECK-DAG: .set test4, bar
+; CHECK-DAG: test4 = bar
 @test4 = alias i32, inttoptr(i64 ptrtoint (ptr @bar to i64) to ptr)
 
-; CHECK-DAG: .set test5, test2-bar
+; CHECK-DAG: test5 = test2-bar
 @test5 = alias i32, inttoptr(i32 sub (i32 ptrtoint (ptr @test2 to i32),
                                  i32 ptrtoint (ptr @bar to i32)) to ptr)
 
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index 1fada58f05ba..dec829fed353 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -1235,7 +1235,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.v
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1248,7 +1248,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.v
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1359,7 +1359,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.v
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1372,7 +1372,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.v
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1952,7 +1952,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX2-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX2-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -1965,7 +1965,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512F-SLOW-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512F-SLOW-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
 ; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -1991,7 +1991,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512DQ-SLOW-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
 ; AVX512DQ-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -2016,7 +2016,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-SLOW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-SLOW-NEXT:    vpbroadcastq %xmm0, %ymm0
+; AVX512BW-SLOW-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -2702,7 +2702,7 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm0 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm0 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -2717,7 +2717,7 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm0 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm0 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -2964,7 +2964,7 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm0 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm0 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -2979,7 +2979,7 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm0, %ymm0
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm0 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm0 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -4209,14 +4209,13 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX2-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
-; AVX2-NEXT:    vpbroadcastd %xmm0, %xmm2
 ; AVX2-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm0
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4,5,6,7]
-; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
-; AVX2-NEXT:    vpaddb 32(%rdx), %ymm2, %ymm1
-; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX2-NEXT:    vmovdqa %ymm1, 32(%rcx)
+; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4,5,6,7]
+; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
+; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)
+; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll b/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
index f2e4da0ac540..3d4cddbb94c7 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -1020,7 +1020,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.e
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1030,7 +1030,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.e
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1116,7 +1116,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.e
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1126,7 +1126,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.e
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1569,7 +1569,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ;
 ; AVX512F-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastq (%rdi), %ymm0
+; AVX512F-NEXT:    vpbroadcastd (%rdi), %ymm0
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rdx)
@@ -1578,7 +1578,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ;
 ; AVX512DQ-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastq (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpbroadcastd (%rdi), %ymm0
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rdx)
@@ -1587,7 +1587,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ;
 ; AVX512BW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastq (%rdi), %ymm0
+; AVX512BW-NEXT:    vpbroadcastd (%rdi), %ymm0
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rdx)
@@ -2125,7 +2125,7 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm0
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (mem & (ymm0 ^ ymm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rdx)
@@ -2137,7 +2137,7 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (mem & (ymm0 ^ ymm1))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, 32(%rdx)
@@ -2346,7 +2346,7 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.e
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm0
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (mem & (ymm0 ^ ymm1))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (m64bcst & (ymm0 ^ ymm1))
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rdx)
@@ -2358,7 +2358,7 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.e
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (mem & (ymm0 ^ ymm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (m64bcst & (ymm0 ^ ymm1))
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, 32(%rdx)
@@ -3367,7 +3367,7 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,0],ymm0[1,3],ymm1[4,4],ymm0[5,7]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = mem[0,1,0,1]
+; AVX-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX-NEXT:    vpaddb 16(%rsi), %xmm2, %xmm2
 ; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
@@ -3467,9 +3467,9 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],mem[2,3,4,5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vbroadcastss (%rdi), %xmm2
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3,4,5],xmm1[6,7]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vpaddb 32(%rsi), %xmm0, %xmm0
@@ -3664,13 +3664,13 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i
 ;
 ; AVX-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa 48(%rdi), %xmm0
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = mem[0,1],xmm0[2,3,4,5,6,7]
-; AVX-NEXT:    vbroadcastss (%rdi), %xmm1
-; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
-; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, 16(%rdx)
-; AVX-NEXT:    vmovdqa %xmm0, (%rdx)
+; AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],mem[2,3,4,5,6,7]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
+; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
+; AVX-NEXT:    vmovdqa %xmm0, 16(%rdx)
 ; AVX-NEXT:    retq
 ;
 ; AVX2-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/apx/push2-pop2-disabled-with-small-stack-alignment.ll b/external/llvm-project/llvm/test/CodeGen/X86/apx/push2-pop2-disabled-with-small-stack-alignment.ll
new file mode 100644
index 000000000000..f07e40b442c0
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/apx/push2-pop2-disabled-with-small-stack-alignment.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+push2pop2 | FileCheck %s --check-prefix=CHECK
+
+; This test is used to check no push2/pop2 emitted if stack alignment is set to
+; the value less than 16 bytes required by push2/pop2 instruction. Here it's set
+; to 8 bytes.
+
+define void @csr1() nounwind {
+; CHECK-LABEL: csr1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  tail call void asm sideeffect "", "~{rbp},~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
+
+define void @csr2() nounwind {
+; CHECK-LABEL: csr2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  tail call void asm sideeffect "", "~{rbp},~{r15},~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
+
+define void @csr3() nounwind {
+; CHECK-LABEL: csr3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  tail call void asm sideeffect "", "~{rbp},~{r15},~{r14},~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
+
+define void @csr4() nounwind {
+; CHECK-LABEL: csr4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  tail call void asm sideeffect "", "~{rbp},~{r15},~{r14},~{r13},~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
+
+define void @csr5() nounwind {
+; CHECK-LABEL: csr5:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  tail call void asm sideeffect "", "~{rbp},~{r15},~{r14},~{r13},~{r12},~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
+
+define void @csr6() nounwind {
+; CHECK-LABEL: csr6:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  tail call void asm sideeffect "", "~{rbp},~{r15},~{r14},~{r13},~{r12},~{rbx},~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"override-stack-alignment", i32 8}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/apx/reloc-opt.ll b/external/llvm-project/llvm/test/CodeGen/X86/apx/reloc-opt.ll
index a5ab94b00d64..ecc3d3297cea 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/apx/reloc-opt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/apx/reloc-opt.ll
@@ -13,11 +13,9 @@
 
 
 ; CHECK-LABEL: test_regclass_not_updated_by_regalloc_1
-; APXREL: movq    (%rip), %r16
-; APXREL-NEXT: R_X86_64_CODE_4_GOTPCRELX gvar-0x4
-; NOAPXREL-NOT: R_X86_64_CODE_4_GOTPCRELX gvar-0x4
-; NOAPXREL: movq    (%rip), %rdi
-; NOAPXREL-NEXT: R_X86_64_REX_GOTPCRELX gvar-0x4
+; CHECK-NOT: R_X86_64_CODE_4_GOTPCRELX gvar-0x4
+; CHECK: movq    (%rip), %rdi
+; CHECK-NEXT: R_X86_64_REX_GOTPCRELX gvar-0x4
 
 @gvar = external global [20000 x i8]
 
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/atomic-idempotent.ll b/external/llvm-project/llvm/test/CodeGen/X86/atomic-idempotent.ll
index 020f9eb79310..01c3e7999a92 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/atomic-idempotent.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/atomic-idempotent.ll
@@ -622,4 +622,333 @@ define void @or8_nouse_seq_cst(ptr %p) #0 {
   ret void
 }
 
+define void @atomic_umin_uint_max(ptr %addr) #0 {
+; X64-LABEL: atomic_umin_uint_max:
+; X64:       # %bb.0:
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_umin_uint_max:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_umin_uint_max:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movl (%eax), %eax
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_umin_uint_max:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw umin ptr %addr, i32 -1 seq_cst
+  ret void
+}
+
+define void @atomic_umax_zero(ptr %addr) #0 {
+; X64-LABEL: atomic_umax_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_umax_zero:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_umax_zero:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movl (%eax), %eax
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_umax_zero:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw umax ptr %addr, i32 0 seq_cst
+  ret void
+}
+
+define void @atomic_min_smax_char(ptr %addr) #0 {
+; X64-LABEL: atomic_min_smax_char:
+; X64:       # %bb.0:
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_min_smax_char:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movzbl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_min_smax_char:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movzbl (%eax), %eax
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_min_smax_char:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movzbl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw min ptr %addr, i8 127 seq_cst
+  ret void
+}
+
+define void @atomic_max_smin_char(ptr %addr) #0 {
+; X64-LABEL: atomic_max_smin_char:
+; X64:       # %bb.0:
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_max_smin_char:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movzbl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_max_smin_char:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movzbl (%eax), %eax
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_max_smin_char:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movzbl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw max ptr %addr, i8 -128 seq_cst
+  ret void
+}
+
+define void @atomic_min_umax_char(ptr %addr) #0 {
+; X64-LABEL: atomic_min_umax_char:
+; X64:       # %bb.0:
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_min_umax_char:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movzbl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_min_umax_char:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movzbl (%eax), %eax
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_min_umax_char:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movzbl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw umin ptr %addr, i8 255 seq_cst
+  ret void
+}
+
+define void @atomic_max_umin_char(ptr %addr) #0 {
+; X64-LABEL: atomic_max_umin_char:
+; X64:       # %bb.0:
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzbl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_max_umin_char:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movzbl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_max_umin_char:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movzbl (%eax), %eax
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_max_umin_char:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movzbl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw umax ptr %addr, i8 0 seq_cst
+  ret void
+}
+
+; TODO: Add floating point support.
+define void @atomic_fadd_zero(ptr %addr) #0 {
+; X64-LABEL: atomic_fadd_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    .p2align 4
+; X64-NEXT:  .LBB21_1: # %atomicrmw.start
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    lock cmpxchgl %eax, (%rdi)
+; X64-NEXT:    jne .LBB21_1
+; X64-NEXT:  # %bb.2: # %atomicrmw.end
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_fadd_zero:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    .p2align 4
+; X86-SSE2-NEXT:  .LBB21_1: # %atomicrmw.start
+; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    jne .LBB21_1
+; X86-SSE2-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_fadd_zero:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    subl $8, %esp
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SLM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SLM-NEXT:    .p2align 4
+; X86-SLM-NEXT:  .LBB21_1: # %atomicrmw.start
+; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-SLM-NEXT:    movss %xmm0, (%esp)
+; X86-SLM-NEXT:    movl (%esp), %eax
+; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-SLM-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-SLM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SLM-NEXT:    jne .LBB21_1
+; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    addl $8, %esp
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_fadd_zero:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-ATOM-NEXT:    .p2align 4
+; X86-ATOM-NEXT:  .LBB21_1: # %atomicrmw.start
+; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-ATOM-NEXT:    movss %xmm0, (%esp)
+; X86-ATOM-NEXT:    movl (%esp), %eax
+; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-ATOM-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-ATOM-NEXT:    jne .LBB21_1
+; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    retl
+  atomicrmw fadd ptr %addr, float -0.0 monotonic
+  ret void
+}
+
+define void @atomic_fsub_zero(ptr %addr) #0 {
+; X64-LABEL: atomic_fsub_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    .p2align 4
+; X64-NEXT:  .LBB22_1: # %atomicrmw.start
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    lock cmpxchgl %eax, (%rdi)
+; X64-NEXT:    jne .LBB22_1
+; X64-NEXT:  # %bb.2: # %atomicrmw.end
+; X64-NEXT:    retq
+;
+; X86-SSE2-LABEL: atomic_fsub_zero:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    .p2align 4
+; X86-SSE2-NEXT:  .LBB22_1: # %atomicrmw.start
+; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    jne .LBB22_1
+; X86-SSE2-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: atomic_fsub_zero:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    subl $8, %esp
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SLM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SLM-NEXT:    .p2align 4
+; X86-SLM-NEXT:  .LBB22_1: # %atomicrmw.start
+; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-SLM-NEXT:    movss %xmm0, (%esp)
+; X86-SLM-NEXT:    movl (%esp), %eax
+; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-SLM-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-SLM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SLM-NEXT:    jne .LBB22_1
+; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    addl $8, %esp
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: atomic_fsub_zero:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-ATOM-NEXT:    .p2align 4
+; X86-ATOM-NEXT:  .LBB22_1: # %atomicrmw.start
+; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-ATOM-NEXT:    movss %xmm0, (%esp)
+; X86-ATOM-NEXT:    movl (%esp), %eax
+; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-ATOM-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-ATOM-NEXT:    jne .LBB22_1
+; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    retl
+  atomicrmw fsub ptr %addr, float 0.0 release
+  ret void
+}
+
 attributes #0 = { nounwind }
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avgfloors.ll b/external/llvm-project/llvm/test/CodeGen/X86/avgfloors.ll
index 0508e5ccb543..9cc55c6f7a81 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avgfloors.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avgfloors.ll
@@ -53,7 +53,7 @@ define <16 x i8> @test_fixed_v16i8(<16 x i8> %a0, <16 x i8> %a1) nounwind {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
@@ -108,7 +108,7 @@ define <16 x i8> @test_ext_v16i8(<16 x i8> %a0, <16 x i8> %a1) nounwind {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
@@ -405,7 +405,7 @@ define <32 x i8> @test_fixed_v32i8(<32 x i8> %a0, <32 x i8> %a1) nounwind {
 ; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vpsrlw $1, %ymm0, %ymm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
 ; AVX512-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
@@ -478,7 +478,7 @@ define <32 x i8> @test_ext_v32i8(<32 x i8> %a0, <32 x i8> %a1) nounwind {
 ; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vpsrlw $1, %ymm0, %ymm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
 ; AVX512-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
@@ -966,7 +966,7 @@ define <64 x i8> @test_fixed_v64i8(<64 x i8> %a0, <64 x i8> %a1) nounwind {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vpsrlw $1, %zmm0, %zmm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m32bcst)
 ; AVX512-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
 ; AVX512-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
@@ -1078,7 +1078,7 @@ define <64 x i8> @test_ext_v64i8(<64 x i8> %a0, <64 x i8> %a1) nounwind {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vpsrlw $1, %zmm0, %zmm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m32bcst)
 ; AVX512-NEXT:    vpaddb %zmm2, %zmm0, %zmm0
 ; AVX512-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx-cvt.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx-cvt.ll
index 1bd25273ecd4..fb30044512fa 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx-cvt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx-cvt.ll
@@ -108,7 +108,7 @@ define <2 x double> @fpext01(<2 x double> %a0, <4 x float> %a1) nounwind {
 define double @funcA(ptr nocapture %e) nounwind uwtable readonly ssp {
 ; CHECK-LABEL: funcA:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %tmp1 = load i64, ptr %e, align 8
   %conv = sitofp i64 %tmp1 to double
@@ -118,7 +118,7 @@ define double @funcA(ptr nocapture %e) nounwind uwtable readonly ssp {
 define double @funcB(ptr nocapture %e) nounwind uwtable readonly ssp {
 ; CHECK-LABEL: funcB:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vcvtsi2sdl (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %tmp1 = load i32, ptr %e, align 4
   %conv = sitofp i32 %tmp1 to double
@@ -128,7 +128,7 @@ define double @funcB(ptr nocapture %e) nounwind uwtable readonly ssp {
 define float @funcC(ptr nocapture %e) nounwind uwtable readonly ssp {
 ; CHECK-LABEL: funcC:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vcvtsi2ssl (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %tmp1 = load i32, ptr %e, align 4
   %conv = sitofp i32 %tmp1 to float
@@ -138,7 +138,7 @@ define float @funcC(ptr nocapture %e) nounwind uwtable readonly ssp {
 define float @funcD(ptr nocapture %e) nounwind uwtable readonly ssp {
 ; CHECK-LABEL: funcD:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %tmp1 = load i64, ptr %e, align 8
   %conv = sitofp i64 %tmp1 to float
@@ -183,7 +183,7 @@ declare float @llvm.floor.f32(float %p)
 define float @floor_f32_load(ptr %aptr) optsize {
 ; CHECK-LABEL: floor_f32_load:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vroundss $9, (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vroundss $9, (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %a = load float, ptr %aptr
   %res = call float @llvm.floor.f32(float %a)
@@ -193,7 +193,7 @@ define float @floor_f32_load(ptr %aptr) optsize {
 define float @floor_f32_load_pgso(ptr %aptr) !prof !14 {
 ; CHECK-LABEL: floor_f32_load_pgso:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vroundss $9, (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vroundss $9, (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %a = load float, ptr %aptr
   %res = call float @llvm.floor.f32(float %a)
@@ -203,7 +203,7 @@ define float @floor_f32_load_pgso(ptr %aptr) !prof !14 {
 define double @nearbyint_f64_load(ptr %aptr) optsize {
 ; CHECK-LABEL: nearbyint_f64_load:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vroundsd $12, (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vroundsd $12, (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %a = load double, ptr %aptr
   %res = call double @llvm.nearbyint.f64(double %a)
@@ -213,7 +213,7 @@ define double @nearbyint_f64_load(ptr %aptr) optsize {
 define double @nearbyint_f64_load_pgso(ptr %aptr) !prof !14 {
 ; CHECK-LABEL: nearbyint_f64_load_pgso:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vroundsd $12, (%rdi), %xmm0, %xmm0
+; CHECK-NEXT:    vroundsd $12, (%rdi), %xmm15, %xmm0
 ; CHECK-NEXT:    retq
   %a = load double, ptr %aptr
   %res = call double @llvm.nearbyint.f64(double %a)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx-insertelt.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx-insertelt.ll
index 18ca01290c91..f8feceb0404b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx-insertelt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx-insertelt.ll
@@ -94,7 +94,7 @@ define <8 x float> @insert_f32_firstelt_of_high_subvector(<8 x float> %x, float
 ; AVX-LABEL: insert_f32_firstelt_of_high_subvector:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
@@ -111,7 +111,7 @@ define <4 x double> @insert_f64_firstelt_of_high_subvector(<4 x double> %x, doub
 ; AVX-LABEL: insert_f64_firstelt_of_high_subvector:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm2[1]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
@@ -202,9 +202,9 @@ define <4 x i64> @insert_i64_firstelt_of_high_subvector(<4 x i64> %x, i64 %s) {
 define <8 x float> @insert_f32_firstelts(<8 x float> %x, float %s) {
 ; AVX-LABEL: insert_f32_firstelts:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm2 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX-NEXT:    retq
 ;
@@ -221,10 +221,9 @@ define <8 x float> @insert_f32_firstelts(<8 x float> %x, float %s) {
 define <4 x double> @insert_f64_firstelts(<4 x double> %x, double %s) {
 ; AVX-LABEL: insert_f64_firstelts:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0,1],xmm0[2,3]
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
 ; AVX-NEXT:    retq
 ;
 ; AVX2-LABEL: insert_f64_firstelts:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
index 966662f5f9f8..f0203b3b889e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
@@ -300,8 +300,8 @@ declare <2 x i64> @llvm.x86.sse2.psrl.dq(<2 x i64>, i32) nounwind readnone
 define <2 x double> @test_x86_sse41_blendpd(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_sse41_blendpd:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vblendps $3, %xmm0, %xmm1, %xmm0 # encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x03]
-; CHECK-NEXT:    # xmm0 = xmm0[0,1],xmm1[2,3]
+; CHECK-NEXT:    vmovsd %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf3,0x10,0xc0]
+; CHECK-NEXT:    # xmm0 = xmm0[0],xmm1[1]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <2 x double> @llvm.x86.sse41.blendpd(<2 x double> %a0, <2 x double> %a1, i8 2) ; <<2 x double>> [#uses=1]
   ret <2 x double> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx10_2_512bf16-arith.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx10_2_512bf16-arith.ll
index 1e2cf4956bd0..c22a394e6c4e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx10_2_512bf16-arith.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx10_2_512bf16-arith.ll
@@ -94,8 +94,8 @@ define <32 x bfloat> @test_int_x86_avx10_maskz_sub_bf16_512(<32 x bfloat> %src,
 ;
 ; X86-LABEL: test_int_x86_avx10_maskz_sub_bf16_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
 ; X86-NEXT:    vsubbf16 %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf5,0x75,0xc9,0x5c,0xc2]
 ; X86-NEXT:    vsubbf16 (%eax), %zmm1, %zmm1 # encoding: [0x62,0xf5,0x75,0x48,0x5c,0x08]
 ; X86-NEXT:    vsubbf16 %zmm1, %zmm0, %zmm0 {%k1} # encoding: [0x62,0xf5,0x7d,0x49,0x5c,0xc1]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll
index 42831a453cb1..435f67a0f1e4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx10_2bf16-arith.ll
@@ -147,8 +147,8 @@ define <16 x bfloat> @test_int_x86_avx10_maskz_sub_bf16_256(<16 x bfloat> %src,
 ;
 ; X86-LABEL: test_int_x86_avx10_maskz_sub_bf16_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
 ; X86-NEXT:    vsubbf16 %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf5,0x75,0xa9,0x5c,0xc2]
 ; X86-NEXT:    vsubbf16 (%eax), %ymm1, %ymm1 # encoding: [0x62,0xf5,0x75,0x28,0x5c,0x08]
 ; X86-NEXT:    vsubbf16 %ymm1, %ymm0, %ymm0 {%k1} # encoding: [0x62,0xf5,0x7d,0x29,0x5c,0xc1]
@@ -201,8 +201,8 @@ define <8 x bfloat> @test_int_x86_avx10_maskz_sub_bf16_128(<8 x bfloat> %src, <8
 ;
 ; X86-LABEL: test_int_x86_avx10_maskz_sub_bf16_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
 ; X86-NEXT:    vsubbf16 %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf5,0x75,0x89,0x5c,0xc2]
 ; X86-NEXT:    vsubbf16 (%eax), %xmm1, %xmm1 # encoding: [0x62,0xf5,0x75,0x08,0x5c,0x08]
 ; X86-NEXT:    vsubbf16 %xmm1, %xmm0, %xmm0 {%k1} # encoding: [0x62,0xf5,0x7d,0x09,0x5c,0xc1]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-cvt.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-cvt.ll
index a78d97782e6a..76c87900b04d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-cvt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-cvt.ll
@@ -22,27 +22,27 @@ define <8 x double> @sltof864(<8 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm0
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
@@ -66,14 +66,14 @@ define <4 x double> @slto4f64(<4 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; NODQ-NEXT:    retq
@@ -97,9 +97,9 @@ define <2 x double> @slto2f64(<2 x i64> %a) {
 ; NODQ-LABEL: slto2f64:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; NODQ-NEXT:    retq
 ;
@@ -123,9 +123,9 @@ define <2 x float> @sltof2f32(<2 x i64> %a) {
 ; NODQ-LABEL: sltof2f32:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; NODQ-NEXT:    retq
 ;
@@ -148,12 +148,12 @@ define <2 x float> @sltof2f32(<2 x i64> %a) {
 define <4 x float> @slto4f32_mem(ptr %a) {
 ; NODQ-LABEL: slto4f32_mem:
 ; NODQ:       # %bb.0:
-; NODQ-NEXT:    vcvtsi2ssq 8(%rdi), %xmm0, %xmm0
-; NODQ-NEXT:    vcvtsi2ssq (%rdi), %xmm1, %xmm1
+; NODQ-NEXT:    vcvtsi2ssq 8(%rdi), %xmm15, %xmm0
+; NODQ-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; NODQ-NEXT:    vcvtsi2ssq 16(%rdi), %xmm2, %xmm1
+; NODQ-NEXT:    vcvtsi2ssq 16(%rdi), %xmm15, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; NODQ-NEXT:    vcvtsi2ssq 24(%rdi), %xmm2, %xmm1
+; NODQ-NEXT:    vcvtsi2ssq 24(%rdi), %xmm15, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
 ; NODQ-NEXT:    retq
 ;
@@ -246,16 +246,16 @@ define <4 x float> @slto4f32(<4 x i64> %a) {
 ; NODQ-LABEL: slto4f32:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; NODQ-NEXT:    vzeroupper
 ; NODQ-NEXT:    retq
@@ -281,16 +281,16 @@ define <4 x float> @ulto4f32(<4 x i64> %a) {
 ; NODQ-LABEL: ulto4f32:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; NODQ-NEXT:    vzeroupper
 ; NODQ-NEXT:    retq
@@ -316,16 +316,16 @@ define <4 x float> @ulto4f32_nneg(<4 x i64> %a) {
 ; NODQ-LABEL: ulto4f32_nneg:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; NODQ-NEXT:    vzeroupper
 ; NODQ-NEXT:    retq
@@ -351,7 +351,7 @@ define <8 x double> @ulto8f64(<8 x i64> %a) {
 ; NODQ-LABEL: ulto8f64:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; NODQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
+; NODQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
 ; NODQ-NEXT:    vpsrlq $32, %zmm0, %zmm0
 ; NODQ-NEXT:    vporq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
 ; NODQ-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
@@ -864,7 +864,7 @@ define <2 x double> @f32tof64_inreg(<2 x double> %a0, <4 x float> %a1) nounwind
 define double @sltof64_load(ptr nocapture %e) {
 ; ALL-LABEL: sltof64_load:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %tmp1 = load i64, ptr %e, align 8
@@ -875,7 +875,7 @@ entry:
 define double @sitof64_load(ptr %e) {
 ; ALL-LABEL: sitof64_load:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtsi2sdl (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %tmp1 = load i32, ptr %e, align 4
@@ -886,7 +886,7 @@ entry:
 define float @sitof32_load(ptr %e) {
 ; ALL-LABEL: sitof32_load:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtsi2ssl (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %tmp1 = load i32, ptr %e, align 4
@@ -897,7 +897,7 @@ entry:
 define float @sltof32_load(ptr %e) {
 ; ALL-LABEL: sltof32_load:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %tmp1 = load i64, ptr %e, align 8
@@ -990,28 +990,28 @@ define <8 x float> @slto8f32(<8 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm0
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; NODQ-NEXT:    retq
@@ -1034,54 +1034,54 @@ define <16 x float> @slto16f32(<16 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0],xmm2[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; NODQ-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm2
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
 ; NODQ-NEXT:    vextractf32x4 $3, %zmm0, %xmm3
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0],xmm2[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm0
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
@@ -1109,27 +1109,27 @@ define <8 x double> @slto8f64(<8 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm0
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
@@ -1153,53 +1153,53 @@ define <16 x double> @slto16f64(<16 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm4
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm0
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm2
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm3[0],xmm4[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm1
+; NODQ-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm4[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm2, %zmm1, %zmm1
@@ -1225,28 +1225,28 @@ define <8 x float> @ulto8f32(<8 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm1
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm0
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; NODQ-NEXT:    retq
@@ -1269,54 +1269,54 @@ define <16 x float> @ulto16f32(<16 x i64> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vextracti32x4 $2, %zmm1, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm2
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
 ; NODQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm3
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm4
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0],xmm2[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; NODQ-NEXT:    vmovq %xmm1, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm1
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; NODQ-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; NODQ-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm2, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm2
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
 ; NODQ-NEXT:    vextractf32x4 $3, %zmm0, %xmm3
 ; NODQ-NEXT:    vmovq %xmm3, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0],xmm2[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm3, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm3
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-NEXT:    vmovq %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm4
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm4
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-NEXT:    vcvtusi2ss %rax, %xmm5, %xmm0
+; NODQ-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[0]
 ; NODQ-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
@@ -1498,7 +1498,7 @@ define i32 @fptoui(float %a) nounwind {
 define float @uitof32(i32 %a) nounwind {
 ; ALL-LABEL: uitof32:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; ALL-NEXT:    retq
   %b = uitofp i32 %a to float
   ret float %b
@@ -1507,7 +1507,7 @@ define float @uitof32(i32 %a) nounwind {
 define double @uitof64(i32 %a) nounwind {
 ; ALL-LABEL: uitof64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; ALL-NEXT:    retq
   %b = uitofp i32 %a to double
   ret double %b
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index a8574c0b7516..30bf1a261f4b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -1843,7 +1843,7 @@ define <2 x double> @test_mm_cvtu64_sd(<2 x double> %__A, i64 %__B) {
 ; X86-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
 ; X86-NEXT:    vshufpd {{.*#+}} xmm2 = xmm1[1,0]
 ; X86-NEXT:    vaddsd %xmm1, %xmm2, %xmm1
-; X86-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_cvtu64_sd:
@@ -1891,7 +1891,7 @@ define <4 x float> @test_mm_cvtu64_ss(<4 x float> %__A, i64 %__B) {
 ; X86-NEXT:    fadds {{\.?LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X86-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    .cfi_def_cfa %esp, 4
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index c1ef500d9d3d..aae48aba93be 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -10483,7 +10483,7 @@ define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss_rm(<4 x float> %x0, <4 x
 ; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_ss_rm:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; CHECK-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; CHECK-NEXT:    vmovss %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x10,0xc1]
 ; CHECK-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
   %q = load float, ptr %ptr_b
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics.ll
index 926af4e9957a..f9b5994a18d3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-intrinsics.ll
@@ -6505,7 +6505,7 @@ define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss_rm(<4 x float> %x0, <4 x
 ; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_ss_rm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %q = load float, ptr %ptr_b
   %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-logic.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-logic.ll
index 23f4fcb1c77c..bdcc524545fb 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -889,7 +889,7 @@ define <16 x i32> @ternlog_xor_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 define <16 x i32> @ternlog_or_and_mask(<16 x i32> %x, <16 x i32> %y) {
 ; ALL-LABEL: ternlog_or_and_mask:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 = (zmm0 & mem) | zmm1
+; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 = (zmm0 & m32bcst) | zmm1
 ; ALL-NEXT:    retq
   %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %b = or <16 x i32> %a, %y
@@ -899,7 +899,7 @@ define <16 x i32> @ternlog_or_and_mask(<16 x i32> %x, <16 x i32> %y) {
 define <8 x i64> @ternlog_xor_and_mask(<8 x i64> %x, <8 x i64> %y) {
 ; ALL-LABEL: ternlog_xor_and_mask:
 ; ALL:       ## %bb.0:
-; ALL-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem)
+; ALL-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m64bcst)
 ; ALL-NEXT:    retq
   %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %b = xor <8 x i64> %a, %y
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
index 88c99a06326a..a664cc7f17a5 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
@@ -1221,17 +1221,17 @@ define dso_local x86_regcallcc i32 @test_argRetMixTypes(double, float, i8 signex
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X32-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
 ; X32-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; X32-NEXT:    vcvtsi2sd %eax, %xmm2, %xmm1
+; X32-NEXT:    vcvtsi2sd %eax, %xmm3, %xmm1
 ; X32-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vcvtsi2sd %ecx, %xmm2, %xmm1
+; X32-NEXT:    vcvtsi2sd %ecx, %xmm3, %xmm1
 ; X32-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vmovd %edx, %xmm1
 ; X32-NEXT:    vpinsrd $1, %edi, %xmm1, %xmm1
 ; X32-NEXT:    vcvtqq2pd %ymm1, %ymm1
 ; X32-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vcvtsi2sd %esi, %xmm2, %xmm1
+; X32-NEXT:    vcvtsi2sd %esi, %xmm3, %xmm1
 ; X32-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vcvtsi2sdl (%ebx), %xmm2, %xmm1
+; X32-NEXT:    vcvtsi2sdl (%ebx), %xmm3, %xmm1
 ; X32-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vcvttsd2si %xmm0, %eax
 ; X32-NEXT:    popl %ebx
@@ -1242,15 +1242,15 @@ define dso_local x86_regcallcc i32 @test_argRetMixTypes(double, float, i8 signex
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
 ; WIN64-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; WIN64-NEXT:    vcvtsi2sd %eax, %xmm2, %xmm1
+; WIN64-NEXT:    vcvtsi2sd %eax, %xmm7, %xmm1
 ; WIN64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; WIN64-NEXT:    vcvtsi2sd %ecx, %xmm2, %xmm1
+; WIN64-NEXT:    vcvtsi2sd %ecx, %xmm7, %xmm1
 ; WIN64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; WIN64-NEXT:    vcvtsi2sd %rdx, %xmm2, %xmm1
+; WIN64-NEXT:    vcvtsi2sd %rdx, %xmm7, %xmm1
 ; WIN64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; WIN64-NEXT:    vcvtsi2sd %edi, %xmm2, %xmm1
+; WIN64-NEXT:    vcvtsi2sd %edi, %xmm7, %xmm1
 ; WIN64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; WIN64-NEXT:    vcvtsi2sdl (%rsi), %xmm2, %xmm1
+; WIN64-NEXT:    vcvtsi2sdl (%rsi), %xmm7, %xmm1
 ; WIN64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; WIN64-NEXT:    vcvttsd2si %xmm0, %eax
 ; WIN64-NEXT:    retq
@@ -1259,15 +1259,15 @@ define dso_local x86_regcallcc i32 @test_argRetMixTypes(double, float, i8 signex
 ; LINUXOSX64:       # %bb.0:
 ; LINUXOSX64-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
 ; LINUXOSX64-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; LINUXOSX64-NEXT:    vcvtsi2sd %eax, %xmm2, %xmm1
+; LINUXOSX64-NEXT:    vcvtsi2sd %eax, %xmm7, %xmm1
 ; LINUXOSX64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; LINUXOSX64-NEXT:    vcvtsi2sd %ecx, %xmm2, %xmm1
+; LINUXOSX64-NEXT:    vcvtsi2sd %ecx, %xmm7, %xmm1
 ; LINUXOSX64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; LINUXOSX64-NEXT:    vcvtsi2sd %rdx, %xmm2, %xmm1
+; LINUXOSX64-NEXT:    vcvtsi2sd %rdx, %xmm7, %xmm1
 ; LINUXOSX64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; LINUXOSX64-NEXT:    vcvtsi2sd %edi, %xmm2, %xmm1
+; LINUXOSX64-NEXT:    vcvtsi2sd %edi, %xmm7, %xmm1
 ; LINUXOSX64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; LINUXOSX64-NEXT:    vcvtsi2sdl (%rsi), %xmm2, %xmm1
+; LINUXOSX64-NEXT:    vcvtsi2sdl (%rsi), %xmm7, %xmm1
 ; LINUXOSX64-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; LINUXOSX64-NEXT:    vcvttsd2si %xmm0, %eax
 ; LINUXOSX64-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-scalar_mask.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-scalar_mask.ll
index 9e9fc57cf7b6..fc2bf8863167 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-scalar_mask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-scalar_mask.ll
@@ -24,7 +24,7 @@ define <4 x float>@test_var_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float>
   ret < 4 x float> %res
 }
 
-; FIXME: we should just return %xmm0 here.
+; just return %xmm0 here.
 define <4 x float>@test_const0_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const0_mask:
 ; CHECK:       ## %bb.0:
@@ -33,7 +33,7 @@ define <4 x float>@test_const0_mask(<4 x float> %v0, <4 x float> %v1, <4 x float
   ret < 4 x float> %res
 }
 
-; FIXME: we should zero the lower element of xmm0 and return it.
+; zero the lower element of xmm0 and return it.
 define <4 x float>@test_const0_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const0_maskz:
 ; CHECK:       ## %bb.0:
@@ -44,7 +44,7 @@ define <4 x float>@test_const0_maskz(<4 x float> %v0, <4 x float> %v1, <4 x floa
   ret < 4 x float> %res
 }
 
-; FIXME: we should just return %xmm0 here.
+; just return %xmm0 here.
 define <4 x float>@test_const2_mask(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const2_mask:
 ; CHECK:       ## %bb.0:
@@ -53,7 +53,7 @@ define <4 x float>@test_const2_mask(<4 x float> %v0, <4 x float> %v1, <4 x float
   ret < 4 x float> %res
 }
 
-; FIXME: we should zero the lower element of xmm0 and return it.
+; zero the lower element of xmm0 and return it.
 define <4 x float>@test_const2_maskz(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
 ; CHECK-LABEL: test_const2_maskz:
 ; CHECK:       ## %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512-shuffles/shuffle-blend.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512-shuffles/shuffle-blend.ll
index 59e9fb1c4a9f..78957d10301c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512-shuffles/shuffle-blend.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512-shuffles/shuffle-blend.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F,X86-AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F,X64-AVX512F
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F
 ; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512BW,X86-AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512BW,X64-AVX512BW
 
@@ -61,27 +61,16 @@ entry:
 }
 
 define <64 x i8> @addb_selectw_64xi8(<64 x i8> %t0, <64 x i8> %t1) {
-; X86-AVX512F-LABEL: addb_selectw_64xi8:
-; X86-AVX512F:       # %bb.0:
-; X86-AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; X86-AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; X86-AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
-; X86-AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm3
-; X86-AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; X86-AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
-; X86-AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}, %zmm2, %zmm0
-; X86-AVX512F-NEXT:    retl
-;
-; X64-AVX512F-LABEL: addb_selectw_64xi8:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; X64-AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; X64-AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
-; X64-AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm3
-; X64-AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; X64-AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
-; X64-AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; X64-AVX512F-NEXT:    retq
+; AVX512F-LABEL: addb_selectw_64xi8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    ret{{[l|q]}}
 ;
 ; X86-AVX512BW-LABEL: addb_selectw_64xi8:
 ; X86-AVX512BW:       # %bb.0:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512copy-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512copy-intrinsics.ll
index a7ca23792e6f..a2af7df44010 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512copy-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512copy-intrinsics.ll
@@ -11,7 +11,7 @@ define <4 x i32> @test_mm_move_epi32(<4 x i32> %a0) nounwind {
 ; NOAVX512MOVZXC-LABEL: test_mm_move_epi32:
 ; NOAVX512MOVZXC:       # %bb.0:
 ; NOAVX512MOVZXC-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf0,0x57,0xc9]
-; NOAVX512MOVZXC-NEXT:    vblendps $1, %xmm0, %xmm1, %xmm0 # encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x01]
+; NOAVX512MOVZXC-NEXT:    vmovss %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf2,0x10,0xc0]
 ; NOAVX512MOVZXC-NEXT:    # xmm0 = xmm0[0],xmm1[1,2,3]
 ; NOAVX512MOVZXC-NEXT:    retq # encoding: [0xc3]
   %res = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-arith.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-arith.ll
index b264f5fc3468..d19c9bb55017 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-arith.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-arith.ll
@@ -384,7 +384,7 @@ declare <8 x half> @llvm.fabs.v8f16(<8 x half>)
 define <8 x half> @fcopysignv8f16(<8 x half> %x, <8 x half> %y) {
 ; CHECK-LABEL: fcopysignv8f16:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (mem & (xmm0 ^ xmm1))
+; CHECK-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (m32bcst & (xmm0 ^ xmm1))
 ; CHECK-NEXT:    retq
   %a = call <8 x half> @llvm.copysign.v8f16(<8 x half> %x, <8 x half> %y)
   ret <8 x half> %a
@@ -439,7 +439,7 @@ declare <16 x half> @llvm.fabs.v16f16(<16 x half>)
 define <16 x half> @fcopysignv16f16(<16 x half> %x, <16 x half> %y) {
 ; CHECK-LABEL: fcopysignv16f16:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (mem & (ymm0 ^ ymm1))
+; CHECK-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (m32bcst & (ymm0 ^ ymm1))
 ; CHECK-NEXT:    retq
   %a = call <16 x half> @llvm.copysign.v16f16(<16 x half> %x, <16 x half> %y)
   ret <16 x half> %a
@@ -494,7 +494,7 @@ declare <32 x half> @llvm.fabs.v32f16(<32 x half>)
 define <32 x half> @fcopysignv32f16(<32 x half> %x, <32 x half> %y) {
 ; CHECK-LABEL: fcopysignv32f16:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (mem & (zmm0 ^ zmm1))
+; CHECK-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (m32bcst & (zmm0 ^ zmm1))
 ; CHECK-NEXT:    retq
   %a = call <32 x half> @llvm.copysign.v32f16(<32 x half> %x, <32 x half> %y)
   ret <32 x half> %a
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-cvt.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
index 26abf51c76b2..3f6ddc6ecfd7 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
@@ -412,7 +412,7 @@ define double @extload_f16_f64(ptr %x) {
 define float @extload_f16_f32_optsize(ptr %x) optsize {
 ; X64-LABEL: extload_f16_f32_optsize:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtsh2ss (%rdi), %xmm0, %xmm0
+; X64-NEXT:    vcvtsh2ss (%rdi), %xmm15, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: extload_f16_f32_optsize:
@@ -420,7 +420,7 @@ define float @extload_f16_f32_optsize(ptr %x) optsize {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsh2ss (%eax), %xmm0, %xmm0
+; X86-NEXT:    vcvtsh2ss (%eax), %xmm7, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
 ; X86-NEXT:    popl %eax
@@ -434,7 +434,7 @@ define float @extload_f16_f32_optsize(ptr %x) optsize {
 define double @extload_f16_f64_optsize(ptr %x) optsize {
 ; X64-LABEL: extload_f16_f64_optsize:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtsh2sd (%rdi), %xmm0, %xmm0
+; X64-NEXT:    vcvtsh2sd (%rdi), %xmm15, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: extload_f16_f64_optsize:
@@ -447,7 +447,7 @@ define double @extload_f16_f64_optsize(ptr %x) optsize {
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    movl 8(%ebp), %eax
-; X86-NEXT:    vcvtsh2sd (%eax), %xmm0, %xmm0
+; X86-NEXT:    vcvtsh2sd (%eax), %xmm7, %xmm0
 ; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
@@ -559,13 +559,13 @@ define half @s8_to_half(i8 %x) {
 ; X64-LABEL: s8_to_half:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movsbl %dil, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: s8_to_half:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
   %a = sitofp i8 %x to half
   ret half %a
@@ -575,13 +575,13 @@ define half @s16_to_half(i16 %x) {
 ; X64-LABEL: s16_to_half:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: s16_to_half:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
   %a = sitofp i16 %x to half
   ret half %a
@@ -590,12 +590,12 @@ define half @s16_to_half(i16 %x) {
 define half @s32_to_half(i32 %x) {
 ; X64-LABEL: s32_to_half:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtsi2sh %edi, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %edi, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: s32_to_half:
 ; X86:       # %bb.0:
-; X86-NEXT:    vcvtsi2shl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2shl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
   %a = sitofp i32 %x to half
   ret half %a
@@ -604,7 +604,7 @@ define half @s32_to_half(i32 %x) {
 define half @s64_to_half(i64 %x) {
 ; X64-LABEL: s64_to_half:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtsi2sh %rdi, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %rdi, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: s64_to_half:
@@ -644,13 +644,13 @@ define half @u8_to_half(i8 %x) {
 ; X64-LABEL: u8_to_half:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: u8_to_half:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
   %a = uitofp i8 %x to half
   ret half %a
@@ -660,13 +660,13 @@ define half @u16_to_half(i16 %x) {
 ; X64-LABEL: u16_to_half:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl %di, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: u16_to_half:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
   %a = uitofp i16 %x to half
   ret half %a
@@ -675,12 +675,12 @@ define half @u16_to_half(i16 %x) {
 define half @u32_to_half(i32 %x) {
 ; X64-LABEL: u32_to_half:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtusi2sh %edi, %xmm0, %xmm0
+; X64-NEXT:    vcvtusi2sh %edi, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: u32_to_half:
 ; X86:       # %bb.0:
-; X86-NEXT:    vcvtusi2shl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vcvtusi2shl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
   %a = uitofp i32 %x to half
   ret half %a
@@ -689,7 +689,7 @@ define half @u32_to_half(i32 %x) {
 define half @u64_to_half(i64 %x) {
 ; X64-LABEL: u64_to_half:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtusi2sh %rdi, %xmm0, %xmm0
+; X64-NEXT:    vcvtusi2sh %rdi, %xmm31, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: u64_to_half:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
index 1d535f93bc86..eac803f83e86 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,avx512vl    | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,avx512vl    | FileCheck %s --check-prefixes=CHECK,HasVL
+; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16    | FileCheck %s --check-prefixes=CHECK,NOVL
 
 declare half @llvm.maxnum.f16(half, half)
 declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>)
@@ -9,61 +10,397 @@ declare <16 x half> @llvm.maxnum.v16f16(<16 x half>, <16 x half>)
 declare <32 x half> @llvm.maxnum.v32f16(<32 x half>, <32 x half>)
 
 define half @test_intrinsic_fmaxh(half %x, half %y) {
-; CHECK-LABEL: test_intrinsic_fmaxh:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmaxsh %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xd0]
-; CHECK-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xd1]
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmaxh:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xd0]
+; HasVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xd1]
+; HasVL-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmaxh:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xd0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xd1]
+; NOVL-NEXT:    vmovaps %xmm2, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc2]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call half @llvm.maxnum.f16(half %x, half %y) readnone
   ret half %z
 }
 
 define <2 x half> @test_intrinsic_fmax_v2f16(<2 x half> %x, <2 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmax_v2f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmaxph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5f,0xd0]
-; CHECK-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmax_v2f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vmaxph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5f,0xd0]
+; HasVL-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmax_v2f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm4 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xe2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm2 # encoding: [0xc5,0xf8,0xc6,0xd0,0xff]
+; NOVL-NEXT:    # xmm2 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm3 # encoding: [0xc5,0xf9,0x70,0xd9,0xff]
+; NOVL-NEXT:    # xmm3 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm5 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xea]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm2 # encoding: [0xc5,0xd1,0x61,0xd4]
+; NOVL-NEXT:    # xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm3 # encoding: [0xc5,0xf9,0xc6,0xd8,0x01]
+; NOVL-NEXT:    # xmm3 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm4 # encoding: [0xc5,0xf1,0xc6,0xe1,0x01]
+; NOVL-NEXT:    # xmm4 = xmm1[1,0]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd1,0x30]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm4 # encoding: [0xc5,0xfa,0x16,0xe1]
+; NOVL-NEXT:    # xmm4 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm4 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xe0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe1]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm4, %xmm0 # encoding: [0xc5,0xd9,0x61,0xc5]
+; NOVL-NEXT:    # xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %x, <2 x half> %y) readnone
   ret <2 x half> %z
 }
 
 define <4 x half> @test_intrinsic_fmax_v4f16(<4 x half> %x, <4 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmax_v4f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmaxph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5f,0xd0]
-; CHECK-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmax_v4f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vmaxph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5f,0xd0]
+; HasVL-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmax_v4f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm4 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xe2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm2 # encoding: [0xc5,0xf8,0xc6,0xd0,0xff]
+; NOVL-NEXT:    # xmm2 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm3 # encoding: [0xc5,0xf9,0x70,0xd9,0xff]
+; NOVL-NEXT:    # xmm3 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm5 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xea]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm2 # encoding: [0xc5,0xd1,0x61,0xd4]
+; NOVL-NEXT:    # xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm3 # encoding: [0xc5,0xf9,0xc6,0xd8,0x01]
+; NOVL-NEXT:    # xmm3 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm4 # encoding: [0xc5,0xf1,0xc6,0xe1,0x01]
+; NOVL-NEXT:    # xmm4 = xmm1[1,0]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd1,0x30]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm4 # encoding: [0xc5,0xfa,0x16,0xe1]
+; NOVL-NEXT:    # xmm4 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm4 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xe0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe1]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm4, %xmm0 # encoding: [0xc5,0xd9,0x61,0xc5]
+; NOVL-NEXT:    # xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %x, <4 x half> %y) readnone
   ret <4 x half> %z
 }
 
 define <8 x half> @test_intrinsic_fmax_v8f16(<8 x half> %x, <8 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmax_v8f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmaxph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5f,0xd0]
-; CHECK-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmax_v8f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vmaxph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5f,0xd0]
+; HasVL-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmax_v8f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm4 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xe2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm2 # encoding: [0xc5,0xf8,0xc6,0xd0,0xff]
+; NOVL-NEXT:    # xmm2 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm3 # encoding: [0xc5,0xf9,0x70,0xd9,0xff]
+; NOVL-NEXT:    # xmm3 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm5 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xea]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm2 # encoding: [0xc5,0xd1,0x61,0xd4]
+; NOVL-NEXT:    # xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm3 # encoding: [0xc5,0xf9,0xc6,0xd8,0x01]
+; NOVL-NEXT:    # xmm3 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm4 # encoding: [0xc5,0xf1,0xc6,0xe1,0x01]
+; NOVL-NEXT:    # xmm4 = xmm1[1,0]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd1,0x30]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm4 # encoding: [0xc5,0xfa,0x16,0xe1]
+; NOVL-NEXT:    # xmm4 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5f,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm4 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xe0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe1]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm4, %xmm0 # encoding: [0xc5,0xd9,0x61,0xc5]
+; NOVL-NEXT:    # xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <8 x half> @llvm.maxnum.v8f16(<8 x half> %x, <8 x half> %y) readnone
   ret <8 x half> %z
 }
 
 define <16 x half> @test_intrinsic_fmax_v16f16(<16 x half> %x, <16 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmax_v16f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmaxph %ymm0, %ymm1, %ymm2 # encoding: [0x62,0xf5,0x74,0x28,0x5f,0xd0]
-; CHECK-NEXT:    vcmpunordph %ymm0, %ymm0, %k1 # encoding: [0x62,0xf3,0x7c,0x28,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %ymm1, %ymm2 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmax_v16f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vmaxph %ymm0, %ymm1, %ymm2 # encoding: [0x62,0xf5,0x74,0x28,0x5f,0xd0]
+; HasVL-NEXT:    vcmpunordph %ymm0, %ymm0, %k1 # encoding: [0x62,0xf3,0x7c,0x28,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %ymm1, %ymm2 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmax_v16f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vextracti128 $1, %ymm0, %xmm2 # encoding: [0xc4,0xe3,0x7d,0x39,0xc2,0x01]
+; NOVL-NEXT:    vpsrldq $14, %xmm2, %xmm4 # encoding: [0xc5,0xd9,0x73,0xda,0x0e]
+; NOVL-NEXT:    # xmm4 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vextracti128 $1, %ymm1, %xmm3 # encoding: [0xc4,0xe3,0x7d,0x39,0xcb,0x01]
+; NOVL-NEXT:    vpsrldq $14, %xmm3, %xmm5 # encoding: [0xc5,0xd1,0x73,0xdb,0x0e]
+; NOVL-NEXT:    # xmm5 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm4, %xmm5, %xmm6 # encoding: [0x62,0xf5,0x56,0x08,0x5f,0xf4]
+; NOVL-NEXT:    vcmpunordsh %xmm4, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcc,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf5]
+; NOVL-NEXT:    vpshufd $255, %xmm2, %xmm4 # encoding: [0xc5,0xf9,0x70,0xe2,0xff]
+; NOVL-NEXT:    # xmm4 = xmm2[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm3, %xmm5 # encoding: [0xc5,0xf9,0x70,0xeb,0xff]
+; NOVL-NEXT:    # xmm5 = xmm3[3,3,3,3]
+; NOVL-NEXT:    vmaxsh %xmm4, %xmm5, %xmm7 # encoding: [0x62,0xf5,0x56,0x08,0x5f,0xfc]
+; NOVL-NEXT:    vcmpunordsh %xmm4, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcc,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfd]
+; NOVL-NEXT:    vpunpcklwd %xmm6, %xmm7, %xmm4 # encoding: [0xc5,0xc1,0x61,0xe6]
+; NOVL-NEXT:    # xmm4 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm2, %xmm5 # encoding: [0xc5,0xd1,0x73,0xda,0x0a]
+; NOVL-NEXT:    # xmm5 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm3, %xmm6 # encoding: [0xc5,0xc9,0x73,0xdb,0x0a]
+; NOVL-NEXT:    # xmm6 = xmm3[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm5, %xmm6, %xmm7 # encoding: [0x62,0xf5,0x4e,0x08,0x5f,0xfd]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfe]
+; NOVL-NEXT:    vshufpd $1, %xmm2, %xmm2, %xmm5 # encoding: [0xc5,0xe9,0xc6,0xea,0x01]
+; NOVL-NEXT:    # xmm5 = xmm2[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm3, %xmm3, %xmm6 # encoding: [0xc5,0xe1,0xc6,0xf3,0x01]
+; NOVL-NEXT:    # xmm6 = xmm3[1,0]
+; NOVL-NEXT:    vmaxsh %xmm5, %xmm6, %xmm8 # encoding: [0x62,0x75,0x4e,0x08,0x5f,0xc5]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc6]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm8, %xmm5 # encoding: [0xc5,0xb9,0x61,0xef]
+; NOVL-NEXT:    # xmm5 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; NOVL-NEXT:    vpunpckldq %xmm4, %xmm5, %xmm4 # encoding: [0xc5,0xd1,0x62,0xe4]
+; NOVL-NEXT:    # xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm5 # encoding: [0xc5,0xd1,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm5 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm6 # encoding: [0xc5,0xc9,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm6 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm5, %xmm6, %xmm7 # encoding: [0x62,0xf5,0x4e,0x08,0x5f,0xfd]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfe]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm5 # encoding: [0xc5,0xf8,0xc6,0xe8,0xff]
+; NOVL-NEXT:    # xmm5 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm6 # encoding: [0xc5,0xf9,0x70,0xf1,0xff]
+; NOVL-NEXT:    # xmm6 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vmaxsh %xmm5, %xmm6, %xmm8 # encoding: [0x62,0x75,0x4e,0x08,0x5f,0xc5]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc6]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm8, %xmm5 # encoding: [0xc5,0xb9,0x61,0xef]
+; NOVL-NEXT:    # xmm5 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm6 # encoding: [0xc5,0xc9,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm6 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm7 # encoding: [0xc5,0xc1,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm7 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vmaxsh %xmm6, %xmm7, %xmm8 # encoding: [0x62,0x75,0x46,0x08,0x5f,0xc6]
+; NOVL-NEXT:    vcmpunordsh %xmm6, %xmm6, %k1 # encoding: [0x62,0xf3,0x4e,0x08,0xc2,0xce,0x03]
+; NOVL-NEXT:    vmovsh %xmm7, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc7]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm6 # encoding: [0xc5,0xf9,0xc6,0xf0,0x01]
+; NOVL-NEXT:    # xmm6 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm7 # encoding: [0xc5,0xf1,0xc6,0xf9,0x01]
+; NOVL-NEXT:    # xmm7 = xmm1[1,0]
+; NOVL-NEXT:    vmaxsh %xmm6, %xmm7, %xmm9 # encoding: [0x62,0x75,0x46,0x08,0x5f,0xce]
+; NOVL-NEXT:    vcmpunordsh %xmm6, %xmm6, %k1 # encoding: [0x62,0xf3,0x4e,0x08,0xc2,0xce,0x03]
+; NOVL-NEXT:    vmovsh %xmm7, %xmm0, %xmm9 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xcf]
+; NOVL-NEXT:    vpunpcklwd %xmm8, %xmm9, %xmm6 # encoding: [0xc4,0xc1,0x31,0x61,0xf0]
+; NOVL-NEXT:    # xmm6 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; NOVL-NEXT:    vpunpckldq %xmm5, %xmm6, %xmm5 # encoding: [0xc5,0xc9,0x62,0xed]
+; NOVL-NEXT:    # xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; NOVL-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm4 # encoding: [0xc4,0xe3,0x55,0x38,0xe4,0x01]
+; NOVL-NEXT:    vpsrlq $48, %xmm2, %xmm5 # encoding: [0xc5,0xd1,0x73,0xd2,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm3, %xmm6 # encoding: [0xc5,0xc9,0x73,0xd3,0x30]
+; NOVL-NEXT:    vmaxsh %xmm5, %xmm6, %xmm7 # encoding: [0x62,0xf5,0x4e,0x08,0x5f,0xfd]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfe]
+; NOVL-NEXT:    vmovshdup %xmm2, %xmm5 # encoding: [0xc5,0xfa,0x16,0xea]
+; NOVL-NEXT:    # xmm5 = xmm2[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm3, %xmm6 # encoding: [0xc5,0xfa,0x16,0xf3]
+; NOVL-NEXT:    # xmm6 = xmm3[1,1,3,3]
+; NOVL-NEXT:    vmaxsh %xmm5, %xmm6, %xmm8 # encoding: [0x62,0x75,0x4e,0x08,0x5f,0xc5]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc6]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm8, %xmm5 # encoding: [0xc5,0xb9,0x61,0xef]
+; NOVL-NEXT:    # xmm5 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm6 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xf2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf3]
+; NOVL-NEXT:    vpsrld $16, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x72,0xd2,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm3, %xmm3 # encoding: [0xc5,0xe1,0x72,0xd3,0x10]
+; NOVL-NEXT:    vmaxsh %xmm2, %xmm3, %xmm7 # encoding: [0x62,0xf5,0x66,0x08,0x5f,0xfa]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfb]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm6, %xmm2 # encoding: [0xc5,0xc9,0x61,0xd7]
+; NOVL-NEXT:    # xmm2 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; NOVL-NEXT:    vpunpckldq %xmm5, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x62,0xd5]
+; NOVL-NEXT:    # xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm5 # encoding: [0xc5,0xd1,0x73,0xd1,0x30]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm5, %xmm6 # encoding: [0x62,0xf5,0x56,0x08,0x5f,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf5]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm5 # encoding: [0xc5,0xfa,0x16,0xe9]
+; NOVL-NEXT:    # xmm5 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vmaxsh %xmm3, %xmm5, %xmm7 # encoding: [0x62,0xf5,0x56,0x08,0x5f,0xfb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfd]
+; NOVL-NEXT:    vpunpcklwd %xmm6, %xmm7, %xmm3 # encoding: [0xc5,0xc1,0x61,0xde]
+; NOVL-NEXT:    # xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vmaxsh %xmm0, %xmm1, %xmm6 # encoding: [0x62,0xf5,0x76,0x08,0x5f,0xf0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf1]
+; NOVL-NEXT:    vpunpcklwd %xmm6, %xmm5, %xmm0 # encoding: [0xc5,0xd1,0x61,0xc6]
+; NOVL-NEXT:    # xmm0 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x38,0xc2,0x01]
+; NOVL-NEXT:    vpunpcklqdq %ymm4, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x6c,0xc4]
+; NOVL-NEXT:    # ymm0 = ymm0[0],ymm4[0],ymm0[2],ymm4[2]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <16 x half> @llvm.maxnum.v16f16(<16 x half> %x, <16 x half> %y) readnone
   ret <16 x half> %z
 }
@@ -81,10 +418,68 @@ define <32 x half> @test_intrinsic_fmax_v32f16(<32 x half> %x, <32 x half> %y) {
 }
 
 define <4 x half> @maxnum_intrinsic_nnan_fmf_f432(<4 x half> %a, <4 x half> %b) {
-; CHECK-LABEL: maxnum_intrinsic_nnan_fmf_f432:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmaxph %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7c,0x08,0x5f,0xc1]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: maxnum_intrinsic_nnan_fmf_f432:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vmaxph %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7c,0x08,0x5f,0xc1]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: maxnum_intrinsic_nnan_fmf_f432:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vcmpltsh %xmm2, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xca,0x01]
+; NOVL-NEXT:    vmovsh %xmm2, %xmm0, %xmm3 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xda]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm2 # encoding: [0xc5,0xf8,0xc6,0xd0,0xff]
+; NOVL-NEXT:    # xmm2 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm4 # encoding: [0xc5,0xf9,0x70,0xe1,0xff]
+; NOVL-NEXT:    # xmm4 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vcmpltsh %xmm2, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xca,0x01]
+; NOVL-NEXT:    vmovsh %xmm2, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe2]
+; NOVL-NEXT:    vpunpcklwd %xmm3, %xmm4, %xmm2 # encoding: [0xc5,0xd9,0x61,0xd3]
+; NOVL-NEXT:    # xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vcmpltsh %xmm3, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcb,0x01]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm3 # encoding: [0xc5,0xf9,0xc6,0xd8,0x01]
+; NOVL-NEXT:    # xmm3 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm5 # encoding: [0xc5,0xf1,0xc6,0xe9,0x01]
+; NOVL-NEXT:    # xmm5 = xmm1[1,0]
+; NOVL-NEXT:    vcmpltsh %xmm3, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcb,0x01]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm3 # encoding: [0xc5,0xd1,0x61,0xdc]
+; NOVL-NEXT:    # xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd1,0x30]
+; NOVL-NEXT:    vcmpltsh %xmm3, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcb,0x01]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm5 # encoding: [0xc5,0xfa,0x16,0xe9]
+; NOVL-NEXT:    # xmm5 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vcmpltsh %xmm3, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcb,0x01]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm3 # encoding: [0xc5,0xd1,0x61,0xdc]
+; NOVL-NEXT:    # xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vcmpltsh %xmm0, %xmm1, %k1 # encoding: [0x62,0xf3,0x76,0x08,0xc2,0xc8,0x01]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x72,0xd1,0x10]
+; NOVL-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xc8]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vcmpltsh %xmm0, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xc8,0x01]
+; NOVL-NEXT:    vmovsh %xmm0, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe0]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0x61,0xc4]
+; NOVL-NEXT:    # xmm0 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %r = tail call nnan <4 x half> @llvm.maxnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %r
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
index b81a6d57bdc6..05509e0099c3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl    | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl    | FileCheck %s --check-prefixes=CHECK,HasVL
+; RUN: llc < %s -verify-machineinstrs --show-mc-encoding -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16    | FileCheck %s --check-prefixes=CHECK,NOVL
 
 declare half @llvm.minnum.f16(half, half)
 declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>)
@@ -9,61 +10,397 @@ declare <16 x half> @llvm.minnum.v16f16(<16 x half>, <16 x half>)
 declare <32 x half> @llvm.minnum.v32f16(<32 x half>, <32 x half>)
 
 define half @test_intrinsic_fminh(half %x, half %y) {
-; CHECK-LABEL: test_intrinsic_fminh:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vminsh %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xd0]
-; CHECK-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xd1]
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fminh:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vminsh %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xd0]
+; HasVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xd1]
+; HasVL-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fminh:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xd0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xd1]
+; NOVL-NEXT:    vmovaps %xmm2, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc2]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call half @llvm.minnum.f16(half %x, half %y) readnone
   ret half %z
 }
 
 define <2 x half> @test_intrinsic_fmin_v2f16(<2 x half> %x, <2 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmin_v2f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vminph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5d,0xd0]
-; CHECK-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmin_v2f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vminph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5d,0xd0]
+; HasVL-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmin_v2f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm4 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xe2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm2 # encoding: [0xc5,0xf8,0xc6,0xd0,0xff]
+; NOVL-NEXT:    # xmm2 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm3 # encoding: [0xc5,0xf9,0x70,0xd9,0xff]
+; NOVL-NEXT:    # xmm3 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm5 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xea]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm2 # encoding: [0xc5,0xd1,0x61,0xd4]
+; NOVL-NEXT:    # xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm3 # encoding: [0xc5,0xf9,0xc6,0xd8,0x01]
+; NOVL-NEXT:    # xmm3 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm4 # encoding: [0xc5,0xf1,0xc6,0xe1,0x01]
+; NOVL-NEXT:    # xmm4 = xmm1[1,0]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd1,0x30]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm4 # encoding: [0xc5,0xfa,0x16,0xe1]
+; NOVL-NEXT:    # xmm4 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm4 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xe0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe1]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm4, %xmm0 # encoding: [0xc5,0xd9,0x61,0xc5]
+; NOVL-NEXT:    # xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <2 x half> @llvm.minnum.v2f16(<2 x half> %x, <2 x half> %y) readnone
   ret <2 x half> %z
 }
 
 define <4 x half> @test_intrinsic_fmin_v4f16(<4 x half> %x, <4 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmin_v4f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vminph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5d,0xd0]
-; CHECK-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmin_v4f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vminph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5d,0xd0]
+; HasVL-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmin_v4f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm4 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xe2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm2 # encoding: [0xc5,0xf8,0xc6,0xd0,0xff]
+; NOVL-NEXT:    # xmm2 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm3 # encoding: [0xc5,0xf9,0x70,0xd9,0xff]
+; NOVL-NEXT:    # xmm3 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm5 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xea]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm2 # encoding: [0xc5,0xd1,0x61,0xd4]
+; NOVL-NEXT:    # xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm3 # encoding: [0xc5,0xf9,0xc6,0xd8,0x01]
+; NOVL-NEXT:    # xmm3 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm4 # encoding: [0xc5,0xf1,0xc6,0xe1,0x01]
+; NOVL-NEXT:    # xmm4 = xmm1[1,0]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd1,0x30]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm4 # encoding: [0xc5,0xfa,0x16,0xe1]
+; NOVL-NEXT:    # xmm4 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm4 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xe0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe1]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm4, %xmm0 # encoding: [0xc5,0xd9,0x61,0xc5]
+; NOVL-NEXT:    # xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <4 x half> @llvm.minnum.v4f16(<4 x half> %x, <4 x half> %y) readnone
   ret <4 x half> %z
 }
 
 define <8 x half> @test_intrinsic_fmin_v8f16(<8 x half> %x, <8 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmin_v8f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vminph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5d,0xd0]
-; CHECK-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmin_v8f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vminph %xmm0, %xmm1, %xmm2 # encoding: [0x62,0xf5,0x74,0x08,0x5d,0xd0]
+; HasVL-NEXT:    vcmpunordph %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7c,0x08,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmin_v8f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm4 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xe2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe3]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm2 # encoding: [0xc5,0xf8,0xc6,0xd0,0xff]
+; NOVL-NEXT:    # xmm2 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm3 # encoding: [0xc5,0xf9,0x70,0xd9,0xff]
+; NOVL-NEXT:    # xmm3 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm5 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xea]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xeb]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm5, %xmm2 # encoding: [0xc5,0xd1,0x61,0xd4]
+; NOVL-NEXT:    # xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm3 # encoding: [0xc5,0xf9,0xc6,0xd8,0x01]
+; NOVL-NEXT:    # xmm3 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm4 # encoding: [0xc5,0xf1,0xc6,0xe1,0x01]
+; NOVL-NEXT:    # xmm4 = xmm1[1,0]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd1,0x30]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm5 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xeb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xec]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm4 # encoding: [0xc5,0xfa,0x16,0xe1]
+; NOVL-NEXT:    # xmm4 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vminsh %xmm3, %xmm4, %xmm6 # encoding: [0x62,0xf5,0x5e,0x08,0x5d,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf4]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm6, %xmm3 # encoding: [0xc5,0xc9,0x61,0xdd]
+; NOVL-NEXT:    # xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm4 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xe0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe1]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpunpcklwd %xmm5, %xmm4, %xmm0 # encoding: [0xc5,0xd9,0x61,0xc5]
+; NOVL-NEXT:    # xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <8 x half> @llvm.minnum.v8f16(<8 x half> %x, <8 x half> %y) readnone
   ret <8 x half> %z
 }
 
 define <16 x half> @test_intrinsic_fmin_v16f16(<16 x half> %x, <16 x half> %y) {
-; CHECK-LABEL: test_intrinsic_fmin_v16f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vminph %ymm0, %ymm1, %ymm2 # encoding: [0x62,0xf5,0x74,0x28,0x5d,0xd0]
-; CHECK-NEXT:    vcmpunordph %ymm0, %ymm0, %k1 # encoding: [0x62,0xf3,0x7c,0x28,0xc2,0xc8,0x03]
-; CHECK-NEXT:    vmovdqu16 %ymm1, %ymm2 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x6f,0xd1]
-; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: test_intrinsic_fmin_v16f16:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vminph %ymm0, %ymm1, %ymm2 # encoding: [0x62,0xf5,0x74,0x28,0x5d,0xd0]
+; HasVL-NEXT:    vcmpunordph %ymm0, %ymm0, %k1 # encoding: [0x62,0xf3,0x7c,0x28,0xc2,0xc8,0x03]
+; HasVL-NEXT:    vmovdqu16 %ymm1, %ymm2 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x6f,0xd1]
+; HasVL-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: test_intrinsic_fmin_v16f16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vextracti128 $1, %ymm0, %xmm2 # encoding: [0xc4,0xe3,0x7d,0x39,0xc2,0x01]
+; NOVL-NEXT:    vpsrldq $14, %xmm2, %xmm4 # encoding: [0xc5,0xd9,0x73,0xda,0x0e]
+; NOVL-NEXT:    # xmm4 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vextracti128 $1, %ymm1, %xmm3 # encoding: [0xc4,0xe3,0x7d,0x39,0xcb,0x01]
+; NOVL-NEXT:    vpsrldq $14, %xmm3, %xmm5 # encoding: [0xc5,0xd1,0x73,0xdb,0x0e]
+; NOVL-NEXT:    # xmm5 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm4, %xmm5, %xmm6 # encoding: [0x62,0xf5,0x56,0x08,0x5d,0xf4]
+; NOVL-NEXT:    vcmpunordsh %xmm4, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcc,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf5]
+; NOVL-NEXT:    vpshufd $255, %xmm2, %xmm4 # encoding: [0xc5,0xf9,0x70,0xe2,0xff]
+; NOVL-NEXT:    # xmm4 = xmm2[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm3, %xmm5 # encoding: [0xc5,0xf9,0x70,0xeb,0xff]
+; NOVL-NEXT:    # xmm5 = xmm3[3,3,3,3]
+; NOVL-NEXT:    vminsh %xmm4, %xmm5, %xmm7 # encoding: [0x62,0xf5,0x56,0x08,0x5d,0xfc]
+; NOVL-NEXT:    vcmpunordsh %xmm4, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcc,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfd]
+; NOVL-NEXT:    vpunpcklwd %xmm6, %xmm7, %xmm4 # encoding: [0xc5,0xc1,0x61,0xe6]
+; NOVL-NEXT:    # xmm4 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm2, %xmm5 # encoding: [0xc5,0xd1,0x73,0xda,0x0a]
+; NOVL-NEXT:    # xmm5 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm3, %xmm6 # encoding: [0xc5,0xc9,0x73,0xdb,0x0a]
+; NOVL-NEXT:    # xmm6 = xmm3[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm5, %xmm6, %xmm7 # encoding: [0x62,0xf5,0x4e,0x08,0x5d,0xfd]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfe]
+; NOVL-NEXT:    vshufpd $1, %xmm2, %xmm2, %xmm5 # encoding: [0xc5,0xe9,0xc6,0xea,0x01]
+; NOVL-NEXT:    # xmm5 = xmm2[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm3, %xmm3, %xmm6 # encoding: [0xc5,0xe1,0xc6,0xf3,0x01]
+; NOVL-NEXT:    # xmm6 = xmm3[1,0]
+; NOVL-NEXT:    vminsh %xmm5, %xmm6, %xmm8 # encoding: [0x62,0x75,0x4e,0x08,0x5d,0xc5]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc6]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm8, %xmm5 # encoding: [0xc5,0xb9,0x61,0xef]
+; NOVL-NEXT:    # xmm5 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; NOVL-NEXT:    vpunpckldq %xmm4, %xmm5, %xmm4 # encoding: [0xc5,0xd1,0x62,0xe4]
+; NOVL-NEXT:    # xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm5 # encoding: [0xc5,0xd1,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm5 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm6 # encoding: [0xc5,0xc9,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm6 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm5, %xmm6, %xmm7 # encoding: [0x62,0xf5,0x4e,0x08,0x5d,0xfd]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfe]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm5 # encoding: [0xc5,0xf8,0xc6,0xe8,0xff]
+; NOVL-NEXT:    # xmm5 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm6 # encoding: [0xc5,0xf9,0x70,0xf1,0xff]
+; NOVL-NEXT:    # xmm6 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vminsh %xmm5, %xmm6, %xmm8 # encoding: [0x62,0x75,0x4e,0x08,0x5d,0xc5]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc6]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm8, %xmm5 # encoding: [0xc5,0xb9,0x61,0xef]
+; NOVL-NEXT:    # xmm5 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm6 # encoding: [0xc5,0xc9,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm6 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm7 # encoding: [0xc5,0xc1,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm7 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vminsh %xmm6, %xmm7, %xmm8 # encoding: [0x62,0x75,0x46,0x08,0x5d,0xc6]
+; NOVL-NEXT:    vcmpunordsh %xmm6, %xmm6, %k1 # encoding: [0x62,0xf3,0x4e,0x08,0xc2,0xce,0x03]
+; NOVL-NEXT:    vmovsh %xmm7, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc7]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm6 # encoding: [0xc5,0xf9,0xc6,0xf0,0x01]
+; NOVL-NEXT:    # xmm6 = xmm0[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm7 # encoding: [0xc5,0xf1,0xc6,0xf9,0x01]
+; NOVL-NEXT:    # xmm7 = xmm1[1,0]
+; NOVL-NEXT:    vminsh %xmm6, %xmm7, %xmm9 # encoding: [0x62,0x75,0x46,0x08,0x5d,0xce]
+; NOVL-NEXT:    vcmpunordsh %xmm6, %xmm6, %k1 # encoding: [0x62,0xf3,0x4e,0x08,0xc2,0xce,0x03]
+; NOVL-NEXT:    vmovsh %xmm7, %xmm0, %xmm9 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xcf]
+; NOVL-NEXT:    vpunpcklwd %xmm8, %xmm9, %xmm6 # encoding: [0xc4,0xc1,0x31,0x61,0xf0]
+; NOVL-NEXT:    # xmm6 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; NOVL-NEXT:    vpunpckldq %xmm5, %xmm6, %xmm5 # encoding: [0xc5,0xc9,0x62,0xed]
+; NOVL-NEXT:    # xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; NOVL-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm4 # encoding: [0xc4,0xe3,0x55,0x38,0xe4,0x01]
+; NOVL-NEXT:    vpsrlq $48, %xmm2, %xmm5 # encoding: [0xc5,0xd1,0x73,0xd2,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm3, %xmm6 # encoding: [0xc5,0xc9,0x73,0xd3,0x30]
+; NOVL-NEXT:    vminsh %xmm5, %xmm6, %xmm7 # encoding: [0x62,0xf5,0x4e,0x08,0x5d,0xfd]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfe]
+; NOVL-NEXT:    vmovshdup %xmm2, %xmm5 # encoding: [0xc5,0xfa,0x16,0xea]
+; NOVL-NEXT:    # xmm5 = xmm2[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm3, %xmm6 # encoding: [0xc5,0xfa,0x16,0xf3]
+; NOVL-NEXT:    # xmm6 = xmm3[1,1,3,3]
+; NOVL-NEXT:    vminsh %xmm5, %xmm6, %xmm8 # encoding: [0x62,0x75,0x4e,0x08,0x5d,0xc5]
+; NOVL-NEXT:    vcmpunordsh %xmm5, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcd,0x03]
+; NOVL-NEXT:    vmovsh %xmm6, %xmm0, %xmm8 {%k1} # encoding: [0x62,0x75,0x7e,0x09,0x10,0xc6]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm8, %xmm5 # encoding: [0xc5,0xb9,0x61,0xef]
+; NOVL-NEXT:    # xmm5 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm6 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xf2]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf3]
+; NOVL-NEXT:    vpsrld $16, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x72,0xd2,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm3, %xmm3 # encoding: [0xc5,0xe1,0x72,0xd3,0x10]
+; NOVL-NEXT:    vminsh %xmm2, %xmm3, %xmm7 # encoding: [0x62,0xf5,0x66,0x08,0x5d,0xfa]
+; NOVL-NEXT:    vcmpunordsh %xmm2, %xmm2, %k1 # encoding: [0x62,0xf3,0x6e,0x08,0xc2,0xca,0x03]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfb]
+; NOVL-NEXT:    vpunpcklwd %xmm7, %xmm6, %xmm2 # encoding: [0xc5,0xc9,0x61,0xd7]
+; NOVL-NEXT:    # xmm2 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; NOVL-NEXT:    vpunpckldq %xmm5, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x62,0xd5]
+; NOVL-NEXT:    # xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd0,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm5 # encoding: [0xc5,0xd1,0x73,0xd1,0x30]
+; NOVL-NEXT:    vminsh %xmm3, %xmm5, %xmm6 # encoding: [0x62,0xf5,0x56,0x08,0x5d,0xf3]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf5]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm3 # encoding: [0xc5,0xfa,0x16,0xd8]
+; NOVL-NEXT:    # xmm3 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm5 # encoding: [0xc5,0xfa,0x16,0xe9]
+; NOVL-NEXT:    # xmm5 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vminsh %xmm3, %xmm5, %xmm7 # encoding: [0x62,0xf5,0x56,0x08,0x5d,0xfb]
+; NOVL-NEXT:    vcmpunordsh %xmm3, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xcb,0x03]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm7 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xfd]
+; NOVL-NEXT:    vpunpcklwd %xmm6, %xmm7, %xmm3 # encoding: [0xc5,0xc1,0x61,0xde]
+; NOVL-NEXT:    # xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm5 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xe8]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm5 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe9]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x72,0xd1,0x10]
+; NOVL-NEXT:    vminsh %xmm0, %xmm1, %xmm6 # encoding: [0x62,0xf5,0x76,0x08,0x5d,0xf0]
+; NOVL-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc8,0x03]
+; NOVL-NEXT:    vmovsh %xmm1, %xmm0, %xmm6 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xf1]
+; NOVL-NEXT:    vpunpcklwd %xmm6, %xmm5, %xmm0 # encoding: [0xc5,0xd1,0x61,0xc6]
+; NOVL-NEXT:    # xmm0 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x38,0xc2,0x01]
+; NOVL-NEXT:    vpunpcklqdq %ymm4, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x6c,0xc4]
+; NOVL-NEXT:    # ymm0 = ymm0[0],ymm4[0],ymm0[2],ymm4[2]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %z = call <16 x half> @llvm.minnum.v16f16(<16 x half> %x, <16 x half> %y) readnone
   ret <16 x half> %z
 }
@@ -81,10 +418,68 @@ define <32 x half> @test_intrinsic_fmin_v32f16(<32 x half> %x, <32 x half> %y) {
 }
 
 define <4 x half> @minnum_intrinsic_nnan_fmf_f432(<4 x half> %a, <4 x half> %b) {
-; CHECK-LABEL: minnum_intrinsic_nnan_fmf_f432:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vminph %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7c,0x08,0x5d,0xc1]
-; CHECK-NEXT:    retq # encoding: [0xc3]
+; HasVL-LABEL: minnum_intrinsic_nnan_fmf_f432:
+; HasVL:       # %bb.0:
+; HasVL-NEXT:    vminph %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7c,0x08,0x5d,0xc1]
+; HasVL-NEXT:    retq # encoding: [0xc3]
+;
+; NOVL-LABEL: minnum_intrinsic_nnan_fmf_f432:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrldq $14, %xmm1, %xmm2 # encoding: [0xc5,0xe9,0x73,0xd9,0x0e]
+; NOVL-NEXT:    # xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $14, %xmm0, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd8,0x0e]
+; NOVL-NEXT:    # xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vcmpltsh %xmm2, %xmm3, %k1 # encoding: [0x62,0xf3,0x66,0x08,0xc2,0xca,0x01]
+; NOVL-NEXT:    vmovsh %xmm3, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xd3]
+; NOVL-NEXT:    vpshufd $255, %xmm1, %xmm3 # encoding: [0xc5,0xf9,0x70,0xd9,0xff]
+; NOVL-NEXT:    # xmm3 = xmm1[3,3,3,3]
+; NOVL-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm4 # encoding: [0xc5,0xf8,0xc6,0xe0,0xff]
+; NOVL-NEXT:    # xmm4 = xmm0[3,3,3,3]
+; NOVL-NEXT:    vcmpltsh %xmm3, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcb,0x01]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm3 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xdc]
+; NOVL-NEXT:    vpunpcklwd %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x61,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; NOVL-NEXT:    vpsrldq $10, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd9,0x0a]
+; NOVL-NEXT:    # xmm3 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vpsrldq $10, %xmm0, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd8,0x0a]
+; NOVL-NEXT:    # xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; NOVL-NEXT:    vcmpltsh %xmm3, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcb,0x01]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm3 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xdc]
+; NOVL-NEXT:    vshufpd $1, %xmm1, %xmm1, %xmm4 # encoding: [0xc5,0xf1,0xc6,0xe1,0x01]
+; NOVL-NEXT:    # xmm4 = xmm1[1,0]
+; NOVL-NEXT:    vshufpd $1, %xmm0, %xmm0, %xmm5 # encoding: [0xc5,0xf9,0xc6,0xe8,0x01]
+; NOVL-NEXT:    # xmm5 = xmm0[1,0]
+; NOVL-NEXT:    vcmpltsh %xmm4, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcc,0x01]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe5]
+; NOVL-NEXT:    vpunpcklwd %xmm3, %xmm4, %xmm3 # encoding: [0xc5,0xd9,0x61,0xdb]
+; NOVL-NEXT:    # xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; NOVL-NEXT:    vpunpckldq %xmm2, %xmm3, %xmm2 # encoding: [0xc5,0xe1,0x62,0xd2]
+; NOVL-NEXT:    # xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm3 # encoding: [0xc5,0xe1,0x73,0xd1,0x30]
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm4 # encoding: [0xc5,0xd9,0x73,0xd0,0x30]
+; NOVL-NEXT:    vcmpltsh %xmm3, %xmm4, %k1 # encoding: [0x62,0xf3,0x5e,0x08,0xc2,0xcb,0x01]
+; NOVL-NEXT:    vmovsh %xmm4, %xmm0, %xmm3 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xdc]
+; NOVL-NEXT:    vmovshdup %xmm1, %xmm4 # encoding: [0xc5,0xfa,0x16,0xe1]
+; NOVL-NEXT:    # xmm4 = xmm1[1,1,3,3]
+; NOVL-NEXT:    vmovshdup %xmm0, %xmm5 # encoding: [0xc5,0xfa,0x16,0xe8]
+; NOVL-NEXT:    # xmm5 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcmpltsh %xmm4, %xmm5, %k1 # encoding: [0x62,0xf3,0x56,0x08,0xc2,0xcc,0x01]
+; NOVL-NEXT:    vmovsh %xmm5, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe5]
+; NOVL-NEXT:    vpunpcklwd %xmm3, %xmm4, %xmm3 # encoding: [0xc5,0xd9,0x61,0xdb]
+; NOVL-NEXT:    # xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; NOVL-NEXT:    vcmpltsh %xmm1, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xc9,0x01]
+; NOVL-NEXT:    vpsrld $16, %xmm1, %xmm4 # encoding: [0xc5,0xd9,0x72,0xd1,0x10]
+; NOVL-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xc8]
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x72,0xd0,0x10]
+; NOVL-NEXT:    vcmpltsh %xmm4, %xmm0, %k1 # encoding: [0x62,0xf3,0x7e,0x08,0xc2,0xcc,0x01]
+; NOVL-NEXT:    vmovsh %xmm0, %xmm0, %xmm4 {%k1} # encoding: [0x62,0xf5,0x7e,0x09,0x10,0xe0]
+; NOVL-NEXT:    vpunpcklwd %xmm4, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0x61,0xc4]
+; NOVL-NEXT:    # xmm0 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; NOVL-NEXT:    vpunpckldq %xmm3, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x62,0xc3]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; NOVL-NEXT:    vpunpcklqdq %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x6c,0xc2]
+; NOVL-NEXT:    # xmm0 = xmm0[0],xmm2[0]
+; NOVL-NEXT:    retq # encoding: [0xc3]
   %r = tail call nnan <4 x half> @llvm.minnum.v4f16(<4 x half> %a, <4 x half> %b)
   ret <4 x half> %r
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-mov.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-mov.ll
index 82efaffe4014..526511c85045 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-mov.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-mov.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK,X64
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK,X64,X64VL
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=CHECK,X64,X64-NOVL
 
 define <8 x half> @broadcastph128(ptr %x) {
 ; X64-LABEL: broadcastph128:
@@ -314,31 +315,47 @@ define <8 x half> @test14(half %x) {
 }
 
 define <16 x half> @test14b(half %x) {
-; X64-LABEL: test14b:
-; X64:       # %bb.0:
-; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
-; X64-NEXT:    retq
+; X64VL-LABEL: test14b:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64VL-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: test14b:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: test14b:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
+; X64-NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; X64-NOVL-NEXT:    retq
    %res = insertelement <16 x half>zeroinitializer, half %x, i32 0
    ret <16 x half>%res
 }
 
 define <32 x half> @test14c(half %x) {
-; X64-LABEL: test14c:
-; X64:       # %bb.0:
-; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
-; X64-NEXT:    retq
+; X64VL-LABEL: test14c:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64VL-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: test14c:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: test14c:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
+; X64-NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; X64-NOVL-NEXT:    retq
    %res = insertelement <32 x half>zeroinitializer, half %x, i32 0
    ret <32 x half>%res
 }
@@ -578,13 +595,13 @@ declare void @llvm.masked.store.v32f16.p0(<32 x half>, ptr, i32, <32 x i1>)
 declare <32 x half> @llvm.masked.load.v32f16.p0(ptr, i32,  <32 x i1>, <32 x half>)
 
 define void @storeu32f16mask(<32 x i1> %mask, ptr %addr, <32 x half> %val) {
-; X64-LABEL: storeu32f16mask:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %ymm0, %ymm0
-; X64-NEXT:    vpmovb2m %ymm0, %k1
-; X64-NEXT:    vmovdqu16 %zmm1, (%rdi) {%k1}
-; X64-NEXT:    vzeroupper
-; X64-NEXT:    retq
+; X64VL-LABEL: storeu32f16mask:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; X64VL-NEXT:    vpmovb2m %ymm0, %k1
+; X64VL-NEXT:    vmovdqu16 %zmm1, (%rdi) {%k1}
+; X64VL-NEXT:    vzeroupper
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: storeu32f16mask:
 ; X86:       # %bb.0:
@@ -594,17 +611,25 @@ define void @storeu32f16mask(<32 x i1> %mask, ptr %addr, <32 x half> %val) {
 ; X86-NEXT:    vmovdqu16 %zmm1, (%eax) {%k1}
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: storeu32f16mask:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; X64-NOVL-NEXT:    vpmovb2m %zmm0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm1, (%rdi) {%k1}
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   call void @llvm.masked.store.v32f16.p0(<32 x half> %val, ptr %addr, i32 4, <32 x i1>%mask)
   ret void
 }
 
 define <32 x half> @maskloadu32f16(ptr %addr, <32 x half> %val, <32 x i1> %mask) {
-; X64-LABEL: maskloadu32f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %ymm1, %ymm1
-; X64-NEXT:    vpmovb2m %ymm1, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskloadu32f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %ymm1, %ymm1
+; X64VL-NEXT:    vpmovb2m %ymm1, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskloadu32f16:
 ; X86:       # %bb.0:
@@ -613,17 +638,24 @@ define <32 x half> @maskloadu32f16(ptr %addr, <32 x half> %val, <32 x i1> %mask)
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %zmm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskloadu32f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $7, %ymm1, %ymm1
+; X64-NOVL-NEXT:    vpmovb2m %zmm1, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1}
+; X64-NOVL-NEXT:    retq
   %res = call <32 x half> @llvm.masked.load.v32f16.p0(ptr %addr, i32 4, <32 x i1> %mask, <32 x half> %val)
   ret <32 x half> %res
 }
 
 define <32 x half> @maskuloadu32f16(ptr %addr, <32 x i1> %mask) {
-; X64-LABEL: maskuloadu32f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %ymm0, %ymm0
-; X64-NEXT:    vpmovb2m %ymm0, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskuloadu32f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; X64VL-NEXT:    vpmovb2m %ymm0, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskuloadu32f16:
 ; X86:       # %bb.0:
@@ -632,17 +664,24 @@ define <32 x half> @maskuloadu32f16(ptr %addr, <32 x i1> %mask) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskuloadu32f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; X64-NOVL-NEXT:    vpmovb2m %zmm0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    retq
   %res = call <32 x half> @llvm.masked.load.v32f16.p0(ptr %addr, i32 4, <32 x i1> %mask, <32 x half> undef)
   ret <32 x half> %res
 }
 
 define <32 x half> @maskzloadu32f16(ptr %addr, <32 x i1> %mask) {
-; X64-LABEL: maskzloadu32f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %ymm0, %ymm0
-; X64-NEXT:    vpmovb2m %ymm0, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskzloadu32f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; X64VL-NEXT:    vpmovb2m %ymm0, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskzloadu32f16:
 ; X86:       # %bb.0:
@@ -651,6 +690,13 @@ define <32 x half> @maskzloadu32f16(ptr %addr, <32 x i1> %mask) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskzloadu32f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; X64-NOVL-NEXT:    vpmovb2m %zmm0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    retq
   %res = call <32 x half> @llvm.masked.load.v32f16.p0(ptr %addr, i32 4, <32 x i1> %mask, <32 x half> zeroinitializer)
   ret <32 x half> %res
 }
@@ -713,11 +759,11 @@ define <16 x half> @load16f16(ptr %a) {
 }
 
 define <16 x half> @load16f16mask(ptr %a, <16 x half> %b, i16 %c) {
-; X64-LABEL: load16f16mask:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: load16f16mask:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: load16f16mask:
 ; X86:       # %bb.0:
@@ -725,6 +771,15 @@ define <16 x half> @load16f16mask(ptr %a, <16 x half> %b, i16 %c) {
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %ymm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: load16f16mask:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqa (%rdi), %ymm1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i16 %c to <16 x i1>
   %res0 = load <16 x half>, ptr %a
   %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %b
@@ -732,11 +787,11 @@ define <16 x half> @load16f16mask(ptr %a, <16 x half> %b, i16 %c) {
 }
 
 define <16 x half> @load16f16maskz(ptr %a, i16 %c) {
-; X64-LABEL: load16f16maskz:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: load16f16maskz:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: load16f16maskz:
 ; X86:       # %bb.0:
@@ -744,6 +799,14 @@ define <16 x half> @load16f16maskz(ptr %a, i16 %c) {
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: load16f16maskz:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqa (%rdi), %ymm0
+; X64-NOVL-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i16 %c to <16 x i1>
   %res0 = load <16 x half>, ptr %a
   %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer
@@ -766,11 +829,11 @@ define <16 x half> @loadu16f16(ptr %a) {
 }
 
 define <16 x half> @loadu16f16mask(ptr %a, <16 x half> %b, i16 %c) {
-; X64-LABEL: loadu16f16mask:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: loadu16f16mask:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: loadu16f16mask:
 ; X86:       # %bb.0:
@@ -778,6 +841,15 @@ define <16 x half> @loadu16f16mask(ptr %a, <16 x half> %b, i16 %c) {
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %ymm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: loadu16f16mask:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqu (%rdi), %ymm1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i16 %c to <16 x i1>
   %res0 = load <16 x half>, ptr %a, align 8
   %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %b
@@ -785,11 +857,11 @@ define <16 x half> @loadu16f16mask(ptr %a, <16 x half> %b, i16 %c) {
 }
 
 define <16 x half> @loadu16f16maskz(ptr %a, i16 %c) {
-; X64-LABEL: loadu16f16maskz:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: loadu16f16maskz:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: loadu16f16maskz:
 ; X86:       # %bb.0:
@@ -797,6 +869,14 @@ define <16 x half> @loadu16f16maskz(ptr %a, i16 %c) {
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: loadu16f16maskz:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqu (%rdi), %ymm0
+; X64-NOVL-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i16 %c to <16 x i1>
   %res0 = load <16 x half>, ptr %a, align 8
   %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer
@@ -841,13 +921,13 @@ declare void @llvm.masked.store.v16f16.p0(<16 x half>, ptr, i32, <16 x i1>)
 declare <16 x half> @llvm.masked.load.v16f16.p0(ptr, i32,  <16 x i1>, <16 x half>)
 
 define void @storeu16f16mask(<16 x i1> %mask, ptr %addr, <16 x half> %val) {
-; X64-LABEL: storeu16f16mask:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %xmm0, %xmm0
-; X64-NEXT:    vpmovb2m %xmm0, %k1
-; X64-NEXT:    vmovdqu16 %ymm1, (%rdi) {%k1}
-; X64-NEXT:    vzeroupper
-; X64-NEXT:    retq
+; X64VL-LABEL: storeu16f16mask:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; X64VL-NEXT:    vpmovb2m %xmm0, %k1
+; X64VL-NEXT:    vmovdqu16 %ymm1, (%rdi) {%k1}
+; X64VL-NEXT:    vzeroupper
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: storeu16f16mask:
 ; X86:       # %bb.0:
@@ -857,17 +937,27 @@ define void @storeu16f16mask(<16 x i1> %mask, ptr %addr, <16 x half> %val) {
 ; X86-NEXT:    vmovdqu16 %ymm1, (%eax) {%k1}
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: storeu16f16mask:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; X64-NOVL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vpmovb2m %zmm0, %k0
+; X64-NOVL-NEXT:    kmovw %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm1, (%rdi) {%k1}
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   call void @llvm.masked.store.v16f16.p0(<16 x half> %val, ptr %addr, i32 4, <16 x i1>%mask)
   ret void
 }
 
 define <16 x half> @maskloadu16f16(ptr %addr, <16 x half> %val, <16 x i1> %mask) {
-; X64-LABEL: maskloadu16f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %xmm1, %xmm1
-; X64-NEXT:    vpmovb2m %xmm1, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskloadu16f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %xmm1, %xmm1
+; X64VL-NEXT:    vpmovb2m %xmm1, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskloadu16f16:
 ; X86:       # %bb.0:
@@ -876,17 +966,27 @@ define <16 x half> @maskloadu16f16(ptr %addr, <16 x half> %val, <16 x i1> %mask)
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %ymm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskloadu16f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X64-NOVL-NEXT:    vpsllw $7, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vpmovb2m %zmm1, %k0
+; X64-NOVL-NEXT:    kmovw %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %res = call <16 x half> @llvm.masked.load.v16f16.p0(ptr %addr, i32 4, <16 x i1> %mask, <16 x half> %val)
   ret <16 x half> %res
 }
 
 define <16 x half> @maskuloadu16f16(ptr %addr, <16 x i1> %mask) {
-; X64-LABEL: maskuloadu16f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %xmm0, %xmm0
-; X64-NEXT:    vpmovb2m %xmm0, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskuloadu16f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; X64VL-NEXT:    vpmovb2m %xmm0, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskuloadu16f16:
 ; X86:       # %bb.0:
@@ -895,17 +995,26 @@ define <16 x half> @maskuloadu16f16(ptr %addr, <16 x i1> %mask) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskuloadu16f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vpmovb2m %zmm0, %k0
+; X64-NOVL-NEXT:    kmovw %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %res = call <16 x half> @llvm.masked.load.v16f16.p0(ptr %addr, i32 4, <16 x i1> %mask, <16 x half> undef)
   ret <16 x half> %res
 }
 
 define <16 x half> @maskzloadu16f16(ptr %addr, <16 x i1> %mask) {
-; X64-LABEL: maskzloadu16f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $7, %xmm0, %xmm0
-; X64-NEXT:    vpmovb2m %xmm0, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskzloadu16f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; X64VL-NEXT:    vpmovb2m %xmm0, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %ymm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskzloadu16f16:
 ; X86:       # %bb.0:
@@ -914,6 +1023,15 @@ define <16 x half> @maskzloadu16f16(ptr %addr, <16 x i1> %mask) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskzloadu16f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vpmovb2m %zmm0, %k0
+; X64-NOVL-NEXT:    kmovw %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %res = call <16 x half> @llvm.masked.load.v16f16.p0(ptr %addr, i32 4, <16 x i1> %mask, <16 x half> zeroinitializer)
   ret <16 x half> %res
 }
@@ -927,34 +1045,51 @@ define <16 x half> @movrr16f16(<16 x half> %a, <16 x half> %b) {
 }
 
 define <16 x half> @movrrk16f16(<16 x half> %a, <16 x half> %b, i16 %msk) {
-; X64-LABEL: movrrk16f16:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: movrrk16f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %edi, %k1
+; X64VL-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: movrrk16f16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: movrrk16f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %edi, %k1
+; X64-NOVL-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %mask = bitcast i16 %msk to <16 x i1>
   %res = select <16 x i1> %mask, <16 x half> %a, <16 x half> %b
   ret <16 x half> %res
 }
 
 define <16 x half> @movrrkz16f16(<16 x half> %a, i16 %msk) {
-; X64-LABEL: movrrkz16f16:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: movrrkz16f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %edi, %k1
+; X64VL-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: movrrkz16f16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: movrrkz16f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %edi, %k1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; X64-NOVL-NEXT:    retq
   %mask = bitcast i16 %msk to <16 x i1>
   %res = select <16 x i1> %mask, <16 x half> %a, <16 x half> zeroinitializer
   ret <16 x half> %res
@@ -976,11 +1111,11 @@ define <8 x half> @load8f16(ptr %a) {
 }
 
 define <8 x half> @load8f16mask(ptr %a, <8 x half> %b, i8 %c) {
-; X64-LABEL: load8f16mask:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: load8f16mask:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: load8f16mask:
 ; X86:       # %bb.0:
@@ -989,6 +1124,16 @@ define <8 x half> @load8f16mask(ptr %a, <8 x half> %b, i8 %c) {
 ; X86-NEXT:    kmovd %ecx, %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %xmm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: load8f16mask:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqa (%rdi), %xmm1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i8 %c to <8 x i1>
   %res0 = load <8 x half>, ptr %a
   %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %b
@@ -996,11 +1141,11 @@ define <8 x half> @load8f16mask(ptr %a, <8 x half> %b, i8 %c) {
 }
 
 define <8 x half> @load8f16maskz(ptr %a, i8 %c) {
-; X64-LABEL: load8f16maskz:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: load8f16maskz:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: load8f16maskz:
 ; X86:       # %bb.0:
@@ -1009,6 +1154,15 @@ define <8 x half> @load8f16maskz(ptr %a, i8 %c) {
 ; X86-NEXT:    kmovd %ecx, %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: load8f16maskz:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqa (%rdi), %xmm0
+; X64-NOVL-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i8 %c to <8 x i1>
   %res0 = load <8 x half>, ptr %a
   %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer
@@ -1031,11 +1185,11 @@ define <8 x half> @loadu8f16(ptr %a) {
 }
 
 define <8 x half> @loadu8f16mask(ptr %a, <8 x half> %b, i8 %c) {
-; X64-LABEL: loadu8f16mask:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: loadu8f16mask:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: loadu8f16mask:
 ; X86:       # %bb.0:
@@ -1044,6 +1198,16 @@ define <8 x half> @loadu8f16mask(ptr %a, <8 x half> %b, i8 %c) {
 ; X86-NEXT:    kmovd %ecx, %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %xmm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: loadu8f16mask:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqu (%rdi), %xmm1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i8 %c to <8 x i1>
   %res0 = load <8 x half>, ptr %a, align 8
   %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %b
@@ -1051,11 +1215,11 @@ define <8 x half> @loadu8f16mask(ptr %a, <8 x half> %b, i8 %c) {
 }
 
 define <8 x half> @loadu8f16maskz(ptr %a, i8 %c) {
-; X64-LABEL: loadu8f16maskz:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: loadu8f16maskz:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %esi, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: loadu8f16maskz:
 ; X86:       # %bb.0:
@@ -1064,6 +1228,15 @@ define <8 x half> @loadu8f16maskz(ptr %a, i8 %c) {
 ; X86-NEXT:    kmovd %ecx, %k1
 ; X86-NEXT:    vmovdqu16 (%eax), %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: loadu8f16maskz:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    kmovd %esi, %k1
+; X64-NOVL-NEXT:    vmovdqu (%rdi), %xmm0
+; X64-NOVL-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %msk = bitcast i8 %c to <8 x i1>
   %res0 = load <8 x half>, ptr %a, align 8
   %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer
@@ -1104,12 +1277,12 @@ declare void @llvm.masked.store.v8f16.p0(<8 x half>, ptr, i32, <8 x i1>)
 declare <8 x half> @llvm.masked.load.v8f16.p0(ptr, i32,  <8 x i1>, <8 x half>)
 
 define void @storeu8f16mask(<8 x i1> %mask, ptr %addr, <8 x half> %val) {
-; X64-LABEL: storeu8f16mask:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $15, %xmm0, %xmm0
-; X64-NEXT:    vpmovw2m %xmm0, %k1
-; X64-NEXT:    vmovdqu16 %xmm1, (%rdi) {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: storeu8f16mask:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $15, %xmm0, %xmm0
+; X64VL-NEXT:    vpmovw2m %xmm0, %k1
+; X64VL-NEXT:    vmovdqu16 %xmm1, (%rdi) {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: storeu8f16mask:
 ; X86:       # %bb.0:
@@ -1118,17 +1291,28 @@ define void @storeu8f16mask(<8 x i1> %mask, ptr %addr, <8 x half> %val) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 %xmm1, (%eax) {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: storeu8f16mask:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; X64-NOVL-NEXT:    vpsllw $15, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vpmovw2m %zmm0, %k0
+; X64-NOVL-NEXT:    kshiftld $24, %k0, %k0
+; X64-NOVL-NEXT:    kshiftrd $24, %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm1, (%rdi) {%k1}
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   call void @llvm.masked.store.v8f16.p0(<8 x half> %val, ptr %addr, i32 4, <8 x i1>%mask)
   ret void
 }
 
 define <8 x half> @maskloadu8f16(ptr %addr, <8 x half> %val, <8 x i1> %mask) {
-; X64-LABEL: maskloadu8f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $15, %xmm1, %xmm1
-; X64-NEXT:    vpmovw2m %xmm1, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskloadu8f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $15, %xmm1, %xmm1
+; X64VL-NEXT:    vpmovw2m %xmm1, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskloadu8f16:
 ; X86:       # %bb.0:
@@ -1137,17 +1321,29 @@ define <8 x half> @maskloadu8f16(ptr %addr, <8 x half> %val, <8 x i1> %mask) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %xmm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskloadu8f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; X64-NOVL-NEXT:    vpsllw $15, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vpmovw2m %zmm1, %k0
+; X64-NOVL-NEXT:    kshiftld $24, %k0, %k0
+; X64-NOVL-NEXT:    kshiftrd $24, %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %res = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %addr, i32 4, <8 x i1> %mask, <8 x half> %val)
   ret <8 x half> %res
 }
 
 define <8 x half> @maskuloadu8f16(ptr %addr, <8 x i1> %mask) {
-; X64-LABEL: maskuloadu8f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $15, %xmm0, %xmm0
-; X64-NEXT:    vpmovw2m %xmm0, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskuloadu8f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $15, %xmm0, %xmm0
+; X64VL-NEXT:    vpmovw2m %xmm0, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskuloadu8f16:
 ; X86:       # %bb.0:
@@ -1156,17 +1352,28 @@ define <8 x half> @maskuloadu8f16(ptr %addr, <8 x i1> %mask) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskuloadu8f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $15, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vpmovw2m %zmm0, %k0
+; X64-NOVL-NEXT:    kshiftld $24, %k0, %k0
+; X64-NOVL-NEXT:    kshiftrd $24, %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %res = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %addr, i32 4, <8 x i1> %mask, <8 x half> undef)
   ret <8 x half> %res
 }
 
 define <8 x half> @maskzloadu8f16(ptr %addr, <8 x i1> %mask) {
-; X64-LABEL: maskzloadu8f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $15, %xmm0, %xmm0
-; X64-NEXT:    vpmovw2m %xmm0, %k1
-; X64-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: maskzloadu8f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpsllw $15, %xmm0, %xmm0
+; X64VL-NEXT:    vpmovw2m %xmm0, %k1
+; X64VL-NEXT:    vmovdqu16 (%rdi), %xmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: maskzloadu8f16:
 ; X86:       # %bb.0:
@@ -1175,6 +1382,17 @@ define <8 x half> @maskzloadu8f16(ptr %addr, <8 x i1> %mask) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovdqu16 (%eax), %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: maskzloadu8f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpsllw $15, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vpmovw2m %zmm0, %k0
+; X64-NOVL-NEXT:    kshiftld $24, %k0, %k0
+; X64-NOVL-NEXT:    kshiftrd $24, %k0, %k1
+; X64-NOVL-NEXT:    vmovdqu16 (%rdi), %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %res = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %addr, i32 4, <8 x i1> %mask, <8 x half> zeroinitializer)
   ret <8 x half> %res
 }
@@ -1188,11 +1406,11 @@ define <8 x half> @movrr8f16(<8 x half> %a, <8 x half> %b) {
 }
 
 define <8 x half> @movrrk8f16(<8 x half> %a, <8 x half> %b, i8 %msk) {
-; X64-LABEL: movrrk8f16:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
-; X64-NEXT:    retq
+; X64VL-LABEL: movrrk8f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %edi, %k1
+; X64VL-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: movrrk8f16:
 ; X86:       # %bb.0:
@@ -1200,17 +1418,27 @@ define <8 x half> @movrrk8f16(<8 x half> %a, <8 x half> %b, i8 %msk) {
 ; X86-NEXT:    kmovd %eax, %k1
 ; X86-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: movrrk8f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %edi, %k1
+; X64-NOVL-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %mask = bitcast i8 %msk to <8 x i1>
   %res = select <8 x i1> %mask, <8 x half> %a, <8 x half> %b
   ret <8 x half> %res
 }
 
 define <8 x half> @movrrkz8f16(<8 x half> %a, i8 %msk) {
-; X64-LABEL: movrrkz8f16:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: movrrkz8f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    kmovd %edi, %k1
+; X64VL-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: movrrkz8f16:
 ; X86:       # %bb.0:
@@ -1218,18 +1446,69 @@ define <8 x half> @movrrkz8f16(<8 x half> %a, i8 %msk) {
 ; X86-NEXT:    kmovd %eax, %k1
 ; X86-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: movrrkz8f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; X64-NOVL-NEXT:    kmovd %edi, %k1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
   %mask = bitcast i8 %msk to <8 x i1>
   %res = select <8 x i1> %mask, <8 x half> %a, <8 x half> zeroinitializer
   ret <8 x half> %res
 }
 
 define <8 x half> @movsh(<8 x half> %a, <8 x half> %b) {
-; CHECK-LABEL: movsh:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]
-; CHECK-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
-; CHECK-NEXT:    vaddph %xmm0, %xmm2, %xmm0
-; CHECK-NEXT:    ret{{[l|q]}}
+; X64VL-LABEL: movsh:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]
+; X64VL-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
+; X64VL-NEXT:    vaddph %xmm0, %xmm2, %xmm0
+; X64VL-NEXT:    retq
+;
+; X86-LABEL: movsh:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]
+; X86-NEXT:    vmovsh %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vaddph %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: movsh:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]
+; X64-NOVL-NEXT:    vmovsh %xmm0, %xmm1, %xmm3
+; X64-NOVL-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-NOVL-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-NOVL-NEXT:    vaddsh %xmm4, %xmm5, %xmm4
+; X64-NOVL-NEXT:    vshufps {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; X64-NOVL-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[3,3,3,3]
+; X64-NOVL-NEXT:    vaddsh %xmm5, %xmm6, %xmm5
+; X64-NOVL-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X64-NOVL-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm3[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-NOVL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
+; X64-NOVL-NEXT:    vaddsh %xmm5, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vshufpd {{.*#+}} xmm5 = xmm3[1,0]
+; X64-NOVL-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; X64-NOVL-NEXT:    vaddsh %xmm5, %xmm6, %xmm5
+; X64-NOVL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; X64-NOVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; X64-NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm4
+; X64-NOVL-NEXT:    vpsrlq $48, %xmm2, %xmm5
+; X64-NOVL-NEXT:    vaddsh %xmm4, %xmm5, %xmm4
+; X64-NOVL-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm3[1,1,3,3]
+; X64-NOVL-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; X64-NOVL-NEXT:    vaddsh %xmm5, %xmm6, %xmm5
+; X64-NOVL-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; X64-NOVL-NEXT:    vaddsh %xmm3, %xmm2, %xmm3
+; X64-NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vpsrld $16, %xmm2, %xmm2
+; X64-NOVL-NEXT:    vaddsh %xmm1, %xmm2, %xmm1
+; X64-NOVL-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X64-NOVL-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; X64-NOVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; X64-NOVL-NEXT:    retq
   %res1 = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 0, i32 7, i32 0, i32 1, i32 2, i32 3, i32 7, i32 5>
   %res2 = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %res = fadd <8 x half> %res1, %res2
@@ -1939,14 +2218,14 @@ define <8 x half> @regression1(<8 x half> %a, <8 x half> %b) {
 }
 
 define <4 x float> @regression2(ptr addrspace(1) %0, <4 x i32> %1, <4 x i32> %2, <4 x float> %3, ptr %4) {
-; X64-LABEL: regression2:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovw (%rsi), %xmm0
-; X64-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X64-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X64-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
-; X64-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
-; X64-NEXT:    retq
+; X64VL-LABEL: regression2:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vmovw (%rsi), %xmm0
+; X64VL-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64VL-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X64VL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
+; X64VL-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: regression2:
 ; X86:       # %bb.0:
@@ -1957,6 +2236,16 @@ define <4 x float> @regression2(ptr addrspace(1) %0, <4 x i32> %1, <4 x i32> %2,
 ; X86-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
 ; X86-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: regression2:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vmovw (%rsi), %xmm0
+; X64-NOVL-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-NOVL-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X64-NOVL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
+; X64-NOVL-NEXT:    vbroadcastss {{.*#+}} xmm1 = [3.92156886E-3,3.92156886E-3,3.92156886E-3,3.92156886E-3]
+; X64-NOVL-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; X64-NOVL-NEXT:    retq
   %6 = load i8, ptr %4, align 1
   %7 = getelementptr i8, ptr %4, i64 1
   %8 = addrspacecast ptr %7 to ptr addrspace(4)
@@ -1973,13 +2262,13 @@ define <4 x float> @regression2(ptr addrspace(1) %0, <4 x i32> %1, <4 x i32> %2,
 ; Make sure load/stores of v4f16 are handled well on 32-bit targets where
 ; default widening legalization can't use i64.
 define void @load_store_v4f16(ptr %x, ptr %y, ptr %z) {
-; X64-LABEL: load_store_v4f16:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; X64-NEXT:    vaddph %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vmovlps %xmm0, (%rdx)
-; X64-NEXT:    retq
+; X64VL-LABEL: load_store_v4f16:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64VL-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64VL-NEXT:    vaddph %xmm1, %xmm0, %xmm0
+; X64VL-NEXT:    vmovlps %xmm0, (%rdx)
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: load_store_v4f16:
 ; X86:       # %bb.0:
@@ -1991,6 +2280,26 @@ define void @load_store_v4f16(ptr %x, ptr %y, ptr %z) {
 ; X86-NEXT:    vaddph %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovlps %xmm0, (%eax)
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: load_store_v4f16:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; X64-NOVL-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-NOVL-NEXT:    vpsrlq $48, %xmm1, %xmm2
+; X64-NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm3
+; X64-NOVL-NEXT:    vaddsh %xmm2, %xmm3, %xmm2
+; X64-NOVL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; X64-NOVL-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; X64-NOVL-NEXT:    vaddsh %xmm3, %xmm4, %xmm3
+; X64-NOVL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-NOVL-NEXT:    vaddsh %xmm1, %xmm0, %xmm3
+; X64-NOVL-NEXT:    vpsrld $16, %xmm1, %xmm1
+; X64-NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vaddsh %xmm1, %xmm0, %xmm0
+; X64-NOVL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; X64-NOVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X64-NOVL-NEXT:    vmovq %xmm0, (%rdx)
+; X64-NOVL-NEXT:    retq
   %a = load <4 x half>, ptr %x
   %b = load <4 x half>, ptr %y
   %c = fadd <4 x half> %a, %b
@@ -2044,20 +2353,20 @@ define <16 x i16> @test22(ptr %mem) nounwind {
 }
 
 define void @pr52560(i8 %0, <2 x i16> %1, ptr %c) nounwind {
-; X64-LABEL: pr52560:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    movsbl %dil, %eax
-; X64-NEXT:    vmovw %eax, %xmm1
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X64-NEXT:    vpcmpgtw %xmm2, %xmm1, %k1
-; X64-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    vmovw %xmm0, %eax
-; X64-NEXT:    testw %ax, %ax
-; X64-NEXT:    je .LBB123_2
-; X64-NEXT:  # %bb.1: # %for.body.preheader
-; X64-NEXT:    movb $0, (%rsi)
-; X64-NEXT:  .LBB123_2: # %for.end
-; X64-NEXT:    retq
+; X64VL-LABEL: pr52560:
+; X64VL:       # %bb.0: # %entry
+; X64VL-NEXT:    movsbl %dil, %eax
+; X64VL-NEXT:    vmovw %eax, %xmm1
+; X64VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64VL-NEXT:    vpcmpgtw %xmm2, %xmm1, %k1
+; X64VL-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
+; X64VL-NEXT:    vmovw %xmm0, %eax
+; X64VL-NEXT:    testw %ax, %ax
+; X64VL-NEXT:    je .LBB123_2
+; X64VL-NEXT:  # %bb.1: # %for.body.preheader
+; X64VL-NEXT:    movb $0, (%rsi)
+; X64VL-NEXT:  .LBB123_2: # %for.end
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: pr52560:
 ; X86:       # %bb.0: # %entry
@@ -2074,6 +2383,23 @@ define void @pr52560(i8 %0, <2 x i16> %1, ptr %c) nounwind {
 ; X86-NEXT:    movb $0, (%eax)
 ; X86-NEXT:  .LBB123_2: # %for.end
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: pr52560:
+; X64-NOVL:       # %bb.0: # %entry
+; X64-NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; X64-NOVL-NEXT:    movsbl %dil, %eax
+; X64-NOVL-NEXT:    vmovw %eax, %xmm1
+; X64-NOVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NOVL-NEXT:    vpcmpgtw %zmm2, %zmm1, %k1
+; X64-NOVL-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z}
+; X64-NOVL-NEXT:    vmovw %xmm0, %eax
+; X64-NOVL-NEXT:    testw %ax, %ax
+; X64-NOVL-NEXT:    je .LBB123_2
+; X64-NOVL-NEXT:  # %bb.1: # %for.body.preheader
+; X64-NOVL-NEXT:    movb $0, (%rsi)
+; X64-NOVL-NEXT:  .LBB123_2: # %for.end
+; X64-NOVL-NEXT:    vzeroupper
+; X64-NOVL-NEXT:    retq
 entry:
   %conv = sext i8 %0 to i16
   %2 = insertelement <2 x i16> <i16 poison, i16 0>, i16 %conv, i32 0
@@ -2092,17 +2418,17 @@ for.end:                                          ; preds = %for.body.preheader,
 }
 
 define <16 x i32> @pr52561(<16 x i32> %a, <16 x i32> %b) "min-legal-vector-width"="256" "prefer-vector-width"="256" nounwind {
-; X64-LABEL: pr52561:
-; X64:       # %bb.0:
-; X64-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
-; X64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [112,112,112,112,112,112,112,112]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
-; X64-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
-; X64-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X64-NEXT:    vmovsh %xmm0, %xmm2, %xmm0
-; X64-NEXT:    retq
+; X64VL-LABEL: pr52561:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
+; X64VL-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; X64VL-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [112,112,112,112,112,112,112,112]
+; X64VL-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; X64VL-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; X64VL-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; X64VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64VL-NEXT:    vmovsh %xmm0, %xmm2, %xmm0
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: pr52561:
 ; X86:       # %bb.0:
@@ -2121,6 +2447,13 @@ define <16 x i32> @pr52561(<16 x i32> %a, <16 x i32> %b) "min-legal-vector-width
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: pr52561:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; X64-NOVL-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; X64-NOVL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; X64-NOVL-NEXT:    retq
   %1 = add <16 x i32> %a, <i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
   %2 = add <16 x i32> %1, %b
   %3 = and <16 x i32> %2, <i32 65535, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 65535>
@@ -2128,13 +2461,13 @@ define <16 x i32> @pr52561(<16 x i32> %a, <16 x i32> %b) "min-legal-vector-width
 }
 
 define <8 x i16> @pr59628_xmm(i16 %arg) {
-; X64-LABEL: pr59628_xmm:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovw %edi, %xmm0
-; X64-NEXT:    vpbroadcastw %edi, %xmm1
-; X64-NEXT:    vpcmpneqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %k1
-; X64-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    retq
+; X64VL-LABEL: pr59628_xmm:
+; X64VL:       # %bb.0:
+; X64VL-NEXT:    vmovw %edi, %xmm0
+; X64VL-NEXT:    vpbroadcastw %edi, %xmm1
+; X64VL-NEXT:    vpcmpneqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %k1
+; X64VL-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
+; X64VL-NEXT:    retq
 ;
 ; X86-LABEL: pr59628_xmm:
 ; X86:       # %bb.0:
@@ -2145,6 +2478,13 @@ define <8 x i16> @pr59628_xmm(i16 %arg) {
 ; X86-NEXT:    vpcmpneqw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %k1
 ; X86-NEXT:    vmovdqu16 %xmm0, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
+;
+; X64-NOVL-LABEL: pr59628_xmm:
+; X64-NOVL:       # %bb.0:
+; X64-NOVL-NEXT:    vmovw %edi, %xmm0
+; X64-NOVL-NEXT:    vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; X64-NOVL-NEXT:    vpandn %xmm0, %xmm1, %xmm0
+; X64-NOVL-NEXT:    retq
   %I1 = insertelement <8 x i16> zeroinitializer, i16 %arg, i16 0
   %I2 = insertelement <8 x i16> %I1, i16 0, i16 %arg
   ret <8 x i16> %I2
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-novl.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-novl.ll
new file mode 100644
index 000000000000..d17cacc0e1ad
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512fp16-novl.ll
@@ -0,0 +1,224 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64 -mattr=+avx512fp16 | FileCheck %s
+
+define <2 x half> @vector_sint64ToHalf(<2 x i64> %int64) {
+; CHECK-LABEL: vector_sint64ToHalf:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vcvtqq2ph %zmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %fp16 = sitofp <2 x i64> %int64 to <2 x half>
+  ret <2 x half> %fp16
+}
+
+define <4 x half> @vector_sint32ToHalf(<4 x i32> %int32) {
+; CHECK-LABEL: vector_sint32ToHalf:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextractps $3, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm1
+; CHECK-NEXT:    vextractps $2, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT:    vextractps $1, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vmovd %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; CHECK-NEXT:    retq
+  %fp16 = sitofp <4 x i32> %int32 to <4 x half>
+  ret <4 x half> %fp16
+}
+
+define <8 x half> @vector_sint16ToHalf(<8 x i16> %int16) {
+; CHECK-LABEL: vector_sint16ToHalf:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpextrw $7, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm1
+; CHECK-NEXT:    vpextrw $6, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT:    vpextrw $5, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vpextrw $4, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm3
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT:    vpextrw $3, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vpextrw $2, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm3
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT:    vpextrw $1, %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm3
+; CHECK-NEXT:    vmovw %xmm0, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+  %fp16 = sitofp <8 x i16> %int16 to <8 x half>
+  ret <8 x half> %fp16
+}
+
+define <2 x half> @vector_uint64ToHalf(<2 x i64> %int64) {
+; CHECK-LABEL: vector_uint64ToHalf:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vcvtuqq2ph %zmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %fp16 = uitofp <2 x i64> %int64 to <2 x half>
+  ret <2 x half> %fp16
+}
+
+define <4 x half> @vector_uint32ToHalf(<4 x i32> %int32) {
+; CHECK-LABEL: vector_uint32ToHalf:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vcvtudq2ph %zmm0, %ymm0
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %fp16 = uitofp <4 x i32> %int32 to <4 x half>
+  ret <4 x half> %fp16
+}
+
+define <8 x half> @vector_uint16ToHalf(<8 x i16> %int16) {
+; CHECK-LABEL: vector_uint16ToHalf:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpextrw $7, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm1
+; CHECK-NEXT:    vpextrw $6, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT:    vpextrw $5, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vpextrw $4, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm3
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT:    vpextrw $3, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm2
+; CHECK-NEXT:    vpextrw $2, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm3
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT:    vpextrw $1, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm3
+; CHECK-NEXT:    vpextrw $0, %xmm0, %eax
+; CHECK-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    retq
+  %fp16 = uitofp <8 x i16> %int16 to <8 x half>
+  ret <8 x half> %fp16
+}
+
+define <8 x half> @select(<8 x half> %x) {
+; CHECK-LABEL: select:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vmovsh {{.*#+}} xmm1 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; CHECK-NEXT:    vucomish %xmm1, %xmm0
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    kmovw %eax, %k0
+; CHECK-NEXT:    vpsrld $16, %xmm0, %xmm2
+; CHECK-NEXT:    vucomish %xmm1, %xmm2
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kshiftlw $15, %k1, %k1
+; CHECK-NEXT:    kshiftrw $14, %k1, %k1
+; CHECK-NEXT:    korw %k1, %k0, %k0
+; CHECK-NEXT:    movw $-5, %ax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kandw %k1, %k0, %k0
+; CHECK-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-NEXT:    vucomish %xmm1, %xmm2
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kshiftlw $15, %k1, %k1
+; CHECK-NEXT:    kshiftrw $13, %k1, %k1
+; CHECK-NEXT:    korw %k1, %k0, %k0
+; CHECK-NEXT:    movw $-9, %ax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kandw %k1, %k0, %k0
+; CHECK-NEXT:    vpsrlq $48, %xmm0, %xmm2
+; CHECK-NEXT:    vucomish %xmm1, %xmm2
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kshiftlw $15, %k1, %k1
+; CHECK-NEXT:    kshiftrw $12, %k1, %k1
+; CHECK-NEXT:    korw %k1, %k0, %k0
+; CHECK-NEXT:    movw $-17, %ax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kandw %k1, %k0, %k0
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm2 = xmm0[1,0]
+; CHECK-NEXT:    vucomish %xmm1, %xmm2
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kshiftlw $15, %k1, %k1
+; CHECK-NEXT:    kshiftrw $11, %k1, %k1
+; CHECK-NEXT:    korw %k1, %k0, %k0
+; CHECK-NEXT:    movw $-33, %ax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kandw %k1, %k0, %k0
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vucomish %xmm1, %xmm2
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kshiftlw $15, %k1, %k1
+; CHECK-NEXT:    kshiftrw $10, %k1, %k1
+; CHECK-NEXT:    korw %k1, %k0, %k0
+; CHECK-NEXT:    movw $-65, %ax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kandw %k1, %k0, %k0
+; CHECK-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; CHECK-NEXT:    vucomish %xmm1, %xmm2
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kshiftlw $6, %k1, %k1
+; CHECK-NEXT:    korw %k1, %k0, %k0
+; CHECK-NEXT:    kshiftlw $9, %k0, %k0
+; CHECK-NEXT:    kshiftrw $9, %k0, %k0
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vucomish %xmm1, %xmm2
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    kshiftlw $7, %k1, %k1
+; CHECK-NEXT:    korw %k1, %k0, %k1
+; CHECK-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; CHECK-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+entry:
+  %c = fcmp ogt <8 x half> %x, splat (half 0xH3C00)
+  %s = select <8 x i1> %c, <8 x half> splat (half 0xH3C00), <8 x half> %x
+  ret <8 x half> %s
+}
+
+define <4 x half> @select2(<4 x i32> %0, <4 x half> %1) {
+; CHECK-LABEL: select2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmovq {{.*#+}} xmm2 = [9.6E+1,9.7E+1,9.8E+1,9.9E+1,0.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; CHECK-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+entry:
+  %2 = icmp eq <4 x i32> %0, zeroinitializer
+  %3 = select <4 x i1> %2, <4 x half> %1, <4 x half> <half 0xH5600, half 0xH5610, half 0xH5620, half 0xH5630>
+  ret <4 x half> %3
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
index 0973824fbb0e..b408aac21810 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
@@ -46,7 +46,7 @@ define <2 x double> @test_compress_pd_128(<2 x double> %data) {
 ; CHECK-LABEL: test_compress_pd_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> undef, <2 x i1> <i1 true, i1 true>)
+  %1 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> poison, <2 x i1> <i1 true, i1 true>)
   ret <2 x double> %1
 }
 
@@ -94,7 +94,7 @@ define <4 x float> @test_compress_ps_128(<4 x float> %data) {
 ; CHECK-LABEL: test_compress_ps_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x float> %1
 }
 
@@ -142,7 +142,7 @@ define <2 x i64> @test_compress_q_128(<2 x i64> %data) {
 ; CHECK-LABEL: test_compress_q_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> undef, <2 x i1> <i1 true, i1 true>)
+  %1 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> poison, <2 x i1> <i1 true, i1 true>)
   ret <2 x i64> %1
 }
 
@@ -190,7 +190,7 @@ define <4 x i32> @test_compress_d_128(<4 x i32> %data) {
 ; CHECK-LABEL: test_compress_d_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x i32> %1
 }
 
@@ -198,7 +198,7 @@ define <2 x double> @test_expand_pd_128(<2 x double> %data) {
 ; CHECK-LABEL: test_expand_pd_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> undef, <2 x i1> <i1 true, i1 true>)
+  %1 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> poison, <2 x i1> <i1 true, i1 true>)
   ret <2 x double> %1
 }
 
@@ -246,7 +246,7 @@ define <4 x float> @test_expand_ps_128(<4 x float> %data) {
 ; CHECK-LABEL: test_expand_ps_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x float> %1
 }
 
@@ -294,7 +294,7 @@ define <2 x i64> @test_expand_q_128(<2 x i64> %data) {
 ; CHECK-LABEL: test_expand_q_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> undef, <2 x i1> <i1 true, i1 true>)
+  %1 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> poison, <2 x i1> <i1 true, i1 true>)
   ret <2 x i64> %1
 }
 
@@ -342,7 +342,7 @@ define <4 x i32> @test_expand_d_128(<4 x i32> %data) {
 ; CHECK-LABEL: test_expand_d_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x i32> %1
 }
 
@@ -430,7 +430,7 @@ define <4 x double> @test_compress_pd_256(<4 x double> %data) {
 ; CHECK-LABEL: test_compress_pd_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x double> %1
 }
 
@@ -476,7 +476,7 @@ define <8 x float> @test_compress_ps_256(<8 x float> %data) {
 ; CHECK-LABEL: test_compress_ps_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> poison, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
   ret <8 x float> %1
 }
 
@@ -524,7 +524,7 @@ define <4 x i64> @test_compress_q_256(<4 x i64> %data) {
 ; CHECK-LABEL: test_compress_q_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x i64> %1
 }
 
@@ -570,7 +570,7 @@ define <8 x i32> @test_compress_d_256(<8 x i32> %data) {
 ; CHECK-LABEL: test_compress_d_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> poison, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
   ret <8 x i32> %1
 }
 
@@ -578,7 +578,7 @@ define <4 x double> @test_expand_pd_256(<4 x double> %data) {
 ; CHECK-LABEL: test_expand_pd_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x double> %1
 }
 
@@ -626,7 +626,7 @@ define <8 x float> @test_expand_ps_256(<8 x float> %data) {
 ; CHECK-LABEL: test_expand_ps_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> poison, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
   ret <8 x float> %1
 }
 
@@ -672,7 +672,7 @@ define <4 x i64> @test_expand_q_256(<4 x i64> %data) {
 ; CHECK-LABEL: test_expand_q_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> poison, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
   ret <4 x i64> %1
 }
 
@@ -720,7 +720,7 @@ define <8 x i32> @test_expand_d_256(<8 x i32> %data) {
 ; CHECK-LABEL: test_expand_d_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %1 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> poison, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
   ret <8 x i32> %1
 }
 
@@ -884,7 +884,7 @@ define <4 x float> @test_mm512_maskz_max_ps_128(<4 x float> %a0, <4 x float> %a1
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   %2 = bitcast i8 %mask to <8 x i1>
-  %extract = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> zeroinitializer
   ret <4 x float> %3
 }
@@ -906,7 +906,7 @@ define <4 x float> @test_mm512_mask_max_ps_128(<4 x float> %a0, <4 x float> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
   %2 = bitcast i8 %mask to <8 x i1>
-  %extract = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %src
   ret <4 x float> %3
 }
@@ -986,7 +986,7 @@ define <4 x float> @test_mm512_maskz_min_ps_128(<4 x float> %a0, <4 x float> %a1
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   %2 = bitcast i8 %mask to <8 x i1>
-  %extract = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> zeroinitializer
   ret <4 x float> %3
 }
@@ -1008,7 +1008,7 @@ define <4 x float> @test_mm512_mask_min_ps_128(<4 x float> %a0, <4 x float> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
   %2 = bitcast i8 %mask to <8 x i1>
-  %extract = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %src
   ret <4 x float> %3
 }
@@ -5223,7 +5223,7 @@ define <2 x i64> @test_x86_avx512_mask_psra_q_128(<2 x i64> %a0, <2 x i64> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> %passthru
   ret <2 x i64> %res2
 }
@@ -5242,7 +5242,7 @@ define <2 x i64> @test_x86_avx512_maskz_psra_q_128(<2 x i64> %a0, <2 x i64> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> zeroinitializer
   ret <2 x i64> %res2
 }
@@ -5274,7 +5274,7 @@ define <4 x i64> @test_x86_avx512_mask_psra_q_256(<4 x i64> %a0, <2 x i64> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> %passthru
   ret <4 x i64> %res2
 }
@@ -5293,7 +5293,7 @@ define <4 x i64> @test_x86_avx512_maskz_psra_q_256(<4 x i64> %a0, <2 x i64> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> zeroinitializer
   ret <4 x i64> %res2
 }
@@ -5325,7 +5325,7 @@ define <2 x i64> @test_x86_avx512_mask_psrai_q_128(<2 x i64> %a0, <2 x i64> %pas
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> %passthru
   ret <2 x i64> %res2
 }
@@ -5344,7 +5344,7 @@ define <2 x i64> @test_x86_avx512_maskz_psrai_q_128(<2 x i64> %a0, i8 %mask) {
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> zeroinitializer
   ret <2 x i64> %res2
 }
@@ -5376,7 +5376,7 @@ define <4 x i64> @test_x86_avx512_mask_psrai_q_256(<4 x i64> %a0, <4 x i64> %pas
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> %passthru
   ret <4 x i64> %res2
 }
@@ -5395,7 +5395,7 @@ define <4 x i64> @test_x86_avx512_maskz_psrai_q_256(<4 x i64> %a0, i8 %mask) {
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> zeroinitializer
   ret <4 x i64> %res2
 }
@@ -5427,7 +5427,7 @@ define <2 x i64> @test_x86_avx512_mask_psrav_q_128(<2 x i64> %a0, <2 x i64> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %a0, <2 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> %a2
   ret <2 x i64> %res2
 }
@@ -5447,7 +5447,7 @@ define <2 x i64> @test_x86_avx512_maskz_psrav_q_128(<2 x i64> %a0, <2 x i64> %a1
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %a0, <2 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <2 x i32> <i32 0, i32 1>
   %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> zeroinitializer
   ret <2 x i64> %res2
 }
@@ -5480,7 +5480,7 @@ define <4 x i64> @test_x86_avx512_mask_psrav_q_256(<4 x i64> %a0, <4 x i64> %a1,
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %a0, <4 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> %a2
   ret <4 x i64> %res2
 }
@@ -5500,7 +5500,7 @@ define <4 x i64> @test_x86_avx512_maskz_psrav_q_256(<4 x i64> %a0, <4 x i64> %a1
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %a0, <4 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
-  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> zeroinitializer
   ret <4 x i64> %res2
 }
@@ -6861,7 +6861,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmb(<4 x float> %a0, <4 x float> %a1,
 ; X64-NEXT:    # xmm0 {%k1} = (xmm1 * xmm0) + mem
 ; X64-NEXT:    retq # encoding: [0xc3]
   %q = load float, ptr %ptr_a2
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
@@ -6889,7 +6889,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmba(<4 x float> %a0, <4 x float> %a1
 ; X64-NEXT:    # xmm0 {%k1} = (xmm1 * xmm0) + mem
 ; X64-NEXT:    retq # encoding: [0xc3]
   %q = load float, ptr %ptr_a2, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
@@ -6914,7 +6914,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmbz(<4 x float> %a0, <4 x float> %a1
 ; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
 ; X64-NEXT:    retq # encoding: [0xc3]
   %q = load float, ptr %ptr_a2
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
@@ -6936,7 +6936,7 @@ define <4 x float> @test_mask_vfmadd128_ps_rmbza(<4 x float> %a0, <4 x float> %a
 ; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
 ; X64-NEXT:    retq # encoding: [0xc3]
   %q = load float, ptr %ptr_a2, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
   %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
   %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
   %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-logic.ll b/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-logic.ll
index 284a0eb33047..c1ae0e36c2c0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-logic.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/avx512vl-logic.ll
@@ -1039,7 +1039,7 @@ define <4 x i32> @ternlog_xor_andn(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
 define <4 x i32> @ternlog_or_and_mask(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: ternlog_or_and_mask:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpternlogd {{.*#+}} xmm0 = (xmm0 & mem) | xmm1
+; CHECK-NEXT:    vpternlogd {{.*#+}} xmm0 = (xmm0 & m32bcst) | xmm1
 ; CHECK-NEXT:    retq
   %a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
   %b = or <4 x i32> %a, %y
@@ -1049,7 +1049,7 @@ define <4 x i32> @ternlog_or_and_mask(<4 x i32> %x, <4 x i32> %y) {
 define <8 x i32> @ternlog_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y) {
 ; CHECK-LABEL: ternlog_or_and_mask_ymm:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpternlogd {{.*#+}} ymm0 = (ymm0 & mem) | ymm1
+; CHECK-NEXT:    vpternlogd {{.*#+}} ymm0 = (ymm0 & m32bcst) | ymm1
 ; CHECK-NEXT:    retq
   %a = and <8 x i32> %x, <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
   %b = or <8 x i32> %a, %y
@@ -1059,7 +1059,7 @@ define <8 x i32> @ternlog_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y) {
 define <2 x i64> @ternlog_xor_and_mask(<2 x i64> %x, <2 x i64> %y) {
 ; CHECK-LABEL: ternlog_xor_and_mask:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; CHECK-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m64bcst)
 ; CHECK-NEXT:    retq
   %a = and <2 x i64> %x, <i64 1099511627775, i64 1099511627775>
   %b = xor <2 x i64> %a, %y
@@ -1069,7 +1069,7 @@ define <2 x i64> @ternlog_xor_and_mask(<2 x i64> %x, <2 x i64> %y) {
 define <4 x i64> @ternlog_xor_and_mask_ymm(<4 x i64> %x, <4 x i64> %y) {
 ; CHECK-LABEL: ternlog_xor_and_mask_ymm:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem)
+; CHECK-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m64bcst)
 ; CHECK-NEXT:    retq
   %a = and <4 x i64> %x, <i64 72057594037927935, i64 72057594037927935, i64 72057594037927935, i64 72057594037927935>
   %b = xor <4 x i64> %a, %y
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/basic-block-sections-cloning-invalid.ll b/external/llvm-project/llvm/test/CodeGen/X86/basic-block-sections-cloning-invalid.ll
index c316ef9f8f26..3c3c4b55e89b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/basic-block-sections-cloning-invalid.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/basic-block-sections-cloning-invalid.ll
@@ -83,5 +83,5 @@ cold:
 ;; Check the warnings
 ; WARN1: warning: block #2 is not a successor of block #0 in function foo
 ; WARN2: warning: no block with id 100 in function foo
-; WARN3: warning: block #6 has its machine block address taken in function foo
+; WARN3: warning: block #6 is a branch target of an 'asm goto' in function foo
 
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/bitselect.ll b/external/llvm-project/llvm/test/CodeGen/X86/bitselect.ll
index 2922113b14ea..4fc0827ac4dd 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/bitselect.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/bitselect.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-bmi | FileCheck %s --check-prefixes=X64,X64-NOBMI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefixes=X64,X64-BMI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-bmi | FileCheck %s --check-prefixes=X64-NOBMI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefixes=X64-BMI
 
 ; PR46472
 ; bitselect(a,b,m) == or(and(a,not(m)),and(b,m))
@@ -17,14 +17,22 @@ define i8 @bitselect_i8(i8 %a, i8 %b, i8 %m) nounwind {
 ; X86-NEXT:    xorb %cl, %al
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: bitselect_i8:
-; X64:       # %bb.0:
-; X64-NEXT:    andl %edx, %esi
-; X64-NEXT:    movl %edx, %eax
-; X64-NEXT:    notb %al
-; X64-NEXT:    andb %dil, %al
-; X64-NEXT:    orb %sil, %al
-; X64-NEXT:    retq
+; X64-NOBMI-LABEL: bitselect_i8:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    xorl %edi, %eax
+; X64-NOBMI-NEXT:    andl %edx, %eax
+; X64-NOBMI-NEXT:    xorl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: bitselect_i8:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    andnl %edi, %edx, %eax
+; X64-BMI-NEXT:    andl %edx, %esi
+; X64-BMI-NEXT:    orl %esi, %eax
+; X64-BMI-NEXT:    # kill: def $al killed $al killed $eax
+; X64-BMI-NEXT:    retq
   %not = xor i8 %m, -1
   %ma = and i8 %a, %not
   %mb = and i8 %b, %m
@@ -35,21 +43,20 @@ define i8 @bitselect_i8(i8 %a, i8 %b, i8 %m) nounwind {
 define i16 @bitselect_i16(i16 %a, i16 %b, i16 %m) nounwind {
 ; X86-LABEL: bitselect_i16:
 ; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    xorw %ax, %cx
-; X86-NEXT:    andw {{[0-9]+}}(%esp), %cx
+; X86-NEXT:    xorw %cx, %ax
+; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
 ; X86-NEXT:    xorl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-NOBMI-LABEL: bitselect_i16:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    movl %edx, %eax
-; X64-NOBMI-NEXT:    andl %edx, %esi
-; X64-NOBMI-NEXT:    notl %eax
-; X64-NOBMI-NEXT:    andl %edi, %eax
-; X64-NOBMI-NEXT:    orl %esi, %eax
+; X64-NOBMI-NEXT:    movl %esi, %eax
+; X64-NOBMI-NEXT:    xorl %edi, %eax
+; X64-NOBMI-NEXT:    andl %edx, %eax
+; X64-NOBMI-NEXT:    xorl %edi, %eax
 ; X64-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NOBMI-NEXT:    retq
 ;
@@ -186,13 +193,12 @@ define i128 @bitselect_i128(i128 %a, i128 %b, i128 %m) nounwind {
 ;
 ; X64-BMI-LABEL: bitselect_i128:
 ; X64-BMI:       # %bb.0:
-; X64-BMI-NEXT:    andnq %rsi, %r9, %rsi
 ; X64-BMI-NEXT:    andnq %rdi, %r8, %rax
-; X64-BMI-NEXT:    andq %r9, %rcx
-; X64-BMI-NEXT:    orq %rcx, %rsi
 ; X64-BMI-NEXT:    andq %r8, %rdx
 ; X64-BMI-NEXT:    orq %rdx, %rax
-; X64-BMI-NEXT:    movq %rsi, %rdx
+; X64-BMI-NEXT:    andnq %rsi, %r9, %rdx
+; X64-BMI-NEXT:    andq %r9, %rcx
+; X64-BMI-NEXT:    orq %rcx, %rdx
 ; X64-BMI-NEXT:    retq
   %not = xor i128 %m, -1
   %ma = and i128 %a, %not
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/bmi-select-distrib.ll b/external/llvm-project/llvm/test/CodeGen/X86/bmi-select-distrib.ll
new file mode 100644
index 000000000000..e5696ded4fbf
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/bmi-select-distrib.ll
@@ -0,0 +1,865 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+sse2,+bmi | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+sse2,+bmi,+bmi2 | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefixes=X64
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2 | FileCheck %s --check-prefixes=X64
+
+; PR131587
+define i32 @and_select_neg_to_blsi1(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_neg_to_blsi1:
+; X86:       # %bb.0:
+; X86-NEXT:    blsil %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_to_blsi1:
+; X64:       # %bb.0:
+; X64-NEXT:    blsil %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 0, %a1
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; PR131587
+define i32 @and_select_neg_to_blsi2(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_neg_to_blsi2:
+; X86:       # %bb.0:
+; X86-NEXT:    blsil %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_to_blsi2:
+; X64:       # %bb.0:
+; X64-NEXT:    blsil %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 0, %a1
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %bls, %a1
+  ret i32 %ret
+}
+
+; PR131587
+define i32 @and_select_neg_to_blsi3(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_neg_to_blsi3:
+; X86:       # %bb.0:
+; X86-NEXT:    blsil %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_to_blsi3:
+; X64:       # %bb.0:
+; X64-NEXT:    blsil %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 0, %a1
+  %bls = select i1 %a0, i32 -1, i32 %sub
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; PR131587
+define i64 @and_select_neg_to_blsi_i64(i1 %a0, i64 %a1) nounwind {
+; X86-LABEL: and_select_neg_to_blsi_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    sbbl %esi, %edx
+; X86-NEXT:    andl %esi, %edx
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %esi, %edx
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_to_blsi_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    blsiq %rsi, %rax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmoveq %rsi, %rax
+; X64-NEXT:    retq
+  %sub = sub i64 0, %a1
+  %bls = select i1 %a0, i64 %sub, i64 -1
+  %ret = and i64 %a1, %bls
+  ret i64 %ret
+}
+
+; The transformation also applies to the following test although we do not emit a BLSI instruction
+; here because the negation instruction is used more than once.
+define i32 @and_neg_select_pos_i32(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_neg_select_pos_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    negl %ecx
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_neg_select_pos_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    andl %eax, %esi
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 0, %a1
+  %bls = select i1 %a0, i32 %a1, i32 -1
+  %ret = and i32 %sub, %bls
+  ret i32 %ret
+}
+
+; Negative test
+define i16 @and_select_neg_i16(i1 %a0, i16 %a1) nounwind {
+; X86-LABEL: and_select_neg_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $1, %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    negl %esi
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpb $1, %cl
+; X86-NEXT:    sbbl %eax, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    andl %edx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    negl %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb $1, %dil
+; X64-NEXT:    sbbl %eax, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %sub = sub i16 0, %a1
+  %bls = select i1 %a0, i16 %sub, i16 -1
+  %ret = and i16 %a1, %bls
+  ret i16 %ret
+}
+
+; Negative test
+define <4 x i32> @and_select_neg_v4xi32(i1 %a0, <4 x i32> %a1) nounwind {
+; X86-LABEL: and_select_neg_v4xi32:
+; X86:       # %bb.0:
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    jne .LBB6_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-NEXT:    pand %xmm1, %xmm0
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB6_1:
+; X86-NEXT:    pxor %xmm1, %xmm1
+; X86-NEXT:    psubd %xmm0, %xmm1
+; X86-NEXT:    pand %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_v4xi32:
+; X64:       # %bb.0:
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    jne .LBB6_1
+; X64-NEXT:  # %bb.2:
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    pand %xmm1, %xmm0
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB6_1:
+; X64-NEXT:    pxor %xmm1, %xmm1
+; X64-NEXT:    psubd %xmm0, %xmm1
+; X64-NEXT:    pand %xmm1, %xmm0
+; X64-NEXT:    retq
+  %sub = sub <4 x i32> zeroinitializer, %a1
+  %bls = select i1 %a0, <4 x i32> %sub, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  %ret = and <4 x i32> %a1, %bls
+  ret <4 x i32> %ret
+}
+
+; Negative test
+define i32 @and_select_no_neg(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_no_neg:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $1, %cl
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpb $1, %cl
+; X86-NEXT:    sbbl %edx, %edx
+; X86-NEXT:    orl %eax, %edx
+; X86-NEXT:    andl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_no_neg:
+; X64:       # %bb.0:
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb $1, %dil
+; X64-NEXT:    sbbl %eax, %eax
+; X64-NEXT:    orl %esi, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 %a1, 0
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; Negative test
+define i32 @and_select_neg_wrong_const(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_neg_wrong_const:
+; X86:       # %bb.0:
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    negl %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    cmovnel %ecx, %edx
+; X86-NEXT:    andl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_wrong_const:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    negl %ecx
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 0, %a1
+  %bls = select i1 %a0, i32 %sub, i32 1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; Negative test
+define i32 @and_select_neg_different_op(i1 %a0, i32 inreg %a1, i32 inreg %a2) nounwind {
+; X86-LABEL: and_select_neg_different_op:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $1, %cl
+; X86-NEXT:    negl %edx
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    cmpb $1, %cl
+; X86-NEXT:    sbbl %esi, %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    andl %esi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_neg_different_op:
+; X64:       # %bb.0:
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    negl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb $1, %dil
+; X64-NEXT:    sbbl %eax, %eax
+; X64-NEXT:    orl %edx, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 0, %a2
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; PR133848
+define i32 @and_select_sub_1_to_blsr1(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_sub_1_to_blsr1:
+; X86:       # %bb.0:
+; X86-NEXT:    blsrl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_to_blsr1:
+; X64:       # %bb.0:
+; X64-NEXT:    blsrl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; PR133848
+define i32 @and_select_sub_1_to_blsr2(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_sub_1_to_blsr2:
+; X86:       # %bb.0:
+; X86-NEXT:    blsrl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_to_blsr2:
+; X64:       # %bb.0:
+; X64-NEXT:    blsrl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %bls, %a1
+  ret i32 %ret
+}
+
+; PR133848
+define i32 @and_select_sub_1_to_blsr3(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_sub_1_to_blsr3:
+; X86:       # %bb.0:
+; X86-NEXT:    blsrl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_to_blsr3:
+; X64:       # %bb.0:
+; X64-NEXT:    blsrl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 -1, i32 %sub
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; PR133848
+define i32 @and_select_sub_1_to_blsr4(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_sub_1_to_blsr4:
+; X86:       # %bb.0:
+; X86-NEXT:    blsrl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_to_blsr4:
+; X64:       # %bb.0:
+; X64-NEXT:    blsrl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 %a1, 1
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; The transformation also applies to the following test although we do not emit a BLSR instruction
+; here because the subtraction instruction is used more than once.
+define i32 @and_sub_1_select_orig(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_sub_1_select_orig:
+; X86:       # %bb.0:
+; X86-NEXT:    leal -1(%eax), %ecx
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_sub_1_select_orig:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    leal -1(%rsi), %eax
+; X64-NEXT:    andl %eax, %esi
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %a1, i32 -1
+  %ret = and i32 %sub, %bls
+  ret i32 %ret
+}
+
+; PR133848
+define i64 @and_select_sub_1_to_blsr_i64(i1 %a0, i64 %a1) nounwind {
+; X86-LABEL: and_select_sub_1_to_blsr_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $-1, %eax
+; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    adcl $-1, %edx
+; X86-NEXT:    andl %esi, %edx
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    cmovel %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_to_blsr_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    blsrq %rsi, %rax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmoveq %rsi, %rax
+; X64-NEXT:    retq
+  %sub = add i64 %a1, -1
+  %bls = select i1 %a0, i64 %sub, i64 -1
+  %ret = and i64 %a1, %bls
+  ret i64 %ret
+}
+
+; Negative test
+define i16 @and_select_sub_1_i16(i1 %a0, i16 %a1) nounwind {
+; X86-LABEL: and_select_sub_1_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $1, %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    leal -1(%edx), %esi
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpb $1, %cl
+; X86-NEXT:    sbbl %eax, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    andl %edx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    leal -1(%rsi), %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb $1, %dil
+; X64-NEXT:    sbbl %eax, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %sub = add i16 %a1, -1
+  %bls = select i1 %a0, i16 %sub, i16 -1
+  %ret = and i16 %a1, %bls
+  ret i16 %ret
+}
+
+; Negative test
+define <4 x i32> @and_select_sub_1_v4xi32(i1 %a0, <4 x i32> %a1) nounwind {
+; X86-LABEL: and_select_sub_1_v4xi32:
+; X86:       # %bb.0:
+; X86-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    je .LBB17_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    paddd %xmm0, %xmm1
+; X86-NEXT:  .LBB17_2:
+; X86-NEXT:    pand %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_v4xi32:
+; X64:       # %bb.0:
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    je .LBB17_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    paddd %xmm0, %xmm1
+; X64-NEXT:  .LBB17_2:
+; X64-NEXT:    pand %xmm1, %xmm0
+; X64-NEXT:    retq
+  %sub = add <4 x i32> %a1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bls = select i1 %a0, <4 x i32> %sub, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  %ret = and <4 x i32> %a1, %bls
+  ret <4 x i32> %ret
+}
+
+; Negative test
+define i32 @and_select_no_sub_1(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_no_sub_1:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $1, %cl
+; X86-NEXT:    leal -2(%eax), %edx
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    cmpb $1, %cl
+; X86-NEXT:    sbbl %esi, %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    andl %esi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_no_sub_1:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    leal -2(%rsi), %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb $1, %dil
+; X64-NEXT:    sbbl %eax, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -2
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; Negative test
+define i32 @and_select_sub_1_wrong_const(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: and_select_sub_1_wrong_const:
+; X86:       # %bb.0:
+; X86-NEXT:    leal -1(%eax), %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    cmovnel %ecx, %edx
+; X86-NEXT:    andl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_wrong_const:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    leal -1(%rsi), %ecx
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %sub, i32 1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+; Negative test
+define i32 @and_select_sub_1_different_op(i1 %a0, i32 inreg %a1, i32 inreg %a2) nounwind {
+; X86-LABEL: and_select_sub_1_different_op:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $1, %cl
+; X86-NEXT:    decl %edx
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    cmpb $1, %cl
+; X86-NEXT:    sbbl %esi, %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    andl %esi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: and_select_sub_1_different_op:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edx killed $edx def $rdx
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    leal -1(%rdx), %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb $1, %dil
+; X64-NEXT:    sbbl %eax, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    andl %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a2, -1
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = and i32 %a1, %bls
+  ret i32 %ret
+}
+
+define i32 @xor_select_sub_1_to_blsmsk1(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_to_blsmsk1:
+; X86:       # %bb.0:
+; X86-NEXT:    blsmskl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_to_blsmsk1:
+; X64:       # %bb.0:
+; X64-NEXT:    blsmskl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %sub, i32 0
+  %ret = xor i32 %a1, %bls
+  ret i32 %ret
+}
+
+define i32 @xor_select_sub_1_to_blsmsk2(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_to_blsmsk2:
+; X86:       # %bb.0:
+; X86-NEXT:    blsmskl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_to_blsmsk2:
+; X64:       # %bb.0:
+; X64-NEXT:    blsmskl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %sub, i32 0
+  %ret = xor i32 %bls, %a1
+  ret i32 %ret
+}
+
+define i32 @xor_select_sub_1_to_blsmsk3(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_to_blsmsk3:
+; X86:       # %bb.0:
+; X86-NEXT:    blsmskl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_to_blsmsk3:
+; X64:       # %bb.0:
+; X64-NEXT:    blsmskl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 0, i32 %sub
+  %ret = xor i32 %a1, %bls
+  ret i32 %ret
+}
+
+define i32 @xor_select_sub_1_to_blsmsk4(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_to_blsmsk4:
+; X86:       # %bb.0:
+; X86-NEXT:    blsmskl %eax, %ecx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_to_blsmsk4:
+; X64:       # %bb.0:
+; X64-NEXT:    blsmskl %esi, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %sub = sub i32 %a1, 1
+  %bls = select i1 %a0, i32 %sub, i32 0
+  %ret = xor i32 %a1, %bls
+  ret i32 %ret
+}
+
+; The transformation also applies to the following test although we do not emit a BLSMSK instruction
+; here because the subtraction instruction is used more than once.
+define i32 @xor_sub_1_select_orig(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: xor_sub_1_select_orig:
+; X86:       # %bb.0:
+; X86-NEXT:    leal -1(%eax), %ecx
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_sub_1_select_orig:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    leal -1(%rsi), %eax
+; X64-NEXT:    xorl %eax, %esi
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %a1, i32 0
+  %ret = xor i32 %sub, %bls
+  ret i32 %ret
+}
+
+define i64 @xor_select_sub_1_to_blsmsk_i64(i1 %a0, i64 %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_to_blsmsk_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $-1, %eax
+; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    adcl $-1, %edx
+; X86-NEXT:    xorl %esi, %edx
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    cmovel %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_to_blsmsk_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    blsmskq %rsi, %rax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmoveq %rsi, %rax
+; X64-NEXT:    retq
+  %sub = add i64 %a1, -1
+  %bls = select i1 %a0, i64 %sub, i64 0
+  %ret = xor i64 %a1, %bls
+  ret i64 %ret
+}
+
+; Negative test
+define i16 @xor_select_sub_1_i16(i1 %a0, i16 %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal -1(%ecx), %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %edx, %eax
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    leal -1(%rsi), %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    xorl %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %sub = add i16 %a1, -1
+  %bls = select i1 %a0, i16 %sub, i16 0
+  %ret = xor i16 %a1, %bls
+  ret i16 %ret
+}
+
+; Negative test
+define <4 x i32> @xor_select_sub_1_v4xi32(i1 %a0, <4 x i32> %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_v4xi32:
+; X86:       # %bb.0:
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    jne .LBB28_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    xorps %xmm1, %xmm1
+; X86-NEXT:    xorps %xmm1, %xmm0
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB28_1:
+; X86-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-NEXT:    paddd %xmm0, %xmm1
+; X86-NEXT:    pxor %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_v4xi32:
+; X64:       # %bb.0:
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    jne .LBB28_1
+; X64-NEXT:  # %bb.2:
+; X64-NEXT:    xorps %xmm1, %xmm1
+; X64-NEXT:    xorps %xmm1, %xmm0
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB28_1:
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    paddd %xmm0, %xmm1
+; X64-NEXT:    pxor %xmm1, %xmm0
+; X64-NEXT:    retq
+  %sub = add <4 x i32> %a1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bls = select i1 %a0, <4 x i32> %sub, <4 x i32> zeroinitializer
+  %ret = xor <4 x i32> %a1, %bls
+  ret <4 x i32> %ret
+}
+
+; Negative test
+define i32 @xor_select_no_sub_1(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: xor_select_no_sub_1:
+; X86:       # %bb.0:
+; X86-NEXT:    leal 1(%eax), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %edx
+; X86-NEXT:    xorl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_no_sub_1:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    leal 1(%rsi), %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    xorl %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, 1
+  %bls = select i1 %a0, i32 %sub, i32 0
+  %ret = xor i32 %a1, %bls
+  ret i32 %ret
+}
+
+; Negative test
+define i32 @xor_select_sub_1_wrong_const(i1 %a0, i32 inreg %a1) nounwind {
+; X86-LABEL: xor_select_sub_1_wrong_const:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $1, %cl
+; X86-NEXT:    leal -1(%eax), %edx
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    cmpb $1, %cl
+; X86-NEXT:    sbbl %esi, %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    xorl %esi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_wrong_const:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    andb $1, %dil
+; X64-NEXT:    leal -1(%rsi), %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb $1, %dil
+; X64-NEXT:    sbbl %eax, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    xorl %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a1, -1
+  %bls = select i1 %a0, i32 %sub, i32 -1
+  %ret = xor i32 %a1, %bls
+  ret i32 %ret
+}
+
+; Negative test
+define i32 @xor_select_sub_1_different_op(i1 %a0, i32 inreg %a1, i32 inreg %a2) nounwind {
+; X86-LABEL: xor_select_sub_1_different_op:
+; X86:       # %bb.0:
+; X86-NEXT:    leal -1(%edx), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %edx
+; X86-NEXT:    xorl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: xor_select_sub_1_different_op:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edx killed $edx def $rdx
+; X64-NEXT:    leal -1(%rdx), %ecx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    xorl %esi, %eax
+; X64-NEXT:    retq
+  %sub = add i32 %a2, -1
+  %bls = select i1 %a0, i32 %sub, i32 0
+  %ret = xor i32 %a1, %bls
+  ret i32 %ret
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/break-false-dep.ll b/external/llvm-project/llvm/test/CodeGen/X86/break-false-dep.ll
index 5acbccf41c5d..6943622fac7f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/break-false-dep.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/break-false-dep.ll
@@ -36,7 +36,7 @@ define dso_local float @t2(ptr nocapture %x) nounwind readonly ssp optsize {
 ;
 ; AVX-LABEL: t2:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsd2ss (%rcx), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsd2ss (%rcx), %xmm5, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load double, ptr %x, align 8
@@ -93,7 +93,7 @@ define dso_local float @squirtf_size(ptr %x) nounwind optsize {
 ;
 ; AVX-LABEL: squirtf_size:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vsqrtss (%rcx), %xmm0, %xmm0
+; AVX-NEXT:    vsqrtss (%rcx), %xmm5, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %z = load float, ptr %x
@@ -114,7 +114,7 @@ define dso_local double @squirt_size(ptr %x) nounwind optsize {
 ;
 ; AVX-LABEL: squirt_size:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vsqrtsd (%rcx), %xmm0, %xmm0
+; AVX-NEXT:    vsqrtsd (%rcx), %xmm5, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %z = load double, ptr %x
@@ -199,8 +199,8 @@ define dso_local float @loopdep1(i32 %m) nounwind uwtable readnone ssp {
 ; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB6_3: # %for.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX1-NEXT:    vcvtsi2ss %eax, %xmm4, %xmm2
-; AVX1-NEXT:    vcvtsi2ss %ecx, %xmm4, %xmm3
+; AVX1-NEXT:    vcvtsi2ss %eax, %xmm5, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %ecx, %xmm5, %xmm3
 ; AVX1-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vaddss %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    incl %eax
@@ -226,9 +226,9 @@ define dso_local float @loopdep1(i32 %m) nounwind uwtable readnone ssp {
 ; AVX512VL-NEXT:    .p2align 4
 ; AVX512VL-NEXT:  .LBB6_3: # %for.body
 ; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX512VL-NEXT:    vcvtsi2ss %eax, %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtsi2ss %eax, %xmm5, %xmm2
 ; AVX512VL-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vcvtsi2ss %ecx, %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtsi2ss %ecx, %xmm5, %xmm2
 ; AVX512VL-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512VL-NEXT:    incl %eax
 ; AVX512VL-NEXT:    decl %ecx
@@ -358,8 +358,8 @@ define i64 @loopdep2(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB7_1: # %loop
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vcvtsi2sd %rcx, %xmm1, %xmm0
+; AVX-NEXT:    vxorps %xmm5, %xmm5, %xmm5
+; AVX-NEXT:    vcvtsi2sd %rcx, %xmm5, %xmm0
 ; AVX-NEXT:    vmovsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX-NEXT:    #APP
 ; AVX-NEXT:    #NO_APP
@@ -566,8 +566,8 @@ define dso_local void @loopdep3() {
 ; AVX-NEXT:  .LBB8_2: # %for.body3
 ; AVX-NEXT:    # Parent Loop BB8_1 Depth=1
 ; AVX-NEXT:    # => This Inner Loop Header: Depth=2
-; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vcvtsi2sdl (%r11), %xmm0, %xmm0
+; AVX-NEXT:    vxorps %xmm5, %xmm5, %xmm5
+; AVX-NEXT:    vcvtsi2sdl (%r11), %xmm5, %xmm0
 ; AVX-NEXT:    vmulsd (%rsi,%rdx), %xmm0, %xmm0
 ; AVX-NEXT:    vmulsd (%rsi,%r8), %xmm0, %xmm0
 ; AVX-NEXT:    vmulsd (%rsi,%r9), %xmm0, %xmm0
@@ -761,8 +761,8 @@ define dso_local double @inlineasmdep(i64 %arg) {
 ; AVX-NEXT:    #NO_APP
 ; AVX-NEXT:    #APP
 ; AVX-NEXT:    #NO_APP
-; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vcvtsi2sd %rcx, %xmm0, %xmm0
+; AVX-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vcvtsi2sd %rcx, %xmm3, %xmm0
 ; AVX-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/bsf.ll b/external/llvm-project/llvm/test/CodeGen/X86/bsf.ll
new file mode 100644
index 000000000000..312f94c04123
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/bsf.ll
@@ -0,0 +1,452 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64
+
+define i8 @cmov_bsf8(i8 %x, i8 %y) nounwind {
+; X86-LABEL: cmov_bsf8:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testb %al, %al
+; X86-NEXT:    je .LBB0_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    orl $256, %eax # imm = 0x100
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB0_1:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsf8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    orl $256, %eax # imm = 0x100
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    testb %dil, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i8 @llvm.cttz.i8(i8 %x, i1 false)
+  %2 = icmp eq i8 %x, 0
+  %3 = select i1 %2, i8 %y, i8 %1
+  ret i8 %3
+}
+
+define i8 @cmov_bsf8_undef(i8 %x, i8 %y) nounwind {
+; X86-LABEL: cmov_bsf8_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testb %al, %al
+; X86-NEXT:    jne .LBB1_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB1_1:
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsf8_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    testb %dil, %dil
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i8 @llvm.cttz.i8(i8 %x, i1 true)
+  %2 = icmp ne i8 %x, 0
+  %3 = select i1 %2, i8 %1, i8 %y
+  ret i8 %3
+}
+
+define i16 @cmov_bsf16(i16 %x, i16 %y) nounwind {
+; X86-LABEL: cmov_bsf16:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testw %ax, %ax
+; X86-NEXT:    jne .LBB2_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB2_1:
+; X86-NEXT:    orl $65536, %eax # imm = 0x10000
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsf16:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    orl $65536, %eax # imm = 0x10000
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    testw %di, %di
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i16 @llvm.cttz.i16(i16 %x, i1 false)
+  %2 = icmp ne i16 %x, 0
+  %3 = select i1 %2, i16 %1, i16 %y
+  ret i16 %3
+}
+
+define i16 @cmov_bsf16_undef(i16 %x, i16 %y) nounwind {
+; X86-LABEL: cmov_bsf16_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testw %ax, %ax
+; X86-NEXT:    je .LBB3_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB3_1:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsf16_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    testw %di, %di
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i16 @llvm.cttz.i16(i16 %x, i1 true)
+  %2 = icmp eq i16 %x, 0
+  %3 = select i1 %2, i16 %y, i16 %1
+  ret i16 %3
+}
+
+define i32 @cmov_bsf32(i32 %x, i32 %y) nounwind {
+; X86-LABEL: cmov_bsf32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    je .LBB4_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %ecx, %eax
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    jne .LBB4_5
+; X86-NEXT:  .LBB4_4:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:  .LBB4_5: # %cond.end
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB4_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    je .LBB4_4
+; X86-NEXT:    jmp .LBB4_5
+;
+; X64-LABEL: cmov_bsf32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    bsfl %edi, %eax
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %1 = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
+  %2 = icmp eq i32 %x, 0
+  %3 = select i1 %2, i32 %y, i32 %1
+  ret i32 %3
+}
+
+define i32 @cmov_bsf32_undef(i32 %x, i32 %y) nounwind {
+; X86-LABEL: cmov_bsf32_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    jne .LBB5_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB5_1:
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsf32_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsfl %edi, %eax
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    retq
+  %1 = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %2 = icmp ne i32 %x, 0
+  %3 = select i1 %2, i32 %1, i32 %y
+  ret i32 %3
+}
+
+define i64 @cmov_bsf64(i64 %x, i64 %y) nounwind {
+; X86-LABEL: cmov_bsf64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    orl %ecx, %eax
+; X86-NEXT:    je .LBB6_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    jne .LBB6_3
+; X86-NEXT:  # %bb.4: # %cond.false
+; X86-NEXT:    rep bsfl %ecx, %eax
+; X86-NEXT:    addl $32, %eax
+; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    je .LBB6_6
+; X86-NEXT:    jmp .LBB6_7
+; X86-NEXT:  .LBB6_1:
+; X86-NEXT:    movl $64, %eax
+; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    jne .LBB6_7
+; X86-NEXT:  .LBB6_6: # %cond.end
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:  .LBB6_7: # %cond.end
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB6_3:
+; X86-NEXT:    rep bsfl %esi, %eax
+; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    je .LBB6_6
+; X86-NEXT:    jmp .LBB6_7
+;
+; X64-LABEL: cmov_bsf64:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $64, %eax
+; X64-NEXT:    bsfq %rdi, %rax
+; X64-NEXT:    cmoveq %rsi, %rax
+; X64-NEXT:    retq
+  %1 = tail call i64 @llvm.cttz.i64(i64 %x, i1 false)
+  %2 = icmp ne i64 %x, 0
+  %3 = select i1 %2, i64 %1, i64 %y
+  ret i64 %3
+}
+
+define i64 @cmov_bsf64_undef(i64 %x, i64 %y) nounwind {
+; X86-LABEL: cmov_bsf64_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    orl %eax, %edx
+; X86-NEXT:    je .LBB7_5
+; X86-NEXT:  # %bb.1: # %select.false.sink
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    jne .LBB7_2
+; X86-NEXT:  # %bb.3: # %select.false.sink
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    addl $32, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB7_5: # %select.end
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB7_2:
+; X86-NEXT:    rep bsfl %ecx, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsf64_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsfq %rdi, %rax
+; X64-NEXT:    cmoveq %rsi, %rax
+; X64-NEXT:    retq
+  %1 = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+  %2 = icmp eq i64 %x, 0
+  %3 = select i1 %2, i64 %y, i64 %1
+  ret i64 %3
+}
+
+define i128 @cmov_bsf128(i128 %x, i128 %y) nounwind {
+; X86-LABEL: cmov_bsf128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    orl %ebp, %edx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    orl %eax, %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    je .LBB8_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    jne .LBB8_3
+; X86-NEXT:  # %bb.4: # %cond.false
+; X86-NEXT:    rep bsfl %edi, %esi
+; X86-NEXT:    addl $32, %esi
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB8_7
+; X86-NEXT:  .LBB8_6:
+; X86-NEXT:    rep bsfl %eax, %edx
+; X86-NEXT:    jmp .LBB8_8
+; X86-NEXT:  .LBB8_1:
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    movl $128, %esi
+; X86-NEXT:    jmp .LBB8_11
+; X86-NEXT:  .LBB8_3:
+; X86-NEXT:    rep bsfl %ecx, %esi
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    jne .LBB8_6
+; X86-NEXT:  .LBB8_7: # %cond.false
+; X86-NEXT:    rep bsfl %ebp, %edx
+; X86-NEXT:    addl $32, %edx
+; X86-NEXT:  .LBB8_8: # %cond.false
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    movl %ecx, %ebx
+; X86-NEXT:    orl %edi, %ebx
+; X86-NEXT:    jne .LBB8_10
+; X86-NEXT:  # %bb.9: # %cond.false
+; X86-NEXT:    addl $64, %edx
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:  .LBB8_10: # %cond.false
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:  .LBB8_11: # %cond.end
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    orl %eax, %edi
+; X86-NEXT:    orl %ecx, %edi
+; X86-NEXT:    jne .LBB8_13
+; X86-NEXT:  # %bb.12:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:  .LBB8_13: # %cond.end
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %edx, 12(%eax)
+; X86-NEXT:    movl %ebx, 8(%eax)
+; X86-NEXT:    movl %ebp, 4(%eax)
+; X86-NEXT:    movl %esi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: cmov_bsf128:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    orq %rsi, %rax
+; X64-NEXT:    je .LBB8_2
+; X64-NEXT:  # %bb.1: # %select.false.sink
+; X64-NEXT:    rep bsfq %rdi, %rcx
+; X64-NEXT:    movl $64, %eax
+; X64-NEXT:    rep bsfq %rsi, %rax
+; X64-NEXT:    addq $64, %rax
+; X64-NEXT:    testq %rdi, %rdi
+; X64-NEXT:    cmovneq %rcx, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB8_2: # %select.end
+; X64-NEXT:    movq %rdx, %rax
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    retq
+  %1 = tail call i128 @llvm.cttz.i128(i128 %x, i1 false)
+  %2 = icmp eq i128 %x, 0
+  %3 = select i1 %2, i128 %y, i128 %1
+  ret i128 %3
+}
+
+define i128 @cmov_bsf128_undef(i128 %x, i128 %y) nounwind {
+; X86-LABEL: cmov_bsf128_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    orl %esi, %edi
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    orl %ebx, %ebp
+; X86-NEXT:    orl %edi, %ebp
+; X86-NEXT:    je .LBB9_11
+; X86-NEXT:  # %bb.1: # %select.true.sink
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    jne .LBB9_2
+; X86-NEXT:  # %bb.3: # %select.true.sink
+; X86-NEXT:    rep bsfl %ecx, %edi
+; X86-NEXT:    addl $32, %edi
+; X86-NEXT:    testl %ebx, %ebx
+; X86-NEXT:    je .LBB9_6
+; X86-NEXT:  .LBB9_5:
+; X86-NEXT:    rep bsfl %ebx, %esi
+; X86-NEXT:    orl %ecx, %edx
+; X86-NEXT:    je .LBB9_8
+; X86-NEXT:    jmp .LBB9_9
+; X86-NEXT:  .LBB9_11: # %select.end
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    jmp .LBB9_10
+; X86-NEXT:  .LBB9_2:
+; X86-NEXT:    rep bsfl %edx, %edi
+; X86-NEXT:    testl %ebx, %ebx
+; X86-NEXT:    jne .LBB9_5
+; X86-NEXT:  .LBB9_6: # %select.true.sink
+; X86-NEXT:    rep bsfl %esi, %esi
+; X86-NEXT:    addl $32, %esi
+; X86-NEXT:    orl %ecx, %edx
+; X86-NEXT:    jne .LBB9_9
+; X86-NEXT:  .LBB9_8: # %select.true.sink
+; X86-NEXT:    addl $64, %esi
+; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:  .LBB9_9: # %select.true.sink
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    movl $0, 12(%eax)
+; X86-NEXT:    movl $0, 8(%eax)
+; X86-NEXT:    movl $0, 4(%eax)
+; X86-NEXT:  .LBB9_10: # %select.true.sink
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: cmov_bsf128_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    orq %rsi, %rax
+; X64-NEXT:    je .LBB9_2
+; X64-NEXT:  # %bb.1: # %select.true.sink
+; X64-NEXT:    rep bsfq %rdi, %rcx
+; X64-NEXT:    rep bsfq %rsi, %rax
+; X64-NEXT:    addq $64, %rax
+; X64-NEXT:    testq %rdi, %rdi
+; X64-NEXT:    cmovneq %rcx, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB9_2: # %select.end
+; X64-NEXT:    movq %rdx, %rax
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    retq
+  %1 = tail call i128 @llvm.cttz.i128(i128 %x, i1 true)
+  %2 = icmp ne i128 %x, 0
+  %3 = select i1 %2, i128 %1, i128 %y
+  ret i128 %3
+}
+
+declare i8 @llvm.cttz.i8(i8, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i128 @llvm.cttz.i128(i128, i1)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/bsr.ll b/external/llvm-project/llvm/test/CodeGen/X86/bsr.ll
new file mode 100644
index 000000000000..fbca4af425ea
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/bsr.ll
@@ -0,0 +1,490 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64
+
+define i8 @cmov_bsr8(i8 %x, i8 %y) nounwind {
+; X86-LABEL: cmov_bsr8:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    testb %cl, %cl
+; X86-NEXT:    je .LBB0_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    movzbl %cl, %eax
+; X86-NEXT:    bsrl %eax, %eax
+; X86-NEXT:    xorl $7, %eax
+; X86-NEXT:    testb %cl, %cl
+; X86-NEXT:    je .LBB0_4
+; X86-NEXT:  .LBB0_5: # %cond.end
+; X86-NEXT:    xorb $7, %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB0_1:
+; X86-NEXT:    movb $8, %al
+; X86-NEXT:    testb %cl, %cl
+; X86-NEXT:    jne .LBB0_5
+; X86-NEXT:  .LBB0_4:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr8:
+; X64:       # %bb.0:
+; X64-NEXT:    movzbl %dil, %ecx
+; X64-NEXT:    movl $15, %eax
+; X64-NEXT:    bsrl %ecx, %eax
+; X64-NEXT:    testb %cl, %cl
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i8 @llvm.ctlz.i8(i8 %x, i1 false)
+  %2 = xor i8 %1, 7
+  %3 = icmp eq i8 %x, 0
+  %4 = select i1 %3, i8 %y, i8 %2
+  ret i8 %4
+}
+
+define i8 @cmov_bsr8_undef(i8 %x, i8 %y) nounwind {
+; X86-LABEL: cmov_bsr8_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    jne .LBB1_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB1_1:
+; X86-NEXT:    bsrl %eax, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr8_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    movzbl %dil, %ecx
+; X64-NEXT:    bsrl %ecx, %eax
+; X64-NEXT:    testb %cl, %cl
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i8 @llvm.ctlz.i8(i8 %x, i1 true)
+  %2 = xor i8 %1, 7
+  %3 = icmp ne i8 %x, 0
+  %4 = select i1 %3, i8 %2, i8 %y
+  ret i8 %4
+}
+
+define i16 @cmov_bsr16(i16 %x, i16 %y) nounwind {
+; X86-LABEL: cmov_bsr16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testw %ax, %ax
+; X86-NEXT:    je .LBB2_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    bsrw %ax, %cx
+; X86-NEXT:    xorl $15, %ecx
+; X86-NEXT:    testw %ax, %ax
+; X86-NEXT:    jne .LBB2_4
+; X86-NEXT:  .LBB2_5: # %cond.end
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB2_1:
+; X86-NEXT:    movw $16, %cx
+; X86-NEXT:    testw %ax, %ax
+; X86-NEXT:    je .LBB2_5
+; X86-NEXT:  .LBB2_4:
+; X86-NEXT:    movzwl %cx, %eax
+; X86-NEXT:    xorl $15, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr16:
+; X64:       # %bb.0:
+; X64-NEXT:    movw $31, %ax
+; X64-NEXT:    bsrw %di, %ax
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i16 @llvm.ctlz.i16(i16 %x, i1 false)
+  %2 = xor i16 %1, 15
+  %3 = icmp ne i16 %x, 0
+  %4 = select i1 %3, i16 %2, i16 %y
+  ret i16 %4
+}
+
+define i16 @cmov_bsr16_undef(i16 %x, i16 %y) nounwind {
+; X86-LABEL: cmov_bsr16_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testw %ax, %ax
+; X86-NEXT:    je .LBB3_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    bsrw %ax, %ax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB3_1:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr16_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsrw %di, %ax
+; X64-NEXT:    cmovel %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+  %1 = tail call i16 @llvm.ctlz.i16(i16 %x, i1 true)
+  %2 = xor i16 %1, 15
+  %3 = icmp eq i16 %x, 0
+  %4 = select i1 %3, i16 %y, i16 %2
+  ret i16 %4
+}
+
+define i32 @cmov_bsr32(i32 %x, i32 %y) nounwind {
+; X86-LABEL: cmov_bsr32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    je .LBB4_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    bsrl %ecx, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    je .LBB4_4
+; X86-NEXT:  .LBB4_5: # %cond.end
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB4_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    jne .LBB4_5
+; X86-NEXT:  .LBB4_4:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    bsrl %edi, %eax
+; X64-NEXT:    retq
+  %1 = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %2 = xor i32 %1, 31
+  %3 = icmp eq i32 %x, 0
+  %4 = select i1 %3, i32 %y, i32 %2
+  ret i32 %4
+}
+
+define i32 @cmov_bsr32_undef(i32 %x, i32 %y) nounwind {
+; X86-LABEL: cmov_bsr32_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    jne .LBB5_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB5_1:
+; X86-NEXT:    bsrl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr32_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    bsrl %edi, %eax
+; X64-NEXT:    retq
+  %1 = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %2 = xor i32 %1, 31
+  %3 = icmp ne i32 %x, 0
+  %4 = select i1 %3, i32 %2, i32 %y
+  ret i32 %4
+}
+
+define i64 @cmov_bsr64(i64 %x, i64 %y) nounwind {
+; X86-LABEL: cmov_bsr64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    orl %ecx, %eax
+; X86-NEXT:    je .LBB6_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    jne .LBB6_3
+; X86-NEXT:  # %bb.4: # %cond.false
+; X86-NEXT:    bsrl %esi, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    orl $32, %eax
+; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    je .LBB6_7
+; X86-NEXT:    jmp .LBB6_6
+; X86-NEXT:  .LBB6_1:
+; X86-NEXT:    movl $64, %eax
+; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    jne .LBB6_6
+; X86-NEXT:  .LBB6_7: # %cond.end
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB6_3:
+; X86-NEXT:    bsrl %ecx, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    orl %ecx, %esi
+; X86-NEXT:    je .LBB6_7
+; X86-NEXT:  .LBB6_6:
+; X86-NEXT:    xorl $63, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    bsrq %rdi, %rax
+; X64-NEXT:    retq
+  %1 = tail call i64 @llvm.ctlz.i64(i64 %x, i1 false)
+  %2 = xor i64 %1, 63
+  %3 = icmp ne i64 %x, 0
+  %4 = select i1 %3, i64 %2, i64 %y
+  ret i64 %4
+}
+
+define i64 @cmov_bsr64_undef(i64 %x, i64 %y) nounwind {
+; X86-LABEL: cmov_bsr64_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    jne .LBB7_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    bsrl %ecx, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    orl $32, %eax
+; X86-NEXT:    orl %edx, %ecx
+; X86-NEXT:    jne .LBB7_5
+; X86-NEXT:  .LBB7_4:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB7_1:
+; X86-NEXT:    bsrl %edx, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    orl %edx, %ecx
+; X86-NEXT:    je .LBB7_4
+; X86-NEXT:  .LBB7_5:
+; X86-NEXT:    xorl $63, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    retl
+;
+; X64-LABEL: cmov_bsr64_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    bsrq %rdi, %rax
+; X64-NEXT:    retq
+  %1 = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %2 = xor i64 %1, 63
+  %3 = icmp eq i64 %x, 0
+  %4 = select i1 %3, i64 %y, i64 %2
+  ret i64 %4
+}
+
+define i128 @cmov_bsr128(i128 %x, i128 %y) nounwind {
+; X86-LABEL: cmov_bsr128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    orl %ebp, %edx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    orl %ebx, %esi
+; X86-NEXT:    orl %edx, %esi
+; X86-NEXT:    je .LBB8_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    testl %ebp, %ebp
+; X86-NEXT:    jne .LBB8_3
+; X86-NEXT:  # %bb.4: # %cond.false
+; X86-NEXT:    bsrl %ebx, %edx
+; X86-NEXT:    xorl $31, %edx
+; X86-NEXT:    orl $32, %edx
+; X86-NEXT:    testl %edi, %edi
+; X86-NEXT:    je .LBB8_7
+; X86-NEXT:  .LBB8_6:
+; X86-NEXT:    bsrl %edi, %esi
+; X86-NEXT:    xorl $31, %esi
+; X86-NEXT:    jmp .LBB8_8
+; X86-NEXT:  .LBB8_1:
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    movl $0, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    movl $128, %edx
+; X86-NEXT:    jmp .LBB8_11
+; X86-NEXT:  .LBB8_3:
+; X86-NEXT:    bsrl %ebp, %edx
+; X86-NEXT:    xorl $31, %edx
+; X86-NEXT:    testl %edi, %edi
+; X86-NEXT:    jne .LBB8_6
+; X86-NEXT:  .LBB8_7: # %cond.false
+; X86-NEXT:    bsrl %ecx, %esi
+; X86-NEXT:    xorl $31, %esi
+; X86-NEXT:    orl $32, %esi
+; X86-NEXT:  .LBB8_8: # %cond.false
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    orl %ebp, %ebx
+; X86-NEXT:    jne .LBB8_10
+; X86-NEXT:  # %bb.9: # %cond.false
+; X86-NEXT:    orl $64, %esi
+; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:  .LBB8_10: # %cond.false
+; X86-NEXT:    movl $0, (%esp) # 4-byte Folded Spill
+; X86-NEXT:  .LBB8_11: # %cond.end
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    orl %eax, %ecx
+; X86-NEXT:    orl %ebp, %edi
+; X86-NEXT:    orl %ecx, %edi
+; X86-NEXT:    je .LBB8_12
+; X86-NEXT:  # %bb.13: # %cond.end
+; X86-NEXT:    xorl $127, %edx
+; X86-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT:    jmp .LBB8_14
+; X86-NEXT:  .LBB8_12:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:  .LBB8_14: # %cond.end
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %ebx, 12(%eax)
+; X86-NEXT:    movl %esi, 8(%eax)
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl %edx, (%eax)
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: cmov_bsr128:
+; X64:       # %bb.0:
+; X64-NEXT:    bsrq %rsi, %r8
+; X64-NEXT:    xorq $63, %r8
+; X64-NEXT:    movl $127, %eax
+; X64-NEXT:    bsrq %rdi, %rax
+; X64-NEXT:    xorq $63, %rax
+; X64-NEXT:    addq $64, %rax
+; X64-NEXT:    testq %rsi, %rsi
+; X64-NEXT:    cmovneq %r8, %rax
+; X64-NEXT:    xorq $127, %rax
+; X64-NEXT:    xorl %r8d, %r8d
+; X64-NEXT:    orq %rsi, %rdi
+; X64-NEXT:    cmoveq %rdx, %rax
+; X64-NEXT:    cmoveq %rcx, %r8
+; X64-NEXT:    movq %r8, %rdx
+; X64-NEXT:    retq
+  %1 = tail call i128 @llvm.ctlz.i128(i128 %x, i1 false)
+  %2 = xor i128 %1, 127
+  %3 = icmp eq i128 %x, 0
+  %4 = select i1 %3, i128 %y, i128 %2
+  ret i128 %4
+}
+
+define i128 @cmov_bsr128_undef(i128 %x, i128 %y) nounwind {
+; X86-LABEL: cmov_bsr128_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    testl %edi, %edi
+; X86-NEXT:    jne .LBB9_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    bsrl %esi, %ecx
+; X86-NEXT:    xorl $31, %ecx
+; X86-NEXT:    orl $32, %ecx
+; X86-NEXT:    jmp .LBB9_3
+; X86-NEXT:  .LBB9_1:
+; X86-NEXT:    bsrl %edi, %ecx
+; X86-NEXT:    xorl $31, %ecx
+; X86-NEXT:  .LBB9_3:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    testl %edx, %edx
+; X86-NEXT:    jne .LBB9_4
+; X86-NEXT:  # %bb.5:
+; X86-NEXT:    bsrl %ebx, %ebp
+; X86-NEXT:    xorl $31, %ebp
+; X86-NEXT:    orl $32, %ebp
+; X86-NEXT:    jmp .LBB9_6
+; X86-NEXT:  .LBB9_4:
+; X86-NEXT:    bsrl %edx, %ebp
+; X86-NEXT:    xorl $31, %ebp
+; X86-NEXT:  .LBB9_6:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl %edi, %esi
+; X86-NEXT:    jne .LBB9_8
+; X86-NEXT:  # %bb.7:
+; X86-NEXT:    orl $64, %ebp
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:  .LBB9_8:
+; X86-NEXT:    orl %edi, %edx
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    orl %edx, %ebx
+; X86-NEXT:    jne .LBB9_9
+; X86-NEXT:  # %bb.10:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    jmp .LBB9_11
+; X86-NEXT:  .LBB9_9:
+; X86-NEXT:    xorl $127, %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    xorl %edi, %edi
+; X86-NEXT:  .LBB9_11:
+; X86-NEXT:    movl %edi, 12(%eax)
+; X86-NEXT:    movl %esi, 8(%eax)
+; X86-NEXT:    movl %edx, 4(%eax)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: cmov_bsr128_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsrq %rsi, %r8
+; X64-NEXT:    xorq $63, %r8
+; X64-NEXT:    bsrq %rdi, %rax
+; X64-NEXT:    xorq $63, %rax
+; X64-NEXT:    orq $64, %rax
+; X64-NEXT:    testq %rsi, %rsi
+; X64-NEXT:    cmovneq %r8, %rax
+; X64-NEXT:    xorq $127, %rax
+; X64-NEXT:    xorl %r8d, %r8d
+; X64-NEXT:    orq %rsi, %rdi
+; X64-NEXT:    cmoveq %rdx, %rax
+; X64-NEXT:    cmoveq %rcx, %r8
+; X64-NEXT:    movq %r8, %rdx
+; X64-NEXT:    retq
+  %1 = tail call i128 @llvm.ctlz.i128(i128 %x, i1 true)
+  %2 = xor i128 %1, 127
+  %3 = icmp ne i128 %x, 0
+  %4 = select i1 %3, i128 %2, i128 %y
+  ret i128 %4
+}
+
+declare i8 @llvm.ctlz.i8(i8, i1)
+declare i16 @llvm.ctlz.i16(i16, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i128 @llvm.ctlz.i128(i128, i1)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/build-vector-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/build-vector-512.ll
index 789196c5e484..69d17fe3ab69 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/build-vector-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/build-vector-512.ll
@@ -578,7 +578,7 @@ define <16 x float> @test_buildvector_16f32_2_var(float %a0, float %a1) {
 ; AVX-32-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [0,17,0,0]
 ; AVX-32-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX-32-NEXT:    vpermi2ps %zmm0, %zmm2, %zmm1
-; AVX-32-NEXT:    vblendps {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3]
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3]
 ; AVX-32-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX-32-NEXT:    vinsertps {{.*#+}} xmm3 = xmm0[0,1,2],xmm2[0]
 ; AVX-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[2,3]
@@ -626,7 +626,7 @@ define <16 x float> @test_buildvector_16f32_2_load(ptr %p0, ptr %p1) {
 ; AVX-32-NEXT:    vbroadcastss (%ecx), %xmm1
 ; AVX-32-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX-32-NEXT:    vpermi2ps %zmm1, %zmm2, %zmm0
-; AVX-32-NEXT:    vblendps {{.*#+}} xmm3 = xmm2[0],xmm1[1,2,3]
+; AVX-32-NEXT:    vmovss {{.*#+}} xmm3 = xmm2[0],xmm1[1,2,3]
 ; AVX-32-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-32-NEXT:    vinsertps {{.*#+}} xmm3 = xmm1[0,1,2],xmm2[0]
 ; AVX-32-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
@@ -640,7 +640,7 @@ define <16 x float> @test_buildvector_16f32_2_load(ptr %p0, ptr %p1) {
 ; AVX-64-NEXT:    vbroadcastss (%rdi), %xmm1
 ; AVX-64-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX-64-NEXT:    vpermi2ps %zmm1, %zmm2, %zmm0
-; AVX-64-NEXT:    vblendps {{.*#+}} xmm3 = xmm2[0],xmm1[1,2,3]
+; AVX-64-NEXT:    vmovss {{.*#+}} xmm3 = xmm2[0],xmm1[1,2,3]
 ; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-64-NEXT:    vinsertps {{.*#+}} xmm3 = xmm1[0,1,2],xmm2[0]
 ; AVX-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/buildvec-extract.ll b/external/llvm-project/llvm/test/CodeGen/X86/buildvec-extract.ll
index 545c57fed4b2..9d856ed7647c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/buildvec-extract.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/buildvec-extract.ll
@@ -42,7 +42,7 @@ define <2 x i64> @extract0_i32_zext_insert0_i64_zero(<4 x i32> %x) {
 ; AVX-LABEL: extract0_i32_zext_insert0_i64_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 0
   %z = zext i32 %e to i64
@@ -85,7 +85,7 @@ define <2 x i64> @extract1_i32_zext_insert0_i64_zero(<4 x i32> %x) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 1
   %z = zext i32 %e to i64
@@ -130,7 +130,7 @@ define <2 x i64> @extract2_i32_zext_insert0_i64_zero(<4 x i32> %x) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 2
   %z = zext i32 %e to i64
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll b/external/llvm-project/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
index 345014edd0e9..5e94598565aa 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
@@ -7,7 +7,6 @@
 define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
 ; SSE2-LABEL: dot_ext_v8i8_v8i32:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    pushq %r14
 ; SSE2-NEXT:    pushq %rbx
 ; SSE2-NEXT:    movzbl (%rdi), %eax
 ; SSE2-NEXT:    movzbl (%rdi,%rsi), %ecx
@@ -18,9 +17,9 @@ define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
 ; SSE2-NEXT:    leaq (%rsi,%rsi,4), %rbx
 ; SSE2-NEXT:    movzbl (%rdi,%rbx), %ebx
 ; SSE2-NEXT:    movzbl (%rdi,%r9,2), %r9d
-; SSE2-NEXT:    leaq (,%rsi,8), %r14
-; SSE2-NEXT:    subq %rsi, %r14
-; SSE2-NEXT:    movzbl (%rdi,%r14), %esi
+; SSE2-NEXT:    leaq (%rdi,%rsi,8), %rdi
+; SSE2-NEXT:    subq %rsi, %rdi
+; SSE2-NEXT:    movzbl (%rdi), %esi
 ; SSE2-NEXT:    shll $16, %ecx
 ; SSE2-NEXT:    orl %eax, %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm0
@@ -38,7 +37,6 @@ define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    popq %rbx
-; SSE2-NEXT:    popq %r14
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: dot_ext_v8i8_v8i32:
@@ -46,7 +44,7 @@ define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
 ; SSE4-NEXT:    movzbl (%rdi), %eax
 ; SSE4-NEXT:    leaq (%rsi,%rsi,4), %rcx
 ; SSE4-NEXT:    leaq (%rsi,%rsi,2), %r8
-; SSE4-NEXT:    leaq (,%rsi,8), %r9
+; SSE4-NEXT:    leaq (%rdi,%rsi,8), %r9
 ; SSE4-NEXT:    subq %rsi, %r9
 ; SSE4-NEXT:    movd %eax, %xmm0
 ; SSE4-NEXT:    pinsrb $2, (%rdi,%rsi), %xmm0
@@ -55,7 +53,7 @@ define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
 ; SSE4-NEXT:    pinsrb $8, (%rdi,%rsi,4), %xmm0
 ; SSE4-NEXT:    pinsrb $10, (%rdi,%rcx), %xmm0
 ; SSE4-NEXT:    pinsrb $12, (%rdi,%r8,2), %xmm0
-; SSE4-NEXT:    pinsrb $14, (%rdi,%r9), %xmm0
+; SSE4-NEXT:    pinsrb $14, (%r9), %xmm0
 ; SSE4-NEXT:    movdqu (%rdx), %xmm1
 ; SSE4-NEXT:    pmaddwd %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
@@ -70,7 +68,7 @@ define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
 ; AVX-NEXT:    movzbl (%rdi), %eax
 ; AVX-NEXT:    leaq (%rsi,%rsi,2), %rcx
 ; AVX-NEXT:    leaq (%rsi,%rsi,4), %r8
-; AVX-NEXT:    leaq (,%rsi,8), %r9
+; AVX-NEXT:    leaq (%rdi,%rsi,8), %r9
 ; AVX-NEXT:    subq %rsi, %r9
 ; AVX-NEXT:    vmovd %eax, %xmm0
 ; AVX-NEXT:    vpinsrb $2, (%rdi,%rsi), %xmm0, %xmm0
@@ -79,7 +77,7 @@ define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
 ; AVX-NEXT:    vpinsrb $8, (%rdi,%rsi,4), %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrb $10, (%rdi,%r8), %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrb $12, (%rdi,%rcx,2), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $14, (%rdi,%r9), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $14, (%r9), %xmm0, %xmm0
 ; AVX-NEXT:    vpmaddwd (%rdx), %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
index 7f91b846447b..db27132d4055 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
@@ -50,7 +50,7 @@ define i32 @foo(i32 %arg, ptr %arg3) nounwind {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  # %bb.4: # %bb17
 ; CHECK-NEXT:    callq widget@PLT
-; CHECK-NEXT:  .LBB0_5: # Block address taken
+; CHECK-NEXT:  .LBB0_5: # Inline asm indirect target
 ; CHECK-NEXT:    # %bb18
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movw $0, 14(%r14)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
index 7ea3e2ce0a7b..3d389523dffb 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
@@ -53,7 +53,7 @@ define dso_local void @n(ptr %o, i32 %p, i32 %u) nounwind {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    jmp .LBB0_9
-; CHECK-NEXT:  .LBB0_7: # Block address taken
+; CHECK-NEXT:  .LBB0_7: # Inline asm indirect target
 ; CHECK-NEXT:    # %if.then20.critedge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl j(%rip), %edi
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-destinations.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-destinations.ll
index 38238d7b254a..71e45f0bab42 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-destinations.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-destinations.ll
@@ -10,7 +10,7 @@ define i32 @duplicate_normal_and_indirect_dest(i32 %a) {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    jmp .LBB0_1
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB0_1: # Block address taken
+; CHECK-NEXT:  .LBB0_1: # Inline asm indirect target
 ; CHECK-NEXT:    # %fail
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $1, %eax
@@ -35,7 +35,7 @@ define i32 @duplicate_indirect_dest(i32 %a) {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  # %bb.1: # %normal
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB1_2: # Block address taken
+; CHECK-NEXT:  .LBB1_2: # Inline asm indirect target
 ; CHECK-NEXT:    # %fail
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $1, %eax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-endbr.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-endbr.ll
new file mode 100644
index 000000000000..133de89d5f3a
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-endbr.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+define i32 @test1(i32 %a) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    endbr64
+; CHECK-NEXT:    addl $4, %edi
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    jmp .LBB0_2
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  # %bb.1: # %normal
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_2: # Inline asm indirect target
+; CHECK-NEXT:    # %fail
+; CHECK-NEXT:    # Label of block must be emitted
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    retq
+entry:
+  %0 = add i32 %a, 4
+  callbr void asm "xorl $0, $0; jmp ${1:l}", "r,!i,~{dirflag},~{fpsr},~{flags}"(i32 %0) to label %normal [label %fail]
+
+normal:
+  ret i32 0
+
+fail:
+  ret i32 1
+}
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 8, !"cf-protection-branch", i32 1}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll
index 103e6a694c33..b369662f8530 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-label-addr.ll
@@ -8,7 +8,7 @@ define i32 @test1(i32 %x) {
 ; CHECK-NEXT:    .quad .Ltmp0
 ; CHECK-NEXT:    .quad .LBB0_1
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB0_1: # Block address taken
+; CHECK-NEXT:  .LBB0_1: # Inline asm indirect target
 ; CHECK-NEXT:    # %bar
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    pushq %rax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel-m32.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel-m32.ll
index ee3889dec005..187298e7b815 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel-m32.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel-m32.ll
@@ -22,7 +22,7 @@ define i8 @emulator_cmpxchg_emulated() {
   ; CHECK-NEXT:   $al = COPY [[SETCCr]]
   ; CHECK-NEXT:   RET 0, $al
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.efaultu64.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.efaultu64.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
   ; CHECK-NEXT:   $al = COPY [[SETCCr1]]
   ; CHECK-NEXT:   RET 0, $al
@@ -62,7 +62,7 @@ define i32 @emulator_cmpxchg_emulated2() {
   ; CHECK-NEXT:   $eax = COPY [[COPY2]]
   ; CHECK-NEXT:   RET 0, $eax
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.efaultu64.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.efaultu64.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   $eax = COPY %3
   ; CHECK-NEXT:   RET 0, $eax
 entry:
@@ -99,7 +99,7 @@ define i64 @multireg() {
   ; CHECK-NEXT:   $edx = COPY [[COPY2]]
   ; CHECK-NEXT:   RET 0, $eax, $edx
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   liveins: $eax, $edx
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gr32 = COPY $eax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll
index 34f822ef5285..433bd254317e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll
@@ -19,7 +19,7 @@ define i32 @test0() {
   ; CHECK-NEXT:   $eax = COPY [[MOV32ri]]
   ; CHECK-NEXT:   RET 0, $eax
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.z.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   $eax = COPY %1
   ; CHECK-NEXT:   RET 0, $eax
   %direct = callbr i32 asm "", "=r,!i"()
@@ -43,7 +43,7 @@ define i32 @test1() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY %4
   ; CHECK-NEXT:   JMP_1 %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.z.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY %4
@@ -77,7 +77,7 @@ define i32 @test2() {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY %5
   ; CHECK-NEXT:   JMP_1 %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.z.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY %5
@@ -112,7 +112,7 @@ define i32 @test3() {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]]
   ; CHECK-NEXT:   JMP_1 %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.z.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT:   liveins: $ebx
   ; CHECK-NEXT: {{  $}}
@@ -150,7 +150,7 @@ define i32 @test4() {
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY]]
   ; CHECK-NEXT:   JMP_1 %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.z.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT:   liveins: $ebx, $edx
   ; CHECK-NEXT: {{  $}}
@@ -187,7 +187,7 @@ define i32 @test5() {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]]
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.cleanup (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.cleanup (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   liveins: $ebx
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr32 = COPY $ebx
@@ -227,7 +227,7 @@ define i64 @test6() {
   ; CHECK-NEXT:   $rax = COPY [[PHI]]
   ; CHECK-NEXT:   RET 0, $rax
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.3.foo.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.3.foo.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT:   liveins: $rdx
   ; CHECK-NEXT: {{  $}}
@@ -235,7 +235,7 @@ define i64 @test6() {
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:gr64 = COPY [[COPY4]]
   ; CHECK-NEXT:   JMP_1 %bb.2
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.4.foo.split2 (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.4.foo.split2 (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT:   liveins: $rbx
   ; CHECK-NEXT: {{  $}}
@@ -286,7 +286,7 @@ define i32 @test7() {
   ; CHECK-NEXT:   $eax = COPY [[COPY2]]
   ; CHECK-NEXT:   RET 0, $eax
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.3.retry.split (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.3.retry.split (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT:   liveins: $edx
   ; CHECK-NEXT: {{  $}}
@@ -321,7 +321,7 @@ define i32 @test8() {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY %1
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1.cleanup (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.1.cleanup (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   $eax = COPY %1
   ; CHECK-NEXT:   RET 0, $eax
 entry:
@@ -348,7 +348,7 @@ define i64 @condition_code() {
   ; CHECK-NEXT:   $rax = COPY [[SUBREG_TO_REG]]
   ; CHECK-NEXT:   RET 0, $rax
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2.c (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.2.c (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
   ; CHECK-NEXT:   [[MOVZX32rr8_1:%[0-9]+]]:gr32 = MOVZX32rr8 killed [[SETCCr1]]
   ; CHECK-NEXT:   [[SUBREG_TO_REG1:%[0-9]+]]:gr64 = SUBREG_TO_REG 0, killed [[MOVZX32rr8_1]], %subreg.sub_32bit
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll
index 64f062bed902..1504516d60cd 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll
@@ -17,12 +17,12 @@
 
 ; Check the second INLINEASM_BR target block is preceded by the block with the
 ; second INLINEASM_BR.
-; CHECK: bb.2 (%ir-block.7, machine-block-address-taken, inlineasm-br-indirect-target):
+; CHECK: bb.2 (%ir-block.7, inlineasm-br-indirect-target):
 ; CHECK-NEXT: predecessors: %bb.1
 
 ; Check the first INLINEASM_BR target block is predecessed by the block with
 ; the first INLINEASM_BR.
-; CHECK: bb.4 (%ir-block.12, machine-block-address-taken, inlineasm-br-indirect-target):
+; CHECK: bb.4 (%ir-block.12, inlineasm-br-indirect-target):
 ; CHECK-NEXT: predecessors: %bb.0
 
 @.str = private unnamed_addr constant [26 x i8] c"inline asm#1 returned %d\0A\00", align 1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs.ll
index aadbda1716ba..6ee43099209d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-outputs.ll
@@ -14,7 +14,7 @@ define i32 @test1(i32 %x) {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  # %bb.1: # %normal
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB0_2: # Block address taken
+; CHECK-NEXT:  .LBB0_2: # Inline asm indirect target
 ; CHECK-NEXT:    # %abnormal
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $1, %eax
@@ -47,7 +47,7 @@ define i32 @test2(i32 %out1, i32 %out2) nounwind {
 ; CHECK-NEXT:    jne .LBB1_2
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    jmp .LBB1_4
-; CHECK-NEXT:  .LBB1_2: # Block address taken
+; CHECK-NEXT:  .LBB1_2: # Inline asm indirect target
 ; CHECK-NEXT:    # %if.then.label_true_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    jmp .LBB1_9
@@ -64,15 +64,15 @@ define i32 @test2(i32 %out1, i32 %out2) nounwind {
 ; CHECK-NEXT:    popl %esi
 ; CHECK-NEXT:    popl %edi
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB1_6: # Block address taken
+; CHECK-NEXT:  .LBB1_6: # Inline asm indirect target
 ; CHECK-NEXT:    # %if.then.return_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
-; CHECK-NEXT:  .LBB1_7: # Block address taken
+; CHECK-NEXT:  .LBB1_7: # Inline asm indirect target
 ; CHECK-NEXT:    # %if.else.return_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $-1, %eax
 ; CHECK-NEXT:    jmp .LBB1_5
-; CHECK-NEXT:  .LBB1_8: # Block address taken
+; CHECK-NEXT:  .LBB1_8: # Inline asm indirect target
 ; CHECK-NEXT:    # %if.else.label_true_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:  .LBB1_9: # %label_true
@@ -130,10 +130,10 @@ define i32 @test3(i1 %cmp) nounwind {
 ; CHECK-NEXT:    popl %esi
 ; CHECK-NEXT:    popl %edi
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB2_5: # Block address taken
+; CHECK-NEXT:  .LBB2_5: # Inline asm indirect target
 ; CHECK-NEXT:    # %true.indirect_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
-; CHECK-NEXT:  .LBB2_6: # Block address taken
+; CHECK-NEXT:  .LBB2_6: # Inline asm indirect target
 ; CHECK-NEXT:    # %false.indirect_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $42, %eax
@@ -175,18 +175,18 @@ define i32 @test4(i32 %out1, i32 %out2) {
 ; CHECK-NEXT:  # %bb.2: # %asm.fallthrough2
 ; CHECK-NEXT:    addl %ecx, %eax
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB3_3: # Block address taken
+; CHECK-NEXT:  .LBB3_3: # Inline asm indirect target
 ; CHECK-NEXT:    # %entry.return_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
-; CHECK-NEXT:  .LBB3_4: # Block address taken
+; CHECK-NEXT:  .LBB3_4: # Inline asm indirect target
 ; CHECK-NEXT:    # %asm.fallthrough.return_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $-1, %eax
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB3_5: # Block address taken
+; CHECK-NEXT:  .LBB3_5: # Inline asm indirect target
 ; CHECK-NEXT:    # %entry.label_true_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
-; CHECK-NEXT:  .LBB3_6: # Block address taken
+; CHECK-NEXT:  .LBB3_6: # Inline asm indirect target
 ; CHECK-NEXT:    # %asm.fallthrough.label_true_crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $-2, %eax
@@ -226,7 +226,7 @@ define dso_local void @test5() {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB4_2: # Block address taken
+; CHECK-NEXT:  .LBB4_2: # Inline asm indirect target
 ; CHECK-NEXT:    # %._crit_edge
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    retl
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-phi-placement.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-phi-placement.ll
index 43776bfac462..1a4f24e3c5b4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-phi-placement.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-phi-placement.ll
@@ -16,7 +16,7 @@ define void @test1(ptr %arg, ptr %mem) nounwind {
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  .LBB0_1: # Block address taken
+; CHECK-NEXT:  .LBB0_1: # Inline asm indirect target
 ; CHECK-NEXT:    # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    # Label of block must be emitted
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-sink.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-sink.ll
index a563838cfdf5..c0a501fba091 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-sink.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm-sink.ll
@@ -16,7 +16,7 @@ define void @klist_dec_and_del(ptr) {
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB0_1: # Block address taken
+; CHECK-NEXT:  .LBB0_1: # Inline asm indirect target
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movq $0, -8(%rax)
 ; CHECK-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm.ll b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm.ll
index 65dc635e43cd..16b23fa81e34 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/callbr-asm.ll
@@ -17,7 +17,7 @@ define i32 @test1(i32 %a) {
 ; CHECK-NEXT:  # %bb.1: # %normal
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB0_2: # Block address taken
+; CHECK-NEXT:  .LBB0_2: # Inline asm indirect target
 ; CHECK-NEXT:    # %fail
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $1, %eax
@@ -48,7 +48,7 @@ define i32 @test1b(i32 %a) {
 ; CHECK-NEXT:  # %bb.1: # %normal
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:  .LBB1_2: # Block address taken
+; CHECK-NEXT:  .LBB1_2: # Inline asm indirect target
 ; CHECK-NEXT:    # %fail
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    movl $1, %eax
@@ -91,14 +91,14 @@ fail:
 define i32 @test3(i32 %a) {
 ; CHECK-LABEL: test3:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:  .LBB3_1: # Block address taken
+; CHECK-NEXT:  .LBB3_1: # Inline asm indirect target
 ; CHECK-NEXT:    # %label01
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB3_2 Depth 2
 ; CHECK-NEXT:    # Child Loop BB3_3 Depth 3
 ; CHECK-NEXT:    # Child Loop BB3_4 Depth 4
 ; CHECK-NEXT:    # Label of block must be emitted
-; CHECK-NEXT:  .LBB3_2: # Block address taken
+; CHECK-NEXT:  .LBB3_2: # Inline asm indirect target
 ; CHECK-NEXT:    # %label02
 ; CHECK-NEXT:    # Parent Loop BB3_1 Depth=1
 ; CHECK-NEXT:    # => This Loop Header: Depth=2
@@ -106,14 +106,14 @@ define i32 @test3(i32 %a) {
 ; CHECK-NEXT:    # Child Loop BB3_4 Depth 4
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    addl $4, {{[0-9]+}}(%esp)
-; CHECK-NEXT:  .LBB3_3: # Block address taken
+; CHECK-NEXT:  .LBB3_3: # Inline asm indirect target
 ; CHECK-NEXT:    # %label03
 ; CHECK-NEXT:    # Parent Loop BB3_1 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB3_2 Depth=2
 ; CHECK-NEXT:    # => This Loop Header: Depth=3
 ; CHECK-NEXT:    # Child Loop BB3_4 Depth 4
 ; CHECK-NEXT:    # Label of block must be emitted
-; CHECK-NEXT:  .LBB3_4: # Block address taken
+; CHECK-NEXT:  .LBB3_4: # Inline asm indirect target
 ; CHECK-NEXT:    # %label04
 ; CHECK-NEXT:    # Parent Loop BB3_1 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB3_2 Depth=2
@@ -177,7 +177,7 @@ define void @test4() {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    ja .LBB4_3
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB4_3: # Block address taken
+; CHECK-NEXT:  .LBB4_3: # Inline asm indirect target
 ; CHECK-NEXT:    # %quux
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    retl
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll b/external/llvm-project/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll
index 556b0deaf4c8..8b3aa2964db0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/canonicalize-vars-f16-type.ll
@@ -51,7 +51,7 @@ define void @v_test_canonicalize__half(half addrspace(1)* %out) nounwind {
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX512-NEXT:    retq
@@ -149,7 +149,7 @@ define half @complex_canonicalize_fmul_half(half %a, half %b) nounwind {
 ; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vsubss %xmm1, %xmm0, %xmm0
@@ -235,12 +235,12 @@ define void @v_test_canonicalize_v2half(<2 x half> addrspace(1)* %out) nounwind
 ; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
 ; AVX512-NEXT:    vmulss %xmm1, %xmm2, %xmm2
 ; AVX512-NEXT:    vxorps %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm3[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm3[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; AVX512-NEXT:    vmovd %xmm0, (%rdi)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/catchret-empty-fallthrough.ll b/external/llvm-project/llvm/test/CodeGen/X86/catchret-empty-fallthrough.ll
index 437d9698ee6b..ab9fa2287ffa 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/catchret-empty-fallthrough.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/catchret-empty-fallthrough.ll
@@ -44,7 +44,7 @@ return:                                           ; preds = %catch, %entry
 ; CHECK: .LBB0_[[catch:[0-9]+]]:
 
 ; CHECK: .seh_handlerdata
-; CHECK-NEXT: .set .Lfoo$parent_frame_offset, 32
+; CHECK-NEXT: .Lfoo$parent_frame_offset = 32
 ; CHECK-NEXT: .long   (.Llsda_end0-.Llsda_begin0)/16
 ; CHECK-NEXT: .Llsda_begin0:
 ; CHECK-NEXT: .long   .Ltmp0@IMGREL
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/coalesce_commute_movsd.ll b/external/llvm-project/llvm/test/CodeGen/X86/coalesce_commute_movsd.ll
index b42fd957d7f4..441c79b3fc31 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/coalesce_commute_movsd.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/coalesce_commute_movsd.ll
@@ -19,12 +19,12 @@ define <2 x double> @insert_f64(double %a0, <2 x double> %a1) {
 ;
 ; AVX-LABEL: insert_f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: insert_f64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX512-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX512-NEXT:    retq
  %1 = insertelement <2 x double> %a1, double %a0, i32 0
  ret <2 x double> %1
@@ -44,12 +44,12 @@ define <4 x float> @insert_f32(float %a0, <4 x float> %a1) {
 ;
 ; AVX-LABEL: insert_f32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: insert_f32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    retq
  %1 = insertelement <4 x float> %a1, float %a0, i32 0
  ret <4 x float> %1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/coalescer-commute1.ll b/external/llvm-project/llvm/test/CodeGen/X86/coalescer-commute1.ll
index 28502782cf64..f4decb7e2e0c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/coalescer-commute1.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/coalescer-commute1.ll
@@ -16,7 +16,7 @@ define void @runcont(ptr %source) nounwind  {
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %bb
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vcvtsi2ssl (%eax,%edx,4), %xmm2, %xmm1
+; CHECK-NEXT:    vcvtsi2ssl (%eax,%edx,4), %xmm7, %xmm1
 ; CHECK-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    incl %edx
 ; CHECK-NEXT:    cmpl %edx, %ecx
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/coff-alias-type.ll b/external/llvm-project/llvm/test/CodeGen/X86/coff-alias-type.ll
index a242cd2d77d7..6cc0638b2d4a 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/coff-alias-type.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/coff-alias-type.ll
@@ -22,4 +22,4 @@ entry:
 ; CHECK-NEXT: .scl     2
 ; CHECK-NEXT: .type    32
 ; CHECK-NEXT: .endef
-; CHECK-NEXT: .set     _ZN8MyStructC1Ev, _ZN8MyStructC2Ev
+; CHECK-NEXT: _ZN8MyStructC1Ev = _ZN8MyStructC2Ev
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/coff-comdat.ll b/external/llvm-project/llvm/test/CodeGen/X86/coff-comdat.ll
index 99b3c0a687af..084a5a71125e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/coff-comdat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/coff-comdat.ll
@@ -89,4 +89,4 @@ $vftable = comdat largest
 ; CHECK: .globl  _f6
 ; CHECK: .section        .rdata,"dr",largest,_vftable
 ; CHECK: .globl  _vftable
-; CHECK: .set _vftable, L_some_name+4
+; CHECK: _vftable = L_some_name+4
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/coff-feat00.ll b/external/llvm-project/llvm/test/CodeGen/X86/coff-feat00.ll
index 21dd04ed34c7..1dcd4276399a 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/coff-feat00.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/coff-feat00.ll
@@ -4,4 +4,4 @@ define i32 @foo() {
   ret i32 0
 }
 
-; CHECK: .set @feat.00, 1
+; CHECK: @feat.00 = 1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/combine-and.ll b/external/llvm-project/llvm/test/CodeGen/X86/combine-and.ll
index e5594dc9c5e3..a476b21979ce 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/combine-and.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/combine-and.ll
@@ -37,7 +37,7 @@ define <4 x i32> @test1(<4 x i32> %A) {
 ; AVX-LABEL: test1:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %1 = and <4 x i32> %A, <i32 -1, i32 0, i32 0, i32 0>
   ret <4 x i32> %1
@@ -127,7 +127,7 @@ define <4 x i32> @test7(<4 x i32> %A) {
 ; SSE-LABEL: test7:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    xorps %xmm1, %xmm1
-; SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test7:
@@ -189,13 +189,13 @@ define <4 x i32> @test11(<4 x i32> %A) {
 ; SSE-LABEL: test11:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    xorps %xmm1, %xmm1
-; SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test11:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %1 = and <4 x i32> %A, <i32 0, i32 -1, i32 -1, i32 -1>
   ret <4 x i32> %1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/combine-bitselect.ll b/external/llvm-project/llvm/test/CodeGen/X86/combine-bitselect.ll
index 25c26d598881..4f1c00b64fa9 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/combine-bitselect.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/combine-bitselect.ll
@@ -283,7 +283,7 @@ define <2 x i64> @bitselect_v2i64_broadcast_rrm(<2 x i64> %a0, <2 x i64> %a1, pt
 ;
 ; AVX512VL-LABEL: bitselect_v2i64_broadcast_rrm:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm1 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm1 ^ (m64bcst & (xmm0 ^ xmm1))
 ; AVX512VL-NEXT:    retq
   %a2 = load i64, ptr %p2
   %1 = insertelement <2 x i64> undef, i64 %a2, i32 0
@@ -604,7 +604,7 @@ define <4 x i64> @bitselect_v4i64_broadcast_rrm(<4 x i64> %a0, <4 x i64> %a1, pt
 ;
 ; AVX512VL-LABEL: bitselect_v4i64_broadcast_rrm:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VL-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm1 ^ (m64bcst & (ymm0 ^ ymm1))
 ; AVX512VL-NEXT:    retq
   %a2 = load i64, ptr %p2
   %1 = insertelement <4 x i64> undef, i64 %a2, i32 0
@@ -975,7 +975,7 @@ define <8 x i64> @bitselect_v8i64_broadcast_rrm(<8 x i64> %a0, <8 x i64> %a1, pt
 ;
 ; AVX512-LABEL: bitselect_v8i64_broadcast_rrm:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm1 ^ (mem & (zmm0 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm1 ^ (m64bcst & (zmm0 ^ zmm1))
 ; AVX512-NEXT:    retq
   %a2 = load i64, ptr %p2
   %1 = insertelement <8 x i64> undef, i64 %a2, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/combine-mask-with-shuffle.ll b/external/llvm-project/llvm/test/CodeGen/X86/combine-mask-with-shuffle.ll
index 268ac3dd31b8..7564e65a428b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/combine-mask-with-shuffle.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/combine-mask-with-shuffle.ll
@@ -67,11 +67,9 @@ define <16 x i32> @combine_mask_with_abs(<16 x i32> %v0) {
 define <16 x i32> @combine_mask_with_umin(<16 x i32> %v0) {
 ; CHECK-LABEL: combine_mask_with_umin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; CHECK-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; CHECK-NEXT:    kmovw %eax, %k1
-; CHECK-NEXT:    vpblendmd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vpminud %zmm2, %zmm1, %zmm1
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm2 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
+; CHECK-NEXT:    vpminud %zmm1, %zmm2, %zmm1
 ; CHECK-NEXT:    movw $-3856, %ax # imm = 0xF0F0
 ; CHECK-NEXT:    kmovw %eax, %k1
 ; CHECK-NEXT:    vpopcntd %zmm0, %zmm1 {%k1}
@@ -88,11 +86,9 @@ define <16 x i32> @combine_mask_with_umin(<16 x i32> %v0) {
 define <16 x i32> @combine_mask_with_umax(<16 x i32> %v0) {
 ; CHECK-LABEL: combine_mask_with_umax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; CHECK-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; CHECK-NEXT:    kmovw %eax, %k1
-; CHECK-NEXT:    vpblendmd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vpmaxud %zmm2, %zmm1, %zmm1
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm2 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
+; CHECK-NEXT:    vpmaxud %zmm1, %zmm2, %zmm1
 ; CHECK-NEXT:    movw $-3856, %ax # imm = 0xF0F0
 ; CHECK-NEXT:    kmovw %eax, %k1
 ; CHECK-NEXT:    vpopcntd %zmm0, %zmm1 {%k1}
@@ -109,11 +105,9 @@ define <16 x i32> @combine_mask_with_umax(<16 x i32> %v0) {
 define <16 x i32> @combine_mask_with_smin(<16 x i32> %v0) {
 ; CHECK-LABEL: combine_mask_with_smin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; CHECK-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; CHECK-NEXT:    kmovw %eax, %k1
-; CHECK-NEXT:    vpblendmd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vpminsd %zmm2, %zmm1, %zmm1
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm2 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
+; CHECK-NEXT:    vpminsd %zmm1, %zmm2, %zmm1
 ; CHECK-NEXT:    movw $-3856, %ax # imm = 0xF0F0
 ; CHECK-NEXT:    kmovw %eax, %k1
 ; CHECK-NEXT:    vpopcntd %zmm0, %zmm1 {%k1}
@@ -130,11 +124,9 @@ define <16 x i32> @combine_mask_with_smin(<16 x i32> %v0) {
 define <16 x i32> @combine_mask_with_smax(<16 x i32> %v0) {
 ; CHECK-LABEL: combine_mask_with_smax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
-; CHECK-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; CHECK-NEXT:    kmovw %eax, %k1
-; CHECK-NEXT:    vpblendmd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vpmaxsd %zmm2, %zmm1, %zmm1
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; CHECK-NEXT:    vpshufd {{.*#+}} zmm2 = zmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
+; CHECK-NEXT:    vpmaxsd %zmm1, %zmm2, %zmm1
 ; CHECK-NEXT:    movw $-3856, %ax # imm = 0xF0F0
 ; CHECK-NEXT:    kmovw %eax, %k1
 ; CHECK-NEXT:    vpopcntd %zmm0, %zmm1 {%k1}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/combine-or-shuffle.ll b/external/llvm-project/llvm/test/CodeGen/X86/combine-or-shuffle.ll
index 95b5fcf8eac5..38ea796c0fcb 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/combine-or-shuffle.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/combine-or-shuffle.ll
@@ -31,15 +31,10 @@ define <2 x i64> @test1(<2 x i64> %a, <2 x i64> %b) {
 
 
 define <4 x i32> @test2(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test2:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test2:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2:
 ; AVX:       # %bb.0:
@@ -53,15 +48,10 @@ define <4 x i32> @test2(<4 x i32> %a, <4 x i32> %b) {
 
 
 define <2 x i64> @test3(<2 x i64> %a, <2 x i64> %b) {
-; SSE2-LABEL: test3:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test3:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test3:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test3:
 ; AVX:       # %bb.0:
@@ -86,10 +76,20 @@ define <4 x i32> @test4(<4 x i32> %a, <4 x i32> %b) {
 ; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; SSE4-NEXT:    retq
 ;
-; AVX-LABEL: test4:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX-NEXT:    retq
+; AVX1-LABEL: test4:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test4:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test4:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    retq
   %shuf1 = shufflevector <4 x i32> %a, <4 x i32> zeroinitializer, <4 x i32><i32 0, i32 4, i32 4, i32 4>
   %shuf2 = shufflevector <4 x i32> %b, <4 x i32> zeroinitializer, <4 x i32><i32 4, i32 1, i32 2, i32 3>
   %or = or <4 x i32> %shuf1, %shuf2
@@ -98,20 +98,25 @@ define <4 x i32> @test4(<4 x i32> %a, <4 x i32> %b) {
 
 
 define <4 x i32> @test5(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test5:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    retq
+; SSE-LABEL: test5:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
-; SSE4-LABEL: test5:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE4-NEXT:    retq
+; AVX1-LABEL: test5:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX1-NEXT:    retq
 ;
-; AVX-LABEL: test5:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; AVX-NEXT:    retq
+; AVX2-LABEL: test5:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test5:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX512-NEXT:    retq
   %shuf1 = shufflevector <4 x i32> %a, <4 x i32> zeroinitializer, <4 x i32><i32 4, i32 1, i32 2, i32 3>
   %shuf2 = shufflevector <4 x i32> %b, <4 x i32> zeroinitializer, <4 x i32><i32 0, i32 4, i32 4, i32 4>
   %or = or <4 x i32> %shuf1, %shuf2
@@ -186,15 +191,10 @@ define <2 x i64> @test8(<2 x i64> %a, <2 x i64> %b) {
 
 
 define <4 x i32> @test9(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test9:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test9:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test9:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test9:
 ; AVX:       # %bb.0:
@@ -208,15 +208,10 @@ define <4 x i32> @test9(<4 x i32> %a, <4 x i32> %b) {
 
 
 define <2 x i64> @test10(<2 x i64> %a, <2 x i64> %b) {
-; SSE2-LABEL: test10:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test10:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test10:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test10:
 ; AVX:       # %bb.0:
@@ -241,10 +236,20 @@ define <4 x i32> @test11(<4 x i32> %a, <4 x i32> %b) {
 ; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; SSE4-NEXT:    retq
 ;
-; AVX-LABEL: test11:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX-NEXT:    retq
+; AVX1-LABEL: test11:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test11:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test11:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    retq
   %and1 = and <4 x i32> %a, <i32 -1, i32 0, i32 0, i32 0>
   %and2 = and <4 x i32> %b, <i32 0, i32 -1, i32 -1, i32 -1>
   %or = or <4 x i32> %and1, %and2
@@ -253,20 +258,25 @@ define <4 x i32> @test11(<4 x i32> %a, <4 x i32> %b) {
 
 
 define <4 x i32> @test12(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test12:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    retq
+; SSE-LABEL: test12:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
-; SSE4-LABEL: test12:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE4-NEXT:    retq
+; AVX1-LABEL: test12:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX1-NEXT:    retq
 ;
-; AVX-LABEL: test12:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; AVX-NEXT:    retq
+; AVX2-LABEL: test12:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test12:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX512-NEXT:    retq
   %and1 = and <4 x i32> %a, <i32 0, i32 -1, i32 -1, i32 -1>
   %and2 = and <4 x i32> %b, <i32 -1, i32 0, i32 0, i32 0>
   %or = or <4 x i32> %and1, %and2
@@ -395,18 +405,18 @@ define <4 x i32> @test18(<4 x i32> %a, <4 x i32> %b) {
 ; AVX1-LABEL: test18:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,0,1,1]
-; AVX1-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test18:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,0,1,1]
-; AVX2-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
@@ -414,7 +424,7 @@ define <4 x i32> @test18(<4 x i32> %a, <4 x i32> %b) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %shuf1 = shufflevector <4 x i32> %a, <4 x i32> zeroinitializer, <4 x i32><i32 4, i32 0, i32 4, i32 4>
@@ -533,20 +543,25 @@ define <2 x i64> @test21(<2 x i64> %a, <2 x i64> %b) {
 ; bitcast to use the mask-or blend combine.
 
 define <2 x double> @test22(<2 x double> %a0, <2 x double> %a1) {
-; SSE2-LABEL: test22:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
+; SSE-LABEL: test22:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
-; SSE4-LABEL: test22:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; AVX1-LABEL: test22:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX1-NEXT:    retq
 ;
-; AVX-LABEL: test22:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX-NEXT:    retq
+; AVX2-LABEL: test22:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test22:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX512-NEXT:    retq
   %bc1 = bitcast <2 x double> %a0 to <2 x i64>
   %bc2 = bitcast <2 x double> %a1 to <2 x i64>
   %and1 = and <2 x i64> %bc1, <i64 0, i64 -1>
@@ -584,20 +599,25 @@ define <4 x float> @test23(<4 x float> %a0, <4 x float> %a1) {
 
 
 define <4 x float> @test24(<4 x float> %a0, <4 x float> %a1) {
-; SSE2-LABEL: test24:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
+; SSE-LABEL: test24:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
-; SSE4-LABEL: test24:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; AVX1-LABEL: test24:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX1-NEXT:    retq
 ;
-; AVX-LABEL: test24:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX-NEXT:    retq
+; AVX2-LABEL: test24:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test24:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX512-NEXT:    retq
   %bc1 = bitcast <4 x float> %a0 to <2 x i64>
   %bc2 = bitcast <4 x float> %a1 to <2 x i64>
   %and1 = and <2 x i64> %bc1, <i64 0, i64 -1>
@@ -677,15 +697,10 @@ define <4 x i8> @test_crash(<4 x i8> %a, <4 x i8> %b) {
 ; Verify that we can fold regardless of which operand is the zeroinitializer
 
 define <4 x i32> @test2b(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test2b:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test2b:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test2b:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2b:
 ; AVX:       # %bb.0:
@@ -698,15 +713,10 @@ define <4 x i32> @test2b(<4 x i32> %a, <4 x i32> %b) {
 }
 
 define <4 x i32> @test2c(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test2c:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test2c:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test2c:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2c:
 ; AVX:       # %bb.0:
@@ -720,15 +730,10 @@ define <4 x i32> @test2c(<4 x i32> %a, <4 x i32> %b) {
 
 
 define <4 x i32> @test2d(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test2d:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test2d:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test2d:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2d:
 ; AVX:       # %bb.0:
@@ -743,15 +748,10 @@ define <4 x i32> @test2d(<4 x i32> %a, <4 x i32> %b) {
 ; Make sure we can have an undef where an index pointing to the zero vector should be
 
 define <4 x i32> @test2e(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test2e:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test2e:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test2e:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2e:
 ; AVX:       # %bb.0:
@@ -764,15 +764,10 @@ define <4 x i32> @test2e(<4 x i32> %a, <4 x i32> %b) {
 }
 
 define <4 x i32> @test2f(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test2f:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE4-LABEL: test2f:
-; SSE4:       # %bb.0:
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE4-NEXT:    retq
+; SSE-LABEL: test2f:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2f:
 ; AVX:       # %bb.0:
@@ -808,7 +803,7 @@ define <2 x i64> @or_and_v2i64(<2 x i64> %a0) {
 ; AVX512-LABEL: or_and_v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [7,7]
-; AVX512-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm1 & (xmm0 | mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm1 & (xmm0 | m64bcst)
 ; AVX512-NEXT:    retq
   %1 = and <2 x i64> %a0, <i64 7, i64 7>
   %2 = or <2 x i64> %1, <i64 3, i64 3>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/commute-blend-sse41.ll b/external/llvm-project/llvm/test/CodeGen/X86/commute-blend-sse41.ll
index 07d6a8ba22bb..4740bf59a69e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/commute-blend-sse41.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/commute-blend-sse41.ll
@@ -57,7 +57,7 @@ define void @baz(ptr %arg, ptr %arg1) optsize {
 ; CHECK-NEXT:    movaps (%rdi), %xmm0
 ; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [3,3]
 ; CHECK-NEXT:    andps %xmm0, %xmm1
-; CHECK-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3]
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
 ; CHECK-NEXT:    movups %xmm1, (%rsi)
 ; CHECK-NEXT:    retq
 bb:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/constant-hoisting-cmp.ll b/external/llvm-project/llvm/test/CodeGen/X86/constant-hoisting-cmp.ll
index b90080003dd0..c11dfe5c11c0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/constant-hoisting-cmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/constant-hoisting-cmp.ll
@@ -1,15 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -O3 -mtriple=x86_64-- |FileCheck %s
-define i64 @foo(i64 %data1, i64 %data2, i64 %data3)
-{
+
 ; If constant 4294967295 is hoisted to a variable, then we won't be able to
 ; use a shift right by 32 to optimize the compare.
+define i64 @foo(i64 %data1, i64 %data2, i64 %data3) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdx, %rax
+; CHECK-NEXT:    shrq $32, %rdi
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    incq %rax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_2: # %L_val2
+; CHECK-NEXT:    shrq $32, %rsi
+; CHECK-NEXT:    je .LBB0_4
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    addq $2, %rax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_4: # %L_val3
+; CHECK-NEXT:    addq $3, %rax
+; CHECK-NEXT:    retq
 entry:
   %val1 = add i64 %data3, 1
   %x = icmp ugt i64 %data1, 4294967295
   br i1 %x, label %End, label %L_val2
 
-; CHECK: shrq    $32, {{.*}}
-; CHECK: shrq    $32, {{.*}}
 L_val2:
   %val2 = add i64 %data3, 2
   %y = icmp ugt i64 %data2, 4294967295
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll b/external/llvm-project/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll
index d9393ba9febb..edd3933fcfc2 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll
@@ -342,7 +342,7 @@ define <4 x i64> @vec256_eltty_i64_source_subvec_1_target_subvec_mask_3_binary(<
 ; CHECK-LABEL: vec256_eltty_i64_source_subvec_1_target_subvec_mask_3_binary:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,2,2,2]
-; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; CHECK-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
 ; CHECK-NEXT:    retq
   %r = shufflevector <4 x i64> %x, <4 x i64> %y, <4 x i32> <i32 0, i32 6, i32 2, i32 6>
   ret <4 x i64> %r
@@ -597,8 +597,8 @@ define <8 x i32> @vec256_eltty_i32_source_subvec_1_target_subvec_mask_3_unary(<8
 define <8 x i32> @vec256_eltty_i32_source_subvec_1_target_subvec_mask_3_binary(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-SLOW-LABEL: vec256_eltty_i32_source_subvec_1_target_subvec_mask_3_binary:
 ; CHECK-SLOW:       # %bb.0:
-; CHECK-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; CHECK-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
+; CHECK-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; CHECK-SLOW-NEXT:    vbroadcastss %xmm1, %ymm1
 ; CHECK-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6],ymm1[7]
 ; CHECK-SLOW-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/dllexport-x86_64.ll b/external/llvm-project/llvm/test/CodeGen/X86/dllexport-x86_64.ll
index 76add98314f5..b640e630e47e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/dllexport-x86_64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/dllexport-x86_64.ll
@@ -105,23 +105,23 @@ define weak_odr dllexport void @weak1() {
 ; MINGW: .ascii " -export:blob_alias"
 
 ; CHECK: .globl alias
-; CHECK: .set alias, notExported
+; CHECK: alias = notExported
 @alias = dllexport alias void(), ptr @notExported
 
 ; CHECK: .globl aliasNotExported
-; CHECK: .set aliasNotExported, f1
+; CHECK: aliasNotExported = f1
 @aliasNotExported = alias void(), ptr @f1
 
 ; CHECK: .globl alias2
-; CHECK: .set alias2, f1
+; CHECK: alias2 = f1
 @alias2 = dllexport alias void(), ptr @f1
 
 ; CHECK: .globl alias3
-; CHECK: .set alias3, notExported
+; CHECK: alias3 = notExported
 @alias3 = dllexport alias void(), ptr @notExported
 
 ; CHECK: .weak weak_alias
-; CHECK: .set weak_alias, f1
+; CHECK: weak_alias = f1
 @weak_alias = weak_odr dllexport alias void(), ptr @f1
 
 @blob = global [6 x i8] c"\B8*\00\00\00\C3", section ".text", align 16
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/dllexport.ll b/external/llvm-project/llvm/test/CodeGen/X86/dllexport.ll
index 09cc03e7729d..53ecb8e7a1b4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/dllexport.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/dllexport.ll
@@ -135,17 +135,17 @@ define weak_odr dllexport void @weak1() {
 ; CHECK-GCC: .ascii " -export:weak_alias"
 
 ; CHECK: .globl _alias
-; CHECK: .set _alias, _notExported
+; CHECK: _alias = _notExported
 @alias = dllexport alias void(), ptr @notExported
 
 ; CHECK: .globl _alias2
-; CHECK: .set _alias2, _f1
+; CHECK: _alias2 = _f1
 @alias2 = dllexport alias void(), ptr @f1
 
 ; CHECK: .globl _alias3
-; CHECK: .set _alias3, _notExported
+; CHECK: _alias3 = _notExported
 @alias3 = dllexport alias void(), ptr @notExported
 
 ; CHECK: .weak _weak_alias
-; CHECK: .set _weak_alias, _f1
+; CHECK: _weak_alias = _f1
 @weak_alias = weak_odr dllexport alias void(), ptr @f1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/dpbusd.ll b/external/llvm-project/llvm/test/CodeGen/X86/dpbusd.ll
index 04d7a9691b64..3aa77c3955c6 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/dpbusd.ll
@@ -317,8 +317,8 @@ define i32 @vpdpbusd_128(ptr%a, ptr%b, i32 %c, i32 %n) {
 ; AVXVNNI-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVXVNNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVXVNNI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVXVNNI-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
-; AVXVNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; AVXVNNI-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVXVNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVXVNNI-NEXT:    {vex} vpdpbusd %xmm1, %xmm0, %xmm2
 ; AVXVNNI-NEXT:    vmovd %xmm2, %eax
 ; AVXVNNI-NEXT:    addl %edx, %eax
@@ -328,9 +328,9 @@ define i32 @vpdpbusd_128(ptr%a, ptr%b, i32 %c, i32 %n) {
 ; AVX512VNNI:       # %bb.0: # %entry
 ; AVX512VNNI-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX512VNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512VNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512VNNI-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512VNNI-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3,4,5,6,7]
+; AVX512VNNI-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; AVX512VNNI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512VNNI-NEXT:    vpdpbusd %zmm0, %zmm1, %zmm2
 ; AVX512VNNI-NEXT:    vmovd %xmm2, %eax
@@ -343,8 +343,8 @@ define i32 @vpdpbusd_128(ptr%a, ptr%b, i32 %c, i32 %n) {
 ; AVX512VLVNNI-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX512VLVNNI-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX512VLVNNI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLVNNI-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]
-; AVX512VLVNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; AVX512VLVNNI-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVX512VLVNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX512VLVNNI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512VLVNNI-NEXT:    vpdpbusd %xmm1, %xmm0, %xmm2
 ; AVX512VLVNNI-NEXT:    vmovd %xmm2, %eax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_const.ll b/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_const.ll
index dfae853f9961..456e6e8f263a 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_const.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_const.ll
@@ -27,7 +27,7 @@ define i32 @mul_4xi8_zc(<4 x i8> %a, i32 %c) {
 ; AVXVNNI-LABEL: mul_4xi8_zc:
 ; AVXVNNI:       # %bb.0: # %entry
 ; AVXVNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVXVNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVXVNNI-NEXT:    {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
 ; AVXVNNI-NEXT:    vmovd %xmm1, %eax
 ; AVXVNNI-NEXT:    addl %edi, %eax
@@ -36,7 +36,7 @@ define i32 @mul_4xi8_zc(<4 x i8> %a, i32 %c) {
 ; AVX512VNNI-LABEL: mul_4xi8_zc:
 ; AVX512VNNI:       # %bb.0: # %entry
 ; AVX512VNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512VNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512VNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512VNNI-NEXT:    vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
 ; AVX512VNNI-NEXT:    vmovd %xmm1, %eax
@@ -47,7 +47,7 @@ define i32 @mul_4xi8_zc(<4 x i8> %a, i32 %c) {
 ; AVX512VLVNNI-LABEL: mul_4xi8_zc:
 ; AVX512VLVNNI:       # %bb.0: # %entry
 ; AVX512VLVNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLVNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512VLVNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512VLVNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512VLVNNI-NEXT:    vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
 ; AVX512VLVNNI-NEXT:    vmovd %xmm1, %eax
@@ -67,7 +67,7 @@ define i32 @mul_4xi4_cz(<4 x i4> %a, i32 %c) {
 ; AVXVNNI-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVXVNNI-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVXVNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVXVNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVXVNNI-NEXT:    {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
 ; AVXVNNI-NEXT:    vmovd %xmm1, %eax
 ; AVXVNNI-NEXT:    addl %edi, %eax
@@ -78,7 +78,7 @@ define i32 @mul_4xi4_cz(<4 x i4> %a, i32 %c) {
 ; AVX512VNNI-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512VNNI-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX512VNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512VNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512VNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512VNNI-NEXT:    vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
 ; AVX512VNNI-NEXT:    vmovd %xmm1, %eax
@@ -107,7 +107,7 @@ define i32 @mul_4xi8_cs(<4 x i8> %a, i32 %c) {
 ; AVXVNNI-LABEL: mul_4xi8_cs:
 ; AVXVNNI:       # %bb.0: # %entry
 ; AVXVNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVXVNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVXVNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVXVNNI-NEXT:    vmovd {{.*#+}} xmm2 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVXVNNI-NEXT:    {vex} vpdpbusd %xmm0, %xmm2, %xmm1
 ; AVXVNNI-NEXT:    vmovd %xmm1, %eax
@@ -117,7 +117,7 @@ define i32 @mul_4xi8_cs(<4 x i8> %a, i32 %c) {
 ; AVX512VNNI-LABEL: mul_4xi8_cs:
 ; AVX512VNNI:       # %bb.0: # %entry
 ; AVX512VNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512VNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512VNNI-NEXT:    vmovd {{.*#+}} xmm1 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512VNNI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512VNNI-NEXT:    vpdpbusd %zmm0, %zmm1, %zmm2
@@ -129,7 +129,7 @@ define i32 @mul_4xi8_cs(<4 x i8> %a, i32 %c) {
 ; AVX512VLVNNI-LABEL: mul_4xi8_cs:
 ; AVX512VLVNNI:       # %bb.0: # %entry
 ; AVX512VLVNNI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLVNNI-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512VLVNNI-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512VLVNNI-NEXT:    vmovd {{.*#+}} xmm1 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512VLVNNI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512VLVNNI-NEXT:    vpdpbusd %xmm0, %xmm1, %xmm2
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_i4.ll b/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_i4.ll
index a212f99680ef..9fbac111ee16 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_i4.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/dpbusd_i4.ll
@@ -54,7 +54,7 @@ define i32 @mul_i4i4(<16 x i4> %a, <16 x i4> %b, i32 %c) {
 ; CHECK-NEXT:    vpsllw $4, %xmm1, %xmm1
 ; CHECK-NEXT:    vpsrlw $4, %xmm1, %xmm1
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; CHECK-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm1
+; CHECK-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm2 ^ (xmm1 & m32bcst)
 ; CHECK-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; CHECK-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
 ; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/ehcontguard.ll b/external/llvm-project/llvm/test/CodeGen/X86/ehcontguard.ll
index 740621bc5d02..e868209babce 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/ehcontguard.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/ehcontguard.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s
 ; EHCont Guard is currently only available on Windows
 
-; CHECK: .set @feat.00, 16384
+; CHECK: @feat.00 = 16384
 
 ; CHECK: .section .gehcont$y
 
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-fptrunc-fpext.ll b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-fptrunc-fpext.ll
index cfca56d35998..00aa9cd8a27f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-fptrunc-fpext.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-fptrunc-fpext.ll
@@ -78,7 +78,7 @@ define double @single_to_double_rm_optsize(ptr %x) optsize {
 ;
 ; AVX-LABEL: single_to_double_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtss2sd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtss2sd (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load float, ptr %x, align 4
@@ -112,7 +112,7 @@ define float @double_to_single_rm_optsize(ptr %x) optsize {
 ;
 ; AVX-LABEL: double_to_single_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsd2ss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsd2ss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load double, ptr %x, align 8
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion-x86-64.ll b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion-x86-64.ll
index 42d65f7cd64b..5bf08f1c523d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion-x86-64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion-x86-64.ll
@@ -12,7 +12,7 @@ define double @long_to_double_rr(i64 %a) {
 ;
 ; AVX-LABEL: long_to_double_rr:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = sitofp i64 %a to double
@@ -27,7 +27,7 @@ define double @long_to_double_rm(ptr %a) {
 ;
 ; AVX-LABEL: long_to_double_rm:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
@@ -43,7 +43,7 @@ define double @long_to_double_rm_optsize(ptr %a) optsize {
 ;
 ; AVX-LABEL: long_to_double_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
@@ -59,7 +59,7 @@ define float @long_to_float_rr(i64 %a) {
 ;
 ; AVX-LABEL: long_to_float_rr:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = sitofp i64 %a to float
@@ -74,7 +74,7 @@ define float @long_to_float_rm(ptr %a) {
 ;
 ; AVX-LABEL: long_to_float_rm:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
@@ -90,7 +90,7 @@ define float @long_to_float_rm_optsize(ptr %a) optsize {
 ;
 ; AVX-LABEL: long_to_float_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion.ll b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion.ll
index 36daba63f08b..b39d9a7a3a6d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-int-float-conversion.ll
@@ -15,7 +15,7 @@ define double @int_to_double_rr(i32 %a) {
 ;
 ; AVX-LABEL: int_to_double_rr:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_double_rr:
@@ -44,7 +44,7 @@ define double @int_to_double_rr(i32 %a) {
 ; AVX_X86-NEXT:    .cfi_def_cfa_register %ebp
 ; AVX_X86-NEXT:    andl $-8, %esp
 ; AVX_X86-NEXT:    subl $8, %esp
-; AVX_X86-NEXT:    vcvtsi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtsi2sdl 8(%ebp), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX_X86-NEXT:    fldl (%esp)
 ; AVX_X86-NEXT:    movl %ebp, %esp
@@ -64,7 +64,7 @@ define double @int_to_double_rm(ptr %a) {
 ;
 ; AVX-LABEL: int_to_double_rm:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sdl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_double_rm:
@@ -95,7 +95,7 @@ define double @int_to_double_rm(ptr %a) {
 ; AVX_X86-NEXT:    andl $-8, %esp
 ; AVX_X86-NEXT:    subl $8, %esp
 ; AVX_X86-NEXT:    movl 8(%ebp), %eax
-; AVX_X86-NEXT:    vcvtsi2sdl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtsi2sdl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX_X86-NEXT:    fldl (%esp)
 ; AVX_X86-NEXT:    movl %ebp, %esp
@@ -116,7 +116,7 @@ define double @int_to_double_rm_optsize(ptr %a) optsize {
 ;
 ; AVX-LABEL: int_to_double_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sdl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_double_rm_optsize:
@@ -147,7 +147,7 @@ define double @int_to_double_rm_optsize(ptr %a) optsize {
 ; AVX_X86-NEXT:    andl $-8, %esp
 ; AVX_X86-NEXT:    subl $8, %esp
 ; AVX_X86-NEXT:    movl 8(%ebp), %eax
-; AVX_X86-NEXT:    vcvtsi2sdl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtsi2sdl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX_X86-NEXT:    fldl (%esp)
 ; AVX_X86-NEXT:    movl %ebp, %esp
@@ -168,7 +168,7 @@ define float @int_to_float_rr(i32 %a) {
 ;
 ; AVX-LABEL: int_to_float_rr:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_float_rr:
@@ -186,7 +186,7 @@ define float @int_to_float_rr(i32 %a) {
 ; AVX_X86:       # %bb.0: # %entry
 ; AVX_X86-NEXT:    pushl %eax
 ; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
-; AVX_X86-NEXT:    vcvtsi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtsi2ssl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX_X86-NEXT:    flds (%esp)
 ; AVX_X86-NEXT:    popl %eax
@@ -205,7 +205,7 @@ define float @int_to_float_rm(ptr %a) {
 ;
 ; AVX-LABEL: int_to_float_rm:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_float_rm:
@@ -225,7 +225,7 @@ define float @int_to_float_rm(ptr %a) {
 ; AVX_X86-NEXT:    pushl %eax
 ; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX_X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; AVX_X86-NEXT:    vcvtsi2ssl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtsi2ssl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX_X86-NEXT:    flds (%esp)
 ; AVX_X86-NEXT:    popl %eax
@@ -245,7 +245,7 @@ define float @int_to_float_rm_optsize(ptr %a) optsize {
 ;
 ; AVX-LABEL: int_to_float_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ssl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_float_rm_optsize:
@@ -265,7 +265,7 @@ define float @int_to_float_rm_optsize(ptr %a) optsize {
 ; AVX_X86-NEXT:    pushl %eax
 ; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX_X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; AVX_X86-NEXT:    vcvtsi2ssl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtsi2ssl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX_X86-NEXT:    flds (%esp)
 ; AVX_X86-NEXT:    popl %eax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion-x86-64.ll b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion-x86-64.ll
index d05bcfe3fd1e..77ef9ee5ad2b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion-x86-64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion-x86-64.ll
@@ -5,7 +5,7 @@
 define double @long_to_double_rr(i64 %a) {
 ; ALL-LABEL: long_to_double_rr:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %0 = uitofp i64 %a to double
@@ -15,7 +15,7 @@ entry:
 define double @long_to_double_rm(ptr %a) {
 ; ALL-LABEL: long_to_double_rm:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtusi2sdq (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2sdq (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
@@ -26,7 +26,7 @@ entry:
 define double @long_to_double_rm_optsize(ptr %a) optsize {
 ; ALL-LABEL: long_to_double_rm_optsize:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtusi2sdq (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2sdq (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
@@ -37,7 +37,7 @@ entry:
 define float @long_to_float_rr(i64 %a) {
 ; ALL-LABEL: long_to_float_rr:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2ss %rdi, %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %0 = uitofp i64 %a to float
@@ -47,7 +47,7 @@ entry:
 define float @long_to_float_rm(ptr %a) {
 ; ALL-LABEL: long_to_float_rm:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtusi2ssq (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2ssq (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
@@ -58,7 +58,7 @@ entry:
 define float @long_to_float_rm_optsize(ptr %a) optsize {
 ; ALL-LABEL: long_to_float_rm_optsize:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vcvtusi2ssq (%rdi), %xmm0, %xmm0
+; ALL-NEXT:    vcvtusi2ssq (%rdi), %xmm15, %xmm0
 ; ALL-NEXT:    retq
 entry:
   %0 = load i64, ptr %a
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion.ll b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion.ll
index b7f9af6165a9..de5765baeb9d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fast-isel-uint-float-conversion.ll
@@ -6,7 +6,7 @@
 define double @int_to_double_rr(i32 %a) {
 ; AVX-LABEL: int_to_double_rr:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX_X86-LABEL: int_to_double_rr:
@@ -18,7 +18,7 @@ define double @int_to_double_rr(i32 %a) {
 ; AVX_X86-NEXT:    .cfi_def_cfa_register %ebp
 ; AVX_X86-NEXT:    andl $-8, %esp
 ; AVX_X86-NEXT:    subl $8, %esp
-; AVX_X86-NEXT:    vcvtusi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtusi2sdl 8(%ebp), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX_X86-NEXT:    fldl (%esp)
 ; AVX_X86-NEXT:    movl %ebp, %esp
@@ -33,7 +33,7 @@ entry:
 define double @int_to_double_rm(ptr %a) {
 ; AVX-LABEL: int_to_double_rm:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtusi2sdl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtusi2sdl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX_X86-LABEL: int_to_double_rm:
@@ -46,7 +46,7 @@ define double @int_to_double_rm(ptr %a) {
 ; AVX_X86-NEXT:    andl $-8, %esp
 ; AVX_X86-NEXT:    subl $8, %esp
 ; AVX_X86-NEXT:    movl 8(%ebp), %eax
-; AVX_X86-NEXT:    vcvtusi2sdl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtusi2sdl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX_X86-NEXT:    fldl (%esp)
 ; AVX_X86-NEXT:    movl %ebp, %esp
@@ -62,7 +62,7 @@ entry:
 define double @int_to_double_rm_optsize(ptr %a) optsize {
 ; AVX-LABEL: int_to_double_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtusi2sdl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtusi2sdl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX_X86-LABEL: int_to_double_rm_optsize:
@@ -75,7 +75,7 @@ define double @int_to_double_rm_optsize(ptr %a) optsize {
 ; AVX_X86-NEXT:    andl $-8, %esp
 ; AVX_X86-NEXT:    subl $8, %esp
 ; AVX_X86-NEXT:    movl 8(%ebp), %eax
-; AVX_X86-NEXT:    vcvtusi2sdl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtusi2sdl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX_X86-NEXT:    fldl (%esp)
 ; AVX_X86-NEXT:    movl %ebp, %esp
@@ -91,14 +91,14 @@ entry:
 define float @int_to_float_rr(i32 %a) {
 ; AVX-LABEL: int_to_float_rr:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX_X86-LABEL: int_to_float_rr:
 ; AVX_X86:       # %bb.0: # %entry
 ; AVX_X86-NEXT:    pushl %eax
 ; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
-; AVX_X86-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX_X86-NEXT:    flds (%esp)
 ; AVX_X86-NEXT:    popl %eax
@@ -112,7 +112,7 @@ entry:
 define float @int_to_float_rm(ptr %a) {
 ; AVX-LABEL: int_to_float_rm:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtusi2ssl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtusi2ssl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX_X86-LABEL: int_to_float_rm:
@@ -120,7 +120,7 @@ define float @int_to_float_rm(ptr %a) {
 ; AVX_X86-NEXT:    pushl %eax
 ; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX_X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; AVX_X86-NEXT:    vcvtusi2ssl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtusi2ssl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX_X86-NEXT:    flds (%esp)
 ; AVX_X86-NEXT:    popl %eax
@@ -135,7 +135,7 @@ entry:
 define float @int_to_float_rm_optsize(ptr %a) optsize {
 ; AVX-LABEL: int_to_float_rm_optsize:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtusi2ssl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vcvtusi2ssl (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX_X86-LABEL: int_to_float_rm_optsize:
@@ -143,7 +143,7 @@ define float @int_to_float_rm_optsize(ptr %a) optsize {
 ; AVX_X86-NEXT:    pushl %eax
 ; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX_X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; AVX_X86-NEXT:    vcvtusi2ssl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vcvtusi2ssl (%eax), %xmm7, %xmm0
 ; AVX_X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX_X86-NEXT:    flds (%esp)
 ; AVX_X86-NEXT:    popl %eax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fastcall-correct-mangling.ll b/external/llvm-project/llvm/test/CodeGen/X86/fastcall-correct-mangling.ll
index 53b4bc8f1df2..4840308a5d49 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fastcall-correct-mangling.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fastcall-correct-mangling.ll
@@ -33,5 +33,5 @@ define private x86_fastcallcc void @dontCrash() {
 }
 
 @alias = alias void(i64, i8, i8, i16), ptr @func
-; CHECK32-LABEL: {{^}}.set @alias@20, @func@20
-; CHECK64-LABEL: {{^}}.set alias, func
+; CHECK32-LABEL: {{^}}@alias@20 = @func@20
+; CHECK64-LABEL: {{^}}alias = func
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fcmp-logic.ll b/external/llvm-project/llvm/test/CodeGen/X86/fcmp-logic.ll
index 7b806bca43c2..98fa725b2ea3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fcmp-logic.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fcmp-logic.ll
@@ -399,11 +399,11 @@ define i1 @PR140534(i32 %a0, i32 %a1, i32 %a2) {
 ; AVX1-LABEL: PR140534:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movl %edi, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    movl %esi, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    movl %edx, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vcmpltsd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vcmpltsd %xmm0, %xmm1, %xmm0
@@ -414,9 +414,9 @@ define i1 @PR140534(i32 %a0, i32 %a1, i32 %a2) {
 ;
 ; AVX512-LABEL: PR140534:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
-; AVX512-NEXT:    vcvtusi2sd %esi, %xmm1, %xmm1
-; AVX512-NEXT:    vcvtusi2sd %edx, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %esi, %xmm15, %xmm1
+; AVX512-NEXT:    vcvtusi2sd %edx, %xmm15, %xmm2
 ; AVX512-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; AVX512-NEXT:    vcmpltsd %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vcmpltsd %xmm0, %xmm1, %k1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fixup-blend.ll b/external/llvm-project/llvm/test/CodeGen/X86/fixup-blend.ll
new file mode 100644
index 000000000000..d64dd6d3114a
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fixup-blend.ll
@@ -0,0 +1,879 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64      | FileCheck %s -check-prefixes=SSE,SSE-MOV,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2   | FileCheck %s -check-prefixes=SSE,SSE4,SSE4-BLEND
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=slm         | FileCheck %s -check-prefixes=SSE,SSE-MOV,SSE4,SSE4-MOV
+
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=sandybridge | FileCheck %s -check-prefixes=AVX,AVX1,AVX-BLEND,AVX1-BLEND
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=btver2      | FileCheck %s -check-prefixes=AVX,AVX1,AVX-MOV,AVX1-MOV
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3   | FileCheck %s -check-prefixes=AVX,AVX2,AVX-BLEND,AVX2-BLEND
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4   | FileCheck %s -check-prefixes=AVX,AVX2,AVX-BLEND,AVX2-BLEND
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=alderlake   | FileCheck %s -check-prefixes=AVX,AVX2,AVX-MOV,AVX2-MOV
+
+;
+; v2f64 patterns
+;
+
+define <2 x double> @test_v2f64_blend_movsd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
+; SSE-MOV-LABEL: test_v2f64_blend_movsd:
+; SSE-MOV:       # %bb.0:
+; SSE-MOV-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-MOV-NEXT:    addpd %xmm2, %xmm0
+; SSE-MOV-NEXT:    retq
+;
+; SSE4-BLEND-LABEL: test_v2f64_blend_movsd:
+; SSE4-BLEND:       # %bb.0:
+; SSE4-BLEND-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE4-BLEND-NEXT:    addpd %xmm2, %xmm0
+; SSE4-BLEND-NEXT:    retq
+;
+; AVX-BLEND-LABEL: test_v2f64_blend_movsd:
+; AVX-BLEND:       # %bb.0:
+; AVX-BLEND-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-BLEND-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
+; AVX-BLEND-NEXT:    retq
+;
+; AVX-MOV-LABEL: test_v2f64_blend_movsd:
+; AVX-MOV:       # %bb.0:
+; AVX-MOV-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-MOV-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
+; AVX-MOV-NEXT:    retq
+  %s = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 2, i32 1>
+  %r = fadd <2 x double> %s, %a2
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_v2f64_blend_movsd_optsize(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) optsize {
+; SSE-LABEL: test_v2f64_blend_movsd_optsize:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    addpd %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2f64_blend_movsd_optsize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %s = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 2, i32 1>
+  %r = fadd <2 x double> %s, %a2
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_v2f64_blend_movsd_load(ptr %p0, <2 x double> %a1, <2 x double> %a2) {
+; SSE2-LABEL: test_v2f64_blend_movsd_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[0],mem[1]
+; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v2f64_blend_movsd_load:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1]
+; SSE4-NEXT:    addpd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX-LABEL: test_v2f64_blend_movsd_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1]
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %a0 = load <2 x double>, ptr %p0
+  %s = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 2, i32 1>
+  %r = fadd <2 x double> %s, %a2
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_v2f64_blend_movsd_load_commute(<2 x double> %a0, ptr %p1, <2 x double> %a2) {
+; SSE2-LABEL: test_v2f64_blend_movsd_load_commute:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movlpd {{.*#+}} xmm0 = mem[0],xmm0[1]
+; SSE2-NEXT:    addpd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v2f64_blend_movsd_load_commute:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    blendpd {{.*#+}} xmm0 = mem[0],xmm0[1]
+; SSE4-NEXT:    addpd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX-LABEL: test_v2f64_blend_movsd_load_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = mem[0],xmm0[1]
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %a1 = load <2 x double>, ptr %p1
+  %s = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 2, i32 1>
+  %r = fadd <2 x double> %s, %a2
+  ret <2 x double> %r
+}
+
+define <2 x double> @test_v2f64_blend_movsd_zero(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
+; SSE-MOV-LABEL: test_v2f64_blend_movsd_zero:
+; SSE-MOV:       # %bb.0:
+; SSE-MOV-NEXT:    xorpd %xmm1, %xmm1
+; SSE-MOV-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-MOV-NEXT:    addpd %xmm2, %xmm0
+; SSE-MOV-NEXT:    retq
+;
+; SSE4-BLEND-LABEL: test_v2f64_blend_movsd_zero:
+; SSE4-BLEND:       # %bb.0:
+; SSE4-BLEND-NEXT:    xorpd %xmm1, %xmm1
+; SSE4-BLEND-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE4-BLEND-NEXT:    addpd %xmm2, %xmm0
+; SSE4-BLEND-NEXT:    retq
+;
+; AVX-BLEND-LABEL: test_v2f64_blend_movsd_zero:
+; AVX-BLEND:       # %bb.0:
+; AVX-BLEND-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX-BLEND-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-BLEND-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
+; AVX-BLEND-NEXT:    retq
+;
+; AVX-MOV-LABEL: test_v2f64_blend_movsd_zero:
+; AVX-MOV:       # %bb.0:
+; AVX-MOV-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX-MOV-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-MOV-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
+; AVX-MOV-NEXT:    retq
+  %s = shufflevector <2 x double> %a0, <2 x double> zeroinitializer, <2 x i32> <i32 2, i32 1>
+  %r = fadd <2 x double> %s, %a2
+  ret <2 x double> %r
+}
+
+;
+; v2i64 patterns
+;
+
+define <2 x i64> @test_v2i64_blend_movsd(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2) {
+; SSE2-LABEL: test_v2i64_blend_movsd:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v2i64_blend_movsd:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddq %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v2i64_blend_movsd:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v2i64_blend_movsd:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 2, i32 1>
+  %r = add <2 x i64> %s, %a2
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @test_v2i64_blend_movsd_optsize(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2) optsize {
+; SSE2-LABEL: test_v2i64_blend_movsd_optsize:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v2i64_blend_movsd_optsize:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddq %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v2i64_blend_movsd_optsize:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v2i64_blend_movsd_optsize:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 2, i32 1>
+  %r = add <2 x i64> %s, %a2
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @test_v2i64_blend_movsd_load(ptr %p0, <2 x i64> %a1, <2 x i64> %a2) {
+; SSE2-LABEL: test_v2i64_blend_movsd_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[0],mem[1]
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v2i64_blend_movsd_load:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],mem[4,5,6,7]
+; SSE4-NEXT:    paddq %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v2i64_blend_movsd_load:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],mem[4,5,6,7]
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v2i64_blend_movsd_load:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %a0 = load <2 x i64>, ptr %p0
+  %s = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 2, i32 1>
+  %r = add <2 x i64> %s, %a2
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @test_v2i64_blend_movsd_load_commute(<2 x i64> %a0, ptr %p1, <2 x i64> %a2) {
+; SSE2-LABEL: test_v2i64_blend_movsd_load_commute:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v2i64_blend_movsd_load_commute:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = mem[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddq %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v2i64_blend_movsd_load_commute:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = mem[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v2i64_blend_movsd_load_commute:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %a1 = load <2 x i64>, ptr %p1
+  %s = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 2, i32 1>
+  %r = add <2 x i64> %s, %a2
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @test_v2i64_blend_movsd_zero(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2) {
+; SSE2-LABEL: test_v2i64_blend_movsd_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    paddq %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v2i64_blend_movsd_zero:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddq %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v2i64_blend_movsd_zero:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v2i64_blend_movsd_zero:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i32> <i32 2, i32 1>
+  %r = add <2 x i64> %s, %a2
+  ret <2 x i64> %r
+}
+
+;
+; v4f32 patterns
+;
+
+define <4 x float> @test_v4f32_blend_movss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; SSE-MOV-LABEL: test_v4f32_blend_movss:
+; SSE-MOV:       # %bb.0:
+; SSE-MOV-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-MOV-NEXT:    addps %xmm2, %xmm0
+; SSE-MOV-NEXT:    retq
+;
+; SSE4-BLEND-LABEL: test_v4f32_blend_movss:
+; SSE4-BLEND:       # %bb.0:
+; SSE4-BLEND-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE4-BLEND-NEXT:    addps %xmm2, %xmm0
+; SSE4-BLEND-NEXT:    retq
+;
+; AVX-BLEND-LABEL: test_v4f32_blend_movss:
+; AVX-BLEND:       # %bb.0:
+; AVX-BLEND-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-BLEND-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-BLEND-NEXT:    retq
+;
+; AVX-MOV-LABEL: test_v4f32_blend_movss:
+; AVX-MOV:       # %bb.0:
+; AVX-MOV-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-MOV-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-MOV-NEXT:    retq
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movsd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; SSE-MOV-LABEL: test_v4f32_blend_movsd:
+; SSE-MOV:       # %bb.0:
+; SSE-MOV-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-MOV-NEXT:    addps %xmm2, %xmm0
+; SSE-MOV-NEXT:    retq
+;
+; SSE4-BLEND-LABEL: test_v4f32_blend_movsd:
+; SSE4-BLEND:       # %bb.0:
+; SSE4-BLEND-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE4-BLEND-NEXT:    addps %xmm2, %xmm0
+; SSE4-BLEND-NEXT:    retq
+;
+; AVX-BLEND-LABEL: test_v4f32_blend_movsd:
+; AVX-BLEND:       # %bb.0:
+; AVX-BLEND-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-BLEND-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-BLEND-NEXT:    retq
+;
+; AVX-MOV-LABEL: test_v4f32_blend_movsd:
+; AVX-MOV:       # %bb.0:
+; AVX-MOV-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-MOV-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-MOV-NEXT:    retq
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movss_optsize(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) optsize {
+; SSE-LABEL: test_v4f32_blend_movss_optsize:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    addps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v4f32_blend_movss_optsize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movsd_optsize(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) optsize {
+; SSE-LABEL: test_v4f32_blend_movsd_optsize:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    addps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v4f32_blend_movsd_optsize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movss_load(ptr %p0, <4 x float> %a1, <4 x float> %a2) {
+; SSE2-LABEL: test_v4f32_blend_movss_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps (%rdi), %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSE2-NEXT:    addps %xmm1, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4f32_blend_movss_load:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],mem[1,2,3]
+; SSE4-NEXT:    addps %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX-LABEL: test_v4f32_blend_movss_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],mem[1,2,3]
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %a0 = load <4 x float>, ptr %p0
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movss_load_commute(<4 x float> %a0, ptr %p1, <4 x float> %a2) {
+; SSE2-LABEL: test_v4f32_blend_movss_load_commute:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps (%rdi), %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; SSE2-NEXT:    addps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4f32_blend_movss_load_commute:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    blendps {{.*#+}} xmm0 = mem[0],xmm0[1,2,3]
+; SSE4-NEXT:    addps %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX-LABEL: test_v4f32_blend_movss_load_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = mem[0],xmm0[1,2,3]
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %a1 = load <4 x float>, ptr %p1
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movsd_load(ptr %p0, <4 x float> %a1, <4 x float> %a2) {
+; SSE2-LABEL: test_v4f32_blend_movsd_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[0],mem[1]
+; SSE2-NEXT:    addps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4f32_blend_movsd_load:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
+; SSE4-NEXT:    addps %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX-LABEL: test_v4f32_blend_movsd_load:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %a0 = load <4 x float>, ptr %p0
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movsd_load_commute(<4 x float> %a0, ptr %p1, <4 x float> %a2) {
+; SSE2-LABEL: test_v4f32_blend_movsd_load_commute:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; SSE2-NEXT:    addps %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4f32_blend_movsd_load_commute:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    blendps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; SSE4-NEXT:    addps %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX-LABEL: test_v4f32_blend_movsd_load_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %a1 = load <4 x float>, ptr %p1
+  %s = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movss_zero(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; SSE-MOV-LABEL: test_v4f32_blend_movss_zero:
+; SSE-MOV:       # %bb.0:
+; SSE-MOV-NEXT:    xorps %xmm1, %xmm1
+; SSE-MOV-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-MOV-NEXT:    addps %xmm2, %xmm0
+; SSE-MOV-NEXT:    retq
+;
+; SSE4-BLEND-LABEL: test_v4f32_blend_movss_zero:
+; SSE4-BLEND:       # %bb.0:
+; SSE4-BLEND-NEXT:    xorps %xmm1, %xmm1
+; SSE4-BLEND-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE4-BLEND-NEXT:    addps %xmm2, %xmm0
+; SSE4-BLEND-NEXT:    retq
+;
+; AVX-BLEND-LABEL: test_v4f32_blend_movss_zero:
+; AVX-BLEND:       # %bb.0:
+; AVX-BLEND-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-BLEND-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-BLEND-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-BLEND-NEXT:    retq
+;
+; AVX-MOV-LABEL: test_v4f32_blend_movss_zero:
+; AVX-MOV:       # %bb.0:
+; AVX-MOV-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-MOV-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-MOV-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-MOV-NEXT:    retq
+  %s = shufflevector <4 x float> %a0, <4 x float> zeroinitializer, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+define <4 x float> @test_v4f32_blend_movsd_zero(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; SSE2-LABEL: test_v4f32_blend_movsd_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    addps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-BLEND-LABEL: test_v4f32_blend_movsd_zero:
+; SSE4-BLEND:       # %bb.0:
+; SSE4-BLEND-NEXT:    xorps %xmm1, %xmm1
+; SSE4-BLEND-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE4-BLEND-NEXT:    addps %xmm2, %xmm0
+; SSE4-BLEND-NEXT:    retq
+;
+; SSE4-MOV-LABEL: test_v4f32_blend_movsd_zero:
+; SSE4-MOV:       # %bb.0:
+; SSE4-MOV-NEXT:    xorps %xmm1, %xmm1
+; SSE4-MOV-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE4-MOV-NEXT:    addps %xmm2, %xmm0
+; SSE4-MOV-NEXT:    retq
+;
+; AVX-BLEND-LABEL: test_v4f32_blend_movsd_zero:
+; AVX-BLEND:       # %bb.0:
+; AVX-BLEND-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-BLEND-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-BLEND-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-BLEND-NEXT:    retq
+;
+; AVX-MOV-LABEL: test_v4f32_blend_movsd_zero:
+; AVX-MOV:       # %bb.0:
+; AVX-MOV-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-MOV-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-MOV-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-MOV-NEXT:    retq
+  %s = shufflevector <4 x float> %a0, <4 x float> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = fadd <4 x float> %s, %a2
+  ret <4 x float> %r
+}
+
+;
+; v4i32 patterns
+;
+
+define <4 x i32> @test_v4i32_blend_movss(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movss:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movss:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; SSE4-NEXT:    paddd %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movss:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movss:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movsd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movsd:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movsd:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddd %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movsd:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movsd:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movss_optsize(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) optsize {
+; SSE2-LABEL: test_v4i32_blend_movss_optsize:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movss_optsize:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; SSE4-NEXT:    paddd %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movss_optsize:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movss_optsize:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movsd_optsize(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) optsize {
+; SSE2-LABEL: test_v4i32_blend_movsd_optsize:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movsd_optsize:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddd %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movsd_optsize:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movsd_optsize:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movss_load(ptr %p0, <4 x i32> %a1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movss_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps (%rdi), %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movss_load:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],mem[2,3,4,5,6,7]
+; SSE4-NEXT:    paddd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movss_load:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],mem[2,3,4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movss_load:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],mem[1,2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %a0 = load <4 x i32>, ptr %p0
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movss_load_commute(<4 x i32> %a0, ptr %p1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movss_load_commute:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps (%rdi), %xmm2
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movss_load_commute:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = mem[0,1],xmm0[2,3,4,5,6,7]
+; SSE4-NEXT:    paddd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movss_load_commute:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = mem[0,1],xmm0[2,3,4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movss_load_commute:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = mem[0],xmm0[1,2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %a1 = load <4 x i32>, ptr %p1
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movsd_load(ptr %p0, <4 x i32> %a1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movsd_load:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[0],mem[1]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movsd_load:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],mem[4,5,6,7]
+; SSE4-NEXT:    paddd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movsd_load:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],mem[4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movsd_load:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %a0 = load <4 x i32>, ptr %p0
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movsd_load_commute(<4 x i32> %a0, ptr %p1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movsd_load_commute:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movsd_load_commute:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = mem[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movsd_load_commute:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = mem[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movsd_load_commute:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %a1 = load <4 x i32>, ptr %p1
+  %s = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movss_zero(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movss_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movss_zero:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; SSE4-NEXT:    paddd %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movss_zero:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movss_zero:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_v4i32_blend_movsd_zero(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
+; SSE2-LABEL: test_v4i32_blend_movsd_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE2-NEXT:    paddd %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: test_v4i32_blend_movsd_zero:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE4-NEXT:    paddd %xmm2, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX1-LABEL: test_v4i32_blend_movsd_zero:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i32_blend_movsd_zero:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %s = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %r = add <4 x i32> %s, %a2
+  ret <4 x i32> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX1-BLEND: {{.*}}
+; AVX1-MOV: {{.*}}
+; AVX2-BLEND: {{.*}}
+; AVX2-MOV: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/external/llvm-project/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index 5945bae94f45..95d350d45d90 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -1343,7 +1343,7 @@ define <2 x double> @test_fminimumnum_vector_nan(<2 x double> %x) {
 ; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vmovhpd {{.*#+}} xmm2 = xmm1[0],mem[0]
 ; AVX-NEXT:    vminpd %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_nan:
@@ -1812,15 +1812,15 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ;
 ; AVX512-LABEL: test_fmaximumnum_v4f16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $72, %rsp
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm0, %xmm8
+; AVX512-NEXT:    subq $56, %rsp
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm5
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm6
 ; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm8[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm6[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
@@ -1837,41 +1837,39 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm2
+; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm5[3,3,3,3]
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    movzwl {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm9
-; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[3,3,3,3]
-; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
-; AVX512-NEXT:    vucomiss %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[3,3,3,3]
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; AVX512-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k2}
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; AVX512-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm3
-; AVX512-NEXT:    vucomiss %xmm3, %xmm2
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
+; AVX512-NEXT:    vucomiss %xmm0, %xmm1
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm4[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
 ; AVX512-NEXT:    vucomiss %xmm2, %xmm2
 ; AVX512-NEXT:    setp %al
@@ -1888,222 +1886,178 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
-; AVX512-NEXT:    vshufpd {{.*#+}} xmm2 = xmm4[1,0]
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm2
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm3
+; AVX512-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm1 = xmm5[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vshufpd {{.*#+}} xmm7 = xmm8[1,0]
-; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm7
-; AVX512-NEXT:    vucomiss %xmm7, %xmm7
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm2 = xmm6[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vucomiss %xmm2, %xmm2
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm2, %xmm7, %xmm7 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm7, %xmm14
-; AVX512-NEXT:    vcvtph2ps %xmm14, %xmm7
-; AVX512-NEXT:    vmovss %xmm7, %xmm2, %xmm2 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; AVX512-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm2, %xmm7
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm13
+; AVX512-NEXT:    vcvtph2ps %xmm13, %xmm2
+; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm12
+; AVX512-NEXT:    vcvtph2ps %xmm12, %xmm1
+; AVX512-NEXT:    vucomiss %xmm1, %xmm2
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm7, %xmm2, %xmm2 {%k1}
-; AVX512-NEXT:    vxorps %xmm15, %xmm15, %xmm15
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm5
-; AVX512-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm3
-; AVX512-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm1
+; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpsrlq $48, %xmm4, %xmm0
+; AVX512-NEXT:    vpsrlq $48, %xmm5, %xmm0
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpsrlq $48, %xmm8, %xmm1
+; AVX512-NEXT:    vpsrlq $48, %xmm6, %xmm1
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
 ; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm12
-; AVX512-NEXT:    vcvtph2ps %xmm12, %xmm1
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm11
+; AVX512-NEXT:    vcvtph2ps %xmm11, %xmm1
 ; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm13
-; AVX512-NEXT:    vcvtph2ps %xmm13, %xmm6
-; AVX512-NEXT:    vucomiss %xmm6, %xmm1
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm15
+; AVX512-NEXT:    vcvtph2ps %xmm15, %xmm7
+; AVX512-NEXT:    vucomiss %xmm7, %xmm1
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm6, %xmm6 {%k1}
-; AVX512-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; AVX512-NEXT:    vmovss %xmm1, %xmm7, %xmm7 {%k1}
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm5[1,1,3,3]
 ; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm8[1,1,3,3]
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm6[1,1,3,3]
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
 ; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm10
+; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm9
+; AVX512-NEXT:    vcvtph2ps %xmm9, %xmm4
+; AVX512-NEXT:    vmovss %xmm4, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm10
 ; AVX512-NEXT:    vcvtph2ps %xmm10, %xmm3
-; AVX512-NEXT:    vmovss %xmm3, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm11
-; AVX512-NEXT:    vcvtph2ps %xmm11, %xmm5
-; AVX512-NEXT:    vucomiss %xmm5, %xmm3
+; AVX512-NEXT:    vucomiss %xmm3, %xmm4
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm3, %xmm5, %xmm5 {%k1}
-; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm0
+; AVX512-NEXT:    vmovss %xmm4, %xmm3, %xmm3 {%k1}
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm0
 ; AVX512-NEXT:    vucomiss %xmm0, %xmm0
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vcvtph2ps %xmm8, %xmm3
-; AVX512-NEXT:    vucomiss %xmm3, %xmm3
+; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm4
+; AVX512-NEXT:    vucomiss %xmm4, %xmm4
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm0, %xmm3, %xmm3 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
-; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm1
+; AVX512-NEXT:    vmovss %xmm0, %xmm4, %xmm4 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm4
+; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm1
 ; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm7
-; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm2
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm8
+; AVX512-NEXT:    vcvtph2ps %xmm8, %xmm2
 ; AVX512-NEXT:    vucomiss %xmm2, %xmm1
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
-; AVX512-NEXT:    vpsrld $16, %xmm4, %xmm1
+; AVX512-NEXT:    vpsrld $16, %xmm5, %xmm1
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX512-NEXT:    vucomiss %xmm1, %xmm1
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vpsrld $16, %xmm8, %xmm4
-; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
-; AVX512-NEXT:    vucomiss %xmm4, %xmm4
+; AVX512-NEXT:    vpsrld $16, %xmm6, %xmm5
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm5, %xmm5
 ; AVX512-NEXT:    setp %al
 ; AVX512-NEXT:    kmovw %eax, %k2
-; AVX512-NEXT:    vmovss %xmm1, %xmm4, %xmm4 {%k2}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm8
-; AVX512-NEXT:    vcvtph2ps %xmm8, %xmm4
-; AVX512-NEXT:    vmovss %xmm4, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm5, %xmm5 {%k2}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm5, %xmm6
+; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm5
+; AVX512-NEXT:    vmovss %xmm5, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm0
-; AVX512-NEXT:    vucomiss %xmm0, %xmm4
+; AVX512-NEXT:    vucomiss %xmm0, %xmm5
 ; AVX512-NEXT:    seta %al
 ; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovss %xmm4, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    vcvtps2ph $4, %xmm6, %xmm4
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
-; AVX512-NEXT:    vmulss %xmm4, %xmm9, %xmm4
-; AVX512-NEXT:    vcvtps2ph $4, %xmm5, %xmm5
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
-; AVX512-NEXT:    vmulss %xmm5, %xmm9, %xmm5
-; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm2
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vmulss %xmm2, %xmm9, %xmm2
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
-; AVX512-NEXT:    vmulss %xmm0, %xmm9, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm15[1,2,3]
-; AVX512-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0],xmm15[1,2,3]
-; AVX512-NEXT:    vblendps {{.*#+}} xmm9 = xmm2[0],xmm15[1,2,3]
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm15[1,2,3]
-; AVX512-NEXT:    vcvtps2ph $4, %xmm4, %xmm2
-; AVX512-NEXT:    vcvtps2ph $4, %xmm5, %xmm6
-; AVX512-NEXT:    vcvtps2ph $4, %xmm9, %xmm4
-; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
+; AVX512-NEXT:    vmovss %xmm5, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vcvtps2ph $4, %xmm7, %xmm7
+; AVX512-NEXT:    vcvtps2ph $4, %xmm3, %xmm3
+; AVX512-NEXT:    vcvtps2ph $4, %xmm2, %xmm5
+; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
 ; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
-; AVX512-NEXT:    vmovdqa (%rsp), %xmm9 # 16-byte Reload
-; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm9 = xmm9[0],mem[0],xmm9[1],mem[1],xmm9[2],mem[2],xmm9[3],mem[3]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm14 # 16-byte Folded Reload
 ; AVX512-NEXT:    # xmm14 = xmm14[0],mem[0],xmm14[1],mem[1],xmm14[2],mem[2],xmm14[3],mem[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm14[0],xmm9[0],xmm14[1],xmm9[1]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1]
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm9[0]
-; AVX512-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX512-NEXT:    vpcmpeqw %xmm3, %xmm8, %xmm9
-; AVX512-NEXT:    vpblendvb %xmm9, %xmm3, %xmm0, %xmm3
+; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm13 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm13 = xmm13[0],mem[0],xmm13[1],mem[1],xmm13[2],mem[2],xmm13[3],mem[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1],xmm9[2],xmm11[2],xmm9[3],xmm11[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm9[0],xmm4[1],xmm9[1]
+; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm13[0]
+; AVX512-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; AVX512-NEXT:    vpcmpeqw %xmm6, %xmm4, %xmm9
+; AVX512-NEXT:    vpblendvb %xmm9, %xmm4, %xmm0, %xmm4
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
 ; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload
 ; AVX512-NEXT:    # xmm9 = xmm9[0],mem[0],xmm9[1],mem[1],xmm9[2],mem[2],xmm9[3],mem[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm10 = xmm10[0],mem[0],xmm10[1],mem[1],xmm10[2],mem[2],xmm10[3],mem[3]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm13[0],xmm11[1],xmm13[1],xmm11[2],xmm13[2],xmm11[3],xmm13[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm11 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm11 = xmm12[0],mem[0],xmm12[1],mem[1],xmm12[2],mem[2],xmm12[3],mem[3]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm15[0],xmm10[1],xmm15[1],xmm10[2],xmm15[2],xmm10[3],xmm15[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3]
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1]
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm9[0]
-; AVX512-NEXT:    vpcmpeqw %xmm1, %xmm8, %xmm7
-; AVX512-NEXT:    vpblendvb %xmm7, %xmm1, %xmm3, %xmm1
-; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm3
+; AVX512-NEXT:    vpcmpeqw %xmm6, %xmm1, %xmm6
+; AVX512-NEXT:    vpblendvb %xmm6, %xmm1, %xmm4, %xmm1
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
 ; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX512-NEXT:    vucomiss %xmm5, %xmm3
+; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    movl $65535, %ecx # imm = 0xFFFF
 ; AVX512-NEXT:    movl $0, %edx
 ; AVX512-NEXT:    cmovel %ecx, %edx
-; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm3
-; AVX512-NEXT:    vucomiss %xmm5, %xmm3
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm2
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    movl $0, %esi
 ; AVX512-NEXT:    cmovel %ecx, %esi
-; AVX512-NEXT:    vcvtph2ps %xmm6, %xmm3
-; AVX512-NEXT:    vucomiss %xmm5, %xmm3
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm2
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    movl $0, %edi
 ; AVX512-NEXT:    cmovel %ecx, %edi
-; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
-; AVX512-NEXT:    vucomiss %xmm5, %xmm2
+; AVX512-NEXT:    vcvtph2ps %xmm7, %xmm2
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    movl $0, %r8d
 ; AVX512-NEXT:    cmovel %ecx, %r8d
 ; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm5, %xmm2
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    movl $0, %r9d
 ; AVX512-NEXT:    cmovel %ecx, %r9d
 ; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm5, %xmm2
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    movl $0, %r10d
 ; AVX512-NEXT:    cmovel %ecx, %r10d
-; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm5, %xmm2
+; AVX512-NEXT:    vcvtph2ps (%rsp), %xmm2 # 16-byte Folded Reload
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    movl $0, %r11d
 ; AVX512-NEXT:    cmovel %ecx, %r11d
 ; AVX512-NEXT:    vcvtph2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; AVX512-NEXT:    vucomiss %xmm5, %xmm2
+; AVX512-NEXT:    vucomiss %xmm4, %xmm2
 ; AVX512-NEXT:    vmovd %esi, %xmm2
 ; AVX512-NEXT:    vpinsrw $1, %edx, %xmm2, %xmm2
 ; AVX512-NEXT:    vpinsrw $2, %edi, %xmm2, %xmm2
@@ -2114,7 +2068,7 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind
 ; AVX512-NEXT:    cmovel %ecx, %eax
 ; AVX512-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm2
 ; AVX512-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    addq $72, %rsp
+; AVX512-NEXT:    addq $56, %rsp
 ; AVX512-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_v4f16:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fmsubadd-combine.ll b/external/llvm-project/llvm/test/CodeGen/X86/fmsubadd-combine.ll
index ddf51b858cdd..674a1d5ad779 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fmsubadd-combine.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fmsubadd-combine.ll
@@ -12,7 +12,7 @@ define <2 x double> @mul_subadd_pd128(<2 x double> %A, <2 x double> %B, <2 x dou
 ; NOFMA-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; NOFMA-NEXT:    vsubpd %xmm2, %xmm0, %xmm1
 ; NOFMA-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
-; NOFMA-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; NOFMA-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; NOFMA-NEXT:    retq
 ;
 ; FMA3-LABEL: mul_subadd_pd128:
@@ -191,7 +191,7 @@ define <2 x double> @mul_subadd_bad_commute(<2 x double> %A, <2 x double> %B, <2
 ; CHECK-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vsubpd %xmm0, %xmm2, %xmm1
 ; CHECK-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
-; CHECK-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; CHECK-NEXT:    retq
 entry:
   %AB = fmul <2 x double> %A, %B
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll b/external/llvm-project/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
index 5519d9b787b7..d59b12c6d123 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -887,14 +887,14 @@ define double @fmul_pow_shl_cnt_fail_maybe_non_pow2(i64 %v, i64 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    movq %rsi, %rcx
 ; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rdi
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    retq
 ;
 ; CHECK-FMA-LABEL: fmul_pow_shl_cnt_fail_maybe_non_pow2:
 ; CHECK-FMA:       # %bb.0:
 ; CHECK-FMA-NEXT:    shlxq %rsi, %rdi, %rax
-; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
+; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    retq
   %shl = shl nuw i64 %v, %cnt
@@ -927,9 +927,9 @@ define <2 x float> @fmul_pow_shl_cnt_vec_fail_expensive_cast(<2 x i64> %cnt) nou
 ; CHECK-AVX2-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [2,2]
 ; CHECK-AVX2-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
 ; CHECK-AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm1
+; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; CHECK-AVX2-NEXT:    vmovq %xmm0, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; CHECK-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.5E+1,1.5E+1,1.5E+1,1.5E+1]
 ; CHECK-AVX2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
@@ -940,9 +940,9 @@ define <2 x float> @fmul_pow_shl_cnt_vec_fail_expensive_cast(<2 x i64> %cnt) nou
 ; CHECK-NO-FASTFMA-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [2,2]
 ; CHECK-NO-FASTFMA-NEXT:    vpsllvq %xmm0, %xmm1, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vpextrq $1, %xmm0, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm1
+; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; CHECK-NO-FASTFMA-NEXT:    vmovq %xmm0, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; CHECK-NO-FASTFMA-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.5E+1,1.5E+1,1.5E+1,1.5E+1]
 ; CHECK-NO-FASTFMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0
@@ -1108,13 +1108,13 @@ define <2 x half> @fmul_pow_shl_cnt_vec_fail_to_large(<2 x i16> %cnt) nounwind {
 ; CHECK-AVX2-NEXT:    vpsllvd %ymm0, %ymm1, %ymm0
 ; CHECK-AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; CHECK-AVX2-NEXT:    vpextrw $2, %xmm0, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    vzeroupper
 ; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-AVX2-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-AVX2-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; CHECK-AVX2-NEXT:    vpextrw $0, %xmm0, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    vzeroupper
 ; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
@@ -1201,7 +1201,7 @@ define double @fmul_pow_shl_cnt_fail_maybe_bad_exp(i64 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
 ; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    retq
 ;
@@ -1209,7 +1209,7 @@ define double @fmul_pow_shl_cnt_fail_maybe_bad_exp(i64 %cnt) nounwind {
 ; CHECK-FMA:       # %bb.0:
 ; CHECK-FMA-NEXT:    movl $1, %eax
 ; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm0, %xmm0
+; CHECK-FMA-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; CHECK-FMA-NEXT:    vmulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    retq
   %shl = shl nuw i64 1, %cnt
@@ -1317,11 +1317,11 @@ define float @fdiv_pow_shl_cnt_fail_maybe_z(i64 %cnt) nounwind {
 ; CHECK-AVX2-NEXT:    testq %rax, %rax
 ; CHECK-AVX2-NEXT:    js .LBB23_1
 ; CHECK-AVX2-NEXT:  # %bb.2:
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    jmp .LBB23_3
 ; CHECK-AVX2-NEXT:  .LBB23_1:
 ; CHECK-AVX2-NEXT:    shrq %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    vaddss %xmm0, %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:  .LBB23_3:
 ; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = [-9.0E+0,0.0E+0,0.0E+0,0.0E+0]
@@ -1334,7 +1334,7 @@ define float @fdiv_pow_shl_cnt_fail_maybe_z(i64 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    movl $8, %eax
 ; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %rax, %xmm0, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = [-9.0E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    retq
@@ -1343,7 +1343,7 @@ define float @fdiv_pow_shl_cnt_fail_maybe_z(i64 %cnt) nounwind {
 ; CHECK-FMA:       # %bb.0:
 ; CHECK-FMA-NEXT:    movl $8, %eax
 ; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtusi2ss %rax, %xmm0, %xmm0
+; CHECK-FMA-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = [-9.0E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-FMA-NEXT:    retq
@@ -1371,7 +1371,7 @@ define float @fdiv_pow_shl_cnt_fail_neg_int(i64 %cnt) nounwind {
 ; CHECK-AVX2-NEXT:    movl $8, %eax
 ; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-AVX2-NEXT:    shlq %cl, %rax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = [-9.0E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-AVX2-NEXT:    retq
@@ -1382,7 +1382,7 @@ define float @fdiv_pow_shl_cnt_fail_neg_int(i64 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    movl $8, %eax
 ; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NO-FASTFMA-NEXT:    shlq %cl, %rax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = [-9.0E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    retq
@@ -1391,7 +1391,7 @@ define float @fdiv_pow_shl_cnt_fail_neg_int(i64 %cnt) nounwind {
 ; CHECK-FMA:       # %bb.0:
 ; CHECK-FMA-NEXT:    movl $8, %eax
 ; CHECK-FMA-NEXT:    shlxq %rdi, %rax, %rax
-; CHECK-FMA-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; CHECK-FMA-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = [-9.0E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-FMA-NEXT:    retq
@@ -1451,7 +1451,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
 ; CHECK-AVX2-NEXT:    movl $1, %eax
 ; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
@@ -1466,7 +1466,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
 ; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
@@ -1478,7 +1478,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bounds(i32 %cnt) nounwind {
 ; CHECK-FMA:       # %bb.0:
 ; CHECK-FMA-NEXT:    movl $1, %eax
 ; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
+; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm15, %xmm0
 ; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = [8.192E+3,0.0E+0,0.0E+0,0.0E+0]
@@ -1562,7 +1562,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
 ; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; CHECK-AVX2-NEXT:    shll %cl, %eax
 ; CHECK-AVX2-NEXT:    movzwl %ax, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-AVX2-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = [2.0E+0,0.0E+0,0.0E+0,0.0E+0]
@@ -1578,7 +1578,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
 ; CHECK-NO-FASTFMA-NEXT:    movzwl %ax, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = [2.0E+0,0.0E+0,0.0E+0,0.0E+0]
@@ -1591,7 +1591,7 @@ define half @fdiv_pow_shl_cnt_fail_out_of_bound2(i16 %cnt) nounwind {
 ; CHECK-FMA-NEXT:    movl $1, %eax
 ; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
 ; CHECK-FMA-NEXT:    movzwl %ax, %eax
-; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-FMA-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-FMA-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = [2.0E+0,0.0E+0,0.0E+0,0.0E+0]
@@ -1648,7 +1648,7 @@ define float @fdiv_pow_shl_cnt32_out_of_bounds2(i32 %cnt) nounwind {
 ; CHECK-AVX2-NEXT:    movl $1, %eax
 ; CHECK-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; CHECK-AVX2-NEXT:    shll %cl, %eax
-; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; CHECK-AVX2-NEXT:    vmovss {{.*#+}} xmm1 = [1.00974148E-28,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-AVX2-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-AVX2-NEXT:    retq
@@ -1659,7 +1659,7 @@ define float @fdiv_pow_shl_cnt32_out_of_bounds2(i32 %cnt) nounwind {
 ; CHECK-NO-FASTFMA-NEXT:    movl $1, %eax
 ; CHECK-NO-FASTFMA-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; CHECK-NO-FASTFMA-NEXT:    shll %cl, %eax
-; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
+; CHECK-NO-FASTFMA-NEXT:    vcvtusi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    vmovss {{.*#+}} xmm1 = [1.00974148E-28,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-NO-FASTFMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-NO-FASTFMA-NEXT:    retq
@@ -1668,7 +1668,7 @@ define float @fdiv_pow_shl_cnt32_out_of_bounds2(i32 %cnt) nounwind {
 ; CHECK-FMA:       # %bb.0:
 ; CHECK-FMA-NEXT:    movl $1, %eax
 ; CHECK-FMA-NEXT:    shlxl %edi, %eax, %eax
-; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm0, %xmm0
+; CHECK-FMA-NEXT:    vcvtusi2ss %eax, %xmm15, %xmm0
 ; CHECK-FMA-NEXT:    vmovss {{.*#+}} xmm1 = [1.00974148E-28,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; CHECK-FMA-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fold-load-unops.ll b/external/llvm-project/llvm/test/CodeGen/X86/fold-load-unops.ll
index d3e34f8d8ffd..35e14e5cf898 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fold-load-unops.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fold-load-unops.ll
@@ -89,7 +89,7 @@ define float @rcpss_size(ptr %a) optsize {
 ;
 ; AVX-LABEL: rcpss_size:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrcpss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrcpss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load float, ptr %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
@@ -106,7 +106,7 @@ define <4 x float> @rcpss_full_size(ptr %a) optsize {
 ;
 ; AVX-LABEL: rcpss_full_size:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrcpss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrcpss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load <4 x float>, ptr %a
     %res = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %ld)
@@ -121,7 +121,7 @@ define float @rcpss_pgso(ptr %a) !prof !14 {
 ;
 ; AVX-LABEL: rcpss_pgso:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrcpss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrcpss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load float, ptr %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
@@ -138,7 +138,7 @@ define <4 x float> @rcpss_full_pgso(ptr %a) !prof !14 {
 ;
 ; AVX-LABEL: rcpss_full_pgso:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrcpss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrcpss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load <4 x float>, ptr %a
     %res = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %ld)
@@ -153,7 +153,7 @@ define float @rsqrtss_size(ptr %a) optsize {
 ;
 ; AVX-LABEL: rsqrtss_size:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrsqrtss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load float, ptr %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
@@ -170,7 +170,7 @@ define <4 x float> @rsqrtss_full_size(ptr %a) optsize {
 ;
 ; AVX-LABEL: rsqrtss_full_size:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrsqrtss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load <4 x float>, ptr %a
     %res = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %ld)
@@ -185,7 +185,7 @@ define float @rsqrtss_pgso(ptr %a) !prof !14 {
 ;
 ; AVX-LABEL: rsqrtss_pgso:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrsqrtss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load float, ptr %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
@@ -202,7 +202,7 @@ define <4 x float> @rsqrtss_full_pgso(ptr %a) !prof !14 {
 ;
 ; AVX-LABEL: rsqrtss_full_pgso:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vrsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vrsqrtss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load <4 x float>, ptr %a
     %res = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %ld)
@@ -217,7 +217,7 @@ define float @sqrtss_size(ptr %a) optsize{
 ;
 ; AVX-LABEL: sqrtss_size:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vsqrtss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load float, ptr %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
@@ -268,7 +268,7 @@ define float @sqrtss_pgso(ptr %a) !prof !14 {
 ;
 ; AVX-LABEL: sqrtss_pgso:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vsqrtss (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vsqrtss (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load float, ptr %a
     %ins = insertelement <4 x float> undef, float %ld, i32 0
@@ -319,7 +319,7 @@ define double @sqrtsd_size(ptr %a) optsize {
 ;
 ; AVX-LABEL: sqrtsd_size:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vsqrtsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vsqrtsd (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load double, ptr %a
     %ins = insertelement <2 x double> undef, double %ld, i32 0
@@ -370,7 +370,7 @@ define double @sqrtsd_pgso(ptr %a) !prof !14 {
 ;
 ; AVX-LABEL: sqrtsd_pgso:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vsqrtsd (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vsqrtsd (%rdi), %xmm15, %xmm0
 ; AVX-NEXT:    retq
     %ld = load double, ptr %a
     %ins = insertelement <2 x double> undef, double %ld, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fold-masked-merge.ll b/external/llvm-project/llvm/test/CodeGen/X86/fold-masked-merge.ll
index b2614c5fe049..05e7b2a2de37 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fold-masked-merge.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fold-masked-merge.ll
@@ -30,18 +30,17 @@ define i32 @masked_merge0(i32 %a0, i32 %a1, i32 %a2) {
 define i16 @masked_merge1(i16 %a0, i16 %a1, i16 %a2) {
 ; NOBMI-LABEL: masked_merge1:
 ; NOBMI:       # %bb.0:
-; NOBMI-NEXT:    movl %edi, %eax
-; NOBMI-NEXT:    andl %edi, %esi
-; NOBMI-NEXT:    notl %eax
-; NOBMI-NEXT:    andl %edx, %eax
-; NOBMI-NEXT:    orl %esi, %eax
+; NOBMI-NEXT:    movl %esi, %eax
+; NOBMI-NEXT:    xorl %edx, %eax
+; NOBMI-NEXT:    andl %edi, %eax
+; NOBMI-NEXT:    xorl %edx, %eax
 ; NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
 ; NOBMI-NEXT:    retq
 ;
 ; BMI-LABEL: masked_merge1:
 ; BMI:       # %bb.0:
-; BMI-NEXT:    andl %edi, %esi
 ; BMI-NEXT:    andnl %edx, %edi, %eax
+; BMI-NEXT:    andl %edi, %esi
 ; BMI-NEXT:    orl %esi, %eax
 ; BMI-NEXT:    # kill: def $ax killed $ax killed $eax
 ; BMI-NEXT:    retq
@@ -53,20 +52,11 @@ define i16 @masked_merge1(i16 %a0, i16 %a1, i16 %a2) {
 }
 
 define i8 @masked_merge2(i8 %a0, i8 %a1, i8 %a2) {
-; NOBMI-LABEL: masked_merge2:
-; NOBMI:       # %bb.0:
-; NOBMI-NEXT:    movl %esi, %eax
-; NOBMI-NEXT:    # kill: def $al killed $al killed $eax
-; NOBMI-NEXT:    retq
-;
-; BMI-LABEL: masked_merge2:
-; BMI:       # %bb.0:
-; BMI-NEXT:    movl %edi, %eax
-; BMI-NEXT:    notb %al
-; BMI-NEXT:    andb %sil, %al
-; BMI-NEXT:    andb %dil, %sil
-; BMI-NEXT:    orb %sil, %al
-; BMI-NEXT:    retq
+; CHECK-LABEL: masked_merge2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
   %not = xor i8 %a0, -1
   %and0 = and i8 %not, %a1
   %and1 = and i8 %a1, %a0
@@ -279,3 +269,27 @@ define i32 @masked_merge_no_transform2(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
   store i32 %and1, ptr %p1
   ret i32 %or
 }
+
+define i32 @pr137641_crash({ i8, i32 } %0) {
+; NOBMI-LABEL: pr137641_crash:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %esi, %eax
+; NOBMI-NEXT:    andl $201, %eax
+; NOBMI-NEXT:    xorl $1, %eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: pr137641_crash:
+; BMI:       # %bb.0:
+; BMI-NEXT:    movl %esi, %eax
+; BMI-NEXT:    notl %eax
+; BMI-NEXT:    andl $1, %eax
+; BMI-NEXT:    andl $200, %esi
+; BMI-NEXT:    orl %esi, %eax
+; BMI-NEXT:    retq
+  %asmresult1.i = extractvalue { i8, i32 } %0, 1
+  %not = xor i32 %asmresult1.i, 1
+  %and = and i32 1, %not
+  %and1 = and i32 %asmresult1.i, 200
+  %2 = or i32 %and, %and1
+  ret i32 %2
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp-intrinsics.ll
index 3577f252f50d..5d69a217fb40 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp-intrinsics.ll
@@ -2018,7 +2018,7 @@ define double @sifdb(i8 %x) #0 {
 ; AVX-LABEL: sifdb:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movsbl %dil, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.sitofp.f64.i8(i8 %x,
@@ -2062,7 +2062,7 @@ define double @sifdw(i16 %x) #0 {
 ; AVX-LABEL: sifdw:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movswl %di, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.sitofp.f64.i16(i16 %x,
@@ -2103,7 +2103,7 @@ define double @sifdi(i32 %x) #0 {
 ;
 ; AVX-LABEL: sifdi:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.sitofp.f64.i32(i32 %x,
@@ -2147,7 +2147,7 @@ define float @siffb(i8 %x) #0 {
 ; AVX-LABEL: siffb:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movsbl %dil, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.sitofp.f32.i8(i8 %x,
@@ -2191,7 +2191,7 @@ define float @siffw(i16 %x) #0 {
 ; AVX-LABEL: siffw:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movswl %di, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.sitofp.f32.i16(i16 %x,
@@ -2232,7 +2232,7 @@ define float @siffi(i32 %x) #0 {
 ;
 ; AVX-LABEL: siffi:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %x,
@@ -2267,7 +2267,7 @@ define double @sifdl(i64 %x) #0 {
 ;
 ; AVX-LABEL: sifdl:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.sitofp.f64.i64(i64 %x,
@@ -2302,7 +2302,7 @@ define float @siffl(i64 %x) #0 {
 ;
 ; AVX-LABEL: siffl:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.sitofp.f32.i64(i64 %x,
@@ -2349,7 +2349,7 @@ define double @uifdb(i8 %x) #0 {
 ; AVX-LABEL: uifdb:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movzbl %dil, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.uitofp.f64.i8(i8 %x,
@@ -2393,7 +2393,7 @@ define double @uifdw(i16 %x) #0 {
 ; AVX-LABEL: uifdw:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movzwl %di, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.uitofp.f64.i16(i16 %x,
@@ -2440,12 +2440,12 @@ define double @uifdi(i32 %x) #0 {
 ; AVX1-LABEL: uifdi:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    movl %edi, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: uifdi:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 %x,
@@ -2518,7 +2518,7 @@ define double @uifdl(i64 %x) #0 {
 ; AVX1-NEXT:    orq %rax, %rcx
 ; AVX1-NEXT:    testq %rdi, %rdi
 ; AVX1-NEXT:    cmovnsq %rdi, %rcx
-; AVX1-NEXT:    vcvtsi2sd %rcx, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rcx, %xmm15, %xmm0
 ; AVX1-NEXT:    jns .LBB48_2
 ; AVX1-NEXT:  # %bb.1:
 ; AVX1-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
@@ -2527,7 +2527,7 @@ define double @uifdl(i64 %x) #0 {
 ;
 ; AVX512-LABEL: uifdl:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call double @llvm.experimental.constrained.uitofp.f64.i64(i64 %x,
@@ -2571,7 +2571,7 @@ define float @uiffb(i8 %x) #0 {
 ; AVX-LABEL: uiffb:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movzbl %dil, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.uitofp.f32.i8(i8 %x,
@@ -2615,7 +2615,7 @@ define float @uiffw(i16 %x) #0 {
 ; AVX-LABEL: uiffw:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    movzwl %di, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.uitofp.f32.i16(i16 %x,
@@ -2662,12 +2662,12 @@ define float @uiffi(i32 %x) #0 {
 ; AVX1-LABEL: uiffi:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    movl %edi, %eax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: uiffi:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %x,
@@ -2740,7 +2740,7 @@ define float @uiffl(i64 %x) #0 {
 ; AVX1-NEXT:    orq %rax, %rcx
 ; AVX1-NEXT:    testq %rdi, %rdi
 ; AVX1-NEXT:    cmovnsq %rdi, %rcx
-; AVX1-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rcx, %xmm15, %xmm0
 ; AVX1-NEXT:    jns .LBB52_2
 ; AVX1-NEXT:  # %bb.1:
 ; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
@@ -2749,7 +2749,7 @@ define float @uiffl(i64 %x) #0 {
 ;
 ; AVX512-LABEL: uiffl:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call float @llvm.experimental.constrained.uitofp.f32.i64(i64 %x,
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp-round.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp-round.ll
index e98fb8e374c0..8595b63fc810 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp-round.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp-round.ll
@@ -52,7 +52,7 @@ define half @round_f16(half %h) {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
 ; AVX512F-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
@@ -98,7 +98,7 @@ define float @round_f32(float %x) {
 ; AVX512F-LABEL: round_f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
 ; AVX512F-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -106,7 +106,7 @@ define float @round_f32(float %x) {
 ; AVX512FP16-LABEL: round_f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
 ; AVX512FP16-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -142,7 +142,7 @@ define double @round_f64(double %x) {
 ; AVX512F-LABEL: round_f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512F-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
 ; AVX512F-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -150,7 +150,7 @@ define double @round_f64(double %x) {
 ; AVX512FP16-LABEL: round_f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
 ; AVX512FP16-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundsd $11, %xmm0, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -208,7 +208,7 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
 ; AVX512F-LABEL: round_v4f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512F-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
 ; AVX512F-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundps $11, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -216,7 +216,7 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
 ; AVX512FP16-LABEL: round_v4f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
 ; AVX512FP16-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundps $11, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -262,7 +262,7 @@ define <2 x double> @round_v2f64(<2 x double> %x) {
 ; AVX512F-LABEL: round_v2f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512F-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
 ; AVX512F-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vroundpd $11, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
@@ -270,7 +270,7 @@ define <2 x double> @round_v2f64(<2 x double> %x) {
 ; AVX512FP16-LABEL: round_v2f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & mem)
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst)
 ; AVX512FP16-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    vroundpd $11, %xmm0, %xmm0
 ; AVX512FP16-NEXT:    retq
@@ -356,7 +356,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; AVX512F-LABEL: round_v8f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
 ; AVX512F-NEXT:    vaddps %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vroundps $11, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -364,7 +364,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; AVX512FP16-LABEL: round_v8f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
 ; AVX512FP16-NEXT:    vaddps %ymm1, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    vroundps $11, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    retq
@@ -426,7 +426,7 @@ define <4 x double> @round_v4f64(<4 x double> %x) {
 ; AVX512F-LABEL: round_v4f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & m64bcst)
 ; AVX512F-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vroundpd $11, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -434,7 +434,7 @@ define <4 x double> @round_v4f64(<4 x double> %x) {
 ; AVX512FP16-LABEL: round_v4f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & mem)
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & m64bcst)
 ; AVX512FP16-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    vroundpd $11, %ymm0, %ymm0
 ; AVX512FP16-NEXT:    retq
@@ -582,7 +582,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; AVX512F-LABEL: round_v16f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
 ; AVX512F-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vrndscaleps $11, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
@@ -590,7 +590,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; AVX512FP16-LABEL: round_v16f32:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512FP16-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
+; AVX512FP16-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm0 & m32bcst)
 ; AVX512FP16-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    vrndscaleps $11, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    retq
@@ -690,7 +690,7 @@ define <8 x double> @round_v8f64(<8 x double> %x) {
 ; AVX512F-LABEL: round_v8f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
 ; AVX512F-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vrndscalepd $11, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
@@ -698,7 +698,7 @@ define <8 x double> @round_v8f64(<8 x double> %x) {
 ; AVX512FP16-LABEL: round_v8f64:
 ; AVX512FP16:       ## %bb.0:
 ; AVX512FP16-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1,4.9999999999999994E-1]
-; AVX512FP16-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
+; AVX512FP16-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & m64bcst)
 ; AVX512FP16-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    vrndscalepd $11, %zmm0, %zmm0
 ; AVX512FP16-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll
index fbc798d8bbe4..b013ddad19a9 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-fp16.ll
@@ -44,7 +44,7 @@ define half @fadd_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -92,7 +92,7 @@ define half @fsub_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX-NEXT:    vsubss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -140,7 +140,7 @@ define half @fmul_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -188,7 +188,7 @@ define half @fdiv_f16(half %a, half %b) nounwind strictfp {
 ; AVX-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; AVX-NEXT:    vdivss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -400,7 +400,7 @@ define void @fsqrt_f16(ptr %a) nounwind strictfp {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX-NEXT:    retq
@@ -469,7 +469,7 @@ define half @fma_f16(half %a, half %b, half %c) nounwind strictfp {
 ; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; F16C-NEXT:    callq fmaf@PLT
 ; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; F16C-NEXT:    popq %rax
 ; F16C-NEXT:    retq
@@ -490,7 +490,7 @@ define half @fma_f16(half %a, half %b, half %c) nounwind strictfp {
 ; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
 ; AVX512-NEXT:    vfmadd213ss {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm0
 ; AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll
index 7c0386f0e784..6312a26db9bf 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp-fp16.ll
@@ -33,9 +33,9 @@ define half @sitofp_i1tof16(i1 %x) #0 {
 ; AVX-NEXT:    andb $1, %dil
 ; AVX-NEXT:    negb %dil
 ; AVX-NEXT:    movsbl %dil, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -45,7 +45,7 @@ define half @sitofp_i1tof16(i1 %x) #0 {
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    negb %al
 ; X86-NEXT:    movsbl %al, %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sitofp_i1tof16:
@@ -53,7 +53,7 @@ define half @sitofp_i1tof16(i1 %x) #0 {
 ; X64-NEXT:    andb $1, %dil
 ; X64-NEXT:    negb %dil
 ; X64-NEXT:    movsbl %dil, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.sitofp.f16.i1(i1 %x,
                                                metadata !"round.dynamic",
@@ -74,22 +74,22 @@ define half @sitofp_i8tof16(i8 %x) #0 {
 ; AVX-LABEL: sitofp_i8tof16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movsbl %dil, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i8tof16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sitofp_i8tof16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movsbl %dil, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.sitofp.f16.i8(i8 %x,
                                                metadata !"round.dynamic",
@@ -110,22 +110,22 @@ define half @sitofp_i16tof16(i16 %x) #0 {
 ; AVX-LABEL: sitofp_i16tof16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movswl %di, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i16tof16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sitofp_i16tof16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movswl %di, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.sitofp.f16.i16(i16 %x,
                                                metadata !"round.dynamic",
@@ -144,20 +144,20 @@ define half @sitofp_i32tof16(i32 %x) #0 {
 ;
 ; AVX-LABEL: sitofp_i32tof16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: sitofp_i32tof16:
 ; X86:       # %bb.0:
-; X86-NEXT:    vcvtsi2shl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2shl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sitofp_i32tof16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtsi2sh %edi, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %edi, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %x,
                                                metadata !"round.dynamic",
@@ -176,9 +176,9 @@ define half @sitofp_i64tof16(i64 %x) #0 {
 ;
 ; AVX-LABEL: sitofp_i64tof16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -190,7 +190,7 @@ define half @sitofp_i64tof16(i64 %x) #0 {
 ;
 ; X64-LABEL: sitofp_i64tof16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtsi2sh %rdi, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %rdi, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.sitofp.f16.i64(i64 %x,
                                                metadata !"round.dynamic",
@@ -211,9 +211,9 @@ define half @uitofp_i1tof16(i1 %x) #0 {
 ; AVX-LABEL: uitofp_i1tof16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    andl $1, %edi
-; AVX-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -222,13 +222,13 @@ define half @uitofp_i1tof16(i1 %x) #0 {
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: uitofp_i1tof16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    andl $1, %edi
-; X64-NEXT:    vcvtsi2sh %edi, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %edi, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.uitofp.f16.i1(i1 %x,
                                                metadata !"round.dynamic",
@@ -249,22 +249,22 @@ define half @uitofp_i8tof16(i8 %x) #0 {
 ; AVX-LABEL: uitofp_i8tof16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movzbl %dil, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i8tof16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: uitofp_i8tof16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.uitofp.f16.i8(i8 %x,
                                                metadata !"round.dynamic",
@@ -285,22 +285,22 @@ define half @uitofp_i16tof16(i16 %x) #0 {
 ; AVX-LABEL: uitofp_i16tof16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movzwl %di, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i16tof16:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X86-NEXT:    vcvtsi2sh %eax, %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: uitofp_i16tof16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl %di, %eax
-; X64-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; X64-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.uitofp.f16.i16(i16 %x,
                                                metadata !"round.dynamic",
@@ -321,28 +321,28 @@ define half @uitofp_i32tof16(i32 %x) #0 {
 ; F16C-LABEL: uitofp_i32tof16:
 ; F16C:       # %bb.0:
 ; F16C-NEXT:    movl %edi, %eax
-; F16C-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; F16C-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: uitofp_i32tof16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; X86-LABEL: uitofp_i32tof16:
 ; X86:       # %bb.0:
-; X86-NEXT:    vcvtusi2shl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vcvtusi2shl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: uitofp_i32tof16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtusi2sh %edi, %xmm0, %xmm0
+; X64-NEXT:    vcvtusi2sh %edi, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.uitofp.f16.i32(i32 %x,
                                                metadata !"round.dynamic",
@@ -381,21 +381,21 @@ define half @uitofp_i64tof16(i64 %x) #0 {
 ; F16C-NEXT:    orq %rax, %rcx
 ; F16C-NEXT:    testq %rdi, %rdi
 ; F16C-NEXT:    cmovnsq %rdi, %rcx
-; F16C-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
+; F16C-NEXT:    vcvtsi2ss %rcx, %xmm15, %xmm0
 ; F16C-NEXT:    jns .LBB9_2
 ; F16C-NEXT:  # %bb.1:
 ; F16C-NEXT:    vaddss %xmm0, %xmm0, %xmm0
 ; F16C-NEXT:  .LBB9_2:
 ; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: uitofp_i64tof16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
@@ -407,7 +407,7 @@ define half @uitofp_i64tof16(i64 %x) #0 {
 ;
 ; X64-LABEL: uitofp_i64tof16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vcvtusi2sh %rdi, %xmm0, %xmm0
+; X64-NEXT:    vcvtusi2sh %rdi, %xmm31, %xmm0
 ; X64-NEXT:    retq
   %result = call half @llvm.experimental.constrained.uitofp.f16.i64(i64 %x,
                                                metadata !"round.dynamic",
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
index 4933a870ddd8..f0aa3827ce93 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
@@ -62,7 +62,7 @@ define float @sitofp_i1tof32(i1 %x) #0 {
 ; AVX-X86-NEXT:    andb $1, %al
 ; AVX-X86-NEXT:    negb %al
 ; AVX-X86-NEXT:    movsbl %al, %eax
-; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX-X86-NEXT:    flds (%esp)
 ; AVX-X86-NEXT:    wait
@@ -75,7 +75,7 @@ define float @sitofp_i1tof32(i1 %x) #0 {
 ; AVX-X64-NEXT:    andb $1, %dil
 ; AVX-X64-NEXT:    negb %dil
 ; AVX-X64-NEXT:    movsbl %dil, %eax
-; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i1tof32:
@@ -123,7 +123,7 @@ define float @sitofp_i8tof32(i8 %x) #0 {
 ; AVX-X86-NEXT:    pushl %eax
 ; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX-X86-NEXT:    flds (%esp)
 ; AVX-X86-NEXT:    wait
@@ -134,7 +134,7 @@ define float @sitofp_i8tof32(i8 %x) #0 {
 ; AVX-X64-LABEL: sitofp_i8tof32:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movsbl %dil, %eax
-; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i8tof32:
@@ -179,7 +179,7 @@ define float @sitofp_i16tof32(i16 %x) #0 {
 ; AVX-X86-NEXT:    pushl %eax
 ; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX-X86-NEXT:    flds (%esp)
 ; AVX-X86-NEXT:    wait
@@ -190,7 +190,7 @@ define float @sitofp_i16tof32(i16 %x) #0 {
 ; AVX-X64-LABEL: sitofp_i16tof32:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movswl %di, %eax
-; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i16tof32:
@@ -232,7 +232,7 @@ define float @sitofp_i32tof32(i32 %x) #0 {
 ; AVX-X86:       # %bb.0:
 ; AVX-X86-NEXT:    pushl %eax
 ; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
-; AVX-X86-NEXT:    vcvtsi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2ssl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX-X86-NEXT:    flds (%esp)
 ; AVX-X86-NEXT:    wait
@@ -242,7 +242,7 @@ define float @sitofp_i32tof32(i32 %x) #0 {
 ;
 ; AVX-X64-LABEL: sitofp_i32tof32:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i32tof32:
@@ -294,7 +294,7 @@ define float @sitofp_i64tof32(i64 %x) #0 {
 ;
 ; AVX-X64-LABEL: sitofp_i64tof32:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i64tof32:
@@ -337,7 +337,7 @@ define float @uitofp_i1tof32(i1 %x) #0 {
 ; AVX-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; AVX-X86-NEXT:    andb $1, %al
 ; AVX-X86-NEXT:    movzbl %al, %eax
-; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX-X86-NEXT:    flds (%esp)
 ; AVX-X86-NEXT:    wait
@@ -348,7 +348,7 @@ define float @uitofp_i1tof32(i1 %x) #0 {
 ; AVX-X64-LABEL: uitofp_i1tof32:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    andl $1, %edi
-; AVX-X64-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i1tof32:
@@ -395,7 +395,7 @@ define float @uitofp_i8tof32(i8 %x) #0 {
 ; AVX-X86-NEXT:    pushl %eax
 ; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX-X86-NEXT:    flds (%esp)
 ; AVX-X86-NEXT:    wait
@@ -406,7 +406,7 @@ define float @uitofp_i8tof32(i8 %x) #0 {
 ; AVX-X64-LABEL: uitofp_i8tof32:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movzbl %dil, %eax
-; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i8tof32:
@@ -451,7 +451,7 @@ define float @uitofp_i16tof32(i16 %x) #0 {
 ; AVX-X86-NEXT:    pushl %eax
 ; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2ss %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX-X86-NEXT:    flds (%esp)
 ; AVX-X86-NEXT:    wait
@@ -462,7 +462,7 @@ define float @uitofp_i16tof32(i16 %x) #0 {
 ; AVX-X64-LABEL: uitofp_i16tof32:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movzwl %di, %eax
-; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i16tof32:
@@ -534,14 +534,14 @@ define float @uitofp_i32tof32(i32 %x) #0 {
 ; AVX1-X64-LABEL: uitofp_i32tof32:
 ; AVX1-X64:       # %bb.0:
 ; AVX1-X64-NEXT:    movl %edi, %eax
-; AVX1-X64-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; AVX1-X64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-X64-NEXT:    retq
 ;
 ; AVX512-X86-LABEL: uitofp_i32tof32:
 ; AVX512-X86:       # %bb.0:
 ; AVX512-X86-NEXT:    pushl %eax
 ; AVX512-X86-NEXT:    .cfi_def_cfa_offset 8
-; AVX512-X86-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512-X86-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; AVX512-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX512-X86-NEXT:    flds (%esp)
 ; AVX512-X86-NEXT:    wait
@@ -551,7 +551,7 @@ define float @uitofp_i32tof32(i32 %x) #0 {
 ;
 ; AVX512-X64-LABEL: uitofp_i32tof32:
 ; AVX512-X64:       # %bb.0:
-; AVX512-X64-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; AVX512-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i32tof32:
@@ -656,7 +656,7 @@ define float @uitofp_i64tof32(i64 %x) #0 {
 ; AVX1-X64-NEXT:    orq %rax, %rcx
 ; AVX1-X64-NEXT:    testq %rdi, %rdi
 ; AVX1-X64-NEXT:    cmovnsq %rdi, %rcx
-; AVX1-X64-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
+; AVX1-X64-NEXT:    vcvtsi2ss %rcx, %xmm15, %xmm0
 ; AVX1-X64-NEXT:    jns .LBB9_2
 ; AVX1-X64-NEXT:  # %bb.1:
 ; AVX1-X64-NEXT:    vaddss %xmm0, %xmm0, %xmm0
@@ -665,7 +665,7 @@ define float @uitofp_i64tof32(i64 %x) #0 {
 ;
 ; AVX512-X64-LABEL: uitofp_i64tof32:
 ; AVX512-X64:       # %bb.0:
-; AVX512-X64-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    vcvtusi2ss %rdi, %xmm15, %xmm0
 ; AVX512-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i64tof32:
@@ -733,7 +733,7 @@ define double @sitofp_i8tof64(i8 %x) #0 {
 ; AVX-X86-NEXT:    andl $-8, %esp
 ; AVX-X86-NEXT:    subl $8, %esp
 ; AVX-X86-NEXT:    movsbl 8(%ebp), %eax
-; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX-X86-NEXT:    fldl (%esp)
 ; AVX-X86-NEXT:    wait
@@ -745,7 +745,7 @@ define double @sitofp_i8tof64(i8 %x) #0 {
 ; AVX-X64-LABEL: sitofp_i8tof64:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movsbl %dil, %eax
-; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i8tof64:
@@ -801,7 +801,7 @@ define double @sitofp_i16tof64(i16 %x) #0 {
 ; AVX-X86-NEXT:    andl $-8, %esp
 ; AVX-X86-NEXT:    subl $8, %esp
 ; AVX-X86-NEXT:    movswl 8(%ebp), %eax
-; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX-X86-NEXT:    fldl (%esp)
 ; AVX-X86-NEXT:    wait
@@ -813,7 +813,7 @@ define double @sitofp_i16tof64(i16 %x) #0 {
 ; AVX-X64-LABEL: sitofp_i16tof64:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movswl %di, %eax
-; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i16tof64:
@@ -866,7 +866,7 @@ define double @sitofp_i32tof64(i32 %x) #0 {
 ; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
 ; AVX-X86-NEXT:    andl $-8, %esp
 ; AVX-X86-NEXT:    subl $8, %esp
-; AVX-X86-NEXT:    vcvtsi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2sdl 8(%ebp), %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX-X86-NEXT:    fldl (%esp)
 ; AVX-X86-NEXT:    wait
@@ -877,7 +877,7 @@ define double @sitofp_i32tof64(i32 %x) #0 {
 ;
 ; AVX-X64-LABEL: sitofp_i32tof64:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i32tof64:
@@ -941,7 +941,7 @@ define double @sitofp_i64tof64(i64 %x) #0 {
 ;
 ; AVX-X64-LABEL: sitofp_i64tof64:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: sitofp_i64tof64:
@@ -995,7 +995,7 @@ define double @uitofp_i1tof64(i1 %x) #0 {
 ; AVX-X86-NEXT:    movzbl 8(%ebp), %eax
 ; AVX-X86-NEXT:    andb $1, %al
 ; AVX-X86-NEXT:    movzbl %al, %eax
-; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX-X86-NEXT:    fldl (%esp)
 ; AVX-X86-NEXT:    wait
@@ -1007,7 +1007,7 @@ define double @uitofp_i1tof64(i1 %x) #0 {
 ; AVX-X64-LABEL: uitofp_i1tof64:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    andl $1, %edi
-; AVX-X64-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i1tof64:
@@ -1065,7 +1065,7 @@ define double @uitofp_i8tof64(i8 %x) #0 {
 ; AVX-X86-NEXT:    andl $-8, %esp
 ; AVX-X86-NEXT:    subl $8, %esp
 ; AVX-X86-NEXT:    movzbl 8(%ebp), %eax
-; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX-X86-NEXT:    fldl (%esp)
 ; AVX-X86-NEXT:    wait
@@ -1077,7 +1077,7 @@ define double @uitofp_i8tof64(i8 %x) #0 {
 ; AVX-X64-LABEL: uitofp_i8tof64:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movzbl %dil, %eax
-; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i8tof64:
@@ -1133,7 +1133,7 @@ define double @uitofp_i16tof64(i16 %x) #0 {
 ; AVX-X86-NEXT:    andl $-8, %esp
 ; AVX-X86-NEXT:    subl $8, %esp
 ; AVX-X86-NEXT:    movzwl 8(%ebp), %eax
-; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X86-NEXT:    vcvtsi2sd %eax, %xmm7, %xmm0
 ; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX-X86-NEXT:    fldl (%esp)
 ; AVX-X86-NEXT:    wait
@@ -1145,7 +1145,7 @@ define double @uitofp_i16tof64(i16 %x) #0 {
 ; AVX-X64-LABEL: uitofp_i16tof64:
 ; AVX-X64:       # %bb.0:
 ; AVX-X64-NEXT:    movzwl %di, %eax
-; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-X64-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i16tof64:
@@ -1217,7 +1217,7 @@ define double @uitofp_i32tof64(i32 %x) #0 {
 ; AVX1-X64-LABEL: uitofp_i32tof64:
 ; AVX1-X64:       # %bb.0:
 ; AVX1-X64-NEXT:    movl %edi, %eax
-; AVX1-X64-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX1-X64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-X64-NEXT:    retq
 ;
 ; AVX512-X86-LABEL: uitofp_i32tof64:
@@ -1229,7 +1229,7 @@ define double @uitofp_i32tof64(i32 %x) #0 {
 ; AVX512-X86-NEXT:    .cfi_def_cfa_register %ebp
 ; AVX512-X86-NEXT:    andl $-8, %esp
 ; AVX512-X86-NEXT:    subl $8, %esp
-; AVX512-X86-NEXT:    vcvtusi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX512-X86-NEXT:    vcvtusi2sdl 8(%ebp), %xmm7, %xmm0
 ; AVX512-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX512-X86-NEXT:    fldl (%esp)
 ; AVX512-X86-NEXT:    wait
@@ -1240,7 +1240,7 @@ define double @uitofp_i32tof64(i32 %x) #0 {
 ;
 ; AVX512-X64-LABEL: uitofp_i32tof64:
 ; AVX512-X64:       # %bb.0:
-; AVX512-X64-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; AVX512-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i32tof64:
@@ -1345,7 +1345,7 @@ define double @uitofp_i64tof64(i64 %x) #0 {
 ; AVX1-X64-NEXT:    orq %rax, %rcx
 ; AVX1-X64-NEXT:    testq %rdi, %rdi
 ; AVX1-X64-NEXT:    cmovnsq %rdi, %rcx
-; AVX1-X64-NEXT:    vcvtsi2sd %rcx, %xmm0, %xmm0
+; AVX1-X64-NEXT:    vcvtsi2sd %rcx, %xmm15, %xmm0
 ; AVX1-X64-NEXT:    jns .LBB18_2
 ; AVX1-X64-NEXT:  # %bb.1:
 ; AVX1-X64-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
@@ -1354,7 +1354,7 @@ define double @uitofp_i64tof64(i64 %x) #0 {
 ;
 ; AVX512-X64-LABEL: uitofp_i64tof64:
 ; AVX512-X64:       # %bb.0:
-; AVX512-X64-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512-X64-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; AVX512-X64-NEXT:    retq
 ;
 ; X87-LABEL: uitofp_i64tof64:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll
index 1ab97dafb851..85a43394a1dc 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll
@@ -31,13 +31,13 @@ define half @fceil32(half %f) #0 {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fceil32:
 ; X86:       # %bb.0:
-; X86-NEXT:    vrndscalesh $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vrndscalesh $10, {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: fceil32:
@@ -67,13 +67,13 @@ define half @ffloor32(half %f) #0 {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: ffloor32:
 ; X86:       # %bb.0:
-; X86-NEXT:    vrndscalesh $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vrndscalesh $9, {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: ffloor32:
@@ -103,13 +103,13 @@ define half @ftrunc32(half %f) #0 {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: ftrunc32:
 ; X86:       # %bb.0:
-; X86-NEXT:    vrndscalesh $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vrndscalesh $11, {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: ftrunc32:
@@ -139,13 +139,13 @@ define half @frint32(half %f) #0 {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    vroundss $4, %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: frint32:
 ; X86:       # %bb.0:
-; X86-NEXT:    vrndscalesh $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vrndscalesh $4, {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: frint32:
@@ -176,13 +176,13 @@ define half @fnearbyint32(half %f) #0 {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    vroundss $12, %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: fnearbyint32:
 ; X86:       # %bb.0:
-; X86-NEXT:    vrndscalesh $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vrndscalesh $12, {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: fnearbyint32:
@@ -213,13 +213,13 @@ define half @froundeven16(half %f) #0 {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    vroundss $8, %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X86-LABEL: froundeven16:
 ; X86:       # %bb.0:
-; X86-NEXT:    vrndscalesh $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vrndscalesh $8, {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: froundeven16:
@@ -251,7 +251,7 @@ define half @fround16(half %f) #0 {
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX-NEXT:    callq roundf@PLT
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
index a85b53ea62ac..a7eea04181f6 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
@@ -452,7 +452,7 @@ define fp128 @fma(fp128 %x, fp128 %y, fp128 %z) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll fmal
+; X86-NEXT:    calll fmaf128
 ; X86-NEXT:    addl $60, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -552,7 +552,7 @@ define fp128 @frem(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll fmodl
+; X86-NEXT:    calll fmodf128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -641,7 +641,7 @@ define fp128 @ceil(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll ceill
+; X86-NEXT:    calll ceilf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -723,7 +723,7 @@ define fp128 @acos(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll acosl
+; X86-NEXT:    calll acosf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -805,7 +805,7 @@ define fp128 @cos(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll cosl
+; X86-NEXT:    calll cosf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -887,7 +887,7 @@ define fp128 @cosh(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll coshl
+; X86-NEXT:    calll coshf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -969,7 +969,7 @@ define fp128 @exp(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll expl
+; X86-NEXT:    calll expf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1051,7 +1051,7 @@ define fp128 @exp2(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll exp2l
+; X86-NEXT:    calll exp2f128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1133,7 +1133,7 @@ define fp128 @floor(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll floorl
+; X86-NEXT:    calll floorf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1215,7 +1215,7 @@ define fp128 @log(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll logl
+; X86-NEXT:    calll logf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1297,7 +1297,7 @@ define fp128 @log10(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll log10l
+; X86-NEXT:    calll log10f128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1379,7 +1379,7 @@ define fp128 @log2(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll log2l
+; X86-NEXT:    calll log2f128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1465,7 +1465,7 @@ define fp128 @maxnum(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll fmaxl
+; X86-NEXT:    calll fmaxf128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1558,7 +1558,7 @@ define fp128 @minnum(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll fminl
+; X86-NEXT:    calll fminf128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1647,7 +1647,7 @@ define fp128 @nearbyint(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll nearbyintl
+; X86-NEXT:    calll nearbyintf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1733,7 +1733,7 @@ define fp128 @pow(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll powl
+; X86-NEXT:    calll powf128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1913,7 +1913,7 @@ define fp128 @rint(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll rintl
+; X86-NEXT:    calll rintf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1995,7 +1995,7 @@ define fp128 @round(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll roundl
+; X86-NEXT:    calll roundf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2077,7 +2077,7 @@ define fp128 @roundeven(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll roundevenl
+; X86-NEXT:    calll roundevenf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2159,7 +2159,7 @@ define fp128 @asin(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll asinl
+; X86-NEXT:    calll asinf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2241,7 +2241,7 @@ define fp128 @sin(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll sinl
+; X86-NEXT:    calll sinf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2323,7 +2323,7 @@ define fp128 @sinh(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll sinhl
+; X86-NEXT:    calll sinhf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2405,7 +2405,7 @@ define fp128 @sqrt(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll sqrtl
+; X86-NEXT:    calll sqrtf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2487,7 +2487,7 @@ define fp128 @atan(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll atanl
+; X86-NEXT:    calll atanf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2573,7 +2573,7 @@ define fp128 @atan2(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll atan2l
+; X86-NEXT:    calll atan2f128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2662,7 +2662,7 @@ define fp128 @tan(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll tanl
+; X86-NEXT:    calll tanf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2744,7 +2744,7 @@ define fp128 @tanh(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll tanhl
+; X86-NEXT:    calll tanhf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2826,7 +2826,7 @@ define fp128 @trunc(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll truncl
+; X86-NEXT:    calll truncf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2903,7 +2903,7 @@ define i32 @lrint(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll lrintl
+; X86-NEXT:    calll lrintf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 ;
@@ -2953,7 +2953,7 @@ define i64 @llrint(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll llrintl
+; X86-NEXT:    calll llrintf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 ;
@@ -3003,7 +3003,7 @@ define i32 @lround(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll lroundl
+; X86-NEXT:    calll lroundf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 ;
@@ -3053,7 +3053,7 @@ define i64 @llround(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll llroundl
+; X86-NEXT:    calll llroundf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 ;
@@ -3443,8 +3443,6 @@ define i64 @cmp_ueq_q(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    calll __unordtf2
 ; X86-NEXT:    addl $32, %esp
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    setne %al
 ; X86-NEXT:    orb %bl, %al
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
@@ -3526,8 +3524,6 @@ define i64 @cmp_ueq_q(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 {
 ; WIN-X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; WIN-X86-NEXT:    calll ___unordtf2
 ; WIN-X86-NEXT:    addl $32, %esp
-; WIN-X86-NEXT:    testl %eax, %eax
-; WIN-X86-NEXT:    setne %al
 ; WIN-X86-NEXT:    orb %bl, %al
 ; WIN-X86-NEXT:    jne LBB39_1
 ; WIN-X86-NEXT:  # %bb.2:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls.ll b/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls.ll
index 05c02a2139b5..f727a7907862 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fp128-libcalls.ll
@@ -841,7 +841,7 @@ define dso_local void @Test128Rem(fp128 %d1, fp128 %d2) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll fmodl
+; X86-NEXT:    calll fmodf128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -933,7 +933,7 @@ define dso_local void @Test128_1Rem(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl vf128+4
 ; X86-NEXT:    pushl vf128
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll fmodl
+; X86-NEXT:    calll fmodf128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1018,7 +1018,7 @@ define dso_local void @Test128Sqrt(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll sqrtl
+; X86-NEXT:    calll sqrtf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1096,7 +1096,7 @@ define dso_local void @Test128Sin(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll sinl
+; X86-NEXT:    calll sinf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1174,7 +1174,7 @@ define dso_local void @Test128Cos(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll cosl
+; X86-NEXT:    calll cosf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1252,7 +1252,7 @@ define dso_local void @Test128Ceil(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll ceill
+; X86-NEXT:    calll ceilf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1330,7 +1330,7 @@ define dso_local void @Test128Floor(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll floorl
+; X86-NEXT:    calll floorf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1408,7 +1408,7 @@ define dso_local void @Test128Trunc(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll truncl
+; X86-NEXT:    calll truncf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1486,7 +1486,7 @@ define dso_local void @Test128Nearbyint(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll nearbyintl
+; X86-NEXT:    calll nearbyintf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1564,7 +1564,7 @@ define dso_local void @Test128Rint(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll rintl
+; X86-NEXT:    calll rintf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1642,7 +1642,7 @@ define dso_local void @Test128Round(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll roundl
+; X86-NEXT:    calll roundf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, vf128
@@ -1723,7 +1723,7 @@ define fp128 @Test128FMA(fp128 %a, fp128 %b, fp128 %c) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll fmal
+; X86-NEXT:    calll fmaf128
 ; X86-NEXT:    addl $60, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1814,7 +1814,7 @@ define fp128 @Test128Acos(fp128 %a) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll acosl
+; X86-NEXT:    calll acosf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1889,7 +1889,7 @@ define fp128 @Test128Asin(fp128 %a) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll asinl
+; X86-NEXT:    calll asinf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -1964,7 +1964,7 @@ define fp128 @Test128Atan(fp128 %a) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll atanl
+; X86-NEXT:    calll atanf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2043,7 +2043,7 @@ define fp128 @Test128Atan2(fp128 %a, fp128 %b) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll atan2l
+; X86-NEXT:    calll atan2f128
 ; X86-NEXT:    addl $44, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2125,7 +2125,7 @@ define fp128 @Test128Cosh(fp128 %a) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll coshl
+; X86-NEXT:    calll coshf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2200,7 +2200,7 @@ define fp128 @Test128Sinh(fp128 %a) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll sinhl
+; X86-NEXT:    calll sinhf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2275,7 +2275,7 @@ define fp128 @Test128Tan(fp128 %a) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll tanl
+; X86-NEXT:    calll tanf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2350,7 +2350,7 @@ define fp128 @Test128Tanh(fp128 %a) nounwind {
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll tanhl
+; X86-NEXT:    calll tanhf128
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    movaps (%esp), %xmm0
 ; X86-NEXT:    movaps %xmm0, (%esi)
@@ -2403,3 +2403,112 @@ define fp128 @Test128Tanh(fp128 %a) nounwind {
   %x = call fp128 @llvm.tanh.f128(fp128 %a)
   ret fp128 %x
 }
+
+define { fp128, fp128 } @Test128Modf(fp128 %a) nounwind {
+; ANDROID-LABEL: Test128Modf:
+; ANDROID:       # %bb.0:
+; ANDROID-NEXT:    subq $24, %rsp
+; ANDROID-NEXT:    movq %rsp, %rdi
+; ANDROID-NEXT:    callq modfl@PLT
+; ANDROID-NEXT:    movaps (%rsp), %xmm1
+; ANDROID-NEXT:    addq $24, %rsp
+; ANDROID-NEXT:    retq
+;
+; GNU-LABEL: Test128Modf:
+; GNU:       # %bb.0:
+; GNU-NEXT:    subq $24, %rsp
+; GNU-NEXT:    movq %rsp, %rdi
+; GNU-NEXT:    callq modff128@PLT
+; GNU-NEXT:    movaps (%rsp), %xmm1
+; GNU-NEXT:    addq $24, %rsp
+; GNU-NEXT:    retq
+;
+; X86-LABEL: Test128Modf:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $40, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %ecx
+; X86-NEXT:    calll modff128
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps {{[0-9]+}}(%esp), %xmm1
+; X86-NEXT:    movaps %xmm1, 16(%esi)
+; X86-NEXT:    movaps %xmm0, (%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $40, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl $4
+;
+; WIN-LABEL: Test128Modf:
+; WIN:       # %bb.0:
+; WIN-NEXT:    subq $72, %rsp
+; WIN-NEXT:    movaps (%rcx), %xmm0
+; WIN-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; WIN-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; WIN-NEXT:    callq modfl
+; WIN-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
+; WIN-NEXT:    addq $72, %rsp
+; WIN-NEXT:    retq
+;
+; WIN-X86-LABEL: Test128Modf:
+; WIN-X86:       # %bb.0:
+; WIN-X86-NEXT:    pushl %ebp
+; WIN-X86-NEXT:    movl %esp, %ebp
+; WIN-X86-NEXT:    pushl %ebx
+; WIN-X86-NEXT:    pushl %edi
+; WIN-X86-NEXT:    pushl %esi
+; WIN-X86-NEXT:    andl $-16, %esp
+; WIN-X86-NEXT:    subl $64, %esp
+; WIN-X86-NEXT:    movl 8(%ebp), %esi
+; WIN-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; WIN-X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; WIN-X86-NEXT:    pushl %eax
+; WIN-X86-NEXT:    pushl 24(%ebp)
+; WIN-X86-NEXT:    pushl 20(%ebp)
+; WIN-X86-NEXT:    pushl 16(%ebp)
+; WIN-X86-NEXT:    pushl 12(%ebp)
+; WIN-X86-NEXT:    pushl %ecx
+; WIN-X86-NEXT:    calll _modfl
+; WIN-X86-NEXT:    addl $24, %esp
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN-X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN-X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN-X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; WIN-X86-NEXT:    movl %edx, 28(%esi)
+; WIN-X86-NEXT:    movl %eax, 24(%esi)
+; WIN-X86-NEXT:    movl %ecx, 20(%esi)
+; WIN-X86-NEXT:    movl %ebx, 16(%esi)
+; WIN-X86-NEXT:    movl %edi, 12(%esi)
+; WIN-X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; WIN-X86-NEXT:    movl %eax, 8(%esi)
+; WIN-X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; WIN-X86-NEXT:    movl %eax, 4(%esi)
+; WIN-X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; WIN-X86-NEXT:    movl %eax, (%esi)
+; WIN-X86-NEXT:    movl %esi, %eax
+; WIN-X86-NEXT:    leal -12(%ebp), %esp
+; WIN-X86-NEXT:    popl %esi
+; WIN-X86-NEXT:    popl %edi
+; WIN-X86-NEXT:    popl %ebx
+; WIN-X86-NEXT:    popl %ebp
+; WIN-X86-NEXT:    retl
+  %x = call { fp128, fp128 } @llvm.modf.f128(fp128 %a)
+  ret { fp128, fp128 } %x
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll b/external/llvm-project/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll
index e89acc6bb268..480a47e57f07 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/fpcmp-soft-fp.ll
@@ -99,8 +99,6 @@ entry:
 ; CHECK: calll __eqdf2
 ; CHECK: sete
 ; CHECK: calll __unorddf2
-; CHECK: setne
-; CHECK: or
 ; CHECK: retl
 
 define i1 @test11(double %d) #0 {
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/ftrunc.ll b/external/llvm-project/llvm/test/CodeGen/X86/ftrunc.ll
index da44b5ec1371..3ed98589767f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/ftrunc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/ftrunc.ll
@@ -514,7 +514,7 @@ define double @trunc_signed_f64_no_fast_math(double %x) {
 ; X64-AVX1-LABEL: trunc_signed_f64_no_fast_math:
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vcvttsd2si %xmm0, %rax
-; X64-AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm0
+; X64-AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; X64-AVX1-NEXT:    retq
 ;
 ; X86-AVX1-LABEL: trunc_signed_f64_no_fast_math:
@@ -695,7 +695,7 @@ define float @trunc_unsigned_f32_disable_via_intrinsic(float %x) #0 {
 ; X64-AVX1-NEXT:    vucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; X64-AVX1-NEXT:    movl $-1, %eax
 ; X64-AVX1-NEXT:    cmovbel %ecx, %eax
-; X64-AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; X64-AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; X64-AVX1-NEXT:    retq
 ;
 ; X86-AVX1-LABEL: trunc_unsigned_f32_disable_via_intrinsic:
@@ -752,7 +752,7 @@ define double @trunc_signed_f64_disable_via_intrinsic(double %x) #0 {
 ; X64-AVX1-NEXT:    xorl %eax, %eax
 ; X64-AVX1-NEXT:    vucomisd %xmm0, %xmm0
 ; X64-AVX1-NEXT:    cmovnpq %rcx, %rax
-; X64-AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm0
+; X64-AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; X64-AVX1-NEXT:    retq
 ;
 ; X86-AVX1-LABEL: trunc_signed_f64_disable_via_intrinsic:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/external/llvm-project/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
index 0ca3380d188b..7001bf7f2807 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
@@ -492,7 +492,7 @@ define <16 x i8> @splatconstant_fshl_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpsllw $3, %xmm0, %xmm2
 ; GFNIAVX512-NEXT:    vpsrlw $5, %xmm1, %xmm0
-; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
   ret <16 x i8> %res
@@ -518,7 +518,7 @@ define <16 x i8> @splatconstant_fshr_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpaddw %xmm0, %xmm0, %xmm2
 ; GFNIAVX512-NEXT:    vpsrlw $7, %xmm1, %xmm0
-; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>)
   ret <16 x i8> %res
@@ -1311,7 +1311,7 @@ define <32 x i8> @splatconstant_fshl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; GFNIAVX512-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <32 x i8> %res
@@ -1349,7 +1349,7 @@ define <32 x i8> @splatconstant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind
 ; GFNIAVX512:       # %bb.0:
 ; GFNIAVX512-NEXT:    vpsllw $2, %ymm0, %ymm2
 ; GFNIAVX512-NEXT:    vpsrlw $6, %ymm1, %ymm0
-; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; GFNIAVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; GFNIAVX512-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> <i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6>)
   ret <32 x i8> %res
@@ -2775,7 +2775,7 @@ define <64 x i8> @splatconstant_fshl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind
 ; GFNIAVX512BW:       # %bb.0:
 ; GFNIAVX512BW-NEXT:    vpaddw %zmm0, %zmm0, %zmm2
 ; GFNIAVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; GFNIAVX512BW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
   ret <64 x i8> %res
@@ -2836,7 +2836,7 @@ define <64 x i8> @splatconstant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind
 ; GFNIAVX512BW:       # %bb.0:
 ; GFNIAVX512BW-NEXT:    vpsllw $6, %zmm0, %zmm2
 ; GFNIAVX512BW-NEXT:    vpsrlw $2, %zmm1, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; GFNIAVX512BW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>)
   ret <64 x i8> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/gfni-lzcnt.ll b/external/llvm-project/llvm/test/CodeGen/X86/gfni-lzcnt.ll
index 8e48950c32cd..6e93f218f1c1 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/gfni-lzcnt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/gfni-lzcnt.ll
@@ -8,40 +8,29 @@
 define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
 ; GFNISSE-LABEL: testv16i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm2
 ; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; GFNISSE-NEXT:    pxor %xmm3, %xmm3
-; GFNISSE-NEXT:    pcmpeqb %xmm0, %xmm3
-; GFNISSE-NEXT:    pand %xmm2, %xmm3
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm1
-; GFNISSE-NEXT:    paddb %xmm3, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    pxor %xmm1, %xmm1
+; GFNISSE-NEXT:    psubb %xmm0, %xmm1
+; GFNISSE-NEXT:    pand %xmm1, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: testv16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
 ; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX1OR2-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
-; GFNIAVX1OR2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
-; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX1OR2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX1OR2-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
+; GFNIAVX1OR2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: testv16i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
 ; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX512-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
-; GFNIAVX512-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
-; GFNIAVX512-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
 ; GFNIAVX512-NEXT:    retq
   %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 0)
   ret <16 x i8> %out
@@ -50,40 +39,29 @@ define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
 define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
 ; GFNISSE-LABEL: testv16i8u:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm2
 ; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; GFNISSE-NEXT:    pxor %xmm3, %xmm3
-; GFNISSE-NEXT:    pcmpeqb %xmm0, %xmm3
-; GFNISSE-NEXT:    pand %xmm2, %xmm3
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm1
-; GFNISSE-NEXT:    paddb %xmm3, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
+; GFNISSE-NEXT:    pxor %xmm1, %xmm1
+; GFNISSE-NEXT:    psubb %xmm0, %xmm1
+; GFNISSE-NEXT:    pand %xmm1, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: testv16i8u:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
 ; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX1OR2-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
-; GFNIAVX1OR2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1OR2-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
-; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX1OR2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX1OR2-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
+; GFNIAVX1OR2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: testv16i8u:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vmovq {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
 ; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX512-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm3
-; GFNIAVX512-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
-; GFNIAVX512-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; GFNIAVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpsubb %xmm0, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
 ; GFNIAVX512-NEXT:    retq
   %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 -1)
   ret <16 x i8> %out
@@ -92,73 +70,53 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
 define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
 ; GFNISSE-LABEL: testv32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm3
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
-; GFNISSE-NEXT:    pxor %xmm5, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm6
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
-; GFNISSE-NEXT:    pand %xmm3, %xmm0
-; GFNISSE-NEXT:    paddb %xmm6, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm3
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
-; GFNISSE-NEXT:    pcmpeqb %xmm1, %xmm5
-; GFNISSE-NEXT:    pand %xmm3, %xmm5
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm2
-; GFNISSE-NEXT:    paddb %xmm5, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    pxor %xmm3, %xmm3
+; GFNISSE-NEXT:    pxor %xmm4, %xmm4
+; GFNISSE-NEXT:    psubb %xmm0, %xmm4
+; GFNISSE-NEXT:    pand %xmm4, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [12307476859704049664,12307476859704049664]
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
+; GFNISSE-NEXT:    psubb %xmm1, %xmm3
+; GFNISSE-NEXT:    pand %xmm3, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm4, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: testv32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm6
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
-; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm0, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm3, %xmm0
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX1-NEXT:    # xmm2 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm2, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm2, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm2
+; GFNIAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; GFNIAVX1-NEXT:    vpsubb %xmm0, %xmm3, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: testv32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX2-NEXT:    # ymm1 = mem[0,1,0,1]
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX2-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX2-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [12307476859704049664,12307476859704049664,12307476859704049664,12307476859704049664]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $8, %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: testv32i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512-NEXT:    # ymm1 = mem[0,1,0,1]
-; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
-; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX512-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
-; GFNIAVX512-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
-; GFNIAVX512-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
+; GFNIAVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 0)
   ret <32 x i8> %out
@@ -167,73 +125,53 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
 define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
 ; GFNISSE-LABEL: testv32i8u:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm3
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
-; GFNISSE-NEXT:    pxor %xmm5, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm6
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
-; GFNISSE-NEXT:    pand %xmm3, %xmm0
-; GFNISSE-NEXT:    paddb %xmm6, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm3
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
-; GFNISSE-NEXT:    pcmpeqb %xmm1, %xmm5
-; GFNISSE-NEXT:    pand %xmm3, %xmm5
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm2
-; GFNISSE-NEXT:    paddb %xmm5, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    pxor %xmm3, %xmm3
+; GFNISSE-NEXT:    pxor %xmm4, %xmm4
+; GFNISSE-NEXT:    psubb %xmm0, %xmm4
+; GFNISSE-NEXT:    pand %xmm4, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [12307476859704049664,12307476859704049664]
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
+; GFNISSE-NEXT:    psubb %xmm1, %xmm3
+; GFNISSE-NEXT:    pand %xmm3, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm4, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: testv32i8u:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm6
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
-; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm5, %xmm0, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm3, %xmm0
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX1-NEXT:    # xmm2 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm2, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm2, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm2
+; GFNIAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; GFNIAVX1-NEXT:    vpsubb %xmm0, %xmm3, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: testv32i8u:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX2-NEXT:    # ymm1 = mem[0,1,0,1]
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX2-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX2-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [12307476859704049664,12307476859704049664,12307476859704049664,12307476859704049664]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $8, %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: testv32i8u:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512-NEXT:    # ymm1 = mem[0,1,0,1]
-; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
-; GFNIAVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; GFNIAVX512-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
-; GFNIAVX512-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; GFNIAVX512-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
-; GFNIAVX512-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; GFNIAVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX512-NEXT:    vpsubb %ymm0, %ymm1, %ymm1
+; GFNIAVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 -1)
   ret <32 x i8> %out
@@ -242,130 +180,93 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
 define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 ; GFNISSE-LABEL: testv64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
-; GFNISSE-NEXT:    movq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm7
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
 ; GFNISSE-NEXT:    pxor %xmm5, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm8
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
-; GFNISSE-NEXT:    pand %xmm7, %xmm0
-; GFNISSE-NEXT:    paddb %xmm8, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm7
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm8
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm1
+; GFNISSE-NEXT:    pxor %xmm6, %xmm6
+; GFNISSE-NEXT:    psubb %xmm0, %xmm6
+; GFNISSE-NEXT:    pand %xmm6, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [12307476859704049664,12307476859704049664]
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    pxor %xmm7, %xmm7
+; GFNISSE-NEXT:    psubb %xmm1, %xmm7
 ; GFNISSE-NEXT:    pand %xmm7, %xmm1
-; GFNISSE-NEXT:    paddb %xmm8, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm2, %xmm7
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    pshufb %xmm2, %xmm8
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    pxor %xmm7, %xmm7
+; GFNISSE-NEXT:    psubb %xmm2, %xmm7
 ; GFNISSE-NEXT:    pand %xmm7, %xmm2
-; GFNISSE-NEXT:    paddb %xmm8, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm4, %xmm7
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm4
-; GFNISSE-NEXT:    pcmpeqb %xmm4, %xmm5
-; GFNISSE-NEXT:    pand %xmm7, %xmm5
-; GFNISSE-NEXT:    pshufb %xmm4, %xmm3
-; GFNISSE-NEXT:    paddb %xmm5, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
+; GFNISSE-NEXT:    psubb %xmm3, %xmm5
+; GFNISSE-NEXT:    pand %xmm5, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: testv64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm0, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm4, %xmm0
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm4
+; GFNIAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX1-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; GFNIAVX1-NEXT:    vpsubb %xmm0, %xmm5, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm1, %xmm5
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
-; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm4, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vandps %ymm0, %ymm4, %ymm0
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [12307476859704049664,12307476859704049664,12307476859704049664,12307476859704049664]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $8, %ymm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm3
+; GFNIAVX1-NEXT:    vpsubb %xmm4, %xmm5, %xmm4
+; GFNIAVX1-NEXT:    vpsubb %xmm1, %xmm5, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vandps %ymm1, %ymm3, %ymm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $8, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: testv64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
-; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX2-NEXT:    vpsubb %ymm0, %ymm3, %ymm4
+; GFNIAVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [12307476859704049664,12307476859704049664,12307476859704049664,12307476859704049664]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $8, %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpsubb %ymm1, %ymm3, %ymm2
+; GFNIAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $8, %ymm4, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512VL-LABEL: testv64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; GFNIAVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512VL-NEXT:    # ymm2 = mem[0,1,0,1]
-; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
-; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2
+; GFNIAVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX512VL-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
+; GFNIAVX512VL-NEXT:    vpsubb %ymm0, %ymm3, %ymm0
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT:    vpandq %zmm0, %zmm2, %zmm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: testv64i8:
 ; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
-; GFNIAVX512BW-NEXT:    vptestnmb %zmm1, %zmm1, %k1
-; GFNIAVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; GFNIAVX512BW-NEXT:    vpshufb %zmm0, %zmm2, %zmm0 {%k1} {z}
-; GFNIAVX512BW-NEXT:    vpshufb %zmm1, %zmm2, %zmm1
-; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX512BW-NEXT:    vpsubb %zmm0, %zmm1, %zmm1
+; GFNIAVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
 ; GFNIAVX512BW-NEXT:    retq
   %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 0)
   ret <64 x i8> %out
@@ -374,130 +275,93 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
 define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
 ; GFNISSE-LABEL: testv64i8u:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
-; GFNISSE-NEXT:    movq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm7
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
 ; GFNISSE-NEXT:    pxor %xmm5, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    pshufb %xmm0, %xmm8
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm0
-; GFNISSE-NEXT:    pand %xmm7, %xmm0
-; GFNISSE-NEXT:    paddb %xmm8, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm7
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    pshufb %xmm1, %xmm8
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm1
+; GFNISSE-NEXT:    pxor %xmm6, %xmm6
+; GFNISSE-NEXT:    psubb %xmm0, %xmm6
+; GFNISSE-NEXT:    pand %xmm6, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [12307476859704049664,12307476859704049664]
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    pxor %xmm7, %xmm7
+; GFNISSE-NEXT:    psubb %xmm1, %xmm7
 ; GFNISSE-NEXT:    pand %xmm7, %xmm1
-; GFNISSE-NEXT:    paddb %xmm8, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm2, %xmm7
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    pshufb %xmm2, %xmm8
-; GFNISSE-NEXT:    pcmpeqb %xmm5, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    pxor %xmm7, %xmm7
+; GFNISSE-NEXT:    psubb %xmm2, %xmm7
 ; GFNISSE-NEXT:    pand %xmm7, %xmm2
-; GFNISSE-NEXT:    paddb %xmm8, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm7
-; GFNISSE-NEXT:    pshufb %xmm4, %xmm7
-; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm4
-; GFNISSE-NEXT:    pcmpeqb %xmm4, %xmm5
-; GFNISSE-NEXT:    pand %xmm7, %xmm5
-; GFNISSE-NEXT:    pshufb %xmm4, %xmm3
-; GFNISSE-NEXT:    paddb %xmm5, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
+; GFNISSE-NEXT:    psubb %xmm3, %xmm5
+; GFNISSE-NEXT:    pand %xmm5, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $8, %xmm6, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: testv64i8u:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vmovq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm0, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm4, %xmm0
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm4
+; GFNIAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; GFNIAVX1-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; GFNIAVX1-NEXT:    vpsubb %xmm0, %xmm5, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpcmpeqb %xmm6, %xmm1, %xmm5
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
-; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm4, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vandps %ymm0, %ymm4, %ymm0
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [12307476859704049664,12307476859704049664,12307476859704049664,12307476859704049664]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $8, %ymm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm3
+; GFNIAVX1-NEXT:    vpsubb %xmm4, %xmm5, %xmm4
+; GFNIAVX1-NEXT:    vpsubb %xmm1, %xmm5, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vandps %ymm1, %ymm3, %ymm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $8, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: testv64i8u:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX2-NEXT:    # ymm2 = mem[0,1,0,1]
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm6
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; GFNIAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
-; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
-; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; GFNIAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX2-NEXT:    vpsubb %ymm0, %ymm3, %ymm4
+; GFNIAVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [12307476859704049664,12307476859704049664,12307476859704049664,12307476859704049664]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $8, %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpsubb %ymm1, %ymm3, %ymm2
+; GFNIAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $8, %ymm4, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512VL-LABEL: testv64i8u:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; GFNIAVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512VL-NEXT:    # ymm2 = mem[0,1,0,1]
-; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16,0,0,0,0,128,64,32,16]
-; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
-; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm3, %ymm0
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2
+; GFNIAVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; GFNIAVX512VL-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
+; GFNIAVX512VL-NEXT:    vpsubb %ymm0, %ymm3, %ymm0
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT:    vpandq %zmm0, %zmm2, %zmm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: testv64i8u:
 ; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
-; GFNIAVX512BW-NEXT:    vptestnmb %zmm1, %zmm1, %k1
-; GFNIAVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; GFNIAVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; GFNIAVX512BW-NEXT:    vpshufb %zmm0, %zmm2, %zmm0 {%k1} {z}
-; GFNIAVX512BW-NEXT:    vpshufb %zmm1, %zmm2, %zmm1
-; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; GFNIAVX512BW-NEXT:    vpsubb %zmm0, %zmm1, %zmm1
+; GFNIAVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
 ; GFNIAVX512BW-NEXT:    retq
   %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 -1)
   ret <64 x i8> %out
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/gfni-shifts.ll b/external/llvm-project/llvm/test/CodeGen/X86/gfni-shifts.ll
index 5cd1a2c76762..cd16651123b0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/gfni-shifts.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/gfni-shifts.ll
@@ -1261,7 +1261,7 @@ define <32 x i8> @constant_shl_v32i8(<32 x i8> %a) nounwind {
 ; GFNIAVX512VL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0]
 ; GFNIAVX512VL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
 ; GFNIAVX512VL-NEXT:    vpsllw $8, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 | (ymm1 & mem)
+; GFNIAVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 | (ymm1 & m32bcst)
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: constant_shl_v32i8:
@@ -2634,7 +2634,7 @@ define <64 x i8> @constant_shl_v64i8(<64 x i8> %a) nounwind {
 ; GFNIAVX512VL-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpsllw $8, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
+; GFNIAVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & m32bcst)
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: constant_shl_v64i8:
@@ -2642,7 +2642,7 @@ define <64 x i8> @constant_shl_v64i8(<64 x i8> %a) nounwind {
 ; GFNIAVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0]
 ; GFNIAVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
 ; GFNIAVX512BW-NEXT:    vpsllw $8, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
+; GFNIAVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & m32bcst)
 ; GFNIAVX512BW-NEXT:    retq
   %shift = shl <64 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <64 x i8> %shift
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/half-constrained.ll b/external/llvm-project/llvm/test/CodeGen/X86/half-constrained.ll
index f1874cc03000..d5f2060ca20e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/half-constrained.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/half-constrained.ll
@@ -194,7 +194,7 @@ define void @float_to_half(float %0) strictfp {
 ; X64-F16C-LABEL: float_to_half:
 ; X64-F16C:       # %bb.0:
 ; X64-F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64-F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X64-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; X64-F16C-NEXT:    movq a@GOTPCREL(%rip), %rax
 ; X64-F16C-NEXT:    vpextrw $0, %xmm0, (%rax)
@@ -350,7 +350,7 @@ define void @add() strictfp {
 ; X86-F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; X86-F16C-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; X86-F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86-F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X86-F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X86-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; X86-F16C-NEXT:    vpextrw $0, %xmm0, c
 ; X86-F16C-NEXT:    retl
@@ -387,7 +387,7 @@ define void @add() strictfp {
 ; X64-F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; X64-F16C-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; X64-F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64-F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X64-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; X64-F16C-NEXT:    movq c@GOTPCREL(%rip), %rax
 ; X64-F16C-NEXT:    vpextrw $0, %xmm0, (%rax)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/half-darwin.ll b/external/llvm-project/llvm/test/CodeGen/X86/half-darwin.ll
index 3cbf5c11235e..8765f7dbe6d3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/half-darwin.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/half-darwin.ll
@@ -105,7 +105,7 @@ define void @strict_truncsfhf(float %in, ptr %ptr) nounwind strictfp {
 ; CHECK-F16C-LABEL: strict_truncsfhf:
 ; CHECK-F16C:       ## %bb.0:
 ; CHECK-F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-F16C-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; CHECK-F16C-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/half.ll b/external/llvm-project/llvm/test/CodeGen/X86/half.ll
index d8686b8b2950..b6a4a12eb0fa 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/half.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/half.ll
@@ -262,7 +262,7 @@ define void @test_sitofp_i64(i64 %a, ptr %p) #0 {
 ;
 ; BWON-F16C-LABEL: test_sitofp_i64:
 ; BWON-F16C:       # %bb.0:
-; BWON-F16C-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; BWON-F16C-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; BWON-F16C-NEXT:    retq
@@ -385,14 +385,14 @@ define void @test_uitofp_i64(i64 %a, ptr %p) #0 {
 ; BWON-F16C-NEXT:    testq %rdi, %rdi
 ; BWON-F16C-NEXT:    js .LBB10_1
 ; BWON-F16C-NEXT:  # %bb.2:
-; BWON-F16C-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; BWON-F16C-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; BWON-F16C-NEXT:    jmp .LBB10_3
 ; BWON-F16C-NEXT:  .LBB10_1:
 ; BWON-F16C-NEXT:    movq %rdi, %rax
 ; BWON-F16C-NEXT:    shrq %rax
 ; BWON-F16C-NEXT:    andl $1, %edi
 ; BWON-F16C-NEXT:    orq %rax, %rdi
-; BWON-F16C-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; BWON-F16C-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; BWON-F16C-NEXT:    vaddss %xmm0, %xmm0, %xmm0
 ; BWON-F16C-NEXT:  .LBB10_3:
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
@@ -843,7 +843,7 @@ define float @test_sitofp_fadd_i32(i32 %a, ptr %b) #0 {
 ; BWON-F16C-LABEL: test_sitofp_fadd_i32:
 ; BWON-F16C:       # %bb.0:
 ; BWON-F16C-NEXT:    vpinsrw $0, (%rsi), %xmm0, %xmm0
-; BWON-F16C-NEXT:    vcvtsi2ss %edi, %xmm1, %xmm1
+; BWON-F16C-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm1
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm1, %xmm1
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/horizontal-sum.ll b/external/llvm-project/llvm/test/CodeGen/X86/horizontal-sum.ll
index 443275e11459..0afc4f784bc5 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/horizontal-sum.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -577,7 +577,7 @@ define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <
 ; AVX-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
 ; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]
 ; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]
-; AVX-SLOW-NEXT:    vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX-SLOW-NEXT:    vmovsd {{.*#+}} xmm2 = xmm5[0],xmm2[1]
 ; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
 ; AVX-SLOW-NEXT:    vaddps %xmm3, %xmm4, %xmm4
 ; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
@@ -596,7 +596,7 @@ define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <
 ; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
 ; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]
 ; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]
-; AVX-FAST-NEXT:    vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX-FAST-NEXT:    vmovsd {{.*#+}} xmm2 = xmm5[0],xmm2[1]
 ; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm4
 ; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
 ; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/ifunc-asm.ll b/external/llvm-project/llvm/test/CodeGen/X86/ifunc-asm.ll
index a4c47da7f4c6..bc8e7e3d7d05 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/ifunc-asm.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/ifunc-asm.ll
@@ -15,7 +15,7 @@ entry:
 @foo_ifunc = ifunc i32 (i32), ptr @foo_resolver
 ; ELF:             .globl foo_ifunc
 ; ELF-NEXT:        .type foo_ifunc,@gnu_indirect_function
-; ELF-NEXT:        .set foo_ifunc, foo_resolver
+; ELF-NEXT:        foo_ifunc = foo_resolver
 
 ; MACHO:           .section __DATA,__data
 ; MACHO-NEXT:      .p2align 3, 0x0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/insertelement-zero.ll b/external/llvm-project/llvm/test/CodeGen/X86/insertelement-zero.ll
index 952940d14180..b66ad07c466e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/insertelement-zero.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/insertelement-zero.ll
@@ -30,13 +30,13 @@ define <2 x double> @insert_v2f64_z1(<2 x double> %a) {
 ; SSE41-LABEL: insert_v2f64_z1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: insert_v2f64_z1:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
   %1 = insertelement <2 x double> %a, double 0.0, i32 0
   ret <2 x double> %1
@@ -68,7 +68,7 @@ define <4 x double> @insert_v4f64_0zz3(<4 x double> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: insert_v4f64_0zz3:
@@ -103,7 +103,7 @@ define <2 x i64> @insert_v2i64_z1(<2 x i64> %a) {
 ; SSE41-LABEL: insert_v2i64_z1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: insert_v2i64_z1:
@@ -137,7 +137,7 @@ define <4 x i64> @insert_v4i64_01z3(<4 x i64> %a) {
 ; SSE41-LABEL: insert_v4i64_01z3:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: insert_v4i64_01z3:
@@ -214,7 +214,7 @@ define <8 x float> @insert_v8f32_z12345z7(<8 x float> %a) {
 ; SSE41-LABEL: insert_v8f32_z12345z7:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
 ; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3]
 ; SSE41-NEXT:    retq
 ;
@@ -287,7 +287,7 @@ define <8 x i32> @insert_v8i32_z12345z7(<8 x i32> %a) {
 ; SSE41-LABEL: insert_v8i32_z12345z7:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
 ; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3]
 ; SSE41-NEXT:    retq
 ;
@@ -508,8 +508,8 @@ define <8 x float> @PR41512_v8f32(float %x, float %y) {
 ; AVX-LABEL: PR41512_v8f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
   %ins1 = insertelement <8 x float> zeroinitializer, float %x, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-call.ll b/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-call.ll
new file mode 100644
index 000000000000..1f3a458ecf3a
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-call.ll
@@ -0,0 +1,10 @@
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: cannot lower calls with arbitrary operand bundles!
+
+declare void @g()
+
+define void @f(i32 %arg) {
+  call void @g() [ "foo"(i32 %arg) ]
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-callbr.ll b/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-callbr.ll
new file mode 100644
index 000000000000..56157d205b1c
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-callbr.ll
@@ -0,0 +1,11 @@
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: cannot lower callbrs with arbitrary operand bundles!
+
+define void @f(i32 %arg) {
+  callbr void asm "", ""() [ "foo"(i32 %arg) ]
+    to label %cont []
+
+cont:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-invoke.ll b/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-invoke.ll
new file mode 100644
index 000000000000..8091a220a44c
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/invalid-operand-bundle-invoke.ll
@@ -0,0 +1,19 @@
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: cannot lower invokes with arbitrary operand bundles!
+
+declare void @g()
+declare i32 @__gxx_personality_v0(...)
+
+define void @f(i32 %arg) personality ptr @__gxx_personality_v0 {
+  invoke void @g() [ "foo"(i32 %arg) ]
+    to label %cont unwind label %lpad
+
+lpad:
+  %l = landingpad {ptr, i32}
+    cleanup
+  resume {ptr, i32} %l
+
+cont:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/isel-fabs-x87.ll b/external/llvm-project/llvm/test/CodeGen/X86/isel-fabs-x87.ll
new file mode 100644
index 000000000000..8b846499946c
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/isel-fabs-x87.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X64
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=X64
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=X86
+
+define x86_fp80 @test_x86_fp80_abs(x86_fp80 %arg) {
+; X64-LABEL: test_x86_fp80_abs:
+; X64:       # %bb.0:
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fabs
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_fp80_abs:
+; X86:       # %bb.0:
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fabs
+; X86-NEXT:    retl
+  %abs = tail call x86_fp80 @llvm.fabs.f80(x86_fp80 %arg)
+  ret x86_fp80 %abs
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/isel-fabs.ll b/external/llvm-project/llvm/test/CodeGen/X86/isel-fabs.ll
new file mode 100644
index 000000000000..10bd5799280a
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/isel-fabs.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- -mattr=-x87 | FileCheck %s --check-prefixes=X64
+; RUN: llc < %s -mtriple=x86_64-- -mattr=-x87 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=X64
+; RUN: llc < %s -mtriple=i686-- -mattr=-x87 | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=i686-- -mattr=-x87 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=FASTISEL-X86
+
+
+define float @test_float_abs(float %arg) {
+; X64-LABEL: test_float_abs:
+; X64:       # %bb.0:
+; X64-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_float_abs:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; FASTISEL-X86-LABEL: test_float_abs:
+; FASTISEL-X86:       # %bb.0:
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; FASTISEL-X86-NEXT:    andl $2147483647, %eax # imm = 0x7FFFFFFF
+; FASTISEL-X86-NEXT:    retl
+    %abs = tail call float @llvm.fabs.f32(float %arg)
+    ret float %abs
+}
+
+define double @test_double_abs(double %arg) {
+; X64-LABEL: test_double_abs:
+; X64:       # %bb.0:
+; X64-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_double_abs:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $2147483647, %edx # imm = 0x7FFFFFFF
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    retl
+;
+; FASTISEL-X86-LABEL: test_double_abs:
+; FASTISEL-X86:       # %bb.0:
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; FASTISEL-X86-NEXT:    andl $2147483647, %edx # imm = 0x7FFFFFFF
+; FASTISEL-X86-NEXT:    retl
+    %abs = tail call double @llvm.fabs.f64(double %arg)
+    ret double %abs
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/isel-fp-to-sint-x87.ll b/external/llvm-project/llvm/test/CodeGen/X86/isel-fp-to-sint-x87.ll
index b0400dff1d3d..49e33141701f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/isel-fp-to-sint-x87.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/isel-fp-to-sint-x87.ll
@@ -2,71 +2,133 @@
 ; NOTE: Fast Isel is not added because it does not support x87 stores.
 
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse,-sse2 -global-isel=0 | FileCheck %s --check-prefixes X64,SDAG-X64
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse,-sse2 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes X64,GISEL-X64
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse,-sse2 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes X64,GISEL-X64
 ; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse,-sse2 -global-isel=0 | FileCheck %s --check-prefixes X86,SDAG-X86
+; TODO: The last RUN line fails GISEL for f64/double cases and will fallback to DAG due to lack of support for
+; loads/stores in X86 mode, support is expected soon enough, for this reason the isel-fp64-to-sint-x86.mir test is added.
 ; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse,-sse2 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes X86,GISEL-X86
 
 define i8 @test_float_to_int8(float %input) nounwind {
-; X64-LABEL: test_float_to_int8:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    flds {{[0-9]+}}(%rsp)
-; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    orl $3072, %eax # imm = 0xC00
-; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    retq
+; SDAG-X64-LABEL: test_float_to_int8:
+; SDAG-X64:       # %bb.0: # %entry
+; SDAG-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; SDAG-X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; SDAG-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SDAG-X64-NEXT:    retq
 ;
-; X86-LABEL: test_float_to_int8:
-; X86:       # %bb.0: # %entry
-; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    flds {{[0-9]+}}(%esp)
-; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    orl $3072, %eax # imm = 0xC00
-; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
-; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-NEXT:    fistps {{[0-9]+}}(%esp)
-; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    addl $8, %esp
-; X86-NEXT:    retl
+; GISEL-X64-LABEL: test_float_to_int8:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    flds {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; GISEL-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT:    # kill: def $al killed $al killed $ax
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: test_float_to_int8:
+; SDAG-X86:       # %bb.0: # %entry
+; SDAG-X86-NEXT:    subl $8, %esp
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; SDAG-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    addl $8, %esp
+; SDAG-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_float_to_int8:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $8, %esp
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; GISEL-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    # kill: def $al killed $al killed $ax
+; GISEL-X86-NEXT:    addl $8, %esp
+; GISEL-X86-NEXT:    retl
 entry:
     %conv = fptosi float %input to i8
     ret i8 %conv
 }
 
 define i8 @test_longdouble_to_int8(x86_fp80 %input) nounwind {
-; X64-LABEL: test_longdouble_to_int8:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    orl $3072, %eax # imm = 0xC00
-; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    retq
+; SDAG-X64-LABEL: test_longdouble_to_int8:
+; SDAG-X64:       # %bb.0: # %entry
+; SDAG-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; SDAG-X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; SDAG-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SDAG-X64-NEXT:    retq
 ;
-; X86-LABEL: test_longdouble_to_int8:
-; X86:       # %bb.0: # %entry
-; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    orl $3072, %eax # imm = 0xC00
-; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
-; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-NEXT:    fistps {{[0-9]+}}(%esp)
-; X86-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    addl $8, %esp
-; X86-NEXT:    retl
+; GISEL-X64-LABEL: test_longdouble_to_int8:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; GISEL-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT:    # kill: def $al killed $al killed $ax
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: test_longdouble_to_int8:
+; SDAG-X86:       # %bb.0: # %entry
+; SDAG-X86-NEXT:    subl $8, %esp
+; SDAG-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; SDAG-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    addl $8, %esp
+; SDAG-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_longdouble_to_int8:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    subl $8, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fnstcw {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    orl $3072, %eax # imm = 0xC00
+; GISEL-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fistps {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldcw {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    # kill: def $al killed $al killed $ax
+; GISEL-X86-NEXT:    addl $8, %esp
+; GISEL-X86-NEXT:    retl
 entry:
     %conv = fptosi x86_fp80 %input to i8
     ret i8 %conv
@@ -281,18 +343,32 @@ entry:
 }
 
 define i8 @test_double_to_int8(double %input) nounwind {
-; X64-LABEL: test_double_to_int8:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    fldl {{[0-9]+}}(%rsp)
-; X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    orl $3072, %eax # imm = 0xC00
-; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:    retq
+; SDAG-X64-LABEL: test_double_to_int8:
+; SDAG-X64:       # %bb.0: # %entry
+; SDAG-X64-NEXT:    fldl {{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; SDAG-X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; SDAG-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SDAG-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: test_double_to_int8:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    fldl {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT:    orl $3072, %eax # imm = 0xC00
+; GISEL-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fistps -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT:    # kill: def $al killed $al killed $ax
+; GISEL-X64-NEXT:    retq
 ;
 ; X86-LABEL: test_double_to_int8:
 ; X86:       # %bb.0: # %entry
@@ -416,8 +492,4 @@ entry:
     %conv = fptosi double %input to i64
     ret i64 %conv
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GISEL-X86: {{.*}}
-; GISEL-X64: {{.*}}
-; SDAG-X86: {{.*}}
-; SDAG-X64: {{.*}}
+
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/isel-int-to-fp.ll b/external/llvm-project/llvm/test/CodeGen/X86/isel-int-to-fp.ll
index fc99ff95788f..5884944e4198 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/isel-int-to-fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/isel-int-to-fp.ll
@@ -33,7 +33,7 @@ define double @test_ui64_to_double(i64 %x) {
 ;
 ; AVX512-LABEL: test_ui64_to_double:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = uitofp i64 %x to double
@@ -49,7 +49,7 @@ define double @test_ui32_to_double(i32 %x) {
 ;
 ; AVX512-LABEL: test_ui32_to_double:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = uitofp i32 %x to double
@@ -64,12 +64,12 @@ define double @test_ui16_to_double(i16 zeroext %x) {
 ;
 ; SDAG-AVX512-LABEL: test_ui16_to_double:
 ; SDAG-AVX512:       # %bb.0: # %entry
-; SDAG-AVX512-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; SDAG-AVX512-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; SDAG-AVX512-NEXT:    retq
 ;
 ; GISEL-AVX512-LABEL: test_ui16_to_double:
 ; GISEL-AVX512:       # %bb.0: # %entry
-; GISEL-AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; GISEL-AVX512-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; GISEL-AVX512-NEXT:    retq
 entry:
   %conv = uitofp i16 %x to double
@@ -84,12 +84,12 @@ define double @test_ui8_to_double(i8 zeroext %x) {
 ;
 ; SDAG-AVX512-LABEL: test_ui8_to_double:
 ; SDAG-AVX512:       # %bb.0: # %entry
-; SDAG-AVX512-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; SDAG-AVX512-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; SDAG-AVX512-NEXT:    retq
 ;
 ; GISEL-AVX512-LABEL: test_ui8_to_double:
 ; GISEL-AVX512:       # %bb.0: # %entry
-; GISEL-AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; GISEL-AVX512-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; GISEL-AVX512-NEXT:    retq
 entry:
   %conv = uitofp i8 %x to double
@@ -135,7 +135,7 @@ define float @test_ui64_to_float(i64 %x) {
 ;
 ; AVX512-LABEL: test_ui64_to_float:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = uitofp i64 %x to float
@@ -151,7 +151,7 @@ define float @test_ui32_to_float(i32 %x) {
 ;
 ; AVX512-LABEL: test_ui32_to_float:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = uitofp i32 %x to float
@@ -166,12 +166,12 @@ define float @test_ui16_to_float(i16 zeroext %x) {
 ;
 ; SDAG-AVX512-LABEL: test_ui16_to_float:
 ; SDAG-AVX512:       # %bb.0: # %entry
-; SDAG-AVX512-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; SDAG-AVX512-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; SDAG-AVX512-NEXT:    retq
 ;
 ; GISEL-AVX512-LABEL: test_ui16_to_float:
 ; GISEL-AVX512:       # %bb.0: # %entry
-; GISEL-AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; GISEL-AVX512-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; GISEL-AVX512-NEXT:    retq
 entry:
   %conv = uitofp i16 %x to float
@@ -186,12 +186,12 @@ define float @test_ui8_to_float(i8 zeroext %x) {
 ;
 ; SDAG-AVX512-LABEL: test_ui8_to_float:
 ; SDAG-AVX512:       # %bb.0: # %entry
-; SDAG-AVX512-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; SDAG-AVX512-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; SDAG-AVX512-NEXT:    retq
 ;
 ; GISEL-AVX512-LABEL: test_ui8_to_float:
 ; GISEL-AVX512:       # %bb.0: # %entry
-; GISEL-AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; GISEL-AVX512-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; GISEL-AVX512-NEXT:    retq
 entry:
   %conv = uitofp i8 %x to float
@@ -206,7 +206,7 @@ define double @test_si64_to_double(i64 %x) {
 ;
 ; AVX512-LABEL: test_si64_to_double:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i64 %x to double
@@ -221,7 +221,7 @@ define double @test_si32_to_double(i32 %x) {
 ;
 ; AVX512-LABEL: test_si32_to_double:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i32 %x to double
@@ -236,7 +236,7 @@ define double @test_si16_to_double(i16 signext %x) {
 ;
 ; AVX512-LABEL: test_si16_to_double:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i16 %x to double
@@ -251,7 +251,7 @@ define double @test_si8_to_double(i8 signext %x) {
 ;
 ; AVX512-LABEL: test_si8_to_double:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i8 %x to double
@@ -270,7 +270,7 @@ define double @test_si31_to_double(i31 %x) {
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    addl %edi, %edi
 ; AVX512-NEXT:    sarl %edi
-; AVX512-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i31 %x to double
@@ -289,7 +289,7 @@ define double @test_si33_to_double(i33 %x) {
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    shlq $31, %rdi
 ; AVX512-NEXT:    sarq $31, %rdi
-; AVX512-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i33 %x to double
@@ -304,7 +304,7 @@ define float @test_si64_to_float(i64 %x) {
 ;
 ; AVX512-LABEL: test_si64_to_float:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i64 %x to float
@@ -319,7 +319,7 @@ define float @test_si32_to_float(i32 %x) {
 ;
 ; AVX512-LABEL: test_si32_to_float:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i32 %x to float
@@ -334,7 +334,7 @@ define float @test_si16_to_float(i16 signext %x) {
 ;
 ; AVX512-LABEL: test_si16_to_float:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i16 %x to float
@@ -349,7 +349,7 @@ define float @test_si8_to_float(i8 signext %x) {
 ;
 ; AVX512-LABEL: test_si8_to_float:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i8 %x to float
@@ -368,7 +368,7 @@ define float @test_si31_to_float(i31 %x) {
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    addl %edi, %edi
 ; AVX512-NEXT:    sarl %edi
-; AVX512-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i31 %x to float
@@ -387,7 +387,7 @@ define float @test_si33_to_float(i33 %x) {
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    shlq $31, %rdi
 ; AVX512-NEXT:    sarq $31, %rdi
-; AVX512-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %conv = sitofp i33 %x to float
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/isel-sint-to-fp-x87.ll b/external/llvm-project/llvm/test/CodeGen/X86/isel-sint-to-fp-x87.ll
index 6e8461598d5e..98fa3c9541fa 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/isel-sint-to-fp-x87.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/isel-sint-to-fp-x87.ll
@@ -2,29 +2,53 @@
 ; NOTE: Fast Isel is not added because it does not support x87 stores.
 
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse,-sse2 -global-isel=0 | FileCheck %s --check-prefixes X64,SDAG-X64
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse,-sse2 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes X64,GISEL-X64
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse,-sse2 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes X64,GISEL-X64
 ; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse,-sse2 -global-isel=0 | FileCheck %s --check-prefixes X86,SDAG-X86
+; TODO: The last RUN line fails GISEL for f64/double cases and will fallback to DAG due to lack of support for
+; loads/stores in X86 mode, support is expected soon enough, for this reason the isel-sint-to-fp64-x86.mir test is added.
 ; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse,-sse2 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes X86,GISEL-X86
 
 define void @test_int8_to_float(i8 %x, ptr %p) nounwind {
-; X64-LABEL: test_int8_to_float:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    movsbl %dil, %eax
-; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fstps (%rsi)
-; X64-NEXT:    retq
+; SDAG-X64-LABEL: test_int8_to_float:
+; SDAG-X64:       # %bb.0: # %entry
+; SDAG-X64-NEXT:    movsbl %dil, %eax
+; SDAG-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fstps (%rsi)
+; SDAG-X64-NEXT:    retq
 ;
-; X86-LABEL: test_int8_to_float:
-; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
-; X86-NEXT:    filds {{[0-9]+}}(%esp)
-; X86-NEXT:    fstps (%ecx)
-; X86-NEXT:    popl %eax
-; X86-NEXT:    retl
+; GISEL-X64-LABEL: test_int8_to_float:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    shlw $8, %di
+; GISEL-X64-NEXT:    sarw $8, %di
+; GISEL-X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstps (%rsi)
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: test_int8_to_float:
+; SDAG-X86:       # %bb.0: # %entry
+; SDAG-X86-NEXT:    pushl %eax
+; SDAG-X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SDAG-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    filds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fstps (%ecx)
+; SDAG-X86-NEXT:    popl %eax
+; SDAG-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_int8_to_float:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    pushl %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    shlw $8, %ax
+; GISEL-X86-NEXT:    sarw $8, %ax
+; GISEL-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    filds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstps (%ecx)
+; GISEL-X86-NEXT:    popl %eax
+; GISEL-X86-NEXT:    retl
 entry:
   %conv = sitofp i8 %x to float
   store float %conv, ptr %p, align 4
@@ -39,16 +63,27 @@ define void @test_int16_to_float(i16 %x, ptr %p) nounwind {
 ; X64-NEXT:    fstps (%rsi)
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: test_int16_to_float:
-; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movw %cx, {{[0-9]+}}(%esp)
-; X86-NEXT:    filds {{[0-9]+}}(%esp)
-; X86-NEXT:    fstps (%eax)
-; X86-NEXT:    popl %eax
-; X86-NEXT:    retl
+; SDAG-X86-LABEL: test_int16_to_float:
+; SDAG-X86:       # %bb.0: # %entry
+; SDAG-X86-NEXT:    pushl %eax
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; SDAG-X86-NEXT:    movw %cx, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    filds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fstps (%eax)
+; SDAG-X86-NEXT:    popl %eax
+; SDAG-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_int16_to_float:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    pushl %eax
+; GISEL-X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    filds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstps (%ecx)
+; GISEL-X86-NEXT:    popl %eax
+; GISEL-X86-NEXT:    retl
 entry:
   %conv = sitofp i16 %x to float
   store float %conv, ptr %p, align 4
@@ -63,16 +98,27 @@ define void @test_int32_to_float(i32 %x, ptr %p) nounwind {
 ; X64-NEXT:    fstps (%rsi)
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: test_int32_to_float:
-; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ecx, (%esp)
-; X86-NEXT:    fildl (%esp)
-; X86-NEXT:    fstps (%eax)
-; X86-NEXT:    popl %eax
-; X86-NEXT:    retl
+; SDAG-X86-LABEL: test_int32_to_float:
+; SDAG-X86:       # %bb.0: # %entry
+; SDAG-X86-NEXT:    pushl %eax
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SDAG-X86-NEXT:    movl %ecx, (%esp)
+; SDAG-X86-NEXT:    fildl (%esp)
+; SDAG-X86-NEXT:    fstps (%eax)
+; SDAG-X86-NEXT:    popl %eax
+; SDAG-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_int32_to_float:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    pushl %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    fildl (%esp)
+; GISEL-X86-NEXT:    fstps (%ecx)
+; GISEL-X86-NEXT:    popl %eax
+; GISEL-X86-NEXT:    retl
 entry:
   %conv = sitofp i32 %x to float
   store float %conv, ptr %p, align 4
@@ -100,21 +146,40 @@ entry:
 }
 
 define x86_fp80 @test_int8to_longdouble(i8 %a) nounwind {
-; X64-LABEL: test_int8to_longdouble:
-; X64:       # %bb.0:
-; X64-NEXT:    movsbl %dil, %eax
-; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
-; X64-NEXT:    retq
+; SDAG-X64-LABEL: test_int8to_longdouble:
+; SDAG-X64:       # %bb.0:
+; SDAG-X64-NEXT:    movsbl %dil, %eax
+; SDAG-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    retq
 ;
-; X86-LABEL: test_int8to_longdouble:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
-; X86-NEXT:    filds {{[0-9]+}}(%esp)
-; X86-NEXT:    popl %eax
-; X86-NEXT:    retl
+; GISEL-X64-LABEL: test_int8to_longdouble:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    shlw $8, %di
+; GISEL-X64-NEXT:    sarw $8, %di
+; GISEL-X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: test_int8to_longdouble:
+; SDAG-X86:       # %bb.0:
+; SDAG-X86-NEXT:    pushl %eax
+; SDAG-X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    filds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    popl %eax
+; SDAG-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_int8to_longdouble:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    pushl %eax
+; GISEL-X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    shlw $8, %ax
+; GISEL-X86-NEXT:    sarw $8, %ax
+; GISEL-X86-NEXT:    movw %ax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    filds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    popl %eax
+; GISEL-X86-NEXT:    retl
   %conv = sitofp i8 %a to x86_fp80
   ret x86_fp80 %conv
 }
@@ -174,13 +239,22 @@ define x86_fp80 @test_int64_to_longdouble(i64 %a, ptr %p) nounwind {
 
 
 define void @test_int8to_double(i8 %x, ptr %p) nounwind {
-; X64-LABEL: test_int8to_double:
-; X64:       # %bb.0: # %entry
-; X64-NEXT:    movsbl %dil, %eax
-; X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fstpl (%rsi)
-; X64-NEXT:    retq
+; SDAG-X64-LABEL: test_int8to_double:
+; SDAG-X64:       # %bb.0: # %entry
+; SDAG-X64-NEXT:    movsbl %dil, %eax
+; SDAG-X64-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; SDAG-X64-NEXT:    fstpl (%rsi)
+; SDAG-X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: test_int8to_double:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    shlw $8, %di
+; GISEL-X64-NEXT:    sarw $8, %di
+; GISEL-X64-NEXT:    movw %di, -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    filds -{{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpl (%rsi)
+; GISEL-X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int8to_double:
 ; X86:       # %bb.0: # %entry
@@ -265,8 +339,3 @@ entry:
   store double %conv, ptr %p, align 4
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GISEL-X86: {{.*}}
-; GISEL-X64: {{.*}}
-; SDAG-X86: {{.*}}
-; SDAG-X64: {{.*}}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll b/external/llvm-project/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll
index b8f0661225f8..5199b1519ebe 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/lea-opt-memop-check-1.ll
@@ -47,9 +47,9 @@ entry:
   call fastcc void @"\01?fin$0@0@test2@@"(ptr %tmp0)
   ret void
 ; CHECK-LABEL: test2:
-; CHECK:	.set Ltest2$frame_escape_0, 8
-; CHECK:	.set Ltest2$frame_escape_1, 4
-; CHECK:	.set Ltest2$frame_escape_2, 0
+; CHECK:	Ltest2$frame_escape_0 = 8
+; CHECK:	Ltest2$frame_escape_1 = 4
+; CHECK:	Ltest2$frame_escape_2 = 0
 ; CHECK:	calll "?fin$0@0@test2@@"
 }
 
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/linux-preemption.ll b/external/llvm-project/llvm/test/CodeGen/X86/linux-preemption.ll
index 8e60b4787975..dc06a34e1c69 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/linux-preemption.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/linux-preemption.ll
@@ -285,18 +285,18 @@ define dso_local ptr @comdat_any_local() comdat {
 ; CHECK-NEXT: .Lstrong_local_global$local:
 
 ; COMMON:      .globl strong_default_alias
-; COMMON-NEXT: .set strong_default_alias, aliasee
+; COMMON-NEXT: strong_default_alias = aliasee
 ; COMMON-NEXT: .globl strong_hidden_alias
 ; COMMON-NEXT: .hidden strong_hidden_alias
-; COMMON-NEXT: .set strong_hidden_alias, aliasee
+; COMMON-NEXT: strong_hidden_alias = aliasee
 ; COMMON-NEXT: .weak weak_default_alias
-; COMMON-NEXT: .set weak_default_alias, aliasee
+; COMMON-NEXT: weak_default_alias = aliasee
 ; COMMON-NEXT: .globl strong_local_alias
-; COMMON-NEXT: .set strong_local_alias, aliasee
-; CHECK-NEXT:  .set .Lstrong_local_alias$local, aliasee
+; COMMON-NEXT: strong_local_alias = aliasee
+; CHECK-NEXT:  .Lstrong_local_alias$local = aliasee
 ; COMMON-NEXT: .weak weak_local_alias
-; COMMON-NEXT: .set weak_local_alias, aliasee
+; COMMON-NEXT: weak_local_alias = aliasee
 ; COMMON-NEXT: .globl strong_preemptable_alias
-; COMMON-NEXT: .set strong_preemptable_alias, aliasee
+; COMMON-NEXT: strong_preemptable_alias = aliasee
 ; COMMON-NEXT: .weak weak_preemptable_alias
-; COMMON-NEXT: .set weak_preemptable_alias, aliasee
+; COMMON-NEXT: weak_preemptable_alias = aliasee
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/llvm.sincos.ll b/external/llvm-project/llvm/test/CodeGen/X86/llvm.sincos.ll
index a429314630e5..5734729a2c50 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/llvm.sincos.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/llvm.sincos.ll
@@ -1,157 +1,262 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp --version 5
-; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-linux-gnu -fast-isel  | FileCheck %s --check-prefixes=X86,FASTISEL-X86
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -fast-isel  | FileCheck %s --check-prefixes=X64,FASTISEL-X64
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel=0 -fast-isel=0  | FileCheck %s --check-prefixes=X86,SDAG-X86
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel=0 -fast-isel=0  | FileCheck %s --check-prefixes=X64,SDAG-X64
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel=1 -global-isel-abort=2 | FileCheck %s --check-prefixes=X86,GISEL-X86
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel=1 -global-isel-abort=2 | FileCheck %s --check-prefixes=X64,GISEL-X64
 
-define void @test_sincos_v4f32(<4 x float> %x, ptr noalias %out_sin, ptr noalias %out_cos) {
-; CHECK-LABEL: test_sincos_v4f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    pushl %esi
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    subl $52, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset %esi, -12
-; CHECK-NEXT:    .cfi_offset %edi, -8
-; CHECK-NEXT:    movl 84(%esp), %esi
-; CHECK-NEXT:    flds 76(%esp)
-; CHECK-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT:    flds 64(%esp)
-; CHECK-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT:    flds 72(%esp)
-; CHECK-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT:    flds 68(%esp)
-; CHECK-NEXT:    movl 80(%esp), %edi
-; CHECK-NEXT:    leal 40(%esp), %eax
-; CHECK-NEXT:    movl %eax, 8(%esp)
-; CHECK-NEXT:    leal 4(%edi), %eax
-; CHECK-NEXT:    movl %eax, 4(%esp)
-; CHECK-NEXT:    fstps (%esp)
-; CHECK-NEXT:    calll sincosf
-; CHECK-NEXT:    leal 44(%esp), %eax
-; CHECK-NEXT:    movl %eax, 8(%esp)
-; CHECK-NEXT:    leal 8(%edi), %eax
-; CHECK-NEXT:    movl %eax, 4(%esp)
-; CHECK-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; CHECK-NEXT:    fstps (%esp)
-; CHECK-NEXT:    calll sincosf
-; CHECK-NEXT:    leal 36(%esp), %eax
-; CHECK-NEXT:    movl %eax, 8(%esp)
-; CHECK-NEXT:    movl %edi, 4(%esp)
-; CHECK-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; CHECK-NEXT:    fstps (%esp)
-; CHECK-NEXT:    calll sincosf
-; CHECK-NEXT:    leal 48(%esp), %eax
-; CHECK-NEXT:    movl %eax, 8(%esp)
-; CHECK-NEXT:    addl $12, %edi
-; CHECK-NEXT:    movl %edi, 4(%esp)
-; CHECK-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; CHECK-NEXT:    fstps (%esp)
-; CHECK-NEXT:    calll sincosf
-; CHECK-NEXT:    flds 36(%esp)
-; CHECK-NEXT:    flds 40(%esp)
-; CHECK-NEXT:    flds 44(%esp)
-; CHECK-NEXT:    flds 48(%esp)
-; CHECK-NEXT:    fstps 12(%esi)
-; CHECK-NEXT:    fstps 8(%esi)
-; CHECK-NEXT:    fstps 4(%esi)
-; CHECK-NEXT:    fstps (%esi)
-; CHECK-NEXT:    addl $52, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    popl %esi
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    popl %edi
-; CHECK-NEXT:    .cfi_def_cfa_offset 4
-; CHECK-NEXT:    retl
-  %result = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x)
-  %result.0 = extractvalue { <4 x float>, <4 x float> } %result, 0
-  %result.1 = extractvalue { <4 x float>, <4 x float> } %result, 1
-  store <4 x float> %result.0, ptr %out_sin, align 4
-  store <4 x float> %result.1, ptr %out_cos, align 4
-  ret void
+define { float, float } @test_sincos_f32(float %Val) nounwind {
+; X86-LABEL: test_sincos_f32:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $28, %esp
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fstps (%esp)
+; X86-NEXT:    calll sincosf
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_sincos_f32:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    movq %rsp, %rsi
+; X64-NEXT:    callq sincosf@PLT
+; X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+  %res = call { float, float } @llvm.sincos.f32(float %Val)
+  ret { float, float } %res
 }
 
-define void @test_sincos_v2f64(<2 x double> %x, ptr noalias %out_sin, ptr noalias %out_cos) {
-; CHECK-LABEL: test_sincos_v2f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    pushl %esi
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    subl $52, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset %esi, -12
-; CHECK-NEXT:    .cfi_offset %edi, -8
-; CHECK-NEXT:    movl 84(%esp), %esi
-; CHECK-NEXT:    fldl 72(%esp)
-; CHECK-NEXT:    fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
-; CHECK-NEXT:    fldl 64(%esp)
-; CHECK-NEXT:    movl 80(%esp), %edi
-; CHECK-NEXT:    leal 24(%esp), %eax
-; CHECK-NEXT:    movl %eax, 12(%esp)
-; CHECK-NEXT:    movl %edi, 8(%esp)
-; CHECK-NEXT:    fstpl (%esp)
-; CHECK-NEXT:    calll sincos
-; CHECK-NEXT:    leal 32(%esp), %eax
-; CHECK-NEXT:    movl %eax, 12(%esp)
-; CHECK-NEXT:    addl $8, %edi
-; CHECK-NEXT:    movl %edi, 8(%esp)
-; CHECK-NEXT:    fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
-; CHECK-NEXT:    fstpl (%esp)
-; CHECK-NEXT:    calll sincos
-; CHECK-NEXT:    fldl 24(%esp)
-; CHECK-NEXT:    fldl 32(%esp)
-; CHECK-NEXT:    fstpl 8(%esi)
-; CHECK-NEXT:    fstpl (%esi)
-; CHECK-NEXT:    addl $52, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    popl %esi
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    popl %edi
-; CHECK-NEXT:    .cfi_def_cfa_offset 4
-; CHECK-NEXT:    retl
-  %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %x)
-  %result.0 = extractvalue { <2 x double>, <2 x double> } %result, 0
-  %result.1 = extractvalue { <2 x double>, <2 x double> } %result, 1
-  store <2 x double> %result.0, ptr %out_sin, align 8
-  store <2 x double> %result.1, ptr %out_cos, align 8
-  ret void
+define { double, double } @test_sincos_f64(double %Val) nounwind  {
+; X86-LABEL: test_sincos_f64:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $44, %esp
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpl (%esp)
+; X86-NEXT:    calll sincos
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_sincos_f64:
+; X64:       # %bb.0:
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; X64-NEXT:    callq sincos@PLT
+; X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+  %res = call { double, double } @llvm.sincos.f64(double %Val)
+  ret { double, double } %res
+}
+
+define { x86_fp80, x86_fp80 } @test_sincos_f80(x86_fp80 %Val) nounwind {
+; X86-LABEL: test_sincos_f80:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $44, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    calll sincosl
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    addl $44, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_sincos_f80:
+; X64:       # %bb.0:
+; X64-NEXT:    subq $56, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; X64-NEXT:    callq sincosl@PLT
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    addq $56, %rsp
+; X64-NEXT:    retq
+  %res = call { x86_fp80, x86_fp80 } @llvm.sincos.f80(x86_fp80 %Val)
+  ret { x86_fp80, x86_fp80 } %res
 }
 
 declare void @foo(ptr, ptr)
 
-define void @can_fold_with_call_in_chain(float %x, ptr noalias %a, ptr noalias %b) {
-; CHECK-LABEL: can_fold_with_call_in_chain:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    pushl %esi
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    subl $20, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset %esi, -12
-; CHECK-NEXT:    .cfi_offset %edi, -8
-; CHECK-NEXT:    flds 32(%esp)
-; CHECK-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT:    movl 36(%esp), %edi
-; CHECK-NEXT:    movl 40(%esp), %esi
-; CHECK-NEXT:    movl %esi, 4(%esp)
-; CHECK-NEXT:    movl %edi, (%esp)
-; CHECK-NEXT:    calll foo@PLT
-; CHECK-NEXT:    leal 16(%esp), %eax
-; CHECK-NEXT:    movl %eax, 8(%esp)
-; CHECK-NEXT:    movl %edi, 4(%esp)
-; CHECK-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; CHECK-NEXT:    fstps (%esp)
-; CHECK-NEXT:    calll sincosf
-; CHECK-NEXT:    flds 16(%esp)
-; CHECK-NEXT:    fstps (%esi)
-; CHECK-NEXT:    addl $20, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    popl %esi
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    popl %edi
-; CHECK-NEXT:    .cfi_def_cfa_offset 4
-; CHECK-NEXT:    retl
+define void @can_fold_with_call_in_chain(float %x, ptr noalias %a, ptr noalias %b) nounwind {
+; FASTISEL-X86-LABEL: can_fold_with_call_in_chain:
+; FASTISEL-X86:       # %bb.0: # %entry
+; FASTISEL-X86-NEXT:    pushl %edi
+; FASTISEL-X86-NEXT:    pushl %esi
+; FASTISEL-X86-NEXT:    subl $20, %esp
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; FASTISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FASTISEL-X86-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; FASTISEL-X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; FASTISEL-X86-NEXT:    movl %edi, (%esp)
+; FASTISEL-X86-NEXT:    calll foo@PLT
+; FASTISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; FASTISEL-X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; FASTISEL-X86-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; FASTISEL-X86-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; FASTISEL-X86-NEXT:    fstps (%esp)
+; FASTISEL-X86-NEXT:    calll sincosf
+; FASTISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; FASTISEL-X86-NEXT:    fstps (%esi)
+; FASTISEL-X86-NEXT:    addl $20, %esp
+; FASTISEL-X86-NEXT:    popl %esi
+; FASTISEL-X86-NEXT:    popl %edi
+; FASTISEL-X86-NEXT:    retl
+;
+; FASTISEL-X64-LABEL: can_fold_with_call_in_chain:
+; FASTISEL-X64:       # %bb.0: # %entry
+; FASTISEL-X64-NEXT:    pushq %r14
+; FASTISEL-X64-NEXT:    pushq %rbx
+; FASTISEL-X64-NEXT:    pushq %rax
+; FASTISEL-X64-NEXT:    movq %rsi, %rbx
+; FASTISEL-X64-NEXT:    movq %rdi, %r14
+; FASTISEL-X64-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
+; FASTISEL-X64-NEXT:    callq sinf@PLT
+; FASTISEL-X64-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FASTISEL-X64-NEXT:    movss (%rsp), %xmm0 # 4-byte Reload
+; FASTISEL-X64-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; FASTISEL-X64-NEXT:    callq cosf@PLT
+; FASTISEL-X64-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
+; FASTISEL-X64-NEXT:    movq %r14, %rdi
+; FASTISEL-X64-NEXT:    movq %rbx, %rsi
+; FASTISEL-X64-NEXT:    callq foo@PLT
+; FASTISEL-X64-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; FASTISEL-X64-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; FASTISEL-X64-NEXT:    movss %xmm0, (%r14)
+; FASTISEL-X64-NEXT:    movss (%rsp), %xmm0 # 4-byte Reload
+; FASTISEL-X64-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; FASTISEL-X64-NEXT:    movss %xmm0, (%rbx)
+; FASTISEL-X64-NEXT:    addq $8, %rsp
+; FASTISEL-X64-NEXT:    popq %rbx
+; FASTISEL-X64-NEXT:    popq %r14
+; FASTISEL-X64-NEXT:    retq
+;
+; SDAG-X86-LABEL: can_fold_with_call_in_chain:
+; SDAG-X86:       # %bb.0: # %entry
+; SDAG-X86-NEXT:    pushl %edi
+; SDAG-X86-NEXT:    pushl %esi
+; SDAG-X86-NEXT:    subl $20, %esp
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; SDAG-X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; SDAG-X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    movl %edi, (%esp)
+; SDAG-X86-NEXT:    calll foo@PLT
+; SDAG-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SDAG-X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; SDAG-X86-NEXT:    fstps (%esp)
+; SDAG-X86-NEXT:    calll sincosf
+; SDAG-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; SDAG-X86-NEXT:    fstps (%esi)
+; SDAG-X86-NEXT:    addl $20, %esp
+; SDAG-X86-NEXT:    popl %esi
+; SDAG-X86-NEXT:    popl %edi
+; SDAG-X86-NEXT:    retl
+;
+; SDAG-X64-LABEL: can_fold_with_call_in_chain:
+; SDAG-X64:       # %bb.0: # %entry
+; SDAG-X64-NEXT:    pushq %r14
+; SDAG-X64-NEXT:    pushq %rbx
+; SDAG-X64-NEXT:    pushq %rax
+; SDAG-X64-NEXT:    movq %rsi, %rbx
+; SDAG-X64-NEXT:    movq %rdi, %r14
+; SDAG-X64-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
+; SDAG-X64-NEXT:    callq foo@PLT
+; SDAG-X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; SDAG-X64-NEXT:    movss (%rsp), %xmm0 # 4-byte Reload
+; SDAG-X64-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; SDAG-X64-NEXT:    movq %r14, %rdi
+; SDAG-X64-NEXT:    callq sincosf@PLT
+; SDAG-X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SDAG-X64-NEXT:    movss %xmm0, (%rbx)
+; SDAG-X64-NEXT:    addq $8, %rsp
+; SDAG-X64-NEXT:    popq %rbx
+; SDAG-X64-NEXT:    popq %r14
+; SDAG-X64-NEXT:    retq
+;
+; GISEL-X86-LABEL: can_fold_with_call_in_chain:
+; GISEL-X86:       # %bb.0: # %entry
+; GISEL-X86-NEXT:    pushl %ebx
+; GISEL-X86-NEXT:    pushl %edi
+; GISEL-X86-NEXT:    pushl %esi
+; GISEL-X86-NEXT:    subl $16, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; GISEL-X86-NEXT:    movl %ebx, (%esp)
+; GISEL-X86-NEXT:    calll sinf
+; GISEL-X86-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; GISEL-X86-NEXT:    movl %ebx, (%esp)
+; GISEL-X86-NEXT:    calll cosf
+; GISEL-X86-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; GISEL-X86-NEXT:    movl %esi, (%esp)
+; GISEL-X86-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    calll foo
+; GISEL-X86-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; GISEL-X86-NEXT:    fstps (%esi)
+; GISEL-X86-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; GISEL-X86-NEXT:    fstps (%edi)
+; GISEL-X86-NEXT:    addl $16, %esp
+; GISEL-X86-NEXT:    popl %esi
+; GISEL-X86-NEXT:    popl %edi
+; GISEL-X86-NEXT:    popl %ebx
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: can_fold_with_call_in_chain:
+; GISEL-X64:       # %bb.0: # %entry
+; GISEL-X64-NEXT:    pushq %r14
+; GISEL-X64-NEXT:    pushq %rbx
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
+; GISEL-X64-NEXT:    movq %rdi, %rbx
+; GISEL-X64-NEXT:    movq %rsi, %r14
+; GISEL-X64-NEXT:    callq sinf
+; GISEL-X64-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; GISEL-X64-NEXT:    movss (%rsp), %xmm0 # 4-byte Reload
+; GISEL-X64-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; GISEL-X64-NEXT:    callq cosf
+; GISEL-X64-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
+; GISEL-X64-NEXT:    movq %rbx, %rdi
+; GISEL-X64-NEXT:    movq %r14, %rsi
+; GISEL-X64-NEXT:    callq foo
+; GISEL-X64-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; GISEL-X64-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; GISEL-X64-NEXT:    movd %xmm0, %eax
+; GISEL-X64-NEXT:    movl %eax, (%rbx)
+; GISEL-X64-NEXT:    movd (%rsp), %xmm0 # 4-byte Folded Reload
+; GISEL-X64-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; GISEL-X64-NEXT:    movd %xmm0, %eax
+; GISEL-X64-NEXT:    movl %eax, (%r14)
+; GISEL-X64-NEXT:    addq $8, %rsp
+; GISEL-X64-NEXT:    popq %rbx
+; GISEL-X64-NEXT:    popq %r14
+; GISEL-X64-NEXT:    retq
 entry:
   %sin = tail call float @llvm.sin.f32(float %x)
   %cos = tail call float @llvm.cos.f32(float %x)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/llvm.sincos.vec.ll b/external/llvm-project/llvm/test/CodeGen/X86/llvm.sincos.vec.ll
new file mode 100644
index 000000000000..834dd788ff7f
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/llvm.sincos.vec.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp --version 5
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  | FileCheck %s
+
+define void @test_sincos_v4f32(<4 x float> %x, ptr noalias %out_sin, ptr noalias %out_cos) nounwind {
+; CHECK-LABEL: test_sincos_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $52, %esp
+; CHECK-NEXT:    movl 84(%esp), %esi
+; CHECK-NEXT:    flds 76(%esp)
+; CHECK-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    flds 64(%esp)
+; CHECK-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    flds 72(%esp)
+; CHECK-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT:    flds 68(%esp)
+; CHECK-NEXT:    movl 80(%esp), %edi
+; CHECK-NEXT:    leal 40(%esp), %eax
+; CHECK-NEXT:    movl %eax, 8(%esp)
+; CHECK-NEXT:    leal 4(%edi), %eax
+; CHECK-NEXT:    movl %eax, 4(%esp)
+; CHECK-NEXT:    fstps (%esp)
+; CHECK-NEXT:    calll sincosf
+; CHECK-NEXT:    leal 44(%esp), %eax
+; CHECK-NEXT:    movl %eax, 8(%esp)
+; CHECK-NEXT:    leal 8(%edi), %eax
+; CHECK-NEXT:    movl %eax, 4(%esp)
+; CHECK-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT:    fstps (%esp)
+; CHECK-NEXT:    calll sincosf
+; CHECK-NEXT:    leal 36(%esp), %eax
+; CHECK-NEXT:    movl %eax, 8(%esp)
+; CHECK-NEXT:    movl %edi, 4(%esp)
+; CHECK-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT:    fstps (%esp)
+; CHECK-NEXT:    calll sincosf
+; CHECK-NEXT:    leal 48(%esp), %eax
+; CHECK-NEXT:    movl %eax, 8(%esp)
+; CHECK-NEXT:    addl $12, %edi
+; CHECK-NEXT:    movl %edi, 4(%esp)
+; CHECK-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT:    fstps (%esp)
+; CHECK-NEXT:    calll sincosf
+; CHECK-NEXT:    flds 36(%esp)
+; CHECK-NEXT:    flds 40(%esp)
+; CHECK-NEXT:    flds 44(%esp)
+; CHECK-NEXT:    flds 48(%esp)
+; CHECK-NEXT:    fstps 12(%esi)
+; CHECK-NEXT:    fstps 8(%esi)
+; CHECK-NEXT:    fstps 4(%esi)
+; CHECK-NEXT:    fstps (%esi)
+; CHECK-NEXT:    addl $52, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+  %result = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x)
+  %result.0 = extractvalue { <4 x float>, <4 x float> } %result, 0
+  %result.1 = extractvalue { <4 x float>, <4 x float> } %result, 1
+  store <4 x float> %result.0, ptr %out_sin, align 4
+  store <4 x float> %result.1, ptr %out_cos, align 4
+  ret void
+}
+
+define void @test_sincos_v2f64(<2 x double> %x, ptr noalias %out_sin, ptr noalias %out_cos) nounwind {
+; CHECK-LABEL: test_sincos_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $52, %esp
+; CHECK-NEXT:    movl 84(%esp), %esi
+; CHECK-NEXT:    fldl 72(%esp)
+; CHECK-NEXT:    fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    fldl 64(%esp)
+; CHECK-NEXT:    movl 80(%esp), %edi
+; CHECK-NEXT:    leal 24(%esp), %eax
+; CHECK-NEXT:    movl %eax, 12(%esp)
+; CHECK-NEXT:    movl %edi, 8(%esp)
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll sincos
+; CHECK-NEXT:    leal 32(%esp), %eax
+; CHECK-NEXT:    movl %eax, 12(%esp)
+; CHECK-NEXT:    addl $8, %edi
+; CHECK-NEXT:    movl %edi, 8(%esp)
+; CHECK-NEXT:    fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT:    fstpl (%esp)
+; CHECK-NEXT:    calll sincos
+; CHECK-NEXT:    fldl 24(%esp)
+; CHECK-NEXT:    fldl 32(%esp)
+; CHECK-NEXT:    fstpl 8(%esi)
+; CHECK-NEXT:    fstpl (%esi)
+; CHECK-NEXT:    addl $52, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    retl
+  %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %x)
+  %result.0 = extractvalue { <2 x double>, <2 x double> } %result, 0
+  %result.1 = extractvalue { <2 x double>, <2 x double> } %result, 1
+  store <2 x double> %result.0, ptr %out_sin, align 8
+  store <2 x double> %result.1, ptr %out_cos, align 8
+  ret void
+}
+
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/localescape.ll b/external/llvm-project/llvm/test/CodeGen/X86/localescape.ll
index aee7613273f7..57369be489af 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/localescape.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/localescape.ll
@@ -76,8 +76,8 @@ define void @alloc_func(i32 %n) {
 ; X64: .seh_stackalloc 16
 ; X64: leaq    16(%rsp), %rbp
 ; X64: .seh_setframe %rbp, 16
-; X64: .set .Lalloc_func$frame_escape_0, -4
-; X64: .set .Lalloc_func$frame_escape_1, -12
+; X64: .Lalloc_func$frame_escape_0 = -4
+; X64: .Lalloc_func$frame_escape_1 = -12
 ; X64: movl $42, -4(%rbp)
 ; X64: movl $13, -12(%rbp)
 ; X64: movq 	%rbp, %rcx
@@ -88,8 +88,8 @@ define void @alloc_func(i32 %n) {
 ; X86: pushl   %ebp
 ; X86: movl    %esp, %ebp
 ; X86: subl    $12, %esp
-; X86: .set Lalloc_func$frame_escape_0, -4
-; X86: .set Lalloc_func$frame_escape_1, -12
+; X86: Lalloc_func$frame_escape_0 = -4
+; X86: Lalloc_func$frame_escape_1 = -12
 ; X86: movl    $42, -4(%ebp)
 ; X86: movl    $13, -12(%ebp)
 ; X86: pushl   %ebp
@@ -118,8 +118,8 @@ define void @alloc_func_no_frameaddr() {
 ; X64: subq    $40, %rsp
 ; X64: .seh_stackalloc 40
 ; X64: .seh_endprologue
-; X64: .set .Lalloc_func_no_frameaddr$frame_escape_0, 36
-; X64: .set .Lalloc_func_no_frameaddr$frame_escape_1, 32
+; X64: .Lalloc_func_no_frameaddr$frame_escape_0 = 36
+; X64: .Lalloc_func_no_frameaddr$frame_escape_1 = 32
 ; X64: movl $42, 36(%rsp)
 ; X64: movl $13, 32(%rsp)
 ; X64: xorl %ecx, %ecx
@@ -131,8 +131,8 @@ define void @alloc_func_no_frameaddr() {
 
 ; X86-LABEL: alloc_func_no_frameaddr:
 ; X86: subl    $8, %esp
-; X86: .set Lalloc_func_no_frameaddr$frame_escape_0, 4
-; X86: .set Lalloc_func_no_frameaddr$frame_escape_1, 0
+; X86: Lalloc_func_no_frameaddr$frame_escape_0 = 4
+; X86: Lalloc_func_no_frameaddr$frame_escape_1 = 0
 ; X86: movl $42, 4(%esp)
 ; X86: movl $13, (%esp)
 ; X86: pushl $0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/masked_expandload.ll b/external/llvm-project/llvm/test/CodeGen/X86/masked_expandload.ll
index 4c5b67962a58..e81a983c0701 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/masked_expandload.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/masked_expandload.ll
@@ -1097,7 +1097,7 @@ define <2 x float> @expandload_v2f32_v2i1(ptr %base, <2 x float> %src0, <2 x i32
 ; SSE42-NEXT:    retq
 ; SSE42-NEXT:  LBB4_1: ## %cond.load
 ; SSE42-NEXT:    movss (%rdi), %xmm1 ## xmm1 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE42-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE42-NEXT:    addq $4, %rdi
 ; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB4_4
@@ -1120,7 +1120,7 @@ define <2 x float> @expandload_v2f32_v2i1(ptr %base, <2 x float> %src0, <2 x i32
 ; AVX1OR2-NEXT:    retq
 ; AVX1OR2-NEXT:  LBB4_1: ## %cond.load
 ; AVX1OR2-NEXT:    vmovss (%rdi), %xmm1 ## xmm1 = mem[0],zero,zero,zero
-; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX1OR2-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX1OR2-NEXT:    addq $4, %rdi
 ; AVX1OR2-NEXT:    testb $2, %al
 ; AVX1OR2-NEXT:    je LBB4_4
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/masked_gather.ll b/external/llvm-project/llvm/test/CodeGen/X86/masked_gather.ll
index 559a7ec0930b..324a371632c4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/masked_gather.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/masked_gather.ll
@@ -65,7 +65,7 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x ptr> %ptr, <4 x i32> %trigger, <
 ; AVX1-NEXT:  # %bb.1: # %cond.load
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB0_2: # %else
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB0_4
@@ -105,7 +105,7 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x ptr> %ptr, <4 x i32> %trigger, <
 ; AVX2-NEXT:  # %bb.1: # %cond.load
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
 ; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB0_2: # %else
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB0_4
@@ -254,7 +254,7 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(ptr %base, <4 x i32> %idx, <4 x i32
 ; AVX1-NEXT:  # %bb.1: # %cond.load
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB1_2: # %else
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB1_4
@@ -299,7 +299,7 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(ptr %base, <4 x i32> %idx, <4 x i32
 ; AVX2-NEXT:  # %bb.1: # %cond.load
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
 ; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB1_2: # %else
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB1_4
@@ -451,7 +451,7 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(ptr %base, <4 x i64> %idx, <4 x i32
 ; AVX1-NEXT:  # %bb.1: # %cond.load
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB2_2: # %else
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB2_4
@@ -495,7 +495,7 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(ptr %base, <4 x i64> %idx, <4 x i32
 ; AVX2-NEXT:  # %bb.1: # %cond.load
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
 ; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; AVX2-NEXT:    vmovss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB2_2: # %else
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB2_4
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/masked_gather_scatter.ll b/external/llvm-project/llvm/test/CodeGen/X86/masked_gather_scatter.ll
index af018d83d520..4e6f666fa05d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/masked_gather_scatter.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/masked_gather_scatter.ll
@@ -3481,7 +3481,7 @@ define <2 x float> @large_index(ptr %base, <2 x i128> %ind, <2 x i1> %mask, <2 x
 ; X86-SKX-NEXT:  .LBB47_1: # %cond.load
 ; X86-SKX-NEXT:    vmovd %xmm0, %ecx
 ; X86-SKX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X86-SKX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; X86-SKX-NEXT:    vmovss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; X86-SKX-NEXT:    testb $2, %al
 ; X86-SKX-NEXT:    je .LBB47_4
 ; X86-SKX-NEXT:  .LBB47_3: # %cond.load1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/masked_load.ll b/external/llvm-project/llvm/test/CodeGen/X86/masked_load.ll
index 89459a2d1017..8c4bab99a5b7 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/masked_load.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/masked_load.ll
@@ -817,7 +817,7 @@ define <2 x float> @load_v2f32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x float>
 ; SSE42-NEXT:    retq
 ; SSE42-NEXT:  LBB7_1: ## %cond.load
 ; SSE42-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE42-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; SSE42-NEXT:    testb $2, %al
 ; SSE42-NEXT:    je LBB7_4
 ; SSE42-NEXT:  LBB7_3: ## %cond.load1
@@ -1220,7 +1220,7 @@ define <8 x float> @load_v8f32_v8i1_zero(<8 x i1> %mask, ptr %addr) {
 ; SSE42-NEXT:    je LBB10_10
 ; SSE42-NEXT:  LBB10_9: ## %cond.load10
 ; SSE42-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE42-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; SSE42-NEXT:    testb $32, %al
 ; SSE42-NEXT:    je LBB10_12
 ; SSE42-NEXT:  LBB10_11: ## %cond.load13
@@ -6424,7 +6424,7 @@ define <4 x i32> @mload_constmask_v4i32(ptr %addr, <4 x i32> %dst) {
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vmovaps {{.*#+}} xmm1 = [0,4294967295,4294967295,4294967295]
 ; AVX1-NEXT:    vmaskmovps (%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: mload_constmask_v4i32:
@@ -6512,7 +6512,7 @@ define <8 x float> @mload_constmask_v8f32(ptr %addr, <8 x float> %dst) {
 ; SSE42-LABEL: mload_constmask_v8f32:
 ; SSE42:       ## %bb.0:
 ; SSE42-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE42-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; SSE42-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; SSE42-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; SSE42-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/matrix-multiply.ll b/external/llvm-project/llvm/test/CodeGen/X86/matrix-multiply.ll
index 9a5bd1ce87c0..388d8528a2b8 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/matrix-multiply.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/matrix-multiply.ll
@@ -119,9 +119,9 @@ define <4 x double> @test_mul2x2_f64(<4 x double> %a0, <4 x double> %a1) nounwin
 ;
 ; AVX1-LABEL: test_mul2x2_f64:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vshufpd {{.*#+}} ymm2 = ymm1[1,1,3,3]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
-; AVX1-NEXT:    vmulpd %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm3 = ymm1[1,1,3,3]
+; AVX1-NEXT:    vmulpd %ymm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
 ; AVX1-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
index ac6b7e54ca5b..a798f4c38f68 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
@@ -2500,7 +2500,7 @@ define <16 x i8> @vec128_i8_signed_reg_reg(<16 x i8> %a1, <16 x i8> %a2) nounwin
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm2, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -2729,7 +2729,7 @@ define <16 x i8> @vec128_i8_unsigned_reg_reg(<16 x i8> %a1, <16 x i8> %a2) nounw
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpternlogq {{.*#+}} xmm2 = ~xmm2
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm2 ^ (xmm1 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm1 = xmm2 ^ (xmm1 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -2961,7 +2961,7 @@ define <16 x i8> @vec128_i8_signed_mem_reg(ptr %a1_addr, <16 x i8> %a2) nounwind
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm2, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm0 ^ (xmm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm0 ^ (xmm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm0, %xmm2, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -3192,7 +3192,7 @@ define <16 x i8> @vec128_i8_signed_reg_mem(<16 x i8> %a1, ptr %a2_addr) nounwind
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm2, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -3432,7 +3432,7 @@ define <16 x i8> @vec128_i8_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm2, %xmm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
index a8021e3164f3..3a4a638c7330 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
@@ -2016,7 +2016,7 @@ define <32 x i8> @vec256_i8_signed_reg_reg(<32 x i8> %a1, <32 x i8> %a2) nounwin
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm3, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm1 ^ (ymm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm1 ^ (ymm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -2194,7 +2194,7 @@ define <32 x i8> @vec256_i8_unsigned_reg_reg(<32 x i8> %a1, <32 x i8> %a2) nounw
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpternlogq {{.*#+}} ymm2 = ~ymm2
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm2 ^ (ymm1 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm2 ^ (ymm1 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -2372,7 +2372,7 @@ define <32 x i8> @vec256_i8_signed_mem_reg(ptr %a1_addr, <32 x i8> %a2) nounwind
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm3, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm0
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm0 ^ (ymm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm0 ^ (ymm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm0, %ymm2, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -2550,7 +2550,7 @@ define <32 x i8> @vec256_i8_signed_reg_mem(<32 x i8> %a1, ptr %a2_addr) nounwind
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm3, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm1 ^ (ymm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm1 ^ (ymm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
@@ -2733,7 +2733,7 @@ define <32 x i8> @vec256_i8_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm3, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
 ; AVX512VL-FALLBACK-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm1 ^ (ymm2 & mem)
+; AVX512VL-FALLBACK-NEXT:    vpternlogd {{.*#+}} ymm2 = ymm1 ^ (ymm2 & m32bcst)
 ; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/external/llvm-project/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 24c884211cf9..d752659f94a5 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -1993,21 +1993,21 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-v
 ; CHECK-SKX:       # %bb.0:
 ; CHECK-SKX-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; CHECK-SKX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-SKX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; CHECK-SKX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; CHECK-SKX-NEXT:    retq
 ;
 ; CHECK-AVX512-LABEL: splatconstant_rotate_v32i8:
 ; CHECK-AVX512:       # %bb.0:
 ; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; CHECK-AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; CHECK-AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; CHECK-AVX512-NEXT:    retq
 ;
 ; CHECK-VBMI1-LABEL: splatconstant_rotate_v32i8:
 ; CHECK-VBMI1:       # %bb.0:
 ; CHECK-VBMI1-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; CHECK-VBMI1-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-VBMI1-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; CHECK-VBMI1-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; CHECK-VBMI1-NEXT:    retq
 ;
 ; CHECK-GFNI-LABEL: splatconstant_rotate_v32i8:
@@ -2025,7 +2025,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le
 ; CHECK-SKX:       # %bb.0:
 ; CHECK-SKX-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; CHECK-SKX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-SKX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; CHECK-SKX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; CHECK-SKX-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
 ; CHECK-SKX-NEXT:    retq
 ;
@@ -2033,7 +2033,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le
 ; CHECK-AVX512:       # %bb.0:
 ; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; CHECK-AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; CHECK-AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; CHECK-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
 ; CHECK-AVX512-NEXT:    retq
 ;
@@ -2041,7 +2041,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le
 ; CHECK-VBMI1:       # %bb.0:
 ; CHECK-VBMI1-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; CHECK-VBMI1-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-VBMI1-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; CHECK-VBMI1-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; CHECK-VBMI1-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
 ; CHECK-VBMI1-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/mul-constant-i64.ll b/external/llvm-project/llvm/test/CodeGen/X86/mul-constant-i64.ll
index 03dd5351c78a..40d591f8d1be 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/mul-constant-i64.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/mul-constant-i64.ll
@@ -166,12 +166,11 @@ define i64 @test_mul_by_6(i64 %x) {
 define i64 @test_mul_by_7(i64 %x) {
 ; X86-LABEL: test_mul_by_7:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    leal (,%eax,8), %ecx
-; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl $7, %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    addl %ecx, %edx
+; X86-NEXT:    leal (%edx,%ecx,8), %edx
+; X86-NEXT:    subl %ecx, %edx
 ; X86-NEXT:    retl
 ;
 ; X86-NOOPT-LABEL: test_mul_by_7:
@@ -733,16 +732,17 @@ define i64 @test_mul_by_22(i64 %x) {
   ret i64 %mul
 }
 
-define i64 @test_mul_by_23(i64 %x) {
+define i64 @test_mul_by_23(i64 %x) nounwind {
 ; X86-LABEL: test_mul_by_23:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    leal (%eax,%eax,2), %ecx
-; X86-NEXT:    shll $3, %ecx
-; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal (%ecx,%ecx,2), %esi
 ; X86-NEXT:    movl $23, %eax
 ; X86-NEXT:    mull {{[0-9]+}}(%esp)
-; X86-NEXT:    addl %ecx, %edx
+; X86-NEXT:    leal (%edx,%esi,8), %edx
+; X86-NEXT:    subl %ecx, %edx
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X86-NOOPT-LABEL: test_mul_by_23:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/musttail-varargs.ll b/external/llvm-project/llvm/test/CodeGen/X86/musttail-varargs.ll
index f9b2ce79a35c..65cd1edd92e3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/musttail-varargs.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/musttail-varargs.ll
@@ -2,6 +2,7 @@
 ; RUN: llc -verify-machineinstrs < %s -enable-tail-merge=0 -mtriple=x86_64-linux | FileCheck %s --check-prefix=LINUX
 ; RUN: llc -verify-machineinstrs < %s -enable-tail-merge=0 -mtriple=x86_64-linux-gnux32 | FileCheck %s --check-prefix=LINUX-X32
 ; RUN: llc -verify-machineinstrs < %s -enable-tail-merge=0 -mtriple=x86_64-windows | FileCheck %s --check-prefix=WINDOWS
+; RUN: llc -verify-machineinstrs < %s -enable-tail-merge=0 -mtriple=x86_64-uefi | FileCheck %s --check-prefix=WINDOWS
 ; RUN: llc -verify-machineinstrs < %s -enable-tail-merge=0 -mtriple=i686-windows | FileCheck %s --check-prefix=X86 --check-prefix=X86-NOSSE
 ; RUN: llc -verify-machineinstrs < %s -enable-tail-merge=0 -mtriple=i686-windows -mattr=+sse2 | FileCheck %s --check-prefix=X86 --check-prefix=X86-SSE
 
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/oddshuffles.ll b/external/llvm-project/llvm/test/CodeGen/X86/oddshuffles.ll
index 8fd8e0e8120c..6b9a86343ea1 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/oddshuffles.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/oddshuffles.ll
@@ -1683,33 +1683,32 @@ define void @interleave_24i32_in(ptr %p, ptr %q1, ptr %q2, ptr %q3) nounwind {
 ;
 ; AVX1-LABEL: interleave_24i32_in:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovupd (%rcx), %ymm0
-; AVX1-NEXT:    vmovups (%rdx), %xmm1
-; AVX1-NEXT:    vmovups 16(%rdx), %xmm2
-; AVX1-NEXT:    vmovups (%rsi), %xmm3
-; AVX1-NEXT:    vmovups 16(%rsi), %xmm4
-; AVX1-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[3,3],xmm2[3,3]
-; AVX1-NEXT:    vunpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,1],xmm4[0,2]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3]
-; AVX1-NEXT:    vshufpd {{.*#+}} ymm4 = ymm4[0,0,3,3]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2,3],ymm4[4],ymm2[5,6],ymm4[7]
-; AVX1-NEXT:    vunpckhpd {{.*#+}} xmm4 = xmm3[1],xmm1[1]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[1,1],xmm4[0,2]
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm3[2,1]
+; AVX1-NEXT:    vmovups (%rdx), %xmm0
+; AVX1-NEXT:    vmovups 16(%rdx), %xmm1
+; AVX1-NEXT:    vmovups (%rsi), %xmm2
+; AVX1-NEXT:    vmovups 16(%rsi), %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[3,3],xmm1[3,3]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1,2,2]
+; AVX1-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
-; AVX1-NEXT:    vbroadcastsd (%rcx), %ymm3
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
-; AVX1-NEXT:    vpermilps {{.*#+}} ymm3 = mem[0,0,3,3,4,4,7,7]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm4 = mem[1,0,2,2]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
-; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1,2],ymm0[3],ymm3[4,5],ymm0[6],ymm3[7]
-; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX1-NEXT:    vmovups %ymm1, (%rdi)
-; AVX1-NEXT:    vmovups %ymm2, 64(%rdi)
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm3 = ymm3[0,0,3,3]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm0[1,1,2,2]
+; AVX1-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2],xmm3[3]
+; AVX1-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastsd (%rcx), %ymm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm2 = mem[0,0,3,3,4,4,7,7]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,0,2,2]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,1,2,2]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]
+; AVX1-NEXT:    vmovups %ymm2, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vmovups %ymm1, 64(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1804,30 +1803,29 @@ define void @interleave_24i32_in(ptr %p, ptr %q1, ptr %q2, ptr %q3) nounwind {
 ; XOP-NEXT:    vmovups (%rsi), %ymm0
 ; XOP-NEXT:    vmovups (%rdx), %ymm1
 ; XOP-NEXT:    vpermil2ps {{.*#+}} ymm0 = ymm0[u,3],ymm1[3],ymm0[u,4],ymm1[4],ymm0[u,5]
-; XOP-NEXT:    vmovups (%rcx), %ymm1
-; XOP-NEXT:    vmovups (%rdx), %xmm2
-; XOP-NEXT:    vmovups 16(%rdx), %xmm3
-; XOP-NEXT:    vmovups (%rsi), %xmm4
-; XOP-NEXT:    vmovups 16(%rsi), %xmm5
-; XOP-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[3,3],xmm3[3,3]
-; XOP-NEXT:    vunpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
-; XOP-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1],xmm5[0,2]
-; XOP-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
-; XOP-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm1[2,3,2,3]
-; XOP-NEXT:    vshufpd {{.*#+}} ymm5 = ymm5[0,0,3,3]
-; XOP-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2,3],ymm5[4],ymm3[5,6],ymm5[7]
-; XOP-NEXT:    vunpckhpd {{.*#+}} xmm5 = xmm4[1],xmm2[1]
-; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm2[1,1],xmm5[0,2]
-; XOP-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; XOP-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,0],xmm4[2,1]
+; XOP-NEXT:    vmovups (%rdx), %xmm1
+; XOP-NEXT:    vmovups 16(%rdx), %xmm2
+; XOP-NEXT:    vmovups (%rsi), %xmm3
+; XOP-NEXT:    vmovups 16(%rsi), %xmm4
+; XOP-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[3,3],xmm2[3,3]
+; XOP-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,1,2,2]
+; XOP-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2],xmm2[3]
 ; XOP-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
-; XOP-NEXT:    vbroadcastsd (%rcx), %ymm4
-; XOP-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7]
-; XOP-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1,1,2,2]
-; XOP-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7]
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; XOP-NEXT:    vshufpd {{.*#+}} ymm4 = ymm4[0,0,3,3]
+; XOP-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2,3],ymm4[4],ymm2[5,6],ymm4[7]
+; XOP-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[1,1,2,2]
+; XOP-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2],xmm4[3]
+; XOP-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; XOP-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,2]
+; XOP-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; XOP-NEXT:    vbroadcastsd (%rcx), %ymm3
+; XOP-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
+; XOP-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,1,2,2]
+; XOP-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0],ymm0[1,2],ymm3[3],ymm0[4,5],ymm3[6],ymm0[7]
 ; XOP-NEXT:    vmovups %ymm0, 32(%rdi)
-; XOP-NEXT:    vmovups %ymm2, (%rdi)
-; XOP-NEXT:    vmovups %ymm3, 64(%rdi)
+; XOP-NEXT:    vmovups %ymm1, (%rdi)
+; XOP-NEXT:    vmovups %ymm2, 64(%rdi)
 ; XOP-NEXT:    vzeroupper
 ; XOP-NEXT:    retq
   %s1 = load <8 x i32>, ptr %q1, align 4
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/opt-pipeline.ll b/external/llvm-project/llvm/test/CodeGen/X86/opt-pipeline.ll
index 540046e6a863..8d155bd57df1 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -121,7 +121,6 @@
 ; CHECK-NEXT:       Peephole Optimizations
 ; CHECK-NEXT:       Remove dead machine instructions
 ; CHECK-NEXT:       Live Range Shrink
-; CHECK-NEXT:       X86 Windows Fixup Buffer Security Check
 ; CHECK-NEXT:       X86 Fixup SetCC
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       X86 LEA Optimize
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pmul.ll b/external/llvm-project/llvm/test/CodeGen/X86/pmul.ll
index c7cc2acaf262..9aee2f11e9ea 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pmul.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pmul.ll
@@ -832,7 +832,7 @@ define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind  {
 ; AVX512F-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsllw $8, %ymm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & m32bcst)
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: mul_v64i8c:
@@ -840,7 +840,7 @@ define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind  {
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 # [117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0]
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]
 ; AVX512BW-NEXT:    vpsllw $8, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & m32bcst)
 ; AVX512BW-NEXT:    retq
 entry:
   %A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr132844.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr132844.ll
index ded100b2accc..dc9f006d93d1 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pr132844.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr132844.ll
@@ -4,12 +4,11 @@
 define  { ptr, i8 } @PR132844(<4 x ptr> %0, <4 x ptr> %1) {
 ; CHECK-LABEL: PR132844:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; CHECK-NEXT:    movb $10, %al
-; CHECK-NEXT:    kmovd %eax, %k1
-; CHECK-NEXT:    vinserti64x2 $1, 16, %ymm2, %ymm0 {%k1}
-; CHECK-NEXT:    vmovdqu %ymm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; CHECK-NEXT:    vinsertf128 $1, 16, %ymm2, %ymm2
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]
+; CHECK-NEXT:    vmovups %ymm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    vzeroupper
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr142513.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr142513.ll
new file mode 100644
index 000000000000..fe969104fcf5
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr142513.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64
+
+define i64 @foo(i64 %x) {
+; X86-LABEL: foo:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrl $15, %eax
+; X86-NEXT:    cmpl $65509, %eax # imm = 0xFFE5
+; X86-NEXT:    je .LBB0_1
+; X86-NEXT:  # %bb.2: # %if.end
+; X86-NEXT:    movl $3, %eax
+; X86-NEXT:    movl $2146598912, %edx # imm = 0x7FF28000
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB0_1: # %if.then
+;
+; X64-LABEL: foo:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    shrq $47, %rdi
+; X64-NEXT:    cmpl $65509, %edi # imm = 0xFFE5
+; X64-NEXT:    je .LBB0_1
+; X64-NEXT:  # %bb.2: # %if.end
+; X64-NEXT:    movabsq $9219572124669181955, %rax # imm = 0x7FF2800000000003
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB0_1: # %if.then
+entry:
+  %shr.mask = and i64 %x, -140737488355328
+  %cmp = icmp eq i64 %shr.mask, 9219572124669181952
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  unreachable
+
+if.end:
+  ret i64 9219572124669181955
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr142937.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr142937.ll
new file mode 100644
index 000000000000..8be99e102ff5
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr142937.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-- -O0   | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-- -O0 | FileCheck %s --check-prefix=X64
+
+define void @public_type_test() {
+; X86-LABEL: public_type_test:
+; X86:       # %bb.0: # %bb
+; X86-NEXT:    movb $1, %al
+; X86-NEXT:  # %bb.1: # %bb1
+; X86-NEXT:    retl
+;
+; X64-LABEL: public_type_test:
+; X64:       # %bb.0: # %bb
+; X64-NEXT:    movb $1, %al
+; X64-NEXT:  # %bb.1: # %bb1
+; X64-NEXT:    retq
+bb:
+  %call = call i1 @llvm.public.type.test(ptr null, metadata !"typeinfo")
+  br label %bb1
+
+bb1:
+  call void @llvm.assume(i1 %call)
+  ret void
+}
+
+define void @type_test() {
+; X86-LABEL: type_test:
+; X86:       # %bb.0: # %bb
+; X86-NEXT:    movb $1, %al
+; X86-NEXT:    testb $1, %al
+; X86-NEXT:    jne .LBB1_2
+; X86-NEXT:  # %bb.1: # %bb1
+; X86-NEXT:    ud1l 2(%eax), %eax
+; X86-NEXT:  .LBB1_2: # %bb2
+; X86-NEXT:    retl
+;
+; X64-LABEL: type_test:
+; X64:       # %bb.0: # %bb
+; X64-NEXT:    movb $1, %al
+; X64-NEXT:    testb $1, %al
+; X64-NEXT:    jne .LBB1_2
+; X64-NEXT:  # %bb.1: # %bb1
+; X64-NEXT:    ud1l 2(%eax), %eax
+; X64-NEXT:  .LBB1_2: # %bb2
+; X64-NEXT:    retq
+bb:
+  %call = tail call i1 @llvm.type.test(ptr null, metadata !"typeinfo")
+  br i1 %call, label %bb2, label %bb1
+
+bb1:
+  tail call void @llvm.ubsantrap(i8 2)
+  unreachable
+
+bb2:
+  ret void
+}
+
+declare i1 @llvm.public.type.test(ptr, metadata)
+
+declare void @llvm.assume(i1 noundef)
+
+declare i1 @llvm.type.test(ptr, metadata)
+
+declare void @llvm.ubsantrap(i8 immarg)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr22019.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr22019.ll
index 4e78bae20442..262ee5fad737 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pr22019.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr22019.ll
@@ -5,9 +5,9 @@ target triple = "x86_64-unknown-linux-gnu"
 module asm "pselect = __pselect"
 module asm "var = __var"
 module asm "alias = __alias"
-; CHECK: .set pselect, __pselect
-; CHECK: .set var, __var
-; CHECK: .set alias, __alias
+; CHECK: pselect = __pselect
+; CHECK: var = __var
+; CHECK: alias = __alias
 
 ; CHECK: pselect:
 ; CHECK: retq
@@ -19,5 +19,5 @@ define void @pselect() {
 ; CHECK: .long 0
 @var = global i32 0
 
-; CHECK: .set alias, var
+; CHECK: alias = var
 @alias = alias i32, ptr @var
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr34080.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr34080.ll
index 436b54db333b..d07d1aaf6fc0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pr34080.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr34080.ll
@@ -124,7 +124,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; AVX-NEXT:    fldt 16(%rbp)
 ; AVX-NEXT:    fld %st(0)
 ; AVX-NEXT:    fisttpl -4(%rbp)
-; AVX-NEXT:    vcvtsi2sdl -4(%rbp), %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sdl -4(%rbp), %xmm15, %xmm0
 ; AVX-NEXT:    vmovsd %xmm0, -48(%rbp)
 ; AVX-NEXT:    vmovsd %xmm0, -24(%rbp)
 ; AVX-NEXT:    fsubl -24(%rbp)
@@ -132,7 +132,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; AVX-NEXT:    fmul %st, %st(1)
 ; AVX-NEXT:    fld %st(1)
 ; AVX-NEXT:    fisttpl -8(%rbp)
-; AVX-NEXT:    vcvtsi2sdl -8(%rbp), %xmm1, %xmm0
+; AVX-NEXT:    vcvtsi2sdl -8(%rbp), %xmm15, %xmm0
 ; AVX-NEXT:    vmovsd %xmm0, -40(%rbp)
 ; AVX-NEXT:    vmovsd %xmm0, -16(%rbp)
 ; AVX-NEXT:    fxch %st(1)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr37879.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr37879.ll
index 60ca7c5b6d22..34cbccca2867 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pr37879.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr37879.ll
@@ -5,7 +5,7 @@ define double @foo(ptr nocapture readonly) #0 {
 ; CHECK-LABEL: foo:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    movq (%rax), %rax
-; CHECK-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm1
+; CHECK-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; CHECK-NEXT:    kmovd %eax, %k1
 ; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
 ; CHECK-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr38803.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr38803.ll
index ebac8121df91..3efe9f8dfa55 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pr38803.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr38803.ll
@@ -17,7 +17,7 @@ define dso_local float @_Z3fn2v() {
 ; CHECK-NEXT:    cmpl $0, c(%rip)
 ; CHECK-NEXT:    je .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %if.then
-; CHECK-NEXT:    vcvtsi2ssl b(%rip), %xmm1, %xmm1
+; CHECK-NEXT:    vcvtsi2ssl b(%rip), %xmm15, %xmm1
 ; CHECK-NEXT:    kmovd %eax, %k1
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr40090.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr40090.ll
index f0aaf09e359d..af933c950e11 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pr40090.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr40090.ll
@@ -4,10 +4,9 @@
 define i64 @foo(i64 %x, i64 %y) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    bsrq %rdi, %rax
-; CHECK-NEXT:    orq $64, %rax
+; CHECK-NEXT:    bsrq %rdi, %rcx
+; CHECK-NEXT:    orq $64, %rcx
 ; CHECK-NEXT:    bsrq %rsi, %rcx
-; CHECK-NEXT:    cmoveq %rax, %rcx
 ; CHECK-NEXT:    movl $63, %eax
 ; CHECK-NEXT:    subq %rcx, %rax
 ; CHECK-NEXT:    retq
@@ -21,4 +20,24 @@ define i64 @foo(i64 %x, i64 %y) {
   ret i64 %7
 }
 
+define i64 @bar(i64 %x, i64 %y) {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $127, %ecx
+; CHECK-NEXT:    bsrq %rdi, %rcx
+; CHECK-NEXT:    xorq $64, %rcx
+; CHECK-NEXT:    bsrq %rsi, %rcx
+; CHECK-NEXT:    movl $63, %eax
+; CHECK-NEXT:    subq %rcx, %rax
+; CHECK-NEXT:    retq
+  %1 = tail call i64 @llvm.ctlz.i64(i64 %x, i1 false)
+  %2 = xor i64 %1, 127
+  %3 = tail call i64 @llvm.ctlz.i64(i64 %y, i1 false)
+  %4 = xor i64 %3, 63
+  %5 = icmp eq i64 %y, 0
+  %6 = select i1 %5, i64 %2, i64 %4
+  %7 = sub nsw i64 63, %6
+  ret i64 %7
+}
+
 declare i64 @llvm.ctlz.i64(i64, i1)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr40730.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr40730.ll
index 164bf203d054..304d071e5d6e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/pr40730.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr40730.ll
@@ -5,7 +5,7 @@ define <8 x i32> @shuffle_v8i32_0dcd3f14(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: shuffle_v8i32_0dcd3f14:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3]
 ; CHECK-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,1,1,0]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
@@ -26,7 +26,7 @@ define <8 x i32> @shuffle_v8i32_0dcd3f14_constant(<8 x i32> %a0)  {
 ; CHECK-LABEL: shuffle_v8i32_0dcd3f14_constant:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm1 = xmm1[0],xmm0[1,2,3]
 ; CHECK-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,1,1,0]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5],ymm0[6,7]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/pr62145.ll b/external/llvm-project/llvm/test/CodeGen/X86/pr62145.ll
new file mode 100644
index 000000000000..38208422be6b
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/pr62145.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64
+
+define void @f(i64 %a, i64 %b) nounwind {
+; X86-LABEL: f:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $-65536, %edi # imm = 0xFFFF0000
+; X86-NEXT:    cmpl $65527, %eax # imm = 0xFFF7
+; X86-NEXT:    jne .LBB0_2
+; X86-NEXT:  # %bb.1: # %if.then
+; X86-NEXT:    calll ext1@PLT
+; X86-NEXT:  .LBB0_2: # %if.end
+; X86-NEXT:    calll ext2@PLT
+; X86-NEXT:    andl %edi, %esi
+; X86-NEXT:    cmpl $-589824, %esi # imm = 0xFFF70000
+; X86-NEXT:    jne .LBB0_3
+; X86-NEXT:  # %bb.4: # %if.then2
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    jmp ext1@PLT # TAILCALL
+; X86-NEXT:  .LBB0_3: # %if.end3
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+;
+; X64-LABEL: f:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %r14
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movq %rsi, %rbx
+; X64-NEXT:    movabsq $-281474976710656, %r14 # imm = 0xFFFF000000000000
+; X64-NEXT:    shrq $48, %rdi
+; X64-NEXT:    cmpl $65527, %edi # imm = 0xFFF7
+; X64-NEXT:    jne .LBB0_2
+; X64-NEXT:  # %bb.1: # %if.then
+; X64-NEXT:    callq ext1@PLT
+; X64-NEXT:  .LBB0_2: # %if.end
+; X64-NEXT:    callq ext2@PLT
+; X64-NEXT:    andq %r14, %rbx
+; X64-NEXT:    movabsq $-2533274790395904, %rax # imm = 0xFFF7000000000000
+; X64-NEXT:    addq $8, %rsp
+; X64-NEXT:    cmpq %rax, %rbx
+; X64-NEXT:    jne .LBB0_3
+; X64-NEXT:  # %bb.4: # %if.then2
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %r14
+; X64-NEXT:    jmp ext1@PLT # TAILCALL
+; X64-NEXT:  .LBB0_3: # %if.end3
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %r14
+; X64-NEXT:    retq
+entry:
+  %shr.mask.i = and i64 %a, -281474976710656
+  %cmp.i = icmp eq i64 %shr.mask.i, -2533274790395904
+  br i1 %cmp.i, label %if.then, label %if.end
+
+if.then:
+  tail call void @ext1()
+  br label %if.end
+
+if.end:
+  tail call void @ext2()
+  %shr.mask.i4 = and i64 %b, -281474976710656
+  %cmp.i5 = icmp eq i64 %shr.mask.i4, -2533274790395904
+  br i1 %cmp.i5, label %if.then2, label %if.end3
+
+if.then2:
+  tail call void @ext1()
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+declare void @ext1()
+declare void @ext2()
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/psubus.ll b/external/llvm-project/llvm/test/CodeGen/X86/psubus.ll
index 73ee28a7fd24..e10b360b35b5 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/psubus.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/psubus.ll
@@ -146,7 +146,7 @@ define <4 x i32> @ashr_xor_and_custom(<4 x i32> %x) nounwind {
 ; AVX512-LABEL: ashr_xor_and_custom:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 ^ mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 ^ m32bcst)
 ; AVX512-NEXT:    retq
   %signsplat = ashr <4 x i32> %x, <i32 undef, i32 31, i32 31, i32 31>
   %flipsign = xor <4 x i32> %x, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
@@ -187,7 +187,7 @@ define <4 x i32> @ashr_add_and_custom(<4 x i32> %x) nounwind {
 ; AVX512-LABEL: ashr_add_and_custom:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 ^ mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 ^ m32bcst)
 ; AVX512-NEXT:    retq
   %signsplat = ashr <4 x i32> %x, <i32 undef, i32 31, i32 31, i32 31>
   %flipsign = add <4 x i32> %x, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
@@ -230,7 +230,7 @@ define <4 x i32> @usubsat_custom(<4 x i32> %x) nounwind {
 ; AVX512-LABEL: usubsat_custom:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsrad $31, %xmm0, %xmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 ^ mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 ^ m32bcst)
 ; AVX512-NEXT:    retq
   %res = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> <i32 2147483648, i32 2147483648, i32 2147483648, i32 undef>)
   ret <4 x i32> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/rex-profile-test.ll b/external/llvm-project/llvm/test/CodeGen/X86/rex-profile-test.ll
new file mode 100644
index 000000000000..379d8faa4fc4
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/rex-profile-test.ll
@@ -0,0 +1,18 @@
+;; Test that the UEFI and Windows targets set the rex64 correctly.
+; RUN: llc -mtriple x86_64-uefi %s -o - | FileCheck %s -check-prefix=REX
+; RUN: llc -mtriple x86_64-windows-msvc %s -o - | FileCheck %s -check-prefix=REX
+; RUN: llc -mtriple x86_64-unknown-linux %s -o - | FileCheck %s -check-prefix=NOREX
+
+define void @test_tailjmp(ptr %fptr) {
+; REX-LABEL:    test_tailjmp:           # @test_tailjmp
+; REX:          # %bb.0:                # %entry
+; REX-NEXT:     rex64   jmpq    *%rcx   # TAILCALL
+;
+; NOREX-LABEL:  test_tailjmp:           # @test_tailjmp
+; NOREX:        .cfi_startproc
+; NOREX-NEXT:   # %bb.0:                # %entry
+; NOREX-NEXT:   jmpq	*%rdi           # TAILCALL
+entry:
+  tail call void %fptr()
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/rounding-ops.ll b/external/llvm-project/llvm/test/CodeGen/X86/rounding-ops.ll
index 04d6d1fdb141..948449c68b3f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/rounding-ops.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/rounding-ops.ll
@@ -221,12 +221,12 @@ define float @test11(ptr %xptr) nounwind optsize {
 ;
 ; CHECK-AVX-LABEL: test11:
 ; CHECK-AVX:       ## %bb.0:
-; CHECK-AVX-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vroundss $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX-NEXT:    retq
 ;
 ; CHECK-AVX512-LABEL: test11:
 ; CHECK-AVX512:       ## %bb.0:
-; CHECK-AVX512-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vroundss $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX512-NEXT:    retq
   %x = load float, ptr %xptr
   %call = tail call float @truncf(float %x) nounwind readnone
@@ -241,12 +241,12 @@ define double @test12(ptr %xptr) nounwind optsize {
 ;
 ; CHECK-AVX-LABEL: test12:
 ; CHECK-AVX:       ## %bb.0:
-; CHECK-AVX-NEXT:    vroundsd $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vroundsd $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX-NEXT:    retq
 ;
 ; CHECK-AVX512-LABEL: test12:
 ; CHECK-AVX512:       ## %bb.0:
-; CHECK-AVX512-NEXT:    vroundsd $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vroundsd $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX512-NEXT:    retq
   %x = load double, ptr %xptr
   %call = tail call double @trunc(double %x) nounwind readnone
@@ -261,12 +261,12 @@ define float @test11_pgso(ptr %xptr) nounwind !prof !14 {
 ;
 ; CHECK-AVX-LABEL: test11_pgso:
 ; CHECK-AVX:       ## %bb.0:
-; CHECK-AVX-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vroundss $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX-NEXT:    retq
 ;
 ; CHECK-AVX512-LABEL: test11_pgso:
 ; CHECK-AVX512:       ## %bb.0:
-; CHECK-AVX512-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vroundss $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX512-NEXT:    retq
   %x = load float, ptr %xptr
   %call = tail call float @truncf(float %x) nounwind readnone
@@ -281,12 +281,12 @@ define double @test12_pgso(ptr %xptr) nounwind !prof !14 {
 ;
 ; CHECK-AVX-LABEL: test12_pgso:
 ; CHECK-AVX:       ## %bb.0:
-; CHECK-AVX-NEXT:    vroundsd $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vroundsd $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX-NEXT:    retq
 ;
 ; CHECK-AVX512-LABEL: test12_pgso:
 ; CHECK-AVX512:       ## %bb.0:
-; CHECK-AVX512-NEXT:    vroundsd $11, (%rdi), %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vroundsd $11, (%rdi), %xmm15, %xmm0
 ; CHECK-AVX512-NEXT:    retq
   %x = load double, ptr %xptr
   %call = tail call double @trunc(double %x) nounwind readnone
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sadd_sat_vec.ll b/external/llvm-project/llvm/test/CodeGen/X86/sadd_sat_vec.ll
index bd563f97b0ac..80b55a364dba 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sadd_sat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sadd_sat_vec.ll
@@ -567,7 +567,7 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; AVX512BW-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    retq
   %z = call <16 x i4> @llvm.sadd.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/scalar-int-to-fp.ll b/external/llvm-project/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
index 0757d30296e2..43c1a84f7cd6 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
@@ -18,7 +18,7 @@ define float @u32_to_f(i32 %a) nounwind {
 ; AVX512_32-LABEL: u32_to_f:
 ; AVX512_32:       # %bb.0:
 ; AVX512_32-NEXT:    pushl %eax
-; AVX512_32-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512_32-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; AVX512_32-NEXT:    vmovss %xmm0, (%esp)
 ; AVX512_32-NEXT:    flds (%esp)
 ; AVX512_32-NEXT:    popl %eax
@@ -26,7 +26,7 @@ define float @u32_to_f(i32 %a) nounwind {
 ;
 ; AVX512_64-LABEL: u32_to_f:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; SSE2_32-LABEL: u32_to_f:
@@ -84,7 +84,7 @@ define float @s32_to_f(i32 %a) nounwind {
 ; AVX512_32-LABEL: s32_to_f:
 ; AVX512_32:       # %bb.0:
 ; AVX512_32-NEXT:    pushl %eax
-; AVX512_32-NEXT:    vcvtsi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX512_32-NEXT:    vcvtsi2ssl {{[0-9]+}}(%esp), %xmm7, %xmm0
 ; AVX512_32-NEXT:    vmovss %xmm0, (%esp)
 ; AVX512_32-NEXT:    flds (%esp)
 ; AVX512_32-NEXT:    popl %eax
@@ -92,7 +92,7 @@ define float @s32_to_f(i32 %a) nounwind {
 ;
 ; AVX512_64-LABEL: s32_to_f:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; SSE_32-LABEL: s32_to_f:
@@ -128,7 +128,7 @@ define double @u32_to_d(i32 %a) nounwind {
 ; AVX512_32-NEXT:    movl %esp, %ebp
 ; AVX512_32-NEXT:    andl $-8, %esp
 ; AVX512_32-NEXT:    subl $8, %esp
-; AVX512_32-NEXT:    vcvtusi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX512_32-NEXT:    vcvtusi2sdl 8(%ebp), %xmm7, %xmm0
 ; AVX512_32-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX512_32-NEXT:    fldl (%esp)
 ; AVX512_32-NEXT:    movl %ebp, %esp
@@ -137,7 +137,7 @@ define double @u32_to_d(i32 %a) nounwind {
 ;
 ; AVX512_64-LABEL: u32_to_d:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; SSE2_32-LABEL: u32_to_d:
@@ -199,7 +199,7 @@ define double @s32_to_d(i32 %a) nounwind {
 ; AVX512_32-NEXT:    movl %esp, %ebp
 ; AVX512_32-NEXT:    andl $-8, %esp
 ; AVX512_32-NEXT:    subl $8, %esp
-; AVX512_32-NEXT:    vcvtsi2sdl 8(%ebp), %xmm0, %xmm0
+; AVX512_32-NEXT:    vcvtsi2sdl 8(%ebp), %xmm7, %xmm0
 ; AVX512_32-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX512_32-NEXT:    fldl (%esp)
 ; AVX512_32-NEXT:    movl %ebp, %esp
@@ -208,7 +208,7 @@ define double @s32_to_d(i32 %a) nounwind {
 ;
 ; AVX512_64-LABEL: s32_to_d:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; SSE2_32-LABEL: s32_to_d:
@@ -308,7 +308,7 @@ define float @u64_to_f(i64 %a) nounwind {
 ;
 ; AVX512_64-LABEL: u64_to_f:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtusi2ss %rdi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; AVX512DQ_32-LABEL: u64_to_f:
@@ -437,7 +437,7 @@ define float @s64_to_f(i64 %a) nounwind {
 ;
 ; AVX512_64-LABEL: s64_to_f:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; AVX512DQ_32-LABEL: s64_to_f:
@@ -502,7 +502,7 @@ define float @s64_to_f_2(i64 %a) nounwind {
 ; AVX512_64-LABEL: s64_to_f_2:
 ; AVX512_64:       # %bb.0:
 ; AVX512_64-NEXT:    addq $5, %rdi
-; AVX512_64-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; AVX512DQ_32-LABEL: s64_to_f_2:
@@ -626,7 +626,7 @@ define double @u64_to_d(i64 %a) nounwind {
 ;
 ; AVX512_64-LABEL: u64_to_d:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; AVX512DQ_32-LABEL: u64_to_d:
@@ -748,7 +748,7 @@ define double @u64_to_d_optsize(i64 %a) nounwind optsize {
 ;
 ; AVX512_64-LABEL: u64_to_d_optsize:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; AVX512DQ_32-LABEL: u64_to_d_optsize:
@@ -869,7 +869,7 @@ define double @s64_to_d(i64 %a) nounwind {
 ;
 ; AVX512_64-LABEL: s64_to_d:
 ; AVX512_64:       # %bb.0:
-; AVX512_64-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; AVX512DQ_32-LABEL: s64_to_d:
@@ -955,7 +955,7 @@ define double @s64_to_d_2(i64 %a) nounwind {
 ; AVX512_64-LABEL: s64_to_d_2:
 ; AVX512_64:       # %bb.0:
 ; AVX512_64-NEXT:    addq $5, %rdi
-; AVX512_64-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX512_64-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX512_64-NEXT:    retq
 ;
 ; AVX512DQ_32-LABEL: s64_to_d_2:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/scalarize-fp.ll b/external/llvm-project/llvm/test/CodeGen/X86/scalarize-fp.ll
index ea9b69f8f5b8..ae24d3487c4b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/scalarize-fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/scalarize-fp.ll
@@ -911,7 +911,7 @@ define <4 x float> @merge_fcmp_cmpeqss_v4f32(<4 x float> %x, <4 x float> %y) {
 ; AVX1-LABEL: merge_fcmp_cmpeqss_v4f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vcmpeqss %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: merge_fcmp_cmpeqss_v4f32:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/seh-catch-all-win32.ll b/external/llvm-project/llvm/test/CodeGen/X86/seh-catch-all-win32.ll
index 3acf999fc423..bd51ca76c59d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/seh-catch-all-win32.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/seh-catch-all-win32.ll
@@ -58,7 +58,7 @@ entry:
 ; CHECK: pushl %edi
 ; CHECK: pushl %esi
 
-; CHECK: .set Lmain$frame_escape_0, [[code_offs:[-0-9]+]]
+; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]]
 ; CHECK: movl %esp, [[reg_offs:[-0-9]+]](%ebp)
 ; CHECK: movl $L__ehtable$main,
 ;       EH state 0
@@ -78,7 +78,7 @@ entry:
 ; CHECK: calll _printf
 
 ; CHECK: .section .xdata,"dr"
-; CHECK: .set Lmain$parent_frame_offset, [[reg_offs]]
+; CHECK: Lmain$parent_frame_offset = [[reg_offs]]
 ; CHECK: .p2align 2
 ; CHECK: L__ehtable$main
 ; CHECK-NEXT: .long -1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/seh-catchpad.ll b/external/llvm-project/llvm/test/CodeGen/X86/seh-catchpad.ll
index 7558c4389be5..d958580e5925 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/seh-catchpad.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/seh-catchpad.ll
@@ -119,7 +119,7 @@ __except.ret:                                     ; preds = %catch.dispatch.7
 ; CHECK:         jmp     .LBB1_[[epilogue]]
 
 ; CHECK:         .seh_handlerdata
-; CHECK-NEXT:         .set .Lmain$parent_frame_offset, 32
+; CHECK-NEXT:         .Lmain$parent_frame_offset = 32
 ; CHECK-NEXT:         .long   (.Llsda_end0-.Llsda_begin0)/16
 ; CHECK-NEXT: .Llsda_begin0:
 ; CHECK-NEXT:         .long   .Ltmp0@IMGREL
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/seh-finally.ll b/external/llvm-project/llvm/test/CodeGen/X86/seh-finally.ll
index 28e5cf68dd27..41823dfb38f0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/seh-finally.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/seh-finally.ll
@@ -26,7 +26,7 @@ lpad:                                             ; preds = %entry
 ; X64: retq
 
 ; X64: .seh_handlerdata
-; X64-NEXT: .set .Lmain$parent_frame_offset, 32
+; X64-NEXT: .Lmain$parent_frame_offset = 32
 ; X64-NEXT: .long   (.Llsda_end0-.Llsda_begin0)/16 # Number of call sites
 ; X64-NEXT: .Llsda_begin0:
 ; X64-NEXT: .long   .Ltmp0@IMGREL # LabelStart
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/seh-no-invokes.ll b/external/llvm-project/llvm/test/CodeGen/X86/seh-no-invokes.ll
index 99b81f0eb1bb..63e91d33d400 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/seh-no-invokes.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/seh-no-invokes.ll
@@ -15,7 +15,7 @@
 ; label. This was PR30431.
 
 ; CHECK-LABEL: _f:                                     # @f
-; CHECK: .set Lf$parent_frame_offset, 0
+; CHECK: Lf$parent_frame_offset = 0
 ; CHECK: retl
 
 ; CHECK-LABEL: "?filt$0@0@f@@":                        # @"\01?filt$0@0@f@@"
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/seh-stack-realign.ll b/external/llvm-project/llvm/test/CodeGen/X86/seh-stack-realign.ll
index 2869bff82231..ae687343cc50 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/seh-stack-realign.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/seh-stack-realign.ll
@@ -51,7 +51,7 @@ entry:
 ; Check that we can get the exception code from eax to the printf.
 
 ; CHECK-LABEL: _main:
-; CHECK: .set Lmain$frame_escape_0, [[code_offs:[-0-9]+]]
+; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]]
 ; CHECK: movl %esp, [[reg_offs:[-0-9]+]](%esi)
 ; CHECK: movl $L__ehtable$main,
 ;       EH state 0
@@ -71,7 +71,7 @@ entry:
 ; CHECK: calll _printf
 
 ; CHECK: .section .xdata,"dr"
-; CHECK: .set Lmain$parent_frame_offset, [[reg_offs]]
+; CHECK: Lmain$parent_frame_offset = [[reg_offs]]
 ; CHECK: L__ehtable$main
 ; CHECK-NEXT: .long -1
 ; CHECK-NEXT: .long _filt$main
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll b/external/llvm-project/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll
index 28b405799dfd..b64bfb38c7a5 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/select-narrow-int-to-fp.ll
@@ -17,13 +17,13 @@ define float @uint8ToFloat(i8 %int8) {
 ; CHECK-NO_FP16-LABEL: uint8ToFloat:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movzbl %dil, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: uint8ToFloat:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movzbl %dil, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = uitofp i8 %int8 to float
     ret float %fp32
@@ -62,14 +62,14 @@ define half @uint8ToHalf(i8 %int8) {
 ; CHECK-NO_FP16-LABEL: uint8ToHalf:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movzbl %dil, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: uint8ToHalf:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movzbl %dil, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = uitofp i8 %int8 to half
     ret half %fp32
@@ -111,13 +111,13 @@ define float @sint8ToFloat(i8 %int8) {
 ; CHECK-NO_FP16-LABEL: sint8ToFloat:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movsbl %dil, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: sint8ToFloat:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movsbl %dil, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = sitofp i8 %int8 to float
     ret float %fp32
@@ -143,14 +143,14 @@ define half @sint8ToHalf(i8 %int8) {
 ; CHECK-NO_FP16-LABEL: sint8ToHalf:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movsbl %dil, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: sint8ToHalf:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movsbl %dil, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = sitofp i8 %int8 to half
     ret half %fp32
@@ -184,13 +184,13 @@ define float @uint16ToFloat(i16 %int16) {
 ; CHECK-NO_FP16-LABEL: uint16ToFloat:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movzwl %di, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: uint16ToFloat:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movzwl %di, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = uitofp i16 %int16 to float
     ret float %fp32
@@ -216,14 +216,14 @@ define half @uint16ToHalf(i16 %int16) {
 ; CHECK-NO_FP16-LABEL: uint16ToHalf:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movzwl %di, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: uint16ToHalf:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movzwl %di, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = uitofp i16 %int16 to half
     ret half %fp32
@@ -249,13 +249,13 @@ define float @sint16ToFloat(i16 %int16) {
 ; CHECK-NO_FP16-LABEL: sint16ToFloat:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movswl %di, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: sint16ToFloat:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movswl %di, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = sitofp i16 %int16 to float
     ret float %fp32
@@ -281,14 +281,14 @@ define half @sint16ToHalf(i16 %int16) {
 ; CHECK-NO_FP16-LABEL: sint16ToHalf:
 ; CHECK-NO_FP16:       # %bb.0:
 ; CHECK-NO_FP16-NEXT:    movswl %di, %eax
-; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; CHECK-NO_FP16-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; CHECK-NO_FP16-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; CHECK-NO_FP16-NEXT:    retq
 ;
 ; CHECK-WITH_FP16-LABEL: sint16ToHalf:
 ; CHECK-WITH_FP16:       # %bb.0:
 ; CHECK-WITH_FP16-NEXT:    movswl %di, %eax
-; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm0, %xmm0
+; CHECK-WITH_FP16-NEXT:    vcvtsi2sh %eax, %xmm31, %xmm0
 ; CHECK-WITH_FP16-NEXT:    retq
     %fp32 = sitofp i16 %int16 to half
     ret half %fp32
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll b/external/llvm-project/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
index b8e9cbb14a15..66bd957b89c4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
@@ -33,7 +33,7 @@ define i32 @test1(i32 %v) {
 ; CHECK-NEXT:    popq %rcx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB0_4: # Block address taken
+; CHECK-NEXT:  .LBB0_4: # Inline asm indirect target
 ; CHECK-NEXT:    # %two
 ; CHECK-NEXT:    # Label of block must be emitted
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 84ae818d9183..05c855ed90b3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -1014,7 +1014,7 @@ define <16 x i8> @trunc_v4i64_to_v4i8_return_v16i8(<4 x i64> %vec) nounwind {
 ; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -1023,7 +1023,7 @@ define <16 x i8> @trunc_v4i64_to_v4i8_return_v16i8(<4 x i64> %vec) nounwind {
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1038,7 +1038,7 @@ define <16 x i8> @trunc_v4i64_to_v4i8_return_v16i8(<4 x i64> %vec) nounwind {
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll b/external/llvm-project/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
index 8a6c2f851a6d..c8e31f7088a4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
@@ -1,21 +1,35 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=X64
 
 define <4 x i64> @autogen_SD88863() {
-; CHECK-LABEL: autogen_SD88863:
-; CHECK:       # %bb.0: # %BB
-; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
-; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3]
-; CHECK-NEXT:    movb $1, %al
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  .LBB0_1: # %CF
-; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_1
-; CHECK-NEXT:  # %bb.2: # %CF240
-; CHECK-NEXT:    ret{{[l|q]}}
+; X86-LABEL: autogen_SD88863:
+; X86:       # %bb.0: # %BB
+; X86-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm7[0,1]
+; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3]
+; X86-NEXT:    movb $1, %al
+; X86-NEXT:    .p2align 4
+; X86-NEXT:  .LBB0_1: # %CF
+; X86-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NEXT:    testb %al, %al
+; X86-NEXT:    jne .LBB0_1
+; X86-NEXT:  # %bb.2: # %CF240
+; X86-NEXT:    retl
+;
+; X64-LABEL: autogen_SD88863:
+; X64:       # %bb.0: # %BB
+; X64-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm15[0,1]
+; X64-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3]
+; X64-NEXT:    movb $1, %al
+; X64-NEXT:    .p2align 4
+; X64-NEXT:  .LBB0_1: # %CF
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    testb %al, %al
+; X64-NEXT:    jne .LBB0_1
+; X64-NEXT:  # %bb.2: # %CF240
+; X64-NEXT:    retq
 BB:
   %I26 = insertelement <4 x i64> undef, i64 undef, i32 2
   br label %CF
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll b/external/llvm-project/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
index 08d9183bd30b..2d0778853fec 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll
@@ -2461,7 +2461,7 @@ define <32 x i1> @pr51133(<32 x i8> %x, <32 x i8> %y) {
 ; CHECK-AVX512VL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2 # [197,0,27,0,1,0,1,0,223,0,205,0,161,0,171,0,171,0,183,0,61,0,127,0,9,0,41,0,1,0,161,0]
 ; CHECK-AVX512VL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3 # [0,205,0,241,0,1,0,163,0,223,0,183,0,1,0,239,0,103,0,171,0,1,0,183,0,0,0,183,0,1,0,221]
 ; CHECK-AVX512VL-NEXT:    vpsllw $8, %ymm3, %ymm3
-; CHECK-AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 | (ymm2 & mem)
+; CHECK-AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 | (ymm2 & m32bcst)
 ; CHECK-AVX512VL-NEXT:    vpaddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm2
 ; CHECK-AVX512VL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; CHECK-AVX512VL-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [128,128,1,1,1,128,1,64,128,1,128,1,128,32,1,1]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse-cvttp2si.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse-cvttp2si.ll
index d08cf120bb4b..09b1d0f8b87d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse-cvttp2si.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse-cvttp2si.ll
@@ -23,7 +23,7 @@ define float @float_to_int_to_float_mem_f32_i32(ptr %p) #0 {
 ; AVX-LABEL: float_to_int_to_float_mem_f32_i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttss2si (%rdi), %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %x = load <4 x float>, ptr %p, align 16
   %fptosi = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> %x)
@@ -42,7 +42,7 @@ define float @float_to_int_to_float_reg_f32_i32(<4 x float> %x) #0 {
 ; AVX-LABEL: float_to_int_to_float_reg_f32_i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttss2si %xmm0, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm1, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %fptosi = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> %x)
   %sitofp = sitofp i32 %fptosi to float
@@ -59,7 +59,7 @@ define float @float_to_int_to_float_mem_f32_i64(ptr %p) #0 {
 ; AVX-LABEL: float_to_int_to_float_mem_f32_i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttss2si (%rdi), %rax
-; AVX-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %x = load <4 x float>, ptr %p, align 16
   %fptosi = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %x)
@@ -78,7 +78,7 @@ define float @float_to_int_to_float_reg_f32_i64(<4 x float> %x) #0 {
 ; AVX-LABEL: float_to_int_to_float_reg_f32_i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttss2si %xmm0, %rax
-; AVX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm0
+; AVX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %fptosi = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %x)
   %sitofp = sitofp i64 %fptosi to float
@@ -95,7 +95,7 @@ define double @float_to_int_to_float_mem_f64_i32(ptr %p) #0 {
 ; AVX-LABEL: float_to_int_to_float_mem_f64_i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttsd2si (%rdi), %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %x = load <2 x double>, ptr %p, align 16
   %fptosi = tail call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %x)
@@ -114,7 +114,7 @@ define double @float_to_int_to_float_reg_f64_i32(<2 x double> %x) #0 {
 ; AVX-LABEL: float_to_int_to_float_reg_f64_i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttsd2si %xmm0, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm1, %xmm0
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %fptosi = tail call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %x)
   %sitofp = sitofp i32 %fptosi to double
@@ -131,7 +131,7 @@ define double @float_to_int_to_float_mem_f64_i64(ptr %p) #0 {
 ; AVX-LABEL: float_to_int_to_float_mem_f64_i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttsd2si (%rdi), %rax
-; AVX-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %x = load <2 x double>, ptr %p, align 16
   %fptosi = tail call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %x)
@@ -150,7 +150,7 @@ define double @float_to_int_to_float_reg_f64_i64(<2 x double> %x) #0 {
 ; AVX-LABEL: float_to_int_to_float_reg_f64_i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttsd2si %xmm0, %rax
-; AVX-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm0
+; AVX-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX-NEXT:    retq
   %fptosi = tail call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %x)
   %sitofp = sitofp i64 %fptosi to double
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt-from-mem.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt-from-mem.ll
index f0af8fc29969..1c3cfd079e9e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt-from-mem.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt-from-mem.ll
@@ -7,22 +7,16 @@
 ; 0'th element insertion into an SSE register.
 
 define <4 x float> @insert_f32_firstelt(<4 x float> %x, ptr %s.addr) {
-; SSE2-LABEL: insert_f32_firstelt:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: insert_f32_firstelt:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: insert_f32_firstelt:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_f32_firstelt:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %s = load float, ptr %s.addr
   %i0 = insertelement <4 x float> %x, float %s, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt.ll
index 34de7e65465d..72e002ed6b7d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse-insertelt.ll
@@ -7,38 +7,28 @@
 ; 0'th element insertion into an SSE register.
 
 define <4 x float> @insert_f32_firstelt(<4 x float> %x, float %s) {
-; SSE2-LABEL: insert_f32_firstelt:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: insert_f32_firstelt:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: insert_f32_firstelt:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_f32_firstelt:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %i0 = insertelement <4 x float> %x, float %s, i32 0
   ret <4 x float> %i0
 }
 
 define <2 x double> @insert_f64_firstelt(<2 x double> %x, double %s) {
-; SSE2-LABEL: insert_f64_firstelt:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: insert_f64_firstelt:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: insert_f64_firstelt:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_f64_firstelt:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
   %i0 = insertelement <2 x double> %x, double %s, i32 0
   ret <2 x double> %i0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
index b1f9872f7b6e..2e2e78a6da51 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
@@ -190,7 +190,7 @@ define <4 x float> @test_mm_cmpge_ss(<4 x float> %a0, <4 x float> %a1) nounwind
 ; AVX-LABEL: test_mm_cmpge_ss:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmpless %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf2,0xc2,0xc8,0x02]
-; AVX-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; AVX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfa,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a1, <4 x float> %a0, i8 2)
@@ -232,7 +232,7 @@ define <4 x float> @test_mm_cmpgt_ss(<4 x float> %a0, <4 x float> %a1) nounwind
 ; AVX-LABEL: test_mm_cmpgt_ss:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmpltss %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf2,0xc2,0xc8,0x01]
-; AVX-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; AVX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfa,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a1, <4 x float> %a0, i8 1)
@@ -382,7 +382,7 @@ define <4 x float> @test_mm_cmpnge_ss(<4 x float> %a0, <4 x float> %a1) nounwind
 ; AVX-LABEL: test_mm_cmpnge_ss:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmpnless %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf2,0xc2,0xc8,0x06]
-; AVX-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; AVX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfa,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a1, <4 x float> %a0, i8 6)
@@ -424,7 +424,7 @@ define <4 x float> @test_mm_cmpngt_ss(<4 x float> %a0, <4 x float> %a1) nounwind
 ; AVX-LABEL: test_mm_cmpngt_ss:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmpnltss %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf2,0xc2,0xc8,0x05]
-; AVX-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; AVX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfa,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a1, <4 x float> %a0, i8 5)
@@ -1603,7 +1603,7 @@ define <4 x float> @test_mm_move_ss(<4 x float> %a0, <4 x float> %a1) {
 ;
 ; AVX-LABEL: test_mm_move_ss:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; AVX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfa,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
@@ -2219,7 +2219,7 @@ define <4 x float> @test_mm_set_ss(float %a0) nounwind {
 ; X86-AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX1-NEXT:    # encoding: [0xc5,0xfa,0x10,0x44,0x24,0x04]
 ; X86-AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf0,0x57,0xc9]
-; X86-AVX1-NEXT:    vblendps $1, %xmm0, %xmm1, %xmm0 # encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x01]
+; X86-AVX1-NEXT:    vmovss %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf2,0x10,0xc0]
 ; X86-AVX1-NEXT:    # xmm0 = xmm0[0],xmm1[1,2,3]
 ; X86-AVX1-NEXT:    retl # encoding: [0xc3]
 ;
@@ -2228,7 +2228,7 @@ define <4 x float> @test_mm_set_ss(float %a0) nounwind {
 ; X86-AVX512-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX512-NEXT:    # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x44,0x24,0x04]
 ; X86-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf0,0x57,0xc9]
-; X86-AVX512-NEXT:    vblendps $1, %xmm0, %xmm1, %xmm0 # encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x01]
+; X86-AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf2,0x10,0xc0]
 ; X86-AVX512-NEXT:    # xmm0 = xmm0[0],xmm1[1,2,3]
 ; X86-AVX512-NEXT:    retl # encoding: [0xc3]
 ;
@@ -2243,7 +2243,7 @@ define <4 x float> @test_mm_set_ss(float %a0) nounwind {
 ; X64-AVX-LABEL: test_mm_set_ss:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf0,0x57,0xc9]
-; X64-AVX-NEXT:    vblendps $1, %xmm0, %xmm1, %xmm0 # encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x01]
+; X64-AVX-NEXT:    vmovss %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf2,0x10,0xc0]
 ; X64-AVX-NEXT:    # xmm0 = xmm0[0],xmm1[1,2,3]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
   %res0  = insertelement <4 x float> undef, float %a0, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse-scalar-fp-arith.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse-scalar-fp-arith.ll
index 47d35f3636d4..325f735b09cd 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse-scalar-fp-arith.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse-scalar-fp-arith.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,X86-SSE,SSE2,X86-SSE2
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X86-SSE,SSE41,X86-SSE41
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,X86-SSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X86-SSE
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,X86-AVX,X86-AVX1
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,X86-AVX,X86-AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,X64-SSE,SSE2,X64-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X64-SSE,SSE41,X64-SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,X64-SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X64-SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,X64-AVX,X64-AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,X64-AVX,X64-AVX512
 
@@ -1150,22 +1150,16 @@ define <4 x float> @insert_test5_add_ss(<4 x float> %a, <4 x float> %b) {
 }
 
 define <4 x float> @insert_test5_sub_ss(<4 x float> %a, <4 x float> %b) {
-; SSE2-LABEL: insert_test5_sub_ss:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    subps %xmm0, %xmm1
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    ret{{[l|q]}}
-;
-; SSE41-LABEL: insert_test5_sub_ss:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    subps %xmm0, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    ret{{[l|q]}}
+; SSE-LABEL: insert_test5_sub_ss:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subps %xmm0, %xmm1
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: insert_test5_sub_ss:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vsubps %xmm0, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}}
   %1 = fsub <4 x float> %b, %a
   %2 = shufflevector <4 x float> %1, <4 x float> %a, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
@@ -1188,22 +1182,16 @@ define <4 x float> @insert_test5_mul_ss(<4 x float> %a, <4 x float> %b) {
 }
 
 define <4 x float> @insert_test5_div_ss(<4 x float> %a, <4 x float> %b) {
-; SSE2-LABEL: insert_test5_div_ss:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    divps %xmm0, %xmm1
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    ret{{[l|q]}}
-;
-; SSE41-LABEL: insert_test5_div_ss:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    divps %xmm0, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    ret{{[l|q]}}
+; SSE-LABEL: insert_test5_div_ss:
+; SSE:       # %bb.0:
+; SSE-NEXT:    divps %xmm0, %xmm1
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: insert_test5_div_ss:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vdivps %xmm0, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}}
   %1 = fdiv <4 x float> %b, %a
   %2 = shufflevector <4 x float> %1, <4 x float> %a, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
@@ -1226,22 +1214,16 @@ define <2 x double> @insert_test5_add_sd(<2 x double> %a, <2 x double> %b) {
 }
 
 define <2 x double> @insert_test5_sub_sd(<2 x double> %a, <2 x double> %b) {
-; SSE2-LABEL: insert_test5_sub_sd:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    subpd %xmm0, %xmm1
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    ret{{[l|q]}}
-;
-; SSE41-LABEL: insert_test5_sub_sd:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    subpd %xmm0, %xmm1
-; SSE41-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE41-NEXT:    ret{{[l|q]}}
+; SSE-LABEL: insert_test5_sub_sd:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subpd %xmm0, %xmm1
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: insert_test5_sub_sd:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vsubpd %xmm0, %xmm1, %xmm1
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    ret{{[l|q]}}
   %1 = fsub <2 x double> %b, %a
   %2 = shufflevector <2 x double> %1, <2 x double> %a, <2 x i32> <i32 0, i32 3>
@@ -1264,22 +1246,16 @@ define <2 x double> @insert_test5_mul_sd(<2 x double> %a, <2 x double> %b) {
 }
 
 define <2 x double> @insert_test5_div_sd(<2 x double> %a, <2 x double> %b) {
-; SSE2-LABEL: insert_test5_div_sd:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    divpd %xmm0, %xmm1
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    ret{{[l|q]}}
-;
-; SSE41-LABEL: insert_test5_div_sd:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    divpd %xmm0, %xmm1
-; SSE41-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE41-NEXT:    ret{{[l|q]}}
+; SSE-LABEL: insert_test5_div_sd:
+; SSE:       # %bb.0:
+; SSE-NEXT:    divpd %xmm0, %xmm1
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: insert_test5_div_sd:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vdivpd %xmm0, %xmm1, %xmm1
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    ret{{[l|q]}}
   %1 = fdiv <2 x double> %b, %a
   %2 = shufflevector <2 x double> %1, <2 x double> %a, <2 x i32> <i32 0, i32 3>
@@ -1287,29 +1263,17 @@ define <2 x double> @insert_test5_div_sd(<2 x double> %a, <2 x double> %b) {
 }
 
 define <4 x float> @add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
-; X86-SSE2-LABEL: add_ss_mask:
-; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    testb $1, {{[0-9]+}}(%esp)
-; X86-SSE2-NEXT:    jne .LBB70_1
-; X86-SSE2-NEXT:  # %bb.2:
-; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
-; X86-SSE2-NEXT:    retl
-; X86-SSE2-NEXT:  .LBB70_1:
-; X86-SSE2-NEXT:    addss %xmm0, %xmm1
-; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; X86-SSE2-NEXT:    retl
-;
-; X86-SSE41-LABEL: add_ss_mask:
-; X86-SSE41:       # %bb.0:
-; X86-SSE41-NEXT:    testb $1, {{[0-9]+}}(%esp)
-; X86-SSE41-NEXT:    jne .LBB70_1
-; X86-SSE41-NEXT:  # %bb.2:
-; X86-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
-; X86-SSE41-NEXT:    retl
-; X86-SSE41-NEXT:  .LBB70_1:
-; X86-SSE41-NEXT:    addss %xmm0, %xmm1
-; X86-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; X86-SSE41-NEXT:    retl
+; X86-SSE-LABEL: add_ss_mask:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    jne .LBB70_1
+; X86-SSE-NEXT:  # %bb.2:
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; X86-SSE-NEXT:    retl
+; X86-SSE-NEXT:  .LBB70_1:
+; X86-SSE-NEXT:    addss %xmm0, %xmm1
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; X86-SSE-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: add_ss_mask:
 ; X86-AVX1:       # %bb.0:
@@ -1318,7 +1282,7 @@ define <4 x float> @add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c,
 ; X86-AVX1-NEXT:  # %bb.1:
 ; X86-AVX1-NEXT:    vaddss %xmm1, %xmm0, %xmm2
 ; X86-AVX1-NEXT:  .LBB70_2:
-; X86-AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; X86-AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX512-LABEL: add_ss_mask:
@@ -1329,29 +1293,17 @@ define <4 x float> @add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c,
 ; X86-AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; X86-AVX512-NEXT:    retl
 ;
-; X64-SSE2-LABEL: add_ss_mask:
-; X64-SSE2:       # %bb.0:
-; X64-SSE2-NEXT:    testb $1, %dil
-; X64-SSE2-NEXT:    jne .LBB70_1
-; X64-SSE2-NEXT:  # %bb.2:
-; X64-SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
-; X64-SSE2-NEXT:    retq
-; X64-SSE2-NEXT:  .LBB70_1:
-; X64-SSE2-NEXT:    addss %xmm0, %xmm1
-; X64-SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; X64-SSE2-NEXT:    retq
-;
-; X64-SSE41-LABEL: add_ss_mask:
-; X64-SSE41:       # %bb.0:
-; X64-SSE41-NEXT:    testb $1, %dil
-; X64-SSE41-NEXT:    jne .LBB70_1
-; X64-SSE41-NEXT:  # %bb.2:
-; X64-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
-; X64-SSE41-NEXT:    retq
-; X64-SSE41-NEXT:  .LBB70_1:
-; X64-SSE41-NEXT:    addss %xmm0, %xmm1
-; X64-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; X64-SSE41-NEXT:    retq
+; X64-SSE-LABEL: add_ss_mask:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    testb $1, %dil
+; X64-SSE-NEXT:    jne .LBB70_1
+; X64-SSE-NEXT:  # %bb.2:
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; X64-SSE-NEXT:    retq
+; X64-SSE-NEXT:  .LBB70_1:
+; X64-SSE-NEXT:    addss %xmm0, %xmm1
+; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX1-LABEL: add_ss_mask:
 ; X64-AVX1:       # %bb.0:
@@ -1360,7 +1312,7 @@ define <4 x float> @add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c,
 ; X64-AVX1-NEXT:  # %bb.1:
 ; X64-AVX1-NEXT:    vaddss %xmm1, %xmm0, %xmm2
 ; X64-AVX1-NEXT:  .LBB70_2:
-; X64-AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; X64-AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
 ; X64-AVX1-NEXT:    retq
 ;
 ; X64-AVX512-LABEL: add_ss_mask:
@@ -1381,29 +1333,17 @@ define <4 x float> @add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c,
 }
 
 define <2 x double> @add_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
-; X86-SSE2-LABEL: add_sd_mask:
-; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    testb $1, {{[0-9]+}}(%esp)
-; X86-SSE2-NEXT:    jne .LBB71_1
-; X86-SSE2-NEXT:  # %bb.2:
-; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X86-SSE2-NEXT:    retl
-; X86-SSE2-NEXT:  .LBB71_1:
-; X86-SSE2-NEXT:    addsd %xmm0, %xmm1
-; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X86-SSE2-NEXT:    retl
-;
-; X86-SSE41-LABEL: add_sd_mask:
-; X86-SSE41:       # %bb.0:
-; X86-SSE41-NEXT:    testb $1, {{[0-9]+}}(%esp)
-; X86-SSE41-NEXT:    jne .LBB71_1
-; X86-SSE41-NEXT:  # %bb.2:
-; X86-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; X86-SSE41-NEXT:    retl
-; X86-SSE41-NEXT:  .LBB71_1:
-; X86-SSE41-NEXT:    addsd %xmm0, %xmm1
-; X86-SSE41-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X86-SSE41-NEXT:    retl
+; X86-SSE-LABEL: add_sd_mask:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    jne .LBB71_1
+; X86-SSE-NEXT:  # %bb.2:
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X86-SSE-NEXT:    retl
+; X86-SSE-NEXT:  .LBB71_1:
+; X86-SSE-NEXT:    addsd %xmm0, %xmm1
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X86-SSE-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: add_sd_mask:
 ; X86-AVX1:       # %bb.0:
@@ -1412,7 +1352,7 @@ define <2 x double> @add_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double>
 ; X86-AVX1-NEXT:  # %bb.1:
 ; X86-AVX1-NEXT:    vaddsd %xmm1, %xmm0, %xmm2
 ; X86-AVX1-NEXT:  .LBB71_2:
-; X86-AVX1-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X86-AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX512-LABEL: add_sd_mask:
@@ -1423,29 +1363,17 @@ define <2 x double> @add_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double>
 ; X86-AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; X86-AVX512-NEXT:    retl
 ;
-; X64-SSE2-LABEL: add_sd_mask:
-; X64-SSE2:       # %bb.0:
-; X64-SSE2-NEXT:    testb $1, %dil
-; X64-SSE2-NEXT:    jne .LBB71_1
-; X64-SSE2-NEXT:  # %bb.2:
-; X64-SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X64-SSE2-NEXT:    retq
-; X64-SSE2-NEXT:  .LBB71_1:
-; X64-SSE2-NEXT:    addsd %xmm0, %xmm1
-; X64-SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X64-SSE2-NEXT:    retq
-;
-; X64-SSE41-LABEL: add_sd_mask:
-; X64-SSE41:       # %bb.0:
-; X64-SSE41-NEXT:    testb $1, %dil
-; X64-SSE41-NEXT:    jne .LBB71_1
-; X64-SSE41-NEXT:  # %bb.2:
-; X64-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; X64-SSE41-NEXT:    retq
-; X64-SSE41-NEXT:  .LBB71_1:
-; X64-SSE41-NEXT:    addsd %xmm0, %xmm1
-; X64-SSE41-NEXT:    blendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X64-SSE41-NEXT:    retq
+; X64-SSE-LABEL: add_sd_mask:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    testb $1, %dil
+; X64-SSE-NEXT:    jne .LBB71_1
+; X64-SSE-NEXT:  # %bb.2:
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X64-SSE-NEXT:    retq
+; X64-SSE-NEXT:  .LBB71_1:
+; X64-SSE-NEXT:    addsd %xmm0, %xmm1
+; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X64-SSE-NEXT:    retq
 ;
 ; X64-AVX1-LABEL: add_sd_mask:
 ; X64-AVX1:       # %bb.0:
@@ -1454,7 +1382,7 @@ define <2 x double> @add_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double>
 ; X64-AVX1-NEXT:  # %bb.1:
 ; X64-AVX1-NEXT:    vaddsd %xmm1, %xmm0, %xmm2
 ; X64-AVX1-NEXT:  .LBB71_2:
-; X64-AVX1-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X64-AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; X64-AVX1-NEXT:    retq
 ;
 ; X64-AVX512-LABEL: add_sd_mask:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
index dbdc45abb24d..3f48b22e2b9f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
@@ -631,7 +631,7 @@ define <2 x double> @test_mm_cmpge_sd(<2 x double> %a0, <2 x double> %a1) nounwi
 ; AVX-LABEL: test_mm_cmpge_sd:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmplesd %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf3,0xc2,0xc8,0x02]
-; AVX-NEXT:    vblendpd $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0d,0xc1,0x01]
+; AVX-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a1, <2 x double> %a0, i8 2)
@@ -748,7 +748,7 @@ define <2 x double> @test_mm_cmpgt_sd(<2 x double> %a0, <2 x double> %a1) nounwi
 ; AVX-LABEL: test_mm_cmpgt_sd:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmpltsd %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf3,0xc2,0xc8,0x01]
-; AVX-NEXT:    vblendpd $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0d,0xc1,0x01]
+; AVX-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a1, <2 x double> %a0, i8 1)
@@ -976,7 +976,7 @@ define <2 x double> @test_mm_cmpnge_sd(<2 x double> %a0, <2 x double> %a1) nounw
 ; AVX-LABEL: test_mm_cmpnge_sd:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmpnlesd %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf3,0xc2,0xc8,0x06]
-; AVX-NEXT:    vblendpd $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0d,0xc1,0x01]
+; AVX-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a1, <2 x double> %a0, i8 6)
@@ -1021,7 +1021,7 @@ define <2 x double> @test_mm_cmpngt_sd(<2 x double> %a0, <2 x double> %a1) nounw
 ; AVX-LABEL: test_mm_cmpngt_sd:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcmpnltsd %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf3,0xc2,0xc8,0x05]
-; AVX-NEXT:    vblendpd $1, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0d,0xc1,0x01]
+; AVX-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x10,0xc1]
 ; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %cmp = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a1, <2 x double> %a0, i8 5)
@@ -3010,8 +3010,8 @@ define <2 x double> @test_mm_move_sd(<2 x double> %a0, <2 x double> %a1) nounwin
 ;
 ; AVX-LABEL: test_mm_move_sd:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps $3, %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x03]
-; AVX-NEXT:    # xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x10,0xc1]
+; AVX-NEXT:    # xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %ext0 = extractelement <2 x double> %a1, i32 0
   %res0 = insertelement <2 x double> undef, double %ext0, i32 0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
index f6b0df153c26..413b4e79257a 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
@@ -724,8 +724,8 @@ define <2 x double> @test_x86_sse2_cvtss2sd_load(<2 x double> %a0, ptr %p1) {
 ; X86-AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-AVX1-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x08]
 ; X86-AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf2,0x5a,0xc9]
-; X86-AVX1-NEXT:    vblendps $3, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x03]
-; X86-AVX1-NEXT:    ## xmm0 = xmm1[0,1],xmm0[2,3]
+; X86-AVX1-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfb,0x10,0xc1]
+; X86-AVX1-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X86-AVX1-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X86-AVX512-LABEL: test_x86_sse2_cvtss2sd_load:
@@ -734,8 +734,8 @@ define <2 x double> @test_x86_sse2_cvtss2sd_load(<2 x double> %a0, ptr %p1) {
 ; X86-AVX512-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x08]
 ; X86-AVX512-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf2,0x5a,0xc9]
-; X86-AVX512-NEXT:    vblendps $3, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x03]
-; X86-AVX512-NEXT:    ## xmm0 = xmm1[0,1],xmm0[2,3]
+; X86-AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfb,0x10,0xc1]
+; X86-AVX512-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: test_x86_sse2_cvtss2sd_load:
@@ -752,8 +752,8 @@ define <2 x double> @test_x86_sse2_cvtss2sd_load(<2 x double> %a0, ptr %p1) {
 ; X64-AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X64-AVX1-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x0f]
 ; X64-AVX1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf2,0x5a,0xc9]
-; X64-AVX1-NEXT:    vblendps $3, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x03]
-; X64-AVX1-NEXT:    ## xmm0 = xmm1[0,1],xmm0[2,3]
+; X64-AVX1-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfb,0x10,0xc1]
+; X64-AVX1-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_x86_sse2_cvtss2sd_load:
@@ -761,8 +761,8 @@ define <2 x double> @test_x86_sse2_cvtss2sd_load(<2 x double> %a0, ptr %p1) {
 ; X64-AVX512-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X64-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x0f]
 ; X64-AVX512-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf2,0x5a,0xc9]
-; X64-AVX512-NEXT:    vblendps $3, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x03]
-; X64-AVX512-NEXT:    ## xmm0 = xmm1[0,1],xmm0[2,3]
+; X64-AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfb,0x10,0xc1]
+; X64-AVX512-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %a1 = load <4 x float>, ptr %p1
   %res = call <2 x double> @llvm.x86.sse2.cvtss2sd(<2 x double> %a0, <4 x float> %a1) ; <<2 x double>> [#uses=1]
@@ -782,7 +782,7 @@ define <2 x double> @test_x86_sse2_cvtss2sd_load_optsize(<2 x double> %a0, ptr %
 ; X86-AVX1-LABEL: test_x86_sse2_cvtss2sd_load_optsize:
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX1-NEXT:    vcvtss2sd (%eax), %xmm1, %xmm1 ## encoding: [0xc5,0xf2,0x5a,0x08]
+; X86-AVX1-NEXT:    vcvtss2sd (%eax), %xmm7, %xmm1 ## encoding: [0xc5,0xc2,0x5a,0x08]
 ; X86-AVX1-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfb,0x10,0xc1]
 ; X86-AVX1-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X86-AVX1-NEXT:    retl ## encoding: [0xc3]
@@ -790,7 +790,7 @@ define <2 x double> @test_x86_sse2_cvtss2sd_load_optsize(<2 x double> %a0, ptr %
 ; X86-AVX512-LABEL: test_x86_sse2_cvtss2sd_load_optsize:
 ; X86-AVX512:       ## %bb.0:
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512-NEXT:    vcvtss2sd (%eax), %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf2,0x5a,0x08]
+; X86-AVX512-NEXT:    vcvtss2sd (%eax), %xmm7, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xc2,0x5a,0x08]
 ; X86-AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfb,0x10,0xc1]
 ; X86-AVX512-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
@@ -804,14 +804,14 @@ define <2 x double> @test_x86_sse2_cvtss2sd_load_optsize(<2 x double> %a0, ptr %
 ;
 ; X64-AVX1-LABEL: test_x86_sse2_cvtss2sd_load_optsize:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vcvtss2sd (%rdi), %xmm1, %xmm1 ## encoding: [0xc5,0xf2,0x5a,0x0f]
+; X64-AVX1-NEXT:    vcvtss2sd (%rdi), %xmm15, %xmm1 ## encoding: [0xc5,0x82,0x5a,0x0f]
 ; X64-AVX1-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfb,0x10,0xc1]
 ; X64-AVX1-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_x86_sse2_cvtss2sd_load_optsize:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vcvtss2sd (%rdi), %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf2,0x5a,0x0f]
+; X64-AVX512-NEXT:    vcvtss2sd (%rdi), %xmm15, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0x82,0x5a,0x0f]
 ; X64-AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfb,0x10,0xc1]
 ; X64-AVX512-NEXT:    ## xmm0 = xmm1[0],xmm0[1]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse2.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse2.ll
index 3e5d76eae0bb..6e77d3e4fd13 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse2.ll
@@ -417,7 +417,7 @@ define void @test12() nounwind {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmovaps 0, %xmm0
 ; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
-; AVX512-NEXT:    vblendps {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3]
+; AVX512-NEXT:    vmovsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
 ; AVX512-NEXT:    vaddps %xmm0, %xmm1, %xmm0
@@ -670,7 +670,7 @@ define <4 x i32> @PR19721(<4 x i32> %i) {
 ; AVX-LABEL: PR19721:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; X64-SSE-LABEL: PR19721:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll
index 47630501864a..c6f0ec493a36 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll
@@ -33,7 +33,7 @@ define <2 x double> @test_mm_blend_pd(<2 x double> %a0, <2 x double> %a1) {
 ;
 ; AVX-LABEL: test_mm_blend_pd:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    ret{{[l|q]}}
   %res = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll
index bdf8033a00b0..137606b7cfee 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll
@@ -18,8 +18,8 @@ define <2 x double> @test_x86_sse41_blendpd(<2 x double> %a0, <2 x double> %a1)
 ;
 ; AVX-LABEL: test_x86_sse41_blendpd:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    vblendps $3, %xmm0, %xmm1, %xmm0 ## encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x03]
-; AVX-NEXT:    ## xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd %xmm0, %xmm1, %xmm0 ## encoding: [0xc5,0xf3,0x10,0xc0]
+; AVX-NEXT:    ## xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
   %res = call <2 x double> @llvm.x86.sse41.blendpd(<2 x double> %a0, <2 x double> %a1, i32 6) ; <<2 x double>> [#uses=1]
   ret <2 x double> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/sse41.ll b/external/llvm-project/llvm/test/CodeGen/X86/sse41.ll
index 2d7258a49f5d..4f5b7ee0eaea 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/sse41.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/sse41.ll
@@ -345,7 +345,7 @@ define <4 x float> @blendps_not_insertps_1(<4 x float> %t1, float %t2) nounwind
 ; X86-SSE:       ## %bb.0:
 ; X86-SSE-NEXT:    movss {{[0-9]+}}(%esp), %xmm1 ## xmm1 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    ## encoding: [0xf3,0x0f,0x10,0x4c,0x24,0x04]
-; X86-SSE-NEXT:    blendps $1, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0c,0xc1,0x01]
+; X86-SSE-NEXT:    movss %xmm1, %xmm0 ## encoding: [0xf3,0x0f,0x10,0xc1]
 ; X86-SSE-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; X86-SSE-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -353,7 +353,7 @@ define <4 x float> @blendps_not_insertps_1(<4 x float> %t1, float %t2) nounwind
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vmovss {{[0-9]+}}(%esp), %xmm1 ## xmm1 = mem[0],zero,zero,zero
 ; X86-AVX1-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x4c,0x24,0x04]
-; X86-AVX1-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; X86-AVX1-NEXT:    vmovss %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x10,0xc1]
 ; X86-AVX1-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; X86-AVX1-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -361,19 +361,19 @@ define <4 x float> @blendps_not_insertps_1(<4 x float> %t1, float %t2) nounwind
 ; X86-AVX512:       ## %bb.0:
 ; X86-AVX512-NEXT:    vmovss {{[0-9]+}}(%esp), %xmm1 ## xmm1 = mem[0],zero,zero,zero
 ; X86-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x4c,0x24,0x04]
-; X86-AVX512-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; X86-AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x10,0xc1]
 ; X86-AVX512-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: blendps_not_insertps_1:
 ; X64-SSE:       ## %bb.0:
-; X64-SSE-NEXT:    blendps $1, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0c,0xc1,0x01]
+; X64-SSE-NEXT:    movss %xmm1, %xmm0 ## encoding: [0xf3,0x0f,0x10,0xc1]
 ; X64-SSE-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: blendps_not_insertps_1:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; X64-AVX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x10,0xc1]
 ; X64-AVX-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; X64-AVX-NEXT:    retq ## encoding: [0xc3]
   %tmp1 = insertelement <4 x float> %t1, float %t2, i32 0
@@ -434,13 +434,13 @@ define <4 x float> @insertps_or_blendps(<4 x float> %t1, float %t2) minsize noun
 define <4 x float> @blendps_not_insertps_2(<4 x float> %t1, <4 x float> %t2) nounwind {
 ; SSE-LABEL: blendps_not_insertps_2:
 ; SSE:       ## %bb.0:
-; SSE-NEXT:    blendps $1, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0c,0xc1,0x01]
+; SSE-NEXT:    movss %xmm1, %xmm0 ## encoding: [0xf3,0x0f,0x10,0xc1]
 ; SSE-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
 ;
 ; AVX-LABEL: blendps_not_insertps_2:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    vblendps $1, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x01]
+; AVX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x10,0xc1]
 ; AVX-NEXT:    ## xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
   %tmp2 = extractelement <4 x float> %t2, i32 0
@@ -1207,7 +1207,7 @@ define <4 x i32> @i32_shuf_X00A(<4 x i32> %x, <4 x i32> %a) {
 ; AVX1-LABEL: i32_shuf_X00A:
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
-; AVX1-NEXT:    vblendps $1, %xmm0, %xmm2, %xmm0 ## encoding: [0xc4,0xe3,0x69,0x0c,0xc0,0x01]
+; AVX1-NEXT:    vmovss %xmm0, %xmm2, %xmm0 ## encoding: [0xc5,0xea,0x10,0xc0]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX1-NEXT:    vshufps $0, %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0xc6,0xc9,0x00]
 ; AVX1-NEXT:    ## xmm1 = xmm1[0,0,0,0]
@@ -1218,7 +1218,7 @@ define <4 x i32> @i32_shuf_X00A(<4 x i32> %x, <4 x i32> %a) {
 ; AVX512-LABEL: i32_shuf_X00A:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
-; AVX512-NEXT:    vblendps $1, %xmm0, %xmm2, %xmm0 ## encoding: [0xc4,0xe3,0x69,0x0c,0xc0,0x01]
+; AVX512-NEXT:    vmovss %xmm0, %xmm2, %xmm0 ## encoding: [0xc5,0xea,0x10,0xc0]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX512-NEXT:    vbroadcastss %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0xc9]
 ; AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/ssub_sat_vec.ll b/external/llvm-project/llvm/test/CodeGen/X86/ssub_sat_vec.ll
index 88df3c175ec9..eb2ad4fdff92 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/ssub_sat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/ssub_sat_vec.ll
@@ -567,7 +567,7 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; AVX512BW-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    retq
   %z = call <16 x i4> @llvm.ssub.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
@@ -601,7 +601,7 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ;
 ; AVX512BW-LABEL: v16i1:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 ^ mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 ^ m32bcst)
 ; AVX512BW-NEXT:    retq
   %z = call <16 x i1> @llvm.ssub.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
   ret <16 x i1> %z
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll b/external/llvm-project/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
index 6625cc4f07a2..665a84a26fea 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
@@ -990,7 +990,7 @@ define double @stack_fold_cvtsi2sd(i32 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    popq %r12
@@ -1034,7 +1034,7 @@ define <2 x double> @stack_fold_cvtsi2sd_int(i32 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2sdl {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
@@ -1080,7 +1080,7 @@ define double @stack_fold_cvtsi642sd(i64 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    popq %r12
@@ -1124,7 +1124,7 @@ define <2 x double> @stack_fold_cvtsi642sd_int(i64 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2sdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
@@ -1170,7 +1170,7 @@ define float @stack_fold_cvtsi2ss(i32 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    popq %r12
@@ -1214,9 +1214,9 @@ define <4 x float> @stack_fold_cvtsi2ss_int(i32 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2ssl {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    popq %r12
@@ -1261,7 +1261,7 @@ define float @stack_fold_cvtsi642ss(i64 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    popq %r12
@@ -1305,9 +1305,9 @@ define <4 x float> @stack_fold_cvtsi642ss_int(i64 %a0) {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vcvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vcvtsi2ssq {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    popq %r12
@@ -2861,8 +2861,8 @@ define double @stack_fold_roundsd(double %a0) optsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vroundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm15, %xmm15, %xmm15
+; CHECK-NEXT:    vroundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call double @llvm.floor.f64(double %a0)
@@ -2876,7 +2876,7 @@ define double @stack_fold_roundsd_minsize(double %a0) minsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vroundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vroundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call double @llvm.floor.f64(double %a0)
@@ -2908,8 +2908,8 @@ define float @stack_fold_roundss(float %a0) optsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vroundss $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm15, %xmm15, %xmm15
+; CHECK-NEXT:    vroundss $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call float @llvm.floor.f32(float %a0)
@@ -3106,8 +3106,8 @@ define double @stack_fold_sqrtsd(double %a0) optsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vsqrtsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm15, %xmm15, %xmm15
+; CHECK-NEXT:    vsqrtsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call double @llvm.sqrt.f64(double %a0)
@@ -3124,8 +3124,8 @@ define float @stack_fold_sqrtss(float %a0) optsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vsqrtss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; CHECK-NEXT:    vxorps %xmm15, %xmm15, %xmm15
+; CHECK-NEXT:    vsqrtss {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = call float @llvm.sqrt.f32(float %a0)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/stack-protector-msvc-oz.ll b/external/llvm-project/llvm/test/CodeGen/X86/stack-protector-msvc-oz.ll
new file mode 100644
index 000000000000..d8a772efbd7e
--- /dev/null
+++ b/external/llvm-project/llvm/test/CodeGen/X86/stack-protector-msvc-oz.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=i386-pc-windows-msvc < %s -o - | FileCheck -check-prefix=MSVC-X86 %s
+; RUN: llc -mtriple=x86_64-pc-windows-msvc < %s -o - | FileCheck -check-prefix=MSVC-X64 %s
+
+; Make sure fastisel falls back and does something secure.
+; RUN: llc -mtriple=i686-pc-windows-msvc -O0 < %s -o - | FileCheck -check-prefix=MSVC-X86-O0 %s
+; RUN: llc -mtriple=x86_64-pc-windows-msvc -O0 < %s -o - | FileCheck -check-prefix=MSVC-X64-O0 %s
+
+@"\01LC" = internal constant [11 x i8] c"buf == %s\0A\00"    ; <ptr> [#uses=1]
+
+define void @test(ptr %a) nounwind ssp minsize {
+; MSVC-X86-LABEL: test:
+; MSVC-X86:       # %bb.0: # %entry
+; MSVC-X86-NEXT:    pushl %esi
+; MSVC-X86-NEXT:    subl $12, %esp
+; MSVC-X86-NEXT:    movl ___security_cookie, %eax
+; MSVC-X86-NEXT:    xorl %esp, %eax
+; MSVC-X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; MSVC-X86-NEXT:    movl %esp, %esi
+; MSVC-X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; MSVC-X86-NEXT:    pushl %esi
+; MSVC-X86-NEXT:    calll _strcpy
+; MSVC-X86-NEXT:    popl %ecx
+; MSVC-X86-NEXT:    popl %edx
+; MSVC-X86-NEXT:    pushl %esi
+; MSVC-X86-NEXT:    pushl $LC
+; MSVC-X86-NEXT:    calll _printf
+; MSVC-X86-NEXT:    popl %ecx
+; MSVC-X86-NEXT:    popl %edx
+; MSVC-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; MSVC-X86-NEXT:    xorl %esp, %ecx
+; MSVC-X86-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-NEXT:    addl $12, %esp
+; MSVC-X86-NEXT:    popl %esi
+; MSVC-X86-NEXT:    retl
+;
+; MSVC-X64-LABEL: test:
+; MSVC-X64:       # %bb.0: # %entry
+; MSVC-X64-NEXT:    pushq %rsi
+; MSVC-X64-NEXT:    subq $64, %rsp
+; MSVC-X64-NEXT:    movq %rcx, %rdx
+; MSVC-X64-NEXT:    movq __security_cookie(%rip), %rax
+; MSVC-X64-NEXT:    xorq %rsp, %rax
+; MSVC-X64-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; MSVC-X64-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
+; MSVC-X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; MSVC-X64-NEXT:    movq %rsi, %rcx
+; MSVC-X64-NEXT:    callq strcpy
+; MSVC-X64-NEXT:    leaq LC(%rip), %rcx
+; MSVC-X64-NEXT:    movq %rsi, %rdx
+; MSVC-X64-NEXT:    callq printf
+; MSVC-X64-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; MSVC-X64-NEXT:    xorq %rsp, %rcx
+; MSVC-X64-NEXT:    callq __security_check_cookie
+; MSVC-X64-NEXT:    addq $64, %rsp
+; MSVC-X64-NEXT:    popq %rsi
+; MSVC-X64-NEXT:    retq
+;
+; MSVC-X86-O0-LABEL: test:
+; MSVC-X86-O0:       # %bb.0: # %entry
+; MSVC-X86-O0-NEXT:    subl $20, %esp
+; MSVC-X86-O0-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; MSVC-X86-O0-NEXT:    movl ___security_cookie, %eax
+; MSVC-X86-O0-NEXT:    xorl %esp, %eax
+; MSVC-X86-O0-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; MSVC-X86-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; MSVC-X86-O0-NEXT:    movl %esp, %eax
+; MSVC-X86-O0-NEXT:    movl %ecx, 4(%eax)
+; MSVC-X86-O0-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; MSVC-X86-O0-NEXT:    movl %ecx, (%eax)
+; MSVC-X86-O0-NEXT:    calll _strcpy
+; MSVC-X86-O0-NEXT:    leal LC, %ecx
+; MSVC-X86-O0-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; MSVC-X86-O0-NEXT:    movl %ecx, (%esp)
+; MSVC-X86-O0-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; MSVC-X86-O0-NEXT:    calll _printf
+; MSVC-X86-O0-NEXT:  # %bb.1: # %return
+; MSVC-X86-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; MSVC-X86-O0-NEXT:    xorl %esp, %ecx
+; MSVC-X86-O0-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-O0-NEXT:    addl $20, %esp
+; MSVC-X86-O0-NEXT:    retl
+;
+; MSVC-X64-O0-LABEL: test:
+; MSVC-X64-O0:       # %bb.0: # %entry
+; MSVC-X64-O0-NEXT:    subq $56, %rsp
+; MSVC-X64-O0-NEXT:    movq __security_cookie(%rip), %rax
+; MSVC-X64-O0-NEXT:    xorq %rsp, %rax
+; MSVC-X64-O0-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; MSVC-X64-O0-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
+; MSVC-X64-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
+; MSVC-X64-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; MSVC-X64-O0-NEXT:    callq strcpy
+; MSVC-X64-O0-NEXT:    leaq LC(%rip), %rcx
+; MSVC-X64-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; MSVC-X64-O0-NEXT:    callq printf
+; MSVC-X64-O0-NEXT:  # %bb.1: # %return
+; MSVC-X64-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; MSVC-X64-O0-NEXT:    xorq %rsp, %rcx
+; MSVC-X64-O0-NEXT:    callq __security_check_cookie
+; MSVC-X64-O0-NEXT:    addq $56, %rsp
+; MSVC-X64-O0-NEXT:    retq
+entry:
+ %a_addr = alloca ptr    ; <ptr> [#uses=2]
+ %buf = alloca [8 x i8]    ; <ptr> [#uses=2]
+ store ptr %a, ptr %a_addr
+ %0 = load ptr, ptr %a_addr, align 4    ; <ptr> [#uses=1]
+ %1 = call ptr @strcpy(ptr %buf, ptr %0) nounwind   ; <ptr> [#uses=0]
+ %2 = call i32 (ptr, ...) @printf(ptr @"\01LC", ptr %buf) nounwind    ; <i32> [#uses=0]
+ br label %return
+
+return:    ; preds = %entry
+ ret void
+}
+
+declare ptr @strcpy(ptr, ptr) nounwind
+
+declare i32 @printf(ptr, ...) nounwind
+
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/stack-protector-msvc.ll b/external/llvm-project/llvm/test/CodeGen/X86/stack-protector-msvc.ll
index d718062d2c48..a868fa549296 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/stack-protector-msvc.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/stack-protector-msvc.ll
@@ -25,12 +25,19 @@ define void @test(ptr %a) nounwind ssp {
 ; MSVC-X86-NEXT:    pushl $LC
 ; MSVC-X86-NEXT:    calll _printf
 ; MSVC-X86-NEXT:    addl $8, %esp
-; MSVC-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; MSVC-X86-NEXT:    xorl %esp, %ecx
-; MSVC-X86-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; MSVC-X86-NEXT:    xorl %esp, %eax
+; MSVC-X86-NEXT:    movl ___security_cookie, %ecx
+; MSVC-X86-NEXT:    cmpl %eax, %ecx
+; MSVC-X86-NEXT:    jne LBB0_2
+; MSVC-X86-NEXT:  # %bb.1: # %return
 ; MSVC-X86-NEXT:    addl $12, %esp
 ; MSVC-X86-NEXT:    popl %esi
 ; MSVC-X86-NEXT:    retl
+; MSVC-X86-NEXT:  LBB0_2: # %return
+; MSVC-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; MSVC-X86-NEXT:    xorl %esp, %ecx
+; MSVC-X86-NEXT:    calll @__security_check_cookie@4
 ;
 ; MSVC-X64-LABEL: test:
 ; MSVC-X64:       # %bb.0: # %entry
@@ -47,17 +54,19 @@ define void @test(ptr %a) nounwind ssp {
 ; MSVC-X64-NEXT:    leaq LC(%rip), %rcx
 ; MSVC-X64-NEXT:    movq %rsi, %rdx
 ; MSVC-X64-NEXT:    callq printf
-; MSVC-X64-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; MSVC-X64-NEXT:    xorq %rsp, %rcx
-; MSVC-X64-NEXT:    cmpq __security_cookie(%rip), %rcx
+; MSVC-X64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; MSVC-X64-NEXT:    xorq %rsp, %rax
+; MSVC-X64-NEXT:    movq __security_cookie(%rip), %rcx
+; MSVC-X64-NEXT:    cmpq %rax, %rcx
 ; MSVC-X64-NEXT:    jne .LBB0_2
-; MSVC-X64-NEXT:  # %bb.1:
+; MSVC-X64-NEXT:  # %bb.1: # %return
 ; MSVC-X64-NEXT:    addq $64, %rsp
 ; MSVC-X64-NEXT:    popq %rsi
 ; MSVC-X64-NEXT:    retq
-; MSVC-X64-NEXT:  .LBB0_2:
+; MSVC-X64-NEXT:  .LBB0_2: # %return
+; MSVC-X64-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; MSVC-X64-NEXT:    xorq %rsp, %rcx
 ; MSVC-X64-NEXT:    callq __security_check_cookie
-; MSVC-X64-NEXT:    int3
 ;
 ; MSVC-X86-O0-LABEL: test:
 ; MSVC-X86-O0:       # %bb.0: # %entry
@@ -80,7 +89,15 @@ define void @test(ptr %a) nounwind ssp {
 ; MSVC-X86-O0-NEXT:  # %bb.1: # %return
 ; MSVC-X86-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; MSVC-X86-O0-NEXT:    xorl %esp, %ecx
+; MSVC-X86-O0-NEXT:    movl ___security_cookie, %eax
+; MSVC-X86-O0-NEXT:    subl %ecx, %eax
+; MSVC-X86-O0-NEXT:    jne LBB0_3
+; MSVC-X86-O0-NEXT:    jmp LBB0_2
+; MSVC-X86-O0-NEXT:  LBB0_3: # %return
+; MSVC-X86-O0-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; MSVC-X86-O0-NEXT:    xorl %esp, %ecx
 ; MSVC-X86-O0-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-O0-NEXT:  LBB0_2: # %return
 ; MSVC-X86-O0-NEXT:    addl $20, %esp
 ; MSVC-X86-O0-NEXT:    retl
 ;
@@ -100,9 +117,18 @@ define void @test(ptr %a) nounwind ssp {
 ; MSVC-X64-O0-NEXT:  # %bb.1: # %return
 ; MSVC-X64-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; MSVC-X64-O0-NEXT:    xorq %rsp, %rcx
+; MSVC-X64-O0-NEXT:    movq __security_cookie(%rip), %rax
+; MSVC-X64-O0-NEXT:    subq %rcx, %rax
+; MSVC-X64-O0-NEXT:    jne .LBB0_3
+; MSVC-X64-O0-NEXT:    jmp .LBB0_2
+; MSVC-X64-O0-NEXT:  .LBB0_3: # %return
+; MSVC-X64-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; MSVC-X64-O0-NEXT:    xorq %rsp, %rcx
 ; MSVC-X64-O0-NEXT:    callq __security_check_cookie
+; MSVC-X64-O0-NEXT:  .LBB0_2: # %return
 ; MSVC-X64-O0-NEXT:    addq $56, %rsp
 ; MSVC-X64-O0-NEXT:    retq
+
 entry:
  %a_addr = alloca ptr    ; <ptr> [#uses=2]
  %buf = alloca [8 x i8]    ; <ptr> [#uses=2]
@@ -134,12 +160,19 @@ define void @test_vla(i32 %n) nounwind ssp {
 ; MSVC-X86-NEXT:    pushl %eax
 ; MSVC-X86-NEXT:    calll _escape
 ; MSVC-X86-NEXT:    addl $4, %esp
-; MSVC-X86-NEXT:    movl -4(%ebp), %ecx
-; MSVC-X86-NEXT:    xorl %ebp, %ecx
-; MSVC-X86-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-NEXT:    movl -4(%ebp), %eax
+; MSVC-X86-NEXT:    xorl %ebp, %eax
+; MSVC-X86-NEXT:    movl ___security_cookie, %ecx
+; MSVC-X86-NEXT:    cmpl %eax, %ecx
+; MSVC-X86-NEXT:    jne LBB1_2
+; MSVC-X86-NEXT:  # %bb.1:
 ; MSVC-X86-NEXT:    movl %ebp, %esp
 ; MSVC-X86-NEXT:    popl %ebp
 ; MSVC-X86-NEXT:    retl
+; MSVC-X86-NEXT:  LBB1_2:
+; MSVC-X86-NEXT:    movl -4(%ebp), %ecx
+; MSVC-X86-NEXT:    xorl %ebp, %ecx
+; MSVC-X86-NEXT:    calll @__security_check_cookie@4
 ;
 ; MSVC-X64-LABEL: test_vla:
 ; MSVC-X64:       # %bb.0:
@@ -158,19 +191,20 @@ define void @test_vla(i32 %n) nounwind ssp {
 ; MSVC-X64-NEXT:    subq $32, %rsp
 ; MSVC-X64-NEXT:    callq escape
 ; MSVC-X64-NEXT:    addq $32, %rsp
-; MSVC-X64-NEXT:    movq -8(%rbp), %rcx
-; MSVC-X64-NEXT:    xorq %rbp, %rcx
-; MSVC-X64-NEXT:    cmpq __security_cookie(%rip), %rcx
+; MSVC-X64-NEXT:    movq -8(%rbp), %rax
+; MSVC-X64-NEXT:    xorq %rbp, %rax
+; MSVC-X64-NEXT:    movq __security_cookie(%rip), %rcx
+; MSVC-X64-NEXT:    cmpq %rax, %rcx
 ; MSVC-X64-NEXT:    jne .LBB1_2
 ; MSVC-X64-NEXT:  # %bb.1:
 ; MSVC-X64-NEXT:    movq %rbp, %rsp
 ; MSVC-X64-NEXT:    popq %rbp
 ; MSVC-X64-NEXT:    retq
 ; MSVC-X64-NEXT:  .LBB1_2:
+; MSVC-X64-NEXT:    movq -8(%rbp), %rcx
+; MSVC-X64-NEXT:    xorq %rbp, %rcx
 ; MSVC-X64-NEXT:    subq $32, %rsp
 ; MSVC-X64-NEXT:    callq __security_check_cookie
-; MSVC-X64-NEXT:    addq $32, %rsp
-; MSVC-X64-NEXT:    int3
 ;
 ; MSVC-X86-O0-LABEL: test_vla:
 ; MSVC-X86-O0:       # %bb.0:
@@ -190,7 +224,15 @@ define void @test_vla(i32 %n) nounwind ssp {
 ; MSVC-X86-O0-NEXT:    addl $4, %esp
 ; MSVC-X86-O0-NEXT:    movl -4(%ebp), %ecx
 ; MSVC-X86-O0-NEXT:    xorl %ebp, %ecx
+; MSVC-X86-O0-NEXT:    movl ___security_cookie, %eax
+; MSVC-X86-O0-NEXT:    subl %ecx, %eax
+; MSVC-X86-O0-NEXT:    jne LBB1_2
+; MSVC-X86-O0-NEXT:    jmp LBB1_1
+; MSVC-X86-O0-NEXT:  LBB1_2:
+; MSVC-X86-O0-NEXT:    movl -4(%ebp), %ecx
+; MSVC-X86-O0-NEXT:    xorl %ebp, %ecx
 ; MSVC-X86-O0-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-O0-NEXT:  LBB1_1:
 ; MSVC-X86-O0-NEXT:    movl %ebp, %esp
 ; MSVC-X86-O0-NEXT:    popl %ebp
 ; MSVC-X86-O0-NEXT:    retl
@@ -215,8 +257,16 @@ define void @test_vla(i32 %n) nounwind ssp {
 ; MSVC-X64-O0-NEXT:    addq $32, %rsp
 ; MSVC-X64-O0-NEXT:    movq -8(%rbp), %rcx
 ; MSVC-X64-O0-NEXT:    xorq %rbp, %rcx
+; MSVC-X64-O0-NEXT:    movq __security_cookie(%rip), %rax
+; MSVC-X64-O0-NEXT:    subq %rcx, %rax
+; MSVC-X64-O0-NEXT:    jne .LBB1_2
+; MSVC-X64-O0-NEXT:    jmp .LBB1_1
+; MSVC-X64-O0-NEXT:  .LBB1_2:
+; MSVC-X64-O0-NEXT:    movq -8(%rbp), %rcx
+; MSVC-X64-O0-NEXT:    xorq %rbp, %rcx
 ; MSVC-X64-O0-NEXT:    subq $32, %rsp
 ; MSVC-X64-O0-NEXT:    callq __security_check_cookie
+; MSVC-X64-O0-NEXT:  .LBB1_1:
 ; MSVC-X64-O0-NEXT:    movq %rbp, %rsp
 ; MSVC-X64-O0-NEXT:    popq %rbp
 ; MSVC-X64-O0-NEXT:    retq
@@ -253,14 +303,21 @@ define void @test_vla_realign(i32 %n) nounwind ssp {
 ; MSVC-X86-NEXT:    pushl %edi
 ; MSVC-X86-NEXT:    calll _escape
 ; MSVC-X86-NEXT:    addl $4, %esp
-; MSVC-X86-NEXT:    movl 12(%esi), %ecx
-; MSVC-X86-NEXT:    xorl %ebp, %ecx
-; MSVC-X86-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-NEXT:    movl 12(%esi), %eax
+; MSVC-X86-NEXT:    xorl %ebp, %eax
+; MSVC-X86-NEXT:    movl ___security_cookie, %ecx
+; MSVC-X86-NEXT:    cmpl %eax, %ecx
+; MSVC-X86-NEXT:    jne LBB2_2
+; MSVC-X86-NEXT:  # %bb.1:
 ; MSVC-X86-NEXT:    leal -8(%ebp), %esp
 ; MSVC-X86-NEXT:    popl %esi
 ; MSVC-X86-NEXT:    popl %edi
 ; MSVC-X86-NEXT:    popl %ebp
 ; MSVC-X86-NEXT:    retl
+; MSVC-X86-NEXT:  LBB2_2:
+; MSVC-X86-NEXT:    movl 12(%esi), %ecx
+; MSVC-X86-NEXT:    xorl %ebp, %ecx
+; MSVC-X86-NEXT:    calll @__security_check_cookie@4
 ;
 ; MSVC-X64-LABEL: test_vla_realign:
 ; MSVC-X64:       # %bb.0:
@@ -286,9 +343,10 @@ define void @test_vla_realign(i32 %n) nounwind ssp {
 ; MSVC-X64-NEXT:    movq %rsi, %rcx
 ; MSVC-X64-NEXT:    callq escape
 ; MSVC-X64-NEXT:    addq $32, %rsp
-; MSVC-X64-NEXT:    movq 24(%rbx), %rcx
-; MSVC-X64-NEXT:    xorq %rbp, %rcx
-; MSVC-X64-NEXT:    cmpq __security_cookie(%rip), %rcx
+; MSVC-X64-NEXT:    movq 24(%rbx), %rax
+; MSVC-X64-NEXT:    xorq %rbp, %rax
+; MSVC-X64-NEXT:    movq __security_cookie(%rip), %rcx
+; MSVC-X64-NEXT:    cmpq %rax, %rcx
 ; MSVC-X64-NEXT:    jne .LBB2_2
 ; MSVC-X64-NEXT:  # %bb.1:
 ; MSVC-X64-NEXT:    movq %rbp, %rsp
@@ -297,10 +355,10 @@ define void @test_vla_realign(i32 %n) nounwind ssp {
 ; MSVC-X64-NEXT:    popq %rbp
 ; MSVC-X64-NEXT:    retq
 ; MSVC-X64-NEXT:  .LBB2_2:
+; MSVC-X64-NEXT:    movq 24(%rbx), %rcx
+; MSVC-X64-NEXT:    xorq %rbp, %rcx
 ; MSVC-X64-NEXT:    subq $32, %rsp
 ; MSVC-X64-NEXT:    callq __security_check_cookie
-; MSVC-X64-NEXT:    addq $32, %rsp
-; MSVC-X64-NEXT:    int3
 ;
 ; MSVC-X86-O0-LABEL: test_vla_realign:
 ; MSVC-X86-O0:       # %bb.0:
@@ -328,7 +386,15 @@ define void @test_vla_realign(i32 %n) nounwind ssp {
 ; MSVC-X86-O0-NEXT:    addl $4, %esp
 ; MSVC-X86-O0-NEXT:    movl 48(%esi), %ecx
 ; MSVC-X86-O0-NEXT:    xorl %ebp, %ecx
+; MSVC-X86-O0-NEXT:    movl ___security_cookie, %eax
+; MSVC-X86-O0-NEXT:    subl %ecx, %eax
+; MSVC-X86-O0-NEXT:    jne LBB2_2
+; MSVC-X86-O0-NEXT:    jmp LBB2_1
+; MSVC-X86-O0-NEXT:  LBB2_2:
+; MSVC-X86-O0-NEXT:    movl 48(%esi), %ecx
+; MSVC-X86-O0-NEXT:    xorl %ebp, %ecx
 ; MSVC-X86-O0-NEXT:    calll @__security_check_cookie@4
+; MSVC-X86-O0-NEXT:  LBB2_1:
 ; MSVC-X86-O0-NEXT:    leal -4(%ebp), %esp
 ; MSVC-X86-O0-NEXT:    popl %esi
 ; MSVC-X86-O0-NEXT:    popl %ebp
@@ -361,8 +427,16 @@ define void @test_vla_realign(i32 %n) nounwind ssp {
 ; MSVC-X64-O0-NEXT:    addq $32, %rsp
 ; MSVC-X64-O0-NEXT:    movq 64(%rbx), %rcx
 ; MSVC-X64-O0-NEXT:    xorq %rbp, %rcx
+; MSVC-X64-O0-NEXT:    movq __security_cookie(%rip), %rax
+; MSVC-X64-O0-NEXT:    subq %rcx, %rax
+; MSVC-X64-O0-NEXT:    jne .LBB2_2
+; MSVC-X64-O0-NEXT:    jmp .LBB2_1
+; MSVC-X64-O0-NEXT:  .LBB2_2:
+; MSVC-X64-O0-NEXT:    movq 64(%rbx), %rcx
+; MSVC-X64-O0-NEXT:    xorq %rbp, %rcx
 ; MSVC-X64-O0-NEXT:    subq $32, %rsp
 ; MSVC-X64-O0-NEXT:    callq __security_check_cookie
+; MSVC-X64-O0-NEXT:  .LBB2_1:
 ; MSVC-X64-O0-NEXT:    leaq 8(%rbp), %rsp
 ; MSVC-X64-O0-NEXT:    popq %rbx
 ; MSVC-X64-O0-NEXT:    popq %rbp
@@ -377,3 +451,4 @@ define void @test_vla_realign(i32 %n) nounwind ssp {
 declare ptr @strcpy(ptr, ptr) nounwind
 
 declare i32 @printf(ptr, ...) nounwind
+
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll b/external/llvm-project/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll
index 05fefbe750e5..7ce983869ce7 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll
@@ -39,7 +39,7 @@ define ptr @test1(ptr %arg1, ptr %arg2) {
   ; CHECK-NEXT:   INLINEASM_BR &"#$0 $1 $2", 9 /* sideeffect mayload attdialect */, 13 /* imm */, 42, 13 /* imm */, 0, 13 /* imm */, %bb.4, 12 /* clobber */, implicit-def early-clobber $df, 12 /* clobber */, implicit-def early-clobber $fpsw, 12 /* clobber */, implicit-def early-clobber $eflags
   ; CHECK-NEXT:   JMP_1 %bb.5
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.4.bb17.i.i.i (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.4.bb17.i.i.i (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.5(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.5.kmem_cache_has_cpu_partial.exit:
@@ -118,7 +118,7 @@ define void @ceph_con_v2_try_read(i32 %__trans_tmp_3.sroa.0.0.copyload, i1 %tobo
   ; CHECK-NEXT:   LIFETIME_END %stack.0.skip.i.i
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.5.if.end.i (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT: bb.5.if.end.i (inlineasm-br-indirect-target):
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   LIFETIME_END %stack.0.skip.i.i
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll b/external/llvm-project/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll
index d8fcf6d86fa4..ecbbaf3ab362 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll
@@ -34,12 +34,12 @@ define i32 @foo(i32 %x) nounwind ssp {
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:    .data_region jt32
-; CHECK-NEXT:  .set L0_0_set_2, LBB0_2-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_3, LBB0_3-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_4, LBB0_4-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_5, LBB0_5-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_6, LBB0_6-LJTI0_0
-; CHECK-NEXT:  .set L0_0_set_7, LBB0_7-LJTI0_0
+; CHECK-NEXT:  L0_0_set_2 = LBB0_2-LJTI0_0
+; CHECK-NEXT:  L0_0_set_3 = LBB0_3-LJTI0_0
+; CHECK-NEXT:  L0_0_set_4 = LBB0_4-LJTI0_0
+; CHECK-NEXT:  L0_0_set_5 = LBB0_5-LJTI0_0
+; CHECK-NEXT:  L0_0_set_6 = LBB0_6-LJTI0_0
+; CHECK-NEXT:  L0_0_set_7 = LBB0_7-LJTI0_0
 ; CHECK-NEXT:  LJTI0_0:
 ; CHECK-NEXT:    .long L0_0_set_2
 ; CHECK-NEXT:    .long L0_0_set_3
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/tailcc-ssp.ll b/external/llvm-project/llvm/test/CodeGen/X86/tailcc-ssp.ll
index 5211e4fe9eef..ac5dda7d69bd 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/tailcc-ssp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/tailcc-ssp.ll
@@ -13,9 +13,10 @@ define tailcc void @tailcall_frame(ptr %0, i64 %1) sspreq {
 ; WINDOWS-NEXT:    movq __security_cookie(%rip), %rax
 ; WINDOWS-NEXT:    xorq %rsp, %rax
 ; WINDOWS-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; WINDOWS-NEXT:    xorq %rsp, %rcx
-; WINDOWS-NEXT:    cmpq __security_cookie(%rip), %rcx
+; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WINDOWS-NEXT:    xorq %rsp, %rax
+; WINDOWS-NEXT:    movq __security_cookie(%rip), %rcx
+; WINDOWS-NEXT:    cmpq %rax, %rcx
 ; WINDOWS-NEXT:    jne .LBB0_1
 ; WINDOWS-NEXT:  # %bb.2:
 ; WINDOWS-NEXT:    xorl %ecx, %ecx
@@ -26,6 +27,8 @@ define tailcc void @tailcall_frame(ptr %0, i64 %1) sspreq {
 ; WINDOWS-NEXT:    .seh_endepilogue
 ; WINDOWS-NEXT:    jmp h # TAILCALL
 ; WINDOWS-NEXT:  .LBB0_1:
+; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; WINDOWS-NEXT:    xorq %rsp, %rcx
 ; WINDOWS-NEXT:    callq __security_check_cookie
 ; WINDOWS-NEXT:    int3
 ; WINDOWS-NEXT:    .seh_endproc
@@ -49,7 +52,6 @@ define tailcc void @tailcall_frame(ptr %0, i64 %1) sspreq {
 ; LINUX-NEXT:  .LBB0_2: # %CallStackCheckFailBlk
 ; LINUX-NEXT:    .cfi_def_cfa_offset 32
 ; LINUX-NEXT:    callq __stack_chk_fail@PLT
-
    tail call tailcc void @h(ptr null, i64 0, ptr null)
    ret void
 }
@@ -65,9 +67,10 @@ define void @tailcall_unrelated_frame() sspreq {
 ; WINDOWS-NEXT:    xorq %rsp, %rax
 ; WINDOWS-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
 ; WINDOWS-NEXT:    callq bar
-; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; WINDOWS-NEXT:    xorq %rsp, %rcx
-; WINDOWS-NEXT:    cmpq __security_cookie(%rip), %rcx
+; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WINDOWS-NEXT:    xorq %rsp, %rax
+; WINDOWS-NEXT:    movq __security_cookie(%rip), %rcx
+; WINDOWS-NEXT:    cmpq %rax, %rcx
 ; WINDOWS-NEXT:    jne .LBB1_1
 ; WINDOWS-NEXT:  # %bb.2:
 ; WINDOWS-NEXT:    .seh_startepilogue
@@ -75,10 +78,12 @@ define void @tailcall_unrelated_frame() sspreq {
 ; WINDOWS-NEXT:    .seh_endepilogue
 ; WINDOWS-NEXT:    jmp bar # TAILCALL
 ; WINDOWS-NEXT:  .LBB1_1:
+; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; WINDOWS-NEXT:    xorq %rsp, %rcx
 ; WINDOWS-NEXT:    callq __security_check_cookie
 ; WINDOWS-NEXT:    int3
 ; WINDOWS-NEXT:    .seh_endproc
-
+;
 ; LINUX-LABEL: tailcall_unrelated_frame:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq %rax
@@ -97,6 +102,7 @@ define void @tailcall_unrelated_frame() sspreq {
 ; LINUX-NEXT:    .cfi_def_cfa_offset 16
 ; LINUX-NEXT:    callq __stack_chk_fail@PLT
 
+
   call void @bar()
   tail call void @bar()
   ret void
@@ -105,18 +111,51 @@ define void @tailcall_unrelated_frame() sspreq {
 declare void @callee()
 define void @caller() sspreq {
 ; WINDOWS-LABEL: caller:
-; WINDOWS: callq   callee
-; WINDOWS: callq   callee
-; WINDOWS: cmpq    __security_cookie(%rip), %rcx
-; WINDOWS: jne
-; WINDOWS: callq   __security_check_cookie
-
+; WINDOWS:       # %bb.0:
+; WINDOWS-NEXT:    subq $40, %rsp
+; WINDOWS-NEXT:    .seh_stackalloc 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    movq __security_cookie(%rip), %rax
+; WINDOWS-NEXT:    xorq %rsp, %rax
+; WINDOWS-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; WINDOWS-NEXT:    callq callee
+; WINDOWS-NEXT:    callq callee
+; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WINDOWS-NEXT:    xorq %rsp, %rax
+; WINDOWS-NEXT:    movq __security_cookie(%rip), %rcx
+; WINDOWS-NEXT:    cmpq %rax, %rcx
+; WINDOWS-NEXT:    jne .LBB2_2
+; WINDOWS-NEXT:  # %bb.1:
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    addq $40, %rsp
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    retq
+; WINDOWS-NEXT:  .LBB2_2:
+; WINDOWS-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; WINDOWS-NEXT:    xorq %rsp, %rcx
+; WINDOWS-NEXT:    callq __security_check_cookie
+; WINDOWS-NEXT:    int3
+; WINDOWS-NEXT:    .seh_endproc
+;
 ; LINUX-LABEL: caller:
-; LINUX: callq   callee@PLT
-; LINUX: callq   callee@PLT
-; LINUX: cmpq
-; LINUX: jne
-; LINUX: callq   __stack_chk_fail@PLT
+; LINUX:       # %bb.0:
+; LINUX-NEXT:    pushq %rax
+; LINUX-NEXT:    .cfi_def_cfa_offset 16
+; LINUX-NEXT:    movq %fs:40, %rax
+; LINUX-NEXT:    movq %rax, (%rsp)
+; LINUX-NEXT:    callq callee@PLT
+; LINUX-NEXT:    callq callee@PLT
+; LINUX-NEXT:    movq %fs:40, %rax
+; LINUX-NEXT:    cmpq (%rsp), %rax
+; LINUX-NEXT:    jne .LBB2_2
+; LINUX-NEXT:  # %bb.1: # %SP_return
+; LINUX-NEXT:    popq %rax
+; LINUX-NEXT:    .cfi_def_cfa_offset 8
+; LINUX-NEXT:    retq
+; LINUX-NEXT:  .LBB2_2: # %CallStackCheckFailBlk
+; LINUX-NEXT:    .cfi_def_cfa_offset 16
+; LINUX-NEXT:    callq __stack_chk_fail@PLT
+
 
   tail call void @callee()
   call void @callee()
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-scalar-variablemask.ll b/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-scalar-variablemask.ll
index 9c9d06921096..6a55d740fe42 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-scalar-variablemask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-scalar-variablemask.ll
@@ -6,21 +6,18 @@
 define i8 @out8(i8 %x, i8 %y, i8 %mask) {
 ; CHECK-NOBMI-LABEL: out8:
 ; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    movl %edx, %eax
-; CHECK-NOBMI-NEXT:    andl %edx, %edi
-; CHECK-NOBMI-NEXT:    notb %al
-; CHECK-NOBMI-NEXT:    andb %sil, %al
-; CHECK-NOBMI-NEXT:    orb %dil, %al
+; CHECK-NOBMI-NEXT:    movl %edi, %eax
+; CHECK-NOBMI-NEXT:    xorl %esi, %eax
+; CHECK-NOBMI-NEXT:    andl %edx, %eax
+; CHECK-NOBMI-NEXT:    xorl %esi, %eax
 ; CHECK-NOBMI-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NOBMI-NEXT:    retq
 ;
 ; CHECK-BMI-LABEL: out8:
 ; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movl %edx, %eax
+; CHECK-BMI-NEXT:    andnl %esi, %edx, %eax
 ; CHECK-BMI-NEXT:    andl %edx, %edi
-; CHECK-BMI-NEXT:    notb %al
-; CHECK-BMI-NEXT:    andb %sil, %al
-; CHECK-BMI-NEXT:    orb %dil, %al
+; CHECK-BMI-NEXT:    orl %edi, %eax
 ; CHECK-BMI-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-BMI-NEXT:    retq
   %mx = and i8 %x, %mask
@@ -33,18 +30,17 @@ define i8 @out8(i8 %x, i8 %y, i8 %mask) {
 define i16 @out16(i16 %x, i16 %y, i16 %mask) {
 ; CHECK-NOBMI-LABEL: out16:
 ; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    movl %edx, %eax
-; CHECK-NOBMI-NEXT:    andl %edx, %edi
-; CHECK-NOBMI-NEXT:    notl %eax
-; CHECK-NOBMI-NEXT:    andl %esi, %eax
-; CHECK-NOBMI-NEXT:    orl %edi, %eax
+; CHECK-NOBMI-NEXT:    movl %edi, %eax
+; CHECK-NOBMI-NEXT:    xorl %esi, %eax
+; CHECK-NOBMI-NEXT:    andl %edx, %eax
+; CHECK-NOBMI-NEXT:    xorl %esi, %eax
 ; CHECK-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NOBMI-NEXT:    retq
 ;
 ; CHECK-BMI-LABEL: out16:
 ; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    andl %edx, %edi
 ; CHECK-BMI-NEXT:    andnl %esi, %edx, %eax
+; CHECK-BMI-NEXT:    andl %edx, %edi
 ; CHECK-BMI-NEXT:    orl %edi, %eax
 ; CHECK-BMI-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-BMI-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll b/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll
index b1194bedc4e1..809c15881cc9 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/unfold-masked-merge-vector-variablemask.ll
@@ -16,11 +16,10 @@
 define <1 x i8> @out_v1i8(<1 x i8> %x, <1 x i8> %y, <1 x i8> %mask) nounwind {
 ; CHECK-LABEL: out_v1i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edx, %eax
-; CHECK-NEXT:    andl %edx, %edi
-; CHECK-NEXT:    notb %al
-; CHECK-NEXT:    andb %sil, %al
-; CHECK-NEXT:    orb %dil, %al
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    xorl %esi, %eax
+; CHECK-NEXT:    andl %edx, %eax
+; CHECK-NEXT:    xorl %esi, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
   %mx = and <1 x i8> %x, %mask
@@ -37,32 +36,28 @@ define <1 x i8> @out_v1i8(<1 x i8> %x, <1 x i8> %y, <1 x i8> %mask) nounwind {
 define <2 x i8> @out_v2i8(<2 x i8> %x, <2 x i8> %y, <2 x i8> %mask) nounwind {
 ; CHECK-BASELINE-LABEL: out_v2i8:
 ; CHECK-BASELINE:       # %bb.0:
-; CHECK-BASELINE-NEXT:    movl %r8d, %eax
+; CHECK-BASELINE-NEXT:    movl %edi, %eax
+; CHECK-BASELINE-NEXT:    xorl %edx, %eax
+; CHECK-BASELINE-NEXT:    andl %r8d, %eax
+; CHECK-BASELINE-NEXT:    xorl %edx, %eax
+; CHECK-BASELINE-NEXT:    xorl %ecx, %esi
 ; CHECK-BASELINE-NEXT:    andl %r9d, %esi
-; CHECK-BASELINE-NEXT:    andl %r8d, %edi
-; CHECK-BASELINE-NEXT:    notb %al
-; CHECK-BASELINE-NEXT:    notb %r9b
-; CHECK-BASELINE-NEXT:    andb %cl, %r9b
-; CHECK-BASELINE-NEXT:    andb %dl, %al
-; CHECK-BASELINE-NEXT:    orb %dil, %al
-; CHECK-BASELINE-NEXT:    orb %sil, %r9b
+; CHECK-BASELINE-NEXT:    xorl %ecx, %esi
 ; CHECK-BASELINE-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-BASELINE-NEXT:    movl %r9d, %edx
+; CHECK-BASELINE-NEXT:    movl %esi, %edx
 ; CHECK-BASELINE-NEXT:    retq
 ;
 ; CHECK-SSE1-LABEL: out_v2i8:
 ; CHECK-SSE1:       # %bb.0:
-; CHECK-SSE1-NEXT:    movl %r8d, %eax
+; CHECK-SSE1-NEXT:    movl %edi, %eax
+; CHECK-SSE1-NEXT:    xorl %edx, %eax
+; CHECK-SSE1-NEXT:    andl %r8d, %eax
+; CHECK-SSE1-NEXT:    xorl %edx, %eax
+; CHECK-SSE1-NEXT:    xorl %ecx, %esi
 ; CHECK-SSE1-NEXT:    andl %r9d, %esi
-; CHECK-SSE1-NEXT:    andl %r8d, %edi
-; CHECK-SSE1-NEXT:    notb %al
-; CHECK-SSE1-NEXT:    notb %r9b
-; CHECK-SSE1-NEXT:    andb %cl, %r9b
-; CHECK-SSE1-NEXT:    andb %dl, %al
-; CHECK-SSE1-NEXT:    orb %dil, %al
-; CHECK-SSE1-NEXT:    orb %sil, %r9b
+; CHECK-SSE1-NEXT:    xorl %ecx, %esi
 ; CHECK-SSE1-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-SSE1-NEXT:    movl %r9d, %edx
+; CHECK-SSE1-NEXT:    movl %esi, %edx
 ; CHECK-SSE1-NEXT:    retq
 ;
 ; CHECK-SSE2-LABEL: out_v2i8:
@@ -86,11 +81,10 @@ define <2 x i8> @out_v2i8(<2 x i8> %x, <2 x i8> %y, <2 x i8> %mask) nounwind {
 define <1 x i16> @out_v1i16(<1 x i16> %x, <1 x i16> %y, <1 x i16> %mask) nounwind {
 ; CHECK-LABEL: out_v1i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edx, %eax
-; CHECK-NEXT:    andl %edx, %edi
-; CHECK-NEXT:    notl %eax
-; CHECK-NEXT:    andl %esi, %eax
-; CHECK-NEXT:    orl %edi, %eax
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    xorl %esi, %eax
+; CHECK-NEXT:    andl %edx, %eax
+; CHECK-NEXT:    xorl %esi, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    retq
   %mx = and <1 x i16> %x, %mask
@@ -235,32 +229,28 @@ define <4 x i8> @out_v4i8_undef(<4 x i8> %x, <4 x i8> %y, <4 x i8> %mask) nounwi
 define <2 x i16> @out_v2i16(<2 x i16> %x, <2 x i16> %y, <2 x i16> %mask) nounwind {
 ; CHECK-BASELINE-LABEL: out_v2i16:
 ; CHECK-BASELINE:       # %bb.0:
-; CHECK-BASELINE-NEXT:    movl %r8d, %eax
+; CHECK-BASELINE-NEXT:    movl %edi, %eax
+; CHECK-BASELINE-NEXT:    xorl %edx, %eax
+; CHECK-BASELINE-NEXT:    andl %r8d, %eax
+; CHECK-BASELINE-NEXT:    xorl %edx, %eax
+; CHECK-BASELINE-NEXT:    xorl %ecx, %esi
 ; CHECK-BASELINE-NEXT:    andl %r9d, %esi
-; CHECK-BASELINE-NEXT:    andl %r8d, %edi
-; CHECK-BASELINE-NEXT:    notl %eax
-; CHECK-BASELINE-NEXT:    notl %r9d
-; CHECK-BASELINE-NEXT:    andl %ecx, %r9d
-; CHECK-BASELINE-NEXT:    orl %esi, %r9d
-; CHECK-BASELINE-NEXT:    andl %edx, %eax
-; CHECK-BASELINE-NEXT:    orl %edi, %eax
+; CHECK-BASELINE-NEXT:    xorl %ecx, %esi
 ; CHECK-BASELINE-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-BASELINE-NEXT:    movl %r9d, %edx
+; CHECK-BASELINE-NEXT:    movl %esi, %edx
 ; CHECK-BASELINE-NEXT:    retq
 ;
 ; CHECK-SSE1-LABEL: out_v2i16:
 ; CHECK-SSE1:       # %bb.0:
-; CHECK-SSE1-NEXT:    movl %r8d, %eax
+; CHECK-SSE1-NEXT:    movl %edi, %eax
+; CHECK-SSE1-NEXT:    xorl %edx, %eax
+; CHECK-SSE1-NEXT:    andl %r8d, %eax
+; CHECK-SSE1-NEXT:    xorl %edx, %eax
+; CHECK-SSE1-NEXT:    xorl %ecx, %esi
 ; CHECK-SSE1-NEXT:    andl %r9d, %esi
-; CHECK-SSE1-NEXT:    andl %r8d, %edi
-; CHECK-SSE1-NEXT:    notl %eax
-; CHECK-SSE1-NEXT:    notl %r9d
-; CHECK-SSE1-NEXT:    andl %ecx, %r9d
-; CHECK-SSE1-NEXT:    orl %esi, %r9d
-; CHECK-SSE1-NEXT:    andl %edx, %eax
-; CHECK-SSE1-NEXT:    orl %edi, %eax
+; CHECK-SSE1-NEXT:    xorl %ecx, %esi
 ; CHECK-SSE1-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-SSE1-NEXT:    movl %r9d, %edx
+; CHECK-SSE1-NEXT:    movl %esi, %edx
 ; CHECK-SSE1-NEXT:    retq
 ;
 ; CHECK-SSE2-LABEL: out_v2i16:
@@ -439,9 +429,12 @@ define <4 x i16> @out_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) nounwin
 ; CHECK-BASELINE-LABEL: out_v4i16:
 ; CHECK-BASELINE:       # %bb.0:
 ; CHECK-BASELINE-NEXT:    movq %rdi, %rax
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %edi
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r11d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r11d
+; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
+; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %si
+; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
 ; CHECK-BASELINE-NEXT:    xorl %r11d, %edx
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %dx
 ; CHECK-BASELINE-NEXT:    xorl %r11d, %edx
@@ -451,21 +444,21 @@ define <4 x i16> @out_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) nounwin
 ; CHECK-BASELINE-NEXT:    xorl %edi, %r8d
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %r8w
 ; CHECK-BASELINE-NEXT:    xorl %edi, %r8d
-; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
-; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %si
-; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
-; CHECK-BASELINE-NEXT:    movw %si, (%rax)
 ; CHECK-BASELINE-NEXT:    movw %r8w, 6(%rax)
 ; CHECK-BASELINE-NEXT:    movw %cx, 4(%rax)
 ; CHECK-BASELINE-NEXT:    movw %dx, 2(%rax)
+; CHECK-BASELINE-NEXT:    movw %si, (%rax)
 ; CHECK-BASELINE-NEXT:    retq
 ;
 ; CHECK-SSE1-LABEL: out_v4i16:
 ; CHECK-SSE1:       # %bb.0:
 ; CHECK-SSE1-NEXT:    movq %rdi, %rax
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %edi
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r11d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r11d
+; CHECK-SSE1-NEXT:    xorl %r9d, %esi
+; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %si
+; CHECK-SSE1-NEXT:    xorl %r9d, %esi
 ; CHECK-SSE1-NEXT:    xorl %r11d, %edx
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %dx
 ; CHECK-SSE1-NEXT:    xorl %r11d, %edx
@@ -475,13 +468,10 @@ define <4 x i16> @out_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) nounwin
 ; CHECK-SSE1-NEXT:    xorl %edi, %r8d
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %r8w
 ; CHECK-SSE1-NEXT:    xorl %edi, %r8d
-; CHECK-SSE1-NEXT:    xorl %r9d, %esi
-; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %si
-; CHECK-SSE1-NEXT:    xorl %r9d, %esi
-; CHECK-SSE1-NEXT:    movw %si, (%rax)
 ; CHECK-SSE1-NEXT:    movw %r8w, 6(%rax)
 ; CHECK-SSE1-NEXT:    movw %cx, 4(%rax)
 ; CHECK-SSE1-NEXT:    movw %dx, 2(%rax)
+; CHECK-SSE1-NEXT:    movw %si, (%rax)
 ; CHECK-SSE1-NEXT:    retq
 ;
 ; CHECK-SSE2-LABEL: out_v4i16:
@@ -506,43 +496,43 @@ define <4 x i16> @out_v4i16_undef(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) n
 ; CHECK-BASELINE-LABEL: out_v4i16_undef:
 ; CHECK-BASELINE:       # %bb.0:
 ; CHECK-BASELINE-NEXT:    movq %rdi, %rax
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %edi
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %cx
+; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
+; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %si
+; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
 ; CHECK-BASELINE-NEXT:    xorl %r10d, %edx
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %dx
 ; CHECK-BASELINE-NEXT:    xorl %r10d, %edx
 ; CHECK-BASELINE-NEXT:    xorl %edi, %r8d
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %r8w
 ; CHECK-BASELINE-NEXT:    xorl %edi, %r8d
-; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
-; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %si
-; CHECK-BASELINE-NEXT:    xorl %r9d, %esi
 ; CHECK-BASELINE-NEXT:    movw %cx, 4(%rax)
-; CHECK-BASELINE-NEXT:    movw %si, (%rax)
 ; CHECK-BASELINE-NEXT:    movw %r8w, 6(%rax)
 ; CHECK-BASELINE-NEXT:    movw %dx, 2(%rax)
+; CHECK-BASELINE-NEXT:    movw %si, (%rax)
 ; CHECK-BASELINE-NEXT:    retq
 ;
 ; CHECK-SSE1-LABEL: out_v4i16_undef:
 ; CHECK-SSE1:       # %bb.0:
 ; CHECK-SSE1-NEXT:    movq %rdi, %rax
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %edi
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %cx
+; CHECK-SSE1-NEXT:    xorl %r9d, %esi
+; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %si
+; CHECK-SSE1-NEXT:    xorl %r9d, %esi
 ; CHECK-SSE1-NEXT:    xorl %r10d, %edx
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %dx
 ; CHECK-SSE1-NEXT:    xorl %r10d, %edx
 ; CHECK-SSE1-NEXT:    xorl %edi, %r8d
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %r8w
 ; CHECK-SSE1-NEXT:    xorl %edi, %r8d
-; CHECK-SSE1-NEXT:    xorl %r9d, %esi
-; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %si
-; CHECK-SSE1-NEXT:    xorl %r9d, %esi
 ; CHECK-SSE1-NEXT:    movw %cx, 4(%rax)
-; CHECK-SSE1-NEXT:    movw %si, (%rax)
 ; CHECK-SSE1-NEXT:    movw %r8w, 6(%rax)
 ; CHECK-SSE1-NEXT:    movw %dx, 2(%rax)
+; CHECK-SSE1-NEXT:    movw %si, (%rax)
 ; CHECK-SSE1-NEXT:    retq
 ;
 ; CHECK-SSE2-LABEL: out_v4i16_undef:
@@ -883,14 +873,14 @@ define <8 x i16> @out_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %mask) nounwin
 ; CHECK-BASELINE-NEXT:    pushq %r12
 ; CHECK-BASELINE-NEXT:    pushq %rbx
 ; CHECK-BASELINE-NEXT:    movq %rdi, %rax
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %edi
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r11d
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %ebx
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %ebp
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r14d
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r15d
-; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r12d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r11d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %ebx
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %ebp
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r14d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r15d
+; CHECK-BASELINE-NEXT:    movl {{[0-9]+}}(%rsp), %r12d
 ; CHECK-BASELINE-NEXT:    xorl %r12d, %esi
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %si
 ; CHECK-BASELINE-NEXT:    xorl %r12d, %esi
@@ -906,16 +896,16 @@ define <8 x i16> @out_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %mask) nounwin
 ; CHECK-BASELINE-NEXT:    xorl %ebx, %r9d
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %r9w
 ; CHECK-BASELINE-NEXT:    xorl %ebx, %r9d
-; CHECK-BASELINE-NEXT:    movl %r11d, %ebx
-; CHECK-BASELINE-NEXT:    xorw {{[0-9]+}}(%rsp), %bx
+; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %ebx
+; CHECK-BASELINE-NEXT:    xorw %r11w, %bx
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %bx
 ; CHECK-BASELINE-NEXT:    xorl %r11d, %ebx
-; CHECK-BASELINE-NEXT:    movl %r10d, %r11d
-; CHECK-BASELINE-NEXT:    xorw {{[0-9]+}}(%rsp), %r11w
+; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r11d
+; CHECK-BASELINE-NEXT:    xorw %r10w, %r11w
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %r11w
 ; CHECK-BASELINE-NEXT:    xorl %r10d, %r11d
-; CHECK-BASELINE-NEXT:    movl %edi, %r10d
-; CHECK-BASELINE-NEXT:    xorw {{[0-9]+}}(%rsp), %r10w
+; CHECK-BASELINE-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; CHECK-BASELINE-NEXT:    xorw %di, %r10w
 ; CHECK-BASELINE-NEXT:    andw {{[0-9]+}}(%rsp), %r10w
 ; CHECK-BASELINE-NEXT:    xorl %edi, %r10d
 ; CHECK-BASELINE-NEXT:    movw %r10w, 14(%rax)
@@ -941,14 +931,14 @@ define <8 x i16> @out_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %mask) nounwin
 ; CHECK-SSE1-NEXT:    pushq %r12
 ; CHECK-SSE1-NEXT:    pushq %rbx
 ; CHECK-SSE1-NEXT:    movq %rdi, %rax
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %edi
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r11d
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %ebx
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %ebp
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r14d
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r15d
-; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r12d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %edi
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r11d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %ebx
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %ebp
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r14d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r15d
+; CHECK-SSE1-NEXT:    movl {{[0-9]+}}(%rsp), %r12d
 ; CHECK-SSE1-NEXT:    xorl %r12d, %esi
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %si
 ; CHECK-SSE1-NEXT:    xorl %r12d, %esi
@@ -964,16 +954,16 @@ define <8 x i16> @out_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %mask) nounwin
 ; CHECK-SSE1-NEXT:    xorl %ebx, %r9d
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %r9w
 ; CHECK-SSE1-NEXT:    xorl %ebx, %r9d
-; CHECK-SSE1-NEXT:    movl %r11d, %ebx
-; CHECK-SSE1-NEXT:    xorw {{[0-9]+}}(%rsp), %bx
+; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %ebx
+; CHECK-SSE1-NEXT:    xorw %r11w, %bx
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %bx
 ; CHECK-SSE1-NEXT:    xorl %r11d, %ebx
-; CHECK-SSE1-NEXT:    movl %r10d, %r11d
-; CHECK-SSE1-NEXT:    xorw {{[0-9]+}}(%rsp), %r11w
+; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r11d
+; CHECK-SSE1-NEXT:    xorw %r10w, %r11w
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %r11w
 ; CHECK-SSE1-NEXT:    xorl %r10d, %r11d
-; CHECK-SSE1-NEXT:    movl %edi, %r10d
-; CHECK-SSE1-NEXT:    xorw {{[0-9]+}}(%rsp), %r10w
+; CHECK-SSE1-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; CHECK-SSE1-NEXT:    xorw %di, %r10w
 ; CHECK-SSE1-NEXT:    andw {{[0-9]+}}(%rsp), %r10w
 ; CHECK-SSE1-NEXT:    xorl %edi, %r10d
 ; CHECK-SSE1-NEXT:    movw %r10w, 14(%rax)
@@ -1759,113 +1749,117 @@ define <16 x i16> @out_v16i16(ptr%px, ptr%py, ptr%pmask) nounwind {
 ; CHECK-BASELINE-NEXT:    pushq %r13
 ; CHECK-BASELINE-NEXT:    pushq %r12
 ; CHECK-BASELINE-NEXT:    pushq %rbx
-; CHECK-BASELINE-NEXT:    movzwl 18(%rdx), %r15d
-; CHECK-BASELINE-NEXT:    movzwl 16(%rdx), %r14d
-; CHECK-BASELINE-NEXT:    movzwl 14(%rdx), %ebp
-; CHECK-BASELINE-NEXT:    movzwl 12(%rdx), %ebx
-; CHECK-BASELINE-NEXT:    movzwl 10(%rdx), %r13d
-; CHECK-BASELINE-NEXT:    movzwl 8(%rdx), %r11d
-; CHECK-BASELINE-NEXT:    movzwl 6(%rdx), %r10d
-; CHECK-BASELINE-NEXT:    movzwl 4(%rdx), %r9d
-; CHECK-BASELINE-NEXT:    movzwl (%rdx), %r8d
-; CHECK-BASELINE-NEXT:    movzwl 2(%rdx), %r12d
-; CHECK-BASELINE-NEXT:    movzwl (%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r8w, %ax
-; CHECK-BASELINE-NEXT:    andw (%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r8d
-; CHECK-BASELINE-NEXT:    movl %r8d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-BASELINE-NEXT:    movzwl 2(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r12w, %ax
-; CHECK-BASELINE-NEXT:    andw 2(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r12d
-; CHECK-BASELINE-NEXT:    movzwl 4(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r9w, %ax
-; CHECK-BASELINE-NEXT:    andw 4(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r9d
-; CHECK-BASELINE-NEXT:    movl %r9d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-BASELINE-NEXT:    movzwl 6(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r10w, %ax
-; CHECK-BASELINE-NEXT:    andw 6(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r10d
-; CHECK-BASELINE-NEXT:    movl %r10d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-BASELINE-NEXT:    movzwl 8(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r11w, %ax
-; CHECK-BASELINE-NEXT:    andw 8(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r11d
-; CHECK-BASELINE-NEXT:    movl %r11d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-BASELINE-NEXT:    movzwl 10(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r13w, %ax
-; CHECK-BASELINE-NEXT:    andw 10(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r13d
-; CHECK-BASELINE-NEXT:    movl %r13d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-BASELINE-NEXT:    movzwl 12(%rsi), %eax
+; CHECK-BASELINE-NEXT:    movq %rcx, %r10
+; CHECK-BASELINE-NEXT:    movq %rdx, %r8
+; CHECK-BASELINE-NEXT:    movq %rsi, %r9
+; CHECK-BASELINE-NEXT:    movq %rdi, %r11
+; CHECK-BASELINE-NEXT:    movzwl 18(%rdx), %ebp
+; CHECK-BASELINE-NEXT:    movl 16(%rdx), %r15d
+; CHECK-BASELINE-NEXT:    movzwl 14(%rdx), %r13d
+; CHECK-BASELINE-NEXT:    movl 12(%rdx), %r12d
+; CHECK-BASELINE-NEXT:    movzwl 10(%rdx), %r14d
+; CHECK-BASELINE-NEXT:    movl 8(%rdx), %ebx
+; CHECK-BASELINE-NEXT:    movzwl 6(%rdx), %eax
+; CHECK-BASELINE-NEXT:    movl (%rdx), %ecx
+; CHECK-BASELINE-NEXT:    movl 4(%rdx), %edx
+; CHECK-BASELINE-NEXT:    movzwl 2(%r8), %esi
+; CHECK-BASELINE-NEXT:    movzwl (%r9), %edi
+; CHECK-BASELINE-NEXT:    xorw %cx, %di
+; CHECK-BASELINE-NEXT:    andw (%r10), %di
+; CHECK-BASELINE-NEXT:    xorl %ecx, %edi
+; CHECK-BASELINE-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-BASELINE-NEXT:    movzwl 2(%r9), %ecx
+; CHECK-BASELINE-NEXT:    xorw %si, %cx
+; CHECK-BASELINE-NEXT:    andw 2(%r10), %cx
+; CHECK-BASELINE-NEXT:    xorl %esi, %ecx
+; CHECK-BASELINE-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-BASELINE-NEXT:    movzwl 4(%r9), %ecx
+; CHECK-BASELINE-NEXT:    xorw %dx, %cx
+; CHECK-BASELINE-NEXT:    andw 4(%r10), %cx
+; CHECK-BASELINE-NEXT:    xorl %edx, %ecx
+; CHECK-BASELINE-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-BASELINE-NEXT:    movzwl 6(%r9), %ecx
+; CHECK-BASELINE-NEXT:    xorw %ax, %cx
+; CHECK-BASELINE-NEXT:    andw 6(%r10), %cx
+; CHECK-BASELINE-NEXT:    xorl %eax, %ecx
+; CHECK-BASELINE-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-BASELINE-NEXT:    movzwl 8(%r9), %eax
 ; CHECK-BASELINE-NEXT:    xorw %bx, %ax
-; CHECK-BASELINE-NEXT:    andw 12(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %ebx
-; CHECK-BASELINE-NEXT:    movzwl 14(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %bp, %ax
-; CHECK-BASELINE-NEXT:    andw 14(%rcx), %ax
+; CHECK-BASELINE-NEXT:    andw 8(%r10), %ax
+; CHECK-BASELINE-NEXT:    xorl %ebx, %eax
+; CHECK-BASELINE-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-BASELINE-NEXT:    movzwl 10(%r9), %ebx
+; CHECK-BASELINE-NEXT:    xorw %r14w, %bx
+; CHECK-BASELINE-NEXT:    andw 10(%r10), %bx
+; CHECK-BASELINE-NEXT:    xorl %r14d, %ebx
+; CHECK-BASELINE-NEXT:    movzwl 12(%r9), %r14d
+; CHECK-BASELINE-NEXT:    xorw %r12w, %r14w
+; CHECK-BASELINE-NEXT:    andw 12(%r10), %r14w
+; CHECK-BASELINE-NEXT:    xorl %r12d, %r14d
+; CHECK-BASELINE-NEXT:    movzwl 14(%r9), %r12d
+; CHECK-BASELINE-NEXT:    xorw %r13w, %r12w
+; CHECK-BASELINE-NEXT:    andw 14(%r10), %r12w
+; CHECK-BASELINE-NEXT:    xorl %r13d, %r12d
+; CHECK-BASELINE-NEXT:    movzwl 16(%r9), %r13d
+; CHECK-BASELINE-NEXT:    xorw %r15w, %r13w
+; CHECK-BASELINE-NEXT:    andw 16(%r10), %r13w
+; CHECK-BASELINE-NEXT:    xorl %r15d, %r13d
+; CHECK-BASELINE-NEXT:    movzwl 18(%r9), %r15d
+; CHECK-BASELINE-NEXT:    xorw %bp, %r15w
+; CHECK-BASELINE-NEXT:    andw 18(%r10), %r15w
+; CHECK-BASELINE-NEXT:    xorl %ebp, %r15d
+; CHECK-BASELINE-NEXT:    movl 20(%r8), %eax
+; CHECK-BASELINE-NEXT:    movzwl 20(%r9), %ebp
+; CHECK-BASELINE-NEXT:    xorw %ax, %bp
+; CHECK-BASELINE-NEXT:    andw 20(%r10), %bp
 ; CHECK-BASELINE-NEXT:    xorl %eax, %ebp
-; CHECK-BASELINE-NEXT:    movzwl 16(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r14w, %ax
-; CHECK-BASELINE-NEXT:    andw 16(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r14d
-; CHECK-BASELINE-NEXT:    movzwl 18(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r15w, %ax
-; CHECK-BASELINE-NEXT:    andw 18(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r15d
-; CHECK-BASELINE-NEXT:    movzwl 20(%rdx), %r13d
-; CHECK-BASELINE-NEXT:    movzwl 20(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r13w, %ax
-; CHECK-BASELINE-NEXT:    andw 20(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r13d
-; CHECK-BASELINE-NEXT:    movzwl 22(%rdx), %r9d
-; CHECK-BASELINE-NEXT:    movzwl 22(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r9w, %ax
-; CHECK-BASELINE-NEXT:    andw 22(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r9d
-; CHECK-BASELINE-NEXT:    movzwl 24(%rdx), %r8d
-; CHECK-BASELINE-NEXT:    movzwl 24(%rsi), %eax
-; CHECK-BASELINE-NEXT:    xorw %r8w, %ax
-; CHECK-BASELINE-NEXT:    andw 24(%rcx), %ax
-; CHECK-BASELINE-NEXT:    xorl %eax, %r8d
-; CHECK-BASELINE-NEXT:    movzwl 26(%rdx), %eax
-; CHECK-BASELINE-NEXT:    movzwl 26(%rsi), %r10d
-; CHECK-BASELINE-NEXT:    xorw %ax, %r10w
-; CHECK-BASELINE-NEXT:    andw 26(%rcx), %r10w
-; CHECK-BASELINE-NEXT:    xorl %r10d, %eax
-; CHECK-BASELINE-NEXT:    movzwl 28(%rdx), %r10d
-; CHECK-BASELINE-NEXT:    movzwl 28(%rsi), %r11d
-; CHECK-BASELINE-NEXT:    xorw %r10w, %r11w
-; CHECK-BASELINE-NEXT:    andw 28(%rcx), %r11w
-; CHECK-BASELINE-NEXT:    xorl %r11d, %r10d
-; CHECK-BASELINE-NEXT:    movzwl 30(%rdx), %edx
-; CHECK-BASELINE-NEXT:    movzwl 30(%rsi), %esi
-; CHECK-BASELINE-NEXT:    xorw %dx, %si
-; CHECK-BASELINE-NEXT:    andw 30(%rcx), %si
-; CHECK-BASELINE-NEXT:    xorl %esi, %edx
-; CHECK-BASELINE-NEXT:    movw %dx, 30(%rdi)
-; CHECK-BASELINE-NEXT:    movw %r10w, 28(%rdi)
-; CHECK-BASELINE-NEXT:    movw %ax, 26(%rdi)
-; CHECK-BASELINE-NEXT:    movw %r8w, 24(%rdi)
-; CHECK-BASELINE-NEXT:    movw %r9w, 22(%rdi)
-; CHECK-BASELINE-NEXT:    movw %r13w, 20(%rdi)
-; CHECK-BASELINE-NEXT:    movw %r15w, 18(%rdi)
-; CHECK-BASELINE-NEXT:    movw %r14w, 16(%rdi)
-; CHECK-BASELINE-NEXT:    movw %bp, 14(%rdi)
-; CHECK-BASELINE-NEXT:    movw %bx, 12(%rdi)
+; CHECK-BASELINE-NEXT:    movzwl 22(%r8), %eax
+; CHECK-BASELINE-NEXT:    movzwl 22(%r9), %esi
+; CHECK-BASELINE-NEXT:    xorw %ax, %si
+; CHECK-BASELINE-NEXT:    andw 22(%r10), %si
+; CHECK-BASELINE-NEXT:    xorl %eax, %esi
+; CHECK-BASELINE-NEXT:    movl 24(%r8), %eax
+; CHECK-BASELINE-NEXT:    movzwl 24(%r9), %edx
+; CHECK-BASELINE-NEXT:    xorw %ax, %dx
+; CHECK-BASELINE-NEXT:    andw 24(%r10), %dx
+; CHECK-BASELINE-NEXT:    xorl %eax, %edx
+; CHECK-BASELINE-NEXT:    movzwl 26(%r8), %eax
+; CHECK-BASELINE-NEXT:    movzwl 26(%r9), %ecx
+; CHECK-BASELINE-NEXT:    xorw %ax, %cx
+; CHECK-BASELINE-NEXT:    andw 26(%r10), %cx
+; CHECK-BASELINE-NEXT:    xorl %eax, %ecx
+; CHECK-BASELINE-NEXT:    movl 28(%r8), %edi
+; CHECK-BASELINE-NEXT:    movzwl 28(%r9), %eax
+; CHECK-BASELINE-NEXT:    xorw %di, %ax
+; CHECK-BASELINE-NEXT:    andw 28(%r10), %ax
+; CHECK-BASELINE-NEXT:    xorl %edi, %eax
+; CHECK-BASELINE-NEXT:    movzwl 30(%r8), %edi
+; CHECK-BASELINE-NEXT:    movzwl 30(%r9), %r8d
+; CHECK-BASELINE-NEXT:    xorw %di, %r8w
+; CHECK-BASELINE-NEXT:    andw 30(%r10), %r8w
+; CHECK-BASELINE-NEXT:    xorl %edi, %r8d
+; CHECK-BASELINE-NEXT:    movw %r8w, 30(%r11)
+; CHECK-BASELINE-NEXT:    movw %ax, 28(%r11)
+; CHECK-BASELINE-NEXT:    movw %cx, 26(%r11)
+; CHECK-BASELINE-NEXT:    movw %dx, 24(%r11)
+; CHECK-BASELINE-NEXT:    movw %si, 22(%r11)
+; CHECK-BASELINE-NEXT:    movw %bp, 20(%r11)
+; CHECK-BASELINE-NEXT:    movw %r15w, 18(%r11)
+; CHECK-BASELINE-NEXT:    movw %r13w, 16(%r11)
+; CHECK-BASELINE-NEXT:    movw %r12w, 14(%r11)
+; CHECK-BASELINE-NEXT:    movw %r14w, 12(%r11)
+; CHECK-BASELINE-NEXT:    movw %bx, 10(%r11)
 ; CHECK-BASELINE-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-BASELINE-NEXT:    movw %ax, 10(%rdi)
+; CHECK-BASELINE-NEXT:    movw %ax, 8(%r11)
 ; CHECK-BASELINE-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-BASELINE-NEXT:    movw %ax, 8(%rdi)
+; CHECK-BASELINE-NEXT:    movw %ax, 6(%r11)
 ; CHECK-BASELINE-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-BASELINE-NEXT:    movw %ax, 6(%rdi)
+; CHECK-BASELINE-NEXT:    movw %ax, 4(%r11)
 ; CHECK-BASELINE-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-BASELINE-NEXT:    movw %ax, 4(%rdi)
-; CHECK-BASELINE-NEXT:    movw %r12w, 2(%rdi)
+; CHECK-BASELINE-NEXT:    movw %ax, 2(%r11)
 ; CHECK-BASELINE-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-BASELINE-NEXT:    movw %ax, (%rdi)
-; CHECK-BASELINE-NEXT:    movq %rdi, %rax
+; CHECK-BASELINE-NEXT:    movw %ax, (%r11)
+; CHECK-BASELINE-NEXT:    movq %r11, %rax
 ; CHECK-BASELINE-NEXT:    popq %rbx
 ; CHECK-BASELINE-NEXT:    popq %r12
 ; CHECK-BASELINE-NEXT:    popq %r13
@@ -1882,113 +1876,117 @@ define <16 x i16> @out_v16i16(ptr%px, ptr%py, ptr%pmask) nounwind {
 ; CHECK-SSE1-NEXT:    pushq %r13
 ; CHECK-SSE1-NEXT:    pushq %r12
 ; CHECK-SSE1-NEXT:    pushq %rbx
-; CHECK-SSE1-NEXT:    movzwl 18(%rdx), %r15d
-; CHECK-SSE1-NEXT:    movzwl 16(%rdx), %r14d
-; CHECK-SSE1-NEXT:    movzwl 14(%rdx), %ebp
-; CHECK-SSE1-NEXT:    movzwl 12(%rdx), %ebx
-; CHECK-SSE1-NEXT:    movzwl 10(%rdx), %r13d
-; CHECK-SSE1-NEXT:    movzwl 8(%rdx), %r11d
-; CHECK-SSE1-NEXT:    movzwl 6(%rdx), %r10d
-; CHECK-SSE1-NEXT:    movzwl 4(%rdx), %r9d
-; CHECK-SSE1-NEXT:    movzwl (%rdx), %r8d
-; CHECK-SSE1-NEXT:    movzwl 2(%rdx), %r12d
-; CHECK-SSE1-NEXT:    movzwl (%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r8w, %ax
-; CHECK-SSE1-NEXT:    andw (%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r8d
-; CHECK-SSE1-NEXT:    movl %r8d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE1-NEXT:    movzwl 2(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r12w, %ax
-; CHECK-SSE1-NEXT:    andw 2(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r12d
-; CHECK-SSE1-NEXT:    movzwl 4(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r9w, %ax
-; CHECK-SSE1-NEXT:    andw 4(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r9d
-; CHECK-SSE1-NEXT:    movl %r9d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE1-NEXT:    movzwl 6(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r10w, %ax
-; CHECK-SSE1-NEXT:    andw 6(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r10d
-; CHECK-SSE1-NEXT:    movl %r10d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE1-NEXT:    movzwl 8(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r11w, %ax
-; CHECK-SSE1-NEXT:    andw 8(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r11d
-; CHECK-SSE1-NEXT:    movl %r11d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE1-NEXT:    movzwl 10(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r13w, %ax
-; CHECK-SSE1-NEXT:    andw 10(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r13d
-; CHECK-SSE1-NEXT:    movl %r13d, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE1-NEXT:    movzwl 12(%rsi), %eax
+; CHECK-SSE1-NEXT:    movq %rcx, %r10
+; CHECK-SSE1-NEXT:    movq %rdx, %r8
+; CHECK-SSE1-NEXT:    movq %rsi, %r9
+; CHECK-SSE1-NEXT:    movq %rdi, %r11
+; CHECK-SSE1-NEXT:    movzwl 18(%rdx), %ebp
+; CHECK-SSE1-NEXT:    movl 16(%rdx), %r15d
+; CHECK-SSE1-NEXT:    movzwl 14(%rdx), %r13d
+; CHECK-SSE1-NEXT:    movl 12(%rdx), %r12d
+; CHECK-SSE1-NEXT:    movzwl 10(%rdx), %r14d
+; CHECK-SSE1-NEXT:    movl 8(%rdx), %ebx
+; CHECK-SSE1-NEXT:    movzwl 6(%rdx), %eax
+; CHECK-SSE1-NEXT:    movl (%rdx), %ecx
+; CHECK-SSE1-NEXT:    movl 4(%rdx), %edx
+; CHECK-SSE1-NEXT:    movzwl 2(%r8), %esi
+; CHECK-SSE1-NEXT:    movzwl (%r9), %edi
+; CHECK-SSE1-NEXT:    xorw %cx, %di
+; CHECK-SSE1-NEXT:    andw (%r10), %di
+; CHECK-SSE1-NEXT:    xorl %ecx, %edi
+; CHECK-SSE1-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE1-NEXT:    movzwl 2(%r9), %ecx
+; CHECK-SSE1-NEXT:    xorw %si, %cx
+; CHECK-SSE1-NEXT:    andw 2(%r10), %cx
+; CHECK-SSE1-NEXT:    xorl %esi, %ecx
+; CHECK-SSE1-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE1-NEXT:    movzwl 4(%r9), %ecx
+; CHECK-SSE1-NEXT:    xorw %dx, %cx
+; CHECK-SSE1-NEXT:    andw 4(%r10), %cx
+; CHECK-SSE1-NEXT:    xorl %edx, %ecx
+; CHECK-SSE1-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE1-NEXT:    movzwl 6(%r9), %ecx
+; CHECK-SSE1-NEXT:    xorw %ax, %cx
+; CHECK-SSE1-NEXT:    andw 6(%r10), %cx
+; CHECK-SSE1-NEXT:    xorl %eax, %ecx
+; CHECK-SSE1-NEXT:    movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE1-NEXT:    movzwl 8(%r9), %eax
 ; CHECK-SSE1-NEXT:    xorw %bx, %ax
-; CHECK-SSE1-NEXT:    andw 12(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %ebx
-; CHECK-SSE1-NEXT:    movzwl 14(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %bp, %ax
-; CHECK-SSE1-NEXT:    andw 14(%rcx), %ax
+; CHECK-SSE1-NEXT:    andw 8(%r10), %ax
+; CHECK-SSE1-NEXT:    xorl %ebx, %eax
+; CHECK-SSE1-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE1-NEXT:    movzwl 10(%r9), %ebx
+; CHECK-SSE1-NEXT:    xorw %r14w, %bx
+; CHECK-SSE1-NEXT:    andw 10(%r10), %bx
+; CHECK-SSE1-NEXT:    xorl %r14d, %ebx
+; CHECK-SSE1-NEXT:    movzwl 12(%r9), %r14d
+; CHECK-SSE1-NEXT:    xorw %r12w, %r14w
+; CHECK-SSE1-NEXT:    andw 12(%r10), %r14w
+; CHECK-SSE1-NEXT:    xorl %r12d, %r14d
+; CHECK-SSE1-NEXT:    movzwl 14(%r9), %r12d
+; CHECK-SSE1-NEXT:    xorw %r13w, %r12w
+; CHECK-SSE1-NEXT:    andw 14(%r10), %r12w
+; CHECK-SSE1-NEXT:    xorl %r13d, %r12d
+; CHECK-SSE1-NEXT:    movzwl 16(%r9), %r13d
+; CHECK-SSE1-NEXT:    xorw %r15w, %r13w
+; CHECK-SSE1-NEXT:    andw 16(%r10), %r13w
+; CHECK-SSE1-NEXT:    xorl %r15d, %r13d
+; CHECK-SSE1-NEXT:    movzwl 18(%r9), %r15d
+; CHECK-SSE1-NEXT:    xorw %bp, %r15w
+; CHECK-SSE1-NEXT:    andw 18(%r10), %r15w
+; CHECK-SSE1-NEXT:    xorl %ebp, %r15d
+; CHECK-SSE1-NEXT:    movl 20(%r8), %eax
+; CHECK-SSE1-NEXT:    movzwl 20(%r9), %ebp
+; CHECK-SSE1-NEXT:    xorw %ax, %bp
+; CHECK-SSE1-NEXT:    andw 20(%r10), %bp
 ; CHECK-SSE1-NEXT:    xorl %eax, %ebp
-; CHECK-SSE1-NEXT:    movzwl 16(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r14w, %ax
-; CHECK-SSE1-NEXT:    andw 16(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r14d
-; CHECK-SSE1-NEXT:    movzwl 18(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r15w, %ax
-; CHECK-SSE1-NEXT:    andw 18(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r15d
-; CHECK-SSE1-NEXT:    movzwl 20(%rdx), %r13d
-; CHECK-SSE1-NEXT:    movzwl 20(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r13w, %ax
-; CHECK-SSE1-NEXT:    andw 20(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r13d
-; CHECK-SSE1-NEXT:    movzwl 22(%rdx), %r9d
-; CHECK-SSE1-NEXT:    movzwl 22(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r9w, %ax
-; CHECK-SSE1-NEXT:    andw 22(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r9d
-; CHECK-SSE1-NEXT:    movzwl 24(%rdx), %r8d
-; CHECK-SSE1-NEXT:    movzwl 24(%rsi), %eax
-; CHECK-SSE1-NEXT:    xorw %r8w, %ax
-; CHECK-SSE1-NEXT:    andw 24(%rcx), %ax
-; CHECK-SSE1-NEXT:    xorl %eax, %r8d
-; CHECK-SSE1-NEXT:    movzwl 26(%rdx), %eax
-; CHECK-SSE1-NEXT:    movzwl 26(%rsi), %r10d
-; CHECK-SSE1-NEXT:    xorw %ax, %r10w
-; CHECK-SSE1-NEXT:    andw 26(%rcx), %r10w
-; CHECK-SSE1-NEXT:    xorl %r10d, %eax
-; CHECK-SSE1-NEXT:    movzwl 28(%rdx), %r10d
-; CHECK-SSE1-NEXT:    movzwl 28(%rsi), %r11d
-; CHECK-SSE1-NEXT:    xorw %r10w, %r11w
-; CHECK-SSE1-NEXT:    andw 28(%rcx), %r11w
-; CHECK-SSE1-NEXT:    xorl %r11d, %r10d
-; CHECK-SSE1-NEXT:    movzwl 30(%rdx), %edx
-; CHECK-SSE1-NEXT:    movzwl 30(%rsi), %esi
-; CHECK-SSE1-NEXT:    xorw %dx, %si
-; CHECK-SSE1-NEXT:    andw 30(%rcx), %si
-; CHECK-SSE1-NEXT:    xorl %esi, %edx
-; CHECK-SSE1-NEXT:    movw %dx, 30(%rdi)
-; CHECK-SSE1-NEXT:    movw %r10w, 28(%rdi)
-; CHECK-SSE1-NEXT:    movw %ax, 26(%rdi)
-; CHECK-SSE1-NEXT:    movw %r8w, 24(%rdi)
-; CHECK-SSE1-NEXT:    movw %r9w, 22(%rdi)
-; CHECK-SSE1-NEXT:    movw %r13w, 20(%rdi)
-; CHECK-SSE1-NEXT:    movw %r15w, 18(%rdi)
-; CHECK-SSE1-NEXT:    movw %r14w, 16(%rdi)
-; CHECK-SSE1-NEXT:    movw %bp, 14(%rdi)
-; CHECK-SSE1-NEXT:    movw %bx, 12(%rdi)
+; CHECK-SSE1-NEXT:    movzwl 22(%r8), %eax
+; CHECK-SSE1-NEXT:    movzwl 22(%r9), %esi
+; CHECK-SSE1-NEXT:    xorw %ax, %si
+; CHECK-SSE1-NEXT:    andw 22(%r10), %si
+; CHECK-SSE1-NEXT:    xorl %eax, %esi
+; CHECK-SSE1-NEXT:    movl 24(%r8), %eax
+; CHECK-SSE1-NEXT:    movzwl 24(%r9), %edx
+; CHECK-SSE1-NEXT:    xorw %ax, %dx
+; CHECK-SSE1-NEXT:    andw 24(%r10), %dx
+; CHECK-SSE1-NEXT:    xorl %eax, %edx
+; CHECK-SSE1-NEXT:    movzwl 26(%r8), %eax
+; CHECK-SSE1-NEXT:    movzwl 26(%r9), %ecx
+; CHECK-SSE1-NEXT:    xorw %ax, %cx
+; CHECK-SSE1-NEXT:    andw 26(%r10), %cx
+; CHECK-SSE1-NEXT:    xorl %eax, %ecx
+; CHECK-SSE1-NEXT:    movl 28(%r8), %edi
+; CHECK-SSE1-NEXT:    movzwl 28(%r9), %eax
+; CHECK-SSE1-NEXT:    xorw %di, %ax
+; CHECK-SSE1-NEXT:    andw 28(%r10), %ax
+; CHECK-SSE1-NEXT:    xorl %edi, %eax
+; CHECK-SSE1-NEXT:    movzwl 30(%r8), %edi
+; CHECK-SSE1-NEXT:    movzwl 30(%r9), %r8d
+; CHECK-SSE1-NEXT:    xorw %di, %r8w
+; CHECK-SSE1-NEXT:    andw 30(%r10), %r8w
+; CHECK-SSE1-NEXT:    xorl %edi, %r8d
+; CHECK-SSE1-NEXT:    movw %r8w, 30(%r11)
+; CHECK-SSE1-NEXT:    movw %ax, 28(%r11)
+; CHECK-SSE1-NEXT:    movw %cx, 26(%r11)
+; CHECK-SSE1-NEXT:    movw %dx, 24(%r11)
+; CHECK-SSE1-NEXT:    movw %si, 22(%r11)
+; CHECK-SSE1-NEXT:    movw %bp, 20(%r11)
+; CHECK-SSE1-NEXT:    movw %r15w, 18(%r11)
+; CHECK-SSE1-NEXT:    movw %r13w, 16(%r11)
+; CHECK-SSE1-NEXT:    movw %r12w, 14(%r11)
+; CHECK-SSE1-NEXT:    movw %r14w, 12(%r11)
+; CHECK-SSE1-NEXT:    movw %bx, 10(%r11)
 ; CHECK-SSE1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-SSE1-NEXT:    movw %ax, 10(%rdi)
+; CHECK-SSE1-NEXT:    movw %ax, 8(%r11)
 ; CHECK-SSE1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-SSE1-NEXT:    movw %ax, 8(%rdi)
+; CHECK-SSE1-NEXT:    movw %ax, 6(%r11)
 ; CHECK-SSE1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-SSE1-NEXT:    movw %ax, 6(%rdi)
+; CHECK-SSE1-NEXT:    movw %ax, 4(%r11)
 ; CHECK-SSE1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-SSE1-NEXT:    movw %ax, 4(%rdi)
-; CHECK-SSE1-NEXT:    movw %r12w, 2(%rdi)
+; CHECK-SSE1-NEXT:    movw %ax, 2(%r11)
 ; CHECK-SSE1-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-SSE1-NEXT:    movw %ax, (%rdi)
-; CHECK-SSE1-NEXT:    movq %rdi, %rax
+; CHECK-SSE1-NEXT:    movw %ax, (%r11)
+; CHECK-SSE1-NEXT:    movq %r11, %rax
 ; CHECK-SSE1-NEXT:    popq %rbx
 ; CHECK-SSE1-NEXT:    popq %r12
 ; CHECK-SSE1-NEXT:    popq %r13
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/usub_sat_vec.ll b/external/llvm-project/llvm/test/CodeGen/X86/usub_sat_vec.ll
index 4e17ca6fbae3..a5f768e48bae 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/usub_sat_vec.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/usub_sat_vec.ll
@@ -543,7 +543,7 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ;
 ; AVX512BW-LABEL: v16i1:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 ^ mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 ^ m32bcst)
 ; AVX512BW-NEXT:    retq
   %z = call <16 x i1> @llvm.usub.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
   ret <16 x i1> %z
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-128-fp16.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-128-fp16.ll
index 2c3d7ceb37d0..35688e59fc9f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-128-fp16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-128-fp16.ll
@@ -93,7 +93,7 @@ define <2 x double> @f12(<2 x double> %a0, <8 x half> %a1) #0 {
 ; CHECK-LABEL: f12:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtsh2sd %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %ext = extractelement <8 x half> %a1, i32 0
   %cvt = call double @llvm.experimental.constrained.fpext.f64.f16(half %ext,
@@ -154,7 +154,7 @@ define <4 x float> @f18(<4 x float> %a0, <8 x half> %a1) #0 {
 ; CHECK-LABEL: f18:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtsh2ss %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %ext = extractelement <8 x half> %a1, i32 0
   %cvt = call float @llvm.experimental.constrained.fpext.f32.f16(half %ext,
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
index 0a9dd78afb8c..bde14e75dfc0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512fp16,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 -O3 | FileCheck %s --check-prefixes=NOVL
 
 declare <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f16(<2 x half>, metadata)
 declare <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f16(<2 x half>, metadata)
@@ -31,9 +32,19 @@ define <2 x i64> @strict_vector_fptosi_v2f16_to_v2i64(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptosi_v2f16_to_v2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2qq %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i64:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttsh2si %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm1
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm0
+; NOVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; NOVL-NEXT:    retq
   %ret = call <2 x i64> @llvm.experimental.constrained.fptosi.v2i64.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i64> %ret
@@ -43,9 +54,19 @@ define <2 x i64> @strict_vector_fptoui_v2f16_to_v2i64(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptoui_v2f16_to_v2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2uqq %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i64:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm1
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm0
+; NOVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; NOVL-NEXT:    retq
   %ret = call <2 x i64> @llvm.experimental.constrained.fptoui.v2i64.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i64> %ret
@@ -55,9 +76,20 @@ define <2 x i32> @strict_vector_fptosi_v2f16_to_v2i32(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptosi_v2f16_to_v2i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2dq %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i32:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i32> %ret
@@ -67,9 +99,20 @@ define <2 x i32> @strict_vector_fptoui_v2f16_to_v2i32(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptoui_v2f16_to_v2i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2udq %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i32:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOVL-NEXT:    vcvttph2udq %ymm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i32> %ret
@@ -79,9 +122,20 @@ define <2 x i16> @strict_vector_fptosi_v2f16_to_v2i16(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptosi_v2f16_to_v2i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2w %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i16> @llvm.experimental.constrained.fptosi.v2i16.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i16> %ret
@@ -91,9 +145,20 @@ define <2 x i16> @strict_vector_fptoui_v2f16_to_v2i16(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptoui_v2f16_to_v2i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2uw %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i16> @llvm.experimental.constrained.fptoui.v2i16.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i16> %ret
@@ -103,10 +168,21 @@ define <2 x i8> @strict_vector_fptosi_v2f16_to_v2i8(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptosi_v2f16_to_v2i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2w %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovwb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i8> %ret
@@ -116,10 +192,21 @@ define <2 x i8> @strict_vector_fptoui_v2f16_to_v2i8(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptoui_v2f16_to_v2i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2uw %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovwb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
+; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i8> %ret
@@ -129,13 +216,28 @@ define <2 x i1> @strict_vector_fptosi_v2f16_to_v2i1(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptosi_v2f16_to_v2i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2w %xmm0, %xmm0
 ; CHECK-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovw2m %xmm0, %k1
 ; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    andl $1, %eax
+; NOVL-NEXT:    kmovw %eax, %k0
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $1, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k1
+; NOVL-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i1> @llvm.experimental.constrained.fptosi.v2i1.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i1> %ret
@@ -145,13 +247,28 @@ define <2 x i1> @strict_vector_fptoui_v2f16_to_v2i1(<2 x half> %a) #0 {
 ; CHECK-LABEL: strict_vector_fptoui_v2f16_to_v2i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    vcvttph2uw %xmm0, %xmm0
 ; CHECK-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovw2m %xmm0, %k1
 ; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    andl $1, %eax
+; NOVL-NEXT:    kmovw %eax, %k0
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $1, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k1
+; NOVL-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <2 x i1> @llvm.experimental.constrained.fptoui.v2i1.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <2 x i1> %ret
@@ -163,6 +280,21 @@ define <4 x i32> @strict_vector_fptosi_v4f16_to_v4i32(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vcvttph2dq %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i32:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    retq
   %ret = call <4 x i32> @llvm.experimental.constrained.fptosi.v4i32.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i32> %ret
@@ -174,6 +306,21 @@ define <4 x i32> @strict_vector_fptoui_v4f16_to_v4i32(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vcvttph2udq %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i32:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2usi %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2usi %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    retq
   %ret = call <4 x i32> @llvm.experimental.constrained.fptoui.v4i32.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i32> %ret
@@ -185,6 +332,16 @@ define <4 x i16> @strict_vector_fptosi_v4f16_to_v4i16(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vcvttph2w %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <4 x i16> @llvm.experimental.constrained.fptosi.v4i16.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i16> %ret
@@ -196,6 +353,16 @@ define <4 x i16> @strict_vector_fptoui_v4f16_to_v4i16(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vcvttph2uw %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <4 x i16> @llvm.experimental.constrained.fptoui.v4i16.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i16> %ret
@@ -208,6 +375,16 @@ define <4 x i8> @strict_vector_fptosi_v4f16_to_v4i8(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vcvttph2w %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovwb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i8> %ret
@@ -220,6 +397,16 @@ define <4 x i8> @strict_vector_fptoui_v4f16_to_v4i8(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vcvttph2uw %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovwb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
+; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i8> %ret
@@ -235,6 +422,37 @@ define <4 x i1> @strict_vector_fptosi_v4f16_to_v4i1(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    andl $1, %eax
+; NOVL-NEXT:    kmovw %eax, %k0
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $15, %k1, %k1
+; NOVL-NEXT:    kshiftrw $14, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k0
+; NOVL-NEXT:    movw $-5, %ax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kandw %k1, %k0, %k0
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $2, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k0
+; NOVL-NEXT:    kshiftlw $13, %k0, %k0
+; NOVL-NEXT:    kshiftrw $13, %k0, %k0
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $3, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k1
+; NOVL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <4 x i1> @llvm.experimental.constrained.fptosi.v4i1.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i1> %ret
@@ -250,6 +468,37 @@ define <4 x i1> @strict_vector_fptoui_v4f16_to_v4i1(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    andl $1, %eax
+; NOVL-NEXT:    kmovw %eax, %k0
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $15, %k1, %k1
+; NOVL-NEXT:    kshiftrw $14, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k0
+; NOVL-NEXT:    movw $-5, %ax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kandw %k1, %k0, %k0
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $2, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k0
+; NOVL-NEXT:    kshiftlw $13, %k0, %k0
+; NOVL-NEXT:    kshiftrw $13, %k0, %k0
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    kmovd %eax, %k1
+; NOVL-NEXT:    kshiftlw $3, %k1, %k1
+; NOVL-NEXT:    korw %k1, %k0, %k1
+; NOVL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <4 x i1> @llvm.experimental.constrained.fptoui.v4i1.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i1> %ret
@@ -260,6 +509,15 @@ define <8 x i16> @strict_vector_fptosi_v8f16_to_v8i16(<8 x half> %a) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttph2w %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v8f16_to_v8i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <8 x i16> @llvm.experimental.constrained.fptosi.v8i16.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i16> %ret
@@ -270,6 +528,15 @@ define <8 x i16> @strict_vector_fptoui_v8f16_to_v8i16(<8 x half> %a) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttph2uw %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v8f16_to_v8i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <8 x i16> @llvm.experimental.constrained.fptoui.v8i16.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i16> %ret
@@ -281,6 +548,15 @@ define <8 x i8> @strict_vector_fptosi_v8f16_to_v8i8(<8 x half> %a) #0 {
 ; CHECK-NEXT:    vcvttph2w %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovwb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v8f16_to_v8i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <8 x i8> @llvm.experimental.constrained.fptosi.v8i8.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i8> %ret
@@ -292,6 +568,15 @@ define <8 x i8> @strict_vector_fptoui_v8f16_to_v8i8(<8 x half> %a) #0 {
 ; CHECK-NEXT:    vcvttph2uw %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovwb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v8f16_to_v8i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
+; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <8 x i8> @llvm.experimental.constrained.fptoui.v8i8.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i8> %ret
@@ -305,6 +590,18 @@ define <8 x i1> @strict_vector_fptosi_v8f16_to_v8i1(<8 x half> %a) #0 {
 ; CHECK-NEXT:    vpmovm2w %k0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v8f16_to_v8i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; NOVL-NEXT:    vpmovm2w %k0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <8 x i1> @llvm.experimental.constrained.fptosi.v8i1.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i1> %ret
@@ -319,6 +616,19 @@ define <8 x i1> @strict_vector_fptoui_v8f16_to_v8i1(<8 x half> %a) #0 {
 ; CHECK-NEXT:    vpmovm2w %k0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v8f16_to_v8i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    vpslld $31, %ymm0, %ymm0
+; NOVL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; NOVL-NEXT:    vpmovm2w %k0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <8 x i1> @llvm.experimental.constrained.fptoui.v8i1.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i1> %ret
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll
index 7bdb6a45bebc..a232122e9c70 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-fptoint-256-fp16.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=avx512fp16,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16,avx512vl -O3 | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 -O3 | FileCheck %s --check-prefixes=NOVL
 
 
 declare <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f16(<4 x half>, metadata)
@@ -20,6 +21,24 @@ define <4 x i64> @strict_vector_fptosi_v4f16_to_v4i64(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vcvttph2qq %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i64:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm2
+; NOVL-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; NOVL-NEXT:    vcvttsh2si %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm2
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm0
+; NOVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; NOVL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; NOVL-NEXT:    retq
   %ret = call <4 x i64> @llvm.experimental.constrained.fptosi.v4i64.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i64> %ret
@@ -31,6 +50,24 @@ define <4 x i64> @strict_vector_fptoui_v4f16_to_v4i64(<4 x half> %a) #0 {
 ; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; CHECK-NEXT:    vcvttph2uqq %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i64:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2usi %xmm1, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2usi %xmm2, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm2
+; NOVL-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm2
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %rax
+; NOVL-NEXT:    vmovq %rax, %xmm0
+; NOVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; NOVL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; NOVL-NEXT:    retq
   %ret = call <4 x i64> @llvm.experimental.constrained.fptoui.v4i64.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <4 x i64> %ret
@@ -41,6 +78,15 @@ define <8 x i32> @strict_vector_fptosi_v8f16_to_v8i32(<8 x half> %a) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttph2dq %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v8f16_to_v8i32:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; NOVL-NEXT:    retq
   %ret = call <8 x i32> @llvm.experimental.constrained.fptosi.v8i32.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i32> %ret
@@ -51,6 +97,15 @@ define <8 x i32> @strict_vector_fptoui_v8f16_to_v8i32(<8 x half> %a) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttph2udq %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v8f16_to_v8i32:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; NOVL-NEXT:    vcvttph2udq %ymm0, %zmm0
+; NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; NOVL-NEXT:    retq
   %ret = call <8 x i32> @llvm.experimental.constrained.fptoui.v8i32.v8f16(<8 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <8 x i32> %ret
@@ -61,6 +116,14 @@ define <16 x i16> @strict_vector_fptosi_v16f16_to_v16i16(<16 x half> %a) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttph2w %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v16f16_to_v16i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf64x4 $0, %ymm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; NOVL-NEXT:    retq
   %ret = call <16 x i16> @llvm.experimental.constrained.fptosi.v16i16.v16f16(<16 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <16 x i16> %ret
@@ -71,6 +134,14 @@ define <16 x i16> @strict_vector_fptoui_v16f16_to_v16i16(<16 x half> %a) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttph2uw %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v16f16_to_v16i16:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; NOVL-NEXT:    vinsertf64x4 $0, %ymm0, %zmm1, %zmm0
+; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
+; NOVL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; NOVL-NEXT:    retq
   %ret = call <16 x i16> @llvm.experimental.constrained.fptoui.v16i16.v16f16(<16 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <16 x i16> %ret
@@ -83,6 +154,13 @@ define <16 x i8> @strict_vector_fptosi_v16f16_to_v16i8(<16 x half> %a) #0 {
 ; CHECK-NEXT:    vpmovdb %zmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v16f16_to_v16i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    vpmovdb %zmm0, %xmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <16 x i8> @llvm.experimental.constrained.fptosi.v16i8.v16f16(<16 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <16 x i8> %ret
@@ -95,6 +173,13 @@ define <16 x i8> @strict_vector_fptoui_v16f16_to_v16i8(<16 x half> %a) #0 {
 ; CHECK-NEXT:    vpmovdb %zmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v16f16_to_v16i8:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    vpmovdb %zmm0, %xmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <16 x i8> @llvm.experimental.constrained.fptoui.v16i8.v16f16(<16 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <16 x i8> %ret
@@ -108,6 +193,15 @@ define <16 x i1> @strict_vector_fptosi_v16f16_to_v16i1(<16 x half> %a) #0 {
 ; CHECK-NEXT:    vpmovm2b %k0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptosi_v16f16_to_v16i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; NOVL-NEXT:    vpmovm2b %k0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <16 x i1> @llvm.experimental.constrained.fptosi.v16i1.v16f16(<16 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <16 x i1> %ret
@@ -122,6 +216,16 @@ define <16 x i1> @strict_vector_fptoui_v16f16_to_v16i1(<16 x half> %a) #0 {
 ; CHECK-NEXT:    vpmovm2b %k0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
+;
+; NOVL-LABEL: strict_vector_fptoui_v16f16_to_v16i1:
+; NOVL:       # %bb.0:
+; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
+; NOVL-NEXT:    vpslld $31, %zmm0, %zmm0
+; NOVL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; NOVL-NEXT:    vpmovm2b %k0, %zmm0
+; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    retq
   %ret = call <16 x i1> @llvm.experimental.constrained.fptoui.v16i1.v16f16(<16 x half> %a,
                                               metadata !"fpexcept.strict") #0
   ret <16 x i1> %ret
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
index 7c788d291a5c..cd4ceca6716b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
@@ -227,9 +227,9 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX-64-LABEL: sitofp_v2i64_v2f32:
 ; AVX-64:       # %bb.0:
 ; AVX-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX-64-NEXT:    vmovq %xmm0, %rax
-; AVX-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX-64-NEXT:    retq
 ;
@@ -246,9 +246,9 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX512DQ-64-LABEL: sitofp_v2i64_v2f32:
 ; AVX512DQ-64:       # %bb.0:
 ; AVX512DQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512DQ-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512DQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512DQ-64-NEXT:    vmovq %xmm0, %rax
-; AVX512DQ-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512DQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512DQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX512DQ-64-NEXT:    retq
 ;
@@ -439,9 +439,9 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX1-64-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; AVX1-64-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
 ; AVX1-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-64-NEXT:    vmovq %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
 ; AVX1-64-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; AVX1-64-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -453,18 +453,18 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX512F-64-LABEL: uitofp_v2i64_v2f32:
 ; AVX512F-64:       # %bb.0:
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX512F-64-NEXT:    retq
 ;
 ; AVX512VL-64-LABEL: uitofp_v2i64_v2f32:
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX512VL-64-NEXT:    retq
 ;
@@ -481,9 +481,9 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX512DQ-64-LABEL: uitofp_v2i64_v2f32:
 ; AVX512DQ-64:       # %bb.0:
 ; AVX512DQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512DQ-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512DQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512DQ-64-NEXT:    vmovq %xmm0, %rax
-; AVX512DQ-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512DQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512DQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX512DQ-64-NEXT:    retq
 ;
@@ -1237,9 +1237,9 @@ define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; AVX-64-LABEL: sitofp_v2i64_v2f64:
 ; AVX-64:       # %bb.0:
 ; AVX-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX-64-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX-64-NEXT:    vmovq %xmm0, %rax
-; AVX-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-64-NEXT:    retq
 ;
@@ -1439,7 +1439,7 @@ define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; AVX1-64-NEXT:    orq %rcx, %rdx
 ; AVX1-64-NEXT:    testq %rax, %rax
 ; AVX1-64-NEXT:    cmovnsq %rax, %rdx
-; AVX1-64-NEXT:    vcvtsi2sd %rdx, %xmm1, %xmm1
+; AVX1-64-NEXT:    vcvtsi2sd %rdx, %xmm15, %xmm1
 ; AVX1-64-NEXT:    jns .LBB21_2
 ; AVX1-64-NEXT:  # %bb.1:
 ; AVX1-64-NEXT:    vaddsd %xmm1, %xmm1, %xmm1
@@ -1452,7 +1452,7 @@ define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; AVX1-64-NEXT:    orq %rcx, %rdx
 ; AVX1-64-NEXT:    testq %rax, %rax
 ; AVX1-64-NEXT:    cmovnsq %rax, %rdx
-; AVX1-64-NEXT:    vcvtsi2sd %rdx, %xmm2, %xmm0
+; AVX1-64-NEXT:    vcvtsi2sd %rdx, %xmm15, %xmm0
 ; AVX1-64-NEXT:    jns .LBB21_4
 ; AVX1-64-NEXT:  # %bb.3:
 ; AVX1-64-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
@@ -1463,18 +1463,18 @@ define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; AVX512F-64-LABEL: uitofp_v2i64_v2f64:
 ; AVX512F-64:       # %bb.0:
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm0
+; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; AVX512F-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512F-64-NEXT:    retq
 ;
 ; AVX512VL-64-LABEL: uitofp_v2i64_v2f64:
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm0
+; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; AVX512VL-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512VL-64-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
index a336d0a01fa7..f790377f3331 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
@@ -670,14 +670,14 @@ define <4 x double> @sitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX1-64:       # %bb.0:
 ; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-64-NEXT:    vmovq %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX1-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-64-NEXT:    vmovq %xmm0, %rax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX1-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-64-NEXT:    retq
@@ -686,14 +686,14 @@ define <4 x double> @sitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX2-64:       # %bb.0:
 ; AVX2-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX2-64-NEXT:    vmovq %xmm1, %rax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX2-64-NEXT:    vmovq %xmm0, %rax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX2-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-64-NEXT:    retq
@@ -702,14 +702,14 @@ define <4 x double> @sitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX512F-64:       # %bb.0:
 ; AVX512F-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vmovq %xmm1, %rax
-; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512F-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512F-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512F-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512F-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-64-NEXT:    retq
@@ -718,14 +718,14 @@ define <4 x double> @sitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vmovq %xmm1, %rax
-; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512VL-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512VL-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512VL-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512VL-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512VL-64-NEXT:    retq
@@ -802,26 +802,26 @@ define <4 x double> @uitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX1-64:       # %bb.0:
 ; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-64-NEXT:    vpextrd $2, %xmm1, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-64-NEXT:    vmovd %xmm1, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm3[0],xmm2[0]
 ; AVX1-64-NEXT:    vextractps $2, %xmm0, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-64-NEXT:    vmovq %xmm0, %rax
 ; AVX1-64-NEXT:    movl %eax, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm4
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm4[0],xmm3[0]
 ; AVX1-64-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-64-NEXT:    vpextrd $3, %xmm1, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-64-NEXT:    vpextrd $1, %xmm1, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm1
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
 ; AVX1-64-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-64-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm0
+; AVX1-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; AVX1-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-64-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
@@ -832,28 +832,28 @@ define <4 x double> @uitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX2-64:       # %bb.0:
 ; AVX2-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX2-64-NEXT:    vextractps $3, %xmm1, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX2-64-NEXT:    vextractps $1, %xmm1, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm3[0],xmm2[0]
 ; AVX2-64-NEXT:    vextractps $3, %xmm0, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX2-64-NEXT:    vextractps $1, %xmm0, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm4
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm4[0],xmm3[0]
 ; AVX2-64-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX2-64-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [4.294967296E+9,4.294967296E+9,4.294967296E+9,4.294967296E+9]
 ; AVX2-64-NEXT:    vmulpd %ymm3, %ymm2, %ymm2
 ; AVX2-64-NEXT:    vextractps $2, %xmm1, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX2-64-NEXT:    vmovd %xmm1, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm1
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
 ; AVX2-64-NEXT:    vextractps $2, %xmm0, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX2-64-NEXT:    vmovq %xmm0, %rax
 ; AVX2-64-NEXT:    movl %eax, %eax
-; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm0
+; AVX2-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX2-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; AVX2-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-64-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
@@ -863,14 +863,14 @@ define <4 x double> @uitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX512F-64:       # %bb.0:
 ; AVX512F-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm2
+; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vmovq %xmm1, %rax
-; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm1
+; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; AVX512F-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm2
+; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm0
+; AVX512F-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; AVX512F-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512F-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-64-NEXT:    retq
@@ -879,14 +879,14 @@ define <4 x double> @uitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm2
+; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vmovq %xmm1, %rax
-; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm1
+; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; AVX512VL-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm2
+; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm0
+; AVX512VL-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; AVX512VL-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512VL-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512VL-64-NEXT:    retq
@@ -947,16 +947,16 @@ define <4 x float> @sitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX1-64-LABEL: sitofp_v4i64_v4f32:
 ; AVX1-64:       # %bb.0:
 ; AVX1-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-64-NEXT:    vmovq %xmm0, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX1-64-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-64-NEXT:    vmovq %xmm0, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX1-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX1-64-NEXT:    vzeroupper
 ; AVX1-64-NEXT:    retq
@@ -964,16 +964,16 @@ define <4 x float> @sitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX2-64-LABEL: sitofp_v4i64_v4f32:
 ; AVX2-64:       # %bb.0:
 ; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX2-64-NEXT:    vmovq %xmm0, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX2-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-64-NEXT:    vmovq %xmm0, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX2-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX2-64-NEXT:    vzeroupper
 ; AVX2-64-NEXT:    retq
@@ -981,16 +981,16 @@ define <4 x float> @sitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX512F-64-LABEL: sitofp_v4i64_v4f32:
 ; AVX512F-64:       # %bb.0:
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512F-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512F-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512F-64-NEXT:    vzeroupper
 ; AVX512F-64-NEXT:    retq
@@ -998,16 +998,16 @@ define <4 x float> @sitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX512VL-64-LABEL: sitofp_v4i64_v4f32:
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512VL-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512VL-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512VL-64-NEXT:    vzeroupper
 ; AVX512VL-64-NEXT:    retq
@@ -1092,16 +1092,16 @@ define <4 x float> @uitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX1-64-NEXT:    vorpd %ymm3, %ymm1, %ymm1
 ; AVX1-64-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX1-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX1-64-NEXT:    vmovq %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; AVX1-64-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-64-NEXT:    vmovq %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; AVX1-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; AVX1-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; AVX1-64-NEXT:    vaddps %xmm1, %xmm1, %xmm3
 ; AVX1-64-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
@@ -1117,16 +1117,16 @@ define <4 x float> @uitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX2-64-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-64-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX2-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-64-NEXT:    vmovq %xmm1, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; AVX2-64-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX2-64-NEXT:    vmovq %xmm1, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; AVX2-64-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm1
+; AVX2-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX2-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
 ; AVX2-64-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; AVX2-64-NEXT:    vextracti128 $1, %ymm0, %xmm3
@@ -1138,16 +1138,16 @@ define <4 x float> @uitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX512F-64-LABEL: uitofp_v4i64_v4f32:
 ; AVX512F-64:       # %bb.0:
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512F-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512F-64-NEXT:    vmovq %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512F-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512F-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512F-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512F-64-NEXT:    vzeroupper
 ; AVX512F-64-NEXT:    retq
@@ -1155,16 +1155,16 @@ define <4 x float> @uitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX512VL-64-LABEL: uitofp_v4i64_v4f32:
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512VL-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512VL-64-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512VL-64-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512VL-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512VL-64-NEXT:    vzeroupper
 ; AVX512VL-64-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
index 0cf945202a2d..59294dd17fbc 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
@@ -323,27 +323,27 @@ define <8 x double> @sitofp_v8i64_v8f64(<8 x i64> %x) #0 {
 ; NODQ-64:       # %bb.0:
 ; NODQ-64-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
 ; NODQ-64-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm1, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-64-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
 ; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vmovq %xmm2, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-64-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; NODQ-64-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vmovq %xmm2, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vmovq %xmm0, %rax
-; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm0
+; NODQ-64-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; NODQ-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-64-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
@@ -452,27 +452,27 @@ define <8 x double> @uitofp_v8i64_v8f64(<8 x i64> %x) #0 {
 ; NODQ-64:       # %bb.0:
 ; NODQ-64-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
 ; NODQ-64-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm2
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm1, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm1
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; NODQ-64-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
 ; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm3
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vmovq %xmm2, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-64-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; NODQ-64-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vmovq %xmm2, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vmovq %xmm0, %rax
-; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm4, %xmm0
+; NODQ-64-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; NODQ-64-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; NODQ-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; NODQ-64-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
@@ -548,28 +548,28 @@ define <8 x float> @sitofp_v8i64_v8f32(<8 x i64> %x) #0 {
 ; NODQ-64:       # %bb.0:
 ; NODQ-64-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
 ; NODQ-64-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm1, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
 ; NODQ-64-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm2, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm0, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; NODQ-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-64-NEXT:    vmovq %xmm0, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm0
+; NODQ-64-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
 ; NODQ-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; NODQ-64-NEXT:    retq
@@ -675,28 +675,28 @@ define <8 x float> @uitofp_v8i64_v8f32(<8 x i64> %x) #0 {
 ; NODQ-64:       # %bb.0:
 ; NODQ-64-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
 ; NODQ-64-NEXT:    vpextrq $1, %xmm1, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm1, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm1
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
 ; NODQ-64-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm2, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm3
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm2, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm2
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; NODQ-64-NEXT:    vmovq %xmm0, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; NODQ-64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; NODQ-64-NEXT:    vmovq %xmm0, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm3
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm3
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; NODQ-64-NEXT:    vpextrq $1, %xmm0, %rax
-; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm4, %xmm0
+; NODQ-64-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; NODQ-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
 ; NODQ-64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; NODQ-64-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec_extract-avx.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec_extract-avx.ll
index 341a703a21bd..4b70933334fb 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec_extract-avx.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec_extract-avx.ll
@@ -119,7 +119,7 @@ define void @legal_vzmovl_2i32_8i32(ptr %in, ptr %out) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X86-NEXT:    vmovaps %ymm0, (%eax)
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
@@ -128,7 +128,7 @@ define void @legal_vzmovl_2i32_8i32(ptr %in, ptr %out) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X64-NEXT:    vmovaps %ymm0, (%rsi)
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
@@ -169,7 +169,7 @@ define void @legal_vzmovl_2f32_8f32(ptr %in, ptr %out) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X86-NEXT:    vmovaps %ymm0, (%eax)
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
@@ -178,7 +178,7 @@ define void @legal_vzmovl_2f32_8f32(ptr %in, ptr %out) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X64-NEXT:    vmovaps %ymm0, (%rsi)
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec_floor.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec_floor.ll
index abb85ac83464..7f4ed3394d10 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec_floor.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec_floor.ll
@@ -828,13 +828,13 @@ define <4 x float> @floor_ss(<4 x float> %x, <4 x float> %y) nounwind {
 ; AVX-LABEL: floor_ss:
 ; AVX:       ## %bb.0:
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_ss:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    retq
   %s = extractelement <4 x float> %x, i32 0
   %call = call float @llvm.floor.f32(float %s)
@@ -853,13 +853,13 @@ define <2 x double> @floor_sd(<2 x double> %x, <2 x double> %y) nounwind {
 ; AVX-LABEL: floor_sd:
 ; AVX:       ## %bb.0:
 ; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_sd:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX512-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX512-NEXT:    retq
   %s = extractelement <2 x double> %x, i32 0
   %call = call double @llvm.floor.f64(double %s)
@@ -1361,7 +1361,7 @@ define <4 x float> @floor_mask_ss(<4 x float> %x, <4 x float> %y, <4 x float> %w
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundss $9, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB52_2:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1372,7 +1372,7 @@ define <4 x float> @floor_mask_ss(<4 x float> %x, <4 x float> %y, <4 x float> %w
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB52_2:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_mask_ss:
@@ -1402,7 +1402,7 @@ define <4 x float> @floor_maskz_ss(<4 x float> %x, <4 x float> %y, i8 %k) nounwi
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundss $9, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB53_2:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1414,7 +1414,7 @@ define <4 x float> @floor_maskz_ss(<4 x float> %x, <4 x float> %y, i8 %k) nounwi
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB53_2:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_maskz_ss:
@@ -1441,7 +1441,7 @@ define <2 x double> @floor_mask_sd(<2 x double> %x, <2 x double> %y, <2 x double
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundsd $9, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB54_2:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1452,7 +1452,7 @@ define <2 x double> @floor_mask_sd(<2 x double> %x, <2 x double> %y, <2 x double
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB54_2:
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_mask_sd:
@@ -1482,7 +1482,7 @@ define <2 x double> @floor_maskz_sd(<2 x double> %x, <2 x double> %y, i8 %k) nou
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundsd $9, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB55_2:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1494,7 +1494,7 @@ define <2 x double> @floor_maskz_sd(<2 x double> %x, <2 x double> %y, i8 %k) nou
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB55_2:
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_maskz_sd:
@@ -1521,7 +1521,7 @@ define <4 x float> @floor_mask_ss_trunc(<4 x float> %x, <4 x float> %y, <4 x flo
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundss $9, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB56_2:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1532,7 +1532,7 @@ define <4 x float> @floor_mask_ss_trunc(<4 x float> %x, <4 x float> %y, <4 x flo
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB56_2:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_mask_ss_trunc:
@@ -1562,7 +1562,7 @@ define <4 x float> @floor_maskz_ss_trunc(<4 x float> %x, <4 x float> %y, i16 %k)
 ; SSE41-NEXT:  LBB57_1:
 ; SSE41-NEXT:    roundss $9, %xmm0, %xmm0
 ; SSE41-NEXT:  LBB57_3:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1572,11 +1572,11 @@ define <4 x float> @floor_maskz_ss_trunc(<4 x float> %x, <4 x float> %y, i16 %k)
 ; AVX-NEXT:    jne LBB57_1
 ; AVX-NEXT:  ## %bb.2:
 ; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ; AVX-NEXT:  LBB57_1:
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_maskz_ss_trunc:
@@ -1602,7 +1602,7 @@ define <2 x double> @floor_mask_sd_trunc(<2 x double> %x, <2 x double> %y, <2 x
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundsd $9, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB58_2:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1613,7 +1613,7 @@ define <2 x double> @floor_mask_sd_trunc(<2 x double> %x, <2 x double> %y, <2 x
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB58_2:
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_mask_sd_trunc:
@@ -1643,7 +1643,7 @@ define <2 x double> @floor_maskz_sd_trunc(<2 x double> %x, <2 x double> %y, i16
 ; SSE41-NEXT:  LBB59_1:
 ; SSE41-NEXT:    roundsd $9, %xmm0, %xmm0
 ; SSE41-NEXT:  LBB59_3:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1653,11 +1653,11 @@ define <2 x double> @floor_maskz_sd_trunc(<2 x double> %x, <2 x double> %y, i16
 ; AVX-NEXT:    jne LBB59_1
 ; AVX-NEXT:  ## %bb.2:
 ; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ; AVX-NEXT:  LBB59_1:
 ; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_maskz_sd_trunc:
@@ -1689,7 +1689,7 @@ define <4 x float> @floor_mask_ss_mask8(<4 x float> %x, <4 x float> %y, <4 x flo
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm3
 ; AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_mask_ss_mask8:
@@ -1723,7 +1723,7 @@ define <4 x float> @floor_maskz_ss_mask8(<4 x float> %x, <4 x float> %y) nounwin
 ; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_maskz_ss_mask8:
@@ -1756,7 +1756,7 @@ define <2 x double> @floor_mask_sd_mask8(<2 x double> %x, <2 x double> %y, <2 x
 ; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm3
 ; AVX-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vblendvpd %xmm0, %xmm3, %xmm2, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_mask_sd_mask8:
@@ -1790,7 +1790,7 @@ define <2 x double> @floor_maskz_sd_mask8(<2 x double> %x, <2 x double> %y) noun
 ; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vandpd %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: floor_maskz_sd_mask8:
@@ -1818,13 +1818,13 @@ define <4 x float> @ceil_ss(<4 x float> %x, <4 x float> %y) nounwind {
 ; AVX-LABEL: ceil_ss:
 ; AVX:       ## %bb.0:
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_ss:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    retq
   %s = extractelement <4 x float> %x, i32 0
   %call = call float @llvm.ceil.f32(float %s)
@@ -1843,13 +1843,13 @@ define <2 x double> @ceil_sd(<2 x double> %x, <2 x double> %y) nounwind {
 ; AVX-LABEL: ceil_sd:
 ; AVX:       ## %bb.0:
 ; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_sd:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX512-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX512-NEXT:    retq
   %s = extractelement <2 x double> %x, i32 0
   %call = call double @llvm.ceil.f64(double %s)
@@ -2351,7 +2351,7 @@ define <4 x float> @ceil_mask_ss(<4 x float> %x, <4 x float> %y, <4 x float> %w,
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB78_2:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2362,7 +2362,7 @@ define <4 x float> @ceil_mask_ss(<4 x float> %x, <4 x float> %y, <4 x float> %w,
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB78_2:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_mask_ss:
@@ -2392,7 +2392,7 @@ define <4 x float> @ceil_maskz_ss(<4 x float> %x, <4 x float> %y, i8 %k) nounwin
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB79_2:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2404,7 +2404,7 @@ define <4 x float> @ceil_maskz_ss(<4 x float> %x, <4 x float> %y, i8 %k) nounwin
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB79_2:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_maskz_ss:
@@ -2431,7 +2431,7 @@ define <2 x double> @ceil_mask_sd(<2 x double> %x, <2 x double> %y, <2 x double>
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB80_2:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2442,7 +2442,7 @@ define <2 x double> @ceil_mask_sd(<2 x double> %x, <2 x double> %y, <2 x double>
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB80_2:
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_mask_sd:
@@ -2472,7 +2472,7 @@ define <2 x double> @ceil_maskz_sd(<2 x double> %x, <2 x double> %y, i8 %k) noun
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB81_2:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2484,7 +2484,7 @@ define <2 x double> @ceil_maskz_sd(<2 x double> %x, <2 x double> %y, i8 %k) noun
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB81_2:
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_maskz_sd:
@@ -2511,7 +2511,7 @@ define <4 x float> @ceil_mask_ss_trunc(<4 x float> %x, <4 x float> %y, <4 x floa
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB82_2:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2522,7 +2522,7 @@ define <4 x float> @ceil_mask_ss_trunc(<4 x float> %x, <4 x float> %y, <4 x floa
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB82_2:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_mask_ss_trunc:
@@ -2552,7 +2552,7 @@ define <4 x float> @ceil_maskz_ss_trunc(<4 x float> %x, <4 x float> %y, i16 %k)
 ; SSE41-NEXT:  LBB83_1:
 ; SSE41-NEXT:    roundss $10, %xmm0, %xmm0
 ; SSE41-NEXT:  LBB83_3:
-; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2562,11 +2562,11 @@ define <4 x float> @ceil_maskz_ss_trunc(<4 x float> %x, <4 x float> %y, i16 %k)
 ; AVX-NEXT:    jne LBB83_1
 ; AVX-NEXT:  ## %bb.2:
 ; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ; AVX-NEXT:  LBB83_1:
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_maskz_ss_trunc:
@@ -2592,7 +2592,7 @@ define <2 x double> @ceil_mask_sd_trunc(<2 x double> %x, <2 x double> %y, <2 x d
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
 ; SSE41-NEXT:  LBB84_2:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2603,7 +2603,7 @@ define <2 x double> @ceil_mask_sd_trunc(<2 x double> %x, <2 x double> %y, <2 x d
 ; AVX-NEXT:  ## %bb.1:
 ; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:  LBB84_2:
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_mask_sd_trunc:
@@ -2633,7 +2633,7 @@ define <2 x double> @ceil_maskz_sd_trunc(<2 x double> %x, <2 x double> %y, i16 %
 ; SSE41-NEXT:  LBB85_1:
 ; SSE41-NEXT:    roundsd $10, %xmm0, %xmm0
 ; SSE41-NEXT:  LBB85_3:
-; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -2643,11 +2643,11 @@ define <2 x double> @ceil_maskz_sd_trunc(<2 x double> %x, <2 x double> %y, i16 %
 ; AVX-NEXT:    jne LBB85_1
 ; AVX-NEXT:  ## %bb.2:
 ; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ; AVX-NEXT:  LBB85_1:
 ; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_maskz_sd_trunc:
@@ -2679,7 +2679,7 @@ define <4 x float> @ceil_mask_ss_mask8(<4 x float> %x, <4 x float> %y, <4 x floa
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm3
 ; AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_mask_ss_mask8:
@@ -2713,7 +2713,7 @@ define <4 x float> @ceil_maskz_ss_mask8(<4 x float> %x, <4 x float> %y) nounwind
 ; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_maskz_ss_mask8:
@@ -2746,7 +2746,7 @@ define <2 x double> @ceil_mask_sd_mask8(<2 x double> %x, <2 x double> %y, <2 x d
 ; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm3
 ; AVX-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vblendvpd %xmm0, %xmm3, %xmm2, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_mask_sd_mask8:
@@ -2780,7 +2780,7 @@ define <2 x double> @ceil_maskz_sd_mask8(<2 x double> %x, <2 x double> %y) nounw
 ; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
 ; AVX-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vandpd %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: ceil_maskz_sd_mask8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index af841cf38b24..62ab5d82bfbb 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -126,27 +126,27 @@ define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; VEX-LABEL: sitofp_2i64_to_2f64:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; VEX-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; VEX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_2i64_to_2f64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_2i64_to_2f64:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512VL-NEXT:    retq
 ;
@@ -352,14 +352,14 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -368,14 +368,14 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX2-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vmovq %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX2-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX2-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX2-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX2-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
@@ -384,14 +384,14 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm1, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -400,14 +400,14 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512VL-NEXT:    vmovq %xmm1, %rax
-; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512VL-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
@@ -1247,27 +1247,27 @@ define <4 x float> @sitofp_2i64_to_4f32(<2 x i64> %a) {
 ; VEX-LABEL: sitofp_2i64_to_4f32:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_2i64_to_4f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_2i64_to_4f32:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -1316,27 +1316,27 @@ define <4 x float> @sitofp_2i64_to_4f32_zero(<2 x i64> %a) {
 ; VEX-LABEL: sitofp_2i64_to_4f32_zero:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_2i64_to_4f32_zero:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_2i64_to_4f32_zero:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -1383,27 +1383,27 @@ define <4 x float> @sitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; VEX-LABEL: sitofp_4i64_to_4f32_undef:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vpextrq $1, %xmm0, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; VEX-NEXT:    vmovq %xmm0, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_4i64_to_4f32_undef:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_4i64_to_4f32_undef:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -1581,16 +1581,16 @@ define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX1-LABEL: sitofp_4i64_to_4f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1598,16 +1598,16 @@ define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX2-LABEL: sitofp_4i64_to_4f32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1615,16 +1615,16 @@ define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX512F-LABEL: sitofp_4i64_to_4f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1632,16 +1632,16 @@ define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX512VL-LABEL: sitofp_4i64_to_4f32:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512VL-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1892,9 +1892,9 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ; VEX-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; VEX-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
 ; VEX-NEXT:    vpextrq $1, %xmm1, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; VEX-NEXT:    vmovq %xmm1, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
 ; VEX-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; VEX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -1906,18 +1906,18 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ; AVX512F-LABEL: uitofp_2i64_to_4f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: uitofp_2i64_to_4f32:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -2007,9 +2007,9 @@ define <4 x float> @uitofp_2i64_to_2f32(<2 x i64> %a) {
 ; VEX-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; VEX-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
 ; VEX-NEXT:    vpextrq $1, %xmm1, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; VEX-NEXT:    vmovq %xmm1, %rax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
 ; VEX-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; VEX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -2022,18 +2022,18 @@ define <4 x float> @uitofp_2i64_to_2f32(<2 x i64> %a) {
 ; AVX512F-LABEL: uitofp_2i64_to_2f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: uitofp_2i64_to_2f32:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -2125,9 +2125,9 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; AVX1-NEXT:    vmovaps %xmm0, %xmm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
 ; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
@@ -2148,16 +2148,16 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vmovq %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX2-NEXT:    vmovq %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm1
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
 ; AVX2-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
@@ -2168,18 +2168,18 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; AVX512F-LABEL: uitofp_4i64_to_4f32_undef:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: uitofp_4i64_to_4f32_undef:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
 ; AVX512VL-NEXT:    retq
 ;
@@ -2494,16 +2494,16 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX1-NEXT:    vorpd %ymm3, %ymm1, %ymm1
 ; AVX1-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm3
 ; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
@@ -2519,16 +2519,16 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vmovq %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX2-NEXT:    vmovq %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm1
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
 ; AVX2-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
@@ -2540,16 +2540,16 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX512F-LABEL: uitofp_4i64_to_4f32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2557,16 +2557,16 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; AVX512VL-LABEL: uitofp_4i64_to_4f32:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512VL-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512VL-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -2836,22 +2836,22 @@ define <2 x double> @sitofp_load_2i64_to_2f64(ptr%a) {
 ;
 ; VEX-LABEL: sitofp_load_2i64_to_2f64:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vcvtsi2sdq 8(%rdi), %xmm0, %xmm0
-; VEX-NEXT:    vcvtsi2sdq (%rdi), %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2sdq 8(%rdi), %xmm15, %xmm0
+; VEX-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_load_2i64_to_2f64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvtsi2sdq 8(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:    vcvtsi2sdq (%rdi), %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2sdq 8(%rdi), %xmm15, %xmm0
+; AVX512F-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_load_2i64_to_2f64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvtsi2sdq 8(%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT:    vcvtsi2sdq (%rdi), %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2sdq 8(%rdi), %xmm15, %xmm0
+; AVX512VL-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX512VL-NEXT:    retq
 ;
@@ -3011,33 +3011,33 @@ define <4 x double> @sitofp_load_4i64_to_4f64(ptr%a) {
 ;
 ; VEX-LABEL: sitofp_load_4i64_to_4f64:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vcvtsi2sdq 24(%rdi), %xmm0, %xmm0
-; VEX-NEXT:    vcvtsi2sdq 16(%rdi), %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2sdq 24(%rdi), %xmm15, %xmm0
+; VEX-NEXT:    vcvtsi2sdq 16(%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; VEX-NEXT:    vcvtsi2sdq 8(%rdi), %xmm2, %xmm1
-; VEX-NEXT:    vcvtsi2sdq (%rdi), %xmm2, %xmm2
+; VEX-NEXT:    vcvtsi2sdq 8(%rdi), %xmm15, %xmm1
+; VEX-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm2
 ; VEX-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; VEX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_load_4i64_to_4f64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvtsi2sdq 24(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:    vcvtsi2sdq 16(%rdi), %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2sdq 24(%rdi), %xmm15, %xmm0
+; AVX512F-NEXT:    vcvtsi2sdq 16(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512F-NEXT:    vcvtsi2sdq 8(%rdi), %xmm2, %xmm1
-; AVX512F-NEXT:    vcvtsi2sdq (%rdi), %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtsi2sdq 8(%rdi), %xmm15, %xmm1
+; AVX512F-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm2
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_load_4i64_to_4f64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvtsi2sdq 24(%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT:    vcvtsi2sdq 16(%rdi), %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2sdq 24(%rdi), %xmm15, %xmm0
+; AVX512VL-NEXT:    vcvtsi2sdq 16(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VL-NEXT:    vcvtsi2sdq 8(%rdi), %xmm2, %xmm1
-; AVX512VL-NEXT:    vcvtsi2sdq (%rdi), %xmm2, %xmm2
+; AVX512VL-NEXT:    vcvtsi2sdq 8(%rdi), %xmm15, %xmm1
+; AVX512VL-NEXT:    vcvtsi2sdq (%rdi), %xmm15, %xmm2
 ; AVX512VL-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX512VL-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512VL-NEXT:    retq
@@ -3776,34 +3776,34 @@ define <4 x float> @sitofp_load_4i64_to_4f32(ptr%a) {
 ;
 ; VEX-LABEL: sitofp_load_4i64_to_4f32:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vcvtsi2ssq 8(%rdi), %xmm0, %xmm0
-; VEX-NEXT:    vcvtsi2ssq (%rdi), %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2ssq 8(%rdi), %xmm15, %xmm0
+; VEX-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; VEX-NEXT:    vcvtsi2ssq 16(%rdi), %xmm2, %xmm1
+; VEX-NEXT:    vcvtsi2ssq 16(%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; VEX-NEXT:    vcvtsi2ssq 24(%rdi), %xmm2, %xmm1
+; VEX-NEXT:    vcvtsi2ssq 24(%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_load_4i64_to_4f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvtsi2ssq 8(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:    vcvtsi2ssq (%rdi), %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2ssq 8(%rdi), %xmm15, %xmm0
+; AVX512F-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512F-NEXT:    vcvtsi2ssq 16(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtsi2ssq 16(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512F-NEXT:    vcvtsi2ssq 24(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtsi2ssq 24(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_load_4i64_to_4f32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvtsi2ssq 8(%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT:    vcvtsi2ssq (%rdi), %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ssq 8(%rdi), %xmm15, %xmm0
+; AVX512VL-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512VL-NEXT:    vcvtsi2ssq 16(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ssq 16(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512VL-NEXT:    vcvtsi2ssq 24(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ssq 24(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
 ; AVX512VL-NEXT:    retq
 ;
@@ -3938,57 +3938,57 @@ define <8 x float> @sitofp_load_8i64_to_8f32(ptr%a) {
 ;
 ; VEX-LABEL: sitofp_load_8i64_to_8f32:
 ; VEX:       # %bb.0:
-; VEX-NEXT:    vcvtsi2ssq 40(%rdi), %xmm0, %xmm0
-; VEX-NEXT:    vcvtsi2ssq 32(%rdi), %xmm1, %xmm1
+; VEX-NEXT:    vcvtsi2ssq 40(%rdi), %xmm15, %xmm0
+; VEX-NEXT:    vcvtsi2ssq 32(%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; VEX-NEXT:    vcvtsi2ssq 48(%rdi), %xmm2, %xmm1
+; VEX-NEXT:    vcvtsi2ssq 48(%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; VEX-NEXT:    vcvtsi2ssq 56(%rdi), %xmm2, %xmm1
+; VEX-NEXT:    vcvtsi2ssq 56(%rdi), %xmm15, %xmm1
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
-; VEX-NEXT:    vcvtsi2ssq 8(%rdi), %xmm2, %xmm1
-; VEX-NEXT:    vcvtsi2ssq (%rdi), %xmm2, %xmm2
+; VEX-NEXT:    vcvtsi2ssq 8(%rdi), %xmm15, %xmm1
+; VEX-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm2
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
-; VEX-NEXT:    vcvtsi2ssq 16(%rdi), %xmm3, %xmm2
+; VEX-NEXT:    vcvtsi2ssq 16(%rdi), %xmm15, %xmm2
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; VEX-NEXT:    vcvtsi2ssq 24(%rdi), %xmm3, %xmm2
+; VEX-NEXT:    vcvtsi2ssq 24(%rdi), %xmm15, %xmm2
 ; VEX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; VEX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: sitofp_load_8i64_to_8f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvtsi2ssq 40(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:    vcvtsi2ssq 32(%rdi), %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2ssq 40(%rdi), %xmm15, %xmm0
+; AVX512F-NEXT:    vcvtsi2ssq 32(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512F-NEXT:    vcvtsi2ssq 48(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtsi2ssq 48(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512F-NEXT:    vcvtsi2ssq 56(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtsi2ssq 56(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
-; AVX512F-NEXT:    vcvtsi2ssq 8(%rdi), %xmm2, %xmm1
-; AVX512F-NEXT:    vcvtsi2ssq (%rdi), %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtsi2ssq 8(%rdi), %xmm15, %xmm1
+; AVX512F-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
-; AVX512F-NEXT:    vcvtsi2ssq 16(%rdi), %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtsi2ssq 16(%rdi), %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; AVX512F-NEXT:    vcvtsi2ssq 24(%rdi), %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtsi2ssq 24(%rdi), %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: sitofp_load_8i64_to_8f32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvtsi2ssq 40(%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT:    vcvtsi2ssq 32(%rdi), %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ssq 40(%rdi), %xmm15, %xmm0
+; AVX512VL-NEXT:    vcvtsi2ssq 32(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512VL-NEXT:    vcvtsi2ssq 48(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ssq 48(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512VL-NEXT:    vcvtsi2ssq 56(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ssq 56(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
-; AVX512VL-NEXT:    vcvtsi2ssq 8(%rdi), %xmm2, %xmm1
-; AVX512VL-NEXT:    vcvtsi2ssq (%rdi), %xmm2, %xmm2
+; AVX512VL-NEXT:    vcvtsi2ssq 8(%rdi), %xmm15, %xmm1
+; AVX512VL-NEXT:    vcvtsi2ssq (%rdi), %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
-; AVX512VL-NEXT:    vcvtsi2ssq 16(%rdi), %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtsi2ssq 16(%rdi), %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; AVX512VL-NEXT:    vcvtsi2ssq 24(%rdi), %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtsi2ssq 24(%rdi), %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; AVX512VL-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512VL-NEXT:    retq
@@ -4235,16 +4235,16 @@ define <4 x float> @uitofp_load_4i64_to_4f32(ptr%a) {
 ; AVX1-NEXT:    vorpd %ymm3, %ymm1, %ymm1
 ; AVX1-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm3
 ; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
@@ -4261,16 +4261,16 @@ define <4 x float> @uitofp_load_4i64_to_4f32(ptr%a) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vmovq %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX2-NEXT:    vmovq %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm1
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
 ; AVX2-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; AVX2-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
@@ -4280,23 +4280,23 @@ define <4 x float> @uitofp_load_4i64_to_4f32(ptr%a) {
 ;
 ; AVX512F-LABEL: uitofp_load_4i64_to_4f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvtusi2ssq 8(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:    vcvtusi2ssq (%rdi), %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2ssq 8(%rdi), %xmm15, %xmm0
+; AVX512F-NEXT:    vcvtusi2ssq (%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512F-NEXT:    vcvtusi2ssq 16(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtusi2ssq 16(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512F-NEXT:    vcvtusi2ssq 24(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtusi2ssq 24(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: uitofp_load_4i64_to_4f32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvtusi2ssq 8(%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT:    vcvtusi2ssq (%rdi), %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ssq 8(%rdi), %xmm15, %xmm0
+; AVX512VL-NEXT:    vcvtusi2ssq (%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512VL-NEXT:    vcvtusi2ssq 16(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ssq 16(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512VL-NEXT:    vcvtusi2ssq 24(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ssq 24(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
 ; AVX512VL-NEXT:    retq
 ;
@@ -4664,16 +4664,16 @@ define <8 x float> @uitofp_load_8i64_to_8f32(ptr%a) {
 ; AVX1-NEXT:    vorps %ymm3, %ymm4, %ymm3
 ; AVX1-NEXT:    vblendvpd %ymm1, %ymm3, %ymm1, %ymm3
 ; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm4
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vmovq %xmm3, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm6
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm6
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
 ; AVX1-NEXT:    vmovq %xmm3, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm6
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm6
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]
 ; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm3
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[0]
 ; AVX1-NEXT:    vaddps %xmm3, %xmm3, %xmm4
 ; AVX1-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
@@ -4686,16 +4686,16 @@ define <8 x float> @uitofp_load_8i64_to_8f32(ptr%a) {
 ; AVX1-NEXT:    vorps %ymm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm0, %ymm2, %ymm0, %ymm2
 ; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm3
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vmovq %xmm2, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm5
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm5
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
 ; AVX1-NEXT:    vmovq %xmm2, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm5
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm5
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm5[0],xmm3[3]
 ; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm7, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[0]
 ; AVX1-NEXT:    vaddps %xmm2, %xmm2, %xmm3
 ; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
@@ -4713,16 +4713,16 @@ define <8 x float> @uitofp_load_8i64_to_8f32(ptr%a) {
 ; AVX2-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm1, %ymm3, %ymm1, %ymm3
 ; AVX2-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX2-NEXT:    vmovq %xmm3, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm5
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm5
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[2,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm3
 ; AVX2-NEXT:    vmovq %xmm3, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm5
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm5
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0],xmm4[3]
 ; AVX2-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[0]
 ; AVX2-NEXT:    vaddps %xmm3, %xmm3, %xmm4
 ; AVX2-NEXT:    vpackssdw 48(%rdi), %xmm1, %xmm1
@@ -4732,16 +4732,16 @@ define <8 x float> @uitofp_load_8i64_to_8f32(ptr%a) {
 ; AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm0, %ymm2
 ; AVX2-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm3
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX2-NEXT:    vmovq %xmm2, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm4
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm2
 ; AVX2-NEXT:    vmovq %xmm2, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm4
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; AVX2-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX2-NEXT:    vcvtsi2ss %rax, %xmm6, %xmm2
+; AVX2-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[0]
 ; AVX2-NEXT:    vaddps %xmm2, %xmm2, %xmm3
 ; AVX2-NEXT:    vpackssdw 16(%rdi), %xmm0, %xmm0
@@ -4751,38 +4751,38 @@ define <8 x float> @uitofp_load_8i64_to_8f32(ptr%a) {
 ;
 ; AVX512F-LABEL: uitofp_load_8i64_to_8f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vcvtusi2ssq 40(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT:    vcvtusi2ssq 32(%rdi), %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2ssq 40(%rdi), %xmm15, %xmm0
+; AVX512F-NEXT:    vcvtusi2ssq 32(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512F-NEXT:    vcvtusi2ssq 48(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtusi2ssq 48(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512F-NEXT:    vcvtusi2ssq 56(%rdi), %xmm2, %xmm1
+; AVX512F-NEXT:    vcvtusi2ssq 56(%rdi), %xmm15, %xmm1
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
-; AVX512F-NEXT:    vcvtusi2ssq 8(%rdi), %xmm2, %xmm1
-; AVX512F-NEXT:    vcvtusi2ssq (%rdi), %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtusi2ssq 8(%rdi), %xmm15, %xmm1
+; AVX512F-NEXT:    vcvtusi2ssq (%rdi), %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
-; AVX512F-NEXT:    vcvtusi2ssq 16(%rdi), %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtusi2ssq 16(%rdi), %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; AVX512F-NEXT:    vcvtusi2ssq 24(%rdi), %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtusi2ssq 24(%rdi), %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: uitofp_load_8i64_to_8f32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vcvtusi2ssq 40(%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT:    vcvtusi2ssq 32(%rdi), %xmm1, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ssq 40(%rdi), %xmm15, %xmm0
+; AVX512VL-NEXT:    vcvtusi2ssq 32(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512VL-NEXT:    vcvtusi2ssq 48(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ssq 48(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
-; AVX512VL-NEXT:    vcvtusi2ssq 56(%rdi), %xmm2, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ssq 56(%rdi), %xmm15, %xmm1
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
-; AVX512VL-NEXT:    vcvtusi2ssq 8(%rdi), %xmm2, %xmm1
-; AVX512VL-NEXT:    vcvtusi2ssq (%rdi), %xmm2, %xmm2
+; AVX512VL-NEXT:    vcvtusi2ssq 8(%rdi), %xmm15, %xmm1
+; AVX512VL-NEXT:    vcvtusi2ssq (%rdi), %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
-; AVX512VL-NEXT:    vcvtusi2ssq 16(%rdi), %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtusi2ssq 16(%rdi), %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; AVX512VL-NEXT:    vcvtusi2ssq 24(%rdi), %xmm3, %xmm2
+; AVX512VL-NEXT:    vcvtusi2ssq 24(%rdi), %xmm15, %xmm2
 ; AVX512VL-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[0]
 ; AVX512VL-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512VL-NEXT:    retq
@@ -5148,7 +5148,7 @@ define float @extract0_sitofp_v4i32_f32i_multiuse1(<4 x i32> %x) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; AVX-NEXT:    incl %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm1, %xmm1
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm1
 ; AVX-NEXT:    vdivss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 0
@@ -5207,7 +5207,7 @@ define float @extract0_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
 ; VEX-LABEL: extract0_uitofp_v4i32_f32:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vmovd %xmm0, %eax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm0
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: extract0_uitofp_v4i32_f32:
@@ -5251,7 +5251,7 @@ define double @extract0_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ; VEX-LABEL: extract0_uitofp_v4i32_f64:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vmovd %xmm0, %eax
-; VEX-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm0
+; VEX-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: extract0_uitofp_v4i32_f64:
@@ -5348,7 +5348,7 @@ define float @extract3_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
 ; VEX-LABEL: extract3_uitofp_v4i32_f32:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vextractps $3, %xmm0, %eax
-; VEX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm0
+; VEX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: extract3_uitofp_v4i32_f32:
@@ -5402,7 +5402,7 @@ define double @extract3_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ; VEX-LABEL: extract3_uitofp_v4i32_f64:
 ; VEX:       # %bb.0:
 ; VEX-NEXT:    vextractps $3, %xmm0, %eax
-; VEX-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm0
+; VEX-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: extract3_uitofp_v4i32_f64:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vec_ss_load_fold.ll b/external/llvm-project/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
index e4304f2cc214..e73d345d0fcd 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vec_ss_load_fold.ll
@@ -38,7 +38,7 @@ define i16 @test1(float %f) nounwind {
 ; X86_AVX1-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86_AVX1-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86_AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86_AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X86_AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X86_AVX1-NEXT:    vminss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86_AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X86_AVX1-NEXT:    vcvttss2si %xmm0, %eax
@@ -50,7 +50,7 @@ define i16 @test1(float %f) nounwind {
 ; X64_AVX1-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; X64_AVX1-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; X64_AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64_AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64_AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X64_AVX1-NEXT:    vminss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; X64_AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
 ; X64_AVX1-NEXT:    vcvttss2si %xmm0, %eax
@@ -63,7 +63,7 @@ define i16 @test1(float %f) nounwind {
 ; X86_AVX512-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86_AVX512-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86_AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X86_AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X86_AVX512-NEXT:    vminss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X86_AVX512-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
@@ -76,7 +76,7 @@ define i16 @test1(float %f) nounwind {
 ; X64_AVX512-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; X64_AVX512-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; X64_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64_AVX512-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64_AVX512-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; X64_AVX512-NEXT:    vminss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; X64_AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64_AVX512-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-blend.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-blend.ll
index bd5c9363794a..2d2fc6b6ee0d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-blend.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-blend.ll
@@ -54,7 +54,7 @@ define <4 x float> @vsel_float2(<4 x float> %v1, <4 x float> %v2) {
 ;
 ; AVX-LABEL: vsel_float2:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 entry:
   %vsel = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x float> %v1, <4 x float> %v2
@@ -172,7 +172,7 @@ define <2 x double> @vsel_double(<2 x double> %v1, <2 x double> %v2) {
 ;
 ; AVX-LABEL: vsel_double:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
 entry:
   %vsel = select <2 x i1> <i1 true, i1 false>, <2 x double> %v1, <2 x double> %v2
@@ -732,23 +732,11 @@ entry:
 }
 
 define <4 x i64> @blend_shufflevector_4xi64(<4 x i64> %a, <4 x i64> %b) {
-; SSE2-LABEL: blend_shufflevector_4xi64:
-; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: blend_shufflevector_4xi64:
-; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movaps %xmm3, %xmm1
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: blend_shufflevector_4xi64:
-; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    movaps %xmm3, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: blend_shufflevector_4xi64:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movaps %xmm3, %xmm1
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: blend_shufflevector_4xi64:
 ; AVX:       # %bb.0: # %entry
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
index 49062eaef318..4a5b4277c3cc 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck %s
 ; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx < %s | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+avx512f < %s | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
@@ -6504,7 +6504,7 @@ define <1 x double> @constrained_vector_sitofp_v1f64_v1i32(<1 x i32> %x) #0 {
 ;
 ; AVX-LABEL: constrained_vector_sitofp_v1f64_v1i32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sd %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call <1 x double>
@@ -6522,7 +6522,7 @@ define <1 x float> @constrained_vector_sitofp_v1f32_v1i32(<1 x i32> %x) #0 {
 ;
 ; AVX-LABEL: constrained_vector_sitofp_v1f32_v1i32:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ss %edi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %edi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call <1 x float>
@@ -6540,7 +6540,7 @@ define <1 x double> @constrained_vector_sitofp_v1f64_v1i64(<1 x i64> %x) #0 {
 ;
 ; AVX-LABEL: constrained_vector_sitofp_v1f64_v1i64:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2sd %rdi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2sd %rdi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call <1 x double>
@@ -6558,7 +6558,7 @@ define <1 x float> @constrained_vector_sitofp_v1f32_v1i64(<1 x i64> %x) #0 {
 ;
 ; AVX-LABEL: constrained_vector_sitofp_v1f32_v1i64:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vcvtsi2ss %rdi, %xmm0, %xmm0
+; AVX-NEXT:    vcvtsi2ss %rdi, %xmm15, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %result = call <1 x float>
@@ -6622,18 +6622,18 @@ define <2 x double> @constrained_vector_sitofp_v2f64_v2i64(<2 x i64> %x) #0 {
 ; AVX1-LABEL: constrained_vector_sitofp_v2f64_v2i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX1-NEXT:    retq
 ;
 ; AVX512F-LABEL: constrained_vector_sitofp_v2f64_v2i64:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512F-NEXT:    retq
 ;
@@ -6668,9 +6668,9 @@ define <2 x float> @constrained_vector_sitofp_v2f32_v2i64(<2 x i64> %x) #0 {
 ; AVX-LABEL: constrained_vector_sitofp_v2f32_v2i64:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm0
+; AVX-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX-NEXT:    retq
 entry:
@@ -6703,12 +6703,12 @@ define <3 x double> @constrained_vector_sitofp_v3f64_v3i32(<3 x i32> %x) #0 {
 ; AVX-LABEL: constrained_vector_sitofp_v3f64_v3i32:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    vextractps $1, %xmm0, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm1, %xmm1
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm1
 ; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm2, %xmm2
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm2
 ; AVX-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vcvtsi2sd %eax, %xmm3, %xmm0
+; AVX-NEXT:    vcvtsi2sd %eax, %xmm15, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX-NEXT:    retq
 entry:
@@ -6740,12 +6740,12 @@ define <3 x float> @constrained_vector_sitofp_v3f32_v3i32(<3 x i32> %x) #0 {
 ; AVX-LABEL: constrained_vector_sitofp_v3f32_v3i32:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    vextractps $1, %xmm0, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm1, %xmm1
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm1
 ; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm2, %xmm2
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm2
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX-NEXT:    vcvtsi2ss %eax, %xmm3, %xmm0
+; AVX-NEXT:    vcvtsi2ss %eax, %xmm15, %xmm0
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX-NEXT:    retq
 entry:
@@ -6770,26 +6770,26 @@ define <3 x double> @constrained_vector_sitofp_v3f64_v3i64(<3 x i64> %x) #0 {
 ; AVX1-LABEL: constrained_vector_sitofp_v3f64_v3i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: constrained_vector_sitofp_v3f64_v3i64:
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
 entry:
@@ -6814,13 +6814,13 @@ define <3 x float> @constrained_vector_sitofp_v3f32_v3i64(<3 x i64> %x) #0 {
 ; AVX1-LABEL: constrained_vector_sitofp_v3f32_v3i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -6828,13 +6828,13 @@ define <3 x float> @constrained_vector_sitofp_v3f32_v3i64(<3 x i64> %x) #0 {
 ; AVX512-LABEL: constrained_vector_sitofp_v3f32_v3i64:
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -6910,14 +6910,14 @@ define <4 x double> @constrained_vector_sitofp_v4f64_v4i64(<4 x i64> %x) #0 {
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -6926,14 +6926,14 @@ define <4 x double> @constrained_vector_sitofp_v4f64_v4i64(<4 x i64> %x) #0 {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm1, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm1
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -6977,16 +6977,16 @@ define <4 x float> @constrained_vector_sitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; AVX1-LABEL: constrained_vector_sitofp_v4f32_v4i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -6994,16 +6994,16 @@ define <4 x float> @constrained_vector_sitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; AVX512F-LABEL: constrained_vector_sitofp_v4f32_v4i64:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -7033,12 +7033,12 @@ define <1 x double> @constrained_vector_uitofp_v1f64_v1i32(<1 x i32> %x) #0 {
 ; AVX1-LABEL: constrained_vector_uitofp_v1f64_v1i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    movl %edi, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: constrained_vector_uitofp_v1f64_v1i32:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2sd %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call <1 x double>
@@ -7058,12 +7058,12 @@ define <1 x float> @constrained_vector_uitofp_v1f32_v1i32(<1 x i32> %x) #0 {
 ; AVX1-LABEL: constrained_vector_uitofp_v1f32_v1i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    movl %edi, %eax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: constrained_vector_uitofp_v1f32_v1i32:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2ss %edi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %edi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call <1 x float>
@@ -7099,7 +7099,7 @@ define <1 x double> @constrained_vector_uitofp_v1f64_v1i64(<1 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rax, %rcx
 ; AVX1-NEXT:    testq %rdi, %rdi
 ; AVX1-NEXT:    cmovnsq %rdi, %rcx
-; AVX1-NEXT:    vcvtsi2sd %rcx, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rcx, %xmm15, %xmm0
 ; AVX1-NEXT:    jns .LBB175_2
 ; AVX1-NEXT:  # %bb.1:
 ; AVX1-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
@@ -7108,7 +7108,7 @@ define <1 x double> @constrained_vector_uitofp_v1f64_v1i64(<1 x i64> %x) #0 {
 ;
 ; AVX512-LABEL: constrained_vector_uitofp_v1f64_v1i64:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call <1 x double>
@@ -7144,7 +7144,7 @@ define <1 x float> @constrained_vector_uitofp_v1f32_v1i64(<1 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rax, %rcx
 ; AVX1-NEXT:    testq %rdi, %rdi
 ; AVX1-NEXT:    cmovnsq %rdi, %rcx
-; AVX1-NEXT:    vcvtsi2ss %rcx, %xmm0, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rcx, %xmm15, %xmm0
 ; AVX1-NEXT:    jns .LBB176_2
 ; AVX1-NEXT:  # %bb.1:
 ; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
@@ -7153,7 +7153,7 @@ define <1 x float> @constrained_vector_uitofp_v1f32_v1i64(<1 x i64> %x) #0 {
 ;
 ; AVX512-LABEL: constrained_vector_uitofp_v1f32_v1i64:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm0, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %rdi, %xmm15, %xmm0
 ; AVX512-NEXT:    retq
 entry:
   %result = call <1 x float>
@@ -7279,7 +7279,7 @@ define <2 x double> @constrained_vector_uitofp_v2f64_v2i64(<2 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm15, %xmm1
 ; AVX1-NEXT:    jns .LBB179_2
 ; AVX1-NEXT:  # %bb.1:
 ; AVX1-NEXT:    vaddsd %xmm1, %xmm1, %xmm1
@@ -7292,7 +7292,7 @@ define <2 x double> @constrained_vector_uitofp_v2f64_v2i64(<2 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm2, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm15, %xmm0
 ; AVX1-NEXT:    jns .LBB179_4
 ; AVX1-NEXT:  # %bb.3:
 ; AVX1-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
@@ -7303,9 +7303,9 @@ define <2 x double> @constrained_vector_uitofp_v2f64_v2i64(<2 x i64> %x) #0 {
 ; AVX512F-LABEL: constrained_vector_uitofp_v2f64_v2i64:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm0
+; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX512F-NEXT:    retq
 ;
@@ -7367,9 +7367,9 @@ define <2 x float> @constrained_vector_uitofp_v2f32_v2i64(<2 x i64> %x) #0 {
 ; AVX1-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
 ; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -7381,9 +7381,9 @@ define <2 x float> @constrained_vector_uitofp_v2f32_v2i64(<2 x i64> %x) #0 {
 ; AVX512-LABEL: constrained_vector_uitofp_v2f32_v2i64:
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX512-NEXT:    retq
 entry:
@@ -7416,24 +7416,24 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i32(<3 x i32> %x) #0 {
 ; AVX1-LABEL: constrained_vector_uitofp_v3f64_v3i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vextractps $1, %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: constrained_vector_uitofp_v3f64_v3i32:
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vextractps $1, %xmm0, %eax
-; AVX512-NEXT:    vcvtusi2sd %eax, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtusi2sd %eax, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vcvtusi2sd %eax, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtusi2sd %eax, %xmm15, %xmm2
 ; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    vcvtusi2sd %eax, %xmm3, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %eax, %xmm15, %xmm0
 ; AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
 entry:
@@ -7465,24 +7465,24 @@ define <3 x float> @constrained_vector_uitofp_v3f32_v3i32(<3 x i32> %x) #0 {
 ; AVX1-LABEL: constrained_vector_uitofp_v3f32_v3i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vextractps $1, %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: constrained_vector_uitofp_v3f32_v3i32:
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vextractps $1, %xmm0, %eax
-; AVX512-NEXT:    vcvtusi2ss %eax, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtusi2ss %eax, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vcvtusi2ss %eax, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtusi2ss %eax, %xmm15, %xmm2
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512-NEXT:    vpextrd $2, %xmm0, %eax
-; AVX512-NEXT:    vcvtusi2ss %eax, %xmm3, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %eax, %xmm15, %xmm0
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX512-NEXT:    retq
 entry:
@@ -7547,7 +7547,7 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i64(<3 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm15, %xmm1
 ; AVX1-NEXT:    jns .LBB183_2
 ; AVX1-NEXT:  # %bb.1:
 ; AVX1-NEXT:    vaddsd %xmm1, %xmm1, %xmm1
@@ -7560,7 +7560,7 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i64(<3 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm15, %xmm2
 ; AVX1-NEXT:    jns .LBB183_4
 ; AVX1-NEXT:  # %bb.3:
 ; AVX1-NEXT:    vaddsd %xmm2, %xmm2, %xmm2
@@ -7575,7 +7575,7 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i64(<3 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rdx, %xmm15, %xmm0
 ; AVX1-NEXT:    jns .LBB183_6
 ; AVX1-NEXT:  # %bb.5:
 ; AVX1-NEXT:    vaddsd %xmm0, %xmm0, %xmm0
@@ -7586,13 +7586,13 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i64(<3 x i64> %x) #0 {
 ; AVX512-LABEL: constrained_vector_uitofp_v3f64_v3i64:
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2sd %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
 entry:
@@ -7657,7 +7657,7 @@ define <3 x float> @constrained_vector_uitofp_v3f32_v3i64(<3 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm1, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm15, %xmm1
 ; AVX1-NEXT:    jns .LBB184_2
 ; AVX1-NEXT:  # %bb.1:
 ; AVX1-NEXT:    vaddss %xmm1, %xmm1, %xmm1
@@ -7670,7 +7670,7 @@ define <3 x float> @constrained_vector_uitofp_v3f32_v3i64(<3 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm15, %xmm2
 ; AVX1-NEXT:    jns .LBB184_4
 ; AVX1-NEXT:  # %bb.3:
 ; AVX1-NEXT:    vaddss %xmm2, %xmm2, %xmm2
@@ -7685,7 +7685,7 @@ define <3 x float> @constrained_vector_uitofp_v3f32_v3i64(<3 x i64> %x) #0 {
 ; AVX1-NEXT:    orq %rcx, %rdx
 ; AVX1-NEXT:    testq %rax, %rax
 ; AVX1-NEXT:    cmovnsq %rax, %rdx
-; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm3, %xmm0
+; AVX1-NEXT:    vcvtsi2ss %rdx, %xmm15, %xmm0
 ; AVX1-NEXT:    jns .LBB184_6
 ; AVX1-NEXT:  # %bb.5:
 ; AVX1-NEXT:    vaddss %xmm0, %xmm0, %xmm0
@@ -7697,13 +7697,13 @@ define <3 x float> @constrained_vector_uitofp_v3f32_v3i64(<3 x i64> %x) #0 {
 ; AVX512-LABEL: constrained_vector_uitofp_v3f32_v3i64:
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -7860,26 +7860,26 @@ define <4 x double> @constrained_vector_uitofp_v4f64_v4i64(<4 x i64> %x) #0 {
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrd $2, %xmm1, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm2, %xmm2
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm2
 ; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm3, %xmm3
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm3[0],xmm2[0]
 ; AVX1-NEXT:    vextractps $2, %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm3
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    movl %eax, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm4, %xmm4
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm3 = xmm4[0],xmm3[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm1
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
 ; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm3
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX1-NEXT:    vcvtsi2sd %rax, %xmm5, %xmm0
+; AVX1-NEXT:    vcvtsi2sd %rax, %xmm15, %xmm0
 ; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vmulpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
@@ -7890,14 +7890,14 @@ define <4 x double> @constrained_vector_uitofp_v4f64_v4i64(<4 x i64> %x) #0 {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm1, %rax
-; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm1
+; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vcvtusi2sd %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -7991,16 +7991,16 @@ define <4 x float> @constrained_vector_uitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; AVX1-NEXT:    vorpd %ymm3, %ymm1, %ymm1
 ; AVX1-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm3
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm3
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm4, %xmm4
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm4
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm4
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vcvtsi2ss %rax, %xmm5, %xmm1
+; AVX1-NEXT:    vcvtsi2ss %rax, %xmm15, %xmm1
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[0]
 ; AVX1-NEXT:    vaddps %xmm1, %xmm1, %xmm3
 ; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
@@ -8011,16 +8011,16 @@ define <4 x float> @constrained_vector_uitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; AVX512F-LABEL: constrained_vector_uitofp_v4f32_v4i64:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm1, %xmm1
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm1
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm2, %xmm2
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512F-NEXT:    vmovq %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm2
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm2
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm3, %xmm0
+; AVX512F-NEXT:    vcvtusi2ss %rax, %xmm15, %xmm0
 ; AVX512F-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-128.ll
index fd0525e6d56a..6b8a03ba5eb7 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -2407,7 +2407,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512F-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2416,14 +2416,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VL-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2432,7 +2432,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
@@ -2441,14 +2441,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VLBW-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-256.ll
index fdd0d68b8900..c6e1aa9cd90c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -2296,7 +2296,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -2304,14 +2304,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512BW-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -2319,7 +2319,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -2327,14 +2327,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VLBW-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX10-LABEL: splatconstant_funnnel_v32i8:
 ; AVX10:       # %bb.0:
 ; AVX10-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX10-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX10-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX10-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; AVX10-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-512.ll
index 1d807fa85ddc..34ad667f0117 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-512.ll
@@ -1124,7 +1124,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
@@ -1137,35 +1137,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index b763b7bac243..e60b56551e58 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -1859,7 +1859,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1868,14 +1868,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1884,14 +1884,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
@@ -1900,7 +1900,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index 9e872cc6d74a..11a02f8cf754 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -443,12 +443,12 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
@@ -463,17 +463,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 | (ymm2 & mem)
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 | (ymm2 & m32bcst)
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
@@ -1641,7 +1641,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -1649,14 +1649,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -1664,14 +1664,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -1679,7 +1679,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index 436fbe31f7a3..4c6680ac4a19 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -721,7 +721,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
@@ -733,35 +733,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-128.ll
index a56b0a6351a3..bf525442a419 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -2412,7 +2412,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512F-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2421,14 +2421,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VL-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2437,7 +2437,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
@@ -2446,14 +2446,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VLBW-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm2))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm2))
 ; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 0fa2c858ff00..9479174d964c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -2096,7 +2096,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -2104,14 +2104,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512BW-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -2119,7 +2119,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -2127,14 +2127,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VLBW-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX10-LABEL: splatconstant_funnnel_v32i8:
 ; AVX10:       # %bb.0:
 ; AVX10-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX10-NEXT:    vpsrlw $4, %ymm1, %ymm0
-; AVX10-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX10-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm2))
 ; AVX10-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-512.ll
index 665223167fbb..3a522ccb6214 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -1166,7 +1166,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
@@ -1179,35 +1179,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index 9ce682306f18..d9799975cd37 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -1928,7 +1928,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1937,14 +1937,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1953,14 +1953,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
@@ -1969,7 +1969,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index 3d4f283260aa..15e09c3b6737 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -469,17 +469,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsllw $6, %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -488,17 +488,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsllw $6, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $1, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
@@ -1693,7 +1693,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -1701,14 +1701,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -1716,14 +1716,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -1731,7 +1731,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index 11ea650e1f02..1d089e427bfa 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -721,7 +721,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
@@ -733,35 +733,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-half-conversions.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-half-conversions.ll
index 1bbf92e45fc6..01159d4135d8 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-half-conversions.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-half-conversions.ll
@@ -5034,7 +5034,7 @@ define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind {
 ; F16C-LABEL: fptoui_2f16_to_4i32:
 ; F16C:       # %bb.0:
 ; F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; F16C-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; F16C-NEXT:    vcvttps2dq %xmm0, %xmm1
 ; F16C-NEXT:    vpsrad $31, %xmm1, %xmm2
@@ -5048,7 +5048,7 @@ define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind {
 ; AVX512F-LABEL: fptoui_2f16_to_4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512F-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
 ; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 3cc17c1f2b86..816d5cace033 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -174,20 +174,19 @@ define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
 ; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
 ; SSE-NEXT:    pmulhw %xmm3, %xmm2
 ; SSE-NEXT:    psrlw $8, %xmm2
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE-NEXT:    pmulhw %xmm3, %xmm1
-; SSE-NEXT:    psrlw $8, %xmm1
-; SSE-NEXT:    packuswb %xmm2, %xmm1
-; SSE-NEXT:    paddb %xmm1, %xmm0
-; SSE-NEXT:    movdqa %xmm0, %xmm1
-; SSE-NEXT:    psrlw $2, %xmm1
-; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; SSE-NEXT:    pxor %xmm2, %xmm1
-; SSE-NEXT:    psrlw $7, %xmm0
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE-NEXT:    pmulhw %xmm3, %xmm4
+; SSE-NEXT:    psrlw $8, %xmm4
+; SSE-NEXT:    packuswb %xmm2, %xmm4
+; SSE-NEXT:    paddb %xmm4, %xmm0
+; SSE-NEXT:    psrlw $2, %xmm0
 ; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    psubb %xmm2, %xmm0
+; SSE-NEXT:    pcmpgtb %xmm0, %xmm1
+; SSE-NEXT:    psubb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: test_div7_16i8:
@@ -197,19 +196,18 @@ define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
 ; AVX1-NEXT:    vpmulhw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX1-NEXT:    vpmulhw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_div7_16i8:
@@ -220,14 +218,14 @@ define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX2NOBW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
-; AVX2NOBW-NEXT:    vpsrlw $2, %xmm0, %xmm1
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpbroadcastb {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX2NOBW-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpsrlw $7, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpsrlw $2, %xmm0, %xmm0
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2NOBW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
-; AVX2NOBW-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX2NOBW-NEXT:    vzeroupper
 ; AVX2NOBW-NEXT:    retq
 ;
@@ -238,14 +236,14 @@ define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
-; AVX512BW-NEXT:    vpsrlw $2, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsrlw $7, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsrlw $2, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
-; AVX512BW-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %res = sdiv <16 x i8> %a, <i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7>
@@ -264,26 +262,25 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
 ; SSE-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [37632,20224,11008,47872,26368,14592,14592,37632]
 ; SSE-NEXT:    psrlw $8, %xmm2
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [37632,33024,14592,26368,47872,11008,20224,37632]
-; SSE-NEXT:    psrlw $8, %xmm1
-; SSE-NEXT:    packuswb %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [37632,33024,14592,26368,47872,11008,20224,37632]
+; SSE-NEXT:    psrlw $8, %xmm3
+; SSE-NEXT:    packuswb %xmm2, %xmm3
 ; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT:    paddb %xmm1, %xmm0
-; SSE-NEXT:    movdqa %xmm0, %xmm1
-; SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; SSE-NEXT:    psraw $8, %xmm1
-; SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [32,64,128,32,64,128,128,64]
-; SSE-NEXT:    psrlw $8, %xmm1
+; SSE-NEXT:    paddb %xmm3, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
 ; SSE-NEXT:    psraw $8, %xmm2
-; SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [64,64,128,64,32,128,64,32]
+; SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [32,64,128,32,64,128,128,64]
 ; SSE-NEXT:    psrlw $8, %xmm2
-; SSE-NEXT:    packuswb %xmm1, %xmm2
-; SSE-NEXT:    psrlw $7, %xmm0
-; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT:    paddb %xmm2, %xmm0
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    psraw $8, %xmm0
+; SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [64,64,128,64,32,128,64,32]
+; SSE-NEXT:    psrlw $8, %xmm0
+; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    pcmpgtb %xmm0, %xmm1
+; SSE-NEXT:    psubb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: test_divconstant_16i8:
@@ -292,24 +289,23 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [37632,20224,11008,47872,26368,14592,14592,37632]
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [37632,33024,14592,26368,47872,11008,20224,37632]
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [37632,33024,14592,26368,47872,11008,20224,37632]
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpsraw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [32,64,128,32,64,128,128,64]
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [32,64,128,32,64,128,128,64]
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_divconstant_16i8:
@@ -321,14 +317,14 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX2NOBW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
-; AVX2NOBW-NEXT:    vpmovsxbw %xmm0, %ymm1
-; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [64,64,128,64,32,128,64,32,32,64,128,32,64,128,128,64]
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2NOBW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX2NOBW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [64,64,128,64,32,128,64,32,32,64,128,32,64,128,128,64]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX2NOBW-NEXT:    vzeroupper
 ; AVX2NOBW-NEXT:    retq
 ;
@@ -341,12 +337,12 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm2
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
-; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm2
-; AVX512BW-NEXT:    vpsravw %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
-; AVX512BW-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %res = sdiv <16 x i8> %a, <i8 7, i8 8, i8 9, i8 10,i8 11, i8 12, i8 13, i8 14, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9,i8 9, i8 7>
@@ -568,25 +564,24 @@ define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
 ; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
 ; SSE-NEXT:    pmulhw %xmm3, %xmm2
 ; SSE-NEXT:    psrlw $8, %xmm2
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE-NEXT:    pmulhw %xmm3, %xmm1
-; SSE-NEXT:    psrlw $8, %xmm1
-; SSE-NEXT:    packuswb %xmm2, %xmm1
-; SSE-NEXT:    paddb %xmm0, %xmm1
-; SSE-NEXT:    movdqa %xmm1, %xmm2
-; SSE-NEXT:    psrlw $2, %xmm2
-; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; SSE-NEXT:    pxor %xmm3, %xmm2
-; SSE-NEXT:    psrlw $7, %xmm1
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE-NEXT:    pmulhw %xmm3, %xmm4
+; SSE-NEXT:    psrlw $8, %xmm4
+; SSE-NEXT:    packuswb %xmm2, %xmm4
+; SSE-NEXT:    paddb %xmm0, %xmm4
+; SSE-NEXT:    psrlw $2, %xmm4
+; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; SSE-NEXT:    pxor %xmm2, %xmm4
+; SSE-NEXT:    psubb %xmm2, %xmm4
+; SSE-NEXT:    pcmpgtb %xmm4, %xmm1
+; SSE-NEXT:    psubb %xmm1, %xmm4
+; SSE-NEXT:    movdqa %xmm4, %xmm1
+; SSE-NEXT:    psllw $3, %xmm1
 ; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE-NEXT:    paddb %xmm2, %xmm1
-; SSE-NEXT:    psubb %xmm3, %xmm1
-; SSE-NEXT:    movdqa %xmm1, %xmm2
-; SSE-NEXT:    psllw $3, %xmm2
-; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT:    psubb %xmm2, %xmm1
-; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    psubb %xmm1, %xmm4
+; SSE-NEXT:    paddb %xmm4, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: test_rem7_16i8:
@@ -596,19 +591,18 @@ define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
 ; AVX1-NEXT:    vpmulhw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX1-NEXT:    vpmulhw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpsllw $3, %xmm1, %xmm2
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
@@ -623,14 +617,14 @@ define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX2NOBW-NEXT:    vpaddb %xmm0, %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpsrlw $2, %xmm1, %xmm2
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX2NOBW-NEXT:    vpbroadcastb {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX2NOBW-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX2NOBW-NEXT:    vpsrlw $7, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpsrlw $2, %xmm1, %xmm1
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
-; AVX2NOBW-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpbroadcastb {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX2NOBW-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; AVX2NOBW-NEXT:    vpsllw $3, %xmm1, %xmm2
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; AVX2NOBW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
@@ -645,14 +639,14 @@ define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vpaddb %xmm0, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsrlw $2, %xmm1, %xmm2
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpsrlw $7, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsrlw $2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512BW-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
-; AVX512BW-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpsllw $3, %xmm1, %xmm2
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
@@ -675,27 +669,26 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; SSE2-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [37632,20224,11008,47872,26368,14592,14592,37632]
 ; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [37632,33024,14592,26368,47872,11008,20224,37632]
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    packuswb %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [37632,33024,14592,26368,47872,11008,20224,37632]
+; SSE2-NEXT:    psrlw $8, %xmm3
+; SSE2-NEXT:    packuswb %xmm1, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,0,255,0,0,255,255,0,0,255,0,0,0,255]
 ; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    paddb %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [32,64,128,32,64,128,128,64]
-; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    paddb %xmm3, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
 ; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [64,64,128,64,32,128,64,32]
+; SSE2-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [32,64,128,32,64,128,128,64]
 ; SSE2-NEXT:    psrlw $8, %xmm3
-; SSE2-NEXT:    packuswb %xmm2, %xmm3
-; SSE2-NEXT:    psrlw $7, %xmm1
-; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    paddb %xmm3, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [64,64,128,64,32,128,64,32]
+; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    psubb %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [14,13,12,11,10,9,9,7]
@@ -710,39 +703,38 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ;
 ; SSE41-LABEL: test_remconstant_16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; SSE41-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [37632,20224,11008,47872,26368,14592,14592,37632]
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE41-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [37632,33024,14592,26368,47872,11008,20224,37632]
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    packuswb %xmm1, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,0,255,0,0,255,255,0,0,255,0,0,0,255]
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    paddb %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; SSE41-NEXT:    psraw $8, %xmm2
-; SSE41-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [32,64,128,32,64,128,128,64]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE41-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [37632,20224,11008,47872,26368,14592,14592,37632]
 ; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; SSE41-NEXT:    psraw $8, %xmm3
-; SSE41-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [64,64,128,64,32,128,64,32]
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE41-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [37632,33024,14592,26368,47872,11008,20224,37632]
 ; SSE41-NEXT:    psrlw $8, %xmm3
 ; SSE41-NEXT:    packuswb %xmm2, %xmm3
-; SSE41-NEXT:    psrlw $7, %xmm1
-; SSE41-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT:    paddb %xmm3, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [0,8,0,10,0,12,0,14,0,13,0,11,0,9,0,7]
-; SSE41-NEXT:    psllw $8, %xmm2
-; SSE41-NEXT:    pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [7,0,9,0,11,0,13,0,14,0,12,0,10,0,9,0]
-; SSE41-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT:    por %xmm2, %xmm1
-; SSE41-NEXT:    psubb %xmm1, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,0,0,255,0,0,255,255,0,0,255,0,0,0,255]
+; SSE41-NEXT:    pand %xmm0, %xmm2
+; SSE41-NEXT:    paddb %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    psraw $8, %xmm3
+; SSE41-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [32,64,128,32,64,128,128,64]
+; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE41-NEXT:    psraw $8, %xmm2
+; SSE41-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [64,64,128,64,32,128,64,32]
+; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
+; SSE41-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE41-NEXT:    psubb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [0,8,0,10,0,12,0,14,0,13,0,11,0,9,0,7]
+; SSE41-NEXT:    psllw $8, %xmm1
+; SSE41-NEXT:    pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [7,0,9,0,11,0,13,0,14,0,12,0,10,0,9,0]
+; SSE41-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE41-NEXT:    por %xmm1, %xmm2
+; SSE41-NEXT:    psubb %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_remconstant_16i8:
@@ -751,24 +743,23 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [37632,20224,11008,47872,26368,14592,14592,37632]
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [37632,33024,14592,26368,47872,11008,20224,37632]
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
-; AVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [32,64,128,32,64,128,128,64]
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [37632,33024,14592,26368,47872,11008,20224,37632]
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
+; AVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [32,64,128,32,64,128,128,64]
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2 # [0,8,0,10,0,12,0,14,0,13,0,11,0,9,0,7]
 ; AVX1-NEXT:    vpsllw $8, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [7,0,9,0,11,0,13,0,14,0,12,0,10,0,9,0]
@@ -786,14 +777,14 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; AVX2NOBW-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpmovsxbw %xmm1, %ymm2
-; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [64,64,128,64,32,128,64,32,32,64,128,32,64,128,128,64]
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX2NOBW-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX2NOBW-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2NOBW-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
+; AVX2NOBW-NEXT:    vpmovsxbw %xmm1, %ymm1
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [64,64,128,64,32,128,64,32,32,64,128,32,64,128,128,64]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm2
+; AVX2NOBW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; AVX2NOBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [7,8,9,10,11,12,13,14,14,13,12,11,10,9,9,7]
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -812,12 +803,12 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm2
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
 ; AVX512BW-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovsxbw %xmm2, %ymm3
-; AVX512BW-NEXT:    vpsravw %zmm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpmovsxbw %xmm2, %ymm2
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
-; AVX512BW-NEXT:    vpsrlw $7, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [7,8,9,10,11,12,13,14,14,13,12,11,10,9,9,7]
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
@@ -1118,3 +1109,103 @@ define <16 x i8> @test_rem_variable_16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
   %res = srem <16 x i8> %a, %b
   ret <16 x i8> %res
 }
+
+define <16 x i8> @PR143238(<16 x i8> %a0) {
+; SSE-LABEL: PR143238:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [26368,47872,11008,20224,37632,35072,33024,30976]
+; SSE-NEXT:    psrlw $8, %xmm2
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE-NEXT:    pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [33024,22016,33024,26368,11008,37632,33024,14592]
+; SSE-NEXT:    psrlw $8, %xmm3
+; SSE-NEXT:    packuswb %xmm2, %xmm3
+; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT:    paddb %xmm3, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE-NEXT:    psraw $8, %xmm2
+; SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [64,32,128,64,32,32,32,32]
+; SSE-NEXT:    psrlw $8, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    psraw $8, %xmm0
+; SSE-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [256,256,128,128,256,64,64,128]
+; SSE-NEXT:    psrlw $8, %xmm0
+; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    pcmpgtb %xmm0, %xmm1
+; SSE-NEXT:    psubb %xmm1, %xmm0
+; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: PR143238:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [26368,47872,11008,20224,37632,35072,33024,30976]
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [33024,22016,33024,26368,11008,37632,33024,14592]
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [64,32,128,64,32,32,32,32]
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [256,256,128,128,256,64,64,128]
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2NOBW-LABEL: PR143238:
+; AVX2NOBW:       # %bb.0:
+; AVX2NOBW-NEXT:    vpmovsxbw %xmm0, %ymm1
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [65409,86,65409,103,43,65427,65409,57,103,65467,43,79,65427,65417,65409,121]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2NOBW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; AVX2NOBW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [256,256,128,128,256,64,64,128,64,32,128,64,32,32,32,32]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2NOBW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2NOBW-NEXT:    vzeroupper
+; AVX2NOBW-NEXT:    retq
+;
+; AVX512BW-LABEL: PR143238:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,0,1,1,0,2,2,1,2,3,1,2,3,3,3,3]
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm2
+; AVX512BW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [65409,86,65409,103,43,65427,65409,57,103,65467,43,79,65427,65417,65409,121]
+; AVX512BW-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm2
+; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
+; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %sdiv = sdiv <16 x i8> %a0, <i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17>
+  %mask = and <16 x i8> %sdiv, splat (i8 1)
+  ret <16 x i8> %mask
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index 3b9ac630ab25..63c69e59d00e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -169,31 +169,28 @@ define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm3
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX1-NEXT:    vpxor %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpsubb %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; AVX1-NEXT:    vpmulhw %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX1-NEXT:    vpmulhw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm2
-; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm6
+; AVX1-NEXT:    vpsubb %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT:    vpackuswb %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpaddb %xmm0, %xmm4, %xmm0
 ; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm7, %xmm0, %xmm0
-; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubb %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -204,19 +201,18 @@ define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vpbroadcastw {{.*#+}} ymm3 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
 ; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
-; AVX2NOBW-NEXT:    vpsrlw $2, %ymm0, %ymm1
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX2NOBW-NEXT:    vpxor %ymm2, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $7, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX2NOBW-NEXT:    vpsrlw $2, %ymm0, %ymm0
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX2NOBW-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX2NOBW-NEXT:    vpxor %ymm2, %ymm0, %ymm0
 ; AVX2NOBW-NEXT:    vpsubb %ymm2, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX2NOBW-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_div7_32i8:
@@ -226,14 +222,14 @@ define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT:    vpsrlw $2, %ymm0, %ymm1
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpxor %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpsrlw $7, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpsrlw $2, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT:    vpsubb %ymm2, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} ymm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    retq
   %res = sdiv <32 x i8> %a, <i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7>
   ret <32 x i8> %res
@@ -246,52 +242,49 @@ define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: test_divconstant_32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
 ; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [37632,20224,11008,47872,26368,14592,33024,37632]
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [47872,12544,26368,6912,14592,30976,33024,35072]
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [47872,12544,26368,6912,14592,30976,33024,35072]
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpaddb %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm4, %xmm4
 ; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4 # [32,64,128,32,64,128,64,64]
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpsraw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5, %xmm5 # [16,64,32,128,64,32,32,32]
-; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [16,64,32,128,64,32,32,32]
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm1, %xmm4
+; AVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4 # [35072,33024,30976,14592,6912,26368,12544,47872]
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [37632,33024,14592,26368,47872,11008,20224,37632]
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [32,32,32,64,128,32,64,16]
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [32,32,32,64,128,32,64,16]
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_divconstant_32i8:
@@ -300,24 +293,23 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX2NOBW-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [35072,33024,30976,14592,6912,26368,12544,47872,37632,20224,11008,47872,26368,14592,33024,37632]
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX2NOBW-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [37632,33024,14592,26368,47872,11008,20224,37632,47872,12544,26368,6912,14592,30976,33024,35072]
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [37632,33024,14592,26368,47872,11008,20224,37632,47872,12544,26368,6912,14592,30976,33024,35072]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
-; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [32,32,32,64,128,32,64,16,32,64,128,32,64,128,64,64]
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
+; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX2NOBW-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [64,64,128,64,32,128,64,32,16,64,32,128,64,32,32,32]
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [32,32,32,64,128,32,64,16,32,64,128,32,64,128,64,64]
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpackuswb %ymm1, %ymm2, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $7, %ymm0, %ymm0
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2NOBW-NEXT:    vpsraw $8, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [64,64,128,64,32,128,64,32,16,64,32,128,64,32,32,32]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2NOBW-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX2NOBW-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_divconstant_32i8:
@@ -328,12 +320,12 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpsrlw $7, %ymm0, %ymm1
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
 ; AVX512BW-NEXT:    vpsravw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    retq
   %res = sdiv <32 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
   ret <32 x i8> %res
@@ -544,52 +536,49 @@ define <16 x i16> @test_rem7_16i16(<16 x i16> %a) nounwind {
 define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: test_rem7_32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
 ; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [37632,37632,37632,37632,37632,37632,37632,37632]
 ; AVX1-NEXT:    vpmulhw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; AVX1-NEXT:    vpmulhw %xmm4, %xmm5, %xmm5
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $7, %xmm3, %xmm5
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $2, %xmm3, %xmm3
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm8 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX1-NEXT:    vpxor %xmm3, %xmm8, %xmm3
-; AVX1-NEXT:    vpaddb %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpsubb %xmm8, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllw $3, %xmm3, %xmm5
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm9 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX1-NEXT:    vpand %xmm5, %xmm9, %xmm5
-; AVX1-NEXT:    vpsubb %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubb %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm1, %xmm7
+; AVX1-NEXT:    vpsubb %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpsllw $3, %xmm3, %xmm7
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm8 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
+; AVX1-NEXT:    vpand %xmm7, %xmm8, %xmm7
+; AVX1-NEXT:    vpsubb %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX1-NEXT:    vpmulhw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX1-NEXT:    vpmulhw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
-; AVX1-NEXT:    vpand %xmm7, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm8, %xmm2
-; AVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubb %xmm8, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllw $3, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm3, %xmm9, %xmm3
-; AVX1-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpaddb %xmm0, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $2, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubb %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpsllw $3, %xmm1, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
+; AVX1-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_rem7_32i8:
@@ -599,19 +588,18 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vpbroadcastw {{.*#+}} ymm3 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
 ; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $2, %ymm1, %ymm2
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm4, %ymm3
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpsrlw $2, %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
 ; AVX2NOBW-NEXT:    vpxor %ymm3, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
-; AVX2NOBW-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsubb %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpcmpgtb %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; AVX2NOBW-NEXT:    vpsllw $3, %ymm1, %ymm2
 ; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
@@ -625,14 +613,14 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vpaddb %ymm0, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpsrlw $2, %ymm1, %ymm2
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpxor %ymm3, %ymm2, %ymm2
-; AVX512BW-NEXT:    vpsrlw $7, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsrlw $2, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512BW-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
-; AVX512BW-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpcmpgtb %ymm1, %ymm2, %ymm2
+; AVX512BW-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpsllw $3, %ymm1, %ymm2
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; AVX512BW-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
@@ -665,18 +653,16 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpsraw $8, %xmm5, %xmm5
 ; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5, %xmm5 # [32,64,128,32,64,128,64,64]
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpsraw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6, %xmm6 # [16,64,32,128,64,32,32,32]
-; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vpsrlw $7, %xmm3, %xmm3
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [16,64,32,128,64,32,32,32]
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm1, %xmm5
+; AVX1-NEXT:    vpsubb %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm5 # [22,0,20,0,18,0,16,0,14,0,12,0,10,0,8,0]
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
 ; AVX1-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [0,21,0,19,0,17,0,15,0,13,0,11,0,9,0,7]
 ; AVX1-NEXT:    vpsllw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpor %xmm3, %xmm5, %xmm3
@@ -684,25 +670,24 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [35072,33024,30976,14592,6912,26368,12544,47872]
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [37632,33024,14592,26368,47872,11008,20224,37632]
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [32,32,32,64,128,32,64,16]
-; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5, %xmm5 # [37632,33024,14592,26368,47872,11008,20224,37632]
+; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddb %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4 # [32,32,32,64,128,32,64,16]
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [64,64,128,64,32,128,64,32]
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 # [7,0,9,0,11,0,13,0,15,0,17,0,19,0,21,0]
-; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [0,8,0,10,0,12,0,14,0,16,0,18,0,20,0,22]
 ; AVX1-NEXT:    vpsllw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
@@ -716,24 +701,23 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX2NOBW-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [35072,33024,30976,14592,6912,26368,12544,47872,37632,20224,11008,47872,26368,14592,33024,37632]
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX2NOBW-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [37632,33024,14592,26368,47872,11008,20224,37632,47872,12544,26368,6912,14592,30976,33024,35072]
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2NOBW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [32,32,32,64,128,32,64,16,32,64,128,32,64,128,64,64]
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [64,64,128,64,32,128,64,32,16,64,32,128,64,32,32,32]
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [37632,33024,14592,26368,47872,11008,20224,37632,47872,12544,26368,6912,14592,30976,33024,35072]
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
-; AVX2NOBW-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
+; AVX2NOBW-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2NOBW-NEXT:    vpsraw $8, %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [32,32,32,64,128,32,64,16,32,64,128,32,64,128,64,64]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2NOBW-NEXT:    vpsraw $8, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [64,64,128,64,32,128,64,32,16,64,32,128,64,32,32,32]
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpcmpgtb %ymm2, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; AVX2NOBW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2 # [0,8,0,10,0,12,0,14,0,16,0,18,0,20,0,22,0,21,0,19,0,17,0,15,0,13,0,11,0,9,0,7]
 ; AVX2NOBW-NEXT:    vpsllw $8, %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [7,0,9,0,11,0,13,0,15,0,17,0,19,0,21,0,22,0,20,0,18,0,16,0,14,0,12,0,10,0,8,0]
@@ -750,12 +734,12 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpmovwb %zmm2, %ymm2
 ; AVX512BW-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
-; AVX512BW-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; AVX512BW-NEXT:    vpmovsxbw %ymm1, %zmm1
 ; AVX512BW-NEXT:    vpsravw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
-; AVX512BW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpcmpgtb %ymm1, %ymm2, %ymm2
+; AVX512BW-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512BW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7]
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 6256c4f2acf7..6bc4fcb6cc1e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -140,31 +140,28 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm3
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm7 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512F-NEXT:    vpxor %ymm7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
-; AVX512F-NEXT:    vpmulhw %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
-; AVX512F-NEXT:    vpmulhw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512F-NEXT:    vpxor %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm2, %ymm6
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm6, %ymm6
+; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm7, %ymm4
+; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpackuswb %ymm6, %ymm4, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm4, %ymm0
 ; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm6, %ymm0, %ymm0
-; AVX512F-NEXT:    vpxor %ymm7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpxor %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsubb %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -180,13 +177,13 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm2 ^ (zmm1 & mem)
-; AVX512BW-NEXT:    vpsrlw $7, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpsubb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m32bcst)
+; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
+; AVX512BW-NEXT:    vpmovm2b %k0, %zmm1
+; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %res = sdiv <64 x i8> %a, <i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7,i8 7, i8 7, i8 7, i8 7>
   ret <64 x i8> %res
@@ -199,52 +196,49 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_divconstant_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
 ; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [35072,18176,37632,4864,20224,10496,11008,45824,37632,20224,11008,47872,26368,14592,33024,37632]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
-; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [6912,28416,14592,15104,30976,32000,33024,34048,47872,12544,26368,6912,14592,30976,33024,35072]
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
+; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [6912,28416,14592,15104,30976,32000,33024,34048,47872,12544,26368,6912,14592,30976,33024,35072]
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpaddb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [16,32,16,128,32,64,64,16,32,64,128,32,64,128,64,64]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5 # [64,16,32,32,16,16,16,16,16,64,32,128,64,32,32,32]
-; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [64,16,32,32,16,16,16,16,16,64,32,128,64,32,32,32]
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm1, %ymm4
+; AVX512F-NEXT:    vpsubb %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [35072,33024,30976,14592,6912,26368,12544,47872,34048,33024,32000,30976,15104,14592,28416,6912]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [37632,33024,14592,26368,47872,11008,20224,37632,45824,11008,10496,20224,4864,37632,18176,35072]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [32,32,32,64,128,32,64,16,16,16,16,16,32,32,16,64]
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [64,64,128,64,32,128,64,32,16,64,64,32,128,16,32,16]
+; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [32,32,32,64,128,32,64,16,16,16,16,16,32,32,16,64]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpsraw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [64,64,128,64,32,128,64,32,16,64,64,32,128,16,32,16]
+; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_divconstant_64i8:
@@ -263,14 +257,14 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsraw $8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
-; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    vpsrlw $7, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpsraw $8, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
+; AVX512BW-NEXT:    vpmovm2b %k0, %zmm1
+; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %res = sdiv <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
   ret <64 x i8> %res
@@ -447,52 +441,49 @@ define <32 x i16> @test_rem7_32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_rem7_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
 ; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
 ; AVX512F-NEXT:    vpmulhw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
 ; AVX512F-NEXT:    vpmulhw %ymm4, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm5
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $2, %ymm3, %ymm3
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm8 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512F-NEXT:    vpxor %ymm3, %ymm8, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsllw $3, %ymm3, %ymm5
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm9 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX512F-NEXT:    vpand %ymm5, %ymm9, %ymm5
-; AVX512F-NEXT:    vpsubb %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512F-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm7
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsllw $3, %ymm3, %ymm7
+; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm8 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
+; AVX512F-NEXT:    vpand %ymm7, %ymm8, %ymm7
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512F-NEXT:    vpmulhw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
-; AVX512F-NEXT:    vpmulhw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm3
-; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpxor %ymm2, %ymm8, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsllw $3, %ymm2, %ymm3
-; AVX512F-NEXT:    vpand %ymm3, %ymm9, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm7, %ymm4
+; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $2, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsllw $3, %ymm1, %ymm3
+; AVX512F-NEXT:    vpand %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_rem7_64i8:
@@ -507,13 +498,13 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsrlw $2, %zmm1, %zmm2
-; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm3 ^ (zmm2 & mem)
-; AVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpaddb %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    vpsubb %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw $2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm2 ^ (zmm1 & m32bcst)
+; AVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpmovb2m %zmm1, %k0
+; AVX512BW-NEXT:    vpmovm2b %k0, %zmm2
+; AVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsllw $3, %zmm1, %zmm2
 ; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm1
@@ -546,18 +537,16 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpsraw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5 # [16,32,16,128,32,64,64,16,32,64,128,32,64,128,64,64]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm6 # [64,16,32,32,16,16,16,16,16,64,32,128,64,32,32,32]
-; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [64,16,32,32,16,16,16,16,16,64,32,128,64,32,32,32]
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm5
+; AVX512F-NEXT:    vpsubb %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm5 # [38,0,36,0,34,0,32,0,30,0,28,0,26,0,24,0,22,0,20,0,18,0,16,0,14,0,12,0,10,0,8,0]
-; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [0,37,0,35,0,33,0,31,0,29,0,27,0,25,0,23,0,21,0,19,0,17,0,15,0,13,0,11,0,9,0,7]
 ; AVX512F-NEXT:    vpsllw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpor %ymm3, %ymm5, %ymm3
@@ -565,25 +554,24 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [35072,33024,30976,14592,6912,26368,12544,47872,34048,33024,32000,30976,15104,14592,28416,6912]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [37632,33024,14592,26368,47872,11008,20224,37632,45824,11008,10496,20224,4864,37632,18176,35072]
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [32,32,32,64,128,32,64,16,16,16,16,16,32,32,16,64]
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512F-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5 # [37632,33024,14592,26368,47872,11008,20224,37632,45824,11008,10496,20224,4864,37632,18176,35072]
+; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [64,64,128,64,32,128,64,32,16,64,64,32,128,16,32,16]
+; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [32,32,32,64,128,32,64,16,16,16,16,16,32,32,16,64]
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [64,64,128,64,32,128,64,32,16,64,64,32,128,16,32,16]
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
 ; AVX512F-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3 # [7,0,9,0,11,0,13,0,15,0,17,0,19,0,21,0,23,0,25,0,27,0,29,0,31,0,33,0,35,0,37,0]
-; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,8,0,10,0,12,0,14,0,16,0,18,0,20,0,22,0,24,0,26,0,28,0,30,0,32,0,34,0,36,0,38]
 ; AVX512F-NEXT:    vpsllw $8, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
@@ -607,18 +595,18 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpsraw $8, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
-; AVX512BW-NEXT:    vpsrlw $8, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpaddb %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpsraw $8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpmovb2m %zmm1, %k0
+; AVX512BW-NEXT:    vpmovm2b %k0, %zmm2
+; AVX512BW-NEXT:    vpsubb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2 # [7,0,9,0,11,0,13,0,15,0,17,0,19,0,21,0,23,0,25,0,27,0,29,0,31,0,33,0,35,0,37,0,38,0,36,0,34,0,32,0,30,0,28,0,26,0,24,0,22,0,20,0,18,0,16,0,14,0,12,0,10,0,8,0]
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,8,0,10,0,12,0,14,0,16,0,18,0,20,0,22,0,24,0,26,0,28,0,30,0,32,0,34,0,36,0,38,0,37,0,35,0,33,0,31,0,29,0,27,0,25,0,23,0,21,0,19,0,17,0,15,0,13,0,11,0,9,0,7]
 ; AVX512BW-NEXT:    vpsllw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm2 & mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm2 & m32bcst)
 ; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %res = srem <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index fd7a4c9b8d5a..9c56894f0c59 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -651,7 +651,7 @@ define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2 # [7,0,9,0,11,0,13,0,15,0,17,0,19,0,21,0,23,0,25,0,27,0,29,0,31,0,33,0,35,0,37,0,38,0,36,0,34,0,32,0,30,0,28,0,26,0,24,0,22,0,20,0,18,0,16,0,14,0,12,0,10,0,8,0]
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,8,0,10,0,12,0,14,0,16,0,18,0,20,0,22,0,24,0,26,0,28,0,30,0,32,0,34,0,36,0,38,0,37,0,35,0,33,0,31,0,29,0,27,0,25,0,23,0,21,0,19,0,17,0,15,0,13,0,11,0,9,0,7]
 ; AVX512BW-NEXT:    vpsllw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm2 & mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 | (zmm2 & m32bcst)
 ; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %res = urem <64 x i8> %a, <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleave.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleave.ll
index 206f7ed43fd6..aec3140c73fb 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleave.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleave.ll
@@ -576,12 +576,12 @@ define void @splat2_i64(ptr %s, ptr %d) {
 ;
 ; AVX1-LABEL: splat2_i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vperm2f128 $51, (%rdi), %ymm0, %ymm0 # ymm0 = mem[2,3,2,3]
+; AVX1-NEXT:    vbroadcastf128 (%rdi), %ymm0 # ymm0 = mem[0,1,0,1]
 ; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0,0,3,3]
-; AVX1-NEXT:    vbroadcastf128 (%rdi), %ymm1 # ymm1 = mem[0,1,0,1]
+; AVX1-NEXT:    vbroadcastf128 16(%rdi), %ymm1 # ymm1 = mem[0,1,0,1]
 ; AVX1-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
-; AVX1-NEXT:    vmovupd %ymm0, 32(%rsi)
-; AVX1-NEXT:    vmovupd %ymm1, (%rsi)
+; AVX1-NEXT:    vmovupd %ymm1, 32(%rsi)
+; AVX1-NEXT:    vmovupd %ymm0, (%rsi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
index b3d8d05f6994..dbb4b9f64f4b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
@@ -1105,19 +1105,18 @@ define void @load_i16_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
 ; AVX512-VL-NEXT:    vmovdqa64 64(%rdi), %zmm1
 ; AVX512-VL-NEXT:    vmovdqa64 128(%rdi), %zmm2
 ; AVX512-VL-NEXT:    vmovdqa64 192(%rdi), %zmm3
-; AVX512-VL-NEXT:    vpmovdw %zmm1, %ymm4
-; AVX512-VL-NEXT:    vpsrld $16, %zmm1, %zmm1
-; AVX512-VL-NEXT:    vpsrld $16, %zmm0, %zmm5
-; AVX512-VL-NEXT:    vpsrld $16, %zmm3, %zmm6
-; AVX512-VL-NEXT:    vpsrld $16, %zmm2, %zmm7
+; AVX512-VL-NEXT:    vpsrld $16, %zmm0, %zmm4
+; AVX512-VL-NEXT:    vpsrld $16, %zmm1, %zmm5
+; AVX512-VL-NEXT:    vpsrld $16, %zmm2, %zmm6
+; AVX512-VL-NEXT:    vpsrld $16, %zmm3, %zmm7
+; AVX512-VL-NEXT:    vpmovdw %zmm1, 32(%rsi)
 ; AVX512-VL-NEXT:    vpmovdw %zmm0, (%rsi)
-; AVX512-VL-NEXT:    vmovdqa %ymm4, 32(%rsi)
-; AVX512-VL-NEXT:    vpmovdw %zmm2, 64(%rsi)
 ; AVX512-VL-NEXT:    vpmovdw %zmm3, 96(%rsi)
-; AVX512-VL-NEXT:    vpmovdw %zmm7, 64(%rdx)
-; AVX512-VL-NEXT:    vpmovdw %zmm6, 96(%rdx)
-; AVX512-VL-NEXT:    vpmovdw %zmm5, (%rdx)
-; AVX512-VL-NEXT:    vpmovdw %zmm1, 32(%rdx)
+; AVX512-VL-NEXT:    vpmovdw %zmm2, 64(%rsi)
+; AVX512-VL-NEXT:    vpmovdw %zmm7, 96(%rdx)
+; AVX512-VL-NEXT:    vpmovdw %zmm6, 64(%rdx)
+; AVX512-VL-NEXT:    vpmovdw %zmm5, 32(%rdx)
+; AVX512-VL-NEXT:    vpmovdw %zmm4, (%rdx)
 ; AVX512-VL-NEXT:    vzeroupper
 ; AVX512-VL-NEXT:    retq
 ;
@@ -1127,19 +1126,18 @@ define void @load_i16_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
 ; AVX512-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm1
 ; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %zmm2
 ; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %zmm3
-; AVX512-FCP-NEXT:    vpmovdw %zmm1, %ymm4
-; AVX512-FCP-NEXT:    vpsrld $16, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpsrld $16, %zmm0, %zmm5
-; AVX512-FCP-NEXT:    vpsrld $16, %zmm3, %zmm6
-; AVX512-FCP-NEXT:    vpsrld $16, %zmm2, %zmm7
+; AVX512-FCP-NEXT:    vpsrld $16, %zmm0, %zmm4
+; AVX512-FCP-NEXT:    vpsrld $16, %zmm1, %zmm5
+; AVX512-FCP-NEXT:    vpsrld $16, %zmm2, %zmm6
+; AVX512-FCP-NEXT:    vpsrld $16, %zmm3, %zmm7
+; AVX512-FCP-NEXT:    vpmovdw %zmm1, 32(%rsi)
 ; AVX512-FCP-NEXT:    vpmovdw %zmm0, (%rsi)
-; AVX512-FCP-NEXT:    vmovdqa %ymm4, 32(%rsi)
-; AVX512-FCP-NEXT:    vpmovdw %zmm2, 64(%rsi)
 ; AVX512-FCP-NEXT:    vpmovdw %zmm3, 96(%rsi)
-; AVX512-FCP-NEXT:    vpmovdw %zmm7, 64(%rdx)
-; AVX512-FCP-NEXT:    vpmovdw %zmm6, 96(%rdx)
-; AVX512-FCP-NEXT:    vpmovdw %zmm5, (%rdx)
-; AVX512-FCP-NEXT:    vpmovdw %zmm1, 32(%rdx)
+; AVX512-FCP-NEXT:    vpmovdw %zmm2, 64(%rsi)
+; AVX512-FCP-NEXT:    vpmovdw %zmm7, 96(%rdx)
+; AVX512-FCP-NEXT:    vpmovdw %zmm6, 64(%rdx)
+; AVX512-FCP-NEXT:    vpmovdw %zmm5, 32(%rdx)
+; AVX512-FCP-NEXT:    vpmovdw %zmm4, (%rdx)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -1149,19 +1147,18 @@ define void @load_i16_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
 ; AVX512DQ-NEXT:    vmovdqa64 64(%rdi), %zmm1
 ; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %zmm2
 ; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %zmm3
-; AVX512DQ-NEXT:    vpmovdw %zmm1, %ymm4
-; AVX512DQ-NEXT:    vpsrld $16, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpsrld $16, %zmm0, %zmm5
-; AVX512DQ-NEXT:    vpsrld $16, %zmm3, %zmm6
-; AVX512DQ-NEXT:    vpsrld $16, %zmm2, %zmm7
+; AVX512DQ-NEXT:    vpsrld $16, %zmm0, %zmm4
+; AVX512DQ-NEXT:    vpsrld $16, %zmm1, %zmm5
+; AVX512DQ-NEXT:    vpsrld $16, %zmm2, %zmm6
+; AVX512DQ-NEXT:    vpsrld $16, %zmm3, %zmm7
+; AVX512DQ-NEXT:    vpmovdw %zmm1, 32(%rsi)
 ; AVX512DQ-NEXT:    vpmovdw %zmm0, (%rsi)
-; AVX512DQ-NEXT:    vmovdqa %ymm4, 32(%rsi)
-; AVX512DQ-NEXT:    vpmovdw %zmm2, 64(%rsi)
 ; AVX512DQ-NEXT:    vpmovdw %zmm3, 96(%rsi)
-; AVX512DQ-NEXT:    vpmovdw %zmm7, 64(%rdx)
-; AVX512DQ-NEXT:    vpmovdw %zmm6, 96(%rdx)
-; AVX512DQ-NEXT:    vpmovdw %zmm5, (%rdx)
-; AVX512DQ-NEXT:    vpmovdw %zmm1, 32(%rdx)
+; AVX512DQ-NEXT:    vpmovdw %zmm2, 64(%rsi)
+; AVX512DQ-NEXT:    vpmovdw %zmm7, 96(%rdx)
+; AVX512DQ-NEXT:    vpmovdw %zmm6, 64(%rdx)
+; AVX512DQ-NEXT:    vpmovdw %zmm5, 32(%rdx)
+; AVX512DQ-NEXT:    vpmovdw %zmm4, (%rdx)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -1171,19 +1168,18 @@ define void @load_i16_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %zmm3
-; AVX512DQ-FCP-NEXT:    vpmovdw %zmm1, %ymm4
-; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm0, %zmm5
-; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm3, %zmm6
-; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm2, %zmm7
+; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm0, %zmm4
+; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm1, %zmm5
+; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm2, %zmm6
+; AVX512DQ-FCP-NEXT:    vpsrld $16, %zmm3, %zmm7
+; AVX512DQ-FCP-NEXT:    vpmovdw %zmm1, 32(%rsi)
 ; AVX512DQ-FCP-NEXT:    vpmovdw %zmm0, (%rsi)
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm4, 32(%rsi)
-; AVX512DQ-FCP-NEXT:    vpmovdw %zmm2, 64(%rsi)
 ; AVX512DQ-FCP-NEXT:    vpmovdw %zmm3, 96(%rsi)
-; AVX512DQ-FCP-NEXT:    vpmovdw %zmm7, 64(%rdx)
-; AVX512DQ-FCP-NEXT:    vpmovdw %zmm6, 96(%rdx)
-; AVX512DQ-FCP-NEXT:    vpmovdw %zmm5, (%rdx)
-; AVX512DQ-FCP-NEXT:    vpmovdw %zmm1, 32(%rdx)
+; AVX512DQ-FCP-NEXT:    vpmovdw %zmm2, 64(%rsi)
+; AVX512DQ-FCP-NEXT:    vpmovdw %zmm7, 96(%rdx)
+; AVX512DQ-FCP-NEXT:    vpmovdw %zmm6, 64(%rdx)
+; AVX512DQ-FCP-NEXT:    vpmovdw %zmm5, 32(%rdx)
+; AVX512DQ-FCP-NEXT:    vpmovdw %zmm4, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll
index 0bf126073843..822d31eb4513 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll
@@ -250,10 +250,10 @@ define void @load_i32_stride4_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,0]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = xmm0[1],xmm1[1],zero,zero
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm5 = xmm5[0],xmm6[1]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm7 = zero,zero,xmm2[2],xmm3[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm6 = xmm6[0],xmm7[1]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[3,0],xmm0[3,0]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3]
@@ -584,14 +584,14 @@ define void @load_i32_stride4_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm10[2,0],ymm9[2,3],ymm10[6,4],ymm9[6,7]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = xmm5[1],xmm6[1],zero,zero
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm10 = xmm10[0],xmm11[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
 ; AVX-NEXT:    vunpckhpd {{.*#+}} ymm10 = ymm1[1],ymm2[1],ymm1[3],ymm2[3]
 ; AVX-NEXT:    vunpckhps {{.*#+}} ymm11 = ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[6],ymm0[6],ymm4[7],ymm0[7]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,0],ymm11[4,5],ymm10[6,4]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm5[2],xmm6[2],xmm5[3],xmm6[3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = zero,zero,xmm7[2],xmm8[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm11[0,1],xmm12[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm11 = xmm11[0],xmm12[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
 ; AVX-NEXT:    vunpckhps {{.*#+}} ymm1 = ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[6],ymm1[6],ymm2[7],ymm1[7]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[3,0],ymm4[3,0],ymm0[7,4],ymm4[7,4]
@@ -1080,7 +1080,7 @@ define void @load_i32_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm10[0],xmm2[0],xmm10[1],xmm2[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm6 = xmm6[0],xmm7[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm3[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps %ymm12, %ymm0
@@ -1094,7 +1094,7 @@ define void @load_i32_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm6 = xmm6[0],xmm7[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
@@ -1105,7 +1105,7 @@ define void @load_i32_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm7 = zero,zero,xmm10[2],xmm3[2]
 ; AVX-NEXT:    vmovaps %xmm10, %xmm14
-; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm6 = xmm6[0],xmm7[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm4[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vunpckhpd {{.*#+}} ymm6 = ymm0[1],ymm12[1],ymm0[3],ymm12[3]
@@ -1115,7 +1115,7 @@ define void @load_i32_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm10[2],xmm8[2],xmm10[3],xmm8[3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,zero,xmm9[2],xmm1[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm7[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm7[0],xmm0[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
 ; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm4 # 32-byte Folded Reload
@@ -2120,7 +2120,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm14 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm14 = xmm9[0],mem[0],xmm9[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm8[0],ymm13[0],ymm8[1],ymm13[1],ymm8[4],ymm13[4],ymm8[5],ymm13[5]
@@ -2131,7 +2131,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm14 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm14 = xmm2[0],mem[0],xmm2[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
@@ -2147,7 +2147,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm14 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm14 = xmm5[0],mem[0],xmm5[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
@@ -2164,7 +2164,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
@@ -2176,7 +2176,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    # xmm1 = xmm1[2],mem[2],xmm1[3],mem[3]
 ; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm14 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm14 = zero,zero,xmm9[2],mem[0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vunpckhpd {{.*#+}} ymm8 = ymm2[1],ymm0[1],ymm2[3],ymm0[3]
@@ -2187,7 +2187,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
 ; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm14 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm14 = zero,zero,xmm5[2],mem[0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -2203,7 +2203,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm2[2],xmm4[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
@@ -2215,7 +2215,7 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm7[2],xmm12[2],xmm7[3],xmm12[3]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm7[2],xmm13[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm14[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm14[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
@@ -4239,7 +4239,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm15 = xmm3[0],xmm13[0],xmm3[1],xmm13[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
@@ -4253,7 +4253,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm15 = xmm6[0],xmm11[0],xmm6[1],xmm11[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4269,7 +4269,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = xmm15[0],mem[0],xmm15[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4285,7 +4285,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = xmm15[0],mem[0],xmm15[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4301,7 +4301,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = xmm15[0],mem[0],xmm15[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4317,7 +4317,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = xmm15[0],mem[0],xmm15[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4333,7 +4333,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = xmm15[0],mem[0],xmm15[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4349,7 +4349,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = xmm15[0],mem[0],xmm15[1],mem[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm2[1],ymm10[1],ymm2[3],ymm10[3]
@@ -4358,7 +4358,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm12[2],xmm4[2],xmm12[3],xmm4[3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm15 = zero,zero,xmm3[2],xmm7[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
@@ -4368,7 +4368,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm15 = zero,zero,xmm6[2],xmm11[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
@@ -4384,7 +4384,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm15 = zero,zero,xmm2[2],xmm3[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
@@ -4400,7 +4400,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = zero,zero,xmm15[2],mem[0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4416,7 +4416,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = zero,zero,xmm15[2],mem[0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4432,7 +4432,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = zero,zero,xmm15[2],mem[0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4448,7 +4448,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = zero,zero,xmm15[2],mem[0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -4464,7 +4464,7 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
 ; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm15 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm15 = zero,zero,xmm15[2],mem[0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm15[1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll
index c08442f9d9d0..4f80140bc6c1 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll
@@ -359,7 +359,7 @@ define void @load_i32_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3]
 ; AVX-NEXT:    vmovaps (%rdi), %xmm3
 ; AVX-NEXT:    vmovaps 32(%rdi), %xmm4
-; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm3[0,1],xmm4[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm5 = xmm3[0],xmm4[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0,1],mem[2],xmm5[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[1,2,3,3]
 ; AVX-NEXT:    vmovaps 64(%rdi), %xmm6
@@ -369,7 +369,7 @@ define void @load_i32_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm7[0,1],mem[2],xmm7[3]
 ; AVX-NEXT:    vshufpd {{.*#+}} xmm7 = xmm7[1,0]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm7 = xmm7[0,1,2],xmm6[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[3,0],mem[1,3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm6[2]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
@@ -787,7 +787,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm9[2,0],ymm7[3,0],ymm9[6,4],ymm7[7,4]
 ; AVX-NEXT:    vmovaps (%rdi), %xmm9
 ; AVX-NEXT:    vmovaps 32(%rdi), %xmm10
-; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm9[0,1],xmm10[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm11 = xmm9[0],xmm10[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm11[0,1],mem[2],xmm11[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm11[0,1,2],ymm7[3,4,5,6,7]
@@ -806,7 +806,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm12[6,7]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm2[0,0],ymm1[3,0],ymm2[4,4],ymm1[7,4]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[2,0],ymm1[2,2],ymm12[6,4],ymm1[6,6]
-; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm9 = xmm10[0],xmm9[1]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[3,0],mem[1,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm12[3,4,5,6,7]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm0[2,0],ymm11[1,0],ymm0[6,4],ymm11[5,4]
@@ -1552,7 +1552,7 @@ define void @load_i32_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[2,0],ymm1[3,0],ymm4[6,4],ymm1[7,4]
 ; AVX-NEXT:    vmovaps (%rdi), %xmm15
 ; AVX-NEXT:    vmovaps 32(%rdi), %xmm10
-; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm15[0,1],xmm10[2,3]
+; AVX-NEXT:    vmovsd %xmm15, %xmm10, %xmm4
 ; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],mem[2],xmm4[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3,4,5,6,7]
@@ -1565,7 +1565,7 @@ define void @load_i32_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0],ymm4[3,0],ymm1[6,4],ymm4[7,4]
 ; AVX-NEXT:    vmovaps 160(%rdi), %xmm9
 ; AVX-NEXT:    vmovaps 192(%rdi), %xmm8
-; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm9[0,1],xmm8[2,3]
+; AVX-NEXT:    vmovsd %xmm9, %xmm8, %xmm4
 ; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],mem[2],xmm4[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3,4,5,6,7]
@@ -1597,7 +1597,7 @@ define void @load_i32_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm0[1,0],ymm3[0,0],ymm0[5,4],ymm3[4,4]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm13[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm15[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm10 = xmm10[0],xmm15[1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm6[0,0],ymm5[3,0],ymm6[4,4],ymm5[7,4]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[2,0],ymm5[2,2],ymm13[6,4],ymm5[6,6]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[3,0],mem[1,3]
@@ -1605,7 +1605,7 @@ define void @load_i32_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm1[2,0],ymm4[1,0],ymm1[6,4],ymm4[5,4]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm13[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm8 = xmm8[0],xmm9[1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm7[0,0],ymm2[3,0],ymm7[4,4],ymm2[7,4]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[2,0],ymm2[2,2],ymm9[6,4],ymm2[6,6]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm8[3,0],mem[1,3]
@@ -3086,7 +3086,7 @@ define void @load_i32_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 192(%rdi), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm2[3,4,5,6,7]
@@ -3102,7 +3102,7 @@ define void @load_i32_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 512(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -3118,7 +3118,7 @@ define void @load_i32_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 32(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -3134,7 +3134,7 @@ define void @load_i32_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 352(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6148,7 +6148,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 192(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6165,7 +6165,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 512(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6182,7 +6182,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 832(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6198,7 +6198,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 1152(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6214,7 +6214,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 32(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6230,7 +6230,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 352(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6247,7 +6247,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 672(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
@@ -6265,7 +6265,7 @@ define void @load_i32_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 992(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],mem[2],xmm1[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,2,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
index 86c932a5bb1f..82481269022b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
@@ -13723,29 +13723,29 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512BW-FCP-LABEL: load_i8_stride7_vf64:
 ; AVX512BW-FCP:       # %bb.0:
-; AVX512BW-FCP-NEXT:    vmovdqa64 320(%rdi), %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm2
+; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqa64 320(%rdi), %zmm3
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [24,17,2,19,28,21,6,31,16,9,26,27,20,13,30,23]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm24
+; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm16
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [8,1,18,11,4,5,22,15,0,25,10,3,12,29,14,7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm13
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm24
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [24,17,2,27,20,5,22,31,16,9,26,19,12,29,30,23]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm25
+; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm17
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,18,11,4,21,14,7,8,25,10,3,28,13,6,15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm12
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm12
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,2,11,4,5,14,7,8,9,26,19,12,29,22,15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm16
+; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm18
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,17,10,3,4,21,14,7,24,9,2,11,28,13,6,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm8
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm8
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,10,3,4,5,14,7,8,25,18,11,12,29,22,15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm4
+; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm7
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,17,10,3,20,13,6,23,24,25,18,27,28,21,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm5
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm3
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm5
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
 ; AVX512BW-FCP-NEXT:    movw $-28382, %ax # imm = 0x9122
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm1 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm1 {%k1}
 ; AVX512BW-FCP-NEXT:    kmovq %k1, %k2
 ; AVX512BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm6
@@ -13756,330 +13756,328 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm5, %ymm1 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa 128(%rdi), %ymm11
-; AVX512BW-FCP-NEXT:    vmovdqa 160(%rdi), %ymm9
+; AVX512BW-FCP-NEXT:    vmovdqa 128(%rdi), %ymm6
+; AVX512BW-FCP-NEXT:    vmovdqa 160(%rdi), %ymm5
 ; AVX512BW-FCP-NEXT:    movw $8772, %ax # imm = 0x2244
-; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm11, %ymm9, %ymm5 {%k1}
-; AVX512BW-FCP-NEXT:    kmovq %k1, %k3
-; AVX512BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm6
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u],zero,zero,xmm6[3,10],zero,zero,zero,xmm6[6,13,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,5,12],zero,zero,xmm5[1,8,15],zero,zero,xmm5[u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm6, %xmm5, %xmm5
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,1,2,4,6]
-; AVX512BW-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm17
-; AVX512BW-FCP-NEXT:    vpermd %ymm17, %ymm6, %ymm6
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
-; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm6[7]
+; AVX512BW-FCP-NEXT:    kmovd %eax, %k6
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm5, %ymm9 {%k6}
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm10
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u],zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,5,12],zero,zero,xmm9[1,8,15],zero,zero,xmm9[u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm10, %xmm9, %xmm9
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,0,0,0,1,2,4,6]
+; AVX512BW-FCP-NEXT:    vmovdqa 192(%rdi), %ymm14
+; AVX512BW-FCP-NEXT:    vpermd %ymm14, %ymm10, %ymm10
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
+; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5,6],ymm10[7]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 240(%rdi), %xmm19
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm19[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm19[5,12,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 224(%rdi), %xmm20
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm20[0,7,14],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm6, %zmm5, %zmm5
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm20[0,7,14],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm10, %xmm11, %xmm10
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm10, %zmm9, %zmm9
 ; AVX512BW-FCP-NEXT:    movabsq $137438429184, %rax # imm = 0x1FFFF80000
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k5
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k5}
-; AVX512BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm6
-; AVX512BW-FCP-NEXT:    vmovdqa 256(%rdi), %ymm5
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm9, %zmm1 {%k5}
+; AVX512BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm10
+; AVX512BW-FCP-NEXT:    vmovdqa 256(%rdi), %ymm9
 ; AVX512BW-FCP-NEXT:    movw $9288, %ax # imm = 0x2448
-; AVX512BW-FCP-NEXT:    kmovd %eax, %k6
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm5, %ymm7 {%k6}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm7[u,u,u,u,u,3,10],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[1,8,15],zero,zero,xmm7[4,11,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm18, %xmm7, %xmm21
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    kmovd %eax, %k3
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm9, %ymm11 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm11[u,u,u,u,u,3,10],zero,zero,zero,xmm11[6,13],zero,zero,xmm11[u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm11, %xmm11
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u],zero,zero,xmm11[1,8,15],zero,zero,xmm11[4,11,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm15, %xmm11, %xmm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    movw $3968, %ax # imm = 0xF80
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k7
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm4, %ymm21 {%k7}
-; AVX512BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm7
-; AVX512BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm7, %ymm21 {%k7}
+; AVX512BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm11
+; AVX512BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm7
 ; AVX512BW-FCP-NEXT:    movw $4644, %ax # imm = 0x1224
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k4
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm4, %ymm18 {%k4}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm22
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm11, %ymm7, %ymm15 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm15, %xmm22
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm22[u,u,u,u,u,u,u],zero,zero,zero,xmm22[6,13],zero,zero,xmm22[2,9]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm18[4,11],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %xmm22, %xmm18, %xmm18
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm22
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm15[4,11],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %xmm22, %xmm15, %xmm15
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm15, %ymm0, %ymm22
 ; AVX512BW-FCP-NEXT:    movl $-8388608, %eax # imm = 0xFF800000
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm18 {%k4}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm23
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm15 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm15, %xmm23
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = zero,zero,zero,xmm23[6,13],zero,zero,xmm23[2,9,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[1,8,15],zero,zero,xmm18[4,11],zero,zero,xmm18[u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm23, %xmm18, %xmm18
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[1,8,15],zero,zero,xmm15[4,11],zero,zero,xmm15[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm23, %xmm15, %xmm15
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    movl $511, %r10d # imm = 0x1FF
 ; AVX512BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm8 {%k1}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm11, %ymm9, %ymm18 {%k6}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm18[u,u,u,6,13],zero,zero,xmm18[2,9],zero,zero,zero,xmm18[u,u,u,u]
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u],zero,zero,xmm18[4,11],zero,zero,xmm18[0,7,14,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm23, %xmm18, %xmm18
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm14
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm18 = [0,0,0,0,1,3,4,6]
-; AVX512BW-FCP-NEXT:    vpermd %ymm17, %ymm18, %ymm18
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
-; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5,6],ymm15[7]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm8 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm5, %ymm15 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm15[u,u,u,6,13],zero,zero,xmm15[2,9],zero,zero,zero,xmm15[u,u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm15, %xmm15
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u],zero,zero,xmm15[4,11],zero,zero,xmm15[0,7,14,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm23, %xmm15, %xmm15
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm23 = [0,0,0,0,1,3,4,6]
+; AVX512BW-FCP-NEXT:    vpermd %ymm14, %ymm23, %ymm23
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm23[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
+; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm19[6,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm20[1,8,15],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm15, %xmm18, %xmm15
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm14, %zmm14
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm8 {%k5}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm14 {%k3}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[2,9],zero,zero,zero,xmm14[5,12],zero,zero,xmm14[u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = zero,zero,xmm14[0,7,14],zero,zero,xmm14[3,10,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm20[1,8,15],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm15, %xmm23, %xmm15
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm13, %zmm13
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm13, %zmm8 {%k5}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm13 {%k6}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm13[2,9],zero,zero,zero,xmm13[5,12],zero,zero,xmm13[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm13
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm13[0,7,14],zero,zero,xmm13[3,10,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm13, %xmm13
 ; AVX512BW-FCP-NEXT:    movl $261632, %r10d # imm = 0x3FE00
 ; AVX512BW-FCP-NEXT:    kmovd %r10d, %k5
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 {%k5} = ymm12[u,u,u,u,u,u,u,u,u,1,8,15,6,13,4,11,18,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm12 {%k2}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm13 {%k5} = ymm12[u,u,u,u,u,u,u,u,u,1,8,15,6,13,4,11,18,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm12 {%k2}
 ; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm15
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u],zero,zero,zero,xmm15[5,12],zero,zero,xmm15[1,8,15,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[u,u,0,7,14],zero,zero,xmm12[3,10],zero,zero,zero,xmm12[u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm12, %xmm12
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
 ; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [0,0,0,0,1,3,5,6]
-; AVX512BW-FCP-NEXT:    vpermd %ymm17, %ymm15, %ymm15
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
-; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6],ymm15[7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm20[2,9],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = zero,zero,xmm19[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm15, %xmm17, %xmm15
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm12, %zmm12
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm12 {%k1}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm14 {%k6}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[3,10],zero,zero,zero,xmm14[6,13],zero,zero,xmm14[u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = zero,zero,xmm14[1,8,15],zero,zero,xmm14[4,11,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 {%k5} = ymm13[u,u,u,u,u,u,u,u,u,2,9,0,7,14,5,12,19,26,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm13 {%k4}
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm15
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u],zero,zero,zero,xmm15[6,13],zero,zero,xmm15[2,9,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpermd %ymm14, %ymm15, %ymm14
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
+; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6],ymm14[7]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm20[2,9],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,xmm19[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm14, %xmm15, %xmm14
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm14, %zmm12, %zmm12
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm13, %zmm12 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm13 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm13[3,10],zero,zero,zero,xmm13[6,13],zero,zero,xmm13[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm13
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm13[1,8,15],zero,zero,xmm13[4,11,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm14, %xmm13, %xmm23
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm23 {%k5} = ymm24[u,u,u,u,u,u,u,u,u,2,9,0,7,14,5,12,19,26,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm13 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm14
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u],zero,zero,zero,xmm14[6,13],zero,zero,xmm14[2,9,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,1,8,15],zero,zero,xmm13[4,11],zero,zero,xmm13[u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm13, %xmm13
+; AVX512BW-FCP-NEXT:    vpor %xmm14, %xmm13, %xmm13
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-FCP-NEXT:    vmovdqa64 208(%rdi), %xmm17
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm17[5,12]
-; AVX512BW-FCP-NEXT:    vmovdqa64 192(%rdi), %xmm18
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm18[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %xmm15, %xmm23, %xmm15
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512BW-FCP-NEXT:    vmovdqa 208(%rdi), %xmm14
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm24 = xmm14[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm14[5,12]
+; AVX512BW-FCP-NEXT:    vmovdqa 192(%rdi), %xmm15
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm25 = xmm15[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %xmm24, %xmm25, %xmm24
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm24, %ymm0, %ymm24
 ; AVX512BW-FCP-NEXT:    movl $-134217728, %r10d # imm = 0xF8000000
 ; AVX512BW-FCP-NEXT:    kmovd %r10d, %k2
 ; AVX512BW-FCP-NEXT:    kmovd %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm13 {%k2}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm20[3,10],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm24, %ymm13 {%k2}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[3,10],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = zero,zero,xmm19[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm15, %xmm19, %xmm15
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm13, %zmm13
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm13 {%k1}
-; AVX512BW-FCP-NEXT:    kmovd %eax, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm22, %ymm21 {%k3}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm0, %zmm14
+; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm19, %xmm19
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm19, %zmm13, %zmm13
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm23, %zmm13 {%k1}
+; AVX512BW-FCP-NEXT:    kmovd %eax, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm22, %ymm21 {%k2}
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm0, %zmm19
 ; AVX512BW-FCP-NEXT:    movabsq $-137438953472, %rax # imm = 0xFFFFFFE000000000
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm1 {%k2}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm14 {%k4}
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm15
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u],zero,zero,xmm15[2,9],zero,zero,zero,xmm15[5,12,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,4,11],zero,zero,xmm14[0,7,14],zero,zero,xmm14[u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm15, %ymm14 {%k7}
+; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm19, %zmm1 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm10, %ymm19 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm19, %xmm20
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u,u,u,u],zero,zero,xmm20[2,9],zero,zero,zero,xmm20[5,12,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u,4,11],zero,zero,xmm19[0,7,14],zero,zero,xmm19[u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm19, %xmm19
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm18 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm18, %ymm19 {%k7}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm11, %ymm7, %ymm18 {%k6}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm18[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm18[5,12],zero,zero
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[0,7,14],zero,zero,xmm18[3,10]
+; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm18, %xmm18
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm18
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm19 {%k2}
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm0, %zmm18
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm18, %zmm8 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm10, %ymm18 {%k6}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm19
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u],zero,zero,xmm19[3,10],zero,zero,zero,xmm19[6,13,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,5,12],zero,zero,xmm18[1,8,15],zero,zero,xmm18[u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm18, %xmm18
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm17 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm17, %ymm18 {%k7}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm11, %ymm7, %ymm17 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm17[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm17[6,13],zero,zero
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u],zero,zero,xmm17[1,8,15],zero,zero,xmm17[4,11]
+; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm17, %xmm17
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm17
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm18 {%k2}
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm0, %zmm17
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm17, %zmm12 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm10, %ymm17 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm17[u,u,u,u,u,6,13],zero,zero,xmm17[2,9],zero,zero,zero,xmm17[u,u]
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u],zero,zero,xmm17[4,11],zero,zero,xmm17[0,7,14,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm18, %xmm17, %xmm17
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm16 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm16, %ymm17 {%k7}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm11, %ymm16 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm18
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[2,9],zero,zero,zero,xmm18[5,12]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u,4,11],zero,zero,xmm16[0,7,14],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %xmm18, %xmm16, %xmm16
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm17 {%k2}
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm0, %zmm16
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm16, %zmm13 {%k1}
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [0,9,2,3,4,13,6,7,24,17,10,11,28,21,14,31]
+; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm16, %zmm16
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [16,25,18,3,28,21,6,23,24,17,10,27,20,13,30,31]
+; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm17, %zmm17
 ; AVX512BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm4, %ymm15 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm15[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm15[5,12],zero,zero
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm15, %xmm15
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,u],zero,zero,xmm15[0,7,14],zero,zero,xmm15[3,10]
-; AVX512BW-FCP-NEXT:    vporq %xmm16, %xmm15, %xmm15
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm14 {%k3}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm8 {%k2}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm14 {%k1}
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm15
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u],zero,zero,xmm15[3,10],zero,zero,zero,xmm15[6,13,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,5,12],zero,zero,xmm14[1,8,15],zero,zero,xmm14[u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm25[u,u,u,u,u,u,u,u,u,u,u,u,u,u,4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm15, %ymm14 {%k7}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm4, %ymm15 {%k6}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm15[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm15[6,13],zero,zero
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm15, %xmm15
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,u],zero,zero,xmm15[1,8,15],zero,zero,xmm15[4,11]
-; AVX512BW-FCP-NEXT:    vporq %xmm16, %xmm15, %xmm15
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm14 {%k3}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm12 {%k2}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm14 {%k6}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[u,u,u,u,u,6,13],zero,zero,xmm14[2,9],zero,zero,zero,xmm14[u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u],zero,zero,xmm14[4,11],zero,zero,xmm14[0,7,14,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm15, %ymm14 {%k7}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm7, %ymm15 {%k4}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm15, %xmm16
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u],zero,zero,xmm16[2,9],zero,zero,zero,xmm16[5,12]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,u,4,11],zero,zero,xmm15[0,7,14],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %xmm16, %xmm15, %xmm15
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm14 {%k3}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm13 {%k2}
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [0,9,2,3,4,13,6,7,24,17,10,11,28,21,14,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm14, %zmm15
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [16,25,18,3,28,21,6,23,24,17,10,27,20,13,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm14, %zmm16
-; AVX512BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm5, %ymm14 {%k2}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm14, %xmm19
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm9, %ymm18 {%k1}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm19
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u],zero,zero,zero,xmm19[5,12],zero,zero,xmm19[1,8,15,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,0,7,14],zero,zero,xmm14[3,10],zero,zero,zero,xmm14[u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm16 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm16, %ymm14 {%k7}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm5, %ymm16 {%k4}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm19
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,0,7,14],zero,zero,xmm18[3,10],zero,zero,zero,xmm18[u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm18, %xmm18
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm17 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm17, %ymm18 {%k7}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm9, %ymm17 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm19
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u],zero,zero,zero,xmm19[6,13],zero,zero,xmm19[2,9,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,1,8,15],zero,zero,xmm16[4,11],zero,zero,xmm16[u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm16, %xmm16
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,1,8,15],zero,zero,xmm17[4,11],zero,zero,xmm17[u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm17, %xmm17
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm19 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    movl $8176, %eax # imm = 0x1FF0
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm15 {%k1}
-; AVX512BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm7, %ymm16 {%k1}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm19
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u,u,u],zero,zero,xmm19[3,10],zero,zero,zero,xmm19[6,13]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u,5,12],zero,zero,xmm16[1,8,15],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm16, %xmm16
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm14 {%k3}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm7, %ymm16 {%k6}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm16[u,u,u,u,u,u,u,6,13],zero,zero,xmm16[2,9],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm16
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u],zero,zero,xmm16[4,11],zero,zero,xmm16[0,7,14]
-; AVX512BW-FCP-NEXT:    vporq %xmm19, %xmm16, %xmm16
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm15 {%k3}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm3, %ymm19 {%k4}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm3, %ymm16 {%k1}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm20 {%k6}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm3 {%k6}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm10 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm11, %ymm9 {%k4}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm10[u,u,2,9],zero,zero,zero,xmm10[5,12],zero,zero,xmm10[u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm10, %xmm10
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u],zero,zero,xmm10[0,7,14],zero,zero,xmm10[3,10,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm11, %xmm10, %xmm10
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm17[6,13]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm18[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %xmm11, %xmm21, %xmm11
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512BW-FCP-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 4-byte Reload
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm11, %ymm10 {%k3}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm20[u,u,3,10],zero,zero,zero,xmm20[6,13],zero,zero,xmm20[u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm19 {%k1}
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm16, %zmm20
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm16, %zmm17
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm2, %ymm16 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = zero,zero,xmm21[2,9],zero,zero,zero,xmm21[5,12,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[4,11],zero,zero,xmm16[0,7,14],zero,zero,xmm16[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm21, %xmm16, %xmm16
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm16 {%k5} = ymm17[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm17 {%k6}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm17[u,u,2,9],zero,zero,zero,xmm17[5,12],zero,zero,xmm17[u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u],zero,zero,xmm17[0,7,14],zero,zero,xmm17[3,10,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm21, %xmm17, %xmm17
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm14[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm14[6,13]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm15[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %xmm17, %xmm22, %xmm17
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm17
+; AVX512BW-FCP-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 4-byte Reload
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm21 {%k7}
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm22 = [1,2,4,6,0,0,0,0]
+; AVX512BW-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm17
+; AVX512BW-FCP-NEXT:    vpermd %ymm17, %ymm22, %ymm22
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm22[0,7,10,13,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm22, %zmm21, %zmm21
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm21, %zmm16 {%k5}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm11, %ymm21 {%k6}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm21, %xmm22
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm22[u,u,u,u,u,u,u],zero,zero,xmm22[3,10],zero,zero,zero,xmm22[6,13]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[u,u,u,u,u,u,u,5,12],zero,zero,xmm21[1,8,15],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %xmm22, %xmm21, %xmm21
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm0, %ymm21
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm21, %ymm18 {%k2}
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm0, %zmm18
+; AVX512BW-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm11, %ymm21 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm21[u,u,u,u,u,u,u,6,13],zero,zero,xmm21[2,9],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm21, %xmm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[u,u,u,u,u,u,u],zero,zero,xmm21[4,11],zero,zero,xmm21[0,7,14]
+; AVX512BW-FCP-NEXT:    vporq %xmm22, %xmm21, %xmm21
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm0, %ymm21
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm21, %ymm19 {%k2}
+; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm18, %zmm16 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm2, %ymm18 {%k6}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = zero,zero,xmm21[3,10],zero,zero,zero,xmm21[6,13,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[5,12],zero,zero,xmm18[1,8,15],zero,zero,xmm18[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm21, %xmm18, %xmm18
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm18 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm20 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm20[u,u,3,10],zero,zero,zero,xmm20[6,13],zero,zero,xmm20[u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm20, %xmm20
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u],zero,zero,xmm20[1,8,15],zero,zero,xmm20[4,11,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm11, %xmm20, %xmm11
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm18[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm17[0,7,14]
-; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm21, %xmm20
+; AVX512BW-FCP-NEXT:    vporq %xmm21, %xmm20, %xmm20
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm20
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm20, %ymm11 {%k3}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm9, %xmm20
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u],zero,zero,xmm20[2,9],zero,zero,zero,xmm20[5,12,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,4,11],zero,zero,xmm9[0,7,14],zero,zero,xmm9[u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm9, %xmm9
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm17[1,8,15]
-; AVX512BW-FCP-NEXT:    vporq %xmm18, %xmm17, %xmm17
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm17
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm9 {%k3}
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm17, %zmm17
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm18 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm18, %zmm18
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm20 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm20, %zmm20
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm19, %xmm2
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[2,9],zero,zero,zero,xmm2[5,12,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[4,11],zero,zero,xmm19[0,7,14],zero,zero,xmm19[u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm2, %xmm19, %xmm2
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm19 = [1,2,4,6,0,0,0,0]
-; AVX512BW-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512BW-FCP-NEXT:    vpermd %ymm20, %ymm19, %ymm19
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[0,7,10,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm19, %zmm10, %zmm10
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm10, %zmm2 {%k5}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm10
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[5,12],zero,zero,xmm16[1,8,15],zero,zero,xmm16[u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm10, %xmm16, %xmm10
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 {%k5} = ymm18[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm16 = [1,3,4,6,0,0,0,0]
-; AVX512BW-FCP-NEXT:    vpermd %ymm20, %ymm16, %ymm16
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[1,4,11,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm16, %zmm11, %zmm11
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm11, %zmm10 {%k5}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm3[6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm11, %xmm3
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 {%k5} = ymm17[u,u,u,u,u,u,u,u,u,5,12,3,10,1,8,15,22,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [1,3,5,6,0,0,0,0]
-; AVX512BW-FCP-NEXT:    vpermd %ymm20, %ymm11, %ymm11
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[2,5,8,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm11, %zmm9, %zmm9
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm3 {%k5}
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm9 = [0,1,10,3,4,13,6,7,8,25,18,11,28,21,14,15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm9, %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm5 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm5[u,u,u,u,2,9],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[u,u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u],zero,zero,xmm5[0,7,14],zero,zero,xmm5[3,10,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm6, %xmm5, %xmm5
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm15[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm14[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm14[0,7,14]
+; AVX512BW-FCP-NEXT:    vporq %xmm21, %xmm22, %xmm21
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm0, %ymm21
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm21, %ymm20 {%k7}
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm21 = [1,3,4,6,0,0,0,0]
+; AVX512BW-FCP-NEXT:    vpermd %ymm17, %ymm21, %ymm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[1,4,11,14,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm21, %zmm20, %zmm20
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm20, %zmm18 {%k5}
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm0, %zmm19
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm19, %zmm18 {%k1}
+; AVX512BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm11, %ymm7 {%k1}
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,1,10,3,4,13,6,7,8,25,18,11,28,21,14,15]
+; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm11, %zmm3
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm9 {%k6}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm9[u,u,u,u,2,9],zero,zero,zero,xmm9[5,12],zero,zero,xmm9[u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm9
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u],zero,zero,xmm9[0,7,14],zero,zero,xmm9[3,10,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm10, %xmm9, %xmm9
 ; AVX512BW-FCP-NEXT:    movl $4186112, %eax # imm = 0x3FE000
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 {%k1} = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,1,8,15,22,29,20,27,18,25,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm7, %ymm4 {%k2}
-; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm10 {%k1}
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[1,8,15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,0,7,14],zero,zero,xmm4[3,10],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm4, %xmm0
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0,1,2],ymm0[3,4,5,6,7],ymm5[8,9,10],ymm0[11,12,13,14,15]
-; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm3 {%k1}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm9 {%k1} = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,1,8,15,22,29,20,27,18,25,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm4, %ymm2 {%k3}
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm5 {%k4}
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm3
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm5[u,u,4,11],zero,zero,xmm5[0,7,14],zero,zero,xmm5[u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm15[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm14[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm14[1,8,15]
+; AVX512BW-FCP-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm4, %ymm3 {%k7}
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm4 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm4, %zmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[6,13],zero,zero,xmm2[2,9],zero,zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm0[u,u,u,u,u,u,u,u,u,5,12,3,10,1,8,15,22,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [1,3,5,6,0,0,0,0]
+; AVX512BW-FCP-NEXT:    vpermd %ymm17, %ymm0, %ymm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,15,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm3, %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k5}
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[1,8,15]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm7[u,u,u,u,u,u,0,7,14],zero,zero,xmm7[3,10],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm3
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm4 = [16,17,18,19,20,21,22,23,24,25,26,43,44,45,46,47]
+; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm3, %zmm4
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,1,2,3,4,5,6,7,8,17,18,19,20,21,22,23]
+; AVX512BW-FCP-NEXT:    vpermi2d %zmm4, %zmm2, %zmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rsi)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm8, (%rdx)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm12, (%rcx)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, (%r8)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, (%r9)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, (%rdi)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, (%r9)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm18, (%rdi)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -14452,361 +14450,359 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512DQ-BW-FCP-LABEL: load_i8_stride7_vf64:
 ; AVX512DQ-BW-FCP:       # %bb.0:
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 320(%rdi), %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 320(%rdi), %zmm3
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [24,17,2,19,28,21,6,31,16,9,26,27,20,13,30,23]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm24
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm24
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [8,1,18,11,4,5,22,15,0,25,10,3,12,29,14,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm13
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [24,17,2,27,20,5,22,31,16,9,26,19,12,29,30,23]
 ; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm25
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [24,17,2,27,20,5,22,31,16,9,26,19,12,29,30,23]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm17
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,18,11,4,21,14,7,8,25,10,3,28,13,6,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm12
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm12
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,2,11,4,5,14,7,8,9,26,19,12,29,22,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm16
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm18
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,17,10,3,4,21,14,7,24,9,2,11,28,13,6,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm8
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm8
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,10,3,4,5,14,7,8,25,18,11,12,29,22,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm1, %zmm7
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,17,10,3,20,13,6,23,24,25,18,27,28,21,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm4
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    movw $-28382, %ax # imm = 0x9122
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm1 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm1 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %k1, %k2
 ; AVX512DQ-BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm6
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[5,12],zero,zero,xmm6[1,8,15,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,7,14],zero,zero,xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm6, %xmm1, %xmm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,6,13,4,11,2,9,16,23,30,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,6,13,4,11,2,9,16,23,30,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm4, %ymm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 128(%rdi), %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 160(%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm5, %ymm1 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 160(%rdi), %ymm5
 ; AVX512DQ-BW-FCP-NEXT:    movw $8772, %ax # imm = 0x2244
-; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm11, %ymm9, %ymm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    kmovq %k1, %k3
-; AVX512DQ-BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm6
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u],zero,zero,xmm6[3,10],zero,zero,zero,xmm6[6,13,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,1,2,4,6]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm17, %ymm6, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
-; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm6[7]
+; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k6
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm5, %ymm9 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u],zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,5,12],zero,zero,xmm9[1,8,15],zero,zero,xmm9[u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm10, %xmm9, %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,0,0,0,1,2,4,6]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 192(%rdi), %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm15, %ymm10, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
+; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5,6],ymm10[7]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 240(%rdi), %xmm19
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm19[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm19[5,12,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 224(%rdi), %xmm20
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm20[0,7,14],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm6, %zmm4, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm20[0,7,14],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm10, %xmm11, %xmm10
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm10, %zmm9, %zmm9
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $137438429184, %rax # imm = 0x1FFFF80000
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm1 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 256(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm9, %zmm1 {%k5}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 256(%rdi), %ymm9
 ; AVX512DQ-BW-FCP-NEXT:    movw $9288, %ax # imm = 0x2448
-; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k6
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm4, %ymm7 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm7[u,u,u,u,u,3,10],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[1,8,15],zero,zero,xmm7[4,11,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm7, %xmm21
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k3
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm9, %ymm11 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm11[u,u,u,u,u,3,10],zero,zero,zero,xmm11[6,13],zero,zero,xmm11[u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm11, %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u],zero,zero,xmm11[1,8,15],zero,zero,xmm11[4,11,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm16, %xmm11, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    movw $3968, %ax # imm = 0xF80
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k7
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm5, %ymm21 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm7
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm7, %ymm21 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm7
 ; AVX512DQ-BW-FCP-NEXT:    movw $4644, %ax # imm = 0x1224
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k4
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm5, %ymm18 {%k4}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm22
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm11, %ymm7, %ymm16 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm22
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm22[u,u,u,u,u,u,u],zero,zero,zero,xmm22[6,13],zero,zero,xmm22[2,9]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm18[4,11],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm22, %xmm18, %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm16[4,11],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm22, %xmm16, %xmm16
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm22
 ; AVX512DQ-BW-FCP-NEXT:    movl $-8388608, %eax # imm = 0xFF800000
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm18 {%k4}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm23
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm16 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm23
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = zero,zero,zero,xmm23[6,13],zero,zero,xmm23[2,9,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[1,8,15],zero,zero,xmm18[4,11],zero,zero,xmm18[u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm23, %xmm18, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[1,8,15],zero,zero,xmm16[4,11],zero,zero,xmm16[u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm23, %xmm16, %xmm16
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    movl $511, %r10d # imm = 0x1FF
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm8 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm11, %ymm9, %ymm18 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm18[u,u,u,6,13],zero,zero,xmm18[2,9],zero,zero,zero,xmm18[u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u],zero,zero,xmm18[4,11],zero,zero,xmm18[0,7,14,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm23, %xmm18, %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm18 = [0,0,0,0,1,3,4,6]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm17, %ymm18, %ymm18
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
-; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5,6],ymm15[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm19[6,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm20[1,8,15],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm15, %xmm18, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm14, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm8 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm14 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[2,9],zero,zero,zero,xmm14[5,12],zero,zero,xmm14[u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = zero,zero,xmm14[0,7,14],zero,zero,xmm14[3,10,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm8 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm5, %ymm16 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm16[u,u,u,6,13],zero,zero,xmm16[2,9],zero,zero,zero,xmm16[u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm16
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u],zero,zero,xmm16[4,11],zero,zero,xmm16[0,7,14,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm23, %xmm16, %xmm16
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm16 = [0,0,0,0,1,3,4,6]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm15, %ymm16, %ymm16
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
+; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5,6],ymm13[7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = zero,zero,zero,xmm19[6,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm20[1,8,15],zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm14, %xmm16, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm14, %zmm13, %zmm13
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm13, %zmm8 {%k5}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm13 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm13[2,9],zero,zero,zero,xmm13[5,12],zero,zero,xmm13[u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm13[0,7,14],zero,zero,xmm13[3,10,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm14, %xmm13, %xmm13
 ; AVX512DQ-BW-FCP-NEXT:    movl $261632, %r10d # imm = 0x3FE00
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k5
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 {%k5} = ymm12[u,u,u,u,u,u,u,u,u,1,8,15,6,13,4,11,18,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm12 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u],zero,zero,zero,xmm15[5,12],zero,zero,xmm15[1,8,15,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm13 {%k5} = ymm12[u,u,u,u,u,u,u,u,u,1,8,15,6,13,4,11,18,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm12 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u],zero,zero,zero,xmm14[5,12],zero,zero,xmm14[1,8,15,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[u,u,0,7,14],zero,zero,xmm12[3,10],zero,zero,zero,xmm12[u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm15, %xmm12, %xmm12
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm14, %xmm12, %xmm12
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [0,0,0,0,1,3,5,6]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm17, %ymm15, %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
-; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6],ymm15[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm20[2,9],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = zero,zero,xmm19[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm15, %xmm17, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm12, %zmm12
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm12 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm3, %ymm10, %ymm14 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[3,10],zero,zero,zero,xmm14[6,13],zero,zero,xmm14[u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = zero,zero,xmm14[1,8,15],zero,zero,xmm14[4,11,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 {%k5} = ymm13[u,u,u,u,u,u,u,u,u,2,9,0,7,14,5,12,19,26,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm13 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [0,0,0,0,1,3,5,6]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm15, %ymm14, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
+; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6],ymm14[7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm20[2,9],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,xmm19[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm14, %xmm15, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm14, %zmm12, %zmm12
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm13, %zmm12 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm2, %ymm4, %ymm13 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm13[3,10],zero,zero,zero,xmm13[6,13],zero,zero,xmm13[u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm13[1,8,15],zero,zero,xmm13[4,11,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm14, %xmm13, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 {%k5} = ymm25[u,u,u,u,u,u,u,u,u,2,9,0,7,14,5,12,19,26,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm13 {%k4}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm15
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u],zero,zero,zero,xmm15[6,13],zero,zero,xmm15[2,9,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,1,8,15],zero,zero,xmm13[4,11],zero,zero,xmm13[u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm15, %xmm13, %xmm13
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 208(%rdi), %xmm17
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm17[5,12]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 192(%rdi), %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm18[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm15, %xmm23, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 208(%rdi), %xmm15
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm15[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm15[5,12]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 192(%rdi), %xmm16
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm25 = xmm16[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm23, %xmm25, %xmm23
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm23, %ymm0, %ymm23
 ; AVX512DQ-BW-FCP-NEXT:    movl $-134217728, %r10d # imm = 0xF8000000
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k2
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm13 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm20[3,10],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm23, %ymm13 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[3,10],zero,zero,zero,xmm20[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = zero,zero,xmm19[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm15, %xmm19, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm13, %zmm13
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm20, %xmm19, %xmm19
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm19, %zmm13, %zmm13
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm13 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm22, %ymm21 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm22, %ymm21 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm0, %zmm14
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-137438953472, %rax # imm = 0xFFFFFFE000000000
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm1 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm6, %ymm14 {%k4}
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u],zero,zero,xmm15[2,9],zero,zero,zero,xmm15[5,12,u,u]
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm1 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm10, %ymm14 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm14, %xmm19
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u],zero,zero,xmm19[2,9],zero,zero,zero,xmm19[5,12,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,4,11],zero,zero,xmm14[0,7,14],zero,zero,xmm14[u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm15, %ymm14 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm5, %ymm15 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm15[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm15[5,12],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm15, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,u],zero,zero,xmm15[0,7,14],zero,zero,xmm15[3,10]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm16, %xmm15, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm14 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm14, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm18 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm18, %ymm14 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm11, %ymm7, %ymm18 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm18[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm18[5,12],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[0,7,14],zero,zero,xmm18[3,10]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm18, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm18
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm14 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm8 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm6, %ymm14 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u],zero,zero,xmm15[3,10],zero,zero,zero,xmm15[6,13,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm8 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm10, %ymm14 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm14, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u],zero,zero,xmm18[3,10],zero,zero,zero,xmm18[6,13,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,5,12],zero,zero,xmm14[1,8,15],zero,zero,xmm14[u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm25[u,u,u,u,u,u,u,u,u,u,u,u,u,u,4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm15, %ymm14 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm5, %ymm15 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm15[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm15[6,13],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm15, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,u],zero,zero,xmm15[1,8,15],zero,zero,xmm15[4,11]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm16, %xmm15, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm14 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm14, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm17 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm17, %ymm14 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm11, %ymm7, %ymm17 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm17[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm17[6,13],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u],zero,zero,xmm17[1,8,15],zero,zero,xmm17[4,11]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm17, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm17
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm14 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm12 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm6, %ymm14 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[u,u,u,u,u,6,13],zero,zero,xmm14[2,9],zero,zero,zero,xmm14[u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm12 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm10, %ymm14 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm14[u,u,u,u,u,6,13],zero,zero,xmm14[2,9],zero,zero,zero,xmm14[u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u],zero,zero,xmm14[4,11],zero,zero,xmm14[0,7,14,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm15, %xmm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm15, %ymm14 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm7, %ymm15 {%k4}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm15, %xmm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u],zero,zero,xmm16[2,9],zero,zero,zero,xmm16[5,12]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,u,4,11],zero,zero,xmm15[0,7,14],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm16, %xmm15, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm15, %ymm14 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm17, %xmm14, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm17 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm17, %ymm14 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm11, %ymm17 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[2,9],zero,zero,zero,xmm18[5,12]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u,4,11],zero,zero,xmm17[0,7,14],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm17, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm17
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm14 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm13 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm13 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [0,9,2,3,4,13,6,7,24,17,10,11,28,21,14,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm14, %zmm15
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [16,25,18,3,28,21,6,23,24,17,10,27,20,13,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm14, %zmm16
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm14, %zmm14
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [16,25,18,3,28,21,6,23,24,17,10,27,20,13,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm17, %zmm18
 ; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm4, %ymm14 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm14, %xmm19
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm9, %ymm17 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm19
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u],zero,zero,zero,xmm19[5,12],zero,zero,xmm19[1,8,15,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,0,7,14],zero,zero,xmm14[3,10],zero,zero,zero,xmm14[u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm16 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm16, %ymm14 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm4, %ymm16 {%k4}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm19
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,0,7,14],zero,zero,xmm17[3,10],zero,zero,zero,xmm17[u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm17, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm18 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm18, %ymm17 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm9, %ymm18 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm19
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u],zero,zero,zero,xmm19[6,13],zero,zero,xmm19[2,9,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,1,8,15],zero,zero,xmm16[4,11],zero,zero,xmm16[u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm16, %xmm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,1,8,15],zero,zero,xmm18[4,11],zero,zero,xmm18[u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm18, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm19 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    movl $8176, %eax # imm = 0x1FF0
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm15 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm7, %ymm16 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm19
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u,u,u],zero,zero,xmm19[3,10],zero,zero,zero,xmm19[6,13]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u,5,12],zero,zero,xmm16[1,8,15],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm16, %xmm16
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm14 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm7, %ymm16 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm16[u,u,u,u,u,u,u,6,13],zero,zero,xmm16[2,9],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u],zero,zero,xmm16[4,11],zero,zero,xmm16[0,7,14]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm19, %xmm16, %xmm16
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm16, %ymm15 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm3, %ymm19 {%k4}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm3, %ymm16 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm20 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm3 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm11, %ymm10 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm11, %ymm9 {%k4}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm10[u,u,2,9],zero,zero,zero,xmm10[5,12],zero,zero,xmm10[u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm10, %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u],zero,zero,xmm10[0,7,14],zero,zero,xmm10[3,10,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm11, %xmm10, %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm17[6,13]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm18[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm11, %xmm21, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-BW-FCP-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 4-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm11, %ymm10 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm20[u,u,3,10],zero,zero,zero,xmm20[6,13],zero,zero,xmm20[u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm19 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm14, %zmm20
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm14, %zmm18
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm11, %ymm14 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm14, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[u,u,u,u,u,u,u],zero,zero,xmm21[3,10],zero,zero,zero,xmm21[6,13]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,u,u,5,12],zero,zero,xmm14[1,8,15],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm21, %xmm14, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm14, %ymm17 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm2, %ymm14 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm14, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = zero,zero,xmm21[2,9],zero,zero,zero,xmm21[5,12,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[4,11],zero,zero,xmm14[0,7,14],zero,zero,xmm14[u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm21, %xmm14, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 {%k5} = ymm18[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm18 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm18[u,u,2,9],zero,zero,zero,xmm18[5,12],zero,zero,xmm18[u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u],zero,zero,xmm18[0,7,14],zero,zero,xmm18[3,10,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm21, %xmm18, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm15[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm15[6,13]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm16[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm22, %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm18
+; AVX512DQ-BW-FCP-NEXT:    kmovd {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 4-byte Reload
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm21 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm22 = [1,2,4,6,0,0,0,0]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm18, %ymm22, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm22[0,7,10,13,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm22, %zmm21, %zmm21
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm21, %zmm14 {%k5}
+; AVX512DQ-BW-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm11, %ymm21 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm21[u,u,u,u,u,u,u,6,13],zero,zero,xmm21[2,9],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm21, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[u,u,u,u,u,u,u],zero,zero,xmm21[4,11],zero,zero,xmm21[0,7,14]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm22, %xmm21, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm0, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm21, %ymm19 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm17, %zmm0, %zmm14 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm2, %ymm17 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = zero,zero,xmm21[3,10],zero,zero,zero,xmm21[6,13,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[5,12],zero,zero,xmm17[1,8,15],zero,zero,xmm17[u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm21, %xmm17, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm17 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm6, %ymm20 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm20[u,u,3,10],zero,zero,zero,xmm20[6,13],zero,zero,xmm20[u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm20, %xmm20
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u],zero,zero,xmm20[1,8,15],zero,zero,xmm20[4,11,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm11, %xmm20, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm18[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm17[0,7,14]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm20, %xmm21, %xmm20
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm21, %xmm20, %xmm20
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm20
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm20, %ymm11 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm9, %xmm20
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u],zero,zero,xmm20[2,9],zero,zero,zero,xmm20[5,12,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,4,11],zero,zero,xmm9[0,7,14],zero,zero,xmm9[u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm20, %xmm9, %xmm9
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm17[1,8,15]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm17, %xmm17
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm9 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm17, %zmm17
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm18 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm18, %zmm18
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm20 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm20, %zmm20
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm19, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[2,9],zero,zero,zero,xmm2[5,12,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[4,11],zero,zero,xmm19[0,7,14],zero,zero,xmm19[u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm2, %xmm19, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm19 = [1,2,4,6,0,0,0,0]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm20, %ymm19, %ymm19
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[0,7,10,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm19, %zmm10, %zmm10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm10, %zmm2 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[5,12],zero,zero,xmm16[1,8,15],zero,zero,xmm16[u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm10, %xmm16, %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 {%k5} = ymm18[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm16 = [1,3,4,6,0,0,0,0]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm20, %ymm16, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[1,4,11,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm16, %zmm11, %zmm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm11, %zmm10 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm3[6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm11, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 {%k5} = ymm17[u,u,u,u,u,u,u,u,u,5,12,3,10,1,8,15,22,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [1,3,5,6,0,0,0,0]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm20, %ymm11, %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[2,5,8,15,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm11, %zmm9, %zmm9
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm3 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm9 = [0,1,10,3,4,13,6,7,8,25,18,11,28,21,14,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm9, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm4[u,u,u,u,2,9],zero,zero,zero,xmm4[5,12],zero,zero,xmm4[u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u],zero,zero,xmm4[0,7,14],zero,zero,xmm4[3,10,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    movl $4186112, %eax # imm = 0x3FE000
-; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 {%k1} = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,1,8,15,22,29,20,27,18,25,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm16[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm15[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm15[0,7,14]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm21, %xmm22, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm0, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm21, %ymm20 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm21 = [1,3,4,6,0,0,0,0]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm18, %ymm21, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[1,4,11,14,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm21, %zmm20, %zmm20
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm20, %zmm17 {%k5}
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm19, %zmm0, %zmm17 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm7, %ymm5 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm11, %ymm7 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,1,10,3,4,13,6,7,8,25,18,11,28,21,14,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm11, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm9 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm9[u,u,u,u,2,9],zero,zero,zero,xmm9[5,12],zero,zero,xmm9[u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u],zero,zero,xmm9[0,7,14],zero,zero,xmm9[3,10,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm10, %xmm9, %xmm9
+; AVX512DQ-BW-FCP-NEXT:    movl $4186112, %eax # imm = 0x3FE000
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm14, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm15, %zmm0, %zmm10 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[1,8,15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,0,7,14],zero,zero,xmm5[3,10],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm5, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1,2],ymm0[3,4,5,6,7],ymm4[8,9,10],ymm0[11,12,13,14,15]
-; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm3 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm9 {%k1} = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,1,8,15,22,29,20,27,18,25,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm4, %ymm2 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm5 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm5[u,u,4,11],zero,zero,xmm5[0,7,14],zero,zero,xmm5[u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm16[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm15[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm15[1,8,15]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm4, %ymm3 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm4 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm4, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[6,13],zero,zero,xmm2[2,9],zero,zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm0[u,u,u,u,u,u,u,u,u,5,12,3,10,1,8,15,22,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [1,3,5,6,0,0,0,0]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm18, %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,15,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm3, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k5}
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[1,8,15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm7[u,u,u,u,u,u,0,7,14],zero,zero,xmm7[3,10],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm4 = [16,17,18,19,20,21,22,23,24,25,26,43,44,45,46,47]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm3, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,1,2,3,4,5,6,7,8,17,18,19,20,21,22,23]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2d %zmm4, %zmm2, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rsi)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm8, (%rdx)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm12, (%rcx)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, (%r8)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, (%r9)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, (%rdi)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm14, (%r9)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm17, (%rdi)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %wide.vec = load <448 x i8>, ptr %in.vec, align 64
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll
index a034363895c0..4a99ebecab5c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll
@@ -445,14 +445,14 @@ define void @store_i16_stride2_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
 ; AVX512-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i16_stride2_vf16:
@@ -461,14 +461,14 @@ define void @store_i16_stride2_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
 ; AVX512-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i16_stride2_vf16:
@@ -477,14 +477,14 @@ define void @store_i16_stride2_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride2_vf16:
@@ -493,14 +493,14 @@ define void @store_i16_stride2_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512DQ-FCP-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i16_stride2_vf16:
@@ -684,22 +684,22 @@ define void @store_i16_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i16_stride2_vf32:
@@ -712,22 +712,22 @@ define void @store_i16_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512-FCP-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
-; AVX512-FCP-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i16_stride2_vf32:
@@ -740,22 +740,22 @@ define void @store_i16_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride2_vf32:
@@ -768,22 +768,22 @@ define void @store_i16_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512DQ-FCP-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i16_stride2_vf32:
@@ -1075,214 +1075,214 @@ define void @store_i16_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ;
 ; AVX512-LABEL: store_i16_stride2_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm1
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX512-NEXT:    vmovdqa 80(%rsi), %xmm1
+; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm16
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-NEXT:    vmovdqa 80(%rsi), %xmm3
-; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm5
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm6
+; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm3
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa 112(%rsi), %xmm5
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm6
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; AVX512-NEXT:    vmovdqa 112(%rsi), %xmm6
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm7
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm6
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm7
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
 ; AVX512-NEXT:    vmovdqa (%rsi), %xmm7
 ; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm9
 ; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm10
 ; AVX512-NEXT:    vmovdqa 48(%rsi), %xmm11
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm12
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm13
 ; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm14
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
-; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm12
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512-NEXT:    vmovdqa %xmm9, 48(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm0, 32(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm7, 16(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm14, (%rdx)
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
+; AVX512-NEXT:    vmovdqa %xmm7, (%rdx)
+; AVX512-NEXT:    vmovdqa %xmm0, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm9, 32(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm13, 48(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm10, 64(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm14, 80(%rdx)
 ; AVX512-NEXT:    vmovdqa %xmm11, 112(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm13, 96(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm10, 80(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm15, 64(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm6, 240(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm8, 224(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm5, 208(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm4, 192(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm3, 176(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm2, 160(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm1, 144(%rdx)
-; AVX512-NEXT:    vmovdqa64 %xmm16, 128(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm15, 96(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm6, 192(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm8, 208(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm5, 240(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm4, 224(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm3, 128(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm2, 144(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm1, 176(%rdx)
+; AVX512-NEXT:    vmovdqa64 %xmm16, 160(%rdx)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i16_stride2_vf64:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa 80(%rsi), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa 80(%rdi), %xmm2
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm16
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT:    vmovdqa 80(%rsi), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa 80(%rdi), %xmm4
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-FCP-NEXT:    vmovdqa 112(%rsi), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm6
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; AVX512-FCP-NEXT:    vmovdqa 112(%rsi), %xmm6
-; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm7
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm7
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm7
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm9
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm10
 ; AVX512-FCP-NEXT:    vmovdqa 48(%rsi), %xmm11
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm12
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm13
 ; AVX512-FCP-NEXT:    vmovdqa 48(%rdi), %xmm14
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
-; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm12
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm9, 48(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, 32(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm7, 16(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm14, (%rdx)
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
+; AVX512-FCP-NEXT:    vmovdqa %xmm7, (%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm9, 32(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm13, 48(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm10, 64(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm14, 80(%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa %xmm11, 112(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm13, 96(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm10, 80(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm15, 64(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm6, 240(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm8, 224(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm5, 208(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, 192(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm3, 176(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, 160(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, 144(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, 128(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm15, 96(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm6, 192(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm8, 208(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, 240(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm4, 224(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, 128(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm2, 144(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, 176(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, 160(%rdx)
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i16_stride2_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm1
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX512DQ-NEXT:    vmovdqa 80(%rsi), %xmm1
+; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm16
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-NEXT:    vmovdqa 80(%rsi), %xmm3
-; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm5
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm6
+; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm3
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa 112(%rsi), %xmm5
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm6
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; AVX512DQ-NEXT:    vmovdqa 112(%rsi), %xmm6
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm7
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm6
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm7
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm7
 ; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm9
 ; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm10
 ; AVX512DQ-NEXT:    vmovdqa 48(%rsi), %xmm11
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm12
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm13
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm14
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
-; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm12
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm9, 48(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm0, 32(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm7, 16(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm14, (%rdx)
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
+; AVX512DQ-NEXT:    vmovdqa %xmm7, (%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm0, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm9, 32(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm13, 48(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm10, 64(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm14, 80(%rdx)
 ; AVX512DQ-NEXT:    vmovdqa %xmm11, 112(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm13, 96(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm10, 80(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm15, 64(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm6, 240(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm8, 224(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm5, 208(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm4, 192(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm3, 176(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm2, 160(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm1, 144(%rdx)
-; AVX512DQ-NEXT:    vmovdqa64 %xmm16, 128(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm15, 96(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm6, 192(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm8, 208(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm5, 240(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm4, 224(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm3, 128(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm2, 144(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm1, 176(%rdx)
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, 160(%rdx)
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride2_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rsi), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rdi), %xmm2
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm16
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rsi), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rsi), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm6
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rsi), %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm7
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm7
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm7
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm9
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm10
 ; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rsi), %xmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rdi), %xmm14
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm12
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, 48(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, 32(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, 16(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm14, (%rdx)
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm12
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, (%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm9, 32(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm13, 48(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm10, 64(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm14, 80(%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm11, 112(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm13, 96(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm10, 80(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm15, 64(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, 240(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, 224(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, 208(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 192(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 176(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, 160(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 144(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, 128(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm15, 96(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, 192(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, 208(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, 240(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 224(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 128(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, 144(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 176(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, 160(%rdx)
 ; AVX512DQ-FCP-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i16_stride2_vf64:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
index 418c987ab9a3..3311a311c8e4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
@@ -1783,7 +1783,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
 ; AVX512-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm6, (%r9)
@@ -1856,7 +1856,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
 ; AVX512-FCP-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, (%r9)
@@ -1932,7 +1932,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm6, (%r9)
@@ -2005,7 +2005,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, (%r9)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index 6f50d61f4d1f..fafb69be0d38 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -1388,7 +1388,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm9[1,3,3,1]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,1,8,9],zero,zero,ymm7[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm7[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} ymm8 = (mem & ~ymm8) | ymm7
+; AVX512-NEXT:    vpternlogd {{.*#+}} ymm8 = (m32bcst & ~ymm8) | ymm7
 ; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm2 ^ (mem & (ymm8 ^ ymm2))
 ; AVX512-NEXT:    vpsrlq $48, %xmm4, %xmm2
 ; AVX512-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm3[1],xmm2[1]
@@ -1448,7 +1448,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm9[1,3,3,1]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,1,8,9],zero,zero,ymm7[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm7[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm8 = (mem & ~ymm8) | ymm7
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm8 = (m32bcst & ~ymm8) | ymm7
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm6 ^ (mem & (ymm8 ^ ymm6))
 ; AVX512-FCP-NEXT:    vpsrlq $48, %xmm3, %xmm3
 ; AVX512-FCP-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
@@ -1511,7 +1511,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm9[1,3,3,1]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,1,8,9],zero,zero,ymm7[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm7[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm8 = (mem & ~ymm8) | ymm7
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm8 = (m32bcst & ~ymm8) | ymm7
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm2 ^ (mem & (ymm8 ^ ymm2))
 ; AVX512DQ-NEXT:    vpsrlq $48, %xmm4, %xmm2
 ; AVX512DQ-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm3[1],xmm2[1]
@@ -1571,7 +1571,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm9[1,3,3,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,1,8,9],zero,zero,ymm7[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm7[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm8 = (mem & ~ymm8) | ymm7
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm8 = (m32bcst & ~ymm8) | ymm7
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm6 ^ (mem & (ymm8 ^ ymm6))
 ; AVX512DQ-FCP-NEXT:    vpsrlq $48, %xmm3, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
@@ -13076,7 +13076,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm16 # 32-byte Folded Reload
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm1 ^ (mem & (zmm16 ^ zmm1))
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = mem ^ (ymm18 & (ymm0 ^ mem))
+; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = m32bcst ^ (ymm18 & (ymm0 ^ m32bcst))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} ymm21 = ymm21 ^ (ymm1 & (ymm21 ^ ymm0))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
@@ -13752,7 +13752,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm18 & (zmm5 ^ zmm6))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm0 & (zmm5 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm2 = mem ^ (ymm0 & (ymm2 ^ mem))
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm2 = m32bcst ^ (ymm0 & (ymm2 ^ m32bcst))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm3 ^ (ymm0 & (ymm3 ^ ymm2))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
@@ -14403,7 +14403,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm16 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm1 ^ (mem & (zmm16 ^ zmm1))
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm0 = mem ^ (ymm18 & (ymm0 ^ mem))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm0 = m32bcst ^ (ymm18 & (ymm0 ^ m32bcst))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm21 = ymm21 ^ (ymm1 & (ymm21 ^ ymm0))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
@@ -15079,7 +15079,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm18 & (zmm5 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm0 & (zmm5 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm2 = mem ^ (ymm0 & (ymm2 ^ mem))
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm2 = m32bcst ^ (ymm0 & (ymm2 ^ m32bcst))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm3 ^ (ymm0 & (ymm3 ^ ymm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
index 9c9dca82f60c..f626dfe5daf0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
@@ -4093,139 +4093,125 @@ define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa (%r8), %xmm5
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm26 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512-NEXT:    vpermd %zmm1, %zmm26, %zmm30
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm27 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512-NEXT:    movw $-30584, %r11w # imm = 0x8888
+; AVX512-NEXT:    vmovdqa (%r10), %xmm0
+; AVX512-NEXT:    vmovdqa (%rax), %xmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm20
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm21
+; AVX512-NEXT:    vmovdqa (%r9), %xmm0
+; AVX512-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm22
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm23
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm0
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm24
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm25
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm26
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm27
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512-NEXT:    vpermt2d %zmm10, %zmm17, %zmm9
+; AVX512-NEXT:    movb $-86, %r11b
 ; AVX512-NEXT:    kmovw %r11d, %k1
-; AVX512-NEXT:    vpermd %zmm0, %zmm27, %zmm30 {%k1}
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm9
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm10
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm11
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm28 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512-NEXT:    vpermd %zmm1, %zmm28, %zmm3
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512-NEXT:    movw $8738, %r11w # imm = 0x2222
-; AVX512-NEXT:    kmovw %r11d, %k2
-; AVX512-NEXT:    vpermd %zmm0, %zmm29, %zmm3 {%k2}
-; AVX512-NEXT:    vmovdqa 32(%r10), %ymm15
-; AVX512-NEXT:    vmovdqa 32(%rax), %ymm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm2[0],ymm15[0],ymm2[1],ymm15[1],ymm2[2],ymm15[2],ymm2[3],ymm15[3],ymm2[8],ymm15[8],ymm2[9],ymm15[9],ymm2[10],ymm15[10],ymm2[11],ymm15[11]
-; AVX512-NEXT:    vmovdqa 32(%r9), %ymm7
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm12
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm12[0],ymm7[0],ymm12[1],ymm7[1],ymm12[2],ymm7[2],ymm12[3],ymm7[3],ymm12[8],ymm7[8],ymm12[9],ymm7[9],ymm12[10],ymm7[10],ymm12[11],ymm7[11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512-NEXT:    vpermd %zmm13, %zmm19, %zmm31
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512-NEXT:    vpermd %zmm6, %zmm18, %zmm31 {%k1}
-; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm13
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm1
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512-NEXT:    vpermd %zmm6, %zmm20, %zmm14
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm6
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm6[0],ymm13[0],ymm6[1],ymm13[1],ymm6[2],ymm13[2],ymm6[3],ymm13[3],ymm6[8],ymm13[8],ymm6[9],ymm13[9],ymm6[10],ymm13[10],ymm6[11],ymm13[11]
-; AVX512-NEXT:    vpermd %zmm4, %zmm21, %zmm14 {%k2}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm15[4],ymm2[5],ymm15[5],ymm2[6],ymm15[6],ymm2[7],ymm15[7],ymm2[12],ymm15[12],ymm2[13],ymm15[13],ymm2[14],ymm15[14],ymm2[15],ymm15[15]
-; AVX512-NEXT:    vmovdqa 32(%r10), %xmm2
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm12 = ymm12[4],ymm7[4],ymm12[5],ymm7[5],ymm12[6],ymm7[6],ymm12[7],ymm7[7],ymm12[12],ymm7[12],ymm12[13],ymm7[13],ymm12[14],ymm7[14],ymm12[15],ymm7[15]
-; AVX512-NEXT:    vmovdqa 32(%rax), %xmm7
-; AVX512-NEXT:    vpermd %zmm12, %zmm19, %zmm17
-; AVX512-NEXT:    vmovdqa 32(%r9), %xmm12
-; AVX512-NEXT:    vpermd %zmm4, %zmm18, %zmm17 {%k1}
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm15
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm6[4],ymm13[4],ymm6[5],ymm13[5],ymm6[6],ymm13[6],ymm6[7],ymm13[7],ymm6[12],ymm13[12],ymm6[13],ymm13[13],ymm6[14],ymm13[14],ymm6[15],ymm13[15]
-; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm13
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm1
-; AVX512-NEXT:    vpermd %zmm0, %zmm20, %zmm16
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512-NEXT:    vmovdqa 32(%r10), %ymm5
+; AVX512-NEXT:    vmovdqa 32(%rax), %ymm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[8],ymm5[8],ymm10[9],ymm5[9],ymm10[10],ymm5[10],ymm10[11],ymm5[11]
+; AVX512-NEXT:    vmovdqa 32(%r9), %ymm13
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm15
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512-NEXT:    vpermt2d %zmm11, %zmm18, %zmm0
+; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm3
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm12
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm3[0],ymm12[1],ymm3[1],ymm12[2],ymm3[2],ymm12[3],ymm3[3],ymm12[8],ymm3[8],ymm12[9],ymm3[9],ymm12[10],ymm3[10],ymm12[11],ymm3[11]
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm2
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512-NEXT:    vpermt2d %zmm14, %zmm19, %zmm11
+; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm5[4],ymm10[5],ymm5[5],ymm10[6],ymm5[6],ymm10[7],ymm5[7],ymm10[12],ymm5[12],ymm10[13],ymm5[13],ymm10[14],ymm5[14],ymm10[15],ymm5[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm18, %zmm5
+; AVX512-NEXT:    vmovdqa 32(%r10), %xmm6
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm3[4],ymm12[5],ymm3[5],ymm12[6],ymm3[6],ymm12[7],ymm3[7],ymm12[12],ymm3[12],ymm12[13],ymm3[13],ymm12[14],ymm3[14],ymm12[15],ymm3[15]
+; AVX512-NEXT:    vmovdqa 32(%rax), %xmm10
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512-NEXT:    vmovdqa 32(%r9), %xmm4
+; AVX512-NEXT:    vpermt2d %zmm3, %zmm19, %zmm13
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm2
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512-NEXT:    vpermt2d %zmm3, %zmm16, %zmm5
+; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm3
+; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm12
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm14
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm0
-; AVX512-NEXT:    vpermd %zmm4, %zmm21, %zmm16 {%k2}
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm15[0],xmm12[0],xmm15[1],xmm12[1],xmm15[2],xmm12[2],xmm15[3],xmm12[3]
-; AVX512-NEXT:    vpermd %zmm4, %zmm26, %zmm23
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
-; AVX512-NEXT:    vpermd %zmm4, %zmm27, %zmm23 {%k1}
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512-NEXT:    vpermd %zmm4, %zmm28, %zmm22
-; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512-NEXT:    vpermd %zmm6, %zmm29, %zmm22 {%k2}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm15[4],xmm12[4],xmm15[5],xmm12[5],xmm15[6],xmm12[6],xmm15[7],xmm12[7]
-; AVX512-NEXT:    vpermd %zmm6, %zmm26, %zmm25
-; AVX512-NEXT:    vpermd %zmm2, %zmm27, %zmm25 {%k1}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm13[4],xmm4[5],xmm13[5],xmm4[6],xmm13[6],xmm4[7],xmm13[7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512-NEXT:    vpermd %zmm0, %zmm28, %zmm24
-; AVX512-NEXT:    vpermd %zmm2, %zmm29, %zmm24 {%k2}
-; AVX512-NEXT:    vmovdqa (%r10), %ymm0
-; AVX512-NEXT:    vmovdqa (%rax), %ymm1
-; AVX512-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm15
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm5
+; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
+; AVX512-NEXT:    vmovdqa (%r10), %ymm8
+; AVX512-NEXT:    vmovdqa (%rax), %ymm7
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
+; AVX512-NEXT:    vmovdqa (%r9), %ymm3
 ; AVX512-NEXT:    vmovdqa (%r8), %ymm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11]
-; AVX512-NEXT:    vpermd %zmm6, %zmm19, %zmm6
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512-NEXT:    vpermd %zmm7, %zmm18, %zmm6 {%k1}
-; AVX512-NEXT:    vmovdqa (%rcx), %ymm7
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm1
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15]
-; AVX512-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512-NEXT:    vpermd %zmm2, %zmm19, %zmm2
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm12
-; AVX512-NEXT:    vpermd %zmm0, %zmm18, %zmm2 {%k1}
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm12[0],ymm4[0],ymm12[1],ymm4[1],ymm12[2],ymm4[2],ymm12[3],ymm4[3],ymm12[8],ymm4[8],ymm12[9],ymm4[9],ymm12[10],ymm4[10],ymm12[11],ymm4[11]
-; AVX512-NEXT:    vpermd %zmm0, %zmm20, %zmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm1[0],ymm7[0],ymm1[1],ymm7[1],ymm1[2],ymm7[2],ymm1[3],ymm7[3],ymm1[8],ymm7[8],ymm1[9],ymm7[9],ymm1[10],ymm7[10],ymm1[11],ymm7[11]
-; AVX512-NEXT:    vpermd %zmm13, %zmm21, %zmm0 {%k2}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm7[4],ymm1[5],ymm7[5],ymm1[6],ymm7[6],ymm1[7],ymm7[7],ymm1[12],ymm7[12],ymm1[13],ymm7[13],ymm1[14],ymm7[14],ymm1[15],ymm7[15]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm12[4],ymm4[4],ymm12[5],ymm4[5],ymm12[6],ymm4[6],ymm12[7],ymm4[7],ymm12[12],ymm4[12],ymm12[13],ymm4[13],ymm12[14],ymm4[14],ymm12[15],ymm4[15]
-; AVX512-NEXT:    vpermd %zmm4, %zmm20, %zmm4
-; AVX512-NEXT:    vpermd %zmm1, %zmm21, %zmm4 {%k2}
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm5 = xmm5[0],mem[0],xmm5[1],mem[1],xmm5[2],mem[2],xmm5[3],mem[3]
-; AVX512-NEXT:    vpermd %zmm5, %zmm26, %zmm5
-; AVX512-NEXT:    vpermd %zmm1, %zmm27, %zmm5 {%k1}
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
-; AVX512-NEXT:    vpermd %zmm7, %zmm28, %zmm7
-; AVX512-NEXT:    vpermd %zmm1, %zmm29, %zmm7 {%k2}
-; AVX512-NEXT:    movb $-86, %al
-; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovdqa64 %zmm30, %zmm3 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm31, %zmm14 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm23, %zmm22 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm25, %zmm24 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm6
+; AVX512-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]
+; AVX512-NEXT:    vmovdqa (%rsi), %ymm1
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512-NEXT:    vpermt2d %zmm14, %zmm19, %zmm10
+; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm10 {%k1}
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512-NEXT:    vpermt2d %zmm6, %zmm18, %zmm3
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm3
+; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm7, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm4, 192(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm0, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm24, 320(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm22, 256(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm16, 448(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm14, 384(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm3, 64(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm10, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm15, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm13, 448(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm11, 384(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm9, 64(%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -4234,139 +4220,123 @@ define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm21
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm22
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm5
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm26 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm26, %zmm25
-; AVX512-FCP-NEXT:    movw $-30584, %r11w # imm = 0x8888
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm2
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm23
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm24
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm25
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm26
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm9
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm27
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
+; AVX512-FCP-NEXT:    vpermi2d %zmm0, %zmm10, %zmm19
+; AVX512-FCP-NEXT:    movb $-86, %r11b
 ; AVX512-FCP-NEXT:    kmovw %r11d, %k1
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm26, %zmm25 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm8
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,1,1,1,1,0,0,2,0,3,0,3,0,0,0]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm6, %zmm29
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm24 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512-FCP-NEXT:    movw $8738, %r11w # imm = 0x2222
-; AVX512-FCP-NEXT:    kmovw %r11d, %k2
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm24, %zmm29 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm19 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm12
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512-FCP-NEXT:    vpermd %zmm11, %zmm16, %zmm27
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512-FCP-NEXT:    vpermd %zmm6, %zmm17, %zmm27 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm14
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm15
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm1[0],ymm15[0],ymm1[1],ymm15[1],ymm1[2],ymm15[2],ymm1[3],ymm15[3],ymm1[8],ymm15[8],ymm1[9],ymm15[9],ymm1[10],ymm15[10],ymm1[11],ymm15[11]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpermd %zmm6, %zmm18, %zmm30
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm0[0],ymm14[0],ymm0[1],ymm14[1],ymm0[2],ymm14[2],ymm0[3],ymm14[3],ymm0[8],ymm14[8],ymm0[9],ymm14[9],ymm0[10],ymm14[10],ymm0[11],ymm14[11]
-; AVX512-FCP-NEXT:    vpermd %zmm6, %zmm19, %zmm30 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %xmm11
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm3
-; AVX512-FCP-NEXT:    vpermd %zmm6, %zmm16, %zmm28
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm6
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm17, %zmm28 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm14
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512-FCP-NEXT:    vpermt2d %zmm10, %zmm17, %zmm15
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm11
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm13
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm3
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm0
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm10
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, %zmm10 {%k1}
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm1[4],ymm5[5],ymm1[5],ymm5[6],ymm1[6],ymm5[7],ymm1[7],ymm5[12],ymm1[12],ymm5[13],ymm1[13],ymm5[14],ymm1[14],ymm5[15],ymm1[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %xmm6
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm4
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm13
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm12 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm2
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm14[4],ymm0[5],ymm14[5],ymm0[6],ymm14[6],ymm0[7],ymm14[7],ymm0[12],ymm14[12],ymm0[13],ymm14[13],ymm0[14],ymm14[14],ymm0[15],ymm14[15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm13
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm15[4],ymm1[5],ymm15[5],ymm1[6],ymm15[6],ymm1[7],ymm15[7],ymm1[12],ymm15[12],ymm1[13],ymm15[13],ymm1[14],ymm15[14],ymm1[15],ymm15[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm11, %zmm18, %zmm12
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm12 {%k1}
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm13[0],xmm2[1],xmm13[1],xmm2[2],xmm13[2],xmm2[3],xmm13[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm16, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm3
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm11
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm15
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm18, %zmm31
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm19, %zmm31 {%k2}
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm1, %zmm21
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm11[0],xmm3[1],xmm11[1],xmm3[2],xmm11[2],xmm3[3],xmm11[3]
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm26, %zmm21 {%k1}
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm1, %zmm20
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm24, %zmm20 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm11[4],xmm3[5],xmm11[5],xmm3[6],xmm11[6],xmm3[7],xmm11[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm26, %zmm23
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm26, %zmm23 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm15[4],xmm12[5],xmm15[5],xmm12[6],xmm15[6],xmm12[7],xmm15[7]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm24, %zmm22
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm24, %zmm22 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa (%r10), %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm1
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm14
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm13[4],xmm2[5],xmm13[5],xmm2[6],xmm13[6],xmm2[7],xmm13[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa (%r10), %ymm8
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm7
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm3
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm6
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm3[0],ymm6[1],ymm3[1],ymm6[2],ymm3[2],ymm6[3],ymm3[3],ymm6[8],ymm3[8],ymm6[9],ymm3[9],ymm6[10],ymm3[10],ymm6[11],ymm3[11]
-; AVX512-FCP-NEXT:    vpermd %zmm11, %zmm16, %zmm11
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm17, %zmm11 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm11
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm13
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm2
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm12
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]
-; AVX512-FCP-NEXT:    vpermd %zmm13, %zmm18, %zmm13
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm15
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm14
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm14[0],ymm15[0],ymm14[1],ymm15[1],ymm14[2],ymm15[2],ymm14[3],ymm15[3],ymm14[8],ymm15[8],ymm14[9],ymm15[9],ymm14[10],ymm15[10],ymm14[11],ymm15[11]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm19, %zmm13 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm6[4],ymm3[4],ymm6[5],ymm3[5],ymm6[6],ymm3[6],ymm6[7],ymm3[7],ymm6[12],ymm3[12],ymm6[13],ymm3[13],ymm6[14],ymm3[14],ymm6[15],ymm3[15]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm16, %zmm1
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm17, %zmm1 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm14[4],ymm15[4],ymm14[5],ymm15[5],ymm14[6],ymm15[6],ymm14[7],ymm15[7],ymm14[12],ymm15[12],ymm14[13],ymm15[13],ymm14[14],ymm15[14],ymm14[15],ymm15[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm18, %zmm2
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm19, %zmm2 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm3 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm3 = xmm5[0],mem[0],xmm5[1],mem[1],xmm5[2],mem[2],xmm5[3],mem[3]
-; AVX512-FCP-NEXT:    vpermd %zmm3, %zmm26, %zmm3
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm26, %zmm3 {%k1}
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm24, %zmm4
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm24, %zmm4 {%k2}
-; AVX512-FCP-NEXT:    movb $-86, %al
-; AVX512-FCP-NEXT:    kmovw %eax, %k1
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm25, %zmm29 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm27, %zmm30 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm28, %zmm31 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm21, %zmm20 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm23, %zmm22 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, %zmm13 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm18, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm6, %zmm17, %zmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm18, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm20, %zmm4
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 128(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm22, 320(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, 256(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm31, 448(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm30, 384(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm29, 64(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 256(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 448(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 384(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rax)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -4374,139 +4344,125 @@ define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm5
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm26 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm26, %zmm30
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm27 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512DQ-NEXT:    movw $-30584, %r11w # imm = 0x8888
+; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm1
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm20
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm21
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm22
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm23
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm1
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm24
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm25
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm26
+; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm27
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512DQ-NEXT:    vpermt2d %zmm10, %zmm17, %zmm9
+; AVX512DQ-NEXT:    movb $-86, %r11b
 ; AVX512DQ-NEXT:    kmovw %r11d, %k1
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm27, %zmm30 {%k1}
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm9
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm10
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm11
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm28 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm28, %zmm3
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-NEXT:    movw $8738, %r11w # imm = 0x2222
-; AVX512DQ-NEXT:    kmovw %r11d, %k2
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm29, %zmm3 {%k2}
-; AVX512DQ-NEXT:    vmovdqa 32(%r10), %ymm15
-; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm2[0],ymm15[0],ymm2[1],ymm15[1],ymm2[2],ymm15[2],ymm2[3],ymm15[3],ymm2[8],ymm15[8],ymm2[9],ymm15[9],ymm2[10],ymm15[10],ymm2[11],ymm15[11]
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm7
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm12
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm12[0],ymm7[0],ymm12[1],ymm7[1],ymm12[2],ymm7[2],ymm12[3],ymm7[3],ymm12[8],ymm7[8],ymm12[9],ymm7[9],ymm12[10],ymm7[10],ymm12[11],ymm7[11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512DQ-NEXT:    vpermd %zmm13, %zmm19, %zmm31
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm18, %zmm31 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm13
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm1
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm20, %zmm14
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm6
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm6[0],ymm13[0],ymm6[1],ymm13[1],ymm6[2],ymm13[2],ymm6[3],ymm13[3],ymm6[8],ymm13[8],ymm6[9],ymm13[9],ymm6[10],ymm13[10],ymm6[11],ymm13[11]
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm21, %zmm14 {%k2}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm15[4],ymm2[5],ymm15[5],ymm2[6],ymm15[6],ymm2[7],ymm15[7],ymm2[12],ymm15[12],ymm2[13],ymm15[13],ymm2[14],ymm15[14],ymm2[15],ymm15[15]
-; AVX512DQ-NEXT:    vmovdqa 32(%r10), %xmm2
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm12 = ymm12[4],ymm7[4],ymm12[5],ymm7[5],ymm12[6],ymm7[6],ymm12[7],ymm7[7],ymm12[12],ymm7[12],ymm12[13],ymm7[13],ymm12[14],ymm7[14],ymm12[15],ymm7[15]
-; AVX512DQ-NEXT:    vmovdqa 32(%rax), %xmm7
-; AVX512DQ-NEXT:    vpermd %zmm12, %zmm19, %zmm17
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm12
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm18, %zmm17 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm15
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm6[4],ymm13[4],ymm6[5],ymm13[5],ymm6[6],ymm13[6],ymm6[7],ymm13[7],ymm6[12],ymm13[12],ymm6[13],ymm13[13],ymm6[14],ymm13[14],ymm6[15],ymm13[15]
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm13
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm1
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm20, %zmm16
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vmovdqa 32(%r10), %ymm5
+; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[8],ymm5[8],ymm10[9],ymm5[9],ymm10[10],ymm5[10],ymm10[11],ymm5[11]
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm13
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm15
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512DQ-NEXT:    vpermt2d %zmm11, %zmm18, %zmm0
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm3
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm12
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm3[0],ymm12[1],ymm3[1],ymm12[2],ymm3[2],ymm12[3],ymm3[3],ymm12[8],ymm3[8],ymm12[9],ymm3[9],ymm12[10],ymm3[10],ymm12[11],ymm3[11]
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm2
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm14, %zmm19, %zmm11
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm5[4],ymm10[5],ymm5[5],ymm10[6],ymm5[6],ymm10[7],ymm5[7],ymm10[12],ymm5[12],ymm10[13],ymm5[13],ymm10[14],ymm5[14],ymm10[15],ymm5[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm18, %zmm5
+; AVX512DQ-NEXT:    vmovdqa 32(%r10), %xmm6
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm3[4],ymm12[5],ymm3[5],ymm12[6],ymm3[6],ymm12[7],ymm3[7],ymm12[12],ymm3[12],ymm12[13],ymm3[13],ymm12[14],ymm3[14],ymm12[15],ymm3[15]
+; AVX512DQ-NEXT:    vmovdqa 32(%rax), %xmm10
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm4
+; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm19, %zmm13
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm16, %zmm5
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm3
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm12
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm14
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm0
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm21, %zmm16 {%k2}
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm15[0],xmm12[0],xmm15[1],xmm12[1],xmm15[2],xmm12[2],xmm15[3],xmm12[3]
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm26, %zmm23
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm27, %zmm23 {%k1}
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm28, %zmm22
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm29, %zmm22 {%k2}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm15[4],xmm12[4],xmm15[5],xmm12[5],xmm15[6],xmm12[6],xmm15[7],xmm12[7]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm26, %zmm25
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm27, %zmm25 {%k1}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm13[4],xmm4[5],xmm13[5],xmm4[6],xmm13[6],xmm4[7],xmm13[7]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm28, %zmm24
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm29, %zmm24 {%k2}
-; AVX512DQ-NEXT:    vmovdqa (%r10), %ymm0
-; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm1
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm15
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm5
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
+; AVX512DQ-NEXT:    vmovdqa (%r10), %ymm8
+; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm7
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm3
 ; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm19, %zmm6
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512DQ-NEXT:    vpermd %zmm7, %zmm18, %zmm6 {%k1}
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm7
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm1
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15]
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm19, %zmm2
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm12
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm18, %zmm2 {%k1}
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm12[0],ymm4[0],ymm12[1],ymm4[1],ymm12[2],ymm4[2],ymm12[3],ymm4[3],ymm12[8],ymm4[8],ymm12[9],ymm4[9],ymm12[10],ymm4[10],ymm12[11],ymm4[11]
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm20, %zmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm1[0],ymm7[0],ymm1[1],ymm7[1],ymm1[2],ymm7[2],ymm1[3],ymm7[3],ymm1[8],ymm7[8],ymm1[9],ymm7[9],ymm1[10],ymm7[10],ymm1[11],ymm7[11]
-; AVX512DQ-NEXT:    vpermd %zmm13, %zmm21, %zmm0 {%k2}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm7[4],ymm1[5],ymm7[5],ymm1[6],ymm7[6],ymm1[7],ymm7[7],ymm1[12],ymm7[12],ymm1[13],ymm7[13],ymm1[14],ymm7[14],ymm1[15],ymm7[15]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm12[4],ymm4[4],ymm12[5],ymm4[5],ymm12[6],ymm4[6],ymm12[7],ymm4[7],ymm12[12],ymm4[12],ymm12[13],ymm4[13],ymm12[14],ymm4[14],ymm12[15],ymm4[15]
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm20, %zmm4
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm21, %zmm4 {%k2}
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX512DQ-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm5 = xmm5[0],mem[0],xmm5[1],mem[1],xmm5[2],mem[2],xmm5[3],mem[3]
-; AVX512DQ-NEXT:    vpermd %zmm5, %zmm26, %zmm5
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm27, %zmm5 {%k1}
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
-; AVX512DQ-NEXT:    vpermd %zmm7, %zmm28, %zmm7
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm29, %zmm7 {%k2}
-; AVX512DQ-NEXT:    movb $-86, %al
-; AVX512DQ-NEXT:    kmovw %eax, %k1
-; AVX512DQ-NEXT:    vmovdqa64 %zmm30, %zmm3 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm31, %zmm14 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm23, %zmm22 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm25, %zmm24 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm6
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm1
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm14, %zmm19, %zmm10
+; AVX512DQ-NEXT:    vmovdqa64 %zmm6, %zmm10 {%k1}
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm6, %zmm18, %zmm3
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm3
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm7, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 192(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm24, 320(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm22, 256(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 448(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 384(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 64(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm15, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm13, 448(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm11, 384(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm9, 64(%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -4515,139 +4471,123 @@ define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm22
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm5
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm26 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm26, %zmm25
-; AVX512DQ-FCP-NEXT:    movw $-30584, %r11w # imm = 0x8888
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm23
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm24
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm26
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm9
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm27
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
+; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm10, %zmm19
+; AVX512DQ-FCP-NEXT:    movb $-86, %r11b
 ; AVX512DQ-FCP-NEXT:    kmovw %r11d, %k1
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm26, %zmm25 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm8
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,1,1,1,1,0,0,2,0,3,0,3,0,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm6, %zmm29
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm24 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-FCP-NEXT:    movw $8738, %r11w # imm = 0x2222
-; AVX512DQ-FCP-NEXT:    kmovw %r11d, %k2
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm24, %zmm29 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm19 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm12
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm13
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm11, %zmm16, %zmm27
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm17, %zmm27 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm14
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm15
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm1[0],ymm15[0],ymm1[1],ymm15[1],ymm1[2],ymm15[2],ymm1[3],ymm15[3],ymm1[8],ymm15[8],ymm1[9],ymm15[9],ymm1[10],ymm15[10],ymm1[11],ymm15[11]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm18, %zmm30
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm0[0],ymm14[0],ymm0[1],ymm14[1],ymm0[2],ymm14[2],ymm0[3],ymm14[3],ymm0[8],ymm14[8],ymm0[9],ymm14[9],ymm0[10],ymm14[10],ymm0[11],ymm14[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm19, %zmm30 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %xmm11
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm3
-; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm16, %zmm28
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm6
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm17, %zmm28 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm14
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm10, %zmm17, %zmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm13
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, %zmm10 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm1[4],ymm5[5],ymm1[5],ymm5[6],ymm1[6],ymm5[7],ymm1[7],ymm5[12],ymm1[12],ymm5[13],ymm1[13],ymm5[14],ymm1[14],ymm5[15],ymm1[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %xmm6
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm13
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm12 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm14[4],ymm0[5],ymm14[5],ymm0[6],ymm14[6],ymm0[7],ymm14[7],ymm0[12],ymm14[12],ymm0[13],ymm14[13],ymm0[14],ymm14[14],ymm0[15],ymm14[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm13
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm15[4],ymm1[5],ymm15[5],ymm1[6],ymm15[6],ymm1[7],ymm15[7],ymm1[12],ymm15[12],ymm1[13],ymm15[13],ymm1[14],ymm15[14],ymm1[15],ymm15[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm11, %zmm18, %zmm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm12 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm13[0],xmm2[1],xmm13[1],xmm2[2],xmm13[2],xmm2[3],xmm13[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm16, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm15
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm18, %zmm31
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm19, %zmm31 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm1, %zmm21
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm11[0],xmm3[1],xmm11[1],xmm3[2],xmm11[2],xmm3[3],xmm11[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm26, %zmm21 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm1, %zmm20
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm24, %zmm20 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm11[4],xmm3[5],xmm11[5],xmm3[6],xmm11[6],xmm3[7],xmm11[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm26, %zmm23
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm26, %zmm23 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm15[4],xmm12[5],xmm15[5],xmm12[6],xmm15[6],xmm12[7],xmm15[7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm24, %zmm22
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm24, %zmm22 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm1
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm13[4],xmm2[5],xmm13[5],xmm2[6],xmm13[6],xmm2[7],xmm13[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm7
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm6
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm3[0],ymm6[1],ymm3[1],ymm6[2],ymm3[2],ymm6[3],ymm3[3],ymm6[8],ymm3[8],ymm6[9],ymm3[9],ymm6[10],ymm3[10],ymm6[11],ymm3[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm11, %zmm16, %zmm11
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm17, %zmm11 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm13
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm12
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm13, %zmm18, %zmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm15
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm14
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm14[0],ymm15[0],ymm14[1],ymm15[1],ymm14[2],ymm15[2],ymm14[3],ymm15[3],ymm14[8],ymm15[8],ymm14[9],ymm15[9],ymm14[10],ymm15[10],ymm14[11],ymm15[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm19, %zmm13 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm6[4],ymm3[4],ymm6[5],ymm3[5],ymm6[6],ymm3[6],ymm6[7],ymm3[7],ymm6[12],ymm3[12],ymm6[13],ymm3[13],ymm6[14],ymm3[14],ymm6[15],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm16, %zmm1
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm17, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm14[4],ymm15[4],ymm14[5],ymm15[5],ymm14[6],ymm15[6],ymm14[7],ymm15[7],ymm14[12],ymm15[12],ymm14[13],ymm15[13],ymm14[14],ymm15[14],ymm14[15],ymm15[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm18, %zmm2
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm19, %zmm2 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm3 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm3 = xmm5[0],mem[0],xmm5[1],mem[1],xmm5[2],mem[2],xmm5[3],mem[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm26, %zmm3
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm26, %zmm3 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm24, %zmm4
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm24, %zmm4 {%k2}
-; AVX512DQ-FCP-NEXT:    movb $-86, %al
-; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm25, %zmm29 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm27, %zmm30 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm28, %zmm31 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm21, %zmm20 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm23, %zmm22 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, %zmm13 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm18, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm6, %zmm17, %zmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm18, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm20, %zmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm22, 320(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm31, 448(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm30, 384(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm29, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 256(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 448(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 384(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
@@ -7777,1095 +7717,959 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512-LABEL: store_i16_stride8_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $504, %rsp # imm = 0x1F8
+; AVX512-NEXT:    subq $392, %rsp # imm = 0x188
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-NEXT:    vmovdqa (%r10), %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%r10), %xmm4
-; AVX512-NEXT:    vmovdqa (%rax), %xmm1
-; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%rax), %xmm3
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512-NEXT:    vmovdqa (%r9), %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa (%r8), %xmm7
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512-NEXT:    vpermd %zmm2, %zmm30, %zmm0
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512-NEXT:    movw $-30584, %r11w # imm = 0x8888
-; AVX512-NEXT:    kmovw %r11d, %k2
-; AVX512-NEXT:    vpermd %zmm1, %zmm29, %zmm0 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    movw $8738, %r11w # imm = 0x2222
+; AVX512-NEXT:    vmovdqa (%r10), %xmm1
+; AVX512-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm20
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm21
+; AVX512-NEXT:    vmovdqa (%r9), %xmm2
+; AVX512-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm22
+; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm23
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm2
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm24
+; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm25
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm26
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm17, %zmm2
+; AVX512-NEXT:    movb $-86, %r11b
 ; AVX512-NEXT:    kmovw %r11d, %k1
-; AVX512-NEXT:    vmovdqa 96(%r10), %ymm2
-; AVX512-NEXT:    vmovdqa 96(%rax), %ymm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
-; AVX512-NEXT:    vmovdqa 96(%r9), %ymm8
-; AVX512-NEXT:    vmovdqa 96(%r8), %ymm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512-NEXT:    vpermd %zmm10, %zmm19, %zmm0
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512-NEXT:    vpermd %zmm1, %zmm18, %zmm0 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 96(%rcx), %ymm10
-; AVX512-NEXT:    vmovdqa 96(%rdx), %ymm11
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 96(%r10), %ymm0
+; AVX512-NEXT:    vmovdqa 96(%rax), %ymm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512-NEXT:    vmovdqa 96(%r9), %ymm3
+; AVX512-NEXT:    vmovdqa 96(%r8), %ymm4
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm5
+; AVX512-NEXT:    vmovdqa 96(%rcx), %ymm2
+; AVX512-NEXT:    vmovdqa 96(%rdx), %ymm6
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
 ; AVX512-NEXT:    vmovdqa 96(%rsi), %ymm12
 ; AVX512-NEXT:    vmovdqa 96(%rdi), %ymm13
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512-NEXT:    vpermd %zmm1, %zmm16, %zmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512-NEXT:    vpermd %zmm14, %zmm17, %zmm0 {%k1}
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
-; AVX512-NEXT:    vpermd %zmm5, %zmm19, %zmm0
-; AVX512-NEXT:    vpermd %zmm2, %zmm18, %zmm0 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[12],ymm10[12],ymm11[13],ymm10[13],ymm11[14],ymm10[14],ymm11[15],ymm10[15]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
-; AVX512-NEXT:    vpermd %zmm5, %zmm16, %zmm31
-; AVX512-NEXT:    vpermd %zmm2, %zmm17, %zmm31 {%k1}
-; AVX512-NEXT:    vmovdqa 96(%r10), %xmm2
-; AVX512-NEXT:    vmovdqa 96(%rax), %xmm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3]
-; AVX512-NEXT:    vmovdqa 96(%r9), %xmm10
-; AVX512-NEXT:    vmovdqa 96(%r8), %xmm11
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
-; AVX512-NEXT:    vpermd %zmm12, %zmm30, %zmm0
-; AVX512-NEXT:    vpermd %zmm9, %zmm29, %zmm0 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
-; AVX512-NEXT:    vpermd %zmm8, %zmm30, %zmm0
-; AVX512-NEXT:    vpermd %zmm2, %zmm29, %zmm0 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%r10), %ymm2
-; AVX512-NEXT:    vmovdqa 64(%rax), %ymm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[2],ymm2[2],ymm9[3],ymm2[3],ymm9[8],ymm2[8],ymm9[9],ymm2[9],ymm9[10],ymm2[10],ymm9[11],ymm2[11]
-; AVX512-NEXT:    vmovdqa 64(%r9), %ymm10
-; AVX512-NEXT:    vmovdqa 64(%r8), %ymm11
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11]
-; AVX512-NEXT:    vpermd %zmm12, %zmm19, %zmm0
-; AVX512-NEXT:    vpermd %zmm8, %zmm18, %zmm0 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 64(%rcx), %ymm12
-; AVX512-NEXT:    vmovdqa 64(%rdx), %ymm15
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
+; AVX512-NEXT:    vpermt2d %zmm10, %zmm19, %zmm7
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm18, %zmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm19, %zmm2
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 96(%r10), %xmm0
+; AVX512-NEXT:    vmovdqa 96(%rax), %xmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512-NEXT:    vmovdqa 96(%r9), %xmm3
+; AVX512-NEXT:    vmovdqa 96(%r8), %xmm4
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm2
+; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm6
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm13
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT:    vpermt2d %zmm10, %zmm17, %zmm7
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm17, %zmm2
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 64(%r10), %ymm6
+; AVX512-NEXT:    vmovdqa 64(%rax), %ymm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
+; AVX512-NEXT:    vmovdqa 64(%r9), %ymm3
+; AVX512-NEXT:    vmovdqa 64(%r8), %ymm4
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm10
+; AVX512-NEXT:    vmovdqa 64(%rcx), %ymm2
+; AVX512-NEXT:    vmovdqa 64(%rdx), %ymm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
 ; AVX512-NEXT:    vmovdqa 64(%rsi), %ymm1
 ; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512-NEXT:    vpermd %zmm8, %zmm16, %zmm26
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm15[0],ymm12[0],ymm15[1],ymm12[1],ymm15[2],ymm12[2],ymm15[3],ymm12[3],ymm15[8],ymm12[8],ymm15[9],ymm12[9],ymm15[10],ymm12[10],ymm15[11],ymm12[11]
-; AVX512-NEXT:    vpermd %zmm8, %zmm17, %zmm26 {%k1}
-; AVX512-NEXT:    vmovdqa 64(%r9), %xmm8
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm9[4],ymm2[4],ymm9[5],ymm2[5],ymm9[6],ymm2[6],ymm9[7],ymm2[7],ymm9[12],ymm2[12],ymm9[13],ymm2[13],ymm9[14],ymm2[14],ymm9[15],ymm2[15]
-; AVX512-NEXT:    vmovdqa 64(%r8), %xmm9
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm10 = ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[12],ymm10[12],ymm11[13],ymm10[13],ymm11[14],ymm10[14],ymm11[15],ymm10[15]
-; AVX512-NEXT:    vpermd %zmm10, %zmm19, %zmm5
-; AVX512-NEXT:    vpermd %zmm2, %zmm18, %zmm5 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm12[4],ymm15[5],ymm12[5],ymm15[6],ymm12[6],ymm15[7],ymm12[7],ymm15[12],ymm12[12],ymm15[13],ymm12[13],ymm15[14],ymm12[14],ymm15[15],ymm12[15]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512-NEXT:    vpermt2d %zmm14, %zmm19, %zmm7
+; AVX512-NEXT:    vmovdqa64 %zmm10, %zmm7 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512-NEXT:    vpermt2d %zmm5, %zmm18, %zmm3
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
-; AVX512-NEXT:    vpermd %zmm0, %zmm16, %zmm24
-; AVX512-NEXT:    vpermd %zmm2, %zmm17, %zmm24 {%k1}
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT:    vpermd %zmm1, %zmm30, %zmm1
-; AVX512-NEXT:    vpermd %zmm0, %zmm29, %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
-; AVX512-NEXT:    vmovdqa 32(%r10), %ymm0
-; AVX512-NEXT:    vmovdqa 32(%rax), %ymm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512-NEXT:    vmovdqa 32(%r9), %ymm4
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm8[0],ymm4[0],ymm8[1],ymm4[1],ymm8[2],ymm4[2],ymm8[3],ymm4[3],ymm8[8],ymm4[8],ymm8[9],ymm4[9],ymm8[10],ymm4[10],ymm8[11],ymm4[11]
-; AVX512-NEXT:    vpermd %zmm9, %zmm19, %zmm28
-; AVX512-NEXT:    vpermd %zmm3, %zmm18, %zmm28 {%k2}
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm9
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm11
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[2],ymm9[2],ymm11[3],ymm9[3],ymm11[8],ymm9[8],ymm11[9],ymm9[9],ymm11[10],ymm9[10],ymm11[11],ymm9[11]
-; AVX512-NEXT:    vpermd %zmm3, %zmm16, %zmm23
-; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm12
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm3[0],ymm12[0],ymm3[1],ymm12[1],ymm3[2],ymm12[2],ymm3[3],ymm12[3],ymm3[8],ymm12[8],ymm3[9],ymm12[9],ymm3[10],ymm12[10],ymm3[11],ymm12[11]
-; AVX512-NEXT:    vpermd %zmm6, %zmm17, %zmm23 {%k1}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm8[4],ymm4[4],ymm8[5],ymm4[5],ymm8[6],ymm4[6],ymm8[7],ymm4[7],ymm8[12],ymm4[12],ymm8[13],ymm4[13],ymm8[14],ymm4[14],ymm8[15],ymm4[15]
-; AVX512-NEXT:    vpermd %zmm1, %zmm19, %zmm25
-; AVX512-NEXT:    vpermd %zmm0, %zmm18, %zmm25 {%k2}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm12[4],ymm3[5],ymm12[5],ymm3[6],ymm12[6],ymm3[7],ymm12[7],ymm3[12],ymm12[12],ymm3[13],ymm12[13],ymm3[14],ymm12[14],ymm3[15],ymm12[15]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm11[4],ymm9[4],ymm11[5],ymm9[5],ymm11[6],ymm9[6],ymm11[7],ymm9[7],ymm11[12],ymm9[12],ymm11[13],ymm9[13],ymm11[14],ymm9[14],ymm11[15],ymm9[15]
-; AVX512-NEXT:    vpermd %zmm0, %zmm16, %zmm21
-; AVX512-NEXT:    vpermd %zmm3, %zmm17, %zmm21 {%k1}
-; AVX512-NEXT:    vmovdqa (%r10), %ymm3
-; AVX512-NEXT:    vmovdqa (%r9), %ymm4
-; AVX512-NEXT:    vmovdqa (%r8), %ymm6
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm6[0],ymm4[0],ymm6[1],ymm4[1],ymm6[2],ymm4[2],ymm6[3],ymm4[3],ymm6[8],ymm4[8],ymm6[9],ymm4[9],ymm6[10],ymm4[10],ymm6[11],ymm4[11]
-; AVX512-NEXT:    vpermd %zmm8, %zmm19, %zmm27
-; AVX512-NEXT:    vmovdqa (%rax), %ymm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm8[0],ymm3[0],ymm8[1],ymm3[1],ymm8[2],ymm3[2],ymm8[3],ymm3[3],ymm8[8],ymm3[8],ymm8[9],ymm3[9],ymm8[10],ymm3[10],ymm8[11],ymm3[11]
-; AVX512-NEXT:    vpermd %zmm9, %zmm18, %zmm27 {%k2}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm8[4],ymm3[4],ymm8[5],ymm3[5],ymm8[6],ymm3[6],ymm8[7],ymm3[7],ymm8[12],ymm3[12],ymm8[13],ymm3[13],ymm8[14],ymm3[14],ymm8[15],ymm3[15]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm6[4],ymm4[4],ymm6[5],ymm4[5],ymm6[6],ymm4[6],ymm6[7],ymm4[7],ymm6[12],ymm4[12],ymm6[13],ymm4[13],ymm6[14],ymm4[14],ymm6[15],ymm4[15]
-; AVX512-NEXT:    vmovdqa (%rcx), %ymm6
-; AVX512-NEXT:    vpermd %zmm4, %zmm19, %zmm20
-; AVX512-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512-NEXT:    vpermd %zmm3, %zmm18, %zmm20 {%k2}
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]
-; AVX512-NEXT:    vpermd %zmm8, %zmm16, %zmm18
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[8],ymm6[8],ymm8[9],ymm6[9],ymm8[10],ymm6[10],ymm8[11],ymm6[11]
-; AVX512-NEXT:    vpermd %zmm9, %zmm17, %zmm18 {%k1}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm8[4],ymm6[4],ymm8[5],ymm6[5],ymm8[6],ymm6[6],ymm8[7],ymm6[7],ymm8[12],ymm6[12],ymm8[13],ymm6[13],ymm8[14],ymm6[14],ymm8[15],ymm6[15]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]
-; AVX512-NEXT:    vmovdqa 32(%r10), %xmm4
-; AVX512-NEXT:    vpermd %zmm3, %zmm16, %zmm16
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vmovdqa 64(%r10), %xmm7
+; AVX512-NEXT:    vmovdqa 64(%rax), %xmm2
+; AVX512-NEXT:    vmovdqa 64(%r9), %xmm8
+; AVX512-NEXT:    vmovdqa 64(%r8), %xmm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
+; AVX512-NEXT:    vpermt2d %zmm4, %zmm16, %zmm5
+; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm4
+; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm6
+; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm14
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm9
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm15
+; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
+; AVX512-NEXT:    vmovdqa 32(%r10), %ymm8
+; AVX512-NEXT:    vmovdqa 32(%rax), %ymm4
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]
+; AVX512-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm6
+; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm7
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm1
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512-NEXT:    vpermt2d %zmm9, %zmm19, %zmm14
+; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]
+; AVX512-NEXT:    vpermt2d %zmm4, %zmm18, %zmm5
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm19, %zmm13
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512-NEXT:    vmovdqa 32(%r10), %xmm10
 ; AVX512-NEXT:    vmovdqa 32(%rax), %xmm3
-; AVX512-NEXT:    vpermd %zmm6, %zmm17, %zmm16 {%k1}
-; AVX512-NEXT:    vmovdqa 32(%r9), %xmm6
-; AVX512-NEXT:    vpermd %zmm10, %zmm30, %zmm19
-; AVX512-NEXT:    vmovdqa 32(%r8), %xmm8
-; AVX512-NEXT:    vpermd %zmm2, %zmm29, %zmm19 {%k2}
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX512-NEXT:    vpermd %zmm2, %zmm30, %zmm10
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT:    vpermd %zmm9, %zmm29, %zmm10 {%k2}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm0
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm13
-; AVX512-NEXT:    vpermd %zmm4, %zmm30, %zmm17
-; AVX512-NEXT:    vpermd %zmm3, %zmm29, %zmm17 {%k2}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm22
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm4 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm6 # 16-byte Folded Reload
-; AVX512-NEXT:    # xmm6 = xmm7[0],mem[0],xmm7[1],mem[1],xmm7[2],mem[2],xmm7[3],mem[3]
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm12
-; AVX512-NEXT:    vpermd %zmm6, %zmm30, %zmm8
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512-NEXT:    vpermd %zmm4, %zmm29, %zmm8 {%k2}
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512-NEXT:    vpermd %zmm6, %zmm29, %zmm7
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512-NEXT:    vpermd %zmm3, %zmm30, %zmm7 {%k1}
-; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm3
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT:    vpermd %zmm6, %zmm29, %zmm6
-; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm1
-; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512-NEXT:    vpermd %zmm14, %zmm30, %zmm6 {%k1}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; AVX512-NEXT:    vpermd %zmm1, %zmm29, %zmm5
-; AVX512-NEXT:    vpermd %zmm0, %zmm30, %zmm5 {%k1}
-; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm1
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpermd %zmm3, %zmm29, %zmm4
-; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm14
-; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512-NEXT:    vpermd %zmm15, %zmm30, %zmm4 {%k1}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-NEXT:    vpermd %zmm1, %zmm29, %zmm3
-; AVX512-NEXT:    vpermd %zmm0, %zmm30, %zmm3 {%k1}
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm1
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpermd %zmm14, %zmm29, %zmm14
-; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm15
-; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
-; AVX512-NEXT:    vpermd %zmm11, %zmm30, %zmm14 {%k1}
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-NEXT:    vpermd %zmm1, %zmm29, %zmm1
-; AVX512-NEXT:    vpermd %zmm0, %zmm30, %zmm1 {%k1}
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
-; AVX512-NEXT:    vpermd %zmm2, %zmm29, %zmm2
-; AVX512-NEXT:    vpermd %zmm0, %zmm30, %zmm2 {%k1}
-; AVX512-NEXT:    movb $-86, %al
-; AVX512-NEXT:    kmovw %eax, %k1
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k1}
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm31 {%k1}
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm6 {%k1}
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm5 {%k1}
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm26 {%k1}
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm24 {%k1}
-; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm19, %zmm3 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm28, %zmm23 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm25, %zmm21 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm10, %zmm14 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm17, %zmm1 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm27, %zmm18 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm20, %zmm16 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
+; AVX512-NEXT:    vmovdqa 32(%r9), %xmm4
+; AVX512-NEXT:    vmovdqa 32(%r8), %xmm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm6
+; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm7
+; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm8
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm12
+; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm3
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm4
+; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512-NEXT:    vmovdqa (%r10), %ymm10
+; AVX512-NEXT:    vmovdqa (%rax), %ymm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]
+; AVX512-NEXT:    vmovdqa (%r9), %ymm5
+; AVX512-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
+; AVX512-NEXT:    vpermt2d %zmm0, %zmm18, %zmm7
+; AVX512-NEXT:    vmovdqa (%rcx), %ymm8
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
+; AVX512-NEXT:    vmovdqa (%rsi), %ymm2
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512-NEXT:    vpermt2d %zmm11, %zmm19, %zmm0
+; AVX512-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
+; AVX512-NEXT:    vpermt2d %zmm3, %zmm18, %zmm5
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512-NEXT:    vpermt2d %zmm3, %zmm19, %zmm1
+; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm3
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm6
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX512-NEXT:    vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm16, 192(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm18, 128(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm1, 320(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm14, 256(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm21, 448(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm23, 384(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm3, 576(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm4, 512(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm24, 704(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm26, 640(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm5, 832(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm6, 768(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm31, 960(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm0, 896(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm7, 64(%rax)
-; AVX512-NEXT:    addq $504, %rsp # imm = 0x1F8
+; AVX512-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm4, 320(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm12, 256(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm13, 448(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm15, 576(%rax)
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 512(%rax)
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 704(%rax)
+; AVX512-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 640(%rax)
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 832(%rax)
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 768(%rax)
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 960(%rax)
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 896(%rax)
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-NEXT:    vmovaps %zmm0, 64(%rax)
+; AVX512-NEXT:    addq $392, %rsp # imm = 0x188
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i16_stride8_vf64:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    subq $264, %rsp # imm = 0x108
+; AVX512-FCP-NEXT:    subq $328, %rsp # imm = 0x148
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm21
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm22
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm14, %zmm1
-; AVX512-FCP-NEXT:    movw $-30584, %r11w # imm = 0x8888
-; AVX512-FCP-NEXT:    kmovw %r11d, %k2
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm1 {%k2}
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,0,1,1,1,1,0,0,2,0,3,0,3,0,0,0]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm2, %zmm17
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512-FCP-NEXT:    movw $8738, %r11w # imm = 0x2222
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm23
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm24
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm25
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm26
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm27
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm28
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
+; AVX512-FCP-NEXT:    vpermi2d %zmm0, %zmm2, %zmm20
+; AVX512-FCP-NEXT:    movb $-86, %r11b
 ; AVX512-FCP-NEXT:    kmovw %r11d, %k1
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm7, %zmm17 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa 96(%r10), %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %ymm4
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]
-; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm13
-; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm15
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm23 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512-FCP-NEXT:    vpermd %zmm11, %zmm23, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm22 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512-FCP-NEXT:    vpermd %zmm6, %zmm22, %zmm0 {%k2}
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm3
-; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm1
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpermd %zmm6, %zmm20, %zmm10
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %ymm0
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
-; AVX512-FCP-NEXT:    vpermd %zmm9, %zmm21, %zmm10 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]
-; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm23, %zmm5
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm22, %zmm5 {%k2}
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm20, %zmm13
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm13 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa 64(%r10), %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 64(%rax), %ymm1
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %ymm3
-; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm4
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm23, %zmm5
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm22, %zmm5 {%k2}
-; AVX512-FCP-NEXT:    vmovdqu64 %zmm5, (%rsp) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm9
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm9[0],ymm5[0],ymm9[1],ymm5[1],ymm9[2],ymm5[2],ymm9[3],ymm5[3],ymm9[8],ymm5[8],ymm9[9],ymm5[9],ymm9[10],ymm5[10],ymm9[11],ymm5[11]
-; AVX512-FCP-NEXT:    vpermd %zmm8, %zmm20, %zmm16
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %ymm8
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm8[0],ymm2[0],ymm8[1],ymm2[1],ymm8[2],ymm2[2],ymm8[3],ymm2[3],ymm8[8],ymm2[8],ymm8[9],ymm2[9],ymm8[10],ymm2[10],ymm8[11],ymm2[11]
-; AVX512-FCP-NEXT:    vpermd %zmm12, %zmm21, %zmm16 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm23, %zmm19
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm22, %zmm19 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm8[4],ymm2[4],ymm8[5],ymm2[5],ymm8[6],ymm2[6],ymm8[7],ymm2[7],ymm8[12],ymm2[12],ymm8[13],ymm2[13],ymm8[14],ymm2[14],ymm8[15],ymm2[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm9[4],ymm5[4],ymm9[5],ymm5[5],ymm9[6],ymm5[6],ymm9[7],ymm5[7],ymm9[12],ymm5[12],ymm9[13],ymm5[13],ymm9[14],ymm5[14],ymm9[15],ymm5[15]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm20, %zmm18
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm18 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %ymm0
-; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa 96(%r10), %ymm0
+; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %ymm1
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm4
+; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm3
+; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm4
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm23, %zmm25
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm22, %zmm25 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm2
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
-; AVX512-FCP-NEXT:    vpermd %zmm8, %zmm20, %zmm24
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm8
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm9
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
-; AVX512-FCP-NEXT:    vpermd %zmm12, %zmm21, %zmm24 {%k1}
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm2
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %ymm6
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %ymm12
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm13
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm23, %zmm27
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm22, %zmm27 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm20, %zmm26
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm26 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa (%r10), %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm2
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
-; AVX512-FCP-NEXT:    vpermd %zmm3, %zmm23, %zmm28
-; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm22, %zmm28 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm2
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm23, %zmm23
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm1
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm22, %zmm23 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512-FCP-NEXT:    vpermd %zmm3, %zmm20, %zmm22
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm21, %zmm22 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
-; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm1
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm20, %zmm20
-; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm0
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm21, %zmm20 {%k1}
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm3, %zmm21
-; AVX512-FCP-NEXT:    vmovdqa 96(%r10), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %xmm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm14, %zmm21 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm1
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm29
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm29 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm0
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm30
-; AVX512-FCP-NEXT:    vmovdqa 64(%r10), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa 64(%rax), %xmm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm14, %zmm30 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm31
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm31 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm2
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm14, %zmm6
-; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm4
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm14, %zmm6 {%k2}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm1
-; AVX512-FCP-NEXT:    vpermd %zmm3, %zmm14, %zmm1 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm2 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm3 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512-FCP-NEXT:    vpermd %zmm3, %zmm14, %zmm9
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm9 {%k2}
-; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm3
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm5, %zmm8
-; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm5
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-FCP-NEXT:    vpermd %zmm12, %zmm7, %zmm8 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-FCP-NEXT:    vpermd %zmm2, %zmm7, %zmm2
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm7, %zmm2 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm1
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm19, %zmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 96(%r10), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %xmm1
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm4
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm7, %zmm5
-; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm12
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm14
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]
-; AVX512-FCP-NEXT:    vpermd %zmm11, %zmm7, %zmm5 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-FCP-NEXT:    vpermd %zmm3, %zmm7, %zmm3
-; AVX512-FCP-NEXT:    vpermd %zmm11, %zmm7, %zmm3 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm11
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; AVX512-FCP-NEXT:    vpermd %zmm12, %zmm7, %zmm12
-; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm14
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512-FCP-NEXT:    vpermd %zmm15, %zmm7, %zmm12 {%k1}
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
-; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm7, %zmm4
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm7, %zmm4 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    # xmm11 = xmm11[0],mem[0],xmm11[1],mem[1],xmm11[2],mem[2],xmm11[3],mem[3]
-; AVX512-FCP-NEXT:    vpermd %zmm11, %zmm7, %zmm11
-; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm7, %zmm11 {%k1}
-; AVX512-FCP-NEXT:    movb $-86, %al
-; AVX512-FCP-NEXT:    kmovw %eax, %k1
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k1}
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm21, %zmm8 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm29, %zmm2 {%k1}
-; AVX512-FCP-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm30, %zmm5 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm31, %zmm3 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm25, %zmm24 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm27, %zmm26 {%k1}
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm2
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm6
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm13
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm15
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm12, %zmm18, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 64(%r10), %ymm6
+; AVX512-FCP-NEXT:    vmovdqa 64(%rax), %ymm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
+; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %ymm3
+; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm12
+; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %ymm2
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %ymm13
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
+; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm0
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm19, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm5, %zmm17, %zmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa 64(%r10), %xmm8
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vmovdqa 64(%rax), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm7
+; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm3
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm16, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm4
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm13
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm9
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm15
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %ymm8
+; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm7
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm1
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm0
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm9, %zmm19, %zmm14
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm17, %zmm5
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm13
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %xmm10
+; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm4
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm7
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm8
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm12
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm28, %zmm22 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm23, %zmm20 {%k1}
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm9, %zmm11 {%k1}
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm3
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa (%r10), %ymm10
+; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm3
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm5
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm8
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm9
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm2
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm17, %zmm5
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm19, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm6
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, 192(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm22, 128(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 320(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm26, 448(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm24, 384(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 576(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 512(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, 704(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, 640(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 832(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 768(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 960(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 896(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, 64(%rax)
-; AVX512-FCP-NEXT:    addq $264, %rsp # imm = 0x108
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 448(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, 576(%rax)
+; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm0, 512(%rax)
+; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm0, 704(%rax)
+; AVX512-FCP-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm0, 640(%rax)
+; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm0, 832(%rax)
+; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm0, 768(%rax)
+; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm0, 960(%rax)
+; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT:    vmovaps %zmm0, 896(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, 64(%rax)
+; AVX512-FCP-NEXT:    addq $328, %rsp # imm = 0x148
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i16_stride8_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $504, %rsp # imm = 0x1F8
+; AVX512DQ-NEXT:    subq $392, %rsp # imm = 0x188
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%r10), %xmm4
-; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm1
-; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%rax), %xmm3
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm7
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm30, %zmm0
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512DQ-NEXT:    movw $-30584, %r11w # imm = 0x8888
-; AVX512DQ-NEXT:    kmovw %r11d, %k2
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm29, %zmm0 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    movw $8738, %r11w # imm = 0x2222
+; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm1
+; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm20
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm21
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm2
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm22
+; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm23
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm2
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm24
+; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm25
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm26
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm17, %zmm2
+; AVX512DQ-NEXT:    movb $-86, %r11b
 ; AVX512DQ-NEXT:    kmovw %r11d, %k1
-; AVX512DQ-NEXT:    vmovdqa 96(%r10), %ymm2
-; AVX512DQ-NEXT:    vmovdqa 96(%rax), %ymm5
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
-; AVX512DQ-NEXT:    vmovdqa 96(%r9), %ymm8
-; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512DQ-NEXT:    vpermd %zmm10, %zmm19, %zmm0
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm18, %zmm0 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %ymm10
-; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %ymm11
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 96(%r10), %ymm0
+; AVX512DQ-NEXT:    vmovdqa 96(%rax), %ymm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512DQ-NEXT:    vmovdqa 96(%r9), %ymm3
+; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm4
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm5
+; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %ymm2
+; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %ymm6
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
 ; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %ymm12
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %ymm13
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm16, %zmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpermd %zmm14, %zmm17, %zmm0 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
-; AVX512DQ-NEXT:    vpermd %zmm5, %zmm19, %zmm0
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm18, %zmm0 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[12],ymm10[12],ymm11[13],ymm10[13],ymm11[14],ymm10[14],ymm11[15],ymm10[15]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
-; AVX512DQ-NEXT:    vpermd %zmm5, %zmm16, %zmm31
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm17, %zmm31 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 96(%r10), %xmm2
-; AVX512DQ-NEXT:    vmovdqa 96(%rax), %xmm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3]
-; AVX512DQ-NEXT:    vmovdqa 96(%r9), %xmm10
-; AVX512DQ-NEXT:    vmovdqa 96(%r8), %xmm11
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
-; AVX512DQ-NEXT:    vpermd %zmm12, %zmm30, %zmm0
-; AVX512DQ-NEXT:    vpermd %zmm9, %zmm29, %zmm0 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
-; AVX512DQ-NEXT:    vpermd %zmm8, %zmm30, %zmm0
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm29, %zmm0 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%r10), %ymm2
-; AVX512DQ-NEXT:    vmovdqa 64(%rax), %ymm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[2],ymm2[2],ymm9[3],ymm2[3],ymm9[8],ymm2[8],ymm9[9],ymm2[9],ymm9[10],ymm2[10],ymm9[11],ymm2[11]
-; AVX512DQ-NEXT:    vmovdqa 64(%r9), %ymm10
-; AVX512DQ-NEXT:    vmovdqa 64(%r8), %ymm11
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11]
-; AVX512DQ-NEXT:    vpermd %zmm12, %zmm19, %zmm0
-; AVX512DQ-NEXT:    vpermd %zmm8, %zmm18, %zmm0 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %ymm12
-; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %ymm15
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm10, %zmm19, %zmm7
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm18, %zmm1
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm19, %zmm2
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 96(%r10), %xmm0
+; AVX512DQ-NEXT:    vmovdqa 96(%rax), %xmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512DQ-NEXT:    vmovdqa 96(%r9), %xmm3
+; AVX512DQ-NEXT:    vmovdqa 96(%r8), %xmm4
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %xmm2
+; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %xmm6
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm13
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm10, %zmm17, %zmm7
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm17, %zmm2
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%r10), %ymm6
+; AVX512DQ-NEXT:    vmovdqa 64(%rax), %ymm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
+; AVX512DQ-NEXT:    vmovdqa 64(%r9), %ymm3
+; AVX512DQ-NEXT:    vmovdqa 64(%r8), %ymm4
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm10
+; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %ymm2
+; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %ymm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
 ; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %ymm1
 ; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512DQ-NEXT:    vpermd %zmm8, %zmm16, %zmm26
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm15[0],ymm12[0],ymm15[1],ymm12[1],ymm15[2],ymm12[2],ymm15[3],ymm12[3],ymm15[8],ymm12[8],ymm15[9],ymm12[9],ymm15[10],ymm12[10],ymm15[11],ymm12[11]
-; AVX512DQ-NEXT:    vpermd %zmm8, %zmm17, %zmm26 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 64(%r9), %xmm8
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm9[4],ymm2[4],ymm9[5],ymm2[5],ymm9[6],ymm2[6],ymm9[7],ymm2[7],ymm9[12],ymm2[12],ymm9[13],ymm2[13],ymm9[14],ymm2[14],ymm9[15],ymm2[15]
-; AVX512DQ-NEXT:    vmovdqa 64(%r8), %xmm9
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm10 = ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[12],ymm10[12],ymm11[13],ymm10[13],ymm11[14],ymm10[14],ymm11[15],ymm10[15]
-; AVX512DQ-NEXT:    vpermd %zmm10, %zmm19, %zmm5
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm18, %zmm5 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm12[4],ymm15[5],ymm12[5],ymm15[6],ymm12[6],ymm15[7],ymm12[7],ymm15[12],ymm12[12],ymm15[13],ymm12[13],ymm15[14],ymm12[14],ymm15[15],ymm12[15]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm14, %zmm19, %zmm7
+; AVX512DQ-NEXT:    vmovdqa64 %zmm10, %zmm7 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm5, %zmm18, %zmm3
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm16, %zmm24
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm17, %zmm24 {%k1}
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm30, %zmm1
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm29, %zmm1 {%k2}
-; AVX512DQ-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
-; AVX512DQ-NEXT:    vmovdqa 32(%r10), %ymm0
-; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm1
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm4
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm8[0],ymm4[0],ymm8[1],ymm4[1],ymm8[2],ymm4[2],ymm8[3],ymm4[3],ymm8[8],ymm4[8],ymm8[9],ymm4[9],ymm8[10],ymm4[10],ymm8[11],ymm4[11]
-; AVX512DQ-NEXT:    vpermd %zmm9, %zmm19, %zmm28
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm18, %zmm28 {%k2}
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm9
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm11
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[2],ymm9[2],ymm11[3],ymm9[3],ymm11[8],ymm9[8],ymm11[9],ymm9[9],ymm11[10],ymm9[10],ymm11[11],ymm9[11]
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm16, %zmm23
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm12
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm3[0],ymm12[0],ymm3[1],ymm12[1],ymm3[2],ymm12[2],ymm3[3],ymm12[3],ymm3[8],ymm12[8],ymm3[9],ymm12[9],ymm3[10],ymm12[10],ymm3[11],ymm12[11]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm17, %zmm23 {%k1}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm8[4],ymm4[4],ymm8[5],ymm4[5],ymm8[6],ymm4[6],ymm8[7],ymm4[7],ymm8[12],ymm4[12],ymm8[13],ymm4[13],ymm8[14],ymm4[14],ymm8[15],ymm4[15]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm19, %zmm25
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm18, %zmm25 {%k2}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm12[4],ymm3[5],ymm12[5],ymm3[6],ymm12[6],ymm3[7],ymm12[7],ymm3[12],ymm12[12],ymm3[13],ymm12[13],ymm3[14],ymm12[14],ymm3[15],ymm12[15]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm11[4],ymm9[4],ymm11[5],ymm9[5],ymm11[6],ymm9[6],ymm11[7],ymm9[7],ymm11[12],ymm9[12],ymm11[13],ymm9[13],ymm11[14],ymm9[14],ymm11[15],ymm9[15]
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm16, %zmm21
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm17, %zmm21 {%k1}
-; AVX512DQ-NEXT:    vmovdqa (%r10), %ymm3
-; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm4
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm6
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm6[0],ymm4[0],ymm6[1],ymm4[1],ymm6[2],ymm4[2],ymm6[3],ymm4[3],ymm6[8],ymm4[8],ymm6[9],ymm4[9],ymm6[10],ymm4[10],ymm6[11],ymm4[11]
-; AVX512DQ-NEXT:    vpermd %zmm8, %zmm19, %zmm27
-; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm8[0],ymm3[0],ymm8[1],ymm3[1],ymm8[2],ymm3[2],ymm8[3],ymm3[3],ymm8[8],ymm3[8],ymm8[9],ymm3[9],ymm8[10],ymm3[10],ymm8[11],ymm3[11]
-; AVX512DQ-NEXT:    vpermd %zmm9, %zmm18, %zmm27 {%k2}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm8[4],ymm3[4],ymm8[5],ymm3[5],ymm8[6],ymm3[6],ymm8[7],ymm3[7],ymm8[12],ymm3[12],ymm8[13],ymm3[13],ymm8[14],ymm3[14],ymm8[15],ymm3[15]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm6[4],ymm4[4],ymm6[5],ymm4[5],ymm6[6],ymm4[6],ymm6[7],ymm4[7],ymm6[12],ymm4[12],ymm6[13],ymm4[13],ymm6[14],ymm4[14],ymm6[15],ymm4[15]
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm6
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm19, %zmm20
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm18, %zmm20 {%k2}
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]
-; AVX512DQ-NEXT:    vpermd %zmm8, %zmm16, %zmm18
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[8],ymm6[8],ymm8[9],ymm6[9],ymm8[10],ymm6[10],ymm8[11],ymm6[11]
-; AVX512DQ-NEXT:    vpermd %zmm9, %zmm17, %zmm18 {%k1}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm8[4],ymm6[4],ymm8[5],ymm6[5],ymm8[6],ymm6[6],ymm8[7],ymm6[7],ymm8[12],ymm6[12],ymm8[13],ymm6[13],ymm8[14],ymm6[14],ymm8[15],ymm6[15]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]
-; AVX512DQ-NEXT:    vmovdqa 32(%r10), %xmm4
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm16, %zmm16
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vmovdqa 64(%r10), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 64(%rax), %xmm2
+; AVX512DQ-NEXT:    vmovdqa 64(%r9), %xmm8
+; AVX512DQ-NEXT:    vmovdqa 64(%r8), %xmm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm4, %zmm16, %zmm5
+; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %xmm4
+; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %xmm6
+; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm14
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm9
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm15
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
+; AVX512DQ-NEXT:    vmovdqa 32(%r10), %ymm8
+; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm4
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm6
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm7
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm1
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm9, %zmm19, %zmm14
+; AVX512DQ-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm4, %zmm18, %zmm5
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm19, %zmm13
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512DQ-NEXT:    vmovdqa 32(%r10), %xmm10
 ; AVX512DQ-NEXT:    vmovdqa 32(%rax), %xmm3
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm17, %zmm16 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm6
-; AVX512DQ-NEXT:    vpermd %zmm10, %zmm30, %zmm19
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm8
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm29, %zmm19 {%k2}
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm30, %zmm10
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT:    vpermd %zmm9, %zmm29, %zmm10 {%k2}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm0
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm13
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm30, %zmm17
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm29, %zmm17 {%k2}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm22
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm4 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm6 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    # xmm6 = xmm7[0],mem[0],xmm7[1],mem[1],xmm7[2],mem[2],xmm7[3],mem[3]
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm12
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm30, %zmm8
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512DQ-NEXT:    vpermd %zmm4, %zmm29, %zmm8 {%k2}
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm29, %zmm7
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm30, %zmm7 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm3
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT:    vpermd %zmm6, %zmm29, %zmm6
-; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %xmm1
-; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512DQ-NEXT:    vpermd %zmm14, %zmm30, %zmm6 {%k1}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm29, %zmm5
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm30, %zmm5 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm1
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpermd %zmm3, %zmm29, %zmm4
-; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %xmm14
-; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-NEXT:    vpermd %zmm15, %zmm30, %zmm4 {%k1}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm29, %zmm3
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm30, %zmm3 {%k1}
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm1
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpermd %zmm14, %zmm29, %zmm14
-; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm15
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
-; AVX512DQ-NEXT:    vpermd %zmm11, %zmm30, %zmm14 {%k1}
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm29, %zmm1
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm30, %zmm1 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
-; AVX512DQ-NEXT:    vpermd %zmm2, %zmm29, %zmm2
-; AVX512DQ-NEXT:    vpermd %zmm0, %zmm30, %zmm2 {%k1}
-; AVX512DQ-NEXT:    movb $-86, %al
-; AVX512DQ-NEXT:    kmovw %eax, %k1
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm31 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm6 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm5 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm26 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm24 {%k1}
-; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm19, %zmm3 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm28, %zmm23 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm25, %zmm21 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm10, %zmm14 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm17, %zmm1 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm27, %zmm18 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm20, %zmm16 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
+; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm4
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm6
+; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm8
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm12
+; AVX512DQ-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm3
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm4
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512DQ-NEXT:    vmovdqa (%r10), %ymm10
+; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]
+; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm5
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm18, %zmm7
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm8
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm2
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512DQ-NEXT:    vpermt2d %zmm11, %zmm19, %zmm0
+; AVX512DQ-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm18, %zmm5
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm19, %zmm1
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm6
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 192(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm18, 128(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 320(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 256(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm21, 448(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm23, 384(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 576(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 512(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm24, 704(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm26, 640(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 832(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 768(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm31, 960(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 896(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm7, 64(%rax)
-; AVX512DQ-NEXT:    addq $504, %rsp # imm = 0x1F8
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 320(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm12, 256(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm13, 448(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm15, 576(%rax)
+; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 512(%rax)
+; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 704(%rax)
+; AVX512DQ-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 640(%rax)
+; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 832(%rax)
+; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 768(%rax)
+; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 960(%rax)
+; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 896(%rax)
+; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-NEXT:    vmovaps %zmm0, 64(%rax)
+; AVX512DQ-NEXT:    addq $392, %rsp # imm = 0x188
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride8_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    subq $264, %rsp # imm = 0x108
+; AVX512DQ-FCP-NEXT:    subq $328, %rsp # imm = 0x148
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm22
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,0,0,0,0,1,1,2,2,2,2,0,0,3,3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm14, %zmm1
-; AVX512DQ-FCP-NEXT:    movw $-30584, %r11w # imm = 0x8888
-; AVX512DQ-FCP-NEXT:    kmovw %r11d, %k2
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm1 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,0,1,1,1,1,0,0,2,0,3,0,3,0,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm2, %zmm17
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [0,0,1,1,1,1,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-FCP-NEXT:    movw $8738, %r11w # imm = 0x2222
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm23
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm24
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm26
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm27
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm28
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
+; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm2, %zmm20
+; AVX512DQ-FCP-NEXT:    movb $-86, %r11b
 ; AVX512DQ-FCP-NEXT:    kmovw %r11d, %k1
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm7, %zmm17 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r10), %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %ymm4
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm15
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm23 = [0,1,4,5,4,5,5,7,2,1,6,5,6,5,7,7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm11, %zmm23, %zmm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm22 = [0,0,4,4,4,4,6,5,0,2,4,6,4,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm22, %zmm0 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [4,5,1,3,5,7,5,7,6,5,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm20, %zmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %ymm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm21 = [4,4,2,1,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm9, %zmm21, %zmm10 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm23, %zmm5
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm22, %zmm5 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm20, %zmm13
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm13 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r10), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rax), %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r10), %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %ymm1
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm23, %zmm5
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm22, %zmm5 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm5, (%rsp) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm9
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm9[0],ymm5[0],ymm9[1],ymm5[1],ymm9[2],ymm5[2],ymm9[3],ymm5[3],ymm9[8],ymm5[8],ymm9[9],ymm5[9],ymm9[10],ymm5[10],ymm9[11],ymm5[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm8, %zmm20, %zmm16
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %ymm8
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm8[0],ymm2[0],ymm8[1],ymm2[1],ymm8[2],ymm2[2],ymm8[3],ymm2[3],ymm8[8],ymm2[8],ymm8[9],ymm2[9],ymm8[10],ymm2[10],ymm8[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm12, %zmm21, %zmm16 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm23, %zmm19
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm22, %zmm19 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm8[4],ymm2[4],ymm8[5],ymm2[5],ymm8[6],ymm2[6],ymm8[7],ymm2[7],ymm8[12],ymm2[12],ymm8[13],ymm2[13],ymm8[14],ymm2[14],ymm8[15],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm9[4],ymm5[4],ymm9[5],ymm5[5],ymm9[6],ymm5[6],ymm9[7],ymm5[7],ymm9[12],ymm5[12],ymm9[13],ymm5[13],ymm9[14],ymm5[14],ymm9[15],ymm5[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm20, %zmm18
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm18 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm1
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm4
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm23, %zmm25
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm22, %zmm25 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm8, %zmm20, %zmm24
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm8
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm9
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm12, %zmm21, %zmm24 {%k1}
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %ymm6
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm23, %zmm27
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm22, %zmm27 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm20, %zmm26
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm21, %zmm26 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm2
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm23, %zmm28
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm22, %zmm28 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm2
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm23, %zmm23
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm1
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm22, %zmm23 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm20, %zmm22
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm21, %zmm22 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm1
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm20, %zmm20
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm0
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm21, %zmm20 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,0,0,0,0,0,1,0,2,2,2,2,0,0,3,3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm3, %zmm21
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r10), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm14, %zmm21 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm1
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm29
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm29 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm0
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm30
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r10), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rax), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm14, %zmm30 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm31
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm31 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm14, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm4
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm14, %zmm6 {%k2}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm14, %zmm1
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm14, %zmm1 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm2 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm3 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm14, %zmm9
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm14, %zmm9 {%k2}
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,0,1,0,0,0,2,2,3,3,3,3,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm5, %zmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm5
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm12, %zmm7, %zmm8 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm2, %zmm7, %zmm2
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm7, %zmm2 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm1
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm19, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r10), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %xmm1
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm4
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm7, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %xmm14
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm11, %zmm7, %zmm5 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm3, %zmm7, %zmm3
-; AVX512DQ-FCP-NEXT:    vpermd %zmm11, %zmm7, %zmm3 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm11
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm12, %zmm7, %zmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm14
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm15, %zmm7, %zmm12 {%k1}
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm7, %zmm4
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm7, %zmm4 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # xmm11 = xmm11[0],mem[0],xmm11[1],mem[1],xmm11[2],mem[2],xmm11[3],mem[3]
-; AVX512DQ-FCP-NEXT:    vpermd %zmm11, %zmm7, %zmm11
-; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm7, %zmm11 {%k1}
-; AVX512DQ-FCP-NEXT:    movb $-86, %al
-; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm21, %zmm8 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm29, %zmm2 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm30, %zmm5 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm31, %zmm3 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm25, %zmm24 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm27, %zmm26 {%k1}
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm6
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm12, %zmm18, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm1
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r10), %ymm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rax), %ymm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm12
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %ymm13
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm19, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm5, %zmm17, %zmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r10), %xmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rax), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm3
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm16, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm9
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm7
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm9, %zmm19, %zmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %xmm10
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm28, %zmm22 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm23, %zmm20 {%k1}
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, %zmm11 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm3
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm8
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm9
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm19, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm6
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, 192(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm22, 128(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 320(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm26, 448(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm24, 384(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 576(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 512(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, 704(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, 640(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 832(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 768(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 960(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 896(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, 64(%rax)
-; AVX512DQ-FCP-NEXT:    addq $264, %rsp # imm = 0x108
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 448(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 384(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, 576(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 512(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 704(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 640(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 832(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 768(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 960(%rax)
+; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 896(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, 64(%rax)
+; AVX512DQ-FCP-NEXT:    addq $328, %rsp # imm = 0x148
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-3.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-3.ll
index ac0522a9e68f..acf9bad81736 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-3.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-3.ll
@@ -243,19 +243,19 @@ define void @store_i32_stride3_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX-NEXT:    vmovaps (%rsi), %xmm1
 ; AVX-NEXT:    vmovaps (%rdx), %xmm2
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm3 = xmm0[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[1,1],xmm3[0,2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0],xmm2[1],xmm3[2,3]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,2],xmm0[2,1]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm2[0],xmm4[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,1,2,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm2[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm0[2],xmm4[3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm2[1],xmm4[2,3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm1[3,3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm2[2,3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0,1,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
 ; AVX-NEXT:    vmovaps %xmm0, 32(%rcx)
-; AVX-NEXT:    vmovaps %ymm3, (%rcx)
-; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    vmovaps %xmm4, 16(%rcx)
+; AVX-NEXT:    vmovaps %xmm3, (%rcx)
 ; AVX-NEXT:    retq
 ;
 ; AVX2-LABEL: store_i32_stride3_vf4:
@@ -458,33 +458,32 @@ define void @store_i32_stride3_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ;
 ; AVX-LABEL: store_i32_stride3_vf8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovapd (%rdx), %ymm0
-; AVX-NEXT:    vmovaps (%rsi), %xmm1
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm2
-; AVX-NEXT:    vmovaps (%rdi), %xmm3
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm4
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm5 = xmm3[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm1[1,1],xmm5[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm3[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
-; AVX-NEXT:    vbroadcastsd (%rdx), %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[3,3],xmm2[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,1],xmm4[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm3[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm3 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm4 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1,2],ymm0[3],ymm3[4,5],ymm0[6],ymm3[7]
-; AVX-NEXT:    vmovaps %ymm0, 32(%rcx)
-; AVX-NEXT:    vmovaps %ymm2, 64(%rcx)
-; AVX-NEXT:    vmovaps %ymm1, (%rcx)
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm1
+; AVX-NEXT:    vmovaps (%rsi), %xmm2
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm3
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm2[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm0[2],xmm4[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastsd (%rdx), %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[3,3],xmm3[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm2[0,0,3,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm2 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]
+; AVX-NEXT:    vmovaps %ymm2, 32(%rcx)
+; AVX-NEXT:    vmovaps %ymm1, 64(%rcx)
+; AVX-NEXT:    vmovaps %ymm0, (%rcx)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -780,60 +779,58 @@ define void @store_i32_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride3_vf16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovapd (%rdx), %ymm1
-; AVX-NEXT:    vmovapd 32(%rdx), %ymm0
-; AVX-NEXT:    vmovaps (%rsi), %xmm2
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm3
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm4
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm5
-; AVX-NEXT:    vmovaps (%rdi), %xmm6
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm7
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm8
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm9 = xmm6[1],xmm2[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm2[1,1],xmm9[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm6[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,0],xmm6[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm2, %ymm2
-; AVX-NEXT:    vbroadcastsd (%rdx), %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7]
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm6
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm6[3,3],xmm5[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm6[1],xmm5[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[1,1],xmm6[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm5, %ymm5
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm6[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3],ymm6[4],ymm5[5,6],ymm6[7]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm8[1],xmm4[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm4[1,1],xmm6[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm8[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[2,0],xmm8[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
-; AVX-NEXT:    vbroadcastsd 32(%rdx), %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm7[3,3],xmm3[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm7 = xmm7[1],xmm3[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1],xmm7[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm1[2,3,2,3]
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm1
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm2
+; AVX-NEXT:    vmovaps (%rsi), %xmm3
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm4
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm5
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm6
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm3[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm7[0,1],xmm0[2],xmm7[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastsd (%rdx), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm3
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm3[3,3],xmm6[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm6[0,1],xmm3[2],xmm6[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm3, %ymm3
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm6[0,0,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm6[1],ymm3[2,3],ymm6[4],ymm3[5,6],ymm6[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm6 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm7 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1,2],ymm1[3],ymm6[4,5],ymm1[6],ymm6[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm6 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm7 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm6[1,2],ymm0[3],ymm6[4,5],ymm0[6],ymm6[7]
-; AVX-NEXT:    vmovaps %ymm0, 128(%rcx)
-; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)
-; AVX-NEXT:    vmovaps %ymm3, 64(%rcx)
-; AVX-NEXT:    vmovaps %ymm4, 96(%rcx)
-; AVX-NEXT:    vmovaps %ymm5, 160(%rcx)
-; AVX-NEXT:    vmovaps %ymm2, (%rcx)
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm2[2],xmm6[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
+; AVX-NEXT:    vbroadcastsd 32(%rdx), %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2],ymm2[3,4],ymm5[5],ymm2[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm1[3,3],xmm4[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm4[0,1],xmm1[2],xmm4[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm4[0,0,3,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2,3],ymm4[4],ymm1[5,6],ymm4[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm4 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm5 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm5 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm5 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm6 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm6 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0],ymm5[1,2],ymm6[3],ymm5[4,5],ymm6[6],ymm5[7]
+; AVX-NEXT:    vmovaps %ymm5, 128(%rcx)
+; AVX-NEXT:    vmovaps %ymm4, 32(%rcx)
+; AVX-NEXT:    vmovaps %ymm1, 64(%rcx)
+; AVX-NEXT:    vmovaps %ymm2, 96(%rcx)
+; AVX-NEXT:    vmovaps %ymm3, 160(%rcx)
+; AVX-NEXT:    vmovaps %ymm0, (%rcx)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -1378,114 +1375,110 @@ define void @store_i32_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride3_vf32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovapd (%rdx), %ymm4
-; AVX-NEXT:    vmovapd 32(%rdx), %ymm2
-; AVX-NEXT:    vmovapd 64(%rdx), %ymm3
-; AVX-NEXT:    vmovapd 96(%rdx), %ymm0
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm3
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm6
 ; AVX-NEXT:    vmovaps (%rsi), %xmm1
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm7
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm10
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm9
-; AVX-NEXT:    vmovaps (%rdi), %xmm5
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm8
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm11
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm5[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[1,1],xmm6[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm5[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX-NEXT:    vbroadcastsd (%rdx), %ymm5
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm5[2],ymm1[3,4],ymm5[5],ymm1[6,7]
-; AVX-NEXT:    vmovaps 80(%rsi), %xmm5
-; AVX-NEXT:    vmovaps 80(%rdi), %xmm6
-; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm6[3,3],xmm5[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm6[1],xmm5[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[1,1],xmm6[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm5, %ymm5
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm3[2,3,2,3]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm6[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3],ymm6[4],ymm5[5,6],ymm6[7]
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm6
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm12
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm13 = xmm12[1],xmm6[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm6[1,1],xmm13[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm12[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[2,0],xmm12[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm6, %ymm6
-; AVX-NEXT:    vbroadcastsd 64(%rdx), %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm12[2],ymm6[3,4],ymm12[5],ymm6[6,7]
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm12
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm12[3,3],xmm9[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm12 = xmm12[1],xmm9[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[1,1],xmm12[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm9, %ymm9
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm2[2,3,2,3]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm12 = ymm12[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm12[1],ymm9[2,3],ymm12[4],ymm9[5,6],ymm12[7]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm12 = xmm11[1],xmm10[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm10[1,1],xmm12[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm10 = xmm10[0],xmm11[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[2,0],xmm11[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm10, %ymm10
-; AVX-NEXT:    vbroadcastsd 32(%rdx), %ymm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7]
-; AVX-NEXT:    vmovaps 112(%rsi), %xmm11
-; AVX-NEXT:    vmovaps 112(%rdi), %xmm12
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm12[3,3],xmm11[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm12 = xmm12[1],xmm11[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[1,1],xmm12[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm11, %ymm11
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm12 = ymm12[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm12[1],ymm11[2,3],ymm12[4],ymm11[5,6],ymm12[7]
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm12
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm13
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm14 = xmm13[1],xmm12[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm12[1,1],xmm14[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm12 = xmm12[0],xmm13[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm12[2,0],xmm13[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm12, %ymm12
-; AVX-NEXT:    vbroadcastsd 96(%rdx), %ymm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm8[3,3],xmm7[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm8 = xmm8[1],xmm7[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[1,1],xmm8[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm7, %ymm7
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm4[2,3,2,3]
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm4
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm7
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm5
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2],xmm2[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastsd (%rdx), %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vmovaps 80(%rsi), %xmm1
+; AVX-NEXT:    vmovaps 80(%rdi), %xmm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm2[3,3],xmm1[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm1, %ymm1
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm2[0,0,3,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm2
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm8
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm8[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm2[2],xmm9[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm2, %ymm2
+; AVX-NEXT:    vbroadcastsd 64(%rdx), %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm8[2],ymm2[3,4],ymm8[5],ymm2[6,7]
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm8
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm8[3,3],xmm5[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm8[2],xmm5[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm5, %ymm5
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm8 = ymm8[0,0,3,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm8[1],ymm5[2,3],ymm8[4],ymm5[5,6],ymm8[7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm7[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm6[2],xmm8[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm6, %ymm6
+; AVX-NEXT:    vbroadcastsd 32(%rdx), %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7]
+; AVX-NEXT:    vmovaps 112(%rsi), %xmm7
+; AVX-NEXT:    vmovaps 112(%rdi), %xmm8
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm8[3,3],xmm7[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm7[0,1],xmm8[2],xmm7[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm7, %ymm7
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm8 = ymm8[0,0,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0],ymm8[1],ymm7[2,3],ymm8[4],ymm7[5,6],ymm8[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm8 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm13[0,1],ymm8[2],ymm13[3,4],ymm8[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm4[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm8[1,2],ymm4[3],ymm8[4,5],ymm4[6],ymm8[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm8 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm13[0,1],ymm8[2],ymm13[3,4],ymm8[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm3[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm8[1,2],ymm3[3],ymm8[4,5],ymm3[6],ymm8[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm8 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm13[0,1],ymm8[2],ymm13[3,4],ymm8[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm8[1,2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm8 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm13[0,1],ymm8[2],ymm13[3,4],ymm8[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm8[1,2],ymm0[3],ymm8[4,5],ymm0[6],ymm8[7]
-; AVX-NEXT:    vmovaps %ymm0, 320(%rcx)
-; AVX-NEXT:    vmovaps %ymm2, 128(%rcx)
-; AVX-NEXT:    vmovaps %ymm3, 224(%rcx)
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm8
+; AVX-NEXT:    vmovaps 96(%rsi), %xmm9
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm8[2],xmm10[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm8[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm8, %ymm8
+; AVX-NEXT:    vbroadcastsd 96(%rdx), %ymm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm3[3,3],xmm4[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm3, %ymm3
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm4[0,0,3,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm4 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm9 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm9[0,1],ymm4[2],ymm9[3,4],ymm4[5],ymm9[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm9 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm9[0],ymm4[1,2],ymm9[3],ymm4[4,5],ymm9[6],ymm4[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm9 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm10 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1],ymm9[2],ymm10[3,4],ymm9[5],ymm10[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm10 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1,2],ymm10[3],ymm9[4,5],ymm10[6],ymm9[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm10 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm11 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1],ymm10[2],ymm11[3,4],ymm10[5],ymm11[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm11 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0],ymm10[1,2],ymm11[3],ymm10[4,5],ymm11[6],ymm10[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm11 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm12 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1],ymm11[2],ymm12[3,4],ymm11[5],ymm12[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm12 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0],ymm11[1,2],ymm12[3],ymm11[4,5],ymm12[6],ymm11[7]
+; AVX-NEXT:    vmovaps %ymm11, 320(%rcx)
+; AVX-NEXT:    vmovaps %ymm10, 128(%rcx)
+; AVX-NEXT:    vmovaps %ymm9, 224(%rcx)
 ; AVX-NEXT:    vmovaps %ymm4, 32(%rcx)
-; AVX-NEXT:    vmovaps %ymm7, 64(%rcx)
-; AVX-NEXT:    vmovaps %ymm12, 288(%rcx)
-; AVX-NEXT:    vmovaps %ymm11, 352(%rcx)
-; AVX-NEXT:    vmovaps %ymm10, 96(%rcx)
-; AVX-NEXT:    vmovaps %ymm9, 160(%rcx)
-; AVX-NEXT:    vmovaps %ymm6, 192(%rcx)
-; AVX-NEXT:    vmovaps %ymm5, 256(%rcx)
-; AVX-NEXT:    vmovaps %ymm1, (%rcx)
+; AVX-NEXT:    vmovaps %ymm3, 64(%rcx)
+; AVX-NEXT:    vmovaps %ymm8, 288(%rcx)
+; AVX-NEXT:    vmovaps %ymm7, 352(%rcx)
+; AVX-NEXT:    vmovaps %ymm6, 96(%rcx)
+; AVX-NEXT:    vmovaps %ymm5, 160(%rcx)
+; AVX-NEXT:    vmovaps %ymm2, 192(%rcx)
+; AVX-NEXT:    vmovaps %ymm1, 256(%rcx)
+; AVX-NEXT:    vmovaps %ymm0, (%rcx)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -2532,234 +2525,224 @@ define void @store_i32_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride3_vf64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $200, %rsp
-; AVX-NEXT:    vmovapd (%rdx), %ymm8
-; AVX-NEXT:    vmovapd 32(%rdx), %ymm9
-; AVX-NEXT:    vmovapd 64(%rdx), %ymm11
-; AVX-NEXT:    vmovapd 96(%rdx), %ymm12
-; AVX-NEXT:    vmovaps (%rsi), %xmm0
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm1
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm2
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm3
-; AVX-NEXT:    vmovaps (%rdi), %xmm4
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm5
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm6
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm7 = xmm4[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm0[1,1],xmm7[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[2,1]
+; AVX-NEXT:    subq $168, %rsp
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm1
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm2
+; AVX-NEXT:    vmovaps (%rsi), %xmm3
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm4
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm5
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm6
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm3[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm7[0,1],xmm0[2],xmm7[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
 ; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
-; AVX-NEXT:    vbroadcastsd (%rdx), %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm4[2],ymm0[3,4],ymm4[5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastsd (%rdx), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[3,3],xmm1[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm4 = xmm5[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[0,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[3,3],xmm4[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm8[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm6[1],xmm2[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[1,1],xmm0[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm6[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm6[2,1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,2]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX-NEXT:    vbroadcastsd 32(%rdx), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 48(%rdi), %xmm0
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[3,3],xmm3[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[1,1],xmm0[0,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[3,3],xmm6[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm6[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm9[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[1,1],xmm2[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm0[2],xmm4[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX-NEXT:    vbroadcastsd 64(%rdx), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 80(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps 80(%rdi), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[3,3],xmm0[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm11[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm1[3,3],xmm0[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
-; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[1,1],xmm2[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 96(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm0[2],xmm6[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
 ; AVX-NEXT:    vbroadcastsd 96(%rdx), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 112(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps 112(%rdi), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[3,3],xmm0[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm12[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm1[3,3],xmm0[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 128(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[1,1],xmm2[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vmovaps 128(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 128(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm0[2],xmm8[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
 ; AVX-NEXT:    vbroadcastsd 128(%rdx), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 144(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps 144(%rdi), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[3,3],xmm0[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,2]
-; AVX-NEXT:    vmovapd 128(%rdx), %ymm6
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm6[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm1[3,3],xmm0[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 160(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[1,1],xmm2[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
+; AVX-NEXT:    vmovaps 160(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 160(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm0[2],xmm10[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm0, %ymm0
 ; AVX-NEXT:    vbroadcastsd 160(%rdx), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovaps 176(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps 176(%rdi), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[3,3],xmm0[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovapd 160(%rdx), %ymm4
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm4[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm1[3,3],xmm0[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
-; AVX-NEXT:    vmovaps 192(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 192(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm3 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm0[1,1],xmm3[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
+; AVX-NEXT:    vmovaps 192(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 192(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm12[0,1],xmm0[2],xmm12[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm0
 ; AVX-NEXT:    vbroadcastsd 192(%rdx), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovaps 208(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps 208(%rdi), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[3,3],xmm0[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vmovapd 192(%rdx), %ymm2
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm2[2,3,2,3]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm3[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7]
-; AVX-NEXT:    vmovaps 224(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 224(%rdi), %xmm3
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm5 = xmm3[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm0[1,1],xmm5[0,2]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[2,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
-; AVX-NEXT:    vbroadcastsd 224(%rdx), %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm1[3,3],xmm0[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
+; AVX-NEXT:    vmovaps 224(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 224(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm1[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm14 = xmm14[0,1],xmm0[2],xmm14[3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastsd 224(%rdx), %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovaps 240(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 240(%rdi), %xmm3
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[3,3],xmm0[3,3]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vmovapd 224(%rdx), %ymm0
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm3[0,0,3,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm1 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1],ymm1[2],ymm13[3,4],ymm1[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm8 = ymm8[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm1 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1],ymm1[2],ymm13[3,4],ymm1[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm9 = ymm9[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm1[1,2],ymm9[3],ymm1[4,5],ymm9[6],ymm1[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm1 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1],ymm1[2],ymm13[3,4],ymm1[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm11 = ymm11[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm11[0],ymm1[1,2],ymm11[3],ymm1[4,5],ymm11[6],ymm1[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm11 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm13[0,1],ymm11[2],ymm13[3,4],ymm11[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm12 = ymm12[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0],ymm11[1,2],ymm12[3],ymm11[4,5],ymm12[6],ymm11[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm12 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm6[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm12[1,2],ymm6[3],ymm12[4,5],ymm6[6],ymm12[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm12 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm4[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm12 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm2[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm12[1,2],ymm2[3],ymm12[4,5],ymm2[6],ymm12[7]
-; AVX-NEXT:    vpermilps {{.*#+}} ymm12 = mem[0,0,3,3,4,4,7,7]
-; AVX-NEXT:    vpermilpd {{.*#+}} ymm13 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1,1,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm12[1,2],ymm0[3],ymm12[4,5],ymm0[6],ymm12[7]
-; AVX-NEXT:    vmovaps %ymm0, 704(%rcx)
-; AVX-NEXT:    vmovaps %ymm2, 608(%rcx)
-; AVX-NEXT:    vmovaps %ymm4, 512(%rcx)
-; AVX-NEXT:    vmovaps %ymm6, 416(%rcx)
-; AVX-NEXT:    vmovaps %ymm11, 320(%rcx)
-; AVX-NEXT:    vmovaps %ymm1, 224(%rcx)
-; AVX-NEXT:    vmovaps %ymm9, 128(%rcx)
-; AVX-NEXT:    vmovaps %ymm8, 32(%rcx)
-; AVX-NEXT:    vmovaps %ymm3, 736(%rcx)
-; AVX-NEXT:    vmovaps %ymm5, 672(%rcx)
-; AVX-NEXT:    vmovaps %ymm7, 640(%rcx)
-; AVX-NEXT:    vmovaps %ymm10, 576(%rcx)
-; AVX-NEXT:    vmovaps %ymm15, 544(%rcx)
-; AVX-NEXT:    vmovaps %ymm14, 480(%rcx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 448(%rcx)
+; AVX-NEXT:    vmovaps 240(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm1[3,3],xmm0[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm0 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm0 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm2 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm2 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm2 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm3 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm3 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm4 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm4 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm4 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm5 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm5 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm5 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm6 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm6 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0],ymm5[1,2],ymm6[3],ymm5[4,5],ymm6[6],ymm5[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm6 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm7 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm7 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0],ymm6[1,2],ymm7[3],ymm6[4,5],ymm7[6],ymm6[7]
+; AVX-NEXT:    vpermilps {{.*#+}} ymm7 = mem[0,0,3,3,4,4,7,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm8 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2],ymm8[3,4],ymm7[5],ymm8[6,7]
+; AVX-NEXT:    vpermilpd {{.*#+}} ymm8 = mem[1,1,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0],ymm7[1,2],ymm8[3],ymm7[4,5],ymm8[6],ymm7[7]
+; AVX-NEXT:    vmovaps %ymm7, 704(%rcx)
+; AVX-NEXT:    vmovaps %ymm6, 608(%rcx)
+; AVX-NEXT:    vmovaps %ymm5, 512(%rcx)
+; AVX-NEXT:    vmovaps %ymm4, 416(%rcx)
+; AVX-NEXT:    vmovaps %ymm3, 320(%rcx)
+; AVX-NEXT:    vmovaps %ymm2, 224(%rcx)
+; AVX-NEXT:    vmovaps %ymm0, 128(%rcx)
+; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)
+; AVX-NEXT:    vmovaps %ymm15, 736(%rcx)
+; AVX-NEXT:    vmovaps %ymm14, 672(%rcx)
+; AVX-NEXT:    vmovaps %ymm13, 640(%rcx)
+; AVX-NEXT:    vmovaps %ymm12, 576(%rcx)
+; AVX-NEXT:    vmovaps %ymm11, 544(%rcx)
+; AVX-NEXT:    vmovaps %ymm10, 480(%rcx)
+; AVX-NEXT:    vmovaps %ymm9, 448(%rcx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 384(%rcx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 352(%rcx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 288(%rcx)
-; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 256(%rcx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 256(%rcx)
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 192(%rcx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 160(%rcx)
@@ -2769,7 +2752,7 @@ define void @store_i32_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vmovaps %ymm0, 64(%rcx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, (%rcx)
-; AVX-NEXT:    addq $200, %rsp
+; AVX-NEXT:    addq $168, %rsp
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-4.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-4.ll
index c15eff9141ff..984654e78df0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-4.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-4.ll
@@ -251,21 +251,21 @@ define void @store_i32_stride4_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX-NEXT:    vmovaps (%rsi), %xmm1
 ; AVX-NEXT:    vmovaps (%rdx), %xmm2
 ; AVX-NEXT:    vmovaps (%rcx), %xmm3
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = xmm0[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm5 = xmm2[0],xmm3[0]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm0[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0],xmm1[1],xmm5[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[3,0],xmm0[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm6[2,0],xmm5[2,3]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm6[0],xmm4[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = zero,zero,xmm2[2],xmm3[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps %ymm0, 32(%r8)
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
+; AVX-NEXT:    vmovaps %xmm0, 48(%r8)
+; AVX-NEXT:    vmovaps %xmm1, 32(%r8)
 ; AVX-NEXT:    vmovaps %ymm4, (%r8)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -495,50 +495,50 @@ define void @store_i32_stride4_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ;
 ; AVX-LABEL: store_i32_stride4_vf8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps (%rdi), %xmm2
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm0
 ; AVX-NEXT:    vmovaps (%rsi), %xmm4
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = xmm2[1],xmm4[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
-; AVX-NEXT:    vmovaps (%rcx), %xmm5
-; AVX-NEXT:    vmovaps 16(%rcx), %xmm6
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm0
+; AVX-NEXT:    vmovaps (%rdi), %xmm5
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm5[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps (%rcx), %xmm6
+; AVX-NEXT:    vmovaps 16(%rcx), %xmm3
 ; AVX-NEXT:    vmovaps (%rdx), %xmm7
 ; AVX-NEXT:    vmovaps 16(%rdx), %xmm8
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm10 = xmm5[0],xmm7[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[0,1,2,0]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[0,1,0,1]
 ; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm9[2,3],ymm3[4,5],ymm9[6,7]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = xmm0[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm10 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm10 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm11 = xmm6[0],xmm8[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[0,1,2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1],ymm10[2,3],ymm9[4,5],ymm10[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,zero,xmm7[2],xmm5[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm5, %ymm5
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm4[3,0],xmm2[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm7, %ymm2
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,zero,xmm8[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[3,0],xmm0[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]
-; AVX-NEXT:    vmovaps %ymm0, 96(%r8)
-; AVX-NEXT:    vmovaps %ymm2, 32(%r8)
-; AVX-NEXT:    vmovaps %ymm9, 64(%r8)
-; AVX-NEXT:    vmovaps %ymm3, (%r8)
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm9[2,3],ymm2[4,5],ymm9[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5],ymm5[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm5
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm8[2],xmm3[2],xmm8[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm6[2,3],ymm5[4,5],ymm6[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0],xmm0[1],xmm6[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm8[0],xmm3[0],xmm8[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps %ymm0, 64(%r8)
+; AVX-NEXT:    vmovaps %ymm5, 96(%r8)
+; AVX-NEXT:    vmovaps %ymm4, 32(%r8)
+; AVX-NEXT:    vmovaps %ymm2, (%r8)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -877,108 +877,94 @@ define void @store_i32_stride4_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride4_vf16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $24, %rsp
-; AVX-NEXT:    vmovaps (%rdi), %xmm7
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm3
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm10
-; AVX-NEXT:    vmovaps (%rsi), %xmm9
+; AVX-NEXT:    vmovaps (%rsi), %xmm3
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps 32(%rsi), %xmm4
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm5
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm7[1],xmm9[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps (%rcx), %xmm14
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm6
-; AVX-NEXT:    vmovaps 48(%rcx), %xmm11
-; AVX-NEXT:    vmovaps (%rdx), %xmm2
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm8
-; AVX-NEXT:    vmovaps 48(%rdx), %xmm12
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm14[0],xmm2[1],xmm14[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm13 = xmm14[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm13[0,1,2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm13, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm10[1],xmm5[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm13 = xmm11[0],xmm12[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm13[0,1,2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm13, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm3[1],xmm4[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm6[0],xmm8[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm13
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm15
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm13[1],xmm15[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm13[0],xmm15[0],xmm13[1],xmm15[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm3
-; AVX-NEXT:    vmovaps 16(%rcx), %xmm1
-; AVX-NEXT:    vmovaps 16(%rdx), %xmm0
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm10 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm10, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm2[2],xmm14[2],xmm2[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm2[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm7[2],xmm9[2],xmm7[3],xmm9[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm9[3,0],xmm7[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,zero,xmm12[2],xmm11[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[3,0],xmm7[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[2,0,2,3]
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm7
+; AVX-NEXT:    vmovaps (%rdi), %xmm6
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm1
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm5
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm8
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm2[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm2, %ymm2
+; AVX-NEXT:    vmovaps (%rcx), %xmm9
+; AVX-NEXT:    vmovaps (%rdx), %xmm10
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm9[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm10[0,1,2],xmm12[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
+; AVX-NEXT:    vmovaps 48(%rcx), %xmm12
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm11[2,3],ymm2[4,5],ymm11[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm6[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm11[0],xmm3[1],xmm11[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
+; AVX-NEXT:    vmovaps 48(%rdx), %xmm13
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm3, %ymm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm6[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm9, %ymm6
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm6[2,3],ymm3[4,5],ymm6[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm8[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0],xmm7[1],xmm6[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm9, %ymm6
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm10
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm9[2,3],ymm6[4,5],ymm9[6,7]
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm9
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm7[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm7, %ymm7
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm8 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm12[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm5[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm8 = xmm8[0],xmm4[1],xmm8[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm11[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
+; AVX-NEXT:    vmovaps 16(%rcx), %xmm12
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm11[2,3],ymm8[4,5],ymm11[6,7]
+; AVX-NEXT:    vmovaps 16(%rdx), %xmm11
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[2,3,2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,zero,xmm8[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[3,0],xmm7[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm5
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,zero,xmm0[2],xmm1[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm13[2],xmm15[2],xmm13[3],xmm15[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm15[3,0],xmm13[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm9[2],xmm10[2],xmm9[3],xmm10[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1,2],xmm10[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm9, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5],ymm5[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0],xmm0[1],xmm5[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm9, %ymm5
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm9[2,3],ymm5[4,5],ymm9[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm11[2],xmm12[2],xmm11[3],xmm12[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm12[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm9, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovaps %ymm0, 96(%r8)
+; AVX-NEXT:    vmovaps %ymm5, 64(%r8)
 ; AVX-NEXT:    vmovaps %ymm4, 160(%r8)
-; AVX-NEXT:    vmovaps %ymm3, 224(%r8)
+; AVX-NEXT:    vmovaps %ymm8, 128(%r8)
+; AVX-NEXT:    vmovaps %ymm7, 224(%r8)
+; AVX-NEXT:    vmovaps %ymm6, 192(%r8)
+; AVX-NEXT:    vmovaps %ymm3, (%r8)
 ; AVX-NEXT:    vmovaps %ymm2, 32(%r8)
-; AVX-NEXT:    vmovaps %ymm10, 64(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 128(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 192(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, (%r8)
-; AVX-NEXT:    addq $24, %rsp
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -1687,244 +1673,194 @@ define void @store_i32_stride4_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride4_vf32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $488, %rsp # imm = 0x1E8
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    subq $40, %rsp
 ; AVX-NEXT:    vmovaps 16(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm9
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm6
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm2
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm10
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm8
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps 16(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 16(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 64(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 80(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 80(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 80(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 80(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm11
+; AVX-NEXT:    vmovaps 16(%rdx), %xmm4
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm4[0,1,2],xmm7[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm7, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm5[2,3],ymm0[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm2[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0],xmm1[1],xmm5[2,3]
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm13
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm13[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 48(%rcx), %xmm10
-; AVX-NEXT:    vmovaps 48(%rdx), %xmm9
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm10[0],xmm9[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 96(%rcx), %xmm7
-; AVX-NEXT:    vmovaps 96(%rdx), %xmm5
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm7[0],xmm5[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 112(%rdi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 112(%rsi), %xmm12
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[1],xmm12[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 112(%rcx), %xmm4
-; AVX-NEXT:    vmovaps 112(%rdx), %xmm3
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm4[0],xmm3[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rdi), %xmm11
-; AVX-NEXT:    vmovaps (%rsi), %xmm8
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm11[1],xmm8[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm11[0],xmm8[0],xmm11[1],xmm8[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps (%rcx), %xmm6
-; AVX-NEXT:    vmovaps (%rdx), %xmm2
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm14 = xmm6[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm14[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm15 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm14[2,3],ymm0[4,5],ymm14[6,7]
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm3
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm4
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm4[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
+; AVX-NEXT:    vmovaps 64(%rcx), %xmm5
+; AVX-NEXT:    vmovaps 64(%rdx), %xmm7
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm12 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm12[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm13, %ymm12
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm12[2,3],ymm2[4,5],ymm12[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm14[2],xmm1[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm1[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm14, %ymm1
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm0[2],xmm15[2],xmm0[3],xmm15[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[3,0],xmm0[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm1[2,3],ymm14[4,5],ymm1[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm7[0,1,2],xmm5[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps (%rsp), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm14[2],xmm1[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm1[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm14, %ymm0
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm1[2],xmm15[3],xmm1[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm1[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm0[2,3],ymm14[4,5],ymm0[6,7]
+; AVX-NEXT:    vmovaps 80(%rsi), %xmm5
+; AVX-NEXT:    vmovaps 80(%rdi), %xmm7
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm7[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm12 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm12, %ymm4
+; AVX-NEXT:    vmovaps 80(%rcx), %xmm13
+; AVX-NEXT:    vmovaps 80(%rdx), %xmm14
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm12 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm12[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm15, %ymm15
+; AVX-NEXT:    vmovaps 48(%rcx), %xmm12
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm15[2,3],ymm4[4,5],ymm15[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm14[2],xmm1[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm1[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm14, %ymm0
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm1[2],xmm15[3],xmm1[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm1[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm14[0,1],ymm0[2,3],ymm14[4,5],ymm0[6,7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm15
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm5[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm13[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm13, %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm7[2,3],ymm5[4,5],ymm7[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm10[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm13, %ymm7
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm15[0],xmm11[0],xmm15[1],xmm11[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm13[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13
+; AVX-NEXT:    vmovaps 48(%rdx), %xmm14
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1],ymm13[2,3],ymm7[4,5],ymm13[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm9 = xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm9
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm15[2],xmm11[2],xmm15[3],xmm11[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm15[0,1,2],xmm11[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1],ymm10[2,3],ymm9[4,5],ymm10[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm8[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0],xmm6[1],xmm10[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm14[0],xmm12[0],xmm14[1],xmm12[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm11[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm13, %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm11[2,3],ymm10[4,5],ymm11[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm8[2],xmm6[2],xmm8[3],xmm6[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm6[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm6, %ymm6
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm8 = xmm14[2],xmm12[2],xmm14[3],xmm12[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm12[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm14[0,1,2],xmm11[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm8[2,3],ymm6[4,5],ymm8[6,7]
+; AVX-NEXT:    vmovaps 96(%rsi), %xmm11
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm12
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm12[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm8 = xmm8[0],xmm11[1],xmm8[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm13, %ymm8
+; AVX-NEXT:    vmovaps 96(%rcx), %xmm13
+; AVX-NEXT:    vmovaps 96(%rdx), %xmm14
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm15 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm15[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm0[2,3],ymm8[4,5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm13[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm14[0,1,2],xmm12[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1],ymm11[2,3],ymm0[4,5],ymm11[6,7]
+; AVX-NEXT:    vmovaps 112(%rsi), %xmm1
+; AVX-NEXT:    vmovaps 112(%rdi), %xmm13
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm13[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm12[0],xmm1[1],xmm12[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm13[0],xmm1[0],xmm13[1],xmm1[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
+; AVX-NEXT:    vmovaps 112(%rcx), %xmm14
+; AVX-NEXT:    vmovaps 112(%rdx), %xmm15
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1],ymm0[2,3],ymm12[4,5],ymm0[6,7]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm13 = zero,zero,xmm13[2],xmm1[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm13, %ymm0
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm13 = xmm14[2],xmm1[2],xmm14[3],xmm1[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm1[3,0],xmm14[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm14[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm13, %ymm13
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm14[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm15[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT:    vmovaps (%rsi), %xmm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps (%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm13 = xmm13[0],xmm2[1],xmm13[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13
+; AVX-NEXT:    vmovaps (%rcx), %xmm14
+; AVX-NEXT:    vmovaps (%rdx), %xmm15
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm0[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1],ymm0[2,3],ymm13[4,5],ymm0[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm13 = xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = zero,zero,xmm9[2],xmm10[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm9, %ymm9
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm1[3,0],xmm13[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm13[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm10, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,3],ymm10[4,5],ymm9[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm5[2],xmm7[2],xmm5[3],xmm7[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,zero,xmm5[2],xmm7[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm5, %ymm5
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm1[3,0],xmm10[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm7, %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = zero,zero,xmm3[2],xmm4[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm3, %ymm3
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm1[2],xmm12[2],xmm1[3],xmm12[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm12[3,0],xmm1[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm4, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = zero,zero,xmm2[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm11[2],xmm8[2],xmm11[3],xmm8[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm8[3,0],xmm11[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm14[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm15[0,1,2],xmm4[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
 ; AVX-NEXT:    vmovaps %ymm1, 32(%r8)
-; AVX-NEXT:    vmovaps %ymm3, 480(%r8)
-; AVX-NEXT:    vmovaps %ymm5, 416(%r8)
-; AVX-NEXT:    vmovaps %ymm9, 224(%r8)
-; AVX-NEXT:    vmovaps %ymm0, 160(%r8)
-; AVX-NEXT:    vmovaps %ymm15, 352(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 288(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 96(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, (%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 448(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 384(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 192(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 128(%r8)
+; AVX-NEXT:    vmovaps %ymm3, 480(%r8)
+; AVX-NEXT:    vmovaps %ymm12, 448(%r8)
+; AVX-NEXT:    vmovaps %ymm11, 416(%r8)
+; AVX-NEXT:    vmovaps %ymm8, 384(%r8)
+; AVX-NEXT:    vmovaps %ymm6, 224(%r8)
+; AVX-NEXT:    vmovaps %ymm10, 192(%r8)
+; AVX-NEXT:    vmovaps %ymm9, 160(%r8)
+; AVX-NEXT:    vmovaps %ymm7, 128(%r8)
+; AVX-NEXT:    vmovaps %ymm5, 352(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 320(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 288(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 256(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 64(%r8)
-; AVX-NEXT:    addq $488, %rsp # imm = 0x1E8
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 96(%r8)
+; AVX-NEXT:    addq $40, %rsp
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -3287,516 +3223,402 @@ define void @store_i32_stride4_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride4_vf64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $1384, %rsp # imm = 0x568
-; AVX-NEXT:    vmovaps (%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm5
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm3
-; AVX-NEXT:    vmovaps (%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    subq $552, %rsp # imm = 0x228
+; AVX-NEXT:    vmovaps (%rsi), %xmm8
 ; AVX-NEXT:    vmovaps 16(%rsi), %xmm6
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm4
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps (%rcx), %xmm11
-; AVX-NEXT:    vmovaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 16(%rcx), %xmm9
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm7
-; AVX-NEXT:    vmovaps (%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 16(%rdx), %xmm10
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm8
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm11[0],xmm2[1],xmm11[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm11[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm5[1],xmm6[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm9[0],xmm10[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,0]
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm3
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm0
+; AVX-NEXT:    vmovaps (%rdi), %xmm11
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm9
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm4
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm11[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm8[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm11[0],xmm8[0],xmm11[1],xmm8[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm7
+; AVX-NEXT:    vmovaps (%rcx), %xmm12
+; AVX-NEXT:    vmovaps 16(%rcx), %xmm10
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm5
+; AVX-NEXT:    vmovaps 48(%rcx), %xmm2
+; AVX-NEXT:    vmovaps (%rdx), %xmm13
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm14[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm15, %ymm14
+; AVX-NEXT:    vmovaps 16(%rdx), %xmm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1],ymm14[2,3],ymm7[4,5],ymm14[6,7]
+; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm7
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm8 = xmm11[2],xmm8[2],xmm11[3],xmm8[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm8[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm8, %ymm8
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm12[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm13[0,1,2],xmm12[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm11[2,3],ymm8[4,5],ymm11[6,7]
+; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm9[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm8 = xmm8[0],xmm6[1],xmm8[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm11
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm15[0],xmm10[0],xmm15[1],xmm10[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm8[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm12, %ymm12
+; AVX-NEXT:    vmovaps 48(%rdx), %xmm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1],ymm12[2,3],ymm11[4,5],ymm12[6,7]
+; AVX-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm9[2],xmm6[2],xmm9[3],xmm6[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm6[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm6, %ymm6
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm9 = xmm15[2],xmm10[2],xmm15[3],xmm10[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm15[0,1,2],xmm10[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm9[2,3],ymm6[4,5],ymm9[6,7]
+; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm4[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0],xmm3[1],xmm6[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm9, %ymm6
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm9[2,3],ymm6[4,5],ymm9[6,7]
+; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm7[0,1,2],xmm5[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
+; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0],xmm0[1],xmm3[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
+; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm8[2],xmm2[2],xmm8[3],xmm2[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm8[0,1,2],xmm2[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm3[1],xmm4[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm7[0],xmm8[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 48(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 48(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX-NEXT:    vmovaps 64(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 64(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 80(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 80(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 80(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 80(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX-NEXT:    vmovaps 80(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 80(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 80(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 96(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX-NEXT:    vmovaps 96(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 96(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 112(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 112(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 112(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 112(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX-NEXT:    vmovaps 112(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 112(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 112(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 128(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 128(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 128(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX-NEXT:    vmovaps 128(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 128(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 128(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm2, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 144(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 144(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 144(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 144(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX-NEXT:    vmovaps 144(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vmovaps 144(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 144(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 160(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vmovaps 160(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 160(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX-NEXT:    vmovaps 160(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 160(%rdx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 176(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 176(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 176(%rcx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 176(%rdx), %xmm13
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm13[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 192(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 192(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 192(%rcx), %xmm10
-; AVX-NEXT:    vmovaps 192(%rdx), %xmm8
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm10[0],xmm8[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 208(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 208(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 208(%rcx), %xmm6
-; AVX-NEXT:    vmovaps 208(%rdx), %xmm4
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm6[0],xmm4[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 224(%rdi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 224(%rsi), %xmm11
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[1],xmm11[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 224(%rcx), %xmm3
-; AVX-NEXT:    vmovaps 224(%rdx), %xmm2
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 240(%rdi), %xmm9
-; AVX-NEXT:    vmovaps 240(%rsi), %xmm7
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm9[1],xmm7[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm12
-; AVX-NEXT:    vmovaps 240(%rcx), %xmm5
-; AVX-NEXT:    vmovaps 240(%rdx), %xmm1
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm14 = xmm5[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm14[0,1,2,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm15 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm12[0,1],ymm14[2,3],ymm12[4,5],ymm14[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm0[2],xmm15[2],xmm0[3],xmm15[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[3,0],xmm0[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps (%rsp), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm14 = zero,zero,xmm0[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm12
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm13 = zero,zero,xmm13[2],xmm0[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm13, %ymm13
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[3,0],xmm15[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1],ymm13[2,3],ymm14[4,5],ymm13[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm14 = xmm8[2],xmm10[2],xmm8[3],xmm10[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm8 = zero,zero,xmm8[2],xmm10[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm8, %ymm8
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm14[2],xmm0[2],xmm14[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm0[3,0],xmm14[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm14[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm10, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1],ymm8[2,3],ymm10[4,5],ymm8[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,zero,xmm4[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm4, %ymm4
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm10[2],xmm0[2],xmm10[3],xmm0[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm0[3,0],xmm10[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm6, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm2[2],xmm3[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm0[2],xmm11[2],xmm0[3],xmm11[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm11[3,0],xmm0[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[2,0,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,zero,xmm1[2],xmm5[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm7[3,0],xmm9[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,0,2,3]
+; AVX-NEXT:    vmovaps 160(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 176(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 176(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 176(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 176(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 192(%rsi), %xmm4
+; AVX-NEXT:    vmovaps 192(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm5
+; AVX-NEXT:    vmovaps 192(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 192(%rdx), %xmm2
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm0[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm14, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm5[0,1],ymm0[2,3],ymm5[4,5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 208(%rsi), %xmm3
+; AVX-NEXT:    vmovaps 208(%rdi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
+; AVX-NEXT:    vmovaps 208(%rcx), %xmm4
+; AVX-NEXT:    vmovaps 208(%rdx), %xmm14
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm14[0],xmm4[0],xmm14[1],xmm4[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm0[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm15, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm14[2],xmm4[2],xmm14[3],xmm4[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm14[0,1,2],xmm3[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 224(%rsi), %xmm3
+; AVX-NEXT:    vmovaps 224(%rdi), %xmm4
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm4[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm14, %ymm1
+; AVX-NEXT:    vmovaps 224(%rcx), %xmm14
+; AVX-NEXT:    vmovaps 224(%rdx), %xmm15
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm0[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm13, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vmovaps %ymm0, 992(%r8)
-; AVX-NEXT:    vmovaps %ymm2, 928(%r8)
-; AVX-NEXT:    vmovaps %ymm4, 864(%r8)
-; AVX-NEXT:    vmovaps %ymm8, 800(%r8)
-; AVX-NEXT:    vmovaps %ymm13, 736(%r8)
-; AVX-NEXT:    vmovaps %ymm12, 672(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 608(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 544(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 480(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 416(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 352(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 288(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 224(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 160(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 96(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 32(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm14[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm15[0,1,2],xmm4[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]
+; AVX-NEXT:    vmovaps 240(%rsi), %xmm3
+; AVX-NEXT:    vmovaps 240(%rdi), %xmm4
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm4[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm13 = xmm13[0],xmm3[1],xmm13[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13
+; AVX-NEXT:    vmovaps 240(%rcx), %xmm14
+; AVX-NEXT:    vmovaps 240(%rdx), %xmm15
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm0[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm12, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1],ymm0[2,3],ymm13[4,5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm14[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm15[0,1,2],xmm12[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm12, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
+; AVX-NEXT:    vmovaps %ymm3, 992(%r8)
 ; AVX-NEXT:    vmovaps %ymm0, 960(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 896(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 832(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 768(%r8)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 704(%r8)
+; AVX-NEXT:    vmovaps %ymm1, 928(%r8)
+; AVX-NEXT:    vmovaps %ymm2, 896(%r8)
+; AVX-NEXT:    vmovaps %ymm5, 864(%r8)
+; AVX-NEXT:    vmovaps %ymm6, 832(%r8)
+; AVX-NEXT:    vmovaps %ymm7, 800(%r8)
+; AVX-NEXT:    vmovaps %ymm8, 768(%r8)
+; AVX-NEXT:    vmovaps %ymm9, 736(%r8)
+; AVX-NEXT:    vmovaps %ymm10, 704(%r8)
+; AVX-NEXT:    vmovaps %ymm11, 672(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 640(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 608(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 576(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 544(%r8)
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 512(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 480(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 448(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 416(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 384(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 352(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 320(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 288(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 256(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 224(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 192(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 160(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 128(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 96(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 64(%r8)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 32(%r8)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, (%r8)
-; AVX-NEXT:    addq $1384, %rsp # imm = 0x568
+; AVX-NEXT:    addq $552, %rsp # imm = 0x228
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
index d591b7ad59b0..d83f969dd033 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
@@ -51,7 +51,7 @@ define void @store_i32_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm4[1,1],xmm0[1,1]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,0]
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm5[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm5[0],xmm0[1,2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[3,3,3,3]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm5[1],xmm1[2,3]
@@ -338,28 +338,28 @@ define void @store_i32_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX-NEXT:    vmovaps (%rsi), %xmm1
 ; AVX-NEXT:    vmovaps (%rdx), %xmm2
 ; AVX-NEXT:    vmovaps (%rcx), %xmm3
-; AVX-NEXT:    vmovaps (%r8), %xmm4
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm5 = xmm2[0],xmm3[0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm6[0,1],xmm5[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm6
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm0[2,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm0[0,1,2],xmm5[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = xmm3[1],zero,xmm5[2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3,4,5],ymm5[6],ymm4[7]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm7[0],xmm6[0]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm7 = zero,xmm0[1],xmm1[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm7 = zero,xmm7[1,2],xmm2[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0],ymm5[1,2,3],ymm6[4],ymm5[5,6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm0[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm4[2],xmm6[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,zero,xmm0[2],xmm1[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[1,1],xmm0[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm4[1],xmm0[2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX-NEXT:    vinsertps {{.*#+}} xmm7 = xmm7[0,1,2],xmm2[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm6
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm6[1,2,3],ymm0[4],ymm6[5,6,7]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3],xmm3[3,3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[3],xmm2[1,2],zero
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[3],xmm2[1,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[3]
 ; AVX-NEXT:    vmovaps %xmm1, 64(%r9)
-; AVX-NEXT:    vmovaps %ymm0, 32(%r9)
-; AVX-NEXT:    vmovaps %ymm5, (%r9)
+; AVX-NEXT:    vmovaps %ymm0, (%r9)
+; AVX-NEXT:    vmovaps %ymm4, 32(%r9)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -452,7 +452,7 @@ define void @store_i32_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FCP-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [7,3,7,3,7,3,7,3]
 ; AVX2-FCP-NEXT:    vpermps %ymm2, %ymm5, %ymm2
 ; AVX2-FCP-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-FCP-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
+; AVX2-FCP-NEXT:    vmovss {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]
 ; AVX2-FCP-NEXT:    vmovaps %ymm4, 32(%r9)
 ; AVX2-FCP-NEXT:    vmovaps %ymm0, (%r9)
 ; AVX2-FCP-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm3[3]
@@ -687,64 +687,63 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ;
 ; AVX-LABEL: store_i32_stride5_vf8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps (%rdi), %ymm1
-; AVX-NEXT:    vmovaps (%rsi), %ymm3
-; AVX-NEXT:    vmovaps (%rdx), %ymm2
-; AVX-NEXT:    vmovaps (%rcx), %ymm4
-; AVX-NEXT:    vmovaps (%r8), %ymm0
-; AVX-NEXT:    vmovaps (%rsi), %xmm6
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm7
-; AVX-NEXT:    vmovaps (%rdi), %xmm8
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm9
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,xmm8[1],xmm6[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = xmm8[0],xmm6[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm10, %ymm5
+; AVX-NEXT:    vmovaps (%rdi), %ymm0
+; AVX-NEXT:    vmovaps (%rsi), %ymm2
+; AVX-NEXT:    vmovaps (%rdx), %ymm1
+; AVX-NEXT:    vmovaps (%rcx), %ymm3
+; AVX-NEXT:    vmovaps (%rsi), %xmm5
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm6
+; AVX-NEXT:    vmovaps (%rdi), %xmm7
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm8
+; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,xmm7[1],xmm5[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm9, %ymm4
+; AVX-NEXT:    vmovaps (%rcx), %xmm9
+; AVX-NEXT:    vmovaps 16(%rcx), %xmm10
 ; AVX-NEXT:    vmovaps (%rdx), %xmm11
-; AVX-NEXT:    vmovaps (%rcx), %xmm12
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm13 = xmm12[0],xmm11[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm13[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 4(%rdx), %xmm14
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm13, %ymm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm13[2,3],ymm5[4,5,6],ymm13[7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm10, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm10[0],ymm5[1,2,3],ymm10[4],ymm5[5,6,7]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = zero,zero,xmm9[2],xmm7[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm9[1,1],xmm7[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm7, %ymm7
-; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm4[1,1],ymm2[1,1],ymm4[5,5],ymm2[5,5]
+; AVX-NEXT:    vmovaps 16(%rdx), %xmm12
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm13, %ymm13
+; AVX-NEXT:    vmovddup {{.*#+}} ymm13 = ymm13[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm13[2,3],ymm4[4,5,6],ymm13[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm13[4],ymm4[5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm11[2],xmm9[2],xmm11[3],xmm9[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm9, %ymm9
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm7[0,1,2],xmm5[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm13[1],ymm5[2,3,4,5],ymm13[6],ymm5[7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm6[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm6 = zero,xmm8[1],xmm6[1],zero
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm6
+; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm3[1,1],ymm1[1,1],ymm3[5,5],ymm1[5,5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm7[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm7[3,4],ymm6[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0],ymm6[1,2,3,4],ymm7[5],ymm6[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm12[3,3],xmm10[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm9 = xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
+; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm2[3,3],ymm0[3,3],ymm2[7,7],ymm0[7,7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm9[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm9[3,4],ymm7[5,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm0[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0],ymm7[1,2,3,4],ymm9[5],ymm7[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm11[2],xmm12[2],xmm11[3],xmm12[3]
-; AVX-NEXT:    vbroadcastss 4(%rcx), %xmm11
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10
-; AVX-NEXT:    vinsertps {{.*#+}} xmm6 = zero,zero,xmm8[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm6, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm10[0,1],ymm6[2,3],ymm10[4,5],ymm6[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm8[1],ymm6[2,3,4,5],ymm8[6],ymm6[7]
-; AVX-NEXT:    vmovaps 16(%rcx), %xmm8
-; AVX-NEXT:    vmovaps 16(%rdx), %xmm10
-; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm10[3,3],xmm8[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm8 = xmm10[2],xmm8[2],xmm10[3],xmm8[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm8, %ymm8
-; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm3[3,3],ymm1[3,3],ymm3[7,7],ymm1[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = ymm10[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm10[3,4],ymm8[5,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm9[2],ymm8[3,4,5,6],ymm9[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4],ymm3[5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm4[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm2[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4,5,6],ymm3[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4,5],ymm2[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm9[3,4],ymm8[5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2],ymm8[3,4,5,6],ymm7[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovaps %ymm0, 64(%r9)
-; AVX-NEXT:    vmovaps %ymm8, 128(%r9)
-; AVX-NEXT:    vmovaps %ymm6, 32(%r9)
-; AVX-NEXT:    vmovaps %ymm7, 96(%r9)
-; AVX-NEXT:    vmovaps %ymm5, (%r9)
+; AVX-NEXT:    vmovaps %ymm7, 128(%r9)
+; AVX-NEXT:    vmovaps %ymm6, 96(%r9)
+; AVX-NEXT:    vmovaps %ymm5, 32(%r9)
+; AVX-NEXT:    vmovaps %ymm4, (%r9)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -1329,130 +1328,126 @@ define void @store_i32_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-LABEL: store_i32_stride5_vf16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovaps (%rdi), %ymm0
-; AVX-NEXT:    vmovaps 32(%rdi), %ymm1
-; AVX-NEXT:    vmovaps 32(%rsi), %ymm3
-; AVX-NEXT:    vmovaps 32(%rdx), %ymm2
-; AVX-NEXT:    vmovaps (%rsi), %xmm6
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm8
-; AVX-NEXT:    vmovaps (%rdi), %xmm10
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm9
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,xmm10[1],xmm6[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = xmm10[0],xmm6[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
-; AVX-NEXT:    vmovaps (%rdx), %xmm11
-; AVX-NEXT:    vmovaps (%rcx), %xmm12
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm7 = xmm12[0],xmm11[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 4(%rdx), %xmm13
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm7, %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm7[2,3],ymm4[4,5,6],ymm7[7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm5, %ymm5
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1,2,3],ymm5[4],ymm4[5,6,7]
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,xmm9[1],xmm8[1],zero
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm13
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm14
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm7 = xmm14[0],xmm13[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 36(%rdx), %xmm15
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm7, %ymm7
-; AVX-NEXT:    vinsertps {{.*#+}} xmm15 = xmm9[0],xmm8[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm15, %ymm5
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm7[2,3],ymm5[4,5,6],ymm7[7]
-; AVX-NEXT:    vmovaps 32(%rcx), %ymm7
-; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm15, %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm15[0],ymm5[1,2,3],ymm15[4],ymm5[5,6,7]
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm11[2],xmm12[2],xmm11[3],xmm12[3]
-; AVX-NEXT:    vbroadcastss 4(%rcx), %xmm12
-; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
-; AVX-NEXT:    vinsertps {{.*#+}} xmm6 = zero,zero,xmm10[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm6, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3],ymm11[4,5],ymm6[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0],ymm10[1],ymm6[2,3,4,5],ymm10[6],ymm6[7]
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm10
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm11
-; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = zero,zero,xmm11[2],xmm10[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm11[1,1],xmm10[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm10, %ymm10
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm7[1,1],ymm2[1,1],ymm7[5,5],ymm2[5,5]
-; AVX-NEXT:    vmovaps %ymm2, %ymm6
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = ymm11[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2],ymm11[3,4],ymm10[5,6,7]
-; AVX-NEXT:    vmovaps 48(%rcx), %xmm11
-; AVX-NEXT:    vmovaps 48(%rdx), %xmm12
-; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm12[3,3],xmm11[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVX-NEXT:    vmovaps 32(%r8), %ymm12
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm11, %ymm11
-; AVX-NEXT:    vshufps {{.*#+}} ymm15 = ymm3[3,3],ymm1[3,3],ymm3[7,7],ymm1[7,7]
-; AVX-NEXT:    vmovaps %ymm1, %ymm2
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm2
+; AVX-NEXT:    vmovaps (%rsi), %ymm0
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 32(%rsi), %ymm4
+; AVX-NEXT:    vmovaps 32(%rdx), %ymm1
+; AVX-NEXT:    vmovaps 32(%rcx), %ymm3
+; AVX-NEXT:    vmovaps (%rcx), %xmm6
+; AVX-NEXT:    vmovaps 16(%rcx), %xmm8
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm11
+; AVX-NEXT:    vmovaps (%rdx), %xmm7
+; AVX-NEXT:    vmovaps 16(%rdx), %xmm9
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm12
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm6[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm13, %ymm10
+; AVX-NEXT:    vmovaps (%rdi), %xmm13
+; AVX-NEXT:    vmovaps (%rsi), %xmm14
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm14[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm15 = xmm13[0,1,2],xmm15[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm15, %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm15[2,3],ymm10[4,5],ymm15[6,7]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm15 = zero,xmm13[1],xmm14[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm14
+; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm13, %ymm13
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm15
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm7
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm10[0],ymm0[1],ymm10[2,3,4,5],ymm0[6],ymm10[7]
+; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovddup {{.*#+}} ymm7 = ymm7[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm13[0,1],ymm7[2,3],ymm13[4,5,6],ymm7[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm0[4],ymm7[5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm11[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm10, %ymm0
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm15[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm14[0,1,2],xmm10[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm10, %ymm10
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm10[2,3],ymm0[4,5],ymm10[6,7]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm13 = zero,xmm14[1],xmm15[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm15 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0],ymm15[1],ymm0[2,3,4,5],ymm15[6],ymm0[7]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm0
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm11, %ymm11
+; AVX-NEXT:    vmovddup {{.*#+}} ymm11 = ymm11[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm11[2,3],ymm0[4,5,6],ymm11[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2,3],ymm15[4],ymm0[5,6,7]
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm12
+; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm12[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm13 = xmm0[0,1,2],xmm13[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm12[1],zero
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm0, %ymm0
+; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm3[1,1],ymm1[1,1],ymm3[5,5],ymm1[5,5]
+; AVX-NEXT:    vmovaps %ymm1, %ymm6
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm12[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm12[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vmovaps 48(%rcx), %xmm12
+; AVX-NEXT:    vmovaps 48(%rdx), %xmm13
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm13[3,3],xmm12[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm12 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm15
+; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm12, %ymm12
+; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm4[3,3],ymm2[3,3],ymm4[7,7],ymm2[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = ymm13[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm12[0,1,2],ymm13[3,4],ymm12[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm14 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0],ymm0[1,2,3,4],ymm14[5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4,5,6],ymm14[7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm9[3,3],xmm8[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm8 = xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm9
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm8, %ymm0
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm1[3,3],ymm5[3,3],ymm1[7,7],ymm5[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm8[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm8[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm9[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm8 = xmm15[0,1,2],xmm8[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = zero,xmm15[1],xmm9[1],zero
+; AVX-NEXT:    vmovaps (%rdx), %ymm14
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
+; AVX-NEXT:    vmovaps (%rcx), %ymm9
+; AVX-NEXT:    vshufps {{.*#+}} ymm15 = ymm9[1,1],ymm14[1,1],ymm9[5,5],ymm14[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm15[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm15[3,4],ymm11[5,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm12[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm15[0],ymm10[1,2,3,4],ymm15[5],ymm10[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1],ymm15[2],ymm11[3,4,5,6],ymm15[7]
-; AVX-NEXT:    vmovaps (%rsi), %ymm15
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm13 = xmm13[2],xmm14[2],xmm13[3],xmm14[3]
-; AVX-NEXT:    vbroadcastss 36(%rcx), %xmm14
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13
-; AVX-NEXT:    vinsertps {{.*#+}} xmm8 = zero,zero,xmm9[2],xmm8[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm8, %ymm8
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm13[0,1],ymm8[2,3],ymm13[4,5],ymm8[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3,4,5],ymm9[6],ymm8[7]
-; AVX-NEXT:    vmovaps 16(%rcx), %xmm9
-; AVX-NEXT:    vmovaps 16(%rdx), %xmm13
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm13[3,3],xmm9[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm9 = xmm13[2],xmm9[2],xmm13[3],xmm9[3]
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm13
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm9, %ymm9
-; AVX-NEXT:    vmovaps %ymm0, %ymm5
-; AVX-NEXT:    vshufps {{.*#+}} ymm14 = ymm15[3,3],ymm0[3,3],ymm15[7,7],ymm0[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm14[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm14[3,4],ymm9[5,6,7]
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm14
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,zero,xmm14[2],xmm13[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm14[1,1],xmm13[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm13, %ymm0
-; AVX-NEXT:    vmovaps (%rdx), %ymm13
-; AVX-NEXT:    vmovaps (%rcx), %ymm14
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm14[1,1],ymm13[1,1],ymm14[5,5],ymm13[5,5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2],ymm4[3,4],ymm0[5,6,7]
-; AVX-NEXT:    vmovaps (%r8), %ymm4
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm4[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1],ymm0[2],ymm9[3,4,5,6],ymm0[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4],ymm0[5],ymm1[6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm6[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4,5,6],ymm3[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4,5],ymm2[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm12[3],ymm1[4,5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm14[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm7 = ymm13[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm7[0,1],ymm3[2],ymm7[3,4,5,6],ymm3[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm15[3,4],ymm8[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm15 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm15[2],ymm0[3,4,5,6],ymm15[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm15[0],ymm8[1,2,3,4],ymm15[5],ymm8[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4],ymm4[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm3[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm6[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4,5,6],ymm4[7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1,2,3],ymm2[4,5],ymm3[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm4[3],ymm2[4,5,6,7]
-; AVX-NEXT:    vmovaps %ymm2, 64(%r9)
-; AVX-NEXT:    vmovaps %ymm1, 224(%r9)
-; AVX-NEXT:    vmovaps %ymm0, 96(%r9)
-; AVX-NEXT:    vmovaps %ymm9, 128(%r9)
-; AVX-NEXT:    vmovaps %ymm8, 192(%r9)
-; AVX-NEXT:    vmovaps %ymm11, 288(%r9)
-; AVX-NEXT:    vmovaps %ymm10, 256(%r9)
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4],ymm1[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm9[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm14[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4,5,6],ymm3[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1,2,3],ymm1[4,5],ymm3[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],mem[3],ymm2[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],mem[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vmovaps %ymm1, 64(%r9)
+; AVX-NEXT:    vmovaps %ymm2, 224(%r9)
+; AVX-NEXT:    vmovaps %ymm8, 96(%r9)
+; AVX-NEXT:    vmovaps %ymm0, 128(%r9)
+; AVX-NEXT:    vmovaps %ymm13, 288(%r9)
+; AVX-NEXT:    vmovaps %ymm12, 256(%r9)
+; AVX-NEXT:    vmovaps %ymm11, 160(%r9)
+; AVX-NEXT:    vmovaps %ymm10, 192(%r9)
+; AVX-NEXT:    vmovaps %ymm7, (%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 32(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 160(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, (%r9)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -2770,290 +2765,272 @@ define void @store_i32_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride5_vf32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $600, %rsp # imm = 0x258
-; AVX-NEXT:    vmovaps (%rsi), %xmm6
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm3
-; AVX-NEXT:    vmovaps (%rdi), %xmm9
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm4
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm9[1],xmm6[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm9[0],xmm6[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps (%rdx), %xmm14
-; AVX-NEXT:    vmovaps (%rcx), %xmm15
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm15[0],xmm14[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 4(%rdx), %xmm5
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5,6],ymm2[7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm4[1],xmm3[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm4[0],xmm3[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm2
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm5
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm7 = xmm5[0],xmm2[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 36(%rdx), %xmm8
-; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm7, %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm7[2,3],ymm0[4,5,6],ymm7[7]
-; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm1
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm0
-; AVX-NEXT:    vmovaps 64(%rdx), %xmm7
-; AVX-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rcx), %xmm8
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm7 = xmm8[0],xmm7[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 68(%rdx), %xmm10
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm7, %ymm7
-; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = zero,xmm0[1],xmm1[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm11 = xmm0[0],xmm1[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm10[0,1],ymm7[2,3],ymm10[4,5,6],ymm7[7]
-; AVX-NEXT:    vinsertf128 $1, 64(%r8), %ymm11, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm10[0],ymm7[1,2,3],ymm10[4],ymm7[5,6,7]
-; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdx), %xmm10
-; AVX-NEXT:    vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rcx), %xmm7
-; AVX-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm7 = xmm7[0],xmm10[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 100(%rdx), %xmm10
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm7, %ymm7
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm13
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm12
-; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = zero,xmm12[1],xmm13[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm11 = xmm12[0],xmm13[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm10[0,1],ymm7[2,3],ymm10[4,5,6],ymm7[7]
-; AVX-NEXT:    vinsertf128 $1, 96(%r8), %ymm11, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm10[0],ymm7[1,2,3],ymm10[4],ymm7[5,6,7]
-; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX-NEXT:    vbroadcastss 4(%rcx), %xmm10
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm10, %ymm7
-; AVX-NEXT:    vinsertps {{.*#+}} xmm6 = zero,zero,xmm9[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm6, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3],ymm7[4,5],ymm6[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3,4,5],ymm7[6],ymm6[7]
-; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 16(%rsi), %xmm6
-; AVX-NEXT:    vmovaps 16(%rdi), %xmm7
-; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = zero,zero,xmm7[2],xmm6[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm7[1,1],xmm6[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm6, %ymm6
-; AVX-NEXT:    vmovaps (%rdx), %ymm15
-; AVX-NEXT:    vmovaps (%rcx), %ymm7
-; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,1],ymm15[1,1],ymm7[5,5],ymm15[5,5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm7[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm7[3,4],ymm6[5,6,7]
-; AVX-NEXT:    vmovaps 16(%rcx), %xmm7
-; AVX-NEXT:    vmovaps 16(%rdx), %xmm9
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[3,3],xmm7[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm7, %ymm7
-; AVX-NEXT:    vmovaps (%rdi), %ymm9
-; AVX-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rsi), %ymm10
-; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm10[3,3],ymm9[3,3],ymm10[7,7],ymm9[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm9[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm9[3,4],ymm7[5,6,7]
-; AVX-NEXT:    vmovaps (%r8), %ymm9
-; AVX-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm9[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm9[0],ymm6[1,2,3,4],ymm9[5],ymm6[6,7]
-; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm9[2],ymm7[3,4,5,6],ymm9[7]
-; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; AVX-NEXT:    vbroadcastss 36(%rcx), %xmm5
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
-; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = zero,zero,xmm4[2],xmm3[2]
+; AVX-NEXT:    subq $456, %rsp # imm = 0x1C8
+; AVX-NEXT:    vmovaps (%rdi), %ymm11
+; AVX-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rsi), %ymm14
+; AVX-NEXT:    vmovaps (%rdx), %ymm12
+; AVX-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rcx), %ymm13
+; AVX-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rsi), %xmm3
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm0
+; AVX-NEXT:    vmovaps (%rdi), %xmm4
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm4[1],xmm3[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm5
+; AVX-NEXT:    vmovaps (%rcx), %xmm6
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm2
+; AVX-NEXT:    vmovaps (%rdx), %xmm7
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm8
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm9, %ymm9
+; AVX-NEXT:    vmovddup {{.*#+}} ymm9 = ymm9[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm9[2,3],ymm5[4,5,6],ymm9[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm9[4],ymm5[5,6,7]
+; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3,4,5],ymm3[6],ymm2[7]
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm2
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm3
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,zero,xmm3[2],xmm2[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[1,1],xmm2[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; AVX-NEXT:    vmovaps 32(%rdx), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3],ymm5[4,5],ymm3[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm9[1],ymm3[2,3,4,5],ymm9[6],ymm3[7]
 ; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 32(%rcx), %ymm4
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm4[1,1],ymm3[1,1],ymm4[5,5],ymm3[5,5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4],ymm2[5,6,7]
-; AVX-NEXT:    vmovaps 48(%rcx), %xmm3
-; AVX-NEXT:    vmovaps 48(%rdx), %xmm4
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[3,3],xmm3[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; AVX-NEXT:    vmovaps 32(%rdi), %ymm4
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 32(%rsi), %ymm14
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm14[3,3],ymm4[3,3],ymm14[7,7],ymm4[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4],ymm3[5,6,7]
-; AVX-NEXT:    vmovaps 32(%r8), %ymm4
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0],ymm2[1,2,3,4],ymm4[5],ymm2[6,7]
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm4[2],ymm3[3,4,5,6],ymm4[7]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm4
+; AVX-NEXT:    vmovddup {{.*#+}} ymm4 = ymm4[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5,6],ymm4[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6,7]
+; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm8[2],xmm2[2],xmm8[3],xmm2[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2,3,4,5],ymm4[6],ymm0[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 64(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 64(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7]
 ; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm8[2],xmm2[3],xmm8[3]
-; AVX-NEXT:    vbroadcastss 68(%rcx), %xmm3
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = zero,zero,xmm0[2],xmm1[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3,4,5],ymm5[6],ymm0[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 96(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 96(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 96(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vmovaps 16(%rsi), %xmm5
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX-NEXT:    vmovaps 16(%rdi), %xmm4
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vmovaps 16(%rcx), %xmm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3,4,5],ymm3[6],ymm0[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm4[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm4[1],xmm5[1],zero
+; AVX-NEXT:    vmovaps 16(%rdx), %xmm3
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[1,1],ymm12[1,1],ymm13[5,5],ymm12[5,5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[3,3],xmm1[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3,4],ymm3[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 80(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 80(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm0
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[3,3],ymm11[3,3],ymm14[7,7],ymm11[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4,5,6],ymm3[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 64(%rdx), %ymm10
-; AVX-NEXT:    vmovaps 64(%rcx), %ymm9
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[1,1],ymm10[1,1],ymm9[5,5],ymm10[5,5]
+; AVX-NEXT:    vmovaps 32(%rdx), %ymm1
+; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 32(%rcx), %ymm13
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[1,1],ymm1[1,1],ymm13[5,5],ymm1[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
-; AVX-NEXT:    vmovaps 80(%rcx), %xmm0
-; AVX-NEXT:    vmovaps 80(%rdx), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[3,3],xmm0[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm1
-; AVX-NEXT:    vmovaps 64(%rdi), %ymm8
-; AVX-NEXT:    vmovaps 64(%rsi), %ymm5
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm5[3,3],ymm8[3,3],ymm5[7,7],ymm8[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm1[0,1,2],ymm4[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 64(%r8), %ymm6
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = ymm6[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm11[0],ymm3[1,2,3,4],ymm11[5],ymm3[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vmovaps 48(%rcx), %xmm1
+; AVX-NEXT:    vmovaps 48(%rdx), %xmm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm12
+; AVX-NEXT:    vmovaps 32(%rsi), %ymm11
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm11[3,3],ymm12[3,3],ymm11[7,7],ymm12[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm11[2],ymm4[3,4,5,6],ymm11[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
+; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vmovaps 80(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 80(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vmovaps 64(%rdx), %ymm9
+; AVX-NEXT:    vmovaps 64(%rcx), %ymm8
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[1,1],ymm9[1,1],ymm8[5,5],ymm9[5,5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vmovaps 80(%rcx), %xmm1
+; AVX-NEXT:    vmovaps 80(%rdx), %xmm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 64(%rdi), %ymm5
+; AVX-NEXT:    vmovaps 64(%rsi), %ymm4
+; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm4[3,3],ymm5[3,3],ymm4[7,7],ymm5[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm6[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm6[3,4],ymm1[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0],ymm0[1,2,3,4],ymm6[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm3 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 100(%rcx), %xmm4
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,zero,xmm12[2],xmm13[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm4, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0],ymm4[1],ymm3[2,3,4,5],ymm4[6],ymm3[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm6[2],ymm1[3,4,5,6],ymm6[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 112(%rsi), %xmm3
-; AVX-NEXT:    vmovaps 112(%rdi), %xmm4
-; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = zero,zero,xmm4[2],xmm3[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[1,1],xmm3[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm3, %ymm3
-; AVX-NEXT:    vmovaps 96(%rdx), %ymm12
-; AVX-NEXT:    vmovaps 96(%rcx), %ymm13
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm13[1,1],ymm12[1,1],ymm13[5,5],ymm12[5,5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm3[0,1,2],ymm4[3,4],ymm3[5,6,7]
-; AVX-NEXT:    vmovaps 112(%rcx), %xmm4
+; AVX-NEXT:    vmovaps 112(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 112(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm6 = xmm0[0,1,2],xmm6[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm6
+; AVX-NEXT:    vmovaps 96(%rdx), %ymm3
+; AVX-NEXT:    vmovaps 96(%rcx), %ymm2
+; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm2[1,1],ymm3[1,1],ymm2[5,5],ymm3[5,5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm7[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm7[3,4],ymm6[5,6,7]
+; AVX-NEXT:    vmovaps 112(%rcx), %xmm7
 ; AVX-NEXT:    vmovaps 112(%rdx), %xmm0
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[3,3],xmm4[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[3,3],xmm7[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm7[2],xmm0[3],xmm7[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps 96(%rdi), %ymm1
-; AVX-NEXT:    vmovaps 96(%rsi), %ymm4
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm4[3,3],ymm1[3,3],ymm4[7,7],ymm1[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = ymm11[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2],ymm11[3,4],ymm0[5,6,7]
-; AVX-NEXT:    vmovaps 96(%r8), %ymm2
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm2[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm3[0],ymm7[1,2,3,4],ymm3[5],ymm7[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm11[0,1],ymm3[2],ymm11[3,4,5,6],ymm3[7]
-; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm11 = mem[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm11 = ymm11[0,1,2,3],mem[4],ymm11[5,6,7]
-; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm0 = mem[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm15 = ymm15[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm15[0,1],ymm0[2],ymm15[3,4,5,6],ymm0[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm11[0],ymm0[1,2,3],ymm11[4,5],ymm0[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm14[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm11 = ymm11[0,1,2,3],mem[4],ymm11[5,6,7]
-; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm14 = mem[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm15 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2],ymm15[3,4,5,6],ymm14[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm14[1,2,3],ymm11[4,5],ymm14[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm11 = ymm11[0,1,2],mem[3],ymm11[4,5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4],ymm5[5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm9[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm9 = ymm10[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2],ymm9[3,4,5,6],ymm8[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm8[1,2,3],ymm5[4,5],ymm8[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm6[3],ymm5[4,5,6,7]
+; AVX-NEXT:    vmovaps 96(%rsi), %ymm7
+; AVX-NEXT:    vshufps {{.*#+}} ymm15 = ymm7[3,3],ymm1[3,3],ymm7[7,7],ymm1[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm15[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0,1,2],ymm15[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0],ymm6[1,2,3,4],ymm0[5],ymm6[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm15[0,1],ymm0[2],ymm15[3,4,5,6],ymm0[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm14[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4],ymm0[5,6,7]
+; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm15 = mem[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm14 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4,5,6],ymm15[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm14[1,2,3],ymm0[4,5],ymm14[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4],ymm11[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm13[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm13 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4,5,6],ymm12[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm12[1,2,3],ymm11[4,5],ymm12[6,7]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4],ymm4[5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm13[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm12[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1],ymm4[2],ymm6[3,4,5,6],ymm4[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm4[1,2,3],ymm1[4,5],ymm4[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4],ymm4[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm8 = ymm9[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm8[0,1],ymm5[2],ymm8[3,4,5,6],ymm5[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1,2,3],ymm4[4,5],ymm5[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm7[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4],ymm5[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm3[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4,5,6],ymm2[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4,5],ymm2[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm11[0,1,2],mem[3],ymm11[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],mem[3],ymm4[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],mem[3],ymm1[4,5,6,7]
 ; AVX-NEXT:    vmovaps %ymm1, 544(%r9)
-; AVX-NEXT:    vmovaps %ymm5, 384(%r9)
-; AVX-NEXT:    vmovaps %ymm11, 224(%r9)
+; AVX-NEXT:    vmovaps %ymm3, 384(%r9)
+; AVX-NEXT:    vmovaps %ymm2, 224(%r9)
 ; AVX-NEXT:    vmovaps %ymm0, 64(%r9)
-; AVX-NEXT:    vmovaps %ymm3, 608(%r9)
-; AVX-NEXT:    vmovaps %ymm7, 576(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 512(%r9)
+; AVX-NEXT:    vmovaps %ymm6, 608(%r9)
+; AVX-NEXT:    vmovaps %ymm10, 576(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 448(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 416(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 352(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 288(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 256(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 192(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 128(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 96(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 32(%r9)
+; AVX-NEXT:    vmovaps %ymm0, 512(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 480(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 352(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 320(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 192(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 160(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 32(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, (%r9)
-; AVX-NEXT:    addq $600, %rsp # imm = 0x258
+; AVX-NEXT:    addq $456, %rsp # imm = 0x1C8
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -5587,285 +5564,283 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride5_vf64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $1784, %rsp # imm = 0x6F8
-; AVX-NEXT:    vmovaps (%rsi), %xmm6
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm14
-; AVX-NEXT:    vmovaps (%rdi), %xmm5
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm15
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm5[1],xmm6[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = xmm5[0],xmm6[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vmovaps (%rdx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps (%rcx), %xmm11
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm11[0],xmm0[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 4(%rdx), %xmm7
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm4, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2,3],ymm2[4,5,6],ymm4[7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0],ymm2[1,2,3],ymm3[4],ymm2[5,6,7]
+; AVX-NEXT:    subq $1512, %rsp # imm = 0x5E8
+; AVX-NEXT:    vmovaps (%rsi), %xmm3
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm0
+; AVX-NEXT:    vmovaps (%rdi), %xmm4
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm4[1],xmm3[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm5
+; AVX-NEXT:    vmovaps (%rcx), %xmm6
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm2
+; AVX-NEXT:    vmovaps (%rdx), %xmm7
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm8
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm9, %ymm9
+; AVX-NEXT:    vmovddup {{.*#+}} ymm9 = ymm9[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm9[2,3],ymm5[4,5,6],ymm9[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm9[4],ymm5[5,6,7]
+; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3],ymm5[4,5],ymm3[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm9[1],ymm3[2,3,4,5],ymm9[6],ymm3[7]
+; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm4
+; AVX-NEXT:    vmovddup {{.*#+}} ymm4 = ymm4[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5,6],ymm4[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6,7]
+; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm8[2],xmm2[2],xmm8[3],xmm2[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2,3,4,5],ymm4[6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm15[1],xmm14[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = xmm15[0],xmm14[0],zero,zero
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 36(%rdx), %xmm7
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm4, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2,3],ymm2[4,5,6],ymm4[7]
-; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0],ymm2[1,2,3],ymm3[4],ymm2[5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm13
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm10
-; AVX-NEXT:    vmovaps 64(%rdx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 68(%rdx), %xmm3
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = zero,xmm10[1],xmm13[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = xmm10[0],xmm13[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5,6],ymm2[7]
-; AVX-NEXT:    vinsertf128 $1, 64(%r8), %ymm4, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0],ymm2[1,2,3],ymm3[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovaps 64(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 64(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3,4,5],ymm5[6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 100(%rdx), %xmm3
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm3
 ; AVX-NEXT:    vmovaps 96(%rsi), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm7
-; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,xmm7[1],xmm0[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = xmm7[0],xmm0[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm9, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3],ymm4[4,5,6],ymm3[7]
-; AVX-NEXT:    vinsertf128 $1, 96(%r8), %ymm9, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0],ymm3[1,2,3],ymm4[4],ymm3[5,6,7]
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 96(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 96(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3,4,5],ymm5[6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rdx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 128(%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 132(%rdx), %xmm4
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm4
 ; AVX-NEXT:    vmovaps 128(%rsi), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 128(%rdi), %xmm8
-; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = zero,xmm8[1],xmm0[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = xmm8[0],xmm0[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm12, %ymm9
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm9[0,1],ymm4[2,3],ymm9[4,5,6],ymm4[7]
-; AVX-NEXT:    vinsertf128 $1, 128(%r8), %ymm12, %ymm9
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0],ymm4[1,2,3],ymm9[4],ymm4[5,6,7]
+; AVX-NEXT:    vmovaps 128(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 128(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 128(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3,4,5],ymm5[6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rdx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 160(%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 164(%rdx), %xmm9
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm4, %ymm9
 ; AVX-NEXT:    vmovaps 160(%rsi), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 160(%rdi), %xmm4
-; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = zero,xmm4[1],xmm0[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm4[0],xmm0[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm12[0,1],ymm9[2,3],ymm12[4,5,6],ymm9[7]
-; AVX-NEXT:    vinsertf128 $1, 160(%r8), %ymm0, %ymm0
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm9[1,2,3],ymm0[4],ymm9[5,6,7]
+; AVX-NEXT:    vmovaps 160(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 160(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 160(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3,4,5],ymm5[6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 192(%rdx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 192(%rcx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 196(%rdx), %xmm9
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm1
 ; AVX-NEXT:    vmovaps 192(%rsi), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 192(%rdi), %xmm9
-; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = zero,xmm9[1],xmm0[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm9[0],xmm0[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm12[0,1],ymm1[2,3],ymm12[4,5,6],ymm1[7]
-; AVX-NEXT:    vinsertf128 $1, 192(%r8), %ymm0, %ymm0
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 224(%rdx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 224(%rcx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,2,0]
-; AVX-NEXT:    vbroadcastss 228(%rdx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
-; AVX-NEXT:    vmovaps 224(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 224(%rdi), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm0[1],xmm1[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = xmm0[0],xmm1[0],zero,zero
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm12, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm3[2,3],ymm1[4,5,6],ymm3[7]
-; AVX-NEXT:    vinsertf128 $1, 224(%r8), %ymm12, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]
+; AVX-NEXT:    vmovaps 192(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 192(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 192(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm1[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm11[2],xmm0[3],xmm11[3]
-; AVX-NEXT:    vbroadcastss 4(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = zero,zero,xmm5[2],xmm6[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 224(%rsi), %xmm0
+; AVX-NEXT:    vmovaps 224(%rdi), %xmm1
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX-NEXT:    vmovaps 224(%rcx), %xmm3
+; AVX-NEXT:    vmovaps 224(%rdx), %xmm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm5
+; AVX-NEXT:    vmovddup {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3],ymm2[4,5,6],ymm5[7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
 ; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm1[4],ymm2[5,6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 16(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps 16(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps (%rdx), %ymm11
-; AVX-NEXT:    vmovaps (%rcx), %ymm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[1],xmm0[1],zero
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vmovaps (%rdx), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1],ymm11[1,1],ymm1[5,5],ymm11[5,5]
+; AVX-NEXT:    vmovaps (%rcx), %ymm13
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[1,1],ymm1[1,1],ymm13[5,5],ymm1[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
 ; AVX-NEXT:    vmovaps 16(%rcx), %xmm1
-; AVX-NEXT:    vmovaps 16(%rdx), %xmm5
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[3,3],xmm1[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX-NEXT:    vmovaps (%rdi), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rsi), %ymm3
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm3[3,3],ymm2[3,3],ymm3[7,7],ymm2[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm5[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps (%r8), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm2[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0],ymm0[1,2,3,4],ymm5[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm5[2],ymm1[3,4,5,6],ymm5[7]
+; AVX-NEXT:    vmovaps 16(%rdx), %xmm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps (%rdi), %ymm14
+; AVX-NEXT:    vmovaps (%rsi), %ymm15
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[3,3],ymm14[3,3],ymm15[7,7],ymm14[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 36(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = zero,zero,xmm15[2],xmm14[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 48(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 48(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 32(%rdx), %ymm14
-; AVX-NEXT:    vmovaps 32(%rcx), %ymm15
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[1,1],ymm14[1,1],ymm15[5,5],ymm14[5,5]
+; AVX-NEXT:    vmovaps 48(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 48(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vmovaps 32(%rdx), %ymm1
+; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 32(%rcx), %ymm2
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[1,1],ymm1[1,1],ymm2[5,5],ymm1[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
 ; AVX-NEXT:    vmovaps 48(%rcx), %xmm1
-; AVX-NEXT:    vmovaps 48(%rdx), %xmm5
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[3,3],xmm1[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 48(%rdx), %xmm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX-NEXT:    vmovaps 32(%rdi), %ymm2
 ; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 32(%rsi), %ymm3
 ; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm3[3,3],ymm2[3,3],ymm3[7,7],ymm2[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm5[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 32(%r8), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm2[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0],ymm0[1,2,3,4],ymm5[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm5[2],ymm1[3,4,5,6],ymm5[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[3,3],ymm2[3,3],ymm3[7,7],ymm2[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 68(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = zero,zero,xmm10[2],xmm13[2]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 80(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 80(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 64(%rdx), %ymm13
-; AVX-NEXT:    vmovaps 64(%rcx), %ymm1
+; AVX-NEXT:    vmovaps 80(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 80(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vmovaps 64(%rdx), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1],ymm13[1,1],ymm1[5,5],ymm13[5,5]
+; AVX-NEXT:    vmovaps 64(%rcx), %ymm2
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[1,1],ymm1[1,1],ymm2[5,5],ymm1[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
 ; AVX-NEXT:    vmovaps 80(%rcx), %xmm1
-; AVX-NEXT:    vmovaps 80(%rdx), %xmm5
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm5[3,3],xmm1[3,3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
+; AVX-NEXT:    vmovaps 80(%rdx), %xmm2
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX-NEXT:    vmovaps 64(%rdi), %ymm2
 ; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 64(%rsi), %ymm3
 ; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm3[3,3],ymm2[3,3],ymm3[7,7],ymm2[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm5[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 64(%r8), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm2[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0],ymm0[1,2,3,4],ymm5[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm5[2],ymm1[3,4,5,6],ymm5[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[3,3],ymm2[3,3],ymm3[7,7],ymm2[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 100(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = zero,zero,xmm7[2],mem[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 112(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 112(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
+; AVX-NEXT:    vmovaps 112(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 112(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps 96(%rdx), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -5876,9 +5851,9 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
 ; AVX-NEXT:    vmovaps 112(%rcx), %xmm1
 ; AVX-NEXT:    vmovaps 112(%rdx), %xmm2
-; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm2[3,3],xmm1[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX-NEXT:    vmovaps 96(%rdi), %ymm3
 ; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 96(%rsi), %ymm2
@@ -5886,29 +5861,16 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,3],ymm3[3,3],ymm2[7,7],ymm3[7,7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 96(%r8), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 132(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = zero,zero,xmm8[2],mem[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 144(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 144(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
+; AVX-NEXT:    vmovaps 144(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 144(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps 128(%rdx), %ymm2
 ; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -5922,42 +5884,28 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX-NEXT:    vmovaps 128(%rdi), %ymm3
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 128(%rdi), %ymm12
 ; AVX-NEXT:    vmovaps 128(%rsi), %ymm2
 ; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,3],ymm3[3,3],ymm2[7,7],ymm3[7,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,3],ymm12[3,3],ymm2[7,7],ymm12[7,7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 128(%r8), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 164(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = zero,zero,xmm4[2],mem[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 176(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 176(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
+; AVX-NEXT:    vmovaps 176(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 176(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 160(%rdx), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rcx), %ymm1
+; AVX-NEXT:    vmovaps 160(%rdx), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1],ymm2[1,1],ymm1[5,5],ymm2[5,5]
+; AVX-NEXT:    vmovaps 160(%rcx), %ymm2
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[1,1],ymm1[1,1],ymm2[5,5],ymm1[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
 ; AVX-NEXT:    vmovaps 176(%rcx), %xmm1
@@ -5965,41 +5913,26 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX-NEXT:    vmovaps 160(%rdi), %ymm12
-; AVX-NEXT:    vmovaps 160(%rsi), %ymm2
+; AVX-NEXT:    vmovaps 160(%rdi), %ymm2
 ; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,3],ymm12[3,3],ymm2[7,7],ymm12[7,7]
+; AVX-NEXT:    vmovaps 160(%rsi), %ymm11
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm11[3,3],ymm2[3,3],ymm11[7,7],ymm2[7,7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 160(%r8), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 196(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = zero,zero,xmm9[2],mem[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 208(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 208(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
+; AVX-NEXT:    vmovaps 208(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 208(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 192(%rdx), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 192(%rcx), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[1,1],ymm1[1,1],ymm2[5,5],ymm1[5,5]
+; AVX-NEXT:    vmovaps 192(%rdx), %ymm10
+; AVX-NEXT:    vmovaps 192(%rcx), %ymm9
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[1,1],ymm10[1,1],ymm9[5,5],ymm10[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
 ; AVX-NEXT:    vmovaps 208(%rcx), %xmm1
@@ -6007,100 +5940,69 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[3,3],xmm1[3,3]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX-NEXT:    vmovaps 192(%rdi), %ymm8
-; AVX-NEXT:    vmovaps 192(%rsi), %ymm9
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[3,3],ymm8[3,3],ymm9[7,7],ymm8[7,7]
+; AVX-NEXT:    vmovaps 192(%rdi), %ymm7
+; AVX-NEXT:    vmovaps 192(%rsi), %ymm5
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm5[3,3],ymm7[3,3],ymm5[7,7],ymm7[7,7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 192(%r8), %ymm10
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm10[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4],ymm2[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6],ymm2[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vbroadcastss 228(%rcx), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX-NEXT:    vinsertps $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = zero,zero,xmm2[2],mem[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 240(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 240(%rdi), %xmm1
-; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = zero,zero,xmm1[2],xmm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,1],xmm0[1,1]
+; AVX-NEXT:    vmovaps 240(%rdi), %xmm0
+; AVX-NEXT:    vmovaps 240(%rsi), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],xmm1[1],zero
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 224(%rdx), %ymm6
-; AVX-NEXT:    vmovaps 224(%rcx), %ymm4
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[1,1],ymm6[1,1],ymm4[5,5],ymm6[5,5]
+; AVX-NEXT:    vmovaps 224(%rdx), %ymm4
+; AVX-NEXT:    vmovaps 224(%rcx), %ymm3
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[1,1],ymm4[1,1],ymm3[5,5],ymm4[5,5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2],ymm1[3,4],ymm0[5,6,7]
 ; AVX-NEXT:    vmovaps 240(%rcx), %xmm0
 ; AVX-NEXT:    vmovaps 240(%rdx), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[3,3],xmm0[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[3,3],xmm0[3,3]
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm1
-; AVX-NEXT:    vmovaps 224(%rdi), %ymm2
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm6
+; AVX-NEXT:    vmovaps 224(%rdi), %ymm1
 ; AVX-NEXT:    vmovaps 224(%rsi), %ymm0
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm0[3,3],ymm2[3,3],ymm0[7,7],ymm2[7,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm1[0,1,2],ymm5[3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps 224(%r8), %ymm1
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm1[2,3,2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm7[0],ymm3[1,2,3,4],ymm7[5],ymm3[6,7]
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm7[2],ymm5[3,4,5,6],ymm7[7]
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = mem[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = ymm3[0,1,2,3],mem[4],ymm3[5,6,7]
-; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm5 = mem[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm7 = ymm11[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1],ymm5[2],ymm7[3,4,5,6],ymm5[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1,2,3],ymm3[4,5],ymm5[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm11 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm11 = ymm3[0,1,2],mem[3],ymm3[4,5,6,7]
-; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = mem[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = ymm3[0,1,2,3],mem[4],ymm3[5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm15[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm7 = ymm14[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1],ymm5[2],ymm7[3,4,5,6],ymm5[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1,2,3],ymm3[4,5],ymm5[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm7 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm7 = ymm3[0,1,2],mem[3],ymm3[4,5,6,7]
-; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = mem[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = ymm3[0,1,2,3],mem[4],ymm3[5,6,7]
-; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm5 = mem[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm15 = ymm13[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm15[0,1],ymm5[2],ymm15[3,4,5,6],ymm5[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1,2,3],ymm3[4,5],ymm5[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm5 = ymm3[0,1,2],mem[3],ymm3[4,5,6,7]
-; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = mem[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = ymm3[0,1,2,3],mem[4],ymm3[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm0[3,3],ymm1[3,3],ymm0[7,7],ymm1[7,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm8[2,3,2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm8[3,4],ymm6[5,6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm8[0],ymm2[1,2,3,4],ymm8[5],ymm2[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1],ymm8[2],ymm6[3,4,5,6],ymm8[7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4],ymm2[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm13[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm8 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1],ymm6[2],ymm8[3,4,5,6],ymm6[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm2[0],ymm6[1,2,3],ymm2[4,5],ymm6[6,7]
+; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm2 = mem[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm2 = ymm2[0,1,2,3],mem[4],ymm2[5,6,7]
+; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm6 = mem[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm15 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm15[0,1],ymm6[2],ymm15[3,4,5,6],ymm6[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm2[0],ymm6[1,2,3],ymm2[4,5],ymm6[6,7]
+; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm2 = mem[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm2 = ymm2[0,1,2,3],mem[4],ymm2[5,6,7]
 ; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
 ; AVX-NEXT:    # ymm15 = mem[0,1,3,0,4,5,7,4]
 ; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX-NEXT:    # ymm14 = mem[1,0,2,2]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4,5,6],ymm15[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm14[1,2,3],ymm3[4,5],ymm14[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = ymm3[0,1,2],mem[3],ymm3[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm14[1,2,3],ymm2[4,5],ymm14[6,7]
 ; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX-NEXT:    # ymm14 = mem[3,0,2,3,7,4,6,7]
 ; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
@@ -6111,8 +6013,6 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    # ymm13 = mem[1,0,2,2]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1],ymm15[2],ymm13[3,4,5,6],ymm15[7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0],ymm13[1,2,3],ymm14[4,5],ymm13[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm13 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm13 = ymm13[0,1,2],mem[3],ymm13[4,5,6,7]
 ; AVX-NEXT:    vpermilps $227, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX-NEXT:    # ymm14 = mem[3,0,2,3,7,4,6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm12[4],ymm14[5,6,7]
@@ -6122,97 +6022,108 @@ define void @store_i32_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    # ymm12 = mem[1,0,2,2]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1],ymm15[2],ymm12[3,4,5,6],ymm15[7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0],ymm12[1,2,3],ymm14[4,5],ymm12[6,7]
-; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm12 = ymm12[0,1,2],mem[3],ymm12[4,5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4],ymm9[5,6,7]
-; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm9 = mem[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm14 = mem[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm14[0,1],ymm9[2],ymm14[3,4,5,6],ymm9[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0],ymm9[1,2,3],ymm8[4,5],ymm9[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm10[3],ymm8[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm11 = ymm11[0,1,2,3],mem[4],ymm11[5,6,7]
+; AVX-NEXT:    vpermilps $52, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm14 = mem[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm15 = mem[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2],ymm15[3,4,5,6],ymm14[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm14[1,2,3],ymm11[4,5],ymm14[6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[3,0,2,3,7,4,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4],ymm5[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm9[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm9 = ymm10[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1],ymm7[2],ymm9[3,4,5,6],ymm7[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm7[1,2,3],ymm5[4,5],ymm7[6,7]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[3,0,2,3,7,4,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4],ymm0[5,6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,1,3,0,4,5,7,4]
-; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm6[1,0,2,2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3,4,5,6],ymm2[7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4,5],ymm2[6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,1,3,0,4,5,7,4]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm4[1,0,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1,2],mem[3],ymm8[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1,2],mem[3],ymm6[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],mem[3],ymm2[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm13[0,1,2],mem[3],ymm13[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm12[0,1,2],mem[3],ymm12[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm11[0,1,2],mem[3],ymm11[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],mem[3],ymm5[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovaps %ymm0, 1184(%r9)
-; AVX-NEXT:    vmovaps %ymm8, 1024(%r9)
-; AVX-NEXT:    vmovaps %ymm12, 864(%r9)
-; AVX-NEXT:    vmovaps %ymm13, 704(%r9)
-; AVX-NEXT:    vmovaps %ymm3, 544(%r9)
-; AVX-NEXT:    vmovaps %ymm5, 384(%r9)
-; AVX-NEXT:    vmovaps %ymm7, 224(%r9)
-; AVX-NEXT:    vmovaps %ymm11, 64(%r9)
+; AVX-NEXT:    vmovaps %ymm5, 1024(%r9)
+; AVX-NEXT:    vmovaps %ymm7, 864(%r9)
+; AVX-NEXT:    vmovaps %ymm6, 704(%r9)
+; AVX-NEXT:    vmovaps %ymm4, 544(%r9)
+; AVX-NEXT:    vmovaps %ymm2, 384(%r9)
+; AVX-NEXT:    vmovaps %ymm3, 224(%r9)
+; AVX-NEXT:    vmovaps %ymm1, 64(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 1248(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 1216(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 1152(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 1088(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 1056(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 992(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 928(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 896(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 832(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 768(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 736(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 672(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 608(%r9)
-; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 576(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 512(%r9)
+; AVX-NEXT:    vmovaps %ymm0, 576(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 448(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 416(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 352(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 288(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 256(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 192(%r9)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 128(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 96(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 32(%r9)
+; AVX-NEXT:    vmovaps %ymm0, 1152(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 1120(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 992(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 960(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 832(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 800(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 672(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 640(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 512(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 480(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 352(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 320(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 192(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 160(%r9)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 32(%r9)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, (%r9)
-; AVX-NEXT:    addq $1784, %rsp # imm = 0x6F8
+; AVX-NEXT:    addq $1512, %rsp # imm = 0x5E8
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
index 7cac6e8e1a5f..2c60eb31e204 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
@@ -372,22 +372,22 @@ define void @store_i32_stride6_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm8[3,3],ymm7[3,3],ymm8[7,7],ymm7[7,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6],ymm6[7]
 ; AVX-NEXT:    vpermilps {{.*#+}} ymm6 = ymm6[0,2,1,3,4,6,7,5]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,3]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm8 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm8, %ymm7
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm5 = xmm4[0,1],xmm7[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm7 = xmm7[0],xmm8[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm7, %ymm5
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = zero,zero,xmm2[2],xmm3[2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[1],xmm3[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovaps %ymm0, 32(%rax)
-; AVX-NEXT:    vmovaps %ymm7, (%rax)
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
+; AVX-NEXT:    vmovaps %xmm1, 32(%rax)
+; AVX-NEXT:    vmovaps %xmm0, 48(%rax)
+; AVX-NEXT:    vmovaps %ymm5, (%rax)
 ; AVX-NEXT:    vmovaps %ymm6, 64(%rax)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -764,77 +764,76 @@ define void @store_i32_stride6_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX-LABEL: store_i32_stride6_vf8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT:    vmovaps (%rdi), %ymm7
-; AVX-NEXT:    vmovaps (%rsi), %ymm8
-; AVX-NEXT:    vmovaps (%rdx), %ymm2
-; AVX-NEXT:    vmovaps (%rcx), %ymm3
-; AVX-NEXT:    vmovaps (%r8), %ymm1
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[4],ymm8[4],ymm7[5],ymm8[5]
+; AVX-NEXT:    vmovaps (%rdi), %ymm3
+; AVX-NEXT:    vmovaps (%rsi), %ymm4
+; AVX-NEXT:    vmovaps (%rdx), %ymm1
+; AVX-NEXT:    vmovaps (%rcx), %ymm2
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2]
-; AVX-NEXT:    vextractf128 $1, %ymm4, %xmm4
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 16(%r9), %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
-; AVX-NEXT:    vmovaps (%rcx), %xmm9
-; AVX-NEXT:    vmovaps (%rdx), %xmm10
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm10[1,2],xmm9[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT:    vmovaps (%rsi), %xmm5
-; AVX-NEXT:    vmovaps (%rdi), %xmm6
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm0, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm12[4,5],ymm4[6,7]
-; AVX-NEXT:    vbroadcastss 4(%r8), %xmm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm12[2,3],ymm4[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 4(%r9), %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm12[3],ymm4[4,5,6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm8 = ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[6],ymm8[6],ymm7[7],ymm8[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm2[1,2],ymm3[1,2],ymm2[5,6],ymm3[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm7[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5],ymm7[6,7]
-; AVX-NEXT:    vbroadcastss 20(%r8), %xmm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1],ymm12[2,3],ymm7[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 20(%r9), %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm12[3],ymm7[4,5,6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm9 = xmm10[2],xmm9[2],xmm10[3],xmm9[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[2,3,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm11, %ymm9
-; AVX-NEXT:    vpermilps {{.*#+}} xmm10 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm10, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,3,4,5],ymm10[6,7]
-; AVX-NEXT:    vmovaps (%r9), %xmm10
-; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm10[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm10[1],ymm9[2,3,4,5,6],ymm10[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[3,0],ymm2[3,0],ymm3[7,4],ymm2[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3,4,5],ymm1[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = mem[2,3,2,3]
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm5 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
+; AVX-NEXT:    vextractf128 $1, %ymm5, %xmm5
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm5[2,3],ymm0[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],mem[4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastss 16(%r9), %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5],ymm0[6,7]
+; AVX-NEXT:    vmovaps (%rcx), %xmm6
+; AVX-NEXT:    vmovaps (%rdx), %xmm7
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm7[1,2],xmm6[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm5, %ymm5
+; AVX-NEXT:    vmovaps (%rsi), %xmm8
+; AVX-NEXT:    vmovaps (%rdi), %xmm9
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm0, %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm11[4,5],ymm5[6,7]
+; AVX-NEXT:    vbroadcastss 4(%r8), %xmm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm11[2,3],ymm5[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 4(%r9), %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm11[3],ymm5[4,5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm10, %ymm6
+; AVX-NEXT:    vpermilps {{.*#+}} xmm7 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm7, %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3,4,5],ymm7[6,7]
+; AVX-NEXT:    vmovaps (%r9), %xmm7
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm7[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm10, %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3,4,5,6],ymm7[7]
+; AVX-NEXT:    vbroadcastss (%rcx), %xmm7
+; AVX-NEXT:    vbroadcastss (%rdx), %xmm10
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1],ymm7[2,3],ymm9[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm8, %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
+; AVX-NEXT:    vbroadcastss (%r9), %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm8[5],ymm7[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} ymm3 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm1[1,2],ymm2[1,2],ymm1[5,6],ymm2[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5],ymm4[6,7]
+; AVX-NEXT:    vbroadcastss 20(%r8), %xmm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm8[2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 20(%r9), %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm8[3],ymm4[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[3,0],ymm1[3,0],ymm2[7,4],ymm1[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5],ymm2[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2,2,3,4,6,6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4,5,6],ymm2[7]
-; AVX-NEXT:    vbroadcastss (%rcx), %xmm2
-; AVX-NEXT:    vbroadcastss (%rdx), %xmm3
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm5
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
-; AVX-NEXT:    vbroadcastss (%r9), %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
-; AVX-NEXT:    vmovaps %ymm2, (%rax)
 ; AVX-NEXT:    vmovaps %ymm1, 160(%rax)
-; AVX-NEXT:    vmovaps %ymm9, 64(%rax)
-; AVX-NEXT:    vmovaps %ymm7, 128(%rax)
-; AVX-NEXT:    vmovaps %ymm4, 32(%rax)
+; AVX-NEXT:    vmovaps %ymm4, 128(%rax)
+; AVX-NEXT:    vmovaps %ymm7, (%rax)
+; AVX-NEXT:    vmovaps %ymm6, 64(%rax)
+; AVX-NEXT:    vmovaps %ymm5, 32(%rax)
 ; AVX-NEXT:    vmovaps %ymm0, 96(%rax)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -1477,171 +1476,165 @@ define void @store_i32_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride6_vf16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $104, %rsp
-; AVX-NEXT:    vmovaps 32(%rdi), %ymm5
+; AVX-NEXT:    subq $40, %rsp
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm11
 ; AVX-NEXT:    vmovaps 32(%rsi), %ymm13
-; AVX-NEXT:    vmovaps 32(%rdx), %ymm7
+; AVX-NEXT:    vmovaps 32(%rdx), %ymm8
 ; AVX-NEXT:    vmovaps 32(%rcx), %ymm9
-; AVX-NEXT:    vmovaps 32(%r8), %ymm11
-; AVX-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rcx), %xmm8
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps (%rdx), %xmm6
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm10
-; AVX-NEXT:    vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm6[1,2],xmm8[1,2]
+; AVX-NEXT:    vmovaps (%rcx), %xmm4
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm5
+; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps (%rdx), %xmm15
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm6
+; AVX-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm15[1,2],xmm4[1,2]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps (%rsi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps (%rdi), %xmm2
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm4 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm1
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
 ; AVX-NEXT:    vbroadcastss 4(%r8), %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vbroadcastss 4(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm5[0],ymm13[0],ymm5[1],ymm13[1],ymm5[4],ymm13[4],ymm5[5],ymm13[5]
+; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[4],ymm13[4],ymm11[5],ymm13[5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm9[0],ymm7[0],ymm9[2],ymm7[2]
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm9[0],ymm8[0],ymm9[2],ymm8[2]
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],mem[4,5],ymm0[6,7]
 ; AVX-NEXT:    vbroadcastss 48(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm10[1,2],xmm3[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm6[1,2],xmm5[1,2]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm3
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm2
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm15 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm0, %ymm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 36(%r8), %xmm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm11[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 36(%r9), %ymm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm11[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rdi), %ymm1
-; AVX-NEXT:    vmovaps (%rsi), %ymm0
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm11 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm11[2,3,2,3]
-; AVX-NEXT:    vmovaps (%rdx), %ymm11
-; AVX-NEXT:    vmovaps (%rcx), %ymm12
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm10 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
-; AVX-NEXT:    vextractf128 $1, %ymm10, %xmm10
-; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm14[0,1],ymm10[2,3],ymm14[4,5,6,7]
-; AVX-NEXT:    vmovaps (%r8), %ymm14
-; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm14[4,5],ymm10[6,7]
-; AVX-NEXT:    vbroadcastss 16(%r9), %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm10[0,1,2,3,4],ymm14[5],ymm10[6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm13 = ymm5[2],ymm13[2],ymm5[3],ymm13[3],ymm5[6],ymm13[6],ymm5[7],ymm13[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm7[1,2],ymm9[1,2],ymm7[5,6],ymm9[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm13[4,5],ymm5[6,7]
-; AVX-NEXT:    vbroadcastss 52(%r8), %xmm10
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm5
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm2
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm1
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm10
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm10[4,5],ymm5[6,7]
+; AVX-NEXT:    vbroadcastss 36(%r8), %xmm10
 ; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm10[2,3],ymm5[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 52(%r9), %ymm10
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm5[0,1,2],ymm10[3],ymm5[4,5,6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm5 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm11[1,2],ymm12[1,2],ymm11[5,6],ymm12[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 20(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 20(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm6[2],xmm8[2],xmm6[3],xmm8[3]
+; AVX-NEXT:    vbroadcastss 36(%r9), %ymm10
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0,1,2],ymm10[3],ymm5[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rdi), %ymm14
+; AVX-NEXT:    vmovaps (%rsi), %ymm5
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm10 = ymm14[0],ymm5[0],ymm14[1],ymm5[1],ymm14[4],ymm5[4],ymm14[5],ymm5[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm10[2,3,2,3]
+; AVX-NEXT:    vmovaps (%rdx), %ymm10
+; AVX-NEXT:    vmovaps (%rcx), %ymm12
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm7 = ymm12[0],ymm10[0],ymm12[2],ymm10[2]
+; AVX-NEXT:    vextractf128 $1, %ymm7, %xmm7
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm7[2,3],ymm0[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],mem[4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastss 16(%r9), %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm0[0,1,2,3,4],ymm7[5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm15[2],xmm4[2],xmm15[3],xmm4[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm4 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3,4,5],ymm4[6,7]
-; AVX-NEXT:    vmovaps (%r9), %xmm4
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm4[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm0[0],ymm4[1],ymm0[2,3,4,5,6],ymm4[7]
-; AVX-NEXT:    vbroadcastss 32(%rcx), %xmm0
-; AVX-NEXT:    vbroadcastss 32(%rdx), %xmm6
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm2, %ymm2
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 32(%r9), %ymm2
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm9[3,0],ymm7[3,0],ymm9[7,4],ymm7[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm13[2,3],ymm0[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3,4,5],ymm3[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vmovaps (%r9), %xmm3
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0],ymm3[1],ymm0[2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vbroadcastss 32(%rcx), %xmm0
+; AVX-NEXT:    vbroadcastss 32(%rdx), %xmm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm1, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastss 32(%r9), %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm11[2],ymm13[2],ymm11[3],ymm13[3],ymm11[6],ymm13[6],ymm11[7],ymm13[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[1,2],ymm9[1,2],ymm8[5,6],ymm9[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 52(%r8), %xmm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 52(%r9), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm1[0,1,2],ymm3[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm6, %ymm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 32(%r9), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
+; AVX-NEXT:    vbroadcastss (%rcx), %xmm1
+; AVX-NEXT:    vbroadcastss (%rdx), %xmm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm15, %ymm3
-; AVX-NEXT:    vpermilps {{.*#+}} xmm6 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm6, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1],ymm3[2,3,4,5],ymm6[6,7]
-; AVX-NEXT:    vmovaps 32(%r9), %xmm6
-; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm6[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm6[1],ymm3[2,3,4,5,6],ymm6[7]
-; AVX-NEXT:    vbroadcastss (%rcx), %xmm6
-; AVX-NEXT:    vbroadcastss (%rdx), %xmm7
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm7 = xmm7[0],mem[0],xmm7[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm8
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,3],ymm8[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm7, %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3],ymm7[4,5],ymm6[6,7]
-; AVX-NEXT:    vbroadcastss (%r9), %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm7[5],ymm6[6,7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm12[3,0],ymm11[3,0],ymm12[7,4],ymm11[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3],ymm7[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm7 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3,4,5],ymm7[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm7[1],ymm5[2,3,4,5,6],ymm7[7]
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm1[2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm3, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3],ymm3[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss (%r9), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5],ymm1[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} ymm3 = ymm14[2],ymm5[2],ymm14[3],ymm5[3],ymm14[6],ymm5[6],ymm14[7],ymm5[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm10[1,2],ymm12[1,2],ymm10[5,6],ymm12[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5],ymm4[6,7]
+; AVX-NEXT:    vbroadcastss 20(%r8), %xmm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 20(%r9), %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3],ymm4[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm12[3,0],ymm10[3,0],ymm12[7,4],ymm10[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3],ymm5[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5],ymm5[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2,3,4,5,6],ymm5[7]
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT:    vmovaps %ymm5, 160(%rax)
-; AVX-NEXT:    vmovaps %ymm6, (%rax)
-; AVX-NEXT:    vmovaps %ymm3, 256(%rax)
-; AVX-NEXT:    vmovaps %ymm0, 352(%rax)
+; AVX-NEXT:    vmovaps %ymm3, 160(%rax)
+; AVX-NEXT:    vmovaps %ymm4, 128(%rax)
+; AVX-NEXT:    vmovaps %ymm1, (%rax)
+; AVX-NEXT:    vmovaps %ymm0, 256(%rax)
+; AVX-NEXT:    vmovaps %ymm8, 352(%rax)
+; AVX-NEXT:    vmovaps %ymm11, 320(%rax)
 ; AVX-NEXT:    vmovaps %ymm2, 192(%rax)
-; AVX-NEXT:    vmovaps %ymm4, 64(%rax)
-; AVX-NEXT:    vmovaps %ymm1, 128(%rax)
-; AVX-NEXT:    vmovaps %ymm10, 320(%rax)
-; AVX-NEXT:    vmovaps %ymm14, 96(%rax)
-; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm15, 64(%rax)
+; AVX-NEXT:    vmovaps %ymm7, 96(%rax)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 224(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 288(%rax)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 32(%rax)
-; AVX-NEXT:    addq $104, %rsp
+; AVX-NEXT:    addq $40, %rsp
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -3224,57 +3217,50 @@ define void @store_i32_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride6_vf32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $1032, %rsp # imm = 0x408
-; AVX-NEXT:    vmovaps (%rdi), %ymm12
-; AVX-NEXT:    vmovaps (%rsi), %ymm8
-; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rdx), %ymm4
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rcx), %ymm6
-; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%r8), %ymm5
-; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    subq $712, %rsp # imm = 0x2C8
+; AVX-NEXT:    vmovaps (%rdi), %ymm7
+; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rsi), %ymm10
+; AVX-NEXT:    vmovups %ymm10, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rdx), %ymm8
+; AVX-NEXT:    vmovaps (%rcx), %ymm4
+; AVX-NEXT:    vmovaps (%rcx), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 32(%rcx), %xmm2
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps (%rdx), %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
+; AVX-NEXT:    vmovaps (%rdx), %xmm14
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm6
+; AVX-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm14[1,2],xmm0[1,2]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps (%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps (%rdi), %xmm7
-; AVX-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX-NEXT:    vmovaps (%rsi), %xmm5
+; AVX-NEXT:    vmovaps (%rdi), %xmm3
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm9 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
 ; AVX-NEXT:    vbroadcastss 4(%r8), %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vbroadcastss 4(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm12[0],ymm8[0],ymm12[1],ymm8[1],ymm12[4],ymm8[4],ymm12[5],ymm8[5]
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm7[0],ymm10[0],ymm7[1],ymm10[1],ymm7[4],ymm10[4],ymm7[5],ymm10[5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm4[0],ymm8[0],ymm4[2],ymm8[2]
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],mem[4,5],ymm0[6,7]
 ; AVX-NEXT:    vbroadcastss 16(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[1,2],xmm2[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm6[1,2],xmm2[1,2]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps 32(%rsi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 32(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
@@ -3289,315 +3275,290 @@ define void @store_i32_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vmovaps 32(%rsi), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 32(%rdx), %ymm8
-; AVX-NEXT:    vmovaps 32(%rcx), %ymm13
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm13[0],ymm8[0],ymm13[2],ymm8[2]
-; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 32(%r8), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 48(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm0[2,3,2,3]
+; AVX-NEXT:    vmovaps 32(%rdx), %ymm2
+; AVX-NEXT:    vmovaps 32(%rcx), %ymm1
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm7 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
+; AVX-NEXT:    vextractf128 $1, %ymm7, %xmm7
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm7[2,3],ymm6[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],mem[4,5],ymm6[6,7]
+; AVX-NEXT:    vbroadcastss 48(%r9), %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3,4],ymm7[5],ymm6[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdx), %xmm0
+; AVX-NEXT:    vmovaps 64(%rcx), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 68(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 68(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vmovaps 64(%rdx), %xmm6
+; AVX-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[1,2],xmm0[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm6, %ymm6
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm7
+; AVX-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm7[2],xmm0[2],xmm7[3],xmm0[3]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdi), %ymm6
-; AVX-NEXT:    vmovaps 64(%rsi), %ymm14
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm6[0],ymm14[0],ymm6[1],ymm14[1],ymm6[4],ymm14[4],ymm6[5],ymm14[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 64(%rdx), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rcx), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 64(%r8), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 80(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5],ymm6[6,7]
+; AVX-NEXT:    vbroadcastss 68(%r8), %xmm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm7[2,3],ymm6[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 68(%r9), %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2],ymm7[3],ymm6[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rcx), %xmm9
-; AVX-NEXT:    vmovaps 96(%rdx), %xmm11
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm11[1,2],xmm9[1,2]
-; AVX-NEXT:    vmovaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm2
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm5
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm4
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX-NEXT:    vmovaps 64(%rdi), %ymm0
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 64(%rsi), %ymm6
+; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm6 = ymm0[0],ymm6[0],ymm0[1],ymm6[1],ymm0[4],ymm6[4],ymm0[5],ymm6[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = ymm6[2,3,2,3]
+; AVX-NEXT:    vmovaps 64(%rdx), %ymm7
+; AVX-NEXT:    vmovaps 64(%rcx), %ymm6
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm11 = ymm6[0],ymm7[0],ymm6[2],ymm7[2]
+; AVX-NEXT:    vextractf128 $1, %ymm11, %xmm11
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm11[2,3],ymm10[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3],mem[4,5],ymm10[6,7]
+; AVX-NEXT:    vbroadcastss 80(%r9), %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm11[5],ymm10[6,7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 96(%rcx), %xmm11
+; AVX-NEXT:    vmovaps 96(%rdx), %xmm12
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm12[1,2],xmm11[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm10, %ymm10
+; AVX-NEXT:    vmovaps 96(%rsi), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm13
+; AVX-NEXT:    vmovaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm13[2],xmm0[2],xmm13[3],xmm0[3]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm13
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm13[4,5],ymm10[6,7]
+; AVX-NEXT:    vbroadcastss 100(%r8), %xmm13
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm13[2,3],ymm10[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 100(%r9), %ymm13
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm10[0,1,2],ymm13[3],ymm10[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5],ymm2[6,7]
-; AVX-NEXT:    vbroadcastss 100(%r8), %xmm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 100(%r9), %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm3[3],ymm2[4,5,6,7]
+; AVX-NEXT:    vmovaps 96(%rdi), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdi), %ymm7
-; AVX-NEXT:    vmovaps 96(%rsi), %ymm3
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm10 = ymm7[0],ymm3[0],ymm7[1],ymm3[1],ymm7[4],ymm3[4],ymm7[5],ymm3[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm10[2,3,2,3]
-; AVX-NEXT:    vmovaps 96(%rdx), %ymm10
-; AVX-NEXT:    vmovaps 96(%rcx), %ymm2
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm15 = ymm2[0],ymm10[0],ymm2[2],ymm10[2]
+; AVX-NEXT:    vmovaps 96(%rsi), %ymm10
 ; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm10 = ymm0[0],ymm10[0],ymm0[1],ymm10[1],ymm0[4],ymm10[4],ymm0[5],ymm10[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm10[2,3,2,3]
+; AVX-NEXT:    vmovaps 96(%rdx), %ymm13
+; AVX-NEXT:    vmovaps 96(%rcx), %ymm10
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm15 = ymm10[0],ymm13[0],ymm10[2],ymm13[2]
 ; AVX-NEXT:    vextractf128 $1, %ymm15, %xmm15
 ; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm15[2,3],ymm1[4,5,6,7]
-; AVX-NEXT:    vmovaps 96(%r8), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm15[2,3],ymm0[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],mem[4,5],ymm0[6,7]
 ; AVX-NEXT:    vbroadcastss 112(%r9), %ymm15
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm1 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm1 = ymm12[2],mem[2],ymm12[3],mem[3],ymm12[6],mem[6],ymm12[7],mem[7]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,2],ymm12[1,2],ymm0[5,6],ymm12[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 20(%r8), %xmm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm15[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 20(%r9), %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm15[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm15 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm8[1,2],ymm13[1,2],ymm8[5,6],ymm13[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm15[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 52(%r8), %xmm8
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm8[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 52(%r9), %ymm8
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm8[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm14[2],ymm6[3],ymm14[3],ymm6[6],ymm14[6],ymm6[7],ymm14[7]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX-NEXT:    vshufps $153, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm0 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm0 = ymm14[1,2],mem[1,2],ymm14[5,6],mem[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 84(%r8), %xmm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm6[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 84(%r9), %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm6[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm9[0,0,0,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm11[0,0,0,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm1[2,3],ymm4[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 96(%r8), %ymm0, %ymm0
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX-NEXT:    vbroadcastss 96(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm11 = ymm7[2],ymm3[2],ymm7[3],ymm3[3],ymm7[6],ymm3[6],ymm7[7],ymm3[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm10[1,2],ymm2[1,2],ymm10[5,6],ymm2[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 116(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 116(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vbroadcastss (%rcx), %xmm0
-; AVX-NEXT:    vbroadcastss (%rdx), %xmm1
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss (%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX-NEXT:    vbroadcastss (%rdx), %xmm15
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0,1],ymm0[2,3],ymm5[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm3, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastss (%r9), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm14[2],mem[2],xmm14[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm9, %ymm0
 ; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = mem[2,1,3,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5],ymm3[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3,4,5],ymm3[6,7]
 ; AVX-NEXT:    vmovaps (%r9), %xmm3
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3,4,5,6],ymm3[7]
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = ymm12[3,0],mem[3,0],ymm12[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 $19, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = mem[2,3],ymm3[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm4 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3,4,5],ymm4[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3,4,5,6],ymm4[7]
-; AVX-NEXT:    vbroadcastss 32(%rcx), %xmm4
-; AVX-NEXT:    vbroadcastss 32(%rdx), %xmm6
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX-NEXT:    vmovaps (%rsp), %xmm5 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm6 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm6 = xmm5[0],mem[0],xmm5[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm7[0,1],ymm4[2,3],ymm7[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm6, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]
-; AVX-NEXT:    vbroadcastss 32(%r9), %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm6[5],ymm4[6,7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm6 = xmm6[2],mem[2],xmm6[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
-; AVX-NEXT:    vpermilps {{.*#+}} xmm7 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm7, %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3,4,5],ymm7[6,7]
-; AVX-NEXT:    vmovaps 32(%r9), %xmm7
-; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm7[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm8, %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm6[0],ymm7[1],ymm6[2,3,4,5,6],ymm7[7]
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm6 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm6 = ymm13[3,0],mem[3,0],ymm13[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm15[2,3],ymm6[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm8 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,3,4,5],ymm8[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm8[1],ymm6[2,3,4,5,6],ymm8[7]
-; AVX-NEXT:    vbroadcastss 64(%rcx), %xmm8
-; AVX-NEXT:    vbroadcastss 64(%rdx), %xmm9
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm9 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm9 = xmm5[0],mem[0],xmm5[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm12[0,1],ymm8[2,3],ymm12[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 64(%r8), %ymm9, %ymm9
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]
-; AVX-NEXT:    vbroadcastss 64(%r9), %ymm9
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm9[5],ymm8[6,7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm9 = xmm9[2],mem[2],xmm9[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm12, %ymm9
-; AVX-NEXT:    vpermilps {{.*#+}} xmm12 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm12, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm12[0,1],ymm9[2,3,4,5],ymm12[6,7]
-; AVX-NEXT:    vmovaps 64(%r9), %xmm12
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm12[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm13, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm12[1],ymm9[2,3,4,5,6],ymm12[7]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm5[3,0],ymm14[3,0],ymm5[7,4],ymm14[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 $19, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm5 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm5 = mem[2,3],ymm12[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm12 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm12[0,1],ymm5[2,3,4,5],ymm12[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3,4,5,6],ymm12[7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm12 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm12 = xmm10[2],mem[2],xmm10[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm12[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm10, %ymm12
-; AVX-NEXT:    vpermilps {{.*#+}} xmm13 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm13, %ymm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2,3,4,5],ymm13[6,7]
-; AVX-NEXT:    vmovaps 96(%r9), %xmm13
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm13[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm13[1],ymm12[2,3,4,5,6],ymm13[7]
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm10 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm10 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm11[2,3],ymm10[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm10 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm10[0,1],ymm2[2,3,4,5],ymm10[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2,3,4,5,6],ymm10[7]
-; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT:    vmovaps %ymm2, 736(%rax)
-; AVX-NEXT:    vmovaps %ymm12, 640(%rax)
-; AVX-NEXT:    vmovaps %ymm5, 544(%rax)
-; AVX-NEXT:    vmovaps %ymm9, 448(%rax)
-; AVX-NEXT:    vmovaps %ymm8, 384(%rax)
-; AVX-NEXT:    vmovaps %ymm6, 352(%rax)
-; AVX-NEXT:    vmovaps %ymm7, 256(%rax)
-; AVX-NEXT:    vmovaps %ymm4, 192(%rax)
-; AVX-NEXT:    vmovaps %ymm3, 160(%rax)
-; AVX-NEXT:    vmovaps %ymm1, 64(%rax)
-; AVX-NEXT:    vmovaps %ymm0, (%rax)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 704(%rax)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 576(%rax)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 512(%rax)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 320(%rax)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 128(%rax)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 672(%rax)
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm3[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 608(%rax)
+; AVX-NEXT:    vunpckhps (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[1,2],ymm4[1,2],ymm8[5,6],ymm4[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm0[4,5],ymm3[6,7]
+; AVX-NEXT:    vbroadcastss 20(%r8), %xmm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm5[2,3],ymm3[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 20(%r9), %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm3, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm4[3,0],ymm8[3,0],ymm4[7,4],ymm8[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm3[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3,4,5],ymm3[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm0[0],ymm3[1],ymm0[2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vbroadcastss 32(%rcx), %xmm0
+; AVX-NEXT:    vbroadcastss 32(%rdx), %xmm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm3, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastss 32(%r9), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm4 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3,4,5],ymm4[6,7]
+; AVX-NEXT:    vmovaps 32(%r9), %xmm4
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm4[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm0[0],ymm4[1],ymm0[2,3,4,5,6],ymm4[7]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 480(%rax)
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm2[1,2],ymm1[1,2],ymm2[5,6],ymm1[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm0[4,5],ymm5[6,7]
+; AVX-NEXT:    vbroadcastss 52(%r8), %xmm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm8[2,3],ymm5[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 52(%r9), %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm8[3],ymm5[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[3,0],ymm2[3,0],ymm1[7,4],ymm2[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
+; AVX-NEXT:    vbroadcastss 64(%rcx), %xmm1
+; AVX-NEXT:    vbroadcastss 64(%rdx), %xmm2
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1],ymm1[2,3],ymm8[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 64(%r8), %ymm2, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 64(%r9), %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm2 = xmm2[2],mem[2],xmm2[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm8, %ymm2
+; AVX-NEXT:    vpermilps {{.*#+}} xmm8 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm8, %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm8[0,1],ymm2[2,3,4,5],ymm8[6,7]
+; AVX-NEXT:    vmovaps 64(%r9), %xmm8
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm8[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm2[0],ymm8[1],ymm2[2,3,4,5,6],ymm8[7]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm2 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm7[1,2],ymm6[1,2],ymm7[5,6],ymm6[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm9[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm2[4,5],ymm9[6,7]
+; AVX-NEXT:    vbroadcastss 84(%r8), %xmm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1],ymm15[2,3],ymm9[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 84(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm15[3],ymm9[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[3,0],ymm7[3,0],ymm6[7,4],ymm7[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3],ymm6[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1],ymm2[2,3,4,5],ymm6[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm2[0],ymm6[1],ymm2[2,3,4,5,6],ymm6[7]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm11[0,0,0,0]
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm12[0,0,0,0]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm15[0,1],ymm7[2,3],ymm15[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 96(%r8), %ymm2, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm7[2,3],ymm2[4,5],ymm7[6,7]
+; AVX-NEXT:    vbroadcastss 96(%r9), %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3,4],ymm7[5],ymm2[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm11, %ymm2
+; AVX-NEXT:    vpermilps {{.*#+}} xmm11 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm11, %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm11[0,1],ymm2[2,3,4,5],ymm11[6,7]
+; AVX-NEXT:    vmovaps 96(%r9), %xmm11
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm11[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm11[1],ymm2[2,3,4,5,6],ymm11[7]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm11 = ymm11[2],mem[2],ymm11[3],mem[3],ymm11[6],mem[6],ymm11[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm13[1,2],ymm10[1,2],ymm13[5,6],ymm10[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm12[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm11[4,5],ymm12[6,7]
+; AVX-NEXT:    vbroadcastss 116(%r8), %xmm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1],ymm15[2,3],ymm12[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 116(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm15[3],ymm12[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[3,0],ymm13[3,0],ymm10[7,4],ymm13[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = ymm11[2,3],ymm10[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,3,4,5],ymm11[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm11[1],ymm10[2,3,4,5,6],ymm11[7]
+; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX-NEXT:    vmovaps %ymm10, 736(%rax)
+; AVX-NEXT:    vmovaps %ymm12, 704(%rax)
+; AVX-NEXT:    vmovaps %ymm2, 640(%rax)
+; AVX-NEXT:    vmovaps %ymm7, 576(%rax)
+; AVX-NEXT:    vmovaps %ymm6, 544(%rax)
+; AVX-NEXT:    vmovaps %ymm9, 512(%rax)
+; AVX-NEXT:    vmovaps %ymm8, 448(%rax)
+; AVX-NEXT:    vmovaps %ymm1, 384(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 352(%rax)
+; AVX-NEXT:    vmovaps %ymm5, 320(%rax)
+; AVX-NEXT:    vmovaps %ymm4, 256(%rax)
+; AVX-NEXT:    vmovaps %ymm3, 192(%rax)
+; AVX-NEXT:    vmovaps %ymm14, 160(%rax)
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 128(%rax)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 64(%rax)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, (%rax)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 672(%rax)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 608(%rax)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 480(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 416(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -3608,7 +3569,7 @@ define void @store_i32_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vmovaps %ymm0, 96(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 32(%rax)
-; AVX-NEXT:    addq $1032, %rsp # imm = 0x408
+; AVX-NEXT:    addq $712, %rsp # imm = 0x2C8
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -6568,32 +6529,27 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride6_vf64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $2504, %rsp # imm = 0x9C8
-; AVX-NEXT:    vmovaps (%rdi), %ymm8
-; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rsi), %ymm9
-; AVX-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rdx), %ymm4
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rcx), %ymm15
-; AVX-NEXT:    vmovaps (%r8), %ymm6
+; AVX-NEXT:    subq $2056, %rsp # imm = 0x808
+; AVX-NEXT:    vmovaps (%rdi), %ymm6
 ; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rsi), %ymm7
+; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps (%rdx), %ymm3
+; AVX-NEXT:    vmovaps (%rcx), %ymm2
 ; AVX-NEXT:    vmovaps (%rcx), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm4
+; AVX-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps (%rdx), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm3
-; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm5
+; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps (%rsi), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps (%rdi), %xmm7
-; AVX-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; AVX-NEXT:    vmovaps (%rsi), %xmm13
+; AVX-NEXT:    vmovaps (%rdi), %xmm12
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm12[2],xmm13[2],xmm12[3],xmm13[3]
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
@@ -6602,25 +6558,24 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vbroadcastss 4(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5]
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[4],ymm7[4],ymm6[5],ymm7[5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm15[0],ymm4[0],ymm15[2],ymm4[2]
-; AVX-NEXT:    vmovups %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm2[0],ymm3[0],ymm2[2],ymm3[2]
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5],ymm0[6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],mem[4,5],ymm0[6,7]
 ; AVX-NEXT:    vbroadcastss 16(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[1,2],xmm2[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[1,2],xmm4[1,2]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps 32(%rsi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm4
+; AVX-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
@@ -6634,414 +6589,348 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vmovaps 32(%rsi), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 32(%rdx), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3]
+; AVX-NEXT:    vmovaps 32(%rdx), %ymm0
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 32(%rcx), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 32(%r8), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 48(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rcx), %xmm1
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX-NEXT:    vextractf128 $1, %ymm5, %xmm5
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],mem[4,5],ymm4[6,7]
+; AVX-NEXT:    vbroadcastss 48(%r9), %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5],ymm4[6,7]
+; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 64(%rcx), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 64(%rdx), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdx), %xmm0
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[1,2],xmm0[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
+; AVX-NEXT:    vmovaps 64(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm1
+; AVX-NEXT:    vmovaps 64(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 68(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 68(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rdi), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rsi), %ymm0
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5],ymm4[6,7]
+; AVX-NEXT:    vbroadcastss 68(%r8), %xmm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 68(%r9), %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3],ymm4[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 64(%rdi), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 64(%rdx), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 64(%rcx), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 64(%r8), %ymm1
+; AVX-NEXT:    vmovaps 64(%rsi), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 80(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rcx), %xmm1
-; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdx), %xmm0
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm4[2,3,2,3]
+; AVX-NEXT:    vmovaps 64(%rdx), %ymm5
+; AVX-NEXT:    vmovaps 64(%rcx), %ymm4
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
+; AVX-NEXT:    vextractf128 $1, %ymm7, %xmm7
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm7[2,3],ymm6[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],mem[4,5],ymm6[6,7]
+; AVX-NEXT:    vbroadcastss 80(%r9), %ymm7
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm7[5],ymm6[6,7]
+; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 96(%rcx), %xmm7
+; AVX-NEXT:    vmovaps 96(%rdx), %xmm6
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm6[1,2],xmm7[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm8[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm8, %ymm8
+; AVX-NEXT:    vmovaps 96(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm1
+; AVX-NEXT:    vmovaps 96(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 100(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 100(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rdi), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rsi), %ymm0
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5],ymm8[6,7]
+; AVX-NEXT:    vbroadcastss 100(%r8), %xmm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm9[2,3],ymm8[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 100(%r9), %ymm9
+; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm9[3],ymm8[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 96(%rdi), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 96(%rdx), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 96(%rcx), %ymm1
+; AVX-NEXT:    vmovaps 96(%rsi), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 96(%r8), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 112(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rcx), %xmm1
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm8 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = ymm8[2,3,2,3]
+; AVX-NEXT:    vmovaps 96(%rdx), %ymm9
+; AVX-NEXT:    vmovaps 96(%rcx), %ymm8
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm11 = ymm8[0],ymm9[0],ymm8[2],ymm9[2]
+; AVX-NEXT:    vextractf128 $1, %ymm11, %xmm11
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm11[2,3],ymm10[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3],mem[4,5],ymm10[6,7]
+; AVX-NEXT:    vbroadcastss 112(%r9), %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm11[5],ymm10[6,7]
+; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 128(%rcx), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 128(%rdx), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 128(%rdx), %xmm0
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm1[1,2],xmm0[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm10 = xmm10[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm10, %ymm10
+; AVX-NEXT:    vmovaps 128(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 128(%rsi), %xmm1
+; AVX-NEXT:    vmovaps 128(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 128(%rdi), %xmm2
-; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 132(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 132(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5],ymm10[6,7]
+; AVX-NEXT:    vbroadcastss 132(%r8), %xmm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm11[2,3],ymm10[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 132(%r9), %ymm11
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2],ymm11[3],ymm10[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 128(%rdi), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rsi), %ymm11
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm11[0],ymm0[1],ymm11[1],ymm0[4],ymm11[4],ymm0[5],ymm11[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 128(%rdx), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rcx), %ymm2
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 128(%r8), %ymm1
+; AVX-NEXT:    vmovaps 128(%rsi), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 144(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm10[2,3,2,3]
+; AVX-NEXT:    vmovaps 128(%rdx), %ymm11
+; AVX-NEXT:    vmovaps 128(%rcx), %ymm10
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm15 = ymm10[0],ymm11[0],ymm10[2],ymm11[2]
+; AVX-NEXT:    vextractf128 $1, %ymm15, %xmm15
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2,3],ymm14[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3],mem[4,5],ymm14[6,7]
+; AVX-NEXT:    vbroadcastss 144(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
+; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps 160(%rcx), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 160(%rdx), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[1,2],xmm0[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 160(%rsi), %xmm7
-; AVX-NEXT:    vmovaps 160(%rdi), %xmm6
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 164(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 164(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm1[1,2],xmm0[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm15, %ymm15
+; AVX-NEXT:    vmovaps 160(%rsi), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 160(%rdi), %xmm1
+; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rdi), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rsi), %ymm0
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm14
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1,2,3],ymm14[4,5],ymm15[6,7]
+; AVX-NEXT:    vbroadcastss 164(%r8), %xmm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2,3],ymm14[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 164(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2],ymm15[3],ymm14[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 160(%rdi), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 160(%rdx), %ymm1
+; AVX-NEXT:    vmovaps 160(%rsi), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rcx), %ymm8
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[2],ymm1[2]
-; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 160(%r8), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 176(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm14[2,3,2,3]
+; AVX-NEXT:    vmovaps 160(%rdx), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 192(%rcx), %xmm1
+; AVX-NEXT:    vmovaps 160(%rcx), %ymm1
+; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm15 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX-NEXT:    vextractf128 $1, %ymm15, %xmm15
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2,3],ymm14[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3],mem[4,5],ymm14[6,7]
+; AVX-NEXT:    vbroadcastss 176(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
+; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 192(%rcx), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 192(%rdx), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 192(%rdx), %xmm0
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm1[1,2],xmm0[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm14[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm14, %ymm14
+; AVX-NEXT:    vmovaps 192(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 192(%rsi), %xmm3
-; AVX-NEXT:    vmovaps 192(%rdi), %xmm2
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 196(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 196(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vmovaps 192(%rdi), %xmm1
+; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 192(%rdi), %ymm1
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4,5],ymm14[6,7]
+; AVX-NEXT:    vbroadcastss 196(%r8), %xmm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2,3],ymm14[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 196(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2],ymm15[3],ymm14[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 192(%rdi), %ymm0
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 192(%rsi), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 192(%rsi), %ymm0
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm14[2,3,2,3]
+; AVX-NEXT:    vmovaps 192(%rdx), %ymm0
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vmovaps 192(%rdx), %ymm4
 ; AVX-NEXT:    vmovaps 192(%rcx), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
-; AVX-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vmovaps 192(%r8), %ymm1
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 208(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 224(%rcx), %xmm1
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm15 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX-NEXT:    vextractf128 $1, %ymm15, %xmm15
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2,3],ymm14[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3],mem[4,5],ymm14[6,7]
+; AVX-NEXT:    vbroadcastss 208(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
+; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 224(%rcx), %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps 224(%rdx), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 224(%rdx), %xmm0
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm1[1,2],xmm0[1,2]
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm14[0,2,1,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm14, %ymm14
+; AVX-NEXT:    vmovaps 224(%rsi), %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[1,2]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX-NEXT:    vmovaps 224(%rsi), %xmm1
+; AVX-NEXT:    vmovaps 224(%rdi), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 224(%rdi), %xmm5
-; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 228(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 228(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 224(%rdi), %ymm9
-; AVX-NEXT:    vmovaps 224(%rsi), %ymm0
-; AVX-NEXT:    vunpcklps {{.*#+}} ymm14 = ymm9[0],ymm0[0],ymm9[1],ymm0[1],ymm9[4],ymm0[4],ymm9[5],ymm0[5]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4,5],ymm14[6,7]
+; AVX-NEXT:    vbroadcastss 228(%r8), %xmm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2,3],ymm14[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 228(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2],ymm15[3],ymm14[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 224(%rdi), %ymm0
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 224(%rsi), %ymm1
+; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm14[2,3,2,3]
-; AVX-NEXT:    vmovaps 224(%rdx), %ymm5
-; AVX-NEXT:    vmovaps 224(%rcx), %ymm1
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm13 = ymm1[0],ymm5[0],ymm1[2],ymm5[2]
+; AVX-NEXT:    vmovaps 224(%rdx), %ymm1
 ; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vextractf128 $1, %ymm13, %xmm13
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm13[0,1,2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1],ymm13[2,3],ymm14[4,5,6,7]
-; AVX-NEXT:    vmovaps 224(%r8), %ymm10
-; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm10[4,5],ymm13[6,7]
-; AVX-NEXT:    vbroadcastss 240(%r9), %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2,3,4],ymm14[5],ymm13[6,7]
-; AVX-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm14 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm14 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm10[1,2],ymm15[1,2],ymm10[5,6],ymm15[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = ymm13[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5],ymm13[6,7]
-; AVX-NEXT:    vbroadcastss 20(%r8), %xmm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1],ymm15[2,3],ymm13[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 20(%r9), %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2],ymm15[3],ymm13[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm13 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm13 = ymm12[2],mem[2],ymm12[3],mem[3],ymm12[6],mem[6],ymm12[7],mem[7]
-; AVX-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX-NEXT:    vshufps $153, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm15 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm15 = ymm12[1,2],mem[1,2],ymm12[5,6],mem[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm15[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm15 = ymm15[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm15[0,1,2,3],ymm13[4,5],ymm15[6,7]
-; AVX-NEXT:    vbroadcastss 52(%r8), %xmm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm15[0,1],ymm12[2,3],ymm15[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 52(%r9), %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm15[3],ymm12[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm13 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm13 = ymm12[2],mem[2],ymm12[3],mem[3],ymm12[6],mem[6],ymm12[7],mem[7]
-; AVX-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX-NEXT:    vshufps $153, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm12 = ymm12[1,2],mem[1,2],ymm12[5,6],mem[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm12[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5],ymm12[6,7]
-; AVX-NEXT:    vbroadcastss 84(%r8), %xmm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1],ymm15[2,3],ymm12[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 84(%r9), %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm15[3],ymm12[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm12 = xmm12[0],mem[0],xmm12[1],mem[1]
-; AVX-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm15 = mem[0,0,0,0]
-; AVX-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm13 = mem[0,0,0,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm15[0,1],ymm13[2,3],ymm15[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 96(%r8), %ymm12, %ymm12
+; AVX-NEXT:    vmovaps 224(%rcx), %ymm0
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm15 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX-NEXT:    vextractf128 $1, %ymm15, %xmm15
+; AVX-NEXT:    vshufps {{.*#+}} xmm15 = xmm15[0,1,2,0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm15[2,3],ymm14[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3],mem[4,5],ymm14[6,7]
+; AVX-NEXT:    vbroadcastss 240(%r9), %ymm15
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
+; AVX-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastss (%rcx), %xmm14
+; AVX-NEXT:    vbroadcastss (%rdx), %xmm15
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm13
+; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1],ymm14[2,3],ymm13[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm12, %ymm12
 ; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1],ymm13[2,3],ymm12[4,5],ymm13[6,7]
-; AVX-NEXT:    vbroadcastss 96(%r9), %ymm13
+; AVX-NEXT:    vbroadcastss (%r9), %ymm13
 ; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4],ymm13[5],ymm12[6,7]
 ; AVX-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm12 = ymm12[2],mem[2],ymm12[3],mem[3],ymm12[6],mem[6],ymm12[7],mem[7]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX-NEXT:    vshufps $153, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm13 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm13 = ymm13[1,2],mem[1,2],ymm13[5,6],mem[5,6]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm12 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm12 = xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm12[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm12
+; AVX-NEXT:    vpermilps {{.*#+}} xmm13 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm13, %ymm13
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2,3,4,5],ymm13[6,7]
+; AVX-NEXT:    vmovaps (%r9), %xmm13
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm13[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm13[1],ymm12[2,3,4,5,6],ymm13[7]
+; AVX-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm12 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm3[1,2],ymm2[1,2],ymm3[5,6],ymm2[5,6]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = ymm13[2,3,2,3]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[0,2,1,3,4,6,5,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm12[4,5],ymm13[6,7]
-; AVX-NEXT:    vbroadcastss 116(%r8), %xmm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1],ymm15[2,3],ymm13[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 116(%r9), %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm15[3],ymm13[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastss 20(%r8), %xmm14
+; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1],ymm14[2,3],ymm13[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 20(%r9), %ymm14
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1,2],ymm14[3],ymm13[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,0],ymm3[3,0],ymm2[7,4],ymm3[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm12[2,3],ymm2[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5],ymm3[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm3[1],ymm2[2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastss 32(%rcx), %xmm2
+; AVX-NEXT:    vbroadcastss 32(%rdx), %xmm3
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm3 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm12
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1],ymm2[2,3],ymm12[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm3, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]
+; AVX-NEXT:    vbroadcastss 32(%r9), %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm2 = xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5],ymm3[6,7]
+; AVX-NEXT:    vmovaps 32(%r9), %xmm3
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm3[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm12, %ymm3
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm3[1],ymm2[2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm2 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm13 = ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[6],ymm11[6],ymm13[7],ymm11[7]
-; AVX-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[1,2],ymm15[1,2],ymm11[5,6],ymm15[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = ymm11[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm13[4,5],ymm11[6,7]
-; AVX-NEXT:    vbroadcastss 148(%r8), %xmm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1],ymm13[2,3],ymm11[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 148(%r9), %ymm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm13[3],ymm11[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm7 = mem[0,0,0,0]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm13[0,0,0,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm11[0],xmm7[0],xmm11[1],xmm7[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm11[0,1],ymm7[2,3],ymm11[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 160(%r8), %ymm6, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm7[2,3],ymm6[4,5],ymm7[6,7]
-; AVX-NEXT:    vbroadcastss 160(%r9), %ymm7
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm7[5],ymm6[6,7]
-; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm7 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm7 = ymm6[2],mem[2],ymm6[3],mem[3],ymm6[6],mem[6],ymm6[7],mem[7]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[1,2],ymm8[1,2],ymm6[5,6],ymm8[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm6[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5],ymm6[6,7]
-; AVX-NEXT:    vbroadcastss 180(%r8), %xmm8
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm8[2,3],ymm6[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 180(%r9), %ymm8
-; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm8[3],ymm6[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm3 = mem[0,0,0,0]
-; AVX-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm6 = mem[0,0,0,0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm6
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1],ymm3[2,3],ymm6[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 192(%r8), %ymm2, %ymm2
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
-; AVX-NEXT:    vbroadcastss 192(%r9), %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
-; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm6 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm6 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm4[1,2],ymm2[1,2],ymm4[5,6],ymm2[5,6]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm13[1,2],ymm1[1,2],ymm13[5,6],ymm1[5,6]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3,2,3]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5],ymm3[6,7]
-; AVX-NEXT:    vbroadcastss 212(%r8), %xmm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 212(%r9), %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5,6,7]
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm3 = ymm9[2],ymm0[2],ymm9[3],ymm0[3],ymm9[6],ymm0[6],ymm9[7],ymm0[7]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm5[1,2],ymm1[1,2],ymm5[5,6],ymm1[5,6]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5],ymm0[6,7]
-; AVX-NEXT:    vmovaps %ymm3, %ymm5
-; AVX-NEXT:    vbroadcastss 244(%r8), %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; AVX-NEXT:    vbroadcastss 244(%r9), %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm2[4,5],ymm3[6,7]
+; AVX-NEXT:    vbroadcastss 52(%r8), %xmm12
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm12[2,3],ymm3[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 52(%r9), %ymm12
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2],ymm12[3],ymm3[4,5,6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vbroadcastss (%rcx), %xmm0
-; AVX-NEXT:    vbroadcastss (%rdx), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm13[3,0],ymm1[7,4],ymm13[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm2[2,3],ymm0[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastss 64(%rcx), %xmm0
+; AVX-NEXT:    vbroadcastss 64(%rdx), %xmm1
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, (%r8), %ymm1, %ymm1
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 64(%r8), %ymm1, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss (%r9), %ymm1
+; AVX-NEXT:    vbroadcastss 64(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -7053,303 +6942,321 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = mem[2,1,3,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
-; AVX-NEXT:    vmovaps (%r9), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX-NEXT:    vmovaps 64(%r9), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[3,0],ymm10[3,0],ymm0[7,4],ymm10[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm14[2,3],ymm0[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm5[1,2],ymm4[1,2],ymm5[5,6],ymm4[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 84(%r8), %xmm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 84(%r9), %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[3,0],ymm5[3,0],ymm4[7,4],ymm5[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vbroadcastss 32(%rcx), %xmm0
-; AVX-NEXT:    vbroadcastss 32(%rdx), %xmm1
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 32(%r8), %ymm1, %ymm1
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 32(%r9), %ymm1
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm7[0,0,0,0]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm6[0,0,0,0]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 96(%r8), %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 96(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = mem[2,1,3,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
-; AVX-NEXT:    vmovaps 32(%r9), %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX-NEXT:    vmovaps 96(%r9), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm0 = ymm0[3,0],mem[3,0],ymm0[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 $19, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm0 = mem[2,3],ymm0[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[1,2],ymm8[1,2],ymm9[5,6],ymm8[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 116(%r8), %xmm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 116(%r9), %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[3,0],ymm9[3,0],ymm8[7,4],ymm9[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vbroadcastss 64(%rcx), %xmm0
-; AVX-NEXT:    vbroadcastss 64(%rdx), %xmm1
+; AVX-NEXT:    vbroadcastss 128(%rcx), %xmm0
+; AVX-NEXT:    vbroadcastss 128(%rdx), %xmm1
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 64(%r8), %ymm1, %ymm1
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 128(%r8), %ymm1, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
-; AVX-NEXT:    vbroadcastss 64(%r9), %ymm1
+; AVX-NEXT:    vbroadcastss 128(%r9), %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm1 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5],ymm3[6,7]
-; AVX-NEXT:    vmovaps 64(%r9), %xmm3
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm3[1],ymm1[2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 128(%r9), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = ymm0[3,0],mem[3,0],ymm0[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 $19, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm3 = mem[2,3],ymm3[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm4 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3,4,5],ymm4[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0],ymm4[1],ymm3[2,3,4,5,6],ymm4[7]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[1,2],ymm10[1,2],ymm11[5,6],ymm10[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 148(%r8), %xmm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 148(%r9), %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[3,0],ymm11[3,0],ymm10[7,4],ymm11[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm3 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
-; AVX-NEXT:    vpermilps {{.*#+}} xmm4 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3,4,5],ymm4[6,7]
-; AVX-NEXT:    vmovaps 96(%r9), %xmm4
-; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm4[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm8, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3,4,5,6],ymm4[7]
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,0,0,0]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[0,0,0,0]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 160(%r8), %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 160(%r9), %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 160(%r9), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm4 = ymm0[3,0],mem[3,0],ymm0[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm12[2,3],ymm4[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm8 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm8[0,1],ymm4[2,3,4,5],ymm8[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm8[1],ymm4[2,3,4,5,6],ymm8[7]
-; AVX-NEXT:    vbroadcastss 128(%rcx), %xmm8
-; AVX-NEXT:    vbroadcastss 128(%rdx), %xmm9
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm9 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm9 = xmm0[0],mem[0],xmm0[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm11[0,1],ymm8[2,3],ymm11[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 128(%r8), %ymm9, %ymm9
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]
-; AVX-NEXT:    vbroadcastss 128(%r9), %ymm9
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm9[5],ymm8[6,7]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[1,2],ymm3[1,2],ymm4[5,6],ymm3[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 180(%r8), %xmm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 180(%r9), %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[3,0],ymm4[3,0],ymm3[7,4],ymm4[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm9 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm9 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm9
-; AVX-NEXT:    vpermilps {{.*#+}} xmm11 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm11, %ymm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm11[0,1],ymm9[2,3,4,5],ymm11[6,7]
-; AVX-NEXT:    vmovaps 128(%r9), %xmm11
-; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm11[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm11[1],ymm9[2,3,4,5,6],ymm11[7]
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm11 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm11 = ymm15[3,0],mem[3,0],ymm15[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 $19, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm10 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm10 = mem[2,3],ymm11[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm11 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,3,4,5],ymm11[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm11 = ymm11[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm11[1],ymm10[2,3,4,5,6],ymm11[7]
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm11 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm11 = xmm13[2],mem[2],xmm13[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[2,3,2,3]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm0, %ymm11
-; AVX-NEXT:    vpermilps {{.*#+}} xmm12 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm12, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1],ymm11[2,3,4,5],ymm12[6,7]
-; AVX-NEXT:    vmovaps 160(%r9), %xmm12
-; AVX-NEXT:    vshufps {{.*#+}} xmm13 = xmm12[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm13, %ymm12
-; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm12[1],ymm11[2,3,4,5,6],ymm12[7]
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,0,0,0]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[0,0,0,0]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 192(%r8), %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 192(%r9), %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vmovaps 192(%r9), %xmm1
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm12 = ymm0[3,0],mem[3,0],ymm0[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm7[2,3],ymm12[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm12 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm12[0,1],ymm7[2,3,4,5],ymm12[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0],ymm12[1],ymm7[2,3,4,5,6],ymm12[7]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm5[1,2],ymm3[1,2],ymm5[5,6],ymm3[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastss 212(%r8), %xmm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 212(%r9), %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[3,0],ymm5[3,0],ymm3[7,4],ymm5[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6],ymm1[7]
+; AVX-NEXT:    vbroadcastss 224(%rcx), %xmm0
+; AVX-NEXT:    vbroadcastss 224(%rdx), %xmm5
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm5 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm5 = xmm1[0],mem[0],xmm1[1],mem[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm6
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5,6,7]
+; AVX-NEXT:    vinsertf128 $1, 224(%r8), %ymm5, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0,1],ymm0[2,3],ymm5[4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastss 224(%r9), %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4],ymm5[5],ymm0[6,7]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm12 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm12 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm12[2,3,2,3]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm5 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm5 = xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[2,3,2,3]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm12
-; AVX-NEXT:    vpermilps {{.*#+}} xmm13 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm13, %ymm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm12[2,3,4,5],ymm13[6,7]
-; AVX-NEXT:    vmovaps 192(%r9), %xmm13
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm13[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13
-; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm13[1],ymm12[2,3,4,5,6],ymm13[7]
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm13 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm13 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm6[2,3],ymm13[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm13 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm13 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm13[0,1],ymm2[2,3,4,5],ymm13[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm13[1],ymm2[2,3,4,5,6],ymm13[7]
-; AVX-NEXT:    vbroadcastss 224(%rcx), %xmm13
-; AVX-NEXT:    vbroadcastss 224(%rdx), %xmm14
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm14 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm14 = xmm0[0],mem[0],xmm0[1],mem[1]
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm0, %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm15[0,1],ymm13[2,3],ymm15[4,5,6,7]
-; AVX-NEXT:    vinsertf128 $1, 224(%r8), %ymm14, %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1],ymm13[2,3],ymm14[4,5],ymm13[6,7]
-; AVX-NEXT:    vbroadcastss 224(%r9), %ymm14
-; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm14[5],ymm13[6,7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm14 # 16-byte Folded Reload
-; AVX-NEXT:    # xmm14 = xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm14[2,3,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
+; AVX-NEXT:    vpermilps {{.*#+}} xmm6 = mem[2,1,3,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm6, %ymm6
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3,4,5],ymm6[6,7]
+; AVX-NEXT:    vmovaps 224(%r9), %xmm6
+; AVX-NEXT:    vshufps {{.*#+}} xmm7 = xmm6[0,2,2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
+; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4,5,6],ymm6[7]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm0, %ymm14
-; AVX-NEXT:    vpermilps {{.*#+}} xmm15 = mem[2,1,3,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm15, %ymm15
-; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5],ymm15[6,7]
-; AVX-NEXT:    vmovaps 224(%r9), %xmm15
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm15[0,2,2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm0, %ymm0
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0],ymm0[1],ymm14[2,3,4,5,6],ymm0[7]
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX-NEXT:    vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm14 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm14 = ymm6[3,0],mem[3,0],ymm6[7,4],mem[7,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3],ymm14[2,3]
-; AVX-NEXT:    vperm2f128 $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm14 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm14 = ymm14[2,1,3,3,6,5,7,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1],ymm5[2,3,4,5],ymm14[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = mem[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm14 = ymm14[0,2,2,3,4,6,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm14[1],ymm5[2,3,4,5,6],ymm14[7]
+; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
+; AVX-NEXT:    # ymm6 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm0[1,2],ymm1[1,2],ymm0[5,6],ymm1[5,6]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm7[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[0,2,1,3,4,6,5,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm6[4,5],ymm7[6,7]
+; AVX-NEXT:    vbroadcastss 244(%r8), %xmm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5,6,7]
+; AVX-NEXT:    vbroadcastss 244(%r9), %ymm8
+; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5,6,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,0,2,3,6,4,6,7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = ymm6[2,3],ymm8[2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,1,3,3,6,5,7,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,3,4,5],ymm8[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[0,2,2,3,4,6,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm8[1],ymm6[2,3,4,5,6],ymm8[7]
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT:    vmovaps %ymm5, 1504(%rax)
-; AVX-NEXT:    vmovaps %ymm0, 1408(%rax)
-; AVX-NEXT:    vmovaps %ymm13, 1344(%rax)
-; AVX-NEXT:    vmovaps %ymm2, 1312(%rax)
-; AVX-NEXT:    vmovaps %ymm12, 1216(%rax)
-; AVX-NEXT:    vmovaps %ymm7, 1120(%rax)
-; AVX-NEXT:    vmovaps %ymm11, 1024(%rax)
-; AVX-NEXT:    vmovaps %ymm10, 928(%rax)
-; AVX-NEXT:    vmovaps %ymm9, 832(%rax)
-; AVX-NEXT:    vmovaps %ymm8, 768(%rax)
-; AVX-NEXT:    vmovaps %ymm4, 736(%rax)
-; AVX-NEXT:    vmovaps %ymm3, 640(%rax)
-; AVX-NEXT:    vmovaps %ymm1, 544(%rax)
+; AVX-NEXT:    vmovaps %ymm6, 1504(%rax)
+; AVX-NEXT:    vmovaps %ymm7, 1472(%rax)
+; AVX-NEXT:    vmovaps %ymm5, 1408(%rax)
+; AVX-NEXT:    vmovaps %ymm2, 1344(%rax)
+; AVX-NEXT:    vmovaps %ymm3, 1312(%rax)
+; AVX-NEXT:    vmovaps %ymm4, 1280(%rax)
+; AVX-NEXT:    vmovaps %ymm9, 1216(%rax)
+; AVX-NEXT:    vmovaps %ymm10, 1152(%rax)
+; AVX-NEXT:    vmovaps %ymm15, 1120(%rax)
+; AVX-NEXT:    vmovaps %ymm14, 1088(%rax)
+; AVX-NEXT:    vmovaps %ymm13, 1024(%rax)
+; AVX-NEXT:    vmovaps %ymm11, 960(%rax)
+; AVX-NEXT:    vmovaps %ymm12, 928(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 448(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 896(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 384(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 832(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 352(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 768(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 256(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 736(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 192(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 704(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 160(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 640(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 64(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 576(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, (%rax)
+; AVX-NEXT:    vmovaps %ymm0, 544(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 1472(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 512(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 1280(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 448(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 1152(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 384(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 1088(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 352(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 960(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 320(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 896(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 256(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 704(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 192(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 576(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 160(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 512(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 128(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 320(%rax)
+; AVX-NEXT:    vmovaps %ymm0, 64(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 128(%rax)
+; AVX-NEXT:    vmovaps %ymm0, (%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 1440(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -7382,7 +7289,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vmovaps %ymm0, 96(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 32(%rax)
-; AVX-NEXT:    addq $2504, %rsp # imm = 0x9C8
+; AVX-NEXT:    addq $2056, %rsp # imm = 0x808
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
index e11352abcaec..bfd1e3ece200 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
@@ -462,45 +462,46 @@ define void @store_i32_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX-NEXT:    vmovaps (%rdi), %xmm3
-; AVX-NEXT:    vmovaps (%rsi), %xmm4
-; AVX-NEXT:    vmovaps (%rdx), %xmm5
+; AVX-NEXT:    vmovaps (%rdi), %xmm4
+; AVX-NEXT:    vmovaps (%rsi), %xmm5
+; AVX-NEXT:    vmovaps (%rdx), %xmm6
 ; AVX-NEXT:    vmovaps (%rcx), %xmm0
 ; AVX-NEXT:    vmovaps (%r8), %xmm2
-; AVX-NEXT:    vmovaps (%r9), %xmm6
+; AVX-NEXT:    vmovaps (%r9), %xmm3
 ; AVX-NEXT:    vmovaps (%r10), %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm7
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm7
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm8
-; AVX-NEXT:    vshufps {{.*#+}} ymm9 = ymm8[1,0],ymm7[1,0],ymm8[5,4],ymm7[5,4]
-; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm9[2,0],ymm7[2,1],ymm9[6,4],ymm7[6,5]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = zero,zero,xmm3[2],xmm4[2]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = zero,xmm5[1],xmm0[1],zero
-; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = xmm4[1],xmm10[1,2],zero
+; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm8[1,0],ymm7[1,0],ymm8[5,4],ymm7[5,4]
+; AVX-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[2,0],ymm2[2,1],ymm7[6,4],ymm2[6,5]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm5[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm4[0,1,2],xmm9[3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = zero,xmm6[1],xmm0[1],zero
+; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = xmm5[1],xmm10[1,2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
 ; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2],ymm7[3,4,5],ymm9[6,7]
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm9 = xmm6[1],xmm2[1]
+; AVX-NEXT:    vunpckhpd {{.*#+}} xmm9 = xmm3[1],xmm2[1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,1],ymm9[2,0],ymm8[6,5],ymm9[6,4]
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm3[3,3],xmm4[3,3]
-; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1,2],xmm5[3]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm4[3,3],xmm5[3,3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1,2],xmm6[3]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm6[2],xmm0[2],xmm6[3],xmm0[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
 ; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3,4],ymm9[5,6,7]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm5 = xmm5[0],xmm0[0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,1],xmm5[0,2]
-; AVX-NEXT:    vbroadcastss 4(%rdi), %xmm4
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm1[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6],ymm3[7]
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3],xmm6[3,3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm2[1,2],zero
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm5 = xmm5[0],xmm6[0]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm5 = xmm5[0],xmm1[0]
+; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
+; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6],ymm4[7]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3],xmm3[3,3]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm2[1,2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX-NEXT:    vmovaps %xmm0, 96(%rax)
 ; AVX-NEXT:    vmovaps %ymm8, 64(%rax)
-; AVX-NEXT:    vmovaps %ymm3, (%rax)
 ; AVX-NEXT:    vmovaps %ymm7, 32(%rax)
+; AVX-NEXT:    vmovaps %ymm4, (%rax)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -650,7 +651,7 @@ define void @store_i32_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FCP-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [7,3,7,3,7,3,7,3]
 ; AVX2-FCP-NEXT:    vpermps %ymm6, %ymm3, %ymm3
 ; AVX2-FCP-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-FCP-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],xmm3[1,2,3]
+; AVX2-FCP-NEXT:    vmovss {{.*#+}} xmm1 = xmm1[0],xmm3[1,2,3]
 ; AVX2-FCP-NEXT:    vmovaps %ymm2, (%rax)
 ; AVX2-FCP-NEXT:    vmovaps %ymm4, 64(%rax)
 ; AVX2-FCP-NEXT:    vmovaps %ymm7, 32(%rax)
@@ -2059,83 +2060,83 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX-LABEL: store_i32_stride7_vf16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $456, %rsp # imm = 0x1C8
+; AVX-NEXT:    subq $472, %rsp # imm = 0x1D8
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX-NEXT:    vmovaps 32(%rdi), %ymm9
-; AVX-NEXT:    vmovaps 32(%rsi), %ymm6
-; AVX-NEXT:    vmovaps 32(%rdx), %ymm7
-; AVX-NEXT:    vmovaps 32(%rcx), %ymm8
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm8
+; AVX-NEXT:    vmovaps 32(%rsi), %ymm5
+; AVX-NEXT:    vmovaps 32(%rdx), %ymm6
+; AVX-NEXT:    vmovaps 32(%rcx), %ymm7
 ; AVX-NEXT:    vmovaps 32(%r8), %ymm0
-; AVX-NEXT:    vmovaps 32(%r9), %ymm1
-; AVX-NEXT:    vmovaps 32(%rax), %ymm2
-; AVX-NEXT:    vunpckhps {{.*#+}} ymm3 = ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[6],ymm8[6],ymm7[7],ymm8[7]
-; AVX-NEXT:    vunpckhpd {{.*#+}} ymm4 = ymm6[1],ymm9[1],ymm6[3],ymm9[3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm9[0,1],ymm4[2,0],ymm9[4,5],ymm4[6,4]
-; AVX-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm4[2,3],ymm3[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm1[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[1,1,2,2,5,5,6,6]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0],ymm3[1,2,3,4,5,6],ymm4[7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3,4],ymm4[5],ymm3[6,7]
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 32(%rax), %ymm1
+; AVX-NEXT:    vunpckhps {{.*#+}} ymm2 = ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[6],ymm7[6],ymm6[7],ymm7[7]
+; AVX-NEXT:    vunpckhpd {{.*#+}} ymm3 = ymm5[1],ymm8[1],ymm5[3],ymm8[3]
+; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[0,1],ymm3[2,0],ymm8[4,5],ymm3[6,4]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm3[2,3],ymm2[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[1,1,2,2,5,5,6,6]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3,4,5,6],ymm3[7]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm1[2,3],ymm2[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3,4],ymm3[5],ymm2[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vmovaps 32(%rax), %xmm4
+; AVX-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 32(%r9), %xmm10
-; AVX-NEXT:    vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm10, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    vmovaps 32(%r8), %xmm11
 ; AVX-NEXT:    vmovaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm13
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm12
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm12[0],xmm13[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[2,0],xmm13[2,1]
-; AVX-NEXT:    vmovaps %xmm13, (%rsp) # 16-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
-; AVX-NEXT:    vmovaps 32(%rcx), %xmm5
-; AVX-NEXT:    vmovaps 32(%rdx), %xmm15
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
-; AVX-NEXT:    vmovaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps %xmm5, %xmm14
-; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,1,0,1]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5,6,7]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; AVX-NEXT:    vmovaps 32(%rdi), %xmm12
+; AVX-NEXT:    vmovaps 32(%rsi), %xmm9
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm9[0],xmm12[0]
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,0],xmm12[2,1]
+; AVX-NEXT:    vmovaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
+; AVX-NEXT:    vmovaps 32(%rcx), %xmm13
+; AVX-NEXT:    vmovaps 32(%rdx), %xmm14
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
+; AVX-NEXT:    vmovaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,1,0,1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5,6,7]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
 ; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
-; AVX-NEXT:    vmovaps 32(%rax), %xmm5
-; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[2],ymm5[2]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6],ymm3[7]
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm10[1,1],xmm11[1,1]
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6],ymm2[7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm10[1,1],xmm11[1,1]
+; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm4[5],ymm2[6,7]
+; AVX-NEXT:    vmovaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vunpckhpd {{.*#+}} xmm3 = xmm12[1],xmm9[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm9[1,1],xmm3[0,2]
 ; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5],ymm3[6,7]
-; AVX-NEXT:    vmovaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm4 = xmm13[1],xmm12[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm12[1,1],xmm4[0,2]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT:    vinsertps {{.*#+}} xmm5 = zero,xmm15[1],xmm14[1],zero
-; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1,2],ymm4[3,4,5,6,7]
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5],ymm4[6,7]
-; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm6[1,1],ymm9[1,1],ymm6[5,5],ymm9[5,5]
-; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm3[2,3,2,3]
-; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm7[1,1],ymm8[1,1],ymm7[5,5],ymm8[5,5]
-; AVX-NEXT:    vmovaps %ymm8, %ymm13
+; AVX-NEXT:    vinsertps {{.*#+}} xmm4 = zero,xmm14[1],xmm13[1],zero
+; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2],ymm3[3,4,5,6,7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5],ymm3[6,7]
+; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm5[1,1],ymm8[1,1],ymm5[5,5],ymm8[5,5]
+; AVX-NEXT:    vmovaps %ymm8, %ymm4
+; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm6[1,1],ymm7[1,1],ymm6[5,5],ymm7[5,5]
 ; AVX-NEXT:    vmovaps %ymm7, %ymm11
 ; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5,6],ymm3[7]
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,1],ymm1[6,4],ymm0[6,5]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm0[2,3]
+; AVX-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6],ymm2[7]
+; AVX-NEXT:    vmovaps 32(%r9), %ymm3
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[2],ymm0[2]
+; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm3[2,0],ymm0[2,1],ymm3[6,4],ymm0[6,5]
+; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm0[2,3]
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3,4,5,6],ymm0[7]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6],ymm0[7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps (%rdi), %xmm5
-; AVX-NEXT:    vmovaps (%rsi), %xmm14
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm5[1],xmm14[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm14[1,1],xmm0[0,2]
+; AVX-NEXT:    vmovaps (%rdi), %xmm14
+; AVX-NEXT:    vmovaps (%rsi), %xmm13
+; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm14[1],xmm13[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm13[1,1],xmm0[0,2]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX-NEXT:    vmovaps (%rcx), %xmm7
 ; AVX-NEXT:    vmovaps (%rdx), %xmm12
@@ -2144,9 +2145,9 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7]
 ; AVX-NEXT:    vmovaps (%r9), %xmm3
 ; AVX-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovaps (%r8), %xmm4
-; AVX-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[1,1],xmm4[1,1]
+; AVX-NEXT:    vmovaps (%r8), %xmm5
+; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[1,1],xmm5[1,1]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX-NEXT:    vmovaps (%rax), %xmm2
 ; AVX-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -2154,12 +2155,11 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5],ymm0[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
-; AVX-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm14[0],xmm5[0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm5[2,1]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm13[0],xmm14[0]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm14[2,1]
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm12[0],xmm7[0],xmm12[1],xmm7[1]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1,0,1]
@@ -2195,10 +2195,10 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3,4,5,6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3,4,5],ymm2[6,7]
 ; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm13[3,3],ymm11[3,3],ymm13[7,7],ymm11[7,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm0 = ymm11[3,3],ymm6[3,3],ymm11[7,7],ymm6[7,7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm11[3,3],ymm6[3,3],ymm11[7,7],ymm6[7,7]
+; AVX-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[3,3],ymm11[3,3],ymm4[7,7],ymm11[7,7]
 ; AVX-NEXT:    vextractf128 $1, %ymm2, %xmm2
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
 ; AVX-NEXT:    vbroadcastss 60(%r8), %ymm2
@@ -2218,7 +2218,7 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,3,1,4,6,7,5]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    vshufps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm0 = xmm0[3,3],mem[3,3]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
@@ -2227,32 +2227,32 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6],ymm1[7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX-NEXT:    vmovaps (%rsp), %xmm6 # 16-byte Reload
 ; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm6[2,2,2,2]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
 ; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3]
 ; AVX-NEXT:    vbroadcastsd 40(%rax), %ymm2
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4],ymm0[5,6,7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3],xmm14[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm14[3,3],xmm13[3,3]
 ; AVX-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm2 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm2 = xmm12[2],mem[2],xmm12[3],mem[3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm2, %ymm2
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5,6],ymm2[7]
-; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm14[2,2,2,2]
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm13[2,2,2,2]
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
 ; AVX-NEXT:    vblendps {{.*#+}} xmm2 = xmm12[0,1,2],xmm2[3]
 ; AVX-NEXT:    vbroadcastsd 8(%rax), %ymm3
 ; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3,4],ymm1[5,6,7]
-; AVX-NEXT:    vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm2 # 32-byte Folded Reload
-; AVX-NEXT:    # ymm2 = ymm11[0],mem[0],ymm11[1],mem[1],ymm11[4],mem[4],ymm11[5],mem[5]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX-NEXT:    vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm11[0],ymm2[1],ymm11[1],ymm2[4],ymm11[4],ymm2[5],ymm11[5]
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX-NEXT:    vunpcklpd {{.*#+}} ymm3 = ymm3[0],ymm13[0],ymm3[2],ymm13[2]
-; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm13[3,1],ymm3[0,2],ymm13[7,5],ymm3[4,6]
+; AVX-NEXT:    vunpcklpd {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[2],ymm5[2]
+; AVX-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[3,1],ymm3[0,2],ymm5[7,5],ymm3[4,6]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5],ymm3[6,7]
 ; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[3,3],xmm6[3,3]
 ; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
@@ -2262,7 +2262,7 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vunpcklpd {{.*#+}} ymm5 = ymm8[0],ymm7[0],ymm8[2],ymm7[2]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm4 = ymm7[3,1],ymm5[0,2],ymm7[7,5],ymm5[4,6]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5],ymm4[6,7]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm12[3,3],xmm14[3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm12[3,3],xmm13[3,3]
 ; AVX-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
 ; AVX-NEXT:    # xmm4 = xmm4[0,1,2],mem[3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4,5,6,7]
@@ -2290,7 +2290,7 @@ define void @store_i32_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vmovaps %ymm0, 416(%rax)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 384(%rax)
-; AVX-NEXT:    addq $456, %rsp # imm = 0x1C8
+; AVX-NEXT:    addq $472, %rsp # imm = 0x1D8
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -4619,7 +4619,7 @@ define void @store_i32_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[0,1],ymm1[2,0],ymm4[4,5],ymm1[6,4]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1,2,2,5,5,6,6]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm0[2,3]
@@ -9613,7 +9613,7 @@ define void @store_i32_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,0],ymm5[4,5],ymm1[6,4]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1,2,2,5,5,6,6]
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7]
 ; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm3[2,3],ymm0[2,3]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll
index cf246e4ede08..29be2b20fa93 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll
@@ -427,50 +427,50 @@ define void @store_i32_stride8_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; AVX-NEXT:    vmovaps (%rdi), %xmm0
-; AVX-NEXT:    vmovaps (%rsi), %xmm1
-; AVX-NEXT:    vmovaps (%rdx), %xmm2
-; AVX-NEXT:    vmovaps (%rcx), %xmm3
+; AVX-NEXT:    vmovaps (%rdi), %xmm2
+; AVX-NEXT:    vmovaps (%rsi), %xmm3
+; AVX-NEXT:    vmovaps (%rdx), %xmm0
+; AVX-NEXT:    vmovaps (%rcx), %xmm1
 ; AVX-NEXT:    vmovaps (%r8), %xmm4
 ; AVX-NEXT:    vmovaps (%r9), %xmm5
 ; AVX-NEXT:    vmovaps (%r11), %xmm6
 ; AVX-NEXT:    vmovaps (%r10), %xmm7
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm8 = xmm7[0],xmm6[0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,0]
-; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm8
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm9 = xmm3[0],xmm2[0]
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm10 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,0]
-; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm9 = xmm4[1],xmm5[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm10 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm10[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm9 = xmm4[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0],xmm5[1],xmm9[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm8[2,3]
 ; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm9
-; AVX-NEXT:    vinsertps {{.*#+}} xmm10 = xmm0[1],xmm1[1],zero,zero
-; AVX-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm11 = zero,zero,xmm6[2],xmm7[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm0, %ymm10
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm12 = zero,zero,xmm2[2],xmm3[2]
-; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm11[0,1],xmm12[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[3,0],xmm5[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,2],xmm6[2,3]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm10 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-NEXT:    vshufps {{.*#+}} xmm11 = xmm2[1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm11[0],xmm3[1],xmm11[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm11 = xmm11[0,1],xmm10[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm11 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm12 = xmm7[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm6[0,1,2],xmm12[3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm12 = xmm11[0,1],xmm12[2,3]
+; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm12
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm13 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm14 = xmm1[2,2,2,2]
+; AVX-NEXT:    vblendps {{.*#+}} xmm14 = xmm0[0,1,2],xmm14[3]
+; AVX-NEXT:    vblendps {{.*#+}} xmm14 = xmm13[0,1],xmm14[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm8[0]
 ; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
-; AVX-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,0],xmm1[3,0]
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm2[0],xmm10[0]
+; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX-NEXT:    vunpckhpd {{.*#+}} xmm3 = xmm11[1],xmm3[1]
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm13[1],xmm0[1]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
 ; AVX-NEXT:    vmovaps %ymm0, 96(%rax)
-; AVX-NEXT:    vmovaps %ymm10, 64(%rax)
+; AVX-NEXT:    vmovaps %ymm2, (%rax)
+; AVX-NEXT:    vmovaps %ymm12, 64(%rax)
 ; AVX-NEXT:    vmovaps %ymm9, 32(%rax)
-; AVX-NEXT:    vmovaps %ymm8, (%rax)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-2.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-2.ll
index 1061c08dcced..3e9fed78b56b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-2.ll
@@ -163,13 +163,11 @@ define void @store_i64_stride2_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ;
 ; AVX-LABEL: store_i64_stride2_vf4:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps (%rsi), %xmm0
-; AVX-NEXT:    vmovaps (%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[3],ymm1[3]
 ; AVX-NEXT:    vmovapd %ymm1, 32(%rdx)
 ; AVX-NEXT:    vmovapd %ymm0, (%rdx)
@@ -343,26 +341,22 @@ define void @store_i64_stride2_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ;
 ; AVX-LABEL: store_i64_stride2_vf8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps (%rsi), %xmm0
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm1
-; AVX-NEXT:    vmovaps (%rdi), %xmm2
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm3
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm4 = xmm2[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm2[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm3[1],xmm1[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm1[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm2 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[3],ymm1[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[3],ymm2[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm3 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[3],ymm3[3]
 ; AVX-NEXT:    vmovapd %ymm3, 96(%rdx)
-; AVX-NEXT:    vmovapd %ymm2, 32(%rdx)
-; AVX-NEXT:    vmovaps %ymm1, 64(%rdx)
-; AVX-NEXT:    vmovapd %ymm0, (%rdx)
+; AVX-NEXT:    vmovapd %ymm2, 64(%rdx)
+; AVX-NEXT:    vmovapd %ymm1, (%rdx)
+; AVX-NEXT:    vmovapd %ymm0, 32(%rdx)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -617,46 +611,38 @@ define void @store_i64_stride2_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ;
 ; AVX-LABEL: store_i64_stride2_vf16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps (%rsi), %xmm0
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm1
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm2
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm3
-; AVX-NEXT:    vmovaps (%rdi), %xmm4
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm5
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm6
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm7
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm8 = xmm7[1],xmm3[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm7[0],xmm3[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm3
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm7 = xmm6[1],xmm2[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm6[0],xmm2[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm2
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm4[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm4[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm4 = xmm5[1],xmm1[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm5[0],xmm1[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm4 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[3],ymm1[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[3],ymm2[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[3],ymm3[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[3],ymm4[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm5 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm5 = ymm6[0],ymm5[0],ymm6[3],ymm5[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm6 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[3],ymm6[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[3],ymm7[3]
 ; AVX-NEXT:    vmovapd %ymm7, 32(%rdx)
 ; AVX-NEXT:    vmovapd %ymm6, 96(%rdx)
-; AVX-NEXT:    vmovapd %ymm5, 160(%rdx)
-; AVX-NEXT:    vmovapd %ymm4, 224(%rdx)
-; AVX-NEXT:    vmovaps %ymm1, 64(%rdx)
-; AVX-NEXT:    vmovapd %ymm0, (%rdx)
-; AVX-NEXT:    vmovaps %ymm2, 128(%rdx)
-; AVX-NEXT:    vmovaps %ymm3, 192(%rdx)
+; AVX-NEXT:    vmovapd %ymm5, 64(%rdx)
+; AVX-NEXT:    vmovapd %ymm4, (%rdx)
+; AVX-NEXT:    vmovapd %ymm3, 160(%rdx)
+; AVX-NEXT:    vmovapd %ymm2, 128(%rdx)
+; AVX-NEXT:    vmovapd %ymm1, 192(%rdx)
+; AVX-NEXT:    vmovapd %ymm0, 224(%rdx)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -1117,70 +1103,54 @@ define void @store_i64_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ;
 ; AVX-LABEL: store_i64_stride2_vf32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps 224(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 224(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
 ; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rsi), %xmm1
-; AVX-NEXT:    vmovaps 128(%rdi), %xmm2
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm3 = xmm2[1],xmm1[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX-NEXT:    vmovaps (%rsi), %xmm2
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm3
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm4
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm5
-; AVX-NEXT:    vmovaps (%rdi), %xmm6
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm7
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm8
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm9
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm10 = xmm6[1],xmm2[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm6[0],xmm2[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm2, %ymm2
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm7[1],xmm3[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm3 = xmm7[0],xmm3[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm8[1],xmm4[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm4 = xmm8[0],xmm4[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm6 = xmm9[1],xmm5[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm5 = xmm9[0],xmm5[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm5
-; AVX-NEXT:    vmovaps 160(%rsi), %xmm6
-; AVX-NEXT:    vmovaps 160(%rdi), %xmm7
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm8 = xmm7[1],xmm6[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm6 = xmm7[0],xmm6[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm6, %ymm6
-; AVX-NEXT:    vmovaps 192(%rsi), %xmm7
-; AVX-NEXT:    vmovaps 192(%rdi), %xmm8
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm9 = xmm8[1],xmm7[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm7 = xmm8[0],xmm7[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm7, %ymm7
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[3],ymm1[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[3],ymm2[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[3],ymm3[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[3],ymm4[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm5 = ymm6[0],ymm5[0],ymm6[3],ymm5[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[3],ymm6[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[3],ymm7[3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[3],ymm8[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[3],ymm9[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[3],ymm10[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm12 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[3],ymm11[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm12 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm12 = ymm13[0],ymm12[0],ymm13[3],ymm12[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm14 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm13 = ymm14[0],ymm13[0],ymm14[3],ymm13[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm14 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm15 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm14 = ymm15[0],ymm14[0],ymm15[3],ymm14[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm15 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm15[0],ymm0[3],ymm15[3]
 ; AVX-NEXT:    vmovapd %ymm0, 416(%rdx)
 ; AVX-NEXT:    vmovapd %ymm14, 352(%rdx)
@@ -1188,17 +1158,17 @@ define void @store_i64_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX-NEXT:    vmovapd %ymm12, 32(%rdx)
 ; AVX-NEXT:    vmovapd %ymm11, 96(%rdx)
 ; AVX-NEXT:    vmovapd %ymm10, 160(%rdx)
-; AVX-NEXT:    vmovapd %ymm9, 288(%rdx)
-; AVX-NEXT:    vmovapd %ymm8, 480(%rdx)
-; AVX-NEXT:    vmovaps %ymm7, 384(%rdx)
-; AVX-NEXT:    vmovaps %ymm6, 320(%rdx)
-; AVX-NEXT:    vmovaps %ymm5, 192(%rdx)
-; AVX-NEXT:    vmovaps %ymm4, 128(%rdx)
-; AVX-NEXT:    vmovaps %ymm3, 64(%rdx)
-; AVX-NEXT:    vmovaps %ymm2, (%rdx)
-; AVX-NEXT:    vmovaps %ymm1, 256(%rdx)
+; AVX-NEXT:    vmovapd %ymm9, 384(%rdx)
+; AVX-NEXT:    vmovapd %ymm8, 320(%rdx)
+; AVX-NEXT:    vmovapd %ymm7, 192(%rdx)
+; AVX-NEXT:    vmovapd %ymm6, 128(%rdx)
+; AVX-NEXT:    vmovapd %ymm5, 64(%rdx)
+; AVX-NEXT:    vmovapd %ymm4, (%rdx)
+; AVX-NEXT:    vmovapd %ymm3, 288(%rdx)
+; AVX-NEXT:    vmovapd %ymm2, 256(%rdx)
+; AVX-NEXT:    vmovapd %ymm1, 448(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 448(%rdx)
+; AVX-NEXT:    vmovaps %ymm0, 480(%rdx)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
@@ -2080,199 +2050,167 @@ define void @store_i64_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
 ; AVX-LABEL: store_i64_stride2_vf64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    subq $424, %rsp # imm = 0x1A8
-; AVX-NEXT:    vmovaps (%rsi), %xmm0
-; AVX-NEXT:    vmovaps 32(%rsi), %xmm1
-; AVX-NEXT:    vmovaps 64(%rsi), %xmm2
-; AVX-NEXT:    vmovaps 96(%rsi), %xmm3
-; AVX-NEXT:    vmovaps (%rdi), %xmm4
-; AVX-NEXT:    vmovaps 32(%rdi), %xmm5
-; AVX-NEXT:    vmovaps 64(%rdi), %xmm6
-; AVX-NEXT:    vmovaps 96(%rdi), %xmm7
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm8 = xmm4[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm4[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm5[1],xmm1[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm5[0],xmm1[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm6[1],xmm2[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm6[0],xmm2[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm7[1],xmm3[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm7[0],xmm3[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 128(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 128(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 160(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 160(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 192(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 192(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 224(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 224(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 256(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 256(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 288(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 288(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 320(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 320(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 352(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 352(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 384(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 384(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX-NEXT:    vmovaps 416(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 416(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 448(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 448(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vmovaps 480(%rsi), %xmm0
-; AVX-NEXT:    vmovaps 480(%rdi), %xmm1
-; AVX-NEXT:    vunpckhpd {{.*#+}} xmm2 = xmm1[1],xmm0[1]
-; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
 ; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
 ; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, (%rsp) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm6 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm4 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm3 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm8[0],ymm0[0],ymm8[3],ymm0[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm8 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[3],ymm8[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm9 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[3],ymm9[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm10 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[3],ymm10[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm11 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm12 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[3],ymm11[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm12 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm12 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm12 = ymm13[0],ymm12[0],ymm13[3],ymm12[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm13 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm14 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm13 = ymm14[0],ymm13[0],ymm14[3],ymm13[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm14 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm14 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm15 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm14 = ymm15[0],ymm14[0],ymm15[3],ymm14[3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm15 = mem[2,3,2,3]
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm7 = mem[2,3,2,3]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm15 = mem[0,1,0,1]
+; AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = mem[0,1,0,1]
 ; AVX-NEXT:    vshufpd {{.*#+}} ymm7 = ymm7[0],ymm15[0],ymm7[3],ymm15[3]
 ; AVX-NEXT:    vmovapd %ymm7, 992(%rdx)
-; AVX-NEXT:    vmovapd %ymm14, 928(%rdx)
-; AVX-NEXT:    vmovapd %ymm13, 864(%rdx)
-; AVX-NEXT:    vmovapd %ymm12, 800(%rdx)
-; AVX-NEXT:    vmovapd %ymm11, 736(%rdx)
-; AVX-NEXT:    vmovapd %ymm10, 672(%rdx)
-; AVX-NEXT:    vmovapd %ymm9, 608(%rdx)
-; AVX-NEXT:    vmovapd %ymm8, 544(%rdx)
-; AVX-NEXT:    vmovapd %ymm0, 480(%rdx)
-; AVX-NEXT:    vmovapd %ymm1, 416(%rdx)
-; AVX-NEXT:    vmovapd %ymm2, 352(%rdx)
-; AVX-NEXT:    vmovapd %ymm3, 288(%rdx)
-; AVX-NEXT:    vmovapd %ymm4, 224(%rdx)
-; AVX-NEXT:    vmovapd %ymm5, 160(%rdx)
-; AVX-NEXT:    vmovapd %ymm6, 96(%rdx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 32(%rdx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 960(%rdx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 896(%rdx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 832(%rdx)
-; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 768(%rdx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 704(%rdx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 640(%rdx)
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX-NEXT:    vmovaps %ymm0, 576(%rdx)
+; AVX-NEXT:    vmovapd %ymm14, 960(%rdx)
+; AVX-NEXT:    vmovapd %ymm13, 928(%rdx)
+; AVX-NEXT:    vmovapd %ymm12, 896(%rdx)
+; AVX-NEXT:    vmovapd %ymm11, 864(%rdx)
+; AVX-NEXT:    vmovapd %ymm10, 832(%rdx)
+; AVX-NEXT:    vmovapd %ymm9, 800(%rdx)
+; AVX-NEXT:    vmovapd %ymm8, 768(%rdx)
+; AVX-NEXT:    vmovapd %ymm0, 736(%rdx)
+; AVX-NEXT:    vmovapd %ymm1, 704(%rdx)
+; AVX-NEXT:    vmovapd %ymm2, 672(%rdx)
+; AVX-NEXT:    vmovapd %ymm3, 640(%rdx)
+; AVX-NEXT:    vmovapd %ymm4, 608(%rdx)
+; AVX-NEXT:    vmovapd %ymm5, 576(%rdx)
+; AVX-NEXT:    vmovapd %ymm6, 544(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 512(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 480(%rdx)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 448(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 416(%rdx)
+; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 384(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 352(%rdx)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 320(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 288(%rdx)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 256(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 224(%rdx)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 192(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 160(%rdx)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 128(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 96(%rdx)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, 64(%rdx)
 ; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX-NEXT:    vmovaps %ymm0, 32(%rdx)
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX-NEXT:    vmovaps %ymm0, (%rdx)
 ; AVX-NEXT:    addq $424, %rsp # imm = 0x1A8
 ; AVX-NEXT:    vzeroupper
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-2.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-2.ll
index 53a6d306ef84..30be6c88514b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-2.ll
@@ -544,14 +544,14 @@ define void @store_i8_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
 ; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; AVX512-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i8_stride2_vf32:
@@ -560,14 +560,14 @@ define void @store_i8_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; AVX512-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i8_stride2_vf32:
@@ -576,14 +576,14 @@ define void @store_i8_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i8_stride2_vf32:
@@ -592,14 +592,14 @@ define void @store_i8_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, 48(%rdx)
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
 ; AVX512DQ-FCP-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i8_stride2_vf32:
@@ -787,22 +787,22 @@ define void @store_i8_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
 ; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
 ; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i8_stride2_vf64:
@@ -815,22 +815,22 @@ define void @store_i8_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512-FCP-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512-FCP-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i8_stride2_vf64:
@@ -843,22 +843,22 @@ define void @store_i8_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512DQ-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i8_stride2_vf64:
@@ -871,22 +871,22 @@ define void @store_i8_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rdi), %xmm7
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 96(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, 112(%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, 64(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, 80(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 48(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, 80(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 112(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, 96(%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, 16(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 32(%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, 48(%rdx)
 ; AVX512DQ-FCP-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i8_stride2_vf64:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
index 8802e8a77933..674bad2c7aa8 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
@@ -740,16 +740,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: store_i8_stride3_vf16:
@@ -763,16 +762,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512-FCP-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i8_stride3_vf16:
@@ -786,16 +784,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512DQ-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512DQ-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512DQ-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512DQ-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: store_i8_stride3_vf16:
@@ -809,16 +806,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512DQ-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512DQ-FCP-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i8_stride3_vf16:
@@ -832,16 +828,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512BW-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512BW-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512BW-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512BW-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512BW-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512BW-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512BW-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512BW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BW-FCP-LABEL: store_i8_stride3_vf16:
@@ -855,16 +850,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512BW-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512BW-FCP-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512BW-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512BW-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512BW-FCP-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512BW-FCP-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512BW-FCP-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
 ; AVX512DQ-BW-LABEL: store_i8_stride3_vf16:
@@ -878,16 +872,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512DQ-BW-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512DQ-BW-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512DQ-BW-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512DQ-BW-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512DQ-BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512DQ-BW-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512DQ-BW-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512DQ-BW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
 ; AVX512DQ-BW-FCP-LABEL: store_i8_stride3_vf16:
@@ -901,16 +894,15 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512DQ-BW-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512DQ-BW-FCP-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512DQ-BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm2, 32(%rcx)
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <16 x i8>, ptr %in.vecptr0, align 64
   %in.vec1 = load <16 x i8>, ptr %in.vecptr1, align 64
@@ -1639,119 +1631,122 @@ define void @store_i8_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ;
 ; AVX-LABEL: store_i8_stride3_vf64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    subq $24, %rsp
-; AVX-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX-NEXT:    subq $40, %rsp
+; AVX-NEXT:    vmovdqa (%rdi), %xmm8
 ; AVX-NEXT:    vmovdqa 16(%rdi), %xmm9
 ; AVX-NEXT:    vmovdqa 32(%rdi), %xmm6
-; AVX-NEXT:    vmovdqa 48(%rdi), %xmm2
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm8 = [11,12,13,14,15,0,1,2,3,4,5,128,128,128,128,128]
-; AVX-NEXT:    vpshufb %xmm8, %xmm7, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm7 = [11,12,13,14,15,0,1,2,3,4,5,128,128,128,128,128]
+; AVX-NEXT:    vpshufb %xmm7, %xmm8, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovdqa (%rdx), %xmm5
 ; AVX-NEXT:    vmovdqa 16(%rdx), %xmm1
-; AVX-NEXT:    vpshufb %xmm8, %xmm9, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpshufb %xmm8, %xmm6, %xmm0
+; AVX-NEXT:    vpshufb %xmm7, %xmm9, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpshufb %xmm8, %xmm2, %xmm8
+; AVX-NEXT:    vpshufb %xmm7, %xmm6, %xmm4
+; AVX-NEXT:    vpshufb %xmm7, %xmm3, %xmm7
 ; AVX-NEXT:    vmovdqa {{.*#+}} xmm10 = [u,u,u,u,u,128,128,128,128,128,128,6,7,8,9,10]
-; AVX-NEXT:    vpshufb %xmm10, %xmm2, %xmm2
+; AVX-NEXT:    vpshufb %xmm10, %xmm3, %xmm3
 ; AVX-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,u,u,u,u,5,6,7,8,9,10,128,128,128,128,128]
-; AVX-NEXT:    vmovdqa 16(%rsi), %xmm12
-; AVX-NEXT:    vmovdqa 32(%rsi), %xmm13
-; AVX-NEXT:    vmovdqa 48(%rsi), %xmm14
-; AVX-NEXT:    vpshufb %xmm11, %xmm14, %xmm15
-; AVX-NEXT:    vpor %xmm2, %xmm15, %xmm0
-; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovdqa 16(%rsi), %xmm13
+; AVX-NEXT:    vmovdqa 32(%rsi), %xmm14
+; AVX-NEXT:    vmovdqa 48(%rsi), %xmm15
+; AVX-NEXT:    vpshufb %xmm11, %xmm15, %xmm12
+; AVX-NEXT:    vpor %xmm3, %xmm12, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vpshufb %xmm10, %xmm6, %xmm6
-; AVX-NEXT:    vpshufb %xmm11, %xmm13, %xmm15
-; AVX-NEXT:    vpor %xmm6, %xmm15, %xmm0
+; AVX-NEXT:    vpshufb %xmm11, %xmm14, %xmm12
+; AVX-NEXT:    vpor %xmm6, %xmm12, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vpshufb %xmm10, %xmm9, %xmm9
-; AVX-NEXT:    vpshufb %xmm11, %xmm12, %xmm15
-; AVX-NEXT:    vpor %xmm9, %xmm15, %xmm0
+; AVX-NEXT:    vpshufb %xmm11, %xmm13, %xmm12
+; AVX-NEXT:    vpor %xmm9, %xmm12, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vmovdqa (%rsi), %xmm15
-; AVX-NEXT:    vpshufb %xmm10, %xmm7, %xmm7
-; AVX-NEXT:    vpshufb %xmm11, %xmm15, %xmm10
-; AVX-NEXT:    vpor %xmm7, %xmm10, %xmm0
+; AVX-NEXT:    vmovdqa (%rsi), %xmm2
+; AVX-NEXT:    vpshufb %xmm10, %xmm8, %xmm8
+; AVX-NEXT:    vpshufb %xmm11, %xmm2, %xmm10
+; AVX-NEXT:    vpor %xmm8, %xmm10, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm3[8],xmm15[8],xmm3[9],xmm15[9],xmm3[10],xmm15[10],xmm3[11],xmm15[11],xmm3[12],xmm15[12],xmm3[13],xmm15[13],xmm3[14],xmm15[14],xmm3[15],xmm15[15]
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm10 = [u,u,u,u,u,4,6,8,10,12,14,7,9,11,13,15]
-; AVX-NEXT:    vpshufb %xmm10, %xmm7, %xmm6
-; AVX-NEXT:    vmovdqa %xmm1, %xmm0
-; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm1[8],xmm12[8],xmm1[9],xmm12[9],xmm1[10],xmm12[10],xmm1[11],xmm12[11],xmm1[12],xmm12[12],xmm1[13],xmm12[13],xmm1[14],xmm12[14],xmm1[15],xmm12[15]
-; AVX-NEXT:    vpshufb %xmm10, %xmm7, %xmm5
-; AVX-NEXT:    vmovdqa 32(%rdx), %xmm11
-; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm11[8],xmm13[8],xmm11[9],xmm13[9],xmm11[10],xmm13[10],xmm11[11],xmm13[11],xmm11[12],xmm13[12],xmm11[13],xmm13[13],xmm11[14],xmm13[14],xmm11[15],xmm13[15]
-; AVX-NEXT:    vpshufb %xmm10, %xmm7, %xmm4
-; AVX-NEXT:    vmovdqa 48(%rdx), %xmm7
-; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm7[8],xmm14[8],xmm7[9],xmm14[9],xmm7[10],xmm14[10],xmm7[11],xmm14[11],xmm7[12],xmm14[12],xmm7[13],xmm14[13],xmm7[14],xmm14[14],xmm7[15],xmm14[15]
-; AVX-NEXT:    vpshufb %xmm10, %xmm9, %xmm9
-; AVX-NEXT:    vpslldq {{.*#+}} xmm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm7[0,1,2,3,4]
-; AVX-NEXT:    vpor %xmm10, %xmm8, %xmm10
-; AVX-NEXT:    vpalignr {{.*#+}} xmm1 = xmm10[5,6,7,8,9,10,11,12,13,14,15],xmm14[0,1,2,3,4]
-; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpslldq {{.*#+}} xmm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm11[0,1,2,3,4]
-; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX-NEXT:    vpor %xmm2, %xmm14, %xmm14
-; AVX-NEXT:    vpalignr {{.*#+}} xmm13 = xmm14[5,6,7,8,9,10,11,12,13,14,15],xmm13[0,1,2,3,4]
-; AVX-NEXT:    vpslldq {{.*#+}} xmm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
-; AVX-NEXT:    vmovdqa %xmm0, %xmm10
+; AVX-NEXT:    vmovdqa %xmm5, %xmm0
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm11 = [u,u,u,u,u,4,6,8,10,12,14,7,9,11,13,15]
+; AVX-NEXT:    vpshufb %xmm11, %xmm8, %xmm8
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm1[8],xmm13[8],xmm1[9],xmm13[9],xmm1[10],xmm13[10],xmm1[11],xmm13[11],xmm1[12],xmm13[12],xmm1[13],xmm13[13],xmm1[14],xmm13[14],xmm1[15],xmm13[15]
+; AVX-NEXT:    vpshufb %xmm11, %xmm10, %xmm6
+; AVX-NEXT:    vmovdqa 32(%rdx), %xmm12
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm12[8],xmm14[8],xmm12[9],xmm14[9],xmm12[10],xmm14[10],xmm12[11],xmm14[11],xmm12[12],xmm14[12],xmm12[13],xmm14[13],xmm12[14],xmm14[14],xmm12[15],xmm14[15]
+; AVX-NEXT:    vpshufb %xmm11, %xmm10, %xmm5
+; AVX-NEXT:    vmovdqa 48(%rdx), %xmm10
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm10[8],xmm15[8],xmm10[9],xmm15[9],xmm10[10],xmm15[10],xmm10[11],xmm15[11],xmm10[12],xmm15[12],xmm10[13],xmm15[13],xmm10[14],xmm15[14],xmm10[15],xmm15[15]
+; AVX-NEXT:    vpshufb %xmm11, %xmm9, %xmm9
+; AVX-NEXT:    vpslldq {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm10[0,1,2,3,4]
+; AVX-NEXT:    vpor %xmm7, %xmm11, %xmm11
+; AVX-NEXT:    vpalignr {{.*#+}} xmm3 = xmm11[5,6,7,8,9,10,11,12,13,14,15],xmm15[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpslldq {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm12[0,1,2,3,4]
+; AVX-NEXT:    vpor %xmm4, %xmm11, %xmm11
+; AVX-NEXT:    vpalignr {{.*#+}} xmm3 = xmm11[5,6,7,8,9,10,11,12,13,14,15],xmm14[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpslldq {{.*#+}} xmm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa %xmm1, %xmm11
 ; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX-NEXT:    vpor %xmm1, %xmm14, %xmm14
-; AVX-NEXT:    vpalignr {{.*#+}} xmm12 = xmm14[5,6,7,8,9,10,11,12,13,14,15],xmm12[0,1,2,3,4]
-; AVX-NEXT:    vpslldq {{.*#+}} xmm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4]
-; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vpalignr {{.*#+}} xmm3 = xmm14[5,6,7,8,9,10,11,12,13,14,15],xmm13[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpslldq {{.*#+}} xmm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa %xmm0, %xmm3
+; AVX-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    vpor %xmm0, %xmm14, %xmm14
-; AVX-NEXT:    vpalignr {{.*#+}} xmm14 = xmm14[5,6,7,8,9,10,11,12,13,14,15],xmm15[0,1,2,3,4]
-; AVX-NEXT:    vpalignr {{.*#+}} xmm8 = xmm9[5,6,7,8,9,10,11,12,13,14,15],xmm8[0,1,2,3,4]
-; AVX-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX-NEXT:    vpalignr {{.*#+}} xmm15 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX-NEXT:    vpalignr {{.*#+}} xmm2 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX-NEXT:    vpalignr {{.*#+}} xmm4 = xmm6[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm5 = [5,128,11,6,128,12,7,128,13,8,128,14,9,128,15,10]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm14 = xmm14[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm2 = xmm9[5,6,7,8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vpalignr {{.*#+}} xmm13 = xmm5[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm15 = xmm6[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX-NEXT:    vpalignr {{.*#+}} xmm1 = xmm8[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm8 = [5,128,11,6,128,12,7,128,13,8,128,14,9,128,15,10]
 ; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
+; AVX-NEXT:    vpshufb %xmm8, %xmm0, %xmm6
 ; AVX-NEXT:    vmovdqa {{.*#+}} xmm9 = [128,5,128,128,6,128,128,7,128,128,8,128,128,9,128,128]
-; AVX-NEXT:    vpshufb %xmm9, %xmm3, %xmm1
-; AVX-NEXT:    vpor %xmm1, %xmm6, %xmm1
+; AVX-NEXT:    vpshufb %xmm9, %xmm3, %xmm2
+; AVX-NEXT:    vpor %xmm2, %xmm6, %xmm2
 ; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
-; AVX-NEXT:    vpshufb %xmm9, %xmm10, %xmm3
-; AVX-NEXT:    vpor %xmm3, %xmm6, %xmm3
+; AVX-NEXT:    vpshufb %xmm8, %xmm0, %xmm6
+; AVX-NEXT:    vpshufb %xmm9, %xmm11, %xmm4
+; AVX-NEXT:    vpor %xmm4, %xmm6, %xmm4
 ; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
-; AVX-NEXT:    vpshufb %xmm9, %xmm11, %xmm11
-; AVX-NEXT:    vpor %xmm6, %xmm11, %xmm6
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX-NEXT:    vpshufb %xmm11, %xmm14, %xmm14
-; AVX-NEXT:    vpshufb %xmm11, %xmm4, %xmm4
-; AVX-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
-; AVX-NEXT:    vpshufb %xmm11, %xmm2, %xmm2
-; AVX-NEXT:    vpshufb %xmm11, %xmm13, %xmm13
-; AVX-NEXT:    vpshufb %xmm11, %xmm15, %xmm0
-; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX-NEXT:    vpshufb %xmm11, %xmm8, %xmm10
-; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX-NEXT:    vpshufb %xmm11, %xmm8, %xmm8
-; AVX-NEXT:    vmovdqa (%rsp), %xmm11 # 16-byte Reload
-; AVX-NEXT:    vpshufb %xmm5, %xmm11, %xmm5
+; AVX-NEXT:    vpshufb %xmm8, %xmm0, %xmm6
+; AVX-NEXT:    vpshufb %xmm9, %xmm12, %xmm12
+; AVX-NEXT:    vpor %xmm6, %xmm12, %xmm6
+; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    vpshufb %xmm8, %xmm0, %xmm8
+; AVX-NEXT:    vpshufb %xmm9, %xmm10, %xmm9
+; AVX-NEXT:    vpor %xmm9, %xmm8, %xmm8
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm9 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX-NEXT:    vpshufb %xmm9, %xmm14, %xmm0
+; AVX-NEXT:    vpshufb %xmm9, %xmm1, %xmm5
+; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vpshufb %xmm9, %xmm1, %xmm10
+; AVX-NEXT:    vpshufb %xmm9, %xmm15, %xmm3
+; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX-NEXT:    vpshufb %xmm9, %xmm1, %xmm11
+; AVX-NEXT:    vpshufb %xmm9, %xmm13, %xmm1
+; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX-NEXT:    vpshufb %xmm9, %xmm7, %xmm12
+; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
 ; AVX-NEXT:    vpshufb %xmm9, %xmm7, %xmm7
-; AVX-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX-NEXT:    vmovdqa %xmm3, 64(%rcx)
-; AVX-NEXT:    vmovdqa %xmm2, 80(%rcx)
-; AVX-NEXT:    vmovdqa %xmm14, (%rcx)
-; AVX-NEXT:    vmovdqa %xmm1, 16(%rcx)
-; AVX-NEXT:    vmovdqa %xmm4, 32(%rcx)
-; AVX-NEXT:    vmovdqa %xmm12, 48(%rcx)
-; AVX-NEXT:    vmovdqa %xmm5, 160(%rcx)
-; AVX-NEXT:    vmovdqa %xmm8, 176(%rcx)
-; AVX-NEXT:    vmovdqa %xmm13, 96(%rcx)
+; AVX-NEXT:    vmovdqa %xmm3, 80(%rcx)
+; AVX-NEXT:    vmovdqa %xmm4, 64(%rcx)
+; AVX-NEXT:    vmovdqa %xmm0, (%rcx)
+; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)
+; AVX-NEXT:    vmovdqa %xmm5, 32(%rcx)
+; AVX-NEXT:    vmovdqa %xmm10, 48(%rcx)
+; AVX-NEXT:    vmovdqa %xmm7, 176(%rcx)
+; AVX-NEXT:    vmovdqa %xmm8, 160(%rcx)
+; AVX-NEXT:    vmovdqa %xmm11, 96(%rcx)
 ; AVX-NEXT:    vmovdqa %xmm6, 112(%rcx)
-; AVX-NEXT:    vmovdqa %xmm0, 128(%rcx)
-; AVX-NEXT:    vmovdqa %xmm10, 144(%rcx)
-; AVX-NEXT:    addq $24, %rsp
+; AVX-NEXT:    vmovdqa %xmm1, 128(%rcx)
+; AVX-NEXT:    vmovdqa %xmm12, 144(%rcx)
+; AVX-NEXT:    addq $40, %rsp
 ; AVX-NEXT:    retq
 ;
 ; AVX2-LABEL: store_i8_stride3_vf64:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index 86efcf9c5761..ad9db98711a6 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -1190,8 +1190,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[2,3,0,1]
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,ymm5[7],zero,zero,zero,zero,ymm5[8],zero,zero,zero,zero,ymm5[9],zero,zero,zero,ymm5[26],zero,zero,zero,zero,ymm5[27],zero,zero,zero,zero,ymm5[28],zero,zero,zero
 ; AVX2-NEXT:    vpor %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255]
-; AVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,2,2]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255]
@@ -1233,8 +1232,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[2,3,0,1]
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,ymm5[7],zero,zero,zero,zero,ymm5[8],zero,zero,zero,zero,ymm5[9],zero,zero,zero,ymm5[26],zero,zero,zero,zero,ymm5[27],zero,zero,zero,zero,ymm5[28],zero,zero,zero
 ; AVX2-FP-NEXT:    vpor %ymm4, %ymm5, %ymm4
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255,u,0,0,255,255]
-; AVX2-FP-NEXT:    vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
+; AVX2-FP-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,2,2]
 ; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255]
@@ -1461,20 +1459,18 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[7],zero,zero,zero,zero,ymm4[8],zero,zero,zero,zero,ymm4[9],zero,zero,zero,ymm4[26],zero,zero,zero,zero,ymm4[27],zero,zero,zero,zero,ymm4[28],zero,zero,zero
 ; AVX512BW-NEXT:    vpor %ymm4, %ymm3, %ymm3
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[0,2,2,0]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[0,8],zero,zero,zero,ymm4[1,9],zero,zero,zero,ymm4[2,10],zero,zero,zero,ymm4[3,19],zero,zero,zero,ymm4[28,20],zero,zero,zero,ymm4[29,21],zero,zero,zero,ymm4[30,22]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
 ; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm1[6],zero,zero,zero,zero,ymm1[7],zero,zero,zero,zero,ymm1[8],zero,zero,zero,zero,ymm1[9,25],zero,zero,zero,zero,ymm1[26],zero,zero,zero,zero,ymm1[27],zero,zero,zero,zero,ymm1[28]
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[2,3,0,1]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm5 = zero,ymm5[6],zero,zero,zero,zero,ymm5[7],zero,zero,zero,zero,ymm5[8],zero,zero,zero,zero,zero,zero,zero,zero,ymm5[26],zero,zero,zero,zero,ymm5[27],zero,zero,zero,zero,ymm5[28],zero
 ; AVX512BW-NEXT:    vpor %ymm4, %ymm5, %ymm4
-; AVX512BW-NEXT:    movl $831283992, %eax # imm = 0x318C6318
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqu8 %ymm3, %ymm4 {%k1}
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[0,2,2,0]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,8],zero,zero,zero,ymm3[1,9],zero,zero,zero,ymm3[2,10],zero,zero,zero,ymm3[3,19],zero,zero,zero,ymm3[28,20],zero,zero,zero,ymm3[29,21],zero,zero,zero,ymm3[30,22]
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[0,2,0,2]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,ymm5[0,8],zero,zero,zero,ymm5[1,9],zero,zero,zero,ymm5[2,10],zero,zero,zero,ymm5[19,27],zero,zero,zero,ymm5[20,28],zero,zero,zero,ymm5[21,29],zero,zero,zero
-; AVX512BW-NEXT:    vpor %ymm3, %ymm5, %ymm3
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
+; AVX512BW-NEXT:    vporq %zmm3, %zmm4, %zmm3
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,2]
 ; AVX512BW-NEXT:    vpermd %zmm2, %zmm4, %zmm4
 ; AVX512BW-NEXT:    movabsq $595056260442243600, %rax # imm = 0x842108421084210
@@ -1531,20 +1527,18 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[7],zero,zero,zero,zero,ymm4[8],zero,zero,zero,zero,ymm4[9],zero,zero,zero,ymm4[26],zero,zero,zero,zero,ymm4[27],zero,zero,zero,zero,ymm4[28],zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vpor %ymm4, %ymm3, %ymm3
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[0,2,2,0]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[0,8],zero,zero,zero,ymm4[1,9],zero,zero,zero,ymm4[2,10],zero,zero,zero,ymm4[3,19],zero,zero,zero,ymm4[28,20],zero,zero,zero,ymm4[29,21],zero,zero,zero,ymm4[30,22]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm1[6],zero,zero,zero,zero,ymm1[7],zero,zero,zero,zero,ymm1[8],zero,zero,zero,zero,ymm1[9,25],zero,zero,zero,zero,ymm1[26],zero,zero,zero,zero,ymm1[27],zero,zero,zero,zero,ymm1[28]
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[2,3,0,1]
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm5 = zero,ymm5[6],zero,zero,zero,zero,ymm5[7],zero,zero,zero,zero,ymm5[8],zero,zero,zero,zero,zero,zero,zero,zero,ymm5[26],zero,zero,zero,zero,ymm5[27],zero,zero,zero,zero,ymm5[28],zero
 ; AVX512DQ-BW-NEXT:    vpor %ymm4, %ymm5, %ymm4
-; AVX512DQ-BW-NEXT:    movl $831283992, %eax # imm = 0x318C6318
-; AVX512DQ-BW-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu8 %ymm3, %ymm4 {%k1}
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[0,2,2,0]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,8],zero,zero,zero,ymm3[1,9],zero,zero,zero,ymm3[2,10],zero,zero,zero,ymm3[3,19],zero,zero,zero,ymm3[28,20],zero,zero,zero,ymm3[29,21],zero,zero,zero,ymm3[30,22]
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[0,2,0,2]
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,ymm5[0,8],zero,zero,zero,ymm5[1,9],zero,zero,zero,ymm5[2,10],zero,zero,zero,ymm5[19,27],zero,zero,zero,ymm5[20,28],zero,zero,zero,ymm5[21,29],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpor %ymm3, %ymm5, %ymm3
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
+; AVX512DQ-BW-NEXT:    vporq %zmm3, %zmm4, %zmm3
 ; AVX512DQ-BW-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,2]
 ; AVX512DQ-BW-NEXT:    vpermd %zmm2, %zmm4, %zmm4
 ; AVX512DQ-BW-NEXT:    movabsq $595056260442243600, %rax # imm = 0x842108421084210
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
index 887db41dcf8a..03f5b90002d3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
@@ -2996,94 +2996,88 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm5
-; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm6
+; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm4
+; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm5
 ; AVX512BW-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512BW-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
-; AVX512BW-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[2],ymm7[2],ymm4[3],ymm7[3],ymm4[4],ymm7[4],ymm4[5],ymm7[5],ymm4[6],ymm7[6],ymm4[7],ymm7[7],ymm4[16],ymm7[16],ymm4[17],ymm7[17],ymm4[18],ymm7[18],ymm4[19],ymm7[19],ymm4[20],ymm7[20],ymm4[21],ymm7[21],ymm4[22],ymm7[22],ymm4[23],ymm7[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15],ymm5[24],ymm6[24],ymm5[25],ymm6[25],ymm5[26],ymm6[26],ymm5[27],ymm6[27],ymm5[28],ymm6[28],ymm5[29],ymm6[29],ymm5[30],ymm6[30],ymm5[31],ymm6[31]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512BW-NEXT:    vpermw %ymm7, %ymm8, %ymm7
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm7
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512BW-NEXT:    vpshufb %ymm8, %ymm6, %ymm4
-; AVX512BW-NEXT:    vpshufb %ymm8, %ymm5, %ymm9
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm9[0],ymm4[0],ymm9[1],ymm4[1],ymm9[2],ymm4[2],ymm9[3],ymm4[3],ymm9[4],ymm4[4],ymm9[5],ymm4[5],ymm9[6],ymm4[6],ymm9[7],ymm4[7],ymm9[16],ymm4[16],ymm9[17],ymm4[17],ymm9[18],ymm4[18],ymm9[19],ymm4[19],ymm9[20],ymm4[20],ymm9[21],ymm4[21],ymm9[22],ymm4[22],ymm9[23],ymm4[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512BW-NEXT:    vpermw %ymm9, %ymm10, %ymm9
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm9, %zmm4, %zmm4
-; AVX512BW-NEXT:    movl $1227114788, %r10d # imm = 0x49244924
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-NEXT:    vpshufb %ymm6, %ymm3, %ymm7
+; AVX512BW-NEXT:    vpshufb %ymm6, %ymm2, %ymm6
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[4],ymm7[4],ymm6[5],ymm7[5],ymm6[6],ymm7[6],ymm6[7],ymm7[7],ymm6[16],ymm7[16],ymm6[17],ymm7[17],ymm6[18],ymm7[18],ymm6[19],ymm7[19],ymm6[20],ymm7[20],ymm6[21],ymm7[21],ymm6[22],ymm7[22],ymm6[23],ymm7[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512BW-NEXT:    vpshufb %ymm7, %ymm5, %ymm8
+; AVX512BW-NEXT:    vpshufb %ymm7, %ymm4, %ymm9
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512BW-NEXT:    movw $18724, %r10w # imm = 0x4924
 ; AVX512BW-NEXT:    kmovd %r10d, %k1
-; AVX512BW-NEXT:    vmovdqu16 %zmm7, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm1, %ymm9
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm0, %ymm10
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512BW-NEXT:    vpermw %ymm10, %ymm11, %ymm10
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512BW-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
-; AVX512BW-NEXT:    kmovd %r10d, %k1
-; AVX512BW-NEXT:    vmovdqu16 %zmm9, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm9
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm9, %xmm10
-; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm11
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm11, %xmm8
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm10
+; AVX512BW-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k1}
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512BW-NEXT:    vpshufb %ymm9, %ymm1, %ymm6
+; AVX512BW-NEXT:    vpshufb %ymm9, %ymm0, %ymm10
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm10[0],ymm6[0],ymm10[1],ymm6[1],ymm10[2],ymm6[2],ymm10[3],ymm6[3],ymm10[4],ymm6[4],ymm10[5],ymm6[5],ymm10[6],ymm6[6],ymm10[7],ymm6[7],ymm10[16],ymm6[16],ymm10[17],ymm6[17],ymm10[18],ymm6[18],ymm10[19],ymm6[19],ymm10[20],ymm6[20],ymm10[21],ymm6[21],ymm10[22],ymm6[22],ymm10[23],ymm6[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512BW-NEXT:    movw $9362, %r10w # imm = 0x2492
+; AVX512BW-NEXT:    kmovd %r10d, %k2
+; AVX512BW-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k2}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,11,27,0,12,28,0,13,29,0,14,30,0,15,31,0]
+; AVX512BW-NEXT:    vpermi2w %ymm6, %ymm10, %ymm11
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm6
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [42,17,18,43,20,21,44,23,24,45,26,27,46,29,30,47]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512BW-NEXT:    vpermi2w %zmm11, %zmm6, %zmm10
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm8
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm11
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm12
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm13, %zmm14
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} xmm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm10, %xmm13
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm12, %xmm8
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm8[8],xmm13[8],xmm8[9],xmm13[9],xmm8[10],xmm13[10],xmm8[11],xmm13[11],xmm8[12],xmm13[12],xmm8[13],xmm13[13],xmm8[14],xmm13[14],xmm8[15],xmm13[15]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3],xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]
-; AVX512BW-NEXT:    vprold $16, %xmm13, %xmm13
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm13, %zmm8
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,0,0,1,4,4,4,5]
-; AVX512BW-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
-; AVX512BW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-NEXT:    vmovdqu16 %zmm8, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%r9), %xmm8
-; AVX512BW-NEXT:    vpshufb %xmm7, %xmm8, %xmm13
-; AVX512BW-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512BW-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
+; AVX512BW-NEXT:    vpshufb %xmm10, %xmm12, %xmm10
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
+; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm11
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm11, %xmm13
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm14, %xmm7
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-NEXT:    vpermi2w %zmm7, %zmm13, %zmm16
-; AVX512BW-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512BW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-NEXT:    vmovdqu16 %zmm16, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[16],ymm6[16],ymm5[17],ymm6[17],ymm5[18],ymm6[18],ymm5[19],ymm6[19],ymm5[20],ymm6[20],ymm5[21],ymm6[21],ymm5[22],ymm6[22],ymm5[23],ymm6[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512BW-NEXT:    vpermi2w %zmm5, %zmm6, %zmm7
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512BW-NEXT:    vmovdqu16 %ymm10, %ymm7 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%r9), %xmm10
+; AVX512BW-NEXT:    vpshufb %xmm9, %xmm10, %xmm13
+; AVX512BW-NEXT:    vmovdqa (%r8), %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm9, %xmm15, %xmm9
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm13[0],xmm9[1],xmm13[1],xmm9[2],xmm13[2],xmm9[3],xmm13[3],xmm9[4],xmm13[4],xmm9[5],xmm13[5],xmm9[6],xmm13[6],xmm9[7],xmm13[7]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512BW-NEXT:    vmovdqu16 %ymm9, %ymm7 {%k2}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3],xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm8[0],xmm12[1],xmm8[1],xmm12[2],xmm8[2],xmm12[3],xmm8[3],xmm12[4],xmm8[4],xmm12[5],xmm8[5],xmm12[6],xmm8[6],xmm12[7],xmm8[7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [0,16,0,1,17,0,2,18,0,3,19,0,4,20,0,5]
+; AVX512BW-NEXT:    vpermi2w %ymm9, %ymm13, %ymm16
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3],xmm15[4],xmm10[4],xmm15[5],xmm10[5],xmm15[6],xmm10[6],xmm15[7],xmm10[7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512BW-NEXT:    vpermi2w %ymm9, %ymm16, %ymm13
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm14[8],xmm11[8],xmm14[9],xmm11[9],xmm14[10],xmm11[10],xmm14[11],xmm11[11],xmm14[12],xmm11[12],xmm14[13],xmm11[13],xmm14[14],xmm11[14],xmm14[15],xmm11[15]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,3,19,0,4,20,0,5,21,0,6,22,0,7,23,0]
+; AVX512BW-NEXT:    vpermi2w %ymm9, %ymm8, %ymm11
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm15[8],xmm10[8],xmm15[9],xmm10[9],xmm15[10],xmm10[10],xmm15[11],xmm10[11],xmm15[12],xmm10[12],xmm15[13],xmm10[13],xmm15[14],xmm10[14],xmm15[15],xmm10[15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm9 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512BW-NEXT:    vpermi2w %ymm8, %ymm11, %ymm9
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm12[8],xmm10[8],xmm12[9],xmm10[9],xmm12[10],xmm10[10],xmm12[11],xmm10[11],xmm12[12],xmm10[12],xmm12[13],xmm10[13],xmm12[14],xmm10[14],xmm12[15],xmm10[15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm3, %zmm5
-; AVX512BW-NEXT:    vmovdqu16 %zmm7, %zmm5 {%k1}
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [8,24,0,9,25,0,10,26,0,11,27,0,12,28,0,13]
+; AVX512BW-NEXT:    vpermi2w %ymm4, %ymm2, %ymm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [16,17,40,19,20,41,22,23,42,25,26,43,28,29,44,31]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm8[8],xmm15[9],xmm8[9],xmm15[10],xmm8[10],xmm15[11],xmm8[11],xmm15[12],xmm8[12],xmm15[13],xmm8[13],xmm15[14],xmm8[14],xmm15[15],xmm8[15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,42,41,40,43,42,41,40,43,42,41,40,43,44,44,44,44]
-; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm2
-; AVX512BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512BW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-NEXT:    vmovdqu16 %zmm2, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm2, %zmm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm9, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 128(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -3092,93 +3086,88 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm3
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm5
-; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %ymm6
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %ymm5
 ; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512BW-FCP-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[2],ymm7[2],ymm4[3],ymm7[3],ymm4[4],ymm7[4],ymm4[5],ymm7[5],ymm4[6],ymm7[6],ymm4[7],ymm7[7],ymm4[16],ymm7[16],ymm4[17],ymm7[17],ymm4[18],ymm7[18],ymm4[19],ymm7[19],ymm4[20],ymm7[20],ymm4[21],ymm7[21],ymm4[22],ymm7[22],ymm4[23],ymm7[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15],ymm5[24],ymm6[24],ymm5[25],ymm6[25],ymm5[26],ymm6[26],ymm5[27],ymm6[27],ymm5[28],ymm6[28],ymm5[29],ymm6[29],ymm5[30],ymm6[30],ymm5[31],ymm6[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm7, %ymm8, %ymm7
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm7
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm6, %ymm4
-; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm5, %ymm9
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm9[0],ymm4[0],ymm9[1],ymm4[1],ymm9[2],ymm4[2],ymm9[3],ymm4[3],ymm9[4],ymm4[4],ymm9[5],ymm4[5],ymm9[6],ymm4[6],ymm9[7],ymm4[7],ymm9[16],ymm4[16],ymm9[17],ymm4[17],ymm9[18],ymm4[18],ymm9[19],ymm4[19],ymm9[20],ymm4[20],ymm9[21],ymm4[21],ymm9[22],ymm4[22],ymm9[23],ymm4[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm9, %ymm10, %ymm9
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm4, %zmm4
-; AVX512BW-FCP-NEXT:    movl $1227114788, %r10d # imm = 0x49244924
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm7
+; AVX512BW-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm6
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[4],ymm7[4],ymm6[5],ymm7[5],ymm6[6],ymm7[6],ymm6[7],ymm7[7],ymm6[16],ymm7[16],ymm6[17],ymm7[17],ymm6[18],ymm7[18],ymm6[19],ymm7[19],ymm6[20],ymm7[20],ymm6[21],ymm7[21],ymm6[22],ymm7[22],ymm6[23],ymm7[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm5, %ymm8
+; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm9
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512BW-FCP-NEXT:    movw $18724, %r10w # imm = 0x4924
 ; AVX512BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm4 {%k1}
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm9
-; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm10
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm10
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512BW-FCP-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
-; AVX512BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm4 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %xmm9
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k1}
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm1, %ymm6
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm10
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm10[0],ymm6[0],ymm10[1],ymm6[1],ymm10[2],ymm6[2],ymm10[3],ymm6[3],ymm10[4],ymm6[4],ymm10[5],ymm6[5],ymm10[6],ymm6[6],ymm10[7],ymm6[7],ymm10[16],ymm6[16],ymm10[17],ymm6[17],ymm10[18],ymm6[18],ymm10[19],ymm6[19],ymm10[20],ymm6[20],ymm10[21],ymm6[21],ymm10[22],ymm6[22],ymm10[23],ymm6[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512BW-FCP-NEXT:    movw $9362, %r10w # imm = 0x2492
+; AVX512BW-FCP-NEXT:    kmovd %r10d, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k2}
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,11,27,0,12,28,0,13,29,0,14,30,0,15,31,0]
+; AVX512BW-FCP-NEXT:    vpermi2w %ymm6, %ymm10, %ymm11
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm6
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [42,17,18,43,20,21,44,23,24,45,26,27,46,29,30,47]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512BW-FCP-NEXT:    vpermi2w %zmm11, %zmm6, %zmm10
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm6
+; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %xmm8
 ; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm11
+; AVX512BW-FCP-NEXT:    vpshufb %xmm10, %xmm8, %xmm11
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm12
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm10, %xmm12, %xmm10
 ; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %xmm11
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm13
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3],xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-FCP-NEXT:    vpermi2w %zmm10, %zmm14, %zmm15
-; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm11, %xmm10
-; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm8
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3],xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm10, %zmm14
-; AVX512BW-FCP-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm15, %zmm14 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa (%r9), %xmm8
-; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm10
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm11, %xmm13
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm14
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm7
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm7 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa (%r9), %xmm10
+; AVX512BW-FCP-NEXT:    vpshufb %xmm9, %xmm10, %xmm13
 ; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3],xmm7[4],xmm10[4],xmm7[5],xmm10[5],xmm7[6],xmm10[6],xmm7[7],xmm10[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-FCP-NEXT:    vpermi2w %zmm7, %zmm10, %zmm16
-; AVX512BW-FCP-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm16, %zmm14 {%k1}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[16],ymm6[16],ymm5[17],ymm6[17],ymm5[18],ymm6[18],ymm5[19],ymm6[19],ymm5[20],ymm6[20],ymm5[21],ymm6[21],ymm5[22],ymm6[22],ymm5[23],ymm6[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm13[8],xmm11[8],xmm13[9],xmm11[9],xmm13[10],xmm11[10],xmm13[11],xmm11[11],xmm13[12],xmm11[12],xmm13[13],xmm11[13],xmm13[14],xmm11[14],xmm13[15],xmm11[15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512BW-FCP-NEXT:    vpermi2w %zmm5, %zmm6, %zmm7
+; AVX512BW-FCP-NEXT:    vpshufb %xmm9, %xmm15, %xmm9
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm13[0],xmm9[1],xmm13[1],xmm9[2],xmm13[2],xmm9[3],xmm13[3],xmm9[4],xmm13[4],xmm9[5],xmm13[5],xmm9[6],xmm13[6],xmm9[7],xmm13[7]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm9, %ymm7 {%k2}
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3],xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm8[0],xmm12[1],xmm8[1],xmm12[2],xmm8[2],xmm12[3],xmm8[3],xmm12[4],xmm8[4],xmm12[5],xmm8[5],xmm12[6],xmm8[6],xmm12[7],xmm8[7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [0,16,0,1,17,0,2,18,0,3,19,0,4,20,0,5]
+; AVX512BW-FCP-NEXT:    vpermi2w %ymm9, %ymm13, %ymm16
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3],xmm15[4],xmm10[4],xmm15[5],xmm10[5],xmm15[6],xmm10[6],xmm15[7],xmm10[7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512BW-FCP-NEXT:    vpermi2w %ymm9, %ymm16, %ymm13
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm14[8],xmm11[8],xmm14[9],xmm11[9],xmm14[10],xmm11[10],xmm14[11],xmm11[11],xmm14[12],xmm11[12],xmm14[13],xmm11[13],xmm14[14],xmm11[14],xmm14[15],xmm11[15]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,3,19,0,4,20,0,5,21,0,6,22,0,7,23,0]
+; AVX512BW-FCP-NEXT:    vpermi2w %ymm9, %ymm8, %ymm11
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm15[8],xmm10[8],xmm15[9],xmm10[9],xmm15[10],xmm10[10],xmm15[11],xmm10[11],xmm15[12],xmm10[12],xmm15[13],xmm10[13],xmm15[14],xmm10[14],xmm15[15],xmm10[15]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm9 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512BW-FCP-NEXT:    vpermi2w %ymm8, %ymm11, %ymm9
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23]
 ; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm12[8],xmm9[8],xmm12[9],xmm9[9],xmm12[10],xmm9[10],xmm12[11],xmm9[11],xmm12[12],xmm9[12],xmm12[13],xmm9[13],xmm12[14],xmm9[14],xmm12[15],xmm9[15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512BW-FCP-NEXT:    vpermi2w %zmm2, %zmm3, %zmm5
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm5 {%k1}
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [8,24,0,9,25,0,10,26,0,11,27,0,12,28,0,13]
+; AVX512BW-FCP-NEXT:    vpermi2w %ymm4, %ymm2, %ymm3
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm2
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [16,17,40,19,20,41,22,23,42,25,26,43,28,29,44,31]
 ; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm8[8],xmm15[9],xmm8[9],xmm15[10],xmm8[10],xmm15[11],xmm8[11],xmm15[12],xmm8[12],xmm15[13],xmm8[13],xmm15[14],xmm8[14],xmm15[15],xmm8[15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,42,41,40,43,42,41,40,43,42,41,40,43,44,44,44,44]
-; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm2
-; AVX512BW-FCP-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm2, %zmm5 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm2, %zmm3
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm9, %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -3187,94 +3176,88 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %ymm3
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %ymm5
-; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %ymm6
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %ymm4
+; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %ymm5
 ; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
-; AVX512DQ-BW-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[2],ymm7[2],ymm4[3],ymm7[3],ymm4[4],ymm7[4],ymm4[5],ymm7[5],ymm4[6],ymm7[6],ymm4[7],ymm7[7],ymm4[16],ymm7[16],ymm4[17],ymm7[17],ymm4[18],ymm7[18],ymm4[19],ymm7[19],ymm4[20],ymm7[20],ymm4[21],ymm7[21],ymm4[22],ymm7[22],ymm4[23],ymm7[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15],ymm5[24],ymm6[24],ymm5[25],ymm6[25],ymm5[26],ymm6[26],ymm5[27],ymm6[27],ymm5[28],ymm6[28],ymm5[29],ymm6[29],ymm5[30],ymm6[30],ymm5[31],ymm6[31]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm7, %ymm8, %ymm7
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm7
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm6, %ymm4
-; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm5, %ymm9
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm9[0],ymm4[0],ymm9[1],ymm4[1],ymm9[2],ymm4[2],ymm9[3],ymm4[3],ymm9[4],ymm4[4],ymm9[5],ymm4[5],ymm9[6],ymm4[6],ymm9[7],ymm4[7],ymm9[16],ymm4[16],ymm9[17],ymm4[17],ymm9[18],ymm4[18],ymm9[19],ymm4[19],ymm9[20],ymm4[20],ymm9[21],ymm4[21],ymm9[22],ymm4[22],ymm9[23],ymm4[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm9, %ymm10, %ymm9
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm9, %zmm4, %zmm4
-; AVX512DQ-BW-NEXT:    movl $1227114788, %r10d # imm = 0x49244924
-; AVX512DQ-BW-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm7, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm1, %ymm9
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm0, %ymm10
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm10, %ymm11, %ymm10
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-BW-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm3, %ymm7
+; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm2, %ymm6
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[4],ymm7[4],ymm6[5],ymm7[5],ymm6[6],ymm7[6],ymm6[7],ymm7[7],ymm6[16],ymm7[16],ymm6[17],ymm7[17],ymm6[18],ymm7[18],ymm6[19],ymm7[19],ymm6[20],ymm7[20],ymm6[21],ymm7[21],ymm6[22],ymm7[22],ymm6[23],ymm7[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm5, %ymm8
+; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm4, %ymm9
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512DQ-BW-NEXT:    movw $18724, %r10w # imm = 0x4924
 ; AVX512DQ-BW-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm9, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm9
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm9, %xmm10
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm11
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm11, %xmm8
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm10
+; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k1}
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm9, %ymm1, %ymm6
+; AVX512DQ-BW-NEXT:    vpshufb %ymm9, %ymm0, %ymm10
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm10[0],ymm6[0],ymm10[1],ymm6[1],ymm10[2],ymm6[2],ymm10[3],ymm6[3],ymm10[4],ymm6[4],ymm10[5],ymm6[5],ymm10[6],ymm6[6],ymm10[7],ymm6[7],ymm10[16],ymm6[16],ymm10[17],ymm6[17],ymm10[18],ymm6[18],ymm10[19],ymm6[19],ymm10[20],ymm6[20],ymm10[21],ymm6[21],ymm10[22],ymm6[22],ymm10[23],ymm6[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512DQ-BW-NEXT:    movw $9362, %r10w # imm = 0x2492
+; AVX512DQ-BW-NEXT:    kmovd %r10d, %k2
+; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k2}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,11,27,0,12,28,0,13,29,0,14,30,0,15,31,0]
+; AVX512DQ-BW-NEXT:    vpermi2w %ymm6, %ymm10, %ymm11
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm6
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [42,17,18,43,20,21,44,23,24,45,26,27,46,29,30,47]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512DQ-BW-NEXT:    vpermi2w %zmm11, %zmm6, %zmm10
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm6
+; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm8
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm11
 ; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm12
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm13, %zmm14
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} xmm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm10, %xmm13
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm12, %xmm8
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm8[8],xmm13[8],xmm8[9],xmm13[9],xmm8[10],xmm13[10],xmm8[11],xmm13[11],xmm8[12],xmm13[12],xmm8[13],xmm13[13],xmm8[14],xmm13[14],xmm8[15],xmm13[15]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3],xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]
-; AVX512DQ-BW-NEXT:    vprold $16, %xmm13, %xmm13
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm13, %zmm8
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,0,0,1,4,4,4,5]
-; AVX512DQ-BW-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm8, %zmm14 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm8
-; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm8, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
+; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm12, %xmm10
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
+; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm11
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm11, %xmm13
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm14, %xmm7
 ; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-NEXT:    vpermi2w %zmm7, %zmm13, %zmm16
-; AVX512DQ-BW-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm16, %zmm14 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[16],ymm6[16],ymm5[17],ymm6[17],ymm5[18],ymm6[18],ymm5[19],ymm6[19],ymm5[20],ymm6[20],ymm5[21],ymm6[21],ymm5[22],ymm6[22],ymm5[23],ymm6[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512DQ-BW-NEXT:    vpermi2w %zmm5, %zmm6, %zmm7
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm10, %ymm7 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm10
+; AVX512DQ-BW-NEXT:    vpshufb %xmm9, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm9, %xmm15, %xmm9
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm13[0],xmm9[1],xmm13[1],xmm9[2],xmm13[2],xmm9[3],xmm13[3],xmm9[4],xmm13[4],xmm9[5],xmm13[5],xmm9[6],xmm13[6],xmm9[7],xmm13[7]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm9, %ymm7 {%k2}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3],xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm8[0],xmm12[1],xmm8[1],xmm12[2],xmm8[2],xmm12[3],xmm8[3],xmm12[4],xmm8[4],xmm12[5],xmm8[5],xmm12[6],xmm8[6],xmm12[7],xmm8[7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [0,16,0,1,17,0,2,18,0,3,19,0,4,20,0,5]
+; AVX512DQ-BW-NEXT:    vpermi2w %ymm9, %ymm13, %ymm16
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3],xmm15[4],xmm10[4],xmm15[5],xmm10[5],xmm15[6],xmm10[6],xmm15[7],xmm10[7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512DQ-BW-NEXT:    vpermi2w %ymm9, %ymm16, %ymm13
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm14[8],xmm11[8],xmm14[9],xmm11[9],xmm14[10],xmm11[10],xmm14[11],xmm11[11],xmm14[12],xmm11[12],xmm14[13],xmm11[13],xmm14[14],xmm11[14],xmm14[15],xmm11[15]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,3,19,0,4,20,0,5,21,0,6,22,0,7,23,0]
+; AVX512DQ-BW-NEXT:    vpermi2w %ymm9, %ymm8, %ymm11
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm15[8],xmm10[8],xmm15[9],xmm10[9],xmm15[10],xmm10[10],xmm15[11],xmm10[11],xmm15[12],xmm10[12],xmm15[13],xmm10[13],xmm15[14],xmm10[14],xmm15[15],xmm10[15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm9 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512DQ-BW-NEXT:    vpermi2w %ymm8, %ymm11, %ymm9
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23]
 ; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm12[8],xmm10[8],xmm12[9],xmm10[9],xmm12[10],xmm10[10],xmm12[11],xmm10[11],xmm12[12],xmm10[12],xmm12[13],xmm10[13],xmm12[14],xmm10[14],xmm12[15],xmm10[15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512DQ-BW-NEXT:    vpermi2w %zmm2, %zmm3, %zmm5
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm7, %zmm5 {%k1}
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [8,24,0,9,25,0,10,26,0,11,27,0,12,28,0,13]
+; AVX512DQ-BW-NEXT:    vpermi2w %ymm4, %ymm2, %ymm3
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm2
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [16,17,40,19,20,41,22,23,42,25,26,43,28,29,44,31]
 ; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm8[8],xmm15[9],xmm8[9],xmm15[10],xmm8[10],xmm15[11],xmm8[11],xmm15[12],xmm8[12],xmm15[13],xmm8[13],xmm15[14],xmm8[14],xmm15[15],xmm8[15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,42,41,40,43,42,41,40,43,42,41,40,43,44,44,44,44]
-; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm2
-; AVX512DQ-BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm2, %zmm5 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm2, %zmm3
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm9, %zmm0
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm6, 128(%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
@@ -3283,93 +3266,88 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %ymm5
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[2],ymm7[2],ymm4[3],ymm7[3],ymm4[4],ymm7[4],ymm4[5],ymm7[5],ymm4[6],ymm7[6],ymm4[7],ymm7[7],ymm4[16],ymm7[16],ymm4[17],ymm7[17],ymm4[18],ymm7[18],ymm4[19],ymm7[19],ymm4[20],ymm7[20],ymm4[21],ymm7[21],ymm4[22],ymm7[22],ymm4[23],ymm7[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15],ymm5[24],ymm6[24],ymm5[25],ymm6[25],ymm5[26],ymm6[26],ymm5[27],ymm6[27],ymm5[28],ymm6[28],ymm5[29],ymm6[29],ymm5[30],ymm6[30],ymm5[31],ymm6[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm7, %ymm8, %ymm7
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm7
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm6, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm5, %ymm9
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm9[0],ymm4[0],ymm9[1],ymm4[1],ymm9[2],ymm4[2],ymm9[3],ymm4[3],ymm9[4],ymm4[4],ymm9[5],ymm4[5],ymm9[6],ymm4[6],ymm9[7],ymm4[7],ymm9[16],ymm4[16],ymm9[17],ymm4[17],ymm9[18],ymm4[18],ymm9[19],ymm4[19],ymm9[20],ymm4[20],ymm9[21],ymm4[21],ymm9[22],ymm4[22],ymm9[23],ymm4[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm9, %ymm10, %ymm9
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm4, %zmm4
-; AVX512DQ-BW-FCP-NEXT:    movl $1227114788, %r10d # imm = 0x49244924
-; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm9
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-BW-FCP-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm7
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[4],ymm7[4],ymm6[5],ymm7[5],ymm6[6],ymm7[6],ymm6[7],ymm7[7],ymm6[16],ymm7[16],ymm6[17],ymm7[17],ymm6[18],ymm7[18],ymm6[19],ymm7[19],ymm6[20],ymm7[20],ymm6[21],ymm7[21],ymm6[22],ymm7[22],ymm6[23],ymm7[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm5, %ymm8
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    movw $18724, %r10w # imm = 0x4924
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm1, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm10[0],ymm6[0],ymm10[1],ymm6[1],ymm10[2],ymm6[2],ymm10[3],ymm6[3],ymm10[4],ymm6[4],ymm10[5],ymm6[5],ymm10[6],ymm6[6],ymm10[7],ymm6[7],ymm10[16],ymm6[16],ymm10[17],ymm6[17],ymm10[18],ymm6[18],ymm10[19],ymm6[19],ymm10[20],ymm6[20],ymm10[21],ymm6[21],ymm10[22],ymm6[22],ymm10[23],ymm6[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    movw $9362, %r10w # imm = 0x2492
+; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm8 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,11,27,0,12,28,0,13,29,0,14,30,0,15,31,0]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm6, %ymm10, %ymm11
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [42,17,18,43,20,21,44,23,24,45,26,27,46,29,30,47]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm11, %zmm6, %zmm10
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %xmm8
 ; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm10, %xmm8, %xmm11
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm12
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm10, %xmm12, %xmm10
 ; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm13
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3],xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm10, %zmm14, %zmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm11, %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm8
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3],xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm10, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm15, %zmm14 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r9), %xmm8
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm11, %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm7
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm7 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r9), %xmm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm9, %xmm10, %xmm13
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3],xmm7[4],xmm10[4],xmm7[5],xmm10[5],xmm7[6],xmm10[6],xmm7[7],xmm10[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm7, %zmm10, %zmm16
-; AVX512DQ-BW-FCP-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm16, %zmm14 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[16],ymm6[16],ymm5[17],ymm6[17],ymm5[18],ymm6[18],ymm5[19],ymm6[19],ymm5[20],ymm6[20],ymm5[21],ymm6[21],ymm5[22],ymm6[22],ymm5[23],ymm6[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm13[8],xmm11[8],xmm13[9],xmm11[9],xmm13[10],xmm11[10],xmm13[11],xmm11[11],xmm13[12],xmm11[12],xmm13[13],xmm11[13],xmm13[14],xmm11[14],xmm13[15],xmm11[15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm5, %zmm6, %zmm7
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm9, %xmm15, %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm9[0],xmm13[0],xmm9[1],xmm13[1],xmm9[2],xmm13[2],xmm9[3],xmm13[3],xmm9[4],xmm13[4],xmm9[5],xmm13[5],xmm9[6],xmm13[6],xmm9[7],xmm13[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm9, %ymm7 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3],xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm8[0],xmm12[1],xmm8[1],xmm12[2],xmm8[2],xmm12[3],xmm8[3],xmm12[4],xmm8[4],xmm12[5],xmm8[5],xmm12[6],xmm8[6],xmm12[7],xmm8[7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [0,16,0,1,17,0,2,18,0,3,19,0,4,20,0,5]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm9, %ymm13, %ymm16
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3],xmm15[4],xmm10[4],xmm15[5],xmm10[5],xmm15[6],xmm10[6],xmm15[7],xmm10[7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm9, %ymm16, %ymm13
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm14[8],xmm11[8],xmm14[9],xmm11[9],xmm14[10],xmm11[10],xmm14[11],xmm11[11],xmm14[12],xmm11[12],xmm14[13],xmm11[13],xmm14[14],xmm11[14],xmm14[15],xmm11[15]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,3,19,0,4,20,0,5,21,0,6,22,0,7,23,0]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm9, %ymm8, %ymm11
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm15[8],xmm10[8],xmm15[9],xmm10[9],xmm15[10],xmm10[10],xmm15[11],xmm10[11],xmm15[12],xmm10[12],xmm15[13],xmm10[13],xmm15[14],xmm10[14],xmm15[15],xmm10[15]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm9 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm8, %ymm11, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23]
 ; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm12[8],xmm9[8],xmm12[9],xmm9[9],xmm12[10],xmm9[10],xmm12[11],xmm9[11],xmm12[12],xmm9[12],xmm12[13],xmm9[13],xmm12[14],xmm9[14],xmm12[15],xmm9[15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm2, %zmm3, %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm5 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [8,24,0,9,25,0,10,26,0,11,27,0,12,28,0,13]
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm4, %ymm2, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [16,17,40,19,20,41,22,23,42,25,26,43,28,29,44,31]
 ; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm8[8],xmm15[9],xmm8[9],xmm15[10],xmm8[10],xmm15[11],xmm8[11],xmm15[12],xmm8[12],xmm15[13],xmm8[13],xmm15[14],xmm8[14],xmm15[15],xmm8[15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,42,41,40,43,42,41,40,43,42,41,40,43,44,44,44,44]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm2, %zmm5 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm2, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm9, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <32 x i8>, ptr %in.vecptr0, align 64
@@ -6368,742 +6346,770 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-LABEL: store_i8_stride6_vf64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm6
-; AVX512BW-NEXT:    vmovdqa 32(%rsi), %ymm1
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-NEXT:    vpshufb %ymm5, %ymm1, %ymm0
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %ymm2
-; AVX512BW-NEXT:    vpshufb %ymm5, %ymm2, %ymm3
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512BW-NEXT:    vpermw %ymm3, %ymm8, %ymm3
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm9
-; AVX512BW-NEXT:    vmovdqa 32(%rcx), %ymm3
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm3, %ymm0
-; AVX512BW-NEXT:    vmovdqa 32(%rdx), %ymm4
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm4, %ymm10
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm10[0],ymm0[0],ymm10[1],ymm0[1],ymm10[2],ymm0[2],ymm10[3],ymm0[3],ymm10[4],ymm0[4],ymm10[5],ymm0[5],ymm10[6],ymm0[6],ymm10[7],ymm0[7],ymm10[16],ymm0[16],ymm10[17],ymm0[17],ymm10[18],ymm0[18],ymm10[19],ymm0[19],ymm10[20],ymm0[20],ymm10[21],ymm0[21],ymm10[22],ymm0[22],ymm10[23],ymm0[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15],ymm4[24],ymm3[24],ymm4[25],ymm3[25],ymm4[26],ymm3[26],ymm4[27],ymm3[27],ymm4[28],ymm3[28],ymm4[29],ymm3[29],ymm4[30],ymm3[30],ymm4[31],ymm3[31]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512BW-NEXT:    vpermw %ymm10, %ymm12, %ymm10
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm0
-; AVX512BW-NEXT:    movl $613566756, %r10d # imm = 0x24924924
-; AVX512BW-NEXT:    kmovd %r10d, %k1
-; AVX512BW-NEXT:    vmovdqu16 %zmm9, %zmm0 {%k1}
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm7[4,5,6,7,4,5,6,7]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
-; AVX512BW-NEXT:    vpshufb %zmm14, %zmm9, %zmm9
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,2,2,3,6,6,6,7]
+; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm1
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm15
+; AVX512BW-NEXT:    vmovdqa 32(%rsi), %xmm4
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} xmm5 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm4, %xmm0
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %xmm18
+; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm6
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm6, %xmm3
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
+; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm7, %zmm3
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %xmm16
+; AVX512BW-NEXT:    vmovdqa 32(%rcx), %xmm10
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm8
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %xmm17
+; AVX512BW-NEXT:    vmovdqa 32(%rdx), %xmm11
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm9
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
+; AVX512BW-NEXT:    vprold $16, %xmm9, %xmm9
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm9, %zmm8
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,0,0,1,4,4,4,5]
 ; AVX512BW-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512BW-NEXT:    kmovd %r10d, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm8, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa 32(%r8), %xmm9
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm8 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512BW-NEXT:    vmovdqa %ymm3, %ymm12
+; AVX512BW-NEXT:    vpermt2w %ymm8, %ymm13, %ymm12
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [16,37,18,19,38,21,22,39,24,25,32,27,28,33,30,31]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm9[2,1,2,3]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm14 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero,xmm14[4],zero,xmm14[5],zero,xmm14[6],zero,xmm14[7],zero
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm8, %zmm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm12, %zmm3
+; AVX512BW-NEXT:    vmovdqa 32(%r9), %xmm12
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm12[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm19 = xmm12[2,1,2,3]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm19[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm20 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
+; AVX512BW-NEXT:    vpermt2w %zmm19, %zmm20, %zmm14
+; AVX512BW-NEXT:    movabsq $585610922974906400, %r10 # imm = 0x820820820820820
+; AVX512BW-NEXT:    kmovq %r10, %k2
+; AVX512BW-NEXT:    vmovdqu8 %zmm14, %zmm3 {%k2}
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm15, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm18, %xmm5
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm14[8],xmm5[9],xmm14[9],xmm5[10],xmm14[10],xmm5[11],xmm14[11],xmm5[12],xmm14[12],xmm5[13],xmm14[13],xmm5[14],xmm14[14],xmm5[15],xmm14[15]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm18[0],xmm15[0],xmm18[1],xmm15[1],xmm18[2],xmm15[2],xmm18[3],xmm15[3],xmm18[4],xmm15[4],xmm18[5],xmm15[5],xmm18[6],xmm15[6],xmm18[7],xmm15[7]
+; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm7, %zmm14
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm7
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512BW-NEXT:    vprold $16, %xmm7, %xmm7
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm5
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[0,0,0,1,4,4,4,5]
+; AVX512BW-NEXT:    vmovdqu16 %zmm5, %zmm14 {%k1}
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX512BW-NEXT:    vpermi2w %ymm7, %ymm14, %ymm13
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[2,1,2,3]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm8, %zmm14
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm7
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm8[2,1,2,3]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm14[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm20, %zmm13
+; AVX512BW-NEXT:    vmovdqu8 %zmm13, %zmm7 {%k2}
+; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm13
+; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm14
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm14[0],ymm13[0],ymm14[1],ymm13[1],ymm14[2],ymm13[2],ymm14[3],ymm13[3],ymm14[4],ymm13[4],ymm14[5],ymm13[5],ymm14[6],ymm13[6],ymm14[7],ymm13[7],ymm14[16],ymm13[16],ymm14[17],ymm13[17],ymm14[18],ymm13[18],ymm14[19],ymm13[19],ymm14[20],ymm13[20],ymm14[21],ymm13[21],ymm14[22],ymm13[22],ymm14[23],ymm13[23]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm21 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
+; AVX512BW-NEXT:    vpermt2w %zmm19, %zmm21, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %ymm16
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm17
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm17[0],ymm16[0],ymm17[1],ymm16[1],ymm17[2],ymm16[2],ymm17[3],ymm16[3],ymm17[4],ymm16[4],ymm17[5],ymm16[5],ymm17[6],ymm16[6],ymm17[7],ymm16[7],ymm17[16],ymm16[16],ymm17[17],ymm16[17],ymm17[18],ymm16[18],ymm17[19],ymm16[19],ymm17[20],ymm16[20],ymm17[21],ymm16[21],ymm17[22],ymm16[22],ymm17[23],ymm16[23]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm18[8],xmm15[8],xmm18[9],xmm15[9],xmm18[10],xmm15[10],xmm18[11],xmm15[11],xmm18[12],xmm15[12],xmm18[13],xmm15[13],xmm18[14],xmm15[14],xmm18[15],xmm15[15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm18 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
+; AVX512BW-NEXT:    vpermt2w %zmm19, %zmm18, %zmm15
+; AVX512BW-NEXT:    movl $613566756, %r10d # imm = 0x24924924
 ; AVX512BW-NEXT:    kmovd %r10d, %k2
-; AVX512BW-NEXT:    vmovdqu16 %zmm9, %zmm0 {%k2}
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm6[4,5,6,7,4,5,6,7]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm9 = zmm9[u,6,u,5,u,8,u,7,u,u,u,9,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,42,u,45,u,44,u,43,u,46,u,u,u,u,u,47,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,2,2,3,6,6,6,7]
-; AVX512BW-NEXT:    movabsq $-9076969306111049208, %r10 # imm = 0x8208208208208208
-; AVX512BW-NEXT:    kmovq %r10, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm9, %zmm0 {%k3}
-; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm9
-; AVX512BW-NEXT:    vpshufb %ymm5, %ymm9, %ymm13
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512BW-NEXT:    vpshufb %ymm5, %ymm10, %ymm5
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm13[0],ymm5[1],ymm13[1],ymm5[2],ymm13[2],ymm5[3],ymm13[3],ymm5[4],ymm13[4],ymm5[5],ymm13[5],ymm5[6],ymm13[6],ymm5[7],ymm13[7],ymm5[16],ymm13[16],ymm5[17],ymm13[17],ymm5[18],ymm13[18],ymm5[19],ymm13[19],ymm5[20],ymm13[20],ymm5[21],ymm13[21],ymm5[22],ymm13[22],ymm5[23],ymm13[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm13 = ymm10[8],ymm9[8],ymm10[9],ymm9[9],ymm10[10],ymm9[10],ymm10[11],ymm9[11],ymm10[12],ymm9[12],ymm10[13],ymm9[13],ymm10[14],ymm9[14],ymm10[15],ymm9[15],ymm10[24],ymm9[24],ymm10[25],ymm9[25],ymm10[26],ymm9[26],ymm10[27],ymm9[27],ymm10[28],ymm9[28],ymm10[29],ymm9[29],ymm10[30],ymm9[30],ymm10[31],ymm9[31]
-; AVX512BW-NEXT:    vpermw %ymm13, %ymm8, %ymm8
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm5, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %ymm16
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm16, %ymm5
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %ymm17
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm17, %ymm13
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm13[0],ymm5[0],ymm13[1],ymm5[1],ymm13[2],ymm5[2],ymm13[3],ymm5[3],ymm13[4],ymm5[4],ymm13[5],ymm5[5],ymm13[6],ymm5[6],ymm13[7],ymm5[7],ymm13[16],ymm5[16],ymm13[17],ymm5[17],ymm13[18],ymm5[18],ymm13[19],ymm5[19],ymm13[20],ymm5[20],ymm13[21],ymm5[21],ymm13[22],ymm5[22],ymm13[23],ymm5[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm13 = ymm17[8],ymm16[8],ymm17[9],ymm16[9],ymm17[10],ymm16[10],ymm17[11],ymm16[11],ymm17[12],ymm16[12],ymm17[13],ymm16[13],ymm17[14],ymm16[14],ymm17[15],ymm16[15],ymm17[24],ymm16[24],ymm17[25],ymm16[25],ymm17[26],ymm16[26],ymm17[27],ymm16[27],ymm17[28],ymm16[28],ymm17[29],ymm16[29],ymm17[30],ymm16[30],ymm17[31],ymm16[31]
-; AVX512BW-NEXT:    vpermw %ymm13, %ymm12, %ymm12
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm5
-; AVX512BW-NEXT:    vmovdqu16 %zmm8, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%r8), %ymm13
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm13, %zmm7, %zmm7
-; AVX512BW-NEXT:    vpshufb %zmm14, %zmm7, %zmm7
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,2,2,3,6,6,6,7]
-; AVX512BW-NEXT:    vmovdqu16 %zmm7, %zmm5 {%k2}
-; AVX512BW-NEXT:    vmovdqa (%r9), %ymm12
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm6, %zmm6
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7]
-; AVX512BW-NEXT:    vmovdqu8 %zmm6, %zmm5 {%k3}
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %xmm21
-; AVX512BW-NEXT:    vmovdqa 32(%rsi), %xmm7
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} xmm20 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-NEXT:    vpshufb %xmm20, %xmm7, %xmm6
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %xmm22
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm8
-; AVX512BW-NEXT:    vpshufb %xmm20, %xmm8, %xmm14
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm14[8],xmm6[8],xmm14[9],xmm6[9],xmm14[10],xmm6[10],xmm14[11],xmm6[11],xmm14[12],xmm6[12],xmm14[13],xmm6[13],xmm14[14],xmm6[14],xmm14[15],xmm6[15]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm27 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm27, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %xmm25
-; AVX512BW-NEXT:    vmovdqa 32(%rcx), %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm14, %xmm15
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %xmm26
-; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %xmm18
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm18, %xmm19
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm19[0],xmm15[0],xmm19[1],xmm15[1],xmm19[2],xmm15[2],xmm19[3],xmm15[3],xmm19[4],xmm15[4],xmm19[5],xmm15[5],xmm19[6],xmm15[6],xmm19[7],xmm15[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm18[0],xmm14[0],xmm18[1],xmm14[1],xmm18[2],xmm14[2],xmm18[3],xmm14[3],xmm18[4],xmm14[4],xmm18[5],xmm14[5],xmm18[6],xmm14[6],xmm18[7],xmm14[7]
-; AVX512BW-NEXT:    vprold $16, %xmm19, %xmm19
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm15, %zmm19, %zmm15
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm15 = zmm15[0,0,0,1,4,4,4,5]
-; AVX512BW-NEXT:    vmovdqu16 %zmm15, %zmm6 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 (%r8), %xmm23
-; AVX512BW-NEXT:    vmovdqa 32(%r8), %xmm15
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm19 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero,xmm15[4],zero,xmm15[5],zero,xmm15[6],zero,xmm15[7],zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm15[2,1,2,3]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm24 = xmm24[0],zero,xmm24[1],zero,xmm24[2],zero,xmm24[3],zero,xmm24[4],zero,xmm24[5],zero,xmm24[6],zero,xmm24[7],zero
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
-; AVX512BW-NEXT:    vpermt2w %zmm24, %zmm28, %zmm19
-; AVX512BW-NEXT:    vmovdqu16 %zmm19, %zmm6 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %xmm24
-; AVX512BW-NEXT:    vmovdqa64 32(%r9), %xmm19
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm29 = xmm19[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm30 = xmm19[2,1,2,3]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm30 = xmm30[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm28, %zmm29
-; AVX512BW-NEXT:    movabsq $585610922974906400, %rcx # imm = 0x820820820820820
-; AVX512BW-NEXT:    kmovq %rcx, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm29, %zmm6 {%k3}
-; AVX512BW-NEXT:    vpshufb %xmm20, %xmm21, %xmm29
-; AVX512BW-NEXT:    vpshufb %xmm20, %xmm22, %xmm20
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm29 = xmm20[8],xmm29[8],xmm20[9],xmm29[9],xmm20[10],xmm29[10],xmm20[11],xmm29[11],xmm20[12],xmm29[12],xmm20[13],xmm29[13],xmm20[14],xmm29[14],xmm20[15],xmm29[15]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm22[0],xmm21[0],xmm22[1],xmm21[1],xmm22[2],xmm21[2],xmm22[3],xmm21[3],xmm22[4],xmm21[4],xmm22[5],xmm21[5],xmm22[6],xmm21[6],xmm22[7],xmm21[7]
-; AVX512BW-NEXT:    vpermt2w %zmm29, %zmm27, %zmm20
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm25, %xmm27
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm26, %xmm11
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm11[0],xmm27[0],xmm11[1],xmm27[1],xmm11[2],xmm27[2],xmm11[3],xmm27[3],xmm11[4],xmm27[4],xmm11[5],xmm27[5],xmm11[6],xmm27[6],xmm11[7],xmm27[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512BW-NEXT:    vprold $16, %xmm27, %xmm27
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm27, %zmm11
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[0,0,0,1,4,4,4,5]
-; AVX512BW-NEXT:    vmovdqu16 %zmm11, %zmm20 {%k2}
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm11 = xmm23[0],zero,xmm23[1],zero,xmm23[2],zero,xmm23[3],zero,xmm23[4],zero,xmm23[5],zero,xmm23[6],zero,xmm23[7],zero
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm27 = xmm23[2,1,2,3]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm27 = xmm27[0],zero,xmm27[1],zero,xmm27[2],zero,xmm27[3],zero,xmm27[4],zero,xmm27[5],zero,xmm27[6],zero,xmm27[7],zero
-; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm28, %zmm11
-; AVX512BW-NEXT:    vmovdqu16 %zmm11, %zmm20 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm24[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm27 = xmm24[2,1,2,3]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm27[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm28, %zmm11
-; AVX512BW-NEXT:    vmovdqu8 %zmm11, %zmm20 {%k3}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm17[0],ymm16[0],ymm17[1],ymm16[1],ymm17[2],ymm16[2],ymm17[3],ymm16[3],ymm17[4],ymm16[4],ymm17[5],ymm16[5],ymm17[6],ymm16[6],ymm17[7],ymm16[7],ymm17[16],ymm16[16],ymm17[17],ymm16[17],ymm17[18],ymm16[18],ymm17[19],ymm16[19],ymm17[20],ymm16[20],ymm17[21],ymm16[21],ymm17[22],ymm16[22],ymm17[23],ymm16[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm17 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512BW-NEXT:    vpermt2w %zmm11, %zmm17, %zmm16
+; AVX512BW-NEXT:    vmovdqu16 %zmm20, %zmm15 {%k2}
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm15, %ymm19
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm20 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
+; AVX512BW-NEXT:    # ymm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpshufb %ymm20, %ymm5, %ymm22
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm22 = ymm22[2,2,2,3]
+; AVX512BW-NEXT:    movw $18724, %r10w # imm = 0x4924
+; AVX512BW-NEXT:    kmovd %r10d, %k3
+; AVX512BW-NEXT:    vmovdqu16 %ymm22, %ymm19 {%k3}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm23 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512BW-NEXT:    vpermt2w %ymm22, %ymm23, %ymm15
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm19, %zmm15, %zmm15
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512BW-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpshufb %ymm19, %ymm8, %ymm22
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm24 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm25 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
+; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm25, %zmm24
+; AVX512BW-NEXT:    movabsq $2342443691899625602, %r10 # imm = 0x2082082082082082
+; AVX512BW-NEXT:    kmovq %r10, %k4
+; AVX512BW-NEXT:    vmovdqu8 %zmm24, %zmm15 {%k4}
+; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %ymm22
+; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %ymm24
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm24[0],ymm22[0],ymm24[1],ymm22[1],ymm24[2],ymm22[2],ymm24[3],ymm22[3],ymm24[4],ymm22[4],ymm24[5],ymm22[5],ymm24[6],ymm22[6],ymm24[7],ymm22[7],ymm24[16],ymm22[16],ymm24[17],ymm22[17],ymm24[18],ymm22[18],ymm24[19],ymm22[19],ymm24[20],ymm22[20],ymm24[21],ymm22[21],ymm24[22],ymm22[22],ymm24[23],ymm22[23]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15]
+; AVX512BW-NEXT:    vpermt2w %zmm26, %zmm21, %zmm10
+; AVX512BW-NEXT:    vmovdqa 32(%rsi), %ymm11
+; AVX512BW-NEXT:    vmovdqa64 32(%rdi), %ymm21
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm21[0],ymm11[0],ymm21[1],ymm11[1],ymm21[2],ymm11[2],ymm21[3],ymm11[3],ymm21[4],ymm11[4],ymm21[5],ymm11[5],ymm21[6],ymm11[6],ymm21[7],ymm11[7],ymm21[16],ymm11[16],ymm21[17],ymm11[17],ymm21[18],ymm11[18],ymm21[19],ymm11[19],ymm21[20],ymm11[20],ymm21[21],ymm11[21],ymm21[22],ymm11[22],ymm21[23],ymm11[23]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; AVX512BW-NEXT:    vpermt2w %zmm26, %zmm18, %zmm4
+; AVX512BW-NEXT:    vmovdqu16 %zmm10, %zmm4 {%k2}
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm4, %ymm6
+; AVX512BW-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512BW-NEXT:    vpshufb %ymm20, %ymm10, %ymm10
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512BW-NEXT:    vmovdqu16 %ymm10, %ymm6 {%k3}
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpermt2w %ymm9, %ymm23, %ymm4
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
+; AVX512BW-NEXT:    vmovdqa 32(%r9), %ymm6
+; AVX512BW-NEXT:    vpshufb %ymm19, %ymm6, %ymm6
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpermt2w %zmm6, %zmm25, %zmm9
+; AVX512BW-NEXT:    vmovdqu8 %zmm9, %zmm4 {%k4}
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-NEXT:    vpshufb %ymm6, %ymm11, %ymm9
+; AVX512BW-NEXT:    vpshufb %ymm6, %ymm21, %ymm10
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm22[8],xmm21[8],xmm22[9],xmm21[9],xmm22[10],xmm21[10],xmm22[11],xmm21[11],xmm22[12],xmm21[12],xmm22[13],xmm21[13],xmm22[14],xmm21[14],xmm22[15],xmm21[15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm11, %zmm10
-; AVX512BW-NEXT:    vmovdqu16 %zmm16, %zmm10 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpshufb %ymm9, %ymm13, %ymm13
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm23[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm21 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
-; AVX512BW-NEXT:    vpermt2w %zmm13, %zmm21, %zmm16
-; AVX512BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512BW-NEXT:    kmovd %ecx, %k2
-; AVX512BW-NEXT:    vmovdqu16 %zmm16, %zmm10 {%k2}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512BW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpshufb %ymm13, %ymm12, %ymm12
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm24[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm21, %zmm16
-; AVX512BW-NEXT:    movabsq $2342443691899625602, %rcx # imm = 0x2082082082082082
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm21[8],ymm11[8],ymm21[9],ymm11[9],ymm21[10],ymm11[10],ymm21[11],ymm11[11],ymm21[12],ymm11[12],ymm21[13],ymm11[13],ymm21[14],ymm11[14],ymm21[15],ymm11[15],ymm21[24],ymm11[24],ymm21[25],ymm11[25],ymm21[26],ymm11[26],ymm21[27],ymm11[27],ymm21[28],ymm11[28],ymm21[29],ymm11[29],ymm21[30],ymm11[30],ymm21[31],ymm11[31]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
+; AVX512BW-NEXT:    vpermw %ymm10, %ymm11, %ymm10
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm22, %ymm10
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm24, %ymm12
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[2],ymm10[2],ymm12[3],ymm10[3],ymm12[4],ymm10[4],ymm12[5],ymm10[5],ymm12[6],ymm10[6],ymm12[7],ymm10[7],ymm12[16],ymm10[16],ymm12[17],ymm10[17],ymm12[18],ymm10[18],ymm12[19],ymm10[19],ymm12[20],ymm10[20],ymm12[21],ymm10[21],ymm12[22],ymm10[22],ymm12[23],ymm10[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm12 = ymm24[8],ymm22[8],ymm24[9],ymm22[9],ymm24[10],ymm22[10],ymm24[11],ymm22[11],ymm24[12],ymm22[12],ymm24[13],ymm22[13],ymm24[14],ymm22[14],ymm24[15],ymm22[15],ymm24[24],ymm22[24],ymm24[25],ymm22[25],ymm24[26],ymm22[26],ymm24[27],ymm22[27],ymm24[28],ymm22[28],ymm24[29],ymm22[29],ymm24[30],ymm22[30],ymm24[31],ymm22[31]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm18 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512BW-NEXT:    vpermw %ymm12, %ymm18, %ymm12
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512BW-NEXT:    vmovdqu16 %zmm9, %zmm10 {%k2}
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[4,5,6,7,4,5,6,7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
+; AVX512BW-NEXT:    vpshufb %zmm9, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7]
+; AVX512BW-NEXT:    vmovdqu16 %zmm2, %zmm10 {%k1}
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[4,5,6,7,4,5,6,7]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,6,u,5,u,8,u,7,u,u,u,9,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,42,u,45,u,44,u,43,u,46,u,u,u,u,u,47,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
+; AVX512BW-NEXT:    movabsq $-9076969306111049208, %rcx # imm = 0x8208208208208208
 ; AVX512BW-NEXT:    kmovq %rcx, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm16, %zmm10 {%k3}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm18[8],xmm14[8],xmm18[9],xmm14[9],xmm18[10],xmm14[10],xmm18[11],xmm14[11],xmm18[12],xmm14[12],xmm18[13],xmm14[13],xmm18[14],xmm14[14],xmm18[15],xmm14[15]
-; AVX512BW-NEXT:    vpermt2w %zmm3, %zmm17, %zmm4
+; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm10 {%k3}
+; AVX512BW-NEXT:    vpshufb %ymm6, %ymm16, %ymm1
+; AVX512BW-NEXT:    vpshufb %ymm6, %ymm17, %ymm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
-; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm11, %zmm2
-; AVX512BW-NEXT:    vmovdqu16 %zmm4, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa 32(%r8), %ymm1
-; AVX512BW-NEXT:    vpshufb %ymm9, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm15[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm21, %zmm3
-; AVX512BW-NEXT:    vmovdqu16 %zmm3, %zmm2 {%k2}
-; AVX512BW-NEXT:    vmovdqa 32(%r9), %ymm1
-; AVX512BW-NEXT:    vpshufb %ymm13, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm19[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm21, %zmm3
-; AVX512BW-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k3}
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm17[8],ymm16[8],ymm17[9],ymm16[9],ymm17[10],ymm16[10],ymm17[11],ymm16[11],ymm17[12],ymm16[12],ymm17[13],ymm16[13],ymm17[14],ymm16[14],ymm17[15],ymm16[15],ymm17[24],ymm16[24],ymm17[25],ymm16[25],ymm17[26],ymm16[26],ymm17[27],ymm16[27],ymm17[28],ymm16[28],ymm17[29],ymm16[29],ymm17[30],ymm16[30],ymm17[31],ymm16[31]
+; AVX512BW-NEXT:    vpermw %ymm2, %ymm11, %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm13, %ymm2
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm14[8],ymm13[8],ymm14[9],ymm13[9],ymm14[10],ymm13[10],ymm14[11],ymm13[11],ymm14[12],ymm13[12],ymm14[13],ymm13[13],ymm14[14],ymm13[14],ymm14[15],ymm13[15],ymm14[24],ymm13[24],ymm14[25],ymm13[25],ymm14[26],ymm13[26],ymm14[27],ymm13[27],ymm14[28],ymm13[28],ymm14[29],ymm13[29],ymm14[30],ymm13[30],ymm14[31],ymm13[31]
+; AVX512BW-NEXT:    vpermw %ymm2, %ymm18, %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
+; AVX512BW-NEXT:    vpshufb %zmm9, %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
+; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
+; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 256(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 192(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BW-FCP-LABEL: store_i8_stride6_vf64:
 ; AVX512BW-FCP:       # %bb.0:
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm6
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm6, %ymm0
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm7
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm2
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11],ymm7[12],ymm6[12],ymm7[13],ymm6[13],ymm7[14],ymm6[14],ymm7[15],ymm6[15],ymm7[24],ymm6[24],ymm7[25],ymm6[25],ymm7[26],ymm6[26],ymm7[27],ymm6[27],ymm7[28],ymm6[28],ymm7[29],ymm6[29],ymm7[30],ymm6[30],ymm7[31],ymm6[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm3, %ymm1, %ymm3
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm5
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm0
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm11
-; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm11, %ymm4
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11],ymm11[12],ymm10[12],ymm11[13],ymm10[13],ymm11[14],ymm10[14],ymm11[15],ymm10[15],ymm11[24],ymm10[24],ymm11[25],ymm10[25],ymm11[26],ymm10[26],ymm11[27],ymm10[27],ymm11[28],ymm10[28],ymm11[29],ymm10[29],ymm11[30],ymm10[30],ymm11[31],ymm10[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm4 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm8, %ymm4, %ymm8
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    movl $613566756, %eax # imm = 0x24924924
-; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm5, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %ymm5
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %xmm23
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm17
-; AVX512BW-FCP-NEXT:    vpshufb %xmm3, %xmm17, %xmm8
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %xmm25
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdx), %xmm21
-; AVX512BW-FCP-NEXT:    vpshufb %xmm3, %xmm21, %xmm9
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm21[0],xmm17[0],xmm21[1],xmm17[1],xmm21[2],xmm17[2],xmm21[3],xmm17[3],xmm21[4],xmm17[4],xmm21[5],xmm17[5],xmm21[6],xmm17[6],xmm21[7],xmm17[7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm8, %zmm13, %zmm9
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm30
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm18
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm15 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm18, %xmm8
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm31
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm24
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm24, %xmm12
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm24[0],xmm18[0],xmm24[1],xmm18[1],xmm24[2],xmm18[2],xmm24[3],xmm18[3],xmm24[4],xmm18[4],xmm24[5],xmm18[5],xmm24[6],xmm18[6],xmm24[7],xmm18[7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm12, %zmm16, %zmm8
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %xmm26
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm20
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} xmm19 = [8,9,0,0,0,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm20, %xmm14
-; AVX512BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm12 = xmm20[0],zero,xmm20[1],zero,xmm20[2],zero,xmm20[3],zero,xmm20[4],zero,xmm20[5],zero,xmm20[6],zero,xmm20[7],zero
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm14, %zmm28, %zmm12
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %xmm27
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm22
-; AVX512BW-FCP-NEXT:    vpshufb %xmm3, %xmm23, %xmm14
-; AVX512BW-FCP-NEXT:    vpshufb %xmm3, %xmm25, %xmm29
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm29 = xmm29[0],xmm14[0],xmm29[1],xmm14[1],xmm29[2],xmm14[2],xmm29[3],xmm14[3],xmm29[4],xmm14[4],xmm29[5],xmm14[5],xmm29[6],xmm14[6],xmm29[7],xmm14[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm25[0],xmm23[0],xmm25[1],xmm23[1],xmm25[2],xmm23[2],xmm25[3],xmm23[3],xmm25[4],xmm23[4],xmm25[5],xmm23[5],xmm25[6],xmm23[6],xmm25[7],xmm23[7]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm29, %zmm13, %zmm14
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm29 = [u,8,u,9,u,10,u,11,u,4,u,5,u,6,u,7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm30, %xmm13
-; AVX512BW-FCP-NEXT:    vpshufb %xmm15, %xmm31, %xmm15
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm31[0],xmm30[0],xmm31[1],xmm30[1],xmm31[2],xmm30[2],xmm31[3],xmm30[3],xmm31[4],xmm30[4],xmm31[5],xmm30[5],xmm31[6],xmm30[6],xmm31[7],xmm30[7]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm15, %zmm16, %zmm13
-; AVX512BW-FCP-NEXT:    vpshufb %xmm29, %xmm22, %xmm16
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm22[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm16, %zmm28, %zmm15
-; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm26, %xmm19
-; AVX512BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm16 = xmm26[0],zero,xmm26[1],zero,xmm26[2],zero,xmm26[3],zero,xmm26[4],zero,xmm26[5],zero,xmm26[6],zero,xmm26[7],zero
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm19, %zmm28, %zmm16
-; AVX512BW-FCP-NEXT:    vpshufb %xmm29, %xmm27, %xmm29
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm27[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm29, %zmm28, %zmm19
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %ymm28
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm29 = ymm28[0],ymm5[0],ymm28[1],ymm5[1],ymm28[2],ymm5[2],ymm28[3],ymm5[3],ymm28[4],ymm5[4],ymm28[5],ymm5[5],ymm28[6],ymm5[6],ymm28[7],ymm5[7],ymm28[16],ymm5[16],ymm28[17],ymm5[17],ymm28[18],ymm5[18],ymm28[19],ymm5[19],ymm28[20],ymm5[20],ymm28[21],ymm5[21],ymm28[22],ymm5[22],ymm28[23],ymm5[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm25 = xmm25[8],xmm23[8],xmm25[9],xmm23[9],xmm25[10],xmm23[10],xmm25[11],xmm23[11],xmm25[12],xmm23[12],xmm25[13],xmm23[13],xmm25[14],xmm23[14],xmm25[15],xmm23[15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm29, %zmm1, %zmm25
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %ymm29
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm31[8],xmm30[8],xmm31[9],xmm30[9],xmm31[10],xmm30[10],xmm31[11],xmm30[11],xmm31[12],xmm30[12],xmm31[13],xmm30[13],xmm31[14],xmm30[14],xmm31[15],xmm30[15]
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm30
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm31 = ymm30[0],ymm29[0],ymm30[1],ymm29[1],ymm30[2],ymm29[2],ymm30[3],ymm29[3],ymm30[4],ymm29[4],ymm30[5],ymm29[5],ymm30[6],ymm29[6],ymm30[7],ymm29[7],ymm30[16],ymm29[16],ymm30[17],ymm29[17],ymm30[18],ymm29[18],ymm30[19],ymm29[19],ymm30[20],ymm29[20],ymm30[21],ymm29[21],ymm30[22],ymm29[22],ymm30[23],ymm29[23]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm31, %zmm2, %zmm23
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[16],ymm10[16],ymm11[17],ymm10[17],ymm11[18],ymm10[18],ymm11[19],ymm10[19],ymm11[20],ymm10[20],ymm11[21],ymm10[21],ymm11[22],ymm10[22],ymm11[23],ymm10[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm21[8],xmm17[8],xmm21[9],xmm17[9],xmm21[10],xmm17[10],xmm21[11],xmm17[11],xmm21[12],xmm17[12],xmm21[13],xmm17[13],xmm21[14],xmm17[14],xmm21[15],xmm17[15]
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %ymm17
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm11, %zmm1, %zmm10
-; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
-; AVX512BW-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm17, %ymm21
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm26[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm26 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm21, %zmm26, %zmm11
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[4],ymm6[4],ymm7[5],ymm6[5],ymm7[6],ymm6[6],ymm7[7],ymm6[7],ymm7[16],ymm6[16],ymm7[17],ymm6[17],ymm7[18],ymm6[18],ymm7[19],ymm6[19],ymm7[20],ymm6[20],ymm7[21],ymm6[21],ymm7[22],ymm6[22],ymm7[23],ymm6[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm24[8],xmm18[8],xmm24[9],xmm18[9],xmm24[10],xmm18[10],xmm24[11],xmm18[11],xmm24[12],xmm18[12],xmm24[13],xmm18[13],xmm24[14],xmm18[14],xmm24[15],xmm18[15]
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %ymm21
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm7, %zmm2, %zmm6
-; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512BW-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm21, %ymm18
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm27[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm18, %zmm26, %zmm7
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r8), %ymm18
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm18, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm24
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm20[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm1, %zmm26, %zmm18
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm1
-; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm2
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm22[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-FCP-NEXT:    vpermt2w %zmm1, %zmm26, %zmm20
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm24[4,5,6,7,4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm22 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
-; AVX512BW-FCP-NEXT:    vpshufb %zmm22, %zmm1, %zmm1
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512BW-FCP-NEXT:    movl $-1840700270, %eax # imm = 0x92492492
-; AVX512BW-FCP-NEXT:    kmovd %eax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[4,5,6,7,4,5,6,7]
+; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm2
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm1
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %xmm15
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm6
+; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm6, %xmm3
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %xmm16
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm8
+; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm8, %xmm4
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3],xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm5, %zmm4
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm17
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm7 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm3
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm18
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm10
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm10, %xmm11
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm12 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm13, %zmm12
+; AVX512BW-FCP-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512BW-FCP-NEXT:    kmovd %r10d, %k1
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm4, %zmm12 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%r8), %xmm11
+; AVX512BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512BW-FCP-NEXT:    vmovdqa %ymm12, %ymm4
+; AVX512BW-FCP-NEXT:    vpermt2w %ymm3, %ymm14, %ymm4
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm19 = [16,37,18,19,38,21,22,39,24,25,32,27,28,33,30,31]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} xmm20 = [8,9,0,0,0,5,6,7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm11, %xmm3
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm19, %zmm12
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm4, %zmm3
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %xmm12
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm21 = [u,8,u,9,u,10,u,11,u,4,u,5,u,6,u,7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm12, %xmm4
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm12[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm23 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm4, %zmm23, %zmm22
+; AVX512BW-FCP-NEXT:    movabsq $585610922974906400, %r10 # imm = 0x820820820820820
+; AVX512BW-FCP-NEXT:    kmovq %r10, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm22, %zmm3 {%k2}
+; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm15, %xmm4
+; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm16, %xmm22
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm22[0],xmm4[0],xmm22[1],xmm4[1],xmm22[2],xmm4[2],xmm22[3],xmm4[3],xmm22[4],xmm4[4],xmm22[5],xmm4[5],xmm22[6],xmm4[6],xmm22[7],xmm4[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3],xmm16[4],xmm15[4],xmm16[5],xmm15[5],xmm16[6],xmm15[6],xmm16[7],xmm15[7]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm4, %zmm5, %zmm22
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm17, %xmm4
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm18, %xmm5
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm18[0],xmm17[0],xmm18[1],xmm17[1],xmm18[2],xmm17[2],xmm18[3],xmm17[3],xmm18[4],xmm17[4],xmm18[5],xmm17[5],xmm18[6],xmm17[6],xmm18[7],xmm17[7]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm4, %zmm13, %zmm5
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm22, %zmm5 {%k1}
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX512BW-FCP-NEXT:    vpermi2w %ymm7, %ymm5, %ymm14
+; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm4, %xmm7
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm7, %zmm19, %zmm5
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm14, %zmm5
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm7, %xmm13
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm13, %zmm23, %zmm14
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm5 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %ymm13
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm14
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm14[0],ymm13[0],ymm14[1],ymm13[1],ymm14[2],ymm13[2],ymm14[3],ymm13[3],ymm14[4],ymm13[4],ymm14[5],ymm13[5],ymm14[6],ymm13[6],ymm14[7],ymm13[7],ymm14[16],ymm13[16],ymm14[17],ymm13[17],ymm14[18],ymm13[18],ymm14[19],ymm13[19],ymm14[20],ymm13[20],ymm14[21],ymm13[21],ymm14[22],ymm13[22],ymm14[23],ymm13[23]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm16[8],xmm15[8],xmm16[9],xmm15[9],xmm16[10],xmm15[10],xmm16[11],xmm15[11],xmm16[12],xmm15[12],xmm16[13],xmm15[13],xmm16[14],xmm15[14],xmm16[15],xmm15[15]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm21 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm19, %zmm21, %zmm20
+; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm15
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm16
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm16[0],ymm15[0],ymm16[1],ymm15[1],ymm16[2],ymm15[2],ymm16[3],ymm15[3],ymm16[4],ymm15[4],ymm16[5],ymm15[5],ymm16[6],ymm15[6],ymm16[7],ymm15[7],ymm16[16],ymm15[16],ymm16[17],ymm15[17],ymm16[18],ymm15[18],ymm16[19],ymm15[19],ymm16[20],ymm15[20],ymm16[21],ymm15[21],ymm16[22],ymm15[22],ymm16[23],ymm15[23]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm18 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm19, %zmm18, %zmm17
+; AVX512BW-FCP-NEXT:    movl $613566756, %r10d # imm = 0x24924924
+; AVX512BW-FCP-NEXT:    kmovd %r10d, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm20, %zmm17 {%k2}
+; AVX512BW-FCP-NEXT:    vextracti64x4 $1, %zmm17, %ymm19
+; AVX512BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm20 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
+; AVX512BW-FCP-NEXT:    # ymm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm20, %ymm4, %ymm22
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm22 = ymm22[2,2,2,3]
+; AVX512BW-FCP-NEXT:    movw $18724, %r10w # imm = 0x4924
+; AVX512BW-FCP-NEXT:    kmovd %r10d, %k3
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm22, %ymm19 {%k3}
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm23 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512BW-FCP-NEXT:    vpermt2w %ymm22, %ymm23, %ymm17
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm17, %zmm17
+; AVX512BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512BW-FCP-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm19, %ymm7, %ymm22
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm24 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm25 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm22, %zmm25, %zmm24
+; AVX512BW-FCP-NEXT:    movabsq $2342443691899625602, %r10 # imm = 0x2082082082082082
+; AVX512BW-FCP-NEXT:    kmovq %r10, %k4
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm24, %zmm17 {%k4}
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rcx), %ymm22
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdx), %ymm24
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm24[0],ymm22[0],ymm24[1],ymm22[1],ymm24[2],ymm22[2],ymm24[3],ymm22[3],ymm24[4],ymm22[4],ymm24[5],ymm22[5],ymm24[6],ymm22[6],ymm24[7],ymm22[7],ymm24[16],ymm22[16],ymm24[17],ymm22[17],ymm24[18],ymm22[18],ymm24[19],ymm22[19],ymm24[20],ymm22[20],ymm24[21],ymm22[21],ymm24[22],ymm22[22],ymm24[23],ymm22[23]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm8[8],xmm6[8],xmm8[9],xmm6[9],xmm8[10],xmm6[10],xmm8[11],xmm6[11],xmm8[12],xmm6[12],xmm8[13],xmm6[13],xmm8[14],xmm6[14],xmm8[15],xmm6[15]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm26, %zmm21, %zmm6
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm8
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdi), %ymm21
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm21[0],ymm8[0],ymm21[1],ymm8[1],ymm21[2],ymm8[2],ymm21[3],ymm8[3],ymm21[4],ymm8[4],ymm21[5],ymm8[5],ymm21[6],ymm8[6],ymm21[7],ymm8[7],ymm21[16],ymm8[16],ymm21[17],ymm8[17],ymm21[18],ymm8[18],ymm21[19],ymm8[19],ymm21[20],ymm8[20],ymm21[21],ymm8[21],ymm21[22],ymm8[22],ymm21[23],ymm8[23]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm26, %zmm18, %zmm9
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm6, %zmm9 {%k2}
+; AVX512BW-FCP-NEXT:    vextracti64x4 $1, %zmm9, %ymm6
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512BW-FCP-NEXT:    vpshufb %ymm20, %ymm10, %ymm10
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm6 {%k3}
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-FCP-NEXT:    vpermt2w %ymm10, %ymm23, %ymm9
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm9, %zmm6
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
+; AVX512BW-FCP-NEXT:    vpshufb %ymm19, %ymm9, %ymm9
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm9, %zmm25, %zmm10
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm6 {%k4}
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm10
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm21, %ymm11
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[16],ymm10[16],ymm11[17],ymm10[17],ymm11[18],ymm10[18],ymm11[19],ymm10[19],ymm11[20],ymm10[20],ymm11[21],ymm10[21],ymm11[22],ymm10[22],ymm11[23],ymm10[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm21[8],ymm8[8],ymm21[9],ymm8[9],ymm21[10],ymm8[10],ymm21[11],ymm8[11],ymm21[12],ymm8[12],ymm21[13],ymm8[13],ymm21[14],ymm8[14],ymm21[15],ymm8[15],ymm21[24],ymm8[24],ymm21[25],ymm8[25],ymm21[26],ymm8[26],ymm21[27],ymm8[27],ymm21[28],ymm8[28],ymm21[29],ymm8[29],ymm21[30],ymm8[30],ymm21[31],ymm8[31]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
+; AVX512BW-FCP-NEXT:    vpermw %ymm8, %ymm11, %ymm8
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm10, %zmm8
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm22, %ymm10
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm24, %ymm12
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[2],ymm10[2],ymm12[3],ymm10[3],ymm12[4],ymm10[4],ymm12[5],ymm10[5],ymm12[6],ymm10[6],ymm12[7],ymm10[7],ymm12[16],ymm10[16],ymm12[17],ymm10[17],ymm12[18],ymm10[18],ymm12[19],ymm10[19],ymm12[20],ymm10[20],ymm12[21],ymm10[21],ymm12[22],ymm10[22],ymm12[23],ymm10[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm12 = ymm24[8],ymm22[8],ymm24[9],ymm22[9],ymm24[10],ymm22[10],ymm24[11],ymm22[11],ymm24[12],ymm22[12],ymm24[13],ymm22[13],ymm24[14],ymm22[14],ymm24[15],ymm22[15],ymm24[24],ymm22[24],ymm24[25],ymm22[25],ymm24[26],ymm22[26],ymm24[27],ymm22[27],ymm24[28],ymm22[28],ymm24[29],ymm22[29],ymm24[30],ymm22[30],ymm24[31],ymm22[31]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm18 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512BW-FCP-NEXT:    vpermw %ymm12, %ymm18, %ymm12
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm8, %zmm10 {%k2}
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[4,5,6,7,4,5,6,7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
+; AVX512BW-FCP-NEXT:    vpshufb %zmm8, %zmm2, %zmm2
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm2, %zmm10 {%k1}
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[4,5,6,7,4,5,6,7]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,6,u,5,u,8,u,7,u,u,u,9,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,42,u,45,u,44,u,43,u,46,u,u,u,u,u,47,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512BW-FCP-NEXT:    movabsq $-9076969306111049208, %rax # imm = 0x8208208208208208
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm26 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm26, %ymm29, %ymm1
-; AVX512BW-FCP-NEXT:    vpshufb %ymm26, %ymm30, %ymm26
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm26[0],ymm1[0],ymm26[1],ymm1[1],ymm26[2],ymm1[2],ymm26[3],ymm1[3],ymm26[4],ymm1[4],ymm26[5],ymm1[5],ymm26[6],ymm1[6],ymm26[7],ymm1[7],ymm26[16],ymm1[16],ymm26[17],ymm1[17],ymm26[18],ymm1[18],ymm26[19],ymm1[19],ymm26[20],ymm1[20],ymm26[21],ymm1[21],ymm26[22],ymm1[22],ymm26[23],ymm1[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm26 = ymm30[8],ymm29[8],ymm30[9],ymm29[9],ymm30[10],ymm29[10],ymm30[11],ymm29[11],ymm30[12],ymm29[12],ymm30[13],ymm29[13],ymm30[14],ymm29[14],ymm30[15],ymm29[15],ymm30[24],ymm29[24],ymm30[25],ymm29[25],ymm30[26],ymm29[26],ymm30[27],ymm29[27],ymm30[28],ymm29[28],ymm30[29],ymm29[29],ymm30[30],ymm29[30],ymm30[31],ymm29[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm27 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm26, %ymm27, %ymm26
+; AVX512BW-FCP-NEXT:    movabsq $-9076969306111049208, %rcx # imm = 0x8208208208208208
+; AVX512BW-FCP-NEXT:    kmovq %rcx, %k3
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm10 {%k3}
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm15, %ymm1
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm16, %ymm2
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm26, %zmm1, %zmm1
-; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm5, %ymm26
-; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm28, %ymm3
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm26[0],ymm3[1],ymm26[1],ymm3[2],ymm26[2],ymm3[3],ymm26[3],ymm3[4],ymm26[4],ymm3[5],ymm26[5],ymm3[6],ymm26[6],ymm3[7],ymm26[7],ymm3[16],ymm26[16],ymm3[17],ymm26[17],ymm3[18],ymm26[18],ymm3[19],ymm26[19],ymm3[20],ymm26[20],ymm3[21],ymm26[21],ymm3[22],ymm26[22],ymm3[23],ymm26[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm28[8],ymm5[8],ymm28[9],ymm5[9],ymm28[10],ymm5[10],ymm28[11],ymm5[11],ymm28[12],ymm5[12],ymm28[13],ymm5[13],ymm28[14],ymm5[14],ymm28[15],ymm5[15],ymm28[24],ymm5[24],ymm28[25],ymm5[25],ymm28[26],ymm5[26],ymm28[27],ymm5[27],ymm28[28],ymm5[28],ymm28[29],ymm5[29],ymm28[30],ymm5[30],ymm28[31],ymm5[31]
-; AVX512BW-FCP-NEXT:    vpermw %ymm5, %ymm4, %ymm4
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm3 {%k1}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm24, %zmm1
-; AVX512BW-FCP-NEXT:    vpshufb %zmm22, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm16[8],ymm15[8],ymm16[9],ymm15[9],ymm16[10],ymm15[10],ymm16[11],ymm15[11],ymm16[12],ymm15[12],ymm16[13],ymm15[13],ymm16[14],ymm15[14],ymm16[15],ymm15[15],ymm16[24],ymm15[24],ymm16[25],ymm15[25],ymm16[26],ymm15[26],ymm16[27],ymm15[27],ymm16[28],ymm15[28],ymm16[29],ymm15[29],ymm16[30],ymm15[30],ymm16[31],ymm15[31]
+; AVX512BW-FCP-NEXT:    vpermw %ymm2, %ymm11, %ymm2
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm2
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm14[8],ymm13[8],ymm14[9],ymm13[9],ymm14[10],ymm13[10],ymm14[11],ymm13[11],ymm14[12],ymm13[12],ymm14[13],ymm13[13],ymm14[14],ymm13[14],ymm14[15],ymm13[15],ymm14[24],ymm13[24],ymm14[25],ymm13[25],ymm14[26],ymm13[26],ymm14[27],ymm13[27],ymm14[28],ymm13[28],ymm14[29],ymm13[29],ymm14[30],ymm13[30],ymm14[31],ymm13[31]
+; AVX512BW-FCP-NEXT:    vpermw %ymm2, %ymm18, %ymm2
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
+; AVX512BW-FCP-NEXT:    vpshufb %zmm8, %zmm4, %zmm1
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm3 {%k2}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm3 {%k3}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm8 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm12, %zmm8 {%k1}
-; AVX512BW-FCP-NEXT:    movabsq $585610922974906400, %rax # imm = 0x820820820820820
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm8 {%k3}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm13 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm16, %zmm13 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm19, %zmm13 {%k3}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm25, %zmm23 {%k1}
-; AVX512BW-FCP-NEXT:    movl $1227133513, %eax # imm = 0x49249249
-; AVX512BW-FCP-NEXT:    kmovd %eax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm11, %zmm23 {%k2}
-; AVX512BW-FCP-NEXT:    movabsq $2342443691899625602, %rax # imm = 0x2082082082082082
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm7, %zmm23 {%k3}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm10, %zmm6 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm18, %zmm6 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm20, %zmm6 {%k3}
-; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm23, 64(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, (%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm8, 192(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, 192(%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
 ; AVX512DQ-BW-LABEL: store_i8_stride6_vf64:
 ; AVX512DQ-BW:       # %bb.0:
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%r8), %zmm7
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %zmm6
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rsi), %ymm1
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm5, %ymm1, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-BW-NEXT:    vpshufb %ymm5, %ymm2, %ymm3
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm3, %ymm8, %ymm3
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm9
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rcx), %ymm3
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm3, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rdx), %ymm4
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm4, %ymm10
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm10[0],ymm0[0],ymm10[1],ymm0[1],ymm10[2],ymm0[2],ymm10[3],ymm0[3],ymm10[4],ymm0[4],ymm10[5],ymm0[5],ymm10[6],ymm0[6],ymm10[7],ymm0[7],ymm10[16],ymm0[16],ymm10[17],ymm0[17],ymm10[18],ymm0[18],ymm10[19],ymm0[19],ymm10[20],ymm0[20],ymm10[21],ymm0[21],ymm10[22],ymm0[22],ymm10[23],ymm0[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15],ymm4[24],ymm3[24],ymm4[25],ymm3[25],ymm4[26],ymm3[26],ymm4[27],ymm3[27],ymm4[28],ymm3[28],ymm4[29],ymm3[29],ymm4[30],ymm3[30],ymm4[31],ymm3[31]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm10, %ymm12, %ymm10
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    movl $613566756, %r10d # imm = 0x24924924
-; AVX512DQ-BW-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm9, %zmm0 {%k1}
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm7[4,5,6,7,4,5,6,7]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
-; AVX512DQ-BW-NEXT:    vpshufb %zmm14, %zmm9, %zmm9
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,2,2,3,6,6,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%r8), %zmm2
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %zmm1
+; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm15
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%rsi), %xmm4
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} xmm5 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm4, %xmm0
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %xmm18
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%rdi), %xmm6
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm6, %xmm3
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm7, %zmm3
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rcx), %xmm16
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%rcx), %xmm10
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm8
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdx), %xmm17
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%rdx), %xmm11
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm9
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
+; AVX512DQ-BW-NEXT:    vprold $16, %xmm9, %xmm9
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm9, %zmm8
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,0,0,1,4,4,4,5]
 ; AVX512DQ-BW-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512DQ-BW-NEXT:    kmovd %r10d, %k1
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm8, %zmm3 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%r8), %xmm9
+; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm8 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512DQ-BW-NEXT:    vmovdqa %ymm3, %ymm12
+; AVX512DQ-BW-NEXT:    vpermt2w %ymm8, %ymm13, %ymm12
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [16,37,18,19,38,21,22,39,24,25,32,27,28,33,30,31]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm9[2,1,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm14 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero,xmm14[4],zero,xmm14[5],zero,xmm14[6],zero,xmm14[7],zero
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm14, %zmm8, %zmm3
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm12, %zmm3
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%r9), %xmm12
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm12[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm19 = xmm12[2,1,2,3]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm19[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm20 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm19, %zmm20, %zmm14
+; AVX512DQ-BW-NEXT:    movabsq $585610922974906400, %r10 # imm = 0x820820820820820
+; AVX512DQ-BW-NEXT:    kmovq %r10, %k2
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm14, %zmm3 {%k2}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm15, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm18, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm14[8],xmm5[9],xmm14[9],xmm5[10],xmm14[10],xmm5[11],xmm14[11],xmm5[12],xmm14[12],xmm5[13],xmm14[13],xmm5[14],xmm14[14],xmm5[15],xmm14[15]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm18[0],xmm15[0],xmm18[1],xmm15[1],xmm18[2],xmm15[2],xmm18[3],xmm15[3],xmm18[4],xmm15[4],xmm18[5],xmm15[5],xmm18[6],xmm15[6],xmm18[7],xmm15[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm5, %zmm7, %zmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm7
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512DQ-BW-NEXT:    vprold $16, %xmm7, %xmm7
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm5
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[0,0,0,1,4,4,4,5]
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm5, %zmm14 {%k1}
+; AVX512DQ-BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX512DQ-BW-NEXT:    vpermi2w %ymm7, %ymm14, %ymm13
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[2,1,2,3]
+; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm7, %zmm8, %zmm14
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm7
+; AVX512DQ-BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm8[2,1,2,3]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm14[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm14, %zmm20, %zmm13
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm13, %zmm7 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %ymm13
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %ymm14
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm14[0],ymm13[0],ymm14[1],ymm13[1],ymm14[2],ymm13[2],ymm14[3],ymm13[3],ymm14[4],ymm13[4],ymm14[5],ymm13[5],ymm14[6],ymm13[6],ymm14[7],ymm13[7],ymm14[16],ymm13[16],ymm14[17],ymm13[17],ymm14[18],ymm13[18],ymm14[19],ymm13[19],ymm14[20],ymm13[20],ymm14[21],ymm13[21],ymm14[22],ymm13[22],ymm14[23],ymm13[23]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm21 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm19, %zmm21, %zmm20
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rsi), %ymm16
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %ymm17
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm17[0],ymm16[0],ymm17[1],ymm16[1],ymm17[2],ymm16[2],ymm17[3],ymm16[3],ymm17[4],ymm16[4],ymm17[5],ymm16[5],ymm17[6],ymm16[6],ymm17[7],ymm16[7],ymm17[16],ymm16[16],ymm17[17],ymm16[17],ymm17[18],ymm16[18],ymm17[19],ymm16[19],ymm17[20],ymm16[20],ymm17[21],ymm16[21],ymm17[22],ymm16[22],ymm17[23],ymm16[23]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm18[8],xmm15[8],xmm18[9],xmm15[9],xmm18[10],xmm15[10],xmm18[11],xmm15[11],xmm18[12],xmm15[12],xmm18[13],xmm15[13],xmm18[14],xmm15[14],xmm18[15],xmm15[15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm18 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm19, %zmm18, %zmm15
+; AVX512DQ-BW-NEXT:    movl $613566756, %r10d # imm = 0x24924924
 ; AVX512DQ-BW-NEXT:    kmovd %r10d, %k2
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm9, %zmm0 {%k2}
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm6[4,5,6,7,4,5,6,7]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm9 = zmm9[u,6,u,5,u,8,u,7,u,u,u,9,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,42,u,45,u,44,u,43,u,46,u,u,u,u,u,47,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,2,2,3,6,6,6,7]
-; AVX512DQ-BW-NEXT:    movabsq $-9076969306111049208, %r10 # imm = 0x8208208208208208
-; AVX512DQ-BW-NEXT:    kmovq %r10, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm9, %zmm0 {%k3}
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %ymm9
-; AVX512DQ-BW-NEXT:    vpshufb %ymm5, %ymm9, %ymm13
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512DQ-BW-NEXT:    vpshufb %ymm5, %ymm10, %ymm5
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm13[0],ymm5[1],ymm13[1],ymm5[2],ymm13[2],ymm5[3],ymm13[3],ymm5[4],ymm13[4],ymm5[5],ymm13[5],ymm5[6],ymm13[6],ymm5[7],ymm13[7],ymm5[16],ymm13[16],ymm5[17],ymm13[17],ymm5[18],ymm13[18],ymm5[19],ymm13[19],ymm5[20],ymm13[20],ymm5[21],ymm13[21],ymm5[22],ymm13[22],ymm5[23],ymm13[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm13 = ymm10[8],ymm9[8],ymm10[9],ymm9[9],ymm10[10],ymm9[10],ymm10[11],ymm9[11],ymm10[12],ymm9[12],ymm10[13],ymm9[13],ymm10[14],ymm9[14],ymm10[15],ymm9[15],ymm10[24],ymm9[24],ymm10[25],ymm9[25],ymm10[26],ymm9[26],ymm10[27],ymm9[27],ymm10[28],ymm9[28],ymm10[29],ymm9[29],ymm10[30],ymm9[30],ymm10[31],ymm9[31]
-; AVX512DQ-BW-NEXT:    vpermw %ymm13, %ymm8, %ymm8
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm5, %zmm8
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rcx), %ymm16
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm16, %ymm5
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdx), %ymm17
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm17, %ymm13
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm13[0],ymm5[0],ymm13[1],ymm5[1],ymm13[2],ymm5[2],ymm13[3],ymm5[3],ymm13[4],ymm5[4],ymm13[5],ymm5[5],ymm13[6],ymm5[6],ymm13[7],ymm5[7],ymm13[16],ymm5[16],ymm13[17],ymm5[17],ymm13[18],ymm5[18],ymm13[19],ymm5[19],ymm13[20],ymm5[20],ymm13[21],ymm5[21],ymm13[22],ymm5[22],ymm13[23],ymm5[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm13 = ymm17[8],ymm16[8],ymm17[9],ymm16[9],ymm17[10],ymm16[10],ymm17[11],ymm16[11],ymm17[12],ymm16[12],ymm17[13],ymm16[13],ymm17[14],ymm16[14],ymm17[15],ymm16[15],ymm17[24],ymm16[24],ymm17[25],ymm16[25],ymm17[26],ymm16[26],ymm17[27],ymm16[27],ymm17[28],ymm16[28],ymm17[29],ymm16[29],ymm17[30],ymm16[30],ymm17[31],ymm16[31]
-; AVX512DQ-BW-NEXT:    vpermw %ymm13, %ymm12, %ymm12
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm5
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm8, %zmm5 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %ymm13
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm13, %zmm7, %zmm7
-; AVX512DQ-BW-NEXT:    vpshufb %zmm14, %zmm7, %zmm7
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,2,2,3,6,6,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm7, %zmm5 {%k2}
-; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %ymm12
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm6, %zmm6
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm6, %zmm5 {%k3}
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rsi), %xmm21
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rsi), %xmm7
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} xmm20 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm20, %xmm7, %xmm6
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %xmm22
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rdi), %xmm8
-; AVX512DQ-BW-NEXT:    vpshufb %xmm20, %xmm8, %xmm14
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm14[8],xmm6[8],xmm14[9],xmm6[9],xmm14[10],xmm6[10],xmm14[11],xmm6[11],xmm14[12],xmm6[12],xmm14[13],xmm6[13],xmm14[14],xmm6[14],xmm14[15],xmm6[15]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm27 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm14, %zmm27, %zmm6
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rcx), %xmm25
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rcx), %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm14, %xmm15
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdx), %xmm26
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %xmm18
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm18, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm19[0],xmm15[0],xmm19[1],xmm15[1],xmm19[2],xmm15[2],xmm19[3],xmm15[3],xmm19[4],xmm15[4],xmm19[5],xmm15[5],xmm19[6],xmm15[6],xmm19[7],xmm15[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm18[0],xmm14[0],xmm18[1],xmm14[1],xmm18[2],xmm14[2],xmm18[3],xmm14[3],xmm18[4],xmm14[4],xmm18[5],xmm14[5],xmm18[6],xmm14[6],xmm18[7],xmm14[7]
-; AVX512DQ-BW-NEXT:    vprold $16, %xmm19, %xmm19
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm15, %zmm19, %zmm15
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm15 = zmm15[0,0,0,1,4,4,4,5]
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm15, %zmm6 {%k2}
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%r8), %xmm23
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%r8), %xmm15
-; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm19 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero,xmm15[4],zero,xmm15[5],zero,xmm15[6],zero,xmm15[7],zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm24 = xmm15[2,1,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm24 = xmm24[0],zero,xmm24[1],zero,xmm24[2],zero,xmm24[3],zero,xmm24[4],zero,xmm24[5],zero,xmm24[6],zero,xmm24[7],zero
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm24, %zmm28, %zmm19
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm19, %zmm6 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %xmm24
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r9), %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm29 = xmm19[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm30 = xmm19[2,1,2,3]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm30 = xmm30[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm28, %zmm29
-; AVX512DQ-BW-NEXT:    movabsq $585610922974906400, %rcx # imm = 0x820820820820820
-; AVX512DQ-BW-NEXT:    kmovq %rcx, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm29, %zmm6 {%k3}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm20, %xmm21, %xmm29
-; AVX512DQ-BW-NEXT:    vpshufb %xmm20, %xmm22, %xmm20
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm29 = xmm20[8],xmm29[8],xmm20[9],xmm29[9],xmm20[10],xmm29[10],xmm20[11],xmm29[11],xmm20[12],xmm29[12],xmm20[13],xmm29[13],xmm20[14],xmm29[14],xmm20[15],xmm29[15]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm22[0],xmm21[0],xmm22[1],xmm21[1],xmm22[2],xmm21[2],xmm22[3],xmm21[3],xmm22[4],xmm21[4],xmm22[5],xmm21[5],xmm22[6],xmm21[6],xmm22[7],xmm21[7]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm29, %zmm27, %zmm20
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm25, %xmm27
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm26, %xmm11
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm11[0],xmm27[0],xmm11[1],xmm27[1],xmm11[2],xmm27[2],xmm11[3],xmm27[3],xmm11[4],xmm27[4],xmm11[5],xmm27[5],xmm11[6],xmm27[6],xmm11[7],xmm27[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512DQ-BW-NEXT:    vprold $16, %xmm27, %xmm27
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm27, %zmm11
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm11 = zmm11[0,0,0,1,4,4,4,5]
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm11, %zmm20 {%k2}
-; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm11 = xmm23[0],zero,xmm23[1],zero,xmm23[2],zero,xmm23[3],zero,xmm23[4],zero,xmm23[5],zero,xmm23[6],zero,xmm23[7],zero
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm27 = xmm23[2,1,2,3]
-; AVX512DQ-BW-NEXT:    vpmovzxbw {{.*#+}} xmm27 = xmm27[0],zero,xmm27[1],zero,xmm27[2],zero,xmm27[3],zero,xmm27[4],zero,xmm27[5],zero,xmm27[6],zero,xmm27[7],zero
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm28, %zmm11
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm11, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm24[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} xmm27 = xmm24[2,1,2,3]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm27[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm28, %zmm11
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm11, %zmm20 {%k3}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm17[0],ymm16[0],ymm17[1],ymm16[1],ymm17[2],ymm16[2],ymm17[3],ymm16[3],ymm17[4],ymm16[4],ymm17[5],ymm16[5],ymm17[6],ymm16[6],ymm17[7],ymm16[7],ymm17[16],ymm16[16],ymm17[17],ymm16[17],ymm17[18],ymm16[18],ymm17[19],ymm16[19],ymm17[20],ymm16[20],ymm17[21],ymm16[21],ymm17[22],ymm16[22],ymm17[23],ymm16[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm17 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm11, %zmm17, %zmm16
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm20, %zmm15 {%k2}
+; AVX512DQ-BW-NEXT:    vextracti64x4 $1, %zmm15, %ymm19
+; AVX512DQ-BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm20 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
+; AVX512DQ-BW-NEXT:    # ymm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm20, %ymm5, %ymm22
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm22 = ymm22[2,2,2,3]
+; AVX512DQ-BW-NEXT:    movw $18724, %r10w # imm = 0x4924
+; AVX512DQ-BW-NEXT:    kmovd %r10d, %k3
+; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm22, %ymm19 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm23 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512DQ-BW-NEXT:    vpermt2w %ymm22, %ymm23, %ymm15
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm19, %zmm15, %zmm15
+; AVX512DQ-BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512DQ-BW-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm19, %ymm8, %ymm22
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm24 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm25 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm22, %zmm25, %zmm24
+; AVX512DQ-BW-NEXT:    movabsq $2342443691899625602, %r10 # imm = 0x2082082082082082
+; AVX512DQ-BW-NEXT:    kmovq %r10, %k4
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm24, %zmm15 {%k4}
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %ymm22
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %ymm24
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm24[0],ymm22[0],ymm24[1],ymm22[1],ymm24[2],ymm22[2],ymm24[3],ymm22[3],ymm24[4],ymm22[4],ymm24[5],ymm22[5],ymm24[6],ymm22[6],ymm24[7],ymm22[7],ymm24[16],ymm22[16],ymm24[17],ymm22[17],ymm24[18],ymm22[18],ymm24[19],ymm22[19],ymm24[20],ymm22[20],ymm24[21],ymm22[21],ymm24[22],ymm22[22],ymm24[23],ymm22[23]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm26, %zmm21, %zmm10
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%rsi), %ymm11
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdi), %ymm21
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm21[0],ymm11[0],ymm21[1],ymm11[1],ymm21[2],ymm11[2],ymm21[3],ymm11[3],ymm21[4],ymm11[4],ymm21[5],ymm11[5],ymm21[6],ymm11[6],ymm21[7],ymm11[7],ymm21[16],ymm11[16],ymm21[17],ymm11[17],ymm21[18],ymm11[18],ymm21[19],ymm11[19],ymm21[20],ymm11[20],ymm21[21],ymm11[21],ymm21[22],ymm11[22],ymm21[23],ymm11[23]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm26, %zmm18, %zmm4
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm10, %zmm4 {%k2}
+; AVX512DQ-BW-NEXT:    vextracti64x4 $1, %zmm4, %ymm6
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512DQ-BW-NEXT:    vpshufb %ymm20, %ymm10, %ymm10
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm10, %ymm6 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-NEXT:    vpermt2w %ymm9, %ymm23, %ymm4
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%r9), %ymm6
+; AVX512DQ-BW-NEXT:    vpshufb %ymm19, %ymm6, %ymm6
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm6, %zmm25, %zmm9
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm9, %zmm4 {%k4}
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm11, %ymm9
+; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm21, %ymm10
 ; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm22[8],xmm21[8],xmm22[9],xmm21[9],xmm22[10],xmm21[10],xmm22[11],xmm21[11],xmm22[12],xmm21[12],xmm22[13],xmm21[13],xmm22[14],xmm21[14],xmm22[15],xmm21[15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm9, %zmm11, %zmm10
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm16, %zmm10 {%k1}
-; AVX512DQ-BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
-; AVX512DQ-BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm9, %ymm13, %ymm13
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm23[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm21 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm13, %zmm21, %zmm16
-; AVX512DQ-BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k2
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm16, %zmm10 {%k2}
-; AVX512DQ-BW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512DQ-BW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm13, %ymm12, %ymm12
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm24[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm12, %zmm21, %zmm16
-; AVX512DQ-BW-NEXT:    movabsq $2342443691899625602, %rcx # imm = 0x2082082082082082
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm21[8],ymm11[8],ymm21[9],ymm11[9],ymm21[10],ymm11[10],ymm21[11],ymm11[11],ymm21[12],ymm11[12],ymm21[13],ymm11[13],ymm21[14],ymm11[14],ymm21[15],ymm11[15],ymm21[24],ymm11[24],ymm21[25],ymm11[25],ymm21[26],ymm11[26],ymm21[27],ymm11[27],ymm21[28],ymm11[28],ymm21[29],ymm11[29],ymm21[30],ymm11[30],ymm21[31],ymm11[31]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
+; AVX512DQ-BW-NEXT:    vpermw %ymm10, %ymm11, %ymm10
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm22, %ymm10
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm24, %ymm12
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[2],ymm10[2],ymm12[3],ymm10[3],ymm12[4],ymm10[4],ymm12[5],ymm10[5],ymm12[6],ymm10[6],ymm12[7],ymm10[7],ymm12[16],ymm10[16],ymm12[17],ymm10[17],ymm12[18],ymm10[18],ymm12[19],ymm10[19],ymm12[20],ymm10[20],ymm12[21],ymm10[21],ymm12[22],ymm10[22],ymm12[23],ymm10[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm12 = ymm24[8],ymm22[8],ymm24[9],ymm22[9],ymm24[10],ymm22[10],ymm24[11],ymm22[11],ymm24[12],ymm22[12],ymm24[13],ymm22[13],ymm24[14],ymm22[14],ymm24[15],ymm22[15],ymm24[24],ymm22[24],ymm24[25],ymm22[25],ymm24[26],ymm22[26],ymm24[27],ymm22[27],ymm24[28],ymm22[28],ymm24[29],ymm22[29],ymm24[30],ymm22[30],ymm24[31],ymm22[31]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm18 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512DQ-BW-NEXT:    vpermw %ymm12, %ymm18, %ymm12
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm9, %zmm10 {%k2}
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[4,5,6,7,4,5,6,7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
+; AVX512DQ-BW-NEXT:    vpshufb %zmm9, %zmm2, %zmm2
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm2, %zmm10 {%k1}
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[4,5,6,7,4,5,6,7]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,6,u,5,u,8,u,7,u,u,u,9,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,42,u,45,u,44,u,43,u,46,u,u,u,u,u,47,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
+; AVX512DQ-BW-NEXT:    movabsq $-9076969306111049208, %rcx # imm = 0x8208208208208208
 ; AVX512DQ-BW-NEXT:    kmovq %rcx, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm16, %zmm10 {%k3}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm18[8],xmm14[8],xmm18[9],xmm14[9],xmm18[10],xmm14[10],xmm18[11],xmm14[11],xmm18[12],xmm14[12],xmm18[13],xmm14[13],xmm18[14],xmm14[14],xmm18[15],xmm14[15]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm3, %zmm17, %zmm4
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm1, %zmm10 {%k3}
+; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm16, %ymm1
+; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm17, %ymm2
 ; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm11, %zmm2
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm4, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%r8), %ymm1
-; AVX512DQ-BW-NEXT:    vpshufb %ymm9, %ymm1, %ymm1
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm15[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm21, %zmm3
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm3, %zmm2 {%k2}
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%r9), %ymm1
-; AVX512DQ-BW-NEXT:    vpshufb %ymm13, %ymm1, %ymm1
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm19[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm21, %zmm3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k3}
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 256(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm10, 64(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm20, (%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm6, 192(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, 128(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm17[8],ymm16[8],ymm17[9],ymm16[9],ymm17[10],ymm16[10],ymm17[11],ymm16[11],ymm17[12],ymm16[12],ymm17[13],ymm16[13],ymm17[14],ymm16[14],ymm17[15],ymm16[15],ymm17[24],ymm16[24],ymm17[25],ymm16[25],ymm17[26],ymm16[26],ymm17[27],ymm16[27],ymm17[28],ymm16[28],ymm17[29],ymm16[29],ymm17[30],ymm16[30],ymm17[31],ymm16[31]
+; AVX512DQ-BW-NEXT:    vpermw %ymm2, %ymm11, %ymm2
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm13, %ymm2
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm14[8],ymm13[8],ymm14[9],ymm13[9],ymm14[10],ymm13[10],ymm14[11],ymm13[11],ymm14[12],ymm13[12],ymm14[13],ymm13[13],ymm14[14],ymm13[14],ymm14[15],ymm13[15],ymm14[24],ymm13[24],ymm14[25],ymm13[25],ymm14[26],ymm13[26],ymm14[27],ymm13[27],ymm14[28],ymm13[28],ymm14[29],ymm13[29],ymm14[30],ymm13[30],ymm14[31],ymm13[31]
+; AVX512DQ-BW-NEXT:    vpermw %ymm2, %ymm18, %ymm2
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
+; AVX512DQ-BW-NEXT:    vpshufb %zmm9, %zmm5, %zmm1
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm10, 320(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, 256(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm15, 64(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, 192(%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
 ; AVX512DQ-BW-FCP-LABEL: store_i8_stride6_vf64:
 ; AVX512DQ-BW-FCP:       # %bb.0:
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm6, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm7
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11],ymm7[12],ymm6[12],ymm7[13],ymm6[13],ymm7[14],ymm6[14],ymm7[15],ymm6[15],ymm7[24],ymm6[24],ymm7[25],ymm6[25],ymm7[26],ymm6[26],ymm7[27],ymm6[27],ymm7[28],ymm6[28],ymm7[29],ymm6[29],ymm7[30],ymm6[30],ymm7[31],ymm6[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm3, %ymm1, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm11, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11],ymm11[12],ymm10[12],ymm11[13],ymm10[13],ymm11[14],ymm10[14],ymm11[15],ymm10[15],ymm11[24],ymm10[24],ymm11[25],ymm10[25],ymm11[26],ymm10[26],ymm11[27],ymm10[27],ymm11[28],ymm10[28],ymm11[29],ymm10[29],ymm11[30],ymm10[30],ymm11[31],ymm10[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm4 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm8, %ymm4, %ymm8
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    movl $613566756, %eax # imm = 0x24924924
-; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm5, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %ymm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %xmm23
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm17
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm3, %xmm17, %xmm8
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %xmm25
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdx), %xmm21
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm3, %xmm21, %xmm9
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm21[0],xmm17[0],xmm21[1],xmm17[1],xmm21[2],xmm17[2],xmm21[3],xmm17[3],xmm21[4],xmm17[4],xmm21[5],xmm17[5],xmm21[6],xmm17[6],xmm21[7],xmm17[7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm8, %zmm13, %zmm9
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm30
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm15 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm18, %xmm8
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm31
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm24
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm24, %xmm12
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm12[8],xmm8[8],xmm12[9],xmm8[9],xmm12[10],xmm8[10],xmm12[11],xmm8[11],xmm12[12],xmm8[12],xmm12[13],xmm8[13],xmm12[14],xmm8[14],xmm12[15],xmm8[15]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm24[0],xmm18[0],xmm24[1],xmm18[1],xmm24[2],xmm18[2],xmm24[3],xmm18[3],xmm24[4],xmm18[4],xmm24[5],xmm18[5],xmm24[6],xmm18[6],xmm24[7],xmm18[7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm12, %zmm16, %zmm8
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %xmm26
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm20
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} xmm19 = [8,9,0,0,0,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm20, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm12 = xmm20[0],zero,xmm20[1],zero,xmm20[2],zero,xmm20[3],zero,xmm20[4],zero,xmm20[5],zero,xmm20[6],zero,xmm20[7],zero
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm14, %zmm28, %zmm12
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %xmm27
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm22
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm3, %xmm23, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm3, %xmm25, %xmm29
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm29 = xmm29[0],xmm14[0],xmm29[1],xmm14[1],xmm29[2],xmm14[2],xmm29[3],xmm14[3],xmm29[4],xmm14[4],xmm29[5],xmm14[5],xmm29[6],xmm14[6],xmm29[7],xmm14[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm25[0],xmm23[0],xmm25[1],xmm23[1],xmm25[2],xmm23[2],xmm25[3],xmm23[3],xmm25[4],xmm23[4],xmm25[5],xmm23[5],xmm25[6],xmm23[6],xmm25[7],xmm23[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm29, %zmm13, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm29 = [u,8,u,9,u,10,u,11,u,4,u,5,u,6,u,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm30, %xmm13
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm15, %xmm31, %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm31[0],xmm30[0],xmm31[1],xmm30[1],xmm31[2],xmm30[2],xmm31[3],xmm30[3],xmm31[4],xmm30[4],xmm31[5],xmm30[5],xmm31[6],xmm30[6],xmm31[7],xmm30[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm15, %zmm16, %zmm13
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm29, %xmm22, %xmm16
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm22[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm16, %zmm28, %zmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm26, %xmm19
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm16 = xmm26[0],zero,xmm26[1],zero,xmm26[2],zero,xmm26[3],zero,xmm26[4],zero,xmm26[5],zero,xmm26[6],zero,xmm26[7],zero
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm19, %zmm28, %zmm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm29, %xmm27, %xmm29
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm27[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm29, %zmm28, %zmm19
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %ymm28
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm29 = ymm28[0],ymm5[0],ymm28[1],ymm5[1],ymm28[2],ymm5[2],ymm28[3],ymm5[3],ymm28[4],ymm5[4],ymm28[5],ymm5[5],ymm28[6],ymm5[6],ymm28[7],ymm5[7],ymm28[16],ymm5[16],ymm28[17],ymm5[17],ymm28[18],ymm5[18],ymm28[19],ymm5[19],ymm28[20],ymm5[20],ymm28[21],ymm5[21],ymm28[22],ymm5[22],ymm28[23],ymm5[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm25 = xmm25[8],xmm23[8],xmm25[9],xmm23[9],xmm25[10],xmm23[10],xmm25[11],xmm23[11],xmm25[12],xmm23[12],xmm25[13],xmm23[13],xmm25[14],xmm23[14],xmm25[15],xmm23[15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm29, %zmm1, %zmm25
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %ymm29
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm31[8],xmm30[8],xmm31[9],xmm30[9],xmm31[10],xmm30[10],xmm31[11],xmm30[11],xmm31[12],xmm30[12],xmm31[13],xmm30[13],xmm31[14],xmm30[14],xmm31[15],xmm30[15]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm30
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm31 = ymm30[0],ymm29[0],ymm30[1],ymm29[1],ymm30[2],ymm29[2],ymm30[3],ymm29[3],ymm30[4],ymm29[4],ymm30[5],ymm29[5],ymm30[6],ymm29[6],ymm30[7],ymm29[7],ymm30[16],ymm29[16],ymm30[17],ymm29[17],ymm30[18],ymm29[18],ymm30[19],ymm29[19],ymm30[20],ymm29[20],ymm30[21],ymm29[21],ymm30[22],ymm29[22],ymm30[23],ymm29[23]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm31, %zmm2, %zmm23
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[16],ymm10[16],ymm11[17],ymm10[17],ymm11[18],ymm10[18],ymm11[19],ymm10[19],ymm11[20],ymm10[20],ymm11[21],ymm10[21],ymm11[22],ymm10[22],ymm11[23],ymm10[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm21[8],xmm17[8],xmm21[9],xmm17[9],xmm21[10],xmm17[10],xmm21[11],xmm17[11],xmm21[12],xmm17[12],xmm21[13],xmm17[13],xmm21[14],xmm17[14],xmm21[15],xmm17[15]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm11, %zmm1, %zmm10
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
-; AVX512DQ-BW-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm17, %ymm21
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm26[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm26 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm21, %zmm26, %zmm11
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[4],ymm6[4],ymm7[5],ymm6[5],ymm7[6],ymm6[6],ymm7[7],ymm6[7],ymm7[16],ymm6[16],ymm7[17],ymm6[17],ymm7[18],ymm6[18],ymm7[19],ymm6[19],ymm7[20],ymm6[20],ymm7[21],ymm6[21],ymm7[22],ymm6[22],ymm7[23],ymm6[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm24[8],xmm18[8],xmm24[9],xmm18[9],xmm24[10],xmm18[10],xmm24[11],xmm18[11],xmm24[12],xmm18[12],xmm24[13],xmm18[13],xmm24[14],xmm18[14],xmm24[15],xmm18[15]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %ymm21
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm7, %zmm2, %zmm6
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
-; AVX512DQ-BW-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm21, %ymm18
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm27[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm18, %zmm26, %zmm7
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r8), %ymm18
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm18, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm24
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm20[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm1, %zmm26, %zmm18
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm22[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm1, %zmm26, %zmm20
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm24[4,5,6,7,4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm22 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm22, %zmm1, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512DQ-BW-FCP-NEXT:    movl $-1840700270, %eax # imm = 0x92492492
-; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[4,5,6,7,4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %xmm15
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm0, %xmm6, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %xmm16
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm8
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm0, %xmm8, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3],xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm5, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm7 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm10, %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm12 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5,32,33,34,35,32,33,34,35,32,33,34,35,36,37,38,39]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm13, %zmm12
+; AVX512DQ-BW-FCP-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm4, %zmm12 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r8), %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm14 = [0,1,16,3,4,17,6,7,18,9,10,19,12,13,20,15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm12, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %ymm3, %ymm14, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm19 = [16,37,18,19,38,21,22,39,24,25,32,27,28,33,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} xmm20 = [8,9,0,0,0,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm11, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm19, %zmm12
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm4, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %xmm12
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm21 = [u,8,u,9,u,10,u,11,u,4,u,5,u,6,u,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm12, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm12[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm23 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4,38,37,32,39,38,37,32,39,38,37,32,39,33,33,33,33]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm4, %zmm23, %zmm22
+; AVX512DQ-BW-FCP-NEXT:    movabsq $585610922974906400, %r10 # imm = 0x820820820820820
+; AVX512DQ-BW-FCP-NEXT:    kmovq %r10, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm22, %zmm3 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm0, %xmm15, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm0, %xmm16, %xmm22
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm22[0],xmm4[0],xmm22[1],xmm4[1],xmm22[2],xmm4[2],xmm22[3],xmm4[3],xmm22[4],xmm4[4],xmm22[5],xmm4[5],xmm22[6],xmm4[6],xmm22[7],xmm4[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3],xmm16[4],xmm15[4],xmm16[5],xmm15[5],xmm16[6],xmm15[6],xmm16[7],xmm15[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm4, %zmm5, %zmm22
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm17, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm18, %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm18[0],xmm17[0],xmm18[1],xmm17[1],xmm18[2],xmm17[2],xmm18[3],xmm17[3],xmm18[4],xmm17[4],xmm18[5],xmm17[5],xmm18[6],xmm17[6],xmm18[7],xmm17[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm4, %zmm13, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm22, %zmm5 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm7, %ymm5, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm4, %xmm7
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm7, %zmm19, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm14, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm7, %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm13, %zmm23, %zmm14
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm5 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %ymm13
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm14[0],ymm13[0],ymm14[1],ymm13[1],ymm14[2],ymm13[2],ymm14[3],ymm13[3],ymm14[4],ymm13[4],ymm14[5],ymm13[5],ymm14[6],ymm13[6],ymm14[7],ymm13[7],ymm14[16],ymm13[16],ymm14[17],ymm13[17],ymm14[18],ymm13[18],ymm14[19],ymm13[19],ymm14[20],ymm13[20],ymm14[21],ymm13[21],ymm14[22],ymm13[22],ymm14[23],ymm13[23]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm16[8],xmm15[8],xmm16[9],xmm15[9],xmm16[10],xmm15[10],xmm16[11],xmm15[11],xmm16[12],xmm15[12],xmm16[13],xmm15[13],xmm16[14],xmm15[14],xmm16[15],xmm15[15]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm21 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,41,40,43,42,41,40,43,42,41,40,43,42,45,44,47,46]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm19, %zmm21, %zmm20
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm16
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm19 = ymm16[0],ymm15[0],ymm16[1],ymm15[1],ymm16[2],ymm15[2],ymm16[3],ymm15[3],ymm16[4],ymm15[4],ymm16[5],ymm15[5],ymm16[6],ymm15[6],ymm16[7],ymm15[7],ymm16[16],ymm15[16],ymm16[17],ymm15[17],ymm16[18],ymm15[18],ymm16[19],ymm15[19],ymm16[20],ymm15[20],ymm16[21],ymm15[21],ymm16[22],ymm15[22],ymm16[23],ymm15[23]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm18 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,40,43,42,41,40,43,42,41,40,43,42,41,44,45,46,45]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm19, %zmm18, %zmm17
+; AVX512DQ-BW-FCP-NEXT:    movl $613566756, %r10d # imm = 0x24924924
+; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm20, %zmm17 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vextracti64x4 $1, %zmm17, %ymm19
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm20 = [2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0]
+; AVX512DQ-BW-FCP-NEXT:    # ymm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm20, %ymm4, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm22 = ymm22[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    movw $18724, %r10w # imm = 0x4924
+; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm22, %ymm19 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm23 = [18,1,2,19,4,5,20,7,8,21,10,11,22,13,14,23]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %ymm22, %ymm23, %ymm17
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm17, %zmm17
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
+; AVX512DQ-BW-FCP-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm19, %ymm7, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm24 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm25 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,40,41,42,43,40,41,42,43,40,41,42,43,44,45,46,47]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm22, %zmm25, %zmm24
+; AVX512DQ-BW-FCP-NEXT:    movabsq $2342443691899625602, %r10 # imm = 0x2082082082082082
+; AVX512DQ-BW-FCP-NEXT:    kmovq %r10, %k4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm24, %zmm17 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rcx), %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdx), %ymm24
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm24[0],ymm22[0],ymm24[1],ymm22[1],ymm24[2],ymm22[2],ymm24[3],ymm22[3],ymm24[4],ymm22[4],ymm24[5],ymm22[5],ymm24[6],ymm22[6],ymm24[7],ymm22[7],ymm24[16],ymm22[16],ymm24[17],ymm22[17],ymm24[18],ymm22[18],ymm24[19],ymm22[19],ymm24[20],ymm22[20],ymm24[21],ymm22[21],ymm24[22],ymm22[22],ymm24[23],ymm22[23]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm8[8],xmm6[8],xmm8[9],xmm6[9],xmm8[10],xmm6[10],xmm8[11],xmm6[11],xmm8[12],xmm6[12],xmm8[13],xmm6[13],xmm8[14],xmm6[14],xmm8[15],xmm6[15]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm26, %zmm21, %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm8
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdi), %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm26 = ymm21[0],ymm8[0],ymm21[1],ymm8[1],ymm21[2],ymm8[2],ymm21[3],ymm8[3],ymm21[4],ymm8[4],ymm21[5],ymm8[5],ymm21[6],ymm8[6],ymm21[7],ymm8[7],ymm21[16],ymm8[16],ymm21[17],ymm8[17],ymm21[18],ymm8[18],ymm21[19],ymm8[19],ymm21[20],ymm8[20],ymm21[21],ymm8[21],ymm21[22],ymm8[22],ymm21[23],ymm8[23]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm26, %zmm18, %zmm9
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm6, %zmm9 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vextracti64x4 $1, %zmm9, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm20, %ymm10, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm6 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %ymm10, %ymm23, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm9, %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm19, %ymm9, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm9, %zmm25, %zmm10
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm6 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm21, %ymm11
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[16],ymm10[16],ymm11[17],ymm10[17],ymm11[18],ymm10[18],ymm11[19],ymm10[19],ymm11[20],ymm10[20],ymm11[21],ymm10[21],ymm11[22],ymm10[22],ymm11[23],ymm10[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm21[8],ymm8[8],ymm21[9],ymm8[9],ymm21[10],ymm8[10],ymm21[11],ymm8[11],ymm21[12],ymm8[12],ymm21[13],ymm8[13],ymm21[14],ymm8[14],ymm21[15],ymm8[15],ymm21[24],ymm8[24],ymm21[25],ymm8[25],ymm21[26],ymm8[26],ymm21[27],ymm8[27],ymm21[28],ymm8[28],ymm21[29],ymm8[29],ymm21[30],ymm8[30],ymm21[31],ymm8[31]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm8, %ymm11, %ymm8
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm10, %zmm8
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm22, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm24, %ymm12
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[2],ymm10[2],ymm12[3],ymm10[3],ymm12[4],ymm10[4],ymm12[5],ymm10[5],ymm12[6],ymm10[6],ymm12[7],ymm10[7],ymm12[16],ymm10[16],ymm12[17],ymm10[17],ymm12[18],ymm10[18],ymm12[19],ymm10[19],ymm12[20],ymm10[20],ymm12[21],ymm10[21],ymm12[22],ymm10[22],ymm12[23],ymm10[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm12 = ymm24[8],ymm22[8],ymm24[9],ymm22[9],ymm24[10],ymm22[10],ymm24[11],ymm22[11],ymm24[12],ymm22[12],ymm24[13],ymm22[13],ymm24[14],ymm22[14],ymm24[15],ymm22[15],ymm24[24],ymm22[24],ymm24[25],ymm22[25],ymm24[26],ymm22[26],ymm24[27],ymm22[27],ymm24[28],ymm22[28],ymm24[29],ymm22[29],ymm24[30],ymm22[30],ymm24[31],ymm22[31]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm18 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm12, %ymm18, %ymm12
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm8, %zmm10 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[4,5,6,7,4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,10,13,12,11,14,0,0,15,10,13,12,11,14,0,0,15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm8, %zmm2, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm2, %zmm10 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[4,5,6,7,4,5,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,6,u,5,u,8,u,7,u,u,u,9,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,42,u,45,u,44,u,43,u,46,u,u,u,u,u,47,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512DQ-BW-FCP-NEXT:    movabsq $-9076969306111049208, %rax # imm = 0x8208208208208208
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm26 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm26, %ymm29, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm26, %ymm30, %ymm26
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm26[0],ymm1[0],ymm26[1],ymm1[1],ymm26[2],ymm1[2],ymm26[3],ymm1[3],ymm26[4],ymm1[4],ymm26[5],ymm1[5],ymm26[6],ymm1[6],ymm26[7],ymm1[7],ymm26[16],ymm1[16],ymm26[17],ymm1[17],ymm26[18],ymm1[18],ymm26[19],ymm1[19],ymm26[20],ymm1[20],ymm26[21],ymm1[21],ymm26[22],ymm1[22],ymm26[23],ymm1[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm26 = ymm30[8],ymm29[8],ymm30[9],ymm29[9],ymm30[10],ymm29[10],ymm30[11],ymm29[11],ymm30[12],ymm29[12],ymm30[13],ymm29[13],ymm30[14],ymm29[14],ymm30[15],ymm29[15],ymm30[24],ymm29[24],ymm30[25],ymm29[25],ymm30[26],ymm29[26],ymm30[27],ymm29[27],ymm30[28],ymm29[28],ymm30[29],ymm29[29],ymm30[30],ymm29[30],ymm30[31],ymm29[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm27 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm26, %ymm27, %ymm26
+; AVX512DQ-BW-FCP-NEXT:    movabsq $-9076969306111049208, %rcx # imm = 0x8208208208208208
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rcx, %k3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm10 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm15, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm16, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm26, %zmm1, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm5, %ymm26
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm28, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm26[0],ymm3[1],ymm26[1],ymm3[2],ymm26[2],ymm3[3],ymm26[3],ymm3[4],ymm26[4],ymm3[5],ymm26[5],ymm3[6],ymm26[6],ymm3[7],ymm26[7],ymm3[16],ymm26[16],ymm3[17],ymm26[17],ymm3[18],ymm26[18],ymm3[19],ymm26[19],ymm3[20],ymm26[20],ymm3[21],ymm26[21],ymm3[22],ymm26[22],ymm3[23],ymm26[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm28[8],ymm5[8],ymm28[9],ymm5[9],ymm28[10],ymm5[10],ymm28[11],ymm5[11],ymm28[12],ymm5[12],ymm28[13],ymm5[13],ymm28[14],ymm5[14],ymm28[15],ymm5[15],ymm28[24],ymm5[24],ymm28[25],ymm5[25],ymm28[26],ymm5[26],ymm28[27],ymm5[27],ymm28[28],ymm5[28],ymm28[29],ymm5[29],ymm28[30],ymm5[30],ymm28[31],ymm5[31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm5, %ymm4, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm3 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm24, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm22, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm16[8],ymm15[8],ymm16[9],ymm15[9],ymm16[10],ymm15[10],ymm16[11],ymm15[11],ymm16[12],ymm15[12],ymm16[13],ymm15[13],ymm16[14],ymm15[14],ymm16[15],ymm15[15],ymm16[24],ymm15[24],ymm16[25],ymm15[25],ymm16[26],ymm15[26],ymm16[27],ymm15[27],ymm16[28],ymm15[28],ymm16[29],ymm15[29],ymm16[30],ymm15[30],ymm16[31],ymm15[31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm2, %ymm11, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm14[8],ymm13[8],ymm14[9],ymm13[9],ymm14[10],ymm13[10],ymm14[11],ymm13[11],ymm14[12],ymm13[12],ymm14[13],ymm13[13],ymm14[14],ymm13[14],ymm14[15],ymm13[15],ymm14[24],ymm13[24],ymm14[25],ymm13[25],ymm14[26],ymm13[26],ymm14[27],ymm13[27],ymm14[28],ymm13[28],ymm14[29],ymm13[29],ymm14[30],ymm13[30],ymm14[31],ymm13[31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm2, %ymm18, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm8, %zmm4, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm3 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,u,21,u,24,u,23,u,u,u,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,58,u,61,u,60,u,59,u,62,u,u,u,u,u,63]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,2,2,3,6,6,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm3 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm8 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm12, %zmm8 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    movabsq $585610922974906400, %rax # imm = 0x820820820820820
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm8 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm13 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm16, %zmm13 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm19, %zmm13 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm25, %zmm23 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    movl $1227133513, %eax # imm = 0x49249249
-; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm11, %zmm23 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    movabsq $2342443691899625602, %rax # imm = 0x2082082082082082
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm7, %zmm23 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm10, %zmm6 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm18, %zmm6 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm20, %zmm6 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm23, 64(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm8, 192(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, 192(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <64 x i8>, ptr %in.vecptr0, align 64
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 9aa3330c5777..25e489eef9d1 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -925,16 +925,14 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,ymm0[4],zero,zero,zero,zero,zero,zero,ymm0[5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[23,31],zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm6 = ymm1[4,12],zero,zero,zero,zero,zero,ymm1[5,13],zero,zero,zero,zero,zero,ymm1[6,14,22],zero,zero,zero,zero,zero,zero,ymm1[23],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpblendvb %ymm6, %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpor %ymm3, %ymm5, %ymm3
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[0,8],zero,zero,zero,zero,zero,ymm4[1,9],zero,zero,zero,zero,zero,ymm4[2,10,18,26],zero,zero,zero,zero,zero,ymm4[19,27],zero,zero,zero,zero,zero,ymm4[20,28]
 ; AVX2-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm0[0],zero,zero,zero,zero,zero,zero,ymm0[1],zero,zero,zero,zero,ymm0[18,26],zero,zero,zero,zero,zero,ymm0[19,27],zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[0,8],zero,zero,zero,zero,zero,ymm1[1,9],zero,zero,zero,zero,zero,zero,zero,ymm1[18],zero,zero,zero,zero,zero,zero,ymm1[19],zero,zero,zero,zero
 ; AVX2-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255]
-; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, 48(%rax)
@@ -967,16 +965,14 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,ymm0[4],zero,zero,zero,zero,zero,zero,ymm0[5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[23,31],zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm1[4,12],zero,zero,zero,zero,zero,ymm1[5,13],zero,zero,zero,zero,zero,ymm1[6,14,22],zero,zero,zero,zero,zero,zero,ymm1[23],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX2-FP-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,u,u,u,u,u,u,u,u]
-; AVX2-FP-NEXT:    vpblendvb %ymm6, %ymm3, %ymm5, %ymm3
+; AVX2-FP-NEXT:    vpor %ymm3, %ymm5, %ymm3
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[0,8],zero,zero,zero,zero,zero,ymm4[1,9],zero,zero,zero,zero,zero,ymm4[2,10,18,26],zero,zero,zero,zero,zero,ymm4[19,27],zero,zero,zero,zero,zero,ymm4[20,28]
 ; AVX2-FP-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,ymm0[0],zero,zero,zero,zero,zero,zero,ymm0[1],zero,zero,zero,zero,ymm0[18,26],zero,zero,zero,zero,zero,ymm0[19,27],zero,zero,zero,zero,zero
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[0,8],zero,zero,zero,zero,zero,ymm1[1,9],zero,zero,zero,zero,zero,zero,zero,ymm1[18],zero,zero,zero,zero,zero,zero,ymm1[19],zero,zero,zero,zero
 ; AVX2-FP-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255]
-; AVX2-FP-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-FP-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-FP-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX2-FP-NEXT:    vextracti128 $1, %ymm3, %xmm0
 ; AVX2-FP-NEXT:    vmovq %xmm0, 48(%rax)
@@ -1205,24 +1201,21 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512BW-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[0,2,1,3,4,6,5,7]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zmm1[0,8],zero,zero,zero,zero,zero,zmm1[1,9],zero,zero,zero,zero,zero,zero,zero,zmm1[18],zero,zero,zero,zero,zero,zero,zmm1[19],zero,zero,zero,zero,zmm1[36,44],zero,zero,zero,zero,zero,zmm1[37,45],zero,zero,zero,zero,zero,zmm1[38,46,54],zero,zero,zero,zero,zero,zero,zmm1[55],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,0,2,5,7,4,6]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zmm0[0],zero,zero,zero,zero,zero,zero,zmm0[1],zero,zero,zero,zero,zmm0[18,26],zero,zero,zero,zero,zero,zmm0[19,27],zero,zero,zero,zero,zero,zero,zero,zmm0[36],zero,zero,zero,zero,zero,zero,zmm0[37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,8],zero,zero,zero,zero,zero,zmm1[1,9],zero,zero,zero,zero,zero,zmm1[2,10,18,26],zero,zero,zero,zero,zero,zmm1[19,27],zero,zero,zero,zero,zero,zmm1[20,28],zero,zero,zero,zmm1[37,45],zero,zero,zero,zero,zero,zmm1[38,46],zero,zero,zero,zero,zero,zero,zero,zmm1[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[2,3,0,1,2,3,0,1]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm2 = zero,zero,zmm2[0,8],zero,zero,zero,zero,zero,zmm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[19,27],zero,zero,zero,zero,zero,zmm2[20,28],zero,zero,zero,zero,zero,zero,zero,zmm2[37,45],zero,zero,zero,zero,zero,zmm2[38,46],zero,zero,zero,zmm2[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    movabsq $63546854584629360, %rcx # imm = 0xE1C3870E1C3870
-; AVX512BW-NEXT:    kmovq %rcx, %k1
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vextracti32x4 $2, %zmm1, 32(%rax)
-; AVX512BW-NEXT:    vextracti32x4 $3, %zmm1, %xmm0
-; AVX512BW-NEXT:    vmovq %xmm0, 48(%rax)
-; AVX512BW-NEXT:    vmovdqa %ymm1, (%rax)
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm2 = zero,zero,zero,zero,zmm2[0,8],zero,zero,zero,zero,zero,zmm2[1,9],zero,zero,zero,zero,zero,zero,zero,zmm2[18],zero,zero,zero,zero,zero,zero,zmm2[19],zero,zero,zero,zero,zmm2[36,44],zero,zero,zero,zero,zero,zmm2[37,45],zero,zero,zero,zero,zero,zmm2[38,46,54],zero,zero,zero,zero,zero,zero,zmm2[55],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,0,2,5,7,4,6]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zmm0[0],zero,zero,zero,zero,zero,zero,zmm0[1],zero,zero,zero,zero,zmm0[18,26],zero,zero,zero,zero,zero,zmm0[19,27],zero,zero,zero,zero,zero,zero,zero,zmm0[36],zero,zero,zero,zero,zero,zero,zmm0[37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm2
+; AVX512BW-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
+; AVX512BW-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
+; AVX512BW-NEXT:    vmovq %xmm1, 48(%rax)
+; AVX512BW-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -1283,24 +1276,21 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512DQ-BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm0[0,2,1,3,4,6,5,7]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zmm1[0,8],zero,zero,zero,zero,zero,zmm1[1,9],zero,zero,zero,zero,zero,zero,zero,zmm1[18],zero,zero,zero,zero,zero,zero,zmm1[19],zero,zero,zero,zero,zmm1[36,44],zero,zero,zero,zero,zero,zmm1[37,45],zero,zero,zero,zero,zero,zmm1[38,46,54],zero,zero,zero,zero,zero,zero,zmm1[55],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,0,2,5,7,4,6]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zmm0[0],zero,zero,zero,zero,zero,zero,zmm0[1],zero,zero,zero,zero,zmm0[18,26],zero,zero,zero,zero,zero,zmm0[19,27],zero,zero,zero,zero,zero,zero,zero,zmm0[36],zero,zero,zero,zero,zero,zero,zmm0[37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm1
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,8],zero,zero,zero,zero,zero,zmm1[1,9],zero,zero,zero,zero,zero,zmm1[2,10,18,26],zero,zero,zero,zero,zero,zmm1[19,27],zero,zero,zero,zero,zero,zmm1[20,28],zero,zero,zero,zmm1[37,45],zero,zero,zero,zero,zero,zmm1[38,46],zero,zero,zero,zero,zero,zero,zero,zmm1[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[2,3,0,1,2,3,0,1]
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm2 = zero,zero,zmm2[0,8],zero,zero,zero,zero,zero,zmm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[19,27],zero,zero,zero,zero,zero,zmm2[20,28],zero,zero,zero,zero,zero,zero,zero,zmm2[37,45],zero,zero,zero,zero,zero,zmm2[38,46],zero,zero,zero,zmm2[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512DQ-BW-NEXT:    movabsq $63546854584629360, %rcx # imm = 0xE1C3870E1C3870
-; AVX512DQ-BW-NEXT:    kmovq %rcx, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT:    vextracti32x4 $2, %zmm1, 32(%rax)
-; AVX512DQ-BW-NEXT:    vextracti32x4 $3, %zmm1, %xmm0
-; AVX512DQ-BW-NEXT:    vmovq %xmm0, 48(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa %ymm1, (%rax)
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm2 = zero,zero,zero,zero,zmm2[0,8],zero,zero,zero,zero,zero,zmm2[1,9],zero,zero,zero,zero,zero,zero,zero,zmm2[18],zero,zero,zero,zero,zero,zero,zmm2[19],zero,zero,zero,zero,zmm2[36,44],zero,zero,zero,zero,zero,zmm2[37,45],zero,zero,zero,zero,zero,zmm2[38,46,54],zero,zero,zero,zero,zero,zero,zmm2[55],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,0,2,5,7,4,6]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zmm0[0],zero,zero,zero,zero,zero,zero,zmm0[1],zero,zero,zero,zero,zmm0[18,26],zero,zero,zero,zero,zero,zmm0[19,27],zero,zero,zero,zero,zero,zero,zero,zmm0[36],zero,zero,zero,zero,zero,zero,zmm0[37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[55,63],zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | zmm1 | zmm2
+; AVX512DQ-BW-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
+; AVX512DQ-BW-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
+; AVX512DQ-BW-NEXT:    vmovq %xmm1, 48(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
@@ -1824,8 +1814,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[2,3,0,1]
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,ymm8[5],zero,zero,zero,zero,zero,zero,ymm8[6],zero,zero,zero,zero,zero,ymm8[23],zero,zero,zero,zero,zero,zero,ymm8[24],zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vpor %ymm7, %ymm8, %ymm7
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm8 = [u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255]
-; AVX2-NEXT:    vpblendvb %ymm8, %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpor %ymm5, %ymm7, %ymm5
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm6[4],zero,zero,zero,zero,zero,zero,ymm6[5],zero,zero,zero,zero,zero,zero,ymm6[6],zero,zero,zero,zero,zero,zero,zero,ymm6[23],zero,zero,zero,zero,zero,zero,ymm6[24],zero,zero
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm6[2,3,0,1]
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[4],zero,zero,zero,zero,zero,zero,ymm8[5],zero,zero,zero,zero,zero,zero,ymm8[6],zero,zero,zero,zero,zero,ymm8[23],zero,zero,zero,zero,zero,zero,ymm8[24],zero,zero,zero
@@ -1903,8 +1892,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,ymm7[5],zero,zero,zero,zero,zero,zero,ymm7[6],zero,zero,zero,zero,zero,ymm7[23],zero,zero,zero,zero,zero,zero,ymm7[24],zero,zero,zero,zero,zero
 ; AVX2-FP-NEXT:    vpor %ymm7, %ymm5, %ymm5
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255]
-; AVX2-FP-NEXT:    vpblendvb %ymm7, %ymm4, %ymm5, %ymm5
+; AVX2-FP-NEXT:    vpor %ymm4, %ymm5, %ymm5
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm6[4],zero,zero,zero,zero,zero,zero,ymm6[5],zero,zero,zero,zero,zero,zero,ymm6[6],zero,zero,zero,zero,zero,zero,zero,ymm6[23],zero,zero,zero,zero,zero,zero,ymm6[24],zero,zero
 ; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[4],zero,zero,zero,zero,zero,zero,ymm7[5],zero,zero,zero,zero,zero,zero,ymm7[6],zero,zero,zero,zero,zero,ymm7[23],zero,zero,zero,zero,zero,zero,ymm7[24],zero,zero,zero
@@ -2323,19 +2311,17 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,ymm6[5],zero,zero,zero,zero,zero,zero,ymm6[6],zero,zero,zero,zero,zero,ymm6[23],zero,zero,zero,zero,zero,zero,ymm6[24],zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpor %ymm5, %ymm6, %ymm5
 ; AVX512BW-NEXT:    vinserti128 $1, (%rsi), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,zero,zero,zero,ymm6[1,9],zero,zero,zero,zero,zero,ymm6[2,10],zero,zero,zero,zero,zero,ymm6[19,27],zero,zero,zero,zero,zero,ymm6[20,28],zero,zero
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,ymm1[5],zero,zero,zero,zero,zero,zero,ymm1[6],zero,zero,zero,zero,zero,zero,zero,ymm1[23],zero,zero,zero,zero,zero,zero,ymm1[24],zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm1[2,3,0,1]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,ymm7[5],zero,zero,zero,zero,zero,zero,ymm7[6],zero,zero,zero,zero,zero,ymm7[23],zero,zero,zero,zero,zero,zero,ymm7[24],zero,zero,zero,zero,zero,zero,ymm7[25]
 ; AVX512BW-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX512BW-NEXT:    movl $202911840, %ecx # imm = 0xC183060
-; AVX512BW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-NEXT:    vmovdqu8 %ymm5, %ymm6 {%k1}
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[0,2,0,2]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[0,8],zero,zero,zero,zero,zero,ymm5[1,9],zero,zero,zero,zero,zero,ymm5[2,10],zero,zero,zero,zero,zero,ymm5[19,27],zero,zero,zero,zero,zero,ymm5[20,28],zero,zero
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[0,2,0,2]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,ymm7[0,8],zero,zero,zero,zero,zero,ymm7[1,9],zero,zero,zero,zero,zero,ymm7[18,26],zero,zero,zero,zero,zero,ymm7[19,27],zero,zero,zero,zero,zero,ymm7[20,28]
-; AVX512BW-NEXT:    vpor %ymm5, %ymm7, %ymm5
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm5
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm7, %zmm6
+; AVX512BW-NEXT:    vporq %zmm5, %zmm6, %zmm5
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm6
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm7 = ymm6[4],zero,zero,zero,zero,zero,zero,ymm6[5],zero,zero,zero,zero,zero,zero,ymm6[6],zero,zero,zero,zero,zero,zero,zero,ymm6[23],zero,zero,zero,zero,zero,zero,ymm6[24],zero,zero
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm8 = ymm6[2,3,0,1]
@@ -2445,12 +2431,9 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,zero,zero,zero,zero,xmm1[13,12],zero,zero,zero,zero,zero,xmm1[15,14],zero
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm4[13],zero,zero,zero,zero,zero,zero,xmm4[14],zero,zero,zero,zero,zero,zero,xmm4[15]
-; AVX512BW-FCP-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX512BW-FCP-NEXT:    movw $-7741, %cx # imm = 0xE1C3
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
+; AVX512BW-FCP-NEXT:    vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
 ; AVX512BW-FCP-NEXT:    vmovdqa %ymm5, 64(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa %xmm0, 96(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa %xmm2, 96(%rax)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm7, (%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
@@ -2470,19 +2453,17 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,ymm6[5],zero,zero,zero,zero,zero,zero,ymm6[6],zero,zero,zero,zero,zero,ymm6[23],zero,zero,zero,zero,zero,zero,ymm6[24],zero,zero,zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vpor %ymm5, %ymm6, %ymm5
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rsi), %ymm1, %ymm1
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,zero,zero,zero,ymm6[1,9],zero,zero,zero,zero,zero,ymm6[2,10],zero,zero,zero,zero,zero,ymm6[19,27],zero,zero,zero,zero,zero,ymm6[20,28],zero,zero
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,ymm1[5],zero,zero,zero,zero,zero,zero,ymm1[6],zero,zero,zero,zero,zero,zero,zero,ymm1[23],zero,zero,zero,zero,zero,zero,ymm1[24],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm1[2,3,0,1]
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,ymm7[5],zero,zero,zero,zero,zero,zero,ymm7[6],zero,zero,zero,zero,zero,ymm7[23],zero,zero,zero,zero,zero,zero,ymm7[24],zero,zero,zero,zero,zero,zero,ymm7[25]
 ; AVX512DQ-BW-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX512DQ-BW-NEXT:    movl $202911840, %ecx # imm = 0xC183060
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu8 %ymm5, %ymm6 {%k1}
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[0,2,0,2]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[0,8],zero,zero,zero,zero,zero,ymm5[1,9],zero,zero,zero,zero,zero,ymm5[2,10],zero,zero,zero,zero,zero,ymm5[19,27],zero,zero,zero,zero,zero,ymm5[20,28],zero,zero
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[0,2,0,2]
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,ymm7[0,8],zero,zero,zero,zero,zero,ymm7[1,9],zero,zero,zero,zero,zero,ymm7[18,26],zero,zero,zero,zero,zero,ymm7[19,27],zero,zero,zero,zero,zero,ymm7[20,28]
-; AVX512DQ-BW-NEXT:    vpor %ymm5, %ymm7, %ymm5
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm5
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm7, %zmm6
+; AVX512DQ-BW-NEXT:    vporq %zmm5, %zmm6, %zmm5
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm6
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm7 = ymm6[4],zero,zero,zero,zero,zero,zero,ymm6[5],zero,zero,zero,zero,zero,zero,ymm6[6],zero,zero,zero,zero,zero,zero,zero,ymm6[23],zero,zero,zero,zero,zero,zero,ymm6[24],zero,zero
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm8 = ymm6[2,3,0,1]
@@ -2592,12 +2573,9 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,zero,zero,zero,zero,xmm1[13,12],zero,zero,zero,zero,zero,xmm1[15,14],zero
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm4[13],zero,zero,zero,zero,zero,zero,xmm4[14],zero,zero,zero,zero,zero,zero,xmm4[15]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-BW-FCP-NEXT:    movw $-7741, %cx # imm = 0xE1C3
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpternlogq {{.*#+}} xmm2 = xmm2 | xmm0 | xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm5, 64(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm0, 96(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm2, 96(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm7, (%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
@@ -3598,24 +3576,24 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    pushq %rax
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT:    vmovdqa (%rdi), %ymm4
-; AVX2-NEXT:    vmovdqa (%rsi), %ymm6
-; AVX2-NEXT:    vmovdqa (%rdx), %ymm3
-; AVX2-NEXT:    vmovdqa (%rcx), %ymm5
-; AVX2-NEXT:    vmovdqa (%r8), %ymm7
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm7
 ; AVX2-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vmovdqa (%r9), %ymm2
-; AVX2-NEXT:    vmovdqa (%rax), %ymm1
+; AVX2-NEXT:    vmovdqa (%rsi), %ymm3
+; AVX2-NEXT:    vmovdqa (%rdx), %ymm1
 ; AVX2-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u,u,30,u,28,u,u,u,u,31,u,29,u]
-; AVX2-NEXT:    vpshufhw {{.*#+}} ymm8 = ymm3[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX2-NEXT:    vmovdqa (%rcx), %ymm2
+; AVX2-NEXT:    vmovdqa (%r8), %ymm5
+; AVX2-NEXT:    vmovdqa (%r9), %ymm6
+; AVX2-NEXT:    vmovdqa (%rax), %ymm4
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u,u,30,u,28,u,u,u,u,31,u,29,u]
+; AVX2-NEXT:    vpshufhw {{.*#+}} ymm8 = ymm1[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
 ; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[0,2,3,3,4,6,7,7]
 ; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [255,0,0,0,0,255,0,255,0,0,0,0,255,0,255,0,255,0,0,0,0,255,0,255,0,0,0,0,255,0,255,0]
 ; AVX2-NEXT:    # ymm9 = mem[0,1,0,1]
 ; AVX2-NEXT:    vpblendvb %ymm9, %ymm0, %ymm8, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,30,u,28,u,u,u,u,31,u,29,u,u,u]
-; AVX2-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm4[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,30,u,28,u,u,u,u,31,u,29,u,u,u]
+; AVX2-NEXT:    vpshufhw {{.*#+}} ymm9 = ymm7[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
 ; AVX2-NEXT:    vpshufd {{.*#+}} ymm9 = ymm9[2,2,3,3,6,6,7,7]
 ; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [0,0,255,0,255,0,0,0,0,255,0,255,0,0,0,0,0,0,255,0,255,0,0,0,0,255,0,255,0,0,0,0]
 ; AVX2-NEXT:    # ymm10 = mem[0,1,0,1]
@@ -3623,13 +3601,13 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,u,u,u,0,0,255,255,u,u,u,0,0,255,255,u,u,u,0,0,255,255,u,u,u,0,0,255,255,u,u,u]
 ; AVX2-NEXT:    vpblendvb %ymm9, %ymm0, %ymm8, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm7[27],zero,ymm7[27,28,29,30],zero,ymm7[28],zero,ymm7[26,27,30,31],zero,ymm7[29]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm9 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm2[27],zero,zero,zero,zero,ymm2[30],zero,ymm2[28],zero,zero,zero,zero,ymm2[31],zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm5[27],zero,ymm5[27,28,29,30],zero,ymm5[28],zero,ymm5[26,27,30,31],zero,ymm5[29]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm9 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm6[27],zero,zero,zero,zero,ymm6[30],zero,ymm6[28],zero,zero,zero,zero,ymm6[31],zero
 ; AVX2-NEXT:    vpor %ymm8, %ymm9, %ymm8
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,0,0,u,255,255,255,255,0,0,u,255,255,255,255,0,0,u,255,255,255,255,0,0,u,255,255,255,255,0,0,u]
 ; AVX2-NEXT:    vpblendvb %ymm9, %ymm0, %ymm8, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,28,29,30,31,30,31,28,29,28,29,30,31]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0]
 ; AVX2-NEXT:    vpblendvb %ymm9, %ymm0, %ymm8, %ymm0
@@ -3698,68 +3676,67 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-NEXT:    vpblendvb %ymm8, %ymm1, %ymm7, %ymm1
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255]
 ; AVX2-NEXT:    vpblendvb %ymm7, %ymm0, %ymm1, %ymm10
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
-; AVX2-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm4[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
+; AVX2-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm12[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
 ; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,1,1,4,4,5,5]
 ; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [0,0,0,255,0,255,0,0,0,0,255,0,255,0,0,0,0,0,0,255,0,255,0,0,0,0,255,0,255,0,0,0]
 ; AVX2-NEXT:    # ymm7 = mem[0,1,0,1]
 ; AVX2-NEXT:    vpblendvb %ymm7, %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm5[18],zero,zero,zero,zero,ymm5[21],zero,ymm5[19],zero,zero,zero,zero,ymm5[22],zero,ymm5[20]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,zero,zero,zero,ymm3[21],zero,ymm3[19],zero,zero,zero,zero,ymm3[22],zero,ymm3[20],zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22],zero,ymm2[20]
+; AVX2-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,zero,zero,zero,ymm13[21],zero,ymm13[19],zero,zero,zero,zero,ymm13[22],zero,ymm13[20],zero
 ; AVX2-NEXT:    vpor %ymm1, %ymm7, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,u,u,u,0,0,255,255,u,u,u,0,0,255,255,u,u,u,0,0,255,255,u,u,u,0,0,255,255,u,u]
 ; AVX2-NEXT:    vpblendvb %ymm7, %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
-; AVX2-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm13[18],zero,zero,zero,zero,ymm13[21],zero,ymm13[19],zero,zero,zero,zero,ymm13[22],zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[20],zero,ymm6[18],zero,zero,zero,zero,ymm6[21],zero,ymm6[19],zero,zero,zero,zero,ymm6[22]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm5[18],zero,zero,zero,zero,ymm5[21],zero,ymm5[19],zero,zero,zero,zero,ymm5[22],zero
 ; AVX2-NEXT:    vpor %ymm1, %ymm7, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX2-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm12[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
+; AVX2-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm4[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
 ; AVX2-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[0,1,1,3,4,5,5,7]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,3,2]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm8 = [u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255]
 ; AVX2-NEXT:    vpblendvb %ymm8, %ymm1, %ymm7, %ymm1
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0]
 ; AVX2-NEXT:    vpblendvb %ymm7, %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[23],zero,ymm4[27,20,21,26],zero,ymm4[24],zero,ymm4[26,27,26,27],zero,ymm4[25]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm6[23],zero,zero,zero,zero,ymm6[26],zero,ymm6[24],zero,zero,zero,zero,ymm6[27],zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm12[23],zero,ymm12[27,20,21,26],zero,ymm12[24],zero,ymm12[26,27,26,27],zero,ymm12[25]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero,zero,zero,ymm3[27],zero
 ; AVX2-NEXT:    vpor %ymm1, %ymm7, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm5[25],zero,ymm5[23],zero,zero,zero,zero,ymm5[26],zero,ymm5[24],zero,zero,zero,zero
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[25],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero,zero,zero,ymm3[27]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero,zero,zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm13[25],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero,zero,ymm13[27]
 ; AVX2-NEXT:    vpor %ymm7, %ymm8, %ymm7
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,3,2,3]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm8 = [u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0]
 ; AVX2-NEXT:    vpblendvb %ymm8, %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm13[25],zero,ymm13[23],zero,zero,zero,zero,ymm13[26],zero,ymm13[24],zero,zero,zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm7 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm6[25],zero,ymm6[23],zero,zero,zero,zero,ymm6[26],zero,ymm6[24],zero,zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm5[25],zero,ymm5[23],zero,zero,zero,zero,ymm5[26],zero,ymm5[24],zero,zero,zero
 ; AVX2-NEXT:    vpor %ymm7, %ymm8, %ymm7
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,3,2,3]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm8 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3]
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u]
 ; AVX2-NEXT:    vpblendvb %ymm11, %ymm7, %ymm8, %ymm7
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm8 = [0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255]
 ; AVX2-NEXT:    vpblendvb %ymm8, %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,ymm6[14],zero,zero,zero,zero,zero,zero,ymm6[15],zero,zero,zero,zero,zero,zero,ymm6[16],zero,zero,zero,zero,zero,zero,ymm6[17],zero,zero,zero,zero,zero,zero,ymm6[18]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,ymm4[14],zero,zero,zero,zero,zero,zero,ymm4[15],zero,zero,zero,zero,zero,zero,ymm4[16],zero,zero,zero,zero,zero,zero,ymm4[17],zero,zero,zero,zero,zero,zero,ymm4[18],zero
-; AVX2-NEXT:    vpor %ymm6, %ymm4, %ymm4
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,ymm5[14],zero,zero,zero,zero,zero,zero,ymm5[15],zero,zero,zero,zero,zero,zero,ymm5[16],zero,zero,zero,zero,zero,zero,ymm5[17],zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpor %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255]
-; AVX2-NEXT:    vpblendvb %ymm5, %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm4 = zero,ymm13[1,2,3,0,1,14],zero,ymm13[0,1,0,1,14,15],zero,ymm13[15,16,17,18,19,16],zero,ymm13[30,31,16,17,16,17],zero,ymm13[31,30,31]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[13],zero,zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero
-; AVX2-NEXT:    vpor %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm4 = ymm12[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u]
-; AVX2-NEXT:    vpblendvb %ymm5, %ymm2, %ymm4, %ymm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255]
-; AVX2-NEXT:    vpblendvb %ymm4, %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm5 = zero,ymm5[1,2,3,0,1,14],zero,ymm5[0,1,0,1,14,15],zero,ymm5[15,16,17,18,19,16],zero,ymm5[30,31,16,17,16,17],zero,ymm5[31,30,31]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[13],zero,zero,zero,zero,zero,zero,ymm6[14],zero,zero,zero,zero,zero,zero,ymm6[15],zero,zero,zero,zero,zero,zero,ymm6[16],zero,zero,zero,zero,zero,zero,ymm6[17],zero,zero,zero
+; AVX2-NEXT:    vpor %ymm5, %ymm6, %ymm5
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm6 = [255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u]
+; AVX2-NEXT:    vpblendvb %ymm6, %ymm5, %ymm4, %ymm4
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero,zero,zero,ymm3[18]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,ymm12[14],zero,zero,zero,zero,zero,zero,ymm12[15],zero,zero,zero,zero,zero,zero,ymm12[16],zero,zero,zero,zero,zero,zero,ymm12[17],zero,zero,zero,zero,zero,zero,ymm12[18],zero
+; AVX2-NEXT:    vpor %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,ymm13[14],zero,zero,zero,zero,zero,zero,ymm13[15],zero,zero,zero,zero,zero,zero,ymm13[16],zero,zero,zero,zero,zero,zero,ymm13[17],zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpor %ymm2, %ymm5, %ymm2
+; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255]
+; AVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm4, %ymm2
 ; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX2-NEXT:    vmovdqa %ymm2, 96(%rax)
 ; AVX2-NEXT:    vmovdqa %ymm1, 160(%rax)
@@ -3905,22 +3882,21 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FP-NEXT:    vpblendvb %ymm15, %ymm13, %ymm14, %ymm13
 ; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm14 = [255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0]
 ; AVX2-FP-NEXT:    vpblendvb %ymm14, %ymm12, %ymm13, %ymm12
+; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,2,3,0,1,14],zero,ymm6[0,1,0,1,14,15],zero,ymm6[15,16,17,18,19,16],zero,ymm6[30,31,16,17,16,17],zero,ymm6[31,30,31]
+; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[13],zero,zero,zero,zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero
+; AVX2-FP-NEXT:    vpor %ymm6, %ymm8, %ymm6
+; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
+; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u]
+; AVX2-FP-NEXT:    vpblendvb %ymm8, %ymm6, %ymm7, %ymm6
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero,zero,zero,ymm3[18]
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero,zero,zero,ymm1[18],zero
 ; AVX2-FP-NEXT:    vpor %ymm3, %ymm1, %ymm1
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero,zero,zero
 ; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,zero,zero
 ; AVX2-FP-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255]
-; AVX2-FP-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,ymm6[1,2,3,0,1,14],zero,ymm6[0,1,0,1,14,15],zero,ymm6[15,16,17,18,19,16],zero,ymm6[30,31,16,17,16,17],zero,ymm6[31,30,31]
-; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm8[13],zero,zero,zero,zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero
-; AVX2-FP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm7[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u]
-; AVX2-FP-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255]
-; AVX2-FP-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FP-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255]
+; AVX2-FP-NEXT:    vpblendvb %ymm1, %ymm0, %ymm6, %ymm0
 ; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX2-FP-NEXT:    vmovdqa %ymm0, 96(%rax)
 ; AVX2-FP-NEXT:    vmovdqa %ymm12, 128(%rax)
@@ -4067,22 +4043,21 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FCP-NEXT:    vpblendvb %ymm15, %ymm13, %ymm14, %ymm13
 ; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255]
 ; AVX2-FCP-NEXT:    vpblendvb %ymm14, %ymm12, %ymm13, %ymm12
+; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,2,3,0,1,14],zero,ymm6[0,1,0,1,14,15],zero,ymm6[15,16,17,18,19,16],zero,ymm6[30,31,16,17,16,17],zero,ymm6[31,30,31]
+; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[13],zero,zero,zero,zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero
+; AVX2-FCP-NEXT:    vpor %ymm6, %ymm8, %ymm6
+; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
+; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u]
+; AVX2-FCP-NEXT:    vpblendvb %ymm8, %ymm6, %ymm7, %ymm6
 ; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,ymm3[14],zero,zero,zero,zero,zero,zero,ymm3[15],zero,zero,zero,zero,zero,zero,ymm3[16],zero,zero,zero,zero,zero,zero,ymm3[17],zero,zero,zero,zero,zero,zero,ymm3[18]
 ; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,ymm1[14],zero,zero,zero,zero,zero,zero,ymm1[15],zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,ymm1[17],zero,zero,zero,zero,zero,zero,ymm1[18],zero
 ; AVX2-FCP-NEXT:    vpor %ymm3, %ymm1, %ymm1
 ; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,ymm2[14],zero,zero,zero,zero,zero,zero,ymm2[15],zero,zero,zero,zero,zero,zero,ymm2[16],zero,zero,zero,zero,zero,zero,ymm2[17],zero,zero,zero,zero,zero
 ; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,zero,zero
 ; AVX2-FCP-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255,255]
-; AVX2-FCP-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,ymm6[1,2,3,0,1,14],zero,ymm6[0,1,0,1,14,15],zero,ymm6[15,16,17,18,19,16],zero,ymm6[30,31,16,17,16,17],zero,ymm6[31,30,31]
-; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm8[13],zero,zero,zero,zero,zero,zero,ymm8[14],zero,zero,zero,zero,zero,zero,ymm8[15],zero,zero,zero,zero,zero,zero,ymm8[16],zero,zero,zero,zero,zero,zero,ymm8[17],zero,zero,zero
-; AVX2-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm7[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
-; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u,u,u,255,255,0,u,u]
-; AVX2-FCP-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
-; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255]
-; AVX2-FCP-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-FCP-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255]
+; AVX2-FCP-NEXT:    vpblendvb %ymm1, %ymm0, %ymm6, %ymm0
 ; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX2-FCP-NEXT:    vmovdqa %ymm0, 96(%rax)
 ; AVX2-FCP-NEXT:    vmovdqa %ymm10, 128(%rax)
@@ -9852,1130 +9827,1096 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-LABEL: store_i8_stride7_vf64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 (%rax), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 (%rax), %zmm14
 ; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %ymm16
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512BW-NEXT:    vpshufb %ymm0, %ymm16, %ymm2
-; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %ymm17
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512BW-NEXT:    vpshufb %ymm1, %ymm17, %ymm4
-; AVX512BW-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
-; AVX512BW-NEXT:    vpshufb %ymm4, %ymm16, %ymm4
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
-; AVX512BW-NEXT:    vpshufb %ymm5, %ymm17, %ymm5
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
+; AVX512BW-NEXT:    vpshufb %ymm2, %ymm16, %ymm1
+; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %ymm18
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
+; AVX512BW-NEXT:    vpshufb %ymm3, %ymm18, %ymm4
+; AVX512BW-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm16, %ymm4
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm18, %ymm5
 ; AVX512BW-NEXT:    vpor %ymm4, %ymm5, %ymm4
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm4
 ; AVX512BW-NEXT:    vmovdqa64 32(%rsi), %ymm24
 ; AVX512BW-NEXT:    vmovdqa64 32(%rdi), %ymm23
-; AVX512BW-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm23[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[0,0,1,1,4,4,5,5]
+; AVX512BW-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm23[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,1,1,4,4,5,5]
 ; AVX512BW-NEXT:    movl $676341840, %r10d # imm = 0x28502850
 ; AVX512BW-NEXT:    kmovd %r10d, %k1
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm4 {%k1} = ymm24[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm23, %ymm5
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512BW-NEXT:    vpshufb %ymm8, %ymm24, %ymm6
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm24[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm8 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
+; AVX512BW-NEXT:    vpshufb %ymm8, %ymm23, %ymm5
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
+; AVX512BW-NEXT:    vpshufb %ymm10, %ymm24, %ymm6
 ; AVX512BW-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm5
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm6
 ; AVX512BW-NEXT:    movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830
 ; AVX512BW-NEXT:    kmovq %r10, %k2
-; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm5 {%k2}
+; AVX512BW-NEXT:    vmovdqu8 %zmm4, %zmm6 {%k2}
 ; AVX512BW-NEXT:    vmovdqa 32(%r9), %ymm4
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
-; AVX512BW-NEXT:    vpshufb %ymm2, %ymm4, %ymm9
-; AVX512BW-NEXT:    vmovdqa 32(%r8), %ymm10
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512BW-NEXT:    vpshufb %ymm6, %ymm10, %ymm11
-; AVX512BW-NEXT:    vpor %ymm9, %ymm11, %ymm9
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512BW-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm5 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
+; AVX512BW-NEXT:    vpshufb %ymm5, %ymm4, %ymm7
+; AVX512BW-NEXT:    vmovdqa 32(%r8), %ymm9
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm9, %ymm11
+; AVX512BW-NEXT:    vpor %ymm7, %ymm11, %ymm11
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm9, %ymm9
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm4, %ymm4
+; AVX512BW-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm9, %zmm4
-; AVX512BW-NEXT:    vmovdqa 32(%rax), %ymm9
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512BW-NEXT:    vpermw %ymm9, %ymm10, %ymm10
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm11, %zmm4
+; AVX512BW-NEXT:    vmovdqa 32(%rax), %ymm11
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512BW-NEXT:    vpermw %ymm11, %ymm0, %ymm12
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
 ; AVX512BW-NEXT:    movabsq $145249953336295682, %rax # imm = 0x204081020408102
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm9, %zmm4 {%k3}
+; AVX512BW-NEXT:    vmovdqu8 %zmm11, %zmm4 {%k3}
 ; AVX512BW-NEXT:    movabsq $-4357498600088870461, %rax # imm = 0xC3870E1C3870E1C3
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm4, %zmm5 {%k3}
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm14
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm14, %ymm4
+; AVX512BW-NEXT:    vmovdqu8 %zmm4, %zmm6 {%k3}
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm13
+; AVX512BW-NEXT:    vpshufb %ymm8, %ymm13, %ymm4
 ; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512BW-NEXT:    vpshufb %ymm8, %ymm15, %ymm7
-; AVX512BW-NEXT:    vpor %ymm4, %ymm7, %ymm4
+; AVX512BW-NEXT:    vpshufb %ymm10, %ymm15, %ymm8
+; AVX512BW-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %xmm25
 ; AVX512BW-NEXT:    vmovdqa 32(%rdi), %xmm8
 ; AVX512BW-NEXT:    vmovdqa64 (%rsi), %xmm26
 ; AVX512BW-NEXT:    vmovdqa 32(%rsi), %xmm10
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm7, %zmm4
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpshufb %ymm0, %ymm18, %ymm0
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm11 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512BW-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,0,1]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm12, %zmm4
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpshufb %ymm2, %ymm17, %ymm2
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpshufb %ymm1, %ymm19, %ymm1
-; AVX512BW-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm19[8],xmm18[8],xmm19[9],xmm18[9],xmm19[10],xmm18[10],xmm19[11],xmm18[11],xmm19[12],xmm18[12],xmm19[13],xmm18[13],xmm19[14],xmm18[14],xmm19[15],xmm18[15]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpshufb %ymm3, %ymm19, %ymm3
+; AVX512BW-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm19[8],xmm17[8],xmm19[9],xmm17[9],xmm19[10],xmm17[10],xmm19[11],xmm17[11],xmm19[12],xmm17[12],xmm19[13],xmm17[13],xmm19[14],xmm17[14],xmm19[15],xmm17[15]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512BW-NEXT:    vpshufb %xmm12, %xmm3, %xmm3
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
 ; AVX512BW-NEXT:    movabsq $435749860008887046, %rax # imm = 0x60C183060C18306
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm4 {%k3}
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %ymm20
-; AVX512BW-NEXT:    vpshufb %ymm2, %ymm20, %ymm0
-; AVX512BW-NEXT:    vmovdqa64 (%r8), %ymm21
-; AVX512BW-NEXT:    vpshufb %ymm6, %ymm21, %ymm1
-; AVX512BW-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %xmm30
-; AVX512BW-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm30[8],xmm2[9],xmm30[9],xmm2[10],xmm30[10],xmm2[11],xmm30[11],xmm2[12],xmm30[12],xmm2[13],xmm30[13],xmm2[14],xmm30[14],xmm2[15],xmm30[15]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
-; AVX512BW-NEXT:    vpermw %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm4 {%k3}
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpshufb %ymm5, %ymm20, %ymm2
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm22, %ymm3
+; AVX512BW-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm22[8],xmm20[8],xmm22[9],xmm20[9],xmm22[10],xmm20[10],xmm22[11],xmm20[11],xmm22[12],xmm20[12],xmm22[13],xmm20[13],xmm22[14],xmm20[14],xmm22[15],xmm20[15]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm9 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512BW-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
+; AVX512BW-NEXT:    vpermw %zmm14, %zmm3, %zmm3
 ; AVX512BW-NEXT:    movabsq $2323999253380730912, %rax # imm = 0x2040810204081020
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
+; AVX512BW-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k3}
 ; AVX512BW-NEXT:    movabsq $4066998693416279096, %rax # imm = 0x3870E1C3870E1C38
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm4 {%k3}
-; AVX512BW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm14[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,3,3,6,6,7,7]
-; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14]
+; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm4 {%k3}
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm13[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,3,3,6,6,7,7]
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14]
 ; AVX512BW-NEXT:    movl $338170920, %eax # imm = 0x14281428
 ; AVX512BW-NEXT:    kmovd %eax, %k3
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm15, %ymm0 {%k3}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm6, %xmm6
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm6[0,1,0,1]
-; AVX512BW-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm18[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,2,3,3,4,6,7,7]
+; AVX512BW-NEXT:    vpshufb %ymm3, %ymm15, %ymm2 {%k3}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} xmm31 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512BW-NEXT:    vpshufb %xmm31, %xmm5, %xmm5
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[2,3,2,3],zmm5[0,1,0,1]
 ; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [11,0,0,0,15,14,13,12,11,0,0,0,15,14,13,12,11,0,0,0,15,14,13,12,11,0,0,0,15,14,13,12]
-; AVX512BW-NEXT:    vpshufb %ymm1, %ymm19, %ymm9
-; AVX512BW-NEXT:    vmovdqu8 %ymm6, %ymm9 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %xmm29
-; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %xmm31
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm31[0],xmm29[0],xmm31[1],xmm29[1],xmm31[2],xmm29[2],xmm31[3],xmm29[3],xmm31[4],xmm29[4],xmm31[5],xmm29[5],xmm31[6],xmm29[6],xmm31[7],xmm29[7]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm11 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm6, %xmm6
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm9[2,3,2,3],zmm6[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm22 {%k2}
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm20[27],zero,zero,zero,zero,ymm20[30],zero,ymm20[28],zero,zero,zero,zero,ymm20[31],zero
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm21[27],zero,zero,zero,zero,ymm21[30],zero,ymm21[28],zero,zero,zero,zero,ymm21[31],zero,ymm21[29]
-; AVX512BW-NEXT:    vpor %ymm0, %ymm6, %ymm0
-; AVX512BW-NEXT:    vmovdqa64 32(%r9), %xmm27
-; AVX512BW-NEXT:    vmovdqa64 32(%r8), %xmm28
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3],xmm28[4],xmm27[4],xmm28[5],xmm27[5],xmm28[6],xmm27[6],xmm28[7],xmm27[7]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm6[0,1,0,1]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
-; AVX512BW-NEXT:    vpermw %zmm3, %zmm6, %zmm6
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm19, %ymm5
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm17[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[0,2,3,3,4,6,7,7]
+; AVX512BW-NEXT:    vmovdqu8 %ymm7, %ymm5 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %xmm27
+; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %xmm29
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm29[0],xmm27[0],xmm29[1],xmm27[1],xmm29[2],xmm27[2],xmm29[3],xmm27[3],xmm29[4],xmm27[4],xmm29[5],xmm27[5],xmm29[6],xmm27[6],xmm29[7],xmm27[7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm7 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm21, %xmm21
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm5[2,3,2,3],zmm21[0,1,0,1]
+; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm21 {%k2}
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm20[27],zero,zero,zero,zero,ymm20[30],zero,ymm20[28],zero,zero,zero,zero,ymm20[31],zero
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm22[27],zero,zero,zero,zero,ymm22[30],zero,ymm22[28],zero,zero,zero,zero,ymm22[31],zero,ymm22[29]
+; AVX512BW-NEXT:    vpor %ymm2, %ymm5, %ymm2
+; AVX512BW-NEXT:    vmovdqa64 32(%r9), %xmm28
+; AVX512BW-NEXT:    vmovdqa64 32(%r8), %xmm30
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm5 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
+; AVX512BW-NEXT:    vpermw %zmm14, %zmm2, %zmm2
 ; AVX512BW-NEXT:    movabsq $580999813345182728, %rax # imm = 0x810204081020408
 ; AVX512BW-NEXT:    kmovq %rax, %k2
-; AVX512BW-NEXT:    vmovdqu8 %zmm6, %zmm0 {%k2}
+; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k2}
 ; AVX512BW-NEXT:    movabsq $1016749673354069774, %rax # imm = 0xE1C3870E1C3870E
 ; AVX512BW-NEXT:    kmovq %rax, %k2
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm22 {%k2}
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm21 {%k2}
 ; AVX512BW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm23[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,3,3,6,6,7,7]
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm24, %ymm0 {%k3}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
-; AVX512BW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm24, %ymm6
+; AVX512BW-NEXT:    vpshufb %ymm3, %ymm24, %ymm0 {%k3}
+; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
+; AVX512BW-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vpshufb %ymm3, %ymm24, %ymm2
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm24 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
 ; AVX512BW-NEXT:    vpshufb %ymm24, %ymm23, %ymm23
-; AVX512BW-NEXT:    vporq %ymm6, %ymm23, %ymm6
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm6[2,3,2,3],zmm0[2,3,2,3]
+; AVX512BW-NEXT:    vporq %ymm2, %ymm23, %ymm2
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm2[2,3,2,3],zmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm16[24,25],zero,ymm16[23],zero,ymm16[21,22,23,26],zero,ymm16[24],zero,ymm16[28,29,26,27]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm6 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm17[25],zero,ymm17[23],zero,zero,zero,zero,ymm17[26],zero,ymm17[24],zero,zero,zero,zero
-; AVX512BW-NEXT:    vpor %ymm0, %ymm6, %ymm0
-; AVX512BW-NEXT:    vpshufb %ymm1, %ymm17, %ymm1
-; AVX512BW-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm16[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,2,3,3,4,6,7,7]
-; AVX512BW-NEXT:    vmovdqu8 %ymm6, %ymm1 {%k1}
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm18[25],zero,ymm18[23],zero,zero,zero,zero,ymm18[26],zero,ymm18[24],zero,zero,zero,zero
+; AVX512BW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm18, %ymm1
+; AVX512BW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm16[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,2,3,3,4,6,7,7]
+; AVX512BW-NEXT:    vmovdqu8 %ymm2, %ymm1 {%k1}
 ; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm1[2,3,2,3]
 ; AVX512BW-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
 ; AVX512BW-NEXT:    kmovq %rax, %k2
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k2}
 ; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm1
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm0[4,5,6,7],zmm1[4,5,6,7]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm6[23],zero,zmm6[23,24,25,26],zero,zmm6[24],zero,zmm6[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm6[59],zero,zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm1[4,5,6,7],zmm0[4,5,6,7]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm16 = zmm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm16[25],zero,zmm16[23],zero,zero,zero,zero,zmm16[26],zero,zmm16[24],zero,zero,zmm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm16[59],zero,zero,zero,zero,zmm16[62],zero,zmm16[60],zero,zero,zero,zero,zmm16[63],zero,zmm16[61]
-; AVX512BW-NEXT:    vporq %zmm6, %zmm16, %zmm6
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm6 = zmm6[2,3,2,3,6,7,6,7]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm0[4,5,6,7],zmm1[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7],zmm0[4,5,6,7]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm2[23],zero,zmm2[23,24,25,26],zero,zmm2[24],zero,zmm2[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm2[59],zero,zero,zero,zero,zmm2[62],zero,zmm2[60],zero,zero,zero,zero,zmm2[63],zero
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm0[25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero,zmm0[61]
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512BW-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
 ; AVX512BW-NEXT:    kmovq %rax, %k2
-; AVX512BW-NEXT:    vmovdqu8 %zmm6, %zmm23 {%k2}
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
-; AVX512BW-NEXT:    vpermw %zmm3, %zmm6, %zmm6
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k2}
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
+; AVX512BW-NEXT:    vpermw %zmm14, %zmm0, %zmm0
 ; AVX512BW-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm6, %zmm23 {%k3}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm26, %xmm16
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm25, %xmm13
-; AVX512BW-NEXT:    vporq %xmm16, %xmm13, %xmm13
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm25[0],xmm26[0],xmm25[1],xmm26[1],xmm25[2],xmm26[2],xmm25[3],xmm26[3],xmm25[4],xmm26[4],xmm25[5],xmm26[5],xmm25[6],xmm26[6],xmm25[7],xmm26[7]
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm16, %xmm12
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm12[0,1,0,1],zmm13[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512BW-NEXT:    vpshufb %xmm25, %xmm19, %xmm12
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm13 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm18, %xmm26
-; AVX512BW-NEXT:    vporq %xmm12, %xmm26, %xmm12
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm26 = xmm18[0],xmm19[0],xmm18[1],xmm19[1],xmm18[2],xmm19[2],xmm18[3],xmm19[3],xmm18[4],xmm19[4],xmm18[5],xmm19[5],xmm18[6],xmm19[6],xmm18[7],xmm19[7]
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm26, %xmm11
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm11 = zmm11[0,1,0,1],zmm12[0,1,0,1]
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k3}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} xmm18 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm26, %xmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm25, %xmm1
+; AVX512BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm25[0],xmm26[0],xmm25[1],xmm26[1],xmm25[2],xmm26[2],xmm25[3],xmm26[3],xmm25[4],xmm26[4],xmm25[5],xmm26[5],xmm25[6],xmm26[6],xmm25[7],xmm26[7]
+; AVX512BW-NEXT:    vpshufb %xmm31, %xmm1, %xmm1
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm19, %xmm1
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} xmm26 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512BW-NEXT:    vpshufb %xmm26, %xmm17, %xmm25
+; AVX512BW-NEXT:    vporq %xmm1, %xmm25, %xmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm17[0],xmm19[0],xmm17[1],xmm19[1],xmm17[2],xmm19[2],xmm17[3],xmm19[3],xmm17[4],xmm19[4],xmm17[5],xmm19[5],xmm17[6],xmm19[6],xmm17[7],xmm19[7]
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm25, %xmm7
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm7[0,1,0,1],zmm1[0,1,0,1]
 ; AVX512BW-NEXT:    movabsq $871499720017774092, %rax # imm = 0xC183060C183060C
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm11, %zmm16 {%k3}
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm11 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm30, %xmm26
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm2, %xmm9
-; AVX512BW-NEXT:    vporq %xmm26, %xmm9, %xmm9
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3],xmm2[4],xmm30[4],xmm2[5],xmm30[5],xmm2[6],xmm30[6],xmm2[7],xmm30[7]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm9[0,1,0,1]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
-; AVX512BW-NEXT:    vpermw %zmm3, %zmm9, %zmm9
+; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm16 {%k3}
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm20, %xmm7
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
+; AVX512BW-NEXT:    vpshufb %xmm25, %xmm22, %xmm31
+; AVX512BW-NEXT:    vporq %xmm7, %xmm31, %xmm7
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm31 = xmm22[0],xmm20[0],xmm22[1],xmm20[1],xmm22[2],xmm20[2],xmm22[3],xmm20[3],xmm22[4],xmm20[4],xmm22[5],xmm20[5],xmm22[6],xmm20[6],xmm22[7],xmm20[7]
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm31, %xmm5
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,0,1],zmm7[0,1,0,1]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
+; AVX512BW-NEXT:    vpermw %zmm14, %zmm7, %zmm7
 ; AVX512BW-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm9, %zmm2 {%k3}
+; AVX512BW-NEXT:    vmovdqu8 %zmm7, %zmm5 {%k3}
 ; AVX512BW-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
 ; AVX512BW-NEXT:    kmovq %rax, %k3
-; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm16 {%k3}
-; AVX512BW-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm14[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,1,1,4,4,5,5]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm15[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
-; AVX512BW-NEXT:    vpshufb %ymm7, %ymm15, %ymm7
-; AVX512BW-NEXT:    vpshufb %ymm24, %ymm14, %ymm9
-; AVX512BW-NEXT:    vpor %ymm7, %ymm9, %ymm7
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[2,3,2,3],zmm7[2,3,2,3]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm18[18],zero,zmm18[18,19,20,21],zero,zmm18[19],zero,zmm18[25,26,27,22],zero,zmm18[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm18[56,57],zero,zmm18[55],zero,zmm18[53,54,55,58],zero,zmm18[56],zero,zmm18[60,61,58,59]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm9 = zmm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm19[18],zero,zero,zero,zero,zmm19[21],zero,zmm19[19],zero,zero,zero,zero,zmm19[22],zero,zmm19[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm19[57],zero,zmm19[55],zero,zero,zero,zero,zmm19[58],zero,zmm19[56],zero,zero,zero,zero
-; AVX512BW-NEXT:    vporq %zmm7, %zmm9, %zmm7
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
-; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm7 {%k2}
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[20],zero,zmm0[18],zero,zmm0[20,21,20,21],zero,zmm0[19],zero,zmm0[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[56,57,56,57],zero,zmm0[55],zero,zmm0[55,56,57,58],zero,zmm0[56],zero,zmm0[62,63]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm1[20],zero,zmm1[18],zero,zero,zero,zero,zmm1[21],zero,zmm1[19],zero,zero,zero,zero,zmm1[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm1[57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero
-; AVX512BW-NEXT:    vporq %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512BW-NEXT:    vpermw %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    vmovdqu8 %zmm5, %zmm16 {%k3}
+; AVX512BW-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm13[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[0,0,1,1,4,4,5,5]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm5 {%k1} = ymm15[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
+; AVX512BW-NEXT:    vpshufb %ymm3, %ymm15, %ymm3
+; AVX512BW-NEXT:    vpshufb %ymm24, %ymm13, %ymm7
+; AVX512BW-NEXT:    vpor %ymm3, %ymm7, %ymm3
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm5[2,3,2,3],zmm3[2,3,2,3]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm17[18],zero,zmm17[18,19,20,21],zero,zmm17[19],zero,zmm17[25,26,27,22],zero,zmm17[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm17[56,57],zero,zmm17[55],zero,zmm17[53,54,55,58],zero,zmm17[56],zero,zmm17[60,61,58,59]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm7 = zmm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm19[18],zero,zero,zero,zero,zmm19[21],zero,zmm19[19],zero,zero,zero,zero,zmm19[22],zero,zmm19[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm19[57],zero,zmm19[55],zero,zero,zero,zero,zmm19[58],zero,zmm19[56],zero,zero,zero,zero
+; AVX512BW-NEXT:    vporq %zmm5, %zmm7, %zmm5
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,3,2,3,6,7,6,7]
+; AVX512BW-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k2}
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm22[20],zero,zmm22[18],zero,zmm22[20,21,20,21],zero,zmm22[19],zero,zmm22[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm22[56,57,56,57],zero,zmm22[55],zero,zmm22[55,56,57,58],zero,zmm22[56],zero,zmm22[62,63]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm7 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm20[20],zero,zmm20[18],zero,zero,zero,zero,zmm20[21],zero,zmm20[19],zero,zero,zero,zero,zmm20[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm20[57],zero,zmm20[55],zero,zero,zero,zero,zmm20[58],zero,zmm20[56],zero,zero
+; AVX512BW-NEXT:    vporq %zmm3, %zmm7, %zmm3
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512BW-NEXT:    vpermw %zmm14, %zmm7, %zmm7
 ; AVX512BW-NEXT:    movabsq $1161999626690365456, %rax # imm = 0x1020408102040810
 ; AVX512BW-NEXT:    kmovq %rax, %k1
-; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqu8 %zmm7, %zmm3 {%k1}
 ; AVX512BW-NEXT:    movabsq $2033499346708139548, %rax # imm = 0x1C3870E1C3870E1C
 ; AVX512BW-NEXT:    kmovq %rax, %k1
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm7 {%k1}
-; AVX512BW-NEXT:    vpshufb %xmm25, %xmm29, %xmm0
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm31, %xmm1
+; AVX512BW-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k1}
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm0
+; AVX512BW-NEXT:    vpshufb %xmm26, %xmm29, %xmm3
+; AVX512BW-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm27[8],xmm29[8],xmm27[9],xmm29[9],xmm27[10],xmm29[10],xmm27[11],xmm29[11],xmm27[12],xmm29[12],xmm27[13],xmm29[13],xmm27[14],xmm29[14],xmm27[15],xmm29[15]
+; AVX512BW-NEXT:    vpshufb %xmm12, %xmm3, %xmm3
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm10, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm2
+; AVX512BW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm8[8],xmm10[9],xmm8[9],xmm10[10],xmm8[10],xmm10[11],xmm8[11],xmm10[12],xmm8[12],xmm10[13],xmm8[13],xmm10[14],xmm8[14],xmm10[15],xmm8[15]
+; AVX512BW-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm3[0,1,0,1]
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm2 {%k2}
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm28, %xmm0
+; AVX512BW-NEXT:    vpshufb %xmm25, %xmm30, %xmm1
 ; AVX512BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm29[8],xmm31[8],xmm29[9],xmm31[9],xmm29[10],xmm31[10],xmm29[11],xmm31[11],xmm29[12],xmm31[12],xmm29[13],xmm31[13],xmm29[14],xmm31[14],xmm29[15],xmm31[15]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
+; AVX512BW-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm1[0,1,0,1]
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm10, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm8, %xmm2
-; AVX512BW-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm10[8],xmm8[8],xmm10[9],xmm8[9],xmm10[10],xmm8[10],xmm10[11],xmm8[11],xmm10[12],xmm8[12],xmm10[13],xmm8[13],xmm10[14],xmm8[14],xmm10[15],xmm8[15]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,0,1],zmm2[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k2}
-; AVX512BW-NEXT:    vpshufb %xmm11, %xmm27, %xmm0
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm28, %xmm2
-; AVX512BW-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm28[8],xmm27[8],xmm28[9],xmm27[9],xmm28[10],xmm27[10],xmm28[11],xmm27[11],xmm28[12],xmm27[12],xmm28[13],xmm27[13],xmm28[14],xmm27[14],xmm28[15],xmm27[15]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm2[0,1,0,1]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
-; AVX512BW-NEXT:    vpermw %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
+; AVX512BW-NEXT:    vpermw %zmm14, %zmm1, %zmm1
 ; AVX512BW-NEXT:    movabsq $290499906672591364, %rax # imm = 0x408102040810204
 ; AVX512BW-NEXT:    kmovq %rax, %k1
-; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387
 ; AVX512BW-NEXT:    kmovq %rax, %k1
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm2 {%k1}
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 256(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 320(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm16, (%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm23, 384(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 192(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, 64(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BW-FCP-LABEL: store_i8_stride7_vf64:
 ; AVX512BW-FCP:       # %bb.0:
-; AVX512BW-FCP-NEXT:    subq $200, %rsp
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm8
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm7
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm5
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm17
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm3
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm3, %ymm6
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm11
-; AVX512BW-FCP-NEXT:    vpor %ymm6, %ymm11, %ymm6
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm3, %ymm3
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm10, %ymm10
-; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm10, %ymm3
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm6, %zmm11
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm6
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm10
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm12
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm14
-; AVX512BW-FCP-NEXT:    vpor %ymm10, %ymm14, %ymm14
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm6, %ymm6
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm12, %ymm12
-; AVX512BW-FCP-NEXT:    vpor %ymm6, %ymm12, %ymm6
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,3,2,3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm10
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm16
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm0
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm3
+; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
+; AVX512BW-FCP-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm3
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm5
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm5, %ymm6
+; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm6, %ymm3
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm5, %ymm5
+; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm5
 ; AVX512BW-FCP-NEXT:    movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830
 ; AVX512BW-FCP-NEXT:    kmovq %r10, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm11, %zmm10 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm14
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm19 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm19, %ymm14, %ymm11
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm12
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm20 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm20, %ymm12, %ymm15
-; AVX512BW-FCP-NEXT:    vpor %ymm11, %ymm15, %ymm15
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm6, %ymm12, %ymm16
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm6, %ymm14, %ymm14
-; AVX512BW-FCP-NEXT:    vporq %ymm16, %ymm14, %ymm14
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,3,2,3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm15, %zmm14
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rax), %ymm15
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10]
-; AVX512BW-FCP-NEXT:    vpermw %ymm15, %ymm16, %ymm16
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm15, %zmm15
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm5 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm6
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm7
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm8
+; AVX512BW-FCP-NEXT:    vpor %ymm6, %ymm8, %ymm8
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm7, %ymm10
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm10, %ymm3
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm3
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rax), %ymm10
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512BW-FCP-NEXT:    vpermw %ymm10, %ymm4, %ymm11
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
 ; AVX512BW-FCP-NEXT:    movabsq $145249953336295682, %rax # imm = 0x204081020408102
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm14 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm3 {%k2}
 ; AVX512BW-FCP-NEXT:    movabsq $-4357498600088870461, %rax # imm = 0xC3870E1C3870E1C3
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm10 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm14
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm14, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %ymm15
-; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm2
-; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %xmm21
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %xmm22
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm22[8],xmm21[8],xmm22[9],xmm21[9],xmm22[10],xmm21[10],xmm22[11],xmm21[11],xmm22[12],xmm21[12],xmm22[13],xmm21[13],xmm22[14],xmm21[14],xmm22[15],xmm21[15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm16
-; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm16, %ymm2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm10
+; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm11
-; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm11, %ymm3
+; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm11, %ymm3
 ; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm23
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm24
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm24[8],xmm23[8],xmm24[9],xmm23[9],xmm24[10],xmm23[10],xmm24[11],xmm23[11],xmm24[12],xmm23[12],xmm24[13],xmm23[13],xmm24[14],xmm23[14],xmm24[15],xmm23[15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm13
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm3
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm18
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm18[8],xmm3[8],xmm18[9],xmm3[9],xmm18[10],xmm3[10],xmm18[11],xmm3[11],xmm18[12],xmm3[12],xmm18[13],xmm3[13],xmm18[14],xmm3[14],xmm18[15],xmm3[15]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm9, %xmm9
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm9
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm2
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm14
+; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm14, %ymm2
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm19 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm14, %xmm14
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm14, %zmm2
 ; AVX512BW-FCP-NEXT:    movabsq $435749860008887046, %rax # imm = 0x60C183060C18306
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm13 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %ymm18
-; AVX512BW-FCP-NEXT:    vpshufb %ymm19, %ymm18, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %ymm19
-; AVX512BW-FCP-NEXT:    vpshufb %ymm20, %ymm19, %ymm2
-; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %xmm25
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %xmm27
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm27[8],xmm25[8],xmm27[9],xmm25[9],xmm27[10],xmm25[10],xmm27[11],xmm25[11],xmm27[12],xmm25[12],xmm27[13],xmm25[13],xmm27[14],xmm25[14],xmm27[15],xmm25[15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm17, %zmm2, %zmm2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm9 {%k2}
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
+; AVX512BW-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm20 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm1, %xmm1
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
+; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    movabsq $2323999253380730912, %rax # imm = 0x2040810204081020
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
 ; AVX512BW-FCP-NEXT:    movabsq $4066998693416279096, %rax # imm = 0x3870E1C3870E1C38
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm13 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm5[4,5,6,7],zmm4[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm1[23],zero,zmm1[21,22,23,26],zero,zmm1[24],zero,zmm1[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero,zmm1[61],zero
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm4[4,5,6,7],zmm5[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm2[25],zero,zmm2[23],zero,zero,zero,zero,zmm2[26],zero,zmm2[24],zero,zero,zero,zero,zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[62],zero,zmm2[60],zero,zero,zero,zero,zmm2[63],zero,zmm2[61],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm7[4,5,6,7],zmm8[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm2[23],zero,zero,zero,zero,zmm2[26],zero,zmm2[24],zero,zero,zero,zero,zmm2[27],zero,zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm2[60],zero,zmm2[62,63,62,63],zero,zmm2[61],zero,zmm2[63,60,61]
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm8[4,5,6,7],zmm7[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[23],zero,zero,zero,zero,zmm3[26],zero,zmm3[24],zero,zero,zero,zero,zmm3[27],zero,zmm3[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm3[62],zero,zmm3[60],zero,zero,zero,zero,zmm3[63],zero,zmm3[61],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vporq %zmm2, %zmm3, %zmm2
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm26
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm2[2,3,2,3,6,7,6,7]
-; AVX512BW-FCP-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm9 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm21 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm18, %xmm0
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm22 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm22, %xmm3, %xmm1
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm18[0],xmm3[1],xmm18[1],xmm3[2],xmm18[2],xmm3[3],xmm18[3],xmm3[4],xmm18[4],xmm3[5],xmm18[5],xmm3[6],xmm18[6],xmm3[7],xmm18[7]
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm26 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm26, %xmm13, %xmm0
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm27 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm27, %xmm12, %xmm1
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3],xmm12[4],xmm13[4],xmm12[5],xmm13[5],xmm12[6],xmm13[6],xmm12[7],xmm13[7]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm23 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512BW-FCP-NEXT:    movabsq $871499720017774092, %rax # imm = 0xC183060C183060C
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm20 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7],zmm26[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm1[23],zero,zmm1[23,24,25,26],zero,zmm1[24],zero,zmm1[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm1[59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm26[4,5,6,7],zmm0[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[25],zero,zmm2[23],zero,zero,zero,zero,zmm2[26],zero,zmm2[24],zero,zero,zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm2[59],zero,zero,zero,zero,zmm2[62],zero,zmm2[60],zero,zero,zero,zero,zmm2[63],zero,zmm2[61]
-; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512BW-FCP-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm24 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm24, %xmm14, %xmm0
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm25, %xmm15, %xmm1
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm29 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm29, %xmm1, %xmm1
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
+; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm20 {%k2}
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm17, %zmm1, %zmm1
-; AVX512BW-FCP-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm20 {%k3}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm16[28,29,30],zero,ymm16[28],zero,ymm16[30,31,30,31],zero,ymm16[29],zero,ymm16[31,28,29]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm28
+; AVX512BW-FCP-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
+; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm8[0],xmm28[0],xmm8[1],xmm28[1],xmm8[2],xmm28[2],xmm8[3],xmm28[3],xmm8[4],xmm28[4],xmm8[5],xmm28[5],xmm8[6],xmm28[6],xmm8[7],xmm28[7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[28,29,30],zero,ymm10[28],zero,ymm10[30,31,30,31],zero,ymm10[29],zero,ymm10[31,28,29]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm11[30],zero,ymm11[28],zero,zero,zero,zero,ymm11[31],zero,ymm11[29],zero,zero,zero
 ; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm7
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %xmm6
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm31 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm31, %xmm3, %xmm3
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm1[2,3,2,3],zmm3[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero,ymm15[29],zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero,ymm14[29],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %ymm1, %ymm29, %ymm29
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm2
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm1
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm29[2,3,2,3],zmm0[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm29 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm18[27],zero,zero,zero,zero,ymm18[30],zero,ymm18[28],zero,zero,zero,zero,ymm18[31],zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm19[27],zero,zero,zero,zero,ymm19[30],zero,ymm19[28],zero,zero,zero,zero,ymm19[31],zero,ymm19[29]
-; AVX512BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm3
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %xmm5
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%r8), %xmm4
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm30 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm30, %xmm0, %xmm0
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm3[2,3,2,3],zmm0[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
-; AVX512BW-FCP-NEXT:    vpermw %zmm17, %zmm3, %zmm3
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm4
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm3
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm23 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm13[30],zero,ymm13[28],zero,zero,zero,zero,ymm13[31],zero,ymm13[29],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm30 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %ymm23, %ymm30, %ymm23
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm23[2,3,2,3],zmm1[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm30
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm31
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm30[0],xmm31[1],xmm30[1],xmm31[2],xmm30[2],xmm31[3],xmm30[3],xmm31[4],xmm30[4],xmm31[5],xmm30[5],xmm31[6],xmm30[6],xmm31[7],xmm30[7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm29, %xmm0, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm14[27],zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero,ymm15[29]
+; AVX512BW-FCP-NEXT:    vporq %ymm1, %ymm29, %ymm1
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
+; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    movabsq $580999813345182728, %rax # imm = 0x810204081020408
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-FCP-NEXT:    movabsq $1016749673354069774, %rax # imm = 0xE1C3870E1C3870E
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm1
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm0[4,5,6,7],zmm1[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7],zmm0[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm29[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm29[23],zero,zmm29[21,22,23,26],zero,zmm29[24],zero,zmm29[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm29[62],zero,zmm29[60],zero,zero,zero,zero,zmm29[63],zero,zmm29[61],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm0[25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zero,zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero,zmm0[61],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm0, %zmm29
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm2
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm2[4,5,6,7],zmm1[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zero,zero,zmm6[27],zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm6[60],zero,zmm6[62,63,62,63],zero,zmm6[61],zero,zmm6[63,60,61]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm1[4,5,6,7],zmm2[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm7 = zmm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm7[23],zero,zero,zero,zero,zmm7[26],zero,zmm7[24],zero,zero,zero,zero,zmm7[27],zero,zmm7[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm7[62],zero,zmm7[60],zero,zero,zero,zero,zmm7[63],zero,zmm7[61],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vporq %zmm6, %zmm7, %zmm6
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm29[2,3,2,3,6,7,6,7]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm29 = zmm6[2,3,2,3,6,7,6,7]
+; AVX512BW-FCP-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
+; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm7, %zmm29 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm6
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm7
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm6[4,5,6,7],zmm7[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm7[4,5,6,7],zmm6[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm0[23],zero,zmm0[23,24,25,26],zero,zmm0[24],zero,zmm0[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm6[25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm6[59],zero,zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero,zmm6[61]
+; AVX512BW-FCP-NEXT:    vporq %zmm0, %zmm6, %zmm0
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
+; AVX512BW-FCP-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
+; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm3
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm2, %xmm28
-; AVX512BW-FCP-NEXT:    vporq %xmm3, %xmm28, %xmm3
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,0,1],zmm1[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm7, %xmm28
-; AVX512BW-FCP-NEXT:    vporq %xmm3, %xmm28, %xmm3
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm28 = zmm3[0,1,0,1],zmm6[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm28 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm3
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm7
-; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm7, %xmm3
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
+; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm0, %zmm0
+; AVX512BW-FCP-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
+; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k2}
+; AVX512BW-FCP-NEXT:    vpshufb %xmm26, %xmm4, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm27, %xmm3, %xmm6
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm6, %xmm0
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm3, %xmm3
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm28, %xmm3
+; AVX512BW-FCP-NEXT:    vpshufb %xmm22, %xmm8, %xmm4
+; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm28[8],xmm8[8],xmm28[9],xmm8[9],xmm28[10],xmm8[10],xmm28[11],xmm8[11],xmm28[12],xmm8[12],xmm28[13],xmm8[13],xmm28[14],xmm8[14],xmm28[15],xmm8[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm4, %xmm4
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,0,1],zmm4[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k1}
+; AVX512BW-FCP-NEXT:    vpshufb %xmm24, %xmm30, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm25, %xmm31, %xmm4
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm31[8],xmm30[8],xmm31[9],xmm30[9],xmm31[10],xmm30[10],xmm31[11],xmm30[11],xmm31[12],xmm30[12],xmm31[13],xmm30[13],xmm31[14],xmm30[14],xmm31[15],xmm30[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm4, %xmm4
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm4[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
-; AVX512BW-FCP-NEXT:    vpermw %zmm17, %zmm4, %zmm4
+; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm4, %zmm4
 ; AVX512BW-FCP-NEXT:    movabsq $290499906672591364, %rax # imm = 0x408102040810204
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k1}
+; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k2}
 ; AVX512BW-FCP-NEXT:    movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm28 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb %xmm9, %xmm22, %xmm3
-; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm21, %xmm4
-; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm21[0],xmm22[0],xmm21[1],xmm22[1],xmm21[2],xmm22[2],xmm21[3],xmm22[3],xmm21[4],xmm22[4],xmm21[5],xmm22[5],xmm21[6],xmm22[6],xmm21[7],xmm22[7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm12, %xmm4, %xmm4
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm4[0,1,0,1],zmm3[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
-; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
-; AVX512BW-FCP-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm23[0],xmm24[0],xmm23[1],xmm24[1],xmm23[2],xmm24[2],xmm23[3],xmm24[3],xmm23[4],xmm24[4],xmm23[5],xmm24[5],xmm23[6],xmm24[6],xmm23[7],xmm24[7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm31, %xmm2, %xmm2
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512BW-FCP-NEXT:    movabsq $871499720017774092, %rax # imm = 0xC183060C183060C
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb %xmm1, %xmm25, %xmm1
-; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm27, %xmm2
-; AVX512BW-FCP-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm27[0],xmm25[0],xmm27[1],xmm25[1],xmm27[2],xmm25[2],xmm27[3],xmm25[3],xmm27[4],xmm25[4],xmm27[5],xmm25[5],xmm27[6],xmm25[6],xmm27[7],xmm25[7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm30, %xmm2, %xmm2
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,0,1],zmm1[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
-; AVX512BW-FCP-NEXT:    vpermw %zmm17, %zmm2, %zmm2
-; AVX512BW-FCP-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
-; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k2}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[18],zero,zmm12[18,19,20,21],zero,zmm12[19],zero,zmm12[25,26,27,22],zero,zmm12[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[56,57],zero,zmm12[55],zero,zmm12[53,54,55,58],zero,zmm12[56],zero,zmm12[60,61,58,59]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[18],zero,zero,zero,zero,zmm13[21],zero,zmm13[19],zero,zero,zero,zero,zmm13[22],zero,zmm13[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm13[57],zero,zmm13[55],zero,zero,zero,zero,zmm13[58],zero,zmm13[56],zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vporq %zmm0, %zmm4, %zmm0
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[18,19,20,21],zero,zmm1[19],zero,zmm1[21,20,21,22],zero,zmm1[20],zero,zmm1[22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero,zero,zero,zmm1[59],zero
-; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm2, %zmm2
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[21],zero,zmm2[19],zero,zero,zero,zero,zmm2[22],zero,zmm2[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[55],zero,zero,zero,zero,zmm2[58],zero,zmm2[56],zero,zero,zero,zero,zmm2[59],zero,zmm2[57]
 ; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm2, %zmm2
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[18],zero,zmm2[18,19,20,21],zero,zmm2[19],zero,zmm2[25,26,27,22],zero,zmm2[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[56,57],zero,zmm2[55],zero,zmm2[53,54,55,58],zero,zmm2[56],zero,zmm2[60,61,58,59]
-; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm3, %zmm3
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[18],zero,zero,zero,zero,zmm3[21],zero,zmm3[19],zero,zero,zero,zero,zmm3[22],zero,zmm3[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm3[57],zero,zmm3[55],zero,zero,zero,zero,zmm3[58],zero,zmm3[56],zero,zero,zero,zero
-; AVX512BW-FCP-NEXT:    vporq %zmm2, %zmm3, %zmm2
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,3,2,3,6,7,6,7]
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm2 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm1, %zmm1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[20],zero,zmm1[18],zero,zmm1[20,21,20,21],zero,zmm1[19],zero,zmm1[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[56,57,56,57],zero,zmm1[55],zero,zmm1[55,56,57,58],zero,zmm1[56],zero,zmm1[62,63]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm26, %zmm3
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[20],zero,zmm3[18],zero,zero,zero,zero,zmm3[21],zero,zmm3[19],zero,zero,zero,zero,zmm3[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm3[57],zero,zmm3[55],zero,zero,zero,zero,zmm3[58],zero,zmm3[56],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm3, %zmm1
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512BW-FCP-NEXT:    vpermw %zmm17, %zmm3, %zmm3
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[20],zero,zmm15[18],zero,zmm15[20,21,20,21],zero,zmm15[19],zero,zmm15[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[56,57,56,57],zero,zmm15[55],zero,zmm15[55,56,57,58],zero,zmm15[56],zero,zmm15[62,63]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm14[20],zero,zmm14[18],zero,zero,zero,zero,zmm14[21],zero,zmm14[19],zero,zero,zero,zero,zmm14[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm14[57],zero,zmm14[55],zero,zero,zero,zero,zmm14[58],zero,zmm14[56],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm2, %zmm2
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
 ; AVX512BW-FCP-NEXT:    movabsq $1161999626690365456, %rax # imm = 0x1020408102040810
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm1 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k1}
 ; AVX512BW-FCP-NEXT:    movabsq $2033499346708139548, %rax # imm = 0x1C3870E1C3870E1C
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm2 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm28, 256(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm29, 192(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm20, 384(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512BW-FCP-NEXT:    addq $200, %rsp
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm29, 384(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm23, 192(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm18, (%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
 ; AVX512DQ-BW-LABEL: store_i8_stride7_vf64:
 ; AVX512DQ-BW:       # %bb.0:
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rax), %zmm3
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rax), %zmm14
 ; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %ymm16
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm16, %ymm2
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %ymm17
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm17, %ymm4
-; AVX512DQ-BW-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm4, %ymm16, %ymm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm5, %ymm17, %ymm5
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm2, %ymm16, %ymm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %ymm18
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm3, %ymm18, %ymm4
+; AVX512DQ-BW-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm16, %ymm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm18, %ymm5
 ; AVX512DQ-BW-NEXT:    vpor %ymm4, %ymm5, %ymm4
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm4
 ; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rsi), %ymm24
 ; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdi), %ymm23
-; AVX512DQ-BW-NEXT:    vpshuflw {{.*#+}} ymm4 = ymm23[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[0,0,1,1,4,4,5,5]
+; AVX512DQ-BW-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm23[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,1,1,4,4,5,5]
 ; AVX512DQ-BW-NEXT:    movl $676341840, %r10d # imm = 0x28502850
 ; AVX512DQ-BW-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm4 {%k1} = ymm24[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm23, %ymm5
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm24, %ymm6
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm24[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm8 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm23, %ymm5
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm10, %ymm24, %ymm6
 ; AVX512DQ-BW-NEXT:    vpor %ymm5, %ymm6, %ymm5
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm5
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm6
 ; AVX512DQ-BW-NEXT:    movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830
 ; AVX512DQ-BW-NEXT:    kmovq %r10, %k2
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm5 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm4, %zmm6 {%k2}
 ; AVX512DQ-BW-NEXT:    vmovdqa 32(%r9), %ymm4
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm2, %ymm4, %ymm9
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%r8), %ymm10
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm10, %ymm11
-; AVX512DQ-BW-NEXT:    vpor %ymm9, %ymm11, %ymm9
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512DQ-BW-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm5 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm5, %ymm4, %ymm7
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%r8), %ymm9
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm9, %ymm11
+; AVX512DQ-BW-NEXT:    vpor %ymm7, %ymm11, %ymm11
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm9, %ymm9
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm4, %ymm4
+; AVX512DQ-BW-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm9, %zmm4
-; AVX512DQ-BW-NEXT:    vmovdqa 32(%rax), %ymm9
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512DQ-BW-NEXT:    vpermw %ymm9, %ymm10, %ymm10
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm11, %zmm4
+; AVX512DQ-BW-NEXT:    vmovdqa 32(%rax), %ymm11
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512DQ-BW-NEXT:    vpermw %ymm11, %ymm0, %ymm12
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
 ; AVX512DQ-BW-NEXT:    movabsq $145249953336295682, %rax # imm = 0x204081020408102
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm9, %zmm4 {%k3}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm11, %zmm4 {%k3}
 ; AVX512DQ-BW-NEXT:    movabsq $-4357498600088870461, %rax # imm = 0xC3870E1C3870E1C3
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm4, %zmm5 {%k3}
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm14
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm14, %ymm4
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm4, %zmm6 {%k3}
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm13
+; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm13, %ymm4
 ; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm15, %ymm7
-; AVX512DQ-BW-NEXT:    vpor %ymm4, %ymm7, %ymm4
+; AVX512DQ-BW-NEXT:    vpshufb %ymm10, %ymm15, %ymm8
+; AVX512DQ-BW-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %xmm25
 ; AVX512DQ-BW-NEXT:    vmovdqa 32(%rdi), %xmm8
 ; AVX512DQ-BW-NEXT:    vmovdqa64 (%rsi), %xmm26
 ; AVX512DQ-BW-NEXT:    vmovdqa 32(%rsi), %xmm10
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm7, %zmm4
-; AVX512DQ-BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm0, %ymm18, %ymm0
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm11 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm4, %zmm12, %zmm4
+; AVX512DQ-BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm2, %ymm17, %ymm2
 ; AVX512DQ-BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm19, %ymm1
-; AVX512DQ-BW-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm19[8],xmm18[8],xmm19[9],xmm18[9],xmm19[10],xmm18[10],xmm19[11],xmm18[11],xmm19[12],xmm18[12],xmm19[13],xmm18[13],xmm19[14],xmm18[14],xmm19[15],xmm18[15]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-BW-NEXT:    vpshufb %ymm3, %ymm19, %ymm3
+; AVX512DQ-BW-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm19[8],xmm17[8],xmm19[9],xmm17[9],xmm19[10],xmm17[10],xmm19[11],xmm17[11],xmm19[12],xmm17[12],xmm19[13],xmm17[13],xmm19[14],xmm17[14],xmm19[15],xmm17[15]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm3, %xmm3
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
 ; AVX512DQ-BW-NEXT:    movabsq $435749860008887046, %rax # imm = 0x60C183060C18306
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm4 {%k3}
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %ymm20
-; AVX512DQ-BW-NEXT:    vpshufb %ymm2, %ymm20, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%r8), %ymm21
-; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm21, %ymm1
-; AVX512DQ-BW-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %xmm30
-; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm30[8],xmm2[9],xmm30[9],xmm2[10],xmm30[10],xmm2[11],xmm30[11],xmm2[12],xmm30[12],xmm2[13],xmm30[13],xmm2[14],xmm30[14],xmm2[15],xmm30[15]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
-; AVX512DQ-BW-NEXT:    vpermw %zmm3, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm4 {%k3}
+; AVX512DQ-BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm5, %ymm20, %ymm2
+; AVX512DQ-BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm22, %ymm3
+; AVX512DQ-BW-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm22[8],xmm20[8],xmm22[9],xmm20[9],xmm22[10],xmm20[10],xmm22[11],xmm20[11],xmm22[12],xmm20[12],xmm22[13],xmm20[13],xmm22[14],xmm20[14],xmm22[15],xmm20[15]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm9 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
+; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm3, %zmm3
 ; AVX512DQ-BW-NEXT:    movabsq $2323999253380730912, %rax # imm = 0x2040810204081020
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k3}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k3}
 ; AVX512DQ-BW-NEXT:    movabsq $4066998693416279096, %rax # imm = 0x3870E1C3870E1C38
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm4 {%k3}
-; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm14[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,3,3,6,6,7,7]
-; AVX512DQ-BW-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm4 {%k3}
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm13[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[2,2,3,3,6,6,7,7]
+; AVX512DQ-BW-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14,13,12,15,14]
 ; AVX512DQ-BW-NEXT:    movl $338170920, %eax # imm = 0x14281428
 ; AVX512DQ-BW-NEXT:    kmovd %eax, %k3
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm15, %ymm0 {%k3}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm6, %xmm6
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm6[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm18[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,2,3,3,4,6,7,7]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm3, %ymm15, %ymm2 {%k3}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3],xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} xmm31 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm31, %xmm5, %xmm5
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[2,3,2,3],zmm5[0,1,0,1]
 ; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [11,0,0,0,15,14,13,12,11,0,0,0,15,14,13,12,11,0,0,0,15,14,13,12,11,0,0,0,15,14,13,12]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm19, %ymm9
-; AVX512DQ-BW-NEXT:    vmovdqu8 %ymm6, %ymm9 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %xmm29
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %xmm31
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm31[0],xmm29[0],xmm31[1],xmm29[1],xmm31[2],xmm29[2],xmm31[3],xmm29[3],xmm31[4],xmm29[4],xmm31[5],xmm29[5],xmm31[6],xmm29[6],xmm31[7],xmm29[7]
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm11 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm6, %xmm6
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm9[2,3,2,3],zmm6[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm22 {%k2}
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm20[27],zero,zero,zero,zero,ymm20[30],zero,ymm20[28],zero,zero,zero,zero,ymm20[31],zero
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm21[27],zero,zero,zero,zero,ymm21[30],zero,ymm21[28],zero,zero,zero,zero,ymm21[31],zero,ymm21[29]
-; AVX512DQ-BW-NEXT:    vpor %ymm0, %ymm6, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r9), %xmm27
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r8), %xmm28
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3],xmm28[4],xmm27[4],xmm28[5],xmm27[5],xmm28[6],xmm27[6],xmm28[7],xmm27[7]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm6[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
-; AVX512DQ-BW-NEXT:    vpermw %zmm3, %zmm6, %zmm6
+; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm19, %ymm5
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm17[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[0,2,3,3,4,6,7,7]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %ymm7, %ymm5 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %xmm27
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %xmm29
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm29[0],xmm27[0],xmm29[1],xmm27[1],xmm29[2],xmm27[2],xmm29[3],xmm27[3],xmm29[4],xmm27[4],xmm29[5],xmm27[5],xmm29[6],xmm27[6],xmm29[7],xmm27[7]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm7 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm21, %xmm21
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm5[2,3,2,3],zmm21[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm21 {%k2}
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm20[27],zero,zero,zero,zero,ymm20[30],zero,ymm20[28],zero,zero,zero,zero,ymm20[31],zero
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm22[27],zero,zero,zero,zero,ymm22[30],zero,ymm22[28],zero,zero,zero,zero,ymm22[31],zero,ymm22[29]
+; AVX512DQ-BW-NEXT:    vpor %ymm2, %ymm5, %ymm2
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r9), %xmm28
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r8), %xmm30
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm5 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[2,3,2,3],zmm0[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
+; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm2, %zmm2
 ; AVX512DQ-BW-NEXT:    movabsq $580999813345182728, %rax # imm = 0x810204081020408
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm6, %zmm0 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k2}
 ; AVX512DQ-BW-NEXT:    movabsq $1016749673354069774, %rax # imm = 0xE1C3870E1C3870E
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm22 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm21 {%k2}
 ; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm23[0,1,2,3,6,7,7,6,8,9,10,11,14,15,15,14]
 ; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,3,3,6,6,7,7]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm24, %ymm0 {%k3}
-; AVX512DQ-BW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
-; AVX512DQ-BW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm24, %ymm6
+; AVX512DQ-BW-NEXT:    vpshufb %ymm3, %ymm24, %ymm0 {%k3}
+; AVX512DQ-BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
+; AVX512DQ-BW-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm3, %ymm24, %ymm2
 ; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} ymm24 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
 ; AVX512DQ-BW-NEXT:    vpshufb %ymm24, %ymm23, %ymm23
-; AVX512DQ-BW-NEXT:    vporq %ymm6, %ymm23, %ymm6
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm6[2,3,2,3],zmm0[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vporq %ymm2, %ymm23, %ymm2
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm2[2,3,2,3],zmm0[2,3,2,3]
 ; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm16[24,25],zero,ymm16[23],zero,ymm16[21,22,23,26],zero,ymm16[24],zero,ymm16[28,29,26,27]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm6 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm17[25],zero,ymm17[23],zero,zero,zero,zero,ymm17[26],zero,ymm17[24],zero,zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpor %ymm0, %ymm6, %ymm0
-; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm17, %ymm1
-; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm16[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,2,3,3,4,6,7,7]
-; AVX512DQ-BW-NEXT:    vmovdqu8 %ymm6, %ymm1 {%k1}
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm18[25],zero,ymm18[23],zero,zero,zero,zero,ymm18[26],zero,ymm18[24],zero,zero,zero,zero
+; AVX512DQ-BW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512DQ-BW-NEXT:    vpshufb %ymm1, %ymm18, %ymm1
+; AVX512DQ-BW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm16[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,2,3,3,4,6,7,7]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %ymm2, %ymm1 {%k1}
 ; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm1[2,3,2,3]
 ; AVX512DQ-BW-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k2
 ; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k2}
 ; AVX512DQ-BW-NEXT:    vmovdqa64 (%r8), %zmm0
 ; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %zmm1
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm0[4,5,6,7],zmm1[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm6[23],zero,zmm6[23,24,25,26],zero,zmm6[24],zero,zmm6[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm6[59],zero,zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm1[4,5,6,7],zmm0[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm16 = zmm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm16[25],zero,zmm16[23],zero,zero,zero,zero,zmm16[26],zero,zmm16[24],zero,zero,zmm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm16[59],zero,zero,zero,zero,zmm16[62],zero,zmm16[60],zero,zero,zero,zero,zmm16[63],zero,zmm16[61]
-; AVX512DQ-BW-NEXT:    vporq %zmm6, %zmm16, %zmm6
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm6 = zmm6[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm0[4,5,6,7],zmm1[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7],zmm0[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm2[23],zero,zmm2[23,24,25,26],zero,zmm2[24],zero,zmm2[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm2[59],zero,zero,zero,zero,zmm2[62],zero,zmm2[60],zero,zero,zero,zero,zmm2[63],zero
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm0[25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero,zmm0[61]
+; AVX512DQ-BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm6, %zmm23 {%k2}
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
-; AVX512DQ-BW-NEXT:    vpermw %zmm3, %zmm6, %zmm6
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k2}
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
+; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm0, %zmm0
 ; AVX512DQ-BW-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm6, %zmm23 {%k3}
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm26, %xmm16
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm6 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm25, %xmm13
-; AVX512DQ-BW-NEXT:    vporq %xmm16, %xmm13, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm25[0],xmm26[0],xmm25[1],xmm26[1],xmm25[2],xmm26[2],xmm25[3],xmm26[3],xmm25[4],xmm26[4],xmm25[5],xmm26[5],xmm25[6],xmm26[6],xmm25[7],xmm26[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm16, %xmm12
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm12[0,1,0,1],zmm13[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm25, %xmm19, %xmm12
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm13 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm18, %xmm26
-; AVX512DQ-BW-NEXT:    vporq %xmm12, %xmm26, %xmm12
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm26 = xmm18[0],xmm19[0],xmm18[1],xmm19[1],xmm18[2],xmm19[2],xmm18[3],xmm19[3],xmm18[4],xmm19[4],xmm18[5],xmm19[5],xmm18[6],xmm19[6],xmm18[7],xmm19[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm26, %xmm11
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm11 = zmm11[0,1,0,1],zmm12[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k3}
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} xmm18 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm26, %xmm0
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm25, %xmm1
+; AVX512DQ-BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm25[0],xmm26[0],xmm25[1],xmm26[1],xmm25[2],xmm26[2],xmm25[3],xmm26[3],xmm25[4],xmm26[4],xmm25[5],xmm26[5],xmm25[6],xmm26[6],xmm25[7],xmm26[7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm31, %xmm1, %xmm1
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm19, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} xmm26 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm26, %xmm17, %xmm25
+; AVX512DQ-BW-NEXT:    vporq %xmm1, %xmm25, %xmm1
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm17[0],xmm19[0],xmm17[1],xmm19[1],xmm17[2],xmm19[2],xmm17[3],xmm19[3],xmm17[4],xmm19[4],xmm17[5],xmm19[5],xmm17[6],xmm19[6],xmm17[7],xmm19[7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm25, %xmm7
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm7[0,1,0,1],zmm1[0,1,0,1]
 ; AVX512DQ-BW-NEXT:    movabsq $871499720017774092, %rax # imm = 0xC183060C183060C
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm11, %zmm16 {%k3}
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm11 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm30, %xmm26
-; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm2, %xmm9
-; AVX512DQ-BW-NEXT:    vporq %xmm26, %xmm9, %xmm9
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3],xmm2[4],xmm30[4],xmm2[5],xmm30[5],xmm2[6],xmm30[6],xmm2[7],xmm30[7]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm9[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
-; AVX512DQ-BW-NEXT:    vpermw %zmm3, %zmm9, %zmm9
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm1, %zmm16 {%k3}
+; AVX512DQ-BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm20, %xmm7
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm25, %xmm22, %xmm31
+; AVX512DQ-BW-NEXT:    vporq %xmm7, %xmm31, %xmm7
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm31 = xmm22[0],xmm20[0],xmm22[1],xmm20[1],xmm22[2],xmm20[2],xmm22[3],xmm20[3],xmm22[4],xmm20[4],xmm22[5],xmm20[5],xmm22[6],xmm20[6],xmm22[7],xmm20[7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm31, %xmm5
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,0,1],zmm7[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
+; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm7, %zmm7
 ; AVX512DQ-BW-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm9, %zmm2 {%k3}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm7, %zmm5 {%k3}
 ; AVX512DQ-BW-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm16 {%k3}
-; AVX512DQ-BW-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm14[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
-; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,0,1,1,4,4,5,5]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm15[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm7, %ymm15, %ymm7
-; AVX512DQ-BW-NEXT:    vpshufb %ymm24, %ymm14, %ymm9
-; AVX512DQ-BW-NEXT:    vpor %ymm7, %ymm9, %ymm7
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[2,3,2,3],zmm7[2,3,2,3]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm18[18],zero,zmm18[18,19,20,21],zero,zmm18[19],zero,zmm18[25,26,27,22],zero,zmm18[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm18[56,57],zero,zmm18[55],zero,zmm18[53,54,55,58],zero,zmm18[56],zero,zmm18[60,61,58,59]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm9 = zmm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm19[18],zero,zero,zero,zero,zmm19[21],zero,zmm19[19],zero,zero,zero,zero,zmm19[22],zero,zmm19[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm19[57],zero,zmm19[55],zero,zero,zero,zero,zmm19[58],zero,zmm19[56],zero,zero,zero,zero
-; AVX512DQ-BW-NEXT:    vporq %zmm7, %zmm9, %zmm7
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm7 {%k2}
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm21, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[20],zero,zmm0[18],zero,zmm0[20,21,20,21],zero,zmm0[19],zero,zmm0[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[56,57,56,57],zero,zmm0[55],zero,zmm0[55,56,57,58],zero,zmm0[56],zero,zmm0[62,63]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm20, %zmm1, %zmm1
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm1[20],zero,zmm1[18],zero,zero,zero,zero,zmm1[21],zero,zmm1[19],zero,zero,zero,zero,zmm1[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm1[57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero
-; AVX512DQ-BW-NEXT:    vporq %zmm0, %zmm1, %zmm0
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512DQ-BW-NEXT:    vpermw %zmm3, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm5, %zmm16 {%k3}
+; AVX512DQ-BW-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm13[1,2,2,3,4,5,6,7,9,10,10,11,12,13,14,15]
+; AVX512DQ-BW-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[0,0,1,1,4,4,5,5]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} ymm5 {%k1} = ymm15[u,u,u,u,5,u,3,u,u,u,u,6,u,4,u,u,u,u,u,u,21,u,19,u,u,u,u,22,u,20,u,u]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm3, %ymm15, %ymm3
+; AVX512DQ-BW-NEXT:    vpshufb %ymm24, %ymm13, %ymm7
+; AVX512DQ-BW-NEXT:    vpor %ymm3, %ymm7, %ymm3
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm5[2,3,2,3],zmm3[2,3,2,3]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm17[18],zero,zmm17[18,19,20,21],zero,zmm17[19],zero,zmm17[25,26,27,22],zero,zmm17[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm17[56,57],zero,zmm17[55],zero,zmm17[53,54,55,58],zero,zmm17[56],zero,zmm17[60,61,58,59]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm7 = zmm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm19[18],zero,zero,zero,zero,zmm19[21],zero,zmm19[19],zero,zero,zero,zero,zmm19[22],zero,zmm19[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm19[57],zero,zmm19[55],zero,zero,zero,zero,zmm19[58],zero,zmm19[56],zero,zero,zero,zero
+; AVX512DQ-BW-NEXT:    vporq %zmm5, %zmm7, %zmm5
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k2}
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm22[20],zero,zmm22[18],zero,zmm22[20,21,20,21],zero,zmm22[19],zero,zmm22[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm22[56,57,56,57],zero,zmm22[55],zero,zmm22[55,56,57,58],zero,zmm22[56],zero,zmm22[62,63]
+; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} zmm7 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm20[20],zero,zmm20[18],zero,zero,zero,zero,zmm20[21],zero,zmm20[19],zero,zero,zero,zero,zmm20[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm20[57],zero,zmm20[55],zero,zero,zero,zero,zmm20[58],zero,zmm20[56],zero,zero
+; AVX512DQ-BW-NEXT:    vporq %zmm3, %zmm7, %zmm3
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm7, %zmm7
 ; AVX512DQ-BW-NEXT:    movabsq $1161999626690365456, %rax # imm = 0x1020408102040810
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm7, %zmm3 {%k1}
 ; AVX512DQ-BW-NEXT:    movabsq $2033499346708139548, %rax # imm = 0x1C3870E1C3870E1C
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm7 {%k1}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm25, %xmm29, %xmm0
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm31, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k1}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm0
+; AVX512DQ-BW-NEXT:    vpshufb %xmm26, %xmm29, %xmm3
+; AVX512DQ-BW-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm27[8],xmm29[8],xmm27[9],xmm29[9],xmm27[10],xmm29[10],xmm27[11],xmm29[11],xmm27[12],xmm29[12],xmm27[13],xmm29[13],xmm27[14],xmm29[14],xmm27[15],xmm29[15]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm3, %xmm3
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm2
+; AVX512DQ-BW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm10[8],xmm8[8],xmm10[9],xmm8[9],xmm10[10],xmm8[10],xmm10[11],xmm8[11],xmm10[12],xmm8[12],xmm10[13],xmm8[13],xmm10[14],xmm8[14],xmm10[15],xmm8[15]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
+; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm3[0,1,0,1]
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm2 {%k2}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm28, %xmm0
+; AVX512DQ-BW-NEXT:    vpshufb %xmm25, %xmm30, %xmm1
 ; AVX512DQ-BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm29[8],xmm31[8],xmm29[9],xmm31[9],xmm29[10],xmm31[10],xmm29[11],xmm31[11],xmm29[12],xmm31[12],xmm29[13],xmm31[13],xmm29[14],xmm31[14],xmm29[15],xmm31[15]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
 ; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm1[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm10, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm8, %xmm2
-; AVX512DQ-BW-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm10[8],xmm8[8],xmm10[9],xmm8[9],xmm10[10],xmm8[10],xmm10[11],xmm8[11],xmm10[12],xmm8[12],xmm10[13],xmm8[13],xmm10[14],xmm8[14],xmm10[15],xmm8[15]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,0,1],zmm2[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k2}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm11, %xmm27, %xmm0
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm28, %xmm2
-; AVX512DQ-BW-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm28[8],xmm27[8],xmm28[9],xmm27[9],xmm28[10],xmm27[10],xmm28[11],xmm27[11],xmm28[12],xmm27[12],xmm28[13],xmm27[13],xmm28[14],xmm27[14],xmm28[15],xmm27[15]
-; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm2[0,1,0,1]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
-; AVX512DQ-BW-NEXT:    vpermw %zmm3, %zmm2, %zmm2
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
+; AVX512DQ-BW-NEXT:    vpermw %zmm14, %zmm1, %zmm1
 ; AVX512DQ-BW-NEXT:    movabsq $290499906672591364, %rax # imm = 0x408102040810204
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512DQ-BW-NEXT:    movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387
 ; AVX512DQ-BW-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqu8 %zmm0, %zmm2 {%k1}
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, 128(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 256(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, 128(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm6, 320(%rax)
 ; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, (%rax)
 ; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm23, 384(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm22, 192(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm21, 192(%rax)
 ; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, 64(%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
 ; AVX512DQ-BW-FCP-LABEL: store_i8_stride7_vf64:
 ; AVX512DQ-BW-FCP:       # %bb.0:
-; AVX512DQ-BW-FCP-NEXT:    subq $200, %rsp
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm8
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm7
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm17
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm3, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm6, %ymm11, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm10, %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm10, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm6, %zmm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm12
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm10, %ymm14, %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm6, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm12, %ymm12
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm6, %ymm12, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,3,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm10
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm16
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm5, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm6, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm5, %ymm5
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm5
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %r10, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm11, %zmm10 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm19 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm19, %ymm14, %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm12
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm20 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm20, %ymm12, %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm11, %ymm15, %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm6, %ymm12, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm6, %ymm14, %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vporq %ymm16, %ymm14, %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,3,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm15, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rax), %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm15, %ymm16, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm15, %zmm15
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm5 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm7
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm8
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm6, %ymm8, %ymm8
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm7, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm10, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rax), %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm10, %ymm4, %ymm11
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $145249953336295682, %rax # imm = 0x204081020408102
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm15, %zmm14 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm3 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-4357498600088870461, %rax # imm = 0xC3870E1C3870E1C3
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm10 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm14, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %xmm21
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %xmm22
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm22[8],xmm21[8],xmm22[9],xmm21[9],xmm22[10],xmm21[10],xmm22[11],xmm21[11],xmm22[12],xmm21[12],xmm22[13],xmm21[13],xmm22[14],xmm21[14],xmm22[15],xmm21[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm16, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm11, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm11, %ymm3
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm23
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm24
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm24[8],xmm23[8],xmm24[9],xmm23[9],xmm24[10],xmm23[10],xmm24[11],xmm23[11],xmm24[12],xmm23[12],xmm24[13],xmm23[13],xmm24[14],xmm23[14],xmm24[15],xmm23[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm13
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm18[8],xmm3[8],xmm18[9],xmm3[9],xmm18[10],xmm3[10],xmm18[11],xmm3[11],xmm18[12],xmm3[12],xmm18[13],xmm3[13],xmm18[14],xmm3[14],xmm18[15],xmm3[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm9, %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm9
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm14, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm19 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm14, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm14, %zmm2
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $435749860008887046, %rax # imm = 0x60C183060C18306
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm13 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %ymm18
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm19, %ymm18, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %ymm19
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm20, %ymm19, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %xmm25
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %xmm27
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm27[8],xmm25[8],xmm27[9],xmm25[9],xmm27[10],xmm25[10],xmm27[11],xmm25[11],xmm27[12],xmm25[12],xmm27[13],xmm25[13],xmm27[14],xmm25[14],xmm27[15],xmm25[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm17, %zmm2, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm9 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm20 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm1, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $2323999253380730912, %rax # imm = 0x2040810204081020
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $4066998693416279096, %rax # imm = 0x3870E1C3870E1C38
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm13 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm5[4,5,6,7],zmm4[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm1[23],zero,zmm1[21,22,23,26],zero,zmm1[24],zero,zmm1[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero,zmm1[61],zero
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm4[4,5,6,7],zmm5[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm2[25],zero,zmm2[23],zero,zero,zero,zero,zmm2[26],zero,zmm2[24],zero,zero,zero,zero,zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[62],zero,zmm2[60],zero,zero,zero,zero,zmm2[63],zero,zmm2[61],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm7[4,5,6,7],zmm8[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm2[23],zero,zero,zero,zero,zmm2[26],zero,zmm2[24],zero,zero,zero,zero,zmm2[27],zero,zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm2[60],zero,zmm2[62,63,62,63],zero,zmm2[61],zero,zmm2[63,60,61]
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm8[4,5,6,7],zmm7[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[23],zero,zero,zero,zero,zmm3[26],zero,zmm3[24],zero,zero,zero,zero,zmm3[27],zero,zmm3[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm3[62],zero,zmm3[60],zero,zero,zero,zero,zmm3[63],zero,zmm3[61],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm2, %zmm3, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm26
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm2[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-FCP-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm9 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm21 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm18, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm22 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm22, %xmm3, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm18[0],xmm3[1],xmm18[1],xmm3[2],xmm18[2],xmm3[3],xmm18[3],xmm3[4],xmm18[4],xmm3[5],xmm18[5],xmm3[6],xmm18[6],xmm3[7],xmm18[7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm26 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm26, %xmm13, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm27 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm27, %xmm12, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3],xmm12[4],xmm13[4],xmm12[5],xmm13[5],xmm12[6],xmm13[6],xmm12[7],xmm13[7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm23 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    movabsq $871499720017774092, %rax # imm = 0xC183060C183060C
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm20 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7],zmm26[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm1[23],zero,zmm1[23,24,25,26],zero,zmm1[24],zero,zmm1[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm1[59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm26[4,5,6,7],zmm0[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[25],zero,zmm2[23],zero,zero,zero,zero,zmm2[26],zero,zmm2[24],zero,zero,zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm2[59],zero,zero,zero,zero,zmm2[62],zero,zmm2[60],zero,zero,zero,zero,zmm2[63],zero,zmm2[61]
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-FCP-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm24 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm24, %xmm14, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm25, %xmm15, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm29 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm29, %xmm1, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm20 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm17, %zmm1, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm20 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm16[28,29,30],zero,ymm16[28],zero,ymm16[30,31,30,31],zero,ymm16[29],zero,ymm16[31,28,29]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm28
+; AVX512DQ-BW-FCP-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm8[0],xmm28[0],xmm8[1],xmm28[1],xmm8[2],xmm28[2],xmm8[3],xmm28[3],xmm8[4],xmm28[4],xmm8[5],xmm28[5],xmm8[6],xmm28[6],xmm8[7],xmm28[7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[28,29,30],zero,ymm10[28],zero,ymm10[30,31,30,31],zero,ymm10[29],zero,ymm10[31,28,29]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm11[30],zero,ymm11[28],zero,zero,zero,zero,ymm11[31],zero,ymm11[29],zero,zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %xmm6
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm31 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm31, %xmm3, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm1[2,3,2,3],zmm3[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero,ymm15[29],zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero,ymm14[29],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %ymm1, %ymm29, %ymm29
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm1
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm29[2,3,2,3],zmm0[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm29 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm18[27],zero,zero,zero,zero,ymm18[30],zero,ymm18[28],zero,zero,zero,zero,ymm18[31],zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm19[27],zero,zero,zero,zero,ymm19[30],zero,ymm19[28],zero,zero,zero,zero,ymm19[31],zero,ymm19[29]
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %xmm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r8), %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm30 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm30, %xmm0, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm3[2,3,2,3],zmm0[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm17, %zmm3, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm23 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm13[30],zero,ymm13[28],zero,zero,zero,zero,ymm13[31],zero,ymm13[29],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm30 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %ymm23, %ymm30, %ymm23
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm23[2,3,2,3],zmm1[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm30
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm31
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm30[0],xmm31[1],xmm30[1],xmm31[2],xmm30[2],xmm31[3],xmm30[3],xmm31[4],xmm30[4],xmm31[5],xmm30[5],xmm31[6],xmm30[6],xmm31[7],xmm30[7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm29, %xmm0, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm14[27],zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero,ymm15[29]
+; AVX512DQ-BW-FCP-NEXT:    vporq %ymm1, %ymm29, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $580999813345182728, %rax # imm = 0x810204081020408
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $1016749673354069774, %rax # imm = 0xE1C3870E1C3870E
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm0[4,5,6,7],zmm1[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7],zmm0[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm29[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm29[23],zero,zmm29[21,22,23,26],zero,zmm29[24],zero,zmm29[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm29[62],zero,zmm29[60],zero,zero,zero,zero,zmm29[63],zero,zmm29[61],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm0[25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zero,zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero,zmm0[61],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm0, %zmm29
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm2[4,5,6,7],zmm1[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zero,zero,zmm6[27],zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm6[60],zero,zmm6[62,63,62,63],zero,zmm6[61],zero,zmm6[63,60,61]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm1[4,5,6,7],zmm2[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm7 = zmm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm7[23],zero,zero,zero,zero,zmm7[26],zero,zmm7[24],zero,zero,zero,zero,zmm7[27],zero,zmm7[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm7[62],zero,zmm7[60],zero,zero,zero,zero,zmm7[63],zero,zmm7[61],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm6, %zmm7, %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm29[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm29 = zmm6[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-FCP-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm7, %zmm29 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm7
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm6[4,5,6,7],zmm7[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm7[4,5,6,7],zmm6[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm0[23],zero,zmm0[23,24,25,26],zero,zmm0[24],zero,zmm0[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm6[25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm6[59],zero,zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero,zmm6[61]
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm0, %zmm6, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-FCP-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm2, %xmm28
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm3, %xmm28, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,0,1],zmm1[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm0, %xmm7, %xmm28
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm3, %xmm28, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm28 = zmm3[0,1,0,1],zmm6[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm28 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm7, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm0, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm26, %xmm4, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm27, %xmm3, %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm6, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm3, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm3[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm28, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm22, %xmm8, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm28[8],xmm8[8],xmm28[9],xmm8[9],xmm28[10],xmm8[10],xmm28[11],xmm8[11],xmm28[12],xmm8[12],xmm28[13],xmm8[13],xmm28[14],xmm8[14],xmm28[15],xmm8[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm4, %xmm4
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,0,1],zmm4[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm24, %xmm30, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm25, %xmm31, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm31[8],xmm30[8],xmm31[9],xmm30[9],xmm31[10],xmm30[10],xmm31[11],xmm30[11],xmm31[12],xmm30[12],xmm31[13],xmm30[13],xmm31[14],xmm30[14],xmm31[15],xmm30[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm4, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm4[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm17, %zmm4, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm4, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $290499906672591364, %rax # imm = 0x408102040810204
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm28 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm9, %xmm22, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm21, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm21[0],xmm22[0],xmm21[1],xmm22[1],xmm21[2],xmm22[2],xmm21[3],xmm22[3],xmm21[4],xmm22[4],xmm21[5],xmm22[5],xmm21[6],xmm22[6],xmm21[7],xmm22[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm12, %xmm4, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm4[0,1,0,1],zmm3[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm23[0],xmm24[0],xmm23[1],xmm24[1],xmm23[2],xmm24[2],xmm23[3],xmm24[3],xmm23[4],xmm24[4],xmm23[5],xmm24[5],xmm23[6],xmm24[6],xmm23[7],xmm24[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm31, %xmm2, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    movabsq $871499720017774092, %rax # imm = 0xC183060C183060C
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm1, %xmm25, %xmm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm27, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm27[0],xmm25[0],xmm27[1],xmm25[1],xmm27[2],xmm25[2],xmm27[3],xmm25[3],xmm27[4],xmm25[4],xmm27[5],xmm25[5],xmm27[6],xmm25[6],xmm27[7],xmm25[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm30, %xmm2, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,0,1],zmm1[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm17, %zmm2, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
-; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[18],zero,zmm12[18,19,20,21],zero,zmm12[19],zero,zmm12[25,26,27,22],zero,zmm12[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[56,57],zero,zmm12[55],zero,zmm12[53,54,55,58],zero,zmm12[56],zero,zmm12[60,61,58,59]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[18],zero,zero,zero,zero,zmm13[21],zero,zmm13[19],zero,zero,zero,zero,zmm13[22],zero,zmm13[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm13[57],zero,zmm13[55],zero,zero,zero,zero,zmm13[58],zero,zmm13[56],zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm0, %zmm4, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[18,19,20,21],zero,zmm1[19],zero,zmm1[21,20,21,22],zero,zmm1[20],zero,zmm1[22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero,zero,zero,zmm1[59],zero
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm2, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[21],zero,zmm2[19],zero,zero,zero,zero,zmm2[22],zero,zmm2[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[55],zero,zero,zero,zero,zmm2[58],zero,zmm2[56],zero,zero,zero,zero,zmm2[59],zero,zmm2[57]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm2, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[18],zero,zmm2[18,19,20,21],zero,zmm2[19],zero,zmm2[25,26,27,22],zero,zmm2[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[56,57],zero,zmm2[55],zero,zmm2[53,54,55,58],zero,zmm2[56],zero,zmm2[60,61,58,59]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm3, %zmm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[18],zero,zero,zero,zero,zmm3[21],zero,zmm3[19],zero,zero,zero,zero,zmm3[22],zero,zmm3[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm3[57],zero,zmm3[55],zero,zero,zero,zero,zmm3[58],zero,zmm3[56],zero,zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm2, %zmm3, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm2 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm1, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[20],zero,zmm1[18],zero,zmm1[20,21,20,21],zero,zmm1[19],zero,zmm1[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[56,57,56,57],zero,zmm1[55],zero,zmm1[55,56,57,58],zero,zmm1[56],zero,zmm1[62,63]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm26, %zmm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[20],zero,zmm3[18],zero,zero,zero,zero,zmm3[21],zero,zmm3[19],zero,zero,zero,zero,zmm3[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm3[57],zero,zmm3[55],zero,zero,zero,zero,zmm3[58],zero,zmm3[56],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm3, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm17, %zmm3, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[20],zero,zmm15[18],zero,zmm15[20,21,20,21],zero,zmm15[19],zero,zmm15[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[56,57,56,57],zero,zmm15[55],zero,zmm15[55,56,57,58],zero,zmm15[56],zero,zmm15[62,63]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm14[20],zero,zmm14[18],zero,zero,zero,zero,zmm14[21],zero,zmm14[19],zero,zero,zero,zero,zmm14[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm14[57],zero,zmm14[55],zero,zero,zero,zero,zmm14[58],zero,zmm14[56],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm2, %zmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $1161999626690365456, %rax # imm = 0x1020408102040810
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm1 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $2033499346708139548, %rax # imm = 0x1C3870E1C3870E1C
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm2 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm28, 256(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm29, 192(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm20, 384(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512DQ-BW-FCP-NEXT:    addq $200, %rsp
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm29, 384(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm23, 192(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm18, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <64 x i8>, ptr %in.vecptr0, align 64
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
index a9da7abaa945..3acc94d6e1fc 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
@@ -2071,9 +2071,7 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm9 = zmm3[0,2,0,2,4,6,4,6]
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [128,128,128,128,128,128,0,8,128,128,128,128,128,128,1,9,128,128,128,128,128,128,18,26,128,128,128,128,128,128,19,27,128,128,128,128,128,128,36,44,128,128,128,128,128,128,37,45,128,128,128,128,128,128,54,62,128,128,128,128,128,128,55,63]
 ; AVX512BW-NEXT:    vpshufb %zmm10, %zmm9, %zmm9
-; AVX512BW-NEXT:    movw $-21846, %cx # imm = 0xAAAA
-; AVX512BW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-NEXT:    vpord %zmm6, %zmm9, %zmm4 {%k1}
+; AVX512BW-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 | zmm4 | zmm6
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,1,3,5,7,5,7]
 ; AVX512BW-NEXT:    vpshufb %zmm5, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[1,3,1,3,5,7,5,7]
@@ -2083,9 +2081,9 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vpshufb %zmm8, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm3[1,3,1,3,5,7,5,7]
 ; AVX512BW-NEXT:    vpshufb %zmm10, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpord %zmm1, %zmm2, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512BW-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 | zmm0 | zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -2117,23 +2115,21 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vpermq %zmm3, %zmm4, %zmm4
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [128,128,128,128,128,128,0,8,128,128,128,128,128,128,1,9,128,128,128,128,128,128,18,26,128,128,128,128,128,128,19,27,128,128,128,128,128,128,36,44,128,128,128,128,128,128,37,45,128,128,128,128,128,128,54,62,128,128,128,128,128,128,55,63]
 ; AVX512BW-FCP-NEXT:    vpshufb %zmm10, %zmm4, %zmm4
-; AVX512BW-FCP-NEXT:    movw $-21846, %cx # imm = 0xAAAA
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vpord %zmm7, %zmm4, %zmm5 {%k1}
-; AVX512BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [1,3,1,3,1,3,1,3]
-; AVX512BW-FCP-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-FCP-NEXT:    vpermq %zmm0, %zmm4, %zmm0
+; AVX512BW-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | zmm5 | zmm7
+; AVX512BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [1,3,1,3,1,3,1,3]
+; AVX512BW-FCP-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpermq %zmm0, %zmm5, %zmm0
 ; AVX512BW-FCP-NEXT:    vpshufb %zmm6, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vpermq %zmm1, %zmm4, %zmm1
+; AVX512BW-FCP-NEXT:    vpermq %zmm1, %zmm5, %zmm1
 ; AVX512BW-FCP-NEXT:    vpshufb %zmm8, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    vporq %zmm0, %zmm1, %zmm0
-; AVX512BW-FCP-NEXT:    vpermq %zmm2, %zmm4, %zmm1
+; AVX512BW-FCP-NEXT:    vpermq %zmm2, %zmm5, %zmm1
 ; AVX512BW-FCP-NEXT:    vpshufb %zmm9, %zmm1, %zmm1
-; AVX512BW-FCP-NEXT:    vpermq %zmm3, %zmm4, %zmm2
+; AVX512BW-FCP-NEXT:    vpermq %zmm3, %zmm5, %zmm2
 ; AVX512BW-FCP-NEXT:    vpshufb %zmm10, %zmm2, %zmm2
-; AVX512BW-FCP-NEXT:    vpord %zmm1, %zmm2, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512BW-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 | zmm0 | zmm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -2167,9 +2163,7 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm9 = zmm3[0,2,0,2,4,6,4,6]
 ; AVX512DQ-BW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [128,128,128,128,128,128,0,8,128,128,128,128,128,128,1,9,128,128,128,128,128,128,18,26,128,128,128,128,128,128,19,27,128,128,128,128,128,128,36,44,128,128,128,128,128,128,37,45,128,128,128,128,128,128,54,62,128,128,128,128,128,128,55,63]
 ; AVX512DQ-BW-NEXT:    vpshufb %zmm10, %zmm9, %zmm9
-; AVX512DQ-BW-NEXT:    movw $-21846, %cx # imm = 0xAAAA
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-NEXT:    vpord %zmm6, %zmm9, %zmm4 {%k1}
+; AVX512DQ-BW-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 | zmm4 | zmm6
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[1,3,1,3,5,7,5,7]
 ; AVX512DQ-BW-NEXT:    vpshufb %zmm5, %zmm0, %zmm0
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[1,3,1,3,5,7,5,7]
@@ -2179,9 +2173,9 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-NEXT:    vpshufb %zmm8, %zmm1, %zmm1
 ; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm2 = zmm3[1,3,1,3,5,7,5,7]
 ; AVX512DQ-BW-NEXT:    vpshufb %zmm10, %zmm2, %zmm2
-; AVX512DQ-BW-NEXT:    vpord %zmm1, %zmm2, %zmm0 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512DQ-BW-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 | zmm0 | zmm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm9, (%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
@@ -2213,23 +2207,21 @@ define void @store_i8_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm3, %zmm4, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [128,128,128,128,128,128,0,8,128,128,128,128,128,128,1,9,128,128,128,128,128,128,18,26,128,128,128,128,128,128,19,27,128,128,128,128,128,128,36,44,128,128,128,128,128,128,37,45,128,128,128,128,128,128,54,62,128,128,128,128,128,128,55,63]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm10, %zmm4, %zmm4
-; AVX512DQ-BW-FCP-NEXT:    movw $-21846, %cx # imm = 0xAAAA
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpord %zmm7, %zmm4, %zmm5 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [1,3,1,3,1,3,1,3]
-; AVX512DQ-BW-FCP-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm0, %zmm4, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | zmm5 | zmm7
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [1,3,1,3,1,3,1,3]
+; AVX512DQ-BW-FCP-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm0, %zmm5, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm6, %zmm0, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm1, %zmm4, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm1, %zmm5, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm8, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vporq %zmm0, %zmm1, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm2, %zmm4, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm2, %zmm5, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm9, %zmm1, %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm3, %zmm4, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpermq %zmm3, %zmm5, %zmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %zmm10, %zmm2, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vpord %zmm1, %zmm2, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 | zmm0 | zmm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <16 x i8>, ptr %in.vecptr0, align 64
@@ -8050,128 +8042,107 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm1
-; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %xmm16
-; AVX512BW-NEXT:    vmovdqa 48(%rcx), %xmm14
-; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm2
-; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %xmm18
-; AVX512BW-NEXT:    vmovdqa64 48(%rdx), %xmm17
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512BW-NEXT:    vmovdqa64 32(%rsi), %xmm20
-; AVX512BW-NEXT:    vmovdqa64 48(%rsi), %xmm19
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512BW-NEXT:    vmovdqa64 32(%rdi), %xmm21
-; AVX512BW-NEXT:    vmovdqa64 48(%rdi), %xmm22
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7,4,4,2,3,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512BW-NEXT:    movl $572662306, %r11d # imm = 0x22222222
-; AVX512BW-NEXT:    kmovd %r11d, %k1
-; AVX512BW-NEXT:    vpermw %zmm4, %zmm6, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%r10), %xmm4
-; AVX512BW-NEXT:    vmovdqa64 48(%r10), %xmm23
-; AVX512BW-NEXT:    vmovdqa (%rax), %xmm7
-; AVX512BW-NEXT:    vmovdqa64 48(%rax), %xmm24
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15]
-; AVX512BW-NEXT:    vmovdqa (%r9), %xmm8
-; AVX512BW-NEXT:    vmovdqa64 48(%r9), %xmm25
-; AVX512BW-NEXT:    vmovdqa (%r8), %xmm10
-; AVX512BW-NEXT:    vmovdqa64 48(%r8), %xmm26
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm10[8],xmm8[8],xmm10[9],xmm8[9],xmm10[10],xmm8[10],xmm10[11],xmm8[11],xmm10[12],xmm8[12],xmm10[13],xmm8[13],xmm10[14],xmm8[14],xmm10[15],xmm8[15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3,0,1,4,5,4,5,5,7,0,1,6,5,6,5,7,7]
-; AVX512BW-NEXT:    vpermw %zmm11, %zmm12, %zmm11
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3,0,1,4,4,4,4,6,5,0,1,4,6,4,6,6,7]
-; AVX512BW-NEXT:    movl $-2004318072, %r11d # imm = 0x88888888
-; AVX512BW-NEXT:    kmovd %r11d, %k2
-; AVX512BW-NEXT:    vpermw %zmm9, %zmm13, %zmm11 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3],xmm17[4],xmm14[4],xmm17[5],xmm14[5],xmm17[6],xmm14[6],xmm17[7],xmm14[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm22[0],xmm19[0],xmm22[1],xmm19[1],xmm22[2],xmm19[2],xmm22[3],xmm19[3],xmm22[4],xmm19[4],xmm22[5],xmm19[5],xmm22[6],xmm19[6],xmm22[7],xmm19[7]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero
-; AVX512BW-NEXT:    vpermw %zmm15, %zmm6, %zmm9 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm24[0],xmm23[0],xmm24[1],xmm23[1],xmm24[2],xmm23[2],xmm24[3],xmm23[3],xmm24[4],xmm23[4],xmm24[5],xmm23[5],xmm24[6],xmm23[6],xmm24[7],xmm23[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512BW-NEXT:    vpermw %zmm15, %zmm12, %zmm15
-; AVX512BW-NEXT:    vpermw %zmm27, %zmm13, %zmm15 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 32(%r10), %xmm27
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm17[8],xmm14[8],xmm17[9],xmm14[9],xmm17[10],xmm14[10],xmm17[11],xmm14[11],xmm17[12],xmm14[12],xmm17[13],xmm14[13],xmm17[14],xmm14[14],xmm17[15],xmm14[15]
-; AVX512BW-NEXT:    vmovdqa64 32(%rax), %xmm28
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm22[8],xmm19[8],xmm22[9],xmm19[9],xmm22[10],xmm19[10],xmm22[11],xmm19[11],xmm22[12],xmm19[12],xmm22[13],xmm19[13],xmm22[14],xmm19[14],xmm22[15],xmm19[15]
-; AVX512BW-NEXT:    vmovdqa64 32(%r9), %xmm29
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm14 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero,xmm14[2],zero,zero,zero,xmm14[3],zero,zero,zero,xmm14[4],zero,zero,zero,xmm14[5],zero,zero,zero,xmm14[6],zero,zero,zero,xmm14[7],zero,zero,zero
-; AVX512BW-NEXT:    vpermw %zmm17, %zmm6, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 32(%r8), %xmm30
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm24[8],xmm23[8],xmm24[9],xmm23[9],xmm24[10],xmm23[10],xmm24[11],xmm23[11],xmm24[12],xmm23[12],xmm24[13],xmm23[13],xmm24[14],xmm23[14],xmm24[15],xmm23[15]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512BW-NEXT:    vpermw %zmm19, %zmm12, %zmm19
-; AVX512BW-NEXT:    vpermw %zmm17, %zmm13, %zmm19 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm18[0],xmm16[0],xmm18[1],xmm16[1],xmm18[2],xmm16[2],xmm18[3],xmm16[3],xmm18[4],xmm16[4],xmm18[5],xmm16[5],xmm18[6],xmm16[6],xmm18[7],xmm16[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm17 = xmm21[0],xmm20[0],xmm21[1],xmm20[1],xmm21[2],xmm20[2],xmm21[3],xmm20[3],xmm21[4],xmm20[4],xmm21[5],xmm20[5],xmm21[6],xmm20[6],xmm21[7],xmm20[7]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm17 = xmm17[0],zero,zero,zero,xmm17[1],zero,zero,zero,xmm17[2],zero,zero,zero,xmm17[3],zero,zero,zero,xmm17[4],zero,zero,zero,xmm17[5],zero,zero,zero,xmm17[6],zero,zero,zero,xmm17[7],zero,zero,zero
-; AVX512BW-NEXT:    vpermw %zmm22, %zmm6, %zmm17 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3],xmm28[4],xmm27[4],xmm28[5],xmm27[5],xmm28[6],xmm27[6],xmm28[7],xmm27[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm30[0],xmm29[0],xmm30[1],xmm29[1],xmm30[2],xmm29[2],xmm30[3],xmm29[3],xmm30[4],xmm29[4],xmm30[5],xmm29[5],xmm30[6],xmm29[6],xmm30[7],xmm29[7]
-; AVX512BW-NEXT:    vpermw %zmm22, %zmm12, %zmm22
-; AVX512BW-NEXT:    vpermw %zmm23, %zmm13, %zmm22 {%k2}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm18[8],xmm16[8],xmm18[9],xmm16[9],xmm18[10],xmm16[10],xmm18[11],xmm16[11],xmm18[12],xmm16[12],xmm18[13],xmm16[13],xmm18[14],xmm16[14],xmm18[15],xmm16[15]
-; AVX512BW-NEXT:    vmovdqa64 16(%rcx), %xmm23
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm21[8],xmm20[8],xmm21[9],xmm20[9],xmm21[10],xmm20[10],xmm21[11],xmm20[11],xmm21[12],xmm20[12],xmm21[13],xmm20[13],xmm21[14],xmm20[14],xmm21[15],xmm20[15]
-; AVX512BW-NEXT:    vmovdqa64 16(%rdx), %xmm21
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm16 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero,xmm16[2],zero,zero,zero,xmm16[3],zero,zero,zero,xmm16[4],zero,zero,zero,xmm16[5],zero,zero,zero,xmm16[6],zero,zero,zero,xmm16[7],zero,zero,zero
-; AVX512BW-NEXT:    vpermw %zmm18, %zmm6, %zmm16 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 16(%rsi), %xmm24
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm28[8],xmm27[8],xmm28[9],xmm27[9],xmm28[10],xmm27[10],xmm28[11],xmm27[11],xmm28[12],xmm27[12],xmm28[13],xmm27[13],xmm28[14],xmm27[14],xmm28[15],xmm27[15]
-; AVX512BW-NEXT:    vmovdqa64 16(%rdi), %xmm25
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm30[8],xmm29[8],xmm30[9],xmm29[9],xmm30[10],xmm29[10],xmm30[11],xmm29[11],xmm30[12],xmm29[12],xmm30[13],xmm29[13],xmm30[14],xmm29[14],xmm30[15],xmm29[15]
-; AVX512BW-NEXT:    vpermw %zmm18, %zmm12, %zmm18
-; AVX512BW-NEXT:    vpermw %zmm20, %zmm13, %zmm18 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm26 = xmm21[0],xmm23[0],xmm21[1],xmm23[1],xmm21[2],xmm23[2],xmm21[3],xmm23[3],xmm21[4],xmm23[4],xmm21[5],xmm23[5],xmm21[6],xmm23[6],xmm21[7],xmm23[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3],xmm25[4],xmm24[4],xmm25[5],xmm24[5],xmm25[6],xmm24[6],xmm25[7],xmm24[7]
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero,xmm20[2],zero,zero,zero,xmm20[3],zero,zero,zero,xmm20[4],zero,zero,zero,xmm20[5],zero,zero,zero,xmm20[6],zero,zero,zero,xmm20[7],zero,zero,zero
-; AVX512BW-NEXT:    vpermw %zmm26, %zmm6, %zmm20 {%k1}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm21[8],xmm23[8],xmm21[9],xmm23[9],xmm21[10],xmm23[10],xmm21[11],xmm23[11],xmm21[12],xmm23[12],xmm21[13],xmm23[13],xmm21[14],xmm23[14],xmm21[15],xmm23[15]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm25[8],xmm24[8],xmm25[9],xmm24[9],xmm25[10],xmm24[10],xmm25[11],xmm24[11],xmm25[12],xmm24[12],xmm25[13],xmm24[13],xmm25[14],xmm24[14],xmm25[15],xmm24[15]
-; AVX512BW-NEXT:    vmovdqa64 16(%r10), %xmm24
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm23 = xmm23[0],zero,zero,zero,xmm23[1],zero,zero,zero,xmm23[2],zero,zero,zero,xmm23[3],zero,zero,zero,xmm23[4],zero,zero,zero,xmm23[5],zero,zero,zero,xmm23[6],zero,zero,zero,xmm23[7],zero,zero,zero
-; AVX512BW-NEXT:    vpermw %zmm21, %zmm6, %zmm23 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 16(%rax), %xmm21
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512BW-NEXT:    vmovdqa 16(%r9), %xmm2
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512BW-NEXT:    vmovdqa 16(%r8), %xmm5
-; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm6, %zmm3 {%k1}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm21[0],xmm24[0],xmm21[1],xmm24[1],xmm21[2],xmm24[2],xmm21[3],xmm24[3],xmm21[4],xmm24[4],xmm21[5],xmm24[5],xmm21[6],xmm24[6],xmm21[7],xmm24[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; AVX512BW-NEXT:    vpermw %zmm6, %zmm12, %zmm6
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm13, %zmm6 {%k2}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm21[8],xmm24[8],xmm21[9],xmm24[9],xmm21[10],xmm24[10],xmm21[11],xmm24[11],xmm21[12],xmm24[12],xmm21[13],xmm24[13],xmm21[14],xmm24[14],xmm21[15],xmm24[15]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
-; AVX512BW-NEXT:    vpermw %zmm2, %zmm12, %zmm2
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm13, %zmm2 {%k2}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3],xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]
-; AVX512BW-NEXT:    vpermw %zmm4, %zmm12, %zmm4
-; AVX512BW-NEXT:    vpermw %zmm1, %zmm13, %zmm4 {%k2}
+; AVX512BW-NEXT:    vmovdqa (%r10), %xmm1
+; AVX512BW-NEXT:    vmovdqa64 32(%r10), %xmm16
+; AVX512BW-NEXT:    vmovdqa 48(%r10), %xmm14
+; AVX512BW-NEXT:    vmovdqa (%rax), %xmm3
+; AVX512BW-NEXT:    vmovdqa64 32(%rax), %xmm17
+; AVX512BW-NEXT:    vmovdqa 48(%rax), %xmm15
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512BW-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512BW-NEXT:    vmovdqa64 32(%r9), %xmm19
+; AVX512BW-NEXT:    vmovdqa64 48(%r9), %xmm18
+; AVX512BW-NEXT:    vmovdqa (%r8), %xmm6
+; AVX512BW-NEXT:    vmovdqa64 32(%r8), %xmm21
+; AVX512BW-NEXT:    vmovdqa64 48(%r8), %xmm20
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,1,0,32,4,5,1,33,2,1,2,34,4,5,3,35,0,1,4,36,4,5,5,37,0,1,6,38,6,5,7,39]
+; AVX512BW-NEXT:    vpermt2w %zmm2, %zmm7, %zmm0
+; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm5
+; AVX512BW-NEXT:    vmovdqa64 48(%rcx), %xmm22
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm8
+; AVX512BW-NEXT:    vmovdqa64 48(%rdx), %xmm23
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15]
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm9
+; AVX512BW-NEXT:    vmovdqa64 48(%rsi), %xmm24
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm10
+; AVX512BW-NEXT:    vmovdqa64 48(%rdi), %xmm25
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]
+; AVX512BW-NEXT:    vpermt2w %zmm11, %zmm12, %zmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm20[0],xmm18[0],xmm20[1],xmm18[1],xmm20[2],xmm18[2],xmm20[3],xmm18[3],xmm20[4],xmm18[4],xmm20[5],xmm18[5],xmm20[6],xmm18[6],xmm20[7],xmm18[7]
+; AVX512BW-NEXT:    vpermt2w %zmm11, %zmm7, %zmm13
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm26 = xmm23[0],xmm22[0],xmm23[1],xmm22[1],xmm23[2],xmm22[2],xmm23[3],xmm22[3],xmm23[4],xmm22[4],xmm23[5],xmm22[5],xmm23[6],xmm22[6],xmm23[7],xmm22[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3],xmm25[4],xmm24[4],xmm25[5],xmm24[5],xmm25[6],xmm24[6],xmm25[7],xmm24[7]
+; AVX512BW-NEXT:    vpermt2w %zmm26, %zmm12, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 32(%rcx), %xmm26
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15]
+; AVX512BW-NEXT:    vmovdqa64 32(%rdx), %xmm27
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm20[8],xmm18[8],xmm20[9],xmm18[9],xmm20[10],xmm18[10],xmm20[11],xmm18[11],xmm20[12],xmm18[12],xmm20[13],xmm18[13],xmm20[14],xmm18[14],xmm20[15],xmm18[15]
+; AVX512BW-NEXT:    vmovdqa64 32(%rsi), %xmm28
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm7, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 32(%rdi), %xmm29
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm23[8],xmm22[8],xmm23[9],xmm22[9],xmm23[10],xmm22[10],xmm23[11],xmm22[11],xmm23[12],xmm22[12],xmm23[13],xmm22[13],xmm23[14],xmm22[14],xmm23[15],xmm22[15]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm25[8],xmm24[8],xmm25[9],xmm24[9],xmm25[10],xmm24[10],xmm25[11],xmm24[11],xmm25[12],xmm24[12],xmm25[13],xmm24[13],xmm25[14],xmm24[14],xmm25[15],xmm24[15]
+; AVX512BW-NEXT:    vpermt2w %zmm18, %zmm12, %zmm14
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm21[0],xmm19[0],xmm21[1],xmm19[1],xmm21[2],xmm19[2],xmm21[3],xmm19[3],xmm21[4],xmm19[4],xmm21[5],xmm19[5],xmm21[6],xmm19[6],xmm21[7],xmm19[7]
+; AVX512BW-NEXT:    vpermt2w %zmm18, %zmm7, %zmm20
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm27[0],xmm26[0],xmm27[1],xmm26[1],xmm27[2],xmm26[2],xmm27[3],xmm26[3],xmm27[4],xmm26[4],xmm27[5],xmm26[5],xmm27[6],xmm26[6],xmm27[7],xmm26[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm29[0],xmm28[0],xmm29[1],xmm28[1],xmm29[2],xmm28[2],xmm29[3],xmm28[3],xmm29[4],xmm28[4],xmm29[5],xmm28[5],xmm29[6],xmm28[6],xmm29[7],xmm28[7]
+; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm12, %zmm18
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512BW-NEXT:    vmovdqa64 16(%r10), %xmm22
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm21[8],xmm19[8],xmm21[9],xmm19[9],xmm21[10],xmm19[10],xmm21[11],xmm19[11],xmm21[12],xmm19[12],xmm21[13],xmm19[13],xmm21[14],xmm19[14],xmm21[15],xmm19[15]
+; AVX512BW-NEXT:    vmovdqa64 16(%rax), %xmm19
+; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm7, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 16(%r9), %xmm21
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm27[8],xmm26[8],xmm27[9],xmm26[9],xmm27[10],xmm26[10],xmm27[11],xmm26[11],xmm27[12],xmm26[12],xmm27[13],xmm26[13],xmm27[14],xmm26[14],xmm27[15],xmm26[15]
+; AVX512BW-NEXT:    vmovdqa64 16(%r8), %xmm24
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm29[8],xmm28[8],xmm29[9],xmm28[9],xmm29[10],xmm28[10],xmm29[11],xmm28[11],xmm29[12],xmm28[12],xmm29[13],xmm28[13],xmm29[14],xmm28[14],xmm29[15],xmm28[15]
+; AVX512BW-NEXT:    vpermt2w %zmm23, %zmm12, %zmm17
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm19[0],xmm22[0],xmm19[1],xmm22[1],xmm19[2],xmm22[2],xmm19[3],xmm22[3],xmm19[4],xmm22[4],xmm19[5],xmm22[5],xmm19[6],xmm22[6],xmm19[7],xmm22[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm24[0],xmm21[0],xmm24[1],xmm21[1],xmm24[2],xmm21[2],xmm24[3],xmm21[3],xmm24[4],xmm21[4],xmm24[5],xmm21[5],xmm24[6],xmm21[6],xmm24[7],xmm21[7]
+; AVX512BW-NEXT:    vpermt2w %zmm23, %zmm7, %zmm25
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm19[8],xmm22[8],xmm19[9],xmm22[9],xmm19[10],xmm22[10],xmm19[11],xmm22[11],xmm19[12],xmm22[12],xmm19[13],xmm22[13],xmm19[14],xmm22[14],xmm19[15],xmm22[15]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm24[8],xmm21[8],xmm24[9],xmm21[9],xmm24[10],xmm21[10],xmm24[11],xmm21[11],xmm24[12],xmm21[12],xmm24[13],xmm21[13],xmm24[14],xmm21[14],xmm24[15],xmm21[15]
+; AVX512BW-NEXT:    vmovdqa64 16(%rcx), %xmm22
+; AVX512BW-NEXT:    vpermt2w %zmm19, %zmm7, %zmm21
+; AVX512BW-NEXT:    vmovdqa64 16(%rdx), %xmm19
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512BW-NEXT:    vmovdqa 16(%rsi), %xmm3
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm6
+; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm7, %zmm4
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm19[0],xmm22[0],xmm19[1],xmm22[1],xmm19[2],xmm22[2],xmm19[3],xmm22[3],xmm19[4],xmm22[4],xmm19[5],xmm22[5],xmm19[6],xmm22[6],xmm19[7],xmm22[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm12, %zmm7
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm19[8],xmm22[8],xmm19[9],xmm22[9],xmm19[10],xmm22[10],xmm19[11],xmm22[11],xmm19[12],xmm22[12],xmm19[13],xmm22[13],xmm19[14],xmm22[14],xmm19[15],xmm22[15]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm6[8],xmm3[8],xmm6[9],xmm3[9],xmm6[10],xmm3[10],xmm6[11],xmm3[11],xmm6[12],xmm3[12],xmm6[13],xmm3[13],xmm6[14],xmm3[14],xmm6[15],xmm3[15]
+; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm12, %zmm3
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm12, %zmm5
 ; AVX512BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa32 %zmm11, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm15, %zmm9 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm19, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm22, %zmm17 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm18, %zmm16 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm6, %zmm20 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm2, %zmm23 {%k1}
-; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm13, %zmm11 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm15, %zmm14 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm20, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm17 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm25, %zmm7 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm21, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k1}
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 256(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 256(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm14, 448(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, 384(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, 384(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -8179,172 +8150,173 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP:       # %bb.0:
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %xmm0
-; AVX512BW-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm20
-; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rsi), %xmm17
+; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm19
+; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rsi), %xmm16
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm21
-; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rdi), %xmm18
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm20
+; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rdi), %xmm17
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512BW-FCP-NEXT:    vpmovsxwq {{.*#+}} ymm4 = [2312,2826,3340,3854]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512BW-FCP-NEXT:    vpmovsxwq {{.*#+}} xmm5 = [1284,1798]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
+; AVX512BW-FCP-NEXT:    vpmovsxwq {{.*#+}} ymm5 = [2312,2826,3340,3854]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpmovsxwq {{.*#+}} xmm6 = [1284,1798]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm0, %xmm4
 ; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm1
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %xmm3
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm22
-; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rcx), %xmm19
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm21
+; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rcx), %xmm18
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm7
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdx), %xmm23
-; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rdx), %xmm24
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7,4,4,2,3,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512BW-FCP-NEXT:    movl $572662306, %r11d # imm = 0x22222222
-; AVX512BW-FCP-NEXT:    kmovd %r11d, %k1
-; AVX512BW-FCP-NEXT:    vpermw %zmm6, %zmm8, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa (%r10), %xmm6
-; AVX512BW-FCP-NEXT:    vmovdqa64 48(%r10), %xmm25
-; AVX512BW-FCP-NEXT:    vmovdqa (%rax), %xmm9
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rdx), %xmm22
+; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rdx), %xmm23
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm4, %ymm4
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,32,2,3,4,33,6,7,8,42,10,11,12,43,14,15,16,36,18,19,20,37,22,23,24,46,26,27,28,47,30,31]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm4, %zmm9, %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqa (%r10), %xmm4
+; AVX512BW-FCP-NEXT:    vmovdqa64 48(%r10), %xmm24
+; AVX512BW-FCP-NEXT:    vmovdqa (%rax), %xmm8
 ; AVX512BW-FCP-NEXT:    vmovdqa64 48(%rax), %xmm26
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm9[8],xmm6[8],xmm9[9],xmm6[9],xmm9[10],xmm6[10],xmm9[11],xmm6[11],xmm9[12],xmm6[12],xmm9[13],xmm6[13],xmm9[14],xmm6[14],xmm9[15],xmm6[15]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm8[8],xmm4[8],xmm8[9],xmm4[9],xmm8[10],xmm4[10],xmm8[11],xmm4[11],xmm8[12],xmm4[12],xmm8[13],xmm4[13],xmm8[14],xmm4[14],xmm8[15],xmm4[15]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm10, %ymm13
 ; AVX512BW-FCP-NEXT:    vmovdqa (%r9), %xmm10
-; AVX512BW-FCP-NEXT:    vmovdqa64 48(%r9), %xmm27
-; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %xmm11
-; AVX512BW-FCP-NEXT:    vmovdqa64 48(%r8), %xmm28
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3,0,1,4,5,4,5,5,7,0,1,6,5,6,5,7,7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm12, %zmm13, %zmm12
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3,0,1,4,4,4,4,6,5,0,1,4,6,4,6,6,7]
-; AVX512BW-FCP-NEXT:    movl $-2004318072, %r11d # imm = 0x88888888
-; AVX512BW-FCP-NEXT:    kmovd %r11d, %k2
-; AVX512BW-FCP-NEXT:    vpermw %zmm15, %zmm14, %zmm12 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm18[0],xmm17[0],xmm18[1],xmm17[1],xmm18[2],xmm17[2],xmm18[3],xmm17[3],xmm18[4],xmm17[4],xmm18[5],xmm17[5],xmm18[6],xmm17[6],xmm18[7],xmm17[7]
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm15, %ymm15, %ymm16
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm16, %ymm16
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm15, %xmm29
-; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm15 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm29, %ymm15, %ymm15
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm15, %zmm15
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm24[0],xmm19[0],xmm24[1],xmm19[1],xmm24[2],xmm19[2],xmm24[3],xmm19[3],xmm24[4],xmm19[4],xmm24[5],xmm19[5],xmm24[6],xmm19[6],xmm24[7],xmm19[7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm8, %zmm15 {%k1}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm29 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3],xmm28[4],xmm27[4],xmm28[5],xmm27[5],xmm28[6],xmm27[6],xmm28[7],xmm27[7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm13, %zmm16
-; AVX512BW-FCP-NEXT:    vpermw %zmm29, %zmm14, %zmm16 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm17, %xmm18
-; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm29 = xmm17[0],zero,zero,zero,xmm17[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm29, %ymm18
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r10), %xmm29
+; AVX512BW-FCP-NEXT:    vmovdqa64 48(%r9), %xmm28
+; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %xmm12
+; AVX512BW-FCP-NEXT:    vmovdqa64 48(%r8), %xmm30
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm12[8],xmm10[8],xmm12[9],xmm10[9],xmm12[10],xmm10[10],xmm12[11],xmm10[11],xmm12[12],xmm10[12],xmm12[13],xmm10[13],xmm12[14],xmm10[14],xmm12[15],xmm10[15]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,1,0,32,4,5,1,33,2,1,2,42,4,5,3,43,0,1,4,36,4,5,5,37,0,1,6,46,6,5,7,47]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm13, %zmm14, %zmm11
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm13, %ymm15
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm15, %ymm15
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm13, %xmm25
+; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm13 = xmm13[0],zero,zero,zero,xmm13[1],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm13, %ymm13
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm13, %zmm13
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm23[0],xmm18[0],xmm23[1],xmm18[1],xmm23[2],xmm18[2],xmm23[3],xmm18[3],xmm23[4],xmm18[4],xmm23[5],xmm18[5],xmm23[6],xmm18[6],xmm23[7],xmm18[7]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm15, %ymm15
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm15, %zmm9, %zmm13
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm26[0],xmm24[0],xmm26[1],xmm24[1],xmm26[2],xmm24[2],xmm26[3],xmm24[3],xmm26[4],xmm24[4],xmm26[5],xmm24[5],xmm26[6],xmm24[6],xmm26[7],xmm24[7]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm15, %ymm15, %ymm25
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm25, %zmm14, %zmm15
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm16, %xmm17
+; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm25, %ymm17
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r10), %xmm25
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm16, %ymm16
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm16, %ymm16
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm17, %zmm16
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rax), %xmm29
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm23[8],xmm18[8],xmm23[9],xmm18[9],xmm23[10],xmm18[10],xmm23[11],xmm18[11],xmm23[12],xmm18[12],xmm23[13],xmm18[13],xmm23[14],xmm18[14],xmm23[15],xmm18[15]
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm27
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm17, %ymm17
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm17, %ymm17
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm18, %zmm17
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rax), %xmm30
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm24[8],xmm19[8],xmm24[9],xmm19[9],xmm24[10],xmm19[10],xmm24[11],xmm19[11],xmm24[12],xmm19[12],xmm24[13],xmm19[13],xmm24[14],xmm19[14],xmm24[15],xmm19[15]
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm31
-; AVX512BW-FCP-NEXT:    vpermw %zmm18, %zmm8, %zmm17 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%r8), %xmm0
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm28[8],xmm27[8],xmm28[9],xmm27[9],xmm28[10],xmm27[10],xmm28[11],xmm27[11],xmm28[12],xmm27[12],xmm28[13],xmm27[13],xmm28[14],xmm27[14],xmm28[15],xmm27[15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm18, %zmm13, %zmm18
-; AVX512BW-FCP-NEXT:    vpermw %zmm19, %zmm14, %zmm18 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm21[0],xmm20[0],xmm21[1],xmm20[1],xmm21[2],xmm20[2],xmm21[3],xmm20[3],xmm21[4],xmm20[4],xmm21[5],xmm20[5],xmm21[6],xmm20[6],xmm21[7],xmm20[7]
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm19, %ymm19, %ymm24
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm24, %ymm24
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm19, %xmm25
-; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm19 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm19, %ymm19
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm24, %zmm19, %zmm19
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm24 = xmm23[0],xmm22[0],xmm23[1],xmm22[1],xmm23[2],xmm22[2],xmm23[3],xmm22[3],xmm23[4],xmm22[4],xmm23[5],xmm22[5],xmm23[6],xmm22[6],xmm23[7],xmm22[7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm24, %zmm8, %zmm19 {%k1}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm30[0],xmm29[0],xmm30[1],xmm29[1],xmm30[2],xmm29[2],xmm30[3],xmm29[3],xmm30[4],xmm29[4],xmm30[5],xmm29[5],xmm30[6],xmm29[6],xmm30[7],xmm29[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm24 = xmm0[0],xmm31[0],xmm0[1],xmm31[1],xmm0[2],xmm31[2],xmm0[3],xmm31[3],xmm0[4],xmm31[4],xmm0[5],xmm31[5],xmm0[6],xmm31[6],xmm0[7],xmm31[7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm24, %zmm13, %zmm24
-; AVX512BW-FCP-NEXT:    vpermw %zmm25, %zmm14, %zmm24 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm21[8],xmm20[8],xmm21[9],xmm20[9],xmm21[10],xmm20[10],xmm21[11],xmm20[11],xmm21[12],xmm20[12],xmm21[13],xmm20[13],xmm21[14],xmm20[14],xmm21[15],xmm20[15]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm20, %xmm21
-; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm25, %ymm21
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm17, %zmm9, %zmm16
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm31
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm26[8],xmm24[8],xmm26[9],xmm24[9],xmm26[10],xmm24[10],xmm26[11],xmm24[11],xmm26[12],xmm24[12],xmm26[13],xmm24[13],xmm26[14],xmm24[14],xmm26[15],xmm24[15]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm17, %ymm18
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm18, %zmm14, %zmm17
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm20[0],xmm19[0],xmm20[1],xmm19[1],xmm20[2],xmm19[2],xmm20[3],xmm19[3],xmm20[4],xmm19[4],xmm20[5],xmm19[5],xmm20[6],xmm19[6],xmm20[7],xmm19[7]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm18, %ymm23
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm23, %ymm23
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm18, %xmm24
+; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm18 = xmm18[0],zero,zero,zero,xmm18[1],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm24, %ymm18, %ymm18
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm23, %zmm18, %zmm18
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm22[0],xmm21[0],xmm22[1],xmm21[1],xmm22[2],xmm21[2],xmm22[3],xmm21[3],xmm22[4],xmm21[4],xmm22[5],xmm21[5],xmm22[6],xmm21[6],xmm22[7],xmm21[7]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm23, %ymm23, %ymm23
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm23, %zmm9, %zmm18
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm29[0],xmm25[0],xmm29[1],xmm25[1],xmm29[2],xmm25[2],xmm29[3],xmm25[3],xmm29[4],xmm25[4],xmm29[5],xmm25[5],xmm29[6],xmm25[6],xmm29[7],xmm25[7]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm23, %ymm23, %ymm24
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm31[0],xmm27[0],xmm31[1],xmm27[1],xmm31[2],xmm27[2],xmm31[3],xmm27[3],xmm31[4],xmm27[4],xmm31[5],xmm27[5],xmm31[6],xmm27[6],xmm31[7],xmm27[7]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm24, %zmm14, %zmm23
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm20[8],xmm19[8],xmm20[9],xmm19[9],xmm20[10],xmm19[10],xmm20[11],xmm19[11],xmm20[12],xmm19[12],xmm20[13],xmm19[13],xmm20[14],xmm19[14],xmm20[15],xmm19[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm19, %xmm20
+; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm24, %ymm20
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm19, %ymm19, %ymm19
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm19, %ymm19
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm20, %zmm19
+; AVX512BW-FCP-NEXT:    vmovdqa64 16(%rsi), %xmm24
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm22[8],xmm21[8],xmm22[9],xmm21[9],xmm22[10],xmm21[10],xmm22[11],xmm21[11],xmm22[12],xmm21[12],xmm22[13],xmm21[13],xmm22[14],xmm21[14],xmm22[15],xmm21[15]
+; AVX512BW-FCP-NEXT:    vmovdqa64 16(%rdi), %xmm22
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm20, %ymm20
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm20, %ymm20
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm21, %zmm20
-; AVX512BW-FCP-NEXT:    vmovdqa64 16(%rsi), %xmm25
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm23[8],xmm22[8],xmm23[9],xmm22[9],xmm23[10],xmm22[10],xmm23[11],xmm22[11],xmm23[12],xmm22[12],xmm23[13],xmm22[13],xmm23[14],xmm22[14],xmm23[15],xmm22[15]
-; AVX512BW-FCP-NEXT:    vmovdqa64 16(%rdi), %xmm23
-; AVX512BW-FCP-NEXT:    vpermw %zmm21, %zmm8, %zmm20 {%k1}
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm20, %zmm9, %zmm19
 ; AVX512BW-FCP-NEXT:    vmovdqa64 16(%rcx), %xmm26
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm30[8],xmm29[8],xmm30[9],xmm29[9],xmm30[10],xmm29[10],xmm30[11],xmm29[11],xmm30[12],xmm29[12],xmm30[13],xmm29[13],xmm30[14],xmm29[14],xmm30[15],xmm29[15]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm31[8],xmm0[9],xmm31[9],xmm0[10],xmm31[10],xmm0[11],xmm31[11],xmm0[12],xmm31[12],xmm0[13],xmm31[13],xmm0[14],xmm31[14],xmm0[15],xmm31[15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm13, %zmm21
-; AVX512BW-FCP-NEXT:    vpermw %zmm22, %zmm14, %zmm21 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm23[0],xmm25[0],xmm23[1],xmm25[1],xmm23[2],xmm25[2],xmm23[3],xmm25[3],xmm23[4],xmm25[4],xmm23[5],xmm25[5],xmm23[6],xmm25[6],xmm23[7],xmm25[7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm22
-; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm27 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm22, %ymm27, %ymm22
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm29[8],xmm25[8],xmm29[9],xmm25[9],xmm29[10],xmm25[10],xmm29[11],xmm25[11],xmm29[12],xmm25[12],xmm29[13],xmm25[13],xmm29[14],xmm25[14],xmm29[15],xmm25[15]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm20, %ymm21
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm31[8],xmm27[8],xmm31[9],xmm27[9],xmm31[10],xmm27[10],xmm31[11],xmm27[11],xmm31[12],xmm27[12],xmm31[13],xmm27[13],xmm31[14],xmm27[14],xmm31[15],xmm27[15]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm21, %zmm14, %zmm20
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm22[0],xmm24[0],xmm22[1],xmm24[1],xmm22[2],xmm24[2],xmm22[3],xmm24[3],xmm22[4],xmm24[4],xmm22[5],xmm24[5],xmm22[6],xmm24[6],xmm22[7],xmm24[7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm21, %xmm25
+; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm27 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm27, %ymm25
 ; AVX512BW-FCP-NEXT:    vmovdqa64 16(%rdx), %xmm27
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm22, %zmm22
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm27[0],xmm26[0],xmm27[1],xmm26[1],xmm27[2],xmm26[2],xmm27[3],xmm26[3],xmm27[4],xmm26[4],xmm27[5],xmm26[5],xmm27[6],xmm26[6],xmm27[7],xmm26[7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm8, %zmm22 {%k1}
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm23[8],xmm25[8],xmm23[9],xmm25[9],xmm23[10],xmm25[10],xmm23[11],xmm25[11],xmm23[12],xmm25[12],xmm23[13],xmm25[13],xmm23[14],xmm25[14],xmm23[15],xmm25[15]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm23
-; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm23, %ymm25, %ymm23
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm23, %zmm0
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm27[8],xmm26[8],xmm27[9],xmm26[9],xmm27[10],xmm26[10],xmm27[11],xmm26[11],xmm27[12],xmm26[12],xmm27[13],xmm26[13],xmm27[14],xmm26[14],xmm27[15],xmm26[15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm23, %zmm8, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
-; AVX512BW-FCP-NEXT:    # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3],xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm2, %ymm2, %ymm23
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm23, %ymm4
-; AVX512BW-FCP-NEXT:    vmovdqa64 16(%r10), %xmm23
-; AVX512BW-FCP-NEXT:    vpshufb %xmm5, %xmm2, %xmm5
-; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
-; AVX512BW-FCP-NEXT:    vmovdqa 16(%rax), %xmm5
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512BW-FCP-NEXT:    vmovdqa 16(%r9), %xmm4
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm21, %ymm21
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm21, %ymm21
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm25, %zmm21
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm27[0],xmm26[0],xmm27[1],xmm26[1],xmm27[2],xmm26[2],xmm27[3],xmm26[3],xmm27[4],xmm26[4],xmm27[5],xmm26[5],xmm27[6],xmm26[6],xmm27[7],xmm26[7]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm25, %ymm25
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm25, %zmm9, %zmm21
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm22[8],xmm24[8],xmm22[9],xmm24[9],xmm22[10],xmm24[10],xmm22[11],xmm24[11],xmm22[12],xmm24[12],xmm22[13],xmm24[13],xmm22[14],xmm24[14],xmm22[15],xmm24[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm22, %xmm24
+; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm22[0],zero,zero,zero,xmm22[1],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm24, %ymm25, %ymm24
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm22, %ymm22, %ymm22
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm22, %ymm22
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm24, %zmm22
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm24 = xmm27[8],xmm26[8],xmm27[9],xmm26[9],xmm27[10],xmm26[10],xmm27[11],xmm26[11],xmm27[12],xmm26[12],xmm27[13],xmm26[13],xmm27[14],xmm26[14],xmm27[15],xmm26[15]
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm24, %ymm24, %ymm24
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm24, %zmm9, %zmm22
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm1, %ymm2
+; AVX512BW-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm2
+; AVX512BW-FCP-NEXT:    vmovdqa 16(%r10), %xmm5
+; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm6
+; AVX512BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm1, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa 16(%rax), %xmm6
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vmovdqa 16(%r9), %xmm2
 ; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
 ; AVX512BW-FCP-NEXT:    vmovdqa 16(%r8), %xmm7
-; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm8, %zmm2 {%k1}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm5[0],xmm23[0],xmm5[1],xmm23[1],xmm5[2],xmm23[2],xmm5[3],xmm23[3],xmm5[4],xmm23[4],xmm5[5],xmm23[5],xmm5[6],xmm23[6],xmm5[7],xmm23[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm8, %zmm13, %zmm8
-; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm14, %zmm8 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm23[8],xmm5[9],xmm23[9],xmm5[10],xmm23[10],xmm5[11],xmm23[11],xmm5[12],xmm23[12],xmm5[13],xmm23[13],xmm5[14],xmm23[14],xmm5[15],xmm23[15]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm4, %zmm13, %zmm4
-; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm14, %zmm4 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3],xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm5, %zmm13, %zmm5
-; AVX512BW-FCP-NEXT:    vpermw %zmm3, %zmm14, %zmm5 {%k2}
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm9, %zmm1
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm14, %zmm9
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm14, %zmm2
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm14, %zmm4
 ; AVX512BW-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm15 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm18, %zmm17 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm24, %zmm19 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm21, %zmm20 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm8, %zmm22 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm11, %zmm0 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm15, %zmm13 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm17, %zmm16 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm23, %zmm18 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm20, %zmm19 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm21 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm2, %zmm22 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm1 {%k1}
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 192(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm22, 128(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm20, 320(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm19, 256(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm17, 448(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm15, 384(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm22, 192(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm21, 128(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm19, 320(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm18, 256(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, 448(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, 384(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -8352,128 +8324,107 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW:       # %bb.0:
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %xmm16
-; AVX512DQ-BW-NEXT:    vmovdqa 48(%rcx), %xmm14
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm2
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %xmm18
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rdx), %xmm17
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rsi), %xmm20
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rsi), %xmm19
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdi), %xmm21
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rdi), %xmm22
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7,4,4,2,3,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512DQ-BW-NEXT:    movl $572662306, %r11d # imm = 0x22222222
-; AVX512DQ-BW-NEXT:    kmovd %r11d, %k1
-; AVX512DQ-BW-NEXT:    vpermw %zmm4, %zmm6, %zmm0 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa (%r10), %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r10), %xmm23
-; AVX512DQ-BW-NEXT:    vmovdqa (%rax), %xmm7
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rax), %xmm24
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15]
-; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm8
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r9), %xmm25
-; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm10
-; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r8), %xmm26
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm10[8],xmm8[8],xmm10[9],xmm8[9],xmm10[10],xmm8[10],xmm10[11],xmm8[11],xmm10[12],xmm8[12],xmm10[13],xmm8[13],xmm10[14],xmm8[14],xmm10[15],xmm8[15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3,0,1,4,5,4,5,5,7,0,1,6,5,6,5,7,7]
-; AVX512DQ-BW-NEXT:    vpermw %zmm11, %zmm12, %zmm11
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3,0,1,4,4,4,4,6,5,0,1,4,6,4,6,6,7]
-; AVX512DQ-BW-NEXT:    movl $-2004318072, %r11d # imm = 0x88888888
-; AVX512DQ-BW-NEXT:    kmovd %r11d, %k2
-; AVX512DQ-BW-NEXT:    vpermw %zmm9, %zmm13, %zmm11 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3],xmm17[4],xmm14[4],xmm17[5],xmm14[5],xmm17[6],xmm14[6],xmm17[7],xmm14[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm22[0],xmm19[0],xmm22[1],xmm19[1],xmm22[2],xmm19[2],xmm22[3],xmm19[3],xmm22[4],xmm19[4],xmm22[5],xmm19[5],xmm22[6],xmm19[6],xmm22[7],xmm19[7]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermw %zmm15, %zmm6, %zmm9 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm27 = xmm24[0],xmm23[0],xmm24[1],xmm23[1],xmm24[2],xmm23[2],xmm24[3],xmm23[3],xmm24[4],xmm23[4],xmm24[5],xmm23[5],xmm24[6],xmm23[6],xmm24[7],xmm23[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512DQ-BW-NEXT:    vpermw %zmm15, %zmm12, %zmm15
-; AVX512DQ-BW-NEXT:    vpermw %zmm27, %zmm13, %zmm15 {%k2}
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r10), %xmm27
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm17[8],xmm14[8],xmm17[9],xmm14[9],xmm17[10],xmm14[10],xmm17[11],xmm14[11],xmm17[12],xmm14[12],xmm17[13],xmm14[13],xmm17[14],xmm14[14],xmm17[15],xmm14[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rax), %xmm28
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm22[8],xmm19[8],xmm22[9],xmm19[9],xmm22[10],xmm19[10],xmm22[11],xmm19[11],xmm22[12],xmm19[12],xmm22[13],xmm19[13],xmm22[14],xmm19[14],xmm22[15],xmm19[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r9), %xmm29
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm14 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero,xmm14[2],zero,zero,zero,xmm14[3],zero,zero,zero,xmm14[4],zero,zero,zero,xmm14[5],zero,zero,zero,xmm14[6],zero,zero,zero,xmm14[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermw %zmm17, %zmm6, %zmm14 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r8), %xmm30
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm24[8],xmm23[8],xmm24[9],xmm23[9],xmm24[10],xmm23[10],xmm24[11],xmm23[11],xmm24[12],xmm23[12],xmm24[13],xmm23[13],xmm24[14],xmm23[14],xmm24[15],xmm23[15]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512DQ-BW-NEXT:    vpermw %zmm19, %zmm12, %zmm19
-; AVX512DQ-BW-NEXT:    vpermw %zmm17, %zmm13, %zmm19 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm18[0],xmm16[0],xmm18[1],xmm16[1],xmm18[2],xmm16[2],xmm18[3],xmm16[3],xmm18[4],xmm16[4],xmm18[5],xmm16[5],xmm18[6],xmm16[6],xmm18[7],xmm16[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm17 = xmm21[0],xmm20[0],xmm21[1],xmm20[1],xmm21[2],xmm20[2],xmm21[3],xmm20[3],xmm21[4],xmm20[4],xmm21[5],xmm20[5],xmm21[6],xmm20[6],xmm21[7],xmm20[7]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm17 = xmm17[0],zero,zero,zero,xmm17[1],zero,zero,zero,xmm17[2],zero,zero,zero,xmm17[3],zero,zero,zero,xmm17[4],zero,zero,zero,xmm17[5],zero,zero,zero,xmm17[6],zero,zero,zero,xmm17[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermw %zmm22, %zmm6, %zmm17 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3],xmm28[4],xmm27[4],xmm28[5],xmm27[5],xmm28[6],xmm27[6],xmm28[7],xmm27[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm30[0],xmm29[0],xmm30[1],xmm29[1],xmm30[2],xmm29[2],xmm30[3],xmm29[3],xmm30[4],xmm29[4],xmm30[5],xmm29[5],xmm30[6],xmm29[6],xmm30[7],xmm29[7]
-; AVX512DQ-BW-NEXT:    vpermw %zmm22, %zmm12, %zmm22
-; AVX512DQ-BW-NEXT:    vpermw %zmm23, %zmm13, %zmm22 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm18[8],xmm16[8],xmm18[9],xmm16[9],xmm18[10],xmm16[10],xmm18[11],xmm16[11],xmm18[12],xmm16[12],xmm18[13],xmm16[13],xmm18[14],xmm16[14],xmm18[15],xmm16[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rcx), %xmm23
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm21[8],xmm20[8],xmm21[9],xmm20[9],xmm21[10],xmm20[10],xmm21[11],xmm20[11],xmm21[12],xmm20[12],xmm21[13],xmm20[13],xmm21[14],xmm20[14],xmm21[15],xmm20[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rdx), %xmm21
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm16 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero,xmm16[2],zero,zero,zero,xmm16[3],zero,zero,zero,xmm16[4],zero,zero,zero,xmm16[5],zero,zero,zero,xmm16[6],zero,zero,zero,xmm16[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermw %zmm18, %zmm6, %zmm16 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rsi), %xmm24
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm28[8],xmm27[8],xmm28[9],xmm27[9],xmm28[10],xmm27[10],xmm28[11],xmm27[11],xmm28[12],xmm27[12],xmm28[13],xmm27[13],xmm28[14],xmm27[14],xmm28[15],xmm27[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rdi), %xmm25
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm30[8],xmm29[8],xmm30[9],xmm29[9],xmm30[10],xmm29[10],xmm30[11],xmm29[11],xmm30[12],xmm29[12],xmm30[13],xmm29[13],xmm30[14],xmm29[14],xmm30[15],xmm29[15]
-; AVX512DQ-BW-NEXT:    vpermw %zmm18, %zmm12, %zmm18
-; AVX512DQ-BW-NEXT:    vpermw %zmm20, %zmm13, %zmm18 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm26 = xmm21[0],xmm23[0],xmm21[1],xmm23[1],xmm21[2],xmm23[2],xmm21[3],xmm23[3],xmm21[4],xmm23[4],xmm21[5],xmm23[5],xmm21[6],xmm23[6],xmm21[7],xmm23[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3],xmm25[4],xmm24[4],xmm25[5],xmm24[5],xmm25[6],xmm24[6],xmm25[7],xmm24[7]
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm20 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero,xmm20[2],zero,zero,zero,xmm20[3],zero,zero,zero,xmm20[4],zero,zero,zero,xmm20[5],zero,zero,zero,xmm20[6],zero,zero,zero,xmm20[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermw %zmm26, %zmm6, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm21[8],xmm23[8],xmm21[9],xmm23[9],xmm21[10],xmm23[10],xmm21[11],xmm23[11],xmm21[12],xmm23[12],xmm21[13],xmm23[13],xmm21[14],xmm23[14],xmm21[15],xmm23[15]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm25[8],xmm24[8],xmm25[9],xmm24[9],xmm25[10],xmm24[10],xmm25[11],xmm24[11],xmm25[12],xmm24[12],xmm25[13],xmm24[13],xmm25[14],xmm24[14],xmm25[15],xmm24[15]
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r10), %xmm24
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm23 = xmm23[0],zero,zero,zero,xmm23[1],zero,zero,zero,xmm23[2],zero,zero,zero,xmm23[3],zero,zero,zero,xmm23[4],zero,zero,zero,xmm23[5],zero,zero,zero,xmm23[6],zero,zero,zero,xmm23[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermw %zmm21, %zmm6, %zmm23 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rax), %xmm21
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-BW-NEXT:    vmovdqa 16(%r9), %xmm2
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512DQ-BW-NEXT:    vmovdqa 16(%r8), %xmm5
-; AVX512DQ-BW-NEXT:    vpmovzxwq {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm6, %zmm3 {%k1}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm21[0],xmm24[0],xmm21[1],xmm24[1],xmm21[2],xmm24[2],xmm21[3],xmm24[3],xmm21[4],xmm24[4],xmm21[5],xmm24[5],xmm21[6],xmm24[6],xmm21[7],xmm24[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; AVX512DQ-BW-NEXT:    vpermw %zmm6, %zmm12, %zmm6
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm13, %zmm6 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm21[8],xmm24[8],xmm21[9],xmm24[9],xmm21[10],xmm24[10],xmm21[11],xmm24[11],xmm21[12],xmm24[12],xmm21[13],xmm24[13],xmm21[14],xmm24[14],xmm21[15],xmm24[15]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
-; AVX512DQ-BW-NEXT:    vpermw %zmm2, %zmm12, %zmm2
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm13, %zmm2 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3],xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]
-; AVX512DQ-BW-NEXT:    vpermw %zmm4, %zmm12, %zmm4
-; AVX512DQ-BW-NEXT:    vpermw %zmm1, %zmm13, %zmm4 {%k2}
+; AVX512DQ-BW-NEXT:    vmovdqa (%r10), %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r10), %xmm16
+; AVX512DQ-BW-NEXT:    vmovdqa 48(%r10), %xmm14
+; AVX512DQ-BW-NEXT:    vmovdqa (%rax), %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rax), %xmm17
+; AVX512DQ-BW-NEXT:    vmovdqa 48(%rax), %xmm15
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r9), %xmm19
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r9), %xmm18
+; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm6
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%r8), %xmm21
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%r8), %xmm20
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,1,0,32,4,5,1,33,2,1,2,34,4,5,3,35,0,1,4,36,4,5,5,37,0,1,6,38,6,5,7,39]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm2, %zmm7, %zmm0
+; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rcx), %xmm22
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm8
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rdx), %xmm23
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15]
+; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm9
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rsi), %xmm24
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm10
+; AVX512DQ-BW-NEXT:    vmovdqa64 48(%rdi), %xmm25
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm11, %zmm12, %zmm2
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm20[0],xmm18[0],xmm20[1],xmm18[1],xmm20[2],xmm18[2],xmm20[3],xmm18[3],xmm20[4],xmm18[4],xmm20[5],xmm18[5],xmm20[6],xmm18[6],xmm20[7],xmm18[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm11, %zmm7, %zmm13
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm26 = xmm23[0],xmm22[0],xmm23[1],xmm22[1],xmm23[2],xmm22[2],xmm23[3],xmm22[3],xmm23[4],xmm22[4],xmm23[5],xmm22[5],xmm23[6],xmm22[6],xmm23[7],xmm22[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3],xmm25[4],xmm24[4],xmm25[5],xmm24[5],xmm25[6],xmm24[6],xmm25[7],xmm24[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm26, %zmm12, %zmm11
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rcx), %xmm26
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15]
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdx), %xmm27
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm20[8],xmm18[8],xmm20[9],xmm18[9],xmm20[10],xmm18[10],xmm20[11],xmm18[11],xmm20[12],xmm18[12],xmm20[13],xmm18[13],xmm20[14],xmm18[14],xmm20[15],xmm18[15]
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rsi), %xmm28
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm14, %zmm7, %zmm15
+; AVX512DQ-BW-NEXT:    vmovdqa64 32(%rdi), %xmm29
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm23[8],xmm22[8],xmm23[9],xmm22[9],xmm23[10],xmm22[10],xmm23[11],xmm22[11],xmm23[12],xmm22[12],xmm23[13],xmm22[13],xmm23[14],xmm22[14],xmm23[15],xmm22[15]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm25[8],xmm24[8],xmm25[9],xmm24[9],xmm25[10],xmm24[10],xmm25[11],xmm24[11],xmm25[12],xmm24[12],xmm25[13],xmm24[13],xmm25[14],xmm24[14],xmm25[15],xmm24[15]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm18, %zmm12, %zmm14
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm20 = xmm21[0],xmm19[0],xmm21[1],xmm19[1],xmm21[2],xmm19[2],xmm21[3],xmm19[3],xmm21[4],xmm19[4],xmm21[5],xmm19[5],xmm21[6],xmm19[6],xmm21[7],xmm19[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm18, %zmm7, %zmm20
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm22 = xmm27[0],xmm26[0],xmm27[1],xmm26[1],xmm27[2],xmm26[2],xmm27[3],xmm26[3],xmm27[4],xmm26[4],xmm27[5],xmm26[5],xmm27[6],xmm26[6],xmm27[7],xmm26[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm29[0],xmm28[0],xmm29[1],xmm28[1],xmm29[2],xmm28[2],xmm29[3],xmm28[3],xmm29[4],xmm28[4],xmm29[5],xmm28[5],xmm29[6],xmm28[6],xmm29[7],xmm28[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm22, %zmm12, %zmm18
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r10), %xmm22
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm21[8],xmm19[8],xmm21[9],xmm19[9],xmm21[10],xmm19[10],xmm21[11],xmm19[11],xmm21[12],xmm19[12],xmm21[13],xmm19[13],xmm21[14],xmm19[14],xmm21[15],xmm19[15]
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rax), %xmm19
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm17, %zmm7, %zmm16
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r9), %xmm21
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm27[8],xmm26[8],xmm27[9],xmm26[9],xmm27[10],xmm26[10],xmm27[11],xmm26[11],xmm27[12],xmm26[12],xmm27[13],xmm26[13],xmm27[14],xmm26[14],xmm27[15],xmm26[15]
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%r8), %xmm24
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm29[8],xmm28[8],xmm29[9],xmm28[9],xmm29[10],xmm28[10],xmm29[11],xmm28[11],xmm29[12],xmm28[12],xmm29[13],xmm28[13],xmm29[14],xmm28[14],xmm29[15],xmm28[15]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm23, %zmm12, %zmm17
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm19[0],xmm22[0],xmm19[1],xmm22[1],xmm19[2],xmm22[2],xmm19[3],xmm22[3],xmm19[4],xmm22[4],xmm19[5],xmm22[5],xmm19[6],xmm22[6],xmm19[7],xmm22[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm24[0],xmm21[0],xmm24[1],xmm21[1],xmm24[2],xmm21[2],xmm24[3],xmm21[3],xmm24[4],xmm21[4],xmm24[5],xmm21[5],xmm24[6],xmm21[6],xmm24[7],xmm21[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm23, %zmm7, %zmm25
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm19[8],xmm22[8],xmm19[9],xmm22[9],xmm19[10],xmm22[10],xmm19[11],xmm22[11],xmm19[12],xmm22[12],xmm19[13],xmm22[13],xmm19[14],xmm22[14],xmm19[15],xmm22[15]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm24[8],xmm21[8],xmm24[9],xmm21[9],xmm24[10],xmm21[10],xmm24[11],xmm21[11],xmm24[12],xmm21[12],xmm24[13],xmm21[13],xmm24[14],xmm21[14],xmm24[15],xmm21[15]
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rcx), %xmm22
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm19, %zmm7, %zmm21
+; AVX512DQ-BW-NEXT:    vmovdqa64 16(%rdx), %xmm19
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512DQ-BW-NEXT:    vmovdqa 16(%rsi), %xmm3
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX512DQ-BW-NEXT:    vmovdqa 16(%rdi), %xmm6
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm7, %zmm4
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm19[0],xmm22[0],xmm19[1],xmm22[1],xmm19[2],xmm22[2],xmm19[3],xmm22[3],xmm19[4],xmm22[4],xmm19[5],xmm22[5],xmm19[6],xmm22[6],xmm19[7],xmm22[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm12, %zmm7
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm19[8],xmm22[8],xmm19[9],xmm22[9],xmm19[10],xmm22[10],xmm19[11],xmm22[11],xmm19[12],xmm22[12],xmm19[13],xmm22[13],xmm19[14],xmm22[14],xmm19[15],xmm22[15]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm6[8],xmm3[8],xmm6[9],xmm3[9],xmm6[10],xmm3[10],xmm6[11],xmm3[11],xmm6[12],xmm3[12],xmm6[13],xmm3[13],xmm6[14],xmm3[14],xmm6[15],xmm3[15]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm12, %zmm3
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
+; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm12, %zmm5
 ; AVX512DQ-BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA
 ; AVX512DQ-BW-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm11, %zmm0 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm15, %zmm9 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm19, %zmm14 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm22, %zmm17 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm18, %zmm16 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm6, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm2, %zmm23 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm3 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm13, %zmm11 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm15, %zmm14 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm20, %zmm18 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm17 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm25, %zmm7 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm21, %zmm3 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k1}
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm23, 192(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm20, 128(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm17, 256(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, 192(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, 128(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm17, 320(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm18, 256(%rax)
 ; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm14, 448(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm9, 384(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm11, 384(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
@@ -8481,172 +8432,173 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP:       # %bb.0:
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm20
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rsi), %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm19
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rsi), %xmm16
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm21
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rdi), %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm20
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rdi), %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxwq {{.*#+}} ymm4 = [2312,2826,3340,3854]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxwq {{.*#+}} xmm5 = [1284,1798]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxwq {{.*#+}} ymm5 = [2312,2826,3340,3854]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxwq {{.*#+}} xmm6 = [1284,1798]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm0, %xmm4
 ; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm22
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rcx), %xmm19
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rcx), %xmm18
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdx), %xmm23
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rdx), %xmm24
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,2,1,2,1,6,7,0,2,2,3,2,3,6,7,4,4,2,3,6,5,6,5,4,6,2,3,6,7,6,7]
-; AVX512DQ-BW-FCP-NEXT:    movl $572662306, %r11d # imm = 0x22222222
-; AVX512DQ-BW-FCP-NEXT:    kmovd %r11d, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm6, %zmm8, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r10), %xmm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%r10), %xmm25
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rax), %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rdx), %xmm22
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rdx), %xmm23
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm4, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,32,2,3,4,33,6,7,8,42,10,11,12,43,14,15,16,36,18,19,20,37,22,23,24,46,26,27,28,47,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm4, %zmm9, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r10), %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%r10), %xmm24
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rax), %xmm8
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%rax), %xmm26
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm15 = xmm9[8],xmm6[8],xmm9[9],xmm6[9],xmm9[10],xmm6[10],xmm9[11],xmm6[11],xmm9[12],xmm6[12],xmm9[13],xmm6[13],xmm9[14],xmm6[14],xmm9[15],xmm6[15]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm8[8],xmm4[8],xmm8[9],xmm4[9],xmm8[10],xmm4[10],xmm8[11],xmm4[11],xmm8[12],xmm4[12],xmm8[13],xmm4[13],xmm8[14],xmm4[14],xmm8[15],xmm4[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm10, %ymm13
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r9), %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%r9), %xmm27
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%r8), %xmm28
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,1,0,1,4,5,1,3,2,1,2,1,4,5,3,3,0,1,4,5,4,5,5,7,0,1,6,5,6,5,7,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm12, %zmm13, %zmm12
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,0,0,0,4,5,2,1,0,2,0,2,4,5,2,3,0,1,4,4,4,4,6,5,0,1,4,6,4,6,6,7]
-; AVX512DQ-BW-FCP-NEXT:    movl $-2004318072, %r11d # imm = 0x88888888
-; AVX512DQ-BW-FCP-NEXT:    kmovd %r11d, %k2
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm15, %zmm14, %zmm12 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm18[0],xmm17[0],xmm18[1],xmm17[1],xmm18[2],xmm17[2],xmm18[3],xmm17[3],xmm18[4],xmm17[4],xmm18[5],xmm17[5],xmm18[6],xmm17[6],xmm18[7],xmm17[7]
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm15, %ymm15, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm16, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm15, %xmm29
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm15 = xmm15[0],zero,zero,zero,xmm15[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm29, %ymm15, %ymm15
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm15, %zmm15
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm24[0],xmm19[0],xmm24[1],xmm19[1],xmm24[2],xmm19[2],xmm24[3],xmm19[3],xmm24[4],xmm19[4],xmm24[5],xmm19[5],xmm24[6],xmm19[6],xmm24[7],xmm19[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm8, %zmm15 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm29 = xmm26[0],xmm25[0],xmm26[1],xmm25[1],xmm26[2],xmm25[2],xmm26[3],xmm25[3],xmm26[4],xmm25[4],xmm26[5],xmm25[5],xmm26[6],xmm25[6],xmm26[7],xmm25[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3],xmm28[4],xmm27[4],xmm28[5],xmm27[5],xmm28[6],xmm27[6],xmm28[7],xmm27[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm13, %zmm16
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm29, %zmm14, %zmm16 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm17, %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm29 = xmm17[0],zero,zero,zero,xmm17[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm29, %ymm18
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r10), %xmm29
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%r9), %xmm28
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %xmm12
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 48(%r8), %xmm30
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm12[8],xmm10[8],xmm12[9],xmm10[9],xmm12[10],xmm10[10],xmm12[11],xmm10[11],xmm12[12],xmm10[12],xmm12[13],xmm10[13],xmm12[14],xmm10[14],xmm12[15],xmm10[15]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,1,0,32,4,5,1,33,2,1,2,42,4,5,3,43,0,1,4,36,4,5,5,37,0,1,6,46,6,5,7,47]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm13, %zmm14, %zmm11
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm17[0],xmm16[0],xmm17[1],xmm16[1],xmm17[2],xmm16[2],xmm17[3],xmm16[3],xmm17[4],xmm16[4],xmm17[5],xmm16[5],xmm17[6],xmm16[6],xmm17[7],xmm16[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm13, %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm15, %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm13, %xmm25
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm13 = xmm13[0],zero,zero,zero,xmm13[1],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm13, %ymm13
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm13, %zmm13
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm23[0],xmm18[0],xmm23[1],xmm18[1],xmm23[2],xmm18[2],xmm23[3],xmm18[3],xmm23[4],xmm18[4],xmm23[5],xmm18[5],xmm23[6],xmm18[6],xmm23[7],xmm18[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm15, %ymm15, %ymm15
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm15, %zmm9, %zmm13
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm26[0],xmm24[0],xmm26[1],xmm24[1],xmm26[2],xmm24[2],xmm26[3],xmm24[3],xmm26[4],xmm24[4],xmm26[5],xmm24[5],xmm26[6],xmm24[6],xmm26[7],xmm24[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm15, %ymm15, %ymm25
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm30[0],xmm28[0],xmm30[1],xmm28[1],xmm30[2],xmm28[2],xmm30[3],xmm28[3],xmm30[4],xmm28[4],xmm30[5],xmm28[5],xmm30[6],xmm28[6],xmm30[7],xmm28[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm25, %zmm14, %zmm15
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm16 = xmm17[8],xmm16[8],xmm17[9],xmm16[9],xmm17[10],xmm16[10],xmm17[11],xmm16[11],xmm17[12],xmm16[12],xmm17[13],xmm16[13],xmm17[14],xmm16[14],xmm17[15],xmm16[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm16, %xmm17
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm16[0],zero,zero,zero,xmm16[1],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm25, %ymm17
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r10), %xmm25
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm16, %ymm16, %ymm16
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm16, %ymm16
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm17, %zmm16
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rax), %xmm29
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm23[8],xmm18[8],xmm23[9],xmm18[9],xmm23[10],xmm18[10],xmm23[11],xmm18[11],xmm23[12],xmm18[12],xmm23[13],xmm18[13],xmm23[14],xmm18[14],xmm23[15],xmm18[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm27
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm17, %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm17, %ymm17
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm18, %zmm17
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rax), %xmm30
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm24[8],xmm19[8],xmm24[9],xmm19[9],xmm24[10],xmm19[10],xmm24[11],xmm19[11],xmm24[12],xmm19[12],xmm24[13],xmm19[13],xmm24[14],xmm19[14],xmm24[15],xmm19[15]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm31
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm18, %zmm8, %zmm17 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r8), %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm26[8],xmm25[8],xmm26[9],xmm25[9],xmm26[10],xmm25[10],xmm26[11],xmm25[11],xmm26[12],xmm25[12],xmm26[13],xmm25[13],xmm26[14],xmm25[14],xmm26[15],xmm25[15]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm18 = xmm28[8],xmm27[8],xmm28[9],xmm27[9],xmm28[10],xmm27[10],xmm28[11],xmm27[11],xmm28[12],xmm27[12],xmm28[13],xmm27[13],xmm28[14],xmm27[14],xmm28[15],xmm27[15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm18, %zmm13, %zmm18
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm19, %zmm14, %zmm18 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm19 = xmm21[0],xmm20[0],xmm21[1],xmm20[1],xmm21[2],xmm20[2],xmm21[3],xmm20[3],xmm21[4],xmm20[4],xmm21[5],xmm20[5],xmm21[6],xmm20[6],xmm21[7],xmm20[7]
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm19, %ymm19, %ymm24
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm24, %ymm24
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm19, %xmm25
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm19 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm19, %ymm19
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm24, %zmm19, %zmm19
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm24 = xmm23[0],xmm22[0],xmm23[1],xmm22[1],xmm23[2],xmm22[2],xmm23[3],xmm22[3],xmm23[4],xmm22[4],xmm23[5],xmm22[5],xmm23[6],xmm22[6],xmm23[7],xmm22[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm24, %zmm8, %zmm19 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm30[0],xmm29[0],xmm30[1],xmm29[1],xmm30[2],xmm29[2],xmm30[3],xmm29[3],xmm30[4],xmm29[4],xmm30[5],xmm29[5],xmm30[6],xmm29[6],xmm30[7],xmm29[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm24 = xmm0[0],xmm31[0],xmm0[1],xmm31[1],xmm0[2],xmm31[2],xmm0[3],xmm31[3],xmm0[4],xmm31[4],xmm0[5],xmm31[5],xmm0[6],xmm31[6],xmm0[7],xmm31[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm24, %zmm13, %zmm24
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm25, %zmm14, %zmm24 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm21[8],xmm20[8],xmm21[9],xmm20[9],xmm21[10],xmm20[10],xmm21[11],xmm20[11],xmm21[12],xmm20[12],xmm21[13],xmm20[13],xmm21[14],xmm20[14],xmm21[15],xmm20[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm20, %xmm21
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm20[0],zero,zero,zero,xmm20[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm25, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm17, %zmm9, %zmm16
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm31
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm26[8],xmm24[8],xmm26[9],xmm24[9],xmm26[10],xmm24[10],xmm26[11],xmm24[11],xmm26[12],xmm24[12],xmm26[13],xmm24[13],xmm26[14],xmm24[14],xmm26[15],xmm24[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm17, %ymm17, %ymm18
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm17 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm18, %zmm14, %zmm17
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm18 = xmm20[0],xmm19[0],xmm20[1],xmm19[1],xmm20[2],xmm19[2],xmm20[3],xmm19[3],xmm20[4],xmm19[4],xmm20[5],xmm19[5],xmm20[6],xmm19[6],xmm20[7],xmm19[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm18, %ymm23
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm23, %ymm23
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm18, %xmm24
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm18 = xmm18[0],zero,zero,zero,xmm18[1],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm24, %ymm18, %ymm18
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm23, %zmm18, %zmm18
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm22[0],xmm21[0],xmm22[1],xmm21[1],xmm22[2],xmm21[2],xmm22[3],xmm21[3],xmm22[4],xmm21[4],xmm22[5],xmm21[5],xmm22[6],xmm21[6],xmm22[7],xmm21[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm23, %ymm23, %ymm23
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm23, %zmm9, %zmm18
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm29[0],xmm25[0],xmm29[1],xmm25[1],xmm29[2],xmm25[2],xmm29[3],xmm25[3],xmm29[4],xmm25[4],xmm29[5],xmm25[5],xmm29[6],xmm25[6],xmm29[7],xmm25[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm23, %ymm23, %ymm24
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm23 = xmm31[0],xmm27[0],xmm31[1],xmm27[1],xmm31[2],xmm27[2],xmm31[3],xmm27[3],xmm31[4],xmm27[4],xmm31[5],xmm27[5],xmm31[6],xmm27[6],xmm31[7],xmm27[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm24, %zmm14, %zmm23
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm19 = xmm20[8],xmm19[8],xmm20[9],xmm19[9],xmm20[10],xmm19[10],xmm20[11],xmm19[11],xmm20[12],xmm19[12],xmm20[13],xmm19[13],xmm20[14],xmm19[14],xmm20[15],xmm19[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm19, %xmm20
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm24 = xmm19[0],zero,zero,zero,xmm19[1],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm24, %ymm20
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm19, %ymm19, %ymm19
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm19, %ymm19
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm20, %zmm19
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 16(%rsi), %xmm24
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm22[8],xmm21[8],xmm22[9],xmm21[9],xmm22[10],xmm21[10],xmm22[11],xmm21[11],xmm22[12],xmm21[12],xmm22[13],xmm21[13],xmm22[14],xmm21[14],xmm22[15],xmm21[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 16(%rdi), %xmm22
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm20, %ymm20
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm20, %ymm20
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm21, %zmm20
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 16(%rsi), %xmm25
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm21 = xmm23[8],xmm22[8],xmm23[9],xmm22[9],xmm23[10],xmm22[10],xmm23[11],xmm22[11],xmm23[12],xmm22[12],xmm23[13],xmm22[13],xmm23[14],xmm22[14],xmm23[15],xmm22[15]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 16(%rdi), %xmm23
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm21, %zmm8, %zmm20 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm20, %zmm9, %zmm19
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 16(%rcx), %xmm26
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm30[8],xmm29[8],xmm30[9],xmm29[9],xmm30[10],xmm29[10],xmm30[11],xmm29[11],xmm30[12],xmm29[12],xmm30[13],xmm29[13],xmm30[14],xmm29[14],xmm30[15],xmm29[15]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm31[8],xmm0[9],xmm31[9],xmm0[10],xmm31[10],xmm0[11],xmm31[11],xmm0[12],xmm31[12],xmm0[13],xmm31[13],xmm0[14],xmm31[14],xmm0[15],xmm31[15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm13, %zmm21
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm22, %zmm14, %zmm21 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm23[0],xmm25[0],xmm23[1],xmm25[1],xmm23[2],xmm25[2],xmm23[3],xmm25[3],xmm23[4],xmm25[4],xmm23[5],xmm25[5],xmm23[6],xmm25[6],xmm23[7],xmm25[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm22
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm27 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm22, %ymm27, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm29[8],xmm25[8],xmm29[9],xmm25[9],xmm29[10],xmm25[10],xmm29[11],xmm25[11],xmm29[12],xmm25[12],xmm29[13],xmm25[13],xmm29[14],xmm25[14],xmm29[15],xmm25[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm20, %ymm20, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm20 = xmm31[8],xmm27[8],xmm31[9],xmm27[9],xmm31[10],xmm27[10],xmm31[11],xmm27[11],xmm31[12],xmm27[12],xmm31[13],xmm27[13],xmm31[14],xmm27[14],xmm31[15],xmm27[15]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm21, %zmm14, %zmm20
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm21 = xmm22[0],xmm24[0],xmm22[1],xmm24[1],xmm22[2],xmm24[2],xmm22[3],xmm24[3],xmm22[4],xmm24[4],xmm22[5],xmm24[5],xmm22[6],xmm24[6],xmm22[7],xmm24[7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm21, %xmm25
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm27 = xmm21[0],zero,zero,zero,xmm21[1],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm27, %ymm25
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 16(%rdx), %xmm27
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm22, %zmm22
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm27[0],xmm26[0],xmm27[1],xmm26[1],xmm27[2],xmm26[2],xmm27[3],xmm26[3],xmm27[4],xmm26[4],xmm27[5],xmm26[5],xmm27[6],xmm26[6],xmm27[7],xmm26[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm8, %zmm22 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm23[8],xmm25[8],xmm23[9],xmm25[9],xmm23[10],xmm25[10],xmm23[11],xmm25[11],xmm23[12],xmm25[12],xmm23[13],xmm25[13],xmm23[14],xmm25[14],xmm23[15],xmm25[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm23
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm23, %ymm25, %ymm23
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm23, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm23 = xmm27[8],xmm26[8],xmm27[9],xmm26[9],xmm27[10],xmm26[10],xmm27[11],xmm26[11],xmm27[12],xmm26[12],xmm27[13],xmm26[13],xmm27[14],xmm26[14],xmm27[15],xmm26[15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm23, %zmm8, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
-; AVX512DQ-BW-FCP-NEXT:    # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3],xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm2, %ymm2, %ymm23
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm23, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 16(%r10), %xmm23
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm5, %xmm2, %xmm5
-; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 16(%rax), %xmm5
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 16(%r9), %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm21, %ymm21, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm21, %ymm21
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm25, %zmm21
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm25 = xmm27[0],xmm26[0],xmm27[1],xmm26[1],xmm27[2],xmm26[2],xmm27[3],xmm26[3],xmm27[4],xmm26[4],xmm27[5],xmm26[5],xmm27[6],xmm26[6],xmm27[7],xmm26[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm25, %ymm25, %ymm25
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm25, %zmm9, %zmm21
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm22 = xmm22[8],xmm24[8],xmm22[9],xmm24[9],xmm22[10],xmm24[10],xmm22[11],xmm24[11],xmm22[12],xmm24[12],xmm22[13],xmm24[13],xmm22[14],xmm24[14],xmm22[15],xmm24[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm22, %xmm24
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm25 = xmm22[0],zero,zero,zero,xmm22[1],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm24, %ymm25, %ymm24
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm22, %ymm22, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm22, %ymm22
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm22, %zmm24, %zmm22
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm24 = xmm27[8],xmm26[8],xmm27[9],xmm26[9],xmm27[10],xmm26[10],xmm27[11],xmm26[11],xmm27[12],xmm26[12],xmm27[13],xmm26[13],xmm27[14],xmm26[14],xmm27[15],xmm26[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm24, %ymm24, %ymm24
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm24, %zmm9, %zmm22
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm1, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 16(%r10), %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 16(%rax), %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 16(%r9), %xmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 16(%r8), %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm8, %zmm2 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm5[0],xmm23[0],xmm5[1],xmm23[1],xmm5[2],xmm23[2],xmm5[3],xmm23[3],xmm5[4],xmm23[4],xmm5[5],xmm23[5],xmm5[6],xmm23[6],xmm5[7],xmm23[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm8, %zmm13, %zmm8
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm14, %zmm8 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm23[8],xmm5[9],xmm23[9],xmm5[10],xmm23[10],xmm5[11],xmm23[11],xmm5[12],xmm23[12],xmm5[13],xmm23[13],xmm5[14],xmm23[14],xmm5[15],xmm23[15]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm4, %zmm13, %zmm4
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm14, %zmm4 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3],xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm5, %zmm13, %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm3, %zmm14, %zmm5 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm9, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm14, %zmm9
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm14, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm14, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm15 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm18, %zmm17 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm24, %zmm19 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm21, %zmm20 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm8, %zmm22 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm11, %zmm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm15, %zmm13 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm17, %zmm16 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm23, %zmm18 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm20, %zmm19 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm21 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm2, %zmm22 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm1 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 192(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm22, 128(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm20, 320(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm19, 256(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm17, 448(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm15, 384(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm22, 192(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm21, 128(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm19, 320(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm18, 256(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, 448(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, 384(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <64 x i8>, ptr %in.vecptr0, align 64
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-mul.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-mul.ll
index 98b5bab98c4f..13b21a747878 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-mul.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-mul.ll
@@ -1579,7 +1579,7 @@ define <2 x i64> @mul_v2i64_0_1(<2 x i64> %a0) nounwind {
 ; SSE4-LABEL: mul_v2i64_0_1:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    xorps %xmm1, %xmm1
-; SSE4-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE4-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE4-NEXT:    ret{{[l|q]}}
 ;
 ; X64-AVX-LABEL: mul_v2i64_0_1:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-pcmp.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-pcmp.ll
index 5b43acbe5237..30eb2279bda8 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-pcmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-pcmp.ll
@@ -1749,7 +1749,7 @@ define <16 x i1> @is_positive_mask_v16i16_v16i1(<16 x i16> %x, <16 x i1> %y) {
 ; AVX512F-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1 {%k1}
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
index 4898ae98faea..983ae594e3ab 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
@@ -112,7 +112,7 @@ define i64 @test_v4i64_v4i16(<4 x i64> %a0) {
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
index 937ac3d2db88..d99b20038558 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
@@ -231,7 +231,7 @@ define i32 @test_v4i32(<4 x i8> %a0) {
 ; AVX2-LABEL: test_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    retq
@@ -239,7 +239,7 @@ define i32 @test_v4i32(<4 x i8> %a0) {
 ; AVX512-LABEL: test_v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add.ll
index 6cc0e1e73fcd..aed4e023e340 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-reduce-add.ll
@@ -1025,19 +1025,28 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: test_v4i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
-; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    # kill: def $al killed $al killed $eax
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_v4i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v4i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v4i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-128.ll
index 993e6afc0eaf..6c79be75550e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -1581,7 +1581,7 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX512NOVLX:       # %bb.0:
 ; AVX512NOVLX-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512NOVLX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512NOVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512NOVLX-NEXT:    vzeroupper
 ; AVX512NOVLX-NEXT:    retq
@@ -1590,7 +1590,7 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VLX:       # %bb.0:
 ; AVX512VLX-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VLX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLX-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VLX-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VLX-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_v16i8:
@@ -1739,7 +1739,7 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $5, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpsrlw $11, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = mem & (xmm0 | xmm1)
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} xmm0 = m32bcst & (xmm0 | xmm1)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v8i16:
@@ -1754,7 +1754,7 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $5, %xmm0, %xmm1
 ; AVX512VLBW-NEXT:    vpsrlw $11, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = mem & (xmm0 | xmm1)
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} xmm0 = m32bcst & (xmm0 | xmm1)
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v8i16:
@@ -1819,7 +1819,7 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind {
 ; AVX512NOVLX:       # %bb.0:
 ; AVX512NOVLX-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512NOVLX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512NOVLX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX512NOVLX-NEXT:    vzeroupper
 ; AVX512NOVLX-NEXT:    retq
@@ -1828,7 +1828,7 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VLX:       # %bb.0:
 ; AVX512VLX-NEXT:    vpsllw $4, %xmm0, %xmm1
 ; AVX512VLX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLX-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm1))
+; AVX512VLX-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm0 ^ (m32bcst & (xmm0 ^ xmm1))
 ; AVX512VLX-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
 ; AVX512VLX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-256.ll
index c2c6a5f7eba5..684721f434eb 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -387,12 +387,12 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm2))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
@@ -407,17 +407,17 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm2))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 | (ymm2 & mem)
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm3 = ymm3 | (ymm2 & m32bcst)
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
@@ -1392,7 +1392,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX512NOVLX:       # %bb.0:
 ; AVX512NOVLX-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512NOVLX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512NOVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512NOVLX-NEXT:    retq
 ;
@@ -1400,7 +1400,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLX:       # %bb.0:
 ; AVX512VLX-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VLX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VLX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VLX-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_rotate_v32i8:
@@ -1566,7 +1566,7 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsllw $5, %ymm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $11, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = mem & (ymm0 | ymm1)
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} ymm0 = m32bcst & (ymm0 | ymm1)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v16i16:
@@ -1581,7 +1581,7 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $5, %ymm0, %ymm1
 ; AVX512VLBW-NEXT:    vpsrlw $11, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = mem & (ymm0 | ymm1)
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} ymm0 = m32bcst & (ymm0 | ymm1)
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v16i16:
@@ -1653,7 +1653,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ; AVX512NOVLX:       # %bb.0:
 ; AVX512NOVLX-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512NOVLX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512NOVLX-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512NOVLX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX512NOVLX-NEXT:    retq
 ;
@@ -1661,7 +1661,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLX:       # %bb.0:
 ; AVX512VLX-NEXT:    vpsllw $4, %ymm0, %ymm1
 ; AVX512VLX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
+; AVX512VLX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
 ; AVX512VLX-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
 ; AVX512VLX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-512.ll
index 8ac0b178a16d..2cde988ed776 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -754,7 +754,7 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_v64i8:
@@ -766,35 +766,35 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
@@ -844,7 +844,7 @@ define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 ; AVX512F-NEXT:    vpsrlw $11, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $11, %ymm2, %ymm2
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = mem & (zmm0 | zmm1)
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v32i16:
@@ -856,21 +856,21 @@ define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $11, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $11, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = mem & (zmm0 | zmm1)
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v32i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $5, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $11, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = mem & (zmm0 | zmm1)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_rotate_mask_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $5, %zmm0, %zmm1
 ; AVX512VLBW-NEXT:    vpsrlw $11, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = mem & (zmm0 | zmm1)
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = m32bcst & (zmm0 | zmm1)
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v32i16:
@@ -902,7 +902,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -915,7 +915,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
@@ -923,7 +923,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -931,7 +931,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLBW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
 ;
@@ -939,7 +939,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VBMI2-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -947,7 +947,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512VLVBMI2-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512VLVBMI2-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index 60295f1c145a..02f0f53a0bb3 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -2021,7 +2021,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512DQVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    retq
 ;
@@ -2029,7 +2029,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512BWVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index 9f3fff34ea20..15855e3bce46 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -2295,7 +2295,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpsrlw $3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem)
+; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
 ; AVX512DQVL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    retq
 ;
@@ -2303,7 +2303,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpsrlw $3, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpbroadcastb {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem)
+; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
 ; AVX512BWVL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
index 74dbee5e5d2c..ea0745b157f5 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
@@ -540,7 +540,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsrlw $3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m32bcst)
 ; AVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index 4f8cbc07243f..f7de8d427150 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -2344,7 +2344,7 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512DQVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    retq
 ;
@@ -2352,7 +2352,7 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512BWVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
@@ -2414,7 +2414,7 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512DQVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    retq
 ;
@@ -2422,7 +2422,7 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512BWVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
@@ -2484,7 +2484,7 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512DQVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    retq
 ;
@@ -2492,7 +2492,7 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem)
+; AVX512BWVL-NEXT:    vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
 ; AVX512BWVL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 4f55f7af20f4..3f238b5739f0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -1369,7 +1369,7 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512DQVL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0]
 ; AVX512DQVL-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
 ; AVX512DQVL-NEXT:    vpsllw $8, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 | (ymm1 & mem)
+; AVX512DQVL-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 | (ymm1 & m32bcst)
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: constant_shift_v32i8:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
index a42056be895e..efd742956ed0 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
@@ -324,7 +324,7 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpsllw $8, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & m32bcst)
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_shift_v64i8:
@@ -332,7 +332,7 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0]
 ; AVX512BW-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
 ; AVX512BW-NEXT:    vpsllw $8, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & m32bcst)
 ; AVX512BW-NEXT:    retq
   %shift = shl <64 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <64 x i8> %shift
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
index 1d389f981722..baaae507ae15 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v2.ll
@@ -242,35 +242,20 @@ define <2 x double> @shuffle_v2f64_03(<2 x double> %a, <2 x double> %b) {
 ;
 ; AVX-LABEL: shuffle_v2f64_03:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
   %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %shuffle
 }
 define <2 x double> @shuffle_v2f64_21(<2 x double> %a, <2 x double> %b) {
-; SSE2-LABEL: shuffle_v2f64_21:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE3-LABEL: shuffle_v2f64_21:
-; SSE3:       # %bb.0:
-; SSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE3-NEXT:    retq
-;
-; SSSE3-LABEL: shuffle_v2f64_21:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: shuffle_v2f64_21:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: shuffle_v2f64_21:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v2f64_21:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
   %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 2, i32 1>
   ret <2 x double> %shuffle
@@ -523,25 +508,10 @@ define <2 x i64> @shuffle_v2i64_20_copy(<2 x i64> %nonce, <2 x i64> %a, <2 x i64
   ret <2 x i64> %shuffle
 }
 define <2 x i64> @shuffle_v2i64_21(<2 x i64> %a, <2 x i64> %b) {
-; SSE2-LABEL: shuffle_v2i64_21:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE3-LABEL: shuffle_v2i64_21:
-; SSE3:       # %bb.0:
-; SSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE3-NEXT:    retq
-;
-; SSSE3-LABEL: shuffle_v2i64_21:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: shuffle_v2i64_21:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: shuffle_v2i64_21:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v2i64_21:
 ; AVX:       # %bb.0:
@@ -572,7 +542,7 @@ define <2 x i64> @shuffle_v2i64_21_copy(<2 x i64> %nonce, <2 x i64> %a, <2 x i64
 ; SSE41-LABEL: shuffle_v2i64_21_copy:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v2i64_21_copy:
@@ -740,7 +710,7 @@ define <2 x i64> @shuffle_v2i64_z1(<2 x i64> %a) {
 ; SSE41-LABEL: shuffle_v2i64_z1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v2i64_z1:
@@ -821,13 +791,13 @@ define <2 x double> @shuffle_v2f64_z1(<2 x double> %a) {
 ; SSE41-LABEL: shuffle_v2f64_z1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v2f64_z1:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
   %shuffle = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <2 x i32> <i32 2, i32 1>
   ret <2 x double> %shuffle
@@ -871,13 +841,13 @@ define <2 x i64> @shuffle_v2i64_bitcast_z123(<2 x i64> %x) {
 ; SSE41-LABEL: shuffle_v2i64_bitcast_z123:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE41-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_v2i64_bitcast_z123:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %bitcast32 = bitcast <2 x i64> %x to <4 x float>
   %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> <float 1.000000e+00, float poison, float poison, float poison>, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
@@ -1102,7 +1072,7 @@ define <2 x double> @insert_reg_lo_v2f64(double %a, <2 x double> %b) {
 ;
 ; AVX-LABEL: insert_reg_lo_v2f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
   %v = insertelement <2 x double> poison, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> <i32 0, i32 3>
@@ -1334,7 +1304,7 @@ define <2 x double> @shuffle_mem_v2f64_21(<2 x double> %a, ptr %pb) {
 ; SSE41-LABEL: shuffle_mem_v2f64_21:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movups (%rdi), %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_mem_v2f64_21:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
index 0eb72c8bc0be..9ec24c447c2c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
@@ -506,7 +506,7 @@ define <4 x i32> @shuffle_v4i32_4012(<4 x i32> %a, <4 x i32> %b) {
 ; AVX1OR2-LABEL: shuffle_v4i32_4012:
 ; AVX1OR2:       # %bb.0:
 ; AVX1OR2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0,1,2]
-; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX1OR2-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v4i32_4012:
@@ -618,7 +618,7 @@ define <4 x float> @shuffle_v4f32_4zzz(<4 x float> %a) {
 ; AVX-LABEL: shuffle_v4f32_4zzz:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %shuffle = shufflevector <4 x float> zeroinitializer, <4 x float> %a, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
   ret <4 x float> %shuffle
@@ -1164,7 +1164,7 @@ define <4 x i32> @shuffle_v4i32_4zzz(<4 x i32> %a) {
 ; AVX-LABEL: shuffle_v4i32_4zzz:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %shuffle = shufflevector <4 x i32> zeroinitializer, <4 x i32> %a, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
   ret <4 x i32> %shuffle
@@ -1202,14 +1202,14 @@ define <4 x i32> @shuffle_v4i32_z4zz(<4 x i32> %a) {
 ; AVX1-LABEL: shuffle_v4i32_z4zz:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,0,1,1]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v4i32_z4zz:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX2-SLOW-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,0,1,1]
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -1258,14 +1258,14 @@ define <4 x i32> @shuffle_v4i32_zz4z(<4 x i32> %a) {
 ; AVX1-LABEL: shuffle_v4i32_zz4z:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v4i32_zz4z:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX2-SLOW-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -2138,7 +2138,7 @@ define <4 x float> @insert_reg_and_zero_v4f32(float %a) {
 ; AVX-LABEL: insert_reg_and_zero_v4f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %v = insertelement <4 x float> poison, float %a, i32 0
   %shuffle = shufflevector <4 x float> %v, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
@@ -2222,7 +2222,7 @@ define <4 x i32> @insert_mem_lo_v4i32(ptr %ptr, <4 x i32> %b) {
 ; SSE41-LABEL: insert_mem_lo_v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: insert_mem_lo_v4i32:
@@ -2295,7 +2295,7 @@ define <4 x float> @insert_reg_lo_v4f32(double %a, <4 x float> %b) {
 ;
 ; AVX-LABEL: insert_reg_lo_v4f32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
   %a.cast = bitcast double %a to <2 x float>
   %v = shufflevector <2 x float> %a.cast, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
@@ -2489,7 +2489,7 @@ define <4 x float> @shuffle_mem_v4f32_4523(<4 x float> %a, ptr %pb) {
 ; SSE41-LABEL: shuffle_mem_v4f32_4523:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movups (%rdi), %xmm1
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; SSE41-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shuffle_mem_v4f32_4523:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index d848a8b87921..94fc982d87e5 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -2376,33 +2376,21 @@ define <4 x i64> @shuffle_v4i64_0zzz_pgso(<4 x i64> %a) !prof !14 {
 }
 
 define <8 x float> @shuffle_v8f32_0zzzzzzz_pgso(<8 x float> %a) !prof !14 {
-; AVX1OR2-LABEL: shuffle_v8f32_0zzzzzzz_pgso:
-; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX1OR2-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v8f32_0zzzzzzz_pgso:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX512VL-NEXT:    retq
+; ALL-LABEL: shuffle_v8f32_0zzzzzzz_pgso:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; ALL-NEXT:    retq
   %b = shufflevector <8 x float> %a, <8 x float> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x float> %b
 }
 
 define <8 x i32> @shuffle_v8i32_0zzzzzzz_pgso(<8 x i32> %a) !prof !14 {
-; AVX1OR2-LABEL: shuffle_v8i32_0zzzzzzz_pgso:
-; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX1OR2-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v8i32_0zzzzzzz_pgso:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX512VL-NEXT:    retq
+; ALL-LABEL: shuffle_v8i32_0zzzzzzz_pgso:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; ALL-NEXT:    retq
   %b = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x i32> %b
 }
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index bd78dbded070..bce50db4d952 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -468,7 +468,7 @@ define <8 x float> @shuffle_v8f32_08991abb(<8 x float> %a, <8 x float> %b) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[1,1]
-; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2,3,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    retq
@@ -2006,8 +2006,8 @@ define <8 x i32> @shuffle_v8i32_81a3c5e7(<8 x i32> %a, <8 x i32> %b) {
 define <8 x i32> @shuffle_v8i32_08080808(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-LABEL: shuffle_v8i32_08080808:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX1-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -2131,8 +2131,8 @@ define <8 x i32> @shuffle_v8i32_08192a3b(<8 x i32> %a, <8 x i32> %b) {
 define <8 x i32> @shuffle_v8i32_08991abb(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-LABEL: shuffle_v8i32_08991abb:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[1,1]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[0,0,1,1]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
 ; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,2,3,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
@@ -3402,7 +3402,7 @@ define <8 x i32> @shuffle_v8i32_0dcd3f14(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-LABEL: shuffle_v8i32_0dcd3f14:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3]
+; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,1,1,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
index 07498c1233b5..db3be98efa53 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
@@ -355,7 +355,7 @@ define <32 x i16> @test_mm512_mask_blend_epi16(<32 x i16> %A, <32 x i16> %W){
 ;
 ; AVX512F-LABEL: test_mm512_mask_blend_epi16:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))
 ; AVX512F-NEXT:    ret{{[l|q]}}
 entry:
   %0 = shufflevector <32 x i16> %A, <32 x i16> %W, <32 x i32>  <i32 32, i32 1, i32 34, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 29, i32 62, i32 31>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
index 38920aa5d7a1..f4f4842e4c69 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
@@ -435,7 +435,7 @@ define <8 x float> @combine_pshufb_as_vzmovl_32(<8 x float> %a0) {
 ; CHECK-LABEL: combine_pshufb_as_vzmovl_32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = bitcast <8 x float> %a0 to <32 x i8>
   %2 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %1, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
index 68967c2ce653..c33776daf18f 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
@@ -964,41 +964,11 @@ define <8 x i64> @combine_vpermvar_insertion_as_broadcast_v8i64(i64 %a0) {
 }
 
 define <16 x i32> @blend_of_permutes_v16i32(<8 x i64> %a0, <8x i64> %a1) {
-; X86-AVX512F-LABEL: blend_of_permutes_v16i32:
-; X86-AVX512F:       # %bb.0:
-; X86-AVX512F-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
-; X86-AVX512F-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
-; X86-AVX512F-NEXT:    movw $-25958, %ax # imm = 0x9A9A
-; X86-AVX512F-NEXT:    kmovw %eax, %k1
-; X86-AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
-; X86-AVX512F-NEXT:    retl
-;
-; X86-AVX512BW-LABEL: blend_of_permutes_v16i32:
-; X86-AVX512BW:       # %bb.0:
-; X86-AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
-; X86-AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
-; X86-AVX512BW-NEXT:    movw $-25958, %ax # imm = 0x9A9A
-; X86-AVX512BW-NEXT:    kmovd %eax, %k1
-; X86-AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
-; X86-AVX512BW-NEXT:    retl
-;
-; X64-AVX512F-LABEL: blend_of_permutes_v16i32:
-; X64-AVX512F:       # %bb.0:
-; X64-AVX512F-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
-; X64-AVX512F-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
-; X64-AVX512F-NEXT:    movw $-25958, %ax # imm = 0x9A9A
-; X64-AVX512F-NEXT:    kmovw %eax, %k1
-; X64-AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: blend_of_permutes_v16i32:
-; X64-AVX512BW:       # %bb.0:
-; X64-AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
-; X64-AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
-; X64-AVX512BW-NEXT:    movw $-25958, %ax # imm = 0x9A9A
-; X64-AVX512BW-NEXT:    kmovd %eax, %k1
-; X64-AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
-; X64-AVX512BW-NEXT:    retq
+; CHECK-LABEL: blend_of_permutes_v16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [4,21,6,23,16,1,2,19,12,29,14,31,24,9,10,27]
+; CHECK-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %s0 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
   %s1 = shufflevector <8 x i64> %a1, <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
   %x0 = bitcast <8 x i64> %s0 to <16 x i32>
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vl.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vl.ll
index 1f4228b1fdec..d5aa7588925d 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vl.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vl.ll
@@ -45,3 +45,123 @@ define <4 x double> @concat_vpermv3_ops_vpermv_swap_v4f64(ptr %p0, <4 x i64> %m)
   %res = tail call noundef <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %lo, <4 x i64> %m, <4 x double> %hi)
   ret <4 x double> %res
 }
+
+define void @PR142995(ptr %p0, ptr %p1, ptr %p2) nounwind #0 {
+; X86-LABEL: PR142995:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movb $17, %bl
+; X86-NEXT:    kmovw %ebx, %k1
+; X86-NEXT:    vmovdqu32 (%edx), %ymm0 {%k1} {z}
+; X86-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT:    movw $6144, %dx # imm = 0x1800
+; X86-NEXT:    kmovw %edx, %k1
+; X86-NEXT:    vmovdqu32 128(%ecx), %zmm2 {%k1} {z}
+; X86-NEXT:    movw $1031, %dx # imm = 0x407
+; X86-NEXT:    kmovw %edx, %k1
+; X86-NEXT:    vmovdqu32 (%ecx), %zmm3 {%k1} {z}
+; X86-NEXT:    vpbroadcastd 252(%ecx), %zmm4
+; X86-NEXT:    vpbroadcastd %xmm1, %xmm5
+; X86-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm5[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero
+; X86-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X86-NEXT:    vpunpckldq {{.*#+}} zmm2 = zmm4[0],zmm2[0],zmm4[1],zmm2[1],zmm4[4],zmm2[4],zmm4[5],zmm2[5],zmm4[8],zmm2[8],zmm4[9],zmm2[9],zmm4[12],zmm2[12],zmm4[13],zmm2[13]
+; X86-NEXT:    vextracti32x4 $3, %zmm2, %xmm2
+; X86-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; X86-NEXT:    vpaddd %xmm1, %xmm5, %xmm1
+; X86-NEXT:    vmovdqu %xmm1, (%eax)
+; X86-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [4,0,10,0,4,4,14,0]
+; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X86-NEXT:    vpermi2d %ymm2, %ymm0, %ymm1
+; X86-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm4[1],ymm1[2,3,4,5,6,7]
+; X86-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6,7]
+; X86-NEXT:    vmovdqu %ymm0, (%eax)
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+;
+; X64-LABEL: PR142995:
+; X64:       # %bb.0:
+; X64-NEXT:    movb $17, %al
+; X64-NEXT:    kmovw %eax, %k1
+; X64-NEXT:    vmovdqu32 (%rdi), %ymm0 {%k1} {z}
+; X64-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X64-NEXT:    movw $6144, %ax # imm = 0x1800
+; X64-NEXT:    kmovw %eax, %k1
+; X64-NEXT:    vmovdqu32 128(%rsi), %zmm2 {%k1} {z}
+; X64-NEXT:    movw $1031, %ax # imm = 0x407
+; X64-NEXT:    kmovw %eax, %k1
+; X64-NEXT:    vmovdqu32 (%rsi), %zmm3 {%k1} {z}
+; X64-NEXT:    vpbroadcastd 252(%rsi), %zmm4
+; X64-NEXT:    vpbroadcastd %xmm1, %xmm5
+; X64-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm5[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero
+; X64-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X64-NEXT:    vpunpckldq {{.*#+}} zmm2 = zmm4[0],zmm2[0],zmm4[1],zmm2[1],zmm4[4],zmm2[4],zmm4[5],zmm2[5],zmm4[8],zmm2[8],zmm4[9],zmm2[9],zmm4[12],zmm2[12],zmm4[13],zmm2[13]
+; X64-NEXT:    vextracti32x4 $3, %zmm2, %xmm2
+; X64-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; X64-NEXT:    vpaddd %xmm1, %xmm5, %xmm1
+; X64-NEXT:    vmovdqu %xmm1, (%rax)
+; X64-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [4,0,10,0,4,4,14,0]
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vpermi2d %ymm2, %ymm0, %ymm1
+; X64-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm4[1],ymm1[2,3,4,5,6,7]
+; X64-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6,7]
+; X64-NEXT:    vmovdqu %ymm0, (%rdx)
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+  %i = tail call <5 x i32> @llvm.masked.load.v5i32.p0(ptr %p0, i32 4, <5 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true>, <5 x i32> poison)
+  %i1 = load <2 x i32>, ptr poison, align 4
+  %i2 = tail call <64 x i32> @llvm.masked.load.v64i32.p0(ptr %p1, i32 4, <64 x i1> <i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <64 x i32> poison)
+  %i3 = shufflevector <2 x i32> %i1, <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+  %i4 = shufflevector <4 x i32> zeroinitializer, <4 x i32> %i3, <4 x i32> <i32 poison, i32 poison, i32 4, i32 3>
+  %i5 = shufflevector <4 x i32> poison, <4 x i32> %i4, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %i6 = add <4 x i32> zeroinitializer, %i5
+  %i7 = shufflevector <2 x i32> %i1, <2 x i32> poison, <64 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %i8 = shufflevector <64 x i32> %i2, <64 x i32> %i7, <4 x i32> <i32 63, i32 44, i32 1, i32 65>
+  %i9 = add <4 x i32> %i6, %i8
+  %i10 = add <4 x i32> %i9, zeroinitializer
+  store <4 x i32> %i10, ptr poison, align 4
+  %i11 = shufflevector <5 x i32> %i, <5 x i32> poison, <6 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 4, i32 poison>
+  %i12 = shufflevector <6 x i32> %i11, <6 x i32> zeroinitializer, <8 x i32> <i32 4, i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 10, i32 0>
+  %i13 = shufflevector <8 x i32> %i12, <8 x i32> poison, <64 x i32> <i32 0, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %i14 = shufflevector <64 x i32> %i13, <64 x i32> %i2, <8 x i32> <i32 0, i32 127, i32 2, i32 poison, i32 poison, i32 poison, i32 6, i32 7>
+  %i15 = shufflevector <8 x i32> %i14, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 10, i32 poison, i32 poison, i32 6, i32 7>
+  %i16 = shufflevector <8 x i32> %i15, <8 x i32> poison, <19 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 poison, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %i17 = shufflevector <19 x i32> %i16, <19 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 19, i32 6, i32 7>
+  %i18 = add <8 x i32> zeroinitializer, %i17
+  %i19 = add <8 x i32> %i18, zeroinitializer
+  store <8 x i32> %i19, ptr %p2, align 4
+  ret void
+}
+declare <5 x i32> @llvm.masked.load.v5i32.p0(ptr captures(none), i32 immarg, <5 x i1>, <5 x i32>)
+declare <64 x i32> @llvm.masked.load.v64i32.p0(ptr captures(none), i32 immarg, <64 x i1>, <64 x i32>)
+
+define <8 x double> @PR143606(ptr %px, ptr %py) {
+; X86-LABEL: PR143606:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovapd (%ecx), %ymm0
+; X86-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0],mem[1,2],ymm0[3]
+; X86-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],mem[0],ymm0[2],mem[3]
+; X86-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: PR143606:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovapd (%rdi), %ymm0
+; X64-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0],mem[1,2],ymm0[3]
+; X64-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],mem[0],ymm0[2],mem[3]
+; X64-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+  %x = load <4 x double>, ptr %px, align 32
+  %y.lo = load <4 x double>, ptr %py, align 32
+  %py.hi = getelementptr inbounds nuw i8, ptr %py, i64 32
+  %y.hi = load <4 x double>, ptr %py.hi, align 32
+  %lo = shufflevector <4 x double> %x, <4 x double> %y.lo, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %hi = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x, <4 x i64> <i64 1, i64 4, i64 2, i64 7>, <4 x double> %y.hi)
+  %res = shufflevector <4 x double> %lo, <4 x double> %hi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x double> %res
+}
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
index 12d494c32b65..bd2710139d58 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
@@ -53,7 +53,7 @@ define <2 x double> @combine_pshufb_as_movsd(<2 x double> %a0, <2 x double> %a1)
 ;
 ; AVX-LABEL: combine_pshufb_as_movsd:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
   %1 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 3, i32 0>
   %2 = bitcast <2 x double> %1 to <16 x i8>
@@ -63,19 +63,14 @@ define <2 x double> @combine_pshufb_as_movsd(<2 x double> %a0, <2 x double> %a1)
 }
 
 define <4 x float> @combine_pshufb_as_movss(<4 x float> %a0, <4 x float> %a1) {
-; SSSE3-LABEL: combine_pshufb_as_movss:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: combine_pshufb_as_movss:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: combine_pshufb_as_movss:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_pshufb_as_movss:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 3, i32 2, i32 1>
   %2 = bitcast <4 x float> %1 to <16 x i8>
@@ -137,7 +132,7 @@ define <4 x float> @combine_pshufb_as_vzmovl_32(<4 x float> %a0) {
 ; AVX-LABEL: combine_pshufb_as_vzmovl_32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %1 = bitcast <4 x float> %a0 to <16 x i8>
   %2 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %1, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll
index f53c7a337017..e8bf5ec2b49a 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll
@@ -106,7 +106,7 @@ define <4 x float> @combine_vpermil2ps_blend_with_zero(<4 x float> %a0, <4 x flo
 ; CHECK-LABEL: combine_vpermil2ps_blend_with_zero:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 8, i32 1, i32 2, i32 3>, i8 2)
   ret <4 x float> %res0
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll
index 2812bf348910..925f8d510451 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll
@@ -173,7 +173,7 @@ define void @concat_a_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movaps (%rdi), %xmm0
 ; SSE42-NEXT:    movaps (%rsi), %xmm1
-; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3]
+; SSE42-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
 ; SSE42-NEXT:    movaps %xmm0, 16(%rdx)
 ; SSE42-NEXT:    movaps %xmm1, (%rdx)
 ; SSE42-NEXT:    retq
@@ -288,7 +288,7 @@ define void @concat_shuf_of_ab_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movaps (%rdi), %xmm0
 ; SSE42-NEXT:    movaps (%rsi), %xmm1
-; SSE42-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3]
+; SSE42-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
 ; SSE42-NEXT:    movaps %xmm1, 16(%rdx)
 ; SSE42-NEXT:    movaps %xmm0, (%rdx)
 ; SSE42-NEXT:    retq
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vector-zmov.ll b/external/llvm-project/llvm/test/CodeGen/X86/vector-zmov.ll
index 09835d797d17..9d84ff8c01ab 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vector-zmov.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vector-zmov.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX
 
@@ -38,32 +38,18 @@ entry:
 }
 
 define <4 x i32> @load_zmov_4i32_to_0zzz_volatile(ptr%ptr) {
-; SSE2-LABEL: load_zmov_4i32_to_0zzz_volatile:
-; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movaps (%rdi), %xmm1
-; SSE2-NEXT:    xorps %xmm0, %xmm0
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: load_zmov_4i32_to_0zzz_volatile:
-; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movaps (%rdi), %xmm1
-; SSSE3-NEXT:    xorps %xmm0, %xmm0
-; SSSE3-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: load_zmov_4i32_to_0zzz_volatile:
-; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    movaps (%rdi), %xmm1
-; SSE41-NEXT:    xorps %xmm0, %xmm0
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: load_zmov_4i32_to_0zzz_volatile:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movaps (%rdi), %xmm1
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: load_zmov_4i32_to_0zzz_volatile:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
 entry:
   %X = load volatile <4 x i32>, ptr %ptr
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vselect-2.ll b/external/llvm-project/llvm/test/CodeGen/X86/vselect-2.ll
index c02cbcf55408..429ae88fe6d6 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vselect-2.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vselect-2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX
 
 define <4 x i32> @test1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test1:
@@ -24,15 +24,10 @@ define <4 x i32> @test1(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define <4 x i32> @test2(<4 x i32> %A, <4 x i32> %B) {
-; SSE2-LABEL: test2:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test2:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test2:
 ; AVX:       # %bb.0:
@@ -55,26 +50,21 @@ define <4 x float> @test3(<4 x float> %A, <4 x float> %B) {
 ;
 ; AVX-LABEL: test3:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
   %select = select <4 x i1><i1 true, i1 true, i1 false, i1 false>, <4 x float> %A, <4 x float> %B
   ret <4 x float> %select
 }
 
 define <4 x float> @test4(<4 x float> %A, <4 x float> %B) {
-; SSE2-LABEL: test4:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test4:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test4:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test4:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
   %select = select <4 x i1><i1 false, i1 false, i1 true, i1 true>, <4 x float> %A, <4 x float> %B
   ret <4 x float> %select
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vselect-pcmp.ll b/external/llvm-project/llvm/test/CodeGen/X86/vselect-pcmp.ll
index ab487ed88898..8543e9fd919b 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vselect-pcmp.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vselect-pcmp.ll
@@ -1763,7 +1763,7 @@ define <64 x i8> @PR110875(<32 x i8> %a0, <32 x i8> %a1, i64 %a2) {
 ; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = mem ^ (zmm0 & (zmm1 ^ mem))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = m32bcst ^ (zmm0 & (zmm1 ^ m32bcst))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: PR110875:
@@ -1780,7 +1780,7 @@ define <64 x i8> @PR110875(<32 x i8> %a0, <32 x i8> %a1, i64 %a2) {
 ; AVX512VL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = mem ^ (zmm0 & (zmm1 ^ mem))
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 = m32bcst ^ (zmm0 & (zmm1 ^ m32bcst))
 ; AVX512VL-NEXT:    retq
 ;
 ; XOP-LABEL: PR110875:
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/vselect.ll b/external/llvm-project/llvm/test/CodeGen/X86/vselect.ll
index be6ee8f68995..f70145d6b21c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/vselect.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/vselect.ll
@@ -69,26 +69,21 @@ define <4 x float> @test2(<4 x float> %a, <4 x float> %b) {
 ;
 ; AVX-LABEL: test2:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
   %1 = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x float> %a, <4 x float> %b
   ret <4 x float> %1
 }
 
 define <4 x float> @test3(<4 x float> %a, <4 x float> %b) {
-; SSE2-LABEL: test3:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test3:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test3:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test3:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
   %1 = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x float> %a, <4 x float> %b
   ret <4 x float> %1
@@ -152,15 +147,10 @@ define <8 x i16> @test7(<8 x i16> %a, <8 x i16> %b) {
 }
 
 define <8 x i16> @test8(<8 x i16> %a, <8 x i16> %b) {
-; SSE2-LABEL: test8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test8:
 ; AVX:       # %bb.0:
@@ -301,72 +291,52 @@ define <8 x i16> @test17(<8 x i16> %a, <8 x i16> %b) {
 }
 
 define <4 x float> @test18(<4 x float> %a, <4 x float> %b) {
-; SSE2-LABEL: test18:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test18:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test18:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test18:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %1 = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> %a, <4 x float> %b
   ret <4 x float> %1
 }
 
 define <4 x i32> @test19(<4 x i32> %a, <4 x i32> %b) {
-; SSE2-LABEL: test19:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test19:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test19:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test19:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX-NEXT:    retq
   %1 = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %1
 }
 
 define <2 x double> @test20(<2 x double> %a, <2 x double> %b) {
-; SSE2-LABEL: test20:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test20:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test20:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test20:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; AVX-NEXT:    retq
   %1 = select <2 x i1> <i1 false, i1 true>, <2 x double> %a, <2 x double> %b
   ret <2 x double> %1
 }
 
 define <2 x i64> @test21(<2 x i64> %a, <2 x i64> %b) {
-; SSE2-LABEL: test21:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test21:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: test21:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test21:
 ; AVX:       # %bb.0:
@@ -390,7 +360,7 @@ define <4 x float> @test22(<4 x float> %a, <4 x float> %b) {
 ;
 ; AVX-LABEL: test22:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %1 = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x float> %a, <4 x float> %b
   ret <4 x float> %1
@@ -410,7 +380,7 @@ define <4 x i32> @test23(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; AVX-LABEL: test23:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
   %1 = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %1
@@ -429,7 +399,7 @@ define <2 x double> @test24(<2 x double> %a, <2 x double> %b) {
 ;
 ; AVX-LABEL: test24:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
 ; AVX-NEXT:    retq
   %1 = select <2 x i1> <i1 true, i1 false>, <2 x double> %a, <2 x double> %b
   ret <2 x double> %1
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/windows-seh-EHa-TryInFinally.ll b/external/llvm-project/llvm/test/CodeGen/X86/windows-seh-EHa-TryInFinally.ll
index 16322cbe9980..9e44299083d4 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/windows-seh-EHa-TryInFinally.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/windows-seh-EHa-TryInFinally.ll
@@ -2,7 +2,7 @@
 
 ; CHECK-LABEL: "?fin$0@0@main@@"
 ; CHECK:      .seh_handlerdata
-; CHECK:      .set ".L?fin$0@0@main@@$parent_frame_offset", 48
+; CHECK:      ".L?fin$0@0@main@@$parent_frame_offset" = 48
 ; CHECK-NEXT:        .long   (.Llsda_end1-.Llsda_begin1)/16
 ; CHECK-NEXT: .Llsda_begin1:
 ; CHECK-NEXT:        .long   .Ltmp
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/external/llvm-project/llvm/test/CodeGen/X86/x86-interleaved-access.ll
index d2c64a462a3e..7cddebdca5cc 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/x86-interleaved-access.ll
@@ -962,16 +962,15 @@ define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX512-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vmovdqu %xmm0, 16(%rdi)
+; AVX512-NEXT:    vmovdqu %xmm1, (%rdi)
 ; AVX512-NEXT:    vmovdqu %xmm2, 32(%rdi)
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqu %ymm0, (%rdi)
-; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 %1 = shufflevector <16 x i8> %a, <16 x i8> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 %2 = shufflevector <16 x i8> %c, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1632,12 +1631,12 @@ ret void
 define void @splat2_v4f64_load_store(ptr %s, ptr %d) nounwind {
 ; AVX1-LABEL: splat2_v4f64_load_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vperm2f128 $51, (%rdi), %ymm0, %ymm0 # ymm0 = mem[2,3,2,3]
+; AVX1-NEXT:    vbroadcastf128 (%rdi), %ymm0 # ymm0 = mem[0,1,0,1]
 ; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0,0,3,3]
-; AVX1-NEXT:    vbroadcastf128 (%rdi), %ymm1 # ymm1 = mem[0,1,0,1]
+; AVX1-NEXT:    vbroadcastf128 16(%rdi), %ymm1 # ymm1 = mem[0,1,0,1]
 ; AVX1-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
-; AVX1-NEXT:    vmovupd %ymm0, 32(%rsi)
-; AVX1-NEXT:    vmovupd %ymm1, (%rsi)
+; AVX1-NEXT:    vmovupd %ymm1, 32(%rsi)
+; AVX1-NEXT:    vmovupd %ymm0, (%rsi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1669,12 +1668,12 @@ define void @splat2_v4f64_load_store(ptr %s, ptr %d) nounwind {
 define void @splat2_v4i64_load_store(ptr %s, ptr %d) nounwind {
 ; AVX1-LABEL: splat2_v4i64_load_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vperm2f128 $51, (%rdi), %ymm0, %ymm0 # ymm0 = mem[2,3,2,3]
+; AVX1-NEXT:    vbroadcastf128 (%rdi), %ymm0 # ymm0 = mem[0,1,0,1]
 ; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0,0,3,3]
-; AVX1-NEXT:    vbroadcastf128 (%rdi), %ymm1 # ymm1 = mem[0,1,0,1]
+; AVX1-NEXT:    vbroadcastf128 16(%rdi), %ymm1 # ymm1 = mem[0,1,0,1]
 ; AVX1-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,3]
-; AVX1-NEXT:    vmovupd %ymm0, 32(%rsi)
-; AVX1-NEXT:    vmovupd %ymm1, (%rsi)
+; AVX1-NEXT:    vmovupd %ymm1, 32(%rsi)
+; AVX1-NEXT:    vmovupd %ymm0, (%rsi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll b/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
index ddd7f1016893..cacc43e96b6e 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
@@ -1329,7 +1329,7 @@ define void @vec128_v4i32_to_v1i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi
 ; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX2-NEXT:    vzeroupper
@@ -1340,7 +1340,7 @@ define void @vec128_v4i32_to_v1i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1351,7 +1351,7 @@ define void @vec128_v4i32_to_v1i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
@@ -2428,7 +2428,7 @@ define void @vec256_v8i32_to_v1i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bi
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX2-NEXT:    vzeroupper
@@ -2439,7 +2439,7 @@ define void @vec256_v8i32_to_v1i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bi
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -2450,7 +2450,7 @@ define void @vec256_v8i32_to_v1i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bi
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
@@ -4996,7 +4996,7 @@ define void @vec384_v12i32_to_v3i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b
 ; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX2-SLOW-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3]
 ; AVX2-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]
 ; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]
@@ -5063,7 +5063,7 @@ define void @vec384_v12i32_to_v3i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b
 ; AVX512BW-SLOW-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} {z}
 ; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -5282,7 +5282,7 @@ define void @vec384_v12i32_to_v1i384_factor12(ptr %in.vec.base.ptr, ptr %in.vec.
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vmovaps 32(%rdx), %ymm1
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)
@@ -5295,7 +5295,7 @@ define void @vec384_v12i32_to_v1i384_factor12(ptr %in.vec.base.ptr, ptr %in.vec.
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm1
 ; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)
@@ -5308,7 +5308,7 @@ define void @vec384_v12i32_to_v1i384_factor12(ptr %in.vec.base.ptr, ptr %in.vec.
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
@@ -7347,9 +7347,9 @@ define void @vec512_v16i32_to_v2i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
 ; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm2, %ymm1
 ; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)
@@ -7362,7 +7362,7 @@ define void @vec512_v16i32_to_v2i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
 ; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
@@ -7376,7 +7376,7 @@ define void @vec512_v16i32_to_v2i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
 ; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
@@ -7405,9 +7405,9 @@ define void @vec512_v16i32_to_v2i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
 ; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512BW-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX512BW-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -7419,7 +7419,7 @@ define void @vec512_v16i32_to_v2i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
 ; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512BW-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-FAST-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
@@ -7491,7 +7491,7 @@ define void @vec512_v16i32_to_v1i512_factor16(ptr %in.vec.base.ptr, ptr %in.vec.
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vmovaps 32(%rdx), %ymm1
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)
@@ -7504,7 +7504,7 @@ define void @vec512_v16i32_to_v1i512_factor16(ptr %in.vec.base.ptr, ptr %in.vec.
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm1
 ; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)
@@ -7517,7 +7517,7 @@ define void @vec512_v16i32_to_v1i512_factor16(ptr %in.vec.base.ptr, ptr %in.vec.
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index 7fbb211b69cc..572ed314ab31 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -1235,7 +1235,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.v
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1248,7 +1248,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.v
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1359,7 +1359,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.v
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1372,7 +1372,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.v
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1952,7 +1952,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX2-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX2-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -1965,7 +1965,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512F-SLOW-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512F-SLOW-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
 ; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -1991,7 +1991,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512DQ-SLOW-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
 ; AVX512DQ-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -2016,7 +2016,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-SLOW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-SLOW-NEXT:    vpbroadcastq %xmm0, %ymm0
+; AVX512BW-SLOW-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -3567,14 +3567,13 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm1
 ; AVX-NEXT:    vpaddb 32(%rdx), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero
 ; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm0
-; AVX-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)
+; AVX-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX-NEXT:    vmovdqa %xmm2, 32(%rcx)
 ; AVX-NEXT:    retq
 ;
@@ -3595,41 +3594,73 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
-; AVX512F-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
-; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
+; AVX512F-SLOW-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
+; AVX512F-SLOW:       # %bb.0:
+; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
+; AVX512F-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-SLOW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
+; AVX512F-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)
+; AVX512F-SLOW-NEXT:    vzeroupper
+; AVX512F-SLOW-NEXT:    retq
 ;
-; AVX512DQ-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
-; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
-; AVX512DQ-NEXT:    vmovdqa %ymm1, 32(%rcx)
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
+; AVX512F-FAST-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
+; AVX512F-FAST:       # %bb.0:
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-FAST-NEXT:    vpbroadcastw %xmm0, %ymm2
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
+; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero
+; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
+; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)
+; AVX512F-FAST-NEXT:    vmovdqa %ymm1, (%rcx)
+; AVX512F-FAST-NEXT:    vzeroupper
+; AVX512F-FAST-NEXT:    retq
+;
+; AVX512DQ-SLOW-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
+; AVX512DQ-SLOW:       # %bb.0:
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
+; AVX512DQ-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512DQ-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)
+; AVX512DQ-SLOW-NEXT:    vzeroupper
+; AVX512DQ-SLOW-NEXT:    retq
+;
+; AVX512DQ-FAST-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
+; AVX512DQ-FAST:       # %bb.0:
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-FAST-NEXT:    vpbroadcastw %xmm0, %ymm2
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
+; AVX512DQ-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero
+; AVX512DQ-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
+; AVX512DQ-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm1, (%rcx)
+; AVX512DQ-FAST-NEXT:    vzeroupper
+; AVX512DQ-FAST-NEXT:    retq
 ;
 ; AVX512BW-SLOW-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512BW-SLOW:       # %bb.0:
@@ -3725,14 +3756,14 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
 ; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm1
-; AVX-NEXT:    vpaddb 32(%rdx), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpaddb 32(%rdx), %xmm3, %xmm3
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
 ; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)
-; AVX-NEXT:    vmovdqa %xmm2, 32(%rcx)
+; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)
 ; AVX-NEXT:    retq
 ;
 ; AVX2-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
@@ -3757,10 +3788,10 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
 ;
 ; AVX512F-SLOW-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
 ; AVX512F-SLOW:       # %bb.0:
-; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
 ; AVX512F-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -3796,10 +3827,10 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
 ;
 ; AVX512DQ-SLOW-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
 ; AVX512DQ-SLOW:       # %bb.0:
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
 ; AVX512DQ-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -3923,10 +3954,9 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
 ; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm1
-; AVX-NEXT:    vpaddb 32(%rdx), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm2
 ; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)
@@ -3987,11 +4017,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-SLOW-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F-SLOW:       # %bb.0:
-; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
 ; AVX512F-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4006,29 +4035,27 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F-FAST:       # %bb.0:
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %ymm0
-; AVX512F-FAST-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm1
-; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-FAST-NEXT:    vpbroadcastw %xmm1, %ymm2
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3],ymm2[4],ymm0[5,6,7],ymm2[8],ymm0[9,10,11],ymm2[12],ymm0[13,14,15]
-; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1],zero,zero,zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero,zero,zero
-; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm1
-; AVX512F-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)
-; AVX512F-FAST-NEXT:    vmovdqa %ymm0, (%rcx)
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512F-FAST-NEXT:    vpbroadcastw %xmm0, %ymm2
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
+; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero
+; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
+; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)
+; AVX512F-FAST-NEXT:    vmovdqa %ymm1, (%rcx)
 ; AVX512F-FAST-NEXT:    vzeroupper
 ; AVX512F-FAST-NEXT:    retq
 ;
 ; AVX512DQ-SLOW-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ-SLOW:       # %bb.0:
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
 ; AVX512DQ-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4043,19 +4070,18 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512DQ-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-FAST-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %xmm1
-; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT:    vpbroadcastw %xmm1, %ymm2
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3],ymm2[4],ymm0[5,6,7],ymm2[8],ymm0[9,10,11],ymm2[12],ymm0[13,14,15]
-; AVX512DQ-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1],zero,zero,zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, (%rcx)
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512DQ-FAST-NEXT:    vpbroadcastw %xmm0, %ymm2
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
+; AVX512DQ-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
+; AVX512DQ-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm1, (%rcx)
 ; AVX512DQ-FAST-NEXT:    vzeroupper
 ; AVX512DQ-FAST-NEXT:    retq
 ;
@@ -4153,17 +4179,16 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3,4,5],xmm2[6],xmm1[7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1]
 ; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4,5,6,7]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4,5,6,7]
 ; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm1
-; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm0
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm2, %xmm2
-; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm2
+; AVX-NEXT:    vpaddb 32(%rdx), %xmm2, %xmm2
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)
 ; AVX-NEXT:    vmovdqa %xmm1, (%rcx)
-; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)
-; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)
+; AVX-NEXT:    vmovdqa %xmm2, 32(%rcx)
 ; AVX-NEXT:    retq
 ;
 ; AVX2-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
@@ -4188,10 +4213,10 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
 ;
 ; AVX512F-SLOW-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
 ; AVX512F-SLOW:       # %bb.0:
-; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
 ; AVX512F-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -4227,10 +4252,10 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
 ;
 ; AVX512DQ-SLOW-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
 ; AVX512DQ-SLOW:       # %bb.0:
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vpbroadcastw %xmm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
 ; AVX512DQ-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -4351,10 +4376,9 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i
 ; AVX-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3,4,5,6,7]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
 ; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm1
-; AVX-NEXT:    vpaddb 32(%rdx), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm2
 ; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm1, (%rcx)
 ; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)
@@ -4489,10 +4513,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX-NEXT:    vmovaps 32(%rdx), %ymm2
 ; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm1
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero
 ; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm0
 ; AVX-NEXT:    vmovaps %ymm2, 32(%rcx)
 ; AVX-NEXT:    vmovdqa %xmm1, (%rcx)
@@ -4649,11 +4672,10 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX2-SLOW-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX2-SLOW-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpbroadcastq %xmm0, %ymm2
-; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %ymm0
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6],ymm2[7]
-; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
 ; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
@@ -4669,7 +4691,7 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX2-FAST-PERLANE-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX2-FAST-PERLANE-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
-; AVX2-FAST-PERLANE-NEXT:    vpbroadcastq %xmm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vpbroadcastd %xmm0, %ymm2
 ; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6],ymm2[7]
@@ -4687,7 +4709,7 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX2-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX2-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpbroadcastq %xmm0, %ymm2
+; AVX2-FAST-NEXT:    vpbroadcastd %xmm0, %ymm2
 ; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4,5,6,7]
 ; AVX2-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6],ymm2[7]
@@ -4853,7 +4875,7 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5],ymm1[6],ymm2[7]
 ; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
 ; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
@@ -5046,7 +5068,7 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)
@@ -6582,7 +6604,7 @@ define void @vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8(ptr %in.
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX2-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
 ; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm1
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
@@ -6825,7 +6847,7 @@ define void @vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2(ptr %i
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm1
 ; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)
diff --git a/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll b/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
index bd6cbbaadc95..239472c5cd1c 100644
--- a/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/external/llvm-project/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -1020,7 +1020,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.e
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1030,7 +1030,7 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.e
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm1 = ymm1 ^ (m32bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1116,7 +1116,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.e
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512F-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512F-NEXT:    vzeroupper
@@ -1126,7 +1126,7 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.e
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm0
 ; AVX512DQ-NEXT:    vpbroadcastb (%rdi), %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (m64bcst & (ymm1 ^ ymm0))
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rdx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -1569,7 +1569,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ;
 ; AVX512F-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastq (%rdi), %ymm0
+; AVX512F-NEXT:    vpbroadcastd (%rdi), %ymm0
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rdx)
@@ -1578,7 +1578,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ;
 ; AVX512DQ-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastq (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpbroadcastd (%rdi), %ymm0
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rdx)
@@ -1587,7 +1587,7 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ;
 ; AVX512BW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastq (%rdi), %ymm0
+; AVX512BW-NEXT:    vpbroadcastd (%rdi), %ymm0
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rdx)
@@ -1884,17 +1884,16 @@ define void @vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24(ptr %in
 ;
 ; AVX2-LABEL: vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero
-; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,u,0,u,0,u,0,u,0,u,0,u,0,u,0,u,16],zero,ymm1[16],zero,ymm1[16],zero,ymm1[16],zero,ymm1[16],zero,ymm1[16],zero,ymm1[16],zero,ymm1[16],zero
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm0[0,u,0,u,0,u,0,u,0,u,0,u,0,u,0,u,16],zero,ymm0[16],zero,ymm0[16],zero,ymm0[16],zero,ymm0[16],zero,ymm0[16],zero,ymm0[16],zero,ymm0[16],zero
 ; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = mem[2,3],ymm1[2,3]
 ; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm3 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; AVX2-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero
 ; AVX2-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
+; AVX2-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rdx)
+; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2133,17 +2132,16 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in
 ;
 ; AVX2-LABEL: vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
-; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,u,u,u,0,u,u,u,0,u,u,u,0,u,u,u,16],zero,zero,zero,ymm1[16],zero,zero,zero,ymm1[16],zero,zero,zero,ymm1[16],zero,zero,zero
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm0[0,u,u,u,0,u,u,u,0,u,u,u,0,u,u,u,16],zero,zero,zero,ymm0[16],zero,zero,zero,ymm0[16],zero,zero,zero,ymm0[16],zero,zero,zero
 ; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = mem[2,3],ymm1[2,3]
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255]
 ; AVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero
 ; AVX2-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
+; AVX2-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rdx)
+; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2384,17 +2382,16 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.e
 ;
 ; AVX2-LABEL: vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,u,u,u,u,u,u,u,0,u,u,u,u,u,u,u,16],zero,zero,zero,zero,zero,zero,zero,ymm1[16],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm0[0,u,u,u,u,u,u,u,0,u,u,u,u,u,u,u,16],zero,zero,zero,zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = mem[2,3],ymm1[2,3]
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
+; AVX2-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rdx)
+; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2636,15 +2633,15 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm1 = [255,0,255,0]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = mem[2,3],ymm0[2,3]
-; AVX2-NEXT:    vpmovsxwq {{.*#+}} ymm3 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpand (%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
-; AVX2-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa %ymm1, 32(%rdx)
-; AVX2-NEXT:    vmovdqa %ymm0, (%rdx)
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm3 = mem[2,3],ymm2[2,3]
+; AVX2-NEXT:    vpmovsxwq {{.*#+}} ymm4 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
+; AVX2-NEXT:    vpblendvb %ymm4, %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddb (%rsi), %ymm2, %ymm1
+; AVX2-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm1, (%rdx)
+; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2871,14 +2868,13 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX-NEXT:    vpaddb 32(%rsi), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero
+; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, 16(%rdx)
+; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
 ; AVX-NEXT:    vmovdqa %xmm2, 32(%rdx)
 ; AVX-NEXT:    retq
 ;
@@ -2989,7 +2985,8 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,xmm2[0,1],zero,zero,zero,zero,xmm2[0,1],zero,zero,zero,zero
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1],xmm3[2,3],xmm0[4],xmm3[5,6],xmm0[7]
 ; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vpaddb 32(%rsi), %xmm2, %xmm2
@@ -3138,9 +3135,8 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = mem[0,1,0,1]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1,2,3],xmm0[4],mem[5,6,7]
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
-; AVX-NEXT:    vpaddb 32(%rsi), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
+; AVX-NEXT:    vpaddb 32(%rsi), %xmm0, %xmm2
 ; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
@@ -3322,13 +3318,12 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1]
 ; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4,5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX-NEXT:    vpaddb 32(%rsi), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, 16(%rdx)
+; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
 ; AVX-NEXT:    vmovdqa %xmm2, 32(%rdx)
 ; AVX-NEXT:    retq
 ;
@@ -3472,9 +3467,8 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1,2,3,4,5,6,7]
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3,4,5,6,7]
-; AVX-NEXT:    vpaddb 32(%rsi), %xmm2, %xmm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
+; AVX-NEXT:    vpaddb 32(%rsi), %xmm0, %xmm2
 ; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
@@ -3484,16 +3478,16 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i
 ;
 ; AVX2-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = mem[0],xmm0[1,2,3,4,5,6,7]
-; AVX2-NEXT:    vmovdqa 48(%rdi), %xmm1
-; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = mem[0,1,0,1]
-; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15]
-; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX2-NEXT:    vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdx)
-; AVX2-NEXT:    vmovdqa %ymm1, (%rdx)
+; AVX2-NEXT:    vmovdqa 48(%rdi), %xmm0
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]
+; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
+; AVX2-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm0, (%rdx)
+; AVX2-NEXT:    vmovdqa %ymm1, 32(%rdx)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3587,9 +3581,8 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1,2,3,4,5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX-NEXT:    vmovaps 32(%rsi), %ymm2
-; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero
 ; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vmovaps %ymm2, 32(%rdx)
@@ -3849,19 +3842,19 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],mem[2,3,4,5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3,4,5],xmm1[6,7]
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
-; AVX-NEXT:    vbroadcastss (%rdi), %xmm3
-; AVX-NEXT:    vpaddb 32(%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,0,1,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX-NEXT:    vpaddb 32(%rsi), %xmm3, %xmm3
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
+; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
-; AVX-NEXT:    vpaddb 16(%rsi), %xmm2, %xmm2
-; AVX-NEXT:    vmovdqa %xmm2, 16(%rdx)
 ; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
-; AVX-NEXT:    vmovdqa %xmm0, 32(%rdx)
+; AVX-NEXT:    vmovdqa %xmm0, 16(%rdx)
+; AVX-NEXT:    vmovdqa %xmm3, 32(%rdx)
 ; AVX-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4:
@@ -4105,17 +4098,17 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i
 ;
 ; AVX-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa 48(%rdi), %xmm0
-; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = mem[0,1],xmm0[2,3,4,5,6,7]
-; AVX-NEXT:    vbroadcastss (%rdi), %xmm1
+; AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],mem[2,3,4,5,6,7]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX-NEXT:    vmovaps 32(%rsi), %ymm2
-; AVX-NEXT:    vxorps %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3]
-; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm1
-; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5],xmm3[6,7]
+; AVX-NEXT:    vpaddb 16(%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vpaddb (%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vmovaps %ymm2, 32(%rdx)
-; AVX-NEXT:    vmovdqa %xmm0, (%rdx)
-; AVX-NEXT:    vmovdqa %xmm1, 16(%rdx)
+; AVX-NEXT:    vmovdqa %xmm1, (%rdx)
+; AVX-NEXT:    vmovdqa %xmm0, 16(%rdx)
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
diff --git a/external/llvm-project/llvm/test/CodeGen/XCore/globals.ll b/external/llvm-project/llvm/test/CodeGen/XCore/globals.ll
index 134bbb3444b5..186cfda97104 100644
--- a/external/llvm-project/llvm/test/CodeGen/XCore/globals.ll
+++ b/external/llvm-project/llvm/test/CodeGen/XCore/globals.ll
@@ -127,4 +127,4 @@ entry:
 
 @array = global [10 x i16] zeroinitializer, align 2
 ; CHECK: .globl  array.globound
-; CHECK: .set array.globound, 10
+; CHECK: array.globound = 10
diff --git a/external/llvm-project/llvm/test/CodeGen/XCore/linkage.ll b/external/llvm-project/llvm/test/CodeGen/XCore/linkage.ll
index 93edf01cf8a9..5bfb83d964df 100644
--- a/external/llvm-project/llvm/test/CodeGen/XCore/linkage.ll
+++ b/external/llvm-project/llvm/test/CodeGen/XCore/linkage.ll
@@ -19,14 +19,14 @@ define protected void @test_protected() {
 }
 
 ; CHECK: .globl array.globound
-; CHECK: .set array.globound, 2
+; CHECK: array.globound = 2
 ; CHECK: .weak array.globound
 ; CHECK: .globl array
 ; CHECK: .weak array
 @array = weak global [2 x i32] zeroinitializer
 
 ; CHECK: .globl ac.globound
-; CHECK: .set ac.globound, 2
+; CHECK: ac.globound = 2
 ; CHECK: .weak ac.globound
 ; CHECK: .globl ac
 ; CHECK: .weak ac
diff --git a/external/llvm-project/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-cfi-directives.s b/external/llvm-project/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-cfi-directives.s
index d742cfc49689..7a6844b06df5 100644
--- a/external/llvm-project/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-cfi-directives.s
+++ b/external/llvm-project/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-cfi-directives.s
@@ -39,7 +39,7 @@ vector_registers_single:
 vector_offsets:
   .cfi_startproc
   s_nop 2
-  ; CHECK: DW_CFA_expression: VGPR40 DW_OP_regx VGPR40, DW_OP_swap, DW_OP_LLVM_user DW_OP_LLVM_offset_uconst 0x100, DW_OP_LLVM_user DW_OP_LLVM_call_frame_entry_reg EXEC, DW_OP_deref_size 0x8, DW_OP_LLVM_user DW_OP_LLVM_select_bit_piece 0x20 0x40
+  ; CHECK: DW_CFA_expression: VGPR40 DW_OP_regx VGPR40, DW_OP_swap, DW_OP_LLVM_user DW_OP_LLVM_offset_uconst 0x100, DW_OP_LLVM_user DW_OP_LLVM_call_frame_entry_reg 0x11, DW_OP_deref_size 0x8, DW_OP_LLVM_user DW_OP_LLVM_select_bit_piece 0x20 0x40
   .cfi_llvm_vector_offset 2600, 32, 17, 64, 256
   s_nop 2
   .cfi_endproc
@@ -49,7 +49,7 @@ vector_offsets:
 vector_register_mask:
   .cfi_startproc
   s_nop 0
-  ; CHECK: DW_CFA_expression: VGPR40 DW_OP_regx VGPR40, DW_OP_regx AGPR0, DW_OP_LLVM_user DW_OP_LLVM_call_frame_entry_reg EXEC, DW_OP_deref_size 0x8, DW_OP_LLVM_user DW_OP_LLVM_select_bit_piece 0x20 0x40
+  ; CHECK: DW_CFA_expression: VGPR40 DW_OP_regx VGPR40, DW_OP_regx AGPR0, DW_OP_LLVM_user DW_OP_LLVM_call_frame_entry_reg 0x11, DW_OP_deref_size 0x8, DW_OP_LLVM_user DW_OP_LLVM_select_bit_piece 0x20 0x40
   .cfi_llvm_vector_register_mask 2600, 3072, 32, 17, 64
   s_nop 0
   .cfi_endproc
diff --git a/external/llvm-project/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-diop-diexpression-address-spaces.ll b/external/llvm-project/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-diop-diexpression-address-spaces.ll
new file mode 100644
index 000000000000..a8e900a7e91c
--- /dev/null
+++ b/external/llvm-project/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-diop-diexpression-address-spaces.ll
@@ -0,0 +1,162 @@
+; RUN: llc -O0 -mcpu=gfx1030 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-dwarfdump --debug-info - | FileCheck %s
+
+; CHECK-LABEL: DW_AT_name ("test_loc_single")
+define void @test_loc_single(ptr addrspace(3) %ptr) #0 !dbg !9 {
+  ; Verify that the right address class attribute is attached to the variable's
+  ; type for a single location:
+  ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+  ; CHECK-NEXT: DW_AT_location (DW_OP_regx {{.*}}, DW_OP_LLVM_user DW_OP_LLVM_push_lane, DW_OP_lit4, DW_OP_mul, DW_OP_LLVM_user DW_OP_LLVM_offset)
+  ; CHECK-NEXT: DW_AT_name ("loc_single_ptr")
+  ; CHECK-NEXT: DW_AT_decl_file
+  ; CHECK-NEXT: DW_AT_decl_line
+  ; CHECK-NEXT: DW_AT_type ([[PTR_AS_3:0x[0-9a-f]+]] "int *")
+
+    #dbg_value(ptr addrspace(3) %ptr, !13, !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr)), !16)
+  ret void, !dbg !17
+}
+
+; CHECK-LABEL: DW_AT_name ("test_loc_multi")
+define void @test_loc_multi(ptr addrspace(3) %loc_ptr) #0 !dbg !18 {
+  ; Verify that no attribute is attached to the variable type if the loclist
+  ; contains entries with different address spaces:
+  ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+  ; CHECK-NEXT:   DW_AT_location (indexed ({{0x[0-9a-f]+}}) loclist =
+  ; CHECK-NEXT:      [{{0x[0-9a-f]+}}, {{0x[0-9a-f]+}}):{{.*}} DW_OP_LLVM_user DW_OP_LLVM_undefined
+  ; CHECK-NEXT:      [{{0x[0-9a-f]+}}, {{0x[0-9a-f]+}}): DW_OP_lit0, DW_OP_stack_value)
+  ; CHECK-NEXT:   DW_AT_name ("ptr_as3_as2")
+  ; CHECK-NEXT:   DW_AT_decl_file
+  ; CHECK-NEXT:   DW_AT_decl_line
+  ; CHECK-NEXT:   DW_AT_type ([[PTR_AS_NONE:0x[0-9a-f]+]] "int *")
+
+  ; Verify that an attribute is attached to the variable type if the loclist
+  ; contains entries with the same address spaces:
+  ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+  ; CHECK-NEXT:   DW_AT_location (indexed ({{0x[0-9a-f]+}}) loclist =
+  ; CHECK-NEXT:      [{{0x[0-9a-f]+}}, {{0x[0-9a-f]+}}): DW_OP_regx
+  ; CHECK-NEXT:      [{{0x[0-9a-f]+}}, {{0x[0-9a-f]+}}): DW_OP_lit0, DW_OP_stack_value)
+  ; CHECK-NEXT:   DW_AT_name ("ptr_all_as3")
+  ; CHECK-NEXT:   DW_AT_decl_file
+  ; CHECK-NEXT:   DW_AT_decl_line
+  ; CHECK-NEXT:   DW_AT_type ([[PTR_AS_3]] "int *")
+
+    #dbg_value(ptr addrspace(3) %loc_ptr, !21, !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr)), !22)
+    #dbg_value(ptr addrspace(3) %loc_ptr, !20, !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr)), !22)
+  tail call void asm sideeffect "s_nop 1", ""(), !dbg !22
+    #dbg_value(ptr null, !21, !DIExpression(DIOpArg(0, ptr)), !23)
+    #dbg_value(ptr addrspace(3) null, !20, !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr)), !23)
+  ret void, !dbg !23
+}
+
+; CHECK-LABEL: DW_AT_name ("test_loc_mmi")
+define void @test_loc_mmi() #0 !dbg !24 {
+  ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+  ; CHECK-NEXT:   DW_AT_location (indexed ({{0x[0-9a-f]+}}) loclist =
+  ; CHECK-NEXT:      [{{0x[0-9a-f]+}}, {{0x[0-9a-f]+}}): DW_OP_regx SGPR{{.*}}, DW_OP_deref_size 0x4, DW_OP_lit5, DW_OP_shr, DW_OP_lit0, DW_OP_plus, DW_OP_stack_value)
+  ; CHECK-NEXT:   DW_AT_name ("ptr_as5")
+  ; CHECK-NEXT:   DW_AT_decl_file
+  ; CHECK-NEXT:   DW_AT_decl_line
+  ; CHECK-NEXT:   DW_AT_type ([[PTR_AS_5:0x[0-9a-f]+]] "int *")
+
+  %ptr = alloca i32, align 4, addrspace(5), !dbg !27
+    #dbg_value(ptr addrspace(5) %ptr, !26, !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpConvert(ptr)), !27)
+  ret void, !dbg !28
+}
+
+; CHECK-LABEL: DW_AT_name ("test_divergent")
+define void @test_divergent(ptr addrspace(5) %p5, ptr addrspace(3) %p3) #0 !dbg !29 {
+  ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+  ; CHECK-NEXT:   DW_AT_location (DW_OP_regx {{.*}}, DW_OP_LLVM_user DW_OP_LLVM_push_lane, DW_OP_lit4, DW_OP_mul, DW_OP_LLVM_user DW_OP_LLVM_offset)
+  ; CHECK-NEXT:   DW_AT_name ("ptr_div_as5")
+  ; CHECK-NEXT:   DW_AT_decl_file
+  ; CHECK-NEXT:   DW_AT_decl_line
+  ; CHECK-NEXT:   DW_AT_type ([[PTR_AS_5]] "int *")
+    #dbg_value(ptr addrspace(5) %p5, !31, !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpConvert(ptr)), !30)
+
+  ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+  ; CHECK-NEXT:   DW_AT_location (DW_OP_regx {{.*}}, DW_OP_LLVM_user DW_OP_LLVM_push_lane, DW_OP_lit4, DW_OP_mul, DW_OP_LLVM_user DW_OP_LLVM_offset)
+  ; CHECK-NEXT:   DW_AT_name ("ptr_div_as3")
+  ; CHECK-NEXT:   DW_AT_decl_file
+  ; CHECK-NEXT:   DW_AT_decl_line
+  ; CHECK-NEXT:   DW_AT_type ([[PTR_AS_3]] "int *")
+    #dbg_value(ptr addrspace(3) %p3, !32, !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr), DIOpReinterpret(i64), DIOpReinterpret(ptr)), !30)
+
+  ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+  ; CHECK-NEXT:   DW_AT_location ({{.*}} DW_OP_LLVM_user DW_OP_LLVM_undefined)
+  ; CHECK-NEXT:   DW_AT_name ("ptr_div_invalid")
+    #dbg_value(ptr addrspace(5) %p5, !33, !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpConvert(ptr), DIOpReinterpret(i64), DIOpConstant(i64 42), DIOpAdd(), DIOpReinterpret(ptr)), !30)
+
+  ret void, !dbg !30
+}
+
+; CHECK-LABEL: DW_AT_name ("test_noop_convert")
+define void @test_noop_convert(ptr addrspace(1) %p1) #0 !dbg !34 {
+ ; Verify that a noop address space conversion doesn't produce a divergent
+ ; address space.
+ ; CHECK: 0x{{[0-9a-f]+}}: DW_TAG_variable
+ ; CHECK-NEXT: DW_AT_location
+ ; CHECK-NEXT: DW_AT_name ("not_divergent")
+ ; CHECK-NEXT: DW_AT_decl_file
+ ; CHECK-NEXT: DW_AT_decl_line
+ ; CHECK-NEXT: DW_AT_type ([[PTR_AS_NONE]] "int *")
+    #dbg_value(ptr addrspace(1) %p1, !36, !DIExpression(DIOpArg(0, ptr addrspace(1)), DIOpConvert(ptr addrspace(1)), DIOpReinterpret(ptr)), !37)
+  ret void, !dbg !37
+}
+
+attributes #0 = { "frame-pointer"="all" }
+
+; CHECK: [[PTR_AS_3]]: DW_TAG_pointer_type
+; CHECK-NEXT: DW_AT_type
+; CHECK-NEXT: DW_AT_address_class (0x00000003)
+; CHECK-NEXT: DW_AT_LLVM_address_space (0x00000003 "DW_ASPACE_LLVM_AMDGPU_local")
+
+; CHECK: [[PTR_AS_NONE]]: DW_TAG_pointer_type
+; CHECK-NEXT: DW_AT_type
+; CHECK-EMPTY:
+
+; CHECK: [[PTR_AS_5]]: DW_TAG_pointer_type
+; CHECK-NEXT: DW_AT_type
+; CHECK-NEXT: DW_AT_address_class (0x00000005)
+; CHECK-NEXT: DW_AT_LLVM_address_space (0x00000005 "DW_ASPACE_LLVM_AMDGPU_private_lane")
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 19.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "t.cpp", directory: "/")
+!2 = !{i32 1, !"amdhsa_code_object_version", i32 500}
+!3 = !{i32 7, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 8, !"PIC Level", i32 2}
+!7 = !{i32 7, !"frame-pointer", i32 2}
+!8 = !{!"clang version 19.0.0"}
+!9 = distinct !DISubprogram(name: "test_loc_single", linkageName: "test_loc_single", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !12)
+!10 = !DISubroutineType(types: !11)
+!11 = !{}
+!12 = !{!13}
+!13 = !DILocalVariable(name: "loc_single_ptr", scope: !9, file: !1, line: 1, type: !14)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !15, size: 64)
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !DILocation(line: 1, column: 14, scope: !9)
+!17 = !DILocation(line: 2, column: 1, scope: !9)
+!18 = distinct !DISubprogram(name: "test_loc_multi", linkageName: "test_loc_multi", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !19)
+!19 = !{!20, !21}
+!20 = !DILocalVariable(name: "ptr_all_as3", scope: !18, file: !1, line: 1, type: !14)
+!21 = !DILocalVariable(name: "ptr_as3_as2", scope: !18, file: !1, line: 1, type: !14)
+!22 = !DILocation(line: 1, column: 1, scope: !18)
+!23 = !DILocation(line: 2, column: 1, scope: !18)
+!24 = distinct !DISubprogram(name: "test_loc_mmi", linkageName: "test_loc_mmi", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !25)
+!25 = !{!26}
+!26 = !DILocalVariable(name: "ptr_as5", scope: !24, file: !1, line: 1, type: !14)
+!27 = !DILocation(line: 1, column: 1, scope: !24)
+!28 = !DILocation(line: 2, column: 1, scope: !24)
+!29 = distinct !DISubprogram(name: "test_divergent", linkageName: "test_divergent", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !19)
+!30 = !DILocation(line: 1, column: 1, scope: !29)
+!31 = !DILocalVariable(name: "ptr_div_as5", scope: !29, file: !1, line: 1, type: !14)
+!32 = !DILocalVariable(name: "ptr_div_as3", scope: !29, file: !1, line: 1, type: !14)
+!33 = !DILocalVariable(name: "ptr_div_invalid", scope: !29, file: !1, line: 1, type: !14)
+!34 = distinct !DISubprogram(name: "test_noop_convert", linkageName: "test_noop_convert", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !35)
+!35 = !{!36}
+!36 = !DILocalVariable(name: "not_divergent", scope: !34, file: !1, line: 1, type: !14)
+!37 = !DILocation(line: 1, column: 1, scope: !34)
diff --git a/external/llvm-project/llvm/test/DebugInfo/AMDGPU/lds-variable-location-info.ll b/external/llvm-project/llvm/test/DebugInfo/AMDGPU/lds-variable-location-info.ll
index dc79dd7950e6..6adac6fdc603 100644
--- a/external/llvm-project/llvm/test/DebugInfo/AMDGPU/lds-variable-location-info.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/AMDGPU/lds-variable-location-info.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs -filetype=obj < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s
-
+; XFAIL: *
 @fun.variable_name = internal addrspace(3) global i32 undef, align 4, !dbg !0
 
 
diff --git a/external/llvm-project/llvm/test/DebugInfo/COFF/dwarf-headers.ll b/external/llvm-project/llvm/test/DebugInfo/COFF/dwarf-headers.ll
index 9d515f6cec64..919068e96604 100644
--- a/external/llvm-project/llvm/test/DebugInfo/COFF/dwarf-headers.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/COFF/dwarf-headers.ll
@@ -43,6 +43,33 @@
 ; DWO-4: 0x00000000: Compile Unit: {{.*}} version = 0x0004, abbr_offset
 ; DWO-4: 0x0000000b: DW_TAG_compile_unit
 
+; Check that basic CodeView compiler info is emitted even when the DWARF debug format is used.
+; RUN: llc -dwarf-version=4 \
+; RUN:     -filetype=obj -O0 -mtriple=x86_64-unknown-windows-msvc < %s \
+; RUN:     | llvm-readobj --codeview - | FileCheck %s --check-prefix=CODEVIEW
+; CODEVIEW:      CodeViewDebugInfo [
+; CODEVIEW-NEXT:   Section: .debug$S (4)
+; CODEVIEW-NEXT:   Magic: 0x4
+; CODEVIEW-NEXT:   Subsection [
+; CODEVIEW-NEXT:     SubSectionType: Symbols (0xF1)
+; CODEVIEW-NEXT:     SubSectionSize: 0x90
+; CODEVIEW-NEXT:     ObjNameSym {
+; CODEVIEW-NEXT:       Kind: S_OBJNAME (0x1101)
+; CODEVIEW-NEXT:       Signature: 0x0
+; CODEVIEW-NEXT:       ObjectName:
+; CODEVIEW-NEXT:     }
+; CODEVIEW-NEXT:     Compile3Sym {
+; CODEVIEW-NEXT:       Kind: S_COMPILE3 (0x113C)
+; CODEVIEW-NEXT:       Language: Cpp (0x1)
+; CODEVIEW-NEXT:       Flags [ (0x0)
+; CODEVIEW-NEXT:       ]
+; CODEVIEW-NEXT:       Machine: X64 (0xD0)
+; CODEVIEW-NEXT:       FrontendVersion: 17.0.0.0
+; CODEVIEW-NEXT:       BackendVersion:
+; CODEVIEW-NEXT:       VersionName: clang version 17.0.0
+; CODEVIEW-NEXT:     }
+; CODEVIEW-NEXT:   ]
+; CODEVIEW-NEXT: ]
 
 ; ModuleID = 't.cpp'
 source_filename = "t.cpp"
diff --git a/external/llvm-project/llvm/test/DebugInfo/COFF/emission-kind-no-codeview.ll b/external/llvm-project/llvm/test/DebugInfo/COFF/emission-kind-no-codeview.ll
new file mode 100644
index 000000000000..792aaeef483f
--- /dev/null
+++ b/external/llvm-project/llvm/test/DebugInfo/COFF/emission-kind-no-codeview.ll
@@ -0,0 +1,38 @@
+; RUN: llc -filetype=obj -o - < %s | llvm-readobj --codeview - | FileCheck %s
+; Check that basic CodeView compiler info is emitted even when the CodeView flag is not set.
+
+; CHECK-NOT:  CodeViewTypes
+; CHECK:      CodeViewDebugInfo [
+; CHECK-NEXT:   Section: .debug$S (4)
+; CHECK-NEXT:   Magic: 0x4
+; CHECK-NEXT:   Subsection [
+; CHECK-NEXT:     SubSectionType: Symbols (0xF1)
+; CHECK-NEXT:     SubSectionSize: 0x2C
+; CHECK-NEXT:     ObjNameSym {
+; CHECK-NEXT:       Kind: S_OBJNAME (0x1101)
+; CHECK-NEXT:       Signature: 0x0
+; CHECK-NEXT:       ObjectName:
+; CHECK-NEXT:     }
+; CHECK-NEXT:     Compile3Sym {
+; CHECK-NEXT:       Kind: S_COMPILE3 (0x113C)
+; CHECK-NEXT:       Language: C (0x0)
+; CHECK-NEXT:       Flags [ (0x0)
+; CHECK-NEXT:       ]
+; CHECK-NEXT:       Machine: X64 (0xD0)
+; CHECK-NEXT:       FrontendVersion:
+; CHECK-NEXT:       BackendVersion:
+; CHECK-NEXT:       VersionName: clang
+; CHECK-NEXT:     }
+; CHECK-NEXT:   ]
+; CHECK-NEXT: ]
+
+source_filename = "empty"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "clang", emissionKind: NoDebug)
+!1 = !DIFile(filename: "empty", directory: "path/to")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/external/llvm-project/llvm/test/DebugInfo/COFF/emission-kind-no-debug.ll b/external/llvm-project/llvm/test/DebugInfo/COFF/emission-kind-no-debug.ll
index 4204df512ac3..94fee0e1812f 100644
--- a/external/llvm-project/llvm/test/DebugInfo/COFF/emission-kind-no-debug.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/COFF/emission-kind-no-debug.ll
@@ -1,8 +1,30 @@
 ; RUN: llc -filetype=obj -o - < %s | llvm-readobj --codeview - | FileCheck %s
-; Check that debug info isn't emitted for CodeView with emissionKind NoDebug
+; Check that only basic compiler info is emitted for CodeView with emissionKind NoDebug
 
-; CHECK-NOT:      CodeViewTypes
-; CHECK-NOT:      CodeViewDebugInfo
+; CHECK-NOT:  CodeViewTypes
+; CHECK:      CodeViewDebugInfo [
+; CHECK-NEXT:   Section: .debug$S (4)
+; CHECK-NEXT:   Magic: 0x4
+; CHECK-NEXT:   Subsection [
+; CHECK-NEXT:     SubSectionType: Symbols (0xF1)
+; CHECK-NEXT:     SubSectionSize: 0x2C
+; CHECK-NEXT:     ObjNameSym {
+; CHECK-NEXT:       Kind: S_OBJNAME (0x1101)
+; CHECK-NEXT:       Signature: 0x0
+; CHECK-NEXT:       ObjectName:
+; CHECK-NEXT:     }
+; CHECK-NEXT:     Compile3Sym {
+; CHECK-NEXT:       Kind: S_COMPILE3 (0x113C)
+; CHECK-NEXT:       Language: C (0x0)
+; CHECK-NEXT:       Flags [ (0x0)
+; CHECK-NEXT:       ]
+; CHECK-NEXT:       Machine: X64 (0xD0)
+; CHECK-NEXT:       FrontendVersion:
+; CHECK-NEXT:       BackendVersion:
+; CHECK-NEXT:       VersionName: clang
+; CHECK-NEXT:     }
+; CHECK-NEXT:   ]
+; CHECK-NEXT: ]
 
 source_filename = "empty"
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/external/llvm-project/llvm/test/DebugInfo/COFF/fission-cu.ll b/external/llvm-project/llvm/test/DebugInfo/COFF/fission-cu.ll
index 3afcb8717e31..dcc3fdd2efa7 100644
--- a/external/llvm-project/llvm/test/DebugInfo/COFF/fission-cu.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/COFF/fission-cu.ll
@@ -107,11 +107,11 @@ source_filename = "test/DebugInfo/X86/fission-cu.ll"
 ; For COFF we should have this set of relocations for the debug info section
 ;
 ; OBJ: .debug_info
-; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_abbrev (6)
-; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_line (26)
-; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_str (10)
-; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_str (10)
-; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_addr (20)
+; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_abbrev (8)
+; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_line (28)
+; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_str (12)
+; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_str (12)
+; OBJ-NEXT: IMAGE_REL_AMD64_SECREL .debug_addr (22)
 ; OBJ-NEXT: }
 
 ; HDR-NOT: .debug_aranges
diff --git a/external/llvm-project/llvm/test/DebugInfo/COFF/fission-sections.ll b/external/llvm-project/llvm/test/DebugInfo/COFF/fission-sections.ll
index 754e2b888c20..c16a4d072909 100644
--- a/external/llvm-project/llvm/test/DebugInfo/COFF/fission-sections.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/COFF/fission-sections.ll
@@ -27,13 +27,14 @@ source_filename = "test/DebugInfo/X86/fission-cu.ll"
 ; OBJ-NEXT:  0 .text
 ; OBJ-NEXT:  1 .data
 ; OBJ-NEXT:  2 .bss
-; OBJ-NEXT:  3 .debug_abbrev
-; OBJ-NEXT:  4 .debug_info
-; OBJ-NEXT:  5 .debug_str
-; OBJ-NEXT:  6 .debug_addr
-; OBJ-NEXT:  7 .debug_pubnames
-; OBJ-NEXT:  8 .debug_pubtypes
-; OBJ-NEXT:  9 .debug_line
+; OBJ-NEXT:  3 .debug$S
+; OBJ-NEXT:  4 .debug_abbrev
+; OBJ-NEXT:  5 .debug_info
+; OBJ-NEXT:  6 .debug_str
+; OBJ-NEXT:  7 .debug_addr
+; OBJ-NEXT:  8 .debug_pubnames
+; OBJ-NEXT:  9 .debug_pubtypes
+; OBJ-NEXT: 10 .debug_line
 
 ; OBJ:     .debug_abbrev
 ; OBJ:     .debug_info
diff --git a/external/llvm-project/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll b/external/llvm-project/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll
index 566d36e87d2b..d0d724910faf 100644
--- a/external/llvm-project/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/COFF/fpo-stack-protect.ll
@@ -15,7 +15,7 @@
 ; CHECK:         subl    $20, %esp
 ; CHECK:         .cv_fpo_stackalloc      20
 ; CHECK:         .cv_fpo_endprologue
-; CHECK:         ___security_cookie
+; CHECK:         movl    ___security_cookie, %ea
 
 ; CHECK:         movl    28(%esp), %esi
 ; CHECK:         movl    %esi, {{[0-9]*}}(%esp)
@@ -24,13 +24,16 @@
 ; CHECK:         movl    %esi, {{[0-9]*}}(%esp)
 
 ; CHECK:         calll   _escape
-; CHECK:         calll   @__security_check_cookie@4
+
+; CHECK:         movl    ___security_cookie, %ecx
+; CHECK:         cmpl    %eax, %ecx
 
 ; CHECK:         movl    %esi, %eax
 ; CHECK:         addl    $20, %esp
 ; CHECK:         popl    %esi
 ; CHECK:         retl
 ; CHECK: Ltmp4:
+; CHECK:         calll   @__security_check_cookie@4
 ; CHECK:         .cv_fpo_endproc
 
 ; ModuleID = 't.c'
diff --git a/external/llvm-project/llvm/test/DebugInfo/COFF/uefi-nodebug.ll b/external/llvm-project/llvm/test/DebugInfo/COFF/uefi-nodebug.ll
new file mode 100644
index 000000000000..92e5fd6b5796
--- /dev/null
+++ b/external/llvm-project/llvm/test/DebugInfo/COFF/uefi-nodebug.ll
@@ -0,0 +1,16 @@
+; RUN: llc -filetype=obj -o - < %s | llvm-readobj --codeview - | FileCheck %s
+; Check that compiler info is not emitted when CodeView flag is not specified
+
+; CHECK-NOT:  CodeViewTypes
+; CHECK-NOT:  CodeViewDebugInfo
+
+source_filename = "empty"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-uefi"
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "clang", emissionKind: NoDebug)
+!1 = !DIFile(filename: "empty", directory: "path/to")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/external/llvm-project/llvm/test/DebugInfo/Generic/assignment-tracking/parse-and-verify/verify.ll b/external/llvm-project/llvm/test/DebugInfo/Generic/assignment-tracking/parse-and-verify/verify.ll
index 0a4b7c255dc7..d1f1e1ce768d 100644
--- a/external/llvm-project/llvm/test/DebugInfo/Generic/assignment-tracking/parse-and-verify/verify.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/Generic/assignment-tracking/parse-and-verify/verify.ll
@@ -8,7 +8,7 @@
 
 define dso_local void @fun2() !dbg !15 {
   ;; DIAssignID copied here from @fun() where it is used by intrinsics.
-  ; CHECK: dbg.assign not in same function as inst
+  ; CHECK: DVRAssign not in same function as inst
   %x = alloca i32, align 4, !DIAssignID !14
   ret void
 }
@@ -17,24 +17,24 @@ define dso_local void @fun() !dbg !7 {
 entry:
   %a = alloca i32, align 4, !DIAssignID !14
   ;; Here something other than a dbg.assign intrinsic is using a DIAssignID.
-  ; CHECK: !DIAssignID should only be used by llvm.dbg.assign intrinsics
+  ; CHECK: !DIAssignID should only be used by Assign DVRs
   call void @llvm.dbg.value(metadata !14, metadata !10, metadata !DIExpression()), !dbg !13
 
   ;; Each following dbg.assign has an argument of the incorrect type.
-  ; CHECK: invalid llvm.dbg.assign intrinsic address/value
+  ; CHECK: invalid #dbg record address/value
   call void @llvm.dbg.assign(metadata !3, metadata !10, metadata !DIExpression(), metadata !14, metadata ptr undef, metadata !DIExpression()), !dbg !13
-  ; CHECK: invalid llvm.dbg.assign intrinsic variable
+  ; CHECK: invalid #dbg record variable
   call void @llvm.dbg.assign(metadata i32 0, metadata !2, metadata !DIExpression(), metadata !14, metadata ptr undef, metadata !DIExpression()), !dbg !13
-  ; CHECK: invalid llvm.dbg.assign intrinsic expression
+  ; CHECK: invalid #dbg record expression
   call void @llvm.dbg.assign(metadata !14, metadata !10, metadata !2, metadata !14, metadata ptr undef, metadata !DIExpression()), !dbg !13
-  ; CHECK: invalid llvm.dbg.assign intrinsic DIAssignID
+  ; CHECK: invalid #dbg_assign DIAssignID
   call void @llvm.dbg.assign(metadata !14, metadata !10, metadata !DIExpression(), metadata !2, metadata ptr undef, metadata !DIExpression()), !dbg !13
-  ; CHECK: invalid llvm.dbg.assign intrinsic address
+  ; CHECK: invalid #dbg_assign address
   call void @llvm.dbg.assign(metadata !14, metadata !10, metadata !DIExpression(), metadata !14, metadata !3, metadata !DIExpression()), !dbg !13
   ;; Empty metadata debug operands are allowed.
-  ; CHECK-NOT: invalid llvm.dbg.assign
+  ; CHECK-NOT: invalid #dbg record
   call void @llvm.dbg.assign(metadata !14, metadata !10, metadata !DIExpression(), metadata !14, metadata !2, metadata !DIExpression()), !dbg !13
-  ; CHECK: invalid llvm.dbg.assign intrinsic address expression
+  ; CHECK: invalid #dbg_assign address expression
   call void @llvm.dbg.assign(metadata !14, metadata !10, metadata !DIExpression(), metadata !14, metadata ptr undef, metadata !2), !dbg !13
   ret void
 }
diff --git a/external/llvm-project/llvm/test/DebugInfo/Generic/directives-only.ll b/external/llvm-project/llvm/test/DebugInfo/Generic/directives-only.ll
index ff9393221e2f..4754df7186fa 100644
--- a/external/llvm-project/llvm/test/DebugInfo/Generic/directives-only.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/Generic/directives-only.ll
@@ -18,7 +18,7 @@
 ; CHECK: .loc 1 4 15
 ; CHECK: .loc 1 5 1
 
-; CHECK-NOT: .section .{{debug.*}}
+; CHECK-NOT: .section .{{debug_.*}}
 
 ; Function Attrs: nounwind uwtable
 define void @f2() #0 !dbg !4 {
diff --git a/external/llvm-project/llvm/test/DebugInfo/Generic/sroa-extract-bits.ll b/external/llvm-project/llvm/test/DebugInfo/Generic/sroa-extract-bits.ll
index f47e495db661..6db453605cb5 100644
--- a/external/llvm-project/llvm/test/DebugInfo/Generic/sroa-extract-bits.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/Generic/sroa-extract-bits.ll
@@ -13,8 +13,8 @@ define i8 @test1(i32 %arg) {
 ; CHECK-NEXT:      #dbg_value(i8 [[PTR_SROA_0_0_EXTRACT_TRUNC]], [[META2:![0-9]+]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 8), [[META7:![0-9]+]])
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i32 [[ARG]], 8
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[PTR_SROA_2_0_EXTRACT_SHIFT]] to i24
-; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META8:![0-9]+]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 8, 16), [[META7]])
-; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META9:![0-9]+]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META8:![0-9]+]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META10:![0-9]+]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 8, 16), [[META7]])
 ; CHECK-NEXT:    ret i8 [[PTR_SROA_0_0_EXTRACT_TRUNC]]
 ;
 entry:
@@ -36,11 +36,11 @@ define i8 @test2(i32 %arg1, i8 %arg2) {
 ; CHECK-NEXT:      #dbg_value(i8 [[PTR_SROA_0_0_EXTRACT_TRUNC]], [[META2]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 8), [[META7]])
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i32 [[ARG1]], 8
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[PTR_SROA_2_0_EXTRACT_SHIFT]] to i16
-; CHECK-NEXT:      #dbg_value(i16 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META9]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 16), [[META7]])
+; CHECK-NEXT:      #dbg_value(i16 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 16), [[META7]])
 ; CHECK-NEXT:    [[PTR_SROA_21_0_EXTRACT_SHIFT:%.*]] = lshr i32 [[ARG1]], 24
 ; CHECK-NEXT:    [[PTR_SROA_21_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[PTR_SROA_21_0_EXTRACT_SHIFT]] to i8
-; CHECK-NEXT:      #dbg_value(i8 [[PTR_SROA_21_0_EXTRACT_TRUNC]], [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 8), [[META7]])
-; CHECK-NEXT:      #dbg_value(i8 [[ARG2]], [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i8 [[PTR_SROA_21_0_EXTRACT_TRUNC]], [[META10]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i8 [[ARG2]], [[META10]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 8), [[META7]])
 ; CHECK-NEXT:    ret i8 [[PTR_SROA_0_0_EXTRACT_TRUNC]]
 ;
 entry:
@@ -84,7 +84,7 @@ define i16 @test4(i32 %arg) {
 ; CHECK-NEXT:      #dbg_value(i16 [[PTR_SROA_0_0_EXTRACT_TRUNC]], [[META2]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 8), [[META7]])
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i32 [[ARG]], 16
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[PTR_SROA_2_0_EXTRACT_SHIFT]] to i16
-; CHECK-NEXT:      #dbg_value(i16 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 8, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i16 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META10]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 8, 8), [[META7]])
 ; CHECK-NEXT:    ret i16 [[PTR_SROA_0_0_EXTRACT_TRUNC]]
 ;
 entry:
@@ -107,8 +107,8 @@ define i8 @test5(i32 %arg) {
 ; CHECK-NEXT:      #dbg_value(i8 [[PTR_SROA_0_0_EXTRACT_TRUNC]], [[META11:![0-9]+]], !DIExpression(), [[META7]])
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i32 [[ARG]], 8
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[PTR_SROA_2_0_EXTRACT_SHIFT]] to i24
-; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 8, 8), [[META7]])
-; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META9]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i24 [[PTR_SROA_2_0_EXTRACT_TRUNC]], [[META10]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 8, 8), [[META7]])
 ; CHECK-NEXT:    ret i8 [[PTR_SROA_0_0_EXTRACT_TRUNC]]
 ;
 entry:
@@ -130,11 +130,11 @@ define i8 @test6(i32 %arg1, i8 %arg2) {
 ; CHECK-NEXT:      #dbg_value(i8 poison, [[META2]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i32 [[ARG1]], 8
 ; CHECK-NEXT:    [[PTR_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[PTR_SROA_2_0_EXTRACT_SHIFT]] to i16
-; CHECK-NEXT:      #dbg_value(i16 poison, [[META9]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 16), [[META7]])
+; CHECK-NEXT:      #dbg_value(i16 poison, [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_zext, 0, 16), [[META7]])
 ; CHECK-NEXT:    [[PTR_SROA_21_0_EXTRACT_SHIFT:%.*]] = lshr i32 [[ARG1]], 24
 ; CHECK-NEXT:    [[PTR_SROA_21_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[PTR_SROA_21_0_EXTRACT_SHIFT]] to i8
-; CHECK-NEXT:      #dbg_value(i8 poison, [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
-; CHECK-NEXT:      #dbg_value(i8 poison, [[META8]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i8 poison, [[META10]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
+; CHECK-NEXT:      #dbg_value(i8 poison, [[META10]], !DIExpression(DW_OP_LLVM_extract_bits_sext, 0, 8), [[META7]])
 ; CHECK-NEXT:    ret i8 [[PTR_SROA_0_0_EXTRACT_TRUNC]]
 ;
 entry:
@@ -197,9 +197,9 @@ entry:
 ; CHECK: [[META5]] = !DIFile(filename: "dbg-bit-piece.cpp", directory: "")
 ; CHECK: [[META6]] = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
 ; CHECK: [[META7]] = !DILocation(line: 0, scope: [[META3]])
-; CHECK: [[META8]] = !DILocalVariable(name: "z", scope: [[META3]], type: [[META6]])
-; CHECK: [[META9]] = !DILocalVariable(name: "y", scope: [[META3]], type: [[META10:![0-9]+]])
-; CHECK: [[META10]] = !DIBasicType(name: "signed int", size: 32, encoding: DW_ATE_signed)
+; CHECK: [[META8]] = !DILocalVariable(name: "y", scope: [[META3]], type: [[META9:![0-9]+]])
+; CHECK: [[META9]] = !DIBasicType(name: "signed int", size: 32, encoding: DW_ATE_signed)
+; CHECK: [[META10]] = !DILocalVariable(name: "z", scope: [[META3]], type: [[META6]])
 ; CHECK: [[META11]] = !DILocalVariable(name: "x", scope: [[META3]], type: [[META12:![0-9]+]])
 ; CHECK: [[META12]] = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
 ;.
diff --git a/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll b/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
index 8a9052c6f98f..e2a9b795420c 100644
--- a/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64-nvidia-cuda | %ptxas-verify %}
-
+; XFAIL: *
 ; CHECK: .target sm_30, debug
 
 ; CHECK: .visible .func use_dbg_declare()
diff --git a/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-value-const-byref.ll b/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-value-const-byref.ll
index 41734f33213e..19d8d23c4aae 100644
--- a/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-value-const-byref.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/NVPTX/dbg-value-const-byref.ll
@@ -1,5 +1,6 @@
 ; RUN: llc -mtriple=nvptx64-nvidia-cuda -mattr=+ptx70 < %s | FileCheck %s
 ; RUN: %if ptxas %{ llc -mtriple=nvptx64-nvidia-cuda -mattr=+ptx70 < %s | %ptxas-verify %}
+; XFAIL: *
 
 ; Generated with -O1 from:
 ; int f1();
diff --git a/external/llvm-project/llvm/test/DebugInfo/NVPTX/debug-info.ll b/external/llvm-project/llvm/test/DebugInfo/NVPTX/debug-info.ll
index 04296cd92cd0..e47c422820c7 100644
--- a/external/llvm-project/llvm/test/DebugInfo/NVPTX/debug-info.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/NVPTX/debug-info.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mattr=+ptx70 | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64-nvidia-cuda -mattr=+ptx70 | %ptxas-verify %}
-
+; XFAIL: *
 ; // Bitcode in this test case is reduced version of compiled code below:
 ;__device__ inline void res(float x, float y, ptr res) { *res = x + y; }
 ;
diff --git a/external/llvm-project/llvm/test/DebugInfo/X86/DW_AT_LLVM_stmt_seq_sec_offset_empty_func.ll b/external/llvm-project/llvm/test/DebugInfo/X86/DW_AT_LLVM_stmt_seq_sec_offset_empty_func.ll
new file mode 100644
index 000000000000..553a05f12082
--- /dev/null
+++ b/external/llvm-project/llvm/test/DebugInfo/X86/DW_AT_LLVM_stmt_seq_sec_offset_empty_func.ll
@@ -0,0 +1,37 @@
+; RUN: llc -filetype=obj -emit-func-debug-line-table-offsets < %s -o %t_yes_off.o
+; RUN: llc -filetype=obj < %s -o %t_no_off.o
+; RUN: llvm-dwarfdump -v -all %t_no_off.o | FileCheck --check-prefix=CHECK-NO-OFF %s
+; RUN: llvm-dwarfdump -v -all %t_yes_off.o | FileCheck --check-prefix=CHECK-YES-OFF %s
+
+; We don't need a line table for an empty function
+; CHECK-NO-OFF-NOT: DW_LNE_set_address
+
+; CHECK-YES-OFF:                 DW_AT_LLVM_stmt_sequence [DW_FORM_sec_offset] ({{.*}})
+; CHECK-YES-OFF:                 Address
+; CHECK-YES-OFF-NEXT:            ------------------
+; CHECK-YES-OFF-NEXT:            DW_LNE_set_address ({{.*}})
+; CHECK-YES-OFF-NEXT:            DW_LNE_end_sequence
+; CHECK-YES-OFF-NEXT:            0x0000000000000000 {{.*}} is_stmt end_sequence
+
+; IR generated by llvm-reduce from LTO repro
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define ptr @_my_test_function() !dbg !4 {
+entry:
+  unreachable
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "repro_bug.cpp", directory: "/tmp/repro", checksumkind: CSK_MD5, checksum: "74e33e88b3108a4f94403da9fdb362f5")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "east", linkageName: "_my_test_function", scope: !6, file: !5, line: 114, type: !8, scopeLine: 114, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !9, retainedNodes: !2)
+!5 = !DIFile(filename: "repro_bug.cpp", directory: "/tmp/repro", checksumkind: CSK_MD5, checksum: "74e33e88b3108a4f94403da9fdb362f5")
+!6 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "rectangle", file: !5, line: 103, size: 256, flags: DIFlagTypePassByReference | DIFlagNonTrivial, elements: !2, vtableHolder: !7)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "shape", file: !5, line: 58, size: 128, flags: DIFlagTypePassByReference | DIFlagNonTrivial, elements: !2, vtableHolder: !7, identifier: "_ZTS5shape")
+!8 = distinct !DISubroutineType(types: !2)
+!9 = !DISubprogram(name: "east", linkageName: "_my_test_function", scope: !6, file: !5, line: 114, type: !8, scopeLine: 114, containingType: !6, virtualIndex: 2, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagVirtual | DISPFlagOptimized)
diff --git a/external/llvm-project/llvm/test/DebugInfo/X86/dbg-value-range.ll b/external/llvm-project/llvm/test/DebugInfo/X86/dbg-value-range.ll
index 0d49b5eeefd1..a6ede2814aba 100644
--- a/external/llvm-project/llvm/test/DebugInfo/X86/dbg-value-range.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/X86/dbg-value-range.ll
@@ -49,9 +49,9 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
 ;CHECK-NEXT: [[CLOBBER:Ltmp[0-9]*]]
 
 ;CHECK:Ldebug_loc0:
-;CHECK-NEXT: .set Lset{{.*}},
+;CHECK-NEXT: Lset{{.*}} =
 ;CHECK-NEXT:	.quad
-;CHECK-NEXT: .set [[CLOBBER_OFF:Lset.*]], [[CLOBBER]]-{{.*}}
+;CHECK-NEXT: [[CLOBBER_OFF:Lset.*]] = [[CLOBBER]]-{{.*}}
 ;CHECK-NEXT:	.quad	[[CLOBBER_OFF]]
 ;CHECK-NEXT:  .short 1 ## Loc expr size
 ;CHECK-NEXT:	.byte	85 ## DW_OP_reg
diff --git a/external/llvm-project/llvm/test/DebugInfo/X86/file-index-across-cu.ll b/external/llvm-project/llvm/test/DebugInfo/X86/file-index-across-cu.ll
new file mode 100644
index 000000000000..3e0bf9d5e789
--- /dev/null
+++ b/external/llvm-project/llvm/test/DebugInfo/X86/file-index-across-cu.ll
@@ -0,0 +1,78 @@
+; RUN: llc %s -filetype=obj -o - | llvm-dwarfdump -verify -
+; RUN: llc %s -filetype=obj -o - | llvm-dwarfdump -debug-info - | FileCheck %s
+
+;; This test exercises the cross-CU production of type information. Due to
+;; unfortunate inputs, when the DW_TAG_variable for the global
+;; @a_struct_field8typeDataE is produced it will be done in the CU for 2.cpp.
+;; However, the type information for the "a_struct" structure type will have
+;; already been produced in 3.cpp's CU. Thus, we'll be creating a DIE in one
+;; CU from another CU. Check that when we do so, we get the source file
+;; identifier correct -- if the FileID came from the wrong CU then we'd end
+;; up with a false declaration file.
+;;
+;; Due to unrelated reasons there are two copies of the "typeData" static member
+;; in this LTO-linked IR -- if that ever gets fitxed, it's fine to drop the
+;; duplicate DW_TAG_variable, but we want to keep testing the DW_AT_decl_file
+;; that's created across a CU boundary.
+;;
+;; https://github.com/llvm/llvm-project/issues/109227
+
+; CHECK-LABEL: DW_TAG_structure_type
+; CHECK-NOT:   DW_TAG
+; CHECK:       DW_AT_name      ("a_struct")
+; CHECK-NOT:   DW_TAG
+; CHECK:       DW_TAG_variable
+; CHECK-NOT:   DW_TAG
+; CHECK:       DW_AT_decl_file       ("C:\Users\gbmorsej\source\bees{{\\|/}}3.cpp")
+; CHECK-NOT:   DW_TAG
+; CHECK:       DW_TAG_variable
+; CHECK-NOT:   DW_TAG
+; CHECK:       DW_AT_decl_file       ("C:\users\gbmorsej\source/bees{{\\|/}}2.cpp")
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-sie-ps5"
+
+%struct.a_struct = type { i8 }
+
+@__trans_tmp_2 = hidden local_unnamed_addr global %struct.a_struct zeroinitializer, align 1, !dbg !0
+@a_struct_field8typeDataE = hidden local_unnamed_addr global i64 0, align 8, !dbg !13
+
+!llvm.dbg.cu = !{!2, !15}
+!llvm.linker.options = !{}
+!llvm.ident = !{!22, !22}
+!llvm.module.flags = !{!23, !24, !25, !26, !27, !28, !29, !30, !31, !32}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "__trans_tmp_2", scope: !2, file: !3, line: 9, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_20, file: !3, producer: "clang version 21.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!3 = !DIFile(filename: "3.cpp", directory: "C:\\Users\\gbmorsej\\source\\bees", checksumkind: CSK_MD5, checksum: "05973c817251e916cc8ba01e728764dc")
+!4 = !{!0}
+!5 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "a_struct", file: !3, line: 4, size: 8, flags: DIFlagTypePassByValue, elements: !6, identifier: "redacted_struct_name")
+!6 = !{!7}
+!7 = !DIDerivedType(tag: DW_TAG_variable, name: "typeData", scope: !5, file: !3, line: 6, baseType: !8, flags: DIFlagStaticMember)
+!8 = !DIDerivedType(tag: DW_TAG_typedef, name: "TypeData", scope: !10, file: !9, line: 6, baseType: !12)
+!9 = !DIFile(filename: ".\\h1.h", directory: "C:\\Users\\gbmorsej\\source\\bees", checksumkind: CSK_MD5, checksum: "0c90de8c8e867df533d869035e11cf8c")
+!10 = !DINamespace(name: "b", scope: !11)
+!11 = !DINamespace(name: "a", scope: null)
+!12 = !DIBasicType(name: "long", size: 64, encoding: DW_ATE_signed)
+!13 = !DIGlobalVariableExpression(var: !14, expr: !DIExpression())
+!14 = distinct !DIGlobalVariable(name: "typeData", linkageName: "a_struct_field8typeDataE", scope: !15, file: !18, line: 10, type: !19, isLocal: false, isDefinition: true, declaration: !21)
+!15 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_20, file: !16, producer: "clang version 21.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !17, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!16 = !DIFile(filename: "C:\\users\\gbmorsej\\source/bees\\2.cpp", directory: "C:\\Users\\gbmorsej\\source\\bees", checksumkind: CSK_MD5, checksum: "231f25b4978512f0d961a1d7baa9cb01")
+!17 = !{!13}
+!18 = !DIFile(filename: "C:\\users\\gbmorsej\\source/bees/2.cpp", directory: "", checksumkind: CSK_MD5, checksum: "231f25b4978512f0d961a1d7baa9cb01")
+!19 = !DIDerivedType(tag: DW_TAG_typedef, name: "TypeData", scope: !10, file: !20, line: 6, baseType: !12)
+!20 = !DIFile(filename: "h1.h", directory: "C:\\Users\\gbmorsej\\source\\bees", checksumkind: CSK_MD5, checksum: "0c90de8c8e867df533d869035e11cf8c")
+!21 = !DIDerivedType(tag: DW_TAG_variable, name: "typeData", scope: !5, file: !18, line: 6, baseType: !19, flags: DIFlagStaticMember)
+!22 = !{!"clang version 21.0.0"}
+!23 = !{i32 7, !"Dwarf Version", i32 5}
+!24 = !{i32 2, !"Debug Info Version", i32 3}
+!25 = !{i32 1, !"wchar_size", i32 2}
+!26 = !{i32 1, !"SIE:somestuff", i32 2}
+!27 = !{i32 8, !"PIC Level", i32 2}
+!28 = !{i32 7, !"uwtable", i32 2}
+!29 = !{i32 7, !"frame-pointer", i32 1}
+!30 = !{i32 1, !"MaxTLSAlign", i32 256}
+!31 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!32 = !{i32 1, !"UnifiedLTO", i32 1}
+
diff --git a/external/llvm-project/llvm/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll b/external/llvm-project/llvm/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll
index 446f31f9a912..8d4d065641fc 100644
--- a/external/llvm-project/llvm/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/X86/stmt-list-multiple-compile-units.ll
@@ -64,11 +64,11 @@
 ; PR15408
 ; ASM: Lcu_begin0:
 ; ASM-NOT: Lcu_begin
-; ASM: .set Lset[[LT:[0-9]+]], Lline_table_start0-Lsection_line ## DW_AT_stmt_list
+; ASM: Lset[[LT:[0-9]+]] = Lline_table_start0-Lsection_line ## DW_AT_stmt_list
 ; ASM-NEXT: .long   Lset[[LT]]
 ; ASM: Lcu_begin1:
 ; ASM-NOT: Lcu_begin
-; ASM: .set Lset[[LT:[0-9]+]], Lline_table_start0-Lsection_line ## DW_AT_stmt_list
+; ASM: Lset[[LT:[0-9]+]] = Lline_table_start0-Lsection_line ## DW_AT_stmt_list
 ; ASM-NEXT: .long   Lset[[LT]]
 define i32 @test(i32 %a) nounwind uwtable ssp !dbg !5 {
 entry:
diff --git a/external/llvm-project/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll b/external/llvm-project/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll
index 55e436b1a93b..4990979f10c5 100644
--- a/external/llvm-project/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -passes=check-debugify < %s 2>&1 | FileCheck %s
-; RUN: opt --experimental-debuginfo-iterators=false -passes=check-debugify < %s 2>&1 | FileCheck %s
 
 define <2 x i64> @test-fun(<2 x i64> %A) !dbg !6 {
   %and = and <2 x i64> %A, <i64 23, i64 42>, !dbg !14
diff --git a/external/llvm-project/llvm/test/DebugInfo/debugify-each.ll b/external/llvm-project/llvm/test/DebugInfo/debugify-each.ll
index 7685b57b5dd1..e9241dedb696 100644
--- a/external/llvm-project/llvm/test/DebugInfo/debugify-each.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/debugify-each.ll
@@ -40,40 +40,6 @@
 ; RUN: opt -debugify-each -passes=globalopt -S -o /dev/null < %s 2> %t
 ; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS-ONE
 
-; Repeat the same checks with debug intrinsics enabled.
-; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -O3 -S -o /dev/null < %s 2> %t
-; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
-; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
-; RUN: opt --experimental-debuginfo-iterators=false -disable-output -debugify-each -passes='default<O3>' %s 2> %t
-; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
-; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
-
-; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -debugify-each -O3 -S -o /dev/null < %s 2> %t
-; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
-; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
-
-; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -passes='instrprof,instrprof,sroa,sccp' -S -o /dev/null < %s 2> %t
-; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
-; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
-
-; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -O1 < %s | opt -O2 -o /dev/null
-
-; RUN: opt --experimental-debuginfo-iterators=false -disable-output -debugify-quiet -debugify-each -O1 < %s 2>&1 | count 0
-
-; RUN: opt --experimental-debuginfo-iterators=false -O1 < %s -S -o %t.before
-; RUN: opt --experimental-debuginfo-iterators=false -O1 -debugify-each < %s -S -o %t.after
-; RUN: diff %t.before %t.after
-
-; RUN: opt --experimental-debuginfo-iterators=false -O1 < %s | llvm-dis -o %t.before
-; RUN: opt --experimental-debuginfo-iterators=false -O1 -debugify-each < %s | llvm-dis -o %t.after
-; RUN: diff %t.before %t.after
-
-; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -passes=instsimplify -S -o /dev/null < %s 2> %t
-; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS-ONE
-
-; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -passes=globalopt -S -o /dev/null < %s 2> %t
-; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS-ONE
-
 define void @foo(i32 %arg) {
   call i32 asm "bswap $0", "=r,r"(i32 %arg)
   ret void
diff --git a/external/llvm-project/llvm/test/DebugInfo/debugify-export.ll b/external/llvm-project/llvm/test/DebugInfo/debugify-export.ll
index 30333ca908b0..6e5952d433da 100644
--- a/external/llvm-project/llvm/test/DebugInfo/debugify-export.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/debugify-export.ll
@@ -1,9 +1,6 @@
 ; RUN: opt %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
 ; RUN: opt %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
 
-; RUN: opt --experimental-debuginfo-iterators=false %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
-; RUN: opt --experimental-debuginfo-iterators=false %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
-
 ; CHECK: Pass Name
 ; CHECK-SAME: # of missing debug values
 ; CHECK-SAME: # of missing locations
diff --git a/external/llvm-project/llvm/test/DebugInfo/debugify-ignore-phi.ll b/external/llvm-project/llvm/test/DebugInfo/debugify-ignore-phi.ll
index 643df1d96048..322ccafa22ac 100644
--- a/external/llvm-project/llvm/test/DebugInfo/debugify-ignore-phi.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/debugify-ignore-phi.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -passes=check-debugify < %s -S 2>&1 | FileCheck %s
-; RUN: opt --experimental-debuginfo-iterators=false -passes=check-debugify < %s -S 2>&1 | FileCheck %s
 
 define void @test_phi(i1 %cond) !dbg !6 {
   br i1 %cond, label %1, label %2, !dbg !11
diff --git a/external/llvm-project/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll b/external/llvm-project/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll
index 4cbbfc5c215e..941b294fb855 100644
--- a/external/llvm-project/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -verify-debuginfo-preserve -passes=instcombine -S -o - < %s 2>&1 | FileCheck %s
-; RUN: opt --experimental-debuginfo-iterators=false -verify-debuginfo-preserve -passes=instcombine -S -o - < %s 2>&1 | FileCheck %s
 
 ; CHECK: ModuleDebugify (original debuginfo): Skipping module without debug info
 ; CHECK-NEXT: CheckModuleDebugify (original debuginfo): Skipping module without debug info
diff --git a/external/llvm-project/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll b/external/llvm-project/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
index 04b7636f025a..1c5daa19c648 100644
--- a/external/llvm-project/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -passes=check-debugify < %s -S -o - 2>&1 | FileCheck %s -implicit-check-not "WARNING: Instruction with empty DebugLoc in function bar"
-; RUN: opt --experimental-debuginfo-iterators=false -passes=check-debugify < %s -S -o - 2>&1 | FileCheck %s -implicit-check-not "WARNING: Instruction with empty DebugLoc in function bar"
 
 ; CHECK: WARNING: Instruction with empty DebugLoc in function foo --   ret void
 define void @foo() !dbg !6 {
diff --git a/external/llvm-project/llvm/test/DebugInfo/debugify.ll b/external/llvm-project/llvm/test/DebugInfo/debugify.ll
index 191015f82593..5ce6795d41b6 100644
--- a/external/llvm-project/llvm/test/DebugInfo/debugify.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/debugify.ll
@@ -25,33 +25,6 @@
 ; RUN: opt -enable-debugify -O1 < %s | opt -O2 -o /dev/null
 ; RUN: opt -passes=debugify,mem2reg,check-debugify < %s | opt -O2 -o /dev/null
 
-;; Perform the same checks again for intrinsic debug info
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify -S -o - < %s | FileCheck %s
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify -S -o - < %s | FileCheck %s
-
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,debugify -S -o - < %s 2>&1 | \
-; RUN:   FileCheck %s -check-prefix=CHECK-REPEAT
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,debugify -S -o - < %s 2>&1 | \
-; RUN:   FileCheck %s -check-prefix=CHECK-REPEAT
-
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,check-debugify -S -o - < %s | \
-; RUN:   FileCheck %s -implicit-check-not="CheckModuleDebugify: FAIL"
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,check-debugify -S -o - < %s | \
-; RUN:   FileCheck %s -implicit-check-not="CheckModuleDebugify: FAIL"
-; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -passes=verify -S -o - < %s | \
-; RUN:   FileCheck %s -implicit-check-not="CheckModuleDebugify: FAIL"
-
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,strip,check-debugify -S -o - < %s 2>&1 | \
-; RUN:   FileCheck %s -check-prefix=CHECK-WARN
-
-; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -passes=strip -S -o - < %s 2>&1 | \
-; RUN:   FileCheck %s -check-prefix=CHECK-WARN
-
-; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -S -o - < %s 2>&1 | FileCheck %s -check-prefix=PASS
-
-; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -O1 < %s | opt -O2 -o /dev/null
-; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,mem2reg,check-debugify < %s | opt -O2 -o /dev/null
-
 ; CHECK-LABEL: define void @foo
 define void @foo() {
 ; CHECK: ret void, !dbg ![[RET1:.*]]
diff --git a/external/llvm-project/llvm/test/DebugInfo/pr37964.ll b/external/llvm-project/llvm/test/DebugInfo/pr37964.ll
index 63db67d2bd37..9581f1a6b35d 100644
--- a/external/llvm-project/llvm/test/DebugInfo/pr37964.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/pr37964.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -disable-output -debugify-each -passes=gvn < %s 2>&1 | FileCheck %s
-; RUN: opt --experimental-debuginfo-iterators=false -disable-output -debugify-each -passes=gvn < %s 2>&1 | FileCheck %s
 
 ; CHECK-NOT: ERROR: Instruction with empty DebugLoc in function _Z3bazv --  {{%.*}} = phi
 ; CHECK: CheckFunctionDebugify [GVNPass]: PASS
diff --git a/external/llvm-project/llvm/test/DebugInfo/verify-di-preserve.ll b/external/llvm-project/llvm/test/DebugInfo/verify-di-preserve.ll
index 92fc62a0b34c..46d17154ebf6 100644
--- a/external/llvm-project/llvm/test/DebugInfo/verify-di-preserve.ll
+++ b/external/llvm-project/llvm/test/DebugInfo/verify-di-preserve.ll
@@ -1,10 +1,8 @@
 ; RUN: opt %s -verify-debuginfo-preserve -passes=instcombine -disable-output 2>&1 | FileCheck --check-prefix=VERIFY %s
-; RUN: opt --experimental-debuginfo-iterators=false %s -verify-debuginfo-preserve -passes=instcombine -disable-output 2>&1 | FileCheck --check-prefix=VERIFY %s
 
 ; VERIFY: CheckModuleDebugify (original debuginfo):
 
 ; RUN: opt %s -verify-each-debuginfo-preserve -O2 -disable-output 2>&1 | FileCheck --check-prefix=VERIFY-EACH %s
-; RUN: opt %s --experimental-debuginfo-iterators=false -verify-each-debuginfo-preserve -O2 -disable-output 2>&1 | FileCheck --check-prefix=VERIFY-EACH %s
 
 ; VERIFY-EACH: DeadArgumentEliminationPass
 ; VERIFY-EACH: GlobalDCEPass
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive-with-duplicate-member-filenames.test b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive-with-duplicate-member-filenames.test
index 1102339a411c..0b663617d5e0 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive-with-duplicate-member-filenames.test
+++ b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive-with-duplicate-member-filenames.test
@@ -8,7 +8,12 @@
 # RUN:     | FileCheck %s
 #
 # FIXME: Enable this test on Windows/arm64 once that backend is available.
-# UNSUPPORTED: target=aarch64-pc-windows-{{.*}}
+# UNSUPPORTED: target=aarch64-{{.*}}-windows-{{.*}}
+#
+# On MinGW targets, when compiling the main() function, it gets
+# an implicitly generated call to __main(), which is missing in
+# this context.
+# XFAIL: target={{.*}}-windows-gnu
 #
 # Check that synthesized archive member names are unambiguous, even if an
 # archive contains multiple files with the same name.
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive.test b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive.test
index 22c60eb6ac14..e67d2eb9c8d9 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive.test
+++ b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/all-load-multifile-archive.test
@@ -7,11 +7,16 @@
 # RUN:     | FileCheck %s
 #
 # FIXME: Enable this test on Windows/arm64 once that backend is available.
-# UNSUPPORTED: target=aarch64-pc-windows-{{.*}}
+# UNSUPPORTED: target=aarch64-{{.*}}-windows-{{.*}}
+#
+# On MinGW targets, when compiling the main() function, it gets
+# an implicitly generated call to __main(), which is missing in
+# this context.
+# XFAIL: target={{.*}}-windows-gnu
 #
 # Check that the llvm-jitlink -all-load option loads all members of
 # multi-file archives.
 #
 # CHECK-DAG: main{{.*}}main.o
 # CHECK-DAG: foo{{.*}}libFoo.a[0](foo.o)
-# CHECK-DAG: bar{{.*}}libFoo.a[1](bar.o)
\ No newline at end of file
+# CHECK-DAG: bar{{.*}}libFoo.a[1](bar.o)
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/sectcreate.test b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/sectcreate.test
index 08b6372dcf2c..bab0a7d4f2e7 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/sectcreate.test
+++ b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/Generic/sectcreate.test
@@ -6,7 +6,7 @@
 # Use -sectcreate to create a section from a data file.
 
 # Jitlink does not support ARM64 COFF files.
-# UNSUPPORTED: target=aarch64-pc-windows-{{.*}}
+# UNSUPPORTED: target=aarch64-{{.*}}-windows-{{.*}}
 
 # On MinGW targets, when compiling the main() function, it gets
 # an implicitly generated call to __main(), which is missing in
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/RISCV/ELF_branch.s b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/RISCV/ELF_branch.s
index 29d3c5906d20..5d3a048ae69c 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/JITLink/RISCV/ELF_branch.s
+++ b/external/llvm-project/llvm/test/ExecutionEngine/JITLink/RISCV/ELF_branch.s
@@ -1,7 +1,7 @@
 # RUN: rm -rf %t && mkdir -p %t
-# RUN: llvm-mc -triple=riscv64 -filetype=obj -riscv-asm-relax-branches=0 \
+# RUN: llvm-mc -triple=riscv64 -filetype=obj \
 # RUN:     -o %t/elf_riscv64_branch.o %s
-# RUN: llvm-mc -triple=riscv32 -filetype=obj -riscv-asm-relax-branches=0 \
+# RUN: llvm-mc -triple=riscv32 -filetype=obj \
 # RUN:     -o %t/elf_riscv32_branch.o %s
 # RUN: llvm-jitlink -noexec \
 # RUN:     -slab-allocate 100Kb -slab-address 0xfff00ff4 -slab-page-size 4096 \
@@ -13,6 +13,8 @@
 # RUN:     -check %s %t/elf_riscv32_branch.o
 #
 
+        .option exact
+
         .text
 # Empty main entry point.
         .globl  main
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/cross-module-a.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/cross-module-a.ll
index 646881ee269f..3bac9bd0b775 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/cross-module-a.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/cross-module-a.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli -jit-kind=mcjit -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target %s > /dev/null
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/eh.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/eh.ll
index b094ec239494..4be7f0b66fda 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/eh.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/eh.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: cxx-shared-library
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target %s
 ; XFAIL: target=arm{{.*}}, target={{.*-(cygwin|windows-msvc|windows-gnu)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/multi-module-a.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/multi-module-a.ll
index 4d44a52ec984..ed1b9c62bdb6 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/multi-module-a.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/multi-module-a.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli -jit-kind=mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target %s > /dev/null
 ; REQUIRES: thread_support
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/simpletest-remote.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/simpletest-remote.ll
index 4042b31584cb..02d450f05fc0 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/simpletest-remote.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/simpletest-remote.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target %s > /dev/null
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/stubs-remote.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/stubs-remote.ll
index 43e61f120628..30c70b95a91c 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/stubs-remote.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/stubs-remote.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target %s
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-common-symbols-remote.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-common-symbols-remote.ll
index d9a8d85efcd4..6c26001601ae 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-common-symbols-remote.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-common-symbols-remote.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -O0 -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -O0 -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target %s
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-data-align-remote.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-data-align-remote.ll
index 0cd513c0254b..162a19ce58a4 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-data-align-remote.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-data-align-remote.ll
@@ -1,4 +1,4 @@
-; RUN:  %lli -jit-kind=mcjit -remote-mcjit -O0 -mcjit-remote-process=lli-child-target%exeext %s
+; RUN:  %lli -jit-kind=mcjit -remote-mcjit -O0 -mcjit-remote-process=lli-child-target %s
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-fp-no-external-funcs-remote.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-fp-no-external-funcs-remote.ll
index 73e66d3eb48d..2d4d47a65d6c 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-fp-no-external-funcs-remote.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-fp-no-external-funcs-remote.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target %s > /dev/null
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-remote.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-remote.ll
index 71617e0960cc..f0faf07c42cf 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-remote.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-remote.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target %s > /dev/null
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll
index c2c79a9d88d2..6bba022c6078 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-global-init-nonzero-sm-pic.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext \
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target \
 ; RUN:   -relocation-model=pic -code-model=small %s > /dev/null
 ; XFAIL: target={{(mips|mipsel)-.*}}, target={{(aarch64|arm).*}}, target={{(i686|i386).*}}
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-remote.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-remote.ll
index 18e6b0319116..e96da853a8a6 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-remote.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-remote.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -O0 -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -O0 -mcjit-remote-process=lli-child-target %s
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
 ; REQUIRES: thread_support
 ; UNSUPPORTED: target=powerpc64-unknown-linux-gnu
diff --git a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll
index db88cbb4f958..f2337f372ca0 100644
--- a/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll
+++ b/external/llvm-project/llvm/test/ExecutionEngine/MCJIT/remote/test-ptr-reloc-sm-pic.ll
@@ -1,4 +1,4 @@
-; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext \
+; RUN: %lli -jit-kind=mcjit -remote-mcjit -mcjit-remote-process=lli-child-target \
 ; RUN:   -O0 -relocation-model=pic -code-model=small %s
 ; XFAIL: target={{(mips|mipsel)-.*}}, target={{(aarch64|arm).*}}, target={{(i686|i386).*}}
 ; XFAIL: target={{.*-windows-(gnu|msvc)}}
diff --git a/external/llvm-project/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll b/external/llvm-project/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll
index 51bef51d6b35..120a4b235f85 100644
--- a/external/llvm-project/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll
@@ -12,7 +12,7 @@ define void @load.v1i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i8 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK:       7:
 ; CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP1]], 7
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 3
@@ -99,7 +99,7 @@ define void @load.v8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i8 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
 ; CHECK:       10:
 ; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP4]], 7
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i8
@@ -115,7 +115,7 @@ define void @load.v8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
 ; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
-; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
 ; CHECK:       22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
 ; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
@@ -149,7 +149,7 @@ define void @load.v16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i8 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
 ; CHECK:       10:
 ; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP4]], 7
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i8
@@ -165,7 +165,7 @@ define void @load.v16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
 ; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
-; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
 ; CHECK:       22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
 ; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
@@ -197,7 +197,7 @@ define void @store.v1i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i8 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; CHECK:       7:
 ; CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP1]], 7
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 3
@@ -284,7 +284,7 @@ define void @store.v8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i8 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
 ; CHECK:       10:
 ; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP4]], 7
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i8
@@ -300,7 +300,7 @@ define void @store.v8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
 ; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
-; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
 ; CHECK:       22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
 ; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
@@ -334,7 +334,7 @@ define void @store.v16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i8 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
 ; CHECK:       10:
 ; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP4]], 7
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i8
@@ -350,7 +350,7 @@ define void @store.v16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
 ; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
-; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[TMP22:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
 ; CHECK:       22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
 ; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
@@ -414,7 +414,7 @@ define void @load.nxv1i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -430,7 +430,7 @@ define void @load.nxv1i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -471,7 +471,7 @@ define void @load.nxv2i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -487,7 +487,7 @@ define void @load.nxv2i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -528,7 +528,7 @@ define void @load.nxv4i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -544,7 +544,7 @@ define void @load.nxv4i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -585,7 +585,7 @@ define void @load.nxv8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -601,7 +601,7 @@ define void @load.nxv8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -642,7 +642,7 @@ define void @load.nxv16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -658,7 +658,7 @@ define void @load.nxv16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -700,7 +700,7 @@ define void @store.nxv1i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -716,7 +716,7 @@ define void @store.nxv1i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -757,7 +757,7 @@ define void @store.nxv2i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -773,7 +773,7 @@ define void @store.nxv2i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -814,7 +814,7 @@ define void @store.nxv4i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -830,7 +830,7 @@ define void @store.nxv4i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -871,7 +871,7 @@ define void @store.nxv8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -887,7 +887,7 @@ define void @store.nxv8i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -928,7 +928,7 @@ define void @store.nxv16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -944,7 +944,7 @@ define void @store.nxv16i32(ptr %p) sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
@@ -989,7 +989,7 @@ define <vscale x 2 x i32> @local_alloca() sanitize_address {
 ; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[TMP11]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
 ; CHECK:       14:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP8]], 7
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i8
@@ -1005,7 +1005,7 @@ define <vscale x 2 x i32> @local_alloca() sanitize_address {
 ; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
 ; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne i8 [[TMP24]], 0
-; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
 ; CHECK:       26:
 ; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP20]], 7
 ; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i8
diff --git a/external/llvm-project/llvm/test/Instrumentation/BoundsChecking/simple.ll b/external/llvm-project/llvm/test/Instrumentation/BoundsChecking/simple.ll
index ce2540cf49cc..5d8f76753e0c 100644
--- a/external/llvm-project/llvm/test/Instrumentation/BoundsChecking/simple.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/BoundsChecking/simple.ll
@@ -35,8 +35,8 @@ define void @f2() nounwind {
 ; CHECK-NEXT:    store i32 3, ptr [[IDX]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6:[0-9]+]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6:[0-9]+]], !nosanitize [[META0:![0-9]+]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = tail call ptr @malloc(i64 32)
   %idx = getelementptr inbounds i32, ptr %1, i64 8
@@ -49,18 +49,18 @@ define void @f3(i64 %x) nounwind {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 4, [[X:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call ptr @calloc(i64 4, i64 [[X]])
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 8
-; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], 32
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], 32
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP3]], 4
-; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = or i1 false, [[TMP6]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], 32, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], 32, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP3]], 4, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 false, [[TMP6]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[TRAP:%.*]], label [[TMP8:%.*]]
 ; CHECK:       8:
 ; CHECK-NEXT:    store i32 3, ptr [[IDX]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = tail call ptr @calloc(i64 4, i64 %x)
   %idx = getelementptr inbounds i32, ptr %1, i64 8
@@ -85,18 +85,18 @@ define void @f4(i64 %x) nounwind {
 ; CHECK-LABEL: @f4(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call ptr @realloc(ptr null, i64 [[X:%.*]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 8
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[X]], 32
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[X]], 32
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP2]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = or i1 false, [[TMP5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[X]], 32, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[X]], 32, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP2]], 4, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 false, [[TMP5]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[TRAP:%.*]], label [[TMP7:%.*]]
 ; CHECK:       7:
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IDX]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = tail call ptr @realloc(ptr null, i64 %x) nounwind
   %idx = getelementptr inbounds i32, ptr %1, i64 8
@@ -108,17 +108,17 @@ define void @f5(i64 %x) nounwind {
 ; CHECK-LABEL: @f5(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 0, [[X:%.*]]
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [8 x i8], ptr @.str, i64 0, i64 [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 8, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 8, [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 8, [[TMP1]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 8, [[TMP1]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP2]], 1, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[TRAP:%.*]], label [[TMP6:%.*]]
 ; CHECK:       6:
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[IDX]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %idx = getelementptr inbounds [8 x i8], ptr @.str, i64 0, i64 %x
   %1 = load i8, ptr %idx, align 4
@@ -130,17 +130,17 @@ define void @f5_as1(i64 %x) nounwind {
 ; CHECK-NEXT:    [[X_C:%.*]] = trunc i64 [[X:%.*]] to i16
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i16 0, [[X_C]]
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [8 x i8], ptr addrspace(1) @.str_as1, i64 0, i64 [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i16 8, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i16 8, [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i16 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i16 8, [[TMP1]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i16 8, [[TMP1]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i16 [[TMP2]], 1, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[TRAP:%.*]], label [[TMP6:%.*]]
 ; CHECK:       6:
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(1) [[IDX]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %idx = getelementptr inbounds [8 x i8], ptr addrspace(1) @.str_as1, i64 0, i64 %x
   %1 = load i8, ptr addrspace(1) %idx, align 4
@@ -152,17 +152,17 @@ define void @f5_as2(i32 %x) nounwind {;
 ; CHECK-NEXT:    [[X_C:%.*]] = sext i32 [[X:%.*]] to i48
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i48 0, [[X_C]]
 ; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [8 x i8], ptr addrspace(2) @.str_as2, i32 0, i32 [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i48 8, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i48 8, [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i48 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i48 8, [[TMP1]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i48 8, [[TMP1]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i48 [[TMP2]], 1, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[TRAP:%.*]], label [[TMP6:%.*]]
 ; CHECK:       6:
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr addrspace(2) [[IDX]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %idx = getelementptr inbounds [8 x i8], ptr addrspace(2) @.str_as2, i32 0, i32 %x
   %1 = load i8, ptr addrspace(2) %idx, align 4
@@ -184,17 +184,17 @@ define void @f7(i64 %x) nounwind {
 ; CHECK-LABEL: @f7(
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 16, [[X:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca i128, i64 [[X]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 16
-; CHECK-NEXT:    [[TMP5:%.*]] = or i1 false, [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = or i1 false, [[TMP5]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], 0, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 16, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 false, [[TMP4]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 false, [[TMP5]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[TRAP:%.*]], label [[TMP7:%.*]]
 ; CHECK:       7:
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i128, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = alloca i128, i64 %x
   %2 = load i128, ptr %1, align 4
@@ -237,17 +237,17 @@ define void @f10(i64 %x, i64 %y) nounwind {
 ; CHECK-NEXT:    [[TMP4:%.*]] = alloca i128, i64 [[Y]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = select i1 undef, i64 [[TMP1]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 undef, ptr [[TMP2]], ptr [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i64 [[TMP7]], 16
-; CHECK-NEXT:    [[TMP9:%.*]] = or i1 false, [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = or i1 false, [[TMP9]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP5]], 0, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i64 [[TMP7]], 16, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 false, [[TMP8]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 false, [[TMP9]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[TRAP:%.*]], label [[TMP11:%.*]]
 ; CHECK:       11:
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i128, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = alloca i128, i64 %x
   %2 = alloca i128, i64 %y
@@ -264,8 +264,8 @@ define void @f11(ptr byval(i128) %x) nounwind {
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = getelementptr inbounds i8, ptr %x, i64 16
   %2 = load i8, ptr %1, align 4
@@ -280,8 +280,8 @@ define void @f11_as1(ptr addrspace(1) byval(i128) %x) nounwind {
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr addrspace(1) [[TMP1]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = getelementptr inbounds i8, ptr addrspace(1) %x, i16 16
   %2 = load i8, ptr addrspace(1) %1, align 4
@@ -295,19 +295,19 @@ define i64 @f12(i64 %x, i64 %y) nounwind {
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = mul i64 [[Y:%.*]], 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 [[Y]]
-; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP5]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt i64 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP1]], [[TMP3]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP5]], 8, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt i64 [[TMP3]], 0, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP9]], [[TMP8]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[TRAP:%.*]], label [[TMP11:%.*]]
 ; CHECK:       11:
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP4]], align 8
 ; CHECK-NEXT:    ret i64 [[TMP12]]
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = tail call ptr @calloc(i64 1, i64 %x)
   %2 = getelementptr inbounds i64, ptr %1, i64 %y
@@ -369,17 +369,17 @@ define i8 @f14(i1 %i) {
 ; CHECK-NEXT:    [[IND:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ -4, [[BB1]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], [[IND]]
 ; CHECK-NEXT:    [[P:%.*]] = getelementptr i8, ptr [[ALLOC]], i64 [[IND]]
-; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP3]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP0]], [[TMP2]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP3]], 1, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[TRAP:%.*]], label [[TMP7:%.*]]
 ; CHECK:       7:
 ; CHECK-NEXT:    [[RET:%.*]] = load i8, ptr [[P]], align 1
 ; CHECK-NEXT:    ret i8 [[RET]]
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
 entry:
   br i1 %i, label %bb1, label %bb2
@@ -411,17 +411,17 @@ define i8 @f15(i1 %i) {
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 100, [[ENTRY:%.*]] ], [ 16, [[BB1]] ]
 ; CHECK-NEXT:    [[ALLOC:%.*]] = phi ptr [ [[G1]], [[ENTRY]] ], [ [[G2]], [[BB1]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 32, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 32, [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 32, [[TMP0]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 32, [[TMP0]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP1]], 1, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[TMP5:%.*]]
 ; CHECK:       5:
 ; CHECK-NEXT:    [[RET:%.*]] = load i8, ptr [[ALLOC]], align 1
 ; CHECK-NEXT:    ret i8 [[RET]]
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
 entry:
   %a1 = alloca [32 x i8]
@@ -445,17 +445,17 @@ define <4 x i32> @load_vector(i64 %y) nounwind {
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = mul i64 [[Y:%.*]], 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 [[Y]]
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 256, [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 256, [[TMP2]]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP4]], 16
-; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 256, [[TMP2]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 256, [[TMP2]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP4]], 16, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[TRAP:%.*]], label [[TMP8:%.*]]
 ; CHECK:       8:
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMP9]]
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = tail call ptr @calloc(i64 1, i64 256)
   %2 = getelementptr inbounds i64, ptr %1, i64 %y
@@ -469,19 +469,19 @@ define <vscale x 1 x i32> @load_scalable_vector(i64 %y) nounwind {
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = mul i64 [[Y:%.*]], 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 [[Y]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 256, [[TMP2]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 256, [[TMP2]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64(), !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 256, [[TMP2]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 256, [[TMP2]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[TRAP:%.*]], label [[TMP10:%.*]]
 ; CHECK:       10:
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <vscale x 1 x i32>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    ret <vscale x 1 x i32> [[TMP11]]
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = tail call ptr @calloc(i64 1, i64 256)
   %2 = getelementptr inbounds i64, ptr %1, i64 %y
@@ -500,21 +500,21 @@ define void @scalable_alloca(i64 %y) nounwind {
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = mul i64 [[Y:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds <vscale x 4 x i16>, ptr [[TMP4]], i64 [[Y]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
-; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i64 [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[TMP11]], [[TMP10]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i64 [[TMP7]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64(), !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[TMP11]], [[TMP10]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i64 [[TMP7]], 0, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[TMP14]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[TRAP:%.*]], label [[TMP17:%.*]]
 ; CHECK:       17:
 ; CHECK-NEXT:    [[TMP18:%.*]] = load <vscale x 4 x i16>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = alloca <vscale x 4 x i16>, i32 5
   %2 = getelementptr inbounds <vscale x 4 x i16>, ptr %1, i64 %y
@@ -533,21 +533,21 @@ define void @scalable_alloca2(i64 %y) nounwind {
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = mul i64 [[Y:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds <vscale x 4 x i64>, ptr [[TMP4]], i64 [[Y]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 32
-; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i64 [[TMP3]], [[TMP7]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[TMP11]], [[TMP10]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i64 [[TMP7]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64(), !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 32, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[TMP11]], [[TMP10]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]], !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i64 [[TMP7]], 0, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[TMP14]], !nosanitize [[META0]]
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[TRAP:%.*]], label [[TMP17:%.*]]
 ; CHECK:       17:
 ; CHECK-NEXT:    [[TMP18:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       trap:
-; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    call void @llvm.trap() #[[ATTR6]], !nosanitize [[META0]]
+; CHECK-NEXT:    unreachable, !nosanitize [[META0]]
 ;
   %1 = alloca <vscale x 4 x i64>
   %2 = getelementptr inbounds <vscale x 4 x i64>, ptr %1, i64 %y
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
index d1655bfbb4d2..c4016e749e70 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
@@ -4,12 +4,6 @@
 ; Handled strictly:
 ; - i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %a0)
 ; - i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %a0)
-; - <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1)
-; - <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1)
-; - <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
-; - <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1)
-; - <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a2)
-; - <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1)
 ; - void @llvm.x86.avx.vzeroall()
 ; - void @llvm.x86.avx.vzeroupper()
 
@@ -956,19 +950,19 @@ define <2 x double> @test_x86_avx_vpermilvar_pd(<2 x double> %a0, <2 x i64> %a1)
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <2 x i64> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A0]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x double> [[RES]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A0:%.*]], <2 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <2 x double> [[RES]]
+; CHECK:       8:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A2:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES1]]
 ;
   %res = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1) ; <<2 x double>> [#uses=1]
   ret <2 x double> %res
@@ -981,19 +975,19 @@ define <4 x double> @test_x86_avx_vpermilvar_pd_256(<4 x double> %a0, <4 x i64>
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0:%.*]], <4 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x double> [[RES]]
+; CHECK:       8:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A2:%.*]], <4 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES1]]
 ;
   %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1) ; <<4 x double>> [#uses=1]
   ret <4 x double> %res
@@ -1004,16 +998,12 @@ define <4 x double> @test_x86_avx_vpermilvar_pd_256_2(<4 x double> %a0) #0 {
 ; CHECK-LABEL: @test_x86_avx_vpermilvar_pd_256_2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
-; CHECK:       3:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       4:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0:%.*]], <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x double> [[RES]]
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0]], <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A1:%.*]], <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES1]]
 ;
   %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 0, i64 2>) ; <<4 x double>> [#uses=1]
   ret <4 x double> %res
@@ -1024,19 +1014,19 @@ define <4 x float> @test_x86_avx_vpermilvar_ps(<4 x float> %a0, <4 x i32> %a1) #
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0:%.*]], <4 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x float> [[RES]]
+; CHECK:       8:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A2:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES1]]
 ;
   %res = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1) ; <<4 x float>> [#uses=1]
   ret <4 x float> %res
@@ -1057,19 +1047,19 @@ define <4 x float> @test_x86_avx_vpermilvar_ps_load(<4 x float> %a0, ptr %a1) #0
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
-; CHECK:       10:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       11:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0:%.*]], <4 x i32> [[A2]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x float> [[RES]]
+; CHECK:       13:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A3:%.*]], <4 x i32> [[A2]])
+; CHECK-NEXT:    store <4 x i32> [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES1]]
 ;
   %a2 = load <4 x i32>, ptr %a1
   %res = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a2) ; <<4 x float>> [#uses=1]
@@ -1083,19 +1073,19 @@ define <8 x float> @test_x86_avx_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A0]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[RES]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <8 x float> [[RES]]
+; CHECK:       8:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A2:%.*]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES1]]
 ;
   %res = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1]
   ret <8 x float> %res
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
index f916130fe53e..9649f2dc71f1 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
@@ -740,8 +740,15 @@ define <32 x i8> @test_x86_avx2_pshuf_b(<32 x i8> %a0, <32 x i8> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[TMP1]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i8> [[RES]]
 ;
@@ -969,8 +976,15 @@ define <8 x i32> @test_x86_avx2_permd(<8 x i32> %a0, <8 x i32> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> [[TMP1]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> [[A0:%.*]], <8 x i32> [[A1]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
 ;
@@ -985,18 +999,18 @@ define <8 x float> @test_x86_avx2_permps(<8 x float> %a0, <8 x i32> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[TMP3]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[TMP6]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[A0:%.*]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES]]
 ;
   %res = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1]
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
index 2ce2c076f5ad..3eeb5886b5fc 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
@@ -8143,19 +8143,19 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x double> [[TMP7]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <8 x double> [[TMP7]]
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[TMP5]]
 ;
   %res = call <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
   ret <8 x double> %res
@@ -8169,27 +8169,27 @@ define <8 x double>@test_int_x86_avx512_mask_vpermilvar_pd_512(<8 x double> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X4:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
-; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP6]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP7]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP17]], <8 x i64> [[TMP12]]
-; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP9]], <8 x double> [[X2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP7]], <8 x double> [[X2]]
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP18]]
 ;
@@ -8204,26 +8204,26 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermilvar_pd_512(<8 x double> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP6]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
-; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP6]], <8 x double> zeroinitializer
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP16]]
 ;
@@ -8238,19 +8238,19 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512(<16 x float> %x0, <16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <16 x float> [[TMP7]]
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[TMP5]]
 ;
   %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
   ret <16 x float> %res
@@ -8264,27 +8264,27 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512(<16 x float> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X4:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
-; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP6]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
-; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[X2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP7]], <16 x float> [[X2]]
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP18]]
 ;
@@ -8300,26 +8300,26 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermilvar_ps_512(<16 x float> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP6]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
-; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP6]], <16 x float> zeroinitializer
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP16]]
 ;
@@ -8335,52 +8335,40 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(<16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP33:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X1:%.*]], <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> zeroinitializer, <16 x i32> [[TMP3]]
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP6]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[TMP33]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP3]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP15]], <16 x i32> [[TMP10]]
-; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP9]], <16 x float> [[TMP7]], <16 x float> [[X2]]
-; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP17]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP3]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
-; CHECK:       18:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       19:
-; CHECK-NEXT:    [[TMP20:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP9]], <16 x float> [[TMP33]], <16 x float> [[X2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP18:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[TMP17]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x float> [[TMP18]] to <16 x i32>
+; CHECK-NEXT:    [[TMP20:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X1]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>)
 ; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i16 [[X3]] to <16 x i1>
-; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP22]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = select <16 x i1> [[TMP22]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x float> [[TMP20]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP25:%.*]] = xor <16 x i32> [[TMP24]], zeroinitializer
-; CHECK-NEXT:    [[TMP26:%.*]] = or <16 x i32> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = or <16 x i32> [[TMP25]], [[TMP19]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = or <16 x i32> [[TMP26]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP27]], <16 x i32> [[TMP23]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = select <16 x i1> [[TMP22]], <16 x float> [[TMP20]], <16 x float> zeroinitializer
-; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP29]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
-; CHECK:       30:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       31:
-; CHECK-NEXT:    [[TMP32:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP30:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[TMP29]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x float> [[TMP30]] to <16 x i32>
+; CHECK-NEXT:    [[TMP32:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X1]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[_MSPROP_SELECT]], [[_MSPROP_SELECT1]]
 ; CHECK-NEXT:    [[RES3:%.*]] = fadd <16 x float> [[TMP16]], [[TMP28]]
-; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> zeroinitializer, [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <16 x i32> [[TMP31]], [[_MSPROP]]
 ; CHECK-NEXT:    [[RES4:%.*]] = fadd <16 x float> [[TMP32]], [[RES3]]
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[RES4]]
@@ -13183,18 +13171,18 @@ define <8 x double>@test_int_x86_avx512_permvar_df_512(<8 x double> %x0, <8 x i6
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[TMP3]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x double> [[TMP6]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP7]]
 ;
   %res = call <8 x double> @llvm.x86.avx512.mask.permvar.df.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
@@ -13209,24 +13197,24 @@ define <8 x double>@test_int_x86_avx512_mask_permvar_df_512(<8 x double> %x0, <8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[TMP5]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP7]], <8 x i64> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP17]], <8 x i64> [[TMP12]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP9]], <8 x double> [[X2]]
@@ -13244,23 +13232,23 @@ define <8 x double>@test_int_x86_avx512_maskz_permvar_df_512(<8 x double> %x0, <
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[TMP4]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x double> [[TMP7]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP6]], <8 x i64> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
@@ -13278,8 +13266,15 @@ define <8 x i64>@test_int_x86_avx512_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
 ;
@@ -13295,8 +13290,15 @@ define <8 x i64>@test_int_x86_avx512_mask_permvar_di_512(<8 x i64> %x0, <8 x i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
@@ -13319,8 +13321,15 @@ define <8 x i64>@test_int_x86_avx512_maskz_permvar_di_512(<8 x i64> %x0, <8 x i6
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
@@ -13343,18 +13352,18 @@ define <16 x float>@test_int_x86_avx512_permvar_sf_512(<16 x float> %x0, <16 x i
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[TMP3]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[TMP6]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP7]]
 ;
   %res = call <16 x float> @llvm.x86.avx512.mask.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
@@ -13369,24 +13378,24 @@ define <16 x float>@test_int_x86_avx512_mask_permvar_sf_512(<16 x float> %x0, <1
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[TMP5]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP7]], <16 x i32> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[X2]]
@@ -13404,23 +13413,23 @@ define <16 x float>@test_int_x86_avx512_maskz_permvar_sf_512(<16 x float> %x0, <
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[TMP4]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP6]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
@@ -13438,8 +13447,15 @@ define <16 x i32>@test_int_x86_avx512_permvar_si_512(<16 x i32> %x0, <16 x i32>
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
 ;
@@ -13455,8 +13471,15 @@ define <16 x i32>@test_int_x86_avx512_mask_permvar_si_512(<16 x i32> %x0, <16 x
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
@@ -13479,8 +13502,15 @@ define <16 x i32>@test_int_x86_avx512_maskz_permvar_si_512(<16 x i32> %x0, <16 x
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
@@ -13712,8 +13742,8 @@ define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
@@ -13726,9 +13756,15 @@ define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP4]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X4:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[TMP4]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP14]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X4:%.*]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
 ;
@@ -13756,9 +13792,15 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP18]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
@@ -13780,25 +13822,23 @@ declare <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double>, <8 x
 define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP4]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP8]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <8 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP9]]
 ;
   %res = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
@@ -13809,32 +13849,30 @@ define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0,
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP5]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP8]], <8 x i64> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x double> [[TMP10]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <8 x i64> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP19]], <8 x i64> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <8 x i1> [[TMP13]], <8 x double> [[TMP10]], <8 x double> [[TMP11]]
@@ -13850,25 +13888,23 @@ declare <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float>, <16 x
 define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP4]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP8]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <16 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP9]]
 ;
   %res = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
@@ -13879,32 +13915,30 @@ define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0,
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> zeroinitializer, <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP8]], <16 x i32> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP19]], <16 x i32> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP13]], <16 x float> [[TMP10]], <16 x float> [[TMP11]]
@@ -13920,12 +13954,18 @@ declare <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64>, <8 x i64>, <
 define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP8]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
 ;
@@ -13937,13 +13977,19 @@ define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP2]]
@@ -13980,9 +14026,15 @@ define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X0:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP18]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
@@ -14011,7 +14063,7 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
 ; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
@@ -14025,26 +14077,24 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[X2INS:%.*]] = insertelement <8 x double> [[EXTRA_PARAM:%.*]], double [[X2S]], i32 0
 ; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[X2:%.*]] = shufflevector <8 x double> [[X2INS]], <8 x double> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP2]] to <8 x double>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP13:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP11]], <8 x i64> [[X0:%.*]], <8 x double> [[TMP24]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[TMP13]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
 ; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP14:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
-; CHECK:       14:
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP25:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; CHECK:       16:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       15:
-; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x double> [[X2]])
+; CHECK:       17:
+; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0]], <8 x double> [[X2]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP14]], <8 x i64> zeroinitializer
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x double> [[TMP15]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i64> [[TMP22]], <8 x i64> [[TMP18]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP17]], <8 x double> [[TMP15]], <8 x double> zeroinitializer
@@ -14064,30 +14114,28 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0,
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X0:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP9]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP20:%.*]], label [[TMP21:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x float> [[X2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP14]], zeroinitializer
-; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP17]], <16 x i32> [[TMP13]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP12]], <16 x float> [[TMP10]], <16 x float> zeroinitializer
@@ -14105,13 +14153,19 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X0:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i64> [[TMP13]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
@@ -14132,12 +14186,18 @@ declare <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32>, <16 x i32>
 define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP8]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
 ;
@@ -14149,13 +14209,19 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[TMP13]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
index d6fa08cd61f5..4b559bc9fb8e 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
@@ -5467,9 +5467,15 @@ define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
 ; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
 ;
@@ -5496,9 +5502,15 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP18]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
@@ -5522,24 +5534,22 @@ declare <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double>, <8 x i64>,
 define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP4]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP8]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP12:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP9]]
 ;
   %1 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)
@@ -5549,32 +5559,30 @@ define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x
 define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP5]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP8]], <8 x i64> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x double> [[TMP10]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <8 x i64> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP19]], <8 x i64> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <8 x i1> [[TMP13]], <8 x double> [[TMP10]], <8 x double> [[TMP11]]
@@ -5593,24 +5601,22 @@ declare <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float>, <16 x i32>
 define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP4]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP8]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP12:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP9]]
 ;
   %1 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)
@@ -5620,32 +5626,30 @@ define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16
 define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> zeroinitializer, <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP8]], <16 x i32> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP19]], <16 x i32> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP13]], <16 x float> [[TMP10]], <16 x float> [[TMP11]]
@@ -5664,12 +5668,18 @@ declare <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i
 define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP8]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
 ;
@@ -5680,13 +5690,19 @@ define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %
 define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP2]]
@@ -5722,9 +5738,15 @@ define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X0:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP18]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
@@ -5753,7 +5775,7 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
 ; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
@@ -5767,26 +5789,24 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[X2INS:%.*]] = insertelement <8 x double> [[EXTRA_PARAM:%.*]], double [[X2S]], i32 0
 ; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP6]], <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[X2:%.*]] = shufflevector <8 x double> [[X2INS]], <8 x double> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i64> [[TMP2]] to <8 x double>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP24]], <8 x i64> [[X0:%.*]], <8 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <8 x double> [[TMP14]] to <8 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
 ; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
-; CHECK:       15:
+; CHECK-NEXT:    br i1 [[_MSCMP4]], label [[TMP26:%.*]], label [[TMP27:%.*]], !prof [[PROF1]]
+; CHECK:       17:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       16:
-; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x double> [[X2]])
+; CHECK:       18:
+; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0]], <8 x double> [[X2]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP25]], <8 x i64> zeroinitializer
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x double> [[TMP15]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP25]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i64> [[TMP22]], <8 x i64> [[TMP18]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP17]], <8 x double> [[TMP15]], <8 x double> zeroinitializer
@@ -5805,30 +5825,28 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X0:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP9]] to i512
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP20:%.*]], label [[TMP21:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x float> [[X2:%.*]])
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP14]], zeroinitializer
-; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP17]], <16 x i32> [[TMP13]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP12]], <16 x float> [[TMP10]], <16 x float> zeroinitializer
@@ -5844,13 +5862,19 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0,
 define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X0:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i64> [[TMP13]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
@@ -5871,12 +5895,18 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x
 define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP8]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
 ;
@@ -5887,13 +5917,19 @@ define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32
 define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x i32> [[TMP13]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
@@ -8657,19 +8693,19 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <8 x double> [[RES]]
+; CHECK:       8:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x double> [[RES1]]
 ;
   %res = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1)
   ret <8 x double> %res
@@ -8682,27 +8718,27 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_mask(<8 x double> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X3:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP6]], <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x double> [[RES1]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP10]]
-; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x double> [[RES]], <8 x double> [[X2]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x double> [[RES1]], <8 x double> [[X2]]
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[RES2]]
 ;
@@ -8718,26 +8754,26 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_maskz(<8 x double> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x i64> [[TMP5]], <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x double> [[RES1]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor <8 x i64> [[TMP10]], zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i64> [[TMP11]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i64> [[TMP12]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i64> [[TMP13]], <8 x i64> [[TMP9]]
-; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x double> [[RES]], <8 x double> zeroinitializer
+; CHECK-NEXT:    [[RES2:%.*]] = select <8 x i1> [[MASK_CAST]], <8 x double> [[RES1]], <8 x double> zeroinitializer
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[RES2]]
 ;
@@ -8754,19 +8790,19 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512(<16 x float> %x0, <16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <16 x float> [[RES]]
+; CHECK:       8:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES1]]
 ;
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1)
   ret <16 x float> %res
@@ -8779,27 +8815,27 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_mask(<16 x float> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X3:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP6]], <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x float> [[RES1]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP10]]
-; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> [[X2]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES1]], <16 x float> [[X2]]
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[RES2]]
 ;
@@ -8815,26 +8851,26 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_maskz(<16 x float> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[RES1]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP10]], zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP13]], <16 x i32> [[TMP9]]
-; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> zeroinitializer
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES1]], <16 x float> zeroinitializer
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[RES2]]
 ;
@@ -8848,16 +8884,12 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool(<16 x fl
 ; CHECK-LABEL: @test_int_x86_avx512_vpermilvar_ps_512_constant_pool(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
-; CHECK:       3:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       4:
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <16 x float> [[RES]]
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X1:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    store <16 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES1]]
 ;
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
   ret <16 x float> %res
@@ -8869,24 +8901,20 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask(<16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X1:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> [[TMP3]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP6]], <16 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x float> [[RES1]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor <16 x i32> [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = or <16 x i32> [[TMP11]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = or <16 x i32> [[TMP12]], [[TMP3]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP13]], <16 x i32> [[TMP8]]
-; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> [[X2]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES1]], <16 x float> [[X2]]
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[RES2]]
 ;
@@ -8901,23 +8929,19 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz(<1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X1:%.*]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP2]] to <16 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x i32> [[TMP5]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[RES1]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i32> [[TMP8]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i32> [[TMP9]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i32> [[TMP10]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP11]], <16 x i32> [[TMP7]]
-; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES]], <16 x float> zeroinitializer
+; CHECK-NEXT:    [[RES2:%.*]] = select <16 x i1> [[MASK_CAST]], <16 x float> [[RES1]], <16 x float> zeroinitializer
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[RES2]]
 ;
@@ -9453,18 +9477,18 @@ define <8 x double>@test_int_x86_avx512_permvar_df_512(<8 x double> %x0, <8 x i6
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[TMP3]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x double> [[TMP6]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP7]]
 ;
   %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %x0, <8 x i64> %x1)
@@ -9478,24 +9502,24 @@ define <8 x double>@test_int_x86_avx512_mask_permvar_df_512(<8 x double> %x0, <8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[TMP5]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP11]], <8 x i64> [[TMP7]], <8 x i64> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[X2:%.*]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i64> [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i64> [[TMP15]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i64> [[TMP16]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP17]], <8 x i64> [[TMP12]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP11]], <8 x double> [[TMP9]], <8 x double> [[X2]]
@@ -9514,23 +9538,23 @@ define <8 x double>@test_int_x86_avx512_maskz_permvar_df_512(<8 x double> %x0, <
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[TMP4]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x double> [[TMP7]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP10]], <8 x i64> [[TMP6]], <8 x i64> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i64> [[TMP12]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i64> [[TMP13]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i64> [[TMP14]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP9]], <8 x i64> [[TMP15]], <8 x i64> [[TMP11]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP10]], <8 x double> [[TMP8]], <8 x double> zeroinitializer
@@ -9550,8 +9574,15 @@ define <8 x i64>@test_int_x86_avx512_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
 ;
@@ -9566,8 +9597,15 @@ define <8 x i64>@test_int_x86_avx512_mask_permvar_di_512(<8 x i64> %x0, <8 x i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP]], <8 x i64> [[TMP4]]
@@ -9591,8 +9629,15 @@ define <8 x i64>@test_int_x86_avx512_maskz_permvar_di_512(<8 x i64> %x0, <8 x i6
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP6]], <8 x i64> [[_MSPROP]], <8 x i64> zeroinitializer
@@ -9617,18 +9662,18 @@ define <16 x float>@test_int_x86_avx512_permvar_sf_512(<16 x float> %x0, <16 x i
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[TMP3]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[TMP6]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP7]]
 ;
   %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1)
@@ -9642,24 +9687,24 @@ define <16 x float>@test_int_x86_avx512_mask_permvar_sf_512(<16 x float> %x0, <1
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[TMP5]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       9:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP7]], <16 x i32> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[X2:%.*]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP13]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP17]], <16 x i32> [[TMP12]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP11]], <16 x float> [[TMP9]], <16 x float> [[X2]]
@@ -9678,23 +9723,23 @@ define <16 x float>@test_int_x86_avx512_maskz_permvar_sf_512(<16 x float> %x0, <
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[TMP4]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP6]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP9]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP10]], <16 x float> [[TMP8]], <16 x float> zeroinitializer
@@ -9714,8 +9759,15 @@ define <16 x i32>@test_int_x86_avx512_permvar_si_512(<16 x i32> %x0, <16 x i32>
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
 ;
@@ -9730,8 +9782,15 @@ define <16 x i32>@test_int_x86_avx512_mask_permvar_si_512(<16 x i32> %x0, <16 x
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP]], <16 x i32> [[TMP4]]
@@ -9755,8 +9814,15 @@ define <16 x i32>@test_int_x86_avx512_maskz_permvar_si_512(<16 x i32> %x0, <16 x
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[TMP1]], <16 x i32> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[_MSPROP]], <16 x i32> zeroinitializer
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
new file mode 100644
index 000000000000..40b5e9338e45
--- /dev/null
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
@@ -0,0 +1,12501 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx512f -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
+
+; Strictly handled instructions:
+; * llvm.x86.avx512.mask.cmp.pd
+; * llvm.x86.avx512.mask.cmp.ps
+; * llvm.x86.avx512.mask.compress
+; * llvm.x86.avx512.mask.cvtpd2dq
+; * llvm.x86.avx512.mask.cvtp
+; * llvm.x86.avx512.mask.cvtpd2udq
+; * llvm.x86.avx512.mask.cvtps2dq
+; * llvm.x86.avx512.mask.cvtps2udq
+; * llvm.x86.avx512.mask.cvttpd2dq
+; * llvm.x86.avx512.mask.cvttpd2udq
+; * llvm.x86.avx512.mask.cvttps2udq
+; * llvm.x86.avx512.mask.expand
+; * llvm.x86.avx512.mask.fixupimm.pd
+; * llvm.x86.avx512.mask.fixupimm.ps
+; * llvm.x86.avx512.mask.getexp.pd
+; * llvm.x86.avx512.mask.getexp.ps
+; * llvm.x86.avx512.mask.getmant.pd
+; * llvm.x86.avx512.mask.getmant.ps
+; * llvm.x86.avx512.mask.pmov.db
+; * llvm.x86.avx512.mask.pmov.db.mem
+; * llvm.x86.avx512.mask.pmov.dw
+; * llvm.x86.avx512.mask.pmov.dw.mem
+; * llvm.x86.avx512.mask.pmov.qb
+; * llvm.x86.avx512.mask.pmov.qb.mem
+; * llvm.x86.avx512.mask.pmov.qd
+; * llvm.x86.avx512.mask.pmov.qd.mem
+; * llvm.x86.avx512.mask.pmov.qw
+; * llvm.x86.avx512.mask.pmov.qw.mem
+; * llvm.x86.avx512.mask.pmovs.db
+; * llvm.x86.avx512.mask.pmovs.db.mem
+; * llvm.x86.avx512.mask.pmovs.dw
+; * llvm.x86.avx512.mask.pmovs.dw.mem
+; * llvm.x86.avx512.mask.pmovs.qb
+; * llvm.x86.avx512.mask.pmovs.qb.mem
+; * llvm.x86.avx512.mask.pmovs.qd
+; * llvm.x86.avx512.mask.pmovs.qd.mem
+; * llvm.x86.avx512.mask.pmovs.qw
+; * llvm.x86.avx512.mask.pmovs.qw.mem
+; * llvm.x86.avx512.mask.pmovus.db
+; * llvm.x86.avx512.mask.pmovus.db.mem
+; * llvm.x86.avx512.mask.pmovus.dw
+; * llvm.x86.avx512.mask.pmovus.dw.mem
+; * llvm.x86.avx512.mask.pmovus.qb
+; * llvm.x86.avx512.mask.pmovus.qb.mem
+; * llvm.x86.avx512.mask.pmovus.qd
+; * llvm.x86.avx512.mask.pmovus.qd.mem
+; * llvm.x86.avx512.mask.pmovus.qw
+; * llvm.x86.avx512.mask.pmovus.qw.mem
+; * llvm.x86.avx512.mask.rndscale.pd
+; * llvm.x86.avx512.mask.rndscale.ps
+; * llvm.x86.avx512.mask.scalef.pd
+; * llvm.x86.avx512.mask.scalef.ps
+; * llvm.x86.avx512.mask.vcvtps2ph
+; * llvm.x86.avx512.maskz.fixupimm.pd
+; * llvm.x86.avx512.maskz.fixupimm.ps
+; * llvm.x86.avx512.pternlog.d
+; * llvm.x86.avx512.pternlog.q
+; * llvm.x86.avx512.rcp14.pd
+; * llvm.x86.avx512.rcp14.ps
+; * llvm.x86.avx512.rsqrt14.pd
+; * llvm.x86.avx512.rsqrt14.ps
+;
+; Heuristically handled instructions:
+; * llvm.fma.v2f64
+; * llvm.fma.v4f32
+; * llvm.fma.v4f64
+; * llvm.fma.v8f32
+; * llvm.x86.avx.max.ps.256
+; * llvm.x86.avx.min.ps.256
+; * llvm.x86.sse.max.ps
+; * llvm.x86.sse.min.ps
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <2 x double> @test_mask_compress_pd_128(<2 x double> %data, <2 x double> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_compress_pd_128(
+; CHECK-SAME: <2 x double> [[DATA:%.*]], <2 x double> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i2 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> [[DATA]], <2 x double> [[PASSTHRU]], <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> %passthru, <2 x i1> %extract)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_maskz_compress_pd_128(<2 x double> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_maskz_compress_pd_128(
+; CHECK-SAME: <2 x double> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> [[DATA]], <2 x double> zeroinitializer, <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> zeroinitializer, <2 x i1> %extract)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_compress_pd_128(<2 x double> %data, <2 x double> %data2) #0 {
+; CHECK-LABEL: define <2 x double> @test_compress_pd_128(
+; CHECK-SAME: <2 x double> [[DATA:%.*]], <2 x double> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> [[DATA]], <2 x double> [[DATA2]], <2 x i1> splat (i1 true))
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> %data2, <2 x i1> <i1 true, i1 true>)
+  ret <2 x double> %1
+}
+
+define <4 x float> @test_mask_compress_ps_128(<4 x float> %data, <4 x float> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_compress_ps_128(
+; CHECK-SAME: <4 x float> [[DATA:%.*]], <4 x float> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> [[DATA]], <4 x float> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> %passthru, <4 x i1> %extract)
+  ret <4 x float> %2
+}
+
+define <4 x float> @test_maskz_compress_ps_128(<4 x float> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_maskz_compress_ps_128(
+; CHECK-SAME: <4 x float> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> [[DATA]], <4 x float> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> zeroinitializer, <4 x i1> %extract)
+  ret <4 x float> %2
+}
+
+define <4 x float> @test_compress_ps_128(<4 x float> %data, <4 x float> %data2) #0 {
+; CHECK-LABEL: define <4 x float> @test_compress_ps_128(
+; CHECK-SAME: <4 x float> [[DATA:%.*]], <4 x float> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> [[DATA]], <4 x float> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x float> %1
+}
+
+define <2 x i64> @test_mask_compress_q_128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_mask_compress_q_128(
+; CHECK-SAME: <2 x i64> [[DATA:%.*]], <2 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i2 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> [[DATA]], <2 x i64> [[PASSTHRU]], <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> %passthru, <2 x i1> %extract)
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_maskz_compress_q_128(<2 x i64> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_maskz_compress_q_128(
+; CHECK-SAME: <2 x i64> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> [[DATA]], <2 x i64> zeroinitializer, <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> zeroinitializer, <2 x i1> %extract)
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_compress_q_128(<2 x i64> %data, <2 x i64> %data2) #0 {
+; CHECK-LABEL: define <2 x i64> @test_compress_q_128(
+; CHECK-SAME: <2 x i64> [[DATA:%.*]], <2 x i64> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> [[DATA]], <2 x i64> [[DATA2]], <2 x i1> splat (i1 true))
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> %data2, <2 x i1> <i1 true, i1 true>)
+  ret <2 x i64> %1
+}
+
+define <4 x i32> @test_mask_compress_d_128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_mask_compress_d_128(
+; CHECK-SAME: <4 x i32> [[DATA:%.*]], <4 x i32> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> [[DATA]], <4 x i32> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> %passthru, <4 x i1> %extract)
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_maskz_compress_d_128(<4 x i32> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_maskz_compress_d_128(
+; CHECK-SAME: <4 x i32> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> [[DATA]], <4 x i32> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_compress_d_128(<4 x i32> %data, <4 x i32> %data2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_compress_d_128(
+; CHECK-SAME: <4 x i32> [[DATA:%.*]], <4 x i32> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> [[DATA]], <4 x i32> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i32> %1
+}
+
+define <2 x double> @test_expand_pd_128(<2 x double> %data, <2 x double> %data2) #0 {
+; CHECK-LABEL: define <2 x double> @test_expand_pd_128(
+; CHECK-SAME: <2 x double> [[DATA:%.*]], <2 x double> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> [[DATA]], <2 x double> [[DATA2]], <2 x i1> splat (i1 true))
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> %data2, <2 x i1> <i1 true, i1 true>)
+  ret <2 x double> %1
+}
+
+define <2 x double> @test_mask_expand_pd_128(<2 x double> %data, <2 x double> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_expand_pd_128(
+; CHECK-SAME: <2 x double> [[DATA:%.*]], <2 x double> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i2 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> [[DATA]], <2 x double> [[PASSTHRU]], <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> %passthru, <2 x i1> %extract)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_maskz_expand_pd_128(<2 x double> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_maskz_expand_pd_128(
+; CHECK-SAME: <2 x double> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> [[DATA]], <2 x double> zeroinitializer, <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> zeroinitializer, <2 x i1> %extract)
+  ret <2 x double> %2
+}
+
+define <4 x float> @test_expand_ps_128(<4 x float> %data, <4 x float> %data2) #0 {
+; CHECK-LABEL: define <4 x float> @test_expand_ps_128(
+; CHECK-SAME: <4 x float> [[DATA:%.*]], <4 x float> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> [[DATA]], <4 x float> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_mask_expand_ps_128(<4 x float> %data, <4 x float> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_expand_ps_128(
+; CHECK-SAME: <4 x float> [[DATA:%.*]], <4 x float> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> [[DATA]], <4 x float> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> %passthru, <4 x i1> %extract)
+  ret <4 x float> %2
+}
+
+define <4 x float> @test_maskz_expand_ps_128(<4 x float> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_maskz_expand_ps_128(
+; CHECK-SAME: <4 x float> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> [[DATA]], <4 x float> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> zeroinitializer, <4 x i1> %extract)
+  ret <4 x float> %2
+}
+
+define <2 x i64> @test_expand_q_128(<2 x i64> %data, <2 x i64> %data2) #0 {
+; CHECK-LABEL: define <2 x i64> @test_expand_q_128(
+; CHECK-SAME: <2 x i64> [[DATA:%.*]], <2 x i64> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> [[DATA]], <2 x i64> [[DATA2]], <2 x i1> splat (i1 true))
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> %data2, <2 x i1> <i1 true, i1 true>)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_mask_expand_q_128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_mask_expand_q_128(
+; CHECK-SAME: <2 x i64> [[DATA:%.*]], <2 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i2 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> [[DATA]], <2 x i64> [[PASSTHRU]], <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> %passthru, <2 x i1> %extract)
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_maskz_expand_q_128(<2 x i64> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_maskz_expand_q_128(
+; CHECK-SAME: <2 x i64> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i1> [[_MSPROP]] to i2
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> [[DATA]], <2 x i64> zeroinitializer, <2 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> zeroinitializer, <2 x i1> %extract)
+  ret <2 x i64> %2
+}
+
+define <4 x i32> @test_expand_d_128(<4 x i32> %data, <4 x i32> %data2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_expand_d_128(
+; CHECK-SAME: <4 x i32> [[DATA:%.*]], <4 x i32> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> [[DATA]], <4 x i32> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_mask_expand_d_128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_mask_expand_d_128(
+; CHECK-SAME: <4 x i32> [[DATA:%.*]], <4 x i32> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> [[DATA]], <4 x i32> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> %passthru, <4 x i1> %extract)
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_maskz_expand_d_128(<4 x i32> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_maskz_expand_d_128(
+; CHECK-SAME: <4 x i32> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> [[DATA]], <4 x i32> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i32> %2
+}
+
+define <4 x double> @test_mask_compress_pd_256(<4 x double> %data, <4 x double> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_compress_pd_256(
+; CHECK-SAME: <4 x double> [[DATA:%.*]], <4 x double> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> [[DATA]], <4 x double> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> %passthru, <4 x i1> %extract)
+  ret <4 x double> %2
+}
+
+define <4 x double> @test_maskz_compress_pd_256(<4 x double> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_maskz_compress_pd_256(
+; CHECK-SAME: <4 x double> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> [[DATA]], <4 x double> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> zeroinitializer, <4 x i1> %extract)
+  ret <4 x double> %2
+}
+
+define <4 x double> @test_compress_pd_256(<4 x double> %data, <4 x double> %data2) #0 {
+; CHECK-LABEL: define <4 x double> @test_compress_pd_256(
+; CHECK-SAME: <4 x double> [[DATA:%.*]], <4 x double> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> [[DATA]], <4 x double> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x double> %1
+}
+
+define <8 x float> @test_mask_compress_ps_256(<8 x float> %data, <8 x float> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mask_compress_ps_256(
+; CHECK-SAME: <8 x float> [[DATA:%.*]], <8 x float> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> [[DATA]], <8 x float> [[PASSTHRU]], <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> %passthru, <8 x i1> %1)
+  ret <8 x float> %2
+}
+
+define <8 x float> @test_maskz_compress_ps_256(<8 x float> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_maskz_compress_ps_256(
+; CHECK-SAME: <8 x float> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> [[DATA]], <8 x float> zeroinitializer, <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> zeroinitializer, <8 x i1> %1)
+  ret <8 x float> %2
+}
+
+define <8 x float> @test_compress_ps_256(<8 x float> %data, <8 x float> %data2) #0 {
+; CHECK-LABEL: define <8 x float> @test_compress_ps_256(
+; CHECK-SAME: <8 x float> [[DATA:%.*]], <8 x float> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> [[DATA]], <8 x float> [[DATA2]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> %data2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x float> %1
+}
+
+define <4 x i64> @test_mask_compress_q_256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_mask_compress_q_256(
+; CHECK-SAME: <4 x i64> [[DATA:%.*]], <4 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> [[DATA]], <4 x i64> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> %passthru, <4 x i1> %extract)
+  ret <4 x i64> %2
+}
+
+define <4 x i64> @test_maskz_compress_q_256(<4 x i64> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_maskz_compress_q_256(
+; CHECK-SAME: <4 x i64> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> [[DATA]], <4 x i64> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i64> %2
+}
+
+define <4 x i64> @test_compress_q_256(<4 x i64> %data, <4 x i64> %data2) #0 {
+; CHECK-LABEL: define <4 x i64> @test_compress_q_256(
+; CHECK-SAME: <4 x i64> [[DATA:%.*]], <4 x i64> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> [[DATA]], <4 x i64> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i64> %1
+}
+
+define <8 x i32> @test_mask_compress_d_256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_mask_compress_d_256(
+; CHECK-SAME: <8 x i32> [[DATA:%.*]], <8 x i32> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> [[DATA]], <8 x i32> [[PASSTHRU]], <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> %passthru, <8 x i1> %1)
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_maskz_compress_d_256(<8 x i32> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_maskz_compress_d_256(
+; CHECK-SAME: <8 x i32> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> [[DATA]], <8 x i32> zeroinitializer, <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> zeroinitializer, <8 x i1> %1)
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_compress_d_256(<8 x i32> %data, <8 x i32> %data2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_compress_d_256(
+; CHECK-SAME: <8 x i32> [[DATA:%.*]], <8 x i32> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> [[DATA]], <8 x i32> [[DATA2]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> %data2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i32> %1
+}
+
+define <4 x double> @test_expand_pd_256(<4 x double> %data, <4 x double> %data2) #0 {
+; CHECK-LABEL: define <4 x double> @test_expand_pd_256(
+; CHECK-SAME: <4 x double> [[DATA:%.*]], <4 x double> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> [[DATA]], <4 x double> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x double> %1
+}
+
+define <4 x double> @test_mask_expand_pd_256(<4 x double> %data, <4 x double> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_expand_pd_256(
+; CHECK-SAME: <4 x double> [[DATA:%.*]], <4 x double> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> [[DATA]], <4 x double> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> %passthru, <4 x i1> %extract)
+  ret <4 x double> %2
+}
+
+define <4 x double> @test_maskz_expand_pd_256(<4 x double> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_maskz_expand_pd_256(
+; CHECK-SAME: <4 x double> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> [[DATA]], <4 x double> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> zeroinitializer, <4 x i1> %extract)
+  ret <4 x double> %2
+}
+
+define <8 x float> @test_expand_ps_256(<8 x float> %data, <8 x float> %data2) #0 {
+; CHECK-LABEL: define <8 x float> @test_expand_ps_256(
+; CHECK-SAME: <8 x float> [[DATA:%.*]], <8 x float> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> [[DATA]], <8 x float> [[DATA2]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> %data2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x float> %1
+}
+
+define <8 x float> @test_mask_expand_ps_256(<8 x float> %data, <8 x float> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mask_expand_ps_256(
+; CHECK-SAME: <8 x float> [[DATA:%.*]], <8 x float> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> [[DATA]], <8 x float> [[PASSTHRU]], <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> %passthru, <8 x i1> %1)
+  ret <8 x float> %2
+}
+
+define <8 x float> @test_maskz_expand_ps_256(<8 x float> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_maskz_expand_ps_256(
+; CHECK-SAME: <8 x float> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> [[DATA]], <8 x float> zeroinitializer, <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> zeroinitializer, <8 x i1> %1)
+  ret <8 x float> %2
+}
+
+define <4 x i64> @test_expand_q_256(<4 x i64> %data, <4 x i64> %data2) #0 {
+; CHECK-LABEL: define <4 x i64> @test_expand_q_256(
+; CHECK-SAME: <4 x i64> [[DATA:%.*]], <4 x i64> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> [[DATA]], <4 x i64> [[DATA2]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> %data2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @test_mask_expand_q_256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_mask_expand_q_256(
+; CHECK-SAME: <4 x i64> [[DATA:%.*]], <4 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i4 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> [[DATA]], <4 x i64> [[PASSTHRU]], <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> %passthru, <4 x i1> %extract)
+  ret <4 x i64> %2
+}
+
+define <4 x i64> @test_maskz_expand_q_256(<4 x i64> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_maskz_expand_q_256(
+; CHECK-SAME: <4 x i64> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i1> [[_MSPROP]] to i4
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i4 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> [[DATA]], <4 x i64> zeroinitializer, <4 x i1> [[EXTRACT]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i64> %2
+}
+
+define <8 x i32> @test_expand_d_256(<8 x i32> %data, <8 x i32> %data2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_expand_d_256(
+; CHECK-SAME: <8 x i32> [[DATA:%.*]], <8 x i32> [[DATA2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> [[DATA]], <8 x i32> [[DATA2]], <8 x i1> splat (i1 true))
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> %data2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @test_mask_expand_d_256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_mask_expand_d_256(
+; CHECK-SAME: <8 x i32> [[DATA:%.*]], <8 x i32> [[PASSTHRU:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> [[DATA]], <8 x i32> [[PASSTHRU]], <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> %passthru, <8 x i1> %1)
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_maskz_expand_d_256(<8 x i32> %data, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_maskz_expand_d_256(
+; CHECK-SAME: <8 x i32> [[DATA:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> [[DATA]], <8 x i32> zeroinitializer, <8 x i1> [[TMP1]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> zeroinitializer, <8 x i1> %1)
+  ret <8 x i32> %2
+}
+
+define i8 @test_cmpps_256(<8 x float> %a, <8 x float> %b) #0 {
+; CHECK-LABEL: define i8 @test_cmpps_256(
+; CHECK-SAME: <8 x float> [[A:%.*]], <8 x float> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i1> @llvm.x86.avx512.mask.cmp.ps.256(<8 x float> [[A]], <8 x float> [[B]], i32 2, <8 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i1> [[RES]] to i8
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = call <8 x i1> @llvm.x86.avx512.mask.cmp.ps.256(<8 x float> %a, <8 x float> %b, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %1 = bitcast <8 x i1> %res to i8
+  ret i8 %1
+}
+declare <8 x i1> @llvm.x86.avx512.mask.cmp.ps.256(<8 x float>, <8 x float>, i32, <8 x i1>)
+
+define i8 @test_cmpps_128(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: define i8 @test_cmpps_128(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP6]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i1> @llvm.x86.avx512.mask.cmp.ps.128(<4 x float> [[A]], <4 x float> [[B]], i32 2, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[RES]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %res = call <4 x i1> @llvm.x86.avx512.mask.cmp.ps.128(<4 x float> %a, <4 x float> %b, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = shufflevector <4 x i1> %res, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+declare <4 x i1> @llvm.x86.avx512.mask.cmp.ps.128(<4 x float>, <4 x float>, i32, <4 x i1>)
+
+define i8 @test_cmppd_256(<4 x double> %a, <4 x double> %b) #0 {
+; CHECK-LABEL: define i8 @test_cmppd_256(
+; CHECK-SAME: <4 x double> [[A:%.*]], <4 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP5]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP6]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i1> @llvm.x86.avx512.mask.cmp.pd.256(<4 x double> [[A]], <4 x double> [[B]], i32 2, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[RES]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %res = call <4 x i1> @llvm.x86.avx512.mask.cmp.pd.256(<4 x double> %a, <4 x double> %b, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %1 = shufflevector <4 x i1> %res, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+declare <4 x i1> @llvm.x86.avx512.mask.cmp.pd.256(<4 x double>, <4 x double>, i32, <4 x i1>)
+
+define i8 @test_cmppd_128(<2 x double> %a, <2 x double> %b) #0 {
+; CHECK-LABEL: define i8 @test_cmppd_128(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP5]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP6]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i1> @llvm.x86.avx512.mask.cmp.pd.128(<2 x double> [[A]], <2 x double> [[B]], i32 2, <2 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i1> [[RES]], <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %res = call <2 x i1> @llvm.x86.avx512.mask.cmp.pd.128(<2 x double> %a, <2 x double> %b, i32 2, <2 x i1> <i1 true, i1 true>)
+  %1 = shufflevector <2 x i1> %res, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+declare <2 x i1> @llvm.x86.avx512.mask.cmp.pd.128(<2 x double>, <2 x double>, i32, <2 x i1>)
+
+define <8 x float> @test_mm512_maskz_max_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mm512_maskz_max_ps_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[_MSPROP]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i32> [[TMP7]], <8 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP8]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> zeroinitializer
+  ret <8 x float> %3
+}
+
+define <8 x float> @test_mm512_mask_max_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mm512_mask_max_ps_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[SRC:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[_MSPROP]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[SRC]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i32> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP14]], <8 x i32> [[TMP13]], <8 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[SRC]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP9]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %src
+  ret <8 x float> %3
+}
+
+define <8 x float> @test_mm512_max_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: define <8 x float> @test_mm512_max_ps_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
+  ret <8 x float> %1
+}
+declare <8 x float> @llvm.x86.avx.max.ps.256(<8 x float>, <8 x float>)
+
+define <4 x float> @test_mm512_maskz_max_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mm512_maskz_max_ps_128(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> splat (i1 true), <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP7]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %1, <4 x float> zeroinitializer
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mm512_mask_max_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mm512_mask_max_ps_128(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[SRC:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> splat (i1 true), <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP13]], <4 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[SRC]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %1 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %src
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mm512_max_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: define <4 x float> @test_mm512_max_ps_128(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
+  ret <4 x float> %1
+}
+declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>)
+
+define <8 x float> @test_mm512_maskz_min_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mm512_maskz_min_ps_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[_MSPROP]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i32> [[TMP7]], <8 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP8]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> zeroinitializer
+  ret <8 x float> %3
+}
+
+define <8 x float> @test_mm512_mask_min_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mm512_mask_min_ps_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[SRC:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[_MSPROP]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[SRC]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i32> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP14]], <8 x i32> [[TMP13]], <8 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[SRC]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP9]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %src
+  ret <8 x float> %3
+}
+
+define <8 x float> @test_mm512_min_ps_256(<8 x float> %a0, <8 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: define <8 x float> @test_mm512_min_ps_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> [[A0]], <8 x float> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
+  ret <8 x float> %1
+}
+declare <8 x float> @llvm.x86.avx.min.ps.256(<8 x float>, <8 x float>)
+
+define <4 x float> @test_mm512_maskz_min_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mm512_maskz_min_ps_128(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP8]], <4 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %1 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = bitcast i8 %mask2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %1, <4 x float> zeroinitializer
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mm512_mask_min_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mm512_mask_min_ps_128(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[SRC:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP8]], [[TMP12]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP16]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[SRC]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP10]]
+;
+  %1 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = bitcast i8 %mask2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %src
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mm512_min_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+; CHECK-LABEL: define <4 x float> @test_mm512_min_ps_128(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> [[A0]], <4 x float> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
+  ret <4 x float> %1
+}
+declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>)
+
+define <4 x double> @test_getexp_pd_256(<4 x double> %a0) #0 {
+; CHECK-LABEL: define <4 x double> @test_getexp_pd_256(
+; CHECK-SAME: <4 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.mask.getexp.pd.256(<4 x double> [[A0]], <4 x double> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.mask.getexp.pd.256(<4 x double> %a0,  <4 x double> zeroinitializer, i8 -1)
+  ret <4 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.getexp.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+
+define <8 x float> @test_getexp_ps_256(<8 x float> %a0) #0 {
+; CHECK-LABEL: define <8 x float> @test_getexp_ps_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float> [[A0]], <8 x float> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+
+declare <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx512_vpermi2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpermi2var_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP6]], <4 x i32> [[X1]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_vpermi2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_vpermi2var_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP8]], <4 x i32> [[X1]], <4 x i32> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP5]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i32> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP15]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP1]], <4 x i32> [[X1]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x1
+  ret <4 x i32> %3
+}
+
+define <4 x i32>@test_int_x86_avx512_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpermt2var_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP6]], <4 x i32> [[X0]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X1]], <4 x i32> [[X0]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_vpermt2var_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP8]], <4 x i32> [[X0]], <4 x i32> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X1]], <4 x i32> [[X0]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP5]], <4 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i32> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP15]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP1]], <4 x i32> [[X1]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x1
+  ret <4 x i32> %3
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_vpermt2var_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP8]], <4 x i32> [[X0]], <4 x i32> [[TMP9]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X1]], <4 x i32> [[X0]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP13]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP6]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> zeroinitializer
+  ret <4 x i32> %3
+}
+
+declare <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx512_vpermi2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpermi2var_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP6]], <8 x i32> [[X1]], <8 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_vpermi2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_vpermi2var_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP8]], <8 x i32> [[X1]], <8 x i32> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP5]], <8 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i32> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i32> [[TMP15]], <8 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[X1]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP7]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x1
+  ret <8 x i32> %3
+}
+
+define <8 x i32>@test_int_x86_avx512_ask_vpermt2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_ask_vpermt2var_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP6]], <8 x i32> [[X0]], <8 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X1]], <8 x i32> [[X0]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_vpermt2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_vpermt2var_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP8]], <8 x i32> [[X0]], <8 x i32> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X1]], <8 x i32> [[X0]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP5]], <8 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i32> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP14]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i32> [[TMP15]], <8 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[X1]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP7]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x1
+  ret <8 x i32> %3
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_maskz_vpermt2var_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP8]], <8 x i32> [[X0]], <8 x i32> [[TMP9]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X1]], <8 x i32> [[X0]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP13]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i32> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i32> [[TMP6]], <8 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP7]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
+  ret <8 x i32> %3
+}
+
+declare <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double>, <2 x i64>, <2 x double>)
+
+define <2 x double>@test_int_x86_avx512_vpermi2var_pd_128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_vpermi2var_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x double> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP9]] to <2 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP4]] to <2 x double>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[TMP8]], <2 x i64> [[X1]], <2 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[X0]], <2 x i64> [[X1]], <2 x double> [[X2]])
+; CHECK-NEXT:    store <2 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2)
+  ret <2 x double> %1
+}
+
+define <2 x double>@test_int_x86_avx512_mask_vpermi2var_pd_128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_vpermi2var_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP11]] to <2 x double>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP8]] to <2 x double>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[TMP9]], <2 x i64> [[X1]], <2 x double> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <2 x double> [[TMP17]] to <2 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP13]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[X0]], <2 x i64> [[X1]], <2 x double> [[X2]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[X1]] to <2 x double>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP18]], <2 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x double> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <2 x i64> [[TMP7]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <2 x i64> [[TMP20]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP21]], <2 x i64> [[TMP16]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP1]], <2 x double> [[TMP2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP10]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2)
+  %2 = bitcast <2 x i64> %x1 to <2 x double>
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <2 x i32> <i32 0, i32 1>
+  %4 = select <2 x i1> %extract, <2 x double> %1, <2 x double> %2
+  ret <2 x double> %4
+}
+
+declare <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double>, <4 x i64>, <4 x double>)
+
+define <4 x double>@test_int_x86_avx512_vpermi2var_pd_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2) #0 {
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_vpermi2var_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x double> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP9]] to <4 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP4]] to <4 x double>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[TMP8]], <4 x i64> [[X1]], <4 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP10]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[X0]], <4 x i64> [[X1]], <4 x double> [[X2]])
+; CHECK-NEXT:    store <4 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2)
+  ret <4 x double> %1
+}
+
+define <4 x double>@test_int_x86_avx512_mask_vpermi2var_pd_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_vpermi2var_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[TMP11]] to <4 x double>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP8]] to <4 x double>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[TMP9]], <4 x i64> [[X1]], <4 x double> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <4 x double> [[TMP17]] to <4 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i64> [[TMP13]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[X0]], <4 x i64> [[X1]], <4 x double> [[X2]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[X1]] to <4 x double>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP18]], <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i64> [[TMP7]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i64> [[TMP20]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP21]], <4 x i64> [[TMP16]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[TMP2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP10]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2)
+  %2 = bitcast <4 x i64> %x1 to <4 x double>
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %2
+  ret <4 x double> %4
+}
+
+declare <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float>, <4 x i32>, <4 x float>)
+
+define <4 x float>@test_int_x86_avx512_vpermi2var_ps_128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_vpermi2var_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x float> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP9]] to <4 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[TMP8]], <4 x i32> [[X1]], <4 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[TMP10]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[X0]], <4 x i32> [[X1]], <4 x float> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2)
+  ret <4 x float> %1
+}
+
+define <4 x float>@test_int_x86_avx512_mask_vpermi2var_ps_128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_vpermi2var_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP11]] to <4 x float>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP8]] to <4 x float>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[TMP9]], <4 x i32> [[X1]], <4 x float> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <4 x float> [[TMP17]] to <4 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP13]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[X0]], <4 x i32> [[X1]], <4 x float> [[X2]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[X1]] to <4 x float>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP18]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP7]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i32> [[TMP20]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP21]], <4 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[TMP2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP10]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2)
+  %2 = bitcast <4 x i32> %x1 to <4 x float>
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %2
+  ret <4 x float> %4
+}
+
+define <4 x float>@test_int_x86_avx512_mask_vpermi2var_ps_128_cast(<4 x float> %x0, <2 x i64> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_vpermi2var_ps_128_cast(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <2 x i64> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP11]] to <4 x i32>
+; CHECK-NEXT:    [[X1CAST:%.*]] = bitcast <2 x i64> [[X1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP12]] to <4 x float>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <4 x i32> [[TMP13]] to <4 x float>
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[TMP16]], <4 x i32> [[X1CAST]], <4 x float> [[TMP18]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x float> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP14]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB11:.*]], label %[[BB12:.*]], !prof [[PROF1]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[X0]], <4 x i32> [[X1CAST]], <4 x float> [[X2]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[X1CAST]] to <4 x float>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP9]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <4 x i32> [[TMP21]], [[TMP14]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP22]], <4 x i32> [[TMP17]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[TMP2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP10]]
+;
+  %x1cast = bitcast <2 x i64> %x1 to <4 x i32>
+  %1 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1cast, <4 x float> %x2)
+  %2 = bitcast <4 x i32> %x1cast to <4 x float>
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %2
+  ret <4 x float> %4
+}
+
+declare <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float>, <8 x i32>, <8 x float>)
+
+define <8 x float>@test_int_x86_avx512_vpermi2var_ps_256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2) #0 {
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_vpermi2var_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x float> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP9]] to <8 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[TMP8]], <8 x i32> [[X1]], <8 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[TMP10]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[X0]], <8 x i32> [[X1]], <8 x float> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2)
+  ret <8 x float> %1
+}
+
+define <8 x float>@test_int_x86_avx512_mask_vpermi2var_ps_256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask_vpermi2var_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP11]] to <8 x float>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[TMP9]], <8 x i32> [[X1]], <8 x float> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x float> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i32> [[TMP13]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[X0]], <8 x i32> [[X1]], <8 x float> [[X2]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[X1]] to <8 x float>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP18]], <8 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i32> [[TMP7]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i32> [[TMP20]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP14]], <8 x i32> [[TMP21]], <8 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP3]], <8 x float> [[TMP1]], <8 x float> [[TMP2]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP10]]
+;
+  %1 = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2)
+  %2 = bitcast <8 x i32> %x1 to <8 x float>
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %4 = select <8 x i1> %3, <8 x float> %1, <8 x float> %2
+  ret <8 x float> %4
+}
+
+declare <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64>, <2 x i64>, <2 x i64>)
+
+define <2 x i64>@test_int_x86_avx512_vpermi2var_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) #0 {
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_vpermi2var_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[X1]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[X1]], <2 x i64> [[X2]])
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
+  ret <2 x i64> %1
+}
+
+define <2 x i64>@test_int_x86_avx512_mask_vpermi2var_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_mask_vpermi2var_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP8]], <2 x i64> [[X1]], <2 x i64> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[X1]], <2 x i64> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP5]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP15]], <2 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP1]], <2 x i64> [[X1]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP7]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x1
+  ret <2 x i64> %3
+}
+
+define <2 x i64>@test_int_x86_avx512_vpermt2var_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) #0 {
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_vpermt2var_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[X0]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X1]], <2 x i64> [[X0]], <2 x i64> [[X2]])
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x2)
+  ret <2 x i64> %1
+}
+
+define <2 x i64>@test_int_x86_avx512_mask_vpermt2var_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_mask_vpermt2var_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP8]], <2 x i64> [[X0]], <2 x i64> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X1]], <2 x i64> [[X0]], <2 x i64> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP5]], <2 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP15]], <2 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP1]], <2 x i64> [[X1]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP7]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x1
+  ret <2 x i64> %3
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_maskz_vpermt2var_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP8]], <2 x i64> [[X0]], <2 x i64> [[TMP9]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X1]], <2 x i64> [[X0]], <2 x i64> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP13]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP6]], <2 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP1]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP7]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> zeroinitializer
+  ret <2 x i64> %3
+}
+
+declare <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64>, <4 x i64>, <4 x i64>)
+
+define <4 x i64>@test_int_x86_avx512_vpermi2var_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) #0 {
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_vpermi2var_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP6]], <4 x i64> [[X1]], <4 x i64> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> [[X1]], <4 x i64> [[X2]])
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
+  ret <4 x i64> %1
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_vpermi2var_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_mask_vpermi2var_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP8]], <4 x i64> [[X1]], <4 x i64> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> [[X1]], <4 x i64> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP5]], <4 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP15]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[X1]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x1
+  ret <4 x i64> %3
+}
+
+define <4 x i64>@test_int_x86_avx512_vpermt2var_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) #0 {
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_vpermt2var_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP6]], <4 x i64> [[X0]], <4 x i64> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X1]], <4 x i64> [[X0]], <4 x i64> [[X2]])
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x2)
+  ret <4 x i64> %1
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_vpermt2var_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_mask_vpermt2var_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP8]], <4 x i64> [[X0]], <4 x i64> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X1]], <4 x i64> [[X0]], <4 x i64> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP5]], <4 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP14]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP15]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[X1]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x1
+  ret <4 x i64> %3
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_maskz_vpermt2var_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP8]], <4 x i64> [[X0]], <4 x i64> [[TMP9]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X1]], <4 x i64> [[X0]], <4 x i64> [[X2]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP13]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP6]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> zeroinitializer
+  ret <4 x i64> %3
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_scalef_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2) #0 {
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_scalef_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[X2]], i8 -1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  ret <2 x double> %res
+}
+
+define <2 x double>@test_int_x86_avx512_mask_scalef_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_scalef_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  ret <2 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_scalef_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2) #0 {
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_scalef_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[X2]], i8 -1)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  ret <4 x double> %res
+}
+
+define <4 x double>@test_int_x86_avx512_mask_scalef_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_scalef_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  ret <4 x double> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_scalef_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_scalef_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[X2]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float>@test_int_x86_avx512_mask_scalef_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_scalef_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  ret <4 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_scalef_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2) #0 {
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_scalef_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[X2]], i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float>@test_int_x86_avx512_mask_scalef_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask_scalef_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  ret <8 x float> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmov_qb_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qb.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qb_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_qb_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qb.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qb.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qb.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qb.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovs_qb_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qb.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qb_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_qb_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovus_qb_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qb.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qb_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_qb_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmov_qb_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qb.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qb_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_qb_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qb.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qb.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qb.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qb.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovs_qb_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qb.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qb_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_qb_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovus_qb_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qb.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qb_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_qb_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmov_qw_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qw.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qw_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_qw_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qw.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qw.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qw.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qw.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovs_qw_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qw.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qw_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_qw_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovus_qw_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qw.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qw_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_qw_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmov_qw_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qw.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qw_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_qw_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qw.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qw.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qw.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qw.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovs_qw_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qw.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qw_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_qw_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovus_qw_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qw.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qw_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_qw_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_pmov_qd_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <4 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <4 x i32> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES4]]
+;
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  %res3 = add <4 x i32> %res0, %res1
+  %res4 = add <4 x i32> %res3, %res2
+  ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qd.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qd_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_qd_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qd.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qd.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qd.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qd.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovs_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_pmovs_qd_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <4 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <4 x i32> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES4]]
+;
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  %res3 = add <4 x i32> %res0, %res1
+  %res4 = add <4 x i32> %res3, %res2
+  ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qd.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qd_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_qd_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovus_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_pmovus_qd_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <4 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <4 x i32> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES4]]
+;
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  %res3 = add <4 x i32> %res0, %res1
+  %res4 = add <4 x i32> %res3, %res2
+  ret <4 x i32> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qd.mem.128(ptr %ptr, <2 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qd_mem_128(ptr %ptr, <2 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_qd_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(ptr [[PTR]], <2 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(ptr %ptr, <2 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(ptr %ptr, <2 x i64> %x1, i8 %x2)
+  ret void
+}
+
+define <4 x i32>@test_int_x86_avx512_pmov_qd_256(<4 x i64> %x0) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_pmov_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <4 x i64> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[X0]] to <4 x i32>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = trunc <4 x i64> %x0 to <4 x i32>
+  ret <4 x i32> %1
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_pmov_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <4 x i64> [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[X0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP8]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i32> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP4]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <4 x i32> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP10]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP1]], <4 x i32> [[X1]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = trunc <4 x i64> %x0 to <4 x i32>
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x1
+  ret <4 x i32> %3
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_pmov_qd_256(<4 x i64> %x0, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_pmov_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <4 x i64> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[X0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP8]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP9]], <8 x i1> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> [[TMP4]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i32> [[TMP6]], <4 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = trunc <4 x i64> %x0 to <4 x i32>
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> zeroinitializer
+  ret <4 x i32> %3
+}
+
+declare void @llvm.x86.avx512.mask.pmov.qd.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_qd_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_qd_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qd.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.qd.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.qd.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.qd.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_pmovs_qd_256(<4 x i64> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_pmovs_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovs_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_pmovs_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_pmovs_qd_256(<4 x i64> %x0, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_pmovs_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.qd.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_qd_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_qd_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_pmovus_qd_256(<4 x i64> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_pmovus_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_pmovus_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_pmovus_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_pmovus_qd_256(<4 x i64> %x0, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_pmovus_qd_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> [[X0]], <4 x i32> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.qd.mem.256(ptr %ptr, <4 x i64>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_qd_mem_256(ptr %ptr, <4 x i64> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_qd_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(ptr [[PTR]], <4 x i64> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(ptr %ptr, <4 x i64> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(ptr %ptr, <4 x i64> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmov_db_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.db.mem.128(ptr %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_db_mem_128(ptr %ptr, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_db_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.db.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.db.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.db.mem.128(ptr %ptr, <4 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.db.mem.128(ptr %ptr, <4 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovs_db_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.db.mem.128(ptr %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_db_mem_128(ptr %ptr, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_db_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.db.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.db.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.db.mem.128(ptr %ptr, <4 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.db.mem.128(ptr %ptr, <4 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovus_db_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.db.mem.128(ptr %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_db_mem_128(ptr %ptr, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_db_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.db.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.db.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.db.mem.128(ptr %ptr, <4 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.db.mem.128(ptr %ptr, <4 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmov_db_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.db.mem.256(ptr %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_db_mem_256(ptr %ptr, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_db_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.db.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.db.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.db.mem.256(ptr %ptr, <8 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.db.mem.256(ptr %ptr, <8 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovs_db_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.db.mem.256(ptr %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_db_mem_256(ptr %ptr, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_db_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.db.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.db.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.db.mem.256(ptr %ptr, <8 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.db.mem.256(ptr %ptr, <8 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32>, <16 x i8>, i8)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <16 x i8> @test_int_x86_avx512_mask_pmovus_db_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <16 x i8> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> [[X0]], <16 x i8> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> [[X0]], <16 x i8> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> [[X0]], <16 x i8> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <16 x i8> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <16 x i8> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES4]]
+;
+  %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2)
+  %res3 = add <16 x i8> %res0, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.db.mem.256(ptr %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_db_mem_256(ptr %ptr, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_db_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.db.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.db.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.db.mem.256(ptr %ptr, <8 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.db.mem.256(ptr %ptr, <8 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmov_dw_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.dw.mem.128(ptr %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_dw_mem_128(ptr %ptr, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_dw_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.dw.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.dw.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.dw.mem.128(ptr %ptr, <4 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.dw.mem.128(ptr %ptr, <4 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovs_dw_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.dw.mem.128(ptr %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_dw_mem_128(ptr %ptr, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_dw_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(ptr %ptr, <4 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(ptr %ptr, <4 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovus_dw_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.dw.mem.128(ptr %ptr, <4 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_dw_mem_128(ptr %ptr, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_dw_mem_128(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(ptr [[PTR]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(ptr %ptr, <4 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(ptr %ptr, <4 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmov_dw_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmov.dw.mem.256(ptr %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmov_dw_mem_256(ptr %ptr, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmov_dw_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.dw.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.dw.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.dw.mem.256(ptr %ptr, <8 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmov.dw.mem.256(ptr %ptr, <8 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovs_dw_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.dw.mem.256(ptr %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovs_dw_mem_256(ptr %ptr, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovs_dw_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(ptr %ptr, <8 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(ptr %ptr, <8 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_int_x86_avx512_mask_pmovus_dw_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i16> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> [[X0]], <8 x i16> [[X1]], i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> [[X0]], <8 x i16> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> [[X0]], <8 x i16> zeroinitializer, i8 [[X2]])
+; CHECK-NEXT:    [[RES3:%.*]] = add <8 x i16> [[RES0]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = add <8 x i16> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES4]]
+;
+  %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i16> %res0, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.dw.mem.256(ptr %ptr, <8 x i32>, i8)
+
+define void @test_int_x86_avx512_mask_pmovus_dw_mem_256(ptr %ptr, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define void @test_int_x86_avx512_mask_pmovus_dw_mem_256(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(ptr [[PTR]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(ptr %ptr, <8 x i32> %x1, i8 -1)
+  call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(ptr %ptr, <8 x i32> %x1, i8 %x2)
+  ret void
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_pd2dq_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = add <4 x i32> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES2]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+define <4 x i32>@test_int_x86_avx512_cvt_pd2dq_128_zext(<2 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvt_pd2dq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = shufflevector <4 x i32> [[RES2]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES3]]
+;
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res3 = shufflevector <4 x i32> %res2, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res3
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_128_zext(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_pd2dq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[RES1:%.*]] = shufflevector <4 x i32> [[RES]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES1]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = shufflevector <4 x i32> %res, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res1
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_cvt_pd2ps(<2 x double> %x0, <4 x float> %x1) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_cvt_pd2ps(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x float> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> [[X0]], <4 x float> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps(<2 x double> %x0, <4 x float> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_cvt_pd2ps(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x float> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> [[X0]], <4 x float> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 %x2)
+  ret <4 x float> %res
+}
+
+define <4 x float>@test_int_x86_avx512_cvt_pd2ps_zext(<2 x double> %x0, <4 x float> %x1) #0 {
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_cvt_pd2ps_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x float> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> [[X0]], <4 x float> [[X1]], i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = shufflevector <4 x float> [[RES2]], <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES3]]
+;
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 -1)
+  %res3 = shufflevector <4 x float> %res2, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x float> %res3
+}
+
+define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps_zext(<2 x double> %x0, <4 x float> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_cvt_pd2ps_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x float> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> [[X0]], <4 x float> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[RES1:%.*]] = shufflevector <4 x float> [[RES]], <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES1]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = shufflevector <4 x float> %res, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x float> %res1
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_cvt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvt_pd2udq_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_pd2udq_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_cvt_pd2udq_128_zext(<2 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvt_pd2udq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = shufflevector <4 x i32> [[RES2]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES3]]
+;
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res3 = shufflevector <4 x i32> %res2, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res3
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_128_zext(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_pd2udq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[RES1:%.*]] = shufflevector <4 x i32> [[RES]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES1]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = shufflevector <4 x i32> %res, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res1
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_cvt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvt_pd2udq_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_pd2udq_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_cvt_ps2dq_128(<4 x float> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvt_ps2dq_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_ps2dq_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_cvt_ps2dq_256(<8 x float> %x0, <8 x i32> %x1) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_cvt_ps2dq_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> [[X0]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_cvt_ps2dq_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> [[X0]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_cvt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvt_ps2udq_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvt_ps2udq_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_cvt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_cvt_ps2udq_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> [[X0]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_cvt_ps2udq_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> [[X0]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_ask_cvtt_pd2dq_128(<2 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_ask_cvtt_pd2dq_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_pd2dq_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_cvtt_pd2dq_128_zext(<2 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvtt_pd2dq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = shufflevector <4 x i32> [[RES2]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES3]]
+;
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res3 = shufflevector <4 x i32> %res2, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res3
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_128_zext(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_pd2dq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[RES1:%.*]] = shufflevector <4 x i32> [[RES]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES1]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = shufflevector <4 x i32> %res, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res1
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_cvtt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvtt_pd2udq_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_pd2udq_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_cvtt_pd2udq_128_zext(<2 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvtt_pd2udq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = shufflevector <4 x i32> [[RES2]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES3]]
+;
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1)
+  %res3 = shufflevector <4 x i32> %res2, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res3
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_128_zext(<2 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_pd2udq_128_zext(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    [[RES1:%.*]] = shufflevector <4 x i32> [[RES]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES1]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = shufflevector <4 x i32> %res, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i32> %res1
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_cvtt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvtt_pd2udq_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_pd2udq_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_cvtt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_cvtt_ps2udq_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> [[X0]], <4 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_cvtt_ps2udq_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> [[X0]], <4 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_cvtt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_cvtt_ps2udq_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> [[X0]], <8 x i32> [[X1]], i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_cvtt_ps2udq_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]], i8 [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> [[X0]], <8 x i32> [[X1]], i8 [[X2]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2)
+  ret <8 x i32> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double>, i32, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_rndscale_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_rndscale_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> [[X0]], i32 4, <2 x double> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> [[X0]], i32 88, <2 x double> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES2]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %x0, i32 4, <2 x double> %x2, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %x0, i32 88, <2 x double> %x2, i8 -1)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double>, i32, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_rndscale_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_rndscale_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> [[X0]], i32 4, <4 x double> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> [[X0]], i32 88, <4 x double> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <4 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES2]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %x0, i32 4, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %x0, i32 88, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float>, i32, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_rndscale_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_rndscale_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> [[X0]], i32 88, <4 x float> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> [[X0]], i32 4, <4 x float> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES2]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %x0, i32 88, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %x0, i32 4, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float>, i32, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask_rndscale_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask_rndscale_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> [[X0]], i32 5, <8 x float> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> [[X0]], i32 66, <8 x float> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <8 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES2]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %x0, i32 5, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %x0, i32 66, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double>, i32, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_getmant_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_getmant_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> [[X0]], i32 11, <2 x double> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> [[X0]], i32 12, <2 x double> zeroinitializer, i8 [[X3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR9:%.*]] = or i1 [[_MSCMP7]], [[_MSCMP8]]
+; CHECK-NEXT:    br i1 [[_MSOR9]], label %[[BB13:.*]], label %[[BB14:.*]], !prof [[PROF1]]
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> [[X0]], i32 13, <2 x double> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[RES2]], [[RES3]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> %x2, i8 %x3)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 12, <2 x double> zeroinitializer, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 13, <2 x double> %x2, i8 -1)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double>, i32, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_getmant_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_getmant_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> [[X0]], i32 11, <4 x double> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> [[X0]], i32 12, <4 x double> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <4 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES2]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 11, <4 x double> %x2, i8 %x3)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 12, <4 x double> %x2, i8 -1)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float>, i32, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_getmant_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_getmant_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> [[X0]], i32 11, <4 x float> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> [[X0]], i32 12, <4 x float> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES2]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> %x0, i32 11, <4 x float> %x2, i8 %x3)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> %x0, i32 12, <4 x float> %x2, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float>, i32, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask_getmant_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask_getmant_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> [[X0]], i32 11, <8 x float> [[X2]], i8 [[X3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label %[[BB10:.*]], label %[[BB11:.*]], !prof [[PROF1]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> [[X0]], i32 12, <8 x float> [[X2]], i8 -1)
+; CHECK-NEXT:    [[RES2:%.*]] = fadd <8 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES2]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 %x3)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> %x0, i32 12, <8 x float> %x2, i8 -1)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32)
+
+define <4 x i32>@test_int_x86_avx512_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) #0 {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_pternlog_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]], i32 33)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
+  ret <4 x i32> %1
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_pternlog_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP8]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP10]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> zeroinitializer, <4 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i32> [[TMP1]], [[X0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP16]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP1]], <4 x i32> [[X0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
+  ret <4 x i32> %3
+}
+
+declare <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32, i8)
+
+define <4 x i32>@test_int_x86_avx512_maskz_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_pternlog_d_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> [[TMP8]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP10]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP12]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP6]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP7]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> zeroinitializer
+  ret <4 x i32> %3
+}
+
+declare <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32)
+
+define <8 x i32>@test_int_x86_avx512_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) #0 {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_pternlog_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]], i32 33)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
+  ret <8 x i32> %1
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_pternlog_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP8]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP10]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> zeroinitializer, <8 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i32> [[TMP1]], [[X0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i32> [[TMP16]], <8 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[X0]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP7]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
+  ret <8 x i32> %3
+}
+
+declare <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32, i8)
+
+define <8 x i32>@test_int_x86_avx512_maskz_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_maskz_pternlog_d_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i32> [[TMP8]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i32> [[TMP10]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP12]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> zeroinitializer, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP11]], <8 x i32> [[TMP6]], <8 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP7]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
+  ret <8 x i32> %3
+}
+
+declare <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i32)
+
+define <2 x i64>@test_int_x86_avx512_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) #0 {
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_pternlog_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP7]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> [[X0]], <2 x i64> [[X1]], <2 x i64> [[X2]], i32 33)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
+  ret <2 x i64> %1
+}
+
+define <2 x i64>@test_int_x86_avx512_mask_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_mask_pternlog_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP8]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> [[X0]], <2 x i64> [[X1]], <2 x i64> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP10]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> zeroinitializer, <2 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP1]], [[X0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <2 x i64> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP16]], <2 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP1]], <2 x i64> [[X0]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP7]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x0
+  ret <2 x i64> %3
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_maskz_pternlog_q_128(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <2 x i64> [[TMP8]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP10]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> [[X0]], <2 x i64> [[X1]], <2 x i64> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP12]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> zeroinitializer, <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP6]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP1]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP7]]
+;
+  %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> zeroinitializer
+  ret <2 x i64> %3
+}
+
+declare <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i32)
+
+define <4 x i64>@test_int_x86_avx512_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) #0 {
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_pternlog_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP7]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> [[X0]], <4 x i64> [[X1]], <4 x i64> [[X2]], i32 33)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
+  ret <4 x i64> %1
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_mask_pternlog_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP8]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> [[X0]], <4 x i64> [[X1]], <4 x i64> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP10]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> zeroinitializer, <4 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[TMP1]], [[X0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i64> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP16]], <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[X0]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x0
+  ret <4 x i64> %3
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_maskz_pternlog_q_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i64> [[TMP8]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i64> [[TMP10]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> [[X0]], <4 x i64> [[X1]], <4 x i64> [[X2]], i32 33)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP12]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> zeroinitializer, <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP6]], <4 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> zeroinitializer
+  ret <4 x i64> %3
+}
+
+define <8 x i16> @test_x86_vcvtps2ph_128(<4 x float> %a0, i8 %mask, <8 x i16> %src) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_x86_vcvtps2ph_128(
+; CHECK-SAME: <4 x float> [[A0:%.*]], i8 [[MASK:%.*]], <8 x i16> [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> [[A0]], i32 2, <8 x i16> zeroinitializer, i8 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> [[A0]], i32 10, <8 x i16> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> [[A0]], i32 11, <8 x i16> [[SRC]], i8 [[MASK]])
+; CHECK-NEXT:    [[RES0:%.*]] = add <8 x i16> [[RES1]], [[RES2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i16> [[RES3]], [[RES0]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> %a0, i32 10, <8 x i16> zeroinitializer, i8 %mask)
+  %res3 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> %a0, i32 11, <8 x i16> %src, i8 %mask)
+  %res0 = add <8 x i16> %res1, %res2
+  %res = add <8 x i16> %res3, %res0
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float>, i32, <8 x i16>, i8) nounwind readonly
+
+define <8 x i16> @test_x86_vcvtps2ph_256(<8 x float> %a0, i8 %mask, <8 x i16> %src) #0 {
+;
+; CHECK-LABEL: define <8 x i16> @test_x86_vcvtps2ph_256(
+; CHECK-SAME: <8 x float> [[A0:%.*]], i8 [[MASK:%.*]], <8 x i16> [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> [[A0]], i32 2, <8 x i16> zeroinitializer, i8 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> [[A0]], i32 11, <8 x i16> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
+; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES3:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> [[A0]], i32 12, <8 x i16> [[SRC]], i8 [[MASK]])
+; CHECK-NEXT:    [[RES0:%.*]] = add <8 x i16> [[RES1]], [[RES2]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i16> [[RES3]], [[RES0]]
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> %a0, i32 11, <8 x i16> zeroinitializer, i8 %mask)
+  %res3 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> %a0, i32 12, <8 x i16> %src, i8 %mask)
+  %res0 = add <8 x i16> %res1, %res2
+  %res = add <8 x i16> %res3, %res0
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float>, i32, <8 x i16>, i8) nounwind readonly
+
+define <8 x float> @test_rsqrt_ps_256_rr(<8 x float> %a0) #0 {
+; CHECK-LABEL: define <8 x float> @test_rsqrt_ps_256_rr(
+; CHECK-SAME: <8 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> [[A0]], <8 x float> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_rsqrt_ps_256_rrkz(<8 x float> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_rsqrt_ps_256_rrkz(
+; CHECK-SAME: <8 x float> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> [[A0]], <8 x float> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_rsqrt_ps_256_rrk(<8 x float> %a0, <8 x float> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_rsqrt_ps_256_rrk(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> [[A0]], <8 x float> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> %a0, <8 x float> %a1, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <4 x float> @test_rsqrt_ps_128_rr(<4 x float> %a0) #0 {
+; CHECK-LABEL: define <4 x float> @test_rsqrt_ps_128_rr(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> [[A0]], <4 x float> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_rsqrt_ps_128_rrkz(<4 x float> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_rsqrt_ps_128_rrkz(
+; CHECK-SAME: <4 x float> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> [[A0]], <4 x float> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_rsqrt_ps_128_rrk(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_rsqrt_ps_128_rrk(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> [[A0]], <4 x float> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> %a0, <4 x float> %a1, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float>, <4 x float>, i8) nounwind readnone
+
+define <8 x float> @test_rcp_ps_256_rr(<8 x float> %a0) #0 {
+; CHECK-LABEL: define <8 x float> @test_rcp_ps_256_rr(
+; CHECK-SAME: <8 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> [[A0]], <8 x float> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_rcp_ps_256_rrkz(<8 x float> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_rcp_ps_256_rrkz(
+; CHECK-SAME: <8 x float> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> [[A0]], <8 x float> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_rcp_ps_256_rrk(<8 x float> %a0, <8 x float> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_rcp_ps_256_rrk(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> [[A0]], <8 x float> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> %a0, <8 x float> %a1, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <4 x float> @test_rcp_ps_128_rr(<4 x float> %a0) #0 {
+; CHECK-LABEL: define <4 x float> @test_rcp_ps_128_rr(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> [[A0]], <4 x float> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_rcp_ps_128_rrkz(<4 x float> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_rcp_ps_128_rrkz(
+; CHECK-SAME: <4 x float> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> [[A0]], <4 x float> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_rcp_ps_128_rrk(<4 x float> %a0, <4 x float> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_rcp_ps_128_rrk(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> [[A0]], <4 x float> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> %a0, <4 x float> %a1, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float>, <4 x float>, i8) nounwind readnone
+
+define <4 x double> @test_rsqrt_pd_256_rr(<4 x double> %a0) #0 {
+; CHECK-LABEL: define <4 x double> @test_rsqrt_pd_256_rr(
+; CHECK-SAME: <4 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> [[A0]], <4 x double> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 -1)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_rsqrt_pd_256_rrkz(<4 x double> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_rsqrt_pd_256_rrkz(
+; CHECK-SAME: <4 x double> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> [[A0]], <4 x double> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 %mask)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_rsqrt_pd_256_rrk(<4 x double> %a0, <4 x double> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_rsqrt_pd_256_rrk(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> [[A0]], <4 x double> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> %a0, <4 x double> %a1, i8 %mask)
+  ret <4 x double> %res
+}
+
+define <2 x double> @test_rsqrt_pd_128_rr(<2 x double> %a0) #0 {
+; CHECK-LABEL: define <2 x double> @test_rsqrt_pd_128_rr(
+; CHECK-SAME: <2 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> [[A0]], <2 x double> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 -1)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_rsqrt_pd_128_rrkz(<2 x double> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_rsqrt_pd_128_rrkz(
+; CHECK-SAME: <2 x double> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> [[A0]], <2 x double> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 %mask)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_rsqrt_pd_128_rrk(<2 x double> %a0, <2 x double> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_rsqrt_pd_128_rrk(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> [[A0]], <2 x double> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> %a0, <2 x double> %a1, i8 %mask)
+  ret <2 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double>, <2 x double>, i8) nounwind readnone
+
+define <4 x double> @test_rcp_pd_256_rr(<4 x double> %a0) #0 {
+; CHECK-LABEL: define <4 x double> @test_rcp_pd_256_rr(
+; CHECK-SAME: <4 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> [[A0]], <4 x double> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 -1)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_rcp_pd_256_rrkz(<4 x double> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_rcp_pd_256_rrkz(
+; CHECK-SAME: <4 x double> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> [[A0]], <4 x double> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 %mask)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_rcp_pd_256_rrk(<4 x double> %a0, <4 x double> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_rcp_pd_256_rrk(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> [[A0]], <4 x double> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> %a0, <4 x double> %a1, i8 %mask)
+  ret <4 x double> %res
+}
+
+define <2 x double> @test_rcp_pd_128_rr(<2 x double> %a0) #0 {
+; CHECK-LABEL: define <2 x double> @test_rcp_pd_128_rr(
+; CHECK-SAME: <2 x double> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> [[A0]], <2 x double> zeroinitializer, i8 -1)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 -1)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_rcp_pd_128_rrkz(<2 x double> %a0, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_rcp_pd_128_rrkz(
+; CHECK-SAME: <2 x double> [[A0:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i8 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> [[A0]], <2 x double> zeroinitializer, i8 [[MASK]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 %mask)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_rcp_pd_128_rrk(<2 x double> %a0, <2 x double> %a1, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_rcp_pd_128_rrk(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> [[A0]], <2 x double> [[A1]], i8 [[MASK]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> %a0, <2 x double> %a1, i8 %mask)
+  ret <2 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double>, <2 x double>, i8) nounwind readnone
+
+declare <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double>, <4 x i64>)
+
+define <4 x double>@test_int_x86_avx512_permvar_df_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2) #0 {
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_permvar_df_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x double> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP5]] to <4 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> [[TMP3]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP6]] to <4 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> [[X0]], <4 x i64> [[X1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %x0, <4 x i64> %x1)
+  ret <4 x double> %1
+}
+
+define <4 x double>@test_int_x86_avx512_mask_permvar_df_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_permvar_df_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i64> [[TMP8]] to <4 x double>
+; CHECK-NEXT:    [[TMP16:%.*]] = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> [[TMP14]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <4 x double> [[TMP16]] to <4 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i64> [[TMP11]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> [[X0]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i64> [[TMP18]], <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[X2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i64> [[TMP6]], [[TMP18]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i64> [[TMP7]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP17]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x double> [[TMP1]], <4 x double> [[X2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP9]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x double> %1, <4 x double> %x2
+  ret <4 x double> %3
+}
+
+define <4 x double>@test_int_x86_avx512_maskz_permvar_df_256(<4 x double> %x0, <4 x i64> %x1, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_maskz_permvar_df_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i64> [[X1:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i64> [[TMP10]] to <4 x double>
+; CHECK-NEXT:    [[TMP15:%.*]] = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> [[TMP13]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x double> [[TMP15]] to <4 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i64> [[TMP11]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> [[X0]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP12]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP9]], <8 x i1> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i64> [[TMP16]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], [[TMP16]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP7]], <4 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x double> [[TMP1]], <4 x double> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP8]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x double> %1, <4 x double> zeroinitializer
+  ret <4 x double> %3
+}
+
+declare <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64>, <4 x i64>)
+
+define <4 x i64>@test_int_x86_avx512_permvar_di_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) #0 {
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_permvar_di_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> [[TMP3]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> [[X0]], <4 x i64> [[X1]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %x0, <4 x i64> %x1)
+  ret <4 x i64> %1
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_permvar_di_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_mask_permvar_di_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> [[TMP5]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> [[X0]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP6]], <8 x i1> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i64> [[_MSPROP]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[TMP1]], [[X2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <4 x i64> [[TMP4]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <4 x i64> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i64> [[TMP11]], <4 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i64> [[TMP1]], <4 x i64> [[X2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
+  ret <4 x i64> %3
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_permvar_di_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_maskz_permvar_di_256(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> [[TMP8]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> [[X0]], <4 x i64> [[X1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i64> [[_MSPROP]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[TMP4]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP1]], <4 x i64> [[TMP6]], <4 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i64> [[TMP1]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> zeroinitializer
+  ret <4 x i64> %3
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.fixupimm.pd.128(<2 x double>, <2 x double>, <2 x i64>, i32, i8)
+
+define <2 x double>@test_int_x86_avx512_mask_fixupimm_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_fixupimm_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.mask.fixupimm.pd.128(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.mask.fixupimm.pd.128(<2 x double> zeroinitializer, <2 x double> [[X1]], <2 x i64> [[X2]], i32 4, i8 [[X4]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <2 x double> @llvm.x86.avx512.mask.fixupimm.pd.128(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> [[X2]], i32 3, i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <2 x double> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES4]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.mask.fixupimm.pd.128(<2 x double> %x0, <2 x double> %x1,<2 x i64> %x2, i32 5, i8 %x4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.pd.128(<2 x double> zeroinitializer, <2 x double> %x1, <2 x i64> %x2, i32 4, i8 %x4)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 3, i8 -1)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res3, %res2
+  ret <2 x double> %res4
+}
+
+declare <2 x double> @llvm.x86.avx512.maskz.fixupimm.pd.128(<2 x double>, <2 x double>, <2 x i64>, i32, i8)
+
+define <2 x double>@test_int_x86_avx512_maskz_fixupimm_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_maskz_fixupimm_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.pd.128(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.pd.128(<2 x double> [[X0]], <2 x double> [[X1]], <2 x i64> zeroinitializer, i32 3, i8 [[X4]])
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <2 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES3]]
+;
+  %res = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4)
+  %res1 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x i64> zeroinitializer, i32 3, i8 %x4)
+  ;%res2 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 4, i8 -1)
+  %res3 = fadd <2 x double> %res, %res1
+  ;%res4 = fadd <2 x double> %res3, %res2
+  ret <2 x double> %res3
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.fixupimm.pd.256(<4 x double>, <4 x double>, <4 x i64>, i32, i8)
+
+define <4 x double>@test_int_x86_avx512_mask_fixupimm_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_fixupimm_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.mask.fixupimm.pd.256(<4 x double> [[X0]], <4 x double> [[X1]], <4 x i64> [[X2]], i32 4, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i256 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx512.mask.fixupimm.pd.256(<4 x double> zeroinitializer, <4 x double> [[X1]], <4 x i64> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i256 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x double> @llvm.x86.avx512.mask.fixupimm.pd.256(<4 x double> [[X0]], <4 x double> [[X1]], <4 x i64> [[X2]], i32 3, i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x double> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES4]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.mask.fixupimm.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x i64> %x2, i32 4, i8 %x4)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.fixupimm.pd.256(<4 x double> zeroinitializer, <4 x double> %x1, <4 x i64> %x2 , i32 5, i8 %x4)
+  %res2 = call <4 x double> @llvm.x86.avx512.mask.fixupimm.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x i64> %x2, i32 3, i8 -1)
+  %res3 = fadd <4 x double> %res, %res1
+  %res4 = fadd <4 x double> %res3, %res2
+  ret <4 x double> %res4
+}
+
+declare <4 x double> @llvm.x86.avx512.maskz.fixupimm.pd.256(<4 x double>, <4 x double>, <4 x i64>, i32, i8)
+
+define <4 x double>@test_int_x86_avx512_maskz_fixupimm_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x i64> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_maskz_fixupimm_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx512.maskz.fixupimm.pd.256(<4 x double> [[X0]], <4 x double> [[X1]], <4 x i64> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i256 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx512.maskz.fixupimm.pd.256(<4 x double> [[X0]], <4 x double> [[X1]], <4 x i64> zeroinitializer, i32 4, i8 [[X4]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i256 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x double> @llvm.x86.avx512.maskz.fixupimm.pd.256(<4 x double> [[X0]], <4 x double> [[X1]], <4 x i64> [[X2]], i32 3, i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x double> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x double> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES4]]
+;
+  %res = call <4 x double> @llvm.x86.avx512.maskz.fixupimm.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x i64> %x2, i32 5, i8 %x4)
+  %res1 = call <4 x double> @llvm.x86.avx512.maskz.fixupimm.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x i64> zeroinitializer, i32 4, i8 %x4)
+  %res2 = call <4 x double> @llvm.x86.avx512.maskz.fixupimm.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x i64> %x2, i32 3, i8 -1)
+  %res3 = fadd <4 x double> %res, %res1
+  %res4 = fadd <4 x double> %res3, %res2
+  ret <4 x double> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float>, <4 x float>, <4 x i32>, i32, i8)
+
+define <4 x float>@test_int_x86_avx512_mask_fixupimm_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_fixupimm_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> zeroinitializer, i32 6, i8 [[X4]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> [[X2]], i32 7, i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 7, i8 -1)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res3, %res2
+  ret <4 x float> %res4
+}
+
+declare <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float>, <4 x float>, <4 x i32>, i32, i8)
+
+define <4 x float>@test_int_x86_avx512_maskz_fixupimm_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_maskz_fixupimm_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> zeroinitializer, i32 6, i8 [[X4]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i128 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i128 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i128 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> [[X0]], <4 x float> [[X1]], <4 x i32> [[X2]], i32 7, i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <4 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <4 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES4]]
+;
+  %res = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4)
+  %res1 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 7, i8 -1)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res3, %res2
+  ret <4 x float> %res4
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float>, <8 x float>, <8 x i32>, i32, i8)
+
+define <8 x float>@test_int_x86_avx512_mask_fixupimm_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask_fixupimm_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x i32> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i256 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x i32> zeroinitializer, i32 6, i8 [[X4]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i256 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x i32> [[X2]], i32 7, i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <8 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <8 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES4]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 %x4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 7, i8 -1)
+  %res3 = fadd <8 x float> %res, %res1
+  %res4 = fadd <8 x float> %res3, %res2
+  ret <8 x float> %res4
+}
+
+declare <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float>, <8 x float>, <8 x i32>, i32, i8)
+
+define <8 x float>@test_int_x86_avx512_maskz_fixupimm_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i8 %x4) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_maskz_fixupimm_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x i32> [[X2]], i32 5, i8 [[X4]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i256 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    [[_MSCMP9:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR10:%.*]] = or i1 [[_MSOR8]], [[_MSCMP9]]
+; CHECK-NEXT:    br i1 [[_MSOR10]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x i32> zeroinitializer, i32 6, i8 [[X4]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP11:%.*]] = icmp ne i256 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP12:%.*]] = icmp ne i256 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR13:%.*]] = or i1 [[_MSCMP11]], [[_MSCMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP14:%.*]] = icmp ne i256 [[TMP16]], 0
+; CHECK-NEXT:    [[_MSOR15:%.*]] = or i1 [[_MSOR13]], [[_MSCMP14]]
+; CHECK-NEXT:    br i1 [[_MSOR15]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[RES2:%.*]] = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> [[X0]], <8 x float> [[X1]], <8 x i32> [[X2]], i32 7, i8 -1)
+; CHECK-NEXT:    [[RES3:%.*]] = fadd <8 x float> [[RES]], [[RES1]]
+; CHECK-NEXT:    [[RES4:%.*]] = fadd <8 x float> [[RES3]], [[RES2]]
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES4]]
+;
+  %res = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 %x4)
+  %res1 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 7, i8 -1)
+  %res3 = fadd <8 x float> %res, %res1
+  %res4 = fadd <8 x float> %res3, %res2
+  ret <8 x float> %res4
+}
+
+define <2 x i64> @test_x86_avx512_psra_q_128(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_psra_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+define <2 x i64> @test_x86_avx512_mask_psra_q_128(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %passthru, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_mask_psra_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], <2 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i128 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i1 [[TMP8]] to i128
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i128 [[TMP9]] to <2 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <2 x i64> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP16]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP14]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[TMP12]], <2 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[RES]], [[PASSTHRU]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <2 x i64> [[TMP4]], [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <2 x i64> [[TMP17]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP18]], <2 x i64> [[TMP15]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[RES]], <2 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES2]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <2 x i32> <i32 0, i32 1>
+  %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> %passthru
+  ret <2 x i64> %res2
+}
+define <2 x i64> @test_x86_avx512_maskz_psra_q_128(<2 x i64> %a0, <2 x i64> %a1, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_maskz_psra_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i128 [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i128
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i128 [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <2 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP15]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP13]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[TMP11]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[TMP4]], [[TMP11]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP6]], <2 x i64> [[TMP14]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[RES]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES2]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <2 x i32> <i32 0, i32 1>
+  %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> zeroinitializer
+  ret <2 x i64> %res2
+}
+declare <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx512_psra_q_256(<4 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_psra_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+define <4 x i64> @test_x86_avx512_mask_psra_q_256(<4 x i64> %a0, <2 x i64> %a1, <4 x i64> %passthru, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_mask_psra_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], <4 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 88) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i128 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i1 [[TMP8]] to i256
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i256 [[TMP9]] to <4 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i64> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP16]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[TMP12]], <4 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[RES]], [[PASSTHRU]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i64> [[TMP4]], [[TMP12]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i64> [[TMP17]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP18]], <4 x i64> [[TMP15]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[RES]], <4 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES2]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> %passthru
+  ret <4 x i64> %res2
+}
+define <4 x i64> @test_x86_avx512_maskz_psra_q_256(<4 x i64> %a0, <2 x i64> %a1, <4 x i64> %passthru, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_maskz_psra_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], <4 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 88) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i128 [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i256
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i256 [[TMP8]] to <4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <4 x i64> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[TMP15]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[TMP11]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[TMP4]], [[TMP11]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP6]], <4 x i64> [[TMP14]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[RES]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES2]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> zeroinitializer
+  ret <4 x i64> %res2
+}
+declare <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64>, <2 x i64>) nounwind readnone
+
+
+define <2 x i64> @test_x86_avx512_psrai_q_128(<2 x i64> %a0) #0 {
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_psrai_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> [[TMP3]], i32 7)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> [[A0]], i32 7)
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+define <2 x i64> @test_x86_avx512_mask_psrai_q_128(<2 x i64> %a0, <2 x i64> %passthru, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_mask_psrai_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> [[TMP5]], i32 7)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> [[A0]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP6]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP7]], <8 x i1> [[TMP8]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[TMP2]], <2 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[RES]], [[PASSTHRU]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <2 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <2 x i64> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP12]], <2 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[RES]], <2 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES2]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <2 x i32> <i32 0, i32 1>
+  %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> %passthru
+  ret <2 x i64> %res2
+}
+define <2 x i64> @test_x86_avx512_maskz_psrai_q_128(<2 x i64> %a0, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_maskz_psrai_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> [[TMP9]], i32 7)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> [[A0]], i32 7)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP10]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP7]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[TMP2]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP6]], <2 x i64> [[TMP8]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[RES]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES2]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <2 x i32> <i32 0, i32 1>
+  %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> zeroinitializer
+  ret <2 x i64> %res2
+}
+declare <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64>, i32) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx512_psrai_q_256(<4 x i64> %a0) #0 {
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_psrai_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> [[TMP3]], i32 7)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> [[A0]], i32 7)
+; CHECK-NEXT:    store <4 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+define <4 x i64> @test_x86_avx512_mask_psrai_q_256(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_mask_psrai_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], <4 x i64> [[PASSTHRU:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> [[TMP5]], i32 7)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> [[A0]], i32 7)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP6]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP7]], <8 x i1> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[TMP2]], <4 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[RES]], [[PASSTHRU]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <4 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i64> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP12]], <4 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[RES]], <4 x i64> [[PASSTHRU]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES2]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> %passthru
+  ret <4 x i64> %res2
+}
+define <4 x i64> @test_x86_avx512_maskz_psrai_q_256(<4 x i64> %a0, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_maskz_psrai_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> [[TMP9]], i32 7)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> [[A0]], i32 7)
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP10]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[TMP2]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP6]], <4 x i64> [[TMP8]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[RES]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES2]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> zeroinitializer
+  ret <4 x i64> %res2
+}
+declare <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64>, i32) nounwind readnone
+
+define <2 x i64> @test_x86_avx512_psrav_q_128(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_psrav_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <2 x i1> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %a0, <2 x i64> %a1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_x86_avx512_mask_psrav_q_128(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_mask_psrav_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], <2 x i64> [[A2:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = sext <2 x i1> [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP11]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[TMP9]], <2 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[RES]], [[A2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP4]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP15]], <2 x i64> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[RES]], <2 x i64> [[A2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES2]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %a0, <2 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <2 x i32> <i32 0, i32 1>
+  %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> %a2
+  ret <2 x i64> %res2
+}
+
+define <2 x i64> @test_x86_avx512_maskz_psrav_q_128(<2 x i64> %a0, <2 x i64> %a1, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <2 x i64> @test_x86_avx512_maskz_psrav_q_128(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = sext <2 x i1> [[TMP13]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> [[TMP1]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <2 x i64> [[TMP7]], [[TMP14]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> [[A0]], <2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP12]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP9]], <8 x i1> [[TMP10]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[TMP8]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP6]], <2 x i64> [[TMP11]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <2 x i1> [[MASK_EXTRACT]], <2 x i64> [[RES]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES2]]
+;
+  %res = call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %a0, <2 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast, <2 x i32> <i32 0, i32 1>
+  %res2 = select <2 x i1> %mask.extract, <2 x i64> %res, <2 x i64> zeroinitializer
+  ret <2 x i64> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64>, <2 x i64>) nounwind readnone
+
+define <4 x i64> @test_x86_avx512_psrav_q_256(<4 x i64> %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_psrav_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> [[TMP1]], <4 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> [[A0]], <4 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %a0, <4 x i64> %a1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_x86_avx512_mask_psrav_q_256(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2, i8 %mask, i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_mask_psrav_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]], <4 x i64> [[A2:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = sext <4 x i1> [[TMP6]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> [[TMP1]], <4 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> [[A0]], <4 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[TMP9]], <4 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[RES]], [[A2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP4]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP14]], [[TMP5]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP15]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[RES]], <4 x i64> [[A2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES2]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %a0, <4 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast , <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> %a2
+  ret <4 x i64> %res2
+}
+
+define <4 x i64> @test_x86_avx512_maskz_psrav_q_256(<4 x i64> %a0, <4 x i64> %a1, i8 %mask,  i8 %mask2) #0 {
+;
+; CHECK-LABEL: define <4 x i64> @test_x86_avx512_maskz_psrav_q_256(
+; CHECK-SAME: <4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]], i8 [[MASK:%.*]], i8 [[MASK2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = sext <4 x i1> [[TMP13]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> [[TMP1]], <4 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i64> [[TMP7]], [[TMP14]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> [[A0]], <4 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP12]] to <8 x i1>
+; CHECK-NEXT:    [[MASK2_CAST:%.*]] = bitcast i8 [[MASK2]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP9]], <8 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[MASK_EXTRACT:%.*]] = shufflevector <8 x i1> [[MASK_CAST]], <8 x i1> [[MASK2_CAST]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[TMP8]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i64> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP6]], <4 x i64> [[TMP11]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <4 x i1> [[MASK_EXTRACT]], <4 x i64> [[RES]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES2]]
+;
+  %res = call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %a0, <4 x i64> %a1)
+  %mask.cast = bitcast i8 %mask to <8 x i1>
+  %mask2.cast = bitcast i8 %mask2 to <8 x i1>
+  %mask.extract = shufflevector <8 x i1> %mask.cast, <8 x i1> %mask2.cast , <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res2 = select <4 x i1> %mask.extract, <4 x i64> %res, <4 x i64> zeroinitializer
+  ret <4 x i64> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64>, <4 x i64>) nounwind readnone
+
+define <8 x float> @test_vfmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-LABEL: define <8 x float> @test_vfmadd256_ps(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A0]], <8 x float> [[A1]], <8 x float> [[A2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  ret <8 x float> %1
+}
+
+define <8 x float> @test_mask_vfmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mask_vfmadd256_ps(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A0]], <8 x float> [[A1]], <8 x float> [[A2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[A0]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i32> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP14]], <8 x i32> [[TMP13]], <8 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[A0]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP9]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %a0
+  ret <8 x float> %3
+}
+
+define <4 x float> @test_vfmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-LABEL: define <4 x float> @test_vfmadd128_ps(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_mask_vfmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP13]], <4 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[A0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %a0
+  ret <4 x float> %3
+}
+
+define <4 x double> @test_fmadd256_pd(<4 x double> %a, <4 x double> %b, <4 x double> %c) #0 {
+; CHECK-LABEL: define <4 x double> @test_fmadd256_pd(
+; CHECK-SAME: <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x double> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A]], <4 x double> [[B]], <4 x double> [[C]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)
+  ret <4 x double> %1
+}
+
+define <4 x double> @test_mask_fmadd256_pd(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_fmadd256_pd(
+; CHECK-SAME: <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x double> [[C:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A]], <4 x double> [[B]], <4 x double> [[C]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP1]], <4 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[A]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i64> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i64> [[TMP13]], <4 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[A]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP9]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %a
+  ret <4 x double> %3
+}
+
+define <2 x double> @test_fmadd128_pd(<2 x double> %a, <2 x double> %b, <2 x double> %c) #0 {
+; CHECK-LABEL: define <2 x double> @test_fmadd128_pd(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A]], <2 x double> [[B]], <2 x double> [[C]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %1
+}
+
+define <2 x double> @test_mask_fmadd128_pd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_fmadd128_pd(
+; CHECK-SAME: <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A]], <2 x double> [[B]], <2 x double> [[C]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP1]], <2 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x double> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x double> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <2 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <2 x i64> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i64> [[TMP12]], [[TMP7]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP2]], <2 x i64> [[TMP13]], <2 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP1]], <2 x double> [[A]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP9]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x double> %1, <2 x double> %a
+  ret <2 x double> %3
+}
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask3_vfmadd_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[X2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP1]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x double> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x double> [[X2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <2 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <2 x i64> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i64> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP2]], <2 x i64> [[TMP13]], <2 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP1]], <2 x double> [[X2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP9]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x double> %1, <2 x double> %x2
+  ret <2 x double> %3
+}
+
+define <2 x double>@test_int_x86_avx512_maskz_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_maskz_vfmadd_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP1]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x double> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <2 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP2]], <2 x i64> [[TMP7]], <2 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP1]], <2 x double> zeroinitializer
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x double> %1, <2 x double> zeroinitializer
+  ret <2 x double> %3
+}
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask3_vfmadd_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[X2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP1]], <4 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[X2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i64> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i64> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i64> [[TMP13]], <4 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[X2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP9]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %x2
+  ret <4 x double> %3
+}
+
+define <4 x double>@test_int_x86_avx512_maskz_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_maskz_vfmadd_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP1]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i64> [[TMP7]], <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP8]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> zeroinitializer
+  ret <4 x double> %3
+}
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask3_vfmadd_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[X2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[X2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP13]], <4 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[X2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %x2
+  ret <4 x float> %3
+}
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_maskz_vfmadd_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP7]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> zeroinitializer
+  ret <4 x float> %3
+}
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask3_vfmadd_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[X2]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[X2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <8 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i32> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP14]], <8 x i32> [[TMP13]], <8 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[X2]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP9]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %x2
+  ret <8 x float> %3
+}
+
+define <8 x float>@test_int_x86_avx512_maskz_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_maskz_vfmadd_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[_MSPROP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <8 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP7]], <8 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP8]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> zeroinitializer
+  ret <8 x float> %3
+}
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask3_vfmsub_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSPROP1]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP2]], <2 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x double> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[X2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i64> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP3]], <2 x i64> [[TMP14]], <2 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP2]], <2 x double> [[X2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP10]]
+;
+  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %2 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %1)
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <2 x i32> <i32 0, i32 1>
+  %4 = select <2 x i1> %extract, <2 x double> %2, <2 x double> %x2
+  ret <2 x double> %4
+}
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask3_vfmsub_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> [[_MSPROP1]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP2]], <4 x i64> [[TMP8]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[X2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i64> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i64> [[TMP14]], <4 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP2]], <4 x double> [[X2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP10]]
+;
+  %1 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %x2
+  %2 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %1)
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x double> %2, <4 x double> %x2
+  ret <4 x double> %4
+}
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask3_vfmsub_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP2]], <4 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[X2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i32> [[TMP14]], <4 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP2]], <4 x float> [[X2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP10]]
+;
+  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %2 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %1)
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %2, <4 x float> %x2
+  ret <4 x float> %4
+}
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask3_vfmsub_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[_MSPROP1]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[_MSPROP2]], <8 x i32> [[TMP8]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[X2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP13]], [[TMP8]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP15]], <8 x i32> [[TMP14]], <8 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP3]], <8 x float> [[TMP2]], <8 x float> [[X2]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP10]]
+;
+  %1 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %2 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %1)
+  %3 = bitcast i8 %x3 to <8 x i1>
+  %4 = select <8 x i1> %3, <8 x float> %2, <8 x float> %x2
+  ret <8 x float> %4
+}
+
+define <8 x float> @test_vfnmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-LABEL: define <8 x float> @test_vfnmadd256_ps(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP5]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[_MSPROP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A0]], <8 x float> [[TMP1]], <8 x float> [[A2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a0, <8 x float> %1, <8 x float> %a2)
+  ret <8 x float> %2
+}
+
+define <8 x float> @test_mask_vfnmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mask_vfnmadd256_ps(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[_MSPROP1]], [[TMP12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A0]], <8 x float> [[TMP1]], <8 x float> [[A2]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[_MSPROP2]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[A0]] to <8 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP13]], [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP15]], <8 x i32> [[TMP14]], <8 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP3]], <8 x float> [[TMP2]], <8 x float> [[A0]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP10]]
+;
+  %1 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a0, <8 x float> %1, <8 x float> %a2)
+  %3 = bitcast i8 %mask to <8 x i1>
+  %4 = select <8 x i1> %3, <8 x float> %2, <8 x float> %a0
+  ret <8 x float> %4
+}
+
+define <4 x float> @test_vfnmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-LABEL: define <4 x float> @test_vfnmadd128_ps(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP5]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSPROP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[TMP1]], <4 x float> [[A2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+;
+  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %1, <4 x float> %a2)
+  ret <4 x float> %2
+}
+
+define <4 x float> @test_mask_vfnmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfnmadd128_ps(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSPROP1]], [[TMP12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[TMP1]], <4 x float> [[A2]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP2]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP13]], [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i32> [[TMP14]], <4 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP2]], <4 x float> [[A0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP10]]
+;
+  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %1, <4 x float> %a2)
+  %3 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x float> %2, <4 x float> %a0
+  ret <4 x float> %4
+}
+
+define <4 x double> @test_vfnmadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-LABEL: define <4 x double> @test_vfnmadd256_pd(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], <4 x double> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[TMP5]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> [[_MSPROP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[TMP1]], <4 x double> [[A2]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %1 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %1, <4 x double> %a2)
+  ret <4 x double> %2
+}
+
+define <4 x double> @test_mask_vfnmadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_vfnmadd256_pd(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], <4 x double> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> [[_MSPROP1]], [[TMP12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[TMP1]], <4 x double> [[A2]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP2]], <4 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[A0]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i64> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP13]], [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i64> [[TMP14]], <4 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP2]], <4 x double> [[A0]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP10]]
+;
+  %1 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %1, <4 x double> %a2)
+  %3 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract, <4 x double> %2, <4 x double> %a0
+  ret <4 x double> %4
+}
+
+define <2 x double> @test_vfnmadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-LABEL: define <2 x double> @test_vfnmadd128_pd(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP5]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSPROP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[TMP1]], <2 x double> [[A2]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %1, <2 x double> %a2)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_mask_vfnmadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_vfnmadd128_pd(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP11]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSPROP1]], [[TMP12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[TMP1]], <2 x double> [[A2]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP2]], <2 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x double> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[A0]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i64> [[TMP7]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP13]], [[TMP11]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP3]], <2 x i64> [[TMP14]], <2 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP2]], <2 x double> [[A0]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP10]]
+;
+  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %1, <2 x double> %a2)
+  %3 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %3, <8 x i1> %3, <2 x i32> <i32 0, i32 1>
+  %4 = select <2 x i1> %extract, <2 x double> %2, <2 x double> %a0
+  ret <2 x double> %4
+}
+
+define <8 x float> @test_vfnmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-LABEL: define <8 x float> @test_vfnmsub256_ps(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A0]], <8 x float> [[TMP1]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %1 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a2
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a0, <8 x float> %1, <8 x float> %2)
+  ret <8 x float> %3
+}
+
+define <8 x float> @test_mask_vfnmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mask_vfnmsub256_ps(
+; CHECK-SAME: <8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], <8 x float> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[TMP12]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A0]], <8 x float> [[TMP1]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP4]], <8 x i32> [[_MSPROP3]], <8 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[A0]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP15]], <8 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP4]], <8 x float> [[TMP3]], <8 x float> [[A0]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP11]]
+;
+  %1 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a2
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a0, <8 x float> %1, <8 x float> %2)
+  %4 = bitcast i8 %mask to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x float> %3, <8 x float> %a0
+  ret <8 x float> %5
+}
+
+define <4 x float> @test_vfnmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-LABEL: define <4 x float> @test_vfnmsub128_ps(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a2
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %1, <4 x float> %2)
+  ret <4 x float> %3
+}
+
+define <4 x float> @test_mask_vfnmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfnmsub128_ps(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <4 x float> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP12]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP3]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP4]], <4 x i32> [[TMP15]], <4 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP3]], <4 x float> [[A0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP11]]
+;
+  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a1
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a2
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %1, <4 x float> %2)
+  %4 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %4, <8 x i1> %4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract, <4 x float> %3, <4 x float> %a0
+  ret <4 x float> %5
+}
+
+define <4 x double> @test_vfnmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-LABEL: define <4 x double> @test_vfnmsub256_pd(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], <4 x double> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[TMP1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP3]]
+;
+  %1 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %1, <4 x double> %2)
+  ret <4 x double> %3
+}
+
+define <4 x double> @test_mask_vfnmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_vfnmsub256_pd(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], <4 x double> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> [[TMP12]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[TMP1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP3]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[TMP3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[A0]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP4]], <4 x i64> [[TMP15]], <4 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP3]], <4 x double> [[A0]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP11]]
+;
+  %1 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %1, <4 x double> %2)
+  %4 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %4, <8 x i1> %4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract, <4 x double> %3, <4 x double> %a0
+  ret <4 x double> %5
+}
+
+define <2 x double> @test_vfnmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-LABEL: define <2 x double> @test_vfnmsub128_pd(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %1, <2 x double> %2)
+  ret <2 x double> %3
+}
+
+define <2 x double> @test_mask_vfnmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_vfnmsub128_pd(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[TMP12]], [[_MSPROP]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> [[TMP16]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP3]], <2 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[A0]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP4]], <2 x i64> [[TMP15]], <2 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP3]], <2 x double> [[A0]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP11]]
+;
+  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a1
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %1, <2 x double> %2)
+  %4 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %4, <8 x i1> %4, <2 x i32> <i32 0, i32 1>
+  %5 = select <2 x i1> %extract, <2 x double> %3, <2 x double> %a0
+  ret <2 x double> %5
+}
+
+define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask3_vfnmsub_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[X0]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[TMP1]], <2 x double> [[X1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> [[TMP16]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP3]], <2 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[X2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP4]], <2 x i64> [[TMP15]], <2 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP3]], <2 x double> [[X2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP11]]
+;
+  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x0
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %1, <2 x double> %x1, <2 x double> %2)
+  %4 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %4, <8 x i1> %4, <2 x i32> <i32 0, i32 1>
+  %5 = select <2 x i1> %extract, <2 x double> %3, <2 x double> %x2
+  ret <2 x double> %5
+}
+
+define <4 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask3_vfnmsub_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[X0]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[TMP1]], <4 x double> [[X1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP3]], <4 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[TMP3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[X2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP4]], <4 x i64> [[TMP15]], <4 x i64> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP3]], <4 x double> [[X2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP11]]
+;
+  %1 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %x0
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %1, <4 x double> %x1, <4 x double> %2)
+  %4 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %4, <8 x i1> %4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract, <4 x double> %3, <4 x double> %x2
+  ret <4 x double> %5
+}
+
+define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask3_vfnmsub_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[X0]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[TMP1]], <4 x float> [[X1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP3]], <4 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[X2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP4]], <4 x i32> [[TMP15]], <4 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP3]], <4 x float> [[X2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP11]]
+;
+  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x0
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %1, <4 x float> %x1, <4 x float> %2)
+  %4 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %4, <8 x i1> %4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract, <4 x float> %3, <4 x float> %x2
+  ret <4 x float> %5
+}
+
+define <8 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask3_vfnmsub_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[X0]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[TMP1]], <8 x float> [[X1]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP13]] to <8 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP4]], <8 x i32> [[_MSPROP3]], <8 x i32> [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[X2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP8]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP15]], <8 x i32> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP4]], <8 x float> [[TMP3]], <8 x float> [[X2]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP11]]
+;
+  %1 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x0
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %1, <8 x float> %x1, <8 x float> %2)
+  %4 = bitcast i8 %x3 to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x float> %3, <8 x float> %x2
+  ret <8 x float> %5
+}
+
+define <8 x float> @test_fmaddsub256_ps(<8 x float> %a, <8 x float> %b, <8 x float> %c) #0 {
+; CHECK-LABEL: define <8 x float> @test_fmaddsub256_ps(
+; CHECK-SAME: <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[C]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> zeroinitializer, [[TMP7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[C]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i32> [[_MSPROP4]], <8 x i32> [[_MSPROP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP4]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %2)
+  %4 = shufflevector <8 x float> %3, <8 x float> %1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+  ret <8 x float> %4
+}
+
+define <8 x float> @test_mask_fmaddsub256_ps(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_mask_fmaddsub256_ps(
+; CHECK-SAME: <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[C]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[C]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i32> [[_MSPROP4]], <8 x i32> [[_MSPROP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[_MSPROP5]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x float> [[A]] to <8 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> [[TMP16]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP5]], <8 x float> [[TMP4]], <8 x float> [[A]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP12]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %2)
+  %4 = shufflevector <8 x float> %3, <8 x float> %1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+  %5 = bitcast i8 %mask to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x float> %4, <8 x float> %a
+  ret <8 x float> %6
+}
+
+define <4 x float> @test_fmaddsub128_ps(<4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
+; CHECK-LABEL: define <4 x float> @test_fmaddsub128_ps(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A]], <4 x float> [[B]], <4 x float> [[C]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> zeroinitializer, [[TMP7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[C]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A]], <4 x float> [[B]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i32> [[_MSPROP4]], <4 x i32> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %2)
+  %4 = shufflevector <4 x float> %3, <4 x float> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mask_fmaddsub128_ps(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_fmaddsub128_ps(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A]], <4 x float> [[B]], <4 x float> [[C]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[C]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A]], <4 x float> [[B]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i32> [[_MSPROP4]], <4 x i32> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP5]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP16]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP4]], <4 x float> [[A]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP12]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %2)
+  %4 = shufflevector <4 x float> %3, <4 x float> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x float> %4, <4 x float> %a
+  ret <4 x float> %6
+}
+
+define <4 x double> @test_vfmaddsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-LABEL: define <4 x double> @test_vfmaddsub256_pd(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], <4 x double> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[A1]], <4 x double> [[A2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> zeroinitializer, [[TMP7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[A1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i64> [[_MSPROP4]], <4 x i64> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP4]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %a1, <4 x double> %2)
+  %4 = shufflevector <4 x double> %3, <4 x double> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x double> %4
+}
+
+define <4 x double> @test_mask_vfmaddsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_vfmaddsub256_pd(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], <4 x double> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[A1]], <4 x double> [[A2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[A1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i64> [[_MSPROP4]], <4 x i64> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP5]], <4 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP4]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x double> [[A0]] to <4 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i64> [[TMP15]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i64> [[TMP16]], <4 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP4]], <4 x double> [[A0]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP12]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %a1, <4 x double> %2)
+  %4 = shufflevector <4 x double> %3, <4 x double> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x double> %4, <4 x double> %a0
+  ret <4 x double> %6
+}
+
+define <2 x double> @test_vfmaddsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-LABEL: define <2 x double> @test_vfmaddsub128_pd(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[A2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> zeroinitializer, [[TMP7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <2 x i64> [[_MSPROP4]], <2 x i64> [[_MSPROP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %a1, <2 x double> %2)
+  %4 = shufflevector <2 x double> %3, <2 x double> %1, <2 x i32> <i32 0, i32 3>
+  ret <2 x double> %4
+}
+
+define <2 x double> @test_mask_vfmaddsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_vfmaddsub128_pd(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], <2 x double> [[A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[A2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[A2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <2 x i64> [[_MSPROP4]], <2 x i64> [[_MSPROP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP5]], <2 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x double> [[A0]] to <2 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <2 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <2 x i64> [[TMP15]], [[TMP6]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP6]], <2 x i64> [[TMP16]], <2 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP4]], <2 x double> [[A0]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP12]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %a1, <2 x double> %2)
+  %4 = shufflevector <2 x double> %3, <2 x double> %1, <2 x i32> <i32 0, i32 3>
+  %5 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x double> %4, <2 x double> %a0
+  ret <2 x double> %6
+}
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask3_vfmaddsub_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <2 x i64> [[_MSPROP4]], <2 x i64> [[_MSPROP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP5]], <2 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x double> [[X2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <2 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <2 x i64> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP6]], <2 x i64> [[TMP16]], <2 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP4]], <2 x double> [[X2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP12]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2)
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %2)
+  %4 = shufflevector <2 x double> %3, <2 x double> %1, <2 x i32> <i32 0, i32 3>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x double> %4, <2 x double> %x2
+  ret <2 x double> %6
+}
+
+define <2 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_maskz_vfmaddsub_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <2 x i64> [[_MSPROP4]], <2 x i64> [[_MSPROP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP5]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <2 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP6]], <2 x i64> [[TMP10]], <2 x i64> [[TMP16]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP4]], <2 x double> zeroinitializer
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP11]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2)
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %2)
+  %4 = shufflevector <2 x double> %3, <2 x double> %1, <2 x i32> <i32 0, i32 3>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x double> %4, <2 x double> zeroinitializer
+  ret <2 x double> %6
+}
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask3_vfmaddsub_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i64> [[_MSPROP4]], <4 x i64> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP5]], <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP4]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x double> [[X2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i64> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i64> [[TMP16]], <4 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP4]], <4 x double> [[X2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP12]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2)
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %2)
+  %4 = shufflevector <4 x double> %3, <4 x double> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x double> %4, <4 x double> %x2
+  ret <4 x double> %6
+}
+
+define <4 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_maskz_vfmaddsub_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i64> [[_MSPROP4]], <4 x i64> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP5]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP4]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <4 x i64> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i64> [[TMP10]], <4 x i64> [[TMP16]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP4]], <4 x double> zeroinitializer
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP11]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2)
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %2)
+  %4 = shufflevector <4 x double> %3, <4 x double> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x double> %4, <4 x double> zeroinitializer
+  ret <4 x double> %6
+}
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask3_vfmaddsub_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i32> [[_MSPROP4]], <4 x i32> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP5]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x float> [[X2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP16]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP4]], <4 x float> [[X2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP12]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2)
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %2)
+  %4 = shufflevector <4 x float> %3, <4 x float> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x float> %4, <4 x float> %x2
+  ret <4 x float> %6
+}
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_maskz_vfmaddsub_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i32> [[_MSPROP4]], <4 x i32> [[_MSPROP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <4 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP10]], <4 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP4]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP11]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2)
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %2)
+  %4 = shufflevector <4 x float> %3, <4 x float> %1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x float> %4, <4 x float> zeroinitializer
+  ret <4 x float> %6
+}
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask3_vfmaddsub_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i32> [[_MSPROP4]], <8 x i32> [[_MSPROP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[_MSPROP5]], <8 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x float> [[X2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> [[TMP16]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP5]], <8 x float> [[TMP4]], <8 x float> [[X2]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP12]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2)
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %2)
+  %4 = shufflevector <8 x float> %3, <8 x float> %1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x float> %4, <8 x float> %x2
+  ret <8 x float> %6
+}
+
+define <8 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_maskz_vfmaddsub_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i32> [[_MSPROP4]], <8 x i32> [[_MSPROP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[_MSPROP5]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <8 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP15]], <8 x i32> [[TMP10]], <8 x i32> [[TMP16]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x float> [[TMP4]], <8 x float> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP11]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2)
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %2)
+  %4 = shufflevector <8 x float> %3, <8 x float> %1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x float> %4, <8 x float> zeroinitializer
+  ret <8 x float> %6
+}
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask3_vfmsubadd_pd_128(
+; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x double> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <2 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <2 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <2 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[X0]], <2 x double> [[X1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <2 x i64> [[_MSPROP1]], <2 x i64> [[_MSPROP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP3]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP5]], <2 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x double> [[X2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <2 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i64> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <2 x i64> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP6]], <2 x i64> [[TMP16]], <2 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP4]], <2 x double> [[X2]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP12]]
+;
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2)
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %2)
+  %4 = shufflevector <2 x double> %1, <2 x double> %3, <2 x i32> <i32 0, i32 3>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x double> %4, <2 x double> %x2
+  ret <2 x double> %6
+}
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask3_vfmsubadd_pd_256(
+; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x double> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x double> splat (double -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i64> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i64> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[X0]], <4 x double> [[X1]], <4 x double> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i64> [[_MSPROP1]], <4 x i64> [[_MSPROP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP5]], <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP4]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x double> [[X2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i64> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i64> [[TMP16]], <4 x i64> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP4]], <4 x double> [[X2]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP12]]
+;
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2)
+  %2 = fsub <4 x double> <double -0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %x2
+  %3 = call <4 x double> @llvm.fma.v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %2)
+  %4 = shufflevector <4 x double> %1, <4 x double> %3, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x double> %4, <4 x double> %x2
+  ret <4 x double> %6
+}
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask3_vfmsubadd_ps_128(
+; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x float> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[X0]], <4 x float> [[X1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <4 x i32> [[_MSPROP1]], <4 x i32> [[_MSPROP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP5]], <8 x i1> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP5]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x float> [[X2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP16]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP4]], <4 x float> [[X2]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP12]]
+;
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2)
+  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %2)
+  %4 = shufflevector <4 x float> %1, <4 x float> %3, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x float> %4, <4 x float> %x2
+  ret <4 x float> %6
+}
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) #0 {
+;
+; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask3_vfmsubadd_ps_256(
+; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x float> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> splat (float -0.000000e+00), [[X2]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP3]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[X0]], <8 x float> [[X1]], <8 x float> [[TMP2]])
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i32> [[_MSPROP1]], <8 x i32> [[_MSPROP4]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP3]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[_MSPROP5]], <8 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x float> [[X2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP9]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> [[TMP16]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP5]], <8 x float> [[TMP4]], <8 x float> [[X2]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[TMP12]]
+;
+  %1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2)
+  %2 = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %3 = call <8 x float> @llvm.fma.v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %2)
+  %4 = shufflevector <8 x float> %1, <8 x float> %3, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x float> %4, <8 x float> %x2
+  ret <8 x float> %6
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmk(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmk(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <4 x float>, ptr [[PTR_A2]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i32> [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP18]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[A0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %a2 = load <4 x float>, ptr %ptr_a2
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %a0
+  ret <4 x float> %3
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmka(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmka(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <4 x float>, ptr [[PTR_A2]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i32> [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP18]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[A0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %a2 = load <4 x float>, ptr %ptr_a2, align 8
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %a0
+  ret <4 x float> %3
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmkz(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmkz(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <4 x float>, ptr [[PTR_A2]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %a2 = load <4 x float>, ptr %ptr_a2
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmkza(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmkza(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <4 x float>, ptr [[PTR_A2]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %a2 = load <4 x float>, ptr %ptr_a2, align 4
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmb(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmb(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_A2]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP15]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> poison, float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x float> [[VECINIT_I]], float [[Q]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <4 x float> [[VECINIT2_I]], float [[Q]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <4 x float> [[VECINIT4_I]], float [[Q]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[VECINIT6_I]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP5]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i32> [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP18]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[A0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %q = load float, ptr %ptr_a2
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %a0
+  ret <4 x float> %3
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmba(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmba(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_A2]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP15]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> poison, float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x float> [[VECINIT_I]], float [[Q]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <4 x float> [[VECINIT2_I]], float [[Q]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <4 x float> [[VECINIT4_I]], float [[Q]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[VECINIT6_I]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP5]], <4 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP6]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i32> [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP18]], <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[A0]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+;
+  %q = load float, ptr %ptr_a2, align 4
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x float> %1, <4 x float> %a0
+  ret <4 x float> %3
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmbz(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmbz(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_A2]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> poison, float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x float> [[VECINIT_I]], float [[Q]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <4 x float> [[VECINIT2_I]], float [[Q]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <4 x float> [[VECINIT4_I]], float [[Q]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[VECINIT6_I]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %q = load float, ptr %ptr_a2
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmbza(<4 x float> %a0, <4 x float> %a1, ptr %ptr_a2) #0 {
+;
+; CHECK-LABEL: define <4 x float> @test_mask_vfmadd128_ps_rmbza(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[PTR_A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[Q:%.*]] = load float, ptr [[PTR_A2]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <4 x float> poison, float [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = insertelement <4 x i32> [[_MSPROP]], i32 [[_MSLD]], i32 1
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <4 x float> [[VECINIT_I]], float [[Q]], i32 1
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = insertelement <4 x i32> [[_MSPROP1]], i32 [[_MSLD]], i32 2
+; CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <4 x float> [[VECINIT2_I]], float [[Q]], i32 2
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = insertelement <4 x i32> [[_MSPROP2]], i32 [[_MSLD]], i32 3
+; CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <4 x float> [[VECINIT4_I]], float [[Q]], i32 3
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[_MSPROP4]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[VECINIT6_I]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %q = load float, ptr %ptr_a2, align 4
+  %vecinit.i = insertelement <4 x float> poison, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i)
+  ret <4 x float> %1
+}
+
+define <2 x double> @test_mask_vfmadd128_pd_rmk(<2 x double> %a0, <2 x double> %a1, ptr %ptr_a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_vfmadd128_pd_rmk(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], ptr [[PTR_A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <2 x double>, ptr [[PTR_A2]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[A2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[_MSPROP1]], <2 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x double> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x double> [[A0]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <2 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <2 x i64> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <2 x i64> [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP2]], <2 x i64> [[TMP18]], <2 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP1]], <2 x double> [[A0]]
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP9]]
+;
+  %a2 = load <2 x double>, ptr %ptr_a2
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x double> %1, <2 x double> %a0
+  ret <2 x double> %3
+}
+
+define <2 x double> @test_mask_vfmadd128_pd_rmkz(<2 x double> %a0, <2 x double> %a1, ptr %ptr_a2) #0 {
+;
+; CHECK-LABEL: define <2 x double> @test_mask_vfmadd128_pd_rmkz(
+; CHECK-SAME: <2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], ptr [[PTR_A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <2 x double>, ptr [[PTR_A2]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[A2]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %a2 = load <2 x double>, ptr %ptr_a2
+  %1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
+  ret <2 x double> %1
+}
+
+define <4 x double> @test_mask_vfmadd256_pd_rmk(<4 x double> %a0, <4 x double> %a1, ptr %ptr_a2, i8 %mask) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_vfmadd256_pd_rmk(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], ptr [[PTR_A2:%.*]], i8 [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <4 x double>, ptr [[PTR_A2]], align 32
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i64>, ptr [[TMP15]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP12]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[A1]], <4 x double> [[A2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP14]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP11]], <8 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[_MSPROP1]], <4 x i64> [[TMP12]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[A0]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i64> [[TMP6]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i64> [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i64> [[TMP18]], <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[A0]]
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP9]]
+;
+  %a2 = load <4 x double>, ptr %ptr_a2
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %a0
+  ret <4 x double> %3
+}
+
+define <4 x double> @test_mask_vfmadd256_pd_rmkz(<4 x double> %a0, <4 x double> %a1, ptr %ptr_a2) #0 {
+;
+; CHECK-LABEL: define <4 x double> @test_mask_vfmadd256_pd_rmkz(
+; CHECK-SAME: <4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], ptr [[PTR_A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[A2:%.*]] = load <4 x double>, ptr [[PTR_A2]], align 32
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_A2]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[A0]], <4 x double> [[A1]], <4 x double> [[A2]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %a2 = load <4 x double>, ptr %ptr_a2
+  %1 = call <4 x double> @llvm.fma.v4f64(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
+  ret <4 x double> %1
+}
+
+define <8 x i32> @combine_vpermi2d_vpermps(<16 x i32> noundef %a) #0 {
+;
+; CHECK-LABEL: define <8 x i32> @combine_vpermi2d_vpermps(
+; CHECK-SAME: <16 x i32> noundef [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> splat (i32 -1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> splat (i32 -1), <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[_MSPROP]], <8 x i32> <i32 14, i32 13, i32 6, i32 3, i32 5, i32 15, i32 0, i32 1>, <8 x i32> [[_MSPROP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP1]], <8 x i32> <i32 14, i32 13, i32 6, i32 3, i32 5, i32 15, i32 0, i32 1>, <8 x i32> [[TMP2]])
+; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
+;
+  %1 = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = shufflevector <16 x i32> %a, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %3 = tail call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %1, <8 x i32> <i32 14, i32 13, i32 6, i32 3, i32 5, i32 15, i32 0, i32 1>, <8 x i32> %2)
+  ret <8 x i32> %3
+}
+
+declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double>, <2 x double>, <2 x i1>)
+declare <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float>, <4 x float>, <4 x i1>)
+declare <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64>, <2 x i64>, <2 x i1>)
+declare <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32>, <4 x i32>, <4 x i1>)
+declare <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double>, <2 x double>, <2 x i1>)
+declare <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float>, <4 x float>, <4 x i1>)
+declare <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64>, <2 x i64>, <2 x i1>)
+declare <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32>, <4 x i32>, <4 x i1>)
+declare <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double>, <4 x double>, <4 x i1>)
+declare <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float>, <8 x float>, <8 x i1>)
+declare <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64>, <4 x i64>, <4 x i1>)
+declare <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32>, <8 x i32>, <8 x i1>)
+declare <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double>, <4 x double>, <4 x i1>)
+declare <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float>, <8 x float>, <8 x i1>)
+declare <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64>, <4 x i64>, <4 x i1>)
+declare <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32>, <8 x i32>, <8 x i1>)
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll
new file mode 100644
index 000000000000..35e1feb3aa20
--- /dev/null
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll
@@ -0,0 +1,779 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx512f -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/Transforms/InstCombine/X86/x86-vpermi2.ll
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+;
+; vXi64
+;
+
+define <2 x i64> @shuffle_vpermv3_v2i64(<2 x i64> %x0, <2 x i64> %x1) #0 {
+; CHECK-LABEL: define <2 x i64> @shuffle_vpermv3_v2i64(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP1]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[X1]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %r = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x0, <2 x i64> <i64 2, i64 0>, <2 x i64> %x1)
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @shuffle_vpermv3_v2i64_unary(<2 x i64> %x0) #0 {
+; CHECK-LABEL: define <2 x i64> @shuffle_vpermv3_v2i64_unary(
+; CHECK-SAME: <2 x i64> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP1]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[X0]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %r = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x0, <2 x i64> <i64 2, i64 0>, <2 x i64> %x0)
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @shuffle_vpermv3_v2i64_demandedbits(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %m) #0 {
+; CHECK-LABEL: define <2 x i64> @shuffle_vpermv3_v2i64_demandedbits(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i64> [[M]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <2 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <2 x i64> [[TMP4]], <i64 -1, i64 -5>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <2 x i64> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <2 x i64> [[M]], <i64 0, i64 4>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[T]], <2 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[T]], <2 x i64> [[X1]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %t = or <2 x i64> %m, <i64 0, i64 4>
+  %r = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x0, <2 x i64> %t, <2 x i64> %x1)
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @shuffle_vpermv3_v2i64_demandedbits_negative(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %m) #0 {
+; CHECK-LABEL: define <2 x i64> @shuffle_vpermv3_v2i64_demandedbits_negative(
+; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i64> [[M]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <2 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <2 x i64> [[TMP4]], <i64 -1, i64 -3>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <2 x i64> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <2 x i64> [[M]], <i64 0, i64 2>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[T]], <2 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[T]], <2 x i64> [[X1]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %t = or <2 x i64> %m, <i64 0, i64 2>
+  %r = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> %x0, <2 x i64> %t, <2 x i64> %x1)
+  ret <2 x i64> %r
+}
+
+define <4 x i64> @shuffle_vpermv3_v4i64(<4 x i64> %x0, <4 x i64> %x1) #0 {
+; CHECK-LABEL: define <4 x i64> @shuffle_vpermv3_v4i64(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP1]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[X1]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %r = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x0, <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> %x1)
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @shuffle_vpermv3_v4i64_unary(<4 x i64> %x0) #0 {
+; CHECK-LABEL: define <4 x i64> @shuffle_vpermv3_v4i64_unary(
+; CHECK-SAME: <4 x i64> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP1]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[X0]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %r = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x0, <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> %x0)
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @shuffle_vpermv3_v4i64_demandedbits(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %m) #0 {
+; CHECK-LABEL: define <4 x i64> @shuffle_vpermv3_v4i64_demandedbits(
+; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i64> [[M]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <4 x i64> [[TMP4]], <i64 -1, i64 -9, i64 -17, i64 -33>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i64> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <4 x i64> [[M]], <i64 0, i64 8, i64 16, i64 32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP6]], <4 x i64> [[T]], <4 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> [[T]], <4 x i64> [[X1]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[R]]
+;
+  %t = or <4 x i64> %m, <i64 0, i64 8, i64 16, i64 32>
+  %r = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> %x0, <4 x i64> %t, <4 x i64> %x1)
+  ret <4 x i64> %r
+}
+
+define <8 x i64> @shuffle_vpermv3_v8i64(<8 x i64> %x0, <8 x i64> %x1) #0 {
+; CHECK-LABEL: define <8 x i64> @shuffle_vpermv3_v8i64(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[R]]
+;
+  %r = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> %x1)
+  ret <8 x i64> %r
+}
+
+define <8 x i64> @shuffle_vpermv3_v8i64_unary(<8 x i64> %x0) #0 {
+; CHECK-LABEL: define <8 x i64> @shuffle_vpermv3_v8i64_unary(
+; CHECK-SAME: <8 x i64> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[X0]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[R]]
+;
+  %r = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> %x0)
+  ret <8 x i64> %r
+}
+
+define <8 x i64> @shuffle_vpermv3_v8i64_demandedbits(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %m) #0 {
+; CHECK-LABEL: define <8 x i64> @shuffle_vpermv3_v8i64_demandedbits(
+; CHECK-SAME: <8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <8 x i64> [[M]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <8 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i64> [[TMP4]], <i64 -1, i64 -17, i64 -33, i64 -65, i64 -257, i64 -513, i64 -1025, i64 15>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <8 x i64> [[M]], <i64 0, i64 16, i64 32, i64 64, i64 256, i64 512, i64 1024, i64 -16>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP6]], <8 x i64> [[T]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP9]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0]], <8 x i64> [[T]], <8 x i64> [[X1]])
+; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[R]]
+;
+  %t = or <8 x i64> %m, <i64 0, i64 16, i64 32, i64 64, i64 256, i64 512, i64 1024, i64 -16>
+  %r = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %t, <8 x i64> %x1)
+  ret <8 x i64> %r
+}
+
+;
+; vXi32
+;
+
+define <4 x i32> @shuffle_vpermv3_v4i32(<4 x i32> %x0, <4 x i32> %x1) #0 {
+; CHECK-LABEL: define <4 x i32> @shuffle_vpermv3_v4i32(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP1]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[X1]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %r = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x0, <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> %x1)
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_vpermv3_v4i32_unary(<4 x i32> %x0) #0 {
+; CHECK-LABEL: define <4 x i32> @shuffle_vpermv3_v4i32_unary(
+; CHECK-SAME: <4 x i32> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP1]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[X0]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %r = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x0, <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> %x0)
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_vpermv3_v4i32_demandedbits(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %m) #0 {
+; CHECK-LABEL: define <4 x i32> @shuffle_vpermv3_v4i32_demandedbits(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[M]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <4 x i32> [[TMP4]], <i32 -1, i32 -9, i32 -17, i32 -33>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <4 x i32> [[M]], <i32 0, i32 8, i32 16, i32 32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP6]], <4 x i32> [[T]], <4 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> [[T]], <4 x i32> [[X1]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t = or <4 x i32> %m, <i32 0, i32 8, i32 16, i32 32>
+  %r = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> %x0, <4 x i32> %t, <4 x i32> %x1)
+  ret <4 x i32> %r
+}
+
+define <8 x i32> @shuffle_vpermv3_v8i32(<8 x i32> %x0, <8 x i32> %x1) #0 {
+; CHECK-LABEL: define <8 x i32> @shuffle_vpermv3_v8i32(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP1]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[X1]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[R]]
+;
+  %r = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x0, <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> %x1)
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @shuffle_vpermv3_v8i32_unary(<8 x i32> %x0) #0 {
+; CHECK-LABEL: define <8 x i32> @shuffle_vpermv3_v8i32_unary(
+; CHECK-SAME: <8 x i32> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP1]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[X0]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[R]]
+;
+  %r = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x0, <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> %x0)
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @shuffle_vpermv3_v8i32_demandedbits(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %m) #0 {
+; CHECK-LABEL: define <8 x i32> @shuffle_vpermv3_v8i32_demandedbits(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <8 x i32> [[M]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i32> [[TMP4]], <i32 -1, i32 -17, i32 -33, i32 -65, i32 -257, i32 -513, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <8 x i32> [[M]], <i32 0, i32 16, i32 32, i32 64, i32 256, i32 512, i32 -16, i32 -32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP6]], <8 x i32> [[T]], <8 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> [[T]], <8 x i32> [[X1]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[R]]
+;
+  %t = or <8 x i32> %m, <i32 0, i32 16, i32 32, i32 64, i32 256, i32 512, i32 -16, i32 -32>
+  %r = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> %x0, <8 x i32> %t, <8 x i32> %x1)
+  ret <8 x i32> %r
+}
+
+define <16 x i32> @shuffle_vpermv3_v16i32(<16 x i32> %x0, <16 x i32> %x1) #0 {
+; CHECK-LABEL: define <16 x i32> @shuffle_vpermv3_v16i32(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[R]]
+;
+  %r = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> %x1)
+  ret <16 x i32> %r
+}
+
+define <16 x i32> @shuffle_vpermv3_v16i32_unary(<16 x i32> %x0) #0 {
+; CHECK-LABEL: define <16 x i32> @shuffle_vpermv3_v16i32_unary(
+; CHECK-SAME: <16 x i32> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[X0]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[R]]
+;
+  %r = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> %x0)
+  ret <16 x i32> %r
+}
+
+define <16 x i32> @shuffle_vpermv3_v16i32_demandedbits(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %m) #0 {
+; CHECK-LABEL: define <16 x i32> @shuffle_vpermv3_v16i32_demandedbits(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <16 x i32> [[M]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i32> [[TMP4]], <i32 -1, i32 -33, i32 -65, i32 -257, i32 -513, i32 -1025, i32 -2049, i32 -4097, i32 -8193, i32 31, i32 63, i32 127, i32 255, i32 511, i32 1023, i32 2047>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <16 x i32> [[M]], <i32 0, i32 32, i32 64, i32 256, i32 512, i32 1024, i32 2048, i32 4096, i32 8192, i32 -32, i32 -64, i32 -128, i32 -256, i32 -512, i32 -1024, i32 -2048>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP6]], <16 x i32> [[T]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP9]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0]], <16 x i32> [[T]], <16 x i32> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[R]]
+;
+  %t = or <16 x i32> %m, <i32 0, i32 32, i32 64, i32 256, i32 512, i32 1024, i32 2048, i32 4096, i32 8192, i32 -32, i32 -64, i32 -128, i32 -256, i32 -512, i32 -1024, i32 -2048>
+  %r = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %t, <16 x i32> %x1)
+  ret <16 x i32> %r
+}
+
+;
+; vXi16
+;
+
+define <8 x i16> @shuffle_vpermv3_v8i16(<8 x i16> %x0, <8 x i16> %x1) #0 {
+; CHECK-LABEL: define <8 x i16> @shuffle_vpermv3_v8i16(
+; CHECK-SAME: <8 x i16> [[X0:%.*]], <8 x i16> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[TMP1]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[X0]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[X1]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[R]]
+;
+  %r = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> %x0, <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> %x1)
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @shuffle_vpermv3_v8i16_unary(<8 x i16> %x0) #0 {
+; CHECK-LABEL: define <8 x i16> @shuffle_vpermv3_v8i16_unary(
+; CHECK-SAME: <8 x i16> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[TMP1]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[X0]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[X0]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[R]]
+;
+  %r = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> %x0, <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> %x0)
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @shuffle_vpermv3_v8i16_demandedbits(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %m) #0 {
+; CHECK-LABEL: define <8 x i16> @shuffle_vpermv3_v8i16_demandedbits(
+; CHECK-SAME: <8 x i16> [[X0:%.*]], <8 x i16> [[X1:%.*]], <8 x i16> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <8 x i16> [[M]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <8 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i16> [[TMP4]], <i16 -1, i16 -17, i16 -33, i16 -65, i16 -257, i16 -513, i16 15, i16 31>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <8 x i16> [[M]], <i16 0, i16 16, i16 32, i16 64, i16 256, i16 512, i16 -16, i16 -32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[TMP6]], <8 x i16> [[T]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[X0]], <8 x i16> [[T]], <8 x i16> [[X1]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[R]]
+;
+  %t = or <8 x i16> %m, <i16 0, i16 16, i16 32, i16 64, i16 256, i16 512, i16 -16, i16 -32>
+  %r = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> %x0, <8 x i16> %t, <8 x i16> %x1)
+  ret <8 x i16> %r
+}
+
+define <16 x i16> @shuffle_vpermv3_v16i16(<16 x i16> %x0, <16 x i16> %x1) #0 {
+; CHECK-LABEL: define <16 x i16> @shuffle_vpermv3_v16i16(
+; CHECK-SAME: <16 x i16> [[X0:%.*]], <16 x i16> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[TMP1]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[X0]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[X1]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[R]]
+;
+  %r = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> %x1)
+  ret <16 x i16> %r
+}
+
+define <16 x i16> @shuffle_vpermv3_v16i16_unary(<16 x i16> %x0) #0 {
+; CHECK-LABEL: define <16 x i16> @shuffle_vpermv3_v16i16_unary(
+; CHECK-SAME: <16 x i16> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[TMP1]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[X0]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[X0]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[R]]
+;
+  %r = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> %x0)
+  ret <16 x i16> %r
+}
+
+define <16 x i16> @shuffle_vpermv3_v16i16_demandedbits(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %m) #0 {
+; CHECK-LABEL: define <16 x i16> @shuffle_vpermv3_v16i16_demandedbits(
+; CHECK-SAME: <16 x i16> [[X0:%.*]], <16 x i16> [[X1:%.*]], <16 x i16> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <16 x i16> [[M]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i16> [[TMP4]], <i16 -1, i16 -33, i16 -65, i16 -257, i16 -513, i16 -1025, i16 -2049, i16 -4097, i16 31, i16 63, i16 127, i16 255, i16 511, i16 1023, i16 2047, i16 4095>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <16 x i16> [[M]], <i16 0, i16 32, i16 64, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 -32, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[TMP6]], <16 x i16> [[T]], <16 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i16> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[X0]], <16 x i16> [[T]], <16 x i16> [[X1]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[R]]
+;
+  %t = or <16 x i16> %m, <i16 0, i16 32, i16 64, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 -32, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096>
+  %r = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> %t, <16 x i16> %x1)
+  ret <16 x i16> %r
+}
+
+define <32 x i16> @shuffle_vpermv3_v32i16(<32 x i16> %x0, <32 x i16> %x1) #0 {
+; CHECK-LABEL: define <32 x i16> @shuffle_vpermv3_v32i16(
+; CHECK-SAME: <32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[X1]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[R]]
+;
+  %r = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> %x1)
+  ret <32 x i16> %r
+}
+
+define <32 x i16> @shuffle_vpermv3_v32i16_unary(<32 x i16> %x0) #0 {
+; CHECK-LABEL: define <32 x i16> @shuffle_vpermv3_v32i16_unary(
+; CHECK-SAME: <32 x i16> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[X0]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[R]]
+;
+  %r = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> %x0)
+  ret <32 x i16> %r
+}
+
+define <32 x i16> @shuffle_vpermv3_v32i16_demandedbits(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %m) #0 {
+; CHECK-LABEL: define <32 x i16> @shuffle_vpermv3_v32i16_demandedbits(
+; CHECK-SAME: <32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]], <32 x i16> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <32 x i16> [[M]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <32 x i16> [[TMP4]], <i16 -1, i16 -65, i16 -129, i16 -257, i16 -513, i16 -1025, i16 -2049, i16 -4097, i16 -1, i16 63, i16 127, i16 255, i16 511, i16 1023, i16 2047, i16 4095, i16 -1, i16 -65, i16 -129, i16 -257, i16 -513, i16 -1025, i16 -2049, i16 -4097, i16 -1, i16 63, i16 127, i16 255, i16 511, i16 1023, i16 2047, i16 4095>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <32 x i16> [[M]], <i16 0, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 0, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096, i16 0, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 0, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP6]], <32 x i16> [[T]], <32 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[TMP9]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0]], <32 x i16> [[T]], <32 x i16> [[X1]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[R]]
+;
+  %t = or <32 x i16> %m, <i16 0, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 0, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096, i16 0, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 0, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096>
+  %r = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %t, <32 x i16> %x1)
+  ret <32 x i16> %r
+}
+
+;
+; vXi8
+;
+
+define <16 x i8> @shuffle_vpermv3_v16i8(<16 x i8> %x0, <16 x i8> %x1) #0 {
+; CHECK-LABEL: define <16 x i8> @shuffle_vpermv3_v16i8(
+; CHECK-SAME: <16 x i8> [[X0:%.*]], <16 x i8> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[TMP1]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[X0]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[X1]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[R]]
+;
+  %r = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %x0, <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> %x1)
+  ret <16 x i8> %r
+}
+
+define <16 x i8> @shuffle_vpermv3_v16i8_unary(<16 x i8> %x0) #0 {
+; CHECK-LABEL: define <16 x i8> @shuffle_vpermv3_v16i8_unary(
+; CHECK-SAME: <16 x i8> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[TMP1]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[X0]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[X0]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[R]]
+;
+  %r = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %x0, <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> %x0)
+  ret <16 x i8> %r
+}
+
+define <16 x i8> @shuffle_vpermv3_v16i8_demandedbits(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %m) #0 {
+; CHECK-LABEL: define <16 x i8> @shuffle_vpermv3_v16i8_demandedbits(
+; CHECK-SAME: <16 x i8> [[X0:%.*]], <16 x i8> [[X1:%.*]], <16 x i8> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <16 x i8> [[M]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <16 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <16 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i8> [[TMP4]], <i8 -1, i8 -33, i8 -65, i8 127, i8 -1, i8 31, i8 63, i8 127, i8 -1, i8 -33, i8 -65, i8 127, i8 -1, i8 31, i8 63, i8 127>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i8> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i8> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <16 x i8> [[M]], <i8 0, i8 32, i8 64, i8 -128, i8 0, i8 -32, i8 -64, i8 -128, i8 0, i8 32, i8 64, i8 -128, i8 0, i8 -32, i8 -64, i8 -128>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[TMP6]], <16 x i8> [[T]], <16 x i8> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i8> [[TMP9]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[X0]], <16 x i8> [[T]], <16 x i8> [[X1]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[R]]
+;
+  %t = or <16 x i8> %m, <i8 0, i8 32, i8 64, i8 128, i8 0, i8 -32, i8 -64, i8 -128, i8 0, i8 32, i8 64, i8 128, i8 0, i8 -32, i8 -64, i8 -128>
+  %r = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %x0, <16 x i8> %t, <16 x i8> %x1)
+  ret <16 x i8> %r
+}
+
+define <32 x i8> @shuffle_vpermv3_v32i8(<32 x i8> %x0, <32 x i8> %x1) #0 {
+; CHECK-LABEL: define <32 x i8> @shuffle_vpermv3_v32i8(
+; CHECK-SAME: <32 x i8> [[X0:%.*]], <32 x i8> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[TMP1]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[X0]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[X1]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[R]]
+;
+  %r = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %x0, <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> %x1)
+  ret <32 x i8> %r
+}
+
+define <32 x i8> @shuffle_vpermv3_v32i8_unary(<32 x i8> %x0) #0 {
+; CHECK-LABEL: define <32 x i8> @shuffle_vpermv3_v32i8_unary(
+; CHECK-SAME: <32 x i8> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[TMP1]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[X0]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[X0]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[R]]
+;
+  %r = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %x0, <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> %x0)
+  ret <32 x i8> %r
+}
+
+define <32 x i8> @shuffle_vpermv3_v32i8_demandedbits(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %m) #0 {
+; CHECK-LABEL: define <32 x i8> @shuffle_vpermv3_v32i8_demandedbits(
+; CHECK-SAME: <32 x i8> [[X0:%.*]], <32 x i8> [[X1:%.*]], <32 x i8> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <32 x i8> [[M]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <32 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <32 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <32 x i8> [[TMP4]], <i8 -1, i8 -1, i8 -65, i8 127, i8 -1, i8 -1, i8 63, i8 127, i8 -1, i8 -1, i8 -65, i8 127, i8 -1, i8 -1, i8 63, i8 127, i8 -1, i8 -1, i8 -65, i8 127, i8 -1, i8 -1, i8 63, i8 127, i8 -1, i8 -1, i8 -65, i8 127, i8 -1, i8 -1, i8 63, i8 127>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i8> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i8> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <32 x i8> [[M]], <i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[TMP6]], <32 x i8> [[T]], <32 x i8> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i8> [[TMP9]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[X0]], <32 x i8> [[T]], <32 x i8> [[X1]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[R]]
+;
+  %t = or <32 x i8> %m, <i8 0, i8 0, i8 64, i8 128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 128, i8 0, i8 0, i8 -64, i8 -128>
+  %r = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %x0, <32 x i8> %t, <32 x i8> %x1)
+  ret <32 x i8> %r
+}
+
+define <64 x i8> @shuffle_vpermv3_v64i8(<64 x i8> %x0, <64 x i8> %x1) #0 {
+; CHECK-LABEL: define <64 x i8> @shuffle_vpermv3_v64i8(
+; CHECK-SAME: <64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[TMP1]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[TMP2]])
+; CHECK-NEXT:    [[R:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[X0]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[X1]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[R]]
+;
+  %r = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %x0, <64 x i8> <i8 128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> %x1)
+  ret <64 x i8> %r
+}
+
+define <64 x i8> @shuffle_vpermv3_v64i8_unary(<64 x i8> %x0) #0 {
+; CHECK-LABEL: define <64 x i8> @shuffle_vpermv3_v64i8_unary(
+; CHECK-SAME: <64 x i8> [[X0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[TMP1]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[X0]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[X0]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[R]]
+;
+  %r = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %x0, <64 x i8> <i8 128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> %x0)
+  ret <64 x i8> %r
+}
+
+define <64 x i8> @shuffle_vpermv3_v64i8_demandedbits(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %m) #0 {
+; CHECK-LABEL: define <64 x i8> @shuffle_vpermv3_v64i8_demandedbits(
+; CHECK-SAME: <64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], <64 x i8> [[M:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <64 x i8> [[M]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = and <64 x i8> [[TMP4]], <i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127, i8 -1, i8 127>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[T:%.*]] = or <64 x i8> [[M]], <i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[TMP6]], <64 x i8> [[T]], <64 x i8> [[TMP3]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <64 x i8> [[TMP9]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[R:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[X0]], <64 x i8> [[T]], <64 x i8> [[X1]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[R]]
+;
+  %t = or <64 x i8> %m, <i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128, i8 0, i8 128, i8 0, i8 -128>
+  %r = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %x0, <64 x i8> %t, <64 x i8> %x1)
+  ret <64 x i8> %r
+}
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
index 06c62dd44c99..613146ff8227 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
@@ -4,12 +4,6 @@
 ; Handled strictly:
 ; - i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %a0)
 ; - i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %a0)
-; - <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1)
-; - <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1)
-; - <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
-; - <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1)
-; - <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a2)
-; - <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1)
 ; - void @llvm.x86.avx.vzeroall()
 ; - void @llvm.x86.avx.vzeroupper()
 
@@ -996,19 +990,19 @@ define <2 x double> @test_x86_avx_vpermilvar_pd(<2 x double> %a0, <2 x i64> %a1)
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <2 x i64> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A0]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[RES]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A0:%.*]], <2 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <2 x double> [[RES]]
+; CHECK:       9:
+; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A2:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES1]]
 ;
   %res = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1) ; <<2 x double>> [#uses=1]
   ret <2 x double> %res
@@ -1022,19 +1016,19 @@ define <4 x double> @test_x86_avx_vpermilvar_pd_256(<4 x double> %a0, <4 x i64>
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0:%.*]], <4 x i64> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x double> [[RES]]
+; CHECK:       9:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A2:%.*]], <4 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES1]]
 ;
   %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1) ; <<4 x double>> [#uses=1]
   ret <4 x double> %res
@@ -1046,16 +1040,12 @@ define <4 x double> @test_x86_avx_vpermilvar_pd_256_2(<4 x double> %a0) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
-; CHECK:       4:
-; CHECK-NEXT:    call void @__msan_warning_noreturn()
-; CHECK-NEXT:    unreachable
-; CHECK:       5:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0:%.*]], <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x double> [[RES]]
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0]], <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A1:%.*]], <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
+; CHECK-NEXT:    store <4 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES1]]
 ;
   %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 0, i64 2>) ; <<4 x double>> [#uses=1]
   ret <4 x double> %res
@@ -1067,19 +1057,19 @@ define <4 x float> @test_x86_avx_vpermilvar_ps(<4 x float> %a0, <4 x i32> %a1) #
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0:%.*]], <4 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x float> [[RES]]
+; CHECK:       9:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A2:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES1]]
 ;
   %res = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1) ; <<4 x float>> [#uses=1]
   ret <4 x float> %res
@@ -1091,7 +1081,7 @@ define <4 x float> @test_x86_avx_vpermilvar_ps_load(<4 x float> %a0, ptr %a1) #0
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
 ; CHECK:       4:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
@@ -1101,19 +1091,19 @@ define <4 x float> @test_x86_avx_vpermilvar_ps_load(<4 x float> %a0, ptr %a1) #0
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -2147483649
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
-; CHECK:       11:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       12:
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0:%.*]], <4 x i32> [[A2]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <4 x float> [[RES]]
+; CHECK:       14:
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A3:%.*]], <4 x i32> [[A2]])
+; CHECK-NEXT:    store <4 x i32> [[TMP11]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES1]]
 ;
   %a2 = load <4 x i32>, ptr %a1
   %res = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a2) ; <<4 x float>> [#uses=1]
@@ -1128,19 +1118,19 @@ define <8 x float> @test_x86_avx_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    [[A0:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A0]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[RES]] to <8 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret <8 x float> [[RES]]
+; CHECK:       9:
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A2:%.*]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES1]]
 ;
   %res = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1]
   ret <8 x float> %res
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
index 5cc56baf0e0d..9d3e9d63eed2 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
@@ -780,8 +780,15 @@ define <32 x i8> @test_x86_avx2_pshuf_b(<32 x i8> %a0, <32 x i8> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[TMP1]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i8> [[RES]]
 ;
@@ -1021,8 +1028,15 @@ define <8 x i32> @test_x86_avx2_permd(<8 x i32> %a0, <8 x i32> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> [[TMP1]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> [[A0:%.*]], <8 x i32> [[A1]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
 ;
@@ -1038,18 +1052,18 @@ define <8 x float> @test_x86_avx2_permps(<8 x float> %a0, <8 x i32> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[TMP7]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[TMP10]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
 ; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[A0:%.*]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES]]
 ;
   %res = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1]
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/partial-poison.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/partial-poison.ll
new file mode 100644
index 000000000000..5164441c17e1
--- /dev/null
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/partial-poison.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -S -passes='msan' 2>&1 | FileCheck %s
+;
+; Test case to show that MSan computes shadows for partially poisoned vectors
+; as fully initialized, resulting in false negatives.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <2 x i64> @left_poison(ptr %add.ptr) sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @left_poison(
+; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> <i64 poison, i64 42>
+;
+  ret <2 x i64> <i64 poison, i64 42>
+}
+
+define <2 x i64> @right_poison(ptr %add.ptr) sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @right_poison(
+; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> <i64 42, i64 poison>
+;
+  ret <2 x i64> <i64 42, i64 poison>
+}
+
+define <2 x i64> @full_poison(ptr %add.ptr) sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @full_poison(
+; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store <2 x i64> splat (i64 -1), ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> poison
+;
+  ret <2 x i64> <i64 poison, i64 poison>
+}
+
+define <2 x i64> @no_poison_or_undef(ptr %add.ptr) sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @no_poison_or_undef(
+; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> splat (i64 42)
+;
+  ret <2 x i64> <i64 42, i64 42>
+}
+
+define <2 x i64> @left_undef(ptr %add.ptr) sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @left_undef(
+; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> <i64 undef, i64 42>
+;
+  ret <2 x i64> <i64 undef, i64 42>
+}
+
+define <2 x i64> @right_undef(ptr %add.ptr) sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @right_undef(
+; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> <i64 42, i64 undef>
+;
+  ret <2 x i64> <i64 42, i64 undef>
+}
+
+define <2 x i64> @full_undef(ptr %add.ptr) sanitize_memory {
+; CHECK-LABEL: define <2 x i64> @full_undef(
+; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store <2 x i64> splat (i64 -1), ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  ret <2 x i64> <i64 undef, i64 undef>
+}
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll
index 2e4bfb76cc9d..0149b4f05559 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll
@@ -20,7 +20,7 @@ define void @load.v1i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0:![0-9]+]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1:![0-9]+]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3:[0-9]+]]
 ; ADDR-NEXT:    unreachable
@@ -62,7 +62,7 @@ define void @load.v2i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -104,7 +104,7 @@ define void @load.v4i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -146,7 +146,7 @@ define void @load.v8i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -188,7 +188,7 @@ define void @load.v16i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -231,7 +231,7 @@ define void @store.v1i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -272,7 +272,7 @@ define void @store.v2i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -313,7 +313,7 @@ define void @store.v4i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -354,7 +354,7 @@ define void @store.v8i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -395,7 +395,7 @@ define void @store.v16i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -436,7 +436,7 @@ define void @load.nxv1i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -478,7 +478,7 @@ define void @load.nxv2i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -520,7 +520,7 @@ define void @load.nxv4i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -562,7 +562,7 @@ define void @load.nxv8i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -604,7 +604,7 @@ define void @load.nxv16i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -647,7 +647,7 @@ define void @store.nxv1i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -669,7 +669,7 @@ define void @store.nxv1i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    store <vscale x 1 x i32> zeroinitializer, ptr [[TMP3]], align 4
 ; ORIGINS-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> zeroinitializer)
 ; ORIGINS-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP6]], 0
-; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF0:![0-9]+]]
+; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1:![0-9]+]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
@@ -707,7 +707,7 @@ define void @store.nxv2i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -729,7 +729,7 @@ define void @store.nxv2i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    store <vscale x 2 x i32> zeroinitializer, ptr [[TMP3]], align 8
 ; ORIGINS-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> zeroinitializer)
 ; ORIGINS-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP6]], 0
-; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF0]]
+; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
@@ -767,7 +767,7 @@ define void @store.nxv4i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -789,7 +789,7 @@ define void @store.nxv4i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr [[TMP3]], align 16
 ; ORIGINS-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> zeroinitializer)
 ; ORIGINS-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP6]], 0
-; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF0]]
+; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
@@ -827,7 +827,7 @@ define void @store.nxv8i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -849,7 +849,7 @@ define void @store.nxv8i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    store <vscale x 8 x i32> zeroinitializer, ptr [[TMP3]], align 32
 ; ORIGINS-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.nxv8i32(<vscale x 8 x i32> zeroinitializer)
 ; ORIGINS-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP6]], 0
-; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF0]]
+; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 32
@@ -887,7 +887,7 @@ define void @store.nxv16i32(ptr %p) sanitize_memory {
 ; ADDR-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; ADDR-NEXT:    call void @llvm.donothing()
 ; ADDR-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; ADDR-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
 ; ADDR:       2:
 ; ADDR-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
 ; ADDR-NEXT:    unreachable
@@ -909,7 +909,7 @@ define void @store.nxv16i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    store <vscale x 16 x i32> zeroinitializer, ptr [[TMP3]], align 64
 ; ORIGINS-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.nxv16i32(<vscale x 16 x i32> zeroinitializer)
 ; ORIGINS-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP6]], 0
-; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF0]]
+; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 64
diff --git a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vscale.ll b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vscale.ll
index 36ca9e516f2d..61114af7ce90 100644
--- a/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vscale.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/MemorySanitizer/vscale.ll
@@ -40,7 +40,7 @@ define void @test_load_store_i32(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN-NEXT:    store <vscale x 4 x i32> [[_MSLD]], ptr [[TMP10]], align 16
 ; ORIGIN-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[_MSLD]])
 ; ORIGIN-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP13]], 0
-; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP21:%.*]], !prof [[PROF0:![0-9]+]]
+; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP21:%.*]], !prof [[PROF1:![0-9]+]]
 ; ORIGIN:       14:
 ; ORIGIN-NEXT:    [[TMP15:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP7]])
 ; ORIGIN-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
@@ -121,7 +121,7 @@ define void @test_load_store_add_int(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN-NEXT:    store <vscale x 8 x i64> [[_MSLD1]], ptr [[TMP21]], align 64
 ; ORIGIN-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vector.reduce.or.nxv8i64(<vscale x 8 x i64> [[_MSLD1]])
 ; ORIGIN-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP24]], 0
-; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP25:%.*]], label [[TMP32:%.*]], !prof [[PROF0]]
+; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP25:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
 ; ORIGIN:       25:
 ; ORIGIN-NEXT:    [[TMP26:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP14]])
 ; ORIGIN-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
@@ -184,7 +184,7 @@ define void @test_load_store_float(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN-NEXT:    store <vscale x 4 x i32> [[_MSLD]], ptr [[TMP10]], align 16
 ; ORIGIN-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[_MSLD]])
 ; ORIGIN-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP13]], 0
-; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP21:%.*]], !prof [[PROF0]]
+; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP21:%.*]], !prof [[PROF1]]
 ; ORIGIN:       14:
 ; ORIGIN-NEXT:    [[TMP15:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP7]])
 ; ORIGIN-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
@@ -265,7 +265,7 @@ define void @test_load_store_add_float(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN-NEXT:    store <vscale x 2 x i32> [[_MSLD1]], ptr [[TMP21]], align 8
 ; ORIGIN-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> [[_MSLD1]])
 ; ORIGIN-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP24]], 0
-; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP25:%.*]], label [[TMP32:%.*]], !prof [[PROF0]]
+; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP25:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
 ; ORIGIN:       25:
 ; ORIGIN-NEXT:    [[TMP26:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP14]])
 ; ORIGIN-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
@@ -359,7 +359,7 @@ define void @test_ret(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN-NEXT:    store <vscale x 2 x i32> [[_MSRET]], ptr [[TMP7]], align 8
 ; ORIGIN-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> [[_MSRET]])
 ; ORIGIN-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP10]], 0
-; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP11:%.*]], label [[TMP18:%.*]], !prof [[PROF0]]
+; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP11:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
 ; ORIGIN:       11:
 ; ORIGIN-NEXT:    [[TMP12:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP4]])
 ; ORIGIN-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
@@ -407,7 +407,7 @@ define void @fn_param(<vscale x 2 x float> %a, ptr %b) sanitize_memory {
 ; ORIGIN-NEXT:    store <vscale x 2 x i32> zeroinitializer, ptr [[TMP3]], align 8
 ; ORIGIN-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> zeroinitializer)
 ; ORIGIN-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP6]], 0
-; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP14:%.*]], !prof [[PROF0]]
+; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
 ; ORIGIN:       7:
 ; ORIGIN-NEXT:    [[TMP8:%.*]] = call i32 @__msan_chain_origin(i32 0)
 ; ORIGIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
@@ -445,7 +445,7 @@ define void @test_param(ptr %a, ptr %b) sanitize_memory {
 ; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> [[_MSLD]])
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5:[0-9]+]]
 ; CHECK-NEXT:    unreachable
@@ -470,7 +470,7 @@ define void @test_param(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN-NEXT:    store i32 [[TMP2]], ptr @__msan_param_origin_tls, align 4
 ; ORIGIN-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> [[_MSLD]])
 ; ORIGIN-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP10]], 0
-; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
+; ORIGIN-NEXT:    br i1 [[_MSCMP]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
 ; ORIGIN:       11:
 ; ORIGIN-NEXT:    call void @__msan_warning_with_origin_noreturn(i32 [[TMP9]]) #[[ATTR5:[0-9]+]]
 ; ORIGIN-NEXT:    unreachable
@@ -590,7 +590,7 @@ entry:
 }
 
 ;.
-; CHECK: [[PROF0]] = !{!"branch_weights", i32 1, i32 1048575}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
 ;.
-; ORIGIN: [[PROF0]] = !{!"branch_weights", i32 1, i32 1048575}
+; ORIGIN: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
 ;.
diff --git a/external/llvm-project/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll b/external/llvm-project/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll
index 40c4bef3bff9..8bcabaecf0fd 100644
--- a/external/llvm-project/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll
+++ b/external/llvm-project/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll
@@ -1,51 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=tsan -S | FileCheck %s
 ; Check that atomic memory operations on floating-point types are converted to calls into ThreadSanitizer runtime.
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 define float @load_float(ptr %fptr) {
+; CHECK-LABEL: define float @load_float(
+; CHECK-SAME: ptr [[FPTR:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.returnaddress(i32 0)
+; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @__tsan_atomic32_load(ptr [[FPTR]], i32 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
+; CHECK-NEXT:    [[V:%.*]] = load atomic float, ptr [[FPTR]] unordered, align 4
+; CHECK-NEXT:    call void @__tsan_func_exit()
+; CHECK-NEXT:    ret float [[TMP3]]
+;
   %v = load atomic float, ptr %fptr unordered, align 4
   ret float %v
-  ; CHECK-LABEL: load_float
-  ; CHECK: call i32 @__tsan_atomic32_load(ptr %{{.+}}, i32 0)
-  ; CHECK: bitcast i32 {{.+}} to float
 }
 
 define double @load_double(ptr %fptr) {
+; CHECK-LABEL: define double @load_double(
+; CHECK-SAME: ptr [[FPTR:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.returnaddress(i32 0)
+; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @__tsan_atomic64_load(ptr [[FPTR]], i32 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[TMP2]] to double
+; CHECK-NEXT:    [[V:%.*]] = load atomic double, ptr [[FPTR]] unordered, align 8
+; CHECK-NEXT:    call void @__tsan_func_exit()
+; CHECK-NEXT:    ret double [[TMP3]]
+;
   %v = load atomic double, ptr %fptr unordered, align 8
   ret double %v
-  ; CHECK-LABEL: load_double
-  ; CHECK: call i64 @__tsan_atomic64_load(ptr %{{.+}}, i32 0)
-  ; CHECK: bitcast i64 {{.+}} to double
 }
 
 define fp128 @load_fp128(ptr %fptr) {
+; CHECK-LABEL: define fp128 @load_fp128(
+; CHECK-SAME: ptr [[FPTR:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.returnaddress(i32 0)
+; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i128 @__tsan_atomic128_load(ptr [[FPTR]], i32 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i128 [[TMP2]] to fp128
+; CHECK-NEXT:    [[V:%.*]] = load atomic fp128, ptr [[FPTR]] unordered, align 16
+; CHECK-NEXT:    call void @__tsan_func_exit()
+; CHECK-NEXT:    ret fp128 [[TMP3]]
+;
   %v = load atomic fp128, ptr %fptr unordered, align 16
   ret fp128 %v
-  ; CHECK-LABEL: load_fp128
-  ; CHECK: call i128 @__tsan_atomic128_load(ptr %{{.+}}, i32 0)
-  ; CHECK: bitcast i128 {{.+}} to fp128
 }
 
 define void @store_float(ptr %fptr, float %v) {
+; CHECK-LABEL: define void @store_float(
+; CHECK-SAME: ptr [[FPTR:%.*]], float [[V:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.returnaddress(i32 0)
+; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[V]] to i32
+; CHECK-NEXT:    call void @__tsan_atomic32_store(ptr [[FPTR]], i32 [[TMP2]], i32 0)
+; CHECK-NEXT:    call void @__tsan_func_exit()
+; CHECK-NEXT:    ret void
+;
   store atomic float %v, ptr %fptr unordered, align 4
   ret void
-  ; CHECK-LABEL: store_float
-  ; CHECK: bitcast float %v to i32
-  ; CHECK: call void @__tsan_atomic32_store(ptr %{{.+}}, i32 %{{.+}}, i32 0)
 }
 
 define void @store_double(ptr %fptr, double %v) {
+; CHECK-LABEL: define void @store_double(
+; CHECK-SAME: ptr [[FPTR:%.*]], double [[V:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.returnaddress(i32 0)
+; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double [[V]] to i64
+; CHECK-NEXT:    call void @__tsan_atomic64_store(ptr [[FPTR]], i64 [[TMP2]], i32 0)
+; CHECK-NEXT:    call void @__tsan_func_exit()
+; CHECK-NEXT:    ret void
+;
   store atomic double %v, ptr %fptr unordered, align 8
   ret void
-  ; CHECK-LABEL: store_double
-  ; CHECK: bitcast double %v to i64
-  ; CHECK: call void @__tsan_atomic64_store(ptr %{{.+}}, i64 %{{.+}}, i32 0)
 }
 
 define void @store_fp128(ptr %fptr, fp128 %v) {
+; CHECK-LABEL: define void @store_fp128(
+; CHECK-SAME: ptr [[FPTR:%.*]], fp128 [[V:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.returnaddress(i32 0)
+; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast fp128 [[V]] to i128
+; CHECK-NEXT:    call void @__tsan_atomic128_store(ptr [[FPTR]], i128 [[TMP2]], i32 0)
+; CHECK-NEXT:    call void @__tsan_func_exit()
+; CHECK-NEXT:    ret void
+;
   store atomic fp128 %v, ptr %fptr unordered, align 16
   ret void
-  ; CHECK-LABEL: store_fp128
-  ; CHECK: bitcast fp128 %v to i128
-  ; CHECK: call void @__tsan_atomic128_store(ptr %{{.+}}, i128 %{{.+}}, i32 0)
 }
diff --git a/external/llvm-project/llvm/test/Linker/opaque.ll b/external/llvm-project/llvm/test/Linker/opaque.ll
index 7d2c49af7797..2b66e3d8c7dd 100644
--- a/external/llvm-project/llvm/test/Linker/opaque.ll
+++ b/external/llvm-project/llvm/test/Linker/opaque.ll
@@ -1,6 +1,7 @@
 ; RUN: llvm-link %p/opaque.ll %p/Inputs/opaque.ll -S -o - | FileCheck %s
 
-; CHECK-DAG: %A =   type {}
+; CHECK-DAG: %A =   type opaque
+; CHECK-DAG: %A.0 = type {}
 ; CHECK-DAG: %B =   type { %C, %C, ptr }
 ; CHECK-DAG: %B.1 = type { %D, %E, ptr }
 ; CHECK-DAG: %C =   type { %A }
@@ -8,10 +9,10 @@
 ; CHECK-DAG: %E =   type opaque
 
 ; CHECK-DAG: @g1 = external global %B
-; CHECK-DAG: @g2 = external global %A
+; CHECK-DAG: @g2 = external global %A.0
 ; CHECK-DAG: @g3 = external global %B.1
 
-; CHECK-DAG: getelementptr %A, ptr null, i32 0
+; CHECK-DAG: getelementptr %A.0, ptr null, i32 0
 
 %A = type opaque
 %B = type { %C, %C, ptr }
diff --git a/external/llvm-project/llvm/test/Linker/pr22807.ll b/external/llvm-project/llvm/test/Linker/pr22807.ll
index 24bcf1732112..69944b84cd50 100644
--- a/external/llvm-project/llvm/test/Linker/pr22807.ll
+++ b/external/llvm-project/llvm/test/Linker/pr22807.ll
@@ -1,6 +1,8 @@
-; RUN: not llvm-link -S -o - %p/pr22807.ll %p/Inputs/pr22807.ll 2>&1 | FileCheck %s
+; RUN: llvm-link -S -o - %p/pr22807.ll %p/Inputs/pr22807.ll 2>&1 | FileCheck %s
 
-; CHECK: error: identified structure type 'struct.A' is recursive
+; CHECK: %struct.B = type { %struct.A }
+; CHECK: %struct.A = type opaque
+; CHECK: @g = external global %struct.B
 
 %struct.B = type { %struct.A }
 %struct.A = type opaque
diff --git a/external/llvm-project/llvm/test/Linker/type-unique-dst-types.ll b/external/llvm-project/llvm/test/Linker/type-unique-dst-types.ll
index c5fcbd3f51fd..54f0ef40333b 100644
--- a/external/llvm-project/llvm/test/Linker/type-unique-dst-types.ll
+++ b/external/llvm-project/llvm/test/Linker/type-unique-dst-types.ll
@@ -2,21 +2,17 @@
 ; RUN:           %p/Inputs/type-unique-dst-types2.ll \
 ; RUN:           %p/Inputs/type-unique-dst-types3.ll -S -o %t1.ll
 ; RUN: cat %t1.ll | FileCheck %s
-; RUN: cat %t1.ll | FileCheck --check-prefix=RENAMED %s
 
-; This tests the importance of keeping track of which types are part of the
-; destination module.
-; When the second input is merged in, the context gets an unused A.11. When
-; the third module is then merged, we should pretend it doesn't exist.
+; The types of @g1 and @g3 can be deduplicated, but @g2 should retain its
+; opaque type, even if it has the same name as a type from a different module.
 
 ; CHECK: %A = type { %B }
 ; CHECK-NEXT: %B = type { i8 }
+; CHECK-NEXT: %A.11 = type opaque
 
 ; CHECK: @g3 = external global %A
 ; CHECK: @g1 = external global %A
-; CHECK: @g2 = external global %A
-
-; RENAMED-NOT: A.11
+; CHECK: @g2 = external global %A.11
 
 %A = type { %B }
 %B = type { i8 }
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s b/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s
index 06bc5ec23397..1e399812b927 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch-negative.s
@@ -21,5 +21,5 @@ bfadd za.h[w8, 3], {z20.h-z21.h}
 .arch armv9-a+sve-aes2+ssve-aes
 .arch armv9-a+nossve-aes
 aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
-// CHECK: error: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
\ No newline at end of file
+// CHECK: error: instruction requires: sve-aes2
+// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s b/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
index 6e185a7ff8cc..659c6b09a58c 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
@@ -23,5 +23,5 @@ bfadd za.h[w8, 3], {z20.h-z21.h}
 .arch_extension ssve-aes
 .arch_extension nossve-aes
 aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
-// CHECK: error: instruction requires: sve2p1 or ssve-aes
-// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
\ No newline at end of file
+// CHECK: error: instruction requires: sve2 or ssve-aes
+// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesd.s b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesd.s
index e2731d221a84..479335dc3616 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesd.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesd.s
@@ -18,36 +18,36 @@
 aesd    {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100010-11101100-00000000
 // CHECK-INST: aesd    { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x22,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4522ec00 <unknown>
 
 aesd    {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110010-11101101-01010100
 // CHECK-INST: aesd    { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xed,0x32,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4532ed54 <unknown>
 
 aesd    {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111010-11101111-11111110
 // CHECK-INST: aesd    { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xef,0x3a,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453aeffe <unknown>
 
 // x4
 aesd    {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100110-11101100-00000000
 // CHECK-INST: aesd    { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x26,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4526ec00 <unknown>
 
 aesd    {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101110-11101101-10110100
 // CHECK-INST: aesd    { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb4,0xed,0x2e,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452eedb4 <unknown>
 
 aesd    {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111110-11101111-11111100
 // CHECK-INST: aesd    { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xef,0x3e,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK-UNKNOWN: 453eeffc <unknown>
\ No newline at end of file
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453eeffc <unknown>
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesdimc.s b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
index 6aa0e0d3d2ba..d1b0b130f9ec 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
@@ -19,36 +19,36 @@
 aesdimc {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100011-11101100-00000000
 // CHECK-INST: aesdimc { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x23,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4523ec00 <unknown>
 
 aesdimc {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110011-11101101-01010100
 // CHECK-INST: aesdimc { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xed,0x33,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4533ed54 <unknown>
 
 aesdimc {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111011-11101111-11111110
 // CHECK-INST: aesdimc { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xef,0x3b,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453beffe <unknown>
 
 // x4
 aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100111-11101100-00000000
 // CHECK-INST: aesdimc { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x27,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4527ec00 <unknown>
 
 aesdimc {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101111-11101101-10110100
 // CHECK-INST: aesdimc { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb4,0xed,0x2f,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452fedb4 <unknown>
 
 aesdimc {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111111-11101111-11111100
 // CHECK-INST: aesdimc { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xef,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK-UNKNOWN: 453feffc <unknown>
\ No newline at end of file
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453feffc <unknown>
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aese.s b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aese.s
index a5272e62e245..1f8ef5e85a15 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aese.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aese.s
@@ -18,36 +18,36 @@
 aese    {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100010-11101000-00000000
 // CHECK-INST: aese    { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x22,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4522e800 <unknown>
 
 aese    {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110010-11101001-01010100
 // CHECK-INST: aese    { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xe9,0x32,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4532e954 <unknown>
 
 aese    {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111010-11101011-11111110
 // CHECK-INST: aese    { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xeb,0x3a,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453aebfe <unknown>
 
 // x4
 aese    {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100110-11101000-00000000
 // CHECK-INST: aese    { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x26,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4526e800 <unknown>
 
 aese    {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101110-11101001-10110100
 // CHECK-INST: aese    { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb4,0xe9,0x2e,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452ee9b4 <unknown>
 
 aese    {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111110-11101011-11111100
 // CHECK-INST: aese    { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xeb,0x3e,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK-UNKNOWN: 453eebfc <unknown>
\ No newline at end of file
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453eebfc <unknown>
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesemc.s b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesemc.s
index 888c82811562..5682ce87737a 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesemc.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/aesemc.s
@@ -18,36 +18,36 @@
 aesemc  {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100011-11101000-00000000
 // CHECK-INST: aesemc  { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x23,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4523e800 <unknown>
 
 aesemc  {z22.b-z23.b}, {z22.b-z23.b}, z13.q[1]  // 01000101-00101011-11101001-10110110
 // CHECK-INST: aesemc  { z22.b, z23.b }, { z22.b, z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb6,0xe9,0x2b,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452be9b6 <unknown>
 
 aesemc  {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111011-11101011-11111110
 // CHECK-INST: aesemc  { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xeb,0x3b,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453bebfe <unknown>
 
 // x4
 aesemc  {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100111-11101000-00000000
 // CHECK-INST: aesemc  { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x27,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4527e800 <unknown>
 
 aesemc  {z20.b-z23.b}, {z20.b-z23.b}, z10.q[2]  // 01000101-00110111-11101001-01010100
 // CHECK-INST: aesemc  { z20.b - z23.b }, { z20.b - z23.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xe9,0x37,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4537e954 <unknown>
 
 aesemc  {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111111-11101011-11111100
 // CHECK-INST: aesemc  { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xeb,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK-UNKNOWN: 453febfc <unknown>
\ No newline at end of file
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453febfc <unknown>
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s
index e82c92df451d..71ce537a26cb 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/directive-arch-negative.s
@@ -15,5 +15,5 @@ bfadd   z23.h, p3/m, z23.h, z13.h
 .arch armv9-a+sve2p1+sve-aes2
 .arch armv9-a+nosve-aes2
 aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
-// CHECK: error: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
\ No newline at end of file
+// CHECK: error: instruction requires: sve-aes2
+// CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmlal.s b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmlal.s
index 0420b230956c..6a3751e9ac25 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmlal.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmlal.s
@@ -17,17 +17,17 @@
 pmlal   {z0.q-z1.q}, z0.d, z0.d  // 01000101-00100000-11111100-00000000
 // CHECK-INST: pmlal   { z0.q, z1.q }, z0.d, z0.d
 // CHECK-ENCODING: [0x00,0xfc,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4520fc00 <unknown>
 
 pmlal   {z22.q-z23.q}, z13.d, z8.d  // 01000101-00101000-11111101-10110110
 // CHECK-INST: pmlal   { z22.q, z23.q }, z13.d, z8.d
 // CHECK-ENCODING: [0xb6,0xfd,0x28,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4528fdb6 <unknown>
 
 pmlal   {z30.q-z31.q}, z31.d, z31.d  // 01000101-00111111-11111111-11111110
 // CHECK-INST: pmlal   { z30.q, z31.q }, z31.d, z31.d
 // CHECK-ENCODING: [0xfe,0xff,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK-UNKNOWN: 453ffffe <unknown>
\ No newline at end of file
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453ffffe <unknown>
diff --git a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmull.s b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmull.s
index 9c3ee16401c1..3686181c35b7 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmull.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/SVE2p1/pmull.s
@@ -17,17 +17,17 @@
 pmull   {z0.q-z1.q}, z0.d, z0.d  // 01000101-00100000-11111000-00000000
 // CHECK-INST: pmull   { z0.q, z1.q }, z0.d, z0.d
 // CHECK-ENCODING: [0x00,0xf8,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4520f800 <unknown>
 
 pmull   {z22.q-z23.q}, z13.d, z8.d  // 01000101-00101000-11111001-10110110
 // CHECK-INST: pmull   { z22.q, z23.q }, z13.d, z8.d
 // CHECK-ENCODING: [0xb6,0xf9,0x28,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4528f9b6 <unknown>
 
 pmull   {z30.q-z31.q}, z31.d, z31.d  // 01000101-00111111-11111011-11111110
 // CHECK-INST: pmull   { z30.q, z31.q }, z31.d, z31.d
 // CHECK-ENCODING: [0xfe,0xfb,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2p1 or ssve-aes sve-aes2
-// CHECK-UNKNOWN: 453ffbfe <unknown>
\ No newline at end of file
+// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-UNKNOWN: 453ffbfe <unknown>
diff --git a/external/llvm-project/llvm/test/MC/AArch64/basic-a64-instructions.s b/external/llvm-project/llvm/test/MC/AArch64/basic-a64-instructions.s
index 14ac11f581a5..b2ec5b6ac367 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/basic-a64-instructions.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/basic-a64-instructions.s
@@ -3349,7 +3349,7 @@ _func:
 
 	.equ equvalue, 0x0001
         movk x1, equvalue, lsl 16
-// CHECK: .set equvalue, 1
+// CHECK: equvalue = 1
 // CHECK-NEXT: movk x1, #1, lsl #16 // encoding: [0x21,0x00,0xa0,0xf2]
 
         movz x2, #:abs_g0:sym
diff --git a/external/llvm-project/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s b/external/llvm-project/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s
index 2ef67fafb2ea..ff53d01e5fed 100644
--- a/external/llvm-project/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s
+++ b/external/llvm-project/llvm/test/MC/AArch64/label-arithmetic-diags-elf.s
@@ -1,5 +1,8 @@
-// RUN: not llvm-mc -triple aarch64-elf -filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: not llvm-mc -triple aarch64-elf -filetype=obj a.s -o /dev/null 2>&1 | FileCheck a.s
+// RUN: not llvm-mc -triple aarch64-elf -filetype=obj b.s -o /dev/null 2>&1 | FileCheck b.s
 
+//--- a.s
   .data
 b:
   .fill 300
@@ -9,6 +12,7 @@ e:
   // CHECK-NEXT: .byte e - b
   // CHECK-NEXT:       ^
 
+//--- b.s
   .section sec_x
 start:
   .space 5000
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
index 93e01954bea5..f1438532d7c5 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
@@ -3808,3 +3808,9 @@ v_trunc_f64 v[5:6], src_scc
 
 v_trunc_f64 v[254:255], 0xaf123456
 // GFX11: v_trunc_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+
+v_trunc_f16 v[5].l, v[1].h
+// GFX11: v_trunc_f16_e32 v5.l, v1.h              ; encoding: [0x81,0xbb,0x0a,0x7e]
+
+v_trunc_f16 v[5:5].l, v[1:1].h
+// GFX11: v_trunc_f16_e32 v5.l, v1.h              ; encoding: [0x81,0xbb,0x0a,0x7e]
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s
index 55a25ad3ec81..d19220867f29 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s
@@ -1231,3 +1231,9 @@ v_trunc_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
 
 v_trunc_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
 // GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
+
+v_ceil_f16_e32 v[5:5].s, 0xfe0b
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+
+v_ceil_f16_e32 v[6:7].l, 0xfe0b
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
index f89833f9da65..8121dd1a8b89 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
@@ -5390,3 +5390,15 @@ v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 row_xmask:0 row_mask:0x1 bank_ma
 
 v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX11: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff]
+// GFX11: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
+// GFX11: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
+// GFX11: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0x00]
+// GFX11: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0x00]
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
index 80ee725db319..dd1d9299470b 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
@@ -3630,3 +3630,9 @@ v_dot2_bf16_bf16_e64_dpp v5.l, -|v1|, -|v2|, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
 
 v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX11: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x67,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
+// GFX11: v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x1a,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
+// GFX11: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x1a,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
index afd4af92b4f2..08b6b26d1f26 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
@@ -1169,23 +1169,35 @@ v_cubetc_f32 v5, -src_scc, |vcc_lo|, -1 mul:4
 v_cubetc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2
 // GFX12: v_cubetc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
-v_cvt_pk_fp8_f32 v1, v2, v3
-// GFX12: v_cvt_pk_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
+v_cvt_pk_fp8_f32 v1.l, v2, v3
+// GFX12: v_cvt_pk_fp8_f32 v1.l, v2, v3           ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
 
-v_cvt_pk_fp8_f32 v1, -v2, |v3|
-// GFX12: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+v_cvt_pk_fp8_f32 v1.l, -v2, |v3|
+// GFX12: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 
-v_cvt_pk_fp8_f32 v1, s2, 3
-// GFX12: v_cvt_pk_fp8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
+v_cvt_pk_fp8_f32 v1.l, s2, 3
+// GFX12: v_cvt_pk_fp8_f32 v1.l, s2, 3            ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
 
-v_cvt_pk_bf8_f32 v1, v2, v3
-// GFX12: v_cvt_pk_bf8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
+v_cvt_pk_fp8_f32 v1.h v2, v3
+// GFX12: v_cvt_pk_fp8_f32 v1.h, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x69,0xd7,0x02,0x07,0x02,0x00]
 
-v_cvt_pk_bf8_f32 v1, -v2, |v3|
-// GFX12: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+v_cvt_pk_fp8_f32 v255.h v2, v3
+// GFX12: v_cvt_pk_fp8_f32 v255.h, v2, v3 op_sel:[0,0,1] ; encoding: [0xff,0x40,0x69,0xd7,0x02,0x07,0x02,0x00]
 
-v_cvt_pk_bf8_f32 v1, s2, 3
-// GFX12: v_cvt_pk_bf8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
+v_cvt_pk_bf8_f32 v1.l, v2, v3
+// GFX12: v_cvt_pk_bf8_f32 v1.l, v2, v3           ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
+
+v_cvt_pk_bf8_f32 v1.l, -v2, |v3|
+// GFX12: v_cvt_pk_bf8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+
+v_cvt_pk_bf8_f32 v1.l, s2, 3
+// GFX12: v_cvt_pk_bf8_f32 v1.l, s2, 3            ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
+
+v_cvt_pk_bf8_f32 v1.h, v2, v3
+// GFX12: v_cvt_pk_bf8_f32 v1.h, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x6a,0xd7,0x02,0x07,0x02,0x00]
+
+v_cvt_pk_bf8_f32 v255.h, -v2, |v3|
+// GFX12: v_cvt_pk_bf8_f32 v255.h, -v2, |v3| op_sel:[0,0,1] ; encoding: [0xff,0x42,0x6a,0xd7,0x02,0x07,0x02,0x20]
 
 v_cvt_sr_fp8_f32 v1, v2, v3
 // GFX12: v_cvt_sr_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6b,0xd7,0x02,0x07,0x02,0x00]
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s b/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
index cfd01ee02aa7..cf497c058b7e 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
@@ -1209,59 +1209,65 @@ v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_m
 v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,1,2,3]
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,1,2,3]
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 
-v_cvt_pk_bf8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+v_cvt_pk_bf8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
-v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,1,2,3]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
+v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0xff,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,1,2,3]
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
-v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+
+v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+
+v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| quad_perm:[0,1,2,3]
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 
 v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
 // GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
@@ -5639,6 +5645,18 @@ v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h row_mirror
 v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
 // GFX12: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff]
+// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
+// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
+// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0x00]
+// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0x00]
+
 v_minimum_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
 
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s b/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
index b6d395c8ede8..f43a1128f521 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
@@ -760,29 +760,35 @@ v_cubetc_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x0e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
-v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
+v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1]
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 
-v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
-v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cvt_pk_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
-v_cvt_pk_bf8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+v_cvt_pk_bf8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_pk_bf8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+v_cvt_pk_bf8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
-v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
-// GFX12: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+v_cvt_pk_bf8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_cvt_pk_bf8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_cvt_pk_bf8_f32_e64_dpp v255.h, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
+// GFX12: v_cvt_pk_bf8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cvt_sr_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -3896,6 +3902,12 @@ v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
 v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX12: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x67,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
+v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
+// GFX12: v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
+
+v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x1a,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
+// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x1a,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
+
 v_minimum_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_minimum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
diff --git a/external/llvm-project/llvm/test/MC/AMDGPU/mai-gfx950.s b/external/llvm-project/llvm/test/MC/AMDGPU/mai-gfx950.s
index cf8ca70c07a3..2d3a56703674 100644
--- a/external/llvm-project/llvm/test/MC/AMDGPU/mai-gfx950.s
+++ b/external/llvm-project/llvm/test/MC/AMDGPU/mai-gfx950.s
@@ -441,6 +441,22 @@ v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], s20, 9
 // ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], 33, 9
 
+// GFX950: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], 4.0, v2 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf6,0x04,0x02,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], 4.0, v2
+
+// GFX950: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], v2, 4.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x02,0xed,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], v2, 4.0
+
+// GFX950: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], -4.0, 1.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf7,0xe4,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], -4.0, 1.0
+
+// GFX950: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], 0.15915494, -16 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf8,0xa0,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], 0.15915494, -16
+
 // GFX950: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[12:19], v[20:23], v24, v25 op_sel_hi:[0,0,0] cbsz:3 blgp:1 ; encoding: [0x00,0x00,0xac,0xd3,0x18,0x33,0x02,0x00,0x00,0x0b,0xad,0xd3,0x04,0x19,0x52,0x24]
 // ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[12:19], v[20:23], v24, v25 cbsz:3 blgp:1
@@ -569,6 +585,22 @@ v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:29], v[32:47], v48, v49
 // ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:21], v[24:29], v[32:47], v48, v49 op_sel:[0,1,0] op_sel_hi:[0,1,0] cbsz:2 blgp:3
 
+// GFX950: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 16, v49 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x90,0x62,0x02,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 16, v49
+
+// GFX950: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], v48, -4.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x30,0xef,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], v48, -4.0
+
+// GFX950: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 4.0, 1.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf6,0xe4,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 4.0, 1.0
+
+// GFX950: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 0.15915494, -16 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf8,0xa0,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+// ERR: :[[@LINE+1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 0.15915494, -16
+
 // op_sel combinations
 
 // GFX950: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], v48, v49 op_sel:[0,1,0] op_sel_hi:[0,0,0] ; encoding: [0x00,0x10,0xac,0xd3,0x30,0x63,0x02,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
diff --git a/external/llvm-project/llvm/test/MC/AsmParser/assignment.s b/external/llvm-project/llvm/test/MC/AsmParser/assignment.s
index 6f84a1c338da..8c8984c12ac3 100644
--- a/external/llvm-project/llvm/test/MC/AsmParser/assignment.s
+++ b/external/llvm-project/llvm/test/MC/AsmParser/assignment.s
@@ -1,22 +1,22 @@
 # RUN: llvm-mc -triple i386-unknown-unknown %s | FileCheck %s
 
 # CHECK: TEST0:
-# CHECK: .set a, 0
+# CHECK: a = 0
 TEST0:
         a = 0
 
 # CHECK: TEST1:
-# CHECK: .set b, 0
+# CHECK: b = 0
 TEST1:
-        .set b, 0
+        b = 0
 
 # CHECK: .globl	_f1
-# CHECK: .set _f1, 0
+# CHECK: _f1 = 0
         .globl _f1
         _f1 = 0
 
 # CHECK: .globl	_f2
-# CHECK: .set _f2, 0
+# CHECK: _f2 = 0
         .globl _f2
-        .set _f2, 0
+        _f2 = 0
 
diff --git a/external/llvm-project/llvm/test/MC/AsmParser/directive_include.s b/external/llvm-project/llvm/test/MC/AsmParser/directive_include.s
index 8d2ef2753b23..f53bc671fc64 100644
--- a/external/llvm-project/llvm/test/MC/AsmParser/directive_include.s
+++ b/external/llvm-project/llvm/test/MC/AsmParser/directive_include.s
@@ -2,7 +2,7 @@
 
 # CHECK: TESTA:
 # CHECK: TEST0:
-# CHECK: .set a, 0
+# CHECK: a = 0
 # CHECK: TESTB:
 TESTA:  
 	.include       "directive\137set.s"   # "\137" is underscore "_"
diff --git a/external/llvm-project/llvm/test/MC/AsmParser/directive_set.s b/external/llvm-project/llvm/test/MC/AsmParser/directive_set.s
index 65dd33d1d54f..4b93de01b309 100644
--- a/external/llvm-project/llvm/test/MC/AsmParser/directive_set.s
+++ b/external/llvm-project/llvm/test/MC/AsmParser/directive_set.s
@@ -1,13 +1,13 @@
 # RUN: llvm-mc -triple i386-unknown-elf %s | FileCheck %s
 
 # CHECK: TEST0:
-# CHECK: .set a, 0
+# CHECK: a = 0
 # CHECK-NOT: .no_dead_strip a
 TEST0:  
-        .set a, 0
+        a = 0
         
 # CHECK: TEST1:
-# CHECK: .set a, 0
+# CHECK: a = 0
 # CHECK-NOT: .no_dead_strip a
 TEST1:  
         .equ a, 0
diff --git a/external/llvm-project/llvm/test/MC/AsmParser/include.ll b/external/llvm-project/llvm/test/MC/AsmParser/include.ll
index 3321f0a6a287..625fdd9e1e59 100644
--- a/external/llvm-project/llvm/test/MC/AsmParser/include.ll
+++ b/external/llvm-project/llvm/test/MC/AsmParser/include.ll
@@ -1,6 +1,4 @@
-; RUN: llc -I %p/Inputs -filetype asm -o - %s | FileCheck %s
-; UNSUPPORTED: target={{.*}}-zos{{.*}},target=nvptx{{.*}}
-; REQUIRES: default_triple
+; RUN: llc -mtriple=x86_64 -I %p/Inputs -filetype asm -o - %s | FileCheck %s
 
 module asm ".include \22module.x\22"
 
@@ -10,5 +8,5 @@ entry:
   ret void
 }
 
-; CHECK: .set MODULE, 1
-; CHECK: .set FUNCTION, 1
+; CHECK: MODULE = 1
+; CHECK: FUNCTION = 1
diff --git a/external/llvm-project/llvm/test/MC/AsmParser/labels.s b/external/llvm-project/llvm/test/MC/AsmParser/labels.s
index 599ce72c44ee..6a9870b655f2 100644
--- a/external/llvm-project/llvm/test/MC/AsmParser/labels.s
+++ b/external/llvm-project/llvm/test/MC/AsmParser/labels.s
@@ -18,12 +18,12 @@ foo:
 // CHECK: addl $24, a$b+10(%eax)
         addl $24, ("a$b" + 10)(%eax)
 
-// CHECK: .set b$c, 10
+// CHECK: b$c = 10
 "b$c" = 10
 // CHECK: addl $10, %eax
         addl $"b$c", %eax
 
-// CHECK: .set "a 0", 11
+// CHECK: "a 0" = 11
         .set "a 0", 11
 
 // CHECK: .long 11
@@ -49,7 +49,7 @@ foo:
 // CHECX: .lsym "a 8",1
 //        .lsym "a 8", 1
 
-// CHECK: .set "a 9", a-b
+// CHECK: "a 9" = a-b
         .set "a 9", a - b
 
 // CHECK: .long "a 9"
diff --git a/external/llvm-project/llvm/test/MC/AsmParser/macro-arg-darwin.s b/external/llvm-project/llvm/test/MC/AsmParser/macro-arg-darwin.s
index 8671107539ce..88c63dd488be 100644
--- a/external/llvm-project/llvm/test/MC/AsmParser/macro-arg-darwin.s
+++ b/external/llvm-project/llvm/test/MC/AsmParser/macro-arg-darwin.s
@@ -38,7 +38,7 @@ bar
     .endif
 .endm
 .macro bottom
-    .set fred, $0
+    fred = $0
 .endm
 
 .text
@@ -49,7 +49,7 @@ top bar, 42
 // CHECK: _foo:
 // CHECK-NOT: fred
 // CHECK: _bar
-// CHECK-NEXT: .set fred, 42
+// CHECK-NEXT: fred = 42
 
 
 .macro foo
diff --git a/external/llvm-project/llvm/test/MC/AsmParser/motorola_integers.s b/external/llvm-project/llvm/test/MC/AsmParser/motorola_integers.s
index c75d9a5e0cb1..1ec2e02e97f0 100644
--- a/external/llvm-project/llvm/test/MC/AsmParser/motorola_integers.s
+++ b/external/llvm-project/llvm/test/MC/AsmParser/motorola_integers.s
@@ -1,10 +1,10 @@
 # RUN: llvm-mc -triple i386-unknown-unknown -motorola-integers %s | FileCheck %s
 
-# CHECK: .set a, 2882400009
-.set a, $aBcDeF09
-# CHECK: .set b, 256
-.set b, $0100
-# CHECK: .set c, 10
-.set c, %01010
-# CHECK: .set d, 1
-.set d, %1
+# CHECK: a = 2882400009
+a = $aBcDeF09
+# CHECK: b = 256
+b = $0100
+# CHECK: c = 10
+c = %01010
+# CHECK: d = 1
+d = %1
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AArch64/armv9.6a-rme-gpc3.txt b/external/llvm-project/llvm/test/MC/Disassembler/AArch64/armv9.6a-rme-gpc3.txt
index d198771c341b..75129ac48566 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AArch64/armv9.6a-rme-gpc3.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AArch64/armv9.6a-rme-gpc3.txt
@@ -9,10 +9,10 @@
 [0xa3,0x21,0x3e,0xd5]
 [0xa4,0x21,0x1e,0xd5]
 
-# CHECK:      	sys	#6, c7, c0, #0, x0
-# CHECK-NEXT: 	sys	#6, c7, c0, #0, x1
-# CHECK-NEXT: 	sys	#6, c7, c0, #0, x2
-# CHECK-NEXT: 	sys	#6, c7, c0, #0, x17
-# CHECK-NEXT: 	sys	#6, c7, c0, #0, x30
+# CHECK:      	apas x0
+# CHECK-NEXT: 	apas x1
+# CHECK-NEXT: 	apas x2
+# CHECK-NEXT: 	apas x17
+# CHECK-NEXT: 	apas x30
 # CHECK-NEXT: 	mrs	x3, GPCBW_EL3
 # CHECK-NEXT: 	msr	GPCBW_EL3, x4
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp16.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp16.txt
index 524623ea8be3..7bf75e7b190d 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp16.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp16.txt
@@ -4,6 +4,9 @@
 # GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x13,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
 0x00,0x45,0x13,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe
 
+# GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x13,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
+0x00,0x45,0x13,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe
+
 # GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf bound_ctrl:1 fi:1 ; encoding: [0x00,0x40,0x13,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x0c,0xff]
 0x00,0x40,0x13,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x0c,0xff
 
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp8.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp8.txt
index 7f26c47b1e88..d5c50fb1925a 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp8.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3p_dpp8.txt
@@ -4,6 +4,9 @@
 # GFX11: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x13,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
 0x00,0x45,0x13,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05
 
+# GFX11: v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
+0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05
+
 # GFX11: v_fma_mix_f32_e64_dpp v0, v1, v2, v3 clamp dpp8:[2,2,2,2,4,4,4,4] fi:1 ; encoding: [0x00,0x80,0x20,0xcc,0xea,0x04,0x0e,0x04,0x01,0x92,0x44,0x92]
 0x00,0x80,0x20,0xcc,0xea,0x04,0x0e,0x04,0x01,0x92,0x44,0x92
 
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
index e7d92af7511f..5980a6820a34 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
@@ -1318,22 +1318,52 @@
 # GFX12: v_cubetc_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x0e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
 
 0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00
-# GFX12: v_cvt_pk_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
+# W32-REAL16: v_cvt_pk_fp8_f32 v1.l, v2, v3           ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
+# W32-FAKE16: v_cvt_pk_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
+# W64-REAL16: v_cvt_pk_fp8_f32 v1.l, v2, v3           ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
+# W64-FAKE16: v_cvt_pk_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x07,0x02,0x00]
 
 0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20
-# GFX12: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W32-REAL16: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W32-FAKE16: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W64-REAL16: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W64-FAKE16: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 
 0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00
-# GFX12: v_cvt_pk_fp8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
+# W32-REAL16: v_cvt_pk_fp8_f32 v1.l, s2, 3            ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
+# W32-FAKE16: v_cvt_pk_fp8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
+# W64-REAL16: v_cvt_pk_fp8_f32 v1.l, s2, 3            ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
+# W64-FAKE16: v_cvt_pk_fp8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x69,0xd7,0x02,0x06,0x01,0x00]
+
+0x01,0x40,0x69,0xd7,0x02,0x07,0x02,0x00
+# W32-REAL16: v_cvt_pk_fp8_f32 v1.h, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x69,0xd7,0x02,0x07,0x02,0x00]
+# W32-FAKE16: v_cvt_pk_fp8_f32 v1, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x69,0xd7,0x02,0x07,0x02,0x00]
+# W64-REAL16: v_cvt_pk_fp8_f32 v1.h, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x69,0xd7,0x02,0x07,0x02,0x00]
+# W64-FAKE16: v_cvt_pk_fp8_f32 v1, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x69,0xd7,0x02,0x07,0x02,0x00]
 
 0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00
-# GFX12: v_cvt_pk_bf8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
+# W32-REAL16: v_cvt_pk_bf8_f32 v1.l, v2, v3           ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
+# W32-FAKE16: v_cvt_pk_bf8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
+# W64-REAL16: v_cvt_pk_bf8_f32 v1.l, v2, v3           ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
+# W64-FAKE16: v_cvt_pk_bf8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x07,0x02,0x00]
 
 0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20
-# GFX12: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W32-REAL16: v_cvt_pk_bf8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W32-FAKE16: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W64-REAL16: v_cvt_pk_bf8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W64-FAKE16: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
 
 0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00
-# GFX12: v_cvt_pk_bf8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
+# W32-REAL16: v_cvt_pk_bf8_f32 v1.l, s2, 3            ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
+# W32-FAKE16: v_cvt_pk_bf8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
+# W64-REAL16: v_cvt_pk_bf8_f32 v1.l, s2, 3            ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
+# W64-FAKE16: v_cvt_pk_bf8_f32 v1, s2, 3              ; encoding: [0x01,0x00,0x6a,0xd7,0x02,0x06,0x01,0x00]
+
+0x01,0x40,0x6a,0xd7,0x02,0x07,0x02,0x00
+# W32-REAL16: v_cvt_pk_bf8_f32 v1.h, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x6a,0xd7,0x02,0x07,0x02,0x00]
+# W32-FAKE16: v_cvt_pk_bf8_f32 v1, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x6a,0xd7,0x02,0x07,0x02,0x00]
+# W64-REAL16: v_cvt_pk_bf8_f32 v1.h, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x6a,0xd7,0x02,0x07,0x02,0x00]
+# W64-FAKE16: v_cvt_pk_bf8_f32 v1, v2, v3 op_sel:[0,0,1] ; encoding: [0x01,0x40,0x6a,0xd7,0x02,0x07,0x02,0x00]
 
 0x01,0x00,0x6b,0xd7,0x02,0x07,0x02,0x00
 # GFX12: v_cvt_sr_fp8_f32 v1, v2, v3             ; encoding: [0x01,0x00,0x6b,0xd7,0x02,0x07,0x02,0x00]
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
index a84e0e1734ff..397852da6ef3 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
@@ -1119,58 +1119,124 @@
 # GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
 
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20
-# GFX12: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W32-REAL16: v_cvt_pk_bf8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W32-FAKE16: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W64-REAL16: v_cvt_pk_bf8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
+# W64-FAKE16: v_cvt_pk_bf8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x6a,0xd7,0x02,0x07,0x02,0x20]
 
 0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
 0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
 0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+
+0x01,0x42,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v1.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20
-# GFX12: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W32-REAL16: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W32-FAKE16: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W64-REAL16: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# W64-FAKE16: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 
 0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+
+0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
 0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
 # GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
index e4bef0a17385..8688410a8c9b 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
@@ -681,28 +681,64 @@
 # GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 
 0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
 
 0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6a,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x6a,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# GFX12: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+0xff,0x43,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+# W32-REAL16: v_cvt_pk_bf8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W32-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-REAL16: v_cvt_pk_bf8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v255, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x6a,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_cvt_sr_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6b,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt
index d97418b279b7..07febf31641f 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt
@@ -7,6 +7,12 @@
 # GFX12: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf bound_ctrl:1 fi:1 ; encoding: [0x00,0x40,0x13,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x0c,0xff]
 0x00,0x40,0x13,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x0c,0xff
 
+# GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
+0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe
+
+# GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf bound_ctrl:1 fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x0c,0xff]
+0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x0c,0xff
+
 # GFX12: v_fma_mix_f32_e64_dpp v0, v1, v2, v3 row_ror:7 row_mask:0xf bank_mask:0x1 bound_ctrl:1 ; encoding: [0x00,0x00,0x20,0xcc,0xfa,0x04,0x0e,0x04,0x01,0x27,0x09,0xf1]
 0x00,0x00,0x20,0xcc,0xfa,0x04,0x0e,0x04,0x01,0x27,0x09,0xf1
 
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt
index 1461065d1c54..9cb888f06255 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt
@@ -4,6 +4,9 @@
 # GFX12: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x13,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
 0x00,0x45,0x13,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05
 
+# GFX12: v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
+0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05
+
 # GFX12: v_fma_mix_f32_e64_dpp v0, v1, v2, v3 clamp dpp8:[2,2,2,2,4,4,4,4] fi:1 ; encoding: [0x00,0x80,0x20,0xcc,0xea,0x04,0x0e,0x04,0x01,0x92,0x44,0x92]
 0x00,0x80,0x20,0xcc,0xea,0x04,0x0e,0x04,0x01,0x92,0x44,0x92
 
diff --git a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx950_mai.txt b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx950_mai.txt
index accd1cc8db03..77b87ac63f33 100644
--- a/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx950_mai.txt
+++ b/external/llvm-project/llvm/test/MC/Disassembler/AMDGPU/gfx950_mai.txt
@@ -467,6 +467,18 @@
 # GFX950:   v_mfma_scale_f32_16x16x128_f8f6f4 v[50:53], v[4:11], v[12:19], v[20:23], v24, v25 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x18,0x33,0x02,0x00,0x32,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
 0x00,0x00,0xac,0xd3,0x18,0x33,0x02,0x00,0x32,0x08,0xad,0xd3,0x04,0x19,0x52,0x04
 
+# GFX950:   v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], 4.0, v2 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf6,0x04,0x02,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+0x00,0x00,0xac,0xd3,0xf6,0x04,0x02,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04
+
+# GFX950:   v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], v2, 4.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x02,0xed,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+0x00,0x00,0xac,0xd3,0x02,0xed,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04
+
+# GFX950:   v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], -4.0, 1.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf7,0xe4,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+0x00,0x00,0xac,0xd3,0xf7,0xe4,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04
+
+# GFX950:   v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[12:19], v[20:23], 0.15915494, -16 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf8,0xa0,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04]
+0x00,0x00,0xac,0xd3,0xf8,0xa0,0x01,0x00,0x00,0x08,0xad,0xd3,0x04,0x19,0x52,0x04
+
 # GFX950:   v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], a[16:19], a[24:27], a[32:47], v48, v49 op_sel_hi:[0,0,0] cbsz:4 blgp:4 ; encoding: [0x00,0x00,0xac,0xd3,0x30,0x63,0x02,0x00,0x00,0x8c,0xae,0xd3,0x10,0x31,0x82,0x9c]
 0x00,0x00,0xac,0xd3,0x30,0x63,0x02,0x00,0x00,0x8c,0xae,0xd3,0x10,0x31,0x82,0x9c
 
@@ -569,6 +581,17 @@
 # GFX950:   v_mfma_scale_f32_32x32x64_f8f6f4 v[50:65], v[16:23], v[24:31], v[32:47], v48, v49 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x30,0x63,0x02,0x00,0x32,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
 0x00,0x00,0xac,0xd3,0x30,0x63,0x02,0x00,0x32,0x08,0xae,0xd3,0x10,0x31,0x82,0x04
 
+# GFX950: 	v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 16, v49 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x90,0x62,0x02,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+0x00,0x00,0xac,0xd3,0x90,0x62,0x02,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04
+
+# GFX950: 	v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], v48, -4.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0x30,0xef,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+0x00,0x00,0xac,0xd3,0x30,0xef,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04
+
+# GFX950: 	v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 4.0, 1.0 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf6,0xe4,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+0x00,0x00,0xac,0xd3,0xf6,0xe4,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04
+
+# GFX950: 	v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[32:47], 0.15915494, -16 op_sel_hi:[0,0,0] ; encoding: [0x00,0x00,0xac,0xd3,0xf8,0xa0,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04]
+0x00,0x00,0xac,0xd3,0xf8,0xa0,0x01,0x00,0x00,0x08,0xae,0xd3,0x10,0x31,0x82,0x04
 
 # GFX950:   v_mfma_i32_16x16x64_i8 a[0:3], a[0:3], a[0:3], a[0:3] ; encoding: [0x00,0x80,0xb6,0xd3,0x00,0x01,0x02,0x1c]
 0x00,0x80,0xb6,0xd3,0x00,0x01,0x02,0x1c
diff --git a/external/llvm-project/llvm/test/MC/ELF/layout-interdependency3.s b/external/llvm-project/llvm/test/MC/ELF/layout-interdependency3.s
new file mode 100644
index 000000000000..bd73890192ee
--- /dev/null
+++ b/external/llvm-project/llvm/test/MC/ELF/layout-interdependency3.s
@@ -0,0 +1,49 @@
+## This contrived .space example previously triggered "invalid number of bytes" error.
+## https://github.com/llvm/llvm-project/issues/123402
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s
+
+# CHECK-LABEL: <p_1st>:
+# CHECK:        e: cli
+# CHECK-LABEL: <q_1st>:
+# CHECK:        25: nop
+
+	.section .p,"ax"
+p_1st:
+0:	pause
+	lfence
+	jmp	0b
+
+	.section .q,"ax"
+q_1st:
+	addl	11,%eax
+	addl	22,%eax
+
+q_cli:
+	cli
+0:	pause
+	lfence
+	jmp	0b
+
+	.section .p
+	.space	(q_cli - q_1st) - (. - p_1st), 0xcc
+	cli
+
+	.section .q
+q_sti:
+	sti
+
+	.section .p
+	.space	(q_sti - q_1st) - (. - p_1st), 0xcc
+	sti
+	addl	33,%eax
+	addl	44,%eax
+p_nop:
+	nop
+
+	.section .q
+0:	pause
+	lfence
+	jmp	0b
+	.space	(p_nop - p_1st) - (. - q_1st), 0xcc
+	nop
diff --git a/external/llvm-project/llvm/test/MC/Lanai/conditional_inst.s b/external/llvm-project/llvm/test/MC/Lanai/conditional_inst.s
index d167d1af00eb..a0a8caf269fe 100644
--- a/external/llvm-project/llvm/test/MC/Lanai/conditional_inst.s
+++ b/external/llvm-project/llvm/test/MC/Lanai/conditional_inst.s
@@ -27,14 +27,14 @@ jump2:
 ! CHECK: encoding: [0b1110110A,A,A,0x01'A']
 ! CHECK-NEXT: fixup A - offset: 0, value: jump1, kind: FIXUP_LANAI_25
 ! CHECK-NEXT: <MCInst #{{[0-9]+}} BRCC{{$}}
-! CHECK-NEXT: <MCOperand Expr:(jump1)>
+! CHECK-NEXT: <MCOperand Expr:specifier(0,jump1)>
 ! CHECK-NEXT: <MCOperand Imm:13>
 
     bpl jump2
 ! CHECK: encoding: [0b1110101A,A,A,A]
 ! CHECK-NEXT: fixup A - offset: 0, value: jump2, kind: FIXUP_LANAI_25
 ! CHECK-NEXT: <MCInst #{{[0-9]+}} BRCC{{$}}
-! CHECK-NEXT: <MCOperand Expr:(jump2)>
+! CHECK-NEXT: <MCOperand Expr:specifier(0,jump2)>
 ! CHECK-NEXT: <MCOperand Imm:10>
 
     bt .
@@ -43,7 +43,7 @@ jump2:
 ! CHECK:      encoding: [0b1110000A,A,A,A]
 ! CHECK-NEXT:   fixup A - offset: 0, value: .Ltmp0, kind: FIXUP_LANAI_25
 ! CHECK-NEXT: <MCInst #{{[0-9]+}} BT{{$}}
-! CHECK-NEXT:   <MCOperand Expr:(.Ltmp0)>
+! CHECK-NEXT:   <MCOperand Expr:.Ltmp0>
 
 ! SCC
     spl %r19
diff --git a/external/llvm-project/llvm/test/MC/Lanai/memory.s b/external/llvm-project/llvm/test/MC/Lanai/memory.s
index 41dc8fba7bf2..0e6234645a80 100644
--- a/external/llvm-project/llvm/test/MC/Lanai/memory.s
+++ b/external/llvm-project/llvm/test/MC/Lanai/memory.s
@@ -235,7 +235,7 @@
 ! CHECK-NEXT: <MCInst #{{[0-9]+}} ADD_I_HI
 ! CHECK-NEXT: <MCOperand Reg:11>
 ! CHECK-NEXT: <MCOperand Reg:7>
-! CHECK-NEXT: <MCOperand Expr:(hi(x))>
+! CHECK-NEXT: <MCOperand Expr:specifier(1,x)>
 
     mov hi(l+4), %r7
 ! CHECK: encoding: [0x03,0x81,A,A]
@@ -243,5 +243,5 @@
 ! CHECK-NEXT: <MCInst #{{[0-9]+}} ADD_I_HI
 ! CHECK-NEXT: <MCOperand Reg:14>
 ! CHECK-NEXT: <MCOperand Reg:7>
-! CHECK-NEXT: <MCOperand Expr:(hi(l)+4)>
+! CHECK-NEXT: <MCOperand Expr:specifier(1,l)+4>
 
diff --git a/external/llvm-project/llvm/test/MC/Mips/cpsetup.s b/external/llvm-project/llvm/test/MC/Mips/cpsetup.s
index 4a027c6e796a..f948d650da94 100644
--- a/external/llvm-project/llvm/test/MC/Mips/cpsetup.s
+++ b/external/llvm-project/llvm/test/MC/Mips/cpsetup.s
@@ -196,7 +196,7 @@ IMM_8 = 8
 
 # ALL-LABEL: <t1b>:
 # ASM-LABEL: t1b:
-# ASM-NEXT: .set IMM_8, 8
+# ASM-NEXT: IMM_8 = 8
 
 # O32-NOT: __cerror
 
diff --git a/external/llvm-project/llvm/test/MC/RISCV/long-jump-disable-relax.s b/external/llvm-project/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
similarity index 86%
rename from external/llvm-project/llvm/test/MC/RISCV/long-jump-disable-relax.s
rename to external/llvm-project/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
index 815c2dfcec07..6e4b5fdd31e1 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/long-jump-disable-relax.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
@@ -1,14 +1,16 @@
-## Test that long branches are not relaxed with -riscv-asm-relax-branches=0
+## Test that long branches are not relaxed with .option exact
 # RUN: split-file %s %t
 # RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c \
-# RUN:       -riscv-asm-relax-branches=0 %t/pass.s \
+# RUN:       %t/pass.s \
 # RUN:     | llvm-objdump -dr -M no-aliases - \
 # RUN:     | FileCheck %t/pass.s
 # RUN: not llvm-mc -filetype=obj -triple=riscv64 -mattr=+c -o /dev/null \
-# RUN:       -riscv-asm-relax-branches=0 %t/fail.s 2>&1 \
+# RUN:       %t/fail.s 2>&1 \
 # RUN:     | FileCheck %t/fail.s
 
 #--- pass.s
+  .option exact
+
   .text
 test_undefined:
 ## Branches to undefined symbols should not be relaxed
@@ -33,6 +35,8 @@ test_defined_in_range:
 bar:
 
 #--- fail.s
+  .option exact
+
   .text
 ## Branches to defined out-of-range symbols should report an error
 test_defined_out_of_range:
diff --git a/external/llvm-project/llvm/test/MC/RISCV/rv32-relaxation-xqci.s b/external/llvm-project/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
new file mode 100644
index 000000000000..03c9914bb37d
--- /dev/null
+++ b/external/llvm-project/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
@@ -0,0 +1,170 @@
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcilb %t/pass.s -o - \
+# RUN:   | llvm-objdump -dr -M no-aliases - --mattr=+experimental-xqcilb | FileCheck %s
+# RUN: not llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcilb %t/fail.s \
+# RUN:   2>&1 | FileCheck %t/fail.s --check-prefix=ERROR
+
+## This testcase shows how `c.j`, `c.jal` and `jal` can be relaxed to `qc.e.j` and `qc.e.jal`
+## with Xqcilb, when the branches are out of range, but also that these can be compressed
+## when referencing close labels.
+
+## The only problem we have here is when `jal` references an out-of-range label, and `rd` is
+## not `x0` or `x1` - for which we have no equivalent sequence, so we just fail to relax, and
+## emit a fixup-value-out-of-range error.
+
+#--- pass.s
+
+EXT_JUMP_NEGATIVE:
+  c.nop
+.space 0x100000
+
+FAR_JUMP_NEGATIVE:
+  c.nop
+.space 0x1000
+
+NEAR_NEGATIVE:
+  c.nop
+
+start:
+  c.j NEAR
+# CHECK: c.j {{0x[0-9a-f]+}} <NEAR>
+  c.j NEAR_NEGATIVE
+# CHECK: c.j {{0x[0-9a-f]+}} <NEAR_NEGATIVE>
+  c.j FAR_JUMP
+# CHECK: jal zero, {{0x[0-9a-f]+}} <FAR_JUMP>
+  c.j FAR_JUMP_NEGATIVE
+# CHECK: jal zero, {{0x[0-9a-f]+}} <FAR_JUMP_NEGATIVE>
+  c.j EXT_JUMP
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <EXT_JUMP>
+  c.j EXT_JUMP_NEGATIVE
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <EXT_JUMP_NEGATIVE>
+  c.j undef
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <start+{{0x[0-9a-f]+}}>
+# CHECK: R_RISCV_CUSTOM195 undef
+
+  c.jal NEAR
+# CHECK: c.jal {{0x[0-9a-f]+}} <NEAR>
+  c.jal NEAR_NEGATIVE
+# CHECK: c.jal {{0x[0-9a-f]+}} <NEAR_NEGATIVE>
+  c.jal FAR_JUMP
+# CHECK: jal ra, {{0x[0-9a-f]+}} <FAR_JUMP>
+  c.jal FAR_JUMP_NEGATIVE
+# CHECK: jal ra, {{0x[0-9a-f]+}} <FAR_JUMP_NEGATIVE>
+  c.jal EXT_JUMP
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <EXT_JUMP>
+  c.jal EXT_JUMP_NEGATIVE
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <EXT_JUMP_NEGATIVE>
+  c.jal undef
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <start+{{0x[0-9a-f]+}}>
+# CHECK: R_RISCV_CUSTOM195 undef
+
+  jal zero, NEAR
+# CHECK: c.j {{0x[0-9a-f]+}} <NEAR>
+  jal zero, NEAR_NEGATIVE
+# CHECK: c.j {{0x[0-9a-f]+}} <NEAR_NEGATIVE>
+  jal zero, FAR_JUMP
+# CHECK: jal zero, {{0x[0-9a-f]+}} <FAR_JUMP>
+  jal zero, FAR_JUMP_NEGATIVE
+# CHECK: jal zero, {{0x[0-9a-f]+}} <FAR_JUMP_NEGATIVE>
+  jal zero, EXT_JUMP
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <EXT_JUMP>
+  jal zero, EXT_JUMP_NEGATIVE
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <EXT_JUMP_NEGATIVE>
+  jal zero, undef
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <start+{{0x[0-9a-f]+}}>
+# CHECK: R_RISCV_CUSTOM195 undef
+
+  jal ra, NEAR
+# CHECK: c.jal {{0x[0-9a-f]+}} <NEAR>
+  jal ra, NEAR_NEGATIVE
+# CHECK: c.jal {{0x[0-9a-f]+}} <NEAR_NEGATIVE>
+  jal ra, FAR_JUMP
+# CHECK: jal ra, {{0x[0-9a-f]+}} <FAR_JUMP>
+  jal ra, FAR_JUMP_NEGATIVE
+# CHECK: jal ra, {{0x[0-9a-f]+}} <FAR_JUMP_NEGATIVE>
+  jal ra, EXT_JUMP
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <EXT_JUMP>
+  jal ra, EXT_JUMP_NEGATIVE
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <EXT_JUMP_NEGATIVE>
+  jal ra, undef
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <start+{{0x[0-9a-f]+}}>
+# CHECK: R_RISCV_CUSTOM195 undef
+
+  qc.e.j NEAR
+# CHECK: c.j {{0x[0-9a-f]+}} <NEAR>
+  qc.e.j NEAR_NEGATIVE
+# CHECK: c.j {{0x[0-9a-f]+}} <NEAR_NEGATIVE>
+  qc.e.j FAR_JUMP
+# CHECK: jal zero, {{0x[0-9a-f]+}} <FAR_JUMP>
+  qc.e.j FAR_JUMP_NEGATIVE
+# CHECK: jal zero, {{0x[0-9a-f]+}} <FAR_JUMP_NEGATIVE>
+  qc.e.j EXT_JUMP
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <EXT_JUMP>
+  qc.e.j EXT_JUMP_NEGATIVE
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <EXT_JUMP_NEGATIVE>
+  qc.e.j undef
+# CHECK: qc.e.j {{0x[0-9a-f]+}} <start+{{0x[0-9a-f]+}}>
+# CHECK: R_RISCV_CUSTOM195 undef
+
+  qc.e.jal NEAR
+# CHECK: c.jal {{0x[0-9a-f]+}} <NEAR>
+  qc.e.jal NEAR_NEGATIVE
+# CHECK: c.jal {{0x[0-9a-f]+}} <NEAR_NEGATIVE>
+  qc.e.jal FAR_JUMP
+# CHECK: jal ra, {{0x[0-9a-f]+}} <FAR_JUMP>
+  qc.e.jal FAR_JUMP_NEGATIVE
+# CHECK: jal ra, {{0x[0-9a-f]+}} <FAR_JUMP_NEGATIVE>
+  qc.e.jal EXT_JUMP
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <EXT_JUMP>
+  qc.e.jal EXT_JUMP_NEGATIVE
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <EXT_JUMP_NEGATIVE>
+  qc.e.jal undef
+# CHECK: qc.e.jal {{0x[0-9a-f]+}} <start+{{0x[0-9a-f]+}}>
+# CHECK: R_RISCV_CUSTOM195 undef
+
+
+
+  jal t1, NEAR
+# CHECK: jal t1, {{0x[0-9a-f]+}} <NEAR>
+  jal t1, NEAR_NEGATIVE
+# CHECK: jal t1, {{0x[0-9a-f]+}} <NEAR_NEGATIVE>
+  jal t1, FAR_JUMP
+# CHECK: jal t1, {{0x[0-9a-f]+}} <FAR_JUMP>
+  jal t1, FAR_JUMP_NEGATIVE
+# CHECK: jal t1, {{0x[0-9a-f]+}} <FAR_JUMP_NEGATIVE>
+
+## The two cases with EXT_JUMP and EXT_JUMP_NEGATIVE are
+## in fail.s, below.
+
+  jal t1, undef
+# CHECK: jal t1, {{0x[0-9a-f]+}} <start+{{0x[0-9a-f]+}}>
+# CHECK: R_RISCV_JAL undef
+
+
+NEAR:
+  c.nop
+.space 0x1000
+FAR_JUMP:
+  c.nop
+.space 0x100000
+EXT_JUMP:
+  c.nop
+
+
+#--- fail.s
+
+
+EXT_JUMP_NEGATIVE:
+  c.nop
+.space 0x100000
+.space 0x1000
+
+  jal t1, EXT_JUMP
+# ERROR: [[@LINE-1]]:3: error: fixup value out of range
+  jal t1, EXT_JUMP_NEGATIVE
+# ERROR: [[@LINE-1]]:3: error: fixup value out of range
+
+.space 0x1000
+.space 0x100000
+EXT_JUMP:
+  c.nop
diff --git a/external/llvm-project/llvm/test/MC/RISCV/rv64c-aliases-valid.s b/external/llvm-project/llvm/test/MC/RISCV/rv64c-aliases-valid.s
index a464c24609fb..e0fd98cf3739 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/rv64c-aliases-valid.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/rv64c-aliases-valid.s
@@ -30,32 +30,32 @@ li x11, 2048
 # CHECK-EXPAND: addi a1, zero, -2048
 li x11, -2048
 # CHECK-EXPAND: c.lui a1, 1
-# CHECK-EXPAND: addiw a1, a1, -2047
+# CHECK-EXPAND: addi a1, a1, -2047
 li x11, 2049
 # CHECK-EXPAND: c.lui a1, 1048575
-# CHECK-EXPAND: addiw a1, a1, 2047
+# CHECK-EXPAND: addi a1, a1, 2047
 li x11, -2049
 # CHECK-EXPAND: c.lui a1, 1
-# CHECK-EXPAND: c.addiw a1, -1
+# CHECK-EXPAND: c.addi a1, -1
 li x11, 4095
 # CHECK-EXPAND: lui a1, 1048575
-# CHECK-EXPAND: c.addiw a1, 1
+# CHECK-EXPAND: c.addi a1, 1
 li x11, -4095
 # CHECK-EXPAND: c.lui a2, 1
 li x12, 4096
 # CHECK-EXPAND: lui a2, 1048575
 li x12, -4096
 # CHECK-EXPAND: c.lui a2, 1
-# CHECK-EXPAND: c.addiw a2, 1
+# CHECK-EXPAND: c.addi a2, 1
 li x12, 4097
 # CHECK-EXPAND: lui a2, 1048575
-# CHECK-EXPAND: c.addiw a2, -1
+# CHECK-EXPAND: c.addi a2, -1
 li x12, -4097
 # CHECK-EXPAND: lui a2, 524288
 # CHECK-EXPAND: c.addiw a2, -1
 li x12, 2147483647
 # CHECK-EXPAND: lui a2, 524288
-# CHECK-EXPAND: c.addiw a2, 1
+# CHECK-EXPAND: c.addi a2, 1
 li x12, -2147483647
 # CHECK-EXPAND: lui a2, 524288
 li x12, -2147483648
@@ -79,7 +79,7 @@ li t1, 0x8000000000000000
 # CHECK-EXPAND: c.slli t1, 63
 li t1, -0x8000000000000000
 # CHECK-EXPAND: lui t2, 9321
-# CHECK-EXPAND: addiw t2, t2, -1329
+# CHECK-EXPAND: addi t2, t2, -1329
 # CHECK-EXPAND: c.slli t2, 35
 li t2, 0x1234567800000000
 # CHECK-EXPAND: c.li t3, 7
@@ -89,7 +89,7 @@ li t2, 0x1234567800000000
 # CHECK-EXPAND: c.addi t3, 15
 li t3, 0x700000000B00000F
 # CHECK-EXPAND: lui t4, 583
-# CHECK-EXPAND: addiw t4, t4, -1875
+# CHECK-EXPAND: addi t4, t4, -1875
 # CHECK-EXPAND: c.slli t4, 14
 # CHECK-EXPAND: addi t4, t4, -947
 # CHECK-EXPAND: c.slli t4, 12
diff --git a/external/llvm-project/llvm/test/MC/RISCV/rv64i-aliases-valid.s b/external/llvm-project/llvm/test/MC/RISCV/rv64i-aliases-valid.s
index b3956c252447..2a37caa0c5c2 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/rv64i-aliases-valid.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/rv64i-aliases-valid.s
@@ -48,32 +48,32 @@ li x11, 2048
 # CHECK-ALIAS: li a1, -2048
 li x11, -2048
 # CHECK-EXPAND: lui a1, 1
-# CHECK-EXPAND: addiw a1, a1, -2047
+# CHECK-EXPAND: addi a1, a1, -2047
 li x11, 2049
 # CHECK-EXPAND: lui a1, 1048575
-# CHECK-EXPAND: addiw a1, a1, 2047
+# CHECK-EXPAND: addi a1, a1, 2047
 li x11, -2049
 # CHECK-EXPAND: lui a1, 1
-# CHECK-EXPAND: addiw a1, a1, -1
+# CHECK-EXPAND: addi a1, a1, -1
 li x11, 4095
 # CHECK-EXPAND: lui a1, 1048575
-# CHECK-EXPAND: addiw a1, a1, 1
+# CHECK-EXPAND: addi a1, a1, 1
 li x11, -4095
 # CHECK-EXPAND: lui a2, 1
 li x12, 4096
 # CHECK-EXPAND: lui a2, 1048575
 li x12, -4096
 # CHECK-EXPAND: lui a2, 1
-# CHECK-EXPAND-NEXT: addiw a2, a2, 1
+# CHECK-EXPAND-NEXT: addi a2, a2, 1
 li x12, 4097
 # CHECK-EXPAND: lui a2, 1048575
-# CHECK-EXPAND: addiw a2, a2, -1
+# CHECK-EXPAND: addi a2, a2, -1
 li x12, -4097
 # CHECK-EXPAND: lui a2, 524288
 # CHECK-EXPAND-NEXT: addiw a2, a2, -1
 li x12, 2147483647
 # CHECK-EXPAND: lui a2, 524288
-# CHECK-EXPAND-NEXT: addiw a2, a2, 1
+# CHECK-EXPAND-NEXT: addi a2, a2, 1
 li x12, -2147483647
 # CHECK-EXPAND: lui a2, 524288
 li x12, -2147483648
@@ -107,7 +107,7 @@ li t1, 0x8000000000000000
 # CHECK-ALIAS-NEXT: slli t1, t1, 63
 li t1, -0x8000000000000000
 # CHECK-EXPAND: lui t2, 9321
-# CHECK-EXPAND-NEXT: addiw t2, t2, -1329
+# CHECK-EXPAND-NEXT: addi t2, t2, -1329
 # CHECK-EXPAND-NEXT: slli t2, t2, 35
 li t2, 0x1234567800000000
 # CHECK-INST: addi t3, zero, 7
@@ -122,7 +122,7 @@ li t2, 0x1234567800000000
 # CHECK-ALIAS-NEXT: addi t3, t3, 15
 li t3, 0x700000000B00000F
 # CHECK-EXPAND: lui t4, 583
-# CHECK-EXPAND-NEXT: addiw t4, t4, -1875
+# CHECK-EXPAND-NEXT: addi t4, t4, -1875
 # CHECK-EXPAND-NEXT: slli t4, t4, 14
 # CHECK-EXPAND-NEXT: addi t4, t4, -947
 # CHECK-EXPAND-NEXT: slli t4, t4, 12
@@ -171,18 +171,18 @@ li x10, 0xE000000001FFFFFF
 li x11, 0xFFFC007FFFFFF7FF
 
 # CHECK-INST: lui a2, 349525
-# CHECK-INST-NEXT: addiw a2, a2, 1365
+# CHECK-INST-NEXT: addi a2, a2, 1365
 # CHECK-INST-NEXT: slli a2, a2, 1
 # CHECK-ALIAS: lui a2, 349525
-# CHECK-ALIAS-NEXT: addiw a2, a2, 1365
+# CHECK-ALIAS-NEXT: addi a2, a2, 1365
 # CHECK-ALIAS-NEXT: slli a2, a2, 1
 li x12, 0xaaaaaaaa
 
 # CHECK-INST: lui a3, 699051
-# CHECK-INST-NEXT: addiw a3, a3, -1365
+# CHECK-INST-NEXT: addi a3, a3, -1365
 # CHECK-INST-NEXT: slli a3, a3, 1
 # CHECK-ALIAS: lui a3, 699051
-# CHECK-ALIAS-NEXT: addiw a3, a3, -1365
+# CHECK-ALIAS-NEXT: addi a3, a3, -1365
 # CHECK-ALIAS-NEXT: slli a3, a3, 1
 li x13, 0xffffffff55555556
 
@@ -205,12 +205,12 @@ li a0, %pcrel_lo(.Lpcrel_hi0)
 
 .equ CONST, 0x123456
 # CHECK-EXPAND: lui a0, 291
-# CHECK-EXPAND: addiw a0, a0, 1110
+# CHECK-EXPAND: addi a0, a0, 1110
 li a0, CONST
 
 .equ CONST, 0x654321
 # CHECK-EXPAND: lui a0, 1620
-# CHECK-EXPAND: addiw a0, a0, 801
+# CHECK-EXPAND: addi a0, a0, 801
 li a0, CONST
 
 .equ CONST, .Lbuf_end - .Lbuf
@@ -255,19 +255,19 @@ lla x11, 2048
 la x11, -2048
 lla x11, -2048
 # CHECK-EXPAND: lui a1, 1
-# CHECK-EXPAND: addiw a1, a1, -2047
+# CHECK-EXPAND: addi a1, a1, -2047
 la x11, 2049
 lla x11, 2049
 # CHECK-EXPAND: lui a1, 1048575
-# CHECK-EXPAND: addiw a1, a1, 2047
+# CHECK-EXPAND: addi a1, a1, 2047
 la x11, -2049
 lla x11, -2049
 # CHECK-EXPAND: lui a1, 1
-# CHECK-EXPAND: addiw a1, a1, -1
+# CHECK-EXPAND: addi a1, a1, -1
 la x11, 4095
 lla x11, 4095
 # CHECK-EXPAND: lui a1, 1048575
-# CHECK-EXPAND: addiw a1, a1, 1
+# CHECK-EXPAND: addi a1, a1, 1
 la x11, -4095
 lla x11, -4095
 # CHECK-EXPAND: lui a2, 1
@@ -277,7 +277,7 @@ lla x12, 4096
 la x12, -4096
 lla x12, -4096
 # CHECK-EXPAND: lui a2, 1
-# CHECK-EXPAND: addiw a2, a2, 1
+# CHECK-EXPAND: addi a2, a2, 1
 la x12, 4097
 lla x12, 4097
 # CHECK-EXPAND: lui a2, 1048575
@@ -289,7 +289,7 @@ lla x12, -4097
 la x12, 2147483647
 lla x12, 2147483647
 # CHECK-EXPAND: lui a2, 524288
-# CHECK-EXPAND: addiw a2, a2, 1
+# CHECK-EXPAND: addi a2, a2, 1
 la x12, -2147483647
 lla x12, -2147483647
 # CHECK-EXPAND: lui a2, 524288
@@ -331,7 +331,7 @@ lla t1, 0x8000000000000000
 la t1, -0x8000000000000000
 lla t1, -0x8000000000000000
 # CHECK-EXPAND: lui t2, 9321
-# CHECK-EXPAND-NEXT: addiw t2, t2, -1329
+# CHECK-EXPAND-NEXT: addi t2, t2, -1329
 # CHECK-EXPAND-NEXT: slli t2, t2, 35
 la t2, 0x1234567800000000
 lla t2, 0x1234567800000000
@@ -348,7 +348,7 @@ lla t2, 0x1234567800000000
 la t3, 0x700000000B00000F
 lla t3, 0x700000000B00000F
 # CHECK-EXPAND: lui t4, 583
-# CHECK-EXPAND-NEXT: addiw t4, t4, -1875
+# CHECK-EXPAND-NEXT: addi t4, t4, -1875
 # CHECK-EXPAND-NEXT: slli t4, t4, 14
 # CHECK-EXPAND-NEXT: addi t4, t4, -947
 # CHECK-EXPAND-NEXT: slli t4, t4, 12
@@ -407,19 +407,19 @@ la x11, 0xFFFC007FFFFFF7FF
 lla x11, 0xFFFC007FFFFFF7FF
 
 # CHECK-INST: lui a2, 349525
-# CHECK-INST-NEXT: addiw a2, a2, 1365
+# CHECK-INST-NEXT: addi a2, a2, 1365
 # CHECK-INST-NEXT: slli a2, a2, 1
 # CHECK-ALIAS: lui a2, 349525
-# CHECK-ALIAS-NEXT: addiw a2, a2, 1365
+# CHECK-ALIAS-NEXT: addi a2, a2, 1365
 # CHECK-ALIAS-NEXT: slli a2, a2, 1
 la x12, 0xaaaaaaaa
 lla x12, 0xaaaaaaaa
 
 # CHECK-INST: lui a3, 699051
-# CHECK-INST-NEXT: addiw a3, a3, -1365
+# CHECK-INST-NEXT: addi a3, a3, -1365
 # CHECK-INST-NEXT: slli a3, a3, 1
 # CHECK-ALIAS: lui a3, 699051
-# CHECK-ALIAS-NEXT: addiw a3, a3, -1365
+# CHECK-ALIAS-NEXT: addi a3, a3, -1365
 # CHECK-ALIAS-NEXT: slli a3, a3, 1
 la x13, 0xffffffff55555556
 lla x13, 0xffffffff55555556
@@ -433,13 +433,13 @@ lla x5, -2147485013
 
 .equ CONSTANT, 0x123456
 # CHECK-EXPAND: lui a0, 291
-# CHECK-EXPAND: addiw a0, a0, 1110
+# CHECK-EXPAND: addi a0, a0, 1110
 la a0, CONSTANT
 lla a0, CONSTANT
 
 .equ CONSTANT, 0x654321
 # CHECK-EXPAND: lui a0, 1620
-# CHECK-EXPAND: addiw a0, a0, 801
+# CHECK-EXPAND: addi a0, a0, 801
 la a0, CONSTANT
 lla a0, CONSTANT
 
diff --git a/external/llvm-project/llvm/test/MC/RISCV/rv64zba-aliases-valid.s b/external/llvm-project/llvm/test/MC/RISCV/rv64zba-aliases-valid.s
index 78ae18b0eaa0..d3baab58c9f1 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/rv64zba-aliases-valid.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/rv64zba-aliases-valid.s
@@ -40,71 +40,71 @@ li x5, 0xbbbbb0007bb
 li x5, 0xbbbbb0000
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 611378
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, 265
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, 265
 # CHECK-S-OBJ-NOALIAS-NEXT: sh1add t1, t1, t1
 # CHECK-S-OBJ: lui t1, 611378
-# CHECK-S-OBJ-NEXT: addiw t1, t1, 265
+# CHECK-S-OBJ-NEXT: addi t1, t1, 265
 # CHECK-S-OBJ-NEXT: sh1add t1, t1, t1
 li x6, -5372288229
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 437198
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, -265
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, -265
 # CHECK-S-OBJ-NOALIAS-NEXT: sh2add t1, t1, t1
 # CHECK-S-OBJ: lui t1, 437198
-# CHECK-S-OBJ-NEXT: addiw t1, t1, -265
+# CHECK-S-OBJ-NEXT: addi t1, t1, -265
 # CHECK-S-OBJ-NEXT: sh2add t1, t1, t1
 li x6, 8953813715
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 611378
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, 265
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, 265
 # CHECK-S-OBJ-NOALIAS-NEXT: sh2add t1, t1, t1
 # CHECK-S-OBJ: lui t1, 611378
-# CHECK-S-OBJ-NEXT: addiw t1, t1, 265
+# CHECK-S-OBJ-NEXT: addi t1, t1, 265
 # CHECK-S-OBJ-NEXT: sh2add t1, t1, t1
 li x6, -8953813715
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 437198
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, -265
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, -265
 # CHECK-S-OBJ-NOALIAS-NEXT: sh3add t1, t1, t1
 # CHECK-S-OBJ: lui t1, 437198
-# CHECK-S-OBJ-NEXT: addiw t1, t1, -265
+# CHECK-S-OBJ-NEXT: addi t1, t1, -265
 # CHECK-S-OBJ-NEXT: sh3add t1, t1, t1
 li x6, 16116864687
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 611378
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, 265
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, 265
 # CHECK-S-OBJ-NOALIAS-NEXT: sh3add t1, t1, t1
 # CHECK-S-OBJ: lui t1, 611378
-# CHECK-S-OBJ-NEXT: addiw t1, t1, 265
+# CHECK-S-OBJ-NEXT: addi t1, t1, 265
 # CHECK-S-OBJ-NEXT: sh3add t1, t1, t1
 li x6, -16116864687
 
 # CHECK-S-OBJ-NOALIAS: lui t2, 768956
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t2, t2, -1093
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t2, t2, -1093
 # CHECK-S-OBJ-NOALIAS-NEXT: slli.uw t2, t2, 12
 # CHECK-S-OBJ-NOALIAS-NEXT: addi t2, t2, 1911
 # CHECK-S-OBJ: lui t2, 768956
-# CHECK-S-OBJ-NEXT: addiw t2, t2, -1093
+# CHECK-S-OBJ-NEXT: addi t2, t2, -1093
 # CHECK-S-OBJ-NEXT: slli.uw t2, t2, 12
 # CHECK-S-OBJ-NEXT: addi t2, t2, 1911
 li x7, 12900936431479
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 768955
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, 273
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, 273
 # CHECK-S-OBJ-NOALIAS-NEXT: slli.uw t1, t1, 12
 # CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, 273
 # CHECK-S-OBJ: lui t1, 768955
-# CHECK-S-OBJ-NEXT: addiw t1, t1, 273
+# CHECK-S-OBJ-NEXT: addi t1, t1, 273
 # CHECK-S-OBJ-NEXT: slli.uw t1, t1, 12
 # CHECK-S-OBJ-NEXT: addi t1, t1, 273
 li x6, 12900925247761
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 768955
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, -1365
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, -1365
 # CHECK-S-OBJ-NOALIAS-NEXT: slli.uw t1, t1, 12
 # CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, -1366
 # CHECK-S-OBJ: lui t1, 768955
-# CHECK-S-OBJ-NEXT: addiw t1, t1, -1365
+# CHECK-S-OBJ-NEXT: addi t1, t1, -1365
 # CHECK-S-OBJ-NEXT: slli.uw t1, t1, 12
 # CHECK-S-OBJ-NEXT: addi t1, t1, -1366
 li x6, 12900918536874
diff --git a/external/llvm-project/llvm/test/MC/RISCV/rv64zbs-aliases-valid.s b/external/llvm-project/llvm/test/MC/RISCV/rv64zbs-aliases-valid.s
index 0379a06ad4c8..133a838a9652 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/rv64zbs-aliases-valid.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/rv64zbs-aliases-valid.s
@@ -38,21 +38,21 @@ bext x5, x6, 8
 li x5, 2147485013
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 572348
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, -1093
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, -1093
 # CHECK-S-OBJ-NOALIAS-NEXT: bclri t1, t1, 44
 # CHECK-S-OBJ-NOALIAS-NEXT: bclri t1, t1, 63
 # CHECK-S-OBJ: lui t1, 572348
-# CHECK-S-OBJ-NEXT: addiw t1, t1, -1093
+# CHECK-S-OBJ-NEXT: addi t1, t1, -1093
 # CHECK-S-OBJ-NEXT: bclri t1, t1, 44
 # CHECK-S-OBJ-NEXT: bclri t1, t1, 63
 li x6, 9223354442718100411
 
 # CHECK-S-OBJ-NOALIAS: lui t1, 506812
-# CHECK-S-OBJ-NOALIAS-NEXT: addiw t1, t1, -1093
+# CHECK-S-OBJ-NOALIAS-NEXT: addi t1, t1, -1093
 # CHECK-S-OBJ-NOALIAS-NEXT: bseti t1, t1, 46
 # CHECK-S-OBJ-NOALIAS-NEXT: bseti t1, t1, 63
 # CHECK-S-OBJ: lui t1, 506812
-# CHECK-S-OBJ-NEXT: addiw t1, t1, -1093
+# CHECK-S-OBJ-NEXT: addi t1, t1, -1093
 # CHECK-S-OBJ-NEXT: bseti t1, t1, 46
 # CHECK-S-OBJ-NEXT: bseti t1, t1, 63
 li x6, -9223301666034697285
diff --git a/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s b/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s
index a9843c350fc8..5168c42da70d 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s
@@ -13,3 +13,9 @@ sf.vfwmacc.4x4x4 v8, v4, v20
 # CHECK-ENCODING: [0x5b,0x14,0x42,0xf3]
 # CHECK-ERROR: instruction requires the following: 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction and 4-by-4))
 # CHECK-UNKNOWN: f342145b <unknown>
+
+sf.vfwmacc.4x4x4 v0, v4, v20
+# CHECK-INST: sf.vfwmacc.4x4x4 v0, v4, v20
+# CHECK-ENCODING: [0x5b,0x10,0x42,0xf3]
+# CHECK-ERROR: instruction requires the following: 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction and 4-by-4))
+# CHECK-UNKNOWN: f342105b <unknown>
diff --git a/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvqmacc.s b/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvqmacc.s
index 81703c847d74..d7d3cba41cbc 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvqmacc.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/rvv/xsfvqmacc.s
@@ -55,3 +55,9 @@ sf.vqmaccsu.4x8x4 v8, v4, v20
 # CHECK-ENCODING: [0x5b,0x24,0x42,0xff]
 # CHECK-ERROR: instruction requires the following: 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
 # CHECK-UNKNOWN: ff42245b <unknown>
+
+sf.vqmaccu.4x8x4 v0, v4, v20
+# CHECK-INST: sf.vqmaccu.4x8x4 v0, v4, v20
+# CHECK-ENCODING: [0x5b,0x20,0x42,0xf3]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
+# CHECK-UNKNOWN: f342205b <unknown>
diff --git a/external/llvm-project/llvm/test/MC/RISCV/vendor-symbol.s b/external/llvm-project/llvm/test/MC/RISCV/vendor-symbol.s
new file mode 100644
index 000000000000..7df3a3efeb64
--- /dev/null
+++ b/external/llvm-project/llvm/test/MC/RISCV/vendor-symbol.s
@@ -0,0 +1,35 @@
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcibi %s \
+# RUN:     -filetype=obj -o - \
+# RUN:     | llvm-readelf -sr - \
+# RUN:     | FileCheck %s
+
+
+## This checks that the vendor identifier symbols required for vendor
+## relocations do not interfere with symbols with identical names that
+## are written in assembly.
+
+  .option exact
+
+  qc.e.bgeui s0, 20, QUALCOMM
+
+  .global QUALCOMM
+QUALCOMM:
+  nop
+
+  qc.e.bgeui s0, 20, QUALCOMM
+
+
+# CHECK-LABEL: Relocation section '.rela.text'
+## Note the different values for the "Sym. Value" Field
+# CHECK: R_RISCV_VENDOR    00000000 QUALCOMM + 0
+# CHECK: R_RISCV_CUSTOM193 00000006 QUALCOMM + 0
+# CHECK: R_RISCV_VENDOR    00000000 QUALCOMM + 0
+# CHECK: R_RISCV_CUSTOM193 00000006 QUALCOMM + 0
+
+
+# CHECK-LABEL: Symbol table '.symtab'
+# CHECK-NOT: QUALCOMM
+# CHECK: 00000000 0 NOTYPE  LOCAL  DEFAULT ABS QUALCOMM
+# CHECK-NOT: QUALCOMM
+# CHECK: 00000006 0 NOTYPE  GLOBAL DEFAULT   2 QUALCOMM
+# CHECK-NOT: QUALCOMM
diff --git a/external/llvm-project/llvm/test/MC/RISCV/xqcibi-long-conditional-jump.s b/external/llvm-project/llvm/test/MC/RISCV/xqcibi-long-conditional-jump.s
index aab664f2a75f..0279c81cea14 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/xqcibi-long-conditional-jump.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/xqcibi-long-conditional-jump.s
@@ -64,59 +64,115 @@ test:
 .L6:
    ret
 
-# CHECK-INST:         qc.e.beqi    a0, 0x1, 0x7a26
+# CHECK-INST:         qc.e.beqi    a0, 0x51, 0x7a26
 # CHECK-INST-NEXT:    jal     zero, 0x8e76
-# CHECK-INST-RELAX:         qc.e.beqi    a0, 0x1, 0x7a26
+# CHECK-INST-RELAX:         qc.e.beqi    a0, 0x51, 0x7a26
 # CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
-   qc.e.bnei a0, 1, .L7
+   qc.e.bnei a0, 81, .L7
 .fill 1300, 4, 0
 .L7:
    ret
 
-# CHECK-INST:         qc.e.bnei    a0, 0x2, 0x8e82
+# CHECK-INST:         qc.e.bnei    a0, 0x3e, 0x8e82
 # CHECK-INST-NEXT:    jal     zero, 0xa2d2
-# CHECK-INST-RELAX:         qc.e.bnei    a0, 0x2, 0x8e82
+# CHECK-INST-RELAX:         qc.e.bnei    a0, 0x3e, 0x8e82
 # CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
-   qc.e.beqi a0, 2, .L8
+   qc.e.beqi a0, 62, .L8
 .fill 1300, 4, 0
 .L8:
    ret
 
-# CHECK-INST:         qc.e.bgei    a0, 0x3, 0xa2de
+# CHECK-INST:         qc.e.bgei    a0, 0x5d, 0xa2de
 # CHECK-INST-NEXT:    jal     zero, 0xb72e
-# CHECK-INST-RELAX:         qc.e.bgei    a0, 0x3, 0xa2de
+# CHECK-INST-RELAX:         qc.e.bgei    a0, 0x5d, 0xa2de
 # CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
-   qc.e.blti a0, 3, .L9
+   qc.e.blti a0, 93, .L9
 .fill 1300, 4, 0
 .L9:
    ret
 
-# CHECK-INST:         qc.e.blti    a0, 0x4, 0xb73a
+# CHECK-INST:         qc.e.blti    a0, 0x2c, 0xb73a
 # CHECK-INST-NEXT:    jal     zero, 0xcb8a
-# CHECK-INST-RELAX:         qc.e.blti    a0, 0x4, 0xb73a
+# CHECK-INST-RELAX:         qc.e.blti    a0, 0x2c, 0xb73a
 # CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
-   qc.e.bgei a0, 4, .L10
+   qc.e.bgei a0, 44, .L10
 .fill 1300, 4, 0
 .L10:
    ret
 
-# CHECK-INST:         qc.e.bgeui    a0, 0x5, 0xcb96
+# CHECK-INST:         qc.e.bgeui    a0, 0x37, 0xcb96
 # CHECK-INST-NEXT:    jal     zero, 0xdfe6
-# CHECK-INST-RELAX:         qc.e.bgeui    a0, 0x5, 0xcb96
+# CHECK-INST-RELAX:         qc.e.bgeui    a0, 0x37, 0xcb96
 # CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
-   qc.e.bltui a0, 5, .L11
+   qc.e.bltui a0, 55, .L11
 .fill 1300, 4, 0
 .L11:
    ret
 
-# CHECK-INST:         qc.e.bltui    a0, 0x6, 0xdff2
+# CHECK-INST:         qc.e.bltui    a0, 0x24, 0xdff2
 # CHECK-INST-NEXT:    jal     zero, 0xf442
-# CHECK-INST-RELAX:         qc.e.bltui    a0, 0x6, 0xdff2
+# CHECK-INST-RELAX:         qc.e.bltui    a0, 0x24, 0xdff2
 # CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
-   qc.e.bgeui a0, 6, .L12
+   qc.e.bgeui a0, 36, .L12
 .fill 1300, 4, 0
 .L12:
    ret
 
+# Check that instructions are first compressed and then relaxed
+
+# CHECK-INST:         qc.beqi    a0, 0xa, 0xf44c
+# CHECK-INST-NEXT:    jal     zero, 0x1089c
+# CHECK-INST-RELAX:         qc.beqi    a0, 0xa, 0xf44c
+# CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
+   qc.e.bnei a0, 10, .L13
+.fill 1300, 4, 0
+.L13:
+   ret
+
+# CHECK-INST:         qc.bnei    a0, 0xa, 0x108a6
+# CHECK-INST-NEXT:    jal     zero, 0x11cf6
+# CHECK-INST-RELAX:         qc.bnei    a0, 0xa, 0x108a6
+# CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
+   qc.e.beqi a0, 10, .L14
+.fill 1300, 4, 0
+.L14:
+   ret
+
+# CHECK-INST:         qc.bgei    a0, 0xa, 0x11d00
+# CHECK-INST-NEXT:    jal     zero, 0x13150
+# CHECK-INST-RELAX:         qc.bgei    a0, 0xa, 0x11d00
+# CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
+   qc.e.blti a0, 10, .L15
+.fill 1300, 4, 0
+.L15:
+   ret
+
+# CHECK-INST:         qc.blti    a0, 0xa, 0x1315a
+# CHECK-INST-NEXT:    jal     zero, 0x145aa
+# CHECK-INST-RELAX:         qc.blti    a0, 0xa, 0x1315a
+# CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
+   qc.e.bgei a0, 10, .L16
+.fill 1300, 4, 0
+.L16:
+   ret
+
+# CHECK-INST:         qc.bgeui    a0, 0xa, 0x145b4
+# CHECK-INST-NEXT:    jal     zero, 0x15a04
+# CHECK-INST-RELAX:         qc.bgeui    a0, 0xa, 0x145b4
+# CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
+   qc.e.bltui a0, 10, .L17
+.fill 1300, 4, 0
+.L17:
+   ret
+
+# CHECK-INST:         qc.bltui    a0, 0xa, 0x15a0e
+# CHECK-INST-NEXT:    jal     zero, 0x16e5e
+# CHECK-INST-RELAX:         qc.bltui    a0, 0xa, 0x15a0e
+# CHECK-INST-RELAX-NEXT:    jal     zero, {{.*}}
+   qc.e.bgeui a0, 10, .L18
+.fill 1300, 4, 0
+.L18:
+   ret
+
 .Lfunc_end0:
        .size   test, .Lfunc_end0-test
diff --git a/external/llvm-project/llvm/test/MC/RISCV/xqcibi-relocations.s b/external/llvm-project/llvm/test/MC/RISCV/xqcibi-relocations.s
index 054a890a4b5e..0f7cc8c5787a 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/xqcibi-relocations.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/xqcibi-relocations.s
@@ -1,38 +1,94 @@
-# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcibi %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR %s
-# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcibi %s -o %t.o
-# RUN: llvm-readobj -r %t.o | FileCheck -check-prefix=RELOC %s
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcibi %s \
+# RUN:     | FileCheck -check-prefix=ASM %s
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcibi %s \
+# RUN:     -filetype=obj -o - \
+# RUN:     | llvm-objdump -dr --mattr=+experimental-xqcibi - \
+# RUN:     | FileCheck -check-prefix=OBJ %s
 
-# Check prefixes:
-# RELOC - Check the relocation in the object.
-# INSTR - Check the instruction is handled properly by the ASMPrinter.
+## This test checks that we emit the right relocations for Xqcibi
+## relative branches. These can be resolved within the same section
+## (when relaxations are disabled) but otherwise require a relocation.
+## The QC.E.B<op>I instructions also require a vendor relocation.
 
-.text
+# This is required so that the conditional branches requiring relocations
+# are not converted into inverted branches with long jumps by the assembler.
+.option exact
 
-# Since foo is undefined, this will be relaxed to (qc.beqi + jal)
-qc.bnei x6, 10, foo
-# RELOC: R_RISCV_JAL foo 0x0
-# INSTR: qc.bnei t1, 10, foo
+# ASM-LABEL: this_section:
+# OBJ-LABEL: <this_section>:
+this_section:
 
-# Since foo is undefined, this will be relaxed to (qc.e.bltui + jal)
-qc.e.bgeui x8, 12, foo
-# RELOC: R_RISCV_JAL foo 0x0
-# INSTR: qc.e.bgeui s0, 12, foo
+# ASM: qc.bnei t1, 10, undef
+# OBJ: qc.bnei t1, 0xa, 0x0 <this_section>
+# OBJ-NEXT: R_RISCV_BRANCH undef{{$}}
+qc.bnei t1, 10, undef
 
-# Check that a label in a different section is handled similar to an undefined
-# symbol and gets relaxed to (qc.e.bgeui + jal)
-qc.e.bltui x4, 9, .bar
-# RELOC: R_RISCV_JAL .bar 0x0
-# INSTR: qc.e.bltui tp, 9, .bar
+# ASM: qc.e.bgeui s0, 20, undef
+# OBJ-NEXT: qc.e.bgeui s0, 0x14, 0x4 <this_section+0x4>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM193 undef{{$}}
+qc.e.bgeui s0, 20, undef
 
-# Check that branches to a defined symbol are handled correctly
-qc.e.beqi x7, 8, .L1
-# INSTR: qc.e.beqi t2, 8, .L1
 
-.L1:
-  ret
+# ASM: qc.bnei t2, 11, same_section
+# OBJ-NEXT: qc.bnei t2, 0xb, 0x28 <same_section>
+qc.bnei t2, 11, same_section
 
-.section .t2
+# ASM: qc.e.bgeui s1, 21, same_section
+# OBJ-NEXT: qc.e.bgeui s1, 0x15, 0x28 <same_section>
+qc.e.bgeui s1, 21, same_section
 
-.bar:
-  ret
+
+# ASM: qc.bnei t2, 12, same_section_extern
+# OBJ-NEXT: qc.bnei t2, 0xc, 0x14 <this_section+0x14>
+# OBJ-NEXT: R_RISCV_BRANCH same_section_extern{{$}}
+qc.bnei t2, 12, same_section_extern
+
+# ASM: qc.e.bgeui s1, 22, same_section_extern
+# OBJ-NEXT: qc.e.bgeui s1, 0x16, 0x18 <this_section+0x18>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM193 same_section_extern{{$}}
+qc.e.bgeui s1, 22, same_section_extern
+
+
+# ASM: qc.bnei t3, 13, other_section
+# OBJ-NEXT: qc.bnei t3, 0xd, 0x1e <this_section+0x1e>
+# OBJ-NEXT: R_RISCV_BRANCH other_section{{$}}
+qc.bnei t3, 13, other_section
+
+# ASM: qc.e.bgeui s2, 23, other_section
+# OBJ-NEXT: qc.e.bgeui s2, 0x17, 0x22 <this_section+0x22>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM193 other_section{{$}}
+qc.e.bgeui s2, 23, other_section
+
+
+# ASM-LABEL: same_section:
+# OBJ-LABEL: <same_section>:
+same_section:
+  nop
+
+# ASM-LABEL: same_section_extern:
+# OBJ-LABEL: <same_section_extern>:
+  .global same_section_extern
+same_section_extern:
+  nop
+
+.option relax
+
+# ASM: qc.bnei t3, 14, same_section
+# OBJ: qc.bnei t3, 0xe, 0x28 <same_section>
+qc.bnei t3, 14, same_section
+
+# ASM: qc.e.bgeui s2, 24, same_section
+# OBJ-NEXT: qc.e.bgeui s2, 0x18, 0x28 <same_section>
+qc.e.bgeui s2, 24, same_section
+
+.option norelax
+
+.section .text.second, "ax", @progbits
+
+# ASM-LABEL: other_section:
+# OBJ-LABEL: <other_section>:
+other_section:
+  nop
diff --git a/external/llvm-project/llvm/test/MC/RISCV/xqcibi-valid.s b/external/llvm-project/llvm/test/MC/RISCV/xqcibi-valid.s
index 63d35f3cafb8..88f7813d5091 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/xqcibi-valid.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/xqcibi-valid.s
@@ -1,11 +1,11 @@
 # Xqcibi - Qualcomm uC Branch Immediate Extension
 # RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcibi -M no-aliases -show-encoding \
-# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST,CHECK-NOALIAS %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcibi < %s \
 # RUN:     | llvm-objdump --mattr=+experimental-xqcibi -M no-aliases -d - \
-# RUN:     | FileCheck -check-prefix=CHECK-OBJ %s
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ %s
 # RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqcibi -show-encoding \
-# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST,CHECK-ALIAS %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcibi < %s \
 # RUN:     | llvm-objdump --mattr=+experimental-xqcibi -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-OBJ %s
@@ -40,10 +40,10 @@ qc.bgeui x12, 11, 128
 # CHECK-ENC: encoding: [0x7b,0x6d,0x71,0x28]
 qc.bltui x2, 7, 666
 
-# CHECK-INST:  qc.e.beqi   ra, 1, 2
-# CHECK-OBJ:  qc.e.beqi   ra, 0x1, 0x1a
-# CHECK-ENC: encoding: [0x1f,0xc1,0x80,0x01,0x01,0x00]
-qc.e.beqi x1, 1, 2
+# CHECK-INST:  qc.e.beqi   ra, 111, 2
+# CHECK-OBJ:  qc.e.beqi   ra, 0x6f, 0x1a
+# CHECK-ENC: encoding: [0x1f,0xc1,0x80,0x01,0x6f,0x00]
+qc.e.beqi x1, 111, 2
 
 # CHECK-INST: qc.e.bnei   tp, 115, 4094
 # CHECK-OBJ: qc.e.bnei   tp, 0x73, 0x101c
@@ -65,7 +65,45 @@ qc.e.blti x1, 32767, 2000
 # CHECK-ENC: encoding: [0x1f,0x40,0xf6,0x09,0xc7,0x02]
 qc.e.bgeui x12, 711, 128
 
-# CHECK-INST: qc.e.bltui  sp, 7, 666
-# CHECK-OBJ: qc.e.bltui  sp, 0x7, 0x2d0
-# CHECK-ENC: encoding: [0x1f,0x4d,0xe1,0x29,0x07,0x00]
-qc.e.bltui x2, 7, 666
+# CHECK-INST: qc.e.bltui  sp, 77, 666
+# CHECK-OBJ: qc.e.bltui  sp, 0x4d, 0x2d0
+# CHECK-ENC: encoding: [0x1f,0x4d,0xe1,0x29,0x4d,0x00]
+qc.e.bltui x2, 77, 666
+
+# Check that compress patterns work as expected
+
+# CHECK-NOALIAS:  beqi   ra, 11, 2
+# CHECK-ALIAS: qc.beqi   ra, 11, 2
+# CHECK-OBJ:  qc.beqi   ra, 0xb, 0x3e
+# CHECK-ENC: encoding: [0x7b,0x81,0xb0,0x00]
+qc.e.beqi x1, 11, 2
+
+# CHECK-NOALIAS:  bnei   ra, 11, 2
+# CHECK-ALIAS: qc.bnei   ra, 11, 2
+# CHECK-OBJ:  qc.bnei   ra, 0xb, 0x42
+# CHECK-ENC: encoding: [0x7b,0x91,0xb0,0x00]
+qc.e.bnei x1, 11, 2
+
+# CHECK-NOALIAS:  bgei   ra, 11, 2
+# CHECK-ALIAS: qc.bgei   ra, 11, 2
+# CHECK-OBJ:  qc.bgei   ra, 0xb, 0x46
+# CHECK-ENC: encoding: [0x7b,0xd1,0xb0,0x00]
+qc.e.bgei x1, 11, 2
+
+# CHECK-NOALIAS:  blti   ra, 11, 2
+# CHECK-ALIAS: qc.blti   ra, 11, 2
+# CHECK-OBJ:  qc.blti   ra, 0xb, 0x4a
+# CHECK-ENC: encoding: [0x7b,0xc1,0xb0,0x00]
+qc.e.blti x1, 11, 2
+
+# CHECK-NOALIAS:  bgeui   ra, 11, 2
+# CHECK-ALIAS: qc.bgeui   ra, 11, 2
+# CHECK-OBJ:  qc.bgeui   ra, 0xb, 0x4e
+# CHECK-ENC: encoding: [0x7b,0xf1,0xb0,0x00]
+qc.e.bgeui x1, 11, 2
+
+# CHECK-NOALIAS:  bltui   ra, 11, 2
+# CHECK-ALIAS: qc.bltui   ra, 11, 2
+# CHECK-OBJ:  qc.bltui   ra, 0xb, 0x52
+# CHECK-ENC: encoding: [0x7b,0xe1,0xb0,0x00]
+qc.e.bltui x1, 11, 2
diff --git a/external/llvm-project/llvm/test/MC/RISCV/xqcilb-relocations.s b/external/llvm-project/llvm/test/MC/RISCV/xqcilb-relocations.s
index 0a5b788a110d..ab08beed9be9 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/xqcilb-relocations.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/xqcilb-relocations.s
@@ -1,46 +1,102 @@
-# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcilb %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR %s
-# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcilb %s -o %t.o
-# RUN: llvm-readobj -r %t.o | FileCheck -check-prefix=RELOC %s
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcilb %s \
+# RUN:     | FileCheck -check-prefix=ASM %s
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcilb %s \
+# RUN:     -filetype=obj -o - \
+# RUN:     | llvm-objdump -dr --mattr=+experimental-xqcilb - \
+# RUN:     | FileCheck -check-prefix=OBJ %s
 
-# Check prefixes:
-# RELOC - Check the relocation in the object.
-# INSTR - Check the instruction is handled properly by the ASMPrinter.
-
-.text
+## This test checks that we emit the right relocations for Xqcilb
+## relative jumps. These can be resolved within the same section
+## (when relaxations are disabled) but otherwise require a
+## vendor-specific relocation pair.
 
+# This is required so that the conditional jumps are not compressed
+# by the assembler
 .option exact
 
-qc.e.j foo
-# RELOC: R_RISCV_CUSTOM195 foo 0x0
-# INSTR: qc.e.j foo
+# ASM-LABEL: this_section:
+# OBJ-LABEL: <this_section>:
+this_section:
+
+# ASM: qc.e.j undef
+# OBJ: qc.e.j 0x0 <this_section>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 undef{{$}}
+qc.e.j undef
+
+# ASM: qc.e.jal undef
+# OBJ-NEXT: qc.e.jal 0x6 <this_section+0x6>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 undef{{$}}
+qc.e.jal undef
+
+
+# ASM: qc.e.j same_section
+# OBJ-NEXT: qc.e.j 0x30 <same_section>
+qc.e.j same_section
+
+# ASM: qc.e.jal same_section
+# OBJ-NEXT: qc.e.jal 0x30 <same_section>
+qc.e.jal same_section
+
+# ASM: qc.e.j same_section_extern
+# OBJ-NEXT: qc.e.j 0x18 <this_section+0x18>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 same_section_extern{{$}}
+qc.e.j same_section_extern
+
+# ASM: qc.e.jal same_section_extern
+# OBJ-NEXT: qc.e.jal 0x1e <this_section+0x1e>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 same_section_extern{{$}}
+qc.e.jal same_section_extern
+
+
+# ASM: qc.e.j other_section
+# OBJ-NEXT: qc.e.j 0x24 <this_section+0x24>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 other_section{{$}}
+qc.e.j other_section
+
+# ASM: qc.e.jal other_section
+# OBJ-NEXT: qc.e.jal 0x2a <this_section+0x2a>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 other_section{{$}}
+qc.e.jal other_section
 
-qc.e.jal foo
-# RELOC: R_RISCV_CUSTOM195 foo 0x0
-# INSTR: qc.e.jal foo
 
-# Check that a label in a different section is handled similar to an undefined symbol
-qc.e.j .bar
-# RELOC: R_RISCV_CUSTOM195 .bar 0x0
-# INSTR: qc.e.j .bar
+# ASM-LABEL: same_section:
+# OBJ-LABEL: <same_section>:
+same_section:
+  nop
 
-qc.e.jal .bar
-# RELOC: R_RISCV_CUSTOM195 .bar 0x0
-# INSTR: qc.e.jal .bar
+# ASM-LABEL: same_section_extern:
+# OBJ-LABEL: <same_section_extern>:
+  .global same_section_extern
+same_section_extern:
+  nop
 
-# Check that jumps to a defined symbol are handled correctly
-qc.e.j .L1
-# INSTR:qc.e.j .L1
+.option relax
 
-qc.e.jal .L1
-# INSTR:qc.e.jal .L1
+# ASM: qc.e.j same_section
+# OBJ: qc.e.j 0x38 <same_section_extern+0x4>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 same_section{{$}}
+# OBJ-NEXT: R_RISCV_RELAX
+qc.e.j same_section
 
-.option noexact
+# ASM: qc.e.jal same_section
+# OBJ-NEXT: qc.e.jal 0x3e <same_section_extern+0xa>
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM195 same_section{{$}}
+# OBJ-NEXT: R_RISCV_RELAX
+qc.e.jal same_section
 
-.L1:
-  ret
+.option norelax
 
-.section .t2
+.section .text.other, "ax", @progbits
 
-.bar:
-  ret
+# ASM-LABEL: other_section:
+# OBJ-LABEL: <other_section>:
+other_section:
+  nop
diff --git a/external/llvm-project/llvm/test/MC/RISCV/xqcili-relocations.s b/external/llvm-project/llvm/test/MC/RISCV/xqcili-relocations.s
index 9c709e68334c..866586236fa4 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/xqcili-relocations.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/xqcili-relocations.s
@@ -1,46 +1,107 @@
-# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcili %s -show-encoding \
-# RUN:     | FileCheck -check-prefix=INSTR %s
-# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqcili %s -o %t.o
-# RUN: llvm-readobj -r %t.o | FileCheck -check-prefix=RELOC %s
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcili %s \
+# RUN:     | FileCheck -check-prefix=ASM %s
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcili %s \
+# RUN:     -filetype=obj -o - \
+# RUN:     | llvm-objdump -dr --mattr=+experimental-xqcili - \
+# RUN:     | FileCheck -check-prefix=OBJ %s
 
-# Check prefixes:
-# RELOC - Check the relocation in the object.
-# INSTR - Check the instruction is handled properly by the ASMPrinter.
-
-.text
+## This test checks that we emit the right relocations for Xqcili
+## immediates. These always require a relocation pair, unless the
+## target is absolute.
 
+# This is required so that the conditional branches requiring relocations
+# are not converted into inverted branches with long jumps by the assembler.
 .option exact
 
-qc.li x4, %qc.abs20(foo)
-# RELOC: R_RISCV_CUSTOM192 foo 0x0
-# INSTR: qc.li tp, %qc.abs20(foo)
+.set abs_symbol, 0x0
+
+# ASM-LABEL: this_section:
+# OBJ-LABEL: <this_section>:
+this_section:
+
+# ASM: qc.li a0, %qc.abs20(undef)
+# OBJ: qc.li a0, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM192 undef{{$}}
+qc.li a0, %qc.abs20(undef)
+
+# ASM: qc.e.li s0, undef
+# OBJ-NEXT: qc.e.li s0, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM194 undef{{$}}
+qc.e.li s0, undef
+
+
+# ASM: qc.li a1, %qc.abs20(0)
+# OBJ-NEXT: qc.li a1, 0x0
+qc.li a1, %qc.abs20(abs_symbol)
+
+# ASM: qc.e.li s1, 0
+# OBJ-NEXT: qc.e.li s1, 0x0
+qc.e.li s1, abs_symbol
+
+
+# ASM: qc.li a2, %qc.abs20(same_section)
+# OBJ-NEXT: qc.li a2, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM192 same_section{{$}}
+qc.li a2, %qc.abs20(same_section)
+
+# ASM: qc.e.li s2, same_section
+# OBJ-NEXT: qc.e.li s2, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM194 same_section{{$}}
+qc.e.li s2, same_section
+
+# ASM: qc.li a3, %qc.abs20(other_section)
+# OBJ-NEXT: qc.li a3, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM192 other_section{{$}}
+qc.li a3, %qc.abs20(other_section)
+
+# ASM: qc.e.li s3, other_section
+# OBJ-NEXT: qc.e.li s3, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM194 other_section{{$}}
+qc.e.li s3, other_section
 
-qc.e.li x5, foo
-# RELOC: R_RISCV_CUSTOM194 foo 0x0
-# INSTR: qc.e.li t0, foo
+# ASM-LABEL: same_section:
+# OBJ-LABEL: <same_section>:
+same_section:
+  nop
 
-# Check that a label in a different section is handled similar to an undefined symbol
-qc.li x9, %qc.abs20(.bar)
-# RELOC: R_RISCV_CUSTOM192 .bar 0x0
-# INSTR: qc.li s1, %qc.abs20(.bar)
+.option relax
 
-qc.e.li x8, .bar
-# RELOC: R_RISCV_CUSTOM194 .bar 0x0
-# INSTR: qc.e.li s0, .bar
+# ASM: qc.li a1, %qc.abs20(0)
+# OBJ: qc.li a1, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM192 *ABS*{{$}}
+# OBJ-NEXT: R_RISCV_RELAX
+qc.li a1, %qc.abs20(abs_symbol)
 
-# Check that branches to a defined symbol are handled correctly
-qc.li x7, %qc.abs20(.L1)
-# INSTR: qc.li t2, %qc.abs20(.L1)
+# ASM: qc.e.li s1, 0
+# OBJ-NEXT: qc.e.li s1, 0x0
+qc.e.li s1, abs_symbol
 
-qc.e.li x6, .L1
-# INSTR: qc.e.li t1, .L1
+# ASM: qc.li a1, %qc.abs20(undef)
+# OBJ-NEXT: qc.li a1, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM192 undef{{$}}
+# OBJ-NEXT: R_RISCV_RELAX
+qc.li a1, %qc.abs20(undef)
 
-.option noexact
+# ASM: qc.e.li s1, undef
+# OBJ-NEXT: qc.e.li s1, 0x0
+# OBJ-NEXT: R_RISCV_VENDOR QUALCOMM{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM194 undef{{$}}
+# OBJ-NEXT: R_RISCV_RELAX
+qc.e.li s1, undef
 
-.L1:
-  ret
+.option norelax
 
-.section .t2
+.section .text.other, "ax", @progbits
 
-.bar:
-  ret
+# ASM-LABEL: other_section:
+# OBJ-LABEL: <other_section>:
+other_section:
+  nop
diff --git a/external/llvm-project/llvm/test/MC/RISCV/xqcilsm-invalid.s b/external/llvm-project/llvm/test/MC/RISCV/xqcilsm-invalid.s
index 15d55021d64e..a3421db0eff4 100644
--- a/external/llvm-project/llvm/test/MC/RISCV/xqcilsm-invalid.s
+++ b/external/llvm-project/llvm/test/MC/RISCV/xqcilsm-invalid.s
@@ -7,6 +7,10 @@
 # CHECK: :[[@LINE+1]]:20: error: expected register
 qc.swm x5, x20, 12(20)
 
+# CHECK-PLUS: :[[@LINE+2]]:8: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:8: error: invalid operand for instruction
+qc.swm x0, x20, 12(x3)
+
 # CHECK-PLUS: :[[@LINE+2]]:12: error: register must be a GPR excluding zero (x0)
 # CHECK-MINUS: :[[@LINE+1]]:12: error: invalid operand for instruction
 qc.swm x5, x0, 12(x3)
@@ -24,6 +28,10 @@ qc.swm x5, x20, 12(x3)
 # CHECK: :[[@LINE+1]]:20: error: expected register
 qc.swmi x10, 4, 20(4)
 
+# CHECK-PLUS: :[[@LINE+2]]:9: error: register must be a GPR excluding zero (x0)
+# CHECK-MINUS: :[[@LINE+1]]:9: error: invalid operand for instruction
+qc.swmi x0, 4, 20(x4)
+
 # CHECK: :[[@LINE+1]]:1: error: too few operands for instruction
 qc.swmi x10, 4, 20
 
diff --git a/external/llvm-project/llvm/test/MC/Sparc/sparc-vis2.s b/external/llvm-project/llvm/test/MC/Sparc/sparc-vis2.s
index 331888438856..077acd774763 100644
--- a/external/llvm-project/llvm/test/MC/Sparc/sparc-vis2.s
+++ b/external/llvm-project/llvm/test/MC/Sparc/sparc-vis2.s
@@ -3,53 +3,53 @@
 
 !! VIS 2 instructions.
 
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: bmask %o0, %o1, %o2                     ! encoding: [0x95,0xb2,0x03,0x29]
 bmask %o0, %o1, %o2
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: bshuffle %f0, %f2, %f4                  ! encoding: [0x89,0xb0,0x09,0x82]
 bshuffle %f0, %f2, %f4
 
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 0                                  ! encoding: [0x81,0xb0,0x10,0x20]
 siam 0
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 1                                  ! encoding: [0x81,0xb0,0x10,0x21]
 siam 1
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 2                                  ! encoding: [0x81,0xb0,0x10,0x22]
 siam 2
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 3                                  ! encoding: [0x81,0xb0,0x10,0x23]
 siam 3
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 4                                  ! encoding: [0x81,0xb0,0x10,0x24]
 siam 4
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 5                                  ! encoding: [0x81,0xb0,0x10,0x25]
 siam 5
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 6                                  ! encoding: [0x81,0xb0,0x10,0x26]
 siam 6
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: siam 7                                  ! encoding: [0x81,0xb0,0x10,0x27]
 siam 7
 
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: edge8n %o0, %o1, %o2                    ! encoding: [0x95,0xb2,0x00,0x29]
 edge8n %o0, %o1, %o2
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: edge8ln %o0, %o1, %o2                   ! encoding: [0x95,0xb2,0x00,0x69]
 edge8ln %o0, %o1, %o2
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: edge16n %o0, %o1, %o2                   ! encoding: [0x95,0xb2,0x00,0xa9]
 edge16n %o0, %o1, %o2
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: edge16ln %o0, %o1, %o2                  ! encoding: [0x95,0xb2,0x00,0xe9]
 edge16ln %o0, %o1, %o2
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: edge32n %o0, %o1, %o2                   ! encoding: [0x95,0xb2,0x01,0x29]
 edge32n %o0, %o1, %o2
-! NO-VIS2: error: instruction requires a CPU feature not currently enabled
+! NO-VIS2: [[#@LINE+2]]:1: error: instruction requires a CPU feature not currently enabled
 ! VIS2: edge32ln %o0, %o1, %o2                  ! encoding: [0x95,0xb2,0x01,0x69]
 edge32ln %o0, %o1, %o2
diff --git a/external/llvm-project/llvm/test/MC/Sparc/sparcv9-instructions.s b/external/llvm-project/llvm/test/MC/Sparc/sparcv9-instructions.s
index 1c174f399ed3..de9fe03201b1 100644
--- a/external/llvm-project/llvm/test/MC/Sparc/sparcv9-instructions.s
+++ b/external/llvm-project/llvm/test/MC/Sparc/sparcv9-instructions.s
@@ -1,69 +1,55 @@
 ! RUN: not llvm-mc %s -triple=sparc   -show-encoding 2>&1 | FileCheck %s --check-prefix=V8
 ! RUN: llvm-mc %s -triple=sparcv9 -show-encoding | FileCheck %s --check-prefix=V9
 
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: addc %g2, %g1, %g3
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9:      addx %g2, %g1, %g3              ! encoding: [0x86,0x40,0x80,0x01]
         addc %g2, %g1, %g3
 
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: addc %g2, 1, %g3
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9:      addx %g2, 1, %g3              ! encoding: [0x86,0x40,0xa0,0x01]
         addc %g2, 1, %g3
 
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: addc 1, %g2, %g3
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9:      addx %g2, 1, %g3              ! encoding: [0x86,0x40,0xa0,0x01]
         addc 1, %g2, %g3
 
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: addccc %g1, %g2, %g3
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9:      addxcc %g1, %g2, %g3            ! encoding: [0x86,0xc0,0x40,0x02]
         addccc %g1, %g2, %g3
 
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: subc %g2, %g1, %g3
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9:      subx %g2, %g1, %g3          ! encoding: [0x86,0x60,0x80,0x01]
         subc %g2, %g1, %g3
 
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: subccc %g1, %g2, %g3
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9:      subxcc %g1, %g2, %g3         ! encoding: [0x86,0xe0,0x40,0x02]
         subccc %g1, %g2, %g3
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: popc %g1, %g2
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9:      popc %g1, %g2                ! encoding: [0x85,0x70,0x00,0x01]
         popc %g1, %g2
 
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: signx %g1, %g2
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: sra %g1, %g0, %g2               ! encoding: [0x85,0x38,0x40,0x00]
         signx %g1, %g2
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: signx %g1
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: sra %g1, %g0, %g1               ! encoding: [0x83,0x38,0x40,0x00]
         signx %g1
 
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: lduw [%i0 + %l6], %o2
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: ld [%i0+%l6], %o2    ! encoding: [0xd4,0x06,0x00,0x16]
         lduw [%i0 + %l6], %o2
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: lduw [%i0 + 32], %o2
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: ld [%i0+32], %o2     ! encoding: [0xd4,0x06,0x20,0x20]
         lduw [%i0 + 32], %o2
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: lduw [%g1], %o2
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: ld [%g1], %o2        ! encoding: [0xd4,0x00,0x40,0x00]
         lduw [%g1], %o2
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: lduwa [%i0 + %l6] 131, %o2
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: lda [%i0+%l6] #ASI_SNF, %o2 ! encoding: [0xd4,0x86,0x10,0x76]
         lduwa [%i0 + %l6] 131, %o2
-        ! V8:      error: invalid instruction mnemonic
-        ! V8-NEXT: lduwa [%i0 + %l6] (130+1), %o2
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: lda [%i0+%l6] #ASI_SNF, %o2 ! encoding: [0xd4,0x86,0x10,0x76]
         lduwa [%i0 + %l6] (130+1), %o2
 
@@ -78,76 +64,60 @@
         ! V9: ldswa [%i0+%l6] #ASI_SNF, %o2 ! encoding: [0xd4,0xc6,0x10,0x76]
         ldswa [%i0 + %l6] (130+1), %o2
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: lda [%l0] 0xf0, %f29
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: lda [%l0] 240, %f29             ! encoding: [0xfb,0x84,0x1e,0x00]
         lda [%l0] 0xf0, %f29
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: lda [%l0] (0xef+0x01), %f29
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: lda [%l0] 240, %f29             ! encoding: [0xfb,0x84,0x1e,0x00]
         lda [%l0] (0xef+0x01), %f29
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: ldda [%l0] 0xf0, %f48
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldda [%l0] 240, %f48            ! encoding: [0xe3,0x9c,0x1e,0x00]
         ldda [%l0] 0xf0, %f48
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: ldda [%l0] (0xef+0x01), %f48
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldda [%l0] 240, %f48            ! encoding: [0xe3,0x9c,0x1e,0x00]
         ldda [%l0] (0xef+0x01), %f48
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: ldqa [%l0] 0xf0, %f48
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldqa [%l0] 240, %f48            ! encoding: [0xe3,0x94,0x1e,0x00]
         ldqa [%l0] 0xf0, %f48
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: ldqa [%l0] (0xef+0x01), %f48
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldqa [%l0] 240, %f48            ! encoding: [0xe3,0x94,0x1e,0x00]
         ldqa [%l0] (0xef+0x01), %f48
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: ldq [%l0], %f48
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldq [%l0], %f48                 ! encoding: [0xe3,0x14,0x00,0x00]
         ldq [%l0], %f48
 
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: sta %f29, [%l0] 0xf0
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: sta %f29, [%l0] 240             ! encoding: [0xfb,0xa4,0x1e,0x00]
         sta %f29, [%l0] 0xf0
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: sta %f29, [%l0] (0xef+0x01)
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: sta %f29, [%l0] 240             ! encoding: [0xfb,0xa4,0x1e,0x00]
         sta %f29, [%l0] (0xef+0x01)
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: stda %f48, [%l0] 0xf0
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stda %f48, [%l0] 240            ! encoding: [0xe3,0xbc,0x1e,0x00]
         stda %f48, [%l0] 0xf0
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: stda %f48, [%l0] (0xef+0x01)
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stda %f48, [%l0] 240            ! encoding: [0xe3,0xbc,0x1e,0x00]
         stda %f48, [%l0] (0xef+0x01)
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: stqa %f48, [%l0] 0xf0
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stqa %f48, [%l0] 240            ! encoding: [0xe3,0xb4,0x1e,0x00]
         stqa %f48, [%l0] 0xf0
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: stqa %f48, [%l0] (0xef+0x01)
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stqa %f48, [%l0] 240            ! encoding: [0xe3,0xb4,0x1e,0x00]
         stqa %f48, [%l0] (0xef+0x01)
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: stq %f48, [%l0]
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stq %f48, [%l0]                 ! encoding: [0xe3,0x34,0x00,0x00]
         stq %f48, [%l0]
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: ldx [%g2 + 20],%fsr
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldx [%g2+20], %fsr    ! encoding: [0xc3,0x08,0xa0,0x14]
         ldx [%g2 + 20],%fsr
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: ldx [%g2 + %i5],%fsr
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldx [%g2+%i5], %fsr   ! encoding: [0xc3,0x08,0x80,0x1d]
         ldx [%g2 + %i5],%fsr
 
@@ -156,13 +126,11 @@
         ! V9: ldxa [%g2+%i5] #ASI_SNF, %g0   ! encoding: [0xc0,0xd8,0x90,0x7d]
         ldxa [%g2 + %i5] (130+1), %g0
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: stx %fsr,[%g2 + 20]
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stx %fsr, [%g2+20]    ! encoding: [0xc3,0x28,0xa0,0x14]
         stx %fsr,[%g2 + 20]
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: stx %fsr,[%g2 + %i5]
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stx %fsr, [%g2+%i5]   ! encoding: [0xc3,0x28,0x80,0x1d]
         stx %fsr,[%g2 + %i5]
 
@@ -171,317 +139,240 @@
         ! V9: stxa %g0, [%g2+%i5] #ASI_SNF   ! encoding: [0xc0,0xf0,0x90,0x7d]
         stxa %g0, [%g2 + %i5] (130+1)
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%tpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %tpc        ! encoding: [0x81,0x91,0x80,0x1e]
         wrpr %g6,%i6,%tpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%tnpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %tnpc       ! encoding: [0x83,0x91,0x80,0x1e]
         wrpr %g6,%i6,%tnpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%tstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %tstate     ! encoding: [0x85,0x91,0x80,0x1e]
         wrpr %g6,%i6,%tstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%tt
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %tt         ! encoding: [0x87,0x91,0x80,0x1e]
         wrpr %g6,%i6,%tt
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%tick
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %tick       ! encoding: [0x89,0x91,0x80,0x1e]
         wrpr %g6,%i6,%tick
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%tba
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %tba        ! encoding: [0x8b,0x91,0x80,0x1e]
         wrpr %g6,%i6,%tba
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%pstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %pstate     ! encoding: [0x8d,0x91,0x80,0x1e]
         wrpr %g6,%i6,%pstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%tl
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %tl         ! encoding: [0x8f,0x91,0x80,0x1e]
         wrpr %g6,%i6,%tl
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%pil
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %pil        ! encoding: [0x91,0x91,0x80,0x1e]
         wrpr %g6,%i6,%pil
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%cwp
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %cwp        ! encoding: [0x93,0x91,0x80,0x1e]
         wrpr %g6,%i6,%cwp
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%cansave
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %cansave    ! encoding: [0x95,0x91,0x80,0x1e]
         wrpr %g6,%i6,%cansave
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%canrestore
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %canrestore ! encoding: [0x97,0x91,0x80,0x1e]
         wrpr %g6,%i6,%canrestore
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%cleanwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %cleanwin   ! encoding: [0x99,0x91,0x80,0x1e]
         wrpr %g6,%i6,%cleanwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%otherwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %otherwin   ! encoding: [0x9b,0x91,0x80,0x1e]
         wrpr %g6,%i6,%otherwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,%i6,%wstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, %fp, %wstate     ! encoding: [0x9d,0x91,0x80,0x1e]
         wrpr %g6,%i6,%wstate
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%tpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %tpc        ! encoding: [0x81,0x91,0xa0,0xff]
         wrpr %g6,255,%tpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%tnpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %tnpc       ! encoding: [0x83,0x91,0xa0,0xff]
         wrpr %g6,255,%tnpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%tstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %tstate     ! encoding: [0x85,0x91,0xa0,0xff]
         wrpr %g6,255,%tstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%tt
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %tt         ! encoding: [0x87,0x91,0xa0,0xff]
         wrpr %g6,255,%tt
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%tick
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %tick       ! encoding: [0x89,0x91,0xa0,0xff]
         wrpr %g6,255,%tick
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%tba
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %tba        ! encoding: [0x8b,0x91,0xa0,0xff]
         wrpr %g6,255,%tba
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%pstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %pstate     ! encoding: [0x8d,0x91,0xa0,0xff]
         wrpr %g6,255,%pstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%tl
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %tl         ! encoding: [0x8f,0x91,0xa0,0xff]
         wrpr %g6,255,%tl
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%pil
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %pil        ! encoding: [0x91,0x91,0xa0,0xff]
         wrpr %g6,255,%pil
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%cwp
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %cwp        ! encoding: [0x93,0x91,0xa0,0xff]
         wrpr %g6,255,%cwp
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%cansave
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %cansave    ! encoding: [0x95,0x91,0xa0,0xff]
         wrpr %g6,255,%cansave
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%canrestore
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %canrestore ! encoding: [0x97,0x91,0xa0,0xff]
         wrpr %g6,255,%canrestore
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%cleanwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %cleanwin   ! encoding: [0x99,0x91,0xa0,0xff]
         wrpr %g6,255,%cleanwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%otherwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %otherwin   ! encoding: [0x9b,0x91,0xa0,0xff]
         wrpr %g6,255,%otherwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %g6,255,%wstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g6, 255, %wstate     ! encoding: [0x9d,0x91,0xa0,0xff]
         wrpr %g6,255,%wstate
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%tpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %tpc        ! encoding: [0x81,0x97,0x80,0x00]
         wrpr %i6,%tpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%tnpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %tnpc       ! encoding: [0x83,0x97,0x80,0x00]
         wrpr %i6,%tnpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%tstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %tstate     ! encoding: [0x85,0x97,0x80,0x00]
         wrpr %i6,%tstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%tt
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %tt         ! encoding: [0x87,0x97,0x80,0x00]
         wrpr %i6,%tt
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%tick
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %tick       ! encoding: [0x89,0x97,0x80,0x00]
         wrpr %i6,%tick
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%tba
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %tba        ! encoding: [0x8b,0x97,0x80,0x00]
         wrpr %i6,%tba
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%pstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %pstate     ! encoding: [0x8d,0x97,0x80,0x00]
         wrpr %i6,%pstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%tl
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %tl         ! encoding: [0x8f,0x97,0x80,0x00]
         wrpr %i6,%tl
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%pil
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %pil        ! encoding: [0x91,0x97,0x80,0x00]
         wrpr %i6,%pil
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%cwp
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %cwp        ! encoding: [0x93,0x97,0x80,0x00]
         wrpr %i6,%cwp
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%cansave
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %cansave    ! encoding: [0x95,0x97,0x80,0x00]
         wrpr %i6,%cansave
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%canrestore
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %canrestore ! encoding: [0x97,0x97,0x80,0x00]
         wrpr %i6,%canrestore
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%cleanwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %cleanwin   ! encoding: [0x99,0x97,0x80,0x00]
         wrpr %i6,%cleanwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%otherwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %otherwin   ! encoding: [0x9b,0x97,0x80,0x00]
         wrpr %i6,%otherwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr %i6,%wstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %fp, %g0, %wstate     ! encoding: [0x9d,0x97,0x80,0x00]
         wrpr %i6,%wstate
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%tpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %tpc        ! encoding: [0x81,0x90,0x20,0xff]
         wrpr 255,%tpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%tnpc
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %tnpc       ! encoding: [0x83,0x90,0x20,0xff]
         wrpr 255,%tnpc
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%tstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %tstate     ! encoding: [0x85,0x90,0x20,0xff]
         wrpr 255,%tstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%tt
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %tt         ! encoding: [0x87,0x90,0x20,0xff]
         wrpr 255,%tt
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%tick
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %tick       ! encoding: [0x89,0x90,0x20,0xff]
         wrpr 255,%tick
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%tba
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %tba        ! encoding: [0x8b,0x90,0x20,0xff]
         wrpr 255,%tba
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%pstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %pstate     ! encoding: [0x8d,0x90,0x20,0xff]
         wrpr 255,%pstate
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%tl
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %tl         ! encoding: [0x8f,0x90,0x20,0xff]
         wrpr 255,%tl
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%pil
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %pil        ! encoding: [0x91,0x90,0x20,0xff]
         wrpr 255,%pil
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%cwp
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %cwp        ! encoding: [0x93,0x90,0x20,0xff]
         wrpr 255,%cwp
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%cansave
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %cansave    ! encoding: [0x95,0x90,0x20,0xff]
         wrpr 255,%cansave
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%canrestore
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %canrestore ! encoding: [0x97,0x90,0x20,0xff]
         wrpr 255,%canrestore
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%cleanwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %cleanwin   ! encoding: [0x99,0x90,0x20,0xff]
         wrpr 255,%cleanwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%otherwin
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %otherwin   ! encoding: [0x9b,0x90,0x20,0xff]
         wrpr 255,%otherwin
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: wrpr 255,%wstate
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: wrpr %g0, 255, %wstate     ! encoding: [0x9d,0x90,0x20,0xff]
         wrpr 255,%wstate
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %tpc,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %tpc, %i5            ! encoding: [0xbb,0x50,0x00,0x00]
         rdpr %tpc,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %tnpc,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %tnpc, %i5           ! encoding: [0xbb,0x50,0x40,0x00]
         rdpr %tnpc,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %tstate,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %tstate, %i5         ! encoding: [0xbb,0x50,0x80,0x00]
         rdpr %tstate,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %tt,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %tt, %i5             ! encoding: [0xbb,0x50,0xc0,0x00]
         rdpr %tt,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %tick,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %tick, %i5           ! encoding: [0xbb,0x51,0x00,0x00]
         rdpr %tick,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %tba,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %tba, %i5            ! encoding: [0xbb,0x51,0x40,0x00]
         rdpr %tba,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %pstate,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %pstate, %i5         ! encoding: [0xbb,0x51,0x80,0x00]
         rdpr %pstate,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %tl,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %tl, %i5             ! encoding: [0xbb,0x51,0xc0,0x00]
         rdpr %tl,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %pil,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %pil, %i5            ! encoding: [0xbb,0x52,0x00,0x00]
         rdpr %pil,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %cwp,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %cwp, %i5            ! encoding: [0xbb,0x52,0x40,0x00]
         rdpr %cwp,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %cansave,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %cansave, %i5        ! encoding: [0xbb,0x52,0x80,0x00]
         rdpr %cansave,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %canrestore,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %canrestore, %i5     ! encoding: [0xbb,0x52,0xc0,0x00]
         rdpr %canrestore,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %cleanwin,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %cleanwin, %i5       ! encoding: [0xbb,0x53,0x00,0x00]
         rdpr %cleanwin,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %otherwin,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %otherwin, %i5       ! encoding: [0xbb,0x53,0x40,0x00]
         rdpr %otherwin,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %wstate,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %wstate, %i5         ! encoding: [0xbb,0x53,0x80,0x00]
         rdpr %wstate,%i5
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %fq,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %fq, %i5             ! encoding: [0xbb,0x53,0xc0,0x00]
         rdpr %fq,%i5
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: rdpr %ver,%i5
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: rdpr %ver, %i5            ! encoding: [0xbb,0x57,0xc0,0x00]
         rdpr %ver,%i5
 
@@ -537,159 +428,127 @@
         ! V9: stxa %g0, [%g2+%i5] #ASI_SNF   ! encoding: [0xc0,0xf0,0x90,0x7d]
         stxa %g0, [%g2 + %i5] #ASI_SNF
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], 1
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #one_read  ! encoding: [0xc3,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], 1
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_reads
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #n_reads  ! encoding: [0xc1,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #n_reads
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_read
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #one_read  ! encoding: [0xc3,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #one_read
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_writes
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #n_writes  ! encoding: [0xc5,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #n_writes
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_write
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #one_write  ! encoding: [0xc7,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #one_write
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #page
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #page  ! encoding: [0xc9,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #page
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #unified
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #unified  ! encoding: [0xe3,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #unified
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_reads_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #n_reads_strong  ! encoding: [0xe9,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #n_reads_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_read_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #one_read_strong  ! encoding: [0xeb,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #one_read_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #n_writes_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #n_writes_strong  ! encoding: [0xed,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #n_writes_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + 0xf80 ], #one_write_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+3968], #one_write_strong  ! encoding: [0xef,0x6e,0x6f,0x80]
         prefetch  [ %i1 + 0xf80 ], #one_write_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], 1
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #one_read  ! encoding: [0xc3,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], 1
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_reads
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #n_reads  ! encoding: [0xc1,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #n_reads
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_read
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #one_read  ! encoding: [0xc3,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #one_read
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_writes
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #n_writes  ! encoding: [0xc5,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #n_writes
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_write
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #one_write  ! encoding: [0xc7,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #one_write
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #page
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #page  ! encoding: [0xc9,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #page
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #unified
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #unified  ! encoding: [0xe3,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #unified
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_reads_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #n_reads_strong  ! encoding: [0xe9,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #n_reads_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_read_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #one_read_strong  ! encoding: [0xeb,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #one_read_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #n_writes_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #n_writes_strong  ! encoding: [0xed,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #n_writes_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetch  [ %i1 + %i2 ], #one_write_strong
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetch  [%i1+%i2], #one_write_strong  ! encoding: [0xef,0x6e,0x40,0x1a]
         prefetch  [ %i1 + %i2 ], #one_write_strong
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetcha  [ %i1 + 0xf80 ] %asi, 1
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetcha [%i1+3968] %asi, #one_read    ! encoding: [0xc3,0xee,0x6f,0x80]
         prefetcha  [ %i1 + 0xf80 ] %asi, 1
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetcha  [ %i1 + 0xf80 ] %asi, #one_read
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetcha [%i1+3968] %asi, #one_read    ! encoding: [0xc3,0xee,0x6f,0x80]
         prefetcha  [ %i1 + 0xf80 ] %asi, #one_read
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] #ASI_SNF, 1
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
         prefetcha  [ %i1 + %i2 ] #ASI_SNF, 1
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] #ASI_SNF, #one_read
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
         prefetcha  [ %i1 + %i2 ] #ASI_SNF, #one_read
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] 131, 1
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
         prefetcha  [ %i1 + %i2 ] 131, 1
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: prefetcha  [ %i1 + %i2 ] 131, #one_read
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: prefetcha [%i1+%i2] #ASI_SNF, #one_read ! encoding: [0xc3,0xee,0x50,0x7a]
         prefetcha  [ %i1 + %i2 ] 131, #one_read
 
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: done
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: done      ! encoding: [0x81,0xf0,0x00,0x00]
         done
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: retry
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: retry     ! encoding: [0x83,0xf0,0x00,0x00]
         retry
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: saved
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: saved     ! encoding: [0x81,0x88,0x00,0x00]
         saved
-        ! V8:      error: instruction requires a CPU feature not currently enabled
-        ! V8-NEXT: restored
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: restored  ! encoding: [0x83,0x88,0x00,0x00]
         restored
diff --git a/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll b/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll
index cc6bf763b2bc..9ac6f5f63d7d 100644
--- a/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll
+++ b/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll
@@ -158,6 +158,7 @@
 ; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
+; CHECK-O-NEXT: Running pass: SimplifyTypeTestsPass
 ; CHECK-O-NEXT: Running pass: CoroCleanupPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: GlobalDCEPass
diff --git a/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
index 9c4c75cf0c32..56bed3e0d149 100644
--- a/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
+++ b/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
@@ -143,6 +143,7 @@
 ; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
+; CHECK-O-NEXT: Running pass: SimplifyTypeTestsPass
 ; CHECK-O-NEXT: Running pass: CoroCleanupPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
diff --git a/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
index 10d14ca4e9d3..8bfd32972992 100644
--- a/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
+++ b/external/llvm-project/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
@@ -152,6 +152,7 @@
 ; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
+; CHECK-O-NEXT: Running pass: SimplifyTypeTestsPass
 ; CHECK-O-NEXT: Running pass: CoroCleanupPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
diff --git a/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td b/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
index 570adbb63261..23e22b8b8418 100644
--- a/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
+++ b/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
@@ -82,6 +82,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
 // CHECK-NEXT:    bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat &Imm) const override;
 // CHECK-NEXT:    const uint8_t *getMatchTable() const override;
 // CHECK-NEXT:    bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI, const MatcherState &State) const override;
+// CHECK-NEXT:    bool testMOPredicate_MO(unsigned PredicateID, const MachineOperand &MO, const MatcherState &State) const override;
 // CHECK-NEXT:    bool testSimplePredicate(unsigned PredicateID) const override;
 // CHECK-NEXT:    bool runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
 // CHECK-NEXT:  #endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL
@@ -157,6 +158,26 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
 // CHECK-NEXT:   GICXXPred_MI_Predicate_frag = GICXXPred_Invalid + 1,
 // CHECK-NEXT:  };
 
+// CHECK-LABEL: // PatFrag predicates.
+// CHECK-NEXT:  enum {
+// CHECK-NEXT:   GICXXPred_MO_Predicate_leaf = GICXXPred_Invalid + 1,
+// CHECK-NEXT:  };
+
+// CHECK-LABEL:  bool MyTargetInstructionSelector::testMOPredicate_MO(unsigned PredicateID, const MachineOperand & MO, const MatcherState &State) const {
+// CHECK-NEXT:    const auto &Operands = State.RecordedOperands; 
+// CHECK-NEXT:    Register Reg = MO.getReg();
+// CHECK-NEXT:    (void)Operands; 
+// CHECK-NEXT:    (void)Reg;
+// CHECK-NEXT:    switch (PredicateID) {
+// CHECK-NEXT:    case GICXXPred_MO_Predicate_leaf: {
+// CHECK-NEXT:       return true;
+// CHECK-NEXT:    }
+// CHECK-NEXT:    }
+// CHECK-NEXT:    llvm_unreachable("Unknown predicate");
+// CHECK-NEXT:    return false;
+// CHECK-NEXT:  }
+
+
 // CHECK-LABEL: // PatFrag predicates.
 // CHECK-NEXT:  enum {
 // CHECK-NEXT:    GICXXPred_I64_Predicate_cimm8 = GICXXPred_Invalid + 1,
@@ -508,12 +529,12 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
 // R00C-NEXT:    GIR_EraseRootFromParent_Done,
 // R00C-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
 //
-// R00O-NEXT:  GIM_Reject,
+// R00O:       GIM_Reject,
 // R00O-NEXT:  // Label [[GROUP_NUM]]: @[[GROUP]]
 // R00O-NEXT:  GIM_Reject,
 // R00O:       // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
 // R00O-NEXT:  GIM_Reject,
-// R00O-NEXT:  }; // Size: 1856 bytes
+// R00O-NEXT:  }; // Size: 1878 bytes
 
 def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
                  [(set GPR32:$dst,
@@ -828,7 +849,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>;
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/1, // src3
 // NOOPT-NEXT:    GIR_RootConstrainSelectedInstOperands,
-// NOOPT-NEXT:    // GIR_Coverage, 28,
+// NOOPT-NEXT:    // GIR_Coverage, 29,
 // NOOPT-NEXT:    GIR_EraseRootFromParent_Done,
 // NOOPT-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
 
@@ -837,6 +858,35 @@ def MULADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3),
                      (mul (add GPR32:$src1, GPR32:$src2), GPR32:$src3))]>,
              Requires<[HasA]>;
 
+//===- Test a simple pattern with a PatLeaf and a predicate. ---------===//
+//
+// NOOPT-NEXT:     /*  882 */ GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(924), // Rule ID 24 //
+// NOOPT-NEXT:     /*  887 */   GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// NOOPT-NEXT:     /*  890 */   GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SUB),
+// NOOPT-NEXT:     /*  894 */   // MIs[0] DstI[dst]
+// NOOPT-NEXT:     /*  894 */   GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32,
+// NOOPT-NEXT:     /*  897 */   GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// NOOPT-NEXT:     /*  901 */   // MIs[0] src1
+// NOOPT-NEXT:     /*  901 */   GIM_RootCheckType, /*Op*/1, /*Type*/GILLT_s32,
+// NOOPT-NEXT:     /*  904 */   GIM_CheckLeafOperandPredicate, /*MI*/0, /*MO*/1, /*Predicate*/GIMT_Encode2(GICXXPred_MO_Predicate_leaf),
+// NOOPT-NEXT:     /*  909 */   // MIs[0] src2
+// NOOPT-NEXT:     /*  909 */   GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
+// NOOPT-NEXT:     /*  912 */   GIM_CheckLeafOperandPredicate, /*MI*/0, /*MO*/2, /*Predicate*/GIMT_Encode2(GICXXPred_MO_Predicate_leaf),
+// NOOPT-NEXT:     /*  917 */   // (sub:{ *:[i32] } GPR32:{ *:[i32] }<<P:Predicate_leaf>>:$src1, GPR32:{ *:[i32] }<<P:Predicate_leaf>>:$src2)  =>  (INSN5:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)
+// NOOPT-NEXT:     /*  917 */   GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INSN5),
+// NOOPT-NEXT:     /*  922 */   GIR_RootConstrainSelectedInstOperands,
+// NOOPT-NEXT:     /*  923 */   // GIR_Coverage, 24,
+// NOOPT-NEXT:     /*  923 */   GIR_Done,
+// NOOPT-NEXT:     /*  924 */ // Label 13: @924
+
+def leaf: PatLeaf<(i32 GPR32:$src), [{ return true; // C++ code }]> {
+  let GISelLeafPredicateCode = [{ return true; }];
+}
+def INSN5 : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), []>;
+def : Pat<(sub leaf:$src1, leaf:$src2), (INSN5 GPR32:$src1, GPR32:$src2)>;
+
+
+
 //===- Test a simple pattern with just a specific leaf immediate. ---------===//
 //
 // NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
@@ -984,7 +1034,7 @@ def LOAD : I<(outs GPR32:$dst), (ins GPR32:$src1),
 // NOOPT-NEXT:    // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src)
 // NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD),
 // NOOPT-NEXT:    GIR_RootConstrainSelectedInstOperands,
-// NOOPT-NEXT:    // GIR_Coverage, 24,
+// NOOPT-NEXT:    // GIR_Coverage, 25,
 // NOOPT-NEXT:    GIR_Done,
 // NOOPT-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
 
@@ -1083,7 +1133,7 @@ def DOUBLE : I<(outs GPR32:$dst), (ins GPR32:$src), [(set GPR32:$dst, (add GPR32
 // NOOPT-NEXT:    // (add:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername)  =>  (InsnWithSpeciallyNamedDef:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername)
 // NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::InsnWithSpeciallyNamedDef),
 // NOOPT-NEXT:    GIR_RootConstrainSelectedInstOperands,
-// NOOPT-NEXT:    // GIR_Coverage, 25,
+// NOOPT-NEXT:    // GIR_Coverage, 26,
 // NOOPT-NEXT:    GIR_Done,
 // NOOPT-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
 
@@ -1106,7 +1156,7 @@ def : Pat<(add i32:$samename, i32:$othername),
 // NOOPT-NEXT:    // (add:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
 // NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ADD),
 // NOOPT-NEXT:    GIR_RootConstrainSelectedInstOperands,
-// NOOPT-NEXT:    // GIR_Coverage, 26,
+// NOOPT-NEXT:    // GIR_Coverage, 27,
 // NOOPT-NEXT:    GIR_Done,
 // NOOPT-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
 
@@ -1157,7 +1207,7 @@ def MUL : I<(outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src1),
 // NOOPT-NEXT:    // (bitconvert:{ *:[i32] } FPR32:{ *:[f32] }:$src1) => (COPY_TO_REGCLASS:{ *:[i32] } FPR32:{ *:[f32] }:$src1, GPR32:{ *:[i32] })
 // NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // NOOPT-NEXT:    GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(MyTarget::GPR32RegClassID),
-// NOOPT-NEXT:    // GIR_Coverage, 27,
+// NOOPT-NEXT:    // GIR_Coverage, 28,
 // NOOPT-NEXT:    GIR_Done,
 // NOOPT-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
 
@@ -1206,5 +1256,5 @@ def BR : I<(outs), (ins unknown:$target),
             [(br bb:$target)]>;
 
 // NOOPT-NEXT:    GIM_Reject,
-// NOOPT-NEXT:  }; // Size: 1459 bytes
+// NOOPT-NEXT:  }; // Size: 1501 bytes
 // NOOPT-NEXT:  return MatchTable0;
diff --git a/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/HwModes.td b/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/HwModes.td
index 510368516739..f112577e4c3c 100644
--- a/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/HwModes.td
+++ b/external/llvm-project/llvm/test/TableGen/GlobalISelEmitter/HwModes.td
@@ -54,6 +54,7 @@ class I<dag OOps, dag IOps, list<dag> Pat>
 // CHECK-NEXT:    bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat &Imm) const override;
 // CHECK-NEXT:    const uint8_t *getMatchTable() const override;
 // CHECK-NEXT:    bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI, const MatcherState &State) const override;
+// CHECK-NEXT:    bool testMOPredicate_MO(unsigned PredicateID, const MachineOperand &MO, const MatcherState &State) const override;
 // CHECK-NEXT:    bool testSimplePredicate(unsigned PredicateID) const override;
 // CHECK-NEXT:    bool runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
 // CHECK-NEXT:  #endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL
diff --git a/external/llvm-project/llvm/test/TableGen/VarLenDecoder.td b/external/llvm-project/llvm/test/TableGen/VarLenDecoder.td
index db7a520533e8..06ff62294a19 100644
--- a/external/llvm-project/llvm/test/TableGen/VarLenDecoder.td
+++ b/external/llvm-project/llvm/test/TableGen/VarLenDecoder.td
@@ -47,19 +47,19 @@ def FOO32 : MyVarInst<MemOp32> {
   );
 }
 
-// CHECK-SMALL:      MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
-// CHECK-SMALL-NEXT: MCD::OPC_FilterValue, 8, 4, 0, // Skip to: 11
-// CHECK-SMALL-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
-// CHECK-SMALL-NEXT: MCD::OPC_FilterValue, 9, 4, 0, // Skip to: 19
-// CHECK-SMALL-NEXT: MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
-// CHECK-SMALL-NEXT: MCD::OPC_Fail,
+// CHECK-SMALL:      /* 0 */       MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
+// CHECK-SMALL-NEXT: /* 3 */       MCD::OPC_FilterValue, 8, 4, 0, // Skip to: 11
+// CHECK-SMALL-NEXT: /* 7 */       MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
+// CHECK-SMALL-NEXT: /* 11 */      MCD::OPC_FilterValueOrFail, 9,
+// CHECK-SMALL-NEXT: /* 13 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
+// CHECK-SMALL-NEXT: /* 17 */      MCD::OPC_Fail,
 
 // CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 3, 5,  // Inst{7-3} ...
 // CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 8, 4, 0, 0, // Skip to: 12
 // CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 0, // Opcode: FOO16
-// CHECK-LARGE-NEXT: /* 12 */      MCD::OPC_FilterValue, 9, 4, 0, 0, // Skip to: 21
-// CHECK-LARGE-NEXT: /* 17 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
-// CHECK-LARGE-NEXT: /* 21 */      MCD::OPC_Fail,
+// CHECK-LARGE-NEXT: /* 12 */      MCD::OPC_FilterValueOrFail, 9,
+// CHECK-LARGE-NEXT: /* 14 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: FOO32
+// CHECK-LARGE-NEXT: /* 18 */      MCD::OPC_Fail,
 
 // Instruction length table
 // CHECK: 27,
@@ -88,7 +88,8 @@ def FOO32 : MyVarInst<MemOp32> {
 // CHECK-LABEL: case MCD::OPC_ExtractField: {
 // CHECK: makeUp(insn, Start + Len);
 
-// CHECK-LABEL: case MCD::OPC_CheckField: {
+// CHECK-LABEL: case MCD::OPC_CheckField:
+// CHECK-NEXT:  case MCD::OPC_CheckFieldOrFail: {
 // CHECK: makeUp(insn, Start + Len);
 
 // CHECK-LABEL: case MCD::OPC_Decode: {
diff --git a/external/llvm-project/llvm/test/TableGen/directive1.td b/external/llvm-project/llvm/test/TableGen/directive1.td
index 3b2b4ca1b703..1d2bd51204e4 100644
--- a/external/llvm-project/llvm/test/TableGen/directive1.td
+++ b/external/llvm-project/llvm/test/TableGen/directive1.td
@@ -14,11 +14,11 @@ def TestDirectiveLanguage : DirectiveLanguage {
   let flangClauseBaseClass = "TdlClause";
 }
 
-def TDLCV_vala : ClauseVal<"vala",1,1> {}
-def TDLCV_valb : ClauseVal<"valb",2,1> {}
-def TDLCV_valc : ClauseVal<"valc",3,0> { let isDefault = 1; }
+def TDLCV_vala : EnumVal<"vala",1,1> {}
+def TDLCV_valb : EnumVal<"valb",2,1> {}
+def TDLCV_valc : EnumVal<"valc",3,0> { let isDefault = 1; }
 
-def TDLC_ClauseA : Clause<"clausea"> {
+def TDLC_ClauseA : Clause<[Spelling<"clausea">]> {
   let enumClauseValue = "AKind";
   let allowedClauseValues = [
     TDLCV_vala,
@@ -27,18 +27,18 @@ def TDLC_ClauseA : Clause<"clausea"> {
   ];
 }
 
-def TDLC_ClauseB : Clause<"clauseb"> {
+def TDLC_ClauseB : Clause<[Spelling<"clauseb">]> {
   let flangClass = "IntExpr";
   let isValueOptional = 1;
   let isDefault = 1;
 }
 
-def TDLC_ClauseC : Clause<"clausec"> {
+def TDLC_ClauseC : Clause<[Spelling<"clausec">, Spelling<"ccccccc">]> {
   let flangClass = "IntExpr";
   let isValueList = 1;
 }
 
-def TDL_DirA : Directive<"dira"> {
+def TDL_DirA : Directive<[Spelling<"dira">]> {
   let allowedClauses = [
     VersionedClause<TDLC_ClauseA>,
     VersionedClause<TDLC_ClauseB>
@@ -53,11 +53,13 @@ def TDL_DirA : Directive<"dira"> {
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
 // CHECK-NEXT:  #include "llvm/ADT/BitmaskEnum.h"
+// CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
+// CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
 // CHECK-NEXT:  #include <cstddef>
+// CHECK-NEXT:  #include <utility>
 // CHECK-EMPTY:
 // CHECK-NEXT:  namespace llvm {
-// CHECK-NEXT:  class StringRef;
 // CHECK-NEXT:  namespace tdl {
 // CHECK-EMPTY:
 // CHECK-NEXT:  LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
@@ -123,13 +125,20 @@ def TDL_DirA : Directive<"dira"> {
 // CHECK-NEXT:  constexpr auto TDLCV_valc = AKind::TDLCV_valc;
 // CHECK-EMPTY:
 // CHECK-NEXT:  // Enumeration helper functions
-// CHECK-NEXT:  LLVM_ABI Directive getTdlDirectiveKind(StringRef Str);
+// CHECK-NEXT:  LLVM_ABI std::pair<Directive, directive::VersionRange> getTdlDirectiveKindAndVersions(StringRef Str);
+// CHECK-NEXT:  inline Directive getTdlDirectiveKind(StringRef Str) {
+// CHECK-NEXT:   return getTdlDirectiveKindAndVersions(Str).first;
+// CHECK-NEXT:  }
 // CHECK-EMPTY:
-// CHECK-NEXT:  LLVM_ABI StringRef getTdlDirectiveName(Directive D);
+// CHECK-NEXT:  LLVM_ABI StringRef getTdlDirectiveName(Directive D, unsigned Ver = 0);
 // CHECK-EMPTY:
-// CHECK-NEXT:  LLVM_ABI Clause getTdlClauseKind(StringRef Str);
+// CHECK-NEXT:  LLVM_ABI std::pair<Clause, directive::VersionRange> getTdlClauseKindAndVersions(StringRef Str);
 // CHECK-EMPTY:
-// CHECK-NEXT:  LLVM_ABI StringRef getTdlClauseName(Clause C);
+// CHECK-NEXT:  inline Clause getTdlClauseKind(StringRef Str) {
+// CHECK-NEXT:    return getTdlClauseKindAndVersions(Str).first;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT:  LLVM_ABI StringRef getTdlClauseName(Clause C, unsigned Ver = 0);
 // CHECK-EMPTY:
 // CHECK-NEXT:  /// Return true if \p C is a valid clause for \p D in version \p Version.
 // CHECK-NEXT:  LLVM_ABI bool isAllowedClauseForDirective(Directive D, Clause C, unsigned Version);
@@ -260,7 +269,8 @@ def TDL_DirA : Directive<"dira"> {
 // IMPL-NEXT:  TYPE_PARSER(
 // IMPL-NEXT:    "clausec" >> construct<TdlClause>(construct<TdlClause::Clausec>(parenthesized(nonemptyList(Parser<IntExpr>{})))) || 
 // IMPL-NEXT:    "clauseb" >> construct<TdlClause>(construct<TdlClause::Clauseb>(maybe(parenthesized(Parser<IntExpr>{})))) ||
-// IMPL-NEXT:    "clausea" >> construct<TdlClause>(construct<TdlClause::Clausea>())
+// IMPL-NEXT:    "clausea" >> construct<TdlClause>(construct<TdlClause::Clausea>()) ||
+// IMPL-NEXT:    "ccccccc" >> construct<TdlClause>(construct<TdlClause::Clausec>(parenthesized(nonemptyList(Parser<IntExpr>{}))))
 // IMPL-NEXT:  )
 // IMPL-EMPTY:
 // IMPL-NEXT:  #endif // GEN_FLANG_CLAUSES_PARSER
@@ -309,15 +319,18 @@ def TDL_DirA : Directive<"dira"> {
 // IMPL:       #ifdef GEN_DIRECTIVES_IMPL
 // IMPL-NEXT:  #undef GEN_DIRECTIVES_IMPL
 // IMPL-EMPTY:
+// IMPL-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // IMPL-NEXT:  #include "llvm/Support/ErrorHandling.h"
+// IMPL-NEXT:  #include <utility>
 // IMPL-EMPTY:
-// IMPL-NEXT:  llvm::tdl::Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) {
-// IMPL-NEXT:    return StringSwitch<Directive>(Str)
-// IMPL-NEXT:      .Case("dira",TDLD_dira)
-// IMPL-NEXT:      .Default(TDLD_dira);
-// IMPL-NEXT:  }
+// IMPL-NEXT: std::pair<llvm::tdl::Directive, llvm::directive::VersionRange> llvm::tdl::getTdlDirectiveKindAndVersions(llvm::StringRef Str) {
+// IMPL-NEXT:   directive::VersionRange All; // Default-initialized to "all versions"
+// IMPL-NEXT:   return StringSwitch<std::pair<Directive, directive::VersionRange>>(Str)
+// IMPL-NEXT:     .Case("dira", {TDLD_dira, All})
+// IMPL-NEXT:     .Default({TDLD_dira, All});
+// IMPL-NEXT: }
 // IMPL-EMPTY:
-// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlDirectiveName(llvm::tdl::Directive Kind) {
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlDirectiveName(llvm::tdl::Directive Kind, unsigned Version) {
 // IMPL-NEXT:    switch (Kind) {
 // IMPL-NEXT:      case TDLD_dira:
 // IMPL-NEXT:        return "dira";
@@ -325,22 +338,29 @@ def TDL_DirA : Directive<"dira"> {
 // IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
 // IMPL-NEXT:  }
 // IMPL-EMPTY:
-// IMPL-NEXT:  llvm::tdl::Clause llvm::tdl::getTdlClauseKind(llvm::StringRef Str) {
-// IMPL-NEXT:    return StringSwitch<Clause>(Str)
-// IMPL-NEXT:      .Case("clausea",TDLC_clausea)
-// IMPL-NEXT:      .Case("clauseb",TDLC_clauseb)
-// IMPL-NEXT:      .Case("clausec",TDLC_clausec)
-// IMPL-NEXT:      .Default(TDLC_clauseb);
-// IMPL-NEXT:  }
-// IMPL-EMPTY:
-// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlClauseName(llvm::tdl::Clause Kind) {
+// IMPL-NEXT: std::pair<llvm::tdl::Clause, llvm::directive::VersionRange> llvm::tdl::getTdlClauseKindAndVersions(llvm::StringRef Str) {
+// IMPL-NEXT:   directive::VersionRange All; // Default-initialized to "all versions"
+// IMPL-NEXT:   return StringSwitch<std::pair<Clause, directive::VersionRange>>(Str)
+// IMPL-NEXT:     .Case("clausea", {TDLC_clausea, All})
+// IMPL-NEXT:     .Case("clauseb", {TDLC_clauseb, All})
+// IMPL-NEXT:     .Case("clausec", {TDLC_clausec, All})
+// IMPL-NEXT:     .Case("ccccccc", {TDLC_clausec, All})
+// IMPL-NEXT:     .Default({TDLC_clauseb, All});
+// IMPL-NEXT: }
+// IMPL-EMPTY:
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlClauseName(llvm::tdl::Clause Kind, unsigned Version) {
 // IMPL-NEXT:    switch (Kind) {
 // IMPL-NEXT:      case TDLC_clausea:
 // IMPL-NEXT:        return "clausea";
 // IMPL-NEXT:      case TDLC_clauseb:
 // IMPL-NEXT:        return "clauseb";
-// IMPL-NEXT:      case TDLC_clausec:
-// IMPL-NEXT:        return "clausec";
+// IMPL-NEXT:      case TDLC_clausec: {
+// IMPL-NEXT:        static constexpr llvm::directive::Spelling TDLC_clausec_spellings[] = {
+// IMPL-NEXT:            {"clausec", {0, 2147483647}},
+// IMPL-NEXT:            {"ccccccc", {0, 2147483647}},
+// IMPL-NEXT:        };
+// IMPL-NEXT:        return llvm::directive::FindName(TDLC_clausec_spellings, Version);
+// IMPL-NEXT:      }
 // IMPL-NEXT:    }
 // IMPL-NEXT:    llvm_unreachable("Invalid Tdl Clause kind");
 // IMPL-NEXT:  }
diff --git a/external/llvm-project/llvm/test/TableGen/directive2.td b/external/llvm-project/llvm/test/TableGen/directive2.td
index 3de66cfea94b..3a64bb3900a3 100644
--- a/external/llvm-project/llvm/test/TableGen/directive2.td
+++ b/external/llvm-project/llvm/test/TableGen/directive2.td
@@ -13,26 +13,26 @@ def TestDirectiveLanguage : DirectiveLanguage {
   let flangClauseBaseClass = "TdlClause";
 }
 
-def TDLC_ClauseA : Clause<"clausea"> {
+def TDLC_ClauseA : Clause<[Spelling<"clausea">]> {
   let isImplicit = 1;
 }
-def TDLC_ClauseB : Clause<"clauseb"> {
+def TDLC_ClauseB : Clause<[Spelling<"clauseb">]> {
   let isDefault = 1;
   let flangClass = "IntExpr";
   let isValueList = 1;
 }
-def TDLC_ClauseC : Clause<"clausec"> {
+def TDLC_ClauseC : Clause<[Spelling<"clausec">]> {
   let clangClass = "ClauseC";
   let flangClass = "Name";
   let defaultValue = "*";
   let isValueOptional = 1;
 }
-def TDLC_ClauseD : Clause<"claused"> {
+def TDLC_ClauseD : Clause<[Spelling<"claused">]> {
   let clangClass = "ClauseD";
   let isImplicit = 1;
 }
 
-def TDL_DirA : Directive<"dira"> {
+def TDL_DirA : Directive<[Spelling<"dira">]> {
   let allowedClauses = [
     VersionedClause<TDLC_ClauseA, 2, 4>,
     VersionedClause<TDLC_ClauseB, 2>
@@ -46,11 +46,13 @@ def TDL_DirA : Directive<"dira"> {
 // CHECK-NEXT:  #define LLVM_Tdl_INC
 // CHECK-EMPTY:
 // CHECK-NEXT:  #include "llvm/ADT/ArrayRef.h"
+// CHECK-NEXT:  #include "llvm/ADT/StringRef.h"
+// CHECK-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // CHECK-NEXT:  #include "llvm/Support/Compiler.h"
 // CHECK-NEXT:  #include <cstddef>
+// CHECK-NEXT:  #include <utility>
 // CHECK-EMPTY:
 // CHECK-NEXT:  namespace llvm {
-// CHECK-NEXT:  class StringRef;
 // CHECK-NEXT:  namespace tdl {
 // CHECK-EMPTY:
 // CHECK-NEXT:  enum class Association {
@@ -99,13 +101,20 @@ def TDL_DirA : Directive<"dira"> {
 // CHECK-NEXT:  static constexpr std::size_t Clause_enumSize = 4;
 // CHECK-EMPTY:
 // CHECK-NEXT:  // Enumeration helper functions
-// CHECK-NEXT:  LLVM_ABI Directive getTdlDirectiveKind(StringRef Str);
+// CHECK-NEXT:  LLVM_ABI std::pair<Directive, directive::VersionRange> getTdlDirectiveKindAndVersions(StringRef Str);
+// CHECK-NEXT:  inline Directive getTdlDirectiveKind(StringRef Str) {
+// CHECK-NEXT:    return getTdlDirectiveKindAndVersions(Str).first;
+// CHECK-NEXT:  }
 // CHECK-EMPTY:
-// CHECK-NEXT:  LLVM_ABI StringRef getTdlDirectiveName(Directive D);
+// CHECK-NEXT:  LLVM_ABI StringRef getTdlDirectiveName(Directive D, unsigned Ver = 0);
 // CHECK-EMPTY:
-// CHECK-NEXT:  LLVM_ABI Clause getTdlClauseKind(StringRef Str);
+// CHECK-NEXT:  LLVM_ABI std::pair<Clause, directive::VersionRange> getTdlClauseKindAndVersions(StringRef Str);
 // CHECK-EMPTY:
-// CHECK-NEXT:  LLVM_ABI StringRef getTdlClauseName(Clause C);
+// CHECK-NEXT:  inline Clause getTdlClauseKind(StringRef Str) {
+// CHECK-NEXT:    return getTdlClauseKindAndVersions(Str).first;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT:  LLVM_ABI StringRef getTdlClauseName(Clause C, unsigned Ver = 0);
 // CHECK-EMPTY:
 // CHECK-NEXT:  /// Return true if \p C is a valid clause for \p D in version \p Version.
 // CHECK-NEXT:  LLVM_ABI bool isAllowedClauseForDirective(Directive D, Clause C, unsigned Version);
@@ -257,15 +266,18 @@ def TDL_DirA : Directive<"dira"> {
 // IMPL:       #ifdef GEN_DIRECTIVES_IMPL
 // IMPL-NEXT:  #undef GEN_DIRECTIVES_IMPL
 // IMPL-EMPTY:
+// IMPL-NEXT:  #include "llvm/Frontend/Directive/Spelling.h"
 // IMPL-NEXT:  #include "llvm/Support/ErrorHandling.h"
+// IMPL-NEXT:  #include <utility>
 // IMPL-EMPTY:
-// IMPL-NEXT:  llvm::tdl::Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) {
-// IMPL-NEXT:    return StringSwitch<Directive>(Str)
-// IMPL-NEXT:      .Case("dira",TDLD_dira)
-// IMPL-NEXT:      .Default(TDLD_dira);
-// IMPL-NEXT:  }
+// IMPL-NEXT: std::pair<llvm::tdl::Directive, llvm::directive::VersionRange> llvm::tdl::getTdlDirectiveKindAndVersions(llvm::StringRef Str) {
+// IMPL-NEXT:   directive::VersionRange All; // Default-initialized to "all versions"
+// IMPL-NEXT:   return StringSwitch<std::pair<Directive, directive::VersionRange>>(Str)
+// IMPL-NEXT:     .Case("dira", {TDLD_dira, All})
+// IMPL-NEXT:     .Default({TDLD_dira, All});
+// IMPL-NEXT: }
 // IMPL-EMPTY:
-// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlDirectiveName(llvm::tdl::Directive Kind) {
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlDirectiveName(llvm::tdl::Directive Kind, unsigned Version) {
 // IMPL-NEXT:    switch (Kind) {
 // IMPL-NEXT:      case TDLD_dira:
 // IMPL-NEXT:        return "dira";
@@ -273,16 +285,17 @@ def TDL_DirA : Directive<"dira"> {
 // IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
 // IMPL-NEXT:  }
 // IMPL-EMPTY:
-// IMPL-NEXT:  llvm::tdl::Clause llvm::tdl::getTdlClauseKind(llvm::StringRef Str) {
-// IMPL-NEXT:    return StringSwitch<Clause>(Str)
-// IMPL-NEXT:      .Case("clausea",TDLC_clauseb)
-// IMPL-NEXT:      .Case("clauseb",TDLC_clauseb)
-// IMPL-NEXT:      .Case("clausec",TDLC_clausec)
-// IMPL-NEXT:      .Case("claused",TDLC_clauseb)
-// IMPL-NEXT:      .Default(TDLC_clauseb);
-// IMPL-NEXT:  }
+// IMPL-NEXT: std::pair<llvm::tdl::Clause, llvm::directive::VersionRange> llvm::tdl::getTdlClauseKindAndVersions(llvm::StringRef Str) {
+// IMPL-NEXT:   directive::VersionRange All; // Default-initialized to "all versions"
+// IMPL-NEXT:   return StringSwitch<std::pair<Clause, directive::VersionRange>>(Str)
+// IMPL-NEXT:     .Case("clausea", {TDLC_clauseb, All})
+// IMPL-NEXT:     .Case("clauseb", {TDLC_clauseb, All})
+// IMPL-NEXT:     .Case("clausec", {TDLC_clausec, All})
+// IMPL-NEXT:     .Case("claused", {TDLC_clauseb, All})
+// IMPL-NEXT:     .Default({TDLC_clauseb, All});
+// IMPL-NEXT: }
 // IMPL-EMPTY:
-// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlClauseName(llvm::tdl::Clause Kind) {
+// IMPL-NEXT:  llvm::StringRef llvm::tdl::getTdlClauseName(llvm::tdl::Clause Kind, unsigned Version) {
 // IMPL-NEXT:    switch (Kind) {
 // IMPL-NEXT:      case TDLC_clausea:
 // IMPL-NEXT:        return "clausea";
diff --git a/external/llvm-project/llvm/test/TableGen/directive3.td b/external/llvm-project/llvm/test/TableGen/directive3.td
index 2a82b5ebcb49..a9e1f04d4dec 100644
--- a/external/llvm-project/llvm/test/TableGen/directive3.td
+++ b/external/llvm-project/llvm/test/TableGen/directive3.td
@@ -7,20 +7,20 @@ def TestDirectiveLanguage : DirectiveLanguage {
   let name = "TdlError";
 }
 
-def TDLC_ClauseA : Clause<"clausea"> {
+def TDLC_ClauseA : Clause<[Spelling<"clausea">]> {
   let isDefault = 1;
 }
 
-def TDLC_ClauseB : Clause<"clauseb"> {
+def TDLC_ClauseB : Clause<[Spelling<"clauseb">]> {
 }
 
-def TDLC_ClauseC : Clause<"clausec"> {
+def TDLC_ClauseC : Clause<[Spelling<"clausec">]> {
 }
 
-def TDLC_ClauseD : Clause<"claused"> {
+def TDLC_ClauseD : Clause<[Spelling<"claused">]> {
 }
 
-def TDL_DirA : Directive<"dira"> {
+def TDL_DirA : Directive<[Spelling<"dira">]> {
   let allowedClauses = [
     VersionedClause<TDLC_ClauseA>,
     VersionedClause<TDLC_ClauseB>,
diff --git a/external/llvm-project/llvm/test/TableGen/true-false.td b/external/llvm-project/llvm/test/TableGen/true-false.td
index 5a59f20b21d2..5fa570231448 100644
--- a/external/llvm-project/llvm/test/TableGen/true-false.td
+++ b/external/llvm-project/llvm/test/TableGen/true-false.td
@@ -67,13 +67,18 @@ def rec7 {
   bits<3> flags = { true, false, true };
 }
 
-// `!and` and `!or` should be short-circuit such that `!tail` on empty list will never
-// be evaluated.
+// `!and` and `!or` should be short-circuited such that any of the `!head` or
+// `!tail` on empty list below will never be evaluated.
 // CHECK: def rec8
+// CHECK:   bit v = 0;
+// CHECK:   int v2 = -1;
 // CHECK:   list<int> newSeq = [];
 // CHECK:   list<int> newSeq2 = [];
 
 class Foo <list<int> seq = []> {
+  bit v = !and(false, !head(seq));
+  int v2 = !or(-1, !head(seq));
+
   bit unresolved = !ne(!find(NAME, "BAR"), -1);
   list<int> newSeq  = !if(!and(false, unresolved), !tail(seq), seq);
   list<int> newSeq2 = !if(!or(-1, unresolved), seq, !tail(seq));
diff --git a/external/llvm-project/llvm/test/TableGen/trydecode-emission.td b/external/llvm-project/llvm/test/TableGen/trydecode-emission.td
index db77d94b2aa5..c3178dd71cf4 100644
--- a/external/llvm-project/llvm/test/TableGen/trydecode-emission.td
+++ b/external/llvm-project/llvm/test/TableGen/trydecode-emission.td
@@ -35,11 +35,11 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
-// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
-// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 19
-// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-NEXT: /* 5 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 17
+// CHECK-NEXT: /* 11 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 17
+// CHECK-NEXT: /* 17 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA, DecodeIdx: {{[0-9]+}}
+// CHECK-NEXT: /* 21 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 
@@ -48,11 +48,11 @@ def InstB : TestInstruction {
 // CHECK-NEXT:  return NumToSkip;
 
 // CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
-// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
-// CHECK-LARGE-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
-// CHECK-LARGE-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-LARGE-NEXT: /* 26 */      MCD::OPC_Fail,
+// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-LARGE-NEXT: /* 5 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 19
+// CHECK-LARGE-NEXT: /* 12 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 19
+// CHECK-LARGE-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA, DecodeIdx: {{[0-9]+}}
+// CHECK-LARGE-NEXT: /* 23 */      MCD::OPC_Fail,
 
 // CHECK-LARGE: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 
diff --git a/external/llvm-project/llvm/test/TableGen/trydecode-emission2.td b/external/llvm-project/llvm/test/TableGen/trydecode-emission2.td
index 914e17c6f820..4c8a95eff5dd 100644
--- a/external/llvm-project/llvm/test/TableGen/trydecode-emission2.td
+++ b/external/llvm-project/llvm/test/TableGen/trydecode-emission2.td
@@ -32,27 +32,27 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 31, 0, // Skip to: 38
-// CHECK-NEXT: /* 7 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
-// CHECK-NEXT: /* 10 */      MCD::OPC_FilterValue, 0, 24, 0, // Skip to: 38
-// CHECK-NEXT: /* 14 */      MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 26
-// CHECK-NEXT: /* 20 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 26
-// CHECK-NEXT: /* 26 */      MCD::OPC_CheckField, 3, 2, 0, 6, 0, // Skip to: 38
-// CHECK-NEXT: /* 32 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, // Opcode: InstA, skip to: 38
-// CHECK-NEXT: /* 38 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-NEXT: /* 5 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-NEXT: /* 8 */       MCD::OPC_FilterValueOrFail, 0
+// CHECK-NEXT: /* 10 */      MCD::OPC_CheckField, 0, 2, 3, 6, 0, // Skip to: 22
+// CHECK-NEXT: /* 16 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 22
+// CHECK-NEXT: /* 22 */      MCD::OPC_CheckFieldOrFail, 3, 2, 0,
+// CHECK-NEXT: /* 26 */      MCD::OPC_TryDecodeOrFail, {{[0-9]+}}, {{[0-9]+}}, 1,
+// CHECK-NEXT: /* 30 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 // CHECK: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 
 // CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 2, 1,  // Inst{2} ...
-// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 36, 0, 0, // Skip to: 44
-// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
-// CHECK-LARGE-NEXT: /* 11 */      MCD::OPC_FilterValue, 0, 28, 0, 0, // Skip to: 44
-// CHECK-LARGE-NEXT: /* 16 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 30
-// CHECK-LARGE-NEXT: /* 23 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 30
-// CHECK-LARGE-NEXT: /* 30 */      MCD::OPC_CheckField, 3, 2, 0, 7, 0, 0, // Skip to: 44
-// CHECK-LARGE-NEXT: /* 37 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 1, 0, 0, 0, // Opcode: InstA, skip to: 44
-// CHECK-LARGE-NEXT: /* 44 */      MCD::OPC_Fail,
+// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-LARGE-NEXT: /* 5 */       MCD::OPC_ExtractField, 5, 3,  // Inst{7-5} ...
+// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-LARGE-NEXT: /* 10 */      MCD::OPC_CheckField, 0, 2, 3, 7, 0, 0, // Skip to: 24
+// CHECK-LARGE-NEXT: /* 17 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 24
+// CHECK-LARGE-NEXT: /* 24 */      MCD::OPC_CheckFieldOrFail, 3, 2, 0,
+// CHECK-LARGE-NEXT: /* 28 */      MCD::OPC_TryDecodeOrFail, {{[0-9]+}}, {{[0-9]+}}, 1,
+// CHECK-LARGE-NEXT: /* 32 */      MCD::OPC_Fail,
 
 // CHECK-LARGE: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 // CHECK-LARGE: if (!Check(S, DecodeInstA(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/external/llvm-project/llvm/test/TableGen/trydecode-emission3.td b/external/llvm-project/llvm/test/TableGen/trydecode-emission3.td
index 141dc5439ca1..f262bfab77c3 100644
--- a/external/llvm-project/llvm/test/TableGen/trydecode-emission3.td
+++ b/external/llvm-project/llvm/test/TableGen/trydecode-emission3.td
@@ -36,19 +36,19 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 16, 0, // Skip to: 23
-// CHECK-NEXT: /* 7 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 19
-// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 19
-// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 3 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-NEXT: /* 5 */       MCD::OPC_CheckField, 2, 2, 0, 6, 0, // Skip to: 17
+// CHECK-NEXT: /* 11 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 17
+// CHECK-NEXT: /* 17 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
+// CHECK-NEXT: /* 21 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstBOp(MI, tmp, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 
 // CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 4, 4,  // Inst{7-4} ...
-// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValue, 0, 18, 0, 0, // Skip to: 26
-// CHECK-LARGE-NEXT: /* 8 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 22
-// CHECK-LARGE-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 22
-// CHECK-LARGE-NEXT: /* 22 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-LARGE-NEXT: /* 26 */      MCD::OPC_Fail,
+// CHECK-LARGE-NEXT: /* 3 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-LARGE-NEXT: /* 5 */       MCD::OPC_CheckField, 2, 2, 0, 7, 0, 0, // Skip to: 19
+// CHECK-LARGE-NEXT: /* 12 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 19
+// CHECK-LARGE-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA, DecodeIdx: {{[0-9]+}}
+// CHECK-LARGE-NEXT: /* 23 */      MCD::OPC_Fail,
 
 // CHECK-LARGE: if (!Check(S, DecodeInstBOp(MI, tmp, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
diff --git a/external/llvm-project/llvm/test/TableGen/trydecode-emission4.td b/external/llvm-project/llvm/test/TableGen/trydecode-emission4.td
index bb44876e77f0..2c63229c053a 100644
--- a/external/llvm-project/llvm/test/TableGen/trydecode-emission4.td
+++ b/external/llvm-project/llvm/test/TableGen/trydecode-emission4.td
@@ -34,21 +34,21 @@ def InstB : TestInstruction {
 }
 
 // CHECK:      /* 0 */       MCD::OPC_ExtractField, 250, 3, 4,  // Inst{509-506} ...
-// CHECK-NEXT: /* 4 */       MCD::OPC_FilterValue, 0, 17, 0, // Skip to: 25
-// CHECK-NEXT: /* 8 */       MCD::OPC_CheckField, 248, 3, 2, 0, 6, 0, // Skip to: 21
-// CHECK-NEXT: /* 15 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, skip to: 21
-// CHECK-NEXT: /* 21 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-NEXT: /* 25 */      MCD::OPC_Fail,
+// CHECK-NEXT: /* 4 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-NEXT: /* 6 */       MCD::OPC_CheckField, 248, 3, 2, 0, 6, 0, // Skip to: 19
+// CHECK-NEXT: /* 13 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 19
+// CHECK-NEXT: /* 19 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA, DecodeIdx: {{[0-9]+}}
+// CHECK-NEXT: /* 23 */      MCD::OPC_Fail,
 
 // CHECK: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 
 
 // CHECK-LARGE:      /* 0 */       MCD::OPC_ExtractField, 250, 3, 4,  // Inst{509-506} ...
-// CHECK-LARGE-NEXT: /* 4 */       MCD::OPC_FilterValue, 0, 19, 0, 0, // Skip to: 28
-// CHECK-LARGE-NEXT: /* 9 */       MCD::OPC_CheckField, 248, 3, 2, 0, 7, 0, 0, // Skip to: 24
-// CHECK-LARGE-NEXT: /* 17 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, skip to: 24
-// CHECK-LARGE-NEXT: /* 24 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA
-// CHECK-LARGE-NEXT: /* 28 */      MCD::OPC_Fail,
+// CHECK-LARGE-NEXT: /* 4 */       MCD::OPC_FilterValueOrFail, 0,
+// CHECK-LARGE-NEXT: /* 6 */       MCD::OPC_CheckField, 248, 3, 2, 0, 7, 0, 0, // Skip to: 21
+// CHECK-LARGE-NEXT: /* 14 */      MCD::OPC_TryDecode, {{[0-9]+}}, {{[0-9]+}}, 0, 0, 0, 0, // Opcode: InstB, DecodeIdx: {{[0-9]+}}, Skip to: 21
+// CHECK-LARGE-NEXT: /* 21 */      MCD::OPC_Decode, {{[0-9]+}}, {{[0-9]+}}, 1, // Opcode: InstA, DecodeIdx: {{[0-9]+}}
+// CHECK-LARGE-NEXT: /* 25 */      MCD::OPC_Fail,
 
 // CHECK-LARGE: if (!Check(S, DecodeInstB(MI, insn, Address, Decoder))) { DecodeComplete = false; return MCDisassembler::Fail; }
 
diff --git a/external/llvm-project/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_foo_tcl.ll b/external/llvm-project/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_foo_tcl.ll
new file mode 100644
index 000000000000..4a696837bc8e
--- /dev/null
+++ b/external/llvm-project/llvm/test/ThinLTO/X86/Inputs/devirt_single_hybrid_foo_tcl.ll
@@ -0,0 +1,31 @@
+; ModuleID = 'foo.cpp'
+source_filename = "foo.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.A = type { ptr }
+
+; Function Attrs: uwtable
+define hidden i32 @_Z3fooP1A(ptr %pA) local_unnamed_addr {
+entry:
+  %vtable = load ptr, ptr %pA, align 8, !tbaa !2
+  %0 = call { ptr, i1 } @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"_ZTS1A")
+  %1 = extractvalue { ptr, i1 } %0, 0
+  %call = tail call i32 %1(ptr %pA)
+  %add = add nsw i32 %call, 10
+  ret i32 %add
+}
+
+declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
+
+; Function Attrs: nounwind willreturn
+declare void @llvm.assume(i1)
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (trunk 373596)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"vtable pointer", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
diff --git a/external/llvm-project/llvm/test/ThinLTO/X86/cfi-devirt.ll b/external/llvm-project/llvm/test/ThinLTO/X86/cfi-devirt.ll
index 0a9935b036b5..c1d0b5fee8cc 100644
--- a/external/llvm-project/llvm/test/ThinLTO/X86/cfi-devirt.ll
+++ b/external/llvm-project/llvm/test/ThinLTO/X86/cfi-devirt.ll
@@ -91,7 +91,7 @@ cont2:
   ; CHECK-IR: br i1 {{.*}}, label %trap, label %cont2
 
   ; We still have to call it as virtual.
-  ; CHECK-IR: %call3 = tail call i32 %4
+  ; CHECK-IR: %call3 = tail call i32 %
   %call3 = tail call i32 %5(ptr nonnull %obj, i32 %call)
   ret i32 %call3
 }
diff --git a/external/llvm-project/llvm/test/ThinLTO/X86/devirt_single_hybrid.ll b/external/llvm-project/llvm/test/ThinLTO/X86/devirt_single_hybrid.ll
index 90fdf0d7dfa0..53c001efc132 100644
--- a/external/llvm-project/llvm/test/ThinLTO/X86/devirt_single_hybrid.ll
+++ b/external/llvm-project/llvm/test/ThinLTO/X86/devirt_single_hybrid.ll
@@ -2,8 +2,32 @@
 ; when we're running hybrid LTO.
 ;
 ; RUN: opt -thinlto-bc -thinlto-split-lto-unit %s -o %t-main.bc
-; RUN: opt -thinlto-bc -thinlto-split-lto-unit %p/Inputs/devirt_single_hybrid_foo.ll -o %t-foo.bc
 ; RUN: opt -thinlto-bc -thinlto-split-lto-unit %p/Inputs/devirt_single_hybrid_bar.ll -o %t-bar.bc
+
+; Test the assume(type.test) case.
+ 
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit %p/Inputs/devirt_single_hybrid_foo.ll -o %t-foo.bc
+; RUN: llvm-lto2 run -save-temps %t-main.bc %t-foo.bc %t-bar.bc -pass-remarks=. -o %t \
+; RUN:   -whole-program-visibility \
+; RUN:    -r=%t-foo.bc,_Z3fooP1A,pl \
+; RUN:    -r=%t-main.bc,main,plx \
+; RUN:    -r=%t-main.bc,_Z3barv,l \
+; RUN:    -r=%t-bar.bc,_Z3barv,pl \
+; RUN:    -r=%t-bar.bc,_Z3fooP1A, \
+; RUN:    -r=%t-bar.bc,_ZNK1A1fEv,pl \
+; RUN:    -r=%t-bar.bc,_ZTV1A,l \
+; RUN:    -r=%t-bar.bc,_ZTVN10__cxxabiv117__class_type_infoE, \
+; RUN:    -r=%t-bar.bc,_ZTS1A,pl \
+; RUN:    -r=%t-bar.bc,_ZTI1A,pl \
+; RUN:    -r=%t-bar.bc,_ZNK1A1fEv, \
+; RUN:    -r=%t-bar.bc,_ZTV1A,pl \
+; RUN:    -r=%t-bar.bc,_ZTI1A, 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: llvm-dis %t.1.3.import.bc -o - | FileCheck %s --check-prefix=IMPORT
+; RUN: llvm-dis %t.1.5.precodegen.bc -o - | FileCheck %s --check-prefix=CODEGEN
+
+; Test the type.checked.load case.
+ 
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit %p/Inputs/devirt_single_hybrid_foo_tcl.ll -o %t-foo.bc
 ; RUN: llvm-lto2 run -save-temps %t-main.bc %t-foo.bc %t-bar.bc -pass-remarks=. -o %t \
 ; RUN:   -whole-program-visibility \
 ; RUN:    -r=%t-foo.bc,_Z3fooP1A,pl \
diff --git a/external/llvm-project/llvm/test/ThinLTO/X86/memprof-report-hinted-partial.ll b/external/llvm-project/llvm/test/ThinLTO/X86/memprof-report-hinted-partial.ll
new file mode 100644
index 000000000000..d4a3f9bca2ca
--- /dev/null
+++ b/external/llvm-project/llvm/test/ThinLTO/X86/memprof-report-hinted-partial.ll
@@ -0,0 +1,73 @@
+;; Test that we get hinted size reporting for just the subset of MIBs that
+;; contain context size info in the metadata.
+
+;; Generate the bitcode including ThinLTO summary. Specify
+;; -memprof-min-percent-max-cold-size (value doesn't matter) to indicate to
+;; the bitcode writer that it should expect and optimize for partial context
+;; size info.
+; RUN: opt -thinlto-bc -memprof-min-percent-max-cold-size=50 %s >%t.o
+
+; RUN: llvm-lto2 run %t.o -enable-memprof-context-disambiguation \
+; RUN:	-supports-hot-cold-new \
+; RUN:	-r=%t.o,main,plx \
+; RUN:	-r=%t.o,_Znam, \
+; RUN:	-memprof-report-hinted-sizes \
+; RUN:	-o %t.out 2>&1 | FileCheck %s --check-prefix=SIZES
+
+;; We should only get these two messages from -memprof-report-hinted-sizes
+;; as they are the only MIBs with recorded context size info.
+; SIZES-NOT: full allocation context
+; SIZES: Cold full allocation context 456 with total size 200 is Cold after cloning (context id 2)
+; SIZES: Cold full allocation context 789 with total size 300 is Cold after cloning (context id 2)
+; SIZES-NOT: full allocation context
+
+source_filename = "memprof-report-hinted-partial.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main() #0 {
+entry:
+  %call = call ptr @_Z3foov(), !callsite !0
+  %call1 = call ptr @_Z3foov(), !callsite !1
+  ret i32 0
+}
+
+define internal ptr @_Z3barv() #0 {
+entry:
+  %call = call ptr @_Znam(i64 0), !memprof !2, !callsite !7
+  ret ptr null
+}
+
+declare ptr @_Znam(i64)
+
+define internal ptr @_Z3bazv() #0 {
+entry:
+  %call = call ptr @_Z3barv(), !callsite !8
+  ret ptr null
+}
+
+define internal ptr @_Z3foov() #0 {
+entry:
+  %call = call ptr @_Z3bazv(), !callsite !9
+  ret ptr null
+}
+
+; uselistorder directives
+uselistorder ptr @_Z3foov, { 1, 0 }
+
+attributes #0 = { noinline optnone }
+
+!0 = !{i64 8632435727821051414}
+!1 = !{i64 -3421689549917153178}
+!2 = !{!3, !5, !13}
+!3 = !{!4, !"notcold"}
+!4 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 8632435727821051414}
+!5 = !{!6, !"cold", !11, !12}
+!6 = !{i64 9086428284934609951, i64 -5964873800580613432, i64 2732490490862098848, i64 -3421689549917153178}
+!7 = !{i64 9086428284934609951}
+!8 = !{i64 -5964873800580613432}
+!9 = !{i64 2732490490862098848}
+!11 = !{i64 456, i64 200}
+!12 = !{i64 789, i64 300}
+!13 = !{!14, !"cold"}
+!14 = !{i64 9086428284934609951, i64 12345}
diff --git a/external/llvm-project/llvm/test/Transforms/AggressiveInstCombine/popcount.ll b/external/llvm-project/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
index 4a89705e1774..f56cab150353 100644
--- a/external/llvm-project/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
+++ b/external/llvm-project/llvm/test/Transforms/AggressiveInstCombine/popcount.ll
@@ -191,3 +191,51 @@ define <4 x i32> @popcount32vec(<4 x i32> %0) {
   %13 = lshr <4 x i32> %12, <i32 24, i32 24, i32 24, i32 24>
   ret <4 x i32> %13
 }
+
+define i32 @popcount64_zext(i32 %x) {
+; CHECK-LABEL: @popcount64_zext(
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.ctpop.i64(i64 [[ZEXT]])
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc nuw nsw i64 [[TMP12]] to i32
+; CHECK-NEXT:    ret i32 [[TMP13]]
+;
+  %zext = zext i32 %x to i64
+  %1 = lshr i64 %zext, 1
+  %2 = and i64 %1, 1431655765
+  %3 = sub nsw i64 %zext, %2
+  %4 = and i64 %3, 3689348814741910323
+  %5 = lshr i64 %3, 2
+  %6 = and i64 %5, 3689348814741910323
+  %7 = add nuw nsw i64 %6, %4
+  %8 = lshr i64 %7, 4
+  %9 = add nuw nsw i64 %8, %7
+  %10 = and i64 %9, 1085102592571150095
+  %11 = mul i64 %10, 72340172838076673
+  %12 = lshr i64 %11, 56
+  %13 = trunc nuw nsw i64 %12 to i32
+  ret i32 %13
+}
+
+define i32 @popcount64_mask(i64 %x) {
+; CHECK-LABEL: @popcount64_mask(
+; CHECK-NEXT:    [[MASK:%.*]] = and i64 [[X:%.*]], -281470681808896
+; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.ctpop.i64(i64 [[MASK]])
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc nuw nsw i64 [[TMP12]] to i32
+; CHECK-NEXT:    ret i32 [[TMP13]]
+;
+  %mask = and i64 %x, -281470681808896 ; 0xffff0000ffff0000
+  %1 = lshr i64 %mask, 1
+  %2 = and i64 %1, 6148820867675914240 ; 0x0x5555000055550000
+  %3 = sub nsw i64 %mask, %2
+  %4 = and i64 %3, 3689348814741910323
+  %5 = lshr i64 %3, 2
+  %6 = and i64 %5, 3689348814741910323
+  %7 = add nuw nsw i64 %6, %4
+  %8 = lshr i64 %7, 4
+  %9 = add nuw nsw i64 %8, %7
+  %10 = and i64 %9, 1085102592571150095
+  %11 = mul i64 %10, 72340172838076673
+  %12 = lshr i64 %11, 56
+  %13 = trunc nuw nsw i64 %12 to i32
+  ret i32 %13
+}
diff --git a/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
index 4aee397a0152..2ca143ffc111 100644
--- a/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
+++ b/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
@@ -801,37 +801,53 @@ define double @test_atomicrmw_fadd_f64_flat_unsafe(ptr %ptr, double %value) #3 {
 ; GFX908-NEXT:    ret double [[RES]]
 ;
 ; GFX90A-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(
-; GFX90A-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX90A-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX90A-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX90A:       atomicrmw.shared:
+; GFX90A-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX90A-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX90A:       atomicrmw.check.private:
+; GFX90A-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
 ; GFX90A-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
 ; GFX90A:       atomicrmw.private:
 ; GFX90A-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
 ; GFX90A-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
 ; GFX90A-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
-; GFX90A-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
 ; GFX90A:       atomicrmw.global:
-; GFX90A-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("wavefront") monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX90A-NEXT:    [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
 ; GFX90A:       atomicrmw.phi:
-; GFX90A-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ]
+; GFX90A-NEXT:    [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ]
 ; GFX90A-NEXT:    br label [[ATOMICRMW_END:%.*]]
 ; GFX90A:       atomicrmw.end:
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
 ; GFX942-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(
-; GFX942-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX942-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX942-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX942:       atomicrmw.shared:
+; GFX942-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX942-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX942:       atomicrmw.check.private:
+; GFX942-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
 ; GFX942-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
 ; GFX942:       atomicrmw.private:
 ; GFX942-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
 ; GFX942-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; GFX942-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX942-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
 ; GFX942-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
-; GFX942-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
 ; GFX942:       atomicrmw.global:
-; GFX942-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("wavefront") monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX942-NEXT:    [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
 ; GFX942:       atomicrmw.phi:
-; GFX942-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ]
+; GFX942-NEXT:    [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ]
 ; GFX942-NEXT:    br label [[ATOMICRMW_END:%.*]]
 ; GFX942:       atomicrmw.end:
 ; GFX942-NEXT:    ret double [[RES]]
@@ -1227,35 +1243,201 @@ define half @test_atomicrmw_fadd_f16_local(ptr addrspace(3) %ptr, half %value) {
 }
 
 define double @test_atomicrmw_fadd_f64_flat(ptr %ptr, double %value) {
-; ALL-LABEL: @test_atomicrmw_fadd_f64_flat(
-; ALL-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
-; ALL-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
-; ALL:       atomicrmw.private:
-; ALL-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
-; ALL-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; ALL-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
-; ALL-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
-; ALL-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
-; ALL:       atomicrmw.global:
-; ALL-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
-; ALL-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; ALL:       atomicrmw.start:
-; ALL-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; ALL-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; ALL-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
-; ALL-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
-; ALL-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1:![0-9]+]]
-; ALL-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
-; ALL-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
-; ALL-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
-; ALL-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
-; ALL:       atomicrmw.end1:
-; ALL-NEXT:    br label [[ATOMICRMW_PHI]]
-; ALL:       atomicrmw.phi:
-; ALL-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
-; ALL-NEXT:    br label [[ATOMICRMW_END:%.*]]
-; ALL:       atomicrmw.end:
-; ALL-NEXT:    ret double [[RES]]
+; CI-LABEL: @test_atomicrmw_fadd_f64_flat(
+; CI-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; CI-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; CI:       atomicrmw.private:
+; CI-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; CI-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; CI-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; CI:       atomicrmw.global:
+; CI-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end1:
+; CI-NEXT:    br label [[ATOMICRMW_PHI]]
+; CI:       atomicrmw.phi:
+; CI-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; CI-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[RES]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX9-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX9-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX9:       atomicrmw.private:
+; GFX9-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX9-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX9-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX9:       atomicrmw.global:
+; GFX9-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end1:
+; GFX9-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX9:       atomicrmw.phi:
+; GFX9-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; GFX9-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX908-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX908-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX908:       atomicrmw.private:
+; GFX908-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX908-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX908-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX908:       atomicrmw.global:
+; GFX908-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end1:
+; GFX908-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX908:       atomicrmw.phi:
+; GFX908-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; GFX908-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[RES]]
+;
+; GFX90A-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX90A-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX90A-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX90A:       atomicrmw.shared:
+; GFX90A-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX90A-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] seq_cst, align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX90A:       atomicrmw.check.private:
+; GFX90A-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
+; GFX90A-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX90A:       atomicrmw.private:
+; GFX90A-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX90A-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
+; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
+; GFX90A-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP3]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX90A:       atomicrmw.global:
+; GFX90A-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX90A-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP6:%.*]] = bitcast double [[NEW2]] to i64
+; GFX90A-NEXT:    [[TMP7:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; GFX90A-NEXT:    [[TMP9]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end1:
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX90A:       atomicrmw.phi:
+; GFX90A-NEXT:    [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]
+; GFX90A-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX942-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX942-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX942:       atomicrmw.shared:
+; GFX942-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX942-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] seq_cst, align 8
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX942:       atomicrmw.check.private:
+; GFX942-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
+; GFX942-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX942:       atomicrmw.private:
+; GFX942-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX942-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
+; GFX942-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
+; GFX942-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP3]], align 8
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX942:       atomicrmw.global:
+; GFX942-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX942-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 8
+; GFX942-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX942:       atomicrmw.start:
+; GFX942-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]
+; GFX942-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX942-NEXT:    [[TMP6:%.*]] = bitcast double [[NEW2]] to i64
+; GFX942-NEXT:    [[TMP7:%.*]] = bitcast double [[LOADED]] to i64
+; GFX942-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] seq_cst seq_cst, align 8
+; GFX942-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; GFX942-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; GFX942-NEXT:    [[TMP9]] = bitcast i64 [[NEWLOADED]] to double
+; GFX942-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX942:       atomicrmw.end1:
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX942:       atomicrmw.phi:
+; GFX942-NEXT:    [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]
+; GFX942-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX942:       atomicrmw.end:
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX11-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX11-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX11:       atomicrmw.private:
+; GFX11-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX11-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX11-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX11:       atomicrmw.global:
+; GFX11-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end1:
+; GFX11-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX11:       atomicrmw.phi:
+; GFX11-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; GFX11-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw fadd ptr %ptr, double %value seq_cst
   ret double %res
@@ -1270,7 +1452,7 @@ define double @test_atomicrmw_fadd_f64_flat__noprivate(ptr %ptr, double %value)
 ; ALL-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
 ; ALL-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
 ; ALL-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; ALL-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]
+; ALL-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1:![0-9]+]]
 ; ALL-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
 ; ALL-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
 ; ALL-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
@@ -2814,69 +2996,396 @@ define float @test_atomicrmw_fadd_f32_flat_system_ret__amdgpu_ignore_denormal_mo
 }
 
 define void @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(ptr %ptr, double %value) #5 {
-; ALL-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(
-; ALL-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
-; ALL-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
-; ALL:       atomicrmw.private:
-; ALL-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
-; ALL-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; ALL-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
-; ALL-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
-; ALL-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
-; ALL:       atomicrmw.global:
-; ALL-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
-; ALL-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; ALL:       atomicrmw.start:
-; ALL-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; ALL-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; ALL-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
-; ALL-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
-; ALL-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
-; ALL-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
-; ALL-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
-; ALL-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
-; ALL-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
-; ALL:       atomicrmw.end1:
-; ALL-NEXT:    br label [[ATOMICRMW_PHI]]
-; ALL:       atomicrmw.phi:
-; ALL-NEXT:    br label [[ATOMICRMW_END:%.*]]
-; ALL:       atomicrmw.end:
-; ALL-NEXT:    ret void
+; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(
+; CI-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; CI-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; CI:       atomicrmw.private:
+; CI-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; CI-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; CI-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; CI:       atomicrmw.global:
+; CI-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end1:
+; CI-NEXT:    br label [[ATOMICRMW_PHI]]
+; CI:       atomicrmw.phi:
+; CI-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret void
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(
+; GFX9-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX9-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX9:       atomicrmw.private:
+; GFX9-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX9-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX9-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX9:       atomicrmw.global:
+; GFX9-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end1:
+; GFX9-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX9:       atomicrmw.phi:
+; GFX9-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret void
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(
+; GFX908-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX908-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX908:       atomicrmw.private:
+; GFX908-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX908-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX908-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX908:       atomicrmw.global:
+; GFX908-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end1:
+; GFX908-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX908:       atomicrmw.phi:
+; GFX908-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret void
+;
+; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(
+; GFX90A-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX90A-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX90A:       atomicrmw.shared:
+; GFX90A-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX90A-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX90A:       atomicrmw.check.private:
+; GFX90A-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
+; GFX90A-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX90A:       atomicrmw.private:
+; GFX90A-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX90A-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
+; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
+; GFX90A-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP3]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX90A:       atomicrmw.global:
+; GFX90A-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX90A-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP6:%.*]] = bitcast double [[NEW2]] to i64
+; GFX90A-NEXT:    [[TMP7:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; GFX90A-NEXT:    [[TMP9]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end1:
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX90A:       atomicrmw.phi:
+; GFX90A-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret void
+;
+; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(
+; GFX942-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX942-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX942:       atomicrmw.shared:
+; GFX942-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX942-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX942:       atomicrmw.check.private:
+; GFX942-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
+; GFX942-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX942:       atomicrmw.private:
+; GFX942-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX942-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
+; GFX942-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
+; GFX942-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP3]], align 8
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX942:       atomicrmw.global:
+; GFX942-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX942-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 8
+; GFX942-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX942:       atomicrmw.start:
+; GFX942-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]
+; GFX942-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX942-NEXT:    [[TMP6:%.*]] = bitcast double [[NEW2]] to i64
+; GFX942-NEXT:    [[TMP7:%.*]] = bitcast double [[LOADED]] to i64
+; GFX942-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 8
+; GFX942-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; GFX942-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; GFX942-NEXT:    [[TMP9]] = bitcast i64 [[NEWLOADED]] to double
+; GFX942-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX942:       atomicrmw.end1:
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX942:       atomicrmw.phi:
+; GFX942-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX942:       atomicrmw.end:
+; GFX942-NEXT:    ret void
+;
+; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(
+; GFX11-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX11-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX11:       atomicrmw.private:
+; GFX11-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX11-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX11-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX11:       atomicrmw.global:
+; GFX11-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end1:
+; GFX11-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX11:       atomicrmw.phi:
+; GFX11-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret void
 ;
   %unused = atomicrmw fadd ptr %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !0
   ret void
 }
 
 define double @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(ptr %ptr, double %value) #5 {
-; ALL-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(
-; ALL-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
-; ALL-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
-; ALL:       atomicrmw.private:
-; ALL-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
-; ALL-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; ALL-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
-; ALL-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
-; ALL-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
-; ALL:       atomicrmw.global:
-; ALL-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
-; ALL-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; ALL:       atomicrmw.start:
-; ALL-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; ALL-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; ALL-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
-; ALL-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
-; ALL-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
-; ALL-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
-; ALL-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
-; ALL-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
-; ALL-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
-; ALL:       atomicrmw.end1:
-; ALL-NEXT:    br label [[ATOMICRMW_PHI]]
-; ALL:       atomicrmw.phi:
-; ALL-NEXT:    [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
-; ALL-NEXT:    br label [[ATOMICRMW_END:%.*]]
-; ALL:       atomicrmw.end:
-; ALL-NEXT:    ret double [[RET]]
+; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(
+; CI-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; CI-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; CI:       atomicrmw.private:
+; CI-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; CI-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; CI-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; CI:       atomicrmw.global:
+; CI-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end1:
+; CI-NEXT:    br label [[ATOMICRMW_PHI]]
+; CI:       atomicrmw.phi:
+; CI-NEXT:    [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; CI-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[RET]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(
+; GFX9-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX9-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX9:       atomicrmw.private:
+; GFX9-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX9-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX9-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX9:       atomicrmw.global:
+; GFX9-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end1:
+; GFX9-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX9:       atomicrmw.phi:
+; GFX9-NEXT:    [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; GFX9-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[RET]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(
+; GFX908-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX908-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX908:       atomicrmw.private:
+; GFX908-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX908-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX908-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX908:       atomicrmw.global:
+; GFX908-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end1:
+; GFX908-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX908:       atomicrmw.phi:
+; GFX908-NEXT:    [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; GFX908-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[RET]]
+;
+; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(
+; GFX90A-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX90A-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX90A:       atomicrmw.shared:
+; GFX90A-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX90A-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX90A:       atomicrmw.check.private:
+; GFX90A-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
+; GFX90A-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX90A:       atomicrmw.private:
+; GFX90A-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX90A-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
+; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
+; GFX90A-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP3]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX90A:       atomicrmw.global:
+; GFX90A-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX90A-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP6:%.*]] = bitcast double [[NEW2]] to i64
+; GFX90A-NEXT:    [[TMP7:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; GFX90A-NEXT:    [[TMP9]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end1:
+; GFX90A-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX90A:       atomicrmw.phi:
+; GFX90A-NEXT:    [[RET:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]
+; GFX90A-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[RET]]
+;
+; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(
+; GFX942-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])
+; GFX942-NEXT:    br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]
+; GFX942:       atomicrmw.shared:
+; GFX942-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX942-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX942:       atomicrmw.check.private:
+; GFX942-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
+; GFX942-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX942:       atomicrmw.private:
+; GFX942-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX942-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
+; GFX942-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
+; GFX942-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP3]], align 8
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX942:       atomicrmw.global:
+; GFX942-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX942-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 8
+; GFX942-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX942:       atomicrmw.start:
+; GFX942-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]
+; GFX942-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX942-NEXT:    [[TMP6:%.*]] = bitcast double [[NEW2]] to i64
+; GFX942-NEXT:    [[TMP7:%.*]] = bitcast double [[LOADED]] to i64
+; GFX942-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 8
+; GFX942-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; GFX942-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; GFX942-NEXT:    [[TMP9]] = bitcast i64 [[NEWLOADED]] to double
+; GFX942-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX942:       atomicrmw.end1:
+; GFX942-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX942:       atomicrmw.phi:
+; GFX942-NEXT:    [[RET:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]
+; GFX942-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX942:       atomicrmw.end:
+; GFX942-NEXT:    ret double [[RET]]
+;
+; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(
+; GFX11-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])
+; GFX11-NEXT:    br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]
+; GFX11:       atomicrmw.private:
+; GFX11-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
+; GFX11-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]
+; GFX11-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_PHI:%.*]]
+; GFX11:       atomicrmw.global:
+; GFX11-NEXT:    [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end1:
+; GFX11-NEXT:    br label [[ATOMICRMW_PHI]]
+; GFX11:       atomicrmw.phi:
+; GFX11-NEXT:    [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]
+; GFX11-NEXT:    br label [[ATOMICRMW_END:%.*]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[RET]]
 ;
   %ret = atomicrmw fadd ptr %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !0
   ret double %ret
diff --git a/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll b/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll
index 7692fd34312f..1bf821fd53ea 100644
--- a/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll
+++ b/external/llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll
@@ -577,6 +577,13 @@ define double @test_flat_atomicrmw_fadd_f64_agent(ptr %ptr, double %value) {
 ;
 ; GFX90A-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
 ; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
+; GFX90A-NEXT:    br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]
+; GFX90A:       [[ATOMICRMW_SHARED]]:
+; GFX90A-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX90A-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
+; GFX90A-NEXT:    br label %[[ATOMICRMW_PHI:.*]]
+; GFX90A:       [[ATOMICRMW_CHECK_PRIVATE]]:
 ; GFX90A-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
 ; GFX90A-NEXT:    br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
 ; GFX90A:       [[ATOMICRMW_PRIVATE]]:
@@ -584,18 +591,26 @@ define double @test_flat_atomicrmw_fadd_f64_agent(ptr %ptr, double %value) {
 ; GFX90A-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
 ; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
 ; GFX90A-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
-; GFX90A-NEXT:    br label %[[ATOMICRMW_PHI:.*]]
+; GFX90A-NEXT:    br label %[[ATOMICRMW_PHI]]
 ; GFX90A:       [[ATOMICRMW_GLOBAL]]:
-; GFX90A-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
+; GFX90A-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX90A-NEXT:    [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
 ; GFX90A-NEXT:    br label %[[ATOMICRMW_PHI]]
 ; GFX90A:       [[ATOMICRMW_PHI]]:
-; GFX90A-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
+; GFX90A-NEXT:    [[RES:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]
 ; GFX90A-NEXT:    br label %[[ATOMICRMW_END:.*]]
 ; GFX90A:       [[ATOMICRMW_END]]:
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
 ; GFX942-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
 ; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
+; GFX942-NEXT:    br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]
+; GFX942:       [[ATOMICRMW_SHARED]]:
+; GFX942-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
+; GFX942-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
+; GFX942-NEXT:    br label %[[ATOMICRMW_PHI:.*]]
+; GFX942:       [[ATOMICRMW_CHECK_PRIVATE]]:
 ; GFX942-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
 ; GFX942-NEXT:    br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
 ; GFX942:       [[ATOMICRMW_PRIVATE]]:
@@ -603,12 +618,13 @@ define double @test_flat_atomicrmw_fadd_f64_agent(ptr %ptr, double %value) {
 ; GFX942-NEXT:    [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
 ; GFX942-NEXT:    [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
 ; GFX942-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
-; GFX942-NEXT:    br label %[[ATOMICRMW_PHI:.*]]
+; GFX942-NEXT:    br label %[[ATOMICRMW_PHI]]
 ; GFX942:       [[ATOMICRMW_GLOBAL]]:
-; GFX942-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
+; GFX942-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; GFX942-NEXT:    [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
 ; GFX942-NEXT:    br label %[[ATOMICRMW_PHI]]
 ; GFX942:       [[ATOMICRMW_PHI]]:
-; GFX942-NEXT:    [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
+; GFX942-NEXT:    [[RES:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]
 ; GFX942-NEXT:    br label %[[ATOMICRMW_END:.*]]
 ; GFX942:       [[ATOMICRMW_END]]:
 ; GFX942-NEXT:    ret double [[RES]]
diff --git a/external/llvm-project/llvm/test/Transforms/AtomicExpand/PowerPC/atomicrmw-fp.ll b/external/llvm-project/llvm/test/Transforms/AtomicExpand/PowerPC/atomicrmw-fp.ll
index f787aa7f6a42..840e2d3eee55 100644
--- a/external/llvm-project/llvm/test/Transforms/AtomicExpand/PowerPC/atomicrmw-fp.ll
+++ b/external/llvm-project/llvm/test/Transforms/AtomicExpand/PowerPC/atomicrmw-fp.ll
@@ -7,19 +7,51 @@ define float @test_atomicrmw_fadd_f32(ptr %ptr, float %value) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; CHECK:       atomicrmw.start:
-; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[NEW]] to i32
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 4
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
-; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
-; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    %loaded = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %cmpxchg.end ]
+; CHECK-NEXT:    %new = fadd float %loaded, %value
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float %new to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float %loaded to i32
+; CHECK-NEXT:    br label %cmpxchg.start
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.start:                                    ; preds = %cmpxchg.trystore, %atomicrmw.start
+; CHECK-NEXT:    %larx = call i32 @llvm.ppc.lwarx(ptr %ptr)
+; CHECK-NEXT:    %should_store = icmp eq i32 %larx, [[TMP3]]
+; CHECK-NEXT:    br i1 %should_store, label %cmpxchg.fencedstore, label %cmpxchg.nostore
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.fencedstore:                              ; preds = %cmpxchg.start
+; CHECK-NEXT:    br label %cmpxchg.trystore
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.trystore:                                 ; preds = %cmpxchg.fencedstore
+; CHECK-NEXT:    %loaded.trystore = phi i32 [ %larx, %cmpxchg.fencedstore ]
+; CHECK-NEXT:    %stcx = call i32 @llvm.ppc.stwcx(ptr %ptr, i32 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 %stcx, 1
+; CHECK-NEXT:    %success1 = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 %success1, label %cmpxchg.success, label %cmpxchg.start
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.releasedload:                             ; No predecessors!
+; CHECK-NEXT:    unreachable
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.success:                                  ; preds = %cmpxchg.trystore
+; CHECK-NEXT:    br label %cmpxchg.end
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.nostore:                                  ; preds = %cmpxchg.start
+; CHECK-NEXT:    %loaded.nostore = phi i32 [ %larx, %cmpxchg.start ]
+; CHECK-NEXT:    br label %cmpxchg.failure
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.failure:                                  ; preds = %cmpxchg.nostore
+; CHECK-NEXT:    %loaded.failure = phi i32 [ %loaded.nostore, %cmpxchg.nostore ]
+; CHECK-NEXT:    br label %cmpxchg.end
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.end:                                      ; preds = %cmpxchg.failure, %cmpxchg.success
+; CHECK-NEXT:    %loaded.exit = phi i32 [ %loaded.trystore, %cmpxchg.success ], [ %loaded.failure, %cmpxchg.failure ]
+; CHECK-NEXT:    %success2 = phi i1 [ true, %cmpxchg.success ], [ false, %cmpxchg.failure ]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 %loaded.exit to float
+; CHECK-NEXT:    br i1 %success2, label %atomicrmw.end, label %atomicrmw.start
+; CHECK-EMPTY:
+; CHECK-NEXT:  atomicrmw.end:                                    ; preds = %cmpxchg.end
 ; CHECK-NEXT:    call void @llvm.ppc.lwsync()
 ; CHECK-NEXT:    ret float [[TMP5]]
-;
+; CHECK-NEXT:  }
   %res = atomicrmw fadd ptr %ptr, float %value seq_cst
   ret float %res
 }
@@ -28,22 +60,56 @@ define float @test_atomicrmw_fsub_f32(ptr %ptr, float %value) {
 ; CHECK-LABEL: @test_atomicrmw_fsub_f32(
 ; CHECK-NEXT:    call void @llvm.ppc.sync()
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4
-; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; CHECK:       atomicrmw.start:
-; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[NEW]] to i32
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 4
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
-; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
-; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    br label %atomicrmw.start 
+; CHECK-EMPTY:
+; CHECK-NEXT:  atomicrmw.start:
+; CHECK-NEXT:    %loaded = phi float [ [[TMP1]], %0 ], [ [[TMP5:%.*]], %cmpxchg.end ]
+; CHECK-NEXT:    %new = fsub float %loaded, %value
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float %new to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float %loaded to i32
+; CHECK-NEXT:    br label %cmpxchg.start
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.start:
+; CHECK-NEXT:    %larx = call i32 @llvm.ppc.lwarx(ptr %ptr)
+; CHECK-NEXT:    %should_store = icmp eq i32 %larx, [[TMP3]]
+; CHECK-NEXT:    br i1 %should_store, label %cmpxchg.fencedstore, label %cmpxchg.nostore
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.fencedstore:                              ; preds = %cmpxchg.start
+; CHECK-NEXT:    br label %cmpxchg.trystore
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.trystore:                                 ; preds = %cmpxchg.fencedstore
+; CHECK-NEXT:    %loaded.trystore = phi i32 [ %larx, %cmpxchg.fencedstore ]
+; CHECK-NEXT:    %stcx = call i32 @llvm.ppc.stwcx(ptr %ptr, i32 %2)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 %stcx, 1
+; CHECK-NEXT:    %success1 = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 %success1, label %cmpxchg.success, label %cmpxchg.start
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.releasedload:                             ; No predecessors!
+; CHECK-NEXT:    unreachable
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.success:                                  ; preds = %cmpxchg.trystore
+; CHECK-NEXT:    br label %cmpxchg.end
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.nostore:                                  ; preds = %cmpxchg.start
+; CHECK-NEXT:    %loaded.nostore = phi i32 [ %larx, %cmpxchg.start ]
+; CHECK-NEXT:    br label %cmpxchg.failure
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.failure:                                  ; preds = %cmpxchg.nostore
+; CHECK-NEXT:    %loaded.failure = phi i32 [ %loaded.nostore, %cmpxchg.nostore ]
+; CHECK-NEXT:    br label %cmpxchg.end
+; CHECK-EMPTY:
+; CHECK-NEXT:  cmpxchg.end:                                      ; preds = %cmpxchg.failure, %cmpxchg.success
+; CHECK-NEXT:    %loaded.exit = phi i32 [ %loaded.trystore, %cmpxchg.success ], [ %loaded.failure, %cmpxchg.failure ]
+; CHECK-NEXT:    %success2 = phi i1 [ true, %cmpxchg.success ], [ false, %cmpxchg.failure ]
+; CHECK-NEXT:    [[TMP5]] = bitcast i32 %loaded.exit to float
+; CHECK-NEXT:    br i1 %success2, label %atomicrmw.end, label %atomicrmw.start
+; CHECK-EMPTY:
+; CHECK-NEXT:  atomicrmw.end:                                    ; preds = %cmpxchg.end
 ; CHECK-NEXT:    call void @llvm.ppc.lwsync()
 ; CHECK-NEXT:    ret float [[TMP5]]
-;
-  %res = atomicrmw fsub ptr %ptr, float %value seq_cst
+; CHECK-NEXT:  }
+
+%res = atomicrmw fsub ptr %ptr, float %value seq_cst
   ret float %res
 }
 
diff --git a/external/llvm-project/llvm/test/Transforms/Attributor/nofpclass.ll b/external/llvm-project/llvm/test/Transforms/Attributor/nofpclass.ll
index cfc787738388..6491f01a6d2a 100644
--- a/external/llvm-project/llvm/test/Transforms/Attributor/nofpclass.ll
+++ b/external/llvm-project/llvm/test/Transforms/Attributor/nofpclass.ll
@@ -1474,7 +1474,7 @@ define float @returned_extractelement_index_oob(<4 x float> nofpclass(nan) %vec)
 
 define float @returned_extractelement_scalable(<vscale x 4 x float> nofpclass(nan) %vec) {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define float @returned_extractelement_scalable
+; CHECK-LABEL: define nofpclass(nan) float @returned_extractelement_scalable
 ; CHECK-SAME: (<vscale x 4 x float> nofpclass(nan) [[VEC:%.*]]) #[[ATTR3]] {
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <vscale x 4 x float> [[VEC]], i32 0
 ; CHECK-NEXT:    ret float [[EXTRACT]]
@@ -2665,7 +2665,7 @@ define [4 x float] @constant_aggregate_zero() {
 
 define <vscale x 4 x float> @scalable_splat_pnorm() {
 ; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define <vscale x 4 x float> @scalable_splat_pnorm
+; CHECK-LABEL: define noundef <vscale x 4 x float> @scalable_splat_pnorm
 ; CHECK-SAME: () #[[ATTR3]] {
 ; CHECK-NEXT:    ret <vscale x 4 x float> splat (float 1.000000e+00)
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-reg-does-not-geps.ll b/external/llvm-project/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-reg-does-not-geps.ll
new file mode 100644
index 000000000000..1640bafbd0bf
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-reg-does-not-geps.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux"
+
+declare i1 @cond(float)
+
+define void @scaled_reg_does_not_dominate_insert_point(ptr %src) {
+; CHECK-LABEL: define void @scaled_reg_does_not_dominate_insert_point(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[SUNKADDR2:%.*]] = mul i64 [[IV_NEXT]], 2
+; CHECK-NEXT:    [[SUNKADDR3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[SUNKADDR2]]
+; CHECK-NEXT:    [[SUNKADDR4:%.*]] = getelementptr i8, ptr [[SUNKADDR3]], i64 6
+; CHECK-NEXT:    [[L_0:%.*]] = load float, ptr [[SUNKADDR4]], align 4
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[SUNKADDR1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[SUNKADDR]]
+; CHECK-NEXT:    [[L_1:%.*]] = load float, ptr [[SUNKADDR1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @cond(float [[L_0]])
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond(float [[L_1]])
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %gep.base = getelementptr i8, ptr %src, i64 8
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %bb ], [ %iv.next, %loop ]
+  %iv.shl = shl i64 %iv, 1
+  %gep.shl = getelementptr i8, ptr %gep.base, i64 %iv.shl
+  %gep.sub = getelementptr i8, ptr %gep.shl, i64 -8
+  %iv.next = add i64 %iv, 1
+  %l.0 = load float, ptr %gep.shl, align 4
+  %l.1 = load float, ptr %gep.sub, align 4
+  call i1 @cond(float %l.0)
+  %c = call i1 @cond(float %l.1)
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @check_dt_after_modifying_cfg(ptr %dst, i64 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: define void @check_dt_after_modifying_cfg(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[OFFSET:%.*]] = lshr i64 [[X]], 2
+; CHECK-NEXT:    [[SEL_FROZEN:%.*]] = freeze i8 [[Z]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[SEL_FROZEN]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[SELECT_END:.*]], label %[[SELECT_FALSE_SINK:.*]]
+; CHECK:       [[SELECT_FALSE_SINK]]:
+; CHECK-NEXT:    [[SMIN:%.*]] = tail call i8 @llvm.smin.i8(i8 [[Y]], i8 0)
+; CHECK-NEXT:    br label %[[SELECT_END]]
+; CHECK:       [[SELECT_END]]:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[SMIN]], %[[SELECT_FALSE_SINK]] ]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET]]
+; CHECK-NEXT:    store i8 [[SEL]], ptr [[SUNKADDR]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %offset = lshr i64 %x, 2
+  %gep.dst = getelementptr i8, ptr %dst, i64 %offset
+  %smin = tail call i8 @llvm.smin.i8(i8 %y, i8 0)
+  %cmp = icmp slt i8 %z, 0
+  %sel = select i1 %cmp, i8 0, i8 %smin
+  store i8 %sel, ptr %gep.dst, align 1
+  ret void
+}
+
+declare i8 @llvm.smin.i8(i8, i8) #0
diff --git a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll
index 4da07c91eb48..deaec7b8d7f8 100644
--- a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll
+++ b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll
@@ -2,18 +2,18 @@
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
 ;
 ; This file is based on coro-debug-frame-variable.ll.
-; CHECK:  define internal fastcc void @f.resume(ptr noundef nonnull align 16 dereferenceable(80) %begin) !dbg ![[RESUME_FN_DBG_NUM:[0-9]+]]
+; CHECK:  define internal fastcc void @_Z3foov.resume(ptr noundef nonnull align 16 dereferenceable(80) %begin) !dbg ![[RESUME_FN_DBG_NUM:[0-9]+]]
 ; CHECK:       await.ready:
 ; CHECK:         #dbg_value(i32 poison, ![[IVAR_RESUME:[0-9]+]], !DIExpression(
 ; CHECK:         #dbg_value(i32 poison, ![[JVAR_RESUME:[0-9]+]], !DIExpression(
 ;
-; CHECK: ![[RESUME_FN_DBG_NUM]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov"
+; CHECK: ![[RESUME_FN_DBG_NUM]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov.resume"
 ; CHECK: ![[IVAR_RESUME]] = !DILocalVariable(name: "i"
 ; CHECK: ![[JVAR_RESUME]] = !DILocalVariable(name: "j"
 
 source_filename = "../llvm/test/Transforms/Coroutines/coro-debug-dbg.values-O2.ll"
 
-define void @f(i32 %i, i32 %j) presplitcoroutine !dbg !8 {
+define void @_Z3foov(i32 %i, i32 %j) presplitcoroutine !dbg !8 {
 entry:
   %__promise = alloca i8, align 8
   %x = alloca [10 x i32], align 16
diff --git a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
index 28592cc67106..5f7701c357ec 100644
--- a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
+++ b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
 ;
 ; This file is based on coro-debug-frame-variable.ll.
-; CHECK-LABEL: define void @f(
+; CHECK-LABEL: define void @_Z3foov(
 ; CHECK:       %[[frame:.*]] = call {{.*}} @llvm.coro.begin
 ; CHECK:       #dbg_value(ptr %[[frame]]
 ; CHECK-SAME:    !DIExpression(DW_OP_plus_uconst, [[OffsetX:[0-9]*]]),
@@ -20,7 +20,7 @@
 ; CHECK:       #dbg_value(ptr %[[frame]]
 ; CHECK-SAME:    !DIExpression(DW_OP_plus_uconst, [[OffsetJ:[0-9]*]], DW_OP_deref),
 
-; CHECK-LABEL: void @f.resume(
+; CHECK-LABEL: void @_Z3foov.resume(
 ; CHECK-SAME:                 ptr {{.*}} %[[frame:.*]])
 ; CHECK-SAME:  !dbg ![[RESUME_FN_DBG_NUM:[0-9]+]]
 ; CHECK:         %[[frame_alloca:.*]] = alloca ptr
@@ -37,7 +37,7 @@
 ; CHECK:         #dbg_value(ptr %[[frame_alloca]], ![[JVAR_RESUME:[0-9]+]],
 ; CHECK-SAME:        !DIExpression(DW_OP_deref, DW_OP_plus_uconst, [[OffsetJ]], DW_OP_deref)
 ;
-; CHECK: ![[RESUME_FN_DBG_NUM]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov"
+; CHECK: ![[RESUME_FN_DBG_NUM]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov.resume"
 ; CHECK: ![[FRAME_DI_NUM]] = !DILocalVariable(name: "__coro_frame"
 ; CHECK: ![[IVAR_RESUME]] = !DILocalVariable(name: "i"
 ; CHECK: ![[XVAR_RESUME]] = !DILocalVariable(name: "x"
@@ -46,7 +46,7 @@
 
 declare void @consume(i32)
 
-define void @f(i32 %i, i32 %j) presplitcoroutine !dbg !8 {
+define void @_Z3foov(i32 %i, i32 %j) presplitcoroutine !dbg !8 {
 entry:
   %__promise = alloca i8, align 8
   %x = alloca [10 x i32], align 16
@@ -257,4 +257,4 @@ attributes #4 = { argmemonly nofree nosync nounwind willreturn writeonly }
 !21 = !DILocation(line: 43, column: 3, scope: !7)
 !22 = !DILocation(line: 43, column: 8, scope: !7)
 !23 = !DILocalVariable(name: "produced", scope: !7, file: !1, line:24, type: !10)
-!30 = distinct !DIAssignID()
\ No newline at end of file
+!30 = distinct !DIAssignID()
diff --git a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
index a3c62b2dd12e..125ec752c834 100644
--- a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
+++ b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
@@ -23,11 +23,11 @@
 ;
 ; The CHECKs verify that dbg.declare intrinsics are created for the coroutine
 ; funclet 'f.resume', and that they reference the address of the variables on
-; the coroutine frame. The debug locations for the original function 'f' are
+; the coroutine frame. The debug locations for the original function 'foo' are
 ; static (!11 and !13), whereas the coroutine funclet will have its own new
 ; ones with identical line and column numbers.
 ;
-; CHECK-LABEL: define void @f() {{.*}} {
+; CHECK-LABEL: define void @_Z3foov() {{.*}} {
 ; CHECK:       entry:
 ; CHECK:         %j = alloca i32, align 4
 ; CHECK:         #dbg_declare(ptr %j, ![[JVAR:[0-9]+]], !DIExpression(), ![[JDBGLOC:[0-9]+]]
@@ -36,7 +36,7 @@
 ; CHECK:         #dbg_declare(ptr %[[MEMORY]], ![[IVAR:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 20), ![[IDBGLOC]]
 ; CHECK:       await.ready:
 ;
-; CHECK-LABEL: define internal fastcc void @f.resume({{.*}}) {{.*}} {
+; CHECK-LABEL: define internal fastcc void @_Z3foov.resume({{.*}}) {{.*}} {
 ; CHECK:       entry.resume:
 ; CHECK-NEXT:    %[[DBG_PTR:.*]] = alloca ptr
 ; CHECK-NEXT:    #dbg_declare(ptr %[[DBG_PTR]], ![[XVAR_RESUME:[0-9]+]],   !DIExpression(DW_OP_deref, DW_OP_plus_uconst, 32),
@@ -58,13 +58,13 @@
 ; CHECK-DAG: ![[JDBGLOC]] = !DILocation(line: 32, column: 7, scope: ![[BLK_SCOPE]])
 
 ; CHECK-DAG: ![[XVAR_RESUME]] = !DILocalVariable(name: "x"
-; CHECK-DAG: ![[RESUME_PROG_SCOPE:[0-9]+]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov"
+; CHECK-DAG: ![[RESUME_PROG_SCOPE:[0-9]+]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov.resume"
 ; CHECK-DAG: ![[IDBGLOC_RESUME]] = !DILocation(line: 24, column: 7, scope: ![[RESUME_BLK_SCOPE:[0-9]+]])
 ; CHECK-DAG: ![[RESUME_BLK_SCOPE]] = distinct !DILexicalBlock(scope: ![[RESUME_PROG_SCOPE]], file: !1, line: 23, column: 12)
 ; CHECK-DAG: ![[IVAR_RESUME]] = !DILocalVariable(name: "i"
 ; CHECK-DAG: ![[JVAR_RESUME]] = !DILocalVariable(name: "j"
 ; CHECK-DAG: ![[JDBGLOC_RESUME]] = !DILocation(line: 32, column: 7, scope: ![[RESUME_BLK_SCOPE]])
-define void @f() presplitcoroutine !dbg !8 {
+define void @_Z3foov() presplitcoroutine !dbg !8 {
 entry:
   %__promise = alloca i8, align 8
   %i = alloca i32, align 4
diff --git a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug.ll b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug.ll
index 17a0b80c5b5e..a220073248ba 100644
--- a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug.ll
+++ b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-debug.ll
@@ -6,12 +6,12 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: noinline nounwind
-define ptr @f(i32 %x) #0 personality i32 0 !dbg !6 {
+define ptr @flink(i32 %x) #0 personality i32 0 !dbg !6 {
 entry:
   %x.addr = alloca i32, align 4
   %coro_hdl = alloca ptr, align 8
   store i32 %x, ptr %x.addr, align 4
-  %0 = call token @llvm.coro.id(i32 0, ptr null, ptr @f, ptr null), !dbg !16
+  %0 = call token @llvm.coro.id(i32 0, ptr null, ptr @flink, ptr null), !dbg !16
   %1 = call i64 @llvm.coro.size.i64(), !dbg !16
   %call = call ptr @malloc(i64 %1), !dbg !16
   %2 = call ptr @llvm.coro.begin(token %0, ptr %call) #7, !dbg !16
@@ -170,8 +170,8 @@ attributes #7 = { noduplicate }
 !31 = !DILocalVariable(name: "allocated", scope: !6, file: !7, line: 55, type: !11)
 !32 = !DILocalVariable(name: "inline_asm", scope: !6, file: !7, line: 55, type: !11)
 
-; CHECK: define ptr @f(i32 %x) #0 personality i32 0 !dbg ![[ORIG:[0-9]+]]
-; CHECK: define internal fastcc void @f.resume(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[RESUME:[0-9]+]]
+; CHECK: define ptr @flink(i32 %x) #0 personality i32 0 !dbg ![[ORIG:[0-9]+]]
+; CHECK: define internal fastcc void @flink.resume(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[RESUME:[0-9]+]]
 ; CHECK: entry.resume:
 ; CHECK: %[[DBG_PTR:.*]] = alloca ptr
 ; CHECK: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_COROHDL:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst,
@@ -194,18 +194,18 @@ attributes #7 = { noduplicate }
 ; CHECK: [[DEFAULT_DEST]]:
 ; CHECK-NOT: {{.*}}:
 ; CHECK: #dbg_value(i32 %[[CALLBR_RES]]
-; CHECK: define internal fastcc void @f.destroy(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[DESTROY:[0-9]+]]
-; CHECK: define internal fastcc void @f.cleanup(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[CLEANUP:[0-9]+]]
+; CHECK: define internal fastcc void @flink.destroy(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[DESTROY:[0-9]+]]
+; CHECK: define internal fastcc void @flink.cleanup(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[CLEANUP:[0-9]+]]
 
 ; CHECK: ![[ORIG]] = distinct !DISubprogram(name: "f", linkageName: "flink"
 
-; CHECK: ![[RESUME]] = distinct !DISubprogram(name: "f", linkageName: "flink"
+; CHECK: ![[RESUME]] = distinct !DISubprogram(name: "f", linkageName: "flink.resume"
 ; CHECK: ![[RESUME_COROHDL]] = !DILocalVariable(name: "coro_hdl", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_X]] = !DILocalVariable(name: "x", arg: 1, scope: ![[RESUME]]
 ; CHECK: ![[RESUME_CONST]] = !DILocalVariable(name: "direct_const", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_DIRECT]] = !DILocalVariable(name: "direct_mem", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_DIRECT_VALUE]] = !DILocalVariable(name: "direct_value", scope: ![[RESUME]]
 
-; CHECK: ![[DESTROY]] = distinct !DISubprogram(name: "f", linkageName: "flink"
+; CHECK: ![[DESTROY]] = distinct !DISubprogram(name: "f", linkageName: "flink.destroy"
 
-; CHECK: ![[CLEANUP]] = distinct !DISubprogram(name: "f", linkageName: "flink"
+; CHECK: ![[CLEANUP]] = distinct !DISubprogram(name: "f", linkageName: "flink.cleanup"
diff --git a/external/llvm-project/llvm/test/Transforms/Coroutines/coro-noop-pacbti.ll b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-noop-pacbti.ll
new file mode 100644
index 000000000000..4f302a6acc64
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/Coroutines/coro-noop-pacbti.ll
@@ -0,0 +1,23 @@
+
+; RUN: opt < %s -S -passes=coro-early | FileCheck %s
+
+; CHECK:      define private fastcc void @__NoopCoro_ResumeDestroy(ptr %0) #1 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+
+; CHECK:      attributes #1 = { "branch-target-enforcement" "sign-return-address"="all" "sign-return-address-key"="a_key" }
+
+define ptr @noop() {
+entry:
+  %n = call ptr @llvm.coro.noop()
+  ret ptr %n
+}
+
+declare ptr @llvm.coro.noop()
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 8, !"branch-target-enforcement", i32 1}
+!1 = !{i32 8, !"sign-return-address", i32 1}
+!2 = !{i32 8, !"sign-return-address-all", i32 1}
diff --git a/external/llvm-project/llvm/test/Transforms/CorrelatedValuePropagation/switch.ll b/external/llvm-project/llvm/test/Transforms/CorrelatedValuePropagation/switch.ll
index a0794d5efe93..7e6aa3eeebe2 100644
--- a/external/llvm-project/llvm/test/Transforms/CorrelatedValuePropagation/switch.ll
+++ b/external/llvm-project/llvm/test/Transforms/CorrelatedValuePropagation/switch.ll
@@ -294,6 +294,42 @@ cleanup:
   ret i32 %retval.0
 }
 
+; Make sure that we don't branch into unreachable.
+
+define void @pr142286() {
+; CHECK-LABEL: define void @pr142286() {
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br label [[LOOP2:%.*]]
+; CHECK:       loop2:
+; CHECK-NEXT:    br label [[LOOP3:%.*]]
+; CHECK:       loop3:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+start:
+  br label %loop
+
+loop:
+  %phi = phi i8 [ -1, %start ], [ 0, %loop3 ]
+  br label %loop2
+
+loop2:
+  br label %loop3
+
+loop3:
+  switch i8 %phi, label %exit [
+  i8 0, label %loop3
+  i8 1, label %loop2
+  i8 2, label %loop
+  ]
+
+exit:
+  ret void
+}
+
 declare i32 @call0()
 declare i32 @call1()
 declare i32 @call2()
diff --git a/external/llvm-project/llvm/test/Transforms/DeadStoreElimination/noop-stores.ll b/external/llvm-project/llvm/test/Transforms/DeadStoreElimination/noop-stores.ll
index 9fc20d76da5e..59a1cc180a19 100644
--- a/external/llvm-project/llvm/test/Transforms/DeadStoreElimination/noop-stores.ll
+++ b/external/llvm-project/llvm/test/Transforms/DeadStoreElimination/noop-stores.ll
@@ -374,6 +374,19 @@ define ptr @notmalloc_memset(i64 %size, ptr %notmalloc) {
   ret ptr %call1
 }
 
+; This should create a customalloc_zeroed call and eliminate the memset
+define ptr @customalloc_memset(i64 %size, i64 %align) {
+; CHECK-LABEL: @customalloc_memset
+; CHECK-NEXT:  [[CALL:%.*]] = call ptr @customalloc_zeroed(i64 [[SIZE:%.*]], i64 [[ALIGN:%.*]])
+; CHECK-NEXT:  ret ptr [[CALL]]
+  %call = call ptr @customalloc(i64 %size, i64 %align)
+  call void @llvm.memset.p0.i64(ptr %call, i8 0, i64 %size, i1 false)
+  ret ptr %call
+}
+
+declare ptr @customalloc(i64, i64) allockind("alloc") "alloc-family"="customalloc" "alloc-variant-zeroed"="customalloc_zeroed"
+declare ptr @customalloc_zeroed(i64, i64) allockind("alloc,zeroed") "alloc-family"="customalloc"
+
 ; This should not create recursive call to calloc.
 define ptr @calloc(i64 %nmemb, i64 %size) inaccessiblememonly {
 ; CHECK-LABEL: @calloc(
diff --git a/external/llvm-project/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll b/external/llvm-project/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
new file mode 100644
index 000000000000..e390d4bdca63
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -passes='dse,verify<memoryssa>' -S | FileCheck %s
+
+declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind
+
+; This should create a declaration for the named variant
+define ptr @undeclared_customalloc(i64 %size, i64 %align) {
+; CHECK-LABEL: @undeclared_customalloc
+; CHECK-NEXT:  [[CALL:%.*]] = call ptr @customalloc2_zeroed(i64 [[SIZE:%.*]], i64 [[ALIGN:%.*]])
+; CHECK-NEXT:  ret ptr [[CALL]]
+  %call = call ptr @customalloc2(i64 %size, i64 %align)
+  call void @llvm.memset.p0.i64(ptr %call, i8 0, i64 %size, i1 false)
+  ret ptr %call
+}
+
+declare ptr @customalloc2(i64, i64) allockind("alloc") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
+; CHECK-DAG: declare ptr @customalloc2_zeroed(i64, i64) #[[CA2ATTR:[0-9]+]]
+; CHECK-DAG: attributes #[[CA2ATTR]] = { allockind("alloc,zeroed") "alloc-family"="customalloc2" }
diff --git a/external/llvm-project/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll b/external/llvm-project/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll
index 2adaf1c7b67a..cf871e5714bf 100644
--- a/external/llvm-project/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll
+++ b/external/llvm-project/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll
@@ -13,7 +13,7 @@ declare i8 @buz.fp(float, float)
 define i8 @same_parent_combine_diff_attrs(i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs(
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C1]], i8 [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -27,7 +27,7 @@ define i8 @same_parent_combine_diff_attrs(i8 %x, i8 %y) {
 define i8 @same_parent_combine_diff_attrs_needs_intersect(i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_needs_intersect(
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -41,7 +41,7 @@ define i8 @same_parent_combine_diff_attrs_needs_intersect(i8 %x, i8 %y) {
 define i8 @same_parent_combine_diff_attrs_fmf(float %x, float %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fmf(
 ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.fp(float [[C1]], float [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -55,7 +55,7 @@ define i8 @same_parent_combine_diff_attrs_fmf(float %x, float %y) {
 define i8 @same_parent_combine_diff_attrs_fmf2(float %x, float %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fmf2(
 ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR1]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.fp(float [[C1]], float [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -69,7 +69,7 @@ define i8 @same_parent_combine_diff_attrs_fmf2(float %x, float %y) {
 define i8 @same_parent_combine_diff_attrs_needs_intersect2(i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_needs_intersect2(
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -83,7 +83,7 @@ define i8 @same_parent_combine_diff_attrs_needs_intersect2(i8 %x, i8 %y) {
 define i8 @same_parent_combine_diff_attrs_really_needs_intersect(i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_really_needs_intersect(
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr noundef [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -112,7 +112,7 @@ define i8 @same_parent_combine_diff_attrs_fail_side_effects(i8 %x, i8 %y) {
 define i8 @same_parent_combine_diff_attrs_quasi_side_effects2(i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_quasi_side_effects2(
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR1]]
 ; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]])
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -127,10 +127,10 @@ define i8 @same_parent_combine_diff_attrs_quasi_side_effects2(i8 %x, i8 %y) {
 define i8 @diff_parent_combine_diff_attrs(i1 %c, i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs(
 ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
 ; CHECK:       [[T]]:
-; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ; CHECK:       [[F]]:
@@ -151,7 +151,7 @@ F:
 define i8 @diff_parent_combine_diff_attrs_preserves_return_attrs(i1 %c, i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_preserves_return_attrs(
 ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call nonnull ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[C1:%.*]] = call nonnull ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]]
 ; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
 ; CHECK:       [[T]]:
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr noundef [[C1]])
@@ -172,8 +172,8 @@ F:
 define i8 @same_parent_combine_diff_attrs_todo(i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_todo(
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
-; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -187,8 +187,8 @@ define i8 @same_parent_combine_diff_attrs_todo(i8 %x, i8 %y) {
 define i8 @same_parent_combine_diff_attrs_fail(i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fail(
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
-; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR4:[0-9]+]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -202,10 +202,10 @@ define i8 @same_parent_combine_diff_attrs_fail(i8 %x, i8 %y) {
 define i8 @diff_parent_combine_diff_attrs_todo(i1 %c, i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_todo(
 ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
 ; CHECK:       [[T]]:
-; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR5:[0-9]+]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ; CHECK:       [[F]]:
@@ -226,10 +226,10 @@ F:
 define i8 @diff_parent_combine_diff_attrs_fail(i1 %c, i8 %x, i8 %y) {
 ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_fail(
 ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
-; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
 ; CHECK:       [[T]]:
-; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR3]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR4]]
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ; CHECK:       [[F]]:
@@ -247,10 +247,26 @@ F:
   ret i8 %r2
 }
 
+define i32 @commutative_intrinsic_intersection_failure(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: define i32 @commutative_intrinsic_intersection_failure(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG]], i32 [[ARG1]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG1]], i32 [[ARG]])
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[CALL2]], [[CALL]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %call = call i32 @llvm.smin.i32(i32 %arg, i32 %arg1) strictfp
+  %call2 = call i32 @llvm.smin.i32(i32 %arg1, i32 %arg)
+  %or = or i32 %call2, %call
+  ret i32 %or
+}
+
 ;.
-; CHECK: attributes #[[ATTR0]] = { memory(none) }
-; CHECK: attributes #[[ATTR1]] = { memory(read) }
-; CHECK: attributes #[[ATTR2]] = { alwaysinline memory(none) }
-; CHECK: attributes #[[ATTR3]] = { strictfp memory(none) }
-; CHECK: attributes #[[ATTR4]] = { noinline optnone memory(none) }
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; CHECK: attributes #[[ATTR1]] = { memory(none) }
+; CHECK: attributes #[[ATTR2]] = { memory(read) }
+; CHECK: attributes #[[ATTR3]] = { alwaysinline memory(none) }
+; CHECK: attributes #[[ATTR4]] = { strictfp memory(none) }
+; CHECK: attributes #[[ATTR5]] = { noinline optnone memory(none) }
+; CHECK: attributes #[[ATTR6]] = { strictfp }
 ;.
diff --git a/external/llvm-project/llvm/test/Transforms/GVN/captures.ll b/external/llvm-project/llvm/test/Transforms/GVN/captures.ll
new file mode 100644
index 000000000000..96fce438356c
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/GVN/captures.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=gvn < %s | FileCheck %s
+
+declare void @capture(ptr)
+declare void @unknown_call()
+
+define i32 @full_capture() {
+; CHECK-LABEL: define i32 @full_capture() {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @capture(ptr [[A]])
+; CHECK-NEXT:    store i32 1, ptr [[A]], align 4
+; CHECK-NEXT:    call void @unknown_call()
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  call void @capture(ptr %a)
+  store i32 1, ptr %a
+  call void @unknown_call()
+  %v = load i32, ptr %a
+  ret i32 %v
+}
+
+define i32 @address_capture() {
+; CHECK-LABEL: define i32 @address_capture() {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @capture(ptr captures(address) [[A]])
+; CHECK-NEXT:    store i32 1, ptr [[A]], align 4
+; CHECK-NEXT:    call void @unknown_call()
+; CHECK-NEXT:    ret i32 1
+;
+  %a = alloca i32
+  call void @capture(ptr captures(address) %a)
+  store i32 1, ptr %a
+  call void @unknown_call()
+  %v = load i32, ptr %a
+  ret i32 %v
+}
+
+define i32 @read_provenance_capture() {
+; CHECK-LABEL: define i32 @read_provenance_capture() {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @capture(ptr captures(address, read_provenance) [[A]])
+; CHECK-NEXT:    store i32 1, ptr [[A]], align 4
+; CHECK-NEXT:    call void @unknown_call()
+; CHECK-NEXT:    ret i32 1
+;
+  %a = alloca i32
+  call void @capture(ptr captures(address, read_provenance) %a)
+  store i32 1, ptr %a
+  call void @unknown_call()
+  %v = load i32, ptr %a
+  ret i32 %v
+}
+
+define i32 @read_provenance_capture_and_full_capture() {
+; CHECK-LABEL: define i32 @read_provenance_capture_and_full_capture() {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @capture(ptr captures(address, read_provenance) [[A]])
+; CHECK-NEXT:    call void @capture(ptr [[A]])
+; CHECK-NEXT:    store i32 1, ptr [[A]], align 4
+; CHECK-NEXT:    call void @unknown_call()
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  call void @capture(ptr captures(address, read_provenance) %a)
+  call void @capture(ptr %a)
+  store i32 1, ptr %a
+  call void @unknown_call()
+  %v = load i32, ptr %a
+  ret i32 %v
+}
+
+define i32 @read_provenance_capture_and_full_capture_commuted() {
+; CHECK-LABEL: define i32 @read_provenance_capture_and_full_capture_commuted() {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @capture(ptr [[A]])
+; CHECK-NEXT:    call void @capture(ptr captures(address, read_provenance) [[A]])
+; CHECK-NEXT:    store i32 1, ptr [[A]], align 4
+; CHECK-NEXT:    call void @unknown_call()
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  call void @capture(ptr %a)
+  call void @capture(ptr captures(address, read_provenance) %a)
+  store i32 1, ptr %a
+  call void @unknown_call()
+  %v = load i32, ptr %a
+  ret i32 %v
+}
diff --git a/external/llvm-project/llvm/test/Transforms/GVN/opt-remark-assert-constant-uselistorder.ll b/external/llvm-project/llvm/test/Transforms/GVN/opt-remark-assert-constant-uselistorder.ll
new file mode 100644
index 000000000000..e793728815a8
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/GVN/opt-remark-assert-constant-uselistorder.ll
@@ -0,0 +1,26 @@
+; RUN: opt -passes='gvn' -pass-remarks-output=%t.yaml %s
+; RUN: FileCheck %s < %t.yaml
+
+; Check that there's no assert from trying to the uses of the constant
+; null.
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            gvn
+; CHECK-NEXT: Name:            LoadClobbered
+; CHECK-NEXT: Function:        c
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          'load of type '
+; CHECK-NEXT:   - Type:            i64
+; CHECK-NEXT:   - String:          ' not eliminated'
+; CHECK-NEXT:   - String:          ' because it is clobbered by '
+; CHECK-NEXT:   - ClobberedBy:     store
+; CHECK-NEXT: ...
+define void @c(ptr addrspace(21) %a) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %load = load i64, ptr addrspace(21) null, align 1
+  store i64 %load, ptr addrspace(21) %a, align 1
+  br label %for.cond
+}
diff --git a/external/llvm-project/llvm/test/Transforms/GVN/trunc-nuw-equality.ll b/external/llvm-project/llvm/test/Transforms/GVN/trunc-nuw-equality.ll
new file mode 100644
index 000000000000..efe2ca393203
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/GVN/trunc-nuw-equality.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=gvn -S < %s | FileCheck %s
+
+define void @test(ptr %p, i64 %v) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[V:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TR:%.*]] = trunc nuw i64 [[V]] to i1
+; CHECK-NEXT:    br i1 [[TR]], label %[[RET:.*]], label %[[STORE:.*]]
+; CHECK:       [[STORE]]:
+; CHECK-NEXT:    store i64 0, ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+; CHECK:       [[RET]]:
+; CHECK-NEXT:    store i64 1, ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tr = trunc nuw i64 %v to i1
+  br i1 %tr, label %ret, label %store
+
+store:
+  store i64 %v, ptr %p
+  ret void
+
+ret:
+  store i64 %v, ptr %p
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/GlobalOpt/global-constructor-complex-constants.ll b/external/llvm-project/llvm/test/Transforms/GlobalOpt/global-constructor-complex-constants.ll
new file mode 100644
index 000000000000..6d9bdc41a004
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/GlobalOpt/global-constructor-complex-constants.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 5
+; RUN: opt -p globalopt -S %s | FileCheck %s
+
+@llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @ctor, ptr null }, { i32, ptr, ptr } { i32 65535, ptr @ctor_nocfi, ptr null }, { i32, ptr, ptr } { i32 65535, ptr @ctor_dso_local_equivalent, ptr null }]
+
+@foo = internal global ptr null
+
+declare void @user(ptr)
+
+;.
+; CHECK: @llvm.global_ctors = appending global [3 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @ctor, ptr null }, { i32, ptr, ptr } { i32 65535, ptr @ctor_nocfi, ptr null }, { i32, ptr, ptr } { i32 65535, ptr @ctor_dso_local_equivalent, ptr null }]
+; CHECK: @foo = internal global ptr null
+;.
+define void @ctor() {
+; CHECK-LABEL: define void @ctor() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DST:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    store ptr ptrauth (ptr @foo, i32 0), ptr [[DST]], align 8
+; CHECK-NEXT:    call void @user(ptr [[DST]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %dst = alloca ptr, align 8
+  store ptr ptrauth (ptr @foo, i32 0), ptr %dst, align 8
+  call void @user(ptr %dst)
+  ret void
+}
+
+define void @ctor_nocfi() {
+; CHECK-LABEL: define void @ctor_nocfi() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DST:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    store ptr no_cfi @foo, ptr [[DST]], align 8
+; CHECK-NEXT:    call void @user(ptr [[DST]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %dst = alloca ptr, align 8
+  store ptr no_cfi @foo, ptr %dst, align 8
+  call void @user(ptr %dst)
+  ret void
+}
+
+define void @fn() {
+; CHECK-LABEL: define void @fn() {
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define void @ctor_dso_local_equivalent() {
+; CHECK-LABEL: define void @ctor_dso_local_equivalent() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DST:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    store ptr dso_local_equivalent @fn, ptr [[DST]], align 8
+; CHECK-NEXT:    call void @user(ptr [[DST]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %dst = alloca ptr, align 8
+  store ptr dso_local_equivalent @fn, ptr %dst, align 8
+  call void @user(ptr %dst)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/IROutliner/outlining-debug-statements.ll b/external/llvm-project/llvm/test/Transforms/IROutliner/outlining-debug-statements.ll
index bf846c310a52..c1140988fa91 100644
--- a/external/llvm-project/llvm/test/Transforms/IROutliner/outlining-debug-statements.ll
+++ b/external/llvm-project/llvm/test/Transforms/IROutliner/outlining-debug-statements.ll
@@ -19,7 +19,7 @@ entry:
   %c = alloca i32, align 4
   store i32 2, ptr %a, align 4
   store i32 3, ptr %b, align 4
-  call void @llvm.dbg.value(metadata i64 0, metadata !14, metadata !DIExpression()), !dbg !14
+  call void @llvm.dbg.value(metadata i64 0, metadata !14, metadata !DIExpression()), !dbg !15
   store i32 4, ptr %c, align 4
   %al = load i32, ptr %a
   %bl = load i32, ptr %b
@@ -62,3 +62,4 @@ entry:
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
 !14 = !DILocalVariable(name: "p_6", arg: 1, scope: !11, line: 117, type: !1)
+!15 = !DILocation(line: 1, scope: !11)
diff --git a/external/llvm-project/llvm/test/Transforms/IndVarSimplify/add-nsw-zext-fold.ll b/external/llvm-project/llvm/test/Transforms/IndVarSimplify/add-nsw-zext-fold.ll
new file mode 100644
index 000000000000..1de41e47a856
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/IndVarSimplify/add-nsw-zext-fold.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p indvars -S %s | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+
+declare void @foo(i32)
+
+define void @add_nsw_zext_fold_results_in_sext(i64 %len) {
+; CHECK-LABEL: define void @add_nsw_zext_fold_results_in_sext(
+; CHECK-SAME: i64 [[LEN:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[LEN_TRUNC:%.*]] = trunc i64 [[LEN]] to i32
+; CHECK-NEXT:    [[LZ:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[LEN_TRUNC]], i1 false)
+; CHECK-NEXT:    [[SUB_I:%.*]] = lshr i32 [[LZ]], 3
+; CHECK-NEXT:    [[ADD_I:%.*]] = sub i32 5, [[SUB_I]]
+; CHECK-NEXT:    [[PRECOND:%.*]] = icmp eq i32 [[SUB_I]], 5
+; CHECK-NEXT:    br i1 [[PRECOND]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
+; CHECK:       [[LOOP_PREHEADER]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[ADD_I]] to i64
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP1]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[IV_NEXT:%.*]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[SH_PROM:%.*]] = zext nneg i32 [[IV_NEXT]] to i64
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i64 1, [[SH_PROM]]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[SHR]] to i32
+; CHECK-NEXT:    call void @foo(i32 [[TMP0]])
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 0
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %len.trunc  = trunc i64 %len to i32
+  %lz = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %len.trunc, i1 false)
+  %sub.i = lshr i32 %lz, 3
+  %add.i = sub nuw nsw i32 5, %sub.i
+  %precond = icmp eq i32 %sub.i, 5
+  br i1 %precond, label %exit, label %loop
+
+loop:
+  %iv = phi i32 [ %add.i, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i32 %iv, 1
+  %sh_prom = zext nneg i32 %iv.next to i64
+  %shr = lshr i64 1, %sh_prom
+  %2 = trunc nuw nsw i64 %shr to i32
+  call void @foo(i32 %2)
+  %ec = icmp eq i32 %iv.next, 0
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @add_nsw_zext_fold_results_in_sext_known_positive(i32 %mask, ptr %src, i1 %c) {
+; CHECK-LABEL: define void @add_nsw_zext_fold_results_in_sext_known_positive(
+; CHECK-SAME: i32 [[MASK:%.*]], ptr [[SRC:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[C]], i32 0, i32 6
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[SPEC_SELECT]], [[MASK]]
+; CHECK-NEXT:    [[PRECOND:%.*]] = icmp slt i32 [[ADD]], 0
+; CHECK-NEXT:    br i1 [[PRECOND]], label %[[EXIT:.*]], label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i32 78, [[SPEC_SELECT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 1
+; CHECK-NEXT:    call void @foo(i32 [[L]])
+; CHECK-NEXT:    br label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %spec.select = select i1 %c, i32 0, i32 6
+  %add = add i32 %spec.select, %mask
+  %precond = icmp slt i32 %add, 0
+  br i1 %precond, label %exit, label %ph
+
+ph:
+  %start = sub i32 79, %spec.select
+  br label %loop
+
+loop:                                     ; preds = %loop, %ph
+  %iv = phi i32 [ %start, %ph ], [ %dec, %loop ]
+  %iv.ext = zext i32 %iv to i64
+  %gep = getelementptr i32, ptr %src, i64 %iv.ext
+  %l = load i32, ptr %gep, align 1
+  call void @foo(i32 %l)
+  %dec = add i32 %iv, 0
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/IndVarSimplify/iv-cmp-sext.ll b/external/llvm-project/llvm/test/Transforms/IndVarSimplify/iv-cmp-sext.ll
new file mode 100644
index 000000000000..34925a6cca95
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/IndVarSimplify/iv-cmp-sext.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+target triple = "riscv64"
+
+define void @foo(ptr %x, i32 %n) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr [[X:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP10:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP10]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT:    br label %[[FOR_COND_CLEANUP]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[CONV]] to i64
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i64 [[INDVARS_IV]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX]], align 2
+; CHECK-NEXT:    br label %[[FOR_INC]]
+; CHECK:       [[FOR_INC]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+;
+entry:
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.011 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %idxprom = zext nneg i32 %i.011 to i64
+  %arrayidx = getelementptr inbounds nuw i16, ptr %x, i64 %idxprom
+  %0 = load i16, ptr %arrayidx, align 2
+  %conv = sext i16 %0 to i32
+  %cmp1 = icmp eq i32 %i.011, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i16 0, ptr %arrayidx, align 2
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.011, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InferAddressSpaces/AMDGPU/diop-diexpression.ll b/external/llvm-project/llvm/test/Transforms/InferAddressSpaces/AMDGPU/diop-diexpression.ll
new file mode 100644
index 000000000000..012c7a60779c
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InferAddressSpaces/AMDGPU/diop-diexpression.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -passes=infer-address-spaces < %s | FileCheck %s
+
+define void @test_glob(ptr addrspace(1) %global) !dbg !3 {
+; CHECK-LABEL: @test_glob(
+; CHECK-NEXT:    [[USE_GLOB_GEN:%.*]] = load i32, ptr addrspace(1) [[GLOBAL:%.*]], align 4
+; CHECK-NEXT:      #dbg_value(ptr addrspace(1) [[GLOBAL]], [[META6:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(1)), DIOpConvert(ptr)), [[META8:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(1) [[GLOBAL]], [[META9:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(1)), DIOpConvert(ptr), DIOpDeref(i32)), [[META8]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(1) [[GLOBAL]], [[META11:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(1)), DIOpConvert(ptr), DIOpReinterpret(i64)), [[META8]])
+; CHECK-NEXT:    ret void, !dbg [[META8]]
+;
+  %glob_gen = addrspacecast ptr addrspace(1) %global to ptr
+  %use_glob_gen = load i32, ptr %glob_gen, align 4
+  #dbg_value(ptr %glob_gen, !6, !DIExpression(DIOpArg(0, ptr)), !8)
+  #dbg_value(ptr %glob_gen, !9, !DIExpression(DIOpArg(0, ptr), DIOpDeref(i32)), !8)
+  #dbg_value(ptr %glob_gen, !11, !DIExpression(DIOpArg(0, ptr), DIOpReinterpret(i64)), !8)
+  ret void, !dbg !8
+}
+
+define void @test_local(ptr addrspace(3) %local) !dbg !13 {
+; CHECK-LABEL: @test_local(
+; CHECK-NEXT:    [[USE_LOC_GEN:%.*]] = load i32, ptr addrspace(3) [[LOCAL:%.*]], align 4
+; CHECK-NEXT:      #dbg_value(ptr addrspace(3) [[LOCAL]], [[META14:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr)), [[META15:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(3) [[LOCAL]], [[META16:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr), DIOpDeref(i32)), [[META15]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(3) [[LOCAL]], [[META17:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(3)), DIOpConvert(ptr), DIOpReinterpret(i64)), [[META15]])
+; CHECK-NEXT:    ret void
+;
+  %loc_gen = addrspacecast ptr addrspace(3) %local to ptr
+  %use_loc_gen = load i32, ptr %loc_gen, align 4
+  #dbg_value(ptr %loc_gen, !14, !DIExpression(DIOpArg(0, ptr)), !15)
+  #dbg_value(ptr %loc_gen, !16, !DIExpression(DIOpArg(0, ptr), DIOpDeref(i32)), !15)
+  #dbg_value(ptr %loc_gen, !17, !DIExpression(DIOpArg(0, ptr), DIOpReinterpret(i64)), !15)
+  ret void
+}
+
+define void @test_constant(ptr addrspace(4) %constant) !dbg !18 {
+; CHECK-LABEL: @test_constant(
+; CHECK-NEXT:    [[USE_CONST_GEN:%.*]] = load i32, ptr addrspace(4) [[CONSTANT:%.*]], align 4
+; CHECK-NEXT:      #dbg_value(ptr addrspace(4) [[CONSTANT]], [[META19:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(4)), DIOpConvert(ptr)), [[META20:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(4) [[CONSTANT]], [[META21:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(4)), DIOpConvert(ptr), DIOpDeref(i32)), [[META20]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(4) [[CONSTANT]], [[META22:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(4)), DIOpConvert(ptr), DIOpReinterpret(i64)), [[META20]])
+; CHECK-NEXT:    ret void
+;
+  %const_gen = addrspacecast ptr addrspace(4) %constant to ptr
+  %use_const_gen = load i32, ptr %const_gen, align 4
+  #dbg_value(ptr %const_gen, !19, !DIExpression(DIOpArg(0, ptr)), !20)
+  #dbg_value(ptr %const_gen, !21, !DIExpression(DIOpArg(0, ptr), DIOpDeref(i32)), !20)
+  #dbg_value(ptr %const_gen, !22, !DIExpression(DIOpArg(0, ptr), DIOpReinterpret(i64)), !20)
+  ret void
+}
+
+; Verify that we can update the address space regardless of whether the new
+; instruction gets inserted before or after the old one.
+define void @test_before_and_after(ptr addrspace(4) %constant) !dbg !23 {
+; CHECK-LABEL: @test_before_and_after(
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr addrspace(4) @llvm.ptrmask.p4.i64(ptr addrspace(4) [[CONSTANT:%.*]], i64 -2)
+; CHECK-NEXT:      #dbg_value(ptr addrspace(4) [[TMP1]], [[META24:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(4)), DIOpConvert(ptr)), [[META25:![0-9]+]])
+; CHECK-NEXT:    [[USE_MASK:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4
+; CHECK-NEXT:    [[BC:%.*]] = getelementptr i32, ptr addrspace(4) [[CONSTANT]], i32 42
+; CHECK-NEXT:      #dbg_value(ptr addrspace(4) [[BC]], [[META26:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(4)), DIOpConvert(ptr)), [[META25]])
+; CHECK-NEXT:    [[USE_BC:%.*]] = load i32, ptr addrspace(4) [[BC]], align 4
+; CHECK-NEXT:    ret void
+;
+  %const_gen = addrspacecast ptr addrspace(4) %constant to ptr
+
+  %mask = call ptr @llvm.ptrmask.p0.i64(ptr %const_gen, i64 -2)
+  #dbg_value(ptr %mask, !24, !DIExpression(DIOpArg(0, ptr)), !26)
+  %use_mask = load i32, ptr %mask, align 4
+
+  %bc = getelementptr i32, ptr %const_gen, i32 42
+  #dbg_value(ptr %bc, !25, !DIExpression(DIOpArg(0, ptr)), !26)
+  %use_bc = load i32, ptr %bc, align 4
+
+  ret void
+}
+
+define void @test_no_DW_OPs(ptr addrspace(3) %local_ptr) !dbg !27 {
+; CHECK-LABEL: @test_no_DW_OPs(
+; CHECK-NEXT:      #dbg_value(ptr poison, [[META28:![0-9]+]], !DIExpression(), [[META29:![0-9]+]])
+; CHECK-NEXT:    [[USE_GEN:%.*]] = load i32, ptr addrspace(3) [[LOCAL_PTR:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  %gen = addrspacecast ptr addrspace(3) %local_ptr to ptr
+  #dbg_value(ptr %gen, !28, !DIExpression(), !29)
+  %use_gen = load i32, ptr %gen, align 4
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "t.c", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "test_glob", linkageName: "test_glob", scope: null, file: !1, line: 1, type: !4, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!4 = !DISubroutineType(types: !5)
+!5 = !{}
+!6 = !DILocalVariable(name: "ptr_var", scope: !3, file: !1, line: 1, type: !7)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!8 = !DILocation(line: 1, column: 1, scope: !3)
+!9 = !DILocalVariable(name: "i32_var", scope: !3, file: !1, line: 2, type: !10)
+!10 = !DIBasicType(name: "i32", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "i64_var", scope: !3, file: !1, line: 2, type: !12)
+!12 = !DIBasicType(name: "i64", size: 64, encoding: DW_ATE_unsigned)
+!13 = distinct !DISubprogram(name: "test_local", linkageName: "test_local", scope: null, file: !1, line: 1, type: !4, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!14 = !DILocalVariable(name: "ptr_var", scope: !13, file: !1, line: 1, type: !7)
+!15 = !DILocation(line: 1, column: 1, scope: !13)
+!16 = !DILocalVariable(name: "i32_var", scope: !13, file: !1, line: 2, type: !10)
+!17 = !DILocalVariable(name: "i64_var", scope: !13, file: !1, line: 2, type: !12)
+!18 = distinct !DISubprogram(name: "test_constant", linkageName: "test_constant", scope: null, file: !1, line: 1, type: !4, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!19 = !DILocalVariable(name: "ptr_var", scope: !18, file: !1, line: 1, type: !7)
+!20 = !DILocation(line: 1, column: 1, scope: !18)
+!21 = !DILocalVariable(name: "i32_var", scope: !18, file: !1, line: 2, type: !10)
+!22 = !DILocalVariable(name: "i64_var", scope: !18, file: !1, line: 2, type: !12)
+!23 = distinct !DISubprogram(name: "test_before_and_after", linkageName: "test_before_and_after", scope: null, file: !1, line: 1, type: !4, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!24 = !DILocalVariable(name: "p1", scope: !23, file: !1, line: 1, type: !7)
+!25 = !DILocalVariable(name: "p2", scope: !23, file: !1, line: 1, type: !7)
+!26 = !DILocation(line: 1, column: 1, scope: !23)
+!27 = distinct !DISubprogram(name: "test_no_DW_OPs", linkageName: "test_no_DW_OPs", scope: null, file: !1, line: 1, type: !4, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!28 = !DILocalVariable(name: "t1", scope: !27, file: !1, line: 1, type: !7)
+!29 = !DILocation(line: 1, column: 1, scope: !27)
diff --git a/external/llvm-project/llvm/test/Transforms/Inline/inline-noalias-unidentify-object.ll b/external/llvm-project/llvm/test/Transforms/Inline/inline-noalias-unidentify-object.ll
index 54e9ee0918ae..b7ba1b32238a 100644
--- a/external/llvm-project/llvm/test/Transforms/Inline/inline-noalias-unidentify-object.ll
+++ b/external/llvm-project/llvm/test/Transforms/Inline/inline-noalias-unidentify-object.ll
@@ -3,15 +3,18 @@
 define i32 @caller(ptr %p) {
 ; CHECK-LABEL: define i32 @caller(ptr %p) {
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
-; CHECK-NEXT:    [[P_8_I:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 8
-; CHECK-NEXT:    [[V_I:%.*]] = load i32, ptr [[P_8_I]], align 4, !alias.scope !0
-; CHECK-NEXT:    [[P_1_I:%.*]] = getelementptr i8, ptr [[P]], i64 1
+; CHECK-NEXT:    [[P_11_I:%.*]] = getelementptr i8, ptr %p, i64 11
+; CHECK-NEXT:    [[V_I:%.*]] = load i32, ptr [[P_11_I]], align 4, !alias.scope !0
+; CHECK-NEXT:    [[P_1_I:%.*]] = getelementptr i8, ptr %p, i64 1
 ; CHECK-NEXT:    [[P_2_I:%.*]] = getelementptr i8, ptr [[P_1_I]], i64 1
 ; CHECK-NEXT:    [[P_3_I:%.*]] = getelementptr i8, ptr [[P_2_I]], i64 1
 ; CHECK-NEXT:    [[P_4_I:%.*]] = getelementptr i8, ptr [[P_3_I]], i64 1
 ; CHECK-NEXT:    [[P_5_I:%.*]] = getelementptr i8, ptr [[P_4_I]], i64 1
 ; CHECK-NEXT:    [[P_6_I:%.*]] = getelementptr i8, ptr [[P_5_I]], i64 1
-; CHECK-NEXT:    [[P_7_I:%.*]] = getelementptr i8, ptr [[P_6_I]], i64 1
+; CHECK-NEXT:    [[P_7_I1:%.*]] = getelementptr i8, ptr [[P_6_I]], i64 1
+; CHECK-NEXT:    [[P_8_I:%.*]] = getelementptr i8, ptr [[P_7_I1]], i64 1
+; CHECK-NEXT:    [[P_9_I:%.*]] = getelementptr i8, ptr [[P_8_I]], i64 1
+; CHECK-NEXT:    [[P_7_I:%.*]] = getelementptr i8, ptr [[P_9_I]], i64 1
 ; CHECK-NEXT:    [[P_8_ALIAS_I:%.*]] = getelementptr i8, ptr [[P_7_I]], i64 1
 ; CHECK-NEXT:    store i32 42, ptr [[P_8_ALIAS_I]], align 4
 ; CHECK-NEXT:    ret i32 [[V_I]]
@@ -21,8 +24,8 @@ define i32 @caller(ptr %p) {
 }
 
 define internal i32 @callee(ptr noalias %p) {
-  %p.8 = getelementptr i8, ptr %p, i64 8
-  %v = load i32, ptr %p.8
+  %p.11 = getelementptr i8, ptr %p, i64 11
+  %v = load i32, ptr %p.11
   %p.1 = getelementptr i8, ptr %p, i64 1
   %p.2 = getelementptr i8, ptr %p.1, i64 1
   %p.3 = getelementptr i8, ptr %p.2, i64 1
@@ -30,7 +33,10 @@ define internal i32 @callee(ptr noalias %p) {
   %p.5 = getelementptr i8, ptr %p.4, i64 1
   %p.6 = getelementptr i8, ptr %p.5, i64 1
   %p.7 = getelementptr i8, ptr %p.6, i64 1
-  %p.8.alias = getelementptr i8, ptr %p.7, i64 1
-  store i32 42, ptr %p.8.alias
+  %p.8 = getelementptr i8, ptr %p.7, i64 1
+  %p.9 = getelementptr i8, ptr %p.8, i64 1
+  %p.10 = getelementptr i8, ptr %p.9, i64 1
+  %p.11.alias = getelementptr i8, ptr %p.10, i64 1
+  store i32 42, ptr %p.11.alias
   ret i32 %v
 }
diff --git a/external/llvm-project/llvm/test/Transforms/Inline/inline-recursive-fn2.ll b/external/llvm-project/llvm/test/Transforms/Inline/inline-recursive-fn2.ll
new file mode 100644
index 000000000000..52d7b21902e8
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/Inline/inline-recursive-fn2.ll
@@ -0,0 +1,49 @@
+; REQUIRES: asserts
+; RUN: opt -passes='cgscc(inline),instcombine,cgscc(inline)' -S -debug-only=inline -disable-output < %s 2>&1 | FileCheck %s
+
+; This test shows that the recursive function will not get simplified
+; unless the caller is the function itself, not another different caller.
+
+; CHECK:  Inlining calls in: test
+; CHECK:      Function size: 2
+; CHECK:      NOT Inlining (cost=never): recursive, Call:   %call = tail call float @inline_rec_true_successor(float %x, float %scale)
+
+; CHECK:  Inlining calls in: inline_rec_true_successor
+; CHECK:      Function size: 10
+; CHECK:      Inlining (cost=-35, threshold=337), Call:   %call = tail call float @inline_rec_true_successor(float %fneg, float %scale)
+; CHECK:      Size after inlining: 17
+; CHECK:      NOT Inlining (cost=never): noinline function attribute, Call:   %call_test = tail call float @test(float %fneg, float %common.ret18.op.i)
+; CHECK:      NOT Inlining (cost=never): noinline function attribute, Call:   %call_test.i = tail call float @test(float %x, float %call.i)
+; CHECK:  Skipping inlining due to history: inline_rec_true_successor -> inline_rec_true_successor
+; CHECK:  Updated inlining SCC: (test, inline_rec_true_successor)
+
+; CHECK:  Inlining calls in: test
+; CHECK:      Function size: 2
+; CHECK:      Inlining (cost=25, threshold=225), Call:   %call = tail call float @inline_rec_true_successor(float %x, float %scale)
+; CHECK:      Size after inlining: 10
+
+define float @test(float %x, float %scale) noinline {
+entry:
+  %call = tail call float @inline_rec_true_successor(float %x, float %scale)
+  ret float %call
+}
+
+define float @inline_rec_true_successor(float %x, float %scale)  {
+entry:
+  %cmp = fcmp olt float %x, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+common.ret18:                                     ; preds = %if.then, %if.end
+  %common.ret18.op = phi float [ %call_test, %if.then ], [ %mul, %if.end ]
+  ret float %common.ret18.op
+
+if.then:                                          ; preds = %entry
+  %fneg = fneg float %x
+  %call = tail call float @inline_rec_true_successor(float %fneg, float %scale)
+  %call_test = tail call float @test(float %fneg, float %call)
+  br label %common.ret18
+
+if.end:                                           ; preds = %entry
+  %mul = fmul float %x, %scale
+  br label %common.ret18
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
index b532c8155673..ef414885bf80 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
@@ -215,7 +215,7 @@ define ptr @_Z3fooRSt6vectorIiSaIiEE(ptr %X) {
 ; IC_SROA-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[__FIRST_ADDR_I_I_SROA_0_0]], i32 4
 ; IC_SROA-NEXT:    br label [[BB18_I_I]]
 ; IC_SROA:       bb18.i.i:
-; IC_SROA-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_1:%.*]] = phi ptr [ [[TMP27]], [[BB17_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB13_I_I]] ]
+; IC_SROA-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_1:%.*]] = phi ptr [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB13_I_I]] ], [ [[TMP27]], [[BB17_I_I]] ]
 ; IC_SROA-NEXT:    [[TMP28:%.*]] = load i32, ptr [[__FIRST_ADDR_I_I_SROA_0_1]], align 4
 ; IC_SROA-NEXT:    [[TMP29:%.*]] = icmp eq i32 [[TMP28]], 42
 ; IC_SROA-NEXT:    br i1 [[TMP29]], label [[BB20_I_I:%.*]], label [[BB21_I_I:%.*]]
@@ -225,7 +225,7 @@ define ptr @_Z3fooRSt6vectorIiSaIiEE(ptr %X) {
 ; IC_SROA-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[__FIRST_ADDR_I_I_SROA_0_1]], i32 4
 ; IC_SROA-NEXT:    br label [[BB22_I_I]]
 ; IC_SROA:       bb22.i.i:
-; IC_SROA-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_2:%.*]] = phi ptr [ [[TMP30]], [[BB21_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB13_I_I]] ]
+; IC_SROA-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_2:%.*]] = phi ptr [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB13_I_I]] ], [ [[TMP30]], [[BB21_I_I]] ]
 ; IC_SROA-NEXT:    [[TMP31:%.*]] = load i32, ptr [[__FIRST_ADDR_I_I_SROA_0_2]], align 4
 ; IC_SROA-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[TMP31]], 42
 ; IC_SROA-NEXT:    br i1 [[TMP32]], label [[BB24_I_I:%.*]], label [[BB25_I_I:%.*]]
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll
index c6695f17b955..8c69d0721b73 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll
@@ -13,6 +13,17 @@ define <16 x i8> @combineXorAeseZeroARM64(<16 x i8> %data, <16 x i8> %key) {
   ret <16 x i8> %data.aes
 }
 
+define <16 x i8> @combineXorAeseZeroLhsARM64(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: define <16 x i8> @combineXorAeseZeroLhsARM64(
+; CHECK-SAME: <16 x i8> [[DATA:%.*]], <16 x i8> [[KEY:%.*]]) {
+; CHECK-NEXT:    [[DATA_AES:%.*]] = tail call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> [[DATA]], <16 x i8> [[KEY]])
+; CHECK-NEXT:    ret <16 x i8> [[DATA_AES]]
+;
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> zeroinitializer, <16 x i8> %data.xor)
+  ret <16 x i8> %data.aes
+}
+
 define <16 x i8> @combineXorAeseNonZeroARM64(<16 x i8> %data, <16 x i8> %key) {
 ; CHECK-LABEL: define <16 x i8> @combineXorAeseNonZeroARM64(
 ; CHECK-SAME: <16 x i8> [[DATA:%.*]], <16 x i8> [[KEY:%.*]]) {
@@ -36,6 +47,17 @@ define <16 x i8> @combineXorAesdZeroARM64(<16 x i8> %data, <16 x i8> %key) {
   ret <16 x i8> %data.aes
 }
 
+define <16 x i8> @combineXorAesdZeroLhsARM64(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: define <16 x i8> @combineXorAesdZeroLhsARM64(
+; CHECK-SAME: <16 x i8> [[DATA:%.*]], <16 x i8> [[KEY:%.*]]) {
+; CHECK-NEXT:    [[DATA_AES:%.*]] = tail call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> [[DATA]], <16 x i8> [[KEY]])
+; CHECK-NEXT:    ret <16 x i8> [[DATA_AES]]
+;
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> zeroinitializer, <16 x i8> %data.xor)
+  ret <16 x i8> %data.aes
+}
+
 define <16 x i8> @combineXorAesdNonZeroARM64(<16 x i8> %data, <16 x i8> %key) {
 ; CHECK-LABEL: define <16 x i8> @combineXorAesdNonZeroARM64(
 ; CHECK-SAME: <16 x i8> [[DATA:%.*]], <16 x i8> [[KEY:%.*]]) {
@@ -51,3 +73,51 @@ define <16 x i8> @combineXorAesdNonZeroARM64(<16 x i8> %data, <16 x i8> %key) {
 declare <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8>, <16 x i8>) #0
 declare <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8>, <16 x i8>) #0
 
+; SVE
+
+define <vscale x 16 x i8> @combineXorAeseZeroLhsSVE(<vscale x 16 x i8> %data, <vscale x 16 x i8> %key) {
+; CHECK-LABEL: define <vscale x 16 x i8> @combineXorAeseZeroLhsSVE(
+; CHECK-SAME: <vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i8> [[KEY:%.*]]) {
+; CHECK-NEXT:    [[DATA_AES:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aese(<vscale x 16 x i8> [[DATA]], <vscale x 16 x i8> [[KEY]])
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[DATA_AES]]
+;
+  %data.xor = xor <vscale x 16 x i8> %data, %key
+  %data.aes = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aese(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> %data.xor)
+  ret <vscale x 16 x i8> %data.aes
+}
+
+define <vscale x 16 x i8> @combineXorAeseZeroRhsSVE(<vscale x 16 x i8> %data, <vscale x 16 x i8> %key) {
+; CHECK-LABEL: define <vscale x 16 x i8> @combineXorAeseZeroRhsSVE(
+; CHECK-SAME: <vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i8> [[KEY:%.*]]) {
+; CHECK-NEXT:    [[DATA_AES:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aese(<vscale x 16 x i8> [[DATA]], <vscale x 16 x i8> [[KEY]])
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[DATA_AES]]
+;
+  %data.xor = xor <vscale x 16 x i8> %data, %key
+  %data.aes = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aese(<vscale x 16 x i8> %data.xor, <vscale x 16 x i8> zeroinitializer)
+  ret <vscale x 16 x i8> %data.aes
+}
+
+define <vscale x 16 x i8> @combineXorAesdZeroLhsSVE(<vscale x 16 x i8> %data, <vscale x 16 x i8> %key) {
+; CHECK-LABEL: define <vscale x 16 x i8> @combineXorAesdZeroLhsSVE(
+; CHECK-SAME: <vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i8> [[KEY:%.*]]) {
+; CHECK-NEXT:    [[DATA_AES:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aesd(<vscale x 16 x i8> [[DATA]], <vscale x 16 x i8> [[KEY]])
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[DATA_AES]]
+;
+  %data.xor = xor <vscale x 16 x i8> %data, %key
+  %data.aes = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aesd(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> %data.xor)
+  ret <vscale x 16 x i8> %data.aes
+}
+
+define <vscale x 16 x i8> @combineXorAesdZeroRhsSVE(<vscale x 16 x i8> %data, <vscale x 16 x i8> %key) {
+; CHECK-LABEL: define <vscale x 16 x i8> @combineXorAesdZeroRhsSVE(
+; CHECK-SAME: <vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i8> [[KEY:%.*]]) {
+; CHECK-NEXT:    [[DATA_AES:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aesd(<vscale x 16 x i8> [[DATA]], <vscale x 16 x i8> [[KEY]])
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[DATA_AES]]
+;
+  %data.xor = xor <vscale x 16 x i8> %data, %key
+  %data.aes = tail call <vscale x 16 x i8> @llvm.aarch64.sve.aesd(<vscale x 16 x i8> %data.xor, <vscale x 16 x i8> zeroinitializer)
+  ret <vscale x 16 x i8> %data.aes
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.aese(<vscale x 16 x i8>, <vscale x 16 x i8>) #0
+declare <vscale x 16 x i8> @llvm.aarch64.sve.aesd(<vscale x 16 x i8>, <vscale x 16 x i8>) #0
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 9375d56e900e..077da9cda652 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -2743,7 +2743,7 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 
 @gv = constant i32 0
 
-define amdgpu_kernel void @readfirstlane_constant(i32 %arg, ptr %ptr) {
+define amdgpu_cs void @readfirstlane_constant(i32 %arg, ptr %ptr) {
 ; CHECK-LABEL: @readfirstlane_constant(
 ; CHECK-NEXT:    [[VAR:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG:%.*]])
 ; CHECK-NEXT:    store volatile i32 [[VAR]], ptr [[PTR:%.*]], align 4
@@ -2829,7 +2829,7 @@ bb1:
 
 declare i32 @llvm.amdgcn.readlane(i32, i32)
 
-define amdgpu_kernel void @readlane_constant(i32 %arg, i32 %lane, ptr %ptr) {
+define amdgpu_cs void @readlane_constant(i32 %arg, i32 %lane, ptr %ptr) {
 ; CHECK-LABEL: @readlane_constant(
 ; CHECK-NEXT:    [[VAR:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG:%.*]], i32 7)
 ; CHECK-NEXT:    store volatile i32 [[VAR]], ptr [[PTR:%.*]], align 4
@@ -3041,14 +3041,12 @@ define amdgpu_kernel void @permlanex16_fetch_invalid_bound_ctrl(ptr addrspace(1)
 ; llvm.amdgcn.permlane64
 ; --------------------------------------------------------------------
 
-define amdgpu_kernel void @permlane64_uniform(ptr addrspace(1) %out, i32 %src0) {
+define amdgpu_kernel void @permlane64_uniform(ptr addrspace(1) %out, i32 %src) {
 ; CHECK-LABEL: @permlane64_uniform(
-; CHECK-NEXT:    [[SRC1:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[SRC0:%.*]])
-; CHECK-NEXT:    store i32 [[SRC1]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-NEXT:    store i32 [[SRC1:%.*]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
-  %src1 = call i32 @llvm.amdgcn.readfirstlane(i32 %src0)
-  %res = call i32 @llvm.amdgcn.permlane64(i32 %src1)
+  %res = call i32 @llvm.amdgcn.permlane64(i32 %src)
   store i32 %res, ptr addrspace(1) %out
   ret void
 }
@@ -6486,7 +6484,7 @@ define i32 @prng_poison_i32() {
 ; llvm.amdgcn.ds.bpermute
 ; --------------------------------------------------------------------
 
-define amdgpu_kernel void @ds_bpermute_uniform_src(ptr addrspace(1) %out, i32 %lane) {
+define void @ds_bpermute_uniform_src(ptr addrspace(1) %out, i32 %lane) {
 ; CHECK-LABEL: @ds_bpermute_uniform_src(
 ; CHECK-NEXT:    store i32 7, ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
@@ -6496,7 +6494,7 @@ define amdgpu_kernel void @ds_bpermute_uniform_src(ptr addrspace(1) %out, i32 %l
   ret void
 }
 
-define amdgpu_kernel void @ds_bpermute_constant_lane(ptr addrspace(1) %out, i32 %src) {
+define void @ds_bpermute_constant_lane(ptr addrspace(1) %out, i32 %src) {
 ; CHECK-LABEL: @ds_bpermute_constant_lane(
 ; CHECK-NEXT:    [[V:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[SRC:%.*]], i32 7)
 ; CHECK-NEXT:    store i32 [[V]], ptr addrspace(1) [[OUT:%.*]], align 4
@@ -6507,7 +6505,7 @@ define amdgpu_kernel void @ds_bpermute_constant_lane(ptr addrspace(1) %out, i32
   ret void
 }
 
-define amdgpu_kernel void @ds_bpermute_uniform_lane(ptr addrspace(1) %out, i32 %lanearg, i32 %src) {
+define void @ds_bpermute_uniform_lane(ptr addrspace(1) %out, i32 %lanearg, i32 %src) {
 ; CHECK-LABEL: @ds_bpermute_uniform_lane(
 ; CHECK-NEXT:    [[LANE:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[LANEARG:%.*]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[LANE]], 2
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/AMDGPU/trivially-uniform.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/AMDGPU/trivially-uniform.ll
new file mode 100644
index 000000000000..2a3e3929d71b
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/AMDGPU/trivially-uniform.ll
@@ -0,0 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn -mcpu=gfx1010 -passes=instcombine -S < %s | FileCheck %s
+
+; Use readfirstlane to demonstrate when InstCombine deems an input to
+; be trivially uniform.
+
+; Constants are trivially uniform.
+define i32 @test_constant() {
+; CHECK-LABEL: define i32 @test_constant(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    ret i32 7
+;
+  %r = call i32 @llvm.amdgcn.readfirstlane(i32 7)
+  ret i32 %r
+}
+
+; The result of an AlwaysUniform intrinsic is trivially uniform.
+define i32 @test_intrinsic(i32 %x) {
+; CHECK-LABEL: define i32 @test_intrinsic(
+; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[X]])
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = call i32 @llvm.amdgcn.readfirstlane(i32 %x)
+  %r = call i32 @llvm.amdgcn.readfirstlane(i32 %y)
+  ret i32 %r
+}
+
+; In compute kernels, all arguments are trivially uniform.
+
+define amdgpu_kernel void @test_compute_i32(ptr %out, i32 %x) {
+; CHECK-LABEL: define amdgpu_kernel void @test_compute_i32(
+; CHECK-SAME: ptr [[OUT:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    store i32 [[X]], ptr [[OUT]], align 4
+; CHECK-NEXT:    ret void
+;
+  %r = call i32 @llvm.amdgcn.readfirstlane(i32 %x)
+  store i32 %r, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @test_compute_i1(ptr %out, i1 %x) {
+; CHECK-LABEL: define amdgpu_kernel void @test_compute_i1(
+; CHECK-SAME: ptr [[OUT:%.*]], i1 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    store i1 [[X]], ptr [[OUT]], align 1
+; CHECK-NEXT:    ret void
+;
+  %r = call i1 @llvm.amdgcn.readfirstlane(i1 %x)
+  store i1 %r, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @test_compute_v32i1(ptr %out, <32 x i1> %x) {
+; CHECK-LABEL: define amdgpu_kernel void @test_compute_v32i1(
+; CHECK-SAME: ptr [[OUT:%.*]], <32 x i1> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    store <32 x i1> [[X]], ptr [[OUT]], align 4
+; CHECK-NEXT:    ret void
+;
+  %r = call <32 x i1> @llvm.amdgcn.readfirstlane(<32 x i1> %x)
+  store <32 x i1> %r, ptr %out
+  ret void
+}
+
+; In graphics shaders, inreg arguments are trivially uniform.
+
+define amdgpu_ps i32 @test_graphics_i32(i32 inreg %x) {
+; CHECK-LABEL: define amdgpu_ps i32 @test_graphics_i32(
+; CHECK-SAME: i32 inreg [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %r = call i32 @llvm.amdgcn.readfirstlane(i32 %x)
+  ret i32 %r
+}
+
+define amdgpu_ps i1 @test_graphics_i1(i1 inreg %x) {
+; CHECK-LABEL: define amdgpu_ps i1 @test_graphics_i1(
+; CHECK-SAME: i1 inreg [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    ret i1 [[X]]
+;
+  %r = call i1 @llvm.amdgcn.readfirstlane(i1 %x)
+  ret i1 %r
+}
+
+define amdgpu_ps <32 x i1> @test_graphics_v32i1(<32 x i1> inreg %x) {
+; CHECK-LABEL: define amdgpu_ps <32 x i1> @test_graphics_v32i1(
+; CHECK-SAME: <32 x i1> inreg [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    ret <32 x i1> [[X]]
+;
+  %r = call <32 x i1> @llvm.amdgcn.readfirstlane(<32 x i1> %x)
+  ret <32 x i1> %r
+}
+
+; In graphics shaders, non-inreg arguments are not trivially uniform.
+
+define amdgpu_ps i32 @test_graphics_i32_negative(i32 %x) {
+; CHECK-LABEL: define amdgpu_ps i32 @test_graphics_i32_negative(
+; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[X]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.amdgcn.readfirstlane(i32 %x)
+  ret i32 %r
+}
+
+define amdgpu_ps i1 @test_graphics_i1_negative(i1 %x) {
+; CHECK-LABEL: define amdgpu_ps i1 @test_graphics_i1_negative(
+; CHECK-SAME: i1 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = call i1 @llvm.amdgcn.readfirstlane.i1(i1 [[X]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %r = call i1 @llvm.amdgcn.readfirstlane(i1 %x)
+  ret i1 %r
+}
+
+define amdgpu_ps <32 x i1> @test_graphics_v32i1_negative(<32 x i1> %x) {
+; CHECK-LABEL: define amdgpu_ps <32 x i1> @test_graphics_v32i1_negative(
+; CHECK-SAME: <32 x i1> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i1> @llvm.amdgcn.readfirstlane.v32i1(<32 x i1> [[X]])
+; CHECK-NEXT:    ret <32 x i1> [[R]]
+;
+  %r = call <32 x i1> @llvm.amdgcn.readfirstlane(<32 x i1> %x)
+  ret <32 x i1> %r
+}
+
+; Test i1 arguments in non-entry functions.
+
+define amdgpu_gfx i1 @test_callable_i1(i1 inreg %x) {
+; CHECK-LABEL: define amdgpu_gfx i1 @test_callable_i1(
+; CHECK-SAME: i1 inreg [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    ret i1 [[X]]
+;
+  %r = call i1 @llvm.amdgcn.readfirstlane(i1 %x)
+  ret i1 %r
+}
+
+define amdgpu_gfx <32 x i1> @test_callable_v32i1(<32 x i1> inreg %x) {
+; CHECK-LABEL: define amdgpu_gfx <32 x i1> @test_callable_v32i1(
+; CHECK-SAME: <32 x i1> inreg [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    ret <32 x i1> [[X]]
+;
+  %r = call <32 x i1> @llvm.amdgcn.readfirstlane(<32 x i1> %x)
+  ret <32 x i1> %r
+}
+
+define amdgpu_gfx i1 @test_callable_i1_negative(i1 %x) {
+; CHECK-LABEL: define amdgpu_gfx i1 @test_callable_i1_negative(
+; CHECK-SAME: i1 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = call i1 @llvm.amdgcn.readfirstlane.i1(i1 [[X]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %r = call i1 @llvm.amdgcn.readfirstlane(i1 %x)
+  ret i1 %r
+}
+
+define amdgpu_gfx <32 x i1> @test_callable_v32i1_negative(<32 x i1> %x) {
+; CHECK-LABEL: define amdgpu_gfx <32 x i1> @test_callable_v32i1_negative(
+; CHECK-SAME: <32 x i1> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = call <32 x i1> @llvm.amdgcn.readfirstlane.v32i1(<32 x i1> [[X]])
+; CHECK-NEXT:    ret <32 x i1> [[R]]
+;
+  %r = call <32 x i1> @llvm.amdgcn.readfirstlane(<32 x i1> %x)
+  ret <32 x i1> %r
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/abs-1.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/abs-1.ll
index 7037647d116b..fd67fc342149 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/abs-1.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/abs-1.ll
@@ -978,3 +978,14 @@ define i32 @abs_diff_signed_slt_no_nsw_swap(i32 %a, i32 %b) {
   %cond = select i1 %cmp, i32 %sub_ba, i32 %sub_ab
   ret i32 %cond
 }
+
+define <2 x i32> @abs_unary_shuffle_ops(<2 x i32> %x) {
+; CHECK-LABEL: @abs_unary_shuffle_ops(
+; CHECK-NEXT:    [[R2:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[R1:%.*]], i1 false)
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[R2]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %a = shufflevector <2 x i32> %x, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  %r = call <2 x i32> @llvm.abs(<2 x i32> %a, i1 false)
+  ret <2 x i32> %r
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
index 68049ca23019..b3093a92624a 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
@@ -5,7 +5,7 @@ define i8 @add_and_sgt(i8 %x) {
 ; CHECK-LABEL: define i8 @add_and_sgt(
 ; CHECK-SAME: i8 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 8)
-; CHECK-NEXT:    [[S:%.*]] = add nuw i8 [[TMP1]], 16
+; CHECK-NEXT:    [[S:%.*]] = add nuw nsw i8 [[TMP1]], 16
 ; CHECK-NEXT:    ret i8 [[S]]
 ;
   %add = add nsw i8 %x, 16
@@ -155,7 +155,7 @@ define i8 @multi_use_cond_and_sel(i8 %x) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X]], 8
 ; CHECK-NEXT:    call void @use(i1 [[CMP]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 8)
-; CHECK-NEXT:    [[S:%.*]] = add nuw i8 [[TMP1]], 16
+; CHECK-NEXT:    [[S:%.*]] = add nuw nsw i8 [[TMP1]], 16
 ; CHECK-NEXT:    call void @use_byte(i8 [[S]])
 ; CHECK-NEXT:    ret i8 [[S]]
 ;
@@ -399,3 +399,133 @@ define i8 @sub_const_on_lhs_negative(i8 %x) {
   %s = select i1 %cmp, i8 %sub, i8 50
   ret i8 %s
 }
+
+define i8 @smin_ugt(i8 %x) {
+; CHECK-LABEL: define i8 @smin_ugt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 50)
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %smin = call i8 @llvm.smin.i8(i8 %x, i8 50)
+  %cmp = icmp ugt i8 %x, 100
+  %s = select i1 %cmp, i8 50, i8 %smin
+  ret i8 %s
+}
+
+define i8 @smax_ugt(i8 %x) {
+; CHECK-LABEL: define i8 @smax_ugt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 100)
+; CHECK-NEXT:    [[S:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP1]], i8 50)
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %smax = call i8 @llvm.smax.i8(i8 %x, i8 50)
+  %cmp = icmp ugt i8 %x, 100
+  %s = select i1 %cmp, i8 100, i8 %smax
+  ret i8 %s
+}
+
+define i8 @umin_slt(i8 %x) {
+; CHECK-LABEL: define i8 @umin_slt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 0)
+; CHECK-NEXT:    [[S:%.*]] = call i8 @llvm.umin.i8(i8 [[TMP1]], i8 100)
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %cmp = icmp slt i8 %x, 0
+  %umin = tail call i8 @llvm.umin.i8(i8 %x, i8 100)
+  %s = select i1 %cmp, i8 0, i8 %umin
+  ret i8 %s
+}
+
+define i8 @umax_sgt(i8 %x) {
+; CHECK-LABEL: define i8 @umax_sgt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 100)
+; CHECK-NEXT:    [[S:%.*]] = call i8 @llvm.umax.i8(i8 [[TMP1]], i8 50)
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %cmp = icmp sgt i8 %x, 100
+  %umax = tail call i8 @llvm.umax.i8(i8 %x, i8 50)
+  %s = select i1 %cmp, i8 100, i8 %umax
+  ret i8 %s
+}
+
+define i8 @add_sgt_nuw_nsw_safe(i8 %x) {
+; CHECK-LABEL: define i8 @add_sgt_nuw_nsw_safe(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 100)
+; CHECK-NEXT:    [[S:%.*]] = add nuw nsw i8 [[TMP1]], 1
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nuw nsw i8 %x, 1
+  %cmp = icmp sgt i8 %x, 100
+  %s = select i1 %cmp, i8 101, i8 %add
+  ret i8 %s
+}
+
+define i8 @add_sgt_nuw_only(i8 %x) {
+; CHECK-LABEL: define i8 @add_sgt_nuw_only(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 100)
+; CHECK-NEXT:    [[S:%.*]] = add nuw i8 [[TMP1]], 50
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nuw nsw i8 %x, 50
+  %cmp = icmp sgt i8 %x, 100
+  %s = select i1 %cmp, i8 150, i8 %add
+  ret i8 %s
+}
+
+define i8 @add_sgt_nsw_only(i8 %x) {
+; CHECK-LABEL: define i8 @add_sgt_nsw_only(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 100)
+; CHECK-NEXT:    [[S:%.*]] = add nsw i8 [[TMP1]], -99
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %add = add nuw nsw i8 %x, -99
+  %cmp = icmp sgt i8 %x, 100
+  %s = select i1 %cmp, i8 1, i8 %add
+  ret i8 %s
+}
+
+
+define i8 @mul_ult_nuw_nsw_safe(i8 %x) {
+; CHECK-LABEL: define i8 @mul_ult_nuw_nsw_safe(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 10)
+; CHECK-NEXT:    [[S:%.*]] = mul nuw nsw i8 [[TMP1]], 3
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %mul = mul nuw nsw i8 %x, 3
+  %cmp = icmp ult i8 %x, 10
+  %s = select i1 %cmp, i8 30, i8 %mul
+  ret i8 %s
+}
+
+define i8 @mul_ult_nuw_only(i8 %x) {
+; CHECK-LABEL: define i8 @mul_ult_nuw_only(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 10)
+; CHECK-NEXT:    [[S:%.*]] = mul nuw i8 [[TMP1]], 25
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %mul = mul nuw nsw i8 %x, 25
+  %cmp = icmp ult i8 %x, 10
+  %s = select i1 %cmp, i8 250, i8 %mul
+  ret i8 %s
+}
+
+define i8 @mul_ult_nsw_only(i8 %x) {
+; CHECK-LABEL: define i8 @mul_ult_nsw_only(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 40)
+; CHECK-NEXT:    [[S:%.*]] = mul nsw i8 [[TMP1]], -2
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %mul = mul nuw nsw i8 %x, -2
+  %cmp = icmp ult i8 %x, 40
+  %s = select i1 %cmp, i8 -80, i8 %mul
+  ret i8 %s
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/div.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/div.ll
index 7e93612150e8..f0fdc5f54366 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/div.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/div.ll
@@ -494,9 +494,7 @@ define <2 x i8> @sdiv_exact_negated_dividend_constant_divisor_vec_splat(<2 x i8>
 
 define i8 @sdiv_negated_dividend_constant_divisor_smin(i8 %x) {
 ; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_smin(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X:%.*]], -128
-; CHECK-NEXT:    [[D:%.*]] = zext i1 [[TMP1]] to i8
-; CHECK-NEXT:    ret i8 [[D]]
+; CHECK-NEXT:    ret i8 0
 ;
   %neg = sub nsw i8 0, %x
   %d = sdiv i8 %neg, -128
@@ -505,9 +503,7 @@ define i8 @sdiv_negated_dividend_constant_divisor_smin(i8 %x) {
 
 define <2 x i8> @sdiv_negated_dividend_constant_divisor_vec_splat_smin(<2 x i8> %x) {
 ; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_vec_splat_smin(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], splat (i8 -128)
-; CHECK-NEXT:    [[D:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
-; CHECK-NEXT:    ret <2 x i8> [[D]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
 ;
   %neg = sub nsw <2 x i8> zeroinitializer, %x
   %d = sdiv <2 x i8> %neg, <i8 -128, i8 -128>
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/double-float-shrink-2.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/double-float-shrink-2.ll
index f2049e2813eb..f884d2bd1ab5 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/double-float-shrink-2.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/double-float-shrink-2.ll
@@ -449,7 +449,7 @@ define float @test_shrink_intrin_fabs_fast_double_src(double %D) {
 ; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call fast double @llvm.fabs.f64(double %D)
-  %F = fptrunc double %E to float
+  %F = fptrunc fast double %E to float
   ret float %F
 }
 
@@ -611,7 +611,7 @@ define half @test_mismatched_type_intrin_fabs_fast_double_src(double %D) {
 ; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call fast double @llvm.fabs.f64(double %D)
-  %F = fptrunc double %E to half
+  %F = fptrunc fast double %E to half
   ret half %F
 }
 
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/fabs.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/fabs.ll
index 0a22d1431b5f..0d9374410a1d 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/fabs.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/fabs.ll
@@ -1522,7 +1522,7 @@ define float @test_fabs_nsz_used_by_frem(float %x) {
 define half @test_fabs_nsz_used_by_fptrunc(float %x) {
 ; CHECK-LABEL: @test_fabs_nsz_used_by_fptrunc(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[X:%.*]] to half
-; CHECK-NEXT:    [[OP:%.*]] = call nsz half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    [[OP:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
 ; CHECK-NEXT:    ret half [[OP]]
 ;
   %cmp = fcmp oge float %x, 0.000000e+00
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/fcmp-select.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/fcmp-select.ll
index 408bc1cdc268..b622c8636ecc 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/fcmp-select.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/fcmp-select.ll
@@ -268,3 +268,38 @@ define i1 @test_fcmp_select_var_const_unordered(double %x, double %y) {
   %cmp2 = fcmp ugt double %sel, 0x3E80000000000000
   ret i1 %cmp2
 }
+
+define i1 @test_fcmp_ord_select_fcmp_oeq_var_const(double %x) {
+; CHECK-LABEL: @test_fcmp_ord_select_fcmp_oeq_var_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp1 = fcmp ord double %x, 0.000000e+00
+  %sel = select i1 %cmp1, double %x, double 0.000000e+00
+  %cmp2 = fcmp oeq double %sel, 1.000000e+00
+  ret i1 %cmp2
+}
+
+; Make sure that we recognize the SPF correctly.
+
+define float @test_select_nnan_nsz_fcmp_olt(float %x) {
+; CHECK-LABEL: @test_select_nnan_nsz_fcmp_olt(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt float [[X:%.*]], -0.000000e+00
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[TMP1]], float [[X]], float -0.000000e+00
+; CHECK-NEXT:    ret float [[SEL1]]
+;
+  %cmp = fcmp olt float %x, 0.000000e+00
+  %sel = select nnan nsz i1 %cmp, float %x, float -0.000000e+00
+  ret float %sel
+}
+
+define float @test_select_nnan_nsz_fcmp_ult(float %x) {
+; CHECK-LABEL: @test_select_nnan_nsz_fcmp_ult(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[DOTINV]], float -0.000000e+00, float [[X]]
+; CHECK-NEXT:    ret float [[SEL1]]
+;
+  %cmp = fcmp ult float %x, 0.000000e+00
+  %sel = select nnan nsz i1 %cmp, float %x, float -0.000000e+00
+  ret float %sel
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/fma.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/fma.ll
index f0d4f776a5d9..e3d3e722bcc2 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/fma.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/fma.ll
@@ -972,6 +972,19 @@ define <2 x half> @fma_negone_vec_partial_undef(<2 x half> %x, <2 x half> %y) {
   ret <2 x half> %sub
 }
 
+define <2 x float> @fmuladd_unary_shuffle_ops(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmuladd_unary_shuffle_ops(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]])
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x float> [[R1]]
+;
+  %a = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+  %b = shufflevector <2 x float> %y, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+  %c = shufflevector <2 x float> %z, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+  %r = call <2 x float> @llvm.fmuladd(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+  ret <2 x float> %r
+}
+
 ; negative tests
 
 define half @fma_non_negone(half %x, half %y) {
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/fneg.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/fneg.ll
index a9d1b9a4ab83..39117f56fa4e 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/fneg.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/fneg.ll
@@ -42,7 +42,7 @@ define float @fmul_fneg(float %x) {
 
 define float @fmul_fsub_fmf(float %x) {
 ; CHECK-LABEL: @fmul_fsub_fmf(
-; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz float [[X:%.*]], -4.200000e+01
+; CHECK-NEXT:    [[R:%.*]] = fmul nsz float [[X:%.*]], -4.200000e+01
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %m = fmul float %x, 42.0
@@ -52,7 +52,7 @@ define float @fmul_fsub_fmf(float %x) {
 
 define float @fmul_fneg_fmf(float %x) {
 ; CHECK-LABEL: @fmul_fneg_fmf(
-; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz float [[X:%.*]], -4.200000e+01
+; CHECK-NEXT:    [[R:%.*]] = fmul nsz float [[X:%.*]], -4.200000e+01
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %m = fmul float %x, 42.0
@@ -1142,4 +1142,166 @@ define <vscale x 2 x double> @test_fneg_select_svec_3(<vscale x 2 x i1> %cond, <
   ret <vscale x 2 x double> %2
 }
 
+define float @test_fneg_ninf_mul_with_anyzero(float %a) {
+; CHECK-LABEL: @test_fneg_ninf_mul_with_anyzero(
+; CHECK-NEXT:    [[F:%.*]] = fmul float [[A:%.*]], -0.000000e+00
+; CHECK-NEXT:    ret float [[F]]
+;
+  %mul = fmul float %a, 0.0
+  %f = fneg ninf float %mul
+  ret float %f
+}
+
+define float @test_fsub_ninf_mul_with_anyzero(float %a) {
+; CHECK-LABEL: @test_fsub_ninf_mul_with_anyzero(
+; CHECK-NEXT:    [[F2:%.*]] = fmul nsz float [[A:%.*]], -0.000000e+00
+; CHECK-NEXT:    ret float [[F2]]
+;
+  %f1 = fmul nsz float %a, 0.000000
+  %f2 = fsub ninf float -0.000000, %f1
+  ret float %f2
+}
+
+define float @test_fneg_nnan_mul_with_anyzero(float %a) {
+; CHECK-LABEL: @test_fneg_nnan_mul_with_anyzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg nnan float [[A:%.*]]
+; CHECK-NEXT:    [[F2:%.*]] = call nnan float @llvm.copysign.f32(float 0.000000e+00, float [[TMP1]])
+; CHECK-NEXT:    ret float [[F2]]
+;
+  %f1 = fmul ninf float %a, 0.000000
+  %f2 = fneg nnan float %f1
+  ret float %f2
+}
+
+define float @test_fneg_nsz_mul_with_anyzero(float %a) {
+; CHECK-LABEL: @test_fneg_nsz_mul_with_anyzero(
+; CHECK-NEXT:    [[F2:%.*]] = fmul nsz float [[A:%.*]], -0.000000e+00
+; CHECK-NEXT:    ret float [[F2]]
+;
+  %f1 = fmul ninf float %a, 0.000000
+  %f2 = fneg nsz float %f1
+  ret float %f2
+}
+
+define float @test_fneg_ninf_mul_nnan_with_const(float %a) {
+; CHECK-LABEL: @test_fneg_ninf_mul_nnan_with_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg float [[A:%.*]]
+; CHECK-NEXT:    [[F2:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[TMP1]])
+; CHECK-NEXT:    ret float [[F2]]
+;
+  %f1 = fmul nnan float %a, 0.000000
+  %f2 = fneg ninf float %f1
+  ret float %f2
+}
+
+define float @test_fneg_ninf_mul_nsz_with_const(float %a) {
+; CHECK-LABEL: @test_fneg_ninf_mul_nsz_with_const(
+; CHECK-NEXT:    [[F2:%.*]] = fmul nsz float [[A:%.*]], -0.000000e+00
+; CHECK-NEXT:    ret float [[F2]]
+;
+  %f1 = fmul nsz float %a, 0.000000
+  %f2 = fneg ninf float %f1
+  ret float %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_nnan_ninf_with_vec_const(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_nnan_ninf_with_vec_const(
+; CHECK-NEXT:    [[F2:%.*]] = fmul nnan <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul nnan <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_nsz_ninf_with_vec_const(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_nsz_ninf_with_vec_const(
+; CHECK-NEXT:    [[F2:%.*]] = fmul nsz <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul nsz <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_ninf_nnan_mul_with_vec_const(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_ninf_nnan_mul_with_vec_const(
+; CHECK-NEXT:    [[F2:%.*]] = fmul nnan <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg nnan ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_nnan_ninf_with_vec_const2(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_nnan_ninf_with_vec_const2(
+; CHECK-NEXT:    [[F2:%.*]] = fmul nnan ninf <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul ninf <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg nnan ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_reassoc_ninf_with_vec_const1(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_reassoc_ninf_with_vec_const1(
+; CHECK-NEXT:    [[F2:%.*]] = fmul <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul reassoc <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_reassoc_ninf_with_vec_const2(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_reassoc_ninf_with_vec_const2(
+; CHECK-NEXT:    [[F2:%.*]] = fmul ninf <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul ninf <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg reassoc ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_reassoc_ninf_with_vec_const3(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_reassoc_ninf_with_vec_const3(
+; CHECK-NEXT:    [[F2:%.*]] = fmul reassoc <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul reassoc <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg reassoc ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_contract_ninf_with_vec_const1(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_contract_ninf_with_vec_const1(
+; CHECK-NEXT:    [[F2:%.*]] = fmul <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul contract <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_contract_ninf_with_vec_const2(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_contract_ninf_with_vec_const2(
+; CHECK-NEXT:    [[F2:%.*]] = fmul ninf <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul ninf <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg contract ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
+define <2 x float> @test_fneg_mul_combine_contract_ninf_with_vec_const3(<2 x float> %a) {
+; CHECK-LABEL: @test_fneg_mul_combine_contract_ninf_with_vec_const3(
+; CHECK-NEXT:    [[F2:%.*]] = fmul contract <2 x float> [[A:%.*]], <float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[F2]]
+;
+  %f1 = fmul contract <2 x float> %a, <float 0.000000, float -0.000000>
+  %f2 = fneg contract ninf <2 x float> %f1
+  ret <2 x float> %f2
+}
+
 !0 = !{}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/fpcast.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/fpcast.ll
index d5290b572aef..1a3faceebf24 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/fpcast.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/fpcast.ll
@@ -32,7 +32,7 @@ define half @test3(float %a) {
 define half @test3_fast(float %a) {
 ; CHECK-LABEL: @test3_fast(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
-; CHECK-NEXT:    [[C:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    [[C:%.*]] = call fast half @llvm.fabs.f16(half [[TMP1]])
 ; CHECK-NEXT:    ret half [[C]]
 ;
   %b = call float @llvm.fabs.f32(float %a)
@@ -40,6 +40,39 @@ define half @test3_fast(float %a) {
   ret half %c
 }
 
+define half @test3_both_ninf(float %a) {
+; CHECK-LABEL: @test3_both_ninf(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = call ninf half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = call ninf float @llvm.fabs.f32(float %a)
+  %c = fptrunc ninf float %b to half
+  ret half %c
+}
+
+define half @test3_fabs_ninf(float %a) {
+; CHECK-LABEL: @test3_fabs_ninf(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = call ninf float @llvm.fabs.f32(float %a)
+  %c = fptrunc float %b to half
+  ret half %c
+}
+
+define half @test3_fptrunc_ninf(float %a) {
+; CHECK-LABEL: @test3_fptrunc_ninf(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = call ninf half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = call float @llvm.fabs.f32(float %a)
+  %c = fptrunc ninf float %b to half
+  ret half %c
+}
+
 define half @fneg_fptrunc(float %a) {
 ; CHECK-LABEL: @fneg_fptrunc(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/fsub.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/fsub.ll
index cffc63405ddc..28cee50d72c1 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/fsub.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/fsub.ll
@@ -98,7 +98,7 @@ define float @sub_sub_nsz(float %x, float %y, float %z) {
 
 define float @sub_add_neg_x(float %x, float %y) {
 ; CHECK-LABEL: @sub_add_neg_x(
-; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz float [[X:%.*]], -5.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = fmul nsz float [[X:%.*]], -5.000000e+00
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %mul = fmul float %x, 5.000000e+00
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/heterogeneous-poison-dbg-rauw.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/heterogeneous-poison-dbg-rauw.ll
index b2864b61d460..50024be9f13f 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/heterogeneous-poison-dbg-rauw.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/heterogeneous-poison-dbg-rauw.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt < %s -passes=instcombine --experimental-debuginfo-iterators=true  -S | FileCheck %s
-; RUN: opt < %s -passes=instcombine --experimental-debuginfo-iterators=false -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 ;; Test replaceAllDbgUsesWith(). InstCombine uses this function when there is a
 ;; cast of a cast it can eliminate (see InstCombinerImpl::commonCastTransforms).
@@ -11,9 +10,16 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @use_i32(i32)
 declare void @use_i64(i32)
 declare void @use_ptr(ptr)
+declare void @use_ptr1(ptr addrspace(1))
 declare void @llvm.dbg.value(metadata, metadata, metadata) #0
 
 define void @test_int_ptr_int(i64 %A) !dbg !5 {
+; CHECK-LABEL: define void @test_int_ptr_int(
+; CHECK-SAME: i64 [[A:%.*]]) !dbg [[DBG5:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(i64 [[A]], [[META9:![0-9]+]], !DIExpression(DIOpArg(0, i64), DIOpReinterpret(ptr)), [[META12:![0-9]+]])
+; CHECK-NEXT:    call void @use_i64(i64 [[A]])
+; CHECK-NEXT:    ret void
+;
   %1 = inttoptr i64 %A to ptr
   tail call void @llvm.dbg.value(metadata ptr %1, metadata !9, metadata !DIExpression(DIOpArg(0, ptr))), !dbg !12
   %2 = ptrtoint ptr %1 to i64
@@ -22,6 +28,12 @@ define void @test_int_ptr_int(i64 %A) !dbg !5 {
 }
 
 define void @test_ptr_int_ptr(ptr %A) !dbg !13 {
+; CHECK-LABEL: define void @test_ptr_int_ptr(
+; CHECK-SAME: ptr [[A:%.*]]) !dbg [[DBG13:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(ptr [[A]], [[META15:![0-9]+]], !DIExpression(DIOpArg(0, ptr), DIOpReinterpret(i64)), [[META17:![0-9]+]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[A]])
+; CHECK-NEXT:    ret void
+;
   %1 = ptrtoint ptr %A to i64
   tail call void @llvm.dbg.value(metadata i64 %1, metadata !15, metadata !DIExpression(DIOpArg(0, i64))), !dbg !17
   %2 = inttoptr i64 %1 to ptr
@@ -30,6 +42,12 @@ define void @test_ptr_int_ptr(ptr %A) !dbg !13 {
 }
 
 define void @test_zext_trunc(i32 %A) !dbg !18 {
+; CHECK-LABEL: define void @test_zext_trunc(
+; CHECK-SAME: i32 [[A:%.*]]) !dbg [[DBG18:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(i32 [[A]], [[META20:![0-9]+]], !DIExpression(DIOpArg(0, i32), DIOpZExt(i64)), [[META23:![0-9]+]])
+; CHECK-NEXT:    call void @use_i32(i32 [[A]])
+; CHECK-NEXT:    ret void
+;
   %1 = zext i32 %A to i64
   tail call void @llvm.dbg.value(metadata i64 %1, metadata !20, metadata !DIExpression(DIOpArg(0, i64))), !dbg !23
   %2 = trunc i64 %1 to i32
@@ -38,6 +56,13 @@ define void @test_zext_trunc(i32 %A) !dbg !18 {
 }
 
 define void @test_trunc_zext(i64 %A) !dbg !24 {
+; CHECK-LABEL: define void @test_trunc_zext(
+; CHECK-SAME: i64 [[A:%.*]]) !dbg [[DBG24:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(i64 [[A]], [[META26:![0-9]+]], !DIExpression(DIOpArg(0, i64), DIOpConvert(i32)), [[META28:![0-9]+]])
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[A]], 4294967295
+; CHECK-NEXT:    call void @use_i64(i64 [[TMP1]])
+; CHECK-NEXT:    ret void
+;
   %1 = trunc i64 %A to i32
   tail call void @llvm.dbg.value(metadata i32 %1, metadata !26, metadata !DIExpression(DIOpArg(0, i32))), !dbg !28
   %2 = zext i32 %1 to i64
@@ -46,6 +71,12 @@ define void @test_trunc_zext(i64 %A) !dbg !24 {
 }
 
 define void @test_sext_trunc(i32 %A) !dbg !29 {
+; CHECK-LABEL: define void @test_sext_trunc(
+; CHECK-SAME: i32 [[A:%.*]]) !dbg [[DBG29:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(i32 [[A]], [[META31:![0-9]+]], !DIExpression(DIOpArg(0, i32), DIOpSExt(i64)), [[META33:![0-9]+]])
+; CHECK-NEXT:    call void @use_i32(i32 [[A]])
+; CHECK-NEXT:    ret void
+;
   %1 = sext i32 %A to i64
   tail call void @llvm.dbg.value(metadata i64 %1, metadata !31, metadata !DIExpression(DIOpArg(0, i64))), !dbg !33
   %2 = trunc i64 %1 to i32
@@ -53,6 +84,28 @@ define void @test_sext_trunc(i32 %A) !dbg !29 {
   ret void
 }
 
+define void @test_asc_asc(ptr addrspace(1) %A, ptr %B) !dbg !34 {
+; CHECK-LABEL: define void @test_asc_asc(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr [[B:%.*]]) !dbg [[DBG34:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(ptr addrspace(1) [[A]], [[META36:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(1)), DIOpConvert(ptr addrspace(4))), [[META38:![0-9]+]])
+; CHECK-NEXT:    call void @use_ptr1(ptr addrspace(1) [[A]])
+; CHECK-NEXT:      #dbg_value(ptr [[B]], [[META39:![0-9]+]], !DIExpression(DIOpArg(0, ptr), DIOpConvert(ptr addrspace(3))), [[META38]])
+; CHECK-NEXT:    call void @use_ptr(ptr [[B]])
+; CHECK-NEXT:    ret void
+;
+  %1 = addrspacecast ptr addrspace(1) %A to ptr addrspace(4)
+  tail call void @llvm.dbg.value(metadata ptr addrspace(4) %1, metadata !36, metadata !DIExpression(DIOpArg(0, ptr addrspace(4)))), !dbg !38
+  %2 = addrspacecast ptr addrspace(4) %1 to ptr addrspace(1)
+  call void @use_ptr1(ptr addrspace(1) %2)
+
+  %3 = addrspacecast ptr %B to ptr addrspace(3)
+  tail call void @llvm.dbg.value(metadata ptr addrspace(3) %3, metadata !39, metadata !DIExpression(DIOpArg(0, ptr addrspace(3)))), !dbg !38
+  %4 = addrspacecast ptr addrspace(3) %3 to ptr
+  call void @use_ptr(ptr %4)
+
+  ret void
+}
+
 !llvm.dbg.cu = !{!0}
 !llvm.debugify = !{!2, !3}
 !llvm.module.flags = !{!4}
@@ -91,5 +144,48 @@ define void @test_sext_trunc(i32 %A) !dbg !29 {
 !31 = !DILocalVariable(name: "9", scope: !29, file: !1, line: 13, type: !32)
 !32 = !DIBasicType(name: "tys32", size: 32, encoding: DW_ATE_signed)
 !33 = !DILocation(line: 13, column: 1, scope: !29)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
+!34 = distinct !DISubprogram(name: "test_asc_asc", linkageName: "test_asc_asc", scope: null, file: !1, line: 13, type: !6, scopeLine: 13, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !35)
+!35 = !{!36}
+!36 = !DILocalVariable(name: "10", scope: !34, file: !1, line: 13, type: !37)
+!37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!38 = !DILocation(line: 13, column: 1, scope: !34)
+!39 = !DILocalVariable(name: "11", scope: !34, file: !1, line: 13, type: !37)
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META1:![0-9]+]], producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}t.c", directory: {{.*}})
+; CHECK: [[DBG5]] = distinct !DISubprogram(name: "test_int_ptr_int", linkageName: "test_int_ptr_int", scope: null, file: [[META1]], line: 1, type: [[META6:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META8:![0-9]+]])
+; CHECK: [[META6]] = !DISubroutineType(types: [[META7:![0-9]+]])
+; CHECK: [[META7]] = !{}
+; CHECK: [[META8]] = !{[[META9]], [[META11:![0-9]+]]}
+; CHECK: [[META9]] = !DILocalVariable(name: "1", scope: [[DBG5]], file: [[META1]], line: 1, type: [[META10:![0-9]+]])
+; CHECK: [[META10]] = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+; CHECK: [[META11]] = !DILocalVariable(name: "2", scope: [[DBG5]], file: [[META1]], line: 2, type: [[META10]])
+; CHECK: [[META12]] = !DILocation(line: 1, column: 1, scope: [[DBG5]])
+; CHECK: [[DBG13]] = distinct !DISubprogram(name: "test_ptr_int_ptr", linkageName: "test_ptr_int_ptr", scope: null, file: [[META1]], line: 5, type: [[META6]], scopeLine: 5, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META14:![0-9]+]])
+; CHECK: [[META14]] = !{[[META15]], [[META16:![0-9]+]]}
+; CHECK: [[META15]] = !DILocalVariable(name: "3", scope: [[DBG13]], file: [[META1]], line: 5, type: [[META10]])
+; CHECK: [[META16]] = !DILocalVariable(name: "4", scope: [[DBG13]], file: [[META1]], line: 6, type: [[META10]])
+; CHECK: [[META17]] = !DILocation(line: 5, column: 1, scope: [[DBG13]])
+; CHECK: [[DBG18]] = distinct !DISubprogram(name: "test_zext_trunc", linkageName: "test_zext_trunc", scope: null, file: [[META1]], line: 9, type: [[META6]], scopeLine: 9, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META19:![0-9]+]])
+; CHECK: [[META19]] = !{[[META20]], [[META21:![0-9]+]]}
+; CHECK: [[META20]] = !DILocalVariable(name: "5", scope: [[DBG18]], file: [[META1]], line: 9, type: [[META10]])
+; CHECK: [[META21]] = !DILocalVariable(name: "6", scope: [[DBG18]], file: [[META1]], line: 10, type: [[META22:![0-9]+]])
+; CHECK: [[META22]] = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+; CHECK: [[META23]] = !DILocation(line: 9, column: 1, scope: [[DBG18]])
+; CHECK: [[DBG24]] = distinct !DISubprogram(name: "test_trunc_zext", linkageName: "test_trunc_zext", scope: null, file: [[META1]], line: 13, type: [[META6]], scopeLine: 13, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META25:![0-9]+]])
+; CHECK: [[META25]] = !{[[META26]], [[META27:![0-9]+]]}
+; CHECK: [[META26]] = !DILocalVariable(name: "7", scope: [[DBG24]], file: [[META1]], line: 13, type: [[META22]])
+; CHECK: [[META27]] = !DILocalVariable(name: "8", scope: [[DBG24]], file: [[META1]], line: 14, type: [[META10]])
+; CHECK: [[META28]] = !DILocation(line: 13, column: 1, scope: [[DBG24]])
+; CHECK: [[DBG29]] = distinct !DISubprogram(name: "test_sext_trunc", linkageName: "test_sext_trunc", scope: null, file: [[META1]], line: 13, type: [[META6]], scopeLine: 13, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META30:![0-9]+]])
+; CHECK: [[META30]] = !{[[META31]]}
+; CHECK: [[META31]] = !DILocalVariable(name: "9", scope: [[DBG29]], file: [[META1]], line: 13, type: [[META32:![0-9]+]])
+; CHECK: [[META32]] = !DIBasicType(name: "tys32", size: 32, encoding: DW_ATE_signed)
+; CHECK: [[META33]] = !DILocation(line: 13, column: 1, scope: [[DBG29]])
+; CHECK: [[DBG34]] = distinct !DISubprogram(name: "test_asc_asc", linkageName: "test_asc_asc", scope: null, file: [[META1]], line: 13, type: [[META6]], scopeLine: 13, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META35:![0-9]+]])
+; CHECK: [[META35]] = !{[[META36]]}
+; CHECK: [[META36]] = !DILocalVariable(name: "10", scope: [[DBG34]], file: [[META1]], line: 13, type: [[META37:![0-9]+]])
+; CHECK: [[META37]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+; CHECK: [[META38]] = !DILocation(line: 13, column: 1, scope: [[DBG34]])
+; CHECK: [[META39]] = !DILocalVariable(name: "11", scope: [[DBG34]], file: [[META1]], line: 13, type: [[META37]])
+;.
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-dom.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-dom.ll
index 6613bbeb8d6a..a72b5e0bbfa0 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-dom.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-dom.ll
@@ -535,16 +535,13 @@ else:
   ret i1 %cmp1
 }
 
-; TODO: X != Y implies X | Y != 0
 define i1 @or_nonzero_from_nonequal(i8 %x, i8 %y) {
 ; CHECK-LABEL: @or_nonzero_from_nonequal(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[OR:%.*]] = or i8 [[X]], [[Y]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[OR]], 0
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 false
 ; CHECK:       if.else:
 ; CHECK-NEXT:    ret i1 false
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-sub.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-sub.ll
index 4143902bc9c4..13ed7ba0c170 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-sub.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-sub.ll
@@ -290,8 +290,7 @@ define i1 @subC_nsw_ne(i32 %x) {
 ; CHECK-LABEL: @subC_nsw_ne(
 ; CHECK-NEXT:    [[SUBX:%.*]] = sub nsw i32 -2147483647, [[X:%.*]]
 ; CHECK-NEXT:    call void @use(i32 [[SUBX]])
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[X]], 2147483603
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %subx = sub nsw i32 -2147483647, %x
   call void @use(i32 %subx)
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-subadd.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-subadd.ll
new file mode 100644
index 000000000000..fd7e1250d893
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp-subadd.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i1 @test-same-operands-sub-add-nsw-icmp-sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-same-operands-sub-add-nsw-icmp-sgt(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[B]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub nsw i8 %a, %b
+  %add = add nsw i8 %a, %b
+  %cmp = icmp sgt i8 %sub, %add
+  ret i1 %cmp
+}
+
+define i1 @test-same-operands-sub-add-nsw-icmp-slt(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-same-operands-sub-add-nsw-icmp-slt(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[B]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub nsw i8 %a, %b
+  %add = add nsw i8 %a, %b
+  %cmp = icmp slt i8 %sub, %add
+  ret i1 %cmp
+}
+
+define i1 @test-same-operands-sub-add-nsw-icmp-sle(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-same-operands-sub-add-nsw-icmp-sle(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[B]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub nsw i8 %a, %b
+  %add = add nsw i8 %a, %b
+  %cmp = icmp sle i8 %sub, %add
+  ret i1 %cmp
+}
+
+define i1 @test-same-operands-sub-add-nsw-nuw-icmp-eq(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-same-operands-sub-add-nsw-nuw-icmp-eq(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[B]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub nsw i8 %a, %b
+  %add = add nuw i8 %a, %b
+  %cmp = icmp eq i8 %sub, %add
+  ret i1 %cmp
+}
+
+define i1 @test-same-operands-sub-add-nsw-icmp-eq(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-same-operands-sub-add-nsw-icmp-eq(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[B]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub nsw i8 %a, %b
+  %add = add nsw i8 %a, %b
+  %cmp = icmp eq i8 %sub, %add
+  ret i1 %cmp
+}
+
+define i1 @test-add-sub-nsw-icmp-sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-add-sub-nsw-icmp-sgt(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[B]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub nsw i8 %a, %b
+  %add = add nsw i8 %a, %b
+  %cmp = icmp sgt i8 %add, %sub
+  ret i1 %cmp
+}
+
+define i1 @test-add-sub-nuw-icmp-uge(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-add-sub-nuw-icmp-uge(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %sub = sub nuw i8 %a, %b
+  %add = add nuw i8 %a, %b
+  %cmp = icmp uge i8 %add, %sub
+  ret i1 %cmp
+}
+
+; Check not folded
+define i1 @test-add-sub-nuw-icmp-sge(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-add-sub-nuw-icmp-sge(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i8 [[A]], [[B]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[A]], [[B]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i8 [[ADD]], [[SUB]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub nuw i8 %a, %b
+  %add = add nuw i8 %a, %b
+  %cmp = icmp sge i8 %add, %sub
+  ret i1 %cmp
+}
+
+define i1 @test-add-swap-sub-nuw-icmp-uge(i8 %a, i8 %b) {
+; CHECK-LABEL: define i1 @test-add-swap-sub-nuw-icmp-uge(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    ret i1 true
+;
+  %sub = sub nuw i8 %a, %b
+  %add = add nuw i8 %b, %a
+  %cmp = icmp uge i8 %add, %sub
+  ret i1 %cmp
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/icmp.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp.ll
index f5df8573d630..365c17b35a46 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/icmp.ll
@@ -506,8 +506,7 @@ define <2 x i1> @test23vec(<2 x i32> %x) {
 ; unsigned overflow does not happen during offset computation
 define i1 @test24_neg_offs(ptr %p, i64 %offs) {
 ; CHECK-LABEL: @test24_neg_offs(
-; CHECK-NEXT:    [[P1_IDX_NEG:%.*]] = mul i64 [[OFFS:%.*]], -4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[P1_IDX_NEG]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[OFFS:%.*]], -2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %p1 = getelementptr inbounds i32, ptr %p, i64 %offs
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/logical-select.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/logical-select.ll
index 050a53406a9c..87e05002665c 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/logical-select.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/logical-select.ll
@@ -807,9 +807,9 @@ define <2 x i16> @bitcast_vec_cond_commute3(<4 x i8> %cond, <2 x i16> %pc, <2 x
 ; CHECK-LABEL: @bitcast_vec_cond_commute3(
 ; CHECK-NEXT:    [[C:%.*]] = mul <2 x i16> [[PC:%.*]], [[PC]]
 ; CHECK-NEXT:    [[D:%.*]] = mul <2 x i16> [[PD:%.*]], [[PD]]
+; CHECK-NEXT:    [[DOTNOT2:%.*]] = icmp slt <4 x i8> [[COND:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i16> [[D]] to <4 x i8>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i16> [[C]] to <4 x i8>
-; CHECK-NEXT:    [[DOTNOT2:%.*]] = icmp slt <4 x i8> [[COND:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[DOTNOT2]], <4 x i8> [[TMP1]], <4 x i8> [[TMP2]]
 ; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i8> [[TMP3]] to <2 x i16>
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
@@ -1069,8 +1069,8 @@ define <2 x i1> @not_d_bools_vector_poison(<2 x i1> %c, <2 x i1> %x, <2 x i1> %y
 
 define i32 @not_d_allSignBits(i32 %cond, i32 %tval, i32 %fval) {
 ; CHECK-LABEL: @not_d_allSignBits(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[FVAL:%.*]], -1
 ; CHECK-NEXT:    [[DOTNOT2:%.*]] = icmp slt i32 [[COND:%.*]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[FVAL:%.*]], -1
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[DOTNOT2]], i32 [[TVAL:%.*]], i32 [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-fold.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-fold.ll
index 3bb1fd60f3af..a98222537062 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-fold.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-fold.ll
@@ -1598,3 +1598,247 @@ define <2 x i32> @test_umax_smax_vec_neg(<2 x i32> %x) {
   %umax = call <2 x i32> @llvm.umax.v2i32(<2 x i32> %smax, <2 x i32> <i32 1, i32 10>)
   ret <2 x i32> %umax
 }
+
+define i32 @test_smin_sub1_nsw(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smin_sub1_nsw(
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[W:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.smin.i32(i32 [[X:%.*]], i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp slt i32 %x, %w
+  %sub = add nsw i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
+
+define i32 @test_smax_add1_nsw(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smax_add1_nsw(
+; CHECK-NEXT:    [[X2:%.*]] = add nsw i32 [[W:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 [[X2]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp sgt i32 %x, %w
+  %add = add nsw i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umax_add1_nuw(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_umax_add1_nuw(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[W:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.umax.i32(i32 [[X:%.*]], i32 [[ADD]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ugt i32 %x, %w
+  %add = add nuw i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umin_sub1_nuw(i32 %x, i32 range(i32 1, 0) %w) {
+; CHECK-LABEL: @test_umin_sub1_nuw(
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[W:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.umin.i32(i32 [[X:%.*]], i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ult i32 %x, %w
+  %sub = add i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
+
+define i32 @test_smin_sub1_nsw_swapped(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smin_sub1_nsw_swapped(
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[W:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.smin.i32(i32 [[X:%.*]], i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp sgt i32 %w, %x
+  %sub = add nsw i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
+
+define i32 @test_smax_add1_nsw_swapped(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smax_add1_nsw_swapped(
+; CHECK-NEXT:    [[X2:%.*]] = add nsw i32 [[W:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 [[X2]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp slt i32 %w, %x
+  %add = add nsw i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umax_add1_nuw_swapped(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_umax_add1_nuw_swapped(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[W:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.umax.i32(i32 [[X:%.*]], i32 [[ADD]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ult i32 %w, %x
+  %add = add nuw i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umin_sub1_nuw_swapped(i32 %x, i32 range(i32 1, 0) %w) {
+; CHECK-LABEL: @test_umin_sub1_nuw_swapped(
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[W:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.umin.i32(i32 [[X:%.*]], i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ugt i32 %w, %x
+  %sub = add i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
+
+define <2 x i16> @test_smin_sub1_nsw_vec(<2 x i16> %x, <2 x i16> %w) {
+; CHECK-LABEL: @test_smin_sub1_nsw_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw <2 x i16> [[W:%.*]], splat (i16 -1)
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i16> @llvm.smin.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[SUB]])
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %cmp = icmp slt <2 x i16> %x, %w
+  %sub = add nsw <2 x i16> %w, splat (i16 -1)
+  %r = select <2 x i1> %cmp, <2 x i16> %x, <2 x i16> %sub
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @test_smax_add1_nsw_vec(<2 x i16> %x, <2 x i16> %w) {
+; CHECK-LABEL: @test_smax_add1_nsw_vec(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <2 x i16> [[W:%.*]], splat (i16 1)
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i16> @llvm.smax.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[ADD]])
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %cmp = icmp sgt <2 x i16> %x, %w
+  %add = add nsw <2 x i16> %w, splat (i16 1)
+  %r = select <2 x i1> %cmp, <2 x i16> %x, <2 x i16> %add
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @test_umax_add1_nuw_vec(<2 x i16> %x, <2 x i16> %w) {
+; CHECK-LABEL: @test_umax_add1_nuw_vec(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw <2 x i16> [[W:%.*]], splat (i16 1)
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i16> @llvm.umax.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[ADD]])
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %cmp = icmp ugt <2 x i16> %x, %w
+  %add = add nuw <2 x i16> %w, splat (i16 1)
+  %r = select <2 x i1> %cmp, <2 x i16> %x, <2 x i16> %add
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @test_umin_sub1_nuw_vec(<2 x i16> %x, <2 x i16> range(i16 1, 0) %w) {
+; CHECK-LABEL: @test_umin_sub1_nuw_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i16> [[W:%.*]], splat (i16 -1)
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[X:%.*]], <2 x i16> [[SUB]])
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %cmp = icmp ult <2 x i16> %x, %w
+  %sub = add <2 x i16> %w, splat (i16 -1)
+  %r = select <2 x i1> %cmp, <2 x i16> %x, <2 x i16> %sub
+  ret <2 x i16> %r
+}
+
+
+define i32 @test_smin_sub1_nsw_drop_flags(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smin_sub1_nsw_drop_flags(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP]], i32 [[X]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp slt i32 %x, %w
+  %sub = add nsw nuw i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
+
+define i32 @test_smax_add1_nsw_drop_flags(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smax_add1_nsw_drop_flags(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[W:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 [[ADD]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp sgt i32 %x, %w
+  %add = add nsw nuw i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umax_add1_nuw_drop_flags(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_umax_add1_nuw_drop_flags(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[W:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.umax.i32(i32 [[X:%.*]], i32 [[ADD]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ugt i32 %x, %w
+  %add = add nuw nsw i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umin_sub1_nuw_drop_flags(i32 %x, i32 range(i32 1, 0) %w) {
+; CHECK-LABEL: @test_umin_sub1_nuw_drop_flags(
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[W:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.umin.i32(i32 [[X:%.*]], i32 [[SUB]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ult i32 %x, %w
+  %sub = add nsw i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
+
+;; Confirm we don't crash on these cases.
+define i32 @test_smin_or_neg1_nsw(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smin_or_neg1_nsw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP]], i32 [[X]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp slt i32 %x, %w
+  %sub = or disjoint i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
+
+define i32 @test_smax_or_1_nsw(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_smax_or_1_nsw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = or disjoint i32 [[W]], 1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp sgt i32 %x, %w
+  %add = or disjoint i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umax_or_1_nuw(i32 %x, i32 %w) {
+; CHECK-LABEL: @test_umax_or_1_nuw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = or disjoint i32 [[W]], 1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ugt i32 %x, %w
+  %add = or disjoint i32 %w, 1
+  %r = select i1 %cmp, i32 %x, i32 %add
+  ret i32 %r
+}
+
+define i32 @test_umin_or_neg1_nuw(i32 %x, i32 range(i32 1, 0) %w) {
+; CHECK-LABEL: @test_umin_or_neg1_nuw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP]], i32 [[X]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp ult i32 %x, %w
+  %sub = or disjoint i32 %w, -1
+  %r = select i1 %cmp, i32 %x, i32 %sub
+  ret i32 %r
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
index 38930956eda2..52bc3636be35 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
@@ -2511,6 +2511,21 @@ define <3 x i8> @smin_unary_shuffle_ops_uses(<3 x i8> %x, <3 x i8> %y) {
   ret <3 x i8> %r
 }
 
+; negative test - too many uses
+
+define <3 x i8> @smin_unary_shuffle_ops_uses_const(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @smin_unary_shuffle_ops_uses_const(
+; CHECK-NEXT:    [[SX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> <i32 1, i32 0, i32 2>
+; CHECK-NEXT:    call void @use_vec(<3 x i8> [[SX]])
+; CHECK-NEXT:    [[R:%.*]] = call <3 x i8> @llvm.smin.v3i8(<3 x i8> [[SX]], <3 x i8> <i8 1, i8 2, i8 3>)
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %sx = shufflevector <3 x i8> %x, <3 x i8> poison, <3 x i32> <i32 1, i32 0, i32 2>
+  call void @use_vec(<3 x i8> %sx)
+  %r = call <3 x i8> @llvm.smin.v3i8(<3 x i8> %sx, <3 x i8> <i8 1, i8 2, i8 3>)
+  ret <3 x i8> %r
+}
+
 ; This would assert/crash because we tried to zext to i1.
 
 @g = external dso_local global i32, align 4
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/narrow-switch.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/narrow-switch.ll
index 05a30b910e5e..90f56a61fa41 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/narrow-switch.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/narrow-switch.ll
@@ -1,15 +1,27 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --prefix-filecheck-ir-name _ --version 5
 ; Vary legal integer types in data layout.
 ; RUN: opt < %s -passes=instcombine -S -data-layout=n32    | FileCheck %s --check-prefix=ALL --check-prefix=CHECK32
 ; RUN: opt < %s -passes=instcombine -S -data-layout=n32:64 | FileCheck %s --check-prefix=ALL --check-prefix=CHECK64
 
 define i32 @positive1(i64 %a) {
-; ALL-LABEL: @positive1(
-; ALL:         switch i32
-; ALL-NEXT:    i32 10, label %return
-; ALL-NEXT:    i32 100, label %sw.bb1
-; ALL-NEXT:    i32 1001, label %sw.bb2
+; ALL-LABEL: define i32 @positive1(
+; ALL-SAME: i64 [[A:%.*]]) {
+; ALL-NEXT:  [[ENTRY:.*]]:
+; ALL-NEXT:    [[TRUNC:%.*]] = trunc i64 [[A]] to i32
+; ALL-NEXT:    switch i32 [[TRUNC]], label %[[SW_DEFAULT:.*]] [
+; ALL-NEXT:      i32 10, label %[[RETURN:.*]]
+; ALL-NEXT:      i32 100, label %[[SW_BB1:.*]]
+; ALL-NEXT:      i32 1001, label %[[SW_BB2:.*]]
 ; ALL-NEXT:    ]
+; ALL:       [[SW_BB1]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_BB2]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_DEFAULT]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[RETURN]]:
+; ALL-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 24, %[[SW_DEFAULT]] ], [ 123, %[[SW_BB2]] ], [ 213, %[[SW_BB1]] ], [ 231, %[[ENTRY]] ]
+; ALL-NEXT:    ret i32 [[RETVAL_0]]
 ;
 entry:
   %and = and i64 %a, 4294967295
@@ -34,12 +46,24 @@ return:
 }
 
 define i32 @negative1(i64 %a) {
-; ALL-LABEL: @negative1(
-; ALL:         switch i32
-; ALL-NEXT:    i32 -10, label %return
-; ALL-NEXT:    i32 -100, label %sw.bb1
-; ALL-NEXT:    i32 -1001, label %sw.bb2
+; ALL-LABEL: define i32 @negative1(
+; ALL-SAME: i64 [[A:%.*]]) {
+; ALL-NEXT:  [[ENTRY:.*]]:
+; ALL-NEXT:    [[TRUNC:%.*]] = trunc i64 [[A]] to i32
+; ALL-NEXT:    switch i32 [[TRUNC]], label %[[SW_DEFAULT:.*]] [
+; ALL-NEXT:      i32 -10, label %[[RETURN:.*]]
+; ALL-NEXT:      i32 -100, label %[[SW_BB1:.*]]
+; ALL-NEXT:      i32 -1001, label %[[SW_BB2:.*]]
 ; ALL-NEXT:    ]
+; ALL:       [[SW_BB1]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_BB2]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_DEFAULT]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[RETURN]]:
+; ALL-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 24, %[[SW_DEFAULT]] ], [ 123, %[[SW_BB2]] ], [ 213, %[[SW_BB1]] ], [ 231, %[[ENTRY]] ]
+; ALL-NEXT:    ret i32 [[RETVAL_0]]
 ;
 entry:
   %or = or i64 %a, -4294967296
@@ -67,12 +91,24 @@ return:
 ; assertion.
 
 define i32 @trunc72to68(i72 %a) {
-; ALL-LABEL: @trunc72to68(
-; ALL:         switch i68
-; ALL-NEXT:    i68 10, label %return
-; ALL-NEXT:    i68 100, label %sw.bb1
-; ALL-NEXT:    i68 1001, label %sw.bb2
+; ALL-LABEL: define i32 @trunc72to68(
+; ALL-SAME: i72 [[A:%.*]]) {
+; ALL-NEXT:  [[ENTRY:.*]]:
+; ALL-NEXT:    [[TRUNC:%.*]] = trunc i72 [[A]] to i68
+; ALL-NEXT:    switch i68 [[TRUNC]], label %[[SW_DEFAULT:.*]] [
+; ALL-NEXT:      i68 10, label %[[RETURN:.*]]
+; ALL-NEXT:      i68 100, label %[[SW_BB1:.*]]
+; ALL-NEXT:      i68 1001, label %[[SW_BB2:.*]]
 ; ALL-NEXT:    ]
+; ALL:       [[SW_BB1]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_BB2]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_DEFAULT]]:
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[RETURN]]:
+; ALL-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 24, %[[SW_DEFAULT]] ], [ 123, %[[SW_BB2]] ], [ 213, %[[SW_BB1]] ], [ 231, %[[ENTRY]] ]
+; ALL-NEXT:    ret i32 [[RETVAL_0]]
 ;
 entry:
   %and = and i72 %a, 295147905179352825855
@@ -103,15 +139,38 @@ return:
 ; because both are illegal.
 
 define void @trunc64to58(i64 %a) {
-; ALL-LABEL: @trunc64to58(
-; CHECK32:         switch i58
-; CHECK32-NEXT:    i58 0, label %sw.bb1
-; CHECK32-NEXT:    i58 18717182647723699, label %sw.bb2
+; CHECK32-LABEL: define void @trunc64to58(
+; CHECK32-SAME: i64 [[A:%.*]]) {
+; CHECK32-NEXT:  [[ENTRY:.*:]]
+; CHECK32-NEXT:    [[TMP0:%.*]] = trunc i64 [[A]] to i58
+; CHECK32-NEXT:    [[TMP1:%.*]] = and i58 [[TMP0]], 15
+; CHECK32-NEXT:    [[TRUNC:%.*]] = mul nuw i58 [[TMP1]], 18717182647723699
+; CHECK32-NEXT:    switch i58 [[TRUNC]], label %[[SW_DEFAULT:.*]] [
+; CHECK32-NEXT:      i58 0, label %[[SW_BB1:.*]]
+; CHECK32-NEXT:      i58 18717182647723699, label %[[SW_BB2:.*]]
 ; CHECK32-NEXT:    ]
-; CHECK64:         switch i64
-; CHECK64-NEXT:    i64 0, label %sw.bb1
-; CHECK64-NEXT:    i64 18717182647723699, label %sw.bb2
+; CHECK32:       [[SW_BB1]]:
+; CHECK32-NEXT:    br label %[[SW_DEFAULT]]
+; CHECK32:       [[SW_BB2]]:
+; CHECK32-NEXT:    br label %[[SW_DEFAULT]]
+; CHECK32:       [[SW_DEFAULT]]:
+; CHECK32-NEXT:    ret void
+;
+; CHECK64-LABEL: define void @trunc64to58(
+; CHECK64-SAME: i64 [[A:%.*]]) {
+; CHECK64-NEXT:  [[ENTRY:.*:]]
+; CHECK64-NEXT:    [[_TMP0:%.*]] = and i64 [[A]], 15
+; CHECK64-NEXT:    [[TMP0:%.*]] = mul nuw nsw i64 [[_TMP0]], 18717182647723699
+; CHECK64-NEXT:    switch i64 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+; CHECK64-NEXT:      i64 0, label %[[SW_BB1:.*]]
+; CHECK64-NEXT:      i64 18717182647723699, label %[[SW_BB2:.*]]
 ; CHECK64-NEXT:    ]
+; CHECK64:       [[SW_BB1]]:
+; CHECK64-NEXT:    br label %[[SW_DEFAULT]]
+; CHECK64:       [[SW_BB2]]:
+; CHECK64-NEXT:    br label %[[SW_DEFAULT]]
+; CHECK64:       [[SW_DEFAULT]]:
+; CHECK64-NEXT:    ret void
 ;
 entry:
   %tmp0 = and i64 %a, 15
@@ -136,18 +195,19 @@ sw.default:
 ; https://llvm.org/bugs/show_bug.cgi?id=31260
 
 define i8 @PR31260(i8 %x) {
-; ALL-LABEL: @PR31260(
-; ALL-NEXT:  entry:
-; ALL-NEXT:    [[T4:%.*]] = and i8 [[X:%.*]], 2
-; ALL-NEXT:    switch i8 [[T4]], label [[EXIT:%.*]] [
-; ALL-NEXT:    i8 0, label [[CASE126:%.*]]
-; ALL-NEXT:    i8 2, label [[CASE124:%.*]]
+; ALL-LABEL: define i8 @PR31260(
+; ALL-SAME: i8 [[X:%.*]]) {
+; ALL-NEXT:  [[ENTRY:.*:]]
+; ALL-NEXT:    [[T4:%.*]] = and i8 [[X]], 2
+; ALL-NEXT:    switch i8 [[T4]], label %[[EXIT:.*]] [
+; ALL-NEXT:      i8 0, label %[[CASE126:.*]]
+; ALL-NEXT:      i8 2, label %[[CASE124:.*]]
 ; ALL-NEXT:    ]
-; ALL:       exit:
+; ALL:       [[EXIT]]:
 ; ALL-NEXT:    ret i8 1
-; ALL:       case126:
+; ALL:       [[CASE126]]:
 ; ALL-NEXT:    ret i8 3
-; ALL:       case124:
+; ALL:       [[CASE124]]:
 ; ALL-NEXT:    ret i8 5
 ;
 entry:
@@ -169,12 +229,33 @@ case124:
 ; Make sure the arithmetic evaluation of the switch
 ; condition is evaluated on the original type
 define i32 @trunc32to16(i32 %a0) #0 {
-; ALL-LABEL: @trunc32to16(
-; ALL:         switch i16
-; ALL-NEXT:    i16 63, label %sw.bb
-; ALL-NEXT:    i16 1, label %sw.bb1
-; ALL-NEXT:    i16 100, label %sw.bb2
+; ALL-LABEL: define i32 @trunc32to16(
+; ALL-SAME: i32 [[A0:%.*]]) {
+; ALL-NEXT:  [[ENTRY:.*:]]
+; ALL-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; ALL-NEXT:    [[XOR:%.*]] = lshr i32 [[A0]], 16
+; ALL-NEXT:    [[TMP0:%.*]] = trunc nuw i32 [[XOR]] to i16
+; ALL-NEXT:    [[TRUNC:%.*]] = xor i16 [[TMP0]], 15784
+; ALL-NEXT:    switch i16 [[TRUNC]], label %[[SW_EPILOG:.*]] [
+; ALL-NEXT:      i16 63, label %[[SW_BB:.*]]
+; ALL-NEXT:      i16 1, label %[[SW_BB1:.*]]
+; ALL-NEXT:      i16 100, label %[[SW_BB2:.*]]
 ; ALL-NEXT:    ]
+; ALL:       [[SW_BB]]:
+; ALL-NEXT:    store i32 90, ptr [[RETVAL]], align 4
+; ALL-NEXT:    br label %[[RETURN:.*]]
+; ALL:       [[SW_BB1]]:
+; ALL-NEXT:    store i32 91, ptr [[RETVAL]], align 4
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_BB2]]:
+; ALL-NEXT:    store i32 92, ptr [[RETVAL]], align 4
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[SW_EPILOG]]:
+; ALL-NEXT:    store i32 113, ptr [[RETVAL]], align 4
+; ALL-NEXT:    br label %[[RETURN]]
+; ALL:       [[RETURN]]:
+; ALL-NEXT:    [[RVAL:%.*]] = load i32, ptr [[RETVAL]], align 4
+; ALL-NEXT:    ret i32 [[RVAL]]
 ;
 entry:
   %retval = alloca i32, align 4
@@ -182,9 +263,9 @@ entry:
   %shr = lshr i32 %xor, 16
   %add = add i32 %shr, -917677090
   switch i32 %add, label %sw.epilog [
-    i32 -917677027, label %sw.bb
-    i32 -917677089, label %sw.bb1
-    i32 -917676990, label %sw.bb2
+  i32 -917677027, label %sw.bb
+  i32 -917677089, label %sw.bb1
+  i32 -917676990, label %sw.bb2
   ]
 
 sw.bb:                                            ; preds = %entry
@@ -219,11 +300,32 @@ declare i32 @goo()
 ; if original type is legal (i32 in this case)
 
 define void @PR29009() {
-; ALL-LABEL: @PR29009(
-; ALL:         switch i32
-; ALL-NEXT:    i32 0, label
-; ALL-NEXT:    i32 3, label
+; ALL-LABEL: define void @PR29009() {
+; ALL-NEXT:    br label %[[BB1:.*]]
+; ALL:       [[BB1]]:
+; ALL-NEXT:    [[TMP2:%.*]] = load volatile i32, ptr @njob, align 4
+; ALL-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP2]], 0
+; ALL-NEXT:    br i1 [[DOTNOT]], label %[[BB10:.*]], label %[[BB3:.*]]
+; ALL:       [[BB3]]:
+; ALL-NEXT:    [[TMP4:%.*]] = call i32 @goo()
+; ALL-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 7
+; ALL-NEXT:    switch i32 [[TMP5]], label %[[BB6:.*]] [
+; ALL-NEXT:      i32 0, label %[[BB7:.*]]
+; ALL-NEXT:      i32 3, label %[[BB8:.*]]
 ; ALL-NEXT:    ]
+; ALL:       [[BB6]]:
+; ALL-NEXT:    store i32 6, ptr @a, align 4
+; ALL-NEXT:    br label %[[BB9:.*]]
+; ALL:       [[BB7]]:
+; ALL-NEXT:    store i32 1, ptr @a, align 4
+; ALL-NEXT:    br label %[[BB9]]
+; ALL:       [[BB8]]:
+; ALL-NEXT:    store i32 2, ptr @a, align 4
+; ALL-NEXT:    br label %[[BB9]]
+; ALL:       [[BB9]]:
+; ALL-NEXT:    br label %[[BB1]]
+; ALL:       [[BB10]]:
+; ALL-NEXT:    ret void
 ;
   br label %1
 
@@ -236,8 +338,8 @@ define void @PR29009() {
   %5 = call i32 @goo()
   %6 = and i32 %5, 7
   switch i32 %6, label %7 [
-    i32 0, label %8
-    i32 3, label %9
+  i32 0, label %8
+  i32 3, label %9
   ]
 
 ; <label>:7:                                      ; preds = %4
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/or-bitmask.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/or-bitmask.ll
index 3b482dc1794d..3c992dfea569 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/or-bitmask.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/or-bitmask.ll
@@ -36,13 +36,9 @@ define i32 @add_select_cmp_and2(i32 %in) {
 
 define i32 @add_select_cmp_and3(i32 %in) {
 ; CHECK-LABEL: @add_select_cmp_and3(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
-; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[TMP1]], 72
-; CHECK-NEXT:    [[BITOP2:%.*]] = and i32 [[IN]], 4
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[BITOP2]], 0
-; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 0, i32 288
-; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP]], [[SEL2]]
-; CHECK-NEXT:    ret i32 [[OUT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 7
+; CHECK-NEXT:    [[TEMP1:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    ret i32 [[TEMP1]]
 ;
   %bitop0 = and i32 %in, 1
   %cmp0 = icmp eq i32 %bitop0, 0
@@ -60,12 +56,9 @@ define i32 @add_select_cmp_and3(i32 %in) {
 
 define i32 @add_select_cmp_and4(i32 %in) {
 ; CHECK-LABEL: @add_select_cmp_and4(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
-; CHECK-NEXT:    [[OUT:%.*]] = mul nuw nsw i32 [[TMP1]], 72
-; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[IN]], 12
-; CHECK-NEXT:    [[TEMP3:%.*]] = mul nuw nsw i32 [[TMP2]], 72
-; CHECK-NEXT:    [[OUT1:%.*]] = or disjoint i32 [[OUT]], [[TEMP3]]
-; CHECK-NEXT:    ret i32 [[OUT1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[IN:%.*]], 15
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[TMP2]], 72
+; CHECK-NEXT:    ret i32 [[TEMP2]]
 ;
   %bitop0 = and i32 %in, 1
   %cmp0 = icmp eq i32 %bitop0, 0
@@ -361,6 +354,103 @@ define i64 @mask_select_types_1(i64 %in) {
   ret i64 %out
 }
 
+define i32 @add_select_cmp_mixed1(i32 %in) {
+; CHECK-LABEL: @add_select_cmp_mixed1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[OUT:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %mask = and i32 %in, 1
+  %sel0 = mul i32 %mask, 72
+  %bitop1 = and i32 %in, 2
+  %cmp1 = icmp eq i32 %bitop1, 0
+  %sel1 = select i1 %cmp1, i32 0, i32 144
+  %out = or disjoint i32 %sel0, %sel1
+  ret i32 %out
+}
+
+define i32 @add_select_cmp_mixed2(i32 %in) {
+; CHECK-LABEL: @add_select_cmp_mixed2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[OUT:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %bitop0 = and i32 %in, 1
+  %cmp0 = icmp eq i32 %bitop0, 0
+  %mask = and i32 %in, 2
+  %sel0 = select i1 %cmp0, i32 0, i32 72
+  %sel1 = mul i32 %mask, 72
+  %out = or disjoint i32 %sel0, %sel1
+  ret i32 %out
+}
+
+define i32 @add_select_cmp_and_mul(i32 %in) {
+; CHECK-LABEL: @add_select_cmp_and_mul(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[OUT:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %mask0 = and i32 %in, 1
+  %sel0 = mul i32 %mask0, 72
+  %mask1 = and i32 %in, 2
+  %sel1 = mul i32 %mask1, 72
+  %out = or disjoint i32 %sel0, %sel1
+  ret i32 %out
+}
+
+define i32 @add_select_cmp_mixed2_mismatch(i32 %in) {
+; CHECK-LABEL: @add_select_cmp_mixed2_mismatch(
+; CHECK-NEXT:    [[BITOP0:%.*]] = and i32 [[IN:%.*]], 1
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i32 [[BITOP0]], 0
+; CHECK-NEXT:    [[MASK:%.*]] = and i32 [[IN]], 2
+; CHECK-NEXT:    [[SEL0:%.*]] = select i1 [[CMP0]], i32 0, i32 73
+; CHECK-NEXT:    [[SEL1:%.*]] = mul nuw nsw i32 [[MASK]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[SEL0]], [[SEL1]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %bitop0 = and i32 %in, 1
+  %cmp0 = icmp eq i32 %bitop0, 0
+  %mask = and i32 %in, 2
+  %sel0 = select i1 %cmp0, i32 0, i32 73
+  %sel1 = mul i32 %mask, 72
+  %out = or disjoint i32 %sel0, %sel1
+  ret i32 %out
+}
+
+define i32 @add_select_cmp_and_mul_mismatch(i32 %in) {
+; CHECK-LABEL: @add_select_cmp_and_mul_mismatch(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[IN:%.*]] to i1
+; CHECK-NEXT:    [[SEL0:%.*]] = select i1 [[TMP1]], i32 73, i32 0
+; CHECK-NEXT:    [[MASK1:%.*]] = and i32 [[IN]], 2
+; CHECK-NEXT:    [[SEL1:%.*]] = mul nuw nsw i32 [[MASK1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[SEL0]], [[SEL1]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %mask0 = and i32 %in, 1
+  %sel0 = mul i32 %mask0, 73
+  %mask1 = and i32 %in, 2
+  %sel1 = mul i32 %mask1, 72
+  %out = or disjoint i32 %sel0, %sel1
+  ret i32 %out
+}
+
+define i32 @and_mul_non_disjoint(i32 %in) {
+; CHECK-LABEL: @and_mul_non_disjoint(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 2
+; CHECK-NEXT:    [[OUT:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[MASK1:%.*]] = and i32 [[IN]], 4
+; CHECK-NEXT:    [[SEL1:%.*]] = mul nuw nsw i32 [[MASK1]], 72
+; CHECK-NEXT:    [[OUT1:%.*]] = or i32 [[OUT]], [[SEL1]]
+; CHECK-NEXT:    ret i32 [[OUT1]]
+;
+  %mask0 = and i32 %in, 2
+  %sel0 = mul i32 %mask0, 72
+  %mask1 = and i32 %in, 4
+  %sel1 = mul i32 %mask1, 72
+  %out = or i32 %sel0, %sel1
+  ret i32 %out
+}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CONSTSPLAT: {{.*}}
 ; CONSTVEC: {{.*}}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/powi.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/powi.ll
index d76f92c1849a..422792a5a2c2 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/powi.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/powi.ll
@@ -564,3 +564,29 @@ define double @powi_fmul_powi_x_overflow(double noundef %x) {
   %mul = fmul reassoc double %p1, %x
   ret double %mul
 }
+
+define <3 x float> @powi_unary_shuffle_ops(<3 x float> %x, i32 %power) {
+; CHECK-LABEL: @powi_unary_shuffle_ops(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <3 x float> @llvm.powi.v3f32.i32(<3 x float> [[X:%.*]], i32 [[POWER:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> <i32 1, i32 0, i32 2>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %sx = shufflevector <3 x float> %x, <3 x float> poison, <3 x i32> <i32 1, i32 0, i32 2>
+  %r = call <3 x float> @llvm.powi(<3 x float> %sx, i32 %power)
+  ret <3 x float> %r
+}
+
+; Negative test - multiple uses
+
+define <3 x float> @powi_unary_shuffle_ops_use(<3 x float> %x, i32 %power, ptr %p) {
+; CHECK-LABEL: @powi_unary_shuffle_ops_use(
+; CHECK-NEXT:    [[SX:%.*]] = shufflevector <3 x float> [[X:%.*]], <3 x float> poison, <3 x i32> <i32 1, i32 0, i32 2>
+; CHECK-NEXT:    store <3 x float> [[SX]], ptr [[P:%.*]], align 16
+; CHECK-NEXT:    [[R:%.*]] = call <3 x float> @llvm.powi.v3f32.i32(<3 x float> [[SX]], i32 [[POWER:%.*]])
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %sx = shufflevector <3 x float> %x, <3 x float> poison, <3 x i32> <i32 1, i32 0, i32 2>
+  store <3 x float> %sx, ptr %p
+  %r = call <3 x float> @llvm.powi(<3 x float> %sx, i32 %power)
+  ret <3 x float> %r
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/pr142518.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/pr142518.ll
new file mode 100644
index 000000000000..980ed24b5eb9
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/pr142518.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+define i8 @pr142518(ptr %p, i8 %x, i1 %c) "instcombine-no-verify-fixpoint" {
+; CHECK-LABEL: define i8 @pr142518(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[X]], -1
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[NOT1:%.*]] = xor i1 [[CMP1]], true
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[NOT1]]
+; CHECK-NEXT:    [[CMP:%.*]] = xor i1 [[OR]], true
+; CHECK-NEXT:    [[EXT2:%.*]] = zext i1 [[CMP]] to i8
+; CHECK-NEXT:    store i8 [[EXT2]], ptr [[P]], align 1
+; CHECK-NEXT:    br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[NOT3:%.*]] = xor i1 [[OR]], true
+; CHECK-NEXT:    [[EXT3:%.*]] = zext i1 [[NOT3]] to i8
+; CHECK-NEXT:    ret i8 [[EXT3]]
+;
+entry:
+  %flag = alloca i8, align 1
+  %cmp = icmp slt i8 %x, -1
+  br label %loop
+
+loop:
+  %phi = phi i1 [ %cmp, %entry ], [ %c, %loop ]
+  %not1 = xor i1 %phi, true
+  %or = or i1 %cmp, %not1
+  %not2 = xor i1 %or, true
+  %ext2 = zext i1 %not2 to i8
+  store i8 %ext2, ptr %p, align 1
+  store i8 1, ptr %flag, align 1
+  %flagv = load i8, ptr %flag, align 1
+  %cond = icmp eq i8 %flagv, 0
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  %not3 = xor i1 %or, true
+  %ext3 = zext i1 %not3 to i8
+  ret i8 %ext3
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/scmp.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/scmp.ll
index 2140a59de3fa..2bf22aeb7a6e 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/scmp.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/scmp.ll
@@ -423,6 +423,19 @@ define i8 @scmp_from_select_eq_and_gt_commuted3(i32 %x, i32 %y) {
   ret i8 %r
 }
 
+define <3 x i2> @scmp_unary_shuffle_ops(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: define <3 x i2> @scmp_unary_shuffle_ops(
+; CHECK-SAME: <3 x i8> [[X:%.*]], <3 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <3 x i2> @llvm.scmp.v3i2.v3i8(<3 x i8> [[X]], <3 x i8> [[Y]])
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i2> [[TMP1]], <3 x i2> poison, <3 x i32> <i32 1, i32 0, i32 2>
+; CHECK-NEXT:    ret <3 x i2> [[R]]
+;
+  %sx = shufflevector <3 x i8> %x, <3 x i8> poison, <3 x i32> <i32 1, i32 0, i32 2>
+  %sy = shufflevector <3 x i8> %y, <3 x i8> poison, <3 x i32> <i32 1, i32 0, i32 2>
+  %r = call <3 x i2> @llvm.scmp(<3 x i8> %sx, <3 x i8> %sy)
+  ret <3 x i2> %r
+}
+
 ; Negative test: true value of outer select is not zero
 define i8 @scmp_from_select_eq_and_gt_neg1(i32 %x, i32 %y) {
 ; CHECK-LABEL: define i8 @scmp_from_select_eq_and_gt_neg1(
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/select-and-or.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/select-and-or.ll
index b14186d737e9..453ca666a747 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/select-and-or.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/select-and-or.ll
@@ -104,6 +104,20 @@ define i1 @logical_or_implies(i32 %x) {
   ret i1 %res
 }
 
+; Safe to convert to or due to poison implication.
+define <vscale x 2 x i1> @logical_or_implies_scalablevec(<vscale x 2 x i32> %x) {
+; CHECK-LABEL: @logical_or_implies_scalablevec(
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq <vscale x 2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq <vscale x 2 x i32> [[X]], splat (i32 42)
+; CHECK-NEXT:    [[RES:%.*]] = or <vscale x 2 x i1> [[C1]], [[C2]]
+; CHECK-NEXT:    ret <vscale x 2 x i1> [[RES]]
+;
+  %c1 = icmp eq <vscale x 2 x i32> %x, zeroinitializer
+  %c2 = icmp eq <vscale x 2 x i32> %x, splat (i32 42)
+  %res = select <vscale x 2 x i1> %c1, <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> %c2
+  ret <vscale x 2 x i1> %res
+}
+
 ; Will fold after conversion to or.
 define i1 @logical_or_implies_folds(i32 %x) {
 ; CHECK-LABEL: @logical_or_implies_folds(
@@ -129,6 +143,20 @@ define i1 @logical_and_implies(i32 %x) {
   ret i1 %res
 }
 
+; Safe to convert to and due to poison implication.
+define <vscale x 2 x i1> @logical_and_implies_scalablevec(<vscale x 2 x i32> %x) {
+; CHECK-LABEL: @logical_and_implies_scalablevec(
+; CHECK-NEXT:    [[C1:%.*]] = icmp ne <vscale x 2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C2:%.*]] = icmp ne <vscale x 2 x i32> [[X]], splat (i32 42)
+; CHECK-NEXT:    [[RES:%.*]] = and <vscale x 2 x i1> [[C1]], [[C2]]
+; CHECK-NEXT:    ret <vscale x 2 x i1> [[RES]]
+;
+  %c1 = icmp ne <vscale x 2 x i32> %x, zeroinitializer
+  %c2 = icmp ne <vscale x 2 x i32> %x, splat (i32 42)
+  %res = select <vscale x 2 x i1> %c1, <vscale x 2 x i1> %c2, <vscale x 2 x i1> zeroinitializer
+  ret <vscale x 2 x i1> %res
+}
+
 ; Will fold after conversion to and.
 define i1 @logical_and_implies_folds(i32 %x) {
 ; CHECK-LABEL: @logical_and_implies_folds(
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/select-cmp-eq-op-fold.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/select-cmp-eq-op-fold.ll
index 760b1ef055ad..2e6bbdf5da74 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/select-cmp-eq-op-fold.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/select-cmp-eq-op-fold.ll
@@ -182,3 +182,23 @@ define i8 @replace_with_y_for_simple_binop_fail(i8 %x, i8 noundef %y, i8 %z, i8
   %sel = select i1 %cmp, i8 %mul, i8 %z
   ret i8 %sel
 }
+
+; Make sure we don't run into an infinite loop.
+define i32 @pr142405(i32 noundef %x) {
+; CHECK-LABEL: @pr142405(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 0)
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[SMAX]], 65535
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], [[MASKED]]
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[SMAX]], 1
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[CMP]], i32 [[TMP0]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+entry:
+  %smax = call i32 @llvm.smax.i32(i32 %x, i32 0)
+  %masked = and i32 %smax, 65535
+  %cmp = icmp eq i32 %x, %masked
+  %sel = select i1 %cmp, i32 %smax, i32 0
+  %and = and i32 %sel, 1
+  ret i32 %and
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/select-to-cmp.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/select-to-cmp.ll
new file mode 100644
index 000000000000..a76d4b138686
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/select-to-cmp.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+; Tests for select to scmp
+
+define i32 @scmp_x_0_inverted(i32 %x) {
+; CHECK-LABEL: define i32 @scmp_x_0_inverted(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 0)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %2 = icmp ne i32 %x, 0
+  %3 = zext i1 %2 to i32
+  %4 = icmp sgt i32 %x, -1
+  %5 = select i1 %4, i32 %3, i32 -1
+  ret i32 %5
+}
+
+; y = -10
+define i32 @scmp_x_0_inverted_const_neg10(i32 %x) {
+; CHECK-LABEL: define i32 @scmp_x_0_inverted_const_neg10(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 -10)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = icmp ne i32 %x, -10
+  %2 = zext i1 %1 to i32
+  %3 = icmp sgt i32 %x, -11
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; y = 7 (i8)
+define i8 @scmp_x_0_inverted_i8(i8 %x) {
+; CHECK-LABEL: define i8 @scmp_x_0_inverted_i8(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.scmp.i8.i8(i8 [[X]], i8 7)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = icmp ne i8 %x, 7
+  %2 = zext i1 %1 to i8
+  %3 = icmp sgt i8 %x, 6
+  %4 = select i1 %3, i8 %2, i8 -1
+  ret i8 %4
+}
+
+; scmp using ints of two kinds- i32 and i64
+define i32 @scmp_x_0_inverted_i64_neq(i32 %x) {
+; CHECK-LABEL: define i32 @scmp_x_0_inverted_i64_neq(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = call i64 @llvm.scmp.i64.i32(i32 [[X]], i32 0)
+; CHECK-NEXT:    [[RET:%.*]] = trunc i64 [[SEL]] to i32
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %x64 = sext i32 %x to i64
+  %cmp1 = icmp ne i64 %x64, 0
+  %zext = zext i1 %cmp1 to i64
+  %cmp2 = icmp sgt i64 %x64, -1
+  %sel = select i1 %cmp2, i64 %zext, i64 -1
+  %ret = trunc i64 %sel to i32
+  ret i32 %ret
+}
+
+; Same example as previous but with inequality
+define i32 @scmp_x_0_inverted_i64_sgt(i32 %x) {
+; CHECK-LABEL: define i32 @scmp_x_0_inverted_i64_sgt(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = call i64 @llvm.scmp.i64.i32(i32 [[X]], i32 0)
+; CHECK-NEXT:    [[RET:%.*]] = trunc i64 [[SEL]] to i32
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %x64 = sext i32 %x to i64
+  %cmp1 = icmp sgt i64 %x64, 0
+  %zext = zext i1 %cmp1 to i64
+  %cmp2 = icmp sgt i64 %x64, -1
+  %sel = select i1 %cmp2, i64 %zext, i64 -1
+  %ret = trunc i64 %sel to i32
+  ret i32 %ret
+}
+
+; y = -1000
+define i32 @scmp_x_0_inverted_const_neg1000(i32 %x) {
+; CHECK-LABEL: define i32 @scmp_x_0_inverted_const_neg1000(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 -1000)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = icmp sgt i32 %x, -1000
+  %2 = zext i1 %1 to i32
+  %3 = icmp sgt i32 %x, -1001
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; y = 1729
+define i32 @scmp_x_0_inverted_const_1729_sgt(i32 %x) {
+; CHECK-LABEL: define i32 @scmp_x_0_inverted_const_1729_sgt(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 1729)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = icmp sgt i32 %x, 1729
+  %2 = zext i1 %1 to i32
+  %3 = icmp sgt i32 %x, 1728
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; ucmp with 10
+define i32 @ucmp_x_10_inverted(i32 %x) {
+; CHECK-LABEL: define i32 @ucmp_x_10_inverted(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ucmp.i32.i32(i32 [[X]], i32 10)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = icmp ne i32 %x, 10
+  %2 = zext i1 %1 to i32
+  %3 = icmp ugt i32 %x, 9
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; ucmp with -3, wraps around
+define i32 @ucmp_x_neg1_inverted(i32 %x) {
+; CHECK-LABEL: define i32 @ucmp_x_neg1_inverted(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ucmp.i32.i32(i32 [[X]], i32 -3)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = icmp ne i32 %x, -3
+  %2 = zext i1 %1 to i32
+  %3 = icmp ugt i32 %x, -4
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; ucmp with -4, wraps around
+define i8 @ucmp_x_neg4_i8_ugt(i8 %x) {
+; CHECK-LABEL: define i8 @ucmp_x_neg4_i8_ugt(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.ucmp.i8.i8(i8 [[X]], i8 -4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = icmp ugt i8 %x, -4
+  %2 = zext i1 %1 to i8
+  %3 = icmp ugt i8 %x, -5
+  %4 = select i1 %3, i8 %2, i8 -1
+  ret i8 %4
+}
+
+; Vector tests
+
+; Test with splat vec
+define <4 x i32> @scmp_x_0_inverted_splat_vec(<4 x i32> %x) {
+; CHECK-LABEL: define <4 x i32> @scmp_x_0_inverted_splat_vec(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> [[X]], <4 x i32> zeroinitializer)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %2 = icmp ne <4 x i32> %x, zeroinitializer
+  %3 = zext <4 x i1> %2 to <4 x i32>
+  %4 = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %5
+}
+
+; Test with non-splat vector and different bitwidth
+define <4 x i32> @non_splat_vec_scmp_diff_bitwidth(<4 x i32> %x) {
+; CHECK-LABEL: define <4 x i32> @non_splat_vec_scmp_diff_bitwidth(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = call <4 x i64> @llvm.scmp.v4i64.v4i32(<4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 -1, i32 5>)
+; CHECK-NEXT:    [[RET:%.*]] = trunc <4 x i64> [[SEL]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+  %x64 = sext <4 x i32> %x to <4 x i64>
+  %cmp1 = icmp slt <4 x i64> %x64, <i64 0, i64 1, i64 -1, i64 5>
+  %sext = sext <4 x i1> %cmp1 to <4 x i64>
+  %cmp2 = icmp slt <4 x i64> %x64, <i64 1, i64 2, i64 0, i64 6>
+  %sel = select <4 x i1> %cmp2, <4 x i64> %sext, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
+  %ret = trunc <4 x i64> %sel to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+; Negative examples
+
+; Not scmp due to wrong RHS of the predicate
+define i32 @scmp_ne_0(i32 %0) {
+; CHECK-LABEL: define i32 @scmp_ne_0(
+; CHECK-SAME: i32 [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP3]], i32 -1
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %2 = icmp ne i32 %0, 0
+  %3 = zext i1 %2 to i32
+  %4 = icmp sgt i32 %0, 1
+  %5 = select i1 %4, i32 %3, i32 -1
+  ret i32 %5
+}
+
+; y = 0 with unsigned compare but RHS wraps
+define i32 @ucmp_x_0_inverted(i32 %x) {
+; CHECK-LABEL: define i32 @ucmp_x_0_inverted(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    ret i32 -1
+;
+  %1 = icmp ne i32 %x, 0
+  %2 = zext i1 %1 to i32
+  %3 = icmp ugt i32 %x, -1
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; Don't fold with INT32_MIN
+define i32 @scmp_x_0_inverted_const_min(i32 %x) {
+; CHECK-LABEL: define i32 @scmp_x_0_inverted_const_min(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X]], -2147483648
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %1 = icmp ne i32 %x, -2147483648
+  %2 = zext i1 %1 to i32
+  %3 = icmp sge i32 %x, -2147483648
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; Unsigned cmp of zext of i32 with i64 -1 should always be -1
+define i32 @ucmp_x_0_inverted_i64_ugt(i32 %x) {
+; CHECK-LABEL: define i32 @ucmp_x_0_inverted_i64_ugt(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    ret i32 -1
+;
+  %x64 = zext i32 %x to i64
+  %cmp1 = icmp ugt i64 %x64, 0
+  %zext = zext i1 %cmp1 to i64
+  %cmp2 = icmp ugt i64 %x64, -1
+  %sel = select i1 %cmp2, i64 %zext, i64 -1
+  %ret = trunc i64 %sel to i32
+  ret i32 %ret
+}
+
+; y = 4294967295 (UINT32_MAX), simply sign extend neq
+define i32 @ucmp_x_const_u32max(i32 %x) {
+; CHECK-LABEL: define i32 @ucmp_x_const_u32max(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %1 = icmp ugt i32 %x, 4294967295
+  %2 = zext i1 %1 to i32
+  %3 = icmp ugt i32 %x, 4294967294
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; Don't fold with different signedness
+define i32 @different_signedness_neg(i32 %x) {
+; CHECK-LABEL: define i32 @different_signedness_neg(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X]], -10
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[X]], -11
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 -1
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %1 = icmp ugt i32 %x, -10
+  %2 = zext i1 %1 to i32
+  %3 = icmp sgt i32 %x, -11
+  %4 = select i1 %3, i32 %2, i32 -1
+  ret i32 %4
+}
+
+; Test with wrong false value
+define <4 x i32> @scmp_x_0_inverted_vec(<4 x i32> %x) {
+; CHECK-LABEL: define <4 x i32> @scmp_x_0_inverted_vec(
+; CHECK-SAME: <4 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i32> [[X]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP2]], <4 x i32> <i32 -1, i32 -2, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %2 = icmp ne <4 x i32> %x, zeroinitializer
+  %3 = zext <4 x i1> %2 to <4 x i32>
+  %4 = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> <i32 -1, i32 -2, i32 -1, i32 -1>
+  ret <4 x i32> %5
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/select.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/select.ll
index e16f6ad2cfc9..ef5874ffd46a 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/select.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/select.ll
@@ -5047,3 +5047,50 @@ define <2 x ptr> @select_freeze_constant_expression_vector_gep(i1 %cond, <2 x pt
   %sel = select i1 %cond, <2 x ptr> %y, <2 x ptr> %freeze
   ret <2 x ptr> %sel
 }
+
+define void @no_fold_masked_min_loop(ptr nocapture readonly %vals, ptr nocapture readonly %masks, ptr nocapture %out, i64 %n) {
+; CHECK-LABEL: @no_fold_masked_min_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXT_INDEX:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i8 [ -1, [[ENTRY]] ], [ [[RES:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[VAL_PTR:%.*]] = getelementptr inbounds i8, ptr [[VALS:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[MASK_PTR:%.*]] = getelementptr inbounds i8, ptr [[MASKS:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i8, ptr [[VAL_PTR]], align 1
+; CHECK-NEXT:    [[MASK:%.*]] = load i8, ptr [[MASK_PTR]], align 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[MASK]], 0
+; CHECK-NEXT:    [[MASKED_VAL:%.*]] = select i1 [[COND]], i8 [[VAL]], i8 -1
+; CHECK-NEXT:    [[RES]] = call i8 @llvm.umin.i8(i8 [[ACC]], i8 [[MASKED_VAL]])
+; CHECK-NEXT:    [[NEXT_INDEX]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXT_INDEX]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store i8 [[RES]], ptr [[OUT:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi i64 [0, %entry], [%next_index, %loop]
+  %acc = phi i8 [255, %entry], [%res, %loop]
+
+  %val_ptr = getelementptr inbounds i8, ptr %vals, i64 %index
+  %mask_ptr = getelementptr inbounds i8, ptr %masks, i64 %index
+
+  %val = load i8, ptr %val_ptr, align 1
+  %mask = load i8, ptr %mask_ptr, align 1
+
+  %cond = icmp eq i8 %mask, 0
+  %masked_val = select i1 %cond, i8 %val, i8 -1
+  %res = call i8 @llvm.umin.i8(i8 %acc, i8 %masked_val)
+
+  %next_index = add i64 %index, 1
+  %done = icmp eq i64 %next_index, %n
+  br i1 %done, label %exit, label %loop
+
+exit:
+  store i8 %res, ptr %out, align 1
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/simple_phi_condition.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/simple_phi_condition.ll
index 544ebaecaa75..e1a90d6a1c68 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/simple_phi_condition.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/simple_phi_condition.ll
@@ -654,293 +654,484 @@ merge:
 
 define i32 @test_phi_to_zext(i8 noundef %0) {
 ; CHECK-LABEL: @test_phi_to_zext(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[BB2:%.*]] [
-; CHECK-NEXT:      i8 -1, label [[BB3:%.*]]
-; CHECK-NEXT:      i8 0, label [[BB4:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB1:%.*]]
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i8 -1, label [[SW_255:%.*]]
+; CHECK-NEXT:      i8 0, label [[SW_0:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW_1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       sw.0:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 1, [[SW_1]] ], [ 0, [[SW_0]] ], [ 255, [[SW_255]] ]
+; CHECK-NEXT:    ret i32 [[DOT0]]
+;
+entry:
+  switch i8 %0, label %default [
+  i8 255, label %sw.255
+  i8 0, label %sw.0
+  i8 1, label %sw.1
+  ]
+
+default:
+  unreachable
+
+sw.255:
+  br label %merge
+
+sw.0:
+  br label %merge
+
+sw.1:
+  br label %merge
+
+merge:
+  %.0 = phi i32 [ 1, %sw.1 ], [ 0, %sw.0 ], [ 255, %sw.255 ]
+  ret i32 %.0
+}
+
+define i32 @test_phi_to_zext_inverted(i8 noundef %0) {
+; CHECK-LABEL: @test_phi_to_zext_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i8 -1, label [[SW_255:%.*]]
+; CHECK-NEXT:      i8 0, label [[SW_0:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW_1:%.*]]
 ; CHECK-NEXT:    ]
-; CHECK:       bb2:
+; CHECK:       default:
 ; CHECK-NEXT:    unreachable
-; CHECK:       bb3:
-; CHECK-NEXT:    br label [[BB5:%.*]]
-; CHECK:       bb4:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 1, [[BB1]] ], [ 0, [[BB4]] ], [ 255, [[BB3]] ]
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       sw.0:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ -256, [[SW_255]] ], [ -2, [[SW_1]] ], [ -1, [[SW_0]] ]
 ; CHECK-NEXT:    ret i32 [[DOT0]]
 ;
-start:
-  switch i8 %0, label %bb2 [
-  i8 -1, label %bb3
-  i8 0, label %bb4
-  i8 1, label %bb1
+entry:
+  switch i8 %0, label %default [
+  i8 255, label %sw.255
+  i8 0, label %sw.0
+  i8 1, label %sw.1
   ]
 
-bb2:                                              ; preds = %start
+default:
   unreachable
 
-bb3:                                              ; preds = %start
-  br label %bb5
+sw.255:
+  br label %merge
 
-bb4:                                              ; preds = %start
-  br label %bb5
+sw.0:
+  br label %merge
 
-bb1:                                              ; preds = %start
-  br label %bb5
+sw.1:
+  br label %merge
 
-bb5:                                              ; preds = %bb1, %bb4, %bb3
-  %.0 = phi i32 [ 1, %bb1 ], [ 0, %bb4 ], [ 255, %bb3 ]
+merge:
+  %.0 = phi i32  [ -256, %sw.255 ], [ -2, %sw.1 ], [ -1, %sw.0 ]
   ret i32 %.0
 }
 
+define i8 @test_multiple_predecessors_phi_to_zext(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @test_multiple_predecessors_phi_to_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF2_TRUE:%.*]], label [[IF2_FALSE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if2.true:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if2.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ 0, [[IF_FALSE]] ], [ 1, [[IF2_TRUE]] ], [ 1, [[IF2_FALSE]] ]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br i1 %cond2, label %if2.true, label %if2.false
+
+if.false:
+  br label %merge
+
+if2.true:
+  br label %merge
+
+if2.false:
+  br label %merge
+
+merge:
+  %ret = phi i8 [ 0, %if.false ], [ 1, %if2.true ], [ 1, %if2.false ]
+  ret i8 %ret
+}
+
 define i32 @test_phi_to_sext(i8 noundef %0) {
 ; CHECK-LABEL: @test_phi_to_sext(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[BB2:%.*]] [
-; CHECK-NEXT:      i8 -1, label [[BB3:%.*]]
-; CHECK-NEXT:      i8 0, label [[BB4:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB1:%.*]]
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i8 -1, label [[SW_255:%.*]]
+; CHECK-NEXT:      i8 0, label [[SW_0:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW_1:%.*]]
 ; CHECK-NEXT:    ]
-; CHECK:       bb2:
+; CHECK:       default:
 ; CHECK-NEXT:    unreachable
-; CHECK:       bb3:
-; CHECK-NEXT:    br label [[BB5:%.*]]
-; CHECK:       bb4:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 1, [[BB1]] ], [ 0, [[BB4]] ], [ -1, [[BB3]] ]
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       sw.0:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 1, [[SW_1]] ], [ 0, [[SW_0]] ], [ -1, [[SW_255]] ]
 ; CHECK-NEXT:    ret i32 [[DOT0]]
 ;
-start:
-  switch i8 %0, label %bb2 [
-  i8 -1, label %bb3
-  i8 0, label %bb4
-  i8 1, label %bb1
+entry:
+  switch i8 %0, label %default [
+  i8 255, label %sw.255
+  i8 0, label %sw.0
+  i8 1, label %sw.1
   ]
 
-bb2:                                              ; preds = %start
+default:
   unreachable
 
-bb3:                                              ; preds = %start
-  br label %bb5
+sw.255:
+  br label %merge
 
-bb4:                                              ; preds = %start
-  br label %bb5
+sw.0:
+  br label %merge
 
-bb1:                                              ; preds = %start
-  br label %bb5
+sw.1:
+  br label %merge
 
-bb5:                                              ; preds = %bb1, %bb4, %bb3
-  %.0 = phi i32 [ 1, %bb1 ], [ 0, %bb4 ], [ -1, %bb3 ]
+merge:
+  %.0 = phi i32 [ 1, %sw.1 ], [ 0, %sw.0 ], [ -1, %sw.255 ]
   ret i32 %.0
 }
 
 define i32 @test_phi_to_sext_inverted(i8 noundef %0) {
 ; CHECK-LABEL: @test_phi_to_sext_inverted(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[BB2:%.*]] [
-; CHECK-NEXT:      i8 -1, label [[BB3:%.*]]
-; CHECK-NEXT:      i8 0, label [[BB4:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB1:%.*]]
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i8 -1, label [[SW_255:%.*]]
+; CHECK-NEXT:      i8 0, label [[SW_0:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW_1:%.*]]
 ; CHECK-NEXT:    ]
-; CHECK:       bb2:
+; CHECK:       default:
 ; CHECK-NEXT:    unreachable
-; CHECK:       bb3:
-; CHECK-NEXT:    br label [[BB5:%.*]]
-; CHECK:       bb4:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ -2, [[BB1]] ], [ -1, [[BB4]] ], [ 0, [[BB3]] ]
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       sw.0:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ -2, [[SW_1]] ], [ -1, [[SW_0]] ], [ 0, [[SW_255]] ]
 ; CHECK-NEXT:    ret i32 [[DOT0]]
 ;
-start:
-  switch i8 %0, label %bb2 [
-  i8 -1, label %bb3
-  i8 0, label %bb4
-  i8 1, label %bb1
+entry:
+  switch i8 %0, label %default [
+  i8 255, label %sw.255
+  i8 0, label %sw.0
+  i8 1, label %sw.1
   ]
 
-bb2:                                              ; preds = %start
+default:
   unreachable
 
-bb3:                                              ; preds = %start
-  br label %bb5
+sw.255:
+  br label %merge
 
-bb4:                                              ; preds = %start
-  br label %bb5
+sw.0:
+  br label %merge
 
-bb1:                                              ; preds = %start
-  br label %bb5
+sw.1:
+  br label %merge
 
-bb5:                                              ; preds = %bb1, %bb4, %bb3
-  %.0 = phi i32 [ -2, %bb1 ], [ -1, %bb4 ], [ 0, %bb3 ]
+merge:
+  %.0 = phi i32 [ -2, %sw.1 ], [ -1, %sw.0 ], [ 0, %sw.255 ]
   ret i32 %.0
 }
 
+define i8 @test_multiple_predecessors_phi_to_sext(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @test_multiple_predecessors_phi_to_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF2_TRUE:%.*]], label [[IF2_FALSE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if2.true:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if2.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ 0, [[IF_FALSE]] ], [ -1, [[IF2_TRUE]] ], [ -1, [[IF2_FALSE]] ]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br i1 %cond2, label %if2.true, label %if2.false
+
+if.false:
+  br label %merge
+
+if2.true:
+  br label %merge
+
+if2.false:
+  br label %merge
+
+merge:
+  %ret = phi i8 [ 0, %if.false ], [ -1, %if2.true ], [ -1, %if2.false ]
+  ret i8 %ret
+}
+
 define i32 @test_neg_value_not_possible_to_zext_or_sext(i8 noundef %0) {
 ; CHECK-LABEL: @test_neg_value_not_possible_to_zext_or_sext(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[BB2:%.*]] [
-; CHECK-NEXT:      i8 -1, label [[BB3:%.*]]
-; CHECK-NEXT:      i8 0, label [[BB4:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB1:%.*]]
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i8 -1, label [[SW_255:%.*]]
+; CHECK-NEXT:      i8 0, label [[SW_0:%.*]]
+; CHECK-NEXT:      i8 1, label [[SW_1:%.*]]
 ; CHECK-NEXT:    ]
-; CHECK:       bb2:
+; CHECK:       default:
 ; CHECK-NEXT:    unreachable
-; CHECK:       bb3:
-; CHECK-NEXT:    br label [[BB5:%.*]]
-; CHECK:       bb4:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 1, [[BB1]] ], [ 0, [[BB4]] ], [ 511, [[BB3]] ]
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       sw.0:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 1, [[SW_1]] ], [ 0, [[SW_0]] ], [ 511, [[SW_255]] ]
 ; CHECK-NEXT:    ret i32 [[DOT0]]
 ;
-start:
-  switch i8 %0, label %bb2 [
-  i8 -1, label %bb3
-  i8 0, label %bb4
-  i8 1, label %bb1
+entry:
+  switch i8 %0, label %default [
+  i8 255, label %sw.255
+  i8 0, label %sw.0
+  i8 1, label %sw.1
   ]
 
-bb2:                                              ; preds = %start
+default:
   unreachable
 
-bb3:                                              ; preds = %start
-  br label %bb5
+sw.255:
+  br label %merge
 
-bb4:                                              ; preds = %start
-  br label %bb5
+sw.0:
+  br label %merge
 
-bb1:                                              ; preds = %start
-  br label %bb5
+sw.1:
+  br label %merge
 
-bb5:                                              ; preds = %bb1, %bb4, %bb3
-  %.0 = phi i32 [ 1, %bb1 ], [ 0, %bb4 ], [ 511, %bb3 ]
+merge:
+  %.0 = phi i32 [ 1, %sw.1 ], [ 0, %sw.0 ], [ 511, %sw.255 ]
   ret i32 %.0
 }
 
+define i8 @test_neg_multiple_predecessors_value_not_possible_to_zext_or_sext(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @test_neg_multiple_predecessors_value_not_possible_to_zext_or_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF2_TRUE:%.*]], label [[IF2_FALSE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if2.true:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if2.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ 0, [[IF_FALSE]] ], [ 1, [[IF2_TRUE]] ], [ -1, [[IF2_FALSE]] ]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+entry:
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br i1 %cond2, label %if2.true, label %if2.false
+
+if.false:
+  br label %merge
+
+if2.true:
+  br label %merge
+
+if2.false:
+  br label %merge
+
+merge:
+  %ret = phi i8 [ 0, %if.false ], [ 1, %if2.true ], [ -1, %if2.false ]
+  ret i8 %ret
+}
+
 define i8 @test_phi_to_trunc(i32 %0) {
 ; CHECK-LABEL: @test_phi_to_trunc(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    switch i32 [[TMP0:%.*]], label [[BB2:%.*]] [
-; CHECK-NEXT:      i32 0, label [[BB5:%.*]]
-; CHECK-NEXT:      i32 1, label [[BB4:%.*]]
-; CHECK-NEXT:      i32 255, label [[BB1:%.*]]
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i32 0, label [[MERGE:%.*]]
+; CHECK-NEXT:      i32 1, label [[SW_1:%.*]]
+; CHECK-NEXT:      i32 255, label [[SW_255:%.*]]
 ; CHECK-NEXT:    ]
-; CHECK:       bb2:
+; CHECK:       default:
 ; CHECK-NEXT:    unreachable
-; CHECK:       bb4:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[DOT0:%.*]] = phi i8 [ -1, [[BB1]] ], [ 1, [[BB4]] ], [ 0, [[START:%.*]] ]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i8 [ -1, [[SW_255]] ], [ 1, [[SW_1]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[DOT0]]
 ;
-start:
-  switch i32 %0, label %bb2 [
-  i32 0, label %bb5
-  i32 1, label %bb4
-  i32 255, label %bb1
+entry:
+  switch i32 %0, label %default [
+  i32 0, label %merge
+  i32 1, label %sw.1
+  i32 255, label %sw.255
   ]
 
-bb2:                                              ; preds = %start
+default:
   unreachable
 
-bb4:                                              ; preds = %start
-  br label %bb5
+sw.1:
+  br label %merge
 
-bb1:                                              ; preds = %start
-  br label %bb5
+sw.255:
+  br label %merge
 
-bb5:                                              ; preds = %start, %bb1, %bb4
-  %.0 = phi i8 [ -1, %bb1 ], [ 1, %bb4 ], [ 0, %start ]
+merge:
+  %.0 = phi i8 [ -1, %sw.255 ], [ 1, %sw.1 ], [ 0, %entry ]
   ret i8 %.0
 }
 
 define i8 @test_phi_to_trunc_inverted(i32 %0) {
 ; CHECK-LABEL: @test_phi_to_trunc_inverted(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    switch i32 [[TMP0:%.*]], label [[BB2:%.*]] [
-; CHECK-NEXT:      i32 0, label [[BB5:%.*]]
-; CHECK-NEXT:      i32 1, label [[BB4:%.*]]
-; CHECK-NEXT:      i32 255, label [[BB1:%.*]]
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i32 0, label [[MERGE:%.*]]
+; CHECK-NEXT:      i32 1, label [[SW_1:%.*]]
+; CHECK-NEXT:      i32 255, label [[SW_255:%.*]]
 ; CHECK-NEXT:    ]
-; CHECK:       bb2:
+; CHECK:       default:
 ; CHECK-NEXT:    unreachable
-; CHECK:       bb4:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[DOT0:%.*]] = phi i8 [ 0, [[BB1]] ], [ -2, [[BB4]] ], [ -1, [[START:%.*]] ]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i8 [ 0, [[SW_255]] ], [ -2, [[SW_1]] ], [ -1, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[DOT0]]
 ;
-start:
-  switch i32 %0, label %bb2 [
-  i32 0, label %bb5
-  i32 1, label %bb4
-  i32 255, label %bb1
+entry:
+  switch i32 %0, label %default [
+  i32 0, label %merge
+  i32 1, label %sw.1
+  i32 255, label %sw.255
   ]
 
-bb2:                                              ; preds = %start
+default:
   unreachable
 
-bb4:                                              ; preds = %start
-  br label %bb5
+sw.1:
+  br label %merge
 
-bb1:                                              ; preds = %start
-  br label %bb5
+sw.255:
+  br label %merge
 
-bb5:                                              ; preds = %start, %bb1, %bb4
-  %.0 = phi i8 [ 0, %bb1 ], [ -2, %bb4 ], [ -1, %start ]
+merge:
+  %.0 = phi i8 [ 0, %sw.255 ], [ -2, %sw.1 ], [ -1, %entry ]
   ret i8 %.0
 }
 
-define i8 @test_neg_value_not_possible_to_trunc(i32 %0) {
-; CHECK-LABEL: @test_neg_value_not_possible_to_trunc(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    switch i32 [[TMP0:%.*]], label [[BB2:%.*]] [
-; CHECK-NEXT:      i32 0, label [[BB5:%.*]]
-; CHECK-NEXT:      i32 1, label [[BB4:%.*]]
-; CHECK-NEXT:      i32 511, label [[BB1:%.*]]
+define i8 @test_neg_phi_to_trunc(i32 %0) {
+; CHECK-LABEL: @test_neg_phi_to_trunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i32 0, label [[MERGE:%.*]]
+; CHECK-NEXT:      i32 1, label [[SW_1:%.*]]
+; CHECK-NEXT:      i32 511, label [[SW_511:%.*]]
 ; CHECK-NEXT:    ]
-; CHECK:       bb2:
+; CHECK:       default:
 ; CHECK-NEXT:    unreachable
-; CHECK:       bb4:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB5]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[DOT0:%.*]] = phi i8 [ -1, [[BB1]] ], [ 1, [[BB4]] ], [ 0, [[START:%.*]] ]
+; CHECK:       sw.1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.511:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i8 [ 2, [[SW_511]] ], [ 1, [[SW_1]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[DOT0]]
 ;
-start:
-  switch i32 %0, label %bb2 [
-  i32 0, label %bb5
-  i32 1, label %bb4
-  i32 511, label %bb1
+entry:
+  switch i32 %0, label %default [
+  i32 0, label %merge
+  i32 1, label %sw.1
+  i32 511, label %sw.511
   ]
 
-bb2:                                              ; preds = %start
+default:
   unreachable
 
-bb4:                                              ; preds = %start
-  br label %bb5
+sw.1:
+  br label %merge
 
-bb1:                                              ; preds = %start
-  br label %bb5
+sw.511:
+  br label %merge
 
-bb5:                                              ; preds = %start, %bb1, %bb4
-  %.0 = phi i8 [ 255, %bb1 ], [ 1, %bb4 ], [ 0, %start ]
+merge:
+  %.0 = phi i8 [ 2, %sw.511 ], [ 1, %sw.1 ], [ 0, %entry ]
+  ret i8 %.0
+}
+
+define i8 @test_neg_phi_to_trunc_multiple_values_trunc_to_same_value(i32 %0) {
+; CHECK-LABEL: @test_neg_phi_to_trunc_multiple_values_trunc_to_same_value(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[TMP0:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i32 0, label [[MERGE:%.*]]
+; CHECK-NEXT:      i32 255, label [[SW_255:%.*]]
+; CHECK-NEXT:      i32 511, label [[SW_511:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.255:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       sw.511:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i8 [ -1, [[SW_255]] ], [ -1, [[SW_511]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i8 [[DOT0]]
+;
+entry:
+  switch i32 %0, label %default [
+  i32 0, label %merge
+  i32 255, label %sw.255
+  i32 511, label %sw.511
+  ]
+
+default:
+  unreachable
+
+sw.255:
+  br label %merge
+
+sw.511:
+  br label %merge
+
+merge:
+  %.0 = phi i8 [ -1, %sw.255 ], [ -1, %sw.511 ], [ 0, %entry ]
   ret i8 %.0
 }
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/sqrt.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/sqrt.ll
index 0f4db3b3a65a..2fda5bc37d02 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/sqrt.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/sqrt.ll
@@ -201,6 +201,17 @@ define <2 x float> @sqrt_exp_vec(<2 x float> %x) {
   ret <2 x float> %res
 }
 
+define <2 x float> @sqrt_unary_shuffle_ops(<2 x float> %x) {
+; CHECK-LABEL: @sqrt_unary_shuffle_ops(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.sqrt.v2f32(<2 x float> [[A:%.*]])
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x float> [[R1]]
+;
+  %a = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+  %r = call <2 x float> @llvm.sqrt(<2 x float> %a)
+  ret <2 x float> %r
+}
+
 declare i32 @foo(double)
 declare double @sqrt(double) readnone
 declare float @sqrtf(float)
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/sub-gep.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/sub-gep.ll
index c86a1a37bd7a..375be8a3d69c 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/sub-gep.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/sub-gep.ll
@@ -80,7 +80,7 @@ define i32 @test_inbounds_nuw_trunc(ptr %base, i64 %idx) {
 
 define i64 @test_inbounds_nuw_swapped(ptr %base, i64 %idx) {
 ; CHECK-LABEL: @test_inbounds_nuw_swapped(
-; CHECK-NEXT:    [[P2_IDX_NEG:%.*]] = mul i64 [[IDX:%.*]], -4
+; CHECK-NEXT:    [[P2_IDX_NEG:%.*]] = mul nsw i64 [[IDX:%.*]], -4
 ; CHECK-NEXT:    ret i64 [[P2_IDX_NEG]]
 ;
   %p2 = getelementptr inbounds [0 x i32], ptr %base, i64 0, i64 %idx
@@ -104,7 +104,7 @@ define i64 @test_inbounds1_nuw_swapped(ptr %base, i64 %idx) {
 
 define i64 @test_inbounds2_nuw_swapped(ptr %base, i64 %idx) {
 ; CHECK-LABEL: @test_inbounds2_nuw_swapped(
-; CHECK-NEXT:    [[P2_IDX_NEG:%.*]] = mul i64 [[IDX:%.*]], -4
+; CHECK-NEXT:    [[P2_IDX_NEG:%.*]] = mul nsw i64 [[IDX:%.*]], -4
 ; CHECK-NEXT:    ret i64 [[P2_IDX_NEG]]
 ;
   %p2 = getelementptr inbounds [0 x i32], ptr %base, i64 0, i64 %idx
@@ -279,8 +279,8 @@ define i16 @test24_as1(ptr addrspace(1) %P, i16 %A) {
 
 define i64 @test24a(ptr %P, i64 %A){
 ; CHECK-LABEL: @test24a(
-; CHECK-NEXT:    [[DIFF_NEG:%.*]] = sub i64 0, [[A:%.*]]
-; CHECK-NEXT:    ret i64 [[DIFF_NEG]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = sub nsw i64 0, [[A:%.*]]
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
 ;
   %B = getelementptr inbounds i8, ptr %P, i64 %A
   %C = ptrtoint ptr %B to i64
@@ -291,8 +291,8 @@ define i64 @test24a(ptr %P, i64 %A){
 
 define i16 @test24a_as1(ptr addrspace(1) %P, i16 %A) {
 ; CHECK-LABEL: @test24a_as1(
-; CHECK-NEXT:    [[DIFF_NEG:%.*]] = sub i16 0, [[A:%.*]]
-; CHECK-NEXT:    ret i16 [[DIFF_NEG]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = sub nsw i16 0, [[A:%.*]]
+; CHECK-NEXT:    ret i16 [[GEPDIFF]]
 ;
   %B = getelementptr inbounds i8, ptr addrspace(1) %P, i16 %A
   %C = ptrtoint ptr addrspace(1) %B to i16
@@ -860,3 +860,168 @@ _Z3fooPKc.exit:
   %tobool = icmp eq i64 %2, 0
   ret i1 %tobool
 }
+
+define i64 @multiple_geps_one_chain(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @multiple_geps_one_chain(
+; CHECK-NEXT:    [[P2_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = shl i64 [[P2_IDX1]], 2
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %p2 = getelementptr inbounds i32, ptr %base, i64 %idx
+  %p3 = getelementptr inbounds i32, ptr %p2, i64 %idx2
+  %i1 = ptrtoint ptr %base to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @multiple_geps_one_chain_commuted(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @multiple_geps_one_chain_commuted(
+; CHECK-NEXT:    [[P2_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul i64 [[P2_IDX1]], -4
+; CHECK-NEXT:    ret i64 [[DOTNEG]]
+;
+  %p2 = getelementptr inbounds i32, ptr %base, i64 %idx
+  %p3 = getelementptr inbounds i32, ptr %p2, i64 %idx2
+  %i1 = ptrtoint ptr %base to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i1, %i2
+  ret i64 %d
+}
+
+define i64 @multiple_geps_two_chains(ptr %base, i64 %idx, i64 %idx2, i64 %idx3) {
+; CHECK-LABEL: @multiple_geps_two_chains(
+; CHECK-NEXT:    [[P2_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[P2_IDX1]], [[IDX3:%.*]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p2 = getelementptr inbounds i32, ptr %base, i64 %idx
+  %p3 = getelementptr inbounds i32, ptr %p2, i64 %idx2
+  %p4 = getelementptr inbounds i32, ptr %base, i64 %idx3
+  %i1 = ptrtoint ptr %p4 to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @multiple_geps_two_chains_commuted(ptr %base, i64 %idx, i64 %idx2, i64 %idx3) {
+; CHECK-LABEL: @multiple_geps_two_chains_commuted(
+; CHECK-NEXT:    [[P2_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[IDX3:%.*]], [[P2_IDX1]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p2 = getelementptr inbounds i32, ptr %base, i64 %idx
+  %p3 = getelementptr inbounds i32, ptr %p2, i64 %idx2
+  %p4 = getelementptr inbounds i32, ptr %base, i64 %idx3
+  %i1 = ptrtoint ptr %p4 to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i1, %i2
+  ret i64 %d
+}
+
+declare void @use(ptr)
+
+define i64 @multiple_geps_two_chains_gep_base(ptr %base, i64 %base.idx, i64 %idx, i64 %idx2, i64 %idx3) {
+; CHECK-LABEL: @multiple_geps_two_chains_gep_base(
+; CHECK-NEXT:    [[GEP_BASE:%.*]] = getelementptr inbounds i32, ptr [[BASE:%.*]], i64 [[BASE_IDX:%.*]]
+; CHECK-NEXT:    call void @use(ptr [[GEP_BASE]])
+; CHECK-NEXT:    [[P2_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[P2_IDX1]], [[IDX3:%.*]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %gep.base = getelementptr inbounds i32, ptr %base, i64 %base.idx
+  call void @use(ptr %gep.base)
+  %p2 = getelementptr inbounds i32, ptr %gep.base, i64 %idx
+  %p3 = getelementptr inbounds i32, ptr %p2, i64 %idx2
+  %p4 = getelementptr inbounds i32, ptr %gep.base, i64 %idx3
+  %i1 = ptrtoint ptr %p4 to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @multiple_geps_inbounds(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @multiple_geps_inbounds(
+; CHECK-NEXT:    [[D:%.*]] = add nsw i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %p2 = getelementptr inbounds i8, ptr %base, i64 %idx
+  %p3 = getelementptr inbounds i8, ptr %p2, i64 %idx2
+  %i1 = ptrtoint ptr %base to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @multiple_geps_nusw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @multiple_geps_nusw(
+; CHECK-NEXT:    [[D:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %p2 = getelementptr nusw i8, ptr %base, i64 %idx
+  %p3 = getelementptr nusw i8, ptr %p2, i64 %idx2
+  %i1 = ptrtoint ptr %base to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @multiple_geps_nuw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @multiple_geps_nuw(
+; CHECK-NEXT:    [[D:%.*]] = add nuw i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %p2 = getelementptr nuw i8, ptr %base, i64 %idx
+  %p3 = getelementptr nuw i8, ptr %p2, i64 %idx2
+  %i1 = ptrtoint ptr %base to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+
+define i64 @multiple_geps_inbounds_nuw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @multiple_geps_inbounds_nuw(
+; CHECK-NEXT:    [[D:%.*]] = add nuw nsw i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %p2 = getelementptr inbounds nuw i8, ptr %base, i64 %idx
+  %p3 = getelementptr inbounds nuw i8, ptr %p2, i64 %idx2
+  %i1 = ptrtoint ptr %base to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define <2 x i64> @splat_geps(ptr %base, <2 x i64> %idx1, <2 x i64> %idx2) {
+; CHECK-LABEL: @splat_geps(
+; CHECK-NEXT:    [[D:%.*]] = sub nsw <2 x i64> [[IDX2:%.*]], [[IDX1:%.*]]
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %gep1 = getelementptr inbounds i8, ptr %base, <2 x i64> %idx1
+  %gep2 = getelementptr inbounds i8, ptr %base, <2 x i64> %idx2
+  %gep1.int = ptrtoint <2 x ptr> %gep1 to <2 x i64>
+  %gep2.int = ptrtoint <2 x ptr> %gep2 to <2 x i64>
+  %d = sub <2 x i64> %gep2.int, %gep1.int
+  ret <2 x i64> %d
+}
+
+define <2 x i64> @splat_geps_multiple(ptr %base, i64 %idx0, <2 x i64> %idx1, <2 x i64> %idx2) {
+; CHECK-LABEL: @splat_geps_multiple(
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[IDX0:%.*]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i64> [[DOTSPLAT]], [[IDX1:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = sub nsw <2 x i64> [[IDX2:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x i64> [[D]]
+;
+  %gep0 = getelementptr inbounds i8, ptr %base, i64 %idx0
+  %gep1 = getelementptr inbounds i8, ptr %gep0, <2 x i64> %idx1
+  %gep2 = getelementptr inbounds i8, ptr %base, <2 x i64> %idx2
+  %gep1.int = ptrtoint <2 x ptr> %gep1 to <2 x i64>
+  %gep2.int = ptrtoint <2 x ptr> %gep2 to <2 x i64>
+  %d = sub <2 x i64> %gep2.int, %gep1.int
+  ret <2 x i64> %d
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/trunc-lshr.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/trunc-lshr.ll
new file mode 100644
index 000000000000..4364b09cfa70
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/trunc-lshr.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+define i1 @test1(i32 %i, ptr %p) {
+; CHECK-LABEL: define i1 @test1(
+; CHECK-SAME: i32 [[I:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[DOTLOBIT:%.*]] = lshr i32 [[I]], 31
+; CHECK-NEXT:    store i32 [[DOTLOBIT]], ptr [[P]], align 1
+; CHECK-NEXT:    ret i1 false
+;
+  %lobit = lshr i32 %i, 31
+  %t = trunc nuw i32 %lobit to i1
+  %b = icmp slt i32 %i, 0
+  %not = xor i1 %t, true
+  %op = select i1 %not, i1 %b, i1 false
+  store i32 %lobit, ptr %p, align 1
+  ret i1 %op
+}
+
+define i1 @test2(i32 %i, ptr %p) {
+; CHECK-LABEL: define i1 @test2(
+; CHECK-SAME: i32 [[I:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[DOTLOBIT:%.*]] = ashr i32 [[I]], 31
+; CHECK-NEXT:    store i32 [[DOTLOBIT]], ptr [[P]], align 1
+; CHECK-NEXT:    ret i1 false
+;
+  %lobit = ashr i32 %i, 31
+  %t = trunc nuw i32 %lobit to i1
+  %b = icmp slt i32 %i, 0
+  %not = xor i1 %t, true
+  %op = select i1 %not, i1 %b, i1 false
+  store i32 %lobit, ptr %p, align 1
+  ret i1 %op
+}
+
+define i1 @test3(i32 %i, ptr %p, ptr %q) {
+; CHECK-LABEL: define i1 @test3(
+; CHECK-SAME: i32 [[I:%.*]], ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT:    [[DOTLOBIT:%.*]] = lshr i32 [[I]], 31
+; CHECK-NEXT:    store i32 [[DOTLOBIT]], ptr [[P]], align 1
+; CHECK-NEXT:    store i32 [[DOTLOBIT]], ptr [[Q]], align 1
+; CHECK-NEXT:    ret i1 false
+;
+  %lobit = lshr i32 %i, 31
+  %t = trunc nuw i32 %lobit to i1
+  %b = icmp slt i32 %i, 0
+  %not = xor i1 %t, true
+  %op = select i1 %not, i1 %b, i1 false
+  store i32 %lobit, ptr %p, align 1
+  store i32 %lobit, ptr %q, align 1
+  ret i1 %op
+}
+
+; Negative Test
+define i1 @test4(i32 %i, ptr %p) {
+; CHECK-LABEL: define i1 @test4(
+; CHECK-SAME: i32 [[I:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[DOTLOBIT:%.*]] = lshr i32 [[I]], 30
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[DOTLOBIT]] to i1
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[I]], 0
+; CHECK-NEXT:    [[NOT_:%.*]] = xor i1 [[T]], true
+; CHECK-NEXT:    [[COMMON_RET1_OP:%.*]] = select i1 [[NOT_]], i1 [[B]], i1 false
+; CHECK-NEXT:    store i32 [[DOTLOBIT]], ptr [[P]], align 1
+; CHECK-NEXT:    ret i1 [[COMMON_RET1_OP]]
+;
+  %lobit = lshr i32 %i, 30 ; should not fold as no. of bits shifted < BitWidth - 1
+  %t = trunc nuw i32 %lobit to i1
+  %b = icmp slt i32 %i, 0
+  %not = xor i1 %t, true
+  %op = select i1 %not, i1 %b, i1 false
+  store i32 %lobit, ptr %p, align 1
+  ret i1 %op
+}
+
+; Negative Test
+define i1 @test5(i32 %i, ptr %p) {
+; CHECK-LABEL: define i1 @test5(
+; CHECK-SAME: i32 [[I:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[DOTLOBIT:%.*]] = ashr i32 [[I]], 30
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[DOTLOBIT]] to i1
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[I]], 0
+; CHECK-NEXT:    [[NOT_:%.*]] = xor i1 [[T]], true
+; CHECK-NEXT:    [[COMMON_RET1_OP:%.*]] = select i1 [[NOT_]], i1 [[B]], i1 false
+; CHECK-NEXT:    store i32 [[DOTLOBIT]], ptr [[P]], align 1
+; CHECK-NEXT:    ret i1 [[COMMON_RET1_OP]]
+;
+  %lobit = ashr i32 %i, 30 ; should not fold as no. of bits shifted < BitWidth - 1
+  %t = trunc nuw i32 %lobit to i1
+  %b = icmp slt i32 %i, 0
+  %not = xor i1 %t, true
+  %op = select i1 %not, i1 %b, i1 false
+  store i32 %lobit, ptr %p, align 1
+  ret i1 %op
+}
+
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
index 89e75be0a556..ad86f2ed2fa5 100644
--- a/external/llvm-project/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
@@ -135,3 +135,28 @@ define float @select_min_uge_2_use_cmp(float %a, float %b) {
   %sel = select nsz i1 %cmp, float %b, float %a
   ret float %sel
 }
+
+; Make sure that we do not recognize the following pattern as a SPF
+; unless the nsz flag is set on the select instruction.
+
+define float @pr141017(float %x) {
+; CHECK-LABEL: @pr141017(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp nsz olt float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], float -0.000000e+00, float [[X]]
+; CHECK-NEXT:    ret float [[SEL]]
+;
+  %cmp = fcmp nsz olt float %x, 0.0
+  %sel = select i1 %cmp, float -0.0, float %x
+  ret float %sel
+}
+
+define float @pr141017_select_nsz(float %x) {
+; CHECK-LABEL: @pr141017_select_nsz(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp ole float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[DOTINV]], float -0.000000e+00, float [[X]]
+; CHECK-NEXT:    ret float [[SEL1]]
+;
+  %cmp = fcmp olt float %x, 0.0
+  %sel = select nsz i1 %cmp, float -0.0, float %x
+  ret float %sel
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstCombine/vp-reverse.ll b/external/llvm-project/llvm/test/Transforms/InstCombine/vp-reverse.ll
new file mode 100644
index 000000000000..79e6c47bdf1b
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstCombine/vp-reverse.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define <vscale x 4 x i32> @binop_reverse_elim(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_elim(
+; CHECK-NEXT:    [[A:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[B1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD1]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %b, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %add = add nsw <vscale x 4 x i32> %a.rev, %b.rev
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %add.rev
+}
+
+define <vscale x 4 x i32> @binop_reverse_elim2(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %m, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_elim2(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i1> [[M:%.*]], i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]], <vscale x 4 x i1> [[M]], i32 [[EVL]])
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <vscale x 4 x i32> [[A_REV]], [[B_REV]]
+; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD]], <vscale x 4 x i1> [[M]], i32 [[EVL]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> %m, i32 %evl)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %b, <vscale x 4 x i1> %m, i32 %evl)
+  %add = add nsw <vscale x 4 x i32> %a.rev, %b.rev
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> %m, i32 %evl)
+  ret <vscale x 4 x i32> %add.rev
+}
+
+define <vscale x 4 x i32> @binop_reverse_elim_diffmask(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %m1, <vscale x 4 x i1> %m2, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_elim_diffmask(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i1> [[M1:%.*]], i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]], <vscale x 4 x i1> [[M1]], i32 [[EVL]])
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <vscale x 4 x i32> [[A_REV]], [[B_REV]]
+; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD]], <vscale x 4 x i1> [[M2:%.*]], i32 10)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> %m1, i32 %evl)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %b, <vscale x 4 x i1> %m1, i32 %evl)
+  %add = add nsw <vscale x 4 x i32> %a.rev, %b.rev
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> %m2, i32 10)
+  ret <vscale x 4 x i32> %add.rev
+}
+
+define <vscale x 4 x i32> @binop_reverse_elim_diffevl(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_elim_diffevl(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <vscale x 4 x i32> [[A_REV]], [[B_REV]]
+; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD]], <vscale x 4 x i1> splat (i1 true), i32 10)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %b, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %add = add nsw <vscale x 4 x i32> %a.rev, %b.rev
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> splat (i1 true), i32 10)
+  ret <vscale x 4 x i32> %add.rev
+}
+
+define <vscale x 4 x i32> @binop_reverse_splat_elim(<vscale x 4 x i32> %a, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_splat_elim(
+; CHECK-NEXT:    [[A:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A]], splat (i32 22)
+; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD1]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %add = add nsw <vscale x 4 x i32> %a.rev, splat (i32 22)
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %add.rev
+}
+
+define <vscale x 4 x i32> @binop_reverse_splat_elim2(<vscale x 4 x i32> %a, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_splat_elim2(
+; CHECK-NEXT:    [[A:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A]], splat (i32 22)
+; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD1]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %add = add nsw <vscale x 4 x i32> splat (i32 22), %a.rev
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %add.rev
+}
+
+define <vscale x 4 x float> @unop_reverse_splat_elim(<vscale x 4 x float> %a, <vscale x 4 x float> %b, i32 %evl) {
+; CHECK-LABEL: @unop_reverse_splat_elim(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[A:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[OP:%.*]] = fneg <vscale x 4 x float> [[A_REV]]
+; CHECK-NEXT:    [[OP_REV:%.*]] = tail call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[OP]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[OP_REV]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %op = fneg <vscale x 4 x float> %a.rev
+  %op.rev = tail call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> %op, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x float> %op.rev
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstSimplify/ConstProp/atan-intrinsic.ll b/external/llvm-project/llvm/test/Transforms/InstSimplify/ConstProp/atan-intrinsic.ll
new file mode 100644
index 000000000000..c5c17d65524c
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstSimplify/ConstProp/atan-intrinsic.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instsimplify < %s | FileCheck %s
+; XFAIL: target={{.*}}-aix{{.*}}
+
+define double @test_atan_0() {
+; CHECK-LABEL: define double @test_atan_0() {
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %result = call double @llvm.atan.f64(double 0.0)
+  ret double %result
+}
+
+define double @test_atan_one() {
+; CHECK-LABEL: define double @test_atan_one() {
+; CHECK-NEXT:    ret double 0x3FE921FB54442D18
+;
+  %res = call double @llvm.atan.f64(double 1.0)
+  ret double %res
+}
+
+define <2 x double> @test_atan_v2() {
+; CHECK-LABEL: define <2 x double> @test_atan_v2() {
+; CHECK-NEXT:    ret <2 x double> zeroinitializer
+;
+  %result = call <2 x double> @llvm.atan.v2f64(<2 x double> zeroinitializer)
+  ret <2 x double> %result
+}
+
+define double @test_atan_neg0() {
+; CHECK-LABEL: define double @test_atan_neg0() {
+; CHECK-NEXT:    ret double -0.000000e+00
+;
+  %res = call double @llvm.atan.f64(double -0.0)
+  ret double %res
+}
+
+define double @test_atan_poison() {
+; CHECK-LABEL: define double @test_atan_poison() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.atan.f64(double poison)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.atan.f64(double poison)
+  ret double %res
+}
+
+define double @test_atan_undef() {
+; CHECK-LABEL: define double @test_atan_undef() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.atan.f64(double undef)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.atan.f64(double undef)
+  ret double %res
+}
+
+define double @test_atan_snan() {
+; CHECK-LABEL: define double @test_atan_snan() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.atan.f64(double 0x7FF0000000000001)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.atan.f64(double 0x7ff0000000000001)
+  ret double %res
+}
+
+define double @test_atan_qnan() {
+; CHECK-LABEL: define double @test_atan_qnan() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.atan.f64(double 0x7FF8000000000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.atan.f64(double 0x7ff8000000000000)
+  ret double %res
+}
+
+define double @test_atan_pos_inf() {
+; CHECK-LABEL: define double @test_atan_pos_inf() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.atan.f64(double 0x7FF0000000000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.atan.f64(double 0x7ff0000000000000)
+  ret double %res
+}
+
+define double @test_atan_neg_inf() {
+; CHECK-LABEL: define double @test_atan_neg_inf() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.atan.f64(double 0xFFF0000000000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.atan.f64(double 0xfff0000000000000)
+  ret double %res
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstSimplify/ConstProp/vector-calls.ll b/external/llvm-project/llvm/test/Transforms/InstSimplify/ConstProp/vector-calls.ll
new file mode 100644
index 000000000000..14543f339db5
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstSimplify/ConstProp/vector-calls.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instsimplify,verify -S | FileCheck %s
+
+define <3 x i32> @fold_vector_extract() {
+; CHECK-LABEL: define <3 x i32> @fold_vector_extract() {
+; CHECK-NEXT:    ret <3 x i32> <i32 3, i32 4, i32 5>
+;
+  %1 = call <3 x i32> @llvm.vector.extract.v3i32.v8i32(<8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i64 3)
+  ret <3 x i32> %1
+}
+
+@a = external global i16, align 1
+
+define <3 x i32> @fold_vector_extract_constexpr() {
+; CHECK-LABEL: define <3 x i32> @fold_vector_extract_constexpr() {
+; CHECK-NEXT:    ret <3 x i32> <i32 ptrtoint (ptr @a to i32), i32 1, i32 2>
+;
+  %1 = call <3 x i32> @llvm.vector.extract.v3i32.v8i32(<8 x i32> <i32 ptrtoint (ptr @a to i32), i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i64 0)
+  ret <3 x i32> %1
+}
+
+define <8 x i32> @fold_vector_extract_nop() {
+; CHECK-LABEL: define <8 x i32> @fold_vector_extract_nop() {
+; CHECK-NEXT:    ret <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+;
+  %1 = call <8 x i32> @llvm.vector.extract.v3i32.v8i32(<8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i64 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @fold_vector_insert() {
+; CHECK-LABEL: define <8 x i32> @fold_vector_insert() {
+; CHECK-NEXT:    ret <8 x i32> <i32 9, i32 10, i32 11, i32 12, i32 5, i32 6, i32 7, i32 8>
+;
+  %1 = call <8 x i32> @llvm.vector.insert.v8i32(<8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>, <4 x i32> <i32 9, i32 10, i32 11, i32 12>, i64 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @fold_vector_insert_nop() {
+; CHECK-LABEL: define <8 x i32> @fold_vector_insert_nop() {
+; CHECK-NEXT:    ret <8 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+;
+  %1 = call <8 x i32> @llvm.vector.insert.v8i32(<8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>, <8 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>, i64 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @fold_vector_interleave2() {
+; CHECK-LABEL: define <8 x i32> @fold_vector_interleave2() {
+; CHECK-NEXT:    ret <8 x i32> <i32 1, i32 5, i32 2, i32 6, i32 3, i32 7, i32 4, i32 8>
+;
+  %1 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>)
+  ret <8 x i32> %1
+}
+
+define {<4 x i32>, <4 x i32>} @fold_vector_deinterleave2() {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @fold_vector_deinterleave2() {
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } { <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8> }
+;
+  %1 = call {<4 x i32>, <4 x i32>} @llvm.vector.deinterleave2.v4i32.v8i32(<8 x i32> <i32 1, i32 5, i32 2, i32 6, i32 3, i32 7, i32 4, i32 8>)
+  ret {<4 x i32>, <4 x i32>} %1
+}
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>} @fold_scalable_vector_deinterleave2() {
+; CHECK-LABEL: define { <vscale x 4 x i32>, <vscale x 4 x i32> } @fold_scalable_vector_deinterleave2() {
+; CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32> } zeroinitializer
+;
+  %1 = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave2.v4i32.v8i32(<vscale x 8 x i32> zeroinitializer)
+  ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %1
+}
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>} @fold_scalable_vector_deinterleave2_splat() {
+; CHECK-LABEL: define { <vscale x 4 x i32>, <vscale x 4 x i32> } @fold_scalable_vector_deinterleave2_splat() {
+; CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32> } { <vscale x 4 x i32> splat (i32 1), <vscale x 4 x i32> splat (i32 1) }
+;
+  %1 = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave2.v4i32.v8i32(<vscale x 8 x i32> splat (i32 1))
+  ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %1
+}
+
+define {<vscale x 4 x float>, <vscale x 4 x float>} @fold_scalable_vector_deinterleave2_splatfp() {
+; CHECK-LABEL: define { <vscale x 4 x float>, <vscale x 4 x float> } @fold_scalable_vector_deinterleave2_splatfp() {
+; CHECK-NEXT:    ret { <vscale x 4 x float>, <vscale x 4 x float> } { <vscale x 4 x float> splat (float 1.000000e+00), <vscale x 4 x float> splat (float 1.000000e+00) }
+;
+  %1 = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.vector.deinterleave2.v4f32.v8f32(<vscale x 8 x float> splat (float 1.0))
+  ret {<vscale x 4 x float>, <vscale x 4 x float>} %1
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstSimplify/compare.ll b/external/llvm-project/llvm/test/Transforms/InstSimplify/compare.ll
index 4192a59dc713..97ac8f2ea47e 100644
--- a/external/llvm-project/llvm/test/Transforms/InstSimplify/compare.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstSimplify/compare.ll
@@ -3465,6 +3465,26 @@ define i1 @icmp_eq_false_by_trunc(i8 %x) {
   ret i1 %cmp
 }
 
+define <vscale x 8 x i1> @icmp_ne_i1_vec_constant_expr() {
+; CHECK-LABEL: @icmp_ne_i1_vec_constant_expr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0)
+;
+entry:
+  %cmp = icmp ne <vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), zeroinitializer
+  ret <vscale x 8 x i1> %cmp
+}
+
+define <vscale x 8 x i1> @icmp_eq_i1_vec_constant_expr_commuted() {
+; CHECK-LABEL: @icmp_eq_i1_vec_constant_expr_commuted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <vscale x 8 x i1> xor (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> splat (i1 true))
+;
+entry:
+  %cmp = icmp eq <vscale x 8 x i1> zeroinitializer, insertelement (<vscale x 8 x i1> poison, i1 true, i64 0)
+  ret <vscale x 8 x i1> %cmp
+}
+
 declare i64 @llvm.vscale.i64()
 
 ; TODO: Add coverage for global aliases, link once, etc..
diff --git a/external/llvm-project/llvm/test/Transforms/InstSimplify/fp-undef-poison.ll b/external/llvm-project/llvm/test/Transforms/InstSimplify/fp-undef-poison.ll
index cb2026df962c..ffab9c94ddf4 100644
--- a/external/llvm-project/llvm/test/Transforms/InstSimplify/fp-undef-poison.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstSimplify/fp-undef-poison.ll
@@ -293,3 +293,53 @@ define double @fmul_nnan_inf_op1(double %x) {
   %r = fmul nnan double %x, 0xfff0000000000000
   ret double %r
 }
+
+define float @sqrt_poison() {
+; CHECK-LABEL: @sqrt_poison(
+; CHECK-NEXT:    ret float poison
+;
+  %sqrt = call float @llvm.sqrt(float poison)
+  ret float %sqrt
+}
+
+define <2 x float> @sqrt_poison_fixed_vec() {
+; CHECK-LABEL: @sqrt_poison_fixed_vec(
+; CHECK-NEXT:    ret <2 x float> poison
+;
+  %sqrt = call <2 x float> @llvm.sqrt(<2 x float> poison)
+  ret <2 x float> %sqrt
+}
+
+define <2 x float> @sqrt_poison_elt_fixed_vec() {
+; CHECK-LABEL: @sqrt_poison_elt_fixed_vec(
+; CHECK-NEXT:    ret <2 x float> <float 1.000000e+00, float poison>
+;
+  %sqrt = call <2 x float> @llvm.sqrt(<2 x float> <float 1.0, float poison>)
+  ret <2 x float> %sqrt
+}
+
+define <vscale x 2 x float> @sqrt_poison_scalable_vec() {
+; CHECK-LABEL: @sqrt_poison_scalable_vec(
+; CHECK-NEXT:    ret <vscale x 2 x float> poison
+;
+  %sqrt = call <vscale x 2 x float> @llvm.sqrt(<vscale x 2 x float> poison)
+  ret <vscale x 2 x float> %sqrt
+}
+
+define float @sqrt_nnan_nan() {
+; CHECK-LABEL: @sqrt_nnan_nan(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nnan float @llvm.sqrt.f32(float 0x7FF8000000000000)
+; CHECK-NEXT:    ret float [[SQRT]]
+;
+  %sqrt = call nnan float @llvm.sqrt(float 0x7ff8000000000000)
+  ret float %sqrt
+}
+
+define float @sqrt_ninf_inf() {
+; CHECK-LABEL: @sqrt_ninf_inf(
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf float @llvm.sqrt.f32(float 0xFFF0000000000000)
+; CHECK-NEXT:    ret float [[SQRT]]
+;
+  %sqrt = call ninf float @llvm.sqrt(float 0xfff0000000000000)
+  ret float %sqrt
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstSimplify/shufflevector.ll b/external/llvm-project/llvm/test/Transforms/InstSimplify/shufflevector.ll
index 0442bd721974..feab024a2921 100644
--- a/external/llvm-project/llvm/test/Transforms/InstSimplify/shufflevector.ll
+++ b/external/llvm-project/llvm/test/Transforms/InstSimplify/shufflevector.ll
@@ -337,3 +337,19 @@ define <4 x i32> @not_fold_identity2(<4 x i32> %x) {
   %revshuf = shufflevector <4 x i32> %shuf, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i32> %revshuf
 }
+
+define <vscale x 2 x i32> @scalable_splat_undef() {
+; CHECK-LABEL: @scalable_splat_undef(
+; CHECK-NEXT:    ret <vscale x 2 x i32> undef
+;
+  %shuf = shufflevector <vscale x 2 x i32> undef, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i32> %shuf
+}
+
+define <vscale x 2 x i32> @scalable_splat_poison() {
+; CHECK-LABEL: @scalable_splat_poison(
+; CHECK-NEXT:    ret <vscale x 2 x i32> poison
+;
+  %shuf = shufflevector <vscale x 2 x i32> poison, <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+  ret <vscale x 2 x i32> %shuf
+}
diff --git a/external/llvm-project/llvm/test/Transforms/InstSimplify/vp-reverse.ll b/external/llvm-project/llvm/test/Transforms/InstSimplify/vp-reverse.ll
new file mode 100644
index 000000000000..3c3bb871dc61
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/InstSimplify/vp-reverse.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instsimplify -S | FileCheck %s
+
+define <vscale x 4 x i32> @rev_of_rev(<vscale x 4 x i32> %a, i32 %evl) {
+; CHECK-LABEL: @rev_of_rev(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A_REV]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[RES]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %res = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a.rev, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @rev_of_rev_diffevl(<vscale x 4 x i32> %a, i32 %evl) {
+; CHECK-LABEL: @rev_of_rev_diffevl(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A_REV]], <vscale x 4 x i1> splat (i1 true), i32 10)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[RES]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %res = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a.rev, <vscale x 4 x i1> splat (i1 true), i32 10)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @rev_of_poison(i32 %evl) {
+; CHECK-LABEL: @rev_of_poison(
+; CHECK-NEXT:    [[REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[REV]]
+;
+  %rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %rev
+}
+
+define <vscale x 4 x i32> @rev_of_undef(i32 %evl) {
+; CHECK-LABEL: @rev_of_undef(
+; CHECK-NEXT:    [[REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[REV]]
+;
+  %rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> undef, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %rev
+}
+
+define <vscale x 4 x i32> @rev_of_zero(i32 %evl) {
+; CHECK-LABEL: @rev_of_zero(
+; CHECK-NEXT:    [[REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[REV]]
+;
+  %rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %rev
+}
+
+define <vscale x 4 x i32> @rev_of_splat(i32 %a, i32 %evl) {
+; CHECK-LABEL: @rev_of_splat(
+; CHECK-NEXT:    [[A_INS:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A_VEC:%.*]] = shufflevector <vscale x 4 x i32> [[A_INS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A_VEC]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[REV]]
+;
+  %a.ins = insertelement <vscale x 4 x i32> poison, i32 %a, i32 0
+  %a.vec = shufflevector <vscale x 4 x i32> %a.ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+  %rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a.vec, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %rev
+}
+
+define <vscale x 4 x i32> @rev_of_splat2(i32 %a, <vscale x 4 x i1> %m, i32 %evl) {
+; CHECK-LABEL: @rev_of_splat2(
+; CHECK-NEXT:    [[A_INS:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A_VEC:%.*]] = shufflevector <vscale x 4 x i32> [[A_INS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A_VEC]], <vscale x 4 x i1> [[M:%.*]], i32 [[EVL:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[REV]]
+;
+  %a.ins = insertelement <vscale x 4 x i32> poison, i32 %a, i32 0
+  %a.vec = shufflevector <vscale x 4 x i32> %a.ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+  %rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a.vec, <vscale x 4 x i1> %m, i32 %evl)
+  ret <vscale x 4 x i32> %rev
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LICM/call-hoisting.ll b/external/llvm-project/llvm/test/Transforms/LICM/call-hoisting.ll
index e6d2e42e34e8..907f13438623 100644
--- a/external/llvm-project/llvm/test/Transforms/LICM/call-hoisting.ll
+++ b/external/llvm-project/llvm/test/Transforms/LICM/call-hoisting.ll
@@ -1,13 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=licm %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<target-ir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' < %s -S | FileCheck %s
 
 declare i32 @load(ptr %p) argmemonly readonly nounwind
 
 define void @test_load(ptr noalias %loc, ptr noalias %sink) {
-; CHECK-LABEL: @test_load
-; CHECK-LABEL: entry:
-; CHECK: call i32 @load
-; CHECK-LABEL: loop:
+; CHECK-LABEL: define void @test_load(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[SINK:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @load(ptr [[LOC]])
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store volatile i32 [[RET]], ptr [[SINK]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -30,15 +41,26 @@ declare i32 @spec(ptr %p, ptr %q) readonly argmemonly nounwind speculatable
 ; However, we need not strip the nonnull attribute since it just propagates
 ; poison if the parameter was indeed null.
 define void @test_strip_attribute(ptr noalias %loc, ptr noalias %sink, ptr %q) {
-; CHECK-LABEL: @test_strip_attribute(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RET:%.*]] = call i32 @load(ptr [[LOC:%.*]])
-; CHECK-NEXT:    [[NULLCHK:%.*]] = icmp eq ptr [[Q:%.*]], null
+; CHECK-LABEL: define void @test_strip_attribute(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[SINK:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @load(ptr [[LOC]])
+; CHECK-NEXT:    [[NULLCHK:%.*]] = icmp eq ptr [[Q]], null
 ; CHECK-NEXT:    [[RET2:%.*]] = call i32 @spec(ptr nonnull [[Q]], ptr [[LOC]])
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[ISNULL:%.*]] ]
-; CHECK-NEXT:    br i1 [[NULLCHK]], label [[ISNULL]], label [[NONNULLBB:%.*]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[ISNULL:.*]] ]
+; CHECK-NEXT:    br i1 [[NULLCHK]], label %[[ISNULL]], label %[[NONNULLBB:.*]]
+; CHECK:       [[NONNULLBB]]:
+; CHECK-NEXT:    br label %[[ISNULL]]
+; CHECK:       [[ISNULL]]:
+; CHECK-NEXT:    store volatile i32 [[RET]], ptr [[SINK]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -65,10 +87,19 @@ exit:
 declare void @store(i32 %val, ptr %p) argmemonly writeonly nounwind
 
 define void @test(ptr %loc) {
-; CHECK-LABEL: @test
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -84,10 +115,23 @@ exit:
 }
 
 define void @test_multiexit(ptr %loc, i1 %earlycnd) {
-; CHECK-LABEL: @test_multiexit
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: backedge:
+; CHECK-LABEL: define void @test_multiexit(
+; CHECK-SAME: ptr [[LOC:%.*]], i1 [[EARLYCND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[EARLYCND]], label %[[EXIT1:.*]], label %[[BACKEDGE]]
+; CHECK:       [[BACKEDGE]]:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT2:.*]]
+; CHECK:       [[EXIT1]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[EXIT2]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -108,10 +152,19 @@ exit2:
 }
 
 define void @neg_lv_value(ptr %loc) {
-; CHECK-LABEL: @neg_lv_value
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_lv_value(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @store(i32 [[IV]], ptr [[LOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -127,10 +180,20 @@ exit:
 }
 
 define void @neg_lv_addr(ptr %loc) {
-; CHECK-LABEL: @neg_lv_addr
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_lv_addr(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[P:%.*]] = getelementptr i32, ptr [[LOC]], i32 [[IV]]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[P]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -147,10 +210,20 @@ exit:
 }
 
 define void @neg_mod(ptr %loc) {
-; CHECK-LABEL: @neg_mod
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_mod(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    store i32 [[IV]], ptr [[LOC]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -167,10 +240,25 @@ exit:
 }
 
 define void @neg_ref(ptr %loc) {
-; CHECK-LABEL: @neg_ref
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: exit1:
+; CHECK-LABEL: define void @neg_ref(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[LOC]], align 4
+; CHECK-NEXT:    [[EARLYCND:%.*]] = icmp eq i32 [[V]], 198
+; CHECK-NEXT:    br i1 [[EARLYCND]], label %[[EXIT1:.*]], label %[[BACKEDGE]]
+; CHECK:       [[BACKEDGE]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT2:.*]]
+; CHECK:       [[EXIT1]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[EXIT2]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -195,10 +283,20 @@ exit2:
 declare void @modref()
 
 define void @neg_modref(ptr %loc) {
-; CHECK-LABEL: @neg_modref
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_modref(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    call void @modref()
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -215,10 +313,20 @@ exit:
 }
 
 define void @neg_fence(ptr %loc) {
-; CHECK-LABEL: @neg_fence
-; CHECK-LABEL: loop:
-; CHECK: call void @store
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_fence(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -239,10 +347,19 @@ declare void @not_argmemonly(i32 %v, ptr %p) writeonly nounwind
 declare void @not_writeonly(i32 %v, ptr %p) argmemonly nounwind
 
 define void @neg_not_nounwind(ptr %loc) {
-; CHECK-LABEL: @neg_not_nounwind
-; CHECK-LABEL: loop:
-; CHECK: call void @not_nounwind
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_not_nounwind(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @not_nounwind(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -258,10 +375,19 @@ exit:
 }
 
 define void @neg_not_argmemonly(ptr %loc) {
-; CHECK-LABEL: @neg_not_argmemonly
-; CHECK-LABEL: loop:
-; CHECK: call void @not_argmemonly
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_not_argmemonly(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @not_argmemonly(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -277,10 +403,19 @@ exit:
 }
 
 define void @neg_not_writeonly(ptr %loc) {
-; CHECK-LABEL: @neg_not_writeonly
-; CHECK-LABEL: loop:
-; CHECK: call void @not_writeonly
-; CHECK-LABEL: exit:
+; CHECK-LABEL: define void @neg_not_writeonly(
+; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @not_writeonly(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
diff --git a/external/llvm-project/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll b/external/llvm-project/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll
new file mode 100644
index 000000000000..df32e60d5065
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-idiom -S %s | FileCheck %s
+
+define void @fold_add_zext_to_sext(ptr %dst, i1 %start) {
+; CHECK-LABEL: define void @fold_add_zext_to_sext(
+; CHECK-SAME: ptr [[DST:%.*]], i1 [[START:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[START_EXT:%.*]] = zext i1 [[START]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i1 [[START]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 25, [[START_EXT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext nneg i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[SCEVGEP]], i8 0, i64 [[TMP4]], i1 false)
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[START_EXT]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_EXT:%.*]] = zext i32 [[IV]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_EXT]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ult i32 [[IV]], 24
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %start.ext = zext i1 %start to i32
+  br label %loop
+
+loop:
+  %iv = phi i32 [ %start.ext, %entry ], [ %iv.next, %loop ]
+  %iv.ext = zext i32 %iv to i64
+  %gep = getelementptr i32, ptr %dst, i64 %iv.ext
+  store i32 0, ptr %gep, align 4
+  %iv.next = add i32 %iv, 1
+  %ec = icmp ult i32 %iv, 24
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopInterchange/confused-dependence.ll b/external/llvm-project/llvm/test/Transforms/LoopInterchange/confused-dependence.ll
new file mode 100644
index 000000000000..49b7b0e4797b
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopInterchange/confused-dependence.ll
@@ -0,0 +1,41 @@
+; REQUIRES: asserts
+; RUN: opt < %s -passes=loop-interchange -verify-dom-info -verify-loop-info \
+; RUN:     -disable-output -debug 2>&1 | FileCheck %s
+
+;; In the following case, p0 and p1 may alias, so the direction vector must be [* *].
+;;
+;; void may_alias(float *p0, float *p1) {
+;;   for (int i = 0; i < 4; i++)
+;;     for (int j = 0; j < 4; j++)
+;;       p0[4 * i + j] = p1[4 * j + i];
+;; }
+
+; CHECK:      Dependency matrix before interchange:
+; CHECK-NEXT: * *
+; CHECK-NEXT: Processing InnerLoopId = 1 and OuterLoopId = 0
+define void @may_alias(ptr %p0, ptr %p1) {
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j
+
+for.j:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j ]
+  %idx.0 = getelementptr inbounds [4 x [4 x float]], ptr %p0, i32 0, i32 %j, i32 %i
+  %idx.1 = getelementptr inbounds [4 x [4 x float]], ptr %p1, i32 0, i32 %i, i32 %j
+  %0 = load float, ptr %idx.0, align 4
+  store float %0, ptr %idx.1, align 4
+  %j.inc = add nuw nsw i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 4
+  br i1 %cmp.j, label %for.j, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add nuw nsw i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 4
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll b/external/llvm-project/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll
index 520e1ee3506d..732b9b276630 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll
@@ -4,7 +4,7 @@
 
 ; Triply nested loop, should be able to do interchange three times
 ; to get the ideal access pattern.
-; void f(int e[10][10][10], int f[10][10][10]) {
+; void f(int e[restrict 10][10][10], int f[restrict 10][10][10]) {
 ;   for (int a = 0; a < 10; a++) {
 ;     for (int b = 0; b < 10; b++) {
 ;       for (int c = 0; c < 10; c++) {
@@ -35,7 +35,7 @@
 ; REMARKS-NEXT: Name:            Interchanged
 ; REMARKS-NEXT: Function:        pr43326-triply-nested
 
-define void @pr43326-triply-nested(ptr %e, ptr %f) {
+define void @pr43326-triply-nested(ptr noalias %e, ptr noalias %f) {
 entry:
   br label %for.outermost.header
 
diff --git a/external/llvm-project/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll b/external/llvm-project/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll
index e5c18830784a..68089b43121c 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -passes=loop-interchange -S -debug 2>&1 | FileCheck %s
 
 ; CHECK:       Dependency matrix before interchange:
-; CHECK-NEXT:  I I
+; CHECK-NEXT:  * *
 ; CHECK-NEXT:  = *
 ; CHECK-NEXT:  < *
 ; CHECK-NEXT:  Processing InnerLoopId
diff --git a/external/llvm-project/llvm/test/Transforms/LoopStrengthReduce/AArch64/postidx-load.ll b/external/llvm-project/llvm/test/Transforms/LoopStrengthReduce/AArch64/postidx-load.ll
new file mode 100644
index 000000000000..5976658ccdf8
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopStrengthReduce/AArch64/postidx-load.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-none-elf | FileCheck %s
+
+; Check that the load in the loop has postindex addressing, regardless of the
+; type or whether the input uses postindex or offset addressing.
+
+define i32 @i32_initially_postidx(ptr %p, i64 %n) {
+; CHECK-LABEL: i32_initially_postidx:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp x1, #1
+; CHECK-NEXT:    b.lt .LBB0_5
+; CHECK-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:  .LBB0_2: // %for.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr w9, [x0], #4
+; CHECK-NEXT:    add w8, w8, w9
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    b.lo .LBB0_5
+; CHECK-NEXT:  // %bb.3: // %for.inc
+; CHECK-NEXT:    // in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    subs x1, x1, #1
+; CHECK-NEXT:    b.ne .LBB0_2
+; CHECK-NEXT:  // %bb.4: // %cleanup
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_5:
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %cmp1 = icmp sgt i64 %n, 0
+  br i1 %cmp1, label %for.body, label %cleanup
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.inc ], [ 0, %entry ]
+  %accum = phi i32 [ %add, %for.inc ], [ 0, %entry ]
+  %ptr = phi ptr [ %ptr.next, %for.inc ], [ %p, %entry ]
+  %val = load i32, ptr %ptr, align 4
+  %ptr.next = getelementptr inbounds nuw i8, ptr %ptr, i64 4
+  %add = add i32 %accum, %val
+  %cmp2 = icmp ult i32 %add, 0
+  br i1 %cmp2, label %cleanup, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %cleanup, label %for.body
+
+cleanup:
+  %ret = phi i32 [ 0, %entry ], [ 0, %for.body ], [ %add, %for.inc ]
+  ret i32 %ret
+}
+
+define i32 @i32_initially_offset(ptr %p, i64 %n) {
+; CHECK-LABEL: i32_initially_offset:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp x1, #1
+; CHECK-NEXT:    b.lt .LBB1_5
+; CHECK-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:  .LBB1_2: // %for.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr w9, [x0], #4
+; CHECK-NEXT:    add w8, w8, w9
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    b.lo .LBB1_5
+; CHECK-NEXT:  // %bb.3: // %for.cond
+; CHECK-NEXT:    // in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    subs x1, x1, #1
+; CHECK-NEXT:    b.ne .LBB1_2
+; CHECK-NEXT:  // %bb.4: // %cleanup
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_5:
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %cmp1 = icmp sgt i64 %n, 0
+  br i1 %cmp1, label %for.body, label %cleanup
+
+for.cond:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %cleanup, label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.cond ], [ 0, %entry ]
+  %accum = phi i32 [ %add, %for.cond ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i32, ptr %p, i64 %iv
+  %val = load i32, ptr %arrayidx, align 4
+  %add = add i32 %accum, %val
+  %cmp2 = icmp ult i32 %add, 0
+  br i1 %cmp2, label %cleanup, label %for.cond
+
+cleanup:
+  %ret = phi i32 [ 0, %entry ], [ 0, %for.body ], [ %add, %for.cond ]
+  ret i32 %ret
+}
+
+define float @float_initially_postidx(ptr %p, i64 %n) {
+; CHECK-LABEL: float_initially_postidx:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi d0, #0000000000000000
+; CHECK-NEXT:    cmp x1, #1
+; CHECK-NEXT:    b.lt .LBB2_3
+; CHECK-NEXT:  .LBB2_1: // %for.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr s1, [x0], #4
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    fcmp s0, #0.0
+; CHECK-NEXT:    b.mi .LBB2_4
+; CHECK-NEXT:  // %bb.2: // %for.inc
+; CHECK-NEXT:    // in Loop: Header=BB2_1 Depth=1
+; CHECK-NEXT:    subs x1, x1, #1
+; CHECK-NEXT:    b.ne .LBB2_1
+; CHECK-NEXT:  .LBB2_3: // %cleanup
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_4:
+; CHECK-NEXT:    movi d0, #0000000000000000
+; CHECK-NEXT:    ret
+entry:
+  %cmp1 = icmp sgt i64 %n, 0
+  br i1 %cmp1, label %for.body, label %cleanup
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.inc ], [ 0, %entry ]
+  %accum = phi float [ %add, %for.inc ], [ 0.000000e+00, %entry ]
+  %ptr = phi ptr [ %ptr.next, %for.inc ], [ %p, %entry ]
+  %val = load float, ptr %ptr, align 4
+  %ptr.next = getelementptr inbounds nuw i8, ptr %ptr, i64 4
+  %add = fadd float %accum, %val
+  %cmp2 = fcmp olt float %add, 0.000000e+00
+  br i1 %cmp2, label %cleanup, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %cleanup, label %for.body
+
+cleanup:
+  %ret = phi float [ 0.000000e+00, %entry ], [ 0.000000e+00, %for.body ], [ %add, %for.inc ]
+  ret float %ret
+}
+
+define float @float_initially_offset(ptr %p, i64 %n) {
+; CHECK-LABEL: float_initially_offset:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi d0, #0000000000000000
+; CHECK-NEXT:    cmp x1, #1
+; CHECK-NEXT:    b.lt .LBB3_3
+; CHECK-NEXT:  .LBB3_1: // %for.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr s1, [x0], #4
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    fcmp s0, #0.0
+; CHECK-NEXT:    b.mi .LBB3_4
+; CHECK-NEXT:  // %bb.2: // %for.cond
+; CHECK-NEXT:    // in Loop: Header=BB3_1 Depth=1
+; CHECK-NEXT:    subs x1, x1, #1
+; CHECK-NEXT:    b.ne .LBB3_1
+; CHECK-NEXT:  .LBB3_3: // %cleanup
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB3_4:
+; CHECK-NEXT:    movi d0, #0000000000000000
+; CHECK-NEXT:    ret
+entry:
+  %cmp1 = icmp sgt i64 %n, 0
+  br i1 %cmp1, label %for.body, label %cleanup
+
+for.cond:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %cleanup, label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.cond ], [ 0, %entry ]
+  %accum = phi float [ %add, %for.cond ], [ 0.000000e+00, %entry ]
+  %arrayidx = getelementptr inbounds nuw float, ptr %p, i64 %iv
+  %val = load float, ptr %arrayidx, align 4
+  %add = fadd float %accum, %val
+  %cmp2 = fcmp olt float %add, 0.000000e+00
+  br i1 %cmp2, label %cleanup, label %for.cond
+
+cleanup:
+  %ret = phi float [ 0.000000e+00, %entry ], [ 0.000000e+00, %for.body ], [ %add, %for.cond ]
+  ret float %ret
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll b/external/llvm-project/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
index 93203ef6e17f..c12d8135e5eb 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
@@ -25,20 +25,18 @@ define i32 @simple(ptr %a, ptr %b, i32 %x) nounwind {
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    addl (%rdi), %eax
 ; X64-NEXT:    addl (%rdi,%rcx), %eax
-; X64-NEXT:    leaq (%rdi,%rcx), %r8
-; X64-NEXT:    addl (%rcx,%r8), %eax
-; X64-NEXT:    addq %rcx, %r8
-; X64-NEXT:    addl (%rcx,%r8), %eax
-; X64-NEXT:    addq %rdx, %r8
-; X64-NEXT:    movq %r8, %rdi
-; X64-NEXT:    cmpq %rsi, %r8
+; X64-NEXT:    addq %rcx, %rdi
+; X64-NEXT:    addl (%rcx,%rdi), %eax
+; X64-NEXT:    addq %rcx, %rdi
+; X64-NEXT:    addl (%rcx,%rdi), %eax
+; X64-NEXT:    addq %rdx, %rdi
+; X64-NEXT:    cmpq %rsi, %rdi
 ; X64-NEXT:    jne .LBB0_1
 ; X64-NEXT:  # %bb.2: # %exit
 ; X64-NEXT:    retq
 ;
 ; X32-LABEL: simple:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
@@ -52,18 +50,16 @@ define i32 @simple(ptr %a, ptr %b, i32 %x) nounwind {
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X32-NEXT:    addl (%edi), %eax
 ; X32-NEXT:    addl (%edi,%edx), %eax
-; X32-NEXT:    leal (%edi,%edx), %ebx
-; X32-NEXT:    addl (%edx,%ebx), %eax
-; X32-NEXT:    addl %edx, %ebx
-; X32-NEXT:    addl (%edx,%ebx), %eax
-; X32-NEXT:    addl %esi, %ebx
-; X32-NEXT:    movl %ebx, %edi
-; X32-NEXT:    cmpl %ecx, %ebx
+; X32-NEXT:    addl %edx, %edi
+; X32-NEXT:    addl (%edx,%edi), %eax
+; X32-NEXT:    addl %edx, %edi
+; X32-NEXT:    addl (%edx,%edi), %eax
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    cmpl %ecx, %edi
 ; X32-NEXT:    jne .LBB0_1
 ; X32-NEXT:  # %bb.2: # %exit
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
 ; X32-NEXT:    retl
 entry:
   br label %loop
diff --git a/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-minmax.ll b/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-minmax.ll
new file mode 100644
index 000000000000..cd098e123b5f
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-minmax.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-unroll -unroll-full-max-count=0 -S %s | FileCheck %s
+
+declare void @foo(i32)
+
+define i32 @umin_unit_step() {
+; CHECK-LABEL: define i32 @umin_unit_step() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 1024, [[IV]]
+; CHECK-NEXT:    [[MINMAX:%.*]] = call i32 @llvm.umin.i32(i32 [[SUB]], i32 1)
+; CHECK-NEXT:    call void @foo(i32 [[MINMAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MINMAX_LCSSA:%.*]] = phi i32 [ [[MINMAX]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[MINMAX_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %sub = sub i32 1024, %iv
+  %minmax = call i32 @llvm.umin(i32 %sub, i32 1)
+  call void @foo(i32 %minmax)
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp ne i32 %iv.next, 1024
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  %minmax.lcssa = phi i32 [ %minmax, %loop ]
+  ret i32 %minmax.lcssa
+}
+
+define i32 @smin_unit_step() {
+; CHECK-LABEL: define i32 @smin_unit_step() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 1024, [[IV]]
+; CHECK-NEXT:    [[MINMAX:%.*]] = call i32 @llvm.smin.i32(i32 [[SUB]], i32 1)
+; CHECK-NEXT:    call void @foo(i32 [[MINMAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EC_PEEL:%.*]] = icmp ne i32 [[IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EC_PEEL]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MINMAX_LCSSA:%.*]] = phi i32 [ [[MINMAX]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[MINMAX_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %sub = sub i32 1024, %iv
+  %minmax = call i32 @llvm.smin(i32 %sub, i32 1)
+  call void @foo(i32 %minmax)
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp ne i32 %iv.next, 1024
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  %minmax.lcssa = phi i32 [ %minmax, %loop ]
+  ret i32 %minmax.lcssa
+}
+
+define i32 @smax_unit_step() {
+; CHECK-LABEL: define i32 @smax_unit_step() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 1024, [[IV]]
+; CHECK-NEXT:    [[MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[SUB]], i32 1)
+; CHECK-NEXT:    call void @foo(i32 [[MINMAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EC_PEEL:%.*]] = icmp ne i32 [[IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EC_PEEL]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MINMAX_LCSSA:%.*]] = phi i32 [ [[MINMAX]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[MINMAX_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %sub = sub i32 1024, %iv
+  %minmax = call i32 @llvm.smax(i32 %sub, i32 1)
+  call void @foo(i32 %minmax)
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp ne i32 %iv.next, 1024
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  %minmax.lcssa = phi i32 [ %minmax, %loop ]
+  ret i32 %minmax.lcssa
+}
+
+define i32 @umax_unit_step() {
+; CHECK-LABEL: define i32 @umax_unit_step() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 1024, [[IV]]
+; CHECK-NEXT:    [[MINMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[SUB]], i32 1)
+; CHECK-NEXT:    call void @foo(i32 [[MINMAX]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[MINMAX_LCSSA:%.*]] = phi i32 [ [[MINMAX]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[MINMAX_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %sub = sub i32 1024, %iv
+  %minmax = call i32 @llvm.umax(i32 %sub, i32 1)
+  call void @foo(i32 %minmax)
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp ne i32 %iv.next, 1024
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  %minmax.lcssa = phi i32 [ %minmax, %loop ]
+  ret i32 %minmax.lcssa
+}
+
diff --git a/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll b/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll
new file mode 100644
index 000000000000..824e23fcf3e6
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll
@@ -0,0 +1,211 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-unroll -unroll-full-max-count=0 -S %s | FileCheck %s
+
+declare void @foo(i32)
+
+define void @peel_with_guard_known_nonnegative_1(i32 %n) {
+; CHECK-LABEL: define void @peel_with_guard_known_nonnegative_1(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[PRECOND:%.*]] = icmp slt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[PRECOND]], label %[[EXIT:.*]], label %[[PH:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[N_1:%.*]] = add i32 [[N]], 1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N_1]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[PH_SPLIT:.*]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN:.*]]
+; CHECK:       [[PH_SPLIT]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[PH_SPLIT]] ], [ [[IV_NEXT1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], 1
+; CHECK-NEXT:    [[EC1:%.*]] = icmp eq i64 [[IV_NEXT1]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[EC1]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT:    [[DOTPH:%.*]] = phi i64 [ [[IV_NEXT1]], %[[LOOP]] ]
+; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT_PEEL_BEGIN]]
+; CHECK:       [[EXIT_LOOPEXIT_PEEL_BEGIN]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[PH]] ], [ [[DOTPH]], %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT:    br label %[[LOOP_PEEL:.*]]
+; CHECK:       [[LOOP_PEEL]]:
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[IV]], [[N_EXT]]
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[C]], i32 10, i32 20
+; CHECK-NEXT:    [[IV_NEXT:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT_LOOPEXIT_PEEL_NEXT:.*]], label %[[EXIT_LOOPEXIT_PEEL_NEXT]]
+; CHECK:       [[EXIT_LOOPEXIT_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK:       [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %precond = icmp slt i32 %n, 0
+  br i1 %precond, label %exit, label %ph
+
+ph:
+  %n.ext = zext i32 %n to i64
+  %n.1 = add i32 %n, 1
+  %wide.trip.count = zext i32 %n.1 to i64
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %c = icmp eq i64 %iv, %n.ext
+  %spec.select = select i1 %c, i32 10, i32 20
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %wide.trip.count
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define i32 @peel_with_guard_known_nonnegative_2(ptr %x, i32 %w) {
+; CHECK-LABEL: define i32 @peel_with_guard_known_nonnegative_2(
+; CHECK-SAME: ptr [[X:%.*]], i32 [[W:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[PRECOND:%.*]] = icmp sgt i32 [[W]], 0
+; CHECK-NEXT:    br i1 [[PRECOND]], label %[[PH:.*]], label %[[EXIT:.*]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[W]], -1
+; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[SUB]] to i64
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[W]] to i64
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ 0, %[[PH]] ], [ [[ADD:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i64 [[IV]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    tail call void @foo()
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP1]], [[RED]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RED_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RED_LCSSA]]
+;
+entry:
+  %precond = icmp sgt i32 %w, 0
+  br i1 %precond, label %ph, label %exit
+
+ph:
+  %sub = add nsw i32 %w, -1
+  %0 = zext nneg i32 %sub to i64
+  %wide.trip.count = zext nneg i32 %w to i64
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop.latch ]
+  %red = phi i32 [ 0, %ph ], [ %add, %loop.latch ]
+  %cmp1 = icmp eq i64 %iv, %0
+  br i1 %cmp1, label %if.then, label %loop.latch
+
+if.then:
+  tail call void @foo()
+  br label %loop.latch
+
+loop.latch:
+  %arrayidx = getelementptr inbounds nuw i32, ptr %x, i64 %iv
+  %1 = load i32, ptr %arrayidx, align 4
+  %add = add nsw i32 %1, %red
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %wide.trip.count
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  %red.lcssa = phi i32 [ 0, %entry ], [ %add, %loop.latch ]
+  ret i32 %red.lcssa
+}
+
+define void @peel_with_guard2(i32 %n) {
+; CHECK-LABEL: define void @peel_with_guard2(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[PRECOND:%.*]] = icmp eq i32 [[SUB]], 0
+; CHECK-NEXT:    br i1 [[PRECOND]], label %[[EXIT:.*]], label %[[LOOP_HEADER_PREHEADER:.*]]
+; CHECK:       [[LOOP_HEADER_PREHEADER]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[LOOP_HEADER_PREHEADER_SPLIT:.*]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN:.*]]
+; CHECK:       [[LOOP_HEADER_PREHEADER_SPLIT]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 1, %[[LOOP_HEADER_PREHEADER_SPLIT]] ]
+; CHECK-NEXT:    br i1 false, label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK:       [[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT:    [[DOTPH:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT_PEEL_BEGIN]]
+; CHECK:       [[EXIT_LOOPEXIT_PEEL_BEGIN]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ 1, %[[LOOP_HEADER_PREHEADER]] ], [ [[DOTPH]], %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER_PEEL:.*]]
+; CHECK:       [[LOOP_HEADER_PEEL]]:
+; CHECK-NEXT:    [[C_PEEL:%.*]] = icmp eq i32 [[TMP3]], [[SUB]]
+; CHECK-NEXT:    br i1 [[C_PEEL]], label %[[THEN_PEEL:.*]], label %[[LOOP_LATCH_PEEL:.*]]
+; CHECK:       [[THEN_PEEL]]:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label %[[LOOP_LATCH_PEEL]]
+; CHECK:       [[LOOP_LATCH_PEEL]]:
+; CHECK-NEXT:    [[IV_NEXT_PEEL:%.*]] = add nuw i32 [[TMP3]], 1
+; CHECK-NEXT:    [[EC_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_PEEL]], [[N]]
+; CHECK-NEXT:    br i1 [[EC_PEEL]], label %[[EXIT_LOOPEXIT_PEEL_NEXT:.*]], label %[[EXIT_LOOPEXIT_PEEL_NEXT]]
+; CHECK:       [[EXIT_LOOPEXIT_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER_PEEL_NEXT:.*]]
+; CHECK:       [[LOOP_HEADER_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %sub = add i32 %n, -1
+  %precond = icmp eq i32 %sub, 0
+  br i1 %precond, label %exit, label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %loop.latch ]
+  %c = icmp eq i32 %iv, %sub
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  call void @foo()
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, %n
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+;.
diff --git a/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-variable-trip-count.ll b/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-variable-trip-count.ll
index a8712e04d43c..5704be33f0f0 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-variable-trip-count.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-variable-trip-count.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -p loop-unroll -unroll-full-max-count=0 -S %s | FileCheck %s
 
-
 declare void @foo(i32)
 
 define i32 @peel_last_with_trip_count_check_lcssa_phi(i32 %n) {
@@ -172,6 +171,155 @@ exit:
 }
 
 
+define i32 @peel_last_with_trip_count_check_lcssa_phi_cmp_not_invar2(i32 %n) {
+; CHECK-LABEL: define i32 @peel_last_with_trip_count_check_lcssa_phi_cmp_not_invar2(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[N]], -2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK:       [[ENTRY_SPLIT]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[IV_NEXT1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[IV]], [[SUB]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i32 1, i32 2
+; CHECK-NEXT:    call void @foo(i32 [[SEL]])
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT:    [[EC1:%.*]] = icmp ne i32 [[IV_NEXT1]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[EC1]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK:       [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT:    [[DOTPH:%.*]] = phi i32 [ [[IV_NEXT1]], %[[LOOP]] ]
+; CHECK-NEXT:    br label %[[EXIT_PEEL_BEGIN]]
+; CHECK:       [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT:    br label %[[LOOP_PEEL:.*]]
+; CHECK:       [[LOOP_PEEL]]:
+; CHECK-NEXT:    [[C_PEEL:%.*]] = icmp ne i32 [[TMP3]], [[SUB]]
+; CHECK-NEXT:    [[SEL_LCSSA:%.*]] = select i1 [[C_PEEL]], i32 1, i32 2
+; CHECK-NEXT:    call void @foo(i32 [[SEL_LCSSA]])
+; CHECK-NEXT:    [[IV_NEXT:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK:       [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK:       [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+entry:
+  %sub = add i32 %n, -2
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %c = icmp ne i32 %iv, %sub
+  %sel = select i1 %c, i32 1, i32 2
+  call void @foo(i32 %sel)
+  %iv.next = add i32 %iv, 1
+  %ec = icmp ne i32 %iv.next, %n
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  %sel.lcssa = phi i32 [ %sel, %loop ]
+  ret i32 %sel.lcssa
+}
+
+
+
+define i32 @peel_last_with_trip_count_check_lcssa_phi_cmp_not_invar3(i32 %n) {
+; CHECK-LABEL: define i32 @peel_last_with_trip_count_check_lcssa_phi_cmp_not_invar3(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ne i32 [[SUB]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK:       [[ENTRY_SPLIT]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @foo(i32 2)
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT:    [[DOTPH:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    br label %[[EXIT_PEEL_BEGIN]]
+; CHECK:       [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT:    br label %[[LOOP_PEEL:.*]]
+; CHECK:       [[LOOP_PEEL]]:
+; CHECK-NEXT:    [[C_PEEL:%.*]] = icmp eq i32 [[TMP2]], [[SUB]]
+; CHECK-NEXT:    [[SEL_PEEL:%.*]] = select i1 [[C_PEEL]], i32 1, i32 2
+; CHECK-NEXT:    call void @foo(i32 [[SEL_PEEL]])
+; CHECK-NEXT:    [[IV_NEXT_PEEL:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT:    [[EC_PEEL:%.*]] = icmp ne i32 [[IV_NEXT_PEEL]], [[N]]
+; CHECK-NEXT:    br i1 [[EC_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK:       [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK:       [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[SEL_PEEL]]
+;
+entry:
+  %sub = add i32 %n, -1
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %c = icmp eq i32 %iv, %sub
+  %sel = select i1 %c, i32 1, i32 2
+  call void @foo(i32 %sel)
+  %iv.next = add i32 %iv, 1
+  %ec = icmp ne i32 %iv.next, %n
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  %sel.lcssa = phi i32 [ %sel, %loop ]
+  ret i32 %sel.lcssa
+}
+
+define i32 @peel_last_with_trip_count_check_lcssa_phi_cmp_not_invar4(i32 %n) {
+; CHECK-LABEL: define i32 @peel_last_with_trip_count_check_lcssa_phi_cmp_not_invar4(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[IV]], [[SUB]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i32 1, i32 2
+; CHECK-NEXT:    call void @foo(i32 [[SEL]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[SEL_LCSSA]]
+;
+entry:
+  %sub = add i32 %n, -1
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %c = icmp ne i32 %iv, %sub
+  %sel = select i1 %c, i32 1, i32 2
+  call void @foo(i32 %sel)
+  %iv.next = add i32 %iv, 1
+  %ec = icmp ne i32 %iv.next, %n
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  %sel.lcssa = phi i32 [ %sel, %loop ]
+  ret i32 %sel.lcssa
+}
+
 define void @peel_last_with_trip_count_check_nested_loop(i32 %n) {
 ; CHECK-LABEL: define void @peel_last_with_trip_count_check_nested_loop(
 ; CHECK-SAME: i32 [[N:%.*]]) {
@@ -215,7 +363,7 @@ define void @peel_last_with_trip_count_check_nested_loop(i32 %n) {
 ; CHECK-NEXT:    [[IV_NEXT1]] = add nuw i32 [[IV1]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[N]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT1:%.*]] = icmp eq i32 [[IV_NEXT1]], [[TMP2]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT1]], label %[[OUTER_HEADER_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]], label %[[INNER_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT1]], label %[[OUTER_HEADER_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]], label %[[INNER_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
 ;
 entry:
   %sub = add i32 %n, -1
@@ -235,11 +383,14 @@ then:
 
 inner.latch:
   %iv.next = add i32 %iv, 1
-  %exitcond.not = icmp eq i32 %iv.next, %n
-  br i1 %exitcond.not, label %outer.header, label %inner.header
+  %ec = icmp eq i32 %iv.next, %n
+  br i1 %ec, label %outer.header, label %inner.header
 }
+
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
 ; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
 ;.
diff --git a/external/llvm-project/llvm/test/Transforms/LoopUnroll/unroll-and-peel-last-iteration.ll b/external/llvm-project/llvm/test/Transforms/LoopUnroll/unroll-and-peel-last-iteration.ll
index 7d38c18d1066..bc20c59c3943 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopUnroll/unroll-and-peel-last-iteration.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopUnroll/unroll-and-peel-last-iteration.ll
@@ -71,8 +71,106 @@ exit:
   ret i32 %1
 }
 
+; Test case for https://github.com/llvm/llvm-project/issues/142895.
+
+define i32 @pr142895_exit_value_is_arg(i32 %arg) {
+; CHECK-LABEL: define i32 @pr142895_exit_value_is_arg(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[INC]], 32
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN:.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK:       [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[INC]], %[[LOOP]] ]
+; CHECK-NEXT:    br label %[[LOOP_PEEL:.*]]
+; CHECK:       [[LOOP_PEEL]]:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 [[TMP0]], 32
+; CHECK-NEXT:    [[SEL_PEEL:%.*]] = select i1 [[CMP1_PEEL]], i32 0, i32 0
+; CHECK-NEXT:    [[SUB_PEEL:%.*]] = sub i32 0, 0
+; CHECK-NEXT:    [[XOR_PEEL:%.*]] = xor i32 0, 0
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    [[EXITCOND_PEEL:%.*]] = icmp ne i32 [[INC_PEEL]], 33
+; CHECK-NEXT:    br i1 [[EXITCOND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK:       [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK:       [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[ARG]]
+;
+entry:
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %cmp1 = icmp eq i32 %indvar, 32
+  %sel = select i1 %cmp1, i32 0, i32 0
+  %sub = sub i32 0, 0
+  %xor = xor i32 0, 0
+  %inc = add i32 %indvar, 1
+  %exitcond = icmp ne i32 %inc, 33
+  br i1 %exitcond, label %loop, label %exit
+
+exit:
+  %exit.lcssa = phi i32 [ %arg, %loop ]
+  ret i32 %exit.lcssa
+}
+
+define i32 @pr142895_exit_value_is_inst(i32 %arg) {
+; CHECK-LABEL: define i32 @pr142895_exit_value_is_inst(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[ARG]], 7
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[INC]], 32
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[INC]], %[[LOOP]] ]
+; CHECK-NEXT:    br label %[[LOOP_PEEL:.*]]
+; CHECK:       [[LOOP_PEEL]]:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 [[TMP0]], 32
+; CHECK-NEXT:    [[SEL_PEEL:%.*]] = select i1 [[CMP1_PEEL]], i32 0, i32 0
+; CHECK-NEXT:    [[SUB_PEEL:%.*]] = sub i32 0, 0
+; CHECK-NEXT:    [[XOR_PEEL:%.*]] = xor i32 0, 0
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    [[EXITCOND_PEEL:%.*]] = icmp ne i32 [[INC_PEEL]], 33
+; CHECK-NEXT:    br i1 [[EXITCOND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK:       [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK:       [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+entry:
+  %mul = mul i32 %arg, 7
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %cmp1 = icmp eq i32 %indvar, 32
+  %sel = select i1 %cmp1, i32 0, i32 0
+  %sub = sub i32 0, 0
+  %xor = xor i32 0, 0
+  %inc = add i32 %indvar, 1
+  %exitcond = icmp ne i32 %inc, 33
+  br i1 %exitcond, label %loop, label %exit
+
+exit:
+  %exit.lcssa = phi i32 [ %mul, %loop ]
+  ret i32 %exit.lcssa
+}
+
 declare void @foo(i32)
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
 ;.
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
new file mode 100644
index 000000000000..9435c544fc81
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br" --filter "^.*:" --version 5
+; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v1 -force-vector-interleave=1 -S < %s |  FileCheck %s -check-prefix=CHECK-V1-IC1
+; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v2 -force-vector-interleave=1 -S < %s |  FileCheck %s -check-prefix=CHECK-V2-IC1
+; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v2 -force-vector-interleave=4 -S < %s |  FileCheck %s -check-prefix=CHECK-V2-IC4
+
+target triple = "aarch64-unknown-linux-gnu"
+
+@a = global [1024 x i32] zeroinitializer, align 16
+@b = global [1024 x i32] zeroinitializer, align 16
+
+; We expect the branch weight computations after vectorisation to use
+; vscale=2 for neoverse-v1 and vscale=1 for neoverse-v2.
+define void @_Z3foov() {
+; CHECK-V1-IC1-LABEL: define void @_Z3foov(
+; CHECK-V1-IC1-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-V1-IC1:  [[ENTRY:.*:]]
+; CHECK-V1-IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-V1-IC1:  [[VECTOR_PH]]:
+; CHECK-V1-IC1:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V1-IC1:  [[VECTOR_BODY]]:
+; CHECK-V1-IC1:    br i1 [[TMP10:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-V1-IC1:  [[MIDDLE_BLOCK]]:
+; CHECK-V1-IC1:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF4:![0-9]+]]
+; CHECK-V1-IC1:  [[SCALAR_PH]]:
+; CHECK-V1-IC1:    br label %[[FOR_BODY:.*]]
+; CHECK-V1-IC1:  [[FOR_BODY]]:
+; CHECK-V1-IC1:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF5:![0-9]+]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-V1-IC1:  [[FOR_COND_CLEANUP]]:
+;
+; CHECK-V2-IC1-LABEL: define void @_Z3foov(
+; CHECK-V2-IC1-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-V2-IC1:  [[ENTRY:.*:]]
+; CHECK-V2-IC1:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-V2-IC1:  [[VECTOR_PH]]:
+; CHECK-V2-IC1:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V2-IC1:  [[VECTOR_BODY]]:
+; CHECK-V2-IC1:    br i1 [[TMP4:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-V2-IC1:  [[MIDDLE_BLOCK]]:
+; CHECK-V2-IC1:    br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF5:![0-9]+]]
+; CHECK-V2-IC1:  [[SCALAR_PH]]:
+; CHECK-V2-IC1:    br label %[[FOR_BODY:.*]]
+; CHECK-V2-IC1:  [[FOR_BODY]]:
+; CHECK-V2-IC1:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-V2-IC1:  [[FOR_COND_CLEANUP]]:
+;
+; CHECK-V2-IC4-LABEL: define void @_Z3foov(
+; CHECK-V2-IC4-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-V2-IC4:  [[VEC_EPILOG_VECTOR_BODY1:.*:]]
+; CHECK-V2-IC4:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-V2-IC4:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-V2-IC4:    br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK-V2-IC4:  [[VECTOR_PH]]:
+; CHECK-V2-IC4:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V2-IC4:  [[VECTOR_BODY]]:
+; CHECK-V2-IC4:    br i1 [[TMP12:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-V2-IC4:  [[MIDDLE_BLOCK]]:
+; CHECK-V2-IC4:    br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF5:![0-9]+]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-V2-IC4:    br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_PH]]:
+; CHECK-V2-IC4:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-V2-IC4:    br i1 [[TMP23:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-V2-IC4:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF8:![0-9]+]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-V2-IC4:    br label %[[FOR_BODY:.*]]
+; CHECK-V2-IC4:  [[FOR_BODY]]:
+; CHECK-V2-IC4:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-V2-IC4:  [[FOR_COND_CLEANUP]]:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %iv
+  %load = load i32, ptr %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %iv
+  store i32 %load, ptr %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !0
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 1023}
+;.
+; CHECK-V1-IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK-V1-IC1: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK-V1-IC1: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-V1-IC1: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-V1-IC1: [[PROF4]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK-V1-IC1: [[PROF5]] = !{!"branch_weights", i32 0, i32 0}
+; CHECK-V1-IC1: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META2]]}
+;.
+; CHECK-V2-IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK-V2-IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 255}
+; CHECK-V2-IC1: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+; CHECK-V2-IC1: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-V2-IC1: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-V2-IC1: [[PROF5]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK-V2-IC1: [[PROF6]] = !{!"branch_weights", i32 0, i32 0}
+; CHECK-V2-IC1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]], [[META3]]}
+;.
+; CHECK-V2-IC4: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK-V2-IC4: [[PROF1]] = !{!"branch_weights", i32 1, i32 63}
+; CHECK-V2-IC4: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+; CHECK-V2-IC4: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-V2-IC4: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-V2-IC4: [[PROF5]] = !{!"branch_weights", i32 1, i32 15}
+; CHECK-V2-IC4: [[PROF6]] = !{!"branch_weights", i32 2, i32 0}
+; CHECK-V2-IC4: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]], [[META4]]}
+; CHECK-V2-IC4: [[PROF8]] = !{!"branch_weights", i32 1, i32 1}
+; CHECK-V2-IC4: [[PROF9]] = !{!"branch_weights", i32 0, i32 0}
+; CHECK-V2-IC4: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]], [[META3]]}
+;.
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
index 31f299b4d8cf..f957279a7944 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
@@ -22,12 +22,12 @@ define void @f1(ptr %A) #0 {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0
-; CHECK-NEXT:    store <vscale x 4 x i32> splat (i32 1), ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
+; CHECK-NEXT:    store <vscale x 4 x i32> splat (i32 1), ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
index d4494089f708..5508a65744c6 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
@@ -60,16 +60,16 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
 ; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i32 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc i32 [[N_VEC5]] to i8
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP15]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i8> [[DOTSPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i8> [[DOTSPLAT]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
-; CHECK-NEXT:    [[DOTSPLATINSERT10:%.*]] = insertelement <8 x i8> poison, i8 [[TMP15]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT10:%.*]] = insertelement <8 x i8> poison, i8 [[BC_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT11:%.*]] = shufflevector <8 x i8> [[DOTSPLATINSERT10]], <8 x i8> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i8> [[DOTSPLAT11]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX6:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT13:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND7:%.*]] = phi <8 x i8> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI9:%.*]] = phi <8 x i8> [ [[DOTSPLAT11]], %[[VEC_EPILOG_PH]] ], [ [[TMP20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI9:%.*]] = phi <8 x i8> [ [[DOTSPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[IV:%.*]] = trunc i32 [[INDEX6]] to i8
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i8 [[IV]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[GEP]], i32 0
@@ -87,12 +87,12 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
 ; CHECK-NEXT:    [[CMP_N16:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC5]]
 ; CHECK-NEXT:    br i1 [[CMP_N16]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL17:%.*]] = phi i8 [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX18:%.*]] = phi i8 [ [[RDX_SELECT15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i8 [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i8 [ [[RDX_SELECT15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i8 [ [[BC_RESUME_VAL17]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX18]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i8 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i8 [[IV1]]
 ; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[GEP1]], align 8
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 3
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
index e3ee24bdd52d..a65d36b51443 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
@@ -38,6 +38,32 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP16:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> [[TMP14]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi double [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_0_LCSSA:%.*]] = phi double [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT:    ret double [[RES_0_LCSSA]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[RES_07:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[OFFSET]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[TMP17]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = load double, ptr [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ADD]] = fadd double [[RES_07]], [[TMP18]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
 ; SVE-LABEL: @test(
 ; SVE-NEXT:  entry:
@@ -70,6 +96,32 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur
 ; SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
 ; SVE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; SVE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SVE:       middle.block:
+; SVE-NEXT:    [[TMP15:%.*]] = call double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> [[TMP9]])
+; SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; SVE:       scalar.ph:
+; SVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; SVE-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
+; SVE-NEXT:    br label [[FOR_BODY:%.*]]
+; SVE:       for.cond.cleanup.loopexit:
+; SVE-NEXT:    [[ADD_LCSSA:%.*]] = phi double [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; SVE-NEXT:    br label [[FOR_COND_CLEANUP]]
+; SVE:       for.cond.cleanup:
+; SVE-NEXT:    [[RES_0_LCSSA:%.*]] = phi double [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; SVE-NEXT:    ret double [[RES_0_LCSSA]]
+; SVE:       for.body:
+; SVE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; SVE-NEXT:    [[RES_07:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD]], [[FOR_BODY]] ]
+; SVE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[OFFSET]], i64 [[INDVARS_IV]]
+; SVE-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; SVE-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[TMP13]] to i64
+; SVE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[IDXPROM1]]
+; SVE-NEXT:    [[TMP14:%.*]] = load double, ptr [[ARRAYIDX2]], align 8
+; SVE-NEXT:    [[ADD]] = fadd double [[RES_07]], [[TMP14]]
+; SVE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; SVE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; SVE-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
 entry:
   %cmp6 = icmp sgt i32 %size, 0
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/i1-reg-usage.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/i1-reg-usage.ll
index 0ec90b75002c..8c0fc6104e9a 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/i1-reg-usage.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/i1-reg-usage.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output -prefer-predicate-over-epilogue=scalar-epilogue 2>&1 < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize,vplan -disable-output -prefer-predicate-over-epilogue=scalar-epilogue 2>&1 < %s | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "aarch64"
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 6ed9c856f50c..4af4929fad52 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -105,12 +105,11 @@ define i64 @pointer_induction_only(ptr %start, ptr %end) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ <i64 poison, i64 0>, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i32 2
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <2 x i32>, ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[TMP9]] = zext <2 x i32> [[WIDE_LOAD4]] to <2 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <2 x i32> [[WIDE_LOAD4]] to <2 x i64>
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -162,12 +161,11 @@ define i64 @int_and_pointer_iv(ptr %start, i32 %N) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i32 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
-; CHECK-NEXT:    [[TMP5]] = zext <4 x i32> [[WIDE_LOAD3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i32> [[WIDE_LOAD3]] to <4 x i64>
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -306,7 +304,6 @@ define i64 @test_ptr_ivs_and_widened_ivs(ptr %src, i32 %N) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
@@ -316,7 +313,7 @@ define i64 @test_ptr_ivs_and_widened_ivs(ptr %src, i32 %N) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[WIDE_LOAD]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i32> [[TMP7]] to <4 x i64>
 ; CHECK-NEXT:    [[TMP9:%.*]] = zext <4 x i32> [[STEP_ADD]] to <4 x i64>
-; CHECK-NEXT:    [[TMP10]] = shl <4 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shl <4 x i64> [[TMP8]], [[TMP9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
index e97bb857fdba..31be8862a887 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
@@ -750,11 +750,10 @@ define void @add_phifail(ptr noalias nocapture readonly %p, ptr noalias nocaptur
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT:    [[TMP3]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw <16 x i32> [[TMP3]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc <16 x i32> [[TMP4]] to <16 x i8>
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]
@@ -832,11 +831,10 @@ define i8 @add_phifail2(ptr noalias nocapture readonly %p, ptr noalias nocapture
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
-; CHECK-NEXT:    [[TMP5]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw <16 x i32> [[TMP5]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP7:%.*]] = trunc <16 x i32> [[TMP6]] to <16 x i8>
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
index 0e5e785a9463..66dbcff2c123 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
@@ -10,8 +10,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -161,8 +160,8 @@ define void @dotp_small_epilogue_vf(i64 %idx.neg, i8 %a) #1 {
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i8> poison, i8 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT6]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IDX_NEG]], [[N_VEC5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = sext <4 x i8> [[BROADCAST_SPLAT7]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext <4 x i8> [[BROADCAST_SPLAT7]] to <4 x i32>
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX9:%.*]] = phi i64 [ [[IV]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
index e6687fe767c0..0fc324f720e6 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
@@ -589,88 +589,100 @@ for.exit:                        ; preds = %for.body
   ret i32 %add
 }
 
-define i32 @not_dotp_not_phi(ptr %a, ptr %b) {
+define i32 @not_dotp_not_phi(ptr %a, ptr noalias %b, ptr noalias %c) {
 ; CHECK-INTERLEAVE1-LABEL: define i32 @not_dotp_not_phi(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = mul <16 x i32> [[TMP6]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP8]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP4]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul <16 x i32> [[TMP5]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7]] = add <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP7]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    store <16 x i32> [[TMP8]], ptr [[TMP10]], align 4
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = extractelement <16 x i32> [[TMP8]], i32 15
-; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP8]], i32 15
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP7]], i32 15
+; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP7]], i32 15
 ; CHECK-INTERLEAVE1-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVE1:       scalar.ph:
 ;
 ; CHECK-INTERLEAVED-LABEL: define i32 @not_dotp_not_phi(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul <16 x i32> [[TMP6]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP8]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP4]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul <16 x i32> [[TMP5]], [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP7]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0
+; CHECK-INTERLEAVED-NEXT:    store <16 x i32> [[TMP8]], ptr [[TMP10]], align 4
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = extractelement <16 x i32> [[TMP8]], i32 15
-; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP8]], i32 15
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP7]], i32 15
+; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP7]], i32 15
 ; CHECK-INTERLEAVED-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVED:       scalar.ph:
 ;
 ; CHECK-MAXBW-LABEL: define i32 @not_dotp_not_phi(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i32> [ <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = mul <16 x i32> [[TMP6]], [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[TMP8]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP4]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul <16 x i32> [[TMP5]], [[TMP2]]
+; CHECK-MAXBW-NEXT:    [[TMP7]] = add <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP7]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0
+; CHECK-MAXBW-NEXT:    store <16 x i32> [[TMP8]], ptr [[TMP10]], align 4
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = extractelement <16 x i32> [[TMP8]], i32 15
-; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP8]], i32 15
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP7]], i32 15
+; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP7]], i32 15
 ; CHECK-MAXBW-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
 ;
@@ -688,6 +700,8 @@ for.body:                                         ; preds = %for.body, %entry
   %ext.b = zext i8 %load.b to i32
   %mul = mul i32 %ext.b, %ext.a
   %add = add i32 %mul, %ext.b
+  %gep.c = getelementptr i32, ptr %c, i64 %iv
+  store i32 %accum, ptr %gep.c
   %iv.next = add i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, 1024
   br i1 %exitcond.not, label %for.exit, label %for.body
@@ -946,6 +960,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[NUM_IN]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
+;
 entry:
   br label %for.body
 
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index 52dcba69d036..b091452e28b4 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -890,14 +890,9 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
-; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 [[TMP8]]
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
@@ -917,7 +912,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD3]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = mul <vscale x 8 x i32> [[TMP22]], [[TMP15]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP25]] = mul <vscale x 8 x i32> [[TMP23]], [[TMP16]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = mul <vscale x 8 x i32> [[TMP23]], [[TMP16]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> [[TMP24]], <vscale x 8 x i32> [[TMP25]], i32 -1)
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = add <vscale x 8 x i32> [[TMP25]], [[TMP26]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
@@ -1020,14 +1015,9 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
-; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
-; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 [[TMP8]]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0
 ; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
@@ -1037,7 +1027,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = mul <vscale x 8 x i32> [[TMP15]], [[TMP12]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP17]] = add <vscale x 8 x i32> [[TMP16]], [[TMP15]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = add <vscale x 8 x i32> [[TMP16]], [[TMP15]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1067,14 +1057,9 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
-; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 [[TMP8]]
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 8
@@ -1088,7 +1073,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP20]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = mul <vscale x 8 x i32> [[TMP22]], [[TMP15]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP21]] = add <vscale x 8 x i32> [[TMP30]], [[TMP22]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = add <vscale x 8 x i32> [[TMP30]], [[TMP22]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1118,14 +1103,9 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
-; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 [[TMP8]]
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP13]], align 1
@@ -1135,7 +1115,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP18]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP14]]
-; CHECK-MAXBW-NEXT:    [[TMP21]] = add <vscale x 8 x i32> [[TMP20]], [[TMP19]]
+; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = add <vscale x 8 x i32> [[TMP20]], [[TMP19]]
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll
index c5b2be33cae8..e51a925040a4 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
 
-; RUN: opt -mtriple arm64-linux -passes=loop-vectorize -mattr=+sve -debug-only=loop-vectorize -disable-output <%s 2>&1 | FileCheck %s
+; RUN: opt -mtriple arm64-linux -passes=loop-vectorize -mattr=+sve -debug-only=loop-vectorize,vplan -disable-output <%s 2>&1 | FileCheck %s
 
 ; Invariant register usage calculation should take into account if the
 ; invariant would be used in widened instructions. Only in such cases, a vector
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index 8ba3debd46c8..82f64e988e6d 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -230,54 +230,54 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP10]]
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP13]]
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP16]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP11]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP18]] = fadd <vscale x 8 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-UNORDERED-NEXT:    [[TMP19]] = fadd <vscale x 8 x float> [[WIDE_LOAD4]], [[VEC_PHI1]]
-; CHECK-UNORDERED-NEXT:    [[TMP20]] = fadd <vscale x 8 x float> [[WIDE_LOAD5]], [[VEC_PHI2]]
-; CHECK-UNORDERED-NEXT:    [[TMP21]] = fadd <vscale x 8 x float> [[WIDE_LOAD6]], [[VEC_PHI3]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP17]] = fadd <vscale x 8 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-UNORDERED-NEXT:    [[TMP18]] = fadd <vscale x 8 x float> [[WIDE_LOAD4]], [[VEC_PHI1]]
+; CHECK-UNORDERED-NEXT:    [[TMP19]] = fadd <vscale x 8 x float> [[WIDE_LOAD5]], [[VEC_PHI2]]
+; CHECK-UNORDERED-NEXT:    [[TMP20]] = fadd <vscale x 8 x float> [[WIDE_LOAD6]], [[VEC_PHI3]]
 ; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP19]], [[TMP18]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX7:%.*]] = fadd <vscale x 8 x float> [[TMP20]], [[BIN_RDX]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX8:%.*]] = fadd <vscale x 8 x float> [[TMP21]], [[BIN_RDX7]]
-; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX8]])
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP18]], [[TMP17]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX7:%.*]] = fadd <vscale x 8 x float> [[TMP19]], [[BIN_RDX]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX8:%.*]] = fadd <vscale x 8 x float> [[TMP20]], [[BIN_RDX7]]
+; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX8]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP22]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD]] = fadd float [[TMP24]], [[SUM_07]]
+; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD]] = fadd float [[TMP23]], [[SUM_07]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_strict_unroll
@@ -297,47 +297,47 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP10]]
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
-; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP13]]
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 24
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP16]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP11]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[WIDE_LOAD]])
-; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP18]], <vscale x 8 x float> [[WIDE_LOAD1]])
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP19]], <vscale x 8 x float> [[WIDE_LOAD2]])
-; CHECK-ORDERED-NEXT:    [[TMP21]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP20]], <vscale x 8 x float> [[WIDE_LOAD3]])
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
+; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[WIDE_LOAD]])
+; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP17]], <vscale x 8 x float> [[WIDE_LOAD1]])
+; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP18]], <vscale x 8 x float> [[WIDE_LOAD2]])
+; CHECK-ORDERED-NEXT:    [[TMP20]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP19]], <vscale x 8 x float> [[WIDE_LOAD3]])
 ; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD]] = fadd float [[TMP23]], [[SUM_07]]
+; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD]] = fadd float [[TMP22]], [[SUM_07]]
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_strict_unroll
@@ -378,47 +378,47 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT13:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP35:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP18]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP21]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP24]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP27]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[WIDE_MASKED_LOAD]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP28]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP29]], <vscale x 8 x float> [[TMP30]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP31]], <vscale x 8 x float> [[TMP32]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP35]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP33]], <vscale x 8 x float> [[TMP34]])
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP19]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP22]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP25]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP17]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP20]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP23]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP26]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[WIDE_MASKED_LOAD]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP27]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP28]], <vscale x 8 x float> [[TMP29]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP30]], <vscale x 8 x float> [[TMP31]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP34]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP32]], <vscale x 8 x float> [[TMP33]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = add i64 [[INDEX]], [[TMP37]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = mul i64 [[TMP39]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = add i64 [[INDEX]], [[TMP40]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = mul i64 [[TMP42]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP43]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = add i64 [[INDEX]], [[TMP36]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = mul i64 [[TMP38]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = add i64 [[INDEX]], [[TMP39]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = mul i64 [[TMP41]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = add i64 [[INDEX]], [[TMP42]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP38]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP41]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP44]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = extractelement <vscale x 8 x i1> [[TMP45]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP46]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP37]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP40]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP43]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = extractelement <vscale x 8 x i1> [[TMP44]], i32 0
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP45]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
@@ -429,13 +429,13 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP47:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD]] = fadd float [[TMP47]], [[SUM_07]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD]] = fadd float [[TMP46]], [[SUM_07]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP35]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP34]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[ADD_LCSSA]]
 ;
 
@@ -643,34 +643,34 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP2]], [[TMP10]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = icmp ugt i64 [[TMP2]], [[TMP10]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP11]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP2]], [[TMP9]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]])
 ; CHECK-ORDERED-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       vector.body:
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-ORDERED-TF-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP14]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP13]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_MASKED_VEC]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP16]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP18]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP17]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP15]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP20]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP19]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP15]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP17]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP16]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP18]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP13]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = extractelement <vscale x 4 x i1> [[TMP21]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP12]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
@@ -683,18 +683,18 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-TF-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDXB1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD1]] = fadd float [[TMP23]], [[ADD_PHI2]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD1]] = fadd float [[TMP22]], [[ADD_PHI2]]
 ; CHECK-ORDERED-TF-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDXB2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OR]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD2]] = fadd float [[TMP24]], [[ADD_PHI1]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD2]] = fadd float [[TMP23]], [[ADD_PHI1]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
-; CHECK-ORDERED-TF-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    store float [[ADD1_LCSSA]], ptr [[A]], align 4
 ; CHECK-ORDERED-TF-NEXT:    store float [[ADD2_LCSSA]], ptr [[ARRAYIDXA]], align 4
 ; CHECK-ORDERED-TF-NEXT:    ret void
@@ -1396,71 +1396,71 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP32:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP10]]
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP13]]
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP16]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP11]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i32 0
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP21]]
-; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP24]]
-; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP27]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP28]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP29]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
-; CHECK-UNORDERED-NEXT:    [[TMP30]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
-; CHECK-UNORDERED-NEXT:    [[TMP31]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
-; CHECK-UNORDERED-NEXT:    [[TMP32]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
+; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
+; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP24]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP27]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP28]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
+; CHECK-UNORDERED-NEXT:    [[TMP29]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
+; CHECK-UNORDERED-NEXT:    [[TMP30]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
+; CHECK-UNORDERED-NEXT:    [[TMP31]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
 ; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP30]], [[TMP29]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd <vscale x 8 x float> [[TMP31]], [[BIN_RDX]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd <vscale x 8 x float> [[TMP32]], [[BIN_RDX11]]
-; CHECK-UNORDERED-NEXT:    [[TMP34:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP29]], [[TMP28]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd <vscale x 8 x float> [[TMP30]], [[BIN_RDX]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd <vscale x 8 x float> [[TMP31]], [[BIN_RDX11]]
+; CHECK-UNORDERED-NEXT:    [[TMP33:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP34]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP33]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP35:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP34:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP36:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP35]], float [[TMP36]], float [[SUM_07]])
+; CHECK-UNORDERED-NEXT:    [[TMP35:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP34]], float [[TMP35]], float [[SUM_07]])
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP34]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP33]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fmuladd_strict
@@ -1480,68 +1480,68 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP10]]
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
-; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP13]]
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 24
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP16]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP11]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i32 0
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
-; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP21]]
-; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 16
-; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP24]]
-; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 24
-; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP27]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP28]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-ORDERED-NEXT:    [[TMP31:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
-; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
-; CHECK-ORDERED-NEXT:    [[TMP33:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP29]])
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP35:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
+; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
+; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
+; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
+; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP24]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP27]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
+; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
+; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; CHECK-ORDERED-NEXT:    [[TMP31:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
+; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP28]])
+; CHECK-ORDERED-NEXT:    [[TMP33:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP32]], <vscale x 8 x float> [[TMP29]])
 ; CHECK-ORDERED-NEXT:    [[TMP34:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP33]], <vscale x 8 x float> [[TMP30]])
-; CHECK-ORDERED-NEXT:    [[TMP35:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP34]], <vscale x 8 x float> [[TMP31]])
-; CHECK-ORDERED-NEXT:    [[TMP36]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP35]], <vscale x 8 x float> [[TMP32]])
+; CHECK-ORDERED-NEXT:    [[TMP35]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP34]], <vscale x 8 x float> [[TMP31]])
 ; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP36]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP35]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP38:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP37:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP39:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP38]], float [[TMP39]], float [[SUM_07]])
+; CHECK-ORDERED-NEXT:    [[TMP38:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP37]], float [[TMP38]], float [[SUM_07]])
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP36]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP35]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fmuladd_strict
@@ -1582,66 +1582,66 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT17:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP50:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP18]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP21]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP24]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP27]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i32 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP31]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP33]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP34]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP37]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP29]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP32]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP35]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP38]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP39]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP43]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP40]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP44]], <vscale x 8 x float> [[TMP45]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP47:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP41]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP48:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP46]], <vscale x 8 x float> [[TMP47]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP49:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP42]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP50]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP48]], <vscale x 8 x float> [[TMP49]])
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP49:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP19]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP22]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP25]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP17]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP20]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP23]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP26]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP30]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP33]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP36]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP28]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP31]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP34]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP37]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP38]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP42]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP39]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP43]], <vscale x 8 x float> [[TMP44]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP40]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP47:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP45]], <vscale x 8 x float> [[TMP46]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP48:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP41]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP49]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP47]], <vscale x 8 x float> [[TMP48]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = mul i64 [[TMP51]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX]], [[TMP52]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = mul i64 [[TMP54]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = add i64 [[INDEX]], [[TMP55]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP57]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP59:%.*]] = add i64 [[INDEX]], [[TMP58]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP50:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = mul i64 [[TMP50]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = add i64 [[INDEX]], [[TMP51]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = mul i64 [[TMP53]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = add i64 [[INDEX]], [[TMP54]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = mul i64 [[TMP56]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = add i64 [[INDEX]], [[TMP57]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP53]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP56]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP59]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP60:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP61:%.*]] = extractelement <vscale x 8 x i1> [[TMP60]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP61]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP52]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP55]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP58]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP59:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-ORDERED-TF-NEXT:    [[TMP60:%.*]] = extractelement <vscale x 8 x i1> [[TMP59]], i32 0
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
@@ -1652,15 +1652,15 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP62:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP61:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP63:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP62]], float [[TMP63]], float [[SUM_07]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP62:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP61]], float [[TMP62]], float [[SUM_07]])
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP49]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 
@@ -1724,71 +1724,71 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP32:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP10]]
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP13]]
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP16]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP11]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i32 0
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP21]]
-; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP24]]
-; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP27]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP28]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP29]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
-; CHECK-UNORDERED-NEXT:    [[TMP30]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
-; CHECK-UNORDERED-NEXT:    [[TMP31]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
-; CHECK-UNORDERED-NEXT:    [[TMP32]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
+; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
+; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP24]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP27]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP28]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
+; CHECK-UNORDERED-NEXT:    [[TMP29]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
+; CHECK-UNORDERED-NEXT:    [[TMP30]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
+; CHECK-UNORDERED-NEXT:    [[TMP31]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
 ; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd nnan <vscale x 8 x float> [[TMP30]], [[TMP29]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd nnan <vscale x 8 x float> [[TMP31]], [[BIN_RDX]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd nnan <vscale x 8 x float> [[TMP32]], [[BIN_RDX11]]
-; CHECK-UNORDERED-NEXT:    [[TMP34:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd nnan <vscale x 8 x float> [[TMP29]], [[TMP28]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd nnan <vscale x 8 x float> [[TMP30]], [[BIN_RDX]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd nnan <vscale x 8 x float> [[TMP31]], [[BIN_RDX11]]
+; CHECK-UNORDERED-NEXT:    [[TMP33:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP34]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP33]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP35:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP34:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP36:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP35]], float [[TMP36]], float [[SUM_07]])
+; CHECK-UNORDERED-NEXT:    [[TMP35:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP34]], float [[TMP35]], float [[SUM_07]])
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP34]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP33]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fmuladd_strict_fmf
@@ -1808,68 +1808,68 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP10]]
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
-; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP13]]
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 24
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 [[TMP16]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP11]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i32 0
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
-; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP21]]
-; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 16
-; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP24]]
-; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 24
-; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP27]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP28]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-ORDERED-NEXT:    [[TMP31:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
-; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
-; CHECK-ORDERED-NEXT:    [[TMP33:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP29]])
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP35:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
+; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
+; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
+; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
+; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP24]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP27]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
+; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
+; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; CHECK-ORDERED-NEXT:    [[TMP31:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
+; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP28]])
+; CHECK-ORDERED-NEXT:    [[TMP33:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP32]], <vscale x 8 x float> [[TMP29]])
 ; CHECK-ORDERED-NEXT:    [[TMP34:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP33]], <vscale x 8 x float> [[TMP30]])
-; CHECK-ORDERED-NEXT:    [[TMP35:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP34]], <vscale x 8 x float> [[TMP31]])
-; CHECK-ORDERED-NEXT:    [[TMP36]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP35]], <vscale x 8 x float> [[TMP32]])
+; CHECK-ORDERED-NEXT:    [[TMP35]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP34]], <vscale x 8 x float> [[TMP31]])
 ; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP36]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP35]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP38:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP37:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP39:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP38]], float [[TMP39]], float [[SUM_07]])
+; CHECK-ORDERED-NEXT:    [[TMP38:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP37]], float [[TMP38]], float [[SUM_07]])
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP36]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP35]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fmuladd_strict_fmf
@@ -1910,66 +1910,66 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT17:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP50:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP18]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP21]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP24]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP27]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i32 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP31]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP33]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP34]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP37]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP29]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP32]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP35]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP38]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP39]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP43]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP40]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP44]], <vscale x 8 x float> [[TMP45]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP47:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP41]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP48:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP46]], <vscale x 8 x float> [[TMP47]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP49:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP42]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP50]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP48]], <vscale x 8 x float> [[TMP49]])
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP49:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP19]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP22]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP25]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP17]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP20]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP23]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP26]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP30]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP33]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP36]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP28]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP31]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP34]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP37]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP38]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP42]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP39]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP43]], <vscale x 8 x float> [[TMP44]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP40]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP47:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP45]], <vscale x 8 x float> [[TMP46]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP48:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP41]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP49]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP47]], <vscale x 8 x float> [[TMP48]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = mul i64 [[TMP51]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX]], [[TMP52]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = mul i64 [[TMP54]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = add i64 [[INDEX]], [[TMP55]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP57]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP59:%.*]] = add i64 [[INDEX]], [[TMP58]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP50:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = mul i64 [[TMP50]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = add i64 [[INDEX]], [[TMP51]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = mul i64 [[TMP53]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = add i64 [[INDEX]], [[TMP54]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = mul i64 [[TMP56]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = add i64 [[INDEX]], [[TMP57]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP53]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP56]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP59]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP60:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP61:%.*]] = extractelement <vscale x 8 x i1> [[TMP60]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP61]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP52]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP55]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP58]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP59:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-ORDERED-TF-NEXT:    [[TMP60:%.*]] = extractelement <vscale x 8 x i1> [[TMP59]], i32 0
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
@@ -1980,15 +1980,15 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP62:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP61:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP63:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP62]], float [[TMP63]], float [[SUM_07]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP62:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP61]], float [[TMP62]], float [[SUM_07]])
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP49]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
new file mode 100644
index 000000000000..cc7b9e26ca25
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
@@ -0,0 +1,338 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -mtriple=arm64-apple-macosx -S %s | FileCheck %s
+
+define i64 @test_vectorize_select_umin_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sgt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ult i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp slt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 6861644fc996..77e713256d24 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -375,8 +375,8 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP10]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP11]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE1]], [[VEC_IND]]
@@ -1479,34 +1479,24 @@ define void @interleave_deinterleave(ptr writeonly noalias %dst, ptr readonly %a
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP7]])
-; CHECK-NEXT:    [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP8]])
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC6]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC7]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC6]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC7]], 1
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 3
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_VEC8:%.*]] = load <vscale x 16 x i32>, ptr [[TMP13]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC9:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC8]])
-; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC9]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC9]], 1
-; CHECK-NEXT:    [[STRIDED_VEC10:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP14]])
-; CHECK-NEXT:    [[STRIDED_VEC11:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP15]])
-; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 0
-; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC11]], 0
-; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC10]], 1
-; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC11]], 1
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC8]])
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
+; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 3
 ; CHECK-NEXT:    [[TMP20:%.*]] = add nsw <vscale x 4 x i32> [[TMP16]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[DST:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = sub nsw <vscale x 4 x i32> [[TMP10]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = shl <vscale x 4 x i32> [[TMP11]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = ashr <vscale x 4 x i32> [[TMP12]], [[TMP19]]
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x i32> [[TMP23]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[TMP24]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC13:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> [[INTERLEAVED_VEC]], <vscale x 8 x i32> [[INTERLEAVED_VEC12]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC13:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[TMP23]], <vscale x 4 x i32> [[TMP24]])
 ; CHECK-NEXT:    store <vscale x 16 x i32> [[INTERLEAVED_VEC13]], ptr [[TMP21]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1595,18 +1585,14 @@ define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A,
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP11]])
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP12]])
-; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC1]], 0
-; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC1]], 1
-; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP14]])
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
 ; CHECK-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]])
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 3
 ; CHECK-NEXT:    [[REVERSE5:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP16]])
 ; CHECK-NEXT:    [[TMP17:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE3]], [[VEC_IND]]
@@ -1622,9 +1608,7 @@ define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A,
 ; CHECK-NEXT:    [[REVERSE7:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP18]])
 ; CHECK-NEXT:    [[REVERSE8:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP19]])
 ; CHECK-NEXT:    [[REVERSE9:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP20]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[REVERSE6]], <vscale x 4 x i32> [[REVERSE8]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[REVERSE7]], <vscale x 4 x i32> [[REVERSE9]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> [[INTERLEAVED_VEC]], <vscale x 8 x i32> [[INTERLEAVED_VEC10]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[REVERSE6]], <vscale x 4 x i32> [[REVERSE7]], <vscale x 4 x i32> [[REVERSE8]], <vscale x 4 x i32> [[REVERSE9]])
 ; CHECK-NEXT:    store <vscale x 16 x i32> [[INTERLEAVED_VEC11]], ptr [[TMP26]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
index 469faf67a71b..3567aff0ace4 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
@@ -469,36 +469,26 @@ define dso_local void @masked_strided_factor4(ptr noalias nocapture readonly %p,
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 2
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK2:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i1> [[INTERLEAVED_MASK1]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK2]], <vscale x 64 x i8> poison)
-; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 32 x i8>, <vscale x 32 x i8> } @llvm.vector.deinterleave2.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 0
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 1
-; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP11]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP12]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 0
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 0
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 1
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
+; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[TMP12]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP15]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]])
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP17]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP8]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP21]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP19]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC5:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP18]], <vscale x 16 x i8> [[TMP20]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC6:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave2.nxv64i8(<vscale x 32 x i8> [[INTERLEAVED_VEC]], <vscale x 32 x i8> [[INTERLEAVED_VEC5]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK7:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK8:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK9:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK7]], <vscale x 32 x i1> [[INTERLEAVED_MASK8]])
-; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC6]], ptr [[TMP22]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK9]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP8]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP19]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]], <vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP18]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP20]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK1]])
 ; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SCALAR_TAIL_FOLDING:       middle.block:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -531,37 +521,27 @@ define dso_local void @masked_strided_factor4(ptr noalias nocapture readonly %p,
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 2
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK2:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i1> [[INTERLEAVED_MASK1]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK2]], <vscale x 64 x i8> poison)
-; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 32 x i8>, <vscale x 32 x i8> } @llvm.vector.deinterleave2.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 0
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } [[STRIDED_VEC]], 1
-; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP11]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[TMP12]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 0
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 0
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC3]], 1
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC4]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[TMP12]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP15]]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP17]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP8]] to i64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP21]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP19]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC5:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP18]], <vscale x 16 x i8> [[TMP20]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC6:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave2.nxv64i8(<vscale x 32 x i8> [[INTERLEAVED_VEC]], <vscale x 32 x i8> [[INTERLEAVED_VEC5]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK7:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK8:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK9:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave2.nxv64i1(<vscale x 32 x i1> [[INTERLEAVED_MASK7]], <vscale x 32 x i1> [[INTERLEAVED_MASK8]])
-; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC6]], ptr [[TMP22]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK9]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP8]] to i64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP19]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]], <vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP18]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP20]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK1]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP4]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP23]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP21]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
 ; PREDICATED_TAIL_FOLDING:       middle.block:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_END:%.*]]
 ; PREDICATED_TAIL_FOLDING:       scalar.ph:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index b2175b4cf22e..166bdc7931cf 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -63,21 +63,20 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[IND_END2:%.*]] = getelementptr i8, ptr [[START_2:%.*]], i64 [[N_VEC]]
-
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 1, [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP8]], 0
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP10]], 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = mul <vscale x 2 x i64> [[TMP13]], splat (i64 1)
-; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP14]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = mul <vscale x 2 x i64> [[TMP14]], splat (i64 1)
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP15]]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 1
@@ -89,7 +88,7 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 2 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; CHECK-NEXT:    store <vscale x 2 x i8> [[TMP20]], ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP10]]
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -98,12 +97,12 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START_1]], [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[START_2]], [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[START_2]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP_BODY:%.*]]
 ; CHECK:       loop.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY]] ]
 ; CHECK-NEXT:    [[PTR_IV_1:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_IV_1_NEXT:%.*]], [[LOOP_BODY]] ]
-; CHECK-NEXT:    [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], [[LOOP_BODY]] ]
+; CHECK-NEXT:    [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], [[LOOP_BODY]] ]
 ; CHECK-NEXT:    [[PTR_IV_1_NEXT]] = getelementptr inbounds ptr, ptr [[PTR_IV_1]], i64 1
 ; CHECK-NEXT:    [[PTR_IV_2_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV_2]], i64 1
 ; CHECK-NEXT:    store ptr [[PTR_IV_2_NEXT]], ptr [[PTR_IV_1]], align 8
@@ -118,6 +117,7 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ;
 
 
+
 entry:
   br label %loop.body
 
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
index 9ab3d99031bb..444c6dd26942 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
@@ -49,7 +49,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi [[VTC]], ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ [[VTC]], middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
@@ -96,7 +96,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi [[VTC]], ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ [[VTC]], middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index 85624a59ec96..46b4762520de 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -47,7 +47,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
@@ -94,7 +94,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
@@ -146,7 +146,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
@@ -193,7 +193,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
@@ -244,7 +244,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
@@ -291,7 +291,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:  EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll
index cb966e4088db..2705d6910bb2 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll
@@ -11,8 +11,7 @@ define void @test_remove_vector_loop_region_epilogue(ptr %dst, i1 %c)  {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 8
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TC]], 64
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 true, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], 64
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
index 4bc5bb0acd64..8095f258ea18 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
@@ -17,12 +17,13 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
 ; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi ir<0>, ir<[[REDUCE:%.+]]> (VF scaled by 1/4)
+; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi vp<[[RDX_START]]>, ir<[[REDUCE:%.+]]> (VF scaled by 1/4)
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
 ; CHECK-NEXT:   CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
 ; CHECK-NEXT:   vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>
@@ -52,8 +53,8 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val> = resume-phi vp<[[VEC_TC]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<%bc.merge.rdx> = resume-phi vp<[[RED_RESULT]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VEC_TC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[RED_RESULT]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
@@ -83,11 +84,12 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
 ; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.body:
 ; CHECK-NEXT:   EMIT-SCALAR vp<[[EP_IV:%.+]]> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<%index.next>, vector.body ]
-; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, ir<%add> (VF scaled by 1/4)
+; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%accum> = phi vp<[[RDX_START]]>, ir<%add> (VF scaled by 1/4)
 ; CHECK-NEXT:   CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[EP_IV]]>
 ; CHECK-NEXT:   vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>
 ; CHECK-NEXT:   WIDEN ir<%load.a> = load vp<[[PTR_A]]>
@@ -114,8 +116,8 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<scalar.ph>:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[EP_RESUME:%.+]]> = resume-phi ir<1024>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[EP_MERGE:%.+]]> = resume-phi vp<[[RED_RESULT]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[EP_RESUME:%.+]]> = phi [ ir<1024>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[EP_MERGE:%.+]]> = phi [ vp<[[RED_RESULT]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
index b9faa8e25232..1f2e91884a5d 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
@@ -45,7 +45,7 @@ target triple = "arm64-apple-ios"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
@@ -92,7 +92,7 @@ target triple = "arm64-apple-ios"
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll
index 4628bdb9b3c3..3d925b57b408 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll
@@ -17,9 +17,42 @@ define protected amdgpu_kernel void @func_int8(ptr addrspace(1) %p_a_grid.coerce
 ; CHECK:       [[FOR_BODY_LR_PH]]:
 ; CHECK-NEXT:    [[MUL15:%.*]] = mul nsw i32 [[ADD]], [[K]]
 ; CHECK-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[ADD12]], [[K]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[K]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[INDEX]], [[MUL15]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[INDEX]], [[MUL17]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P_A_GRID_COERCE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr addrspace(1) [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P_B_GRID_COERCE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP6]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <2 x i8>, ptr addrspace(1) [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = zext <2 x i8> [[WIDE_LOAD]] to <2 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <2 x i8> [[WIDE_LOAD1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw nsw <2 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11]] = add <2 x i32> [[TMP10]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP11]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_LR_PH]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK:       [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT:    [[ADD24_LCSSA:%.*]] = phi i32 [ [[ADD24:%.*]], %[[FOR_BODY]] ]
+; CHECK:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT:    [[ADD24_LCSSA:%.*]] = phi i32 [ [[ADD24:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[TMP15:%.*]] = trunc i32 [[ADD24_LCSSA]] to i8
 ; CHECK-NEXT:    br label %[[FOR_COND_CLEANUP]]
 ; CHECK:       [[FOR_COND_CLEANUP]]:
@@ -31,8 +64,8 @@ define protected amdgpu_kernel void @func_int8(ptr addrspace(1) %p_a_grid.coerce
 ; CHECK-NEXT:    store i8 [[V_ACC_0_LCSSA]], ptr addrspace(1) [[ARRAYIDX28]], align 1
 ; CHECK-NEXT:    br label %[[IF_END]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[K_IDX_046:%.*]] = phi i32 [ 0, %[[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[V_ACC_045:%.*]] = phi i32 [ 0, %[[FOR_BODY_LR_PH]] ], [ [[ADD24]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[K_IDX_046:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[V_ACC_045:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD24]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[K_IDX_046]], [[MUL15]]
 ; CHECK-NEXT:    [[ADD18:%.*]] = add nsw i32 [[K_IDX_046]], [[MUL17]]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ADD16]] to i64
@@ -47,7 +80,7 @@ define protected amdgpu_kernel void @func_int8(ptr addrspace(1) %p_a_grid.coerce
 ; CHECK-NEXT:    [[ADD24]] = add i32 [[MUL23]], [[V_ACC_045]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[K_IDX_046]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[K]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       [[IF_END]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -99,3 +132,9 @@ for.body:                                         ; preds = %for.body, %for.body
 if.end:                                           ; preds = %for.cond.cleanup, %entry
   ret void
 }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META1]]}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll
new file mode 100644
index 000000000000..4c29a3a0d1d0
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt -mattr=+mve -passes=loop-vectorize < %s -S -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-unknown-none-eabihf"
+
+; Even though it has high register pressure, this example should still vectorise since the mul+add chains become VMLAs.
+
+define void @fn(i32 noundef %n, ptr %in, ptr %out) #0 {
+; CHECK-LABEL: define void @fn(
+; CHECK-SAME: i32 noundef [[N:%.*]], ptr [[IN:%.*]], ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP46_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP46_NOT]], [[EXIT:label %.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[N]], 3
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i32 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[IN]], i32 [[TMP0]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 3
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[IN]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI2:%.*]] = phi ptr [ [[OUT]], %[[VECTOR_PH]] ], [ [[PTR_IND3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:    [[VECTOR_GEP4:%.*]] = getelementptr i8, ptr [[POINTER_PHI2]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 1
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[VECTOR_GEP]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 2
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[TMP1]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[TMP2]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 19595)
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER5]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw nsw <4 x i32> [[TMP5]], splat (i32 38470)
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER6]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw nsw <4 x i32> [[TMP7]], splat (i32 7471)
+; CHECK-NEXT:    [[TMP9:%.*]] = add nuw nsw <4 x i32> [[TMP4]], splat (i32 32768)
+; CHECK-NEXT:    [[TMP10:%.*]] = add nuw nsw <4 x i32> [[TMP9]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add nuw nsw <4 x i32> [[TMP10]], [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = lshr <4 x i32> [[TMP11]], splat (i32 16)
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc <4 x i32> [[TMP12]] to <4 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 32767)
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw <4 x i32> [[TMP5]], splat (i32 16762097)
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw <4 x i32> [[TMP7]], splat (i32 16759568)
+; CHECK-NEXT:    [[TMP17:%.*]] = add nuw nsw <4 x i32> [[TMP14]], splat (i32 32768)
+; CHECK-NEXT:    [[TMP18:%.*]] = add nuw <4 x i32> [[TMP17]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = add <4 x i32> [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = lshr <4 x i32> [[TMP19]], splat (i32 16)
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i8>
+; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 13282)
+; CHECK-NEXT:    [[TMP23:%.*]] = mul nuw <4 x i32> [[TMP5]], splat (i32 16744449)
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nuw nsw <4 x i32> [[TMP7]], splat (i32 19485)
+; CHECK-NEXT:    [[TMP25:%.*]] = add nuw nsw <4 x i32> [[TMP22]], splat (i32 32768)
+; CHECK-NEXT:    [[TMP26:%.*]] = add nuw <4 x i32> [[TMP25]], [[TMP23]]
+; CHECK-NEXT:    [[TMP27:%.*]] = add nuw <4 x i32> [[TMP26]], [[TMP24]]
+; CHECK-NEXT:    [[TMP28:%.*]] = lshr <4 x i32> [[TMP27]], splat (i32 16)
+; CHECK-NEXT:    [[TMP29:%.*]] = trunc <4 x i32> [[TMP28]] to <4 x i8>
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 1
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP13]], <4 x ptr> [[VECTOR_GEP4]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 2
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP21]], <4 x ptr> [[TMP30]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP29]], <4 x ptr> [[TMP31]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 12
+; CHECK-NEXT:    [[PTR_IND3]] = getelementptr i8, ptr [[POINTER_PHI2]], i32 12
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  %cmp46.not = icmp eq i32 %n, 0
+  br i1 %cmp46.not, label %exit, label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %ptr.iv.1 = phi ptr [ %in, %entry ], [ %ptr.iv.1.next, %for.body ]
+  %ptr.iv.2 = phi ptr [ %out, %entry ], [ %ptr.iv.2.next, %for.body ]
+  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+  %incdec.ptr = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 1
+  %0 = load i8, ptr %ptr.iv.1, align 1
+  %incdec.ptr1 = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 2
+  %1 = load i8, ptr %incdec.ptr, align 1
+  %ptr.iv.1.next = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 3
+  %2 = load i8, ptr %incdec.ptr1, align 1
+  %conv = zext i8 %0 to i32
+  %mul = mul nuw nsw i32 %conv, 19595
+  %conv3 = zext i8 %1 to i32
+  %mul4 = mul nuw nsw i32 %conv3, 38470
+  %conv5 = zext i8 %2 to i32
+  %mul6 = mul nuw nsw i32 %conv5, 7471
+  %add = add nuw nsw i32 %mul, 32768
+  %add7 = add nuw nsw i32 %add, %mul4
+  %add8 = add nuw nsw i32 %add7, %mul6
+  %shr = lshr i32 %add8, 16
+  %conv9 = trunc nuw i32 %shr to i8
+  %mul11 = mul nuw nsw i32 %conv, 32767
+  %mul13 = mul nuw i32 %conv3, 16762097
+  %mul16 = mul nuw i32 %conv5, 16759568
+  %add14 = add nuw nsw i32 %mul11, 32768
+  %add17 = add nuw i32 %add14, %mul13
+  %add18 = add i32 %add17, %mul16
+  %shr19 = lshr i32 %add18, 16
+  %conv20 = trunc i32 %shr19 to i8
+  %mul22 = mul nuw nsw i32 %conv, 13282
+  %mul24 = mul nuw i32 %conv3, 16744449
+  %mul27 = mul nuw nsw i32 %conv5, 19485
+  %add25 = add nuw nsw i32 %mul22, 32768
+  %add28 = add nuw i32 %add25, %mul24
+  %add29 = add nuw i32 %add28, %mul27
+  %shr30 = lshr i32 %add29, 16
+  %conv31 = trunc i32 %shr30 to i8
+  %incdec.ptr32 = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 1
+  store i8 %conv9, ptr %ptr.iv.2, align 1
+  %incdec.ptr33 = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 2
+  store i8 %conv20, ptr %incdec.ptr32, align 1
+  %ptr.iv.2.next = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 3
+  store i8 %conv31, ptr %incdec.ptr33, align 1
+  %iv.next = add nuw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll
index 5baf1e013a50..de49337c185a 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll
@@ -1,9 +1,9 @@
 ; REQUIRES: asserts
 ; RUN: opt --passes=loop-vectorize --mtriple loongarch64-linux-gnu \
-; RUN:   --mattr=+lsx -debug-only=loop-vectorize --force-vector-width=1 \
+; RUN:   --mattr=+lsx -debug-only=loop-vectorize,vplan --force-vector-width=1 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALAR
 ; RUN: opt --passes=loop-vectorize --mtriple loongarch64-linux-gnu \
-; RUN:   --mattr=+lsx -debug-only=loop-vectorize --force-vector-width=4 \
+; RUN:   --mattr=+lsx -debug-only=loop-vectorize,vplan --force-vector-width=4 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-VECTOR
 
 define void @bar(ptr %A, i32 signext %n) {
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
index f1947dec2ea2..b4987127a513 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
@@ -153,10 +153,10 @@ define i1 @select_exit_cond(ptr %start, ptr %end, i64 %N) {
 ; CHECK-NEXT:    [[N_MOD_VF24:%.*]] = urem i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_VEC25:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF24]]
 ; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC25]]
+; CHECK-NEXT:    [[TMP57:%.*]] = insertelement <2 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>
-; CHECK-NEXT:    [[TMP57:%.*]] = insertelement <2 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX38:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
index 280b3af04a4d..2c2a60ecc47b 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -debug-only=loop-vectorize -passes='function(loop-vectorize),default<O2>' -vectorizer-maximize-bandwidth -mtriple=powerpc64-unknown-linux -S -mcpu=pwr8 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR8
-; RUN: opt < %s -debug-only=loop-vectorize -passes='function(loop-vectorize),default<O2>' -vectorizer-maximize-bandwidth -mtriple=powerpc64le-unknown-linux -S -mcpu=pwr9 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR9
+; RUN: opt < %s -debug-only=loop-vectorize,vplan -passes='function(loop-vectorize),default<O2>' -vectorizer-maximize-bandwidth -mtriple=powerpc64-unknown-linux -S -mcpu=pwr8 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR8
+; RUN: opt < %s -debug-only=loop-vectorize,vplan -passes='function(loop-vectorize),default<O2>' -vectorizer-maximize-bandwidth -mtriple=powerpc64le-unknown-linux -S -mcpu=pwr9 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR9
 ; REQUIRES: asserts
 
 @a = global [1024 x i8] zeroinitializer, align 16
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
index ea8df1669288..d41caca97e1f 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -31,15 +31,10 @@ define void @block_with_dead_inst_1(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP10]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
-; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> splat (i1 true))
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
@@ -121,15 +116,10 @@ define void @block_with_dead_inst_2(ptr %src) #0 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 0, i32 [[TMP16]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> splat (i1 true))
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
@@ -211,15 +201,10 @@ define void @multiple_blocks_with_dead_insts_3(ptr %src) #0 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 0, i32 [[TMP16]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> splat (i1 true))
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
@@ -311,15 +296,10 @@ define void @multiple_blocks_with_dead_insts_4(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP10]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
-; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> splat (i1 true))
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
@@ -413,15 +393,10 @@ define void @multiple_blocks_with_dead_inst_multiple_successors_5(ptr %src) #0 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 1, i32 [[TMP16]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> splat (i1 true))
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
@@ -523,15 +498,10 @@ define void @multiple_blocks_with_dead_inst_multiple_successors_6(ptr %src, i1 %
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP10]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
-; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> splat (i1 true))
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
@@ -621,14 +591,9 @@ define void @empty_block_with_phi_1(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 [[TMP8]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i16, ptr [[TMP10]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP11]], align 2
@@ -705,14 +670,9 @@ define void @empty_block_with_phi_2(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 [[TMP8]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i16, ptr [[TMP10]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP11]], align 2
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
index 1d12f11b20e1..d34098545716 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
@@ -6,9 +6,9 @@ target triple = "riscv64-unknown-linux-gnu"
 
 ; Make sure we do not pick <vscale x 1 x i64> as VF for a loop with a
 ; first-order recurrence.
-define i64 @pr97452_scalable_vf1_for(ptr %src) #0 {
+define i64 @pr97452_scalable_vf1_for(ptr %src, ptr noalias %dst) #0 {
 ; CHECK-LABEL: define i64 @pr97452_scalable_vf1_for(
-; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -17,8 +17,17 @@ define i64 @pr97452_scalable_vf1_for(ptr %src) #0 {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD1:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD1]] = load <4 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> [[WIDE_LOAD1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 4
+; CHECK-NEXT:    store <4 x i64> [[TMP3]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -31,15 +40,17 @@ define i64 @pr97452_scalable_vf1_for(ptr %src) #0 {
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[FOR]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 22
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[SCALAR_RECUR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -49,8 +60,10 @@ loop:
   %for = phi i64 [ 0, %entry ], [ %l, %loop ]
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
   %iv.next = add i64 %iv, 1
-  %gep = getelementptr inbounds i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep, align 8
+  %gep.src = getelementptr inbounds i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep.src, align 8
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv
+  store i64 %for, ptr %gep.dst
   %ec = icmp eq i64 %iv, 22
   br i1 %ec, label %exit, label %loop
 
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index f48691bd5441..61a3e3561ad9 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -9,7 +9,7 @@ define void @load_store_factor2_i32(ptr %p) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -17,86 +17,86 @@ define void @load_store_factor2_i32(ptr %p) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 1)
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 2)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]])
-; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], splat (i32 1)
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]])
+; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
 ; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q2]], align 4
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; CHECK-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
 ; CHECK-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
-; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor2_i32(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[Q0]], align 4
+; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 1
+; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; FIXED-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; FIXED-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
-; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
-; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; FIXED-NEXT:    [[TMP2:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
+; FIXED-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
+; FIXED-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; FIXED-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP1:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
-; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
 ; FIXED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q2]], align 4
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; FIXED-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
 ; FIXED-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
-; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -105,7 +105,7 @@ define void @load_store_factor2_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -113,43 +113,43 @@ define void @load_store_factor2_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 1)
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 2)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]])
-; SCALABLE-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
-; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], splat (i32 1)
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]])
+; SCALABLE-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q2]], align 4
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
 ; SCALABLE-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
-; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -183,7 +183,7 @@ define void @load_store_factor2_i64(ptr %p) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -191,86 +191,86 @@ define void @load_store_factor2_i64(ptr %p) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 1)
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 2)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]])
-; CHECK-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]])
+; CHECK-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; CHECK-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q2]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
 ; CHECK-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
-; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor2_i64(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 1
+; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; FIXED-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
-; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; FIXED-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; FIXED-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; FIXED-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; FIXED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP1:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; FIXED-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q2]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
 ; FIXED-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
-; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -279,7 +279,7 @@ define void @load_store_factor2_i64(ptr %p) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -287,43 +287,43 @@ define void @load_store_factor2_i64(ptr %p) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 2)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]])
-; SCALABLE-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]])
+; SCALABLE-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q2]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
 ; SCALABLE-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
-; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -354,41 +354,49 @@ exit:
 define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[Q0]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; CHECK-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP7]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
-; CHECK-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP8]], splat (i32 1)
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 3)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP13]])
+; CHECK-NEXT:    store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
-; CHECK-NEXT:    [[Q3:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET3]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q3]], align 4
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
 ; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q3]], align 4
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
+; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; CHECK-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
@@ -398,49 +406,49 @@ define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-NEXT:    [[X2:%.*]] = load i32, ptr [[Q2]], align 4
 ; CHECK-NEXT:    [[Y2:%.*]] = add i32 [[X2]], 3
 ; CHECK-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
-; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor3_i32(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[Q0]], align 4
+; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP1]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
 ; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; FIXED-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; FIXED-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
-; FIXED-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP8]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
-; FIXED-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
-; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
-; FIXED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; FIXED-NEXT:    [[TMP2:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
+; FIXED-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
+; FIXED-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; FIXED-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP7]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; FIXED-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP1:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; FIXED-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
-; FIXED-NEXT:    [[Q3:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET3]]
-; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q3]], align 4
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
 ; FIXED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q3]], align 4
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
+; FIXED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; FIXED-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
@@ -450,49 +458,57 @@ define void @load_store_factor3_i32(ptr %p) {
 ; FIXED-NEXT:    [[X2:%.*]] = load i32, ptr [[Q2]], align 4
 ; FIXED-NEXT:    [[Y2:%.*]] = add i32 [[X2]], 3
 ; FIXED-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
-; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
 ; SCALABLE-LABEL: @load_store_factor3_i32(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[Q0]], align 4
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; SCALABLE-NEXT:    [[TMP2:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
-; SCALABLE-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
-; SCALABLE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SCALABLE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP7]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
-; SCALABLE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[Q0]], align 4
-; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
-; SCALABLE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP1]], align 4
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP8]], splat (i32 1)
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 3)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP13]])
+; SCALABLE-NEXT:    store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; SCALABLE-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
-; SCALABLE-NEXT:    [[Q3:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET3]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q3]], align 4
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q3]], align 4
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
+; SCALABLE-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i32 [[X1]], 2
@@ -502,9 +518,9 @@ define void @load_store_factor3_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[X2:%.*]] = load i32, ptr [[Q2]], align 4
 ; SCALABLE-NEXT:    [[Y2:%.*]] = add i32 [[X2]], 3
 ; SCALABLE-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
-; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -541,41 +557,49 @@ exit:
 define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[Q0]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> [[TMP6]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP7]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 3)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]])
+; CHECK-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
-; CHECK-NEXT:    [[Q3:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET3]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q3]], align 8
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; CHECK-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q3]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
+; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -585,49 +609,49 @@ define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-NEXT:    [[X2:%.*]] = load i64, ptr [[Q2]], align 8
 ; CHECK-NEXT:    [[Y2:%.*]] = add i64 [[X2]], 3
 ; CHECK-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
-; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor3_i64(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
 ; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; FIXED-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> [[TMP7]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP8]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; FIXED-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
-; FIXED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; FIXED-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
+; FIXED-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; FIXED-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> [[TMP6]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP7]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; FIXED-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; FIXED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP1:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; FIXED-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
-; FIXED-NEXT:    [[Q3:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET3]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q3]], align 8
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; FIXED-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q3]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
+; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -637,49 +661,57 @@ define void @load_store_factor3_i64(ptr %p) {
 ; FIXED-NEXT:    [[X2:%.*]] = load i64, ptr [[Q2]], align 8
 ; FIXED-NEXT:    [[Y2:%.*]] = add i64 [[X2]], 3
 ; FIXED-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
-; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
 ; SCALABLE-LABEL: @load_store_factor3_i64(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 2
+; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; SCALABLE-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SCALABLE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> [[TMP6]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP7]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; SCALABLE-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 4
-; SCALABLE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 3)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]])
+; SCALABLE-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; SCALABLE-NEXT:    [[OFFSET3:%.*]] = mul i64 [[I1]], 3
-; SCALABLE-NEXT:    [[Q3:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET3]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q3]], align 8
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q3]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET3]], 1
+; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -689,9 +721,9 @@ define void @load_store_factor3_i64(ptr %p) {
 ; SCALABLE-NEXT:    [[X2:%.*]] = load i64, ptr [[Q2]], align 8
 ; SCALABLE-NEXT:    [[Y2:%.*]] = add i64 [[X2]], 3
 ; SCALABLE-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
-; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -745,22 +777,16 @@ define void @load_store_factor4(ptr %p) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i64>, <vscale x 4 x i64> } @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP8]])
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP9]])
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 1
-; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 3
 ; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 1)
 ; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP11]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[TMP12]], splat (i64 3)
 ; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[TMP13]], splat (i64 4)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP16]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP17]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> [[INTERLEAVED_VEC]], <vscale x 4 x i64> [[INTERLEAVED_VEC3]])
+; CHECK-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> [[TMP17]])
 ; CHECK-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC4]], ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -877,22 +903,16 @@ define void @load_store_factor4(ptr %p) {
 ; SCALABLE-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 4
 ; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
 ; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i64>, <vscale x 4 x i64> } @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP8]])
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP9]])
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 0
-; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 0
-; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 1
-; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 1
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 3
 ; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 1)
 ; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP11]], splat (i64 2)
 ; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[TMP12]], splat (i64 3)
 ; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[TMP13]], splat (i64 4)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP16]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP17]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> [[INTERLEAVED_VEC]], <vscale x 4 x i64> [[INTERLEAVED_VEC3]])
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> [[TMP17]])
 ; SCALABLE-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC4]], ptr [[TMP7]], align 8
 ; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; SCALABLE-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -970,38 +990,41 @@ exit:
 define void @load_store_factor5(ptr %p) {
 ; CHECK-LABEL: @load_store_factor5(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 5
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <10 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 0, i32 5>
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 1, i32 6>
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 2, i32 7>
-; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 3, i32 8>
-; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 4, i32 9>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> [[TMP10]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <10 x i64> [[TMP11]], <10 x i64> poison, <10 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 1, i32 3, i32 5, i32 7, i32 9>
-; CHECK-NEXT:    store <10 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 5 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave5.nxv5i64(<vscale x 5 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
+; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.vector.interleave5.nxv5i64(<vscale x 1 x i64> [[TMP10]], <vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]])
+; CHECK-NEXT:    store <vscale x 5 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -1106,38 +1129,41 @@ define void @load_store_factor5(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor5(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
+; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 5
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <10 x i64>, ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 0, i32 5>
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 1, i32 6>
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 2, i32 7>
-; SCALABLE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 3, i32 8>
-; SCALABLE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 4, i32 9>
-; SCALABLE-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SCALABLE-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> [[TMP10]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <10 x i64> [[TMP11]], <10 x i64> poison, <10 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 1, i32 3, i32 5, i32 7, i32 9>
-; SCALABLE-NEXT:    store <10 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 5 x i64>, ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave5.nxv5i64(<vscale x 5 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.vector.interleave5.nxv5i64(<vscale x 1 x i64> [[TMP10]], <vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]])
+; SCALABLE-NEXT:    store <vscale x 5 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; SCALABLE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
 ; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -1217,41 +1243,43 @@ exit:
 define void @load_store_factor6(ptr %p) {
 ; CHECK-LABEL: @load_store_factor6(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 6
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 5, i32 11>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; CHECK-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11>
-; CHECK-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave6.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
+; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave6.nxv6i64(<vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]])
+; CHECK-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -1369,41 +1397,43 @@ define void @load_store_factor6(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor6(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
+; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 6
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 0, i32 6>
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 1, i32 7>
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 2, i32 8>
-; SCALABLE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 3, i32 9>
-; SCALABLE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 4, i32 10>
-; SCALABLE-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 5, i32 11>
-; SCALABLE-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; SCALABLE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SCALABLE-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11>
-; SCALABLE-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave6.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
+; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave6.nxv6i64(<vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]])
+; SCALABLE-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
 ; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -1494,45 +1524,45 @@ exit:
 define void @load_store_factor7(ptr %p) {
 ; CHECK-LABEL: @load_store_factor7(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[INDEX]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP3]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <14 x i64>, ptr [[TMP4]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 0, i32 7>
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 1, i32 8>
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 2, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 3, i32 10>
-; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 4, i32 11>
-; CHECK-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 5, i32 12>
-; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 6, i32 13>
-; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; CHECK-NEXT:    [[TMP18:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; CHECK-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; CHECK-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; CHECK-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP19]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP11]], <4 x i64> [[TMP13]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <6 x i64> [[TMP14]], <6 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <8 x i64> [[TMP12]], <8 x i64> [[TMP15]], <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <14 x i64> [[TMP16]], <14 x i64> poison, <14 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13>
-; CHECK-NEXT:    store <14 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 7 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave7.nxv7i64(<vscale x 7 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
+; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
+; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
+; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
+; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 7)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.vector.interleave7.nxv7i64(<vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]])
+; CHECK-NEXT:    store <vscale x 7 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -1664,45 +1694,45 @@ define void @load_store_factor7(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor7(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
+; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[INDEX]], 7
-; SCALABLE-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP3]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <14 x i64>, ptr [[TMP4]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 0, i32 7>
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 1, i32 8>
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 2, i32 9>
-; SCALABLE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 3, i32 10>
-; SCALABLE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 4, i32 11>
-; SCALABLE-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 5, i32 12>
-; SCALABLE-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 6, i32 13>
-; SCALABLE-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP18:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; SCALABLE-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
-; SCALABLE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP19]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SCALABLE-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP11]], <4 x i64> [[TMP13]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; SCALABLE-NEXT:    [[TMP15:%.*]] = shufflevector <6 x i64> [[TMP14]], <6 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP16:%.*]] = shufflevector <8 x i64> [[TMP12]], <8 x i64> [[TMP15]], <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <14 x i64> [[TMP16]], <14 x i64> poison, <14 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13>
-; SCALABLE-NEXT:    store <14 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 7
+; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 7 x i64>, ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave7.nxv7i64(<vscale x 7 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
+; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
+; SCALABLE-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 7)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.vector.interleave7.nxv7i64(<vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]])
+; SCALABLE-NEXT:    store <vscale x 7 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; SCALABLE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
 ; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -1806,72 +1836,54 @@ define void @load_store_factor8(ptr %p) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[Q0]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i64>, <vscale x 4 x i64> } @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP5]])
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP6]])
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 1
-; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP7]])
-; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP8]])
-; CHECK-NEXT:    [[STRIDED_VEC5:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP9]])
-; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP10]])
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 0
-; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 0
-; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 1
-; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 1
-; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 1
-; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 1
-; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 1)
-; CHECK-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 2)
-; CHECK-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 3)
-; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 4)
-; CHECK-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 5)
-; CHECK-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 6)
-; CHECK-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 7)
-; CHECK-NEXT:    [[TMP27:%.*]] = add <vscale x 1 x i64> [[TMP19]], splat (i64 8)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP24]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC7:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP25]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC8:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP26]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC9:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP27]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC]], <vscale x 2 x i64> [[INTERLEAVED_VEC8]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC7]], <vscale x 2 x i64> [[INTERLEAVED_VEC9]])
-; CHECK-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> [[INTERLEAVED_VEC10]], <vscale x 4 x i64> [[INTERLEAVED_VEC11]])
-; CHECK-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[Q0]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP2]]
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP3]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave8.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 7
+; CHECK-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 1)
+; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 2)
+; CHECK-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 3)
+; CHECK-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 4)
+; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 5)
+; CHECK-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 6)
+; CHECK-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 7)
+; CHECK-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 8)
+; CHECK-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave8.nxv8i64(<vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP24]], <vscale x 1 x i64> [[TMP25]], <vscale x 1 x i64> [[TMP26]])
+; CHECK-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OFFSET8:%.*]] = shl i64 [[I1]], 3
-; CHECK-NEXT:    [[Q8:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET8]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q8]], align 8
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; CHECK-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q8]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET8]], 1
+; CHECK-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -1906,22 +1918,22 @@ define void @load_store_factor8(ptr %p) {
 ; CHECK-NEXT:    [[X7:%.*]] = load i64, ptr [[Q7]], align 8
 ; CHECK-NEXT:    [[Y7:%.*]] = add i64 [[X7]], 8
 ; CHECK-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
-; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @load_store_factor8(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 3
+; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
 ; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
@@ -1930,39 +1942,39 @@ define void @load_store_factor8(ptr %p) {
 ; FIXED-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
 ; FIXED-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
 ; FIXED-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
-; FIXED-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; FIXED-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; FIXED-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; FIXED-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; FIXED-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
-; FIXED-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC7]], splat (i64 8)
-; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i64> [[TMP9]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i64> [[TMP11]], <4 x i64> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i64> [[TMP13]], <4 x i64> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT:    [[TMP17:%.*]] = shufflevector <8 x i64> [[TMP15]], <8 x i64> [[TMP16]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP17]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[Q0]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 2
-; FIXED-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; FIXED-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
+; FIXED-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
+; FIXED-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
+; FIXED-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
+; FIXED-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
+; FIXED-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
+; FIXED-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
+; FIXED-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[STRIDED_VEC7]], splat (i64 8)
+; FIXED-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP16:%.*]] = shufflevector <8 x i64> [[TMP14]], <8 x i64> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP16]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; FIXED-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP1:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; FIXED-NEXT:    [[OFFSET8:%.*]] = shl i64 [[I1]], 3
-; FIXED-NEXT:    [[Q8:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET8]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q8]], align 8
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; FIXED-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q8]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET8]], 1
+; FIXED-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -1997,9 +2009,9 @@ define void @load_store_factor8(ptr %p) {
 ; FIXED-NEXT:    [[X7:%.*]] = load i64, ptr [[Q7]], align 8
 ; FIXED-NEXT:    [[Y7:%.*]] = add i64 [[X7]], 8
 ; FIXED-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
-; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP19:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -2007,72 +2019,54 @@ define void @load_store_factor8(ptr %p) {
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i64>, <vscale x 4 x i64> } @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i64>, <vscale x 4 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP5]])
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[TMP6]])
-; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 0
-; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 0
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC1]], 1
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC2]], 1
-; SCALABLE-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP7]])
-; SCALABLE-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP8]])
-; SCALABLE-NEXT:    [[STRIDED_VEC5:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP9]])
-; SCALABLE-NEXT:    [[STRIDED_VEC6:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave2.nxv2i64(<vscale x 2 x i64> [[TMP10]])
-; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 0
-; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 0
-; SCALABLE-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 0
-; SCALABLE-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 0
-; SCALABLE-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC3]], 1
-; SCALABLE-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC4]], 1
-; SCALABLE-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC5]], 1
-; SCALABLE-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC6]], 1
-; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 6)
-; SCALABLE-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 7)
-; SCALABLE-NEXT:    [[TMP27:%.*]] = add <vscale x 1 x i64> [[TMP19]], splat (i64 8)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP24]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC7:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP25]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC8:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP26]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC9:%.*]] = call <vscale x 2 x i64> @llvm.vector.interleave2.nxv2i64(<vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP27]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC]], <vscale x 2 x i64> [[INTERLEAVED_VEC8]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[INTERLEAVED_VEC7]], <vscale x 2 x i64> [[INTERLEAVED_VEC9]])
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> [[INTERLEAVED_VEC10]], <vscale x 4 x i64> [[INTERLEAVED_VEC11]])
-; SCALABLE-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[Q0]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP2]]
-; SCALABLE-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; SCALABLE-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP3]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP4]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave8.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; SCALABLE-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; SCALABLE-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 7
+; SCALABLE-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 5)
+; SCALABLE-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 6)
+; SCALABLE-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 7)
+; SCALABLE-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 8)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave8.nxv8i64(<vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP24]], <vscale x 1 x i64> [[TMP25]], <vscale x 1 x i64> [[TMP26]])
+; SCALABLE-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[TMP4]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; SCALABLE-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; SCALABLE-NEXT:    [[OFFSET8:%.*]] = shl i64 [[I1]], 3
-; SCALABLE-NEXT:    [[Q8:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET8]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q8]], align 8
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
 ; SCALABLE-NEXT:    [[Y0:%.*]] = add i64 [[X0]], 1
-; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q8]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET8]], 1
+; SCALABLE-NEXT:    store i64 [[Y0]], ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[Y1:%.*]] = add i64 [[X1]], 2
@@ -2107,9 +2101,9 @@ define void @load_store_factor8(ptr %p) {
 ; SCALABLE-NEXT:    [[X7:%.*]] = load i64, ptr [[Q7]], align 8
 ; SCALABLE-NEXT:    [[Y7:%.*]] = add i64 [[X7]], 8
 ; SCALABLE-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
-; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP19:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -2179,7 +2173,7 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -2187,92 +2181,92 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[I]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP11]], ptr [[TMP13]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP10]], ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[X0]], [[X1]]
-; CHECK-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I1]]
+; CHECK-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]]
 ; CHECK-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
-; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @combine_load_factor2_i32(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[TMP1:%.*]] = add i64 [[I]], 8
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP3]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[Q0]], align 4
+; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 8
+; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX]], 1
+; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0]], 1
+; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
+; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP2]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP3]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP5]], align 4
+; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP4]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
-; FIXED-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
-; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[I]]
-; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[TMP8]], i32 0
-; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP8]], i32 8
+; FIXED-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; FIXED-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
+; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]]
+; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
+; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[TMP7]], i32 8
+; FIXED-NEXT:    store <8 x i32> [[TMP5]], ptr [[TMP8]], align 4
 ; FIXED-NEXT:    store <8 x i32> [[TMP6]], ptr [[TMP9]], align 4
-; FIXED-NEXT:    store <8 x i32> [[TMP7]], ptr [[TMP10]], align 4
-; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 16
-; FIXED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; FIXED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP1:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
-; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; FIXED-NEXT:    [[RES:%.*]] = add i32 [[X0]], [[X1]]
-; FIXED-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I1]]
+; FIXED-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]]
 ; FIXED-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
-; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP21:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -2281,7 +2275,7 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
@@ -2289,42 +2283,42 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[TMP10]]
-; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[I]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0
-; SCALABLE-NEXT:    store <vscale x 4 x i32> [[TMP11]], ptr [[TMP13]], align 4
-; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[TMP9]]
+; SCALABLE-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]]
+; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0
+; SCALABLE-NEXT:    store <vscale x 4 x i32> [[TMP10]], ptr [[TMP12]], align 4
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q2]], align 4
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i32, ptr [[Q1]], align 4
 ; SCALABLE-NEXT:    [[RES:%.*]] = add i32 [[X0]], [[X1]]
-; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I1]]
+; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]]
 ; SCALABLE-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
-; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP21:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -2359,7 +2353,7 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -2367,92 +2361,92 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[TMP12]], i32 0
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], align 8
-; CHECK-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; CHECK-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
-; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
-; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; CHECK-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; CHECK-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; CHECK-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I1]]
+; CHECK-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
 ; CHECK-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
-; CHECK-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 ; FIXED-LABEL: @combine_load_factor2_i64(
 ; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[LOOP:%.*]]
+; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; FIXED-NEXT:    [[TMP1:%.*]] = add i64 [[I]], 4
-; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; FIXED-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; FIXED-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX]], 1
+; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0]], 1
+; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x i64>, ptr [[TMP5]], align 8
+; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
-; FIXED-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
-; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]]
-; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0
-; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i32 4
+; FIXED-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
+; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[INDEX]]
+; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 0
+; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[TMP7]], i32 4
+; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP8]], align 8
 ; FIXED-NEXT:    store <4 x i64> [[TMP6]], ptr [[TMP9]], align 8
-; FIXED-NEXT:    store <4 x i64> [[TMP7]], ptr [[TMP10]], align 8
-; FIXED-NEXT:    [[NEXTI]] = add nuw i64 [[I]], 8
-; FIXED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; FIXED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; FIXED-NEXT:    br label [[LOOP1:%.*]]
+; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; FIXED-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; FIXED-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
-; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
-; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; FIXED-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; FIXED-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; FIXED-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; FIXED-NEXT:    [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; FIXED-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I1]]
+; FIXED-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
 ; FIXED-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
-; FIXED-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP23:![0-9]+]]
+; FIXED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; FIXED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; FIXED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
 ; FIXED:       exit:
 ; FIXED-NEXT:    ret void
 ;
@@ -2461,7 +2455,7 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
@@ -2469,42 +2463,42 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; SCALABLE-NEXT:    br label [[LOOP:%.*]]
+; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
-; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
 ; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], [[TMP10]]
-; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[TMP12]], i32 0
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], align 8
-; SCALABLE-NEXT:    [[NEXTI]] = add nuw i64 [[I]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[NEXTI]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], [[TMP9]]
+; SCALABLE-NEXT:    [[TMP11:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[INDEX]]
+; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr i64, ptr [[TMP11]], i32 0
+; SCALABLE-NEXT:    store <vscale x 2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; SCALABLE:       middle.block:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
-; SCALABLE-NEXT:    br label [[LOOP1:%.*]]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI1:%.*]], [[LOOP1]] ]
-; SCALABLE-NEXT:    [[OFFSET2:%.*]] = shl i64 [[I1]], 1
-; SCALABLE-NEXT:    [[Q2:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET2]]
-; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q2]], align 8
-; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET2]], 1
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
+; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
+; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
+; SCALABLE-NEXT:    [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
 ; SCALABLE-NEXT:    [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
 ; SCALABLE-NEXT:    [[X1:%.*]] = load i64, ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[RES:%.*]] = add i64 [[X0]], [[X1]]
-; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I1]]
+; SCALABLE-NEXT:    [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
 ; SCALABLE-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
-; SCALABLE-NEXT:    [[NEXTI1]] = add i64 [[I1]], 1
-; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI1]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP23:![0-9]+]]
+; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
new file mode 100644
index 000000000000..b7c9612e57ae
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -0,0 +1,408 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 2
+; RUN: opt -mtriple=riscv64-none-linux-gnu -S -passes=loop-vectorize,instcombine -mattr=+v -prefer-predicate-over-epilogue=scalar-epilogue %s 2>&1 | FileCheck %s -check-prefix=SCALAR_EPILOGUE
+; RUN: opt -mtriple=riscv64-none-linux-gnu -S -passes=loop-vectorize,instcombine -mattr=+v -prefer-predicate-over-epilogue=predicate-dont-vectorize %s 2>&1 | FileCheck %s -check-prefix=PREDICATED_TAIL_FOLDING
+; RUN: opt -mtriple=riscv64-none-linux-gnu -S -passes=loop-vectorize,instcombine -mattr=+v -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-tail-folding-style=data-with-evl %s 2>&1 | FileCheck %s -check-prefix=PREDICATED_EVL
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) {
+; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor2
+; SCALAR_EPILOGUE-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
+; SCALAR_EPILOGUE-NEXT:  entry:
+; SCALAR_EPILOGUE-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; SCALAR_EPILOGUE-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALAR_EPILOGUE:       vector.ph:
+; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; SCALAR_EPILOGUE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[DOTSPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALAR_EPILOGUE:       vector.body:
+; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP9]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP10]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP12]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER1]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP15]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP14]], <vscale x 16 x ptr> [[TMP16]], i32 1, <vscale x 16 x i1> [[TMP7]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP14]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP18]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x ptr> [[TMP19]], i32 1, <vscale x 16 x i1> [[TMP7]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALAR_EPILOGUE:       middle.block:
+; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
+; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; SCALAR_EPILOGUE:       scalar.ph:
+;
+; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided_factor2
+; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
+; PREDICATED_TAIL_FOLDING-NEXT:  entry:
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PREDICATED_TAIL_FOLDING:       vector.ph:
+; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
+; PREDICATED_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[DOTSPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PREDICATED_TAIL_FOLDING:       vector.body:
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP8]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP9]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP12]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER1]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP14]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP13]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP17]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP16]], <vscale x 16 x ptr> [[TMP18]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PREDICATED_TAIL_FOLDING:       middle.block:
+; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_END:%.*]]
+; PREDICATED_TAIL_FOLDING:       scalar.ph:
+;
+; PREDICATED_EVL-LABEL: define void @masked_strided_factor2
+; PREDICATED_EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
+; PREDICATED_EVL-NEXT:  entry:
+; PREDICATED_EVL-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; PREDICATED_EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; PREDICATED_EVL:       for.body:
+; PREDICATED_EVL-NEXT:    [[IX_024:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; PREDICATED_EVL-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_024]], [[CONV]]
+; PREDICATED_EVL-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; PREDICATED_EVL:       if.then:
+; PREDICATED_EVL-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IX_024]], 1
+; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[MUL]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
+; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; PREDICATED_EVL-NEXT:    [[ADD:%.*]] = or disjoint i32 [[MUL]], 1
+; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[ADD]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
+; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP1]], i8 [[TMP3]])
+; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = zext nneg i32 [[MUL]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP4]]
+; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I]], ptr [[ARRAYIDX6]], align 1
+; PREDICATED_EVL-NEXT:    [[SUB:%.*]] = sub i8 0, [[SPEC_SELECT_I]]
+; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[ADD]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP5]]
+; PREDICATED_EVL-NEXT:    store i8 [[SUB]], ptr [[ARRAYIDX11]], align 1
+; PREDICATED_EVL-NEXT:    br label [[FOR_INC]]
+; PREDICATED_EVL:       for.inc:
+; PREDICATED_EVL-NEXT:    [[INC]] = add nuw nsw i32 [[IX_024]], 1
+; PREDICATED_EVL-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
+; PREDICATED_EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; PREDICATED_EVL:       for.end:
+; PREDICATED_EVL-NEXT:    ret void
+;
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.024 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.024, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.024, 1
+  %arrayidx = getelementptr inbounds i8, ptr %p, i32 %mul
+  %0 = load i8, ptr %arrayidx, align 1
+  %add = or disjoint i32 %mul, 1
+  %arrayidx4 = getelementptr inbounds i8, ptr %p, i32 %add
+  %1 = load i8, ptr %arrayidx4, align 1
+  %cmp.i = icmp slt i8 %0, %1
+  %spec.select.i = select i1 %cmp.i, i8 %1, i8 %0
+  %arrayidx6 = getelementptr inbounds i8, ptr %q, i32 %mul
+  store i8 %spec.select.i, ptr %arrayidx6, align 1
+  %sub = sub i8 0, %spec.select.i
+  %arrayidx11 = getelementptr inbounds i8, ptr %q, i32 %add
+  store i8 %sub, ptr %arrayidx11, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.024, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+
+define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) {
+; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor4
+; SCALAR_EPILOGUE-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
+; SCALAR_EPILOGUE-NEXT:  entry:
+; SCALAR_EPILOGUE-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; SCALAR_EPILOGUE-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALAR_EPILOGUE:       vector.ph:
+; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; SCALAR_EPILOGUE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[DOTSPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SCALAR_EPILOGUE:       vector.body:
+; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 2)
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 3)
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP12]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP14]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP16]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP17]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP18]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP19]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER1]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP21:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP20]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP22:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER2]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP23:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP22]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP24:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP24]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> [[TMP25]], i32 1, <vscale x 16 x i1> [[TMP7]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP26:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP26]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> [[TMP27]], i32 1, <vscale x 16 x i1> [[TMP7]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP28:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP28]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> [[TMP29]], i32 1, <vscale x 16 x i1> [[TMP7]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP30:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP30]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP23]], <vscale x 16 x ptr> [[TMP31]], i32 1, <vscale x 16 x i1> [[TMP7]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALAR_EPILOGUE:       middle.block:
+; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
+; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; SCALAR_EPILOGUE:       scalar.ph:
+;
+; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided_factor4
+; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
+; PREDICATED_TAIL_FOLDING-NEXT:  entry:
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PREDICATED_TAIL_FOLDING:       vector.ph:
+; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
+; PREDICATED_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[DOTSPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PREDICATED_TAIL_FOLDING:       vector.body:
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 2)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 3)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP12]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP13]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP14]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP15]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP16]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP17]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP18]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER1]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER2]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP22:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP21]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP23:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP23]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP19]], <vscale x 16 x ptr> [[TMP24]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP25:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP25]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> [[TMP26]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP27:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP27]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> [[TMP28]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP29:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP29]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> [[TMP30]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; PREDICATED_TAIL_FOLDING:       middle.block:
+; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_END:%.*]]
+; PREDICATED_TAIL_FOLDING:       scalar.ph:
+;
+; PREDICATED_EVL-LABEL: define void @masked_strided_factor4
+; PREDICATED_EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
+; PREDICATED_EVL-NEXT:  entry:
+; PREDICATED_EVL-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
+; PREDICATED_EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; PREDICATED_EVL:       for.body:
+; PREDICATED_EVL-NEXT:    [[IX_024:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; PREDICATED_EVL-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_024]], [[CONV]]
+; PREDICATED_EVL-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; PREDICATED_EVL:       if.then:
+; PREDICATED_EVL-NEXT:    [[IDX0:%.*]] = shl nuw nsw i32 [[IX_024]], 2
+; PREDICATED_EVL-NEXT:    [[IDX1:%.*]] = or disjoint i32 [[IDX0]], 1
+; PREDICATED_EVL-NEXT:    [[IDX2:%.*]] = or disjoint i32 [[IDX0]], 2
+; PREDICATED_EVL-NEXT:    [[IDX3:%.*]] = or disjoint i32 [[IDX0]], 3
+; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[IDX0]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY1IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
+; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAY1IDX0]], align 1
+; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[IDX1]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY1IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
+; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAY1IDX1]], align 1
+; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = zext nneg i32 [[IDX2]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY1IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP4]]
+; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = load i8, ptr [[ARRAY1IDX2]], align 1
+; PREDICATED_EVL-NEXT:    [[TMP6:%.*]] = zext nneg i32 [[IDX3]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY1IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP6]]
+; PREDICATED_EVL-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAY1IDX3]], align 1
+; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I1:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP1]], i8 [[TMP3]])
+; PREDICATED_EVL-NEXT:    [[SUB1:%.*]] = sub i8 0, [[SPEC_SELECT_I1]]
+; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I2:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP5]], i8 [[TMP7]])
+; PREDICATED_EVL-NEXT:    [[SUB2:%.*]] = sub i8 0, [[SPEC_SELECT_I2]]
+; PREDICATED_EVL-NEXT:    [[TMP8:%.*]] = zext nneg i32 [[IDX0]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY3IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP8]]
+; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I1]], ptr [[ARRAY3IDX0]], align 1
+; PREDICATED_EVL-NEXT:    [[TMP9:%.*]] = zext nneg i32 [[IDX1]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY3IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP9]]
+; PREDICATED_EVL-NEXT:    store i8 [[SUB1]], ptr [[ARRAY3IDX1]], align 1
+; PREDICATED_EVL-NEXT:    [[TMP10:%.*]] = zext nneg i32 [[IDX2]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY3IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP10]]
+; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I2]], ptr [[ARRAY3IDX2]], align 1
+; PREDICATED_EVL-NEXT:    [[TMP11:%.*]] = zext nneg i32 [[IDX3]] to i64
+; PREDICATED_EVL-NEXT:    [[ARRAY3IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP11]]
+; PREDICATED_EVL-NEXT:    store i8 [[SUB2]], ptr [[ARRAY3IDX3]], align 1
+; PREDICATED_EVL-NEXT:    br label [[FOR_INC]]
+; PREDICATED_EVL:       for.inc:
+; PREDICATED_EVL-NEXT:    [[INC]] = add nuw nsw i32 [[IX_024]], 1
+; PREDICATED_EVL-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
+; PREDICATED_EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; PREDICATED_EVL:       for.end:
+; PREDICATED_EVL-NEXT:    ret void
+;
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.024 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.024, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %idx0 = shl nuw nsw i32 %ix.024, 2
+  %idx1 = add i32 %idx0, 1
+  %idx2 = add i32 %idx0, 2
+  %idx3 = add i32 %idx0, 3
+
+  %array1idx0 = getelementptr inbounds i8, ptr %p, i32 %idx0
+  %0 = load i8, ptr %array1idx0, align 1
+  %array1idx1 = getelementptr inbounds i8, ptr %p, i32 %idx1
+  %1 = load i8, ptr %array1idx1, align 1
+  %array1idx2 = getelementptr inbounds i8, ptr %p, i32 %idx2
+  %2 = load i8, ptr %array1idx2, align 1
+  %array1idx3 = getelementptr inbounds i8, ptr %p, i32 %idx3
+  %3 = load i8, ptr %array1idx3, align 1
+
+  %cmp.i1 = icmp slt i8 %0, %1
+  %spec.select.i1 = select i1 %cmp.i1, i8 %1, i8 %0
+  %sub1 = sub i8 0, %spec.select.i1
+  %cmp.i2 = icmp slt i8 %2, %3
+  %spec.select.i2 = select i1 %cmp.i2, i8 %3, i8 %2
+  %sub2 = sub i8 0, %spec.select.i2
+
+  %array3idx0 = getelementptr inbounds i8, ptr %q, i32 %idx0
+  store i8 %spec.select.i1, ptr %array3idx0, align 1
+  %array3idx1 = getelementptr inbounds i8, ptr %q, i32 %idx1
+  store i8 %sub1, ptr %array3idx1, align 1
+  %array3idx2 = getelementptr inbounds i8, ptr %q, i32 %idx2
+  store i8 %spec.select.i2, ptr %array3idx2, align 1
+  %array3idx3 = getelementptr inbounds i8, ptr %q, i32 %idx3
+  store i8 %sub2, ptr %array3idx3, align 1
+
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.024, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index 847c4ba0bebf..8c29da02b813 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -351,79 +351,153 @@ for.exit:                        ; preds = %for.body
 
 
 define i32 @vqdotsu(ptr %a, ptr %b) #0 {
-; CHECK-LABEL: define i32 @vqdotsu(
-; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
-; CHECK-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; CHECK-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
+; V-LABEL: define i32 @vqdotsu(
+; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; V-NEXT:  entry:
+; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; V-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; V:       vector.ph:
+; V-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; V-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; V-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; V-NEXT:    br label [[VECTOR_BODY:%.*]]
+; V:       vector.body:
+; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; V-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; V-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
+; V-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; V-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; V-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
+; V-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; V-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; V-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
+; V-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
+; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; V:       middle.block:
+; V-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; V-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; V-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; V:       scalar.ph:
+;
+; ZVQDOTQ-LABEL: define i32 @vqdotsu(
+; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; ZVQDOTQ-NEXT:  entry:
+; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; ZVQDOTQ:       vector.ph:
+; ZVQDOTQ-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
+; ZVQDOTQ:       vector.body:
+; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ZVQDOTQ-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; ZVQDOTQ-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; ZVQDOTQ-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; ZVQDOTQ-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
+; ZVQDOTQ-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; ZVQDOTQ-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; ZVQDOTQ-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
+; ZVQDOTQ-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; ZVQDOTQ-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; ZVQDOTQ-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
+; ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.experimental.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
+; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; ZVQDOTQ-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; ZVQDOTQ-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; ZVQDOTQ:       middle.block:
+; ZVQDOTQ-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; ZVQDOTQ-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; ZVQDOTQ-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; ZVQDOTQ:       scalar.ph:
 ;
-; FIXED-LABEL: define i32 @vqdotsu(
-; FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
-; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 8
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; FIXED-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; FIXED-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
-; FIXED-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
-; FIXED-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i32 0
-; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP5]], i32 8
-; FIXED-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1
-; FIXED-NEXT:    [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
-; FIXED-NEXT:    [[TMP8:%.*]] = sext <8 x i8> [[WIDE_LOAD3]] to <8 x i32>
-; FIXED-NEXT:    [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-NEXT:    [[TMP10:%.*]] = mul <8 x i32> [[TMP8]], [[TMP3]]
-; FIXED-NEXT:    [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
-; FIXED-NEXT:    [[TMP12]] = add <8 x i32> [[TMP10]], [[VEC_PHI]]
-; FIXED-NEXT:    [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; FIXED:       middle.block:
-; FIXED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP13]], [[TMP12]]
-; FIXED-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
-; FIXED-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; FIXED:       scalar.ph:
+; FIXED-V-LABEL: define i32 @vqdotsu(
+; FIXED-V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; FIXED-V-NEXT:  entry:
+; FIXED-V-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-V:       vector.ph:
+; FIXED-V-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-V:       vector.body:
+; FIXED-V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; FIXED-V-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; FIXED-V-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 8
+; FIXED-V-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; FIXED-V-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; FIXED-V-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; FIXED-V-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i32 0
+; FIXED-V-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP5]], i32 8
+; FIXED-V-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1
+; FIXED-V-NEXT:    [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
+; FIXED-V-NEXT:    [[TMP8:%.*]] = sext <8 x i8> [[WIDE_LOAD3]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP10:%.*]] = mul <8 x i32> [[TMP8]], [[TMP3]]
+; FIXED-V-NEXT:    [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
+; FIXED-V-NEXT:    [[TMP12]] = add <8 x i32> [[TMP10]], [[VEC_PHI]]
+; FIXED-V-NEXT:    [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
+; FIXED-V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; FIXED-V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; FIXED-V:       middle.block:
+; FIXED-V-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP13]], [[TMP12]]
+; FIXED-V-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; FIXED-V-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; FIXED-V:       scalar.ph:
+;
+; FIXED-ZVQDOTQ-LABEL: define i32 @vqdotsu(
+; FIXED-ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; FIXED-ZVQDOTQ-NEXT:  entry:
+; FIXED-ZVQDOTQ-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-ZVQDOTQ:       vector.ph:
+; FIXED-ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-ZVQDOTQ:       vector.body:
+; FIXED-ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVQDOTQ-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVQDOTQ-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; FIXED-ZVQDOTQ-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 8
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i32 0
+; FIXED-ZVQDOTQ-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP5]], i32 8
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[TMP8:%.*]] = sext <8 x i8> [[WIDE_LOAD3]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP10:%.*]] = mul <8 x i32> [[TMP8]], [[TMP3]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
+; FIXED-ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.experimental.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP10]])
+; FIXED-ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE5]] = call <2 x i32> @llvm.experimental.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP11]])
+; FIXED-ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; FIXED-ZVQDOTQ-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-ZVQDOTQ-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; FIXED-ZVQDOTQ:       middle.block:
+; FIXED-ZVQDOTQ-NEXT:    [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
+; FIXED-ZVQDOTQ-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; FIXED-ZVQDOTQ:       scalar.ph:
 ;
 entry:
   br label %for.body
@@ -448,79 +522,153 @@ for.exit:                        ; preds = %for.body
 }
 
 define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
-; CHECK-LABEL: define i32 @vqdotsu2(
-; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
-; CHECK-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
-; CHECK-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
-; CHECK-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
+; V-LABEL: define i32 @vqdotsu2(
+; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; V-NEXT:  entry:
+; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; V-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; V:       vector.ph:
+; V-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; V-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; V-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; V-NEXT:    br label [[VECTOR_BODY:%.*]]
+; V:       vector.body:
+; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; V-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; V-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; V-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
+; V-NEXT:    [[TMP8:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; V-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; V-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
+; V-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; V-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; V-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
+; V-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
+; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; V:       middle.block:
+; V-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; V-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; V-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; V:       scalar.ph:
+;
+; ZVQDOTQ-LABEL: define i32 @vqdotsu2(
+; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; ZVQDOTQ-NEXT:  entry:
+; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; ZVQDOTQ:       vector.ph:
+; ZVQDOTQ-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
+; ZVQDOTQ:       vector.body:
+; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ZVQDOTQ-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; ZVQDOTQ-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; ZVQDOTQ-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; ZVQDOTQ-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
+; ZVQDOTQ-NEXT:    [[TMP8:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; ZVQDOTQ-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; ZVQDOTQ-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
+; ZVQDOTQ-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; ZVQDOTQ-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; ZVQDOTQ-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
+; ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.experimental.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
+; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; ZVQDOTQ-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; ZVQDOTQ-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; ZVQDOTQ:       middle.block:
+; ZVQDOTQ-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; ZVQDOTQ-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; ZVQDOTQ-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; ZVQDOTQ:       scalar.ph:
 ;
-; FIXED-LABEL: define i32 @vqdotsu2(
-; FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT:  entry:
-; FIXED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; FIXED:       vector.ph:
-; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; FIXED:       vector.body:
-; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
-; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 8
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; FIXED-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; FIXED-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
-; FIXED-NEXT:    [[TMP4:%.*]] = sext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
-; FIXED-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i32 0
-; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP5]], i32 8
-; FIXED-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1
-; FIXED-NEXT:    [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
-; FIXED-NEXT:    [[TMP8:%.*]] = zext <8 x i8> [[WIDE_LOAD3]] to <8 x i32>
-; FIXED-NEXT:    [[TMP9:%.*]] = zext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
-; FIXED-NEXT:    [[TMP10:%.*]] = mul <8 x i32> [[TMP8]], [[TMP3]]
-; FIXED-NEXT:    [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
-; FIXED-NEXT:    [[TMP12]] = add <8 x i32> [[TMP10]], [[VEC_PHI]]
-; FIXED-NEXT:    [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; FIXED:       middle.block:
-; FIXED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP13]], [[TMP12]]
-; FIXED-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
-; FIXED-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; FIXED:       scalar.ph:
+; FIXED-V-LABEL: define i32 @vqdotsu2(
+; FIXED-V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; FIXED-V-NEXT:  entry:
+; FIXED-V-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-V:       vector.ph:
+; FIXED-V-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-V:       vector.body:
+; FIXED-V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; FIXED-V-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; FIXED-V-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; FIXED-V-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 8
+; FIXED-V-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; FIXED-V-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; FIXED-V-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP4:%.*]] = sext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; FIXED-V-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i32 0
+; FIXED-V-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP5]], i32 8
+; FIXED-V-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1
+; FIXED-V-NEXT:    [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
+; FIXED-V-NEXT:    [[TMP8:%.*]] = zext <8 x i8> [[WIDE_LOAD3]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP9:%.*]] = zext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
+; FIXED-V-NEXT:    [[TMP10:%.*]] = mul <8 x i32> [[TMP8]], [[TMP3]]
+; FIXED-V-NEXT:    [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
+; FIXED-V-NEXT:    [[TMP12]] = add <8 x i32> [[TMP10]], [[VEC_PHI]]
+; FIXED-V-NEXT:    [[TMP13]] = add <8 x i32> [[TMP11]], [[VEC_PHI1]]
+; FIXED-V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; FIXED-V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; FIXED-V:       middle.block:
+; FIXED-V-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP13]], [[TMP12]]
+; FIXED-V-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; FIXED-V-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; FIXED-V:       scalar.ph:
+;
+; FIXED-ZVQDOTQ-LABEL: define i32 @vqdotsu2(
+; FIXED-ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; FIXED-ZVQDOTQ-NEXT:  entry:
+; FIXED-ZVQDOTQ-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; FIXED-ZVQDOTQ:       vector.ph:
+; FIXED-ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FIXED-ZVQDOTQ:       vector.body:
+; FIXED-ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVQDOTQ-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVQDOTQ-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; FIXED-ZVQDOTQ-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 8
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP4:%.*]] = sext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i32 0
+; FIXED-ZVQDOTQ-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP5]], i32 8
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
+; FIXED-ZVQDOTQ-NEXT:    [[TMP8:%.*]] = zext <8 x i8> [[WIDE_LOAD3]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP9:%.*]] = zext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
+; FIXED-ZVQDOTQ-NEXT:    [[TMP10:%.*]] = mul <8 x i32> [[TMP8]], [[TMP3]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP11:%.*]] = mul <8 x i32> [[TMP9]], [[TMP4]]
+; FIXED-ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.experimental.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP10]])
+; FIXED-ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE5]] = call <2 x i32> @llvm.experimental.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP11]])
+; FIXED-ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; FIXED-ZVQDOTQ-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-ZVQDOTQ-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; FIXED-ZVQDOTQ:       middle.block:
+; FIXED-ZVQDOTQ-NEXT:    [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; FIXED-ZVQDOTQ-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
+; FIXED-ZVQDOTQ-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; FIXED-ZVQDOTQ:       scalar.ph:
 ;
 entry:
   br label %for.body
@@ -543,3 +691,6 @@ for.body:                                         ; preds = %for.body, %entry
 for.exit:                        ; preds = %for.body
   ret i32 %add
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
+; FIXED: {{.*}}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
index 8de1beea8e57..5a67b54c7a3d 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
@@ -1,10 +1,9 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -debug-only=loop-vectorize --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s
 
 define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
 ; CHECK-LABEL: add
-; CHECK:       LV(REG): VF = 8
-; CHECK-NEXT:  LV(REG): Found max usage: 2 item
+; CHECK:  LV(REG): Found max usage: 2 item
 ; CHECK-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
 ; CHECK-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 4 registers
 ; CHECK-NEXT:  LV(REG): Found invariant usage: 1 item
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
index 2005e82e9f27..d4909fa61b4f 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
@@ -1,17 +1,15 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfh -debug-only=loop-vectorize --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFH
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -debug-only=loop-vectorize --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFHMIN
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfh -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFH
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFHMIN
 
 define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
 ; CHECK-LABEL: add
-; ZVFH:       LV(REG): VF = 8
-; ZVFH-NEXT:  LV(REG): Found max usage: 2 item
+; ZVFH:  LV(REG): Found max usage: 2 item
 ; ZVFH-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
 ; ZVFH-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 2 registers
 ; ZVFH-NEXT:  LV(REG): Found invariant usage: 1 item
 ; ZVFH-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; ZVFHMIN:       LV(REG): VF = 8
-; ZVFHMIN-NEXT:  LV(REG): Found max usage: 2 item
+; ZVFHMIN:  LV(REG): Found max usage: 2 item
 ; ZVFHMIN-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
 ; ZVFHMIN-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 4 registers
 ; ZVFHMIN-NEXT:  LV(REG): Found invariant usage: 1 item
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 15665fbd9e31..cee0b1222b6b 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -1,22 +1,22 @@
 ; REQUIRES: asserts
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -force-vector-width=1 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALAR
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=1 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL1
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=2 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL2
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=4 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL4
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v,+d -debug-only=loop-vectorize --disable-output \
+; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
 ; RUN:   -riscv-v-vector-bits-min=128 -riscv-v-register-bit-width-lmul=8 \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL8
 
@@ -28,28 +28,24 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture rea
 ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::FPRRC, 2 registers
 ; CHECK-SCALAR-NEXT: LV(REG): Found invariant usage: 1 item
 ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK-LMUL1:       LV(REG): VF = 2
-; CHECK-LMUL1-NEXT:  LV(REG): Found max usage: 2 item
+; CHECK-LMUL1:       LV(REG): Found max usage: 2 item
 ; CHECK-LMUL1-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
 ; CHECK-LMUL1-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 2 registers
 ; CHECK-LMUL1-NEXT:  LV(REG): Found invariant usage: 1 item
 ; CHECK-LMUL1-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK-LMUL2:       LV(REG): VF = 4
-; CHECK-LMUL2-NEXT:  LV(REG): Found max usage: 2 item
+; CHECK-LMUL2:       LV(REG): Found max usage: 2 item
 ; CHECK-LMUL2-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
-; CHECK-LMUL2-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; CHECK-LMUL2-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 4 registers
 ; CHECK-LMUL2-NEXT:  LV(REG): Found invariant usage: 1 item
 ; CHECK-LMUL2-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK-LMUL4:       LV(REG): VF = 8
-; CHECK-LMUL4-NEXT:  LV(REG): Found max usage: 2 item
+; CHECK-LMUL4:       LV(REG): Found max usage: 2 item
 ; CHECK-LMUL4-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
-; CHECK-LMUL4-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; CHECK-LMUL4-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 8 registers
 ; CHECK-LMUL4-NEXT:  LV(REG): Found invariant usage: 1 item
 ; CHECK-LMUL4-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK-LMUL8:       LV(REG): VF = 16
-; CHECK-LMUL8-NEXT:  LV(REG): Found max usage: 2 item
+; CHECK-LMUL8:       LV(REG): Found max usage: 2 item
 ; CHECK-LMUL8-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
-; CHECK-LMUL8-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; CHECK-LMUL8-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 16 registers
 ; CHECK-LMUL8-NEXT:  LV(REG): Found invariant usage: 1 item
 ; CHECK-LMUL8-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
 
@@ -80,21 +76,17 @@ define void @goo(ptr nocapture noundef %a, i32 noundef signext %n) {
 ; CHECK-SCALAR:      LV(REG): VF = 1
 ; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item
 ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
-; CHECK-LMUL1:       LV(REG): VF = 2
-; CHECK-LMUL1-NEXT:  LV(REG): Found max usage: 2 item
-; CHECK-LMUL1-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
-; CHECK-LMUL1-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 2 registers
-; CHECK-LMUL2:       LV(REG): VF = 4
-; CHECK-LMUL2-NEXT:  LV(REG): Found max usage: 2 item
-; CHECK-LMUL2-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
+; CHECK-LMUL1:       LV(REG): Found max usage: 2 item
+; CHECK-LMUL1-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 2 registers
+; CHECK-LMUL1-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 1 registers
+; CHECK-LMUL2:       LV(REG): Found max usage: 2 item
+; CHECK-LMUL2-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 2 registers
 ; CHECK-LMUL2-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 2 registers
-; CHECK-LMUL4:       LV(REG): VF = 8
-; CHECK-LMUL4-NEXT:  LV(REG): Found max usage: 2 item
-; CHECK-LMUL4-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
+; CHECK-LMUL4:       LV(REG): Found max usage: 2 item
+; CHECK-LMUL4-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 2 registers
 ; CHECK-LMUL4-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 4 registers
-; CHECK-LMUL8:       LV(REG): VF = 16
-; CHECK-LMUL8-NEXT:  LV(REG): Found max usage: 2 item
-; CHECK-LMUL8-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 3 registers
+; CHECK-LMUL8:       LV(REG): Found max usage: 2 item
+; CHECK-LMUL8-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 2 registers
 ; CHECK-LMUL8-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 8 registers
 entry:
   %cmp3 = icmp sgt i32 %n, 0
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 2e461ec658e6..b026e6868581 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; This is the loop in c++ being vectorize in this file with
 ;vector.reverse
 ;  #pragma clang loop vectorize_width(4, scalable)
@@ -7,7 +7,7 @@
 
 ; REQUIRES: asserts
 ; RUN: opt -passes=loop-vectorize,dce,instcombine -mtriple riscv64-linux-gnu \
-; RUN:   -mattr=+v -debug-only=loop-vectorize -scalable-vectorization=on \
+; RUN:   -mattr=+v -debug-only=loop-vectorize,vplan -scalable-vectorization=on \
 ; RUN:   -riscv-v-vector-bits-min=128 -disable-output < %s 2>&1 | FileCheck %s
 
 define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
@@ -46,66 +46,100 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
 ; CHECK-NEXT:  LV: Using user VF vscale x 4.
+; CHECK-NEXT:  Creating VPBasicBlock for for.body
+; CHECK-NEXT:  VPlan 'Plain CFG
+; CHECK-NEXT:   for UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.body.preheader>:
+; CHECK-NEXT:    IR %0 = zext i32 %n to i64
+; CHECK-NEXT:  Successor(s): for.body
+; CHECK-EMPTY:
+; CHECK-NEXT:  for.body:
+; CHECK-NEXT:    WIDEN-PHI ir<%indvars.iv> = phi [ ir<%indvars.iv.next>, for.body ], [ ir<%0>, ir-bb<for.body.preheader> ]
+; CHECK-NEXT:    WIDEN-PHI ir<%i.0.in8> = phi [ ir<%i.0>, for.body ], [ ir<%n>, ir-bb<for.body.preheader> ]
+; CHECK-NEXT:    EMIT ir<%i.0> = add ir<%i.0.in8>, ir<-1>
+; CHECK-NEXT:    EMIT ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT:    EMIT ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom>
+; CHECK-NEXT:    EMIT ir<%1> = load ir<%arrayidx>
+; CHECK-NEXT:    EMIT ir<%add9> = add ir<%1>, ir<1>
+; CHECK-NEXT:    EMIT ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom>
+; CHECK-NEXT:    EMIT store ir<%add9>, ir<%arrayidx3>
+; CHECK-NEXT:    EMIT ir<%cmp> = icmp ir<%indvars.iv>, ir<1>
+; CHECK-NEXT:    EMIT ir<%indvars.iv.next> = add ir<%indvars.iv>, ir<-1>
+; CHECK-NEXT:    EMIT branch-on-cond ir<%cmp>
+; CHECK-NEXT:  Successor(s): for.body, ir-bb<for.cond.cleanup.loopexit>
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
+; CHECK-NEXT:  }
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
-; CHECK:       LV: Scalarizing: %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:  LV: Scalarizing: %i.0 = add nsw i32 %i.0.in8, -1
 ; CHECK-NEXT:  LV: Scalarizing: %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:  LV: Scalarizing: %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
 ; CHECK-NEXT:  LV: Scalarizing: %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
-; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
+; CHECK-NEXT:  Live-in vp<%0> = VF
+; CHECK-NEXT:  Live-in vp<%1> = VF * UF
+; CHECK-NEXT:  Live-in vp<%2> = vector-trip-count
+; CHECK-NEXT:  vp<%3> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
-; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:    EMIT vp<%3> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.ph:
-; CHECK-NEXT:    vp<[[END1:%.+]]> = DERIVED-IV ir<%0> + vp<[[VEC_TC]]> * ir<-1>
-; CHECK-NEXT:    vp<[[END2:%.+]]> = DERIVED-IV ir<%n> + vp<[[VEC_TC]]> * ir<-1>
+; CHECK-NEXT:    vp<%4> = DERIVED-IV ir<%0> + vp<%2> * ir<-1>
+; CHECK-NEXT:    vp<%5> = DERIVED-IV ir<%n> + vp<%2> * ir<-1>
 ; CHECK-NEXT:  Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
-; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT:      EMIT vp<%6> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT:      vp<%7> = DERIVED-IV ir<%n> + vp<%6> * ir<-1>
+; CHECK-NEXT:      vp<%8> = SCALAR-STEPS vp<%7>, ir<-1>, vp<%0>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%8>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
-; CHECK-NEXT:      WIDEN ir<%1> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT:      vp<%9> = vector-end-pointer inbounds ir<%arrayidx>, vp<%0>
+; CHECK-NEXT:      WIDEN ir<%1> = load vp<%9>
 ; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%1>, ir<1>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-end-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
-; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
-; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:      vp<%10> = vector-end-pointer inbounds ir<%arrayidx3>, vp<%0>
+; CHECK-NEXT:      WIDEN store vp<%10>, ir<%add9>
+; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<%6>, vp<%1>
+; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<%2>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq vp<%3>, vp<%2>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME1:%.+]]> = resume-phi vp<[[END1]]>, ir<%0>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = resume-phi vp<[[END2]]>, ir<%n>
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%4>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%5>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
-; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME1]]> from scalar.ph)
-; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME2]]>.1 from scalar.ph)
-; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
+; CHECK-NEXT:    IR %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT:    IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] (extra operand: vp<%bc.resume.val>.1 from scalar.ph)
+; CHECK-NEXT:    IR %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:    IR %idxprom = zext i32 %i.0 to i64
+; CHECK-NEXT:    IR %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
+; CHECK-NEXT:    IR %1 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    IR %add9 = add i32 %1, 1
+; CHECK-NEXT:    IR %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
+; CHECK-NEXT:    IR store i32 %add9, ptr %arrayidx3, align 4
+; CHECK-NEXT:    IR %cmp = icmp ugt i64 %indvars.iv, 1
+; CHECK-NEXT:    IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
@@ -151,85 +185,212 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Interleaving is not beneficial.
 ; CHECK-NEXT:  LV: Found a vectorizable loop (vscale x 4) in <stdin>
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
-; CHECK:       Executing best plan with VF=vscale x 4, UF=1
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT:  Live-in ir<[[VF:%.+]]> = VF
-; CHECK-NEXT:  Live-in ir<[[VFxUF:%.+]]>.1 = VF * UF
-; CHECK-NEXT:  Live-in ir<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT:  ir<%0> = original trip-count
+; CHECK-NEXT:  Live-in ir<%18> = VF
+; CHECK-NEXT:  Live-in ir<%18>.1 = VF * UF
+; CHECK-NEXT:  Live-in ir<%n.vec> = vector-trip-count
+; CHECK-NEXT:  Live-in ir<%0> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
 ; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.scevcheck>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.scevcheck>:
-; CHECK-NEXT:    IR   %3 = add nsw i64 %0, -1
-; CHECK-NEXT:    IR   %4 = add i32 %n, -1
-; CHECK-NEXT:    IR   %5 = trunc i64 %3 to i32
-; CHECK-NEXT:    IR   %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5)
-; CHECK-NEXT:    IR   %mul.result = extractvalue { i32, i1 } %mul, 0
-; CHECK-NEXT:    IR   %mul.overflow = extractvalue { i32, i1 } %mul, 1
-; CHECK-NEXT:    IR   %6 = sub i32 %4, %mul.result
-; CHECK-NEXT:    IR   %7 = icmp ugt i32 %6, %4
-; CHECK-NEXT:    IR   %8 = or i1 %7, %mul.overflow
-; CHECK-NEXT:    IR   %9 = icmp ugt i64 %3, 4294967295
-; CHECK-NEXT:    IR   %10 = or i1 %8, %9
+; CHECK-NEXT:    IR %3 = add nsw i64 %0, -1
+; CHECK-NEXT:    IR %4 = add i32 %n, -1
+; CHECK-NEXT:    IR %5 = trunc i64 %3 to i32
+; CHECK-NEXT:    IR %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5)
+; CHECK-NEXT:    IR %mul.result = extractvalue { i32, i1 } %mul, 0
+; CHECK-NEXT:    IR %mul.overflow = extractvalue { i32, i1 } %mul, 1
+; CHECK-NEXT:    IR %6 = sub i32 %4, %mul.result
+; CHECK-NEXT:    IR %7 = icmp ugt i32 %6, %4
+; CHECK-NEXT:    IR %8 = or i1 %7, %mul.overflow
+; CHECK-NEXT:    IR %9 = icmp ugt i64 %3, 4294967295
+; CHECK-NEXT:    IR %10 = or i1 %8, %9
 ; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.memcheck>:
-; CHECK-NEXT:    IR   %11 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR   %12 = mul i64 %11, 4
-; CHECK-NEXT:    IR   %13 = mul i64 %12, 4
-; CHECK-NEXT:    IR   %14 = sub i64 %B1, %A2
-; CHECK-NEXT:    IR   %diff.check = icmp ult i64 %14, %13
+; CHECK-NEXT:    IR %11 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    IR %12 = mul i64 %11, 4
+; CHECK-NEXT:    IR %13 = mul i64 %12, 4
+; CHECK-NEXT:    IR %14 = sub i64 %B1, %A2
+; CHECK-NEXT:    IR %diff.check = icmp ult i64 %14, %13
 ; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.ph>:
-; CHECK-NEXT:    IR   %15 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR   %16 = mul i64 %15, 4
-; CHECK-NEXT:    IR   %n.mod.vf = urem i64 %0, %16
-; CHECK-NEXT:    IR   %n.vec = sub i64 %0, %n.mod.vf
-; CHECK-NEXT:    IR   %17 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR   %18 = mul i64 %17, 4
-; CHECK-NEXT:    vp<[[END1:%.+]]> = DERIVED-IV ir<%0> + ir<[[VEC_TC]]> * ir<-1>
-; CHECK-NEXT:    vp<[[END2:%.+]]> = DERIVED-IV ir<%n> + ir<[[VEC_TC]]> * ir<-1>
+; CHECK-NEXT:    IR %15 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    IR %16 = mul i64 %15, 4
+; CHECK-NEXT:    IR %n.mod.vf = urem i64 %0, %16
+; CHECK-NEXT:    IR %n.vec = sub i64 %0, %n.mod.vf
+; CHECK-NEXT:    IR %17 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    IR %18 = mul i64 %17, 4
+; CHECK-NEXT:    vp<%1> = DERIVED-IV ir<%0> + ir<%n.vec> * ir<-1>
+; CHECK-NEXT:    vp<%2> = DERIVED-IV ir<%n> + ir<%n.vec> * ir<-1>
 ; CHECK-NEXT:  Successor(s): vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.body:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[CAN_IV:%.+]]> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[CAN_IV_NEXT:%.+]]>, vector.body ]
-; CHECK-NEXT:    vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<[[DEV_IV]]>, ir<-1>
+; CHECK-NEXT:    EMIT-SCALAR vp<%index> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT:    vp<%3> = DERIVED-IV ir<%n> + vp<%index> * ir<-1>
+; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<%3>, ir<-1>
 ; CHECK-NEXT:    CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:    CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, ir<[[VF]]>
-; CHECK-NEXT:    WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT:    WIDEN ir<%add9> = add ir<[[L]]>, ir<1>
+; CHECK-NEXT:    vp<%4> = vector-end-pointer inbounds ir<%arrayidx>, ir<%18>
+; CHECK-NEXT:    WIDEN ir<%19> = load vp<%4>
+; CHECK-NEXT:    WIDEN ir<%add9> = add ir<%19>, ir<1>
 ; CHECK-NEXT:    CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR2:%.+]]> = vector-end-pointer inbounds ir<%arrayidx3>, ir<[[VF]]>
-; CHECK-NEXT:    WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
-; CHECK-NEXT:    EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>.1
-; CHECK-NEXT:    EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VEC_TC]]>
+; CHECK-NEXT:    vp<%5> = vector-end-pointer inbounds ir<%arrayidx3>, ir<%18>
+; CHECK-NEXT:    WIDEN store vp<%5>, ir<%add9>
+; CHECK-NEXT:    EMIT vp<%index.next> = add nuw vp<%index>, ir<%18>.1
+; CHECK-NEXT:    EMIT branch-on-count vp<%index.next>, ir<%n.vec>
 ; CHECK-NEXT:  Successor(s): middle.block, vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq ir<%0>, ir<[[VEC_TC]]>
-; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<%0>, ir<%n.vec>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<scalar.ph>:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_1:%.+]]> = resume-phi vp<[[END1]]>, ir<%0>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_2:%.+]]>.1 = resume-phi vp<[[END2]]>, ir<%n>
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%1>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ], [ ir<%0>, ir-bb<vector.scevcheck> ], [ ir<%0>, ir-bb<vector.memcheck> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%2>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ], [ ir<%n>, ir-bb<vector.scevcheck> ], [ ir<%n>, ir-bb<vector.memcheck> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
-; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME_1]]> from ir-bb<scalar.ph>)
-; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME_2]]>.1 from ir-bb<scalar.ph>)
-; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
+; CHECK-NEXT:    IR %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<%bc.resume.val> from ir-bb<scalar.ph>)
+; CHECK-NEXT:    IR %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<%bc.resume.val>.1 from ir-bb<scalar.ph>)
+; CHECK-NEXT:    IR %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:    IR %idxprom = zext i32 %i.0 to i64
+; CHECK-NEXT:    IR %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
+; CHECK-NEXT:    IR %19 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    IR %add9 = add i32 %19, 1
+; CHECK-NEXT:    IR %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
+; CHECK-NEXT:    IR store i32 %add9, ptr %arrayidx3, align 4
+; CHECK-NEXT:    IR %cmp = icmp ugt i64 %indvars.iv, 1
+; CHECK-NEXT:    IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body.preheader> in BB:for.body.preheader
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  for.body.preheader: ; preds = %entry
+; CHECK-NEXT:    %0 = zext i32 %n to i64
+; CHECK-NEXT:    %1 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %2 = mul i64 %1, 4
+; CHECK-NEXT:    %min.iters.check = icmp ult i64 %0, %2
+; CHECK-NEXT:    br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.scevcheck> in BB:vector.scevcheck
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.scevcheck: ; preds = %for.body.preheader
+; CHECK-NEXT:    %3 = add nsw i64 %0, -1
+; CHECK-NEXT:    %4 = add i32 %n, -1
+; CHECK-NEXT:    %5 = trunc i64 %3 to i32
+; CHECK-NEXT:    %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5)
+; CHECK-NEXT:    %mul.result = extractvalue { i32, i1 } %mul, 0
+; CHECK-NEXT:    %mul.overflow = extractvalue { i32, i1 } %mul, 1
+; CHECK-NEXT:    %6 = sub i32 %4, %mul.result
+; CHECK-NEXT:    %7 = icmp ugt i32 %6, %4
+; CHECK-NEXT:    %8 = or i1 %7, %mul.overflow
+; CHECK-NEXT:    %9 = icmp ugt i64 %3, 4294967295
+; CHECK-NEXT:    %10 = or i1 %8, %9
+; CHECK-NEXT:    br i1 %10, label %scalar.ph, label %vector.memcheck
+; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.memcheck> in BB:vector.memcheck
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.memcheck: ; preds = %vector.scevcheck
+; CHECK-NEXT:    %11 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %12 = mul i64 %11, 4
+; CHECK-NEXT:    %13 = mul i64 %12, 4
+; CHECK-NEXT:    %14 = sub i64 %B1, %A2
+; CHECK-NEXT:    %diff.check = icmp ult i64 %14, %13
+; CHECK-NEXT:    br i1 %diff.check, label %scalar.ph, label %vector.ph
+; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.ph> in BB:vector.ph
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.ph: ; preds = %vector.memcheck
+; CHECK-NEXT:    %15 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %16 = mul i64 %15, 4
+; CHECK-NEXT:    %n.mod.vf = urem i64 %0, %16
+; CHECK-NEXT:    %n.vec = sub i64 %0, %n.mod.vf
+; CHECK-NEXT:    %17 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %18 = mul i64 %17, 4
+; CHECK-NEXT:    %19 = sub i64 %0, %n.vec
+; CHECK-NEXT:    %.cast = trunc i64 %n.vec to i32
+; CHECK-NEXT:    %20 = sub i32 %n, %.cast
+; CHECK-NEXT:    br
+; CHECK-NEXT:  LV: draw edge fromvector.memcheck
+; CHECK-NEXT:  LV: created vector.body
+; CHECK-NEXT:  LV: draw edge fromvector.ph
+; CHECK-NEXT:  LV: vectorizing VPBB:vector.body in BB:vector.body
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.body: ; preds = %vector.body, %vector.ph
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ]
+; CHECK-NEXT:    %.cast3 = trunc i64 %index to i32
+; CHECK-NEXT:    %offset.idx = sub i32 %n, %.cast3
+; CHECK-NEXT:    %21 = add nsw i32 %offset.idx, -1
+; CHECK-NEXT:    %22 = zext i32 %21 to i64
+; CHECK-NEXT:    %23 = getelementptr inbounds i32, ptr %B, i64 %22
+; CHECK-NEXT:    %24 = mul i64 0, %18
+; CHECK-NEXT:    %25 = sub i64 1, %18
+; CHECK-NEXT:    %26 = getelementptr inbounds i32, ptr %23, i64 %24
+; CHECK-NEXT:    %27 = getelementptr inbounds i32, ptr %26, i64 %25
+; CHECK-NEXT:    %wide.load = load <vscale x 4 x i32>, ptr %27, align 4
+; CHECK-NEXT:    %reverse = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %wide.load)
+; CHECK-NEXT:    %28 = add <vscale x 4 x i32> %reverse, splat (i32 1)
+; CHECK-NEXT:    %29 = getelementptr inbounds i32, ptr %A, i64 %22
+; CHECK-NEXT:    %30 = mul i64 0, %18
+; CHECK-NEXT:    %31 = sub i64 1, %18
+; CHECK-NEXT:    %32 = getelementptr inbounds i32, ptr %29, i64 %30
+; CHECK-NEXT:    %33 = getelementptr inbounds i32, ptr %32, i64 %31
+; CHECK-NEXT:    %reverse4 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %28)
+; CHECK-NEXT:    store <vscale x 4 x i32> %reverse4, ptr %33, align 4
+; CHECK-NEXT:    %index.next = add nuw i64 %index, %18
+; CHECK-NEXT:    %34 = icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1 %34, <null operand!>, label %vector.body
+; CHECK-NEXT:  LV: created middle.block
+; CHECK-NEXT:  LV: draw edge fromvector.body
+; CHECK-NEXT:  LV: vectorizing VPBB:middle.block in BB:middle.block
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  middle.block: ; preds = %vector.body
+; CHECK-NEXT:    %cmp.n = icmp eq i64 %0, %n.vec
+; CHECK-NEXT:    br i1 %cmp.n, <null operand!>, <null operand!>
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.cond.cleanup.loopexit> in BB:for.cond.cleanup.loopexit
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  for.cond.cleanup.loopexit: ; preds = %for.body
+; CHECK-NEXT:    br label %for.cond.cleanup
+; CHECK-NEXT:  LV: draw edge frommiddle.block
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<scalar.ph> in BB:scalar.ph
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  scalar.ph: ; preds = %vector.memcheck, %vector.scevcheck, %for.body.preheader
+; CHECK-NEXT:    %bc.resume.val = phi i64 [ %19, %middle.block ], [ %0, %for.body.preheader ], [ %0, %vector.scevcheck ], [ %0, %vector.memcheck ]
+; CHECK-NEXT:    %bc.resume.val5 = phi i32 [ %20, %middle.block ], [ %n, %for.body.preheader ], [ %n, %vector.scevcheck ], [ %n, %vector.memcheck ]
+; CHECK-NEXT:    br label %for.body
+; CHECK-NEXT:  LV: draw edge frommiddle.block
+; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
+; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
+; CHECK-NEXT:  LV: draw edge fromvector.memcheck
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body> in BB:for.body
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  for.body: ; preds = %for.body, %scalar.ph
+; CHECK-NEXT:    %indvars.iv = phi i64 [ %bc.resume.val, %scalar.ph ], [ %indvars.iv.next, %for.body ]
+; CHECK-NEXT:    %i.0.in8 = phi i32 [ %bc.resume.val5, %scalar.ph ], [ %i.0, %for.body ]
+; CHECK-NEXT:    %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:    %idxprom = zext i32 %i.0 to i64
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
+; CHECK-NEXT:    %35 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    %add9 = add i32 %35, 1
+; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
+; CHECK-NEXT:    store i32 %add9, ptr %arrayidx3, align 4
+; CHECK-NEXT:    %cmp = icmp ugt i64 %indvars.iv, 1
+; CHECK-NEXT:    %indvars.iv.next = add nsw i64 %indvars.iv, -1
+; CHECK-NEXT:    br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
+; CHECK-NEXT:  LV: draw edge fromscalar.ph
+; CHECK-NEXT:  LV: Interleaving disabled by the pass manager
+; CHECK-NEXT:  LV: Vectorizing: innermost loop.
+; CHECK-EMPTY:
 ;
 entry:
   %cmp7 = icmp sgt i32 %n, 0
@@ -294,66 +455,100 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
 ; CHECK-NEXT:  LV: Using user VF vscale x 4.
+; CHECK-NEXT:  Creating VPBasicBlock for for.body
+; CHECK-NEXT:  VPlan 'Plain CFG
+; CHECK-NEXT:   for UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.body.preheader>:
+; CHECK-NEXT:    IR %0 = zext i32 %n to i64
+; CHECK-NEXT:  Successor(s): for.body
+; CHECK-EMPTY:
+; CHECK-NEXT:  for.body:
+; CHECK-NEXT:    WIDEN-PHI ir<%indvars.iv> = phi [ ir<%indvars.iv.next>, for.body ], [ ir<%0>, ir-bb<for.body.preheader> ]
+; CHECK-NEXT:    WIDEN-PHI ir<%i.0.in8> = phi [ ir<%i.0>, for.body ], [ ir<%n>, ir-bb<for.body.preheader> ]
+; CHECK-NEXT:    EMIT ir<%i.0> = add ir<%i.0.in8>, ir<-1>
+; CHECK-NEXT:    EMIT ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT:    EMIT ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom>
+; CHECK-NEXT:    EMIT ir<%1> = load ir<%arrayidx>
+; CHECK-NEXT:    EMIT ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
+; CHECK-NEXT:    EMIT ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom>
+; CHECK-NEXT:    EMIT store ir<%conv1>, ir<%arrayidx3>
+; CHECK-NEXT:    EMIT ir<%cmp> = icmp ir<%indvars.iv>, ir<1>
+; CHECK-NEXT:    EMIT ir<%indvars.iv.next> = add ir<%indvars.iv>, ir<-1>
+; CHECK-NEXT:    EMIT branch-on-cond ir<%cmp>
+; CHECK-NEXT:  Successor(s): for.body, ir-bb<for.cond.cleanup.loopexit>
+; CHECK-EMPTY:
+; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT:  No successors
+; CHECK-NEXT:  }
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
-; CHECK:       LV: Scalarizing: %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:  LV: Scalarizing: %i.0 = add nsw i32 %i.0.in8, -1
 ; CHECK-NEXT:  LV: Scalarizing: %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:  LV: Scalarizing: %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
 ; CHECK-NEXT:  LV: Scalarizing: %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<[[VF:%.+]]> = VF
-; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
+; CHECK-NEXT:  Live-in vp<%0> = VF
+; CHECK-NEXT:  Live-in vp<%1> = VF * UF
+; CHECK-NEXT:  Live-in vp<%2> = vector-trip-count
+; CHECK-NEXT:  vp<%3> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
-; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:    EMIT vp<%3> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.ph:
-; CHECK-NEXT:    vp<[[END1:%.+]]> = DERIVED-IV ir<%0> + vp<[[VEC_TC]]> * ir<-1>
-; CHECK-NEXT:    vp<[[END2:%.+]]> = DERIVED-IV ir<%n> + vp<[[VEC_TC]]> * ir<-1>
+; CHECK-NEXT:    vp<%4> = DERIVED-IV ir<%0> + vp<%2> * ir<-1>
+; CHECK-NEXT:    vp<%5> = DERIVED-IV ir<%n> + vp<%2> * ir<-1>
 ; CHECK-NEXT:  Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
-; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
+; CHECK-NEXT:      EMIT vp<%6> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT:      vp<%7> = DERIVED-IV ir<%n> + vp<%6> * ir<-1>
+; CHECK-NEXT:      vp<%8> = SCALAR-STEPS vp<%7>, ir<-1>, vp<%0>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%8>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
-; CHECK-NEXT:      WIDEN ir<%1> = load vp<[[VEC_PTR]]>
+; CHECK-NEXT:      vp<%9> = vector-end-pointer inbounds ir<%arrayidx>, vp<%0>
+; CHECK-NEXT:      WIDEN ir<%1> = load vp<%9>
 ; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-end-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
-; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1>
-; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:      vp<%10> = vector-end-pointer inbounds ir<%arrayidx3>, vp<%0>
+; CHECK-NEXT:      WIDEN store vp<%10>, ir<%conv1>
+; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<%6>, vp<%1>
+; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<%2>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq vp<%3>, vp<%2>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME1:%.+]]> = resume-phi vp<[[END1]]>, ir<%0>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = resume-phi vp<[[END2]]>, ir<%n>
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%4>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%5>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
-; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME1]]> from scalar.ph)
-; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME2]]>.1 from scalar.ph)
-; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
+; CHECK-NEXT:    IR %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT:    IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] (extra operand: vp<%bc.resume.val>.1 from scalar.ph)
+; CHECK-NEXT:    IR %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:    IR %idxprom = zext i32 %i.0 to i64
+; CHECK-NEXT:    IR %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
+; CHECK-NEXT:    IR %1 = load float, ptr %arrayidx, align 4
+; CHECK-NEXT:    IR %conv1 = fadd float %1, 1.000000e+00
+; CHECK-NEXT:    IR %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
+; CHECK-NEXT:    IR store float %conv1, ptr %arrayidx3, align 4
+; CHECK-NEXT:    IR %cmp = icmp ugt i64 %indvars.iv, 1
+; CHECK-NEXT:    IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
@@ -399,85 +594,211 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Interleaving is not beneficial.
 ; CHECK-NEXT:  LV: Found a vectorizable loop (vscale x 4) in <stdin>
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
-; CHECK:       Executing best plan with VF=vscale x 4, UF=1
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  LV: Loop does not require scalar epilogue
+; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT:  Live-in ir<[[VF:%.+]]> = VF
-; CHECK-NEXT:  Live-in ir<[[VFxUF:%.+]]>.1 = VF * UF
-; CHECK-NEXT:  Live-in ir<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT:  ir<%0> = original trip-count
+; CHECK-NEXT:  Live-in ir<%18> = VF
+; CHECK-NEXT:  Live-in ir<%18>.1 = VF * UF
+; CHECK-NEXT:  Live-in ir<%n.vec> = vector-trip-count
+; CHECK-NEXT:  Live-in ir<%0> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
 ; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.scevcheck>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.scevcheck>:
-; CHECK-NEXT:    IR   %3 = add nsw i64 %0, -1
-; CHECK-NEXT:    IR   %4 = add i32 %n, -1
-; CHECK-NEXT:    IR   %5 = trunc i64 %3 to i32
-; CHECK-NEXT:    IR   %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5)
-; CHECK-NEXT:    IR   %mul.result = extractvalue { i32, i1 } %mul, 0
-; CHECK-NEXT:    IR   %mul.overflow = extractvalue { i32, i1 } %mul, 1
-; CHECK-NEXT:    IR   %6 = sub i32 %4, %mul.result
-; CHECK-NEXT:    IR   %7 = icmp ugt i32 %6, %4
-; CHECK-NEXT:    IR   %8 = or i1 %7, %mul.overflow
-; CHECK-NEXT:    IR   %9 = icmp ugt i64 %3, 4294967295
-; CHECK-NEXT:    IR   %10 = or i1 %8, %9
+; CHECK-NEXT:    IR %3 = add nsw i64 %0, -1
+; CHECK-NEXT:    IR %4 = add i32 %n, -1
+; CHECK-NEXT:    IR %5 = trunc i64 %3 to i32
+; CHECK-NEXT:    IR %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5)
+; CHECK-NEXT:    IR %mul.result = extractvalue { i32, i1 } %mul, 0
+; CHECK-NEXT:    IR %mul.overflow = extractvalue { i32, i1 } %mul, 1
+; CHECK-NEXT:    IR %6 = sub i32 %4, %mul.result
+; CHECK-NEXT:    IR %7 = icmp ugt i32 %6, %4
+; CHECK-NEXT:    IR %8 = or i1 %7, %mul.overflow
+; CHECK-NEXT:    IR %9 = icmp ugt i64 %3, 4294967295
+; CHECK-NEXT:    IR %10 = or i1 %8, %9
 ; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.memcheck>:
-; CHECK-NEXT:    IR   %11 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR   %12 = mul i64 %11, 4
-; CHECK-NEXT:    IR   %13 = mul i64 %12, 4
-; CHECK-NEXT:    IR   %14 = sub i64 %B1, %A2
-; CHECK-NEXT:    IR   %diff.check = icmp ult i64 %14, %13
+; CHECK-NEXT:    IR %11 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    IR %12 = mul i64 %11, 4
+; CHECK-NEXT:    IR %13 = mul i64 %12, 4
+; CHECK-NEXT:    IR %14 = sub i64 %B1, %A2
+; CHECK-NEXT:    IR %diff.check = icmp ult i64 %14, %13
 ; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.ph>:
-; CHECK-NEXT:    IR   %15 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR   %16 = mul i64 %15, 4
-; CHECK-NEXT:    IR   %n.mod.vf = urem i64 %0, %16
-; CHECK-NEXT:    IR   %n.vec = sub i64 %0, %n.mod.vf
-; CHECK-NEXT:    IR   %17 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR   %18 = mul i64 %17, 4
-; CHECK-NEXT:    vp<[[END1:%.+]]> = DERIVED-IV ir<%0> + ir<[[VEC_TC]]> * ir<-1>
-; CHECK-NEXT:    vp<[[END2:%.+]]> = DERIVED-IV ir<%n> + ir<[[VEC_TC]]> * ir<-1>
+; CHECK-NEXT:    IR %15 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    IR %16 = mul i64 %15, 4
+; CHECK-NEXT:    IR %n.mod.vf = urem i64 %0, %16
+; CHECK-NEXT:    IR %n.vec = sub i64 %0, %n.mod.vf
+; CHECK-NEXT:    IR %17 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    IR %18 = mul i64 %17, 4
+; CHECK-NEXT:    vp<%1> = DERIVED-IV ir<%0> + ir<%n.vec> * ir<-1>
+; CHECK-NEXT:    vp<%2> = DERIVED-IV ir<%n> + ir<%n.vec> * ir<-1>
 ; CHECK-NEXT:  Successor(s): vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.body:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[CAN_IV:%.+]]> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[CAN_IV_NEXT:%.+]]>, vector.body ]
-; CHECK-NEXT:    vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<[[DEV_IV]]>, ir<-1>
+; CHECK-NEXT:    EMIT-SCALAR vp<%index> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT:    vp<%3> = DERIVED-IV ir<%n> + vp<%index> * ir<-1>
+; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<%3>, ir<-1>
 ; CHECK-NEXT:    CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:    CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, ir<[[VF]]>
-; CHECK-NEXT:    WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT:    WIDEN ir<%conv1> = fadd ir<[[L]]>, ir<1.000000e+00>
+; CHECK-NEXT:    vp<%4> = vector-end-pointer inbounds ir<%arrayidx>, ir<%18>
+; CHECK-NEXT:    WIDEN ir<%19> = load vp<%4>
+; CHECK-NEXT:    WIDEN ir<%conv1> = fadd ir<%19>, ir<1.000000e+00>
 ; CHECK-NEXT:    CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:    vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx3>, ir<[[VF]]>
-; CHECK-NEXT:    WIDEN store vp<[[VEC_PTR]]>, ir<%conv1>
-; CHECK-NEXT:    EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>.1
-; CHECK-NEXT:    EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VEC_TC]]>
+; CHECK-NEXT:    vp<%5> = vector-end-pointer inbounds ir<%arrayidx3>, ir<%18>
+; CHECK-NEXT:    WIDEN store vp<%5>, ir<%conv1>
+; CHECK-NEXT:    EMIT vp<%index.next> = add nuw vp<%index>, ir<%18>.1
+; CHECK-NEXT:    EMIT branch-on-count vp<%index.next>, ir<%n.vec>
 ; CHECK-NEXT:  Successor(s): middle.block, vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq ir<%0>, ir<[[VEC_TC]]>
-; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<%0>, ir<%n.vec>
+; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<scalar.ph>:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME1:%.+]]> = resume-phi vp<[[END1]]>, ir<%0>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = resume-phi vp<[[END2]]>, ir<%n>
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%1>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ], [ ir<%0>, ir-bb<vector.scevcheck> ], [ ir<%0>, ir-bb<vector.memcheck> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%2>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ], [ ir<%n>, ir-bb<vector.scevcheck> ], [ ir<%n>, ir-bb<vector.memcheck> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
-; CHECK-NEXT:    IR   %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME1]]> from ir-bb<scalar.ph>)
-; CHECK-NEXT:    IR   %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME2]]>.1 from ir-bb<scalar.ph>)
-; CHECK:         IR   %indvars.iv.next = add nsw i64 %indvars.iv, -1
+; CHECK-NEXT:    IR %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<%bc.resume.val> from ir-bb<scalar.ph>)
+; CHECK-NEXT:    IR %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<%bc.resume.val>.1 from ir-bb<scalar.ph>)
+; CHECK-NEXT:    IR %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:    IR %idxprom = zext i32 %i.0 to i64
+; CHECK-NEXT:    IR %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
+; CHECK-NEXT:    IR %19 = load float, ptr %arrayidx, align 4
+; CHECK-NEXT:    IR %conv1 = fadd float %19, 1.000000e+00
+; CHECK-NEXT:    IR %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
+; CHECK-NEXT:    IR store float %conv1, ptr %arrayidx3, align 4
+; CHECK-NEXT:    IR %cmp = icmp ugt i64 %indvars.iv, 1
+; CHECK-NEXT:    IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body.preheader> in BB:for.body.preheader
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  for.body.preheader: ; preds = %entry
+; CHECK-NEXT:    %0 = zext i32 %n to i64
+; CHECK-NEXT:    %1 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %2 = mul i64 %1, 4
+; CHECK-NEXT:    %min.iters.check = icmp ult i64 %0, %2
+; CHECK-NEXT:    br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.scevcheck> in BB:vector.scevcheck
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.scevcheck: ; preds = %for.body.preheader
+; CHECK-NEXT:    %3 = add nsw i64 %0, -1
+; CHECK-NEXT:    %4 = add i32 %n, -1
+; CHECK-NEXT:    %5 = trunc i64 %3 to i32
+; CHECK-NEXT:    %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5)
+; CHECK-NEXT:    %mul.result = extractvalue { i32, i1 } %mul, 0
+; CHECK-NEXT:    %mul.overflow = extractvalue { i32, i1 } %mul, 1
+; CHECK-NEXT:    %6 = sub i32 %4, %mul.result
+; CHECK-NEXT:    %7 = icmp ugt i32 %6, %4
+; CHECK-NEXT:    %8 = or i1 %7, %mul.overflow
+; CHECK-NEXT:    %9 = icmp ugt i64 %3, 4294967295
+; CHECK-NEXT:    %10 = or i1 %8, %9
+; CHECK-NEXT:    br i1 %10, label %scalar.ph, label %vector.memcheck
+; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.memcheck> in BB:vector.memcheck
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.memcheck: ; preds = %vector.scevcheck
+; CHECK-NEXT:    %11 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %12 = mul i64 %11, 4
+; CHECK-NEXT:    %13 = mul i64 %12, 4
+; CHECK-NEXT:    %14 = sub i64 %B1, %A2
+; CHECK-NEXT:    %diff.check = icmp ult i64 %14, %13
+; CHECK-NEXT:    br i1 %diff.check, label %scalar.ph, label %vector.ph
+; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.ph> in BB:vector.ph
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.ph: ; preds = %vector.memcheck
+; CHECK-NEXT:    %15 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %16 = mul i64 %15, 4
+; CHECK-NEXT:    %n.mod.vf = urem i64 %0, %16
+; CHECK-NEXT:    %n.vec = sub i64 %0, %n.mod.vf
+; CHECK-NEXT:    %17 = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    %18 = mul i64 %17, 4
+; CHECK-NEXT:    %19 = sub i64 %0, %n.vec
+; CHECK-NEXT:    %.cast = trunc i64 %n.vec to i32
+; CHECK-NEXT:    %20 = sub i32 %n, %.cast
+; CHECK-NEXT:    br
+; CHECK-NEXT:  LV: draw edge fromvector.memcheck
+; CHECK-NEXT:  LV: created vector.body
+; CHECK-NEXT:  LV: draw edge fromvector.ph
+; CHECK-NEXT:  LV: vectorizing VPBB:vector.body in BB:vector.body
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  vector.body: ; preds = %vector.body, %vector.ph
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ]
+; CHECK-NEXT:    %.cast3 = trunc i64 %index to i32
+; CHECK-NEXT:    %offset.idx = sub i32 %n, %.cast3
+; CHECK-NEXT:    %21 = add nsw i32 %offset.idx, -1
+; CHECK-NEXT:    %22 = zext i32 %21 to i64
+; CHECK-NEXT:    %23 = getelementptr inbounds float, ptr %B, i64 %22
+; CHECK-NEXT:    %24 = mul i64 0, %18
+; CHECK-NEXT:    %25 = sub i64 1, %18
+; CHECK-NEXT:    %26 = getelementptr inbounds float, ptr %23, i64 %24
+; CHECK-NEXT:    %27 = getelementptr inbounds float, ptr %26, i64 %25
+; CHECK-NEXT:    %wide.load = load <vscale x 4 x float>, ptr %27, align 4
+; CHECK-NEXT:    %reverse = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %wide.load)
+; CHECK-NEXT:    %28 = fadd <vscale x 4 x float> %reverse, splat (float 1.000000e+00)
+; CHECK-NEXT:    %29 = getelementptr inbounds float, ptr %A, i64 %22
+; CHECK-NEXT:    %30 = mul i64 0, %18
+; CHECK-NEXT:    %31 = sub i64 1, %18
+; CHECK-NEXT:    %32 = getelementptr inbounds float, ptr %29, i64 %30
+; CHECK-NEXT:    %33 = getelementptr inbounds float, ptr %32, i64 %31
+; CHECK-NEXT:    %reverse4 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %28)
+; CHECK-NEXT:    store <vscale x 4 x float> %reverse4, ptr %33, align 4
+; CHECK-NEXT:    %index.next = add nuw i64 %index, %18
+; CHECK-NEXT:    %34 = icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1 %34, <null operand!>, label %vector.body
+; CHECK-NEXT:  LV: created middle.block
+; CHECK-NEXT:  LV: draw edge fromvector.body
+; CHECK-NEXT:  LV: vectorizing VPBB:middle.block in BB:middle.block
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  middle.block: ; preds = %vector.body
+; CHECK-NEXT:    %cmp.n = icmp eq i64 %0, %n.vec
+; CHECK-NEXT:    br i1 %cmp.n, <null operand!>, <null operand!>
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.cond.cleanup.loopexit> in BB:for.cond.cleanup.loopexit
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  for.cond.cleanup.loopexit: ; preds = %for.body
+; CHECK-NEXT:    br label %for.cond.cleanup
+; CHECK-NEXT:  LV: draw edge frommiddle.block
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<scalar.ph> in BB:scalar.ph
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  scalar.ph: ; preds = %vector.memcheck, %vector.scevcheck, %for.body.preheader
+; CHECK-NEXT:    %bc.resume.val = phi i64 [ %19, %middle.block ], [ %0, %for.body.preheader ], [ %0, %vector.scevcheck ], [ %0, %vector.memcheck ]
+; CHECK-NEXT:    %bc.resume.val5 = phi i32 [ %20, %middle.block ], [ %n, %for.body.preheader ], [ %n, %vector.scevcheck ], [ %n, %vector.memcheck ]
+; CHECK-NEXT:    br label %for.body
+; CHECK-NEXT:  LV: draw edge frommiddle.block
+; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
+; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
+; CHECK-NEXT:  LV: draw edge fromvector.memcheck
+; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body> in BB:for.body
+; CHECK-NEXT:  LV: filled BB:
+; CHECK-NEXT:  for.body: ; preds = %for.body, %scalar.ph
+; CHECK-NEXT:    %indvars.iv = phi i64 [ %bc.resume.val, %scalar.ph ], [ %indvars.iv.next, %for.body ]
+; CHECK-NEXT:    %i.0.in8 = phi i32 [ %bc.resume.val5, %scalar.ph ], [ %i.0, %for.body ]
+; CHECK-NEXT:    %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT:    %idxprom = zext i32 %i.0 to i64
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
+; CHECK-NEXT:    %35 = load float, ptr %arrayidx, align 4
+; CHECK-NEXT:    %conv1 = fadd float %35, 1.000000e+00
+; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
+; CHECK-NEXT:    store float %conv1, ptr %arrayidx3, align 4
+; CHECK-NEXT:    %cmp = icmp ugt i64 %indvars.iv, 1
+; CHECK-NEXT:    %indvars.iv.next = add nsw i64 %indvars.iv, -1
+; CHECK-NEXT:    br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
+; CHECK-NEXT:  LV: draw edge fromscalar.ph
+; CHECK-NEXT:  LV: Interleaving disabled by the pass manager
+; CHECK-NEXT:  LV: Vectorizing: innermost loop.
 ;
 entry:
   %cmp7 = icmp sgt i32 %n, 0
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
index 01a7ea4ffcd0..3f17c95f7ca9 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
@@ -277,9 +277,9 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
 ; IF-EVL-OUTLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -581,8 +581,8 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
-; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
+; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP14:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
 ; NO-VP-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP14]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP10]] to i32
@@ -771,8 +771,8 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
-; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
+; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP14:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
 ; NO-VP-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP14]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP10]] to i32
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
index 990ac0bdb72a..d9b1981869b2 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
@@ -59,7 +59,7 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[ADD]] = add nsw i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -154,7 +154,7 @@ define i32 @mul(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[MUL]] = mul i32 [[TMP5]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 4
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -169,7 +169,7 @@ define i32 @mul(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[MUL1]] = mul nsw i32 [[TMP0]], [[RDX1]]
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL1]], [[FOR_BODY1]] ], [ [[MUL]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[MUL_LCSSA]]
@@ -269,7 +269,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -284,7 +284,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[OR]] = or i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[OR_LCSSA]]
@@ -384,7 +384,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -399,7 +399,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[AND]] = and i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[AND_LCSSA]]
@@ -499,7 +499,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -514,7 +514,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[XOR]] = xor i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[XOR_LCSSA:%.*]] = phi i32 [ [[XOR]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[XOR_LCSSA]]
@@ -614,7 +614,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -630,7 +630,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[SMIN]] = select i1 [[CMP_I]], i32 [[TMP17]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[SMIN_LCSSA:%.*]] = phi i32 [ [[SMIN]], [[FOR_BODY]] ], [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[SMIN_LCSSA]]
@@ -732,7 +732,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -748,7 +748,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[SMAX]] = select i1 [[CMP_I]], i32 [[TMP17]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[SMAX_LCSSA:%.*]] = phi i32 [ [[SMAX]], [[FOR_BODY]] ], [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[SMAX_LCSSA]]
@@ -850,7 +850,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -866,7 +866,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[UMIN]] = select i1 [[CMP_I]], i32 [[TMP17]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[UMIN_LCSSA:%.*]] = phi i32 [ [[UMIN]], [[FOR_BODY]] ], [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[UMIN_LCSSA]]
@@ -968,7 +968,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -984,7 +984,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[UMAX]] = select i1 [[CMP_I]], i32 [[TMP17]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[UMAX_LCSSA:%.*]] = phi i32 [ [[UMAX]], [[FOR_BODY]] ], [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[UMAX_LCSSA]]
@@ -1086,7 +1086,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -1101,7 +1101,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[ADD]] = fadd reassoc float [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[ADD_LCSSA]]
@@ -1196,7 +1196,7 @@ define float @fmul(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MUL]] = fmul reassoc float [[TMP5]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 4
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -1211,7 +1211,7 @@ define float @fmul(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MUL1]] = fmul reassoc float [[TMP0]], [[RDX1]]
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP23:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP24:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL1]], [[FOR_BODY1]] ], [ [[MUL]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MUL_LCSSA]]
@@ -1312,7 +1312,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -1328,7 +1328,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MIN]] = select i1 [[CMP]], float [[TMP17]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
@@ -1432,7 +1432,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -1448,7 +1448,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MAX]] = select i1 [[CMP]], float [[TMP17]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
@@ -1546,7 +1546,7 @@ define float @fminimum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> [[TMP5]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1562,7 +1562,7 @@ define float @fminimum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MIN]] = tail call float @llvm.minimum.f32(float [[RDX]], float [[TMP0]])
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP29:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP30:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
@@ -1658,7 +1658,7 @@ define float @fmaximum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> [[TMP5]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1674,7 +1674,7 @@ define float @fmaximum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MAX]] = tail call float @llvm.maximum.f32(float [[RDX]], float [[TMP0]])
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP31:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP32:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
@@ -1779,7 +1779,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -1796,7 +1796,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MULADD]] = tail call reassoc float @llvm.fmuladd.f32(float [[TMP21]], float [[TMP22]], float [[RDX]])
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MULADD_LCSSA]]
@@ -1904,7 +1904,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
@@ -1923,7 +1923,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
@@ -2028,7 +2028,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = freeze i1 [[TMP19]]
@@ -2047,7 +2047,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index 7660d06a35d4..af36f184ea82 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -59,7 +59,7 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[ADD]] = add nsw i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ADD_LCSSA]]
@@ -137,9 +137,9 @@ define i32 @mul(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> splat (i32 1), i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
-; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> splat (i32 1), i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -155,7 +155,7 @@ define i32 @mul(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP5]], <8 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[TMP6]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -171,7 +171,7 @@ define i32 @mul(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[MUL]] = mul nsw i32 [[TMP0]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY1]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[MUL_LCSSA]]
@@ -272,7 +272,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -288,7 +288,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[OR]] = or i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[OR_LCSSA]]
@@ -390,7 +390,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -406,7 +406,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[AND]] = and i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[AND_LCSSA]]
@@ -508,7 +508,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -524,7 +524,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[XOR]] = xor i32 [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[XOR_LCSSA:%.*]] = phi i32 [ [[XOR]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[XOR_LCSSA]]
@@ -628,7 +628,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -645,7 +645,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[SMIN]] = select i1 [[CMP_I]], i32 [[TMP19]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[SMIN_LCSSA:%.*]] = phi i32 [ [[SMIN]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[SMIN_LCSSA]]
@@ -753,7 +753,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -770,7 +770,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[SMAX]] = select i1 [[CMP_I]], i32 [[TMP19]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[SMAX_LCSSA:%.*]] = phi i32 [ [[SMAX]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[SMAX_LCSSA]]
@@ -878,7 +878,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -895,7 +895,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[UMIN]] = select i1 [[CMP_I]], i32 [[TMP19]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[UMIN_LCSSA:%.*]] = phi i32 [ [[UMIN]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[UMIN_LCSSA]]
@@ -1003,7 +1003,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1020,7 +1020,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[UMAX]] = select i1 [[CMP_I]], i32 [[TMP19]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[UMAX_LCSSA:%.*]] = phi i32 [ [[UMAX]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[UMAX_LCSSA]]
@@ -1126,7 +1126,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP15]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1142,7 +1142,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[ADD]] = fadd reassoc float [[TMP18]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[ADD_LCSSA]]
@@ -1220,9 +1220,9 @@ define float @fmul(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <8 x float> splat (float 1.000000e+00), float [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
-; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <8 x float> splat (float 1.000000e+00), float [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -1238,7 +1238,7 @@ define float @fmul(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = select reassoc <8 x i1> [[TMP2]], <8 x float> [[TMP5]], <8 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call reassoc float @llvm.vector.reduce.fmul.v8f32(float 1.000000e+00, <8 x float> [[TMP6]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1254,7 +1254,7 @@ define float @fmul(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MUL]] = fmul reassoc float [[TMP0]], [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP23:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP24:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL]], [[FOR_BODY1]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MUL_LCSSA]]
@@ -1357,7 +1357,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1374,7 +1374,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MIN]] = select i1 [[CMP]], float [[TMP19]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
@@ -1482,7 +1482,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1499,7 +1499,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[MAX]] = select i1 [[CMP]], float [[TMP19]], float [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
@@ -1599,7 +1599,7 @@ define float @fminimum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> [[TMP5]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1615,7 +1615,7 @@ define float @fminimum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MIN]] = tail call float @llvm.minimum.f32(float [[RDX]], float [[TMP0]])
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP29:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP30:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MIN_LCSSA:%.*]] = phi float [ [[MIN]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MIN_LCSSA]]
@@ -1711,7 +1711,7 @@ define float @fmaximum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP4]], <8 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 8
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> [[TMP5]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1727,7 +1727,7 @@ define float @fmaximum(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MAX]] = tail call float @llvm.maximum.f32(float [[RDX]], float [[TMP0]])
 ; IF-EVL-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP31:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP32:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MAX_LCSSA:%.*]] = phi float [ [[MAX]], [[FOR_BODY1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MAX_LCSSA]]
@@ -1832,7 +1832,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP17]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
@@ -1850,7 +1850,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[MULADD]] = tail call reassoc float @llvm.fmuladd.f32(float [[TMP21]], float [[TMP22]], float [[RDX]])
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret float [[MULADD_LCSSA]]
@@ -1958,7 +1958,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP15]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = freeze i1 [[TMP18]]
@@ -1977,7 +1977,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
@@ -2082,7 +2082,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP15]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = freeze i1 [[TMP18]]
@@ -2101,7 +2101,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[ANYOF]] = select i1 [[CMP_I]], i32 [[INV]], i32 [[RDX]]
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    [[ANYOF_LCSSA:%.*]] = phi i32 [ [[ANYOF]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-NEXT:    ret i32 [[ANYOF_LCSSA]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
index 7bdb67b8a0fb..4d8166eaa46f 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
@@ -67,7 +67,7 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:    store i32 [[TMP]], ptr [[GEPS]], align 4
 ; IF-EVL-NEXT:    [[INC]] = add i32 [[I]], 1
 ; IF-EVL-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[INC]], 1024
-; IF-EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[LOOPEND]], !llvm.loop [[LOOP3:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[LOOPEND]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       loopend:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -155,7 +155,7 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP28]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; IF-EVL-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[LOOPEND:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -179,7 +179,7 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL:       for.inc:
 ; IF-EVL-NEXT:    [[INC]] = add i32 [[I]], 1
 ; IF-EVL-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[INC]], 1024
-; IF-EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[LOOPEND]], !llvm.loop [[LOOP5:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[LOOPEND]], !llvm.loop [[LOOP6:![0-9]+]]
 ; IF-EVL:       loopend:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -288,7 +288,7 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP27]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; IF-EVL-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -306,7 +306,7 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    store i8 [[Y]], ptr [[GEP_D]], align 1
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], -1
 ; IF-EVL-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; IF-EVL-NEXT:    br i1 [[CMP_NOT]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[CMP_NOT]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL:       exit:
 ; IF-EVL-NEXT:    ret void
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
index fb2ec8c61d74..ee32127f2889 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
@@ -57,7 +57,7 @@ define void @test(ptr %p) {
 ; IF-EVL-NEXT:    store i64 [[V]], ptr [[A2]], align 8
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       exit:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -114,7 +114,7 @@ define void @test_may_clobber1(ptr %p) {
 ; IF-EVL-NEXT:    store <4 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 32
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; IF-EVL:       scalar.ph:
@@ -129,7 +129,7 @@ define void @test_may_clobber1(ptr %p) {
 ; IF-EVL-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
 ; IF-EVL:       exit:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -254,7 +254,7 @@ define void @test_may_clobber3(ptr %p) {
 ; IF-EVL-NEXT:    store <2 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 32
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; IF-EVL:       scalar.ph:
@@ -269,7 +269,7 @@ define void @test_may_clobber3(ptr %p) {
 ; IF-EVL-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL:       exit:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -357,7 +357,7 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP13]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -372,7 +372,7 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; IF-EVL-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL:       exit:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -441,7 +441,7 @@ define void @no_high_lmul_or_interleave(ptr %p) {
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IF-EVL-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
@@ -456,7 +456,7 @@ define void @no_high_lmul_or_interleave(ptr %p) {
 ; IF-EVL-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 3001
-; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP12:![0-9]+]]
 ; IF-EVL:       exit:
 ; IF-EVL-NEXT:    ret void
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
index 99e4d7539b93..579bc450b83d 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
@@ -63,7 +63,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       for.cond.cleanup:
 ; IF-EVL-NEXT:    ret void
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index 832cd928e626..79f490aa16a9 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -37,13 +37,14 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT: Successor(s): scalar.ph, vector.ph
 ; IF-EVL-OUTLOOP-EMPTY:
 ; IF-EVL-OUTLOOP-NEXT: vector.ph:
+; IF-EVL-OUTLOOP-NEXT:  EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>
 ; IF-EVL-OUTLOOP-NEXT: Successor(s): vector loop
 ; IF-EVL-OUTLOOP-EMPTY:
 ; IF-EVL-OUTLOOP-NEXT: <x1> vector loop: {
 ; IF-EVL-OUTLOOP-NEXT:  vector.body:
 ; IF-EVL-OUTLOOP-NEXT:    EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
 ; IF-EVL-OUTLOOP-NEXT:    EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
-; IF-EVL-OUTLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi ir<%start>, vp<[[RDX_SELECT:%.+]]>
+; IF-EVL-OUTLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
 ; IF-EVL-OUTLOOP-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%n>, vp<[[EVL_PHI]]>
 ; IF-EVL-OUTLOOP-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
 ; IF-EVL-OUTLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
@@ -77,13 +78,14 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT: Live-in ir<%n> = original trip-count
 ; IF-EVL-INLOOP-EMPTY:
 ; IF-EVL-INLOOP:      vector.ph:
+; IF-EVL-INLOOP-NEXT:   EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>
 ; IF-EVL-INLOOP-NEXT: Successor(s): vector loop
 ; IF-EVL-INLOOP-EMPTY:
 ; IF-EVL-INLOOP-NEXT: <x1> vector loop: {
 ; IF-EVL-INLOOP-NEXT:  vector.body:
 ; IF-EVL-INLOOP-NEXT:    EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
 ; IF-EVL-INLOOP-NEXT:    EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
-; IF-EVL-INLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi ir<%start>, ir<[[RDX_NEXT:%.+]]>
+; IF-EVL-INLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; IF-EVL-INLOOP-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%n>, vp<[[EVL_PHI]]>
 ; IF-EVL-INLOOP-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
 ; IF-EVL-INLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
@@ -116,12 +118,13 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT: Live-in ir<%n> = original trip-count
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP:      vector.ph:
+; NO-VP-OUTLOOP-NEXT:   EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>
 ; NO-VP-OUTLOOP-NEXT: Successor(s): vector loop
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: <x1> vector loop: {
 ; NO-VP-OUTLOOP-NEXT:  vector.body:
 ; NO-VP-OUTLOOP-NEXT:    EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; NO-VP-OUTLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi ir<%start>, ir<[[RDX_NEXT:%.+]]>
+; NO-VP-OUTLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; NO-VP-OUTLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
 ; NO-VP-OUTLOOP-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
 ; NO-VP-OUTLOOP-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
@@ -145,8 +148,8 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT: No successors
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: scalar.ph:
-; NO-VP-OUTLOOP-NEXT:   EMIT-SCALAR vp<[[IV_RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
-; NO-VP-OUTLOOP-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
+; NO-VP-OUTLOOP-NEXT:   EMIT-SCALAR vp<[[IV_RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; NO-VP-OUTLOOP-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = phi [ vp<[[RDX]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]
 ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: ir-bb<for.body>:
@@ -164,12 +167,13 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT: Live-in ir<%n> = original trip-count
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP:      vector.ph:
+; NO-VP-INLOOP-NEXT:   EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>
 ; NO-VP-INLOOP-NEXT: Successor(s): vector loop
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: <x1> vector loop: {
 ; NO-VP-INLOOP-NEXT:  vector.body:
 ; NO-VP-INLOOP-NEXT:    EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; NO-VP-INLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi ir<%start>, ir<[[RDX_NEXT:%.+]]>
+; NO-VP-INLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; NO-VP-INLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
 ; NO-VP-INLOOP-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
 ; NO-VP-INLOOP-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
@@ -193,8 +197,8 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT: No successors
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: scalar.ph:
-; NO-VP-INLOOP-NEXT:   EMIT-SCALAR vp<[[IV_RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
-; NO-VP-INLOOP-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
+; NO-VP-INLOOP-NEXT:   EMIT-SCALAR vp<[[IV_RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; NO-VP-INLOOP-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = phi [ vp<[[RDX]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]
 ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.body>
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: ir-bb<for.body>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 7c42c3d9cd52..2c6fe4f5c808 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -1271,12 +1271,12 @@ define i32 @g(i64 %n) {
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP20]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF7:%.*]] = urem i32 [[TMP1]], 4
 ; CHECK-NEXT:    [[N_VEC8:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF7]]
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT9]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[BC_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX9:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index 83e9d6146755..3361068c9922 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -217,14 +217,12 @@ define i64 @test_pr62954_scalar_epilogue_required(ptr %A, ptr noalias %B, ptr %C
 ; CHECK-NEXT:    [[REC_START:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <2 x i64> poison, i64 [[REC_START]], i32 1
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    [[TMP1]] = sub nsw <2 x i64> zeroinitializer, [[STEP_ADD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <2 x i64> zeroinitializer, [[STEP_ADD]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
 ; CHECK-NEXT:    store i64 [[TMP2]], ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -299,13 +297,12 @@ define void @for_iv_trunc_optimized(ptr %dst) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 1>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP0]], splat (i32 3)
-; CHECK-NEXT:    [[TMP3]] = or <4 x i32> [[TMP1]], splat (i32 3)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], splat (i32 3)
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP5]], i32 3
 ; CHECK-NEXT:    store i32 [[TMP6]], ptr [[DST:%.*]], align 4
@@ -364,11 +361,9 @@ define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 8, i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP4]]
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x ptr> poison, ptr [[A:%.*]], i32 3
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 2
@@ -377,7 +372,7 @@ define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 5
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 6
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 7
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr nusw [3 x float], ptr [[A:%.*]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP8]]
@@ -392,7 +387,7 @@ define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr
 ; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP21]], i32 0
 ; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x ptr> [[TMP25]], ptr [[TMP22]], i32 1
 ; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x ptr> [[TMP26]], ptr [[TMP23]], i32 2
-; CHECK-NEXT:    [[TMP28]] = insertelement <4 x ptr> [[TMP27]], ptr [[TMP24]], i32 3
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x ptr> [[TMP27]], ptr [[TMP24]], i32 3
 ; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x ptr> [[TMP20]], <4 x ptr> [[TMP28]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x float>, ptr [[TMP21]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/i1-reg-usage.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/i1-reg-usage.ll
index 3445d4ceff5e..84a48dba6ae4 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/i1-reg-usage.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/i1-reg-usage.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output 2>&1 < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize,vplan -disable-output 2>&1 < %s | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "x86_64"
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 19424e44a902..7aeb32afe43b 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -407,11 +407,10 @@ define i16 @iv_and_step_trunc() {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <2 x i16> [ <i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[TMP0]] to <2 x i16>
-; CHECK-NEXT:    [[TMP2]] = mul <2 x i16> [[VEC_IND1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <2 x i16> [[VEC_IND1]], [[TMP1]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
 ; CHECK-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i16> [[VEC_IND1]], splat (i16 2)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr131359-dead-for-splice.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr131359-dead-for-splice.ll
index bcfa212cf364..c02ec91c4a0c 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr131359-dead-for-splice.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr131359-dead-for-splice.ll
@@ -15,9 +15,8 @@ define void @no_use() {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, %[[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40
@@ -60,9 +59,8 @@ define void @dead_use() {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, %[[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
new file mode 100644
index 000000000000..9a70ed451cf4
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -S %s | FileCheck %s
+
+target triple = "x86_64"
+
+define i8 @pr141968(i1 %cond, i8 %v) {
+; CHECK-LABEL: define i8 @pr141968(
+; CHECK-SAME: i1 [[COND:%.*]], i8 [[V:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[ZEXT_TRUE:%.*]] = zext i1 true to i16
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i8 [[V]] to i16
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[COND]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = xor <16 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE30:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i1> [[TMP0]], i32 0
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK:       [[PRED_SDIV_IF]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE]]
+; CHECK:       [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i1> [[TMP0]], i32 1
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; CHECK:       [[PRED_SDIV_IF1]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE2]]
+; CHECK:       [[PRED_SDIV_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i1> [[TMP0]], i32 2
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
+; CHECK:       [[PRED_SDIV_IF3]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE4]]
+; CHECK:       [[PRED_SDIV_CONTINUE4]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i1> [[TMP0]], i32 3
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
+; CHECK:       [[PRED_SDIV_IF5]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE6]]
+; CHECK:       [[PRED_SDIV_CONTINUE6]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i1> [[TMP0]], i32 4
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_SDIV_IF7:.*]], label %[[PRED_SDIV_CONTINUE8:.*]]
+; CHECK:       [[PRED_SDIV_IF7]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE8]]
+; CHECK:       [[PRED_SDIV_CONTINUE8]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i1> [[TMP0]], i32 5
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_SDIV_IF9:.*]], label %[[PRED_SDIV_CONTINUE10:.*]]
+; CHECK:       [[PRED_SDIV_IF9]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE10]]
+; CHECK:       [[PRED_SDIV_CONTINUE10]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i1> [[TMP0]], i32 6
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_SDIV_IF11:.*]], label %[[PRED_SDIV_CONTINUE12:.*]]
+; CHECK:       [[PRED_SDIV_IF11]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE12]]
+; CHECK:       [[PRED_SDIV_CONTINUE12]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i1> [[TMP0]], i32 7
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_SDIV_IF13:.*]], label %[[PRED_SDIV_CONTINUE14:.*]]
+; CHECK:       [[PRED_SDIV_IF13]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE14]]
+; CHECK:       [[PRED_SDIV_CONTINUE14]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i1> [[TMP0]], i32 8
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_SDIV_IF15:.*]], label %[[PRED_SDIV_CONTINUE16:.*]]
+; CHECK:       [[PRED_SDIV_IF15]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE16]]
+; CHECK:       [[PRED_SDIV_CONTINUE16]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i1> [[TMP0]], i32 9
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_SDIV_IF17:.*]], label %[[PRED_SDIV_CONTINUE18:.*]]
+; CHECK:       [[PRED_SDIV_IF17]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE18]]
+; CHECK:       [[PRED_SDIV_CONTINUE18]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x i1> [[TMP0]], i32 10
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_SDIV_IF19:.*]], label %[[PRED_SDIV_CONTINUE20:.*]]
+; CHECK:       [[PRED_SDIV_IF19]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE20]]
+; CHECK:       [[PRED_SDIV_CONTINUE20]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i1> [[TMP0]], i32 11
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_SDIV_IF21:.*]], label %[[PRED_SDIV_CONTINUE22:.*]]
+; CHECK:       [[PRED_SDIV_IF21]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE22]]
+; CHECK:       [[PRED_SDIV_CONTINUE22]]:
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i1> [[TMP0]], i32 12
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_SDIV_IF23:.*]], label %[[PRED_SDIV_CONTINUE24:.*]]
+; CHECK:       [[PRED_SDIV_IF23]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE24]]
+; CHECK:       [[PRED_SDIV_CONTINUE24]]:
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i1> [[TMP0]], i32 13
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_SDIV_IF25:.*]], label %[[PRED_SDIV_CONTINUE26:.*]]
+; CHECK:       [[PRED_SDIV_IF25]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE26]]
+; CHECK:       [[PRED_SDIV_CONTINUE26]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i1> [[TMP0]], i32 14
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[PRED_SDIV_IF27:.*]], label %[[PRED_SDIV_CONTINUE28:.*]]
+; CHECK:       [[PRED_SDIV_IF27]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE28]]
+; CHECK:       [[PRED_SDIV_CONTINUE28]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i1> [[TMP0]], i32 15
+; CHECK-NEXT:    br i1 [[TMP16]], label %[[PRED_SDIV_IF29:.*]], label %[[PRED_SDIV_CONTINUE30]]
+; CHECK:       [[PRED_SDIV_IF29]]:
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE30]]
+; CHECK:       [[PRED_SDIV_CONTINUE30]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT31:%.*]] = insertelement <16 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT32:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT31]], <16 x i8> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <16 x i1> [[BROADCAST_SPLAT]], <16 x i8> zeroinitializer, <16 x i8> [[BROADCAST_SPLAT32]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 15
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP_LATCH]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[SDIV:%.*]] = sdiv i16 [[SEXT]], [[ZEXT_TRUE]]
+; CHECK-NEXT:    [[SDIV_TRUNC:%.*]] = trunc i16 [[SDIV]] to i8
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ [[SDIV_TRUNC]], %[[COND_FALSE]] ], [ 0, %[[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i8 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i8 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RET_LCSSA:%.*]] = phi i8 [ [[RET]], %[[LOOP_LATCH]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i8 [[RET_LCSSA]]
+;
+entry:
+  %zext.true = zext i1 true to i16
+  %sext = sext i8 %v to i16
+  br label %loop.header
+
+loop.header:                                      ; preds = %loop.latch, %entry
+  %iv = phi i8 [ %iv.next, %loop.latch ], [ 0, %entry ]
+  br i1 %cond, label %loop.latch, label %cond.false
+
+cond.false:                                       ; preds = %loop.header
+  %sdiv = sdiv i16 %sext, %zext.true
+  %sdiv.trunc = trunc i16 %sdiv to i8
+  br label %loop.latch
+
+loop.latch:                                       ; preds = %cond.false, %loop.header
+  %ret = phi i8 [ %sdiv.trunc, %cond.false ], [ 0, %loop.header ]
+  %iv.next = add i8 %iv, 1
+  %exitcond = icmp eq i8 %iv.next, 0
+  br i1 %exitcond, label %exit, label %loop.header
+
+exit:                                             ; preds = %loop.latch
+  ret i8 %ret
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
index 0cd746590e0f..368842634c37 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/pr72969.ll
@@ -56,7 +56,6 @@ define void @test(ptr %p) {
 ; VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC:       vector.body:
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VEC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 1>, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
 ; VEC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 1, i16 2, i16 3, i16 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VEC-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 0
 ; VEC-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX]], 1
@@ -75,7 +74,7 @@ define void @test(ptr %p) {
 ; VEC-NEXT:    store i64 0, ptr [[TMP25]], align 8
 ; VEC-NEXT:    store i64 0, ptr [[TMP26]], align 8
 ; VEC-NEXT:    [[TMP27:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 1)
-; VEC-NEXT:    [[TMP28]] = zext <4 x i16> [[TMP27]] to <4 x i64>
+; VEC-NEXT:    [[TMP28:%.*]] = zext <4 x i16> [[TMP27]] to <4 x i64>
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VEC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
 ; VEC-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
index 04271ff3c997..2cda2533e80e 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
@@ -33,7 +33,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; CHECK: Cost of 1 for VF 2: exit condition instruction   %exitcond = icmp eq i32 %lftr.wideiv, %n
 ; CHECK: Cost of 0 for VF 2: exit condition instruction   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
 ; CHECK: Cost of 0 for VF 2: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%sum.013> = phi ir<0>, vp<[[EXT:%.+]]>
+; CHECK: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%sum.013> = phi vp<{{.+}}>, vp<[[EXT:%.+]]>
 ; CHECK: Cost of 0 for VF 2: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK: Cost of 0 for VF 2: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
 ; CHECK: Cost of 0 for VF 2: vp<[[VECP1:%.+]]> = vector-pointer ir<%arrayidx>
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
index 164188db6ccf..530ff175c81c 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -debug-only=loop-vectorize -passes=loop-vectorize -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
+; RUN: opt < %s -debug-only=loop-vectorize,vplan -passes=loop-vectorize -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Test that the register usage estimation is not affected by the presence of
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
index 3cf44947ea46..98995994cc98 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -debug-only=loop-vectorize -passes=loop-vectorize -vectorizer-maximize-bandwidth -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
-; RUN: opt < %s -debug-only=loop-vectorize -passes=loop-vectorize -vectorizer-maximize-bandwidth -mtriple=x86_64-unknown-linux -mattr=+avx512f -S 2>&1 | FileCheck %s --check-prefix=AVX512F
+; RUN: opt < %s -debug-only=loop-vectorize,vplan -passes=loop-vectorize -vectorizer-maximize-bandwidth -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
+; RUN: opt < %s -debug-only=loop-vectorize,vplan -passes=loop-vectorize -vectorizer-maximize-bandwidth -mtriple=x86_64-unknown-linux -mattr=+avx512f -S 2>&1 | FileCheck %s --check-prefix=AVX512F
 ; REQUIRES: asserts
 
 @a = global [1024 x i8] zeroinitializer, align 16
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/check-prof-info.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/check-prof-info.ll
index 17013c590806..40741941d4b0 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/check-prof-info.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/check-prof-info.ll
@@ -1,24 +1,60 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br" --filter "^.*:"
 ; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=1 -S < %s |  FileCheck %s
 ; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=4 -S < %s |  FileCheck %s -check-prefix=CHECK-MASKED
+; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=1 \
+; RUN:   -scalable-vectorization=on -force-target-supports-scalable-vectors -S < %s |  FileCheck %s -check-prefix=CHECK-SCALABLE
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
-@a = dso_local global [1024 x i32] zeroinitializer, align 16
-@b = dso_local global [1024 x i32] zeroinitializer, align 16
+@a = global [1024 x i32] zeroinitializer, align 16
+@b = global [1024 x i32] zeroinitializer, align 16
 
 ; Check correctness of profile info for vectorization without epilog.
-; Function Attrs: nofree norecurse nounwind uwtable
-define dso_local void @_Z3foov() local_unnamed_addr #0 {
+define void @_Z3foov() {
 ; CHECK-LABEL: @_Z3foov(
-; CHECK:  [[VECTOR_BODY:vector\.body]]:
-; CHECK:    br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_255:\!.*]],
-; CHECK:  [[FOR_BODY:for\.body]]:
-; CHECK:    br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP0_0:\!.*]],
-; CHECK-MASKED:  [[VECTOR_BODY:vector\.body]]:
-; CHECK-MASKED:    br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_63:\!.*]],
-; CHECK-MASKED:  [[FOR_BODY:for\.body]]:
-; CHECK-MASKED:    br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP0_0:\!.*]],
+; CHECK:  entry:
+; CHECK:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:  vector.ph:
+; CHECK:    br label [[VECTOR_BODY:%.*]]
+; CHECK:  vector.body:
+; CHECK:    br i1 [[TMP6:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK:  middle.block:
+; CHECK:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5:![0-9]+]]
+; CHECK:  scalar.ph:
+; CHECK:    br label [[FOR_BODY:%.*]]
+; CHECK:  for.cond.cleanup:
+; CHECK:  for.body:
+; CHECK:    br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+; CHECK-MASKED-LABEL: @_Z3foov(
+; CHECK-MASKED:  entry:
+; CHECK-MASKED:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-MASKED:  vector.ph:
+; CHECK-MASKED:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MASKED:  vector.body:
+; CHECK-MASKED:    br i1 [[TMP18:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-MASKED:  middle.block:
+; CHECK-MASKED:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5:![0-9]+]]
+; CHECK-MASKED:  scalar.ph:
+; CHECK-MASKED:    br label [[FOR_BODY:%.*]]
+; CHECK-MASKED:  for.cond.cleanup:
+; CHECK-MASKED:  for.body:
+; CHECK-MASKED:    br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+; CHECK-SCALABLE-LABEL: @_Z3foov(
+; CHECK-SCALABLE:  entry:
+; CHECK-SCALABLE:    br i1 [[MIN_ITERS_CHECK:%.*]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-SCALABLE:  vector.ph:
+; CHECK-SCALABLE:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SCALABLE:  vector.body:
+; CHECK-SCALABLE:    br i1 [[TMP16:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-SCALABLE:  middle.block:
+; CHECK-SCALABLE:    br i1 [[CMP_N:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5:![0-9]+]]
+; CHECK-SCALABLE:  scalar.ph:
+; CHECK-SCALABLE:    br label [[FOR_BODY:%.*]]
+; CHECK-SCALABLE:  for.cond.cleanup:
+; CHECK-SCALABLE:  for.body:
+; CHECK-SCALABLE:    br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
 ;
 entry:
   br label %for.body
@@ -27,32 +63,66 @@ for.cond.cleanup:                                 ; preds = %for.body
   ret void
 
 for.body:                                         ; preds = %for.body, %entry
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4, !tbaa !2
-  %1 = trunc i64 %indvars.iv to i32
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %1 = trunc i64 %iv to i32
   %mul = mul nsw i32 %0, %1
-  %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %indvars.iv
-  %2 = load i32, ptr %arrayidx2, align 4, !tbaa !2
+  %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %iv
+  %2 = load i32, ptr %arrayidx2, align 4
   %add = add nsw i32 %2, %mul
-  store i32 %add, ptr %arrayidx2, align 4, !tbaa !2
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 1024
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !6
+  store i32 %add, ptr %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !0
 }
 
 ; Check correctness of profile info for vectorization with epilog.
-; Function Attrs: nofree norecurse nounwind uwtable
-define dso_local void @_Z3foo2v() local_unnamed_addr #0 {
+define void @_Z3foo2v() {
 ; CHECK-LABEL: @_Z3foo2v(
-; CHECK:  [[VECTOR_BODY:vector\.body]]:
-; CHECK:    br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_255:\!.*]],
-; CHECK:  [[FOR_BODY:for\.body]]:
-; CHECK:    br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP1_2:\!.*]],
-; CHECK-MASKED:  [[VECTOR_BODY:vector\.body]]:
-; CHECK-MASKED:    br i1 [[TMP:%.*]], label [[MIDDLE_BLOCK:%.*]], label %[[VECTOR_BODY]], !prof [[LP1_63:\!.*]],
-; CHECK-MASKED:  [[FOR_BODY:for\.body]]:
-; CHECK-MASKED:    br i1 [[EXITCOND:%.*]], label [[FOR_END_LOOPEXIT:%.*]], label %[[FOR_BODY]], !prof [[LP1_2:\!.*]],
+; CHECK:  entry:
+; CHECK:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0]]
+; CHECK:  vector.ph:
+; CHECK:    br label [[VECTOR_BODY:%.*]]
+; CHECK:  vector.body:
+; CHECK:    br i1 [[TMP6:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:  middle.block:
+; CHECK:    br i1 false, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5]]
+; CHECK:  scalar.ph:
+; CHECK:    br label [[FOR_BODY:%.*]]
+; CHECK:  for.cond.cleanup:
+; CHECK:  for.body:
+; CHECK:    br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
+;
+; CHECK-MASKED-LABEL: @_Z3foo2v(
+; CHECK-MASKED:  entry:
+; CHECK-MASKED:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0]]
+; CHECK-MASKED:  vector.ph:
+; CHECK-MASKED:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MASKED:  vector.body:
+; CHECK-MASKED:    br i1 [[TMP18:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-MASKED:  middle.block:
+; CHECK-MASKED:    br i1 false, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5]]
+; CHECK-MASKED:  scalar.ph:
+; CHECK-MASKED:    br label [[FOR_BODY:%.*]]
+; CHECK-MASKED:  for.cond.cleanup:
+; CHECK-MASKED:  for.body:
+; CHECK-MASKED:    br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
+;
+; CHECK-SCALABLE-LABEL: @_Z3foo2v(
+; CHECK-SCALABLE:  entry:
+; CHECK-SCALABLE:    br i1 [[MIN_ITERS_CHECK:%.*]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0]]
+; CHECK-SCALABLE:  vector.ph:
+; CHECK-SCALABLE:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SCALABLE:  vector.body:
+; CHECK-SCALABLE:    br i1 [[TMP16:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-SCALABLE:  middle.block:
+; CHECK-SCALABLE:    br i1 [[CMP_N:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5]]
+; CHECK-SCALABLE:  scalar.ph:
+; CHECK-SCALABLE:    br label [[FOR_BODY:%.*]]
+; CHECK-SCALABLE:  for.cond.cleanup:
+; CHECK-SCALABLE:  for.body:
+; CHECK-SCALABLE:    br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
 ;
 entry:
   br label %for.body
@@ -61,36 +131,20 @@ for.cond.cleanup:                                 ; preds = %for.body
   ret void
 
 for.body:                                         ; preds = %for.body, %entry
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %indvars.iv
-  %0 = load i32, ptr %arrayidx, align 4, !tbaa !2
-  %1 = trunc i64 %indvars.iv to i32
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %iv
+  %0 = load i32, ptr %arrayidx, align 4
+  %1 = trunc i64 %iv to i32
   %mul = mul nsw i32 %0, %1
-  %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %indvars.iv
-  %2 = load i32, ptr %arrayidx2, align 4, !tbaa !2
+  %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %iv
+  %2 = load i32, ptr %arrayidx2, align 4
   %add = add nsw i32 %2, %mul
-  store i32 %add, ptr %arrayidx2, align 4, !tbaa !2
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 1027
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !7
+  store i32 %add, ptr %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1027
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !1
 }
 
-attributes #0 = { "use-soft-float"="false" }
-
-!llvm.module.flags = !{!0}
-!llvm.ident = !{!1}
-
-; CHECK: [[LP1_255]] = !{!"branch_weights", i32 1, i32 255}
-; CHECK: [[LP0_0]] = !{!"branch_weights", i32 0, i32 0}
-; CHECK-MASKED: [[LP1_63]] = !{!"branch_weights", i32 1, i32 63}
-; CHECK-MASKED: [[LP0_0]] = !{!"branch_weights", i32 0, i32 0}
-; CHECK: [[LP1_2]] = !{!"branch_weights", i32 1, i32 2}
 
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project c292b5b5e059e6ce3e6449e6827ef7e1037c21c4)"}
-!2 = !{!3, !3, i64 0}
-!3 = !{!"int", !4, i64 0}
-!4 = !{!"omnipotent char", !5, i64 0}
-!5 = !{!"Simple C++ TBAA"}
-!6 = !{!"branch_weights", i32 1, i32 1023}
-!7 = !{!"branch_weights", i32 1, i32 1026}
+!0 = !{!"branch_weights", i32 1, i32 1023}
+!1 = !{!"branch_weights", i32 1, i32 1026}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
index 800b6f3f28b7..6bf8883fbf12 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
@@ -44,16 +44,16 @@ define i64 @select_icmp_const(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = select i1 [[TMP14]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX1]]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_MERGE_RDX]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT9:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT8]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT9]], <i64 0, i64 1, i64 2, i64 3>
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND5:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i64> [ [[DOTSPLAT9]], %[[VEC_EPILOG_PH]] ], [ [[TMP11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i64> [ [[DOTSPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
@@ -70,12 +70,12 @@ define i64 @select_icmp_const(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX14:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[L]], 3
@@ -148,16 +148,16 @@ define i64 @select_fcmp_const_fast(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = select i1 [[TMP14]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX1]]
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_MERGE_RDX]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT9:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT8]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT9]], <i64 0, i64 1, i64 2, i64 3>
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND5:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i64> [ [[DOTSPLAT9]], %[[VEC_EPILOG_PH]] ], [ [[TMP11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i64> [ [[DOTSPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP8]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP9]], align 4
@@ -174,12 +174,12 @@ define i64 @select_fcmp_const_fast(ptr %a, i64 %n) {
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 2, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX14:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 2, %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = fcmp fast ueq float [[L]], 3.000000e+00
@@ -261,16 +261,16 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i32 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = trunc i32 [[N_VEC3]] to i8
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[BC_RESUME_VAL]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP12]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i8> [[DOTSPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i8> [[DOTSPLAT]], <i8 0, i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <4 x i8> poison, i8 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <4 x i8> poison, i8 [[BC_RESUME_VAL]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT9:%.*]] = shufflevector <4 x i8> [[DOTSPLATINSERT8]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i8> [[DOTSPLAT9]], <i8 0, i8 1, i8 2, i8 3>
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND5:%.*]] = phi <4 x i8> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i8> [ [[DOTSPLAT9]], %[[VEC_EPILOG_PH]] ], [ [[TMP17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i8> [ [[DOTSPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX4]] to i8
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i8 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
@@ -288,12 +288,12 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
 ; CHECK-NEXT:    [[CMP_N14:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[CMP_N14]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi i8 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX16:%.*]] = phi i8 [ [[RDX_SELECT13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i8 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX14:%.*]] = phi i8 [ [[RDX_SELECT13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i8 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 3
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index 86a75267dd2a..1376a687b38b 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -47,9 +47,9 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_1_P:%.*]]> = phi [ vp<[[RESUME_1]]>, middle.block ], [ ir<22>, ir-bb<entry> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_2_P:%.*]]>.1 = phi [ vp<[[RESUME_2]]>.1, middle.block ], [ ir<33>, ir-bb<entry> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -128,10 +128,10 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_3_P:%.*]]>.2 = resume-phi vp<[[RESUME_3]]>.2, ir<33>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_1_P:%.*]]> = phi [ vp<[[RESUME_1]]>, middle.block ], [ ir<22>, ir-bb<entry> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_2_P:%.*]]>.1 = phi [ vp<[[RESUME_2]]>.1, middle.block ], [ ir<33>, ir-bb<entry> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_3_P:%.*]]>.2 = phi [ vp<[[RESUME_3]]>.2, middle.block ], [ ir<33>, ir-bb<entry> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -213,9 +213,9 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VTC]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_X:%.+]]> = phi [ vp<[[EXT_X]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_Y:%.+]]>.1 = phi [ vp<[[EXT_Y]]>.1, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
@@ -292,9 +292,9 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = resume-phi vp<[[VTC]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.*]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_X:%.+]]> = phi [ vp<[[EXT_X]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_Y:%.+]]>.1 = phi [ vp<[[EXT_Y]]>.1, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
index 89268ac25c34..0c5784b32fc9 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
@@ -1,26 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
 ; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
 
 define i16 @test_chained_first_order_recurrences_1(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrences_1
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    store <4 x i16> [[TMP6]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define i16 @test_chained_first_order_recurrences_1(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    store <4 x i16> [[TMP4]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP8]], label %middle.block, label %vector.body
-; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI2:%.*]] = extractelement <4 x i16> [[TMP2]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP2]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT4:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT3]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT4]], %[[SCALAR_PH]] ], [ [[FOR_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[FOR_1]], [[FOR_2]]
+; CHECK-NEXT:    store i16 [[ADD]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i16 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi i16 [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI2]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = add i16 [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    ret i16 [[RES]]
 ;
 entry:
   br label %loop
@@ -43,26 +71,53 @@ exit:
 }
 
 define i16 @test_chained_first_order_recurrences_2(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrences_2
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    store <4 x i16> [[TMP6]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define i16 @test_chained_first_order_recurrences_2(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    store <4 x i16> [[TMP4]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP8]], label %middle.block, label %vector.body, !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK:      middle.block:
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP2]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI2:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP2]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT4:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT3]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1]] = phi i16 [ [[SCALAR_RECUR_INIT4]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[FOR_1]], [[FOR_2]]
+; CHECK-NEXT:    store i16 [[ADD]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi i16 [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i16 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI2]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = add i16 [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    ret i16 [[RES]]
 ;
 entry:
   br label %loop
@@ -85,31 +140,63 @@ exit:
 }
 
 define i16 @test_chained_first_order_recurrences_3(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrences_3
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP5:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i16> [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i16> [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    store <4 x i16> [[TMP8]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define i16 @test_chained_first_order_recurrences_3(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i16> [[TMP6]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:      middle.block:
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP2]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT5:%.*]] = extractelement <4 x i16> [[TMP2]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT7:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT5]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT6]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT7]], %[[SCALAR_PH]] ], [ [[FOR_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_3:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT8]], %[[SCALAR_PH]] ], [ [[FOR_2]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i16 [[FOR_1]], [[FOR_2]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add i16 [[ADD_1]], [[FOR_3]]
+; CHECK-NEXT:    store i16 [[ADD_2]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i16 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi i16 [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI3]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_3_LCSSA:%.*]] = phi i16 [ [[FOR_3]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES_1:%.*]] = add i16 [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    [[RES_2:%.*]] = add i16 [[RES_1]], [[FOR_3_LCSSA]]
+; CHECK-NEXT:    ret i16 [[RES_2]]
 ;
 entry:
   br label %loop
@@ -135,8 +222,23 @@ exit:
 }
 
 define void @test_cyclic_phis(ptr %ptr) {
-; CHECK-LABEL: @test_cyclic_phis
-; CHECK-NOT: vector.body:
+; CHECK-LABEL: define void @test_cyclic_phis(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i16 [ 22, %[[ENTRY]] ], [ [[FOR_2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2]] = phi i16 [ 33, %[[ENTRY]] ], [ [[FOR_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT:%.*]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[FOR_1]], [[FOR_2]]
+; CHECK-NEXT:    store i16 [[ADD]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -158,19 +260,47 @@ exit:
 }
 
 define void @test_first_order_recurrences_incoming_cycle_preheader(ptr %ptr) {
-; CHECK-LABEL: @test_first_order_recurrences_incoming_cycle_preheader
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP4]], splat (i16 10)
-; CHECK-NEXT:    store <4 x i16> [[TMP5]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define void @test_first_order_recurrences_incoming_cycle_preheader(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    br i1 true, label %[[LOOP_PREHEADER:.*]], label %[[LOOP_1]]
+; CHECK:       [[LOOP_PREHEADER]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i16> [[TMP2]], splat (i16 10)
+; CHECK-NEXT:    store <4 x i16> [[TMP3]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP7]], label %middle.block, label %vector.body
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i16 [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[FOR_1]], 10
+; CHECK-NEXT:    store i16 [[ADD]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop.1
@@ -195,31 +325,63 @@ exit:
 }
 
 define i16 @test_chained_first_order_recurrences_3_reordered_1(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrences_3_reordered_1
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP5:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i16> [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i16> [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    store <4 x i16> [[TMP8]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define i16 @test_chained_first_order_recurrences_3_reordered_1(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i16> [[TMP6]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:      middle.block:
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP2]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT5:%.*]] = extractelement <4 x i16> [[TMP2]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT7:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT5]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT6]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_3:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2]] = phi i16 [ [[SCALAR_RECUR_INIT7]], %[[SCALAR_PH]] ], [ [[FOR_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1]] = phi i16 [ [[SCALAR_RECUR_INIT8]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i16 [[FOR_1]], [[FOR_2]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add i16 [[ADD_1]], [[FOR_3]]
+; CHECK-NEXT:    store i16 [[ADD_2]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_3_LCSSA:%.*]] = phi i16 [ [[FOR_3]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi i16 [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI3]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i16 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES_1:%.*]] = add i16 [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    [[RES_2:%.*]] = add i16 [[RES_1]], [[FOR_3_LCSSA]]
+; CHECK-NEXT:    ret i16 [[RES_2]]
 ;
 entry:
   br label %loop
@@ -245,31 +407,63 @@ exit:
 }
 
 define i16 @test_chained_first_order_recurrences_3_reordered_2(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrences_3_reordered_2
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP5:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i16> [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i16> [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    store <4 x i16> [[TMP8]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define i16 @test_chained_first_order_recurrences_3_reordered_2(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i16> [[TMP6]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:      middle.block:
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[TMP2]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP2]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT5:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT7:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT5]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT6]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_3:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT7]], %[[SCALAR_PH]] ], [ [[FOR_2]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1]] = phi i16 [ [[SCALAR_RECUR_INIT8]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i16 [[FOR_1]], [[FOR_2]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add i16 [[ADD_1]], [[FOR_3]]
+; CHECK-NEXT:    store i16 [[ADD_2]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi i16 [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_3_LCSSA:%.*]] = phi i16 [ [[FOR_3]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI3]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i16 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES_1:%.*]] = add i16 [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    [[RES_2:%.*]] = add i16 [[RES_1]], [[FOR_3_LCSSA]]
+; CHECK-NEXT:    ret i16 [[RES_2]]
 ;
 entry:
   br label %loop
@@ -295,31 +489,63 @@ exit:
 }
 
 define i16 @test_chained_first_order_recurrences_3_for2_no_other_uses(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrences_3_for2_no_other_uses
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP5:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i16> [[TMP4]], splat (i16 10)
-; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i16> [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    store <4 x i16> [[TMP8]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define i16 @test_chained_first_order_recurrences_3_for2_no_other_uses(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP2]], splat (i16 10)
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    store <4 x i16> [[TMP6]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:      middle.block:
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP2]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT5:%.*]] = extractelement <4 x i16> [[TMP2]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT7:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT5]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT6]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT7]], %[[SCALAR_PH]] ], [ [[FOR_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_3:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT8]], %[[SCALAR_PH]] ], [ [[FOR_2]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i16 [[FOR_1]], 10
+; CHECK-NEXT:    [[ADD_2:%.*]] = add i16 [[ADD_1]], [[FOR_3]]
+; CHECK-NEXT:    store i16 [[ADD_2]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i16 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi i16 [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI3]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_3_LCSSA:%.*]] = phi i16 [ [[FOR_3]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES_1:%.*]] = add i16 [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    [[RES_2:%.*]] = add i16 [[RES_1]], [[FOR_3_LCSSA]]
+; CHECK-NEXT:    ret i16 [[RES_2]]
 ;
 entry:
   br label %loop
@@ -345,30 +571,61 @@ exit:
 }
 
 define i16 @test_chained_first_order_recurrences_3_for1_for2_no_other_uses(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrences_3_for1_for2_no_other_uses
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %vector.ph ], [ [[TMP5:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i16> [[TMP6]], splat (i16 10)
-; CHECK-NEXT:    store <4 x i16> [[TMP8]], ptr [[TMP2]], align 2
+; CHECK-LABEL: define i16 @test_chained_first_order_recurrences_3_for1_for2_no_other_uses(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 22>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 33>, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3]] = shufflevector <4 x i16> [[VECTOR_RECUR1]], <4 x i16> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR2]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP4]], splat (i16 10)
+; CHECK-NEXT:    store <4 x i16> [[TMP5]], ptr [[TMP1]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP10]], label %middle.block, label %vector.body, !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:      middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI8:%.*]] = extractelement <4 x i16> [[TMP5]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x i16> [[TMP2]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI4:%.*]] = extractelement <4 x i16> [[TMP3]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i16> [[TMP5]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT5:%.*]] = extractelement <4 x i16> [[TMP2]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <4 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT7:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT5]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT6]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT7]], %[[SCALAR_PH]] ], [ [[FOR_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_3:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT8]], %[[SCALAR_PH]] ], [ [[FOR_2]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i16 [[FOR_3]], 10
+; CHECK-NEXT:    store i16 [[ADD_1]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i16 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi i16 [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI3]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_3_LCSSA:%.*]] = phi i16 [ [[FOR_3]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES_1:%.*]] = add i16 [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    [[RES_2:%.*]] = add i16 [[RES_1]], [[FOR_3_LCSSA]]
+; CHECK-NEXT:    ret i16 [[RES_2]]
 ;
 entry:
   br label %loop
@@ -393,28 +650,56 @@ exit:
 }
 
 define double @test_chained_first_order_recurrence_sink_users_1(ptr %ptr) {
-; CHECK-LABEL: @test_chained_first_order_recurrence_sink_users_1
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 1.000000e+01>, %vector.ph ], [ [[WIDE_LOAD:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 2.000000e+01>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
+; CHECK-LABEL: define double @test_chained_first_order_recurrence_sink_users_1(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 1.000000e+01>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 2.000000e+01>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[PTR:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x double>, ptr [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR1]], <4 x double> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x double> splat (double 1.000000e+01), [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP6]], [[TMP4]]
-; CHECK-NEXT:    store <4 x double> [[TMP7]], ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <4 x double>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR1]], <4 x double> [[TMP2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x double> splat (double 1.000000e+01), [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x double> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    store <4 x double> [[TMP5]], ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
-; CHECK-NEXT:    br i1 [[TMP9]], label %middle.block, label %vector.body, !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[WIDE_LOAD]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI3:%.*]] = extractelement <4 x double> [[TMP4]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI2:%.*]] = extractelement <4 x double> [[TMP2]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[WIDE_LOAD]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x double> [[TMP4]], i32 3
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <4 x double> [[TMP2]], i32 3
+; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1.000000e+01, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT4:%.*]] = phi double [ [[VECTOR_RECUR_EXTRACT3]], %[[MIDDLE_BLOCK]] ], [ 2.000000e+01, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi double [ [[SCALAR_RECUR_INIT4]], %[[SCALAR_PH]] ], [ [[FOR_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd double 1.000000e+01, [[FOR_2]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd double [[ADD_1]], [[FOR_1]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[FOR_1_NEXT]] = load double, ptr [[GEP_PTR]], align 8
+; CHECK-NEXT:    store double [[ADD_2]], ptr [[GEP_PTR]], align 8
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi double [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi double [ [[FOR_2]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI2]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = fadd double [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    ret double [[RES]]
 ;
 entry:
   br label %loop
@@ -438,8 +723,25 @@ exit:
 }
 
 define void @test_first_order_recurrences_and_reduction(ptr %ptr) {
-; CHECK-LABEL: @test_first_order_recurrences_and_reduction(
-; CHECK-NOT:   vector.body:
+; CHECK-LABEL: define void @test_first_order_recurrences_and_reduction(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i16 [ 22, %[[ENTRY]] ], [ [[RED:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED]] = phi i16 [ 33, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[LV:%.*]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[FOR_1_NEXT:%.*]] = load i16, ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i16 [[FOR_1]], 10
+; CHECK-NEXT:    [[RED_NEXT]] = add i16 [[RED]], [[LV]]
+; CHECK-NEXT:    store i16 [[ADD_1]], ptr [[GEP_PTR]], align 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -463,24 +765,46 @@ exit:
 }
 
 define i64 @test_first_order_recurrences_and_induction(ptr %ptr) {
-; CHECK-LABEL: @test_first_order_recurrences_and_induction(
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 22>, %vector.ph ], [ [[VEC_IND:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VEC_IND]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK-LABEL: define i64 @test_first_order_recurrences_and_induction(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 22>, %[[VECTOR_PH]] ], [ [[VEC_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[TMP1]], splat (i64 10)
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
 ; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP5]], label %middle.block, label %vector.body
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
-; CHECK-NEXT:    br i1 true
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[IV:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i64 [[FOR_1]], 10
+; CHECK-NEXT:    store i64 [[ADD_1]], ptr [[GEP_PTR]], align 4
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i64 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[FOR_1_LCSSA]]
+;
 
 entry:
   br label %loop
@@ -502,24 +826,45 @@ exit:
 ; Same as @test_first_order_recurrences_and_induction but with order of phis
 ; flipped.
 define i64 @test_first_order_recurrences_and_induction2(ptr %ptr) {
-; CHECK-LABEL: @test_first_order_recurrences_and_induction2(
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 22>, %vector.ph ], [ [[VEC_IND]], %vector.body ]
+; CHECK-LABEL: define i64 @test_first_order_recurrences_and_induction2(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 22>, %[[VECTOR_PH]] ], [ [[VEC_IND]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[TMP1]], splat (i64 10)
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
 ; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP5]], label %middle.block, label %vector.body
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
-; CHECK-NEXT:    br i1 true
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 22, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i64 [[FOR_1]], 10
+; CHECK-NEXT:    store i64 [[ADD_1]], ptr [[GEP_PTR]], align 4
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi i64 [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[FOR_1_LCSSA]]
 ;
 entry:
   br label %loop
@@ -539,26 +884,50 @@ exit:
 }
 
 define ptr @test_first_order_recurrences_and_pointer_induction1(ptr %ptr) {
-; CHECK-LABEL: @test_first_order_recurrences_and_pointer_induction1(
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 4000
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ <ptr poison, ptr poison, ptr poison, ptr null>, %vector.ph ], [ [[TMP0:%.*]], %vector.body ]
-; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[PTR]], %vector.ph ], [ [[PTR_IND:%.*]], %vector.body ]
+; CHECK-LABEL: define ptr @test_first_order_recurrences_and_pointer_induction1(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[PTR]], i64 4000
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ <ptr poison, ptr poison, ptr poison, ptr null>, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[PTR]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x ptr> [[VECTOR_RECUR]], <4 x ptr> [[TMP0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[PTR]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds ptr, ptr [[TMP3]], i32 0
 ; CHECK-NEXT:    store <4 x ptr> [[TMP0]], ptr [[TMP4]], align 8
+; CHECK-NEXT:    store <4 x ptr> [[TMP1]], ptr [[TMP4]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP5]], label %middle.block, label %vector.body
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x ptr> [[TMP0]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x ptr> [[TMP0]], i32 3
-; CHECK-NEXT:    br i1 true
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi ptr [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ null, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[PTR]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi ptr [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[PTR_IV:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds ptr, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT:    store ptr [[PTR_IV]], ptr [[GEP_PTR]], align 8
+; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i32, ptr [[PTR_IV]], i64 1
+; CHECK-NEXT:    store ptr [[FOR_1]], ptr [[GEP_PTR]], align 8
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi ptr [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret ptr [[FOR_1_LCSSA]]
 ;
 entry:
   br label %loop
@@ -571,6 +940,7 @@ loop:
   %gep.ptr = getelementptr inbounds ptr, ptr %ptr, i64 %iv
   store ptr %ptr.iv, ptr %gep.ptr
   %ptr.iv.next = getelementptr i32, ptr %ptr.iv, i64 1
+  store ptr %for.1, ptr %gep.ptr
   %exitcond.not = icmp eq i64 %iv.next, 1000
   br i1 %exitcond.not, label %exit, label %loop
 
@@ -581,26 +951,50 @@ exit:
 ; same as @test_first_order_recurrences_and_pointer_induction1 but with order
 ; of phis flipped.
 define ptr @test_first_order_recurrences_and_pointer_induction2(ptr %ptr) {
-; CHECK-LABEL: @test_first_order_recurrences_and_pointer_induction2(
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 4000
-; CHECK-NEXT:    br label %vector.body
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[PTR]], %vector.ph ], [ [[PTR_IND:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ <ptr poison, ptr poison, ptr poison, ptr null>, %vector.ph ], [ [[TMP0:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP0]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[PTR]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds ptr, ptr [[TMP3]], i32 0
-; CHECK-NEXT:    store <4 x ptr> [[TMP0]], ptr [[TMP4]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-LABEL: define ptr @test_first_order_recurrences_and_pointer_induction2(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 4000
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[PTR]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x ptr> [ <ptr poison, ptr poison, ptr poison, ptr null>, %[[VECTOR_PH]] ], [ [[VECTOR_GEP:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_GEP]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x ptr> [[VECTOR_RECUR]], <4 x ptr> [[VECTOR_GEP]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds ptr, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    store <4 x ptr> [[VECTOR_GEP]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    store <4 x ptr> [[TMP1]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], 4
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 1000
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x ptr> [[VECTOR_GEP]], i32 2
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x ptr> [[VECTOR_GEP]], i32 3
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[PTR]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi ptr [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ null, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi ptr [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[PTR_IV]], %[[LOOP]] ]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NEXT:    store ptr [[PTR_IV]], ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i32, ptr [[PTR_IV]], i64 1
+; CHECK-NEXT:    store ptr [[FOR_1]], ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP5]], label %middle.block, label %vector.body
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x ptr> [[TMP0]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x ptr> [[TMP0]], i32 3
-; CHECK-NEXT:    br i1 true
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi ptr [ [[FOR_1]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret ptr [[FOR_1_LCSSA]]
 ;
 entry:
   br label %loop
@@ -613,6 +1007,7 @@ loop:
   %gep.ptr = getelementptr inbounds ptr, ptr %ptr, i64 %iv
   store ptr %ptr.iv, ptr %gep.ptr
   %ptr.iv.next = getelementptr i32, ptr %ptr.iv, i64 1
+  store ptr %for.1, ptr %gep.ptr
   %exitcond.not = icmp eq i64 %iv.next, 1000
   br i1 %exitcond.not, label %exit, label %loop
 
@@ -623,39 +1018,64 @@ exit:
 ; In this test case, %USE_2_FORS uses 2 different fixed-order recurrences and
 ; it needs to be sunk past the previous value for both recurrences.
 define double @test_resinking_required(ptr %p, ptr noalias %a, ptr noalias %b) {
-; CHECK-LABEL: @test_resinking_required(
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %vector.ph ], [ [[BROADCAST_SPLAT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %vector.ph ], [ [[BROADCAST_SPLAT4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %vector.ph ], [ [[TMP4:%.*]], %vector.body ]
-; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr %a, align 8
+; CHECK-LABEL: define double @test_resinking_required(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR2:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[A]], align 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fdiv <4 x double> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr %b, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr [[B]], align 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x double> poison, double [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT3]], <4 x double> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4]] = shufflevector <4 x double> [[VECTOR_RECUR1]], <4 x double> [[BROADCAST_SPLAT4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP2]], i32 3
-; CHECK-NEXT:    store double [[TMP6]], ptr [[P:%.*]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR2]], <4 x double> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x double> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP8]], i32 3
+; CHECK-NEXT:    store double [[TMP6]], ptr [[P]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
-; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI10:%.*]] = extractelement <4 x double> [[TMP4]], i32 2
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT9:%.*]] = extractelement <4 x double> [[TMP4]], i32 3
-; CHECK-NEXT:    br i1 true, label %End, label %scalar.ph
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    phi double [ [[TMP0]], %middle.block ], [ 0.000000e+00, %Entry ]
-; CHECK-NEXT:    phi double [ [[TMP3]], %middle.block ], [ 0.000000e+00, %Entry ]
-; CHECK-NEXT:    phi double [ [[VECTOR_RECUR_EXTRACT9]], %middle.block ], [ 0.000000e+00, %Entry ]
-; CHECK-NEXT:    %bc.resume.val = phi i64 [ 1000, %middle.block ], [ 0, %Entry ]
-; CHECK:      End:
-; CHECK-NEXT:    = phi double [ {{.+}}, %Loop ], [ [[TMP0]], %middle.block ]
-; CHECK-NEXT:    = phi double [ {{.+}}, %Loop ], [ [[TMP3]], %middle.block ]
-; CHECK-NEXT:    = phi double [ {{.+}}, %Loop ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI10]], %middle.block ]
+; CHECK-NEXT:    br i1 true, label %[[END:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT5:%.*]] = phi double [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT6:%.*]] = phi double [ [[VECTOR_RECUR_EXTRACT9]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi double [ [[L1:%.*]], %[[LOOP]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi double [ [[L2:%.*]], %[[LOOP]] ], [ [[SCALAR_RECUR_INIT5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[FOR_3:%.*]] = phi double [ [[FOR_2]], %[[LOOP]] ], [ [[SCALAR_RECUR_INIT6]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[USE_2_FORS:%.*]] = fdiv double [[FOR_3]], [[FOR_1]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 0.000000e+00, [[FOR_1]]
+; CHECK-NEXT:    [[L1]] = load double, ptr [[A]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[L2]] = load double, ptr [[B]], align 8
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[DIV]], [[FOR_3]]
+; CHECK-NEXT:    store double [[ADD]], ptr [[P]], align 8
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label %[[END]], label %[[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[FOR_1_LCSSA:%.*]] = phi double [ [[FOR_1]], %[[LOOP]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_2_LCSSA:%.*]] = phi double [ [[FOR_2]], %[[LOOP]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_3_LCSSA:%.*]] = phi double [ [[FOR_3]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES_1:%.*]] = fadd double [[FOR_1_LCSSA]], [[FOR_2_LCSSA]]
+; CHECK-NEXT:    [[RES_2:%.*]] = fadd double [[RES_1]], [[FOR_3_LCSSA]]
+; CHECK-NEXT:    ret double [[RES_2]]
 ;
 Entry:
   br label %Loop
@@ -670,7 +1090,8 @@ Loop:
   %l1 = load double, ptr %a, align 8
   %iv.next= add nuw nsw i64 %iv, 1
   %l2 = load double, ptr %b, align 8
-  store double %div, ptr %p, align 8
+  %add = fadd double %div, %for.3
+  store double %add, ptr %p, align 8
   %cond = icmp eq i64 %iv.next, 1000
   br i1 %cond, label %End, label %Loop
 
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll
index 6b7736fa9f61..53113b2bdf49 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll
@@ -11,13 +11,12 @@ define float @for_load_interleave_only(ptr %src) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT:    [[TMP3]] = load float, ptr [[NEXT_GEP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[NEXT_GEP2]], align 4
 ; CHECK-NEXT:    store float 0.000000e+00, ptr [[NEXT_GEP]], align 4
 ; CHECK-NEXT:    store float 0.000000e+00, ptr [[NEXT_GEP2]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -27,12 +26,12 @@ define float @for_load_interleave_only(ptr %src) {
 ; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1001, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[ENTRY]] ]
 ; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi float [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[FOR:%.*]] = phi float [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 16
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 0b5074b3a130..9d2719a6153c 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -189,6 +189,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<1234>, ir<-1>, ir<1>
 ; CHECK-NEXT:   WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
@@ -196,7 +197,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-NEXT: vector.body:
 ; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
 ; CHECK-NEXT:   FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
-; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
+; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%and.red> = phi vp<[[RDX_START]]>, ir<%and.red.next>
 ; CHECK-NEXT:   EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]>, vp<[[BTC]]>
 ; CHECK-NEXT:   EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next>
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 594b8ff70feb..13dc53559d28 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -870,7 +870,7 @@ for.end:
 ; }
 ;
 ;
-define i32 @PR27246() {
+define i32 @PR27246(ptr %dst) {
 ; UNROLL-NO-IC-LABEL: @PR27246(
 ; UNROLL-NO-IC-NEXT:  entry:
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
@@ -882,21 +882,23 @@ define i32 @PR27246() {
 ; UNROLL-NO-IC:       vector.ph:
 ; UNROLL-NO-IC-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[I_016]], 8
 ; UNROLL-NO-IC-NEXT:    [[N_VEC:%.*]] = sub i32 [[I_016]], [[N_MOD_VF]]
-; UNROLL-NO-IC-NEXT:    [[IND_END:%.*]] = sub i32 [[I_016]], [[N_VEC]]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[E_015]], i32 3
+; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = sub i32 [[I_016]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[I_016]]
 ; UNROLL-NO-IC-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[I_016]], i64 0
 ; UNROLL-NO-IC-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; UNROLL-NO-IC-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3>
 ; UNROLL-NO-IC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
+; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
+; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP3]], ptr [[TMP1]], align 4
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 -4)
-; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; UNROLL-NO-IC:       middle.block:
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 2
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 3
@@ -904,19 +906,21 @@ define i32 @PR27246() {
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
 ; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ]
-; UNROLL-NO-IC-NEXT:    br label [[FOR_COND1:%.*]]
+; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ]
+; UNROLL-NO-IC-NEXT:    br label [[FOR_COND2:%.*]]
 ; UNROLL-NO-IC:       for.cond.cleanup:
 ; UNROLL-NO-IC-NEXT:    [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ]
 ; UNROLL-NO-IC-NEXT:    ret i32 [[E_1_LCSSA_LCSSA]]
 ; UNROLL-NO-IC:       for.cond1:
-; UNROLL-NO-IC-NEXT:    [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; UNROLL-NO-IC-NEXT:    [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_016]]
 ; UNROLL-NO-IC-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1
 ; UNROLL-NO-IC-NEXT:    [[DEC]] = add nsw i32 [[K_0]], -1
-; UNROLL-NO-IC-NEXT:    br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP9:![0-9]+]]
+; UNROLL-NO-IC-NEXT:    store i32 [[E_1]], ptr [[GEP_DST]], align 4
+; UNROLL-NO-IC-NEXT:    br i1 [[CMP2]], label [[FOR_COND2]], label [[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP9:![0-9]+]]
 ; UNROLL-NO-IC:       for.cond.cleanup3:
-; UNROLL-NO-IC-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-IC-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND2]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    [[INC]] = add nuw nsw i32 [[I_016]], 1
 ; UNROLL-NO-IC-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49
 ; UNROLL-NO-IC-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
@@ -932,34 +936,38 @@ define i32 @PR27246() {
 ; UNROLL-NO-VF:       vector.ph:
 ; UNROLL-NO-VF-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[I_016]], 2
 ; UNROLL-NO-VF-NEXT:    [[N_VEC:%.*]] = sub i32 [[I_016]], [[N_MOD_VF]]
-; UNROLL-NO-VF-NEXT:    [[IND_END:%.*]] = sub i32 [[I_016]], [[N_VEC]]
-; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; UNROLL-NO-VF-NEXT:    [[TMP0:%.*]] = sub i32 [[I_016]], [[N_VEC]]
+; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[I_016]]
+; UNROLL-NO-VF-NEXT:    br label [[FOR_COND1:%.*]]
 ; UNROLL-NO-VF:       vector.body:
-; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ [[E_015]], [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_COND1]] ]
 ; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[I_016]], [[INDEX]]
-; UNROLL-NO-VF-NEXT:    [[TMP1]] = add i32 [[OFFSET_IDX]], -1
+; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 0
+; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
+; UNROLL-NO-VF-NEXT:    store i32 [[TMP2]], ptr [[TMP1]], align 4
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NO-VF-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_COND1]], !llvm.loop [[LOOP8:![0-9]+]]
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ]
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ]
-; UNROLL-NO-VF-NEXT:    br label [[FOR_COND1:%.*]]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ]
+; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ]
+; UNROLL-NO-VF-NEXT:    br label [[FOR_COND2:%.*]]
 ; UNROLL-NO-VF:       for.cond.cleanup:
 ; UNROLL-NO-VF-NEXT:    [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ]
 ; UNROLL-NO-VF-NEXT:    ret i32 [[E_1_LCSSA_LCSSA]]
 ; UNROLL-NO-VF:       for.cond1:
-; UNROLL-NO-VF-NEXT:    [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; UNROLL-NO-VF-NEXT:    [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_016]]
 ; UNROLL-NO-VF-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1
 ; UNROLL-NO-VF-NEXT:    [[DEC]] = add nsw i32 [[K_0]], -1
-; UNROLL-NO-VF-NEXT:    br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP9:![0-9]+]]
+; UNROLL-NO-VF-NEXT:    store i32 [[E_1]], ptr [[GEP_DST]], align 4
+; UNROLL-NO-VF-NEXT:    br i1 [[CMP2]], label [[FOR_COND2]], label [[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP9:![0-9]+]]
 ; UNROLL-NO-VF:       for.cond.cleanup3:
-; UNROLL-NO-VF-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ], [ [[OFFSET_IDX]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-VF-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND2]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-VF-NEXT:    [[INC]] = add nuw nsw i32 [[I_016]], 1
 ; UNROLL-NO-VF-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49
 ; UNROLL-NO-VF-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
@@ -975,20 +983,24 @@ define i32 @PR27246() {
 ; SINK-AFTER:       vector.ph:
 ; SINK-AFTER-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[I_016]], 4
 ; SINK-AFTER-NEXT:    [[N_VEC:%.*]] = sub i32 [[I_016]], [[N_MOD_VF]]
-; SINK-AFTER-NEXT:    [[IND_END:%.*]] = sub i32 [[I_016]], [[N_VEC]]
+; SINK-AFTER-NEXT:    [[TMP0:%.*]] = sub i32 [[I_016]], [[N_VEC]]
+; SINK-AFTER-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[I_016]]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[E_015]], i32 3
 ; SINK-AFTER-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[I_016]], i64 0
 ; SINK-AFTER-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SINK-AFTER-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 -1, i32 -2, i32 -3>
-; SINK-AFTER-NEXT:    br label [[VECTOR_BODY:%.*]]
+; SINK-AFTER-NEXT:    br label [[FOR_COND1:%.*]]
 ; SINK-AFTER:       vector.body:
-; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VEC_IND]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_COND1]] ]
+; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[FOR_COND1]] ]
+; SINK-AFTER-NEXT:    [[VEC_IND]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_COND1]] ]
+; SINK-AFTER-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
+; SINK-AFTER-NEXT:    store i32 [[TMP3]], ptr [[TMP1]], align 4
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
-; SINK-AFTER-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SINK-AFTER-NEXT:    br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SINK-AFTER-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SINK-AFTER-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_COND1]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SINK-AFTER:       middle.block:
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[VEC_IND]], i32 2
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[VEC_IND]], i32 3
@@ -996,19 +1008,21 @@ define i32 @PR27246() {
 ; SINK-AFTER-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
 ; SINK-AFTER:       scalar.ph:
 ; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[E_015]], [[FOR_COND1_PREHEADER]] ]
-; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ]
-; SINK-AFTER-NEXT:    br label [[FOR_COND1:%.*]]
+; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ]
+; SINK-AFTER-NEXT:    br label [[FOR_COND2:%.*]]
 ; SINK-AFTER:       for.cond.cleanup:
 ; SINK-AFTER-NEXT:    [[E_1_LCSSA_LCSSA:%.*]] = phi i32 [ [[E_1_LCSSA]], [[FOR_COND_CLEANUP3]] ]
 ; SINK-AFTER-NEXT:    ret i32 [[E_1_LCSSA_LCSSA]]
 ; SINK-AFTER:       for.cond1:
-; SINK-AFTER-NEXT:    [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND1]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; SINK-AFTER-NEXT:    [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND1]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[E_1:%.*]] = phi i32 [ [[K_0:%.*]], [[FOR_COND2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[K_0]] = phi i32 [ [[DEC:%.*]], [[FOR_COND2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_016]]
 ; SINK-AFTER-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[K_0]], 1
 ; SINK-AFTER-NEXT:    [[DEC]] = add nsw i32 [[K_0]], -1
-; SINK-AFTER-NEXT:    br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP9:![0-9]+]]
+; SINK-AFTER-NEXT:    store i32 [[E_1]], ptr [[GEP_DST]], align 4
+; SINK-AFTER-NEXT:    br i1 [[CMP2]], label [[FOR_COND2]], label [[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP9:![0-9]+]]
 ; SINK-AFTER:       for.cond.cleanup3:
-; SINK-AFTER-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND1]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]
+; SINK-AFTER-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[E_1]], [[FOR_COND2]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]
 ; SINK-AFTER-NEXT:    [[INC]] = add nuw nsw i32 [[I_016]], 1
 ; SINK-AFTER-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49
 ; SINK-AFTER-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
@@ -1028,8 +1042,10 @@ for.cond.cleanup:
 for.cond1:
   %e.1 = phi i32 [ %k.0, %for.cond1 ], [ %e.015, %for.cond1.preheader ]
   %k.0 = phi i32 [ %dec, %for.cond1 ], [ %i.016, %for.cond1.preheader ]
+  %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %i.016
   %cmp2 = icmp sgt i32 %k.0, 1
   %dec = add nsw i32 %k.0, -1
+  store i32 %e.1, ptr %gep.dst
   br i1 %cmp2, label %for.cond1, label %for.cond.cleanup3
 
 for.cond.cleanup3:
@@ -1052,32 +1068,26 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-IC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8
 ; UNROLL-NO-IC-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; UNROLL-NO-IC-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 2
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD:%.*]], i32 3
 ; UNROLL-NO-IC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 8
-; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 10
-; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 12
-; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 14
-; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 2
+; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8
+; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 10
+; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 12
+; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 14
 ; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[TMP4]], 2
 ; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[TMP5]], 2
 ; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 2
-; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP7]]
-; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
+; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = add nuw nsw i64 [[TMP7]], 2
+; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP8]]
 ; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
 ; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
-; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP11]], align 4
+; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
 ; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP12]], align 4
-; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
-; UNROLL-NO-IC-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
-; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP15]], i32 0
-; UNROLL-NO-IC-NEXT:    [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 1
-; UNROLL-NO-IC-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i32 2
-; UNROLL-NO-IC-NEXT:    [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i32 3
+; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP13]], align 4
+; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP14]], align 4
+; UNROLL-NO-IC-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP15]], align 4
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -1086,7 +1096,7 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
 ; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], [[ENTRY]] ]
+; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ [[PRE_LOAD:%.*]], [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[SCALAR_BODY:%.*]]
 ; UNROLL-NO-IC:       scalar.body:
 ; UNROLL-NO-IC-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
@@ -1114,7 +1124,6 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
 ; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ [[PRE_LOAD:%.*]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 2
@@ -1122,7 +1131,7 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-VF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP5]]
 ; UNROLL-NO-VF-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
 ; UNROLL-NO-VF-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4
-; UNROLL-NO-VF-NEXT:    [[TMP10]] = load i32, ptr [[TMP8]], align 4
+; UNROLL-NO-VF-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP8]], align 4
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -1131,7 +1140,7 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-VF-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
 ; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], [[ENTRY]] ]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[PRE_LOAD:%.*]], [[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[SCALAR_BODY:%.*]]
 ; UNROLL-NO-VF:       scalar.body:
 ; UNROLL-NO-VF-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
@@ -1156,32 +1165,26 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; SINK-AFTER-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
 ; SINK-AFTER-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; SINK-AFTER-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 2
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD:%.*]], i32 3
 ; SINK-AFTER-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SINK-AFTER:       vector.body:
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; SINK-AFTER-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0
-; SINK-AFTER-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2
-; SINK-AFTER-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4
-; SINK-AFTER-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6
-; SINK-AFTER-NEXT:    [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 2
+; SINK-AFTER-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 0
+; SINK-AFTER-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 2
+; SINK-AFTER-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 4
+; SINK-AFTER-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; SINK-AFTER-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[TMP4]], 2
 ; SINK-AFTER-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[TMP5]], 2
 ; SINK-AFTER-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 2
-; SINK-AFTER-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP7]]
-; SINK-AFTER-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
+; SINK-AFTER-NEXT:    [[TMP11:%.*]] = add nuw nsw i64 [[TMP7]], 2
+; SINK-AFTER-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP8]]
 ; SINK-AFTER-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
 ; SINK-AFTER-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
-; SINK-AFTER-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP11]], align 4
+; SINK-AFTER-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
 ; SINK-AFTER-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP12]], align 4
-; SINK-AFTER-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
-; SINK-AFTER-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
-; SINK-AFTER-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP15]], i32 0
-; SINK-AFTER-NEXT:    [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP16]], i32 1
-; SINK-AFTER-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i32 2
-; SINK-AFTER-NEXT:    [[TMP22]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i32 3
+; SINK-AFTER-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP13]], align 4
+; SINK-AFTER-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP14]], align 4
+; SINK-AFTER-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP15]], align 4
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -1190,7 +1193,7 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; SINK-AFTER-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; SINK-AFTER:       scalar.ph:
 ; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], [[ENTRY]] ]
+; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ [[PRE_LOAD:%.*]], [[ENTRY]] ]
 ; SINK-AFTER-NEXT:    br label [[SCALAR_BODY:%.*]]
 ; SINK-AFTER:       scalar.body:
 ; SINK-AFTER-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
@@ -1229,7 +1232,6 @@ define i64 @constant_folded_previous_value() {
 ; UNROLL-NO-IC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ splat (i64 1), [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1257,7 +1259,6 @@ define i64 @constant_folded_previous_value() {
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
 ; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ 1, [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1285,7 +1286,6 @@ define i64 @constant_folded_previous_value() {
 ; SINK-AFTER-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SINK-AFTER:       vector.body:
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ splat (i64 1), [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
 ; SINK-AFTER-NEXT:    br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1338,9 +1338,8 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
-; UNROLL-NO-IC-NEXT:    [[TMP0]] = add <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT]]
+; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT]]
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
 ; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
@@ -1371,30 +1370,29 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; UNROLL-NO-VF:       vector.ph:
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
-; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i32 [[INDEX]], [[X:%.*]]
-; UNROLL-NO-VF-NEXT:    [[TMP3]] = add i32 [[TMP1]], [[X]]
-; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; UNROLL-NO-VF-NEXT:    [[TMP0:%.*]] = add i32 [[VECTOR_RECUR]], 1
+; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = add i32 [[VECTOR_RECUR]], [[X:%.*]]
+; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i32 [[TMP0]], [[X]]
+; UNROLL-NO-VF-NEXT:    [[TMP3]] = add nuw i32 [[VECTOR_RECUR]], 2
+; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 96
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT1:%.*]] = phi i32 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-VF:       for.body:
-; UNROLL-NO-VF-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[ADDX:%.*]], [[FOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[INC]] = add i32 [[INC_PHI]], 1
-; UNROLL-NO-VF-NEXT:    [[BC:%.*]] = zext i32 [[INC_PHI]] to i64
-; UNROLL-NO-VF-NEXT:    [[ADDX]] = add i32 [[INC_PHI]], [[X]]
-; UNROLL-NO-VF-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC_PHI]], 95
+; UNROLL-NO-VF-NEXT:    [[VAL_PHI1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT1]], [[SCALAR_PH]] ], [ [[ADDX1:%.*]], [[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[ADDX]] = add i32 [[VAL_PHI]], 1
+; UNROLL-NO-VF-NEXT:    [[BC:%.*]] = zext i32 [[VAL_PHI]] to i64
+; UNROLL-NO-VF-NEXT:    [[ADDX1]] = add i32 [[VAL_PHI]], [[X]]
+; UNROLL-NO-VF-NEXT:    [[CMP:%.*]] = icmp eq i32 [[VAL_PHI]], 95
 ; UNROLL-NO-VF-NEXT:    br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; UNROLL-NO-VF:       for.end:
-; UNROLL-NO-VF-NEXT:    [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[VAL_PHI]], [[FOR_BODY]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-VF-NEXT:    [[VAL_PHI_LCSSA:%.*]] = phi i32 [ [[VAL_PHI1]], [[FOR_BODY]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-VF-NEXT:    ret i32 [[VAL_PHI_LCSSA]]
 ;
 ; SINK-AFTER-LABEL: @extract_second_last_iteration(
@@ -1407,8 +1405,7 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; SINK-AFTER:       vector.body:
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[TMP0]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SINK-AFTER-NEXT:    [[TMP0:%.*]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; SINK-AFTER-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
@@ -2488,13 +2485,12 @@ define void @sink_dead_inst(ptr %a) {
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 -27>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
 ; UNROLL-NO-IC-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
 ; UNROLL-NO-IC-NEXT:    [[OFFSET_IDX:%.*]] = add i16 -27, [[DOTCAST]]
 ; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 1)
 ; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 1)
-; UNROLL-NO-IC-NEXT:    [[TMP3]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
 ; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = add <4 x i16> [[TMP1]], splat (i16 5)
 ; UNROLL-NO-IC-NEXT:    [[TMP5]] = add <4 x i16> [[TMP2]], splat (i16 5)
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
@@ -2542,13 +2538,12 @@ define void @sink_dead_inst(ptr %a) {
 ; UNROLL-NO-VF:       vector.body:
 ; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR1:%.*]] = phi i32 [ -27, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
 ; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = add i16 -27, [[DOTCAST]]
 ; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1
 ; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], 1
 ; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = add i16 [[TMP1]], 1
-; UNROLL-NO-VF-NEXT:    [[TMP4]] = zext i16 [[TMP3]] to i32
+; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; UNROLL-NO-VF-NEXT:    [[TMP5:%.*]] = add i16 [[TMP2]], 5
 ; UNROLL-NO-VF-NEXT:    [[TMP6]] = add i16 [[TMP3]], 5
 ; UNROLL-NO-VF-NEXT:    [[TMP7:%.*]] = sub i16 [[VECTOR_RECUR]], 10
@@ -2591,11 +2586,10 @@ define void @sink_dead_inst(ptr %a) {
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR1:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 -27>, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
 ; SINK-AFTER-NEXT:    [[OFFSET_IDX:%.*]] = add i16 -27, [[DOTCAST]]
 ; SINK-AFTER-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 1)
-; SINK-AFTER-NEXT:    [[TMP2]] = zext <4 x i16> [[TMP1]] to <4 x i32>
+; SINK-AFTER-NEXT:    [[TMP2:%.*]] = zext <4 x i16> [[TMP1]] to <4 x i32>
 ; SINK-AFTER-NEXT:    [[TMP3]] = add <4 x i16> [[TMP1]], splat (i16 5)
 ; SINK-AFTER-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP5:%.*]] = sub <4 x i16> [[TMP4]], splat (i16 10)
@@ -2650,7 +2644,7 @@ for.end:
   ret void
 }
 
-define i32 @sink_into_replication_region(i32 %y) {
+define i32 @sink_into_replication_region(i32 %y, ptr %dst) {
 ; UNROLL-NO-IC-LABEL: @sink_into_replication_region(
 ; UNROLL-NO-IC-NEXT:  bb:
 ; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add i32 [[Y:%.*]], 1
@@ -2741,18 +2735,74 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-IC:       pred.udiv.continue18:
 ; UNROLL-NO-IC-NEXT:    [[TMP38:%.*]] = phi <4 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP37]], [[PRED_UDIV_IF17]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP39:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP39]], label [[PRED_UDIV_IF19:%.*]], label [[PRED_UDIV_CONTINUE20]]
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP39]], label [[PRED_UDIV_IF19:%.*]], label [[PRED_UDIV_CONTINUE21:%.*]]
 ; UNROLL-NO-IC:       pred.udiv.if19:
 ; UNROLL-NO-IC-NEXT:    [[TMP40:%.*]] = add i32 [[OFFSET_IDX]], -7
 ; UNROLL-NO-IC-NEXT:    [[TMP41:%.*]] = udiv i32 219220132, [[TMP40]]
 ; UNROLL-NO-IC-NEXT:    [[TMP42:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP41]], i32 3
-; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE20]]
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE21]]
 ; UNROLL-NO-IC:       pred.udiv.continue20:
 ; UNROLL-NO-IC-NEXT:    [[TMP43]] = phi <4 x i32> [ [[TMP38]], [[PRED_UDIV_CONTINUE18]] ], [ [[TMP42]], [[PRED_UDIV_IF19]] ]
 ; UNROLL-NO-IC-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP23]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[TMP45:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> [[TMP43]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[TMP46]] = add <4 x i32> [[VEC_PHI]], [[TMP44]]
 ; UNROLL-NO-IC-NEXT:    [[TMP47]] = add <4 x i32> [[VEC_PHI1]], [[TMP45]]
+; UNROLL-NO-IC-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP64]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; UNROLL-NO-IC:       pred.store.if:
+; UNROLL-NO-IC-NEXT:    [[TMP65:%.*]] = extractelement <4 x i32> [[TMP44]], i32 0
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP65]], ptr [[DST:%.*]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; UNROLL-NO-IC:       pred.store.continue:
+; UNROLL-NO-IC-NEXT:    [[TMP66:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP66]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
+; UNROLL-NO-IC:       pred.store.if21:
+; UNROLL-NO-IC-NEXT:    [[TMP67:%.*]] = extractelement <4 x i32> [[TMP44]], i32 1
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP67]], ptr [[DST]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE22]]
+; UNROLL-NO-IC:       pred.store.continue22:
+; UNROLL-NO-IC-NEXT:    [[TMP52:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP52]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
+; UNROLL-NO-IC:       pred.store.if23:
+; UNROLL-NO-IC-NEXT:    [[TMP53:%.*]] = extractelement <4 x i32> [[TMP44]], i32 2
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP53]], ptr [[DST]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE24]]
+; UNROLL-NO-IC:       pred.store.continue24:
+; UNROLL-NO-IC-NEXT:    [[TMP54:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP54]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
+; UNROLL-NO-IC:       pred.store.if25:
+; UNROLL-NO-IC-NEXT:    [[TMP55:%.*]] = extractelement <4 x i32> [[TMP44]], i32 3
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP55]], ptr [[DST]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE26]]
+; UNROLL-NO-IC:       pred.store.continue26:
+; UNROLL-NO-IC-NEXT:    [[TMP56:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP56]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
+; UNROLL-NO-IC:       pred.store.if27:
+; UNROLL-NO-IC-NEXT:    [[TMP57:%.*]] = extractelement <4 x i32> [[TMP45]], i32 0
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP57]], ptr [[DST]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE28]]
+; UNROLL-NO-IC:       pred.store.continue28:
+; UNROLL-NO-IC-NEXT:    [[TMP58:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP58]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30:%.*]]
+; UNROLL-NO-IC:       pred.store.if29:
+; UNROLL-NO-IC-NEXT:    [[TMP59:%.*]] = extractelement <4 x i32> [[TMP45]], i32 1
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP59]], ptr [[DST]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE30]]
+; UNROLL-NO-IC:       pred.store.continue30:
+; UNROLL-NO-IC-NEXT:    [[TMP60:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP60]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]]
+; UNROLL-NO-IC:       pred.store.if31:
+; UNROLL-NO-IC-NEXT:    [[TMP61:%.*]] = extractelement <4 x i32> [[TMP45]], i32 2
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP61]], ptr [[DST]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_STORE_CONTINUE32]]
+; UNROLL-NO-IC:       pred.store.continue32:
+; UNROLL-NO-IC-NEXT:    [[TMP62:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP62]], label [[PRED_STORE_IF33:%.*]], label [[PRED_UDIV_CONTINUE20]]
+; UNROLL-NO-IC:       pred.store.if33:
+; UNROLL-NO-IC-NEXT:    [[TMP63:%.*]] = extractelement <4 x i32> [[TMP45]], i32 3
+; UNROLL-NO-IC-NEXT:    store i32 [[TMP63]], ptr [[DST]], align 4
+; UNROLL-NO-IC-NEXT:    br label [[PRED_UDIV_CONTINUE20]]
+; UNROLL-NO-IC:       pred.store.continue34:
 ; UNROLL-NO-IC-NEXT:    [[TMP48:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP46]], <4 x i32> [[VEC_PHI]]
 ; UNROLL-NO-IC-NEXT:    [[TMP49:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP47]], <4 x i32> [[VEC_PHI1]]
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -2777,6 +2827,7 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-IC-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; UNROLL-NO-IC-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
 ; UNROLL-NO-IC-NEXT:    [[VAR8]] = add nsw i32 [[VAR3]], -1
+; UNROLL-NO-IC-NEXT:    store i32 [[VAR4]], ptr [[DST]], align 4
 ; UNROLL-NO-IC-NEXT:    [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2
 ; UNROLL-NO-IC-NEXT:    br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF29:![0-9]+]], !llvm.loop [[LOOP30:![0-9]+]]
 ;
@@ -2808,15 +2859,25 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-VF-NEXT:    br label [[PRED_UDIV_CONTINUE]]
 ; UNROLL-NO-VF:       pred.udiv.continue:
 ; UNROLL-NO-VF-NEXT:    [[TMP6:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
-; UNROLL-NO-VF-NEXT:    br i1 [[TMP3]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP3]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE5:%.*]]
 ; UNROLL-NO-VF:       pred.udiv.if3:
 ; UNROLL-NO-VF-NEXT:    [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -1
 ; UNROLL-NO-VF-NEXT:    [[TMP8:%.*]] = udiv i32 219220132, [[TMP7]]
-; UNROLL-NO-VF-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
+; UNROLL-NO-VF-NEXT:    br label [[PRED_UDIV_CONTINUE5]]
 ; UNROLL-NO-VF:       pred.udiv.continue4:
 ; UNROLL-NO-VF-NEXT:    [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
 ; UNROLL-NO-VF-NEXT:    [[TMP10]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
 ; UNROLL-NO-VF-NEXT:    [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP6]]
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; UNROLL-NO-VF:       pred.store.if:
+; UNROLL-NO-VF-NEXT:    store i32 [[VECTOR_RECUR]], ptr [[DST:%.*]], align 4
+; UNROLL-NO-VF-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; UNROLL-NO-VF:       pred.store.continue:
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF5:%.*]], label [[PRED_UDIV_CONTINUE4]]
+; UNROLL-NO-VF:       pred.store.if5:
+; UNROLL-NO-VF-NEXT:    store i32 [[TMP6]], ptr [[DST]], align 4
+; UNROLL-NO-VF-NEXT:    br label [[PRED_UDIV_CONTINUE4]]
+; UNROLL-NO-VF:       pred.store.continue6:
 ; UNROLL-NO-VF-NEXT:    [[TMP12:%.*]] = select i1 [[TMP2]], i32 [[TMP10]], i32 [[VEC_PHI]]
 ; UNROLL-NO-VF-NEXT:    [[TMP13:%.*]] = select i1 [[TMP3]], i32 [[TMP11]], i32 [[VEC_PHI1]]
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
@@ -2840,6 +2901,7 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-VF-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; UNROLL-NO-VF-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
 ; UNROLL-NO-VF-NEXT:    [[VAR8]] = add nsw i32 [[VAR3]], -1
+; UNROLL-NO-VF-NEXT:    store i32 [[VAR4]], ptr [[DST]], align 4
 ; UNROLL-NO-VF-NEXT:    [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2
 ; UNROLL-NO-VF-NEXT:    br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF29:![0-9]+]], !llvm.loop [[LOOP30:![0-9]+]]
 ;
@@ -2894,16 +2956,44 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; SINK-AFTER:       pred.udiv.continue6:
 ; SINK-AFTER-NEXT:    [[TMP17:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ]
 ; SINK-AFTER-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
-; SINK-AFTER-NEXT:    br i1 [[TMP18]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
+; SINK-AFTER-NEXT:    br i1 [[TMP18]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE9:%.*]]
 ; SINK-AFTER:       pred.udiv.if7:
 ; SINK-AFTER-NEXT:    [[TMP19:%.*]] = add i32 [[OFFSET_IDX]], -3
 ; SINK-AFTER-NEXT:    [[TMP20:%.*]] = udiv i32 219220132, [[TMP19]]
 ; SINK-AFTER-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP20]], i32 3
-; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE8]]
+; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE9]]
 ; SINK-AFTER:       pred.udiv.continue8:
 ; SINK-AFTER-NEXT:    [[TMP22]] = phi <4 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ]
 ; SINK-AFTER-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP22]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]]
+; SINK-AFTER-NEXT:    [[TMP33:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
+; SINK-AFTER-NEXT:    br i1 [[TMP33]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; SINK-AFTER:       pred.store.if:
+; SINK-AFTER-NEXT:    [[TMP34:%.*]] = extractelement <4 x i32> [[TMP23]], i32 0
+; SINK-AFTER-NEXT:    store i32 [[TMP34]], ptr [[DST:%.*]], align 4
+; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; SINK-AFTER:       pred.store.continue:
+; SINK-AFTER-NEXT:    [[TMP35:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
+; SINK-AFTER-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
+; SINK-AFTER:       pred.store.if9:
+; SINK-AFTER-NEXT:    [[TMP28:%.*]] = extractelement <4 x i32> [[TMP23]], i32 1
+; SINK-AFTER-NEXT:    store i32 [[TMP28]], ptr [[DST]], align 4
+; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE10]]
+; SINK-AFTER:       pred.store.continue10:
+; SINK-AFTER-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; SINK-AFTER-NEXT:    br i1 [[TMP29]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]
+; SINK-AFTER:       pred.store.if11:
+; SINK-AFTER-NEXT:    [[TMP30:%.*]] = extractelement <4 x i32> [[TMP23]], i32 2
+; SINK-AFTER-NEXT:    store i32 [[TMP30]], ptr [[DST]], align 4
+; SINK-AFTER-NEXT:    br label [[PRED_STORE_CONTINUE12]]
+; SINK-AFTER:       pred.store.continue12:
+; SINK-AFTER-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
+; SINK-AFTER-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF13:%.*]], label [[PRED_UDIV_CONTINUE8]]
+; SINK-AFTER:       pred.store.if13:
+; SINK-AFTER-NEXT:    [[TMP32:%.*]] = extractelement <4 x i32> [[TMP23]], i32 3
+; SINK-AFTER-NEXT:    store i32 [[TMP32]], ptr [[DST]], align 4
+; SINK-AFTER-NEXT:    br label [[PRED_UDIV_CONTINUE8]]
+; SINK-AFTER:       pred.store.continue14:
 ; SINK-AFTER-NEXT:    [[TMP25:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP24]], <4 x i32> [[VEC_PHI]]
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -2926,6 +3016,7 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; SINK-AFTER-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; SINK-AFTER-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
 ; SINK-AFTER-NEXT:    [[VAR8]] = add nsw i32 [[VAR3]], -1
+; SINK-AFTER-NEXT:    store i32 [[VAR4]], ptr [[DST]], align 4
 ; SINK-AFTER-NEXT:    [[VAR9:%.*]] = icmp slt i32 [[VAR3]], 2
 ; SINK-AFTER-NEXT:    br i1 [[VAR9]], label [[BB1]], label [[BB2]], !prof [[PROF29:![0-9]+]], !llvm.loop [[LOOP30:![0-9]+]]
 ;
@@ -2943,6 +3034,7 @@ bb:
   %var6 = add i32 %var5, %var4
   %var7 = udiv i32 219220132, %var3
   %var8 = add nsw i32 %var3, -1
+  store i32 %var4, ptr %dst
   %var9 = icmp slt i32 %var3, 2
   br i1 %var9, label %bb1, label %bb2, !prof !2
 }
@@ -3383,12 +3475,11 @@ define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
 ; UNROLL-NO-IC-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
 ; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 1)
 ; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = or <4 x i16> [[TMP1]], [[TMP1]]
-; UNROLL-NO-IC-NEXT:    [[TMP3]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
 ; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[A_PTR:%.*]], i16 [[OFFSET_IDX]]
 ; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], i32 0
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[TMP4]], i32 4
@@ -3430,28 +3521,27 @@ define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; UNROLL-NO-VF:       vector.ph:
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
-; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
+; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[VECTOR_RECUR]] to i16
 ; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1
-; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], 1
-; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = add i16 [[TMP1]], 1
+; UNROLL-NO-VF-NEXT:    [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 1
+; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i16 [[TMP1]], 1
+; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = or i16 [[TMP5]], [[TMP5]]
 ; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = or i16 [[TMP2]], [[TMP2]]
-; UNROLL-NO-VF-NEXT:    [[TMP5:%.*]] = or i16 [[TMP3]], [[TMP3]]
-; UNROLL-NO-VF-NEXT:    [[TMP6:%.*]] = zext i16 [[TMP4]] to i32
-; UNROLL-NO-VF-NEXT:    [[TMP7]] = zext i16 [[TMP5]] to i32
+; UNROLL-NO-VF-NEXT:    [[TMP6:%.*]] = zext i16 [[TMP3]] to i32
+; UNROLL-NO-VF-NEXT:    [[TMP10:%.*]] = zext i16 [[TMP4]] to i32
 ; UNROLL-NO-VF-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[A_PTR:%.*]], i16 [[OFFSET_IDX]]
 ; UNROLL-NO-VF-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[A_PTR]], i16 [[TMP1]]
 ; UNROLL-NO-VF-NEXT:    store i32 0, ptr [[TMP8]], align 4
 ; UNROLL-NO-VF-NEXT:    store i32 0, ptr [[TMP9]], align 4
-; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
-; UNROLL-NO-VF-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; UNROLL-NO-VF-NEXT:    [[TMP7]] = add nuw i32 [[VECTOR_RECUR]], 2
+; UNROLL-NO-VF-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP7]], 16
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
 ; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-VF:       loop:
 ; UNROLL-NO-VF-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -3479,11 +3569,10 @@ define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; SINK-AFTER:       vector.body:
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
 ; SINK-AFTER-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 1)
 ; SINK-AFTER-NEXT:    [[TMP2:%.*]] = or <4 x i16> [[TMP1]], [[TMP1]]
-; SINK-AFTER-NEXT:    [[TMP3]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; SINK-AFTER-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
 ; SINK-AFTER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[A_PTR:%.*]], i16 [[OFFSET_IDX]]
 ; SINK-AFTER-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], i32 0
 ; SINK-AFTER-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP5]], align 4
@@ -3551,10 +3640,9 @@ define void @unused_recurrence(ptr %a) {
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
 ; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 1)
-; UNROLL-NO-IC-NEXT:    [[TMP1]] = add <4 x i16> [[TMP0]], splat (i16 5)
+; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[TMP0]], splat (i16 5)
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], splat (i16 4)
 ; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
@@ -3584,12 +3672,11 @@ define void @unused_recurrence(ptr %a) {
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
 ; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
 ; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = add i16 -27, [[DOTCAST]]
 ; UNROLL-NO-VF-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 1
 ; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], 1
-; UNROLL-NO-VF-NEXT:    [[TMP2]] = add i16 [[TMP1]], 5
+; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i16 [[TMP1]], 5
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1028
 ; UNROLL-NO-VF-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
@@ -3618,9 +3705,8 @@ define void @unused_recurrence(ptr %a) {
 ; SINK-AFTER:       vector.body:
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -27, i16 -26, i16 -25, i16 -24>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 1)
-; SINK-AFTER-NEXT:    [[TMP1]] = add <4 x i16> [[TMP0]], splat (i16 5)
+; SINK-AFTER-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[TMP0]], splat (i16 5)
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
 ; SINK-AFTER-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1028
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 5837a49bf0ef..5c1f628bef6d 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -313,9 +313,9 @@ define void @bug18724(i1 %cond, ptr %ptr, i1 %cond.2, i64 %v.1, i32 %v.2) {
 ; VEC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 2
 ; VEC-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
 ; VEC-NEXT:    [[IND_END:%.*]] = add i64 [[V_1]], [[N_VEC]]
+; VEC-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[V_2:%.*]], i32 0
 ; VEC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[COND_2:%.*]], i64 0
 ; VEC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; VEC-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[V_2:%.*]], i32 0
 ; VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VEC:       vector.body:
 ; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-reduction.ll
index e6c2242dd0c4..b4aba5619f1e 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-reduction.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -1803,86 +1803,6 @@ for.end:                                          ; preds = %for.body, %entry
   ret i32 %1
 }
 
-@table = constant [13 x i16] [i16 10, i16 35, i16 69, i16 147, i16 280, i16 472, i16 682, i16 1013, i16 1559, i16 2544, i16 4553, i16 6494, i16 10000], align 1
-
-define i16 @non_reduction_index(i16 noundef %val) {
-; CHECK-LABEL: define i16 @non_reduction_index(
-; CHECK-SAME: i16 noundef [[VAL:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK:       [[FOR_COND_CLEANUP:.*]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
-; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[I_05:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[SUB:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[K_04:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[I_05]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i16 [[TMP0]], [[VAL]]
-; CHECK-NEXT:    [[SUB]] = add nsw i16 [[I_05]], -1
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP1]], i16 [[SUB]], i16 [[K_04]]
-; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i16 [[SUB]], 0
-; CHECK-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
-;
-entry:
-  br label %for.body
-
-for.cond.cleanup:                                 ; preds = %for.body
-  %spec.select.lcssa = phi i16 [ %spec.select, %for.body ]
-  ret i16 %spec.select.lcssa
-
-for.body:                                         ; preds = %entry, %for.body
-  %i.05 = phi i16 [ 12, %entry ], [ %sub, %for.body ]
-  %k.04 = phi i16 [ 0, %entry ], [ %spec.select, %for.body ]
-  %arrayidx = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %i.05
-  %0 = load i16, ptr %arrayidx, align 1
-  %cmp1 = icmp ugt i16 %0, %val
-  %sub = add nsw i16 %i.05, -1
-  %spec.select = select i1 %cmp1, i16 %sub, i16 %k.04
-  %cmp.not = icmp eq i16 %sub, 0
-  br i1 %cmp.not, label %for.cond.cleanup, label %for.body
-}
-
-@tablef = constant [13 x half] [half 10.0, half 35.0, half 69.0, half 147.0, half 280.0, half 472.0, half 682.0, half 1013.0, half 1559.0, half 2544.0, half 4556.0, half 6496.0, half 10000.0], align 1
-
-define i16 @non_reduction_index_half(half noundef %val) {
-; CHECK-LABEL: define i16 @non_reduction_index_half(
-; CHECK-SAME: half noundef [[VAL:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK:       [[FOR_COND_CLEANUP:.*]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
-; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[I_05:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[SUB:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[K_04:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[I_05]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load half, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[FCMP1:%.*]] = fcmp ugt half [[TMP0]], [[VAL]]
-; CHECK-NEXT:    [[SUB]] = add nsw i16 [[I_05]], -1
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[FCMP1]], i16 [[SUB]], i16 [[K_04]]
-; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i16 [[SUB]], 0
-; CHECK-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
-;
-entry:
-  br label %for.body
-
-for.cond.cleanup:                                 ; preds = %for.body
-  %spec.select.lcssa = phi i16 [ %spec.select, %for.body ]
-  ret i16 %spec.select.lcssa
-
-for.body:                                         ; preds = %entry, %for.body
-  %i.05 = phi i16 [ 12, %entry ], [ %sub, %for.body ]
-  %k.04 = phi i16 [ 0, %entry ], [ %spec.select, %for.body ]
-  %arrayidx = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %i.05
-  %0 = load half, ptr %arrayidx, align 1
-  %fcmp1 = fcmp ugt half %0, %val
-  %sub = add nsw i16 %i.05, -1
-  %spec.select = select i1 %fcmp1, i16 %sub, i16 %k.04
-  %cmp.not = icmp eq i16 %sub, 0
-  br i1 %cmp.not, label %for.cond.cleanup, label %for.body
-}
-
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/induction.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/induction.ll
index 1517ec84e9e0..5a5b06de6955 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -3117,9 +3117,9 @@ define i32 @testoverflowcheck() {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP3]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> splat (i32 -1), i32 [[C_PROMOTED_I]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> splat (i32 -1), i32 [[C_PROMOTED_I]], i32 0
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -3161,9 +3161,9 @@ define i32 @testoverflowcheck() {
 ; IND-NEXT:    [[N_VEC:%.*]] = and i32 [[TMP3]], 510
 ; IND-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8
 ; IND-NEXT:    [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]]
+; IND-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 -1>, i32 [[C_PROMOTED_I]], i64 0
 ; IND-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
 ; IND-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
-; IND-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 -1>, i32 [[C_PROMOTED_I]], i64 0
 ; IND-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IND:       vector.body:
 ; IND-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -3203,9 +3203,9 @@ define i32 @testoverflowcheck() {
 ; UNROLL-NEXT:    [[N_VEC:%.*]] = and i32 [[TMP3]], 508
 ; UNROLL-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8
 ; UNROLL-NEXT:    [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]]
+; UNROLL-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 -1>, i32 [[C_PROMOTED_I]], i64 0
 ; UNROLL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
 ; UNROLL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
-; UNROLL-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 -1>, i32 [[C_PROMOTED_I]], i64 0
 ; UNROLL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL:       vector.body:
 ; UNROLL-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -3246,9 +3246,9 @@ define i32 @testoverflowcheck() {
 ; UNROLL-NO-IC-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP3]], [[N_MOD_VF]]
 ; UNROLL-NO-IC-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8
 ; UNROLL-NO-IC-NEXT:    [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]]
+; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> splat (i32 -1), i32 [[C_PROMOTED_I]], i32 0
 ; UNROLL-NO-IC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
 ; UNROLL-NO-IC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> splat (i32 -1), i32 [[C_PROMOTED_I]], i32 0
 ; UNROLL-NO-IC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -3293,9 +3293,9 @@ define i32 @testoverflowcheck() {
 ; INTERLEAVE-NEXT:    [[N_VEC:%.*]] = and i32 [[TMP3]], 504
 ; INTERLEAVE-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8
 ; INTERLEAVE-NEXT:    [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]]
+; INTERLEAVE-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> <i32 poison, i32 -1, i32 -1, i32 -1>, i32 [[C_PROMOTED_I]], i64 0
 ; INTERLEAVE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
 ; INTERLEAVE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; INTERLEAVE-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> <i32 poison, i32 -1, i32 -1, i32 -1>, i32 [[C_PROMOTED_I]], i64 0
 ; INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; INTERLEAVE:       vector.body:
 ; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index 460eafbd1cd2..db25e7bede5c 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -121,8 +121,8 @@ declare i32 @llvm.smin.i32(i32, i32)
 ; DBG-NEXT: No successors
 ; DBG-EMPTY:
 ; DBG-NEXT: scalar.ph:
-; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME1:%.+]]> = resume-phi vp<[[VEC_TC]]>, ir<0>
-; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = resume-phi vp<[[END]]>, ir<false>
+; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME1:%.+]]> = phi [ vp<[[VEC_TC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = phi [ vp<[[END]]>, middle.block ], [ ir<false>, ir-bb<entry> ]
 ; DBG-NEXT: Successor(s): ir-bb<loop.header>
 ; DBG-EMPTY:
 ; DBG-NEXT: ir-bb<loop.header>:
@@ -228,8 +228,8 @@ exit:
 ; DBG-NEXT: No successors
 ; DBG-EMPTY:
 ; DBG-NEXT: scalar.ph:
-; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
-; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
+; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; DBG-NEXT:  EMIT-SCALAR vp<[[RESUME_P:%.*]]> = phi [ vp<[[RESUME_1]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; DBG-NEXT: Successor(s): ir-bb<loop>
 ; DBG-EMPTY:
 ; DBG-NEXT: ir-bb<loop>:
@@ -243,10 +243,9 @@ define void @first_order_recurrence_using_induction(i32 %n, ptr %dst) {
 ; CHECK-LABEL: @first_order_recurrence_using_induction(
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDUCTION1:%.*]], %vector.body ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[INDEX]] to i32
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add i32 [[TMP3]], 0
-; CHECK-NEXT:    [[INDUCTION1]] = add i32 [[TMP3]], 1
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add i32 [[TMP3]], 1
 ; CHECK-NEXT:    store i32 [[INDUCTION]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], %n.vec
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
new file mode 100644
index 000000000000..cf10d32e063a
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -0,0 +1,202 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s
+
+define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
+; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; CHECK-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %entry, %loop
+  %iv = phi i64 [ 19999, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i64 [ 331, %entry ], [ %spec.select, %loop ]
+  %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv
+  %ld.a = load i64, ptr %gep.a.iv, align 8
+  %cmp.a.3 = icmp sgt i64 %ld.a, 3
+  %spec.select = select i1 %cmp.a.3, i64 %iv, i64 %rdx
+  %iv.next = add nsw i64 %iv, -1
+  %exit.cond = icmp eq i64 %iv, 0
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i64 %spec.select
+}
+
+@table = constant [13 x i16] [i16 10, i16 35, i16 69, i16 147, i16 280, i16 472, i16 682, i16 1013, i16 1559, i16 2544, i16 4553, i16 6494, i16 10000], align 1
+
+define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
+; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
+; CHECK-SAME: i16 noundef [[VAL:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; CHECK-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
+; CHECK-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %entry, %loop
+  %iv = phi i16 [ 12, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i16 [ 0, %entry ], [ %spec.select, %loop ]
+  %gep.table.iv = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %iv
+  %ld.table = load i16, ptr %gep.table.iv, align 1
+  %cmp.table.val = icmp ugt i16 %ld.table, %val
+  %iv.next = add nsw i16 %iv, -1
+  %spec.select = select i1 %cmp.table.val, i16 %iv.next, i16 %rdx
+  %exit.cond = icmp eq i16 %iv.next, 0
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  %spec.select.lcssa = phi i16 [ %spec.select, %loop ]
+  ret i16 %spec.select.lcssa
+}
+
+@tablef = constant [13 x half] [half 10.0, half 35.0, half 69.0, half 147.0, half 280.0, half 472.0, half 682.0, half 1013.0, half 1559.0, half 2544.0, half 4556.0, half 6496.0, half 10000.0], align 1
+
+define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
+; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
+; CHECK-SAME: half noundef [[VAL:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; CHECK-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
+; CHECK-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %entry, %loop
+  %iv = phi i16 [ 12, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i16 [ 0, %entry ], [ %spec.select, %loop ]
+  %gep.table.iv = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %iv
+  %ld.table = load half, ptr %gep.table.iv, align 1
+  %cmp.table.val = fcmp ugt half %ld.table, %val
+  %iv.next = add nsw i16 %iv, -1
+  %spec.select = select i1 %cmp.table.val, i16 %iv.next, i16 %rdx
+  %exit.cond = icmp eq i16 %iv.next, 0
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  %spec.select.lcssa = phi i16 [ %spec.select, %loop ]
+  ret i16 %spec.select.lcssa
+}
+
+define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
+; CHECK-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[N]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[LOOP]] ], [ [[RDX_START]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; CHECK-NEXT:    [[GEP_B_IV:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV_NEXT]]
+; CHECK-NEXT:    [[LD_B:%.*]] = load i64, ptr [[GEP_B_IV]], align 8
+; CHECK-NEXT:    [[CMP_A_B:%.*]] = icmp sgt i64 [[LD_A]], [[LD_B]]
+; CHECK-NEXT:    [[COND]] = select i1 [[CMP_A_B]], i64 [[IV_NEXT]], i64 [[RDX]]
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp ugt i64 [[IV]], 1
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[COND_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %entry, %loop
+  %iv = phi i64 [ %iv.next, %loop ], [ %n, %entry ]
+  %rdx = phi i64 [ %cond, %loop ], [ %rdx.start, %entry ]
+  %iv.next = add nsw i64 %iv, -1
+  %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv.next
+  %ld.a = load i64, ptr %gep.a.iv, align 8
+  %gep.b.iv = getelementptr inbounds i64, ptr %b, i64 %iv.next
+  %ld.b = load i64, ptr %gep.b.iv, align 8
+  %cmp.a.b = icmp sgt i64 %ld.a, %ld.b
+  %cond = select i1 %cmp.a.b, i64 %iv.next, i64 %rdx
+  %exit.cond = icmp ugt i64 %iv, 1
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  ret i64 %cond
+}
+
+; The sentinel value for decreasing-IV vectorization is LONG_MAX, and since
+; the IV hits this value, it is impossible to vectorize this case.
+define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(ptr %a) {
+; CHECK-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; CHECK-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %entry, %loop
+  %iv = phi i64 [ 9223372036854775807, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i64 [ 331, %entry ], [ %spec.select, %loop ]
+  %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv
+  %ld.a = load i64, ptr %gep.a.iv, align 8
+  %cmp.a.3 = icmp sgt i64 %ld.a, 3
+  %spec.select = select i1 %cmp.a.3, i64 %iv, i64 %rdx
+  %iv.next = add nsw i64 %iv, -1
+  %exit.cond = icmp eq i64 %iv, 0
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i64 %spec.select
+}
+
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
index 1521f9502879..07c720e9dcb1 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
@@ -1756,160 +1756,6 @@ exit:                                             ; preds = %for.body
   ret float %cond
 }
 
-define i64 @not_vectorized_select_decreasing_induction_icmp_const_start(ptr %a) {
-; CHECK-VF4IC1-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_const_start(
-; CHECK-VF4IC1-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF4IC1:       [[FOR_BODY]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[DEC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP0]], 3
-; CHECK-VF4IC1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF4IC1-NEXT:    [[DEC]] = add nsw i64 [[IV]], -1
-; CHECK-VF4IC1-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-VF4IC1-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
-;
-; CHECK-VF4IC4-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_const_start(
-; CHECK-VF4IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC4-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF4IC4:       [[FOR_BODY]]:
-; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[DEC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF4IC4-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP0]], 3
-; CHECK-VF4IC4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF4IC4-NEXT:    [[DEC]] = add nsw i64 [[IV]], -1
-; CHECK-VF4IC4-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-VF4IC4-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
-; CHECK-VF4IC4:       [[EXIT]]:
-; CHECK-VF4IC4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
-;
-; CHECK-VF1IC4-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_const_start(
-; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC4-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF1IC4:       [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[DEC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF1IC4-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP0]], 3
-; CHECK-VF1IC4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF1IC4-NEXT:    [[DEC]] = add nsw i64 [[IV]], -1
-; CHECK-VF1IC4-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-VF1IC4-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
-; CHECK-VF1IC4:       [[EXIT]]:
-; CHECK-VF1IC4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %entry, %for.body
-  %iv = phi i64 [ 19999, %entry ], [ %dec, %for.body ]
-  %rdx = phi i64 [ 331, %entry ], [ %spec.select, %for.body ]
-  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
-  %0 = load i64, ptr %arrayidx, align 8
-  %cmp = icmp sgt i64 %0, 3
-  %spec.select = select i1 %cmp, i64 %iv, i64 %rdx
-  %dec = add nsw i64 %iv, -1
-  %cmp.not = icmp eq i64 %iv, 0
-  br i1 %cmp.not, label %exit, label %for.body
-
-exit:                                             ; preds = %for.body
-  ret i64 %spec.select
-}
-
-define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
-; CHECK-VF4IC1-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(
-; CHECK-VF4IC1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF4IC1:       [[FOR_BODY]]:
-; CHECK-VF4IC1-NEXT:    [[I_0_IN10:%.*]] = phi i64 [ [[IV:%.*]], %[[FOR_BODY]] ], [ [[N]], %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[FOR_BODY]] ], [ [[RDX_START]], %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    [[IV]] = add nsw i64 [[I_0_IN10]], -1
-; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF4IC1-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
-; CHECK-VF4IC1-NEXT:    [[CMP2:%.*]] = icmp sgt i64 [[TMP0]], [[TMP1]]
-; CHECK-VF4IC1-NEXT:    [[COND]] = select i1 [[CMP2]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[I_0_IN10]], 1
-; CHECK-VF4IC1-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[EXIT:.*]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[COND_LCSSA]]
-;
-; CHECK-VF4IC4-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(
-; CHECK-VF4IC4-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC4-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF4IC4:       [[FOR_BODY]]:
-; CHECK-VF4IC4-NEXT:    [[I_0_IN10:%.*]] = phi i64 [ [[IV:%.*]], %[[FOR_BODY]] ], [ [[N]], %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[FOR_BODY]] ], [ [[RDX_START]], %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT:    [[IV]] = add nsw i64 [[I_0_IN10]], -1
-; CHECK-VF4IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF4IC4-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-VF4IC4-NEXT:    [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
-; CHECK-VF4IC4-NEXT:    [[CMP2:%.*]] = icmp sgt i64 [[TMP0]], [[TMP1]]
-; CHECK-VF4IC4-NEXT:    [[COND]] = select i1 [[CMP2]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF4IC4-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[I_0_IN10]], 1
-; CHECK-VF4IC4-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[EXIT:.*]]
-; CHECK-VF4IC4:       [[EXIT]]:
-; CHECK-VF4IC4-NEXT:    [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    ret i64 [[COND_LCSSA]]
-;
-; CHECK-VF1IC4-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(
-; CHECK-VF1IC4-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC4-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF1IC4:       [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT:    [[I_0_IN10:%.*]] = phi i64 [ [[IV:%.*]], %[[FOR_BODY]] ], [ [[N]], %[[ENTRY]] ]
-; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[FOR_BODY]] ], [ [[RDX_START]], %[[ENTRY]] ]
-; CHECK-VF1IC4-NEXT:    [[IV]] = add nsw i64 [[I_0_IN10]], -1
-; CHECK-VF1IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF1IC4-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-VF1IC4-NEXT:    [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
-; CHECK-VF1IC4-NEXT:    [[CMP2:%.*]] = icmp sgt i64 [[TMP0]], [[TMP1]]
-; CHECK-VF1IC4-NEXT:    [[COND]] = select i1 [[CMP2]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF1IC4-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[I_0_IN10]], 1
-; CHECK-VF1IC4-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[EXIT:.*]]
-; CHECK-VF1IC4:       [[EXIT]]:
-; CHECK-VF1IC4-NEXT:    [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    ret i64 [[COND_LCSSA]]
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %entry, %for.body
-  %i.0.in10 = phi i64 [ %iv, %for.body ], [ %n, %entry ]
-  %rdx = phi i64 [ %cond, %for.body ], [ %rdx.start, %entry ]
-  %iv = add nsw i64 %i.0.in10, -1
-  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
-  %0 = load i64, ptr %arrayidx, align 8
-  %arrayidx1 = getelementptr inbounds i64, ptr %b, i64 %iv
-  %1 = load i64, ptr %arrayidx1, align 8
-  %cmp2 = icmp sgt i64 %0, %1
-  %cond = select i1 %cmp2, i64 %iv, i64 %rdx
-  %cmp = icmp ugt i64 %i.0.in10, 1
-  br i1 %cmp, label %for.body, label %exit
-
-exit:                                             ; preds = %for.body
-  ret i64 %cond
-}
-
 ; The sentinel value for increasing-IV vectorization is -LONG_MAX, and since
 ; the IV hits this value, it is impossible to vectorize this case.
 define i64 @not_vectorized_select_icmp_iv_out_of_bound(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
@@ -2001,81 +1847,6 @@ exit:                                             ; preds = %for.body
   ret i64 %cond
 }
 
-; The sentinel value for decreasing-IV vectorization is LONG_MAX, and since
-; the IV hits this value, it is impossible to vectorize this case.
-define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(ptr %a) {
-; CHECK-VF4IC1-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(
-; CHECK-VF4IC1-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF4IC1:       [[FOR_BODY]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ], [ [[DEC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF4IC1-NEXT:    [[CMP1:%.*]] = icmp sgt i64 [[TMP0]], 3
-; CHECK-VF4IC1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP1]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF4IC1-NEXT:    [[DEC]] = add nsw i64 [[IV]], -1
-; CHECK-VF4IC1-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-VF4IC1-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
-;
-; CHECK-VF4IC4-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(
-; CHECK-VF4IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC4-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF4IC4:       [[FOR_BODY]]:
-; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ], [ [[DEC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF4IC4-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF4IC4-NEXT:    [[CMP1:%.*]] = icmp sgt i64 [[TMP0]], 3
-; CHECK-VF4IC4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP1]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF4IC4-NEXT:    [[DEC]] = add nsw i64 [[IV]], -1
-; CHECK-VF4IC4-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-VF4IC4-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
-; CHECK-VF4IC4:       [[EXIT]]:
-; CHECK-VF4IC4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
-;
-; CHECK-VF1IC4-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(
-; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC4-NEXT:    br label %[[FOR_BODY:.*]]
-; CHECK-VF1IC4:       [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ], [ [[DEC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-VF1IC4-NEXT:    [[CMP1:%.*]] = icmp sgt i64 [[TMP0]], 3
-; CHECK-VF1IC4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP1]], i64 [[IV]], i64 [[RDX]]
-; CHECK-VF1IC4-NEXT:    [[DEC]] = add nsw i64 [[IV]], -1
-; CHECK-VF1IC4-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-VF1IC4-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
-; CHECK-VF1IC4:       [[EXIT]]:
-; CHECK-VF1IC4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %entry, %for.body
-  %iv = phi i64 [ 9223372036854775807, %entry ], [ %dec, %for.body ]
-  %rdx = phi i64 [ 331, %entry ], [ %spec.select, %for.body ]
-  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
-  %0 = load i64, ptr %arrayidx, align 8
-  %cmp1 = icmp sgt i64 %0, 3
-  %spec.select = select i1 %cmp1, i64 %iv, i64 %rdx
-  %dec = add nsw i64 %iv, -1
-  %cmp.not = icmp eq i64 %iv, 0
-  br i1 %cmp.not, label %exit, label %for.body
-
-exit:                                             ; preds = %for.body
-  ret i64 %spec.select
-}
-
 define i64 @not_vectorized_select_icmp_non_const_iv_start_value(ptr %a, ptr %b, i64 %ivstart, i64 %rdx.start, i64 %n) {
 ; CHECK-VF4IC1-LABEL: define i64 @not_vectorized_select_icmp_non_const_iv_start_value(
 ; CHECK-VF4IC1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[IVSTART:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
new file mode 100644
index 000000000000..a622193290c8
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
@@ -0,0 +1,386 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 5
+; RUN: opt -p loop-vectorize -force-widen-divrem-via-safe-divisor=false -force-vector-width=2 -S %s | FileCheck %s
+
+target datalayout="p:16:16"
+
+@src = global [16 x i64] zeroinitializer
+
+; Test cases for https://github.com/llvm/llvm-project/issues/142957
+
+; FIXME: Cannot speculatively execute %l, because %div may trigger UB and must be
+; predicated.
+define void @ptr_depends_on_sdiv(ptr noalias %dst, i16 noundef %off) {
+; CHECK-LABEL: define void @ptr_depends_on_sdiv(
+; CHECK-SAME: ptr noalias [[DST:%.*]], i16 noundef [[OFF:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE4]] ]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK:       [[PRED_SDIV_IF]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = sdiv i16 24316, [[OFF]]
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE]]
+; CHECK:       [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; CHECK:       [[PRED_SDIV_IF1]]:
+; CHECK-NEXT:    [[TMP18:%.*]] = sdiv i16 24316, [[OFF]]
+; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE2]]
+; CHECK:       [[PRED_SDIV_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP13:%.*]] = add i16 [[OFFSET_IDX]], 16383
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i16 [[TMP3]], 14
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i16 [[TMP13]], [[TMP7]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr @src, i16 [[TMP8]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i64, ptr [[TMP20]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP21]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP10]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP11]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
+; CHECK:       [[PRED_STORE_IF3]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP16]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
+; CHECK:       [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i16 [ 9, %entry ], [ %iv.next, %loop.latch ]
+  %cmp1 = icmp eq i16 %iv, 10
+  br i1 %cmp1, label %if.then, label %loop.latch
+
+if.then:
+  %div = sdiv i16 24316, %off
+  %add.iv = add i16 %iv, 16383
+  %s = shl i16 %div, 14
+  %sub6 = sub i16 %add.iv, %s
+  %gep.src = getelementptr inbounds i64, ptr @src, i16 %sub6
+  %l = load i64, ptr %gep.src, align 1
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i16 %iv
+  store i64 %l, ptr %gep.dst, align 1
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i16 %iv, 1
+  %ec = icmp eq i16 %iv.next, 11
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+; FIXME: Cannot speculatively execute %l, because %off may be poison.
+define void @ptr_depends_on_possibly_poison_value(ptr noalias %dst, i16 %off) {
+; CHECK-LABEL: define void @ptr_depends_on_possibly_poison_value(
+; CHECK-SAME: ptr noalias [[DST:%.*]], i16 [[OFF:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i16 1, [[OFF]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], [[OFF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr @src, i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP8]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_IF1]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    store i64 [[TMP13]], ptr [[TMP12]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i16 [ 9, %entry ], [ %iv.next, %loop.latch ]
+  %cmp1 = icmp eq i16 %iv, 10
+  br i1 %cmp1, label %if.then, label %loop.latch
+
+if.then:
+  %sub = sub i16 1, %off
+  %add = add i16 %sub, %off
+  %add.iv = add i16 %iv, %add
+  %gep.src = getelementptr inbounds i64, ptr @src, i16 %add.iv
+  %l = load i64, ptr %gep.src, align 1
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i16 %iv
+  store i64 %l, ptr %gep.dst, align 1
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i16 %iv, 1
+  %ec = icmp eq i16 %iv.next, 11
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @ptr_doesnt_depend_on_poison_or_ub(ptr noalias %dst, i16 noundef %off) {
+; CHECK-LABEL: define void @ptr_doesnt_depend_on_poison_or_ub(
+; CHECK-SAME: ptr noalias [[DST:%.*]], i16 noundef [[OFF:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i16 1, [[OFF]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], [[OFF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr @src, i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP8]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_IF1]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    store i64 [[TMP13]], ptr [[TMP12]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i16 [ 9, %entry ], [ %iv.next, %loop.latch ]
+  %cmp1 = icmp eq i16 %iv, 10
+  br i1 %cmp1, label %if.then, label %loop.latch
+
+if.then:
+  %sub = sub i16 1, %off
+  %add = add i16 %sub, %off
+  %add.iv = add i16 %iv, %add
+  %gep.src = getelementptr inbounds i64, ptr @src, i16 %add.iv
+  %l = load i64, ptr %gep.src, align 1
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i16 %iv
+  store i64 %l, ptr %gep.dst, align 1
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i16 %iv, 1
+  %ec = icmp eq i16 %iv.next, 11
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+; FIXME: Cannot speculatively execute %l, because %off may be poison.
+define void @ptr_depends_on_possibly_poison_value_from_load(ptr noalias %dst) {
+; CHECK-LABEL: define void @ptr_depends_on_possibly_poison_value_from_load(
+; CHECK-SAME: ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @src, align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i16 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i16 [[OFFSET_IDX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr @src, i16 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    store i64 [[TMP10]], ptr [[TMP9]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_IF1]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP13]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i16 [ 9, %entry ], [ %iv.next, %loop.latch ]
+  %off = load i16, ptr @src, align 1
+  %cmp1 = icmp eq i16 %iv, 10
+  br i1 %cmp1, label %if.then, label %loop.latch
+
+if.then:
+  %sub = sub i16 1, %off
+  %add = add i16 %sub, %off
+  %add.iv = add i16 %iv, %add
+  %gep.src = getelementptr inbounds i64, ptr @src, i16 %add.iv
+  %l = load i64, ptr %gep.src, align 1
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i16 %iv
+  store i64 %l, ptr %gep.dst, align 1
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i16 %iv, 1
+  %ec = icmp eq i16 %iv.next, 11
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+; Can speculatively execute %l, because %off cannot be poison due to !noundef.
+define void @ptr_depends_on_noundef_load(ptr noalias %dst) {
+; CHECK-LABEL: define void @ptr_depends_on_noundef_load(
+; CHECK-SAME: ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 9, i16 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @src, align 1, !noundef [[META10:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i16 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i16 [[OFFSET_IDX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr @src, i16 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    store i64 [[TMP10]], ptr [[TMP9]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_IF1]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP13]], align 1
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; CHECK:       [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i16 [ 9, %entry ], [ %iv.next, %loop.latch ]
+  %off = load i16, ptr @src, align 1, !noundef !{}
+  %cmp1 = icmp eq i16 %iv, 10
+  br i1 %cmp1, label %if.then, label %loop.latch
+
+if.then:
+  %sub = sub i16 1, %off
+  %add = add i16 %sub, %off
+  %add.iv = add i16 %iv, %add
+  %gep.src = getelementptr inbounds i64, ptr @src, i16 %add.iv
+  %l = load i64, ptr %gep.src, align 1
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i16 %iv
+  store i64 %l, ptr %gep.dst, align 1
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i16 %iv, 1
+  %ec = icmp eq i16 %iv.next, 11
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/optsize.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/optsize.ll
index c7149b084598..ebddca2294d9 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -621,8 +621,7 @@ define i32 @pr45526_pgso() !prof !14 {
 ; NPGSO:       [[VECTOR_BODY]]:
 ; NPGSO-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; NPGSO-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NPGSO-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ]
-; NPGSO-NEXT:    [[TMP0]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
+; NPGSO-NEXT:    [[TMP0:%.*]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
 ; NPGSO-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; NPGSO-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; NPGSO-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 508
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
index 59727aeb8249..a70d8f72c8a3 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "(for.body:|scalar.body:)" --filter-out-after "for.body:" --version 5
 ; RUN: opt -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=4 -force-vector-interleave=1 -force-target-supports-scalable-vectors=true -S < %s | FileCheck %s --check-prefix=CHECK-VF4UF1
 ; RUN: opt -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=4 -force-vector-interleave=2 -force-target-supports-scalable-vectors=true -S < %s | FileCheck %s --check-prefix=CHECK-VF4UF2
 
@@ -7,30 +8,150 @@
 ; }
 ;
 define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
-; CHECK-VF4UF1-LABEL: @recurrence_1
-; CHECK-VF4UF1: for.preheader
-; CHECK-VF4UF1: %[[SUB_1:.*]] = add i32 %n, -1
-; CHECK-VF4UF1: %[[ZEXT:.*]] = zext i32 %[[SUB_1]] to i64
-; CHECK-VF4UF1: %[[ADD:.*]] = add nuw nsw i64 %[[ZEXT]], 1
-; CHECK-VF4UF1: vector.ph:
-; CHECK-VF4UF1: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], 4
-; CHECK-VF4UF1: %[[SUB1:.*]] = sub i32 %[[MUL1]], 1
-; CHECK-VF4UF1: %[[VEC_RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %pre_load, i32 %[[SUB1]]
-; CHECK-VF4UF1: vector.body:
-; CHECK-VF4UF1: %[[INDEX:.*]] = phi i64 [ 0, %vector.ph ], [ %[[NEXT_IDX:.*]], %vector.body ]
-; CHECK-VF4UF1: %[[VEC_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[VEC_RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-VF4UF1: %[[LOAD]] = load <vscale x 4 x i32>, ptr
-; CHECK-VF4UF1: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> %[[VEC_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 -1)
-; CHECK-VF4UF1: middle.block:
-; CHECK-VF4UF1: %[[VSCALE2:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1: %[[MUL2:.*]] = mul i32 %[[VSCALE2]], 4
-; CHECK-VF4UF1: %[[SUB3:.*]] = sub i32 %[[MUL2]], 2
-; CHECK-VF4UF1: %[[VEC_RECUR_FOR_PHI:.*]] =  extractelement <vscale x 4 x i32> %[[LOAD]], i32 %[[SUB3]]
-; CHECK-VF4UF1: %[[VSCALE3:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1: %[[MUL3:.*]] = mul i32 %[[VSCALE3]], 4
-; CHECK-VF4UF1: %[[SUB3:.*]] = sub i32 %[[MUL3]], 1
-; CHECK-VF4UF1: %[[VEC_RECUR_EXT:.*]] = extractelement <vscale x 4 x i32> %[[LOAD]], i32 %[[SUB3]]
+; CHECK-VF4UF1-LABEL: define i32 @recurrence_1(
+; CHECK-VF4UF1-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-VF4UF1-NEXT:  [[ENTRY:.*:]]
+; CHECK-VF4UF1-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-VF4UF1-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-VF4UF1-NEXT:    br label %[[FOR_PREHEADER:.*]]
+; CHECK-VF4UF1:       [[FOR_PREHEADER]]:
+; CHECK-VF4UF1-NEXT:    [[PRE_LOAD:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = add i64 [[B1]], -4
+; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
+; CHECK-VF4UF1-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], [[TMP7]]
+; CHECK-VF4UF1-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF1:       [[VECTOR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP11]]
+; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[PRE_LOAD]], i32 [[TMP16]]
+; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF1:       [[VECTOR_BODY]]:
+; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[TMP17:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP17]]
+; CHECK-VF4UF1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 0
+; CHECK-VF4UF1-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP19]], align 4
+; CHECK-VF4UF1-NEXT:    [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-VF4UF1-NEXT:    [[TMP22:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP20]]
+; CHECK-VF4UF1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i32 0
+; CHECK-VF4UF1-NEXT:    store <vscale x 4 x i32> [[TMP22]], ptr [[TMP23]], align 4
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]]
+; CHECK-VF4UF1-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF1-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP26]], 2
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP27]]
+; CHECK-VF4UF1-NEXT:    [[TMP28:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP30:%.*]] = sub i32 [[TMP29]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP30]]
+; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP_N]], [[FOR_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF1:       [[SCALAR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], %[[FOR_PREHEADER]] ], [ [[PRE_LOAD]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF1-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF1:       [[SCALAR_BODY]]:
+;
+; CHECK-VF4UF2-LABEL: define i32 @recurrence_1(
+; CHECK-VF4UF2-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-VF4UF2-NEXT:  [[ENTRY:.*:]]
+; CHECK-VF4UF2-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-VF4UF2-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-VF4UF2-NEXT:    br label %[[FOR_PREHEADER:.*]]
+; CHECK-VF4UF2:       [[FOR_PREHEADER]]:
+; CHECK-VF4UF2-NEXT:    [[PRE_LOAD:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = add i64 [[B1]], -4
+; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
+; CHECK-VF4UF2-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], [[TMP7]]
+; CHECK-VF4UF2-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF2:       [[VECTOR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP11]]
+; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[PRE_LOAD]], i32 [[TMP16]]
+; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF2:       [[VECTOR_BODY]]:
+; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP17]]
+; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 0
+; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 [[TMP21]]
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP19]], align 4
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD3]] = load <vscale x 4 x i32>, ptr [[TMP22]], align 4
+; CHECK-VF4UF2-NEXT:    [[TMP23:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP24:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD3]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-VF4UF2-NEXT:    [[TMP26:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP23]]
+; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD3]], [[TMP24]]
+; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0
+; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP30]]
+; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP26]], ptr [[TMP28]], align 4
+; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP27]], ptr [[TMP31]], align 4
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]]
+; CHECK-VF4UF2-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF2-NEXT:    [[TMP33:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = mul i32 [[TMP33]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP35:%.*]] = sub i32 [[TMP34]], 2
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD3]], i32 [[TMP35]]
+; CHECK-VF4UF2-NEXT:    [[TMP36:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = mul i32 [[TMP36]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP37]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD3]], i32 [[TMP38]]
+; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP_N]], [[FOR_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF2:       [[SCALAR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], %[[FOR_PREHEADER]] ], [ [[PRE_LOAD]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF2-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF2:       [[SCALAR_BODY]]:
+;
 entry:
   br label %for.preheader
 
@@ -63,21 +184,142 @@ for.exit:
 ; }
 ;
 define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
-; CHECK-VF4UF1-LABEL: @recurrence_2
-; CHECK-VF4UF1: vector.ph:
-; CHECK-VF4UF1: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], 4
-; CHECK-VF4UF1: %[[SUB1:.*]] = sub i32 %[[MUL1]], 1
-; CHECK-VF4UF1: %[[VEC_RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %.pre, i32 %[[SUB1]]
-; CHECK-VF4UF1: vector.body:
-; CHECK-VF4UF1: %[[VEC_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[VEC_RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-VF4UF1: %[[LOAD]] = load <vscale x 4 x i32>, ptr
-; CHECK-VF4UF1: %[[REVERSE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> %[[VEC_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 -1)
-; CHECK-VF4UF1: middle.block:
-; CHECK-VF4UF1: %[[VSCALE2:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1: %[[MUL2:.*]] = mul i32 %[[VSCALE2]], 4
-; CHECK-VF4UF1: %[[SUB2:.*]] = sub i32 %[[MUL2]], 1
-; CHECK-VF4UF1: %[[VEC_RECUR_EXT:.*]] = extractelement <vscale x 4 x i32> %[[LOAD]], i32 %[[SUB2]]
+; CHECK-VF4UF1-LABEL: define i32 @recurrence_2(
+; CHECK-VF4UF1-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF1-NEXT:    [[CMP27:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP27]], label %[[FOR_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-VF4UF1:       [[FOR_PREHEADER]]:
+; CHECK-VF4UF1-NEXT:    [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -1
+; CHECK-VF4UF1-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4
+; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF1:       [[VECTOR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP8]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP9]]
+; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF1:       [[VECTOR_BODY]]:
+; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ undef, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i32 0
+; CHECK-VF4UF1-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP11]], align 4
+; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP12]]
+; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = icmp sgt <vscale x 4 x i32> [[TMP13]], zeroinitializer
+; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> [[TMP13]], <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI]], [[TMP15]]
+; CHECK-VF4UF1-NEXT:    [[TMP17]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP15]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF1-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP17]])
+; CHECK-VF4UF1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP22:%.*]] = sub i32 [[TMP21]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP22]]
+; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF1:       [[SCALAR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[FOR_PREHEADER]] ]
+; CHECK-VF4UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ]
+; CHECK-VF4UF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ undef, %[[FOR_PREHEADER]] ]
+; CHECK-VF4UF1-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF1:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-VF4UF1-NEXT:    [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], %[[SCALAR_BODY]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4UF1-NEXT:    br label %[[FOR_COND_CLEANUP]]
+; CHECK-VF4UF1:       [[FOR_COND_CLEANUP]]:
+; CHECK-VF4UF1-NEXT:    [[MINMAX_0_LCSSA:%.*]] = phi i32 [ undef, %[[ENTRY]] ], [ [[MINMAX_0_COND_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-VF4UF1-NEXT:    ret i32 [[MINMAX_0_LCSSA]]
+; CHECK-VF4UF1:       [[SCALAR_BODY]]:
+;
+; CHECK-VF4UF2-LABEL: define i32 @recurrence_2(
+; CHECK-VF4UF2-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) {
+; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF2-NEXT:    [[CMP27:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP27]], label %[[FOR_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-VF4UF2:       [[FOR_PREHEADER]]:
+; CHECK-VF4UF2-NEXT:    [[ARRAYIDX2_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -1
+; CHECK-VF4UF2-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4
+; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF2:       [[VECTOR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP8]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP9]]
+; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF2:       [[VECTOR_BODY]]:
+; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ undef, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ undef, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i32 0
+; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 [[TMP13]]
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP11]], align 4
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD2]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
+; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD2]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP15]]
+; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_LOAD2]], [[TMP16]]
+; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = icmp sgt <vscale x 4 x i32> [[TMP17]], zeroinitializer
+; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = icmp sgt <vscale x 4 x i32> [[TMP18]], zeroinitializer
+; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[TMP17]], <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = select <vscale x 4 x i1> [[TMP20]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF2-NEXT:    [[TMP23:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI]], [[TMP21]]
+; CHECK-VF4UF2-NEXT:    [[TMP24:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI1]], [[TMP22]]
+; CHECK-VF4UF2-NEXT:    [[TMP25]] = select <vscale x 4 x i1> [[TMP23]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP21]]
+; CHECK-VF4UF2-NEXT:    [[TMP26]] = select <vscale x 4 x i1> [[TMP24]], <vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> [[TMP22]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF2-NEXT:    [[RDX_MINMAX:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[TMP25]], <vscale x 4 x i32> [[TMP26]])
+; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[RDX_MINMAX]])
+; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP29]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP30]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD2]], i32 [[TMP31]]
+; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF2:       [[SCALAR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[FOR_PREHEADER]] ]
+; CHECK-VF4UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ]
+; CHECK-VF4UF2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ undef, %[[FOR_PREHEADER]] ]
+; CHECK-VF4UF2-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF2:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-VF4UF2-NEXT:    [[MINMAX_0_COND_LCSSA:%.*]] = phi i32 [ [[MINMAX_0_COND:%.*]], %[[SCALAR_BODY]] ], [ [[TMP28]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4UF2-NEXT:    br label %[[FOR_COND_CLEANUP]]
+; CHECK-VF4UF2:       [[FOR_COND_CLEANUP]]:
+; CHECK-VF4UF2-NEXT:    [[MINMAX_0_LCSSA:%.*]] = phi i32 [ undef, %[[ENTRY]] ], [ [[MINMAX_0_COND_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-VF4UF2-NEXT:    ret i32 [[MINMAX_0_LCSSA]]
+; CHECK-VF4UF2:       [[SCALAR_BODY]]:
+;
 entry:
   %cmp27 = icmp sgt i32 %n, 0
   br i1 %cmp27, label %for.preheader, label %for.cond.cleanup
@@ -113,23 +355,180 @@ scalar.body:
 }
 
 define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, float %f, i16 %p) {
-; CHECK-VF4UF1: vector.ph:
-; CHECK-VF4UF1: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], 4
-; CHECK-VF4UF1: %[[SUB1:.*]] = sub i32 %[[MUL1]], 1
-; CHECK-VF4UF1: %vector.recur.init = insertelement <vscale x 4 x i16> poison, i16 %0, i32 %[[SUB1]]
-; CHECK-VF4UF1: vector.body:
-; CHECK-VF4UF1: %vector.recur = phi <vscale x 4 x i16> [ %vector.recur.init, %vector.ph ], [ %[[L1:.*]], %vector.body ]
-; CHECK-VF4UF1: %[[L1]] = load <vscale x 4 x i16>, ptr
-; CHECK-VF4UF1: %[[SPLICE:.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> %vector.recur, <vscale x 4 x i16> %[[L1]], i32 -1)
+; CHECK-VF4UF1-LABEL: define void @recurrence_3(
+; CHECK-VF4UF1-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]], float [[F:%.*]], i16 [[P:%.*]]) {
+; CHECK-VF4UF1-NEXT:  [[ENTRY:.*:]]
+; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-VF4UF1-NEXT:    [[CONV:%.*]] = sitofp i16 [[TMP0]] to double
+; CHECK-VF4UF1-NEXT:    [[CONV1:%.*]] = fpext float [[F]] to double
+; CHECK-VF4UF1-NEXT:    [[CONV2:%.*]] = sitofp i16 [[P]] to double
+; CHECK-VF4UF1-NEXT:    [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]]
+; CHECK-VF4UF1-NEXT:    [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]]
+; CHECK-VF4UF1-NEXT:    store double [[SUB]], ptr [[B]], align 8
+; CHECK-VF4UF1-NEXT:    [[CMP25:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP25]], label %[[FOR_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-VF4UF1:       [[FOR_PREHEADER]]:
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF1-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 8
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = add i32 [[N]], -2
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 3
+; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 16
+; CHECK-VF4UF1-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-VF4UF1-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 2
+; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP9]], 4
+; CHECK-VF4UF1-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-VF4UF1-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
+; CHECK-VF4UF1-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
+; CHECK-VF4UF1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-VF4UF1-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF1:       [[VECTOR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP12]]
+; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x double> poison, double [[CONV1]], i64 0
+; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x double> [[BROADCAST_SPLATINSERT]], <vscale x 4 x double> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP17:%.*]] = mul i32 [[TMP16]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP17]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[TMP0]], i32 [[TMP18]]
+; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF1:       [[VECTOR_BODY]]:
+; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-VF4UF1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-VF4UF1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i16, ptr [[TMP19]], i32 0
+; CHECK-VF4UF1-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i16>, ptr [[TMP20]], align 2, !alias.scope [[META6:![0-9]+]]
+; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> [[VECTOR_RECUR]], <vscale x 4 x i16> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF1-NEXT:    [[TMP22:%.*]] = sitofp <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x double>
+; CHECK-VF4UF1-NEXT:    [[TMP23:%.*]] = sitofp <vscale x 4 x i16> [[TMP21]] to <vscale x 4 x double>
+; CHECK-VF4UF1-NEXT:    [[TMP24:%.*]] = fmul fast <vscale x 4 x double> [[TMP23]], [[BROADCAST_SPLAT]]
+; CHECK-VF4UF1-NEXT:    [[TMP25:%.*]] = fsub fast <vscale x 4 x double> [[TMP22]], [[TMP24]]
+; CHECK-VF4UF1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-VF4UF1-NEXT:    [[TMP27:%.*]] = getelementptr inbounds double, ptr [[TMP26]], i32 0
+; CHECK-VF4UF1-NEXT:    store <vscale x 4 x double> [[TMP25]], ptr [[TMP27]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]]
+; CHECK-VF4UF1-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF1-NEXT:    [[TMP29:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP29]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP30]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD]], i32 [[TMP31]]
+; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF1:       [[SCALAR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF1-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF1:       [[SCALAR_BODY]]:
+;
+; CHECK-VF4UF2-LABEL: define void @recurrence_3(
+; CHECK-VF4UF2-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]], float [[F:%.*]], i16 [[P:%.*]]) {
+; CHECK-VF4UF2-NEXT:  [[ENTRY:.*:]]
+; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-VF4UF2-NEXT:    [[CONV:%.*]] = sitofp i16 [[TMP0]] to double
+; CHECK-VF4UF2-NEXT:    [[CONV1:%.*]] = fpext float [[F]] to double
+; CHECK-VF4UF2-NEXT:    [[CONV2:%.*]] = sitofp i16 [[P]] to double
+; CHECK-VF4UF2-NEXT:    [[MUL:%.*]] = fmul fast double [[CONV2]], [[CONV1]]
+; CHECK-VF4UF2-NEXT:    [[SUB:%.*]] = fsub fast double [[CONV]], [[MUL]]
+; CHECK-VF4UF2-NEXT:    store double [[SUB]], ptr [[B]], align 8
+; CHECK-VF4UF2-NEXT:    [[CMP25:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP25]], label %[[FOR_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-VF4UF2:       [[FOR_PREHEADER]]:
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF2-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 8
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = add i32 [[N]], -2
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 3
+; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 16
+; CHECK-VF4UF2-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-VF4UF2-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 2
+; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP9]], 4
+; CHECK-VF4UF2-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-VF4UF2-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
+; CHECK-VF4UF2-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
+; CHECK-VF4UF2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-VF4UF2-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF2:       [[VECTOR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
+; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP12]]
+; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x double> poison, double [[CONV1]], i64 0
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x double> [[BROADCAST_SPLATINSERT]], <vscale x 4 x double> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = mul i32 [[TMP16]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP17]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[TMP0]], i32 [[TMP18]]
+; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF2:       [[VECTOR_BODY]]:
+; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i16, ptr [[TMP19]], i32 0
+; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i16, ptr [[TMP19]], i64 [[TMP22]]
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP20]], align 2, !alias.scope [[META6:![0-9]+]]
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD4]] = load <vscale x 4 x i16>, ptr [[TMP23]], align 2, !alias.scope [[META6]]
+; CHECK-VF4UF2-NEXT:    [[TMP24:%.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> [[VECTOR_RECUR]], <vscale x 4 x i16> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP25:%.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> [[WIDE_LOAD4]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP26:%.*]] = sitofp <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x double>
+; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = sitofp <vscale x 4 x i16> [[WIDE_LOAD4]] to <vscale x 4 x double>
+; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = sitofp <vscale x 4 x i16> [[TMP24]] to <vscale x 4 x double>
+; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = sitofp <vscale x 4 x i16> [[TMP25]] to <vscale x 4 x double>
+; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = fmul fast <vscale x 4 x double> [[TMP28]], [[BROADCAST_SPLAT]]
+; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = fmul fast <vscale x 4 x double> [[TMP29]], [[BROADCAST_SPLAT]]
+; CHECK-VF4UF2-NEXT:    [[TMP32:%.*]] = fsub fast <vscale x 4 x double> [[TMP26]], [[TMP30]]
+; CHECK-VF4UF2-NEXT:    [[TMP33:%.*]] = fsub fast <vscale x 4 x double> [[TMP27]], [[TMP31]]
+; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-VF4UF2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds double, ptr [[TMP34]], i32 0
+; CHECK-VF4UF2-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds double, ptr [[TMP34]], i64 [[TMP37]]
+; CHECK-VF4UF2-NEXT:    store <vscale x 4 x double> [[TMP32]], ptr [[TMP35]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
+; CHECK-VF4UF2-NEXT:    store <vscale x 4 x double> [[TMP33]], ptr [[TMP38]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]]
+; CHECK-VF4UF2-NEXT:    [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF2-NEXT:    [[TMP40:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP41:%.*]] = mul i32 [[TMP40]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP41]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD4]], i32 [[TMP42]]
+; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF2:       [[SCALAR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF2-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF2:       [[SCALAR_BODY]]:
+;
 ; Check also that the casts were not moved needlessly.
-; CHECK-VF4UF1: sitofp <vscale x 4 x i16> %[[L1]] to <vscale x 4 x double>
-; CHECK-VF4UF1: sitofp <vscale x 4 x i16> %[[SPLICE]] to <vscale x 4 x double>
-; CHECK-VF4UF1: middle.block:
-; CHECK-VF4UF1: %[[VSCALE2:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1: %[[MUL2:.*]] = mul i32 %[[VSCALE2]], 4
-; CHECK-VF4UF1: %[[SUB2:.*]] = sub i32 %[[MUL2]], 1
-; CHECK-VF4UF1: %vector.recur.extract = extractelement <vscale x 4 x i16> %[[L1]], i32 %[[SUB2]]
 entry:
   %0 = load i16, ptr %a, align 2
   %conv = sitofp i16 %0 to double
@@ -168,10 +567,62 @@ for.end:
 }
 
 define i64 @constant_folded_previous_value() {
-; CHECK-VF4UF2-LABEL: @constant_folded_previous_value
-; CHECK-VF4UF2: vector.body
-; CHECK-VF4UF2: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i64> [ %vector.recur.init, %vector.ph ], [ splat (i64 1), %vector.body ]
-; CHECK-VF4UF2: br i1 {{.*}}, label %middle.block, label %vector.body
+; CHECK-VF4UF1-LABEL: define i64 @constant_folded_previous_value() {
+; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1000, [[TMP1]]
+; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF1:       [[VECTOR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1000, [[TMP3]]
+; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 1000, [[N_MOD_VF]]
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF1:       [[VECTOR_BODY]]:
+; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF1:       [[SCALAR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 1, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF1-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF1:       [[SCALAR_BODY]]:
+;
+; CHECK-VF4UF2-LABEL: define i64 @constant_folded_previous_value() {
+; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1000, [[TMP1]]
+; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF2:       [[VECTOR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1000, [[TMP3]]
+; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 1000, [[N_MOD_VF]]
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF2:       [[VECTOR_BODY]]:
+; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF2:       [[SCALAR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 1, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF2-NEXT:    br label %[[SCALAR_BODY:.*]]
+; CHECK-VF4UF2:       [[SCALAR_BODY]]:
+;
 entry:
   br label %scalar.body
 
@@ -180,7 +631,7 @@ scalar.body:
   %tmp2 = phi i64 [ 0, %entry ], [ %tmp3, %scalar.body ]
   %tmp3 = add i64 0, 1
   %i.next = add nuw nsw i64 %i, 1
-  %cond = icmp eq i64 %i.next, undef
+  %cond = icmp eq i64 %i.next, 1000
   br i1 %cond, label %for.end, label %scalar.body, !llvm.loop !0
 
 for.end:
@@ -193,28 +644,103 @@ for.end:
 ; the first order recurrence phi is used outside the loop, so we require the phi
 ; itself and not its update (addx).
 define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
-; CHECK-VF4UF2-LABEL: @extract_second_last_iteration
-; CHECK-VF4UF2: vector.ph
-; CHECK-VF4UF2: call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2: call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2: %[[SPLAT_INS1:.*]] = insertelement <vscale x 4 x i32> poison, i32 %x, i64 0
-; CHECK-VF4UF2: %[[SPLAT1:.*]] = shufflevector <vscale x 4 x i32> %[[SPLAT_INS1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-VF4UF2: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], 4
-; CHECK-VF4UF2: %[[SUB1:.*]] = sub i32 %[[MUL1]], 1
-; CHECK-VF4UF2: %[[VEC_RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 %[[SUB1]]
-; ; CHECK-VF4UF2: vector.body
-; CHECK-VF4UF2: %[[VEC_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[VEC_RECUR_INIT]], %vector.ph ], [ %[[ADD2:.*]], %vector.body ]
-; CHECK-VF4UF2: %[[ADD1:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[SPLAT1]]
-; CHECK-VF4UF2: middle.block
-; CHECK-VF4UF2: %[[VSCALE2:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2: %[[MUL2:.*]] = mul i32 %[[VSCALE2]], 4
-; CHECK-VF4UF2: %[[SUB3:.*]] = sub i32 %[[MUL2]], 2
-; CHECK-VF4UF2: %vector.recur.extract.for.phi = extractelement <vscale x 4 x i32> %[[ADD2]], i32 %[[SUB3]]
-; CHECK-VF4UF2: %[[VSCALE3:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2: %[[MUL3:.*]] = mul i32 %[[VSCALE3]], 4
-; CHECK-VF4UF2: %[[SUB2:.*]] = sub i32 %[[MUL3]], 1
-; CHECK-VF4UF2: %vector.recur.extract = extractelement <vscale x 4 x i32> %[[ADD2]], i32 %[[SUB2]]
+; CHECK-VF4UF1-LABEL: define i32 @extract_second_last_iteration(
+; CHECK-VF4UF1-SAME: ptr [[CVAL:%.*]], i32 [[X:%.*]]) {
+; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 96, [[TMP1]]
+; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF1:       [[VECTOR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 4
+; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i32 96, [[TMP3]]
+; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i32 96, [[N_MOD_VF]]
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
+; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i32> [[TMP6]], splat (i32 1)
+; CHECK-VF4UF1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP7]]
+; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul i32 1, [[TMP5]]
+; CHECK-VF4UF1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-VF4UF1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF1:       [[VECTOR_BODY]]:
+; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; CHECK-VF4UF1-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 2
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP13]]
+; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP16]]
+; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i32 96, [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF1:       [[SCALAR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF1-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-VF4UF1:       [[FOR_BODY]]:
+;
+; CHECK-VF4UF2-LABEL: define i32 @extract_second_last_iteration(
+; CHECK-VF4UF2-SAME: ptr [[CVAL:%.*]], i32 [[X:%.*]]) {
+; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 96, [[TMP1]]
+; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF2:       [[VECTOR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 8
+; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i32 96, [[TMP3]]
+; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i32 96, [[N_MOD_VF]]
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], 2
+; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP7]], splat (i32 1)
+; CHECK-VF4UF2-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP8]]
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF2:       [[VECTOR_BODY]]:
+; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[STEP_ADD:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[VEC_IND_NEXT:%.*]] = add <vscale x 4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT2]]
+; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[VEC_IND_NEXT]], [[BROADCAST_SPLAT]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP6]]
+; CHECK-VF4UF2-NEXT:    [[VEC_IND_NEXT1]] = add <vscale x 4 x i32> [[VEC_IND_NEXT]], [[BROADCAST_SPLAT2]]
+; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 2
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP13]]
+; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP16]]
+; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i32 96, [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF2:       [[SCALAR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4UF2-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-VF4UF2:       [[FOR_BODY]]:
+;
 entry:
   br label %for.body
 
@@ -238,13 +764,140 @@ for.end:
 
 ; Check that the sext sank after the load in the vector loop.
 define void @sink_after(ptr %a, ptr %b, i64 %n) {
-; CHECK-VF4UF1-LABEL: @sink_after
-; CHECK-VF4UF1: vector.body
-; CHECK-VF4UF1: %[[VEC_RECUR:.*]] = phi <vscale x 4 x i16> [ %vector.recur.init, %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-VF4UF1: %[[LOAD]] = load <vscale x 4 x i16>, ptr
-; CHECK-VF4UF1-NEXT: %[[SPLICE:.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> %[[VEC_RECUR]], <vscale x 4 x i16> %[[LOAD]], i32 -1)
-; CHECK-VF4UF1-NEXT: sext <vscale x 4 x i16> %[[SPLICE]] to <vscale x 4 x i32>
-; CHECK-VF4UF1-NEXT: sext <vscale x 4 x i16> %[[LOAD]] to <vscale x 4 x i32>
+; CHECK-VF4UF1-LABEL: define void @sink_after(
+; CHECK-VF4UF1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF1-NEXT:    [[DOTPRE:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = shl i64 [[N]], 2
+; CHECK-VF4UF1-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-VF4UF1-NEXT:    [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 2
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = shl i64 [[N]], 1
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 2
+; CHECK-VF4UF1-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF4UF1-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
+; CHECK-VF4UF1-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]
+; CHECK-VF4UF1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-VF4UF1-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF1:       [[VECTOR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
+; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[DOTPRE]], i32 [[TMP11]]
+; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF1:       [[VECTOR_BODY]]:
+; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[TMP12]]
+; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i32 0
+; CHECK-VF4UF1-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i16>, ptr [[TMP14]], align 2, !alias.scope [[META17:![0-9]+]]
+; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> [[VECTOR_RECUR]], <vscale x 4 x i16> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = sext <vscale x 4 x i16> [[TMP15]] to <vscale x 4 x i32>
+; CHECK-VF4UF1-NEXT:    [[TMP17:%.*]] = sext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP17]], [[TMP16]]
+; CHECK-VF4UF1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-VF4UF1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0
+; CHECK-VF4UF1-NEXT:    store <vscale x 4 x i32> [[TMP18]], ptr [[TMP20]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF1-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF1-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD]], i32 [[TMP24]]
+; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4UF1-NEXT:    br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF1:       [[SCALAR_PH]]:
+; CHECK-VF4UF1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ], [ [[DOTPRE]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF1-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-VF4UF1:       [[FOR_BODY]]:
+;
+; CHECK-VF4UF2-LABEL: define void @sink_after(
+; CHECK-VF4UF2-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4UF2-NEXT:    [[DOTPRE:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = shl i64 [[N]], 2
+; CHECK-VF4UF2-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-VF4UF2-NEXT:    [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 2
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = shl i64 [[N]], 1
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 2
+; CHECK-VF4UF2-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF4UF2-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
+; CHECK-VF4UF2-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]
+; CHECK-VF4UF2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-VF4UF2-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4UF2:       [[VECTOR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
+; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[DOTPRE]], i32 [[TMP11]]
+; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4UF2:       [[VECTOR_BODY]]:
+; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[TMP12]]
+; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i32 0
+; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i64 [[TMP16]]
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP14]], align 2, !alias.scope [[META17:![0-9]+]]
+; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD3]] = load <vscale x 4 x i16>, ptr [[TMP17]], align 2, !alias.scope [[META17]]
+; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> [[VECTOR_RECUR]], <vscale x 4 x i16> [[WIDE_LOAD]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> [[WIDE_LOAD3]], i32 -1)
+; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = sext <vscale x 4 x i16> [[TMP18]] to <vscale x 4 x i32>
+; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = sext <vscale x 4 x i16> [[TMP19]] to <vscale x 4 x i32>
+; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = sext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-VF4UF2-NEXT:    [[TMP23:%.*]] = sext <vscale x 4 x i16> [[WIDE_LOAD3]] to <vscale x 4 x i32>
+; CHECK-VF4UF2-NEXT:    [[TMP24:%.*]] = mul nsw <vscale x 4 x i32> [[TMP22]], [[TMP20]]
+; CHECK-VF4UF2-NEXT:    [[TMP25:%.*]] = mul nsw <vscale x 4 x i32> [[TMP23]], [[TMP21]]
+; CHECK-VF4UF2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i32 0
+; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i64 [[TMP29]]
+; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP24]], ptr [[TMP27]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
+; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP25]], ptr [[TMP30]], align 4, !alias.scope [[META20]], !noalias [[META17]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
+; CHECK-VF4UF2-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VF4UF2-NEXT:    [[TMP33:%.*]] = mul i32 [[TMP32]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP33]], 1
+; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD3]], i32 [[TMP34]]
+; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4UF2-NEXT:    br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK-VF4UF2:       [[SCALAR_PH]]:
+; CHECK-VF4UF2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ], [ [[DOTPRE]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-VF4UF2-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-VF4UF2:       [[FOR_BODY]]:
+;
 entry:
   %.pre = load i16, ptr %a
   br label %for.body
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
index 223acfa2e3a2..04b4c6fe58db 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
@@ -5,18 +5,32 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-LABEL: @reduction_add_trunc(
-; CHECK:         call i32 @llvm.vscale.i32()
-; CHECK:         call i32 @llvm.vscale.i32()
-; CHECK:         [[TMP30:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], 16
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 256, [[TMP31]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 16
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 256, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 256, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 16
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ insertelement (<vscale x 8 x i32> zeroinitializer, i32 255, i32 0), %vector.ph ], [ [[TMP34:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, %vector.ph ], [ [[TMP36:%.*]], %vector.body ]
-; CHECK:         [[TMP14:%.*]] = and <vscale x 8 x i32> [[VEC_PHI]], splat (i32 255)
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ insertelement (<vscale x 8 x i32> zeroinitializer, i32 255, i32 0), [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <vscale x 8 x i32> [[VEC_PHI]], splat (i32 255)
 ; CHECK-NEXT:    [[TMP15:%.*]] = and <vscale x 8 x i32> [[VEC_PHI1]], splat (i32 255)
-; CHECK:         [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr
-; CHECK:         [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[TMP11]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[TMP26:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP27:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP28:%.*]] = add <vscale x 8 x i32> [[TMP14]], [[TMP26]]
@@ -25,15 +39,36 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP35:%.*]] = trunc <vscale x 8 x i32> [[TMP29]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP34]] = zext <vscale x 8 x i8> [[TMP33]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP36]] = zext <vscale x 8 x i8> [[TMP35]] to <vscale x 8 x i32>
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP31]]
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], {{%.*}}
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
 ; CHECK-NEXT:    [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
-; CHECK-NEXT:    %cmp.n = icmp eq i32 256, %n.vec
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP40]], [[MIDDLE_BLOCK]] ], [ 255, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_02P:%.*]] = phi i32 [ [[L9:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = and i32 [[SUM_02P]], 255
+; CHECK-NEXT:    [[L2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDVARS_IV]]
+; CHECK-NEXT:    [[L3:%.*]] = load i8, ptr [[L2]], align 4
+; CHECK-NEXT:    [[L3E:%.*]] = zext i8 [[L3]] to i32
+; CHECK-NEXT:    [[L9]] = add i32 [[SUM_02]], [[L3E]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], 256
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[L9]], [[LOOP]] ], [ [[TMP40]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RET:%.*]] = trunc i32 [[SUM_0_LCSSA]] to i8
+; CHECK-NEXT:    ret i8 [[RET]]
 ;
 entry:
   br label %loop
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-index-interleaving.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-index-interleaving.ll
new file mode 100644
index 000000000000..6b44586286c7
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-index-interleaving.ll
@@ -0,0 +1,338 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
+
+define i64 @test_vectorize_select_umin_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sgt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ult i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_first_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_first_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp slt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smax.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-min-index.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-min-index.ll
deleted file mode 100644
index 12dfbe4d8f56..000000000000
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-min-index.ll
+++ /dev/null
@@ -1,1084 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=CHECK-VF4IC1
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s --check-prefix=CHECK-VF4IC2
-; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s | FileCheck %s --check-prefix=CHECK-VF1IC2
-
-; Test cases for selecting the index with the minimum value.
-
-define i64 @test_vectorize_select_umin_idx(ptr %src, i64 %n) {
-; CHECK-VF4IC1-LABEL: define i64 @test_vectorize_select_umin_idx(
-; CHECK-VF4IC1-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_vectorize_select_umin_idx(
-; CHECK-VF4IC2-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_vectorize_select_umin_idx(
-; CHECK-VF1IC2-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF1IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %gep = getelementptr i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep
-  %cmp = icmp ugt i64 %min.val, %l
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %n
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  ret i64 %res
-}
-
-define i64 @test_vectorize_select_umin_idx_all_exit_inst(ptr %src, ptr %umin, i64 %n) {
-; CHECK-VF4IC1-LABEL: define i64 @test_vectorize_select_umin_idx_all_exit_inst(
-; CHECK-VF4IC1-SAME: ptr [[SRC:%.*]], ptr [[UMIN:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[RES_UMIN:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    store i64 [[RES_UMIN]], ptr [[UMIN]], align 4
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_vectorize_select_umin_idx_all_exit_inst(
-; CHECK-VF4IC2-SAME: ptr [[SRC:%.*]], ptr [[UMIN:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[RES_UMIN:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    store i64 [[RES_UMIN]], ptr [[UMIN]], align 4
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_vectorize_select_umin_idx_all_exit_inst(
-; CHECK-VF1IC2-SAME: ptr [[SRC:%.*]], ptr [[UMIN:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF1IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[RES_UMIN:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    store i64 [[RES_UMIN]], ptr [[UMIN]], align 4
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %gep = getelementptr i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep
-  %cmp = icmp ugt i64 %min.val, %l
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %n
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  %res.umin = phi i64 [ %min.val.next, %loop ]
-  store i64 %res.umin, ptr %umin
-  ret i64 %res
-}
-
-define i64 @test_vectorize_select_umin_idx_min_ops_switched(ptr %src, i64 %n) {
-; CHECK-VF4IC1-LABEL: define i64 @test_vectorize_select_umin_idx_min_ops_switched(
-; CHECK-VF4IC1-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[L]], i64 [[MIN_VAL]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_vectorize_select_umin_idx_min_ops_switched(
-; CHECK-VF4IC2-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[L]], i64 [[MIN_VAL]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_vectorize_select_umin_idx_min_ops_switched(
-; CHECK-VF1IC2-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF1IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[L]], i64 [[MIN_VAL]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %gep = getelementptr i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep
-  %cmp = icmp ugt i64 %min.val, %l
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %l, i64 %min.val)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %n
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  ret i64 %res
-}
-
-define i64 @test_not_vectorize_select_no_min_reduction(ptr %src, i64 %n) {
-; CHECK-VF4IC1-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
-; CHECK-VF4IC1-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-VF4IC1:       [[VECTOR_PH]]:
-; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK-VF4IC1:       [[VECTOR_BODY]]:
-; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[TMP1]], i32 0
-; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 4
-; CHECK-VF4IC1-NEXT:    [[TMP3]] = add <4 x i64> [[WIDE_LOAD]], splat (i64 1)
-; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-VF4IC1-NEXT:    [[TMP5:%.*]] = icmp ugt <4 x i64> [[TMP4]], [[WIDE_LOAD]]
-; CHECK-VF4IC1-NEXT:    [[TMP6]] = select <4 x i1> [[TMP5]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-VF4IC1-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VF4IC1-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-VF4IC1:       [[MIDDLE_BLOCK]]:
-; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP6]])
-; CHECK-VF4IC1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
-; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP8]], i64 0
-; CHECK-VF4IC1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-VF4IC1:       [[SCALAR_PH]]:
-; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = add i64 [[L]], 1
-; CHECK-VF4IC1-NEXT:    [[FOO:%.*]] = call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
-; CHECK-VF4IC2-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
-; CHECK-VF4IC2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-VF4IC2:       [[VECTOR_PH]]:
-; CHECK-VF4IC2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
-; CHECK-VF4IC2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF4IC2-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK-VF4IC2:       [[VECTOR_BODY]]:
-; CHECK-VF4IC2-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC2-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC2-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC2-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC2-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4IC2-NEXT:    [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-VF4IC2-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP0]]
-; CHECK-VF4IC2-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[TMP1]], i32 0
-; CHECK-VF4IC2-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP1]], i32 4
-; CHECK-VF4IC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 4
-; CHECK-VF4IC2-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP3]], align 4
-; CHECK-VF4IC2-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[WIDE_LOAD]], splat (i64 1)
-; CHECK-VF4IC2-NEXT:    [[TMP5]] = add <4 x i64> [[WIDE_LOAD2]], splat (i64 1)
-; CHECK-VF4IC2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-VF4IC2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-VF4IC2-NEXT:    [[TMP8:%.*]] = icmp ugt <4 x i64> [[TMP6]], [[WIDE_LOAD]]
-; CHECK-VF4IC2-NEXT:    [[TMP9:%.*]] = icmp ugt <4 x i64> [[TMP7]], [[WIDE_LOAD2]]
-; CHECK-VF4IC2-NEXT:    [[TMP10]] = select <4 x i1> [[TMP8]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-VF4IC2-NEXT:    [[TMP11]] = select <4 x i1> [[TMP9]], <4 x i64> [[STEP_ADD]], <4 x i64> [[VEC_PHI1]]
-; CHECK-VF4IC2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 8
-; CHECK-VF4IC2-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-VF4IC2-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VF4IC2-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-VF4IC2:       [[MIDDLE_BLOCK]]:
-; CHECK-VF4IC2-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> [[TMP10]], <4 x i64> [[TMP11]])
-; CHECK-VF4IC2-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[RDX_MINMAX]])
-; CHECK-VF4IC2-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP13]], -9223372036854775808
-; CHECK-VF4IC2-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP13]], i64 0
-; CHECK-VF4IC2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3
-; CHECK-VF4IC2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-VF4IC2-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-VF4IC2:       [[SCALAR_PH]]:
-; CHECK-VF4IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = add i64 [[L]], 1
-; CHECK-VF4IC2-NEXT:    [[FOO:%.*]] = call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
-; CHECK-VF1IC2-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-VF1IC2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-VF1IC2:       [[VECTOR_PH]]:
-; CHECK-VF1IC2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-VF1IC2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF1IC2-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK-VF1IC2:       [[VECTOR_BODY]]:
-; CHECK-VF1IC2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF1IC2-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ -9223372036854775808, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF1IC2-NEXT:    [[VEC_PHI1:%.*]] = phi i64 [ -9223372036854775808, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF1IC2-NEXT:    [[VECTOR_RECUR:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF1IC2-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-VF1IC2-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-VF1IC2-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-VF1IC2-NEXT:    [[TMP4:%.*]] = load i64, ptr [[TMP2]], align 4
-; CHECK-VF1IC2-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP3]], align 4
-; CHECK-VF1IC2-NEXT:    [[TMP6:%.*]] = add i64 [[TMP4]], 1
-; CHECK-VF1IC2-NEXT:    [[TMP7]] = add i64 [[TMP5]], 1
-; CHECK-VF1IC2-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[VECTOR_RECUR]], [[TMP4]]
-; CHECK-VF1IC2-NEXT:    [[TMP9:%.*]] = icmp ugt i64 [[TMP6]], [[TMP5]]
-; CHECK-VF1IC2-NEXT:    [[TMP10]] = select i1 [[TMP8]], i64 [[INDEX]], i64 [[VEC_PHI]]
-; CHECK-VF1IC2-NEXT:    [[TMP11]] = select i1 [[TMP9]], i64 [[TMP1]], i64 [[VEC_PHI1]]
-; CHECK-VF1IC2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-VF1IC2-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VF1IC2-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-VF1IC2:       [[MIDDLE_BLOCK]]:
-; CHECK-VF1IC2-NEXT:    [[RDX_MINMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP10]], i64 [[TMP11]])
-; CHECK-VF1IC2-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[RDX_MINMAX]], -9223372036854775808
-; CHECK-VF1IC2-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[RDX_MINMAX]], i64 0
-; CHECK-VF1IC2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-VF1IC2-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-VF1IC2:       [[SCALAR_PH]]:
-; CHECK-VF1IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF1IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF1IC2-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF1IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = add i64 [[L]], 1
-; CHECK-VF1IC2-NEXT:    [[FOO:%.*]] = call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %gep = getelementptr i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep
-  %cmp = icmp ugt i64 %min.val, %l
-  %min.val.next = add i64 %l, 1
-  %foo = call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %n
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  ret i64 %res
-}
-
-
-define i64 @test_not_vectorize_cmp_value(i64 %x, i64 %n) {
-; CHECK-VF4IC1-LABEL: define i64 @test_not_vectorize_cmp_value(
-; CHECK-VF4IC1-SAME: i64 [[X:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[X]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_not_vectorize_cmp_value(
-; CHECK-VF4IC2-SAME: i64 [[X:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[X]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_not_vectorize_cmp_value(
-; CHECK-VF1IC2-SAME: i64 [[X:%.*]], i64 [[N:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[X]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %cmp = icmp ugt i64 %min.val, %x
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 0)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %n
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  ret i64 %res
-}
-
-define i32 @test_vectorize_select_umin_idx_with_trunc(i64 %n) {
-; CHECK-VF4IC1-LABEL: define i32 @test_vectorize_select_umin_idx_with_trunc(
-; CHECK-VF4IC1-SAME: i64 [[N:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], 0
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF4IC1-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret i32 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i32 @test_vectorize_select_umin_idx_with_trunc(
-; CHECK-VF4IC2-SAME: i64 [[N:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], 0
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF4IC2-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret i32 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i32 @test_vectorize_select_umin_idx_with_trunc(
-; CHECK-VF1IC2-SAME: i64 [[N:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], 0
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF1IC2-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret i32 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %cmp = icmp ugt i64 %min.val, 0
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 0)
-  %trunc = trunc i64 %iv to i32
-  %min.idx.next = select i1 %cmp, i32 %trunc, i32 %min.idx
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, %n
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i32 [ %min.idx.next, %loop ]
-  ret i32 %res
-}
-
-define ptr @test_with_ptr_index(ptr %start, ptr %end) {
-; CHECK-VF4IC1-LABEL: define ptr @test_with_ptr_index(
-; CHECK-VF4IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[CMP7_US:%.*]] = icmp ult i64 0, 0
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret ptr [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define ptr @test_with_ptr_index(
-; CHECK-VF4IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[CMP7_US:%.*]] = icmp ult i64 0, 0
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret ptr [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define ptr @test_with_ptr_index(
-; CHECK-VF1IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[CMP7_US:%.*]] = icmp ult i64 0, 0
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 0)
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret ptr [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi ptr [ null, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %cmp7.us = icmp ult i64 0, 0
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 0)
-  %min.idx.next = select i1 %cmp7.us, ptr %iv, ptr %min.idx
-  %iv.next = getelementptr i32, ptr %iv, i64 1
-  %exitcond.not = icmp eq ptr %iv.next, %end
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi ptr [ %min.idx.next, %loop ]
-  ret ptr %res
-}
-
-define void @pointer_index(ptr %start) {
-; CHECK-VF4IC1-LABEL: define void @pointer_index(
-; CHECK-VF4IC1-SAME: ptr [[START:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[PTR_IDX:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[CMP_I_I_I_I2531:%.*]] = icmp ult i16 0, 0
-; CHECK-VF4IC1-NEXT:    [[PTR_SELECT]] = select i1 [[CMP_I_I_I_I2531]], ptr [[PTR_IV]], ptr [[PTR_IDX]]
-; CHECK-VF4IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i16, ptr [[PTR_IV]], i64 1
-; CHECK-VF4IC1-NEXT:    [[CMP_I_I10_NOT_I_I_I:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], null
-; CHECK-VF4IC1-NEXT:    br i1 [[CMP_I_I10_NOT_I_I_I]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    ret void
-;
-; CHECK-VF4IC2-LABEL: define void @pointer_index(
-; CHECK-VF4IC2-SAME: ptr [[START:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[PTR_IDX:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[CMP_I_I_I_I2531:%.*]] = icmp ult i16 0, 0
-; CHECK-VF4IC2-NEXT:    [[PTR_SELECT]] = select i1 [[CMP_I_I_I_I2531]], ptr [[PTR_IV]], ptr [[PTR_IDX]]
-; CHECK-VF4IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i16, ptr [[PTR_IV]], i64 1
-; CHECK-VF4IC2-NEXT:    [[CMP_I_I10_NOT_I_I_I:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], null
-; CHECK-VF4IC2-NEXT:    br i1 [[CMP_I_I10_NOT_I_I_I]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    ret void
-;
-; CHECK-VF1IC2-LABEL: define void @pointer_index(
-; CHECK-VF1IC2-SAME: ptr [[START:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[PTR_IDX:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[CMP_I_I_I_I2531:%.*]] = icmp ult i16 0, 0
-; CHECK-VF1IC2-NEXT:    [[PTR_SELECT]] = select i1 [[CMP_I_I_I_I2531]], ptr [[PTR_IV]], ptr [[PTR_IDX]]
-; CHECK-VF1IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i16, ptr [[PTR_IV]], i64 1
-; CHECK-VF1IC2-NEXT:    [[CMP_I_I10_NOT_I_I_I:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], null
-; CHECK-VF1IC2-NEXT:    br i1 [[CMP_I_I10_NOT_I_I_I]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    ret void
-;
-entry:
-  br label %loop
-
-loop:
-  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ]
-  %ptr.idx = phi ptr [ %start, %entry ], [ %ptr.select, %loop ]
-  %cmp.i.i.i.i2531 = icmp ult i16 0, 0
-  %ptr.select = select i1 %cmp.i.i.i.i2531, ptr %ptr.iv, ptr %ptr.idx
-  %ptr.iv.next = getelementptr inbounds i16, ptr %ptr.iv, i64 1
-  %cmp.i.i10.not.i.i.i = icmp eq ptr %ptr.iv.next, null
-  br i1 %cmp.i.i10.not.i.i.i, label %exit, label %loop
-
-exit:
-  ret void
-}
-
-define ptr @pointer_index_2(ptr %start, ptr %end) {
-; CHECK-VF4IC1-LABEL: define ptr @pointer_index_2(
-; CHECK-VF4IC1-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[CMP_I_I_I_I:%.*]] = icmp ult i16 0, [[MIN_VAL]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = call i16 @llvm.umin.i16(i16 0, i16 [[MIN_VAL]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP_I_I_I_I]], ptr [[PTR_IV]], ptr [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i16, ptr [[PTR_IV]], i64 1
-; CHECK-VF4IC1-NEXT:    [[EXIT_COND:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; CHECK-VF4IC1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret ptr [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define ptr @pointer_index_2(
-; CHECK-VF4IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[CMP_I_I_I_I:%.*]] = icmp ult i16 0, [[MIN_VAL]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = call i16 @llvm.umin.i16(i16 0, i16 [[MIN_VAL]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP_I_I_I_I]], ptr [[PTR_IV]], ptr [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i16, ptr [[PTR_IV]], i64 1
-; CHECK-VF4IC2-NEXT:    [[EXIT_COND:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; CHECK-VF4IC2-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret ptr [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define ptr @pointer_index_2(
-; CHECK-VF1IC2-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[CMP_I_I_I_I:%.*]] = icmp ult i16 0, [[MIN_VAL]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = call i16 @llvm.umin.i16(i16 0, i16 [[MIN_VAL]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP_I_I_I_I]], ptr [[PTR_IV]], ptr [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i16, ptr [[PTR_IV]], i64 1
-; CHECK-VF1IC2-NEXT:    [[EXIT_COND:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; CHECK-VF1IC2-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret ptr [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %min.val  = phi i16 [ 0, %entry ], [ %min.val.next, %loop ]
-  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ]
-  %min.idx = phi ptr [ %start, %entry ], [ %min.idx.next, %loop ]
-  %cmp.i.i.i.i = icmp ult i16 0, %min.val
-  %min.val.next = call i16 @llvm.umin.i16(i16 0, i16 %min.val)
-  %min.idx.next = select i1 %cmp.i.i.i.i, ptr %ptr.iv, ptr %min.idx
-  %ptr.iv.next = getelementptr inbounds i16, ptr %ptr.iv, i64 1
-  %exit.cond = icmp eq ptr %ptr.iv.next, %end
-  br i1 %exit.cond, label %exit, label %loop
-
-exit:
-  %res = phi ptr [ %min.idx.next, %loop ]
-  ret ptr %res
-}
-
-define i64 @test_no_vectorize_select_iv_decrement(ptr %src) {
-; CHECK-VF4IC1-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
-; CHECK-VF4IC1-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
-; CHECK-VF4IC2-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
-; CHECK-VF1IC2-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF1IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %gep = getelementptr i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep
-  %cmp = icmp ugt i64 %min.val, %l
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = add nuw nsw i64 %iv, -1
-  %exitcond.not = icmp eq i64 %iv.next, 0
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  ret i64 %res
-}
-
-define i64 @test_no_vectorize_select_iv_sub(ptr %src) {
-; CHECK-VF4IC1-LABEL: define i64 @test_no_vectorize_select_iv_sub(
-; CHECK-VF4IC1-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_no_vectorize_select_iv_sub(
-; CHECK-VF4IC2-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_no_vectorize_select_iv_sub(
-; CHECK-VF1IC2-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF1IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %gep = getelementptr i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep
-  %cmp = icmp ugt i64 %min.val, %l
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = sub i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 0
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  ret i64 %res
-}
-
-define i64 @test_no_vectorize_select_iv_mul(ptr %src) {
-; CHECK-VF4IC1-LABEL: define i64 @test_no_vectorize_select_iv_mul(
-; CHECK-VF4IC1-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC1-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC1:       [[LOOP]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC1-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC1-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC1-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC1-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
-; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC1:       [[EXIT]]:
-; CHECK-VF4IC1-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC1-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF4IC2-LABEL: define i64 @test_no_vectorize_select_iv_mul(
-; CHECK-VF4IC2-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF4IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF4IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF4IC2:       [[LOOP]]:
-; CHECK-VF4IC2-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF4IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF4IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF4IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF4IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF4IC2-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
-; CHECK-VF4IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-VF4IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF4IC2:       [[EXIT]]:
-; CHECK-VF4IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF4IC2-NEXT:    ret i64 [[RES]]
-;
-; CHECK-VF1IC2-LABEL: define i64 @test_no_vectorize_select_iv_mul(
-; CHECK-VF1IC2-SAME: ptr [[SRC:%.*]]) {
-; CHECK-VF1IC2-NEXT:  [[ENTRY:.*]]:
-; CHECK-VF1IC2-NEXT:    br label %[[LOOP:.*]]
-; CHECK-VF1IC2:       [[LOOP]]:
-; CHECK-VF1IC2-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-VF1IC2-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
-; CHECK-VF1IC2-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
-; CHECK-VF1IC2-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-VF1IC2-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-VF1IC2-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
-; CHECK-VF1IC2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-VF1IC2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK-VF1IC2:       [[EXIT]]:
-; CHECK-VF1IC2-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-VF1IC2-NEXT:    ret i64 [[RES]]
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
-  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
-  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
-  %gep = getelementptr i64, ptr %src, i64 %iv
-  %l = load i64, ptr %gep
-  %cmp = icmp ugt i64 %min.val, %l
-  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
-  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
-  %iv.next = mul i64 %iv, 2
-  %exitcond.not = icmp eq i64 %iv.next, 128
-  br i1 %exitcond.not, label %exit, label %loop
-
-exit:
-  %res = phi i64 [ %min.idx.next, %loop ]
-  ret i64 %res
-}
-
-declare i64 @llvm.umin.i64(i64, i64)
-declare i16 @llvm.umin.i16(i16, i16)
-;.
-; CHECK-VF4IC1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-VF4IC1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-VF4IC1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-VF4IC1: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-;.
-; CHECK-VF4IC2: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-VF4IC2: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-VF4IC2: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-VF4IC2: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-;.
-; CHECK-VF1IC2: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-VF1IC2: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-VF1IC2: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-VF1IC2: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
-;.
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-smax-last-index.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-smax-last-index.ll
new file mode 100644
index 000000000000..48d2eee60015
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-smax-last-index.ll
@@ -0,0 +1,555 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+define i64 @test_vectorize_select_smax_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_idx_cond_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_idx_cond_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %l, %max.val
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_idx_select_ops_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_idx_select_ops_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[MIN_IDX]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %l, %max.val
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %max.idx, i64 %iv
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_via_select_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_via_select_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = select i1 [[CMP]], i64 [[L]], i64 [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = select i1 %cmp, i64 %l, i64 %max.val
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_idx_all_exit_inst(ptr %src, ptr %smax, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_idx_all_exit_inst(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[SMAX:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RES_SMAX:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 [[RES_SMAX]], ptr [[SMAX]], align 4
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  %res.smax = phi i64 [ %max.val.next, %loop ]
+  store i64 %res.smax, ptr %smax
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smax_idx_min_ops_switched(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smax_idx_min_ops_switched(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[L]], i64 [[MIN_VAL]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %l, i64 %max.val)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_not_vectorize_select_no_min_reduction(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[RED_VAL_NEXT]] = add i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %red.val = phi i64 [ 0, %entry ], [ %red.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %red.val, %l
+  %red.val.next = add i64 %red.val, %l
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_cmp_and_smax_use_different_values(ptr %src, i64 %x, i64 %n) {
+; CHECK-LABEL: define i64 @test_cmp_and_smax_use_different_values(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[X:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[X]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %x
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i32 @test_vectorize_select_smax_idx_with_trunc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @test_vectorize_select_smax_idx_with_trunc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %trunc = trunc i64 %iv to i32
+  %max.idx.next = select i1 %cmp, i32 %trunc, i32 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i32 [ %max.idx.next, %loop ]
+  ret i32 %res
+}
+
+define ptr @test_with_ptr_index(ptr %start, ptr %end) {
+; CHECK-LABEL: define ptr @test_with_ptr_index(
+; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[IV]], align 4
+; CHECK-NEXT:    [[CMP7_US:%.*]] = icmp sle i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi ptr [ null, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %l = load i64, ptr %iv
+  %cmp7.us = icmp sle i64 %l, %max.val
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp7.us, ptr %iv, ptr %max.idx
+  %iv.next = getelementptr i32, ptr %iv, i64 1
+  %exitcond.not = icmp eq ptr %iv.next, %end
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi ptr [ %max.idx.next, %loop ]
+  ret ptr %res
+}
+
+define i64 @test_no_vectorize_select_iv_decrement(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, -1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_sub(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_sub(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = sub i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_mul(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_mul(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = mul i64 %iv, 2
+  %exitcond.not = icmp eq i64 %iv.next, 128
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+declare i64 @llvm.smax.i64(i64, i64)
+declare i16 @llvm.smax.i16(i16, i16)
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-smin-last-index.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-smin-last-index.ll
new file mode 100644
index 000000000000..bca6bf38da8d
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-smin-last-index.ll
@@ -0,0 +1,557 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+; Test cases for selecting the index with the minimum value.
+
+define i64 @test_vectorize_select_smin_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_idx_cond_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_idx_cond_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_idx_select_ops_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_idx_select_ops_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[MIN_IDX]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sle i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %min.idx, i64 %iv
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_via_select_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_via_select_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = select i1 [[CMP]], i64 [[L]], i64 [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = select i1 %cmp, i64 %l, i64 %min.val
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_idx_all_exit_inst(ptr %src, ptr %smin, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_idx_all_exit_inst(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[SMIN:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RES_SMIN:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 [[RES_SMIN]], ptr [[SMIN]], align 4
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  %res.smin = phi i64 [ %min.val.next, %loop ]
+  store i64 %res.smin, ptr %smin
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_smin_idx_min_ops_switched(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_smin_idx_min_ops_switched(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[L]], i64 [[MIN_VAL]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %l, i64 %min.val)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_not_vectorize_select_no_min_reduction(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[RED_VAL_NEXT]] = add i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %red.val = phi i64 [ 0, %entry ], [ %red.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %red.val, %l
+  %red.val.next = add i64 %red.val, %l
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_cmp_and_smin_use_different_values(ptr %src, i64 %x, i64 %n) {
+; CHECK-LABEL: define i64 @test_cmp_and_smin_use_different_values(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[X:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[X]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %x
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i32 @test_vectorize_select_smin_idx_with_trunc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @test_vectorize_select_smin_idx_with_trunc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %trunc = trunc i64 %iv to i32
+  %min.idx.next = select i1 %cmp, i32 %trunc, i32 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i32 [ %min.idx.next, %loop ]
+  ret i32 %res
+}
+
+define ptr @test_with_ptr_index(ptr %start, ptr %end) {
+; CHECK-LABEL: define ptr @test_with_ptr_index(
+; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[IV]], align 4
+; CHECK-NEXT:    [[CMP7_US:%.*]] = icmp sle i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi ptr [ null, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %l = load i64, ptr %iv
+  %cmp7.us = icmp sle i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp7.us, ptr %iv, ptr %min.idx
+  %iv.next = getelementptr i32, ptr %iv, i64 1
+  %exitcond.not = icmp eq ptr %iv.next, %end
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi ptr [ %min.idx.next, %loop ]
+  ret ptr %res
+}
+
+define i64 @test_no_vectorize_select_iv_decrement(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, -1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_sub(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_sub(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = sub i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_mul(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_mul(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp sge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = mul i64 %iv, 2
+  %exitcond.not = icmp eq i64 %iv.next, 128
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+declare i64 @llvm.smin.i64(i64, i64)
+declare i16 @llvm.smin.i16(i16, i16)
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umax-last-index.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umax-last-index.ll
new file mode 100644
index 000000000000..cabb868a1dbe
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umax-last-index.ll
@@ -0,0 +1,555 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+define i64 @test_vectorize_select_umax_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_idx_cond_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_cond_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %l, %max.val
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_idx_select_ops_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_select_ops_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[MIN_IDX]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %l, %max.val
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %max.idx, i64 %iv
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_via_select_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_via_select_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = select i1 [[CMP]], i64 [[L]], i64 [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = select i1 %cmp, i64 %l, i64 %max.val
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_idx_all_exit_inst(ptr %src, ptr %umax, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_all_exit_inst(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[UMAX:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RES_UMAX:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 [[RES_UMAX]], ptr [[UMAX]], align 4
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  %res.umax = phi i64 [ %max.val.next, %loop ]
+  store i64 %res.umax, ptr %umax
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umax_idx_min_ops_switched(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_min_ops_switched(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[L]], i64 [[MIN_VAL]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %l, i64 %max.val)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_not_vectorize_select_no_min_reduction(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[RED_VAL_NEXT]] = add i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %red.val = phi i64 [ 0, %entry ], [ %red.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %red.val, %l
+  %red.val.next = add i64 %red.val, %l
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_cmp_and_umax_use_different_values(ptr %src, i64 %x, i64 %n) {
+; CHECK-LABEL: define i64 @test_cmp_and_umax_use_different_values(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[X:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[X]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %x
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i32 @test_vectorize_select_umax_idx_with_trunc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @test_vectorize_select_umax_idx_with_trunc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %trunc = trunc i64 %iv to i32
+  %max.idx.next = select i1 %cmp, i32 %trunc, i32 %max.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i32 [ %max.idx.next, %loop ]
+  ret i32 %res
+}
+
+define ptr @test_with_ptr_index(ptr %start, ptr %end) {
+; CHECK-LABEL: define ptr @test_with_ptr_index(
+; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[IV]], align 4
+; CHECK-NEXT:    [[CMP7_US:%.*]] = icmp ule i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi ptr [ null, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %l = load i64, ptr %iv
+  %cmp7.us = icmp ule i64 %l, %max.val
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp7.us, ptr %iv, ptr %max.idx
+  %iv.next = getelementptr i32, ptr %iv, i64 1
+  %exitcond.not = icmp eq ptr %iv.next, %end
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi ptr [ %max.idx.next, %loop ]
+  ret ptr %res
+}
+
+define i64 @test_no_vectorize_select_iv_decrement(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = add nuw nsw i64 %iv, -1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_sub(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_sub(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = sub i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_mul(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_mul(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+  %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+  %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %max.val, %l
+  %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
+  %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+  %iv.next = mul i64 %iv, 2
+  %exitcond.not = icmp eq i64 %iv.next, 128
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %max.idx.next, %loop ]
+  ret i64 %res
+}
+
+declare i64 @llvm.umax.i64(i64, i64)
+declare i16 @llvm.umax.i16(i16, i16)
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umin-first-index.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umin-first-index.ll
new file mode 100644
index 000000000000..3011fe3e7393
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umin-first-index.ll
@@ -0,0 +1,557 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+; Test cases for selecting the index with the minimum value.
+
+define i64 @test_vectorize_select_umin_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_cond_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_cond_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ult i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_select_ops_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_select_ops_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[MIN_IDX]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ult i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %min.idx, i64 %iv
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_via_select_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_via_select_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = select i1 [[CMP]], i64 [[L]], i64 [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = select i1 %cmp, i64 %l, i64 %min.val
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_all_exit_inst(ptr %src, ptr %umin, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_all_exit_inst(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[UMIN:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RES_UMIN:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 [[RES_UMIN]], ptr [[UMIN]], align 4
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  %res.umin = phi i64 [ %min.val.next, %loop ]
+  store i64 %res.umin, ptr %umin
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_min_ops_switched(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_min_ops_switched(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[L]], i64 [[MIN_VAL]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %l, i64 %min.val)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_not_vectorize_select_no_min_reduction(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[RED_VAL_NEXT]] = add i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %red.val = phi i64 [ 0, %entry ], [ %red.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %red.val, %l
+  %red.val.next = add i64 %red.val, %l
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_cmp_and_umin_use_different_values(ptr %src, i64 %x, i64 %n) {
+; CHECK-LABEL: define i64 @test_cmp_and_umin_use_different_values(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[X:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[X]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %x
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i32 @test_vectorize_select_umin_idx_with_trunc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @test_vectorize_select_umin_idx_with_trunc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %trunc = trunc i64 %iv to i32
+  %min.idx.next = select i1 %cmp, i32 %trunc, i32 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i32 [ %min.idx.next, %loop ]
+  ret i32 %res
+}
+
+define ptr @test_with_ptr_index(ptr %start, ptr %end) {
+; CHECK-LABEL: define ptr @test_with_ptr_index(
+; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[IV]], align 4
+; CHECK-NEXT:    [[CMP7_US:%.*]] = icmp ult i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi ptr [ null, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %l = load i64, ptr %iv
+  %cmp7.us = icmp ult i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp7.us, ptr %iv, ptr %min.idx
+  %iv.next = getelementptr i32, ptr %iv, i64 1
+  %exitcond.not = icmp eq ptr %iv.next, %end
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi ptr [ %min.idx.next, %loop ]
+  ret ptr %res
+}
+
+define i64 @test_no_vectorize_select_iv_decrement(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, -1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_sub(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_sub(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = sub i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_mul(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_mul(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ugt i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = mul i64 %iv, 2
+  %exitcond.not = icmp eq i64 %iv.next, 128
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+declare i64 @llvm.umin.i64(i64, i64)
+declare i16 @llvm.umin.i16(i16, i16)
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umin-last-index.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umin-last-index.ll
new file mode 100644
index 000000000000..549ee0167f2f
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/select-umin-last-index.ll
@@ -0,0 +1,557 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+; Test cases for selecting the index with the minimum value.
+
+define i64 @test_vectorize_select_umin_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_cond_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_cond_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_select_ops_flipped(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_select_ops_flipped(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[MIN_IDX]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp ule i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %min.idx, i64 %iv
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_via_select_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_via_select_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = select i1 [[CMP]], i64 [[L]], i64 [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = select i1 %cmp, i64 %l, i64 %min.val
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_all_exit_inst(ptr %src, ptr %umin, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_all_exit_inst(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[UMIN:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RES_UMIN:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 [[RES_UMIN]], ptr [[UMIN]], align 4
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  %res.umin = phi i64 [ %min.val.next, %loop ]
+  store i64 %res.umin, ptr %umin
+  ret i64 %res
+}
+
+define i64 @test_vectorize_select_umin_idx_min_ops_switched(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_min_ops_switched(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[L]], i64 [[MIN_VAL]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %l, i64 %min.val)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_not_vectorize_select_no_min_reduction(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[RED_VAL_NEXT]] = add i64 [[RED_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %red.val = phi i64 [ 0, %entry ], [ %red.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %red.val, %l
+  %red.val.next = add i64 %red.val, %l
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_cmp_and_umin_use_different_values(ptr %src, i64 %x, i64 %n) {
+; CHECK-LABEL: define i64 @test_cmp_and_umin_use_different_values(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[X:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[X]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %x
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i32 @test_vectorize_select_umin_idx_with_trunc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @test_vectorize_select_umin_idx_with_trunc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %trunc = trunc i64 %iv to i32
+  %min.idx.next = select i1 %cmp, i32 %trunc, i32 %min.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i32 [ %min.idx.next, %loop ]
+  ret i32 %res
+}
+
+define ptr @test_with_ptr_index(ptr %start, ptr %end) {
+; CHECK-LABEL: define ptr @test_with_ptr_index(
+; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[IV]], align 4
+; CHECK-NEXT:    [[CMP7_US:%.*]] = icmp ule i64 [[L]], [[MIN_VAL]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret ptr [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi ptr [ null, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %l = load i64, ptr %iv
+  %cmp7.us = icmp ule i64 %l, %min.val
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp7.us, ptr %iv, ptr %min.idx
+  %iv.next = getelementptr i32, ptr %iv, i64 1
+  %exitcond.not = icmp eq ptr %iv.next, %end
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi ptr [ %min.idx.next, %loop ]
+  ret ptr %res
+}
+
+define i64 @test_no_vectorize_select_iv_decrement(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_decrement(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = add nuw nsw i64 %iv, -1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_sub(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_sub(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = sub i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 0
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+define i64 @test_no_vectorize_select_iv_mul(ptr %src) {
+; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_mul(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = mul i64 [[IV]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+  %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+  %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
+  %gep = getelementptr i64, ptr %src, i64 %iv
+  %l = load i64, ptr %gep
+  %cmp = icmp uge i64 %min.val, %l
+  %min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
+  %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+  %iv.next = mul i64 %iv, 2
+  %exitcond.not = icmp eq i64 %iv.next, 128
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  %res = phi i64 [ %min.idx.next, %loop ]
+  ret i64 %res
+}
+
+declare i64 @llvm.umin.i64(i64, i64)
+declare i16 @llvm.umin.i16(i16, i16)
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/single-scalar-cast-minbw.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/single-scalar-cast-minbw.ll
new file mode 100644
index 000000000000..b8da9ac84a80
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/single-scalar-cast-minbw.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+define void @minbw_cast(ptr %dst, i64 %n, i1 %bool1, i1 %bool2) {
+; CHECK-LABEL: define void @minbw_cast(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i1 [[BOOL1:%.*]], i1 [[BOOL2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[BOOL1_EXT:%.*]] = zext i1 [[BOOL1]] to i32
+; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[BOOL2]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[BOOL1_EXT]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT2]] to <4 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i1> [[BROADCAST_SPLAT]] to <4 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i8> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
+; CHECK-NEXT:    store i8 [[TMP3]], ptr [[DST]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[BOOL2_EXT:%.*]] = zext i1 [[BOOL2]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[BOOL1_EXT]], [[BOOL2_EXT]]
+; CHECK-NEXT:    [[XOR_TRUNC:%.*]] = trunc i32 [[XOR]] to i8
+; CHECK-NEXT:    store i8 [[XOR_TRUNC]], ptr [[DST]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %bool1.ext = zext i1 %bool1 to i32
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %bool2.ext = zext i1 %bool2 to i32
+  %xor = xor i32 %bool1.ext, %bool2.ext
+  %xor.trunc = trunc i32 %xor to i8
+  store i8 %xor.trunc, ptr %dst, align 1
+  %iv.next = add i64 %iv, 1
+  %cmp = icmp ult i64 %iv.next, %n
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
index d1490eaa8be0..3bbcb2b62c38 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
@@ -59,7 +59,7 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: ir-bb<loop.header>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop.header>:
@@ -142,7 +142,7 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values() {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: ir-bb<loop.header>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop.header>:
@@ -222,7 +222,7 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values_2() {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT: ir-bb<loop.header>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop.header>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-branch-weights.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-branch-weights.ll
new file mode 100644
index 000000000000..d5acf5c38f76
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-branch-weights.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF8UF1 %s
+; RUN: opt -p loop-vectorize -force-vector-width=8 -force-vector-interleave=2 -S %s | FileCheck --check-prefixes=VF8UF2 %s
+; RUN: opt -p loop-vectorize -force-vector-width=16 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF16UF1 %s
+
+; Check if the vector loop condition can be simplified to true for a given
+; VF/IC combination.
+define void @test_tc_between_8_and_17(ptr %A, i64 range(i64 8, 17) %N) {
+; VF8UF1-LABEL: define void @test_tc_between_8_and_17(
+; VF8UF1-SAME: ptr [[A:%.*]], i64 range(i64 8, 17) [[N:%.*]]) {
+; VF8UF1-NEXT:  [[ENTRY:.*]]:
+; VF8UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; VF8UF1:       [[VECTOR_PH]]:
+; VF8UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; VF8UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF8UF1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
+; VF8UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF8UF1:       [[VECTOR_BODY]]:
+; VF8UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF8UF1-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; VF8UF1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; VF8UF1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; VF8UF1-NEXT:    [[TMP2:%.*]] = add nsw <8 x i8> [[WIDE_LOAD]], splat (i8 10)
+; VF8UF1-NEXT:    store <8 x i8> [[TMP2]], ptr [[TMP1]], align 1
+; VF8UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; VF8UF1-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF8UF1-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; VF8UF1:       [[MIDDLE_BLOCK]]:
+; VF8UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF8UF1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF5:![0-9]+]]
+; VF8UF1:       [[SCALAR_PH]]:
+; VF8UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VF8UF1-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; VF8UF1-NEXT:    br label %[[LOOP:.*]]
+; VF8UF1:       [[LOOP]]:
+; VF8UF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF1-NEXT:    [[P_SRC:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF1-NEXT:    [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 1
+; VF8UF1-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC]], align 1
+; VF8UF1-NEXT:    [[ADD:%.*]] = add nsw i8 [[L]], 10
+; VF8UF1-NEXT:    store i8 [[ADD]], ptr [[P_SRC]], align 1
+; VF8UF1-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; VF8UF1-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; VF8UF1-NEXT:    br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+; VF8UF1:       [[EXIT]]:
+; VF8UF1-NEXT:    ret void
+;
+; VF8UF2-LABEL: define void @test_tc_between_8_and_17(
+; VF8UF2-SAME: ptr [[A:%.*]], i64 range(i64 8, 17) [[N:%.*]]) {
+; VF8UF2-NEXT:  [[ENTRY:.*]]:
+; VF8UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; VF8UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; VF8UF2:       [[VECTOR_PH]]:
+; VF8UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; VF8UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF8UF2-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
+; VF8UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF8UF2:       [[VECTOR_BODY]]:
+; VF8UF2-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i32 0
+; VF8UF2-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[A]], i32 8
+; VF8UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; VF8UF2-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; VF8UF2-NEXT:    [[TMP3:%.*]] = add nsw <8 x i8> [[WIDE_LOAD]], splat (i8 10)
+; VF8UF2-NEXT:    [[TMP4:%.*]] = add nsw <8 x i8> [[WIDE_LOAD1]], splat (i8 10)
+; VF8UF2-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i32 0
+; VF8UF2-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i32 8
+; VF8UF2-NEXT:    store <8 x i8> [[TMP3]], ptr [[TMP5]], align 1
+; VF8UF2-NEXT:    store <8 x i8> [[TMP4]], ptr [[TMP6]], align 1
+; VF8UF2-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
+; VF8UF2:       [[MIDDLE_BLOCK]]:
+; VF8UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF8UF2-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VF8UF2:       [[SCALAR_PH]]:
+; VF8UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VF8UF2-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; VF8UF2-NEXT:    br label %[[LOOP:.*]]
+; VF8UF2:       [[LOOP]]:
+; VF8UF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF2-NEXT:    [[P_SRC:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF2-NEXT:    [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 1
+; VF8UF2-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC]], align 1
+; VF8UF2-NEXT:    [[ADD:%.*]] = add nsw i8 [[L]], 10
+; VF8UF2-NEXT:    store i8 [[ADD]], ptr [[P_SRC]], align 1
+; VF8UF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; VF8UF2-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; VF8UF2-NEXT:    br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; VF8UF2:       [[EXIT]]:
+; VF8UF2-NEXT:    ret void
+;
+; VF16UF1-LABEL: define void @test_tc_between_8_and_17(
+; VF16UF1-SAME: ptr [[A:%.*]], i64 range(i64 8, 17) [[N:%.*]]) {
+; VF16UF1-NEXT:  [[ENTRY:.*]]:
+; VF16UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; VF16UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; VF16UF1:       [[VECTOR_PH]]:
+; VF16UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; VF16UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; VF16UF1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
+; VF16UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF16UF1:       [[VECTOR_BODY]]:
+; VF16UF1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i32 0
+; VF16UF1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; VF16UF1-NEXT:    [[TMP2:%.*]] = add nsw <16 x i8> [[WIDE_LOAD]], splat (i8 10)
+; VF16UF1-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i32 0
+; VF16UF1-NEXT:    store <16 x i8> [[TMP2]], ptr [[TMP3]], align 1
+; VF16UF1-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
+; VF16UF1:       [[MIDDLE_BLOCK]]:
+; VF16UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; VF16UF1-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VF16UF1:       [[SCALAR_PH]]:
+; VF16UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VF16UF1-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; VF16UF1-NEXT:    br label %[[LOOP:.*]]
+; VF16UF1:       [[LOOP]]:
+; VF16UF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF16UF1-NEXT:    [[P_SRC:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]
+; VF16UF1-NEXT:    [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 1
+; VF16UF1-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC]], align 1
+; VF16UF1-NEXT:    [[ADD:%.*]] = add nsw i8 [[L]], 10
+; VF16UF1-NEXT:    store i8 [[ADD]], ptr [[P_SRC]], align 1
+; VF16UF1-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; VF16UF1-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; VF16UF1-NEXT:    br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; VF16UF1:       [[EXIT]]:
+; VF16UF1-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ]
+  %p.src.next = getelementptr inbounds i8, ptr %p.src, i64 1
+  %l = load i8, ptr %p.src, align 1
+  %add = add nsw i8 %l, 10
+  store i8 %add, ptr %p.src
+  %iv.next = add nsw i64 %iv, 1
+  %cmp = icmp eq i64 %iv.next, %N
+  br i1 %cmp, label %exit, label %loop, !prof !0
+
+exit:
+  ret void
+}
+
+!0 = !{!"branch_weights", !"expected", i32 1, i32 2000}
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index a8d90fa30854..528f2448616e 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -20,7 +20,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-NEXT:  N0 -> N2 [ label="F"]
 ; CHECK-NEXT:  N1 [label =
 ; CHECK-NEXT:    "scalar.ph:\l" +
-; CHECK-NEXT:    "  EMIT-SCALAR vp\<%bc.resume.val\> = resume-phi vp\<%2\>, ir\<0\>\l" +
+; CHECK-NEXT:    "  EMIT-SCALAR vp\<%bc.resume.val\> = phi [ vp\<%2\>, middle.block ], [ ir\<0\>, ir-bb\<for.body.preheader\> ]\l" +
 ; CHECK-NEXT:    "Successor(s): ir-bb\<for.body\>\l"
 ; CHECK-NEXT:  ]
 ; CHECK-NEXT:  N1 -> N3 [ label=""]
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 6b8311d10252..59277186195f 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -38,7 +38,7 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[VEC_TC]]>, ir<0>
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[VEC_TC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop.header>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop.header>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
index ab69e02cc778..d441e4123975 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
@@ -91,7 +91,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<scalar.ph>:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[END]]>, ir<%start>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[END]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop.header>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop.header>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index 783cc255139a..91e70d2467fd 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -49,8 +49,8 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME1:%.+]]> = resume-phi vp<[[END1]]>, ir<%and>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = resume-phi vp<[[END2]]>, ir<%A>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME1:%.+]]> = phi [ vp<[[END1]]>, middle.block ], [ ir<%and>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = phi [ vp<[[END2]]>, middle.block ], [ ir<%A>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
@@ -98,8 +98,8 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<scalar.ph>:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME1:%.+]]> = resume-phi vp<[[END1]]>, ir<%and>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = resume-phi vp<[[END2]]>, ir<%A>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME1:%.+]]> = phi [ vp<[[END1]]>, middle.block ], [ ir<%and>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = phi [ vp<[[END2]]>, middle.block ], [ ir<%A>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
index c22ecda266fd..978f1b80d26d 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
@@ -16,12 +16,13 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
 ; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[RDX_START]]>, ir<%red.next>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:   vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
@@ -45,8 +46,8 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = phi [ vp<[[RED_RES]]>, middle.block ], [ ir<0.000000e+00>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -84,12 +85,13 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
 ; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[RDX_START]]>, ir<%red.next>
 ; CHECK-NEXT:   vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
 ; CHECK-NEXT:   vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
@@ -112,8 +114,8 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = phi [ vp<[[RED_RES]]>, middle.block ], [ ir<0.000000e+00>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -153,12 +155,13 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector nnan ninf nsz ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
 ; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd>
+; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%sum.07> = phi vp<[[RDX_START]]>, ir<%muladd>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
 ; CHECK-NEXT:   vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
@@ -186,8 +189,8 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = phi [ vp<[[RED_RES]]>, middle.block ], [ ir<0.000000e+00>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -237,7 +240,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<%cond>
+; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
 ; CHECK-NEXT:   EMIT vp<[[EXT:%.+]]> = extract-last-element vp<[[RDX_RES]]>
 ; CHECK-NEXT:   EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<%cmp.n>
@@ -248,8 +251,8 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val> = resume-phi vp<{{.+}}>, ir<0>
-; CHECK-NEXT:   EMIT-SCALAR vp<%bc.merge.rdx> = resume-phi vp<[[RDX_RES]]>, ir<%start>
+; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<{{.+}}>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[RDX_RES]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]
 ;
 entry:
   br label %loop
@@ -278,12 +281,13 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
 ; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK:      vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi ir<0>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR:%.+]]> = vector-pointer ir<%arrayidx>
@@ -322,12 +326,13 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
 ; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK:      vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi ir<0>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX0:%.+]]> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR0:%.+]]> = vector-pointer ir<[[ARRAYIDX0]]>
@@ -371,12 +376,13 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
 ; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK:      vector.ph:
+; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi ir<0>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX0:%.+]]> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR0:%.+]]> = vector-pointer ir<[[ARRAYIDX0]]>
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 704778f8f968..732214aa1449 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -46,7 +46,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<for.body.preheader> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
@@ -117,7 +117,7 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<for.body.preheader> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
@@ -206,7 +206,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
@@ -289,7 +289,7 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[IV_END]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[IV_END]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
@@ -394,7 +394,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -479,7 +479,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[IV_END]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[IV_END]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -548,7 +548,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -616,7 +616,7 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -686,7 +686,7 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -776,7 +776,7 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
@@ -855,7 +855,7 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
@@ -972,8 +972,8 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_P:%.*]]> = phi [ vp<[[RESUME_1]]>, middle.block ], [ ir<22>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<loop>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index ca27ce0af667..9550454b1a3c 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -58,8 +58,8 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  scalar.ph:
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME1:%.+]]> = resume-phi vp<[[VEC_TC]]>, ir<0>
-; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = resume-phi vp<[[END]]>, ir<1024>
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME1:%.+]]> = phi [ vp<[[VEC_TC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:    EMIT-SCALAR vp<[[RESUME2:%.+]]>.1 = phi [ vp<[[END]]>, middle.block ], [ ir<1024>, ir-bb<entry> ]
 ; CHECK-NEXT:  Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index c78568e8a6f7..4a4bda254bf8 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -1090,7 +1090,7 @@ define void @merge_with_dead_gep_between_regions(i32 %n, i32 %k, ptr noalias %sr
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[END]]>, ir<%n>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[END]]>, middle.block ], [ ir<%n>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop>:
@@ -1184,7 +1184,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = resume-phi vp<[[END]]>, ir<%start>
+; CHECK-NEXT:   EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[END]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<loop.header>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<loop.header>:
diff --git a/external/llvm-project/llvm/test/Transforms/LoopVersioningLICM/load-from-unknown-address.ll b/external/llvm-project/llvm/test/Transforms/LoopVersioningLICM/load-from-unknown-address.ll
index e9b295403919..89576f1684c3 100644
--- a/external/llvm-project/llvm/test/Transforms/LoopVersioningLICM/load-from-unknown-address.ll
+++ b/external/llvm-project/llvm/test/Transforms/LoopVersioningLICM/load-from-unknown-address.ll
@@ -7,15 +7,25 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i6
 ; accesses but not others.
 
 ; Load from a gep whose bounds can't be calculated as the offset is loaded from memory
-; FIXME: Not knowing the bounds of the gep shouldn't stop us from hoisting the load of rval
 define void @gep_loaded_offset(ptr %p, ptr %q, ptr %r, i32 %n) {
 ; CHECK-LABEL: define void @gep_loaded_offset(
 ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr [[R:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
-; CHECK:       [[WHILE_BODY]]:
-; CHECK-NEXT:    [[N_ADDR:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[P_ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[P]], %[[ENTRY]] ]
+; CHECK-NEXT:  [[WHILE_BODY_LVER_CHECK:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[R]], i64 8
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[R]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[WHILE_BODY_PH_LVER_ORIG:.*]], label %[[WHILE_BODY_PH:.*]]
+; CHECK:       [[WHILE_BODY_PH_LVER_ORIG]]:
+; CHECK-NEXT:    br label %[[WHILE_BODY_LVER_ORIG:.*]]
+; CHECK:       [[WHILE_BODY_LVER_ORIG]]:
+; CHECK-NEXT:    [[N_ADDR:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[N]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
+; CHECK-NEXT:    [[P_ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[P]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
 ; CHECK-NEXT:    [[DEC]] = add nsw i32 [[N_ADDR]], -1
 ; CHECK-NEXT:    [[RVAL:%.*]] = load i64, ptr [[R]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[RVAL]]
@@ -23,7 +33,24 @@ define void @gep_loaded_offset(ptr %p, ptr %q, ptr %r, i32 %n) {
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[P_ADDR]], i64 4
 ; CHECK-NEXT:    store i32 [[VAL]], ptr [[P_ADDR]], align 4
 ; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[WHILE_BODY_LVER_ORIG]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[WHILE_BODY_PH]]:
+; CHECK-NEXT:    [[RVAL1:%.*]] = load i64, ptr [[R]], align 4, !alias.scope [[META2:![0-9]+]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[RVAL1]]
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[N_ADDR1:%.*]] = phi i32 [ [[DEC1:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[P_ADDR1:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[WHILE_BODY]] ], [ [[P]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[DEC1]] = add nsw i32 [[N_ADDR1]], -1
+; CHECK-NEXT:    [[VAL1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[P_ADDR1]], i64 4
+; CHECK-NEXT:    store i32 [[VAL1]], ptr [[P_ADDR1]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META2]]
+; CHECK-NEXT:    [[TOBOOL_NOT1:%.*]] = icmp eq i32 [[DEC1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT1]], label %[[WHILE_END_LOOPEXIT2:.*]], label %[[WHILE_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_END_LOOPEXIT2]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
 ; CHECK:       [[WHILE_END]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -89,15 +116,25 @@ while.end:
 }
 
 ; Load from a gep whose bounds can't be calculated as the pointer is loaded from memory
-; FIXME: Not knowing the bounds of the gep shouldn't stop us from hoisting the load of rval
 define void @gep_loaded_base(ptr %p, ptr %q, ptr %r, i32 %n) {
 ; CHECK-LABEL: define void @gep_loaded_base(
 ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr [[R:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
-; CHECK:       [[WHILE_BODY]]:
-; CHECK-NEXT:    [[N_ADDR:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[P_ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[P]], %[[ENTRY]] ]
+; CHECK-NEXT:  [[WHILE_BODY_LVER_CHECK:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[R]], i64 8
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[R]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[WHILE_BODY_PH_LVER_ORIG:.*]], label %[[WHILE_BODY_PH:.*]]
+; CHECK:       [[WHILE_BODY_PH_LVER_ORIG]]:
+; CHECK-NEXT:    br label %[[WHILE_BODY_LVER_ORIG:.*]]
+; CHECK:       [[WHILE_BODY_LVER_ORIG]]:
+; CHECK-NEXT:    [[N_ADDR:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[N]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
+; CHECK-NEXT:    [[P_ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[P]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
 ; CHECK-NEXT:    [[DEC]] = add nsw i32 [[N_ADDR]], -1
 ; CHECK-NEXT:    [[RVAL:%.*]] = load ptr, ptr [[R]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[RVAL]], i64 0
@@ -105,7 +142,24 @@ define void @gep_loaded_base(ptr %p, ptr %q, ptr %r, i32 %n) {
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[P_ADDR]], i64 4
 ; CHECK-NEXT:    store i32 [[VAL]], ptr [[P_ADDR]], align 4
 ; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[WHILE_BODY_LVER_ORIG]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[WHILE_BODY_PH]]:
+; CHECK-NEXT:    [[RVAL1:%.*]] = load ptr, ptr [[R]], align 4, !alias.scope [[META10:![0-9]+]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[RVAL1]], i64 0
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[N_ADDR1:%.*]] = phi i32 [ [[DEC1:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[P_ADDR1:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[WHILE_BODY]] ], [ [[P]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[DEC1]] = add nsw i32 [[N_ADDR1]], -1
+; CHECK-NEXT:    [[VAL1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[P_ADDR1]], i64 4
+; CHECK-NEXT:    store i32 [[VAL1]], ptr [[P_ADDR1]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META10]]
+; CHECK-NEXT:    [[TOBOOL_NOT1:%.*]] = icmp eq i32 [[DEC1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT1]], label %[[WHILE_END_LOOPEXIT2:.*]], label %[[WHILE_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_END_LOOPEXIT2]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
 ; CHECK:       [[WHILE_END]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -129,15 +183,25 @@ while.end:
 }
 
 ; Load from a gep with an offset that scalar evolution can't describe
-; FIXME: Not knowing the bounds of the gep shouldn't stop us from hoisting the load of qval
 define void @gep_strange_offset(ptr %p, ptr %q, ptr %r, i32 %n) {
 ; CHECK-LABEL: define void @gep_strange_offset(
 ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr [[R:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
-; CHECK:       [[WHILE_BODY]]:
-; CHECK-NEXT:    [[N_ADDR:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[P_ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[P]], %[[ENTRY]] ]
+; CHECK-NEXT:  [[WHILE_BODY_LVER_CHECK:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[Q]], i64 4
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[WHILE_BODY_PH_LVER_ORIG:.*]], label %[[WHILE_BODY_PH:.*]]
+; CHECK:       [[WHILE_BODY_PH_LVER_ORIG]]:
+; CHECK-NEXT:    br label %[[WHILE_BODY_LVER_ORIG:.*]]
+; CHECK:       [[WHILE_BODY_LVER_ORIG]]:
+; CHECK-NEXT:    [[N_ADDR:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[N]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
+; CHECK-NEXT:    [[P_ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[P]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
 ; CHECK-NEXT:    [[DEC]] = add nsw i32 [[N_ADDR]], -1
 ; CHECK-NEXT:    [[QVAL:%.*]] = load i32, ptr [[Q]], align 4
 ; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[DEC]], 2
@@ -148,7 +212,27 @@ define void @gep_strange_offset(ptr %p, ptr %q, ptr %r, i32 %n) {
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[P_ADDR]], i64 4
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[P_ADDR]], align 4
 ; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[WHILE_BODY_LVER_ORIG]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       [[WHILE_BODY_PH]]:
+; CHECK-NEXT:    [[QVAL1:%.*]] = load i32, ptr [[Q]], align 4, !alias.scope [[META17:![0-9]+]]
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[N_ADDR1:%.*]] = phi i32 [ [[DEC1:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[P_ADDR1:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[WHILE_BODY]] ], [ [[P]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[DEC1]] = add nsw i32 [[N_ADDR1]], -1
+; CHECK-NEXT:    [[REM1:%.*]] = srem i32 [[DEC1]], 2
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[REM1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[R]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[VAL1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[VAL1]], [[QVAL1]]
+; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[P_ADDR1]], i64 4
+; CHECK-NEXT:    store i32 [[ADD1]], ptr [[P_ADDR1]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
+; CHECK-NEXT:    [[TOBOOL_NOT1:%.*]] = icmp eq i32 [[DEC1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT1]], label %[[WHILE_END_LOOPEXIT2:.*]], label %[[WHILE_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_END_LOOPEXIT2]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
 ; CHECK:       [[WHILE_END]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -175,15 +259,24 @@ while.end:
 }
 
 ; A memcpy-like loop where the source address is loaded from a pointer
-; FIXME: We should be able to hoist the load of the source address pointer
 define void @memcpy_load_src(ptr %dst, ptr %src, i32 %n) {
 ; CHECK-LABEL: define void @memcpy_load_src(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
-; CHECK:       [[WHILE_BODY]]:
-; CHECK-NEXT:    [[N_VAL:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[DST_VAL:%.*]] = phi ptr [ [[DST_VAL_NEXT:%.*]], %[[WHILE_BODY]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-NEXT:  [[WHILE_BODY_LVER_CHECK:.*:]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 8
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[WHILE_BODY_PH_LVER_ORIG:.*]], label %[[WHILE_BODY_PH:.*]]
+; CHECK:       [[WHILE_BODY_PH_LVER_ORIG]]:
+; CHECK-NEXT:    br label %[[WHILE_BODY_LVER_ORIG:.*]]
+; CHECK:       [[WHILE_BODY_LVER_ORIG]]:
+; CHECK-NEXT:    [[N_VAL:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[N]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
+; CHECK-NEXT:    [[DST_VAL:%.*]] = phi ptr [ [[DST_VAL_NEXT:%.*]], %[[WHILE_BODY_LVER_ORIG]] ], [ [[DST]], %[[WHILE_BODY_PH_LVER_ORIG]] ]
 ; CHECK-NEXT:    [[DEC]] = add nsw i32 [[N_VAL]], -1
 ; CHECK-NEXT:    [[SRC_VAL:%.*]] = load ptr, ptr [[SRC]], align 8
 ; CHECK-NEXT:    [[SRC_VAL_NEXT:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_VAL]], i64 1
@@ -192,7 +285,26 @@ define void @memcpy_load_src(ptr %dst, ptr %src, i32 %n) {
 ; CHECK-NEXT:    [[VAL:%.*]] = load i8, ptr [[SRC_VAL]], align 1
 ; CHECK-NEXT:    store i8 [[VAL]], ptr [[DST_VAL]], align 1
 ; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[DEC]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[WHILE_BODY_LVER_ORIG]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       [[WHILE_BODY_PH]]:
+; CHECK-NEXT:    [[SRC_PROMOTED:%.*]] = load ptr, ptr [[SRC]], align 8, !alias.scope [[META24:![0-9]+]], !noalias [[META27:![0-9]+]]
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[SRC_VAL_NEXT3:%.*]] = phi ptr [ [[SRC_VAL_NEXT1:%.*]], %[[WHILE_BODY]] ], [ [[SRC_PROMOTED]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[N_VAL1:%.*]] = phi i32 [ [[DEC1:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[DST_VAL1:%.*]] = phi ptr [ [[DST_VAL_NEXT1:%.*]], %[[WHILE_BODY]] ], [ [[DST]], %[[WHILE_BODY_PH]] ]
+; CHECK-NEXT:    [[DEC1]] = add nsw i32 [[N_VAL1]], -1
+; CHECK-NEXT:    [[SRC_VAL_NEXT1]] = getelementptr inbounds nuw i8, ptr [[SRC_VAL_NEXT3]], i64 1
+; CHECK-NEXT:    [[DST_VAL_NEXT1]] = getelementptr inbounds nuw i8, ptr [[DST_VAL1]], i64 1
+; CHECK-NEXT:    store ptr [[SRC_VAL_NEXT1]], ptr [[SRC]], align 8, !alias.scope [[META24]], !noalias [[META27]]
+; CHECK-NEXT:    [[VAL1:%.*]] = load i8, ptr [[SRC_VAL_NEXT3]], align 1
+; CHECK-NEXT:    store i8 [[VAL1]], ptr [[DST_VAL1]], align 1, !alias.scope [[META27]]
+; CHECK-NEXT:    [[TOBOOL_NOT1:%.*]] = icmp eq i32 [[DEC1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT1]], label %[[WHILE_END_LOOPEXIT2:.*]], label %[[WHILE_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_END_LOOPEXIT2]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
 ; CHECK:       [[WHILE_END]]:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/binop.ll b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/binop.ll
index 9160ced2715a..265a142db9aa 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/binop.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/binop.ll
@@ -11,17 +11,15 @@ define void @add_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = add <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -35,17 +33,15 @@ define void @fadd_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x float>, ptr %lhs
   %rhsv = load <4 x float>, ptr %rhs
   %op = fadd <4 x float> %lhsv, %rhsv
-  %opt  = call <4 x float> @llvm.matrix.transpose(<4 x float> %op, i32 2, i32 2)
-  %optt = call <4 x float> @llvm.matrix.transpose(<4 x float> %opt, i32 2, i32 2)
-  store <4 x float> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -59,17 +55,15 @@ define void @sub_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = sub <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -83,17 +77,15 @@ define void @fsub_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub nnan <2 x float> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fsub nnan <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x float>, ptr %lhs
   %rhsv = load <4 x float>, ptr %rhs
   %op = fsub nnan <4 x float> %lhsv, %rhsv
-  %opt  = call <4 x float> @llvm.matrix.transpose(<4 x float> %op, i32 2, i32 2)
-  %optt = call <4 x float> @llvm.matrix.transpose(<4 x float> %opt, i32 2, i32 2)
-  store <4 x float> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -107,17 +99,15 @@ define void @mul_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = mul <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -131,17 +121,15 @@ define void @fmul_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract <2 x float> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul contract <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x float>, ptr %lhs
   %rhsv = load <4 x float>, ptr %rhs
   %op = fmul contract <4 x float> %lhsv, %rhsv
-  %opt  = call <4 x float> @llvm.matrix.transpose(<4 x float> %op, i32 2, i32 2)
-  %optt = call <4 x float> @llvm.matrix.transpose(<4 x float> %opt, i32 2, i32 2)
-  store <4 x float> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -155,17 +143,15 @@ define void @udiv_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = udiv <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -179,17 +165,15 @@ define void @sdiv_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = sdiv <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -203,17 +187,15 @@ define void @fdiv_2x2(ptr %num, ptr %denom, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = fdiv nnan <2 x double> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fdiv nnan <2 x double> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 32
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr double, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[VEC_GEP5]], align 16
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[VEC_GEP5]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %numv = load <4 x double>, ptr %num
   %denomv = load <4 x double>, ptr %denom
-  %div = fdiv nnan <4 x double> %numv, %denomv
-  %divt  = call <4 x double> @llvm.matrix.transpose(<4 x double> %div, i32 2, i32 2)
-  %divtt = call <4 x double> @llvm.matrix.transpose(<4 x double> %divt, i32 2, i32 2)
-  store <4 x double> %divtt, ptr %out
+  %op = fdiv nnan <4 x double> %numv, %denomv
+  call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -227,17 +209,15 @@ define void @urem_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = urem <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = urem <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -251,17 +231,15 @@ define void @srem_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = srem <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = srem <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = srem <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -275,17 +253,15 @@ define void @frem_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = frem fast <2 x float> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = frem fast <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x float>, ptr %lhs
   %rhsv = load <4 x float>, ptr %rhs
   %op = frem fast <4 x float> %lhsv, %rhsv
-  %opt  = call <4 x float> @llvm.matrix.transpose(<4 x float> %op, i32 2, i32 2)
-  %optt = call <4 x float> @llvm.matrix.transpose(<4 x float> %opt, i32 2, i32 2)
-  store <4 x float> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -299,17 +275,15 @@ define void @shl_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = shl <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -323,17 +297,15 @@ define void @lshr_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = lshr <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -347,17 +319,15 @@ define void @ashr_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = ashr <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = ashr <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -371,17 +341,15 @@ define void @and_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = and <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -395,17 +363,15 @@ define void @or_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = or <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
 
@@ -419,16 +385,50 @@ define void @xor_2x2(ptr %lhs, ptr %rhs, ptr %out) {
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %lhsv = load <4 x i32>, ptr %lhs
   %rhsv = load <4 x i32>, ptr %rhs
   %op = xor <4 x i32> %lhsv, %rhsv
-  %opt  = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %op, i32 2, i32 2)
-  %optt = call <4 x i32> @llvm.matrix.transpose(<4 x i32> %opt, i32 2, i32 2)
-  store <4 x i32> %optt, ptr %out
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @fabs_2x2f64(ptr %in, ptr %out) {
+; CHECK-LABEL: @fabs_2x2f64(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[IN:%.*]], align 32
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[COL_LOAD]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[COL_LOAD1]])
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    ret void
+;
+  %load = load <4 x double>, ptr %in
+  %op = call <4 x double> @llvm.fabs.v4f64(<4 x double> %load)
+  call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @abs_2x2i32(ptr %in, ptr %out) {
+; CHECK-LABEL: @abs_2x2i32(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[IN:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i32, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[COL_LOAD]], i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[COL_LOAD1]], i1 false)
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %load = load <4 x i32>, ptr %in
+  %op = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %load, i1 false)
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
   ret void
 }
diff --git a/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/flatten.ll b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/flatten.ll
new file mode 100644
index 000000000000..8a928c36baff
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/flatten.ll
@@ -0,0 +1,61 @@
+; RUN: opt -passes=lower-matrix-intrinsics -debug-only=lower-matrix-intrinsics -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK
+; REQUIRES: asserts
+
+define void @diag_3x3(ptr %in, ptr %out) {
+  %inv = call <9 x float> @llvm.matrix.column.major.load(ptr %in, i64 3, i1 false, i32 3, i32 3)
+  %diag = shufflevector <9 x float> %inv, <9 x float> poison, <3 x i32> <i32 0, i32 4, i32 8>
+  store <3 x float> %diag, ptr %out
+  ret void
+}
+; CHECK-LABEL: flattening a 3x3 matrix:
+; CHECK-NEXT: %{{.*}} = call <9 x float> @llvm.matrix.column.major.load.v9f32.i64(ptr %{{.*}}, i64 3, i1 false, i32 3, i32 3)
+; CHECK-NEXT: because we do not have a shape-aware lowering for its user:
+; CHECK-NEXT: %{{.*}} = shufflevector <9 x float> %{{.*}}, <9 x float> poison, <3 x i32> <i32 0, i32 4, i32 8>
+
+define void @reshape(ptr %in, ptr %out) {
+entry:
+  %0 = load <4 x double>, ptr %in, align 8
+  %1 = tail call <4 x double> @llvm.matrix.transpose.v4f64(<4 x double> %0, i32 4, i32 1)
+  %2 = tail call <4 x double> @llvm.matrix.transpose.v4f64(<4 x double> %1, i32 1, i32 4)
+  %3 = tail call <4 x double> @llvm.matrix.transpose.v4f64(<4 x double> %2, i32 2, i32 2)
+  %4 = tail call <4 x double> @llvm.matrix.transpose.v4f64(<4 x double> %3, i32 2, i32 2)
+  %5 = tail call <4 x double> @llvm.matrix.transpose.v4f64(<4 x double> %4, i32 2, i32 2)
+  store <4 x double> %5, ptr %out, align 8
+  ret void
+}
+; CHECK-LABEL: matrix reshape from 4x1 to 2x2 using at least 2 shuffles on behalf of:
+; CHECK-NEXT: %{{.*}} = load <4 x double>, ptr %{{.*}}, align 8
+
+define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
+entry:
+  %a = load <6 x double>, ptr %A, align 16
+  %b = load <6 x double>, ptr %B, align 16
+  %c = load <8 x double>, ptr %C, align 16
+  %b_t = call <6 x double> @llvm.matrix.transpose.v6f64.v6f64(<6 x double> %b, i32 2, i32 3)
+  %c_t = call <8 x double> @llvm.matrix.transpose.v8f64.v8f64(<8 x double> %c, i32 4, i32 2)
+  %m1 = call <12 x double> @llvm.matrix.multiply.v12f64.v6f64.v8f64(<6 x double> %b_t, <8 x double> %c_t, i32 3, i32 2, i32 4)
+  %m2 = call <8 x double> @llvm.matrix.multiply.v8f64.v6f64.v12f64(<6 x double> %a, <12 x double> %m1, i32 2, i32 3, i32 4)
+  store <8 x double> %m2, ptr %R, align 16
+  ret void
+}
+; CHECK-LABEL: flattening a 2x3 matrix:
+; CHECK-NEXT: %{{.*}} = load <6 x double>, ptr %{{.*}}, align 16
+; CHECK-NEXT: because we do not have a shape-aware lowering for its user:
+; CHECK-NEXT: %{{.*}} = shufflevector <6 x double> %{{.*}}, <6 x double> poison, <2 x i32> <i32 4, i32 5>
+
+; CHECK-LABEL: flattening a 4x3 matrix:
+; CHECK-NEXT: %{{.*}} = call <12 x double> @llvm.matrix.multiply.v12f64.v8f64.v6f64(<8 x double> %{{.*}}, <6 x double> %{{.*}}, i32 4, i32 2, i32 3)
+; CHECK-NEXT: because we do not have a shape-aware lowering for its user:
+; CHECK-NEXT: %{{.*}} = shufflevector <12 x double> %{{.*}}, <12 x double> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+
+
+define void @redundant_transpose_of_shuffle(<4 x float> %m, ptr %dst) {
+entry:
+  %shuffle = shufflevector <4 x float> %m, <4 x float> zeroinitializer, <4 x i32> zeroinitializer
+  %t = tail call <4 x float> @llvm.matrix.transpose.v3f32(<4 x float> %shuffle, i32 1, i32 4)
+  store <4 x float> %t, ptr %dst, align 4
+  ret void
+}
+
+; CHECK-LABEL: splitting a 4x1 matrix with 1 shuffles beacuse we do not have a shape-aware lowering for its def:
+; CHECK-NEXT: %{{.*}} = shufflevector <4 x float> %{{.*}}, <4 x float> zeroinitializer, <4 x i32> zeroinitializer
\ No newline at end of file
diff --git a/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/select.ll b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/select.ll
new file mode 100644
index 000000000000..bd97915759aa
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/select.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define void @select_2x2_bot(i1 %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_bot(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr float, ptr [[RHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[COND:%.*]], <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[COND]], <2 x float> [[COL_LOAD1]], <2 x float> [[COL_LOAD4]]
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 4
+; CHECK-NEXT:    ret void
+;
+  %lhsv = load <4 x float>, ptr %lhs
+  %rhsv = load <4 x float>, ptr %rhs
+  %op = select i1 %cond, <4 x float> %lhsv, <4 x float> %rhsv
+  call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @select_2x2_lhs(i1 %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_lhs(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 4
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr float, ptr [[RHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[COND:%.*]], <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[COND]], <2 x float> [[COL_LOAD1]], <2 x float> [[COL_LOAD4]]
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    ret void
+;
+  %lhsv = call <4 x float> @llvm.matrix.column.major.load(ptr %lhs, i64 2, i1 false, i32 2, i32 2)
+  %rhsv = load <4 x float>, ptr %rhs
+  %op = select i1 %cond, <4 x float> %lhsv, <4 x float> %rhsv
+  store <4 x float> %op, ptr %out
+  ret void
+}
+
+define void @select_2x2_rhs(i1 %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_rhs(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[RHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[RHS1:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr float, ptr [[RHS1]], i64 2
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[COND:%.*]], <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[COND]], <2 x float> [[COL_LOAD1]], <2 x float> [[COL_LOAD4]]
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    ret void
+;
+  %lhsv = load <4 x float>, ptr %lhs
+  %rhsv = call <4 x float> @llvm.matrix.column.major.load(ptr %rhs, i64 2, i1 false, i32 2, i32 2)
+  %op = select i1 %cond, <4 x float> %lhsv, <4 x float> %rhsv
+  store <4 x float> %op, ptr %out
+  ret void
+}
+
+define void @select_2x2_vcond_shape1(ptr %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_vcond_shape1(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[CONDV:%.*]] = load <4 x i1>, ptr [[COND:%.*]], align 1
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x float>, ptr [[RHS1:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr float, ptr [[RHS1]], i64 2
+; CHECK-NEXT:    [[COL_LOAD7:%.*]] = load <2 x float>, ptr [[VEC_GEP6]], align 4
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = shufflevector <4 x i1> [[CONDV]], <4 x i1> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = shufflevector <4 x i1> [[CONDV]], <4 x i1> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = select <2 x i1> [[COL_LOAD2]], <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[COL_LOAD4]], <2 x float> [[COL_LOAD1]], <2 x float> [[COL_LOAD7]]
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP8:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP8]], align 8
+; CHECK-NEXT:    ret void
+;
+  %lhsv = load <4 x float>, ptr %lhs
+  %condv = load <4 x i1>, ptr %cond
+  %rhsv = call <4 x float> @llvm.matrix.column.major.load(ptr %rhs, i64 2, i1 false, i32 2, i32 2)
+  %op = select <4 x i1> %condv, <4 x float> %lhsv, <4 x float> %rhsv
+  store <4 x float> %op, ptr %out
+  ret void
+}
+
+define void @select_2x2_vcond_shape2(ptr %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_vcond_shape2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x i1>, ptr [[COND:%.*]], align 1
+; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr i1, ptr [[COND]], i64 2
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x i1>, ptr [[VEC_GEP3]], align 1
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr float, ptr [[RHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD7:%.*]] = load <2 x float>, ptr [[VEC_GEP6]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = select <2 x i1> [[COL_LOAD2]], <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD5]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[COL_LOAD4]], <2 x float> [[COL_LOAD1]], <2 x float> [[COL_LOAD7]]
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP8:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP8]], align 8
+; CHECK-NEXT:    ret void
+;
+  %lhsv = load <4 x float>, ptr %lhs
+  %condv = call <4 x i1> @llvm.matrix.column.major.load(ptr %cond, i64 2, i1 false, i32 2, i32 2)
+  %rhsv = call <4 x float> @llvm.matrix.column.major.load(ptr %rhs, i64 2, i1 false, i32 2, i32 2)
+  %op = select <4 x i1> %condv, <4 x float> %lhsv, <4 x float> %rhsv
+  store <4 x float> %op, ptr %out
+  ret void
+}
+
+define void @select_2x2_vcond_shape3(ptr %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_vcond_shape3(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <4 x i1>, ptr [[COND:%.*]], align 1
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP4:%.*]] = getelementptr float, ptr [[RHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x float>, ptr [[VEC_GEP4]], align 4
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x i1> [[COL_LOAD2]], <4 x i1> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT6:%.*]] = shufflevector <4 x i1> [[COL_LOAD2]], <4 x i1> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = select <2 x i1> [[SPLIT]], <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[SPLIT6]], <2 x float> [[COL_LOAD1]], <2 x float> [[COL_LOAD5]]
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT:    ret void
+;
+  %lhsv = load <4 x float>, ptr %lhs
+  %condv = call <4 x i1> @llvm.matrix.column.major.load(ptr %cond, i64 4, i1 false, i32 4, i32 1)
+  %rhsv = call <4 x float> @llvm.matrix.column.major.load(ptr %rhs, i64 2, i1 false, i32 2, i32 2)
+  %op = select <4 x i1> %condv, <4 x float> %lhsv, <4 x float> %rhsv
+  store <4 x float> %op, ptr %out
+  ret void
+}
+
+define void @select_2x2_vcond_shape4(ptr %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_vcond_shape4(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <4 x float>, ptr [[LHS:%.*]], align 16
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <4 x i1>, ptr [[COND:%.*]], align 1
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <4 x float>, ptr [[RHS:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[COL_LOAD1]], <4 x float> [[COL_LOAD]], <4 x float> [[COL_LOAD2]]
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+  %lhsv = load <4 x float>, ptr %lhs
+  %condv = call <4 x i1> @llvm.matrix.column.major.load(ptr %cond, i64 4, i1 false, i32 4, i32 1)
+  %rhsv = call <4 x float> @llvm.matrix.column.major.load(ptr %rhs, i64 4, i1 false, i32 4, i32 1)
+  %op = select <4 x i1> %condv, <4 x float> %lhsv, <4 x float> %rhsv
+  store <4 x float> %op, ptr %out
+  ret void
+}
+
+define void @select_2x2_vcond_shape5(ptr %cond, ptr %lhs, ptr %rhs, ptr %out) {
+; CHECK-LABEL: @select_2x2_vcond_shape5(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <1 x float>, ptr [[LHS:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 1
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <1 x float>, ptr [[VEC_GEP]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr float, ptr [[LHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <1 x float>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    [[VEC_GEP4:%.*]] = getelementptr float, ptr [[LHS]], i64 3
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <1 x float>, ptr [[VEC_GEP4]], align 4
+; CHECK-NEXT:    [[COL_LOAD6:%.*]] = load <1 x i1>, ptr [[COND:%.*]], align 1
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr i1, ptr [[COND]], i64 1
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <1 x i1>, ptr [[VEC_GEP7]], align 1
+; CHECK-NEXT:    [[VEC_GEP9:%.*]] = getelementptr i1, ptr [[COND]], i64 2
+; CHECK-NEXT:    [[COL_LOAD10:%.*]] = load <1 x i1>, ptr [[VEC_GEP9]], align 1
+; CHECK-NEXT:    [[VEC_GEP11:%.*]] = getelementptr i1, ptr [[COND]], i64 3
+; CHECK-NEXT:    [[COL_LOAD12:%.*]] = load <1 x i1>, ptr [[VEC_GEP11]], align 1
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <1 x float>, ptr [[RHS:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP14:%.*]] = getelementptr float, ptr [[RHS]], i64 1
+; CHECK-NEXT:    [[COL_LOAD15:%.*]] = load <1 x float>, ptr [[VEC_GEP14]], align 4
+; CHECK-NEXT:    [[VEC_GEP16:%.*]] = getelementptr float, ptr [[RHS]], i64 2
+; CHECK-NEXT:    [[COL_LOAD17:%.*]] = load <1 x float>, ptr [[VEC_GEP16]], align 4
+; CHECK-NEXT:    [[VEC_GEP18:%.*]] = getelementptr float, ptr [[RHS]], i64 3
+; CHECK-NEXT:    [[COL_LOAD19:%.*]] = load <1 x float>, ptr [[VEC_GEP18]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = select <1 x i1> [[COL_LOAD6]], <1 x float> [[COL_LOAD]], <1 x float> [[COL_LOAD13]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <1 x i1> [[COL_LOAD8]], <1 x float> [[COL_LOAD1]], <1 x float> [[COL_LOAD15]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select <1 x i1> [[COL_LOAD10]], <1 x float> [[COL_LOAD3]], <1 x float> [[COL_LOAD17]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <1 x i1> [[COL_LOAD12]], <1 x float> [[COL_LOAD5]], <1 x float> [[COL_LOAD19]]
+; CHECK-NEXT:    store <1 x float> [[TMP1]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP20:%.*]] = getelementptr float, ptr [[OUT]], i64 1
+; CHECK-NEXT:    store <1 x float> [[TMP2]], ptr [[VEC_GEP20]], align 4
+; CHECK-NEXT:    [[VEC_GEP21:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <1 x float> [[TMP3]], ptr [[VEC_GEP21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr float, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <1 x float> [[TMP4]], ptr [[VEC_GEP22]], align 4
+; CHECK-NEXT:    ret void
+;
+  %lhsv = load <4 x float>, ptr %lhs
+  %condv = call <4 x i1> @llvm.matrix.column.major.load(ptr %cond, i64 1, i1 false, i32 1, i32 4)
+  %rhsv = call <4 x float> @llvm.matrix.column.major.load(ptr %rhs, i64 1, i1 false, i32 1, i32 4)
+  %op = select <4 x i1> %condv, <4 x float> %lhsv, <4 x float> %rhsv
+  store <4 x float> %op, ptr %out
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-fold-store.ll b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-fold-store.ll
new file mode 100644
index 000000000000..3a01c516f4b7
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-fold-store.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p lower-matrix-intrinsics -S %s | FileCheck %s
+
+define void @redundant_transpose_of_shuffle(<4 x float> %m, ptr %dst) {
+; CHECK-LABEL: define void @redundant_transpose_of_shuffle(
+; CHECK-SAME: <4 x float> [[M:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[M]], <4 x float> zeroinitializer, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[SHUFFLE]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x float> [[SPLIT]], ptr [[DST]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %shuffle = shufflevector <4 x float> %m, <4 x float> zeroinitializer, <4 x i32> zeroinitializer
+  %t = tail call <4 x float> @llvm.matrix.transpose.v3f32(<4 x float> %shuffle, i32 1, i32 4)
+  store <4 x float> %t, ptr %dst, align 4
+  ret void
+}
+
+define void @redundant_transpose_of_shuffle_2(<4 x float> %m, ptr %dst) {
+; CHECK-LABEL: define void @redundant_transpose_of_shuffle_2(
+; CHECK-SAME: <4 x float> [[M:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[M]], <4 x float> zeroinitializer, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[SHUFFLE]], <4 x float> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x float> [[SHUFFLE]], <4 x float> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x float> [[SHUFFLE]], <4 x float> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x float> [[SHUFFLE]], <4 x float> poison, <1 x i32> <i32 3>
+; CHECK-NEXT:    store <1 x float> [[SPLIT]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[DST]], i64 1
+; CHECK-NEXT:    store <1 x float> [[SPLIT1]], ptr [[VEC_GEP]], align 4
+; CHECK-NEXT:    [[VEC_GEP4:%.*]] = getelementptr float, ptr [[DST]], i64 2
+; CHECK-NEXT:    store <1 x float> [[SPLIT2]], ptr [[VEC_GEP4]], align 4
+; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr float, ptr [[DST]], i64 3
+; CHECK-NEXT:    store <1 x float> [[SPLIT3]], ptr [[VEC_GEP5]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %shuffle = shufflevector <4 x float> %m, <4 x float> zeroinitializer, <4 x i32> zeroinitializer
+  %t = tail call <4 x float> @llvm.matrix.transpose.v3f32(<4 x float> %shuffle, i32 4, i32 1)
+  store <4 x float> %t, ptr %dst, align 4
+  ret void
+}
+
+declare <4 x float> @llvm.matrix.transpose.v3f32(<4 x float>, i32, i32)
diff --git a/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/unary.ll b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/unary.ll
new file mode 100644
index 000000000000..a4bd516868bc
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/unary.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define void @fneg_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @fneg_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[IN:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x float> [[COL_LOAD]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fneg <2 x float> [[COL_LOAD1]]
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x float>, ptr %in
+  %op = fneg <4 x float> %inv
+  call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @trunc_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @trunc_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i64>, ptr [[IN:%.*]], align 32
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i64, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x i64>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[COL_LOAD]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i64> [[COL_LOAD1]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x i64>, ptr %in
+  %op = trunc <4 x i64> %inv to <4 x i32>
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @zext_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @zext_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i16>, ptr [[IN:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i16, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x i16>, ptr [[VEC_GEP]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[COL_LOAD]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <2 x i16> [[COL_LOAD1]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x i16>, ptr %in
+  %op = zext <4 x i16> %inv to <4 x i32>
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @sext_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @sext_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i8>, ptr [[IN:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i8, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x i8>, ptr [[VEC_GEP]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i8> [[COL_LOAD]] to <2 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i8> [[COL_LOAD1]] to <2 x i16>
+; CHECK-NEXT:    store <2 x i16> [[TMP1]], ptr [[OUT:%.*]], align 2
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr i16, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x i16> [[TMP2]], ptr [[VEC_GEP2]], align 2
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x i8>, ptr %in
+  %op = sext <4 x i8> %inv to <4 x i16>
+  call void @llvm.matrix.column.major.store(<4 x i16> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @fptoui_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @fptoui_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[IN:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = fptoui <2 x float> [[COL_LOAD]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = fptoui <2 x float> [[COL_LOAD1]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x float>, ptr %in
+  %op = fptoui <4 x float> %inv to <4 x i32>
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @fptosi_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @fptosi_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[IN:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = fptosi <2 x float> [[COL_LOAD]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi <2 x float> [[COL_LOAD1]] to <2 x i32>
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x float>, ptr %in
+  %op = fptosi <4 x float> %inv to <4 x i32>
+  call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @uitofp_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @uitofp_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i64>, ptr [[IN:%.*]], align 32
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i64, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x i64>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = uitofp <2 x i64> [[COL_LOAD]] to <2 x double>
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp <2 x i64> [[COL_LOAD1]] to <2 x double>
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x i64>, ptr %in
+  %op = uitofp <4 x i64> %inv to <4 x double>
+  call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @sitofp_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @sitofp_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i64>, ptr [[IN:%.*]], align 32
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i64, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x i64>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = sitofp <2 x i64> [[COL_LOAD]] to <2 x double>
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <2 x i64> [[COL_LOAD1]] to <2 x double>
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x i64>, ptr %in
+  %op = sitofp <4 x i64> %inv to <4 x double>
+  call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @fptrunc_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @fptrunc_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[IN:%.*]], align 32
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc nnan <2 x double> [[COL_LOAD]] to <2 x float>
+; CHECK-NEXT:    [[TMP2:%.*]] = fptrunc nnan <2 x double> [[COL_LOAD1]] to <2 x float>
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x double>, ptr %in
+  %op = fptrunc nnan <4 x double> %inv to <4 x float>
+  call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @fpext_2x2(ptr %in, ptr %out) {
+; CHECK-LABEL: @fpext_2x2(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[IN:%.*]], align 16
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x float> [[COL_LOAD]] to <2 x double>
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext <2 x float> [[COL_LOAD1]] to <2 x double>
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x float>, ptr %in
+  %op = fpext <4 x float> %inv to <4 x double>
+  call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @bitcast_2x2_v4f64_to_v4i64(ptr %in, ptr %out) {
+; CHECK-LABEL: @bitcast_2x2_v4f64_to_v4i64(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[IN:%.*]], align 32
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x double> [[COL_LOAD]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x double> [[COL_LOAD1]] to <2 x i64>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr i64, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], ptr [[VEC_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x double>, ptr %in
+  %op = bitcast <4 x double> %inv to <4 x i64>
+  call void @llvm.matrix.column.major.store(<4 x i64> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @bitcast_2x2_v4f64_to_v8i32(ptr %in, ptr %out) {
+; CHECK-LABEL: @bitcast_2x2_v4f64_to_v8i32(
+; CHECK-NEXT:    [[INV:%.*]] = load <4 x double>, ptr [[IN:%.*]], align 32
+; CHECK-NEXT:    [[OP:%.*]] = bitcast <4 x double> [[INV]] to <8 x i32>
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x i32> [[OP]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x i32> [[OP]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <4 x i32> [[SPLIT]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT]], i64 4
+; CHECK-NEXT:    store <4 x i32> [[SPLIT1]], ptr [[VEC_GEP]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = load <4 x double>, ptr %in
+  %op = bitcast <4 x double> %inv to <8 x i32>
+  call void @llvm.matrix.column.major.store(<8 x i32> %op, ptr %out, i64 4, i1 false, i32 4, i32 2)
+  ret void
+}
+
+define void @bitcast_2x2_i256_to_v4i64(ptr %in, ptr %out) {
+; CHECK-LABEL: @bitcast_2x2_i256_to_v4i64(
+; CHECK-NEXT:    [[INV:%.*]] = load i256, ptr [[IN:%.*]], align 4
+; CHECK-NEXT:    [[OP:%.*]] = bitcast i256 [[INV]] to <4 x double>
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x double> [[OP]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x double> [[OP]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    store <2 x double> [[SPLIT]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[OUT]], i64 2
+; CHECK-NEXT:    store <2 x double> [[SPLIT1]], ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    ret void
+;
+  %inv = load i256, ptr %in
+  %op = bitcast i256 %inv to <4 x double>
+  call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)
+  ret void
+}
+
+define void @bitcast_2x2_4i64_to_i256(ptr %in, ptr %out) {
+; CHECK-LABEL: @bitcast_2x2_4i64_to_i256(
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[IN:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN]], i64 2
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> [[COL_LOAD1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[OP:%.*]] = bitcast <4 x double> [[TMP1]] to i256
+; CHECK-NEXT:    store i256 [[OP]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  %inv = call <4 x double> @llvm.matrix.column.major.load(ptr %in, i64 2, i1 false, i32 2, i32 2)
+  %op = bitcast <4 x double> %inv to i256
+  store i256 %op, ptr %out
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll
index c932236dffac..8a90174bb3ff 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll
@@ -41,7 +41,7 @@ define i1 @foo(ptr %p) {
 ; AARCH64-LABEL: define i1 @foo
 ; AARCH64-SAME: (ptr [[P:%.*]]) {
 ; AARCH64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; AARCH64-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @.cfi.jumptable to i64)
+; AARCH64-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr getelementptr (i8, ptr @.cfi.jumptable, i64 8) to i64), [[TMP1]]
 ; AARCH64-NEXT:    [[TMP3:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 3)
 ; AARCH64-NEXT:    [[TMP4:%.*]] = icmp ule i64 [[TMP3]], 1
 ; AARCH64-NEXT:    ret i1 [[TMP4]]
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/asm.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/asm.ll
new file mode 100644
index 000000000000..e6a8efb393e8
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/asm.ll
@@ -0,0 +1,19 @@
+; REQUIRES: x86-registered-target
+; RUN: opt -mtriple=x86_64-unknown-linux -S -passes=lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import.yaml %s | llc | FileCheck %s
+
+define void @call(ptr %p) {
+  ; CHECK:      movl $__typeid_allones7_global_addr, %eax
+  ; CHECK-NEXT: subq %rdi, %rax
+  ; CHECK-NEXT: rorq $__typeid_allones7_align, %rax
+  ; CHECK-NEXT: cmpq $__typeid_allones7_size_m1@ABS8, %rax
+  %x = call i1 @llvm.type.test(ptr %p, metadata !"allones7")
+  br i1 %x, label %t, label %f
+
+t:
+  call void %p()
+  ret void
+
+f:
+  ret void
+}
+
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-allones.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-allones.ll
index 9708f4d76192..b5b4f8c5360c 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-allones.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-allones.ll
@@ -141,12 +141,12 @@
 
 ; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer
 
-; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[G]]
-; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to ptr)
+; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 2)
+; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i64 1 to ptr)
 ; X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 1 to ptr)
 
-; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 4)
-; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to ptr)
+; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 516)
+; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i64 2 to ptr)
 ; X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 128 to ptr)
 
 ; ARM-NOT: alias {{.*}} inttoptr
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-bytearray.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-bytearray.ll
index fef93d016267..89b3f0663be1 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-bytearray.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-bytearray.ll
@@ -14,14 +14,14 @@
 ; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer
 ; CHECK: [[B:@[0-9]+]] = private constant [258 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01"
 
-; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[G]]
-; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to ptr)
+; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 130)
+; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i64 1 to ptr)
 ; X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 65 to ptr)
 ; CHECK: @__typeid_typeid1_byte_array = hidden alias i8, ptr @bits.1
 ; X86: @__typeid_typeid1_bit_mask = hidden alias i8, inttoptr (i8 2 to ptr)
 
-; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 4)
-; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to ptr)
+; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 1032)
+; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i64 2 to ptr)
 ; X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 257 to ptr)
 ; CHECK: @__typeid_typeid2_byte_array = hidden alias i8, ptr @bits
 ; X86: @__typeid_typeid2_bit_mask = hidden alias i8, inttoptr (i8 1 to ptr)
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-icall.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-icall.ll
index bb766fc2ea73..47156deb57de 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-icall.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-icall.ll
@@ -36,8 +36,8 @@ define void @f3(i32 %x) !type !8 {
 !8 = !{i64 0, !"typeid3"}
 
 
-; CHECK-DAG: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[JT1:.*]]
-; CHECK-DAG: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 3 to ptr)
+; CHECK-DAG: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[JT1:.*]], i64 32)
+; CHECK-DAG: @__typeid_typeid1_align = hidden alias i8, inttoptr (i64 3 to ptr)
 ; CHECK-DAG: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 4 to ptr)
 
 ; CHECK-DAG: @h                    = alias void (i8), ptr [[JT1]]
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-inline.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-inline.ll
index 23d96a0c8684..72275b900208 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-inline.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/export-inline.ll
@@ -1,8 +1,13 @@
-; RUN: opt -mtriple=x86_64-unknown-linux -S -passes=lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t %s | FileCheck --check-prefix=CHECK %s
-; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86 %s < %t
+; REQUIRES: x86-registered-target
 
-; RUN: opt -mtriple=aarch64-unknown-linux -S -passes=lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t %s | FileCheck --check-prefix=CHECK %s
-; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-ARM %s < %t
+; RUN: opt -mtriple=i686-unknown-linux -S -passes=lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t %s | FileCheck --check-prefixes=CHECK,CHECK-X86-32 %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86,SUMMARY-X86-32 %s < %t
+
+; RUN: opt -mtriple=x86_64-unknown-linux -S -passes=lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t %s | FileCheck --check-prefixes=CHECK,CHECK-64 %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86,SUMMARY-64 %s < %t
+
+; RUN: opt -mtriple=aarch64-unknown-linux -S -passes=lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t %s | FileCheck --check-prefixes=CHECK,CHECK-64 %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-64,SUMMARY-ARM %s < %t
 
 @foo = constant [2048 x i8] zeroinitializer, !type !0, !type !1, !type !2, !type !3
 
@@ -13,15 +18,19 @@
 
 ; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer
 
-; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[G]]
+; CHECK-X86-32: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i32 6)
+; CHECK-64: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 6)
 ; CHECK-X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to ptr)
 ; CHECK-X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 3 to ptr)
 ; CHECK-X86: @__typeid_typeid1_inline_bits = hidden alias i8, inttoptr (i32 9 to ptr)
 
-; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 4)
+; CHECK-X86-32: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i32 136)
+; CHECK-64: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 136)
 ; CHECK-X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to ptr)
 ; CHECK-X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 33 to ptr)
-; CHECK-X86: @__typeid_typeid2_inline_bits = hidden alias i8, inttoptr (i64 8589934593 to ptr)
+; CHECK-X86-64: @__typeid_typeid2_inline_bits = hidden alias i8, inttoptr (i64 8589934593 to ptr)
+; CHECK-X86-32: @__typeid_typeid2_byte_array = hidden alias i8, ptr @bits
+; CHECK-X86-32: @__typeid_typeid2_bit_mask = hidden alias i8, inttoptr (i8 1 to ptr)
 
 ; CHECK: @foo = alias [2048 x i8], ptr [[G]]
 
@@ -41,8 +50,10 @@
 ; SUMMARY-NEXT:     WPDRes:
 ; SUMMARY-NEXT:   typeid2:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Inline
-; SUMMARY-NEXT:       SizeM1BitWidth:  6
+; SUMMARY-X86-32-NEXT: Kind:           ByteArray
+; SUMMARY-X86-32-NEXT: SizeM1BitWidth: 7
+; SUMMARY-64-NEXT:    Kind:            Inline
+; SUMMARY-64-NEXT:    SizeM1BitWidth:  6
 ; SUMMARY-X86-NEXT:   AlignLog2:       0
 ; SUMMARY-X86-NEXT:   SizeM1:          0
 ; SUMMARY-X86-NEXT:   BitMask:         0
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/function.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/function.ll
index f80e99ebfba2..5b0852c82ea6 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/function.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/function.ll
@@ -1,21 +1,21 @@
-; RUN: opt -S -passes=lowertypetests -mtriple=i686-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE %s
-; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE %s
-; RUN: opt -S -passes=lowertypetests -mtriple=i686-pc-win32 %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE %s
-; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-pc-win32 %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE %s
-; RUN: opt -S -passes=lowertypetests -mtriple=riscv32-unknown-linux-gnu %s | FileCheck --check-prefixes=RISCV,NATIVE %s
-; RUN: opt -S -passes=lowertypetests -mtriple=riscv64-unknown-linux-gnu %s | FileCheck --check-prefixes=RISCV,NATIVE %s
+; RUN: opt -S -passes=lowertypetests -mtriple=i686-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=i686-pc-win32 %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-pc-win32 %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=riscv32-unknown-linux-gnu %s | FileCheck --check-prefixes=RISCV,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=riscv64-unknown-linux-gnu %s | FileCheck --check-prefixes=RISCV,NATIVE,JT8 %s
 ; RUN: opt -S -passes=lowertypetests -mtriple=wasm32-unknown-unknown %s | FileCheck --check-prefix=WASM32 %s
-; RUN: opt -S -passes=lowertypetests -mtriple=loongarch64-unknown-linux-gnu %s | FileCheck --check-prefixes=LOONGARCH64,NATIVE %s
+; RUN: opt -S -passes=lowertypetests -mtriple=loongarch64-unknown-linux-gnu %s | FileCheck --check-prefixes=LOONGARCH64,NATIVE,JT8 %s
 
 ; The right format for Arm jump tables depends on the selected
 ; subtarget, so we can't get these tests right without the Arm target
 ; compiled in.
-; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=arm-unknown-linux-gnu %s | FileCheck --check-prefixes=ARM,NATIVE %s %}
-; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv7m-unknown-linux-gnu %s | FileCheck --check-prefixes=THUMB,NATIVE %s %}
-; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv8m.base-unknown-linux-gnu %s | FileCheck --check-prefixes=THUMB,NATIVE %s %}
-; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv6m-unknown-linux-gnu %s | FileCheck --check-prefixes=THUMBV6M,NATIVE %s %}
-; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv5-unknown-linux-gnu %s | FileCheck --check-prefixes=ARM,NATIVE %s %}
-; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=aarch64-unknown-linux-gnu %s | FileCheck --check-prefixes=ARM,NATIVE %s %}
+; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=arm-unknown-linux-gnu %s | FileCheck --check-prefixes=ARM,NATIVE,JT4 %s %}
+; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv7m-unknown-linux-gnu %s | FileCheck --check-prefixes=THUMB,NATIVE,JT4 %s %}
+; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv8m.base-unknown-linux-gnu %s | FileCheck --check-prefixes=THUMB,NATIVE,JT4 %s %}
+; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv6m-unknown-linux-gnu %s | FileCheck --check-prefixes=THUMBV6M,NATIVE,JT16 %s %}
+; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=thumbv5-unknown-linux-gnu %s | FileCheck --check-prefixes=ARM,NATIVE,JT4 %s %}
+; RUN: %if arm-registered-target %{ opt -S -passes=lowertypetests -mtriple=aarch64-unknown-linux-gnu %s | FileCheck --check-prefixes=ARM,NATIVE,JT4 %s %}
 
 ; Tests that we correctly handle bitsets containing 2 or more functions.
 
@@ -54,20 +54,18 @@ define internal void @g() !type !0 {
 declare i1 @llvm.type.test(ptr %ptr, metadata %bitset) noinline readnone
 
 define i1 @foo(ptr %p) {
-  ; NATIVE: sub i64 {{.*}}, ptrtoint (ptr @[[JT]] to i64)
-  ; WASM32: sub i64 {{.*}}, ptrtoint (ptr getelementptr (i8, ptr null, i64 1) to i64)
+  ; JT4: sub i64 ptrtoint (ptr getelementptr (i8, ptr @[[JT]], i64 4) to i64), {{.*}}
+  ; JT8: sub i64 ptrtoint (ptr getelementptr (i8, ptr @[[JT]], i64 8) to i64), {{.*}}
+  ; JT16: sub i64 ptrtoint (ptr getelementptr (i8, ptr @[[JT]], i64 16) to i64), {{.*}}
+  ; WASM32: sub i64 ptrtoint (ptr getelementptr (i8, ptr null, i64 2) to i64), {{.*}}
   ; WASM32: icmp ule i64 {{.*}}, 1
   %x = call i1 @llvm.type.test(ptr %p, metadata !"typeid1")
   ret i1 %x
 }
 
-; X86-LINUX:   define private void @[[JT]]() #[[ATTR:.*]] align 8 {
-; X86-WIN32:   define private void @[[JT]]() #[[ATTR:.*]] align 8 {
-; ARM:         define private void @[[JT]]() #[[ATTR:.*]] align 4 {
-; THUMB:       define private void @[[JT]]() #[[ATTR:.*]] align 4 {
-; THUMBV6M:    define private void @[[JT]]() #[[ATTR:.*]] align 16 {
-; RISCV:       define private void @[[JT]]() #[[ATTR:.*]] align 8 {
-; LOONGARCH64: define private void @[[JT]]() #[[ATTR:.*]] align 8 {
+; JT4:  define private void @[[JT]]() #[[ATTR:.*]] align 4 {
+; JT8:  define private void @[[JT]]() #[[ATTR:.*]] align 8 {
+; JT16: define private void @[[JT]]() #[[ATTR:.*]] align 16 {
 
 ; X86:      jmp ${0:c}@plt
 ; X86-SAME: int3
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/import.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/import.ll
index 3c97f087d5c9..819ede96f997 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/import.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/import.ll
@@ -36,16 +36,15 @@ define i1 @allones7(ptr %p) {
 ; X86-LABEL: define i1 @allones7(
 ; X86-SAME: ptr [[P:%.*]]) {
 ; X86-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; X86-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_allones7_global_addr to i64)
-; X86-NEXT:    [[TMP3:%.*]] = zext i8 ptrtoint (ptr @__typeid_allones7_align to i8) to i64
-; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 [[TMP3]])
+; X86-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_allones7_global_addr to i64), [[TMP1]]
+; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 ptrtoint (ptr @__typeid_allones7_align to i64))
 ; X86-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], ptrtoint (ptr @__typeid_allones7_size_m1 to i64)
 ; X86-NEXT:    ret i1 [[TMP8]]
 ;
 ; ARM-LABEL: define i1 @allones7(
 ; ARM-SAME: ptr [[P:%.*]]) {
 ; ARM-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; ARM-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_allones7_global_addr to i64)
+; ARM-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_allones7_global_addr to i64), [[TMP1]]
 ; ARM-NEXT:    [[TMP5:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 1)
 ; ARM-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[TMP5]], 42
 ; ARM-NEXT:    ret i1 [[TMP6]]
@@ -58,16 +57,15 @@ define i1 @allones32(ptr %p) {
 ; X86-LABEL: define i1 @allones32(
 ; X86-SAME: ptr [[P:%.*]]) {
 ; X86-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; X86-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_allones32_global_addr to i64)
-; X86-NEXT:    [[TMP3:%.*]] = zext i8 ptrtoint (ptr @__typeid_allones32_align to i8) to i64
-; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 [[TMP3]])
+; X86-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_allones32_global_addr to i64), [[TMP1]]
+; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 ptrtoint (ptr @__typeid_allones32_align to i64))
 ; X86-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], ptrtoint (ptr @__typeid_allones32_size_m1 to i64)
 ; X86-NEXT:    ret i1 [[TMP8]]
 ;
 ; ARM-LABEL: define i1 @allones32(
 ; ARM-SAME: ptr [[P:%.*]]) {
 ; ARM-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; ARM-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_allones32_global_addr to i64)
+; ARM-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_allones32_global_addr to i64), [[TMP1]]
 ; ARM-NEXT:    [[TMP5:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 2)
 ; ARM-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[TMP5]], 12345
 ; ARM-NEXT:    ret i1 [[TMP6]]
@@ -80,25 +78,24 @@ define i1 @bytearray7(ptr %p) {
 ; X86-LABEL: define i1 @bytearray7(
 ; X86-SAME: ptr [[P:%.*]]) {
 ; X86-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; X86-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_bytearray7_global_addr to i64)
-; X86-NEXT:    [[TMP3:%.*]] = zext i8 ptrtoint (ptr @__typeid_bytearray7_align to i8) to i64
-; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 [[TMP3]])
+; X86-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_bytearray7_global_addr to i64), [[TMP1]]
+; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 ptrtoint (ptr @__typeid_bytearray7_align to i64))
 ; X86-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], ptrtoint (ptr @__typeid_bytearray7_size_m1 to i64)
-; X86-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP14:%.*]]
-; X86:       6:
+; X86-NEXT:    br i1 [[TMP8]], label %[[TMP9:.*]], label %[[TMP14:.*]]
+; X86:       [[TMP9]]:
 ; X86-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr @__typeid_bytearray7_byte_array, i64 [[TMP7]]
 ; X86-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; X86-NEXT:    [[TMP12:%.*]] = and i8 [[TMP11]], ptrtoint (ptr @__typeid_bytearray7_bit_mask to i8)
 ; X86-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; X86-NEXT:    br label [[TMP14]]
-; X86:       11:
-; X86-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP13]], [[TMP9]] ]
+; X86-NEXT:    br label %[[TMP14]]
+; X86:       [[TMP14]]:
+; X86-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP13]], %[[TMP9]] ]
 ; X86-NEXT:    ret i1 [[TMP15]]
 ;
 ; ARM-LABEL: define i1 @bytearray7(
 ; ARM-SAME: ptr [[P:%.*]]) {
 ; ARM-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; ARM-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_bytearray7_global_addr to i64)
+; ARM-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_bytearray7_global_addr to i64), [[TMP1]]
 ; ARM-NEXT:    [[TMP5:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 3)
 ; ARM-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[TMP5]], 43
 ; ARM-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP12:%.*]]
@@ -120,25 +117,24 @@ define i1 @bytearray32(ptr %p) {
 ; X86-LABEL: define i1 @bytearray32(
 ; X86-SAME: ptr [[P:%.*]]) {
 ; X86-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; X86-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_bytearray32_global_addr to i64)
-; X86-NEXT:    [[TMP3:%.*]] = zext i8 ptrtoint (ptr @__typeid_bytearray32_align to i8) to i64
-; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 [[TMP3]])
+; X86-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_bytearray32_global_addr to i64), [[TMP1]]
+; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 ptrtoint (ptr @__typeid_bytearray32_align to i64))
 ; X86-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], ptrtoint (ptr @__typeid_bytearray32_size_m1 to i64)
-; X86-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP14:%.*]]
-; X86:       6:
+; X86-NEXT:    br i1 [[TMP8]], label %[[TMP9:.*]], label %[[TMP14:.*]]
+; X86:       [[TMP9]]:
 ; X86-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr @__typeid_bytearray32_byte_array, i64 [[TMP7]]
 ; X86-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; X86-NEXT:    [[TMP12:%.*]] = and i8 [[TMP11]], ptrtoint (ptr @__typeid_bytearray32_bit_mask to i8)
 ; X86-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; X86-NEXT:    br label [[TMP14]]
-; X86:       11:
-; X86-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP13]], [[TMP9]] ]
+; X86-NEXT:    br label %[[TMP14]]
+; X86:       [[TMP14]]:
+; X86-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP13]], %[[TMP9]] ]
 ; X86-NEXT:    ret i1 [[TMP15]]
 ;
 ; ARM-LABEL: define i1 @bytearray32(
 ; ARM-SAME: ptr [[P:%.*]]) {
 ; ARM-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; ARM-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_bytearray32_global_addr to i64)
+; ARM-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_bytearray32_global_addr to i64), [[TMP1]]
 ; ARM-NEXT:    [[TMP5:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 4)
 ; ARM-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[TMP5]], 12346
 ; ARM-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP12:%.*]]
@@ -160,26 +156,25 @@ define i1 @inline5(ptr %p) {
 ; X86-LABEL: define i1 @inline5(
 ; X86-SAME: ptr [[P:%.*]]) {
 ; X86-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; X86-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_inline5_global_addr to i64)
-; X86-NEXT:    [[TMP3:%.*]] = zext i8 ptrtoint (ptr @__typeid_inline5_align to i8) to i64
-; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 [[TMP3]])
+; X86-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_inline5_global_addr to i64), [[TMP1]]
+; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 ptrtoint (ptr @__typeid_inline5_align to i64))
 ; X86-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], ptrtoint (ptr @__typeid_inline5_size_m1 to i64)
-; X86-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP15:%.*]]
-; X86:       6:
+; X86-NEXT:    br i1 [[TMP8]], label %[[TMP9:.*]], label %[[TMP15:.*]]
+; X86:       [[TMP9]]:
 ; X86-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP7]] to i32
 ; X86-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 31
 ; X86-NEXT:    [[TMP12:%.*]] = shl i32 1, [[TMP11]]
 ; X86-NEXT:    [[TMP13:%.*]] = and i32 ptrtoint (ptr @__typeid_inline5_inline_bits to i32), [[TMP12]]
 ; X86-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
-; X86-NEXT:    br label [[TMP15]]
-; X86:       12:
-; X86-NEXT:    [[TMP16:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP14]], [[TMP9]] ]
+; X86-NEXT:    br label %[[TMP15]]
+; X86:       [[TMP15]]:
+; X86-NEXT:    [[TMP16:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP14]], %[[TMP9]] ]
 ; X86-NEXT:    ret i1 [[TMP16]]
 ;
 ; ARM-LABEL: define i1 @inline5(
 ; ARM-SAME: ptr [[P:%.*]]) {
 ; ARM-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; ARM-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_inline5_global_addr to i64)
+; ARM-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_inline5_global_addr to i64), [[TMP1]]
 ; ARM-NEXT:    [[TMP5:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 5)
 ; ARM-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[TMP5]], 31
 ; ARM-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP13:%.*]]
@@ -202,25 +197,24 @@ define i1 @inline6(ptr %p) {
 ; X86-LABEL: define i1 @inline6(
 ; X86-SAME: ptr [[P:%.*]]) {
 ; X86-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; X86-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_inline6_global_addr to i64)
-; X86-NEXT:    [[TMP3:%.*]] = zext i8 ptrtoint (ptr @__typeid_inline6_align to i8) to i64
-; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 [[TMP3]])
+; X86-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_inline6_global_addr to i64), [[TMP1]]
+; X86-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 ptrtoint (ptr @__typeid_inline6_align to i64))
 ; X86-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], ptrtoint (ptr @__typeid_inline6_size_m1 to i64)
-; X86-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP14:%.*]]
-; X86:       6:
+; X86-NEXT:    br i1 [[TMP8]], label %[[TMP9:.*]], label %[[TMP14:.*]]
+; X86:       [[TMP9]]:
 ; X86-NEXT:    [[TMP10:%.*]] = and i64 [[TMP7]], 63
 ; X86-NEXT:    [[TMP11:%.*]] = shl i64 1, [[TMP10]]
 ; X86-NEXT:    [[TMP12:%.*]] = and i64 ptrtoint (ptr @__typeid_inline6_inline_bits to i64), [[TMP11]]
 ; X86-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
-; X86-NEXT:    br label [[TMP14]]
-; X86:       11:
-; X86-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP13]], [[TMP9]] ]
+; X86-NEXT:    br label %[[TMP14]]
+; X86:       [[TMP14]]:
+; X86-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[TMP0:%.*]] ], [ [[TMP13]], %[[TMP9]] ]
 ; X86-NEXT:    ret i1 [[TMP15]]
 ;
 ; ARM-LABEL: define i1 @inline6(
 ; ARM-SAME: ptr [[P:%.*]]) {
 ; ARM-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; ARM-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_inline6_global_addr to i64)
+; ARM-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_inline6_global_addr to i64), [[TMP1]]
 ; ARM-NEXT:    [[TMP5:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 6)
 ; ARM-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[TMP5]], 63
 ; ARM-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP12:%.*]]
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simple.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simple.ll
index 94617bab798a..6fb8f6f900e0 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simple.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simple.ll
@@ -18,7 +18,7 @@ target datalayout = "e-p:32:32"
 ; CHECK-NODISCARD: !type
 ; CHECK-NODISCARD: !type
 
-; CHECK: [[BA:@[^ ]*]] = private constant [68 x i8] c"\03\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02\00\01"
+; CHECK: [[BA:@[^ ]*]] = private constant [68 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02\01\01"
 
 ; Offset 0, 4 byte alignment
 !0 = !{i32 0, !"typeid1"}
@@ -49,7 +49,7 @@ define i1 @foo(ptr %p) {
   ; CHECK-NOT: llvm.type.test
 
   ; CHECK: [[R1:%[^ ]*]] = ptrtoint ptr %p to i32
-  ; CHECK: [[R2:%[^ ]*]] = sub i32 [[R1]], ptrtoint (ptr [[G]] to i32)
+  ; CHECK: [[R2:%[^ ]*]] = sub i32 ptrtoint (ptr getelementptr (i8, ptr [[G]], i32 268) to i32), [[R1]]
   ; CHECK: [[R5:%[^ ]*]] = call i32 @llvm.fshr.i32(i32 [[R2]], i32 [[R2]], i32 2)
   ; CHECK: [[R6:%[^ ]*]] = icmp ule i32 [[R5]], 67
   ; CHECK: br i1 [[R6]]
@@ -72,7 +72,7 @@ define i1 @foo(ptr %p) {
 ; CHECK: @bar(ptr [[B0:%[^ ]*]])
 define i1 @bar(ptr %p) {
   ; CHECK: [[S1:%[^ ]*]] = ptrtoint ptr %p to i32
-  ; CHECK: [[S2:%[^ ]*]] = sub i32 [[S1]], ptrtoint (ptr getelementptr (i8, ptr [[G]], i32 4) to i32)
+  ; CHECK: [[S2:%[^ ]*]] = sub i32 ptrtoint (ptr getelementptr (i8, ptr [[G]], i32 260) to i32), [[S1]]
   ; CHECK: [[S5:%[^ ]*]] = call i32 @llvm.fshr.i32(i32 [[S2]], i32 [[S2]], i32 8)
   ; CHECK: [[S6:%[^ ]*]] = icmp ule i32 [[S5]], 1
   %x = call i1 @llvm.type.test(ptr %p, metadata !"typeid2")
@@ -84,7 +84,7 @@ define i1 @bar(ptr %p) {
 ; CHECK: @baz(ptr [[C0:%[^ ]*]])
 define i1 @baz(ptr %p) {
   ; CHECK: [[T1:%[^ ]*]] = ptrtoint ptr %p to i32
-  ; CHECK: [[T2:%[^ ]*]] = sub i32 [[T1]], ptrtoint (ptr [[G]] to i32)
+  ; CHECK: [[T2:%[^ ]*]] = sub i32 ptrtoint (ptr getelementptr (i8, ptr [[G]], i32 260) to i32), [[T1]]
   ; CHECK: [[T5:%[^ ]*]] = call i32 @llvm.fshr.i32(i32 [[T2]], i32 [[T2]], i32 2)
   ; CHECK: [[T6:%[^ ]*]] = icmp ule i32 [[T5]], 65
   ; CHECK: br i1 [[T6]]
diff --git a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simplify.ll b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simplify.ll
index 5f2140caca27..3f7385ab683a 100644
--- a/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simplify.ll
+++ b/external/llvm-project/llvm/test/Transforms/LowerTypeTests/simplify.ll
@@ -10,20 +10,19 @@ define i1 @bytearray7(ptr %p) {
 ; CHECK-LABEL: define i1 @bytearray7(
 ; CHECK-SAME: ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @__typeid_bytearray7_global_addr to i64)
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 ptrtoint (ptr @__typeid_bytearray7_align to i8) to i64
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 [[TMP3]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 ptrtoint (ptr @__typeid_bytearray7_global_addr to i64), [[TMP1]] 
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 ptrtoint (ptr @__typeid_bytearray7_align to i64))
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], ptrtoint (ptr @__typeid_bytearray7_size_m1 to i64)
-; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[F:%.*]]
-; CHECK:       6:
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[TMP9:.*]], label %[[F:.*]]
+; CHECK:       [[TMP9]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr @__typeid_bytearray7_byte_array, i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr [[TMP10]], align 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = and i8 [[TMP11]], ptrtoint (ptr @__typeid_bytearray7_bit_mask to i8)
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i8 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[T:%.*]], label [[F]]
-; CHECK:       t:
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[T:.*]], label %[[F]]
+; CHECK:       [[T]]:
 ; CHECK-NEXT:    ret i1 true
-; CHECK:       f:
+; CHECK:       [[F]]:
 ; CHECK-NEXT:    ret i1 false
 ;
   %x = call i1 @llvm.type.test(ptr %p, metadata !"bytearray7")
diff --git a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
index efdbdce401b7..601498e36a7a 100644
--- a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
+++ b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
@@ -1,9 +1,11 @@
 ; RUN: opt < %s -S -passes=memcpyopt | FileCheck --match-full-lines %s
 
+declare void @use(ptr)
+
 ; Make sure callslot optimization merges alias.scope metadata correctly when it merges instructions.
 ; Merging here naively generates:
 ;  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 1, i1 false), !alias.scope !3
-;  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %src), !noalias !0
+;  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %src), !noalias !0
 ;   ...
 ;  !0 = !{!1}
 ;  !1 = distinct !{!1, !2, !"callee1: %a"}
@@ -18,12 +20,13 @@ define i8 @test(i8 %input) {
   %src = alloca i8
 ; NOTE: we're matching the full line and looking for the lack of !alias.scope here
 ; CHECK:   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 1, i1 false)
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %src), !noalias !3
+  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %src), !noalias !3
   store i8 %input, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %src, i64 1, i1 false), !alias.scope !0
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %src), !noalias !3
+  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %src), !noalias !3
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 1, i1 false), !alias.scope !3
   %ret_value = load i8, ptr %dst
+  call void @use(ptr %src)
   ret i8 %ret_value
 }
 
diff --git a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
index 1c3896407e95..0c16f34590fc 100644
--- a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
+++ b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
@@ -187,6 +187,53 @@ define void @test_write_before_memset_in_both_regions(ptr %result) {
   ret void
 }
 
+define void @test_negative_offset_memset(ptr %result) {
+; CHECK-LABEL: @test_negative_offset_memset(
+; CHECK-NEXT:    [[A1:%.*]] = alloca [16 x i8], align 8
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i8, ptr [[A1]], i32 4
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[A]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[RESULT:%.*]], ptr align 8 [[A1]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca [ 16 x i8 ], align 8
+  %b = getelementptr i8, ptr %a, i32 4
+  call void @llvm.memset.p0.i64(ptr align 8 %b, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr %result, ptr align 8 %a, i64 12, i1 false)
+  ret void
+}
+
+define void @test_offset_memsetcpy(ptr %result) {
+; CHECK-LABEL: @test_offset_memsetcpy(
+; CHECK-NEXT:    [[A1:%.*]] = alloca [16 x i8], align 8
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i8, ptr [[A1]], i32 4
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[A1]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[RESULT:%.*]], i8 0, i64 8, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca [ 16 x i8 ], align 8
+  %b = getelementptr i8, ptr %a, i32 4
+  call void @llvm.memset.p0.i64(ptr align 8 %a, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr %result, ptr align 8 %b, i64 12, i1 false)
+  ret void
+}
+
+define void @test_two_memset(ptr %result) {
+; CHECK-LABEL: @test_two_memset(
+; CHECK-NEXT:    [[A:%.*]] = alloca [16 x i8], align 8
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i8, ptr [[A]], i32 12
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[A]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[B]], i8 1, i64 4, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[RESULT:%.*]], ptr align 8 [[A]], i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca [ 16 x i8 ], align 8
+  %b = getelementptr i8, ptr %a, i32 12
+  call void @llvm.memset.p0.i64(ptr align 8 %a, i8 0, i64 12, i1 false)
+  call void @llvm.memset.p0.i64(ptr align 8 %b, i8 1, i64 4, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr %result, ptr align 8 %a, i64 16, i1 false)
+  ret void
+}
+
 declare ptr @malloc(i64)
 declare void @free(ptr)
 
diff --git a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
index 47474e8dac05..18488f03a2d8 100644
--- a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
+++ b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
@@ -73,11 +73,10 @@ define void @test_different_source_gep(ptr %dst1, ptr %dst2, i8 %c) {
 ; CHECK-LABEL: @test_different_source_gep(
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[DST1:%.*]], i8 [[C:%.*]], i64 128, i1 false)
 ; CHECK-NEXT:    [[P:%.*]] = getelementptr i8, ptr [[DST1]], i64 64
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DST2:%.*]], ptr [[P]], i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[DST2:%.*]], i8 [[C]], i64 64, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.p0.i64(ptr %dst1, i8 %c, i64 128, i1 false)
-  ; FIXME: We could optimize this as well.
   %p = getelementptr i8, ptr %dst1, i64 64
   call void @llvm.memcpy.p0.p0.i64(ptr %dst2, ptr %p, i64 64, i1 false)
   ret void
diff --git a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll
index 5e13432746bf..0e312bc42d46 100644
--- a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll
+++ b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll
@@ -19,7 +19,7 @@ define i32 @foo(i1 %z) {
 ; CHECK:       for.body3.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_INC7_1]]
 ; CHECK:       for.inc7.1:
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[A]], ptr align 4 [[SCEVGEP]], i64 4, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[A]], i8 0, i64 4, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/variable-sized-memset-memcpy.ll b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/variable-sized-memset-memcpy.ll
index a834d2465dfa..4b44f8b44f74 100644
--- a/external/llvm-project/llvm/test/Transforms/MemCpyOpt/variable-sized-memset-memcpy.ll
+++ b/external/llvm-project/llvm/test/Transforms/MemCpyOpt/variable-sized-memset-memcpy.ll
@@ -18,13 +18,13 @@ define void @test(ptr %src, i8 %c, i64 %size) {
   ret void
 }
 
-; Differing sizes, so left as it is.
-define void @negative_test(ptr %src, i8 %c, i64 %size1, i64 %size2) {
-; CHECK-LABEL: @negative_test(
+; Differing sizes, but would be UB if size1 < size2 since the memcpy would reference outside of the first alloca
+define void @dynsize_test(ptr %src, i8 %c, i64 %size1, i64 %size2) {
+; CHECK-LABEL: @dynsize_test(
 ; CHECK-NEXT:    [[DST1:%.*]] = alloca i8, i64 [[SIZE1:%.*]], align 1
 ; CHECK-NEXT:    [[DST2:%.*]] = alloca i8, i64 [[SIZE2:%.*]], align 1
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[DST1]], i8 [[C:%.*]], i64 [[SIZE1]], i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST2]], ptr align 8 [[DST1]], i64 [[SIZE2]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[DST2]], i8 [[C]], i64 [[SIZE2]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %dst1 = alloca i8, i64 %size1
diff --git a/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll b/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll
index 93557e963752..18521c013d55 100644
--- a/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll
+++ b/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll
@@ -64,6 +64,10 @@
 ; RUN:  --check-prefix=STATS --check-prefix=REMARKS
 
 ; RUN:  cat %t.ccg.prestackupdate.dot | FileCheck %s --check-prefix=DOTPRE
+;; There are no changes for this test case in between from immediately after
+;; the stack update through the end of graph finalization, so the
+;; corresponding dot graphs should be identical.
+; RUN:  cat %t.ccg.poststackupdate.dot | FileCheck %s --check-prefix=DOTPOST
 ; RUN:  cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOTPOST
 ;; We should clone D once for the cold allocations via C.
 ; RUN:  cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED
@@ -313,8 +317,8 @@ attributes #6 = { builtin }
 ; DOTPRE: }
 
 
-; DOTPOST:digraph "postbuild" {
-; DOTPOST:	label="postbuild";
+; DOTPOST:digraph "post
+; DOTPOST:	label="post
 ; DOTPOST:	Node[[D:0x[a-z0-9]+]] [shape=record,tooltip="N[[D]] ContextIds: 1 2 3 4",fillcolor="mediumorchid1",style="filled",label="{OrigId: Alloc0\n_Z1Dv -\> _Znam}"];
 ; DOTPOST:	Node[[F:0x[a-z0-9]+]] [shape=record,tooltip="N[[F]] ContextIds: 2",fillcolor="brown1",style="filled",label="{OrigId: 13543580133643026784\n_Z1Fv -\> _Z1Dv}"];
 ; DOTPOST:	Node[[F]] -> Node[[D]][tooltip="ContextIds: 2",fillcolor="brown1"
diff --git a/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/tailcall-nonunique.ll b/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/tailcall-nonunique.ll
index 985c381ad42f..97e433ccefa9 100644
--- a/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/tailcall-nonunique.ll
+++ b/external/llvm-project/llvm/test/Transforms/MemProfContextDisambiguation/tailcall-nonunique.ll
@@ -6,11 +6,20 @@
 ; REQUIRES: asserts
 
 ; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
+; RUN:	-memprof-export-to-dot -memprof-dot-file-path-prefix=%t. \
 ; RUN:  -stats -debug %s -S 2>&1 | FileCheck %s --check-prefix=STATS \
 ; RUN:  --check-prefix=IR --check-prefix=DEBUG
 
 ; DEBUG: Not found through unique tail call chain: _Z3barv from main that actually called xyz (found multiple possible chains)
 
+;; Graph fixup should have nulled the call in the node that could not be
+;; resolved via tail call fixup. That happens in between the initial
+;; stack update and the end of graph building.
+; RUN:  cat %t.ccg.poststackupdate.dot | FileCheck %s --check-prefix=DOTPOSTSTACKUPDATE
+; DOTPOSTSTACKUPDATE: {OrigId: 15025054523792398438\nmain -\> xyz}
+; RUN:  cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOTPOSTBUILD
+; DOTPOSTBUILD: {OrigId: 15025054523792398438\nnull call (external)}
+
 ;; Check that all calls in the IR are to the original functions, leading to a
 ;; non-cold operator new call.
 
diff --git a/external/llvm-project/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll b/external/llvm-project/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
index bb4b2ccacd5e..7767c2b0236d 100644
--- a/external/llvm-project/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
+++ b/external/llvm-project/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
@@ -181,7 +181,7 @@ attributes #3 = { noreturn nounwind }
 ; LOWERTYPETESTS-NEXT:    store ptr [[COND]], ptr [[FP]], align 8
 ; LOWERTYPETESTS-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FP]], align 8
 ; LOWERTYPETESTS-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[TMP2]] to i64
-; LOWERTYPETESTS-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], ptrtoint (ptr @.cfi.jumptable to i64)
+; LOWERTYPETESTS-NEXT:    [[TMP4:%.*]] = sub i64 ptrtoint (ptr getelementptr (i8, ptr @.cfi.jumptable, i64 8) to i64), [[TMP3]]
 ; LOWERTYPETESTS-NEXT:    [[TMP5:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP4]], i64 [[TMP4]], i64 3)
 ; LOWERTYPETESTS-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[TMP5]], 1
 ; LOWERTYPETESTS-NEXT:    br i1 [[TMP6]], label [[CONT:%.*]], label [[TRAP:%.*]], !nosanitize [[META4:![0-9]+]]
diff --git a/external/llvm-project/llvm/test/Transforms/MergeFunc/debuginfo-iterators.ll b/external/llvm-project/llvm/test/Transforms/MergeFunc/debuginfo-iterators.ll
deleted file mode 100644
index 67b51031bed6..000000000000
--- a/external/llvm-project/llvm/test/Transforms/MergeFunc/debuginfo-iterators.ll
+++ /dev/null
@@ -1,54 +0,0 @@
-; RUN: opt -S -passes=mergefunc,inline --experimental-debuginfo-iterators < %s | FileCheck %s
-;; Ensure that the MergeFunctions pass creates thunks with the appropriate debug
-;; info format set (which would otherwise assert when inlining those thunks).
-
-declare void @f1()
-declare void @f2()
-
-define void @f3() {
-  call void @f1()
-  call void @f2()
-  ret void
-}
-
-;; MergeFunctions will replace f4 with a thunk that calls f3. Inlining will
-;; inline f3 into that thunk, which would assert if the thunk had the incorrect
-;; debug info format.
-define void @f4() {
-  call void @f1()
-  call void @f2()
-  ret void
-}
-
-; CHECK-LABEL: define void @f4() {
-; CHECK-NEXT:    call void @f1()
-; CHECK-NEXT:    call void @f2()
-; CHECK-NEXT:    ret void
-; CHECK-NEXT: }
-
-;; Both of these are interposable, so MergeFunctions will create a common thunk
-;; that both will call. Inlining will inline that thunk back, which would assert
-;; if the thunk had the incorrect debug info format.
-define weak void @f5() {
-  call void @f2()
-  call void @f1()
-  ret void
-}
-
-define weak void @f6() {
-  call void @f2()
-  call void @f1()
-  ret void
-}
-
-; CHECK-LABEL: define weak void @f6() {
-; CHECK-NEXT:    call void @f2()
-; CHECK-NEXT:    call void @f1()
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
-
-; CHECK-LABEL: define weak void @f5() {
-; CHECK-NEXT:    call void @f2()
-; CHECK-NEXT:    call void @f1()
-; CHECK-NEXT:    ret void
-; CHECK-NEXT:  }
diff --git a/external/llvm-project/llvm/test/Transforms/ObjCARC/code-motion.ll b/external/llvm-project/llvm/test/Transforms/ObjCARC/code-motion.ll
index 499ee77bc654..9009b98b4b1e 100644
--- a/external/llvm-project/llvm/test/Transforms/ObjCARC/code-motion.ll
+++ b/external/llvm-project/llvm/test/Transforms/ObjCARC/code-motion.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=objc-arc -S < %s | FileCheck %s
+; RUN: opt -passes=objc-arc -S < %s 2>&1 | FileCheck %s '--implicit-check-not=ignoring invalid debug'
 
 declare void @alterRefCount()
 declare void @use(ptr)
@@ -17,7 +17,7 @@ define i32 @test(ptr %x, ptr %y, i8 %z, i32 %i) {
   store i32 %i, ptr %i.addr, align 4
   %v1 = tail call ptr @llvm.objc.retain(ptr %x)
   store i8 %z, ptr %x
-  call void @llvm.dbg.declare(metadata ptr %i.addr, metadata !9, metadata !DIExpression()), !dbg !10
+  call void @llvm.dbg.declare(metadata ptr %i.addr, metadata !11, metadata !DIExpression()), !dbg !10
   call void @alterRefCount()
   tail call void @llvm.objc.release(ptr %x)
   ret i32 %i
@@ -64,7 +64,7 @@ define void @test3(ptr %obj, i1 %cond) {
 ; CHECK-NEXT:    call void @use(ptr [[OBJ]])
 ; CHECK-NEXT:    br label [[JOIN]]
 ; CHECK:       join:
-; CHECK-NEXT:    call void @llvm.objc.release(ptr [[OBJ]]) {{.*}}, !clang.imprecise_release !2
+; CHECK-NEXT:    call void @llvm.objc.release(ptr [[OBJ]]) {{.*}}, !clang.imprecise_release ![[EMPTYMETA:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %v0 = call ptr @llvm.objc.retain(ptr %obj)
@@ -102,8 +102,8 @@ define void @test4(ptr %obj0, ptr %obj1, i1 %cond) {
 ; CHECK-NEXT:    call void @use(ptr [[OBJ1]])
 ; CHECK-NEXT:    br label [[JOIN]]
 ; CHECK:       join:
-; CHECK-NEXT:    call void @llvm.objc.release(ptr [[OBJ0]]) {{.*}}, !clang.imprecise_release !2
-; CHECK-NEXT:    call void @llvm.objc.release(ptr [[OBJ1]]) {{.*}}, !clang.imprecise_release !2
+; CHECK-NEXT:    call void @llvm.objc.release(ptr [[OBJ0]]) {{.*}}, !clang.imprecise_release ![[EMPTYMETA]]
+; CHECK-NEXT:    call void @llvm.objc.release(ptr [[OBJ1]]) {{.*}}, !clang.imprecise_release ![[EMPTYMETA]]
 ; CHECK-NEXT:    ret void
 ;
   %v0 = call ptr @llvm.objc.retain(ptr %obj0)
@@ -190,6 +190,8 @@ attributes #0 = { readonly }
 
 !llvm.module.flags = !{!0, !1}
 
+; CHECK: ![[EMPTYMETA]] = !{}
+
 !0 = !{i32 2, !"Dwarf Version", i32 4}
 !1 = !{i32 2, !"Debug Info Version", i32 3}
 !2 = !DILocalVariable(name: "i", arg: 1, scope: !3, file: !4, line: 1, type: !7)
@@ -201,3 +203,4 @@ attributes #0 = { readonly }
 !8 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !4, isOptimized: false, runtimeVersion: 2, emissionKind: FullDebug, enums: !9, nameTableKind: None)
 !9 = !{}
 !10 = !DILocation(line: 1, column: 14, scope: !3)
+!11 = !DILocalVariable(name: "foo", scope: !3, type: !7)
diff --git a/external/llvm-project/llvm/test/Transforms/OpenMP/nested_parallelism.ll b/external/llvm-project/llvm/test/Transforms/OpenMP/nested_parallelism.ll
index 1679a27fdae8..412e5ea5211e 100644
--- a/external/llvm-project/llvm/test/Transforms/OpenMP/nested_parallelism.ll
+++ b/external/llvm-project/llvm/test/Transforms/OpenMP/nested_parallelism.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
-; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s
+; RUN: opt -S -mtriple=nvptx64 -passes=openmp-opt < %s | FileCheck %s
+; REQUIRES: nvptx-registered-target
 
 ; void foo1(int i) {
 ;   #pragma omp parallel
@@ -20,8 +21,6 @@
 ;     foo1(i);
 ; }
 
-target triple = "nvptx64"
-
 %struct.ident_t = type { i32, i32, i32, i32, ptr }
 %struct.KernelEnvironmentTy = type { %struct.ConfigurationEnvironmentTy, ptr, ptr }
 %struct.ConfigurationEnvironmentTy = type { i8, i8, i8, i32, i32, i32, i32, i32, i32 }
@@ -64,7 +63,8 @@ define weak_odr protected ptx_kernel void @__omp_offloading_10302_bd7e0_main_l13
 ; CHECK-NEXT:    br label [[_Z3FOOI_INTERNALIZED_EXIT]]
 ; CHECK:       _Z3fooi.internalized.exit:
 ; CHECK-NEXT:    tail call void @__kmpc_barrier_simple_spmd(ptr nonnull @[[GLOB1]], i32 [[TMP2]]) #[[ATTR2]]
-; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i_shared to ptr), ptr [[CAPTURED_VARS_ADDRS_I]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS_I]] to ptr addrspace(5)
+; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i_shared to ptr), ptr addrspace(5) [[TMP4]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__, ptr nonnull @__omp_outlined___wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    call void @__kmpc_target_deinit()
@@ -109,7 +109,8 @@ define hidden void @_Z3fooi(i32 noundef %i1) local_unnamed_addr #1 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[I:%.*]] = tail call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[I1:%.*]], ptr [[I]], align 16
-; CHECK-NEXT:    store ptr [[I]], ptr [[CAPTURED_VARS_ADDRS]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS]] to ptr addrspace(5)
+; CHECK-NEXT:    store ptr [[I]], ptr addrspace(5) [[TMP1]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__, ptr nonnull @__omp_outlined___wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS]], i64 1)
 ; CHECK-NEXT:    call void @__kmpc_free_shared(ptr [[I]], i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    ret void
@@ -141,7 +142,8 @@ define weak_odr protected ptx_kernel void @__omp_offloading_10302_bd7e0_main_l16
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(3) @i.i_shared, align 16
-; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), ptr [[CAPTURED_VARS_ADDRS_I]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS_I]] to ptr addrspace(5)
+; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), ptr addrspace(5) [[TMP2]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    call void @__kmpc_target_deinit()
@@ -175,7 +177,8 @@ define hidden void @_Z4foo1i(i32 noundef %i1) local_unnamed_addr #1 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[I:%.*]] = tail call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[I1:%.*]], ptr [[I]], align 16
-; CHECK-NEXT:    store ptr [[I]], ptr [[CAPTURED_VARS_ADDRS]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS]] to ptr addrspace(5)
+; CHECK-NEXT:    store ptr [[I]], ptr addrspace(5) [[TMP1]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS]], i64 1)
 ; CHECK-NEXT:    call void @__kmpc_free_shared(ptr [[I]], i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    ret void
@@ -202,7 +205,8 @@ define internal void @__omp_outlined__(ptr noalias nocapture readnone %.global_t
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[I_I:%.*]] = tail call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr [[I_I]], align 16
-; CHECK-NEXT:    store ptr [[I_I]], ptr [[CAPTURED_VARS_ADDRS_I]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS_I]] to ptr addrspace(5)
+; CHECK-NEXT:    store ptr [[I_I]], ptr addrspace(5) [[TMP2]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
 ; CHECK-NEXT:    call void @__kmpc_free_shared(ptr [[I_I]], i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
@@ -228,15 +232,17 @@ define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 %1) #5 {
 ; CHECK-NEXT:    [[CAPTURED_VARS_ADDRS_I_I:%.*]] = alloca [1 x ptr], align 8
 ; CHECK-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    call void @__kmpc_get_shared_variables(ptr nonnull [[GLOBAL_ARGS]])
-; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = addrspacecast ptr [[GLOBAL_ARGS]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr addrspace(5) [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]])
-; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[I_I_I:%.*]] = call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr [[I_I_I]], align 16
-; CHECK-NEXT:    store ptr [[I_I_I]], ptr [[CAPTURED_VARS_ADDRS_I_I]], align 8
-; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP5]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]], i64 1)
+; CHECK-NEXT:    [[TMP7:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS_I_I]] to ptr addrspace(5)
+; CHECK-NEXT:    store ptr [[I_I_I]], ptr addrspace(5) [[TMP7]], align 8
+; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP6]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]], i64 1)
 ; CHECK-NEXT:    call void @__kmpc_free_shared(ptr [[I_I_I]], i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]])
 ; CHECK-NEXT:    ret void
@@ -287,7 +293,8 @@ define internal void @__omp_outlined__1_wrapper(i16 zeroext %0, i32 %1) #5 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    call void @__kmpc_get_shared_variables(ptr nonnull [[GLOBAL_ARGS]])
-; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = addrspacecast ptr [[GLOBAL_ARGS]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr addrspace(5) [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INC_I:%.*]] = add nsw i32 [[TMP4]], 1
diff --git a/external/llvm-project/llvm/test/Transforms/OpenMP/remove_globalization.ll b/external/llvm-project/llvm/test/Transforms/OpenMP/remove_globalization.ll
index 4c953e65ff3f..fdb450051890 100644
--- a/external/llvm-project/llvm/test/Transforms/OpenMP/remove_globalization.ll
+++ b/external/llvm-project/llvm/test/Transforms/OpenMP/remove_globalization.ll
@@ -1,9 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
 ; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s
-; RUN: opt -passes=openmp-opt -pass-remarks=openmp-opt -pass-remarks-missed=openmp-opt -disable-output < %s 2>&1 | FileCheck %s -check-prefix=CHECK-REMARKS
-; RUN: opt -openmp-opt-disable-deglobalization -S -passes=openmp-opt < %s | FileCheck %s --check-prefix=CHECK-DISABLED
-target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
-target triple = "nvptx64"
+; RUN: opt -mtriple=nvptx64 -passes=openmp-opt -pass-remarks=openmp-opt -pass-remarks-missed=openmp-opt -disable-output < %s 2>&1 | FileCheck %s -check-prefix=CHECK-REMARKS
+; RUN: opt -mtriple=nvptx64 -openmp-opt-disable-deglobalization -S -passes=openmp-opt < %s | FileCheck %s --check-prefix=CHECK-DISABLED
+; REQUIRES: nvptx-registered-target
 
 @S = external local_unnamed_addr global ptr
 %struct.KernelEnvironmentTy = type { %struct.ConfigurationEnvironmentTy, ptr, ptr }
@@ -183,17 +182,22 @@ define internal void @convert_and_move_alloca() {
 ; CHECK-DISABLED-NEXT:  entry:
 ; CHECK-DISABLED-NEXT:    [[DOTH2S:%.*]] = alloca i8, i64 4, align 4
 ; CHECK-DISABLED-NEXT:    [[IV_PTR:%.*]] = alloca i32, align 4
+; CHECK-DISABLED-NEXT:    [[UB_PTR:%.*]] = alloca i32, align 4
+; CHECK-DISABLED-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[UB_PTR]] to ptr addrspace(5)
 ; CHECK-DISABLED-NEXT:    br label [[INITLOOP:%.*]]
 ; CHECK-DISABLED:       initloop:
-; CHECK-DISABLED-NEXT:    store i32 0, ptr [[IV_PTR]], align 4
+; CHECK-DISABLED-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[IV_PTR]] to ptr addrspace(5)
+; CHECK-DISABLED-NEXT:    store i32 0, ptr addrspace(5) [[TMP1]], align 4
 ; CHECK-DISABLED-NEXT:    br label [[LOOPBODY:%.*]]
 ; CHECK-DISABLED:       loopbody:
-; CHECK-DISABLED-NEXT:    [[IV:%.*]] = load i32, ptr [[IV_PTR]], align 4
-; CHECK-DISABLED-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IV]], 10
-; CHECK-DISABLED-NEXT:    br i1 [[TMP0]], label [[EXIT:%.*]], label [[LOOPINC:%.*]]
+; CHECK-DISABLED-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[IV_PTR]] to ptr addrspace(5)
+; CHECK-DISABLED-NEXT:    [[IV:%.*]] = load i32, ptr addrspace(5) [[TMP2]], align 4
+; CHECK-DISABLED-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[IV]], 10
+; CHECK-DISABLED-NEXT:    br i1 [[TMP3]], label [[EXIT:%.*]], label [[LOOPINC:%.*]]
 ; CHECK-DISABLED:       loopinc:
 ; CHECK-DISABLED-NEXT:    [[INC:%.*]] = add i32 [[IV]], 1
-; CHECK-DISABLED-NEXT:    store i32 [[INC]], ptr [[IV_PTR]], align 4
+; CHECK-DISABLED-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[IV_PTR]] to ptr addrspace(5)
+; CHECK-DISABLED-NEXT:    store i32 [[INC]], ptr addrspace(5) [[TMP4]], align 4
 ; CHECK-DISABLED-NEXT:    br label [[LOOPBODY]]
 ; CHECK-DISABLED:       exit:
 ; CHECK-DISABLED-NEXT:    ret void
@@ -268,7 +272,7 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 ; CHECK-DISABLED: attributes #[[ATTR6]] = { nounwind }
 ;.
 ; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]], splitDebugInlining: false, nameTableKind: None)
-; CHECK: [[META1]] = !DIFile(filename: "remove_globalization.c", directory: {{.*}})
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}remove_globalization.c", directory: {{.*}})
 ; CHECK: [[META2]] = !{}
 ; CHECK: [[META3:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3}
 ; CHECK: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
@@ -279,7 +283,7 @@ declare void @unknown_no_openmp() "llvm.assume"="omp_no_openmp"
 ; CHECK: [[META9]] = !DISubroutineType(types: [[META2]])
 ;.
 ; CHECK-DISABLED: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]], splitDebugInlining: false, nameTableKind: None)
-; CHECK-DISABLED: [[META1]] = !DIFile(filename: "remove_globalization.c", directory: {{.*}})
+; CHECK-DISABLED: [[META1]] = !DIFile(filename: "{{.*}}remove_globalization.c", directory: {{.*}})
 ; CHECK-DISABLED: [[META2]] = !{}
 ; CHECK-DISABLED: [[META3:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3}
 ; CHECK-DISABLED: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
diff --git a/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding.ll b/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
index 210ef84dcba7..d605a8661463 100644
--- a/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
+++ b/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
@@ -70,28 +70,29 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK:       user_code.entry:
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[N]], 42
 ; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[C]], ptr [[AL32]], ptr addrspacecast (ptr addrspace(5) @LocGlob to ptr)
-; CHECK-NEXT:    store ptr [[SELECT]], ptr [[LOC]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR6]]
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[LOC]] to ptr addrspace(5)
+; CHECK-NEXT:    store ptr [[SELECT]], ptr addrspace(5) [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR6]]
 ; CHECK-NEXT:    [[ARRAYIDX1_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 1
 ; CHECK-NEXT:    [[SEXT:%.*]] = shl i64 [[N]], 32
 ; CHECK-NEXT:    [[IDXPROM_I:%.*]] = ashr exact i64 [[SEXT]], 32
 ; CHECK-NEXT:    [[ARRAYIDX2_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID:%.*]]
 ; CHECK:       region.check.tid:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
 ; CHECK:       region.guarded:
 ; CHECK-NEXT:    store i32 0, ptr [[X]], align 4, !noalias [[META7:![0-9]+]]
-; CHECK-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[ARRAYIDX1_I]] to ptr addrspace(1)
-; CHECK-NEXT:    store i32 1, ptr addrspace(1) [[TMP4]], align 4, !noalias [[META7]]
-; CHECK-NEXT:    [[TMP5:%.*]] = addrspacecast ptr [[ARRAYIDX2_I]] to ptr addrspace(1)
-; CHECK-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(1) [[TMP5]], align 4, !noalias [[META7]]
+; CHECK-NEXT:    [[TMP5:%.*]] = addrspacecast ptr [[ARRAYIDX1_I]] to ptr addrspace(1)
+; CHECK-NEXT:    store i32 1, ptr addrspace(1) [[TMP5]], align 4, !noalias [[META7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = addrspacecast ptr [[ARRAYIDX2_I]] to ptr addrspace(1)
+; CHECK-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(1) [[TMP6]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END:%.*]]
 ; CHECK:       region.guarded.end:
 ; CHECK-NEXT:    br label [[REGION_BARRIER]]
 ; CHECK:       region.barrier:
-; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP2]])
+; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP3]])
 ; CHECK-NEXT:    br label [[REGION_EXIT:%.*]]
 ; CHECK:       region.exit:
 ; CHECK-NEXT:    call void @usei8ptr(ptr captures(none) [[HEAP2STACK_H2S]]) #[[ATTR9:[0-9]+]]
@@ -107,19 +108,20 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-NEXT:    [[ARRAYIDX5_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM4_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID5:%.*]]
 ; CHECK:       region.check.tid5:
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[REGION_GUARDED4:%.*]], label [[REGION_BARRIER2:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[REGION_GUARDED4:%.*]], label [[REGION_BARRIER2:%.*]]
 ; CHECK:       region.guarded4:
-; CHECK-NEXT:    [[TMP8:%.*]] = addrspacecast ptr [[ARRAYIDX5_I]] to ptr addrspace(1)
-; CHECK-NEXT:    store i32 [[SUB3_I]], ptr addrspace(1) [[TMP8]], align 4, !noalias [[META7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = addrspacecast ptr [[ARRAYIDX5_I]] to ptr addrspace(1)
+; CHECK-NEXT:    store i32 [[SUB3_I]], ptr addrspace(1) [[TMP9]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END1:%.*]]
 ; CHECK:       region.guarded.end1:
 ; CHECK-NEXT:    br label [[REGION_BARRIER2]]
 ; CHECK:       region.barrier2:
-; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP6]])
+; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP7]])
 ; CHECK-NEXT:    br label [[REGION_EXIT3]]
 ; CHECK:       region.exit3:
+; CHECK-NEXT:    [[TMP10:%.*]] = addrspacecast ptr [[SELECT]] to ptr addrspace(5)
 ; CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
 ; CHECK-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       __omp_outlined__.exit:
@@ -129,17 +131,17 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-NEXT:    [[ARRAYIDX7_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM6_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID10:%.*]]
 ; CHECK:       region.check.tid10:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP9]], 0
-; CHECK-NEXT:    br i1 [[TMP10]], label [[REGION_GUARDED9:%.*]], label [[REGION_BARRIER7:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[TMP12]], label [[REGION_GUARDED9:%.*]], label [[REGION_BARRIER7:%.*]]
 ; CHECK:       region.guarded9:
-; CHECK-NEXT:    [[TMP11:%.*]] = addrspacecast ptr [[ARRAYIDX7_I]] to ptr addrspace(1)
-; CHECK-NEXT:    store i32 [[CALL_I]], ptr addrspace(1) [[TMP11]], align 4, !noalias [[META7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = addrspacecast ptr [[ARRAYIDX7_I]] to ptr addrspace(1)
+; CHECK-NEXT:    store i32 [[CALL_I]], ptr addrspace(1) [[TMP13]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END6:%.*]]
 ; CHECK:       region.guarded.end6:
 ; CHECK-NEXT:    br label [[REGION_BARRIER7]]
 ; CHECK:       region.barrier7:
-; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP9]])
+; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP11]])
 ; CHECK-NEXT:    br label [[REGION_EXIT8:%.*]]
 ; CHECK:       region.exit8:
 ; CHECK-NEXT:    [[CALL8_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
@@ -147,17 +149,17 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-NEXT:    [[ARRAYIDX10_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM9_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID15:%.*]]
 ; CHECK:       region.check.tid15:
-; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[REGION_GUARDED14:%.*]], label [[REGION_BARRIER12:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[TMP15]], label [[REGION_GUARDED14:%.*]], label [[REGION_BARRIER12:%.*]]
 ; CHECK:       region.guarded14:
-; CHECK-NEXT:    [[TMP14:%.*]] = addrspacecast ptr [[ARRAYIDX10_I]] to ptr addrspace(1)
-; CHECK-NEXT:    store i32 [[CALL8_I]], ptr addrspace(1) [[TMP14]], align 4, !noalias [[META7]]
+; CHECK-NEXT:    [[TMP16:%.*]] = addrspacecast ptr [[ARRAYIDX10_I]] to ptr addrspace(1)
+; CHECK-NEXT:    store i32 [[CALL8_I]], ptr addrspace(1) [[TMP16]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END11:%.*]]
 ; CHECK:       region.guarded.end11:
 ; CHECK-NEXT:    br label [[REGION_BARRIER12]]
 ; CHECK:       region.barrier12:
-; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP12]])
+; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP14]])
 ; CHECK-NEXT:    br label [[REGION_EXIT13:%.*]]
 ; CHECK:       region.exit13:
 ; CHECK-NEXT:    [[CALL11_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
@@ -165,17 +167,17 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-NEXT:    [[ARRAYIDX13_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM12_I]]
 ; CHECK-NEXT:    br label [[REGION_CHECK_TID20:%.*]]
 ; CHECK:       region.check.tid20:
-; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
-; CHECK-NEXT:    br i1 [[TMP16]], label [[REGION_GUARDED19:%.*]], label [[REGION_BARRIER17:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP17]], 0
+; CHECK-NEXT:    br i1 [[TMP18]], label [[REGION_GUARDED19:%.*]], label [[REGION_BARRIER17:%.*]]
 ; CHECK:       region.guarded19:
-; CHECK-NEXT:    [[TMP17:%.*]] = addrspacecast ptr [[ARRAYIDX13_I]] to ptr addrspace(1)
-; CHECK-NEXT:    store i32 [[CALL11_I]], ptr addrspace(1) [[TMP17]], align 4, !noalias [[META7]]
+; CHECK-NEXT:    [[TMP19:%.*]] = addrspacecast ptr [[ARRAYIDX13_I]] to ptr addrspace(1)
+; CHECK-NEXT:    store i32 [[CALL11_I]], ptr addrspace(1) [[TMP19]], align 4, !noalias [[META7]]
 ; CHECK-NEXT:    br label [[REGION_GUARDED_END16:%.*]]
 ; CHECK:       region.guarded.end16:
 ; CHECK-NEXT:    br label [[REGION_BARRIER17]]
 ; CHECK:       region.barrier17:
-; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP15]])
+; CHECK-NEXT:    call void @__kmpc_barrier_simple_spmd(ptr @[[GLOB2]], i32 [[TMP17]])
 ; CHECK-NEXT:    br label [[REGION_EXIT18:%.*]]
 ; CHECK:       region.exit18:
 ; CHECK-NEXT:    [[CALL14_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR10]], !noalias [[META7]]
@@ -199,17 +201,18 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-DISABLED:       user_code.entry:
 ; CHECK-DISABLED-NEXT:    [[C:%.*]] = icmp eq i64 [[N]], 42
 ; CHECK-DISABLED-NEXT:    [[SELECT:%.*]] = select i1 [[C]], ptr [[AL32]], ptr addrspacecast (ptr addrspace(5) @LocGlob to ptr)
-; CHECK-DISABLED-NEXT:    store ptr [[SELECT]], ptr [[LOC]], align 8
-; CHECK-DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR6]]
+; CHECK-DISABLED-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[LOC]] to ptr addrspace(5)
+; CHECK-DISABLED-NEXT:    store ptr [[SELECT]], ptr addrspace(5) [[TMP1]], align 8
+; CHECK-DISABLED-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR6]]
 ; CHECK-DISABLED-NEXT:    store i32 0, ptr [[X]], align 4, !noalias [[META7:![0-9]+]]
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX1_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 1
-; CHECK-DISABLED-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[ARRAYIDX1_I]] to ptr addrspace(1)
-; CHECK-DISABLED-NEXT:    store i32 1, ptr addrspace(1) [[TMP2]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[ARRAYIDX1_I]] to ptr addrspace(1)
+; CHECK-DISABLED-NEXT:    store i32 1, ptr addrspace(1) [[TMP3]], align 4, !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[SEXT:%.*]] = shl i64 [[N]], 32
 ; CHECK-DISABLED-NEXT:    [[IDXPROM_I:%.*]] = ashr exact i64 [[SEXT]], 32
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX2_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM_I]]
-; CHECK-DISABLED-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[ARRAYIDX2_I]] to ptr addrspace(1)
-; CHECK-DISABLED-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(1) [[TMP3]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[ARRAYIDX2_I]] to ptr addrspace(1)
+; CHECK-DISABLED-NEXT:    store i32 [[N_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(1) [[TMP4]], align 4, !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    call void @usei8ptr(ptr captures(none) [[HEAP2STACK_H2S]]) #[[ATTR8:[0-9]+]]
 ; CHECK-DISABLED-NEXT:    br label [[FOR_COND_I:%.*]]
 ; CHECK-DISABLED:       for.cond.i:
@@ -221,8 +224,9 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-DISABLED-NEXT:    [[SUB3_I:%.*]] = add nsw i32 [[I_0_I]], -1
 ; CHECK-DISABLED-NEXT:    [[IDXPROM4_I:%.*]] = zext i32 [[I_0_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX5_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM4_I]]
-; CHECK-DISABLED-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[ARRAYIDX5_I]] to ptr addrspace(1)
-; CHECK-DISABLED-NEXT:    store i32 [[SUB3_I]], ptr addrspace(1) [[TMP4]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[TMP5:%.*]] = addrspacecast ptr [[ARRAYIDX5_I]] to ptr addrspace(1)
+; CHECK-DISABLED-NEXT:    store i32 [[SUB3_I]], ptr addrspace(1) [[TMP5]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[TMP6:%.*]] = addrspacecast ptr [[SELECT]] to ptr addrspace(5)
 ; CHECK-DISABLED-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
 ; CHECK-DISABLED-NEXT:    br label [[FOR_COND_I]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-DISABLED:       __omp_outlined__.exit:
@@ -230,18 +234,18 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-DISABLED-NEXT:    [[CALL_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR9:[0-9]+]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[IDXPROM6_I:%.*]] = sext i32 [[CALL_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX7_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM6_I]]
-; CHECK-DISABLED-NEXT:    [[TMP5:%.*]] = addrspacecast ptr [[ARRAYIDX7_I]] to ptr addrspace(1)
-; CHECK-DISABLED-NEXT:    store i32 [[CALL_I]], ptr addrspace(1) [[TMP5]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[TMP7:%.*]] = addrspacecast ptr [[ARRAYIDX7_I]] to ptr addrspace(1)
+; CHECK-DISABLED-NEXT:    store i32 [[CALL_I]], ptr addrspace(1) [[TMP7]], align 4, !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[CALL8_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR9]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[IDXPROM9_I:%.*]] = sext i32 [[CALL8_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX10_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM9_I]]
-; CHECK-DISABLED-NEXT:    [[TMP6:%.*]] = addrspacecast ptr [[ARRAYIDX10_I]] to ptr addrspace(1)
-; CHECK-DISABLED-NEXT:    store i32 [[CALL8_I]], ptr addrspace(1) [[TMP6]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[TMP8:%.*]] = addrspacecast ptr [[ARRAYIDX10_I]] to ptr addrspace(1)
+; CHECK-DISABLED-NEXT:    store i32 [[CALL8_I]], ptr addrspace(1) [[TMP8]], align 4, !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[CALL11_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR9]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[IDXPROM12_I:%.*]] = sext i32 [[CALL11_I]] to i64
 ; CHECK-DISABLED-NEXT:    [[ARRAYIDX13_I:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM12_I]]
-; CHECK-DISABLED-NEXT:    [[TMP7:%.*]] = addrspacecast ptr [[ARRAYIDX13_I]] to ptr addrspace(1)
-; CHECK-DISABLED-NEXT:    store i32 [[CALL11_I]], ptr addrspace(1) [[TMP7]], align 4, !noalias [[META7]]
+; CHECK-DISABLED-NEXT:    [[TMP9:%.*]] = addrspacecast ptr [[ARRAYIDX13_I]] to ptr addrspace(1)
+; CHECK-DISABLED-NEXT:    store i32 [[CALL11_I]], ptr addrspace(1) [[TMP9]], align 4, !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[CALL14_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR9]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[CALL15_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR9]], !noalias [[META7]]
 ; CHECK-DISABLED-NEXT:    [[CALL16_I:%.*]] = call i32 @no_openmp(ptr nonnull [[X]]) #[[ATTR9]], !noalias [[META7]]
diff --git a/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll b/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll
index a644fe1b2f82..821f5b194843 100644
--- a/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll
+++ b/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s --check-prefixes=CHECK
-; RUN: opt -S -passes=openmp-opt -openmp-opt-disable-spmdization < %s | FileCheck %s --check-prefixes=CHECK-DISABLE-SPMDIZATION
+; RUN: opt -S -mtriple=nvptx64 -passes=openmp-opt < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt -S -mtriple=nvptx64 -passes=openmp-opt -openmp-opt-disable-spmdization < %s | FileCheck %s --check-prefixes=CHECK-DISABLE-SPMDIZATION
+; REQUIRES: nvptx-registered-target
 ;
 ; int G;
 ;
@@ -26,8 +27,6 @@
 ;   generic_helper();
 ; }
 ;
-target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
-target triple = "nvptx64"
 
 %struct.ident_t = type { i32, i32, i32, i32, ptr }
 %struct.ConfigurationEnvironmentTy = type { i8, i8, i8, i32, i32, i32, i32, i32, i32 }
@@ -195,6 +194,8 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[DOTGLOBAL_TID__ADDR]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[DOTBOUND_TID__ADDR]] to ptr addrspace(5)
 ; CHECK-NEXT:    call void @unknown() #[[ATTR8:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
@@ -203,6 +204,8 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK-DISABLE-SPMDIZATION-NEXT:  entry:
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[DOTGLOBAL_TID__ADDR]] to ptr addrspace(5)
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[DOTBOUND_TID__ADDR]] to ptr addrspace(5)
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @unknown() #[[ATTR8:[0-9]+]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    ret void
 ;
@@ -224,6 +227,9 @@ define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 %1) #2 {
 ; CHECK-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[DOTADDR]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[DOTADDR1]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[DOTZERO_ADDR]] to ptr addrspace(5)
 ; CHECK-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; CHECK-NEXT:    call void @__omp_outlined__(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret void
@@ -235,6 +241,9 @@ define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 %1) #2 {
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[DOTADDR]] to ptr addrspace(5)
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[DOTADDR1]] to ptr addrspace(5)
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[DOTZERO_ADDR]] to ptr addrspace(5)
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__omp_outlined__(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR3]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    ret void
diff --git a/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll b/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
index d68c79ce4b6c..63405828c1ce 100644
--- a/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
+++ b/external/llvm-project/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s --check-prefixes=CHECK
-; RUN: opt -S -passes=openmp-opt -openmp-opt-disable-spmdization < %s | FileCheck %s --check-prefixes=CHECK-DISABLE-SPMDIZATION
+; RUN: opt -S -mtriple=nvptx64 -passes=openmp-opt < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt -S -mtriple=nvptx64 -passes=openmp-opt -openmp-opt-disable-spmdization < %s | FileCheck %s --check-prefixes=CHECK-DISABLE-SPMDIZATION
+; REQUIRES: nvptx-registered-target
 ;
 ; __local int G;
 ;
@@ -27,8 +28,6 @@
 ;   generic_helper();
 ; }
 ;
-target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
-target triple = "nvptx64"
 
 %struct.ident_t = type { i32, i32, i32, i32, ptr }
 %struct.ConfigurationEnvironmentTy = type { i8, i8, i8, i32, i32, i32, i32, i32, i32 }
@@ -189,6 +188,8 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[DOTGLOBAL_TID__ADDR]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[DOTBOUND_TID__ADDR]] to ptr addrspace(5)
 ; CHECK-NEXT:    call void @leaf() #[[ATTR8]]
 ; CHECK-NEXT:    ret void
 ;
@@ -197,6 +198,8 @@ define internal void @__omp_outlined__(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK-DISABLE-SPMDIZATION-NEXT:  entry:
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[DOTGLOBAL_TID__ADDR]] to ptr addrspace(5)
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[DOTBOUND_TID__ADDR]] to ptr addrspace(5)
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @leaf() #[[ATTR8]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    ret void
 ;
@@ -218,6 +221,9 @@ define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 %1) #2 {
 ; CHECK-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[DOTADDR]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[DOTADDR1]] to ptr addrspace(5)
+; CHECK-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[DOTZERO_ADDR]] to ptr addrspace(5)
 ; CHECK-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; CHECK-NEXT:    call void @__omp_outlined__(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR8]]
 ; CHECK-NEXT:    ret void
@@ -229,6 +235,9 @@ define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 %1) #2 {
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTADDR1:%.*]] = alloca i32, align 4
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[DOTADDR]] to ptr addrspace(5)
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP3:%.*]] = addrspacecast ptr [[DOTADDR1]] to ptr addrspace(5)
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[DOTZERO_ADDR]] to ptr addrspace(5)
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__omp_outlined__(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]]) #[[ATTR8]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    ret void
diff --git a/external/llvm-project/llvm/test/Transforms/PGOProfile/ctx-instrumentation-aliases.ll b/external/llvm-project/llvm/test/Transforms/PGOProfile/ctx-instrumentation-aliases.ll
new file mode 100644
index 000000000000..39e2d138fb97
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/PGOProfile/ctx-instrumentation-aliases.ll
@@ -0,0 +1,25 @@
+; REQUIRES: x86-registered-target
+; Test that calls to aliases are instrumented, and the assembly references the
+; aliased function.
+;
+; RUN: opt -passes=ctx-instr-gen,assign-guid,ctx-instr-lower -profile-context-root=an_entrypoint \
+; RUN:   -profile-context-root=another_entrypoint_no_callees \
+; RUN:   -S %s -o %t.ll
+; RUN: llc < %t.ll | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo_alias = weak_odr unnamed_addr alias void (), ptr @foo
+
+define void @foo(i32) {
+  ret void
+}
+
+define void @call_alias(ptr %a) {
+entry:
+  call void @foo(i32 0, ptr %a)
+  ret void
+}
+
+; CHECK-LABEL:   call_alias:
+; CHECK:         movq    foo@GOTPCREL(%rip), [[REG:%r[a-z0-9]+]]
+; CHECK-NEXT:    movq    [[REG]], %fs:__llvm_ctx_profile_expected_callee@TPOFF{{.*}}
diff --git a/external/llvm-project/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/external/llvm-project/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
index 71d54f98d26e..a2f3d52d713d 100644
--- a/external/llvm-project/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
+++ b/external/llvm-project/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
@@ -8,6 +8,8 @@
 
 declare void @bar()
 
+;.
+; INSTRUMENT: @foo_alias = weak_odr unnamed_addr alias void (i32, ptr), ptr @foo
 ;.
 ; LOWERING: @__llvm_ctx_profile_callsite = external hidden thread_local global ptr
 ; LOWERING: @__llvm_ctx_profile_expected_callee = external hidden thread_local global ptr
@@ -19,6 +21,8 @@ declare void @bar()
 ; LOWERING: @[[GLOB5:[0-9]+]] = internal global { ptr, ptr, ptr, ptr, i8 } zeroinitializer
 ; LOWERING: @[[GLOB6:[0-9]+]] = internal global { ptr, ptr, ptr, ptr, i8 } zeroinitializer
 ; LOWERING: @[[GLOB7:[0-9]+]] = internal global { ptr, ptr, ptr, ptr, i8 } { ptr null, ptr null, ptr inttoptr (i64 1 to ptr), ptr null, i8 0 }
+; LOWERING: @[[GLOB8:[0-9]+]] = internal global { ptr, ptr, ptr, ptr, i8 } zeroinitializer
+; LOWERING: @foo_alias = weak_odr unnamed_addr alias void (i32, ptr), ptr @foo
 ;.
 define void @foo(i32 %a, ptr %fct) {
 ; INSTRUMENT-LABEL: define void @foo(
@@ -335,6 +339,41 @@ define void @unreachable() {
 ;
   unreachable
 }
+
+@foo_alias = weak_odr unnamed_addr alias void (i32, ptr), ptr @foo
+
+define void @call_alias(ptr %a) {
+; INSTRUMENT-LABEL: define void @call_alias(
+; INSTRUMENT-SAME: ptr [[A:%.*]]) {
+; INSTRUMENT-NEXT:  entry:
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @call_alias, i64 742261418966908927, i32 1, i32 0)
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.callsite(ptr @call_alias, i64 742261418966908927, i32 1, i32 0, ptr @foo_alias)
+; INSTRUMENT-NEXT:    call void @foo_alias(i32 0, ptr [[A]])
+; INSTRUMENT-NEXT:    ret void
+;
+; LOWERING-LABEL: define void @call_alias(
+; LOWERING-SAME: ptr [[A:%.*]]) !guid [[META10:![0-9]+]] {
+; LOWERING-NEXT:  entry:
+; LOWERING-NEXT:    [[TMP0:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @[[GLOB8]], ptr @call_alias, i64 2172368043968427688, i32 1, i32 1)
+; LOWERING-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; LOWERING-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], 1
+; LOWERING-NEXT:    [[TMP3:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_expected_callee)
+; LOWERING-NEXT:    [[TMP4:%.*]] = getelementptr ptr, ptr [[TMP3]], i64 [[TMP2]]
+; LOWERING-NEXT:    [[TMP5:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_callsite)
+; LOWERING-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i64 [[TMP2]]
+; LOWERING-NEXT:    [[TMP7:%.*]] = and i64 [[TMP1]], -2
+; LOWERING-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; LOWERING-NEXT:    store volatile ptr @foo_alias, ptr [[TMP4]], align 8
+; LOWERING-NEXT:    [[TMP9:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [1 x i64], [1 x ptr] }, ptr [[TMP0]], i32 0, i32 2, i32 0
+; LOWERING-NEXT:    store volatile ptr [[TMP9]], ptr [[TMP6]], align 8
+; LOWERING-NEXT:    call void @foo_alias(i32 0, ptr [[A]])
+; LOWERING-NEXT:    call void @__llvm_ctx_profile_release_context(ptr @[[GLOB8]])
+; LOWERING-NEXT:    ret void
+;
+entry:
+  call void @foo_alias(i32 0, ptr %a)
+  ret void
+}
 ;.
 ; LOWERING: attributes #[[ATTR0]] = { noreturn }
 ; LOWERING: attributes #[[ATTR1:[0-9]+]] = { nounwind }
@@ -353,4 +392,5 @@ define void @unreachable() {
 ; LOWERING: [[META7]] = !{i64 -4680624981836544329}
 ; LOWERING: [[META8]] = !{i64 5519225910966780583}
 ; LOWERING: [[META9]] = !{i64 -565652589829076809}
+; LOWERING: [[META10]] = !{i64 2172368043968427688}
 ;.
diff --git a/external/llvm-project/llvm/test/Transforms/PGOProfile/memprof.ll b/external/llvm-project/llvm/test/Transforms/PGOProfile/memprof.ll
index e48da36a6d97..c69d0311e038 100644
--- a/external/llvm-project/llvm/test/Transforms/PGOProfile/memprof.ll
+++ b/external/llvm-project/llvm/test/Transforms/PGOProfile/memprof.ll
@@ -111,13 +111,13 @@
 ; RUN: opt < %s -passes='memprof-use<profile-filename=%t.memprofdata>' -pgo-warn-missing-function -S -memprof-min-ave-lifetime-access-density-hot-threshold=0 2>&1 | FileCheck %s --check-prefixes=MEMPROF,ALL
 
 ; MEMPROFMATCHINFO: MemProf notcold context with id 1093248920606587996 has total profiled size 10 is matched with 1 frames
-; MEMPROFMATCHINFO: MemProf cold context with id 8525406123785421946 has total profiled size 10 is matched with 1 frames
-; MEMPROFMATCHINFO: MemProf cold context with id 16342802530253093571 has total profiled size 10 is matched with 1 frames
-; MEMPROFMATCHINFO: MemProf cold context with id 18254812774972004394 has total profiled size 10 is matched with 1 frames
-; MEMPROFMATCHINFO: MemProf cold context with id 11714230664165068698 has total profiled size 10 is matched with 1 frames
 ; MEMPROFMATCHINFO: MemProf notcold context with id 5725971306423925017 has total profiled size 10 is matched with 1 frames
 ; MEMPROFMATCHINFO: MemProf notcold context with id 6792096022461663180 has total profiled size 10 is matched with 1 frames
+; MEMPROFMATCHINFO: MemProf cold context with id 8525406123785421946 has total profiled size 10 is matched with 1 frames
+; MEMPROFMATCHINFO: MemProf cold context with id 11714230664165068698 has total profiled size 10 is matched with 1 frames
 ; MEMPROFMATCHINFO: MemProf cold context with id 15737101490731057601 has total profiled size 10 is matched with 1 frames
+; MEMPROFMATCHINFO: MemProf cold context with id 16342802530253093571 has total profiled size 10 is matched with 1 frames
+; MEMPROFMATCHINFO: MemProf cold context with id 18254812774972004394 has total profiled size 10 is matched with 1 frames
 ; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 748269490701775343
 ; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 1544787832369987002
 ; MEMPROFMATCHINFO: MemProf callsite match for inline call stack 2061451396820446691
diff --git a/external/llvm-project/llvm/test/Transforms/PGOProfile/memprof_max_cold_threshold.test b/external/llvm-project/llvm/test/Transforms/PGOProfile/memprof_max_cold_threshold.test
new file mode 100644
index 000000000000..8f1cabf9a4e8
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/PGOProfile/memprof_max_cold_threshold.test
@@ -0,0 +1,165 @@
+;; Test the -memprof-min-percent-max-cold-size flag for annotating only the
+;; largest contexts with size info.
+
+; REQUIRES: x86_64-linux
+
+; RUN: split-file %s %t
+
+;; Specify version 4 so that a summary is generated.
+; RUN: llvm-profdata merge --memprof-version=4 %t/memprof_max_cold_threshold.yaml -o %t/memprof_max_cold_threshold.memprofdata
+
+; RUN: llvm-profdata show --memory %t/memprof_max_cold_threshold.memprofdata | FileCheck %s --check-prefixes=SUMMARY
+; SUMMARY: # MemProfSummary:
+; SUMMARY: #   Total contexts: 5
+; SUMMARY: #   Total cold contexts: 4
+; SUMMARY: #   Total hot contexts: 0
+; SUMMARY: #   Maximum cold context total size: 400
+; SUMMARY: #   Maximum warm context total size: 500
+; SUMMARY: #   Maximum hot context total size: 0
+
+;; Test with various thresholds, starting with the baseline of no context size
+;; info.
+; RUN: opt < %t/memprof_max_cold_threshold.ll -passes='memprof-use<profile-filename=%t/memprof_max_cold_threshold.memprofdata>' -S | FileCheck %s --check-prefixes=ALL,NONE
+; RUN: opt < %t/memprof_max_cold_threshold.ll -memprof-min-percent-max-cold-size=99 -passes='memprof-use<profile-filename=%t/memprof_max_cold_threshold.memprofdata>' -S | FileCheck %s --check-prefixes=ALL,THRESH99
+; RUN: opt < %t/memprof_max_cold_threshold.ll -memprof-min-percent-max-cold-size=75 -passes='memprof-use<profile-filename=%t/memprof_max_cold_threshold.memprofdata>' -S | FileCheck %s --check-prefixes=ALL,THRESH75
+; RUN: opt < %t/memprof_max_cold_threshold.ll -memprof-min-percent-max-cold-size=50 -passes='memprof-use<profile-filename=%t/memprof_max_cold_threshold.memprofdata>' -S | FileCheck %s --check-prefixes=ALL,THRESH50
+
+;; Make sure serializing / deserializing through bitcode works
+; RUN: opt < %t/memprof_max_cold_threshold.ll -memprof-min-percent-max-cold-size=75 -passes='memprof-use<profile-filename=%t/memprof_max_cold_threshold.memprofdata>' -o %t/out.bc
+; RUN: llvm-dis %t/out.bc -o - | FileCheck %s --check-prefixes=ALL,THRESH75
+
+;--- memprof_max_cold_threshold.yaml
+---
+HeapProfileRecords:
+  - GUID:            _Z3foov
+    AllocSites:
+      - Callstack:
+          - { Function: _Z3foov, LineOffset: 0, Column: 22, IsInlineFrame: false }
+          - { Function: main, LineOffset: 2, Column: 5, IsInlineFrame: false }
+        MemInfoBlock:
+	  # Smallest cold, 25% of largest cold size
+          TotalSize:                  100
+          AllocCount:                 1
+          TotalLifetimeAccessDensity: 1
+          TotalLifetime:              1000000
+      - Callstack:
+          - { Function: _Z3foov, LineOffset: 0, Column: 22, IsInlineFrame: false }
+          - { Function: main, LineOffset: 3, Column: 5, IsInlineFrame: false }
+        MemInfoBlock:
+	  # Largest cold
+          TotalSize:                  400
+          AllocCount:                 1
+          TotalLifetimeAccessDensity: 1
+          TotalLifetime:              1000000
+      - Callstack:
+          - { Function: _Z3foov, LineOffset: 0, Column: 22, IsInlineFrame: false }
+          - { Function: main, LineOffset: 4, Column: 5, IsInlineFrame: false }
+        MemInfoBlock:
+	  # Second largest cold, 75% of largest cold size
+          TotalSize:                  300
+          AllocCount:                 1
+          TotalLifetimeAccessDensity: 1
+          TotalLifetime:              1000000
+      - Callstack:
+          - { Function: _Z3foov, LineOffset: 0, Column: 22, IsInlineFrame: false }
+          - { Function: main, LineOffset: 5, Column: 5, IsInlineFrame: false }
+        MemInfoBlock:
+	  # Second smallest cold, 50% of largest cold size
+          TotalSize:                  200
+          AllocCount:                 1
+          TotalLifetimeAccessDensity: 1
+          TotalLifetime:              1000000
+      - Callstack:
+          - { Function: _Z3foov, LineOffset: 0, Column: 22, IsInlineFrame: false }
+          - { Function: main, LineOffset: 6, Column: 5, IsInlineFrame: false }
+        MemInfoBlock:
+	  # Largest context, which is non-cold (due to short lifetime)
+          TotalSize:                  500
+          AllocCount:                 1
+          TotalLifetimeAccessDensity: 1
+          TotalLifetime:              1
+    CallSites:       []
+...
+;--- memprof_max_cold_threshold.ll
+define dso_local ptr @_Z3foov() !dbg !4 {
+entry:
+  %call = call ptr @_Znam(i64 4) #0, !dbg !5
+; ALL: call ptr @_Znam(i64 4) {{.*}} !memprof ![[M1:[0-9]+]]
+  ret ptr %call
+}
+
+;; No MIBs get context size info without option.
+; NONE: ![[M1]] = !{![[M2:[0-9]+]], ![[M3:[0-9]+]], ![[M4:[0-9]+]], ![[M5:[0-9]+]], ![[M6:[0-9]+]]}
+; NONE: ![[M2]] = !{![[C2:[0-9]+]], !"cold"}
+; NONE: ![[C2]] = !{i64 590523745590780990, i64 720385627691022109}
+; NONE: ![[M3]] = !{![[C3:[0-9]+]], !"cold"}
+; NONE: ![[C3]] = !{i64 590523745590780990, i64 8256520048276991898}
+; NONE: ![[M4]] = !{![[C4:[0-9]+]], !"cold"}
+; NONE: ![[C4]] = !{i64 590523745590780990, i64 -6953100768213558995}
+; NONE: ![[M5]] = !{![[C5:[0-9]+]], !"cold"}
+; NONE: ![[C5]] = !{i64 590523745590780990, i64 -6435117705372285425}
+; NONE: ![[M6]] = !{![[C6:[0-9]+]], !"notcold"}
+; NONE: ![[C6]] = !{i64 590523745590780990, i64 -2847840206325626610}
+
+; THRESH99: ![[M1]] = !{![[M2:[0-9]+]], ![[M3:[0-9]+]], ![[M4:[0-9]+]], ![[M5:[0-9]+]], ![[M6:[0-9]+]]}
+; THRESH99: ![[M2]] = !{![[C2:[0-9]+]], !"cold"}
+; THRESH99: ![[C2]] = !{i64 590523745590780990, i64 720385627691022109}
+;; MIB with size 400 is now included with threshold 99%
+; THRESH99: ![[M3]] = !{![[C3:[0-9]+]], !"cold", ![[S3:[0-9]+]]}
+; THRESH99: ![[C3]] = !{i64 590523745590780990, i64 8256520048276991898}
+; THRESH99: ![[S3]] = !{i64 6509573709067523871, i64 400}
+; THRESH99: ![[M4]] = !{![[C4:[0-9]+]], !"cold"}
+; THRESH99: ![[C4]] = !{i64 590523745590780990, i64 -6953100768213558995}
+; THRESH99: ![[M5]] = !{![[C5:[0-9]+]], !"cold"}
+; THRESH99: ![[C5]] = !{i64 590523745590780990, i64 -6435117705372285425}
+; THRESH99: ![[M6]] = !{![[C6:[0-9]+]], !"notcold"}
+; THRESH99: ![[C6]] = !{i64 590523745590780990, i64 -2847840206325626610}
+
+; THRESH75: ![[M1]] = !{![[M2:[0-9]+]], ![[M3:[0-9]+]], ![[M4:[0-9]+]], ![[M5:[0-9]+]], ![[M6:[0-9]+]]}
+; THRESH75: ![[M2]] = !{![[C2:[0-9]+]], !"cold"}
+; THRESH75: ![[C2]] = !{i64 590523745590780990, i64 720385627691022109}
+; THRESH75: ![[M3]] = !{![[C3:[0-9]+]], !"cold", ![[S3:[0-9]+]]}
+; THRESH75: ![[C3]] = !{i64 590523745590780990, i64 8256520048276991898}
+; THRESH75: ![[S3]] = !{i64 6509573709067523871, i64 400}
+; THRESH75: ![[M4]] = !{![[C4:[0-9]+]], !"cold"}
+; THRESH75: ![[C4]] = !{i64 590523745590780990, i64 -6953100768213558995}
+;; MIB with size 300 is now included with threshold 75%
+; THRESH75: ![[M5]] = !{![[C5:[0-9]+]], !"cold", ![[S5:[0-9]+]]}
+; THRESH75: ![[C5]] = !{i64 590523745590780990, i64 -6435117705372285425}
+; THRESH75: ![[S5]] = !{i64 86248815258696712, i64 300}
+; THRESH75: ![[M6]] = !{![[C6:[0-9]+]], !"notcold"}
+; THRESH75: ![[C6]] = !{i64 590523745590780990, i64 -2847840206325626610}
+
+; THRESH50: ![[M1]] = !{![[M2:[0-9]+]], ![[M3:[0-9]+]], ![[M4:[0-9]+]], ![[M5:[0-9]+]], ![[M6:[0-9]+]]}
+;; The smallest never gets context size info.
+; THRESH50: ![[M2]] = !{![[C2:[0-9]+]], !"cold"}
+; THRESH50: ![[C2]] = !{i64 590523745590780990, i64 720385627691022109}
+; THRESH50: ![[M3]] = !{![[C3:[0-9]+]], !"cold", ![[S3:[0-9]+]]}
+; THRESH50: ![[C3]] = !{i64 590523745590780990, i64 8256520048276991898}
+; THRESH50: ![[S3]] = !{i64 6509573709067523871, i64 400}
+;; MIB with size 200 is now included with threshold 50%
+; THRESH50: ![[M4]] = !{![[C4:[0-9]+]], !"cold", ![[S4:[0-9]+]]}
+; THRESH50: ![[C4]] = !{i64 590523745590780990, i64 -6953100768213558995}
+; THRESH50: ![[S4]] = !{i64 -2629930016428010893, i64 200}
+; THRESH50: ![[M5]] = !{![[C5:[0-9]+]], !"cold", ![[S5:[0-9]+]]}
+; THRESH50: ![[C5]] = !{i64 590523745590780990, i64 -6435117705372285425}
+; THRESH50: ![[S5]] = !{i64 86248815258696712, i64 300}
+;; Non cold context never gets context size info
+; THRESH50: ![[M6]] = !{![[C6:[0-9]+]], !"notcold"}
+; THRESH50: ![[C6]] = !{i64 590523745590780990, i64 -2847840206325626610}
+
+declare ptr @_Znam(i64)
+
+attributes #0 = { builtin allocsize(0) }
+
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1)
+!1 = !DIFile(filename: "t", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 1, unit: !0)
+!5 = !DILocation(line: 1, column: 22, scope: !4)
+!6 = !DILocation(line: 2, column: 22, scope: !4)
+!7 = !DILocation(line: 3, column: 22, scope: !4)
+!8 = !DILocation(line: 4, column: 22, scope: !4)
diff --git a/external/llvm-project/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll b/external/llvm-project/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
index 57f68e4fc55f..c7098d2ce96c 100644
--- a/external/llvm-project/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
+++ b/external/llvm-project/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
@@ -13,7 +13,7 @@ define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef
 ; CHECK:       [[FOR_BODY_PREHEADER]]:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[RAND_BLOCK_LENGTH]] to i64
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[RAND_BLOCK_LENGTH]], 8
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER23:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER22:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483640
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[Y]], i64 0
@@ -41,10 +41,8 @@ define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef
 ; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt <4 x double> [[TMP7]], zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast <4 x double> [[TMP6]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast <4 x double> [[TMP7]], [[TMP7]]
-; CHECK-NEXT:    [[TMP24:%.*]] = fcmp fast ole <4 x double> [[TMP6]], splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ole <4 x double> [[TMP7]], splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP12:%.*]] = select nnan ninf <4 x i1> [[TMP24]], <4 x double> splat (double -0.000000e+00), <4 x double> [[TMP6]]
-; CHECK-NEXT:    [[TMP13:%.*]] = select nnan ninf <4 x i1> [[TMP25]], <4 x double> splat (double -0.000000e+00), <4 x double> [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[TMP8]], <4 x double> [[TMP6]], <4 x double> splat (double -0.000000e+00)
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[TMP9]], <4 x double> [[TMP7]], <4 x double> splat (double -0.000000e+00)
 ; CHECK-NEXT:    [[TMP14]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI16]], [[TMP12]]
 ; CHECK-NEXT:    [[TMP15]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI17]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[TMP8]], <4 x double> [[TMP10]], <4 x double> splat (double -0.000000e+00)
@@ -60,16 +58,16 @@ define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef
 ; CHECK-NEXT:    [[BIN_RDX21:%.*]] = fadd reassoc arcp contract afn <4 x double> [[TMP15]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = tail call reassoc arcp contract afn double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[BIN_RDX21]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY_PREHEADER23]]
-; CHECK:       [[FOR_BODY_PREHEADER23]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY_PREHEADER22]]
+; CHECK:       [[FOR_BODY_PREHEADER22]]:
 ; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    [[V1_012_PH:%.*]] = phi double [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    [[V0_011_PH:%.*]] = phi double [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[V1_011_PH:%.*]] = phi double [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[V0_010_PH:%.*]] = phi double [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER23]] ]
-; CHECK-NEXT:    [[V1_012:%.*]] = phi double [ [[V1_2:%.*]], %[[FOR_BODY]] ], [ [[V1_012_PH]], %[[FOR_BODY_PREHEADER23]] ]
-; CHECK-NEXT:    [[V0_011:%.*]] = phi double [ [[V0_2:%.*]], %[[FOR_BODY]] ], [ [[V0_011_PH]], %[[FOR_BODY_PREHEADER23]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER22]] ]
+; CHECK-NEXT:    [[V1_012:%.*]] = phi double [ [[V1_2:%.*]], %[[FOR_BODY]] ], [ [[V1_011_PH]], %[[FOR_BODY_PREHEADER22]] ]
+; CHECK-NEXT:    [[V0_011:%.*]] = phi double [ [[V0_2:%.*]], %[[FOR_BODY]] ], [ [[V0_010_PH]], %[[FOR_BODY_PREHEADER22]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP0]] to double
@@ -77,8 +75,7 @@ define nofpclass(nan inf) double @monte_simple(i32 noundef %nblocks, i32 noundef
 ; CHECK-NEXT:    [[SUB:%.*]] = fsub fast double [[MUL]], [[Z]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp fast ogt double [[SUB]], 0.000000e+00
 ; CHECK-NEXT:    [[MUL3:%.*]] = fmul fast double [[SUB]], [[SUB]]
-; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole double [[SUB]], -0.000000e+00
-; CHECK-NEXT:    [[ADD8:%.*]] = select nnan ninf i1 [[DOTINV]], double -0.000000e+00, double [[SUB]]
+; CHECK-NEXT:    [[ADD8:%.*]] = select i1 [[CMP1]], double [[SUB]], double -0.000000e+00
 ; CHECK-NEXT:    [[V0_2]] = fadd reassoc arcp contract afn double [[V0_011]], [[ADD8]]
 ; CHECK-NEXT:    [[ADD4:%.*]] = select i1 [[CMP1]], double [[MUL3]], double -0.000000e+00
 ; CHECK-NEXT:    [[V1_2]] = fadd reassoc arcp contract afn double [[V1_012]], [[ADD4]]
@@ -232,10 +229,8 @@ define nofpclass(nan inf) double @monte_exp(i32 noundef %nblocks, i32 noundef %R
 ; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt <4 x double> [[TMP11]], zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = fmul fast <4 x double> [[TMP10]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = fmul fast <4 x double> [[TMP11]], [[TMP11]]
-; CHECK-NEXT:    [[TMP28:%.*]] = fcmp fast ole <4 x double> [[TMP10]], splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ole <4 x double> [[TMP11]], splat (double -0.000000e+00)
-; CHECK-NEXT:    [[TMP16:%.*]] = select nnan ninf <4 x i1> [[TMP28]], <4 x double> splat (double -0.000000e+00), <4 x double> [[TMP10]]
-; CHECK-NEXT:    [[TMP17:%.*]] = select nnan ninf <4 x i1> [[TMP29]], <4 x double> splat (double -0.000000e+00), <4 x double> [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[TMP12]], <4 x double> [[TMP10]], <4 x double> splat (double -0.000000e+00)
+; CHECK-NEXT:    [[TMP17:%.*]] = select <4 x i1> [[TMP13]], <4 x double> [[TMP11]], <4 x double> splat (double -0.000000e+00)
 ; CHECK-NEXT:    [[TMP18]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI32]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP19]] = fadd reassoc arcp contract afn <4 x double> [[VEC_PHI33]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <4 x i1> [[TMP12]], <4 x double> [[TMP14]], <4 x double> splat (double -0.000000e+00)
@@ -253,13 +248,13 @@ define nofpclass(nan inf) double @monte_exp(i32 noundef %nblocks, i32 noundef %R
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND1_FOR_INC8_CRIT_EDGE_US]], label %[[FOR_BODY3_US_PREHEADER]]
 ; CHECK:       [[FOR_BODY3_US_PREHEADER]]:
 ; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_US]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    [[V1_116_US_PH:%.*]] = phi double [ [[V1_021_US]], %[[FOR_BODY_US]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    [[V0_115_US_PH:%.*]] = phi double [ [[V0_020_US]], %[[FOR_BODY_US]] ], [ [[TMP26]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[V1_114_US_PH:%.*]] = phi double [ [[V1_021_US]], %[[FOR_BODY_US]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[V0_113_US_PH:%.*]] = phi double [ [[V0_020_US]], %[[FOR_BODY_US]] ], [ [[TMP26]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY3_US:.*]]
 ; CHECK:       [[FOR_BODY3_US]]:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY3_US]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
-; CHECK-NEXT:    [[V1_116_US:%.*]] = phi double [ [[V1_2_US:%.*]], %[[FOR_BODY3_US]] ], [ [[V1_116_US_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
-; CHECK-NEXT:    [[V0_115_US:%.*]] = phi double [ [[V0_2_US:%.*]], %[[FOR_BODY3_US]] ], [ [[V0_115_US_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
+; CHECK-NEXT:    [[V1_116_US:%.*]] = phi double [ [[V1_2_US:%.*]], %[[FOR_BODY3_US]] ], [ [[V1_114_US_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
+; CHECK-NEXT:    [[V0_115_US:%.*]] = phi double [ [[V0_2_US:%.*]], %[[FOR_BODY3_US]] ], [ [[V0_113_US_PH]], %[[FOR_BODY3_US_PREHEADER]] ]
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds nuw float, ptr [[SAMPLES]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX_US]], align 4
 ; CHECK-NEXT:    [[CONV_US:%.*]] = fpext float [[TMP0]] to double
@@ -268,8 +263,7 @@ define nofpclass(nan inf) double @monte_exp(i32 noundef %nblocks, i32 noundef %R
 ; CHECK-NEXT:    [[SUB_US:%.*]] = fsub fast double [[MUL_US]], [[Z]]
 ; CHECK-NEXT:    [[CMP4_US:%.*]] = fcmp fast ogt double [[SUB_US]], 0.000000e+00
 ; CHECK-NEXT:    [[ADD7_US:%.*]] = fmul fast double [[SUB_US]], [[SUB_US]]
-; CHECK-NEXT:    [[DOTINV_US:%.*]] = fcmp fast ole double [[SUB_US]], -0.000000e+00
-; CHECK-NEXT:    [[ADD12_US:%.*]] = select nnan ninf i1 [[DOTINV_US]], double -0.000000e+00, double [[SUB_US]]
+; CHECK-NEXT:    [[ADD12_US:%.*]] = select i1 [[CMP4_US]], double [[SUB_US]], double -0.000000e+00
 ; CHECK-NEXT:    [[V0_2_US]] = fadd reassoc arcp contract afn double [[V0_115_US]], [[ADD12_US]]
 ; CHECK-NEXT:    [[ADD7_US1:%.*]] = select i1 [[CMP4_US]], double [[ADD7_US]], double -0.000000e+00
 ; CHECK-NEXT:    [[V1_2_US]] = fadd reassoc arcp contract afn double [[V1_116_US]], [[ADD7_US1]]
diff --git a/external/llvm-project/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll b/external/llvm-project/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
index 254136b0b841..2ec48a8637da 100644
--- a/external/llvm-project/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
+++ b/external/llvm-project/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
@@ -325,3 +325,79 @@ cleanup:
   %retval.0 = phi i1 [ false, %if.then ], [ true, %if.end ]
   ret i1 %retval.0
 }
+
+define i8 @masked_min_reduction(ptr %data, ptr %mask) {
+; CHECK-LABEL: @masked_min_reduction(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <32 x i8> [ splat (i8 -1), [[ENTRY]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <32 x i8> [ splat (i8 -1), [[ENTRY]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <32 x i8> [ splat (i8 -1), [[ENTRY]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <32 x i8> [ splat (i8 -1), [[ENTRY]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[DATA:%.*]] = getelementptr i8, ptr [[DATA1:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[DATA]], i64 32
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[DATA]], i64 64
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[DATA]], i64 96
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[DATA]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <32 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <32 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[MASK:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP7]], i64 32
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP7]], i64 64
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[TMP7]], i64 96
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <32 x i8>, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <32 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD10:%.*]] = load <32 x i8>, ptr [[TMP22]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <32 x i8> [[WIDE_LOAD7]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <32 x i8> [[WIDE_LOAD8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <32 x i8> [[WIDE_LOAD9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq <32 x i8> [[WIDE_LOAD10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP8]], <32 x i8> [[WIDE_LOAD]], <32 x i8> splat (i8 -1)
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP9]], <32 x i8> [[WIDE_LOAD4]], <32 x i8> splat (i8 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = select <32 x i1> [[TMP10]], <32 x i8> [[WIDE_LOAD5]], <32 x i8> splat (i8 -1)
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP11]], <32 x i8> [[WIDE_LOAD6]], <32 x i8> splat (i8 -1)
+; CHECK-NEXT:    [[TMP16]] = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> [[VEC_PHI]], <32 x i8> [[TMP12]])
+; CHECK-NEXT:    [[TMP17]] = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> [[VEC_PHI1]], <32 x i8> [[TMP13]])
+; CHECK-NEXT:    [[TMP18]] = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> [[VEC_PHI2]], <32 x i8> [[TMP14]])
+; CHECK-NEXT:    [[TMP19]] = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> [[VEC_PHI3]], <32 x i8> [[TMP15]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> [[TMP16]], <32 x i8> [[TMP17]])
+; CHECK-NEXT:    [[RDX_MINMAX11:%.*]] = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> [[RDX_MINMAX]], <32 x i8> [[TMP18]])
+; CHECK-NEXT:    [[RDX_MINMAX12:%.*]] = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> [[RDX_MINMAX11]], <32 x i8> [[TMP19]])
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> [[RDX_MINMAX12]])
+; CHECK-NEXT:    ret i8 [[TMP21]]
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i64 [ 0, %entry ], [ %next, %loop ]
+  %acc = phi i8 [ 255, %entry ], [ %acc_next, %loop ]
+
+  %ptr_i = getelementptr i8, ptr %data, i64 %i
+  %val = load i8, ptr %ptr_i, align 1
+
+  %mask_ptr = getelementptr i8, ptr %mask, i64 %i
+  %m = load i8, ptr %mask_ptr, align 1
+  %cond = icmp eq i8 %m, 0
+
+  ; Use select to implement masking
+  %masked_val = select i1 %cond, i8 %val, i8 255
+
+  ; min reduction
+  %acc_next = call i8 @llvm.umin.i8(i8 %acc, i8 %masked_val)
+
+  %next = add i64 %i, 1
+  %cmp = icmp ult i64 %next, 1024
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i8 %acc_next
+}
diff --git a/external/llvm-project/llvm/test/Transforms/SCCP/conditions-ranges-with-undef.ll b/external/llvm-project/llvm/test/Transforms/SCCP/conditions-ranges-with-undef.ll
index 8b4dea462757..b49a40abaaeb 100644
--- a/external/llvm-project/llvm/test/Transforms/SCCP/conditions-ranges-with-undef.ll
+++ b/external/llvm-project/llvm/test/Transforms/SCCP/conditions-ranges-with-undef.ll
@@ -36,8 +36,6 @@ false:
 
 declare void @use.i32(i32)
 
-; It is not allowed to use the range information from the condition to remove
-; %a.127 = and ... in the true block, as %a could be undef.
 define void @val_undef_range() {
 ; CHECK-LABEL: @val_undef_range(
 ; CHECK-NEXT:  entry:
@@ -46,8 +44,7 @@ define void @val_undef_range() {
 ; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @use(i1 false)
-; CHECK-NEXT:    [[A_127:%.*]] = and i32 [[A]], 127
-; CHECK-NEXT:    call void @use.i32(i32 [[A_127]])
+; CHECK-NEXT:    call void @use.i32(i32 [[A]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       false:
 ; CHECK-NEXT:    ret void
@@ -82,8 +79,7 @@ define void @val_singlecrfromundef_range(i1 %cond) {
 ; CHECK-NEXT:    br label [[TRUE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @use(i1 false)
-; CHECK-NEXT:    [[P_127:%.*]] = and i32 10, 127
-; CHECK-NEXT:    call void @use.i32(i32 [[P_127]])
+; CHECK-NEXT:    call void @use.i32(i32 10)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -113,9 +109,6 @@ false:
   ret void
 }
 
-
-; It is not allowed to use the information from the condition ([0, 128))
-; to remove a.127.2 = and i32 %p, 127, as %p might be undef.
 define void @val_undef_to_cr_to_overdef_range(i32 %a, i1 %cond) {
 ; CHECK-LABEL: @val_undef_to_cr_to_overdef_range(
 ; CHECK-NEXT:  entry:
@@ -131,8 +124,7 @@ define void @val_undef_to_cr_to_overdef_range(i32 %a, i1 %cond) {
 ; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    call void @use(i1 false)
-; CHECK-NEXT:    [[P_127:%.*]] = and i32 [[P]], 127
-; CHECK-NEXT:    call void @use.i32(i32 [[P_127]])
+; CHECK-NEXT:    call void @use.i32(i32 [[P]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       false:
 ; CHECK-NEXT:    ret void
@@ -164,6 +156,43 @@ false:
   ret void
 }
 
+; It is not allowed to use the range information from the condition to remove
+; %p.127 = and i32 %p, 127, as %p could be undef.
+define void @masked_incoming_val_with_undef(i32 %a, i1 %cond) {
+; CHECK-LABEL: @masked_incoming_val_with_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_127:%.*]] = and i32 [[A:%.*]], 127
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[INC1:%.*]], label [[INC2:%.*]]
+; CHECK:       inc1:
+; CHECK-NEXT:    br label [[IF:%.*]]
+; CHECK:       inc2:
+; CHECK-NEXT:    br label [[IF]]
+; CHECK:       if:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[A_127]], [[INC1]] ], [ undef, [[INC2]] ]
+; CHECK-NEXT:    [[P_127:%.*]] = and i32 [[P]], 127
+; CHECK-NEXT:    call void @use.i32(i32 [[P_127]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.127 = and i32 %a, 127
+  br i1 %cond, label %inc1, label %inc2
+
+inc1:
+  br label %if
+
+inc2:
+  br label %if
+
+if:
+  %p = phi i32 [ %a.127, %inc1 ], [ undef, %inc2 ]
+  %p.127 = and i32 %p, 127
+  call void @use.i32(i32 %p.127)
+  ret void
+
+false:
+  ret void
+}
+
 ; All uses of %p can be replaced by a constant (10), we are allowed to use it
 ; as a bound too.
 define void @bound_singlecrfromundef(i32 %a, i1 %cond) {
diff --git a/external/llvm-project/llvm/test/Transforms/SCCP/range-and.ll b/external/llvm-project/llvm/test/Transforms/SCCP/range-and.ll
index ef8758fcba9e..f3452f45e1c0 100644
--- a/external/llvm-project/llvm/test/Transforms/SCCP/range-and.ll
+++ b/external/llvm-project/llvm/test/Transforms/SCCP/range-and.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --verbose
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=sccp %s | FileCheck %s
 
 declare void @use(i1)
@@ -140,9 +140,8 @@ define i64 @constant_range_and_255_100(i1 %cond, i64 %a) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[R_1]], [[BB1]] ], [ [[R_2]], [[BB2]] ]
-; CHECK-NEXT:    [[P_AND:%.*]] = and i64 [[P]], 255
 ; CHECK-NEXT:    call void @use(i1 true)
-; CHECK-NEXT:    ret i64 [[P_AND]]
+; CHECK-NEXT:    ret i64 [[P]]
 ;
 entry:
   br i1 %cond, label %bb1, label %bb2
diff --git a/external/llvm-project/llvm/test/Transforms/SCCP/range-with-undef.ll b/external/llvm-project/llvm/test/Transforms/SCCP/range-with-undef.ll
index 9b8d41517114..774d95ceff42 100644
--- a/external/llvm-project/llvm/test/Transforms/SCCP/range-with-undef.ll
+++ b/external/llvm-project/llvm/test/Transforms/SCCP/range-with-undef.ll
@@ -12,8 +12,7 @@ define i8 @test_binop(i1 %cond, i8 %a) {
 ; CHECK-NEXT:    [[A_EXT:%.*]] = zext i8 [[A]] to i16
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i16 [ undef, %[[ENTRY]] ], [ [[A_EXT]], %[[IF]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i16 [[PHI]], -1
+; CHECK-NEXT:    [[AND:%.*]] = phi i16 [ undef, %[[ENTRY]] ], [ [[A_EXT]], %[[IF]] ]
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i16 [[AND]] to i8
 ; CHECK-NEXT:    ret i8 [[TRUNC]]
 ;
diff --git a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
index 2cd29c524118..c48dac19368f 100644
--- a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
+++ b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
@@ -264,11 +264,8 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:  bb:
 ; GFX8-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX8-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+; GFX8-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -276,11 +273,8 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+; GFX9-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -323,24 +317,20 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX8-LABEL: @uadd_sat_v4i16(
 ; GFX8-NEXT:  bb:
-; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
+; GFX8-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
 ; GFX8-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP3]], <2 x i16> [[TMP4]])
-; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX8-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
+; GFX9-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
 ; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP3]], <2 x i16> [[TMP4]])
-; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 bb:
diff --git a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
index 4653e47f4a5f..9da610b48f74 100644
--- a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
+++ b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
@@ -264,11 +264,8 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:  bb:
 ; GFX8-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX8-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+; GFX8-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -276,11 +273,8 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+; GFX9-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -323,24 +317,20 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX8-LABEL: @uadd_sat_v4i16(
 ; GFX8-NEXT:  bb:
-; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
+; GFX8-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
 ; GFX8-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP3]], <2 x i16> [[TMP4]])
-; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX8-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
+; GFX9-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
 ; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP3]], <2 x i16> [[TMP4]])
-; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 bb:
diff --git a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/external-shuffle.ll b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/external-shuffle.ll
new file mode 100644
index 000000000000..ce9e47a03dee
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/AMDGPU/external-shuffle.ll
@@ -0,0 +1,261 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer,dce < %s | FileCheck -check-prefixes=GCN %s
+
+; The insertelements in the exit block use the various parts of the vectorized tree. These external uses are just creating an identity vector using a sequence
+;  of insert elements. Since these insertelements are just recreating the same vectors that were produced during vectorization, they should not increase the cost of vectorization.
+
+define void @phi_4(ptr addrspace(3) %inptr0, ptr addrspace(3) %inptr1, ptr %out, ptr %out1, ptr %out2, i32 %flag) {
+; GCN-LABEL: define void @phi_4(
+; GCN-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], ptr [[OUT2:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0:[0-9]+]] {
+; GCN-NEXT:  [[ENTRY:.*]]:
+; GCN-NEXT:    [[TMP0:%.*]] = load <2 x i16>, ptr addrspace(3) [[INPTR0]], align 8
+; GCN-NEXT:    [[GEP2:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 2
+; GCN-NEXT:    [[GEP3:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 3
+; GCN-NEXT:    [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP2]], align 2
+; GCN-NEXT:    [[GEP4:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 4
+; GCN-NEXT:    [[GEP5:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 5
+; GCN-NEXT:    [[TMP2:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP4]], align 8
+; GCN-NEXT:    [[GEP6:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 6
+; GCN-NEXT:    [[GEP7:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 7
+; GCN-NEXT:    [[TMP3:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP6]], align 2
+; GCN-NEXT:    [[GEP8:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 8
+; GCN-NEXT:    [[GEP9:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 9
+; GCN-NEXT:    [[TMP4:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP8]], align 8
+; GCN-NEXT:    [[GEP10:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 10
+; GCN-NEXT:    [[GEP11:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 11
+; GCN-NEXT:    [[TMP5:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP10]], align 2
+; GCN-NEXT:    [[GEP12:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 12
+; GCN-NEXT:    [[GEP13:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 13
+; GCN-NEXT:    [[TMP6:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP12]], align 8
+; GCN-NEXT:    [[GEP14:%.*]] = getelementptr i16, ptr addrspace(3) [[INPTR0]], i32 14
+; GCN-NEXT:    [[TMP7:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP14]], align 2
+; GCN-NEXT:    [[TMP8:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0
+; GCN-NEXT:    [[TMP9:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1
+; GCN-NEXT:    [[TMP10:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0
+; GCN-NEXT:    [[TMP11:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1
+; GCN-NEXT:    [[TMP12:%.*]] = extractelement <2 x i16> [[TMP3]], i32 0
+; GCN-NEXT:    [[TMP13:%.*]] = extractelement <2 x i16> [[TMP3]], i32 1
+; GCN-NEXT:    [[TMP14:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0
+; GCN-NEXT:    [[TMP15:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1
+; GCN-NEXT:    [[TMP24:%.*]] = extractelement <2 x i16> [[TMP5]], i32 0
+; GCN-NEXT:    [[TMP26:%.*]] = extractelement <2 x i16> [[TMP5]], i32 1
+; GCN-NEXT:    [[TMP28:%.*]] = extractelement <2 x i16> [[TMP6]], i32 0
+; GCN-NEXT:    [[TMP38:%.*]] = extractelement <2 x i16> [[TMP6]], i32 1
+; GCN-NEXT:    br label %[[DO_BODY:.*]]
+; GCN:       [[DO_BODY]]:
+; GCN-NEXT:    [[PHI2:%.*]] = phi i16 [ [[TMP8]], %[[ENTRY]] ], [ [[TMP30:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI3:%.*]] = phi i16 [ [[TMP9]], %[[ENTRY]] ], [ [[OTHERELE3:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI4:%.*]] = phi i16 [ [[TMP10]], %[[ENTRY]] ], [ [[TMP39:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI5:%.*]] = phi i16 [ [[TMP11]], %[[ENTRY]] ], [ [[OTHERELE5:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI6:%.*]] = phi i16 [ [[TMP12]], %[[ENTRY]] ], [ [[TMP32:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI7:%.*]] = phi i16 [ [[TMP13]], %[[ENTRY]] ], [ [[OTHERELE7:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI8:%.*]] = phi i16 [ [[TMP14]], %[[ENTRY]] ], [ [[TMP40:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI9:%.*]] = phi i16 [ [[TMP15]], %[[ENTRY]] ], [ [[OTHERELE9:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI10:%.*]] = phi i16 [ [[TMP24]], %[[ENTRY]] ], [ [[TMP34:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI11:%.*]] = phi i16 [ [[TMP26]], %[[ENTRY]] ], [ [[OTHERELE11:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI12:%.*]] = phi i16 [ [[TMP28]], %[[ENTRY]] ], [ [[TMP35:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[PHI13:%.*]] = phi i16 [ [[TMP38]], %[[ENTRY]] ], [ [[OTHERELE13:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[TMP41:%.*]] = phi <2 x i16> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP16:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[TMP42:%.*]] = phi <2 x i16> [ [[TMP7]], %[[ENTRY]] ], [ [[TMP23:%.*]], %[[DO_BODY]] ]
+; GCN-NEXT:    [[TMP16]] = load <2 x i16>, ptr addrspace(3) [[INPTR0]], align 8
+; GCN-NEXT:    [[OTHERELE3]] = load i16, ptr addrspace(3) [[GEP3]], align 1
+; GCN-NEXT:    [[TMP17:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP2]], align 2
+; GCN-NEXT:    [[OTHERELE5]] = load i16, ptr addrspace(3) [[GEP5]], align 1
+; GCN-NEXT:    [[TMP18:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP4]], align 8
+; GCN-NEXT:    [[OTHERELE7]] = load i16, ptr addrspace(3) [[GEP7]], align 1
+; GCN-NEXT:    [[TMP19:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP6]], align 2
+; GCN-NEXT:    [[OTHERELE9]] = load i16, ptr addrspace(3) [[GEP9]], align 1
+; GCN-NEXT:    [[TMP20:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP8]], align 8
+; GCN-NEXT:    [[OTHERELE11]] = load i16, ptr addrspace(3) [[GEP11]], align 1
+; GCN-NEXT:    [[TMP21:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP10]], align 2
+; GCN-NEXT:    [[OTHERELE13]] = load i16, ptr addrspace(3) [[GEP13]], align 1
+; GCN-NEXT:    [[TMP22:%.*]] = load <2 x i16>, ptr addrspace(3) [[GEP12]], align 8
+; GCN-NEXT:    [[TMP23]] = load <2 x i16>, ptr addrspace(3) [[GEP14]], align 2
+; GCN-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GCN-NEXT:    [[TMP30]] = extractelement <2 x i16> [[TMP17]], i32 0
+; GCN-NEXT:    [[TMP39]] = extractelement <2 x i16> [[TMP18]], i32 0
+; GCN-NEXT:    [[TMP32]] = extractelement <2 x i16> [[TMP19]], i32 0
+; GCN-NEXT:    [[TMP40]] = extractelement <2 x i16> [[TMP20]], i32 0
+; GCN-NEXT:    [[TMP34]] = extractelement <2 x i16> [[TMP21]], i32 0
+; GCN-NEXT:    [[TMP35]] = extractelement <2 x i16> [[TMP22]], i32 0
+; GCN-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GCN:       [[EXIT]]:
+; GCN-NEXT:    [[TMP36:%.*]] = shufflevector <2 x i16> [[TMP16]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[TMP37:%.*]] = shufflevector <2 x i16> [[TMP17]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC038:%.*]] = shufflevector <16 x i16> [[TMP36]], <16 x i16> [[TMP37]], <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP25:%.*]] = shufflevector <2 x i16> [[TMP18]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC059:%.*]] = shufflevector <16 x i16> [[VEC038]], <16 x i16> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP27:%.*]] = shufflevector <2 x i16> [[TMP19]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC0710:%.*]] = shufflevector <16 x i16> [[VEC059]], <16 x i16> [[TMP27]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP29:%.*]] = shufflevector <2 x i16> [[TMP20]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC0911:%.*]] = shufflevector <16 x i16> [[VEC0710]], <16 x i16> [[TMP29]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP31:%.*]] = shufflevector <2 x i16> [[TMP21]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC01112:%.*]] = shufflevector <16 x i16> [[VEC0911]], <16 x i16> [[TMP31]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP33:%.*]] = shufflevector <2 x i16> [[TMP22]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[TMP58:%.*]] = shufflevector <16 x i16> [[VEC01112]], <16 x i16> [[TMP33]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
+; GCN-NEXT:    [[TMP60:%.*]] = shufflevector <2 x i16> [[TMP23]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC2157:%.*]] = shufflevector <16 x i16> [[TMP58]], <16 x i16> [[TMP60]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; GCN-NEXT:    [[TMP50:%.*]] = shufflevector <2 x i16> [[TMP0]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[TMP51:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC231:%.*]] = shufflevector <16 x i16> [[TMP50]], <16 x i16> [[TMP51]], <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP52:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC252:%.*]] = shufflevector <16 x i16> [[VEC231]], <16 x i16> [[TMP52]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP53:%.*]] = shufflevector <2 x i16> [[TMP3]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC273:%.*]] = shufflevector <16 x i16> [[VEC252]], <16 x i16> [[TMP53]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP54:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC294:%.*]] = shufflevector <16 x i16> [[VEC273]], <16 x i16> [[TMP54]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP55:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC2115:%.*]] = shufflevector <16 x i16> [[VEC294]], <16 x i16> [[TMP55]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
+; GCN-NEXT:    [[TMP56:%.*]] = shufflevector <2 x i16> [[TMP6]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC2136:%.*]] = shufflevector <16 x i16> [[VEC2115]], <16 x i16> [[TMP56]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
+; GCN-NEXT:    [[TMP59:%.*]] = shufflevector <2 x i16> [[TMP7]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC2151:%.*]] = shufflevector <16 x i16> [[VEC2136]], <16 x i16> [[TMP59]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; GCN-NEXT:    [[TMP57:%.*]] = shufflevector <2 x i16> [[TMP41]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC22:%.*]] = insertelement <16 x i16> [[TMP57]], i16 [[PHI2]], i64 2
+; GCN-NEXT:    [[VEC23:%.*]] = insertelement <16 x i16> [[VEC22]], i16 [[PHI3]], i64 3
+; GCN-NEXT:    [[VEC24:%.*]] = insertelement <16 x i16> [[VEC23]], i16 [[PHI4]], i64 4
+; GCN-NEXT:    [[VEC25:%.*]] = insertelement <16 x i16> [[VEC24]], i16 [[PHI5]], i64 5
+; GCN-NEXT:    [[VEC26:%.*]] = insertelement <16 x i16> [[VEC25]], i16 [[PHI6]], i64 6
+; GCN-NEXT:    [[VEC27:%.*]] = insertelement <16 x i16> [[VEC26]], i16 [[PHI7]], i64 7
+; GCN-NEXT:    [[VEC28:%.*]] = insertelement <16 x i16> [[VEC27]], i16 [[PHI8]], i64 8
+; GCN-NEXT:    [[VEC29:%.*]] = insertelement <16 x i16> [[VEC28]], i16 [[PHI9]], i64 9
+; GCN-NEXT:    [[VEC210:%.*]] = insertelement <16 x i16> [[VEC29]], i16 [[PHI10]], i64 10
+; GCN-NEXT:    [[VEC211:%.*]] = insertelement <16 x i16> [[VEC210]], i16 [[PHI11]], i64 11
+; GCN-NEXT:    [[VEC212:%.*]] = insertelement <16 x i16> [[VEC211]], i16 [[PHI12]], i64 12
+; GCN-NEXT:    [[VEC213:%.*]] = insertelement <16 x i16> [[VEC212]], i16 [[PHI13]], i64 13
+; GCN-NEXT:    [[TMP61:%.*]] = shufflevector <2 x i16> [[TMP42]], <2 x i16> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GCN-NEXT:    [[VEC2152:%.*]] = shufflevector <16 x i16> [[VEC213]], <16 x i16> [[TMP61]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; GCN-NEXT:    store <16 x i16> [[VEC2151]], ptr [[OUT]], align 32
+; GCN-NEXT:    store <16 x i16> [[VEC2157]], ptr [[OUT1]], align 32
+; GCN-NEXT:    store <16 x i16> [[VEC2152]], ptr [[OUT2]], align 32
+; GCN-NEXT:    ret void
+;
+entry:
+  %ele0 = load i16, ptr addrspace(3) %inptr0, align 8
+  %gep1 = getelementptr i16, ptr addrspace(3) %inptr0, i32 1
+  %ele1 = load i16, ptr addrspace(3) %gep1, align 1
+  %gep2 = getelementptr i16, ptr addrspace(3) %inptr0, i32 2
+  %ele2 = load i16, ptr addrspace(3) %gep2, align 2
+  %gep3 = getelementptr i16, ptr addrspace(3) %inptr0, i32 3
+  %ele3 = load i16, ptr addrspace(3) %gep3, align 1
+  %gep4 = getelementptr i16, ptr addrspace(3) %inptr0, i32 4
+  %ele4 = load i16, ptr addrspace(3) %gep4, align 8
+  %gep5 = getelementptr i16, ptr addrspace(3) %inptr0, i32 5
+  %ele5 = load i16, ptr addrspace(3) %gep5, align 1
+  %gep6 = getelementptr i16, ptr addrspace(3) %inptr0, i32 6
+  %ele6 = load i16, ptr addrspace(3) %gep6, align 2
+  %gep7 = getelementptr i16, ptr addrspace(3) %inptr0, i32 7
+  %ele7 = load i16, ptr addrspace(3) %gep7, align 1
+  %gep8 = getelementptr i16, ptr addrspace(3) %inptr0, i32 8
+  %ele8 = load i16, ptr addrspace(3) %gep8, align 8
+  %gep9 = getelementptr i16, ptr addrspace(3) %inptr0, i32 9
+  %ele9 = load i16, ptr addrspace(3) %gep9, align 1
+  %gep10 = getelementptr i16, ptr addrspace(3) %inptr0, i32 10
+  %ele10 = load i16, ptr addrspace(3) %gep10, align 2
+  %gep11 = getelementptr i16, ptr addrspace(3) %inptr0, i32 11
+  %ele11 = load i16, ptr addrspace(3) %gep11, align 1
+  %gep12 = getelementptr i16, ptr addrspace(3) %inptr0, i32 12
+  %ele12 = load i16, ptr addrspace(3) %gep12, align 8
+  %gep13 = getelementptr i16, ptr addrspace(3) %inptr0, i32 13
+  %ele13 = load i16, ptr addrspace(3) %gep13, align 1
+  %gep14 = getelementptr i16, ptr addrspace(3) %inptr0, i32 14
+  %ele14 = load i16, ptr addrspace(3) %gep14, align 2
+  %gep15 = getelementptr i16, ptr addrspace(3) %inptr0, i32 15
+  %ele15 = load i16, ptr addrspace(3) %gep15, align 1
+  br label %do.body
+
+do.body:
+  %phi0 = phi i16 [ %ele0, %entry ], [ %otherele0, %do.body ]
+  %phi1 = phi i16 [ %ele1, %entry ], [ %otherele1, %do.body ]
+  %phi2 = phi i16 [ %ele2, %entry ], [ %otherele2, %do.body ]
+  %phi3 = phi i16 [ %ele3, %entry ], [ %otherele3, %do.body ]
+  %phi4 = phi i16 [ %ele4, %entry ], [ %otherele4, %do.body ]
+  %phi5 = phi i16 [ %ele5, %entry ], [ %otherele5, %do.body ]
+  %phi6 = phi i16 [ %ele6, %entry ], [ %otherele6, %do.body ]
+  %phi7 = phi i16 [ %ele7, %entry ], [ %otherele7, %do.body ]
+  %phi8 = phi i16 [ %ele8, %entry ], [ %otherele8, %do.body ]
+  %phi9 = phi i16 [ %ele9, %entry ], [ %otherele9, %do.body ]
+  %phi10 = phi i16 [ %ele10, %entry ], [ %otherele10, %do.body ]
+  %phi11 = phi i16 [ %ele11, %entry ], [ %otherele11, %do.body ]
+  %phi12 = phi i16 [ %ele12, %entry ], [ %otherele12, %do.body ]
+  %phi13 = phi i16 [ %ele13, %entry ], [ %otherele13, %do.body ]
+  %phi14 = phi i16 [ %ele14, %entry ], [ %otherele14, %do.body ]
+  %phi15 = phi i16 [ %ele15, %entry ], [ %otherele15, %do.body ]
+
+  %otherele0 = load i16, ptr addrspace(3) %inptr0, align 8
+  %otherele1 = load i16, ptr addrspace(3) %gep1, align 1
+  %otherele2 = load i16, ptr addrspace(3) %gep2, align 2
+  %otherele3 = load i16, ptr addrspace(3) %gep3, align 1
+  %otherele4 = load i16, ptr addrspace(3) %gep4, align 8
+  %otherele5 = load i16, ptr addrspace(3) %gep5, align 1
+  %otherele6 = load i16, ptr addrspace(3) %gep6, align 2
+  %otherele7 = load i16, ptr addrspace(3) %gep7, align 1
+  %otherele8 = load i16, ptr addrspace(3) %gep8, align 8
+  %otherele9 = load i16, ptr addrspace(3) %gep9, align 1
+  %otherele10 = load i16, ptr addrspace(3) %gep10, align 2
+  %otherele11 = load i16, ptr addrspace(3) %gep11, align 1
+  %otherele12 = load i16, ptr addrspace(3) %gep12, align 8
+  %otherele13 = load i16, ptr addrspace(3) %gep13, align 1
+  %otherele14 = load i16, ptr addrspace(3) %gep14, align 2
+  %otherele15 = load i16, ptr addrspace(3) %gep15, align 1
+  %cmp = icmp eq i32 %flag, 0
+  br i1 %cmp, label %exit, label %do.body
+
+exit:
+  %vec00 = insertelement <16 x i16> poison, i16 %otherele0, i64 0
+  %vec01 = insertelement <16 x i16> %vec00, i16 %otherele1, i64 1
+  %vec02 = insertelement <16 x i16> %vec01, i16 %otherele2, i64 2
+  %vec03 = insertelement <16 x i16> %vec02, i16 %otherele3, i64 3
+  %vec04 = insertelement <16 x i16> %vec03, i16 %otherele4, i64 4
+  %vec05 = insertelement <16 x i16> %vec04, i16 %otherele5, i64 5
+  %vec06 = insertelement <16 x i16> %vec05, i16 %otherele6, i64 6
+  %vec07 = insertelement <16 x i16> %vec06, i16 %otherele7, i64 7
+  %vec08 = insertelement <16 x i16> %vec07, i16 %otherele8, i64 8
+  %vec09 = insertelement <16 x i16> %vec08, i16 %otherele9, i64 9
+  %vec010 = insertelement <16 x i16> %vec09, i16 %otherele10, i64 10
+  %vec011 = insertelement <16 x i16> %vec010, i16 %otherele11, i64 11
+  %vec012 = insertelement <16 x i16> %vec011, i16 %otherele12, i64 12
+  %vec013 = insertelement <16 x i16> %vec012, i16 %otherele13, i64 13
+  %vec014 = insertelement <16 x i16> %vec013, i16 %otherele14, i64 14
+  %vec015 = insertelement <16 x i16> %vec014, i16 %otherele15, i64 15
+
+  %vec10 = insertelement <16 x i16> poison, i16 %ele0, i64 0
+  %vec11 = insertelement <16 x i16> %vec10, i16 %ele1, i64 1
+  %vec12 = insertelement <16 x i16> %vec11, i16 %ele2, i64 2
+  %vec13 = insertelement <16 x i16> %vec12, i16 %ele3, i64 3
+  %vec14 = insertelement <16 x i16> %vec13, i16 %ele4, i64 4
+  %vec15 = insertelement <16 x i16> %vec14, i16 %ele5, i64 5
+  %vec16 = insertelement <16 x i16> %vec15, i16 %ele6, i64 6
+  %vec17 = insertelement <16 x i16> %vec16, i16 %ele7, i64 7
+  %vec18 = insertelement <16 x i16> %vec17, i16 %ele8, i64 8
+  %vec19 = insertelement <16 x i16> %vec18, i16 %ele9, i64 9
+  %vec110 = insertelement <16 x i16> %vec19, i16 %ele10, i64 10
+  %vec111 = insertelement <16 x i16> %vec110, i16 %ele11, i64 11
+  %vec112 = insertelement <16 x i16> %vec111, i16 %ele12, i64 12
+  %vec113 = insertelement <16 x i16> %vec112, i16 %ele13, i64 13
+  %vec114 = insertelement <16 x i16> %vec113, i16 %ele14, i64 14
+  %vec115 = insertelement <16 x i16> %vec114, i16 %ele15, i64 15
+
+  %vec20 = insertelement <16 x i16> poison, i16 %phi0, i64 0
+  %vec21 = insertelement <16 x i16> %vec20, i16 %phi1, i64 1
+  %vec22 = insertelement <16 x i16> %vec21, i16 %phi2, i64 2
+  %vec23 = insertelement <16 x i16> %vec22, i16 %phi3, i64 3
+  %vec24 = insertelement <16 x i16> %vec23, i16 %phi4, i64 4
+  %vec25 = insertelement <16 x i16> %vec24, i16 %phi5, i64 5
+  %vec26 = insertelement <16 x i16> %vec25, i16 %phi6, i64 6
+  %vec27 = insertelement <16 x i16> %vec26, i16 %phi7, i64 7
+  %vec28 = insertelement <16 x i16> %vec27, i16 %phi8, i64 8
+  %vec29 = insertelement <16 x i16> %vec28, i16 %phi9, i64 9
+  %vec210 = insertelement <16 x i16> %vec29, i16 %phi10, i64 10
+  %vec211 = insertelement <16 x i16> %vec210, i16 %phi11, i64 11
+  %vec212 = insertelement <16 x i16> %vec211, i16 %phi12, i64 12
+  %vec213 = insertelement <16 x i16> %vec212, i16 %phi13, i64 13
+  %vec214 = insertelement <16 x i16> %vec213, i16 %phi14, i64 14
+  %vec215 = insertelement <16 x i16> %vec214, i16 %phi15, i64 15
+
+  store <16 x i16> %vec115, ptr %out
+  store <16 x i16> %vec015, ptr %out1
+  store <16 x i16> %vec215, ptr %out2
+
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
index b790e6f3c99c..77d36f010766 100644
--- a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
+++ b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
@@ -38,13 +38,13 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
 ; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
 ; SLM-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; SLM-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
+; SLM-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
 ; SLM-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SLM-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SLM-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
 ; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -59,13 +59,13 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
 ; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
 ; AVX-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; AVX-NEXT:    [[TMP9:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP8]])
+; AVX-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
 ; AVX-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP1]])
-; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP3]])
+; AVX-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
 ; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -80,13 +80,13 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
 ; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
 ; AVX2-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; AVX2-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
+; AVX2-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
 ; AVX2-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX2-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
 ; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
index ef1a67032c23..18d79752b0b4 100644
--- a/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
+++ b/external/llvm-project/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -38,13 +38,13 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
 ; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
 ; SLM-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; SLM-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
+; SLM-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
 ; SLM-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SLM-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SLM-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
 ; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -59,13 +59,13 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
 ; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
 ; AVX-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; AVX-NEXT:    [[TMP9:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP8]])
+; AVX-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
 ; AVX-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP1]])
-; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP3]])
+; AVX-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
 ; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -80,13 +80,13 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
 ; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
 ; AVX2-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
-; AVX2-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
+; AVX2-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
 ; AVX2-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX2-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
+; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
 ; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/external/llvm-project/llvm/test/Transforms/SROA/heterogeneous-poison.ll b/external/llvm-project/llvm/test/Transforms/SROA/heterogeneous-poison.ll
index 7ef813a3e2ee..d8aa9d7a4b51 100644
--- a/external/llvm-project/llvm/test/Transforms/SROA/heterogeneous-poison.ll
+++ b/external/llvm-project/llvm/test/Transforms/SROA/heterogeneous-poison.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes='sroa<preserve-cfg>' -S --experimental-debuginfo-iterators=true  < %s | FileCheck %s
-; RUN: opt -passes='sroa<preserve-cfg>' -S --experimental-debuginfo-iterators=false < %s | FileCheck %s
+; RUN: opt -passes='sroa<preserve-cfg>' -S  < %s | FileCheck %s
 
 source_filename = "test/Transforms/SROA/heterogeneous-poison.ll"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
@@ -8,6 +7,11 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:
 %struct.pair = type { i32, i32 }
 
 define i32 @t1() !dbg !9 {
+; CHECK-LABEL: define i32 @t1(
+; CHECK-SAME: ) !dbg [[DBG9:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(i32 2, [[META13:![0-9]+]], !DIExpression(DIOpArg(0, i32)), [[META15:![0-9]+]])
+; CHECK-NEXT:    ret i32 2
+;
   %local = alloca i32, align 4
   tail call void @llvm.dbg.declare(metadata ptr %local, metadata !13, metadata !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(i32))), !dbg !15
   store i32 2, ptr %local, align 4
@@ -16,6 +20,20 @@ define i32 @t1() !dbg !9 {
 }
 
 define i32 @t2(i1 %cond) !dbg !16 {
+; CHECK-LABEL: define i32 @t2(
+; CHECK-SAME: i1 [[COND:%.*]]) !dbg [[DBG16:![0-9]+]] {
+; CHECK-NEXT:    br i1 [[COND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:      #dbg_value(i32 42, [[META17:![0-9]+]], !DIExpression(DIOpArg(0, i32)), [[META18:![0-9]+]])
+; CHECK-NEXT:    br label [[JOIN:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:      #dbg_value(i32 2, [[META17]], !DIExpression(DIOpArg(0, i32)), [[META18]])
+; CHECK-NEXT:    br label [[JOIN]]
+; CHECK:       join:
+; CHECK-NEXT:    [[LOCAL_0:%.*]] = phi i32 [ 42, [[THEN]] ], [ 2, [[ELSE]] ]
+; CHECK-NEXT:      #dbg_value(i32 [[LOCAL_0]], [[META17]], !DIExpression(DIOpArg(0, i32)), [[META18]])
+; CHECK-NEXT:    ret i32 [[LOCAL_0]]
+;
   %local = alloca i32, align 4
   tail call void @llvm.dbg.declare(metadata ptr %local, metadata !17, metadata !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(i32))), !dbg !18
   br i1 %cond, label %then, label %else
@@ -34,6 +52,12 @@ join:                                             ; preds = %else, %then
 }
 
 define void @t3() !dbg !19 {
+; CHECK-LABEL: define void @t3(
+; CHECK-SAME: ) !dbg [[DBG19:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(i32 42, [[META20:![0-9]+]], !DIExpression(DIOpArg(0, i32), DIOpFragment(0, 32)), [[META25:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(i32 43, [[META20]], !DIExpression(DIOpArg(0, i32), DIOpFragment(32, 32)), [[META25]])
+; CHECK-NEXT:    ret void
+;
   %local = alloca %struct.pair, align 4
   tail call void @llvm.dbg.declare(metadata ptr %local, metadata !20, metadata !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(%struct.pair))), !dbg !25
   %first = getelementptr inbounds %struct.pair, ptr %local, i32 0, i32 0
@@ -44,9 +68,13 @@ define void @t3() !dbg !19 {
 }
 
 define i32 @t4() !dbg !26 {
-
   ;; FIXME(diexpression-poison): We could probably preserve debug info for the dbg.value here if
   ;; necessary. Check that we at least do something sensible with it for now.
+; CHECK-LABEL: define i32 @t4(
+; CHECK-SAME: ) !dbg [[DBG26:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(ptr poison, [[META27:![0-9]+]], !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(i32)), [[META28:![0-9]+]])
+; CHECK-NEXT:    ret i32 42
+;
   %local = alloca i32, align 4
   tail call void @llvm.dbg.value(metadata ptr %local, metadata !27, metadata !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(i32))), !dbg !28
   store i32 42, ptr %local, align 4
@@ -55,11 +83,23 @@ define i32 @t4() !dbg !26 {
 }
 
 define i16 @t5(i1 %cond) !dbg !29 {
-
   ;; Verify that we still convert if the new value doesn't cover the entire size
   ;; of the variable !30. This is something that old-style DIExpressions don't
   ;; support.
-
+; CHECK-LABEL: define i16 @t5(
+; CHECK-SAME: i1 [[COND:%.*]]) !dbg [[DBG29:![0-9]+]] {
+; CHECK-NEXT:    br i1 [[COND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:      #dbg_value(i16 42, [[META30:![0-9]+]], !DIExpression(DIOpArg(0, i16), DIOpSExt(i32)), [[META31:![0-9]+]])
+; CHECK-NEXT:    br label [[JOIN:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:      #dbg_value(i16 43, [[META30]], !DIExpression(DIOpArg(0, i16), DIOpSExt(i32)), [[META31]])
+; CHECK-NEXT:    br label [[JOIN]]
+; CHECK:       join:
+; CHECK-NEXT:    [[LOCAL_0:%.*]] = phi i16 [ 42, [[THEN]] ], [ 43, [[ELSE]] ]
+; CHECK-NEXT:      #dbg_value(i16 [[LOCAL_0]], [[META30]], !DIExpression(DIOpArg(0, i16), DIOpSExt(i32)), [[META31]])
+; CHECK-NEXT:    ret i16 [[LOCAL_0]]
+;
   %local = alloca i16, align 4
   tail call void @llvm.dbg.declare(metadata ptr %local, metadata !30, metadata !DIExpression(DIOpArg(0, ptr addrspace(5)), DIOpDeref(i16), DIOpSExt(i32))), !dbg !31
   br i1 %cond, label %then, label %else
@@ -80,7 +120,14 @@ join:                                             ; preds = %else, %then
 %struct.pair.pair = type { %struct.pair, %struct.pair }
 
 define void @t6() !dbg !32 {
-
+; CHECK-LABEL: define void @t6(
+; CHECK-SAME: ) !dbg [[DBG32:![0-9]+]] {
+; CHECK-NEXT:      #dbg_value(i32 0, [[META33:![0-9]+]], !DIExpression(DIOpArg(0, i32), DIOpFragment(0, 32)), [[META38:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(i32 1, [[META33]], !DIExpression(DIOpArg(0, i32), DIOpFragment(32, 32)), [[META38]])
+; CHECK-NEXT:      #dbg_value(i32 2, [[META33]], !DIExpression(DIOpArg(0, i32), DIOpFragment(64, 32)), [[META38]])
+; CHECK-NEXT:      #dbg_value(i32 3, [[META33]], !DIExpression(DIOpArg(0, i32), DIOpFragment(96, 32)), [[META38]])
+; CHECK-NEXT:    ret void
+;
   %first = alloca %struct.pair, align 4
   %second = alloca %struct.pair, align 4
   tail call void @llvm.dbg.declare(metadata ptr %first, metadata !37, metadata !DIExpression(DIOpArg(0, ptr), DIOpDeref(%struct.pair), DIOpFragment(0, 64))), !dbg !38
@@ -148,5 +195,37 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !37 = !DILocalVariable(name: "local", scope: !32, file: !1, line: 1, type: !33)
 !38 = !DILocation(line: 1, column: 1, scope: !32)
 
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: [[META1:![0-9]+]], producer: "clang 19", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}t.cpp", directory: {{.*}})
+; CHECK: [[DBG9]] = distinct !DISubprogram(name: "t1", linkageName: "t1", scope: [[META1]], file: [[META1]], line: 7, type: [[META10:![0-9]+]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12:![0-9]+]])
+; CHECK: [[META10]] = !DISubroutineType(types: [[META11:![0-9]+]])
+; CHECK: [[META11]] = !{null}
+; CHECK: [[META12]] = !{[[META13]]}
+; CHECK: [[META13]] = !DILocalVariable(name: "local", scope: [[DBG9]], file: [[META1]], line: 8, type: [[META14:![0-9]+]])
+; CHECK: [[META14]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+; CHECK: [[META15]] = !DILocation(line: 0, scope: [[DBG9]])
+; CHECK: [[DBG16]] = distinct !DISubprogram(name: "t2", linkageName: "t2", scope: [[META1]], file: [[META1]], line: 7, type: [[META10]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12]])
+; CHECK: [[META17]] = !DILocalVariable(name: "local", scope: [[DBG16]], file: [[META1]], line: 1, type: [[META14]])
+; CHECK: [[META18]] = !DILocation(line: 0, scope: [[DBG16]])
+; CHECK: [[DBG19]] = distinct !DISubprogram(name: "t3", linkageName: "t3", scope: [[META1]], file: [[META1]], line: 7, type: [[META10]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12]])
+; CHECK: [[META20]] = !DILocalVariable(name: "local", scope: [[DBG19]], file: [[META1]], line: 1, type: [[META21:![0-9]+]])
+; CHECK: [[META21]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "pair", file: [[META1]], line: 2, size: 64, flags: DIFlagTypePassByValue, elements: [[META22:![0-9]+]], identifier: "pair")
+; CHECK: [[META22]] = !{[[META23:![0-9]+]], [[META24:![0-9]+]]}
+; CHECK: [[META23]] = !DIDerivedType(tag: DW_TAG_member, name: "s1", scope: [[META21]], file: [[META1]], line: 3, baseType: [[META14]], size: 32)
+; CHECK: [[META24]] = !DIDerivedType(tag: DW_TAG_member, name: "s2", scope: [[META21]], file: [[META1]], line: 4, baseType: [[META14]], size: 32, offset: 32)
+; CHECK: [[META25]] = !DILocation(line: 0, scope: [[DBG19]])
+; CHECK: [[DBG26]] = distinct !DISubprogram(name: "t4", linkageName: "t4", scope: [[META1]], file: [[META1]], line: 7, type: [[META10]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12]])
+; CHECK: [[META27]] = !DILocalVariable(name: "local", scope: [[DBG26]], file: [[META1]], line: 1, type: [[META14]])
+; CHECK: [[META28]] = !DILocation(line: 1, column: 1, scope: [[DBG26]])
+; CHECK: [[DBG29]] = distinct !DISubprogram(name: "t5", linkageName: "t5", scope: [[META1]], file: [[META1]], line: 7, type: [[META10]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12]])
+; CHECK: [[META30]] = !DILocalVariable(name: "local_i16", scope: [[DBG29]], file: [[META1]], line: 1, type: [[META14]])
+; CHECK: [[META31]] = !DILocation(line: 0, scope: [[DBG29]])
+; CHECK: [[DBG32]] = distinct !DISubprogram(name: "t6", linkageName: "t56", scope: [[META1]], file: [[META1]], line: 7, type: [[META10]], scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META12]])
+; CHECK: [[META33]] = !DILocalVariable(name: "local", scope: [[DBG32]], file: [[META1]], line: 1, type: [[META34:![0-9]+]])
+; CHECK: [[META34]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "pair_pair", file: [[META1]], line: 2, size: 128, flags: DIFlagTypePassByValue, elements: [[META35:![0-9]+]], identifier: "pair_pair")
+; CHECK: [[META35]] = !{[[META36:![0-9]+]], [[META37:![0-9]+]]}
+; CHECK: [[META36]] = !DIDerivedType(tag: DW_TAG_member, name: "s1", scope: [[META34]], file: [[META1]], line: 3, baseType: [[META21]], size: 64)
+; CHECK: [[META37]] = !DIDerivedType(tag: DW_TAG_member, name: "s2", scope: [[META34]], file: [[META1]], line: 4, baseType: [[META21]], size: 64, offset: 64)
+; CHECK: [[META38]] = !DILocation(line: 0, scope: [[DBG32]])
+;.
diff --git a/external/llvm-project/llvm/test/Transforms/SROA/scalable-vectors-with-known-vscale.ll b/external/llvm-project/llvm/test/Transforms/SROA/scalable-vectors-with-known-vscale.ll
new file mode 100644
index 000000000000..85715e406e06
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/SROA/scalable-vectors-with-known-vscale.ll
@@ -0,0 +1,349 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes='sroa<preserve-cfg>' -S | FileCheck %s --check-prefixes=CHECK,CHECK-PRESERVE-CFG
+; RUN: opt < %s -passes='sroa<modify-cfg>' -S | FileCheck %s --check-prefixes=CHECK,CHECK-MODIFY-CFG
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+; This test checks that SROA runs mem2reg on scalable vectors.
+
+define <vscale x 16 x i1> @alloca_nxv16i1(<vscale x 16 x i1> %pg) vscale_range(1) {
+; CHECK-LABEL: @alloca_nxv16i1(
+; CHECK-NEXT:    ret <vscale x 16 x i1> [[PG:%.*]]
+;
+  %pg.addr = alloca <vscale x 16 x i1>
+  store <vscale x 16 x i1> %pg, ptr %pg.addr
+  %1 = load <vscale x 16 x i1>, ptr %pg.addr
+  ret <vscale x 16 x i1> %1
+}
+
+define <vscale x 16 x i8> @alloca_nxv16i8(<vscale x 16 x i8> %vec) vscale_range(1) {
+; CHECK-LABEL: @alloca_nxv16i8(
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[VEC:%.*]]
+;
+  %vec.addr = alloca <vscale x 16 x i8>
+  store <vscale x 16 x i8> %vec, ptr %vec.addr
+  %1 = load <vscale x 16 x i8>, ptr %vec.addr
+  ret <vscale x 16 x i8> %1
+}
+
+; Test scalable alloca that can't be promoted. Mem2Reg only considers
+; non-volatile loads and stores for promotion.
+define <vscale x 16 x i8> @unpromotable_alloca(<vscale x 16 x i8> %vec) vscale_range(1) {
+; CHECK-LABEL: @unpromotable_alloca(
+; CHECK-NEXT:    [[VEC_ADDR:%.*]] = alloca <vscale x 16 x i8>, align 16
+; CHECK-NEXT:    store volatile <vscale x 16 x i8> [[VEC:%.*]], ptr [[VEC_ADDR]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load volatile <vscale x 16 x i8>, ptr [[VEC_ADDR]], align 16
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+;
+  %vec.addr = alloca <vscale x 16 x i8>
+  store volatile <vscale x 16 x i8> %vec, ptr %vec.addr
+  %1 = load volatile <vscale x 16 x i8>, ptr %vec.addr
+  ret <vscale x 16 x i8> %1
+}
+
+; Test we bail out when using an alloca of a fixed-length vector (VLS) that was
+; bitcasted to a scalable vector.
+define <vscale x 4 x i32> @cast_alloca_to_svint32_t(<vscale x 4 x i32> %type.coerce) vscale_range(1) {
+; CHECK-LABEL: @cast_alloca_to_svint32_t(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TYPE_COERCE:%.*]], i64 0)
+; CHECK-NEXT:    [[TYPE_0_VEC_EXPAND:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TYPE_0_VECBLEND:%.*]] = select <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <16 x i32> [[TYPE_0_VEC_EXPAND]], <16 x i32> undef
+; CHECK-NEXT:    [[TYPE_ADDR_0_VEC_EXTRACT:%.*]] = shufflevector <16 x i32> [[TYPE_0_VECBLEND]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> poison, <4 x i32> [[TYPE_ADDR_0_VEC_EXTRACT]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
+;
+  %type = alloca <16 x i32>
+  %type.addr = alloca <16 x i32>
+  store <vscale x 4 x i32> %type.coerce, ptr %type
+  %type1 = load <16 x i32>, ptr %type
+  store <16 x i32> %type1, ptr %type.addr
+  %1 = load <16 x i32>, ptr %type.addr
+  %2 = load <vscale x 4 x i32>, ptr %type.addr
+  ret <vscale x 4 x i32> %2
+}
+
+; When casting from VLA to VLS via memory check we bail out when producing a
+; GEP where the element type is a scalable vector.
+define <vscale x 4 x i32> @cast_alloca_from_svint32_t() vscale_range(1) {
+; CHECK-LABEL: @cast_alloca_from_svint32_t(
+; CHECK-NEXT:    [[RETVAL_COERCE:%.*]] = alloca <vscale x 4 x i32>, align 16
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[RETVAL_COERCE]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <vscale x 4 x i32>, ptr [[RETVAL_COERCE]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+;
+  %retval = alloca <16 x i32>
+  store <16 x i32> zeroinitializer, ptr %retval
+  %retval.coerce = alloca <vscale x 4 x i32>
+  call void @llvm.memcpy.p0.p0.i64(ptr align 16 %retval.coerce, ptr align 16 %retval, i64 64, i1 false)
+  %1 = load <vscale x 4 x i32>, ptr %retval.coerce
+  ret <vscale x 4 x i32> %1
+}
+
+; Test we bail out when using an alloca of a fixed-length vector (VLS) that was
+; bitcasted to a scalable vector.
+define void @select_load_alloca_to_svdouble_t() vscale_range(1) {
+; CHECK-LABEL: @select_load_alloca_to_svdouble_t(
+; CHECK-NEXT:    [[Z:%.*]] = alloca <16 x half>, align 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 0, 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], ptr [[Z]], ptr null
+; CHECK-NEXT:    [[VAL:%.*]] = load <vscale x 2 x double>, ptr [[COND]], align 16
+; CHECK-NEXT:    ret void
+;
+  %z = alloca <16 x half>
+  %cmp = icmp eq i32 0, 0
+  %cond = select i1 %cmp, ptr %z, ptr null
+  %val = load <vscale x 2 x double>, ptr %cond, align 16
+  ret void
+}
+
+define void @select_store_alloca_to_svdouble_t(<vscale x 2 x double> %val) vscale_range(1) {
+; CHECK-LABEL: @select_store_alloca_to_svdouble_t(
+; CHECK-NEXT:    [[Z:%.*]] = alloca <16 x half>, align 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 0, 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], ptr [[Z]], ptr null
+; CHECK-NEXT:    store <vscale x 2 x double> [[VAL:%.*]], ptr [[COND]], align 16
+; CHECK-NEXT:    ret void
+;
+  %z = alloca <16 x half>
+  %cmp = icmp eq i32 0, 0
+  %cond = select i1 %cmp, ptr %z, ptr null
+  store <vscale x 2 x double> %val, ptr %cond, align 16
+  ret void
+}
+
+define <4 x i32> @fixed_alloca_fixed_from_scalable(<vscale x 4 x i32> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[A:%.*]], i64 0)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %tmp = alloca <4 x i32>
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load <4 x i32>, ptr %tmp
+  ret <4 x i32> %cast
+}
+
+define <2 x i8> @fixed_alloca_fixed_from_scalable_requires_bitcast(<vscale x 16 x i1> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_requires_bitcast(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 16 x i1> [[A:%.*]] to <vscale x 2 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i8> @llvm.vector.extract.v2i8.nxv2i8(<vscale x 2 x i8> [[TMP1]], i64 0)
+; CHECK-NEXT:    ret <2 x i8> [[TMP2]]
+;
+  %tmp = alloca <2 x i8>
+  store <vscale x 16 x i1> %a, ptr %tmp
+  %cast = load <2 x i8>, ptr %tmp
+  ret <2 x i8> %cast
+}
+
+define <2 x ptr> @fixed_alloca_fixed_from_scalable_inttoptr(<vscale x 4 x i32> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_inttoptr(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[A:%.*]] to <vscale x 2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> [[TMP1]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr <2 x i64> [[TMP3]] to <2 x ptr>
+; CHECK-NEXT:    ret <2 x ptr> [[TMP2]]
+;
+  %tmp = alloca <4 x i32>
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load <2 x ptr>, ptr %tmp
+  ret <2 x ptr> %cast
+}
+
+define <4 x i32> @fixed_alloca_fixed_from_scalable_ptrtoint(<vscale x 2 x ptr> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_ptrtoint(
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <vscale x 2 x ptr> [[A:%.*]] to <vscale x 2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i64> [[TMP1]] to <vscale x 4 x i32>
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP2]], i64 0)
+; CHECK-NEXT:    ret <4 x i32> [[TMP_0_CAST]]
+;
+  %tmp = alloca <4 x i32>
+  store <vscale x 2 x ptr> %a, ptr %tmp
+  %cast = load <4 x i32>, ptr %tmp
+  ret <4 x i32> %cast
+}
+
+define <2 x ptr> @fixed_alloca_fixed_from_scalable_ptrtoptr(<vscale x 2 x ptr> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_ptrtoptr(
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.nxv2p0(<vscale x 2 x ptr> [[A:%.*]], i64 0)
+; CHECK-NEXT:    ret <2 x ptr> [[TMP_0_CAST]]
+;
+  %tmp = alloca <2 x ptr>
+  store <vscale x 2 x ptr> %a, ptr %tmp
+  %cast = load <2 x ptr>, ptr %tmp
+  ret <2 x ptr> %cast
+}
+
+define <2 x ptr> @fixed_alloca_fixed_from_scalable_ptrtoptr_different_addrspace(<vscale x 2 x ptr addrspace(1)> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_ptrtoptr_different_addrspace(
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <vscale x 2 x ptr addrspace(1)> [[A:%.*]] to <vscale x 2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> [[TMP1]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x ptr>
+; CHECK-NEXT:    ret <2 x ptr> [[TMP3]]
+;
+  %tmp = alloca <2 x ptr>
+  store <vscale x 2 x ptr addrspace(1)> %a, ptr %tmp
+  %cast = load <2 x ptr>, ptr %tmp
+  ret <2 x ptr> %cast
+}
+
+define <vscale x 4 x i32> @fixed_alloca_scalable_from_fixed(<4 x i32> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> poison, <4 x i32> [[A:%.*]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+;
+  %tmp = alloca <4 x i32>
+  store <4 x i32> %a, ptr %tmp
+  %cast = load <vscale x 4 x i32>, ptr %tmp
+  ret <vscale x 4 x i32> %cast
+}
+
+define <vscale x 16 x i1> @fixed_alloca_scalable_from_fixed_requires_bitcast(<2 x i8> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_requires_bitcast(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> poison, <2 x i8> [[A:%.*]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 2 x i8> [[TMP1]] to <vscale x 16 x i1>
+; CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
+;
+  %tmp = alloca <2 x i8>
+  store <2 x i8> %a, ptr %tmp
+  %cast = load <vscale x 16 x i1>, ptr %tmp
+  ret <vscale x 16 x i1> %cast
+}
+
+define <vscale x 2 x ptr> @fixed_alloca_scalable_from_fixed_inttoptr(<4 x i32> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_inttoptr(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> poison, <4 x i32> [[A:%.*]], i64 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <vscale x 4 x i32> [[TMP1]] to <vscale x 2 x i64>
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = inttoptr <vscale x 2 x i64> [[TMP2]] to <vscale x 2 x ptr>
+; CHECK-NEXT:    ret <vscale x 2 x ptr> [[TMP_0_CAST]]
+;
+  %tmp = alloca <4 x i32>
+  store <4 x i32> %a, ptr %tmp
+  %cast = load <vscale x 2 x ptr>, ptr %tmp
+  ret <vscale x 2 x ptr> %cast
+}
+
+define <vscale x 4 x i32> @fixed_alloca_scalable_from_fixed_ptrtoint(<2 x ptr> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_ptrtoint(
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <2 x ptr> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> [[TMP1]], i64 0)
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = bitcast <vscale x 2 x i64> [[TMP2]] to <vscale x 4 x i32>
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP_0_CAST]]
+;
+  %tmp = alloca <4 x i32>
+  store <2 x ptr> %a, ptr %tmp
+  %cast = load <vscale x 4 x i32>, ptr %tmp
+  ret <vscale x 4 x i32> %cast
+}
+
+define <vscale x 2 x ptr> @fixed_alloca_scalable_from_fixed_ptrtoptr(<2 x ptr> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_ptrtoptr(
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = call <vscale x 2 x ptr> @llvm.vector.insert.nxv2p0.v2p0(<vscale x 2 x ptr> poison, <2 x ptr> [[A:%.*]], i64 0)
+; CHECK-NEXT:    ret <vscale x 2 x ptr> [[TMP_0_CAST]]
+;
+  %tmp = alloca <2 x ptr>
+  store <2 x ptr> %a, ptr %tmp
+  %cast = load <vscale x 2 x ptr>, ptr %tmp
+  ret <vscale x 2 x ptr> %cast
+}
+
+define <vscale x 2 x ptr addrspace(1)> @fixed_alloca_scalable_from_fixed_ptrtoptr_different_addrspace(<2 x ptr> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_ptrtoptr_different_addrspace(
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <2 x ptr> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> [[TMP1]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr <vscale x 2 x i64> [[TMP2]] to <vscale x 2 x ptr addrspace(1)>
+; CHECK-NEXT:    ret <vscale x 2 x ptr addrspace(1)> [[TMP3]]
+;
+  %tmp = alloca <2 x ptr>
+  store <2 x ptr> %a, ptr %tmp
+  %cast = load <vscale x 2 x ptr addrspace(1)>, ptr %tmp
+  ret <vscale x 2 x ptr addrspace(1)> %cast
+}
+
+define <4 x i32> @scalable_alloca_fixed_from_scalable(<vscale x 4 x i32> %a) vscale_range(1) {
+; CHECK-LABEL: @scalable_alloca_fixed_from_scalable(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <vscale x 4 x i32>, align 16
+; CHECK-NEXT:    store <vscale x 4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[CAST:%.*]] = load <4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <4 x i32> [[CAST]]
+;
+  %tmp = alloca <vscale x 4 x i32>
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load <4 x i32>, ptr %tmp
+  ret <4 x i32> %cast
+}
+
+define <vscale x 4 x i32> @scalable_alloca_scalable_from_fixed(<4 x i32> %a) vscale_range(1) {
+; CHECK-LABEL: @scalable_alloca_scalable_from_fixed(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <vscale x 4 x i32>, align 16
+; CHECK-NEXT:    store <4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[CAST:%.*]] = load <vscale x 4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST]]
+;
+  %tmp = alloca <vscale x 4 x i32>
+  store <4 x i32> %a, ptr %tmp
+  %cast = load <vscale x 4 x i32>, ptr %tmp
+  ret <vscale x 4 x i32> %cast
+}
+
+define i16 @scalar_alloca_scalar_from_scalable(<vscale x 16 x i1> %a) vscale_range(1) {
+; CHECK-LABEL: @scalar_alloca_scalar_from_scalable(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store <vscale x 16 x i1> [[A:%.*]], ptr [[TMP]], align 2
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load i16, ptr [[TMP]], align 2
+; CHECK-NEXT:    ret i16 [[TMP_0_CAST]]
+;
+  %tmp = alloca i16
+  store <vscale x 16 x i1> %a, ptr %tmp
+  %cast = load i16, ptr %tmp
+  ret i16 %cast
+}
+
+define <vscale x 16 x i1> @scalar_alloca_scalable_from_scalar(i16 %a) vscale_range(1) {
+; CHECK-LABEL: @scalar_alloca_scalable_from_scalar(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 [[A:%.*]], ptr [[TMP]], align 2
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load <vscale x 16 x i1>, ptr [[TMP]], align 2
+; CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP_0_CAST]]
+;
+  %tmp = alloca i16
+  store i16 %a, ptr %tmp
+  %cast = load <vscale x 16 x i1>, ptr %tmp
+  ret <vscale x 16 x i1> %cast
+}
+
+define { <2 x i32>, <2 x i32> } @fixed_struct_alloca_fixed_from_scalable(<vscale x 4 x i32> %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_struct_alloca_fixed_from_scalable(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca { <2 x i32>, <2 x i32> }, align 8
+; CHECK-NEXT:    store <vscale x 4 x i32> [[A:%.*]], ptr [[TMP]], align 8
+; CHECK-NEXT:    [[TMP_0_CAST_FCA_0_LOAD:%.*]] = load <2 x i32>, ptr [[TMP]], align 8
+; CHECK-NEXT:    [[CAST_FCA_0_INSERT:%.*]] = insertvalue { <2 x i32>, <2 x i32> } poison, <2 x i32> [[TMP_0_CAST_FCA_0_LOAD]], 0
+; CHECK-NEXT:    [[TMP_8_CAST_FCA_1_GEP_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 8
+; CHECK-NEXT:    [[TMP_8_CAST_FCA_1_LOAD:%.*]] = load <2 x i32>, ptr [[TMP_8_CAST_FCA_1_GEP_SROA_IDX]], align 8
+; CHECK-NEXT:    [[CAST_FCA_1_INSERT:%.*]] = insertvalue { <2 x i32>, <2 x i32> } [[CAST_FCA_0_INSERT]], <2 x i32> [[TMP_8_CAST_FCA_1_LOAD]], 1
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i32> } [[CAST_FCA_1_INSERT]]
+;
+  %tmp = alloca { <2 x i32>, <2 x i32> }
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load { <2 x i32>, <2 x i32> }, ptr %tmp
+  ret { <2 x i32>, <2 x i32> } %cast
+}
+
+define <vscale x 4 x i64> @fixed_struct_alloca_scalable_from_fixed({ <2 x ptr>, <2 x ptr> } %a) vscale_range(1) {
+; CHECK-LABEL: @fixed_struct_alloca_scalable_from_fixed(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca { <2 x ptr>, <2 x ptr> }, align 16
+; CHECK-NEXT:    [[A_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[A:%.*]], 0
+; CHECK-NEXT:    store <2 x ptr> [[A_FCA_0_EXTRACT]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[A_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[A]], 1
+; CHECK-NEXT:    [[TMP_16_A_FCA_1_GEP_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 16
+; CHECK-NEXT:    store <2 x ptr> [[A_FCA_1_EXTRACT]], ptr [[TMP_16_A_FCA_1_GEP_SROA_IDX]], align 16
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load <vscale x 4 x i64>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP_0_CAST]]
+;
+  %tmp = alloca { <2 x ptr>, <2 x ptr> }
+  store { <2 x ptr>, <2 x ptr> } %a, ptr %tmp
+  %cast = load <vscale x 4 x i64>, ptr %tmp
+  ret <vscale x 4 x i64> %cast
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-MODIFY-CFG: {{.*}}
+; CHECK-PRESERVE-CFG: {{.*}}
diff --git a/external/llvm-project/llvm/test/Transforms/SROA/scalable-vectors.ll b/external/llvm-project/llvm/test/Transforms/SROA/scalable-vectors.ll
index d892883ce9dc..346814d9f630 100644
--- a/external/llvm-project/llvm/test/Transforms/SROA/scalable-vectors.ll
+++ b/external/llvm-project/llvm/test/Transforms/SROA/scalable-vectors.ll
@@ -2,6 +2,8 @@
 ; RUN: opt < %s -passes='sroa<preserve-cfg>' -S | FileCheck %s --check-prefixes=CHECK,CHECK-PRESERVE-CFG
 ; RUN: opt < %s -passes='sroa<modify-cfg>' -S | FileCheck %s --check-prefixes=CHECK,CHECK-MODIFY-CFG
 
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
 ; This test checks that SROA runs mem2reg on scalable vectors.
 
 define <vscale x 16 x i1> @alloca_nxv16i1(<vscale x 16 x i1> %pg) {
@@ -67,11 +69,12 @@ define <vscale x 4 x i32> @cast_alloca_to_svint32_t(<vscale x 4 x i32> %type.coe
 define <vscale x 4 x i32> @cast_alloca_from_svint32_t() {
 ; CHECK-LABEL: @cast_alloca_from_svint32_t(
 ; CHECK-NEXT:    [[RETVAL_COERCE:%.*]] = alloca <vscale x 4 x i32>, align 16
-; CHECK-NEXT:    store <16 x i32> undef, ptr [[RETVAL_COERCE]], align 16
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[RETVAL_COERCE]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <vscale x 4 x i32>, ptr [[RETVAL_COERCE]], align 16
 ; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 ;
   %retval = alloca <16 x i32>
+  store <16 x i32> zeroinitializer, ptr %retval
   %retval.coerce = alloca <vscale x 4 x i32>
   call void @llvm.memcpy.p0.p0.i64(ptr align 16 %retval.coerce, ptr align 16 %retval, i64 64, i1 false)
   %1 = load <vscale x 4 x i32>, ptr %retval.coerce
@@ -110,6 +113,224 @@ define void @select_store_alloca_to_svdouble_t(<vscale x 2 x double> %val) {
   ret void
 }
 
+define <4 x i32> @fixed_alloca_fixed_from_scalable(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i32>, align 16
+; CHECK-NEXT:    store <vscale x 4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %tmp = alloca <4 x i32>
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load <4 x i32>, ptr %tmp
+  ret <4 x i32> %cast
+}
+
+define <2 x i8> @fixed_alloca_fixed_from_scalable_requires_bitcast(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_requires_bitcast(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <2 x i8>, align 2
+; CHECK-NEXT:    store <vscale x 16 x i1> [[A:%.*]], ptr [[TMP]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i8>, ptr [[TMP]], align 2
+; CHECK-NEXT:    ret <2 x i8> [[TMP2]]
+;
+  %tmp = alloca <2 x i8>
+  store <vscale x 16 x i1> %a, ptr %tmp
+  %cast = load <2 x i8>, ptr %tmp
+  ret <2 x i8> %cast
+}
+
+define <2 x ptr> @fixed_alloca_fixed_from_scalable_inttoptr(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_inttoptr(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i32>, align 16
+; CHECK-NEXT:    store <vscale x 4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x ptr>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <2 x ptr> [[TMP2]]
+;
+  %tmp = alloca <4 x i32>
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load <2 x ptr>, ptr %tmp
+  ret <2 x ptr> %cast
+}
+
+define <4 x i32> @fixed_alloca_fixed_from_scalable_ptrtoint(<vscale x 2 x ptr> %a) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_ptrtoint(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i32>, align 16
+; CHECK-NEXT:    store <vscale x 2 x ptr> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load <4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <4 x i32> [[TMP_0_CAST]]
+;
+  %tmp = alloca <4 x i32>
+  store <vscale x 2 x ptr> %a, ptr %tmp
+  %cast = load <4 x i32>, ptr %tmp
+  ret <4 x i32> %cast
+}
+
+define <2 x ptr> @fixed_alloca_fixed_from_scalable_ptrtoptr(<vscale x 2 x ptr> %a) {
+; CHECK-LABEL: @fixed_alloca_fixed_from_scalable_ptrtoptr(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <2 x ptr>, align 16
+; CHECK-NEXT:    store <vscale x 2 x ptr> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[CAST:%.*]] = load <2 x ptr>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <2 x ptr> [[CAST]]
+;
+  %tmp = alloca <2 x ptr>
+  store <vscale x 2 x ptr> %a, ptr %tmp
+  %cast = load <2 x ptr>, ptr %tmp
+  ret <2 x ptr> %cast
+}
+
+define <vscale x 4 x i32> @fixed_alloca_scalable_from_fixed(<4 x i32> %a) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i32>, align 16
+; CHECK-NEXT:    store <4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+;
+  %tmp = alloca <4 x i32>
+  store <4 x i32> %a, ptr %tmp
+  %cast = load <vscale x 4 x i32>, ptr %tmp
+  ret <vscale x 4 x i32> %cast
+}
+
+define <vscale x 16 x i1> @fixed_alloca_scalable_from_fixed_requires_bitcast(<2 x i8> %a) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_requires_bitcast(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <2 x i8>, align 2
+; CHECK-NEXT:    store <2 x i8> [[A:%.*]], ptr [[TMP]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load <vscale x 16 x i1>, ptr [[TMP]], align 2
+; CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP2]]
+;
+  %tmp = alloca <2 x i8>
+  store <2 x i8> %a, ptr %tmp
+  %cast = load <vscale x 16 x i1>, ptr %tmp
+  ret <vscale x 16 x i1> %cast
+}
+
+define <vscale x 2 x ptr> @fixed_alloca_scalable_from_fixed_inttoptr(<4 x i32> %a) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_inttoptr(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i32>, align 16
+; CHECK-NEXT:    store <4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load <vscale x 2 x ptr>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <vscale x 2 x ptr> [[TMP_0_CAST]]
+;
+  %tmp = alloca <4 x i32>
+  store <4 x i32> %a, ptr %tmp
+  %cast = load <vscale x 2 x ptr>, ptr %tmp
+  ret <vscale x 2 x ptr> %cast
+}
+
+define <vscale x 4 x i32> @fixed_alloca_scalable_from_fixed_ptrtoint(<2 x ptr> %a) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_ptrtoint(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <4 x i32>, align 16
+; CHECK-NEXT:    store <2 x ptr> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load <vscale x 4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP_0_CAST]]
+;
+  %tmp = alloca <4 x i32>
+  store <2 x ptr> %a, ptr %tmp
+  %cast = load <vscale x 4 x i32>, ptr %tmp
+  ret <vscale x 4 x i32> %cast
+}
+
+define <vscale x 2 x ptr> @fixed_alloca_scalable_from_fixed_ptrtoptr(<2 x ptr> %a) {
+; CHECK-LABEL: @fixed_alloca_scalable_from_fixed_ptrtoptr(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <2 x ptr>, align 16
+; CHECK-NEXT:    store <2 x ptr> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[CAST:%.*]] = load <vscale x 2 x ptr>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <vscale x 2 x ptr> [[CAST]]
+;
+  %tmp = alloca <2 x ptr>
+  store <2 x ptr> %a, ptr %tmp
+  %cast = load <vscale x 2 x ptr>, ptr %tmp
+  ret <vscale x 2 x ptr> %cast
+}
+
+define <4 x i32> @scalable_alloca_fixed_from_scalable(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @scalable_alloca_fixed_from_scalable(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <vscale x 4 x i32>, align 16
+; CHECK-NEXT:    store <vscale x 4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[CAST:%.*]] = load <4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <4 x i32> [[CAST]]
+;
+  %tmp = alloca <vscale x 4 x i32>
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load <4 x i32>, ptr %tmp
+  ret <4 x i32> %cast
+}
+
+define <vscale x 4 x i32> @scalable_alloca_scalable_from_fixed(<4 x i32> %a) {
+; CHECK-LABEL: @scalable_alloca_scalable_from_fixed(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <vscale x 4 x i32>, align 16
+; CHECK-NEXT:    store <4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[CAST:%.*]] = load <vscale x 4 x i32>, ptr [[TMP]], align 16
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[CAST]]
+;
+  %tmp = alloca <vscale x 4 x i32>
+  store <4 x i32> %a, ptr %tmp
+  %cast = load <vscale x 4 x i32>, ptr %tmp
+  ret <vscale x 4 x i32> %cast
+}
+
+define i16 @scalar_alloca_scalar_from_scalable(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: @scalar_alloca_scalar_from_scalable(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store <vscale x 16 x i1> [[A:%.*]], ptr [[TMP]], align 2
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load i16, ptr [[TMP]], align 2
+; CHECK-NEXT:    ret i16 [[TMP_0_CAST]]
+;
+  %tmp = alloca i16
+  store <vscale x 16 x i1> %a, ptr %tmp
+  %cast = load i16, ptr %tmp
+  ret i16 %cast
+}
+
+define <vscale x 16 x i1> @scalar_alloca_scalable_from_scalar(i16 %a) {
+; CHECK-LABEL: @scalar_alloca_scalable_from_scalar(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    store i16 [[A:%.*]], ptr [[TMP]], align 2
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load <vscale x 16 x i1>, ptr [[TMP]], align 2
+; CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP_0_CAST]]
+;
+  %tmp = alloca i16
+  store i16 %a, ptr %tmp
+  %cast = load <vscale x 16 x i1>, ptr %tmp
+  ret <vscale x 16 x i1> %cast
+}
+
+define { <2 x i32>, <2 x i32> } @fixed_struct_alloca_fixed_from_scalable(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @fixed_struct_alloca_fixed_from_scalable(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca { <2 x i32>, <2 x i32> }, align 8
+; CHECK-NEXT:    store <vscale x 4 x i32> [[A:%.*]], ptr [[TMP]], align 16
+; CHECK-NEXT:    [[CAST_FCA_0_GEP:%.*]] = getelementptr inbounds { <2 x i32>, <2 x i32> }, ptr [[TMP]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP_0_CAST_FCA_0_LOAD:%.*]] = load <2 x i32>, ptr [[CAST_FCA_0_GEP]], align 8
+; CHECK-NEXT:    [[CAST_FCA_0_INSERT:%.*]] = insertvalue { <2 x i32>, <2 x i32> } poison, <2 x i32> [[TMP_0_CAST_FCA_0_LOAD]], 0
+; CHECK-NEXT:    [[TMP_8_CAST_FCA_1_GEP_SROA_IDX:%.*]] = getelementptr inbounds { <2 x i32>, <2 x i32> }, ptr [[TMP]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP_8_CAST_FCA_1_LOAD:%.*]] = load <2 x i32>, ptr [[TMP_8_CAST_FCA_1_GEP_SROA_IDX]], align 8
+; CHECK-NEXT:    [[CAST_FCA_1_INSERT:%.*]] = insertvalue { <2 x i32>, <2 x i32> } [[CAST_FCA_0_INSERT]], <2 x i32> [[TMP_8_CAST_FCA_1_LOAD]], 1
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i32> } [[CAST_FCA_1_INSERT]]
+;
+  %tmp = alloca { <2 x i32>, <2 x i32> }
+  store <vscale x 4 x i32> %a, ptr %tmp
+  %cast = load { <2 x i32>, <2 x i32> }, ptr %tmp
+  ret { <2 x i32>, <2 x i32> } %cast
+}
+
+define <vscale x 4 x i64> @fixed_struct_alloca_scalable_from_fixed({ <2 x ptr>, <2 x ptr> } %a) {
+; CHECK-LABEL: @fixed_struct_alloca_scalable_from_fixed(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca { <2 x ptr>, <2 x ptr> }, align 16
+; CHECK-NEXT:    [[A_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[A:%.*]], 0
+; CHECK-NEXT:    [[A_FCA_0_GEP:%.*]] = getelementptr inbounds { <2 x ptr>, <2 x ptr> }, ptr [[TMP]], i32 0, i32 0
+; CHECK-NEXT:    store <2 x ptr> [[A_FCA_0_EXTRACT]], ptr [[A_FCA_0_GEP]], align 16
+; CHECK-NEXT:    [[A_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[A]], 1
+; CHECK-NEXT:    [[TMP_16_A_FCA_1_GEP_SROA_IDX:%.*]] = getelementptr inbounds { <2 x ptr>, <2 x ptr> }, ptr [[TMP]], i32 0, i32 1
+; CHECK-NEXT:    store <2 x ptr> [[A_FCA_1_EXTRACT]], ptr [[TMP_16_A_FCA_1_GEP_SROA_IDX]], align 16
+; CHECK-NEXT:    [[TMP_0_CAST:%.*]] = load <vscale x 4 x i64>, ptr [[TMP]], align 32
+; CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP_0_CAST]]
+;
+  %tmp = alloca { <2 x ptr>, <2 x ptr> }
+  store { <2 x ptr>, <2 x ptr> } %a, ptr %tmp
+  %cast = load <vscale x 4 x i64>, ptr %tmp
+  ret <vscale x 4 x i64> %cast
+}
+
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK-MODIFY-CFG: {{.*}}
diff --git a/external/llvm-project/llvm/test/Transforms/SandboxVectorizer/pack_reuse_basic.ll b/external/llvm-project/llvm/test/Transforms/SandboxVectorizer/pack_reuse_basic.ll
new file mode 100644
index 000000000000..33cdaf54252c
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/SandboxVectorizer/pack_reuse_basic.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-passes="regions-from-metadata<pack-reuse>" %s -S | FileCheck %s
+
+define void @pack_reuse(i8 %v0, i8 %v1, ptr %ptr) {
+; CHECK-LABEL: define void @pack_reuse(
+; CHECK-SAME: i8 [[V0:%.*]], i8 [[V1:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[PACKA0:%.*]] = insertelement <2 x i8> poison, i8 [[V0]], i64 0, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT:    [[PACKA1:%.*]] = insertelement <2 x i8> [[PACKA0]], i8 [[V1]], i64 1, !sandboxvec [[META0]]
+; CHECK-NEXT:    store <2 x i8> [[PACKA1]], ptr [[PTR]], align 2, !sandboxvec [[META0]]
+; CHECK-NEXT:    store <2 x i8> [[PACKA1]], ptr [[PTR]], align 2, !sandboxvec [[META0]]
+; CHECK-NEXT:    [[PACKC0:%.*]] = insertelement <2 x i8> poison, i8 [[V1]], i64 0, !sandboxvec [[META0]]
+; CHECK-NEXT:    [[PACKC1:%.*]] = insertelement <2 x i8> [[PACKC0]], i8 [[V0]], i64 1, !sandboxvec [[META0]]
+; CHECK-NEXT:    store <2 x i8> [[PACKC1]], ptr [[PTR]], align 2, !sandboxvec [[META0]]
+; CHECK-NEXT:    ret void
+;
+  %PackA0 = insertelement <2 x i8> poison, i8 %v0, i64 0, !sandboxvec !0
+  %PackA1 = insertelement <2 x i8> %PackA0, i8 %v1, i64 1, !sandboxvec !0
+  store <2 x i8> %PackA1, ptr %ptr, !sandboxvec !0
+
+  ; Should reuse PackA1.
+  %PackB0 = insertelement <2 x i8> poison, i8 %v0, i64 0, !sandboxvec !0
+  %PackB1 = insertelement <2 x i8> %PackB0, i8 %v1, i64 1, !sandboxvec !0
+  store <2 x i8> %PackB1, ptr %ptr, !sandboxvec !0
+
+  ; Should remain.
+  %PackC0 = insertelement <2 x i8> poison, i8 %v1, i64 0, !sandboxvec !0
+  %PackC1 = insertelement <2 x i8> %PackC0, i8 %v0, i64 1, !sandboxvec !0
+  store <2 x i8> %PackC1, ptr %ptr, !sandboxvec !0
+  ret void
+}
+
+; TODO: For now we don't support reusing packs from earlier BBs.
+define void @pack_cross_bb(i8 %v0, i8 %v1, ptr %ptr) {
+; CHECK-LABEL: define void @pack_cross_bb(
+; CHECK-SAME: i8 [[V0:%.*]], i8 [[V1:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[PACKA0:%.*]] = insertelement <2 x i8> poison, i8 [[V0]], i64 0, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT:    [[PACKA1:%.*]] = insertelement <2 x i8> [[PACKA0]], i8 [[V1]], i64 1, !sandboxvec [[META1]]
+; CHECK-NEXT:    store <2 x i8> [[PACKA1]], ptr [[PTR]], align 2, !sandboxvec [[META1]]
+; CHECK-NEXT:    br label %[[BB:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[PACKB0:%.*]] = insertelement <2 x i8> poison, i8 [[V0]], i64 0, !sandboxvec [[META1]]
+; CHECK-NEXT:    [[PACKB1:%.*]] = insertelement <2 x i8> [[PACKB0]], i8 [[V1]], i64 1, !sandboxvec [[META1]]
+; CHECK-NEXT:    store <2 x i8> [[PACKB1]], ptr [[PTR]], align 2, !sandboxvec [[META1]]
+; CHECK-NEXT:    [[PACKC0:%.*]] = insertelement <2 x i8> poison, i8 [[V1]], i64 0, !sandboxvec [[META1]]
+; CHECK-NEXT:    [[PACKC1:%.*]] = insertelement <2 x i8> [[PACKC0]], i8 [[V0]], i64 1, !sandboxvec [[META1]]
+; CHECK-NEXT:    store <2 x i8> [[PACKC1]], ptr [[PTR]], align 2, !sandboxvec [[META1]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %PackA0 = insertelement <2 x i8> poison, i8 %v0, i64 0, !sandboxvec !0
+  %PackA1 = insertelement <2 x i8> %PackA0, i8 %v1, i64 1, !sandboxvec !0
+  store <2 x i8> %PackA1, ptr %ptr, !sandboxvec !0
+  br label %bb
+
+bb:
+  %PackB0 = insertelement <2 x i8> poison, i8 %v0, i64 0, !sandboxvec !0
+  %PackB1 = insertelement <2 x i8> %PackB0, i8 %v1, i64 1, !sandboxvec !0
+  store <2 x i8> %PackB1, ptr %ptr, !sandboxvec !0
+
+  %PackC0 = insertelement <2 x i8> poison, i8 %v1, i64 0, !sandboxvec !0
+  %PackC1 = insertelement <2 x i8> %PackC0, i8 %v0, i64 1, !sandboxvec !0
+  store <2 x i8> %PackC1, ptr %ptr, !sandboxvec !0
+  ret void
+}
+
+!0 = distinct !{!"sandboxregion"}
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+;.
diff --git a/external/llvm-project/llvm/test/Transforms/SandboxVectorizer/pack_reuse_end_to_end.ll b/external/llvm-project/llvm/test/Transforms/SandboxVectorizer/pack_reuse_end_to_end.ll
new file mode 100644
index 000000000000..0861c7827c91
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/SandboxVectorizer/pack_reuse_end_to_end.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix NOREUSE
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<tr-save,bottom-up-vec,pack-reuse,tr-accept>" %s -S | FileCheck %s --check-prefix PKREUSE
+
+define void @pack_reuse(ptr %ptr, ptr %ptrX, ptr %ptrY) {
+; NOREUSE-LABEL: define void @pack_reuse(
+; NOREUSE-SAME: ptr [[PTR:%.*]], ptr [[PTRX:%.*]], ptr [[PTRY:%.*]]) {
+; NOREUSE-NEXT:    [[LDX:%.*]] = load float, ptr [[PTRX]], align 4
+; NOREUSE-NEXT:    [[LDY:%.*]] = load float, ptr [[PTRY]], align 4
+; NOREUSE-NEXT:    [[PACK2:%.*]] = insertelement <2 x float> poison, float [[LDX]], i32 0, !sandboxvec [[META0:![0-9]+]]
+; NOREUSE-NEXT:    [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[LDY]], i32 1, !sandboxvec [[META0]]
+; NOREUSE-NEXT:    [[PACK:%.*]] = insertelement <2 x float> poison, float [[LDX]], i32 0, !sandboxvec [[META0]]
+; NOREUSE-NEXT:    [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[LDY]], i32 1, !sandboxvec [[META0]]
+; NOREUSE-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; NOREUSE-NEXT:    [[VEC:%.*]] = fsub <2 x float> [[PACK1]], [[PACK3]], !sandboxvec [[META0]]
+; NOREUSE-NEXT:    store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
+; NOREUSE-NEXT:    ret void
+;
+; PKREUSE-LABEL: define void @pack_reuse(
+; PKREUSE-SAME: ptr [[PTR:%.*]], ptr [[PTRX:%.*]], ptr [[PTRY:%.*]]) {
+; PKREUSE-NEXT:    [[LDX:%.*]] = load float, ptr [[PTRX]], align 4
+; PKREUSE-NEXT:    [[LDY:%.*]] = load float, ptr [[PTRY]], align 4
+; PKREUSE-NEXT:    [[PACK2:%.*]] = insertelement <2 x float> poison, float [[LDX]], i32 0, !sandboxvec [[META0:![0-9]+]]
+; PKREUSE-NEXT:    [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[LDY]], i32 1, !sandboxvec [[META0]]
+; PKREUSE-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
+; PKREUSE-NEXT:    [[VEC:%.*]] = fsub <2 x float> [[PACK3]], [[PACK3]], !sandboxvec [[META0]]
+; PKREUSE-NEXT:    store <2 x float> [[VEC]], ptr [[PTR0]], align 4, !sandboxvec [[META0]]
+; PKREUSE-NEXT:    ret void
+;
+  %ldX = load float, ptr %ptrX
+  %ldY = load float, ptr %ptrY
+
+  %ptr0 = getelementptr float, ptr %ptr, i32 0
+  %ptr1 = getelementptr float, ptr %ptr, i32 1
+  %sub0 = fsub float %ldX, %ldX
+  %sub1 = fsub float %ldY, %ldY
+  store float %sub0, ptr %ptr0
+  store float %sub1, ptr %ptr1
+  ret void
+}
+;.
+; NOREUSE: [[META0]] = distinct !{!"sandboxregion"}
+;.
+; PKREUSE: [[META0]] = distinct !{!"sandboxregion"}
+;.
diff --git a/external/llvm-project/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll b/external/llvm-project/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
index 349a18148460..a29c6a2fdd64 100644
--- a/external/llvm-project/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
+++ b/external/llvm-project/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
@@ -43,7 +43,7 @@ define i32 @foo(i32 %x) #0 section ".tcm_text" {
 ; DISABLE:       sw.default:
 ; DISABLE-NEXT:    br label [[RETURN]]
 ; DISABLE:       return:
-; DISABLE-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 19, [[SW_DEFAULT]] ], [ 33, [[SW_BB5]] ], [ 12, [[SW_BB4]] ], [ 22, [[SW_BB3]] ], [ 14, [[SW_BB2]] ], [ 20, [[SW_BB1]] ], [ 9, [[ENTRY:%.*]] ]
+; DISABLE-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 19, [[SW_DEFAULT]] ], [ 20, [[SW_BB1]] ], [ 14, [[SW_BB2]] ], [ 22, [[SW_BB3]] ], [ 12, [[SW_BB4]] ], [ 33, [[SW_BB5]] ], [ 9, [[ENTRY:%.*]] ]
 ; DISABLE-NEXT:    ret i32 [[RETVAL_0]]
 ;
 entry:
diff --git a/external/llvm-project/llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll b/external/llvm-project/llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
index aae1ab032f36..5b439a23c358 100644
--- a/external/llvm-project/llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
+++ b/external/llvm-project/llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
@@ -238,12 +238,13 @@ else:
 }
 
 declare ptr @fn_nonnull_noundef_arg(ptr nonnull noundef %p)
-declare ptr @fn_nonnull_deref_arg(ptr nonnull dereferenceable(4) %p)
+declare ptr @fn_deref_arg(ptr dereferenceable(4) %p)
 declare ptr @fn_nonnull_deref_or_null_arg(ptr nonnull dereferenceable_or_null(4) %p)
 declare ptr @fn_nonnull_arg(ptr nonnull %p)
 declare ptr @fn_noundef_arg(ptr noundef %p)
 declare ptr @fn_ptr_arg(ptr)
 declare ptr @fn_ptr_arg_nounwind_willreturn(ptr) nounwind willreturn
+declare void @fn_arg_vec(<2 x ptr>)
 
 define void @test9(i1 %X, ptr %Y) {
 ; CHECK-LABEL: @test9(
@@ -271,7 +272,7 @@ define void @test9_deref(i1 %X, ptr %Y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[X:%.*]], true
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP0]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @fn_nonnull_deref_arg(ptr [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @fn_deref_arg(ptr [[Y:%.*]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -282,7 +283,7 @@ if:
 
 else:
   %phi = phi ptr [ %Y, %entry ], [ null, %if ]
-  call ptr @fn_nonnull_deref_arg(ptr %phi)
+  call ptr @fn_deref_arg(ptr %phi)
   ret void
 }
 
@@ -290,9 +291,8 @@ else:
 define void @test9_deref_or_null(i1 %X, ptr %Y) {
 ; CHECK-LABEL: @test9_deref_or_null(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[X:%.*]], true
-; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP0]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @fn_nonnull_deref_or_null_arg(ptr [[Y:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = select i1 [[X:%.*]], ptr null, ptr [[Y1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @fn_nonnull_deref_or_null_arg(ptr [[Y]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -918,6 +918,25 @@ bb5:                                              ; preds = %bb3, %bb
   ret i32 %i7
 }
 
+define void @test9_gep_splat(i1 %X, ptr %Y) {
+; CHECK-LABEL: @test9_gep_splat(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[X:%.*]], ptr null, ptr [[Y:%.*]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[SPEC_SELECT]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    call void @fn_arg_vec(<2 x ptr> [[GEP]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %X, label %if, label %else
+if:
+  br label %else
+else:
+  %phi = phi ptr [ %Y, %entry ], [ null, %if ]
+  %gep = getelementptr i8, ptr %phi, <2 x i64> zeroinitializer
+  call void @fn_arg_vec(<2 x ptr> %gep)
+  ret void
+}
+
 declare void @side.effect()
 declare i8 @get.i8()
 
diff --git a/external/llvm-project/llvm/test/Transforms/SimplifyTypeTests/basic.ll b/external/llvm-project/llvm/test/Transforms/SimplifyTypeTests/basic.ll
new file mode 100644
index 000000000000..852774c0dff1
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/SimplifyTypeTests/basic.ll
@@ -0,0 +1,46 @@
+; Test that a lowered type test for a type is simplified to true
+; if the target is a constant member of that type.
+
+; RUN: opt -S %s -passes=simplify-type-tests | FileCheck %s
+
+; Test that the simplification does not occur if the type is wrong.
+ 
+; RUN: sed -e 's/"_ZTSFvvE"/"wrongtype"/g' %s | opt -S -passes=simplify-type-tests | FileCheck --check-prefix=WRONGTYPE %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@__typeid__ZTSFvvE_global_addr = external hidden global [0 x i8], code_model "small"
+
+define void @_Z2fpv.cfi() !type !0 {
+  ret void
+}
+
+define i64 @main() {
+  %1 = icmp eq ptr @_Z2fpv, @__typeid__ZTSFvvE_global_addr
+  ; CHECK: br i1 true
+  ; WRONGTYPE: br i1 %
+  br i1 %1, label %3, label %2
+
+2:
+  tail call void @llvm.ubsantrap(i8 2)
+  unreachable
+
+3:
+  ; CHECK: br i1 true
+  ; WRONGTYPE: br i1 %
+  %c = icmp eq i64 ptrtoint (ptr @_Z2fpv to i64), ptrtoint (ptr @__typeid__ZTSFvvE_global_addr to i64)
+  br i1 %c, label %4, label %2
+
+4:
+  tail call void @_Z2fpv()
+  ; CHECK: ret i64 0
+  ; WRONGTYPE: ret i64 sub
+  ret i64 sub (i64 ptrtoint (ptr @__typeid__ZTSFvvE_global_addr to i64), i64 ptrtoint (ptr @_Z2fpv to i64))
+}
+
+declare void @llvm.ubsantrap(i8 immarg)
+
+declare hidden void @_Z2fpv()
+
+!0 = !{i64 0, !"_ZTSFvvE"}
diff --git a/external/llvm-project/llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll b/external/llvm-project/llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll
index 0f3fd566f9b1..13dcefcb70cb 100644
--- a/external/llvm-project/llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll
+++ b/external/llvm-project/llvm/test/Transforms/ThinLTOBitcodeWriter/unique-source-file-names.ll
@@ -19,4 +19,5 @@ define internal void @f() {
 !0 = !{i32 0, !"typeid"}
 
 !llvm.module.flags = !{!1}
-!1 = !{i32 1, !"Unique Source File Names", i32 1}
+!1 = !{i32 5, !"Unique Source File Identifier", !2}
+!2 = !{!"unique-source-file-names.c"}
diff --git a/external/llvm-project/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll b/external/llvm-project/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
new file mode 100644
index 000000000000..09c03991ad7c
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
@@ -0,0 +1,268 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p vector-combine -mtriple=arm64-apple-macosx -S %s | FileCheck %s
+
+declare void @use.i32(i32)
+declare void @use.i64(i64)
+declare void @use.v4i32(<4 x i32>)
+
+define void @zext_v4i8_all_lanes_used(<4 x i8> %src) {
+; CHECK-LABEL: define void @zext_v4i8_all_lanes_used(
+; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i8> %src to <4 x i32>
+  %ext.0 = extractelement <4 x i32> %ext9, i64 0
+  %ext.1 = extractelement <4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <4 x i32> %ext9, i64 2
+  %ext.3 = extractelement <4 x i32> %ext9, i64 3
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
+
+define void @sext_v4i8_all_lanes_used(<4 x i8> %src) {
+; CHECK-LABEL: define void @sext_v4i8_all_lanes_used(
+; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = sext <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = sext <4 x i8> %src to <4 x i32>
+  %ext.0 = extractelement <4 x i32> %ext9, i64 0
+  %ext.1 = extractelement <4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <4 x i32> %ext9, i64 2
+  %ext.3 = extractelement <4 x i32> %ext9, i64 3
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
+
+
+define void @zext_v4i8_3_lanes_used_1(<4 x i8> %src) {
+; CHECK-LABEL: define void @zext_v4i8_3_lanes_used_1(
+; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i8> %src to <4 x i32>
+  %ext.1 = extractelement <4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <4 x i32> %ext9, i64 2
+  %ext.3 = extractelement <4 x i32> %ext9, i64 3
+
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
+
+define void @zext_v4i8_3_lanes_used_2(<4 x i8> %src) {
+; CHECK-LABEL: define void @zext_v4i8_3_lanes_used_2(
+; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i8> %src to <4 x i32>
+  %ext.0 = extractelement <4 x i32> %ext9, i64 0
+  %ext.1 = extractelement <4 x i32> %ext9, i64 1
+  %ext.3 = extractelement <4 x i32> %ext9, i64 3
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
+
+define void @zext_v4i8_2_lanes_used_1(<4 x i8> %src) {
+; CHECK-LABEL: define void @zext_v4i8_2_lanes_used_1(
+; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i8> %src to <4 x i32>
+  %ext.1 = extractelement <4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <4 x i32> %ext9, i64 2
+
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  ret void
+}
+
+define void @zext_v4i8_2_lanes_used_2(<4 x i8> %src) {
+; CHECK-LABEL: define void @zext_v4i8_2_lanes_used_2(
+; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i8> %src to <4 x i32>
+  %ext.0 = extractelement <4 x i32> %ext9, i64 0
+  %ext.2 = extractelement <4 x i32> %ext9, i64 2
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.2)
+  ret void
+}
+
+define void @zext_v4i8_all_lanes_used_noundef(<4 x i8> noundef %src) {
+; CHECK-LABEL: define void @zext_v4i8_all_lanes_used_noundef(
+; CHECK-SAME: <4 x i8> noundef [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i8> %src to <4 x i32>
+  %ext.0 = extractelement <4 x i32> %ext9, i64 0
+  %ext.1 = extractelement <4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <4 x i32> %ext9, i64 2
+  %ext.3 = extractelement <4 x i32> %ext9, i64 3
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
+
+define void @zext_v4i8_all_lanes_used_zext_other_users(<4 x i8> %src) {
+; CHECK-LABEL: define void @zext_v4i8_all_lanes_used_zext_other_users(
+; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.v4i32(<4 x i32> [[EXT9]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i8> %src to <4 x i32>
+  %ext.0 = extractelement <4 x i32> %ext9, i64 0
+  %ext.1 = extractelement <4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <4 x i32> %ext9, i64 2
+  %ext.3 = extractelement <4 x i32> %ext9, i64 3
+
+  call void @use.v4i32(<4 x i32> %ext9)
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
+
+define void @zext_v4i16_all_lanes_used(<4 x i16> %src) {
+; CHECK-LABEL: define void @zext_v4i16_all_lanes_used(
+; CHECK-SAME: <4 x i16> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i16> [[SRC]] to <4 x i64>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i64> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i64> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i64> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i64> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i64(i64 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i64(i64 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i64(i64 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i64(i64 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <4 x i16> %src to <4 x i64>
+  %ext.0 = extractelement <4 x i64> %ext9, i64 0
+  %ext.1 = extractelement <4 x i64> %ext9, i64 1
+  %ext.2 = extractelement <4 x i64> %ext9, i64 2
+  %ext.3 = extractelement <4 x i64> %ext9, i64 3
+
+  call void @use.i64(i64 %ext.0)
+  call void @use.i64(i64 %ext.1)
+  call void @use.i64(i64 %ext.2)
+  call void @use.i64(i64 %ext.3)
+  ret void
+}
+
+define void @zext_v2i32_all_lanes_used(<2 x i32> %src) {
+; CHECK-LABEL: define void @zext_v2i32_all_lanes_used(
+; CHECK-SAME: <2 x i32> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <2 x i32> [[SRC]] to <2 x i64>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <2 x i64> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <2 x i64> [[EXT9]], i64 1
+; CHECK-NEXT:    call void @use.i64(i64 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i64(i64 [[EXT_1]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <2 x i32> %src to <2 x i64>
+  %ext.0 = extractelement <2 x i64> %ext9, i64 0
+  %ext.1 = extractelement <2 x i64> %ext9, i64 1
+
+  call void @use.i64(i64 %ext.0)
+  call void @use.i64(i64 %ext.1)
+  ret void
+}
diff --git a/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-ignore-preferred-alignment.ll b/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-ignore-preferred-alignment.ll
new file mode 100644
index 000000000000..6bee92301e33
--- /dev/null
+++ b/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-ignore-preferred-alignment.ll
@@ -0,0 +1,110 @@
+;; Demonstrate that the ABI alignment is used over the preferred alignment.
+;;
+;; In both runs, pointers are 32-bit but we can only store the function returing
+;; the 64-bit constant in the vtable if the ABI alignment for an i64 is 32 since
+;; we cannot guarantee a 64-bit ABI alignment if the vtable is 32-bit aligned.
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility --data-layout="e-p:32:32-i64:32:64" %s | FileCheck %s --check-prefixes=COMMON,ABI32
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility --data-layout="e-p:32:32-i64:64:64" %s | FileCheck %s --check-prefixes=COMMON,ABI64
+
+; ABI32:      [[VT6DATA:@[^ ]*]] = {{.*}} { [4 x i8], [2 x ptr], [8 x i8] } 
+; ABI32-SAME:   [8 x i8] c"\05\00\00\00\00\00\00\00"
+; ABI64:      [[VT6DATA:@[^ ]*]] = {{.*}} { [4 x i8], [2 x ptr], [0 x i8] } 
+; ABI64-SAME:   zeroinitializer
+; COMMON-SAME: }, !type [[T:![0-9]+]]
+@vt6 = constant [2 x ptr] [
+ptr @vf10i8,
+ptr @vf5i64
+], !type !1
+
+; ABI32:      [[VT6DATA:@[^ ]*]] = {{.*}} { [4 x i8], [2 x ptr], [8 x i8] } 
+; ABI32-SAME:   [8 x i8] c"\06\00\00\00\00\00\00\00"
+; ABI64:      [[VT6DATA:@[^ ]*]] = {{.*}} { [4 x i8], [2 x ptr], [0 x i8] } 
+; ABI64-SAME:   zeroinitializer
+; COMMON-SAME: }, !type [[T]]
+@vt7 = constant [2 x ptr] [
+ptr @vf9i8,
+ptr @vf6i64
+], !type !1
+
+define i1 @vf0i1(ptr %this) readnone {
+  ret i1 0
+}
+
+define i1 @vf1i1(ptr %this) readnone {
+  ret i1 1
+}
+
+define i8 @vf0i8(ptr %this) readnone {
+  ret i8 2
+}
+
+define i8 @vf1i8(ptr %this) readnone {
+  ret i8 3
+}
+
+define i32 @vf1i32(ptr %this) readnone {
+  ret i32 1
+}
+
+define i32 @vf2i32(ptr %this) readnone {
+  ret i32 2
+}
+
+define i32 @vf3i32(ptr %this) readnone {
+  ret i32 3
+}
+
+define i32 @vf4i32(ptr %this) readnone {
+  ret i32 4
+}
+
+define i64 @vf5i64(ptr %this) readnone {
+  ret i64 5
+}
+
+define i64 @vf6i64(ptr %this) readnone {
+  ret i64 6
+}
+
+define i8 @vf9i8(ptr %this) readnone {
+  ret i8 10
+}
+
+define i8 @vf10i8(ptr %this) readnone {
+  ret i8 11
+}
+
+; COMMON-LABEL: define i8 @call0(
+define i8 @call0(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x ptr], ptr %vtable, i32 0, i32 0
+  %fptr = load ptr, ptr %fptrptr
+  %result = call i8 %fptr(ptr %obj)
+  ret i8 %result
+  ; COMMON: [[VTGEP:%[^ ]*]] = getelementptr i8, ptr %vtable, i32 -1
+  ; COMMON: [[VTLOAD:%[^ ]*]] = load i8, ptr [[VTGEP]]
+  ; COMMON: ret i8 [[VTLOAD]]
+}
+
+; COMMON-LABEL: define i64 @call1(
+define i64 @call1(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x ptr], ptr %vtable, i32 0, i32 1
+  %fptr = load ptr, ptr %fptrptr
+  %result = call i64 %fptr(ptr %obj)
+  ret i64 %result
+  ; ABI32: [[VTGEP:%[^ ]*]] = getelementptr i8, ptr %vtable, i32 8
+  ; ABI32-NEXT: [[VTLOAD:%[^ ]*]] = load i64, ptr [[VTGEP]]
+  ; ABI64: [[VTGEP:%[^ ]*]] = getelementptr [3 x ptr], ptr %vtable, i32 0, i32 1
+  ; ABI64-NEXT: [[FUNC:%[^ ]*]] = load ptr, ptr [[VTGEP]], align 4
+  ; ABI64-NEXT: [[VTLOAD:%[^ ]*]] = call i64 [[FUNC]](ptr %obj)
+  ; COMMON-NEXT: ret i64 [[VTLOAD]]
+}
+
+; COMMON: [[T]] = !{i32 4, !"typeid"}
+
+!1 = !{i32 0, !"typeid"}
diff --git a/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-32.ll b/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-32.ll
index ab76f2c22e34..314b1ee13a80 100644
--- a/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-32.ll
+++ b/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-32.ll
@@ -1,7 +1,9 @@
-; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility %s | FileCheck %s
-
 ;; This target uses 32-bit sized and aligned pointers.
-target datalayout = "e-p:32:32"
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility --data-layout="e-p:32:32-i64:64:64" %s | FileCheck %s
+
+;; The tests should be the exact same even with different preferred alignments since
+;; the ABI alignment is used.
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility --data-layout="e-p:32:32-i64:64:128" %s | FileCheck %s
 
 ;; Constant propagation should be agnostic towards sections.
 ;; Also the new global should be in the original vtable's section.
diff --git a/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-64.ll b/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-64.ll
index c83fbc6ed5a1..9f39b65e2665 100644
--- a/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-64.ll
+++ b/external/llvm-project/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-small-alignment-64.ll
@@ -1,7 +1,9 @@
-; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility %s | FileCheck %s
-
 ;; This target uses 64-bit sized and aligned pointers.
-target datalayout = "e-p:64:64"
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility --data-layout="e-p:64:64-i64:64:64" %s | FileCheck %s
+
+;; The tests should be the exact same even with different preferred alignments since
+;; the ABI alignment is used.
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility --data-layout="e-p:64:64-i64:64:128" %s | FileCheck %s
 
 ;; Constant propagation should be agnostic towards sections.
 ;; Also the new global should be in the original vtable's section.
diff --git a/external/llvm-project/llvm/test/Verifier/RemoveDI/invalid-dbg-declare-operands.ll b/external/llvm-project/llvm/test/Verifier/RemoveDI/invalid-dbg-declare-operands.ll
new file mode 100644
index 000000000000..cdc9d8df82aa
--- /dev/null
+++ b/external/llvm-project/llvm/test/Verifier/RemoveDI/invalid-dbg-declare-operands.ll
@@ -0,0 +1,46 @@
+; RUN: llvm-as %s -o - 2>&1 | FileCheck %s
+; CHECK: invalid #dbg record expression
+;
+; Fossilised debug-info with only two arguments to dbg.declare have been
+; spotted in LLVMs test suite (debug-info-always-inline.ll), test that this
+; does not cause a crash. LLVM needs to be able to autoupgrade invalid
+; dbg.declares to invalid #dbg_declares because this occurs before the
+; Verifier runs.
+
+; ModuleID = 'out.ll'
+source_filename = "llvm/test/DebugInfo/Generic/debug-info-always-inline.ll"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+; Function Attrs: alwaysinline nounwind sspstrong
+define i32 @_Z3foov() !dbg !7 {
+entry:
+  %sum = alloca i32, align 4, !dbg !11
+  call void @llvm.dbg.declare(metadata ptr %sum,  metadata !26), !dbg !11
+  ret i32 0, !dbg !15
+}
+
+declare void @_Z3barv()
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+!llvm.dbg.cu = !{!3}
+!llvm.debugify = !{!5, !6}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{!"clang version 3.6.0 (217844)"}
+!3 = distinct !DICompileUnit(language: DW_LANG_C, file: !4, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!4 = !DIFile(filename: "/fast/fs/llvm-main/llvm/test/DebugInfo/Generic/debug-info-always-inline.ll", directory: "/")
+!5 = !{i32 14}
+!6 = !{i32 7}
+!7 = distinct !DISubprogram(name: "_Z3foov", linkageName: "_Z3foov", scope: null, file: !4, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !3, retainedNodes: !9)
+!8 = !DISubroutineType(types: !9)
+!9 = !{}
+!11 = !DILocation(line: 2, column: 1, scope: !7)
+!15 = !DILocation(line: 6, column: 1, scope: !7)
+!25 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!26 = !DILocalVariable(name: "b", scope: !7, file: !4, line: 1234, type: !25)
+
diff --git a/external/llvm-project/llvm/test/Verifier/alloc-variant-zeroed.ll b/external/llvm-project/llvm/test/Verifier/alloc-variant-zeroed.ll
new file mode 100644
index 000000000000..03cd7be4d9ac
--- /dev/null
+++ b/external/llvm-project/llvm/test/Verifier/alloc-variant-zeroed.ll
@@ -0,0 +1,19 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: 'alloc-variant-zeroed' must not be empty
+declare ptr @a(i64) "alloc-variant-zeroed"=""
+
+; CHECK: 'alloc-variant-zeroed' must not be empty
+declare ptr @b(i64) "alloc-variant-zeroed"=""
+
+; CHECK: 'alloc-variant-zeroed' must name a function belonging to the same 'alloc-family'
+declare ptr @c(i64) "alloc-variant-zeroed"="c_zeroed" "alloc-family"="C"
+declare ptr @c_zeroed(i64)
+
+; CHECK: 'alloc-variant-zeroed' must name a function with 'allockind("zeroed")'
+declare ptr @d(i64) "alloc-variant-zeroed"="d_zeroed" "alloc-family"="D"
+declare ptr @d_zeroed(i64) "alloc-family"="D"
+
+; CHECK: 'alloc-variant-zeroed' must name a function with the same signature
+declare ptr @e(i64) "alloc-variant-zeroed"="e_zeroed" "alloc-family"="E"
+declare ptr @e_zeroed(i64, i64) "alloc-family"="E" allockind("zeroed")
diff --git a/external/llvm-project/llvm/test/Verifier/dbg-declare-invalid-debug-loc.ll b/external/llvm-project/llvm/test/Verifier/dbg-declare-invalid-debug-loc.ll
new file mode 100644
index 000000000000..c521a9b8eb11
--- /dev/null
+++ b/external/llvm-project/llvm/test/Verifier/dbg-declare-invalid-debug-loc.ll
@@ -0,0 +1,42 @@
+; RUN: opt %s -o /dev/null -S 2>&1 | FileCheck %s
+;
+; The last dbg.declare intrinsic in this file has an illegal DILocation -- this
+; needs to pass through the autoupgrade to #dbg_declare process and then get
+; caught by the verifier.
+;
+; CHECK:      invalid #dbg record DILocation
+; CHECK-NEXT: #dbg_declare(ptr %1, ![[VAR:[0-9]+]], !DIExpression(), ![[PROG:[0-9]+]])
+; CHECK-NEXT: ![[PROG]] = distinct !DISubprogram(name: "IgnoreIntrinsicTest",
+; CHECK-NEXT: label %0
+; CHECK-NEXT: ptr @IgnoreIntrinsicTest
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define i32 @IgnoreIntrinsicTest() !dbg !10 {
+  %1 = alloca i32, align 4
+  call void @llvm.dbg.declare(metadata ptr %1, metadata !14, metadata !DIExpression()), !dbg !10
+  store volatile i32 1, ptr %1, align 4
+  %2 = load volatile i32, ptr %1, align 4
+  %3 = mul nsw i32 %2, 42
+  ret i32 %3
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.4 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !5, globals: !5, imports: !5)
+!1 = !DIFile(filename: "<unknown>", directory: "/Users/matt/ryan_bug")
+!2 = !{!3}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, scope: !4, file: !1, line: 20, size: 32, align: 32, elements: !6)
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", file: !1, line: 19, size: 8, align: 8, elements: !5)
+!5 = !{}
+!6 = !{!7}
+!7 = !DIEnumerator(name: "max_frame_size", value: 0)
+!8 = !{i32 2, !"Dwarf Version", i32 2}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = distinct !DISubprogram(name: "IgnoreIntrinsicTest", linkageName: "IgnoreIntrinsicTest", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !5)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!13}
+!13 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DILocalVariable(name: "x", scope: !10, file: !1, line: 2, type: !13)
+!15 = !DILocation(line: 2, column: 16, scope: !10)
diff --git a/external/llvm-project/llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll b/external/llvm-project/llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll
index 652e6667bfc5..1a28f0ec519f 100644
--- a/external/llvm-project/llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll
+++ b/external/llvm-project/llvm/test/Verifier/diexpression-entry-value-llvm-ir.ll
@@ -1,9 +1,9 @@
 ; RUN: llvm-as -disable-output <%s 2>&1| FileCheck %s
 
-; CHECK-NOT: llvm.dbg.value
+; CHECK-NOT: #dbg_value
 ; CHECK: Entry values are only allowed in MIR unless they target a swiftasync Argument
-; CHECK: call void @llvm.dbg.value(metadata i32 %param, metadata !{{.*}}, metadata !DIExpression(DW_OP_LLVM_entry_value, 1))
-; CHECK-NOT: llvm.dbg.value
+; CHECK: #dbg_value(i32 %param, !{{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1),
+; CHECK-NOT: #dbg_value
 ; CHECK-NOT: Entry values are only allowed
 ; CHECK: warning: ignoring invalid debug info
 
diff --git a/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-address.ll b/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-address.ll
index 219f9ca0a667..251526b4c321 100644
--- a/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-address.ll
+++ b/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-address.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s
-; CHECK: invalid llvm.dbg.declare intrinsic address/value
-; CHECK-NEXT: call void @llvm.dbg.declare({{.*}})
+; CHECK: invalid #dbg record address/value
+; CHECK-NEXT: #dbg_declare({{.*}})
 ; CHECK-NEXT: !""
 ; CHECK: warning: ignoring invalid debug info
 
diff --git a/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-expression.ll b/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-expression.ll
index 671ec2178008..de65bb570677 100644
--- a/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-expression.ll
+++ b/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-expression.ll
@@ -1,7 +1,6 @@
 ; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s
-; CHECK: invalid llvm.dbg.declare intrinsic expression
-; CHECK-NEXT: call void @llvm.dbg.declare({{.*}})
-; CHECK-NEXT: !""
+; CHECK: invalid #dbg record expression
+; CHECK-NEXT: #dbg_declare({{.*}})
 ; CHECK: warning: ignoring invalid debug info
 
 define void @foo(i32 %a) {
diff --git a/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-variable.ll b/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-variable.ll
index 4f0ae4daa822..601fab190d36 100644
--- a/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-variable.ll
+++ b/external/llvm-project/llvm/test/Verifier/llvm.dbg.declare-variable.ll
@@ -1,13 +1,16 @@
 ; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s
-; CHECK: invalid llvm.dbg.declare intrinsic variable
-; CHECK-NEXT: call void @llvm.dbg.declare({{.*}})
-; CHECK-NEXT: !""
+; CHECK: invalid #dbg record variable
+; CHECK-NEXT: #dbg_declare({{.*}})
+; CHECK-NEXT: DISubprogram
 ; CHECK: warning: ignoring invalid debug info
 
+;; This test ensures we report an illegal variable as illegal, but also that
+;; the illegal MDNode is printed out (DISubprogram) to help localise.
+
 define void @foo(i32 %a) {
 entry:
   %s = alloca i32
-  call void @llvm.dbg.declare(metadata ptr %s, metadata !"", metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  call void @llvm.dbg.declare(metadata ptr %s, metadata !1, metadata !DIExpression()), !dbg !DILocation(scope: !1)
   ret void
 }
 
diff --git a/external/llvm-project/llvm/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll b/external/llvm-project/llvm/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll
index 5d82f490e055..b1e22b20d086 100644
--- a/external/llvm-project/llvm/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll
+++ b/external/llvm-project/llvm/test/Verifier/llvm.dbg.intrinsic-dbg-attachment.ll
@@ -5,8 +5,8 @@ entry:
       metadata ptr undef,
       metadata !DILocalVariable(scope: !1),
       metadata !DIExpression())
-; CHECK-LABEL: llvm.dbg.value intrinsic requires a !dbg attachment
-; CHECK-NEXT: call void @llvm.dbg.value({{.*}})
+; CHECK-LABEL: invalid #dbg record DILocation
+; CHECK-NEXT: #dbg_value({{.*}})
 ; CHECK-NEXT: label %entry
 ; CHECK-NEXT: ptr @foo
 
@@ -14,8 +14,8 @@ entry:
       metadata ptr undef,
       metadata !DILocalVariable(scope: !1),
       metadata !DIExpression())
-; CHECK-LABEL: llvm.dbg.declare intrinsic requires a !dbg attachment
-; CHECK-NEXT: call void @llvm.dbg.declare({{.*}})
+; CHECK-LABEL: invalid #dbg record DILocation
+; CHECK-NEXT: #dbg_declare({{.*}})
 ; CHECK-NEXT: label %entry
 ; CHECK-NEXT: ptr @foo
 
@@ -24,8 +24,8 @@ entry:
       metadata !DILocalVariable(scope: !1),
       metadata !DIExpression()),
     !dbg !DILocation(scope: !2)
-; CHECK-LABEL: mismatched subprogram between llvm.dbg.value variable and !dbg attachment
-; CHECK-NEXT: call void @llvm.dbg.value({{[^,]+}}, metadata ![[VAR:[0-9]+]], {{[^,]+}}), !dbg ![[LOC:[0-9]+]]
+; CHECK-LABEL: mismatched subprogram between #dbg record variable and DILocation
+; CHECK-NEXT: #dbg_value({{[^,]+}}, ![[VAR:[0-9]+]], {{[^,]+}}, ![[LOC:[0-9]+]])
 ; CHECK-NEXT: label %entry
 ; CHECK-NEXT: ptr @foo
 ; CHECK-NEXT: ![[VAR]] = !DILocalVariable({{.*}}scope: ![[VARSP:[0-9]+]]
@@ -38,8 +38,8 @@ entry:
       metadata !DILocalVariable(scope: !1),
       metadata !DIExpression()),
     !dbg !DILocation(scope: !2)
-; CHECK-LABEL: mismatched subprogram between llvm.dbg.declare variable and !dbg attachment
-; CHECK-NEXT: call void @llvm.dbg.declare({{[^,]+}}, metadata ![[VAR:[0-9]+]], {{.*[^,]+}}), !dbg ![[LOC:[0-9]+]]
+; CHECK-LABEL: mismatched subprogram between #dbg record variable and DILocation
+; CHECK-NEXT: #dbg_declare({{[^,]+}}, ![[VAR:[0-9]+]], {{.*[^,]+}}, ![[LOC:[0-9]+]])
 ; CHECK-NEXT: label %entry
 ; CHECK-NEXT: ptr @foo
 ; CHECK-NEXT: ![[VAR]] = !DILocalVariable({{.*}}scope: ![[VARSP:[0-9]+]]
diff --git a/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-expression.ll b/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-expression.ll
index cc45af2e8e7c..92fd2add700e 100644
--- a/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-expression.ll
+++ b/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-expression.ll
@@ -1,7 +1,6 @@
 ; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s
-; CHECK: invalid llvm.dbg.value intrinsic expression
-; CHECK-NEXT: call void @llvm.dbg.value({{.*}})
-; CHECK-NEXT: !""
+; CHECK: invalid #dbg record expression
+; CHECK-NEXT: #dbg_value({{.*}})
 ; CHECK: warning: ignoring invalid debug info
 
 define void @foo(i32 %a) {
diff --git a/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-value.ll b/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-value.ll
index 8b0ec1fed05c..c390e530653c 100644
--- a/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-value.ll
+++ b/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-value.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s
-; CHECK: invalid llvm.dbg.value intrinsic address/value
-; CHECK-NEXT: call void @llvm.dbg.value({{.*}})
+; CHECK: invalid #dbg record address/value
+; CHECK-NEXT: #dbg_value({{.*}})
 ; CHECK-NEXT: !""
 ; CHECK: warning: ignoring invalid debug info
 
diff --git a/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-variable.ll b/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-variable.ll
index 4388e20797ce..603a4b5c47e7 100644
--- a/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-variable.ll
+++ b/external/llvm-project/llvm/test/Verifier/llvm.dbg.value-variable.ll
@@ -1,7 +1,6 @@
 ; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s
-; CHECK: invalid llvm.dbg.value intrinsic variable
-; CHECK-NEXT: call void @llvm.dbg.value({{.*}})
-; CHECK-NEXT: !""
+; CHECK: invalid #dbg record variable
+; CHECK-NEXT: #dbg_value({{.*}})
 ; CHECK: warning: ignoring invalid debug info
 
 define void @foo(i32 %a) {
diff --git a/external/llvm-project/llvm/test/Verifier/target-ext-vector-invalid.ll b/external/llvm-project/llvm/test/Verifier/target-ext-vector-invalid.ll
new file mode 100644
index 000000000000..59e3e78276a0
--- /dev/null
+++ b/external/llvm-project/llvm/test/Verifier/target-ext-vector-invalid.ll
@@ -0,0 +1,8 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: invalid vector element type
+
+define void @bad() {
+  %v = alloca <2 x target("spirv.Image")>
+  ret void
+}
\ No newline at end of file
diff --git a/external/llvm-project/llvm/test/Verifier/target-ext-vector.ll b/external/llvm-project/llvm/test/Verifier/target-ext-vector.ll
new file mode 100644
index 000000000000..433715bc6c64
--- /dev/null
+++ b/external/llvm-project/llvm/test/Verifier/target-ext-vector.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=verify -S %s | FileCheck %s
+
+define <2 x target("llvm.test.vectorelement")> @vec_ops(<2 x target("llvm.test.vectorelement")> %x) {
+; CHECK-LABEL: define <2 x target("llvm.test.vectorelement")> @vec_ops(
+; CHECK-SAME: <2 x target("llvm.test.vectorelement")> [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = alloca <2 x target("llvm.test.vectorelement")>{{.*}}
+; CHECK-NEXT:    store <2 x target("llvm.test.vectorelement")> [[X]], ptr [[A]], {{.*}}
+; CHECK-NEXT:    [[LOAD:%.*]] = load <2 x target("llvm.test.vectorelement")>, ptr [[A]], {{.*}}
+; CHECK-NEXT:    [[ELT:%.*]] = extractelement <2 x target("llvm.test.vectorelement")> [[LOAD]], i64 0
+; CHECK-NEXT:    [[RES:%.*]] = insertelement <2 x target("llvm.test.vectorelement")> poison, target("llvm.test.vectorelement") [[ELT]], i64 1
+; CHECK-NEXT:    ret <2 x target("llvm.test.vectorelement")> [[RES]]
+;
+  %a = alloca <2 x target("llvm.test.vectorelement")>
+  store <2 x target("llvm.test.vectorelement")> %x, ptr %a
+  %load = load <2 x target("llvm.test.vectorelement")>, ptr %a
+  %elt = extractelement <2 x target("llvm.test.vectorelement")> %load, i64 0
+  %res = insertelement <2 x target("llvm.test.vectorelement")> poison, target("llvm.test.vectorelement") %elt, i64 1
+  ret <2 x target("llvm.test.vectorelement")> %res
+}
diff --git a/external/llvm-project/llvm/test/tools/dxil-dis/debug-info.ll b/external/llvm-project/llvm/test/tools/dxil-dis/debug-info.ll
index b35258f3797f..f96edca223b2 100644
--- a/external/llvm-project/llvm/test/tools/dxil-dis/debug-info.ll
+++ b/external/llvm-project/llvm/test/tools/dxil-dis/debug-info.ll
@@ -1,4 +1,4 @@
-; RUN: llc --filetype=obj %s -o  - -experimental-debuginfo-iterators=false | dxil-dis -o - | FileCheck %s
+; RUN: llc --filetype=obj %s -o  - | dxil-dis -o - | FileCheck %s
 ; Note: LLVM has soft disabled experimental-debuginfo-iterators in commit 6a45fce
 ; XFAIL: *
 
diff --git a/external/llvm-project/llvm/test/tools/llvm-cgdata/empty.test b/external/llvm-project/llvm/test/tools/llvm-cgdata/empty.test
index bea78d512a6d..0d2b0e848a2c 100644
--- a/external/llvm-project/llvm/test/tools/llvm-cgdata/empty.test
+++ b/external/llvm-project/llvm/test/tools/llvm-cgdata/empty.test
@@ -16,7 +16,7 @@ RUN: llvm-cgdata --show %t_emptyheader.cgdata | count 0
 
 # The version number appears when asked, as it's in the header
 RUN: llvm-cgdata --show --cgdata-version %t_emptyheader.cgdata | FileCheck %s --check-prefix=VERSION
-VERSION: Version: 2
+VERSION: Version: 3
 
 # When converting a binary file (w/ the header only) to a text file, it's an empty file as the text format does not have an explicit header.
 RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0
@@ -30,7 +30,7 @@ RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0
 #   uint64_t StableFunctionMapOffset;
 # }
 RUN: printf '\xffcgdata\x81' > %t_header.cgdata
-RUN: printf '\x02\x00\x00\x00' >> %t_header.cgdata
+RUN: printf '\x03\x00\x00\x00' >> %t_header.cgdata
 RUN: printf '\x00\x00\x00\x00' >> %t_header.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata
diff --git a/external/llvm-project/llvm/test/tools/llvm-cgdata/error.test b/external/llvm-project/llvm/test/tools/llvm-cgdata/error.test
index 2caa3aef4039..92ff484e31ca 100644
--- a/external/llvm-project/llvm/test/tools/llvm-cgdata/error.test
+++ b/external/llvm-project/llvm/test/tools/llvm-cgdata/error.test
@@ -22,9 +22,9 @@ RUN: printf '\xffcgdata\x81' > %t_corrupt.cgdata
 RUN: not llvm-cgdata --show %t_corrupt.cgdata 2>&1 | FileCheck %s  --check-prefix=CORRUPT
 CORRUPT: {{.}}cgdata: invalid codegen data (file header is corrupt)
 
-# The current version 2 while the header says 3.
+# The current version 3 while the header says 4.
 RUN: printf '\xffcgdata\x81' > %t_version.cgdata
-RUN: printf '\x03\x00\x00\x00' >> %t_version.cgdata
+RUN: printf '\x04\x00\x00\x00' >> %t_version.cgdata
 RUN: printf '\x00\x00\x00\x00' >> %t_version.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata
diff --git a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test
index f7e078ba8efa..b060872113b1 100644
--- a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test
+++ b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test
@@ -63,4 +63,4 @@ CHECK-NEXT:  Mergeable function Count: 0
 
 ;--- merge-both-template.ll
 @.data1 = private unnamed_addr constant [72 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_outline"
-@.data2 = private unnamed_addr constant [60 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data2 = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test
index c1881bc870aa..293608632102 100644
--- a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test
+++ b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test
@@ -65,7 +65,7 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-1-template.ll
-@.data = private unnamed_addr constant [60 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
 
 ;--- raw-2.cgtext
 :stable_function_map
@@ -80,4 +80,4 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-2-template.ll
-@.data = private unnamed_addr constant [60 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test
index 301ee6dcf21c..d2965456a199 100644
--- a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test
+++ b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test
@@ -74,5 +74,5 @@ MAP-NEXT: ...
 ; In an linked executable (as opposed to an object file), cgdata in __llvm_merge might be concatenated.
 ; Although this is not a typical workflow, we simply support this case to parse cgdata that is concatenated.
 ; In other words, the following two trees are encoded back-to-back in a binary format.
-@.data1 = private unnamed_addr constant [60 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
-@.data2 = private unnamed_addr constant [60 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data1 = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data2 = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test
index 98a91487aa84..8277e3272d77 100644
--- a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test
+++ b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test
@@ -61,7 +61,7 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-1-template.ll
-@.data = private unnamed_addr constant [60 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
 
 ;--- raw-2.cgtext
 :stable_function_map
@@ -76,4 +76,4 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-2-template.ll
-@.data = private unnamed_addr constant [60 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test
index 2075face72ab..9469f1cbda33 100644
--- a/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test
+++ b/external/llvm-project/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test
@@ -33,4 +33,4 @@ CHECK-NEXT:  Mergeable function Count: 0
 ...
 
 ;--- merge-single-template.ll
-@.data = private unnamed_addr constant [60 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
diff --git a/external/llvm-project/llvm/test/tools/llvm-cov/showLineExecutionCounts-lcov-baseline.test b/external/llvm-project/llvm/test/tools/llvm-cov/showLineExecutionCounts-lcov-baseline.test
new file mode 100644
index 000000000000..bce886bdf510
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-cov/showLineExecutionCounts-lcov-baseline.test
@@ -0,0 +1,37 @@
+// FULL: SF:{{.*}}showLineExecutionCounts.cpp
+// FULL: FN:6,main
+// FULL: FNDA:0,main
+// FULL: FNF:1
+// FULL: FNH:0
+int main() {                              // FULL: DA:[[@LINE]],0
+  int x = 0;                              // FULL: DA:[[@LINE]],0
+                                          // FULL: DA:[[@LINE]],0
+  if (x) {                                // FULL: DA:[[@LINE]],0
+    x = 0;                                // FULL: DA:[[@LINE]],0
+  } else {                                // FULL: DA:[[@LINE]],0
+    x = 1;                                // FULL: DA:[[@LINE]],0
+  }                                       // FULL: DA:[[@LINE]],0
+                                          // FULL: DA:[[@LINE]],0
+  for (int i = 0; i < 100; ++i) {         // FULL: DA:[[@LINE]],0
+    x = 1;                                // FULL: DA:[[@LINE]],0
+  }                                       // FULL: DA:[[@LINE]],0
+                                          // FULL: DA:[[@LINE]],0
+  x = x < 10 ? x + 1 : x - 1;             // FULL: DA:[[@LINE]],0
+  x = x > 10 ?                            // FULL: DA:[[@LINE]],0
+        x - 1:                            // FULL: DA:[[@LINE]],0
+        x + 1;                            // FULL: DA:[[@LINE]],0
+                                          // FULL: DA:[[@LINE]],0
+  return 0;                               // FULL: DA:[[@LINE]],0
+}                                         // FULL: DA:[[@LINE]],0
+// FULL: LF:20
+// FULL: LH:0
+// FULL: end_of_record
+// RUN: llvm-cov export -format=lcov %S/Inputs/lineExecutionCounts.covmapping -empty-profile %s | FileCheck -check-prefixes=FULL %s
+
+// RUN: llvm-cov export -format=lcov -summary-only %S/Inputs/lineExecutionCounts.covmapping -empty-profile %s | FileCheck -check-prefixes=SUMMARYONLY %s
+// SUMMARYONLY: SF:{{.*}}showLineExecutionCounts.cpp
+// SUMMARYONLY: FNF:1
+// SUMMARYONLY: FNH:0
+// SUMMARYONLY: LF:20
+// SUMMARYONLY: LH:0
+// SUMMARYONLY: end_of_record
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/02-coff-logical-lines.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/02-coff-logical-lines.test
index d2058c6b6be3..7bd816abe905 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/02-coff-logical-lines.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/02-coff-logical-lines.test
@@ -15,7 +15,7 @@
 ; The logical views shows the intermixed lines and assembler instructions,
 ; allowing to compare the code generated by the different toolchains.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --print=lines,instructions \
 ; RUN:                         %p/Inputs/hello-world-codeview-clang.o \
 ; RUN:                         %p/Inputs/hello-world-codeview-msvc.o 2>&1 | \
@@ -26,6 +26,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'hello-world.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]               {Function} extern not_inlined 'main' -> 'int'
 ; ONE-NEXT: [003]     4           {Line}
 ; ONE-NEXT: [003]                 {Code} 'subq	$0x28, %rsp'
@@ -43,6 +44,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'hello-world.cpp'
 ; ONE-NEXT: [002]               {Producer} 'Microsoft (R) Optimizing Compiler'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]               {Function} extern not_inlined 'main' -> 'int'
 ; ONE-NEXT: [003]     4           {Line}
 ; ONE-NEXT: [003]                 {Code} 'subq	$0x28, %rsp'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/03-coff-incorrect-lexical-scope-typedef.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/03-coff-incorrect-lexical-scope-typedef.test
index b7c646a10301..25f765b48f7d 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/03-coff-incorrect-lexical-scope-typedef.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/03-coff-incorrect-lexical-scope-typedef.test
@@ -30,7 +30,7 @@
 ; emits both typedefs at the same lexical scope (3), which is wrong.
 ; GCC and MSVC emit correct lexical scope for both typedefs.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --output-sort=kind \
 ; RUN:                         --print=symbols,types,lines \
 ; RUN:                         %p/Inputs/pr-44884-codeview-clang.o \
@@ -42,6 +42,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-44884.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]               {Function} extern not_inlined 'bar' -> 'int'
 ; ONE-NEXT: [003]                 {Parameter} 'Input' -> 'float'
 ; ONE-NEXT: [003]     1           {Line}
@@ -63,6 +64,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-44884.cpp'
 ; ONE-NEXT: [002]               {Producer} 'Microsoft (R) Optimizing Compiler'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]               {Function} extern not_inlined 'bar' -> 'int'
 ; ONE-NEXT: [003]                 {Variable} 'Input' -> 'float'
 ; ONE-NEXT: [003]     1           {Line}
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/04-coff-missing-nested-enumerators.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/04-coff-missing-nested-enumerators.test
index a6862e31ad5d..aecc0914b33a 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/04-coff-missing-nested-enumerators.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/04-coff-missing-nested-enumerators.test
@@ -25,7 +25,7 @@
 ; references to the enumerators 'RED' and 'BLUE'. The CodeView generated
 ; by GCC and MSVC, does include such references.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer,size \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer,size \
 ; RUN:                         --output-sort=name \
 ; RUN:                         --print=symbols,types \
 ; RUN:                         %p/Inputs/pr-46466-codeview-clang.o \
@@ -37,6 +37,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-46466.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]               {Variable} extern 'S' -> 'Struct'
 ; ONE-NEXT: [002]     1         {Struct} 'Struct' [Size = 1]
 ; ONE-NEXT: [003]                 {Member} public 'U' -> 'Union'
@@ -50,6 +51,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-46466.cpp'
 ; ONE-NEXT: [002]               {Producer} 'Microsoft (R) Optimizing Compiler'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]               {Variable} extern 'S' -> 'Struct'
 ; ONE-NEXT: [002]     1         {Struct} 'Struct' [Size = 1]
 ; ONE-NEXT: [003]                 {Member} public 'U' -> 'Union'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/05-coff-incorrect-lexical-scope-variable.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/05-coff-incorrect-lexical-scope-variable.test
index dea30c9f2b37..aa628f70bc50 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/05-coff-incorrect-lexical-scope-variable.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/05-coff-incorrect-lexical-scope-variable.test
@@ -31,7 +31,7 @@
 ; The CodeView generated by MSVC, show those variables at the correct
 ; lexical scope: '3' and '4' respectively.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --output-sort=name \
 ; RUN:                         --print=symbols \
 ; RUN:                         %p/Inputs/pr-43860-codeview-clang.o \
@@ -43,6 +43,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-43860.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]     2         {Function} inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [003]                 {Parameter} '' -> 'int'
 ; ONE-NEXT: [002]               {Function} extern not_inlined 'test' -> 'int'
@@ -59,6 +60,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-43860.cpp'
 ; ONE-NEXT: [002]               {Producer} 'Microsoft (R) Optimizing Compiler'
+; ONE-NEXT: [002]               {Language} 'Cpp'
 ; ONE-NEXT: [002]               {Function} extern declared_inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [003]                 {Block}
 ; ONE-NEXT: [004]                   {Variable} 'Var_2' -> 'int'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/06-coff-full-logical-view.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/06-coff-full-logical-view.test
index 42c6f124ce70..34b9de71084d 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/06-coff-full-logical-view.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/COFF/06-coff-full-logical-view.test
@@ -28,6 +28,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [0x0000000000][001]              {CompileUnit} 'test.cpp'
 ; ONE-NEXT: [0x0000000000][002]                {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [0x0000000000][002]                {Language} 'Cpp'
 ; ONE-NEXT:                                    {Directory} 'test.cpp'
 ; ONE-NEXT:                                    {Directory} 'x:/tests/input'
 ; ONE-NEXT:                                    {File} 'general'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/02-dwarf-logical-lines.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/02-dwarf-logical-lines.test
index 62eaf593a16c..533914f00282 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/02-dwarf-logical-lines.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/02-dwarf-logical-lines.test
@@ -15,7 +15,7 @@
 ; The logical views shows the intermixed lines and assembler instructions,
 ; allowing to compare the code generated by the different toolchains.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --print=lines,instructions \
 ; RUN:                         %p/Inputs/hello-world-dwarf-clang.o \
 ; RUN:                         %p/Inputs/hello-world-dwarf-gcc.o 2>&1 | \
@@ -26,6 +26,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'hello-world.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     3         {Function} extern not_inlined 'main' -> 'int'
 ; ONE-NEXT: [003]     4           {Line}
 ; ONE-NEXT: [003]                 {Code} 'pushq	%rbp'
@@ -48,6 +49,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'hello-world.cpp'
 ; ONE-NEXT: [002]               {Producer} 'GNU C++14 10.3.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus'
 ; ONE-NEXT: [002]     3         {Function} extern not_inlined 'main' -> 'int'
 ; ONE-NEXT: [003]     4           {Line}
 ; ONE-NEXT: [003]                 {Code} 'endbr64'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/03-dwarf-incorrect-lexical-scope-typedef.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/03-dwarf-incorrect-lexical-scope-typedef.test
index f252a9a046f0..dc57d01f3b8b 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/03-dwarf-incorrect-lexical-scope-typedef.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/03-dwarf-incorrect-lexical-scope-typedef.test
@@ -30,7 +30,7 @@
 ; emits both typedefs at the same lexical scope (3), which is wrong.
 ; GCC emit correct lexical scope for both typedefs.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --output-sort=kind \
 ; RUN:                         --print=symbols,types,lines \
 ; RUN:                         %p/Inputs/pr-44884-dwarf-clang.o \
@@ -42,6 +42,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-44884.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     1         {Function} extern not_inlined 'bar' -> 'int'
 ; ONE-NEXT: [003]     1           {Parameter} 'Input' -> 'float'
 ; ONE-NEXT: [003]     1           {Line}
@@ -76,6 +77,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-44884.cpp'
 ; ONE-NEXT: [002]               {Producer} 'GNU C++14 10.3.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus'
 ; ONE-NEXT: [002]     1         {Function} extern not_inlined 'bar' -> 'int'
 ; ONE-NEXT: [003]     1           {Parameter} 'Input' -> 'float'
 ; ONE-NEXT: [003]     1           {Line}
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/04-dwarf-missing-nested-enumerators.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/04-dwarf-missing-nested-enumerators.test
index 15d426f88a37..cd1ee436557f 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/04-dwarf-missing-nested-enumerators.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/04-dwarf-missing-nested-enumerators.test
@@ -25,7 +25,7 @@
 ; references to the enumerators 'RED' and 'BLUE'. The DWARF generated
 ; by GCC, does include such references.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer,size \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer,size \
 ; RUN:                         --output-sort=name \
 ; RUN:                         --print=symbols,types \
 ; RUN:                         %p/Inputs/pr-46466-dwarf-clang.o \
@@ -37,6 +37,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-46466.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     8         {Variable} extern 'S' -> 'Struct'
 ; ONE-NEXT: [002]     1         {Struct} 'Struct' [Size = 1]
 ; ONE-NEXT: [003]     5           {Member} public 'U' -> 'Union'
@@ -46,6 +47,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-46466.cpp'
 ; ONE-NEXT: [002]               {Producer} 'GNU C++14 10.3.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus'
 ; ONE-NEXT: [002]     8         {Variable} extern 'S' -> 'Struct'
 ; ONE-NEXT: [002]     1         {Struct} 'Struct' [Size = 1]
 ; ONE-NEXT: [003]     5           {Member} public 'U' -> 'Union'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test
index 3c3c5dcbda52..97f1d212bac4 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/05-dwarf-incorrect-lexical-scope-variable.test
@@ -31,7 +31,7 @@
 ; The DWARF generated by GCC/Clang show those variables at the correct
 ; lexical scope: '3' and '4' respectively.
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --output-sort=name \
 ; RUN:                         --print=symbols \
 ; RUN:                         %p/Inputs/pr-43860-dwarf-clang.o \
@@ -43,6 +43,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-43860.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     2         {Function} extern inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [003]                 {Block}
 ; ONE-NEXT: [004]     5             {Variable} 'Var_2' -> 'int'
@@ -63,6 +64,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-43860.cpp'
 ; ONE-NEXT: [002]               {Producer} 'GNU C++14 10.3.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus'
 ; ONE-NEXT: [002]     2         {Function} extern declared_inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [003]                 {Block}
 ; ONE-NEXT: [004]     5             {Variable} 'Var_2' -> 'int'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/06-dwarf-full-logical-view.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/06-dwarf-full-logical-view.test
index 1d0843df5250..6616710a1004 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/06-dwarf-full-logical-view.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/06-dwarf-full-logical-view.test
@@ -28,6 +28,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [0x000000000b][001]              {CompileUnit} 'test.cpp'
 ; ONE-NEXT: [0x000000000b][002]                {Producer} 'clang version 15.0.0 {{.*}}'
+; ONE-NEXT: [0x000000000b][002]                {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT:                                    {Directory} '/data/projects/tests/input/general'
 ; ONE-NEXT:                                    {File} 'test.cpp'
 ; ONE-NEXT:                                    {Public} 'foo' [0x0000000000:0x000000003a]
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-ignored-DW_FORM_implicit_const.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-ignored-DW_FORM_implicit_const.test
index 9df058c55980..6e8e720722f0 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-ignored-DW_FORM_implicit_const.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-ignored-DW_FORM_implicit_const.test
@@ -28,7 +28,7 @@
 ;  8    return ParamUnsigned;
 ;  9  }
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --print=scopes,symbols,types \
 ; RUN:                         %p/Inputs/pr-57040-test-dwarf-clang.o 2>&1 | \
 ; RUN: FileCheck --strict-whitespace -check-prefix=ONE %s
@@ -38,6 +38,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'test.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 14.0.6'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     1         {TypeAlias} 'INTPTR' -> '* const int'
 ; ONE-NEXT: [002]     2         {Function} extern not_inlined 'foo' -> 'int'
 ; ONE-NEXT: [003]                 {Block}
@@ -47,7 +48,7 @@
 ; ONE-NEXT: [003]     2           {Parameter} 'ParamUnsigned' -> 'unsigned int'
 ; ONE-NEXT: [003]     4           {TypeAlias} 'INTEGER' -> 'int'
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --print=scopes,symbols,types \
 ; RUN:                         %p/Inputs/pr-57040-test-dwarf-gcc.o 2>&1 | \
 ; RUN: FileCheck --strict-whitespace -check-prefix=TWO %s
@@ -57,6 +58,7 @@
 ; TWO-EMPTY:
 ; TWO-NEXT: [001]             {CompileUnit} 'test.cpp'
 ; TWO-NEXT: [002]               {Producer} 'GNU C++17 11.3.0 {{.*}}'
+; TWO-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; TWO-NEXT: [002]     1         {TypeAlias} 'INTPTR' -> '* const int'
 ; TWO-NEXT: [002]     2         {Function} extern not_inlined 'foo' -> 'int'
 ; TWO-NEXT: [003]                 {Block}
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-incorrect-function-compare.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-incorrect-function-compare.test
index 46fbc703f7de..278d4f4850f5 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-incorrect-function-compare.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/DWARF/pr-57040-incorrect-function-compare.test
@@ -35,7 +35,7 @@
 ;  8    return ParamUnsigned;
 ;  9  }
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,producer \
 ; RUN:                         --compare=types \
 ; RUN:                         --report=view \
 ; RUN:                         --print=symbols,types \
@@ -51,6 +51,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT:  [001]             {CompileUnit} 'test.cpp'
 ; ONE-NEXT:  [002]               {Producer} 'clang version 14.0.6'
+; ONE-NEXT:  [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT:  [002]     1         {TypeAlias} 'INTPTR' -> '* const int'
 ; ONE-NEXT:  [002]     2         {Function} extern not_inlined 'foo' -> 'int'
 ; ONE-NEXT:  [003]                 {Block}
@@ -90,7 +91,7 @@
 
 ; Changing the 'Reference' and 'Target' order:
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,producer \
 ; RUN:                         --compare=types \
 ; RUN:                         --report=view \
 ; RUN:                         --print=symbols,types \
@@ -106,6 +107,7 @@
 ; THR-EMPTY:
 ; THR-NEXT:  [001]             {CompileUnit} 'test.cpp'
 ; THR-NEXT:  [002]               {Producer} 'GNU C++17 11.3.0 {{.*}}'
+; THR-NEXT:  [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; THR-NEXT:  [002]     1         {TypeAlias} 'INTPTR' -> '* const int'
 ; THR-NEXT:  [002]     2         {Function} extern not_inlined 'foo' -> 'int'
 ; THR-NEXT:  [003]                 {Block}
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/02-wasm-logical-lines.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/02-wasm-logical-lines.test
index 101f6abdc96b..50a531ad7d82 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/02-wasm-logical-lines.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/02-wasm-logical-lines.test
@@ -18,7 +18,7 @@
 ; RUN: llvm-mc -arch=wasm32 -filetype=obj \
 ; RUN:         %p/Inputs/hello-world-clang.s -o %t.hello-world-clang.o
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --print=lines,instructions \
 ; RUN:                         %t.hello-world-clang.o 2>&1 | \
 ; RUN: FileCheck --strict-whitespace -check-prefix=ONE %s
@@ -28,6 +28,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'hello-world.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 19{{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     3         {Function} extern not_inlined 'main' -> 'int'
 ; ONE-NEXT: [003]     4           {Line}
 ; ONE-NEXT: [003]                 {Code} 'nop'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/03-wasm-incorrect-lexical-scope-typedef.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/03-wasm-incorrect-lexical-scope-typedef.test
index eb05ecae8ccf..1192a0cb7ca5 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/03-wasm-incorrect-lexical-scope-typedef.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/03-wasm-incorrect-lexical-scope-typedef.test
@@ -33,7 +33,7 @@
 ; RUN: llvm-mc -arch=wasm32 -filetype=obj \
 ; RUN:         %p/Inputs/pr-44884-clang.s -o %t.pr-44884-clang.o
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --output-sort=kind \
 ; RUN:                         --print=symbols,types,lines \
 ; RUN:                         %t.pr-44884-clang.o \
@@ -45,6 +45,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-44884.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 19{{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     1         {Function} extern not_inlined 'bar' -> 'int'
 ; ONE-NEXT: [003]     1           {Parameter} 'Input' -> 'float'
 ; ONE-NEXT: [003]     1           {Line}
@@ -84,6 +85,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-44884.cpp'
 ; ONE-NEXT: [002]               {Producer} 'GNU C++14 10.3.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus'
 ; ONE-NEXT: [002]     1         {Function} extern not_inlined 'bar' -> 'int'
 ; ONE-NEXT: [003]     1           {Parameter} 'Input' -> 'float'
 ; ONE-NEXT: [003]     1           {Line}
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/04-wasm-missing-nested-enumerators.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/04-wasm-missing-nested-enumerators.test
index cafa51c41966..58f80e06bd27 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/04-wasm-missing-nested-enumerators.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/04-wasm-missing-nested-enumerators.test
@@ -28,7 +28,7 @@
 ; RUN: llvm-mc -arch=wasm32 -filetype=obj \
 ; RUN:         %p/Inputs/pr-46466-clang.s -o %t.pr-46466-clang.o
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --output-sort=name \
 ; RUN:                         --print=symbols,types \
 ; RUN:                         %t.pr-46466-clang.o \
@@ -40,6 +40,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-46466.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 19{{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     8         {Variable} extern 'S' -> 'Struct'
 ; ONE-NEXT: [002]     1         {Struct} 'Struct'
 ; ONE-NEXT: [003]     5           {Member} public 'U' -> 'Union'
@@ -49,6 +50,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-46466.cpp'
 ; ONE-NEXT: [002]               {Producer} 'GNU C++14 10.3.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus'
 ; ONE-NEXT: [002]     8         {Variable} extern 'S' -> 'Struct'
 ; ONE-NEXT: [002]     1         {Struct} 'Struct'
 ; ONE-NEXT: [003]     5           {Member} public 'U' -> 'Union'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/05-wasm-incorrect-lexical-scope-variable.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/05-wasm-incorrect-lexical-scope-variable.test
index 4348161f8c5b..32a186004a36 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/05-wasm-incorrect-lexical-scope-variable.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/05-wasm-incorrect-lexical-scope-variable.test
@@ -34,7 +34,7 @@
 ; RUN: llvm-mc -arch=wasm32 -filetype=obj \
 ; RUN:         %p/Inputs/pr-43860-clang.s -o %t.pr-43860-clang.o
 
-; RUN: llvm-debuginfo-analyzer --attribute=level,format,producer \
+; RUN: llvm-debuginfo-analyzer --attribute=language,level,format,producer \
 ; RUN:                         --output-sort=name \
 ; RUN:                         --print=symbols \
 ; RUN:                         %t.pr-43860-clang.o \
@@ -46,6 +46,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-43860.cpp'
 ; ONE-NEXT: [002]               {Producer} 'clang version 19{{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT: [002]     2         {Function} extern inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [003]                 {Block}
 ; ONE-NEXT: [004]     5             {Variable} 'Var_2' -> 'int'
@@ -66,6 +67,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [001]             {CompileUnit} 'pr-43860.cpp'
 ; ONE-NEXT: [002]               {Producer} 'GNU C++14 10.3.0 {{.*}}'
+; ONE-NEXT: [002]               {Language} 'DW_LANG_C_plus_plus'
 ; ONE-NEXT: [002]     2         {Function} extern declared_inlined 'InlineFunction' -> 'int'
 ; ONE-NEXT: [003]                 {Block}
 ; ONE-NEXT: [004]     5             {Variable} 'Var_2' -> 'int'
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/06-wasm-full-logical-view.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/06-wasm-full-logical-view.test
index 81b78bac0321..ac4873f0a4d3 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/06-wasm-full-logical-view.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/06-wasm-full-logical-view.test
@@ -31,6 +31,7 @@
 ; ONE-EMPTY:
 ; ONE-NEXT: [0x000000000b][001]              {CompileUnit} 'test.cpp'
 ; ONE-NEXT: [0x000000000b][002]                {Producer} 'clang version 19{{.*}}'
+; ONE-NEXT: [0x000000000b][002]                {Language} 'DW_LANG_C_plus_plus_14'
 ; ONE-NEXT:                                    {Directory} '{{.*}}/general'
 ; ONE-NEXT:                                    {File} 'test.cpp'
 ; ONE-NEXT:                                    {Public} 'foo' [0x0000000002:0x000000007f]
diff --git a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/cmdline.test b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/cmdline.test
index 000c6e6b5ae3..776342614247 100644
--- a/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/cmdline.test
+++ b/external/llvm-project/llvm/test/tools/llvm-debuginfo-analyzer/cmdline.test
@@ -51,6 +51,7 @@ HELP-ALL:     =gaps                    -   Missing debug location (gaps).
 HELP-ALL:     =generated               -   Compiler generated elements.
 HELP-ALL:     =global                  -   Element referenced across Compile Units.
 HELP-ALL:     =inserted                -   Generated inlined abstract references.
+HELP-ALL:     =language                -   Source language name.
 HELP-ALL:     =level                   -   Lexical scope level (File=0, Compile Unit=1).
 HELP-ALL:     =linkage                 -   Linkage name.
 HELP-ALL:     =local                   -   Element referenced only in the Compile Unit.
diff --git a/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/DW_OP_GNU_implicit_pointer.yaml b/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/DW_OP_GNU_implicit_pointer.yaml
new file mode 100644
index 000000000000..56dcbc036cbb
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/DW_OP_GNU_implicit_pointer.yaml
@@ -0,0 +1,87 @@
+# Test that we can decode `DW_OP_GNU_implicit_pointer` (0xf2)
+# RUN: yaml2obj %s | llvm-dwarfdump - | FileCheck %s
+
+# CHECK:      DW_TAG_variable
+# CHECK-NEXT:   DW_AT_location (DW_OP_GNU_implicit_pointer 0x2a +4)
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+DWARF:
+  debug_abbrev:
+    - Table:
+        - Code:            0x00000001
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_language
+              Form:            DW_FORM_data2
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+        - Code:            0x00000002
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_data4
+            - Attribute:       DW_AT_frame_base
+              Form:            DW_FORM_exprloc
+        - Code:            0x00000003
+          Tag:             DW_TAG_formal_parameter
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_exprloc
+        - Code:            0x00000004
+          Tag:             DW_TAG_variable
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_location
+              Form:            DW_FORM_exprloc
+  debug_info:
+    - Length:          52
+      Version:         5
+      UnitType:        DW_UT_compile
+      AbbrOffset:      0
+      AddrSize:        8
+      Entries:
+        - AbbrCode:        0x00000001
+          Values:
+            - Value:           0x000000000000000C
+            - Value:           0x0000000100000F50
+            - Value:           0x0000000000000034
+        - AbbrCode:        0x00000002
+          Values:
+            - Value:           0x0000000100000F50
+            - Value:           0x0000000000000034
+            - Value:           0x0000000000000001
+              BlockData:
+                - 0x56
+        - AbbrCode:        0x00000003
+          Values:
+            - Value:           0x0000000000000002
+              BlockData:
+                - 0x91
+                - 0x78
+        - AbbrCode:        0x00000004
+          Values:
+            - Value:           0x0000000000000006
+              BlockData:
+                - 0xf2 # DW_OP_GNU_implicit_pointer
+                - 0x2a # Section offset of parameter in the previous entry
+                - 0x00
+                - 0x00
+                - 0x00
+                - 0x04 # Pointer references location 4 bytes into value of previous entry
+        - AbbrCode:        0x00000000
+          Values:
+        - AbbrCode:        0x00000000
+          Values:
+...
diff --git a/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/debug_info_addrx.s b/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/debug_info_addrx.s
index ec1ad227bd56..51f61496d71f 100644
--- a/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/debug_info_addrx.s
+++ b/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/debug_info_addrx.s
@@ -1,6 +1,8 @@
 # RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux -o %t.o
 # RUN: llvm-dwarfdump -debug-info %t.o | FileCheck %s
 # RUN: llvm-dwarfdump -debug-info %t.o -v | FileCheck --check-prefix=VERBOSE %s
+# RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux -o %t.o -crel
+# RUN: llvm-dwarfdump -debug-info %t.o | FileCheck %s
 
 # CHECK: DW_TAG_compile_unit
 # CHECK:   DW_AT_low_pc                                              (0x0000000000000000)
diff --git a/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/debug_info_crel.yaml b/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/debug_info_crel.yaml
deleted file mode 100644
index 308f21136b79..000000000000
--- a/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/debug_info_crel.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
-# Test llvm-dwarfdump handles CREL relocation sections correctly.
-
-# RUN: yaml2obj %s | llvm-dwarfdump - | FileCheck %s
-
-# CHECK: DW_AT_producer    ("clang version
-
---- !ELF
-FileHeader:
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_REL
-  Machine:         EM_X86_64
-  SectionHeaderStringTable: .strtab
-Sections:
-  - Name:            .debug_abbrev
-    Type:            SHT_PROGBITS
-    AddressAlign:    0x1
-    Content:         011101252513050325721710171B25111B120673170000022E00111B120640187A196E2503253A0B3B0B49133F19000003240003253E0B0B0B000000
-  - Name:            .debug_info
-    Type:            SHT_PROGBITS
-    AddressAlign:    0x1
-    Content:         '3400000005000108000000000100210001000000000000000002000600000000000000020006000000015703040001330000000305050400'
-  - Name:            .debug_str_offsets
-    Type:            SHT_PROGBITS
-    AddressAlign:    0x1
-    Content:         1C00000005000000000000000000000000000000000000000000000000000000
-  - Name:            .crel.debug_info
-    Type:            SHT_CREL
-    Flags:           [ SHF_INFO_LINK ]
-    Link:            .symtab
-    AddressAlign:    0x1
-    EntSize:         0x1
-    Info:            .debug_info
-    Relocations:
-      - Offset:          0x11
-        Symbol:          .debug_str_offsets
-        Type:            R_X86_64_32
-        Addend:          8
-  - Name:            .crel.debug_str_offsets
-    Type:            SHT_CREL
-    Flags:           [ SHF_INFO_LINK ]
-    Link:            .symtab
-    AddressAlign:    0x1
-    EntSize:         0x1
-    Info:            .debug_str_offsets
-    Relocations:
-      - Offset:          0x8
-        Symbol:          .debug_str
-        Type:            R_X86_64_32
-  - Type:            SectionHeaderTable
-    Sections:
-      - Name:            .strtab
-      - Name:            .debug_abbrev
-      - Name:            .debug_info
-      - Name:            .crel.debug_info
-      - Name:            .debug_str_offsets
-      - Name:            .crel.debug_str_offsets
-      - Name:            .debug_str
-      - Name:            .symtab
-Symbols:
-  - Name:            foo.cpp
-    Type:            STT_FILE
-    Index:           SHN_ABS
-  - Name:            .debug_abbrev
-    Type:            STT_SECTION
-    Section:         .debug_abbrev
-  - Name:            .debug_str_offsets
-    Type:            STT_SECTION
-    Section:         .debug_str_offsets
-  - Name:            .debug_str
-    Type:            STT_SECTION
-    Section:         .debug_str
-DWARF:
-  debug_str:
-    - 'clang version 21.0.0git'
-...
diff --git a/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/heterogeneous_proposal.s b/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/heterogeneous_proposal.s
index 1d0134f89bef..ee506607331a 100644
--- a/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/heterogeneous_proposal.s
+++ b/external/llvm-project/llvm/test/tools/llvm-dwarfdump/X86/heterogeneous_proposal.s
@@ -22,11 +22,11 @@ foo:
  .cfi_escape 0x10, 0x00, 0x03, 0xe9, 0x05, 0x00
  # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_bit_offset
  .cfi_escape 0x10, 0x00, 0x02, 0xe9, 0x06
- # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_call_frame_entry_reg EAX
+ # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_call_frame_entry_reg 0x0
  .cfi_escape 0x10, 0x00, 0x03, 0xe9, 0x07, 0x00
  # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_undefined
  .cfi_escape 0x10, 0x00, 0x02, 0xe9, 0x08
- # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_aspace_bregx EAX+2
+ # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_aspace_bregx 0x0 0x2
  .cfi_escape 0x10, 0x00, 0x04, 0xe9, 0x09, 0x0, 0x2
  # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_piece_end
  .cfi_escape 0x10, 0x00, 0x02, 0xe9, 0x0a
diff --git a/external/llvm-project/llvm/test/tools/llvm-exegesis/AArch64/skip_unsupported_instructions.s b/external/llvm-project/llvm/test/tools/llvm-exegesis/AArch64/skip_unsupported_instructions.s
deleted file mode 100644
index 72009756ed1d..000000000000
--- a/external/llvm-project/llvm/test/tools/llvm-exegesis/AArch64/skip_unsupported_instructions.s
+++ /dev/null
@@ -1,14 +0,0 @@
-llvm/test/tools/llvm-exegesis/AArch64/skip_unsupported_instructions.s
-
-# TODO: This is failing on some systems that have hardware support for
-# pointer authentication. This needs to be fixed before reenabling.
-# REQUIRES: disabled
-
-# REQUIRES: aarch64-registered-target
-
-# Check for skipping of illegal instruction errors (AUT and LDGM)
-# RUN: llvm-exegesis -mcpu=neoverse-v2 -mode=latency --opcode-name=AUTIA --benchmark-phase=assemble-measured-code 2>&1 | FileCheck %s --check-prefix=CHECK-AUTIA
-# CHECK-AUTIA-NOT: snippet crashed while running: Illegal instruction
-
-# RUN: llvm-exegesis -mcpu=neoverse-v2 -mode=latency --opcode-name=LDGM --benchmark-phase=assemble-measured-code 2>&1 | FileCheck %s --check-prefix=CHECK-LDGM
-# CHECK-LDGM: LDGM: Unsupported opcode: load tag multiple
diff --git a/external/llvm-project/llvm/test/tools/llvm-lto2/print-guid.test b/external/llvm-project/llvm/test/tools/llvm-lto2/print-guid.test
new file mode 100644
index 000000000000..a3d3f202ea43
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-lto2/print-guid.test
@@ -0,0 +1,2 @@
+# RUN: llvm-lto2 print-guid foo | FileCheck %s
+# CHECK: 6699318081062747564
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s
index 581dad6b68dc..54b5f1644be4 100644
--- a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-basic-instructions.s
@@ -2536,14 +2536,14 @@ drps
 # CHECK-NEXT:  1      2     0.50                        bics	x3, xzr, x3, lsl #1
 # CHECK-NEXT:  1      2     0.50                        tst	w3, w7, lsl #31
 # CHECK-NEXT:  1      2     0.50                        tst	x2, x20, asr #2
-# CHECK-NEXT:  1      0     0.06                        mov	x3, x6
-# CHECK-NEXT:  1      0     0.06                        mov	x3, xzr
-# CHECK-NEXT:  1      0     0.06                        mov	wzr, w2
-# CHECK-NEXT:  1      0     0.06                        mov	w3, w5
+# CHECK-NEXT:  1      0     0.17                        mov	x3, x6
+# CHECK-NEXT:  1      0     0.17                        mov	x3, xzr
+# CHECK-NEXT:  1      0     0.17                        mov	wzr, w2
+# CHECK-NEXT:  1      0     0.17                        mov	w3, w5
 # CHECK-NEXT:  1      1     0.17                        movz	w2, #0, lsl #16
 # CHECK-NEXT:  1      1     0.17                        mov	w2, #-1235
 # CHECK-NEXT:  1      1     0.17                        mov	x2, #5299989643264
-# CHECK-NEXT:  1      0     0.06                        mov	x2, #0
+# CHECK-NEXT:  1      0     0.17                        mov	x2, #0
 # CHECK-NEXT:  1      1     0.17                        movk	w3, #0
 # CHECK-NEXT:  1      1     0.17                        movz	x4, #0, lsl #16
 # CHECK-NEXT:  1      1     0.17                        movk	w5, #0, lsl #16
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s
index fbf65e26e99a..3398331a67f5 100644
--- a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-clear-upper-regs.s
@@ -58,7 +58,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -116,8 +116,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ldr	b0, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ldr	b0, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ldr	b0, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -126,9 +126,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ldr	b0, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ldr	b0, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [1] Code Region - FPR16-bit
 
@@ -137,7 +137,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -195,8 +195,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ldr	h0, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ldr	h0, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ldr	h0, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -205,9 +205,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ldr	h0, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ldr	h0, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [2] Code Region - FPR32-bit
 
@@ -216,7 +216,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -274,8 +274,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ldr	s0, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ldr	s0, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ldr	s0, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -284,9 +284,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ldr	s0, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ldr	s0, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [3] Code Region - FPR64-bit
 
@@ -295,7 +295,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -353,8 +353,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ldr	d0, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ldr	d0, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ldr	d0, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -363,9 +363,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ldr	d0, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ldr	d0, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [4] Code Region - FPR128-bit
 
@@ -374,7 +374,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -432,8 +432,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ldr	q0, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ldr	q0, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ldr	q0, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -442,9 +442,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ldr	q0, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ldr	q0, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [5] Code Region - SIMD64-bit-b
 
@@ -453,7 +453,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -511,8 +511,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ld1	{ v0.8b }, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ld1	{ v0.8b }, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ld1	{ v0.8b }, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -521,9 +521,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ld1	{ v0.8b }, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ld1	{ v0.8b }, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [6] Code Region - SIMD64-bit-h
 
@@ -532,7 +532,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -590,8 +590,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ld1	{ v0.4h }, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ld1	{ v0.4h }, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ld1	{ v0.4h }, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -600,9 +600,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ld1	{ v0.4h }, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ld1	{ v0.4h }, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [7] Code Region - SIMD64-bit-s
 
@@ -611,7 +611,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -669,8 +669,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ld1	{ v0.2s }, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ld1	{ v0.2s }, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ld1	{ v0.2s }, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -679,9 +679,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ld1	{ v0.2s }, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ld1	{ v0.2s }, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [8] Code Region - SIMD64-bit-d
 
@@ -690,7 +690,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      44
 # CHECK-NEXT: Total uOps:        200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.55
 # CHECK-NEXT: IPC:               4.55
 # CHECK-NEXT: Block RThroughput: 0.3
@@ -748,8 +748,8 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D======eeER.   add	z0.d, z0.d, z0.d
 # CHECK-NEXT: [2,0]     DeeeeeeE--R.   ld1	{ v0.1d }, [sp]
 # CHECK-NEXT: [2,1]     D======eeER.   add	z0.d, z0.d, z0.d
-# CHECK-NEXT: [3,0]     D=eeeeeeE-R.   ld1	{ v0.1d }, [sp]
-# CHECK-NEXT: [3,1]     D=======eeER   add	z0.d, z0.d, z0.d
+# CHECK-NEXT: [3,0]     .DeeeeeeE-R.   ld1	{ v0.1d }, [sp]
+# CHECK-NEXT: [3,1]     .D======eeER   add	z0.d, z0.d, z0.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -758,9 +758,9 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     1.3    1.3    1.3       ld1	{ v0.1d }, [sp]
-# CHECK-NEXT: 1.     4     7.3    0.0    0.0       add	z0.d, z0.d, z0.d
-# CHECK-NEXT:        4     4.3    0.6    0.6       <total>
+# CHECK-NEXT: 0.     4     1.0    1.0    1.3       ld1	{ v0.1d }, [sp]
+# CHECK-NEXT: 1.     4     7.0    0.0    0.0       add	z0.d, z0.d, z0.d
+# CHECK-NEXT:        4     4.0    0.5    0.6       <total>
 
 # CHECK:      [9] Code Region - insr
 
@@ -769,7 +769,7 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: Total Cycles:      803
 # CHECK-NEXT: Total uOps:        300
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.37
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -825,10 +825,10 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [0,1]     D======eeER    .    .    .    .   .   add	z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,0]     D========eeeeeeER   .    .    .   .   insr	z0.s, w0
 # CHECK-NEXT: [1,1]     D==============eeER .    .    .   .   add	z0.s, z0.s, z0.s
-# CHECK-NEXT: [2,0]     D================eeeeeeER.    .   .   insr	z0.s, w0
-# CHECK-NEXT: [2,1]     D======================eeER   .   .   add	z0.s, z0.s, z0.s
-# CHECK-NEXT: [3,0]     D========================eeeeeeER .   insr	z0.s, w0
-# CHECK-NEXT: [3,1]     D==============================eeER   add	z0.s, z0.s, z0.s
+# CHECK-NEXT: [2,0]     .D===============eeeeeeER.    .   .   insr	z0.s, w0
+# CHECK-NEXT: [2,1]     .D=====================eeER   .   .   add	z0.s, z0.s, z0.s
+# CHECK-NEXT: [3,0]     .D=======================eeeeeeER .   insr	z0.s, w0
+# CHECK-NEXT: [3,1]     .D=============================eeER   add	z0.s, z0.s, z0.s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -837,6 +837,6 @@ add z0.s, z0.s, z0.s
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     4     13.0   0.3    0.0       insr	z0.s, w0
-# CHECK-NEXT: 1.     4     19.0   0.0    0.0       add	z0.s, z0.s, z0.s
-# CHECK-NEXT:        4     16.0   0.1    0.0       <total>
+# CHECK-NEXT: 0.     4     12.5   0.3    0.0       insr	z0.s, w0
+# CHECK-NEXT: 1.     4     18.5   0.0    0.0       add	z0.s, z0.s, z0.s
+# CHECK-NEXT:        4     15.5   0.1    0.0       <total>
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-forwarding.s b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-forwarding.s
index 0f5ab183f535..39a779b27fe7 100644
--- a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-forwarding.s
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-forwarding.s
@@ -315,7 +315,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.57
 # CHECK-NEXT: IPC:               0.57
 # CHECK-NEXT: Block RThroughput: 3.0
@@ -330,8 +330,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=====eeER.    ..   madd	x0, x0, x0, x0
 # CHECK-NEXT: [1,0]     D=======eeER   ..   mul	x0, x0, x0
 # CHECK-NEXT: [1,1]     D=========eeER ..   madd	x0, x1, x2, x0
-# CHECK-NEXT: [1,2]     D==========eeER..   madd	x0, x1, x2, x0
-# CHECK-NEXT: [1,3]     D============eeER   madd	x0, x0, x0, x0
+# CHECK-NEXT: [1,2]     .D=========eeER..   madd	x0, x1, x2, x0
+# CHECK-NEXT: [1,3]     .D===========eeER   madd	x0, x0, x0, x0
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -342,9 +342,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     4.5    0.5    0.0       mul	x0, x0, x0
 # CHECK-NEXT: 1.     2     6.5    0.0    0.0       madd	x0, x1, x2, x0
-# CHECK-NEXT: 2.     2     7.5    0.0    0.0       madd	x0, x1, x2, x0
-# CHECK-NEXT: 3.     2     9.5    0.0    0.0       madd	x0, x0, x0, x0
-# CHECK-NEXT:        2     7.0    0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     7.0    0.0    0.0       madd	x0, x1, x2, x0
+# CHECK-NEXT: 3.     2     9.0    0.0    0.0       madd	x0, x0, x0, x0
+# CHECK-NEXT:        2     6.8    0.1    0.0       <total>
 
 # CHECK:      [1] Code Region - smaddl
 
@@ -353,7 +353,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.57
 # CHECK-NEXT: IPC:               0.57
 # CHECK-NEXT: Block RThroughput: 3.0
@@ -368,8 +368,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=====eeER.    ..   smaddl	x0, w0, w0, x0
 # CHECK-NEXT: [1,0]     D=======eeER   ..   mul	x0, x0, x0
 # CHECK-NEXT: [1,1]     D=========eeER ..   smaddl	x0, w1, w2, x0
-# CHECK-NEXT: [1,2]     D==========eeER..   smaddl	x0, w1, w2, x0
-# CHECK-NEXT: [1,3]     D============eeER   smaddl	x0, w0, w0, x0
+# CHECK-NEXT: [1,2]     .D=========eeER..   smaddl	x0, w1, w2, x0
+# CHECK-NEXT: [1,3]     .D===========eeER   smaddl	x0, w0, w0, x0
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -380,9 +380,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     4.5    0.5    0.0       mul	x0, x0, x0
 # CHECK-NEXT: 1.     2     6.5    0.0    0.0       smaddl	x0, w1, w2, x0
-# CHECK-NEXT: 2.     2     7.5    0.0    0.0       smaddl	x0, w1, w2, x0
-# CHECK-NEXT: 3.     2     9.5    0.0    0.0       smaddl	x0, w0, w0, x0
-# CHECK-NEXT:        2     7.0    0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     7.0    0.0    0.0       smaddl	x0, w1, w2, x0
+# CHECK-NEXT: 3.     2     9.0    0.0    0.0       smaddl	x0, w0, w0, x0
+# CHECK-NEXT:        2     6.8    0.1    0.0       <total>
 
 # CHECK:      [2] Code Region - fmadd
 
@@ -391,7 +391,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1703
 # CHECK-NEXT: Total uOps:        600
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.35
 # CHECK-NEXT: IPC:               0.35
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -406,12 +406,12 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=======eeeeER .    .    .    .    ..   fmadd	d0, d1, d2, d0
 # CHECK-NEXT: [0,4]     D=========eeeeER    .    .    .    ..   fmadd	d0, d1, d2, d0
 # CHECK-NEXT: [0,5]     D=============eeeeER.    .    .    ..   fmadd	d0, d0, d1, d2
-# CHECK-NEXT: [1,0]     D=================eeER   .    .    ..   fadd	d0, d0, d0
-# CHECK-NEXT: [1,1]     D===================eeeeER    .    ..   fmadd	d0, d1, d2, d0
-# CHECK-NEXT: [1,2]     D=======================eeeER .    ..   fmul	d0, d0, d0
-# CHECK-NEXT: [1,3]     D========================eeeeER    ..   fmadd	d0, d1, d2, d0
-# CHECK-NEXT: [1,4]     D==========================eeeeER  ..   fmadd	d0, d1, d2, d0
-# CHECK-NEXT: [1,5]     D==============================eeeeER   fmadd	d0, d0, d1, d2
+# CHECK-NEXT: [1,0]     .D================eeER   .    .    ..   fadd	d0, d0, d0
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .    ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [1,2]     .D======================eeeER .    ..   fmul	d0, d0, d0
+# CHECK-NEXT: [1,3]     .D=======================eeeeER    ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [1,4]     .D=========================eeeeER  ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [1,5]     .D=============================eeeeER   fmadd	d0, d0, d1, d2
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -420,13 +420,13 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     9.5    0.5    0.0       fadd	d0, d0, d0
-# CHECK-NEXT: 1.     2     11.5   0.0    0.0       fmadd	d0, d1, d2, d0
-# CHECK-NEXT: 2.     2     15.5   0.0    0.0       fmul	d0, d0, d0
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmadd	d0, d1, d2, d0
-# CHECK-NEXT: 4.     2     18.5   0.0    0.0       fmadd	d0, d1, d2, d0
-# CHECK-NEXT: 5.     2     22.5   0.0    0.0       fmadd	d0, d0, d1, d2
-# CHECK-NEXT:        2     15.7   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     9.0    0.5    0.0       fadd	d0, d0, d0
+# CHECK-NEXT: 1.     2     11.0   0.0    0.0       fmadd	d0, d1, d2, d0
+# CHECK-NEXT: 2.     2     15.0   0.0    0.0       fmul	d0, d0, d0
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       fmadd	d0, d1, d2, d0
+# CHECK-NEXT: 4.     2     18.0   0.0    0.0       fmadd	d0, d1, d2, d0
+# CHECK-NEXT: 5.     2     22.0   0.0    0.0       fmadd	d0, d0, d1, d2
+# CHECK-NEXT:        2     15.2   0.1    0.0       <total>
 
 # CHECK:      [3] Code Region - saba
 
@@ -435,7 +435,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -450,8 +450,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   saba	v0.4s, v0.4s, v1.4s
 # CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D=================eeeeER .  .   saba	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   saba	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: [1,3]     D======================eeeeER   saba	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   saba	v0.4s, v0.4s, v1.4s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -462,9 +462,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       saba	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       saba	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       saba	v0.4s, v0.4s, v1.4s
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       saba	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
 
 # CHECK:      [4] Code Region - sdot
 
@@ -473,7 +473,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1103
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.36
 # CHECK-NEXT: Block RThroughput: 0.8
@@ -488,8 +488,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D========eeeER .    .   .   sdot	v0.4s, v0.16b, v1.16b
 # CHECK-NEXT: [1,0]     D===========eeeeER  .   .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D===============eeeER   .   sdot	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: [1,2]     D================eeeER  .   sdot	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: [1,3]     D===================eeeER   sdot	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT: [1,2]     .D===============eeeER  .   sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [1,3]     .D==================eeeER   sdot	v0.4s, v0.16b, v1.16b
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -500,9 +500,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     10.5   0.0    0.0       sdot	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: 2.     2     11.5   0.0    0.0       sdot	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: 3.     2     14.5   0.0    0.0       sdot	v0.4s, v0.16b, v1.16b
-# CHECK-NEXT:        2     10.8   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     11.0   0.0    0.0       sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: 3.     2     14.0   0.0    0.0       sdot	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT:        2     10.5   0.1    0.0       <total>
 
 # CHECK:      [5] Code Region - smmla
 
@@ -511,7 +511,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1103
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.36
 # CHECK-NEXT: Block RThroughput: 0.8
@@ -526,8 +526,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D========eeeER .    .   .   smmla	v0.4s, v0.16b, v1.16b
 # CHECK-NEXT: [1,0]     D===========eeeeER  .   .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D===============eeeER   .   smmla	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: [1,2]     D================eeeER  .   smmla	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: [1,3]     D===================eeeER   smmla	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT: [1,2]     .D===============eeeER  .   smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [1,3]     .D==================eeeER   smmla	v0.4s, v0.16b, v1.16b
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -538,9 +538,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     10.5   0.0    0.0       smmla	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: 2.     2     11.5   0.0    0.0       smmla	v0.4s, v1.16b, v2.16b
-# CHECK-NEXT: 3.     2     14.5   0.0    0.0       smmla	v0.4s, v0.16b, v1.16b
-# CHECK-NEXT:        2     10.8   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     11.0   0.0    0.0       smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: 3.     2     14.0   0.0    0.0       smmla	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT:        2     10.5   0.1    0.0       <total>
 
 # CHECK:      [6] Code Region - mla
 
@@ -549,7 +549,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 2.0
@@ -564,8 +564,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   mla	v0.4s, v0.4s, v1.4s
 # CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D=================eeeeER .  .   mla	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   mla	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: [1,3]     D======================eeeeER   mla	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   mla	v0.4s, v0.4s, v1.4s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -576,9 +576,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       mla	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       mla	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       mla	v0.4s, v0.4s, v1.4s
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       mla	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
 
 # CHECK:      [7] Code Region - sqrdmlah
 
@@ -587,7 +587,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.29
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 3.5
@@ -602,8 +602,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   sqrdmlah	v0.4s, v0.4s, v1.4s
 # CHECK-NEXT: [1,0]     D==============eeeeER    .    .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D==================eeeeER.    .   sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: [1,3]     D========================eeeeER   sqrdmlah	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   sqrdmlah	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   sqrdmlah	v0.4s, v0.4s, v1.4s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -614,9 +614,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     12.0   0.0    0.0       sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       sqrdmlah	v0.4s, v0.4s, v1.4s
-# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       sqrdmlah	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       sqrdmlah	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [8] Code Region - smlal2
 
@@ -625,7 +625,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 2.0
@@ -640,8 +640,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   smlal2	v0.4s, v0.8h, v1.8h
 # CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D=================eeeeER .  .   smlal2	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   smlal2	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,3]     D======================eeeeER   smlal2	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   smlal2	v0.4s, v0.8h, v1.8h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -652,9 +652,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       smlal2	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       smlal2	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       smlal2	v0.4s, v0.8h, v1.8h
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       smlal2	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
 
 # CHECK:      [9] Code Region - sadalp
 
@@ -663,7 +663,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -678,8 +678,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   sadalp	v0.2d, v0.4s
 # CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D=================eeeeER .  .   sadalp	v0.2d, v1.4s
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   sadalp	v0.2d, v1.4s
-# CHECK-NEXT: [1,3]     D======================eeeeER   sadalp	v0.2d, v0.4s
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   sadalp	v0.2d, v1.4s
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   sadalp	v0.2d, v0.4s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -690,9 +690,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       sadalp	v0.2d, v1.4s
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       sadalp	v0.2d, v1.4s
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       sadalp	v0.2d, v0.4s
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       sadalp	v0.2d, v1.4s
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       sadalp	v0.2d, v0.4s
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
 
 # CHECK:      [10] Code Region - ssra
 
@@ -701,7 +701,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -716,8 +716,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   ssra	v0.2d, v0.2d, #1
 # CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D=================eeeeER .  .   ssra	v0.2d, v1.2d, #1
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   ssra	v0.2d, v1.2d, #1
-# CHECK-NEXT: [1,3]     D======================eeeeER   ssra	v0.2d, v0.2d, #1
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   ssra	v0.2d, v0.2d, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -728,9 +728,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       ssra	v0.2d, v1.2d, #1
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       ssra	v0.2d, v1.2d, #1
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       ssra	v0.2d, v0.2d, #1
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       ssra	v0.2d, v0.2d, #1
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
 
 # CHECK:      [11] Code Region - fcmla
 
@@ -739,7 +739,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -754,8 +754,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   fcmla	v0.2d, v0.2d, v1.2d, #90
 # CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: [1,1]     D================eeeeER  .  .   fcmla	v0.2d, v1.2d, v2.2d, #90
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   fcmla	v0.2d, v1.2d, v2.2d, #90
-# CHECK-NEXT: [1,3]     D======================eeeeER   fcmla	v0.2d, v0.2d, v1.2d, #90
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   fcmla	v0.2d, v0.2d, v1.2d, #90
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -766,9 +766,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT: 1.     2     10.5   0.0    0.0       fcmla	v0.2d, v1.2d, v2.2d, #90
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       fcmla	v0.2d, v1.2d, v2.2d, #90
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fcmla	v0.2d, v0.2d, v1.2d, #90
-# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       fcmla	v0.2d, v0.2d, v1.2d, #90
+# CHECK-NEXT:        2     11.5   0.1    0.0       <total>
 
 # CHECK:      [12] Code Region - fmla
 
@@ -777,7 +777,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1703
 # CHECK-NEXT: Total uOps:        600
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.35
 # CHECK-NEXT: IPC:               0.35
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -792,12 +792,12 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=======eeeeER .    .    .    .    ..   fmla	v0.2d, v1.2d, v2.2d
 # CHECK-NEXT: [0,4]     D=========eeeeER    .    .    .    ..   fmla	v0.2d, v1.2d, v2.2d
 # CHECK-NEXT: [0,5]     D=============eeeeER.    .    .    ..   fmla	v0.2d, v0.2d, v1.2d
-# CHECK-NEXT: [1,0]     D=================eeeER  .    .    ..   fmul	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: [1,1]     D==================eeeeER.    .    ..   fmla	v0.2d, v1.2d, v2.2d
-# CHECK-NEXT: [1,2]     D======================eeER   .    ..   fadd	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: [1,3]     D========================eeeeER    ..   fmla	v0.2d, v1.2d, v2.2d
-# CHECK-NEXT: [1,4]     D==========================eeeeER  ..   fmla	v0.2d, v1.2d, v2.2d
-# CHECK-NEXT: [1,5]     D==============================eeeeER   fmla	v0.2d, v0.2d, v1.2d
+# CHECK-NEXT: [1,0]     .D================eeeER  .    .    ..   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,1]     .D=================eeeeER.    .    ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [1,2]     .D=====================eeER   .    ..   fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,3]     .D=======================eeeeER    ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [1,4]     .D=========================eeeeER  ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [1,5]     .D=============================eeeeER   fmla	v0.2d, v0.2d, v1.2d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -806,13 +806,13 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     9.5    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: 1.     2     10.5   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
-# CHECK-NEXT: 2.     2     14.5   0.0    0.0       fadd	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
-# CHECK-NEXT: 4.     2     18.5   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
-# CHECK-NEXT: 5.     2     22.5   0.0    0.0       fmla	v0.2d, v0.2d, v1.2d
-# CHECK-NEXT:        2     15.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     9.0    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 1.     2     10.0   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: 2.     2     14.0   0.0    0.0       fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: 4.     2     18.0   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: 5.     2     22.0   0.0    0.0       fmla	v0.2d, v0.2d, v1.2d
+# CHECK-NEXT:        2     14.8   0.1    0.0       <total>
 
 # CHECK:      [13] Code Region - fmlal
 
@@ -821,7 +821,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1903
 # CHECK-NEXT: Total uOps:        600
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.32
 # CHECK-NEXT: IPC:               0.32
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -836,12 +836,12 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .    .    .    .   fmlal	v0.4s, v1.4h, v2.4h
 # CHECK-NEXT: [0,4]     D===========eeeeER  .    .    .    .    .   fmlal	v0.4s, v1.4h, v2.4h
 # CHECK-NEXT: [0,5]     D===============eeeeER   .    .    .    .   fmlal	v0.4s, v0.4h, v1.4h
-# CHECK-NEXT: [1,0]     D===================eeeER.    .    .    .   fmul	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: [1,1]     D======================eeeeER .    .    .   fmlal	v0.4s, v1.4h, v2.4h
-# CHECK-NEXT: [1,2]     D==========================eeER    .    .   fadd	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: [1,3]     D============================eeeeER.    .   fmlal	v0.4s, v1.4h, v2.4h
-# CHECK-NEXT: [1,4]     D==============================eeeeER   .   fmlal	v0.4s, v1.4h, v2.4h
-# CHECK-NEXT: [1,5]     D==================================eeeeER   fmlal	v0.4s, v0.4h, v1.4h
+# CHECK-NEXT: [1,0]     .D==================eeeER.    .    .    .   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,1]     .D=====================eeeeER .    .    .   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [1,2]     .D=========================eeER    .    .   fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,3]     .D===========================eeeeER.    .   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [1,4]     .D=============================eeeeER   .   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [1,5]     .D=================================eeeeER   fmlal	v0.4s, v0.4h, v1.4h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -850,13 +850,13 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     10.5   0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: 1.     2     13.5   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
-# CHECK-NEXT: 2.     2     17.5   0.0    0.0       fadd	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT: 3.     2     19.5   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
-# CHECK-NEXT: 4.     2     21.5   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
-# CHECK-NEXT: 5.     2     25.5   0.0    0.0       fmlal	v0.4s, v0.4h, v1.4h
-# CHECK-NEXT:        2     18.0   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     10.0   0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 1.     2     13.0   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: 2.     2     17.0   0.0    0.0       fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 3.     2     19.0   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: 4.     2     21.0   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: 5.     2     25.0   0.0    0.0       fmlal	v0.4s, v0.4h, v1.4h
+# CHECK-NEXT:        2     17.5   0.1    0.0       <total>
 
 # CHECK:      [14] Code Region - bfdot
 
@@ -865,7 +865,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1603
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.25
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -880,8 +880,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D===========eeeeeER .    .    .   .   bfdot	v0.4s, v0.8h, v1.8h
 # CHECK-NEXT: [1,0]     D================eeeER   .    .   .   fmul	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT: [1,1]     D===================eeeeeER   .   .   bfdot	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,2]     D======================eeeeeER.   .   bfdot	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,3]     D===========================eeeeeER   bfdot	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,2]     .D=====================eeeeeER.   .   bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     .D==========================eeeeeER   bfdot	v0.4s, v0.8h, v1.8h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -892,9 +892,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     9.0    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT: 1.     2     12.0   0.0    0.0       bfdot	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 2.     2     15.0   0.0    0.0       bfdot	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 3.     2     20.0   0.0    0.0       bfdot	v0.4s, v0.8h, v1.8h
-# CHECK-NEXT:        2     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     14.5   0.0    0.0       bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     19.5   0.0    0.0       bfdot	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     13.8   0.1    0.0       <total>
 
 # CHECK:      [15] Code Region - bfmmla
 
@@ -903,7 +903,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1903
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.21
 # CHECK-NEXT: IPC:               0.21
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -918,8 +918,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=============eeeeeeER   .    .    .    .   bfmmla	v0.4s, v0.8h, v1.8h
 # CHECK-NEXT: [1,0]     D===================eeeER.    .    .    .   fmul	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT: [1,1]     D======================eeeeeeER    .    .   bfmmla	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,2]     D==========================eeeeeeER.    .   bfmmla	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,3]     D================================eeeeeeER   bfmmla	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,2]     .D=========================eeeeeeER.    .   bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     .D===============================eeeeeeER   bfmmla	v0.4s, v0.8h, v1.8h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -930,9 +930,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     10.5   0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT: 1.     2     13.5   0.0    0.0       bfmmla	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 2.     2     17.5   0.0    0.0       bfmmla	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 3.     2     23.5   0.0    0.0       bfmmla	v0.4s, v0.8h, v1.8h
-# CHECK-NEXT:        2     16.3   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     17.0   0.0    0.0       bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     23.0   0.0    0.0       bfmmla	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     16.0   0.1    0.0       <total>
 
 # CHECK:      [16] Code Region - bfmlalb
 
@@ -941,7 +941,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1503
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.27
 # CHECK-NEXT: IPC:               0.27
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -956,8 +956,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    . .   bfmlalb	v0.4s, v0.8h, v1.8h
 # CHECK-NEXT: [1,0]     D===============eeeER    .    . .   fmul	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT: [1,1]     D==================eeeeeER    . .   bfmlalb	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,2]     D====================eeeeeER  . .   bfmlalb	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: [1,3]     D=========================eeeeeER   bfmlalb	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,2]     .D===================eeeeeER  . .   bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     .D========================eeeeeER   bfmlalb	v0.4s, v0.8h, v1.8h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -968,9 +968,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     8.5    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       bfmlalb	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 2.     2     13.5   0.0    0.0       bfmlalb	v0.4s, v1.8h, v2.8h
-# CHECK-NEXT: 3.     2     18.5   0.0    0.0       bfmlalb	v0.4s, v0.8h, v1.8h
-# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     13.0   0.0    0.0       bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     18.0   0.0    0.0       bfmlalb	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [17] Code Region - crc32b
 
@@ -979,7 +979,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.57
 # CHECK-NEXT: IPC:               0.57
 # CHECK-NEXT: Block RThroughput: 3.0
@@ -994,8 +994,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=====eeER.    ..   crc32b	w0, w0, w0
 # CHECK-NEXT: [1,0]     D=======eeER   ..   mul	w0, w0, w0
 # CHECK-NEXT: [1,1]     D=========eeER ..   crc32b	w0, w0, w1
-# CHECK-NEXT: [1,2]     D==========eeER..   crc32b	w0, w0, w1
-# CHECK-NEXT: [1,3]     D============eeER   crc32b	w0, w0, w0
+# CHECK-NEXT: [1,2]     .D=========eeER..   crc32b	w0, w0, w1
+# CHECK-NEXT: [1,3]     .D===========eeER   crc32b	w0, w0, w0
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1006,9 +1006,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     4.5    0.5    0.0       mul	w0, w0, w0
 # CHECK-NEXT: 1.     2     6.5    0.0    0.0       crc32b	w0, w0, w1
-# CHECK-NEXT: 2.     2     7.5    0.0    0.0       crc32b	w0, w0, w1
-# CHECK-NEXT: 3.     2     9.5    0.0    0.0       crc32b	w0, w0, w0
-# CHECK-NEXT:        2     7.0    0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     7.0    0.0    0.0       crc32b	w0, w0, w1
+# CHECK-NEXT: 3.     2     9.0    0.0    0.0       crc32b	w0, w0, w0
+# CHECK-NEXT:        2     6.8    0.1    0.0       <total>
 
 # CHECK:      [18] Code Region - Z saba
 
@@ -1017,7 +1017,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -1030,10 +1030,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   saba	z0.d, z1.d, z2.d
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   saba	z0.d, z1.d, z2.d
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   saba	z0.d, z0.d, z1.d
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   saba	z0.d, z1.d, z2.d
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   saba	z0.d, z1.d, z2.d
-# CHECK-NEXT: [1,3]     D========================eeeeER   saba	z0.d, z0.d, z1.d
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   saba	z0.d, z1.d, z2.d
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   saba	z0.d, z1.d, z2.d
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   saba	z0.d, z0.d, z1.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1042,11 +1042,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       saba	z0.d, z1.d, z2.d
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       saba	z0.d, z1.d, z2.d
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       saba	z0.d, z0.d, z1.d
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       saba	z0.d, z1.d, z2.d
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       saba	z0.d, z1.d, z2.d
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       saba	z0.d, z0.d, z1.d
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [19] Code Region - Z sadalp
 
@@ -1055,7 +1055,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -1068,10 +1068,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   sadalp	z0.d, p0/m, z1.s
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   sadalp	z0.d, p0/m, z1.s
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   sadalp	z0.d, p0/m, z0.s
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   sadalp	z0.d, p0/m, z1.s
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   sadalp	z0.d, p0/m, z1.s
-# CHECK-NEXT: [1,3]     D========================eeeeER   sadalp	z0.d, p0/m, z0.s
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   sadalp	z0.d, p0/m, z1.s
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   sadalp	z0.d, p0/m, z1.s
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   sadalp	z0.d, p0/m, z0.s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1080,11 +1080,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       sadalp	z0.d, p0/m, z1.s
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       sadalp	z0.d, p0/m, z1.s
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       sadalp	z0.d, p0/m, z0.s
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       sadalp	z0.d, p0/m, z1.s
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       sadalp	z0.d, p0/m, z1.s
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       sadalp	z0.d, p0/m, z0.s
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [20] Code Region - Z ssra
 
@@ -1093,7 +1093,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 1.5
@@ -1106,10 +1106,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   ssra	z0.d, z1.d, #1
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   ssra	z0.d, z1.d, #1
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   ssra	z0.d, z0.d, #1
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   ssra	z0.d, z1.d, #1
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   ssra	z0.d, z1.d, #1
-# CHECK-NEXT: [1,3]     D========================eeeeER   ssra	z0.d, z0.d, #1
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   ssra	z0.d, z1.d, #1
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   ssra	z0.d, z1.d, #1
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   ssra	z0.d, z0.d, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1118,11 +1118,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       ssra	z0.d, z1.d, #1
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       ssra	z0.d, z1.d, #1
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       ssra	z0.d, z0.d, #1
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       ssra	z0.d, z1.d, #1
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       ssra	z0.d, z1.d, #1
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       ssra	z0.d, z0.d, #1
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [21] Code Region - Z cdot.s
 
@@ -1131,7 +1131,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.42
 # CHECK-NEXT: IPC:               0.33
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1144,10 +1144,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeER    .    .    ..   cdot	z0.s, z1.b, z2.b, #90
 # CHECK-NEXT: [0,2]     D======eeeER   .    .    ..   cdot	z0.s, z1.b, z2.b, #90
 # CHECK-NEXT: [0,3]     D=========eeeER.    .    ..   cdot	z0.s, z0.b, z1.b, #90
-# CHECK-NEXT: [1,0]     D============eeeeeER.    ..   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D=================eeeER  ..   cdot	z0.s, z1.b, z2.b, #90
-# CHECK-NEXT: [1,2]     D==================eeeER ..   cdot	z0.s, z1.b, z2.b, #90
-# CHECK-NEXT: [1,3]     D=====================eeeER   cdot	z0.s, z0.b, z1.b, #90
+# CHECK-NEXT: [1,0]     .D===========eeeeeER.    ..   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D================eeeER  ..   cdot	z0.s, z1.b, z2.b, #90
+# CHECK-NEXT: [1,2]     .D=================eeeER ..   cdot	z0.s, z1.b, z2.b, #90
+# CHECK-NEXT: [1,3]     .D====================eeeER   cdot	z0.s, z0.b, z1.b, #90
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1156,11 +1156,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     7.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     12.0   0.0    0.0       cdot	z0.s, z1.b, z2.b, #90
-# CHECK-NEXT: 2.     2     13.0   0.0    0.0       cdot	z0.s, z1.b, z2.b, #90
-# CHECK-NEXT: 3.     2     16.0   0.0    0.0       cdot	z0.s, z0.b, z1.b, #90
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       cdot	z0.s, z1.b, z2.b, #90
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       cdot	z0.s, z1.b, z2.b, #90
+# CHECK-NEXT: 3.     2     15.5   0.0    0.0       cdot	z0.s, z0.b, z1.b, #90
+# CHECK-NEXT:        2     11.5   0.1    0.0       <total>
 
 # CHECK:      [22] Code Region - Z cdot.d
 
@@ -1169,7 +1169,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -1182,10 +1182,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   cdot	z0.d, z1.h, z2.h, #90
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   cdot	z0.d, z1.h, z2.h, #90
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   cdot	z0.d, z0.h, z1.h, #90
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   cdot	z0.d, z1.h, z2.h, #90
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   cdot	z0.d, z1.h, z2.h, #90
-# CHECK-NEXT: [1,3]     D========================eeeeER   cdot	z0.d, z0.h, z1.h, #90
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   cdot	z0.d, z1.h, z2.h, #90
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   cdot	z0.d, z1.h, z2.h, #90
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   cdot	z0.d, z0.h, z1.h, #90
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1194,11 +1194,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       cdot	z0.d, z1.h, z2.h, #90
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       cdot	z0.d, z1.h, z2.h, #90
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       cdot	z0.d, z0.h, z1.h, #90
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       cdot	z0.d, z1.h, z2.h, #90
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       cdot	z0.d, z1.h, z2.h, #90
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       cdot	z0.d, z0.h, z1.h, #90
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [23] Code Region - Z cmla.b
 
@@ -1207,7 +1207,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -1220,10 +1220,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   cmla	z0.b, z1.b, z2.b, #90
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   cmla	z0.b, z1.b, z2.b, #90
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   cmla	z0.b, z0.b, z1.b, #90
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   cmla	z0.b, z1.b, z2.b, #90
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   cmla	z0.b, z1.b, z2.b, #90
-# CHECK-NEXT: [1,3]     D========================eeeeER   cmla	z0.b, z0.b, z1.b, #90
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   cmla	z0.b, z1.b, z2.b, #90
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   cmla	z0.b, z1.b, z2.b, #90
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   cmla	z0.b, z0.b, z1.b, #90
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1232,11 +1232,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       cmla	z0.b, z1.b, z2.b, #90
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       cmla	z0.b, z1.b, z2.b, #90
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       cmla	z0.b, z0.b, z1.b, #90
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       cmla	z0.b, z1.b, z2.b, #90
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       cmla	z0.b, z1.b, z2.b, #90
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       cmla	z0.b, z0.b, z1.b, #90
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [24] Code Region - Z cmla.d
 
@@ -1245,7 +1245,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1803
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.28
 # CHECK-NEXT: IPC:               0.22
 # CHECK-NEXT: Block RThroughput: 4.0
@@ -1258,10 +1258,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeeER  .    .    .    .    .  .   cmla	z0.d, z1.d, z2.d, #90
 # CHECK-NEXT: [0,2]     D========eeeeeER    .    .    .    .  .   cmla	z0.d, z1.d, z2.d, #90
 # CHECK-NEXT: [0,3]     D=============eeeeeER    .    .    .  .   cmla	z0.d, z0.d, z1.d, #90
-# CHECK-NEXT: [1,0]     D==================eeeeeER    .    .  .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D=======================eeeeeER    .  .   cmla	z0.d, z1.d, z2.d, #90
-# CHECK-NEXT: [1,2]     D==========================eeeeeER .  .   cmla	z0.d, z1.d, z2.d, #90
-# CHECK-NEXT: [1,3]     D===============================eeeeeER   cmla	z0.d, z0.d, z1.d, #90
+# CHECK-NEXT: [1,0]     .D=================eeeeeER    .    .  .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D======================eeeeeER    .  .   cmla	z0.d, z1.d, z2.d, #90
+# CHECK-NEXT: [1,2]     .D=========================eeeeeER .  .   cmla	z0.d, z1.d, z2.d, #90
+# CHECK-NEXT: [1,3]     .D==============================eeeeeER   cmla	z0.d, z0.d, z1.d, #90
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1270,11 +1270,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     10.0   0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     15.0   0.0    0.0       cmla	z0.d, z1.d, z2.d, #90
-# CHECK-NEXT: 2.     2     18.0   0.0    0.0       cmla	z0.d, z1.d, z2.d, #90
-# CHECK-NEXT: 3.     2     23.0   0.0    0.0       cmla	z0.d, z0.d, z1.d, #90
-# CHECK-NEXT:        2     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     14.5   0.0    0.0       cmla	z0.d, z1.d, z2.d, #90
+# CHECK-NEXT: 2.     2     17.5   0.0    0.0       cmla	z0.d, z1.d, z2.d, #90
+# CHECK-NEXT: 3.     2     22.5   0.0    0.0       cmla	z0.d, z0.d, z1.d, #90
+# CHECK-NEXT:        2     16.0   0.1    0.0       <total>
 
 # CHECK:      [25] Code Region - Z sdot.s
 
@@ -1283,7 +1283,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.42
 # CHECK-NEXT: IPC:               0.33
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1296,10 +1296,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeER    .    .    ..   sdot	z0.s, z1.b, z2.b
 # CHECK-NEXT: [0,2]     D======eeeER   .    .    ..   sdot	z0.s, z1.b, z2.b
 # CHECK-NEXT: [0,3]     D=========eeeER.    .    ..   sdot	z0.s, z0.b, z1.b
-# CHECK-NEXT: [1,0]     D============eeeeeER.    ..   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D=================eeeER  ..   sdot	z0.s, z1.b, z2.b
-# CHECK-NEXT: [1,2]     D==================eeeER ..   sdot	z0.s, z1.b, z2.b
-# CHECK-NEXT: [1,3]     D=====================eeeER   sdot	z0.s, z0.b, z1.b
+# CHECK-NEXT: [1,0]     .D===========eeeeeER.    ..   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D================eeeER  ..   sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: [1,2]     .D=================eeeER ..   sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: [1,3]     .D====================eeeER   sdot	z0.s, z0.b, z1.b
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1308,11 +1308,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     7.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     12.0   0.0    0.0       sdot	z0.s, z1.b, z2.b
-# CHECK-NEXT: 2.     2     13.0   0.0    0.0       sdot	z0.s, z1.b, z2.b
-# CHECK-NEXT: 3.     2     16.0   0.0    0.0       sdot	z0.s, z0.b, z1.b
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: 3.     2     15.5   0.0    0.0       sdot	z0.s, z0.b, z1.b
+# CHECK-NEXT:        2     11.5   0.1    0.0       <total>
 
 # CHECK:      [26] Code Region - Z sudot
 
@@ -1321,7 +1321,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.42
 # CHECK-NEXT: IPC:               0.33
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1334,10 +1334,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeER    .    .    ..   sdot	z0.s, z1.b, z2.b[1]
 # CHECK-NEXT: [0,2]     D======eeeER   .    .    ..   sdot	z0.s, z1.b, z2.b[1]
 # CHECK-NEXT: [0,3]     D=========eeeER.    .    ..   sdot	z0.s, z0.b, z1.b[1]
-# CHECK-NEXT: [1,0]     D============eeeeeER.    ..   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D=================eeeER  ..   sdot	z0.s, z1.b, z2.b[1]
-# CHECK-NEXT: [1,2]     D==================eeeER ..   sdot	z0.s, z1.b, z2.b[1]
-# CHECK-NEXT: [1,3]     D=====================eeeER   sdot	z0.s, z0.b, z1.b[1]
+# CHECK-NEXT: [1,0]     .D===========eeeeeER.    ..   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D================eeeER  ..   sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: [1,2]     .D=================eeeER ..   sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: [1,3]     .D====================eeeER   sdot	z0.s, z0.b, z1.b[1]
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1346,11 +1346,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     7.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     12.0   0.0    0.0       sdot	z0.s, z1.b, z2.b[1]
-# CHECK-NEXT: 2.     2     13.0   0.0    0.0       sdot	z0.s, z1.b, z2.b[1]
-# CHECK-NEXT: 3.     2     16.0   0.0    0.0       sdot	z0.s, z0.b, z1.b[1]
-# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: 3.     2     15.5   0.0    0.0       sdot	z0.s, z0.b, z1.b[1]
+# CHECK-NEXT:        2     11.5   0.1    0.0       <total>
 
 # CHECK:      [27] Code Region - Z sdot.d
 
@@ -1359,7 +1359,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -1372,10 +1372,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   sdot	z0.d, z1.h, z2.h
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   sdot	z0.d, z1.h, z2.h
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   sdot	z0.d, z0.h, z1.h
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   sdot	z0.d, z1.h, z2.h
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   sdot	z0.d, z1.h, z2.h
-# CHECK-NEXT: [1,3]     D========================eeeeER   sdot	z0.d, z0.h, z1.h
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   sdot	z0.d, z0.h, z1.h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1384,11 +1384,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       sdot	z0.d, z1.h, z2.h
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       sdot	z0.d, z1.h, z2.h
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       sdot	z0.d, z0.h, z1.h
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       sdot	z0.d, z0.h, z1.h
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [28] Code Region - Z smmla
 
@@ -1397,7 +1397,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1103
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.36
 # CHECK-NEXT: Block RThroughput: 0.8
@@ -1412,8 +1412,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D========eeeER .    .   .   smmla	z0.s, z0.b, z1.b
 # CHECK-NEXT: [1,0]     D===========eeeeER  .   .   mul	z0.s, z0.s, z0.s
 # CHECK-NEXT: [1,1]     D===============eeeER   .   smmla	z0.s, z1.b, z2.b
-# CHECK-NEXT: [1,2]     D================eeeER  .   smmla	z0.s, z1.b, z2.b
-# CHECK-NEXT: [1,3]     D===================eeeER   smmla	z0.s, z0.b, z1.b
+# CHECK-NEXT: [1,2]     .D===============eeeER  .   smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: [1,3]     .D==================eeeER   smmla	z0.s, z0.b, z1.b
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1424,9 +1424,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	z0.s, z0.s, z0.s
 # CHECK-NEXT: 1.     2     10.5   0.0    0.0       smmla	z0.s, z1.b, z2.b
-# CHECK-NEXT: 2.     2     11.5   0.0    0.0       smmla	z0.s, z1.b, z2.b
-# CHECK-NEXT: 3.     2     14.5   0.0    0.0       smmla	z0.s, z0.b, z1.b
-# CHECK-NEXT:        2     10.8   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     11.0   0.0    0.0       smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: 3.     2     14.0   0.0    0.0       smmla	z0.s, z0.b, z1.b
+# CHECK-NEXT:        2     10.5   0.1    0.0       <total>
 
 # CHECK:      [29] Code Region - Z mla.b
 
@@ -1435,7 +1435,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 4.0
@@ -1448,10 +1448,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   mla	z0.b, p0/m, z1.b, z2.b
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   mla	z0.b, p0/m, z1.b, z2.b
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   mla	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   mla	z0.b, p0/m, z1.b, z2.b
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   mla	z0.b, p0/m, z1.b, z2.b
-# CHECK-NEXT: [1,3]     D========================eeeeER   mla	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   mla	z0.b, p0/m, z1.b, z2.b
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   mla	z0.b, p0/m, z1.b, z2.b
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   mla	z0.b, p0/m, z0.b, z1.b
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1460,11 +1460,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       mla	z0.b, p0/m, z1.b, z2.b
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       mla	z0.b, p0/m, z1.b, z2.b
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       mla	z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       mla	z0.b, p0/m, z1.b, z2.b
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       mla	z0.b, p0/m, z1.b, z2.b
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       mla	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [30] Code Region - Z mla.d
 
@@ -1473,7 +1473,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1803
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.28
 # CHECK-NEXT: IPC:               0.22
 # CHECK-NEXT: Block RThroughput: 4.0
@@ -1486,10 +1486,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeeER  .    .    .    .    .  .   mla	z0.d, p0/m, z1.d, z2.d
 # CHECK-NEXT: [0,2]     D========eeeeeER    .    .    .    .  .   mla	z0.d, p0/m, z1.d, z2.d
 # CHECK-NEXT: [0,3]     D=============eeeeeER    .    .    .  .   mla	z0.d, p0/m, z0.d, z1.d
-# CHECK-NEXT: [1,0]     D==================eeeeeER    .    .  .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D=======================eeeeeER    .  .   mla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: [1,2]     D==========================eeeeeER .  .   mla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: [1,3]     D===============================eeeeeER   mla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT: [1,0]     .D=================eeeeeER    .    .  .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D======================eeeeeER    .  .   mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,2]     .D=========================eeeeeER .  .   mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,3]     .D==============================eeeeeER   mla	z0.d, p0/m, z0.d, z1.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1498,11 +1498,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     10.0   0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     15.0   0.0    0.0       mla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: 2.     2     18.0   0.0    0.0       mla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: 3.     2     23.0   0.0    0.0       mla	z0.d, p0/m, z0.d, z1.d
-# CHECK-NEXT:        2     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     14.5   0.0    0.0       mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 2.     2     17.5   0.0    0.0       mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 3.     2     22.5   0.0    0.0       mla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT:        2     16.0   0.1    0.0       <total>
 
 # CHECK:      [31] Code Region - Z smlalb
 
@@ -1511,7 +1511,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.36
 # CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -1524,10 +1524,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   smlalb	z0.d, z1.s, z2.s
 # CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   smlalb	z0.d, z1.s, z2.s
 # CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   smlalb	z0.d, z0.s, z1.s
-# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D===================eeeeER    .   smlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: [1,2]     D====================eeeeER   .   smlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: [1,3]     D========================eeeeER   smlalb	z0.d, z0.s, z1.s
+# CHECK-NEXT: [1,0]     .D=============eeeeeER   .    .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D==================eeeeER    .   smlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: [1,2]     .D===================eeeeER   .   smlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: [1,3]     .D=======================eeeeER   smlalb	z0.d, z0.s, z1.s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1536,11 +1536,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.0   0.0    0.0       smlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: 2.     2     14.0   0.0    0.0       smlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: 3.     2     18.0   0.0    0.0       smlalb	z0.d, z0.s, z1.s
-# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.5   0.0    0.0       smlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       smlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: 3.     2     17.5   0.0    0.0       smlalb	z0.d, z0.s, z1.s
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [32] Code Region - Z sqdmlalb
 
@@ -1549,7 +1549,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1503
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.33
 # CHECK-NEXT: IPC:               0.27
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -1562,10 +1562,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    . .   sqdmlalb	z0.d, z1.s, z2.s
 # CHECK-NEXT: [0,2]     D=======eeeeER .    .    .    . .   sqdmlalb	z0.d, z1.s, z2.s
 # CHECK-NEXT: [0,3]     D===========eeeeER  .    .    . .   sqdmlalb	z0.d, z0.s, z1.s
-# CHECK-NEXT: [1,0]     D===============eeeeeER  .    . .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D====================eeeeER   . .   sqdmlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: [1,2]     D======================eeeeER . .   sqdmlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: [1,3]     D==========================eeeeER   sqdmlalb	z0.d, z0.s, z1.s
+# CHECK-NEXT: [1,0]     .D==============eeeeeER  .    . .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D===================eeeeER   . .   sqdmlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: [1,2]     .D=====================eeeeER . .   sqdmlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: [1,3]     .D=========================eeeeER   sqdmlalb	z0.d, z0.s, z1.s
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1574,11 +1574,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.5    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.5   0.0    0.0       sqdmlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: 2.     2     15.5   0.0    0.0       sqdmlalb	z0.d, z1.s, z2.s
-# CHECK-NEXT: 3.     2     19.5   0.0    0.0       sqdmlalb	z0.d, z0.s, z1.s
-# CHECK-NEXT:        2     14.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     13.0   0.0    0.0       sqdmlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: 2.     2     15.0   0.0    0.0       sqdmlalb	z0.d, z1.s, z2.s
+# CHECK-NEXT: 3.     2     19.0   0.0    0.0       sqdmlalb	z0.d, z0.s, z1.s
+# CHECK-NEXT:        2     13.8   0.1    0.0       <total>
 
 # CHECK:      [33] Code Region - Z sqrdmlah.b
 
@@ -1587,7 +1587,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1503
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.33
 # CHECK-NEXT: IPC:               0.27
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -1600,10 +1600,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    . .   sqrdmlah	z0.b, z1.b, z2.b
 # CHECK-NEXT: [0,2]     D=======eeeeER .    .    .    . .   sqrdmlah	z0.b, z1.b, z2.b
 # CHECK-NEXT: [0,3]     D===========eeeeER  .    .    . .   sqrdmlah	z0.b, z0.b, z1.b
-# CHECK-NEXT: [1,0]     D===============eeeeeER  .    . .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D====================eeeeER   . .   sqrdmlah	z0.b, z1.b, z2.b
-# CHECK-NEXT: [1,2]     D======================eeeeER . .   sqrdmlah	z0.b, z1.b, z2.b
-# CHECK-NEXT: [1,3]     D==========================eeeeER   sqrdmlah	z0.b, z0.b, z1.b
+# CHECK-NEXT: [1,0]     .D==============eeeeeER  .    . .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D===================eeeeER   . .   sqrdmlah	z0.b, z1.b, z2.b
+# CHECK-NEXT: [1,2]     .D=====================eeeeER . .   sqrdmlah	z0.b, z1.b, z2.b
+# CHECK-NEXT: [1,3]     .D=========================eeeeER   sqrdmlah	z0.b, z0.b, z1.b
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1612,11 +1612,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     8.5    0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     13.5   0.0    0.0       sqrdmlah	z0.b, z1.b, z2.b
-# CHECK-NEXT: 2.     2     15.5   0.0    0.0       sqrdmlah	z0.b, z1.b, z2.b
-# CHECK-NEXT: 3.     2     19.5   0.0    0.0       sqrdmlah	z0.b, z0.b, z1.b
-# CHECK-NEXT:        2     14.3   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     13.0   0.0    0.0       sqrdmlah	z0.b, z1.b, z2.b
+# CHECK-NEXT: 2.     2     15.0   0.0    0.0       sqrdmlah	z0.b, z1.b, z2.b
+# CHECK-NEXT: 3.     2     19.0   0.0    0.0       sqrdmlah	z0.b, z0.b, z1.b
+# CHECK-NEXT:        2     13.8   0.1    0.0       <total>
 
 # CHECK:      [34] Code Region - Z sqrdmlah.d
 
@@ -1625,7 +1625,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1803
 # CHECK-NEXT: Total uOps:        500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.28
 # CHECK-NEXT: IPC:               0.22
 # CHECK-NEXT: Block RThroughput: 4.0
@@ -1638,10 +1638,10 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,1]     D=====eeeeeER  .    .    .    .    .  .   sqrdmlah	z0.d, z1.d, z2.d
 # CHECK-NEXT: [0,2]     D========eeeeeER    .    .    .    .  .   sqrdmlah	z0.d, z1.d, z2.d
 # CHECK-NEXT: [0,3]     D=============eeeeeER    .    .    .  .   sqrdmlah	z0.d, z0.d, z1.d
-# CHECK-NEXT: [1,0]     D==================eeeeeER    .    .  .   mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: [1,1]     D=======================eeeeeER    .  .   sqrdmlah	z0.d, z1.d, z2.d
-# CHECK-NEXT: [1,2]     D==========================eeeeeER .  .   sqrdmlah	z0.d, z1.d, z2.d
-# CHECK-NEXT: [1,3]     D===============================eeeeeER   sqrdmlah	z0.d, z0.d, z1.d
+# CHECK-NEXT: [1,0]     .D=================eeeeeER    .    .  .   mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     .D======================eeeeeER    .  .   sqrdmlah	z0.d, z1.d, z2.d
+# CHECK-NEXT: [1,2]     .D=========================eeeeeER .  .   sqrdmlah	z0.d, z1.d, z2.d
+# CHECK-NEXT: [1,3]     .D==============================eeeeeER   sqrdmlah	z0.d, z0.d, z1.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1650,11 +1650,11 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     10.0   0.5    0.0       mul	z0.d, z0.d, z0.d
-# CHECK-NEXT: 1.     2     15.0   0.0    0.0       sqrdmlah	z0.d, z1.d, z2.d
-# CHECK-NEXT: 2.     2     18.0   0.0    0.0       sqrdmlah	z0.d, z1.d, z2.d
-# CHECK-NEXT: 3.     2     23.0   0.0    0.0       sqrdmlah	z0.d, z0.d, z1.d
-# CHECK-NEXT:        2     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       mul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     14.5   0.0    0.0       sqrdmlah	z0.d, z1.d, z2.d
+# CHECK-NEXT: 2.     2     17.5   0.0    0.0       sqrdmlah	z0.d, z1.d, z2.d
+# CHECK-NEXT: 3.     2     22.5   0.0    0.0       sqrdmlah	z0.d, z0.d, z1.d
+# CHECK-NEXT:        2     16.0   0.1    0.0       <total>
 
 # CHECK:      [35] Code Region - Z fcmla ZPmZZ
 
@@ -1663,7 +1663,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1503
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.27
 # CHECK-NEXT: IPC:               0.27
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1678,8 +1678,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    . .   fcmla	z0.d, p0/m, z0.d, z1.d, #90
 # CHECK-NEXT: [1,0]     D===============eeeER    .    . .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D==================eeeeeER    . .   fcmla	z0.d, p0/m, z1.d, z2.d, #90
-# CHECK-NEXT: [1,2]     D====================eeeeeER  . .   fcmla	z0.d, p0/m, z1.d, z2.d, #90
-# CHECK-NEXT: [1,3]     D=========================eeeeeER   fcmla	z0.d, p0/m, z0.d, z1.d, #90
+# CHECK-NEXT: [1,2]     .D===================eeeeeER  . .   fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: [1,3]     .D========================eeeeeER   fcmla	z0.d, p0/m, z0.d, z1.d, #90
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1690,9 +1690,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     8.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       fcmla	z0.d, p0/m, z1.d, z2.d, #90
-# CHECK-NEXT: 2.     2     13.5   0.0    0.0       fcmla	z0.d, p0/m, z1.d, z2.d, #90
-# CHECK-NEXT: 3.     2     18.5   0.0    0.0       fcmla	z0.d, p0/m, z0.d, z1.d, #90
-# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     13.0   0.0    0.0       fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: 3.     2     18.0   0.0    0.0       fcmla	z0.d, p0/m, z0.d, z1.d, #90
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [36] Code Region - Z fcmla ZZZI
 
@@ -1701,7 +1701,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1503
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.27
 # CHECK-NEXT: IPC:               0.27
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1716,8 +1716,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    . .   fcmla	z0.s, z0.s, z1.s[1], #90
 # CHECK-NEXT: [1,0]     D===============eeeER    .    . .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D==================eeeeeER    . .   fcmla	z0.s, z1.s, z2.s[1], #90
-# CHECK-NEXT: [1,2]     D====================eeeeeER  . .   fcmla	z0.s, z1.s, z2.s[1], #90
-# CHECK-NEXT: [1,3]     D=========================eeeeeER   fcmla	z0.s, z0.s, z1.s[1], #90
+# CHECK-NEXT: [1,2]     .D===================eeeeeER  . .   fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: [1,3]     .D========================eeeeeER   fcmla	z0.s, z0.s, z1.s[1], #90
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1728,9 +1728,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     8.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       fcmla	z0.s, z1.s, z2.s[1], #90
-# CHECK-NEXT: 2.     2     13.5   0.0    0.0       fcmla	z0.s, z1.s, z2.s[1], #90
-# CHECK-NEXT: 3.     2     18.5   0.0    0.0       fcmla	z0.s, z0.s, z1.s[1], #90
-# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     13.0   0.0    0.0       fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: 3.     2     18.0   0.0    0.0       fcmla	z0.s, z0.s, z1.s[1], #90
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
 
 # CHECK:      [37] Code Region - Z fmla ZPmZZ
 
@@ -1739,7 +1739,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1754,8 +1754,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   fmla	z0.d, p0/m, z0.d, z1.d
 # CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D================eeeeER  .  .   fmla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   fmla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: [1,3]     D======================eeeeER   fmla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   fmla	z0.d, p0/m, z0.d, z1.d
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1766,9 +1766,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     10.5   0.0    0.0       fmla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       fmla	z0.d, p0/m, z1.d, z2.d
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmla	z0.d, p0/m, z0.d, z1.d
-# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       fmla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT:        2     11.5   0.1    0.0       <total>
 
 # CHECK:      [38] Code Region - Z fmla ZZZI
 
@@ -1777,7 +1777,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1792,8 +1792,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   fmla	z0.d, z0.d, z1.d[1]
 # CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D================eeeeER  .  .   fmla	z0.d, z1.d, z2.d[1]
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   fmla	z0.d, z1.d, z2.d[1]
-# CHECK-NEXT: [1,3]     D======================eeeeER   fmla	z0.d, z0.d, z1.d[1]
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   fmla	z0.d, z0.d, z1.d[1]
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1804,9 +1804,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     10.5   0.0    0.0       fmla	z0.d, z1.d, z2.d[1]
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       fmla	z0.d, z1.d, z2.d[1]
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmla	z0.d, z0.d, z1.d[1]
-# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       fmla	z0.d, z0.d, z1.d[1]
+# CHECK-NEXT:        2     11.5   0.1    0.0       <total>
 
 # CHECK:      [39] Code Region - Z fmlalb ZZZ
 
@@ -1815,7 +1815,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1303
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.31
 # CHECK-NEXT: IPC:               0.31
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1830,8 +1830,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   fmlalb	z0.s, z0.h, z1.h
 # CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D================eeeeER  .  .   fmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,2]     D==================eeeeER.  .   fmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,3]     D======================eeeeER   fmlalb	z0.s, z0.h, z1.h
+# CHECK-NEXT: [1,2]     .D=================eeeeER.  .   fmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,3]     .D=====================eeeeER   fmlalb	z0.s, z0.h, z1.h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1842,9 +1842,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     10.5   0.0    0.0       fmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: 2.     2     12.5   0.0    0.0       fmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmlalb	z0.s, z0.h, z1.h
-# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     12.0   0.0    0.0       fmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       fmlalb	z0.s, z0.h, z1.h
+# CHECK-NEXT:        2     11.5   0.1    0.0       <total>
 
 # CHECK:      [40] Code Region - Z bfdot
 
@@ -1853,7 +1853,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1603
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.25
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1868,8 +1868,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D===========eeeeeER .    .    .   .   bfdot	z0.s, z0.h, z1.h
 # CHECK-NEXT: [1,0]     D================eeeER   .    .   .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D===================eeeeeER   .   .   bfdot	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,2]     D======================eeeeeER.   .   bfdot	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,3]     D===========================eeeeeER   bfdot	z0.s, z0.h, z1.h
+# CHECK-NEXT: [1,2]     .D=====================eeeeeER.   .   bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,3]     .D==========================eeeeeER   bfdot	z0.s, z0.h, z1.h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1880,9 +1880,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     9.0    0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     12.0   0.0    0.0       bfdot	z0.s, z1.h, z2.h
-# CHECK-NEXT: 2.     2     15.0   0.0    0.0       bfdot	z0.s, z1.h, z2.h
-# CHECK-NEXT: 3.     2     20.0   0.0    0.0       bfdot	z0.s, z0.h, z1.h
-# CHECK-NEXT:        2     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     14.5   0.0    0.0       bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: 3.     2     19.5   0.0    0.0       bfdot	z0.s, z0.h, z1.h
+# CHECK-NEXT:        2     13.8   0.1    0.0       <total>
 
 # CHECK:      [41] Code Region - Z bfmmla
 
@@ -1891,7 +1891,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1903
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.21
 # CHECK-NEXT: IPC:               0.21
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1906,8 +1906,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D=============eeeeeeER   .    .    .    .   bfmmla	z0.s, z0.h, z1.h
 # CHECK-NEXT: [1,0]     D===================eeeER.    .    .    .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D======================eeeeeeER    .    .   bfmmla	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,2]     D==========================eeeeeeER.    .   bfmmla	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,3]     D================================eeeeeeER   bfmmla	z0.s, z0.h, z1.h
+# CHECK-NEXT: [1,2]     .D=========================eeeeeeER.    .   bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,3]     .D===============================eeeeeeER   bfmmla	z0.s, z0.h, z1.h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1918,9 +1918,9 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     10.5   0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     13.5   0.0    0.0       bfmmla	z0.s, z1.h, z2.h
-# CHECK-NEXT: 2.     2     17.5   0.0    0.0       bfmmla	z0.s, z1.h, z2.h
-# CHECK-NEXT: 3.     2     23.5   0.0    0.0       bfmmla	z0.s, z0.h, z1.h
-# CHECK-NEXT:        2     16.3   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     17.0   0.0    0.0       bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: 3.     2     23.0   0.0    0.0       bfmmla	z0.s, z0.h, z1.h
+# CHECK-NEXT:        2     16.0   0.1    0.0       <total>
 
 # CHECK:      [42] Code Region - bfmlalb
 
@@ -1929,7 +1929,7 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: Total Cycles:      1503
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.27
 # CHECK-NEXT: IPC:               0.27
 # CHECK-NEXT: Block RThroughput: 1.0
@@ -1944,8 +1944,8 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    . .   bfmlalb	z0.s, z0.h, z1.h
 # CHECK-NEXT: [1,0]     D===============eeeER    .    . .   fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: [1,1]     D==================eeeeeER    . .   bfmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,2]     D====================eeeeeER  . .   bfmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: [1,3]     D=========================eeeeeER   bfmlalb	z0.s, z0.h, z1.h
+# CHECK-NEXT: [1,2]     .D===================eeeeeER  . .   bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,3]     .D========================eeeeeER   bfmlalb	z0.s, z0.h, z1.h
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1956,6 +1956,6 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     8.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
 # CHECK-NEXT: 1.     2     11.5   0.0    0.0       bfmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: 2.     2     13.5   0.0    0.0       bfmlalb	z0.s, z1.h, z2.h
-# CHECK-NEXT: 3.     2     18.5   0.0    0.0       bfmlalb	z0.s, z0.h, z1.h
-# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 2.     2     13.0   0.0    0.0       bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: 3.     2     18.0   0.0    0.0       bfmlalb	z0.s, z0.h, z1.h
+# CHECK-NEXT:        2     12.8   0.1    0.0       <total>
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s
index 6cba45cdd42b..49af4df3d8ff 100644
--- a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-sve-instructions.s
@@ -5071,19 +5071,19 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  2      2     1.00                        movs	p0.b, p0/z, p0.b
 # CHECK-NEXT:  2      2     1.00                        movs	p15.b, p15.b
 # CHECK-NEXT:  2      2     1.00                        movs	p15.b, p15/z, p15.b
-# CHECK-NEXT:  1      1     0.06                  U     mrs	x3, ID_AA64ZFR0_EL1
-# CHECK-NEXT:  1      1     0.06                  U     mrs	x3, ZCR_EL1
-# CHECK-NEXT:  1      1     0.06                  U     mrs	x3, ZCR_EL12
-# CHECK-NEXT:  1      1     0.06                  U     mrs	x3, ZCR_EL2
-# CHECK-NEXT:  1      1     0.06                  U     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  1      1     0.17                  U     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  1      1     0.17                  U     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  1      1     0.17                  U     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  1      1     0.17                  U     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  1      1     0.17                  U     mrs	x3, ZCR_EL3
 # CHECK-NEXT:  1      4     1.00                        msb	z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  1      5     1.00                        msb	z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      4     1.00                        msb	z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      4     1.00                        msb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  1      1     0.06                  U     msr	ZCR_EL1, x3
-# CHECK-NEXT:  1      1     0.06                  U     msr	ZCR_EL12, x3
-# CHECK-NEXT:  1      1     0.06                  U     msr	ZCR_EL2, x3
-# CHECK-NEXT:  1      1     0.06                  U     msr	ZCR_EL3, x3
+# CHECK-NEXT:  1      1     0.17                  U     msr	ZCR_EL1, x3
+# CHECK-NEXT:  1      1     0.17                  U     msr	ZCR_EL12, x3
+# CHECK-NEXT:  1      1     0.17                  U     msr	ZCR_EL2, x3
+# CHECK-NEXT:  1      1     0.17                  U     msr	ZCR_EL3, x3
 # CHECK-NEXT:  1      4     0.50                        mul	z0.b, p7/m, z0.b, z31.b
 # CHECK-NEXT:  1      4     0.50                        mul	z0.b, z1.b, z2.b
 # CHECK-NEXT:  2      5     1.00                        mul	z0.d, p7/m, z0.d, z31.d
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-writeback.s b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-writeback.s
index 1ef746813966..c7a93d1b4ed3 100644
--- a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-writeback.s
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-writeback.s
@@ -733,7 +733,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.97
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 1.7
@@ -745,8 +745,8 @@ ldr  x2, [x1], #254
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.2d }, [x27], #16
 # CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,4]     .D===eeeeeeER   ld1	{ v1.4s }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -758,9 +758,9 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
 # CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
 
 # CHECK:      [1] Code Region - G02
 
@@ -769,7 +769,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.97
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 1.7
@@ -781,8 +781,8 @@ ldr  x2, [x1], #254
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8b }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.8h }, [x27], #16
 # CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,4]     .D===eeeeeeER   ld1	{ v1.2d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -794,9 +794,9 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
 # CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
 
 # CHECK:      [2] Code Region - G03
 
@@ -805,7 +805,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.97
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 1.7
@@ -817,8 +817,8 @@ ldr  x2, [x1], #254
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.4h }, [x27], x28
 # CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     .D===eeeeeeER   ld1	{ v1.8h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -830,9 +830,9 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
 # CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
 
 # CHECK:      [3] Code Region - G04
 
@@ -841,7 +841,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.76
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 3.0
@@ -852,9 +852,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.16b }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -865,10 +865,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [4] Code Region - G05
 
@@ -877,7 +877,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 3.3
@@ -888,9 +888,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4s, v2.4s }, [x27], #32
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -901,10 +901,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [5] Code Region - G06
 
@@ -913,7 +913,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 3.3
@@ -924,9 +924,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2d, v2.2d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -937,10 +937,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [6] Code Region - G07
 
@@ -949,7 +949,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1800
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.54
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 4.3
@@ -960,9 +960,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8h, v2.8h }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,4]     .D===eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,3]     . D=eeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,4]     .  D=eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -973,10 +973,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT:        1     1.8    0.2    0.0       <total>
 
 # CHECK:      [7] Code Region - G08
 
@@ -985,7 +985,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -995,10 +995,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,4]     .D===eeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     .DeeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,4]     .   DeeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1008,11 +1008,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [8] Code Region - G09
 
@@ -1021,7 +1021,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               0.98
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -1031,10 +1031,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1044,11 +1044,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [9] Code Region - G10
 
@@ -1057,7 +1057,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      608
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.62
 # CHECK-NEXT: IPC:               0.82
 # CHECK-NEXT: Block RThroughput: 5.7
@@ -1067,10 +1067,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeER .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeER.  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D==eeeeeeeER.   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,4]     .D===eeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeeeeeeER.  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1080,11 +1080,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [10] Code Region - G11
 
@@ -1093,7 +1093,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      675
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.70
 # CHECK-NEXT: IPC:               0.74
 # CHECK-NEXT: Block RThroughput: 6.7
@@ -1103,10 +1103,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eeeeeeeER   .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,2]     D==eeeeeeeER  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,3]     .D===eeeeeeeER.   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,4]     .D====eeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeeeeeeeER   .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,2]     . DeeeeeeeER  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     .  D=eeeeeeeER.   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,4]     .   D=eeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1116,11 +1116,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 3.     1     4.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT:        1     3.0    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT:        1     1.4    0.4    0.0       <total>
 
 # CHECK:      [11] Code Region - G12
 
@@ -1129,7 +1129,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      675
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.70
 # CHECK-NEXT: IPC:               0.74
 # CHECK-NEXT: Block RThroughput: 6.7
@@ -1139,10 +1139,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,1]     D=eeeeeeeER   .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeeER  .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eeeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,4]     .D====eeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeER   .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeER  .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  D=eeeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,4]     .   D=eeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1152,11 +1152,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT:        1     1.4    0.4    0.0       <total>
 
 # CHECK:      [12] Code Region - G13
 
@@ -1165,7 +1165,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      1210
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.90
 # CHECK-NEXT: IPC:               0.41
 # CHECK-NEXT: Block RThroughput: 5.7
@@ -1175,10 +1175,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789          01
 
 # CHECK:      [0,0]     DeeeeeeeER.    .    ..   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeeER    .    ..   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeeER   .    ..   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eeeeeeeER .    ..   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,4]     .D==========eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,1]     .DeeeeeeeER    .    ..   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .    ..   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  D=eeeeeeeER .    ..   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,4]     .   D=======eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1188,11 +1188,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT:        1     4.2    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 4.     1     8.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT:        1     2.6    0.4    0.0       <total>
 
 # CHECK:      [13] Code Region - G14
 
@@ -1201,10 +1201,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.37
 # CHECK-NEXT: IPC:               0.12
-# CHECK-NEXT: Block RThroughput: 1.7
+# CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
@@ -1212,9 +1212,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
 # CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,2]     D================eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     D========================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,4]     D================================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,2]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,4]     . D==============================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1225,10 +1225,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
 # CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 2.     1     17.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     25.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 4.     1     33.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT:        1     17.0   0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     16.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     24.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 4.     1     31.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT:        1     16.2   0.2    0.0       <total>
 
 # CHECK:      [14] Code Region - G15
 
@@ -1237,10 +1237,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.37
 # CHECK-NEXT: IPC:               0.12
-# CHECK-NEXT: Block RThroughput: 1.7
+# CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
@@ -1248,9 +1248,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.h }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,2]     D================eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,3]     D========================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,4]     D================================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,2]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,3]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,4]     . D==============================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1261,10 +1261,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
 # CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 2.     1     17.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 3.     1     25.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 4.     1     33.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT:        1     17.0   0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     16.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: 3.     1     24.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 4.     1     31.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT:        1     16.2   0.2    0.0       <total>
 
 # CHECK:      [15] Code Region - G16
 
@@ -1273,10 +1273,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.25
 # CHECK-NEXT: IPC:               0.42
-# CHECK-NEXT: Block RThroughput: 1.7
+# CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1284,9 +1284,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1	{ v1.d }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: [0,3]     D===eeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: [0,4]     D====eeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1297,10 +1297,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [16] Code Region - G17
 
@@ -1309,10 +1309,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.94
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 1.7
+# CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1320,9 +1320,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.4s }, [x27], #4
 # CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: [0,3]     D===eeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: [0,4]     D====eeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1333,10 +1333,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [17] Code Region - G18
 
@@ -1345,10 +1345,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.94
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 1.7
+# CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1356,9 +1356,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.2d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,4]     D====eeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1369,10 +1369,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [18] Code Region - G19
 
@@ -1381,10 +1381,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        1900
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.73
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 2.0
+# CHECK-NEXT: Block RThroughput: 3.2
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1392,9 +1392,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.8h }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,3]     D===eeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,4]     .D===eeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,3]     . D=eeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,4]     .  D=eeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1405,10 +1405,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT:        1     1.8    0.2    0.0       <total>
 
 # CHECK:      [19] Code Region - G20
 
@@ -1417,20 +1417,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.71
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,4]     .D===eeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1440,11 +1440,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [20] Code Region - G21
 
@@ -1453,20 +1453,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.31
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1476,11 +1476,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [21] Code Region - G22
 
@@ -1489,20 +1489,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      3310
 # CHECK-NEXT: Total uOps:        2100
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.63
 # CHECK-NEXT: IPC:               0.15
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,2]     D================eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,3]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,4]     .D===============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     .D=======eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,2]     . D==============eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,3]     .  D=====================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .   D============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1512,11 +1512,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 2.     1     17.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 3.     1     24.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 4.     1     32.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT:        1     16.6   0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     8.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 2.     1     15.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 3.     1     22.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: 4.     1     29.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT:        1     15.0   0.2    0.0       <total>
 
 # CHECK:      [22] Code Region - G23
 
@@ -1525,20 +1525,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.50
 # CHECK-NEXT: IPC:               0.12
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,2]     D================eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D========================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,4]     .D===============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,1]     .D=======eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,2]     . D==============eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D=====================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,4]     .   D============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1548,11 +1548,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 2.     1     17.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     25.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 4.     1     32.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT:        1     16.8   0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     8.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 2.     1     15.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 3.     1     22.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: 4.     1     29.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT:        1     15.0   0.2    0.0       <total>
 
 # CHECK:      [23] Code Region - G24
 
@@ -1561,20 +1561,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      2603
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.77
 # CHECK-NEXT: IPC:               0.19
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
 # CHECK-NEXT: Index     0123456789          012345678
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,2]     D================eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D=================eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,4]     .D=================eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,1]     .D=======eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,2]     . D==============eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D==============eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,4]     .   D==============eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1584,11 +1584,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 2.     1     17.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     18.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 4.     1     18.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT:        1     12.6   0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     8.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 2.     1     15.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 3.     1     15.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT:        1     10.8   0.2    0.0       <total>
 
 # CHECK:      [24] Code Region - G25
 
@@ -1597,20 +1597,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.92
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,3]     D===eeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,4]     .D===eeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,3]     .  DeeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1620,11 +1620,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [25] Code Region - G26
 
@@ -1633,20 +1633,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.92
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1656,11 +1656,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [26] Code Region - G27
 
@@ -1669,20 +1669,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.51
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 2.8
+# CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1692,33 +1692,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [27] Code Region - G28
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      709
 # CHECK-NEXT: Total uOps:        3200
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.27
-# CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.51
+# CHECK-NEXT: IPC:               0.71
+# CHECK-NEXT: Block RThroughput: 5.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER   .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     .   DeeeeeeeeER.   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1728,33 +1728,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT:        1     1.0    0.4    0.0       <total>
 
 # CHECK:      [28] Code Region - G29
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      809
 # CHECK-NEXT: Total uOps:        3300
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.47
-# CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 4.3
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.08
+# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     0123456
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  ..   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DeeeeeeeeER.   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .DeeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1764,33 +1764,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.6    0.0       <total>
 
 # CHECK:      [29] Code Region - G30
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1910
+# CHECK-NEXT: Total Cycles:      1911
 # CHECK-NEXT: Total uOps:        3200
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    1.68
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    1.67
 # CHECK-NEXT: IPC:               0.26
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK-NEXT: Block RThroughput: 5.3
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
+# CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER   .    .    .  .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,2]     .D=eeeeeeeeER  .    .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=========eeeeeeeeER    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,4]     . D================eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .   .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER   .    .    .   .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     .  DeeeeeeeeER .    .    .   .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    D======eeeeeeeeER   .   .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,4]     .    .D=============eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1800,11 +1800,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT:        1     6.4    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT:        1     4.8    0.4    0.0       <total>
 
 # CHECK:      [30] Code Region - G31
 
@@ -1813,20 +1813,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.12
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,2]     .D===============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,3]     .D=======================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,4]     . D==============================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,2]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,3]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,4]     .   D============================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1836,11 +1836,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 2.     1     16.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 3.     1     24.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 4.     1     31.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT:        1     16.2   0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     8.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 2.     1     15.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: 3.     1     22.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 4.     1     29.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT:        1     15.0   0.2    0.0       <total>
 
 # CHECK:      [31] Code Region - G32
 
@@ -1849,20 +1849,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    0.75
 # CHECK-NEXT: IPC:               0.12
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,2]     .D===============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D=======================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,4]     . D==============================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,2]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,4]     .   D============================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1872,33 +1872,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 2.     1     16.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     24.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 4.     1     31.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT:        1     16.2   0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     8.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 2.     1     15.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     22.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 4.     1     29.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT:        1     15.0   0.2    0.0       <total>
 
 # CHECK:      [32] Code Region - G33
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      709
 # CHECK-NEXT: Total uOps:        3200
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.27
-# CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.51
+# CHECK-NEXT: IPC:               0.71
+# CHECK-NEXT: Block RThroughput: 5.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER   .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,2]     .  DeeeeeeeeER .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,3]     .   DeeeeeeeeER.   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1908,33 +1908,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT:        1     1.0    0.4    0.0       <total>
 
 # CHECK:      [33] Code Region - G34
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      809
 # CHECK-NEXT: Total uOps:        3300
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.47
-# CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 4.3
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.08
+# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     0123456
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER   ..   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,2]     .  DeeeeeeeeER ..   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,3]     .    DeeeeeeeeER.   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .DeeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1944,33 +1944,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.6    0.0       <total>
 
 # CHECK:      [34] Code Region - G35
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      709
 # CHECK-NEXT: Total uOps:        3200
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.27
-# CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.51
+# CHECK-NEXT: IPC:               0.71
+# CHECK-NEXT: Block RThroughput: 5.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeeER   .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .   DeeeeeeeeER.   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1980,33 +1980,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.4    0.0       <total>
 
 # CHECK:      [35] Code Region - G36
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      710
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        4500
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.34
-# CHECK-NEXT: IPC:               0.70
-# CHECK-NEXT: Block RThroughput: 7.0
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.46
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     0123456789
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeeeeeeeeeER  ..   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,2]     . DeeeeeeeeER  ..   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,3]     . D=eeeeeeeeER ..   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,4]     .  D==eeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeeER .   .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.   .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2016,33 +2016,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT:        1     1.6    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
 
 # CHECK:      [36] Code Region - G37
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      810
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4900
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.05
-# CHECK-NEXT: IPC:               0.62
-# CHECK-NEXT: Block RThroughput: 8.0
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.86
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 8.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    . .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     .DeeeeeeeeeER  . .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,2]     . DeeeeeeeeeER . .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,3]     .  DeeeeeeeeeER. .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,4]     .   D===eeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,1]     . DeeeeeeeeeER .  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeeER  .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeeER.   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2052,33 +2052,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    3.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT:        1     1.6    0.8    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
 
 # CHECK:      [37] Code Region - G38
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      809
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        4900
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.06
-# CHECK-NEXT: IPC:               0.62
-# CHECK-NEXT: Block RThroughput: 8.0
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.85
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 8.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     0123456789
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeeeeeeeeeER  ..   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,2]     . DeeeeeeeeER  ..   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  DeeeeeeeeeER..   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,4]     .   D=eeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeeER .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeeER .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2088,11 +2088,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT:        1     1.2    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
 
 # CHECK:      [38] Code Region - G39
 
@@ -2101,20 +2101,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.00
 # CHECK-NEXT: IPC:               0.12
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,2]     .D===============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D=======================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,4]     . D==============================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,2]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,4]     .    .  D========================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2124,11 +2124,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 2.     1     16.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 3.     1     24.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 4.     1     31.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT:        1     16.2   0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
 
 # CHECK:      [39] Code Region - G40
 
@@ -2137,20 +2137,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.00
 # CHECK-NEXT: IPC:               0.12
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,2]     .D===============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,3]     .D=======================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,4]     . D==============================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,2]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,4]     .    .  D========================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2160,11 +2160,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 2.     1     16.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 3.     1     24.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 4.     1     31.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT:        1     16.2   0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
 
 # CHECK:      [40] Code Region - G41
 
@@ -2173,20 +2173,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      1903
 # CHECK-NEXT: Total uOps:        4100
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.15
 # CHECK-NEXT: IPC:               0.26
-# CHECK-NEXT: Block RThroughput: 5.3
+# CHECK-NEXT: Block RThroughput: 6.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
 # CHECK-NEXT: Index     0123456789          01
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    ..   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     D========eeeeeeeeER ..   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,2]     .D========eeeeeeeeER..   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,3]     . D========eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,4]     .  D========eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER ..   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,2]     .   D=====eeeeeeeeER..   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     .    .D====eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,4]     .    .  D===eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2196,33 +2196,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT:        1     7.4    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     5.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT:        1     4.6    0.2    0.0       <total>
 
 # CHECK:      [41] Code Region - G42
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      659
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4300
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.53
-# CHECK-NEXT: IPC:               0.76
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.26
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: [0,1]     .DeeeeeeeeER   .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: [0,3]     .  DeeeeeeeeER .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: [0,4]     .   D=eeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2232,33 +2232,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT:        1     1.2    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
 
 # CHECK:      [42] Code Region - G43
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      610
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4200
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.89
-# CHECK-NEXT: IPC:               0.82
-# CHECK-NEXT: Block RThroughput: 5.7
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.16
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeeeeeeeeER   .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     . D=eeeeeeeeER .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,4]     .  D==eeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2268,33 +2268,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT:        1     1.6    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
 
 # CHECK:      [43] Code Region - G44
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total Cycles:      808
 # CHECK-NEXT: Total uOps:        3400
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    6.69
-# CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 4.3
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.21
+# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: Block RThroughput: 5.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeeeeeeeeER.   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,2]     . DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     . D=eeeeeeE-R   ldp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,4]     .  D=eeeeeeER   ldp	d1, d2, [x27], #496
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeeER.   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,4]     .    . DeeeeeeER   ldp	d1, d2, [x27], #496
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2304,11 +2304,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    1.0       ldp	s1, s2, [x27], #248
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	d1, d2, [x27], #496
-# CHECK-NEXT:        1     1.4    0.2    0.2       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT:        1     1.0    0.8    0.0       <total>
 
 # CHECK:      [44] Code Region - G45
 
@@ -2317,20 +2317,20 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.54
 # CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 2.3
+# CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeER ..   ldp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D=eeeeeeER..   ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,2]     D==eeeeeeER.   ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,3]     .D==eeeeeeER   ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,4]     .D===eeeeE-R   ldp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,1]     .DeeeeeeER..   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,2]     . DeeeeeeER.   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,3]     .  DeeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,4]     .   DeeeeE-R   ldp	w1, w2, [x27], #248
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2340,11 +2340,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 4.     1     4.0    0.0    1.0       ldp	w1, w2, [x27], #248
-# CHECK-NEXT:        1     2.6    0.2    0.2       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 4.     1     1.0    0.0    1.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT:        1     1.0    0.2    0.2       <total>
 
 # CHECK:      [45] Code Region - G46
 
@@ -2353,10 +2353,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2100
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.14
 # CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01
@@ -2364,9 +2364,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeER   ..   ldp	x1, x2, [x27], #496
 # CHECK-NEXT: [0,1]     D=eeeeER  ..   ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,2]     D==eeeeER ..   ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,3]     D===eeeeeER.   ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: [0,4]     .D===eeeeeER   ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: [0,2]     .D=eeeeER ..   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,3]     . D=eeeeeER.   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,4]     .  D=eeeeeER   ldpsw	x1, x2, [x27, #248]!
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2377,10 +2377,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	x1, x2, [x27], #496
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT:        1     1.8    0.2    0.0       <total>
 
 # CHECK:      [46] Code Region - G47
 
@@ -2389,10 +2389,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 1.7
+# CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -2400,9 +2400,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ldr	h1, [x27], #254
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ldr	s1, [x27], #254
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ldr	d1, [x27], #254
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ldr	q1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   ldr	s1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ldr	d1, [x27], #254
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   ldr	q1, [x27], #254
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2413,10 +2413,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldr	h1, [x27], #254
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldr	s1, [x27], #254
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ldr	d1, [x27], #254
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ldr	q1, [x27], #254
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	s1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldr	d1, [x27], #254
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	q1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [47] Code Region - G48
 
@@ -2425,10 +2425,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               0.98
-# CHECK-NEXT: Block RThroughput: 1.7
+# CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -2436,9 +2436,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eeeeeeER. .   ldr	h1, [x27, #254]!
-# CHECK-NEXT: [0,2]     D==eeeeeeER .   ldr	s1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D===eeeeeeER.   ldr	d1, [x27, #254]!
-# CHECK-NEXT: [0,4]     D====eeeeeeER   ldr	q1, [x27, #254]!
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   ldr	q1, [x27, #254]!
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2449,10 +2449,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldr	h1, [x27, #254]!
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldr	s1, [x27, #254]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ldr	d1, [x27, #254]!
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ldr	q1, [x27, #254]!
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [48] Code Region - G49
 
@@ -2461,7 +2461,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 1.7
@@ -2473,8 +2473,8 @@ ldr  x2, [x1], #254
 # CHECK:      [0,0]     DeeeeER   .   ldr	w1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eeeeER  .   ldr	x1, [x27], #254
 # CHECK-NEXT: [0,2]     D==eeeeER .   ldr	w1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D===eeeeER.   ldr	x1, [x27, #254]!
-# CHECK-NEXT: [0,4]     D====eeeeER   ldrb	w1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eeeeER.   ldr	x1, [x27, #254]!
+# CHECK-NEXT: [0,4]     .D===eeeeER   ldrb	w1, [x27], #254
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2486,9 +2486,9 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	w1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldr	x1, [x27], #254
 # CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldr	w1, [x27, #254]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ldr	x1, [x27, #254]!
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ldrb	w1, [x27], #254
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldr	x1, [x27, #254]!
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ldrb	w1, [x27], #254
+# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
 
 # CHECK:      [49] Code Region - G50
 
@@ -2497,7 +2497,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 1.7
@@ -2509,8 +2509,8 @@ ldr  x2, [x1], #254
 # CHECK:      [0,0]     DeeeeER   .   ldrb	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eeeeER  .   ldrh	w1, [x27], #254
 # CHECK-NEXT: [0,2]     D==eeeeER .   ldrh	w1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D===eeeeER.   ldrsb	w1, [x27], #254
-# CHECK-NEXT: [0,4]     D====eeeeER   ldrsb	x1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eeeeER.   ldrsb	w1, [x27], #254
+# CHECK-NEXT: [0,4]     .D===eeeeER   ldrsb	x1, [x27], #254
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2522,9 +2522,9 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrb	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldrh	w1, [x27], #254
 # CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldrh	w1, [x27, #254]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ldrsb	w1, [x27], #254
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ldrsb	x1, [x27], #254
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldrsb	w1, [x27], #254
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ldrsb	x1, [x27], #254
+# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
 
 # CHECK:      [50] Code Region - G51
 
@@ -2533,7 +2533,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    1.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 1.7
@@ -2545,8 +2545,8 @@ ldr  x2, [x1], #254
 # CHECK:      [0,0]     DeeeeER   .   ldrsb	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eeeeER  .   ldrsb	x1, [x27, #254]!
 # CHECK-NEXT: [0,2]     D==eeeeER .   ldrsh	w1, [x27], #254
-# CHECK-NEXT: [0,3]     D===eeeeER.   ldrsh	x1, [x27], #254
-# CHECK-NEXT: [0,4]     D====eeeeER   ldrsh	w1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D==eeeeER.   ldrsh	x1, [x27], #254
+# CHECK-NEXT: [0,4]     .D===eeeeER   ldrsh	w1, [x27, #254]!
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2558,9 +2558,9 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsb	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldrsb	x1, [x27, #254]!
 # CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldrsh	w1, [x27], #254
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       ldrsh	x1, [x27], #254
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ldrsh	w1, [x27, #254]!
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldrsh	x1, [x27], #254
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       ldrsh	w1, [x27, #254]!
+# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
 
 # CHECK:      [51] Code Region - G52
 
@@ -2569,10 +2569,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1200
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.38
 # CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 1.0
+# CHECK-NEXT: Block RThroughput: 2.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
@@ -2580,8 +2580,8 @@ ldr  x2, [x1], #254
 # CHECK:      [0,0]     DeeeeER .   ldrsh	x1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eeeeER.   ldrsw	x1, [x27], #254
 # CHECK-NEXT: [0,2]     D==eeeeER   ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D===eeE-R   st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,4]     D====eeER   st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,3]     .D==eeE-R   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,4]     .D===eeER   st1	{ v1.2d }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2593,9 +2593,9 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsh	x1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldrsw	x1, [x27], #254
 # CHECK-NEXT: 2.     1     3.0    0.0    0.0       ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: 3.     1     4.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT:        1     3.0    0.2    0.2       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT:        1     2.6    0.2    0.2       <total>
 
 # CHECK:      [52] Code Region - G53
 
@@ -2604,7 +2604,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -2614,9 +2614,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.2s }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eeER  .   st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,2]     D==eeER .   st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,3]     D===eeER.   st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,4]     D====eeER   st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeER .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D==eeER.   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,4]     . D==eeER   st1	{ v1.8h }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2627,10 +2627,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [53] Code Region - G54
 
@@ -2639,7 +2639,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -2649,9 +2649,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.16b }, [x27], #16
 # CHECK-NEXT: [0,1]     D=eeER  .   st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeER .   st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeER.   st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,4]     D====eeER   st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeER .   st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeER.   st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeER   st1	{ v1.4h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2662,10 +2662,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [54] Code Region - G55
 
@@ -2674,7 +2674,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -2684,9 +2684,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.4s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eeER  .   st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeER .   st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeER.   st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,4]     D====eeER   st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeER .   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeER.   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeER   st1	{ v1.1d, v2.1d }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2697,10 +2697,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [55] Code Region - G56
 
@@ -2709,7 +2709,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1900
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.77
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 3.5
@@ -2718,10 +2718,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,2]     D==eeER .   st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,3]     D===eeER.   st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,4]     .D===eeER   st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,1]     .DeeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeER .   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     . D=eeER.   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,4]     .  D=eeER   st1	{ v1.8b, v2.8b }, [x27], #16
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2731,11 +2731,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT:        1     1.6    0.2    0.0       <total>
 
 # CHECK:      [56] Code Region - G57
 
@@ -2744,7 +2744,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2100
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.17
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 4.0
@@ -2753,10 +2753,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,2]     D==eeER .   st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D==eeER.   st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeER   st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,2]     . DeeER .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeER.   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeER   st1	{ v1.2s, v2.2s }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2766,11 +2766,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT:        1     2.6    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [57] Code Region - G58
 
@@ -2779,7 +2779,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2100
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.17
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 4.0
@@ -2788,10 +2788,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,2]     D==eeER .   st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D===eeER.   st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeER   st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeER .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeER.   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeER   st1	{ v1.16b, v2.16b }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2801,11 +2801,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [58] Code Region - G59
 
@@ -2814,7 +2814,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        2900
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.13
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 6.0
@@ -2823,10 +2823,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeER.   .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eeER   .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,2]     .D=eeER  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,3]     .D===eeER.   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,4]     . D===eeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     .DeeER   .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,2]     . DeeER  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,3]     .  D=eeER.   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,4]     .   D=eeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2836,32 +2836,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 3.     1     4.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT:        1     2.6    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT:        1     1.4    0.4    0.0       <total>
 
 # CHECK:      [59] Code Region - G60
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        3100
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    4.41
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.40
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
+# CHECK-NEXT:                     0
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eeER   .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,2]     .D=eeER  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,3]     .D===eeER.   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,4]     . D===eeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK:      [0,0]     DeeER.    .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeeER    .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,2]     .  DeeER  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,3]     .   DeeER .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,4]     .    D=eeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2871,11 +2872,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 3.     1     4.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT:        1     2.6    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT:        1     1.2    0.6    0.0       <total>
 
 # CHECK:      [60] Code Region - G61
 
@@ -2884,7 +2885,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        2900
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.13
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 6.0
@@ -2893,10 +2894,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeER.   .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeER   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,2]     .D=eeER  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D==eeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,4]     . D===eeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeER   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeER  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     .   D=eeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2906,11 +2907,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT:        1     2.4    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT:        1     1.2    0.4    0.0       <total>
 
 # CHECK:      [61] Code Region - G62
 
@@ -2919,7 +2920,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        3100
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.40
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 6.5
@@ -2929,10 +2930,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeER.    .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeER    .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,2]     .D==eeER  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     .D===eeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,4]     . D====eeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeeER    .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,2]     . D=eeER  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .   DeeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,4]     .    D=eeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2942,33 +2943,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 2.     1     3.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 4.     1     5.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT:        1     3.0    0.6    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT:        1     1.4    0.6    0.0       <total>
 
 # CHECK:      [62] Code Region - G63
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      804
+# CHECK-NEXT: Total Cycles:      805
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.60
 # CHECK-NEXT: IPC:               0.62
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
+# CHECK-NEXT:                     012
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    ..   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,1]     D=eeER    ..   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,2]     .D==eeER  ..   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     . D==eeER ..   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,4]     . D=====eeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     . DeeER   . .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,2]     .  D=eeER . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .    DeeER. .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,4]     .    . D=eeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2978,32 +2979,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 2.     1     3.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 4.     1     6.0    2.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT:        1     3.0    0.8    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 4.     1     2.0    2.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT:        1     1.4    1.0    0.0       <total>
 
 # CHECK:      [63] Code Region - G64
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        3300
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    4.69
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
+# CHECK-NEXT:                     0
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeER   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,2]     .D==eeER .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eeER.   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,4]     . D===eeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK:      [0,0]     DeeER.    .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeER   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,2]     .  D=eeER .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .   D=eeER.   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .DeeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3013,33 +3015,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 2.     1     3.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT:        1     2.8    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT:        1     1.4    0.6    0.0       <total>
 
 # CHECK:      [64] Code Region - G65
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      706
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    4.25
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    4.24
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
+# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeeER    . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,2]     .D===eeeeER .   st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,3]     . D===eeeeER.   st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,4]     . D====eeeeER   st1	{ v1.b }[0], [x27], x28
+# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeER   .  .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeER  .   st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,3]     .    D=eeeeER.   st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,4]     .    .D=eeeeER   st1	{ v1.b }[0], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3049,11 +3051,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 2.     1     4.0    2.0    0.0       st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT:        1     3.0    0.6    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 3.     1     2.0    1.0    0.0       st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT:        1     1.4    0.8    0.0       <total>
 
 # CHECK:      [65] Code Region - G66
 
@@ -3062,7 +3064,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -3072,10 +3074,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeER   .   st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeER  .   st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,2]     D==eeeeER .   st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,3]     D===eeeeER.   st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeeeER   st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeER  .   st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,2]     . DeeeeER .   st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,3]     .  DeeeeER.   st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeER   st1	{ v1.h }[4], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3085,11 +3087,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [66] Code Region - G67
 
@@ -3098,7 +3100,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      605
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.80
 # CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 6.0
@@ -3108,10 +3110,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeER   .   st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D=eeeeER  .   st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeER .   st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,3]     D===eeeeER.   st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,4]     .D===eeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeeeeER  .   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeER .   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .  DeeeeER.   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3121,11 +3123,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [67] Code Region - G68
 
@@ -3134,7 +3136,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      705
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.69
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 7.0
@@ -3144,10 +3146,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeER   ..   st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,1]     D=eeeeER  ..   st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,2]     D==eeeeER ..   st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,3]     .D==eeeeER..   st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,4]     .D====eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeeeeER  ..   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,2]     . DeeeeER ..   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeeeeER..   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,4]     .   D=eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3157,33 +3159,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 4.     1     5.0    1.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT:        1     2.8    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT:        1     1.2    0.4    0.0       <total>
 
 # CHECK:      [68] Code Region - G69
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      805
+# CHECK-NEXT: Total Cycles:      806
 # CHECK-NEXT: Total uOps:        2900
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.60
 # CHECK-NEXT: IPC:               0.62
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
+# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   . .   st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eeeeER  . .   st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,2]     .D===eeeeER .   st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D====eeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,4]     .D=====eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK:      [0,0]     DeeeeER   .  .   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     . DeeeeER .  .   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     .  D=eeeeER  .   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .   D==eeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     .    D==eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3193,33 +3195,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 2.     1     4.0    2.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT:        1     3.6    0.6    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    1.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT:        1     2.0    0.8    0.0       <total>
 
 # CHECK:      [69] Code Region - G70
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      706
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.68
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
+# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   . .   st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeER  . .   st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,2]     .D=eeeeER . .   st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D====eeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,4]     .D=====eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK:      [0,0]     DeeeeER   .  .   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeER  .  .   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     .  DeeeeER.  .   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .   D=eeeeER .   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,4]     .    D==eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3229,11 +3231,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    2.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT:        1     3.2    0.6    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT:        1     1.6    0.8    0.0       <total>
 
 # CHECK:      [70] Code Region - G71
 
@@ -3242,7 +3244,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -3252,10 +3254,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,2]     D==eeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,3]     D===eeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,4]     .D===eeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,3]     .  DeeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,4]     .   DeeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3265,11 +3267,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [71] Code Region - G72
 
@@ -3278,7 +3280,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -3288,10 +3290,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,2]     D==eeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D===eeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,4]     .D===eeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,2]     . DeeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,4]     .   DeeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3301,11 +3303,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [72] Code Region - G73
 
@@ -3314,7 +3316,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      807
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.72
 # CHECK-NEXT: IPC:               0.62
 # CHECK-NEXT: Block RThroughput: 7.0
@@ -3325,9 +3327,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeER .    .   .   st2g	x26, [x27], #4064
 # CHECK-NEXT: [0,1]     D=eER.    .   .   st2g	x26, [x27, #4064]!
-# CHECK-NEXT: [0,2]     D==eeeeeeER   .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,3]     .D==eeeeeER   .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,4]     .D======eeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,2]     .D=eeeeeeER   .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,3]     .  DeeeeeER   .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,4]     .    D==eeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3338,32 +3340,32 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2g	x26, [x27], #4064
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2g	x26, [x27, #4064]!
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 4.     1     7.0    3.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT:        1     3.2    0.8    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 4.     1     3.0    3.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT:        1     1.8    0.8    0.0       <total>
 
 # CHECK:      [73] Code Region - G74
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1405
+# CHECK-NEXT: Total Cycles:      1406
 # CHECK-NEXT: Total uOps:        4700
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    3.35
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    3.34
 # CHECK-NEXT: IPC:               0.36
 # CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeeER .    .   .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,1]     .DeeeeeER .    .   .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,2]     . D===eeeeeeER .   .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,3]     .  D===eeeeeeER.   .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,4]     .   D=======eeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK:      [0,0]     DeeeeeeER .    .    .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     . DeeeeeER.    .    .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,2]     .   D==eeeeeeER.    .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     .    .D=eeeeeeER    .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,4]     .    .  D====eeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3373,33 +3375,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 2.     1     4.0    3.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 4.     1     8.0    4.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT:        1     3.6    1.6    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 4.     1     5.0    4.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT:        1     2.4    1.8    0.0       <total>
 
 # CHECK:      [74] Code Region - G75
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1206
+# CHECK-NEXT: Total Cycles:      1207
 # CHECK-NEXT: Total uOps:        4100
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.40
 # CHECK-NEXT: IPC:               0.41
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    . .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeER .    . .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,2]     .D===eeeeeeER  . .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     . D===eeeeeER  . .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,4]     .  D======eeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK:      [0,0]     DeeeeeER  .    .  .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeER.    .  .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeER  .  .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeER .  .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  D==eeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3409,33 +3411,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 2.     1     4.0    2.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 4.     1     7.0    3.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT:        1     3.6    1.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    3.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT:        1     1.4    1.4    0.0       <total>
 
 # CHECK:      [75] Code Region - G76
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1106
+# CHECK-NEXT: Total Cycles:      1107
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    3.44
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    3.43
 # CHECK-NEXT: IPC:               0.45
 # CHECK-NEXT: Block RThroughput: 11.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     01234567
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeeeeeER .    ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,2]     .D====eeeeeER  ..   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,3]     . D====eeeeeER ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,4]     . D=======eeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK:      [0,0]     DeeeeeeER .    . .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeER.    . .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,2]     .   D==eeeeeER . .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,3]     .    .D=eeeeeER. .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .    .  D==eeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3445,20 +3447,20 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 2.     1     5.0    3.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 4.     1     8.0    2.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT:        1     4.0    1.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT:        1     2.0    1.4    0.0       <total>
 
 # CHECK:      [76] Code Region - G77
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1005
+# CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        3500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.48
 # CHECK-NEXT: IPC:               0.50
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -3468,10 +3470,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeER  .    .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,1]     D=eeeeeER .    .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,2]     .D===eeeeeER   .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D====eeeeeER  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,4]     . D======eeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,1]     . DeeeeeER.    .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,2]     .   DeeeeeER   .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeER .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3481,33 +3483,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 2.     1     4.0    2.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 4.     1     7.0    2.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT:        1     3.8    1.0    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
 
 # CHECK:      [77] Code Region - G78
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1304
+# CHECK-NEXT: Total Cycles:      1305
 # CHECK-NEXT: Total uOps:        4300
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.30
 # CHECK-NEXT: IPC:               0.38
 # CHECK-NEXT: Block RThroughput: 13.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     01234567
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    ..   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeER .    ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,2]     .D===eeeeeER   ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     . D===eeeeeER  ..   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,4]     .  D=====eeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK:      [0,0]     DeeeeeER  .    . .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeER.    . .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,2]     .   DeeeeeER   . .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeER . .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,4]     .    .   DeeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3517,33 +3519,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 2.     1     4.0    2.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 4.     1     6.0    2.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT:        1     3.4    1.0    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
 
 # CHECK:      [78] Code Region - G79
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      2399
+# CHECK-NEXT: Total Cycles:      2400
 # CHECK-NEXT: Total uOps:        6900
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.88
 # CHECK-NEXT: IPC:               0.21
 # CHECK-NEXT: Block RThroughput: 24.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012
+# CHECK-NEXT: Index     0123456789          0123
 
-# CHECK:      [0,0]     DeeeeeeER .    .    . .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,1]     .DeeeeeeeER    .    . .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,2]     . D====eeeeeeER.    . .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,3]     .  D=========eeeeeeeER.   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,4]     .    D========eeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK:      [0,0]     DeeeeeeER .    .    .  .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     . DeeeeeeeER   .    .  .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,2]     .    D=eeeeeeER.    .  .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .    . D=====eeeeeeeER .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,4]     .    .    D====eeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3553,33 +3555,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 2.     1     5.0    4.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 3.     1     10.0   5.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT:        1     5.2    2.0    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 2.     1     2.0    2.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     6.0    5.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 4.     1     5.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT:        1     3.0    2.0    0.0       <total>
 
 # CHECK:      [79] Code Region - G80
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1903
+# CHECK-NEXT: Total Cycles:      1904
 # CHECK-NEXT: Total uOps:        5700
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    3.00
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    2.99
 # CHECK-NEXT: IPC:               0.26
 # CHECK-NEXT: Block RThroughput: 19.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01
+# CHECK-NEXT: Index     0123456789          012
 
-# CHECK:      [0,0]     DeeeeeER  .    .    ..   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeeeeeeER.    .    ..   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,2]     . D=====eeeeeeER    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=====eeeeeeeER  ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,4]     .   D=========eeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK:      [0,0]     DeeeeeER  .    .    . .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DeeeeeeER   .    . .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,2]     .    D===eeeeeeER   . .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .    . D==eeeeeeeER . .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .    D====eeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3589,33 +3591,33 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 2.     1     6.0    5.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 4.     1     10.0   4.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT:        1     4.8    2.0    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 2.     1     4.0    4.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     5.0    4.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT:        1     2.8    2.0    0.0       <total>
 
 # CHECK:      [80] Code Region - G81
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      500
-# CHECK-NEXT: Total Cycles:      1658
+# CHECK-NEXT: Total Cycles:      1659
 # CHECK-NEXT: Total uOps:        4900
 
-# CHECK:      Dispatch Width:    16
-# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               0.30
 # CHECK-NEXT: Block RThroughput: 16.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT: Index     0123456789          01234
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .  .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     . DeeeeeeeER   .    .  .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,2]     .  D=========eeeeeeER  .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,3]     .  D===========eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,4]     .  D============eeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK:      [0,0]     DeeeeeeeER.    .    .   .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DeeeeeeeER  .    .   .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,2]     .    .D======eeeeeeER   .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,3]     .    . D========eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,4]     .    .  D========eeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3626,10 +3628,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
 # CHECK-NEXT: 1.     1     1.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 2.     1     10.0   9.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 3.     1     12.0   1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT:        1     7.4    2.4    0.0       <total>
+# CHECK-NEXT: 2.     1     7.0    7.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 3.     1     9.0    2.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT:        1     5.4    2.2    0.0       <total>
 
 # CHECK:      [81] Code Region - G82
 
@@ -3638,7 +3640,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      757
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.30
 # CHECK-NEXT: IPC:               0.66
 # CHECK-NEXT: Block RThroughput: 7.5
@@ -3648,10 +3650,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeER .   .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D=eeeeeeER.   .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,2]     D===eeeeeeER  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,3]     .D===eeeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,4]     .D=====eeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeER.   .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,2]     . D=eeeeeeER  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,3]     .  D=eeeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .   D==eeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3661,11 +3663,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 2.     1     4.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 4.     1     6.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT:        1     3.4    0.6    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT:        1     1.8    0.6    0.0       <total>
 
 # CHECK:      [82] Code Region - G83
 
@@ -3674,7 +3676,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        2700
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.84
 # CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 7.0
@@ -3684,10 +3686,10 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,1]     D=eeeeeeER.   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,2]     .D==eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,3]     .D===eeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,4]     . D===eE--R   stg	x26, [x27], #4064
+# CHECK-NEXT: [0,1]     .DeeeeeeER.   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,2]     . D=eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,3]     .   DeeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .    DeE--R   stg	x26, [x27], #4064
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3697,11 +3699,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 2.     1     3.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 4.     1     4.0    0.0    2.0       stg	x26, [x27], #4064
-# CHECK-NEXT:        1     2.8    0.4    0.4       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    2.0       stg	x26, [x27], #4064
+# CHECK-NEXT:        1     1.2    0.4    0.4       <total>
 
 # CHECK:      [83] Code Region - G84
 
@@ -3710,19 +3712,19 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1700
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.37
 # CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 2.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   stg	x26, [x27, #4064]!
 # CHECK-NEXT: [0,1]     D=eER.  .   stgp	x1, x2, [x27], #992
-# CHECK-NEXT: [0,2]     D==eER  .   stgp	x1, x2, [x27, #992]!
-# CHECK-NEXT: [0,3]     D===eeER.   stp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,4]     .D===eeER   stp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,2]     .D=eER  .   stgp	x1, x2, [x27, #992]!
+# CHECK-NEXT: [0,3]     . D=eeER.   stp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,4]     .  D=eeER   stp	d1, d2, [x27], #496
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3733,10 +3735,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stg	x26, [x27, #4064]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       stgp	x1, x2, [x27], #992
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stgp	x1, x2, [x27, #992]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       stp	s1, s2, [x27], #248
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       stp	d1, d2, [x27], #496
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stgp	x1, x2, [x27, #992]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       stp	s1, s2, [x27], #248
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stp	d1, d2, [x27], #496
+# CHECK-NEXT:        1     1.8    0.2    0.0       <total>
 
 # CHECK:      [84] Code Region - G85
 
@@ -3745,19 +3747,19 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.84
 # CHECK-NEXT: IPC:               0.71
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeER.   .   stp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D==eeER  .   stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,2]     D===eeER .   stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,3]     .D===eeER.   stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,4]     .D=====eER   stp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,1]     .D=eeER  .   stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,2]     . D=eeER .   stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,3]     .  D=eeER.   stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,4]     .   D==eER   stp	w1, w2, [x27], #248
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3767,11 +3769,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       stp	w1, w2, [x27], #248
-# CHECK-NEXT:        1     3.6    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       stp	w1, w2, [x27], #248
+# CHECK-NEXT:        1     2.0    0.2    0.0       <total>
 
 # CHECK:      [85] Code Region - G86
 
@@ -3780,19 +3782,19 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1700
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.37
 # CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 2.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   stp	x1, x2, [x27], #496
 # CHECK-NEXT: [0,1]     D=eER.  .   stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,2]     D==eER  .   stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,3]     D===eeER.   str	b1, [x27], #254
-# CHECK-NEXT: [0,4]     .D===eeER   str	h1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eER  .   stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,3]     . D=eeER.   str	b1, [x27], #254
+# CHECK-NEXT: [0,4]     .  D=eeER   str	h1, [x27], #254
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3803,10 +3805,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	x1, x2, [x27], #496
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       str	b1, [x27], #254
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       str	h1, [x27], #254
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       str	b1, [x27], #254
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	h1, [x27], #254
+# CHECK-NEXT:        1     1.8    0.2    0.0       <total>
 
 # CHECK:      [86] Code Region - G87
 
@@ -3815,19 +3817,19 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   str	s1, [x27], #254
-# CHECK-NEXT: [0,1]     D=eeER  .   str	d1, [x27], #254
-# CHECK-NEXT: [0,2]     D==eeER .   str	q1, [x27], #254
-# CHECK-NEXT: [0,3]     D===eeER.   str	b1, [x27, #254]!
-# CHECK-NEXT: [0,4]     .D===eeER   str	h1, [x27, #254]!
+# CHECK-NEXT: [0,1]     .DeeER  .   str	d1, [x27], #254
+# CHECK-NEXT: [0,2]     . DeeER .   str	q1, [x27], #254
+# CHECK-NEXT: [0,3]     .  DeeER.   str	b1, [x27, #254]!
+# CHECK-NEXT: [0,4]     .   DeeER   str	h1, [x27, #254]!
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3837,11 +3839,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27], #254
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       str	d1, [x27], #254
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       str	q1, [x27], #254
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       str	b1, [x27, #254]!
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       str	h1, [x27, #254]!
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       str	d1, [x27], #254
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	q1, [x27], #254
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       str	b1, [x27, #254]!
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       str	h1, [x27, #254]!
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
 
 # CHECK:      [87] Code Region - G88
 
@@ -3850,19 +3852,19 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      503
 # CHECK-NEXT: Total uOps:        1800
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.58
 # CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     01234567
 
 # CHECK:      [0,0]     DeeER. .   str	s1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D=eeER .   str	d1, [x27, #254]!
-# CHECK-NEXT: [0,2]     D==eeER.   str	q1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D===eER.   str	w1, [x27], #254
-# CHECK-NEXT: [0,4]     .D===eER   str	x1, [x27], #254
+# CHECK-NEXT: [0,1]     .DeeER .   str	d1, [x27, #254]!
+# CHECK-NEXT: [0,2]     . DeeER.   str	q1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .  DeER.   str	w1, [x27], #254
+# CHECK-NEXT: [0,4]     .  D=eER   str	x1, [x27], #254
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3872,11 +3874,11 @@ ldr  x2, [x1], #254
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27, #254]!
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       str	d1, [x27, #254]!
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       str	q1, [x27, #254]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       str	w1, [x27], #254
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       str	x1, [x27], #254
-# CHECK-NEXT:        1     2.8    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       str	d1, [x27, #254]!
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	q1, [x27, #254]!
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       str	w1, [x27], #254
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	x1, [x27], #254
+# CHECK-NEXT:        1     1.2    0.2    0.0       <total>
 
 # CHECK:      [88] Code Region - G89
 
@@ -3885,7 +3887,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      503
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -3895,9 +3897,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeER . .   str	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER. .   str	x1, [x27, #254]!
-# CHECK-NEXT: [0,2]     D==eER .   strb	w1, [x27], #254
-# CHECK-NEXT: [0,3]     D===eER.   strb	w1, [x27, #254]!
-# CHECK-NEXT: [0,4]     D====eER   strh	w1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eER .   strb	w1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eER.   strb	w1, [x27, #254]!
+# CHECK-NEXT: [0,4]     . D==eER   strh	w1, [x27], #254
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3908,10 +3910,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       str	x1, [x27, #254]!
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       strb	w1, [x27], #254
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       strb	w1, [x27, #254]!
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       strh	w1, [x27], #254
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       strb	w1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       strb	w1, [x27, #254]!
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       strh	w1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [89] Code Region - G90
 
@@ -3920,7 +3922,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      503
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    2.98
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 2.5
@@ -3930,9 +3932,9 @@ ldr  x2, [x1], #254
 
 # CHECK:      [0,0]     DeER . .   strh	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER. .   stz2g	x26, [x27], #4064
-# CHECK-NEXT: [0,2]     D==eER .   stz2g	x26, [x27, #4064]!
-# CHECK-NEXT: [0,3]     D===eER.   stzg	x26, [x27], #4064
-# CHECK-NEXT: [0,4]     D====eER   stzg	x26, [x27, #4064]!
+# CHECK-NEXT: [0,2]     .D=eER .   stz2g	x26, [x27, #4064]!
+# CHECK-NEXT: [0,3]     .D==eER.   stzg	x26, [x27], #4064
+# CHECK-NEXT: [0,4]     . D==eER   stzg	x26, [x27, #4064]!
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3943,10 +3945,10 @@ ldr  x2, [x1], #254
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       strh	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       stz2g	x26, [x27], #4064
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stz2g	x26, [x27, #4064]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       stzg	x26, [x27], #4064
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       stzg	x26, [x27, #4064]!
-# CHECK-NEXT:        1     3.0    0.2    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stz2g	x26, [x27, #4064]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       stzg	x26, [x27], #4064
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       stzg	x26, [x27, #4064]!
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
 
 # CHECK:      [90] Code Region - G91
 
@@ -3955,7 +3957,7 @@ ldr  x2, [x1], #254
 # CHECK-NEXT: Total Cycles:      110
 # CHECK-NEXT: Total uOps:        400
 
-# CHECK:      Dispatch Width:    16
+# CHECK:      Dispatch Width:    6
 # CHECK-NEXT: uOps Per Cycle:    3.64
 # CHECK-NEXT: IPC:               1.82
 # CHECK-NEXT: Block RThroughput: 0.7
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s
index 1690d9669b39..3ddb52532701 100644
--- a/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/AArch64/Neoverse/V2-zero-lat-movs.s
@@ -23,18 +23,18 @@ mov  x1, x2
 # CHECK-NEXT: [6]: HasSideEffects (U)
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
-# CHECK-NEXT:  1      0     0.06                        mov	x1, #0
-# CHECK-NEXT:  1      0     0.06                        mov	x1, xzr
-# CHECK-NEXT:  1      0     0.06                        mov	w1, #0
-# CHECK-NEXT:  1      0     0.06                        mov	w1, wzr
-# CHECK-NEXT:  1      0     0.06                        fmov	h1, wzr
-# CHECK-NEXT:  1      0     0.06                        fmov	h1, xzr
-# CHECK-NEXT:  1      0     0.06                        fmov	s1, wzr
-# CHECK-NEXT:  1      0     0.06                        fmov	d1, xzr
-# CHECK-NEXT:  1      0     0.06                        movi	d1, #0000000000000000
-# CHECK-NEXT:  1      0     0.06                        movi	v1.2d, #0000000000000000
-# CHECK-NEXT:  1      0     0.06                        mov	w1, w2
-# CHECK-NEXT:  1      0     0.06                        mov	x1, x2
+# CHECK-NEXT:  1      0     0.17                        mov	x1, #0
+# CHECK-NEXT:  1      0     0.17                        mov	x1, xzr
+# CHECK-NEXT:  1      0     0.17                        mov	w1, #0
+# CHECK-NEXT:  1      0     0.17                        mov	w1, wzr
+# CHECK-NEXT:  1      0     0.17                        fmov	h1, wzr
+# CHECK-NEXT:  1      0     0.17                        fmov	h1, xzr
+# CHECK-NEXT:  1      0     0.17                        fmov	s1, wzr
+# CHECK-NEXT:  1      0     0.17                        fmov	d1, xzr
+# CHECK-NEXT:  1      0     0.17                        movi	d1, #0000000000000000
+# CHECK-NEXT:  1      0     0.17                        movi	v1.2d, #0000000000000000
+# CHECK-NEXT:  1      0     0.17                        mov	w1, w2
+# CHECK-NEXT:  1      0     0.17                        mov	x1, x2
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0.0] - V2UnitB
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/RISCV/Andes45/fpr.s b/external/llvm-project/llvm/test/tools/llvm-mca/RISCV/Andes45/fpr.s
new file mode 100644
index 000000000000..d1ab4b3b6a7e
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/RISCV/Andes45/fpr.s
@@ -0,0 +1,129 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=andes-nx45 -timeline -iterations=1 < %s | FileCheck %s
+
+// Can be dispatched in the same cycle if different functional units are used.
+fadd.s ft0, fa0, fa1
+fdiv.s ft0, fa0, fa1
+
+// FMAC
+fadd.s ft0, fa0, fa1
+fmul.s ft0, fa0, fa1
+fmadd.s ft0, fa0, fa1, fa2
+
+// FDIV
+fdiv.s ft0, fa0, fa1
+fsqrt.s ft0, fa0
+
+// FMV
+fsgnj.s ft0, fa0, fa1
+fmv.x.w a0, fa0
+
+// FMISC
+fmin.s ft0, fa0, fa1
+fclass.s a0, fa0
+feq.s a0, fa0, fa1
+fcvt.s.w ft0, a0
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      13
+# CHECK-NEXT: Total Cycles:      63
+# CHECK-NEXT: Total uOps:        13
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.21
+# CHECK-NEXT: IPC:               0.21
+# CHECK-NEXT: Block RThroughput: 56.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      4     1.00                        fadd.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      19    19.00                       fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      4     1.00                        fadd.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      4     1.00                        fmul.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      4     1.00                        fmadd.s	ft0, fa0, fa1, fa2
+# CHECK-NEXT:  1      19    19.00                       fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      18    18.00                       fsqrt.s	ft0, fa0
+# CHECK-NEXT:  1      1     1.00                        fsgnj.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      1     1.00                        fmv.x.w	a0, fa0
+# CHECK-NEXT:  1      2     1.00                        fmin.s	ft0, fa0, fa1
+# CHECK-NEXT:  1      2     1.00                        fclass.s	a0, fa0
+# CHECK-NEXT:  1      2     1.00                        feq.s	a0, fa0, fa1
+# CHECK-NEXT:  1      2     1.00                        fcvt.s.w	ft0, a0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - Andes45ALU
+# CHECK-NEXT: [0.1] - Andes45ALU
+# CHECK-NEXT: [1]   - Andes45CSR
+# CHECK-NEXT: [2]   - Andes45FDIV
+# CHECK-NEXT: [3]   - Andes45FMAC
+# CHECK-NEXT: [4]   - Andes45FMISC
+# CHECK-NEXT: [5]   - Andes45FMV
+# CHECK-NEXT: [6]   - Andes45LSU
+# CHECK-NEXT: [7]   - Andes45MDU
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7]
+# CHECK-NEXT:  -      -      -     56.00  4.00   4.00   2.00    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     fadd.s	ft0, fa0, fa1
+# CHECK-NEXT:  -      -      -     19.00   -      -      -      -      -     fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     fadd.s	ft0, fa0, fa1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     fmul.s	ft0, fa0, fa1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     fmadd.s	ft0, fa0, fa1, fa2
+# CHECK-NEXT:  -      -      -     19.00   -      -      -      -      -     fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT:  -      -      -     18.00   -      -      -      -      -     fsqrt.s	ft0, fa0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsgnj.s	ft0, fa0, fa1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmv.x.w	a0, fa0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     fmin.s	ft0, fa0, fa1
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     fclass.s	a0, fa0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     feq.s	a0, fa0, fa1
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -     fcvt.s.w	ft0, a0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789          0123456789
+# CHECK-NEXT: Index     0123456789          0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeE.    .    .    .    .    .    .    .    .    .    .    . .   fadd.s	ft0, fa0, fa1
+# CHECK-NEXT: [0,1]     DeeeeeeeeeeeeeeeeeeE.    .    .    .    .    .    .    .    . .   fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT: [0,2]     .    .    .    DeeeE.    .    .    .    .    .    .    .    . .   fadd.s	ft0, fa0, fa1
+# CHECK-NEXT: [0,3]     .    .    .    .DeeeE    .    .    .    .    .    .    .    . .   fmul.s	ft0, fa0, fa1
+# CHECK-NEXT: [0,4]     .    .    .    . DeeeE   .    .    .    .    .    .    .    . .   fmadd.s	ft0, fa0, fa1, fa2
+# CHECK-NEXT: [0,5]     .    .    .    .   DeeeeeeeeeeeeeeeeeeE .    .    .    .    . .   fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT: [0,6]     .    .    .    .    .    .    .    .  DeeeeeeeeeeeeeeeeeE   . .   fsqrt.s	ft0, fa0
+# CHECK-NEXT: [0,7]     .    .    .    .    .    .    .    .    .    .    .    DE   . .   fsgnj.s	ft0, fa0, fa1
+# CHECK-NEXT: [0,8]     .    .    .    .    .    .    .    .    .    .    .    .DE  . .   fmv.x.w	a0, fa0
+# CHECK-NEXT: [0,9]     .    .    .    .    .    .    .    .    .    .    .    .DeE . .   fmin.s	ft0, fa0, fa1
+# CHECK-NEXT: [0,10]    .    .    .    .    .    .    .    .    .    .    .    . DeE. .   fclass.s	a0, fa0
+# CHECK-NEXT: [0,11]    .    .    .    .    .    .    .    .    .    .    .    .  DeE .   feq.s	a0, fa0, fa1
+# CHECK-NEXT: [0,12]    .    .    .    .    .    .    .    .    .    .    .    .    DeE   fcvt.s.w	ft0, a0
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     0.0    0.0    0.0       fadd.s	ft0, fa0, fa1
+# CHECK-NEXT: 1.     1     0.0    0.0    0.0       fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT: 2.     1     0.0    0.0    0.0       fadd.s	ft0, fa0, fa1
+# CHECK-NEXT: 3.     1     0.0    0.0    0.0       fmul.s	ft0, fa0, fa1
+# CHECK-NEXT: 4.     1     0.0    0.0    0.0       fmadd.s	ft0, fa0, fa1, fa2
+# CHECK-NEXT: 5.     1     0.0    0.0    0.0       fdiv.s	ft0, fa0, fa1
+# CHECK-NEXT: 6.     1     0.0    0.0    0.0       fsqrt.s	ft0, fa0
+# CHECK-NEXT: 7.     1     0.0    0.0    0.0       fsgnj.s	ft0, fa0, fa1
+# CHECK-NEXT: 8.     1     0.0    0.0    0.0       fmv.x.w	a0, fa0
+# CHECK-NEXT: 9.     1     0.0    0.0    0.0       fmin.s	ft0, fa0, fa1
+# CHECK-NEXT: 10.    1     0.0    0.0    0.0       fclass.s	a0, fa0
+# CHECK-NEXT: 11.    1     0.0    0.0    0.0       feq.s	a0, fa0, fa1
+# CHECK-NEXT: 12.    1     0.0    0.0    0.0       fcvt.s.w	ft0, a0
+# CHECK-NEXT:        1     0.0    0.0    0.0       <total>
diff --git a/external/llvm-project/llvm/test/tools/llvm-mca/RISCV/Andes45/gpr.s b/external/llvm-project/llvm/test/tools/llvm-mca/RISCV/Andes45/gpr.s
new file mode 100644
index 000000000000..f6dc6eef3f0f
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-mca/RISCV/Andes45/gpr.s
@@ -0,0 +1,252 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=andes-nx45 -mattr=+zbc -timeline -iterations=1 < %s | FileCheck %s
+
+# Two ALUs without dependency can be dispatched in the same cycle.
+add a0, a0, a0
+sub a1, a1, a1
+
+# Otherwise, they can't be dispatched in the same cycle.
+addw a0, a0, a0
+subw a0, a0, a0
+
+// ALU and Shift
+slli a0, a0, 4
+slliw a0, a0, 4
+srl a0, a0, a0
+srlw a0, a0, a0
+
+// MDU
+mul a0, a0, a0
+mulw a0, a0, a0
+div a0, a0, a0
+divw a0, a0, a0
+
+// Memory
+lb a0, 4(a1)
+lh a0, 4(a1)
+lw a0, 4(a1)
+ld a0, 4(a1)
+
+flw fa0, 4(a1)
+fld fa0, 4(a1)
+
+sb a0, 4(a1)
+sh a0, 4(a1)
+sw a0, 4(a1)
+sd a0, 4(a1)
+
+// Atomic Memory
+amoswap.w a0, a1, (a0)
+amoswap.d a0, a1, (a0)
+lr.w a0, (a0)
+lr.d a0, (a0)
+sc.w a0, a1, (a0)
+sc.d a0, a1, (a0)
+
+// CSR
+csrrw a0, mstatus, zero
+
+// Bitmanip
+sh1add a0, a0, a0
+sh1add.uw a0, a0, a0
+rori a0, a0, 4
+roriw a0, a0, 4
+rol a0, a0, a0
+rolw a0, a0, a0
+clz a0, a0
+clzw a0, a0
+clmul a0, a0, a0
+bclri a0, a0, 4
+bclr a0, a0, a0
+bexti a0, a0, 4
+bext a0, a0, a0
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      42
+# CHECK-NEXT: Total Cycles:      158
+# CHECK-NEXT: Total uOps:        42
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.27
+# CHECK-NEXT: IPC:               0.27
+# CHECK-NEXT: Block RThroughput: 80.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        add	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        sub	a1, a1, a1
+# CHECK-NEXT:  1      1     0.50                        addw	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        subw	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        slli	a0, a0, 4
+# CHECK-NEXT:  1      1     0.50                        slliw	a0, a0, 4
+# CHECK-NEXT:  1      1     0.50                        srl	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        srlw	a0, a0, a0
+# CHECK-NEXT:  1      3     1.00                        mul	a0, a0, a0
+# CHECK-NEXT:  1      3     1.00                        mulw	a0, a0, a0
+# CHECK-NEXT:  1      39    39.00                       div	a0, a0, a0
+# CHECK-NEXT:  1      39    39.00                       divw	a0, a0, a0
+# CHECK-NEXT:  1      5     1.00    *                   lb	a0, 4(a1)
+# CHECK-NEXT:  1      5     1.00    *                   lh	a0, 4(a1)
+# CHECK-NEXT:  1      3     1.00    *                   lw	a0, 4(a1)
+# CHECK-NEXT:  1      3     1.00    *                   ld	a0, 4(a1)
+# CHECK-NEXT:  1      3     1.00    *                   flw	fa0, 4(a1)
+# CHECK-NEXT:  1      3     1.00    *                   fld	fa0, 4(a1)
+# CHECK-NEXT:  1      1     1.00           *            sb	a0, 4(a1)
+# CHECK-NEXT:  1      1     1.00           *            sh	a0, 4(a1)
+# CHECK-NEXT:  1      1     1.00           *            sw	a0, 4(a1)
+# CHECK-NEXT:  1      1     1.00           *            sd	a0, 4(a1)
+# CHECK-NEXT:  1      9     1.00    *      *            amoswap.w	a0, a1, (a0)
+# CHECK-NEXT:  1      9     1.00    *      *            amoswap.d	a0, a1, (a0)
+# CHECK-NEXT:  1      9     1.00    *                   lr.w	a0, (a0)
+# CHECK-NEXT:  1      9     1.00    *                   lr.d	a0, (a0)
+# CHECK-NEXT:  1      3     1.00           *            sc.w	a0, a1, (a0)
+# CHECK-NEXT:  1      3     1.00           *            sc.d	a0, a1, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     csrrw	a0, mstatus, zero
+# CHECK-NEXT:  1      1     0.50                        sh1add	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        sh1add.uw	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        rori	a0, a0, 4
+# CHECK-NEXT:  1      1     0.50                        roriw	a0, a0, 4
+# CHECK-NEXT:  1      1     0.50                        rol	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        rolw	a0, a0, a0
+# CHECK-NEXT:  1      3     0.50                        clz	a0, a0
+# CHECK-NEXT:  1      3     0.50                        clzw	a0, a0
+# CHECK-NEXT:  1      3     0.50                        clmul	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        bclri	a0, a0, 4
+# CHECK-NEXT:  1      1     0.50                        bclr	a0, a0, a0
+# CHECK-NEXT:  1      1     0.50                        bexti	a0, a0, 4
+# CHECK-NEXT:  1      1     0.50                        bext	a0, a0, a0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - Andes45ALU
+# CHECK-NEXT: [0.1] - Andes45ALU
+# CHECK-NEXT: [1]   - Andes45CSR
+# CHECK-NEXT: [2]   - Andes45FDIV
+# CHECK-NEXT: [3]   - Andes45FMAC
+# CHECK-NEXT: [4]   - Andes45FMISC
+# CHECK-NEXT: [5]   - Andes45FMV
+# CHECK-NEXT: [6]   - Andes45LSU
+# CHECK-NEXT: [7]   - Andes45MDU
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7]
+# CHECK-NEXT: 10.00  11.00  1.00    -      -      -      -     16.00  80.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     add	a0, a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	a1, a1, a1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     addw	a0, a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subw	a0, a0, a0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     slli	a0, a0, 4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     slliw	a0, a0, 4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     srl	a0, a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     srlw	a0, a0, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   mul	a0, a0, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   mulw	a0, a0, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     39.00  div	a0, a0, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     39.00  divw	a0, a0, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lb	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lh	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lw	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ld	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     flw	fa0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fld	fa0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sb	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sh	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sw	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sd	a0, 4(a1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     amoswap.w	a0, a1, (a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     amoswap.d	a0, a1, (a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lr.w	a0, (a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lr.d	a0, (a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sc.w	a0, a1, (a0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sc.d	a0, a1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -     csrrw	a0, mstatus, zero
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     sh1add	a0, a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sh1add.uw	a0, a0, a0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     rori	a0, a0, 4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     roriw	a0, a0, 4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     rol	a0, a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rolw	a0, a0, a0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     clz	a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     clzw	a0, a0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     clmul	a0, a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bclri	a0, a0, 4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     bclr	a0, a0, a0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bexti	a0, a0, 4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     bext	a0, a0, a0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789          012
+# CHECK-NEXT: Index     0123456789          0123456789          0123456789
+
+# CHECK:      [0,0]     DE   .    .    .    .    .    .    .    .    .    . .   add	a0, a0, a0
+# CHECK-NEXT: [0,1]     DE   .    .    .    .    .    .    .    .    .    . .   sub	a1, a1, a1
+# CHECK-NEXT: [0,2]     .DE  .    .    .    .    .    .    .    .    .    . .   addw	a0, a0, a0
+# CHECK-NEXT: [0,3]     . DE .    .    .    .    .    .    .    .    .    . .   subw	a0, a0, a0
+# CHECK-NEXT: [0,4]     .  DE.    .    .    .    .    .    .    .    .    . .   slli	a0, a0, 4
+# CHECK-NEXT: [0,5]     .   DE    .    .    .    .    .    .    .    .    . .   slliw	a0, a0, 4
+# CHECK-NEXT: [0,6]     .    DE   .    .    .    .    .    .    .    .    . .   srl	a0, a0, a0
+# CHECK-NEXT: [0,7]     .    .DE  .    .    .    .    .    .    .    .    . .   srlw	a0, a0, a0
+# CHECK-NEXT: [0,8]     .    . DeeE    .    .    .    .    .    .    .    . .   mul	a0, a0, a0
+# CHECK-NEXT: [0,9]     .    .    DeeE .    .    .    .    .    .    .    . .   mulw	a0, a0, a0
+# CHECK-NEXT: [0,10]    .    .    .  DeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeE   div	a0, a0, a0
+# CHECK-NEXT: Truncated display due to cycle limit
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     0.0    0.0    0.0       add	a0, a0, a0
+# CHECK-NEXT: 1.     1     0.0    0.0    0.0       sub	a1, a1, a1
+# CHECK-NEXT: 2.     1     0.0    0.0    0.0       addw	a0, a0, a0
+# CHECK-NEXT: 3.     1     0.0    0.0    0.0       subw	a0, a0, a0
+# CHECK-NEXT: 4.     1     0.0    0.0    0.0       slli	a0, a0, 4
+# CHECK-NEXT: 5.     1     0.0    0.0    0.0       slliw	a0, a0, 4
+# CHECK-NEXT: 6.     1     0.0    0.0    0.0       srl	a0, a0, a0
+# CHECK-NEXT: 7.     1     0.0    0.0    0.0       srlw	a0, a0, a0
+# CHECK-NEXT: 8.     1     0.0    0.0    0.0       mul	a0, a0, a0
+# CHECK-NEXT: 9.     1     0.0    0.0    0.0       mulw	a0, a0, a0
+# CHECK-NEXT: 10.    1     0.0    0.0    0.0       div	a0, a0, a0
+# CHECK-NEXT: 11.    1     0.0    0.0    0.0       divw	a0, a0, a0
+# CHECK-NEXT: 12.    1     0.0    0.0    0.0       lb	a0, 4(a1)
+# CHECK-NEXT: 13.    1     0.0    0.0    0.0       lh	a0, 4(a1)
+# CHECK-NEXT: 14.    1     0.0    0.0    0.0       lw	a0, 4(a1)
+# CHECK-NEXT: 15.    1     0.0    0.0    0.0       ld	a0, 4(a1)
+# CHECK-NEXT: 16.    1     0.0    0.0    0.0       flw	fa0, 4(a1)
+# CHECK-NEXT: 17.    1     0.0    0.0    0.0       fld	fa0, 4(a1)
+# CHECK-NEXT: 18.    1     0.0    0.0    0.0       sb	a0, 4(a1)
+# CHECK-NEXT: 19.    1     0.0    0.0    0.0       sh	a0, 4(a1)
+# CHECK-NEXT: 20.    1     0.0    0.0    0.0       sw	a0, 4(a1)
+# CHECK-NEXT: 21.    1     0.0    0.0    0.0       sd	a0, 4(a1)
+# CHECK-NEXT: 22.    1     0.0    0.0    0.0       amoswap.w	a0, a1, (a0)
+# CHECK-NEXT: 23.    1     0.0    0.0    0.0       amoswap.d	a0, a1, (a0)
+# CHECK-NEXT: 24.    1     0.0    0.0    0.0       lr.w	a0, (a0)
+# CHECK-NEXT: 25.    1     0.0    0.0    0.0       lr.d	a0, (a0)
+# CHECK-NEXT: 26.    1     0.0    0.0    0.0       sc.w	a0, a1, (a0)
+# CHECK-NEXT: 27.    1     0.0    0.0    0.0       sc.d	a0, a1, (a0)
+# CHECK-NEXT: 28.    1     0.0    0.0    0.0       csrrw	a0, mstatus, zero
+# CHECK-NEXT: 29.    1     0.0    0.0    0.0       sh1add	a0, a0, a0
+# CHECK-NEXT: 30.    1     0.0    0.0    0.0       sh1add.uw	a0, a0, a0
+# CHECK-NEXT: 31.    1     0.0    0.0    0.0       rori	a0, a0, 4
+# CHECK-NEXT: 32.    1     0.0    0.0    0.0       roriw	a0, a0, 4
+# CHECK-NEXT: 33.    1     0.0    0.0    0.0       rol	a0, a0, a0
+# CHECK-NEXT: 34.    1     0.0    0.0    0.0       rolw	a0, a0, a0
+# CHECK-NEXT: 35.    1     0.0    0.0    0.0       clz	a0, a0
+# CHECK-NEXT: 36.    1     0.0    0.0    0.0       clzw	a0, a0
+# CHECK-NEXT: 37.    1     0.0    0.0    0.0       clmul	a0, a0, a0
+# CHECK-NEXT: 38.    1     0.0    0.0    0.0       bclri	a0, a0, 4
+# CHECK-NEXT: 39.    1     0.0    0.0    0.0       bclr	a0, a0, a0
+# CHECK-NEXT: 40.    1     0.0    0.0    0.0       bexti	a0, a0, 4
+# CHECK-NEXT: 41.    1     0.0    0.0    0.0       bext	a0, a0, a0
+# CHECK-NEXT:        1     0.0    0.0    0.0       <total>
diff --git a/external/llvm-project/llvm/test/tools/llvm-pdbutil/Inputs/register-records.yaml b/external/llvm-project/llvm/test/tools/llvm-pdbutil/Inputs/register-records.yaml
new file mode 100644
index 000000000000..2e7d707529eb
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-pdbutil/Inputs/register-records.yaml
@@ -0,0 +1,21 @@
+---
+DbiStream:
+  VerHeader:       V70
+  Age:             1
+  BuildNumber:     36363
+  PdbDllVersion:   0
+  PdbDllRbld:      0
+  Flags:           0
+  MachineType:     Amd64
+  Modules:
+    - Module:          '/tmp/test.obj'
+      Modi:
+        Signature:       4
+        Records:
+          - Kind:            S_REGREL32
+            RegRelativeSym:
+              Offset:          56
+              Type:            4494
+              Register:        RSP
+              VarName:         this
+...
diff --git a/external/llvm-project/llvm/test/tools/llvm-pdbutil/register-records.test b/external/llvm-project/llvm/test/tools/llvm-pdbutil/register-records.test
new file mode 100644
index 000000000000..60a02732ba94
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-pdbutil/register-records.test
@@ -0,0 +1,18 @@
+; RUN: llvm-pdbutil yaml2pdb %p/Inputs/register-records.yaml --pdb=%t.pdb
+; RUN: llvm-pdbutil dump --symbols %t.pdb | FileCheck --check-prefix=CHECK_YAML2PDB %s
+
+; RUN: llvm-pdbutil pdb2yaml --module-syms %t.pdb > %t.yaml
+; RUN: FileCheck --input-file=%t.yaml --check-prefix=CHECK_PDB2YAML %s
+
+CHECK_YAML2PDB:                           Symbols                           
+CHECK_YAML2PDB: ============================================================
+CHECK_YAML2PDB:   Mod 0000 | `/tmp/test.obj`:
+CHECK_YAML2PDB:        4 | S_REGREL32 [size = 20] `this`
+CHECK_YAML2PDB:            type = 0x118E (<unknown UDT>), register = RSP, offset = 56
+
+CHECK_PDB2YAML: - Kind:            S_REGREL32
+CHECK_PDB2YAML:   RegRelativeSym:
+CHECK_PDB2YAML:    Offset:          56
+CHECK_PDB2YAML:    Type:            4494
+CHECK_PDB2YAML:    Register:        RSP
+CHECK_PDB2YAML:    VarName:         this
diff --git a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/parser-expr.rc b/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/parser-expr.rc
index 8e69c1cd1fa1..2f8e4b2d344a 100644
--- a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/parser-expr.rc
+++ b/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/parser-expr.rc
@@ -5,6 +5,11 @@ LANGUAGE 1|1&0, 0&0|1
 LANGUAGE 3+4-5, 3-4+5
 LANGUAGE 1+2|3, 3|1+2
 LANGUAGE 6&~5, 6&-8
+LANGUAGE 7/3, 7*3
+LANGUAGE 5/2*2, 5*3/2
+LANGUAGE 1+2*3, (1+2)*3
+LANGUAGE 100/12/5*5, 1+1+1+1*4
+LANGUAGE 9/(1+3), (4+5)/4
 LANGUAGE -1, --1
 LANGUAGE ----1, -----1
 LANGUAGE ~1, ~~1
diff --git a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tag-accelerators-ascii-alt.rc b/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tag-accelerators-ascii-alt.rc
deleted file mode 100644
index 363263bfe4cf..000000000000
--- a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tag-accelerators-ascii-alt.rc
+++ /dev/null
@@ -1,4 +0,0 @@
-2 ACCELERATORS {
-  "A", 15, ASCII, ALT
-}
-
diff --git a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tag-accelerators.rc b/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tag-accelerators.rc
index 90e7f926cc08..bcfc35bdeab6 100644
--- a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tag-accelerators.rc
+++ b/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tag-accelerators.rc
@@ -110,5 +110,6 @@ LANGUAGE 5, 1
   "7", 71, VIRTKEY, NOINVERT, CONTROL, SHIFT, ALT
   "^j", 72, ASCII
   "^j", 73, ASCII, NOINVERT
+  "A", 15, ASCII, ALT
 }
 
diff --git a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tokens.rc b/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tokens.rc
index 6a781202a7e3..20f77912477d 100644
--- a/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tokens.rc
+++ b/external/llvm-project/llvm/test/tools/llvm-rc/Inputs/tokens.rc
@@ -1,4 +1,5 @@
 1 + 2 - 3214L & 0x120894 032173 2|&~+(-7){0xabcdef 0xABCDEFl} Begin End
+1*3/4
 He11o LLVM
 identifier-with-dashes
 
diff --git a/external/llvm-project/llvm/test/tools/llvm-rc/parser-expr.test b/external/llvm-project/llvm/test/tools/llvm-rc/parser-expr.test
index ed6796529fdf..14a299c9e3e9 100644
--- a/external/llvm-project/llvm/test/tools/llvm-rc/parser-expr.test
+++ b/external/llvm-project/llvm/test/tools/llvm-rc/parser-expr.test
@@ -7,6 +7,11 @@
 ; CHECK-NEXT:  Language: 2, Sublanguage: 4
 ; CHECK-NEXT:  Language: 3, Sublanguage: 5
 ; CHECK-NEXT:  Language: 2, Sublanguage: 0
+; CHECK-NEXT:  Language: 2, Sublanguage: 21
+; CHECK-NEXT:  Language: 4, Sublanguage: 7
+; CHECK-NEXT:  Language: 7, Sublanguage: 9
+; CHECK-NEXT:  Language: 5, Sublanguage: 7
+; CHECK-NEXT:  Language: 2, Sublanguage: 2
 ; CHECK-NEXT:  Language: 4294967295, Sublanguage: 1
 ; CHECK-NEXT:  Language: 1, Sublanguage: 4294967295
 ; CHECK-NEXT:  Language: 4294967294, Sublanguage: 1
diff --git a/external/llvm-project/llvm/test/tools/llvm-rc/tag-accelerators.test b/external/llvm-project/llvm/test/tools/llvm-rc/tag-accelerators.test
index 336727f61768..4f44aebc7501 100644
--- a/external/llvm-project/llvm/test/tools/llvm-rc/tag-accelerators.test
+++ b/external/llvm-project/llvm/test/tools/llvm-rc/tag-accelerators.test
@@ -37,7 +37,7 @@
 ; ACCELERATORS-NEXT: Version (major): 0
 ; ACCELERATORS-NEXT: Version (minor): 0
 ; ACCELERATORS-NEXT: Characteristics: 0
-; ACCELERATORS-NEXT: Data size: 592
+; ACCELERATORS-NEXT: Data size: 600
 ; ACCELERATORS-NEXT: Data: (
 ; ACCELERATORS-NEXT:   0000: 00002A00 00000000 01002A00 01000000  |..*.......*.....|
 ; ACCELERATORS-NEXT:   0010: 02002A00 02000000 03002A00 03000000  |..*.......*.....|
@@ -75,7 +75,8 @@
 ; ACCELERATORS-NEXT:   0210: 15003700 42000000 0F003700 43000000  |..7.B.....7.C...|
 ; ACCELERATORS-NEXT:   0220: 1B003700 44000000 17003700 45000000  |..7.D.....7.E...|
 ; ACCELERATORS-NEXT:   0230: 1D003700 46000000 1F003700 47000000  |..7.F.....7.G...|
-; ACCELERATORS-NEXT:   0240: 00000A00 48000000 82000A00 49000000  |....H.......I...|
+; ACCELERATORS-NEXT:   0240: 00000A00 48000000 02000A00 49000000  |....H.......I...|
+; ACCELERATORS-NEXT:   0250: 90004100 0F000000                    |..A.....|
 ; ACCELERATORS-NEXT: )
 
 
@@ -94,19 +95,13 @@
 ; RUN: not llvm-rc -no-preprocess /FO %t -- %p/Inputs/tag-accelerators-ascii-control.rc 2>&1 | FileCheck %s --check-prefix ASCII2
 
 ; ASCII2: llvm-rc: Error in ACCELERATORS statement (ID 2):
-; ASCII2-NEXT: Accelerator ID 15: Can only apply ALT, SHIFT or CONTROL to VIRTKEY accelerators
+; ASCII2-NEXT: Accelerator ID 15: Can only apply SHIFT or CONTROL to VIRTKEY accelerators
 
 
 ; RUN: not llvm-rc -no-preprocess /FO %t -- %p/Inputs/tag-accelerators-ascii-shift.rc 2>&1 | FileCheck %s --check-prefix ASCII3
 
 ; ASCII3: llvm-rc: Error in ACCELERATORS statement (ID 2):
-; ASCII3-NEXT: Accelerator ID 15: Can only apply ALT, SHIFT or CONTROL to VIRTKEY accelerators
-
-
-; RUN: not llvm-rc -no-preprocess /FO %t -- %p/Inputs/tag-accelerators-ascii-alt.rc 2>&1 | FileCheck %s --check-prefix ASCII4
-
-; ASCII4: llvm-rc: Error in ACCELERATORS statement (ID 2):
-; ASCII4-NEXT: Accelerator ID 15: Can only apply ALT, SHIFT or CONTROL to VIRTKEY accelerators
+; ASCII3-NEXT: Accelerator ID 15: Can only apply SHIFT or CONTROL to VIRTKEY accelerators
 
 
 ; RUN: not llvm-rc -no-preprocess /FO %t -- %p/Inputs/tag-accelerators-bad-key-id.rc 2>&1 | FileCheck %s --check-prefix BADKEYID
diff --git a/external/llvm-project/llvm/test/tools/llvm-rc/tokenizer.test b/external/llvm-project/llvm/test/tools/llvm-rc/tokenizer.test
index 8486f8bd7869..3062e2bf6462 100644
--- a/external/llvm-project/llvm/test/tools/llvm-rc/tokenizer.test
+++ b/external/llvm-project/llvm/test/tools/llvm-rc/tokenizer.test
@@ -25,6 +25,11 @@
 ; CHECK-NEXT:  BlockEnd: }
 ; CHECK-NEXT:  BlockBegin: Begin
 ; CHECK-NEXT:  BlockEnd: End
+; CHECK-NEXT:  Int: 1; int value = 1
+; CHECK-NEXT:  Asterisk: *
+; CHECK-NEXT:  Int: 3; int value = 3
+; CHECK-NEXT:  Slash: /
+; CHECK-NEXT:  Int: 4; int value = 4
 ; CHECK-NEXT:  Identifier: He11o
 ; CHECK-NEXT:  Identifier: LLVM
 ; CHECK-NEXT:  Identifier: identifier-with-dashes
diff --git a/external/llvm-project/llvm/test/tools/llvm-readobj/COFF/string-table.test b/external/llvm-project/llvm/test/tools/llvm-readobj/COFF/string-table.test
new file mode 100644
index 000000000000..57de89f8009d
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-readobj/COFF/string-table.test
@@ -0,0 +1,37 @@
+# RUN: yaml2obj %s -o %t.obj
+# RUN: llvm-readobj --string-table %t.obj \
+# RUN:   %p/Inputs/coff-load-config-x64.dll \
+# RUN:   %p/Inputs/zero-string-table.obj.coff-i386 | FileCheck %s
+
+# CHECK-LABEL: File: {{.*}}string-table.test.tmp.obj
+# CHECK:       StringTable {
+# CHECK-NEXT:    Length: 31
+# CHECK-NEXT:    [     4] .debug_str
+# CHECK-NEXT:    [     f] _main_test_test
+# CHECK-NEXT:  }
+
+# CHECK-LABEL: File: {{.*}}coff-load-config-x64.dll
+# CHECK:       StringTable {
+# CHECK-NEXT:    Length: 0
+# CHECK-NEXT:  }
+
+# CHECK-LABEL: File: {{.*}}zero-string-table.obj.coff-i386
+# CHECK:       StringTable {
+# CHECK-NEXT:    Length: 4
+# CHECK-NEXT:  }
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+  - Name:            .debug_str
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+symbols:
+  - Name:            _main_test_test
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
diff --git a/external/llvm-project/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-compressed-zstd.test b/external/llvm-project/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-compressed-zstd.test
new file mode 100644
index 000000000000..5a379760e409
--- /dev/null
+++ b/external/llvm-project/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-compressed-zstd.test
@@ -0,0 +1,142 @@
+# REQUIRES: zstd
+## This test checks that we can read compressed SHT_LLVM_BB_ADDR_MAP sections.
+
+## Check 64-bit:
+# RUN: yaml2obj --docnum=1 %s -DBITS=64 -DADDR=0x999999999 -o %t1.x64.o
+# RUN: llvm-objcopy %t1.x64.o --compress-sections .llvm_bb_addr_map=zstd %t1.x64.compressed.o
+# RUN: llvm-readobj %t1.x64.compressed.o --bb-addr-map 2>&1 | FileCheck %s -DADDR=0x999999999 -DFILE=%t1.x64.compressed.o --check-prefix=CHECK
+
+## Check 32-bit:
+# RUN: yaml2obj --docnum=1 %s -DBITS=32 -o %t1.x32.o
+# RUN: llvm-objcopy %t1.x32.o --compress-sections .llvm_bb_addr_map=zstd %t1.x32.compressed.o
+# RUN: llvm-readobj %t1.x32.compressed.o --bb-addr-map 2>&1 | FileCheck %s -DADDR=0x11111 -DFILE=%t1.x32.compressed.o --check-prefix=CHECK
+
+# CHECK:      BBAddrMap [
+# CHECK-NEXT:   Function {
+# CHECK-NEXT:     At: [[ADDR]]
+# CHECK-NEXT: warning: '[[FILE]]': could not identify function symbol for address ([[ADDR]]) in SHT_LLVM_BB_ADDR_MAP section with index 3
+# CHECK-NEXT:     Name: <?>
+# CHECK-NEXT:     BB Ranges [
+# CHECK-NEXT:       {
+# CHECK-NEXT:         Base Address: [[ADDR]]
+# CHECK-NEXT:         BB Entries [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             ID: 0
+# CHECK-NEXT:             Offset: 0x0
+# CHECK-NEXT:             Size: 0x1
+# CHECK-NEXT:             HasReturn: No
+# CHECK-NEXT:             HasTailCall: Yes
+# CHECK-NEXT:             IsEHPad: No
+# CHECK-NEXT:             CanFallThrough: No
+# CHECK-NEXT:             HasIndirectBranch: No
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ]
+# CHECK-NEXT:       }
+# CHECK-NEXT:       {
+# CHECK-NEXT:         Base Address: 0x44444
+# CHECK-NEXT:         BB Entries [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             ID: 2
+# CHECK-NEXT:             Offset: 0x3
+# CHECK-NEXT:             Size: 0x4
+# CHECK-NEXT:             HasReturn: Yes
+# CHECK-NEXT:             HasTailCall: No
+# CHECK-NEXT:             IsEHPad: Yes
+# CHECK-NEXT:             CanFallThrough: No
+# CHECK-NEXT:             HasIndirectBranch: Yes
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ]
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Function {
+# CHECK-NEXT:     At: 0x22222
+# CHECK-NEXT:     Name: foo
+# CHECK-NEXT:     BB Ranges [
+# CHECK-NEXT:       {
+# CHECK-NEXT:         Base Address: 0x22222
+# CHECK-NEXT:         BB Entries [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             ID: 4
+# CHECK-NEXT:             Offset: 0x6
+# CHECK-NEXT:             Size: 0x7
+# CHECK-NEXT:             HasReturn: No
+# CHECK-NEXT:             HasTailCall: No
+# CHECK-NEXT:             IsEHPad: No
+# CHECK-NEXT:             CanFallThrough: Yes
+# CHECK-NEXT:            HasIndirectBranch: No
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ]
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS[[BITS]]
+  Data:  ELFDATA2LSB
+  Type:  ET_EXEC
+Sections:
+  - Name:   .text
+    Type:   SHT_PROGBITS
+    Flags:  [SHF_ALLOC]
+  - Name:   .text.bar
+    Type:   SHT_PROGBITS
+    Flags:  [SHF_ALLOC]
+  - Name:   .llvm_bb_addr_map
+    Type:   SHT_LLVM_BB_ADDR_MAP
+    ShSize: [[SIZE=<none>]]
+    Link:   .text
+    Entries:
+      - Version: 2
+        Feature: 0x8
+        BBRanges:
+          - BaseAddress: [[ADDR=0x11111]]
+            BBEntries:
+              - ID:            0
+                AddressOffset: 0x0
+                Size:          0x1
+                Metadata:      [[METADATA=0x2]]
+          - BaseAddress: 0x44444
+            BBEntries:
+              - ID:            2
+                AddressOffset: 0x3
+                Size:          0x4
+                Metadata:      0x15
+      - Version: 2
+        BBRanges:
+          - BaseAddress: 0x22222
+            BBEntries:
+              - ID:            4
+                AddressOffset: 0x6
+                Size:          0x7
+                Metadata:      0x8
+  - Name: dummy_section
+    Type: SHT_PROGBITS
+    Size: 16
+  - Name: '.llvm_bb_addr_map (1)'
+    Type: SHT_LLVM_BB_ADDR_MAP
+    Link: .text.bar
+    Entries:
+      - Version: 2
+        BBRanges:
+          - BaseAddress: 0x33333
+            BBEntries:
+              - ID:            6
+                AddressOffset: 0x9
+                Size:          0xa
+                Metadata:      0x1b
+              - ID:            7
+                AddressOffset: 0xc
+                Size:          0xd
+                Metadata:      0xe
+Symbols:
+  - Name:    foo
+    Section: .text
+    Type:    STT_FUNC
+    Value:   0x22222
+  - Name:    bar
+    Section: .text.bar
+    Type:    STT_FUNC
+    Value:   0x33333
diff --git a/external/llvm-project/llvm/test/tools/llvm-remarkutil/convert.test b/external/llvm-project/llvm/test/tools/llvm-remarkutil/convert.test
index 83023c8ce6a8..0d7ab8e4682a 100644
--- a/external/llvm-project/llvm/test/tools/llvm-remarkutil/convert.test
+++ b/external/llvm-project/llvm/test/tools/llvm-remarkutil/convert.test
@@ -1,6 +1,7 @@
-RUN: llvm-remarkutil bitstream2yaml %p/Inputs/two-remarks.bitstream -o - | FileCheck %s -strict-whitespace
-RUN: llvm-remarkutil yaml2bitstream %p/Inputs/two-remarks.yaml -o %t
-RUN: llvm-remarkutil bitstream2yaml %t -o - | FileCheck %s -strict-whitespace
+RUN: llvm-remarkutil bitstream2yaml %p/Inputs/two-remarks.bitstream -o %t.yaml
+RUN: FileCheck %s -strict-whitespace < %t.yaml
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/two-remarks.yaml -o %t.bitstream
+RUN: llvm-remarkutil bitstream2yaml %t.bitstream -o - | FileCheck %s -strict-whitespace
 
 ; CHECK: --- !Analysis
 ; CHECK-NEXT: Pass:            prologepilog
diff --git a/external/llvm-project/llvm/test/tools/llvm-symbolizer/symbol-search.test b/external/llvm-project/llvm/test/tools/llvm-symbolizer/symbol-search.test
index b785cf1596fa..bb07c54e100b 100644
--- a/external/llvm-project/llvm/test/tools/llvm-symbolizer/symbol-search.test
+++ b/external/llvm-project/llvm/test/tools/llvm-symbolizer/symbol-search.test
@@ -66,7 +66,7 @@ RUN: llvm-addr2line --obj=%p/Inputs/symbols.so func_01+0A | FileCheck --check-pr
 
 # If '+' is not preceded by a symbol, it is part of a symbol name, not an offset separator.
 RUN: llvm-symbolizer --obj=%p/Inputs/symbols.so +0x1138 | FileCheck --check-prefix=NONEXISTENT %s
-# in addr2line address starting with a `+` sign is a valid address
+# In addr2line address starting with a `+` sign is a valid address.
 RUN: llvm-addr2line --obj=%p/Inputs/symbols.so +0x1138 | FileCheck --check-prefix=CODE-CMD %s
 
 # Show that C++ mangled names may be specified.
diff --git a/external/llvm-project/llvm/tools/llvm-as/llvm-as.cpp b/external/llvm-project/llvm/tools/llvm-as/llvm-as.cpp
index 081bcb6faa71..21648674b51f 100644
--- a/external/llvm-project/llvm/tools/llvm-as/llvm-as.cpp
+++ b/external/llvm-project/llvm/tools/llvm-as/llvm-as.cpp
@@ -66,7 +66,6 @@ static cl::opt<std::string> ClDataLayout("data-layout",
                                          cl::desc("data layout string to use"),
                                          cl::value_desc("layout-string"),
                                          cl::init(""), cl::cat(AsCat));
-extern cl::opt<bool> UseNewDbgInfoFormat;
 
 static void WriteOutputFile(const Module *M, const ModuleSummaryIndex *Index) {
   // Infer the output filename if needed.
@@ -140,10 +139,7 @@ int main(int argc, char **argv) {
     return 1;
   }
 
-  // Convert to new debug format if requested.
-  M->setIsNewDbgInfoFormat(UseNewDbgInfoFormat);
-  if (M->IsNewDbgInfoFormat)
-    M->removeDebugIntrinsicDeclarations();
+  M->removeDebugIntrinsicDeclarations();
 
   std::unique_ptr<ModuleSummaryIndex> Index = std::move(ModuleAndIndex.Index);
 
diff --git a/external/llvm-project/llvm/tools/llvm-cfi-verify/lib/FileAnalysis.cpp b/external/llvm-project/llvm/tools/llvm-cfi-verify/lib/FileAnalysis.cpp
index e6f36735e84b..25ad985c5258 100644
--- a/external/llvm-project/llvm/tools/llvm-cfi-verify/lib/FileAnalysis.cpp
+++ b/external/llvm-project/llvm/tools/llvm-cfi-verify/lib/FileAnalysis.cpp
@@ -278,8 +278,7 @@ Expected<DIInliningInfo>
 FileAnalysis::symbolizeInlinedCode(object::SectionedAddress Address) {
   assert(Symbolizer != nullptr && "Symbolizer is invalid.");
 
-  return Symbolizer->symbolizeInlinedCode(std::string(Object->getFileName()),
-                                          Address);
+  return Symbolizer->symbolizeInlinedCode(Object->getFileName(), Address);
 }
 
 CFIProtectionStatus
@@ -523,9 +522,8 @@ void FileAnalysis::parseSectionContents(ArrayRef<uint8_t> SectionBytes,
 
     // Check if this instruction exists in the range of the DWARF metadata.
     if (!IgnoreDWARFFlag) {
-      auto LineInfo =
-          Symbolizer->symbolizeCode(std::string(Object->getFileName()),
-                                    {VMAddress, Address.SectionIndex});
+      auto LineInfo = Symbolizer->symbolizeCode(
+          Object->getFileName(), {VMAddress, Address.SectionIndex});
       if (!LineInfo) {
         handleAllErrors(LineInfo.takeError(), [](const ErrorInfoBase &E) {
           errs() << "Symbolizer failed to get line: " << E.message() << "\n";
diff --git a/external/llvm-project/llvm/tools/llvm-cgdata/llvm-cgdata.cpp b/external/llvm-project/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
index 5c028311b06b..98fa5c565735 100644
--- a/external/llvm-project/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
+++ b/external/llvm-project/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
@@ -75,8 +75,8 @@ class CGDataOptTable : public opt::GenericOptTable {
 
 // Options
 static StringRef ToolName;
-static StringRef OutputFilename = "-";
-static StringRef Filename;
+static std::string OutputFilename = "-";
+static std::string Filename;
 static bool ShowCGDataVersion;
 static bool SkipTrim;
 static CGDataAction Action;
diff --git a/external/llvm-project/llvm/tools/llvm-cov/CodeCoverage.cpp b/external/llvm-project/llvm/tools/llvm-cov/CodeCoverage.cpp
index 1f2484cd4dda..6c66858c4de8 100644
--- a/external/llvm-project/llvm/tools/llvm-cov/CodeCoverage.cpp
+++ b/external/llvm-project/llvm/tools/llvm-cov/CodeCoverage.cpp
@@ -153,7 +153,7 @@ class CodeCoverageTool {
   bool HadSourceFiles = false;
 
   /// The path to the indexed profile.
-  std::string PGOFilename;
+  std::optional<std::string> PGOFilename;
 
   /// A list of input source files.
   std::vector<std::string> SourceFiles;
@@ -455,10 +455,12 @@ static bool modifiedTimeGT(StringRef LHS, StringRef RHS) {
 }
 
 std::unique_ptr<CoverageMapping> CodeCoverageTool::load() {
-  for (StringRef ObjectFilename : ObjectFilenames)
-    if (modifiedTimeGT(ObjectFilename, PGOFilename))
-      warning("profile data may be out of date - object is newer",
-              ObjectFilename);
+  if (PGOFilename) {
+    for (StringRef ObjectFilename : ObjectFilenames)
+      if (modifiedTimeGT(ObjectFilename, PGOFilename.value()))
+        warning("profile data may be out of date - object is newer",
+                ObjectFilename);
+  }
   auto FS = vfs::getRealFileSystem();
   auto CoverageOrErr = CoverageMapping::load(
       ObjectFilenames, PGOFilename, *FS, CoverageArches,
@@ -668,11 +670,16 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) {
       "dump-collected-paths", cl::Optional, cl::Hidden,
       cl::desc("Show the collected paths to source files"));
 
-  cl::opt<std::string, true> PGOFilename(
-      "instr-profile", cl::Required, cl::location(this->PGOFilename),
+  cl::opt<std::string> PGOFilename(
+      "instr-profile", cl::Optional,
       cl::desc(
           "File with the profile data obtained after an instrumented run"));
 
+  cl::opt<bool> EmptyProfile(
+      "empty-profile", cl::Optional,
+      cl::desc("Use a synthetic profile with no data to generate "
+               "baseline coverage"));
+
   cl::list<std::string> Arches(
       "arch", cl::desc("architectures of the coverage mapping binaries"));
 
@@ -805,6 +812,15 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) {
     }
     this->CheckBinaryIDs = CheckBinaryIDs;
 
+    if (!PGOFilename.empty() == EmptyProfile) {
+      error(
+          "exactly one of -instr-profile and -empty-profile must be specified");
+      return 1;
+    }
+    if (!PGOFilename.empty()) {
+      this->PGOFilename = std::make_optional(PGOFilename.getValue());
+    }
+
     if (!CovFilename.empty())
       ObjectFilenames.emplace_back(CovFilename);
     for (const std::string &Filename : CovFilenames)
@@ -1116,20 +1132,22 @@ int CodeCoverageTool::doShow(int argc, const char **argv,
     }
   }
 
-  sys::fs::file_status Status;
-  if (std::error_code EC = sys::fs::status(PGOFilename, Status)) {
-    error("could not read profile data!" + EC.message(), PGOFilename);
-    return 1;
-  }
+  if (PGOFilename) {
+    sys::fs::file_status Status;
+    if (std::error_code EC = sys::fs::status(PGOFilename.value(), Status)) {
+      error("could not read profile data!" + EC.message(), PGOFilename.value());
+      return 1;
+    }
 
-  if (ShowCreatedTime) {
-    auto ModifiedTime = Status.getLastModificationTime();
-    std::string ModifiedTimeStr = to_string(ModifiedTime);
-    size_t found = ModifiedTimeStr.rfind(':');
-    ViewOpts.CreatedTimeStr =
-        (found != std::string::npos)
-            ? "Created: " + ModifiedTimeStr.substr(0, found)
-            : "Created: " + ModifiedTimeStr;
+    if (ShowCreatedTime) {
+      auto ModifiedTime = Status.getLastModificationTime();
+      std::string ModifiedTimeStr = to_string(ModifiedTime);
+      size_t found = ModifiedTimeStr.rfind(':');
+      ViewOpts.CreatedTimeStr =
+          (found != std::string::npos)
+              ? "Created: " + ModifiedTimeStr.substr(0, found)
+              : "Created: " + ModifiedTimeStr;
+    }
   }
 
   auto Coverage = load();
@@ -1238,10 +1256,12 @@ int CodeCoverageTool::doReport(int argc, const char **argv,
     return 1;
   }
 
-  sys::fs::file_status Status;
-  if (std::error_code EC = sys::fs::status(PGOFilename, Status)) {
-    error("could not read profile data!" + EC.message(), PGOFilename);
-    return 1;
+  if (PGOFilename) {
+    sys::fs::file_status Status;
+    if (std::error_code EC = sys::fs::status(PGOFilename.value(), Status)) {
+      error("could not read profile data!" + EC.message(), PGOFilename.value());
+      return 1;
+    }
   }
 
   auto Coverage = load();
@@ -1303,10 +1323,12 @@ int CodeCoverageTool::doExport(int argc, const char **argv,
     return 1;
   }
 
-  sys::fs::file_status Status;
-  if (std::error_code EC = sys::fs::status(PGOFilename, Status)) {
-    error("could not read profile data!" + EC.message(), PGOFilename);
-    return 1;
+  if (PGOFilename) {
+    sys::fs::file_status Status;
+    if (std::error_code EC = sys::fs::status(PGOFilename.value(), Status)) {
+      error("could not read profile data!" + EC.message(), PGOFilename.value());
+      return 1;
+    }
   }
 
   auto Coverage = load();
diff --git a/external/llvm-project/llvm/tools/llvm-cov/SourceCoverageViewText.cpp b/external/llvm-project/llvm/tools/llvm-cov/SourceCoverageViewText.cpp
index 765f8bbbd8d1..df8eb1d87187 100644
--- a/external/llvm-project/llvm/tools/llvm-cov/SourceCoverageViewText.cpp
+++ b/external/llvm-project/llvm/tools/llvm-cov/SourceCoverageViewText.cpp
@@ -186,8 +186,8 @@ void SourceCoverageViewText::renderLine(raw_ostream &OS, LineRef L,
     if (getOptions().Debug && Highlight)
       HighlightedRanges.push_back(std::make_pair(Col, End));
     Col = End;
-    if ((!S->IsGapRegion || (Highlight && *Highlight == raw_ostream::RED)) &&
-        S->HasCount && S->Count == 0)
+    if ((!S->IsGapRegion || Highlight == raw_ostream::RED) && S->HasCount &&
+        S->Count == 0)
       Highlight = raw_ostream::RED;
     else if (Col == ExpansionCol)
       Highlight = raw_ostream::CYAN;
diff --git a/external/llvm-project/llvm/tools/llvm-debuginfo-analyzer/Options.cpp b/external/llvm-project/llvm/tools/llvm-debuginfo-analyzer/Options.cpp
index 86ca87376348..79e2edccc50b 100644
--- a/external/llvm-project/llvm/tools/llvm-debuginfo-analyzer/Options.cpp
+++ b/external/llvm-project/llvm/tools/llvm-debuginfo-analyzer/Options.cpp
@@ -89,6 +89,8 @@ cl::list<LVAttributeKind> cmdline::AttributeOptions(
                       "Element referenced across Compile Units."),
            clEnumValN(LVAttributeKind::Inserted, "inserted",
                       "Generated inlined abstract references."),
+           clEnumValN(LVAttributeKind::Language, "language",
+                      "Source language name."),
            clEnumValN(LVAttributeKind::Level, "level",
                       "Lexical scope level (File=0, Compile Unit=1)."),
            clEnumValN(LVAttributeKind::Linkage, "linkage", "Linkage name."),
diff --git a/external/llvm-project/llvm/tools/llvm-dis/llvm-dis.cpp b/external/llvm-project/llvm/tools/llvm-dis/llvm-dis.cpp
index 8c1aaf6fd604..422eb855ba2c 100644
--- a/external/llvm-project/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/external/llvm-project/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -96,8 +96,6 @@ static cl::opt<bool> PrintThinLTOIndexOnly(
     cl::desc("Only read thinlto index and print the index as LLVM assembly."),
     cl::init(false), cl::Hidden, cl::cat(DisCategory));
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace {
 
 static void printDebugLoc(const DebugLoc &DL, formatted_raw_ostream &OS) {
@@ -270,9 +268,7 @@ int main(int argc, char **argv) {
       // All that llvm-dis does is write the assembly to a file.
       if (!DontPrint) {
         if (M) {
-          M->setIsNewDbgInfoFormat(UseNewDbgInfoFormat);
-          if (UseNewDbgInfoFormat)
-            M->removeDebugIntrinsicDeclarations();
+          M->removeDebugIntrinsicDeclarations();
           M->print(Out->os(), Annotator.get(), PreserveAssemblyUseListOrder);
         }
         if (Index)
diff --git a/external/llvm-project/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp b/external/llvm-project/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp
index a1eb5a46f21f..d6f4c5220f71 100644
--- a/external/llvm-project/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp
+++ b/external/llvm-project/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp
@@ -12,12 +12,6 @@
 #if defined(__aarch64__) && defined(__linux__)
 #include <linux/prctl.h> // For PR_PAC_* constants
 #include <sys/prctl.h>
-#ifndef PR_PAC_SET_ENABLED_KEYS
-#define PR_PAC_SET_ENABLED_KEYS 60
-#endif
-#ifndef PR_PAC_GET_ENABLED_KEYS
-#define PR_PAC_GET_ENABLED_KEYS 61
-#endif
 #ifndef PR_PAC_APIAKEY
 #define PR_PAC_APIAKEY (1UL << 0)
 #endif
@@ -38,47 +32,6 @@
 namespace llvm {
 namespace exegesis {
 
-bool isPointerAuth(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-
-  // FIXME: Pointer Authentication instructions.
-  // We would like to measure these instructions, but they can behave
-  // differently on different platforms, and maybe the snippets need to look
-  // different for these instructions,
-  // Platform-specific handling:  On Linux, we disable authentication, may
-  // interfere with measurements. On non-Linux platforms, disable opcodes for
-  // now.
-  case AArch64::AUTDA:
-  case AArch64::AUTDB:
-  case AArch64::AUTDZA:
-  case AArch64::AUTDZB:
-  case AArch64::AUTIA:
-  case AArch64::AUTIA1716:
-  case AArch64::AUTIASP:
-  case AArch64::AUTIAZ:
-  case AArch64::AUTIB:
-  case AArch64::AUTIB1716:
-  case AArch64::AUTIBSP:
-  case AArch64::AUTIBZ:
-  case AArch64::AUTIZA:
-  case AArch64::AUTIZB:
-    return true;
-  }
-}
-
-bool isLoadTagMultiple(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-
-  // Load tag multiple instruction
-  case AArch64::LDGM:
-    return true;
-  }
-}
-
 static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
   switch (RegBitWidth) {
   case 32:
@@ -198,35 +151,6 @@ class ExegesisAArch64Target : public ExegesisTarget {
     // Function return is a pseudo-instruction that needs to be expanded
     PM.add(createAArch64ExpandPseudoPass());
   }
-
-  const char *getIgnoredOpcodeReasonOrNull(const LLVMState &State,
-                                           unsigned Opcode) const override {
-    if (const char *Reason =
-            ExegesisTarget::getIgnoredOpcodeReasonOrNull(State, Opcode))
-      return Reason;
-
-    if (isPointerAuth(Opcode)) {
-#if defined(__aarch64__) && defined(__linux__)
-      // Disable all PAC keys. Note that while we expect the measurements to
-      // be the same with PAC keys disabled, they could potentially be lower
-      // since authentication checks are bypassed.
-      if (prctl(PR_PAC_SET_ENABLED_KEYS,
-                PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY |
-                    PR_PAC_APDBKEY, // all keys
-                0,                  // disable all
-                0, 0) < 0) {
-        return "Failed to disable PAC keys";
-      }
-#else
-      return "Unsupported opcode: isPointerAuth";
-#endif
-    }
-
-    if (isLoadTagMultiple(Opcode))
-      return "Unsupported opcode: load tag multiple";
-
-    return nullptr;
-  }
 };
 
 } // namespace
diff --git a/external/llvm-project/llvm/tools/llvm-link/llvm-link.cpp b/external/llvm-project/llvm/tools/llvm-link/llvm-link.cpp
index ac5b5f52741a..22ea54e68358 100644
--- a/external/llvm-project/llvm/tools/llvm-link/llvm-link.cpp
+++ b/external/llvm-project/llvm/tools/llvm-link/llvm-link.cpp
@@ -129,8 +129,6 @@ static cl::opt<bool> IgnoreNonBitcode(
     cl::desc("Do not report an error for non-bitcode files in archives"),
     cl::Hidden);
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 static ExitOnError ExitOnErr;
 
 // Read the specified bitcode file in and return it. This routine searches the
@@ -525,16 +523,10 @@ int main(int argc, char **argv) {
 
   if (Verbose)
     errs() << "Writing bitcode...\n";
-  auto SetFormat = [&](bool NewFormat) {
-    Composite->setIsNewDbgInfoFormat(NewFormat);
-    if (NewFormat)
-      Composite->removeDebugIntrinsicDeclarations();
-  };
+  Composite->removeDebugIntrinsicDeclarations();
   if (OutputAssembly) {
-    SetFormat(UseNewDbgInfoFormat);
     Composite->print(Out.os(), nullptr, PreserveAssemblyUseListOrder);
   } else if (Force || !CheckBitcodeOutputToConsole(Out.os())) {
-    SetFormat(UseNewDbgInfoFormat);
     WriteBitcodeToFile(*Composite, Out.os(), PreserveBitcodeUseListOrder);
   }
 
diff --git a/external/llvm-project/llvm/tools/llvm-lto2/llvm-lto2.cpp b/external/llvm-project/llvm/tools/llvm-lto2/llvm-lto2.cpp
index 2bbb65be3b31..fbde66666a59 100644
--- a/external/llvm-project/llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ b/external/llvm-project/llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -247,7 +247,7 @@ template <typename T> static T check(ErrorOr<T> E, std::string Msg) {
 }
 
 static int usage() {
-  errs() << "Available subcommands: dump-symtab run\n";
+  errs() << "Available subcommands: dump-symtab run print-guid\n";
   return 1;
 }
 
@@ -610,5 +610,11 @@ int main(int argc, char **argv) {
     return dumpSymtab(argc - 1, argv + 1);
   if (Subcommand == "run")
     return run(argc - 1, argv + 1);
+  if (Subcommand == "print-guid" && argc > 2) {
+    // Note the name of the function we're calling: this won't return the right
+    // answer for internal linkage symbols.
+    outs() << GlobalValue::getGUIDAssumingExternalLinkage(argv[2]) << '\n';
+    return 0;
+  }
   return usage();
 }
diff --git a/external/llvm-project/llvm/tools/llvm-objdump/SourcePrinter.cpp b/external/llvm-project/llvm/tools/llvm-objdump/SourcePrinter.cpp
index 33d494bfd1ac..bf3363795a4d 100644
--- a/external/llvm-project/llvm/tools/llvm-objdump/SourcePrinter.cpp
+++ b/external/llvm-project/llvm/tools/llvm-objdump/SourcePrinter.cpp
@@ -45,7 +45,7 @@ void LiveVariable::print(raw_ostream &OS, const MCRegisterInfo &MRI) const {
     return {};
   };
 
-  Expression.printCompact(OS, GetRegName);
+  DWARFExpressionPrinter::printCompact(&Expression, OS, GetRegName);
 }
 
 void LiveVariablePrinter::addVariable(DWARFDie FuncDie, DWARFDie VarDie) {
diff --git a/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.cpp b/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.cpp
index a26241967b5a..fac1d8932161 100644
--- a/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.cpp
+++ b/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.cpp
@@ -155,6 +155,13 @@ void MappingTraits<PdbDbiStream>::mapping(IO &IO, PdbDbiStream &Obj) {
   IO.mapOptional("PdbDllRbld", Obj.PdbDllRbld, uint16_t(0U));
   IO.mapOptional("Flags", Obj.Flags, uint16_t(1U));
   IO.mapOptional("MachineType", Obj.MachineType, PDB_Machine::x86);
+  // This is a workaround for IO not having document context with the
+  // machine type. The machine type is needed to properly parse Register enums
+  // in the PDB.
+  if (!IO.getContext()) {
+    Obj.FakeHeader.Machine = static_cast<uint16_t>(Obj.MachineType);
+    IO.setContext(&Obj.FakeHeader);
+  }
   IO.mapOptional("Modules", Obj.ModInfos);
 }
 
diff --git a/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.h b/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.h
index 21658e9d0e75..d5111a9e8d3a 100644
--- a/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.h
+++ b/external/llvm-project/llvm/tools/llvm-pdbutil/PdbYaml.h
@@ -11,6 +11,7 @@
 
 #include "OutputStyle.h"
 
+#include "llvm/BinaryFormat/COFF.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecord.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/MSF/MSFCommon.h"
@@ -80,6 +81,7 @@ struct PdbDbiStream {
   PDB_Machine MachineType = PDB_Machine::x86;
 
   std::vector<PdbDbiModuleInfo> ModInfos;
+  COFF::header FakeHeader;
 };
 
 struct PdbTpiStream {
diff --git a/external/llvm-project/llvm/tools/llvm-pdbutil/YAMLOutputStyle.cpp b/external/llvm-project/llvm/tools/llvm-pdbutil/YAMLOutputStyle.cpp
index 80b76657facc..ecb4c2175e49 100644
--- a/external/llvm-project/llvm/tools/llvm-pdbutil/YAMLOutputStyle.cpp
+++ b/external/llvm-project/llvm/tools/llvm-pdbutil/YAMLOutputStyle.cpp
@@ -11,6 +11,7 @@
 #include "PdbYaml.h"
 #include "llvm-pdbutil.h"
 
+#include "llvm/BinaryFormat/COFF.h"
 #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h"
 #include "llvm/DebugInfo/CodeView/DebugSubsection.h"
 #include "llvm/DebugInfo/CodeView/DebugUnknownSubsection.h"
@@ -73,7 +74,15 @@ Error YAMLOutputStyle::dump() {
   if (auto EC = dumpPublics())
     return EC;
 
+  // Fake Coff header for dumping register enumerations.
+  COFF::header Header;
+  auto MachineType =
+      Obj.DbiStream ? Obj.DbiStream->MachineType : PDB_Machine::Unknown;
+  Header.Machine = static_cast<uint16_t>(MachineType);
+  Out.setContext(&Header);
   flush();
+  Out.setContext(nullptr);
+
   return Error::success();
 }
 
diff --git a/external/llvm-project/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp b/external/llvm-project/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
index 4cb64bdbe8ef..e50d19a994b6 100644
--- a/external/llvm-project/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
+++ b/external/llvm-project/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
@@ -1375,7 +1375,6 @@ static void mergePdbs() {
 }
 
 static void explain() {
-  std::unique_ptr<IPDBSession> Session;
   InputFile IF =
       ExitOnErr(InputFile::open(opts::explain::InputFilename.front(), true));
 
diff --git a/external/llvm-project/llvm/tools/llvm-rc/ResourceFileWriter.cpp b/external/llvm-project/llvm/tools/llvm-rc/ResourceFileWriter.cpp
index dc21f63a682a..2c35c5ad365d 100644
--- a/external/llvm-project/llvm/tools/llvm-rc/ResourceFileWriter.cpp
+++ b/external/llvm-project/llvm/tools/llvm-rc/ResourceFileWriter.cpp
@@ -631,8 +631,8 @@ Error ResourceFileWriter::writeSingleAccelerator(
   if (IsASCII && IsVirtKey)
     return createAccError("Accelerator can't be both ASCII and VIRTKEY");
 
-  if (!IsVirtKey && (Obj.Flags & (Opt::ALT | Opt::SHIFT | Opt::CONTROL)))
-    return createAccError("Can only apply ALT, SHIFT or CONTROL to VIRTKEY"
+  if (!IsVirtKey && (Obj.Flags & (Opt::SHIFT | Opt::CONTROL)))
+    return createAccError("Can only apply SHIFT or CONTROL to VIRTKEY"
                           " accelerators");
 
   if (Obj.Event.isInt()) {
diff --git a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.cpp b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.cpp
index 69798152c1f2..e4efc83c933b 100644
--- a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.cpp
+++ b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.cpp
@@ -132,12 +132,13 @@ void RCParser::consume() {
 //
 // The following grammar is used to parse the expressions Exp1:
 //   Exp1 ::= Exp2 || Exp1 + Exp2 || Exp1 - Exp2 || Exp1 | Exp2 || Exp1 & Exp2
-//   Exp2 ::= -Exp2 || ~Exp2 || not Expr2 || Int || (Exp1).
-// (More conveniently, Exp1 is a non-empty sequence of Exp2 expressions,
-// separated by binary operators.)
+//   Exp2 ::= Exp3 || Exp3 * Exp3 || Exp3 / Exp3
+//   Exp3 ::= -Exp3 || ~Exp3 || not Expr3 || Int || (Exp1)
+// (More conveniently, Exp1 and Exp2 are non-empty sequences of Exp3
+// expressions, separated by binary operators.)
 //
-// Expressions of type Exp1 are read by parseIntExpr1(Inner) method, while Exp2
-// is read by parseIntExpr2().
+// Expressions of type Exp1 are read by parseIntExpr1(Inner) method, Exp2
+// is read by parseIntExpr2() and Exp3 is read by parseIntExpr3().
 //
 // The original Microsoft tool handles multiple unary operators incorrectly.
 // For example, in 16-bit little-endian integers:
@@ -158,7 +159,7 @@ Expected<IntWithNotMask> RCParser::parseIntExpr1() {
   ASSIGN_OR_RETURN(FirstResult, parseIntExpr2());
   IntWithNotMask Result = *FirstResult;
 
-  while (!isEof() && look().isBinaryOp()) {
+  while (!isEof() && look().isLowPrecedenceBinaryOp()) {
     auto OpToken = read();
     ASSIGN_OR_RETURN(NextResult, parseIntExpr2());
 
@@ -180,7 +181,7 @@ Expected<IntWithNotMask> RCParser::parseIntExpr1() {
       break;
 
     default:
-      llvm_unreachable("Already processed all binary ops.");
+      llvm_unreachable("Already processed all low precedence binary ops.");
     }
   }
 
@@ -188,7 +189,33 @@ Expected<IntWithNotMask> RCParser::parseIntExpr1() {
 }
 
 Expected<IntWithNotMask> RCParser::parseIntExpr2() {
-  // Exp2 ::= -Exp2 || ~Exp2 || not Expr2 || Int || (Exp1).
+  // Exp2 ::= Exp3 || Exp3 * Exp3 || Exp3 / Exp3.
+  ASSIGN_OR_RETURN(FirstResult, parseIntExpr3());
+  IntWithNotMask Result = *FirstResult;
+
+  while (!isEof() && look().isHighPrecedenceBinaryOp()) {
+    auto OpToken = read();
+    ASSIGN_OR_RETURN(NextResult, parseIntExpr3());
+
+    switch (OpToken.kind()) {
+    case Kind::Asterisk:
+      Result *= *NextResult;
+      break;
+
+    case Kind::Slash:
+      Result /= *NextResult;
+      break;
+
+    default:
+      llvm_unreachable("Already processed all high precedence binary ops.");
+    }
+  }
+
+  return Result;
+}
+
+Expected<IntWithNotMask> RCParser::parseIntExpr3() {
+  // Exp3 ::= -Exp3 || ~Exp3 || not Expr3 || Int || (Exp1).
   static const char ErrorMsg[] = "'-', '~', integer or '('";
 
   if (isEof())
@@ -197,13 +224,13 @@ Expected<IntWithNotMask> RCParser::parseIntExpr2() {
   switch (look().kind()) {
   case Kind::Minus: {
     consume();
-    ASSIGN_OR_RETURN(Result, parseIntExpr2());
+    ASSIGN_OR_RETURN(Result, parseIntExpr3());
     return -(*Result);
   }
 
   case Kind::Tilde: {
     consume();
-    ASSIGN_OR_RETURN(Result, parseIntExpr2());
+    ASSIGN_OR_RETURN(Result, parseIntExpr3());
     return ~(*Result);
   }
 
@@ -220,7 +247,7 @@ Expected<IntWithNotMask> RCParser::parseIntExpr2() {
   case Kind::Identifier: {
     if (!read().value().equals_insensitive("not"))
       return getExpectedError(ErrorMsg, true);
-    ASSIGN_OR_RETURN(Result, parseIntExpr2());
+    ASSIGN_OR_RETURN(Result, parseIntExpr3());
     return IntWithNotMask(0, (*Result).getValue());
   }
 
diff --git a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.h b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.h
index aa7f847187c4..1e7618c84142 100644
--- a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.h
+++ b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptParser.h
@@ -88,6 +88,7 @@ class RCParser {
   // Helper integer expression parsing methods.
   Expected<IntWithNotMask> parseIntExpr1();
   Expected<IntWithNotMask> parseIntExpr2();
+  Expected<IntWithNotMask> parseIntExpr3();
 
   // Advance the state by one, discarding the current token.
   // If the discarded token had an incorrect type, fail.
diff --git a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptStmt.h b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptStmt.h
index 8f099202c0b4..a81e384fda36 100644
--- a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptStmt.h
+++ b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptStmt.h
@@ -49,6 +49,16 @@ class RCInt {
     return *this;
   }
 
+  RCInt &operator*=(const RCInt &Rhs) {
+    std::tie(Val, Long) = std::make_pair(Val * Rhs.Val, Long | Rhs.Long);
+    return *this;
+  }
+
+  RCInt &operator/=(const RCInt &Rhs) {
+    std::tie(Val, Long) = std::make_pair(Val / Rhs.Val, Long | Rhs.Long);
+    return *this;
+  }
+
   RCInt &operator|=(const RCInt &Rhs) {
     std::tie(Val, Long) = std::make_pair(Val | Rhs.Val, Long | Rhs.Long);
     return *this;
@@ -98,6 +108,20 @@ class IntWithNotMask {
     return *this;
   }
 
+  IntWithNotMask &operator*=(const IntWithNotMask &Rhs) {
+    Value &= ~Rhs.NotMask;
+    Value *= Rhs.Value;
+    NotMask |= Rhs.NotMask;
+    return *this;
+  }
+
+  IntWithNotMask &operator/=(const IntWithNotMask &Rhs) {
+    Value &= ~Rhs.NotMask;
+    Value /= Rhs.Value;
+    NotMask |= Rhs.NotMask;
+    return *this;
+  }
+
   IntWithNotMask &operator|=(const IntWithNotMask &Rhs) {
     Value &= ~Rhs.NotMask;
     Value |= Rhs.Value;
diff --git a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.cpp b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.cpp
index aad1060c4a38..0070037e63e6 100644
--- a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.cpp
+++ b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.cpp
@@ -64,7 +64,7 @@ StringRef RCToken::value() const { return TokenValue; }
 
 Kind RCToken::kind() const { return TokenKind; }
 
-bool RCToken::isBinaryOp() const {
+bool RCToken::isLowPrecedenceBinaryOp() const {
   switch (TokenKind) {
   case Kind::Plus:
   case Kind::Minus:
@@ -76,6 +76,16 @@ bool RCToken::isBinaryOp() const {
   }
 }
 
+bool RCToken::isHighPrecedenceBinaryOp() const {
+  switch (TokenKind) {
+  case Kind::Asterisk:
+  case Kind::Slash:
+    return true;
+  default:
+    return false;
+  }
+}
+
 static Error getStringError(const Twine &message) {
   return make_error<StringError>("Error parsing file: " + message,
                                  inconvertibleErrorCode());
diff --git a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.h b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.h
index 29f7502f89ef..3dcdfafd2d57 100644
--- a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.h
+++ b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptToken.h
@@ -56,8 +56,11 @@ class RCToken {
   StringRef value() const;
   Kind kind() const;
 
-  // Check if a token describes a binary operator.
-  bool isBinaryOp() const;
+  // Check if a token describes a low precedence binary operator.
+  bool isLowPrecedenceBinaryOp() const;
+
+  // Check if a token describes a high precedence binary operator.
+  bool isHighPrecedenceBinaryOp() const;
 
 private:
   Kind TokenKind;
diff --git a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptTokenList.def b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptTokenList.def
index a61a96461f0f..6ee13b2815d3 100644
--- a/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptTokenList.def
+++ b/external/llvm-project/llvm/tools/llvm-rc/ResourceScriptTokenList.def
@@ -29,6 +29,8 @@ SHORT_TOKEN(BlockEnd, '}')     // End of the block; can also be END.
 SHORT_TOKEN(Comma, ',')        // Comma - resource arguments separator.
 SHORT_TOKEN(Plus, '+')         // Addition operator.
 SHORT_TOKEN(Minus, '-')        // Subtraction operator.
+SHORT_TOKEN(Asterisk, '*')     // Multiplication operator.
+SHORT_TOKEN(Slash, '/')        // Division operator.
 SHORT_TOKEN(Pipe, '|')         // Bitwise-OR operator.
 SHORT_TOKEN(Amp, '&')          // Bitwise-AND operator.
 SHORT_TOKEN(Tilde, '~')        // Bitwise-NOT operator.
diff --git a/external/llvm-project/llvm/tools/llvm-readobj/COFFDumper.cpp b/external/llvm-project/llvm/tools/llvm-readobj/COFFDumper.cpp
index 5d1d2a34d644..dce8e60bda1e 100644
--- a/external/llvm-project/llvm/tools/llvm-readobj/COFFDumper.cpp
+++ b/external/llvm-project/llvm/tools/llvm-readobj/COFFDumper.cpp
@@ -108,6 +108,7 @@ class COFFDumper : public ObjDumper {
   void printStackMap() const override;
   void printAddrsig() override;
   void printCGProfile() override;
+  void printStringTable() override;
 
 private:
   StringRef getSymbolName(uint32_t Index);
@@ -2219,6 +2220,17 @@ void COFFDumper::printCGProfile() {
   }
 }
 
+void COFFDumper::printStringTable() {
+  DictScope DS(W, "StringTable");
+  StringRef StrTable = Obj->getStringTable();
+  uint32_t StrTabSize = StrTable.size();
+  W.printNumber("Length", StrTabSize);
+  // Print strings from the fifth byte, since the first four bytes contain the
+  // length (in bytes) of the string table (including the length field).
+  if (StrTabSize > 4)
+    printAsStringList(StrTable, 4);
+}
+
 StringRef COFFDumper::getSymbolName(uint32_t Index) {
   Expected<COFFSymbolRef> Sym = Obj->getSymbol(Index);
   if (!Sym)
diff --git a/external/llvm-project/llvm/tools/llvm-readobj/ObjDumper.h b/external/llvm-project/llvm/tools/llvm-readobj/ObjDumper.h
index cd744e3bbfb7..a76afbe9c88c 100644
--- a/external/llvm-project/llvm/tools/llvm-readobj/ObjDumper.h
+++ b/external/llvm-project/llvm/tools/llvm-readobj/ObjDumper.h
@@ -157,8 +157,10 @@ class ObjDumper {
                      llvm::codeview::GlobalTypeTableBuilder &GlobalCVTypes,
                      bool GHash) {}
 
-  // Only implemented for XCOFF.
+  // Only implemented for XCOFF/COFF.
   virtual void printStringTable() {}
+
+  // Only implemented for XCOFF.
   virtual void printAuxiliaryHeader() {}
   virtual void printExceptionSection() {}
   virtual void printLoaderSection(bool PrintHeader, bool PrintSymbols,
diff --git a/external/llvm-project/llvm/tools/llvm-readobj/Opts.td b/external/llvm-project/llvm/tools/llvm-readobj/Opts.td
index 7d574d875d22..f95461aaca1a 100644
--- a/external/llvm-project/llvm/tools/llvm-readobj/Opts.td
+++ b/external/llvm-project/llvm/tools/llvm-readobj/Opts.td
@@ -44,7 +44,7 @@ defm sort_symbols : Eq<"sort-symbols", "Specify the keys to sort the symbols bef
 def stack_sizes : FF<"stack-sizes", "Display contents of all stack sizes sections. This option has no effect for GNU style output">;
 def stackmap : FF<"stackmap", "Display contents of stackmap section">;
 defm string_dump : Eq<"string-dump", "Display the specified section(s) as a list of strings">, MetaVarName<"<name or index>">;
-def string_table : FF<"string-table", "Display the string table (only for XCOFF now)">;
+def string_table : FF<"string-table", "Display the string table (only for XCOFF/COFF now)">;
 def symbols : FF<"symbols", "Display the symbol table. Also display the dynamic symbol table when using GNU output style for ELF">;
 def unwind : FF<"unwind", "Display unwind information">;
 
diff --git a/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceGlobalObjects.cpp b/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceGlobalObjects.cpp
index 64bf711f23d5..5d958dd2d82a 100644
--- a/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceGlobalObjects.cpp
+++ b/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceGlobalObjects.cpp
@@ -13,18 +13,26 @@ using namespace llvm;
 
 static bool shouldReduceSection(GlobalObject &GO) { return GO.hasSection(); }
 
-static bool shouldReduceAlign(GlobalObject &GO) {
-  return GO.getAlign().has_value();
+static bool shouldReduceAlign(GlobalVariable *GV) {
+  return GV->getAlign().has_value();
 }
 
+static bool shouldReduceAlign(Function *F) { return F->getAlign().has_value(); }
+
 static bool shouldReduceComdat(GlobalObject &GO) { return GO.hasComdat(); }
 
 void llvm::reduceGlobalObjectsDeltaPass(Oracle &O, ReducerWorkItem &Program) {
   for (auto &GO : Program.getModule().global_objects()) {
     if (shouldReduceSection(GO) && !O.shouldKeep())
       GO.setSection("");
-    if (shouldReduceAlign(GO) && !O.shouldKeep())
-      GO.setAlignment(MaybeAlign());
+    if (auto *GV = dyn_cast<GlobalVariable>(&GO)) {
+      if (shouldReduceAlign(GV) && !O.shouldKeep())
+        GV->setAlignment(MaybeAlign());
+    }
+    if (auto *F = dyn_cast<Function>(&GO)) {
+      if (shouldReduceAlign(F) && !O.shouldKeep())
+        F->setAlignment(MaybeAlign());
+    }
     if (shouldReduceComdat(GO) && !O.shouldKeep())
       GO.setComdat(nullptr);
   }
diff --git a/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceOperandsSkip.cpp b/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceOperandsSkip.cpp
index c99122d8c199..498588fb8d09 100644
--- a/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceOperandsSkip.cpp
+++ b/external/llvm-project/llvm/tools/llvm-reduce/deltas/ReduceOperandsSkip.cpp
@@ -150,8 +150,7 @@ opportunities(Function &F,
       // Regardless whether referenced, add the function arguments as
       // replacement possibility with the goal of reducing the number of (used)
       // function arguments, possibly created by the operands-to-args.
-      for (Argument &Arg : F.args())
-        ReferencedVals.insert(&Arg);
+      ReferencedVals.insert_range(llvm::make_pointer_range(F.args()));
 
       // After all candidates have been added, it doesn't need to be a set
       // anymore.
diff --git a/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkConvert.cpp b/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkConvert.cpp
index 35d8dcd99b4a..207c5e0a8048 100644
--- a/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkConvert.cpp
+++ b/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkConvert.cpp
@@ -133,6 +133,7 @@ static Error tryBitstream2YAML() {
   if (!E.isA<EndOfFileError>())
     return E;
   consumeError(std::move(E));
+  OF->keep();
   return Error::success();
 }
 } // namespace bitstream2yaml
diff --git a/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkSizeDiff.cpp b/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkSizeDiff.cpp
index 35ea43232200..d3cb134279a2 100644
--- a/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkSizeDiff.cpp
+++ b/external/llvm-project/llvm/tools/llvm-remarkutil/RemarkSizeDiff.cpp
@@ -498,4 +498,4 @@ static Error trySizeSiff() {
 }
 
 static CommandRegistration RemarkSizeSiffRegister(&RemarkSizeDiffUtil,
-                                                  trySizeSiff);
\ No newline at end of file
+                                                  trySizeSiff);
diff --git a/external/llvm-project/llvm/tools/llvm-shlib/CMakeLists.txt b/external/llvm-project/llvm/tools/llvm-shlib/CMakeLists.txt
index 089255f36104..9a2015f61f2b 100644
--- a/external/llvm-project/llvm/tools/llvm-shlib/CMakeLists.txt
+++ b/external/llvm-project/llvm/tools/llvm-shlib/CMakeLists.txt
@@ -66,7 +66,7 @@ if(LLVM_BUILD_LLVM_DYLIB)
     else()
       # GNU ld doesn't resolve symbols in the version script.
       set(LIB_NAMES -Wl,--whole-archive ${LIB_NAMES} -Wl,--no-whole-archive)
-      if (NOT LLVM_LINKER_IS_SOLARISLD AND NOT MINGW)
+      if (NOT LLVM_LINKER_IS_SOLARISLD AND NOT MINGW AND NOT CYGWIN)
         # Solaris ld does not accept global: *; so there is no way to version *all* global symbols
         set(LIB_NAMES -Wl,--version-script,${LLVM_LIBRARY_DIR}/tools/llvm-shlib/simple_version_script.map ${LIB_NAMES})
       endif()
diff --git a/external/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/external/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
index a682a0762c39..4784dafeb294 100644
--- a/external/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
+++ b/external/llvm-project/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
@@ -238,7 +238,7 @@ static Error parseCommand(StringRef BinaryName, bool IsAddr2Line,
   bool StartsWithDigit = std::isdigit(AddrSpec.front());
 
   // GNU addr2line assumes the address is hexadecimal and allows a redundant
-  // "0x" or "0X" prefix or with an optional `+` sign; do the same for
+  // "0x", "0X" prefix or an optional `+` sign; do the same for
   // compatibility.
   if (IsAddr2Line) {
     AddrSpec.consume_front_insensitive("0x") ||
diff --git a/external/llvm-project/llvm/tools/sancov/sancov.cpp b/external/llvm-project/llvm/tools/sancov/sancov.cpp
index 2cc84b47de6b..aebb5effd0be 100644
--- a/external/llvm-project/llvm/tools/sancov/sancov.cpp
+++ b/external/llvm-project/llvm/tools/sancov/sancov.cpp
@@ -889,8 +889,7 @@ symbolize(const RawCoverage &Data, const std::string ObjectFile) {
   }
 
   std::set<uint64_t> AllAddrs = findCoveragePointAddrs(ObjectFile);
-  if (!std::includes(AllAddrs.begin(), AllAddrs.end(), Data.Addrs->begin(),
-                     Data.Addrs->end())) {
+  if (!llvm::includes(AllAddrs, *Data.Addrs)) {
     fail("Coverage points in binary and .sancov file do not match.");
   }
   Coverage->Points = getCoveragePoints(ObjectFile, AllAddrs, *Data.Addrs);
diff --git a/external/llvm-project/llvm/unittests/ADT/DAGDeltaAlgorithmTest.cpp b/external/llvm-project/llvm/unittests/ADT/DAGDeltaAlgorithmTest.cpp
index 66a67d96d153..f54394789939 100644
--- a/external/llvm-project/llvm/unittests/ADT/DAGDeltaAlgorithmTest.cpp
+++ b/external/llvm-project/llvm/unittests/ADT/DAGDeltaAlgorithmTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/DAGDeltaAlgorithm.h"
+#include "llvm/ADT/STLExtras.h"
 #include "gtest/gtest.h"
 #include <algorithm>
 #include <cstdarg>
@@ -23,8 +24,7 @@ class FixedDAGDeltaAlgorithm : public DAGDeltaAlgorithm {
 protected:
   bool ExecuteOneTest(const changeset_ty &Changes) override {
     ++NumTests;
-    return std::includes(Changes.begin(), Changes.end(),
-                         FailingSet.begin(), FailingSet.end());
+    return llvm::includes(Changes, FailingSet);
   }
 
 public:
diff --git a/external/llvm-project/llvm/unittests/ADT/DeltaAlgorithmTest.cpp b/external/llvm-project/llvm/unittests/ADT/DeltaAlgorithmTest.cpp
index 5e284129180a..24e18f42eb33 100644
--- a/external/llvm-project/llvm/unittests/ADT/DeltaAlgorithmTest.cpp
+++ b/external/llvm-project/llvm/unittests/ADT/DeltaAlgorithmTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/DeltaAlgorithm.h"
+#include "llvm/ADT/STLExtras.h"
 #include "gtest/gtest.h"
 #include <algorithm>
 #include <cstdarg>
@@ -38,8 +39,7 @@ class FixedDeltaAlgorithm final : public DeltaAlgorithm {
 protected:
   bool ExecuteOneTest(const changeset_ty &Changes) override {
     ++NumTests;
-    return std::includes(Changes.begin(), Changes.end(),
-                         FailingSet.begin(), FailingSet.end());
+    return llvm::includes(Changes, FailingSet);
   }
 
 public:
diff --git a/external/llvm-project/llvm/unittests/ADT/STLExtrasTest.cpp b/external/llvm-project/llvm/unittests/ADT/STLExtrasTest.cpp
index 0101be47a686..286cfa745fd1 100644
--- a/external/llvm-project/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/external/llvm-project/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -1567,6 +1567,30 @@ TEST(STLExtrasTest, Mismatch) {
   }
 }
 
+TEST(STLExtrasTest, Includes) {
+  {
+    std::vector<int> V1 = {1, 2};
+    std::vector<int> V2;
+    EXPECT_TRUE(includes(V1, V2));
+    EXPECT_FALSE(includes(V2, V1));
+    V2.push_back(1);
+    EXPECT_TRUE(includes(V1, V2));
+    V2.push_back(3);
+    EXPECT_FALSE(includes(V1, V2));
+  }
+
+  {
+    std::vector<int> V1 = {3, 2, 1};
+    std::vector<int> V2;
+    EXPECT_TRUE(includes(V1, V2, std::greater<>()));
+    EXPECT_FALSE(includes(V2, V1, std::greater<>()));
+    V2.push_back(3);
+    EXPECT_TRUE(includes(V1, V2, std::greater<>()));
+    V2.push_back(0);
+    EXPECT_FALSE(includes(V1, V2, std::greater<>()));
+  }
+}
+
 struct Foo;
 struct Bar {};
 
diff --git a/external/llvm-project/llvm/unittests/Analysis/IR2VecTest.cpp b/external/llvm-project/llvm/unittests/Analysis/IR2VecTest.cpp
index 0158038b59b6..90d07d080443 100644
--- a/external/llvm-project/llvm/unittests/Analysis/IR2VecTest.cpp
+++ b/external/llvm-project/llvm/unittests/Analysis/IR2VecTest.cpp
@@ -14,6 +14,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/JSON.h"
 
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -28,94 +29,213 @@ namespace {
 
 class TestableEmbedder : public Embedder {
 public:
-  TestableEmbedder(const Function &F, const Vocab &V, unsigned Dim)
-      : Embedder(F, V, Dim) {}
+  TestableEmbedder(const Function &F, const Vocab &V) : Embedder(F, V) {}
   void computeEmbeddings() const override {}
   void computeEmbeddings(const BasicBlock &BB) const override {}
   using Embedder::lookupVocab;
-  static void addVectors(Embedding &Dst, const Embedding &Src) {
-    Embedder::addVectors(Dst, Src);
+};
+
+TEST(EmbeddingTest, ConstructorsAndAccessors) {
+  // Default constructor
+  {
+    Embedding E;
+    EXPECT_TRUE(E.empty());
+    EXPECT_EQ(E.size(), 0u);
   }
-  static void addScaledVector(Embedding &Dst, const Embedding &Src,
-                              float Factor) {
-    Embedder::addScaledVector(Dst, Src, Factor);
+
+  // Constructor with const std::vector<double>&
+  {
+    std::vector<double> Data = {1.0, 2.0, 3.0};
+    Embedding E(Data);
+    EXPECT_FALSE(E.empty());
+    ASSERT_THAT(E, SizeIs(3u));
+    EXPECT_THAT(E.getData(), ElementsAre(1.0, 2.0, 3.0));
+    EXPECT_EQ(E[0], 1.0);
+    EXPECT_EQ(E[1], 2.0);
+    EXPECT_EQ(E[2], 3.0);
   }
-};
 
-TEST(IR2VecTest, CreateSymbolicEmbedder) {
-  Vocab V = {{"foo", {1.0, 2.0}}};
+  // Constructor with std::vector<double>&&
+  {
+    Embedding E(std::vector<double>({4.0, 5.0}));
+    ASSERT_THAT(E, SizeIs(2u));
+    EXPECT_THAT(E.getData(), ElementsAre(4.0, 5.0));
+  }
 
-  LLVMContext Ctx;
-  Module M("M", Ctx);
-  FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), false);
-  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
+  // Constructor with std::initializer_list<double>
+  {
+    Embedding E({6.0, 7.0, 8.0, 9.0});
+    ASSERT_THAT(E, SizeIs(4u));
+    EXPECT_THAT(E.getData(), ElementsAre(6.0, 7.0, 8.0, 9.0));
+    EXPECT_EQ(E[0], 6.0);
+    E[0] = 6.5;
+    EXPECT_EQ(E[0], 6.5);
+  }
 
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V, 2);
-  EXPECT_TRUE(static_cast<bool>(Result));
+  // Constructor with size_t
+  {
+    Embedding E(5);
+    ASSERT_THAT(E, SizeIs(5u));
+    EXPECT_THAT(E.getData(), ElementsAre(0.0, 0.0, 0.0, 0.0, 0.0));
+  }
 
-  auto *Emb = Result->get();
-  EXPECT_NE(Emb, nullptr);
-}
+  // Constructor with size_t and double
+  {
+    Embedding E(5, 1.5);
+    ASSERT_THAT(E, SizeIs(5u));
+    EXPECT_THAT(E.getData(), ElementsAre(1.5, 1.5, 1.5, 1.5, 1.5));
+  }
 
-TEST(IR2VecTest, CreateInvalidMode) {
-  Vocab V = {{"foo", {1.0, 2.0}}};
+  // Test iterators
+  {
+    Embedding E({6.5, 7.0, 8.0, 9.0});
+    std::vector<double> VecE;
+    for (double Val : E) {
+      VecE.push_back(Val);
+    }
+    EXPECT_THAT(VecE, ElementsAre(6.5, 7.0, 8.0, 9.0));
+
+    const Embedding CE = E;
+    std::vector<double> VecCE;
+    for (const double &Val : CE) {
+      VecCE.push_back(Val);
+    }
+    EXPECT_THAT(VecCE, ElementsAre(6.5, 7.0, 8.0, 9.0));
+
+    EXPECT_EQ(*E.begin(), 6.5);
+    EXPECT_EQ(*(E.end() - 1), 9.0);
+    EXPECT_EQ(*CE.cbegin(), 6.5);
+    EXPECT_EQ(*(CE.cend() - 1), 9.0);
+  }
+}
 
-  LLVMContext Ctx;
-  Module M("M", Ctx);
-  FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), false);
-  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
+TEST(EmbeddingTest, AddVectors) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = {0.5, 1.5, -1.0};
 
-  // static_cast an invalid int to IR2VecKind
-  auto Result = Embedder::create(static_cast<IR2VecKind>(-1), *F, V, 2);
-  EXPECT_FALSE(static_cast<bool>(Result));
+  E1 += E2;
+  EXPECT_THAT(E1, ElementsAre(1.5, 3.5, 2.0));
 
-  std::string ErrMsg;
-  llvm::handleAllErrors(
-      Result.takeError(),
-      [&](const llvm::ErrorInfoBase &EIB) { ErrMsg = EIB.message(); });
-  EXPECT_NE(ErrMsg.find("Unknown IR2VecKind"), std::string::npos);
+  // Check that E2 is unchanged
+  EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
 }
 
-TEST(IR2VecTest, AddVectors) {
+TEST(EmbeddingTest, SubtractVectors) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {0.5, 1.5, -1.0};
 
-  TestableEmbedder::addVectors(E1, E2);
-  EXPECT_THAT(E1, ElementsAre(1.5, 3.5, 2.0));
+  E1 -= E2;
+  EXPECT_THAT(E1, ElementsAre(0.5, 0.5, 4.0));
 
   // Check that E2 is unchanged
   EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
 }
 
-TEST(IR2VecTest, AddScaledVector) {
+TEST(EmbeddingTest, AddScaledVector) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {2.0, 0.5, -1.0};
 
-  TestableEmbedder::addScaledVector(E1, E2, 0.5f);
+  E1.scaleAndAdd(E2, 0.5f);
   EXPECT_THAT(E1, ElementsAre(2.0, 2.25, 2.5));
 
   // Check that E2 is unchanged
   EXPECT_THAT(E2, ElementsAre(2.0, 0.5, -1.0));
 }
 
+TEST(EmbeddingTest, ApproximatelyEqual) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = {1.0000001, 2.0000001, 3.0000001};
+  EXPECT_TRUE(E1.approximatelyEquals(E2)); // Diff = 1e-7
+
+  Embedding E3 = {1.00002, 2.00002, 3.00002}; // Diff = 2e-5
+  EXPECT_FALSE(E1.approximatelyEquals(E3));
+  EXPECT_TRUE(E1.approximatelyEquals(E3, 3e-5));
+
+  Embedding E_clearly_within = {1.0000005, 2.0000005, 3.0000005}; // Diff = 5e-7
+  EXPECT_TRUE(E1.approximatelyEquals(E_clearly_within));
+
+  Embedding E_clearly_outside = {1.00001, 2.00001, 3.00001}; // Diff = 1e-5
+  EXPECT_FALSE(E1.approximatelyEquals(E_clearly_outside));
+
+  Embedding E4 = {1.0, 2.0, 3.5}; // Large diff
+  EXPECT_FALSE(E1.approximatelyEquals(E4, 0.01));
+
+  Embedding E5 = {1.0, 2.0, 3.0};
+  EXPECT_TRUE(E1.approximatelyEquals(E5, 0.0));
+  EXPECT_TRUE(E1.approximatelyEquals(E5));
+}
+
 #if GTEST_HAS_DEATH_TEST
 #ifndef NDEBUG
-TEST(IR2VecTest, MismatchedDimensionsAddVectors) {
+TEST(EmbeddingTest, AccessOutOfBounds) {
+  Embedding E = {1.0, 2.0, 3.0};
+  EXPECT_DEATH(E[3], "Index out of bounds");
+  EXPECT_DEATH(E[-1], "Index out of bounds");
+  EXPECT_DEATH(E[4] = 4.0, "Index out of bounds");
+}
+
+TEST(EmbeddingTest, MismatchedDimensionsAddVectors) {
   Embedding E1 = {1.0, 2.0};
   Embedding E2 = {1.0};
-  EXPECT_DEATH(TestableEmbedder::addVectors(E1, E2),
-               "Vectors must have the same dimension");
+  EXPECT_DEATH(E1 += E2, "Vectors must have the same dimension");
 }
 
-TEST(IR2VecTest, MismatchedDimensionsAddScaledVector) {
+TEST(EmbeddingTest, MismatchedDimensionsSubtractVectors) {
   Embedding E1 = {1.0, 2.0};
   Embedding E2 = {1.0};
-  EXPECT_DEATH(TestableEmbedder::addScaledVector(E1, E2, 1.0f),
+  EXPECT_DEATH(E1 -= E2, "Vectors must have the same dimension");
+}
+
+TEST(EmbeddingTest, MismatchedDimensionsAddScaledVector) {
+  Embedding E1 = {1.0, 2.0};
+  Embedding E2 = {1.0};
+  EXPECT_DEATH(E1.scaleAndAdd(E2, 1.0f),
+               "Vectors must have the same dimension");
+}
+
+TEST(EmbeddingTest, MismatchedDimensionsApproximatelyEqual) {
+  Embedding E1 = {1.0, 2.0};
+  Embedding E2 = {1.010};
+  EXPECT_DEATH(E1.approximatelyEquals(E2),
                "Vectors must have the same dimension");
 }
 #endif // NDEBUG
 #endif // GTEST_HAS_DEATH_TEST
 
+TEST(IR2VecTest, CreateSymbolicEmbedder) {
+  Vocab V = {{"foo", {1.0, 2.0}}};
+
+  LLVMContext Ctx;
+  Module M("M", Ctx);
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), false);
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
+
+  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  EXPECT_TRUE(static_cast<bool>(Result));
+
+  auto *Emb = Result->get();
+  EXPECT_NE(Emb, nullptr);
+}
+
+TEST(IR2VecTest, CreateInvalidMode) {
+  Vocab V = {{"foo", {1.0, 2.0}}};
+
+  LLVMContext Ctx;
+  Module M("M", Ctx);
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), false);
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
+
+  // static_cast an invalid int to IR2VecKind
+  auto Result = Embedder::create(static_cast<IR2VecKind>(-1), *F, V);
+  EXPECT_FALSE(static_cast<bool>(Result));
+
+  std::string ErrMsg;
+  llvm::handleAllErrors(
+      Result.takeError(),
+      [&](const llvm::ErrorInfoBase &EIB) { ErrMsg = EIB.message(); });
+  EXPECT_NE(ErrMsg.find("Unknown IR2VecKind"), std::string::npos);
+}
+
 TEST(IR2VecTest, LookupVocab) {
   Vocab V = {{"foo", {1.0, 2.0}}, {"bar", {3.0, 4.0}}};
   LLVMContext Ctx;
@@ -123,7 +243,7 @@ TEST(IR2VecTest, LookupVocab) {
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), false);
   Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
 
-  TestableEmbedder E(*F, V, 2);
+  TestableEmbedder E(*F, V);
   auto V_foo = E.lookupVocab("foo");
   EXPECT_EQ(V_foo.size(), 2u);
   EXPECT_THAT(V_foo, ElementsAre(1.0, 2.0));
@@ -137,8 +257,9 @@ TEST(IR2VecTest, ZeroDimensionEmbedding) {
   Embedding E1;
   Embedding E2;
   // Should be no-op, but not crash
-  TestableEmbedder::addVectors(E1, E2);
-  TestableEmbedder::addScaledVector(E1, E2, 1.0f);
+  E1 += E2;
+  E1 -= E2;
+  E1.scaleAndAdd(E2, 1.0f);
   EXPECT_TRUE(E1.empty());
 }
 
@@ -160,25 +281,30 @@ TEST(IR2VecTest, IR2VecVocabResultValidity) {
   EXPECT_EQ(validResult.getDimension(), 2u);
 }
 
-// Helper to create a minimal function and embedder for getter tests
-struct GetterTestEnv {
-  Vocab V = {};
+// Fixture for IR2Vec tests requiring IR setup and weight management.
+class IR2VecTestFixture : public ::testing::Test {
+protected:
+  Vocab V;
   LLVMContext Ctx;
-  std::unique_ptr<Module> M = nullptr;
+  std::unique_ptr<Module> M;
   Function *F = nullptr;
   BasicBlock *BB = nullptr;
-  Instruction *Add = nullptr;
-  Instruction *Ret = nullptr;
-  std::unique_ptr<Embedder> Emb = nullptr;
+  Instruction *AddInst = nullptr;
+  Instruction *RetInst = nullptr;
+
+  float OriginalOpcWeight = ::OpcWeight;
+  float OriginalTypeWeight = ::TypeWeight;
+  float OriginalArgWeight = ::ArgWeight;
 
-  GetterTestEnv() {
+  void SetUp() override {
     V = {{"add", {1.0, 2.0}},
          {"integerTy", {0.5, 0.5}},
          {"constant", {0.2, 0.3}},
          {"variable", {0.0, 0.0}},
          {"unknownTy", {0.0, 0.0}}};
 
-    M = std::make_unique<Module>("M", Ctx);
+    // Setup IR
+    M = std::make_unique<Module>("TestM", Ctx);
     FunctionType *FTy = FunctionType::get(
         Type::getInt32Ty(Ctx), {Type::getInt32Ty(Ctx), Type::getInt32Ty(Ctx)},
         false);
@@ -187,61 +313,82 @@ struct GetterTestEnv {
     Argument *Arg = F->getArg(0);
     llvm::Value *Const = ConstantInt::get(Type::getInt32Ty(Ctx), 42);
 
-    Add = BinaryOperator::CreateAdd(Arg, Const, "add", BB);
-    Ret = ReturnInst::Create(Ctx, Add, BB);
+    AddInst = BinaryOperator::CreateAdd(Arg, Const, "add", BB);
+    RetInst = ReturnInst::Create(Ctx, AddInst, BB);
+  }
+
+  void setWeights(float OpcWeight, float TypeWeight, float ArgWeight) {
+    ::OpcWeight = OpcWeight;
+    ::TypeWeight = TypeWeight;
+    ::ArgWeight = ArgWeight;
+  }
 
-    auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V, 2);
-    EXPECT_TRUE(static_cast<bool>(Result));
-    Emb = std::move(*Result);
+  void TearDown() override {
+    // Restore original global weights
+    ::OpcWeight = OriginalOpcWeight;
+    ::TypeWeight = OriginalTypeWeight;
+    ::ArgWeight = OriginalArgWeight;
   }
 };
 
-TEST(IR2VecTest, GetInstVecMap) {
-  GetterTestEnv Env;
-  const auto &InstMap = Env.Emb->getInstVecMap();
+TEST_F(IR2VecTestFixture, GetInstVecMap) {
+  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Result));
+  auto Emb = std::move(*Result);
+
+  const auto &InstMap = Emb->getInstVecMap();
 
   EXPECT_EQ(InstMap.size(), 2u);
-  EXPECT_TRUE(InstMap.count(Env.Add));
-  EXPECT_TRUE(InstMap.count(Env.Ret));
+  EXPECT_TRUE(InstMap.count(AddInst));
+  EXPECT_TRUE(InstMap.count(RetInst));
 
-  EXPECT_EQ(InstMap.at(Env.Add).size(), 2u);
-  EXPECT_EQ(InstMap.at(Env.Ret).size(), 2u);
+  EXPECT_EQ(InstMap.at(AddInst).size(), 2u);
+  EXPECT_EQ(InstMap.at(RetInst).size(), 2u);
 
   // Check values for add: {1.29, 2.31}
-  EXPECT_THAT(InstMap.at(Env.Add),
+  EXPECT_THAT(InstMap.at(AddInst),
               ElementsAre(DoubleNear(1.29, 1e-6), DoubleNear(2.31, 1e-6)));
 
   // Check values for ret: {0.0, 0.}; Neither ret nor voidTy are present in
   // vocab
-  EXPECT_THAT(InstMap.at(Env.Ret), ElementsAre(0.0, 0.0));
+  EXPECT_THAT(InstMap.at(RetInst), ElementsAre(0.0, 0.0));
 }
 
-TEST(IR2VecTest, GetBBVecMap) {
-  GetterTestEnv Env;
-  const auto &BBMap = Env.Emb->getBBVecMap();
+TEST_F(IR2VecTestFixture, GetBBVecMap) {
+  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Result));
+  auto Emb = std::move(*Result);
+
+  const auto &BBMap = Emb->getBBVecMap();
 
   EXPECT_EQ(BBMap.size(), 1u);
-  EXPECT_TRUE(BBMap.count(Env.BB));
-  EXPECT_EQ(BBMap.at(Env.BB).size(), 2u);
+  EXPECT_TRUE(BBMap.count(BB));
+  EXPECT_EQ(BBMap.at(BB).size(), 2u);
 
   // BB vector should be sum of add and ret: {1.29, 2.31} + {0.0, 0.0} =
   // {1.29, 2.31}
-  EXPECT_THAT(BBMap.at(Env.BB),
+  EXPECT_THAT(BBMap.at(BB),
               ElementsAre(DoubleNear(1.29, 1e-6), DoubleNear(2.31, 1e-6)));
 }
 
-TEST(IR2VecTest, GetBBVector) {
-  GetterTestEnv Env;
-  const auto &BBVec = Env.Emb->getBBVector(*Env.BB);
+TEST_F(IR2VecTestFixture, GetBBVector) {
+  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Result));
+  auto Emb = std::move(*Result);
+
+  const auto &BBVec = Emb->getBBVector(*BB);
 
   EXPECT_EQ(BBVec.size(), 2u);
   EXPECT_THAT(BBVec,
               ElementsAre(DoubleNear(1.29, 1e-6), DoubleNear(2.31, 1e-6)));
 }
 
-TEST(IR2VecTest, GetFunctionVector) {
-  GetterTestEnv Env;
-  const auto &FuncVec = Env.Emb->getFunctionVector();
+TEST_F(IR2VecTestFixture, GetFunctionVector) {
+  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Result));
+  auto Emb = std::move(*Result);
+
+  const auto &FuncVec = Emb->getFunctionVector();
 
   EXPECT_EQ(FuncVec.size(), 2u);
 
@@ -250,4 +397,45 @@ TEST(IR2VecTest, GetFunctionVector) {
               ElementsAre(DoubleNear(1.29, 1e-6), DoubleNear(2.31, 1e-6)));
 }
 
+TEST_F(IR2VecTestFixture, GetFunctionVectorWithCustomWeights) {
+  setWeights(1.0, 1.0, 1.0);
+
+  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Result));
+  auto Emb = std::move(*Result);
+
+  const auto &FuncVec = Emb->getFunctionVector();
+
+  EXPECT_EQ(FuncVec.size(), 2u);
+
+  // Expected: 1*([1.0 2.0] + [0.0 0.0]) + 1*([0.5 0.5] + [0.0 0.0]) + 1*([0.2
+  // 0.3] + [0.0 0.0])
+  EXPECT_THAT(FuncVec,
+              ElementsAre(DoubleNear(1.7, 1e-6), DoubleNear(2.8, 1e-6)));
+}
+
+TEST(IR2VecTest, IR2VecVocabAnalysisWithPrepopulatedVocab) {
+  Vocab InitialVocab = {{"key1", {1.1, 2.2}}, {"key2", {3.3, 4.4}}};
+  Vocab ExpectedVocab = InitialVocab;
+  unsigned ExpectedDim = InitialVocab.begin()->second.size();
+
+  IR2VecVocabAnalysis VocabAnalysis(std::move(InitialVocab));
+
+  LLVMContext TestCtx;
+  Module TestMod("TestModuleForVocabAnalysis", TestCtx);
+  ModuleAnalysisManager MAM;
+  IR2VecVocabResult Result = VocabAnalysis.run(TestMod, MAM);
+
+  EXPECT_TRUE(Result.isValid());
+  ASSERT_FALSE(Result.getVocabulary().empty());
+  EXPECT_EQ(Result.getDimension(), ExpectedDim);
+
+  const auto &ResultVocab = Result.getVocabulary();
+  EXPECT_EQ(ResultVocab.size(), ExpectedVocab.size());
+  for (const auto &pair : ExpectedVocab) {
+    EXPECT_TRUE(ResultVocab.count(pair.first));
+    EXPECT_THAT(ResultVocab.at(pair.first), ElementsAreArray(pair.second));
+  }
+}
+
 } // end anonymous namespace
diff --git a/external/llvm-project/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp b/external/llvm-project/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
index 9edb21f71d0a..03009d53d63f 100644
--- a/external/llvm-project/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
+++ b/external/llvm-project/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
@@ -24,9 +24,6 @@
 using namespace llvm;
 using namespace IRSimilarity;
 
-LLVM_ABI extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-LLVM_ABI extern cl::opt<bool> UseNewDbgInfoFormat;
-
 static std::unique_ptr<Module> makeLLVMModule(LLVMContext &Context,
                                               StringRef ModuleStr) {
   SMDiagnostic Err;
diff --git a/external/llvm-project/llvm/unittests/Analysis/ValueTrackingTest.cpp b/external/llvm-project/llvm/unittests/Analysis/ValueTrackingTest.cpp
index a5050542b818..6031898f7f67 100644
--- a/external/llvm-project/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/external/llvm-project/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -187,6 +187,15 @@ TEST_F(MatchSelectPatternTest, FastFMin) {
       "  %A = select i1 %1, float %a, float 5.0\n"
       "  ret float %A\n"
       "}\n");
+  expectPattern({SPF_FMINNUM, SPNB_RETURNS_ANY, true});
+}
+
+TEST_F(MatchSelectPatternTest, FastFMinUnordered) {
+  parseAssembly("define float @test(float %a) {\n"
+                "  %1 = fcmp nnan ult float %a, 5.0\n"
+                "  %A = select i1 %1, float %a, float 5.0\n"
+                "  ret float %A\n"
+                "}\n");
   expectPattern({SPF_FMINNUM, SPNB_RETURNS_ANY, false});
 }
 
@@ -202,12 +211,11 @@ TEST_F(MatchSelectPatternTest, FMinConstantZero) {
 }
 
 TEST_F(MatchSelectPatternTest, FMinConstantZeroNsz) {
-  parseAssembly(
-      "define float @test(float %a) {\n"
-      "  %1 = fcmp nsz ole float %a, 0.0\n"
-      "  %A = select i1 %1, float %a, float 0.0\n"
-      "  ret float %A\n"
-      "}\n");
+  parseAssembly("define float @test(float %a) {\n"
+                "  %1 = fcmp nsz ole float %a, 0.0\n"
+                "  %A = select nsz i1 %1, float %a, float 0.0\n"
+                "  ret float %A\n"
+                "}\n");
   // But this should be, because we've ignored signed zeroes.
   expectPattern({SPF_FMINNUM, SPNB_RETURNS_OTHER, true});
 }
@@ -2016,27 +2024,27 @@ TEST_F(ComputeKnownFPClassTest, SqrtNszSignBit) {
 
   {
     KnownFPClass UseInstrInfo =
-        computeKnownFPClass(A, M->getDataLayout(), fcAllFlags, 0, nullptr,
-                            nullptr, nullptr, nullptr, /*UseInstrInfo=*/true);
+        computeKnownFPClass(A, M->getDataLayout(), fcAllFlags, nullptr, nullptr,
+                            nullptr, nullptr, /*UseInstrInfo=*/true);
     EXPECT_EQ(SqrtMask, UseInstrInfo.KnownFPClasses);
     EXPECT_EQ(std::nullopt, UseInstrInfo.SignBit);
 
     KnownFPClass NoUseInstrInfo =
-        computeKnownFPClass(A, M->getDataLayout(), fcAllFlags, 0, nullptr,
-                            nullptr, nullptr, nullptr, /*UseInstrInfo=*/false);
+        computeKnownFPClass(A, M->getDataLayout(), fcAllFlags, nullptr, nullptr,
+                            nullptr, nullptr, /*UseInstrInfo=*/false);
     EXPECT_EQ(SqrtMask, NoUseInstrInfo.KnownFPClasses);
     EXPECT_EQ(std::nullopt, NoUseInstrInfo.SignBit);
   }
 
   {
     KnownFPClass UseInstrInfoNSZ =
-        computeKnownFPClass(A2, M->getDataLayout(), fcAllFlags, 0, nullptr,
+        computeKnownFPClass(A2, M->getDataLayout(), fcAllFlags, nullptr,
                             nullptr, nullptr, nullptr, /*UseInstrInfo=*/true);
     EXPECT_EQ(NszSqrtMask, UseInstrInfoNSZ.KnownFPClasses);
     EXPECT_EQ(std::nullopt, UseInstrInfoNSZ.SignBit);
 
     KnownFPClass NoUseInstrInfoNSZ =
-        computeKnownFPClass(A2, M->getDataLayout(), fcAllFlags, 0, nullptr,
+        computeKnownFPClass(A2, M->getDataLayout(), fcAllFlags, nullptr,
                             nullptr, nullptr, nullptr, /*UseInstrInfo=*/false);
     EXPECT_EQ(SqrtMask, NoUseInstrInfoNSZ.KnownFPClasses);
     EXPECT_EQ(std::nullopt, NoUseInstrInfoNSZ.SignBit);
@@ -2044,14 +2052,14 @@ TEST_F(ComputeKnownFPClassTest, SqrtNszSignBit) {
 
   {
     KnownFPClass UseInstrInfoNoNan =
-        computeKnownFPClass(A3, M->getDataLayout(), fcAllFlags, 0, nullptr,
+        computeKnownFPClass(A3, M->getDataLayout(), fcAllFlags, nullptr,
                             nullptr, nullptr, nullptr, /*UseInstrInfo=*/true);
     EXPECT_EQ(fcPositive | fcNegZero | fcQNan,
               UseInstrInfoNoNan.KnownFPClasses);
     EXPECT_EQ(std::nullopt, UseInstrInfoNoNan.SignBit);
 
     KnownFPClass NoUseInstrInfoNoNan =
-        computeKnownFPClass(A3, M->getDataLayout(), fcAllFlags, 0, nullptr,
+        computeKnownFPClass(A3, M->getDataLayout(), fcAllFlags, nullptr,
                             nullptr, nullptr, nullptr, /*UseInstrInfo=*/false);
     EXPECT_EQ(fcPositive | fcNegZero | fcQNan,
               NoUseInstrInfoNoNan.KnownFPClasses);
@@ -2060,13 +2068,13 @@ TEST_F(ComputeKnownFPClassTest, SqrtNszSignBit) {
 
   {
     KnownFPClass UseInstrInfoNSZNoNan =
-        computeKnownFPClass(A4, M->getDataLayout(), fcAllFlags, 0, nullptr,
+        computeKnownFPClass(A4, M->getDataLayout(), fcAllFlags, nullptr,
                             nullptr, nullptr, nullptr, /*UseInstrInfo=*/true);
     EXPECT_EQ(fcPositive | fcQNan, UseInstrInfoNSZNoNan.KnownFPClasses);
     EXPECT_EQ(std::nullopt, UseInstrInfoNSZNoNan.SignBit);
 
     KnownFPClass NoUseInstrInfoNSZNoNan =
-        computeKnownFPClass(A4, M->getDataLayout(), fcAllFlags, 0, nullptr,
+        computeKnownFPClass(A4, M->getDataLayout(), fcAllFlags, nullptr,
                             nullptr, nullptr, nullptr, /*UseInstrInfo=*/false);
     EXPECT_EQ(fcPositive | fcNegZero | fcQNan,
               NoUseInstrInfoNSZNoNan.KnownFPClasses);
@@ -2086,8 +2094,7 @@ TEST_F(ComputeKnownFPClassTest, Constants) {
 
   {
     KnownFPClass ConstAggZero = computeKnownFPClass(
-        ConstantAggregateZero::get(V4F32), M->getDataLayout(), fcAllFlags, 0,
-        nullptr, nullptr, nullptr, nullptr);
+        ConstantAggregateZero::get(V4F32), M->getDataLayout(), fcAllFlags);
 
     EXPECT_EQ(fcPosZero, ConstAggZero.KnownFPClasses);
     ASSERT_TRUE(ConstAggZero.SignBit);
@@ -2095,17 +2102,15 @@ TEST_F(ComputeKnownFPClassTest, Constants) {
   }
 
   {
-    KnownFPClass Undef =
-        computeKnownFPClass(UndefValue::get(F32), M->getDataLayout(),
-                            fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    KnownFPClass Undef = computeKnownFPClass(UndefValue::get(F32),
+                                             M->getDataLayout(), fcAllFlags);
     EXPECT_EQ(fcAllFlags, Undef.KnownFPClasses);
     EXPECT_FALSE(Undef.SignBit);
   }
 
   {
-    KnownFPClass Poison =
-        computeKnownFPClass(PoisonValue::get(F32), M->getDataLayout(),
-                            fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    KnownFPClass Poison = computeKnownFPClass(PoisonValue::get(F32),
+                                              M->getDataLayout(), fcAllFlags);
     EXPECT_EQ(fcNone, Poison.KnownFPClasses);
     ASSERT_TRUE(Poison.SignBit);
     EXPECT_FALSE(*Poison.SignBit);
@@ -2116,9 +2121,9 @@ TEST_F(ComputeKnownFPClassTest, Constants) {
     Constant *ZeroF32 = ConstantFP::getZero(F32);
     Constant *PoisonF32 = PoisonValue::get(F32);
 
-    KnownFPClass PartiallyPoison = computeKnownFPClass(
-        ConstantVector::get({ZeroF32, PoisonF32}), M->getDataLayout(),
-        fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    KnownFPClass PartiallyPoison =
+        computeKnownFPClass(ConstantVector::get({ZeroF32, PoisonF32}),
+                            M->getDataLayout(), fcAllFlags);
     EXPECT_EQ(fcPosZero, PartiallyPoison.KnownFPClasses);
     ASSERT_TRUE(PartiallyPoison.SignBit);
     EXPECT_FALSE(*PartiallyPoison.SignBit);
@@ -2129,9 +2134,9 @@ TEST_F(ComputeKnownFPClassTest, Constants) {
     Constant *NegZeroF32 = ConstantFP::getZero(F32, true);
     Constant *PoisonF32 = PoisonValue::get(F32);
 
-    KnownFPClass PartiallyPoison = computeKnownFPClass(
-        ConstantVector::get({NegZeroF32, PoisonF32}), M->getDataLayout(),
-        fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    KnownFPClass PartiallyPoison =
+        computeKnownFPClass(ConstantVector::get({NegZeroF32, PoisonF32}),
+                            M->getDataLayout(), fcAllFlags);
     EXPECT_EQ(fcNegZero, PartiallyPoison.KnownFPClasses);
     ASSERT_TRUE(PartiallyPoison.SignBit);
     EXPECT_TRUE(*PartiallyPoison.SignBit);
@@ -2142,9 +2147,9 @@ TEST_F(ComputeKnownFPClassTest, Constants) {
     Constant *NegZeroF32 = ConstantFP::getZero(F32, true);
     Constant *PoisonF32 = PoisonValue::get(F32);
 
-    KnownFPClass PartiallyPoison = computeKnownFPClass(
-        ConstantVector::get({PoisonF32, NegZeroF32}), M->getDataLayout(),
-        fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    KnownFPClass PartiallyPoison =
+        computeKnownFPClass(ConstantVector::get({PoisonF32, NegZeroF32}),
+                            M->getDataLayout(), fcAllFlags);
     EXPECT_EQ(fcNegZero, PartiallyPoison.KnownFPClasses);
     EXPECT_TRUE(PartiallyPoison.SignBit);
   }
@@ -2454,8 +2459,8 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsPtrToIntTrunc) {
       "}\n"
       "declare void @llvm.assume(i1)\n");
   AssumptionCache AC(*F);
-  KnownBits Known = computeKnownBits(
-      A, M->getDataLayout(), /* Depth */ 0, &AC, F->front().getTerminator());
+  KnownBits Known =
+      computeKnownBits(A, M->getDataLayout(), &AC, F->front().getTerminator());
   EXPECT_TRUE(Known.isUnknown());
 }
 
@@ -2472,8 +2477,8 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsPtrToIntZext) {
       "}\n"
       "declare void @llvm.assume(i1)\n");
   AssumptionCache AC(*F);
-  KnownBits Known = computeKnownBits(
-      A, M->getDataLayout(), /* Depth */ 0, &AC, F->front().getTerminator());
+  KnownBits Known =
+      computeKnownBits(A, M->getDataLayout(), &AC, F->front().getTerminator());
   EXPECT_TRUE(Known.isUnknown());
 }
 
@@ -2489,8 +2494,8 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsFreeze) {
                 "declare void @llvm.assume(i1)\n"
                 "declare i32 @any_num()\n");
   AssumptionCache AC(*F);
-  KnownBits Known = computeKnownBits(A, M->getDataLayout(), /* Depth */ 0, &AC,
-                                     F->front().getTerminator());
+  KnownBits Known =
+      computeKnownBits(A, M->getDataLayout(), &AC, F->front().getTerminator());
   EXPECT_EQ(Known.Zero.getZExtValue(), 31u);
   EXPECT_EQ(Known.One.getZExtValue(), 0u);
 }
@@ -2519,12 +2524,12 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsAddWithRange) {
                 "}\n"
                 "declare void @llvm.assume(i1)\n");
   AssumptionCache AC(*F);
-  KnownBits Known = computeKnownBits(A, M->getDataLayout(), /* Depth */ 0, &AC,
-                                     F->front().getTerminator());
+  KnownBits Known =
+      computeKnownBits(A, M->getDataLayout(), &AC, F->front().getTerminator());
   EXPECT_EQ(Known.Zero.getZExtValue(), ~(65536llu - 1));
   EXPECT_EQ(Known.One.getZExtValue(), 0u);
   Instruction &APlus512 = findInstructionByName(F, "APlus512");
-  Known = computeKnownBits(&APlus512, M->getDataLayout(), /* Depth */ 0, &AC,
+  Known = computeKnownBits(&APlus512, M->getDataLayout(), &AC,
                            F->front().getTerminator());
   // We know of one less zero because 512 may have produced a 1 that
   // got carried all the way to the first trailing zero.
@@ -2543,7 +2548,7 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsUnknownVScale) {
                                                       {Builder.getInt32Ty()});
   CallInst *CI = Builder.CreateCall(TheFn, {}, {}, "");
 
-  KnownBits Known = computeKnownBits(CI, M.getDataLayout(), /* Depth */ 0);
+  KnownBits Known = computeKnownBits(CI, M.getDataLayout());
   // There is no parent function so we cannot look up the vscale_range
   // attribute to determine the number of bits.
   EXPECT_EQ(Known.One.getZExtValue(), 0u);
@@ -2551,7 +2556,7 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsUnknownVScale) {
 
   BasicBlock *BB = BasicBlock::Create(Context);
   CI->insertInto(BB, BB->end());
-  Known = computeKnownBits(CI, M.getDataLayout(), /* Depth */ 0);
+  Known = computeKnownBits(CI, M.getDataLayout());
   // There is no parent function so we cannot look up the vscale_range
   // attribute to determine the number of bits.
   EXPECT_EQ(Known.One.getZExtValue(), 0u);
@@ -2574,12 +2579,12 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsAddWithRangeNoOverlap) {
                 "}\n"
                 "declare void @llvm.assume(i1)\n");
   AssumptionCache AC(*F);
-  KnownBits Known = computeKnownBits(A, M->getDataLayout(), /* Depth */ 0, &AC,
-                                     F->front().getTerminator());
+  KnownBits Known =
+      computeKnownBits(A, M->getDataLayout(), &AC, F->front().getTerminator());
   EXPECT_EQ(Known.Zero.getZExtValue(), ~(64llu - 1));
   EXPECT_EQ(Known.One.getZExtValue(), 32u);
   Instruction &APlus512 = findInstructionByName(F, "APlus512");
-  Known = computeKnownBits(&APlus512, M->getDataLayout(), /* Depth */ 0, &AC,
+  Known = computeKnownBits(&APlus512, M->getDataLayout(), &AC,
                            F->front().getTerminator());
   EXPECT_EQ(Known.Zero.getZExtValue(), ~512llu & ~(64llu - 1));
   EXPECT_EQ(Known.One.getZExtValue(), 512u | 32u);
@@ -2601,12 +2606,12 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsGEPWithRange) {
       "}\n"
       "declare void @llvm.assume(i1)\n");
   AssumptionCache AC(*F);
-  KnownBits Known = computeKnownBits(A, M->getDataLayout(), /* Depth */ 0, &AC,
-                                     F->front().getTerminator());
+  KnownBits Known =
+      computeKnownBits(A, M->getDataLayout(), &AC, F->front().getTerminator());
   EXPECT_EQ(Known.Zero.getZExtValue(), ~(65536llu - 1));
   EXPECT_EQ(Known.One.getZExtValue(), 0u);
   Instruction &APtrPlus512 = findInstructionByName(F, "APtrPlus512");
-  Known = computeKnownBits(&APtrPlus512, M->getDataLayout(), /* Depth */ 0, &AC,
+  Known = computeKnownBits(&APtrPlus512, M->getDataLayout(), &AC,
                            F->front().getTerminator());
   // We know of one less zero because 512 may have produced a 1 that
   // got carried all the way to the first trailing zero.
@@ -2634,12 +2639,12 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsGEPWithRangeNoOverlap) {
       "}\n"
       "declare void @llvm.assume(i1)\n");
   AssumptionCache AC(*F);
-  KnownBits Known = computeKnownBits(A, M->getDataLayout(), /* Depth */ 0, &AC,
-                                     F->front().getTerminator());
+  KnownBits Known =
+      computeKnownBits(A, M->getDataLayout(), &AC, F->front().getTerminator());
   EXPECT_EQ(Known.Zero.getZExtValue(), ~(64llu - 1));
   EXPECT_EQ(Known.One.getZExtValue(), 32u);
   Instruction &APtrPlus512 = findInstructionByName(F, "APtrPlus512");
-  Known = computeKnownBits(&APtrPlus512, M->getDataLayout(), /* Depth */ 0, &AC,
+  Known = computeKnownBits(&APtrPlus512, M->getDataLayout(), &AC,
                            F->front().getTerminator());
   EXPECT_EQ(Known.Zero.getZExtValue(), ~512llu & ~(64llu - 1));
   EXPECT_EQ(Known.One.getZExtValue(), 512u | 32u);
diff --git a/external/llvm-project/llvm/unittests/CGData/StableFunctionMapRecordTest.cpp b/external/llvm-project/llvm/unittests/CGData/StableFunctionMapRecordTest.cpp
index f5c9afe449da..caaf6475ee50 100644
--- a/external/llvm-project/llvm/unittests/CGData/StableFunctionMapRecordTest.cpp
+++ b/external/llvm-project/llvm/unittests/CGData/StableFunctionMapRecordTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CGData/StableFunctionMapRecord.h"
+#include "llvm/CGData/CodeGenDataWriter.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
@@ -77,7 +78,10 @@ TEST(StableFunctionMapRecordTest, Serialize) {
   // Serialize and deserialize the map.
   SmallVector<char> Out;
   raw_svector_ostream OS(Out);
-  MapRecord1.serialize(OS);
+  std::vector<CGDataPatchItem> PatchItems;
+  MapRecord1.serialize(OS, PatchItems);
+  CGDataOStream COS(OS);
+  COS.patch(PatchItems);
 
   StableFunctionMapRecord MapRecord2;
   const uint8_t *Data = reinterpret_cast<const uint8_t *>(Out.data());
diff --git a/external/llvm-project/llvm/unittests/CodeGen/GlobalISel/KnownFPClassTest.cpp b/external/llvm-project/llvm/unittests/CodeGen/GlobalISel/KnownFPClassTest.cpp
index 6ee571804e69..040f0cfc9207 100644
--- a/external/llvm-project/llvm/unittests/CodeGen/GlobalISel/KnownFPClassTest.cpp
+++ b/external/llvm-project/llvm/unittests/CodeGen/GlobalISel/KnownFPClassTest.cpp
@@ -96,6 +96,104 @@ TEST_F(AArch64GISelMITest, TestFPClassCstVecNegZero) {
   EXPECT_EQ(true, Known.SignBit);
 }
 
+TEST_F(AArch64GISelMITest, TestFPClassCstZeroFPExt) {
+  StringRef MIRString = R"(
+   %c0:_(s32) = G_FCONSTANT float 0.0
+   %ext:_(s64) = nnan ninf G_FPEXT %c0
+   %copy_vector:_(s64) = COPY %ext
+)";
+
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyReg = Copies[Copies.size() - 1];
+  MachineInstr *FinalCopy = MRI->getVRegDef(CopyReg);
+  Register SrcReg = FinalCopy->getOperand(1).getReg();
+
+  GISelValueTracking Info(*MF);
+
+  KnownFPClass Known = Info.computeKnownFPClass(SrcReg);
+
+  EXPECT_EQ(fcZero | fcNormal, Known.KnownFPClasses);
+  EXPECT_EQ(std::nullopt, Known.SignBit);
+}
+
+TEST_F(AArch64GISelMITest, TestFPClassCstVecZeroFPExt) {
+  StringRef MIRString = R"(
+   %c0:_(s32) = G_FCONSTANT float 0.0
+   %c1:_(s32) = G_FCONSTANT float 0.0
+   %c2:_(s32) = G_FCONSTANT float 0.0
+   %vector:_(<3 x s32>) = G_BUILD_VECTOR %c0, %c1, %c2
+   %ext:_(<3 x s64>) = nnan ninf G_FPEXT %vector
+   %copy_vector:_(<3 x s64>) = COPY %ext
+)";
+
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyReg = Copies[Copies.size() - 1];
+  MachineInstr *FinalCopy = MRI->getVRegDef(CopyReg);
+  Register SrcReg = FinalCopy->getOperand(1).getReg();
+
+  GISelValueTracking Info(*MF);
+
+  KnownFPClass Known = Info.computeKnownFPClass(SrcReg);
+
+  EXPECT_EQ(fcZero | fcNormal, Known.KnownFPClasses);
+  EXPECT_EQ(std::nullopt, Known.SignBit);
+}
+
+TEST_F(AArch64GISelMITest, TestFPClassCstZeroFPTrunc) {
+  StringRef MIRString = R"(
+   %c0:_(s64) = G_FCONSTANT double 0.0
+   %trunc:_(s32) = nnan ninf G_FPTRUNC %c0
+   %copy_vector:_(s32) = COPY %trunc
+)";
+
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyReg = Copies[Copies.size() - 1];
+  MachineInstr *FinalCopy = MRI->getVRegDef(CopyReg);
+  Register SrcReg = FinalCopy->getOperand(1).getReg();
+
+  GISelValueTracking Info(*MF);
+
+  KnownFPClass Known = Info.computeKnownFPClass(SrcReg);
+
+  EXPECT_EQ(fcZero | fcPosSubnormal | fcPosNormal, Known.KnownFPClasses);
+  EXPECT_EQ(false, Known.SignBit);
+}
+
+TEST_F(AArch64GISelMITest, TestFPClassCstVecZeroFPTrunc) {
+  StringRef MIRString = R"(
+   %c0:_(s64) = G_FCONSTANT double 0.0
+   %c1:_(s64) = G_FCONSTANT double 0.0
+   %c2:_(s64) = G_FCONSTANT double 0.0
+   %vector:_(<3 x s64>) = G_BUILD_VECTOR %c0, %c1, %c2
+   %trunc:_(<3 x s32>) = nnan ninf G_FPTRUNC %vector
+   %copy_vector:_(<3 x s32>) = COPY %trunc
+)";
+
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyReg = Copies[Copies.size() - 1];
+  MachineInstr *FinalCopy = MRI->getVRegDef(CopyReg);
+  Register SrcReg = FinalCopy->getOperand(1).getReg();
+
+  GISelValueTracking Info(*MF);
+
+  KnownFPClass Known = Info.computeKnownFPClass(SrcReg);
+
+  EXPECT_EQ(fcZero | fcPosSubnormal | fcPosNormal, Known.KnownFPClasses);
+  EXPECT_EQ(false, Known.SignBit);
+}
+
 TEST_F(AArch64GISelMITest, TestFPClassSelectPos0) {
   StringRef MIRString = R"(
     %ptr:_(p0) = G_IMPLICIT_DEF
diff --git a/external/llvm-project/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/external/llvm-project/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index 67a0d948dcac..1b590aa33bd8 100644
--- a/external/llvm-project/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/external/llvm-project/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -567,7 +567,7 @@ struct VPMatchContext : public SDPatternMatch::BasicMatchContext {
       return OpVal->getOpcode() == Opc;
 
     auto BaseOpc = ISD::getBaseOpcodeForVP(OpVal->getOpcode(), false);
-    return BaseOpc.has_value() && *BaseOpc == Opc;
+    return BaseOpc == Opc;
   }
 
   unsigned getNumOperands(SDValue N) const {
diff --git a/external/llvm-project/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp b/external/llvm-project/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
index f61fc819d500..d5ae07eebe5d 100644
--- a/external/llvm-project/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
+++ b/external/llvm-project/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
@@ -70,7 +70,7 @@ void DWARFExpressionCompactPrinterTest::TestExprPrinter(
     return {};
   };
 
-  Expr.printCompact(OS, GetRegName);
+  DWARFExpressionPrinter::printCompact(&Expr, OS, GetRegName);
   EXPECT_EQ(OS.str(), Expected);
 }
 
diff --git a/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp b/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
index 613a417db0f9..f9aa050832c0 100644
--- a/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
+++ b/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
@@ -79,6 +79,10 @@ void checkElementPropertiesClangCodeview(LVReader *Reader) {
   EXPECT_EQ(CompileUnit->getBaseAddress(), 0u);
   EXPECT_TRUE(CompileUnit->getProducer().starts_with("clang"));
   EXPECT_EQ(CompileUnit->getName(), "test.cpp");
+  LVSourceLanguage Language = CompileUnit->getSourceLanguage();
+  EXPECT_TRUE(Language.isValid());
+  ASSERT_EQ(Language, LVSourceLanguage::CV_LANG_Cpp);
+  ASSERT_EQ(Language.getName(), "Cpp");
 
   EXPECT_EQ(Function->lineCount(), 16u);
   EXPECT_EQ(Function->scopeCount(), 1u);
@@ -453,6 +457,7 @@ void elementProperties(SmallString<128> &InputsDir) {
   ReaderOptions.setAttributeProducer();
   ReaderOptions.setAttributePublics();
   ReaderOptions.setAttributeRange();
+  ReaderOptions.setAttributeLanguage();
   ReaderOptions.setAttributeLocation();
   ReaderOptions.setAttributeSize();
   ReaderOptions.setPrintAll();
diff --git a/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/DWARFReaderTest.cpp b/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/DWARFReaderTest.cpp
index c409f24130d8..544c39a3c7b2 100644
--- a/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/DWARFReaderTest.cpp
+++ b/external/llvm-project/llvm/unittests/DebugInfo/LogicalView/DWARFReaderTest.cpp
@@ -76,6 +76,10 @@ void checkElementProperties(LVReader *Reader) {
   EXPECT_EQ(CompileUnit->getBaseAddress(), 0u);
   EXPECT_TRUE(CompileUnit->getProducer().starts_with("clang"));
   EXPECT_EQ(CompileUnit->getName(), "test.cpp");
+  LVSourceLanguage Language = CompileUnit->getSourceLanguage();
+  EXPECT_TRUE(Language.isValid());
+  EXPECT_EQ(Language, LVSourceLanguage::DW_LANG_C_plus_plus_14);
+  EXPECT_EQ(Language.getName(), "DW_LANG_C_plus_plus_14");
 
   EXPECT_EQ(CompileUnit->lineCount(), 0u);
   EXPECT_EQ(CompileUnit->scopeCount(), 1u);
@@ -315,6 +319,7 @@ void elementProperties(SmallString<128> &InputsDir) {
   ReaderOptions.setAttributeProducer();
   ReaderOptions.setAttributePublics();
   ReaderOptions.setAttributeRange();
+  ReaderOptions.setAttributeLanguage();
   ReaderOptions.setAttributeLocation();
   ReaderOptions.setAttributeInserted();
   ReaderOptions.setAttributeSize();
diff --git a/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp b/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
index f91fbdf01cac..d3fcb948fafb 100644
--- a/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
+++ b/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
@@ -533,7 +533,7 @@ TEST_F(OrcCAPITestBase, AddObjectBuffer) {
 
 // JITLink debug support plugins put information about JITed code in this GDB
 // JIT Interface global from OrcTargetProcess.
-extern "C" struct jit_descriptor __jit_debug_descriptor;
+extern "C" LLVM_ABI struct jit_descriptor __jit_debug_descriptor;
 
 static void *findLastDebugDescriptorEntryPtr() {
   struct jit_code_entry *Last = __jit_debug_descriptor.first_entry;
diff --git a/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp b/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
index ed1feed6a5a9..a0ae9308d33a 100644
--- a/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
+++ b/external/llvm-project/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
@@ -50,9 +50,10 @@ static bool testSetProcessAllSections(std::unique_ptr<MemoryBuffer> Obj,
   auto &JD = ES.createBareJITDylib("main");
   auto Foo = ES.intern("foo");
 
-  RTDyldObjectLinkingLayer ObjLayer(ES, [&NonAllocSectionSeen]() {
-    return std::make_unique<MemoryManagerWrapper>(NonAllocSectionSeen);
-  });
+  RTDyldObjectLinkingLayer ObjLayer(
+      ES, [&NonAllocSectionSeen](const MemoryBuffer &) {
+        return std::make_unique<MemoryManagerWrapper>(NonAllocSectionSeen);
+      });
 
   auto OnResolveDoNothing = [](Expected<SymbolMap> R) {
     cantFail(std::move(R));
@@ -156,8 +157,9 @@ TEST(RTDyldObjectLinkingLayerTest, TestOverrideObjectFlags) {
   ExecutionSession ES{std::make_unique<UnsupportedExecutorProcessControl>()};
   auto &JD = ES.createBareJITDylib("main");
   auto Foo = ES.intern("foo");
-  RTDyldObjectLinkingLayer ObjLayer(
-      ES, []() { return std::make_unique<SectionMemoryManager>(); });
+  RTDyldObjectLinkingLayer ObjLayer(ES, [](const MemoryBuffer &) {
+    return std::make_unique<SectionMemoryManager>();
+  });
   IRCompileLayer CompileLayer(ES, ObjLayer,
                               std::make_unique<FunkySimpleCompiler>(*TM));
 
@@ -226,8 +228,9 @@ TEST(RTDyldObjectLinkingLayerTest, TestAutoClaimResponsibilityForSymbols) {
   ExecutionSession ES{std::make_unique<UnsupportedExecutorProcessControl>()};
   auto &JD = ES.createBareJITDylib("main");
   auto Foo = ES.intern("foo");
-  RTDyldObjectLinkingLayer ObjLayer(
-      ES, []() { return std::make_unique<SectionMemoryManager>(); });
+  RTDyldObjectLinkingLayer ObjLayer(ES, [](const MemoryBuffer &) {
+    return std::make_unique<SectionMemoryManager>();
+  });
   IRCompileLayer CompileLayer(ES, ObjLayer,
                               std::make_unique<FunkySimpleCompiler>(*TM));
 
@@ -244,4 +247,63 @@ TEST(RTDyldObjectLinkingLayerTest, TestAutoClaimResponsibilityForSymbols) {
     ES.reportError(std::move(Err));
 }
 
+TEST(RTDyldObjectLinkingLayerTest, TestMemoryBufferNamePropagation) {
+  OrcNativeTarget::initialize();
+
+  std::unique_ptr<TargetMachine> TM(
+      EngineBuilder().selectTarget(Triple("x86_64-unknown-linux-gnu"), "", "",
+                                   SmallVector<std::string, 1>()));
+
+  if (!TM)
+    GTEST_SKIP();
+
+  // Create a module with two void() functions: foo and bar.
+  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
+  ThreadSafeModule M;
+  {
+    ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
+    MB.getModule()->setDataLayout(TM->createDataLayout());
+
+    Function *FooImpl = MB.createFunctionDecl(
+        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
+        "foo");
+    BasicBlock *FooEntry =
+        BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
+    IRBuilder<> B1(FooEntry);
+    B1.CreateRetVoid();
+
+    M = ThreadSafeModule(MB.takeModule(), std::move(TSCtx));
+  }
+
+  ExecutionSession ES{std::make_unique<UnsupportedExecutorProcessControl>()};
+  auto &JD = ES.createBareJITDylib("main");
+  auto Foo = ES.intern("foo");
+  std::string ObjectIdentifer;
+
+  RTDyldObjectLinkingLayer ObjLayer(
+      ES, [&ObjectIdentifer](const MemoryBuffer &Obj) {
+        // Capture the name of the object so that we can confirm that it
+        // contains the module name.
+        ObjectIdentifer = Obj.getBufferIdentifier().str();
+        return std::make_unique<SectionMemoryManager>();
+      });
+  IRCompileLayer CompileLayer(ES, ObjLayer,
+                              std::make_unique<SimpleCompiler>(*TM));
+
+  // Capture the module name before we move the module.
+  std::string ModuleName = M.getModuleUnlocked()->getName().str();
+
+  cantFail(CompileLayer.add(JD, std::move(M)));
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD), SymbolLookupSet(Foo),
+      SymbolState::Resolved,
+      [](Expected<SymbolMap> R) { cantFail(std::move(R)); },
+      NoDependenciesToRegister);
+
+  if (auto Err = ES.endSession())
+    ES.reportError(std::move(Err));
+
+  EXPECT_TRUE(ObjectIdentifer.find(ModuleName) != std::string::npos);
+}
+
 } // end anonymous namespace
diff --git a/external/llvm-project/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp b/external/llvm-project/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
index 3f92fa0f0579..90e6cd0a80d6 100644
--- a/external/llvm-project/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
+++ b/external/llvm-project/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
 #include "gtest/gtest.h"
 
 using namespace llvm::hlsl::rootsig;
diff --git a/external/llvm-project/llvm/unittests/IR/CMakeLists.txt b/external/llvm-project/llvm/unittests/IR/CMakeLists.txt
index bea6b1b46f57..b66eae93f933 100644
--- a/external/llvm-project/llvm/unittests/IR/CMakeLists.txt
+++ b/external/llvm-project/llvm/unittests/IR/CMakeLists.txt
@@ -50,7 +50,6 @@ add_llvm_unittest(IRTests
   ValueHandleTest.cpp
   ValueMapTest.cpp
   ValueTest.cpp
-  VectorBuilderTest.cpp
   VectorTypeUtilsTest.cpp
   VectorTypesTest.cpp
   VerifierTest.cpp
diff --git a/external/llvm-project/llvm/unittests/IR/DebugInfoTest.cpp b/external/llvm-project/llvm/unittests/IR/DebugInfoTest.cpp
index 1dc0645f401d..ac7914fe6b9f 100644
--- a/external/llvm-project/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/external/llvm-project/llvm/unittests/IR/DebugInfoTest.cpp
@@ -19,6 +19,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Transforms/Utils/Local.h"
 
@@ -26,8 +27,6 @@
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
   SMDiagnostic Err;
   std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
@@ -240,8 +239,6 @@ TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
 // Duplicate of above test, but in DbgVariableRecord representation.
 TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
   LLVMContext C;
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = true;
 
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
@@ -284,23 +281,19 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
   EXPECT_EQ(DVRs[1]->getNumVariableLocationOps(), 2u);
   EXPECT_TRUE(isa<UndefValue>(DVRs[1]->getVariableLocationOp(1)));
   EXPECT_TRUE(DVRs[1]->isKillLocation());
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 // Ensure that the order of dbg.value intrinsics returned by findDbgValues, and
 // their corresponding DbgVariableRecord representation, are consistent.
 TEST(MetadataTest, OrderingOfDbgVariableRecords) {
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
   LLVMContext C;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       %b = add i16 %a, 1, !dbg !11
-      call void @llvm.dbg.value(metadata i16 %b, metadata !9, metadata !DIExpression()), !dbg !11
-      call void @llvm.dbg.value(metadata i16 %b, metadata !12, metadata !DIExpression()), !dbg !11
+        #dbg_value(i16 %b, !9, !DIExpression(), !11)
+        #dbg_value(i16 %b, !12, !DIExpression(), !11)
       ret i16 0, !dbg !11
     }
-    declare void @llvm.dbg.value(metadata, metadata, metadata) #0
     attributes #0 = { nounwind readnone speculatable willreturn }
 
     !llvm.dbg.cu = !{!0}
@@ -323,33 +316,20 @@ TEST(MetadataTest, OrderingOfDbgVariableRecords) {
 
   SmallVector<DbgValueInst *, 2> DVIs;
   SmallVector<DbgVariableRecord *, 2> DVRs;
-  findDbgValues(DVIs, &I, &DVRs);
-  ASSERT_EQ(DVIs.size(), 2u);
-  ASSERT_EQ(DVRs.size(), 0u);
-
-  // The correct order of dbg.values is given by their use-list, which becomes
-  // the reverse order of creation. Thus the dbg.values should come out as
-  // "bar" and then "foo".
-  DILocalVariable *Var0 = DVIs[0]->getVariable();
-  EXPECT_TRUE(Var0->getName() == "bar");
-  DILocalVariable *Var1 = DVIs[1]->getVariable();
-  EXPECT_TRUE(Var1->getName() == "foo");
-
-  // Now try again, but in DbgVariableRecord form.
-  DVIs.clear();
 
-  M->convertToNewDbgValues();
   findDbgValues(DVIs, &I, &DVRs);
   ASSERT_EQ(DVIs.size(), 0u);
   ASSERT_EQ(DVRs.size(), 2u);
 
-  Var0 = DVRs[0]->getVariable();
+  // The correct order of dbg.values is given by their use-list, which becomes
+  // the reverse order of creation. Thus the dbg.values should come out as
+  // "bar" and then "foo".
+  const DILocalVariable *Var0 = DVRs[0]->getVariable();
   EXPECT_TRUE(Var0->getName() == "bar");
-  Var1 = DVRs[1]->getVariable();
+  const DILocalVariable *Var1 = DVRs[1]->getVariable();
   EXPECT_TRUE(Var1->getName() == "foo");
 
   M->convertFromNewDbgValues();
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(DIBuilder, CreateFile) {
@@ -923,8 +903,6 @@ TEST(AssignmentTrackingTest, InstrMethods) {
 // dbg.values that have been converted to a non-instruction format.
 TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
   LLVMContext C;
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -954,6 +932,12 @@ TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
+  // The IR above will be autoupgraded to debug records: but this test is about
+  // the conversion routines, so convert it back. This test will have value
+  // going forwards for the purpose of testing the conversion routine, which
+  // some compatibility tools (DXIL?) wish to use.
+  M->convertFromNewDbgValues();
+
   // Find the first dbg.value,
   Instruction &I = *M->getFunction("f")->getEntryBlock().getFirstNonPHIIt();
   const DILocalVariable *Var = nullptr;
@@ -1008,7 +992,6 @@ TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
   Instruction *RetInst = &*std::next(FirstInst->getIterator());
 
   // Set-up DbgMarkers in this block.
-  ExitBlock->IsNewDbgInfoFormat = true;
   ExitBlock->createMarker(FirstInst);
   ExitBlock->createMarker(RetInst);
 
@@ -1099,15 +1082,11 @@ TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
   // The record of those trailing DbgVariableRecords would dangle and cause an
   // assertion failure if it lived until the end of the LLVMContext.
   ExitBlock->deleteTrailingDbgRecords();
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   LLVMContext C;
 
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -1137,18 +1116,17 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
-  // For the purpose of this test, set and un-set the command line option
-  // corresponding to UseNewDbgInfoFormat, but only after parsing, to ensure
-  // that the IR starts off in the old format.
-  UseNewDbgInfoFormat = true;
+  // This test exists to check we can convert back and forth between old and new
+  // debug info formats (dbg.value intrinsics versus #dbg_value records).
+  // We're ripping out support for debug intrinsics, but the conversions will
+  // live on in bitcode autoupgrade and possibly DXIL autodowngrade. Thus, this
+  // test is still valuable. Begin by starting in the intrinsic format:
+  M->convertFromNewDbgValues();
 
-  // Check that the conversion routines and utilities between dbg.value
-  // debug-info format and DbgVariableRecords works.
   Function *F = M->getFunction("f");
   BasicBlock *BB1 = &F->getEntryBlock();
   // First instruction should be a dbg.value.
   EXPECT_TRUE(isa<DbgValueInst>(BB1->front()));
-  EXPECT_FALSE(BB1->IsNewDbgInfoFormat);
   // Validating the block for DbgVariableRecords / DbgMarkers shouldn't fail --
   // there's no data stored right now.
   bool BrokenDebugInfo = false;
@@ -1156,15 +1134,8 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   EXPECT_FALSE(Error);
   EXPECT_FALSE(BrokenDebugInfo);
 
-  // Function and module should be marked as not having the new format too.
-  EXPECT_FALSE(F->IsNewDbgInfoFormat);
-  EXPECT_FALSE(M->IsNewDbgInfoFormat);
-
   // Now convert.
   M->convertToNewDbgValues();
-  EXPECT_TRUE(M->IsNewDbgInfoFormat);
-  EXPECT_TRUE(F->IsNewDbgInfoFormat);
-  EXPECT_TRUE(BB1->IsNewDbgInfoFormat);
 
   // There should now be no dbg.value instructions!
   // Ensure the first instruction exists, the test all of them.
@@ -1201,7 +1172,6 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   // There should be no DbgVariableRecords / DbgMarkers in the second block, but
   // it should be marked as being in the new format.
   BasicBlock *BB2 = BB1->getNextNode();
-  EXPECT_TRUE(BB2->IsNewDbgInfoFormat);
   for (auto &Inst : *BB2)
     // Either there should be no marker, or it should be empty.
     EXPECT_TRUE(!Inst.DebugMarker ||
@@ -1228,9 +1198,6 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
 
   // Convert everything back to the "old" format and ensure it's right.
   M->convertFromNewDbgValues();
-  EXPECT_FALSE(M->IsNewDbgInfoFormat);
-  EXPECT_FALSE(F->IsNewDbgInfoFormat);
-  EXPECT_FALSE(BB1->IsNewDbgInfoFormat);
 
   EXPECT_EQ(BB1->size(), 4u);
   ASSERT_TRUE(isa<DbgValueInst>(BB1->front()));
@@ -1245,8 +1212,6 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   EXPECT_EQ(DVI1->getExpression(), Expr1);
   EXPECT_EQ(DVI2->getVariable(), DLV2);
   EXPECT_EQ(DVI2->getExpression(), Expr2);
-
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 // Test that the hashing function for DISubprograms representing methods produce
diff --git a/external/llvm-project/llvm/unittests/IR/IRBuilderTest.cpp b/external/llvm-project/llvm/unittests/IR/IRBuilderTest.cpp
index b7eb0af72833..520735dfc326 100644
--- a/external/llvm-project/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/external/llvm-project/llvm/unittests/IR/IRBuilderTest.cpp
@@ -212,14 +212,6 @@ TEST_F(IRBuilderTest, IntrinsicsWithScalableVectors) {
     EXPECT_EQ(FTy->getParamType(i), Args[i]->getType());
 }
 
-TEST_F(IRBuilderTest, CreateVScale) {
-  IRBuilder<> Builder(BB);
-
-  Constant *Zero = Builder.getInt32(0);
-  Value *VScale = Builder.CreateVScale(Zero);
-  EXPECT_TRUE(isa<ConstantInt>(VScale) && cast<ConstantInt>(VScale)->isZero());
-}
-
 TEST_F(IRBuilderTest, CreateStepVector) {
   IRBuilder<> Builder(BB);
 
@@ -896,14 +888,9 @@ TEST_F(IRBuilderTest, DIBuilder) {
   };
 
   auto ExpectOrder = [&](DbgInstPtr First, BasicBlock::iterator Second) {
-    if (M->IsNewDbgInfoFormat) {
-      EXPECT_TRUE(isa<DbgRecord *>(First));
-      EXPECT_FALSE(Second->getDbgRecordRange().empty());
-      EXPECT_EQ(GetLastDbgRecord(&*Second), cast<DbgRecord *>(First));
-    } else {
-      EXPECT_TRUE(isa<Instruction *>(First));
-      EXPECT_EQ(&*std::prev(Second), cast<Instruction *>(First));
-    }
+    EXPECT_TRUE(isa<DbgRecord *>(First));
+    EXPECT_FALSE(Second->getDbgRecordRange().empty());
+    EXPECT_EQ(GetLastDbgRecord(&*Second), cast<DbgRecord *>(First));
   };
 
   auto RunTest = [&]() {
@@ -1011,18 +998,8 @@ TEST_F(IRBuilderTest, DIBuilder) {
     EXPECT_TRUE(verifyModule(*M));
   };
 
-  // Test in new-debug mode.
-  EXPECT_TRUE(M->IsNewDbgInfoFormat);
   RunTest();
-
-  // Test in old-debug mode.
-  // Reset the test then call convertFromNewDbgValues to flip the flag
-  // on the test's Module, Function and BasicBlock.
   TearDown();
-  SetUp();
-  M->convertFromNewDbgValues();
-  EXPECT_FALSE(M->IsNewDbgInfoFormat);
-  RunTest();
 }
 
 TEST_F(IRBuilderTest, createArtificialSubprogram) {
diff --git a/external/llvm-project/llvm/unittests/IR/InstructionsTest.cpp b/external/llvm-project/llvm/unittests/IR/InstructionsTest.cpp
index a194d259ef32..126db4d4c162 100644
--- a/external/llvm-project/llvm/unittests/IR/InstructionsTest.cpp
+++ b/external/llvm-project/llvm/unittests/IR/InstructionsTest.cpp
@@ -27,13 +27,12 @@
 #include "llvm/IR/NoFolder.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
 #include <memory>
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace llvm {
 namespace {
 
@@ -1451,8 +1450,6 @@ TEST(InstructionsTest, GetSplat) {
 
 TEST(InstructionsTest, SkipDebug) {
   LLVMContext C;
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
   std::unique_ptr<Module> M = parseIR(C,
                                       R"(
       declare void @llvm.dbg.value(metadata, metadata, metadata)
@@ -1479,6 +1476,8 @@ TEST(InstructionsTest, SkipDebug) {
   )");
   ASSERT_TRUE(M);
   Function *F = cast<Function>(M->getNamedValue("f"));
+  // This test wants to see dbg.values.
+  F->convertFromNewDbgValues();
   BasicBlock &BB = F->front();
 
   // The first non-debug instruction is the terminator.
@@ -1488,7 +1487,6 @@ TEST(InstructionsTest, SkipDebug) {
 
   // After the terminator, there are no non-debug instructions.
   EXPECT_EQ(nullptr, Term->getNextNonDebugInstruction());
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(InstructionsTest, PhiMightNotBeFPMathOperator) {
diff --git a/external/llvm-project/llvm/unittests/IR/MetadataTest.cpp b/external/llvm-project/llvm/unittests/IR/MetadataTest.cpp
index e685ee18aca0..c788a80a560a 100644
--- a/external/llvm-project/llvm/unittests/IR/MetadataTest.cpp
+++ b/external/llvm-project/llvm/unittests/IR/MetadataTest.cpp
@@ -22,13 +22,14 @@
 #include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 #include <optional>
 using namespace llvm;
 
 namespace llvm {
-extern cl::opt<bool> PickMergedSourceLocations;
+LLVM_ABI extern cl::opt<bool> PickMergedSourceLocations;
 } // namespace llvm
 
 namespace {
diff --git a/external/llvm-project/llvm/unittests/IR/PatternMatch.cpp b/external/llvm-project/llvm/unittests/IR/PatternMatch.cpp
index 639332c9086d..91746b289333 100644
--- a/external/llvm-project/llvm/unittests/IR/PatternMatch.cpp
+++ b/external/llvm-project/llvm/unittests/IR/PatternMatch.cpp
@@ -2355,25 +2355,6 @@ TEST_F(PatternMatchTest, VectorLogicalSelects) {
   EXPECT_FALSE(match(MixedTypeOr, m_LogicalOr(m_Value(), m_Value())));
 }
 
-TEST_F(PatternMatchTest, VScale) {
-  DataLayout DL = M->getDataLayout();
-
-  Type *VecTy = ScalableVectorType::get(IRB.getInt8Ty(), 1);
-  Value *NullPtrVec =
-      Constant::getNullValue(PointerType::getUnqual(VecTy->getContext()));
-  Value *GEP = IRB.CreateGEP(VecTy, NullPtrVec, IRB.getInt64(1));
-  Value *PtrToInt = IRB.CreatePtrToInt(GEP, DL.getIntPtrType(GEP->getType()));
-  EXPECT_TRUE(match(PtrToInt, m_VScale()));
-
-  Type *VecTy2 = ScalableVectorType::get(IRB.getInt8Ty(), 2);
-  Value *NullPtrVec2 =
-      Constant::getNullValue(PointerType::getUnqual(VecTy2->getContext()));
-  Value *GEP2 = IRB.CreateGEP(VecTy, NullPtrVec2, IRB.getInt64(1));
-  Value *PtrToInt2 =
-      IRB.CreatePtrToInt(GEP2, DL.getIntPtrType(GEP2->getType()));
-  EXPECT_TRUE(match(PtrToInt2, m_VScale()));
-}
-
 TEST_F(PatternMatchTest, NotForbidPoison) {
   Type *ScalarTy = IRB.getInt8Ty();
   Type *VectorTy = FixedVectorType::get(ScalarTy, 3);
diff --git a/external/llvm-project/llvm/unittests/IR/ValueTest.cpp b/external/llvm-project/llvm/unittests/IR/ValueTest.cpp
index ca45e9abcf22..4d28fe019ecd 100644
--- a/external/llvm-project/llvm/unittests/IR/ValueTest.cpp
+++ b/external/llvm-project/llvm/unittests/IR/ValueTest.cpp
@@ -15,12 +15,11 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace {
 
 TEST(ValueTest, UsedInBasicBlock) {
@@ -255,74 +254,6 @@ TEST(ValueTest, getLocalSlotDeath) {
 #endif
 
 TEST(ValueTest, replaceUsesOutsideBlock) {
-  // Check that Value::replaceUsesOutsideBlock(New, BB) replaces uses outside
-  // BB, including dbg.* uses of MetadataAsValue(ValueAsMetadata(this)).
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
-  const auto *IR = R"(
-    define i32 @f() !dbg !6 {
-    entry:
-      %a = add i32 0, 1, !dbg !15
-      %b = add i32 0, 2, !dbg !15
-      %c = add i32 %a, 2, !dbg !15
-      call void @llvm.dbg.value(metadata i32 %a, metadata !9, metadata !DIExpression()), !dbg !15
-      br label %exit, !dbg !15
-
-    exit:
-      call void @llvm.dbg.value(metadata i32 %a, metadata !11, metadata !DIExpression()), !dbg !16
-      ret i32 %a, !dbg !16
-    }
-
-    declare void @llvm.dbg.value(metadata, metadata, metadata)
-
-    !llvm.dbg.cu = !{!0}
-    !llvm.module.flags = !{!5}
-
-    !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
-    !1 = !DIFile(filename: "test.ll", directory: "/")
-    !2 = !{}
-    !5 = !{i32 2, !"Debug Info Version", i32 3}
-    !6 = distinct !DISubprogram(name: "f", linkageName: "f", scope: null, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !8)
-    !7 = !DISubroutineType(types: !2)
-    !8 = !{!9, !11}
-    !9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
-    !10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_signed)
-    !11 = !DILocalVariable(name: "2", scope: !6, file: !1, line: 2, type: !12)
-    !12 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_signed)
-    !15 = !DILocation(line: 1, column: 1, scope: !6)
-    !16 = !DILocation(line: 5, column: 1, scope: !6)
-  )";
-  LLVMContext Ctx;
-  SMDiagnostic Err;
-  std::unique_ptr<Module> M = parseAssemblyString(IR, Err, Ctx);
-  if (!M)
-    Err.print("ValueTest", errs());
-
-  auto GetNext = [](auto *I) { return &*++I->getIterator(); };
-
-  Function *F = M->getFunction("f");
-  // Entry.
-  BasicBlock *Entry = &F->front();
-  Instruction *A = &Entry->front();
-  Instruction *B = GetNext(A);
-  Instruction *C = GetNext(B);
-  auto *EntryDbg = cast<DbgValueInst>(GetNext(C));
-  // Exit.
-  BasicBlock *Exit = GetNext(Entry);
-  auto *ExitDbg = cast<DbgValueInst>(&Exit->front());
-  Instruction *Ret = GetNext(ExitDbg);
-
-  A->replaceUsesOutsideBlock(B, Entry);
-  // These users are in Entry so shouldn't be changed.
-  ASSERT_TRUE(C->getOperand(0) == cast<Value>(A));
-  ASSERT_TRUE(EntryDbg->getValue(0) == cast<Value>(A));
-  // These users are outside Entry so should be changed.
-  ASSERT_TRUE(ExitDbg->getValue(0) == cast<Value>(B));
-  ASSERT_TRUE(Ret->getOperand(0) == cast<Value>(B));
-  UseNewDbgInfoFormat = OldDbgValueMode;
-}
-
-TEST(ValueTest, replaceUsesOutsideBlockDbgVariableRecord) {
   // Check that Value::replaceUsesOutsideBlock(New, BB) replaces uses outside
   // BB, including DbgVariableRecords.
   const auto *IR = R"(
diff --git a/external/llvm-project/llvm/unittests/IR/VectorBuilderTest.cpp b/external/llvm-project/llvm/unittests/IR/VectorBuilderTest.cpp
deleted file mode 100644
index e01378a2755f..000000000000
--- a/external/llvm-project/llvm/unittests/IR/VectorBuilderTest.cpp
+++ /dev/null
@@ -1,279 +0,0 @@
-//===--------- VectorBuilderTest.cpp - VectorBuilder unit tests -----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/IR/VectorBuilder.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-
-namespace {
-
-static unsigned VectorNumElements = 8;
-
-class VectorBuilderTest : public testing::Test {
-protected:
-  LLVMContext Context;
-
-  VectorBuilderTest() : Context() {}
-
-  std::unique_ptr<Module> createBuilderModule(Function *&Func, BasicBlock *&BB,
-                                              Value *&Mask, Value *&EVL) {
-    auto Mod = std::make_unique<Module>("TestModule", Context);
-    auto *Int32Ty = Type::getInt32Ty(Context);
-    auto *Mask8Ty =
-        FixedVectorType::get(Type::getInt1Ty(Context), VectorNumElements);
-    auto *VoidFuncTy =
-        FunctionType::get(Type::getVoidTy(Context), {Mask8Ty, Int32Ty}, false);
-    Func =
-        Function::Create(VoidFuncTy, GlobalValue::ExternalLinkage, "bla", *Mod);
-    Mask = Func->getArg(0);
-    EVL = Func->getArg(1);
-    BB = BasicBlock::Create(Context, "entry", Func);
-
-    return Mod;
-  }
-};
-
-/// Check that creating binary arithmetic VP intrinsics works.
-TEST_F(VectorBuilderTest, TestCreateBinaryInstructions) {
-  Function *F;
-  BasicBlock *BB;
-  Value *Mask, *EVL;
-  auto Mod = createBuilderModule(F, BB, Mask, EVL);
-
-  IRBuilder<> Builder(BB);
-  VectorBuilder VBuild(Builder);
-  VBuild.setMask(Mask).setEVL(EVL);
-
-  auto *FloatVecTy =
-      FixedVectorType::get(Type::getFloatTy(Context), VectorNumElements);
-  auto *IntVecTy =
-      FixedVectorType::get(Type::getInt32Ty(Context), VectorNumElements);
-
-#define HANDLE_BINARY_INST(NUM, OPCODE, INSTCLASS)                             \
-  {                                                                            \
-    auto VPID = VPIntrinsic::getForOpcode(Instruction::OPCODE);                \
-    bool IsFP = (#INSTCLASS)[0] == 'F';                                        \
-    auto *ValueTy = IsFP ? FloatVecTy : IntVecTy;                              \
-    Value *Op = PoisonValue::get(ValueTy);                                      \
-    auto *I = VBuild.createVectorInstruction(Instruction::OPCODE, ValueTy,     \
-                                             {Op, Op});                        \
-    ASSERT_TRUE(isa<VPIntrinsic>(I));                                          \
-    auto *VPIntrin = cast<VPIntrinsic>(I);                                     \
-    ASSERT_EQ(VPIntrin->getIntrinsicID(), VPID);                               \
-    ASSERT_EQ(VPIntrin->getMaskParam(), Mask);                                 \
-    ASSERT_EQ(VPIntrin->getVectorLengthParam(), EVL);                          \
-  }
-#include "llvm/IR/Instruction.def"
-}
-
-static bool isAllTrueMask(Value *Val, unsigned NumElements) {
-  auto *ConstMask = dyn_cast<Constant>(Val);
-  if (!ConstMask)
-    return false;
-
-  // Structure check.
-  if (!ConstMask->isAllOnesValue())
-    return false;
-
-  // Type check.
-  auto *MaskVecTy = cast<FixedVectorType>(ConstMask->getType());
-  if (MaskVecTy->getNumElements() != NumElements)
-    return false;
-
-  return MaskVecTy->getElementType()->isIntegerTy(1);
-}
-
-/// Check that creating binary arithmetic VP intrinsics works.
-TEST_F(VectorBuilderTest, TestCreateBinaryInstructions_FixedVector_NoMask) {
-  Function *F;
-  BasicBlock *BB;
-  Value *Mask, *EVL;
-  auto Mod = createBuilderModule(F, BB, Mask, EVL);
-
-  IRBuilder<> Builder(BB);
-  VectorBuilder VBuild(Builder);
-  VBuild.setEVL(EVL).setStaticVL(VectorNumElements);
-
-  auto *FloatVecTy =
-      FixedVectorType::get(Type::getFloatTy(Context), VectorNumElements);
-  auto *IntVecTy =
-      FixedVectorType::get(Type::getInt32Ty(Context), VectorNumElements);
-
-#define HANDLE_BINARY_INST(NUM, OPCODE, INSTCLASS)                             \
-  {                                                                            \
-    auto VPID = VPIntrinsic::getForOpcode(Instruction::OPCODE);                \
-    bool IsFP = (#INSTCLASS)[0] == 'F';                                        \
-    Type *ValueTy = IsFP ? FloatVecTy : IntVecTy;                              \
-    Value *Op = PoisonValue::get(ValueTy);                                      \
-    auto *I = VBuild.createVectorInstruction(Instruction::OPCODE, ValueTy,     \
-                                             {Op, Op});                        \
-    ASSERT_TRUE(isa<VPIntrinsic>(I));                                          \
-    auto *VPIntrin = cast<VPIntrinsic>(I);                                     \
-    ASSERT_EQ(VPIntrin->getIntrinsicID(), VPID);                               \
-    ASSERT_TRUE(isAllTrueMask(VPIntrin->getMaskParam(), VectorNumElements));   \
-    ASSERT_EQ(VPIntrin->getVectorLengthParam(), EVL);                          \
-  }
-#include "llvm/IR/Instruction.def"
-}
-
-static bool isLegalConstEVL(Value *Val, unsigned ExpectedEVL) {
-  auto *ConstEVL = dyn_cast<ConstantInt>(Val);
-  if (!ConstEVL)
-    return false;
-
-  // Value check.
-  if (ConstEVL->getZExtValue() != ExpectedEVL)
-    return false;
-
-  // Type check.
-  return ConstEVL->getType()->isIntegerTy(32);
-}
-
-/// Check that creating binary arithmetic VP intrinsics works.
-TEST_F(VectorBuilderTest, TestCreateBinaryInstructions_FixedVector_NoEVL) {
-  Function *F;
-  BasicBlock *BB;
-  Value *Mask, *EVL;
-  auto Mod = createBuilderModule(F, BB, Mask, EVL);
-
-  IRBuilder<> Builder(BB);
-  VectorBuilder VBuild(Builder);
-  VBuild.setMask(Mask).setStaticVL(VectorNumElements);
-
-  auto *FloatVecTy =
-      FixedVectorType::get(Type::getFloatTy(Context), VectorNumElements);
-  auto *IntVecTy =
-      FixedVectorType::get(Type::getInt32Ty(Context), VectorNumElements);
-
-#define HANDLE_BINARY_INST(NUM, OPCODE, INSTCLASS)                             \
-  {                                                                            \
-    auto VPID = VPIntrinsic::getForOpcode(Instruction::OPCODE);                \
-    bool IsFP = (#INSTCLASS)[0] == 'F';                                        \
-    Type *ValueTy = IsFP ? FloatVecTy : IntVecTy;                              \
-    Value *Op = PoisonValue::get(ValueTy);                                      \
-    auto *I = VBuild.createVectorInstruction(Instruction::OPCODE, ValueTy,     \
-                                             {Op, Op});                        \
-    ASSERT_TRUE(isa<VPIntrinsic>(I));                                          \
-    auto *VPIntrin = cast<VPIntrinsic>(I);                                     \
-    ASSERT_EQ(VPIntrin->getIntrinsicID(), VPID);                               \
-    ASSERT_EQ(VPIntrin->getMaskParam(), Mask);                                 \
-    ASSERT_TRUE(                                                               \
-        isLegalConstEVL(VPIntrin->getVectorLengthParam(), VectorNumElements)); \
-  }
-#include "llvm/IR/Instruction.def"
-}
-
-/// Check that creating binary arithmetic VP intrinsics works.
-TEST_F(VectorBuilderTest,
-       TestCreateBinaryInstructions_FixedVector_NoMask_NoEVL) {
-  Function *F;
-  BasicBlock *BB;
-  Value *Mask, *EVL;
-  auto Mod = createBuilderModule(F, BB, Mask, EVL);
-
-  IRBuilder<> Builder(BB);
-  VectorBuilder VBuild(Builder);
-  VBuild.setStaticVL(VectorNumElements);
-
-  auto *FloatVecTy =
-      FixedVectorType::get(Type::getFloatTy(Context), VectorNumElements);
-  auto *IntVecTy =
-      FixedVectorType::get(Type::getInt32Ty(Context), VectorNumElements);
-
-#define HANDLE_BINARY_INST(NUM, OPCODE, INSTCLASS)                             \
-  {                                                                            \
-    auto VPID = VPIntrinsic::getForOpcode(Instruction::OPCODE);                \
-    bool IsFP = (#INSTCLASS)[0] == 'F';                                        \
-    Type *ValueTy = IsFP ? FloatVecTy : IntVecTy;                              \
-    Value *Op = PoisonValue::get(ValueTy);                                      \
-    auto *I = VBuild.createVectorInstruction(Instruction::OPCODE, ValueTy,     \
-                                             {Op, Op});                        \
-    ASSERT_TRUE(isa<VPIntrinsic>(I));                                          \
-    auto *VPIntrin = cast<VPIntrinsic>(I);                                     \
-    ASSERT_EQ(VPIntrin->getIntrinsicID(), VPID);                               \
-    ASSERT_TRUE(isAllTrueMask(VPIntrin->getMaskParam(), VectorNumElements));   \
-    ASSERT_TRUE(                                                               \
-        isLegalConstEVL(VPIntrin->getVectorLengthParam(), VectorNumElements)); \
-  }
-#include "llvm/IR/Instruction.def"
-}
-/// Check that creating vp.load/vp.store works.
-TEST_F(VectorBuilderTest, TestCreateLoadStore) {
-  Function *F;
-  BasicBlock *BB;
-  Value *Mask, *EVL;
-  auto Mod = createBuilderModule(F, BB, Mask, EVL);
-
-  IRBuilder<> Builder(BB);
-  VectorBuilder VBuild(Builder);
-  VBuild.setMask(Mask).setEVL(EVL);
-
-  auto *FloatVecTy =
-      FixedVectorType::get(Type::getFloatTy(Context), VectorNumElements);
-
-  Value *FloatVecPtr = PoisonValue::get(Builder.getPtrTy(0));
-  Value *FloatVec = PoisonValue::get(FloatVecTy);
-
-  // vp.load
-  auto LoadVPID = VPIntrinsic::getForOpcode(Instruction::Load);
-  auto *LoadIntrin = VBuild.createVectorInstruction(Instruction::Load,
-                                                    FloatVecTy, {FloatVecPtr});
-  ASSERT_TRUE(isa<VPIntrinsic>(LoadIntrin));
-  auto *VPLoad = cast<VPIntrinsic>(LoadIntrin);
-  ASSERT_EQ(VPLoad->getIntrinsicID(), LoadVPID);
-  ASSERT_EQ(VPLoad->getMemoryPointerParam(), FloatVecPtr);
-
-  // vp.store
-  auto *VoidTy = Builder.getVoidTy();
-  auto StoreVPID = VPIntrinsic::getForOpcode(Instruction::Store);
-  auto *StoreIntrin = VBuild.createVectorInstruction(Instruction::Store, VoidTy,
-                                                     {FloatVec, FloatVecPtr});
-  ASSERT_TRUE(isa<VPIntrinsic>(LoadIntrin));
-  auto *VPStore = cast<VPIntrinsic>(StoreIntrin);
-  ASSERT_EQ(VPStore->getIntrinsicID(), StoreVPID);
-  ASSERT_EQ(VPStore->getMemoryPointerParam(), FloatVecPtr);
-  ASSERT_EQ(VPStore->getMemoryDataParam(), FloatVec);
-}
-
-/// Check that the SilentlyReturnNone error handling mode works.
-TEST_F(VectorBuilderTest, TestFail_SilentlyReturnNone) {
-  Function *F;
-  BasicBlock *BB;
-  Value *Mask, *EVL;
-  auto Mod = createBuilderModule(F, BB, Mask, EVL);
-
-  IRBuilder<> Builder(BB);
-  auto *VoidTy = Builder.getVoidTy();
-  VectorBuilder VBuild(Builder, VectorBuilder::Behavior::SilentlyReturnNone);
-  VBuild.setMask(Mask).setEVL(EVL);
-  auto *Val = VBuild.createVectorInstruction(Instruction::Br, VoidTy, {});
-  ASSERT_EQ(Val, nullptr);
-}
-
-/// Check that the ReportAndFail error handling mode aborts as advertised.
-TEST_F(VectorBuilderTest, TestFail_ReportAndAbort) {
-  Function *F;
-  BasicBlock *BB;
-  Value *Mask, *EVL;
-  auto Mod = createBuilderModule(F, BB, Mask, EVL);
-
-  IRBuilder<> Builder(BB);
-  auto *VoidTy = Builder.getVoidTy();
-  VectorBuilder VBuild(Builder, VectorBuilder::Behavior::ReportAndAbort);
-  VBuild.setMask(Mask).setEVL(EVL);
-  ASSERT_DEATH({ VBuild.createVectorInstruction(Instruction::Br, VoidTy, {}); },
-               "No VPIntrinsic for this opcode");
-}
-
-} // end anonymous namespace
diff --git a/external/llvm-project/llvm/unittests/Object/DXContainerTest.cpp b/external/llvm-project/llvm/unittests/Object/DXContainerTest.cpp
index 68c955300c6b..28012073d3c7 100644
--- a/external/llvm-project/llvm/unittests/Object/DXContainerTest.cpp
+++ b/external/llvm-project/llvm/unittests/Object/DXContainerTest.cpp
@@ -1190,12 +1190,12 @@ TEST(RootSignature, ParseStaticSamplers) {
     ASSERT_EQ(Sampler.AddressU, 1u);
     ASSERT_EQ(Sampler.AddressV, 2u);
     ASSERT_EQ(Sampler.AddressW, 5u);
-    ASSERT_FLOAT_EQ(Sampler.MipLODBias, 1.23);
+    ASSERT_FLOAT_EQ(Sampler.MipLODBias, 1.23f);
     ASSERT_EQ(Sampler.MaxAnisotropy, 20u);
     ASSERT_EQ(Sampler.ComparisonFunc, 4u);
     ASSERT_EQ(Sampler.BorderColor, 0u);
-    ASSERT_FLOAT_EQ(Sampler.MinLOD, 4.56);
-    ASSERT_FLOAT_EQ(Sampler.MaxLOD, 8.9);
+    ASSERT_FLOAT_EQ(Sampler.MinLOD, 4.56f);
+    ASSERT_FLOAT_EQ(Sampler.MaxLOD, 8.9f);
     ASSERT_EQ(Sampler.ShaderRegister, 31u);
     ASSERT_EQ(Sampler.RegisterSpace, 32u);
     ASSERT_EQ(Sampler.ShaderVisibility, 7u);
diff --git a/external/llvm-project/llvm/unittests/ProfileData/CoverageMappingTest.cpp b/external/llvm-project/llvm/unittests/ProfileData/CoverageMappingTest.cpp
index 46f881ecddb5..ec81e5f274ef 100644
--- a/external/llvm-project/llvm/unittests/ProfileData/CoverageMappingTest.cpp
+++ b/external/llvm-project/llvm/unittests/ProfileData/CoverageMappingTest.cpp
@@ -277,7 +277,9 @@ struct CoverageMappingTest : ::testing::TestWithParam<std::tuple<bool, bool>> {
       CoverageReaders.push_back(
           std::make_unique<CoverageMappingReaderMock>(Funcs));
     }
-    return CoverageMapping::load(CoverageReaders, *ProfileReader);
+    auto ProfileReaderRef = std::make_optional(
+        std::reference_wrapper<IndexedInstrProfReader>(*ProfileReader));
+    return CoverageMapping::load(CoverageReaders, ProfileReaderRef);
   }
 
   Error loadCoverageMapping(bool EmitFilenames = true) {
diff --git a/external/llvm-project/llvm/unittests/ProfileData/DataAccessProfTest.cpp b/external/llvm-project/llvm/unittests/ProfileData/DataAccessProfTest.cpp
index 13af3390557d..2f93e16f28c1 100644
--- a/external/llvm-project/llvm/unittests/ProfileData/DataAccessProfTest.cpp
+++ b/external/llvm-project/llvm/unittests/ProfileData/DataAccessProfTest.cpp
@@ -92,10 +92,10 @@ TEST(MemProf, DataAccessProfile) {
 
     EXPECT_THAT(
         Data.getProfileRecord("foo.llvm.123"),
-        ValueIs(AllOf(
-            Field(&DataAccessProfRecord::SymHandle,
-                  testing::VariantWith<std::string>(testing::Eq("foo"))),
-            Field(&DataAccessProfRecord::Locations, testing::IsEmpty()))));
+        ValueIs(
+            AllOf(Field(&DataAccessProfRecord::SymHandle,
+                        testing::VariantWith<std::string>(testing::Eq("foo"))),
+                  Field(&DataAccessProfRecord::Locations, IsEmpty()))));
     EXPECT_THAT(
         Data.getProfileRecord("bar.__uniq.321"),
         ValueIs(AllOf(
@@ -130,7 +130,7 @@ TEST(MemProf, DataAccessProfile) {
         reinterpret_cast<const unsigned char *>(serializedData.data());
     ASSERT_THAT(llvm::to_vector(llvm::make_first_range(
                     deserializedData.getStrToIndexMapRef())),
-                testing::IsEmpty());
+                IsEmpty());
     EXPECT_FALSE(deserializedData.deserialize(p));
 
     EXPECT_THAT(
@@ -153,11 +153,10 @@ TEST(MemProf, DataAccessProfile) {
     EXPECT_THAT(
         Records,
         ElementsAre(
-            AllOf(
-                Field(&DataAccessProfRecordRef::SymbolID, 0),
-                Field(&DataAccessProfRecordRef::AccessCount, 100),
-                Field(&DataAccessProfRecordRef::IsStringLiteral, false),
-                Field(&DataAccessProfRecordRef::Locations, testing::IsEmpty())),
+            AllOf(Field(&DataAccessProfRecordRef::SymbolID, 0),
+                  Field(&DataAccessProfRecordRef::AccessCount, 100),
+                  Field(&DataAccessProfRecordRef::IsStringLiteral, false),
+                  Field(&DataAccessProfRecordRef::Locations, IsEmpty())),
             AllOf(Field(&DataAccessProfRecordRef::SymbolID, 2),
                   Field(&DataAccessProfRecordRef::AccessCount, 123),
                   Field(&DataAccessProfRecordRef::IsStringLiteral, false),
diff --git a/external/llvm-project/llvm/unittests/ProfileData/MemProfTest.cpp b/external/llvm-project/llvm/unittests/ProfileData/MemProfTest.cpp
index 1dbafea19573..b57567e29f1c 100644
--- a/external/llvm-project/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/external/llvm-project/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -19,16 +19,18 @@
 #include "llvm/ProfileData/MemProfData.inc"
 #include "llvm/ProfileData/MemProfRadixTree.h"
 #include "llvm/ProfileData/MemProfReader.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 #include <initializer_list>
 
-extern llvm::cl::opt<float> MemProfLifetimeAccessDensityColdThreshold;
-extern llvm::cl::opt<unsigned> MemProfAveLifetimeColdThreshold;
-extern llvm::cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold;
-extern llvm::cl::opt<bool> MemProfUseHotHints;
+LLVM_ABI extern llvm::cl::opt<float> MemProfLifetimeAccessDensityColdThreshold;
+LLVM_ABI extern llvm::cl::opt<unsigned> MemProfAveLifetimeColdThreshold;
+LLVM_ABI extern llvm::cl::opt<unsigned>
+    MemProfMinAveLifetimeAccessDensityHotThreshold;
+LLVM_ABI extern llvm::cl::opt<bool> MemProfUseHotHints;
 
 namespace llvm {
 namespace memprof {
diff --git a/external/llvm-project/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/external/llvm-project/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 18882add5994..33928ac118e0 100644
--- a/external/llvm-project/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/external/llvm-project/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -1051,29 +1051,6 @@ define void @foo() {
   auto *Call = cast<sandboxir::CallInst>(&*It++);
   // Check classof(), creation.
   auto *GO = cast<sandboxir::GlobalObject>(Call->getCalledOperand());
-  // Check getAlignment().
-  EXPECT_EQ(GO->getAlignment(), LLVMGO->getAlignment());
-  // Check getAlign().
-  EXPECT_EQ(GO->getAlign(), LLVMGO->getAlign());
-  // Check setAlignment().
-  auto OrigMaybeAlign = GO->getAlign();
-  auto NewMaybeAlign = MaybeAlign(128);
-  EXPECT_NE(NewMaybeAlign, OrigMaybeAlign);
-  GO->setAlignment(NewMaybeAlign);
-  EXPECT_EQ(GO->getAlign(), NewMaybeAlign);
-  GO->setAlignment(OrigMaybeAlign);
-  EXPECT_EQ(GO->getAlign(), OrigMaybeAlign);
-  // Check getGlobalObjectSubClassData().
-  EXPECT_EQ(GO->getGlobalObjectSubClassData(),
-            LLVMGO->getGlobalObjectSubClassData());
-  // Check setGlobalObjectSubClassData().
-  auto OrigGOSCD = GO->getGlobalObjectSubClassData();
-  auto NewGOSCD = 1u;
-  EXPECT_NE(NewGOSCD, OrigGOSCD);
-  GO->setGlobalObjectSubClassData(NewGOSCD);
-  EXPECT_EQ(GO->getGlobalObjectSubClassData(), NewGOSCD);
-  GO->setGlobalObjectSubClassData(OrigGOSCD);
-  EXPECT_EQ(GO->getGlobalObjectSubClassData(), OrigGOSCD);
   // Check hasSection().
   EXPECT_EQ(GO->hasSection(), LLVMGO->hasSection());
   // Check getSection().
@@ -1284,6 +1261,16 @@ define void @foo() {
   EXPECT_EQ(GV0->getCodeModelRaw(), LLVMGV0->getCodeModelRaw());
   // Check getCodeModel().
   EXPECT_EQ(GV0->getCodeModel(), LLVMGV0->getCodeModel());
+  // Check getAlign().
+  EXPECT_EQ(GV0->getAlign(), LLVMGV0->getAlign());
+  // Check setAlignment().
+  auto OrigMaybeAlign = GV0->getAlign();
+  auto NewMaybeAlign = MaybeAlign(128);
+  EXPECT_NE(NewMaybeAlign, OrigMaybeAlign);
+  GV0->setAlignment(NewMaybeAlign);
+  EXPECT_EQ(GV0->getAlign(), NewMaybeAlign);
+  GV0->setAlignment(OrigMaybeAlign);
+  EXPECT_EQ(GV0->getAlign(), OrigMaybeAlign);
 }
 
 TEST_F(SandboxIRTest, GlobalAlias) {
@@ -1855,6 +1842,17 @@ void @foo0(i32 %arg0, i32 %arg1) {
 )IR");
   }
 #endif // NDEBUG
+
+  // Check getAlign().
+  EXPECT_EQ(F0->getAlign(), F0->getAlign());
+  // Check setAlignment().
+  auto OrigMaybeAlign = F0->getAlign();
+  auto NewMaybeAlign = MaybeAlign(128);
+  EXPECT_NE(NewMaybeAlign, OrigMaybeAlign);
+  F0->setAlignment(NewMaybeAlign);
+  EXPECT_EQ(F0->getAlign(), NewMaybeAlign);
+  F0->setAlignment(OrigMaybeAlign);
+  EXPECT_EQ(F0->getAlign(), OrigMaybeAlign);
 }
 
 TEST_F(SandboxIRTest, Module) {
diff --git a/external/llvm-project/llvm/unittests/Support/ARMAttributeParser.cpp b/external/llvm-project/llvm/unittests/Support/ARMAttributeParser.cpp
index e06f7fab85fa..305f4b3d1d0e 100644
--- a/external/llvm-project/llvm/unittests/Support/ARMAttributeParser.cpp
+++ b/external/llvm-project/llvm/unittests/Support/ARMAttributeParser.cpp
@@ -38,7 +38,7 @@ bool testBuildAttr(unsigned Tag, unsigned Value,
   cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
   std::optional<unsigned> Attr = Parser.getAttributeValue("", ExpectedTag);
-  return Attr && *Attr == ExpectedValue;
+  return Attr == ExpectedValue;
 }
 
 void testParseError(ArrayRef<uint8_t> bytes, const char *msg) {
diff --git a/external/llvm-project/llvm/unittests/TargetParser/TripleTest.cpp b/external/llvm-project/llvm/unittests/TargetParser/TripleTest.cpp
index bbd12e6d0e41..0f6d07657c93 100644
--- a/external/llvm-project/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/external/llvm-project/llvm/unittests/TargetParser/TripleTest.cpp
@@ -2354,6 +2354,79 @@ TEST(TripleTest, getOSVersion) {
   T.getMacOSXVersion(Version);
   EXPECT_EQ(VersionTuple(10, 16), Version);
 
+  // 16 forms a valid triple, even though it's not
+  // a version of a macOS.
+  T = Triple("x86_64-apple-macos16.0");
+  EXPECT_TRUE(T.isMacOSX());
+  T.getMacOSXVersion(Version);
+  EXPECT_EQ(VersionTuple(16, 0), Version);
+
+  T = Triple("arm64-apple-macosx26.1");
+  EXPECT_TRUE(T.isMacOSX());
+  EXPECT_FALSE(T.isiOS());
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
+  T.getMacOSXVersion(Version);
+  EXPECT_EQ(VersionTuple(26, 1), Version);
+
+  // 19.0 forms a valid triple, even though it's not
+  // a version of a iOS.
+  T = Triple("arm64-apple-ios19.0");
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_TRUE(T.isiOS());
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
+  T.getiOSVersion();
+  EXPECT_EQ(VersionTuple(19, 0), T.getOSVersion());
+
+  T = Triple("arm64-apple-ios27.0");
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_TRUE(T.isiOS());
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
+  EXPECT_EQ(VersionTuple(27, 0), T.getiOSVersion());
+
+  T = Triple("arm64-apple-watchos12.0");
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_TRUE(T.isWatchOS());
+  EXPECT_TRUE(T.isArch64Bit());
+  EXPECT_EQ(VersionTuple(26, 0), T.getiOSVersion());
+
+  T = Triple("arm64-apple-visionos3.0");
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_TRUE(T.isXROS());
+  EXPECT_TRUE(T.isArch64Bit());
+  EXPECT_EQ(VersionTuple(26, 0), T.getiOSVersion());
+
+  T = Triple("x86_64-apple-darwin26");
+  EXPECT_TRUE(T.isMacOSX());
+  T.getMacOSXVersion(Version);
+  EXPECT_EQ(VersionTuple(27), Version);
+
+  // Check invalid ranges are remapped.
+  T = Triple("arm64-apple-visionos6.0");
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_TRUE(T.isXROS());
+  EXPECT_TRUE(T.isArch64Bit());
+  EXPECT_EQ(VersionTuple(29, 0), T.getiOSVersion());
+
+  T = Triple("arm64-apple-watchos14.0");
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_TRUE(T.isWatchOS());
+  EXPECT_TRUE(T.isArch64Bit());
+  EXPECT_EQ(VersionTuple(28, 0), T.getiOSVersion());
+
+  T = Triple("arm64-apple-ios21.1");
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_TRUE(T.isiOS());
+  EXPECT_FALSE(T.isArch16Bit());
+  EXPECT_FALSE(T.isArch32Bit());
+  EXPECT_TRUE(T.isArch64Bit());
+  EXPECT_EQ(VersionTuple(28, 1), T.getiOSVersion());
+
   T = Triple("x86_64-apple-darwin20");
   EXPECT_TRUE(T.isMacOSX());
   T.getMacOSXVersion(Version);
@@ -2468,11 +2541,76 @@ TEST(TripleTest, isMacOSVersionLT) {
 TEST(TripleTest, CanonicalizeOSVersion) {
   EXPECT_EQ(VersionTuple(10, 15, 4),
             Triple::getCanonicalVersionForOS(Triple::MacOSX,
-                                             VersionTuple(10, 15, 4)));
-  EXPECT_EQ(VersionTuple(11, 0), Triple::getCanonicalVersionForOS(
-                                     Triple::MacOSX, VersionTuple(10, 16)));
+                                             VersionTuple(10, 15, 4),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(11, 0),
+            Triple::getCanonicalVersionForOS(
+                Triple::MacOSX, VersionTuple(10, 16), /*IsInValidRange=*/true));
   EXPECT_EQ(VersionTuple(20),
-            Triple::getCanonicalVersionForOS(Triple::Darwin, VersionTuple(20)));
+            Triple::getCanonicalVersionForOS(Triple::Darwin, VersionTuple(20),
+                                             /*IsInValidRange=*/true));
+
+  // Validate mappings for all expected OS versions remapping to 26.
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::MacOSX, VersionTuple(16),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::IOS, VersionTuple(19),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::TvOS, VersionTuple(19),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::WatchOS, VersionTuple(12),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::XROS, VersionTuple(3),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::Darwin, VersionTuple(26),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::BridgeOS, VersionTuple(26),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(26),
+            Triple::getCanonicalVersionForOS(Triple::BridgeOS, VersionTuple(26),
+                                             /*IsInValidRange=*/true));
+  EXPECT_EQ(VersionTuple(10),
+            Triple::getCanonicalVersionForOS(Triple::BridgeOS, VersionTuple(10),
+                                             /*IsInValidRange=*/true));
+
+  // Verify invalid ranges can be remapped.
+  EXPECT_EQ(VersionTuple(28),
+            Triple::getCanonicalVersionForOS(Triple::MacOSX, VersionTuple(18),
+                                             /*IsInValidRange=*/false));
+  EXPECT_EQ(VersionTuple(28),
+            Triple::getCanonicalVersionForOS(Triple::IOS, VersionTuple(21),
+                                             /*IsInValidRange=*/false));
+  EXPECT_EQ(VersionTuple(28),
+            Triple::getCanonicalVersionForOS(Triple::WatchOS, VersionTuple(14),
+                                             /*IsInValidRange=*/false));
+  EXPECT_EQ(VersionTuple(28),
+            Triple::getCanonicalVersionForOS(Triple::TvOS, VersionTuple(21),
+                                             /*IsInValidRange=*/false));
+}
+
+TEST(TripleTest, CheckValidOSVersion) {
+  EXPECT_TRUE(Triple::isValidVersionForOS(Triple::MacOSX, VersionTuple(16)));
+  EXPECT_TRUE(Triple::isValidVersionForOS(Triple::IOS, VersionTuple(19)));
+  EXPECT_TRUE(Triple::isValidVersionForOS(Triple::TvOS, VersionTuple(19)));
+  EXPECT_TRUE(Triple::isValidVersionForOS(Triple::TvOS, VersionTuple(18, 2)));
+  EXPECT_TRUE(Triple::isValidVersionForOS(Triple::WatchOS, VersionTuple(12)));
+  EXPECT_TRUE(Triple::isValidVersionForOS(Triple::XROS, VersionTuple(3)));
+  EXPECT_TRUE(Triple::isValidVersionForOS(Triple::Darwin, VersionTuple(26)));
+  EXPECT_TRUE(
+      Triple::isValidVersionForOS(Triple::BridgeOS, VersionTuple(26, 1, 1)));
+
+  EXPECT_FALSE(
+      Triple::isValidVersionForOS(Triple::MacOSX, VersionTuple(16, 1)));
+  EXPECT_FALSE(Triple::isValidVersionForOS(Triple::MacOSX, VersionTuple(18)));
+  EXPECT_FALSE(Triple::isValidVersionForOS(Triple::IOS, VersionTuple(21)));
+  EXPECT_FALSE(Triple::isValidVersionForOS(Triple::WatchOS, VersionTuple(14)));
+  EXPECT_FALSE(Triple::isValidVersionForOS(Triple::TvOS, VersionTuple(21)));
 }
 
 TEST(TripleTest, FileFormat) {
diff --git a/external/llvm-project/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp b/external/llvm-project/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp
index 1602826b7252..fc41b036ffb6 100644
--- a/external/llvm-project/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp
+++ b/external/llvm-project/llvm/unittests/Transforms/IPO/LowerTypeTests.cpp
@@ -30,10 +30,10 @@ TEST(LowerTypeTests, BitSetBuilder) {
       {{0, 4}, {0, 1}, 0, 2, 2, false, true},
       {{0, uint64_t(1) << 33}, {0, 1}, 0, 2, 33, false, true},
       {{3, 7}, {0, 1}, 3, 2, 2, false, true},
-      {{0, 1, 7}, {0, 1, 7}, 0, 8, 0, false, false},
-      {{0, 2, 14}, {0, 1, 7}, 0, 8, 1, false, false},
-      {{0, 1, 8}, {0, 1, 8}, 0, 9, 0, false, false},
-      {{0, 2, 16}, {0, 1, 8}, 0, 9, 1, false, false},
+      {{0, 1, 7}, {0, 6, 7}, 0, 8, 0, false, false},
+      {{0, 2, 14}, {0, 6, 7}, 0, 8, 1, false, false},
+      {{0, 1, 8}, {0, 7, 8}, 0, 9, 0, false, false},
+      {{0, 2, 16}, {0, 7, 8}, 0, 9, 1, false, false},
       {{0, 1, 2, 3, 4, 5, 6, 7},
        {0, 1, 2, 3, 4, 5, 6, 7},
        0,
diff --git a/external/llvm-project/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp b/external/llvm-project/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
index c4f38863e6f0..0dd819a10e0b 100644
--- a/external/llvm-project/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
+++ b/external/llvm-project/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Transforms/Instrumentation/MemProfUse.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/LLVMContext.h"
@@ -17,7 +18,6 @@
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Testing/Support/Error.h"
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
 
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
diff --git a/external/llvm-project/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/external/llvm-project/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
index 5c062df8112f..1751701967e6 100644
--- a/external/llvm-project/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
+++ b/external/llvm-project/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -20,6 +20,7 @@
 #include "llvm/SandboxIR/Function.h"
 #include "llvm/SandboxIR/Type.h"
 #include "llvm/Support/SourceMgr.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -563,3 +564,52 @@ TEST_F(VecUtilsTest, FloorPowerOf2) {
   EXPECT_EQ(sandboxir::VecUtils::getFloorPowerOf2(8), 8u);
   EXPECT_EQ(sandboxir::VecUtils::getFloorPowerOf2(9), 8u);
 }
+
+TEST_F(VecUtilsTest, MatchPackScalar) {
+  parseIR(R"IR(
+define void @foo(i8 %v0, i8 %v1) {
+bb0:
+  %NotPack = insertelement <2 x i8> poison, i8 %v0, i64 0
+  br label %bb1
+
+bb1:
+  %Pack0 = insertelement <2 x i8> poison, i8 %v0, i64 0
+  %Pack1 = insertelement <2 x i8> %Pack0, i8 %v1, i64 1
+
+  %NotPack0 = insertelement <2 x i8> poison, i8 %v0, i64 0
+  %NotPack1 = insertelement <2 x i8> %NotPack0, i8 %v1, i64 0
+  %NotPack2 = insertelement <2 x i8> %NotPack1, i8 %v1, i64 1
+
+  %NotPackBB = insertelement <2 x i8> %NotPack, i8 %v1, i64 1
+
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto &BB = getBasicBlockByName(F, "bb1");
+  auto It = BB.begin();
+  auto *Pack0 = cast<sandboxir::InsertElementInst>(&*It++);
+  auto *Pack1 = cast<sandboxir::InsertElementInst>(&*It++);
+  auto *NotPack0 = cast<sandboxir::InsertElementInst>(&*It++);
+  auto *NotPack1 = cast<sandboxir::InsertElementInst>(&*It++);
+  auto *NotPack2 = cast<sandboxir::InsertElementInst>(&*It++);
+  auto *NotPackBB = cast<sandboxir::InsertElementInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+  auto *Arg0 = F.getArg(0);
+  auto *Arg1 = F.getArg(1);
+  EXPECT_FALSE(sandboxir::VecUtils::matchPack(Pack0));
+  EXPECT_FALSE(sandboxir::VecUtils::matchPack(Ret));
+  {
+    auto PackOpt = sandboxir::VecUtils::matchPack(Pack1);
+    EXPECT_TRUE(PackOpt);
+    EXPECT_THAT(PackOpt->Instrs, testing::ElementsAre(Pack1, Pack0));
+    EXPECT_THAT(PackOpt->Operands, testing::ElementsAre(Arg0, Arg1));
+  }
+  {
+    for (auto *NotPack : {NotPack0, NotPack1, NotPack2, NotPackBB})
+      EXPECT_FALSE(sandboxir::VecUtils::matchPack(NotPack));
+  }
+}
diff --git a/external/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index 9792eb41ea5d..b6d9c9f3a158 100644
--- a/external/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -388,7 +388,7 @@ struct MatchableInfo {
     StringRef Token;
 
     /// The unique class instance this operand should match.
-    ClassInfo *Class;
+    ClassInfo *Class = nullptr;
 
     /// The operand name this is, if anything.
     StringRef SrcOpName;
@@ -397,18 +397,17 @@ struct MatchableInfo {
     StringRef OrigSrcOpName;
 
     /// The suboperand index within SrcOpName, or -1 for the entire operand.
-    int SubOpIdx;
+    int SubOpIdx = -1;
 
     /// Whether the token is "isolated", i.e., it is preceded and followed
     /// by separators.
     bool IsIsolatedToken;
 
     /// Register record if this token is singleton register.
-    const Record *SingletonReg;
+    const Record *SingletonReg = nullptr;
 
     explicit AsmOperand(bool IsIsolatedToken, StringRef T)
-        : Token(T), Class(nullptr), SubOpIdx(-1),
-          IsIsolatedToken(IsIsolatedToken), SingletonReg(nullptr) {}
+        : Token(T), IsIsolatedToken(IsIsolatedToken) {}
   };
 
   /// ResOperand - This represents a single operand in the result instruction
@@ -1330,8 +1329,7 @@ void AsmMatcherInfo::buildRegisterClasses(
   for (const RegisterSet &RS : RegisterSets) {
     ClassInfo *CI = RegisterSetClasses[RS];
     for (const RegisterSet &RS2 : RegisterSets)
-      if (RS != RS2 && std::includes(RS2.begin(), RS2.end(), RS.begin(),
-                                     RS.end(), LessRecordByID()))
+      if (RS != RS2 && llvm::includes(RS2, RS, LessRecordByID()))
         CI->SuperClasses.push_back(RegisterSetClasses[RS2]);
   }
 
diff --git a/external/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp
index ebf1894b0d21..a3ea0f5e32e9 100644
--- a/external/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -1138,10 +1138,10 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
       uint32_t UnescapedSize = 0;
       std::string EncodedAsmString = IAP->formatAliasString(UnescapedSize);
       auto Insertion =
-          AsmStringOffsets.insert({EncodedAsmString, AsmStringsSize});
+          AsmStringOffsets.try_emplace(EncodedAsmString, AsmStringsSize);
       if (Insertion.second) {
         // If the string is new, add it to the vector.
-        AsmStrings.push_back({AsmStringsSize, EncodedAsmString});
+        AsmStrings.emplace_back(AsmStringsSize, EncodedAsmString);
         AsmStringsSize += UnescapedSize + 1;
       }
       unsigned AsmStrOffset = Insertion.first->second;
diff --git a/external/llvm-project/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
index 3d1795c5a6ff..177eecebce9a 100644
--- a/external/llvm-project/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Basic/DirectiveEmitter.cpp
@@ -24,6 +24,7 @@
 #include "llvm/TableGen/TableGenBackend.h"
 
 #include <numeric>
+#include <string>
 #include <vector>
 
 using namespace llvm;
@@ -76,6 +77,19 @@ static std::string getIdentifierName(const Record *Rec, StringRef Prefix) {
   return Prefix.str() + BaseRecord(Rec).getFormattedName();
 }
 
+using RecordWithSpelling = std::pair<const Record *, Spelling::Value>;
+
+static std::vector<RecordWithSpelling>
+getSpellings(ArrayRef<const Record *> Records) {
+  std::vector<RecordWithSpelling> List;
+  for (const Record *R : Records) {
+    BaseRecord Rec(R);
+    llvm::transform(Rec.getSpellings(), std::back_inserter(List),
+                    [R](Spelling::Value V) { return std::make_pair(R, V); });
+  }
+  return List;
+}
+
 static void generateEnumExports(ArrayRef<const Record *> Records,
                                 raw_ostream &OS, StringRef Enum,
                                 StringRef Prefix) {
@@ -147,7 +161,7 @@ static void generateEnumBitmask(ArrayRef<const Record *> Records,
 
 // Generate enums for values that clauses can take.
 // Also generate function declarations for get<Enum>Name(StringRef Str).
-static void generateEnumClauseVal(ArrayRef<const Record *> Records,
+static void generateClauseEnumVal(ArrayRef<const Record *> Records,
                                   raw_ostream &OS,
                                   const DirectiveLanguage &DirLang,
                                   std::string &EnumHelperFuncs) {
@@ -166,8 +180,8 @@ static void generateEnumClauseVal(ArrayRef<const Record *> Records,
 
     OS << "\n";
     OS << "enum class " << Enum << " {\n";
-    for (const ClauseVal CVal : ClauseVals)
-      OS << "  " << CVal.getRecordName() << "=" << CVal.getValue() << ",\n";
+    for (const EnumVal Val : ClauseVals)
+      OS << "  " << Val.getRecordName() << "=" << Val.getValue() << ",\n";
     OS << "};\n";
 
     if (DirLang.hasMakeEnumAvailableInNamespace()) {
@@ -193,10 +207,11 @@ static bool hasDuplicateClauses(ArrayRef<const Record *> Clauses,
                                 StringSet<> &CrtClauses) {
   bool HasError = false;
   for (const VersionedClause VerClause : Clauses) {
-    const auto InsRes = CrtClauses.insert(VerClause.getClause().getName());
+    StringRef Name = VerClause.getClause().getRecordName();
+    const auto InsRes = CrtClauses.insert(Name);
     if (!InsRes.second) {
-      PrintError("Clause " + VerClause.getClause().getRecordName() +
-                 " already defined on directive " + Directive.getRecordName());
+      PrintError("Clause " + Name + " already defined on directive " +
+                 Directive.getRecordName());
       HasError = true;
     }
   }
@@ -267,11 +282,13 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   if (DirLang.hasEnableBitmaskEnumInNamespace())
     OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
 
+  OS << "#include \"llvm/ADT/StringRef.h\"\n";
+  OS << "#include \"llvm/Frontend/Directive/Spelling.h\"\n";
   OS << "#include \"llvm/Support/Compiler.h\"\n";
   OS << "#include <cstddef>\n"; // for size_t
+  OS << "#include <utility>\n"; // for std::pair
   OS << "\n";
   OS << "namespace llvm {\n";
-  OS << "class StringRef;\n";
 
   // Open namespaces defined in the directive language
   SmallVector<StringRef, 2> Namespaces;
@@ -306,29 +323,39 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
                     DirLang.getClausePrefix(),
                     DirLang.hasMakeEnumAvailableInNamespace());
 
-  // Emit ClauseVal enumeration
+  // Emit ClauseVals enumeration
   std::string EnumHelperFuncs;
-  generateEnumClauseVal(DirLang.getClauses(), OS, DirLang, EnumHelperFuncs);
+  generateClauseEnumVal(DirLang.getClauses(), OS, DirLang, EnumHelperFuncs);
 
   // Generic function signatures
   OS << "\n";
   OS << "// Enumeration helper functions\n";
-  OS << "LLVM_ABI Directive get" << Lang << "DirectiveKind(StringRef Str);\n";
+
+  OS << "LLVM_ABI std::pair<Directive, directive::VersionRange> get" << Lang
+     << "DirectiveKindAndVersions(StringRef Str);\n";
+
+  OS << "inline Directive get" << Lang << "DirectiveKind(StringRef Str) {\n";
+  OS << "  return get" << Lang << "DirectiveKindAndVersions(Str).first;\n";
+  OS << "}\n";
   OS << "\n";
 
-  // For OpenMP the signature is
-  //   getOpenMPDirectiveName(Directive D, unsigned V)
-  OS << "LLVM_ABI StringRef get" << DirLang.getName()
-     << "DirectiveName(Directive D";
-  if (DirLang.getCppNamespace() == "omp")
-    OS << ", unsigned = 0";
-  OS << ");\n";
+  OS << "LLVM_ABI StringRef get" << Lang
+     << "DirectiveName(Directive D, unsigned Ver = 0);\n";
   OS << "\n";
 
-  OS << "LLVM_ABI Clause get" << Lang << "ClauseKind(StringRef Str);\n";
+  OS << "LLVM_ABI std::pair<Clause, directive::VersionRange> get" << Lang
+     << "ClauseKindAndVersions(StringRef Str);\n";
   OS << "\n";
-  OS << "LLVM_ABI StringRef get" << Lang << "ClauseName(Clause C);\n";
+
+  OS << "inline Clause get" << Lang << "ClauseKind(StringRef Str) {\n";
+  OS << "  return get" << Lang << "ClauseKindAndVersions(Str).first;\n";
+  OS << "}\n";
+  OS << "\n";
+
+  OS << "LLVM_ABI StringRef get" << Lang
+     << "ClauseName(Clause C, unsigned Ver = 0);\n";
   OS << "\n";
+
   OS << "/// Return true if \\p C is a valid clause for \\p D in version \\p "
      << "Version.\n";
   OS << "LLVM_ABI bool isAllowedClauseForDirective(Directive D, "
@@ -353,31 +380,60 @@ static void emitDirectivesDecl(const RecordKeeper &Records, raw_ostream &OS) {
   OS << "#endif // LLVM_" << Lang << "_INC\n";
 }
 
+// Given a list of spellings (for a given clause/directive), order them
+// in a way that allows the use of binary search to locate a spelling
+// for a specified version.
+static std::vector<Spelling::Value>
+orderSpellings(ArrayRef<Spelling::Value> Spellings) {
+  std::vector<Spelling::Value> List(Spellings.begin(), Spellings.end());
+
+  llvm::stable_sort(List,
+                    [](const Spelling::Value &A, const Spelling::Value &B) {
+                      return A.Versions < B.Versions;
+                    });
+  return List;
+}
+
 // Generate function implementation for get<Enum>Name(StringRef Str)
 static void generateGetName(ArrayRef<const Record *> Records, raw_ostream &OS,
                             StringRef Enum, const DirectiveLanguage &DirLang,
                             StringRef Prefix) {
   StringRef Lang = DirLang.getName();
   std::string Qual = getQualifier(DirLang);
-  // For OpenMP the "Directive" signature is
-  //   getOpenMPDirectiveName(Directive D, unsigned V)
   OS << "\n";
   OS << "llvm::StringRef " << Qual << "get" << Lang << Enum << "Name(" << Qual
-     << Enum << " Kind";
-  if (DirLang.getCppNamespace() == "omp" && Enum == "Directive")
-    OS << ", unsigned";
-  OS << ") {\n";
+     << Enum << " Kind, unsigned Version) {\n";
   OS << "  switch (Kind) {\n";
   for (const Record *R : Records) {
-    OS << "    case " << getIdentifierName(R, Prefix) << ":\n";
-    OS << "      return \"" << BaseRecord(R).getName() << "\";\n";
+    BaseRecord Rec(R);
+    std::string Ident = getIdentifierName(R, Prefix);
+    OS << "    case " << Ident << ":";
+    std::vector<Spelling::Value> Spellings(orderSpellings(Rec.getSpellings()));
+    assert(Spellings.size() != 0 && "No spellings for this item");
+    if (Spellings.size() == 1) {
+      OS << "\n";
+      OS << "      return \"" << Spellings.front().Name << "\";\n";
+    } else {
+      OS << " {\n";
+      std::string SpellingsName = Ident + "_spellings";
+      OS << "      static constexpr llvm::directive::Spelling " << SpellingsName
+         << "[] = {\n";
+      for (auto &S : Spellings) {
+        OS << "          {\"" << S.Name << "\", {" << S.Versions.Min << ", "
+           << S.Versions.Max << "}},\n";
+      }
+      OS << "      };\n";
+      OS << "      return llvm::directive::FindName(" << SpellingsName
+         << ", Version);\n";
+      OS << "    }\n";
+    }
   }
   OS << "  }\n"; // switch
   OS << "  llvm_unreachable(\"Invalid " << Lang << " " << Enum << " kind\");\n";
   OS << "}\n";
 }
 
-// Generate function implementation for get<Enum>Kind(StringRef Str)
+// Generate function implementation for get<Enum>KindAndVersions(StringRef Str)
 static void generateGetKind(ArrayRef<const Record *> Records, raw_ostream &OS,
                             StringRef Enum, const DirectiveLanguage &DirLang,
                             StringRef Prefix, bool ImplicitAsUnknown) {
@@ -394,27 +450,42 @@ static void generateGetKind(ArrayRef<const Record *> Records, raw_ostream &OS,
   std::string Qual = getQualifier(DirLang);
   std::string DefaultName = getIdentifierName(*DefaultIt, Prefix);
 
+  // std::pair<<Enum>, VersionRange>
+  // get<DirLang><Enum>KindAndVersions(StringRef Str);
   OS << "\n";
-  OS << Qual << Enum << " " << Qual << "get" << DirLang.getName() << Enum
-     << "Kind(llvm::StringRef Str) {\n";
-  OS << "  return StringSwitch<" << Enum << ">(Str)\n";
+  OS << "std::pair<" << Qual << Enum << ", llvm::directive::VersionRange> "
+     << Qual << "get" << DirLang.getName() << Enum
+     << "KindAndVersions(llvm::StringRef Str) {\n";
+  OS << "  directive::VersionRange All; // Default-initialized to \"all "
+        "versions\"\n";
+  OS << "  return StringSwitch<std::pair<" << Enum << ", "
+     << "directive::VersionRange>>(Str)\n";
+
+  directive::VersionRange All;
 
   for (const Record *R : Records) {
     BaseRecord Rec(R);
-    if (ImplicitAsUnknown && R->getValueAsBit("isImplicit")) {
-      OS << "    .Case(\"" << Rec.getName() << "\"," << DefaultName << ")\n";
-    } else {
-      OS << "    .Case(\"" << Rec.getName() << "\","
-         << getIdentifierName(R, Prefix) << ")\n";
+    std::string Ident = ImplicitAsUnknown && R->getValueAsBit("isImplicit")
+                            ? DefaultName
+                            : getIdentifierName(R, Prefix);
+
+    for (auto &[Name, Versions] : Rec.getSpellings()) {
+      OS << "    .Case(\"" << Name << "\", {" << Ident << ", ";
+      if (Versions.Min == All.Min && Versions.Max == All.Max)
+        OS << "All})\n";
+      else
+        OS << "{" << Versions.Min << ", " << Versions.Max << "}})\n";
     }
   }
-  OS << "    .Default(" << DefaultName << ");\n";
+  OS << "    .Default({" << DefaultName << ", All});\n";
   OS << "}\n";
 }
 
-// Generate function implementation for get<ClauseVal>Kind(StringRef Str)
-static void generateGetKindClauseVal(const DirectiveLanguage &DirLang,
-                                     raw_ostream &OS) {
+// Generate function implementations for
+//   <enumClauseValue> get<enumClauseValue>(StringRef Str) and
+//   StringRef get<enumClauseValue>Name(<enumClauseValue>)
+static void generateGetClauseVal(const DirectiveLanguage &DirLang,
+                                 raw_ostream &OS) {
   StringRef Lang = DirLang.getName();
   std::string Qual = getQualifier(DirLang);
 
@@ -428,7 +499,7 @@ static void generateGetKindClauseVal(const DirectiveLanguage &DirLang,
     });
 
     if (DefaultIt == ClauseVals.end()) {
-      PrintError("At least one val in Clause " + C.getFormattedName() +
+      PrintError("At least one val in Clause " + C.getRecordName() +
                  " must be defined as default.");
       return;
     }
@@ -436,8 +507,8 @@ static void generateGetKindClauseVal(const DirectiveLanguage &DirLang,
 
     StringRef Enum = C.getEnumName();
     if (Enum.empty()) {
-      PrintError("enumClauseValue field not set in Clause" +
-                 C.getFormattedName() + ".");
+      PrintError("enumClauseValue field not set in Clause" + C.getRecordName() +
+                 ".");
       return;
     }
 
@@ -445,10 +516,9 @@ static void generateGetKindClauseVal(const DirectiveLanguage &DirLang,
     OS << Qual << Enum << " " << Qual << "get" << Enum
        << "(llvm::StringRef Str) {\n";
     OS << "  return StringSwitch<" << Enum << ">(Str)\n";
-    for (const auto &CV : ClauseVals) {
-      ClauseVal CVal(CV);
-      OS << "    .Case(\"" << CVal.getFormattedName() << "\"," << CV->getName()
-         << ")\n";
+    for (const EnumVal Val : ClauseVals) {
+      OS << "    .Case(\"" << Val.getFormattedName() << "\","
+         << Val.getRecordName() << ")\n";
     }
     OS << "    .Default(" << DefaultName << ");\n";
     OS << "}\n";
@@ -457,10 +527,9 @@ static void generateGetKindClauseVal(const DirectiveLanguage &DirLang,
     OS << "llvm::StringRef " << Qual << "get" << Lang << Enum << "Name(" << Qual
        << Enum << " x) {\n";
     OS << "  switch (x) {\n";
-    for (const auto &CV : ClauseVals) {
-      ClauseVal CVal(CV);
-      OS << "    case " << CV->getName() << ":\n";
-      OS << "      return \"" << CVal.getFormattedName() << "\";\n";
+    for (const EnumVal Val : ClauseVals) {
+      OS << "    case " << Val.getRecordName() << ":\n";
+      OS << "      return \"" << Val.getFormattedName() << "\";\n";
     }
     OS << "  }\n"; // switch
     OS << "  llvm_unreachable(\"Invalid " << Lang << " " << Enum
@@ -599,7 +668,10 @@ static void emitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
   // (such as "end declare target").
   DenseSet<int> EndDirectives;
   for (auto [Rec, Id] : DirId) {
-    if (Directive(Rec).getName().starts_with_insensitive("end "))
+    // FIXME: This will need to recognize different spellings for different
+    // versions.
+    StringRef Name = Directive(Rec).getSpellingForIdentifier();
+    if (Name.starts_with_insensitive("end "))
       EndDirectives.insert(Id);
   }
 
@@ -608,12 +680,13 @@ static void emitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
   std::vector<int> Ordering(Directives.size());
   std::iota(Ordering.begin(), Ordering.end(), 0);
 
-  sort(Ordering, [&](int A, int B) {
+  llvm::sort(Ordering, [&](int A, int B) {
     auto &LeavesA = LeafTable[A];
     auto &LeavesB = LeafTable[B];
     int DirA = LeavesA[0], DirB = LeavesB[0];
     // First of all, end directives compare greater than non-end directives.
-    int IsEndA = EndDirectives.count(DirA), IsEndB = EndDirectives.count(DirB);
+    bool IsEndA = EndDirectives.contains(DirA);
+    bool IsEndB = EndDirectives.contains(DirB);
     if (IsEndA != IsEndB)
       return IsEndA < IsEndB;
     if (LeavesA[1] == 0 && LeavesB[1] == 0)
@@ -653,7 +726,7 @@ static void emitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
 
   // Emit a marker where the first "end directive" is.
   auto FirstE = find_if(Ordering, [&](int RowIdx) {
-    return EndDirectives.count(LeafTable[RowIdx][0]);
+    return EndDirectives.contains(LeafTable[RowIdx][0]);
   });
   OS << "[[maybe_unused]] static auto " << TableName
      << "EndDirective = " << TableName << " + "
@@ -710,7 +783,7 @@ static void generateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
   };
 
   auto ErrorPrefixFor = [&](Directive D) -> std::string {
-    return (Twine("Directive '") + D.getName() + "' in namespace '" +
+    return (Twine("Directive '") + D.getRecordName() + "' in namespace '" +
             DirLang.getCppNamespace() + "' ")
         .str();
   };
@@ -752,7 +825,6 @@ static void generateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
     // Compute the association from leaf constructs.
     std::vector<const Record *> Leaves = D.getLeafConstructs();
     if (Leaves.empty()) {
-      errs() << D.getName() << '\n';
       PrintFatalError(ErrorPrefixFor(D) +
                       "requests association to be computed from leaves, "
                       "but it has no leaves");
@@ -839,7 +911,7 @@ static void generateGetDirectiveLanguages(const DirectiveLanguage &DirLang,
         D.getSourceLanguages(), OS,
         [&](const Record *L) {
           StringRef N = L->getValueAsString("name");
-          OS << "SourceLanguage::" << BaseRecord::formatName(N);
+          OS << "SourceLanguage::" << BaseRecord::getSnakeName(N);
         },
         " | ");
     OS << ";\n";
@@ -899,7 +971,7 @@ static void generateDirectiveClauseSets(const DirectiveLanguage &DirLang,
 
   for (const Directive Dir : DirLang.getDirectives()) {
     OS << "\n";
-    OS << "// Sets for " << Dir.getName() << "\n";
+    OS << "// Sets for " << Dir.getSpellingForIdentifier() << "\n";
 
     generateClauseSet(Dir.getAllowedClauses(), OS, "allowedClauses_", Dir,
                       DirLang, FE);
@@ -1034,8 +1106,11 @@ static void generateFlangClauseUnparse(const DirectiveLanguage &DirLang,
   for (const Clause Clause : DirLang.getClauses()) {
     if (Clause.skipFlangUnparser())
       continue;
+    // The unparser doesn't know the effective version, so just pick some
+    // spelling.
+    StringRef SomeSpelling = Clause.getSpellingForIdentifier();
     std::string Parser = Clause.getFormattedParserClassName();
-    std::string Upper = Clause.getName().upper();
+    std::string Upper = SomeSpelling.upper();
 
     if (!Clause.getFlangClass().empty()) {
       if (Clause.isValueOptional() && Clause.getDefaultValue().empty()) {
@@ -1113,59 +1188,45 @@ static void generateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
      << " Parser clause\");\n";
 }
 
-static bool compareClauseName(const Record *R1, const Record *R2) {
-  Clause C1(R1);
-  Clause C2(R2);
-  return (C1.getName() > C2.getName());
-}
-
 // Generate the parser for the clauses.
 static void generateFlangClausesParser(const DirectiveLanguage &DirLang,
                                        raw_ostream &OS) {
   std::vector<const Record *> Clauses = DirLang.getClauses();
-  // Sort clauses in reverse alphabetical order so with clauses with same
-  // beginning, the longer option is tried before.
-  sort(Clauses, compareClauseName);
+  // Sort clauses in the reverse alphabetical order with respect to their
+  // names and aliases, so that longer names are tried before shorter ones.
+  std::vector<RecordWithSpelling> Names = getSpellings(Clauses);
+  llvm::sort(Names, [](const auto &A, const auto &B) {
+    return A.second.Name > B.second.Name;
+  });
   IfDefScope Scope("GEN_FLANG_CLAUSES_PARSER", OS);
   StringRef Base = DirLang.getFlangClauseBaseClass();
 
+  unsigned LastIndex = Names.size() - 1;
   OS << "\n";
-  unsigned Index = 0;
-  unsigned LastClauseIndex = Clauses.size() - 1;
   OS << "TYPE_PARSER(\n";
-  for (const Clause Clause : Clauses) {
-    const std::vector<StringRef> &Aliases = Clause.getAliases();
-    if (Aliases.empty()) {
-      OS << "  \"" << Clause.getName() << "\"";
-    } else {
-      OS << "  ("
-         << "\"" << Clause.getName() << "\"_tok";
-      for (StringRef Alias : Aliases) {
-        OS << " || \"" << Alias << "\"_tok";
-      }
-      OS << ")";
-    }
+  for (auto [Index, RecSp] : llvm::enumerate(Names)) {
+    auto [R, S] = RecSp;
+    Clause C(R);
 
-    StringRef FlangClass = Clause.getFlangClass();
-    OS << " >> construct<" << Base << ">(construct<" << Base
-       << "::" << Clause.getFormattedParserClassName() << ">(";
+    StringRef FlangClass = C.getFlangClass();
+    OS << "  \"" << S.Name << "\" >> construct<" << Base << ">(construct<"
+       << Base << "::" << C.getFormattedParserClassName() << ">(";
     if (FlangClass.empty()) {
       OS << "))";
-      if (Index != LastClauseIndex)
+      if (Index != LastIndex)
         OS << " ||";
       OS << "\n";
-      ++Index;
       continue;
     }
 
-    if (Clause.isValueOptional())
+    if (C.isValueOptional())
       OS << "maybe(";
     OS << "parenthesized(";
-    if (Clause.isValueList())
+    if (C.isValueList())
       OS << "nonemptyList(";
 
-    if (!Clause.getPrefix().empty())
-      OS << "\"" << Clause.getPrefix() << ":\" >> ";
+    if (!C.getPrefix().empty())
+      OS << "\"" << C.getPrefix() << ":\" >> ";
 
     // The common Flang parser are used directly. Their name is identical to
     // the Flang class with first letter as lowercase. If the Flang class is
@@ -1181,19 +1242,18 @@ static void generateFlangClausesParser(const DirectiveLanguage &DirLang,
             .Case("ScalarLogicalExpr", "scalarLogicalExpr")
             .Default(("Parser<" + FlangClass + ">{}").toStringRef(Scratch));
     OS << Parser;
-    if (!Clause.getPrefix().empty() && Clause.isPrefixOptional())
+    if (!C.getPrefix().empty() && C.isPrefixOptional())
       OS << " || " << Parser;
-    if (Clause.isValueList()) // close nonemptyList(.
+    if (C.isValueList()) // close nonemptyList(.
       OS << ")";
     OS << ")"; // close parenthesized(.
 
-    if (Clause.isValueOptional()) // close maybe(.
+    if (C.isValueOptional()) // close maybe(.
       OS << ")";
     OS << "))";
-    if (Index != LastClauseIndex)
+    if (Index != LastIndex)
       OS << " ||";
     OS << "\n";
-    ++Index;
   }
   OS << ")\n";
 }
@@ -1302,7 +1362,10 @@ void emitDirectivesBasicImpl(const DirectiveLanguage &DirLang,
   StringRef DPrefix = DirLang.getDirectivePrefix();
   StringRef CPrefix = DirLang.getClausePrefix();
 
-  OS << "\n#include \"llvm/Support/ErrorHandling.h\"\n";
+  OS << "\n";
+  OS << "#include \"llvm/Frontend/Directive/Spelling.h\"\n";
+  OS << "#include \"llvm/Support/ErrorHandling.h\"\n";
+  OS << "#include <utility>\n";
 
   // getDirectiveKind(StringRef Str)
   generateGetKind(DirLang.getDirectives(), OS, "Directive", DirLang, DPrefix,
@@ -1318,8 +1381,9 @@ void emitDirectivesBasicImpl(const DirectiveLanguage &DirLang,
   // getClauseName(Clause Kind)
   generateGetName(DirLang.getClauses(), OS, "Clause", DirLang, CPrefix);
 
-  // get<ClauseVal>Kind(StringRef Str)
-  generateGetKindClauseVal(DirLang, OS);
+  // <enumClauseValue> get<enumClauseValue>(StringRef Str) ; string -> value
+  // StringRef get<enumClauseValue>Name(<enumClauseValue>) ; value -> string
+  generateGetClauseVal(DirLang, OS);
 
   // isAllowedClauseForDirective(Directive D, Clause C, unsigned Version)
   generateIsAllowedClause(DirLang, OS);
diff --git a/external/llvm-project/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
index 897788fe0f91..2876a0e2ad54 100644
--- a/external/llvm-project/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
@@ -728,7 +728,7 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     // Get the map for this target prefix.
     auto &[Map, CommonPrefix] = BuiltinMap[Int.TargetPrefix];
 
-    if (!Map.insert({BuiltinName, Int.EnumName}).second)
+    if (!Map.try_emplace(BuiltinName, Int.EnumName).second)
       PrintFatalError(Int.TheDef->getLoc(),
                       "Intrinsic '" + Int.TheDef->getName() + "': duplicate " +
                           CompilerName + " builtin name!");
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 3029604adcc3..3a4ca1b45156 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -147,7 +147,7 @@ bool TypeSetByHwMode::constrain(const TypeSetByHwMode &VTS) {
       unsigned M = I.first;
       if (M == DefaultMode || hasMode(M))
         continue;
-      Map.insert({M, Map.at(DefaultMode)});
+      Map.try_emplace(M, Map.at(DefaultMode));
       Changed = true;
     }
   }
@@ -730,8 +730,8 @@ bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) {
   // processed identically.
   auto NoLength = [](const SmallDenseSet<ElementCount> &Lengths,
                      MVT T) -> bool {
-    return !Lengths.count(T.isVector() ? T.getVectorElementCount()
-                                       : ElementCount());
+    return !Lengths.contains(T.isVector() ? T.getVectorElementCount()
+                                          : ElementCount());
   };
 
   SmallVector<unsigned, 4> Modes;
@@ -778,7 +778,7 @@ bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) {
   typedef SmallSet<TypeSize, 2, TypeSizeComparator> TypeSizeSet;
 
   auto NoSize = [](const TypeSizeSet &Sizes, MVT T) -> bool {
-    return !Sizes.count(T.getSizeInBits());
+    return !Sizes.contains(T.getSizeInBits());
   };
 
   SmallVector<unsigned, 4> Modes;
@@ -899,6 +899,11 @@ TreePredicateFn::TreePredicateFn(TreePattern *N) : PatFragRec(N) {
   assert(
       (!hasPredCode() || !hasImmCode()) &&
       ".td file corrupt: can't have a node predicate *and* an imm predicate");
+
+  if (hasGISelPredicateCode() && hasGISelLeafPredicateCode())
+    PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
+                    ".td file corrupt: can't have GISelPredicateCode *and* "
+                    "GISelLeafPredicateCode");
 }
 
 bool TreePredicateFn::hasPredCode() const {
@@ -1293,8 +1298,20 @@ bool TreePredicateFn::hasGISelPredicateCode() const {
 }
 
 std::string TreePredicateFn::getGISelPredicateCode() const {
-  return std::string(
-      PatFragRec->getRecord()->getValueAsString("GISelPredicateCode"));
+  return PatFragRec->getRecord()->getValueAsString("GISelPredicateCode").str();
+}
+
+bool TreePredicateFn::hasGISelLeafPredicateCode() const {
+  return PatFragRec->getRecord()
+      ->getValueAsOptionalString("GISelLeafPredicateCode")
+      .has_value();
+}
+
+std::string TreePredicateFn::getGISelLeafPredicateCode() const {
+  return PatFragRec->getRecord()
+      ->getValueAsOptionalString("GISelLeafPredicateCode")
+      .value_or(StringRef())
+      .str();
 }
 
 StringRef TreePredicateFn::getImmType() const {
@@ -1807,8 +1824,8 @@ bool TreePatternNode::UpdateNodeTypeFromInst(unsigned ResNo,
 }
 
 bool TreePatternNode::ContainsUnresolvedType(TreePattern &TP) const {
-  for (unsigned i = 0, e = Types.size(); i != e; ++i)
-    if (!TP.getInfer().isConcrete(Types[i], true))
+  for (const TypeSetByHwMode &Type : Types)
+    if (!TP.getInfer().isConcrete(Type, true))
       return true;
   for (const TreePatternNode &Child : children())
     if (Child.ContainsUnresolvedType(TP))
@@ -3297,14 +3314,14 @@ void CodeGenDAGPatterns::ParseNodeTransforms() {
        reverse(Records.getAllDerivedDefinitions("SDNodeXForm"))) {
     const Record *SDNode = XFormNode->getValueAsDef("Opcode");
     StringRef Code = XFormNode->getValueAsString("XFormFunction");
-    SDNodeXForms.insert({XFormNode, NodeXForm(SDNode, Code.str())});
+    SDNodeXForms.try_emplace(XFormNode, NodeXForm(SDNode, Code.str()));
   }
 }
 
 void CodeGenDAGPatterns::ParseComplexPatterns() {
   for (const Record *R :
        reverse(Records.getAllDerivedDefinitions("ComplexPattern")))
-    ComplexPatterns.insert({R, R});
+    ComplexPatterns.try_emplace(R, R);
 }
 
 /// ParsePatternFragments - Parse all of the PatFrag definitions in the .td
@@ -3331,7 +3348,7 @@ void CodeGenDAGPatterns::ParsePatternFragments(bool OutFrags) {
     auto ArgsCopy = Args;
     SmallDenseSet<StringRef, 4> OperandsSet(llvm::from_range, ArgsCopy);
 
-    if (OperandsSet.count(""))
+    if (OperandsSet.contains(""))
       P->error("Cannot have unnamed 'node' values in pattern fragment!");
 
     // Parse the operands list.
@@ -3587,16 +3604,14 @@ class InstAnalyzer {
   const CodeGenDAGPatterns &CDP;
 
 public:
-  bool hasSideEffects;
-  bool mayStore;
-  bool mayLoad;
-  bool isBitcast;
-  bool isVariadic;
-  bool hasChain;
-
-  InstAnalyzer(const CodeGenDAGPatterns &cdp)
-      : CDP(cdp), hasSideEffects(false), mayStore(false), mayLoad(false),
-        isBitcast(false), isVariadic(false), hasChain(false) {}
+  bool hasSideEffects = false;
+  bool mayStore = false;
+  bool mayLoad = false;
+  bool isBitcast = false;
+  bool isVariadic = false;
+  bool hasChain = false;
+
+  InstAnalyzer(const CodeGenDAGPatterns &cdp) : CDP(cdp) {}
 
   void Analyze(const PatternToMatch &Pat) {
     const TreePatternNode &N = Pat.getSrcPattern();
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
index 725414f8f68d..64fec275faa6 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
@@ -590,6 +590,11 @@ class TreePredicateFn {
   bool hasGISelPredicateCode() const;
   std::string getGISelPredicateCode() const;
 
+  // If true, indicates that GlobalISel-based C++ code was supplied for checking
+  // register operands.
+  bool hasGISelLeafPredicateCode() const;
+  std::string getGISelLeafPredicateCode() const;
+
 private:
   bool hasPredCode() const;
   bool hasImmCode() const;
@@ -742,8 +747,8 @@ class TreePatternNode : public RefCountedBase<TreePatternNode> {
 
   /// hasChild - Return true if N is any of our children.
   bool hasChild(const TreePatternNode *N) const {
-    for (unsigned i = 0, e = Children.size(); i != e; ++i)
-      if (Children[i].get() == N)
+    for (const TreePatternNodePtr &Child : Children)
+      if (Child.get() == N)
         return true;
     return false;
   }
@@ -1166,9 +1171,9 @@ class CodeGenDAGPatterns {
   }
 
   const CodeGenIntrinsic &getIntrinsic(const Record *R) const {
-    for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i)
-      if (Intrinsics[i].TheDef == R)
-        return Intrinsics[i];
+    for (const CodeGenIntrinsic &Intrinsic : Intrinsics)
+      if (Intrinsic.TheDef == R)
+        return Intrinsic;
     llvm_unreachable("Unknown intrinsic!");
   }
 
@@ -1275,8 +1280,8 @@ class CodeGenDAGPatterns {
 inline bool SDNodeInfo::ApplyTypeConstraints(TreePatternNode &N,
                                              TreePattern &TP) const {
   bool MadeChange = false;
-  for (unsigned i = 0, e = TypeConstraints.size(); i != e; ++i)
-    MadeChange |= TypeConstraints[i].ApplyTypeConstraint(N, *this, TP);
+  for (const SDTypeConstraint &TypeConstraint : TypeConstraints)
+    MadeChange |= TypeConstraint.ApplyTypeConstraint(N, *this, TP);
   return MadeChange;
 }
 
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenInstruction.h b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenInstruction.h
index e38979af3909..3a5abc55319b 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenInstruction.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenInstruction.h
@@ -127,8 +127,7 @@ class CGIOperandList {
     /// getTiedOperand - If this operand is tied to another one, return the
     /// other operand number.  Otherwise, return -1.
     int getTiedRegister() const {
-      for (unsigned j = 0, e = Constraints.size(); j != e; ++j) {
-        const CGIOperandList::ConstraintInfo &CI = Constraints[j];
+      for (const CGIOperandList::ConstraintInfo &CI : Constraints) {
         if (CI.isTied())
           return CI.getTiedOperand();
       }
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index 0f9cd039b252..57a243158692 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -99,7 +99,7 @@ void CodeGenSubRegIndex::updateComponents(CodeGenRegBank &RegBank) {
   if (!Parts.empty()) {
     if (Parts.size() < 2)
       PrintFatalError(TheDef->getLoc(),
-                      "CoveredBySubRegs must have two or more entries");
+                      "CoveringSubRegIndices must have two or more entries");
     SmallVector<CodeGenSubRegIndex *, 8> IdxParts;
     for (const Record *Part : Parts)
       IdxParts.push_back(RegBank.getSubRegIdx(Part));
@@ -164,8 +164,8 @@ CodeGenRegister::CodeGenRegister(const Record *R, unsigned Enum)
     : TheDef(R), EnumValue(Enum),
       CostPerUse(R->getValueAsListOfInts("CostPerUse")),
       CoveredBySubRegs(R->getValueAsBit("CoveredBySubRegs")),
-      HasDisjunctSubRegs(false), Constant(R->getValueAsBit("isConstant")),
-      SubRegsComplete(false), SuperRegsComplete(false), TopoSig(~0u) {
+      Constant(R->getValueAsBit("isConstant")), SubRegsComplete(false),
+      SuperRegsComplete(false), TopoSig(~0u) {
   Artificial = R->getValueAsBit("isArtificial");
 }
 
@@ -466,7 +466,7 @@ void CodeGenRegister::computeSecondarySubRegs(CodeGenRegBank &RegBank) {
 
   std::queue<std::pair<CodeGenSubRegIndex *, CodeGenRegister *>> SubRegQueue;
   for (auto [SRI, SubReg] : SubRegs)
-    SubRegQueue.push({SRI, SubReg});
+    SubRegQueue.emplace(SRI, SubReg);
 
   // Look at the leading super-registers of each sub-register. Those are the
   // candidates for new sub-registers, assuming they are fully contained in
@@ -703,7 +703,7 @@ struct TupleExpander : SetTheory::Expander {
           RV.setValue(BitInit::get(RK, true));
 
         // Copy fields from the RegisterTuples def.
-        if (Field == "SubRegIndices" || Field == "CompositeIndices") {
+        if (Field == "SubRegIndices") {
           NewReg->addValue(*Def->getValue(Field));
           continue;
         }
@@ -933,9 +933,7 @@ bool CodeGenRegisterClass::Key::operator<(
 static bool testSubClass(const CodeGenRegisterClass *A,
                          const CodeGenRegisterClass *B) {
   return A->RSI.isSubClassOf(B->RSI) &&
-         std::includes(A->getMembers().begin(), A->getMembers().end(),
-                       B->getMembers().begin(), B->getMembers().end(),
-                       deref<std::less<>>());
+         llvm::includes(A->getMembers(), B->getMembers(), deref<std::less<>>());
 }
 
 /// Sorting predicate for register classes.  This provides a topological
@@ -1333,7 +1331,7 @@ CodeGenSubRegIndex *CodeGenRegBank::getSubRegIdx(const Record *Def) {
 
 const CodeGenSubRegIndex *
 CodeGenRegBank::findSubRegIdx(const Record *Def) const {
-  return Def2SubRegIdx.lookup(Def);
+  return Def2SubRegIdx.at(Def);
 }
 
 CodeGenRegister *CodeGenRegBank::getReg(const Record *Def) {
@@ -1461,7 +1459,7 @@ void CodeGenRegBank::computeComposites() {
   for (const CodeGenRegister &R : Registers) {
     const CodeGenRegister::SubRegMap &SM = R.getSubRegs();
     for (auto [SRI, SubReg] : SM)
-      SubRegAction[SRI].insert({&R, SubReg});
+      SubRegAction[SRI].try_emplace(&R, SubReg);
   }
 
   // Calculate the composition of two subregisters as compositions of their
@@ -1471,10 +1469,10 @@ void CodeGenRegBank::computeComposites() {
     RegMap C;
     const RegMap &Img1 = SubRegAction.at(Sub1);
     const RegMap &Img2 = SubRegAction.at(Sub2);
-    for (auto [SRI, SubReg] : Img1) {
+    for (auto [R, SubReg] : Img1) {
       auto F = Img2.find(SubReg);
       if (F != Img2.end())
-        C.insert({SRI, F->second});
+        C.try_emplace(R, F->second);
     }
     return C;
   };
@@ -1532,7 +1530,7 @@ void CodeGenRegBank::computeComposites() {
         if (CodeGenSubRegIndex *Prev =
                 Idx1->addComposite(Idx2, Idx3, getHwModes())) {
           // If the composition was not user-defined, always emit a warning.
-          if (!UserDefined.count({Idx1, Idx2}) ||
+          if (!UserDefined.contains({Idx1, Idx2}) ||
               agree(compose(Idx1, Idx2), SubRegAction.at(Idx3)))
             PrintWarning(Twine("SubRegIndex ") + Idx1->getQualifiedName() +
                          " and " + Idx2->getQualifiedName() +
@@ -1849,26 +1847,21 @@ static void computeUberWeights(MutableArrayRef<UberRegSet> UberSets,
   // Skip the first unallocatable set.
   for (UberRegSet &S : UberSets.drop_front()) {
     // Initialize all unit weights in this set, and remember the max units/reg.
-    const CodeGenRegister *Reg = nullptr;
-    unsigned MaxWeight = 0, Weight = 0;
-    for (RegUnitIterator UnitI(S.Regs); UnitI.isValid(); ++UnitI) {
-      if (Reg != UnitI.getReg()) {
-        if (Weight > MaxWeight)
-          MaxWeight = Weight;
-        Reg = UnitI.getReg();
-        Weight = 0;
-      }
-      if (!RegBank.getRegUnit(*UnitI).Artificial) {
-        unsigned UWeight = RegBank.getRegUnit(*UnitI).Weight;
-        if (!UWeight) {
-          UWeight = 1;
-          RegBank.increaseRegUnitWeight(*UnitI, UWeight);
+    unsigned MaxWeight = 0;
+    for (const CodeGenRegister *R : S.Regs) {
+      unsigned Weight = 0;
+      for (unsigned U : R->getRegUnits()) {
+        if (!RegBank.getRegUnit(U).Artificial) {
+          unsigned UWeight = RegBank.getRegUnit(U).Weight;
+          if (!UWeight) {
+            UWeight = 1;
+            RegBank.increaseRegUnitWeight(U, UWeight);
+          }
+          Weight += UWeight;
         }
-        Weight += UWeight;
       }
+      MaxWeight = std::max(MaxWeight, Weight);
     }
-    if (Weight > MaxWeight)
-      MaxWeight = Weight;
     if (S.Weight != MaxWeight) {
       LLVM_DEBUG({
         dbgs() << "UberSet " << &S - UberSets.begin() << " Weight "
@@ -1995,8 +1988,7 @@ findRegUnitSet(const std::vector<RegUnitSet> &UniqueSets,
 // Return true if the RUSubSet is a subset of RUSuperSet.
 static bool isRegUnitSubSet(const std::vector<unsigned> &RUSubSet,
                             const std::vector<unsigned> &RUSuperSet) {
-  return std::includes(RUSuperSet.begin(), RUSuperSet.end(), RUSubSet.begin(),
-                       RUSubSet.end());
+  return llvm::includes(RUSuperSet, RUSubSet);
 }
 
 /// Iteratively prune unit sets. Prune subsets that are close to the superset,
@@ -2408,7 +2400,7 @@ void CodeGenRegBank::inferMatchingSuperRegClass(
     if (RC->getSubClassWithSubReg(SubIdx) != RC)
       continue;
 
-    if (ImpliedSubRegIndices.count(SubIdx))
+    if (ImpliedSubRegIndices.contains(SubIdx))
       continue;
 
     // Build list of (Sub, Super) pairs for this SubIdx, sorted by Sub. Note
@@ -2623,6 +2615,55 @@ CodeGenRegBank::getMinimalPhysRegClass(const Record *RegRecord,
   return BestRC;
 }
 
+const CodeGenRegisterClass *
+CodeGenRegBank::getSuperRegForSubReg(const ValueTypeByHwMode &ValueTy,
+                                     const CodeGenSubRegIndex *SubIdx,
+                                     bool MustBeAllocatable) const {
+  std::vector<const CodeGenRegisterClass *> Candidates;
+  auto &RegClasses = getRegClasses();
+
+  // Try to find a register class which supports ValueTy, and also contains
+  // SubIdx.
+  for (const CodeGenRegisterClass &RC : RegClasses) {
+    // Is there a subclass of this class which contains this subregister index?
+    const CodeGenRegisterClass *SubClassWithSubReg =
+        RC.getSubClassWithSubReg(SubIdx);
+    if (!SubClassWithSubReg)
+      continue;
+
+    // We have a class. Check if it supports this value type.
+    if (!llvm::is_contained(SubClassWithSubReg->VTs, ValueTy))
+      continue;
+
+    // If necessary, check that it is allocatable.
+    if (MustBeAllocatable && !SubClassWithSubReg->Allocatable)
+      continue;
+
+    // We have a register class which supports both the value type and
+    // subregister index. Remember it.
+    Candidates.push_back(SubClassWithSubReg);
+  }
+
+  // If we didn't find anything, we're done.
+  if (Candidates.empty())
+    return nullptr;
+
+  // Find and return the largest of our candidate classes.
+  llvm::stable_sort(Candidates, [&](const CodeGenRegisterClass *A,
+                                    const CodeGenRegisterClass *B) {
+    if (A->getMembers().size() > B->getMembers().size())
+      return true;
+
+    if (A->getMembers().size() < B->getMembers().size())
+      return false;
+
+    // Order by name as a tie-breaker.
+    return StringRef(A->getName()) < B->getName();
+  });
+
+  return Candidates[0];
+}
+
 BitVector
 CodeGenRegBank::computeCoveredRegisters(ArrayRef<const Record *> Regs) {
   SetVector<const CodeGenRegister *> Set;
@@ -2639,13 +2680,13 @@ CodeGenRegBank::computeCoveredRegisters(ArrayRef<const Record *> Regs) {
   // Second, find all super-registers that are completely covered by the set.
   for (unsigned i = 0; i != Set.size(); ++i) {
     for (const CodeGenRegister *Super : Set[i]->getSuperRegs()) {
-      if (!Super->CoveredBySubRegs || Set.count(Super))
+      if (!Super->CoveredBySubRegs || Set.contains(Super))
         continue;
       // This new super-register is covered by its sub-registers.
       bool AllSubsInSet = true;
       const CodeGenRegister::SubRegMap &SRM = Super->getSubRegs();
       for (auto [_, SR] : SRM)
-        if (!Set.count(SR)) {
+        if (!Set.contains(SR)) {
           AllSubsInSet = false;
           break;
         }
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.h b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.h
index 75d9a3f7a2c0..bbcd44ce2cc5 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenRegisters.h
@@ -564,7 +564,7 @@ struct RegUnit {
   // Weight assigned to this RegUnit for estimating register pressure.
   // This is useful when equalizing weights in register classes with mixed
   // register topologies.
-  unsigned Weight;
+  unsigned Weight = 0;
 
   // Each native RegUnit corresponds to one or two root registers. The full
   // set of registers containing this unit can be computed as the union of
@@ -573,14 +573,12 @@ struct RegUnit {
 
   // Index into RegClassUnitSets where we can find the list of UnitSets that
   // contain this unit.
-  unsigned RegClassUnitSetsIdx;
+  unsigned RegClassUnitSetsIdx = 0;
   // A register unit is artificial if at least one of its roots is
   // artificial.
-  bool Artificial;
+  bool Artificial = false;
 
-  RegUnit() : Weight(0), RegClassUnitSetsIdx(0), Artificial(false) {
-    Roots[0] = Roots[1] = nullptr;
-  }
+  RegUnit() { Roots[0] = Roots[1] = nullptr; }
 
   ArrayRef<const CodeGenRegister *> getRoots() const {
     assert(!(Roots[1] && !Roots[0]) && "Invalid roots array");
@@ -831,6 +829,13 @@ class CodeGenRegBank {
   getMinimalPhysRegClass(const Record *RegRecord,
                          ValueTypeByHwMode *VT = nullptr);
 
+  /// Return the largest register class which supports \p Ty and covers \p
+  /// SubIdx if it exists.
+  const CodeGenRegisterClass *
+  getSuperRegForSubReg(const ValueTypeByHwMode &Ty,
+                       const CodeGenSubRegIndex *SubIdx,
+                       bool MustBeAllocatable = false) const;
+
   // Get the sum of unit weights.
   unsigned getRegUnitSetWeight(const std::vector<unsigned> &Units) const {
     unsigned Weight = 0;
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.cpp b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
index 8cd8fb6ba90c..af7e43929bcf 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
@@ -1637,7 +1637,7 @@ static void inferFromTransitions(ArrayRef<PredTransition> LastTransitions,
     // Transition should not contain processor indices already assigned to
     // InstRWs in this scheduling class.
     const CodeGenSchedClass &FromSC = SchedModels.getSchedClass(FromClassIdx);
-    if (FromSC.InstRWProcIndices.count(LastTransition.ProcIndex))
+    if (FromSC.InstRWProcIndices.contains(LastTransition.ProcIndex))
       continue;
     SCTrans.ProcIndex = LastTransition.ProcIndex;
     SCTrans.ToClassIdx =
@@ -2176,7 +2176,7 @@ bool CodeGenProcModel::isUnsupported(const CodeGenInstruction &Inst) const {
 }
 
 bool CodeGenProcModel::hasReadOfWrite(const Record *WriteDef) const {
-  return ReadOfWriteSet.count(WriteDef);
+  return ReadOfWriteSet.contains(WriteDef);
 }
 
 #ifndef NDEBUG
@@ -2232,13 +2232,10 @@ void PredTransitions::dump() const {
       dbgs() << LS << SchedModels.getSchedRW(PC.RWIdx, PC.IsRead).Name << ":"
              << PC.Predicate->getName();
     dbgs() << "},\n  => {";
-    for (SmallVectorImpl<SmallVector<unsigned, 4>>::const_iterator
-             WSI = TI.WriteSequences.begin(),
-             WSE = TI.WriteSequences.end();
-         WSI != WSE; ++WSI) {
+    for (const auto &WS : TI.WriteSequences) {
       dbgs() << "(";
       ListSeparator LS;
-      for (unsigned N : *WSI)
+      for (unsigned N : WS)
         dbgs() << LS << SchedModels.getSchedWrite(N).Name;
       dbgs() << "),";
     }
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.h b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.h
index 697a1ce8f75a..1d5e953cf70c 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenSchedule.h
@@ -193,7 +193,7 @@ struct CodeGenRegisterFile {
   unsigned MaxMovesEliminatedPerCycle;
   bool AllowZeroMoveEliminationOnly;
 
-  unsigned NumPhysRegs;
+  unsigned NumPhysRegs = 0;
   std::vector<CodeGenRegisterCost> Costs;
 
   CodeGenRegisterFile(StringRef name, const Record *def,
@@ -201,7 +201,7 @@ struct CodeGenRegisterFile {
                       bool AllowZeroMoveElimOnly = false)
       : Name(name), RegisterFileDef(def),
         MaxMovesEliminatedPerCycle(MaxMoveElimPerCy),
-        AllowZeroMoveEliminationOnly(AllowZeroMoveElimOnly), NumPhysRegs(0) {}
+        AllowZeroMoveEliminationOnly(AllowZeroMoveElimOnly) {}
 
   bool hasDefaultCosts() const { return Costs.empty(); }
 };
@@ -261,16 +261,16 @@ struct CodeGenProcModel {
   std::vector<CodeGenRegisterFile> RegisterFiles;
 
   // Optional Retire Control Unit definition.
-  const Record *RetireControlUnit;
+  const Record *RetireControlUnit = nullptr;
 
   // Load/Store queue descriptors.
-  const Record *LoadQueue;
-  const Record *StoreQueue;
+  const Record *LoadQueue = nullptr;
+  const Record *StoreQueue = nullptr;
 
   CodeGenProcModel(unsigned Idx, std::string Name, const Record *MDef,
                    const Record *IDef)
-      : Index(Idx), ModelName(std::move(Name)), ModelDef(MDef), ItinsDef(IDef),
-        RetireControlUnit(nullptr), LoadQueue(nullptr), StoreQueue(nullptr) {}
+      : Index(Idx), ModelName(std::move(Name)), ModelDef(MDef), ItinsDef(IDef) {
+  }
 
   bool hasItineraries() const {
     return !ItinsDef->getValueAsListOfDefs("IID").empty();
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.cpp b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.cpp
index 303589d7a934..f519582387db 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.cpp
@@ -160,54 +160,6 @@ CodeGenRegBank &CodeGenTarget::getRegBank() const {
   return *RegBank;
 }
 
-const CodeGenRegisterClass *CodeGenTarget::getSuperRegForSubReg(
-    const ValueTypeByHwMode &ValueTy, CodeGenRegBank &RegBank,
-    const CodeGenSubRegIndex *SubIdx, bool MustBeAllocatable) const {
-  std::vector<const CodeGenRegisterClass *> Candidates;
-  auto &RegClasses = RegBank.getRegClasses();
-
-  // Try to find a register class which supports ValueTy, and also contains
-  // SubIdx.
-  for (const CodeGenRegisterClass &RC : RegClasses) {
-    // Is there a subclass of this class which contains this subregister index?
-    const CodeGenRegisterClass *SubClassWithSubReg =
-        RC.getSubClassWithSubReg(SubIdx);
-    if (!SubClassWithSubReg)
-      continue;
-
-    // We have a class. Check if it supports this value type.
-    if (!llvm::is_contained(SubClassWithSubReg->VTs, ValueTy))
-      continue;
-
-    // If necessary, check that it is allocatable.
-    if (MustBeAllocatable && !SubClassWithSubReg->Allocatable)
-      continue;
-
-    // We have a register class which supports both the value type and
-    // subregister index. Remember it.
-    Candidates.push_back(SubClassWithSubReg);
-  }
-
-  // If we didn't find anything, we're done.
-  if (Candidates.empty())
-    return nullptr;
-
-  // Find and return the largest of our candidate classes.
-  llvm::stable_sort(Candidates, [&](const CodeGenRegisterClass *A,
-                                    const CodeGenRegisterClass *B) {
-    if (A->getMembers().size() > B->getMembers().size())
-      return true;
-
-    if (A->getMembers().size() < B->getMembers().size())
-      return false;
-
-    // Order by name as a tie-breaker.
-    return StringRef(A->getName()) < B->getName();
-  });
-
-  return Candidates[0];
-}
-
 /// getRegisterByName - If there is a register with the specific AsmName,
 /// return it.
 const CodeGenRegister *CodeGenTarget::getRegisterByName(StringRef Name) const {
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.h b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.h
index da2f3e060591..52871f33a301 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/CodeGenTarget.h
@@ -122,13 +122,6 @@ class CodeGenTarget {
   /// getRegBank - Return the register bank description.
   CodeGenRegBank &getRegBank() const;
 
-  /// Return the largest register class on \p RegBank which supports \p Ty and
-  /// covers \p SubIdx if it exists.
-  const CodeGenRegisterClass *
-  getSuperRegForSubReg(const ValueTypeByHwMode &Ty, CodeGenRegBank &RegBank,
-                       const CodeGenSubRegIndex *SubIdx,
-                       bool MustBeAllocatable = false) const;
-
   /// getRegisterByName - If there is a register with the specific AsmName,
   /// return it.
   const CodeGenRegister *getRegisterByName(StringRef Name) const;
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/DAGISelMatcher.cpp b/external/llvm-project/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
index 8780c4f5b61c..3543bb5a55c6 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
@@ -286,11 +286,11 @@ void EmitNodeMatcherCommon::printImpl(raw_ostream &OS, indent Indent) const {
   OS << (isa<MorphNodeToMatcher>(this) ? "MorphNodeTo: " : "EmitNode: ")
      << CGI.Namespace << "::" << CGI.TheDef->getName() << ": <todo flags> ";
 
-  for (unsigned i = 0, e = VTs.size(); i != e; ++i)
-    OS << ' ' << getEnumName(VTs[i]);
+  for (MVT::SimpleValueType VT : VTs)
+    OS << ' ' << getEnumName(VT);
   OS << '(';
-  for (unsigned i = 0, e = Operands.size(); i != e; ++i)
-    OS << Operands[i] << ' ';
+  for (unsigned Operand : Operands)
+    OS << Operand << ' ';
   OS << ")\n";
 }
 
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 2cb357927250..327ac5f87777 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -33,6 +33,8 @@ Error failUnsupported(const Twine &Reason) {
 std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
   if (Predicate.hasGISelPredicateCode())
     return "GICXXPred_MI_" + Predicate.getFnName();
+  if (Predicate.hasGISelLeafPredicateCode())
+    return "GICXXPred_MO_" + Predicate.getFnName();
   return "GICXXPred_" + Predicate.getImmTypeIdentifier().str() + "_" +
          Predicate.getFnName();
 }
@@ -1326,6 +1328,19 @@ void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
         << MatchTable::LineBreak;
 }
 
+//===- OperandLeafPredicateMatcher
+//-----------------------------------------===//
+
+void OperandLeafPredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckLeafOperandPredicate")
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("MO") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::Comment("Predicate")
+        << MatchTable::NamedValue(2, getEnumNameForPredicate(Predicate))
+        << MatchTable::LineBreak;
+}
+
 //===- OperandMatcher -----------------------------------------------------===//
 
 std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const {
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
index 9f17882cdae4..620f88db6610 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
@@ -501,13 +501,13 @@ class RuleMatcher : public Matcher {
 
   /// ID for the next instruction variable defined with
   /// implicitlyDefineInsnVar()
-  unsigned NextInsnVarID;
+  unsigned NextInsnVarID = 0;
 
   /// ID for the next output instruction allocated with allocateOutputInsnID()
-  unsigned NextOutputInsnID;
+  unsigned NextOutputInsnID = 0;
 
   /// ID for the next temporary register ID allocated with allocateTempRegID()
-  unsigned NextTempRegID;
+  unsigned NextTempRegID = 0;
 
   /// ID for the next recorded type. Starts at -1 and counts down.
   TempTypeIdx NextTempTypeIdx = -1;
@@ -545,9 +545,7 @@ class RuleMatcher : public Matcher {
                              StringRef FlagName, GISelFlags FlagBit);
 
 public:
-  RuleMatcher(ArrayRef<SMLoc> SrcLoc)
-      : NextInsnVarID(0), NextOutputInsnID(0), NextTempRegID(0), SrcLoc(SrcLoc),
-        RuleID(NextRuleID++) {}
+  RuleMatcher(ArrayRef<SMLoc> SrcLoc) : SrcLoc(SrcLoc), RuleID(NextRuleID++) {}
   RuleMatcher(RuleMatcher &&Other) = default;
   RuleMatcher &operator=(RuleMatcher &&Other) = default;
 
@@ -824,6 +822,7 @@ class PredicateMatcher {
     IPM_OneUse,
     IPM_GenericPredicate,
     IPM_MIFlags,
+    OPM_LeafPredicate,
     OPM_SameOperand,
     OPM_ComplexPattern,
     OPM_IntrinsicID,
@@ -1255,6 +1254,26 @@ class OperandImmPredicateMatcher : public OperandPredicateMatcher {
                             RuleMatcher &Rule) const override;
 };
 
+/// Generates code to check that this operand is a register whose value meets
+/// the predicate.
+class OperandLeafPredicateMatcher : public OperandPredicateMatcher {
+protected:
+  TreePredicateFn Predicate;
+
+public:
+  OperandLeafPredicateMatcher(unsigned InsnVarID, unsigned OpIdx,
+                              const TreePredicateFn &Predicate)
+      : OperandPredicateMatcher(OPM_LeafPredicate, InsnVarID, OpIdx),
+        Predicate(Predicate) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_LeafPredicate;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
 /// Generates code to check that a set of predicates match for a particular
 /// operand.
 class OperandMatcher : public PredicateListMatcher<OperandPredicateMatcher> {
@@ -1400,7 +1419,7 @@ class InstructionOpcodeMatcher : public InstructionPredicateMatcher {
   }
 
   bool hasValue() const override {
-    return Insts.size() == 1 && OpcodeValues.count(Insts[0]);
+    return Insts.size() == 1 && OpcodeValues.contains(Insts[0]);
   }
 
   // TODO: This is used for the SwitchMatcher optimization. We should be able to
@@ -2018,12 +2037,12 @@ class CopyConstantAsImmRenderer : public OperandRenderer {
   unsigned NewInsnID;
   /// The name of the operand.
   const std::string SymbolicName;
-  bool Signed;
+  bool Signed = true;
 
 public:
   CopyConstantAsImmRenderer(unsigned NewInsnID, StringRef SymbolicName)
       : OperandRenderer(OR_CopyConstantAsImm), NewInsnID(NewInsnID),
-        SymbolicName(SymbolicName), Signed(true) {}
+        SymbolicName(SymbolicName) {}
 
   static bool classof(const OperandRenderer *R) {
     return R->getKind() == OR_CopyConstantAsImm;
@@ -2338,7 +2357,7 @@ class BuildMIAction : public MatchAction {
 private:
   unsigned InsnID;
   const CodeGenInstruction *I;
-  InstructionMatcher *Matched;
+  InstructionMatcher *Matched = nullptr;
   std::vector<std::unique_ptr<OperandRenderer>> OperandRenderers;
   SmallPtrSet<const Record *, 4> DeadImplicitDefs;
 
@@ -2351,7 +2370,7 @@ class BuildMIAction : public MatchAction {
 
 public:
   BuildMIAction(unsigned InsnID, const CodeGenInstruction *I)
-      : MatchAction(AK_BuildMI), InsnID(InsnID), I(I), Matched(nullptr) {}
+      : MatchAction(AK_BuildMI), InsnID(InsnID), I(I) {}
 
   static bool classof(const MatchAction *A) {
     return A->getKind() == AK_BuildMI;
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
index ffab2fdedc5a..333d956a4ff3 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
@@ -182,6 +182,7 @@ void GlobalISelMatchTableExecutorEmitter::emitExecutorImpl(
   emitSubtargetFeatureBitsetImpl(OS, Rules);
   emitComplexPredicates(OS, ComplexOperandMatchers);
   emitMIPredicateFns(OS);
+  emitLeafPredicateFns(OS);
   emitI64ImmPredicateFns(OS);
   emitAPFloatImmPredicateFns(OS);
   emitAPIntImmPredicateFns(OS);
@@ -234,6 +235,9 @@ void GlobalISelMatchTableExecutorEmitter::emitTemporariesDecl(
      << "  bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI"
         ", const MatcherState &State) "
         "const override;\n"
+     << "  bool testMOPredicate_MO(unsigned PredicateID, const MachineOperand "
+        "&MO, const MatcherState &State) "
+        "const override;\n"
      << "  bool testSimplePredicate(unsigned PredicateID) const override;\n"
      << "  bool runCustomAction(unsigned FnID, const MatcherState &State, "
         "NewMIVector &OutMIs) "
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
index 862f1e83c169..1f66d73f7757 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
@@ -79,8 +79,8 @@ class GlobalISelMatchTableExecutorEmitter {
       raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType,
       StringRef ArgName, StringRef AdditionalArgs,
       StringRef AdditionalDeclarations, ArrayRef<PredicateObject> Predicates,
-      std::function<StringRef(PredicateObject)> GetPredEnumName,
-      std::function<StringRef(PredicateObject)> GetPredCode,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredEnumName,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredCode,
       StringRef Comment) {
     if (!Comment.empty())
       OS << "// " << Comment << "\n";
@@ -135,14 +135,34 @@ class GlobalISelMatchTableExecutorEmitter {
   void emitMIPredicateFnsImpl(
       raw_ostream &OS, StringRef AdditionalDecls,
       ArrayRef<PredicateObject> Predicates,
-      std::function<StringRef(PredicateObject)> GetPredEnumName,
-      std::function<StringRef(PredicateObject)> GetPredCode,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredEnumName,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredCode,
       StringRef Comment = "") {
     return emitCxxPredicateFns(
         OS, "MI", "const MachineInstr &", "MI", ", const MatcherState &State",
         AdditionalDecls, Predicates, GetPredEnumName, GetPredCode, Comment);
   }
 
+  /// Emits `testMOPredicate_MO`.
+  /// \tparam PredicateObject An object representing a predicate to emit.
+  /// \param OS Output stream.
+  /// \param AdditionalDecls Additional C++ variable declarations.
+  /// \param Predicates Predicates to emit.
+  /// \param GetPredEnumName Returns an enum name for a given predicate.
+  /// \param GetPredCode Returns the C++ code of a given predicate.
+  /// \param Comment Optional comment for the enum declaration.
+  template <typename PredicateObject>
+  void emitLeafPredicateFnsImpl(
+      raw_ostream &OS, StringRef AdditionalDecls,
+      ArrayRef<PredicateObject> Predicates,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredEnumName,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredCode,
+      StringRef Comment = "") {
+    return emitCxxPredicateFns(
+        OS, "MO", "const MachineOperand &", "MO", ", const MatcherState &State",
+        AdditionalDecls, Predicates, GetPredEnumName, GetPredCode, Comment);
+  }
+
   /// Helper function to emit the following executor functions:
   ///   * testImmPredicate_I64      (TypeIdentifier=I64)
   ///   * testImmPredicate_APInt    (TypeIdentifier=APInt)
@@ -160,8 +180,8 @@ class GlobalISelMatchTableExecutorEmitter {
   void emitImmPredicateFnsImpl(
       raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType,
       ArrayRef<PredicateObject> Predicates,
-      std::function<StringRef(PredicateObject)> GetPredEnumName,
-      std::function<StringRef(PredicateObject)> GetPredCode,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredEnumName,
+      llvm::function_ref<StringRef(PredicateObject)> GetPredCode,
       StringRef Comment = "") {
     return emitCxxPredicateFns(OS, TypeIdentifier, ArgType, "Imm", "", "",
                                Predicates, GetPredEnumName, GetPredCode,
@@ -189,6 +209,10 @@ class GlobalISelMatchTableExecutorEmitter {
   /// Note: `emitMIPredicateFnsImpl` can be used to do most of the work.
   virtual void emitMIPredicateFns(raw_ostream &OS) = 0;
 
+  /// Emit the `testLeafPredicate` function
+  /// Note `emitLeafPredicateFnsImpl` can be used to do most of the work.
+  virtual void emitLeafPredicateFns(raw_ostream &OS) = 0;
+
   /// Emit the `testImmPredicate_I64` function.
   /// Note: `emitImmPredicateFnsImpl` can be used to do most of the work.
   virtual void emitI64ImmPredicateFns(raw_ostream &OS) = 0;
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
index 981761821300..07db6baef84a 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
@@ -581,7 +581,7 @@ bool PatFrag::checkSemantics() {
 
   StringSet<> SeenOps;
   for (const auto &Op : in_params()) {
-    if (SeenOps.count(Op.Name)) {
+    if (SeenOps.contains(Op.Name)) {
       PrintError("duplicate parameter '" + Op.Name + "'");
       return false;
     }
@@ -609,7 +609,7 @@ bool PatFrag::checkSemantics() {
       return false;
     }
 
-    if (SeenOps.count(Op.Name)) {
+    if (SeenOps.contains(Op.Name)) {
       PrintError("duplicate parameter '" + Op.Name + "'");
       return false;
     }
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.cpp b/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.cpp
index c2368cb31dbb..cb4f8876c648 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.cpp
@@ -32,7 +32,7 @@ ValueTypeByHwMode::ValueTypeByHwMode(const Record *R,
                                      const CodeGenHwModes &CGH) {
   const HwModeSelect &MS = CGH.getHwModeSelect(R);
   for (const HwModeSelect::PairType &P : MS.Items) {
-    auto I = Map.insert({P.first, MVT(llvm::getValueType(P.second))});
+    auto I = Map.try_emplace(P.first, MVT(llvm::getValueType(P.second)));
     assert(I.second && "Duplicate entry?");
     (void)I;
   }
@@ -142,7 +142,7 @@ RegSizeInfoByHwMode::RegSizeInfoByHwMode(const Record *R,
                                          const CodeGenHwModes &CGH) {
   const HwModeSelect &MS = CGH.getHwModeSelect(R);
   for (const HwModeSelect::PairType &P : MS.Items) {
-    auto I = Map.insert({P.first, RegSizeInfo(P.second)});
+    auto I = Map.try_emplace(P.first, RegSizeInfo(P.second));
     assert(I.second && "Duplicate entry?");
     (void)I;
   }
@@ -195,7 +195,7 @@ SubRegRangeByHwMode::SubRegRangeByHwMode(const Record *R,
                                          const CodeGenHwModes &CGH) {
   const HwModeSelect &MS = CGH.getHwModeSelect(R);
   for (const HwModeSelect::PairType &P : MS.Items) {
-    auto I = Map.insert({P.first, SubRegRange(P.second)});
+    auto I = Map.try_emplace(P.first, SubRegRange(P.second));
     assert(I.second && "Duplicate entry?");
     (void)I;
   }
@@ -207,7 +207,7 @@ EncodingInfoByHwMode::EncodingInfoByHwMode(const Record *R,
   for (const HwModeSelect::PairType &P : MS.Items) {
     assert(P.second && P.second->isSubClassOf("InstructionEncoding") &&
            "Encoding must subclass InstructionEncoding");
-    auto I = Map.insert({P.first, P.second});
+    auto I = Map.try_emplace(P.first, P.second);
     assert(I.second && "Duplicate entry?");
     (void)I;
   }
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.h b/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.h
index bff164c6a6aa..7925599a98a0 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/InfoByHwMode.h
@@ -118,7 +118,7 @@ template <typename InfoT> struct InfoByHwMode {
 
     // Copy and insert the default mode which should be first.
     assert(hasDefault());
-    auto P = Map.insert({Mode, Map.begin()->second});
+    auto P = Map.try_emplace(Mode, Map.begin()->second);
     return P.first->second;
   }
   const InfoT &get(unsigned Mode) const {
@@ -154,7 +154,7 @@ template <typename InfoT> struct InfoByHwMode {
 struct ValueTypeByHwMode : public InfoByHwMode<MVT> {
   ValueTypeByHwMode(const Record *R, const CodeGenHwModes &CGH);
   ValueTypeByHwMode(const Record *R, MVT T);
-  ValueTypeByHwMode(MVT T) { Map.insert({DefaultMode, T}); }
+  ValueTypeByHwMode(MVT T) { Map.try_emplace(DefaultMode, T); }
   ValueTypeByHwMode() = default;
 
   bool operator==(const ValueTypeByHwMode &T) const;
@@ -229,7 +229,9 @@ struct SubRegRange {
 
 struct SubRegRangeByHwMode : public InfoByHwMode<SubRegRange> {
   SubRegRangeByHwMode(const Record *R, const CodeGenHwModes &CGH);
-  SubRegRangeByHwMode(SubRegRange Range) { Map.insert({DefaultMode, Range}); }
+  SubRegRangeByHwMode(SubRegRange Range) {
+    Map.try_emplace(DefaultMode, Range);
+  }
   SubRegRangeByHwMode() = default;
 
   void insertSubRegRangeForMode(unsigned Mode, SubRegRange Info) {
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/PredicateExpander.h b/external/llvm-project/llvm/utils/TableGen/Common/PredicateExpander.h
index 0c3a8718a473..4439327af2b0 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/PredicateExpander.h
+++ b/external/llvm-project/llvm/utils/TableGen/Common/PredicateExpander.h
@@ -25,9 +25,9 @@ namespace llvm {
 class Record;
 
 class PredicateExpander {
-  bool EmitCallsByRef;
-  bool NegatePredicate;
-  bool ExpandForMC;
+  bool EmitCallsByRef = true;
+  bool NegatePredicate = false;
+  bool ExpandForMC = false;
   StringRef TargetName;
 
   PredicateExpander(const PredicateExpander &) = delete;
@@ -38,8 +38,7 @@ class PredicateExpander {
 
 public:
   explicit PredicateExpander(StringRef Target, unsigned Indent = 1)
-      : EmitCallsByRef(true), NegatePredicate(false), ExpandForMC(false),
-        TargetName(Target), Indent(Indent, 2) {}
+      : TargetName(Target), Indent(Indent, 2) {}
   bool isByRef() const { return EmitCallsByRef; }
   bool shouldNegate() const { return NegatePredicate; }
   bool shouldExpandForMC() const { return ExpandForMC; }
diff --git a/external/llvm-project/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp b/external/llvm-project/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
index 9de6a585eb4d..e5e03b8bd8bd 100644
--- a/external/llvm-project/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
@@ -241,21 +241,21 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
         const CodeGenHwModes &HWM = Target.getHwModes();
         EncodingInfoByHwMode EBM(DI->getDef(), HWM);
         for (const auto [Mode, EncodingDef] : EBM) {
-          Modes.insert({Mode, "_" + HWM.getMode(Mode).Name.str()});
+          Modes.try_emplace(Mode, "_" + HWM.getMode(Mode).Name.str());
           const RecordVal *RV = EncodingDef->getValue("Inst");
           const DagInit *DI = cast<DagInit>(RV->getValue());
-          VarLenInsts[R].insert({Mode, VarLenInst(DI, RV)});
+          VarLenInsts[R].try_emplace(Mode, VarLenInst(DI, RV));
         }
         continue;
       }
     }
     const RecordVal *RV = R->getValue("Inst");
     const DagInit *DI = cast<DagInit>(RV->getValue());
-    VarLenInsts[R].insert({Universal, VarLenInst(DI, RV)});
+    VarLenInsts[R].try_emplace(Universal, VarLenInst(DI, RV));
   }
 
   if (Modes.empty())
-    Modes.insert({Universal, ""}); // Base case, skip suffix.
+    Modes.try_emplace(Universal, ""); // Base case, skip suffix.
 
   // Emit function declaration
   OS << "void " << Target.getName()
diff --git a/external/llvm-project/llvm/utils/TableGen/CompressInstEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/CompressInstEmitter.cpp
index b981d38e283f..4a0b6d79c53d 100644
--- a/external/llvm-project/llvm/utils/TableGen/CompressInstEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/CompressInstEmitter.cpp
@@ -541,9 +541,9 @@ getReqFeatures(std::set<std::pair<bool, StringRef>> &FeaturesSet,
           !cast<DefInit>(Arg)->getDef()->isSubClassOf("SubtargetFeature"))
         PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!");
       if (IsOr)
-        AnyOfSet.insert({IsNot, cast<DefInit>(Arg)->getDef()->getName()});
+        AnyOfSet.emplace(IsNot, cast<DefInit>(Arg)->getDef()->getName());
       else
-        FeaturesSet.insert({IsNot, cast<DefInit>(Arg)->getDef()->getName()});
+        FeaturesSet.emplace(IsNot, cast<DefInit>(Arg)->getDef()->getName());
     }
 
     if (IsOr)
diff --git a/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 13ab21630c69..49c33dc5ca14 100644
--- a/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -151,7 +151,7 @@ class MatcherTableEmitter {
         Uses += PredicateUsage[TP];
 
       // We only add the first predicate here since they are with the same code.
-      PredicateList.push_back({TPs[0], Uses});
+      PredicateList.emplace_back(TPs[0], Uses);
     }
 
     stable_sort(PredicateList, [](const auto &A, const auto &B) {
diff --git a/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index a1ab06defb45..227311b0a3bc 100644
--- a/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -76,7 +76,7 @@ class MatcherGen {
   /// NextRecordedOperandNo - As we emit opcodes to record matched values in
   /// the RecordedNodes array, this keeps track of which slot will be next to
   /// record into.
-  unsigned NextRecordedOperandNo;
+  unsigned NextRecordedOperandNo = 0;
 
   /// MatchedChainNodes - This maintains the position in the recorded nodes
   /// array of all of the recorded input nodes that have chains.
@@ -94,11 +94,11 @@ class MatcherGen {
   SmallVector<std::pair<const Record *, unsigned>, 2> PhysRegInputs;
 
   /// Matcher - This is the top level of the generated matcher, the result.
-  Matcher *TheMatcher;
+  Matcher *TheMatcher = nullptr;
 
   /// CurPredicate - As we emit matcher nodes, this points to the latest check
   /// which should have future checks stuck into its Next position.
-  Matcher *CurPredicate;
+  Matcher *CurPredicate = nullptr;
 
 public:
   MatcherGen(const PatternToMatch &pattern, const CodeGenDAGPatterns &cgp);
@@ -147,8 +147,7 @@ class MatcherGen {
 
 MatcherGen::MatcherGen(const PatternToMatch &pattern,
                        const CodeGenDAGPatterns &cgp)
-    : Pattern(pattern), CGP(cgp), NextRecordedOperandNo(0), TheMatcher(nullptr),
-      CurPredicate(nullptr) {
+    : Pattern(pattern), CGP(cgp) {
   // We need to produce the matcher tree for the patterns source pattern.  To
   // do this we need to match the structure as well as the types.  To do the
   // type matching, we want to figure out the fewest number of type checks we
@@ -526,23 +525,20 @@ void MatcherGen::EmitMatchCode(const TreePatternNode &N,
     EmitOperatorMatchCode(N, NodeNoTypes);
 
   // If there are node predicates for this node, generate their checks.
-  for (unsigned i = 0, e = N.getPredicateCalls().size(); i != e; ++i) {
-    const TreePredicateCall &Pred = N.getPredicateCalls()[i];
+  for (const TreePredicateCall &Pred : N.getPredicateCalls()) {
     SmallVector<unsigned, 4> Operands;
     if (Pred.Fn.usesOperands()) {
       TreePattern *TP = Pred.Fn.getOrigPatFragRecord();
-      for (unsigned i = 0; i < TP->getNumArgs(); ++i) {
-        std::string Name =
-            ("pred:" + Twine(Pred.Scope) + ":" + TP->getArgName(i)).str();
+      for (const std::string &Arg : TP->getArgList()) {
+        std::string Name = ("pred:" + Twine(Pred.Scope) + ":" + Arg).str();
         Operands.push_back(getNamedArgumentSlot(Name));
       }
     }
     AddMatcher(new CheckPredicateMatcher(Pred.Fn, Operands));
   }
 
-  for (unsigned i = 0, e = ResultsToTypeCheck.size(); i != e; ++i)
-    AddMatcher(new CheckTypeMatcher(N.getSimpleType(ResultsToTypeCheck[i]),
-                                    ResultsToTypeCheck[i]));
+  for (unsigned I : ResultsToTypeCheck)
+    AddMatcher(new CheckTypeMatcher(N.getSimpleType(I), I));
 }
 
 /// EmitMatcherCode - Generate the code that matches the predicate of this
@@ -730,7 +726,6 @@ void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode &N,
       // 7 bit and we cannot use StringInteger.
       if (RB.getSubRegIndices().size() > 127) {
         const CodeGenSubRegIndex *I = RB.findSubRegIdx(Def);
-        assert(I && "Cannot find subreg index by name!");
         if (I->EnumValue > 127) {
           AddMatcher(new EmitIntegerMatcher(I->EnumValue, MVT::i32,
                                             NextRecordedOperandNo));
@@ -837,8 +832,8 @@ void MatcherGen::EmitResultInstructionAsOperand(
       // overridden, or which we aren't letting it override; emit the 'default
       // ops' operands.
       const DAGDefaultOperand &DefaultOp = CGP.getDefaultOperand(OperandNode);
-      for (unsigned i = 0, e = DefaultOp.DefaultOps.size(); i != e; ++i)
-        EmitResultOperand(*DefaultOp.DefaultOps[i], InstOps);
+      for (const TreePatternNodePtr &Op : DefaultOp.DefaultOps)
+        EmitResultOperand(*Op, InstOps);
       continue;
     }
 
@@ -887,10 +882,10 @@ void MatcherGen::EmitResultInstructionAsOperand(
   if (isRoot && !PhysRegInputs.empty()) {
     // Emit all of the CopyToReg nodes for the input physical registers.  These
     // occur in patterns like (mul:i8 AL:i8, GR8:i8:$src).
-    for (unsigned i = 0, e = PhysRegInputs.size(); i != e; ++i) {
+    for (const auto &PhysRegInput : PhysRegInputs) {
       const CodeGenRegister *Reg =
-          CGP.getTargetInfo().getRegBank().getReg(PhysRegInputs[i].first);
-      AddMatcher(new EmitCopyToRegMatcher(PhysRegInputs[i].second, Reg));
+          CGP.getTargetInfo().getRegBank().getReg(PhysRegInput.first);
+      AddMatcher(new EmitCopyToRegMatcher(PhysRegInput.second, Reg));
     }
 
     // Even if the node has no other glue inputs, the resultant node must be
@@ -978,8 +973,8 @@ void MatcherGen::EmitResultInstructionAsOperand(
                                  NumFixedArityOperands, NextRecordedOperandNo));
 
   // The non-chain and non-glue results of the newly emitted node get recorded.
-  for (unsigned i = 0, e = ResultVTs.size(); i != e; ++i) {
-    if (ResultVTs[i] == MVT::Other || ResultVTs[i] == MVT::Glue)
+  for (MVT::SimpleValueType ResultVT : ResultVTs) {
+    if (ResultVT == MVT::Other || ResultVT == MVT::Glue)
       break;
     OutputOps.push_back(NextRecordedOperandNo++);
   }
diff --git a/external/llvm-project/llvm/utils/TableGen/DecoderEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/DecoderEmitter.cpp
index 82ba93534cbd..7489d369c993 100644
--- a/external/llvm-project/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -90,9 +90,9 @@ STATISTIC(NumInstructions, "Number of instructions considered");
 STATISTIC(NumEncodingsSupported, "Number of encodings supported");
 STATISTIC(NumEncodingsOmitted, "Number of encodings omitted");
 
-namespace {
+static unsigned getNumToSkipInBytes() { return LargeTable ? 3 : 2; }
 
-unsigned getNumToSkipInBytes() { return LargeTable ? 3 : 2; }
+namespace {
 
 struct EncodingField {
   unsigned Base, Width, Offset;
@@ -104,10 +104,9 @@ struct OperandInfo {
   std::vector<EncodingField> Fields;
   std::string Decoder;
   bool HasCompleteDecoder;
-  uint64_t InitValue;
+  uint64_t InitValue = 0;
 
-  OperandInfo(std::string D, bool HCD)
-      : Decoder(D), HasCompleteDecoder(HCD), InitValue(0) {}
+  OperandInfo(std::string D, bool HCD) : Decoder(D), HasCompleteDecoder(HCD) {}
 
   void addField(unsigned Base, unsigned Width, unsigned Offset) {
     Fields.push_back(EncodingField(Base, Width, Offset));
@@ -182,6 +181,8 @@ struct DecoderTableInfo {
   FixupScopeList FixupStack;
   PredicateSet Predicates;
   DecoderSet Decoders;
+
+  bool isOutermostScope() const { return FixupStack.size() == 1; }
 };
 
 struct EncodingAndInst {
@@ -777,11 +778,21 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
 
       PrevFilter = 0; // Don't re-process the filter's fallthrough.
     } else {
-      Table.push_back(MCD::OPC_FilterValue);
+      // The last filtervalue emitted can be OPC_FilterValue if we are at
+      // outermost scope.
+      const uint8_t DecoderOp =
+          FilterVal == LastFilter && TableInfo.isOutermostScope()
+              ? MCD::OPC_FilterValueOrFail
+              : MCD::OPC_FilterValue;
+      Table.push_back(DecoderOp);
       Table.insertULEB128(FilterVal);
-      // Reserve space for the NumToSkip entry. We'll backpatch the value
-      // later.
-      PrevFilter = Table.insertNumToSkip();
+      if (DecoderOp == MCD::OPC_FilterValue) {
+        // Reserve space for the NumToSkip entry. We'll backpatch the value
+        // later.
+        PrevFilter = Table.insertNumToSkip();
+      } else {
+        PrevFilter = 0;
+      }
     }
 
     // We arrive at a category of instructions with the same segment value.
@@ -796,9 +807,10 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
       Table.patchNumToSkip(PrevFilter, Table.size());
   }
 
-  // If there is no fallthrough, then the final filter should get fixed
-  // up according to the enclosing scope rather than the current position.
-  if (!HasFallthrough)
+  // If there is no fallthrough and the final filter was not in the outermost
+  // scope, then it must be fixed up according to the enclosing scope rather
+  // than the current position.
+  if (PrevFilter)
     TableInfo.FixupStack.back().push_back(PrevFilter);
 }
 
@@ -863,6 +875,15 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
   DecoderTable::const_iterator I = Table.begin();
   DecoderTable::const_iterator E = Table.end();
   const uint8_t *const EndPtr = Table.data() + Table.size();
+
+  auto emitNumToSkipComment = [&](uint32_t NumToSkip, bool InComment = false) {
+    uint32_t Index = ((I - Table.begin()) + NumToSkip);
+    OS << (InComment ? ", " : "// ");
+    OS << "Skip to: " << Index;
+    if (*(I + NumToSkip) == MCD::OPC_Fail)
+      OS << " (Fail)";
+  };
+
   while (I != E) {
     assert(I < E && "incomplete decode table entry!");
 
@@ -873,7 +894,8 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
     const uint8_t DecoderOp = *I++;
     switch (DecoderOp) {
     default:
-      PrintFatalError("Invalid decode table opcode: " + Twine(DecoderOp));
+      PrintFatalError("Invalid decode table opcode: " + Twine((int)DecoderOp) +
+                      " at index " + Twine(Pos));
     case MCD::OPC_ExtractField: {
       OS << Indent << "MCD::OPC_ExtractField, ";
 
@@ -890,17 +912,24 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << Start << "} ...\n";
       break;
     }
-    case MCD::OPC_FilterValue: {
-      OS << Indent << "MCD::OPC_FilterValue, ";
+    case MCD::OPC_FilterValue:
+    case MCD::OPC_FilterValueOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_FilterValueOrFail;
+      OS << Indent << "MCD::OPC_FilterValue" << (IsFail ? "OrFail, " : ", ");
       // The filter value is ULEB128 encoded.
       emitULEB128(I, OS);
 
-      uint32_t NumToSkip = emitNumToSkip(I, OS);
-      OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
+      if (!IsFail) {
+        uint32_t NumToSkip = emitNumToSkip(I, OS);
+        emitNumToSkipComment(NumToSkip);
+      }
+      OS << '\n';
       break;
     }
-    case MCD::OPC_CheckField: {
-      OS << Indent << "MCD::OPC_CheckField, ";
+    case MCD::OPC_CheckField:
+    case MCD::OPC_CheckFieldOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_CheckFieldOrFail;
+      OS << Indent << "MCD::OPC_CheckField" << (IsFail ? "OrFail, " : ", ");
       // ULEB128 encoded start value.
       emitULEB128(I, OS);
       // 8-bit length.
@@ -909,30 +938,44 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       // ULEB128 encoded field value.
       emitULEB128(I, OS);
 
-      uint32_t NumToSkip = emitNumToSkip(I, OS);
-      OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
+      if (!IsFail) {
+        uint32_t NumToSkip = emitNumToSkip(I, OS);
+        emitNumToSkipComment(NumToSkip);
+      }
+      OS << '\n';
       break;
     }
-    case MCD::OPC_CheckPredicate: {
-      OS << Indent << "MCD::OPC_CheckPredicate, ";
+    case MCD::OPC_CheckPredicate:
+    case MCD::OPC_CheckPredicateOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_CheckPredicateOrFail;
+
+      OS << Indent << "MCD::OPC_CheckPredicate" << (IsFail ? "OrFail, " : ", ");
       emitULEB128(I, OS);
 
-      uint32_t NumToSkip = emitNumToSkip(I, OS);
-      OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
+      if (!IsFail) {
+        uint32_t NumToSkip = emitNumToSkip(I, OS);
+        emitNumToSkipComment(NumToSkip);
+      }
+      OS << '\n';
       break;
     }
     case MCD::OPC_Decode:
-    case MCD::OPC_TryDecode: {
-      bool IsTry = DecoderOp == MCD::OPC_TryDecode;
+    case MCD::OPC_TryDecode:
+    case MCD::OPC_TryDecodeOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_TryDecodeOrFail;
+      bool IsTry = DecoderOp == MCD::OPC_TryDecode || IsFail;
       // Decode the Opcode value.
       const char *ErrMsg = nullptr;
       unsigned Opc = decodeULEB128(&*I, nullptr, EndPtr, &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
 
-      OS << Indent << "MCD::OPC_" << (IsTry ? "Try" : "") << "Decode, ";
+      OS << Indent << "MCD::OPC_" << (IsTry ? "Try" : "") << "Decode"
+         << (IsFail ? "OrFail, " : ", ");
       emitULEB128(I, OS);
 
       // Decoder index.
+      unsigned DecodeIdx = decodeULEB128(&*I, nullptr, EndPtr, &ErrMsg);
+      assert(ErrMsg == nullptr && "ULEB128 value too large!");
       emitULEB128(I, OS);
 
       auto EncI = OpcodeToEncodingID.find(Opc);
@@ -940,16 +983,19 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       auto EncodingID = EncI->second;
 
       if (!IsTry) {
-        OS << "// Opcode: " << NumberedEncodings[EncodingID] << "\n";
+        OS << "// Opcode: " << NumberedEncodings[EncodingID]
+           << ", DecodeIdx: " << DecodeIdx << '\n';
         break;
       }
 
       // Fallthrough for OPC_TryDecode.
-
-      uint32_t NumToSkip = emitNumToSkip(I, OS);
-
-      OS << "// Opcode: " << NumberedEncodings[EncodingID]
-         << ", skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
+      if (!IsFail) {
+        uint32_t NumToSkip = emitNumToSkip(I, OS);
+        OS << "// Opcode: " << NumberedEncodings[EncodingID]
+           << ", DecodeIdx: " << DecodeIdx;
+        emitNumToSkipComment(NumToSkip, /*InComment=*/true);
+      }
+      OS << '\n';
       break;
     }
     case MCD::OPC_SoftFail: {
@@ -971,11 +1017,10 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       OS << '\n';
       break;
     }
-    case MCD::OPC_Fail: {
+    case MCD::OPC_Fail:
       OS << Indent << "MCD::OPC_Fail,\n";
       break;
     }
-    }
   }
   OS << Indent << "0\n";
 
@@ -1337,11 +1382,16 @@ void FilterChooser::emitPredicateTableEntry(DecoderTableInfo &TableInfo,
   // computed.
   unsigned PIdx = getPredicateIndex(TableInfo, PS.str());
 
-  TableInfo.Table.push_back(MCD::OPC_CheckPredicate);
+  const uint8_t DecoderOp = TableInfo.isOutermostScope()
+                                ? MCD::OPC_CheckPredicateOrFail
+                                : MCD::OPC_CheckPredicate;
+  TableInfo.Table.push_back(DecoderOp);
   TableInfo.Table.insertULEB128(PIdx);
 
-  // Push location for NumToSkip backpatching.
-  TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
+  if (DecoderOp == MCD::OPC_CheckPredicate) {
+    // Push location for NumToSkip backpatching.
+    TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
+  }
 }
 
 void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo,
@@ -1411,17 +1461,22 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
   for (const Island &Ilnd : reverse(Islands)) {
     unsigned NumBits = Ilnd.NumBits;
     assert(isUInt<8>(NumBits) && "NumBits overflowed uint8 table entry!");
-    TableInfo.Table.push_back(MCD::OPC_CheckField);
+    const uint8_t DecoderOp = TableInfo.isOutermostScope()
+                                  ? MCD::OPC_CheckFieldOrFail
+                                  : MCD::OPC_CheckField;
+    TableInfo.Table.push_back(DecoderOp);
 
     TableInfo.Table.insertULEB128(Ilnd.StartBit);
     TableInfo.Table.push_back(NumBits);
     TableInfo.Table.insertULEB128(Ilnd.FieldVal);
 
-    // Allocate space in the table for fixup so all our relative position
-    // calculations work OK even before we fully resolve the real value here.
+    if (DecoderOp == MCD::OPC_CheckField) {
+      // Allocate space in the table for fixup so all our relative position
+      // calculations work OK even before we fully resolve the real value here.
 
-    // Push location for NumToSkip backpatching.
-    TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
+      // Push location for NumToSkip backpatching.
+      TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
+    }
   }
 
   // Check for soft failure of the match.
@@ -1439,13 +1494,16 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
   // decoder method indicates that additional processing should be done to see
   // if there is any other instruction that also matches the bitpattern and
   // can decode it.
-  TableInfo.Table.push_back(HasCompleteDecoder ? MCD::OPC_Decode
-                                               : MCD::OPC_TryDecode);
+  const uint8_t DecoderOp = HasCompleteDecoder ? MCD::OPC_Decode
+                                               : (TableInfo.isOutermostScope()
+                                                      ? MCD::OPC_TryDecodeOrFail
+                                                      : MCD::OPC_TryDecode);
+  TableInfo.Table.push_back(DecoderOp);
   NumEncodingsSupported++;
   TableInfo.Table.insertULEB128(Opc.Opcode);
   TableInfo.Table.insertULEB128(DIdx);
 
-  if (!HasCompleteDecoder) {
+  if (DecoderOp == MCD::OPC_TryDecode) {
     // Push location for NumToSkip backpatching.
     TableInfo.FixupStack.back().push_back(TableInfo.Table.insertNumToSkip());
   }
@@ -2188,9 +2246,11 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
   DecodeStatus S = MCDisassembler::Success;
   while (true) {
     ptrdiff_t Loc = Ptr - DecodeTable;
-    switch (*Ptr++) {
+    const uint8_t DecoderOp = *Ptr++;
+    switch (DecoderOp) {
     default:
-      errs() << Loc << ": Unexpected decode table opcode!\n";
+      errs() << Loc << ": Unexpected decode table opcode: "
+             << (int)DecoderOp << '\n';
       return MCDisassembler::Fail;
     case MCD::OPC_ExtractField: {
       // Decode the start value.
@@ -2204,22 +2264,34 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
                    << Len << "): " << CurFieldValue << "\n");
       break;
     }
-    case MCD::OPC_FilterValue: {
+    case MCD::OPC_FilterValue:
+    case MCD::OPC_FilterValueOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_FilterValueOrFail;
       // Decode the field value.
       uint64_t Val = decodeULEB128AndIncUnsafe(Ptr);
       bool Failed = Val != CurFieldValue;
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      unsigned NumToSkip = IsFail ? 0 : decodeNumToSkip(Ptr);
+
+      // Note: Print NumToSkip even for OPC_FilterValueOrFail to simplify debug
+      // prints.
+      LLVM_DEBUG({
+        StringRef OpName = IsFail ? "OPC_FilterValueOrFail" : "OPC_FilterValue";
+        dbgs() << Loc << ": " << OpName << '(' << Val << ", " << NumToSkip
+                << ") " << (Failed ? "FAIL:" : "PASS:")
+                << " continuing at " << (Ptr - DecodeTable) << '\n';
+      });
 
       // Perform the filter operation.
-      if (Failed)
+      if (Failed) {
+        if (IsFail)
+          return MCDisassembler::Fail;
         Ptr += NumToSkip;
-      LLVM_DEBUG(dbgs() << Loc << ": OPC_FilterValue(" << Val << ", " << NumToSkip
-                   << "): " << (Failed ? "FAIL:" : "PASS:")
-                   << " continuing at " << (Ptr - DecodeTable) << "\n");
-
+      }
       break;
     }
-    case MCD::OPC_CheckField: {
+    case MCD::OPC_CheckField:
+    case MCD::OPC_CheckFieldOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_CheckFieldOrFail;
       // Decode the start value.
       unsigned Start = decodeULEB128AndIncUnsafe(Ptr);
       unsigned Len = *Ptr;)";
@@ -2232,29 +2304,44 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       uint64_t ExpectedValue = decodeULEB128(++Ptr, &PtrLen);
       Ptr += PtrLen;
       bool Failed = ExpectedValue != FieldValue;
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
-
-      // If the actual and expected values don't match, skip.
-      if (Failed)
+      unsigned NumToSkip = IsFail ? 0 : decodeNumToSkip(Ptr);
+
+      LLVM_DEBUG({
+        StringRef OpName = IsFail ? "OPC_CheckFieldOrFail" : "OPC_CheckField";
+        dbgs() << Loc << ": " << OpName << '(' << Start << ", " << Len << ", "
+                << ExpectedValue << ", " << NumToSkip << "): FieldValue = "
+                << FieldValue << ", ExpectedValue = " << ExpectedValue << ": "
+                << (Failed ? "FAIL\n" : "PASS\n");
+      });
+
+      // If the actual and expected values don't match, skip or fail.
+      if (Failed) {
+        if (IsFail)
+          return MCDisassembler::Fail;
         Ptr += NumToSkip;
-      LLVM_DEBUG(dbgs() << Loc << ": OPC_CheckField(" << Start << ", "
-                   << Len << ", " << ExpectedValue << ", " << NumToSkip
-                   << "): FieldValue = " << FieldValue << ", ExpectedValue = "
-                   << ExpectedValue << ": "
-                   << (Failed ? "FAIL\n" : "PASS\n"));
+      }
       break;
     }
-    case MCD::OPC_CheckPredicate: {
+    case MCD::OPC_CheckPredicate:
+    case MCD::OPC_CheckPredicateOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_CheckPredicateOrFail;
       // Decode the Predicate Index value.
       unsigned PIdx = decodeULEB128AndIncUnsafe(Ptr);
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      unsigned NumToSkip = IsFail ? 0 : decodeNumToSkip(Ptr);
       // Check the predicate.
       bool Failed = !checkDecoderPredicate(PIdx, Bits);
-      if (Failed)
-        Ptr += NumToSkip;
-      LLVM_DEBUG(dbgs() << Loc << ": OPC_CheckPredicate(" << PIdx << "): "
-            << (Failed ? "FAIL\n" : "PASS\n"));
 
+      LLVM_DEBUG({
+        StringRef OpName = IsFail ? "OPC_CheckPredicateOrFail" : "OPC_CheckPredicate";
+        dbgs() << Loc << ": " << OpName << '(' << PIdx << ", " << NumToSkip
+               << "): " << (Failed ? "FAIL\n" : "PASS\n");
+      });
+
+      if (Failed) {
+        if (IsFail)
+          return MCDisassembler::Fail;
+        Ptr += NumToSkip;
+      }
       break;
     }
     case MCD::OPC_Decode: {
@@ -2275,14 +2362,16 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
 
       LLVM_DEBUG(dbgs() << Loc << ": OPC_Decode: opcode " << Opc
                    << ", using decoder " << DecodeIdx << ": "
-                   << (S != MCDisassembler::Fail ? "PASS" : "FAIL") << "\n");
+                   << (S != MCDisassembler::Fail ? "PASS\n" : "FAIL\n"));
       return S;
     }
-    case MCD::OPC_TryDecode: {
+    case MCD::OPC_TryDecode:
+    case MCD::OPC_TryDecodeOrFail: {
+      bool IsFail = DecoderOp == MCD::OPC_TryDecodeOrFail;
       // Decode the Opcode value.
       unsigned Opc = decodeULEB128AndIncUnsafe(Ptr);
       unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
-      unsigned NumToSkip = decodeNumToSkip(Ptr);
+      unsigned NumToSkip = IsFail ? 0 : decodeNumToSkip(Ptr);
 
       // Perform the decode operation.
       MCInst TmpMI;
@@ -2294,28 +2383,31 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
 
       if (DecodeComplete) {
         // Decoding complete.
-        LLVM_DEBUG(dbgs() << (S != MCDisassembler::Fail ? "PASS" : "FAIL") << "\n");
+        LLVM_DEBUG(dbgs() << (S != MCDisassembler::Fail ? "PASS\n" : "FAIL\n"));
         MI = TmpMI;
         return S;
-      } else {
-        assert(S == MCDisassembler::Fail);
-        // If the decoding was incomplete, skip.
-        Ptr += NumToSkip;
-        LLVM_DEBUG(dbgs() << "FAIL: continuing at " << (Ptr - DecodeTable) << "\n");
-        // Reset decode status. This also drops a SoftFail status that could be
-        // set before the decode attempt.
-        S = MCDisassembler::Success;
       }
+      assert(S == MCDisassembler::Fail);
+      if (IsFail) {
+        LLVM_DEBUG(dbgs() << "FAIL: returning FAIL\n");
+        return MCDisassembler::Fail;
+      }
+      // If the decoding was incomplete, skip.
+      Ptr += NumToSkip;
+      LLVM_DEBUG(dbgs() << "FAIL: continuing at " << (Ptr - DecodeTable) << "\n");
+      // Reset decode status. This also drops a SoftFail status that could be
+      // set before the decode attempt.
+      S = MCDisassembler::Success;
       break;
     }
     case MCD::OPC_SoftFail: {
       // Decode the mask values.
       uint64_t PositiveMask = decodeULEB128AndIncUnsafe(Ptr);
       uint64_t NegativeMask = decodeULEB128AndIncUnsafe(Ptr);
-      bool Fail = (insn & PositiveMask) != 0 || (~insn & NegativeMask) != 0;
-      if (Fail)
+      bool Failed = (insn & PositiveMask) != 0 || (~insn & NegativeMask) != 0;
+      if (Failed)
         S = MCDisassembler::SoftFail;
-      LLVM_DEBUG(dbgs() << Loc << ": OPC_SoftFail: " << (Fail ? "FAIL\n" : "PASS\n"));
+      LLVM_DEBUG(dbgs() << Loc << ": OPC_SoftFail: " << (Failed ? "FAIL\n" : "PASS\n"));
       break;
     }
     case MCD::OPC_Fail: {
diff --git a/external/llvm-project/llvm/utils/TableGen/FastISelEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/FastISelEmitter.cpp
index a8b6f79c176a..694d89a5ada3 100644
--- a/external/llvm-project/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -86,10 +86,10 @@ namespace {
 struct OperandsSignature {
   class OpKind {
     enum { OK_Reg, OK_FP, OK_Imm, OK_Invalid = -1 };
-    char Repr;
+    char Repr = OK_Invalid;
 
   public:
-    OpKind() : Repr(OK_Invalid) {}
+    OpKind() {}
 
     bool operator<(OpKind RHS) const { return Repr < RHS.Repr; }
     bool operator==(OpKind RHS) const { return Repr == RHS.Repr; }
diff --git a/external/llvm-project/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index 090e6bb37c3a..f62b26587cf4 100644
--- a/external/llvm-project/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -1161,7 +1161,7 @@ bool CombineRuleBuilder::buildPermutationsToEmit() {
     PermutationsToEmit.clear();
 
     for (const auto &Perm : CurPerms) {
-      assert(!Perm.count(Pat.get()) && "Pattern already emitted?");
+      assert(!Perm.contains(Pat.get()) && "Pattern already emitted?");
       for (unsigned K = 0; K < NumAlts; ++K) {
         PatternAlternatives NewPerm = Perm;
         NewPerm[Pat.get()] = K;
@@ -2414,6 +2414,7 @@ class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter {
   void emitAdditionalImpl(raw_ostream &OS) override;
 
   void emitMIPredicateFns(raw_ostream &OS) override;
+  void emitLeafPredicateFns(raw_ostream &OS) override;
   void emitI64ImmPredicateFns(raw_ostream &OS) override;
   void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
   void emitAPIntImmPredicateFns(raw_ostream &OS) override;
@@ -2581,6 +2582,12 @@ void GICombinerEmitter::emitMIPredicateFns(raw_ostream &OS) {
       [](const CXXPredicateCode *C) -> StringRef { return C->Code; });
 }
 
+void GICombinerEmitter::emitLeafPredicateFns(raw_ostream &OS) {
+  // Unused, but still needs to be called.
+  emitLeafPredicateFnsImpl<unsigned>(
+      OS, "", {}, [](unsigned) { return ""; }, [](unsigned) { return ""; });
+}
+
 void GICombinerEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
   // Unused, but still needs to be called.
   emitImmPredicateFnsImpl<unsigned>(
diff --git a/external/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp
index edaf5299efc3..6772043c2105 100644
--- a/external/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -321,6 +321,7 @@ class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
   void emitAdditionalImpl(raw_ostream &OS) override;
 
   void emitMIPredicateFns(raw_ostream &OS) override;
+  void emitLeafPredicateFns(raw_ostream &OS) override;
   void emitI64ImmPredicateFns(raw_ostream &OS) override;
   void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
   void emitAPIntImmPredicateFns(raw_ostream &OS) override;
@@ -1110,8 +1111,16 @@ Error GlobalISelEmitter::importChildMatcher(
     return Error::success();
   }
 
-  if (SrcChild.hasAnyPredicate())
-    return failedImport("Src pattern child has unsupported predicate");
+  if (SrcChild.hasAnyPredicate()) {
+    for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
+      const TreePredicateFn &Predicate = Call.Fn;
+
+      if (!Predicate.hasGISelLeafPredicateCode())
+        return failedImport("Src pattern child has unsupported predicate");
+      OM.addPredicate<OperandLeafPredicateMatcher>(Predicate);
+    }
+    return Error::success();
+  }
 
   // Check for constant immediates.
   if (auto *ChildInt = dyn_cast<IntInit>(SrcChild.getLeafValue())) {
@@ -1333,7 +1342,7 @@ Error GlobalISelEmitter::importLeafNodeRenderer(
     }
 
     if (R->isSubClassOf("SubRegIndex")) {
-      const CodeGenSubRegIndex *SubRegIndex = CGRegs.getSubRegIdx(R);
+      const CodeGenSubRegIndex *SubRegIndex = CGRegs.findSubRegIdx(R);
       MIBuilder.addRenderer<ImmRenderer>(SubRegIndex->EnumValue);
       return Error::success();
     }
@@ -1606,7 +1615,8 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
     if (!SubRegInit)
       return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
 
-    CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
+    const CodeGenSubRegIndex *SubIdx =
+        CGRegs.findSubRegIdx(SubRegInit->getDef());
     const TreePatternNode &ValChild = Dst.getChild(0);
     if (!ValChild.isLeaf()) {
       // We really have to handle the source instruction, and then insert a
@@ -1675,7 +1685,8 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
 
       if (const DefInit *SubRegInit =
               dyn_cast<DefInit>(SubRegChild.getLeafValue())) {
-        CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
+        const CodeGenSubRegIndex *SubIdx =
+            CGRegs.findSubRegIdx(SubRegInit->getDef());
 
         if (Error Err = importNodeRenderer(M, DstMIBuilder, ValChild, InsertPt))
           return Err;
@@ -2004,11 +2015,11 @@ const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClass(
   const DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode.getLeafValue());
   if (!SubRegInit)
     return nullptr;
-  const CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
+  const CodeGenSubRegIndex *SubIdx = CGRegs.findSubRegIdx(SubRegInit->getDef());
 
   // Use the information we found above to find a minimal register class which
   // supports the subregister and type we want.
-  return Target.getSuperRegForSubReg(Ty.getValueTypeByHwMode(), CGRegs, SubIdx,
+  return CGRegs.getSuperRegForSubReg(Ty.getValueTypeByHwMode(), SubIdx,
                                      /*MustBeAllocatable=*/true);
 }
 
@@ -2034,7 +2045,7 @@ const CodeGenSubRegIndex *GlobalISelEmitter::inferSubRegIndexForNode(
   const DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode.getLeafValue());
   if (!SubRegInit)
     return nullptr;
-  return CGRegs.getSubRegIdx(SubRegInit->getDef());
+  return CGRegs.findSubRegIdx(SubRegInit->getDef());
 }
 
 Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
@@ -2291,6 +2302,27 @@ void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
       "PatFrag predicates.");
 }
 
+void GlobalISelEmitter::emitLeafPredicateFns(raw_ostream &OS) {
+  std::vector<const Record *> MatchedRecords;
+  llvm::copy_if(AllPatFrags, std::back_inserter(MatchedRecords),
+                [](const Record *R) {
+                  return (!R->getValueAsOptionalString("GISelLeafPredicateCode")
+                               .value_or(std::string())
+                               .empty());
+                });
+  emitLeafPredicateFnsImpl<const Record *>(
+      OS,
+      "  const auto &Operands = State.RecordedOperands;\n"
+      "  Register Reg = MO.getReg();\n"
+      "  (void)Operands;\n"
+      "  (void)Reg;",
+      ArrayRef<const Record *>(MatchedRecords), &getPatFragPredicateEnumName,
+      [](const Record *R) {
+        return R->getValueAsString("GISelLeafPredicateCode");
+      },
+      "PatFrag predicates.");
+}
+
 void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
   std::vector<const Record *> MatchedRecords;
   llvm::copy_if(AllPatFrags, std::back_inserter(MatchedRecords),
diff --git a/external/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp
index ef7620a32331..b6f16789bf95 100644
--- a/external/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -203,7 +203,7 @@ InstrInfoEmitter::CollectOperandInfo(OperandInfoListTy &OperandInfoList,
   unsigned Offset = 0;
   for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
     OperandInfoTy OperandInfo = GetOperandInfo(*Inst);
-    if (OperandInfoMap.insert({OperandInfo, Offset}).second) {
+    if (OperandInfoMap.try_emplace(OperandInfo, Offset).second) {
       OperandInfoList.push_back(OperandInfo);
       Offset += OperandInfo.size();
     }
@@ -503,7 +503,8 @@ void InstrInfoEmitter::emitLogicalOperandSizeMappings(
     LogicalOpListSize = std::max(LogicalOpList.size(), LogicalOpListSize);
 
     auto I =
-        LogicalOpSizeMap.insert({LogicalOpList, LogicalOpSizeMap.size()}).first;
+        LogicalOpSizeMap.try_emplace(LogicalOpList, LogicalOpSizeMap.size())
+            .first;
     InstMap[I->second].push_back(
         (Namespace + "::" + Inst->TheDef->getName()).str());
   }
@@ -850,7 +851,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
     std::vector<const Record *> ImplicitOps = Inst->ImplicitUses;
     llvm::append_range(ImplicitOps, Inst->ImplicitDefs);
-    if (EmittedLists.insert({ImplicitOps, ImplicitListSize}).second) {
+    if (EmittedLists.try_emplace(ImplicitOps, ImplicitListSize).second) {
       ImplicitLists.push_back(ImplicitOps);
       ImplicitListSize += ImplicitOps.size();
     }
diff --git a/external/llvm-project/llvm/utils/TableGen/OptionParserEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/OptionParserEmitter.cpp
index 85b220070d1f..f4964282934e 100644
--- a/external/llvm-project/llvm/utils/TableGen/OptionParserEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/OptionParserEmitter.cpp
@@ -264,11 +264,11 @@ static void emitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
   typedef SmallVector<SmallString<2>, 2> PrefixKeyT;
   typedef std::map<PrefixKeyT, unsigned> PrefixesT;
   PrefixesT Prefixes;
-  Prefixes.insert({PrefixKeyT(), 0});
+  Prefixes.try_emplace(PrefixKeyT(), 0);
   for (const Record &R : llvm::make_pointee_range(Opts)) {
     std::vector<StringRef> RPrefixes = R.getValueAsListOfStrings("Prefixes");
     PrefixKeyT PrefixKey(RPrefixes.begin(), RPrefixes.end());
-    Prefixes.insert({PrefixKey, 0});
+    Prefixes.try_emplace(PrefixKey, 0);
   }
 
   DenseSet<StringRef> PrefixesUnionSet;
diff --git a/external/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/external/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 6392514bf415..7d24c0f80cdd 100644
--- a/external/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -726,11 +726,12 @@ void RegisterInfoEmitter::emitComposeSubRegIndices(raw_ostream &OS,
   // Output the rows.
   OS << "  static const " << getMinimalTypeForRange(SubRegIndicesSize + 1, 32)
      << " Rows[" << Rows.size() << "][" << SubRegIndicesSize << "] = {\n";
-  for (unsigned r = 0, re = Rows.size(); r != re; ++r) {
+  for (const auto &Row : Rows) {
     OS << "    { ";
-    for (unsigned i = 0, e = SubRegIndicesSize; i != e; ++i)
-      if (Rows[r][i])
-        OS << Rows[r][i]->getQualifiedName() << ", ";
+    for (const llvm::CodeGenSubRegIndex *Elem :
+         ArrayRef(&Row[0], SubRegIndicesSize))
+      if (Elem)
+        OS << Elem->getQualifiedName() << ", ";
       else
         OS << "0, ";
     OS << "},\n";
@@ -830,8 +831,7 @@ void RegisterInfoEmitter::emitComposeSubRegIndexLaneMask(raw_ostream &OS,
   for (size_t s = 0, se = Sequences.size(); s != se; ++s) {
     OS << "    ";
     const SmallVectorImpl<MaskRolPair> &Sequence = Sequences[s];
-    for (size_t p = 0, pe = Sequence.size(); p != pe; ++p) {
-      const MaskRolPair &P = Sequence[p];
+    for (const MaskRolPair &P : Sequence) {
       printMask(OS << "{ ", P.Mask);
       OS << format(", %2u }, ", P.RotateLeft);
     }
@@ -1403,10 +1403,10 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS) {
     for (const auto &RC : RegisterClasses) {
       if (!RC.AltOrderSelect.empty()) {
         OS << "\nstatic inline unsigned " << RC.getName()
-           << "AltOrderSelect(const MachineFunction &MF) {" << RC.AltOrderSelect
-           << "}\n\n"
+           << "AltOrderSelect(const MachineFunction &MF, bool Rev) {"
+           << RC.AltOrderSelect << "}\n\n"
            << "static ArrayRef<MCPhysReg> " << RC.getName()
-           << "GetRawAllocationOrder(const MachineFunction &MF) {\n";
+           << "GetRawAllocationOrder(const MachineFunction &MF, bool Rev) {\n";
         for (unsigned oi = 1, oe = RC.getNumOrders(); oi != oe; ++oi) {
           ArrayRef<const Record *> Elems = RC.getOrder(oi);
           if (!Elems.empty()) {
@@ -1426,8 +1426,8 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS) {
           else
             OS << "),\n    ArrayRef(AltOrder" << oi;
         OS << ")\n  };\n  const unsigned Select = " << RC.getName()
-           << "AltOrderSelect(MF);\n  assert(Select < " << RC.getNumOrders()
-           << ");\n  return Order[Select];\n}\n";
+           << "AltOrderSelect(MF, Rev);\n  assert(Select < "
+           << RC.getNumOrders() << ");\n  return Order[Select];\n}\n";
       }
     }
 
diff --git a/external/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp b/external/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp
index 36f752a1ebe6..3c422a32dcaf 100644
--- a/external/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp
@@ -882,9 +882,9 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o,
     N = ++OperandSetNum;
 
     o << "  { /* " << (OperandSetNum - 1) << " */\n";
-    for (unsigned i = 0, e = OperandList.size(); i != e; ++i) {
-      const char *Encoding = stringForOperandEncoding(OperandList[i].first);
-      const char *Type = stringForOperandType(OperandList[i].second);
+    for (const auto &[Enc, Ty] : OperandList) {
+      const char *Encoding = stringForOperandEncoding(Enc);
+      const char *Type = stringForOperandType(Ty);
       o << "    { " << Encoding << ", " << Type << " },\n";
     }
     o << "  },\n";
diff --git a/external/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp b/external/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp
index f93e5fbcc4c2..a56e07a8939e 100644
--- a/external/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp
+++ b/external/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp
@@ -1038,131 +1038,134 @@ OperandType RecognizableInstr::typeFromString(StringRef Str, bool hasREX_W,
     // immediate encoding is special.
     Switch.Case("GR32", TYPE_Rv);
   }
-  OperandType Type = Switch.Case("i16mem", TYPE_M)
-                         .Case("i16imm", TYPE_IMM)
-                         .Case("i16i8imm", TYPE_IMM)
-                         .Case("GR16", TYPE_R16)
-                         .Case("GR16orGR32orGR64", TYPE_R16)
-                         .Case("i32mem", TYPE_M)
-                         .Case("i32imm", TYPE_IMM)
-                         .Case("i32i8imm", TYPE_IMM)
-                         .Case("GR32", TYPE_R32)
-                         .Case("GR32orGR64", TYPE_R32)
-                         .Case("i64mem", TYPE_M)
-                         .Case("i64i32imm", TYPE_IMM)
-                         .Case("i64i8imm", TYPE_IMM)
-                         .Case("GR64", TYPE_R64)
-                         .Case("i8mem", TYPE_M)
-                         .Case("i8imm", TYPE_IMM)
-                         .Case("u4imm", TYPE_UIMM8)
-                         .Case("u8imm", TYPE_UIMM8)
-                         .Case("i16u8imm", TYPE_UIMM8)
-                         .Case("i32u8imm", TYPE_UIMM8)
-                         .Case("i64u8imm", TYPE_UIMM8)
-                         .Case("GR8", TYPE_R8)
-                         .Case("VR128", TYPE_XMM)
-                         .Case("VR128X", TYPE_XMM)
-                         .Case("f128mem", TYPE_M)
-                         .Case("f256mem", TYPE_M)
-                         .Case("f512mem", TYPE_M)
-                         .Case("FR128", TYPE_XMM)
-                         .Case("FR64", TYPE_XMM)
-                         .Case("FR64X", TYPE_XMM)
-                         .Case("f64mem", TYPE_M)
-                         .Case("sdmem", TYPE_M)
-                         .Case("FR16X", TYPE_XMM)
-                         .Case("FR32", TYPE_XMM)
-                         .Case("FR32X", TYPE_XMM)
-                         .Case("f32mem", TYPE_M)
-                         .Case("f16mem", TYPE_M)
-                         .Case("ssmem", TYPE_M)
-                         .Case("shmem", TYPE_M)
-                         .Case("RST", TYPE_ST)
-                         .Case("RSTi", TYPE_ST)
-                         .Case("i128mem", TYPE_M)
-                         .Case("i256mem", TYPE_M)
-                         .Case("i512mem", TYPE_M)
-                         .Case("i512mem_GR16", TYPE_M)
-                         .Case("i512mem_GR32", TYPE_M)
-                         .Case("i512mem_GR64", TYPE_M)
-                         .Case("i64i32imm_brtarget", TYPE_REL)
-                         .Case("i8imm_brtarget", TYPE_REL)
-                         .Case("i16imm_brtarget", TYPE_REL)
-                         .Case("i32imm_brtarget", TYPE_REL)
-                         .Case("ccode", TYPE_IMM)
-                         .Case("cflags", TYPE_IMM)
-                         .Case("AVX512RC", TYPE_IMM)
-                         .Case("brtarget32", TYPE_REL)
-                         .Case("brtarget16", TYPE_REL)
-                         .Case("brtarget8", TYPE_REL)
-                         .Case("f80mem", TYPE_M)
-                         .Case("lea64_8mem", TYPE_M)
-                         .Case("lea64_16mem", TYPE_M)
-                         .Case("lea64_32mem", TYPE_M)
-                         .Case("lea64mem", TYPE_M)
-                         .Case("VR64", TYPE_MM64)
-                         .Case("i64imm", TYPE_IMM)
-                         .Case("anymem", TYPE_M)
-                         .Case("opaquemem", TYPE_M)
-                         .Case("sibmem", TYPE_MSIB)
-                         .Case("SEGMENT_REG", TYPE_SEGMENTREG)
-                         .Case("DEBUG_REG", TYPE_DEBUGREG)
-                         .Case("CONTROL_REG", TYPE_CONTROLREG)
-                         .Case("srcidx8", TYPE_SRCIDX)
-                         .Case("srcidx16", TYPE_SRCIDX)
-                         .Case("srcidx32", TYPE_SRCIDX)
-                         .Case("srcidx64", TYPE_SRCIDX)
-                         .Case("dstidx8", TYPE_DSTIDX)
-                         .Case("dstidx16", TYPE_DSTIDX)
-                         .Case("dstidx32", TYPE_DSTIDX)
-                         .Case("dstidx64", TYPE_DSTIDX)
-                         .Case("offset16_8", TYPE_MOFFS)
-                         .Case("offset16_16", TYPE_MOFFS)
-                         .Case("offset16_32", TYPE_MOFFS)
-                         .Case("offset32_8", TYPE_MOFFS)
-                         .Case("offset32_16", TYPE_MOFFS)
-                         .Case("offset32_32", TYPE_MOFFS)
-                         .Case("offset32_64", TYPE_MOFFS)
-                         .Case("offset64_8", TYPE_MOFFS)
-                         .Case("offset64_16", TYPE_MOFFS)
-                         .Case("offset64_32", TYPE_MOFFS)
-                         .Case("offset64_64", TYPE_MOFFS)
-                         .Case("VR256", TYPE_YMM)
-                         .Case("VR256X", TYPE_YMM)
-                         .Case("VR512", TYPE_ZMM)
-                         .Case("VK1", TYPE_VK)
-                         .Case("VK1WM", TYPE_VK)
-                         .Case("VK2", TYPE_VK)
-                         .Case("VK2WM", TYPE_VK)
-                         .Case("VK4", TYPE_VK)
-                         .Case("VK4WM", TYPE_VK)
-                         .Case("VK8", TYPE_VK)
-                         .Case("VK8WM", TYPE_VK)
-                         .Case("VK16", TYPE_VK)
-                         .Case("VK16WM", TYPE_VK)
-                         .Case("VK32", TYPE_VK)
-                         .Case("VK32WM", TYPE_VK)
-                         .Case("VK64", TYPE_VK)
-                         .Case("VK64WM", TYPE_VK)
-                         .Case("VK1Pair", TYPE_VK_PAIR)
-                         .Case("VK2Pair", TYPE_VK_PAIR)
-                         .Case("VK4Pair", TYPE_VK_PAIR)
-                         .Case("VK8Pair", TYPE_VK_PAIR)
-                         .Case("VK16Pair", TYPE_VK_PAIR)
-                         .Case("vx32mem", TYPE_MVSIBX)
-                         .Case("vx64mem", TYPE_MVSIBX)
-                         .Case("vy32mem", TYPE_MVSIBY)
-                         .Case("vy64mem", TYPE_MVSIBY)
-                         .Case("vx32xmem", TYPE_MVSIBX)
-                         .Case("vx64xmem", TYPE_MVSIBX)
-                         .Case("vy32xmem", TYPE_MVSIBY)
-                         .Case("vy64xmem", TYPE_MVSIBY)
-                         .Case("vz32mem", TYPE_MVSIBZ)
-                         .Case("vz64mem", TYPE_MVSIBZ)
-                         .Case("BNDR", TYPE_BNDR)
-                         .Case("TILE", TYPE_TMM)
-                         .Case("TILEPair", TYPE_TMM_PAIR)
-                         .Default(TYPE_NONE);
+  // clang-format off
+  OperandType Type =
+      Switch.Case("i16mem", TYPE_M)
+          .Case("i16imm", TYPE_IMM)
+          .Case("i16i8imm", TYPE_IMM)
+          .Case("GR16", TYPE_R16)
+          .Case("GR16orGR32orGR64", TYPE_R16)
+          .Case("i32mem", TYPE_M)
+          .Case("i32imm", TYPE_IMM)
+          .Case("i32i8imm", TYPE_IMM)
+          .Case("GR32", TYPE_R32)
+          .Case("GR32orGR64", TYPE_R32)
+          .Case("i64mem", TYPE_M)
+          .Case("i64i32imm", TYPE_IMM)
+          .Case("i64i8imm", TYPE_IMM)
+          .Case("GR64", TYPE_R64)
+          .Case("i8mem", TYPE_M)
+          .Case("i8imm", TYPE_IMM)
+          .Case("u4imm", TYPE_UIMM8)
+          .Case("u8imm", TYPE_UIMM8)
+          .Case("i16u8imm", TYPE_UIMM8)
+          .Case("i32u8imm", TYPE_UIMM8)
+          .Case("i64u8imm", TYPE_UIMM8)
+          .Case("GR8", TYPE_R8)
+          .Case("VR128", TYPE_XMM)
+          .Case("VR128X", TYPE_XMM)
+          .Case("f128mem", TYPE_M)
+          .Case("f256mem", TYPE_M)
+          .Case("f512mem", TYPE_M)
+          .Case("FR128", TYPE_XMM)
+          .Case("FR64", TYPE_XMM)
+          .Case("FR64X", TYPE_XMM)
+          .Case("f64mem", TYPE_M)
+          .Case("sdmem", TYPE_M)
+          .Case("FR16X", TYPE_XMM)
+          .Case("FR32", TYPE_XMM)
+          .Case("FR32X", TYPE_XMM)
+          .Case("f32mem", TYPE_M)
+          .Case("f16mem", TYPE_M)
+          .Case("ssmem", TYPE_M)
+          .Case("shmem", TYPE_M)
+          .Case("RST", TYPE_ST)
+          .Case("RSTi", TYPE_ST)
+          .Case("i128mem", TYPE_M)
+          .Case("i256mem", TYPE_M)
+          .Case("i512mem", TYPE_M)
+          .Case("i512mem_GR16", TYPE_M)
+          .Case("i512mem_GR32", TYPE_M)
+          .Case("i512mem_GR64", TYPE_M)
+          .Case("i64i32imm_brtarget", TYPE_REL)
+          .Case("i8imm_brtarget", TYPE_REL)
+          .Case("i16imm_brtarget", TYPE_REL)
+          .Case("i32imm_brtarget", TYPE_REL)
+          .Case("ccode", TYPE_IMM)
+          .Case("cflags", TYPE_IMM)
+          .Case("AVX512RC", TYPE_IMM)
+          .Case("brtarget32", TYPE_REL)
+          .Case("brtarget16", TYPE_REL)
+          .Case("brtarget8", TYPE_REL)
+          .Case("f80mem", TYPE_M)
+          .Case("lea64_8mem", TYPE_M)
+          .Case("lea64_16mem", TYPE_M)
+          .Case("lea64_32mem", TYPE_M)
+          .Case("lea64mem", TYPE_M)
+          .Case("VR64", TYPE_MM64)
+          .Case("i64imm", TYPE_IMM)
+          .Case("anymem", TYPE_M)
+          .Case("opaquemem", TYPE_M)
+          .Case("sibmem", TYPE_MSIB)
+          .Case("SEGMENT_REG", TYPE_SEGMENTREG)
+          .Case("DEBUG_REG", TYPE_DEBUGREG)
+          .Case("CONTROL_REG", TYPE_CONTROLREG)
+          .Case("srcidx8", TYPE_SRCIDX)
+          .Case("srcidx16", TYPE_SRCIDX)
+          .Case("srcidx32", TYPE_SRCIDX)
+          .Case("srcidx64", TYPE_SRCIDX)
+          .Case("dstidx8", TYPE_DSTIDX)
+          .Case("dstidx16", TYPE_DSTIDX)
+          .Case("dstidx32", TYPE_DSTIDX)
+          .Case("dstidx64", TYPE_DSTIDX)
+          .Case("offset16_8", TYPE_MOFFS)
+          .Case("offset16_16", TYPE_MOFFS)
+          .Case("offset16_32", TYPE_MOFFS)
+          .Case("offset32_8", TYPE_MOFFS)
+          .Case("offset32_16", TYPE_MOFFS)
+          .Case("offset32_32", TYPE_MOFFS)
+          .Case("offset32_64", TYPE_MOFFS)
+          .Case("offset64_8", TYPE_MOFFS)
+          .Case("offset64_16", TYPE_MOFFS)
+          .Case("offset64_32", TYPE_MOFFS)
+          .Case("offset64_64", TYPE_MOFFS)
+          .Case("VR256", TYPE_YMM)
+          .Case("VR256X", TYPE_YMM)
+          .Case("VR512", TYPE_ZMM)
+          .Case("VK1", TYPE_VK)
+          .Case("VK1WM", TYPE_VK)
+          .Case("VK2", TYPE_VK)
+          .Case("VK2WM", TYPE_VK)
+          .Case("VK4", TYPE_VK)
+          .Case("VK4WM", TYPE_VK)
+          .Case("VK8", TYPE_VK)
+          .Case("VK8WM", TYPE_VK)
+          .Case("VK16", TYPE_VK)
+          .Case("VK16WM", TYPE_VK)
+          .Case("VK32", TYPE_VK)
+          .Case("VK32WM", TYPE_VK)
+          .Case("VK64", TYPE_VK)
+          .Case("VK64WM", TYPE_VK)
+          .Case("VK1Pair", TYPE_VK_PAIR)
+          .Case("VK2Pair", TYPE_VK_PAIR)
+          .Case("VK4Pair", TYPE_VK_PAIR)
+          .Case("VK8Pair", TYPE_VK_PAIR)
+          .Case("VK16Pair", TYPE_VK_PAIR)
+          .Case("vx32mem", TYPE_MVSIBX)
+          .Case("vx64mem", TYPE_MVSIBX)
+          .Case("vy32mem", TYPE_MVSIBY)
+          .Case("vy64mem", TYPE_MVSIBY)
+          .Case("vx32xmem", TYPE_MVSIBX)
+          .Case("vx64xmem", TYPE_MVSIBX)
+          .Case("vy32xmem", TYPE_MVSIBY)
+          .Case("vy64xmem", TYPE_MVSIBY)
+          .Case("vz32mem", TYPE_MVSIBZ)
+          .Case("vz64mem", TYPE_MVSIBZ)
+          .Case("BNDR", TYPE_BNDR)
+          .Case("TILE", TYPE_TMM)
+          .Case("TILEPair", TYPE_TMM_PAIR)
+          .Default(TYPE_NONE);
+  // clang-format on
 
   if (Type != TYPE_NONE)
     return Type;
@@ -1178,6 +1181,8 @@ OperandEncoding RecognizableInstr::immediateEncodingFromString(StringRef Str,
     // immediate encoding is special.
     Switch.Case("i16imm", ENCODING_IW);
   }
+
+  // clang-format off
   OperandEncoding Encoding =
       Switch.Case("i32i8imm", ENCODING_IB)
           .Case("AVX512RC", ENCODING_IRC)
@@ -1209,6 +1214,7 @@ OperandEncoding RecognizableInstr::immediateEncodingFromString(StringRef Str,
           .Case("VR512", ENCODING_IB)
           .Case("TILE", ENCODING_IB)
           .Default(ENCODING_NONE);
+  // clang-format on
 
   if (Encoding != ENCODING_NONE)
     return Encoding;
@@ -1218,38 +1224,41 @@ OperandEncoding RecognizableInstr::immediateEncodingFromString(StringRef Str,
 
 OperandEncoding
 RecognizableInstr::rmRegisterEncodingFromString(StringRef Str, uint8_t OpSize) {
-  auto Encoding = StringSwitch<OperandEncoding>(Str)
-                      .Case("RST", ENCODING_FP)
-                      .Case("RSTi", ENCODING_FP)
-                      .Case("GR16", ENCODING_RM)
-                      .Case("GR16orGR32orGR64", ENCODING_RM)
-                      .Case("GR32", ENCODING_RM)
-                      .Case("GR32orGR64", ENCODING_RM)
-                      .Case("GR64", ENCODING_RM)
-                      .Case("GR8", ENCODING_RM)
-                      .Case("VR128", ENCODING_RM)
-                      .Case("VR128X", ENCODING_RM)
-                      .Case("FR128", ENCODING_RM)
-                      .Case("FR64", ENCODING_RM)
-                      .Case("FR32", ENCODING_RM)
-                      .Case("FR64X", ENCODING_RM)
-                      .Case("FR32X", ENCODING_RM)
-                      .Case("FR16X", ENCODING_RM)
-                      .Case("VR64", ENCODING_RM)
-                      .Case("VR256", ENCODING_RM)
-                      .Case("VR256X", ENCODING_RM)
-                      .Case("VR512", ENCODING_RM)
-                      .Case("VK1", ENCODING_RM)
-                      .Case("VK2", ENCODING_RM)
-                      .Case("VK4", ENCODING_RM)
-                      .Case("VK8", ENCODING_RM)
-                      .Case("VK16", ENCODING_RM)
-                      .Case("VK32", ENCODING_RM)
-                      .Case("VK64", ENCODING_RM)
-                      .Case("BNDR", ENCODING_RM)
-                      .Case("TILE", ENCODING_RM)
-                      .Case("TILEPair", ENCODING_RM)
-                      .Default(ENCODING_NONE);
+  // clang-format off
+  auto Encoding =
+      StringSwitch<OperandEncoding>(Str)
+          .Case("RST", ENCODING_FP)
+          .Case("RSTi", ENCODING_FP)
+          .Case("GR16", ENCODING_RM)
+          .Case("GR16orGR32orGR64", ENCODING_RM)
+          .Case("GR32", ENCODING_RM)
+          .Case("GR32orGR64", ENCODING_RM)
+          .Case("GR64", ENCODING_RM)
+          .Case("GR8", ENCODING_RM)
+          .Case("VR128", ENCODING_RM)
+          .Case("VR128X", ENCODING_RM)
+          .Case("FR128", ENCODING_RM)
+          .Case("FR64", ENCODING_RM)
+          .Case("FR32", ENCODING_RM)
+          .Case("FR64X", ENCODING_RM)
+          .Case("FR32X", ENCODING_RM)
+          .Case("FR16X", ENCODING_RM)
+          .Case("VR64", ENCODING_RM)
+          .Case("VR256", ENCODING_RM)
+          .Case("VR256X", ENCODING_RM)
+          .Case("VR512", ENCODING_RM)
+          .Case("VK1", ENCODING_RM)
+          .Case("VK2", ENCODING_RM)
+          .Case("VK4", ENCODING_RM)
+          .Case("VK8", ENCODING_RM)
+          .Case("VK16", ENCODING_RM)
+          .Case("VK32", ENCODING_RM)
+          .Case("VK64", ENCODING_RM)
+          .Case("BNDR", ENCODING_RM)
+          .Case("TILE", ENCODING_RM)
+          .Case("TILEPair", ENCODING_RM)
+          .Default(ENCODING_NONE);
+  // clang-format on
   if (Encoding != ENCODING_NONE)
     return Encoding;
   errs() << "Unhandled R/M register encoding " << Str << "\n";
@@ -1258,51 +1267,54 @@ RecognizableInstr::rmRegisterEncodingFromString(StringRef Str, uint8_t OpSize) {
 
 OperandEncoding
 RecognizableInstr::roRegisterEncodingFromString(StringRef Str, uint8_t OpSize) {
-  auto Encoding = StringSwitch<OperandEncoding>(Str)
-                      .Case("GR16", ENCODING_REG)
-                      .Case("GR16orGR32orGR64", ENCODING_REG)
-                      .Case("GR32", ENCODING_REG)
-                      .Case("GR32orGR64", ENCODING_REG)
-                      .Case("GR64", ENCODING_REG)
-                      .Case("GR8", ENCODING_REG)
-                      .Case("VR128", ENCODING_REG)
-                      .Case("FR128", ENCODING_REG)
-                      .Case("FR64", ENCODING_REG)
-                      .Case("FR32", ENCODING_REG)
-                      .Case("VR64", ENCODING_REG)
-                      .Case("SEGMENT_REG", ENCODING_REG)
-                      .Case("DEBUG_REG", ENCODING_REG)
-                      .Case("CONTROL_REG", ENCODING_REG)
-                      .Case("VR256", ENCODING_REG)
-                      .Case("VR256X", ENCODING_REG)
-                      .Case("VR128X", ENCODING_REG)
-                      .Case("FR64X", ENCODING_REG)
-                      .Case("FR32X", ENCODING_REG)
-                      .Case("FR16X", ENCODING_REG)
-                      .Case("VR512", ENCODING_REG)
-                      .Case("VK1", ENCODING_REG)
-                      .Case("VK2", ENCODING_REG)
-                      .Case("VK4", ENCODING_REG)
-                      .Case("VK8", ENCODING_REG)
-                      .Case("VK16", ENCODING_REG)
-                      .Case("VK32", ENCODING_REG)
-                      .Case("VK64", ENCODING_REG)
-                      .Case("VK1Pair", ENCODING_REG)
-                      .Case("VK2Pair", ENCODING_REG)
-                      .Case("VK4Pair", ENCODING_REG)
-                      .Case("VK8Pair", ENCODING_REG)
-                      .Case("VK16Pair", ENCODING_REG)
-                      .Case("VK1WM", ENCODING_REG)
-                      .Case("VK2WM", ENCODING_REG)
-                      .Case("VK4WM", ENCODING_REG)
-                      .Case("VK8WM", ENCODING_REG)
-                      .Case("VK16WM", ENCODING_REG)
-                      .Case("VK32WM", ENCODING_REG)
-                      .Case("VK64WM", ENCODING_REG)
-                      .Case("BNDR", ENCODING_REG)
-                      .Case("TILE", ENCODING_REG)
-                      .Case("TILEPair", ENCODING_REG)
-                      .Default(ENCODING_NONE);
+  // clang-format off
+  auto Encoding =
+      StringSwitch<OperandEncoding>(Str)
+          .Case("GR16", ENCODING_REG)
+          .Case("GR16orGR32orGR64", ENCODING_REG)
+          .Case("GR32", ENCODING_REG)
+          .Case("GR32orGR64", ENCODING_REG)
+          .Case("GR64", ENCODING_REG)
+          .Case("GR8", ENCODING_REG)
+          .Case("VR128", ENCODING_REG)
+          .Case("FR128", ENCODING_REG)
+          .Case("FR64", ENCODING_REG)
+          .Case("FR32", ENCODING_REG)
+          .Case("VR64", ENCODING_REG)
+          .Case("SEGMENT_REG", ENCODING_REG)
+          .Case("DEBUG_REG", ENCODING_REG)
+          .Case("CONTROL_REG", ENCODING_REG)
+          .Case("VR256", ENCODING_REG)
+          .Case("VR256X", ENCODING_REG)
+          .Case("VR128X", ENCODING_REG)
+          .Case("FR64X", ENCODING_REG)
+          .Case("FR32X", ENCODING_REG)
+          .Case("FR16X", ENCODING_REG)
+          .Case("VR512", ENCODING_REG)
+          .Case("VK1", ENCODING_REG)
+          .Case("VK2", ENCODING_REG)
+          .Case("VK4", ENCODING_REG)
+          .Case("VK8", ENCODING_REG)
+          .Case("VK16", ENCODING_REG)
+          .Case("VK32", ENCODING_REG)
+          .Case("VK64", ENCODING_REG)
+          .Case("VK1Pair", ENCODING_REG)
+          .Case("VK2Pair", ENCODING_REG)
+          .Case("VK4Pair", ENCODING_REG)
+          .Case("VK8Pair", ENCODING_REG)
+          .Case("VK16Pair", ENCODING_REG)
+          .Case("VK1WM", ENCODING_REG)
+          .Case("VK2WM", ENCODING_REG)
+          .Case("VK4WM", ENCODING_REG)
+          .Case("VK8WM", ENCODING_REG)
+          .Case("VK16WM", ENCODING_REG)
+          .Case("VK32WM", ENCODING_REG)
+          .Case("VK64WM", ENCODING_REG)
+          .Case("BNDR", ENCODING_REG)
+          .Case("TILE", ENCODING_REG)
+          .Case("TILEPair", ENCODING_REG)
+          .Default(ENCODING_NONE);
+  // clang-format on
 
   if (Encoding != ENCODING_NONE)
     return Encoding;
@@ -1314,33 +1326,35 @@ RecognizableInstr::roRegisterEncodingFromString(StringRef Str, uint8_t OpSize) {
 OperandEncoding
 RecognizableInstr::vvvvRegisterEncodingFromString(StringRef Str,
                                                   uint8_t OpSize) {
-  auto Encoding = StringSwitch<OperandEncoding>(Str)
-                      .Case("GR8", ENCODING_VVVV)
-                      .Case("GR16", ENCODING_VVVV)
-                      .Case("GR32", ENCODING_VVVV)
-                      .Case("GR64", ENCODING_VVVV)
-                      .Case("FR32", ENCODING_VVVV)
-                      .Case("FR128", ENCODING_VVVV)
-                      .Case("FR64", ENCODING_VVVV)
-                      .Case("VR128", ENCODING_VVVV)
-                      .Case("VR256", ENCODING_VVVV)
-                      .Case("FR16X", ENCODING_VVVV)
-                      .Case("FR32X", ENCODING_VVVV)
-                      .Case("FR64X", ENCODING_VVVV)
-                      .Case("VR128X", ENCODING_VVVV)
-                      .Case("VR256X", ENCODING_VVVV)
-                      .Case("VR512", ENCODING_VVVV)
-                      .Case("VK1", ENCODING_VVVV)
-                      .Case("VK2", ENCODING_VVVV)
-                      .Case("VK4", ENCODING_VVVV)
-                      .Case("VK8", ENCODING_VVVV)
-                      .Case("VK16", ENCODING_VVVV)
-                      .Case("VK32", ENCODING_VVVV)
-                      .Case("VK64", ENCODING_VVVV)
-                      .Case("TILE", ENCODING_VVVV)
-                      .Case("TILEPair", ENCODING_VVVV)
-                      .Default(ENCODING_NONE);
-
+  // clang-format off
+  auto Encoding =
+      StringSwitch<OperandEncoding>(Str)
+          .Case("GR8", ENCODING_VVVV)
+          .Case("GR16", ENCODING_VVVV)
+          .Case("GR32", ENCODING_VVVV)
+          .Case("GR64", ENCODING_VVVV)
+          .Case("FR32", ENCODING_VVVV)
+          .Case("FR128", ENCODING_VVVV)
+          .Case("FR64", ENCODING_VVVV)
+          .Case("VR128", ENCODING_VVVV)
+          .Case("VR256", ENCODING_VVVV)
+          .Case("FR16X", ENCODING_VVVV)
+          .Case("FR32X", ENCODING_VVVV)
+          .Case("FR64X", ENCODING_VVVV)
+          .Case("VR128X", ENCODING_VVVV)
+          .Case("VR256X", ENCODING_VVVV)
+          .Case("VR512", ENCODING_VVVV)
+          .Case("VK1", ENCODING_VVVV)
+          .Case("VK2", ENCODING_VVVV)
+          .Case("VK4", ENCODING_VVVV)
+          .Case("VK8", ENCODING_VVVV)
+          .Case("VK16", ENCODING_VVVV)
+          .Case("VK32", ENCODING_VVVV)
+          .Case("VK64", ENCODING_VVVV)
+          .Case("TILE", ENCODING_VVVV)
+          .Case("TILEPair", ENCODING_VVVV)
+          .Default(ENCODING_NONE);
+  // clang-format on
   if (Encoding != ENCODING_NONE)
     return Encoding;
 
@@ -1351,15 +1365,18 @@ RecognizableInstr::vvvvRegisterEncodingFromString(StringRef Str,
 OperandEncoding
 RecognizableInstr::writemaskRegisterEncodingFromString(StringRef Str,
                                                        uint8_t OpSize) {
-  auto Encoding = StringSwitch<OperandEncoding>(Str)
-                      .Case("VK1WM", ENCODING_WRITEMASK)
-                      .Case("VK2WM", ENCODING_WRITEMASK)
-                      .Case("VK4WM", ENCODING_WRITEMASK)
-                      .Case("VK8WM", ENCODING_WRITEMASK)
-                      .Case("VK16WM", ENCODING_WRITEMASK)
-                      .Case("VK32WM", ENCODING_WRITEMASK)
-                      .Case("VK64WM", ENCODING_WRITEMASK)
-                      .Default(ENCODING_NONE);
+  // clang-format off
+  auto Encoding =
+      StringSwitch<OperandEncoding>(Str)
+          .Case("VK1WM", ENCODING_WRITEMASK)
+          .Case("VK2WM", ENCODING_WRITEMASK)
+          .Case("VK4WM", ENCODING_WRITEMASK)
+          .Case("VK8WM", ENCODING_WRITEMASK)
+          .Case("VK16WM", ENCODING_WRITEMASK)
+          .Case("VK32WM", ENCODING_WRITEMASK)
+          .Case("VK64WM", ENCODING_WRITEMASK)
+          .Default(ENCODING_NONE);
+  // clang-format on
 
   if (Encoding != ENCODING_NONE)
     return Encoding;
@@ -1370,45 +1387,48 @@ RecognizableInstr::writemaskRegisterEncodingFromString(StringRef Str,
 
 OperandEncoding RecognizableInstr::memoryEncodingFromString(StringRef Str,
                                                             uint8_t OpSize) {
-  auto Encoding = StringSwitch<OperandEncoding>(Str)
-                      .Case("i16mem", ENCODING_RM)
-                      .Case("i32mem", ENCODING_RM)
-                      .Case("i64mem", ENCODING_RM)
-                      .Case("i8mem", ENCODING_RM)
-                      .Case("shmem", ENCODING_RM)
-                      .Case("ssmem", ENCODING_RM)
-                      .Case("sdmem", ENCODING_RM)
-                      .Case("f128mem", ENCODING_RM)
-                      .Case("f256mem", ENCODING_RM)
-                      .Case("f512mem", ENCODING_RM)
-                      .Case("f64mem", ENCODING_RM)
-                      .Case("f32mem", ENCODING_RM)
-                      .Case("f16mem", ENCODING_RM)
-                      .Case("i128mem", ENCODING_RM)
-                      .Case("i256mem", ENCODING_RM)
-                      .Case("i512mem", ENCODING_RM)
-                      .Case("i512mem_GR16", ENCODING_RM)
-                      .Case("i512mem_GR32", ENCODING_RM)
-                      .Case("i512mem_GR64", ENCODING_RM)
-                      .Case("f80mem", ENCODING_RM)
-                      .Case("lea64_8mem", ENCODING_RM)
-                      .Case("lea64_16mem", ENCODING_RM)
-                      .Case("lea64_32mem", ENCODING_RM)
-                      .Case("lea64mem", ENCODING_RM)
-                      .Case("anymem", ENCODING_RM)
-                      .Case("opaquemem", ENCODING_RM)
-                      .Case("sibmem", ENCODING_SIB)
-                      .Case("vx32mem", ENCODING_VSIB)
-                      .Case("vx64mem", ENCODING_VSIB)
-                      .Case("vy32mem", ENCODING_VSIB)
-                      .Case("vy64mem", ENCODING_VSIB)
-                      .Case("vx32xmem", ENCODING_VSIB)
-                      .Case("vx64xmem", ENCODING_VSIB)
-                      .Case("vy32xmem", ENCODING_VSIB)
-                      .Case("vy64xmem", ENCODING_VSIB)
-                      .Case("vz32mem", ENCODING_VSIB)
-                      .Case("vz64mem", ENCODING_VSIB)
-                      .Default(ENCODING_NONE);
+  // clang-format off
+  auto Encoding =
+      StringSwitch<OperandEncoding>(Str)
+          .Case("i16mem", ENCODING_RM)
+          .Case("i32mem", ENCODING_RM)
+          .Case("i64mem", ENCODING_RM)
+          .Case("i8mem", ENCODING_RM)
+          .Case("shmem", ENCODING_RM)
+          .Case("ssmem", ENCODING_RM)
+          .Case("sdmem", ENCODING_RM)
+          .Case("f128mem", ENCODING_RM)
+          .Case("f256mem", ENCODING_RM)
+          .Case("f512mem", ENCODING_RM)
+          .Case("f64mem", ENCODING_RM)
+          .Case("f32mem", ENCODING_RM)
+          .Case("f16mem", ENCODING_RM)
+          .Case("i128mem", ENCODING_RM)
+          .Case("i256mem", ENCODING_RM)
+          .Case("i512mem", ENCODING_RM)
+          .Case("i512mem_GR16", ENCODING_RM)
+          .Case("i512mem_GR32", ENCODING_RM)
+          .Case("i512mem_GR64", ENCODING_RM)
+          .Case("f80mem", ENCODING_RM)
+          .Case("lea64_8mem", ENCODING_RM)
+          .Case("lea64_16mem", ENCODING_RM)
+          .Case("lea64_32mem", ENCODING_RM)
+          .Case("lea64mem", ENCODING_RM)
+          .Case("anymem", ENCODING_RM)
+          .Case("opaquemem", ENCODING_RM)
+          .Case("sibmem", ENCODING_SIB)
+          .Case("vx32mem", ENCODING_VSIB)
+          .Case("vx64mem", ENCODING_VSIB)
+          .Case("vy32mem", ENCODING_VSIB)
+          .Case("vy64mem", ENCODING_VSIB)
+          .Case("vx32xmem", ENCODING_VSIB)
+          .Case("vx64xmem", ENCODING_VSIB)
+          .Case("vy32xmem", ENCODING_VSIB)
+          .Case("vy64xmem", ENCODING_VSIB)
+          .Case("vz32mem", ENCODING_VSIB)
+          .Case("vz64mem", ENCODING_VSIB)
+          .Default(ENCODING_NONE);
+  // clang-format on
 
   if (Encoding != ENCODING_NONE)
     return Encoding;
@@ -1427,45 +1447,48 @@ RecognizableInstr::relocationEncodingFromString(StringRef Str, uint8_t OpSize) {
     Switch.Case("i16imm", ENCODING_IW);
   }
 
-  OperandEncoding Encoding = Switch.Case("i16imm", ENCODING_Iv)
-                                 .Case("i16i8imm", ENCODING_IB)
-                                 .Case("i32imm", ENCODING_Iv)
-                                 .Case("i32i8imm", ENCODING_IB)
-                                 .Case("i64i32imm", ENCODING_ID)
-                                 .Case("i64i8imm", ENCODING_IB)
-                                 .Case("i8imm", ENCODING_IB)
-                                 .Case("u8imm", ENCODING_IB)
-                                 .Case("i16u8imm", ENCODING_IB)
-                                 .Case("i32u8imm", ENCODING_IB)
-                                 .Case("i64u8imm", ENCODING_IB)
-                                 .Case("i64i32imm_brtarget", ENCODING_ID)
-                                 .Case("i16imm_brtarget", ENCODING_IW)
-                                 .Case("i32imm_brtarget", ENCODING_ID)
-                                 .Case("i8imm_brtarget", ENCODING_IB)
-                                 .Case("brtarget32", ENCODING_ID)
-                                 .Case("brtarget16", ENCODING_IW)
-                                 .Case("brtarget8", ENCODING_IB)
-                                 .Case("i64imm", ENCODING_IO)
-                                 .Case("offset16_8", ENCODING_Ia)
-                                 .Case("offset16_16", ENCODING_Ia)
-                                 .Case("offset16_32", ENCODING_Ia)
-                                 .Case("offset32_8", ENCODING_Ia)
-                                 .Case("offset32_16", ENCODING_Ia)
-                                 .Case("offset32_32", ENCODING_Ia)
-                                 .Case("offset32_64", ENCODING_Ia)
-                                 .Case("offset64_8", ENCODING_Ia)
-                                 .Case("offset64_16", ENCODING_Ia)
-                                 .Case("offset64_32", ENCODING_Ia)
-                                 .Case("offset64_64", ENCODING_Ia)
-                                 .Case("srcidx8", ENCODING_SI)
-                                 .Case("srcidx16", ENCODING_SI)
-                                 .Case("srcidx32", ENCODING_SI)
-                                 .Case("srcidx64", ENCODING_SI)
-                                 .Case("dstidx8", ENCODING_DI)
-                                 .Case("dstidx16", ENCODING_DI)
-                                 .Case("dstidx32", ENCODING_DI)
-                                 .Case("dstidx64", ENCODING_DI)
-                                 .Default(ENCODING_NONE);
+  // clang-format off
+  OperandEncoding Encoding =
+      Switch.Case("i16imm", ENCODING_Iv)
+          .Case("i16i8imm", ENCODING_IB)
+          .Case("i32imm", ENCODING_Iv)
+          .Case("i32i8imm", ENCODING_IB)
+          .Case("i64i32imm", ENCODING_ID)
+          .Case("i64i8imm", ENCODING_IB)
+          .Case("i8imm", ENCODING_IB)
+          .Case("u8imm", ENCODING_IB)
+          .Case("i16u8imm", ENCODING_IB)
+          .Case("i32u8imm", ENCODING_IB)
+          .Case("i64u8imm", ENCODING_IB)
+          .Case("i64i32imm_brtarget", ENCODING_ID)
+          .Case("i16imm_brtarget", ENCODING_IW)
+          .Case("i32imm_brtarget", ENCODING_ID)
+          .Case("i8imm_brtarget", ENCODING_IB)
+          .Case("brtarget32", ENCODING_ID)
+          .Case("brtarget16", ENCODING_IW)
+          .Case("brtarget8", ENCODING_IB)
+          .Case("i64imm", ENCODING_IO)
+          .Case("offset16_8", ENCODING_Ia)
+          .Case("offset16_16", ENCODING_Ia)
+          .Case("offset16_32", ENCODING_Ia)
+          .Case("offset32_8", ENCODING_Ia)
+          .Case("offset32_16", ENCODING_Ia)
+          .Case("offset32_32", ENCODING_Ia)
+          .Case("offset32_64", ENCODING_Ia)
+          .Case("offset64_8", ENCODING_Ia)
+          .Case("offset64_16", ENCODING_Ia)
+          .Case("offset64_32", ENCODING_Ia)
+          .Case("offset64_64", ENCODING_Ia)
+          .Case("srcidx8", ENCODING_SI)
+          .Case("srcidx16", ENCODING_SI)
+          .Case("srcidx32", ENCODING_SI)
+          .Case("srcidx64", ENCODING_SI)
+          .Case("dstidx8", ENCODING_DI)
+          .Case("dstidx16", ENCODING_DI)
+          .Case("dstidx32", ENCODING_DI)
+          .Case("dstidx64", ENCODING_DI)
+          .Default(ENCODING_NONE);
+  // clang-format on
 
   if (Encoding != ENCODING_NONE)
     return Encoding;
@@ -1477,13 +1500,16 @@ RecognizableInstr::relocationEncodingFromString(StringRef Str, uint8_t OpSize) {
 OperandEncoding
 RecognizableInstr::opcodeModifierEncodingFromString(StringRef Str,
                                                     uint8_t OpSize) {
-  auto Encoding = StringSwitch<OperandEncoding>(Str)
-                      .Case("GR32", ENCODING_Rv)
-                      .Case("GR64", ENCODING_RO)
-                      .Case("GR16", ENCODING_Rv)
-                      .Case("GR8", ENCODING_RB)
-                      .Case("ccode", ENCODING_CC)
-                      .Default(ENCODING_NONE);
+  // clang-format off
+  auto Encoding =
+      StringSwitch<OperandEncoding>(Str)
+          .Case("GR32", ENCODING_Rv)
+          .Case("GR64", ENCODING_RO)
+          .Case("GR16", ENCODING_Rv)
+          .Case("GR8", ENCODING_RB)
+          .Case("ccode", ENCODING_CC)
+          .Default(ENCODING_NONE);
+  // clang-format on
   if (Encoding != ENCODING_NONE)
     return Encoding;
 
diff --git a/external/llvm-project/llvm/utils/UpdateTestChecks/asm.py b/external/llvm-project/llvm/utils/UpdateTestChecks/asm.py
index da7e7ecc24bd..3754aa2eeba8 100644
--- a/external/llvm-project/llvm/utils/UpdateTestChecks/asm.py
+++ b/external/llvm-project/llvm/utils/UpdateTestChecks/asm.py
@@ -561,6 +561,7 @@ def get_run_handler(triple):
         "aarch64": (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_RE),
         "aarch64-apple-darwin": (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_DARWIN_RE),
         "aarch64-apple-ios": (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_DARWIN_RE),
+        "aarch64-apple-macosx": (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_DARWIN_RE),
         "bpf": (scrub_asm_bpf, ASM_FUNCTION_BPF_RE),
         "bpfel": (scrub_asm_bpf, ASM_FUNCTION_BPF_RE),
         "bpfeb": (scrub_asm_bpf, ASM_FUNCTION_BPF_RE),
diff --git a/external/llvm-project/llvm/utils/demangle_tree.py b/external/llvm-project/llvm/utils/demangle_tree.py
index 9d9603e63b83..6fbb8c6a36ca 100644
--- a/external/llvm-project/llvm/utils/demangle_tree.py
+++ b/external/llvm-project/llvm/utils/demangle_tree.py
@@ -28,7 +28,7 @@ def parse_line(line):
     if open_paren == -1:
         return None, None
     close_paren = line.rfind(")", open_paren)
-    if open_paren == -1:
+    if close_paren == -1:
         return None, None
     mangled = line[question:open_paren]
     demangled = line[open_paren + 1 : close_paren]
diff --git a/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/clang-doc/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/clang-doc/BUILD.gn
index 3794e15189e5..5815537318b3 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/clang-doc/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/clang-doc/BUILD.gn
@@ -23,6 +23,7 @@ static_library("clang-doc") {
     "Generators.cpp",
     "HTMLGenerator.cpp",
     "HTMLMustacheGenerator.cpp",
+    "JSONGenerator.cpp",
     "MDGenerator.cpp",
     "Mapper.cpp",
     "Representation.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/unittests/clang-doc/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/unittests/clang-doc/BUILD.gn
index 2c6428402b8b..4cbd3b96d8ff 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/unittests/clang-doc/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/clang-tools-extra/unittests/clang-doc/BUILD.gn
@@ -1,13 +1,11 @@
-import("//third-party/unittest/unittest.gni")
 import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//third-party/unittest/unittest.gni")
 
 write_cmake_config("config") {
   input = "config.h.cmake"
   output = "$target_gen_dir/config.h"
-  values = [
-    "CLANG_DOC_TEST_ASSET_DIR=" +
-      rebase_path("//clang-tools-extra/clang-doc/assets"),
-  ]
+  values = [ "CLANG_DOC_TEST_ASSET_DIR=" +
+             rebase_path("//clang-tools-extra/clang-doc/assets") ]
 }
 
 unittest("ClangDocTests") {
@@ -43,6 +41,7 @@ unittest("ClangDocTests") {
     "GeneratorTest.cpp",
     "HTMLGeneratorTest.cpp",
     "HTMLMustacheGeneratorTest.cpp",
+    "JSONGeneratorTest.cpp",
     "MDGeneratorTest.cpp",
     "MergeTest.cpp",
     "SerializeTest.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 803247bd7881..41516d677c45 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -1457,6 +1457,7 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/extent.h",
       "__type_traits/has_unique_object_representation.h",
       "__type_traits/has_virtual_destructor.h",
+      "__type_traits/integer_traits.h",
       "__type_traits/integral_constant.h",
       "__type_traits/invoke.h",
       "__type_traits/is_abstract.h",
@@ -1507,7 +1508,6 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/is_same.h",
       "__type_traits/is_scalar.h",
       "__type_traits/is_signed.h",
-      "__type_traits/is_signed_integer.h",
       "__type_traits/is_specialization.h",
       "__type_traits/is_standard_layout.h",
       "__type_traits/is_swappable.h",
@@ -1521,7 +1521,6 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/is_unbounded_array.h",
       "__type_traits/is_union.h",
       "__type_traits/is_unsigned.h",
-      "__type_traits/is_unsigned_integer.h",
       "__type_traits/is_valid_expansion.h",
       "__type_traits/is_void.h",
       "__type_traits/is_volatile.h",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/lld/Common/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/lld/Common/BUILD.gn
index b5286545f867..2075d5750aea 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/lld/Common/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/lld/Common/BUILD.gn
@@ -43,6 +43,7 @@ static_library("Common") {
     "Strings.cpp",
     "TargetOptionsCommandFlags.cpp",
     "Timer.cpp",
+    "Utils.cpp",
     "Version.cpp",
   ]
 }
diff --git a/external/llvm-project/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
index 1df45f3ed5ee..b00442d8e1eb 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
@@ -16,6 +16,7 @@ static_library("Host") {
   ]
   public_deps = [ "//llvm/utils/gn/build/libs/xml" ]
   sources = [
+    "aix/Support.cpp",
     "common/File.cpp",
     "common/FileAction.cpp",
     "common/FileCache.cpp",
@@ -26,6 +27,7 @@ static_library("Host") {
     "common/HostNativeThreadBase.cpp",
     "common/HostProcess.cpp",
     "common/HostThread.cpp",
+    "common/JSONTransport.cpp",
     "common/LZMA.cpp",
     "common/LockFileBase.cpp",
     "common/MainLoopBase.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/lldb/test/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/lldb/test/BUILD.gn
index e961a47d2c75..15b57f7d85fc 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/lldb/test/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/lldb/test/BUILD.gn
@@ -142,6 +142,7 @@ write_lit_cfg("lit_shell_site_cfg") {
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
     "LLVM_USE_SANITIZER=",
     "Python3_EXECUTABLE=$python_path",
+    "Python3_ROOT_DIR=",  # FIXME
   ]
 
   if (llvm_enable_zlib) {
@@ -167,6 +168,7 @@ write_lit_cfg("lit_unit_site_cfg") {
     "LLVM_BUILD_MODE=.",
     "LLVM_USE_SANITIZER=",
     "Python3_EXECUTABLE=$python_path",
+    "Python3_ROOT_DIR=",  # FIXME
   ]
 }
 
diff --git a/external/llvm-project/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
index 9c2768f457ff..c96042404258 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
@@ -72,6 +72,7 @@ static_library("lib") {
     "OutputRedirector.cpp",
     "ProgressEvent.cpp",
     "Protocol/ProtocolBase.cpp",
+    "Protocol/ProtocolEvents.cpp",
     "Protocol/ProtocolRequests.cpp",
     "Protocol/ProtocolTypes.cpp",
     "ProtocolUtils.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index fec917c25b19..c1d107eefdf9 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -298,7 +298,8 @@ write_cmake_config("llvm-config") {
     "LLVM_BUILD_SHARED_LIBS=",
     "LLVM_ENABLE_TELEMETRY=",
     "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple",
-    "LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=",
+    "LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE=",
+    "LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN=",
     "LLVM_ENABLE_DUMP=",
     "LLVM_ENABLE_HTTPLIB=",
     "LLVM_FORCE_USE_OLD_TOOLCHAIN=",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index daf745119bf5..c9935aface63 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -55,6 +55,7 @@ static_library("Analysis") {
     "FunctionPropertiesAnalysis.cpp",
     "GlobalsModRef.cpp",
     "GuardUtils.cpp",
+    "HashRecognize.cpp",
     "HeatUtils.cpp",
     "IR2Vec.cpp",
     "IRSimilarityIdentifier.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
index 9729398bfeb2..fc071e5471d0 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
@@ -11,9 +11,9 @@ static_library("DWARF") {
     "DWARFAbbreviationDeclaration.cpp",
     "DWARFAcceleratorTable.cpp",
     "DWARFAddressRange.cpp",
+    "DWARFCFIProgram.cpp",
     "DWARFCompileUnit.cpp",
     "DWARFContext.cpp",
-    "DWARFDataExtractor.cpp",
     "DWARFDebugAbbrev.cpp",
     "DWARFDebugAddr.cpp",
     "DWARFDebugArangeSet.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
index 12b9540cc6d9..cca2cb7ee501 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
@@ -22,6 +22,7 @@ static_library("LogicalView") {
     "Core/LVReader.cpp",
     "Core/LVScope.cpp",
     "Core/LVSort.cpp",
+    "Core/LVSourceLanguage.cpp",
     "Core/LVSupport.cpp",
     "Core/LVSymbol.cpp",
     "Core/LVType.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/Directive/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/Directive/BUILD.gn
new file mode 100644
index 000000000000..74ee5fdb1470
--- /dev/null
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/Directive/BUILD.gn
@@ -0,0 +1,7 @@
+static_library("Directive") {
+  output_name = "LLVMFrontendDirective"
+  deps = [
+    "//llvm/lib/Support",
+  ]
+  sources = [ "Spelling.cpp" ]
+}
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
index 67e4d4e807eb..7346de64f6c9 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
@@ -7,6 +7,6 @@ static_library("HLSL") {
   sources = [
     "CBuffer.cpp",
     "HLSLResource.cpp",
-    "HLSLRootSignature.cpp",
+    "HLSLRootSignatureUtils.cpp",
   ]
 }
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
index 5e354a79fe30..74a8d07cbaf0 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
@@ -1,6 +1,9 @@
 static_library("OpenACC") {
   output_name = "LLVMFrontendOpenACC"
-  deps = [ "//llvm/lib/Support" ]
+  deps = [
+    "//llvm/lib/Frontend/Directive",
+    "//llvm/lib/Support",
+  ]
   public_deps = [ "//llvm/include/llvm/Frontend/OpenACC:acc_gen" ]
   sources = [ "ACC.cpp" ]
 }
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
index 4f103d30f300..22aa0b641813 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
@@ -90,7 +90,6 @@ static_library("IR") {
     "VFABIDemangler.cpp",
     "Value.cpp",
     "ValueSymbolTable.cpp",
-    "VectorBuilder.cpp",
     "VectorTypeUtils.cpp",
     "Verifier.cpp",
   ]
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
index 981639faf71d..698607f3a226 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
@@ -67,7 +67,6 @@ static_library("MCTargetDesc") {
     "ARMInstPrinter.cpp",
     "ARMMCAsmInfo.cpp",
     "ARMMCCodeEmitter.cpp",
-    "ARMMCExpr.cpp",
     "ARMMCTargetDesc.cpp",
     "ARMMachORelocationInfo.cpp",
     "ARMMachObjectWriter.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/MCTargetDesc/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/MCTargetDesc/BUILD.gn
index a52132e69cc3..874cdc1b7839 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/MCTargetDesc/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/MCTargetDesc/BUILD.gn
@@ -56,7 +56,6 @@ static_library("MCTargetDesc") {
     "LanaiInstPrinter.cpp",
     "LanaiMCAsmInfo.cpp",
     "LanaiMCCodeEmitter.cpp",
-    "LanaiMCExpr.cpp",
     "LanaiMCTargetDesc.cpp",
   ]
 }
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
index 4182b4911538..360cdc5f10e6 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
@@ -70,7 +70,6 @@ static_library("MCTargetDesc") {
     "SystemZMCAsmBackend.cpp",
     "SystemZMCAsmInfo.cpp",
     "SystemZMCCodeEmitter.cpp",
-    "SystemZMCExpr.cpp",
     "SystemZMCTargetDesc.cpp",
     "SystemZTargetStreamer.cpp",
   ]
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
index bd8e997dbe77..377997579a1e 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
@@ -143,7 +143,6 @@ static_library("LLVMX86CodeGen") {
     "X86VZeroUpper.cpp",
     "X86WinEHState.cpp",
     "X86WinEHUnwindV2.cpp",
-    "X86WinFixupBufferSecurityCheck.cpp",
   ]
 }
 
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn
index 2155835583a0..a8eb834c1da2 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn
@@ -22,7 +22,8 @@ static_library("Instrumentation") {
     "InstrProfiling.cpp",
     "KCFI.cpp",
     "LowerAllowCheckPass.cpp",
-    "MemProfiler.cpp",
+    "MemProfInstrumentation.cpp",
+    "MemProfUse.cpp",
     "MemorySanitizer.cpp",
     "NumericalStabilitySanitizer.cpp",
     "PGOCtxProfFlattening.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 8ac3ab251f08..822e3cec4823 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -19,6 +19,7 @@ static_library("Vectorize") {
     "SandboxVectorizer/Interval.cpp",
     "SandboxVectorizer/Legality.cpp",
     "SandboxVectorizer/Passes/BottomUpVec.cpp",
+    "SandboxVectorizer/Passes/PackReuse.cpp",
     "SandboxVectorizer/Passes/RegionsFromBBs.cpp",
     "SandboxVectorizer/Passes/RegionsFromMetadata.cpp",
     "SandboxVectorizer/Passes/SeedCollection.cpp",
diff --git a/external/llvm-project/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn b/external/llvm-project/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
index 0f34231ae321..0d162ff0f9d5 100644
--- a/external/llvm-project/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
+++ b/external/llvm-project/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
@@ -54,7 +54,6 @@ unittest("IRTests") {
     "ValueHandleTest.cpp",
     "ValueMapTest.cpp",
     "ValueTest.cpp",
-    "VectorBuilderTest.cpp",
     "VectorTypeUtilsTest.cpp",
     "VectorTypesTest.cpp",
     "VerifierTest.cpp",
diff --git a/external/llvm-project/llvm/utils/lit/lit/Test.py b/external/llvm-project/llvm/utils/lit/lit/Test.py
index 051062706f85..1bd5ba8dc207 100644
--- a/external/llvm-project/llvm/utils/lit/lit/Test.py
+++ b/external/llvm-project/llvm/utils/lit/lit/Test.py
@@ -151,7 +151,9 @@ def toMetricValue(value):
 class Result(object):
     """Wrapper for the results of executing an individual test."""
 
-    def __init__(self, code, output="", elapsed=None):
+    def __init__(
+        self, code, output="", elapsed=None, attempts=1, max_allowed_attempts=None
+    ):
         # The result code.
         self.code = code
         # The test output.
@@ -164,6 +166,10 @@ def __init__(self, code, output="", elapsed=None):
         self.metrics = {}
         # The micro-test results reported by this test.
         self.microResults = {}
+        # How often was the test run?
+        self.attempts = attempts
+        # How many attempts were allowed for this test
+        self.max_allowed_attempts = max_allowed_attempts
 
     def addMetric(self, name, value):
         """
diff --git a/external/llvm-project/llvm/utils/lit/lit/TestRunner.py b/external/llvm-project/llvm/utils/lit/lit/TestRunner.py
index 16e9c7fbf45c..73db67aedb73 100644
--- a/external/llvm-project/llvm/utils/lit/lit/TestRunner.py
+++ b/external/llvm-project/llvm/utils/lit/lit/TestRunner.py
@@ -2292,7 +2292,9 @@ def runOnce(
     if err:
         output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
 
-    return lit.Test.Result(status, output)
+    return lit.Test.Result(
+        status, output, attempts=i + 1, max_allowed_attempts=attempts
+    )
 
 
 def executeShTest(
diff --git a/external/llvm-project/llvm/utils/lit/lit/display.py b/external/llvm-project/llvm/utils/lit/lit/display.py
index 7de5a298d230..3a224f734bf7 100644
--- a/external/llvm-project/llvm/utils/lit/lit/display.py
+++ b/external/llvm-project/llvm/utils/lit/lit/display.py
@@ -117,9 +117,20 @@ def clear(self, interrupted):
     def print_result(self, test):
         # Show the test result line.
         test_name = test.getFullName()
+
+        extra_info = ""
+        if test.result.attempts > 1:
+            extra_info = f", {test.result.attempts} of {test.result.max_allowed_attempts} attempts"
+
         print(
-            "%s: %s (%d of %d)"
-            % (test.result.code.name, test_name, self.completed, self.num_tests)
+            "%s: %s (%d of %d%s)"
+            % (
+                test.result.code.name,
+                test_name,
+                self.completed,
+                self.num_tests,
+                extra_info,
+            )
         )
 
         # Show the test failure output, if requested.
diff --git a/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py b/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
index f2333a7de455..4227b89a1b45 100644
--- a/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
+++ b/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 8
 # RUN: "%python" "%s" "%counter"
 
 import sys
diff --git a/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py b/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
index f2333a7de455..cf3c5f849d1a 100644
--- a/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
+++ b/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py
@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 10
 # RUN: "%python" "%s" "%counter"
 
 import sys
diff --git a/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg b/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
index 2279d293849a..cf15d1f442af 100644
--- a/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
+++ b/external/llvm-project/llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg
@@ -9,4 +9,4 @@ config.test_exec_root = None
 config.substitutions.append(("%python", lit_config.params.get("python", "")))
 config.substitutions.append(("%counter", lit_config.params.get("counter", "")))
 
-config.test_retry_attempts = 3
\ No newline at end of file
+config.test_retry_attempts = 9
\ No newline at end of file
diff --git a/external/llvm-project/llvm/utils/lit/tests/allow-retries.py b/external/llvm-project/llvm/utils/lit/tests/allow-retries.py
index ba95d34a09a4..98eed142964a 100644
--- a/external/llvm-project/llvm/utils/lit/tests/allow-retries.py
+++ b/external/llvm-project/llvm/utils/lit/tests/allow-retries.py
@@ -22,7 +22,7 @@
 # RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py -v |\
 # RUN:   FileCheck --check-prefix=CHECK-TEST3 -match-full-lines %s
 #
-#       CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1)
+#       CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1, 4 of 4 attempts)
 #  CHECK-TEST3-NEXT: {{\**}} TEST 'allow-retries :: does-not-succeed-within-limit.py' FAILED {{\**}}
 #  CHECK-TEST3-NEXT: Exit Code: 1
 # CHECK-TEST3-EMPTY:
@@ -72,20 +72,21 @@
 #     CHECK-TEST7: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
-#  * The "--max-retries-per-test" is specified high enough (3).
+#  * The "--max-retries-per-test" is specified high enough (7).
 #  * No ALLOW_RETRIES keyword is used in the test script.
 #  * No config.test_retry_attempts is adjusted in the test suite config file.
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
-# RUN:   --max-retries-per-test=3 \
+# RUN:   --max-retries-per-test=7 \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
+# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 8 attempts)
 # CHECK-TEST8: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (2).
-#  * ALLOW_RETRIES is specified high enough (3)
+#  * ALLOW_RETRIES is specified high enough (8)
 #  * No config.test_retry_attempts is adjusted in the test suite config file.
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
@@ -93,28 +94,31 @@
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
+# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 9 attempts)
 # CHECK-TEST9: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (2).
 #  * No ALLOW_RETRIES keyword is used in the test script.
-#  * config.test_retry_attempts is set high enough (3).
+#  * config.test_retry_attempts is set high enough (9).
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=2 \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
+# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 10 attempts)
 # CHECK-TEST10: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (1).
-#  * ALLOW_RETRIES keyword set high enough (3).
-#  * config.test_retry_attempts is set too low enough (2).
+#  * ALLOW_RETRIES keyword is set high enough (10)
+#  * config.test_retry_attempts is set too low (2).
 # RUN: rm -f %t.counter
-# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
+# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=1 \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
+# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 11 attempts)
 # CHECK-TEST11: Passed With Retry: 1
diff --git a/external/llvm-project/llvm/utils/lit/tests/timeout-hang.py b/external/llvm-project/llvm/utils/lit/tests/timeout-hang.py
index 486f07983708..edb23affad57 100644
--- a/external/llvm-project/llvm/utils/lit/tests/timeout-hang.py
+++ b/external/llvm-project/llvm/utils/lit/tests/timeout-hang.py
@@ -8,20 +8,29 @@
 # throwing an exception. We expect this to fail immediately, rather than
 # timeout.
 
-# DEFINE: %{timeout}=1
+# lit should return immediately once it fails to execute the non-existent file.
+# This will take a variable amount of time depending on process scheduling, but
+# it should always be significantly less than the hard timeout, which is the
+# point where lit would cancel the test.
+# DEFINE: %{grace_period}=5
+# DEFINE: %{hard_timeout}=15
 
 # RUN: not %{lit} %{inputs}/timeout-hang/run-nonexistent.txt \
-# RUN: --timeout=%{timeout} --param external=0 | %{python} %s %{timeout}
+# RUN: --timeout=%{hard_timeout} --param external=0 | %{python} %s %{grace_period}
 
 import sys
 import re
 
-timeout_time = float(sys.argv[1])
+grace_time = float(sys.argv[1])
 testing_time = float(re.search(r"Testing Time: (.*)s", sys.stdin.read()).group(1))
 
-if testing_time < timeout_time:
-    print("Testing took less than timeout")
+if testing_time <= grace_time:
+    print("Testing finished within the grace period")
     sys.exit(0)
 else:
-    print("Testing took as long or longer than timeout")
+    print(
+        "Testing took {}s, which is beyond the grace period of {}s".format(
+            testing_time, grace_time
+        )
+    )
     sys.exit(1)
diff --git a/external/llvm-project/llvm/utils/lldbDataFormatters.py b/external/llvm-project/llvm/utils/lldbDataFormatters.py
index 988827ab4aa5..c5cd627c5314 100644
--- a/external/llvm-project/llvm/utils/lldbDataFormatters.py
+++ b/external/llvm-project/llvm/utils/lldbDataFormatters.py
@@ -3,6 +3,7 @@
 
 Load into LLDB with 'command script import /path/to/lldbDataFormatters.py'
 """
+
 from __future__ import annotations
 
 import collections
@@ -82,6 +83,11 @@ def __lldb_init_module(debugger, internal_dict):
         f"-l {__name__}.DenseMapSynthetic "
         '-x "^llvm::DenseMap<.+>$"'
     )
+    debugger.HandleCommand(
+        "type synthetic add -w llvm "
+        f"-l {__name__}.DenseSetSynthetic "
+        '-x "^llvm::DenseSet<.+>$"'
+    )
 
     debugger.HandleCommand(
         "type synthetic add -w llvm "
@@ -372,7 +378,8 @@ def update(self):
         # For each key, collect a list of buckets it appears in.
         key_buckets: dict[str, list[int]] = collections.defaultdict(list)
         for index in range(num_buckets):
-            key = buckets.GetValueForExpressionPath(f"[{index}].first")
+            bucket = buckets.GetValueForExpressionPath(f"[{index}]")
+            key = bucket.GetChildAtIndex(0)
             key_buckets[str(key.data)].append(index)
 
         # Heuristic: This is not a multi-map, any repeated (non-unique) keys are
@@ -383,6 +390,26 @@ def update(self):
                 self.child_buckets.append(indexes[0])
 
 
+class DenseSetSynthetic:
+    valobj: lldb.SBValue
+    map: lldb.SBValue
+
+    def __init__(self, valobj: lldb.SBValue, _) -> None:
+        self.valobj = valobj
+
+    def num_children(self) -> int:
+        return self.map.num_children
+
+    def get_child_at_index(self, idx: int) -> lldb.SBValue:
+        map_entry = self.map.child[idx]
+        set_entry = map_entry.GetChildAtIndex(0)
+        return set_entry.Clone(f"[{idx}]")
+
+    def update(self):
+        raw_map = self.valobj.GetChildMemberWithName("TheMap")
+        self.map = raw_map.GetSyntheticValue()
+
+
 class ExpectedSynthetic:
     # The llvm::Expected<T> value.
     expected: lldb.SBValue
diff --git a/external/llvm-project/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt b/external/llvm-project/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt
new file mode 100644
index 000000000000..dc1aa73371ff
--- /dev/null
+++ b/external/llvm-project/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_llvm_utility(llvm-test-mustache-spec
+  llvm-test-mustache-spec.cpp
+)
+
+target_link_libraries(llvm-test-mustache-spec PRIVATE LLVMSupport)
diff --git a/external/llvm-project/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp b/external/llvm-project/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp
new file mode 100644
index 000000000000..1f566e13f070
--- /dev/null
+++ b/external/llvm-project/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp
@@ -0,0 +1,270 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Simple drivers to test the mustache spec found at:
+// https://github.com/mustache/spec
+//
+// It is used to verify that the current implementation conforms to the spec.
+// Simply download the spec and pass the test JSON files to the driver. Each
+// spec file should have a list of tests for compliance with the spec. These
+// are loaded as test cases, and rendered with our Mustache implementation,
+// which is then compared against the expected output from the spec.
+//
+// The current implementation only supports non-optional parts of the spec, so
+// we do not expect any of the dynamic-names, inheritance, or lambda tests to
+// pass. Additionally, Triple Mustache is not supported. Unsupported tests are
+// marked as XFail and are removed from the XFail list as they are fixed.
+//
+// Usage:
+//  llvm-test-mustache-spec path/to/test/file.json path/to/test/file2.json ...
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Mustache.h"
+#include "llvm/Support/Path.h"
+#include <string>
+
+using namespace llvm;
+using namespace llvm::json;
+using namespace llvm::mustache;
+
+#define DEBUG_TYPE "llvm-test-mustache-spec"
+
+static cl::OptionCategory Cat("llvm-test-mustache-spec Options");
+
+static cl::list<std::string>
+    InputFiles(cl::Positional, cl::desc("<input files>"), cl::OneOrMore);
+
+static cl::opt<bool> ReportErrors("report-errors",
+                                  cl::desc("Report errors in spec tests"),
+                                  cl::cat(Cat));
+
+static ExitOnError ExitOnErr;
+
+static int NumXFail = 0;
+static int NumSuccess = 0;
+
+static const StringMap<StringSet<>> XFailTestNames = {{
+    {"delimiters.json",
+     {
+         "Pair Behavior",
+         "Special Characters",
+         "Sections",
+         "Inverted Sections",
+         "Partial Inheritence",
+         "Post-Partial Behavior",
+         "Standalone Tag",
+         "Indented Standalone Tag",
+         "Standalone Line Endings",
+         "Standalone Without Previous Line",
+         "Standalone Without Newline",
+     }},
+    {"~dynamic-names.json",
+     {
+         "Basic Behavior - Partial",
+         "Basic Behavior - Name Resolution",
+         "Context",
+         "Dotted Names",
+         "Dotted Names - Failed Lookup",
+         "Dotted names - Context Stacking",
+         "Dotted names - Context Stacking Under Repetition",
+         "Dotted names - Context Stacking Failed Lookup",
+         "Recursion",
+         "Surrounding Whitespace",
+         "Inline Indentation",
+         "Standalone Line Endings",
+         "Standalone Without Previous Line",
+         "Standalone Without Newline",
+         "Standalone Indentation",
+         "Padding Whitespace",
+     }},
+    {"~inheritance.json",
+     {
+         "Default",
+         "Variable",
+         "Triple Mustache",
+         "Sections",
+         "Negative Sections",
+         "Mustache Injection",
+         "Inherit",
+         "Overridden content",
+         "Data does not override block default",
+         "Two overridden parents",
+         "Override parent with newlines",
+         "Inherit indentation",
+         "Only one override",
+         "Parent template",
+         "Recursion",
+         "Multi-level inheritance, no sub child",
+         "Text inside parent",
+         "Text inside parent",
+         "Block scope",
+         "Standalone parent",
+         "Standalone block",
+         "Block reindentation",
+         "Intrinsic indentation",
+         "Nested block reindentation",
+
+     }},
+    {"~lambdas.json",
+     {
+         "Interpolation",
+         "Interpolation - Expansion",
+         "Interpolation - Alternate Delimiters",
+         "Interpolation - Multiple Calls",
+         "Escaping",
+         "Section",
+         "Section - Expansion",
+         "Section - Alternate Delimiters",
+         "Section - Multiple Calls",
+
+     }},
+    {"interpolation.json",
+     {
+         "Triple Mustache",
+         "Triple Mustache Integer Interpolation",
+         "Triple Mustache Decimal Interpolation",
+         "Triple Mustache Null Interpolation",
+         "Triple Mustache Context Miss Interpolation",
+         "Dotted Names - Triple Mustache Interpolation",
+         "Implicit Iterators - Triple Mustache",
+         "Triple Mustache - Surrounding Whitespace",
+         "Triple Mustache - Standalone",
+         "Triple Mustache With Padding",
+     }},
+    {"partials.json", {"Standalone Indentation"}},
+    {"sections.json", {"Implicit Iterator - Triple mustache"}},
+}};
+
+struct TestData {
+  TestData() = default;
+  explicit TestData(const json::Object &TestCase)
+      : TemplateStr(*TestCase.getString("template")),
+        ExpectedStr(*TestCase.getString("expected")),
+        Name(*TestCase.getString("name")), Data(TestCase.get("data")),
+        Partials(TestCase.get("partials")) {}
+
+  static Expected<TestData> createTestData(json::Object *TestCase,
+                                           StringRef InputFile) {
+    // If any of the needed elements are missing, we cannot continue.
+    // NOTE: partials are optional in the test schema.
+    if (!TestCase || !TestCase->getString("template") ||
+        !TestCase->getString("expected") || !TestCase->getString("name") ||
+        !TestCase->get("data"))
+      return createStringError(
+          llvm::inconvertibleErrorCode(),
+          "invalid JSON schema in test file: " + InputFile + "\n");
+
+    return TestData(*TestCase);
+  }
+
+  StringRef TemplateStr;
+  StringRef ExpectedStr;
+  StringRef Name;
+  const Value *Data;
+  const Value *Partials;
+};
+
+static void reportTestFailure(const TestData &TD, StringRef ActualStr,
+                              bool IsXFail) {
+  LLVM_DEBUG(dbgs() << "Template: " << TD.TemplateStr << "\n");
+  if (TD.Partials) {
+    LLVM_DEBUG(dbgs() << "Partial: ");
+    LLVM_DEBUG(TD.Partials->print(dbgs()));
+    LLVM_DEBUG(dbgs() << "\n");
+  }
+  LLVM_DEBUG(dbgs() << "JSON Data: ");
+  LLVM_DEBUG(TD.Data->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "\n");
+  outs() << formatv("Test {}: {}\n", (IsXFail ? "XFailed" : "Failed"), TD.Name);
+  if (ReportErrors) {
+    outs() << "  Expected: \'" << TD.ExpectedStr << "\'\n"
+           << "  Actual: \'" << ActualStr << "\'\n"
+           << " ====================\n";
+  }
+}
+
+static void registerPartials(const Value *Partials, Template &T) {
+  if (!Partials)
+    return;
+  for (const auto &[Partial, Str] : *Partials->getAsObject())
+    T.registerPartial(Partial.str(), Str.getAsString()->str());
+}
+
+static json::Value readJsonFromFile(StringRef &InputFile) {
+  std::unique_ptr<MemoryBuffer> Buffer =
+      ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile)));
+  return ExitOnErr(parse(Buffer->getBuffer()));
+}
+
+static bool isTestXFail(StringRef FileName, StringRef TestName) {
+  auto P = llvm::sys::path::filename(FileName);
+  auto It = XFailTestNames.find(P);
+  return It != XFailTestNames.end() && It->second.contains(TestName);
+}
+
+static bool evaluateTest(StringRef &InputFile, TestData &TestData,
+                         std::string &ActualStr) {
+  bool IsXFail = isTestXFail(InputFile, TestData.Name);
+  bool Matches = TestData.ExpectedStr == ActualStr;
+  if ((Matches && IsXFail) || (!Matches && !IsXFail)) {
+    reportTestFailure(TestData, ActualStr, IsXFail);
+    return false;
+  }
+  IsXFail ? NumXFail++ : NumSuccess++;
+  return true;
+}
+
+static void runTest(StringRef InputFile) {
+  NumXFail = 0;
+  NumSuccess = 0;
+  outs() << "Running Tests: " << InputFile << "\n";
+  json::Value Json = readJsonFromFile(InputFile);
+
+  json::Object *Obj = Json.getAsObject();
+  Array *TestArray = Obj->getArray("tests");
+  // Even though we parsed the JSON, it can have a bad format, so check it.
+  if (!TestArray)
+    ExitOnErr(createStringError(
+        llvm::inconvertibleErrorCode(),
+        "invalid JSON schema in test file: " + InputFile + "\n"));
+
+  const size_t Total = TestArray->size();
+
+  for (Value V : *TestArray) {
+    auto TestData =
+        ExitOnErr(TestData::createTestData(V.getAsObject(), InputFile));
+    Template T(TestData.TemplateStr);
+    registerPartials(TestData.Partials, T);
+
+    std::string ActualStr;
+    raw_string_ostream OS(ActualStr);
+    T.render(*TestData.Data, OS);
+    evaluateTest(InputFile, TestData, ActualStr);
+  }
+
+  const int NumFailed = Total - NumSuccess - NumXFail;
+  outs() << formatv("===Results===\n"
+                    " Suceeded: {}\n"
+                    " Expectedly Failed: {}\n"
+                    " Failed: {}\n"
+                    " Total: {}\n",
+                    NumSuccess, NumXFail, NumFailed, Total);
+}
+
+int main(int argc, char **argv) {
+  ExitOnErr.setBanner(std::string(argv[0]) + " error: ");
+  cl::ParseCommandLineOptions(argc, argv);
+  for (const auto &FileName : InputFiles)
+    runTest(FileName);
+  return 0;
+}
diff --git a/external/llvm-project/llvm/utils/release/bump-version.py b/external/llvm-project/llvm/utils/release/bump-version.py
index 5db62e88fec1..1a55328bb2d5 100755
--- a/external/llvm-project/llvm/utils/release/bump-version.py
+++ b/external/llvm-project/llvm/utils/release/bump-version.py
@@ -169,7 +169,7 @@ def process_line(self, line: str) -> str:
 
     verstr = args.version
 
-    # parse the version string with distutils.
+    # parse the version string.
     # note that -rc will end up as version.pre here
     # since it's a prerelease
     version = packaging.version.parse(verstr)
diff --git a/external/llvm-project/llvm/utils/update_analyze_test_checks.py b/external/llvm-project/llvm/utils/update_analyze_test_checks.py
index 6c194f0923d1..3f14452767f9 100755
--- a/external/llvm-project/llvm/utils/update_analyze_test_checks.py
+++ b/external/llvm-project/llvm/utils/update_analyze_test_checks.py
@@ -109,10 +109,13 @@ def update_test(opt_basename: str, ti: common.TestInfo):
                     prefixes,
                 )
         elif (
-            re.search(r"(LV|LDist): Checking a loop in ", raw_tool_outputs) is not None
+            re.search(
+                r"(LV|LDist|HashRecognize): Checking a loop in ", raw_tool_outputs
+            )
+            is not None
         ):
             for raw_tool_output in re.split(
-                r"(LV|LDist): Checking a loop in ", raw_tool_outputs
+                r"(LV|LDist|HashRecognize): Checking a loop in ", raw_tool_outputs
             ):
                 builder.process_run_line(
                     common.LOOP_PASS_DEBUG_RE,
diff --git a/external/llvm-project/mlir/docs/DefiningDialects/Constraints.md b/external/llvm-project/mlir/docs/DefiningDialects/Constraints.md
index 52a4283d6084..40863e7aecf4 100644
--- a/external/llvm-project/mlir/docs/DefiningDialects/Constraints.md
+++ b/external/llvm-project/mlir/docs/DefiningDialects/Constraints.md
@@ -24,8 +24,8 @@ code is generated for type/attribute constraints. Type constraints can not only
 be used when defining operation arguments, but also when defining type
 parameters.
 
-Optionally, C++ functions can be generated, so that type constraints can be
-checked from C++. The name of the C++ function must be specified in the
+Optionally, C++ functions can be generated, so that type/attribute constraints
+can be checked from C++. The name of the C++ function must be specified in the
 `cppFunctionName` field. If no function name is specified, no C++ function is
 emitted.
 
@@ -43,17 +43,20 @@ bool isValidVectorTypeElementType(::mlir::Type type) {
 }
 ```
 
-An extra TableGen rule is needed to emit C++ code for type constraints. This
-will generate only the declarations/definitions of the type constaraints that
-are defined in the specified `.td` file, but not those that are in included
-`.td` files.
+An extra TableGen rule is needed to emit C++ code for type/attribute
+constraints. This will generate only the declarations/definitions of the
+type/attribute constaraints that are defined in the specified `.td` file, but
+not those that are in included `.td` files.
 
 ```cmake
 mlir_tablegen(<Your Dialect>TypeConstraints.h.inc -gen-type-constraint-decls)
 mlir_tablegen(<Your Dialect>TypeConstraints.cpp.inc -gen-type-constraint-defs)
+mlir_tablegen(<Your Dialect>AttrConstraints.h.inc -gen-attr-constraint-decls)
+mlir_tablegen(<Your Dialect>AttrConstraints.cpp.inc -gen-attr-constraint-defs)
 ```
 
-The generated `<Your Dialect>TypeConstraints.h.inc` will need to be included
-whereever you are referencing the type constraint in C++. Note that no C++
-namespace will be emitted by the code generator. The `#include` statements of
-the `.h.inc`/`.cpp.inc` files should be wrapped in C++ namespaces by the user.
+The generated `<Your Dialect>TypeConstraints.h.inc` respectivelly
+`<Your Dialect>AttrConstraints.h.inc` will need to be included whereever you are
+referencing the type/attributes constraint in C++. Note that no C++ namespace
+will be emitted by the code generator. The `#include` statements of the
+`.h.inc`/`.cpp.inc` files should be wrapped in C++ namespaces by the user.
diff --git a/external/llvm-project/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h b/external/llvm-project/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
index 1b2c67917610..3f8874d02afa 100644
--- a/external/llvm-project/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
+++ b/external/llvm-project/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
@@ -235,6 +235,30 @@ class AbstractSparseForwardDataFlowAnalysis : public DataFlowAnalysis {
   /// Join the lattice element and propagate and update if it changed.
   void join(AbstractSparseLattice *lhs, const AbstractSparseLattice &rhs);
 
+  /// Visits a call operation. Given the operand lattices, sets the result
+  /// lattices. Performs interprocedural data flow as follows: if the call
+  /// operation targets an external function, or if the solver is not
+  /// interprocedural, attempts to infer the results from the call arguments
+  /// using the user-provided `visitExternalCallImpl`. Otherwise, computes the
+  /// result lattices from the return sites if all return sites are known;
+  /// otherwise, conservatively marks the result lattices as having reached
+  /// their pessimistic fixpoints.
+  /// This method can be overridden to, for example, be less conservative and
+  /// propagate the information even if some return sites are unknown.
+  virtual LogicalResult
+  visitCallOperation(CallOpInterface call,
+                     ArrayRef<const AbstractSparseLattice *> operandLattices,
+                     ArrayRef<AbstractSparseLattice *> resultLattices);
+
+  /// Visits a callable operation. Computes the argument lattices from call
+  /// sites if all call sites are known; otherwise, conservatively marks them
+  /// as having reached their pessimistic fixpoints.
+  /// This method can be overridden to, for example, be less conservative and
+  /// propagate the information even if some call sites are unknown.
+  virtual void
+  visitCallableOperation(CallableOpInterface callable,
+                         ArrayRef<AbstractSparseLattice *> argLattices);
+
 private:
   /// Recursively initialize the analysis on nested operations and blocks.
   LogicalResult initializeRecursively(Operation *op);
@@ -430,6 +454,16 @@ class AbstractSparseBackwardDataFlowAnalysis : public DataFlowAnalysis {
   /// Join the lattice element and propagate and update if it changed.
   void meet(AbstractSparseLattice *lhs, const AbstractSparseLattice &rhs);
 
+  /// Visits a callable operation. If all the call sites are known computes the
+  /// operand lattices of `op` from the result lattices of all the call sites;
+  /// otherwise, conservatively marks them as having reached their pessimistic
+  /// fixpoints.
+  /// This method can be overridden to, for example, be less conservative and
+  /// propagate the information even if some call sites are unknown.
+  virtual LogicalResult
+  visitCallableOperation(Operation *op, CallableOpInterface callable,
+                         ArrayRef<AbstractSparseLattice *> operandLattices);
+
 private:
   /// Recursively initialize the analysis on nested operations and blocks.
   LogicalResult initializeRecursively(Operation *op);
diff --git a/external/llvm-project/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h b/external/llvm-project/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
index 7e946495e3e7..503a2a7e6f0c 100644
--- a/external/llvm-project/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
+++ b/external/llvm-project/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
@@ -92,6 +92,10 @@ class ConvertToLLVMPattern : public ConversionPattern {
                        PatternBenefit benefit = 1);
 
 protected:
+  /// See `ConversionPattern::ConversionPattern` for information on the other
+  /// available constructors.
+  using ConversionPattern::ConversionPattern;
+
   /// Returns the LLVM dialect.
   LLVM::LLVMDialect &getDialect() const;
 
@@ -234,6 +238,47 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
   using ConvertToLLVMPattern::matchAndRewrite;
 };
 
+/// Utility class for operation conversions targeting the LLVM dialect that
+/// allows for matching and rewriting against an instance of an OpInterface
+/// class.
+template <typename SourceOp>
+class ConvertOpInterfaceToLLVMPattern : public ConvertToLLVMPattern {
+public:
+  explicit ConvertOpInterfaceToLLVMPattern(
+      const LLVMTypeConverter &typeConverter, PatternBenefit benefit = 1)
+      : ConvertToLLVMPattern(typeConverter, Pattern::MatchInterfaceOpTypeTag(),
+                             SourceOp::getInterfaceID(), benefit,
+                             &typeConverter.getContext()) {}
+
+  /// Wrappers around the RewritePattern methods that pass the derived op type.
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    return matchAndRewrite(cast<SourceOp>(op), operands, rewriter);
+  }
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    return matchAndRewrite(cast<SourceOp>(op), operands, rewriter);
+  }
+
+  /// Methods that operate on the SourceOp type. One of these must be
+  /// overridden by the derived pattern class.
+  virtual LogicalResult
+  matchAndRewrite(SourceOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const {
+    llvm_unreachable("matchAndRewrite is not implemented");
+  }
+  virtual LogicalResult
+  matchAndRewrite(SourceOp op, ArrayRef<ValueRange> operands,
+                  ConversionPatternRewriter &rewriter) const {
+    return matchAndRewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+  }
+
+private:
+  using ConvertToLLVMPattern::matchAndRewrite;
+};
+
 /// Generic implementation of one-to-one conversion from "SourceOp" to
 /// "TargetOp" where the latter belongs to the LLVM dialect or an equivalent.
 /// Upholds a convention that multi-result operations get converted into an
diff --git a/external/llvm-project/mlir/include/mlir/Conversion/Passes.td b/external/llvm-project/mlir/include/mlir/Conversion/Passes.td
index 3cb984f295d5..ff0d841ffe36 100644
--- a/external/llvm-project/mlir/include/mlir/Conversion/Passes.td
+++ b/external/llvm-project/mlir/include/mlir/Conversion/Passes.td
@@ -1431,6 +1431,10 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
            "bool", /*default=*/"false",
            "Enables the use of ArmSVE dialect while lowering the vector "
        "dialect.">,
+    Option<"armI8MM", "enable-arm-i8mm",
+           "bool", /*default=*/"false",
+           "Enables the use of Arm FEAT_I8MM instructions while lowering "
+           "the vector dialect.">,
     Option<"x86Vector", "enable-x86vector",
            "bool", /*default=*/"false",
            "Enables the use of X86Vector dialect while lowering the vector "
diff --git a/external/llvm-project/mlir/include/mlir/Debug/BreakpointManagers/TagBreakpointManager.h b/external/llvm-project/mlir/include/mlir/Debug/BreakpointManagers/TagBreakpointManager.h
index 85fdb9a63286..af138da84e02 100644
--- a/external/llvm-project/mlir/include/mlir/Debug/BreakpointManagers/TagBreakpointManager.h
+++ b/external/llvm-project/mlir/include/mlir/Debug/BreakpointManagers/TagBreakpointManager.h
@@ -48,7 +48,7 @@ class TagBreakpointManager
   /// If a breakpoint already exists for the given tag, return the existing
   /// instance.
   TagBreakpoint *addBreakpoint(StringRef tag) {
-    auto result = breakpoints.insert({tag, nullptr});
+    auto result = breakpoints.try_emplace(tag);
     auto &it = result.first;
     if (result.second)
       it->second = std::make_unique<TagBreakpoint>(tag.str());
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/external/llvm-project/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 02308568c1ad..d58558ac3288 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -112,6 +112,38 @@ def AMDGPU_ExtPackedFp8Op :
   }];
 }
 
+def AMDGPU_ScaledExtPackedOp
+    : AMDGPU_Op<"scaled_ext_packed", [Pure]>,
+      Arguments<(
+          ins AnyTypeOf<[VectorOfLengthAndType<[1, 2, 3, 4], [F8E5M2, F8E4M3FN]>,
+                         VectorOfLengthAndType<[1, 2, 3, 4, 5, 6, 7, 8],
+                                               [F4E2M1FN]>]>:$source,
+          F32:$scale,
+          ConfinedAttr<I32Attr, [IntNonNegative, IntMaxValue<7>]>:$index)>,
+      Results<(
+          outs AnyTypeOf<[FixedVectorOfLengthAndType<[2], [F32]>,
+                          FixedVectorOfLengthAndType<[2], [F16]>,
+                          FixedVectorOfLengthAndType<[2], [BF16]>]>:$res)> {
+  let summary = "Extend a vector of packed floating point values";
+
+  let description = [{
+    Extend and scale two packed floats in `source[index]` to two floats and 
+    return them.
+
+    This rather unusual signature arises from the fact that AMD GPUs cannot
+    easily work with sub 32-bit quantities, so the compiler intrinsics for
+    extending 8-bit floats (which are, currently, the only way to work with
+    this operation) take packed vectors of 2 such floats.
+
+    If the passed-in vector has fewer than two elements, or the input is scalar,
+    the remaining values in the <2 x i8> will be filled with
+    undefined values as needed.
+  }];
+  let assemblyFormat = [{
+    attr-dict $source `[` $index `]` `,` $scale `:` type($source) `to` type($res)
+  }];
+}
+
 def AMDGPU_PackedTrunc2xFp8Op :
     AMDGPU_Op<"packed_trunc_2xfp8", [Pure, AttrSizedOperandSegments]>,
     Arguments<(ins F32:$sourceA,
@@ -139,6 +171,36 @@ def AMDGPU_PackedTrunc2xFp8Op :
   let hasVerifier = 1;
 }
 
+def AMDGPU_PackedScaledTruncOp
+    : AMDGPU_Op<"packed_scaled_trunc", [Pure]>,
+      Arguments<(ins VectorOfLengthAndType<[1, 2], [F32, F16, BF16]>:$source,
+          F32:$scale,
+          ConfinedAttr<I32Attr, [IntNonNegative, IntMaxValue<7>]>:$index,
+          Optional<AnyTypeOf<
+              [FixedVectorOfLengthAndType<[4], [F8E5M2, F8E4M3FN]>,
+               FixedVectorOfLengthAndType<[8], [F4E2M1FN]>]>>:$existing)>,
+      Results<(
+          outs AnyTypeOf<[FixedVectorOfLengthAndType<[4], [F8E5M2, F8E4M3FN]>,
+                          FixedVectorOfLengthAndType<[8], [F4E2M1FN]>]>:$res)> {
+  let summary = "Round two floats into a packed vector of floats";
+  let description = [{
+    Scale and round the inputs `source` (which is undefined if not
+    specified) into the low or high word (bottom two or top two) elements
+    of the returned vector, keeping the other two elements of `existing`
+    unchanged if present (or undefined if it was not passed in).
+
+    The reason for this odd signature is that AMD GPUs cannot easily work with
+    sub-registers, and so the conversion intrinsics take 32-bit wide
+    packed vectors of float values.
+  }];
+  let assemblyFormat = [{
+    attr-dict $source `into` ($existing^):(`undef`)? `[` $index `]`
+    `,` $scale
+    `:` type($source) `to` type($res) (`into` type($existing)^)?
+  }];
+  let hasVerifier = 1;
+}
+
 def AMDGPU_PackedStochRoundFp8Op :
     AMDGPU_Op<"packed_stoch_round_fp8", [Pure]>,
     Arguments<(ins F32:$source,
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/AMX/Transforms.h b/external/llvm-project/mlir/include/mlir/Dialect/AMX/Transforms.h
index 4a751d99ceee..7391ec2ff6b1 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/AMX/Transforms.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/AMX/Transforms.h
@@ -25,6 +25,9 @@ void populateAMXLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter,
 /// intrinsics.
 void configureAMXLegalizeForExportTarget(LLVMConversionTarget &target);
 
+/// Register LLVM conversion interface for AMX dialect.
+void registerConvertAMXToLLVMInterface(DialectRegistry &registry);
+
 } // namespace mlir
 
 #endif // MLIR_DIALECT_AMX_TRANSFORMS_H
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h b/external/llvm-project/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
index bf830a29613f..5c538d28c183 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
@@ -32,6 +32,20 @@ enum class BoundType;
 
 namespace affine {
 class AffineApplyOp;
+class AffineDelinearizeIndexOp;
+class AffineLinearizeIndexOp;
+
+/// Lowers `affine.delinearize_index` into a sequence of division and remainder
+/// operations.
+LogicalResult lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
+                                            AffineDelinearizeIndexOp op);
+
+/// Lowers `affine.linearize_index` into a sequence of multiplications and
+/// additions. Make a best effort to sort the input indices so that
+/// the most loop-invariant terms are at the left of the additions
+/// to enable loop-invariant code motion.
+LogicalResult lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
+                                          AffineLinearizeIndexOp op);
 
 /// Populate patterns that expand affine index operations into more fundamental
 /// operations (not necessarily restricted to Affine dialect).
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/external/llvm-project/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index 599b3b982ec7..adc27ae6bdaf 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -1215,6 +1215,58 @@ def Arith_ExtFOp : Arith_FToFCastOp<"extf", [DeclareOpInterfaceMethods<ArithFast
                           attr-dict `:` type($in) `to` type($out) }];
 }
 
+//===----------------------------------------------------------------------===//
+// Scaling ExtFOp
+//===----------------------------------------------------------------------===//
+def Arith_ScalingExtFOp
+    : Arith_Op<
+          "scaling_extf", [Pure, SameInputOutputTensorDims,
+                           DeclareOpInterfaceMethods<ArithFastMathInterface>,
+                           DeclareOpInterfaceMethods<CastOpInterface>]>,
+      Arguments<(ins FloatLike:$in, FloatLike:$scale,
+          OptionalAttr<Arith_FastMathAttr>:$fastmath)>,
+      Results<(outs FloatLike:$out)> {
+  let summary = "Upcasts input floats using provided scales values following "
+                "OCP MXFP Spec";
+  let description = [{
+  This operation upcasts input floating-point values using provided scale 
+  values. It expects both scales and the input operand to be of the same shape, 
+  making the operation elementwise. Scales are usually calculated per block 
+  following the OCP MXFP spec as described in https://arxiv.org/abs/2310.10537.
+
+  If scales are calculated per block where blockSize != 1, then scales may 
+  require broadcasting to make this operation elementwise. For example, let's 
+  say the input is of shape `<dim1 x dim2 x ... dimN>`. Given blockSize != 1 and 
+  assuming quantization happens on the last axis, the input can be reshaped to 
+  `<dim1 x dim2 x ... (dimN/blockSize) x blockSize>`. Scales will be calculated 
+  per block on the last axis. Therefore, scales will be of shape 
+  `<dim1 x dim2 x ... (dimN/blockSize) x 1>`. Scales could also be of some other 
+  shape as long as it is broadcast compatible with the input, e.g., 
+  `<1 x 1 x ... (dimN/blockSize) x 1>`.
+
+  In this example, before calling into `arith.scaling_extf`, scales must be 
+  broadcasted to `<dim1 x dim2 x dim3 ... (dimN/blockSize) x blockSize>`. Note 
+  that there could be multiple quantization axes. Internally, 
+  `arith.scaling_extf` would perform the following:
+ 
+    ```
+    resultTy = get_type(result) 
+    scaleTy  = get_type(scale)
+    inputTy = get_type(input)
+    scale.exponent = arith.truncf(scale) : scaleTy to f8E8M0
+    scale.extf = arith.extf(scale.exponent) : f8E8M0 to resultTy
+    input.extf = arith.extf(input) : inputTy to resultTy
+    result = arith.mulf(scale.extf, input.extf)
+    ```
+    It propagates NaN values. Therefore, if either scale or the input element 
+    contains NaN, then the output element value will also be a NaN.
+  }];
+  let hasVerifier = 1;
+  let assemblyFormat =
+      [{ $in `,` $scale (`fastmath` `` $fastmath^)? attr-dict `:` 
+      type($in) `,` type($scale) `to` type($out)}];
+}
+
 //===----------------------------------------------------------------------===//
 // TruncIOp
 //===----------------------------------------------------------------------===//
@@ -1280,6 +1332,63 @@ def Arith_TruncFOp :
                           attr-dict `:` type($in) `to` type($out) }];
 }
 
+//===----------------------------------------------------------------------===//
+// Scaling TruncFOp
+//===----------------------------------------------------------------------===//
+
+def Arith_ScalingTruncFOp
+    : Arith_Op<"scaling_truncf",
+               [Pure, SameInputOutputTensorDims,
+                DeclareOpInterfaceMethods<ArithRoundingModeInterface>,
+                DeclareOpInterfaceMethods<ArithFastMathInterface>,
+                DeclareOpInterfaceMethods<CastOpInterface>]>,
+      Arguments<(ins FloatLike:$in, FloatLike:$scale,
+          OptionalAttr<Arith_RoundingModeAttr>:$roundingmode,
+          OptionalAttr<Arith_FastMathAttr>:$fastmath)>,
+      Results<(outs FloatLike:$out)> {
+  let summary = "Downcasts input floating point values using provided scales "
+                "values following OCP MXFP Spec";
+  let description = [{
+    This operation downcasts input using the provided scale values. It expects 
+    both scales and the input operand to be of the same shape and, therefore, 
+    makes the operation elementwise. Scales are usually calculated per block 
+    following the OCP MXFP spec as described in https://arxiv.org/abs/2310.10537.
+    Users are required to normalize and clamp the scales as necessary before calling
+    passing them to this operation.  OCP MXFP spec also does the flushing of denorms
+    on the input operand, which should be handled during lowering by passing appropriate 
+    fastMath flag to this operation. 
+
+    If scales are calculated per block where blockSize != 1, scales may require 
+    broadcasting to make this operation elementwise. For example, let's say the 
+    input is of shape `<dim1 x dim2 x ... dimN>`. Given blockSize != 1 and 
+    assuming quantization happens on the last axis, the input can be reshaped to 
+    `<dim1 x dim2 x ... (dimN/blockSize) x blockSize>`. Scales will be calculated 
+    per block on the last axis. Therefore, scales will be of shape 
+    `<dim1 x dim2 x ... (dimN/blockSize) x 1>`. Scales could also be of some other 
+    shape as long as it is broadcast compatible with the input, e.g., 
+    `<1 x 1 x ... (dimN/blockSize) x 1>`.
+
+    In this example, before calling into `arith.scaling_truncf`, scales must be 
+    broadcasted to `<dim1 x dim2 x dim3 ... (dimN/blockSize) x blockSize>`. Note 
+    that there could be multiple quantization axes. Internally, 
+    `arith.scaling_truncf` would perform the following:
+
+    ```
+    scaleTy = get_type(scale)
+    inputTy = get_type(input)
+    resultTy = get_type(result)
+    scale.exponent = arith.truncf(scale) : scaleTy to f8E8M0
+    scale.extf = arith.extf(scale.exponent) : f8E8M0 to inputTy
+    result = arith.divf(input, scale.extf)
+    result.cast = arith.truncf(result, resultTy)
+    ```
+  }];
+  let hasVerifier = 1;
+  let assemblyFormat =
+      [{ $in `,` $scale ($roundingmode^)? (`fastmath` `` $fastmath^)? attr-dict `:` 
+      type($in) `,` type($scale) `to` type($out)}];
+}
+
 //===----------------------------------------------------------------------===//
 // UIToFPOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h b/external/llvm-project/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
index 5aaac8d8e3dc..e0a4567d6f40 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
@@ -62,6 +62,9 @@ void populateExpandBFloat16Patterns(RewritePatternSet &patterns);
 /// Add patterns to expand Arith f8e8m0 patterns to lower level bitcasts/shifts.
 void populateExpandF8E8M0Patterns(RewritePatternSet &patterns);
 
+/// Add patterns to expand scaling ExtF/TruncF ops to equivalent arith ops
+void populateExpandScalingExtTruncPatterns(RewritePatternSet &patterns);
+
 /// Add patterns to expand Arith ops.
 void populateArithExpandOpsPatterns(RewritePatternSet &patterns);
 
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/CMakeLists.txt b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/CMakeLists.txt
index 1c679bcd049b..3de3ec3f3a0e 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/CMakeLists.txt
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/CMakeLists.txt
@@ -4,3 +4,5 @@ add_mlir_doc(ArmNeon ArmNeon Dialects/ -gen-dialect-doc -dialect=arm_neon)
 set(LLVM_TARGET_DEFINITIONS ArmNeon.td)
 mlir_tablegen(ArmNeonConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRArmNeonConversionsIncGen)
+
+add_subdirectory(TransformOps)
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.h b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.h
new file mode 100644
index 000000000000..ad57e6e1f8ff
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.h
@@ -0,0 +1,31 @@
+//===- ArmNeonVectorTransformOps.h - Vector transform ops -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ARM_NEON_TRANSFORMOPS_VECTORTRANSFORMOPS_H
+#define MLIR_DIALECT_ARM_NEON_TRANSFORMOPS_VECTORTRANSFORMOPS_H
+
+#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/OpImplementation.h"
+
+//===----------------------------------------------------------------------===//
+// ArmNeon Vector Transform Operations
+//===----------------------------------------------------------------------===//
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.h.inc"
+
+namespace mlir {
+class DialectRegistry;
+
+namespace arm_neon {
+void registerTransformDialectExtension(DialectRegistry &registry);
+
+} // namespace arm_neon
+} // namespace mlir
+
+#endif // MLIR_DIALECT_ARM_NEON_TRANSFORMOPS_VECTORTRANSFORMOPS_H
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.td b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.td
new file mode 100644
index 000000000000..bcaca7da967f
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.td
@@ -0,0 +1,27 @@
+//===- ArmNeonVectorTransformOps.td - Arm Neon TD ops ------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef ARM_NEON_VECTOR_TRANSFORM_OPS
+#define ARM_NEON_VECTOR_TRANSFORM_OPS
+
+include "mlir/Dialect/Transform/IR/TransformAttrs.td"
+include "mlir/Dialect/Transform/IR/TransformDialect.td"
+include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td"
+
+def ApplyArmNeonContractionToI8MMPatternsOp
+    : Op<Transform_Dialect,
+         "apply_patterns.arm_neon.vector_contract_to_i8mm",
+         [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+    Indicates that vector.contract operations should be lowered to
+    finer-grained vector primitives from the ArmNeon dialect.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
+
+#endif // ARM_NEON_VECTOR_TRANSFORM_OPS
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/CMakeLists.txt b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/CMakeLists.txt
new file mode 100644
index 000000000000..b8bc72a2bb73
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmNeon/TransformOps/CMakeLists.txt
@@ -0,0 +1,6 @@
+set(LLVM_TARGET_DEFINITIONS ArmNeonVectorTransformOps.td)
+mlir_tablegen(ArmNeonVectorTransformOps.h.inc -gen-op-decls)
+mlir_tablegen(ArmNeonVectorTransformOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRArmNeonVectorTransformOpsIncGen)
+
+add_mlir_doc(ArmNeonVectorTransformOps ArmNeonVectorTransformOps Dialects/ -gen-op-doc)
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt
index 9f57627c321f..cb1e9d01821a 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(IR)
 add_subdirectory(Transforms)
+add_subdirectory(TransformOps)
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.h b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.h
new file mode 100644
index 000000000000..7f22cd1fe643
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.h
@@ -0,0 +1,31 @@
+//===- ArmSVEVectorTransformOps.h - Vector transform ops --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ARM_SVE_VECTOR_TRANSFORMOPS_H
+#define MLIR_DIALECT_ARM_SVE_VECTOR_TRANSFORMOPS_H
+
+#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/OpImplementation.h"
+
+//===----------------------------------------------------------------------===//
+// ArmSVE Vector Transform Operations
+//===----------------------------------------------------------------------===//
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.h.inc"
+
+namespace mlir {
+class DialectRegistry;
+
+namespace arm_sve {
+void registerTransformDialectExtension(DialectRegistry &registry);
+
+} // namespace arm_sve
+} // namespace mlir
+
+#endif // MLIR_DIALECT_ARM_SVE_VECTOR_TRANSFORMOPS_H
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.td b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.td
new file mode 100644
index 000000000000..53784982be6d
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.td
@@ -0,0 +1,26 @@
+//===- ArmSVEVectorTransformOps.td - Arm SVE transform ops--*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef ARMSVE_VECTOR_TRANSFORM_OPS
+#define ARMSVE_VECTOR_TRANSFORM_OPS
+
+include "mlir/Dialect/Transform/IR/TransformAttrs.td"
+include "mlir/Dialect/Transform/IR/TransformDialect.td"
+include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td"
+
+def ApplyArmSVELowerContractionPatternsOp
+    : Op<Transform_Dialect, "apply_patterns.arm_sve.vector_contract_to_i8mm",
+         [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+    Indicates that vector contraction-like operations should be lowered to
+    finer-grained vector primitives using the ArmSVE dialect.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
+
+#endif // ARMSVE_VECTOR_TRANSFORM_OPS
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/CMakeLists.txt b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/CMakeLists.txt
new file mode 100644
index 000000000000..ce8d8fea7f18
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/TransformOps/CMakeLists.txt
@@ -0,0 +1,6 @@
+set(LLVM_TARGET_DEFINITIONS ArmSVEVectorTransformOps.td)
+mlir_tablegen(ArmSVEVectorTransformOps.h.inc -gen-op-decls)
+mlir_tablegen(ArmSVEVectorTransformOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRArmSVEVectorTransformOpsIncGen)
+
+add_mlir_doc(ArmSVEVectorTransformOps ArmSVEVectorTransformOps Dialects/ -gen-op-doc)
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/Transforms/Transforms.h b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/Transforms/Transforms.h
index 8665c8224cc4..232e2be29e57 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/Transforms/Transforms.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/ArmSVE/Transforms/Transforms.h
@@ -20,6 +20,9 @@ class RewritePatternSet;
 void populateArmSVELegalizeForLLVMExportPatterns(
     const LLVMTypeConverter &converter, RewritePatternSet &patterns);
 
+void populateLowerContractionToSVEI8MMPatternPatterns(
+    RewritePatternSet &patterns);
+
 /// Configure the target to support lowering ArmSVE ops to ops that map to LLVM
 /// intrinsics.
 void configureArmSVELegalizeForExportTarget(LLVMConversionTarget &target);
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td b/external/llvm-project/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
index d6d15a08073e..6dbcdefbc933 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
@@ -15,7 +15,6 @@
 
 include "mlir/Dialect/Async/IR/AsyncDialect.td"
 include "mlir/Dialect/Async/IR/AsyncTypes.td"
-include "mlir/Interfaces/CallInterfaces.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
@@ -248,10 +247,6 @@ def Async_CallOp : Async_Op<"call",
     operand_range getArgOperands() {
       return {arg_operand_begin(), arg_operand_end()};
     }
-    
-    operand_range getCallOperands() {
-      return {arg_operand_begin(), arg_operand_end()};
-    }
 
     MutableOperandRange getArgOperandsMutable() {
       return getOperandsMutable();
@@ -381,7 +376,7 @@ def Async_CreateGroupOp : Async_Op<"create_group", [Pure]> {
 }
 
 def Async_AddToGroupOp : Async_Op<"add_to_group", []> {
-  let summary = "adds and async token or value to the group";
+  let summary = "adds an async token or value to the group";
   let description = [{
     The `async.add_to_group` adds an async token or value to the async group.
     Returns the rank of the added element in the group. This rank is fixed
@@ -660,7 +655,7 @@ def Async_RuntimeLoadOp : Async_Op<"runtime.load",
 }
 
 def Async_RuntimeAddToGroupOp : Async_Op<"runtime.add_to_group", []> {
-  let summary = "adds and async token or value to the group";
+  let summary = "adds an async token or value to the group";
   let description = [{
     The `async.runtime.add_to_group` adds an async token or value to the async
     group. Returns the rank of the added element in the group.
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h b/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h
index 752a4a2c6f42..5d33817c7d9f 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h
@@ -104,7 +104,7 @@ struct DeallocationOptions {
 /// BufferDeallocation pass.
 class DeallocationState {
 public:
-  DeallocationState(Operation *op);
+  DeallocationState(Operation *op, SymbolTableCollection &symbolTables);
 
   // The state should always be passed by reference.
   DeallocationState(const DeallocationState &) = delete;
@@ -189,7 +189,7 @@ class DeallocationState {
 private:
   // Symbol cache to lookup functions from call operations to check attributes
   // on the function operation.
-  SymbolTableCollection symbolTable;
+  SymbolTableCollection &symbolTable;
 
   // Mapping from each SSA value with MemRef type to the associated ownership in
   // each block.
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
index 3d4dcdee2663..6051aea84997 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
@@ -131,9 +131,8 @@ def Bufferization_AllocTensorOp : Bufferization_Op<"alloc_tensor",
       assert(!getCopy() && "no dim sizes specified when copying a tensor");
       assert(isDynamicDim(idx) && "expected dynamic size");
       ArrayRef<int64_t> shape = getType().getShape();
-      return std::count_if(
-          shape.begin(), shape.begin() + idx,
-          [&](int64_t size) { return ShapedType::isDynamic(size); });
+      return std::count_if(shape.begin(), shape.begin() + idx,
+                           ShapedType::isDynamic);
     }
 
     // Return the Value of the dynamic size of the tensor at dimension
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h b/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
index a8cc37a103f0..596c470ef6d2 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
@@ -120,8 +120,10 @@ func::FuncOp buildDeallocationLibraryFunction(OpBuilder &builder, Location loc,
                                               SymbolTable &symbolTable);
 
 /// Run the ownership-based buffer deallocation.
-LogicalResult deallocateBuffersOwnershipBased(FunctionOpInterface op,
-                                              DeallocationOptions options);
+LogicalResult
+deallocateBuffersOwnershipBased(FunctionOpInterface op,
+                                DeallocationOptions options,
+                                SymbolTableCollection &symbolTables);
 
 // Options struct for BufferResultsToOutParams pass.
 // Note: defined only here, not in tablegen.
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/external/llvm-project/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index d4aea52a0d48..e53d3e45875d 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -1304,7 +1304,7 @@ def EmitC_VerbatimOp : EmitC_Op<"verbatim"> {
     FailureOr<SmallVector<::mlir::emitc::ReplacementItem>> parseFormatString();
   }];
 
-  let arguments = (ins StrAttr:$value, Variadic<EmitCType>:$fmtArgs);
+  let arguments = (ins StrAttr:$value, Variadic<AnyTypeOf<[EmitCType, EmitC_LValueType]>>:$fmtArgs);
 
   let builders = [OpBuilder<(ins "::mlir::StringAttr":$value),
                             [{ build($_builder, $_state, value, {}); }]>];
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index c0324d561b77..eda1d544cd81 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1893,6 +1893,7 @@ def LLVM_LLVMFuncOp : LLVM_Op<"func", [
     OptionalAttr<FramePointerKindAttr>:$frame_pointer,
     OptionalAttr<StrAttr>:$target_cpu,
     OptionalAttr<StrAttr>:$tune_cpu,
+    OptionalAttr<StrAttr>:$reciprocal_estimates,
     OptionalAttr<StrAttr>:$prefer_vector_width,
     OptionalAttr<LLVM_TargetFeaturesAttr>:$target_features,
     OptionalAttr<BoolAttr>:$unsafe_fp_math,
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
index fc38a3fb2d38..6137bb087c57 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
@@ -36,12 +36,16 @@ constexpr int kSharedMemoryAlignmentBit = 128;
 
 /// NVVM memory space identifiers.
 enum NVVMMemorySpace {
+  /// Generic memory space identifier.
+  kGenericMemorySpace = 0,
   /// Global memory space identifier.
   kGlobalMemorySpace = 1,
   /// Shared memory space identifier.
   kSharedMemorySpace = 3,
   /// Constant memory space identifier.
   kConstantMemorySpace = 4,
+  /// Local memory space identifier.
+  kLocalMemorySpace = 5,
   /// Tensor memory space identifier.
   /// Tensor memory is available only in arch-accelerated
   /// variants from sm100 onwards.
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 2424e3af80d2..2dd7ac29cfed 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -25,6 +25,7 @@ include "mlir/Dialect/LLVMIR/LLVMTypes.td"
 def LLVM_PointerGeneric : LLVM_PointerInAddressSpace<0>;
 def LLVM_PointerGlobal : LLVM_PointerInAddressSpace<1>;
 def LLVM_PointerShared : LLVM_PointerInAddressSpace<3>;
+def LLVM_PointerLocal : LLVM_PointerInAddressSpace<5>;
 def LLVM_PointerTensor : LLVM_PointerInAddressSpace<6>;
 def LLVM_PointerSharedCluster : LLVM_PointerInAddressSpace<7>;
 
@@ -118,6 +119,25 @@ class NVVM_Attr<string attrName, string attrMnemonic, list<Trait> traits = []>
   let mnemonic = attrMnemonic;
 }
 
+// Cache Eviction Priority enum definitions
+def EvictNormal : I32EnumCase<"EvictNormal", 0, "evict_normal">;
+def EvictFirst : I32EnumCase<"EvictFirst", 1, "evict_first">;
+def EvictLast : I32EnumCase<"EvictLast", 2, "evict_last">;
+def EvictUnchanged : I32EnumCase<"EvictUnchanged", 3, "evict_unchanged">;
+def NoAllocate : I32EnumCase<"NoAllocate", 4, "no_allocate">;
+
+def CacheEvictionPriority : I32Enum<"CacheEvictionPriority",
+                                    "NVVM Cache Eviction Priority",
+                                    [EvictNormal, EvictFirst, EvictLast,   
+                                     EvictUnchanged, NoAllocate]> {
+  let cppNamespace = "::mlir::NVVM";
+}
+
+def CacheEvictionPriorityAttr : EnumAttr<NVVM_Dialect, CacheEvictionPriority, 
+                                         "cache_eviction_priority"> {
+  let assemblyFormat = "$value";
+}
+
 //===----------------------------------------------------------------------===//
 // NVVM intrinsic operations
 //===----------------------------------------------------------------------===//
@@ -2333,6 +2353,60 @@ def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp :
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// NVVM Prefetch Op
+//===----------------------------------------------------------------------===//
+
+def PrefetchCacheLevelL1 : I32EnumCase<"L1", 0, "L1">;
+def PrefetchCacheLevelL2 : I32EnumCase<"L2", 1, "L2">;
+
+def PrefetchCacheLevel : I32Enum<"PrefetchCacheLevel",
+                                 "NVVM Prefetch Cache Level",
+                                 [PrefetchCacheLevelL1, PrefetchCacheLevelL2]> {
+  let cppNamespace = "::mlir::NVVM";
+}
+
+def PrefetchCacheLevelAttr : EnumAttr<NVVM_Dialect, PrefetchCacheLevel, "prefetch_cache_level"> {
+  let assemblyFormat = "$value";
+}
+
+def NVVM_PrefetchOp : NVVM_Op<"prefetch"> {
+  let summary = "Brings the cache line containing an address into the specified cache level";
+  let description = [{
+    Operand `addr` can be a global, local or generic address pointer. No 
+    operation is performed if `addr` maps to a `shared` memory location.
+
+    The `cacheLevel` attribute specifies the cache level to which the cache line
+    containing the specified address is brought.
+    
+    `uniform` can be specified after the `cacheLevel` to indicate that the 
+    prefetch is performed to the specified uniform cache level. If `uniform` is 
+    specified, `addr` must be a generic address pointer and no operation is 
+    performed if `addr` maps to a `const`, `local`, or `shared` memory location.
+
+    The `evictPriority` attribute is optional and specifies the cache eviction
+    priority when `cacheLevel` is L2.
+
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prefetch-prefetchu)
+  }];
+  let arguments = (ins PrefetchCacheLevelAttr:$cacheLevel,
+                       UnitAttr:$uniform,
+                       AnyTypeOf<[LLVM_PointerGlobal,
+                                  LLVM_PointerLocal,
+                                  LLVM_PointerGeneric]>:$addr,
+                       OptionalAttr<CacheEvictionPriorityAttr>:$evictPriority);
+  let assemblyFormat = "`level` `=` $cacheLevel (`uniform` $uniform^)? `,` $addr (`,` `evict_priority` `=` $evictPriority^)? attr-dict `:` type($addr)";
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    static llvm::Intrinsic::ID getIntrinsicID(NVVM::PrefetchOp &op);
+  }];
+  let llvmBuilder = [{
+    auto intId = NVVM::PrefetchOp::getIntrinsicID(op);
+    createIntrinsicCall(builder, intId, $addr);
+  }];
+}
+
 def NVVM_PrefetchTensorMapOp : NVVM_Op<"prefetch.tensormap",
                     [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,
   Arguments<(ins LLVM_AnyPointer:$tmaDescriptor, PtxPredicate:$predicate)> {
@@ -3703,11 +3777,65 @@ def NVVM_DotAccumulate4WayOp : NVVM_Op<"dot.accumulate.4way"> {
   }];
 }
 
+def NVVM_DotAccumulate2WayOp : NVVM_Op<"dot.accumulate.2way"> {
+  let summary = "Two-way 16-bit to 8-bit dot product-accumulate instruction";
+  let description = [{
+    Performs a two-way 16-bit to 8-bit dot-product which is accumulated in a 
+    32-bit result.
+    Operand `a` is a vector of two 16-bit elements and operand `b` a vector 
+    of four 8-bit elements between which the dot product is computed.
+
+    The `a_type` and `b_type` attributes specify the type of the elements in `a`
+    and `b` respectively.
+    If `a_type` or `b_type` is `s`, then the elements in the corresponding 
+    vector are sign-extended to 32-bit before the dot product is computed.
+    If `a_type` or `b_type` is `u`, then the elements in the corresponding 
+    vector are zero-extended to 32-bit instead.
+
+    The `b_hi` boolean attribute specifies which two bytes of `b` are used for 
+    the dot product. If `b_hi` is true, then the dot product is computed 
+    between  `a` and elements at indices 2 and 3 of `b`. If `b_hi` is false, 
+    then the dot product is computed between `a` and elements at indices 0 and 
+    1 of `b`.
+
+    Operand `c` is a 32-bit integer to which the result is accumulated. It is
+    treated as holding a signed integer if any of `a_type` or `b_type` is 
+    signed.
+    
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp2a)
+  }];
+
+  let arguments = (ins
+    VectorOfLengthAndType<[2], [I16]>:$a,
+    DotAccumulateTypeAttr:$a_type,
+    VectorOfLengthAndType<[4], [I8]>:$b,
+    DotAccumulateTypeAttr:$b_type,
+    I32:$c,
+    BoolAttr:$b_hi
+  );
+
+  let results = (outs I32:$res);
+
+  let assemblyFormat = "$a $a_type `,` $b $b_type `,` $c attr-dict `:` type($a) `,` type($b)";
+  
+  let extraClassDeclaration = [{
+    static mlir::NVVM::IDArgPair
+    getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
+                          llvm::IRBuilderBase &builder);
+  }];
+  
+  string llvmBuilder = [{
+    auto [id, args] = NVVM::DotAccumulate2WayOp::getIntrinsicIDAndArgs(
+                        *op, moduleTranslation, builder);
+    $res = createIntrinsicCall(builder, id, args);
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // NVVM target attribute.
 //===----------------------------------------------------------------------===//
 
-def NVVM_TargettAttr : NVVM_Attr<"NVVMTarget", "target", 
+def NVVM_TargetAttr : NVVM_Attr<"NVVMTarget", "target", 
   [DeclareAttrInterfaceMethods<GPUTargetAttrVerifyInterface>]> {
   let description = [{
     GPU target attribute for controlling compilation of NVIDIA targets. All
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
index c2a82ffc1c43..ce1fe5a03c49 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
@@ -13,8 +13,8 @@
 // pointed to here. However the following links contain more information about
 // ROCDL (ROCm-Device-Library)
 //
-// https://github.com/RadeonOpenCompute/ROCm-Device-Libs/blob/amd-stg-open/doc/OCML.md
-// https://github.com/RadeonOpenCompute/ROCm-Device-Libs/blob/amd-stg-open/doc/OCKL.md
+// https://github.com/ROCm/llvm-project/blob/amd-staging/amd/device-libs/doc/OCML.md
+// https://github.com/ROCm/llvm-project/blob/amd-staging/amd/device-libs/doc/OCKL.md
 // https://llvm.org/docs/AMDGPUUsage.html
 //
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/external/llvm-project/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index f33ecb28d27c..77e3074661ab 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -174,6 +174,7 @@ def AssumeAlignmentOp : MemRef_Op<"assume_alignment", [
   }];
 
   let hasVerifier = 1;
+  let hasFolder = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index ff5845343313..4eb666239d4e 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -151,6 +151,12 @@ mlir::ValueRange getDataOperands(mlir::Operation *accOp);
 /// Used to get a mutable range iterating over the data operands.
 mlir::MutableOperandRange getMutableDataOperands(mlir::Operation *accOp);
 
+/// Used to obtain the enclosing compute construct operation that contains
+/// the provided `region`. Returns nullptr if no compute construct operation
+/// is found. The returns operation is one of types defined by
+///`ACC_COMPUTE_CONSTRUCT_OPS`.
+mlir::Operation *getEnclosingComputeOp(mlir::Region &region);
+
 /// Used to check whether the provided `type` implements the `PointerLikeType`
 /// interface.
 inline bool isPointerLikeType(mlir::Type type) {
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 083a18d80704..34312655115a 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2246,6 +2246,14 @@ def OpenACC_LoopOp : OpenACC_Op<"loop",
     // device_types. This is for the case where there is no expression specified
     // in a 'gang'.
     void addEmptyGang(MLIRContext *, llvm::ArrayRef<DeviceType>);
+
+    // Return whether this LoopOp has an auto, seq, or independent for the
+    // specified device-type.
+    bool hasParallelismFlag(DeviceType);
+
+    // Return whether this LoopOp has a gang, worker, or vector applying to the
+    // 'default'/None device-type.
+    bool hasDefaultGangWorkerVector();
   }];
 
   let hasCustomAssemblyFormat = 1;
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/external/llvm-project/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 036c6a6e350a..ac80926053a2 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -889,7 +889,7 @@ def FlushOp : OpenMP_Op<"flush", clauses = [
     specified or implied.
   }] # clausesDescription;
 
-  let arguments = !con((ins Variadic<OpenMP_PointerLikeType>:$varList),
+  let arguments = !con((ins Variadic<AnyType>:$varList),
                        clausesArgs);
 
   // Override inherited assembly format to include `varList`.
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h b/external/llvm-project/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
index 33a43ce2ee7b..f686ae07b9a9 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
@@ -136,15 +136,17 @@ struct SCFTilingResult {
   SmallVector<Value> initialValues;
   /// The `scf.for` operations that iterate over the tiles.
   SmallVector<LoopLikeOpInterface> loops;
-  /// The result generated by the loop nest in tiling, may hold partial results,
-  /// which need to be merged to match the computation of the untiled operation.
-  /// `mergeResult` contains the operations used to perform this merge from
-  /// partial results and the values that can be used as replacements of
-  /// the untiled operation.
-  MergeResult mergeResult;
+  /// Values to use as replacements for the untiled op. Is the same size as the
+  /// number of results of the untiled op.
+  SmallVector<Value> replacements;
   /// Slices generated after tiling that can be used for fusing with the tiled
   /// producer.
   SmallVector<Operation *> generatedSlices;
+  /// In cases where there as an additional merge step after tiling
+  /// return the merged ops after tiling. This list is empty when reduction
+  /// tiling strategy is
+  /// `scf::SCFTilingOptions::ReductionTilingStrategy::FullReduction.
+  SmallVector<Operation *> mergeOps;
 };
 
 /// Method to tile an op that implements the `TilingInterface` using
@@ -362,7 +364,7 @@ lowerToLoopsUsingSCFForOp(RewriterBase &rewriter, TilingInterface op);
 /// ```
 FailureOr<scf::SCFTilingResult>
 tileReductionUsingScf(RewriterBase &b, PartialReductionOpInterface op,
-                      ArrayRef<OpFoldResult> tileSize);
+                      ArrayRef<OpFoldResult> tileSizes);
 
 } // namespace scf
 } // namespace mlir
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td
index 22d5afcd7738..309079e54984 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td
@@ -23,7 +23,7 @@ class SPIRV_ArithmeticBinaryOp<string mnemonic, Type type,
       // Operands type same as result type.
       SPIRV_BinaryOp<mnemonic, type, type,
                    !listconcat(traits,
-                               [Pure, SameOperandsAndResultType])> {
+                               [Pure, AllTypesMatch<["operand1", "operand2", "result"]>])> {
   // In addition to normal types arithmetic instructions can support cooperative
   // matrix.
   let arguments = (ins
@@ -42,7 +42,7 @@ class SPIRV_ArithmeticUnaryOp<string mnemonic, Type type,
       // Operand type same as result type.
       SPIRV_UnaryOp<mnemonic, type, type,
                    !listconcat(traits,
-                               [Pure, SameOperandsAndResultType])> {
+                               [Pure, AllTypesMatch<["operand", "result"]>])> {
   // In addition to normal types arithmetic instructions can support cooperative
   // matrix.
   let arguments = (ins
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
index 8fd533db83d9..d2ba76cdad90 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
@@ -344,6 +344,7 @@ def SPV_KHR_subgroup_rotate                  : I32EnumAttrCase<"SPV_KHR_subgroup
 def SPV_KHR_non_semantic_info                : I32EnumAttrCase<"SPV_KHR_non_semantic_info", 29>;
 def SPV_KHR_terminate_invocation             : I32EnumAttrCase<"SPV_KHR_terminate_invocation", 30>;
 def SPV_KHR_cooperative_matrix               : I32EnumAttrCase<"SPV_KHR_cooperative_matrix", 31>;
+def SPV_KHR_bfloat16                         : I32EnumAttrCase<"SPV_KHR_bfloat16", 32>;
 
 def SPV_EXT_demote_to_helper_invocation  : I32EnumAttrCase<"SPV_EXT_demote_to_helper_invocation", 1000>;
 def SPV_EXT_descriptor_indexing          : I32EnumAttrCase<"SPV_EXT_descriptor_indexing", 1001>;
@@ -436,7 +437,7 @@ def SPIRV_ExtensionAttr :
       SPV_KHR_fragment_shader_barycentric, SPV_KHR_ray_cull_mask,
       SPV_KHR_uniform_group_instructions, SPV_KHR_subgroup_rotate,
       SPV_KHR_non_semantic_info, SPV_KHR_terminate_invocation,
-      SPV_KHR_cooperative_matrix,
+      SPV_KHR_cooperative_matrix, SPV_KHR_bfloat16,
       SPV_EXT_demote_to_helper_invocation, SPV_EXT_descriptor_indexing,
       SPV_EXT_fragment_fully_covered, SPV_EXT_fragment_invocation_density,
       SPV_EXT_fragment_shader_interlock, SPV_EXT_physical_storage_buffer,
@@ -1412,6 +1413,23 @@ def SPIRV_C_ShaderStereoViewNV                          : I32EnumAttrCase<"Shade
     Extension<[SPV_NV_stereo_view_rendering]>
   ];
 }
+def SPIRV_C_BFloat16TypeKHR                             : I32EnumAttrCase<"BFloat16TypeKHR", 5116> {
+  list<Availability> availability = [
+    Extension<[SPV_KHR_bfloat16]>
+  ];
+}
+def SPIRV_C_BFloat16DotProductKHR                       : I32EnumAttrCase<"BFloat16DotProductKHR", 5117> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_BFloat16TypeKHR];
+  list<Availability> availability = [
+    Extension<[SPV_KHR_bfloat16]>
+  ];
+}
+def SPIRV_C_BFloat16CooperativeMatrixKHR                : I32EnumAttrCase<"BFloat16CooperativeMatrixKHR", 5118> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_BFloat16TypeKHR, SPIRV_C_CooperativeMatrixKHR];
+  list<Availability> availability = [
+    Extension<[SPV_KHR_bfloat16]>
+  ];
+}
 
 def SPIRV_C_Bfloat16ConversionINTEL                         : I32EnumAttrCase<"Bfloat16ConversionINTEL", 6115> {
   list<Availability> availability = [
@@ -1518,7 +1536,8 @@ def SPIRV_CapabilityAttr :
       SPIRV_C_StorageTexelBufferArrayNonUniformIndexing,
       SPIRV_C_ShaderViewportIndexLayerEXT, SPIRV_C_ShaderViewportMaskNV,
       SPIRV_C_ShaderStereoViewNV, SPIRV_C_Bfloat16ConversionINTEL,
-      SPIRV_C_CacheControlsINTEL
+      SPIRV_C_CacheControlsINTEL, SPIRV_C_BFloat16TypeKHR,
+      SPIRV_C_BFloat16DotProductKHR, SPIRV_C_BFloat16CooperativeMatrixKHR
     ]>;
 
 def SPIRV_AM_Logical                 : I32EnumAttrCase<"Logical", 0>;
@@ -3217,6 +3236,16 @@ def SPIRV_ExecutionModelAttr :
       SPIRV_EM_TaskEXT, SPIRV_EM_MeshEXT
     ]>;
 
+def SPIRV_FPE_BFloat16KHR : I32EnumAttrCase<"BFloat16KHR", 0> {
+  list<Availability> availability = [
+    Capability<[SPIRV_C_BFloat16TypeKHR]>
+  ];
+}
+def SPIRV_FPEncodingAttr :
+    SPIRV_I32EnumAttr<"FPEncoding", "valid SPIR-V FPEncoding", "f_p_encoding", [
+      SPIRV_FPE_BFloat16KHR
+    ]>;
+
 def SPIRV_FC_None         : I32BitEnumAttrCaseNone<"None">;
 def SPIRV_FC_Inline       : I32BitEnumAttrCaseBit<"Inline", 0>;
 def SPIRV_FC_DontInline   : I32BitEnumAttrCaseBit<"DontInline", 1>;
@@ -4161,10 +4190,12 @@ def SPIRV_Integer : AnyIntOfWidths<[8, 16, 32, 64]>;
 def SPIRV_Int16 : TypeAlias<I16, "Int16">;
 def SPIRV_Int32 : TypeAlias<I32, "Int32">;
 def SPIRV_Float32 : TypeAlias<F32, "Float32">;
+def SPIRV_BFloat16KHR : TypeAlias<BF16, "BFloat16">;
 def SPIRV_Float : FloatOfWidths<[16, 32, 64]>;
 def SPIRV_Float16or32 : FloatOfWidths<[16, 32]>;
+def SPIRV_AnyFloat : AnyTypeOf<[SPIRV_Float, SPIRV_BFloat16KHR]>;
 def SPIRV_Vector : VectorOfLengthAndType<[2, 3, 4, 8, 16],
-                                       [SPIRV_Bool, SPIRV_Integer, SPIRV_Float]>;
+                                       [SPIRV_Bool, SPIRV_Integer, SPIRV_AnyFloat]>;
 // Component type check is done in the type parser for the following SPIR-V
 // dialect-specific types so we use "Any" here.
 def SPIRV_AnyPtr : DialectType<SPIRV_Dialect, SPIRV_IsPtrType,
@@ -4187,16 +4218,17 @@ def SPIRV_AnyStruct : DialectType<SPIRV_Dialect, SPIRV_IsStructType,
 def SPIRV_AnySampledImage : DialectType<SPIRV_Dialect, SPIRV_IsSampledImageType,
                                 "any SPIR-V sampled image type">;
 
-def SPIRV_Numerical : AnyTypeOf<[SPIRV_Integer, SPIRV_Float]>;
+def SPIRV_Numerical : AnyTypeOf<[SPIRV_Integer, SPIRV_AnyFloat]>;
 def SPIRV_Scalar : AnyTypeOf<[SPIRV_Numerical, SPIRV_Bool]>;
 def SPIRV_Aggregate : AnyTypeOf<[SPIRV_AnyArray, SPIRV_AnyRTArray, SPIRV_AnyStruct]>;
 def SPIRV_Composite :
     AnyTypeOf<[SPIRV_Vector, SPIRV_AnyArray, SPIRV_AnyRTArray, SPIRV_AnyStruct,
                SPIRV_AnyCooperativeMatrix, SPIRV_AnyMatrix]>;
 def SPIRV_Type : AnyTypeOf<[
-    SPIRV_Void, SPIRV_Bool, SPIRV_Integer, SPIRV_Float, SPIRV_Vector,
+    SPIRV_Void, SPIRV_Bool, SPIRV_Integer, SPIRV_AnyFloat, SPIRV_Vector,
     SPIRV_AnyPtr, SPIRV_AnyArray, SPIRV_AnyRTArray, SPIRV_AnyStruct,
-    SPIRV_AnyCooperativeMatrix, SPIRV_AnyMatrix, SPIRV_AnySampledImage
+    SPIRV_AnyCooperativeMatrix, SPIRV_AnyMatrix, SPIRV_AnySampledImage,
+    SPIRV_AnyImage
   ]>;
 
 def SPIRV_SignedInt : SignedIntOfWidths<[8, 16, 32, 64]>;
@@ -4338,6 +4370,7 @@ def SPIRV_OC_OpImageSampleImplicitLod         : I32EnumAttrCase<"OpImageSampleIm
 def SPIRV_OC_OpImageSampleExplicitLod         : I32EnumAttrCase<"OpImageSampleExplicitLod", 88>;
 def SPIRV_OC_OpImageSampleProjDrefImplicitLod : I32EnumAttrCase<"OpImageSampleProjDrefImplicitLod", 93>;
 def SPIRV_OC_OpImageDrefGather                : I32EnumAttrCase<"OpImageDrefGather", 97>;
+def SPIRV_OC_OpImageRead                      : I32EnumAttrCase<"OpImageRead", 98>;
 def SPIRV_OC_OpImageWrite                     : I32EnumAttrCase<"OpImageWrite", 99>;
 def SPIRV_OC_OpImage                          : I32EnumAttrCase<"OpImage", 100>;
 def SPIRV_OC_OpImageQuerySize                 : I32EnumAttrCase<"OpImageQuerySize", 104>;
@@ -4545,7 +4578,8 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpCompositeInsert, SPIRV_OC_OpTranspose,
       SPIRV_OC_OpImageSampleImplicitLod, SPIRV_OC_OpImageSampleExplicitLod,
       SPIRV_OC_OpImageSampleProjDrefImplicitLod, SPIRV_OC_OpImageDrefGather,
-      SPIRV_OC_OpImageWrite, SPIRV_OC_OpImage, SPIRV_OC_OpImageQuerySize,
+      SPIRV_OC_OpImageRead, SPIRV_OC_OpImageWrite, SPIRV_OC_OpImage,
+      SPIRV_OC_OpImageQuerySize,
       SPIRV_OC_OpConvertFToU, SPIRV_OC_OpConvertFToS, SPIRV_OC_OpConvertSToF,
       SPIRV_OC_OpConvertUToF, SPIRV_OC_OpUConvert, SPIRV_OC_OpSConvert,
       SPIRV_OC_OpFConvert, SPIRV_OC_OpConvertPtrToU, SPIRV_OC_OpConvertUToPtr,
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCastOps.td b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCastOps.td
index b05ee0251df5..a5c8aa8fb450 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCastOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCastOps.td
@@ -86,7 +86,7 @@ def SPIRV_BitcastOp : SPIRV_Op<"Bitcast", [Pure]> {
 
 // -----
 
-def SPIRV_ConvertFToSOp : SPIRV_CastOp<"ConvertFToS", SPIRV_Integer, SPIRV_Float, []> {
+def SPIRV_ConvertFToSOp : SPIRV_CastOp<"ConvertFToS", SPIRV_Integer, SPIRV_AnyFloat, []> {
   let summary = [{
     Convert value numerically from floating point to signed integer, with
     round toward 0.0.
@@ -111,7 +111,7 @@ def SPIRV_ConvertFToSOp : SPIRV_CastOp<"ConvertFToS", SPIRV_Integer, SPIRV_Float
 
 // -----
 
-def SPIRV_ConvertFToUOp : SPIRV_CastOp<"ConvertFToU", SPIRV_Integer, SPIRV_Float, []> {
+def SPIRV_ConvertFToUOp : SPIRV_CastOp<"ConvertFToU", SPIRV_Integer, SPIRV_AnyFloat, []> {
   let summary = [{
     Convert value numerically from floating point to unsigned integer, with
     round toward 0.0.
@@ -138,7 +138,7 @@ def SPIRV_ConvertFToUOp : SPIRV_CastOp<"ConvertFToU", SPIRV_Integer, SPIRV_Float
 // -----
 
 def SPIRV_ConvertSToFOp : SPIRV_CastOp<"ConvertSToF",
-                                   SPIRV_Float,
+                                   SPIRV_AnyFloat,
                                    SPIRV_Integer,
                                    [SignedOp]> {
   let summary = [{
@@ -165,7 +165,7 @@ def SPIRV_ConvertSToFOp : SPIRV_CastOp<"ConvertSToF",
 // -----
 
 def SPIRV_ConvertUToFOp : SPIRV_CastOp<"ConvertUToF",
-                                   SPIRV_Float,
+                                   SPIRV_AnyFloat,
                                    SPIRV_Integer,
                                    [UnsignedOp]> {
   let summary = [{
@@ -192,8 +192,8 @@ def SPIRV_ConvertUToFOp : SPIRV_CastOp<"ConvertUToF",
 // -----
 
 def SPIRV_FConvertOp : SPIRV_CastOp<"FConvert",
-                                SPIRV_Float,
-                                SPIRV_Float,
+                                SPIRV_AnyFloat,
+                                SPIRV_AnyFloat,
                                 [UsableInSpecConstantOp]> {
   let summary = [{
     Convert value numerically from one floating-point width to another
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td
index 057dfac4d630..2ec61758ba8e 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLOps.td
@@ -767,6 +767,135 @@ def SPIRV_GLTanhOp : SPIRV_GLUnaryArithmeticOp<"Tanh", 21, SPIRV_Float16or32> {
 
 // -----
 
+def SPIRV_GLAsinhOp : SPIRV_GLUnaryArithmeticOp<"Asinh", 22, SPIRV_Float16or32> {
+  let summary = "Arc hyperbolic sine of operand in radians.";
+
+  let description = [{
+    Arc hyperbolic sine; result is the inverse of sinh.
+
+    The operand x must be a scalar or vector whose component type is 16-bit or
+    32-bit floating-point.
+
+    Result Type and the type of x must be the same type. Results are computed
+    per component.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.GL.Asinh %0 : f32
+    %3 = spirv.GL.Asinh %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_GLAcoshOp : SPIRV_GLUnaryArithmeticOp<"Acosh", 23, SPIRV_Float16or32> {
+  let summary = "Arc hyperbolic cosine of operand in radians.";
+
+  let description = [{
+    Arc hyperbolic cosine; result is the non-negative inverse of cosh. The resulting
+    value is NaN if x < 1.
+
+    The operand x must be a scalar or vector whose component type is 16-bit or
+    32-bit floating-point.
+
+    Result Type and the type of x must be the same type. Results are computed
+    per component.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.GL.Acosh %0 : f32
+    %3 = spirv.GL.Acosh %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_GLAtanhOp : SPIRV_GLUnaryArithmeticOp<"Atanh", 24, SPIRV_Float16or32> {
+  let summary = "Arc hyperbolic tangent of operand in radians.";
+
+  let description = [{
+    Arc hyperbolic tangent; result is the inverse of tanh. The resulting value
+    is NaN if abs x ≥ 1.
+
+    The operand x must be a scalar or vector whose component type is 16-bit or
+    32-bit floating-point.
+
+    Result Type and the type of x must be the same type. Results are computed
+    per component.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.GL.Atanh %0 : f32
+    %3 = spirv.GL.Atanh %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_GLExp2Op : SPIRV_GLUnaryArithmeticOp<"Exp2", 29, SPIRV_Float16or32> {
+  let summary = "Result is 2 raised to the x power";
+
+  let description = [{
+    Result is 2 raised to the x power; 2**x.
+
+    ```
+    exp2(Inf) = Inf.
+    exp2(-Inf) = +0.
+    ```
+
+    The operand x must be a scalar or vector whose component type is 16-bit or
+    32-bit floating-point.
+
+    Result Type and the type of x must be the same type. Results are computed
+    per component.
+
+    #### Example:
+
+    ```mlir
+    %2 = spirv.GL.Exp2 %0 : f32
+    %3 = spirv.GL.Exp2 %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPIRV_GLLog2Op : SPIRV_GLUnaryArithmeticOp<"Log2", 30, SPIRV_Float16or32> {
+  let summary = "Result is the base-2 logarithm of x";
+
+  let description = [{
+    Result is the base-2 logarithm of x, i.e., the value y which satisfies the
+    equation x = 2**y. The resulting value is NaN if x < 0. Moreover:
+
+    ```
+    log(Inf) = Inf
+    log(1.0) = +0
+    log(±0) = -Inf
+    ```
+    
+    The operand x must be a scalar or vector whose component type is 16-bit or
+    32-bit floating-point.
+
+    Result Type and the type of x must be the same type. Results are computed
+    per component.
+
+    #### Example:
+    
+    ```mlir
+    %2 = spirv.GL.Log2 %0 : f32
+    %3 = spirv.GL.Log2 %1 : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
 def SPIRV_GLFClampOp : SPIRV_GLTernaryArithmeticOp<"FClamp", 43, SPIRV_Float> {
   let summary = "Clamp x between min and max values.";
 
@@ -1145,6 +1274,41 @@ def SPIRV_GLReflectOp : SPIRV_GLBinaryArithmeticOp<"Reflect", 71, SPIRV_Float> {
 
 // ----
 
+def SPIRV_GLFindILsbOp : SPIRV_GLUnaryArithmeticOp<"FindILsb", 73, SPIRV_Integer> {
+  let summary = "Integer least-significant bit";
+
+  let description = [{
+    Results in the bit number of the least-significant 1-bit in the binary
+    representation of Value. If Value is 0, the result is -1.
+
+    Result Type and the type of Value must both be integer scalar or
+    integer vector types. Result Type and operand types must have the
+    same number of components with the same component width. Results are
+    computed per component.
+  }];
+}
+
+// ----
+
+def SPIRV_GLFindSMsbOp : SPIRV_GLUnaryArithmeticOp<"FindSMsb", 74, SPIRV_Int32> {
+  let summary = "Signed-integer most-significant bit, with Value interpreted as a signed integer";
+
+  let description = [{
+    For positive numbers, the result will be the bit number of the most significant
+    1-bit. For negative numbers, the result will be the bit number of the most
+    significant 0-bit. For a Value of 0 or -1, the result is -1.
+
+    Result Type and the type of Value must both be integer scalar or
+    integer vector types. Result Type and operand types must have the
+    same number of components with the same component width. Results are
+    computed per component.
+
+    This instruction is currently limited to 32-bit width components.
+  }];
+}
+
+// ----
+
 def SPIRV_GLFindUMsbOp : SPIRV_GLUnaryArithmeticOp<"FindUMsb", 75, SPIRV_Int32> {
   let summary = "Unsigned-integer most-significant bit";
 
@@ -1188,4 +1352,88 @@ def SPIRV_GLFractOp : SPIRV_GLUnaryArithmeticOp<"Fract", 10, SPIRV_Float> {
   }];
 }
 
+// -----
+
+def SPIRV_GLPackHalf2x16Op : SPIRV_GLOp<"PackHalf2x16", 58, [Pure]> {
+  let summary = "Pack two-component vector of 32-bit floats into a 32-bit integer";
+
+  let description = [{
+    Result is the unsigned integer obtained by converting the components of a
+    two-component floating-point vector to the 16-bit OpTypeFloat, and then packing
+    these two 16-bit integers into a 32-bit unsigned integer. The first vector
+    component specifies the 16 least-significant bits of the result; the second
+    component specifies the 16 most-significant bits.
+
+    The RelaxedPrecision Decoration only affects the conversion step of the instruction.
+
+    The v operand must be a vector of 2 components whose type is a 32-bit floating-point.
+
+    Result Type must be a 32-bit integer type.
+
+    #### Example:
+
+    ```mlir
+    %1 = spirv.GL.PackHalf2x16 %0 : vector<2xf32> -> i32
+    ```
+  }];
+
+  let arguments = (ins
+    VectorOfLengthAndType<[2], [SPIRV_Float32]>:$operand
+  );
+
+  let results = (outs
+    SPIRV_Int32:$result
+  );
+
+  let assemblyFormat = [{
+    attr-dict $operand `:` type($operand) `->` type($result)
+  }];
+
+  let hasVerifier = 0;
+}
+
+// -----
+
+def SPIRV_GLUnpackHalf2x16Op : SPIRV_GLOp<"UnpackHalf2x16", 62, [Pure]> {
+  let summary = "Unpack 32-bit integer into two-component vector of 32-bit floats";
+
+  let description = [{
+    Result is the two-component floating-point vector with components obtained by
+    unpacking a 32-bit unsigned integer into a pair of 16-bit values, interpreting
+    those values as 16-bit floating-point numbers according to the OpenGL
+    Specification, and converting them to 32-bit floating-point values. Subnormal
+    numbers are either preserved or flushed to zero, consistently within an
+    implementation.
+
+    The first component of the vector is obtained from the 16 least-significant bits
+    of v; the second component is obtained from the 16 most-significant bits of v.
+
+    The RelaxedPrecision Decoration only affects the conversion step of the instruction.
+
+    The v operand must be a scalar with 32-bit integer type.
+
+    Result Type must be a vector of 2 components whose type is 32-bit floating point.
+
+    #### Example:
+
+    ```mlir
+    %1 = spirv.GL.UnpackHalf2x16 %0 : i32 -> vector<2xf32>
+    ```
+  }];
+
+  let arguments = (ins
+    SPIRV_Int32:$operand
+  );
+
+  let results = (outs
+    VectorOfLengthAndType<[2], [SPIRV_Float32]>:$result
+  );
+
+  let assemblyFormat = [{
+    attr-dict $operand `:` type($operand) `->` type($result)
+  }];
+
+  let hasVerifier = 0;
+}
+
 #endif // MLIR_DIALECT_SPIRV_IR_GL_OPS
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td
index 9999e5cc07b8..7610966b84be 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVImageOps.td
@@ -186,6 +186,63 @@ def SPIRV_ImageQuerySizeOp : SPIRV_Op<"ImageQuerySize", [Pure]> {
 
 // -----
 
+def SPIRV_ImageReadOp : SPIRV_Op<"ImageRead",
+    [SPIRV_SampledOperandIs<"image", ["SamplerUnknown", "NoSampler"]>,
+     SPIRV_NoneOrElementMatchImage<"result", "image">]> {
+  let summary = "Read a texel from an image without a sampler.";
+
+  let description = [{
+    Result Type must be a scalar or vector of floating-point type or integer
+    type. It must be a scalar or vector with component type the same as Sampled
+    Type of the OpTypeImage (unless that Sampled Type is OpTypeVoid).
+
+    Image must be an object whose type is OpTypeImage with a Sampled operand of
+    0 or 2. If the Arrayed operand is 1, then additional capabilities may be
+    required; e.g., ImageCubeArray, or ImageMSArray.
+
+    Coordinate must be a scalar or vector of floating-point type or integer
+    type. It contains non-normalized texel coordinates (u[, v] ... [, array
+    layer]) as needed by the definition of Image. See the client API
+    specification for handling of coordinates outside the image.
+
+    If the Image Dim operand is SubpassData, Coordinate is relative to the
+    current fragment location. See the client API specification for more detail
+    on how these coordinates are applied.
+
+    If the Image Dim operand is not SubpassData, the Image Format must not be
+    Unknown, unless the StorageImageReadWithoutFormat Capability was declared.
+
+    Image Operands encodes what operands follow, as per Image Operands.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    %0 = spirv.ImageRead %1, %2 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, R32f>, vector<2xsi32> -> vector<4xf32>
+    ```
+  }];
+
+  let arguments = (ins
+    SPIRV_AnyImage:$image,
+    AnyTypeOf<[SPIRV_ScalarOrVectorOf<SPIRV_Float>, SPIRV_ScalarOrVectorOf<SPIRV_Integer>]>:$coordinate,
+    OptionalAttr<SPIRV_ImageOperandsAttr>:$image_operands,
+    Variadic<SPIRV_Type>:$operand_arguments
+  );
+
+  let results = (outs
+    AnyTypeOf<[SPIRV_ScalarOrVectorOf<SPIRV_Float>, SPIRV_ScalarOrVectorOf<SPIRV_Integer>]>:$result
+  );
+
+  let assemblyFormat = [{
+    $image `,` $coordinate custom<ImageOperands>($image_operands) ( `,` $operand_arguments^ )? attr-dict
+    `:` type($image) `,` type($coordinate) ( `,` type($operand_arguments)^ )?
+    `->` type($result)
+  }];
+}
+
+// -----
+
 def SPIRV_ImageWriteOp : SPIRV_Op<"ImageWrite",
     [SPIRV_SampledOperandIs<"image", ["SamplerUnknown", "NoSampler"]>,
      SPIRV_DimIsNot<"image", ["SubpassData"]>,
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
index 2e29e9afaabf..787535d0a6bd 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
@@ -394,7 +394,8 @@ hash_value(const StructType::MemberDecorationInfo &memberDecorationInfo);
 // SPIR-V KHR cooperative matrix type
 class CooperativeMatrixType
     : public Type::TypeBase<CooperativeMatrixType, CompositeType,
-                            detail::CooperativeMatrixTypeStorage> {
+                            detail::CooperativeMatrixTypeStorage,
+                            ShapedType::Trait> {
 public:
   using Base::Base;
 
@@ -418,6 +419,22 @@ class CooperativeMatrixType
                      std::optional<StorageClass> storage = std::nullopt);
   void getCapabilities(SPIRVType::CapabilityArrayRefVector &capabilities,
                        std::optional<StorageClass> storage = std::nullopt);
+
+  operator ShapedType() const { return llvm::cast<ShapedType>(*this); }
+
+  ArrayRef<int64_t> getShape() const;
+
+  bool hasRank() const { return true; }
+
+  CooperativeMatrixType cloneWith(std::optional<ArrayRef<int64_t>> shape,
+                                  Type elementType) const {
+    if (!shape)
+      return get(elementType, getRows(), getColumns(), getScope(), getUse());
+
+    assert(shape.value().size() == 2);
+    return get(elementType, shape.value()[0], shape.value()[1], getScope(),
+               getUse());
+  }
 };
 
 // SPIR-V matrix type
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h b/external/llvm-project/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
index eb550bb469b9..e8e1342ef36f 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
@@ -176,6 +176,10 @@ void populateFoldConstantExtractSlicePatterns(
           return false;
         });
 
+/// Patterns to fold extracts of a collapse_shaped tensor to an extract of the
+/// source tensor.
+void populateFoldCollapseExtractPatterns(RewritePatternSet &patterns);
+
 } // namespace tensor
 } // namespace mlir
 
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Tensor/Transforms/RuntimeOpVerification.h b/external/llvm-project/mlir/include/mlir/Dialect/Tensor/Transforms/RuntimeOpVerification.h
new file mode 100644
index 000000000000..812c4dd6d2cb
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Tensor/Transforms/RuntimeOpVerification.h
@@ -0,0 +1,21 @@
+//===- RuntimeOpVerification.h - Op Verification ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_TENSOR_RUNTIMEOPVERIFICATION_H
+#define MLIR_DIALECT_TENSOR_RUNTIMEOPVERIFICATION_H
+
+namespace mlir {
+class DialectRegistry;
+
+namespace tensor {
+void registerRuntimeVerifiableOpInterfaceExternalModels(
+    DialectRegistry &registry);
+} // namespace tensor
+} // namespace mlir
+
+#endif // MLIR_DIALECT_TENSOR_RUNTIMEOPVERIFICATION_H
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
index 78ed3a1fbcbd..59280893fd31 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
@@ -209,6 +209,16 @@ def Tosa_PadOpQuantInfoBuilder : OpBuilder<
                             input, paddings);
   }]>;
 
+// This builder is called on the TOSA variable operator with a variable type
+// and optional initial value. The builder will extract var_shape and element type
+// attributes from variable type.
+def Tosa_VariableOpBuilder : OpBuilder<
+  (ins "StringRef":$name, "Type":$variable_type, "Attribute":$initial_value),
+  [{
+    buildVariableOp($_builder, $_state, name, variable_type, initial_value);
+  }]>;
+
+
 // Wrapper over base I32EnumAttr to set common fields.
 class Tosa_I32Enum<string name, string description, list<I32EnumAttrCase> cases>
      : I32EnumAttr<name, description, cases> {
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
index 6fa4aedc1f0b..a15f073bc5fc 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
@@ -44,10 +44,14 @@ class PatternRewriter;
 
 namespace tosa {
 
-ParseResult parseTypeOrAttr(OpAsmParser &parser, TypeAttr &typeAttr,
-                            Attribute &attr);
-void printTypeOrAttr(OpAsmPrinter &p, Operation *op, TypeAttr type,
-                     Attribute attr);
+ParseResult parseVariableOpTypeOrInitialValue(OpAsmParser &parser,
+                                              DenseElementsAttr &varShapeAttr,
+                                              TypeAttr &typeAttr,
+                                              Attribute &initialValueAttr);
+void printVariableOpTypeOrInitialValue(OpAsmPrinter &p, Operation *op,
+                                       DenseElementsAttr varShapeAttr,
+                                       TypeAttr typeAttr,
+                                       Attribute initialValueAttr);
 
 #include "mlir/Dialect/Tosa/IR/TosaInterfaces.h.inc"
 
@@ -172,6 +176,9 @@ std::optional<Value> createZeroPointTensor(OpBuilder &builder, Location loc,
 Value createPadConstTensor(OpBuilder &builder, Location loc, Value src,
                            int32_t val = 0);
 
+// returns type of variable op
+RankedTensorType getVariableType(VariableOp variableOp);
+
 } // namespace tosa
 } // namespace mlir
 
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
index 5f9916290794..c8f2907f8dd1 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
@@ -92,6 +92,7 @@ def Tosa_VariableOp : Tosa_Op<"variable", []> {
 
   let arguments = (ins
     SymbolNameAttr:$name,
+    IndexElementsAttr:$var_shape,
     TypeAttr:$type,
     OptionalAttr<AnyAttr>:$initial_value
   );
@@ -101,12 +102,16 @@ def Tosa_VariableOp : Tosa_Op<"variable", []> {
     Extension<[Tosa_EXT_VARIABLE]>,
   ];
 
+  let hasCustomAssemblyFormat = 1;
+
   let assemblyFormat = [{
     $name
     attr-dict
-    custom<TypeOrAttr>($type, $initial_value)
+    custom<VariableOpTypeOrInitialValue>($var_shape, $type, $initial_value)
   }];
 
+  let builders = [Tosa_VariableOpBuilder];
+
   let hasVerifier = 1;
 }
 
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h
index 096510a09e32..6f3b0916a7a6 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Tosa/Utils/ConversionUtils.h
@@ -243,6 +243,11 @@ bool getConstShapeValues(Operation *op,
 // returns a small vector of int64_t values that attr contains
 SmallVector<int64_t> convertFromIntAttr(const DenseElementsAttr &attr,
                                         const int rank);
+
+// returns true iff constant indices for scatter op contains unique indices
+// per batch
+bool hasUniqueConstantScatterIndices(ShapedType indicesType,
+                                     DenseIntElementsAttr indicesAttr);
 } // namespace tosa
 } // namespace mlir
 
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt
index df5af7ae710d..9acab9228f10 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt
@@ -20,6 +20,10 @@ mlir_tablegen(TransformDialectEnums.h.inc -gen-enum-decls)
 mlir_tablegen(TransformDialectEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRTransformDialectEnumIncGen)
 add_dependencies(mlir-headers MLIRTransformDialectEnumIncGen)
+mlir_tablegen(TransformAttrs.h.inc -gen-attrdef-decls)
+mlir_tablegen(TransformAttrs.cpp.inc -gen-attrdef-defs)
+add_public_tablegen_target(MLIRTransformDialectAttributesIncGen)
+add_dependencies(mlir-headers MLIRTransformDialectAttributesIncGen)
 
 add_mlir_dialect(TransformOps transform)
 add_mlir_doc(TransformOps TransformOps Dialects/ -gen-op-doc -dialect=transform)
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.h b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.h
index 3cb935003b4c..379af932ca48 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.h
@@ -17,4 +17,7 @@
 
 #include "mlir/Dialect/Transform/IR/TransformDialectEnums.h.inc"
 
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/Transform/IR/TransformAttrs.h.inc"
+
 #endif // MLIR_DIALECT_TRANSFORM_IR_TRANSFORMATTRS_H
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.td b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.td
index ebad2994880e..e67a9444c24a 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformAttrs.td
@@ -10,6 +10,14 @@
 #define MLIR_DIALECT_TRANSFORM_IR_TRANSFORMATTRS
 
 include "mlir/IR/EnumAttr.td"
+include "mlir/Dialect/Transform/IR/TransformDialect.td"
+
+class Transform_Attr<string name, string attrMnemonic,
+                     list<Trait> traits = [],
+                     string baseCppClass = "::mlir::Attribute">
+    : AttrDef<Transform_Dialect, name, traits, baseCppClass> {
+  let mnemonic = attrMnemonic;
+}
 
 def PropagateFailuresCase : I32EnumAttrCase<"Propagate", 1, "propagate">;
 def SuppressFailuresCase : I32EnumAttrCase<"Suppress", 2, "suppress">;
@@ -33,4 +41,15 @@ def MatchCmpIPredicateAttr : I32EnumAttr<
   let cppNamespace = "::mlir::transform";
 }
 
+def ParamOperandAttr : Transform_Attr<"ParamOperand", "param_operand"> {
+  let description = [{
+    Used to refer to a specific param-operand (via its index) from within an
+    attribute on a transform operation.
+  }];
+  let parameters = (ins
+    "IntegerAttr":$index
+  );
+  let assemblyFormat = "`<` `index` `=` $index `>`";
+}
+
 #endif  // MLIR_DIALECT_TRANSFORM_IR_TRANSFORMATTRS
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td
index d03049e186f9..c7ea5ade72ac 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td
@@ -19,6 +19,7 @@ def Transform_Dialect : Dialect {
   let cppNamespace = "::mlir::transform";
 
   let hasOperationAttrVerify = 1;
+  let useDefaultAttributePrinterParser = 1;
   let extraClassDeclaration = [{
     /// Symbol name for the default entry point "named sequence".
     constexpr const static ::llvm::StringLiteral
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
index e4eb67c8e14c..62e66b3dabee 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
@@ -399,15 +399,32 @@ def ApplyLoopInvariantCodeMotionOp : TransformDialectOp<"apply_licm",
 }
 
 def ApplyRegisteredPassOp : TransformDialectOp<"apply_registered_pass",
-    [TransformOpInterface, TransformEachOpTrait,
-     FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface]> {
+    [DeclareOpInterfaceMethods<TransformOpInterface>,
+     DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
   let summary = "Applies the specified registered pass or pass pipeline";
   let description = [{
     This transform applies the specified pass or pass pipeline to the targeted
     ops. The name of the pass/pipeline is specified as a string attribute, as
-    set during pass/pipeline registration. Optionally, pass options may be
-    specified as a string attribute. The pass options syntax is identical to the
-    one used with "mlir-opt".
+    set during pass/pipeline registration.
+
+    Optionally, pass options may be specified via a DictionaryAttr. This
+    dictionary is converted to a string -- formatted `key=value ...` -- which
+    is expected to be in the exact format used by the pass on the commandline.
+    Values are either attributes or (SSA-values of) Transform Dialect params.
+    For example:
+
+    ```mlir
+    transform.apply_registered_pass "canonicalize"
+        with options = { "top-down" = false,
+                         "max-iterations" = %max_iter,
+                         "test-convergence" = true,
+                         "max-num-rewrites" = %max_rewrites }
+        to %module
+    : (!transform.any_param, !transform.any_param, !transform.any_op) -> !transform.any_op
+    ```
+
+    Options' values which are `ArrayAttr`s are converted to comma-separated
+    lists of options. Likewise for params which associate multiple values.
 
     This op first looks for a pass pipeline with the specified name. If no such
     pipeline exists, it looks for a pass with the specified name. If no such
@@ -422,19 +439,15 @@ def ApplyRegisteredPassOp : TransformDialectOp<"apply_registered_pass",
 
   let arguments = (ins TransformHandleTypeInterface:$target,
                        StrAttr:$pass_name,
-                       DefaultValuedAttr<StrAttr, "\"\"">:$options);
+                       DefaultValuedAttr<DictionaryAttr, "{}">:$options,
+                       Variadic<TransformParamTypeInterface>:$dynamic_options);
   let results = (outs TransformHandleTypeInterface:$result);
   let assemblyFormat = [{
-    $pass_name `to` $target attr-dict `:` functional-type(operands, results)
-  }];
-
-  let extraClassDeclaration = [{
-    ::mlir::DiagnosedSilenceableFailure applyToOne(
-      ::mlir::transform::TransformRewriter &rewriter,
-      ::mlir::Operation *target,
-      ::mlir::transform::ApplyToEachResultList &results,
-      ::mlir::transform::TransformState &state);
+    $pass_name (`with` `options` `=`
+      custom<ApplyRegisteredPassOptions>($options, $dynamic_options)^)?
+      `to` $target attr-dict `:` functional-type(operands, results)
   }];
+  let hasVerifier = 1;
 }
 
 def CastOp : TransformDialectOp<"cast",
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h b/external/llvm-project/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
index af575e10acc8..61c2a50e514c 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
@@ -387,11 +387,14 @@ struct ComposeExpandOfCollapseOp : public OpRewritePattern<ExpandOpTy> {
       auto resultSubShape =
           resultShape.slice(resultIndices.front(), resultIndices.size());
 
+      if (llvm::count_if(srcSubShape, ShapedType::isDynamic) >= 2 &&
+          llvm::count_if(resultSubShape, ShapedType::isDynamic) >= 2)
+        return std::nullopt;
+
       if (srcSubShape.size() == resultSubShape.size()) {
-        if (srcSubShape != resultSubShape ||
-            llvm::count_if(srcSubShape, ShapedType::isDynamic) >= 2) {
+        if (srcSubShape != resultSubShape)
           return std::nullopt;
-        }
+
         for (auto index : llvm::seq<int64_t>(0, srcSubShape.size())) {
           composedReassociation.emplace_back(1, srcIndices.front() + index);
         }
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/external/llvm-project/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 5e8421ed67d6..8353314ed958 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -347,6 +347,7 @@ def Vector_MultiDimReductionOp :
 
 def Vector_BroadcastOp :
   Vector_Op<"broadcast", [Pure,
+     DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>,
      DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
      PredOpTrait<"source operand and result have same element type",
                  TCresVTEtIsSameAsOpBase<0, 0>>]>,
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
index 032ce5bc1833..84c1dc1373ee 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
@@ -295,11 +295,17 @@ def XeGPU_LayoutAttr : XeGPUAttr<"Layout", "layout"> {
     }
 
     LayoutAttr dropSgLayoutAndData() {
+      // avoid every field of the attribute is nullptr, which may lead to segment fault
+      if (!getInstData() && !getLaneLayout())
+        return nullptr;
       return LayoutAttr::get(getContext(), nullptr, nullptr, getInstData(),
                              getLaneLayout(), getLaneData(), getOrder());
     }
 
     LayoutAttr dropInstData() {
+      // avoid every field of the attribute is nullptr, which may lead to segment fault
+      if (!getSgLayout() && !getLaneLayout())
+        return nullptr;
       return LayoutAttr::get(getContext(), getSgLayout(), getSgData(), nullptr,
                              getLaneLayout(), getLaneData(), getOrder());
     }
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index 238bb1567d30..e6c7efc47593 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -205,7 +205,7 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
 
       auto memrefType = llvm::dyn_cast<MemRefType>(getSourceType());
       assert(memrefType && "Incorrect use of getStaticStrides");
-      auto [strides, offset] = memrefType.getStridesAndOffset();
+      auto [strides, _] = memrefType.getStridesAndOffset();
       // reuse the storage of ConstStridesAttr since strides from
       // memref is not persistant
       setConstStrides(strides);
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td
index 6f585f9ceb29..8bdf19ac0e47 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td
+++ b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td
@@ -45,4 +45,17 @@ def XeGPUWgToSgDistribute : Pass<"xegpu-wg-to-sg-distribute"> {
                            "gpu::GPUDialect", "index::IndexDialect"];
 }
 
+def XeGPUBlocking: Pass<"xegpu-blocking"> {
+  let summary = "Block XeGPU ops into smaller size.";
+  let description = [{
+    This pass partitions operations that process large shapes into multiple
+    operations on smaller shapes, as specified by the inst_data in the layout
+    attribute. This enables each resulting operation to be efficiently mapped
+    to a hardware instruction.
+  }];
+  let dependentDialects = [
+      "memref::MemRefDialect", "xegpu::XeGPUDialect", "vector::VectorDialect"
+  ];
+}
+
 #endif // MLIR_DIALECT_XEGPU_TRANSFORMS_PASSES_TD
diff --git a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
index 3616fa614e7f..6fea10185402 100644
--- a/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
+++ b/external/llvm-project/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
@@ -13,6 +13,12 @@
 namespace mlir {
 
 class VectorType;
+class OpOperand;
+class OpResult;
+class OpBuilder;
+class ValueRange;
+class TypeConverter;
+
 namespace xegpu {
 class LayoutAttr;
 class TensorDescType;
@@ -20,6 +26,9 @@ class TensorDescType;
 
 namespace xegpu {
 
+/// Flatten a set of ValueRange into a single SmallVector<Value>
+SmallVector<Value> flattenValues(ArrayRef<ValueRange> values);
+
 /// If tensor descriptor has a layout attribute it is used in SIMT mode.
 /// In this mode, the distributed vector shape is determined as follows:
 /// Definitions:
@@ -50,6 +59,59 @@ FailureOr<VectorType> getDistributedVectorType(xegpu::TensorDescType tdescTy);
 FailureOr<VectorType> getDistributedVectorType(VectorType originalType,
                                                LayoutAttr layout);
 
+/// Return the attribute name for the OpOperand to attach LayoutAttr
+std::string getLayoutName(const OpOperand &operand);
+
+/// Return the attribute name for the OpResult to attach LayoutAttr
+std::string getLayoutName(const OpResult result);
+
+/// Retrieves the LayoutAttr associated with a given Value. For TensorDescType
+/// values, the LayoutAttr is extracted from the TensorDescType itself. For
+/// other values, it is obtained from the attributes of the defining operation.
+/// Returns nullptr if no LayoutAttr is found.
+LayoutAttr getLayoutAttr(const Value value);
+
+/// Retrieves the LayoutAttr associated with a given OpOperand. It will
+/// first check the operand_layout_{id} of the owner operation. If not found,
+/// it will check the operand itself and its defining op.
+LayoutAttr getLayoutAttr(const OpOperand &opr);
+
+/// Sets the LayoutAttr for a given OpOperand or OpResult by attaching
+/// it to the owner's dictionary attributes
+template <typename T,
+          typename = std::enable_if_t<std::is_same_v<T, OpOperand> ||
+                                      std::is_same_v<T, OpResult>>>
+void setLayoutAttr(const T &operandOrResult, const LayoutAttr layout);
+
+/// Set the LayoutAttr for each OpOperand and OpResult of the given operation.
+/// If the operation contains regions, it is also applied recursively to the
+/// contained operations
+void setLayoutAttrs(Operation *op,
+                    function_ref<LayoutAttr(Value)> getLayoutImpl);
+
+/// Extract a set of small vectors from a value with a given shape using
+/// vector.extract_stride_slice
+SmallVector<Value> extractVectorsWithShapeFromValue(OpBuilder &builder,
+                                                    Location loc, Value value,
+                                                    ArrayRef<int64_t> shape);
+
+/// Create a vector of shape from a set of values using
+/// vector.insert_stride_slice.
+Value createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
+                                      ValueRange values,
+                                      ArrayRef<int64_t> shape);
+
+/// Do type conversion for SCF structural ops, e.g., scf.for using SCF structure
+/// type convertion patterns. Since VectorType cannot carry the layout
+/// attribute, which is needed to guide the type conversion for XeGPU, they are
+/// first converted into RankedTensorType, where the layout attribute can be
+/// attached. And then upstream SCF structural type conversion patterns are
+/// applied with the provided converter.
+/// TODO: This is a temporary solution. We should refactor it when context-aware
+/// type conversion is available.
+void doSCFStructuralTypeConversionWithTensorType(Operation *op,
+                                                 TypeConverter converter);
+
 } // namespace xegpu
 
 } // namespace mlir
diff --git a/external/llvm-project/mlir/include/mlir/ExecutionEngine/MemRefUtils.h b/external/llvm-project/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
index 918647d9feac..f355dfb8648e 100644
--- a/external/llvm-project/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
+++ b/external/llvm-project/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
@@ -48,7 +48,8 @@ inline std::array<int64_t, N> makeStrides(ArrayRef<int64_t> shape) {
   std::array<int64_t, N> res;
   int64_t running = 1;
   for (int64_t idx = N - 1; idx >= 0; --idx) {
-    assert(shape[idx] && "size must be non-negative for all shape dimensions");
+    assert(shape[idx] >= 0 &&
+           "size must be non-negative for all shape dimensions");
     res[idx] = running;
     running *= shape[idx];
   }
diff --git a/external/llvm-project/mlir/include/mlir/IR/Builders.h b/external/llvm-project/mlir/include/mlir/IR/Builders.h
index 3f7b3268dd08..d68dbdb1efee 100644
--- a/external/llvm-project/mlir/include/mlir/IR/Builders.h
+++ b/external/llvm-project/mlir/include/mlir/IR/Builders.h
@@ -60,6 +60,7 @@ class Builder {
                        Attribute metadata = Attribute());
 
   // Types.
+  FloatType getF8E8M0Type();
   FloatType getBF16Type();
   FloatType getF16Type();
   FloatType getTF32Type();
diff --git a/external/llvm-project/mlir/include/mlir/IR/Constraints.td b/external/llvm-project/mlir/include/mlir/IR/Constraints.td
index 33e8581ecd35..0d59fffce9df 100644
--- a/external/llvm-project/mlir/include/mlir/IR/Constraints.td
+++ b/external/llvm-project/mlir/include/mlir/IR/Constraints.td
@@ -148,6 +148,15 @@ class Constraint<Pred pred, string desc = ""> {
   string summary = desc;
 }
 
+// Base class for constraints on types and attributes.
+class AttrTypeConstraint<Pred pred, string summary = "",
+                         string cppFunctionNameParam = ""> :
+    Constraint<pred, summary> {
+  // The name of the C++ function that is generated for this constraint.
+  // If empty, no C++ function is generated.
+  string cppFunctionName = cppFunctionNameParam;
+}
+
 // Subclasses used to differentiate different constraint kinds. These are used
 // as markers for the TableGen backend to handle different constraint kinds
 // differently if needed. Constraints not deriving from the following subclasses
@@ -157,17 +166,15 @@ class Constraint<Pred pred, string desc = ""> {
 class TypeConstraint<Pred predicate, string summary = "",
                      string cppTypeParam = "::mlir::Type",
                      string cppFunctionNameParam = ""> :
-    Constraint<predicate, summary> {
+    AttrTypeConstraint<predicate, summary, cppFunctionNameParam> {
   // The name of the C++ Type class if known, or Type if not.
   string cppType = cppTypeParam;
-  // The name of the C++ function that is generated for this type constraint.
-  // If empty, no C++ function is generated.
-  string cppFunctionName = cppFunctionNameParam;
 }
 
 // Subclass for constraints on an attribute.
-class AttrConstraint<Pred predicate, string summary = ""> :
-    Constraint<predicate, summary>;
+class AttrConstraint<Pred predicate, string summary = "",
+                     string cppFunctionNameParam = ""> :
+    AttrTypeConstraint<predicate, summary, cppFunctionNameParam>;
 
 // Subclass for constraints on a property.
 class PropConstraint<Pred predicate, string summary = "", string interfaceTypeParam = ""> :
diff --git a/external/llvm-project/mlir/include/mlir/IR/Location.h b/external/llvm-project/mlir/include/mlir/IR/Location.h
index 5b1cf300295e..d286befe1920 100644
--- a/external/llvm-project/mlir/include/mlir/IR/Location.h
+++ b/external/llvm-project/mlir/include/mlir/IR/Location.h
@@ -21,6 +21,7 @@ namespace mlir {
 
 class Location;
 class WalkResult;
+class UnknownLoc;
 
 //===----------------------------------------------------------------------===//
 // LocationAttr
@@ -53,6 +54,15 @@ class LocationAttr : public Attribute {
     return result;
   }
 
+  /// Return an instance of the given location type if one is nested under the
+  /// current location else return unknown location.
+  template <typename T, typename UnknownT = UnknownLoc>
+  LocationAttr findInstanceOfOrUnknown() {
+    if (T result = findInstanceOf<T>())
+      return result;
+    return UnknownT::get(getContext());
+  }
+
   /// Methods for support type inquiry through isa, cast, and dyn_cast.
   static bool classof(Attribute attr);
 };
diff --git a/external/llvm-project/mlir/include/mlir/IR/OperationSupport.h b/external/llvm-project/mlir/include/mlir/IR/OperationSupport.h
index 0046d977c68f..65e6d4f64e36 100644
--- a/external/llvm-project/mlir/include/mlir/IR/OperationSupport.h
+++ b/external/llvm-project/mlir/include/mlir/IR/OperationSupport.h
@@ -1322,7 +1322,14 @@ struct OperationEquivalence {
     // When provided, the location attached to the operation are ignored.
     IgnoreLocations = 1,
 
-    LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ IgnoreLocations)
+    // When provided, the discardable attributes attached to the operation are
+    // ignored.
+    IgnoreDiscardableAttrs = 2,
+
+    // When provided, the properties attached to the operation are ignored.
+    IgnoreProperties = 4,
+
+    LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ IgnoreProperties)
   };
 
   /// Compute a hash for the given operation.
diff --git a/external/llvm-project/mlir/include/mlir/IR/SymbolTable.h b/external/llvm-project/mlir/include/mlir/IR/SymbolTable.h
index a22d6f3a0426..557a2ba85cd6 100644
--- a/external/llvm-project/mlir/include/mlir/IR/SymbolTable.h
+++ b/external/llvm-project/mlir/include/mlir/IR/SymbolTable.h
@@ -282,10 +282,14 @@ raw_ostream &operator<<(raw_ostream &os, SymbolTable::Visibility visibility);
 /// unnecessary tables.
 class SymbolTableCollection {
 public:
+  virtual ~SymbolTableCollection() = default;
+
   /// Look up a symbol with the specified name within the specified symbol table
   /// operation, returning null if no such name exists.
-  Operation *lookupSymbolIn(Operation *symbolTableOp, StringAttr symbol);
-  Operation *lookupSymbolIn(Operation *symbolTableOp, SymbolRefAttr name);
+  virtual Operation *lookupSymbolIn(Operation *symbolTableOp,
+                                    StringAttr symbol);
+  virtual Operation *lookupSymbolIn(Operation *symbolTableOp,
+                                    SymbolRefAttr name);
   template <typename T, typename NameT>
   T lookupSymbolIn(Operation *symbolTableOp, NameT &&name) {
     return dyn_cast_or_null<T>(
@@ -295,15 +299,18 @@ class SymbolTableCollection {
   /// by a given SymbolRefAttr when resolved within the provided symbol table
   /// operation. Returns failure if any of the nested references could not be
   /// resolved.
-  LogicalResult lookupSymbolIn(Operation *symbolTableOp, SymbolRefAttr name,
-                               SmallVectorImpl<Operation *> &symbols);
+  virtual LogicalResult lookupSymbolIn(Operation *symbolTableOp,
+                                       SymbolRefAttr name,
+                                       SmallVectorImpl<Operation *> &symbols);
 
   /// Returns the operation registered with the given symbol name within the
   /// closest parent operation of, or including, 'from' with the
   /// 'OpTrait::SymbolTable' trait. Returns nullptr if no valid symbol was
   /// found.
-  Operation *lookupNearestSymbolFrom(Operation *from, StringAttr symbol);
-  Operation *lookupNearestSymbolFrom(Operation *from, SymbolRefAttr symbol);
+  virtual Operation *lookupNearestSymbolFrom(Operation *from,
+                                             StringAttr symbol);
+  virtual Operation *lookupNearestSymbolFrom(Operation *from,
+                                             SymbolRefAttr symbol);
   template <typename T>
   T lookupNearestSymbolFrom(Operation *from, StringAttr symbol) {
     return dyn_cast_or_null<T>(lookupNearestSymbolFrom(from, symbol));
@@ -314,14 +321,14 @@ class SymbolTableCollection {
   }
 
   /// Lookup, or create, a symbol table for an operation.
-  SymbolTable &getSymbolTable(Operation *op);
+  virtual SymbolTable &getSymbolTable(Operation *op);
 
   /// Invalidate the cached symbol table for an operation.
   /// This is important when doing IR modifications that erase and also create
   /// operations having the 'OpTrait::SymbolTable' trait. If a symbol table of
   /// an erased operation is not invalidated, a new operation sharing the same
   /// address would be associated with outdated, and wrong, information.
-  void invalidateSymbolTable(Operation *op);
+  virtual void invalidateSymbolTable(Operation *op);
 
 private:
   friend class LockedSymbolTableCollection;
@@ -348,13 +355,15 @@ class LockedSymbolTableCollection : public SymbolTableCollection {
 
   /// Look up a symbol with the specified name within the specified symbol table
   /// operation, returning null if no such name exists.
-  Operation *lookupSymbolIn(Operation *symbolTableOp, StringAttr symbol);
+  Operation *lookupSymbolIn(Operation *symbolTableOp,
+                            StringAttr symbol) override;
   /// Look up a symbol with the specified name within the specified symbol table
   /// operation, returning null if no such name exists.
   Operation *lookupSymbolIn(Operation *symbolTableOp, FlatSymbolRefAttr symbol);
   /// Look up a potentially nested symbol within the specified symbol table
   /// operation, returning null if no such symbol exists.
-  Operation *lookupSymbolIn(Operation *symbolTableOp, SymbolRefAttr name);
+  Operation *lookupSymbolIn(Operation *symbolTableOp,
+                            SymbolRefAttr name) override;
 
   /// Lookup a symbol of a particular kind within the specified symbol table,
   /// returning null if the symbol was not found.
@@ -369,14 +378,14 @@ class LockedSymbolTableCollection : public SymbolTableCollection {
   /// operation. Returns failure if any of the nested references could not be
   /// resolved.
   LogicalResult lookupSymbolIn(Operation *symbolTableOp, SymbolRefAttr name,
-                               SmallVectorImpl<Operation *> &symbols);
+                               SmallVectorImpl<Operation *> &symbols) override;
 
 private:
   /// Get the symbol table for the symbol table operation, constructing if it
   /// does not exist. This function provides thread safety over `collection`
   /// by locking when performing the lookup and when inserting
   /// lazily-constructed symbol tables.
-  SymbolTable &getSymbolTable(Operation *symbolTableOp);
+  SymbolTable &getSymbolTable(Operation *symbolTableOp) override;
 
   /// The symbol tables to manage.
   SymbolTableCollection &collection;
diff --git a/external/llvm-project/mlir/include/mlir/InitAllDialects.h b/external/llvm-project/mlir/include/mlir/InitAllDialects.h
index ea285ac7f16e..261b0e00bdf8 100644
--- a/external/llvm-project/mlir/include/mlir/InitAllDialects.h
+++ b/external/llvm-project/mlir/include/mlir/InitAllDialects.h
@@ -84,6 +84,7 @@
 #include "mlir/Dialect/Tensor/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h"
 #include "mlir/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.h"
+#include "mlir/Dialect/Tensor/Transforms/RuntimeOpVerification.h"
 #include "mlir/Dialect/Tensor/Transforms/SubsetInsertionOpInterfaceImpl.h"
 #include "mlir/Dialect/Tosa/IR/ShardingInterfaceImpl.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
@@ -186,6 +187,7 @@ inline void registerAllDialects(DialectRegistry &registry) {
   tensor::registerBufferizableOpInterfaceExternalModels(registry);
   tensor::registerFindPayloadReplacementOpInterfaceExternalModels(registry);
   tensor::registerInferTypeOpInterfaceExternalModels(registry);
+  tensor::registerRuntimeVerifiableOpInterfaceExternalModels(registry);
   tensor::registerSubsetOpInterfaceExternalModels(registry);
   tensor::registerTilingInterfaceExternalModels(registry);
   tensor::registerValueBoundsOpInterfaceExternalModels(registry);
diff --git a/external/llvm-project/mlir/include/mlir/InitAllExtensions.h b/external/llvm-project/mlir/include/mlir/InitAllExtensions.h
index 1e3f7c649a8b..f356b91b1b6c 100644
--- a/external/llvm-project/mlir/include/mlir/InitAllExtensions.h
+++ b/external/llvm-project/mlir/include/mlir/InitAllExtensions.h
@@ -32,7 +32,10 @@
 #include "mlir/Conversion/SCFToEmitC/SCFToEmitC.h"
 #include "mlir/Conversion/UBToLLVM/UBToLLVM.h"
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
+#include "mlir/Dialect/AMX/Transforms.h"
 #include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.h"
+#include "mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.h"
+#include "mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.h"
 #include "mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.h"
 #include "mlir/Dialect/DLTI/TransformOps/DLTITransformOps.h"
 #include "mlir/Dialect/Func/Extensions/AllExtensions.h"
@@ -83,6 +86,7 @@ inline void registerAllExtensions(DialectRegistry &registry) {
   registerConvertOpenMPToLLVMInterface(registry);
   registerConvertSCFToEmitCInterface(registry);
   ub::registerConvertUBToLLVMInterface(registry);
+  registerConvertAMXToLLVMInterface(registry);
   gpu::registerConvertGpuToLLVMInterface(registry);
   NVVM::registerConvertGpuToNVVMInterface(registry);
   vector::registerConvertVectorToLLVMInterface(registry);
@@ -104,6 +108,8 @@ inline void registerAllExtensions(DialectRegistry &registry) {
   transform::registerLoopExtension(registry);
   transform::registerPDLExtension(registry);
   vector::registerTransformDialectExtension(registry);
+  arm_neon::registerTransformDialectExtension(registry);
+  arm_sve::registerTransformDialectExtension(registry);
 
   // Translation extensions need to be registered by calling
   // `registerAllToLLVMIRTranslations` (see All.h).
diff --git a/external/llvm-project/mlir/include/mlir/Interfaces/TilingInterface.td b/external/llvm-project/mlir/include/mlir/Interfaces/TilingInterface.td
index 50b69b8f8d83..cdf3d01ce8a8 100644
--- a/external/llvm-project/mlir/include/mlir/Interfaces/TilingInterface.td
+++ b/external/llvm-project/mlir/include/mlir/Interfaces/TilingInterface.td
@@ -363,7 +363,8 @@ def TilingInterface : OpInterface<"TilingInterface"> {
   ];
 }
 
-def PartialReductionOpInterface : OpInterface<"PartialReductionOpInterface"> {
+def PartialReductionOpInterface :
+    OpInterface<"PartialReductionOpInterface", [TilingInterface]> {
   let description = [{
     Interface for allowing operations to expose information needed to
     tile reductions using partial reduction followed by merge. This is
diff --git a/external/llvm-project/mlir/include/mlir/Pass/Pass.h b/external/llvm-project/mlir/include/mlir/Pass/Pass.h
index 9e8ba7158f1b..16893c6db87b 100644
--- a/external/llvm-project/mlir/include/mlir/Pass/Pass.h
+++ b/external/llvm-project/mlir/include/mlir/Pass/Pass.h
@@ -118,8 +118,9 @@ class Pass {
                     function_ref<LogicalResult(const Twine &)> errorHandler);
 
   /// Prints out the pass in the textual representation of pipelines. If this is
-  /// an adaptor pass, print its pass managers.
-  void printAsTextualPipeline(raw_ostream &os);
+  /// an adaptor pass, print its pass managers. When `pretty` is true, the
+  /// printed pipeline is formatted for readability.
+  void printAsTextualPipeline(raw_ostream &os, bool pretty = false);
 
   //===--------------------------------------------------------------------===//
   // Statistics
diff --git a/external/llvm-project/mlir/include/mlir/Pass/PassManager.h b/external/llvm-project/mlir/include/mlir/Pass/PassManager.h
index 2287b0a33f47..6e59b0f32ac6 100644
--- a/external/llvm-project/mlir/include/mlir/Pass/PassManager.h
+++ b/external/llvm-project/mlir/include/mlir/Pass/PassManager.h
@@ -140,10 +140,12 @@ class OpPassManager {
   detail::OpPassManagerImpl &getImpl();
 
   /// Prints out the passes of the pass manager as the textual representation
-  /// of pipelines.
+  /// of pipelines. When `pretty` is true, the printed pipeline is formatted
+  /// for readability.
+  ///
   /// Note: The quality of the string representation depends entirely on the
   /// the correctness of per-pass overrides of Pass::printAsTextualPipeline.
-  void printAsTextualPipeline(raw_ostream &os) const;
+  void printAsTextualPipeline(raw_ostream &os, bool pretty = false) const;
 
   /// Raw dump of the pass manager to llvm::errs().
   void dump();
diff --git a/external/llvm-project/mlir/include/mlir/Target/SPIRV/Deserialization.h b/external/llvm-project/mlir/include/mlir/Target/SPIRV/Deserialization.h
index a346a7fd1e5f..68eb863b4a6f 100644
--- a/external/llvm-project/mlir/include/mlir/Target/SPIRV/Deserialization.h
+++ b/external/llvm-project/mlir/include/mlir/Target/SPIRV/Deserialization.h
@@ -23,12 +23,19 @@ class MLIRContext;
 namespace spirv {
 class ModuleOp;
 
+struct DeserializationOptions {
+  // Whether to structurize control flow into `spirv.mlir.selection` and
+  // `spirv.mlir.loop`.
+  bool enableControlFlowStructurization = true;
+};
+
 /// Deserializes the given SPIR-V `binary` module and creates a MLIR ModuleOp
 /// in the given `context`. Returns the ModuleOp on success; otherwise, reports
 /// errors to the error handler registered with `context` and returns a null
 /// module.
-OwningOpRef<spirv::ModuleOp> deserialize(ArrayRef<uint32_t> binary,
-                                         MLIRContext *context);
+OwningOpRef<spirv::ModuleOp>
+deserialize(ArrayRef<uint32_t> binary, MLIRContext *context,
+            const DeserializationOptions &options = {});
 
 } // namespace spirv
 } // namespace mlir
diff --git a/external/llvm-project/mlir/include/mlir/Target/SPIRV/SPIRVBinaryUtils.h b/external/llvm-project/mlir/include/mlir/Target/SPIRV/SPIRVBinaryUtils.h
index e46a576f1d48..4a4116312981 100644
--- a/external/llvm-project/mlir/include/mlir/Target/SPIRV/SPIRVBinaryUtils.h
+++ b/external/llvm-project/mlir/include/mlir/Target/SPIRV/SPIRVBinaryUtils.h
@@ -30,6 +30,13 @@ constexpr uint32_t kMagicNumber = 0x07230203;
 /// The serializer tool ID registered to the Khronos Group
 constexpr uint32_t kGeneratorNumber = 22;
 
+/// Max number of words
+/// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_universal_limits
+constexpr uint32_t kMaxWordCount = 65535;
+
+/// Max number of words for literal
+constexpr uint32_t kMaxLiteralWordCount = kMaxWordCount - 3;
+
 /// Appends a SPRI-V module header to `header` with the given `version` and
 /// `idBound`.
 void appendModuleHeader(SmallVectorImpl<uint32_t> &header,
diff --git a/external/llvm-project/mlir/include/mlir/Tools/lsp-server-support/Protocol.h b/external/llvm-project/mlir/include/mlir/Tools/lsp-server-support/Protocol.h
index 5d2eb01a523a..cc06dbfedb42 100644
--- a/external/llvm-project/mlir/include/mlir/Tools/lsp-server-support/Protocol.h
+++ b/external/llvm-project/mlir/include/mlir/Tools/lsp-server-support/Protocol.h
@@ -158,6 +158,12 @@ struct ClientCapabilities {
   /// Client supports CodeAction return value for textDocument/codeAction.
   /// textDocument.codeAction.codeActionLiteralSupport.
   bool codeActionStructure = false;
+
+  /// Client supports server-initiated progress via the
+  /// window/workDoneProgress/create method.
+  ///
+  /// window.workDoneProgress
+  bool workDoneProgress = false;
 };
 
 /// Add support for JSON serialization.
diff --git a/external/llvm-project/mlir/include/mlir/Tools/mlir-lsp-server/MlirLspRegistryFunction.h b/external/llvm-project/mlir/include/mlir/Tools/mlir-lsp-server/MlirLspRegistryFunction.h
new file mode 100644
index 000000000000..4811ecb5e92b
--- /dev/null
+++ b/external/llvm-project/mlir/include/mlir/Tools/mlir-lsp-server/MlirLspRegistryFunction.h
@@ -0,0 +1,30 @@
+//===- MlirLspRegistryFunction.h - LSP registry functions -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Registry function types for MLIR LSP.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIR_LSP_SERVER_MLIRLSPREGISTRYFUNCTION_H
+#define MLIR_TOOLS_MLIR_LSP_SERVER_MLIRLSPREGISTRYFUNCTION_H
+
+namespace llvm {
+template <typename Fn>
+class function_ref;
+} // namespace llvm
+
+namespace mlir {
+class DialectRegistry;
+namespace lsp {
+class URIForFile;
+using DialectRegistryFn =
+    llvm::function_ref<DialectRegistry &(const URIForFile &uri)>;
+} // namespace lsp
+} // namespace mlir
+
+#endif // MLIR_TOOLS_MLIR_LSP_SERVER_MLIRLSPREGISTRYFUNCTION_H
diff --git a/external/llvm-project/mlir/include/mlir/Tools/mlir-lsp-server/MlirLspServerMain.h b/external/llvm-project/mlir/include/mlir/Tools/mlir-lsp-server/MlirLspServerMain.h
index 66d5d40a6d28..a461fc470294 100644
--- a/external/llvm-project/mlir/include/mlir/Tools/mlir-lsp-server/MlirLspServerMain.h
+++ b/external/llvm-project/mlir/include/mlir/Tools/mlir-lsp-server/MlirLspServerMain.h
@@ -12,20 +12,27 @@
 
 #ifndef MLIR_TOOLS_MLIR_LSP_SERVER_MLIRLSPSERVERMAIN_H
 #define MLIR_TOOLS_MLIR_LSP_SERVER_MLIRLSPSERVERMAIN_H
+#include "mlir/Tools/mlir-lsp-server/MlirLspRegistryFunction.h"
 
 namespace llvm {
 struct LogicalResult;
 } // namespace llvm
 
 namespace mlir {
-class DialectRegistry;
 
 /// Implementation for tools like `mlir-lsp-server`.
 /// - registry should contain all the dialects that can be parsed in source IR
-/// passed to the server.
+///   passed to the server.
 llvm::LogicalResult MlirLspServerMain(int argc, char **argv,
                                       DialectRegistry &registry);
 
+/// Implementation for tools like `mlir-lsp-server`.
+/// - registry should contain all the dialects that can be parsed in source IR
+///   passed to the server and may register different dialects depending on the
+///   input URI.
+llvm::LogicalResult MlirLspServerMain(int argc, char **argv,
+                                      lsp::DialectRegistryFn registry_fn);
+
 } // namespace mlir
 
 #endif // MLIR_TOOLS_MLIR_LSP_SERVER_MLIRLSPSERVERMAIN_H
diff --git a/external/llvm-project/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp b/external/llvm-project/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
index 0b39d1404249..016e59dcb744 100644
--- a/external/llvm-project/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
+++ b/external/llvm-project/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
@@ -128,34 +128,8 @@ AbstractSparseForwardDataFlowAnalysis::visitOperation(Operation *op) {
     operandLattices.push_back(operandLattice);
   }
 
-  if (auto call = dyn_cast<CallOpInterface>(op)) {
-    // If the call operation is to an external function, attempt to infer the
-    // results from the call arguments.
-    auto callable =
-        dyn_cast_if_present<CallableOpInterface>(call.resolveCallable());
-    if (!getSolverConfig().isInterprocedural() ||
-        (callable && !callable.getCallableRegion())) {
-      visitExternalCallImpl(call, operandLattices, resultLattices);
-      return success();
-    }
-
-    // Otherwise, the results of a call operation are determined by the
-    // callgraph.
-    const auto *predecessors = getOrCreateFor<PredecessorState>(
-        getProgramPointAfter(op), getProgramPointAfter(call));
-    // If not all return sites are known, then conservatively assume we can't
-    // reason about the data-flow.
-    if (!predecessors->allPredecessorsKnown()) {
-      setAllToEntryStates(resultLattices);
-      return success();
-    }
-    for (Operation *predecessor : predecessors->getKnownPredecessors())
-      for (auto &&[operand, resLattice] :
-           llvm::zip(predecessor->getOperands(), resultLattices))
-        join(resLattice,
-             *getLatticeElementFor(getProgramPointAfter(op), operand));
-    return success();
-  }
+  if (auto call = dyn_cast<CallOpInterface>(op))
+    return visitCallOperation(call, operandLattices, resultLattices);
 
   // Invoke the operation transfer function.
   return visitOperationImpl(op, operandLattices, resultLattices);
@@ -183,24 +157,8 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) {
   if (block->isEntryBlock()) {
     // Check if this block is the entry block of a callable region.
     auto callable = dyn_cast<CallableOpInterface>(block->getParentOp());
-    if (callable && callable.getCallableRegion() == block->getParent()) {
-      const auto *callsites = getOrCreateFor<PredecessorState>(
-          getProgramPointBefore(block), getProgramPointAfter(callable));
-      // If not all callsites are known, conservatively mark all lattices as
-      // having reached their pessimistic fixpoints.
-      if (!callsites->allPredecessorsKnown() ||
-          !getSolverConfig().isInterprocedural()) {
-        return setAllToEntryStates(argLattices);
-      }
-      for (Operation *callsite : callsites->getKnownPredecessors()) {
-        auto call = cast<CallOpInterface>(callsite);
-        for (auto it : llvm::zip(call.getArgOperands(), argLattices))
-          join(std::get<1>(it),
-               *getLatticeElementFor(getProgramPointBefore(block),
-                                     std::get<0>(it)));
-      }
-      return;
-    }
+    if (callable && callable.getCallableRegion() == block->getParent())
+      return visitCallableOperation(callable, argLattices);
 
     // Check if the lattices can be determined from region control flow.
     if (auto branch = dyn_cast<RegionBranchOpInterface>(block->getParentOp())) {
@@ -248,6 +206,59 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) {
   }
 }
 
+LogicalResult AbstractSparseForwardDataFlowAnalysis::visitCallOperation(
+    CallOpInterface call,
+    ArrayRef<const AbstractSparseLattice *> operandLattices,
+    ArrayRef<AbstractSparseLattice *> resultLattices) {
+  // If the call operation is to an external function, attempt to infer the
+  // results from the call arguments.
+  auto callable =
+      dyn_cast_if_present<CallableOpInterface>(call.resolveCallable());
+  if (!getSolverConfig().isInterprocedural() ||
+      (callable && !callable.getCallableRegion())) {
+    visitExternalCallImpl(call, operandLattices, resultLattices);
+    return success();
+  }
+
+  // Otherwise, the results of a call operation are determined by the
+  // callgraph.
+  const auto *predecessors = getOrCreateFor<PredecessorState>(
+      getProgramPointAfter(call), getProgramPointAfter(call));
+  // If not all return sites are known, then conservatively assume we can't
+  // reason about the data-flow.
+  if (!predecessors->allPredecessorsKnown()) {
+    setAllToEntryStates(resultLattices);
+    return success();
+  }
+  for (Operation *predecessor : predecessors->getKnownPredecessors())
+    for (auto &&[operand, resLattice] :
+         llvm::zip(predecessor->getOperands(), resultLattices))
+      join(resLattice,
+           *getLatticeElementFor(getProgramPointAfter(call), operand));
+  return success();
+}
+
+void AbstractSparseForwardDataFlowAnalysis::visitCallableOperation(
+    CallableOpInterface callable,
+    ArrayRef<AbstractSparseLattice *> argLattices) {
+  Block *entryBlock = &callable.getCallableRegion()->front();
+  const auto *callsites = getOrCreateFor<PredecessorState>(
+      getProgramPointBefore(entryBlock), getProgramPointAfter(callable));
+  // If not all callsites are known, conservatively mark all lattices as
+  // having reached their pessimistic fixpoints.
+  if (!callsites->allPredecessorsKnown() ||
+      !getSolverConfig().isInterprocedural()) {
+    return setAllToEntryStates(argLattices);
+  }
+  for (Operation *callsite : callsites->getKnownPredecessors()) {
+    auto call = cast<CallOpInterface>(callsite);
+    for (auto it : llvm::zip(call.getArgOperands(), argLattices))
+      join(std::get<1>(it),
+           *getLatticeElementFor(getProgramPointBefore(entryBlock),
+                                 std::get<0>(it)));
+  }
+}
+
 void AbstractSparseForwardDataFlowAnalysis::visitRegionSuccessors(
     ProgramPoint *point, RegionBranchOpInterface branch,
     RegionBranchPoint successor, ArrayRef<AbstractSparseLattice *> lattices) {
@@ -512,31 +523,34 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
   if (op->hasTrait<OpTrait::ReturnLike>()) {
     // Going backwards, the operands of the return are derived from the
     // results of all CallOps calling this CallableOp.
-    if (auto callable = dyn_cast<CallableOpInterface>(op->getParentOp())) {
-      const PredecessorState *callsites = getOrCreateFor<PredecessorState>(
-          getProgramPointAfter(op), getProgramPointAfter(callable));
-      if (callsites->allPredecessorsKnown()) {
-        for (Operation *call : callsites->getKnownPredecessors()) {
-          SmallVector<const AbstractSparseLattice *> callResultLattices =
-              getLatticeElementsFor(getProgramPointAfter(op),
-                                    call->getResults());
-          for (auto [op, result] :
-               llvm::zip(operandLattices, callResultLattices))
-            meet(op, *result);
-        }
-      } else {
-        // If we don't know all the callers, we can't know where the
-        // returned values go. Note that, in particular, this will trigger
-        // for the return ops of any public functions.
-        setAllToExitStates(operandLattices);
-      }
-      return success();
-    }
+    if (auto callable = dyn_cast<CallableOpInterface>(op->getParentOp()))
+      return visitCallableOperation(op, callable, operandLattices);
   }
 
   return visitOperationImpl(op, operandLattices, resultLattices);
 }
 
+LogicalResult AbstractSparseBackwardDataFlowAnalysis::visitCallableOperation(
+    Operation *op, CallableOpInterface callable,
+    ArrayRef<AbstractSparseLattice *> operandLattices) {
+  const PredecessorState *callsites = getOrCreateFor<PredecessorState>(
+      getProgramPointAfter(op), getProgramPointAfter(callable));
+  if (callsites->allPredecessorsKnown()) {
+    for (Operation *call : callsites->getKnownPredecessors()) {
+      SmallVector<const AbstractSparseLattice *> callResultLattices =
+          getLatticeElementsFor(getProgramPointAfter(op), call->getResults());
+      for (auto [op, result] : llvm::zip(operandLattices, callResultLattices))
+        meet(op, *result);
+    }
+  } else {
+    // If we don't know all the callers, we can't know where the
+    // returned values go. Note that, in particular, this will trigger
+    // for the return ops of any public functions.
+    setAllToExitStates(operandLattices);
+  }
+  return success();
+}
+
 void AbstractSparseBackwardDataFlowAnalysis::visitRegionSuccessors(
     RegionBranchOpInterface branch,
     ArrayRef<AbstractSparseLattice *> operandLattices) {
diff --git a/external/llvm-project/mlir/lib/Bytecode/Writer/IRNumbering.cpp b/external/llvm-project/mlir/lib/Bytecode/Writer/IRNumbering.cpp
index 1bc02e172157..8e8e7148ee70 100644
--- a/external/llvm-project/mlir/lib/Bytecode/Writer/IRNumbering.cpp
+++ b/external/llvm-project/mlir/lib/Bytecode/Writer/IRNumbering.cpp
@@ -308,7 +308,7 @@ void IRNumberingState::computeGlobalNumberingState(Operation *rootOp) {
 }
 
 void IRNumberingState::number(Attribute attr) {
-  auto it = attrs.insert({attr, nullptr});
+  auto it = attrs.try_emplace(attr);
   if (!it.second) {
     ++it.first->second->refCount;
     return;
@@ -475,7 +475,7 @@ void IRNumberingState::number(OperationName opName) {
 }
 
 void IRNumberingState::number(Type type) {
-  auto it = types.insert({type, nullptr});
+  auto it = types.try_emplace(type);
   if (!it.second) {
     ++it.first->second->refCount;
     return;
diff --git a/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index c5094799bbef..074404add47f 100644
--- a/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -24,6 +24,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <optional>
 
 namespace mlir {
@@ -1100,7 +1101,7 @@ struct GatherToLDSOpLowering : public ConvertOpToLLVMPattern<GatherToLDSOp> {
     Location loc = op.getLoc();
 
     auto srcMemRefType = cast<MemRefType>(op.getSrc().getType());
-    auto dstMemRefType = cast<MemRefType>(op.getSrc().getType());
+    auto dstMemRefType = cast<MemRefType>(op.getDst().getType());
 
     // TODO: instead of only transfering one element per thread, we could
     // augment it to transfer multiple elements per thread by issuing multiple
@@ -1174,6 +1175,32 @@ struct PackedStochRoundFp8OpLowering final
                   PackedStochRoundFp8OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override;
 };
+
+struct ScaledExtPackedOpLowering final
+    : public ConvertOpToLLVMPattern<ScaledExtPackedOp> {
+  ScaledExtPackedOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
+      : ConvertOpToLLVMPattern<amdgpu::ScaledExtPackedOp>(converter),
+        chipset(chipset) {}
+  Chipset chipset;
+
+  LogicalResult
+  matchAndRewrite(ScaledExtPackedOp op, ScaledExtPackedOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
+struct PackedScaledTruncOpLowering final
+    : public ConvertOpToLLVMPattern<PackedScaledTruncOp> {
+  PackedScaledTruncOpLowering(const LLVMTypeConverter &converter,
+                              Chipset chipset)
+      : ConvertOpToLLVMPattern<amdgpu::PackedScaledTruncOp>(converter),
+        chipset(chipset) {}
+  Chipset chipset;
+
+  LogicalResult
+  matchAndRewrite(PackedScaledTruncOp op, PackedScaledTruncOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
 } // end namespace
 
 LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
@@ -1230,6 +1257,165 @@ LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
   return success();
 }
 
+LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
+    ScaledExtPackedOp op, ScaledExtPackedOpAdaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  Location loc = op.getLoc();
+  if (chipset != kGfx950)
+    return rewriter.notifyMatchFailure(
+        loc, "Scaled fp conversion instructions are not available on target "
+             "architecture and their emulation is not implemented");
+  Type i32 = getTypeConverter()->convertType(rewriter.getI32Type());
+
+  Value source = adaptor.getSource();
+  Value scale = adaptor.getScale();
+
+  VectorType sourceVecType = cast<VectorType>(op.getSource().getType());
+  Type sourceElemType = sourceVecType.getElementType();
+  VectorType destVecType = cast<VectorType>(op.getResult().getType());
+  Type destElemType = destVecType.getElementType();
+
+  VectorType packedVecType;
+  if (isa<Float8E5M2Type, Float8E4M3FNType>(sourceElemType)) {
+    VectorType v4i8 = VectorType::get(4, rewriter.getI8Type());
+    packedVecType = cast<VectorType>(getTypeConverter()->convertType(v4i8));
+  } else if (isa<Float4E2M1FNType>(sourceElemType)) {
+    VectorType v8i4 = VectorType::get(8, rewriter.getI4Type());
+    packedVecType = cast<VectorType>(getTypeConverter()->convertType(v8i4));
+  } else {
+    llvm_unreachable("invalid element type for scaled ext");
+  }
+
+  // Extend to a packedVectorType
+  if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
+    Value longVec = rewriter.create<LLVM::ZeroOp>(loc, packedVecType);
+    if (!sourceVecType) {
+      longVec = rewriter.create<LLVM::InsertElementOp>(
+          loc, longVec, source, createI32Constant(rewriter, loc, 0));
+    } else {
+      for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
+        Value idx = createI32Constant(rewriter, loc, i);
+        Value elem = rewriter.create<LLVM::ExtractElementOp>(loc, source, idx);
+        longVec =
+            rewriter.create<LLVM::InsertElementOp>(loc, longVec, elem, idx);
+      }
+    }
+    source = longVec;
+  }
+  Value i32Source = rewriter.create<LLVM::BitcastOp>(loc, i32, source);
+
+  if (isa<Float8E5M2Type>(sourceElemType) && destElemType.isF32())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Bf8Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float8E5M2Type>(sourceElemType) && destElemType.isF16())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF16Bf8Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float8E5M2Type>(sourceElemType) && destElemType.isBF16())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkBf16Bf8Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float8E4M3FNType>(sourceElemType) && destElemType.isF32())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Fp8Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float8E4M3FNType>(sourceElemType) && destElemType.isF16())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF16Fp8Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float8E4M3FNType>(sourceElemType) && destElemType.isBF16())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkBf16Fp8Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float4E2M1FNType>(sourceElemType) && destElemType.isF32())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Fp4Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float4E2M1FNType>(sourceElemType) && destElemType.isF16())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF16Fp4Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else if (isa<Float4E2M1FNType>(sourceElemType) && destElemType.isBF16())
+    rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkBf16Fp4Op>(
+        op, destVecType, i32Source, scale, op.getIndex());
+  else
+    return failure();
+
+  return success();
+}
+
+LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
+    PackedScaledTruncOp op, PackedScaledTruncOpAdaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  Location loc = op.getLoc();
+  if (chipset != kGfx950)
+    return rewriter.notifyMatchFailure(
+        loc, "Scaled fp conversion instructions are not available on target "
+             "architecture and their emulation is not implemented");
+  Type v2i16 = getTypeConverter()->convertType(
+      VectorType::get(2, rewriter.getI16Type()));
+  Type i32 = getTypeConverter()->convertType(rewriter.getI32Type());
+
+  Type resultType = op.getResult().getType();
+  Type resultElemType = getElementTypeOrSelf(resultType);
+  VectorType sourceVecType = cast<VectorType>(op.getSource().getType());
+  Type sourceElemType = sourceVecType.getElementType();
+
+  Type intResultType = isa<Float4E2M1FNType>(resultElemType) ? i32 : v2i16;
+
+  Value source = adaptor.getSource();
+  Value scale = adaptor.getScale();
+  Value existing = adaptor.getExisting();
+  if (existing)
+    existing = rewriter.create<LLVM::BitcastOp>(loc, intResultType, existing);
+  else
+    existing = rewriter.create<LLVM::ZeroOp>(loc, intResultType);
+
+  if (sourceVecType.getNumElements() < 2) {
+    Value c0 = createI32Constant(rewriter, loc, 0);
+    Value elem0 = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
+    VectorType v2 = VectorType::get(2, sourceElemType);
+    source = rewriter.create<LLVM::ZeroOp>(loc, v2);
+    source = rewriter.create<LLVM::InsertElementOp>(loc, source, elem0, c0);
+  }
+
+  Value sourceA, sourceB;
+  if (sourceElemType.isF32()) {
+    Value c0 = createI32Constant(rewriter, loc, 0);
+    Value c1 = createI32Constant(rewriter, loc, 1);
+    sourceA = rewriter.create<LLVM::ExtractElementOp>(loc, source, c0);
+    sourceB = rewriter.create<LLVM::ExtractElementOp>(loc, source, c1);
+  }
+
+  Value result;
+  if (sourceElemType.isF32() && isa<Float8E5M2Type>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F32Op>(
+        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+  else if (sourceElemType.isF16() && isa<Float8E5M2Type>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkBf8F16Op>(
+        loc, intResultType, existing, source, scale, op.getIndex());
+  else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkBf8Bf16Op>(
+        loc, intResultType, existing, source, scale, op.getIndex());
+  else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F32Op>(
+        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+  else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkFp8F16Op>(
+        loc, intResultType, existing, source, scale, op.getIndex());
+  else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkFp8Bf16Op>(
+        loc, intResultType, existing, source, scale, op.getIndex());
+  else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F32Op>(
+        loc, intResultType, existing, sourceA, sourceB, scale, op.getIndex());
+  else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkFp4F16Op>(
+        loc, intResultType, existing, source, scale, op.getIndex());
+  else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(resultElemType))
+    result = rewriter.create<ROCDL::CvtScaleF32PkFp4Bf16Op>(
+        loc, intResultType, existing, source, scale, op.getIndex());
+  else
+    return failure();
+
+  result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
+      op, getTypeConverter()->convertType(resultType), result);
+  return success();
+}
+
 LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
     PackedTrunc2xFp8Op op, PackedTrunc2xFp8OpAdaptor adaptor,
     ConversionPatternRewriter &rewriter) const {
@@ -1547,7 +1733,8 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
                                ROCDL::RawPtrBufferAtomicCmpSwap>,
            AMDGPUDPPLowering, LDSBarrierOpLowering, SchedBarrierOpLowering,
            MFMAOpLowering, ScaledMFMAOpLowering, WMMAOpLowering,
-           ExtPackedFp8OpLowering, PackedTrunc2xFp8OpLowering,
+           ExtPackedFp8OpLowering, ScaledExtPackedOpLowering,
+           PackedScaledTruncOpLowering, PackedTrunc2xFp8OpLowering,
            PackedStochRoundFp8OpLowering, GatherToLDSOpLowering>(converter,
                                                                  chipset);
   patterns.add<AMDGPUSwizzleBitModeLowering>(converter);
diff --git a/external/llvm-project/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/external/llvm-project/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index dea3d99704d9..01ca5e99a9af 100644
--- a/external/llvm-project/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -543,14 +543,20 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   // the device code, not the host code
   auto moduleOp = gpuPrintfOp->getParentOfType<gpu::GPUModuleOp>();
 
+  // Create a valid global location removing any metadata attached to the
+  // location as debug info metadata inside of a function cannot be used outside
+  // of that function.
+  Location globalLoc = loc->findInstanceOfOrUnknown<FileLineColLoc>();
+
   auto vprintfType =
       LLVM::LLVMFunctionType::get(rewriter.getI32Type(), {ptrType, ptrType});
-  LLVM::LLVMFuncOp vprintfDecl =
-      getOrDefineFunction(moduleOp, loc, rewriter, "vprintf", vprintfType);
+  LLVM::LLVMFuncOp vprintfDecl = getOrDefineFunction(
+      moduleOp, globalLoc, rewriter, "vprintf", vprintfType);
 
   // Create the global op or find an existing one.
-  LLVM::GlobalOp global = getOrCreateStringConstant(
-      rewriter, loc, moduleOp, llvmI8, "printfFormat_", adaptor.getFormat());
+  LLVM::GlobalOp global =
+      getOrCreateStringConstant(rewriter, globalLoc, moduleOp, llvmI8,
+                                "printfFormat_", adaptor.getFormat());
 
   // Get a pointer to the format string's first element
   Value globalPtr = rewriter.create<LLVM::AddressOfOp>(loc, global);
diff --git a/external/llvm-project/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/external/llvm-project/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index b4c31a338ce9..34150c4d1308 100644
--- a/external/llvm-project/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/external/llvm-project/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -199,4 +199,4 @@ struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 
 } // namespace mlir
 
-#endif // MLIR_CONVERSION_GPUCOMMON_OPTOFUNCCALLLOWERING_H_
\ No newline at end of file
+#endif // MLIR_CONVERSION_GPUCOMMON_OPTOFUNCCALLLOWERING_H_
diff --git a/external/llvm-project/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/external/llvm-project/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 65c9c114e883..d6d66d9f4e56 100644
--- a/external/llvm-project/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -10,7 +10,6 @@
 // GPU operations.
 //
 //===----------------------------------------------------------------------===//
-#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
 
 #include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
 #include "mlir/Dialect/Arith/Transforms/Passes.h"
@@ -18,11 +17,6 @@
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/Passes.h"
 
-#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
-#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
-#include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
-#include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
-#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 #include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
@@ -48,7 +42,6 @@
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 
 #include "../GPUCommon/GPUOpsLowering.h"
 #include "../GPUCommon/IndexIntrinsicsOpLowering.h"
@@ -58,8 +51,6 @@ namespace mlir {
 #include "mlir/Conversion/Passes.h.inc"
 } // namespace mlir
 
-#include "mlir/Dialect/LLVMIR/Transforms/Passes.h"
-
 using namespace mlir;
 
 // Truncate or extend the result depending on the index bitwidth specified
@@ -384,11 +375,6 @@ struct LowerGpuOpsToROCDLOpsPass final
 
     populateAMDGPUToROCDLConversionPatterns(converter, llvmPatterns,
                                             *maybeChipset);
-    // TODO (rocmlir): remove hardcoded passes
-    // related PR: https://github.com/llvm/llvm-project/pull/124439
-    mlir::vector::populateVectorInsertExtractStridedSliceTransforms(llvmPatterns);
-    // TODO: ends here
-
     populateGpuToROCDLConversionPatterns(converter, llvmPatterns, runtime,
                                          *maybeChipset);
     configureGpuToROCDLConversionLegality(target);
@@ -424,9 +410,7 @@ void mlir::configureGpuToROCDLConversionLegality(ConversionTarget &target) {
   target.addIllegalOp<func::FuncOp>();
   target.addLegalDialect<::mlir::LLVM::LLVMDialect>();
   target.addLegalDialect<ROCDL::ROCDLDialect>();
-  // TODO (rocmlir): remove vector::VectorDialect
-  // related PR: https://github.com/llvm/llvm-project/pull/124439
-  target.addIllegalDialect<gpu::GPUDialect, vector::VectorDialect>();
+  target.addIllegalDialect<gpu::GPUDialect>();
   target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FCeilOp,
                       LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp, LLVM::Log10Op,
                       LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp>();
diff --git a/external/llvm-project/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/external/llvm-project/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index 78e6ebb523a4..46db5d3fdca3 100644
--- a/external/llvm-project/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -464,27 +464,39 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
 
 template <typename UniformOp, typename NonUniformOp>
 static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc,
-                                     Value arg, bool isGroup, bool isUniform) {
+                                     Value arg, bool isGroup, bool isUniform,
+                                     std::optional<uint32_t> clusterSize) {
   Type type = arg.getType();
   auto scope = mlir::spirv::ScopeAttr::get(builder.getContext(),
                                            isGroup ? spirv::Scope::Workgroup
                                                    : spirv::Scope::Subgroup);
-  auto groupOp = spirv::GroupOperationAttr::get(builder.getContext(),
-                                                spirv::GroupOperation::Reduce);
+  auto groupOp = spirv::GroupOperationAttr::get(
+      builder.getContext(), clusterSize.has_value()
+                                ? spirv::GroupOperation::ClusteredReduce
+                                : spirv::GroupOperation::Reduce);
   if (isUniform) {
     return builder.create<UniformOp>(loc, type, scope, groupOp, arg)
         .getResult();
   }
-  return builder.create<NonUniformOp>(loc, type, scope, groupOp, arg, Value{})
+
+  Value clusterSizeValue;
+  if (clusterSize.has_value())
+    clusterSizeValue = builder.create<spirv::ConstantOp>(
+        loc, builder.getI32Type(),
+        builder.getIntegerAttr(builder.getI32Type(), *clusterSize));
+
+  return builder
+      .create<NonUniformOp>(loc, type, scope, groupOp, arg, clusterSizeValue)
       .getResult();
 }
 
-static std::optional<Value> createGroupReduceOp(OpBuilder &builder,
-                                                Location loc, Value arg,
-                                                gpu::AllReduceOperation opType,
-                                                bool isGroup, bool isUniform) {
+static std::optional<Value>
+createGroupReduceOp(OpBuilder &builder, Location loc, Value arg,
+                    gpu::AllReduceOperation opType, bool isGroup,
+                    bool isUniform, std::optional<uint32_t> clusterSize) {
   enum class ElemType { Float, Boolean, Integer };
-  using FuncT = Value (*)(OpBuilder &, Location, Value, bool, bool);
+  using FuncT = Value (*)(OpBuilder &, Location, Value, bool, bool,
+                          std::optional<uint32_t>);
   struct OpHandler {
     gpu::AllReduceOperation kind;
     ElemType elemType;
@@ -548,7 +560,7 @@ static std::optional<Value> createGroupReduceOp(OpBuilder &builder,
 
   for (const OpHandler &handler : handlers)
     if (handler.kind == opType && elementType == handler.elemType)
-      return handler.func(builder, loc, arg, isGroup, isUniform);
+      return handler.func(builder, loc, arg, isGroup, isUniform, clusterSize);
 
   return std::nullopt;
 }
@@ -571,7 +583,7 @@ class GPUAllReduceConversion final
 
     auto result =
         createGroupReduceOp(rewriter, op.getLoc(), adaptor.getValue(), *opType,
-                            /*isGroup*/ true, op.getUniform());
+                            /*isGroup*/ true, op.getUniform(), std::nullopt);
     if (!result)
       return failure();
 
@@ -589,16 +601,17 @@ class GPUSubgroupReduceConversion final
   LogicalResult
   matchAndRewrite(gpu::SubgroupReduceOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    if (op.getClusterSize())
+    if (op.getClusterStride() > 1) {
       return rewriter.notifyMatchFailure(
-          op, "lowering for clustered reduce not implemented");
+          op, "lowering for cluster stride > 1 is not implemented");
+    }
 
     if (!isa<spirv::ScalarType>(adaptor.getValue().getType()))
       return rewriter.notifyMatchFailure(op, "reduction type is not a scalar");
 
-    auto result = createGroupReduceOp(rewriter, op.getLoc(), adaptor.getValue(),
-                                      adaptor.getOp(),
-                                      /*isGroup=*/false, adaptor.getUniform());
+    auto result = createGroupReduceOp(
+        rewriter, op.getLoc(), adaptor.getValue(), adaptor.getOp(),
+        /*isGroup=*/false, adaptor.getUniform(), op.getClusterSize());
     if (!result)
       return failure();
 
diff --git a/external/llvm-project/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/external/llvm-project/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
index b42bb773f53e..f4d69ce8235b 100644
--- a/external/llvm-project/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp
@@ -73,6 +73,8 @@ using ATanOpLowering = ConvertFMFMathToLLVMPattern<math::AtanOp, LLVM::ATanOp>;
 using ATan2OpLowering =
     ConvertFMFMathToLLVMPattern<math::Atan2Op, LLVM::ATan2Op>;
 // A `CtLz/CtTz/absi(a)` is converted into `CtLz/CtTz/absi(a, false)`.
+// TODO: Result and operand types match for `absi` as opposed to `ct*z`, so it
+// may be better to separate the patterns.
 template <typename MathOp, typename LLVMOp>
 struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
   using ConvertOpToLLVMPattern<MathOp>::ConvertOpToLLVMPattern;
@@ -81,26 +83,29 @@ struct IntOpWithFlagLowering : public ConvertOpToLLVMPattern<MathOp> {
   LogicalResult
   matchAndRewrite(MathOp op, typename MathOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
+    const auto &typeConverter = *this->getTypeConverter();
     auto operandType = adaptor.getOperand().getType();
-
-    if (!operandType || !LLVM::isCompatibleType(operandType))
+    auto llvmOperandType = typeConverter.convertType(operandType);
+    if (!llvmOperandType)
       return failure();
 
     auto loc = op.getLoc();
     auto resultType = op.getResult().getType();
+    auto llvmResultType = typeConverter.convertType(resultType);
+    if (!llvmResultType)
+      return failure();
 
-    if (!isa<LLVM::LLVMArrayType>(operandType)) {
-      rewriter.replaceOpWithNewOp<LLVMOp>(op, resultType, adaptor.getOperand(),
-                                          false);
+    if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
+      rewriter.replaceOpWithNewOp<LLVMOp>(op, llvmResultType,
+                                          adaptor.getOperand(), false);
       return success();
     }
 
-    auto vectorType = dyn_cast<VectorType>(resultType);
-    if (!vectorType)
+    if (!isa<VectorType>(llvmResultType))
       return failure();
 
     return LLVM::detail::handleMultidimensionalVectors(
-        op.getOperation(), adaptor.getOperands(), *this->getTypeConverter(),
+        op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
           return rewriter.create<LLVMOp>(loc, llvm1DVectorTy, operands[0],
                                          false);
@@ -123,40 +128,42 @@ struct ExpM1OpLowering : public ConvertOpToLLVMPattern<math::ExpM1Op> {
   LogicalResult
   matchAndRewrite(math::ExpM1Op op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
+    const auto &typeConverter = *this->getTypeConverter();
     auto operandType = adaptor.getOperand().getType();
-
-    if (!operandType || !LLVM::isCompatibleType(operandType))
+    auto llvmOperandType = typeConverter.convertType(operandType);
+    if (!llvmOperandType)
       return failure();
 
     auto loc = op.getLoc();
     auto resultType = op.getResult().getType();
-    auto floatType = cast<FloatType>(getElementTypeOrSelf(resultType));
+    auto floatType = cast<FloatType>(
+        typeConverter.convertType(getElementTypeOrSelf(resultType)));
     auto floatOne = rewriter.getFloatAttr(floatType, 1.0);
     ConvertFastMath<math::ExpM1Op, LLVM::ExpOp> expAttrs(op);
     ConvertFastMath<math::ExpM1Op, LLVM::FSubOp> subAttrs(op);
 
-    if (!isa<LLVM::LLVMArrayType>(operandType)) {
+    if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
-      if (LLVM::isCompatibleVectorType(operandType)) {
+      if (LLVM::isCompatibleVectorType(llvmOperandType)) {
         one = rewriter.create<LLVM::ConstantOp>(
-            loc, operandType,
-            SplatElementsAttr::get(cast<ShapedType>(resultType), floatOne));
+            loc, llvmOperandType,
+            SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
+                                   floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, operandType, floatOne);
+        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
       }
       auto exp = rewriter.create<LLVM::ExpOp>(loc, adaptor.getOperand(),
                                               expAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
-          op, operandType, ValueRange{exp, one}, subAttrs.getAttrs());
+          op, llvmOperandType, ValueRange{exp, one}, subAttrs.getAttrs());
       return success();
     }
 
-    auto vectorType = dyn_cast<VectorType>(resultType);
-    if (!vectorType)
+    if (!isa<VectorType>(resultType))
       return rewriter.notifyMatchFailure(op, "expected vector result type");
 
     return LLVM::detail::handleMultidimensionalVectors(
-        op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
+        op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
           auto numElements = LLVM::getVectorNumElements(llvm1DVectorTy);
           auto splatAttr = SplatElementsAttr::get(
@@ -181,41 +188,43 @@ struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
   LogicalResult
   matchAndRewrite(math::Log1pOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
+    const auto &typeConverter = *this->getTypeConverter();
     auto operandType = adaptor.getOperand().getType();
-
-    if (!operandType || !LLVM::isCompatibleType(operandType))
+    auto llvmOperandType = typeConverter.convertType(operandType);
+    if (!llvmOperandType)
       return rewriter.notifyMatchFailure(op, "unsupported operand type");
 
     auto loc = op.getLoc();
     auto resultType = op.getResult().getType();
-    auto floatType = cast<FloatType>(getElementTypeOrSelf(resultType));
+    auto floatType = cast<FloatType>(
+        typeConverter.convertType(getElementTypeOrSelf(resultType)));
     auto floatOne = rewriter.getFloatAttr(floatType, 1.0);
     ConvertFastMath<math::Log1pOp, LLVM::FAddOp> addAttrs(op);
     ConvertFastMath<math::Log1pOp, LLVM::LogOp> logAttrs(op);
 
-    if (!isa<LLVM::LLVMArrayType>(operandType)) {
+    if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one =
-          LLVM::isCompatibleVectorType(operandType)
+          isa<VectorType>(llvmOperandType)
               ? rewriter.create<LLVM::ConstantOp>(
-                    loc, operandType,
-                    SplatElementsAttr::get(cast<ShapedType>(resultType),
+                    loc, llvmOperandType,
+                    SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
                                            floatOne))
-              : rewriter.create<LLVM::ConstantOp>(loc, operandType, floatOne);
+              : rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType,
+                                                  floatOne);
 
       auto add = rewriter.create<LLVM::FAddOp>(
-          loc, operandType, ValueRange{one, adaptor.getOperand()},
+          loc, llvmOperandType, ValueRange{one, adaptor.getOperand()},
           addAttrs.getAttrs());
-      rewriter.replaceOpWithNewOp<LLVM::LogOp>(op, operandType, ValueRange{add},
-                                               logAttrs.getAttrs());
+      rewriter.replaceOpWithNewOp<LLVM::LogOp>(
+          op, llvmOperandType, ValueRange{add}, logAttrs.getAttrs());
       return success();
     }
 
-    auto vectorType = dyn_cast<VectorType>(resultType);
-    if (!vectorType)
+    if (!isa<VectorType>(resultType))
       return rewriter.notifyMatchFailure(op, "expected vector result type");
 
     return LLVM::detail::handleMultidimensionalVectors(
-        op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
+        op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
           auto numElements = LLVM::getVectorNumElements(llvm1DVectorTy);
           auto splatAttr = SplatElementsAttr::get(
@@ -241,40 +250,42 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
   LogicalResult
   matchAndRewrite(math::RsqrtOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
+    const auto &typeConverter = *this->getTypeConverter();
     auto operandType = adaptor.getOperand().getType();
-
-    if (!operandType || !LLVM::isCompatibleType(operandType))
+    auto llvmOperandType = typeConverter.convertType(operandType);
+    if (!llvmOperandType)
       return failure();
 
     auto loc = op.getLoc();
     auto resultType = op.getResult().getType();
-    auto floatType = cast<FloatType>(getElementTypeOrSelf(resultType));
+    auto floatType = cast<FloatType>(
+        typeConverter.convertType(getElementTypeOrSelf(resultType)));
     auto floatOne = rewriter.getFloatAttr(floatType, 1.0);
     ConvertFastMath<math::RsqrtOp, LLVM::SqrtOp> sqrtAttrs(op);
     ConvertFastMath<math::RsqrtOp, LLVM::FDivOp> divAttrs(op);
 
-    if (!isa<LLVM::LLVMArrayType>(operandType)) {
+    if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
       LLVM::ConstantOp one;
-      if (LLVM::isCompatibleVectorType(operandType)) {
+      if (isa<VectorType>(llvmOperandType)) {
         one = rewriter.create<LLVM::ConstantOp>(
-            loc, operandType,
-            SplatElementsAttr::get(cast<ShapedType>(resultType), floatOne));
+            loc, llvmOperandType,
+            SplatElementsAttr::get(cast<ShapedType>(llvmOperandType),
+                                   floatOne));
       } else {
-        one = rewriter.create<LLVM::ConstantOp>(loc, operandType, floatOne);
+        one = rewriter.create<LLVM::ConstantOp>(loc, llvmOperandType, floatOne);
       }
       auto sqrt = rewriter.create<LLVM::SqrtOp>(loc, adaptor.getOperand(),
                                                 sqrtAttrs.getAttrs());
       rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
-          op, operandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
+          op, llvmOperandType, ValueRange{one, sqrt}, divAttrs.getAttrs());
       return success();
     }
 
-    auto vectorType = dyn_cast<VectorType>(resultType);
-    if (!vectorType)
+    if (!isa<VectorType>(resultType))
       return failure();
 
     return LLVM::detail::handleMultidimensionalVectors(
-        op.getOperation(), adaptor.getOperands(), *getTypeConverter(),
+        op.getOperation(), adaptor.getOperands(), typeConverter,
         [&](Type llvm1DVectorTy, ValueRange operands) {
           auto numElements = LLVM::getVectorNumElements(llvm1DVectorTy);
           auto splatAttr = SplatElementsAttr::get(
@@ -298,13 +309,15 @@ struct IsNaNOpLowering : public ConvertOpToLLVMPattern<math::IsNaNOp> {
   LogicalResult
   matchAndRewrite(math::IsNaNOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto operandType = adaptor.getOperand().getType();
-
-    if (!operandType || !LLVM::isCompatibleType(operandType))
+    const auto &typeConverter = *this->getTypeConverter();
+    auto operandType =
+        typeConverter.convertType(adaptor.getOperand().getType());
+    auto resultType = typeConverter.convertType(op.getResult().getType());
+    if (!operandType || !resultType)
       return failure();
 
     rewriter.replaceOpWithNewOp<LLVM::IsFPClass>(
-        op, op.getType(), adaptor.getOperand(), llvm::fcNan);
+        op, resultType, adaptor.getOperand(), llvm::fcNan);
     return success();
   }
 };
@@ -315,13 +328,15 @@ struct IsFiniteOpLowering : public ConvertOpToLLVMPattern<math::IsFiniteOp> {
   LogicalResult
   matchAndRewrite(math::IsFiniteOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto operandType = adaptor.getOperand().getType();
-
-    if (!operandType || !LLVM::isCompatibleType(operandType))
+    const auto &typeConverter = *this->getTypeConverter();
+    auto operandType =
+        typeConverter.convertType(adaptor.getOperand().getType());
+    auto resultType = typeConverter.convertType(op.getResult().getType());
+    if (!operandType || !resultType)
       return failure();
 
     rewriter.replaceOpWithNewOp<LLVM::IsFPClass>(
-        op, op.getType(), adaptor.getOperand(), llvm::fcFinite);
+        op, resultType, adaptor.getOperand(), llvm::fcFinite);
     return success();
   }
 };
diff --git a/external/llvm-project/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp b/external/llvm-project/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
index 1d2b09be87a4..e065c804507a 100644
--- a/external/llvm-project/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
@@ -148,4 +148,4 @@ void ConvertMathToROCDLPass::runOnOperation() {
                       LLVM::SqrtOp>();
   if (failed(applyPartialConversion(m, target, std::move(patterns))))
     signalPassFailure();
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/external/llvm-project/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 1b83794b5f45..501bfa223fb1 100644
--- a/external/llvm-project/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -509,7 +509,15 @@ void populateMathToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
            CheckedElementwiseOpPattern<math::RsqrtOp, spirv::GLInverseSqrtOp>,
            CheckedElementwiseOpPattern<math::SinOp, spirv::GLSinOp>,
            CheckedElementwiseOpPattern<math::SqrtOp, spirv::GLSqrtOp>,
-           CheckedElementwiseOpPattern<math::TanhOp, spirv::GLTanhOp>>(
+           CheckedElementwiseOpPattern<math::TanhOp, spirv::GLTanhOp>,
+           CheckedElementwiseOpPattern<math::TanOp, spirv::GLTanOp>,
+           CheckedElementwiseOpPattern<math::AsinOp, spirv::GLAsinOp>,
+           CheckedElementwiseOpPattern<math::AcosOp, spirv::GLAcosOp>,
+           CheckedElementwiseOpPattern<math::SinhOp, spirv::GLSinhOp>,
+           CheckedElementwiseOpPattern<math::CoshOp, spirv::GLCoshOp>,
+           CheckedElementwiseOpPattern<math::AsinhOp, spirv::GLAsinhOp>,
+           CheckedElementwiseOpPattern<math::AcoshOp, spirv::GLAcoshOp>,
+           CheckedElementwiseOpPattern<math::AtanhOp, spirv::GLAtanhOp>>(
           typeConverter, patterns.getContext());
 
   // OpenCL patterns
@@ -533,7 +541,15 @@ void populateMathToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
                CheckedElementwiseOpPattern<math::RsqrtOp, spirv::CLRsqrtOp>,
                CheckedElementwiseOpPattern<math::SinOp, spirv::CLSinOp>,
                CheckedElementwiseOpPattern<math::SqrtOp, spirv::CLSqrtOp>,
-               CheckedElementwiseOpPattern<math::TanhOp, spirv::CLTanhOp>>(
+               CheckedElementwiseOpPattern<math::TanhOp, spirv::CLTanhOp>,
+               CheckedElementwiseOpPattern<math::TanOp, spirv::CLTanOp>,
+               CheckedElementwiseOpPattern<math::AsinOp, spirv::CLAsinOp>,
+               CheckedElementwiseOpPattern<math::AcosOp, spirv::CLAcosOp>,
+               CheckedElementwiseOpPattern<math::SinhOp, spirv::CLSinhOp>,
+               CheckedElementwiseOpPattern<math::CoshOp, spirv::CLCoshOp>,
+               CheckedElementwiseOpPattern<math::AsinhOp, spirv::CLAsinhOp>,
+               CheckedElementwiseOpPattern<math::AcoshOp, spirv::CLAcoshOp>,
+               CheckedElementwiseOpPattern<math::AtanhOp, spirv::CLAtanhOp>>(
       typeConverter, patterns.getContext());
 }
 
diff --git a/external/llvm-project/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/external/llvm-project/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index fdf799a20efd..ff5b762a969d 100644
--- a/external/llvm-project/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -307,6 +307,17 @@ class CastPattern final : public OpConversionPattern<memref::CastOp> {
   }
 };
 
+/// Converts memref.extract_aligned_pointer_as_index to spirv.ConvertPtrToU.
+class ExtractAlignedPointerAsIndexOpPattern final
+    : public OpConversionPattern<memref::ExtractAlignedPointerAsIndexOp> {
+public:
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(memref::ExtractAlignedPointerAsIndexOp extractOp,
+                  OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
 } // namespace
 
 //===----------------------------------------------------------------------===//
@@ -921,6 +932,20 @@ LogicalResult ReinterpretCastPattern::matchAndRewrite(
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// ExtractAlignedPointerAsIndexOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult ExtractAlignedPointerAsIndexOpPattern::matchAndRewrite(
+    memref::ExtractAlignedPointerAsIndexOp extractOp, OpAdaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>();
+  Type indexType = typeConverter.getIndexType();
+  rewriter.replaceOpWithNewOp<spirv::ConvertPtrToUOp>(extractOp, indexType,
+                                                      adaptor.getSource());
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Pattern population
 //===----------------------------------------------------------------------===//
@@ -928,10 +953,11 @@ LogicalResult ReinterpretCastPattern::matchAndRewrite(
 namespace mlir {
 void populateMemRefToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
                                    RewritePatternSet &patterns) {
-  patterns.add<AllocaOpPattern, AllocOpPattern, AtomicRMWOpPattern,
-               DeallocOpPattern, IntLoadOpPattern, IntStoreOpPattern,
-               LoadOpPattern, MemorySpaceCastOpPattern, StoreOpPattern,
-               ReinterpretCastPattern, CastPattern>(typeConverter,
-                                                    patterns.getContext());
+  patterns
+      .add<AllocaOpPattern, AllocOpPattern, AtomicRMWOpPattern,
+           DeallocOpPattern, IntLoadOpPattern, IntStoreOpPattern, LoadOpPattern,
+           MemorySpaceCastOpPattern, StoreOpPattern, ReinterpretCastPattern,
+           CastPattern, ExtractAlignedPointerAsIndexOpPattern>(
+          typeConverter, patterns.getContext());
 }
 } // namespace mlir
diff --git a/external/llvm-project/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/external/llvm-project/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
index dab15d23f6e0..ac8ed4fdff7c 100644
--- a/external/llvm-project/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
@@ -173,7 +173,7 @@ getTreePredicates(std::vector<PositionalPredicate> &predList, Value val,
 
       // Ignore the specified operand, usually because this position was
       // visited in an upward traversal via an iterative choice.
-      if (ignoreOperand && *ignoreOperand == operandIt.index())
+      if (ignoreOperand == operandIt.index())
         continue;
 
       Position *pos =
diff --git a/external/llvm-project/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp b/external/llvm-project/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
index 310566e69220..7dbccd19a051 100644
--- a/external/llvm-project/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgram.cpp
@@ -26,8 +26,9 @@ class VariableOpConverter : public OpRewritePattern<tosa::VariableOp> {
 
   LogicalResult matchAndRewrite(tosa::VariableOp op,
                                 PatternRewriter &rewriter) const final {
+    auto variableType = tosa::getVariableType(op);
     auto newVariable = rewriter.create<mlir::ml_program::GlobalOp>(
-        op.getLoc(), op.getName(), op.getType(), /*is_mutable=*/true,
+        op.getLoc(), op.getName(), variableType, /*is_mutable=*/true,
         op.getInitialValueAttr(), /*sym_visibility=*/nullptr);
     newVariable.setPrivate();
     rewriter.replaceOp(op, newVariable);
diff --git a/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt b/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
index 330474a718e3..8e2620029c35 100644
--- a/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
@@ -35,6 +35,7 @@ add_mlir_conversion_library(MLIRVectorToLLVMPass
   MLIRVectorToLLVM
 
   MLIRArmNeonDialect
+  MLIRArmNeonTransforms
   MLIRArmSVEDialect
   MLIRArmSVETransforms
   MLIRAMXDialect
diff --git a/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index 0ee6dce9ee94..293e01a5bf4d 100644
--- a/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/AMX/Transforms.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/ArmNeon/Transforms.h"
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
@@ -82,6 +83,12 @@ void ConvertVectorToLLVMPass::runOnOperation() {
     populateVectorStepLoweringPatterns(patterns);
     populateVectorRankReducingFMAPattern(patterns);
     populateVectorGatherLoweringPatterns(patterns);
+    if (armI8MM) {
+      if (armNeon)
+        arm_neon::populateLowerContractionToSMMLAPatternPatterns(patterns);
+      if (armSVE)
+        populateLowerContractionToSVEI8MMPatternPatterns(patterns);
+    }
     (void)applyPatternsGreedily(getOperation(), std::move(patterns));
   }
 
diff --git a/external/llvm-project/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/external/llvm-project/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index a0a98a4e8672..0d0add309466 100644
--- a/external/llvm-project/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -60,6 +60,15 @@ LogicalResult PackedStochRoundFp8Op::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// mxfp float ops
+//===----------------------------------------------------------------------===//
+LogicalResult PackedScaledTruncOp::verify() {
+  if (getExisting() && getExisting().getType() != getResult().getType())
+    return emitOpError("existing values must have same type as result");
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // FatRawBufferCastOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp b/external/llvm-project/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
index 7471dc797e0f..06e5f7c2196d 100644
--- a/external/llvm-project/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
@@ -24,27 +24,18 @@ namespace {
 /// Generic one-to-one conversion of simply mappable operations into calls
 /// to their respective LLVM intrinsics.
 struct AMXIntrinsicOpConversion
-    : public OpInterfaceConversionPattern<amx::AMXIntrinsicOp> {
-  using OpInterfaceConversionPattern<
-      amx::AMXIntrinsicOp>::OpInterfaceConversionPattern;
-
-  AMXIntrinsicOpConversion(const LLVMTypeConverter &typeConverter,
-                           PatternBenefit benefit = 1)
-      : OpInterfaceConversionPattern(typeConverter, &typeConverter.getContext(),
-                                     benefit),
-        typeConverter(typeConverter) {}
+    : public ConvertOpInterfaceToLLVMPattern<amx::AMXIntrinsicOp> {
+  using ConvertOpInterfaceToLLVMPattern::ConvertOpInterfaceToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(amx::AMXIntrinsicOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
+    const LLVMTypeConverter &typeConverter = *getTypeConverter();
     return LLVM::detail::intrinsicRewrite(
         op, rewriter.getStringAttr(op.getIntrinsicName()),
         op.getIntrinsicOperands(operands, typeConverter, rewriter),
         typeConverter, rewriter);
   }
-
-private:
-  const LLVMTypeConverter &typeConverter;
 };
 
 } // namespace
@@ -60,3 +51,22 @@ void mlir::populateAMXLegalizeForLLVMExportPatterns(
 void mlir::configureAMXLegalizeForExportTarget(LLVMConversionTarget &target) {
   target.addIllegalDialect<AMXDialect>();
 }
+
+namespace {
+/// Implement the interface to convert AMX to LLVM.
+struct AMXToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
+  using ConvertToLLVMPatternInterface::ConvertToLLVMPatternInterface;
+
+  void populateConvertToLLVMConversionPatterns(
+      ConversionTarget &target, LLVMTypeConverter &typeConverter,
+      RewritePatternSet &patterns) const final {
+    populateAMXLegalizeForLLVMExportPatterns(typeConverter, patterns);
+  }
+};
+} // namespace
+
+void mlir::registerConvertAMXToLLVMInterface(DialectRegistry &registry) {
+  registry.addExtension(+[](MLIRContext *ctx, amx::AMXDialect *dialect) {
+    dialect->addInterfaces<AMXToLLVMDialectInterface>();
+  });
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp b/external/llvm-project/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
index fe53d0324936..01cc50014838 100644
--- a/external/llvm-project/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
@@ -402,9 +402,7 @@ isVectorizableLoopBodyWithOpCond(AffineForOp loop,
           return !VectorType::isValidElementType(type);
         }))
       return true;
-    return llvm::any_of(op.getResultTypes(), [](Type type) {
-      return !VectorType::isValidElementType(type);
-    });
+    return !llvm::all_of(op.getResultTypes(), VectorType::isValidElementType);
   });
   SmallVector<NestedMatch, 8> opsMatched;
   types.match(forOp, &opsMatched);
diff --git a/external/llvm-project/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/external/llvm-project/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 2364f8957992..3d09c6a9b2c2 100644
--- a/external/llvm-project/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -2367,7 +2367,7 @@ struct AffineForEmptyLoopFolder : public OpRewritePattern<AffineForOp> {
     if (forOp.getNumResults() == 0)
       return success();
     std::optional<uint64_t> tripCount = getTrivialConstantTripCount(forOp);
-    if (tripCount && *tripCount == 0) {
+    if (tripCount == 0) {
       // The initial values of the iteration arguments would be the op's
       // results.
       rewriter.replaceOp(forOp, forOp.getInits());
@@ -2447,7 +2447,7 @@ void AffineForOp::getSuccessorRegions(
 
   // From the loop body, if the trip count is one, we can only branch back to
   // the parent.
-  if (!point.isParent() && tripCount && *tripCount == 1) {
+  if (!point.isParent() && tripCount == 1) {
     regions.push_back(RegionSuccessor(getResults()));
     return;
   }
@@ -2460,8 +2460,7 @@ void AffineForOp::getSuccessorRegions(
 
 /// Returns true if the affine.for has zero iterations in trivial cases.
 static bool hasTrivialZeroTripCount(AffineForOp op) {
-  std::optional<uint64_t> tripCount = getTrivialConstantTripCount(op);
-  return tripCount && *tripCount == 0;
+  return getTrivialConstantTripCount(op) == 0;
 }
 
 LogicalResult AffineForOp::fold(FoldAdaptor adaptor,
@@ -2667,7 +2666,7 @@ AffineParallelOp mlir::affine::getAffineParallelInductionVarOwner(Value val) {
   if (!ivArg || !ivArg.getOwner())
     return nullptr;
   Operation *containingOp = ivArg.getOwner()->getParentOp();
-  auto parallelOp = dyn_cast<AffineParallelOp>(containingOp);
+  auto parallelOp = dyn_cast_if_present<AffineParallelOp>(containingOp);
   if (parallelOp && llvm::is_contained(parallelOp.getIVs(), val))
     return parallelOp;
   return nullptr;
@@ -4789,7 +4788,7 @@ struct DropUnitExtentBasis
          llvm::enumerate(delinearizeOp.getPaddedBasis())) {
       std::optional<int64_t> basisVal =
           basis ? getConstantIntValue(basis) : std::nullopt;
-      if (basisVal && *basisVal == 1)
+      if (basisVal == 1)
         replacements[index] = getZero();
       else
         newBasis.push_back(basis);
diff --git a/external/llvm-project/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/external/llvm-project/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index 35205a6ca2ee..c0ef28c648ac 100644
--- a/external/llvm-project/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -84,126 +84,130 @@ static SmallVector<Value> computeStrides(Location loc, RewriterBase &rewriter,
   return result;
 }
 
+LogicalResult
+affine::lowerAffineDelinearizeIndexOp(RewriterBase &rewriter,
+                                      AffineDelinearizeIndexOp op) {
+  Location loc = op.getLoc();
+  Value linearIdx = op.getLinearIndex();
+  unsigned numResults = op.getNumResults();
+  ArrayRef<int64_t> staticBasis = op.getStaticBasis();
+  if (numResults == staticBasis.size())
+    staticBasis = staticBasis.drop_front();
+
+  if (numResults == 1) {
+    rewriter.replaceOp(op, linearIdx);
+    return success();
+  }
+
+  SmallVector<Value> results;
+  results.reserve(numResults);
+  SmallVector<Value> strides =
+      computeStrides(loc, rewriter, op.getDynamicBasis(), staticBasis,
+                     /*knownNonNegative=*/true);
+
+  Value zero = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
+
+  Value initialPart =
+      rewriter.create<arith::FloorDivSIOp>(loc, linearIdx, strides.front());
+  results.push_back(initialPart);
+
+  auto emitModTerm = [&](Value stride) -> Value {
+    Value remainder = rewriter.create<arith::RemSIOp>(loc, linearIdx, stride);
+    Value remainderNegative = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::slt, remainder, zero);
+    // If the correction is relevant, this term is <= stride, which is known
+    // to be positive in `index`. Otherwise, while 2 * stride might overflow,
+    // this branch won't be taken, so the risk of `poison` is fine.
+    Value corrected = rewriter.create<arith::AddIOp>(
+        loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
+    Value mod = rewriter.create<arith::SelectOp>(loc, remainderNegative,
+                                                 corrected, remainder);
+    return mod;
+  };
+
+  // Generate all the intermediate parts
+  for (size_t i = 0, e = strides.size() - 1; i < e; ++i) {
+    Value thisStride = strides[i];
+    Value nextStride = strides[i + 1];
+    Value modulus = emitModTerm(thisStride);
+    // We know both inputs are positive, so floorDiv == div.
+    // This could potentially be a divui, but it's not clear if that would
+    // cause issues.
+    Value divided = rewriter.create<arith::DivSIOp>(loc, modulus, nextStride);
+    results.push_back(divided);
+  }
+
+  results.push_back(emitModTerm(strides.back()));
+
+  rewriter.replaceOp(op, results);
+  return success();
+}
+
+LogicalResult affine::lowerAffineLinearizeIndexOp(RewriterBase &rewriter,
+                                                  AffineLinearizeIndexOp op) {
+  // Should be folded away, included here for safety.
+  if (op.getMultiIndex().empty()) {
+    rewriter.replaceOpWithNewOp<arith::ConstantIndexOp>(op, 0);
+    return success();
+  }
+
+  Location loc = op.getLoc();
+  ValueRange multiIndex = op.getMultiIndex();
+  size_t numIndexes = multiIndex.size();
+  ArrayRef<int64_t> staticBasis = op.getStaticBasis();
+  if (numIndexes == staticBasis.size())
+    staticBasis = staticBasis.drop_front();
+
+  SmallVector<Value> strides =
+      computeStrides(loc, rewriter, op.getDynamicBasis(), staticBasis,
+                     /*knownNonNegative=*/op.getDisjoint());
+  SmallVector<std::pair<Value, int64_t>> scaledValues;
+  scaledValues.reserve(numIndexes);
+
+  // Note: strides doesn't contain a value for the final element (stride 1)
+  // and everything else lines up. We use the "mutable" accessor so we can get
+  // our hands on an `OpOperand&` for the loop invariant counting function.
+  for (auto [stride, idxOp] :
+       llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
+    Value scaledIdx = rewriter.create<arith::MulIOp>(
+        loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
+    int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
+    scaledValues.emplace_back(scaledIdx, numHoistableLoops);
+  }
+  scaledValues.emplace_back(
+      multiIndex.back(),
+      numEnclosingInvariantLoops(op.getMultiIndexMutable()[numIndexes - 1]));
+
+  // Sort by how many enclosing loops there are, ties implicitly broken by
+  // size of the stride.
+  llvm::stable_sort(scaledValues,
+                    [&](auto l, auto r) { return l.second > r.second; });
+
+  Value result = scaledValues.front().first;
+  for (auto [scaledValue, numHoistableLoops] : llvm::drop_begin(scaledValues)) {
+    std::ignore = numHoistableLoops;
+    result = rewriter.create<arith::AddIOp>(loc, result, scaledValue,
+                                            arith::IntegerOverflowFlags::nsw);
+  }
+  rewriter.replaceOp(op, result);
+  return success();
+}
+
 namespace {
-/// Lowers `affine.delinearize_index` into a sequence of division and remainder
-/// operations.
 struct LowerDelinearizeIndexOps
     : public OpRewritePattern<AffineDelinearizeIndexOp> {
   using OpRewritePattern<AffineDelinearizeIndexOp>::OpRewritePattern;
   LogicalResult matchAndRewrite(AffineDelinearizeIndexOp op,
                                 PatternRewriter &rewriter) const override {
-    Location loc = op.getLoc();
-    Value linearIdx = op.getLinearIndex();
-    unsigned numResults = op.getNumResults();
-    ArrayRef<int64_t> staticBasis = op.getStaticBasis();
-    if (numResults == staticBasis.size())
-      staticBasis = staticBasis.drop_front();
-
-    if (numResults == 1) {
-      rewriter.replaceOp(op, linearIdx);
-      return success();
-    }
-
-    SmallVector<Value> results;
-    results.reserve(numResults);
-    SmallVector<Value> strides =
-        computeStrides(loc, rewriter, op.getDynamicBasis(), staticBasis,
-                       /*knownNonNegative=*/true);
-
-    Value zero = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
-
-    Value initialPart =
-        rewriter.create<arith::FloorDivSIOp>(loc, linearIdx, strides.front());
-    results.push_back(initialPart);
-
-    auto emitModTerm = [&](Value stride) -> Value {
-      Value remainder = rewriter.create<arith::RemSIOp>(loc, linearIdx, stride);
-      Value remainderNegative = rewriter.create<arith::CmpIOp>(
-          loc, arith::CmpIPredicate::slt, remainder, zero);
-      // If the correction is relevant, this term is <= stride, which is known
-      // to be positive in `index`. Otherwise, while 2 * stride might overflow,
-      // this branch won't be taken, so the risk of `poison` is fine.
-      Value corrected = rewriter.create<arith::AddIOp>(
-          loc, remainder, stride, arith::IntegerOverflowFlags::nsw);
-      Value mod = rewriter.create<arith::SelectOp>(loc, remainderNegative,
-                                                   corrected, remainder);
-      return mod;
-    };
-
-    // Generate all the intermediate parts
-    for (size_t i = 0, e = strides.size() - 1; i < e; ++i) {
-      Value thisStride = strides[i];
-      Value nextStride = strides[i + 1];
-      Value modulus = emitModTerm(thisStride);
-      // We know both inputs are positive, so floorDiv == div.
-      // This could potentially be a divui, but it's not clear if that would
-      // cause issues.
-      Value divided = rewriter.create<arith::DivSIOp>(loc, modulus, nextStride);
-      results.push_back(divided);
-    }
-
-    results.push_back(emitModTerm(strides.back()));
-
-    rewriter.replaceOp(op, results);
-    return success();
+    return affine::lowerAffineDelinearizeIndexOp(rewriter, op);
   }
 };
 
-/// Lowers `affine.linearize_index` into a sequence of multiplications and
-/// additions. Make a best effort to sort the input indices so that
-/// the most loop-invariant terms are at the left of the additions
-/// to enable loop-invariant code motion.
 struct LowerLinearizeIndexOps final : OpRewritePattern<AffineLinearizeIndexOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(AffineLinearizeIndexOp op,
                                 PatternRewriter &rewriter) const override {
-    // Should be folded away, included here for safety.
-    if (op.getMultiIndex().empty()) {
-      rewriter.replaceOpWithNewOp<arith::ConstantIndexOp>(op, 0);
-      return success();
-    }
-
-    Location loc = op.getLoc();
-    ValueRange multiIndex = op.getMultiIndex();
-    size_t numIndexes = multiIndex.size();
-    ArrayRef<int64_t> staticBasis = op.getStaticBasis();
-    if (numIndexes == staticBasis.size())
-      staticBasis = staticBasis.drop_front();
-
-    SmallVector<Value> strides =
-        computeStrides(loc, rewriter, op.getDynamicBasis(), staticBasis,
-                       /*knownNonNegative=*/op.getDisjoint());
-    SmallVector<std::pair<Value, int64_t>> scaledValues;
-    scaledValues.reserve(numIndexes);
-
-    // Note: strides doesn't contain a value for the final element (stride 1)
-    // and everything else lines up. We use the "mutable" accessor so we can get
-    // our hands on an `OpOperand&` for the loop invariant counting function.
-    for (auto [stride, idxOp] :
-         llvm::zip_equal(strides, llvm::drop_end(op.getMultiIndexMutable()))) {
-      Value scaledIdx = rewriter.create<arith::MulIOp>(
-          loc, idxOp.get(), stride, arith::IntegerOverflowFlags::nsw);
-      int64_t numHoistableLoops = numEnclosingInvariantLoops(idxOp);
-      scaledValues.emplace_back(scaledIdx, numHoistableLoops);
-    }
-    scaledValues.emplace_back(
-        multiIndex.back(),
-        numEnclosingInvariantLoops(op.getMultiIndexMutable()[numIndexes - 1]));
-
-    // Sort by how many enclosing loops there are, ties implicitly broken by
-    // size of the stride.
-    llvm::stable_sort(scaledValues,
-                      [&](auto l, auto r) { return l.second > r.second; });
-
-    Value result = scaledValues.front().first;
-    for (auto [scaledValue, numHoistableLoops] :
-         llvm::drop_begin(scaledValues)) {
-      std::ignore = numHoistableLoops;
-      result = rewriter.create<arith::AddIOp>(loc, result, scaledValue,
-                                              arith::IntegerOverflowFlags::nsw);
-    }
-    rewriter.replaceOp(op, result);
-    return success();
+    return affine::lowerAffineLinearizeIndexOp(rewriter, op);
   }
 };
 
diff --git a/external/llvm-project/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/external/llvm-project/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 0d4ba3940c48..4aa1fe318efa 100644
--- a/external/llvm-project/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -1015,8 +1015,7 @@ LogicalResult mlir::affine::loopUnrollByFactor(
 
   std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
   if (unrollFactor == 1) {
-    if (mayBeConstantTripCount && *mayBeConstantTripCount == 1 &&
-        failed(promoteIfSingleIteration(forOp)))
+    if (mayBeConstantTripCount == 1 && failed(promoteIfSingleIteration(forOp)))
       return failure();
     return success();
   }
@@ -1103,8 +1102,7 @@ LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
 
   std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
   if (unrollJamFactor == 1) {
-    if (mayBeConstantTripCount && *mayBeConstantTripCount == 1 &&
-        failed(promoteIfSingleIteration(forOp)))
+    if (mayBeConstantTripCount == 1 && failed(promoteIfSingleIteration(forOp)))
       return failure();
     return success();
   }
diff --git a/external/llvm-project/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/external/llvm-project/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 41f2d0f3425e..9e53e195274a 100644
--- a/external/llvm-project/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -1451,6 +1451,19 @@ bool arith::ExtFOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
 
 LogicalResult arith::ExtFOp::verify() { return verifyExtOp<FloatType>(*this); }
 
+//===----------------------------------------------------------------------===//
+// ScalingExtFOp
+//===----------------------------------------------------------------------===//
+
+bool arith::ScalingExtFOp::areCastCompatible(TypeRange inputs,
+                                             TypeRange outputs) {
+  return checkWidthChangeCast<std::greater, FloatType>(inputs.front(), outputs);
+}
+
+LogicalResult arith::ScalingExtFOp::verify() {
+  return verifyExtOp<FloatType>(*this);
+}
+
 //===----------------------------------------------------------------------===//
 // TruncIOp
 //===----------------------------------------------------------------------===//
@@ -1565,6 +1578,19 @@ LogicalResult arith::TruncFOp::verify() {
   return verifyTruncateOp<FloatType>(*this);
 }
 
+//===----------------------------------------------------------------------===//
+// ScalingTruncFOp
+//===----------------------------------------------------------------------===//
+
+bool arith::ScalingTruncFOp::areCastCompatible(TypeRange inputs,
+                                               TypeRange outputs) {
+  return checkWidthChangeCast<std::less, FloatType>(inputs.front(), outputs);
+}
+
+LogicalResult arith::ScalingTruncFOp::verify() {
+  return verifyTruncateOp<FloatType>(*this);
+}
+
 //===----------------------------------------------------------------------===//
 // AndIOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/external/llvm-project/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index 95546bb09e76..534aff9562b7 100644
--- a/external/llvm-project/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Arith/Transforms/Passes.h"
-
 #include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Arith/Transforms/Passes.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/BuiltinTypeInterfaces.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/DialectConversion.h"
@@ -31,7 +31,6 @@ static Value createConst(Location loc, Type type, int value,
     return rewriter.create<arith::ConstantOp>(
         loc, DenseElementsAttr::get(shapedTy, attr));
   }
-
   return rewriter.create<arith::ConstantOp>(loc, attr);
 }
 
@@ -357,9 +356,10 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
     f32Bits = b.create<arith::SelectOp>(isNan, cF32NaN, f32Bits);
     Value result = b.create<arith::BitcastOp>(f32Ty, f32Bits);
     if (resultETy.getIntOrFloatBitWidth() < 32) {
-      result = b.create<arith::TruncFOp>(resultTy, result);
+      result = b.create<arith::TruncFOp>(resultTy, result, nullptr,
+                                         op.getFastmathAttr());
     } else if (resultETy.getIntOrFloatBitWidth() > 32) {
-      result = b.create<arith::ExtFOp>(resultTy, result);
+      result = b.create<arith::ExtFOp>(resultTy, result, op.getFastmathAttr());
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -395,9 +395,10 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
     Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
 
     if (operandETy.getIntOrFloatBitWidth() < 32) {
-      operand = b.create<arith::ExtFOp>(f32Ty, operand);
+      operand = b.create<arith::ExtFOp>(f32Ty, operand, op.getFastmathAttr());
     } else if (operandETy.getIntOrFloatBitWidth() > 32) {
-      operand = b.create<arith::TruncFOp>(f32Ty, operand);
+      operand = b.create<arith::TruncFOp>(
+          f32Ty, operand, op.getRoundingmodeAttr(), op.getFastmathAttr());
     }
     Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operand);
     Value cF32MantissaWidth = createConst(op->getLoc(), i32Ty, 23, rewriter);
@@ -409,6 +410,83 @@ struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
   }
 };
 
+struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(arith::ScalingExtFOp op,
+                                PatternRewriter &rewriter) const final {
+    ImplicitLocOpBuilder b(op.getLoc(), rewriter);
+    Value inputOperand = op.getIn();
+    Value scaleOperand = op.getScale();
+    Type scaleTy = scaleOperand.getType();
+    Type scaleETy = getElementTypeOrSelf(scaleOperand);
+    // allow implicit exponent extraction from 16/32 bits floats
+    if (scaleETy.getIntOrFloatBitWidth() >= 16) {
+      scaleETy = b.getF8E8M0Type();
+      scaleTy = cloneToShapedType(scaleTy, scaleETy);
+      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+                                               op.getFastmathAttr());
+    }
+    if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
+      return rewriter.notifyMatchFailure(
+          op, "scaling_extf is using scales of type which can not be converted "
+              "to f8E8M0FNU");
+    }
+    Type resultTy = op.getType();
+    // extf on scale will essentially create floating point number
+    // of type resulTy that is 2^scale and will also propagate NaNs
+    Value scaleExt =
+        b.create<arith::ExtFOp>(resultTy, scaleOperand, op.getFastmathAttr());
+    Value inputExt =
+        b.create<arith::ExtFOp>(resultTy, inputOperand, op.getFastmathAttr());
+    Value result =
+        b.create<arith::MulFOp>(inputExt, scaleExt, op.getFastmathAttr());
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
+/*
+Expands arith.ScalingTruncFOp(in, scale) into
+  scale = arith.truncf(scale) : scaleTy -> f8E8M0FNU
+  result = arith.truncf(in / (2^scale))
+ */
+struct ScalingTruncFOpConverter
+    : public OpRewritePattern<arith::ScalingTruncFOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(arith::ScalingTruncFOp op,
+                                PatternRewriter &rewriter) const final {
+    ImplicitLocOpBuilder b(op.getLoc(), rewriter);
+    Value inputOperand = op.getIn();
+    Value scaleOperand = op.getScale();
+    Type scaleTy = scaleOperand.getType();
+    Type scaleETy = getElementTypeOrSelf(scaleOperand);
+    // allow implicit exponent extraction from 16/32 bits floats
+    if (scaleETy.getIntOrFloatBitWidth() >= 16) {
+      scaleETy = b.getF8E8M0Type();
+      scaleTy = cloneToShapedType(scaleTy, scaleETy);
+      scaleOperand = b.create<arith::TruncFOp>(scaleTy, scaleOperand, nullptr,
+                                               op.getFastmathAttr());
+    }
+    if (!llvm::isa<Float8E8M0FNUType>(scaleETy)) {
+      return rewriter.notifyMatchFailure(
+          op, "scaling_truncf is using scales type which can not be converted "
+              "to f8E8M0FNU");
+    }
+    Type resultTy = op.getType();
+    Type inputTy = inputOperand.getType();
+    // this will create a floating point number of type
+    // inputTy that is 2^scale and will also propagate NaNs
+    scaleOperand =
+        b.create<arith::ExtFOp>(inputTy, scaleOperand, op.getFastmathAttr());
+    Value result = b.create<arith::DivFOp>(inputOperand, scaleOperand,
+                                           op.getFastmathAttr());
+    Value resultCast = b.create<arith::TruncFOp>(
+        resultTy, result, op.getRoundingmodeAttr(), op.getFastmathAttr());
+    rewriter.replaceOp(op, resultCast);
+    return success();
+  }
+};
+
 struct ArithExpandOpsPass
     : public arith::impl::ArithExpandOpsPassBase<ArithExpandOpsPass> {
   using ArithExpandOpsPassBase::ArithExpandOpsPassBase;
@@ -432,7 +510,9 @@ struct ArithExpandOpsPass
       arith::MaximumFOp,
       arith::MinimumFOp,
       arith::MaxNumFOp,
-      arith::MinNumFOp
+      arith::MinNumFOp,
+      arith::ScalingExtFOp,
+      arith::ScalingTruncFOp
     >();
 
     if (includeBf16) {
@@ -492,8 +572,15 @@ void mlir::arith::populateExpandF8E8M0Patterns(RewritePatternSet &patterns) {
       patterns.getContext());
 }
 
+void mlir::arith::populateExpandScalingExtTruncPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<ScalingExtFOpConverter, ScalingTruncFOpConverter>(
+      patterns.getContext());
+}
+
 void mlir::arith::populateArithExpandOpsPatterns(RewritePatternSet &patterns) {
   populateCeilFloorDivExpandOpsPatterns(patterns);
+  populateExpandScalingExtTruncPatterns(patterns);
   // clang-format off
   patterns.add<
     MaxMinIOpConverter<MaxSIOp, arith::CmpIPredicate::sgt>,
@@ -503,7 +590,7 @@ void mlir::arith::populateArithExpandOpsPatterns(RewritePatternSet &patterns) {
     MaximumMinimumFOpConverter<MaximumFOp, arith::CmpFPredicate::UGT>,
     MaximumMinimumFOpConverter<MinimumFOp, arith::CmpFPredicate::ULT>,
     MaxNumMinNumFOpConverter<MaxNumFOp, arith::CmpFPredicate::UGT>,
-    MaxNumMinNumFOpConverter<MinNumFOp, arith::CmpFPredicate::ULT>
+    MaxNumMinNumFOpConverter<MinNumFOp, arith::CmpFPredicate::ULT> 
    >(patterns.getContext());
   // clang-format on
 }
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmNeon/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
index 9f57627c321f..cb1e9d01821a 100644
--- a/external/llvm-project/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(IR)
 add_subdirectory(Transforms)
+add_subdirectory(TransformOps)
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp b/external/llvm-project/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
new file mode 100644
index 000000000000..e81fc6a8b598
--- /dev/null
+++ b/external/llvm-project/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
@@ -0,0 +1,54 @@
+//===- ArmNeonVectorTransformOps.cpp - Implementation transform ops -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.h"
+
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/ArmNeon/Transforms.h"
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+
+using namespace mlir;
+
+//===----------------------------------------------------------------------===//
+// Apply...PatternsOp
+//===----------------------------------------------------------------------===//
+
+void transform::ApplyArmNeonContractionToI8MMPatternsOp::populatePatterns(
+    RewritePatternSet &patterns) {
+  arm_neon::populateLowerContractionToSMMLAPatternPatterns(patterns);
+}
+
+//===----------------------------------------------------------------------===//
+// Transform op registration
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ArmNeonVectorTransformDialectExtension
+    : public transform::TransformDialectExtension<
+          ArmNeonVectorTransformDialectExtension> {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
+      ArmNeonVectorTransformDialectExtension)
+
+  ArmNeonVectorTransformDialectExtension() {
+    declareGeneratedDialect<arm_neon::ArmNeonDialect>();
+    registerTransformOps<
+#define GET_OP_LIST
+#include "mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp.inc"
+        >();
+  }
+};
+} // namespace
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp.inc"
+
+void mlir::arm_neon::registerTransformDialectExtension(
+    DialectRegistry &registry) {
+  registry.addExtensions<ArmNeonVectorTransformDialectExtension>();
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmNeon/TransformOps/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/ArmNeon/TransformOps/CMakeLists.txt
new file mode 100644
index 000000000000..69d2143ad4e1
--- /dev/null
+++ b/external/llvm-project/mlir/lib/Dialect/ArmNeon/TransformOps/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_dialect_library(MLIRArmNeonVectorTransformOps
+  ArmNeonVectorTransformOps.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/ArmNeon/TransformOps
+
+  DEPENDS
+  MLIRArmNeonVectorTransformOpsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRLLVMCommonConversion
+  MLIRLLVMDialect
+  MLIRVectorDialect
+  MLIRTransformDialect
+  MLIRArmNeonDialect
+  MLIRArmNeonTransforms
+  )
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp b/external/llvm-project/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
index 2a1271dfd6bd..5ce3d2b28aeb 100644
--- a/external/llvm-project/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
@@ -56,6 +56,10 @@ class LowerContractionToSMMLAPattern
     // Avoid 0-D vectors and 1-D rhs:
     if (!lhsType.hasRank() || !rhsType.hasRank() || rhsType.getRank() < 2)
       return failure();
+    // This codegen does not work for scalable vectors. Return failure so this
+    // pattern is not accidentally chosen over patterns that lower to ArmSVE.
+    if (lhsType.isScalable() || rhsType.isScalable())
+      return failure();
     auto dimM = lhsType.getRank() == 1 ? 1 : lhsType.getDimSize(0);
     auto dimN = rhsType.getDimSize(0);
     auto dimK = rhsType.getDimSize(1);
@@ -238,5 +242,5 @@ class LowerContractionToSMMLAPattern
 void mlir::arm_neon::populateLowerContractionToSMMLAPatternPatterns(
     RewritePatternSet &patterns) {
   MLIRContext *context = patterns.getContext();
-  patterns.add<LowerContractionToSMMLAPattern>(context, /*benefit=*/1);
+  patterns.add<LowerContractionToSMMLAPattern>(context, /*benefit=*/2);
 }
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/external/llvm-project/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index 72a05ffe97ac..e6c9adba62f3 100644
--- a/external/llvm-project/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -528,8 +528,7 @@ chooseSpillUsingHeuristics(OverlappingRangesIterator overlappingRanges,
            a.end() < b.end();
   };
   LiveRange &latestEndingLiveRange =
-      *std::max_element(overlappingRanges.begin(), overlappingRanges.end(),
-                        isSmallerTileTypeOrEndsEarlier);
+      *llvm::max_element(overlappingRanges, isSmallerTileTypeOrEndsEarlier);
   if (!isSmallerTileTypeOrEndsEarlier(latestEndingLiveRange, *newRange))
     return &latestEndingLiveRange;
   return newRange;
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmSVE/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/ArmSVE/CMakeLists.txt
index 9f57627c321f..cb1e9d01821a 100644
--- a/external/llvm-project/mlir/lib/Dialect/ArmSVE/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/ArmSVE/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(IR)
 add_subdirectory(Transforms)
+add_subdirectory(TransformOps)
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp b/external/llvm-project/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
new file mode 100644
index 000000000000..b2ca4fc1eaa8
--- /dev/null
+++ b/external/llvm-project/mlir/lib/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp
@@ -0,0 +1,54 @@
+//===- ArmSVEVectorTransformOps.cpp - Implementation transform ops -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.h"
+
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
+#include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+
+using namespace mlir;
+
+//===----------------------------------------------------------------------===//
+// Apply...PatternsOp
+//===----------------------------------------------------------------------===//
+
+void transform::ApplyArmSVELowerContractionPatternsOp::populatePatterns(
+    RewritePatternSet &patterns) {
+  mlir::populateLowerContractionToSVEI8MMPatternPatterns(patterns);
+}
+
+//===----------------------------------------------------------------------===//
+// Transform op registration
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ArmSVEVectorTransformDialectExtension
+    : public transform::TransformDialectExtension<
+          ArmSVEVectorTransformDialectExtension> {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
+      ArmSVEVectorTransformDialectExtension)
+
+  ArmSVEVectorTransformDialectExtension() {
+    declareGeneratedDialect<arm_sve::ArmSVEDialect>();
+    registerTransformOps<
+#define GET_OP_LIST
+#include "mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp.inc"
+        >();
+  }
+};
+} // namespace
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp.inc"
+
+void mlir::arm_sve::registerTransformDialectExtension(
+    DialectRegistry &registry) {
+  registry.addExtensions<ArmSVEVectorTransformDialectExtension>();
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmSVE/TransformOps/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/ArmSVE/TransformOps/CMakeLists.txt
new file mode 100644
index 000000000000..235fe8f5ca0c
--- /dev/null
+++ b/external/llvm-project/mlir/lib/Dialect/ArmSVE/TransformOps/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_mlir_dialect_library(MLIRArmSVEVectorTransformOps
+  ArmSVEVectorTransformOps.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/ArmSVE/TransformOps
+
+  DEPENDS
+  MLIRArmSVEVectorTransformOpsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRLLVMCommonConversion
+  MLIRLLVMDialect
+  MLIRVectorDialect
+  MLIRTransformDialect
+  MLIRArmSVEDialect
+  MLIRArmSVETransforms
+  )
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt
index a70c489a51fe..65f98b44b1b6 100644
--- a/external/llvm-project/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_mlir_dialect_library(MLIRArmSVETransforms
   LegalizeForLLVMExport.cpp
   LegalizeVectorStorage.cpp
+  LowerContractionToSVEI8MMPattern.cpp
 
   DEPENDS
   MLIRArmSVEConversionsIncGen
diff --git a/external/llvm-project/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/external/llvm-project/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
new file mode 100644
index 000000000000..b1233c5c06eb
--- /dev/null
+++ b/external/llvm-project/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -0,0 +1,365 @@
+//===- LowerContractionToSMMLAPattern.cpp - Contract to SMMLA ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements lowering patterns from vector.contract to operations
+// that map to instructions from the SVE FEAT_I8MM extension.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
+#include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/AffineMap.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#include "mlir/Dialect/UB/IR/UBOps.h"
+
+#define DEBUG_TYPE "lower-contract-to-arm-sve-i8mm"
+
+using namespace mlir;
+
+namespace {
+// Get the operand of a `vector.contract`. This function is intended to abstract
+// away from the particular way a value is extended before feeding it into the
+// `vector.contract` - via zero-extend or an explicit or implicit sign-extend
+// (for implicit sign-extension see `vector.contract` documentation).
+//
+// The template parameter `Op` indicates the extension operation (explicit or
+// implicit) for which we are checking.
+//
+// Return success only for extensions from `i8` to `i32`.
+template <typename Op>
+std::optional<Value> getExtOperand(Value v, Type i8Ty, Type i32Ty) {
+
+  static_assert(llvm::is_one_of<Op, arith::ExtSIOp, arith::ExtUIOp>::value,
+                "Must be instantiated with either sign- or zero- extension op");
+
+  // If the operand is not defined by an explicit extend operation of the
+  // accepted operation type allow for an implicit sign-extension.
+  auto extOp = dyn_cast_or_null<Op>(v.getDefiningOp());
+  if (!extOp) {
+    if constexpr (std::is_same<Op, arith::ExtSIOp>::value) {
+      auto vTy = cast<VectorType>(v.getType());
+      if (vTy.getElementType() != i8Ty)
+        return {};
+      return v;
+    }
+    return {};
+  }
+
+  // If the operand is defined by an explicit extend operation of the accepted
+  // operation type, check it's extended from `i8` to `i32`.
+  auto inOp = extOp.getIn();
+  auto inTy = dyn_cast<VectorType>(inOp.getType());
+  if (!inTy || inTy.getElementType() != i8Ty)
+    return {};
+
+  auto outTy = dyn_cast<VectorType>(extOp.getType());
+  if (!outTy || outTy.getElementType() != i32Ty)
+    return {};
+
+  return inOp;
+}
+
+// Designate the operation (resp. instruction) used to do sub-tile matrix
+// multiplications.
+enum class MMLA {
+  Signed,      // smmla
+  Unsigned,    // ummla
+  Mixed,       // usmmla
+  MixedSwapped // usmmla with LHS and RHS swapped
+};
+
+// Create the matrix mulitply and accumulate operation according to `op`.
+Value createMMLA(PatternRewriter &rewriter, MMLA op, Location loc,
+                 mlir::VectorType accType, Value acc, Value lhs, Value rhs) {
+  switch (op) {
+  case MMLA::Signed:
+    return rewriter.create<arm_sve::SmmlaOp>(loc, accType, acc, lhs, rhs);
+  case MMLA::Unsigned:
+    return rewriter.create<arm_sve::UmmlaOp>(loc, accType, acc, lhs, rhs);
+  case MMLA::Mixed:
+    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, lhs, rhs);
+  case MMLA::MixedSwapped:
+    // The accumulator comes transposed and the result will be transposed
+    // later, so all we have to do here is swap the operands.
+    return rewriter.create<arm_sve::UsmmlaOp>(loc, accType, acc, rhs, lhs);
+  }
+}
+
+/// Lower a contraction operation that performs a matrix multiplication
+/// of two 8-bit integer matrix tiles with logical dimensions <Mx8> and <8x[N]>
+/// for the left-hand side and the right-hand side, respectively,
+/// yielding a <Mx[N]> 32-bit integer result.
+///
+/// The operands' shapes are such that the operands can be evenly split into
+/// sub-tiles with dimensions as expected by the targeted FEAT_I8MM
+/// instructions. The intent is that M and N are chosen (by higher level
+/// transforms) in such a way as to maximise register usage. The main use case
+/// we envision as of now is MMT4D, thus the RHS operand is expected
+/// pre-transposed.
+///
+/// The matrix multiplication is performed by unrolling the usual tiled matrix
+/// multiplication algorithm using sub-tiles with dimensions <2x8> for the LHS,
+/// <8x[2]> for the RHS, and <2x[2]> for the result and the input accumulator.
+///
+/// One way to illustrate the operation is as follows:
+///
+/// RHS<8x[N]>:       <8x[2]> <8x[2]> ... <8x[2]>
+///                 +-----------------------------
+/// LHS<Mx8>: <2x8> | <2x[2]> <2x[2]> ... <2x[2]>
+///           <2x8> | <2x[2]> <2x[2]> ... <2x[2]>
+///            ...  |   ...     ...   ...   ...
+///           <2x8> | <2x[2]> <2x[2]> ... <2x[2]>
+///
+/// The RHS operand is unpacked into N/2 values, each representing a sequence of
+/// VSCALE number of sub-tiles with dimensions <8x2>.
+/// The LHS operand is initially unpacked into M/2 values, each representing a
+/// sub-tile with dimensions <2x8>, and then each such sub-tile is replicated
+/// VSCALE times.
+/// Multiplying thus replicated LHS sub-tile by the corresponding RHS sub-tile
+/// correctly computes an entire result sub-tile.
+class LowerContractionToSVEI8MMPattern
+    : public OpRewritePattern<vector::ContractionOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(vector::ContractionOp op,
+                                PatternRewriter &rewriter) const override {
+
+    Location loc = op.getLoc();
+    mlir::VectorType lhsType = op.getLhsType();
+    mlir::VectorType rhsType = op.getRhsType();
+
+    // Check the rank the types so we can safely examine their dimensions.
+    if (lhsType.getRank() != 2 || rhsType.getRank() != 2)
+      return rewriter.notifyMatchFailure(op, "non-matching operand shape");
+
+    auto M = lhsType.getDimSize(0);
+    auto N = rhsType.getDimSize(0);
+    auto K = rhsType.getDimSize(1);
+
+    // Check the operands have the expected shape:
+    //  * for LHS: fixed vector MxK
+    //  * for RHS: scalable vector [N]xK
+    //  * K == 8
+    //  * M and N even and at least 2
+    if (lhsType.isScalable() || !rhsType.getScalableDims()[0] ||
+        rhsType.getScalableDims()[1] || lhsType.getDimSize(1) != K || K != 8 ||
+        M < 2 || M % 2 != 0 || N < 2 || N % 2 != 0 ||
+        !rhsType.getScalableDims()[0])
+      return rewriter.notifyMatchFailure(op, "non-matching operand shape");
+
+    // Check permutation maps. For now only accept
+    //   lhs: (d0, d1, d2) -> (d0, d2)
+    //   rhs: (d0, d1, d2) -> (d1, d2)
+    //   acc: (d0, d1, d2) -> (d0, d1)
+    // This corresponds to matrix multiplication with transposed RHS.
+    if (op.getIndexingMapsArray()[0] !=
+            AffineMap::getMultiDimMapWithTargets(3, ArrayRef{0u, 2u},
+                                                 op.getContext()) ||
+        op.getIndexingMapsArray()[1] !=
+            AffineMap::getMultiDimMapWithTargets(3, ArrayRef{1u, 2u},
+                                                 op.getContext()) ||
+        op.getIndexingMapsArray()[2] !=
+            AffineMap::getMultiDimMapWithTargets(3, ArrayRef{0u, 1u},
+                                                 op.getContext()))
+      return rewriter.notifyMatchFailure(op, "non-matching permutation maps");
+
+    // Check iterator types for matrix multiplication.
+    auto itTypes = op.getIteratorTypesArray();
+    if (itTypes.size() != 3 || itTypes[0] != vector::IteratorType::parallel ||
+        itTypes[1] != vector::IteratorType::parallel ||
+        itTypes[2] != vector::IteratorType::reduction)
+      return rewriter.notifyMatchFailure(
+          op, "iterator types do not correspond to matrix multiplication");
+
+    // Check the combining kind is addition.
+    if (op.getKind() != vector::CombiningKind::ADD)
+      return rewriter.notifyMatchFailure(op,
+                                         "combining kind is not an addition");
+
+    // Check the output is a vector of i32 elements.
+    auto outTy = dyn_cast<VectorType>(op.getResultType());
+    if (!outTy || outTy.getElementType() != rewriter.getI32Type())
+      return rewriter.notifyMatchFailure(op,
+                                         "output type is not a vector of i32");
+
+    // Check inputs are sign-/zero- extensions from i8 to i32. Get the values
+    // before the extension. All four signed/unsigned combinations for input
+    // operands are supported, but they are lowered to different operations.
+    // Determine which is the appropriate operation to lower to.
+    MMLA mmlaOp = MMLA::Signed;
+    auto maybeLhs = getExtOperand<arith::ExtSIOp>(
+        op.getLhs(), rewriter.getI8Type(), rewriter.getI32Type());
+    if (!maybeLhs) {
+      mmlaOp = MMLA::Unsigned;
+      maybeLhs = getExtOperand<arith::ExtUIOp>(
+          op.getLhs(), rewriter.getI8Type(), rewriter.getI32Type());
+    }
+    if (!maybeLhs)
+      return rewriter.notifyMatchFailure(
+          op, "LHS is not a sign- or zero- extended i8");
+
+    auto maybeRhs = getExtOperand<arith::ExtSIOp>(
+        op.getRhs(), rewriter.getI8Type(), rewriter.getI32Type());
+    if (maybeRhs) {
+      if (mmlaOp == MMLA::Unsigned)
+        mmlaOp = MMLA::Mixed;
+    } else {
+      if (mmlaOp == MMLA::Signed)
+        mmlaOp = MMLA::MixedSwapped;
+      maybeRhs = getExtOperand<arith::ExtUIOp>(
+          op.getRhs(), rewriter.getI8Type(), rewriter.getI32Type());
+    }
+    if (!maybeRhs)
+      return rewriter.notifyMatchFailure(
+          op, "RHS is not a sign- or zero- extended i8");
+
+    // One-dimensional vector types for arm_sve.*mmla
+    auto nxv16i8 = VectorType::get(/*shape=*/16, rewriter.getI8Type(),
+                                   /*scalableDims=*/{true});
+    auto nxv4i32 = VectorType::get(/*shape=*/4, rewriter.getI32Type(),
+                                   /*scalableDims=*/{true});
+
+    // Extract LHS sub-tiles with logicall shape <2x8>.
+    SmallVector<Value> lhsTile;
+    for (int64_t i = 0; i < M; i += 2) {
+      // Extract two consecutive rows of the LHS tile.
+      auto r0 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+                                                   ArrayRef<int64_t>{i});
+      auto r1 = rewriter.create<vector::ExtractOp>(loc, *maybeLhs,
+                                                   ArrayRef<int64_t>{i + 1});
+      // Concatenate to obtain a 16 x i8 flattened sub-tile.
+      auto t = rewriter.create<vector::ShuffleOp>(
+          loc, r0, r1,
+          llvm::ArrayRef<int64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
+                                  14, 15});
+      // Turn it into a scalable vector.
+      auto s = rewriter.create<vector::ScalableInsertOp>(
+          loc, t, rewriter.create<ub::PoisonOp>(loc, nxv16i8), 0);
+      // Replicate the sub-tile VSCALE times to fill the entire vector.
+      auto r = rewriter.create<arm_sve::DupQLaneOp>(loc, s, 0);
+      lhsTile.push_back(r);
+    }
+
+    // "Flatten" the RHS tile from <[N]x8> to <[8*N]>.
+    auto rhs = rewriter.create<vector::ShapeCastOp>(
+        maybeRhs->getLoc(),
+        VectorType::get(/*shape=*/8 * N, rewriter.getI8Type(),
+                        /*scalableDims=*/{true}),
+        *maybeRhs);
+
+    // Extract the RHS sub-tiles with logical shape <8x[2]>.
+    SmallVector<Value> rhsTile;
+    for (int64_t j = 0; j < N; j += 2)
+      rhsTile.push_back(
+          rewriter.create<vector::ScalableExtractOp>(loc, nxv16i8, rhs, j * 8));
+
+    // Handy types for packing/unpacking of the accumulator tile.
+    auto accRowTy = VectorType::get(/*shape=*/N, rewriter.getI32Type(),
+                                    /*scalableDims=*/{true});
+    auto accRowX2Ty = VectorType::get(/*shape=*/2 * N, rewriter.getI32Type(),
+                                      /*scalableDims=*/{true});
+    auto accRow64Ty = VectorType::get(/*shape=*/N / 2, rewriter.getI64Type(),
+                                      /*scalableDims=*/{true});
+    auto accRowX264Ty = VectorType::get(/*shape=*/N, rewriter.getI64Type(),
+                                        /*scalableDims=*/{true});
+
+    // Extract and pack the ACC sub-tiles.
+    SmallVector<Value> accTile;
+    for (int64_t i = 0; i < M; i += 2) {
+      // Extract two consecutive rows of the accumulator tile.
+      auto r0 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+                                                   ArrayRef<int64_t>{i});
+      auto r1 = rewriter.create<vector::ExtractOp>(loc, op.getAcc(),
+                                                   ArrayRef<int64_t>{i + 1});
+      Value accTileVec;
+      if (mmlaOp == MMLA::MixedSwapped) {
+        // We need to swap the positions of the LHS and RHS (since we don't have
+        // a signed * unsigned operation), but then each individual 2x2 tile of
+        // the acumulator and (later) the result need to be transposed.
+        accTileVec = rewriter.create<vector::InterleaveOp>(loc, r0, r1);
+      } else {
+        // Bitcast them to 64-bit elements, so subsequent
+        // interleave/deinterleave work on pairs of 32-bit numbers.
+        auto r0I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r0);
+        auto r1I64 = rewriter.create<vector::BitCastOp>(loc, accRow64Ty, r1);
+
+        // Interleave the rows, effectively flattening each 2x2 tile into 4
+        // consecutive elements.
+        auto intrI64 = rewriter.create<vector::InterleaveOp>(loc, r0I64, r1I64);
+
+        // Bitcast back to 32-bit elements.
+        accTileVec =
+            rewriter.create<vector::BitCastOp>(loc, accRowX2Ty, intrI64);
+      }
+      // Extract ACC sub-tiles.
+      for (int64_t j = 0; j < N; j += 2)
+        accTile.push_back(rewriter.create<vector::ScalableExtractOp>(
+            loc, nxv4i32, accTileVec, j * 2));
+    }
+
+    // Emit sub-tile matrix multiplications.
+    SmallVector<Value> outTile;
+    for (int64_t i = 0; i < M / 2; ++i)
+      for (int64_t j = 0; j < N / 2; ++j) {
+        Value mmla = createMMLA(rewriter, mmlaOp, loc, nxv4i32,
+                                accTile[i * N / 2 + j], lhsTile[i], rhsTile[j]);
+        outTile.push_back(mmla);
+      }
+
+    // Unpack the OUT sub-tiles and insert into the result.
+    Value result = rewriter.create<ub::PoisonOp>(loc, op.getResultType());
+    for (int64_t i = 0; i < M / 2; ++i) {
+      // Collect a number of sub-tiles in a row.
+      Value row = rewriter.create<ub::PoisonOp>(loc, accRowX2Ty);
+      for (int64_t j = 0; j < N / 2; ++j)
+        row = rewriter.create<vector::ScalableInsertOp>(
+            loc, outTile[i * N / 2 + j], row, j * 4);
+
+      // Unpack the row to obtain two rows of the output. If we have the out
+      // sub-tiles transposed we obtain two consecutive output rows by
+      // separating even and odd elements, i.e. a simple deinterleave.
+      // Otherwise, the interleave is by pairs.
+      Value out0, out1;
+      if (mmlaOp == MMLA::MixedSwapped) {
+        auto tmp = rewriter.create<vector::DeinterleaveOp>(loc, row);
+        out0 = tmp.getRes1();
+        out1 = tmp.getRes2();
+      } else {
+        // Deinterleave by pairs.
+        auto row64 = rewriter.create<vector::BitCastOp>(loc, accRowX264Ty, row);
+        auto deintr64 = rewriter.create<vector::DeinterleaveOp>(loc, row64);
+
+        // Bitcast back into 32-bit elements and insert into the result.
+        out0 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+                                                  deintr64.getRes1());
+        out1 = rewriter.create<vector::BitCastOp>(loc, accRowTy,
+                                                  deintr64.getRes2());
+      }
+      result = rewriter.create<vector::InsertOp>(loc, out0, result, i * 2);
+      result = rewriter.create<vector::InsertOp>(loc, out1, result, i * 2 + 1);
+    }
+
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
+} // namespace
+
+void mlir::populateLowerContractionToSVEI8MMPatternPatterns(
+    RewritePatternSet &patterns) {
+  MLIRContext *context = patterns.getContext();
+  patterns.add<LowerContractionToSVEI8MMPattern>(context, /*benefit=*/2);
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/external/llvm-project/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index eed7a56fff8a..ca914df8b789 100644
--- a/external/llvm-project/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -94,7 +94,9 @@ void Ownership::combine(Ownership other) { *this = getCombined(other); }
 // DeallocationState
 //===----------------------------------------------------------------------===//
 
-DeallocationState::DeallocationState(Operation *op) : liveness(op) {}
+DeallocationState::DeallocationState(Operation *op,
+                                     SymbolTableCollection &symbolTables)
+    : symbolTable(symbolTables), liveness(op) {}
 
 void DeallocationState::updateOwnership(Value memref, Ownership ownership,
                                         Block *block) {
diff --git a/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt
index 83d30f33aabc..7c38621be1bb 100644
--- a/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt
@@ -24,7 +24,6 @@ add_mlir_dialect_library(MLIRBufferizationTransforms
   MLIRBufferizationEnumsIncGen
 
   LINK_LIBS PUBLIC
-  MLIRAsyncDialect
   MLIRArithDialect
   MLIRBufferizationDialect
   MLIRCallInterfaces
diff --git a/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp b/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp
index dee2af8271ce..d1d106220a38 100644
--- a/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OneShotModuleBufferize.cpp
@@ -310,21 +310,19 @@ static bool hasTensorSignature(func::FuncOp funcOp) {
 /// any func::CallOp.
 static LogicalResult getFuncOpsOrderedByCalls(
     ModuleOp moduleOp, SmallVectorImpl<func::FuncOp> &orderedFuncOps,
-    SmallVectorImpl<func::FuncOp> &remainingFuncOps, FuncCallerMap &callerMap) {
+    SmallVectorImpl<func::FuncOp> &remainingFuncOps, FuncCallerMap &callerMap,
+    SymbolTableCollection &symbolTables) {
   // For each FuncOp, the set of functions called by it (i.e. the union of
   // symbols of all nested func::CallOp).
   DenseMap<func::FuncOp, DenseSet<func::FuncOp>> calledBy;
   // For each FuncOp, the number of func::CallOp it contains.
   DenseMap<func::FuncOp, unsigned> numberCallOpsContainedInFuncOp;
 
-  // TODO Avoid recomputing the symbol tables every time.
-  mlir::SymbolTableCollection symbolTable;
-
   for (func::FuncOp funcOp : moduleOp.getOps<func::FuncOp>()) {
     // Collect function calls and populate the caller map.
     numberCallOpsContainedInFuncOp[funcOp] = 0;
     WalkResult res = funcOp.walk([&](func::CallOp callOp) -> WalkResult {
-      func::FuncOp calledFunction = getCalledFunction(callOp, symbolTable);
+      func::FuncOp calledFunction = getCalledFunction(callOp, symbolTables);
       assert(calledFunction && "could not retrieved called func::FuncOp");
       // If the called function does not have any tensors in its signature, then
       // it is not necessary to bufferize the callee before the caller.
@@ -343,15 +341,25 @@ static LogicalResult getFuncOpsOrderedByCalls(
 
   // Iteratively remove function operations that do not call any of the
   // functions remaining in the callCounter map and add them to ordered list.
-  while (!numberCallOpsContainedInFuncOp.empty()) {
-    auto it = llvm::find_if(numberCallOpsContainedInFuncOp,
-                            [](auto entry) { return entry.getSecond() == 0; });
-    if (it == numberCallOpsContainedInFuncOp.end())
-      break;
-    orderedFuncOps.push_back(it->getFirst());
-    for (auto callee : calledBy[it->getFirst()])
-      numberCallOpsContainedInFuncOp[callee]--;
-    numberCallOpsContainedInFuncOp.erase(it);
+  SmallVector<func::FuncOp> worklist;
+
+  for (const auto &entry : numberCallOpsContainedInFuncOp) {
+    if (entry.second == 0)
+      worklist.push_back(entry.first);
+  }
+
+  while (!worklist.empty()) {
+    func::FuncOp func = worklist.pop_back_val();
+    orderedFuncOps.push_back(func);
+
+    for (func::FuncOp caller : calledBy[func]) {
+      auto &count = numberCallOpsContainedInFuncOp[caller];
+
+      if (--count == 0)
+        worklist.push_back(caller);
+    }
+
+    numberCallOpsContainedInFuncOp.erase(func);
   }
 
   // Put all other functions in the list of remaining functions. These are
@@ -458,7 +466,8 @@ mlir::bufferization::analyzeModuleOp(ModuleOp moduleOp,
   FuncCallerMap callerMap;
 
   if (failed(getFuncOpsOrderedByCalls(moduleOp, orderedFuncOps,
-                                      remainingFuncOps, callerMap)))
+                                      remainingFuncOps, callerMap,
+                                      funcState.symbolTables)))
     return failure();
 
   // Analyze functions in order. Starting with functions that are not calling
@@ -534,7 +543,8 @@ LogicalResult mlir::bufferization::bufferizeModuleOp(
   // each other recursively are bufferized in an unspecified order at the end.
   // We may use unnecessarily "complex" (in terms of layout map) buffer types.
   if (failed(getFuncOpsOrderedByCalls(moduleOp, orderedFuncOps,
-                                      remainingFuncOps, callerMap)))
+                                      remainingFuncOps, callerMap,
+                                      state.getSymbolTables())))
     return failure();
   llvm::append_range(orderedFuncOps, remainingFuncOps);
 
diff --git a/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index c5b2f3020712..1eeafc4df8cf 100644
--- a/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -166,8 +166,9 @@ namespace {
 /// program have a corresponding de-allocation.
 class BufferDeallocation {
 public:
-  BufferDeallocation(Operation *op, DeallocationOptions options)
-      : state(op), options(options) {}
+  BufferDeallocation(Operation *op, DeallocationOptions options,
+                     SymbolTableCollection &symbolTables)
+      : state(op, symbolTables), options(options) {}
 
   /// Performs the actual placement/creation of all dealloc operations.
   LogicalResult deallocate(FunctionOpInterface op);
@@ -1027,11 +1028,13 @@ struct OwnershipBasedBufferDeallocationPass
     DeallocationOptions options;
     options.privateFuncDynamicOwnership = privateFuncDynamicOwnership;
 
+    mlir::SymbolTableCollection symbolTables;
+
     auto status = getOperation()->walk([&](func::FuncOp func) {
       if (func.isExternal())
         return WalkResult::skip();
 
-      if (failed(deallocateBuffersOwnershipBased(func, options)))
+      if (failed(deallocateBuffersOwnershipBased(func, options, symbolTables)))
         return WalkResult::interrupt();
 
       return WalkResult::advance();
@@ -1047,11 +1050,11 @@ struct OwnershipBasedBufferDeallocationPass
 // Implement bufferization API
 //===----------------------------------------------------------------------===//
 
-LogicalResult
-bufferization::deallocateBuffersOwnershipBased(FunctionOpInterface op,
-                                               DeallocationOptions options) {
+LogicalResult bufferization::deallocateBuffersOwnershipBased(
+    FunctionOpInterface op, DeallocationOptions options,
+    SymbolTableCollection &symbolTables) {
   // Gather all required allocation nodes and prepare the deallocation phase.
-  BufferDeallocation deallocation(op, options);
+  BufferDeallocation deallocation(op, options, symbolTables);
 
   // Place all required temporary clone and dealloc nodes.
   return deallocation.deallocate(op);
diff --git a/external/llvm-project/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/external/llvm-project/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 648b6b087e59..aaf6b0593c2e 100644
--- a/external/llvm-project/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -1205,6 +1205,42 @@ LogicalResult NVVM::VoteSyncOp::verify() {
   return success();
 }
 
+LogicalResult NVVM::PrefetchOp::verify() {
+  using MemSpace = NVVM::NVVMMemorySpace;
+  using CacheLevel = NVVM::PrefetchCacheLevel;
+
+  unsigned addressSpace =
+      llvm::cast<LLVM::LLVMPointerType>(getAddr().getType()).getAddressSpace();
+  std::optional<NVVM::CacheEvictionPriority> evictPriority = getEvictPriority();
+
+  if (getUniform()) {
+    if (getCacheLevel() != CacheLevel::L1)
+      return emitOpError("unsupported cache level, the only supported uniform "
+                         "cache level is L1");
+
+    if (addressSpace != MemSpace::kGenericMemorySpace)
+      return emitOpError(
+          "prefetch to uniform cache requires a generic pointer");
+  }
+
+  if (evictPriority) {
+    if (getCacheLevel() != CacheLevel::L2)
+      return emitOpError(
+          "cache eviction priority supported only for cache level L2");
+
+    if (addressSpace != MemSpace::kGlobalMemorySpace)
+      return emitOpError("cache eviction priority requires a global pointer");
+
+    if (*evictPriority != NVVM::CacheEvictionPriority::EvictNormal &&
+        *evictPriority != NVVM::CacheEvictionPriority::EvictLast)
+      return emitOpError(
+          "unsupported cache eviction priority, only evict_last and "
+          "evict_normal are supported");
+  }
+
+  return success();
+}
+
 /// Packs the given `field` into the `result`.
 /// The `result` is 64-bits and each `field` can be 32-bits or narrower.
 static llvm::Value *
@@ -1712,6 +1748,70 @@ NVVM::IDArgPair DotAccumulate4WayOp::getIntrinsicIDAndArgs(
   return {ids[type], args};
 }
 
+NVVM::IDArgPair DotAccumulate2WayOp::getIntrinsicIDAndArgs(
+    Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
+  auto curOp = cast<NVVM::DotAccumulate2WayOp>(op);
+
+  llvm::SmallVector<llvm::Value *> args;
+  args.push_back(getAsPackedI32(mt.lookupValue(curOp.getA()), builder));
+  args.push_back(getAsPackedI32(mt.lookupValue(curOp.getB()), builder));
+  args.push_back(builder.getInt1(curOp.getBHi()));
+  args.push_back(mt.lookupValue(curOp.getC()));
+
+  bool isASigned = curOp.getAType() == NVVM::DotAccumulateType::SIGNED;
+  bool isBSigned = curOp.getBType() == NVVM::DotAccumulateType::SIGNED;
+  unsigned type = (isASigned << 1) | isBSigned;
+  const llvm::Intrinsic::ID ids[] = {
+      llvm::Intrinsic::nvvm_idp2a_u_u,
+      llvm::Intrinsic::nvvm_idp2a_u_s,
+      llvm::Intrinsic::nvvm_idp2a_s_u,
+      llvm::Intrinsic::nvvm_idp2a_s_s,
+  };
+  return {ids[type], args};
+}
+
+llvm::Intrinsic::ID PrefetchOp::getIntrinsicID(NVVM::PrefetchOp &op) {
+  using MemSpace = NVVM::NVVMMemorySpace;
+  using CacheLevel = NVVM::PrefetchCacheLevel;
+
+  NVVM::PrefetchCacheLevel cacheLevel = op.getCacheLevel();
+  std::optional<NVVM::CacheEvictionPriority> evictPriority =
+      op.getEvictPriority();
+  unsigned addressSpace =
+      llvm::cast<LLVM::LLVMPointerType>(op.getAddr().getType())
+          .getAddressSpace();
+
+  if (op.getUniform() && cacheLevel == CacheLevel::L1)
+    return llvm::Intrinsic::nvvm_prefetchu_L1;
+
+  if (evictPriority && cacheLevel == CacheLevel::L2) {
+    switch (*evictPriority) {
+    case NVVM::CacheEvictionPriority::EvictLast:
+      return llvm::Intrinsic::nvvm_prefetch_global_L2_evict_last;
+    case NVVM::CacheEvictionPriority::EvictNormal:
+      return llvm::Intrinsic::nvvm_prefetch_global_L2_evict_normal;
+    default:
+      llvm_unreachable("Invalid cache eviction priority");
+    }
+  }
+
+  switch (addressSpace) {
+  case MemSpace::kGenericMemorySpace:
+    return cacheLevel == CacheLevel::L1 ? llvm::Intrinsic::nvvm_prefetch_L1
+                                        : llvm::Intrinsic::nvvm_prefetch_L2;
+  case MemSpace::kGlobalMemorySpace:
+    return cacheLevel == CacheLevel::L1
+               ? llvm::Intrinsic::nvvm_prefetch_global_L1
+               : llvm::Intrinsic::nvvm_prefetch_global_L2;
+  case MemSpace::kLocalMemorySpace:
+    return cacheLevel == CacheLevel::L1
+               ? llvm::Intrinsic::nvvm_prefetch_local_L1
+               : llvm::Intrinsic::nvvm_prefetch_local_L2;
+  default:
+    llvm_unreachable("Invalid pointer address space");
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // NVVMDialect initialization, type parsing, and registration.
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 58e5bc8e396a..0cd563b687ce 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -1278,8 +1278,9 @@ LogicalResult GenericOp::verify() { return success(); }
 
 namespace {
 
-/// Remove any linalg operation (on tensors) that are just copying
-/// the values from inputs to the results. Requirements are
+/// Remove linalg operations that are just copying the values from inputs to
+/// results. In the memref case, the operation must be copying to and from the
+/// same value. Requirements are:
 /// 1) All iterator types are parallel
 /// 2) The body contains just a yield operation with the yielded values being
 ///    the arguments corresponding to the operands.
@@ -1304,18 +1305,27 @@ struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
 
     // In the buffer case, we need to check exact buffer equality.
     if (linalgOp.hasPureBufferSemantics()) {
-      if (linalgOp.getNumDpsInputs() == 1 && linalgOp.getNumDpsInits() == 1 &&
-          linalgOp.getDpsInputOperand(0)->get() ==
+      if (linalgOp.getNumDpsInputs() != 1 || linalgOp.getNumDpsInits() != 1 ||
+          linalgOp.getDpsInputOperand(0)->get() !=
               linalgOp.getDpsInitOperand(0)->get()) {
-        rewriter.eraseOp(linalgOp);
-        return success();
+        return rewriter.notifyMatchFailure(
+            linalgOp, "expected single input and output to be the same value");
       }
-      return failure();
+
+      auto yieldArg = dyn_cast<BlockArgument>(yieldOp.getOperand(0));
+      if (!yieldArg || yieldArg.getOwner() != &body) {
+        return rewriter.notifyMatchFailure(linalgOp,
+                                           "cannot fold fill-like op");
+      }
+
+      rewriter.eraseOp(linalgOp);
+      return success();
     }
 
-    // Mixed semantics is not supported yet.
-    if (!linalgOp.hasPureTensorSemantics())
-      return failure();
+    if (!linalgOp.hasPureTensorSemantics()) {
+      return rewriter.notifyMatchFailure(
+          linalgOp, "mixed semantics is not supported yet");
+    }
 
     // Get the argument number of the returned values. That is the operand
     // number to use for replacing uses of this operation.
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 1c3b62182831..b2c28f5eed33 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -2381,7 +2381,7 @@ transform::ScalarizeOp::applyToOne(transform::TransformRewriter &rewriter,
     return emitDefaultDefiniteFailure(target);
 
   if (target->getNumResults())
-    rewriter.replaceOp(target, maybeTilingResult->mergeResult.replacements);
+    rewriter.replaceOp(target, maybeTilingResult->replacements);
   else
     rewriter.eraseOp(target);
 
@@ -2800,12 +2800,12 @@ DiagnosedSilenceableFailure transform::TileReductionUsingForOp::applyToOne(
 
   if (failed(result))
     return emitDefaultSilenceableFailure(target);
-  rewriter.replaceOp(target, result->mergeResult.replacements);
+  rewriter.replaceOp(target, result->replacements);
   for (Value initValue : result->initialValues)
     results.push_back(initValue.getDefiningOp());
   for (auto parallelTiledOp : result->tiledOps)
     results.push_back(parallelTiledOp);
-  for (auto mergeOp : result->mergeResult.mergeOps)
+  for (auto mergeOp : result->mergeOps)
     results.push_back(mergeOp);
   results.push_back(result->loops.front());
   return DiagnosedSilenceableFailure::success();
@@ -3229,7 +3229,7 @@ transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
     if (failed(maybeTilingResult))
       return DiagnosedSilenceableFailure::definiteFailure();
 
-    rewriter.replaceOp(op, maybeTilingResult->mergeResult.replacements);
+    rewriter.replaceOp(op, maybeTilingResult->replacements);
 
     tiled.append(maybeTilingResult->tiledOps);
     for (const auto &en2 : llvm::enumerate(maybeTilingResult->loops))
@@ -3465,7 +3465,7 @@ DiagnosedSilenceableFailure transform::tileToForallOpImpl(
   if (failed(maybeTilingResult))
     return transformOp.emitDefaultSilenceableFailure(tileableOp);
 
-  rewriter.replaceOp(tileableOp, maybeTilingResult->mergeResult.replacements);
+  rewriter.replaceOp(tileableOp, maybeTilingResult->replacements);
 
   tilingResult = *maybeTilingResult;
 
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index dd8ef9608a82..7188987e5e93 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -876,11 +876,8 @@ class BubbleUpPackOpThroughReshapeOp final
       return failure();
     }
     // Currently only support static inner tile sizes.
-    if (llvm::any_of(packOp.getStaticTiles(), [](int64_t size) {
-          return ShapedType::isDynamic(size);
-        })) {
+    if (llvm::any_of(packOp.getStaticTiles(), ShapedType::isDynamic))
       return failure();
-    }
 
     // User controlled propagation function.
     if (!controlFn(&packOp.getSourceMutable()))
@@ -1002,11 +999,8 @@ class PushDownUnPackOpThroughReshapeOp final
       return failure();
     }
     // Currently only support static inner tile sizes.
-    if (llvm::any_of(unPackOp.getStaticTiles(), [](int64_t size) {
-          return ShapedType::isDynamic(size);
-        })) {
+    if (llvm::any_of(unPackOp.getStaticTiles(), ShapedType::isDynamic))
       return failure();
-    }
 
     Operation *consumerOp = *result.user_begin();
     return TypeSwitch<Operation *, LogicalResult>(consumerOp)
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index bd4ffabfbb92..5e6dde36d7f9 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -606,8 +606,7 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
     int64_t padRank = sourceShape.size();
 
     auto isStaticZero = [](OpFoldResult f) {
-      std::optional<int64_t> maybeInt = getConstantIntValue(f);
-      return maybeInt && *maybeInt == 0;
+      return getConstantIntValue(f) == 0;
     };
 
     llvm::SmallDenseSet<unsigned> unitDimsFilter(allowedUnitDims.begin(),
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 1f5af39e604e..f97ed3d6d511 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -231,8 +231,7 @@ static void generateFusedElementwiseOpRegion(
   // `consumerToProducerLoopsMap` to map the producer indices.
   if (producer.hasIndexSemantics()) {
     // Add an index operation for every fused loop dimension.
-    unsigned numFusedOpLoops =
-        std::max(producer.getNumLoops(), consumer.getNumLoops());
+    unsigned numFusedOpLoops = fusedOp.getNumLoops();
     SmallVector<Value> fusedIndices;
     fusedIndices.reserve(numFusedOpLoops);
     llvm::transform(llvm::seq<uint64_t>(0, numFusedOpLoops),
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 8718c57b9e86..615d1f66414b 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -227,8 +227,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   // 1. Filter out NYI cases.
   auto packedTensorType =
       cast<RankedTensorType>(packOp->getResultTypes().front());
-  if (llvm::any_of(packOp.getStaticInnerTiles(),
-                   [](int64_t size) { return ShapedType::isDynamic(size); })) {
+  if (llvm::any_of(packOp.getStaticInnerTiles(), ShapedType::isDynamic)) {
     return rewriter.notifyMatchFailure(
         packOp,
         "non-static shape NYI, needs a more powerful tensor.expand_shape op");
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index c5b62227777a..ff28bd7c4834 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -820,9 +820,8 @@ tensorExtractVectorizationPrecondition(Operation *op, bool vectorizeNDExtract) {
       return failure();
   }
 
-  if (llvm::any_of(extractOp->getResultTypes(), [](Type type) {
-        return !VectorType::isValidElementType(type);
-      })) {
+  if (!llvm::all_of(extractOp->getResultTypes(),
+                    VectorType::isValidElementType)) {
     return failure();
   }
 
@@ -1506,101 +1505,192 @@ static SmallVector<int64_t> getTiledPackShape(linalg::PackOp packOp,
   return applyPermutation(destShape, linalg::getPackInverseDestPerm(packOp));
 }
 
+/// Determines whether a mask for xfer_write is trivially "all true"
+///
+/// Given all the inputs required to generate a mask (mask sizes and shapes),
+/// and an xfer_write operation (write indices and the destination tensor
+/// shape), determines whether the corresponding mask would be trivially
+/// foldable (i.e., trivially "all true").
+///
+/// Use this method to avoid generating spurious masks and relaying on
+/// vectorization post-processing to remove them.
+///
+/// Pre-conditions for a mask to be trivially foldable:
+///   * All involved shapes (mask + destination tensor) are static.
+///   * All write indices are constant.
+///   * All mask sizes are constant (including `arith.constant`).
+///
+/// If the pre-conditions are met, the method checks for each destination
+/// dimension `d`:
+///   (1) destDimSize[rankDiff + d] <= maskShape[d]
+///   (2) destDimSize[rankDiff + d] <= writeIndex[d] + maskSize[d]
+///
+/// rankDiff = rank(dest) - rank(mask).
+///
+/// This method takes a conservative view: it may return false even if the mask
+/// is technically foldable.
+///
+/// EXAMPLE 1 (trivially foldable, all shapes match, mask sizes match the shape
+/// of the dest tensor):
+///   %c0 = arith.constant 0 : index
+///   %mask = vector.create_mask 5, 1
+///   vector.mask %mask {
+///     vector.transfer_write %vecToStore_1, %dest{[%c0, %c0]
+///       {in_bounds = [true, true]}
+///     : vector<5x1xi32>, tensor<5x1xi32>
+///   }
+///
+/// EXAMPLE 2 (not trivially foldable - vector shape exceeds the tensor shape,
+/// mask is required to avoid out-of-bounds write):
+///   %c0 = arith.constant 0 : index
+///   %mask = vector.create_mask 5, 1
+///   vector.mask %mask {
+///     vector.transfer_write %vecToStore_2, %dest[%c0, %c0]
+///      {in_bounds = [true, true]}
+///     : vector<8x1xi32>, tensor<5x1xi32>
+///   }
+///
+/// TODO: Re-use in createReadOrMaskedRead
+static bool isMaskTriviallyFoldable(SmallVector<OpFoldResult> &maskSizes,
+                                    SmallVector<Value> &writeIdxs,
+                                    ArrayRef<int64_t> destShape,
+                                    ArrayRef<int64_t> maskShape) {
+  // Masking is unavoidable in the case of dynamic tensors.
+  if (ShapedType::isDynamicShape(destShape))
+    return false;
+
+  // Collect all constant mask sizes.
+  SmallVector<int64_t, 4> cstMaskSizes;
+  for (auto [i, dimSize] : llvm::enumerate(maskSizes)) {
+    if (auto intSize = getConstantIntValue(dimSize)) {
+      cstMaskSizes.push_back(*intSize);
+    }
+  }
+
+  // If any of the mask sizes is non-constant, bail out.
+  if (cstMaskSizes.size() != maskShape.size())
+    return false;
+
+  // Collect all constant write indices.
+  SmallVector<int64_t, 4> cstWriteIdxs;
+  for (auto [i, idx] : llvm::enumerate(writeIdxs)) {
+    APSInt intVal;
+    if (matchPattern(idx, m_ConstantInt(&intVal))) {
+      cstWriteIdxs.push_back(intVal.getSExtValue());
+    }
+  }
+
+  // If any of the write indices is non-constant, bail out.
+  if (cstWriteIdxs.size() != destShape.size())
+    return false;
+
+  // Go over all destination dims and check (1) and (2). Take into account that:
+  //  * The number of mask sizes will match the rank of the vector to store.
+  //    This could be lower than the rank of the destination tensor.
+  //  * Mask sizes could be larger than the corresponding mask shape (hence
+  //  `clamp`).
+  // TODO: The 2nd item should be rejected by the verifier.
+  int64_t rankDiff = destShape.size() - cstMaskSizes.size();
+  for (auto [i, idx] : llvm::enumerate(cstMaskSizes)) {
+    if (/*(1)*/ maskShape[i] > destShape[rankDiff + i] ||
+        /*(2)*/ destShape[rankDiff + i] <
+            (std::clamp(cstMaskSizes[i], int64_t(0), maskShape[i]) +
+             cstWriteIdxs[i]))
+      return false;
+  }
+
+  return true;
+}
+
 /// Creates an optionally masked TransferWriteOp
 ///
 /// Generates the following operation:
-///   %res = vector.transfer_write %vectorToStore into %dest
+///   %res = vector.transfer_write %vecToStore into %dest
 ///
-/// If the leading N dimensions of the destination tensor do not match
-/// `inputVecSizesForLeadingDims` (N = rank(inputVecSizesForLeadingDims)),
-/// masking is applied to ensure correctness:
+/// If shape(vecToStore) != shape(dest), masking is used to ensure correctness:
 ///
-///   %mask = vector.create_mask(%destShape)
+///   %mask = vector.create_mask(%destShape) : %vecToStoreShape
 ///   %res = vector.mask %mask {
-///     vector.transfer_write %vectorToStore into %dest
+///     vector.transfer_write %vecToStore into %dest
 ///   }
 ///
+/// The mask shape is identical to `vecToStore` (with the element type ==
+/// i1), and the mask values are based on the shape of the `dest` tensor.
+///
 /// If `useInBoundsInsteadOfMasking` is set to `true`, the `in_bounds` attribute
 /// is used instead of masking:
 ///
-///   %write = vector.transfer_write %vectorToStore into %dest
+///   %write = vector.transfer_write %vecToStore into %dest
 ///   in_bounds_flags = (...)
 ///   %res = vector.transfer_write %input into %dest
 ///       {in_bounds = in_bounds_flags}
 ///
-/// NOTE: All write offsets are set to 0.
-/// TODO: Allow specyfying write offsets.
-/// NOTE: When N < rank(input), the missing vector sizes are effectively
-/// extracted from the trailing sizes of `destSizes`. This means those sizes
-/// must be static.
-/// TODO: Support cases where an arbitrary dim is dynamic - this will require
-/// specifying all the vector sizes.
+/// Finally, `writeIndices` specifies the offsets to use. If empty, all indices
+/// are set to 0.
 static Operation *
-createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vectorToStore,
-                         Value dest,
-                         ArrayRef<int64_t> inputVecSizesForLeadingDims,
+createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
+                         Value dest, SmallVector<Value> writeIndices = {},
                          bool useInBoundsInsteadOfMasking = false) {
 
   ShapedType destType = cast<ShapedType>(dest.getType());
-  assert(cast<VectorType>(vectorToStore.getType()).getRank() ==
-             static_cast<int64_t>(destType.getRank()) &&
-         "Rank mismatch!");
-  (void)destType;
+  int64_t destRank = destType.getRank();
+  auto destShape = destType.getShape();
 
-  int64_t rank = cast<ShapedType>(dest.getType()).getRank();
-  auto destShape = cast<ShapedType>(dest.getType()).getShape();
+  VectorType vecToStoreType = cast<VectorType>(vecToStore.getType());
+  int64_t vecToStoreRank = vecToStoreType.getRank();
+  auto vecToStoreShape = vecToStoreType.getShape();
 
   // Compute the in_bounds attribute
-  SmallVector<bool> inBoundsVal(rank, true);
+  SmallVector<bool> inBoundsVal(vecToStoreRank, true);
   if (useInBoundsInsteadOfMasking) {
-    // In this case, assume that all the required vector sizes have been
-    // provided.
-    assert(inputVecSizesForLeadingDims.size() ==
-               static_cast<size_t>(destType.getRank()) &&
-           "Insufficient number of input vector sizes!");
     // Update the inBounds attribute.
-    for (unsigned i = 0; i < rank; i++)
-      inBoundsVal[i] = (destShape[i] == inputVecSizesForLeadingDims[i]) &&
-                       !ShapedType::isDynamic(destShape[i]);
+    // FIXME: This computation is too weak - it ignores the write indices.
+    for (unsigned i = 0; i < vecToStoreRank; i++)
+      inBoundsVal[i] =
+          (destShape[destRank - vecToStoreRank + i] >= vecToStoreShape[i]) &&
+          !ShapedType::isDynamic(destShape[destRank - vecToStoreRank + i]);
+  }
+
+  // If missing, initialize the write indices to 0.
+  assert(writeIndices.empty() ||
+         writeIndices.size() == static_cast<size_t>(destRank) &&
+             "Invalid number of write indices!");
+  if (writeIndices.empty()) {
+    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    writeIndices.assign(destRank, zero);
   }
 
   // Generate the xfer_write Op
-  auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
-  Operation *write = builder.create<vector::TransferWriteOp>(
-      loc,
-      /*vector=*/vectorToStore,
-      /*source=*/dest,
-      /*indices=*/SmallVector<Value>(rank, zero),
-      /*inBounds=*/inBoundsVal);
-  assert(llvm::none_of(
-             destShape.drop_front(inputVecSizesForLeadingDims.size()),
-             [](int64_t size) { return size == ShapedType::kDynamic; }) &&
-         "Only dims aligned with inputVecSizesForLeadingDims may be dynamic");
+  Operation *write =
+      builder.create<vector::TransferWriteOp>(loc,
+                                              /*vector=*/vecToStore,
+                                              /*source=*/dest,
+                                              /*indices=*/writeIndices,
+                                              /*inBounds=*/inBoundsVal);
 
   // If masking is disabled, exit.
   if (useInBoundsInsteadOfMasking)
     return write;
 
-  // Check if masking is needed.
-  bool needMaskForWrite =
-      !llvm::equal(inputVecSizesForLeadingDims,
-                   destShape.take_front(inputVecSizesForLeadingDims.size()));
+  // Check if masking is needed. If not, exit.
+  if (llvm::equal(vecToStoreShape, destShape.take_back(vecToStoreRank)))
+    return write;
 
-  // If masking is needed, generate the mask and mask the operation.
-  if (needMaskForWrite) {
-    SmallVector<int64_t> writeMaskShape;
-    writeMaskShape.append(inputVecSizesForLeadingDims.begin(),
-                          inputVecSizesForLeadingDims.end());
-    writeMaskShape.append(destShape.begin() +
-                              inputVecSizesForLeadingDims.size(),
-                          destShape.end());
-    auto writeMaskType = VectorType::get(writeMaskShape, builder.getI1Type());
-    Value maskForWrite = builder.create<vector::CreateMaskOp>(
-        loc, writeMaskType, tensor::getMixedSizes(builder, loc, dest));
-    write = mlir::vector::maskOperation(builder, write, maskForWrite);
-  }
+  // Compute the mask and mask the write Op.
+  auto writeMaskType = VectorType::get(vecToStoreShape, builder.getI1Type());
 
-  return write;
+  SmallVector<OpFoldResult> destSizes =
+      tensor::getMixedSizes(builder, loc, dest);
+  SmallVector<OpFoldResult> maskSizes(destSizes.end() - vecToStoreRank,
+                                      destSizes.end());
+
+  if (isMaskTriviallyFoldable(maskSizes, writeIndices, destShape,
+                              vecToStoreShape))
+    return write;
+
+  Value maskForWrite =
+      builder.createOrFold<vector::CreateMaskOp>(loc, writeMaskType, maskSizes);
+  return mlir::vector::maskOperation(builder, write, maskForWrite);
 }
 
 /// Vectorize linalg::PackOp with (1) static inner_tiles (2) constant
@@ -1701,9 +1791,7 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
       loc, reifiedReturnShapes[0],
       transposeOp.getResult().getType().getElementType());
   Operation *write =
-      createWriteOrMaskedWrite(rewriter, loc, transposeOp.getResult(), dest,
-                               /*inputVecSizesForLeadingDims=*/inputVectorSizes,
-                               /*useInBoundsInsteadOfMasking=*/false);
+      createWriteOrMaskedWrite(rewriter, loc, transposeOp.getResult(), dest);
   newResults.push_back(write->getResult(0));
   return success();
 }
@@ -1839,10 +1927,9 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   Value dest = rewriter.create<tensor::EmptyOp>(
       loc, reifiedRetShapes[0],
       shapeCastOp.getResult().getType().getElementType());
-  Operation *write =
-      createWriteOrMaskedWrite(rewriter, loc, shapeCastOp.getResult(), dest,
-                               /*inputVecSizesForLeadingDims=*/writeVectorSizes,
-                               useInBoundsInsteadOfMasking);
+  Operation *write = createWriteOrMaskedWrite(
+      rewriter, loc, shapeCastOp.getResult(), dest,
+      /*writeIndices=*/{}, useInBoundsInsteadOfMasking);
   newResults.push_back(write->getResult(0));
   return success();
 }
@@ -1874,10 +1961,7 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
   // Create Xfer write Op
   Value dest = rewriter.create<tensor::EmptyOp>(
       loc, reifiedReturnShapes[0], padOp.getResultType().getElementType());
-  Operation *write =
-      createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest,
-                               /*inputVecSizesForLeadingDims=*/inputVectorSizes,
-                               /*useInBoundsInsteadOfMasking=*/false);
+  Operation *write = createWriteOrMaskedWrite(rewriter, loc, maskedRead, dest);
   newResults.push_back(write->getResult(0));
   return success();
 }
@@ -2163,14 +2247,12 @@ static LogicalResult vectorizeLinalgOpPrecondition(
             })) {
       continue;
     }
-    if (llvm::any_of(innerOp.getOperandTypes(), [](Type type) {
-          return !VectorType::isValidElementType(type);
-        })) {
+    if (!llvm::all_of(innerOp.getOperandTypes(),
+                      VectorType::isValidElementType)) {
       return failure();
     }
-    if (llvm::any_of(innerOp.getResultTypes(), [](Type type) {
-          return !VectorType::isValidElementType(type);
-        })) {
+    if (!llvm::all_of(innerOp.getResultTypes(),
+                      VectorType::isValidElementType)) {
       return failure();
     }
   }
@@ -2883,22 +2965,14 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
         sliceOp.getLoc(), elemType, rewriter.getZeroAttr(elemType));
   }
 
-  // 2. Get the vector shape and in-bounds attributes
+  // 2. Get the vector shape
   SmallVector<int64_t> vecShape;
-  SmallVector<bool> readInBounds;
-  SmallVector<bool> writeInBounds;
   size_t rankDiff = resultType.getRank() - sourceType.getRank();
   for (int64_t i = 0, end = sourceType.getRank(); i < end; ++i) {
     if (!inputVectorSizes.empty()) {
       vecShape.push_back(inputVectorSizes[i]);
-      readInBounds.push_back(false);
-      writeInBounds.push_back(false);
     } else if (!sourceType.isDynamicDim(i)) {
       vecShape.push_back(sourceType.getDimSize(i));
-      // Source shape is statically known: Neither read nor write are
-      // out-of-bounds.
-      readInBounds.push_back(true);
-      writeInBounds.push_back(true);
     } else if (!resultType.isDynamicDim(i)) {
       // Source shape is not statically known, but result shape is.
       // Vectorize with size of result shape. This may be larger than the
@@ -2906,69 +2980,30 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
       // FIXME: Using rankDiff implies that the source tensor is inserted at
       // the end of the destination tensor. However, that's not required.
       vecShape.push_back(resultType.getDimSize(rankDiff + i));
-      // Read may be out-of-bounds because the result size could be larger
-      // than the source size.
-      readInBounds.push_back(false);
-      // Write will be in-bounds provided that the corresponding write idx is 0.
-      // To keep this logic simple, conservatively mark as out-of-bounds.
-      writeInBounds.push_back(false);
     } else {
       // Neither source nor result dim of padOp is static. Cannot vectorize
       // the copy.
-      // TODO: Add support for masking
       return failure();
     }
   }
   auto vecType = VectorType::get(vecShape, sourceType.getElementType());
 
   // 3. Generate TransferReadOp + TransferWriteOp
-  ReifiedRankedShapedTypeDims reifiedSrcSizes;
-  Value maskOp;
-
-  // If vector sizes are user provided, make sure to mask. First, generate the
-  // mask.
-  if (!inputVectorSizes.empty()) {
-    auto *srcDefOp = source.getDefiningOp();
-    if (!srcDefOp) {
-      LDBG("Unable to get the defining Op of " << sliceOp);
-      return failure();
-    }
-
-    LogicalResult status =
-        cast<ReifyRankedShapedTypeOpInterface>(srcDefOp).reifyResultShapes(
-            rewriter, reifiedSrcSizes);
-    if (status.failed()) {
-      LDBG("Unable to reify result shapes of " << srcDefOp);
-      return failure();
-    }
-
-    // Create the mask
-    auto readMaskType = VectorType::get(inputVectorSizes, rewriter.getI1Type());
-    maskOp = rewriter.create<vector::CreateMaskOp>(
-        sliceOp.getLoc(), readMaskType, reifiedSrcSizes[0]);
-  }
+  auto loc = sliceOp.getLoc();
 
+  // Create read
   SmallVector<Value> readIndices(
-      vecType.getRank(),
-      rewriter.create<arith::ConstantIndexOp>(sliceOp.getLoc(), 0));
-  Operation *read = rewriter.create<vector::TransferReadOp>(
-      sliceOp.getLoc(), vecType, source, readIndices, padValue,
-      ArrayRef<bool>{readInBounds});
-
-  if (maskOp) {
-    read = mlir::vector::maskOperation(rewriter, read, maskOp);
-  }
-
-  auto writeIndices = getValueOrCreateConstantIndexOp(
-      rewriter, sliceOp.getLoc(), sliceOp.getMixedOffsets());
-
-  Operation *write = rewriter.create<vector::TransferWriteOp>(
-      sliceOp.getLoc(), read->getResult(0), sliceOp.getDest(), writeIndices,
-      ArrayRef<bool>{writeInBounds});
-
-  if (maskOp) {
-    write = mlir::vector::maskOperation(rewriter, write, maskOp);
-  }
+      vecType.getRank(), rewriter.create<arith::ConstantIndexOp>(loc, 0));
+  Value read = mlir::vector::createReadOrMaskedRead(
+      rewriter, loc, source, vecType.getShape(), padValue,
+      /*useInBoundsInsteadOfMasking=*/inputVectorSizes.empty());
+
+  // Create write
+  auto writeIndices =
+      getValueOrCreateConstantIndexOp(rewriter, loc, sliceOp.getMixedOffsets());
+  Operation *write =
+      createWriteOrMaskedWrite(rewriter, loc, read, sliceOp.getDest(),
+                               writeIndices, inputVectorSizes.empty());
 
   // 4. Finalize
   newResults.push_back(write->getResult(0));
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index c6ebd3a53d98..e4221d474841 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -904,6 +904,10 @@ static bool hasAllOneValues(DenseIntElementsAttr attr) {
 static FailureOr<Operation *>
 winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
                      int64_t m, int64_t r) {
+  if (!convOp.hasPureTensorSemantics())
+    return rewriter.notifyMatchFailure(
+        convOp, "expected pure tensor semantics for linalg.conv_2d_nhwc_fhwc");
+
   Value input = convOp.getInputs()[0];
   Value filter = convOp.getInputs()[1];
   Value output = convOp.getOutputs()[0];
diff --git a/external/llvm-project/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/external/llvm-project/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index bae06c003fd9..2527d90cfa2e 100644
--- a/external/llvm-project/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -688,7 +688,7 @@ computeSliceParameters(OpBuilder &builder, Location loc, Value valueToTile,
     //    tensors with "0" dimensions would never be constructed.
     int64_t shapeSize = shape[r];
     std::optional<int64_t> sizeCst = getConstantIntValue(size);
-    auto hasTileSizeOne = sizeCst && *sizeCst == 1;
+    auto hasTileSizeOne = sizeCst == 1;
     auto dividesEvenly = sizeCst && !ShapedType::isDynamic(shapeSize) &&
                          ((shapeSize % *sizeCst) == 0);
     if (!hasTileSizeOne && !dividesEvenly) {
diff --git a/external/llvm-project/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/external/llvm-project/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index aa9587510670..d56b32193765 100644
--- a/external/llvm-project/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -533,6 +533,15 @@ void AssumeAlignmentOp::getAsmResultNames(
   setNameFn(getResult(), "assume_align");
 }
 
+OpFoldResult AssumeAlignmentOp::fold(FoldAdaptor adaptor) {
+  auto source = getMemref().getDefiningOp<AssumeAlignmentOp>();
+  if (!source)
+    return {};
+  if (source.getAlignment() != getAlignment())
+    return {};
+  return getMemref();
+}
+
 //===----------------------------------------------------------------------===//
 // CastOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/external/llvm-project/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 658ad28477ac..0dfead98b7e7 100644
--- a/external/llvm-project/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -2461,10 +2461,34 @@ LogicalResult acc::LoopOp::verify() {
   if (hasDuplicateDeviceTypes(getAuto_(), deviceTypes) ||
       hasDuplicateDeviceTypes(getIndependent(), deviceTypes) ||
       hasDuplicateDeviceTypes(getSeq(), deviceTypes)) {
-    return emitError() << "only one of \"" << acc::LoopOp::getAutoAttrStrName()
-                       << "\", " << getIndependentAttrName() << ", "
-                       << getSeqAttrName()
-                       << " can be present at the same time";
+    return emitError() << "only one of auto, independent, seq can be present "
+                          "at the same time";
+  }
+
+  // Check that at least one of auto, independent, or seq is present
+  // for the device-independent default clauses.
+  auto hasDeviceNone = [](mlir::acc::DeviceTypeAttr attr) -> bool {
+    return attr.getValue() == mlir::acc::DeviceType::None;
+  };
+  bool hasDefaultSeq =
+      getSeqAttr()
+          ? llvm::any_of(getSeqAttr().getAsRange<mlir::acc::DeviceTypeAttr>(),
+                         hasDeviceNone)
+          : false;
+  bool hasDefaultIndependent =
+      getIndependentAttr()
+          ? llvm::any_of(
+                getIndependentAttr().getAsRange<mlir::acc::DeviceTypeAttr>(),
+                hasDeviceNone)
+          : false;
+  bool hasDefaultAuto =
+      getAuto_Attr()
+          ? llvm::any_of(getAuto_Attr().getAsRange<mlir::acc::DeviceTypeAttr>(),
+                         hasDeviceNone)
+          : false;
+  if (!hasDefaultSeq && !hasDefaultIndependent && !hasDefaultAuto) {
+    return emitError()
+           << "at least one of auto, independent, seq must be present";
   }
 
   // Gang, worker and vector are incompatible with seq.
@@ -2482,8 +2506,7 @@ LogicalResult acc::LoopOp::verify() {
                        deviceTypeAttr.getValue()) ||
           getGangValue(mlir::acc::GangArgType::Static,
                        deviceTypeAttr.getValue()))
-        return emitError()
-               << "gang, worker or vector cannot appear with the seq attr";
+        return emitError() << "gang, worker or vector cannot appear with seq";
     }
   }
 
@@ -2839,6 +2862,30 @@ void acc::LoopOp::addEmptyGang(
                                                  effectiveDeviceTypes));
 }
 
+bool acc::LoopOp::hasParallelismFlag(DeviceType dt) {
+  auto hasDevice = [=](DeviceTypeAttr attr) -> bool {
+    return attr.getValue() == dt;
+  };
+  auto testFromArr = [=](ArrayAttr arr) -> bool {
+    return llvm::any_of(arr.getAsRange<DeviceTypeAttr>(), hasDevice);
+  };
+
+  if (ArrayAttr arr = getSeqAttr(); arr && testFromArr(arr))
+    return true;
+  if (ArrayAttr arr = getIndependentAttr(); arr && testFromArr(arr))
+    return true;
+  if (ArrayAttr arr = getAuto_Attr(); arr && testFromArr(arr))
+    return true;
+
+  return false;
+}
+
+bool acc::LoopOp::hasDefaultGangWorkerVector() {
+  return hasVector() || getVectorValue() || hasWorker() || getWorkerValue() ||
+         hasGang() || getGangValue(GangArgType::Num) ||
+         getGangValue(GangArgType::Dim) || getGangValue(GangArgType::Static);
+}
+
 void acc::LoopOp::addGangOperands(
     MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes,
     llvm::ArrayRef<GangArgType> argTypes, mlir::ValueRange values) {
@@ -3820,3 +3867,14 @@ mlir::acc::getMutableDataOperands(mlir::Operation *accOp) {
           .Default([&](mlir::Operation *) { return nullptr; })};
   return dataOperands;
 }
+
+mlir::Operation *mlir::acc::getEnclosingComputeOp(mlir::Region &region) {
+  mlir::Operation *parentOp = region.getParentOp();
+  while (parentOp) {
+    if (mlir::isa<ACC_COMPUTE_CONSTRUCT_OPS>(parentOp)) {
+      return parentOp;
+    }
+    parentOp = parentOp->getParentOp();
+  }
+  return nullptr;
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/external/llvm-project/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 57ee0f52e749..3f29dd3ac5e4 100644
--- a/external/llvm-project/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -1058,48 +1058,54 @@ mlir::scf::tileUsingSCF(RewriterBase &rewriter, TilingInterface op,
   assert(succeeded(tilingResult) &&
          "expected tiling result to be computed after loop generation");
 
-  SmallVector<Value> partialResults;
   if (loops.empty()) {
     // If loops are empty, the tiled op is used as the replacement for the
     // untiled op.
-    partialResults = tilingResult->tiledValues;
-  } else {
-    partialResults = llvm::map_to_vector(loops.front()->getResults(),
+    return scf::SCFTilingResult{tilingResult->tiledOps,
+                                initTensors,
+                                loops,
+                                tilingResult->tiledValues,
+                                tilingResult->generatedSlices,
+                                {}};
+  }
+
+  auto loopResults = llvm::map_to_vector(loops.front()->getResults(),
                                          [](OpResult r) -> Value { return r; });
+
+  // For the full reduction case, there is nothing more to do.
+  if (options.reductionStrategy ==
+      scf::SCFTilingOptions::ReductionTilingStrategy::FullReduction) {
+    return scf::SCFTilingResult{
+        tilingResult->tiledOps,        initTensors, loops, loopResults,
+        tilingResult->generatedSlices, {}};
   }
 
+  // The results of the loop needs to be merged.
   FailureOr<MergeResult> mergeResult =
-      mergeTilingResults(rewriter, op, partialResults, options);
+      mergeTilingResults(rewriter, op, loopResults, options);
   if (failed(mergeResult)) {
     return rewriter.notifyMatchFailure(
         op, "Failed to merge partial results from tiling");
   }
-
-  return scf::SCFTilingResult{tilingResult->tiledOps, initTensors, loops,
-                              mergeResult.value(),
-                              tilingResult->generatedSlices};
+  return scf::SCFTilingResult{tilingResult->tiledOps,
+                              initTensors,
+                              loops,
+                              mergeResult->replacements,
+                              tilingResult->generatedSlices,
+                              mergeResult->mergeOps};
 }
 
 FailureOr<scf::SCFTilingResult>
 mlir::scf::tileReductionUsingScf(RewriterBase &b,
                                  PartialReductionOpInterface op,
-                                 ArrayRef<OpFoldResult> tileSizes) {
-  SCFTilingOptions options;
-  options.setLoopType(SCFTilingOptions::LoopType::ForOp);
-  options.setReductionTilingStrategy(SCFTilingOptions::ReductionTilingStrategy::
-                                         PartialReductionOuterReduction);
-  options.setTileSizes(tileSizes);
-
-  TilingInterface tilingInterfaceOp =
-      dyn_cast<TilingInterface>(op.getOperation());
-  if (!tilingInterfaceOp) {
-    return b.notifyMatchFailure(
-        op,
-        "Operation implementing PartialReductionOpInterface should implement "
-        "TilingInterface");
-  }
-
-  return tileUsingSCF(b, tilingInterfaceOp, options);
+                                 ArrayRef<OpFoldResult> tileSize) {
+  scf::SCFTilingOptions options;
+  options.setLoopType(scf::SCFTilingOptions::LoopType::ForOp);
+  options.setReductionTilingStrategy(
+      scf::SCFTilingOptions::ReductionTilingStrategy::
+          PartialReductionOuterReduction);
+  options.setTileSizes(tileSize);
+  return tileUsingSCF(b, op, options);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1539,8 +1545,8 @@ mlir::scf::tileConsumerAndFuseProducersUsingSCF(
   tiledAndFusedOps.insert_range(tilingResult->tiledOps);
 
   DenseMap<Value, Value> replacements;
-  for (auto [origVal, replacement] : llvm::zip_equal(
-           consumer->getResults(), tilingResult->mergeResult.replacements)) {
+  for (auto [origVal, replacement] :
+       llvm::zip_equal(consumer->getResults(), tilingResult->replacements)) {
     replacements[origVal] = replacement;
   }
 
diff --git a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/ImageOps.cpp b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/ImageOps.cpp
index a021931425fb..f7af79ceefa8 100644
--- a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/ImageOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/ImageOps.cpp
@@ -204,6 +204,23 @@ LogicalResult spirv::ImageDrefGatherOp::verify() {
                              getOperandArguments());
 }
 
+//===----------------------------------------------------------------------===//
+// spirv.ImageReadOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult spirv::ImageReadOp::verify() {
+  // TODO: Do we need check for: "If the Arrayed operand is 1, then additional
+  // capabilities may be required; e.g., ImageCubeArray, or ImageMSArray."?
+
+  // TODO: Ideally it should be somewhere verified that "If the Image Dim
+  // operand is not SubpassData, the Image Format must not be Unknown, unless
+  // the StorageImageReadWithoutFormat Capability was declared." This function
+  // however may not be the suitable place for such verification.
+
+  return verifyImageOperands(getOperation(), getImageOperandsAttr(),
+                             getOperandArguments());
+}
+
 //===----------------------------------------------------------------------===//
 // spirv.ImageWriteOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index e36d4b910193..03af61c81ae6 100644
--- a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -326,7 +326,6 @@ void spirv::UMulExtendedOp::getCanonicalizationPatterns(
 
 // The transformation is only applied if one divisor is a multiple of the other.
 
-// TODO(https://github.com/llvm/llvm-project/issues/63174): Add support for vector constants
 struct UModSimplification final : OpRewritePattern<spirv::UModOp> {
   using OpRewritePattern::OpRewritePattern;
 
@@ -336,19 +335,29 @@ struct UModSimplification final : OpRewritePattern<spirv::UModOp> {
     if (!prevUMod)
       return failure();
 
-    IntegerAttr prevValue;
-    IntegerAttr currValue;
+    TypedAttr prevValue;
+    TypedAttr currValue;
     if (!matchPattern(prevUMod.getOperand(1), m_Constant(&prevValue)) ||
         !matchPattern(umodOp.getOperand(1), m_Constant(&currValue)))
       return failure();
 
-    APInt prevConstValue = prevValue.getValue();
-    APInt currConstValue = currValue.getValue();
+    // Ensure that previous divisor is a multiple of the current divisor. If
+    // not, fail the transformation.
+    bool isApplicable = false;
+    if (auto prevInt = dyn_cast<IntegerAttr>(prevValue)) {
+      auto currInt = cast<IntegerAttr>(currValue);
+      isApplicable = prevInt.getValue().urem(currInt.getValue()) == 0;
+    } else if (auto prevVec = dyn_cast<DenseElementsAttr>(prevValue)) {
+      auto currVec = cast<DenseElementsAttr>(currValue);
+      isApplicable = llvm::all_of(llvm::zip_equal(prevVec.getValues<APInt>(),
+                                                  currVec.getValues<APInt>()),
+                                  [](const auto &pair) {
+                                    auto &[prev, curr] = pair;
+                                    return prev.urem(curr) == 0;
+                                  });
+    }
 
-    // Ensure that one divisor is a multiple of the other. If not, fail the
-    // transformation.
-    if (prevConstValue.urem(currConstValue) != 0 &&
-        currConstValue.urem(prevConstValue) != 0)
+    if (!isApplicable)
       return failure();
 
     // The transformation is safe. Replace the existing UMod operation with a
diff --git a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
index 0cf5f0823be6..a21acef1c4b4 100644
--- a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
@@ -175,10 +175,7 @@ static Type parseAndVerifyType(SPIRVDialect const &dialect,
 
   // Check other allowed types
   if (auto t = llvm::dyn_cast<FloatType>(type)) {
-    if (type.isBF16()) {
-      parser.emitError(typeLoc, "cannot use 'bf16' to compose SPIR-V types");
-      return Type();
-    }
+    // TODO: All float types are allowed for now, but this should be fixed.
   } else if (auto t = llvm::dyn_cast<IntegerType>(type)) {
     if (!ScalarType::isValid(t)) {
       parser.emitError(typeLoc,
diff --git a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 3d2cb1dd7a03..7148027dae78 100644
--- a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -558,6 +558,13 @@ void spirv::ConstantOp::print(OpAsmPrinter &printer) {
 
 static LogicalResult verifyConstantType(spirv::ConstantOp op, Attribute value,
                                         Type opType) {
+  if (isa<spirv::CooperativeMatrixType>(opType)) {
+    auto denseAttr = dyn_cast<DenseElementsAttr>(value);
+    if (!denseAttr || !denseAttr.isSplat())
+      return op.emitOpError("expected a splat dense attribute for cooperative "
+                            "matrix constant, but found ")
+             << denseAttr;
+  }
   if (llvm::isa<IntegerAttr, FloatAttr>(value)) {
     auto valueType = llvm::cast<TypedAttr>(value).getType();
     if (valueType != opType)
diff --git a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
index 337df3a5a65f..93e0c9b33c54 100644
--- a/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
@@ -194,8 +194,21 @@ std::optional<int64_t> CompositeType::getSizeInBytes() {
 //===----------------------------------------------------------------------===//
 
 struct spirv::detail::CooperativeMatrixTypeStorage final : TypeStorage {
+  // In the specification dimensions of the Cooperative Matrix are 32-bit
+  // integers --- the initial implementation kept those values as such. However,
+  // the `ShapedType` expects the shape to be `int64_t`. We could keep the shape
+  // as 32-bits and expose it as int64_t through `getShape`, however, this
+  // method returns an `ArrayRef`, so returning `ArrayRef<int64_t>` having two
+  // 32-bits integers would require an extra logic and storage. So, we diverge
+  // from the spec and internally represent the dimensions as 64-bit integers,
+  // so we can easily return an `ArrayRef` from `getShape` without any extra
+  // logic. Alternatively, we could store both rows and columns (both 32-bits)
+  // and shape (64-bits), assigning rows and columns to shape whenever
+  // `getShape` is called. This would be at the cost of extra logic and storage.
+  // Note: Because `ArrayRef` is returned we cannot construct an object in
+  // `getShape` on the fly.
   using KeyTy =
-      std::tuple<Type, uint32_t, uint32_t, Scope, CooperativeMatrixUseKHR>;
+      std::tuple<Type, int64_t, int64_t, Scope, CooperativeMatrixUseKHR>;
 
   static CooperativeMatrixTypeStorage *
   construct(TypeStorageAllocator &allocator, const KeyTy &key) {
@@ -204,17 +217,17 @@ struct spirv::detail::CooperativeMatrixTypeStorage final : TypeStorage {
   }
 
   bool operator==(const KeyTy &key) const {
-    return key == KeyTy(elementType, rows, columns, scope, use);
+    return key == KeyTy(elementType, shape[0], shape[1], scope, use);
   }
 
   CooperativeMatrixTypeStorage(const KeyTy &key)
-      : elementType(std::get<0>(key)), rows(std::get<1>(key)),
-        columns(std::get<2>(key)), scope(std::get<3>(key)),
+      : elementType(std::get<0>(key)),
+        shape({std::get<1>(key), std::get<2>(key)}), scope(std::get<3>(key)),
         use(std::get<4>(key)) {}
 
   Type elementType;
-  uint32_t rows;
-  uint32_t columns;
+  // [#rows, #columns]
+  std::array<int64_t, 2> shape;
   Scope scope;
   CooperativeMatrixUseKHR use;
 };
@@ -231,10 +244,18 @@ Type CooperativeMatrixType::getElementType() const {
   return getImpl()->elementType;
 }
 
-uint32_t CooperativeMatrixType::getRows() const { return getImpl()->rows; }
+uint32_t CooperativeMatrixType::getRows() const {
+  assert(getImpl()->shape[0] != ShapedType::kDynamic);
+  return static_cast<uint32_t>(getImpl()->shape[0]);
+}
 
 uint32_t CooperativeMatrixType::getColumns() const {
-  return getImpl()->columns;
+  assert(getImpl()->shape[1] != ShapedType::kDynamic);
+  return static_cast<uint32_t>(getImpl()->shape[1]);
+}
+
+ArrayRef<int64_t> CooperativeMatrixType::getShape() const {
+  return getImpl()->shape;
 }
 
 Scope CooperativeMatrixType::getScope() const { return getImpl()->scope; }
@@ -505,7 +526,7 @@ bool ScalarType::classof(Type type) {
 }
 
 bool ScalarType::isValid(FloatType type) {
-  return llvm::is_contained({16u, 32u, 64u}, type.getWidth()) && !type.isBF16();
+  return llvm::is_contained({16u, 32u, 64u}, type.getWidth());
 }
 
 bool ScalarType::isValid(IntegerType type) {
@@ -514,6 +535,11 @@ bool ScalarType::isValid(IntegerType type) {
 
 void ScalarType::getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
                                std::optional<StorageClass> storage) {
+  if (isa<BFloat16Type>(*this)) {
+    static const Extension ext = Extension::SPV_KHR_bfloat16;
+    extensions.push_back(ext);
+  }
+
   // 8- or 16-bit integer/floating-point numbers will require extra extensions
   // to appear in interface storage classes. See SPV_KHR_16bit_storage and
   // SPV_KHR_8bit_storage for more details.
@@ -619,7 +645,16 @@ void ScalarType::getCapabilities(
   } else {
     assert(llvm::isa<FloatType>(*this));
     switch (bitwidth) {
-      WIDTH_CASE(Float, 16);
+    case 16: {
+      if (isa<BFloat16Type>(*this)) {
+        static const Capability cap = Capability::BFloat16TypeKHR;
+        capabilities.push_back(cap);
+      } else {
+        static const Capability cap = Capability::Float16;
+        capabilities.push_back(cap);
+      }
+      break;
+    }
       WIDTH_CASE(Float, 64);
     case 32:
       break;
diff --git a/external/llvm-project/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/external/llvm-project/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index 62a24646d066..1e7bb046d375 100644
--- a/external/llvm-project/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -737,7 +737,7 @@ static spirv::GlobalVariableOp getBuiltinVariable(Block &body,
             spirv::SPIRVDialect::getAttributeName(
                 spirv::Decoration::BuiltIn))) {
       auto varBuiltIn = spirv::symbolizeBuiltIn(builtinAttr.getValue());
-      if (varBuiltIn && *varBuiltIn == builtin) {
+      if (varBuiltIn == builtin) {
         return varOp;
       }
     }
@@ -1020,22 +1020,22 @@ struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
     SmallVector<Location> locs(convertedTypes.size(), newFuncOp.getLoc());
     entryBlock.addArguments(convertedTypes, locs);
 
-    // Replace the placeholder values with the new arguments. We assume there is
-    // only one block for now.
+    // Replace all uses of placeholders for initially legal arguments with their
+    // original function arguments (that were added to `newFuncOp`).
+    for (auto &[placeholderOp, argIdx] : tmpOps) {
+      if (!placeholderOp)
+        continue;
+      Value replacement = newFuncOp.getArgument(argIdx);
+      rewriter.replaceAllUsesWith(placeholderOp->getResult(0), replacement);
+    }
+
+    // Replace dummy operands of new `vector.insert_strided_slice` ops with
+    // their corresponding new function arguments. The new
+    // `vector.insert_strided_slice` ops are inserted only into the entry block,
+    // so iterating over that block is sufficient.
     size_t unrolledInputIdx = 0;
     for (auto [count, op] : enumerate(entryBlock.getOperations())) {
-      // We first look for operands that are placeholders for initially legal
-      // arguments.
       Operation &curOp = op;
-      for (auto [operandIdx, operandVal] : llvm::enumerate(op.getOperands())) {
-        Operation *operandOp = operandVal.getDefiningOp();
-        if (auto it = tmpOps.find(operandOp); it != tmpOps.end()) {
-          size_t idx = operandIdx;
-          rewriter.modifyOpInPlace(&curOp, [&curOp, &newFuncOp, it, idx] {
-            curOp.setOperand(idx, newFuncOp.getArgument(it->second));
-          });
-        }
-      }
       // Since all newly created operations are in the beginning, reaching the
       // end of them means that any later `vector.insert_strided_slice` should
       // not be touched.
diff --git a/external/llvm-project/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/external/llvm-project/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 219619981629..34ae83b25c39 100644
--- a/external/llvm-project/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -799,8 +799,7 @@ LogicalResult SparseTensorEncodingAttr::verify(
                             "before singleton level";
 
     auto *curCOOEnd = std::find_if_not(it, lvlTypes.end(), isSingletonLT);
-    if (!std::all_of(it, curCOOEnd,
-                     [](LevelType i) { return isSingletonLT(i); }))
+    if (!std::all_of(it, curCOOEnd, isSingletonLT))
       return emitError() << "expected all singleton lvlTypes "
                             "following a singleton level";
     // We can potentially support mixed SoA/AoS singleton levels.
@@ -833,8 +832,7 @@ LogicalResult SparseTensorEncodingAttr::verify(
       it != std::end(lvlTypes)) {
     if (it != lvlTypes.end() - 1)
       return emitError() << "expected n_out_of_m to be the last level type";
-    if (!std::all_of(lvlTypes.begin(), it,
-                     [](LevelType i) { return isDenseLT(i); }))
+    if (!std::all_of(lvlTypes.begin(), it, isDenseLT))
       return emitError() << "expected all dense lvlTypes "
                             "before a n_out_of_m level";
     if (dimToLvl && (dimToLvl.getNumDims() != dimToLvl.getNumResults())) {
diff --git a/external/llvm-project/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/external/llvm-project/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 3d963dea2f57..359590f2434d 100644
--- a/external/llvm-project/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -198,14 +198,14 @@ static Value genVectorReducInit(PatternRewriter &rewriter, Location loc,
   case vector::CombiningKind::ADD:
   case vector::CombiningKind::XOR:
     // Initialize reduction vector to: | 0 | .. | 0 | r |
-    return rewriter.create<vector::InsertElementOp>(
-        loc, r, constantZero(rewriter, loc, vtp),
-        constantIndex(rewriter, loc, 0));
+    return rewriter.create<vector::InsertOp>(loc, r,
+                                             constantZero(rewriter, loc, vtp),
+                                             constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::MUL:
     // Initialize reduction vector to: | 1 | .. | 1 | r |
-    return rewriter.create<vector::InsertElementOp>(
-        loc, r, constantOne(rewriter, loc, vtp),
-        constantIndex(rewriter, loc, 0));
+    return rewriter.create<vector::InsertOp>(loc, r,
+                                             constantOne(rewriter, loc, vtp),
+                                             constantIndex(rewriter, loc, 0));
   case vector::CombiningKind::AND:
   case vector::CombiningKind::OR:
     // Initialize reduction vector to: | r | .. | r | r |
@@ -628,31 +628,49 @@ struct ForOpRewriter : public OpRewritePattern<scf::ForOp> {
   const VL vl;
 };
 
+static LogicalResult cleanReducChain(PatternRewriter &rewriter, Operation *op,
+                                     Value inp) {
+  if (auto redOp = inp.getDefiningOp<vector::ReductionOp>()) {
+    if (auto forOp = redOp.getVector().getDefiningOp<scf::ForOp>()) {
+      if (forOp->hasAttr(LoopEmitter::getLoopEmitterLoopAttrName())) {
+        rewriter.replaceOp(op, redOp.getVector());
+        return success();
+      }
+    }
+  }
+  return failure();
+}
+
 /// Reduction chain cleanup.
 ///   v = for { }
-///   s = vsum(v)               v = for { }
-///   u = expand(s)       ->    for (v) { }
+///   s = vsum(v)                  v = for { }
+///   u = broadcast(s)       ->    for (v) { }
 ///   for (u) { }
-template <typename VectorOp>
-struct ReducChainRewriter : public OpRewritePattern<VectorOp> {
+struct ReducChainBroadcastRewriter
+    : public OpRewritePattern<vector::BroadcastOp> {
 public:
-  using OpRewritePattern<VectorOp>::OpRewritePattern;
+  using OpRewritePattern<vector::BroadcastOp>::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(VectorOp op,
+  LogicalResult matchAndRewrite(vector::BroadcastOp op,
                                 PatternRewriter &rewriter) const override {
-    Value inp = op.getSource();
-    if (auto redOp = inp.getDefiningOp<vector::ReductionOp>()) {
-      if (auto forOp = redOp.getVector().getDefiningOp<scf::ForOp>()) {
-        if (forOp->hasAttr(LoopEmitter::getLoopEmitterLoopAttrName())) {
-          rewriter.replaceOp(op, redOp.getVector());
-          return success();
-        }
-      }
-    }
-    return failure();
+    return cleanReducChain(rewriter, op, op.getSource());
   }
 };
 
+/// Reduction chain cleanup.
+///   v = for { }
+///   s = vsum(v)               v = for { }
+///   u = insert(s)       ->    for (v) { }
+///   for (u) { }
+struct ReducChainInsertRewriter : public OpRewritePattern<vector::InsertOp> {
+public:
+  using OpRewritePattern<vector::InsertOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::InsertOp op,
+                                PatternRewriter &rewriter) const override {
+    return cleanReducChain(rewriter, op, op.getValueToStore());
+  }
+};
 } // namespace
 
 //===----------------------------------------------------------------------===//
@@ -668,6 +686,6 @@ void mlir::populateSparseVectorizationPatterns(RewritePatternSet &patterns,
   vector::populateVectorStepLoweringPatterns(patterns);
   patterns.add<ForOpRewriter>(patterns.getContext(), vectorLength,
                               enableVLAVectorization, enableSIMDIndex32);
-  patterns.add<ReducChainRewriter<vector::InsertElementOp>,
-               ReducChainRewriter<vector::BroadcastOp>>(patterns.getContext());
+  patterns.add<ReducChainInsertRewriter, ReducChainBroadcastRewriter>(
+      patterns.getContext());
 }
diff --git a/external/llvm-project/mlir/lib/Dialect/Tensor/Extensions/AllExtensions.cpp b/external/llvm-project/mlir/lib/Dialect/Tensor/Extensions/AllExtensions.cpp
index 93e1a2021857..0421a6c0ff80 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tensor/Extensions/AllExtensions.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tensor/Extensions/AllExtensions.cpp
@@ -13,4 +13,4 @@ using namespace mlir;
 
 void mlir::tensor::registerAllExtensions(DialectRegistry &registry) {
   registerShardingInterfaceExternalModels(registry);
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
index 5425615dac39..66b73f9bead5 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
@@ -30,6 +30,7 @@ add_mlir_dialect_library(MLIRTensorDialect
   MLIRInferIntRangeInterface
   MLIRInferTypeOpInterface
   MLIRParallelCombiningOpInterface
+  MLIRRuntimeVerifiableOpInterface
   MLIRShapedOpInterfaces
   MLIRSideEffectInterfaces
   MLIRSupport
diff --git a/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp b/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp
index e7d8f52d309c..5b65e47bc937 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/Interfaces/RuntimeVerifiableOpInterface.h"
 #include "mlir/Interfaces/SubsetOpInterface.h"
 #include "mlir/Transforms/InliningUtils.h"
 
@@ -58,6 +59,8 @@ void TensorDialect::initialize() {
                             InsertSliceOp, ReshapeOp>();
   declarePromisedInterfaces<ReifyRankedShapedTypeOpInterface, ExpandShapeOp,
                             CollapseShapeOp, PadOp>();
+  declarePromisedInterfaces<RuntimeVerifiableOpInterface, CastOp, DimOp,
+                            ExtractOp, InsertOp, ExtractSliceOp>();
   declarePromisedInterfaces<SubsetOpInterface, ExtractSliceOp, InsertSliceOp,
                             ParallelInsertSliceOp>();
   declarePromisedInterfaces<SubsetInsertionOpInterface, InsertSliceOp,
diff --git a/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 30ca20fc0d88..04242cad9ecb 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -22,6 +22,7 @@
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/DestinationStyleOpInterface.h"
 #include "mlir/Interfaces/InferIntRangeInterface.h"
@@ -33,10 +34,12 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/MathExtras.h"
 #include <algorithm>
 #include <optional>
+#include <vector>
 
 using namespace mlir;
 using namespace mlir::tensor;
@@ -1058,8 +1061,7 @@ void DimOp::getCanonicalizationPatterns(RewritePatternSet &results,
 void EmptyOp::build(OpBuilder &builder, OperationState &result,
                     ArrayRef<int64_t> staticShape, Type elementType,
                     Attribute encoding) {
-  assert(all_of(staticShape,
-                [](int64_t sz) { return !ShapedType::isDynamic(sz); }) &&
+  assert(none_of(staticShape, ShapedType::isDynamic) &&
          "expected only static sizes");
   build(builder, result, staticShape, elementType, ValueRange{}, encoding);
 }
@@ -1288,6 +1290,68 @@ struct ExtractFromTensorCast : public OpRewritePattern<tensor::ExtractOp> {
   }
 };
 
+/// Canonicalizes the pattern of the form
+///
+/// %val = tensor.collapse_shape %src[[0, 1]] : tensor<3x4xf64> into
+/// tensor<12xf64>
+/// %extracted_element = tensor.extract %val[%c10] :
+/// tensor<12xf64>
+///
+/// to
+///
+/// %extracted_element = tensor.extract %src[%c2, %c2] : tensor<3x4xf64>
+struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
+  using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tensor::ExtractOp extractOp,
+                                PatternRewriter &rewriter) const final {
+    auto collapseOp =
+        extractOp.getTensor().getDefiningOp<tensor::CollapseShapeOp>();
+    if (!collapseOp)
+      return failure();
+    if (!collapseOp.getSrcType().hasStaticShape())
+      return failure();
+
+    auto sourceSizes = collapseOp.getSrcType().getShape();
+
+    SmallVector<Value> indices(extractOp.getIndices().begin(),
+                               extractOp.getIndices().end());
+    SmallVector<Value> sourceIndices;
+    for (auto [index, group] :
+         llvm::zip(indices, collapseOp.getReassociationIndices())) {
+      assert(!group.empty() && "association indices groups cannot be empty");
+      auto groupSize = group.size();
+
+      if (groupSize == 1) {
+        sourceIndices.push_back(index);
+        continue;
+      }
+
+      SmallVector<int64_t> basis =
+          llvm::map_to_vector(group, [&](int64_t d) { return sourceSizes[d]; });
+      auto delinearize = rewriter.create<affine::AffineDelinearizeIndexOp>(
+          extractOp.getLoc(), index, basis, /*hasOuterBound=*/true);
+      llvm::append_range(sourceIndices, delinearize.getResults());
+    }
+    if (collapseOp.getReassociationIndices().empty()) {
+      auto zeroAffineMap = rewriter.getConstantAffineMap(0);
+      int64_t srcRank =
+          cast<RankedTensorType>(collapseOp.getSrcType()).getRank();
+      OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
+          rewriter, extractOp.getLoc(), zeroAffineMap,
+          ArrayRef<OpFoldResult>{});
+      for (int64_t i = 0; i < srcRank; i++) {
+        sourceIndices.push_back(
+            getValueOrCreateConstantIndexOp(rewriter, extractOp.getLoc(), ofr));
+      }
+    }
+
+    rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
+        extractOp, collapseOp.getSrc(), sourceIndices);
+    return success();
+  }
+};
+
 } // namespace
 
 void ExtractOp::getAsmResultNames(
@@ -1303,6 +1367,23 @@ LogicalResult ExtractOp::verify() {
   return success();
 }
 
+/// If we have an ExtractOp consuming an InsertOp with the same
+/// indices, we can return the InsertOp's scalar directly.
+// TODO: This only checks the immediate producer; extend to go up the
+// insert/extract chain if the slices are disjoint.
+static Value foldExtractAfterInsert(ExtractOp extractOp) {
+  auto insertOp = extractOp.getTensor().getDefiningOp<InsertOp>();
+
+  auto isSame = [](Value a, Value b) {
+    return getAsOpFoldResult(a) == getAsOpFoldResult(b);
+  };
+  if (insertOp && insertOp.getScalar().getType() == extractOp.getType() &&
+      llvm::equal(insertOp.getIndices(), extractOp.getIndices(), isSame))
+    return insertOp.getScalar();
+
+  return {};
+}
+
 OpFoldResult ExtractOp::fold(FoldAdaptor adaptor) {
   if (Attribute tensor = adaptor.getTensor()) {
     // If this is a splat elements attribute, simply return the value.
@@ -1350,6 +1431,9 @@ OpFoldResult ExtractOp::fold(FoldAdaptor adaptor) {
       return elementsAttr.getValues<Attribute>()[indices];
   }
 
+  if (Value result = foldExtractAfterInsert(*this))
+    return result;
+
   return {};
 }
 
@@ -1358,6 +1442,11 @@ void ExtractOp::getCanonicalizationPatterns(RewritePatternSet &results,
   results.add<ExtractFromTensorCast>(context);
 }
 
+void mlir::tensor::populateFoldCollapseExtractPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<ExtractFromCollapseShape>(patterns.getContext());
+}
+
 //===----------------------------------------------------------------------===//
 // FromElementsOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
index 7880d1c5a0c5..99e1c4fec846 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
@@ -8,6 +8,7 @@ add_mlir_dialect_library(MLIRTensorTransforms
   MergeConsecutiveInsertExtractSlicePatterns.cpp
   ReshapePatterns.cpp
   RewriteAsConstant.cpp
+  RuntimeOpVerification.cpp
   SwapExtractSliceWithProducerPatterns.cpp
   SubsetInsertionOpInterfaceImpl.cpp
 
@@ -28,6 +29,7 @@ add_mlir_dialect_library(MLIRTensorTransforms
   MLIRLinalgDialect
   MLIRMemRefDialect
   MLIRPass
+  MLIRRuntimeVerifiableOpInterface
   MLIRSCFDialect
   MLIRSubsetOpInterface
   MLIRTensorDialect
diff --git a/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 2b229d60c691..657624b817af 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -185,13 +185,11 @@ struct BubbleUpExpandThroughParallelCollapse
         ArrayRef<int64_t> collapsedStaticShapes = staticSourceSize.slice(
             collapseReassociation.front(), collapseReassociation.size());
         int64_t numCollapsedDynamic =
-            llvm::count_if(collapsedStaticShapes,
-                           [](int64_t d) { return ShapedType::isDynamic(d); });
+            llvm::count_if(collapsedStaticShapes, ShapedType::isDynamic);
         ArrayRef<int64_t> expandedStaticShapes = staticResultSize.slice(
             expandReassociation.front(), expandReassociation.size());
         int64_t numExpandedDynamic =
-            llvm::count_if(expandedStaticShapes,
-                           [](int64_t d) { return ShapedType::isDynamic(d); });
+            llvm::count_if(expandedStaticShapes, ShapedType::isDynamic);
         if (numCollapsedDynamic > 1 || numExpandedDynamic > 1 ||
             collapsedStaticShapes != expandedStaticShapes) {
           return failure();
diff --git a/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp b/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
new file mode 100644
index 000000000000..6138821ee8c6
--- /dev/null
+++ b/external/llvm-project/mlir/lib/Dialect/Tensor/Transforms/RuntimeOpVerification.cpp
@@ -0,0 +1,208 @@
+//===- RuntimeOpVerification.cpp - Op Verification ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Tensor/Transforms/RuntimeOpVerification.h"
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Arith/Utils/Utils.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
+#include "mlir/Interfaces/RuntimeVerifiableOpInterface.h"
+
+using namespace mlir;
+
+namespace mlir {
+namespace tensor {
+namespace {
+/// Generate a runtime check for lb <= value < ub.
+Value generateInBoundsCheck(OpBuilder &builder, Location loc, Value value,
+                            Value lb, Value ub) {
+  Value inBounds1 = builder.createOrFold<arith::CmpIOp>(
+      loc, arith::CmpIPredicate::sge, value, lb);
+  Value inBounds2 = builder.createOrFold<arith::CmpIOp>(
+      loc, arith::CmpIPredicate::slt, value, ub);
+  Value inBounds =
+      builder.createOrFold<arith::AndIOp>(loc, inBounds1, inBounds2);
+  return inBounds;
+}
+
+struct CastOpInterface
+    : public RuntimeVerifiableOpInterface::ExternalModel<CastOpInterface,
+                                                         CastOp> {
+  void generateRuntimeVerification(Operation *op, OpBuilder &builder,
+                                   Location loc) const {
+    auto castOp = cast<CastOp>(op);
+    auto srcType = cast<TensorType>(castOp.getSource().getType());
+
+    // Nothing to check if the result is an unranked tensor.
+    auto resultType = dyn_cast<RankedTensorType>(castOp.getType());
+    if (!resultType)
+      return;
+
+    if (isa<UnrankedTensorType>(srcType)) {
+      // Check rank.
+      Value srcRank = builder.create<RankOp>(loc, castOp.getSource());
+      Value resultRank =
+          builder.create<arith::ConstantIndexOp>(loc, resultType.getRank());
+      Value isSameRank = builder.create<arith::CmpIOp>(
+          loc, arith::CmpIPredicate::eq, srcRank, resultRank);
+      builder.create<cf::AssertOp>(
+          loc, isSameRank,
+          RuntimeVerifiableOpInterface::generateErrorMessage(op,
+                                                             "rank mismatch"));
+    }
+
+    // Check dimension sizes.
+    for (const auto &it : llvm::enumerate(resultType.getShape())) {
+      // Static dim size -> static/dynamic dim size does not need verification.
+      if (auto rankedSrcType = dyn_cast<RankedTensorType>(srcType))
+        if (!rankedSrcType.isDynamicDim(it.index()))
+          continue;
+
+      // Static/dynamic dim size -> dynamic dim size does not need verification.
+      if (resultType.isDynamicDim(it.index()))
+        continue;
+
+      Value srcDimSz =
+          builder.create<DimOp>(loc, castOp.getSource(), it.index());
+      Value resultDimSz =
+          builder.create<arith::ConstantIndexOp>(loc, it.value());
+      Value isSameSz = builder.create<arith::CmpIOp>(
+          loc, arith::CmpIPredicate::eq, srcDimSz, resultDimSz);
+      builder.create<cf::AssertOp>(
+          loc, isSameSz,
+          RuntimeVerifiableOpInterface::generateErrorMessage(
+              op, "size mismatch of dim " + std::to_string(it.index())));
+    }
+  }
+};
+
+struct DimOpInterface
+    : public RuntimeVerifiableOpInterface::ExternalModel<DimOpInterface,
+                                                         DimOp> {
+  void generateRuntimeVerification(Operation *op, OpBuilder &builder,
+                                   Location loc) const {
+    auto dimOp = cast<DimOp>(op);
+    Value rank = builder.create<RankOp>(loc, dimOp.getSource());
+    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    builder.create<cf::AssertOp>(
+        loc, generateInBoundsCheck(builder, loc, dimOp.getIndex(), zero, rank),
+        RuntimeVerifiableOpInterface::generateErrorMessage(
+            op, "index is out of bounds"));
+  }
+};
+
+/// Verifies that the indices on extract/insert ops are in-bounds of the
+/// tensor's index space: 0 <= index#i < dim#i
+template <typename OpTy>
+struct ExtractInsertOpInterface
+    : public RuntimeVerifiableOpInterface::ExternalModel<
+          ExtractInsertOpInterface<OpTy>, OpTy> {
+  void generateRuntimeVerification(Operation *op, OpBuilder &builder,
+                                   Location loc) const {
+    auto extractInsertOp = cast<OpTy>(op);
+
+    Value tensor;
+    if constexpr (std::is_same_v<OpTy, ExtractOp>) {
+      tensor = extractInsertOp.getTensor();
+    } else if constexpr (std::is_same_v<OpTy, InsertOp>) {
+      tensor = extractInsertOp.getDest();
+    } else {
+      llvm_unreachable("invalid op");
+    }
+    auto tensorType = cast<RankedTensorType>(tensor.getType());
+    auto rank = tensorType.getRank();
+    if (rank == 0) {
+      // Nothing to check for 0-d tensors.
+      return;
+    }
+
+    auto indices = extractInsertOp.getIndices();
+    auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    Value assertCond;
+    for (auto i : llvm::seq<int64_t>(0, rank)) {
+      Value dimOp = builder.createOrFold<tensor::DimOp>(loc, tensor, i);
+      Value inBounds =
+          generateInBoundsCheck(builder, loc, indices[i], zero, dimOp);
+      assertCond =
+          i > 0 ? builder.createOrFold<arith::AndIOp>(loc, assertCond, inBounds)
+                : inBounds;
+    }
+    builder.create<cf::AssertOp>(
+        loc, assertCond,
+        RuntimeVerifiableOpInterface::generateErrorMessage(
+            op, "out-of-bounds access"));
+  }
+};
+
+struct ExtractSliceOpInterface
+    : public RuntimeVerifiableOpInterface::ExternalModel<
+          ExtractSliceOpInterface, ExtractSliceOp> {
+  void generateRuntimeVerification(Operation *op, OpBuilder &builder,
+                                   Location loc) const {
+    auto extractSliceOp = cast<ExtractSliceOp>(op);
+    RankedTensorType sourceType = extractSliceOp.getSource().getType();
+
+    // For each dimension, assert that:
+    // 0 <= offset < dim_size
+    // 0 <= offset + (size - 1) * stride < dim_size
+    Value zero = builder.create<arith::ConstantIndexOp>(loc, 0);
+    Value one = builder.create<arith::ConstantIndexOp>(loc, 1);
+    for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
+      Value offset = getValueOrCreateConstantIndexOp(
+          builder, loc, extractSliceOp.getMixedOffsets()[i]);
+      Value size = getValueOrCreateConstantIndexOp(
+          builder, loc, extractSliceOp.getMixedSizes()[i]);
+      Value stride = getValueOrCreateConstantIndexOp(
+          builder, loc, extractSliceOp.getMixedStrides()[i]);
+
+      // Verify that offset is in-bounds.
+      Value dimSize = builder.createOrFold<tensor::DimOp>(
+          loc, extractSliceOp.getSource(), i);
+      Value offsetInBounds =
+          generateInBoundsCheck(builder, loc, offset, zero, dimSize);
+      builder.create<cf::AssertOp>(
+          loc, offsetInBounds,
+          RuntimeVerifiableOpInterface::generateErrorMessage(
+              op, "offset " + std::to_string(i) + " is out-of-bounds"));
+
+      // Verify that slice does not run out-of-bounds.
+      Value sizeMinusOne = builder.create<arith::SubIOp>(loc, size, one);
+      Value sizeMinusOneTimesStride =
+          builder.create<arith::MulIOp>(loc, sizeMinusOne, stride);
+      Value lastPos =
+          builder.create<arith::AddIOp>(loc, offset, sizeMinusOneTimesStride);
+      Value lastPosInBounds =
+          generateInBoundsCheck(builder, loc, lastPos, zero, dimSize);
+      builder.create<cf::AssertOp>(
+          loc, lastPosInBounds,
+          RuntimeVerifiableOpInterface::generateErrorMessage(
+              op, "extract_slice runs out-of-bounds along dimension " +
+                      std::to_string(i)));
+    }
+  }
+};
+} // namespace
+} // namespace tensor
+} // namespace mlir
+
+void mlir::tensor::registerRuntimeVerifiableOpInterfaceExternalModels(
+    DialectRegistry &registry) {
+  registry.addExtension(+[](MLIRContext *ctx, tensor::TensorDialect *dialect) {
+    CastOp::attachInterface<CastOpInterface>(*ctx);
+    DimOp::attachInterface<DimOpInterface>(*ctx);
+    ExtractOp::attachInterface<ExtractInsertOpInterface<ExtractOp>>(*ctx);
+    ExtractSliceOp::attachInterface<ExtractSliceOpInterface>(*ctx);
+    InsertOp::attachInterface<ExtractInsertOpInterface<InsertOp>>(*ctx);
+
+    // Load additional dialects of which ops may get created.
+    ctx->loadDialect<arith::ArithDialect, cf::ControlFlowDialect>();
+  });
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/external/llvm-project/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index e0aab9ad1930..f5333a3bb07c 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -131,6 +131,24 @@ SmallVector<Region *> tosa::WhileOp::getLoopRegions() {
   return {&getBodyGraph()};
 }
 
+//===----------------------------------------------------------------------===//
+// TOSA variable operator support.
+//===----------------------------------------------------------------------===//
+
+static SmallVector<int64_t> convertToMlirShape(ArrayRef<int64_t> shape) {
+  return to_vector(llvm::map_range(shape, [](int64_t dim) {
+    return dim == -1 ? ShapedType::kDynamic : dim;
+  }));
+}
+
+// returns type of variable op
+RankedTensorType mlir::tosa::getVariableType(tosa::VariableOp variableOp) {
+  Type elementType = variableOp.getType();
+  DenseIntElementsAttr varShapeAttr = variableOp.getVarShape();
+  auto shape = convertToMlirShape(to_vector(varShapeAttr.getValues<int64_t>()));
+  return RankedTensorType::get(shape, elementType);
+}
+
 //===----------------------------------------------------------------------===//
 // Tosa dialect initialization.
 //===----------------------------------------------------------------------===//
@@ -177,42 +195,80 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
 // Parsers and printers
 //===----------------------------------------------------------------------===//
 
-ParseResult mlir::tosa::parseTypeOrAttr(OpAsmParser &parser, TypeAttr &typeAttr,
-                                        Attribute &attr) {
+namespace {
+
+ParseResult getShapeAndElementType(OpAsmParser &parser, Type parsedType,
+                                   DenseElementsAttr &varShapeAttr,
+                                   TypeAttr &typeAttr) {
+  if (auto shapedType = dyn_cast<ShapedType>(parsedType)) {
+    if (!shapedType.hasRank())
+      return parser.emitError(parser.getCurrentLocation())
+             << "expected ranked type";
+
+    auto elementType = shapedType.getElementType();
+    typeAttr = TypeAttr::get(elementType);
+    ArrayRef<int64_t> shape = shapedType.getShape();
+    Builder builder(parser.getContext());
+    varShapeAttr = builder.getIndexTensorAttr(convertFromMlirShape(shape));
+    return success();
+  }
+  return parser.emitError(parser.getCurrentLocation())
+         << "expected shaped type";
+}
+
+} // namespace
+
+// parses the optional initial value or type for a tosa variable
+//  with initial value:
+//    tosa.variable @name = dense<0.0> : tensor<1x8xf32>
+//
+//  without initial value:
+//    tosa.variable @name : tensor<1x8xf32>
+ParseResult mlir::tosa::parseVariableOpTypeOrInitialValue(
+    OpAsmParser &parser, DenseElementsAttr &varShapeAttr, TypeAttr &typeAttr,
+    Attribute &initialValueAttr) {
   if (succeeded(parser.parseOptionalEqual())) {
-    if (failed(parser.parseAttribute(attr))) {
+    if (failed(parser.parseAttribute(initialValueAttr))) {
       return parser.emitError(parser.getCurrentLocation())
              << "expected attribute";
     }
-    if (auto typedAttr = dyn_cast<TypedAttr>(attr)) {
-      typeAttr = TypeAttr::get(typedAttr.getType());
+    if (auto typedAttr = dyn_cast<TypedAttr>(initialValueAttr)) {
+      return getShapeAndElementType(parser, typedAttr.getType(), varShapeAttr,
+                                    typeAttr);
     }
-    return success();
+    return parser.emitError(parser.getCurrentLocation())
+           << "expected Typed attr";
   }
 
-  Type type;
-  if (failed(parser.parseColonType(type))) {
-    return parser.emitError(parser.getCurrentLocation()) << "expected type";
+  initialValueAttr = nullptr;
+  Type parsedType;
+  if (failed(parser.parseColonType(parsedType))) {
+    return parser.emitError(parser.getCurrentLocation())
+           << "expected type after colon";
   }
-  typeAttr = TypeAttr::get(type);
-
-  return success();
+  return getShapeAndElementType(parser, parsedType, varShapeAttr, typeAttr);
 }
 
-void mlir::tosa::printTypeOrAttr(OpAsmPrinter &p, Operation *op, TypeAttr type,
-                                 Attribute attr) {
+void mlir::tosa::printVariableOpTypeOrInitialValue(
+    OpAsmPrinter &p, Operation *op, DenseElementsAttr varShapeAttr,
+    TypeAttr typeAttr, Attribute initialValueAttr) {
   bool needsSpace = false;
-  auto typedAttr = dyn_cast_or_null<TypedAttr>(attr);
-  if (!typedAttr || typedAttr.getType() != type.getValue()) {
+  if (!dyn_cast_or_null<TypedAttr>(initialValueAttr)) {
+    auto shape =
+        convertToMlirShape(to_vector(varShapeAttr.getValues<int64_t>()));
+    Type elementType = typeAttr.getValue();
+    RankedTensorType tensorType =
+        RankedTensorType::get(ArrayRef<int64_t>(shape), elementType);
+    auto tensorTypeAttr = TypeAttr::get(tensorType);
     p << ": ";
-    p.printAttribute(type);
+    p.printAttribute(tensorTypeAttr);
     needsSpace = true; // subsequent attr value needs a space separator
   }
-  if (attr) {
+  if (initialValueAttr) {
     if (needsSpace)
       p << ' ';
     p << "= ";
-    p.printAttribute(attr);
+    p.printAttribute(initialValueAttr);
   }
 }
 
@@ -414,19 +470,7 @@ static LogicalResult verifyConvOpModes(T op) {
   if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(resultEType))
     resultEType = quantType.getStorageType();
 
-  // check allowed input/result element types combinations
-  // if ((inputEType.isInteger(8) && resultEType.isInteger(32)) ||
-  //     (inputEType.isInteger(16) && resultEType.isInteger(48)) ||
-  //     (isa<Float8E5M2Type>(inputEType) && resultEType.isF16()) ||
-  //     (isa<Float8E4M3FNType>(inputEType) && resultEType.isF16()) ||
-  //     (inputEType.isF16() && resultEType.isF16()) ||
-  //     (inputEType.isBF16() && resultEType.isBF16()) ||
-  //     (inputEType.isF32() && resultEType.isF32()))
-  //   return success();
-  // TODO: try to update the standard with more options?
   return success();
-
-  return op.emitOpError("input/output element types are incompatible.");
 }
 
 //===----------------------------------------------------------------------===//
@@ -486,13 +530,15 @@ static LogicalResult verifyConvOpErrorIf(T op) {
   //              << dimName << " - 1 + pad_" << padBeforeName << " + pad_"
   //              << padAfterName << " - (kernel_" << dimName
   //              << " - 1) * dilation_" << dimAxis
-  //              << " to be wholly divisible by stride_" << dimAxis << ", got ("
+  //              << " to be wholly divisible by stride_" << dimAxis << ", got
+  //              ("
   //              << inputSize << " - 1 + " << padBefore << " + " << padAfter
   //              << " - (" << kernelSize << " - 1) * " << dilation << ") / "
   //              << stride;
 
-  //     const int64_t calculatedOutSize = calculatedOutSizeMinusOne.value() + 1;
-  //     if (outputSize != ShapedType::kDynamic && calculatedOutSize != outputSize)
+  //     const int64_t calculatedOutSize = calculatedOutSizeMinusOne.value() +
+  //     1; if (outputSize != ShapedType::kDynamic && calculatedOutSize !=
+  //     outputSize)
   //       return op.emitOpError("calculated output ")
   //              << dimName << " did not match expected: "
   //              << "calculated=" << calculatedOutSize
@@ -531,8 +577,8 @@ static LogicalResult verifyConvOpErrorIf(T op) {
   //       return failure();
   //   }
 
-  //   // input = [_,ID,IH,IW,_], weight = [_,KD,KH,KW,_], output = [_,OD,OH,OW,_]
-  //   if constexpr (std::is_same<T, tosa::Conv3DOp>::value) {
+  //   // input = [_,ID,IH,IW,_], weight = [_,KD,KH,KW,_], output =
+  //   [_,OD,OH,OW,_] if constexpr (std::is_same<T, tosa::Conv3DOp>::value) {
   //     if (failed(verifyOutputSize(
   //             inputType.getDimSize(1), weightType.getDimSize(1),
   //             outputType.getDimSize(1), padding[0], padding[1], strides[0],
@@ -670,8 +716,9 @@ static LogicalResult verifyVariableOpErrorIf(T op, Type type, StringRef name) {
            << symName << "' has not been declared by 'tosa.variable'";
 
   // Verify type and shape
-  Type varType = cast<tosa::VariableOp>(varOp.value()).getType();
-  if (errorIfTypeOrShapeMismatch(op, type, name, varType, "the input tensor")
+  auto variableType = getVariableType(varOp.value());
+  if (errorIfTypeOrShapeMismatch(op, type, name, variableType,
+                                 "the input tensor")
           .failed())
     return failure();
 
@@ -1130,6 +1177,33 @@ static void buildPadOpWithQuantInfo(OpBuilder &builder, OperationState &result,
   result.types.push_back(outputType);
 }
 
+static void buildVariableOp(OpBuilder &builder, OperationState &result,
+                            StringRef name, Type variableType,
+                            Attribute initialValue) {
+  const Location loc{result.location};
+  auto nameAttr = builder.getStringAttr(name);
+
+  auto shapedType = dyn_cast<ShapedType>(variableType);
+  if (!shapedType) {
+    (void)emitError(loc, "variable type must be a shaped type");
+    return;
+  }
+  if (!shapedType.hasRank()) {
+    (void)emitError(loc, "variable type must be a ranked type");
+    return;
+  }
+
+  auto elementType = shapedType.getElementType();
+  auto elementTypeAttr = TypeAttr::get(elementType);
+  ArrayRef<int64_t> shape = shapedType.getShape();
+  auto varShapeAttr = builder.getIndexTensorAttr(convertFromMlirShape(shape));
+
+  result.addAttribute("name", nameAttr);
+  result.addAttribute("var_shape", varShapeAttr);
+  result.addAttribute("type", elementTypeAttr);
+  result.addAttribute("initial_value", initialValue);
+}
+
 //===----------------------------------------------------------------------===//
 // TOSA Operator Return Type Inference.
 //===----------------------------------------------------------------------===//
@@ -1703,12 +1777,6 @@ LogicalResult tosa::PadOp::verify() {
   return success();
 }
 
-static SmallVector<int64_t> convertToMlirShape(ArrayRef<int64_t> shape) {
-  return to_vector(llvm::map_range(shape, [](int64_t dim) {
-    return dim == -1 ? ShapedType::kDynamic : dim;
-  }));
-}
-
 LogicalResult tosa::SliceOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
     SliceOp::Adaptor adaptor,
@@ -1806,7 +1874,8 @@ LogicalResult tosa::MulOp::inferReturnTypeComponents(
 }
 
 LogicalResult tosa::MulOp::verify() {
-  auto resElemType = getElementTypeOrSelf(getOutput());
+  const Value output = getOutput();
+  auto resElemType = getElementTypeOrSelf(output);
 
   // Verify if the element type among operands and result match tosa
   // specification.
@@ -1846,59 +1915,39 @@ LogicalResult tosa::MulOp::verify() {
   // Verify the op has same ranks for all main operands (excludes extra operands
   // such as shift of mul op, so this is the only difference with the built-in
   // `SameOperandsAndResultRank` trait) and results types, if known.
-
-  // delegate function that returns true if type is a shaped type with known
-  // rank
-  auto hasRank = [](const Type type) {
-    if (auto shaped_type = dyn_cast<ShapedType>(type))
-      return shaped_type.hasRank();
-
-    return false;
-  };
-
-  auto rankedOperandTypes =
-      llvm::to_vector(llvm::make_filter_range(getOperandTypes(), hasRank));
-
-  auto rankedResultTypes =
-      llvm::make_filter_range(getOperation()->getResultTypes(), hasRank);
-
-  // If all operands and results are unranked, then no further verification.
-  if (rankedOperandTypes.empty() && rankedResultTypes.empty())
+  TypeRange operandTypes = getOperandTypes();
+  ShapedType aType = cast<ShapedType>(operandTypes[0]);
+  ShapedType bType = cast<ShapedType>(operandTypes[1]);
+
+  const bool aHasRank = aType.hasRank();
+  const bool bHasRank = bType.hasRank();
+  if (aHasRank && bHasRank) {
+    const int64_t aRank = aType.getRank();
+    const int64_t bRank = bType.getRank();
+    if (aRank != bRank)
+      return emitOpError("a and b operands don't have matching ranks, got ")
+             << aRank << " and " << bRank;
+
+    // check for broadcast compatible shapes
+    SmallVector<int64_t> resultShape;
+    if (!mlir::OpTrait::util::getBroadcastedShape(
+            aType.getShape(), bType.getShape(), resultShape))
+      return emitOpError("a and b operands don't have broadcast-compatible "
+                         "shapes, got ")
+             << aType << " and " << bType;
+  }
+
+  ShapedType resultType = cast<ShapedType>(output.getType());
+  if (!resultType.hasRank())
     return success();
 
-  // delegate function that returns rank of shaped type with known rank
-  auto getRank = [](const Type type) {
-    return cast<ShapedType>(type).getRank();
-  };
-
-  auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin())
-                                          : getRank(*rankedResultTypes.begin());
-
-  for (size_t i = 0; i < 2; ++i) {
-    if (rank != getRank(rankedOperandTypes[i])) {
-      return emitOpError("operands don't have matching ranks");
-    }
-  }
-
-  for (const auto type : rankedResultTypes) {
-    if (rank != getRank(type)) {
-      return emitOpError("result type has different rank than operands");
-    }
-  }
-
-  // check for broadcast compatible shapes in first two operands (ignoring
-  // shift)
-
-  // delegate function that returns shape of shaped type
-  auto getShape = [](const Type type) {
-    return mlir::cast<ShapedType>(type).getShape();
-  };
-  SmallVector<int64_t> resultShape;
-  if (!mlir::OpTrait::util::getBroadcastedShape(getShape(rankedOperandTypes[0]),
-                                                getShape(rankedOperandTypes[1]),
-                                                resultShape)) {
-    return emitOpError("operands don't have broadcast-compatible shapes");
-  }
+  const int64_t resultRank = resultType.getRank();
+  if (aHasRank && resultRank != aType.getRank())
+    return emitOpError("result type has different rank than a, got ")
+           << resultRank << " vs " << aType.getRank();
+  if (bHasRank && resultRank != bType.getRank())
+    return emitOpError("result type has different rank than b, got ")
+           << resultRank << " vs " << bType.getRank();
 
   return success();
 }
@@ -2641,6 +2690,73 @@ LogicalResult tosa::ScatterOp::verify() {
           .failed()) {
     return failure();
   }
+
+  const ShapeAdaptor valuesInShape(getValuesIn().getType());
+  const ShapeAdaptor indicesShape(getIndices().getType());
+  const ShapeAdaptor inputShape(getInput().getType());
+  const ShapeAdaptor outputShape(getValuesOut().getType());
+
+  int64_t N = ShapedType::kDynamic;
+  int64_t K = ShapedType::kDynamic;
+  int64_t W = ShapedType::kDynamic;
+  int64_t C = ShapedType::kDynamic;
+  if (valuesInShape.hasRank()) {
+    N = valuesInShape.getDimSize(0);
+    K = valuesInShape.getDimSize(1);
+    C = valuesInShape.getDimSize(2);
+  }
+  if (indicesShape.hasRank()) {
+    const int64_t indicesN = indicesShape.getDimSize(0);
+    W = indicesShape.getDimSize(1);
+    if (N == ShapedType::kDynamic)
+      N = indicesN;
+    else if (indicesN != ShapedType::kDynamic && N != indicesN)
+      return emitOpError() << "requires indices dimension 0 to have size " << N
+                           << ", got " << indicesN;
+  }
+  if (inputShape.hasRank()) {
+    const int64_t inputN = inputShape.getDimSize(0);
+    const int64_t inputW = inputShape.getDimSize(1);
+    const int64_t inputC = inputShape.getDimSize(2);
+    if (N == ShapedType::kDynamic)
+      N = inputN;
+    else if (inputN != ShapedType::kDynamic && N != inputN)
+      return emitOpError() << "requires input dimension 0 to have size " << N
+                           << ", got " << inputN;
+    if (W == ShapedType::kDynamic)
+      W = inputW;
+    else if (inputW != ShapedType::kDynamic && W != inputW)
+      return emitOpError() << "requires input dimension 1 to have size " << W
+                           << ", got " << inputW;
+
+    if (C == ShapedType::kDynamic)
+      C = inputC;
+    else if (inputC != ShapedType::kDynamic && C != inputC)
+      return emitOpError() << "requires input dimension 2 to have size " << C
+                           << ", got " << inputC;
+  }
+  if (outputShape.hasRank()) {
+    const int64_t outputN = outputShape.getDimSize(0);
+    const int64_t outputK = outputShape.getDimSize(1);
+    const int64_t outputC = outputShape.getDimSize(2);
+    if (N != ShapedType::kDynamic && outputN != ShapedType::kDynamic &&
+        N != outputN)
+      return emitOpError() << "requires values_out dimension 0 to have size "
+                           << N << ", got " << outputN;
+    if (K == ShapedType::kDynamic)
+      K = outputK;
+    else if (outputK != ShapedType::kDynamic && K != outputK)
+      return emitOpError() << "requires values_out dimension 1 to have size "
+                           << K << ", got " << outputK;
+    if (C != ShapedType::kDynamic && outputC != ShapedType::kDynamic &&
+        C != outputC)
+      return emitOpError() << "requires values_out dimension 2 to have size "
+                           << C << ", got " << outputC;
+  }
+  if (K != ShapedType::kDynamic && W != ShapedType::kDynamic && !(K >= W))
+    return emitOpError() << "requires dimensions K >= W, got K=" << K
+                         << " and W=" << W;
+
   return success();
 }
 
diff --git a/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index 9aa0051070bd..29ec9f8db261 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -344,4 +344,4 @@ struct TosaInferShapes
     validateSameOperandsAndResultRankTrait(func.getBody());
   }
 };
-} // namespace
\ No newline at end of file
+} // namespace
diff --git a/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp b/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
index 1a896c1464e1..de08e7e9a439 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
@@ -215,15 +215,8 @@ LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::MatMulOp op) {
 
 template <>
 LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::VariableOp op) {
-  ::mlir::Attribute attr = op.getInitialValueAttr();
-  if (attr == nullptr)
-    return failure();
-
-  if (auto typedAttr = dyn_cast<TypedAttr>(attr)) {
-    addType(getElementTypeOrSelf(typedAttr));
-    return success();
-  }
-  return failure();
+  addType(op.getType());
+  return success();
 }
 
 template <>
diff --git a/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index 50272b187a74..229f42d3178b 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -238,10 +238,10 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
     return true;
   }
 
-  template <typename T>
-  bool levelCheckRank(Operation *op, const T &v,
+  // Perform the Level Rank check on the tensor type.
+  bool levelCheckRank(Operation *op, const Type typeToCheck,
                       const StringRef operandOrResult, int32_t highest_rank) {
-    if (ShapedType type = dyn_cast<ShapedType>(v.getType())) {
+    if (ShapedType type = dyn_cast<ShapedType>(typeToCheck)) {
       if (!type.hasRank()) {
         op->emitOpError() << "failed level check: unranked tensor";
         return false;
@@ -255,10 +255,22 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
     return true;
   }
 
-  // Perform the Level tensor size check on the input tensor.
-  bool levelCheckSize(Operation *op, const Value &v,
+  // Perform the Level Rank check on the tensor value.
+  bool levelCheckRank(Operation *op, const Value &v,
+                      const StringRef operandOrResult, int32_t highest_rank) {
+    return levelCheckRank(op, v.getType(), operandOrResult, highest_rank);
+  }
+
+  // Perform the Level tensor size check on the tensor type.
+  bool levelCheckSize(Operation *op, const Type &typeToCheck,
                       const StringRef operandOrResult);
 
+  // Perform the Level tensor size check on the tensor value.
+  bool levelCheckSize(Operation *op, const Value &v,
+                      const StringRef operandOrResult) {
+    return levelCheckSize(op, v.getType(), operandOrResult);
+  }
+
   // Level check sizes of all operands and results of the operation.
   template <typename T>
   bool levelCheckSizes(T tosaOp) {
@@ -284,15 +296,6 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
         return false;
     }
 
-    if (!op->getAttrs().empty()) {
-      for (NamedAttribute attr : op->getAttrs()) {
-        if (auto elemAttr = dyn_cast<ElementsAttr>(attr.getValue())) {
-          if (!levelCheckRank(op, elemAttr, "attribute", tosaLevel.MAX_RANK))
-            return false;
-        }
-      }
-    }
-
     for (auto v : op->getResults()) {
       if (!levelCheckRank(op, v, "result", tosaLevel.MAX_RANK))
         return false;
@@ -596,6 +599,26 @@ bool TosaValidation::levelCheckRanks(tosa::IfOp tosaOp) {
   return true;
 }
 
+template <>
+bool TosaValidation::levelCheckRanks(tosa::VariableOp tosaOp) {
+  auto op = tosaOp.getOperation();
+  auto variableType = getVariableType(tosaOp);
+  if (!levelCheckRank(op, variableType, "variable type", tosaLevel.MAX_RANK))
+    return false;
+
+  return true;
+}
+
+template <>
+bool TosaValidation::levelCheckSizes(tosa::VariableOp tosaOp) {
+  auto op = tosaOp.getOperation();
+  auto variableType = getVariableType(tosaOp);
+  if (!levelCheckSize(op, variableType, "variable type"))
+    return false;
+
+  return true;
+}
+
 bool TosaValidation::levelCheckRanksAndSizes(Operation *op) {
 #define CHECK_RANKS_AND_SIZES(tosaOp)                                          \
   if (isa<tosa::tosaOp##Op>(op)) {                                             \
@@ -714,10 +737,10 @@ bool TosaValidation::levelCheckRanksAndSizes(Operation *op) {
   return true;
 }
 
-// Perform the Level tensor size check
-bool TosaValidation::levelCheckSize(Operation *op, const Value &v,
+// Perform the Level tensor size check on the tensor type.
+bool TosaValidation::levelCheckSize(Operation *op, const Type &typeToCheck,
                                     const StringRef operandOrResult) {
-  if (ShapedType type = dyn_cast<ShapedType>(v.getType())) {
+  if (ShapedType type = dyn_cast<ShapedType>(typeToCheck)) {
     if (!type.hasRank()) {
       op->emitOpError() << "failed level check: unranked tensor";
       return false;
@@ -800,18 +823,21 @@ inline bool CompatibleTypes(const mlir::Type &type,
 }
 
 bool TosaValidation::CheckVariable(Operation *op) {
-  if (isa<mlir::tosa::VariableOp>(op)) {
-    mlir::StringAttr nameAttr = cast<mlir::StringAttr>(op->getAttr("name"));
+  if (auto variableOp = dyn_cast<mlir::tosa::VariableOp>(op)) {
+    mlir::StringAttr nameAttr = variableOp.getNameAttr();
 
     if (variablesMap.count(nameAttr)) {
       op->emitOpError() << "name has already been declared";
       return false;
     }
 
-    auto typeAttr = cast<mlir::TypeAttr>(op->getAttr("type"));
-    mlir::Type type = typeAttr.getValue();
+    auto elementType = variableOp.getType();
+    DenseIntElementsAttr varShapeAttr = variableOp.getVarShape();
+    SmallVector<int64_t> shape = to_vector(varShapeAttr.getValues<int64_t>());
+    RankedTensorType variableType =
+        RankedTensorType::get(ArrayRef<int64_t>(shape), elementType);
 
-    variablesMap[nameAttr] = type;
+    variablesMap[nameAttr] = variableType;
   }
 
   return true;
@@ -1167,10 +1193,87 @@ bool checkErrorIfPad(Operation *op) {
   return true;
 }
 
+// Returns true if the operation takes no input operands, excluding attributes.
+static bool isNullaryOperation(Operation *op) {
+  if (isa<tosa::ConstOp>(op) || isa<tosa::ConstShapeOp>(op) ||
+      isa<tosa::YieldOp>(op) || isa<tosa::VariableOp>(op))
+    return true;
+  return false;
+}
+
+bool checkErrorIfCondIf(Operation *op) {
+  auto ifOp = dyn_cast<tosa::IfOp>(op);
+  if (!ifOp)
+    return true;
+
+  // Whether the types and shapes of operands between the input/output list and
+  // internal regions are validated by the operation verifier. However, with
+  // support for the simplified form - where redundant operand notations are
+  // omitted - is not conformant to the specification. According to the
+  // specification, all operands passed into an operation must be explicitly
+  // declared at each operation's structure. This code section verify that the
+  // operation's form complies with this requirement.
+
+  // Returns true if the region uses no external input operands.
+  auto isNullaryRegion = [](Region &region) -> bool {
+    bool noLiveInValue = true;
+    region.walk([&noLiveInValue](Operation *op) {
+      if (!isNullaryOperation(op)) {
+        noLiveInValue = false;
+        return WalkResult::interrupt();
+      }
+      return WalkResult::advance();
+    });
+    return noLiveInValue;
+  };
+
+  mlir::Region &thenGraph = ifOp.getThenGraph();
+  mlir::Region &elseGraph = ifOp.getElseGraph();
+  bool isThenGraphNullaryRegion = isNullaryRegion(thenGraph);
+  bool isElseGraphNullaryRegion = isNullaryRegion(elseGraph);
+  bool isInputListEmpty = ifOp.getInputList().size() == 0;
+
+  if ((isInputListEmpty != isThenGraphNullaryRegion) ||
+      (isInputListEmpty != isElseGraphNullaryRegion)) {
+    op->emitOpError()
+        << "the current simplified form is not strictly conformant to the "
+           "spec, please use the generic format\n";
+    return false;
+  }
+
+  return true;
+}
+
+bool checkErrorIfScatter(Operation *op) {
+  auto scatterOp = dyn_cast<tosa::ScatterOp>(op);
+  if (!scatterOp)
+    return true;
+
+  // for constant indices, check that there are no duplicate values
+  DenseIntElementsAttr indicesAttr;
+  if (!matchPattern(scatterOp.getIndices(), m_Constant(&indicesAttr)))
+    return true;
+
+  auto const indicesType =
+      dyn_cast<ShapedType>(scatterOp.getIndices().getType());
+  if (!indicesType || !indicesType.hasRank()) {
+    op->emitOpError("expect ranked indices tensor");
+    return false;
+  }
+
+  if (!hasUniqueConstantScatterIndices(indicesType, indicesAttr)) {
+    op->emitOpError("indices values contain duplicates");
+    return false;
+  }
+
+  return true;
+}
+
 LogicalResult TosaValidation::applyErrorIfCheck(Operation *op) {
   if (!checkErrorIfResize(op) || !checkErrorIfMul(op) ||
       !checkErrorIfTable(op) || !checkErrorIfRescale(op) ||
-      !checkErrorIfPad(op))
+      !checkErrorIfPad(op) || !checkErrorIfCondIf(op) ||
+      !checkErrorIfScatter(op))
     return failure();
   return success();
 }
@@ -1178,7 +1281,7 @@ LogicalResult TosaValidation::applyErrorIfCheck(Operation *op) {
 bool TosaValidation::isValidElementType(Type type, const bool allowUnsigned) {
   if (isa<FloatType>(type)) {
     return isa<Float32Type, Float16Type, BFloat16Type, Float8E4M3FNType,
-               Float8E5M2Type, Float8E4M3FNUZType, Float8E5M2FNUZType>(type);
+               Float8E5M2Type>(type);
   } else if (auto intTy = dyn_cast<IntegerType>(type)) {
     if (intTy.isSignless()) {
       switch (intTy.getWidth()) {
diff --git a/external/llvm-project/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp b/external/llvm-project/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
index e1b3be74b50f..9844abcc34cb 100644
--- a/external/llvm-project/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Tosa/Utils/ConversionUtils.cpp
@@ -213,3 +213,30 @@ mlir::tosa::convertFromIntAttr(const DenseElementsAttr &attr, const int rank) {
   }
   return {};
 }
+
+bool mlir::tosa::hasUniqueConstantScatterIndices(
+    ShapedType indicesType, DenseIntElementsAttr indicesAttr) {
+  llvm::ArrayRef<int64_t> const indicesShape = indicesType.getShape();
+  const unsigned int indicesRank = indicesShape.size();
+  const unsigned int lastDimSize = indicesShape[indicesRank - 1];
+
+  // check each batch of indices from the flat indicesAttr values
+  // for duplicates
+  auto const indicesValues = indicesAttr.getValues<int32_t>();
+  assert(
+      (indicesValues.size() % lastDimSize == 0) &&
+      "Constant indices data length should be a multiple of indicesShape[-1]");
+
+  std::vector<uint64_t> indices(lastDimSize);
+  for (auto beg = indicesValues.begin(); beg < indicesValues.end();
+       beg += lastDimSize) {
+    std::copy(beg, beg + lastDimSize, indices.begin());
+    std::sort(indices.begin(), indices.end());
+    if (std::adjacent_find(indices.begin(), indices.end()) != indices.end()) {
+      // found duplicate values in indices in batch
+      return false;
+    }
+  }
+
+  return true;
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp b/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp
index 497ceb19f1a2..4a95fe7459e8 100644
--- a/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp
@@ -8,17 +8,22 @@
 
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Analysis/CallGraph.h"
+#include "mlir/Dialect/Transform/IR/TransformAttrs.h"
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/IR/Utils.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/SCCIterator.h"
+#include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
 
 #include "mlir/Dialect/Transform/IR/TransformDialect.cpp.inc"
 
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/Transform/IR/TransformAttrs.cpp.inc"
+
 #ifndef NDEBUG
 void transform::detail::checkImplementsTransformOpInterface(
     StringRef name, MLIRContext *context) {
@@ -66,6 +71,10 @@ void transform::TransformDialect::initialize() {
 #include "mlir/Dialect/Transform/IR/TransformOps.cpp.inc"
       >();
   initializeTypes();
+  addAttributes<
+#define GET_ATTRDEF_LIST
+#include "mlir/Dialect/Transform/IR/TransformAttrs.cpp.inc"
+      >();
   initializeLibraryModule();
 }
 
diff --git a/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 673743f22249..bb9bdd70625e 100644
--- a/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -53,6 +53,13 @@
 
 using namespace mlir;
 
+static ParseResult parseApplyRegisteredPassOptions(
+    OpAsmParser &parser, DictionaryAttr &options,
+    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dynamicOptions);
+static void printApplyRegisteredPassOptions(OpAsmPrinter &printer,
+                                            Operation *op,
+                                            DictionaryAttr options,
+                                            ValueRange dynamicOptions);
 static ParseResult parseSequenceOpOperands(
     OpAsmParser &parser, std::optional<OpAsmParser::UnresolvedOperand> &root,
     Type &rootType,
@@ -766,17 +773,63 @@ void transform::ApplyLoopInvariantCodeMotionOp::getEffects(
 // ApplyRegisteredPassOp
 //===----------------------------------------------------------------------===//
 
-DiagnosedSilenceableFailure transform::ApplyRegisteredPassOp::applyToOne(
-    transform::TransformRewriter &rewriter, Operation *target,
-    ApplyToEachResultList &results, transform::TransformState &state) {
-  // Make sure that this transform is not applied to itself. Modifying the
-  // transform IR while it is being interpreted is generally dangerous. Even
-  // more so when applying passes because they may perform a wide range of IR
-  // modifications.
-  DiagnosedSilenceableFailure payloadCheck =
-      ensurePayloadIsSeparateFromTransform(*this, target);
-  if (!payloadCheck.succeeded())
-    return payloadCheck;
+void transform::ApplyRegisteredPassOp::getEffects(
+    SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
+  consumesHandle(getTargetMutable(), effects);
+  onlyReadsHandle(getDynamicOptionsMutable(), effects);
+  producesHandle(getOperation()->getOpResults(), effects);
+  modifiesPayload(effects);
+}
+
+DiagnosedSilenceableFailure
+transform::ApplyRegisteredPassOp::apply(transform::TransformRewriter &rewriter,
+                                        transform::TransformResults &results,
+                                        transform::TransformState &state) {
+  // Obtain a single options-string to pass to the pass(-pipeline) from options
+  // passed in as a dictionary of keys mapping to values which are either
+  // attributes or param-operands pointing to attributes.
+  OperandRange dynamicOptions = getDynamicOptions();
+
+  std::string options;
+  llvm::raw_string_ostream optionsStream(options); // For "printing" attrs.
+
+  // A helper to convert an option's attribute value into a corresponding
+  // string representation, with the ability to obtain the attr(s) from a param.
+  std::function<void(Attribute)> appendValueAttr = [&](Attribute valueAttr) {
+    if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(valueAttr)) {
+      // The corresponding value attribute(s) is/are passed in via a param.
+      // Obtain the param-operand via its specified index.
+      size_t dynamicOptionIdx = paramOperand.getIndex().getInt();
+      assert(dynamicOptionIdx < dynamicOptions.size() &&
+             "the number of ParamOperandAttrs in the options DictionaryAttr"
+             "should be the same as the number of options passed as params");
+      ArrayRef<Attribute> attrsAssociatedToParam =
+          state.getParams(dynamicOptions[dynamicOptionIdx]);
+      // Recursive so as to append all attrs associated to the param.
+      llvm::interleave(attrsAssociatedToParam, optionsStream, appendValueAttr,
+                       ",");
+    } else if (auto arrayAttr = dyn_cast<ArrayAttr>(valueAttr)) {
+      // Recursive so as to append all nested attrs of the array.
+      llvm::interleave(arrayAttr, optionsStream, appendValueAttr, ",");
+    } else if (auto strAttr = dyn_cast<StringAttr>(valueAttr)) {
+      // Convert to unquoted string.
+      optionsStream << strAttr.getValue().str();
+    } else {
+      // For all other attributes, ask the attr to print itself (without type).
+      valueAttr.print(optionsStream, /*elideType=*/true);
+    }
+  };
+
+  // Convert the options DictionaryAttr into a single string.
+  llvm::interleave(
+      getOptions(), optionsStream,
+      [&](auto namedAttribute) {
+        optionsStream << namedAttribute.getName().str(); // Append the key.
+        optionsStream << "="; // And the key-value separator.
+        appendValueAttr(namedAttribute.getValue()); // And the attr's str repr.
+      },
+      " ");
+  optionsStream.flush();
 
   // Get pass or pass pipeline from registry.
   const PassRegistryEntry *info = PassPipelineInfo::lookup(getPassName());
@@ -786,9 +839,9 @@ DiagnosedSilenceableFailure transform::ApplyRegisteredPassOp::applyToOne(
     return emitDefiniteFailure()
            << "unknown pass or pass pipeline: " << getPassName();
 
-  // Create pass manager and run the pass or pass pipeline.
+  // Create pass manager and add the pass or pass pipeline.
   PassManager pm(getContext());
-  if (failed(info->addToPipeline(pm, getOptions(), [&](const Twine &msg) {
+  if (failed(info->addToPipeline(pm, options, [&](const Twine &msg) {
         emitError(msg);
         return failure();
       }))) {
@@ -796,16 +849,204 @@ DiagnosedSilenceableFailure transform::ApplyRegisteredPassOp::applyToOne(
            << "failed to add pass or pass pipeline to pipeline: "
            << getPassName();
   }
-  if (failed(pm.run(target))) {
-    auto diag = emitSilenceableError() << "pass pipeline failed";
-    diag.attachNote(target->getLoc()) << "target op";
-    return diag;
+
+  auto targets = SmallVector<Operation *>(state.getPayloadOps(getTarget()));
+  for (Operation *target : targets) {
+    // Make sure that this transform is not applied to itself. Modifying the
+    // transform IR while it is being interpreted is generally dangerous. Even
+    // more so when applying passes because they may perform a wide range of IR
+    // modifications.
+    DiagnosedSilenceableFailure payloadCheck =
+        ensurePayloadIsSeparateFromTransform(*this, target);
+    if (!payloadCheck.succeeded())
+      return payloadCheck;
+
+    // Run the pass or pass pipeline on the current target operation.
+    if (failed(pm.run(target))) {
+      auto diag = emitSilenceableError() << "pass pipeline failed";
+      diag.attachNote(target->getLoc()) << "target op";
+      return diag;
+    }
   }
 
-  results.push_back(target);
+  // The applied pass will have directly modified the payload IR(s).
+  results.set(llvm::cast<OpResult>(getResult()), targets);
   return DiagnosedSilenceableFailure::success();
 }
 
+static ParseResult parseApplyRegisteredPassOptions(
+    OpAsmParser &parser, DictionaryAttr &options,
+    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dynamicOptions) {
+  // Construct the options DictionaryAttr per a `{ key = value, ... }` syntax.
+  SmallVector<NamedAttribute> keyValuePairs;
+  size_t dynamicOptionsIdx = 0;
+
+  // Helper for allowing parsing of option values which can be of the form:
+  // - a normal attribute
+  // - an operand (which would be converted to an attr referring to the operand)
+  // - ArrayAttrs containing the foregoing (in correspondence with ListOptions)
+  std::function<ParseResult(Attribute &)> parseValue =
+      [&](Attribute &valueAttr) -> ParseResult {
+    // Allow for array syntax, e.g. `[0 : i64, %param, true, %other_param]`:
+    if (succeeded(parser.parseOptionalLSquare())) {
+      SmallVector<Attribute> attrs;
+
+      // Recursively parse the array's elements, which might be operands.
+      if (parser.parseCommaSeparatedList(
+              AsmParser::Delimiter::None,
+              [&]() -> ParseResult { return parseValue(attrs.emplace_back()); },
+              " in options dictionary") ||
+          parser.parseRSquare())
+        return failure(); // NB: Attempted parse should've output error message.
+
+      valueAttr = ArrayAttr::get(parser.getContext(), attrs);
+
+      return success();
+    }
+
+    // Parse the value, which can be either an attribute or an operand.
+    OptionalParseResult parsedValueAttr =
+        parser.parseOptionalAttribute(valueAttr);
+    if (!parsedValueAttr.has_value()) {
+      OpAsmParser::UnresolvedOperand operand;
+      ParseResult parsedOperand = parser.parseOperand(operand);
+      if (failed(parsedOperand))
+        return failure(); // NB: Attempted parse should've output error message.
+      // To make use of the operand, we need to store it in the options dict.
+      // As SSA-values cannot occur in attributes, what we do instead is store
+      // an attribute in its place that contains the index of the param-operand,
+      // so that an attr-value associated to the param can be resolved later on.
+      dynamicOptions.push_back(operand);
+      auto wrappedIndex = IntegerAttr::get(
+          IntegerType::get(parser.getContext(), 64), dynamicOptionsIdx++);
+      valueAttr =
+          transform::ParamOperandAttr::get(parser.getContext(), wrappedIndex);
+    } else if (failed(parsedValueAttr.value())) {
+      return failure(); // NB: Attempted parse should have output error message.
+    } else if (isa<transform::ParamOperandAttr>(valueAttr)) {
+      return parser.emitError(parser.getCurrentLocation())
+             << "the param_operand attribute is a marker reserved for "
+             << "indicating a value will be passed via params and is only used "
+             << "in the generic print format";
+    }
+
+    return success();
+  };
+
+  // Helper for `key = value`-pair parsing where `key` is a bare identifier or a
+  // string and `value` looks like either an attribute or an operand-in-an-attr.
+  std::function<ParseResult()> parseKeyValuePair = [&]() -> ParseResult {
+    std::string key;
+    Attribute valueAttr;
+
+    if (failed(parser.parseOptionalKeywordOrString(&key)) || key.empty())
+      return parser.emitError(parser.getCurrentLocation())
+             << "expected key to either be an identifier or a string";
+
+    if (failed(parser.parseEqual()))
+      return parser.emitError(parser.getCurrentLocation())
+             << "expected '=' after key in key-value pair";
+
+    if (failed(parseValue(valueAttr)))
+      return parser.emitError(parser.getCurrentLocation())
+             << "expected a valid attribute or operand as value associated "
+             << "to key '" << key << "'";
+
+    keyValuePairs.push_back(NamedAttribute(key, valueAttr));
+
+    return success();
+  };
+
+  if (parser.parseCommaSeparatedList(AsmParser::Delimiter::Braces,
+                                     parseKeyValuePair,
+                                     " in options dictionary"))
+    return failure(); // NB: Attempted parse should have output error message.
+
+  if (DictionaryAttr::findDuplicate(
+          keyValuePairs, /*isSorted=*/false) // Also sorts the keyValuePairs.
+          .has_value())
+    return parser.emitError(parser.getCurrentLocation())
+           << "duplicate keys found in options dictionary";
+
+  options = DictionaryAttr::getWithSorted(parser.getContext(), keyValuePairs);
+
+  return success();
+}
+
+static void printApplyRegisteredPassOptions(OpAsmPrinter &printer,
+                                            Operation *op,
+                                            DictionaryAttr options,
+                                            ValueRange dynamicOptions) {
+  if (options.empty())
+    return;
+
+  std::function<void(Attribute)> printOptionValue = [&](Attribute valueAttr) {
+    if (auto paramOperandAttr =
+            dyn_cast<transform::ParamOperandAttr>(valueAttr)) {
+      // Resolve index of param-operand to its actual SSA-value and print that.
+      printer.printOperand(
+          dynamicOptions[paramOperandAttr.getIndex().getInt()]);
+    } else if (auto arrayAttr = dyn_cast<ArrayAttr>(valueAttr)) {
+      // This case is so that ArrayAttr-contained operands are pretty-printed.
+      printer << "[";
+      llvm::interleaveComma(arrayAttr, printer, printOptionValue);
+      printer << "]";
+    } else {
+      printer.printAttribute(valueAttr);
+    }
+  };
+
+  printer << "{";
+  llvm::interleaveComma(options, printer, [&](NamedAttribute namedAttribute) {
+    printer << namedAttribute.getName();
+    printer << " = ";
+    printOptionValue(namedAttribute.getValue());
+  });
+  printer << "}";
+}
+
+LogicalResult transform::ApplyRegisteredPassOp::verify() {
+  // Check that there is a one-to-one correspondence between param operands
+  // and references to dynamic options in the options dictionary.
+
+  auto dynamicOptions = SmallVector<Value>(getDynamicOptions());
+
+  // Helper for option values to mark seen operands as having been seen (once).
+  std::function<LogicalResult(Attribute)> checkOptionValue =
+      [&](Attribute valueAttr) -> LogicalResult {
+    if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(valueAttr)) {
+      size_t dynamicOptionIdx = paramOperand.getIndex().getInt();
+      if (dynamicOptionIdx < 0 || dynamicOptionIdx >= dynamicOptions.size())
+        return emitOpError()
+               << "dynamic option index " << dynamicOptionIdx
+               << " is out of bounds for the number of dynamic options: "
+               << dynamicOptions.size();
+      if (dynamicOptions[dynamicOptionIdx] == nullptr)
+        return emitOpError() << "dynamic option index " << dynamicOptionIdx
+                             << " is already used in options";
+      dynamicOptions[dynamicOptionIdx] = nullptr; // Mark this option as used.
+    } else if (auto arrayAttr = dyn_cast<ArrayAttr>(valueAttr)) {
+      // Recurse into ArrayAttrs as they may contain references to operands.
+      for (auto eltAttr : arrayAttr)
+        if (failed(checkOptionValue(eltAttr)))
+          return failure();
+    }
+    return success();
+  };
+
+  for (NamedAttribute namedAttr : getOptions())
+    if (failed(checkOptionValue(namedAttr.getValue())))
+      return failure();
+
+  // All dynamicOptions-params seen in the dict will have been set to null.
+  for (Value dynamicOption : dynamicOptions)
+    if (dynamicOption)
+      return emitOpError() << "a param operand does not have a corresponding "
+                           << "param_operand attr in the options dict";
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // CastOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp b/external/llvm-project/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
index ed40a080441b..aa566c0086a2 100644
--- a/external/llvm-project/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
@@ -10,6 +10,10 @@
 
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypeInterfaces.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/LogicalResult.h"
 
 #include <numeric>
 #include <optional>
@@ -28,67 +32,327 @@ mlir::getReassociationIndicesForReshape(ShapedType sourceType,
   return std::nullopt;
 }
 
-std::optional<SmallVector<ReassociationIndices>>
-mlir::getReassociationIndicesForCollapse(ArrayRef<int64_t> sourceShape,
-                                         ArrayRef<int64_t> targetShape) {
-  if (sourceShape.size() <= targetShape.size())
-    return std::nullopt;
-  unsigned sourceDim = 0;
-  SmallVector<ReassociationIndices> reassociationMap;
-  reassociationMap.reserve(targetShape.size());
+namespace {
+/// A simple struct to represent ReassociationIndices as an inclusive interval.
+/// It's designed to be feasibly minimal, so the call sites should manage the
+/// validity of the range manually.
+struct ReassociationIndexRange {
+  /// FIXME: Signed type is used for consistency with ReassociationIndices.
+  /// We should consider refactoring all reassociation utilities to use unsigned
+  /// types.
+  int64_t leftIdx = 0, rightIdx = 0;
+
+  /// Util for manual checks of the range's validity
+  LogicalResult verify() const {
+    return leftIdx >= 0 && (leftIdx <= rightIdx) ? success() : failure();
+  }
+
+  /// Checks range's containment within another range. Treats the edges
+  /// non-exclusively.
+  bool isInRange(const ReassociationIndexRange &outerRange) const {
+    return leftIdx >= outerRange.leftIdx && rightIdx <= outerRange.rightIdx;
+  }
 
-  ReassociationIndices currIndices;
+  unsigned size() const {
+    assert(succeeded(verify()));
+    return rightIdx - leftIdx + 1;
+  }
+  bool containsSingleIndex() const { return size() == 1; }
+
+  /// Collects indices that do not overlap between this and another range.
+  ReassociationIndices
+  getNonOverlappingIndicesWith(ReassociationIndexRange &rhs) const {
+    if (rightIdx < rhs.leftIdx) {
+      // The intervals do not overlap - concatenate the indices from both.
+      auto jointFullIndices = getFullIndices();
+      jointFullIndices.append(rhs.getFullIndices());
+      return jointFullIndices;
+    }
+    ReassociationIndices result;
+    // Handle the chunk left of the overlapping range.
+    int64_t leftStart = std::min(leftIdx, rhs.leftIdx);
+    int64_t leftEnd = std::max(leftIdx, rhs.leftIdx);
+    llvm::append_range(result, llvm::seq(leftStart, leftEnd));
+    // Handle the chunk right of the overlapping range. Symmetrically, we should
+    // skip the edge of the overlap AND include the rightmost index.
+    int64_t rightStart = std::min(rightIdx, rhs.rightIdx) + 1;
+    int64_t rightEnd = std::max(rightIdx, rhs.rightIdx);
+    if (rightStart < rightEnd)
+      llvm::append_range(result, llvm::seq_inclusive(rightStart, rightEnd));
+    return result;
+  }
+
+  /// Converts the range into ReassociationIndices.
+  ReassociationIndices getFullIndices() const {
+    ReassociationIndices result;
+    for (int64_t idx = leftIdx; idx <= rightIdx; ++idx) {
+      result.push_back(idx);
+    }
+    return result;
+  }
+};
+} // namespace
+
+/// Starting from `sourceStartIdx`, searches `sourceShape` for the first
+/// sequence that can be collapsed into a dynamic dimension (at least one must
+/// be present in the source).
+/// By default, lazily returns once the first dynamic dimension has been found.
+/// Setting `matchGreedily` as `true` will also mark all subsequent
+/// source dimensions for collapsing into the target.
+static FailureOr<ReassociationIndexRange>
+findReassociationRangeForDynamicDim(ArrayRef<int64_t> sourceShape,
+                                    int64_t sourceStartIdx,
+                                    bool matchGreedily = false) {
+  const unsigned numSourceDims = sourceShape.size();
+  ReassociationIndexRange sourceShapeAsRange{0, numSourceDims - 1};
+  std::optional<ReassociationIndexRange> resultRange = std::nullopt;
+
+  ReassociationIndexRange iterationRange{sourceStartIdx, sourceStartIdx};
+  for (; iterationRange.isInRange(sourceShapeAsRange);
+       iterationRange.rightIdx++) {
+    int64_t sourceSize = sourceShape[iterationRange.rightIdx];
+    if (sourceSize == ShapedType::kDynamic) {
+      resultRange = iterationRange;
+      break;
+    }
+  }
+  if (!resultRange)
+    return failure();
+  if (matchGreedily)
+    resultRange->rightIdx = sourceShapeAsRange.rightIdx;
+  return *resultRange;
+}
+
+/// Starting from `sourceStartIdx`, searches `sourceShape` for the first
+/// sequence of static dimensions such that their product matches `targetSize`.
+/// By default, lazily returns once the product matches the target size. Setting
+/// `matchGreedily` as `true` will append all neighboring unit dimensions
+/// (dimensions of 1) to the match.
+static FailureOr<ReassociationIndexRange>
+findReassociationRangeForSize(ArrayRef<int64_t> sourceShape,
+                              int64_t sourceStartIdx, int64_t targetSize,
+                              bool matchGreedily = false) {
+  const unsigned numSourceDims = sourceShape.size();
+  ReassociationIndexRange sourceShapeAsRange{0, numSourceDims - 1};
+  std::optional<ReassociationIndexRange> resultRange = std::nullopt;
+
+  ReassociationIndexRange iterationRange{sourceStartIdx, sourceStartIdx};
   int64_t prodOfCollapsedDims = 1;
-  while (sourceDim < sourceShape.size()) {
-    unsigned targetDim = reassociationMap.size();
-    // If we have mapped all the target dimensions stop and handle the remaining
-    // tail of size-1 dimensions explicitly.
-    if (targetDim == targetShape.size())
+  while (iterationRange.isInRange(sourceShapeAsRange)) {
+    int64_t sourceSize = sourceShape[iterationRange.rightIdx];
+    if (sourceSize == ShapedType::kDynamic) {
+      // Reassociation for a static dim cannot include a dynamic dim. Reset
+      // induction variables to essentially restart the loop from the next
+      // source dimension.
+      prodOfCollapsedDims = 1;
+      iterationRange = {iterationRange.rightIdx + 1,
+                        iterationRange.rightIdx + 1};
+      continue;
+    }
+    prodOfCollapsedDims *= sourceSize;
+    // If the target size has been exceeded without matching, we need to shift
+    // the range start right. From the start of the range, roll back the
+    // multiplication until the target size exceeds the product again.
+    while (prodOfCollapsedDims > targetSize &&
+           !iterationRange.containsSingleIndex()) {
+      int64_t frontSourceSize = sourceShape[iterationRange.leftIdx];
+      prodOfCollapsedDims /= frontSourceSize;
+      // Shrink the range rightwards
+      iterationRange.leftIdx++;
+    }
+    // We could've reached the target size with the current dimension,
+    // also as a result of the above shift to right.
+    if (prodOfCollapsedDims == targetSize) {
+      resultRange = iterationRange;
       break;
+    }
+    // Increment the iteration range
+    iterationRange.rightIdx++;
+  }
+  if (!resultRange)
+    return failure();
+  if (matchGreedily) {
+    // We now want to collect all unit dimensions directly after the target
+    // product match. Advance the iterator to avoid OOB when the product match
+    // happens at the last element.
+    iterationRange.rightIdx++;
+    while (iterationRange.isInRange(sourceShapeAsRange) &&
+           sourceShape[iterationRange.rightIdx] == 1) {
+      resultRange = iterationRange;
+      iterationRange.rightIdx++;
+    }
+  }
+  return *resultRange;
+}
+
+/// Attempts to find a valid collapsing reassociation of `sourceShape` into
+/// `targetShape` through a simple traversal. If successful, an array of source
+/// index ranges is returned, correspondingly to each dimension in the target
+/// shape. The resulting indices shall fully cover the `sourceShape` without
+/// overlaps.
+///
+/// The algorithm is essentially a lazy one, searching for non-greedy matches -
+/// it will only yield a greedy match for the last target dimension.
+/// FIXME: The algorithm can only backtrack when it needs to append an offset
+/// for a static target dimension to the preceding dynamic one (this retains the
+/// linear complexity). As feasible, consider adding further backtracking
+/// routines to enable more reassociations, e.g.:
+/// - ?x2x?x2 into ?x2
+static FailureOr<SmallVector<ReassociationIndexRange>>
+findReassociationRangesForCollapse(ArrayRef<int64_t> sourceShape,
+                                   ArrayRef<int64_t> targetShape) {
+  unsigned numSourceDims = sourceShape.size(),
+           numTargetDims = targetShape.size();
+  assert(numSourceDims > numTargetDims);
+  ReassociationIndexRange sourceShapeAsRange{0, numSourceDims - 1};
+
+  SmallVector<ReassociationIndexRange> reassocRanges;
+  reassocRanges.reserve(numTargetDims);
+  // We'll iterate in strides of 2 to enable pseudo-backtracking for simple
+  // cases, e.g.:
+  // - ?x2x3x5 into ?x15
+  std::optional<int64_t> prevTargetSize = std::nullopt;
+  for (unsigned targetDimIdx = 0, sourceDimIdx = 0;
+       targetDimIdx < numTargetDims; ++targetDimIdx) {
+    int64_t targetSize = targetShape[targetDimIdx];
+    // Simply check if there are any subsequent target dimensions left - if not,
+    // the match must be made greedily.
+    bool shouldMatchGreedily = targetDimIdx == numTargetDims - 1;
+    FailureOr<ReassociationIndexRange> sourceRange;
+    if (targetSize == ShapedType::kDynamic) {
+      sourceRange = findReassociationRangeForDynamicDim(
+          sourceShape, sourceDimIdx, shouldMatchGreedily);
+    } else {
+      sourceRange = findReassociationRangeForSize(
+          sourceShape, sourceDimIdx, targetSize, shouldMatchGreedily);
+    }
 
-    int64_t currTargetShape = targetShape[targetDim];
-    while (sourceDim < (sourceShape.size() - 1) &&
-           sourceShape[sourceDim] != ShapedType::kDynamic &&
-           prodOfCollapsedDims * sourceShape[sourceDim] < currTargetShape) {
-      prodOfCollapsedDims *= sourceShape[sourceDim];
-      currIndices.push_back(sourceDim++);
+    // Run sanity checks on the returned index range.
+    if (failed(sourceRange) || failed(sourceRange->verify()) ||
+        !sourceRange->isInRange(sourceShapeAsRange))
+      return failure();
+    if (sourceRange->leftIdx > sourceDimIdx) {
+      // If some source dimensions had to be skipped in order to find a match,
+      // they must be collapsed into the directly preceding dynamic dimension.
+      if (!prevTargetSize || prevTargetSize != ShapedType::kDynamic)
+        return failure();
+      reassocRanges.back().rightIdx = sourceRange->leftIdx - 1;
     }
 
-    // If the current expanded dimension is dynamic, then the collapsed
-    // dimensions should also be dynamic and product of all previous unprocessed
-    // dimensions of the expanded shape should be 1.
-    if (sourceShape[sourceDim] == ShapedType::kDynamic &&
-        (currTargetShape != ShapedType::kDynamic || prodOfCollapsedDims != 1))
-      return std::nullopt;
-
-    // If the collapsed dim is dynamic, the current expanded dim should also
-    // be dynamic.
-    if (currTargetShape == ShapedType::kDynamic &&
-        sourceShape[sourceDim] != ShapedType::kDynamic)
-      return std::nullopt;
-
-    // For static shapes, if the product of dimensions of the expanded shape
-    // should match the collapsed dimension shape.
-    if (prodOfCollapsedDims * sourceShape[sourceDim] != currTargetShape)
-      return std::nullopt;
-
-    currIndices.push_back(sourceDim++);
-    reassociationMap.emplace_back(ReassociationIndices{});
-    std::swap(reassociationMap.back(), currIndices);
-    prodOfCollapsedDims = 1;
+    // Store the gathered information as required for the next iteration.
+    prevTargetSize = targetSize;
+    sourceDimIdx = sourceRange->rightIdx + 1;
+    reassocRanges.push_back(*sourceRange);
+  }
+  // Fail if the source shape wasn't a full match for the target shape. We only
+  // need to check the last recorded index - any other gaps should have been
+  // mended by the main loop.
+  if (reassocRanges.back().rightIdx < sourceShapeAsRange.rightIdx)
+    return failure();
+  return reassocRanges;
+}
+
+/// A variant of `findReassociationRangesForCollapse(...)` that can also scan
+/// the shapes right-to-left.
+static FailureOr<SmallVector<ReassociationIndexRange>>
+findReassociationRangesForCollapse(ArrayRef<int64_t> sourceShape,
+                                   ArrayRef<int64_t> targetShape,
+                                   bool iterateRightToLeft) {
+  if (!iterateRightToLeft)
+    return findReassociationRangesForCollapse(sourceShape, targetShape);
+  // NB: To iterate right-to-left, we currently reverse the shapes and then
+  // reverse the result back. The reversed shapes must not be temporary, as
+  // we're passing through an ArrayRef.
+  // FIXME: It would be preferable to avoid the expensive copies. At the moment,
+  // this approach is chosen for readability of the main implementation.
+  std::vector<int64_t> sourceToReverse = sourceShape.vec(),
+                       targetToReverse = targetShape.vec();
+  std::reverse(sourceToReverse.begin(), sourceToReverse.end());
+  std::reverse(targetToReverse.begin(), targetToReverse.end());
+  auto invertedRanges =
+      findReassociationRangesForCollapse(sourceToReverse, targetToReverse);
+  if (failed(invertedRanges))
+    return failure();
+  SmallVector<ReassociationIndexRange> &rangesToInvert = *invertedRanges;
+  unsigned numSourceDims = sourceShape.size();
+  // We have received the ranges for inverted shapes. Now we have to invert
+  // the ranges back to correspond with the original source shape.
+  for (auto &range : rangesToInvert) {
+    int64_t invLeftIdx = range.leftIdx, invRightIdx = range.rightIdx;
+    range.leftIdx = numSourceDims - 1 - invRightIdx;
+    range.rightIdx = numSourceDims - 1 - invLeftIdx;
   }
-  // All the dimensions in the target must have been processed.
-  if (reassociationMap.size() != targetShape.size())
+  // Also invert the ordering of the ranges to correspond with the original
+  // target shape.
+  std::reverse(rangesToInvert.begin(), rangesToInvert.end());
+  return rangesToInvert;
+}
+
+std::optional<SmallVector<ReassociationIndices>>
+mlir::getReassociationIndicesForCollapse(ArrayRef<int64_t> sourceShape,
+                                         ArrayRef<int64_t> targetShape) {
+  unsigned numSourceDims = sourceShape.size(),
+           numTargetDims = targetShape.size();
+  // We're supposed to search for a collapsing reassociation. If the sizes
+  // match, there's no actual collapsing taking place - it's either a no-op or a
+  // `tensor.reshape`-style reassociation (that would be beyond the scope of
+  // this utility).
+  if (numSourceDims <= numTargetDims)
     return std::nullopt;
-  // Process any remaining entries in the source shape. They all need to be
-  // 1 or dynamic.
-  for (; sourceDim < sourceShape.size(); sourceDim++) {
-    if (sourceShape[sourceDim] != ShapedType::kDynamic &&
-        sourceShape[sourceDim] != 1)
-      return std::nullopt;
-    // The map is empty when the target type is a scalar.
-    if (!reassociationMap.empty())
-      reassociationMap.back().push_back(sourceDim);
+  // Early handling for scalar target types. We should report an invalid
+  // reassociation for non-unit static dimensions - no chance to collapse these
+  // into a scalar.
+  if (numTargetDims == 0) {
+    for (unsigned sourceDimIdx = 0; sourceDimIdx < numSourceDims;
+         ++sourceDimIdx) {
+      int64_t sourceSize = sourceShape[sourceDimIdx];
+      if (sourceSize != 1 && sourceSize != ShapedType::kDynamic)
+        return std::nullopt;
+    }
+    return SmallVector<ReassociationIndices>{};
+  }
+
+  // Collect source ranges by iterating over the target shape left-to-right.
+  FailureOr<SmallVector<ReassociationIndexRange>> maybeForwardRanges =
+      findReassociationRangesForCollapse(sourceShape, targetShape);
+  if (failed(maybeForwardRanges))
+    return std::nullopt;
+  auto &ranges = *maybeForwardRanges;
+  // Now do the same in reverse. We need to get another valid reassociation
+  // through some other strategy, and then compare the results in order to
+  // disambiguate mixed subshapes, such as:
+  // ?x?x? into ?x?, ?x2x? into ?x?, ?x2x3x6x? into ?x6x?
+  // This leads us to lose some of the reassociation opportunities that can only
+  // be found by iterating in a certain direction, e.g. 2x2x? into 2x? - without
+  // backtracking, the algorithm will fail right-to-left. However, this is the
+  // best way to preserve correctness.
+  FailureOr<SmallVector<ReassociationIndexRange>> maybeReverseRanges =
+      findReassociationRangesForCollapse(sourceShape, targetShape,
+                                         /*iterateRightToLeft=*/true);
+  if (failed(maybeReverseRanges))
+    return std::nullopt;
+  auto &reverseRanges = *maybeReverseRanges;
+
+  if (ranges.size() != numTargetDims || reverseRanges.size() != numTargetDims)
+    return std::nullopt;
+  // Now we can check for ambiguity of each target dimension's reassociation. If
+  // successful, we put the full indices into our result map for the target
+  // shape.
+  SmallVector<ReassociationIndices> reassociationMap(numTargetDims);
+  for (unsigned targetDimIdx = 0; targetDimIdx < numTargetDims;
+       ++targetDimIdx) {
+    ReassociationIndexRange &range = ranges[targetDimIdx];
+    ReassociationIndexRange &reverseRange = reverseRanges[targetDimIdx];
+    // Get non-overlapping indices between the ranges
+    ReassociationIndices nonMatchingIndices =
+        range.getNonOverlappingIndicesWith(reverseRange);
+    // Unit dimensions can be collapsed wherever - this is the only ambiguity
+    // that we allow.
+    for (int64_t sourceDimIdx : nonMatchingIndices) {
+      if (sourceShape[sourceDimIdx] != 1)
+        return std::nullopt;
+    }
+    reassociationMap[targetDimIdx] = range.getFullIndices();
   }
   return reassociationMap;
 }
@@ -315,11 +579,11 @@ SmallVector<Range> SliceFromCollapseHelper::getExtractSliceParams(
     // have proven that these are not sliced. In this case we just take
     // the full extent of each dimension in the reassociation list.
     if (linearizedDimensions[it.index()]) {
-      llvm::append_range(
-          offsetsSizesAndStrides,
-          llvm::map_range(it.value(), [&](int64_t idx) -> Range {
-            return {zeroAttr, collapseShapeInputShape[idx], oneAttr};
-          }));
+      llvm::append_range(offsetsSizesAndStrides,
+                         llvm::map_range(it.value(), [&](int64_t idx) -> Range {
+                           return {zeroAttr, collapseShapeInputShape[idx],
+                                   oneAttr};
+                         }));
       continue;
     }
 
diff --git a/external/llvm-project/mlir/lib/Dialect/Utils/StaticValueUtils.cpp b/external/llvm-project/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
index 29f7bd6857c2..8e3f796af54d 100644
--- a/external/llvm-project/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
@@ -142,8 +142,7 @@ getConstantIntValues(ArrayRef<OpFoldResult> ofrs) {
 }
 
 bool isConstantIntValue(OpFoldResult ofr, int64_t value) {
-  auto val = getConstantIntValue(ofr);
-  return val && *val == value;
+  return getConstantIntValue(ofr) == value;
 }
 
 bool areAllConstantIntValue(ArrayRef<OpFoldResult> ofrs, int64_t value) {
diff --git a/external/llvm-project/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/external/llvm-project/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 890a5e9e5c9b..2a2357319bd2 100644
--- a/external/llvm-project/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -33,6 +33,7 @@
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
+#include "mlir/IR/ValueRange.h"
 #include "mlir/Interfaces/SubsetOpInterface.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
 #include "mlir/Support/LLVM.h"
@@ -2387,9 +2388,129 @@ static LogicalResult rewriteFromElementsAsSplat(FromElementsOp fromElementsOp,
   return success();
 }
 
+/// Rewrite from_elements on multiple scalar extracts as a shape_cast
+/// on a single extract. Example:
+///   %0 = vector.extract %source[0, 0] : i8 from vector<2x2xi8>
+///   %1 = vector.extract %source[0, 1] : i8 from vector<2x2xi8>
+///   %2 = vector.from_elements %0, %1 : vector<2xi8>
+///
+/// becomes
+///   %1 = vector.extract %source[0] : vector<1x2xi8> from vector<2x2xi8>
+///   %2 = vector.shape_cast %1 : vector<1x2xi8> to vector<2xi8>
+///
+/// The requirements for this to be valid are
+///
+///   i) The elements are extracted from the same vector (%source).
+///
+///  ii) The elements form a suffix of %source. Specifically, the number
+///      of elements is the same as the product of the last N dimension sizes
+///      of %source, for some N.
+///
+/// iii) The elements are extracted contiguously in ascending order.
+
+class FromElementsToShapeCast : public OpRewritePattern<FromElementsOp> {
+
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(FromElementsOp fromElements,
+                                PatternRewriter &rewriter) const override {
+
+    // Handled by `rewriteFromElementsAsSplat`
+    if (fromElements.getType().getNumElements() == 1)
+      return failure();
+
+    // The common source that all elements are extracted from, if one exists.
+    TypedValue<VectorType> source;
+    // The position of the combined extract operation, if one is created.
+    ArrayRef<int64_t> combinedPosition;
+    // The expected index of extraction of the current element in the loop, if
+    // elements are extracted contiguously in ascending order.
+    SmallVector<int64_t> expectedPosition;
+
+    for (auto [insertIndex, element] :
+         llvm::enumerate(fromElements.getElements())) {
+
+      // Check that the element is from a vector.extract operation.
+      auto extractOp =
+          dyn_cast_if_present<vector::ExtractOp>(element.getDefiningOp());
+      if (!extractOp) {
+        return rewriter.notifyMatchFailure(fromElements,
+                                           "element not from vector.extract");
+      }
+
+      // Check condition (i) by checking that all elements have the same source
+      // as the first element.
+      if (insertIndex == 0) {
+        source = extractOp.getVector();
+      } else if (extractOp.getVector() != source) {
+        return rewriter.notifyMatchFailure(fromElements,
+                                           "element from different vector");
+      }
+
+      ArrayRef<int64_t> position = extractOp.getStaticPosition();
+      int64_t rank = position.size();
+      assert(rank == source.getType().getRank() &&
+             "scalar extract must have full rank position");
+
+      // Check condition (ii) by checking that the position that the first
+      // element is extracted from has sufficient trailing 0s. For example, in
+      //
+      //   %elm0 = vector.extract %source[1, 0, 0] : i8 from vector<2x3x4xi8>
+      //   [...]
+      //   %elms = vector.from_elements %elm0, [...] : vector<12xi8>
+      //
+      // The 2 trailing 0s in the position of extraction of %elm0 cover 3*4 = 12
+      // elements, which is the number of elements of %n, so this is valid.
+      if (insertIndex == 0) {
+        const int64_t numElms = fromElements.getType().getNumElements();
+        int64_t numSuffixElms = 1;
+        int64_t index = rank;
+        while (index > 0 && position[index - 1] == 0 &&
+               numSuffixElms < numElms) {
+          numSuffixElms *= source.getType().getDimSize(index - 1);
+          --index;
+        }
+        if (numSuffixElms != numElms) {
+          return rewriter.notifyMatchFailure(
+              fromElements, "elements do not form a suffix of source");
+        }
+        expectedPosition = llvm::to_vector(position);
+        combinedPosition = position.drop_back(rank - index);
+      }
+
+      // Check condition (iii).
+      else if (expectedPosition != position) {
+        return rewriter.notifyMatchFailure(
+            fromElements, "elements not in ascending order (static order)");
+      }
+      increment(expectedPosition, source.getType().getShape());
+    }
+
+    auto extracted = rewriter.createOrFold<vector::ExtractOp>(
+        fromElements.getLoc(), source, combinedPosition);
+
+    rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
+        fromElements, fromElements.getType(), extracted);
+
+    return success();
+  }
+
+  /// Increments n-D `indices` by 1 starting from the innermost dimension.
+  static void increment(MutableArrayRef<int64_t> indices,
+                        ArrayRef<int64_t> shape) {
+    for (int dim : llvm::reverse(llvm::seq<int>(0, indices.size()))) {
+      indices[dim] += 1;
+      if (indices[dim] < shape[dim])
+        break;
+      indices[dim] = 0;
+    }
+  }
+};
+
 void FromElementsOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                                  MLIRContext *context) {
   results.add(rewriteFromElementsAsSplat);
+  results.add<FromElementsToShapeCast>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2401,6 +2522,10 @@ void BroadcastOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges,
   setResultRanges(getResult(), argRanges.front());
 }
 
+std::optional<SmallVector<int64_t, 4>> BroadcastOp::getShapeForUnroll() {
+  return llvm::to_vector<4>(getResultVectorType().getShape());
+}
+
 /// Return the dimensions of the result vector that were formerly ones in the
 /// source tensor and thus correspond to "dim-1" broadcasting.
 static llvm::SetVector<int64_t>
@@ -4543,12 +4668,15 @@ struct TransferReadAfterWriteToBroadcast
 
   LogicalResult matchAndRewrite(TransferReadOp readOp,
                                 PatternRewriter &rewriter) const override {
-    if (readOp.hasOutOfBoundsDim() ||
-        !llvm::isa<RankedTensorType>(readOp.getShapedType()))
-      return failure();
     auto defWrite = readOp.getBase().getDefiningOp<vector::TransferWriteOp>();
     if (!defWrite)
       return failure();
+    // Bail if we need an alias analysis.
+    if (!readOp.hasPureTensorSemantics() || !defWrite.hasPureTensorSemantics())
+      return failure();
+    // Bail if we need a bounds analysis.
+    if (readOp.hasOutOfBoundsDim() || defWrite.hasOutOfBoundsDim())
+      return failure();
     // TODO: If the written transfer chunk is a superset of the read transfer
     // chunk we could do an extract_strided_slice.
     if (readOp.getTransferChunkAccessed() !=
@@ -4559,15 +4687,28 @@ struct TransferReadAfterWriteToBroadcast
     if (getUnusedDimsBitVector({readOp.getPermutationMap()}) !=
         getUnusedDimsBitVector({defWrite.getPermutationMap()}))
       return failure();
-    if (readOp.getIndices() != defWrite.getIndices() ||
-        readOp.getMask() != defWrite.getMask())
+    // This pattern should only catch the broadcast case, the non-broadcast case
+    // should be done separately to keep application conditions clean and
+    // separate.
+    AffineMap readMap = compressUnusedDims(readOp.getPermutationMap());
+    AffineMap writeMap = compressUnusedDims(defWrite.getPermutationMap());
+    bool bcast = !readMap.getBroadcastDims().empty() ||
+                 !writeMap.getBroadcastDims().empty();
+    if (!bcast)
+      return failure();
+    // At this point, we know we have a bcast.
+    // Bail in the masked case (too complex atm and needed to properly account
+    // for padding).
+    if (readOp.getMask() || defWrite.getMask())
+      return failure();
+    // If indices are not the same a shift may be required, bail.
+    if (readOp.getIndices() != defWrite.getIndices())
       return failure();
+
     Value vec = defWrite.getVector();
     // TODO: loop through the chain of transfer_write if we can prove that they
     // don't overlap with the transfer_read. This requires improving
     // `isDisjointTransferIndices` helper.
-    AffineMap readMap = compressUnusedDims(readOp.getPermutationMap());
-    AffineMap writeMap = compressUnusedDims(defWrite.getPermutationMap());
     AffineMap map = readMap.compose(writeMap);
     if (map.getNumResults() == 0)
       return failure();
@@ -4913,7 +5054,7 @@ void TransferWriteOp::getEffects(
     SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
         &effects) {
   if (llvm::isa<MemRefType>(getShapedType()))
-    effects.emplace_back(MemoryEffects::Write::get(), &getValueToStoreMutable(),
+    effects.emplace_back(MemoryEffects::Write::get(), &getBaseMutable(),
                          SideEffects::DefaultResource::get());
 }
 
diff --git a/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp b/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
index 23324a007377..39c16fab21c4 100644
--- a/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/LowerVectorShapeCast.cpp
@@ -21,177 +21,298 @@
 #include "mlir/IR/Location.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
+#include <numeric>
 
 #define DEBUG_TYPE "vector-shape-cast-lowering"
 
 using namespace mlir;
-using namespace mlir::vector;
-
-/// Increments n-D `indices` by `step` starting from the innermost dimension.
-static void incIdx(SmallVectorImpl<int64_t> &indices, VectorType vecType,
-                   int step = 1) {
-  for (int dim : llvm::reverse(llvm::seq<int>(0, indices.size()))) {
-    assert(indices[dim] < vecType.getDimSize(dim) &&
-           "Indices are out of bound");
-    indices[dim] += step;
-    if (indices[dim] < vecType.getDimSize(dim))
-      break;
 
-    indices[dim] = 0;
-    step = 1;
+/// Perform the inplace update
+///    rhs <- lhs + rhs
+///
+/// where `rhs` is a number expressed in mixed base `base` with most signficant
+/// dimensions on the left. For example if `rhs` is {a,b,c} and `base` is
+/// {5,3,2} then `rhs` has value a*3*2 + b*2 + c.
+///
+/// Some examples where `base` is {5,3,2}:
+/// rhs = {0,0,0}, lhs = 1  --> rhs = {0,0,1}
+/// rhs = {0,0,1}, lhs = 1  --> rhs = {0,1,0}
+/// rhs = {0,0,0}, lhs = 25 --> rhs = {4,0,1}
+///
+/// Invalid:
+/// rhs = {0,0,2}, lhs = 1 : rhs not in base {5,3,2}
+///
+/// Overflows not handled correctly:
+/// rhs = {4,2,1}, lhs = 2 --> rhs = {0,0,0} (not {0,0,1})
+static void inplaceAdd(int64_t lhs, ArrayRef<int64_t> base,
+                       MutableArrayRef<int64_t> rhs) {
+
+  // For dimensions in [numIndices - 1, ..., 3, 2, 1, 0]:
+  for (int dim : llvm::reverse(llvm::seq<int>(0, rhs.size()))) {
+    int64_t dimBase = base[dim];
+    assert(rhs[dim] < dimBase && "rhs not in base");
+
+    int64_t incremented = rhs[dim] + lhs;
+
+    // If the incremented value excedes the dimension base, we must spill to the
+    // next most significant dimension and repeat (we might need to spill to
+    // more significant dimensions multiple times).
+    lhs = incremented / dimBase;
+    rhs[dim] = incremented % dimBase;
+    if (lhs == 0)
+      break;
   }
 }
 
 namespace {
-/// ShapeOp n-D -> 1-D downcast serves the purpose of flattening N-D to 1-D
-/// vectors progressively. This iterates over the n-1 major dimensions of the
-/// n-D vector and performs rewrites into:
-///   vector.extract from n-D + vector.insert_strided_slice offset into 1-D
-class ShapeCastOpNDDownCastRewritePattern
-    : public OpRewritePattern<vector::ShapeCastOp> {
-public:
-  using OpRewritePattern::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(vector::ShapeCastOp op,
-                                PatternRewriter &rewriter) const override {
-    auto sourceVectorType = op.getSourceVectorType();
-    auto resultVectorType = op.getResultVectorType();
-    if (sourceVectorType.isScalable() || resultVectorType.isScalable())
-      return failure();
+/// shape_cast is converted to a sequence of extract, extract_strided_slice,
+/// insert_strided_slice, and insert operations. The running example will be:
+///
+/// %0 = vector.shape_cast %arg0 :
+///         vector<2x2x3x4x7x11xi8> to vector<8x6x7x11xi8>
+///
+/// In this example the source and result shapes share a common suffix of 7x11.
+/// This means we can always decompose the shape_cast into extract, insert, and
+/// their strided equivalents, on vectors with shape suffix 7x11.
+///
+/// The greatest common divisor (gcd) of the first dimension preceding the
+/// common suffix is gcd(4,6) = 2. The algorithm implemented here will operate
+/// on vectors with shapes that are `multiples` of (what we define as) the
+/// 'atomic shape', 2x7x11. The atomic shape is `gcd` x `common-suffix`.
+///
+///         vector<2x2x3x4x7x11xi8> to
+///             vector<8x6x7x11xi8>
+///                      | ||||
+///                      | ++++------------> common suffix of 7x11
+///                      +----------------->    gcd(4,6) is 2 | |
+///                                                         | | |
+///                                                         v v v
+///                                 atomic shape   <-----   2x7x11
+///
+///
+///
+/// The decomposition implemented in this pattern consists of a sequence of
+/// repeated steps:
+///
+///  (1) Extract vectors from the suffix of the source.
+///      In our example this is 2x2x3x4x7x11 -> 4x7x11.
+///
+///  (2) Do extract_strided_slice down to the atomic shape.
+///      In our example this is 4x7x11 -> 2x7x11.
+///
+///  (3) Do insert_strided_slice to the suffix of the result.
+///      In our example this is 2x7x11 -> 6x7x11.
+///
+///  (4) insert these vectors into the result vector.
+///      In our example this is 6x7x11 -> 8x6x7x11.
+///
+/// These steps occur with different periods. In this example
+///  (1) occurs 12 times,
+///  (2) and (3) occur 24 times, and
+///  (4) occurs 8 times.
+///
+/// Two special cases are handled independently in this pattern
+///  (i) A shape_cast that just does leading 1 insertion/removal
+/// (ii) A shape_cast where the gcd is 1.
+///
+/// These 2 cases can have more compact IR generated by not using the generic
+/// algorithm described above.
+///
+class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
+
+  // Case (i) of description.
+  // Assumes source and result shapes are identical up to some leading ones.
+  static LogicalResult leadingOnesLowering(vector::ShapeCastOp shapeCast,
+                                           PatternRewriter &rewriter) {
+
+    const Location loc = shapeCast.getLoc();
+    const VectorType sourceType = shapeCast.getSourceVectorType();
+    const VectorType resultType = shapeCast.getResultVectorType();
+
+    const int64_t sourceRank = sourceType.getRank();
+    const int64_t resultRank = resultType.getRank();
+    const int64_t delta = sourceRank - resultRank;
+    const int64_t sourceLeading = delta > 0 ? delta : 0;
+    const int64_t resultLeading = delta > 0 ? 0 : -delta;
+
+    const Value source = shapeCast.getSource();
+    const Value poison = rewriter.create<ub::PoisonOp>(loc, resultType);
+    const Value extracted = rewriter.create<vector::ExtractOp>(
+        loc, source, SmallVector<int64_t>(sourceLeading, 0));
+    const Value result = rewriter.create<vector::InsertOp>(
+        loc, extracted, poison, SmallVector<int64_t>(resultLeading, 0));
+
+    rewriter.replaceOp(shapeCast, result);
+    return success();
+  }
 
-    int64_t srcRank = sourceVectorType.getRank();
-    int64_t resRank = resultVectorType.getRank();
-    if (srcRank < 2 || resRank != 1)
-      return failure();
+  // Case (ii) of description.
+  // Assumes a shape_cast where the suffix shape of the source starting at
+  // `sourceDim` and the suffix shape of the result starting at `resultDim` are
+  // identical.
+  static LogicalResult noStridedSliceLowering(vector::ShapeCastOp shapeCast,
+                                              int64_t sourceDim,
+                                              int64_t resultDim,
+                                              PatternRewriter &rewriter) {
 
-    // Compute the number of 1-D vector elements involved in the reshape.
-    int64_t numElts = 1;
-    for (int64_t dim = 0; dim < srcRank - 1; ++dim)
-      numElts *= sourceVectorType.getDimSize(dim);
+    const Location loc = shapeCast.getLoc();
 
-    auto loc = op.getLoc();
-    SmallVector<int64_t> srcIdx(srcRank - 1, 0);
-    SmallVector<int64_t> resIdx(resRank, 0);
-    int64_t extractSize = sourceVectorType.getShape().back();
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
+    const Value source = shapeCast.getSource();
+    const ArrayRef<int64_t> sourceShape =
+        shapeCast.getSourceVectorType().getShape();
 
-    // Compute the indices of each 1-D vector element of the source extraction
-    // and destination slice insertion and generate such instructions.
-    for (int64_t i = 0; i < numElts; ++i) {
-      if (i != 0) {
-        incIdx(srcIdx, sourceVectorType, /*step=*/1);
-        incIdx(resIdx, resultVectorType, /*step=*/extractSize);
-      }
+    const VectorType resultType = shapeCast.getResultVectorType();
+    const ArrayRef<int64_t> resultShape = resultType.getShape();
 
-      Value extract =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), srcIdx);
-      result = rewriter.create<vector::InsertStridedSliceOp>(
-          loc, extract, result,
-          /*offsets=*/resIdx, /*strides=*/1);
-    }
+    const int64_t nSlices =
+        std::accumulate(sourceShape.begin(), sourceShape.begin() + sourceDim, 1,
+                        std::multiplies<int64_t>());
 
-    rewriter.replaceOp(op, result);
-    return success();
-  }
-};
+    SmallVector<int64_t> extractIndex(sourceDim, 0);
+    SmallVector<int64_t> insertIndex(resultDim, 0);
+    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
 
-/// ShapeOp 1-D -> n-D upcast serves the purpose of unflattening n-D from 1-D
-/// vectors progressively. This iterates over the n-1 major dimension of the n-D
-/// vector and performs rewrites into:
-///   vector.extract_strided_slice from 1-D + vector.insert into n-D
-/// Note that 1-D extract_strided_slice are lowered to efficient vector.shuffle.
-class ShapeCastOpNDUpCastRewritePattern
-    : public OpRewritePattern<vector::ShapeCastOp> {
-public:
-  using OpRewritePattern::OpRewritePattern;
+    for (int i = 0; i < nSlices; ++i) {
+      Value extracted =
+          rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
 
-  LogicalResult matchAndRewrite(vector::ShapeCastOp op,
-                                PatternRewriter &rewriter) const override {
-    auto sourceVectorType = op.getSourceVectorType();
-    auto resultVectorType = op.getResultVectorType();
-    if (sourceVectorType.isScalable() || resultVectorType.isScalable())
-      return failure();
-
-    int64_t srcRank = sourceVectorType.getRank();
-    int64_t resRank = resultVectorType.getRank();
-    if (srcRank != 1 || resRank < 2)
-      return failure();
-
-    // Compute the number of 1-D vector elements involved in the reshape.
-    int64_t numElts = 1;
-    for (int64_t dim = 0; dim < resRank - 1; ++dim)
-      numElts *= resultVectorType.getDimSize(dim);
-
-    // Compute the indices of each 1-D vector element of the source slice
-    // extraction and destination insertion and generate such instructions.
-    auto loc = op.getLoc();
-    SmallVector<int64_t> srcIdx(srcRank, 0);
-    SmallVector<int64_t> resIdx(resRank - 1, 0);
-    int64_t extractSize = resultVectorType.getShape().back();
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
-    for (int64_t i = 0; i < numElts; ++i) {
-      if (i != 0) {
-        incIdx(srcIdx, sourceVectorType, /*step=*/extractSize);
-        incIdx(resIdx, resultVectorType, /*step=*/1);
-      }
+      result = rewriter.create<vector::InsertOp>(loc, extracted, result,
+                                                 insertIndex);
 
-      Value extract = rewriter.create<vector::ExtractStridedSliceOp>(
-          loc, op.getSource(), /*offsets=*/srcIdx, /*sizes=*/extractSize,
-          /*strides=*/1);
-      result = rewriter.create<vector::InsertOp>(loc, extract, result, resIdx);
+      inplaceAdd(1, sourceShape.take_front(sourceDim), extractIndex);
+      inplaceAdd(1, resultShape.take_front(resultDim), insertIndex);
     }
-    rewriter.replaceOp(op, result);
+    rewriter.replaceOp(shapeCast, result);
     return success();
   }
-};
 
-// We typically should not lower general shape cast operations into data
-// movement instructions, since the assumption is that these casts are
-// optimized away during progressive lowering. For completeness, however,
-// we fall back to a reference implementation that moves all elements
-// into the right place if we get here.
-class ShapeCastOpRewritePattern : public OpRewritePattern<vector::ShapeCastOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
 
   LogicalResult matchAndRewrite(vector::ShapeCastOp op,
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
-    auto sourceVectorType = op.getSourceVectorType();
-    auto resultVectorType = op.getResultVectorType();
+    VectorType sourceType = op.getSourceVectorType();
+    VectorType resultType = op.getResultVectorType();
+
+    if (sourceType.isScalable() || resultType.isScalable())
+      return rewriter.notifyMatchFailure(
+          op,
+          "shape_cast where vectors are scalable not handled by this pattern");
+
+    const ArrayRef<int64_t> sourceShape = sourceType.getShape();
+    const ArrayRef<int64_t> resultShape = resultType.getShape();
+    const int64_t sourceRank = sourceType.getRank();
+    const int64_t resultRank = resultType.getRank();
+    const int64_t numElms = sourceType.getNumElements();
+    const Value source = op.getSource();
+
+    // Set the first dimension (starting at the end) in the source and result
+    // respectively where the dimension sizes differ. Using the running example:
+    //
+    //  dimensions:  [0 1 2 3 4 5 ]    [0 1 2 3 ]
+    //  shapes:      (2,2,3,4,7,11) -> (8,6,7,11)
+    //                      ^             ^
+    //                      |             |
+    //        sourceSuffixStartDim is 3   |
+    //                                    |
+    //                               resultSuffixStartDim is 1
+    int64_t sourceSuffixStartDim = sourceRank - 1;
+    int64_t resultSuffixStartDim = resultRank - 1;
+    while (sourceSuffixStartDim >= 0 && resultSuffixStartDim >= 0 &&
+           (sourceType.getDimSize(sourceSuffixStartDim) ==
+            resultType.getDimSize(resultSuffixStartDim))) {
+      --sourceSuffixStartDim;
+      --resultSuffixStartDim;
+    }
 
-    if (sourceVectorType.isScalable() || resultVectorType.isScalable())
-      return failure();
-
-    // Special case for n-D / 1-D lowerings with better implementations.
-    int64_t srcRank = sourceVectorType.getRank();
-    int64_t resRank = resultVectorType.getRank();
-    if ((srcRank > 1 && resRank == 1) || (srcRank == 1 && resRank > 1))
-      return failure();
-
-    // Generic ShapeCast lowering path goes all the way down to unrolled scalar
-    // extract/insert chains.
-    int64_t numElts = 1;
-    for (int64_t r = 0; r < srcRank; r++)
-      numElts *= sourceVectorType.getDimSize(r);
-    // Replace with data movement operations:
-    //    x[0,0,0] = y[0,0]
-    //    x[0,0,1] = y[0,1]
-    //    x[0,1,0] = y[0,2]
-    // etc., incrementing the two index vectors "row-major"
-    // within the source and result shape.
-    SmallVector<int64_t> srcIdx(srcRank, 0);
-    SmallVector<int64_t> resIdx(resRank, 0);
-    Value result = rewriter.create<ub::PoisonOp>(loc, resultVectorType);
-    for (int64_t i = 0; i < numElts; i++) {
-      if (i != 0) {
-        incIdx(srcIdx, sourceVectorType);
-        incIdx(resIdx, resultVectorType);
+    // This is the case (i) where there are just some leading ones to contend
+    // with in the source or result. It can be handled with a single
+    // extract/insert pair.
+    if (resultSuffixStartDim < 0 || sourceSuffixStartDim < 0)
+      return leadingOnesLowering(op, rewriter);
+
+    const int64_t sourceSuffixStartDimSize =
+        sourceType.getDimSize(sourceSuffixStartDim);
+    const int64_t resultSuffixStartDimSize =
+        resultType.getDimSize(resultSuffixStartDim);
+    const int64_t greatestCommonDivisor =
+        std::gcd(sourceSuffixStartDimSize, resultSuffixStartDimSize);
+    const int64_t stridedSliceRank = sourceRank - sourceSuffixStartDim;
+    const size_t extractPeriod =
+        sourceSuffixStartDimSize / greatestCommonDivisor;
+    const size_t insertPeriod =
+        resultSuffixStartDimSize / greatestCommonDivisor;
+
+    SmallVector<int64_t> atomicShape(sourceShape.begin() + sourceSuffixStartDim,
+                                     sourceShape.end());
+    atomicShape[0] = greatestCommonDivisor;
+
+    const int64_t numAtomicElms = std::accumulate(
+        atomicShape.begin(), atomicShape.end(), 1, std::multiplies<int64_t>());
+    const size_t nAtomicSlices = numElms / numAtomicElms;
+
+    // This is the case (ii) where the strided dimension size is 1. More compact
+    // IR is generated in this case if we just extract and insert the elements
+    // directly. In other words, we don't use extract_strided_slice and
+    // insert_strided_slice.
+    if (greatestCommonDivisor == 1)
+      return noStridedSliceLowering(op, sourceSuffixStartDim + 1,
+                                    resultSuffixStartDim + 1, rewriter);
+
+    // The insert_strided_slice result's type
+    const ArrayRef<int64_t> insertStridedShape =
+        resultShape.drop_front(resultSuffixStartDim);
+    const VectorType insertStridedType =
+        VectorType::get(insertStridedShape, resultType.getElementType());
+
+    SmallVector<int64_t> extractIndex(sourceSuffixStartDim, 0);
+    SmallVector<int64_t> insertIndex(resultSuffixStartDim, 0);
+    SmallVector<int64_t> extractOffsets(stridedSliceRank, 0);
+    SmallVector<int64_t> insertOffsets(stridedSliceRank, 0);
+    const SmallVector<int64_t> sizes(stridedSliceRank, 1);
+
+    Value extracted = {};
+    Value extractedStrided = {};
+    Value insertedSlice = {};
+    Value result = rewriter.create<ub::PoisonOp>(loc, resultType);
+    const Value partResult =
+        rewriter.create<ub::PoisonOp>(loc, insertStridedType);
+
+    for (size_t i = 0; i < nAtomicSlices; ++i) {
+
+      const size_t extractStridedPhase = i % extractPeriod;
+      const size_t insertStridedPhase = i % insertPeriod;
+
+      // vector.extract
+      if (extractStridedPhase == 0) {
+        extracted =
+            rewriter.create<vector::ExtractOp>(loc, source, extractIndex);
+        inplaceAdd(1, sourceShape.take_front(sourceSuffixStartDim),
+                   extractIndex);
       }
 
-      Value extract =
-          rewriter.create<vector::ExtractOp>(loc, op.getSource(), srcIdx);
-      result = rewriter.create<vector::InsertOp>(loc, extract, result, resIdx);
+      // vector.extract_strided_slice
+      extractOffsets[0] = extractStridedPhase * greatestCommonDivisor;
+      extractedStrided = rewriter.create<vector::ExtractStridedSliceOp>(
+          loc, extracted, extractOffsets, atomicShape, sizes);
+
+      // vector.insert_strided_slice
+      if (insertStridedPhase == 0) {
+        insertedSlice = partResult;
+      }
+      insertOffsets[0] = insertStridedPhase * greatestCommonDivisor;
+      insertedSlice = rewriter.create<vector::InsertStridedSliceOp>(
+          loc, extractedStrided, insertedSlice, insertOffsets, sizes);
+
+      // vector.insert
+      if (insertStridedPhase + 1 == insertPeriod) {
+        result = rewriter.create<vector::InsertOp>(loc, insertedSlice, result,
+                                                   insertIndex);
+        inplaceAdd(1, resultType.getShape().take_front(resultSuffixStartDim),
+                   insertIndex);
+      }
     }
     rewriter.replaceOp(op, result);
     return success();
@@ -252,7 +373,8 @@ class ScalableShapeCastOpRewritePattern
     // from >= 2-D scalable vectors or scalable vectors of fixed vectors.
     if (!isTrailingDimScalable(sourceVectorType) ||
         !isTrailingDimScalable(resultVectorType)) {
-      return failure();
+      return rewriter.notifyMatchFailure(
+          op, "trailing dims are not scalable, not handled by this pattern");
     }
 
     // The sizes of the trailing dimension of the source and result vectors, the
@@ -329,8 +451,8 @@ class ScalableShapeCastOpRewritePattern
 
       // 4. Increment the insert/extract indices, stepping by minExtractionSize
       // for the trailing dimensions.
-      incIdx(srcIdx, sourceVectorType, /*step=*/minExtractionSize);
-      incIdx(resIdx, resultVectorType, /*step=*/minExtractionSize);
+      inplaceAdd(minExtractionSize, sourceVectorType.getShape(), srcIdx);
+      inplaceAdd(minExtractionSize, resultVectorType.getShape(), resIdx);
     }
 
     rewriter.replaceOp(op, result);
@@ -347,8 +469,6 @@ class ScalableShapeCastOpRewritePattern
 
 void mlir::vector::populateVectorShapeCastLoweringPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
-  patterns.add<ShapeCastOpNDDownCastRewritePattern,
-               ShapeCastOpNDUpCastRewritePattern, ShapeCastOpRewritePattern,
-               ScalableShapeCastOpRewritePattern>(patterns.getContext(),
-                                                  benefit);
+  patterns.add<ShapeCastOpRewritePattern, ScalableShapeCastOpRewritePattern>(
+      patterns.getContext(), benefit);
 }
diff --git a/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 1cc477d9dca9..fc443ab0d138 100644
--- a/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -631,14 +631,78 @@ struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
   vector::UnrollVectorOptions options;
 };
 
+struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
+  UnrollBroadcastPattern(MLIRContext *context,
+                         const vector::UnrollVectorOptions &options,
+                         PatternBenefit benefit = 1)
+      : OpRewritePattern<vector::BroadcastOp>(context, benefit),
+        options(options) {}
+
+  LogicalResult matchAndRewrite(vector::BroadcastOp broadcastOp,
+                                PatternRewriter &rewriter) const override {
+    auto targetShape = getTargetShape(options, broadcastOp);
+    if (!targetShape)
+      return failure();
+
+    Location loc = broadcastOp.getLoc();
+    VectorType srcType = dyn_cast<VectorType>(broadcastOp.getSourceType());
+    VectorType resType = broadcastOp.getResultVectorType();
+    VectorType targetType =
+        resType.cloneWith(*targetShape, resType.getElementType());
+    Value result = rewriter.create<arith::ConstantOp>(
+        loc, resType, rewriter.getZeroAttr(resType));
+
+    SmallVector<int64_t> originalShape = *broadcastOp.getShapeForUnroll();
+    SmallVector<int64_t> strides(originalShape.size(), 1);
+
+    for (SmallVector<int64_t> offsets :
+         StaticTileOffsetRange(originalShape, *targetShape)) {
+      Value newSrc;
+      if (!srcType) {
+        // Scalar to vector broadcast.
+        newSrc = broadcastOp.getSource();
+      } else {
+        // Vector to vector broadcast.
+        int64_t rank = srcType.getRank();
+        SmallVector<int64_t> srcOffsets(offsets.end() - rank, offsets.end());
+        SmallVector<int64_t> srcShape(targetShape->end() - rank,
+                                      targetShape->end());
+        SmallVector<int64_t> srcStrides(strides.end() - rank, strides.end());
+        // adjust the offset and shape for src if the corresponding dim is 1.
+        for (int64_t i = 0; i < rank; ++i) {
+          if (srcType.getDimSize(i) == 1) {
+            srcOffsets[i] = 0;
+            srcShape[i] = 1;
+          }
+        }
+        newSrc = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
+            loc, broadcastOp.getSource(), srcOffsets, srcShape, srcStrides);
+      }
+
+      Operation *newOp = cloneOpWithOperandsAndTypes(rewriter, loc, broadcastOp,
+                                                     newSrc, targetType);
+
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
+          loc, newOp->getResult(0), result, offsets, strides);
+    }
+
+    rewriter.replaceOp(broadcastOp, result);
+    return success();
+  }
+
+private:
+  vector::UnrollVectorOptions options;
+};
+
 } // namespace
 
 void mlir::vector::populateVectorUnrollPatterns(
     RewritePatternSet &patterns, const UnrollVectorOptions &options,
     PatternBenefit benefit) {
-  patterns.add<UnrollTransferReadPattern, UnrollTransferWritePattern,
-               UnrollContractionPattern, UnrollElementwisePattern,
-               UnrollReductionPattern, UnrollMultiReductionPattern,
-               UnrollTransposePattern, UnrollGatherPattern>(
-      patterns.getContext(), options, benefit);
+  patterns
+      .add<UnrollTransferReadPattern, UnrollTransferWritePattern,
+           UnrollContractionPattern, UnrollElementwisePattern,
+           UnrollReductionPattern, UnrollMultiReductionPattern,
+           UnrollTransposePattern, UnrollGatherPattern, UnrollBroadcastPattern>(
+          patterns.getContext(), options, benefit);
 }
diff --git a/external/llvm-project/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/external/llvm-project/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index d5dd6f2027be..590d244daef4 100644
--- a/external/llvm-project/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -337,15 +337,16 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
   auto sourceShape = sourceShapedType.getShape();
   assert(sourceShape.size() == inputVectorSizes.size() &&
          "expected same ranks.");
-  auto maskType = VectorType::get(inputVectorSizes, builder.getI1Type());
   auto vectorType = VectorType::get(inputVectorSizes, padValue.getType());
   assert(padValue.getType() == sourceShapedType.getElementType() &&
          "expected same pad element type to match source element type");
   int64_t readRank = inputVectorSizes.size();
   auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
   SmallVector<bool> inBoundsVal(readRank, true);
+
   if (useInBoundsInsteadOfMasking) {
     // Update the inBounds attribute.
+    // FIXME: This computation is too weak - it ignores the read indices.
     for (unsigned i = 0; i < readRank; i++)
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
                        !ShapedType::isDynamic(sourceShape[i]);
@@ -362,6 +363,8 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
     return transferReadOp;
   SmallVector<OpFoldResult> mixedSourceDims =
       tensor::getMixedSizes(builder, loc, source);
+
+  auto maskType = VectorType::get(inputVectorSizes, builder.getI1Type());
   Value mask =
       builder.create<vector::CreateMaskOp>(loc, maskType, mixedSourceDims);
   return mlir::vector::maskOperation(builder, transferReadOp, mask)
diff --git a/external/llvm-project/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp b/external/llvm-project/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
index b2fc2f3f40e8..8e062488f58c 100644
--- a/external/llvm-project/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
@@ -23,27 +23,18 @@ namespace {
 /// Generic one-to-one conversion of simply mappable operations into calls
 /// to their respective LLVM intrinsics.
 struct X86IntrinsicOpConversion
-    : public OpInterfaceConversionPattern<x86vector::X86IntrinsicOp> {
-  using OpInterfaceConversionPattern<
-      x86vector::X86IntrinsicOp>::OpInterfaceConversionPattern;
-
-  X86IntrinsicOpConversion(const LLVMTypeConverter &typeConverter,
-                           PatternBenefit benefit = 1)
-      : OpInterfaceConversionPattern(typeConverter, &typeConverter.getContext(),
-                                     benefit),
-        typeConverter(typeConverter) {}
+    : public ConvertOpInterfaceToLLVMPattern<x86vector::X86IntrinsicOp> {
+  using ConvertOpInterfaceToLLVMPattern::ConvertOpInterfaceToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(x86vector::X86IntrinsicOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
+    const LLVMTypeConverter &typeConverter = *getTypeConverter();
     return LLVM::detail::intrinsicRewrite(
         op, rewriter.getStringAttr(op.getIntrinsicName()),
         op.getIntrinsicOperands(operands, typeConverter, rewriter),
         typeConverter, rewriter);
   }
-
-private:
-  const LLVMTypeConverter &typeConverter;
 };
 
 } // namespace
diff --git a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt
index 7d9b5584b0b2..af0d7f6bd907 100644
--- a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_mlir_dialect_library(MLIRXeGPUTransforms
+  XeGPUBlocking.cpp
   XeGPUFoldAliasOps.cpp
   XeGPUSubgroupDistribute.cpp
   XeGPUUnroll.cpp
diff --git a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
new file mode 100644
index 000000000000..a3826c56e1f6
--- /dev/null
+++ b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
@@ -0,0 +1,343 @@
+//===---- XeGPUBlocking.cpp ---- XeGPU Blocking Pass ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
+
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
+#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
+#include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
+#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
+#include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/ADT/STLExtras.h"
+
+namespace mlir {
+namespace xegpu {
+#define GEN_PASS_DEF_XEGPUBLOCKING
+#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
+} // namespace xegpu
+} // namespace mlir
+
+#define DEBUG_TYPE "xegpu-blocking"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
+#define LDBG(X) LLVM_DEBUG(DBGS() << X << "\n")
+
+using namespace mlir;
+
+namespace {
+
+// reslove the unrealized conversion cast ops generated when doing SCF
+// Structural Type Conversion. It will have two formats, N:1 vector
+// cast and 1:N vector cast. vector::insert_strided_slice ops will be
+// used for the first case, and vector::extract_strided_slice ops will be
+// used for the second case.
+static void
+resolveUnrealizedConversionCastOp(UnrealizedConversionCastOp castOp) {
+  ValueRange inputs = castOp.getInputs();
+  ValueRange outputs = castOp.getOutputs();
+
+  auto hasIdenticalVectorTypes = [](ValueRange values) {
+    auto types = values.getTypes();
+    return llvm::all_of(types, [&](Type type) {
+      return isa<VectorType>(type) && type == types.front();
+    });
+  };
+
+  // We only interest in the case where all inputs and outputs have the
+  // identical VectorTypes
+  if (!hasIdenticalVectorTypes(inputs) || !hasIdenticalVectorTypes(outputs)) {
+    LDBG("skip unrealized conversion cast op not emulating pack/unpack.");
+    return;
+  }
+
+  VectorType outputTy = dyn_cast<VectorType>(outputs[0].getType());
+  OpBuilder builder(castOp);
+  if (inputs.size() > 1 && outputs.size() == 1) {
+    // the castOp is emulating an unpack op
+    ArrayRef<int64_t> shape = outputTy.getShape();
+    Value result = xegpu::createVectorWithShapeFromValues(
+        builder, castOp.getLoc(), inputs, shape);
+    castOp->replaceAllUsesWith(ValueRange(result));
+    castOp->erase();
+  } else if (castOp.getNumResults() > 1 && castOp.getNumOperands() == 1) {
+    // the castOp is emulating a pack op
+    ArrayRef<int64_t> tileShape = outputTy.getShape();
+    SmallVector<Value> results = xegpu::extractVectorsWithShapeFromValue(
+        builder, castOp.getLoc(), inputs[0], tileShape);
+    castOp->replaceAllUsesWith(results);
+    castOp->erase();
+  }
+}
+
+//===------------------------------------------------------------------------===//
+// The XeGPUBlockingPass leverages the unroll patterns for XeGPU and Vector ops
+// to partition operations that process large shapes into multiple operations on
+// smaller shapes, as specified by the inst_data in the layout attribute. This
+// enables each resulting operation to be efficiently mapped to a hardware
+// instruction.
+//===------------------------------------------------------------------------===//
+
+class XeGPUBlockingPass final
+    : public xegpu::impl::XeGPUBlockingBase<XeGPUBlockingPass> {
+public:
+  void runOnOperation() override;
+
+private:
+  // Get the tile shape for a given OpOperand or OpResult by examining the
+  // corresponding layout attribute. If layout is not present or is not a
+  // subgroup level layout, it returns std::nullopt.
+  template <typename T,
+            typename = std::enable_if_t<std::is_same_v<T, OpOperand> ||
+                                        std::is_same_v<T, OpResult>>>
+  std::optional<SmallVector<int64_t>>
+  getTileShape(const T &operandOrResult) const;
+
+  // Get the tile shape for a given operation.
+  std::optional<SmallVector<int64_t>> getTileShape(Operation *op) const;
+
+  // Determine if the operation requires unrolling. Return false if all operands
+  // and results have tile shapes identical to their original types. Otherwise,
+  // return true.
+  bool needsUnroll(Operation *op) const;
+};
+} // namespace
+
+template <typename T, typename>
+std::optional<SmallVector<int64_t>>
+XeGPUBlockingPass::getTileShape(const T &operandOrResult) const {
+  Value value;
+  if constexpr (std::is_same_v<T, OpOperand>)
+    value = operandOrResult.get();
+  else
+    value = (Value)operandOrResult;
+
+  xegpu::LayoutAttr layout = xegpu::getLayoutAttr(operandOrResult);
+  if (layout && layout.isSgLayout()) {
+    if (auto inst_data = layout.getInstData())
+      return llvm::to_vector_of<int64_t>(inst_data.asArrayRef());
+
+    if (auto type = dyn_cast<ShapedType>(value.getType()))
+      return llvm::to_vector(type.getShape());
+  }
+  LDBG("failed to getTileShape for: " << value);
+  return std::nullopt;
+}
+
+std::optional<SmallVector<int64_t>>
+XeGPUBlockingPass::getTileShape(Operation *op) const {
+  if (isa<xegpu::CreateNdDescOp, xegpu::UpdateNdOffsetOp>(op))
+    return getTileShape(op->getOpResult(0));
+  if (isa<xegpu::PrefetchNdOp, xegpu::LoadNdOp>(op))
+    return getTileShape(op->getOpOperand(0));
+  if (isa<xegpu::StoreNdOp>(op))
+    return getTileShape(op->getOpOperand(1));
+
+  if (isa<xegpu::DpasOp>(op)) {
+    std::optional<SmallVector<int64_t>> aTile =
+        getTileShape(op->getOpOperand(0));
+    std::optional<SmallVector<int64_t>> bTile =
+        getTileShape(op->getOpOperand(1));
+
+    if (!aTile || aTile->size() != 2 || !bTile || bTile->size() != 2)
+      return std::nullopt;
+
+    // semantic check for A and B
+    if ((*aTile)[1] != (*bTile)[0])
+      return std::nullopt;
+
+    // semantic check for C
+    if (op->getNumOperands() == 3) {
+      std::optional<SmallVector<int64_t>> cTile =
+          getTileShape(op->getOpOperand(2));
+      int64_t expectedCTile[2] = {(*aTile)[0], (*bTile)[1]};
+      if (!cTile || !llvm::equal(*cTile, expectedCTile))
+        return std::nullopt;
+    }
+
+    return SmallVector<int64_t>({(*aTile)[0], (*aTile)[1], (*bTile)[1]});
+  }
+
+  if (OpTrait::hasElementwiseMappableTraits(op) && op->getNumResults() == 1)
+    return getTileShape(op->getOpResult(0));
+
+  if (isa<vector::MultiDimReductionOp>(op))
+    return getTileShape(op->getOpOperand(0));
+
+  if (isa<vector::TransposeOp, vector::BroadcastOp>(op))
+    return getTileShape(op->getOpResult(0));
+
+  return std::nullopt;
+}
+
+bool XeGPUBlockingPass::needsUnroll(Operation *op) const {
+  // skip the op if any of its operands or results has workgroup level layouts
+  bool hasWgLayoutOperands =
+      llvm::any_of(op->getOpOperands(), [](OpOperand &opr) {
+        xegpu::LayoutAttr layout = xegpu::getLayoutAttr(opr);
+        return layout && layout.isWgLayout();
+      });
+  bool hasWgLayoutResults =
+      llvm::any_of(op->getOpResults(), [](OpResult result) {
+        xegpu::LayoutAttr layout = xegpu::getLayoutAttr(result);
+        return layout && layout.isWgLayout();
+      });
+  if (hasWgLayoutOperands || hasWgLayoutResults) {
+    LDBG("skip unrolling for op with workgroup level layout: " << *op);
+    return false;
+  }
+
+  auto isUnrollable = [](Value value, ArrayRef<int64_t> tileShape) {
+    Type valTy = value.getType();
+    if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(valTy)) {
+      xegpu::LayoutAttr layout = tdescTy.getLayoutAttr();
+      return layout && layout.getInstData();
+    }
+    auto shapedType = dyn_cast<ShapedType>(valTy);
+    return shapedType && !llvm::equal(tileShape, shapedType.getShape());
+  };
+
+  bool hasUnrollableOperands =
+      llvm::any_of(op->getOpOperands(), [&](OpOperand &opr) {
+        std::optional<SmallVector<int64_t>> tileShape = getTileShape(opr);
+        return tileShape.has_value() && isUnrollable(opr.get(), *tileShape);
+      });
+  bool hasUnrollableResults =
+      llvm::any_of(op->getOpResults(), [&](OpResult result) {
+        std::optional<SmallVector<int64_t>> tileShape = getTileShape(result);
+        return tileShape.has_value() && isUnrollable(result, *tileShape);
+      });
+  return hasUnrollableOperands || hasUnrollableResults;
+}
+
+void XeGPUBlockingPass::runOnOperation() {
+  MLIRContext *ctx = &getContext();
+  Operation *op = getOperation();
+
+  // Preserve the LayoutAttr for each operand to the owner's DictionaryAttr.
+  // This ensures that the LayoutAttr remains accessible even if the defining
+  // operation is replaced.
+  xegpu::setLayoutAttrs(op, [](Value v) { return xegpu::getLayoutAttr(v); });
+
+  auto getTileShapeAndCount = [](llvm::ArrayRef<int64_t> shape,
+                                 xegpu::LayoutAttr layout) {
+    int count = 1;
+    SmallVector<int64_t> tileShape(shape);
+    if (layout && layout.getInstData()) {
+      DenseI32ArrayAttr instData = layout.getInstData();
+      tileShape = llvm::to_vector_of<int64_t>(instData.asArrayRef());
+      count = computeProduct(shape) / computeProduct(tileShape);
+    }
+    return std::make_pair(tileShape, count);
+  };
+
+  // Perform type conversion for SCF control folow ops
+  TypeConverter converter;
+  converter.addConversion([](Type type) -> Type { return type; });
+  converter.addConversion(
+      [&](RankedTensorType type,
+          SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
+        Type elemTy = type.getElementType();
+        ArrayRef<int64_t> shape = type.getShape();
+
+        auto layout =
+            llvm::dyn_cast_if_present<xegpu::LayoutAttr>(type.getEncoding());
+        if (layout && layout.isWgLayout())
+          return failure();
+
+        int count;
+        SmallVector<int64_t> subShape;
+        std::tie(subShape, count) = getTileShapeAndCount(shape, layout);
+        auto newTy = VectorType::get(subShape, elemTy);
+        result.append(count, newTy);
+        return success();
+      });
+  converter.addConversion(
+      [&](xegpu::TensorDescType type,
+          SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
+        Type elemTy = type.getElementType();
+        ArrayRef<int64_t> shape = type.getShape();
+
+        xegpu::LayoutAttr layout = type.getLayoutAttr();
+        if (layout && layout.isWgLayout())
+          return failure();
+
+        int count;
+        SmallVector<int64_t> subShape;
+        std::tie(subShape, count) = getTileShapeAndCount(shape, layout);
+
+        if (layout)
+          layout = layout.dropInstData();
+
+        auto newTy = xegpu::TensorDescType::get(
+            type.getContext(), subShape, elemTy, type.getEncoding(), layout);
+        result.append(count, newTy);
+        return success();
+      });
+
+  xegpu::doSCFStructuralTypeConversionWithTensorType(op, converter);
+
+  xegpu::UnrollOptions options;
+  options.setFilterConstraint(
+      [&](Operation *op) -> LogicalResult { return success(needsUnroll(op)); });
+
+  options.setNativeShapeFn([&](Operation *op) { return getTileShape(op); });
+
+  options.setUnrolledTypesFn([&](ShapedType type, ArrayRef<int64_t> tileShape) {
+    Type elemTy = type.getElementType();
+    Type newTy;
+
+    if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(type))
+      newTy = xegpu::TensorDescType::get(
+          ctx, tileShape, elemTy, tdescTy.getEncoding(),
+          tdescTy.getLayoutAttr().dropInstData());
+    else
+      newTy = type.clone(tileShape, elemTy);
+
+    std::optional<SmallVector<int64_t>> ratio =
+        computeShapeRatio(type.getShape(), tileShape);
+    assert(ratio && "The shape of the type must be a multiple of tileShape.");
+    return SmallVector<Type>(computeProduct(*ratio), newTy);
+  });
+
+  RewritePatternSet patterns(ctx);
+
+  vector::UnrollVectorOptions vectorOptions;
+  vectorOptions.setNativeShapeFn(options.nativeShape);
+
+  populateXeGPUUnrollPatterns(patterns, options);
+  vector::populateVectorUnrollPatterns(patterns, vectorOptions);
+
+  (void)applyPatternsGreedily(op, std::move(patterns));
+
+  op->walk([](Operation *op) {
+    // Remove the layout attributes cached per operands.
+    for (OpOperand &opr : op->getOpOperands()) {
+      std::string name = xegpu::getLayoutName(opr);
+      if (op->hasAttrOfType<xegpu::LayoutAttr>(name))
+        op->removeAttr(name);
+    }
+
+    // Update the layout attributes per result.
+    for (OpResult result : op->getOpResults()) {
+      std::string name = xegpu::getLayoutName(result);
+      if (auto layout = op->getAttrOfType<xegpu::LayoutAttr>(name)) {
+        op->removeAttr(name);
+        if (!isa<LoopLikeOpInterface>(op))
+          xegpu::setLayoutAttr(result, layout.dropInstData());
+      }
+    }
+
+    // Resolve unrealized conversion cast ops emulating pack/unpack
+    if (auto castOp = dyn_cast<UnrealizedConversionCastOp>(op))
+      resolveUnrealizedConversionCastOp(castOp);
+  });
+}
diff --git a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index 992700524146..c84906cc4556 100644
--- a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -62,8 +62,6 @@ constexpr unsigned packedSizeInBitsForDefault =
     16; // Minimum packing size per register for DPAS A.
 constexpr unsigned packedSizeInBitsForDpasB =
     32; // Minimum packing size per register for DPAS B.
-static const char *const operandLayoutNamePrefix = "layout_operand_";
-static const char *const resultLayoutNamePrefix = "layout_result_";
 
 namespace {
 
@@ -729,10 +727,7 @@ class LayoutAttrAssignment {
 void LayoutAttrAssignment::assignToUsers(Value v, xegpu::LayoutAttr layout) {
   for (OpOperand &user : v.getUses()) {
     Operation *owner = user.getOwner();
-    unsigned operandNumber = user.getOperandNumber();
-    // Use a generic name for ease of querying the layout attribute later.
-    std::string attrName =
-        operandLayoutNamePrefix + std::to_string(operandNumber);
+    std::string attrName = xegpu::getLayoutName(user);
     owner->setAttr(attrName, layout);
   }
 }
@@ -806,10 +801,10 @@ LogicalResult LayoutAttrAssignment::assign(Operation *op) {
     return success();
   }
   // Otherwise simply attach the layout to the op itself.
-  for (auto [i, r] : llvm::enumerate(op->getResults())) {
+  for (auto r : op->getOpResults()) {
     xegpu::LayoutAttr layoutInfo = getLayoutAttrForValue(r);
     if (layoutInfo) {
-      std::string attrName = resultLayoutNamePrefix + std::to_string(i);
+      std::string attrName = xegpu::getLayoutName(r);
       op->setAttr(attrName, layoutInfo);
       // Attach the layout attribute to the users of the result.
       assignToUsers(r, layoutInfo);
@@ -929,11 +924,8 @@ static SmallVector<NamedAttribute>
 removeTemporaryLayoutAttributes(ArrayRef<NamedAttribute> attrs) {
   SmallVector<NamedAttribute> newAttrs;
   for (NamedAttribute attr : attrs) {
-    if (attr.getName().strref().contains(operandLayoutNamePrefix) ||
-        attr.getName().strref().contains(resultLayoutNamePrefix)) {
-      continue;
-    }
-    newAttrs.push_back(attr);
+    if (!isa<xegpu::LayoutAttr>(attr.getValue()))
+      newAttrs.push_back(attr);
   }
   return newAttrs;
 }
@@ -1336,11 +1328,10 @@ struct DpasDistribution final : public gpu::WarpDistributionPattern {
 
     auto dpasOp = operand->get().getDefiningOp<xegpu::DpasOp>();
     unsigned operandIdx = operand->getOperandNumber();
-    std::string layoutAName =
-        llvm::formatv("{0}{1}", operandLayoutNamePrefix, 0).str();
-    std::string layoutBName =
-        llvm::formatv("{0}{1}", operandLayoutNamePrefix, 1).str();
-    auto layoutCName = llvm::formatv("{0}{1}", resultLayoutNamePrefix, 0).str();
+    std::string layoutAName = xegpu::getLayoutName(dpasOp->getOpOperand(0));
+    std::string layoutBName = xegpu::getLayoutName(dpasOp->getOpOperand(1));
+    std::string layoutCName = xegpu::getLayoutName(dpasOp->getOpResult(0));
+
     xegpu::LayoutAttr layoutA =
         dpasOp->getAttrOfType<xegpu::LayoutAttr>(layoutAName);
     xegpu::LayoutAttr layoutB =
diff --git a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 44d45dd2eaec..885477fe4cbd 100644
--- a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
+#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
@@ -74,17 +75,7 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
       assert(vecTy.getRank() == static_cast<int64_t>(blockSize.size()) &&
              "Expecting blockSize size to match the rank of destTy.");
       auto shape = vecTy.getShape();
-      auto zeroAttr = rewriter.getZeroAttr(vecTy.getElementType());
-
-      Value result = rewriter.create<arith::ConstantOp>(
-          loc, vecTy, DenseElementsAttr::get(vecTy, zeroAttr));
-      for (auto [src, offsets] :
-           llvm::zip_equal(srcs, StaticTileOffsetRange(shape, blockSize))) {
-        SmallVector<int64_t> staticStrides(offsets.size(), 1);
-        result = rewriter.create<vector::InsertStridedSliceOp>(
-            loc, src, result, offsets, staticStrides);
-      }
-      return result;
+      return xegpu::createVectorWithShapeFromValues(rewriter, loc, srcs, shape);
     }
 
     if (isa<xegpu::TensorDescType>(destTy)) {
@@ -109,16 +100,8 @@ struct UnrollPattern : public OpRewritePattern<SourceOp> {
     if (auto vecTy = dyn_cast<VectorType>(src.getType())) {
       assert(vecTy.getRank() == static_cast<int64_t>(blockSize.size()) &&
              "Expecting blockSize size to match the rank of src.");
-      auto shape = vecTy.getShape();
-      SmallVector<Value> results;
-      for (SmallVector<int64_t> offsets :
-           StaticTileOffsetRange(shape, blockSize)) {
-        SmallVector<int64_t> staticStrides(offsets.size(), 1);
-        auto slice = rewriter.create<vector::ExtractStridedSliceOp>(
-            loc, src, offsets, blockSize, staticStrides);
-        results.push_back(slice);
-      }
-      return results;
+      return xegpu::extractVectorsWithShapeFromValue(rewriter, loc, src,
+                                                     blockSize);
     }
 
     if (isa<xegpu::TensorDescType>(src.getType())) {
@@ -153,7 +136,7 @@ struct UnrollCreateNdOp : public UnrollPattern<xegpu::CreateNdDescOp> {
     ArrayRef<int64_t> shape = tdescTy.getShape();
 
     std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
-    if (!targetShape || llvm::equal(*targetShape, shape))
+    if (!targetShape)
       return failure();
 
     auto newTdescTy = getUnrolledTypes(tdescTy, *targetShape)[0];
@@ -204,10 +187,9 @@ struct UnrollUpdateNdOffsetOp : public UnrollPattern<xegpu::UpdateNdOffsetOp> {
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     xegpu::TensorDescType tdescTy = op.getTensorDescType();
-    ArrayRef<int64_t> shape = tdescTy.getShape();
 
     std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
-    if (!targetShape || llvm::equal(*targetShape, shape))
+    if (!targetShape)
       return failure();
 
     SmallVector<Type> convertedTdescTypes =
@@ -233,10 +215,9 @@ struct UnrollPrefetchNdOp : public UnrollPattern<xegpu::PrefetchNdOp> {
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     xegpu::TensorDescType tdescTy = op.getTensorDescType();
-    ArrayRef<int64_t> shape = tdescTy.getShape();
 
     std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
-    if (!targetShape || llvm::equal(*targetShape, shape))
+    if (!targetShape)
       return failure();
 
     SmallVector<Type> convertedTdescTypes =
@@ -260,10 +241,9 @@ struct UnrollLoadNdOp : public UnrollPattern<xegpu::LoadNdOp> {
     Location loc = op.getLoc();
     VectorType valueTy = op.getType();
     xegpu::TensorDescType tdescTy = op.getTensorDescType();
-    ArrayRef<int64_t> shape = tdescTy.getShape();
 
     std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
-    if (!targetShape || llvm::equal(*targetShape, shape))
+    if (!targetShape)
       return failure();
 
     Type elemTy = tdescTy.getElementType();
@@ -295,10 +275,9 @@ struct UnrollStoreNdOp : public UnrollPattern<xegpu::StoreNdOp> {
     Location loc = op.getLoc();
     VectorType valueTy = op.getValueType();
     xegpu::TensorDescType tdescTy = op.getTensorDescType();
-    ArrayRef<int64_t> shape = tdescTy.getShape();
 
     std::optional<SmallVector<int64_t>> targetShape = getTargetShape(op);
-    if (!targetShape || llvm::equal(*targetShape, shape))
+    if (!targetShape)
       return failure();
 
     SmallVector<Type> convertedValTypes =
diff --git a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index 3bf76af674ba..a26c6b52f0dd 100644
--- a/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -13,9 +13,11 @@
 #include "mlir/Dialect/Index/IR/IndexDialect.h"
 #include "mlir/Dialect/Index/IR/IndexOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/SCF/Transforms/Patterns.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
+#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
@@ -29,6 +31,29 @@ using namespace mlir;
 
 namespace {
 
+static std::pair<SmallVector<int64_t>, int>
+getSgShapeAndCount(ArrayRef<int64_t> shape, xegpu::LayoutAttr layout) {
+  int count = 1;
+  SmallVector<int64_t> sgShape(shape);
+
+  if (layout && layout.isWgLayout()) {
+    DenseI32ArrayAttr sgLayoutAttr = layout.getSgLayout();
+    auto sgLayout = llvm::to_vector_of<int64_t>(sgLayoutAttr.asArrayRef());
+    if (DenseI32ArrayAttr sgDataAttr = layout.getSgData())
+      sgShape = llvm::to_vector_of<int64_t>(sgDataAttr.asArrayRef());
+    else
+      sgShape = computeShapeRatio(shape, sgLayout).value_or(sgShape);
+    SmallVector<int64_t> distUnit = computeElementwiseMul(sgLayout, sgShape);
+    // Clamp distUnit to the original shape to handle cases where data is
+    // shared among subgroups, which may cause distUnit to exceed the original
+    // shape.
+    for (size_t i = 0; i < distUnit.size(); ++i)
+      distUnit[i] = std::min(shape[i], distUnit[i]);
+    count = computeProduct(shape) / computeProduct(distUnit);
+  }
+  return std::make_pair(sgShape, count);
+}
+
 /// This pattern transforms the CreateNdDescOp to create a subgroup descriptor
 /// from a workgroup descriptor. It replaces the offsets and sizes with
 /// appropriate values for the subgroup.
@@ -129,18 +154,7 @@ struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
       return rewriter.notifyMatchFailure(
           op, "sgLayout attribute is required in layout");
 
-    SmallVector<int64_t> sgShape;
-    if (auto sgDataAttr = layout.getSgData()) {
-      sgShape = llvm::to_vector_of<int64_t>(sgDataAttr.asArrayRef());
-    } else {
-      assert(wgShape.size() == sgLayout.size() &&
-             "sgLayout and wgShape must have the same rank");
-      sgShape.reserve(wgShape.size());
-      for (size_t i = 0; i < wgShape.size(); ++i) {
-        assert(sgLayout[i] != 0 && "sgLayout elements must be non-zero");
-        sgShape.push_back(wgShape[i] / sgLayout[i]);
-      }
-    }
+    SmallVector<int64_t> sgShape = getSgShapeAndCount(wgShape, layout).first;
 
     // TODO : Handle order attribute
     // Get the subgroup ID
@@ -266,15 +280,15 @@ struct WgToSgDpasOp : public OpConversionPattern<xegpu::DpasOp> {
     if (resultTy.getRank() != 2)
       return failure();
 
-    auto originalLayout =
-        llvm::dyn_cast_or_null<xegpu::LayoutAttr>(op->getAttr("layout"));
+    auto originalLayout = xegpu::getLayoutAttr(op.getResult());
     if (!originalLayout)
       return failure();
 
-    SmallVector<Value> newDpasOps;
     size_t i = 0;
+    SmallVector<Value> newDpasOps;
     for (auto aVec : adaptor.getLhs()) {
       for (auto bVec : adaptor.getRhs()) {
+
         llvm::SmallVector<Value> operands({aVec, bVec});
         Value tmpC;
         if (op.getAcc()) {
@@ -288,10 +302,10 @@ struct WgToSgDpasOp : public OpConversionPattern<xegpu::DpasOp> {
             llvm::cast<VectorType>(bVec.getType()).getShape();
         VectorType resTy = VectorType::get({aVecShape[0], bVecShape[1]},
                                            resultTy.getElementType());
-        tmpC = rewriter.create<xegpu::DpasOp>(
-            loc, resTy, operands,
-            llvm::ArrayRef<NamedAttribute>(
-                {"layout_result_0", originalLayout.dropSgLayoutAndData()}));
+        tmpC = rewriter.create<xegpu::DpasOp>(loc, resTy, operands);
+        xegpu::setLayoutAttr(cast<OpResult>(tmpC),
+                             originalLayout.dropSgLayoutAndData());
+
         newDpasOps.push_back(tmpC);
       }
     }
@@ -314,14 +328,90 @@ struct WgToSgPrefetchNdOp : public OpConversionPattern<xegpu::PrefetchNdOp> {
   }
 };
 
+// Handles UnrealizedConversionCastOp generated during
+// SCFStructuralTypeConversions (step 1). This op may appear as either a
+// target or source materialization for Vector values, e.g.:
+// 1. unrealized_cast %1 : vector<256xf32> to vector<16xf32>, ...
+// 2. unrealized_cast %1 : vector<16xf32>, ... to vector<256xf32>
+// it could be either 1:N or N:1 cast. In both cases, the pattern
+// simply forwards the inputs to the outputs using 1:1 or 1:N interface.
+// for example, the following scf::forOp
+// ```
+// %for = scf.for ... iter_args(%arg1 = %0)->(vector<128x128xf16>) {
+//     %n = use(%arg1): vector<128x128xf16>
+//     scf.yield %n : vector<128x128xf16>
+// }
+// ```
+// Could be converted to:
+// ```
+// %1 = unrealized_conversion_cast %0
+//          : vector<128x128xf16> to vector<16x16xf16>, vector<16x16xf16>
+// %for:2 = scf.for ... iter_args(%arg1 = %1#1, %arg2 = %1#2)
+//                    -> (vector<16x16xf16>, vector<16x16xf16) {
+//     %m = unrealized_conversion_cast %arg1, %arg2
+//            : vector<16x16xf16>, vector<16x16xf16> to vector<128x128xf16>
+//     %n = use(%m): vector<128x128xf16>
+//     %b = unrealized_conversion_cast %n
+//            : vector<128x128xf16> to vector<16x16xf16>, vector<16x16xf16>
+//     scf.yield %b#1, %b#2 : vector<16x16xf16>, vector<16x16xf16>
+// }
+// %cast = unrealized_conversion_cast %for:2
+//          : vector<16x16xf16>, vector<16x16xf16> to vector<128x128xf16>
+// ```
+// TODO: remove it when context-aware type converter is ready.
+struct UnrealizedConversionCastOpPattern
+    : public OpConversionPattern<mlir::UnrealizedConversionCastOp> {
+  using OpConversionPattern<
+      mlir::UnrealizedConversionCastOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::UnrealizedConversionCastOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    SmallVector<Value> inputs = xegpu::flattenValues(adaptor.getInputs());
+
+    auto inputTy = dyn_cast<VectorType>(inputs[0].getType());
+    auto outputTy = dyn_cast<VectorType>(op->getOpResult(0).getType());
+
+    if (!inputTy || !outputTy || !llvm::all_equal(op->getResultTypes()) ||
+        !llvm::all_equal(ValueRange(inputs).getTypes()))
+      return failure();
+
+    // Handles the case "cast %1 : vector<256xf32> to vector<16xf32>, ...".
+    // It is generated by source materialization (e.g., inits to scf forOp).
+    // The input values provided by the adaptor should already be distributed,
+    // and their types should correspond exactly to the result types of the
+    // operation.
+    if (op.getNumOperands() == 1 &&
+        llvm::equal(ValueRange(inputs).getTypes(), op->getResultTypes())) {
+      rewriter.replaceOp(op, inputs);
+      return success();
+    }
+
+    // Handles the case "cast %1 : vector<16xf32>, ... to vector<256xf32>".
+    // It is generated by target materialization (e.g., arguments/results
+    // of scf forOp). All input values must have the same vector type, and
+    // their shape must be evenly divisible by the output vector's shape
+    // (determined by the nature of the workgroup to subgroup distribution).
+    // TODO: it is not safe to do such forward, since such N:1 cast could be
+    // from others.
+    if (op.getNumResults() == 1 &&
+        computeShapeRatio(outputTy.getShape(), inputTy.getShape())) {
+      rewriter.replaceOpWithMultiple(op, {inputs});
+      return success();
+    }
+
+    return mlir::failure();
+  }
+};
+
 } // namespace
 
 namespace mlir {
 namespace xegpu {
 void populateXeGPUWgToSgDistributePatterns(RewritePatternSet &patterns) {
   patterns.add<WgToSgCreateNdOp, WgToSgLoadNdOp, WgToSgStoreNdOp,
-               WgToSgUpdateNdOffsetOp, WgToSgDpasOp, WgToSgPrefetchNdOp>(
-      patterns.getContext());
+               WgToSgUpdateNdOffsetOp, WgToSgDpasOp, WgToSgPrefetchNdOp,
+               UnrealizedConversionCastOpPattern>(patterns.getContext());
 }
 } // namespace xegpu
 } // namespace mlir
@@ -334,9 +424,68 @@ struct XeGPUWgToSgDistributePass
 } // namespace
 
 void XeGPUWgToSgDistributePass::runOnOperation() {
+  // Track existing UnrealizedConversionCastOps
+  SmallVector<Operation *> existingCastOps;
+  getOperation()->walk([&](UnrealizedConversionCastOp castOp) {
+    existingCastOps.push_back(castOp.getOperation());
+  });
+
+  {
+    // Step 1: Apply SCFStructuralTypeConversions to SCF operations with
+    // VectorType operands. This first converts such operands to
+    // RankedTensorType, propagates the layout attribute into the encoding
+    // attribute, and finally converts the RankedTensorType to VectorType based
+    // on the encoding.
+
+    TypeConverter converter;
+    converter.addConversion([&](Type type) -> Type { return type; });
+    converter.addConversion(
+        [&](RankedTensorType type,
+            SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
+          Type elemTy = type.getElementType();
+          ArrayRef<int64_t> shape = type.getShape();
+
+          int count;
+          SmallVector<int64_t> subShape;
+          std::tie(subShape, count) = getSgShapeAndCount(
+              shape,
+              dyn_cast_if_present<xegpu::LayoutAttr>(type.getEncoding()));
+
+          auto newTy = VectorType::get(subShape, elemTy);
+          result.append(count, newTy);
+          return success();
+        });
+
+    xegpu::doSCFStructuralTypeConversionWithTensorType(getOperation(),
+                                                       converter);
+  }
+
+  // Step 2: Perform workgroup to subgroup distribution for TensorDesc values,
+  // as well as XeGPU, Arith, and Vector operations.
   MLIRContext *ctx = &getContext();
   RewritePatternSet patterns(ctx);
   ConversionTarget target(*ctx);
+  TypeConverter converter;
+  converter.addConversion([&](Type type) -> Type { return type; });
+  converter.addConversion(
+      [&](xegpu::TensorDescType type,
+          SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
+        Type elemTy = type.getElementType();
+        ArrayRef<int64_t> shape = type.getShape();
+
+        int count;
+        SmallVector<int64_t> subShape;
+        xegpu::LayoutAttr layout = type.getLayoutAttr();
+        std::tie(subShape, count) = getSgShapeAndCount(shape, layout);
+
+        if (layout)
+          layout = layout.dropSgLayoutAndData();
+
+        auto newTy = xegpu::TensorDescType::get(
+            type.getContext(), subShape, elemTy, type.getEncoding(), layout);
+        result.append(count, newTy);
+        return success();
+      });
 
   auto getTensorDescType = [](Operation *op) -> xegpu::TensorDescType {
     if (auto createOp = dyn_cast<xegpu::CreateNdDescOp>(op))
@@ -353,26 +502,49 @@ void XeGPUWgToSgDistributePass::runOnOperation() {
   };
 
   auto isLegal = [&](xegpu::LayoutAttr layout) -> bool {
-    return !layout || layout.getSgLayout() == nullptr;
+    return !layout || !layout.isWgLayout();
   };
 
   target.addDynamicallyLegalOp<xegpu::CreateNdDescOp, xegpu::LoadNdOp,
                                xegpu::StoreNdOp, xegpu::UpdateNdOffsetOp,
                                xegpu::PrefetchNdOp>([=](Operation *op) -> bool {
     auto tdescTy = getTensorDescType(op);
-    auto layout = dyn_cast_or_null<xegpu::LayoutAttr>(tdescTy.getLayout());
+    auto layout = dyn_cast_if_present<xegpu::LayoutAttr>(tdescTy.getLayout());
     return isLegal(layout);
   });
 
   target.addDynamicallyLegalOp<xegpu::DpasOp>([=](xegpu::DpasOp op) -> bool {
-    auto layout = dyn_cast_or_null<xegpu::LayoutAttr>(op->getAttr("layout"));
+    auto layout = xegpu::getLayoutAttr(op.getResult());
     return isLegal(layout);
   });
 
+  target.addDynamicallyLegalOp<UnrealizedConversionCastOp>(
+      [=](UnrealizedConversionCastOp op) {
+        return llvm::is_contained(existingCastOps, op.getOperation());
+      });
+
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
 
+  scf::populateSCFStructuralTypeConversionsAndLegality(converter, patterns,
+                                                       target);
   xegpu::populateXeGPUWgToSgDistributePatterns(patterns);
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
     return signalPassFailure();
+
+  // Remove sg_layout and sg_data attributes from the Layout
+  // attribute for each VectorType result of the operation.
+  // For Structured Control Flow ops, the layout is simply removed,
+  // since in 1:N case, the layout for new results are missing.
+  // Layout propagation pass will activated.
+  getOperation()->walk([](Operation *op) {
+    for (OpResult result : op->getOpResults()) {
+      std::string name = xegpu::getLayoutName(result);
+      if (auto layout = op->getAttrOfType<xegpu::LayoutAttr>(name)) {
+        op->removeAttr(name);
+        if (!isa<scf::IfOp, scf::ForOp, scf::WhileOp, scf::ConditionOp>(op))
+          op->setAttr(name, layout.dropSgLayoutAndData());
+      }
+    }
+  });
 }
diff --git a/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt b/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt
index afd8e2d5c4df..98e84a442072 100644
--- a/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt
@@ -6,5 +6,6 @@ add_mlir_dialect_library(MLIRXeGPUUtils
 
   LINK_LIBS PUBLIC
   MLIRIR
+  MLIRSCFTransforms
   MLIRXeGPUDialect
   )
diff --git a/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 6b45ed0ae4ce..6b85a66a8bd3 100644
--- a/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/external/llvm-project/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -11,12 +11,29 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
+#include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/ValueRange.h"
+#include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/FormatVariadic.h"
 #include <cstdint>
 #include <numeric>
 
 using namespace mlir;
 
+/// convert ArrayRef<ValueRange> into SmallVector<Value>
+SmallVector<Value> xegpu::flattenValues(ArrayRef<ValueRange> values) {
+  SmallVector<Value> result;
+  for (const auto &vals : values)
+    llvm::append_range(result, vals);
+  return result;
+}
+
 FailureOr<VectorType>
 mlir::xegpu::getDistributedVectorType(xegpu::TensorDescType tdescTy) {
   auto layout = llvm::dyn_cast_if_present<LayoutAttr>(tdescTy.getLayout());
@@ -83,3 +100,278 @@ mlir::xegpu::getDistributedVectorType(VectorType originalType,
       /*memory_space=*/xegpu::MemorySpace::Global, layout);
   return xegpu::getDistributedVectorType(helperTdescTy);
 }
+
+std::string xegpu::getLayoutName(const OpOperand &operand) {
+  const StringRef prefix("layout_operand_");
+  unsigned idx = const_cast<OpOperand &>(operand).getOperandNumber();
+  return llvm::formatv("{0}{1}", prefix, idx).str();
+}
+
+std::string xegpu::getLayoutName(const OpResult result) {
+  const StringRef prefix = "layout_result_";
+  return llvm::formatv("{0}{1}", prefix, result.getResultNumber()).str();
+}
+
+xegpu::LayoutAttr xegpu::getLayoutAttr(const Value value) {
+  if (!value)
+    return nullptr;
+
+  if (auto tdescTy =
+          dyn_cast_if_present<xegpu::TensorDescType>(value.getType()))
+    return tdescTy.getLayoutAttr();
+
+  if (auto result = dyn_cast<OpResult>(value)) {
+    Operation *defOp = result.getDefiningOp();
+    assert(defOp && "result must have a defining op");
+
+    // for LoadNdOp, the layout is stored in the tensor descriptor
+    if (auto loadNd = dyn_cast<xegpu::LoadNdOp>(defOp))
+      return getLayoutAttr(loadNd.getTensorDesc());
+
+    std::string layoutName = getLayoutName(result);
+    if (defOp->hasAttr(layoutName))
+      return defOp->getAttrOfType<xegpu::LayoutAttr>(layoutName);
+  }
+
+  if (auto arg = dyn_cast<BlockArgument>(value)) {
+    auto parentOp = arg.getOwner()->getParentOp();
+    if (auto loop = dyn_cast<LoopLikeOpInterface>(parentOp)) {
+      OpOperand *tiedInit = loop.getTiedLoopInit(arg);
+      return getLayoutAttr(tiedInit->get());
+    }
+  }
+
+  return nullptr;
+}
+
+xegpu::LayoutAttr xegpu::getLayoutAttr(const OpOperand &opr) {
+  Operation *op = opr.getOwner();
+  std::string layoutName = xegpu::getLayoutName(opr);
+  if (op->hasAttr(layoutName))
+    return op->getAttrOfType<xegpu::LayoutAttr>(layoutName);
+  return getLayoutAttr(opr.get());
+}
+
+template <typename T, typename>
+void xegpu::setLayoutAttr(const T &operandOrResult, const LayoutAttr layout) {
+  Operation *owner = operandOrResult.getOwner();
+  std::string name = xegpu::getLayoutName(operandOrResult);
+  if (layout && !owner->hasAttrOfType<LayoutAttr>(name))
+    owner->setAttr(name, layout);
+}
+
+// Explicit instantiation for OpResult
+template void
+xegpu::setLayoutAttr<mlir::OpResult>(const mlir::OpResult &result,
+                                     const mlir::xegpu::LayoutAttr layout);
+
+// Explicit instantiation for OpOperand
+template void
+xegpu::setLayoutAttr<mlir::OpOperand>(const mlir::OpOperand &operand,
+                                      const mlir::xegpu::LayoutAttr layout);
+
+void xegpu::setLayoutAttrs(Operation *op,
+                           function_ref<LayoutAttr(Value)> getLayoutImpl) {
+  op->walk([&](Operation *nestOp) {
+    for (OpOperand &opr : nestOp->getOpOperands()) {
+      auto layout = getLayoutImpl(opr.get());
+      setLayoutAttr(opr, layout);
+    }
+    for (OpResult result : nestOp->getOpResults()) {
+      auto layout = getLayoutImpl(result);
+      setLayoutAttr(result, layout);
+    }
+  });
+}
+
+SmallVector<Value>
+xegpu::extractVectorsWithShapeFromValue(OpBuilder &builder, Location loc,
+                                        Value value, ArrayRef<int64_t> shape) {
+  auto vecTy = dyn_cast<VectorType>(value.getType());
+  if (!vecTy)
+    return {value};
+
+  ArrayRef<int64_t> srcShape = vecTy.getShape();
+  if (!computeShapeRatio(srcShape, shape))
+    return {value};
+
+  SmallVector<Value> result;
+  for (SmallVector<int64_t> offsets : StaticTileOffsetRange(srcShape, shape)) {
+    SmallVector<int64_t> staticStrides(offsets.size(), 1);
+    result.push_back(builder.create<vector::ExtractStridedSliceOp>(
+        loc, value, offsets, shape, staticStrides));
+  }
+
+  return result;
+}
+
+Value xegpu::createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
+                                             ValueRange values,
+                                             ArrayRef<int64_t> shape) {
+  VectorType inputTy = dyn_cast<VectorType>(values[0].getType());
+  assert(llvm::all_of(values.getTypes(),
+                      [&](Type type) { return type == inputTy; }) &&
+         "values must be of the same VectorType");
+
+  Type elemTy = inputTy.getElementType();
+  ArrayRef<int64_t> tileShape = inputTy.getShape();
+
+  VectorType resultTy = VectorType::get(shape, elemTy);
+  auto zeroAttr = builder.getZeroAttr(elemTy);
+  Value result = builder.create<arith::ConstantOp>(
+      loc, resultTy, DenseElementsAttr::get(resultTy, zeroAttr));
+
+  for (auto [src, offsets] :
+       llvm::zip_equal(values, StaticTileOffsetRange(shape, tileShape))) {
+    SmallVector<int64_t> staticStrides(offsets.size(), 1);
+    result = builder.create<vector::InsertStridedSliceOp>(
+        loc, src, result, offsets, staticStrides);
+  }
+  return result;
+}
+
+void xegpu::doSCFStructuralTypeConversionWithTensorType(
+    Operation *op, TypeConverter converter) {
+  MLIRContext *context = op->getContext();
+
+  auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
+                            Location loc) -> Value {
+    return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+        .getResult(0);
+  };
+
+  { // convert VectorType to RankedTensorType for SCF Structural ops
+    TypeConverter converter;
+    converter.addConversion([](Type type) -> Type { return type; });
+    converter.addConversion([](VectorType type) -> Type {
+      return RankedTensorType::get(type.getShape(), type.getElementType());
+    });
+    converter.addSourceMaterialization(materializeCast);
+    converter.addTargetMaterialization(materializeCast);
+
+    mlir::ConversionTarget target(*context);
+    target.addLegalOp<UnrealizedConversionCastOp>();
+
+    mlir::RewritePatternSet patterns(context);
+    scf::populateSCFStructuralTypeConversionsAndLegality(converter, patterns,
+                                                         target);
+    (void)mlir::applyPartialConversion(op, target, std::move(patterns));
+  }
+
+  { // propagate the layout attribute to RankedTensorType by checking
+    // BuiltInUnrealizedCastOps
+    // for VectorType to RankedTensorType cast.
+    op->walk([](UnrealizedConversionCastOp castOp) {
+      if (castOp.getNumOperands() != 1 || castOp.getNumResults() != 1)
+        return WalkResult::skip();
+
+      Value input = castOp.getInputs()[0];
+      Value result = castOp.getResults()[0];
+      auto inputTy = dyn_cast<VectorType>(input.getType());
+      auto resultTy = dyn_cast<RankedTensorType>(result.getType());
+
+      // Only look at ops casting from VectorType to RankedTensorType
+      if (!inputTy || !resultTy)
+        return WalkResult::skip();
+
+      xegpu::LayoutAttr layout = xegpu::getLayoutAttr(input);
+      if (!layout)
+        return WalkResult::skip();
+
+      RankedTensorType newTy = resultTy.cloneWithEncoding(layout);
+      result.setType(newTy);
+
+      // update the arguments if user is a LoopLike op.
+      for (OpOperand &use : result.getUses()) {
+        if (auto loop = dyn_cast<LoopLikeOpInterface>(use.getOwner())) {
+          BlockArgument arg = loop.getTiedLoopRegionIterArg(&use);
+          arg.setType(newTy);
+        }
+        // whileOp has two regions, the BlockArgument of the after region
+        // is not exposed by LoopLikeOpInterface
+        if (auto whileOp = dyn_cast<scf::WhileOp>(use.getOwner())) {
+          unsigned idx = use.getOperandNumber();
+          BlockArgument arg = whileOp.getAfterArguments()[idx];
+          arg.setType(newTy);
+        }
+      }
+      return WalkResult::advance();
+    });
+
+    // using yieldOp as anchor to update the result type of its ParentOp
+    op->walk([](scf::YieldOp yieldOp) {
+      Operation *parentOp = yieldOp->getParentOp();
+      for (OpResult r : parentOp->getOpResults()) {
+        unsigned idx = r.getResultNumber();
+        Type resultTy = r.getType();
+        Type yieldTy = yieldOp.getResults()[idx].getType();
+        if (isa<RankedTensorType>(resultTy) && yieldTy != resultTy)
+          r.setType(yieldTy);
+      }
+    });
+  }
+
+  { // perform the conversion from RankedTensorType to VectorType based on the
+    // LayoutAttr
+
+    // Handle the UnrealizedConversionCastOp introduced by the first step.
+    // For vector->RankedTensorType, it will simply forward the inputs.
+    // For RankedTensorType->vector, it will update the inputs with the
+    // one from the adaptor.
+    class UnrealizedConversionCastOpPattern
+        : public OpConversionPattern<mlir::UnrealizedConversionCastOp> {
+      using OpConversionPattern<
+          mlir::UnrealizedConversionCastOp>::OpConversionPattern;
+
+      mlir::LogicalResult
+      matchAndRewrite(mlir::UnrealizedConversionCastOp op,
+                      OneToNOpAdaptor adaptor,
+                      ConversionPatternRewriter &rewriter) const override {
+        auto inputs = op.getOperands();
+        auto outputs = op.getOutputs();
+
+        if (inputs.size() != 1 || outputs.size() != 1)
+          return failure();
+
+        auto inputTy = inputs[0].getType();
+        auto outputTy = outputs[0].getType();
+
+        if (isa<VectorType>(inputTy) && isa<RankedTensorType>(outputTy)) {
+          rewriter.replaceOpWithMultiple(op, adaptor.getInputs());
+          return success();
+        }
+
+        if (isa<RankedTensorType>(inputTy) && isa<VectorType>(outputTy)) {
+          SmallVector<Value> values = xegpu::flattenValues(adaptor.getInputs());
+          auto newOp = rewriter.create<UnrealizedConversionCastOp>(
+              op.getLoc(), outputTy, values);
+          rewriter.replaceOp(op, newOp);
+          return success();
+        }
+        return failure();
+      }
+    };
+
+    converter.addSourceMaterialization(materializeCast);
+    converter.addTargetMaterialization([&](OpBuilder &builder, TypeRange type,
+                                           ValueRange inputs, Location loc) {
+      return builder.create<UnrealizedConversionCastOp>(loc, type, inputs)
+          .getResults();
+    });
+
+    mlir::ConversionTarget target(*context);
+    target.addDynamicallyLegalOp<UnrealizedConversionCastOp>(
+        [](UnrealizedConversionCastOp op) {
+          auto isTensorTy = [](Type type) {
+            return isa<RankedTensorType>(type);
+          };
+          return llvm::none_of(op->getOperandTypes(), isTensorTy) &&
+                 llvm::none_of(op->getResultTypes(), isTensorTy);
+        });
+    mlir::RewritePatternSet patterns(context);
+    patterns.insert<UnrealizedConversionCastOpPattern>(context);
+    scf::populateSCFStructuralTypeConversionsAndLegality(converter, patterns,
+                                                         target);
+    (void)mlir::applyPartialConversion(op, target, std::move(patterns));
+  }
+}
diff --git a/external/llvm-project/mlir/lib/ExecutionEngine/ExecutionEngine.cpp b/external/llvm-project/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
index 39f5c003320d..f704fbfbe8ff 100644
--- a/external/llvm-project/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/external/llvm-project/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -315,7 +315,8 @@ ExecutionEngine::create(Operation *m, const ExecutionEngineOptions &options,
   // process and dynamically linked libraries.
   auto objectLinkingLayerCreator = [&](ExecutionSession &session) {
     auto objectLayer = std::make_unique<RTDyldObjectLinkingLayer>(
-        session, [sectionMemoryMapper = options.sectionMemoryMapper]() {
+        session, [sectionMemoryMapper =
+                      options.sectionMemoryMapper](const MemoryBuffer &) {
           return std::make_unique<SectionMemoryManager>(sectionMemoryMapper);
         });
 
diff --git a/external/llvm-project/mlir/lib/IR/AsmPrinter.cpp b/external/llvm-project/mlir/lib/IR/AsmPrinter.cpp
index fc1806900c0a..c7cc6a02ad20 100644
--- a/external/llvm-project/mlir/lib/IR/AsmPrinter.cpp
+++ b/external/llvm-project/mlir/lib/IR/AsmPrinter.cpp
@@ -1146,8 +1146,7 @@ template <typename T, typename... PrintArgs>
 std::pair<size_t, size_t> AliasInitializer::visitImpl(
     T value, llvm::MapVector<const void *, InProgressAliasInfo> &aliases,
     bool canBeDeferred, PrintArgs &&...printArgs) {
-  auto [it, inserted] =
-      aliases.insert({value.getAsOpaquePointer(), InProgressAliasInfo()});
+  auto [it, inserted] = aliases.try_emplace(value.getAsOpaquePointer());
   size_t aliasIndex = std::distance(aliases.begin(), it);
   if (!inserted) {
     // Make sure that the alias isn't deferred if we don't permit it.
diff --git a/external/llvm-project/mlir/lib/IR/Builders.cpp b/external/llvm-project/mlir/lib/IR/Builders.cpp
index 89102115cdc4..5f7bc50afc41 100644
--- a/external/llvm-project/mlir/lib/IR/Builders.cpp
+++ b/external/llvm-project/mlir/lib/IR/Builders.cpp
@@ -34,6 +34,8 @@ Location Builder::getFusedLoc(ArrayRef<Location> locs, Attribute metadata) {
 // Types.
 //===----------------------------------------------------------------------===//
 
+FloatType Builder::getF8E8M0Type() { return Float8E8M0FNUType::get(context); }
+
 FloatType Builder::getBF16Type() { return BFloat16Type::get(context); }
 
 FloatType Builder::getF16Type() { return Float16Type::get(context); }
diff --git a/external/llvm-project/mlir/lib/IR/MLIRContext.cpp b/external/llvm-project/mlir/lib/IR/MLIRContext.cpp
index e0f4c3611529..d6febc39f451 100644
--- a/external/llvm-project/mlir/lib/IR/MLIRContext.cpp
+++ b/external/llvm-project/mlir/lib/IR/MLIRContext.cpp
@@ -811,7 +811,7 @@ OperationName::OperationName(StringRef name, MLIRContext *context) {
   // Acquire a writer-lock so that we can safely create the new instance.
   ScopedWriterLock lock(ctxImpl.operationInfoMutex, isMultithreadingEnabled);
 
-  auto it = ctxImpl.operations.insert({name, nullptr});
+  auto it = ctxImpl.operations.try_emplace(name);
   if (it.second) {
     auto nameAttr = StringAttr::get(context, name);
     it.first->second = std::make_unique<UnregisteredOpModel>(
diff --git a/external/llvm-project/mlir/lib/IR/OperationSupport.cpp b/external/llvm-project/mlir/lib/IR/OperationSupport.cpp
index 7c9e6c89d4d8..d3e9aebc4dc2 100644
--- a/external/llvm-project/mlir/lib/IR/OperationSupport.cpp
+++ b/external/llvm-project/mlir/lib/IR/OperationSupport.cpp
@@ -680,9 +680,13 @@ llvm::hash_code OperationEquivalence::computeHash(
   //   - Operation Name
   //   - Attributes
   //   - Result Types
+  DictionaryAttr dictAttrs;
+  if (!(flags & Flags::IgnoreDiscardableAttrs))
+    dictAttrs = op->getRawDictionaryAttrs();
   llvm::hash_code hash =
-      llvm::hash_combine(op->getName(), op->getRawDictionaryAttrs(),
-                         op->getResultTypes(), op->hashProperties());
+      llvm::hash_combine(op->getName(), dictAttrs, op->getResultTypes());
+  if (!(flags & Flags::IgnoreProperties))
+    hash = llvm::hash_combine(hash, op->hashProperties());
 
   //   - Location if required
   if (!(flags & Flags::IgnoreLocations))
@@ -836,14 +840,19 @@ OperationEquivalence::isRegionEquivalentTo(Region *lhs, Region *rhs,
     return true;
 
   // 1. Compare the operation properties.
+  if (!(flags & IgnoreDiscardableAttrs) &&
+      lhs->getRawDictionaryAttrs() != rhs->getRawDictionaryAttrs())
+    return false;
+
   if (lhs->getName() != rhs->getName() ||
-      lhs->getRawDictionaryAttrs() != rhs->getRawDictionaryAttrs() ||
       lhs->getNumRegions() != rhs->getNumRegions() ||
       lhs->getNumSuccessors() != rhs->getNumSuccessors() ||
       lhs->getNumOperands() != rhs->getNumOperands() ||
-      lhs->getNumResults() != rhs->getNumResults() ||
-      !lhs->getName().compareOpProperties(lhs->getPropertiesStorage(),
-                                          rhs->getPropertiesStorage()))
+      lhs->getNumResults() != rhs->getNumResults())
+    return false;
+  if (!(flags & IgnoreProperties) &&
+      !(lhs->getName().compareOpProperties(lhs->getPropertiesStorage(),
+                                           rhs->getPropertiesStorage())))
     return false;
   if (!(flags & IgnoreLocations) && lhs->getLoc() != rhs->getLoc())
     return false;
diff --git a/external/llvm-project/mlir/lib/IR/SymbolTable.cpp b/external/llvm-project/mlir/lib/IR/SymbolTable.cpp
index 075a0ba15d7c..aaa4d5617eb4 100644
--- a/external/llvm-project/mlir/lib/IR/SymbolTable.cpp
+++ b/external/llvm-project/mlir/lib/IR/SymbolTable.cpp
@@ -1100,7 +1100,7 @@ void SymbolUserMap::replaceAllUsesWith(Operation *symbol,
   if (newSymbol != symbol) {
     // Transfer over the users to the new symbol.  The reference to the old one
     // is fetched again as the iterator is invalidated during the insertion.
-    auto newIt = symbolToUsers.try_emplace(newSymbol, SetVector<Operation *>{});
+    auto newIt = symbolToUsers.try_emplace(newSymbol);
     auto oldIt = symbolToUsers.find(symbol);
     assert(oldIt != symbolToUsers.end() && "missing old users list");
     if (newIt.second)
diff --git a/external/llvm-project/mlir/lib/Interfaces/ViewLikeInterface.cpp b/external/llvm-project/mlir/lib/Interfaces/ViewLikeInterface.cpp
index 70dd7b4aec88..d70cfebbe53e 100644
--- a/external/llvm-project/mlir/lib/Interfaces/ViewLikeInterface.cpp
+++ b/external/llvm-project/mlir/lib/Interfaces/ViewLikeInterface.cpp
@@ -27,9 +27,7 @@ LogicalResult mlir::verifyListOfOperandsOrIntegers(Operation *op,
     return op->emitError("expected ") << numElements << " " << name
                                       << " values, got " << staticVals.size();
   unsigned expectedNumDynamicEntries =
-      llvm::count_if(staticVals, [](int64_t staticVal) {
-        return ShapedType::isDynamic(staticVal);
-      });
+      llvm::count_if(staticVals, ShapedType::isDynamic);
   if (values.size() != expectedNumDynamicEntries)
     return op->emitError("expected ")
            << expectedNumDynamicEntries << " dynamic " << name << " values";
@@ -270,5 +268,5 @@ bool mlir::detail::sameOffsetsSizesAndStrides(
 unsigned mlir::detail::getNumDynamicEntriesUpToIdx(ArrayRef<int64_t> staticVals,
                                                    unsigned idx) {
   return std::count_if(staticVals.begin(), staticVals.begin() + idx,
-                       [&](int64_t val) { return ShapedType::isDynamic(val); });
+                       ShapedType::isDynamic);
 }
diff --git a/external/llvm-project/mlir/lib/Pass/Pass.cpp b/external/llvm-project/mlir/lib/Pass/Pass.cpp
index e0e9b5f54042..d1371d2b508f 100644
--- a/external/llvm-project/mlir/lib/Pass/Pass.cpp
+++ b/external/llvm-project/mlir/lib/Pass/Pass.cpp
@@ -18,6 +18,7 @@
 #include "mlir/IR/Threading.h"
 #include "mlir/IR/Verifier.h"
 #include "mlir/Support/FileUtilities.h"
+#include "mlir/Support/IndentedOstream.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
@@ -80,14 +81,19 @@ void Pass::copyOptionValuesFrom(const Pass *other) {
 }
 
 /// Prints out the pass in the textual representation of pipelines. If this is
-/// an adaptor pass, print its pass managers.
-void Pass::printAsTextualPipeline(raw_ostream &os) {
+/// an adaptor pass, print its pass managers. When `pretty` is true, the
+/// printed pipeline is formatted for readability.
+void Pass::printAsTextualPipeline(raw_ostream &os, bool pretty) {
   // Special case for adaptors to print its pass managers.
   if (auto *adaptor = dyn_cast<OpToOpPassAdaptor>(this)) {
     llvm::interleave(
         adaptor->getPassManagers(),
-        [&](OpPassManager &pm) { pm.printAsTextualPipeline(os); },
-        [&] { os << ","; });
+        [&](OpPassManager &pm) { pm.printAsTextualPipeline(os, pretty); },
+        [&] {
+          os << ",";
+          if (pretty)
+            os << "\n";
+        });
     return;
   }
   // Otherwise, print the pass argument followed by its options. If the pass
@@ -390,27 +396,51 @@ StringRef OpPassManager::getOpAnchorName() const {
 }
 
 /// Prints out the passes of the pass manager as the textual representation
-/// of pipelines.
+/// of pipelines. When `pretty` is true, the printed pipeline is formatted for
+/// readability.
 void printAsTextualPipeline(
-    raw_ostream &os, StringRef anchorName,
-    const llvm::iterator_range<OpPassManager::pass_iterator> &passes) {
+    raw_indented_ostream &os, StringRef anchorName,
+    const llvm::iterator_range<OpPassManager::pass_iterator> &passes,
+    bool pretty = false) {
   os << anchorName << "(";
+  if (pretty) {
+    os << "\n";
+    os.indent();
+  }
   llvm::interleave(
-      passes, [&](mlir::Pass &pass) { pass.printAsTextualPipeline(os); },
-      [&]() { os << ","; });
+      passes,
+      [&](mlir::Pass &pass) { pass.printAsTextualPipeline(os, pretty); },
+      [&]() {
+        os << ",";
+        if (pretty)
+          os << "\n";
+      });
+  if (pretty) {
+    os << "\n";
+    os.unindent();
+  }
   os << ")";
 }
-void OpPassManager::printAsTextualPipeline(raw_ostream &os) const {
+void printAsTextualPipeline(
+    raw_ostream &os, StringRef anchorName,
+    const llvm::iterator_range<OpPassManager::pass_iterator> &passes,
+    bool pretty) {
+  raw_indented_ostream indentedOS(os);
+  printAsTextualPipeline(indentedOS, anchorName, passes, pretty);
+}
+void OpPassManager::printAsTextualPipeline(raw_ostream &os, bool pretty) const {
   StringRef anchorName = getOpAnchorName();
+  raw_indented_ostream indentedOS(os);
   ::printAsTextualPipeline(
-      os, anchorName,
+      indentedOS, anchorName,
       {MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.begin(),
-       MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.end()});
+       MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.end()},
+      pretty);
 }
 
 void OpPassManager::dump() {
   llvm::errs() << "Pass Manager with " << impl->passes.size() << " passes:\n";
-  printAsTextualPipeline(llvm::errs());
+  printAsTextualPipeline(llvm::errs(), /*pretty=*/true);
   llvm::errs() << "\n";
 }
 
@@ -466,7 +496,6 @@ llvm::hash_code OpPassManager::hash() {
   return hashCode;
 }
 
-
 //===----------------------------------------------------------------------===//
 // OpToOpPassAdaptor
 //===----------------------------------------------------------------------===//
@@ -871,7 +900,8 @@ LogicalResult PassManager::run(Operation *op) {
   // Initialize all of the passes within the pass manager with a new generation.
   llvm::hash_code newInitKey = context->getRegistryHash();
   llvm::hash_code pipelineKey = hash();
-  if (newInitKey != initializationKey || pipelineKey != pipelineInitializationKey) {
+  if (newInitKey != initializationKey ||
+      pipelineKey != pipelineInitializationKey) {
     if (failed(initialize(context, impl->initializationGeneration + 1)))
       return failure();
     initializationKey = newInitKey;
diff --git a/external/llvm-project/mlir/lib/Pass/PassCrashRecovery.cpp b/external/llvm-project/mlir/lib/Pass/PassCrashRecovery.cpp
index 8c6d865cb31d..b048ff946239 100644
--- a/external/llvm-project/mlir/lib/Pass/PassCrashRecovery.cpp
+++ b/external/llvm-project/mlir/lib/Pass/PassCrashRecovery.cpp
@@ -443,7 +443,8 @@ makeReproducerStreamFactory(StringRef outputFile) {
 
 void printAsTextualPipeline(
     raw_ostream &os, StringRef anchorName,
-    const llvm::iterator_range<OpPassManager::pass_iterator> &passes);
+    const llvm::iterator_range<OpPassManager::pass_iterator> &passes,
+    bool pretty = false);
 
 std::string mlir::makeReproducer(
     StringRef anchorName,
diff --git a/external/llvm-project/mlir/lib/Reducer/OptReductionPass.cpp b/external/llvm-project/mlir/lib/Reducer/OptReductionPass.cpp
index fbbe75d30388..7eb29a02702a 100644
--- a/external/llvm-project/mlir/lib/Reducer/OptReductionPass.cpp
+++ b/external/llvm-project/mlir/lib/Reducer/OptReductionPass.cpp
@@ -88,4 +88,4 @@ void OptReductionPass::runOnOperation() {
   moduleVariant->destroy();
 
   LLVM_DEBUG(llvm::dbgs() << "Pass Complete\n\n");
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/mlir/lib/Target/LLVM/CMakeLists.txt b/external/llvm-project/mlir/lib/Target/LLVM/CMakeLists.txt
index d03214c1550a..83fbf7a5fe5f 100644
--- a/external/llvm-project/mlir/lib/Target/LLVM/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Target/LLVM/CMakeLists.txt
@@ -176,7 +176,6 @@ add_mlir_dialect_library(MLIRROCDLTarget
   LINK_COMPONENTS
   FrontendOffloading
   MCParser
-  Object
   ${AMDGPU_LIBS}
 
   LINK_LIBS PUBLIC
diff --git a/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
index 187e2a9b75a9..2dd0640f794e 100644
--- a/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
+++ b/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
@@ -70,8 +70,7 @@ void registerFromLLVMIRTranslation() {
           return nullptr;
 
         // Debug records are not currently supported in the LLVM IR translator.
-        if (llvmModule->IsNewDbgInfoFormat)
-          llvmModule->convertFromNewDbgValues();
+        llvmModule->convertFromNewDbgValues();
 
         return translateLLVMIRToModule(
             std::move(llvmModule), context, emitExpensiveWarnings,
diff --git a/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp b/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
index e3fa7c883c52..75170bffcdf2 100644
--- a/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
+++ b/external/llvm-project/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
@@ -20,8 +20,6 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-
 using namespace mlir;
 
 namespace mlir {
@@ -34,13 +32,7 @@ void registerToLLVMIRTranslation() {
         if (!llvmModule)
           return failure();
 
-        // When printing LLVM IR, we should convert the module to the debug info
-        // format that LLVM expects us to print.
-        // See https://llvm.org/docs/RemoveDIsDebugInfo.html
-        llvm::ScopedDbgInfoFormatSetter formatSetter(*llvmModule,
-                                                     UseNewDbgInfoFormat);
-        if (UseNewDbgInfoFormat)
-          llvmModule->removeDebugIntrinsicDeclarations();
+        llvmModule->removeDebugIntrinsicDeclarations();
         llvmModule->print(output, nullptr);
         return success();
       },
diff --git a/external/llvm-project/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/external/llvm-project/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e4a822294a07..a60da06fb3a1 100644
--- a/external/llvm-project/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/external/llvm-project/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3550,24 +3550,26 @@ static void collectMapDataFromMapOperands(
               mapOp, mapOp.getMapperIdAttr()));
     else
       mapData.Mappers.push_back(nullptr);
-    mapData.IsAMapping.push_back(true);
+    bool hasMapDescriptor =
+        (mapData.Types.back() &
+         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DESCRIPTOR) ==
+        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DESCRIPTOR;
+    mapData.IsAMapping.push_back(!hasMapDescriptor);
     mapData.IsAMember.push_back(checkIsAMember(mapVars, mapOp));
   }
 
   auto findMapInfo = [&mapData](llvm::Value *val,
-                                llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy,
-                                size_t memberCount) {
+                                llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy) {
+    unsigned index = 0;
     bool found = false;
-    for (size_t index = 0; index < mapData.OriginalValue.size(); ++index) {
-      auto mapInfoOp = dyn_cast<omp::MapInfoOp>(mapData.MapClause[index]);
-      if (mapData.IsAMapping[index] && mapData.Pointers[index] == val &&
-          mapData.BasePointers[index] == val &&
-          memberCount == mapInfoOp.getMembers().size()) {
+    for (llvm::Value *basePtr : mapData.OriginalValue) {
+      if (basePtr == val && mapData.IsAMapping[index]) {
         found = true;
         mapData.Types[index] |=
             llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
         mapData.DevicePointers[index] = devInfoTy;
       }
+      index++;
     }
     return found;
   };
@@ -3582,7 +3584,7 @@ static void collectMapDataFromMapOperands(
       llvm::Value *origValue = moduleTranslation.lookupValue(offloadPtr);
 
       // Check if map info is already present for this entry.
-      if (!findMapInfo(origValue, devInfoTy, mapOp.getMembers().size())) {
+      if (!findMapInfo(origValue, devInfoTy)) {
         mapData.OriginalValue.push_back(origValue);
         mapData.Pointers.push_back(mapData.OriginalValue.back());
         mapData.IsDeclareTarget.push_back(false);
@@ -5977,7 +5979,13 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
             llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
                 ompBuilder->createBarrier(builder.saveIP(),
                                           llvm::omp::OMPD_barrier);
-            return handleError(afterIP, *op);
+            LogicalResult res = handleError(afterIP, *op);
+            if (res.succeeded()) {
+              // If the barrier generated a cancellation check, the insertion
+              // point might now need to be changed to a new continuation block
+              builder.restoreIP(*afterIP);
+            }
+            return res;
           })
           .Case([&](omp::TaskyieldOp op) {
             if (failed(checkImplementationStatus(*op)))
diff --git a/external/llvm-project/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/external/llvm-project/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
index 1dde457b5c34..54ae01c77c44 100644
--- a/external/llvm-project/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
+++ b/external/llvm-project/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
@@ -280,7 +280,7 @@ LoopAnnotationTranslation::translateLoopAnnotation(LoopAnnotationAttr attr,
 llvm::MDNode *
 LoopAnnotationTranslation::getAccessGroup(AccessGroupAttr accessGroupAttr) {
   auto [result, inserted] =
-      accessGroupMetadataMapping.insert({accessGroupAttr, nullptr});
+      accessGroupMetadataMapping.try_emplace(accessGroupAttr);
   if (inserted)
     result->second = llvm::MDNode::getDistinct(llvmModule.getContext(), {});
   return result->second;
diff --git a/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 85417da798b2..803ecb39f4da 100644
--- a/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -2636,6 +2636,11 @@ void ModuleImport::processFunctionAttributes(llvm::Function *func,
     funcOp.setTargetFeaturesAttr(
         LLVM::TargetFeaturesAttr::get(context, attr.getValueAsString()));
 
+  if (llvm::Attribute attr = func->getFnAttribute("reciprocal-estimates");
+      attr.isStringAttribute())
+    funcOp.setReciprocalEstimatesAttr(
+        StringAttr::get(context, attr.getValueAsString()));
+
   if (llvm::Attribute attr = func->getFnAttribute("prefer-vector-width");
       attr.isStringAttribute())
     funcOp.setPreferVectorWidth(attr.getValueAsString());
diff --git a/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 4cc419c7cde5..e5ca147ea98f 100644
--- a/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/external/llvm-project/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -67,8 +67,6 @@ using namespace mlir;
 using namespace mlir::LLVM;
 using namespace mlir::LLVM::detail;
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-
 #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc"
 
 namespace {
@@ -1549,6 +1547,9 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
   if (auto tuneCpu = func.getTuneCpu())
     llvmFunc->addFnAttr("tune-cpu", *tuneCpu);
 
+  if (auto reciprocalEstimates = func.getReciprocalEstimates())
+    llvmFunc->addFnAttr("reciprocal-estimates", *reciprocalEstimates);
+
   if (auto preferVectorWidth = func.getPreferVectorWidth())
     llvmFunc->addFnAttr("prefer-vector-width", *preferVectorWidth);
 
@@ -2230,9 +2231,6 @@ prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext,
                   StringRef name) {
   m->getContext()->getOrLoadDialect<LLVM::LLVMDialect>();
   auto llvmModule = std::make_unique<llvm::Module>(name, llvmContext);
-  // ModuleTranslation can currently only construct modules in the old debug
-  // info format, so set the flag accordingly.
-  llvmModule->setNewDbgInfoFormatFlag(false);
   if (auto dataLayoutAttr =
           m->getDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName())) {
     llvmModule->setDataLayout(cast<StringAttr>(dataLayoutAttr).getValue());
@@ -2328,7 +2326,7 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
   // Once we've finished constructing elements in the module, we should convert
   // it to use the debug info format desired by LLVM.
   // See https://llvm.org/docs/RemoveDIsDebugInfo.html
-  translator.llvmModule->setIsNewDbgInfoFormat(UseNewDbgInfoFormat);
+  translator.llvmModule->convertToNewDbgValues();
 
   // Add the necessary debug info module flags, if they were not encoded in MLIR
   // beforehand.
diff --git a/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserialization.cpp b/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserialization.cpp
index 7bb876266059..b82c61cafc8a 100644
--- a/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserialization.cpp
+++ b/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserialization.cpp
@@ -12,9 +12,10 @@
 
 using namespace mlir;
 
-OwningOpRef<spirv::ModuleOp> spirv::deserialize(ArrayRef<uint32_t> binary,
-                                                MLIRContext *context) {
-  Deserializer deserializer(binary, context);
+OwningOpRef<spirv::ModuleOp>
+spirv::deserialize(ArrayRef<uint32_t> binary, MLIRContext *context,
+                   const DeserializationOptions &options) {
+  Deserializer deserializer(binary, context, options);
 
   if (failed(deserializer.deserialize()))
     return nullptr;
diff --git a/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 7afd6e9b25b7..b9d9a9015eb6 100644
--- a/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -49,9 +49,10 @@ static inline bool isFnEntryBlock(Block *block) {
 //===----------------------------------------------------------------------===//
 
 spirv::Deserializer::Deserializer(ArrayRef<uint32_t> binary,
-                                  MLIRContext *context)
+                                  MLIRContext *context,
+                                  const spirv::DeserializationOptions &options)
     : binary(binary), context(context), unknownLoc(UnknownLoc::get(context)),
-      module(createModuleOp()), opBuilder(module->getRegion())
+      module(createModuleOp()), opBuilder(module->getRegion()), options(options)
 #ifndef NDEBUG
       ,
       logger(llvm::dbgs())
@@ -866,11 +867,15 @@ LogicalResult spirv::Deserializer::processType(spirv::Opcode opcode,
     typeMap[operands[0]] = IntegerType::get(context, operands[1], sign);
   } break;
   case spirv::Opcode::OpTypeFloat: {
-    if (operands.size() != 2)
-      return emitError(unknownLoc, "OpTypeFloat must have bitwidth parameter");
+    if (operands.size() != 2 && operands.size() != 3)
+      return emitError(unknownLoc,
+                       "OpTypeFloat expects either 2 operands (type, bitwidth) "
+                       "or 3 operands (type, bitwidth, encoding), but got ")
+             << operands.size();
+    uint32_t bitWidth = operands[1];
 
     Type floatTy;
-    switch (operands[1]) {
+    switch (bitWidth) {
     case 16:
       floatTy = opBuilder.getF16Type();
       break;
@@ -882,8 +887,20 @@ LogicalResult spirv::Deserializer::processType(spirv::Opcode opcode,
       break;
     default:
       return emitError(unknownLoc, "unsupported OpTypeFloat bitwidth: ")
-             << operands[1];
+             << bitWidth;
     }
+
+    if (operands.size() == 3) {
+      if (spirv::FPEncoding(operands[2]) != spirv::FPEncoding::BFloat16KHR)
+        return emitError(unknownLoc, "unsupported OpTypeFloat FP encoding: ")
+               << operands[2];
+      if (bitWidth != 16)
+        return emitError(unknownLoc,
+                         "invalid OpTypeFloat bitwidth for bfloat16 encoding: ")
+               << bitWidth << " (expected 16)";
+      floatTy = opBuilder.getBF16Type();
+    }
+
     typeMap[operands[0]] = floatTy;
   } break;
   case spirv::Opcode::OpTypeVector: {
@@ -1061,12 +1078,30 @@ LogicalResult spirv::Deserializer::processCooperativeMatrixTypeKHR(
            << operands[2];
   }
 
-  unsigned rows = getConstantInt(operands[3]).getInt();
-  unsigned columns = getConstantInt(operands[4]).getInt();
+  IntegerAttr rowsAttr = getConstantInt(operands[3]);
+  IntegerAttr columnsAttr = getConstantInt(operands[4]);
+  IntegerAttr useAttr = getConstantInt(operands[5]);
+
+  if (!rowsAttr)
+    return emitError(unknownLoc, "OpTypeCooperativeMatrixKHR `Rows` references "
+                                 "undefined constant <id> ")
+           << operands[3];
+
+  if (!columnsAttr)
+    return emitError(unknownLoc, "OpTypeCooperativeMatrixKHR `Columns` "
+                                 "references undefined constant <id> ")
+           << operands[4];
+
+  if (!useAttr)
+    return emitError(unknownLoc, "OpTypeCooperativeMatrixKHR `Use` references "
+                                 "undefined constant <id> ")
+           << operands[5];
+
+  unsigned rows = rowsAttr.getInt();
+  unsigned columns = columnsAttr.getInt();
 
   std::optional<spirv::CooperativeMatrixUseKHR> use =
-      spirv::symbolizeCooperativeMatrixUseKHR(
-          getConstantInt(operands[5]).getInt());
+      spirv::symbolizeCooperativeMatrixUseKHR(useAttr.getInt());
   if (!use) {
     return emitError(
                unknownLoc,
@@ -1380,6 +1415,9 @@ LogicalResult spirv::Deserializer::processConstant(ArrayRef<uint32_t> operands,
     } else if (floatType.isF16()) {
       APInt data(16, operands[2]);
       value = APFloat(APFloat::IEEEhalf(), data);
+    } else if (floatType.isBF16()) {
+      APInt data(16, operands[2]);
+      value = APFloat(APFloat::BFloat(), data);
     }
 
     auto attr = opBuilder.getFloatAttr(floatType, value);
@@ -1449,11 +1487,11 @@ spirv::Deserializer::processConstantComposite(ArrayRef<uint32_t> operands) {
   }
 
   auto resultID = operands[1];
-  if (auto vectorType = dyn_cast<VectorType>(resultType)) {
-    auto attr = DenseElementsAttr::get(vectorType, elements);
+  if (auto shapedType = dyn_cast<ShapedType>(resultType)) {
+    auto attr = DenseElementsAttr::get(shapedType, elements);
     // For normal constants, we just record the attribute (and its type) for
     // later materialization at use sites.
-    constantMap.try_emplace(resultID, attr, resultType);
+    constantMap.try_emplace(resultID, attr, shapedType);
   } else if (auto arrayType = dyn_cast<spirv::ArrayType>(resultType)) {
     auto attr = opBuilder.getArrayAttr(elements);
     constantMap.try_emplace(resultID, attr, resultType);
@@ -2361,6 +2399,16 @@ LogicalResult spirv::Deserializer::splitConditionalBlocks() {
 }
 
 LogicalResult spirv::Deserializer::structurizeControlFlow() {
+  if (!options.enableControlFlowStructurization) {
+    LLVM_DEBUG(
+        {
+          logger.startLine()
+              << "//----- [cf] skip structurizing control flow -----//\n";
+          logger.indent();
+        });
+    return success();
+  }
+
   LLVM_DEBUG({
     logger.startLine()
         << "//----- [cf] start structurizing control flow -----//\n";
diff --git a/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h b/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
index bcc78e3e6508..e4556e7652b1 100644
--- a/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
+++ b/external/llvm-project/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/Target/SPIRV/Deserialization.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -121,7 +122,8 @@ class Deserializer {
 public:
   /// Creates a deserializer for the given SPIR-V `binary` module.
   /// The SPIR-V ModuleOp will be created into `context.
-  explicit Deserializer(ArrayRef<uint32_t> binary, MLIRContext *context);
+  explicit Deserializer(ArrayRef<uint32_t> binary, MLIRContext *context,
+                        const DeserializationOptions &options);
 
   /// Deserializes the remembered SPIR-V binary module.
   LogicalResult deserialize();
@@ -622,6 +624,9 @@ class Deserializer {
   /// A list of all structs which have unresolved member types.
   SmallVector<DeferredStructTypeInfo, 0> deferredStructTypesInfos;
 
+  /// Deserialization options.
+  DeserializationOptions options;
+
 #ifndef NDEBUG
   /// A logger used to emit information during the deserialzation process.
   llvm::ScopedPrinter logger;
diff --git a/external/llvm-project/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp b/external/llvm-project/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
index 31205d8f408f..0ec468d4c166 100644
--- a/external/llvm-project/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
+++ b/external/llvm-project/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
@@ -13,6 +13,9 @@
 #include "mlir/Target/SPIRV/SPIRVBinaryUtils.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_VERSION_MAJOR
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "spirv-binary-utils"
 
 using namespace mlir;
 
@@ -67,8 +70,19 @@ uint32_t spirv::getPrefixedOpcode(uint32_t wordCount, spirv::Opcode opcode) {
 void spirv::encodeStringLiteralInto(SmallVectorImpl<uint32_t> &binary,
                                     StringRef literal) {
   // We need to encode the literal and the null termination.
-  auto encodingSize = literal.size() / 4 + 1;
-  auto bufferStartSize = binary.size();
+  size_t encodingSize = literal.size() / 4 + 1;
+  size_t sizeOfDataToCopy = literal.size();
+  if (encodingSize >= kMaxLiteralWordCount) {
+    // Reserve one word for the null termination.
+    encodingSize = kMaxLiteralWordCount - 1;
+    // Do not override the last word (null termination) when copying.
+    sizeOfDataToCopy = (encodingSize - 1) * 4;
+    LLVM_DEBUG(llvm::dbgs()
+               << "Truncating string literal to max size ("
+               << (kMaxLiteralWordCount - 1) << "): " << literal << "\n");
+  }
+  size_t bufferStartSize = binary.size();
   binary.resize(bufferStartSize + encodingSize, 0);
-  std::memcpy(binary.data() + bufferStartSize, literal.data(), literal.size());
+  std::memcpy(binary.data() + bufferStartSize, literal.data(),
+              sizeOfDataToCopy);
 }
diff --git a/external/llvm-project/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/external/llvm-project/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
index 15e06616f449..d258bfd85296 100644
--- a/external/llvm-project/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
+++ b/external/llvm-project/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
@@ -523,6 +523,9 @@ LogicalResult Serializer::prepareBasicType(
   if (auto floatType = dyn_cast<FloatType>(type)) {
     typeEnum = spirv::Opcode::OpTypeFloat;
     operands.push_back(floatType.getWidth());
+    if (floatType.isBF16()) {
+      operands.push_back(static_cast<uint32_t>(spirv::FPEncoding::BFloat16KHR));
+    }
     return success();
   }
 
@@ -845,18 +848,44 @@ Serializer::prepareDenseElementsConstant(Location loc, Type constType,
     return 0;
   }
 
+  int64_t numberOfConstituents = shapedType.getDimSize(dim);
   uint32_t resultID = getNextID();
   SmallVector<uint32_t, 4> operands = {typeID, resultID};
-  operands.reserve(shapedType.getDimSize(dim) + 2);
   auto elementType = cast<spirv::CompositeType>(constType).getElementType(0);
-  for (int i = 0; i < shapedType.getDimSize(dim); ++i) {
-    index[dim] = i;
+
+  // "If the Result Type is a cooperative matrix type, then there must be only
+  // one Constituent, with scalar type matching the cooperative matrix Component
+  // Type, and all components of the matrix are initialized to that value."
+  // (https://github.khronos.org/SPIRV-Registry/extensions/KHR/SPV_KHR_cooperative_matrix.html)
+  if (isa<spirv::CooperativeMatrixType>(constType)) {
+    if (!valueAttr.isSplat()) {
+      emitError(
+          loc,
+          "cannot serialize a non-splat value for a cooperative matrix type");
+      return 0;
+    }
+    // numberOfConstituents is 1, so we only need one more elements in the
+    // SmallVector, so the total is 3 (1 + 2).
+    operands.reserve(3);
+    // We set dim directly to `shapedType.getRank()` so the recursive call
+    // directly returns the scalar type.
     if (auto elementID = prepareDenseElementsConstant(
-            loc, elementType, valueAttr, dim + 1, index)) {
+            loc, elementType, valueAttr, /*dim=*/shapedType.getRank(), index)) {
       operands.push_back(elementID);
     } else {
       return 0;
     }
+  } else {
+    operands.reserve(numberOfConstituents + 2);
+    for (int i = 0; i < numberOfConstituents; ++i) {
+      index[dim] = i;
+      if (auto elementID = prepareDenseElementsConstant(
+              loc, elementType, valueAttr, dim + 1, index)) {
+        operands.push_back(elementID);
+      } else {
+        return 0;
+      }
+    }
   }
   spirv::Opcode opcode = spirv::Opcode::OpConstantComposite;
   encodeInstructionInto(typesGlobalValues, opcode, operands);
@@ -996,21 +1025,23 @@ uint32_t Serializer::prepareConstantFp(Location loc, FloatAttr floatAttr,
 
   auto resultID = getNextID();
   APFloat value = floatAttr.getValue();
+  const llvm::fltSemantics *semantics = &value.getSemantics();
 
   auto opcode =
       isSpec ? spirv::Opcode::OpSpecConstant : spirv::Opcode::OpConstant;
 
-  if (&value.getSemantics() == &APFloat::IEEEsingle()) {
+  if (semantics == &APFloat::IEEEsingle()) {
     uint32_t word = llvm::bit_cast<uint32_t>(value.convertToFloat());
     encodeInstructionInto(typesGlobalValues, opcode, {typeID, resultID, word});
-  } else if (&value.getSemantics() == &APFloat::IEEEdouble()) {
+  } else if (semantics == &APFloat::IEEEdouble()) {
     struct DoubleWord {
       uint32_t word1;
       uint32_t word2;
     } words = llvm::bit_cast<DoubleWord>(value.convertToDouble());
     encodeInstructionInto(typesGlobalValues, opcode,
                           {typeID, resultID, words.word1, words.word2});
-  } else if (&value.getSemantics() == &APFloat::IEEEhalf()) {
+  } else if (semantics == &APFloat::IEEEhalf() ||
+             semantics == &APFloat::BFloat()) {
     uint32_t word =
         static_cast<uint32_t>(value.bitcastToAPInt().getZExtValue());
     encodeInstructionInto(typesGlobalValues, opcode, {typeID, resultID, word});
diff --git a/external/llvm-project/mlir/lib/Target/SPIRV/TranslateRegistration.cpp b/external/llvm-project/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
index ff34f02d07b7..682fff278477 100644
--- a/external/llvm-project/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
+++ b/external/llvm-project/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
@@ -37,7 +37,8 @@ using namespace mlir;
 // Deserializes the SPIR-V binary module stored in the file named as
 // `inputFilename` and returns a module containing the SPIR-V module.
 static OwningOpRef<Operation *>
-deserializeModule(const llvm::MemoryBuffer *input, MLIRContext *context) {
+deserializeModule(const llvm::MemoryBuffer *input, MLIRContext *context,
+                  const spirv::DeserializationOptions &options) {
   context->loadDialect<spirv::SPIRVDialect>();
 
   // Make sure the input stream can be treated as a stream of SPIR-V words
@@ -51,17 +52,26 @@ deserializeModule(const llvm::MemoryBuffer *input, MLIRContext *context) {
 
   auto binary = llvm::ArrayRef(reinterpret_cast<const uint32_t *>(start),
                                size / sizeof(uint32_t));
-  return spirv::deserialize(binary, context);
+  return spirv::deserialize(binary, context, options);
 }
 
 namespace mlir {
 void registerFromSPIRVTranslation() {
+  static llvm::cl::opt<bool> enableControlFlowStructurization(
+      "spirv-structurize-control-flow",
+      llvm::cl::desc(
+          "Enable control flow structurization into `spirv.mlir.selection` and "
+          "`spirv.mlir.loop`. This may need to be disabled to support "
+          "deserialization of early exits (see #138688)"),
+      llvm::cl::init(true));
+
   TranslateToMLIRRegistration fromBinary(
       "deserialize-spirv", "deserializes the SPIR-V module",
       [](llvm::SourceMgr &sourceMgr, MLIRContext *context) {
         assert(sourceMgr.getNumBuffers() == 1 && "expected one buffer");
         return deserializeModule(
-            sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID()), context);
+            sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID()), context,
+            {enableControlFlowStructurization});
       });
 }
 } // namespace mlir
diff --git a/external/llvm-project/mlir/lib/Tools/lsp-server-support/Protocol.cpp b/external/llvm-project/mlir/lib/Tools/lsp-server-support/Protocol.cpp
index e4eb2518e079..0054dc37b39c 100644
--- a/external/llvm-project/mlir/lib/Tools/lsp-server-support/Protocol.cpp
+++ b/external/llvm-project/mlir/lib/Tools/lsp-server-support/Protocol.cpp
@@ -289,6 +289,11 @@ bool mlir::lsp::fromJSON(const llvm::json::Value &value,
       if (codeAction->getObject("codeActionLiteralSupport"))
         result.codeActionStructure = true;
     }
+    if (auto *window = textDocument->getObject("window")) {
+      if (std::optional<bool> workDoneProgressSupport =
+              window->getBoolean("workDoneProgress"))
+        result.workDoneProgress = *workDoneProgressSupport;
+    }
   }
   return true;
 }
diff --git a/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp b/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
index 4e19274c3da4..61987525a5ca 100644
--- a/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
+++ b/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
@@ -997,7 +997,7 @@ namespace {
 class MLIRTextFile {
 public:
   MLIRTextFile(const lsp::URIForFile &uri, StringRef fileContents,
-               int64_t version, DialectRegistry &registry,
+               int64_t version, lsp::DialectRegistryFn registry_fn,
                std::vector<lsp::Diagnostic> &diagnostics);
 
   /// Return the current version of this text file.
@@ -1046,9 +1046,9 @@ class MLIRTextFile {
 } // namespace
 
 MLIRTextFile::MLIRTextFile(const lsp::URIForFile &uri, StringRef fileContents,
-                           int64_t version, DialectRegistry &registry,
+                           int64_t version, lsp::DialectRegistryFn registry_fn,
                            std::vector<lsp::Diagnostic> &diagnostics)
-    : context(registry, MLIRContext::Threading::DISABLED),
+    : context(registry_fn(uri), MLIRContext::Threading::DISABLED),
       contents(fileContents.str()), version(version) {
   context.allowUnregisteredDialects();
 
@@ -1263,11 +1263,11 @@ MLIRTextFileChunk &MLIRTextFile::getChunkFor(lsp::Position &pos) {
 //===----------------------------------------------------------------------===//
 
 struct lsp::MLIRServer::Impl {
-  Impl(DialectRegistry &registry) : registry(registry) {}
+  Impl(lsp::DialectRegistryFn registry_fn) : registry_fn(registry_fn) {}
 
-  /// The registry containing dialects that can be recognized in parsed .mlir
-  /// files.
-  DialectRegistry &registry;
+  /// The registry factory for containing dialects that can be recognized in
+  /// parsed .mlir files.
+  lsp::DialectRegistryFn registry_fn;
 
   /// The files held by the server, mapped by their URI file name.
   llvm::StringMap<std::unique_ptr<MLIRTextFile>> files;
@@ -1277,15 +1277,15 @@ struct lsp::MLIRServer::Impl {
 // MLIRServer
 //===----------------------------------------------------------------------===//
 
-lsp::MLIRServer::MLIRServer(DialectRegistry &registry)
-    : impl(std::make_unique<Impl>(registry)) {}
+lsp::MLIRServer::MLIRServer(lsp::DialectRegistryFn registry_fn)
+    : impl(std::make_unique<Impl>(registry_fn)) {}
 lsp::MLIRServer::~MLIRServer() = default;
 
 void lsp::MLIRServer::addOrUpdateDocument(
     const URIForFile &uri, StringRef contents, int64_t version,
     std::vector<Diagnostic> &diagnostics) {
   impl->files[uri.file()] = std::make_unique<MLIRTextFile>(
-      uri, contents, version, impl->registry, diagnostics);
+      uri, contents, version, impl->registry_fn, diagnostics);
 }
 
 std::optional<int64_t> lsp::MLIRServer::removeDocument(const URIForFile &uri) {
@@ -1348,7 +1348,7 @@ void lsp::MLIRServer::getCodeActions(const URIForFile &uri, const Range &pos,
 
 llvm::Expected<lsp::MLIRConvertBytecodeResult>
 lsp::MLIRServer::convertFromBytecode(const URIForFile &uri) {
-  MLIRContext tempContext(impl->registry);
+  MLIRContext tempContext(impl->registry_fn(uri));
   tempContext.allowUnregisteredDialects();
 
   // Collect any errors during parsing.
diff --git a/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.h b/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.h
index 979be615b82c..85e69e69f663 100644
--- a/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.h
+++ b/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MLIRServer.h
@@ -10,6 +10,7 @@
 #define LIB_MLIR_TOOLS_MLIRLSPSERVER_SERVER_H_
 
 #include "mlir/Support/LLVM.h"
+#include "mlir/Tools/mlir-lsp-server/MlirLspRegistryFunction.h"
 #include "llvm/Support/Error.h"
 #include <memory>
 #include <optional>
@@ -28,15 +29,14 @@ struct Location;
 struct MLIRConvertBytecodeResult;
 struct Position;
 struct Range;
-class URIForFile;
 
 /// This class implements all of the MLIR related functionality necessary for a
 /// language server. This class allows for keeping the MLIR specific logic
 /// separate from the logic that involves LSP server/client communication.
 class MLIRServer {
 public:
-  /// Construct a new server with the given dialect regitstry.
-  MLIRServer(DialectRegistry &registry);
+  /// Construct a new server with the given dialect registry function.
+  MLIRServer(DialectRegistryFn registry_fn);
   ~MLIRServer();
 
   /// Add or update the document, with the provided `version`, at the given URI.
diff --git a/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MlirLspServerMain.cpp b/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MlirLspServerMain.cpp
index 259bd2613a6c..b1bbf98ce769 100644
--- a/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MlirLspServerMain.cpp
+++ b/external/llvm-project/mlir/lib/Tools/mlir-lsp-server/MlirLspServerMain.cpp
@@ -19,7 +19,7 @@ using namespace mlir;
 using namespace mlir::lsp;
 
 LogicalResult mlir::MlirLspServerMain(int argc, char **argv,
-                                      DialectRegistry &registry) {
+                                      DialectRegistryFn registry_fn) {
   llvm::cl::opt<JSONStreamStyle> inputStyle{
       "input-style",
       llvm::cl::desc("Input JSON stream encoding"),
@@ -72,6 +72,15 @@ LogicalResult mlir::MlirLspServerMain(int argc, char **argv,
   URIForFile::registerSupportedScheme("mlir.bytecode-mlir");
 
   // Configure the servers and start the main language server.
-  MLIRServer server(registry);
+  MLIRServer server(registry_fn);
   return runMlirLSPServer(server, transport);
 }
+
+llvm::LogicalResult mlir::MlirLspServerMain(int argc, char **argv,
+                                            DialectRegistry &registry) {
+  auto registry_fn =
+      [&registry](const lsp::URIForFile &uri) -> DialectRegistry & {
+    return registry;
+  };
+  return MlirLspServerMain(argc, argv, registry_fn);
+}
diff --git a/external/llvm-project/mlir/lib/Transforms/Utils/CFGToSCF.cpp b/external/llvm-project/mlir/lib/Transforms/Utils/CFGToSCF.cpp
index eefdf1d4e393..7c1781044d2a 100644
--- a/external/llvm-project/mlir/lib/Transforms/Utils/CFGToSCF.cpp
+++ b/external/llvm-project/mlir/lib/Transforms/Utils/CFGToSCF.cpp
@@ -427,8 +427,7 @@ class ReturnLikeExitCombiner {
   /// region with an instance of `returnLikeOp`s kind.
   void combineExit(Operation *returnLikeOp,
                    function_ref<Value(unsigned)> getSwitchValue) {
-    auto [iter, inserted] =
-        returnLikeToCombinedExit.insert({returnLikeOp, nullptr});
+    auto [iter, inserted] = returnLikeToCombinedExit.try_emplace(returnLikeOp);
     if (!inserted && iter->first == returnLikeOp)
       return;
 
@@ -710,7 +709,7 @@ transformToReduceLoop(Block *loopHeader, Block *exitBlock,
     llvm::SmallDenseMap<Block *, bool> dominanceCache;
     // Returns true if `loopBlock` dominates `block`.
     auto loopBlockDominates = [&](Block *block) {
-      auto [iter, inserted] = dominanceCache.insert({block, false});
+      auto [iter, inserted] = dominanceCache.try_emplace(block);
       if (!inserted)
         return iter->second;
       iter->second = dominanceInfo.dominates(loopBlock, block);
@@ -1284,7 +1283,7 @@ FailureOr<bool> mlir::transformCFGToSCF(Region &region,
 
   DenseMap<Type, Value> typedUndefCache;
   auto getUndefValue = [&](Type type) {
-    auto [iter, inserted] = typedUndefCache.insert({type, nullptr});
+    auto [iter, inserted] = typedUndefCache.try_emplace(type);
     if (!inserted)
       return iter->second;
 
diff --git a/external/llvm-project/mlir/lib/Transforms/Utils/CMakeLists.txt b/external/llvm-project/mlir/lib/Transforms/Utils/CMakeLists.txt
index 3939a00104a5..3ca16239ba33 100644
--- a/external/llvm-project/mlir/lib/Transforms/Utils/CMakeLists.txt
+++ b/external/llvm-project/mlir/lib/Transforms/Utils/CMakeLists.txt
@@ -14,9 +14,6 @@ add_mlir_library(MLIRTransformUtils
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Transforms
 
-  DEPENDS
-  MLIRTransformsPassIncGen
-
   LINK_LIBS PUBLIC
   MLIRAnalysis
   MLIRCallInterfaces
@@ -26,5 +23,4 @@ add_mlir_library(MLIRTransformUtils
   MLIRSideEffectInterfaces
   MLIRSubsetOpInterface
   MLIRRewrite
-  MLIRFuncDialect
   )
diff --git a/external/llvm-project/mlir/lib/Transforms/Utils/Inliner.cpp b/external/llvm-project/mlir/lib/Transforms/Utils/Inliner.cpp
index 54b5c788a352..ae34c4ad49e2 100644
--- a/external/llvm-project/mlir/lib/Transforms/Utils/Inliner.cpp
+++ b/external/llvm-project/mlir/lib/Transforms/Utils/Inliner.cpp
@@ -45,7 +45,7 @@ static void walkReferencedSymbolNodes(
 
   Operation *symbolTableOp = op->getParentOp();
   for (const SymbolTable::SymbolUse &use : *symbolUses) {
-    auto refIt = resolvedRefs.insert({use.getSymbolRef(), nullptr});
+    auto refIt = resolvedRefs.try_emplace(use.getSymbolRef());
     CallGraphNode *&node = refIt.first->second;
 
     // If this is the first instance of this reference, try to resolve a
diff --git a/external/llvm-project/mlir/python/mlir/dialects/transform/__init__.py b/external/llvm-project/mlir/python/mlir/dialects/transform/__init__.py
index 5b158ec6b65f..b075919d1ef0 100644
--- a/external/llvm-project/mlir/python/mlir/dialects/transform/__init__.py
+++ b/external/llvm-project/mlir/python/mlir/dialects/transform/__init__.py
@@ -18,7 +18,12 @@
 except ImportError as e:
     raise RuntimeError("Error loading imports from extension module") from e
 
-from typing import Optional, Sequence, Union, NewType
+from typing import Dict, Optional, Sequence, Union, NewType
+
+
+@register_attribute_builder("ParamOperandAttr")
+def _paramOperandAttr(x: int, context) -> Attribute:
+    return Attribute.parse(f"#transform.param_operand<index={x}>", context=context)
 
 
 @_ods_cext.register_operation(_Dialect, replace=True)
@@ -214,6 +219,86 @@ def __init__(
         super().__init__(_get_op_results_or_values(operands), loc=loc, ip=ip)
 
 
+OptionValueTypes = Union[
+    Sequence["OptionValueTypes"], Attribute, Value, Operation, OpView, str, int, bool
+]
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ApplyRegisteredPassOp(ApplyRegisteredPassOp):
+    def __init__(
+        self,
+        result: Type,
+        target: Union[Operation, Value, OpView],
+        pass_name: Union[str, StringAttr],
+        *,
+        options: Optional[Dict[Union[str, StringAttr], OptionValueTypes]] = None,
+        loc=None,
+        ip=None,
+    ):
+        options_dict = {}
+        dynamic_options = []
+
+        ParamOperandAttr = AttrBuilder.get("ParamOperandAttr")
+        context = (loc and loc.context) or Context.current
+
+        cur_param_operand_idx = 0
+
+        def option_value_to_attr(value):
+            nonlocal cur_param_operand_idx
+            if isinstance(value, (Value, Operation, OpView)):
+                dynamic_options.append(_get_op_result_or_value(value))
+                cur_param_operand_idx += 1
+                return ParamOperandAttr(cur_param_operand_idx - 1, context)
+            elif isinstance(value, Attribute):
+                return value
+            # The following cases auto-convert Python values to attributes.
+            elif isinstance(value, bool):
+                return BoolAttr.get(value)
+            elif isinstance(value, int):
+                default_int_type = IntegerType.get_signless(64, context)
+                return IntegerAttr.get(default_int_type, value)
+            elif isinstance(value, str):
+                return StringAttr.get(value)
+            elif isinstance(value, Sequence):
+                return ArrayAttr.get([option_value_to_attr(elt) for elt in value])
+            else:
+                raise TypeError(f"Unsupported option type: {type(value)}")
+
+        for key, value in options.items() if options is not None else {}:
+            if isinstance(key, StringAttr):
+                key = key.value
+            options_dict[key] = option_value_to_attr(value)
+        super().__init__(
+            result,
+            _get_op_result_or_value(target),
+            pass_name,
+            dynamic_options,
+            options=DictAttr.get(options_dict),
+            loc=loc,
+            ip=ip,
+        )
+
+
+def apply_registered_pass(
+    result: Type,
+    target: Union[Operation, Value, OpView],
+    pass_name: Union[str, StringAttr],
+    *,
+    options: Optional[Dict[Union[str, StringAttr], OptionValueTypes]] = None,
+    loc=None,
+    ip=None,
+) -> Value:
+    return ApplyRegisteredPassOp(
+        result=result,
+        pass_name=pass_name,
+        target=target,
+        options=options,
+        loc=loc,
+        ip=ip,
+    ).result
+
+
 AnyOpTypeT = NewType("AnyOpType", AnyOpType)
 
 
diff --git a/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir b/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir
index cb3539dd11be..581346e03b89 100644
--- a/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir
+++ b/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/load_lds.mlir
@@ -31,8 +31,8 @@ func.func @global_load_to_rocdl_f32(%global : memref<128x72xf32, #gpu_global_add
   // CHECK: %[[GLOBAL_PTR:.*]] = llvm.getelementptr %[[GLOBAL_BASE]][%[[SRC_OFFSET]]]
   // CHECK: %[[LDS_BASE:.*]] = llvm.extractvalue %[[LDS_DESC]][1]
 
-  // CHECK: %[[C72_1:.*]] = llvm.mlir.constant(72 : index) : i64
-  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C72_1]] : i64
+  // CHECK: %[[C64:.*]] = llvm.mlir.constant(64 : index) : i64
+  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C64]] : i64
   // CHECK: %[[DST_OFFSET:.*]] = llvm.add %[[MUL_2]], %[[IC0]] : i64
 
   // CHECK: %[[LDS_PTR:.*]] = llvm.getelementptr %[[LDS_BASE]][%[[DST_OFFSET]]]
@@ -65,8 +65,8 @@ func.func @global_load_to_rocdl_i8(%global : memref<128x72xi8, #gpu_global_addrs
   // CHECK: %[[GLOBAL_PTR:.*]] = llvm.getelementptr %[[GLOBAL_BASE]][%[[SRC_OFFSET]]]
   // CHECK: %[[LDS_BASE:.*]] = llvm.extractvalue %[[LDS_DESC]][1]
 
-  // CHECK: %[[C72_1:.*]] = llvm.mlir.constant(72 : index) : i64
-  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C72_1]] : i64
+  // CHECK: %[[C64:.*]] = llvm.mlir.constant(64 : index) : i64
+  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C64]] : i64
   // CHECK: %[[DST_OFFSET:.*]] = llvm.add %[[MUL_2]], %[[IC0]] : i64
 
   // CHECK: %[[LDS_PTR:.*]] = llvm.getelementptr %[[LDS_BASE]][%[[DST_OFFSET]]]
@@ -103,8 +103,8 @@ func.func @global_load_to_rocdl_vec(%global : memref<128x72xi16, #gpu_global_add
   // CHECK: %[[GLOBAL_PTR:.*]] = llvm.getelementptr %[[GLOBAL_BASE]][%[[SRC_OFFSET]]]
   // CHECK: %[[LDS_BASE:.*]] = llvm.extractvalue %[[LDS_DESC]][1]
 
-  // CHECK: %[[C72_1:.*]] = llvm.mlir.constant(72 : index) : i64
-  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C72_1]] : i64
+  // CHECK: %[[C128:.*]] = llvm.mlir.constant(128 : index) : i64
+  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C128]] : i64
   // CHECK: %[[DST_OFFSET:.*]] = llvm.add %[[MUL_2]], %[[IC0]] : i64
 
   // CHECK: %[[LDS_PTR:.*]] = llvm.getelementptr %[[LDS_BASE]][%[[DST_OFFSET]]]
@@ -130,7 +130,9 @@ func.func @global_load_to_rocdl_dynamic_indices(%global : memref<512xi32, #gpu_g
   // CHECK: %[[GLOBAL_BASE:.*]] = llvm.extractvalue %[[GLOBAL_DESC]][1]
   // CHECK: %[[GLOBAL_PTR:.*]] = llvm.getelementptr %[[GLOBAL_BASE]][%[[SRCIDX_CAST]]]
   // CHECK: %[[LDS_BASE:.*]] = llvm.extractvalue %[[LDS_DESC]][1]
-  // CHECK: %[[LDS_PTR:.*]] = llvm.getelementptr %[[LDS_BASE]][%[[DSTIDX_CAST]]]
+  // CHECK: %[[C64:.*]] = llvm.mlir.constant(64 : index) : i64
+  // CHECK: %[[DSTIDX:.*]] = llvm.mul %[[DSTIDX_CAST]], %[[C64]] : i64
+  // CHECK: %[[LDS_PTR:.*]] = llvm.getelementptr %[[LDS_BASE]][%[[DSTIDX]]]
   // CHECK: rocdl.load.to.lds %[[GLOBAL_PTR]], %[[LDS_PTR]], 4
   %alloc = memref.alloc() : memref<4x64xi32, #gpu_lds_addrspace>
   %c0 = arith.constant 0 : index
@@ -166,8 +168,8 @@ func.func @fat_buffer_load_to_rocdl_f32(%global : memref<128x72xf32, #amdgpu_fat
   // CHECK: %[[GLOBAL_PTR:.*]] = llvm.getelementptr %[[GLOBAL_BASE]][%[[SRC_OFFSET]]]
   // CHECK: %[[LDS_BASE:.*]] = llvm.extractvalue %[[LDS_DESC]][1]
 
-  // CHECK: %[[C72_1:.*]] = llvm.mlir.constant(72 : index) : i64
-  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C72_1]] : i64
+  // CHECK: %[[C64:.*]] = llvm.mlir.constant(64 : index) : i64
+  // CHECK: %[[MUL_2:.*]] = llvm.mul %[[IC32]], %[[C64]] : i64
   // CHECK: %[[DST_OFFSET:.*]] = llvm.add %[[MUL_2]], %[[IC0]] : i64
 
   // CHECK: %[[LDS_PTR:.*]] = llvm.getelementptr %[[LDS_BASE]][%[[DST_OFFSET]]]
diff --git a/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/packed-ext.mlir b/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/packed-ext.mlir
new file mode 100644
index 000000000000..ad2e7684afc4
--- /dev/null
+++ b/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/packed-ext.mlir
@@ -0,0 +1,492 @@
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_f32
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_full_f8e4m3_f32(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_f16
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_full_f8e4m3_f16(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_bf16
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_full_f8e4m3_bf16(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E4M3FN> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_half_f8e4m3_f32(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E4M3FN> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_half_f8e4m3_f16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E4M3FN> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_half_f8e4m3_bf16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E4M3FN> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_scalar_f8e4m3_f32(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E4M3FN> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_scalar_f8e4m3_f16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E4M3FN> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_scalar_f8e4m3_bf16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_f32
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.bf8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_full_f8e5m2_f32(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_f16
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.bf8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_full_f8e5m2_f16(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_bf16
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.bf8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_full_f8e5m2_bf16(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E5M2> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.bf8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_half_f8e5m2_f32(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E5M2> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.bf8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_half_f8e5m2_f16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E5M2> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.bf8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_half_f8e5m2_bf16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E5M2> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.bf8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_scalar_f8e5m2_f32(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E5M2> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.bf8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_scalar_f8e5m2_f16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf8E5M2> to vector<2xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi8>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.bf8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_scalar_f8e5m2_bf16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_f32
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp4 [[BITCAST]][0], %arg1 : vector<2xf32>
+func.func @scaled_ext_full_f4e2m1_f32(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_f16
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp4 [[BITCAST]][0], %arg1 : vector<2xf16>
+func.func @scaled_ext_full_f4e2m1_f16(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_bf16
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp4 [[BITCAST]][0], %arg1 : vector<2xbf16>
+func.func @scaled_ext_full_f4e2m1_bf16(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_f32
+// CHECK-DAG:   [[CAST:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[BITCAST:%.+]] = llvm.bitcast [[CAST]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp4 [[BITCAST]][0], %arg1 : vector<2xf32>
+func.func @scaled_ext_half_f4e2m1_f32(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf4E2M1FN> to vector<4xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C2:%.+]] = llvm.mlir.constant(2 : i32) : i32
+// CHECK:       [[ELEM_2:%.+]] = llvm.extractelement [[V]]{{\[}}[[C2]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_2:%.+]] = llvm.insertelement [[ELEM_2]], [[VEC_1]]{{\[}}[[C2]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C3:%.+]] = llvm.mlir.constant(3 : i32) : i32
+// CHECK:       [[ELEM_3:%.+]] = llvm.extractelement [[V]]{{\[}}[[C3]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_3:%.+]] = llvm.insertelement [[ELEM_3]], [[VEC_2]]{{\[}}[[C3]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_3]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp4 [[BITCAST]][0], %arg1 : vector<2xf16>
+func.func @scaled_ext_half_f4e2m1_f16(%v: vector<4xf4E2M1FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf4E2M1FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<4xf4E2M1FN> to vector<4xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C2:%.+]] = llvm.mlir.constant(2 : i32) : i32
+// CHECK:       [[ELEM_2:%.+]] = llvm.extractelement [[V]]{{\[}}[[C2]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_2:%.+]] = llvm.insertelement [[ELEM_2]], [[VEC_1]]{{\[}}[[C2]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C3:%.+]] = llvm.mlir.constant(3 : i32) : i32
+// CHECK:       [[ELEM_3:%.+]] = llvm.extractelement [[V]]{{\[}}[[C3]] : i32] : vector<4xi4>
+// CHECK:       [[VEC_3:%.+]] = llvm.insertelement [[ELEM_3]], [[VEC_2]]{{\[}}[[C3]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_3]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp4 [[BITCAST]][0], %arg1 : vector<2xbf16>
+func.func @scaled_ext_half_f4e2m1_bf16(%v: vector<4xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf4E2M1FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf4E2M1FN> to vector<2xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi4>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp4 [[BITCAST]][0], %arg1 : vector<2xf32>
+func.func @scaled_ext_scalar_f4e2m1_f32(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf4E2M1FN> to vector<2xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi4>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp4 [[BITCAST]][0], %arg1 : vector<2xf16>
+func.func @scaled_ext_scalar_f4e2m1_f16(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<2xf4E2M1FN> to vector<2xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<2xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM_1:%.+]] = llvm.extractelement [[V]]{{\[}}[[C1]] : i32] : vector<2xi4>
+// CHECK:       [[VEC_1:%.+]] = llvm.insertelement [[ELEM_1]], [[VEC_0]]{{\[}}[[C1]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_1]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp4 [[BITCAST]][0], %arg1 : vector<2xbf16>
+func.func @scaled_ext_scalar_f4e2m1_bf16(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f8e4m3_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf8E4M3FN> to vector<1xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_one_f8e4m3_f32(%v: vector<1xf8E4M3FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf8E4M3FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f8e4m3_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf8E4M3FN> to vector<1xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_one_f8e4m3_f16(%v: vector<1xf8E4M3FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf8E4M3FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f8e4m3_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf8E4M3FN> to vector<1xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_one_f8e4m3_bf16(%v: vector<1xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf8E4M3FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f8e5m2_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf8E5M2> to vector<1xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.bf8 [[BITCAST]][false], %arg1 : vector<2xf32>
+func.func @scaled_ext_one_f8e5m2_f32(%v: vector<1xf8E5M2>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf8E5M2> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f8e5m2_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf8E5M2> to vector<1xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.bf8 [[BITCAST]][false], %arg1 : vector<2xf16>
+func.func @scaled_ext_one_f8e5m2_f16(%v: vector<1xf8E5M2>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf8E5M2> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f8e5m2_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf8E5M2> to vector<1xi8>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<4xi8>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi8>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<4xi8>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<4xi8> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.bf8 [[BITCAST]][false], %arg1 : vector<2xbf16>
+func.func @scaled_ext_one_f8e5m2_bf16(%v: vector<1xf8E5M2>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf8E5M2> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f4e2m1_f32
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf4E2M1FN> to vector<1xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f32.fp4 [[BITCAST]][0], %arg1 : vector<2xf32>
+func.func @scaled_ext_one_f4e2m1_f32(%v: vector<1xf4E2M1FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf4E2M1FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f4e2m1_f16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf4E2M1FN> to vector<1xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.f16.fp4 [[BITCAST]][0], %arg1 : vector<2xf16>
+func.func @scaled_ext_one_f4e2m1_f16(%v: vector<1xf4E2M1FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf4E2M1FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_one_f4e2m1_bf16
+// CHECK:       [[V:%.+]] = builtin.unrealized_conversion_cast %arg0 : vector<1xf4E2M1FN> to vector<1xi4>
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<8xi4>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[ELEM_0:%.+]] = llvm.extractelement [[V]]{{\[}}[[C0]] : i32] : vector<1xi4>
+// CHECK:       [[VEC_0:%.+]] = llvm.insertelement [[ELEM_0]], [[ZERO]]{{\[}}[[C0]] : i32] : vector<8xi4>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[VEC_0]] : vector<8xi4> to i32
+// CHECK:       rocdl.cvt.scalef32.pk.bf16.fp4 [[BITCAST]][0], %arg1 : vector<2xbf16>
+func.func @scaled_ext_one_f4e2m1_bf16(%v: vector<1xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<1xf4E2M1FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
diff --git a/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc.mlir b/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc.mlir
new file mode 100644
index 000000000000..e9764d34cefa
--- /dev/null
+++ b/external/llvm-project/mlir/test/Conversion/AMDGPUToROCDL/packed-trunc.mlir
@@ -0,0 +1,535 @@
+// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f32
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_f8e4m3_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f32_vec1
+// CHECK-DAG:   [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>
+// CHECK:       [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>
+// CHECK-DAG:   [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_f8e4m3_f32_vec1(%v: vector<1xf32>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf32> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f32
+// CHECK-DAG:   [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_into_f8e4m3_f32(%v: vector<2xf32>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f32_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>
+// CHECK:       [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>
+// CHECK-DAG:   [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_into_f8e4m3_f32_vec1(%v: vector<1xf32>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf32> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f16
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_f8e4m3_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f16_vec1
+// CHECK-DAG:   [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>
+// CHECK:       [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_f8e4m3_f16_vec1(%v: vector<1xf16>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf16> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f16
+// CHECK-DAG:   [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_into_f8e4m3_f16(%v: vector<2xf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f16_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>
+// CHECK:       [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_into_f8e4m3_f16_vec1(%v: vector<1xf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_bf16
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_f8e4m3_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_bf16_vec1
+// CHECK-DAG:   [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>
+// CHECK:       [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_f8e4m3_bf16_vec1(%v: vector<1xbf16>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xbf16> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_bf16
+// CHECK-DAG:   [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_into_f8e4m3_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_bf16_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>
+// CHECK:       [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>
+// CHECK:       return [[CAST]] : vector<4xf8E4M3FN>
+func.func @packed_scaled_trunc_into_f8e4m3_bf16_vec1(%v: vector<1xbf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xbf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f32
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_f8e5m2_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f32_vec1
+// CHECK-DAG:   [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>
+// CHECK:       [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>
+// CHECK-DAG:   [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_f8e5m2_f32_vec1(%v: vector<1xf32>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf32> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f32
+// CHECK-DAG:   [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_into_f8e5m2_f32(%v: vector<2xf32>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f32_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>
+// CHECK:       [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>
+// CHECK-DAG:   [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_into_f8e5m2_f32_vec1(%v: vector<1xf32>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf32> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f16
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_f8e5m2_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f16_vec1
+// CHECK-DAG:   [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>
+// CHECK:       [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_f8e5m2_f16_vec1(%v: vector<1xf16>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf16> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f16
+// CHECK-DAG:   [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_into_f8e5m2_f16(%v: vector<2xf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f16_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>
+// CHECK:       [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_into_f8e5m2_f16_vec1(%v: vector<1xf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf16> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_bf16
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_f8e5m2_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_bf16_vec1
+// CHECK-DAG:   [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>
+// CHECK:       [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_f8e5m2_bf16_vec1(%v: vector<1xbf16>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xbf16> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_bf16
+// CHECK-DAG:   [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_into_f8e5m2_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_bf16_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>
+// CHECK:       [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>
+// CHECK:       return [[CAST]] : vector<4xf8E5M2>
+func.func @packed_scaled_trunc_into_f8e5m2_bf16_vec1(%v: vector<1xbf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xbf16> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f32
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : i32
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_f4e2m1_f32(%v: vector<2xf32>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f32_vec1
+// CHECK-DAG:   [[ZERO_I32:%.+]] = llvm.mlir.zero : i32
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>
+// CHECK:       [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>
+// CHECK-DAG:   [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO_I32]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_f4e2m1_f32_vec1(%v: vector<1xf32>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf32> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f32
+// CHECK-DAG:   [[BITCAST_I4:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[BITCAST_I32:%.+]] = llvm.bitcast [[BITCAST_I4]] : vector<8xi4> to i32
+// CHECK-DAG:   [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[BITCAST_I32]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_into_f4e2m1_f32(%v: vector<2xf32>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f32_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<8xi4> to i32
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>
+// CHECK:       [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>
+// CHECK-DAG:   [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK-DAG:   [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK:       [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>
+// CHECK:       [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_into_f4e2m1_f32_vec1(%v: vector<1xf32>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf32> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f16
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : i32
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 %arg0, %arg1 -> [[ZERO]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_f4e2m1_f16(%v: vector<2xf16>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f16_vec1
+// CHECK-DAG:   [[ZERO_I32:%.+]] = llvm.mlir.zero : i32
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>
+// CHECK:       [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 [[INSERT]], %arg1 -> [[ZERO_I32]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_f4e2m1_f16_vec1(%v: vector<1xf16>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf16> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f16
+// CHECK-DAG:   [[BITCAST_I4:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[BITCAST_I32:%.+]] = llvm.bitcast [[BITCAST_I4]] : vector<8xi4> to i32
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 %arg0, %arg2 -> [[BITCAST_I32]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_into_f4e2m1_f16(%v: vector<2xf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f16_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<8xi4> to i32
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>
+// CHECK:       [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_into_f4e2m1_f16_vec1(%v: vector<1xf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_bf16
+// CHECK-DAG:   [[ZERO:%.+]] = llvm.mlir.zero : i32
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 %arg0, %arg1 -> [[ZERO]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_f4e2m1_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_bf16_vec1
+// CHECK-DAG:   [[ZERO_I32:%.+]] = llvm.mlir.zero : i32
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>
+// CHECK:       [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 [[INSERT]], %arg1 -> [[ZERO_I32]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_f4e2m1_bf16_vec1(%v: vector<1xbf16>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xbf16> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_bf16
+// CHECK-DAG:   [[BITCAST_I4:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[BITCAST_I32:%.+]] = llvm.bitcast [[BITCAST_I4]] : vector<8xi4> to i32
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 %arg0, %arg2 -> [[BITCAST_I32]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_into_f4e2m1_bf16(%v: vector<2xbf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_bf16_vec1
+// CHECK-DAG:   [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>
+// CHECK-DAG:   [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<8xi4> to i32
+// CHECK-DAG:   [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:       [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>
+// CHECK:       [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>
+// CHECK:       [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>
+// CHECK:       [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][0] : i32
+// CHECK:       [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>
+// CHECK:       [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>
+// CHECK:       return [[CAST]] : vector<8xf4E2M1FN>
+func.func @packed_scaled_trunc_into_f4e2m1_bf16_vec1(%v: vector<1xbf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xbf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
diff --git a/external/llvm-project/mlir/test/Conversion/ConvertToSPIRV/func-signature-vector-unroll.mlir b/external/llvm-project/mlir/test/Conversion/ConvertToSPIRV/func-signature-vector-unroll.mlir
index c018ccb92498..211d6c90243b 100644
--- a/external/llvm-project/mlir/test/Conversion/ConvertToSPIRV/func-signature-vector-unroll.mlir
+++ b/external/llvm-project/mlir/test/Conversion/ConvertToSPIRV/func-signature-vector-unroll.mlir
@@ -189,3 +189,76 @@ func.func @unsupported_scalable(%arg0 : vector<[8]xi32>) -> (vector<[8]xi32>) {
   return %arg0 : vector<[8]xi32>
 }
 
+// -----
+
+// Check that already legal function parameters are properly preserved across multiple blocks.
+
+// CHECK-LABEL: func.func @legal_params_multiple_blocks_simple
+// CHECK-SAME: (%[[ARG0:.+]]: i32, %[[ARG1:.+]]: i32) -> i32
+func.func @legal_params_multiple_blocks_simple(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK: %[[ADD0:.*]] = arith.addi %[[ARG0]], %[[ARG1]] : i32
+  // CHECK: %[[ADD1:.*]] = arith.addi %[[ADD0]], %[[ARG1]] : i32
+  // CHECK: return %[[ADD1]] : i32
+  cf.br ^bb1(%arg0 : i32)
+^bb1(%acc0: i32):
+  %acc1_val = arith.addi %acc0, %arg1 : i32
+  cf.br ^bb2(%acc1_val : i32)
+^bb2(%acc1: i32):
+  %acc2_val = arith.addi %acc1, %arg1 : i32
+  cf.br ^bb3(%acc2_val : i32)
+^bb3(%acc_final: i32):
+  return %acc_final : i32
+}
+
+// -----
+
+// Check that legal parameters and existing `vector.insert_strided_slice`s are properly preserved across multiple blocks.
+
+// CHECK-LABEL: func.func @legal_params_with_vec_insert_multiple_blocks
+// CHECK-SAME: (%[[ARG0:.+]]: i32, %[[ARG1:.+]]: i32, %[[ARG2:.+]]: vector<4xi32>) -> vector<4xi32>
+func.func @legal_params_with_vec_insert_multiple_blocks(%arg0: i32, %arg1: i32, %arg2: vector<4xi32>) -> vector<4xi32> {
+  // CHECK: %[[ADD0:.*]] = arith.addi %[[ARG0]], %[[ARG1]] : i32
+  // CHECK: %[[ADD1:.*]] = arith.addi %[[ADD0]], %[[ARG1]] : i32
+  // CHECK: %[[VEC1D:.*]] = vector.broadcast %[[ADD1]] : i32 to vector<1xi32>
+  // CHECK: %[[VEC0:.*]] = vector.insert_strided_slice %[[VEC1D]], %[[ARG2]] {offsets = [1], strides = [1]} : vector<1xi32> into vector<4xi32>
+  // CHECK: %[[VEC1:.*]] = vector.insert_strided_slice %[[VEC1D]], %[[VEC0]] {offsets = [2], strides = [1]} : vector<1xi32> into vector<4xi32>
+  // CHECK: %[[RESULT:.*]] = vector.insert_strided_slice %[[VEC1D]], %[[VEC1]] {offsets = [3], strides = [1]} : vector<1xi32> into vector<4xi32>
+  // CHECK: return %[[RESULT]] : vector<4xi32>
+  cf.br ^bb1(%arg0 : i32)
+^bb1(%acc0: i32):
+  %acc1_val = arith.addi %acc0, %arg1 : i32
+  cf.br ^bb2(%acc1_val : i32)
+^bb2(%acc1: i32):
+  %acc2_val = arith.addi %acc1, %arg1 : i32
+  cf.br ^bb3(%acc2_val : i32)
+^bb3(%acc_final: i32):
+  %scalar_vec = vector.broadcast %acc_final : i32 to vector<1xi32>
+  %vec0 = vector.insert_strided_slice %scalar_vec, %arg2 {offsets = [1], strides = [1]} : vector<1xi32> into vector<4xi32>
+  %vec1 = vector.insert_strided_slice %scalar_vec, %vec0 {offsets = [2], strides = [1]} : vector<1xi32> into vector<4xi32>
+  %result = vector.insert_strided_slice %scalar_vec, %vec1 {offsets = [3], strides = [1]} : vector<1xi32> into vector<4xi32>
+  return %result : vector<4xi32>
+}
+
+// -----
+
+// Check that already legal function parameters are preserved across a loop (which contains multiple blocks).
+
+// CHECK-LABEL: @legal_params_for_loop
+// CHECK-SAME: (%[[ARG0:.+]]: i32, %[[ARG1:.+]]: i32, %[[ARG2:.+]]: i32)
+func.func @legal_params_for_loop(%arg0: i32, %arg1: i32, %arg2: i32) -> i32 {
+  // CHECK: %[[CST0:.*]] = arith.constant 0 : index
+  // CHECK: %[[CST1:.*]] = arith.constant 1 : index
+  // CHECK: %[[UB:.*]] = arith.index_cast %[[ARG2]] : i32 to index
+  // CHECK: %[[RESULT:.*]] = scf.for %[[STEP:.*]] = %[[CST0]] to %[[UB]] step %[[CST1]] iter_args(%[[ACC:.*]] = %[[ARG0]]) -> (i32) {
+  // CHECK:   %[[ADD:.*]] = arith.addi %[[ACC]], %[[ARG1]] : i32
+  // CHECK:   scf.yield %[[ADD]] : i32
+  // CHECK: return %[[RESULT]] : i32
+  %zero = arith.constant 0 : index
+  %one = arith.constant 1 : index
+  %ub = arith.index_cast %arg2 : i32 to index
+  %result = scf.for %i = %zero to %ub step %one iter_args(%acc = %arg0) -> (i32) {
+    %new_acc = arith.addi %acc, %arg1 : i32
+    scf.yield %new_acc : i32
+  }
+  return %result : i32
+}
diff --git a/external/llvm-project/mlir/test/Conversion/FuncToSPIRV/types-to-spirv.mlir b/external/llvm-project/mlir/test/Conversion/FuncToSPIRV/types-to-spirv.mlir
index 82d750755ffe..1737f4a906bf 100644
--- a/external/llvm-project/mlir/test/Conversion/FuncToSPIRV/types-to-spirv.mlir
+++ b/external/llvm-project/mlir/test/Conversion/FuncToSPIRV/types-to-spirv.mlir
@@ -173,6 +173,12 @@ func.func @float16(%arg0: f16) { return }
 // NOEMU-SAME: f64
 func.func @float64(%arg0: f64) { return }
 
+// CHECK-LABEL: spirv.func @bfloat16
+// CHECK-SAME: f32
+// NOEMU-LABEL: func.func @bfloat16
+// NOEMU-SAME: bf16
+func.func @bfloat16(%arg0: bf16) { return }
+
 // f80 is not supported by SPIR-V.
 // CHECK-LABEL: func.func @float80
 // CHECK-SAME: f80
@@ -206,18 +212,6 @@ func.func @float64(%arg0: f64) { return }
 
 // -----
 
-// Check that bf16 is not supported.
-module attributes {
-  spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [], []>, #spirv.resource_limits<>>
-} {
-
-// CHECK-NOT: spirv.func @bf16_type
-func.func @bf16_type(%arg0: bf16) { return }
-
-} // end module
-
-// -----
-
 //===----------------------------------------------------------------------===//
 // Complex types
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm-debuginfo.mlir b/external/llvm-project/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm-debuginfo.mlir
new file mode 100644
index 000000000000..08c5800fe93b
--- /dev/null
+++ b/external/llvm-project/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm-debuginfo.mlir
@@ -0,0 +1,25 @@
+// RUN: mlir-opt %s -convert-gpu-to-nvvm='has-redux=1' -mlir-print-debuginfo | FileCheck %s
+
+#di_file = #llvm.di_file<"foo.mlir" in "/tmp/">
+#di_compile_unit = #llvm.di_compile_unit<
+  id = distinct[0]<>, sourceLanguage = DW_LANG_C, file = #di_file,
+  producer = "MLIR", isOptimized = true, emissionKind = Full
+>
+#di_subprogram = #llvm.di_subprogram<
+  compileUnit = #di_compile_unit, scope = #di_file, name = "test_const_printf_with_loc",
+  file = #di_file, subprogramFlags = "Definition"
+>
+
+// CHECK-DAG: [[LOC:#[a-zA-Z0-9_]+]] = loc("foo.mlir":0:0)
+#loc = loc("foo.mlir":0:0)
+
+// Check that debug info metadata from the function is removed from the global location.
+gpu.module @test_module_1 {
+  // CHECK-DAG: llvm.mlir.global internal constant @[[$PRINT_GLOBAL0:[A-Za-z0-9_]+]]("Hello, world with location\0A\00") {addr_space = 0 : i32} loc([[LOC]])
+  // CHECK-DAG: llvm.func @vprintf(!llvm.ptr, !llvm.ptr) -> i32 loc([[LOC]])
+
+  gpu.func @test_const_printf_with_loc() {
+    gpu.printf "Hello, world with location\n" loc(fused<#di_subprogram>[#loc])
+    gpu.return
+  }
+}
diff --git a/external/llvm-project/mlir/test/Conversion/GPUToSPIRV/reductions.mlir b/external/llvm-project/mlir/test/Conversion/GPUToSPIRV/reductions.mlir
index ae834b9915d5..e7e0fa296c98 100644
--- a/external/llvm-project/mlir/test/Conversion/GPUToSPIRV/reductions.mlir
+++ b/external/llvm-project/mlir/test/Conversion/GPUToSPIRV/reductions.mlir
@@ -789,3 +789,44 @@ gpu.module @kernels {
   }
 }
 }
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.3, [Kernel, Addresses, Groups, GroupUniformArithmeticKHR, GroupNonUniformClustered], []>, #spirv.resource_limits<>>
+} {
+
+gpu.module @kernels {
+  // CHECK-LABEL:  spirv.func @test_subgroup_reduce_clustered
+  //  CHECK-SAME: (%[[ARG:.*]]: f32)
+  //  CHECK: %[[CLUSTER_SIZE:.*]] = spirv.Constant 8 : i32
+  gpu.func @test_subgroup_reduce_clustered(%arg : f32) kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    // CHECK: %{{.*}} = spirv.GroupNonUniformFAdd <Subgroup> <ClusteredReduce> %[[ARG]] cluster_size(%[[CLUSTER_SIZE]]) : f32, i32 -> f32
+    %reduced = gpu.subgroup_reduce add %arg cluster(size = 8) : (f32) -> (f32)
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+// Subgrop reduce with cluster stride > 1 is not yet supported.
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.3, [Kernel, Addresses, Groups, GroupUniformArithmeticKHR, GroupNonUniformClustered], []>, #spirv.resource_limits<>>
+} {
+
+gpu.module @kernels {
+  gpu.func @test_invalid_subgroup_reduce_clustered_stride(%arg : f32) kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    // expected-error @+1 {{failed to legalize operation 'gpu.subgroup_reduce'}}
+    %reduced = gpu.subgroup_reduce add %arg cluster(size = 8, stride = 2) : (f32) -> (f32)
+    gpu.return
+  }
+}
+
+}
diff --git a/external/llvm-project/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir b/external/llvm-project/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
index 537fb967ef0e..92904082a6f4 100644
--- a/external/llvm-project/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
+++ b/external/llvm-project/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
@@ -19,6 +19,8 @@ func.func @ops(%arg0: f32, %arg1: f32, %arg2: i32, %arg3: i32, %arg4: f64) {
 
 // -----
 
+// CHECK-LABEL: func @absi(
+// CHECK-SAME: i32
 func.func @absi(%arg0: i32) -> i32 {
   // CHECK: = "llvm.intr.abs"(%{{.*}}) <{is_int_min_poison = false}> : (i32) -> i32
   %0 = math.absi %arg0 : i32
@@ -27,6 +29,17 @@ func.func @absi(%arg0: i32) -> i32 {
 
 // -----
 
+// CHECK-LABEL: func @absi_0dvector(
+// CHECK-SAME: vector<i32>
+func.func @absi_0dvector(%arg0 : vector<i32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<i32> to vector<1xi32>
+  // CHECK: "llvm.intr.abs"(%[[CAST]]) <{is_int_min_poison = false}> : (vector<1xi32>) -> vector<1xi32>
+  %0 = math.absi %arg0 : vector<i32>
+  func.return
+}
+
+// -----
+
 // CHECK-LABEL: func @log1p(
 // CHECK-SAME: f32
 func.func @log1p(%arg0 : f32) {
@@ -89,6 +102,19 @@ func.func @log1p_scalable_vector(%arg0 : vector<[4]xf32>) -> vector<[4]xf32> {
 
 // -----
 
+// CHECK-LABEL: func @log1p_0dvector(
+// CHECK-SAME: vector<f32>
+func.func @log1p_0dvector(%arg0 : vector<f32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<f32> to vector<1xf32>
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(dense<1.000000e+00> : vector<1xf32>) : vector<1xf32>
+  // CHECK: %[[ADD:.*]] = llvm.fadd %[[ONE]], %[[CAST]]  : vector<1xf32>
+  // CHECK: %[[LOG:.*]] = llvm.intr.log(%[[ADD]])  : (vector<1xf32>) -> vector<1xf32>
+  %0 = math.log1p %arg0 : vector<f32>
+  func.return
+}
+
+// -----
+
 // CHECK-LABEL: func @expm1(
 // CHECK-SAME: f32
 func.func @expm1(%arg0 : f32) {
@@ -149,6 +175,19 @@ func.func @expm1_vector_fmf(%arg0 : vector<4xf32>) {
 
 // -----
 
+// CHECK-LABEL: func @expm1_0dvector(
+// CHECK-SAME: vector<f32>
+func.func @expm1_0dvector(%arg0 : vector<f32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<f32> to vector<1xf32>
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(dense<1.000000e+00> : vector<1xf32>) : vector<1xf32>
+  // CHECK: %[[EXP:.*]] = llvm.intr.exp(%[[CAST]]) : (vector<1xf32>) -> vector<1xf32>
+  // CHECK: %[[SUB:.*]] = llvm.fsub %[[EXP]], %[[ONE]] : vector<1xf32>
+  %0 = math.expm1 %arg0 : vector<f32>
+  func.return
+}
+
+// -----
+
 // CHECK-LABEL: func @rsqrt(
 // CHECK-SAME: f32
 func.func @rsqrt(%arg0 : f32) {
@@ -161,6 +200,19 @@ func.func @rsqrt(%arg0 : f32) {
 
 // -----
 
+// CHECK-LABEL: func @rsqrt_0dvector(
+// CHECK-SAME: vector<f32>
+func.func @rsqrt_0dvector(%arg0 : vector<f32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<f32> to vector<1xf32>
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(dense<1.000000e+00> : vector<1xf32>) : vector<1xf32>
+  // CHECK: %[[SQRT:.*]] = llvm.intr.sqrt(%[[CAST]]) : (vector<1xf32>) -> vector<1xf32>
+  // CHECK: %[[SUB:.*]] = llvm.fdiv %[[ONE]], %[[SQRT]] : vector<1xf32>
+  %0 = math.rsqrt %arg0 : vector<f32>
+  func.return
+}
+
+// -----
+
 // CHECK-LABEL: func @trigonometrics
 // CHECK-SAME: [[ARG0:%.+]]: f32
 func.func @trigonometrics(%arg0: f32) {
@@ -279,6 +331,15 @@ func.func @ctlz(%arg0 : i32) {
   func.return
 }
 
+// CHECK-LABEL: func @ctlz_0dvector(
+// CHECK-SAME: vector<i32>
+func.func @ctlz_0dvector(%arg0 : vector<i32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<i32> to vector<1xi32>
+  // CHECK: "llvm.intr.ctlz"(%[[CAST]]) <{is_zero_poison = false}> : (vector<1xi32>) -> vector<1xi32>
+  %0 = math.ctlz %arg0 : vector<i32>
+  func.return
+}
+
 // -----
 
 // CHECK-LABEL: func @cttz(
@@ -291,6 +352,17 @@ func.func @cttz(%arg0 : i32) {
 
 // -----
 
+// CHECK-LABEL: func @cttz_0dvector(
+// CHECK-SAME: vector<i32>
+func.func @cttz_0dvector(%arg0 : vector<i32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<i32> to vector<1xi32>
+  // CHECK: "llvm.intr.cttz"(%[[CAST]]) <{is_zero_poison = false}> : (vector<1xi32>) -> vector<1xi32>
+  %0 = math.cttz %arg0 : vector<i32>
+  func.return
+}
+
+// -----
+
 // CHECK-LABEL: func @cttz_vec(
 // CHECK-SAME: i32
 func.func @cttz_vec(%arg0 : vector<4xi32>) {
@@ -351,6 +423,17 @@ func.func @isnan_double(%arg0 : f64) {
 
 // -----
 
+// CHECK-LABEL: func @isnan_0dvector(
+// CHECK-SAME: vector<f32>
+func.func @isnan_0dvector(%arg0 : vector<f32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<f32> to vector<1xf32>
+  // CHECK: "llvm.intr.is.fpclass"(%[[CAST]]) <{bit = 3 : i32}> : (vector<1xf32>) -> vector<1xi1>
+  %0 = math.isnan %arg0 : vector<f32>
+  func.return
+}
+
+// -----
+
 // CHECK-LABEL: func @isfinite_double(
 // CHECK-SAME: f64
 func.func @isfinite_double(%arg0 : f64) {
@@ -361,6 +444,17 @@ func.func @isfinite_double(%arg0 : f64) {
 
 // -----
 
+// CHECK-LABEL: func @isfinite_0dvector(
+// CHECK-SAME: vector<f32>
+func.func @isfinite_0dvector(%arg0 : vector<f32>) {
+  // CHECK: %[[CAST:.+]] = builtin.unrealized_conversion_cast %arg0 : vector<f32> to vector<1xf32>
+  // CHECK: "llvm.intr.is.fpclass"(%[[CAST]]) <{bit = 504 : i32}> : (vector<1xf32>) -> vector<1xi1>
+  %0 = math.isfinite %arg0 : vector<f32>
+  func.return
+}
+
+// -----
+
 // CHECK-LABEL: func @rsqrt_double(
 // CHECK-SAME: f64
 func.func @rsqrt_double(%arg0 : f64) {
diff --git a/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir b/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
index 5c6561c10438..b8e001c9f695 100644
--- a/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
+++ b/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
@@ -46,6 +46,22 @@ func.func @float32_unary_scalar(%arg0: f32) {
   %14 = math.ceil %arg0 : f32
   // CHECK: spirv.GL.Floor %{{.*}}: f32
   %15 = math.floor %arg0 : f32
+  // CHECK: spirv.GL.Tan %{{.*}}: f32
+  %16 = math.tan %arg0 : f32
+  // CHECK: spirv.GL.Asin %{{.*}}: f32
+  %17 = math.asin %arg0 : f32
+  // CHECK: spirv.GL.Acos %{{.*}}: f32
+  %18 = math.acos %arg0 : f32
+  // CHECK: spirv.GL.Sinh %{{.*}}: f32
+  %19 = math.sinh %arg0 : f32
+  // CHECK: spirv.GL.Cosh %{{.*}}: f32
+  %20 = math.cosh %arg0 : f32
+  // CHECK: spirv.GL.Asinh %{{.*}}: f32
+  %21 = math.asinh %arg0 : f32
+  // CHECK: spirv.GL.Acosh %{{.*}}: f32
+  %22 = math.acosh %arg0 : f32
+  // CHECK: spirv.GL.Atanh %{{.*}}: f32
+  %23 = math.atanh %arg0 : f32
   return
 }
 
@@ -85,6 +101,22 @@ func.func @float32_unary_vector(%arg0: vector<3xf32>) {
   %11 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Sin %{{.*}}: vector<3xf32>
   %12 = math.sin %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Tan %{{.*}}: vector<3xf32>
+  %13 = math.tan %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Asin %{{.*}}: vector<3xf32>
+  %14 = math.asin %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Acos %{{.*}}: vector<3xf32>
+  %15 = math.acos %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Sinh %{{.*}}: vector<3xf32>
+  %16 = math.sinh %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Cosh %{{.*}}: vector<3xf32>
+  %17 = math.cosh %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Asinh %{{.*}}: vector<3xf32>
+  %18 = math.asinh %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Acosh %{{.*}}: vector<3xf32>
+  %19 = math.acosh %arg0 : vector<3xf32>
+  // CHECK: spirv.GL.Atanh %{{.*}}: vector<3xf32>
+  %20 = math.atanh %arg0 : vector<3xf32>
   return
 }
 
diff --git a/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir b/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
index 393a910c1fb1..56a0d4dafec8 100644
--- a/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
+++ b/external/llvm-project/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
@@ -48,6 +48,22 @@ func.func @float32_unary_scalar(%arg0: f32) {
   %16 = math.erf %arg0 : f32
   // CHECK: spirv.CL.round %{{.*}}: f32
   %17 = math.round %arg0 : f32
+  // CHECK: spirv.CL.tan %{{.*}}: f32
+  %18 = math.tan %arg0 : f32
+  // CHECK: spirv.CL.asin %{{.*}}: f32
+  %19 = math.asin %arg0 : f32
+  // CHECK: spirv.CL.acos %{{.*}}: f32
+  %20 = math.acos %arg0 : f32
+  // CHECK: spirv.CL.sinh %{{.*}}: f32
+  %21 = math.sinh %arg0 : f32
+  // CHECK: spirv.CL.cosh %{{.*}}: f32
+  %22 = math.cosh %arg0 : f32
+  // CHECK: spirv.CL.asinh %{{.*}}: f32
+  %23 = math.asinh %arg0 : f32
+  // CHECK: spirv.CL.acosh %{{.*}}: f32
+  %24 = math.acosh %arg0 : f32
+  // CHECK: spirv.CL.atanh %{{.*}}: f32
+  %25 = math.atanh %arg0 : f32
   return
 }
 
@@ -87,6 +103,22 @@ func.func @float32_unary_vector(%arg0: vector<3xf32>) {
   %11 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.CL.sin %{{.*}}: vector<3xf32>
   %12 = math.sin %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.tan %{{.*}}: vector<3xf32>
+  %13 = math.tan %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.asin %{{.*}}: vector<3xf32>
+  %14 = math.asin %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.acos %{{.*}}: vector<3xf32>
+  %15 = math.acos %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.sinh %{{.*}}: vector<3xf32>
+  %16 = math.sinh %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.cosh %{{.*}}: vector<3xf32>
+  %17 = math.cosh %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.asinh %{{.*}}: vector<3xf32>
+  %18 = math.asinh %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.acosh %{{.*}}: vector<3xf32>
+  %19 = math.acosh %arg0 : vector<3xf32>
+  // CHECK: spirv.CL.atanh %{{.*}}: vector<3xf32>
+  %20 = math.atanh %arg0 : vector<3xf32>
   return
 }
 
diff --git a/external/llvm-project/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm-with-transforms.mlir b/external/llvm-project/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm-with-transforms.mlir
new file mode 100644
index 000000000000..f6d0524fce39
--- /dev/null
+++ b/external/llvm-project/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm-with-transforms.mlir
@@ -0,0 +1,10 @@
+// RUN: mlir-opt -test-memref-to-llvm-with-transforms %s | FileCheck %s
+
+// Checks that the program does not crash. The functionality of the pattern is
+// already checked in test/Dialect/MemRef/*.mlir
+
+func.func @subview_folder(%arg0: memref<100x100xf32>, %arg1: index, %arg2: index, %arg3: index, %arg4: index) -> memref<?x?xf32, strided<[100, 1], offset: ?>> {
+  %subview = memref.subview %arg0[%arg1, %arg2] [%arg3, %arg4] [1, 1] : memref<100x100xf32> to memref<?x?xf32, strided<[100, 1], offset: ?>>
+  return %subview : memref<?x?xf32, strided<[100, 1], offset: ?>>
+}
+// CHECK-LABEL: llvm.func @subview_folder
diff --git a/external/llvm-project/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/external/llvm-project/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
index 8c863bb2d3d6..acfc18857425 100644
--- a/external/llvm-project/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
+++ b/external/llvm-project/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
@@ -189,7 +189,7 @@ func.func @assume_alignment(%0 : memref<4x4xf16>) {
   // CHECK-NEXT: %[[ALIGN:.*]] = llvm.mlir.constant(16 : index) : i64
   // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[PTR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1
   // CHECK-INTERFACE: llvm.intr.assume
-  memref.assume_alignment %0, 16 : memref<4x4xf16>
+  %1 = memref.assume_alignment %0, 16 : memref<4x4xf16>
   return
 }
 
@@ -205,7 +205,7 @@ func.func @assume_alignment_w_offset(%0 : memref<4x4xf16, strided<[?, ?], offset
   // CHECK-DAG: %[[ALIGN:.*]] = llvm.mlir.constant(16 : index) : i64
   // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[BUFF_ADDR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1
   // CHECK-INTERFACE: llvm.intr.assume
-  memref.assume_alignment %0, 16 : memref<4x4xf16, strided<[?, ?], offset: ?>>
+  %1 = memref.assume_alignment %0, 16 : memref<4x4xf16, strided<[?, ?], offset: ?>>
   return
 }
 // -----
diff --git a/external/llvm-project/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir b/external/llvm-project/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir
index 8906de9db372..d0ddac8cd801 100644
--- a/external/llvm-project/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir
+++ b/external/llvm-project/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir
@@ -1,4 +1,6 @@
-// RUN: mlir-opt --split-input-file --convert-memref-to-spirv="bool-num-bits=8" --cse %s | FileCheck %s
+// RUN: mlir-opt --split-input-file -pass-pipeline="builtin.module(convert-memref-to-spirv{bool-num-bits=8}, cse)" %s | FileCheck %s
+// RUN: mlir-opt --split-input-file -pass-pipeline="builtin.module(convert-memref-to-spirv{bool-num-bits=8 use-64bit-index=true}, cse)" %s \
+// RUN: | FileCheck --check-prefix=CHECK64 %s
 
 // Check that with proper compute and storage extensions, we don't need to
 // perform special tricks.
@@ -420,6 +422,43 @@ func.func @cast_to_static_zero_elems(%arg: memref<?xf32, #spirv.storage_class<Cr
 
 }
 
+// -----
+
+module attributes {
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.5, [Kernel, Int64, Addresses, PhysicalStorageBufferAddresses], []>, #spirv.resource_limits<>>
+} {
+// CHECK-LABEL: func @extract_aligned_pointer_as_index_kernel
+func.func @extract_aligned_pointer_as_index_kernel(%m: memref<?xf32, #spirv.storage_class<CrossWorkgroup>>) -> index {
+  %0 = memref.extract_aligned_pointer_as_index %m: memref<?xf32, #spirv.storage_class<CrossWorkgroup>> -> index
+  // CHECK: %[[I32:.*]] = spirv.ConvertPtrToU {{%.*}} : !spirv.ptr<f32, CrossWorkgroup> to i32
+  // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I32]] : i32 to index
+  // CHECK64: %[[I64:.*]] = spirv.ConvertPtrToU {{%.*}} : !spirv.ptr<f32, CrossWorkgroup> to i64
+  // CHECK64: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index
+
+  // CHECK: return %[[R:.*]] : index
+  return %0: index
+}
+}
+
+// -----
+
+module attributes {
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.5, [Shader, Int64, Addresses, PhysicalStorageBufferAddresses], []>, #spirv.resource_limits<>>
+} {
+// CHECK-LABEL: func @extract_aligned_pointer_as_index_shader
+func.func @extract_aligned_pointer_as_index_shader(%m: memref<?xf32, #spirv.storage_class<CrossWorkgroup>>) -> index {
+  %0 = memref.extract_aligned_pointer_as_index %m: memref<?xf32, #spirv.storage_class<CrossWorkgroup>> -> index
+  // CHECK: %[[I32:.*]] = spirv.ConvertPtrToU {{%.*}} : !spirv.ptr<!spirv.struct<(!spirv.rtarray<f32>)>, CrossWorkgroup> to i32
+  // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I32]] : i32 to index
+  // CHECK64: %[[I64:.*]] = spirv.ConvertPtrToU {{%.*}} : !spirv.ptr<!spirv.struct<(!spirv.rtarray<f32>)>, CrossWorkgroup> to i64
+  // CHECK64: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index
+
+  // CHECK: return %[[R:.*]] : index
+  return %0: index
+}
+}
+
+
 // -----
 
 // Check nontemporal attribute
diff --git a/external/llvm-project/mlir/test/Conversion/TosaToMLProgram/tosa-to-mlprogram.mlir b/external/llvm-project/mlir/test/Conversion/TosaToMLProgram/tosa-to-mlprogram.mlir
index 365b05ff084d..d2092753f1f5 100644
--- a/external/llvm-project/mlir/test/Conversion/TosaToMLProgram/tosa-to-mlprogram.mlir
+++ b/external/llvm-project/mlir/test/Conversion/TosaToMLProgram/tosa-to-mlprogram.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt --tosa-to-mlprogram %s -o -| FileCheck %s
+// RUN: mlir-opt --tosa-to-mlprogram %s -split-input-file -o -| FileCheck %s
 
 module {
   // CHECK: ml_program.global private mutable @var_x(dense<7.000000e+00> : tensor<1xf32>) : tensor<1xf32>
@@ -10,4 +10,18 @@ module {
     %0 = tosa.variable_read @var_x : tensor<1xf32>
     return %0 : tensor<1xf32>
   }
+}
+
+// -----
+
+module {
+  // CHECK: ml_program.global private mutable @var_x : tensor<f32>
+  tosa.variable @var_x : tensor<f32>
+  func.func @test_stateful_ops(%arg0: tensor<f32>) -> (tensor<f32>) {
+    // CHECK: ml_program.global_store @var_x = %arg0 : tensor<f32>
+    tosa.variable_write @var_x, %arg0 : tensor<f32>
+    // CHECK: %[[LOAD:.+]] = ml_program.global_load @var_x : tensor<f32>
+    %0 = tosa.variable_read @var_x : tensor<f32>
+    return %0 : tensor<f32>
+  }
 }
\ No newline at end of file
diff --git a/external/llvm-project/mlir/test/Dialect/AMDGPU/ops.mlir b/external/llvm-project/mlir/test/Dialect/AMDGPU/ops.mlir
index 188cfcc4eb38..6c3ffb575f7c 100644
--- a/external/llvm-project/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -32,6 +32,321 @@ func.func @packed_stoch_round_fp8(%v1: f32, %stoch: i32, %others: vector<4xf8E5M
   func.return %ret : vector<4xf8E5M2FNUZ>
 }
 
+// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f8e4m3_f32(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f8e4m3_f16(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f8e4m3_bf16(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f8e4m3_f32(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f8e4m3_f16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f8e4m3_bf16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f8e4m3_f32(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f8e4m3_f16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f8e4m3_bf16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f8e5m2_f32(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f8e5m2_f16(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f8e5m2_bf16(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f8e5m2_f32(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f8e5m2_f16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f8e5m2_bf16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f8e5m2_f32(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f8e5m2_f16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f8e5m2_bf16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f4e2m1_f32(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f4e2m1_f16(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_full_f4e2m1_bf16(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f4e2m1_f32(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f4e2m1_f16(%v: vector<4xf4E2M1FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf4E2M1FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_half_f4e2m1_bf16(%v: vector<4xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf4E2M1FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_f32
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f4e2m1_f32(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xf32> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xf32>
+  func.return %ret : vector<2xf32>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_f16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f4e2m1_f16(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xf16>
+  func.return %ret : vector<2xf16>
+}
+
+// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_bf16
+// CHECK: amdgpu.scaled_ext_packed
+func.func @scaled_ext_scalar_f4e2m1_bf16(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {
+  %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xbf16>
+  func.return %ret : vector<2xbf16>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f32
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f8e4m3_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f32
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f8e4m3_f32(%v: vector<2xf32>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f8e4m3_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f8e4m3_f16(%v: vector<2xf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_bf16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f8e4m3_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_bf16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f8e4m3_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>
+  func.return %ret : vector<4xf8E4M3FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f32
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f8e5m2_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f32
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f8e5m2_f32(%v: vector<2xf32>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f8e5m2_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f8e5m2_f16(%v: vector<2xf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_bf16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f8e5m2_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_bf16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f8e5m2_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E5M2> into vector<4xf8E5M2>
+  func.return %ret : vector<4xf8E5M2>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f32
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f4e2m1_f32(%v: vector<2xf32>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f32
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f4e2m1_f32(%v: vector<2xf32>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f4e2m1_f16(%v: vector<2xf16>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f4e2m1_f16(%v: vector<2xf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_bf16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_f4e2m1_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
+// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_bf16
+// CHECK: amdgpu.packed_scaled_trunc
+func.func @packed_scaled_trunc_into_f4e2m1_bf16(%v: vector<2xbf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {
+  %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>
+  func.return %ret : vector<8xf4E2M1FN>
+}
+
 // CHECK-LABEL: func @fat_raw_buffer_cast_easy
 // CHECK: amdgpu.fat_raw_buffer_cast
 func.func @fat_raw_buffer_cast_easy(%m: memref<8xi32>) -> memref<8xi32, #amdgpu.address_space<fat_raw_buffer>> {
diff --git a/external/llvm-project/mlir/test/Dialect/Arith/expand-ops.mlir b/external/llvm-project/mlir/test/Dialect/Arith/expand-ops.mlir
index 5b6badf13d76..db1349feaff3 100644
--- a/external/llvm-project/mlir/test/Dialect/Arith/expand-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Arith/expand-ops.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-opt %s -arith-expand="include-bf16=true include-f8e8m0=true" -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -arith-expand="include-bf16=true include-f8e8m0=true" -verify-diagnostics -split-input-file | FileCheck %s 
+// RUN: mlir-opt %s -arith-expand -split-input-file -verify-diagnostics | FileCheck %s --check-prefix=SCHECK
 
 // Test ceil divide with signed integer
 // CHECK-LABEL:       func @ceildivi
@@ -253,7 +254,7 @@ func.func @truncf_f32_to_f8E8M0FNU(%arg0 : f32) -> f8E8M0FNU {
     %0 = arith.truncf %arg0 : f32 to f8E8M0FNU
     return %0 : f8E8M0FNU
 }
-// CHECK-LABLE: @truncf_f32_to_f8E8M0FNU
+// CHECK-LABEL: @truncf_f32_to_f8E8M0FNU
 // CHECK: %[[BITCAST:.+]] = arith.bitcast %arg0 : f32 to i32
 // CHECK: %[[C23_i32:.+]] = arith.constant 23 : i32
 // CHECK: %[[SHRUI:.+]] = arith.shrui %[[BITCAST]], %[[C23_i32]] : i32
@@ -267,7 +268,7 @@ func.func @truncf_f16_to_f8E8M0FNU(%arg0 : f16) -> f8E8M0FNU {
     %0 = arith.truncf %arg0 : f16 to f8E8M0FNU
     return %0 : f8E8M0FNU
 }
-// CHECK-LABLE: @truncf_f16_to_f8E8M0FNU
+// CHECK-LABEL: @truncf_f16_to_f8E8M0FNU
 // CHECK: %[[EXTF:.+]] = arith.extf %arg0 : f16 to f32
 // CHECK: %[[BITCAST:.+]] = arith.bitcast %[[EXTF]] : f32 to i32
 // CHECK: %[[C23_i32:.+]] = arith.constant 23 : i32
@@ -305,9 +306,76 @@ func.func @truncf_vector_bf16_to_f8E8M0FNU(%arg0 : vector<4xbf16>) -> vector<4xf
 
 // CHECK-LABEL: @truncf_vector_bf16_to_f8E8M0FNU
 // CHECK-NOT: arith.truncf
+// CHECK: return
 
+// -----
+
+func.func @scaling_truncf_f32_to_f4E2M1FN(%arg0 : f32, %arg1: f8E8M0FNU) -> f4E2M1FN {
+    %0 = arith.scaling_truncf %arg0, %arg1 : f32, f8E8M0FNU to f4E2M1FN
+    return %0 : f4E2M1FN
+}
+
+// SCHECK-LABEL: @scaling_truncf_f32_to_f4E2M1FN
+// SCHECK: %[[SCALEF32:.+]] = arith.extf %arg1 : f8E8M0FNU to f32
+// SCHECK: %[[DIVF:.+]] = arith.divf %arg0, %[[SCALEF32]] : f32
+// SCHECK: %[[RESULT:.+]] = arith.truncf %[[DIVF]] : f32 to f4E2M1FN
+// SCHECK: return %[[RESULT]]
+
+// -----
+
+func.func @scaling_truncf_vector_f16_to_f6E3M2FN(%arg0 : vector<4xf16>, %arg1: vector<4xf8E8M0FNU>) -> vector<4xf6E3M2FN> {
+    %0 = arith.scaling_truncf %arg0, %arg1 : vector<4xf16>, vector<4xf8E8M0FNU> to vector<4xf6E3M2FN>
+    return %0 : vector<4xf6E3M2FN>
+}
+
+// SCHECK-LABEL: @scaling_truncf_vector_f16_to_f6E3M2FN
+// SCHECK: %[[SCALEF16:.+]] = arith.extf %arg1 : vector<4xf8E8M0FNU> to vector<4xf16>
+// SCHECK: %[[DIVF:.+]] = arith.divf %arg0, %[[SCALEF16]] : vector<4xf16>
+// SCHECK: %[[RESULT:.+]] = arith.truncf %[[DIVF]] : vector<4xf16> to vector<4xf6E3M2FN>
+// SCHECK: return %[[RESULT]] : vector<4xf6E3M2FN>
 
 // -----
+
+func.func @scaling_truncf_propagate_rounding_mode_fast_math(%arg0 : vector<4xf16>, %arg1: vector<4xf16>) -> vector<4xf6E3M2FN> {
+    %0 = arith.scaling_truncf %arg0, %arg1 to_nearest_even fastmath<fast> : vector<4xf16>, vector<4xf16> to vector<4xf6E3M2FN>
+    return %0 : vector<4xf6E3M2FN>
+}
+// SCHECK-LABEL: @scaling_truncf_propagate_rounding_mode_fast_math
+// SCHECK: %[[SCALEF8:.+]] = arith.truncf %arg1 fastmath<fast> : vector<4xf16> to vector<4xf8E8M0FNU>
+// SCHECK: %[[SCALEINTY:.+]] = arith.extf %[[SCALEF8]] fastmath<fast> : vector<4xf8E8M0FNU> to vector<4xf16>
+// SCHECK: %[[DIVF:.+]] = arith.divf %arg0, %[[SCALEINTY]] fastmath<fast> : vector<4xf16>
+// SCHECK: %[[TRUNCF:.+]] = arith.truncf [[_:%[a-zA-Z0-9_]+]] to_nearest_even fastmath<fast> : vector<4xf16> to vector<4xf6E3M2FN>
+// SCHECK: return %[[TRUNCF]] : vector<4xf6E3M2FN>
+
+// -----
+
+func.func @scaling_truncf_f16_to_f4E2M1FN_using_f16_scales(%arg0: f16, %arg1 : f16) -> f4E2M1FN {
+    %0 = arith.scaling_truncf %arg0, %arg1 : f16, f16 to f4E2M1FN
+    return %0 : f4E2M1FN
+}
+// SCHECK-LABEL: @scaling_truncf_f16_to_f4E2M1FN_using_f16_scales
+// SCHECK: %[[SCALETRUNCF:.+]] = arith.truncf %arg1 : f16 to f8E8M0FN
+// SCHECK: return
+
+// -----
+func.func @scaling_truncf_vector_f16_to_f4E2M1FN_using_f16_scales(%arg0: vector<4xf16>, %arg1 : vector<4xf16>) -> vector<4xf4E2M1FN> {
+    %0 = arith.scaling_truncf %arg0, %arg1 : vector<4xf16>, vector<4xf16> to vector<4xf4E2M1FN>
+    return %0 : vector<4xf4E2M1FN>
+}
+// SCHECK-LABEL: @scaling_truncf_vector_f16_to_f4E2M1FN_using_f16_scales
+// SCHECK: %[[SCALETRUNCF:.+]] = arith.truncf %arg1 : vector<4xf16> to vector<4xf8E8M0FNU>
+// SCHECK: return
+
+// -----
+
+func.func @invalid_scaling_truncf_to_f4E2M1FN(%arg0: f16, %arg1 : f8E5M2FNUZ) -> f4E2M1FN {
+    // expected-error@+1 {{failed to legalize operation 'arith.scaling_truncf' that was explicitly marked illegal}}
+    %0 = arith.scaling_truncf %arg0, %arg1 : f16, f8E5M2FNUZ to f4E2M1FN
+    return %0 : f4E2M1FN
+}
+
+// -----
+
 func.func @extf_f8E8M0FNU_to_f32(%arg0 : f8E8M0FNU) -> f32 {
     %0 = arith.extf %arg0 : f8E8M0FNU to f32
     return %0 : f32
@@ -332,7 +400,7 @@ func.func @extf_f8E8M0FNU_to_f16(%arg0 : f8E8M0FNU) -> f16 {
     return %0 : f16
 }
 
-// CHECK-LABLE: @extf_f8E8M0FNU_to_f16
+// CHECK-LABEL: @extf_f8E8M0FNU_to_f16
 // CHECK: %[[BITCAST:.+]] = arith.bitcast %arg0 : f8E8M0FNU to i8
 // CHECK-DAG: %[[CF8NAN:.+]] = arith.constant -1 : i8
 // CHECK-DAG: %[[CF32NAN:.+]] = arith.constant -1 : i32
@@ -374,7 +442,109 @@ func.func @extf_vector_f8E8M0FNU_to_bf16(%arg0 : vector<4xf8E8M0FNU>) -> vector<
 
 // CHECK-LABEL: @extf_vector_f8E8M0FNU_to_bf16
 // CHECK-NOT: arith.extf
+// CHECK: return
+
+// -----
+
+func.func @scaling_extf_to_f32(%arg0: f4E2M1FN, %arg1 : f8E8M0FNU) -> f32 {
+    %0 = arith.scaling_extf %arg0, %arg1 : f4E2M1FN, f8E8M0FNU to f32
+    return %0 : f32 
+}
+
+// SCHECK-LABEL: @scaling_extf_to_f32
+// SCHECK: %[[EXT_SCALE:.+]] = arith.extf %arg1 : f8E8M0FNU to f32
+// SCHECK: %[[EXT_INPUT:.+]] = arith.extf %arg0 : f4E2M1FN to f32
+// SCHECK: %[[RESULT:.+]] = arith.mulf %[[EXT_INPUT]], %[[EXT_SCALE]] : f32
+// SCHECK: return %[[RESULT]]
+
+// -----
+
+func.func @scaling_extf_to_f32_using_f16_scales(%arg0: f4E2M1FN, %arg1 : f16) -> f32 {
+    %0 = arith.scaling_extf %arg0, %arg1 : f4E2M1FN, f16 to f32
+    return %0 : f32 
+}
+
+// SCHECK-LABEL: @scaling_extf_to_f32_using_f16_scales
+// SCHECK: %[[TRUNCF_SCALE:.+]] = arith.truncf %arg1 : f16 to f8E8M0FNU
+// SCHECK: %[[EXT_SCALE:.+]] = arith.extf %[[TRUNCF_SCALE]] : f8E8M0FNU to f32
+// SCHECK: %[[EXT_INPUT:.+]] = arith.extf %arg0 : f4E2M1FN to f32
+// SCHECK: %[[RESULT:.+]] = arith.mulf %[[EXT_INPUT]], %[[EXT_SCALE]] : f32
+// SCHECK: return %[[RESULT]]
+
+// -----
+
+func.func @invalid_scaling_extf_to_f32(%arg0: f4E2M1FN, %arg1 : f8E5M2FNUZ) -> f32 {
+    // expected-error@+1 {{failed to legalize operation 'arith.scaling_extf' that was explicitly marked illegal}}
+    %0 = arith.scaling_extf %arg0, %arg1 : f4E2M1FN, f8E5M2FNUZ to f32
+    return %0 : f32
+}
+
+// -----
+
+func.func @scaling_extf_vector_to_f32(%arg0: vector<4xf4E2M1FN>, %arg1 : vector<4xf8E8M0FNU>) -> vector<4xf32> {
+    %0 = arith.scaling_extf %arg0, %arg1 : vector<4xf4E2M1FN>, vector<4xf8E8M0FNU> to vector<4xf32>
+    return %0 : vector<4xf32>
+}
+
+// SCHECK-LABEL: @scaling_extf_vector_to_f32
+// SCHECK: %[[EXT_SCALE:.+]] = arith.extf %arg1 : vector<4xf8E8M0FNU> to vector<4xf32>
+// SCHECK: %[[EXT_INPUT:.+]] = arith.extf %arg0 : vector<4xf4E2M1FN> to vector<4xf32>
+// SCHECK: %[[RESULT:.+]] = arith.mulf %[[EXT_INPUT]], %[[EXT_SCALE]] : vector<4xf32> 
+// SCHECK: return %[[RESULT]]
+
+// -----
+
+func.func @scaling_extf_vector_to_f16(%arg0: vector<4xf4E2M1FN>, %arg1 : vector<4xf8E8M0FNU>) -> vector<4xf16> {
+    %0 = arith.scaling_extf %arg0, %arg1 : vector<4xf4E2M1FN>, vector<4xf8E8M0FNU> to vector<4xf16>
+    return %0 : vector<4xf16>
+}
+
+// SCHECK-LABEL: @scaling_extf_vector_to_f16
+// SCHECK: %[[EXT_SCALE:.+]] = arith.extf %arg1 : vector<4xf8E8M0FNU> to vector<4xf16>
+// SCHECK: %[[EXT_INPUT:.+]] = arith.extf %arg0 : vector<4xf4E2M1FN> to vector<4xf16>
+// SCHECK: %[[RESULT:.+]] = arith.mulf %[[EXT_INPUT]], %[[EXT_SCALE]] : vector<4xf16> 
+// SCHECK: return %[[RESULT]]
+
+// -----
+
+func.func @scaling_extf_vector_to_bf16(%arg0: vector<4xf4E2M1FN>, %arg1 : vector<4xf8E8M0FNU>) -> vector<4xbf16> {
+    %0 = arith.scaling_extf %arg0, %arg1 : vector<4xf4E2M1FN>, vector<4xf8E8M0FNU> to vector<4xbf16>
+    return %0 : vector<4xbf16>
+}
+
+// SCHECK-LABEL: @scaling_extf_vector_to_bf16
+// SCHECK: %[[EXT_SCALE:.+]] = arith.extf %arg1 : vector<4xf8E8M0FNU> to vector<4xbf16>
+// SCHECK: %[[EXT_INPUT:.+]] = arith.extf %arg0 : vector<4xf4E2M1FN> to vector<4xbf16>
+// SCHECK: %[[RESULT:.+]] = arith.mulf %[[EXT_INPUT]], %[[EXT_SCALE]] : vector<4xbf16> 
+// SCHECK: return %[[RESULT]]
+
+// -----
+
+func.func @scaling_extf_vector_to_f32_using_f16_scales(%arg0: vector<4xf4E2M1FN>, %arg1 : vector<4xf16>) -> vector<4xf32> {
+    %0 = arith.scaling_extf %arg0, %arg1 : vector<4xf4E2M1FN>, vector<4xf16> to vector<4xf32>
+    return %0 : vector<4xf32>
+}
+
+// SCHECK-LABEL: @scaling_extf_vector_to_f32_using_f16_scales
+// SCHECK: %[[TRUNCF_SCALE:.+]] = arith.truncf %arg1 : vector<4xf16> to vector<4xf8E8M0FNU>
+// SCHECK: %[[EXT_SCALE:.+]] = arith.extf %[[TRUNCF_SCALE]] : vector<4xf8E8M0FNU> to vector<4xf32>
+// SCHECK: %[[EXT_INPUT:.+]] = arith.extf %arg0 : vector<4xf4E2M1FN> to vector<4xf32>
+// SCHECK: %[[RESULT:.+]] = arith.mulf %[[EXT_INPUT]], %[[EXT_SCALE]] : vector<4xf32>
+// SCHECK: return %[[RESULT]]
+
+// -----
+
+func.func @scaling_extf_vector_to_f32_using_f16_scales_fastmath(%arg0: vector<4xf4E2M1FN>, %arg1 : vector<4xf16>) -> vector<4xf32> {
+    %0 = arith.scaling_extf %arg0, %arg1 fastmath<fast> : vector<4xf4E2M1FN>, vector<4xf16> to vector<4xf32>
+    return %0 : vector<4xf32>
+}
 
+// SCHECK-LABEL: @scaling_extf_vector_to_f32_using_f16_scales_fastmath
+// SCHECK: %[[TRUNCF_SCALE:.+]] = arith.truncf %arg1 fastmath<fast> : vector<4xf16> to vector<4xf8E8M0FNU>
+// SCHECK: %[[EXT_SCALE:.+]] = arith.extf %[[TRUNCF_SCALE]] fastmath<fast> : vector<4xf8E8M0FNU> to vector<4xf32>
+// SCHECK: %[[EXT_INPUT:.+]] = arith.extf %arg0 fastmath<fast> : vector<4xf4E2M1FN> to vector<4xf32>
+// SCHECK: %[[RESULT:.+]] = arith.mulf %[[EXT_INPUT]], %[[EXT_SCALE]] fastmath<fast> : vector<4xf32>
+// SCHECK: return %[[RESULT]]
 
 // -----
 
diff --git a/external/llvm-project/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir b/external/llvm-project/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
index 297be91e7728..e4f7ea150c85 100644
--- a/external/llvm-project/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
+++ b/external/llvm-project/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-lower-to-arm-neon -verify-diagnostics -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -transform-interpreter  %s | FileCheck %s
 
 // CHECK-LABEL: vector_arm_neon_mixed_types
 // CHECK-SAME:    %[[A0:.*]]: vector<2x8xi8>, %[[A1:.*]]: vector<2x8xi4>, %[[A2:.*]]: vector<2x2xi32>
@@ -354,3 +354,121 @@ func.func @vector_arm_neon_k_unroll_vecmat(%lhs: vector<1x32xi8>, %rhs: vector<2
   %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<1x32xi32>, vector<2x32xi32> into vector<1x2xi32>
   return %res : vector<1x2xi32>
 }
+
+// -----
+
+// Test with more than one iteration across all of the M, N and K dimensions.
+// Shows multiple independent accumulators as well as accumulator reuse.
+// For each iterarion [I, J, K] sub-tiles are extracted from offsets as follows:
+//   LHS: [2*I, 8*K]
+//   RHS: [2*J, 8*K]
+//   ACC: [2*I, 2*J]
+// Sub-tile insert offsets for the result are as like ACC (there are redundant
+// inserts).
+
+// CHECK-LABEL: @vector_arm_neon_mnk_unroll
+// CHECK-SAME:    %[[LHS:arg0]]: vector<4x16xi8>
+// CHECK-SAME:    %[[RHS:arg1]]: vector<4x16xi8>
+// CHECK-SAME:    %[[ACC:arg2]]: vector<4x4xi32>
+// CHECK-SAME:    -> vector<4x4xi32> {
+// CHECK-NEXT:  %cst = arith.constant dense<0> : vector<4x4xi32>
+
+// Iteration: [0, 0, 0]
+// CHECK-NEXT:  %[[VAL_0:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_1:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_2:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_3:[0-9]+]] = vector.shape_cast %[[VAL_0]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_4:[0-9]+]] = vector.shape_cast %[[VAL_1]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_5:[0-9]+]] = vector.shape_cast %[[VAL_2]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-NEXT:  %[[KACC_0_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_5]], %[[VAL_3]], %[[VAL_4]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_7:[0-9]+]] = vector.shape_cast %[[KACC_0_v0]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_8:[0-9]+]] = vector.insert_strided_slice %[[VAL_7]], %cst {offsets = [0, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// Iteration: [0, 0, 1]
+// CHECK-NEXT:  %[[VAL_9:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_10:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_11:[0-9]+]] = vector.shape_cast %[[VAL_9]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_12:[0-9]+]] = vector.shape_cast %[[VAL_10]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[KACC_0_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_0_v0]], %[[VAL_11]], %[[VAL_12]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_14:[0-9]+]] = vector.shape_cast %[[KACC_0_v1]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_15:[0-9]+]] = vector.insert_strided_slice %[[VAL_14]], %[[VAL_8]] {offsets = [0, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// Iteration: [0, 1, 0]
+// CHECK-NEXT:  %[[VAL_16:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_17:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_18:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_19:[0-9]+]] = vector.shape_cast %[[VAL_16]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_20:[0-9]+]] = vector.shape_cast %[[VAL_17]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_21:[0-9]+]] = vector.shape_cast %[[VAL_18]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-NEXT:  %[[KACC_1_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_21]], %[[VAL_19]], %[[VAL_20]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_23:[0-9]+]] = vector.shape_cast %[[KACC_1_v0]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_24:[0-9]+]] = vector.insert_strided_slice %[[VAL_23]], %[[VAL_15]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// Iteration: [0, 1, 1]
+// CHECK-NEXT:  %[[VAL_25:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_26:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_27:[0-9]+]] = vector.shape_cast %[[VAL_25]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_28:[0-9]+]] = vector.shape_cast %[[VAL_26]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[KACC_1_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_1_v0]], %[[VAL_27]], %[[VAL_28]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_30:[0-9]+]] = vector.shape_cast %[[KACC_1_v1]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_31:[0-9]+]] = vector.insert_strided_slice %[[VAL_30]], %[[VAL_24]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// Iteration: [1, 0, 0]
+// CHECK-NEXT:  %[[VAL_32:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_33:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_34:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_35:[0-9]+]] = vector.shape_cast %[[VAL_32]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_36:[0-9]+]] = vector.shape_cast %[[VAL_33]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_37:[0-9]+]] = vector.shape_cast %[[VAL_34]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-NEXT:  %[[KACC_2_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_37]], %[[VAL_35]], %[[VAL_36]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_39:[0-9]+]] = vector.shape_cast %[[KACC_2_v0]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_40:[0-9]+]] = vector.insert_strided_slice %[[VAL_39]], %[[VAL_31]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// Iteration: [1, 0, 1]
+// CHECK-NEXT:  %[[VAL_41:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_42:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_43:[0-9]+]] = vector.shape_cast %[[VAL_41]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_44:[0-9]+]] = vector.shape_cast %[[VAL_42]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[KACC_2_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_2_v0]], %[[VAL_43]], %[[VAL_44]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_46:[0-9]+]] = vector.shape_cast %[[KACC_2_v1]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_47:[0-9]+]] = vector.insert_strided_slice %[[VAL_46]], %[[VAL_40]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// Iteration: [1, 1, 0]
+// CHECK-NEXT:  %[[VAL_48:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_49:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_50:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_51:[0-9]+]] = vector.shape_cast %[[VAL_48]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_52:[0-9]+]] = vector.shape_cast %[[VAL_49]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_53:[0-9]+]] = vector.shape_cast %[[VAL_50]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-NEXT:  %[[KACC_3_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_53]], %[[VAL_51]], %[[VAL_52]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_55:[0-9]+]] = vector.shape_cast %[[KACC_3_v0]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_56:[0-9]+]] = vector.insert_strided_slice %[[VAL_55]], %[[VAL_47]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// Iteration: [1, 1, 1]
+// CHECK-NEXT:  %[[VAL_57:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_58:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
+// CHECK-NEXT:  %[[VAL_59:[0-9]+]] = vector.shape_cast %[[VAL_57]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[VAL_60:[0-9]+]] = vector.shape_cast %[[VAL_58]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-NEXT:  %[[KACC_3_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_3_v0]], %[[VAL_59]], %[[VAL_60]] : vector<16xi8> to vector<4xi32>
+// CHECK-NEXT:  %[[VAL_62:[0-9]+]] = vector.shape_cast %[[KACC_3_v1]] : vector<4xi32> to vector<2x2xi32>
+// CHECK-NEXT:  %[[VAL_63:[0-9]+]] = vector.insert_strided_slice %[[VAL_62]], %[[VAL_56]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
+
+// CHECK-NEXT:  return %[[VAL_63]] : vector<4x4xi32>
+func.func @vector_arm_neon_mnk_unroll(%lhs: vector<4x16xi8>, %rhs: vector<4x16xi8>, %acc : vector<4x4xi32>) -> vector<4x4xi32> {
+  %lhs_extsi = arith.extsi %lhs : vector<4x16xi8> to vector<4x16xi32>
+  %rhs_extsi = arith.extsi %rhs : vector<4x16xi8> to vector<4x16xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<4x16xi32>, vector<4x16xi32> into vector<4x4xi32>
+  return %res : vector<4x4xi32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func {
+      transform.apply_patterns.arm_neon.vector_contract_to_i8mm
+    } : !transform.op<"func.func">
+
+    transform.yield
+  }
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir b/external/llvm-project/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir
index 508535f98cd2..2efb5893c851 100644
--- a/external/llvm-project/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir
@@ -660,13 +660,12 @@ func.func private @maybe_writing_func(%ptr : tensor<*xf32>)
 
 // Test if other callables are left intact and don't cause trouble.
 
-// TODO: enable this test back
-// llvm.func @llvm_func()
+llvm.func @llvm_func()
 
-// func.func @call_llvm_func() {
-//   llvm.call @llvm_func() : () -> ()
-//   return
-// }
+func.func @call_llvm_func() {
+  llvm.call @llvm_func() : () -> ()
+  return
+}
 
 // -----
 
diff --git a/external/llvm-project/mlir/test/Dialect/Bufferization/side-effects.mlir b/external/llvm-project/mlir/test/Dialect/Bufferization/side-effects.mlir
index 841490e9f323..129fc8b32c27 100644
--- a/external/llvm-project/mlir/test/Dialect/Bufferization/side-effects.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Bufferization/side-effects.mlir
@@ -1,9 +1,9 @@
 // RUN: mlir-opt %s --test-side-effects --verify-diagnostics
 
 func.func @test_side_effects(%arg0: memref<2xi32>) -> memref<2xi32> {
-  // expected-remark @below {{found an instance of 'read' on a op operand, on resource '<Default>'}}
-  // expected-remark @below {{found an instance of 'write' on a op result, on resource '<Default>'}}
-  // expected-remark @below {{found an instance of 'allocate' on a op result, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'write' on op result 0, on resource '<Default>'}}
+  // expected-remark @below {{found an instance of 'allocate' on op result 0, on resource '<Default>'}}
   %0 = bufferization.clone %arg0 : memref<2xi32> to memref<2xi32>
   return %0 : memref<2xi32>
 }
diff --git a/external/llvm-project/mlir/test/Dialect/EmitC/ops.mlir b/external/llvm-project/mlir/test/Dialect/EmitC/ops.mlir
index 36d12e763afc..ad40313f95df 100644
--- a/external/llvm-project/mlir/test/Dialect/EmitC/ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/EmitC/ops.mlir
@@ -246,12 +246,20 @@ emitc.verbatim "typedef float f32;"
 // The value is not interpreted as format string if there are no operands.
 emitc.verbatim "{} {  }"
 
-func.func @test_verbatim(%arg0 : !emitc.ptr<i32>, %arg1 : i32) {
+func.func @test_verbatim(%arg0 : !emitc.ptr<i32>, %arg1 : i32, %arg2: !emitc.array<3x!emitc.ptr<i32>>) {
+  %a = "emitc.variable"() <{value = #emitc.opaque<"1">}> : () -> !emitc.lvalue<i32>
+
+  // Check that the lvalue type can be used by verbatim.
+  emitc.verbatim "++{};" args %a : !emitc.lvalue<i32>
+
+  // Check that the array type can be used by verbatim.
+  emitc.verbatim "*{}[0] = 1;" args %arg2 : !emitc.array<3x!emitc.ptr<i32>>
+
   emitc.verbatim "{} + {};" args %arg0, %arg1 : !emitc.ptr<i32>, i32
 
-  // Check there is no ambiguity whether %a is the argument to the emitc.verbatim op.
-  emitc.verbatim "a"
-  %a = "emitc.constant"(){value = 42 : i32} : () -> i32
+  // Check there is no ambiguity whether %b is the argument to the emitc.verbatim op.
+  emitc.verbatim "b"
+  %b = "emitc.constant"(){value = 42 : i32} : () -> i32
 
   return
 }
diff --git a/external/llvm-project/mlir/test/Dialect/LLVMIR/nvvm.mlir b/external/llvm-project/mlir/test/Dialect/LLVMIR/nvvm.mlir
index 77b302155cb1..c7fa41c98ac9 100644
--- a/external/llvm-project/mlir/test/Dialect/LLVMIR/nvvm.mlir
+++ b/external/llvm-project/mlir/test/Dialect/LLVMIR/nvvm.mlir
@@ -587,6 +587,38 @@ func.func @dot_accumulate_4way(%a_vec: vector<4xi8>, %b_vec: vector<4xi8>, %c: i
   return
 }
 
+// CHECK-LABEL: @dot_accumulate_2way
+func.func @dot_accumulate_2way(%a_vec: vector<2xi16>, %b_vec: vector<4xi8>, %c: i32) {
+  // CHECK:   nvvm.dot.accumulate.2way %{{.*}}, %{{.*}}, %{{.*}} {b_hi = false} : vector<2xi16>, vector<4xi8>
+  %1 = nvvm.dot.accumulate.2way %a_vec <unsigned>, %b_vec <unsigned>, %c {b_hi = false}: vector<2xi16>, vector<4xi8>
+  // CHECK:   nvvm.dot.accumulate.2way %{{.*}}, %{{.*}}, %{{.*}} {b_hi = true} : vector<2xi16>, vector<4xi8>
+  %3 = nvvm.dot.accumulate.2way %a_vec <signed>, %b_vec <signed>, %c {b_hi = true}: vector<2xi16>, vector<4xi8>
+  return
+}
+
+// CHECK-LABEL: @prefetch
+func.func @prefetch(%gen_ptr: !llvm.ptr, %local_ptr: !llvm.ptr<5>, %global_ptr: !llvm.ptr<1>) {
+  // CHECK:   nvvm.prefetch level = L1, %{{.*}}
+  nvvm.prefetch level = L1, %gen_ptr : !llvm.ptr<0>
+  // CHECK:   nvvm.prefetch level = L1, %{{.*}}
+  nvvm.prefetch level = L1, %local_ptr : !llvm.ptr<5>
+  // CHECK:   nvvm.prefetch level = L1, %{{.*}}
+  nvvm.prefetch level = L1, %global_ptr : !llvm.ptr<1>
+  // CHECK:   nvvm.prefetch level = L2, %{{.*}}
+  nvvm.prefetch level = L2, %gen_ptr : !llvm.ptr<0>
+  // CHECK:   nvvm.prefetch level = L2, %{{.*}}
+  nvvm.prefetch level = L2, %local_ptr : !llvm.ptr<5>
+  // CHECK:   nvvm.prefetch level = L2, %{{.*}}
+  nvvm.prefetch level = L2, %global_ptr : !llvm.ptr<1>
+  // CHECK:   nvvm.prefetch level = L2, %{{.*}}
+  nvvm.prefetch level = L2, %global_ptr, evict_priority = evict_last : !llvm.ptr<1>
+  // CHECK:   nvvm.prefetch level = L2, %{{.*}}
+  nvvm.prefetch level = L2, %global_ptr, evict_priority = evict_normal : !llvm.ptr<1>
+  // CHECK:   nvvm.prefetch level = L1 uniform, %{{.*}}
+  nvvm.prefetch level = L1 uniform, %gen_ptr : !llvm.ptr
+  return
+}
+
 // -----
 
 // Just check these don't emit errors.
diff --git a/external/llvm-project/mlir/test/Dialect/Linalg/canonicalize.mlir b/external/llvm-project/mlir/test/Dialect/Linalg/canonicalize.mlir
index 3daf221f4402..7284ae7dbd67 100644
--- a/external/llvm-project/mlir/test/Dialect/Linalg/canonicalize.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Linalg/canonicalize.mlir
@@ -495,7 +495,7 @@ func.func @dynamic_fill_pack(%arg0: tensor<?x?xf32>) -> tensor<?x?x16x16xf32> {
 
 // -----
 
-// CHECK: func @fold_self_copy
+// CHECK-LABEL: func @fold_self_copy
 func.func @fold_self_copy(%0 : memref<4x16xf32>) {
 // CHECK-NEXT: return
   linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
@@ -511,6 +511,36 @@ func.func @fold_self_copy(%0 : memref<4x16xf32>) {
 
 // -----
 
+// CHECK-LABEL: func @no_fold_fill_like_memref
+//  CHECK-NEXT:   linalg.generic 
+func.func @no_fold_fill_like_memref(%in_out : memref<4x16xf32>, %fill_val : f32) {
+  linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
+                                   affine_map<(d0, d1) -> (d0, d1)>],
+                  iterator_types = ["parallel", "parallel"]}
+    ins(%in_out : memref<4x16xf32>)
+    outs(%in_out : memref<4x16xf32>) {
+      ^bb0(%arg0: f32, %arg1: f32):
+        linalg.yield %fill_val : f32
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @no_fold_fill_like_tensor
+//  CHECK-NEXT:   linalg.generic 
+func.func @no_fold_fill_like_tensor(%in_out : tensor<4x16xf32>, %fill_val : f32) -> tensor<4x16xf32> {
+  %result = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
+                                   affine_map<(d0, d1) -> (d0, d1)>],
+                  iterator_types = ["parallel", "parallel"]}
+    ins(%in_out : tensor<4x16xf32>)
+    outs(%in_out : tensor<4x16xf32>) {
+      ^bb0(%arg0: f32, %arg1: f32):
+        linalg.yield %fill_val : f32
+  } -> tensor<4x16xf32>
+  return %result : tensor<4x16xf32>
+}
+
 // CHECK-LABEL: func @fold_static_pad_fill
 //       CHECK:   %[[F0:.+]] = arith.constant 0.000000e+00 : f32
 //       CHECK:   %[[INIT:.+]] = tensor.empty() : tensor<412x276xf32>
diff --git a/external/llvm-project/mlir/test/Dialect/Linalg/fusion-elementwise-ops.mlir b/external/llvm-project/mlir/test/Dialect/Linalg/fusion-elementwise-ops.mlir
index 28e1291bce1f..66fc55fadf8f 100644
--- a/external/llvm-project/mlir/test/Dialect/Linalg/fusion-elementwise-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Linalg/fusion-elementwise-ops.mlir
@@ -860,6 +860,43 @@ func.func @fusion_different_axes(%arg0 : tensor<5000xi64>, %arg1 : tensor<5000xi
 
 // -----
 
+func.func @fusion_different_axes_indexed(%arg0: tensor<2x2xi32>) ->  tensor<2xi32> {
+  %0 = tensor.empty() : tensor<2x2xi32>
+  %1 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>],
+        iterator_types = ["parallel", "parallel"]}
+        ins(%arg0 : tensor<2x2xi32>) outs(%0 : tensor<2x2xi32>) {
+          ^bb0(%in: i32, %out: i32):
+            %2 = linalg.index 1 : index
+            %3 = arith.index_cast %2 : index to i32
+            linalg.yield %3 : i32
+        } -> tensor<2x2xi32>
+  %4 = tensor.empty() : tensor<2xi32>
+  %5 = linalg.generic {
+        indexing_maps = [affine_map<(d0) -> (d0, 1)>, affine_map<(d0) -> (d0)>],
+        iterator_types = ["parallel"]}
+        ins(%1 : tensor<2x2xi32>) outs(%4 : tensor<2xi32>) {
+          ^bb0(%in: i32, %out: i32):
+            linalg.yield %in : i32
+        } -> tensor<2xi32>
+  return %5 : tensor<2xi32>
+}
+
+//  CHECK-DAG: #[[MAP:.+]] = affine_map<(d0) -> (d0)>
+//      CHECK: func @fusion_different_axes_indexed(
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<2x2xi32>
+//  CHECK-DAG:   %[[CST:.+]] = arith.constant 1 : i32
+//  CHECK-DAG:   %[[INIT:.+]] = tensor.empty() : tensor<2xi32>
+//      CHECK:   %[[RESULT:.+]] = linalg.generic
+// CHECK-SAME:       indexing_maps = [#[[MAP]]]
+// CHECK-SAME:       outs(%[[INIT]] :
+// CHECK-NEXT:   ^bb0(
+// CHECK-SAME:       %[[B0:.+]]: i32
+//      CHECK:     linalg.yield %[[CST]] : i32
+//      CHECK:   return %[[RESULT]]
+
+// -----
+
 // CHECK-LABEL: func @fold_fill_generic_basic
 //  CHECK-SAME: (%[[ARG0:.*]]: tensor<?xf32>) -> tensor<?xf32> {
 //   CHECK-NOT: linalg.fill
diff --git a/external/llvm-project/mlir/test/Dialect/Linalg/simplify-pack-unpack.mlir b/external/llvm-project/mlir/test/Dialect/Linalg/simplify-pack-unpack.mlir
index 51350e5bc849..6979770154ba 100644
--- a/external/llvm-project/mlir/test/Dialect/Linalg/simplify-pack-unpack.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Linalg/simplify-pack-unpack.mlir
@@ -158,8 +158,8 @@ func.func @unpack_to_partial_slice(%arg0: tensor<8x32xf32>) -> tensor<255xf32> {
 // -----
 
 // CHECK-LABEL: func.func @unpack_dynamic
-// CHECK-NOT:     tensor.collapse
-// CHECK:         linalg.unpack
+// CHECK:     tensor.collapse
+// CHECK-NOT:         linalg.unpack
 func.func @unpack_dynamic(%arg0: tensor<?x32xf32>) -> tensor<?xf32> {
   %c32 = arith.constant 32 : index
   %c0 = arith.constant 0 : index
diff --git a/external/llvm-project/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir b/external/llvm-project/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir
index c10e0ccebfd7..1de861e65300 100644
--- a/external/llvm-project/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir
@@ -61,6 +61,22 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+func.func @conv2d_unsupported_type(%arg0: memref<2x10x10x5xf32>, %arg1: memref<2x3x3x5xf32>, %arg2: memref<2x8x8x2xf32>) {
+  linalg.conv_2d_nhwc_fhwc {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} ins(%arg0, %arg1 : memref<2x10x10x5xf32>, memref<2x3x3x5xf32>) outs(%arg2 : memref<2x8x8x2xf32>)
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_fhwc"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    // expected-error @+1 {{apply Winograd Conv2D failed}}
+    %1 = transform.structured.winograd_conv2d %0 { m = 4, r = 3 } : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// -----
+
 func.func @conv2d(%arg0: tensor<2x?x?x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<1xf32>, %arg3: tensor<2x?x?x2xf32>) -> tensor<2x?x?x2xf32> {
   %0 = linalg.conv_2d_nhwc_fhwc {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} ins(%arg0, %arg1 : tensor<2x?x?x5xf32>, tensor<2x3x3x5xf32>) outs(%arg3 : tensor<2x?x?x2xf32>) -> tensor<2x?x?x2xf32>
   return %0 : tensor<2x?x?x2xf32>
diff --git a/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/insert-slice.mlir b/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/insert-slice.mlir
index ddd4f433b365..0563c21f220e 100644
--- a/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/insert-slice.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/insert-slice.mlir
@@ -20,22 +20,26 @@ func.func private @insert_slice_static_sizes(%source: tensor<?x3x?x1xi32>) -> te
 // CHECK:           %[[INIT:.*]] = tensor.empty() : tensor<5x3xi32>
 // CHECK:           %[[SRC_SLICE:.*]] = tensor.extract_slice %[[SEC]][0, %[[C_2]], 0, 0] [1, 1, 5, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<5x1xi32>
 // CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[C_0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C_0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[C_5:.*]] = arith.constant 5 : index
 // CHECK-DAG:       %[[C_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[MASK:.*]] = vector.create_mask %[[C_5]], %[[C_1]] : vector<8x1xi1>
-// CHECK:           %[[C0:.*]] = arith.constant 0 : index
-// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C0]], %[[C0]]], %[[PAD]] : tensor<5x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
-// CHECK:           %[[C_0:.*]] = arith.constant 0 : index
-// CHECK:           %[[RES:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0]], %[[C_2]]] : vector<8x1xi32>, tensor<5x3xi32> } : vector<8x1xi1> -> tensor<5x3xi32>
+// CHECK:           %[[MASK_READ:.*]] = vector.create_mask %[[C_5]], %[[C_1]] : vector<8x1xi1>
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK_READ]] { vector.transfer_read %[[SRC_SLICE]][%[[C_0]], %[[C_0]]], %[[PAD]] {{.*}} : tensor<5x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK:           %[[C_0_1:.*]] = arith.constant 0 : index
+// CHECK:           %[[C_5_1:.*]] = arith.constant 5 : index
+// CHECK:           %[[C_3:.*]] = arith.constant 3 : index
+// CHECK:           %[[MASK_WRITE:.*]] = vector.create_mask %[[C_5_1]], %[[C_3]] : vector<8x1xi1>
+// CHECK:           %[[RES:.*]] = vector.mask %[[MASK_WRITE]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]]  {in_bounds = [true, true]} : vector<8x1xi32>, tensor<5x3xi32> } : vector<8x1xi1> -> tensor<5x3xi32>
 // CHECK:           return %[[RES]] : tensor<5x3xi32>
 
- module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg0 : (!transform.any_op) -> !transform.any_op
-    transform.structured.vectorize %0 vector_sizes [8, 1] : !transform.any_op
-    transform.yield
-  }
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+   %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+   transform.structured.vectorize %0 vector_sizes [8, 1] : !transform.any_op
+   transform.yield
  }
+}
 
 // -----
 
@@ -59,11 +63,17 @@ func.func private @insert_slice_dynamic_src_dim(%source: tensor<?x3x?x1xi32>, %s
 // CHECK:           %[[SRC_SLICE:.*]] = tensor.extract_slice %[[SRC]][0, %[[C_2]], 0, 0] [1, 1, %[[SIZE]], 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<?x1xi32>
 // CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
 // CHECK-DAG:       %[[C_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[MASK:.*]] = vector.create_mask %[[SIZE]], %[[C_1]] : vector<8x1xi1>
-// CHECK:           %[[C_0:.*]] = arith.constant 0 : index
-// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C_0]], %[[C_0]]], %[[PAD]] : tensor<?x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK-DAG:       %[[C_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C_0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C_0_2:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[D0:.*]] = tensor.dim %[[SRC_SLICE]], %[[C_0_2]] : tensor<?x1xi32>
+// CHECK:           %[[MASK:.*]] = vector.create_mask %[[D0]], %[[C_1]] : vector<8x1xi1>
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C_0_1]], %[[C_0_1]]], %[[PAD]] {{.*}} : tensor<?x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
 // CHECK:           %[[C_0_1:.*]] = arith.constant 0 : index
-// CHECK:           %[[RES:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]] : vector<8x1xi32>, tensor<5x3xi32> } : vector<8x1xi1> -> tensor<5x3xi32>
+// CHECK:           %[[C_5_1:.*]] = arith.constant 5 : index
+// CHECK:           %[[C_3:.*]] = arith.constant 3 : index
+// CHECK:           %[[MASK_WRITE:.*]] = vector.create_mask %[[C_5_1]], %[[C_3]] : vector<8x1xi1>
+// CHECK:           %[[RES:.*]] = vector.mask %[[MASK_WRITE]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]]  {in_bounds = [true, true]} : vector<8x1xi32>, tensor<5x3xi32> } : vector<8x1xi1> -> tensor<5x3xi32>
 // CHECK:           return %[[RES]] : tensor<5x3xi32>
 
  module attributes {transform.with_named_sequence} {
@@ -94,15 +104,20 @@ func.func private @insert_slice_dynamic_dest_dim(%source: tensor<?x3x?x1xi32>, %
 // CHECK:           %[[C_2:.*]] = arith.constant 2 : index
 // CHECK:           %[[INIT:.*]] = tensor.empty(%[[SIZE]]) : tensor<?x3xi32>
 // CHECK:           %[[SRC_SLICE:.*]] = tensor.extract_slice %[[SRC]][0, %[[C_2]], 0, 0] [1, 1, 5, 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<5x1xi32>
-// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
-// CHECK:           %[[C_5:.*]] = arith.constant 5 : index
-// CHECK:           %[[C_1:.*]] = arith.constant 1 : index
-// CHECK:           %[[MASK:.*]] = vector.create_mask %[[C_5]], %[[C_1]] : vector<8x1xi1>
-// CHECK:           %[[C_0:.*]] = arith.constant 0 : index
-// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C_0]], %[[C_0]]], %[[PAD]] : tensor<5x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[C_5:.*]] = arith.constant 5 : index
+// CHECK-DAG:       %[[C_1:.*]] = arith.constant 1 : index
+// CHECK-DAG:       %[[MASK:.*]] = vector.create_mask %[[C_5]], %[[C_1]] : vector<8x1xi1>
+// CHECK-DAG:       %[[C_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C_0_1:.*]] = arith.constant 0 : index
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C_0_1]], %[[C_0_1]]], %[[PAD]] {{.*}} : tensor<5x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
 // CHECK:           %[[C_0_1:.*]] = arith.constant 0 : index
-// CHECK:           %[[WRITE:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]] : vector<8x1xi32>, tensor<?x3xi32> } : vector<8x1xi1> -> tensor<?x3xi32>
-// CHECK:           return %[[WRITE]] : tensor<?x3xi32>
+// CHECK:           %[[C_0_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[DIM:.*]] = tensor.dim %[[INIT]], %[[C_0_2]] : tensor<?x3xi32>
+// CHECK:           %[[C_3:.*]] = arith.constant 3 : index
+// CHECK:           %[[MASK_WRITE:.*]] = vector.create_mask %[[DIM]], %[[C_3]] : vector<8x1xi1>
+// CHECK:           %[[RES:.*]] = vector.mask %[[MASK_WRITE]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]]  {in_bounds = [true, true]} : vector<8x1xi32>, tensor<?x3xi32> } : vector<8x1xi1> -> tensor<?x3xi32>
+// CHECK:           return %[[RES]] : tensor<?x3xi32>
 
  module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
@@ -131,15 +146,21 @@ func.func private @insert_slice_dynamic_source_and_dest_dim(%source: tensor<?x3x
 // CHECK-SAME:      %[[SIZE:.*]]: index) -> tensor<?x3xi32> {
 // CHECK:           %[[C_2:.*]] = arith.constant 2 : index
 // CHECK:           %[[INIT:.*]] = tensor.empty(%[[SIZE]]) : tensor<?x3xi32>
-// CHECK:           %[[SRC_SIZE:.*]] = tensor.extract_slice %[[SRC]][0, %[[C_2]], 0, 0] [1, 1, %[[SIZE]], 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<?x1xi32>
-// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK:           %[[SRC_SLICE:.*]] = tensor.extract_slice %[[SRC]][0, %[[C_2]], 0, 0] [1, 1, %[[SIZE]], 1] [1, 1, 1, 1] : tensor<?x3x?x1xi32> to tensor<?x1xi32>
+// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C0_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[D0:.*]] = tensor.dim %[[SRC_SLICE]], %[[C0_2]] : tensor<?x1xi32>
 // CHECK:           %[[C1:.*]] = arith.constant 1 : index
-// CHECK:           %[[MASK:.*]] = vector.create_mask %[[SIZE]], %[[C1]] : vector<8x1xi1>
-// CHECK:           %[[C0:.*]] = arith.constant 0 : index
-// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SIZE]]{{\[}}%[[C0]], %[[C0]]], %[[PAD]] : tensor<?x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
+// CHECK:           %[[MASK:.*]] = vector.create_mask %[[D0]], %[[C1]] : vector<8x1xi1>
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[SRC_SLICE]][%[[C0_1]], %[[C0_1]]], %[[PAD]] {{.*}} : tensor<?x1xi32>, vector<8x1xi32> } : vector<8x1xi1> -> vector<8x1xi32>
 // CHECK:           %[[C_0_1:.*]] = arith.constant 0 : index
-// CHECK:           %[[WRITE:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[READ]], %[[INIT]]{{\[}}%[[C_0_1]], %[[C_2]]] : vector<8x1xi32>, tensor<?x3xi32> } : vector<8x1xi1> -> tensor<?x3xi32>
-// CHECK:           return %[[WRITE]] : tensor<?x3xi32>
+// CHECK:           %[[C_0_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[DIM:.*]] = tensor.dim %[[INIT]], %[[C_0_2]] : tensor<?x3xi32>
+// CHECK:           %[[C_3:.*]] = arith.constant 3 : index
+// CHECK:           %[[MASK_WRITE:.*]] = vector.create_mask %[[DIM]], %[[C_3]] : vector<8x1xi1>
+// CHECK:           %[[RES:.*]] = vector.mask %[[MASK_WRITE]] { vector.transfer_write %[[READ]], %[[INIT]][%[[C_0_1]], %[[C_2]]]  {in_bounds = [true, true]} : vector<8x1xi32>, tensor<?x3xi32> } : vector<8x1xi1> -> tensor<?x3xi32>
 
  module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
diff --git a/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir b/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
index fa9daad1dcbf..6722de817f6b 100644
--- a/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
@@ -1394,4 +1394,3 @@ func.func @test_vectorize_unpack_no_vector_sizes_permute(%source: tensor<4x7x4xf
     transform.yield
   }
  }
-
diff --git a/external/llvm-project/mlir/test/Dialect/MemRef/canonicalize.mlir b/external/llvm-project/mlir/test/Dialect/MemRef/canonicalize.mlir
index da6faf490653..7a267ae8a2c9 100644
--- a/external/llvm-project/mlir/test/Dialect/MemRef/canonicalize.mlir
+++ b/external/llvm-project/mlir/test/Dialect/MemRef/canonicalize.mlir
@@ -1177,3 +1177,16 @@ func.func @cannot_fold_transpose_cast(%arg0: memref<?x4xf32>) -> memref<?x?xf32,
     // CHECK: return %[[TRANSPOSE]]
     return %transpose : memref<?x?xf32, #transpose_map>
 }
+
+// -----
+
+// CHECK-LABEL: func @fold_assume_alignment_chain
+//  CHECK-SAME:   %[[ARG0:[a-zA-Z0-9]+]]
+func.func @fold_assume_alignment_chain(%0: memref<128xf32>) -> memref<128xf32> {
+  // CHECK: %[[ALIGN:.+]] = memref.assume_alignment %[[ARG0]], 16
+  %1 = memref.assume_alignment %0, 16 : memref<128xf32>
+  // CHECK-NOT: memref.assume_alignment
+  %2 = memref.assume_alignment %1, 16 : memref<128xf32>
+  // CHECK: return %[[ALIGN]]
+  return %2 : memref<128xf32>
+}
diff --git a/external/llvm-project/mlir/test/Dialect/MemRef/emulate-narrow-type.mlir b/external/llvm-project/mlir/test/Dialect/MemRef/emulate-narrow-type.mlir
index 111a02abcc74..3378d329e820 100644
--- a/external/llvm-project/mlir/test/Dialect/MemRef/emulate-narrow-type.mlir
+++ b/external/llvm-project/mlir/test/Dialect/MemRef/emulate-narrow-type.mlir
@@ -63,7 +63,7 @@ func.func @memref_load_i4(%arg0: index) -> i4 {
 
 func.func @memref_load_i4_rank2(%arg0: index, %arg1: index) -> i4 {
     %0 = memref.alloc() : memref<3x125xi4>
-    %align0 =memref.assume_alignment %0, 64 : memref<3x125xi4>
+    %align0 = memref.assume_alignment %0, 64 : memref<3x125xi4>
     %1 = memref.load %align0[%arg0,%arg1] : memref<3x125xi4>
     return %1 : i4
 }
diff --git a/external/llvm-project/mlir/test/Dialect/MemRef/ops.mlir b/external/llvm-project/mlir/test/Dialect/MemRef/ops.mlir
index 38ee363a7d42..13fdf3cf1351 100644
--- a/external/llvm-project/mlir/test/Dialect/MemRef/ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/MemRef/ops.mlir
@@ -283,7 +283,7 @@ func.func @memref_view(%arg0 : index, %arg1 : index, %arg2 : index) {
 // CHECK-LABEL: func @assume_alignment
 // CHECK-SAME: %[[MEMREF:.*]]: memref<4x4xf16>
 func.func @assume_alignment(%0: memref<4x4xf16>) {
-  // CHECK: memref.assume_alignment %[[MEMREF]], 16 : memref<4x4xf16>
+  // CHECK: %{{.*}} = memref.assume_alignment %[[MEMREF]], 16 : memref<4x4xf16>
   %1 = memref.assume_alignment %0, 16 : memref<4x4xf16>
   return
 }
diff --git a/external/llvm-project/mlir/test/Dialect/OpenACC/canonicalize.mlir b/external/llvm-project/mlir/test/Dialect/OpenACC/canonicalize.mlir
index e43a27f6b9e8..fdc8e6b5cae6 100644
--- a/external/llvm-project/mlir/test/Dialect/OpenACC/canonicalize.mlir
+++ b/external/llvm-project/mlir/test/Dialect/OpenACC/canonicalize.mlir
@@ -116,10 +116,10 @@ func.func @testhostdataop(%a: memref<f32>, %ifCond: i1) -> () {
   acc.host_data dataOperands(%0 : memref<f32>) if(%false) {
     acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
       acc.yield
-    } attributes { inclusiveUpperbound = array<i1: true> }
+    } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
     acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
       acc.yield
-    } attributes { inclusiveUpperbound = array<i1: true> }
+    } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
     acc.terminator
   }
   return
diff --git a/external/llvm-project/mlir/test/Dialect/OpenACC/invalid.mlir b/external/llvm-project/mlir/test/Dialect/OpenACC/invalid.mlir
index aadf18927321..8f6e961a0616 100644
--- a/external/llvm-project/mlir/test/Dialect/OpenACC/invalid.mlir
+++ b/external/llvm-project/mlir/test/Dialect/OpenACC/invalid.mlir
@@ -2,7 +2,7 @@
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}}
+// expected-error@+1 {{gang, worker or vector cannot appear with seq}}
 acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
   "test.openacc_dummy_op"() : () -> ()
   acc.yield
@@ -12,7 +12,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}}
+// expected-error@+1 {{gang, worker or vector cannot appear with seq}}
 acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
   "test.openacc_dummy_op"() : () -> ()
   acc.yield
@@ -22,7 +22,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}}
+// expected-error@+1 {{gang, worker or vector cannot appear with seq}}
 acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
   "test.openacc_dummy_op"() : () -> ()
   acc.yield
@@ -32,7 +32,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}}
+// expected-error@+1 {{gang, worker or vector cannot appear with seq}}
 acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
   "test.openacc_dummy_op"() : () -> ()
   acc.yield
@@ -42,7 +42,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}}
+// expected-error@+1 {{gang, worker or vector cannot appear with seq}}
 acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
   "test.openacc_dummy_op"() : () -> ()
   acc.yield
@@ -52,7 +52,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}}
+// expected-error@+1 {{gang, worker or vector cannot appear with seq}}
 acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
   "test.openacc_dummy_op"() : () -> ()
   acc.yield
@@ -62,7 +62,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}}
+// expected-error@+1 {{gang, worker or vector cannot appear with seq}}
 acc.loop {
   "test.openacc_dummy_op"() : () -> ()
   acc.yield
@@ -72,7 +72,7 @@ acc.loop {
 
 // expected-error@+1 {{expected non-empty body.}}
 acc.loop {
-}
+} attributes {independent = [#acc.device_type<none>]}
 
 // -----
 
@@ -99,7 +99,7 @@ acc.loop {
 
 %1 = arith.constant 1 : i32
 %2 = arith.constant 10 : i32
-// expected-error@+1 {{only one of "auto", "independent", "seq" can be present at the same time}}
+// expected-error@+1 {{only one of auto, independent, seq can be present at the same time}}
 acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
   acc.yield
 } attributes {auto_ = [#acc.device_type<none>], seq = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
@@ -168,7 +168,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32){
 // expected-error@+1 {{'acc.init' op cannot be nested in a compute operation}}
   acc.init
   acc.yield
-} attributes {inclusiveUpperbound = array<i1: true>}
+} attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
 // -----
 
@@ -186,7 +186,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
 // expected-error@+1 {{'acc.shutdown' op cannot be nested in a compute operation}}
   acc.shutdown
   acc.yield
-} attributes {inclusiveUpperbound = array<i1: true>}
+} attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
 // -----
 
@@ -198,7 +198,7 @@ acc.loop control(%iv : i32) = (%1 : i32) to (%2 : i32) step (%1 : i32) {
     acc.shutdown
   }) : () -> ()
   acc.yield
-} attributes {inclusiveUpperbound = array<i1: true>}
+} attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
 // -----
 
@@ -797,7 +797,7 @@ func.func @acc_loop_container() {
         scf.yield
     }
     acc.yield
-  } attributes { collapse = [2], collapseDeviceType = [#acc.device_type<none>] }
+  } attributes { collapse = [2], collapseDeviceType = [#acc.device_type<none>], independent = [#acc.device_type<none>]}
   return
 }
 
@@ -816,6 +816,6 @@ func.func @acc_loop_container() {
       scf.yield
     }
     acc.yield
-  } attributes { collapse = [3], collapseDeviceType = [#acc.device_type<none>] }
+  } attributes { collapse = [3], collapseDeviceType = [#acc.device_type<none>], independent = [#acc.device_type<none>]}
   return
 }
diff --git a/external/llvm-project/mlir/test/Dialect/OpenACC/legalize-data.mlir b/external/llvm-project/mlir/test/Dialect/OpenACC/legalize-data.mlir
index 28ef6761a6ef..40604dcc736d 100644
--- a/external/llvm-project/mlir/test/Dialect/OpenACC/legalize-data.mlir
+++ b/external/llvm-project/mlir/test/Dialect/OpenACC/legalize-data.mlir
@@ -96,7 +96,7 @@ func.func @test(%a: memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
       acc.yield
-    }
+    } attributes {independent = [#acc.device_type<none>]}
     acc.yield
   }
   return
@@ -109,7 +109,7 @@ func.func @test(%a: memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[CREATE:.*]][%[[I]]] : memref<10xf32>
 // CHECK:     acc.yield
-// CHECK:   }
+// CHECK:   } attributes {independent = [#acc.device_type<none>]}
 // CHECK:   acc.yield
 // CHECK: }
 
@@ -134,7 +134,7 @@ func.func @test(%a: memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
       acc.yield
-    }
+    } attributes {independent = [#acc.device_type<none>]}
     acc.yield
   }
   return
@@ -147,7 +147,7 @@ func.func @test(%a: memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
 // CHECK:     acc.yield
-// CHECK:   }
+// CHECK:   } attributes {independent = [#acc.device_type<none>]}
 // CHECK:   acc.yield
 // CHECK: }
 
@@ -172,7 +172,7 @@ func.func @test(%a: memref<10xf32>) {
     acc.loop private(@privatization_memref_10_f32 -> %p1 : memref<10xf32>) control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
       acc.yield
-    }
+    } attributes {independent = [#acc.device_type<none>]}
     acc.yield
   }
   return
@@ -185,7 +185,7 @@ func.func @test(%a: memref<10xf32>) {
 // CHECK:   acc.loop private(@privatization_memref_10_f32 -> %[[PRIVATE]] : memref<10xf32>) control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
 // CHECK:     acc.yield
-// CHECK:   }
+// CHECK:   } attributes {independent = [#acc.device_type<none>]}
 // CHECK:   acc.yield
 // CHECK: }
 
@@ -210,7 +210,7 @@ func.func @test(%a: memref<10xf32>) {
     acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
       %ci = memref.load %a[%i] : memref<10xf32>
       acc.yield
-    }
+    } attributes {seq = [#acc.device_type<none>]}
     acc.yield
   }
   return
@@ -223,7 +223,7 @@ func.func @test(%a: memref<10xf32>) {
 // CHECK:   acc.loop control(%[[I:.*]] : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index) {
 // DEVICE:    %{{.*}} = memref.load %[[PRIVATE:.*]][%[[I]]] : memref<10xf32>
 // CHECK:     acc.yield
-// CHECK:   }
+// CHECK:   } attributes {seq = [#acc.device_type<none>]}
 // CHECK:   acc.yield
 // CHECK: }
 
diff --git a/external/llvm-project/mlir/test/Dialect/OpenACC/ops.mlir b/external/llvm-project/mlir/test/Dialect/OpenACC/ops.mlir
index 550f295f074a..97278f869534 100644
--- a/external/llvm-project/mlir/test/Dialect/OpenACC/ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/OpenACC/ops.mlir
@@ -19,7 +19,7 @@ func.func @compute1(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x
       %co = arith.addf %cij, %p : f32
       memref.store %co, %C[%arg3, %arg4] : memref<10x10xf32>
       acc.yield
-    } attributes { collapse = [3], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true, true>}
+    } attributes { collapse = [3], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true, true>, independent = [#acc.device_type<none>]}
     acc.yield
   }
 
@@ -40,7 +40,7 @@ func.func @compute1(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x
 //  CHECK-NEXT:       %{{.*}} = arith.addf %{{.*}}, %{{.*}} : f32
 //  CHECK-NEXT:       memref.store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] : memref<10x10xf32>
 //  CHECK-NEXT:       acc.yield
-//  CHECK-NEXT:     } attributes {collapse = [3], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true, true>}
+//  CHECK-NEXT:     } attributes {collapse = [3], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true, true>, independent = [#acc.device_type<none>]}
 //  CHECK-NEXT:     acc.yield
 //  CHECK-NEXT:   }
 //  CHECK-NEXT:   return %{{.*}} : memref<10x10xf32>
@@ -129,7 +129,7 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
           %tmp = arith.addf %axy, %bxy : f32
           memref.store %tmp, %c[%y] : memref<10xf32>
           acc.yield
-        } attributes {inclusiveUpperbound = array<i1: true>}
+        } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 
         acc.loop control(%i : index) = (%lb : index) to (%c10 : index) step (%st : index) {
           // for i = 0 to 10 step 1
@@ -139,9 +139,9 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
           %z = arith.addf %ci, %dx : f32
           memref.store %z, %d[%x] : memref<10xf32>
           acc.yield
-        } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<nvidia>]}
+        } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>], seq = [#acc.device_type<nvidia>]}
         acc.yield
-      } attributes {inclusiveUpperbound = array<i1: true>}
+      } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
       acc.yield
     }
     acc.terminator
@@ -166,16 +166,16 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x
 // CHECK-NEXT:           %{{.*}} = arith.addf %{{.*}}, %{{.*}} : f32
 // CHECK-NEXT:           memref.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
 // CHECK-NEXT:           acc.yield
-// CHECK-NEXT:         } attributes {inclusiveUpperbound = array<i1: true>}
+// CHECK-NEXT:         } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 // CHECK-NEXT:         acc.loop control(%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) {
 // CHECK-NEXT:           %{{.*}} = memref.load %{{.*}}[%{{.*}}] : memref<10xf32>
 // CHECK-NEXT:           %{{.*}} = memref.load %{{.*}}[%{{.*}}] : memref<10xf32>
 // CHECK-NEXT:           %{{.*}} = arith.addf %{{.*}}, %{{.*}} : f32
 // CHECK-NEXT:           memref.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
 // CHECK-NEXT:           acc.yield
-// CHECK-NEXT:         } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<nvidia>]}
+// CHECK-NEXT:         } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>], seq = [#acc.device_type<nvidia>]}
 // CHECK-NEXT:         acc.yield
-// CHECK-NEXT:       } attributes {inclusiveUpperbound = array<i1: true>}
+// CHECK-NEXT:       } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 // CHECK-NEXT:       acc.yield
 // CHECK-NEXT:     }
 // CHECK-NEXT:     acc.terminator
@@ -196,72 +196,72 @@ func.func @testloopop(%a : memref<10xf32>) -> () {
   acc.loop gang vector worker control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop gang({num=%i64Value: i64}) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop gang({static=%i64Value: i64}) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop worker(%i64Value: i64) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop worker(%i32Value: i32) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop worker(%idxValue: index) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop vector(%i64Value: i64) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop vector(%i32Value: i32) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop vector(%idxValue: index) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop gang({num=%i64Value: i64}) worker vector control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop gang({num=%i64Value: i64, static=%i64Value: i64}) worker(%i64Value: i64) vector(%i64Value: i64) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop gang({num=%i32Value: i32, static=%idxValue: index}) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop tile({%i64Value : i64, %i64Value : i64}) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop tile({%i32Value : i32, %i32Value : i32}) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop gang({static=%i64Value: i64, num=%i64Value: i64}) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   acc.loop gang({dim=%i64Value : i64, static=%i64Value: i64}) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   %b = acc.cache varPtr(%a : memref<10xf32>) varType(tensor<10xf32>) -> memref<10xf32>
   acc.loop cache(%b : memref<10xf32>) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
     "test.openacc_dummy_op"() : () -> ()
     acc.yield
-  } attributes {inclusiveUpperbound = array<i1: true>}
+  } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
   return
 }
 
@@ -271,7 +271,7 @@ func.func @testloopop(%a : memref<10xf32>) -> () {
 // CHECK:      acc.loop
 // CHECK-NEXT:   "test.openacc_dummy_op"() : () -> ()
 // CHECK-NEXT:   acc.yield
-// CHECK-NEXT: attributes {inclusiveUpperbound = array<i1: true>}
+// CHECK-NEXT: attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
 // CHECK:      acc.loop gang({num=[[I64VALUE]] : i64})
 // CHECK-NEXT:   "test.openacc_dummy_op"() : () -> ()
 // CHECK-NEXT:   acc.yield
@@ -343,7 +343,7 @@ func.func @acc_loop_multiple_block() {
       cf.br ^bb1(%22 : index)
     ^bb3:
       acc.yield
-    } attributes {inclusiveUpperbound = array<i1: true>}
+    } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
     acc.yield
   }
   return
@@ -1477,7 +1477,7 @@ func.func @acc_reduc_test(%a : i64) -> () {
   acc.parallel reduction(@reduction_add_i64 -> %a : i64) {
     acc.loop reduction(@reduction_add_i64 -> %a : i64) control(%iv : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
       acc.yield
-    } attributes { inclusiveUpperbound = array<i1: true> }
+    } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
     acc.yield
   }
   return
@@ -1869,21 +1869,21 @@ func.func @acc_combined() {
   acc.parallel combined(loop) {
     acc.loop combined(parallel) control(%arg3 : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
       acc.yield
-    }
+    } attributes {independent = [#acc.device_type<none>]}
     acc.terminator
   }
 
   acc.kernels combined(loop) {
     acc.loop combined(kernels) control(%arg3 : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
       acc.yield
-    }
+    } attributes {auto_ = [#acc.device_type<none>]}
     acc.terminator
   }
 
   acc.serial combined(loop) {
     acc.loop combined(serial) control(%arg3 : index) = (%c0 : index) to (%c10 : index) step (%c1 : index) {
       acc.yield
-    }
+    } attributes {seq = [#acc.device_type<none>]}
     acc.terminator
   }
 
@@ -1949,7 +1949,7 @@ func.func @acc_loop_container() {
       scf.yield
     }
     acc.yield
-  }
+  } attributes {independent = [#acc.device_type<none>]}
   return
 }
 
@@ -1971,7 +1971,7 @@ func.func @acc_loop_container() {
       scf.yield
     }
     acc.yield
-  } attributes { collapse = [2], collapseDeviceType = [#acc.device_type<none>] }
+  } attributes { collapse = [2], collapseDeviceType = [#acc.device_type<none>], independent = [#acc.device_type<none>]}
   return
 }
 
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/arithmetic-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/arithmetic-ops.mlir
index 2d0c86e08de5..d58c27598f2b 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/arithmetic-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/arithmetic-ops.mlir
@@ -12,6 +12,14 @@ func.func @fadd_scalar(%arg: f32) -> f32 {
 
 // -----
 
+func.func @fadd_bf16_scalar(%arg: bf16) -> bf16 {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %0 = spirv.FAdd %arg, %arg : bf16
+  return %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.FDiv
 //===----------------------------------------------------------------------===//
@@ -24,6 +32,14 @@ func.func @fdiv_scalar(%arg: f32) -> f32 {
 
 // -----
 
+func.func @fdiv_bf16_scalar(%arg: bf16) -> bf16 {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %0 = spirv.FDiv %arg, %arg : bf16
+  return %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.FMod
 //===----------------------------------------------------------------------===//
@@ -36,6 +52,14 @@ func.func @fmod_scalar(%arg: f32) -> f32 {
 
 // -----
 
+func.func @fmod_bf16_scalar(%arg: bf16) -> bf16 {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %0 = spirv.FMod %arg, %arg : bf16
+  return %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.FMul
 //===----------------------------------------------------------------------===//
@@ -70,6 +94,14 @@ func.func @fmul_bf16(%arg: bf16) -> bf16 {
 
 // -----
 
+func.func @fmul_bf16_vector(%arg: vector<4xbf16>) -> vector<4xbf16> {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %0 = spirv.FMul %arg, %arg : vector<4xbf16>
+  return %0 : vector<4xbf16>
+}
+
+// -----
+
 func.func @fmul_tensor(%arg: tensor<4xf32>) -> tensor<4xf32> {
   // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
   %0 = spirv.FMul %arg, %arg : tensor<4xf32>
@@ -90,6 +122,14 @@ func.func @fnegate_scalar(%arg: f32) -> f32 {
 
 // -----
 
+func.func @fnegate_bf16_scalar(%arg: bf16) -> bf16 {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %0 = spirv.FNegate %arg : bf16
+  return %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.FRem
 //===----------------------------------------------------------------------===//
@@ -102,6 +142,14 @@ func.func @frem_scalar(%arg: f32) -> f32 {
 
 // -----
 
+func.func @frem_bf16_scalar(%arg: bf16) -> bf16 {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %0 = spirv.FRem %arg, %arg : bf16
+  return %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.FSub
 //===----------------------------------------------------------------------===//
@@ -114,6 +162,14 @@ func.func @fsub_scalar(%arg: f32) -> f32 {
 
 // -----
 
+func.func @fsub_bf16_scalar(%arg: bf16) -> bf16 {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %0 = spirv.FSub %arg, %arg : bf16
+  return %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.IAdd
 //===----------------------------------------------------------------------===//
@@ -489,3 +545,11 @@ func.func @vector_times_scalar(%vector: vector<4xf32>, %scalar: f32) -> vector<3
   %0 = spirv.VectorTimesScalar %vector, %scalar : (vector<4xf32>, f32) -> vector<3xf32>
   return %0 : vector<3xf32>
 }
+
+// -----
+
+func.func @vector_bf16_times_scalar_bf16(%vector: vector<4xbf16>, %scalar: bf16) -> vector<4xbf16> {
+  // expected-error @+1 {{op operand #0 must be vector of 16/32/64-bit float values of length 2/3/4}}
+  %0 = spirv.VectorTimesScalar %vector, %scalar : (vector<4xbf16>, bf16) -> vector<4xbf16>
+  return %0 : vector<4xbf16>
+}
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/atomic-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/atomic-ops.mlir
index cc0abd3a42dc..661497d5fff3 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/atomic-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/atomic-ops.mlir
@@ -272,3 +272,11 @@ func.func @atomic_fadd(%ptr : !spirv.ptr<f32, StorageBuffer>, %value : f32) -> f
   %0 = spirv.EXT.AtomicFAdd <Device> <Acquire|Release> %ptr, %value : !spirv.ptr<f32, StorageBuffer>
   return %0 : f32
 }
+
+// -----
+
+func.func @atomic_bf16_fadd(%ptr : !spirv.ptr<bf16, StorageBuffer>, %value : bf16) -> bf16 {
+  // expected-error @+1 {{op operand #1 must be 16/32/64-bit float, but got 'bf16'}}
+  %0 = spirv.EXT.AtomicFAdd <Device> <None> %ptr, %value : !spirv.ptr<bf16, StorageBuffer>
+  return %0 : bf16
+}
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/cast-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/cast-ops.mlir
index 34d0109e6bb4..4480a1f3720f 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/cast-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/cast-ops.mlir
@@ -110,6 +110,14 @@ func.func @convert_f_to_s_vector(%arg0 : vector<3xf32>) -> vector<3xi32> {
 
 // -----
 
+func.func @convert_bf16_to_s32_scalar(%arg0 : bf16) -> i32 {
+  // CHECK: {{%.*}} = spirv.ConvertFToS {{%.*}} : bf16 to i32
+  %0 = spirv.ConvertFToS %arg0 : bf16 to i32
+  spirv.ReturnValue %0 : i32
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.ConvertFToU
 //===----------------------------------------------------------------------===//
@@ -146,6 +154,14 @@ func.func @convert_f_to_u.coopmatrix(%arg0 : !spirv.coopmatrix<8x16xf32, Subgrou
 
 // -----
 
+func.func @convert_bf16_to_u32_scalar(%arg0 : bf16) -> i32 {
+  // CHECK: {{%.*}} = spirv.ConvertFToU {{%.*}} : bf16 to i32
+  %0 = spirv.ConvertFToU %arg0 : bf16 to i32
+  spirv.ReturnValue %0 : i32
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.ConvertSToF
 //===----------------------------------------------------------------------===//
@@ -174,6 +190,14 @@ func.func @convert_s_to_f_vector(%arg0 : vector<3xi32>) -> vector<3xf32> {
 
 // -----
 
+func.func @convert_s32_to_bf16_scalar(%arg0 : i32) -> bf16 {
+  // CHECK: {{%.*}} = spirv.ConvertSToF {{%.*}} : i32 to bf16
+  %0 = spirv.ConvertSToF %arg0 : i32 to bf16
+  spirv.ReturnValue %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.ConvertUToF
 //===----------------------------------------------------------------------===//
@@ -202,6 +226,14 @@ func.func @convert_u_to_f_vector(%arg0 : vector<3xi32>) -> vector<3xf32> {
 
 // -----
 
+func.func @convert_u32_to_bf16_scalar(%arg0 : i32) -> bf16 {
+  // CHECK: {{%.*}} = spirv.ConvertUToF {{%.*}} : i32 to bf16
+  %0 = spirv.ConvertUToF %arg0 : i32 to bf16
+  spirv.ReturnValue %0 : bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.FConvert
 //===----------------------------------------------------------------------===//
@@ -238,6 +270,30 @@ func.func @f_convert_vector(%arg0 : f32) -> f32 {
 
 // -----
 
+func.func @f_convert_bf16_to_f32_scalar(%arg0 : bf16) -> f32 {
+  // CHECK: {{%.*}} = spirv.FConvert {{%.*}} : bf16 to f32
+  %0 = spirv.FConvert %arg0 : bf16 to f32
+  spirv.ReturnValue %0 : f32
+}
+
+// -----
+
+func.func @f_convert_f32_to_bf16_vector(%arg0 : vector<3xf32>) -> vector<3xbf16> {
+  // CHECK: {{%.*}} = spirv.FConvert {{%.*}} : vector<3xf32> to vector<3xbf16>
+  %0 = spirv.FConvert %arg0 : vector<3xf32> to vector<3xbf16>
+  spirv.ReturnValue %0 : vector<3xbf16>
+}
+
+// -----
+
+func.func @f_convert_f32_to_bf16_coop_matrix(%arg0 : !spirv.coopmatrix<8x16xf32, Subgroup, MatrixA>) -> !spirv.coopmatrix<8x16xbf16, Subgroup, MatrixA> {
+  // CHECK: {{%.*}} = spirv.FConvert {{%.*}} : !spirv.coopmatrix<8x16xf32, Subgroup, MatrixA> to !spirv.coopmatrix<8x16xbf16, Subgroup, MatrixA>
+  %0 = spirv.FConvert %arg0 : !spirv.coopmatrix<8x16xf32, Subgroup, MatrixA> to !spirv.coopmatrix<8x16xbf16, Subgroup, MatrixA>
+  spirv.ReturnValue %0 : !spirv.coopmatrix<8x16xbf16, Subgroup, MatrixA>
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.SConvert
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/composite-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/composite-ops.mlir
index 3fc8dfb2767d..e71b545de11d 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/composite-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/composite-ops.mlir
@@ -11,6 +11,13 @@ func.func @composite_construct_vector(%arg0: f32, %arg1: f32, %arg2 : f32) -> ve
   return %0: vector<3xf32>
 }
 
+// CHECK-LABEL: func @composite_construct_bf16_vector
+func.func @composite_construct_bf16_vector(%arg0: bf16, %arg1: bf16, %arg2 : bf16) -> vector<3xbf16> {
+  // CHECK: spirv.CompositeConstruct {{%.*}}, {{%.*}}, {{%.*}} : (bf16, bf16, bf16) -> vector<3xbf16>
+  %0 = spirv.CompositeConstruct %arg0, %arg1, %arg2 : (bf16, bf16, bf16) -> vector<3xbf16>
+  return %0: vector<3xbf16>
+}
+
 // CHECK-LABEL: func @composite_construct_struct
 func.func @composite_construct_struct(%arg0: vector<3xf32>, %arg1: !spirv.array<4xf32>, %arg2 : !spirv.struct<(f32)>) -> !spirv.struct<(vector<3xf32>, !spirv.array<4xf32>, !spirv.struct<(f32)>)> {
   // CHECK: spirv.CompositeConstruct
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir
index 0be047932c1f..642346cc40b0 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/gl-ops.mlir
@@ -50,6 +50,14 @@ func.func @exp(%arg0 : i32) -> () {
 
 // -----
 
+func.func @exp_bf16(%arg0 : bf16) -> () {
+  // expected-error @+1 {{op operand #0 must be 16/32-bit float or vector of 16/32-bit float values of length 2/3/4}}
+  %2 = spirv.GL.Exp %arg0 : bf16
+  return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.GL.{F|S|U}{Max|Min}
 //===----------------------------------------------------------------------===//
@@ -92,6 +100,15 @@ func.func @iminmax(%arg0: i32, %arg1: i32) {
 
 // -----
 
+func.func @fmaxminbf16vec(%arg0 : vector<3xbf16>, %arg1 : vector<3xbf16>) {
+  // expected-error @+1 {{operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values}}
+  %1 = spirv.GL.FMax %arg0, %arg1 : vector<3xbf16>
+  %2 = spirv.GL.FMin %arg0, %arg1 : vector<3xbf16>
+  return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.GL.InverseSqrt
 //===----------------------------------------------------------------------===//
@@ -254,6 +271,54 @@ func.func @coshvec(%arg0 : vector<3xf16>) -> () {
   return
 }
 
+//===----------------------------------------------------------------------===//
+// spirv.GL.Asinh
+//===----------------------------------------------------------------------===//
+
+func.func @asinh(%arg0 : f32) -> () {
+  // CHECK: spirv.GL.Asinh {{%.*}} : f32
+  %2 = spirv.GL.Asinh %arg0 : f32
+  return
+}
+
+func.func @asinhvec(%arg0 : vector<3xf16>) -> () {
+  // CHECK: spirv.GL.Asinh {{%.*}} : vector<3xf16>
+  %2 = spirv.GL.Asinh %arg0 : vector<3xf16>
+  return
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.Acosh
+//===----------------------------------------------------------------------===//
+
+func.func @acosh(%arg0 : f32) -> () {
+  // CHECK: spirv.GL.Acosh {{%.*}} : f32
+  %2 = spirv.GL.Acosh %arg0 : f32
+  return
+}
+
+func.func @acoshvec(%arg0 : vector<3xf16>) -> () {
+  // CHECK: spirv.GL.Acosh {{%.*}} : vector<3xf16>
+  %2 = spirv.GL.Acosh %arg0 : vector<3xf16>
+  return
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.Atanh
+//===----------------------------------------------------------------------===//
+
+func.func @atanh(%arg0 : f32) -> () {
+  // CHECK: spirv.GL.Atanh {{%.*}} : f32
+  %2 = spirv.GL.Atanh %arg0 : f32
+  return
+}
+
+func.func @atanhvec(%arg0 : vector<3xf16>) -> () {
+  // CHECK: spirv.GL.Atanh {{%.*}} : vector<3xf16>
+  %2 = spirv.GL.Atanh %arg0 : vector<3xf16>
+  return
+}
+
 //===----------------------------------------------------------------------===//
 // spirv.GL.Pow
 //===----------------------------------------------------------------------===//
@@ -519,7 +584,71 @@ func.func @fmix_vector(%arg0 : vector<3xf32>, %arg1 : vector<3xf32>, %arg2 : vec
 // -----
 
 //===----------------------------------------------------------------------===//
-// spirv.GL.Exp
+// spirv.GL.FindILsb
+//===----------------------------------------------------------------------===//
+
+func.func @findimsb_scalar_i32(%arg0 : i32) -> () {
+  // CHECK: spirv.GL.FindILsb {{%.*}} : i32
+  %2 = spirv.GL.FindILsb %arg0 : i32
+  return
+}
+
+func.func @findimsb_vector_i32(%arg0 : vector<3xi32>) -> () {
+  // CHECK: spirv.GL.FindILsb {{%.*}} : vector<3xi32>
+  %2 = spirv.GL.FindILsb %arg0 : vector<3xi32>
+  return
+}
+
+func.func @findimsb_scalar_i16(%arg0 : i16) -> () {
+  // CHECK: spirv.GL.FindILsb {{%.*}} : i16
+  %2 = spirv.GL.FindILsb %arg0 : i16
+  return
+}
+
+func.func @findimsb_vector_i64(%arg0 : vector<3xi64>) -> () {
+  // CHECK: spirv.GL.FindILsb {{%.*}} : vector<3xi64>
+  %2 = spirv.GL.FindILsb %arg0 : vector<3xi64>
+  return
+}
+
+// -----
+
+func.func @findimsb_error_scalar_float(%arg0 : f32) -> () {
+  // expected-error @+1 {{operand #0 must be 8/16/32/64-bit integer or vector of 8/16/32/64-bit integer values of length 2/3/4/8/1}}
+  %2 = spirv.GL.FindILsb %arg0 : f32
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.FindSMsb
+//===----------------------------------------------------------------------===//
+
+func.func @findsmsb_scalar(%arg0 : i32) -> () {
+  // CHECK: spirv.GL.FindSMsb {{%.*}} : i32
+  %2 = spirv.GL.FindSMsb %arg0 : i32
+  return
+}
+
+func.func @findsmsb_vector(%arg0 : vector<3xi32>) -> () {
+  // CHECK: spirv.GL.FindSMsb {{%.*}} : vector<3xi32>
+  %2 = spirv.GL.FindSMsb %arg0 : vector<3xi32>
+  return
+}
+
+// -----
+
+func.func @findsmsb_error_scalar_i64(%arg0 : i64) -> () {
+  // expected-error @+1 {{operand #0 must be Int32 or vector of Int32}}
+  %2 = spirv.GL.FindSMsb %arg0 : i64
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.FindUMsb
 //===----------------------------------------------------------------------===//
 
 func.func @findumsb(%arg0 : i32) -> () {
@@ -689,3 +818,185 @@ func.func @fract_invalid_type(%arg0 : i32) {
   %0 = spirv.GL.Fract %arg0 : i32
   return
 }
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.Log2
+//===----------------------------------------------------------------------===//
+
+func.func @log2(%arg0 : f32) -> () {
+  // CHECK: spirv.GL.Log2 {{%.*}} : f32
+  %0 = spirv.GL.Log2 %arg0 : f32
+  return
+}
+
+func.func @log2vec(%arg0 : vector<3xf16>) -> () {
+  // CHECK: spirv.GL.Log2 {{%.*}} : vector<3xf16>
+  %0 = spirv.GL.Log2 %arg0 : vector<3xf16>
+  return
+}
+
+// -----
+
+func.func @log2_invalid_type(%arg0 : i32) -> () {
+  // expected-error @+1 {{op operand #0 must be 16/32-bit float or vector of 16/32-bit float values}}
+  %0 = spirv.GL.Log2 %arg0 : i32
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.Tanh
+//===----------------------------------------------------------------------===//
+
+func.func @tanh(%arg0 : f32) -> () {
+  // CHECK: spirv.GL.Tanh {{%.*}} : f32
+  %0 = spirv.GL.Tanh %arg0 : f32
+  return
+}
+
+func.func @tanhvec(%arg0 : vector<3xf16>) -> () {
+  // CHECK: spirv.GL.Tanh {{%.*}} : vector<3xf16>
+  %0 = spirv.GL.Tanh %arg0 : vector<3xf16>
+  return
+}
+
+// -----
+
+func.func @tanh_invalid_type(%arg0 : i32) -> () {
+  // expected-error @+1 {{op operand #0 must be 16/32-bit float or vector of 16/32-bit float values}}
+  %0 = spirv.GL.Tanh %arg0 : i32
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.Exp2
+//===----------------------------------------------------------------------===//
+
+func.func @exp2(%arg0 : f32) -> () {
+  // CHECK: spirv.GL.Exp2 {{%.*}} : f32
+  %0 = spirv.GL.Exp2 %arg0 : f32
+  return
+}
+
+func.func @exp2vec(%arg0 : vector<3xf16>) -> () {
+  // CHECK: spirv.GL.Exp2 {{%.*}} : vector<3xf16>
+  %0 = spirv.GL.Exp2 %arg0 : vector<3xf16>
+  return
+}
+
+// -----
+
+func.func @exp2_invalid_type(%arg0 : i32) -> () {
+  // expected-error @+1 {{op operand #0 must be 16/32-bit float or vector of 16/32-bit float values}}
+  %0 = spirv.GL.Exp2 %arg0 : i32
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.PackHalf2x16 
+//===----------------------------------------------------------------------===//
+
+func.func @pack_half_2x16(%arg0 : vector<2xf32>) -> () {
+  // CHECK: spirv.GL.PackHalf2x16 {{%.*}} : vector<2xf32> -> i32
+  %0 = spirv.GL.PackHalf2x16 %arg0 : vector<2xf32> -> i32
+  return
+}
+
+// -----
+
+func.func @pack_half_2x16_i16_output(%arg0 : vector<2xf32>) -> () {
+  // expected-error @+1 {{op result #0 must be Int32, but got 'i16'}}
+  %0 = spirv.GL.PackHalf2x16 %arg0 : vector<2xf32> -> i16
+  return
+}
+
+// -----
+
+func.func @pack_half_2x16_wrong_vec_size(%arg0 : vector<3xf32>) -> () {
+  // expected-error @+1 {{op operand #0 must be vector of Float32 values of length 2, but got 'vector<3xf32>'}}
+  %0 = spirv.GL.PackHalf2x16 %arg0 : vector<3xf32> -> i32
+  return
+}
+
+// -----
+
+func.func @pack_half_2x16_wrong_vec_type(%arg0 : vector<2xi32>) -> () {
+  // expected-error @+1 {{op operand #0 must be vector of Float32 values of length 2, but got 'vector<2xi32>'}}
+  %0 = spirv.GL.PackHalf2x16 %arg0 : vector<2xi32> -> i32
+  return
+}
+
+// -----
+
+func.func @pack_half_2x16_scalar_in(%arg0 : f32) -> () {
+  // expected-error @+1 {{invalid kind of type specified: expected builtin.vector, but found 'f32'}}
+  %0 = spirv.GL.PackHalf2x16 %arg0 : f32 -> i32
+  return
+}
+
+// -----
+
+func.func @unpack_half_2x16_vector_out(%arg0 : vector<2xf32>) -> () {
+  // expected-error @+1 {{invalid kind of type specified: expected builtin.integer, but found 'vector<2xf32>'}}
+  %0 = spirv.GL.UnpackHalf2x16 %arg0 : vector<2xf32> -> vector<2xi32>
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GL.UnpackHalf2x16 
+//===----------------------------------------------------------------------===//
+
+func.func @unpack_half_2x16(%arg0 : i32) -> () {
+  // CHECK: spirv.GL.UnpackHalf2x16 {{%.*}} : i32 -> vector<2xf32>
+  %0 = spirv.GL.UnpackHalf2x16 %arg0 : i32 -> vector<2xf32>
+  return
+}
+
+// -----
+
+func.func @unpack_half_2x16_i16_input(%arg0 : i16) -> () {
+  // expected-error @+1 {{op operand #0 must be Int32, but got 'i16'}}
+  %0 = spirv.GL.UnpackHalf2x16 %arg0 : i16 -> vector<2xf32>
+  return
+}
+
+// -----
+
+func.func @unpack_half_2x16_wrong_vec_size(%arg0 : i32) -> () {
+  // expected-error @+1 {{op result #0 must be vector of Float32 values of length 2, but got 'vector<3xf32>'}}
+  %0 = spirv.GL.UnpackHalf2x16 %arg0 : i32 -> vector<3xf32>
+  return
+}
+
+// -----
+
+func.func @unpack_half_2x16_wrong_vec_type(%arg0 : i32) -> () {
+  // expected-error @+1 {{op result #0 must be vector of Float32 values of length 2, but got 'vector<2xi32>'}}
+  %0 = spirv.GL.UnpackHalf2x16 %arg0 : i32 -> vector<2xi32>
+  return
+}
+
+// -----
+
+func.func @unpack_half_2x16_vec_in(%arg0 : vector<2xf32>) -> () {
+  // expected-error @+1 {{invalid kind of type specified: expected builtin.integer, but found 'vector<2xf32>'}}
+  %0 = spirv.GL.UnpackHalf2x16 %arg0 : vector<2xf32> -> vector<2xf32>
+  return
+}
+
+// -----
+
+func.func @unpack_half_2x16_scalar_out(%arg0 : i32) -> () {
+  // expected-error @+1 {{invalid kind of type specified: expected builtin.vector, but found 'f32'}}
+  %0 = spirv.GL.UnpackHalf2x16 %arg0 : i32 -> f32
+  return
+}
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/image-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/image-ops.mlir
index 1ebdfdb41de1..484a54023edc 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/image-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/image-ops.mlir
@@ -116,6 +116,34 @@ func.func @image_query_size_error_result2(%arg0 : !spirv.image<f32, Buffer, NoDe
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// spirv.ImageRead
+//===----------------------------------------------------------------------===//
+
+func.func @image_read(%arg0: !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, %arg1: vector<2xsi32>) -> () {
+  // CHECK: {{%.*}} = spirv.ImageRead {{%.*}}, {{%.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, vector<2xsi32> -> vector<4xf32>
+  %0 = spirv.ImageRead %arg0, %arg1 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, vector<2xsi32> -> vector<4xf32>
+  spirv.Return
+}
+
+// -----
+
+func.func @image_read_type_mismatch(%arg0: !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, %arg1: vector<2xsi32>) -> () {
+  // expected-error @+1 {{op failed to verify that the result component type must match the image sampled type}}
+  %0 = spirv.ImageRead %arg0, %arg1 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, vector<2xsi32> -> vector<4xf16>
+  spirv.Return
+}
+
+// -----
+
+func.func @image_read_need_sampler(%arg0: !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NeedSampler, Rgba8>, %arg1: vector<2xsi32>) -> () {
+  // expected-error @+1 {{op failed to verify that the sampled operand of the underlying image must be SamplerUnknown or NoSampler}}
+  %0 = spirv.ImageRead %arg0, %arg1 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NeedSampler, Rgba8>, vector<2xsi32> -> vector<4xf16>
+  spirv.Return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.ImageWrite
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
index d3e1dbc229ef..8733ff93768a 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
@@ -524,7 +524,7 @@ spirv.func @matrix_times_scalar(%a: !matA_f32, %b: f32) "None" {
 
 spirv.func @iadd(%a: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>,
                  %b: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixB>) "None" {
-  // expected-error @+1 {{op requires the same type for all operands and results}}
+  // expected-error @+1 {{failed to verify that all of {operand1, operand2, result} have same type}}
   %q = "spirv.IAdd"(%a, %b) :
     (!spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>, !spirv.coopmatrix<2x2xi32, Subgroup, MatrixB>)
     -> !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>
@@ -535,7 +535,7 @@ spirv.func @iadd(%a: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>,
 
 spirv.func @fadd(%a: !spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>,
                  %b: !spirv.coopmatrix<2x2xf32, Subgroup, MatrixAcc>) "None" {
-  // expected-error @+1 {{op requires the same type for all operands and results}}
+  // expected-error @+1 {{failed to verify that all of {operand1, operand2, result} have same type}}
   %q = "spirv.FAdd"(%a, %b) :
     (!spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>, !spirv.coopmatrix<2x2xf32, Subgroup, MatrixAcc>)
     -> !spirv.coopmatrix<2x2xf32, Subgroup, MatrixAcc>
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/logical-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/logical-ops.mlir
index 5c24f0e6a7d3..d6c34645f574 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/logical-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/logical-ops.mlir
@@ -201,6 +201,14 @@ func.func @select_op_float(%arg0: i1) -> () {
   return
 }
 
+func.func @select_op_bfloat16(%arg0: i1) -> () {
+  %0 = spirv.Constant 2.0 : bf16
+  %1 = spirv.Constant 3.0 : bf16
+  // CHECK: spirv.Select {{%.*}}, {{%.*}}, {{%.*}} : i1, bf16
+  %2 = spirv.Select %arg0, %0, %1 : i1, bf16
+  return
+}
+
 func.func @select_op_ptr(%arg0: i1) -> () {
   %0 = spirv.Variable : !spirv.ptr<f32, Function>
   %1 = spirv.Variable : !spirv.ptr<f32, Function>
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir
index 57ff94762ff6..a3b96c698a34 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/memory-ops.mlir
@@ -356,6 +356,16 @@ spirv.module Logical GLSL450 {
 
 // -----
 
+// CHECK-LABEL: @image_load
+func.func @image_load() -> () {
+  %0 = spirv.Variable : !spirv.ptr<!spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, Function>
+  // CHECK: spirv.Load "Function" %{{.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>
+  %1 = spirv.Load "Function" %0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>
+  return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.StoreOp
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
index 5f56de6ad1fa..7ab94f17360d 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
@@ -184,6 +184,14 @@ func.func @group_non_uniform_fmul_clustered_reduce(%val: vector<2xf32>) -> vecto
 
 // -----
 
+func.func @group_non_uniform_bf16_fmul_reduce(%val: bf16) -> bf16 {
+  // expected-error @+1 {{op operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values of length 2/3/4/8/16, but got 'bf16'}}
+  %0 = spirv.GroupNonUniformFMul <Workgroup> <Reduce> %val : bf16 -> bf16
+  return %0: bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.GroupNonUniformFMax
 //===----------------------------------------------------------------------===//
@@ -197,6 +205,14 @@ func.func @group_non_uniform_fmax_reduce(%val: f32) -> f32 {
 
 // -----
 
+func.func @group_non_uniform_bf16_fmax_reduce(%val: bf16) -> bf16 {
+  // expected-error @+1 {{op operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values of length 2/3/4/8/16, but got 'bf16'}}
+  %0 = spirv.GroupNonUniformFMax <Workgroup> <Reduce> %val : bf16 -> bf16
+  return %0: bf16
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.GroupNonUniformFMin
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir
index 5e98b9fdb3c5..207549afdda9 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir
@@ -62,6 +62,10 @@ func.func @const() -> () {
   // CHECK: spirv.Constant dense<1.000000e+00> : tensor<2x3xf32> : !spirv.array<2 x !spirv.array<3 x f32>>
   // CHECK: spirv.Constant dense<{{\[}}[1, 2, 3], [4, 5, 6]]> : tensor<2x3xi32> : !spirv.array<2 x !spirv.array<3 x i32>>
   // CHECK: spirv.Constant dense<{{\[}}[1.000000e+00, 2.000000e+00, 3.000000e+00], [4.000000e+00, 5.000000e+00, 6.000000e+00]]> : tensor<2x3xf32> : !spirv.array<2 x !spirv.array<3 x f32>>
+  // CHECK: spirv.Constant dense<0.000000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+  // CHECK: spirv.Constant dense<4.200000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+  // CHECK: spirv.Constant dense<0> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+  // CHECK: spirv.Constant dense<4> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
 
   %0 = spirv.Constant true
   %1 = spirv.Constant 42 : i32
@@ -73,6 +77,10 @@ func.func @const() -> () {
   %7 = spirv.Constant dense<[[1, 2, 3], [4, 5, 6]]> : tensor<2x3xi32> : !spirv.array<2 x !spirv.array<3 x i32>>
   %8 = spirv.Constant dense<[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]> : tensor<2x3xf32> : !spirv.array<2 x !spirv.array<3 x f32>>
   %9 = spirv.Constant [[dense<3.0> : vector<2xf32>]] : !spirv.array<1 x !spirv.array<1xvector<2xf32>>>
+  %10 = spirv.Constant dense<0.000000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+  %11 = spirv.Constant dense<4.200000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+  %12 = spirv.Constant dense<0> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+  %13 = spirv.Constant dense<4> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
   return
 }
 
@@ -132,6 +140,31 @@ func.func @value_result_num_elements_mismatch() -> () {
 
 // -----
 
+func.func @coop_matrix_const_non_splat() -> () {
+    // expected-error @+1 {{expected a splat dense attribute for cooperative matrix constant, but found}}
+    %0 = spirv.Constant dense<[[1.0, 2.0], [3.0, 4.0]]> : !spirv.coopmatrix<2x2xf32, Subgroup, MatrixAcc>
+    return
+}
+
+// -----
+
+func.func @coop_matrix_const_non_dense() -> () {
+    // expected-error @+2 {{floating point value not valid for specified type}}
+    %0 = spirv.Constant 0.000000e+00 : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+    return
+}
+
+// -----
+
+func.func @coop_matrix_const_wrong_type() -> () {
+    // expected-error @below {{unexpected decimal integer literal for a floating point value}}
+    // expected-note @+1 {{add a trailing dot to make the literal a float}}
+    %0 = spirv.Constant dense<4> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+    return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.EntryPoint
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/types.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/types.mlir
index b63a08d96e6a..c23894c62826 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/IR/types.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/IR/types.mlir
@@ -15,6 +15,9 @@ func.func private @vector_array_type(!spirv.array< 32 x vector<4xf32> >) -> ()
 // CHECK: func private @array_type_stride(!spirv.array<4 x !spirv.array<4 x f32, stride=4>, stride=128>)
 func.func private @array_type_stride(!spirv.array< 4 x !spirv.array<4 x f32, stride=4>, stride = 128>) -> ()
 
+// CHECK: func private @vector_array_type_bf16(!spirv.array<32 x vector<4xbf16>>)
+func.func private @vector_array_type_bf16(!spirv.array<32 x vector<4xbf16> >) -> ()
+
 // -----
 
 // expected-error @+1 {{expected '<'}}
@@ -57,11 +60,6 @@ func.func private @tensor_type(!spirv.array<4xtensor<4xf32>>) -> ()
 
 // -----
 
-// expected-error @+1 {{cannot use 'bf16' to compose SPIR-V types}}
-func.func private @bf16_type(!spirv.array<4xbf16>) -> ()
-
-// -----
-
 // expected-error @+1 {{only 1/8/16/32/64-bit integer type allowed but found 'i256'}}
 func.func private @i256_type(!spirv.array<4xi256>) -> ()
 
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
index 0fd6c18a6c24..722c27586aa6 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
@@ -967,17 +967,17 @@ func.func @umod_fold(%arg0: i32) -> (i32, i32) {
   return %0, %1: i32, i32
 }
 
-// CHECK-LABEL: @umod_fail_vector_fold
+// CHECK-LABEL: @umod_vector_fold
 // CHECK-SAME: (%[[ARG:.*]]: vector<4xi32>)
-func.func @umod_fail_vector_fold(%arg0: vector<4xi32>) -> (vector<4xi32>, vector<4xi32>) {
+func.func @umod_vector_fold(%arg0: vector<4xi32>) -> (vector<4xi32>, vector<4xi32>) {
   // CHECK: %[[CONST4:.*]] = spirv.Constant dense<4> : vector<4xi32>
   // CHECK: %[[CONST32:.*]] = spirv.Constant dense<32> : vector<4xi32>
   %const1 = spirv.Constant dense<32> : vector<4xi32>
   %0 = spirv.UMod %arg0, %const1 : vector<4xi32>
-  // CHECK: %[[UMOD0:.*]] = spirv.UMod %[[ARG]], %[[CONST32]]
   %const2 = spirv.Constant dense<4> : vector<4xi32>
   %1 = spirv.UMod %0, %const2 : vector<4xi32>
-  // CHECK: %[[UMOD1:.*]] = spirv.UMod %[[UMOD0]], %[[CONST4]]
+  // CHECK: %[[UMOD0:.*]] = spirv.UMod %[[ARG]], %[[CONST32]]
+  // CHECK: %[[UMOD1:.*]] = spirv.UMod %[[ARG]], %[[CONST4]]
   // CHECK: return %[[UMOD0]], %[[UMOD1]]
   return %0, %1: vector<4xi32>, vector<4xi32>
 } 
@@ -996,9 +996,9 @@ func.func @umod_fold_same_divisor(%arg0: i32) -> (i32, i32) {
   return %0, %1: i32, i32
 }
 
-// CHECK-LABEL: @umod_fail_fold
+// CHECK-LABEL: @umod_fail_1_fold
 // CHECK-SAME: (%[[ARG:.*]]: i32)
-func.func @umod_fail_fold(%arg0: i32) -> (i32, i32) {
+func.func @umod_fail_1_fold(%arg0: i32) -> (i32, i32) {
   // CHECK: %[[CONST5:.*]] = spirv.Constant 5
   // CHECK: %[[CONST32:.*]] = spirv.Constant 32
   %const1 = spirv.Constant 32 : i32
@@ -1011,6 +1011,51 @@ func.func @umod_fail_fold(%arg0: i32) -> (i32, i32) {
   return %0, %1: i32, i32
 }
 
+// CHECK-LABEL: @umod_fail_2_fold
+// CHECK-SAME: (%[[ARG:.*]]: i32)
+func.func @umod_fail_2_fold(%arg0: i32) -> (i32, i32) {
+  // CHECK: %[[CONST32:.*]] = spirv.Constant 32
+  // CHECK: %[[CONST4:.*]] = spirv.Constant 4
+  %const1 = spirv.Constant 4 : i32
+  %0 = spirv.UMod %arg0, %const1 : i32
+  // CHECK: %[[UMOD0:.*]] = spirv.UMod %[[ARG]], %[[CONST4]]
+  %const2 = spirv.Constant 32 : i32
+  %1 = spirv.UMod %0, %const2 : i32
+  // CHECK: %[[UMOD1:.*]] = spirv.UMod %[[UMOD0]], %[[CONST32]]
+  // CHECK: return %[[UMOD0]], %[[UMOD1]]
+  return %0, %1: i32, i32
+}
+
+// CHECK-LABEL: @umod_vector_fail_1_fold
+// CHECK-SAME: (%[[ARG:.*]]: vector<4xi32>)
+func.func @umod_vector_fail_1_fold(%arg0: vector<4xi32>) -> (vector<4xi32>, vector<4xi32>) {
+  // CHECK: %[[CONST9:.*]] = spirv.Constant dense<9> : vector<4xi32>
+  // CHECK: %[[CONST64:.*]] = spirv.Constant dense<64> : vector<4xi32>
+  %const1 = spirv.Constant dense<64> : vector<4xi32>
+  %0 = spirv.UMod %arg0, %const1 : vector<4xi32>
+  // CHECK: %[[UMOD0:.*]] = spirv.UMod %[[ARG]], %[[CONST64]]
+  %const2 = spirv.Constant dense<9> : vector<4xi32>
+  %1 = spirv.UMod %0, %const2 : vector<4xi32>
+  // CHECK: %[[UMOD1:.*]] = spirv.UMod %[[UMOD0]], %[[CONST9]]
+  // CHECK: return %[[UMOD0]], %[[UMOD1]]
+  return %0, %1: vector<4xi32>, vector<4xi32>
+}
+
+// CHECK-LABEL: @umod_vector_fail_2_fold
+// CHECK-SAME: (%[[ARG:.*]]: vector<4xi32>)
+func.func @umod_vector_fail_2_fold(%arg0: vector<4xi32>) -> (vector<4xi32>, vector<4xi32>) {
+  // CHECK: %[[CONST32:.*]] = spirv.Constant dense<32> : vector<4xi32>
+  // CHECK: %[[CONST4:.*]] = spirv.Constant dense<4> : vector<4xi32>
+  %const1 = spirv.Constant dense<4> : vector<4xi32>
+  %0 = spirv.UMod %arg0, %const1 : vector<4xi32>
+  // CHECK: %[[UMOD0:.*]] = spirv.UMod %[[ARG]], %[[CONST4]]
+  %const2 = spirv.Constant dense<32> : vector<4xi32>
+  %1 = spirv.UMod %0, %const2 : vector<4xi32>
+  // CHECK: %[[UMOD1:.*]] = spirv.UMod %[[UMOD0]], %[[CONST32]]
+  // CHECK: return %[[UMOD0]], %[[UMOD1]]
+  return %0, %1: vector<4xi32>, vector<4xi32>
+}
+
 // -----
 
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/vce-deduction.mlir b/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/vce-deduction.mlir
index ff5ac7cea8fc..2b237665ffc4 100644
--- a/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/vce-deduction.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SPIRV/Transforms/vce-deduction.mlir
@@ -217,3 +217,17 @@ spirv.module Logical GLSL450 attributes {
   spirv.GlobalVariable @data : !spirv.ptr<!spirv.struct<(i8 [0], f16 [2], i64 [4])>, Uniform>
   spirv.GlobalVariable @img  : !spirv.ptr<!spirv.image<f32, Buffer, NoDepth, NonArrayed, SingleSampled, SamplerUnknown, Rg32f>, UniformConstant>
 }
+
+// Using bfloat16 requires BFloat16TypeKHR capability and SPV_KHR_bfloat16 extension.
+// CHECK: requires #spirv.vce<v1.0, [StorageBuffer16BitAccess, Shader, BFloat16TypeKHR], [SPV_KHR_bfloat16, SPV_KHR_16bit_storage, SPV_KHR_storage_buffer_storage_class]>
+spirv.module Logical GLSL450 attributes {
+  spirv.target_env = #spirv.target_env<
+    #spirv.vce<v1.0, [Shader, StorageBuffer16BitAccess, BFloat16TypeKHR], [SPV_KHR_bfloat16, SPV_KHR_16bit_storage, SPV_KHR_storage_buffer_storage_class]>,
+    #spirv.resource_limits<>
+  >
+} {
+  spirv.func @load_bf16(%ptr : !spirv.ptr<bf16, StorageBuffer>) -> bf16 "None" {
+    %val = spirv.Load "StorageBuffer" %ptr : bf16
+    spirv.ReturnValue %val : bf16
+  }
+}
diff --git a/external/llvm-project/mlir/test/Dialect/SparseTensor/minipipeline_vector.mlir b/external/llvm-project/mlir/test/Dialect/SparseTensor/minipipeline_vector.mlir
index 2475aa5139da..b2dfbeb53fde 100755
--- a/external/llvm-project/mlir/test/Dialect/SparseTensor/minipipeline_vector.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SparseTensor/minipipeline_vector.mlir
@@ -22,7 +22,7 @@
 // CHECK-NOVEC:       }
 //
 // CHECK-VEC-LABEL: func.func @sum_reduction
-// CHECK-VEC:       vector.insertelement
+// CHECK-VEC:       vector.insert
 // CHECK-VEC:       scf.for
 // CHECK-VEC:         vector.create_mask
 // CHECK-VEC:         vector.maskedload
diff --git a/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector.mlir b/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector.mlir
index 364ba6e71ff3..64235c722780 100644
--- a/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector.mlir
@@ -241,7 +241,7 @@ func.func @mul_s(%arga: tensor<1024xf32, #SparseVector>,
 // CHECK-VEC16-DAG:   %[[c1024:.*]] = arith.constant 1024 : index
 // CHECK-VEC16-DAG:   %[[v0:.*]] = arith.constant dense<0.000000e+00> : vector<16xf32>
 // CHECK-VEC16:       %[[l:.*]] = memref.load %{{.*}}[] : memref<f32>
-// CHECK-VEC16:       %[[r:.*]] = vector.insertelement %[[l]], %[[v0]][%[[c0]] : index] : vector<16xf32>
+// CHECK-VEC16:       %[[r:.*]] = vector.insert %[[l]], %[[v0]] [0] : f32 into vector<16xf32>
 // CHECK-VEC16:       %[[red:.*]] = scf.for %[[i:.*]] = %[[c0]] to %[[c1024]] step %[[c16]] iter_args(%[[red_in:.*]] = %[[r]]) -> (vector<16xf32>) {
 // CHECK-VEC16:         %[[la:.*]] = vector.load %{{.*}}[%[[i]]] : memref<?xf32>, vector<16xf32>
 // CHECK-VEC16:         %[[lb:.*]] = vector.load %{{.*}}[%[[i]]] : memref<1024xf32>, vector<16xf32>
@@ -258,7 +258,7 @@ func.func @mul_s(%arga: tensor<1024xf32, #SparseVector>,
 // CHECK-VEC16-IDX32-DAG:   %[[c1024:.*]] = arith.constant 1024 : index
 // CHECK-VEC16-IDX32-DAG:   %[[v0:.*]] = arith.constant dense<0.000000e+00> : vector<16xf32>
 // CHECK-VEC16-IDX32:       %[[l:.*]] = memref.load %{{.*}}[] : memref<f32>
-// CHECK-VEC16-IDX32:       %[[r:.*]] = vector.insertelement %[[l]], %[[v0]][%[[c0]] : index] : vector<16xf32>
+// CHECK-VEC16-IDX32:       %[[r:.*]] = vector.insert %[[l]], %[[v0]] [0] : f32 into vector<16xf32>
 // CHECK-VEC16-IDX32:       %[[red:.*]] = scf.for %[[i:.*]] = %[[c0]] to %[[c1024]] step %[[c16]] iter_args(%[[red_in:.*]] = %[[r]]) -> (vector<16xf32>) {
 // CHECK-VEC16-IDX32:         %[[la:.*]] = vector.load %{{.*}}[%[[i]]] : memref<?xf32>, vector<16xf32>
 // CHECK-VEC16-IDX32:         %[[lb:.*]] = vector.load %{{.*}}[%[[i]]] : memref<1024xf32>, vector<16xf32>
@@ -278,7 +278,7 @@ func.func @mul_s(%arga: tensor<1024xf32, #SparseVector>,
 // CHECK-VEC4-SVE:       %[[l:.*]] = memref.load %{{.*}}[] : memref<f32>
 // CHECK-VEC4-SVE:       %[[vscale:.*]] = vector.vscale
 // CHECK-VEC4-SVE:       %[[step:.*]] = arith.muli %[[vscale]], %[[c4]] : index
-// CHECK-VEC4-SVE:       %[[r:.*]] = vector.insertelement %[[l]], %[[v0]][%[[c0]] : index] : vector<[4]xf32>
+// CHECK-VEC4-SVE:       %[[r:.*]] = vector.insert %[[l]], %[[v0]] [0] : f32 into vector<[4]xf32>
 // CHECK-VEC4-SVE:       %[[red:.*]] = scf.for %[[i:.*]] = %[[c0]] to %[[c1024]] step %[[step]] iter_args(%[[red_in:.*]] = %[[r]]) -> (vector<[4]xf32>) {
 // CHECK-VEC4-SVE:         %[[sub:.*]] = affine.min #[[$map]](%[[c1024]], %[[i]])[%[[step]]]
 // CHECK-VEC4-SVE:         %[[mask:.*]] = vector.create_mask %[[sub]] : vector<[4]xi1>
diff --git a/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector_chain.mlir b/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector_chain.mlir
index f4b565c7f9c8..0ab72897d7bc 100644
--- a/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector_chain.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SparseTensor/sparse_vector_chain.mlir
@@ -82,7 +82,7 @@
 // CHECK:               %[[VAL_57:.*]] = arith.select %[[VAL_39]], %[[VAL_56]], %[[VAL_32]] : index
 // CHECK:               scf.yield %[[VAL_55]], %[[VAL_57]], %[[VAL_58:.*]] : index, index, f64
 // CHECK:             } attributes {"Emitted from" = "linalg.generic"}
-// CHECK:             %[[VAL_59:.*]] = vector.insertelement %[[VAL_60:.*]]#2, %[[VAL_4]]{{\[}}%[[VAL_6]] : index] : vector<8xf64>
+// CHECK:             %[[VAL_59:.*]] = vector.insert %[[VAL_60:.*]]#2, %[[VAL_4]] [0] : f64 into vector<8xf64>
 // CHECK:             %[[VAL_61:.*]] = scf.for %[[VAL_62:.*]] = %[[VAL_60]]#0 to %[[VAL_21]] step %[[VAL_3]] iter_args(%[[VAL_63:.*]] = %[[VAL_59]]) -> (vector<8xf64>) {
 // CHECK:               %[[VAL_64:.*]] = affine.min #map(%[[VAL_21]], %[[VAL_62]]){{\[}}%[[VAL_3]]]
 // CHECK:               %[[VAL_65:.*]] = vector.create_mask %[[VAL_64]] : vector<8xi1>
diff --git a/external/llvm-project/mlir/test/Dialect/SparseTensor/vectorize_reduction.mlir b/external/llvm-project/mlir/test/Dialect/SparseTensor/vectorize_reduction.mlir
index 01b717090e87..6effbbf98abb 100644
--- a/external/llvm-project/mlir/test/Dialect/SparseTensor/vectorize_reduction.mlir
+++ b/external/llvm-project/mlir/test/Dialect/SparseTensor/vectorize_reduction.mlir
@@ -172,7 +172,7 @@ func.func @sparse_reduction_ori_accumulator_on_rhs(%argx: tensor<i13>,
 // CHECK-ON:           %[[VAL_9:.*]] = memref.load %[[VAL_8]][] : memref<i32>
 // CHECK-ON:           %[[VAL_10:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_3]]] : memref<?xindex>
 // CHECK-ON:           %[[VAL_11:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_5]]] : memref<?xindex>
-// CHECK-ON:           %[[VAL_12:.*]] = vector.insertelement %[[VAL_9]], %[[VAL_4]]{{\[}}%[[VAL_3]] : index] : vector<8xi32>
+// CHECK-ON:           %[[VAL_12:.*]] = vector.insert %[[VAL_9]], %[[VAL_4]] [0] : i32 into vector<8xi32>
 // CHECK-ON:           %[[VAL_13:.*]] = scf.for %[[VAL_14:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_2]] iter_args(%[[VAL_15:.*]] = %[[VAL_12]]) -> (vector<8xi32>) {
 // CHECK-ON:             %[[VAL_16:.*]] = affine.min #map(%[[VAL_11]], %[[VAL_14]]){{\[}}%[[VAL_2]]]
 // CHECK-ON:             %[[VAL_17:.*]] = vector.create_mask %[[VAL_16]] : vector<8xi1>
@@ -247,7 +247,7 @@ func.func @sparse_reduction_subi(%argx: tensor<i32>,
 // CHECK-ON:  %[[VAL_9:.*]] = memref.load %[[VAL_8]][] : memref<i32>
 // CHECK-ON:  %[[VAL_10:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<?xindex>
 // CHECK-ON:  %[[VAL_11:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_5]]] : memref<?xindex>
-// CHECK-ON:  %[[VAL_12:.*]] = vector.insertelement %[[VAL_9]], %[[VAL_3]]{{\[}}%[[VAL_4]] : index] : vector<8xi32>
+// CHECK-ON:  %[[VAL_12:.*]] = vector.insert %[[VAL_9]], %[[VAL_3]] [0] : i32 into vector<8xi32>
 // CHECK-ON:  %[[VAL_13:.*]] = scf.for %[[VAL_14:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_2]] iter_args(%[[VAL_15:.*]] = %[[VAL_12]]) -> (vector<8xi32>) {
 // CHECK-ON:    %[[VAL_16:.*]] = affine.min #map(%[[VAL_11]], %[[VAL_14]]){{\[}}%[[VAL_2]]]
 // CHECK-ON:    %[[VAL_17:.*]] = vector.create_mask %[[VAL_16]] : vector<8xi1>
@@ -323,7 +323,7 @@ func.func @sparse_reduction_xor(%argx: tensor<i32>,
 // CHECK-ON:   %[[VAL_9:.*]] = memref.load %[[VAL_8]][] : memref<i32>
 // CHECK-ON:   %[[VAL_10:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<?xindex>
 // CHECK-ON:   %[[VAL_11:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_5]]] : memref<?xindex>
-// CHECK-ON:   %[[VAL_12:.*]] = vector.insertelement %[[VAL_9]], %[[VAL_3]]{{\[}}%[[VAL_4]] : index] : vector<8xi32>
+// CHECK-ON:   %[[VAL_12:.*]] = vector.insert %[[VAL_9]], %[[VAL_3]] [0] : i32 into vector<8xi32>
 // CHECK-ON:   %[[VAL_13:.*]] = scf.for %[[VAL_14:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_2]] iter_args(%[[VAL_15:.*]] = %[[VAL_12]]) -> (vector<8xi32>) {
 // CHECK-ON:     %[[VAL_16:.*]] = affine.min #map(%[[VAL_11]], %[[VAL_14]]){{\[}}%[[VAL_2]]]
 // CHECK-ON:     %[[VAL_17:.*]] = vector.create_mask %[[VAL_16]] : vector<8xi1>
@@ -399,7 +399,7 @@ func.func @sparse_reduction_addi(%argx: tensor<i32>,
 // CHECK-ON:   %[[VAL_9:.*]] = memref.load %[[VAL_8]][] : memref<f32>
 // CHECK-ON:   %[[VAL_10:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<?xindex>
 // CHECK-ON:   %[[VAL_11:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_5]]] : memref<?xindex>
-// CHECK-ON:   %[[VAL_12:.*]] = vector.insertelement %[[VAL_9]], %[[VAL_3]]{{\[}}%[[VAL_4]] : index] : vector<8xf32>
+// CHECK-ON:   %[[VAL_12:.*]] = vector.insert %[[VAL_9]], %[[VAL_3]] [0] : f32 into vector<8xf32>
 // CHECK-ON:   %[[VAL_13:.*]] = scf.for %[[VAL_14:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_2]] iter_args(%[[VAL_15:.*]] = %[[VAL_12]]) -> (vector<8xf32>) {
 // CHECK-ON:     %[[VAL_16:.*]] = affine.min #map(%[[VAL_11]], %[[VAL_14]]){{\[}}%[[VAL_2]]]
 // CHECK-ON:     %[[VAL_17:.*]] = vector.create_mask %[[VAL_16]] : vector<8xi1>
@@ -475,7 +475,7 @@ func.func @sparse_reduction_subf(%argx: tensor<f32>,
 // CHECK-ON:   %[[VAL_9:.*]] = memref.load %[[VAL_8]][] : memref<f32>
 // CHECK-ON:   %[[VAL_10:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<?xindex>
 // CHECK-ON:   %[[VAL_11:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_5]]] : memref<?xindex>
-// CHECK-ON:   %[[VAL_12:.*]] = vector.insertelement %[[VAL_9]], %[[VAL_3]]{{\[}}%[[VAL_4]] : index] : vector<8xf32>
+// CHECK-ON:   %[[VAL_12:.*]] = vector.insert %[[VAL_9]], %[[VAL_3]] [0] : f32 into vector<8xf32>
 // CHECK-ON:   %[[VAL_13:.*]] = scf.for %[[VAL_14:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_2]] iter_args(%[[VAL_15:.*]] = %[[VAL_12]]) -> (vector<8xf32>) {
 // CHECK-ON:     %[[VAL_16:.*]] = affine.min #map(%[[VAL_11]], %[[VAL_14]]){{\[}}%[[VAL_2]]]
 // CHECK-ON:     %[[VAL_17:.*]] = vector.create_mask %[[VAL_16]] : vector<8xi1>
diff --git a/external/llvm-project/mlir/test/Dialect/Tensor/canonicalize.mlir b/external/llvm-project/mlir/test/Dialect/Tensor/canonicalize.mlir
index cdcd7f305d2d..3251c5a4a2bf 100644
--- a/external/llvm-project/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -163,7 +163,7 @@ func.func @infer_concat_return_type(%arg0: tensor<5x12xi32>, %arg1: tensor<?x12x
 // -----
 
 // CHECK-LABEL: func @fold_extract
-func.func @fold_extract(%arg0 : index) -> (f32, f16, f16, i32, complex<f32>) {
+func.func @fold_extract(%arg0 : index) -> (f32, f16, f16, i32, complex<f32>, i32) {
   %const_0 = arith.constant 0 : index
   %const_1 = arith.constant 1 : index
   %const_3 = arith.constant 3 : index
@@ -193,8 +193,15 @@ func.func @fold_extract(%arg0 : index) -> (f32, f16, f16, i32, complex<f32>) {
   %4 = arith.constant dense<(1.2, 2.3)> : tensor<complex<f32>>
   %ext_5 = tensor.extract %4[] : tensor<complex<f32>>
 
-  // CHECK-NEXT: return [[C4]], [[CM2]], [[C0]], [[C64]], [[C5]]
-  return %ext_1, %ext_2, %ext_3, %ext_4, %ext_5 : f32, f16, f16, i32, complex<f32>
+  // Fold an extract after an insert.
+  // CHECK-DAG: [[C6:%.+]] = arith.constant 4 : i32
+  %c4_i32 = arith.constant 4 : i32
+  %5 = arith.constant dense<[[1, 3], [0, 2]]> : tensor<2x2xi32>
+  %inserted = tensor.insert %c4_i32 into %5[%const_1, %const_0] : tensor<2x2xi32>
+  %ext_6 = tensor.extract %inserted[%const_1, %const_0] : tensor<2x2xi32>
+
+  // CHECK-NEXT: return [[C4]], [[CM2]], [[C0]], [[C64]], [[C5]], [[C6]]
+  return %ext_1, %ext_2, %ext_3, %ext_4, %ext_5, %ext_6 : f32, f16, f16, i32, complex<f32>, i32
 }
 
 // -----
@@ -1094,7 +1101,7 @@ func.func @fold_expand_of_collapse(%arg0 : tensor<3x4x4xf32>) -> tensor<3x4x4xf3
 
 // -----
 
-func.func @fold_expand_of_collapse_dynamic(%arg0 : tensor<?x4x?xf32>, %arg1: index, %arg2: index)
+func.func @fold_expand_of_collapse_mixed_subshape(%arg0 : tensor<?x4x?xf32>, %arg1: index, %arg2: index)
     -> tensor<?x4x?xf32> {
   %0 = tensor.collapse_shape %arg0 [[0, 1], [2]]
       : tensor<?x4x?xf32> into tensor<?x?xf32>
@@ -1102,12 +1109,28 @@ func.func @fold_expand_of_collapse_dynamic(%arg0 : tensor<?x4x?xf32>, %arg1: ind
       : tensor<?x?xf32> into tensor<?x4x?xf32>
   return %1 : tensor<?x4x?xf32>
 }
-// CHECK-LABEL: @fold_expand_of_collapse_dynamic
+// CHECK-LABEL: @fold_expand_of_collapse_mixed_subshape
 //   CHECK-NOT:   tensor.{{.*}}_shape
 
 // -----
 
-func.func @no_fold_expand_of_collapse_dynamic(%arg0 : tensor<?x?x?xf32>, %arg1: index, %arg2: index, %arg3: index)
+func.func @fold_expand_of_collapse_mixed_target_subshape(%arg0 : tensor<?x4x?x2xf32>, %arg1: index, %arg2: index)
+    -> tensor<?x4x?xf32> {
+  %0 = tensor.collapse_shape %arg0 [[0, 1], [2, 3]]
+      : tensor<?x4x?x2xf32> into tensor<?x?xf32>
+  %1 = tensor.expand_shape %0 [[0, 1], [2]] output_shape [%arg1, 4, %arg2]
+      : tensor<?x?xf32> into tensor<?x4x?xf32>
+  return %1 : tensor<?x4x?xf32>
+}
+// CHECK-LABEL: @fold_expand_of_collapse_mixed_target_subshape
+//   CHECK-NOT:   tensor.expand_shape
+//       CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %arg0 {{\[}}[0], [1], [2, 3]]
+//  CHECK-SAME:     : tensor<?x4x?x2xf32> into tensor<?x4x?xf32>
+//  CHECK-NEXT:   return %[[COLLAPSE]]
+
+// -----
+
+func.func @no_fold_expand_of_collapse_fully_dynamic(%arg0 : tensor<?x?x?xf32>, %arg1: index, %arg2: index, %arg3: index)
     -> tensor<?x?x?xf32> {
   %0 = tensor.collapse_shape %arg0 [[0, 1], [2]]
       : tensor<?x?x?xf32> into tensor<?x?xf32>
@@ -1115,7 +1138,22 @@ func.func @no_fold_expand_of_collapse_dynamic(%arg0 : tensor<?x?x?xf32>, %arg1:
       : tensor<?x?xf32> into tensor<?x?x?xf32>
   return %1 : tensor<?x?x?xf32>
 }
-// CHECK-LABEL: @no_fold_expand_of_collapse_dynamic
+// CHECK-LABEL: @no_fold_expand_of_collapse_fully_dynamic
+//       CHECK:   tensor.collapse_shape
+//       CHECK:   %[[EXPAND:.+]] = tensor.expand_shape
+//       CHECK:   return %[[EXPAND]]
+
+// -----
+
+func.func @no_fold_expand_of_collapse_adjacent_dynamic(%arg0 : tensor<?x?x?xf32>, %arg1: index, %arg2: index)
+    -> tensor<?x?xf32> {
+  %0 = tensor.collapse_shape %arg0 [[0, 1, 2]]
+      : tensor<?x?x?xf32> into tensor<?xf32>
+  %1 = tensor.expand_shape %0 [[0, 1]] output_shape [%arg1, %arg2]
+      : tensor<?xf32> into tensor<?x?xf32>
+  return %1 : tensor<?x?xf32>
+}
+// CHECK-LABEL: @no_fold_expand_of_collapse_adjacent_dynamic
 //       CHECK:   tensor.collapse_shape
 //       CHECK:   %[[EXPAND:.+]] = tensor.expand_shape
 //       CHECK:   return %[[EXPAND]]
@@ -1265,6 +1303,20 @@ func.func @compose_expand_of_collapse_dynamic(%arg0 : tensor<4x?x10x64x2xf16>, %
 
 // -----
 
+func.func @no_compose_collapse_of_expand_dynamic(%arg0 : tensor<?x8x128x?xf16>, %arg1: index) -> tensor<?x128x?xf16> {
+  %collapse = tensor.collapse_shape %arg0 [[0, 1, 2, 3]] : tensor<?x8x128x?xf16> into tensor<?xf16>
+  %expanded_19 = tensor.expand_shape %collapse [[0, 1, 2]] output_shape [%arg1, 8, %arg1] : tensor<?xf16> into tensor<?x128x?xf16>
+  return %expanded_19 : tensor<?x128x?xf16>
+}
+// CHECK-LABEL: func @no_compose_collapse_of_expand_dynamic
+//  CHECK-SAME:   %[[ARG0:.+]]: tensor
+//  CHECK-SAME:   %[[ARG1:.+]]: index
+//       CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[ARG0]]
+//       CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %[[COLLAPSE]]
+//       CHECK:   return %[[EXPAND]]
+
+// -----
+
 // CHECK-LABEL: func @zero_rank_reshape_multi
 func.func @zero_rank_reshape_multi(%arg0: tensor<f32>) -> tensor<f32> {
   // CHECK: return %arg0
diff --git a/external/llvm-project/mlir/test/Dialect/Tensor/extract-from-collapse-shape.mlir b/external/llvm-project/mlir/test/Dialect/Tensor/extract-from-collapse-shape.mlir
new file mode 100644
index 000000000000..c301f494a7c8
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/Tensor/extract-from-collapse-shape.mlir
@@ -0,0 +1,31 @@
+// RUN: mlir-opt -split-input-file -test-tensor-transform-patterns=test-fold-extract-from-collapse-shape %s | FileCheck %s
+
+// CHECK-LABEL: @extract_from_collapse_shape
+// CHECK-SAME: (%[[ARG0:.*]]: tensor<1x1x8xi8>)
+func.func @extract_from_collapse_shape(%arg0: tensor<1x1x8xi8>) -> (i8, i8) {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %collapsed = tensor.collapse_shape %arg0 [[0, 1, 2]] : tensor<1x1x8xi8> into tensor<8xi8>
+  %extracted = tensor.extract %collapsed[%c0] : tensor<8xi8>
+  %extracted_0 = tensor.extract %collapsed[%c1] : tensor<8xi8>
+  func.return %extracted, %extracted_0 : i8, i8
+}
+
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[RESULT0:.*]] = tensor.extract %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]] : tensor<1x1x8xi8>
+// CHECK-DAG: %[[RESULT1:.*]] = tensor.extract %[[ARG0]][%[[C0]], %[[C0]], %[[C1]]] : tensor<1x1x8xi8>
+// CHECK-NEXT: return %[[RESULT0]], %[[RESULT1]] : i8, i8
+
+// -----
+
+// CHECK-LABEL: @extract_from_static_shape
+// CHECK-SAME: (%[[ARG0:.*]]: tensor<2x6x32xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index)
+func.func @extract_from_static_shape(%arg0 : tensor<2x6x32xf32>, %arg1 : index, %arg2 : index) -> f32 {
+  %0 = tensor.collapse_shape %arg0 [[0, 1], [2]] : tensor<2x6x32xf32> into tensor<12x32xf32>
+  %1 = tensor.extract %0[%arg1, %arg2] : tensor<12x32xf32>
+  return %1 : f32
+}
+// CHECK-NEXT: %[[MODIFIED_INDEXES:.*]]:2 = affine.delinearize_index %[[ARG1]] into (2, 6)
+// CHECK-NEXT: %[[RESULT:.*]] = tensor.extract %[[ARG0]][%[[MODIFIED_INDEXES]]#0, %[[MODIFIED_INDEXES]]#1, %[[ARG2]]] : tensor<2x6x32xf32>
+// CHECK-NEXT: return %[[RESULT]] : f32
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/availability.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/availability.mlir
index 75126a11ac50..0176fc288351 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/availability.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/availability.mlir
@@ -583,11 +583,11 @@ func.func @test_gather(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>) ->
 
 // -----
 // CHECK-LABEL: scatter
-func.func @test_scatter(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x21x3xf32> {
+func.func @test_scatter(%arg0: tensor<13x28x3xf32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x28x3xf32> {
   // CHECK: profiles: [ [pro_int, pro_fp] ]
   // CHECK: extensions: [ [fp8e4m3, fp8e5m2, bf16] ]
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf32>, tensor<13x26xi32>, tensor<13x26x3xf32>) -> tensor<13x21x3xf32>
-  return %0 : tensor<13x21x3xf32>
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x28x3xf32>, tensor<13x26xi32>, tensor<13x26x3xf32>) -> tensor<13x28x3xf32>
+  return %0 : tensor<13x28x3xf32>
 }
 
 // -----
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/error_if_check.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/error_if_check.mlir
index ac161128694c..1f25132d6bcf 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/error_if_check.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/error_if_check.mlir
@@ -225,3 +225,17 @@ func.func @test_error_i32_unsigned_output(%arg0: tensor<1xi8>) -> tensor<1xi32>
   %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = false, input_unsigned = false, output_unsigned = true} : (tensor<1xi8>, tensor<1xi16>, tensor<1xi8>, tensor<1xi8>, tensor<1xi32>) -> tensor<1xi32>
   return %0 : tensor<1xi32>
 }
+
+// -----
+// CHECK-LABEL: cond_if_simplified_form
+func.func @test_cond_if_simplified_form(%arg0: tensor<f32>, %arg1: tensor<f32>, %arg2: tensor<i1>) -> tensor<f32> {
+  // expected-error@+1 {{'tosa.cond_if' op the current simplified form is not strictly conformant to the spec, please use the generic format}}
+  %0 = tosa.cond_if %arg2 -> (tensor<f32>) {
+    %1 = tosa.add %arg0, %arg1 : (tensor<f32>, tensor<f32>) -> tensor<f32>
+    tosa.yield %1 : tensor<f32>
+  } else {
+    %1 = tosa.sub %arg0, %arg1 : (tensor<f32>, tensor<f32>) -> tensor<f32>
+    tosa.yield %1 : tensor<f32>
+  }
+  return %0 : tensor<f32>
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/invalid.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/invalid.mlir
index 229273574b34..d2409f3a43da 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/invalid.mlir
@@ -254,7 +254,6 @@ func.func @test_conv2d_quant_any(%arg0: tensor<1x4x4x4x!quant.any<i8<-8:7>>>, %a
 
 // -----
 
-
 func.func @test_concat(%arg0 : tensor<2x1xf32>, %arg1 : tensor<2x2xf32>) -> tensor<?x?xf32> {
   // expected-error@+1 {{'tosa.concat' op expect all operand shapes to have the same sizes on non-axis dimensions, but got 2 vs 1 at index 1 on operands 0 and 1}}
   %0 = tosa.concat %arg0, %arg1 {axis = 0 : i32} : (tensor<2x1xf32>, tensor<2x2xf32>) -> tensor<?x?xf32>
@@ -565,6 +564,23 @@ func.func @test_avg_pool2d_zero_dim_input(%arg0: tensor<1x0x?x9xf32>, %arg1: ten
 
 // -----
 
+func.func @test_variable_unranked(%arg0: tensor<2x4x8xi8>) -> () {
+  tosa.variable @stored_var : tensor<*xi8>
+  // expected-error@+1 {{custom op 'tosa.variable' expected ranked type}}
+  return
+}
+
+// -----
+
+func.func @test_variable_unranked_initial_value(%arg0: tensor<2x4x8xi8>) -> () {
+  // expected-error@+1 {{elements literal type must have static shape}}
+  tosa.variable @stored_var = dense<0> : tensor<*xi8>
+  // expected-error@+1 {{custom op 'tosa.variable' expected attribute}}
+  return
+}
+
+// -----
+
 func.func @test_variable_duplicates(%arg0: tensor<2x4x8xi8>) -> () {
   tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi8>
   // expected-error@+1 {{'tosa.variable' op illegal to have multiple declaration of 'stored_var'}}
@@ -1118,11 +1134,38 @@ func.func @test_mul_non_scalar_shift_1d(%arg0: tensor<13x21x3xf32>, %arg1: tenso
 // CHECK-LABEL: test_mul_non_broadcast
 func.func @test_mul_non_broadcast(%arg0: tensor<13x21x2xf32>, %arg1: tensor<3x1x3xf32>) -> tensor<13x21x3xf32> {
   %shift = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
-  // expected-error@+1 {{'tosa.mul' op operands don't have broadcast-compatible shapes}}
+  // expected-error@+1 {{'tosa.mul' op a and b operands don't have broadcast-compatible shapes, got 'tensor<13x21x2xf32>' and 'tensor<3x1x3xf32>'}}
   %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x2xf32>, tensor<3x1x3xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
+// -----
+// CHECK-LABEL: test_mul_different_operand_ranks
+func.func @test_mul_different_operand_ranks(%arg0: tensor<13x21xf32>, %arg1: tensor<3x1x3xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.mul' op a and b operands don't have matching ranks, got 2 and 3}}
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21xf32>, tensor<3x1x3xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+// CHECK-LABEL: test_mul_different_a_and_result_ranks
+func.func @test_mul_different_a_and_result_ranks(%arg0: tensor<13x21xf32>, %arg1: tensor<*xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.mul' op result type has different rank than a, got 3 vs 2}}
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21xf32>, tensor<*xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+// CHECK-LABEL: test_mul_different_b_and_result_ranks
+func.func @test_mul_different_b_and_result_ranks(%arg0: tensor<*xf32>, %arg1: tensor<13x12xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.mul' op result type has different rank than b, got 3 vs 2}}
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<*xf32>, tensor<13x12xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
 // -----
 // CHECK-LABEL: test_resize_invalid_scale_values
 func.func @test_resize_invalid_scale_values(%arg0: tensor<1x8x8x8xf32>) -> tensor<?x?x?x?xf32> {
@@ -1936,3 +1979,13 @@ func.func @test_rescale_output_unsigned(%arg0: tensor<1x1xi8>) -> (tensor<1x1xui
   %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xui8>
   return %r : tensor<1x1xui8>
 }
+
+// -----
+
+// CHECK-LABEL: test_scatter_duplicate_indices
+func.func @test_scatter_duplicate_indices(%arg0: tensor<2x52x3xf32>, %arg2: tensor<2x12x3xf32>) -> tensor<2x52x3xf32> {
+  %indices = "tosa.const"() { values = dense<[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [1, 2, 3, 4, 5, 6, 7, 8, 9, 3, 11, 12]]> : tensor<2x12xi32> } : () -> tensor<2x12xi32>
+  // expected-error@+1 {{'tosa.scatter' op indices values contain duplicates}}
+  %0 = tosa.scatter %arg0, %indices, %arg2 : (tensor<2x52x3xf32>, tensor<2x12xi32>, tensor<2x12x3xf32>) -> tensor<2x52x3xf32>
+  return %0 : tensor<2x52x3xf32>
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/invalid_extension.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/invalid_extension.mlir
index 2364985442e4..5630c33639d8 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/invalid_extension.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/invalid_extension.mlir
@@ -243,10 +243,10 @@ func.func @test_gather(%arg0: tensor<13x21x3xbf16>, %arg1: tensor<13x26xi32>) ->
 }
 
 // -----
-func.func @test_scatter(%arg0: tensor<13x21x3xbf16>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xbf16>) -> tensor<13x21x3xbf16> {
+func.func @test_scatter(%arg0: tensor<13x26x3xbf16>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xbf16>) -> tensor<13x26x3xbf16> {
   // expected-error@+1 {{'tosa.scatter' op illegal: requires [bf16] but not enabled in target}}
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xbf16>, tensor<13x26xi32>, tensor<13x26x3xbf16>) -> tensor<13x21x3xbf16>
-  return %0 : tensor<13x21x3xbf16>
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x26x3xbf16>, tensor<13x26xi32>, tensor<13x26x3xbf16>) -> tensor<13x26x3xbf16>
+  return %0 : tensor<13x26x3xbf16>
 }
 
 // -----
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/level_check.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/level_check.mlir
index e7d0a0e1fa4e..0dddf26fb1f8 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/level_check.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/level_check.mlir
@@ -443,7 +443,7 @@ func.func @test_rescale_rank_invalid(%arg0: tensor<1x1x1x1x13x21x3xi8>) -> tenso
 
 // -----
 func.func @test_const(%arg0 : tensor<1x1xi32>) -> tensor<1x1x1x1x1x1x1xi32> {
-  // expected-error@+1 {{'tosa.const' op failed level check: attribute rank(shape) <= MAX_RANK}}
+  // expected-error@+1 {{'tosa.const' op failed level check: result rank(shape) <= MAX_RANK}}
   %0 = "tosa.const"() {values = dense<0> : tensor<1x1x1x1x1x1x1xi32>} : () -> tensor<1x1x1x1x1x1x1xi32>
   return %0: tensor<1x1x1x1x1x1x1xi32>
 }
@@ -1080,16 +1080,17 @@ func.func @test_gather_tensor_size_invalid(%arg0: tensor<268435456x21x3xf32>, %a
 
 // -----
 
-func.func @test_scatter_tensor_size_invalid(%arg0: tensor<13x210000000x3xf32>, %arg1: tensor<13x260000000xi32>, %arg2: tensor<13x260000000x3xf32>) -> tensor<13x210000000x3xf32> {
+func.func @test_scatter_tensor_size_invalid(%arg0: tensor<13x260000000x3xf32>, %arg1: tensor<13x260000000xi32>, %arg2: tensor<13x260000000x3xf32>) -> tensor<13x260000000x3xf32> {
   // expected-error@+1 {{'tosa.scatter' op failed level check: operand tensor size (in bytes) <= (1 << MAX_LOG2_SIZE - 1)}}
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x210000000x3xf32>, tensor<13x260000000xi32>, tensor<13x260000000x3xf32>) -> tensor<13x210000000x3xf32>
-  return %0 : tensor<13x210000000x3xf32>
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x260000000x3xf32>, tensor<13x260000000xi32>, tensor<13x260000000x3xf32>) -> tensor<13x260000000x3xf32>
+  return %0 : tensor<13x260000000x3xf32>
 }
 
 // -----
 
 func.func @test_variable_read_write_tensor_size_invalid() -> () {
-  tosa.variable @stored_var = dense<3.14> : tensor<536870912xf32>
+  // expected-error@+1 {{'tosa.variable' op failed level check: variable type tensor size (in bytes) <= (1 << MAX_LOG2_SIZE - 1)}}
+  tosa.variable @stored_var : tensor<536870912xf32>
   // expected-error@+1 {{'tosa.variable_read' op failed level check: result tensor size (in bytes) <= (1 << MAX_LOG2_SIZE - 1)}}
   %0 = tosa.variable_read @stored_var : tensor<536870912xf32>
   // expected-error@+1 {{'tosa.variable_write' op failed level check: operand tensor size (in bytes) <= (1 << MAX_LOG2_SIZE - 1)}}
@@ -1156,8 +1157,8 @@ func.func @test_cond_if_rank_invalid(%arg0: tensor<1x1x1x1x1x1x1x1xf32>, %arg1:
 // -----
 
 func.func @test_variable_read_write_rank_invalid() -> () {
-  // expected-error@+1 {{'tosa.variable' op failed level check: attribute rank(shape) <= MAX_RANK}}
-  tosa.variable @stored_var = dense<3.14> : tensor<1x1x1x1x1x1x1x1xf32>
+  // expected-error@+1 {{'tosa.variable' op failed level check: variable type rank(shape) <= MAX_RANK}}
+  tosa.variable @stored_var : tensor<1x1x1x1x1x1x1x1xf32>
   // expected-error@+1 {{'tosa.variable_read' op failed level check: result rank(shape) <= MAX_RANK}}
   %0 = tosa.variable_read @stored_var : tensor<1x1x1x1x1x1x1x1xf32>
   // expected-error@+1 {{'tosa.variable_write' op failed level check: operand rank(shape) <= MAX_RANK}}
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/ops.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/ops.mlir
index 767fa833dedd..ef51197e86d5 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/ops.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/ops.mlir
@@ -424,6 +424,22 @@ func.func @test_mul_relaxed_result_type(%arg0: tensor<13x21x3xi16>, %arg1: tenso
   return %0 : tensor<13x21x3xi16>
 }
 
+// -----
+// CHECK-LABEL: test_mul_unranked_b
+func.func @test_mul_unranked_b(%arg0: tensor<13x21x3xf32>, %arg1: tensor<*xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<13x21x3xf32>, tensor<*xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+// CHECK-LABEL: test_mul_unranked_a_and_b
+func.func @test_mul_unranked_a_and_b(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<13x21x3xf32> {
+  %shift = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.mul %arg0, %arg1, %shift : (tensor<*xf32>, tensor<*xf32>, tensor<1xi8>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
 // -----
 // CHECK-LABEL: pow
 func.func @test_pow(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x21x1xf32>) -> tensor<13x21x3xf32> {
@@ -714,9 +730,9 @@ func.func @test_gather(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>) ->
 
 // -----
 // CHECK-LABEL: scatter
-func.func @test_scatter(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf32>, tensor<13x26xi32>, tensor<13x26x3xf32>) -> tensor<13x21x3xf32>
-  return %0 : tensor<13x21x3xf32>
+func.func @test_scatter(%arg0: tensor<13x52x3xf32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x52x3xf32> {
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x52x3xf32>, tensor<13x26xi32>, tensor<13x26x3xf32>) -> tensor<13x52x3xf32>
+  return %0 : tensor<13x52x3xf32>
 }
 
 // -----
@@ -728,8 +744,8 @@ func.func @test_gather_unranked_indices(%arg0: tensor<13x21x3xf32>, %arg1: tenso
 
 // -----
 // CHECK-LABEL: scatter_unranked_indices
-func.func @test_scatter_unranked_indices(%arg0: tensor<13x21x3xf32>, %arg1: tensor<*xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf32>, tensor<*xi32>, tensor<13x26x3xf32>) -> tensor<13x21x3xf32>
+func.func @test_scatter_unranked_indices(%arg0: tensor<13x21x3xf32>, %arg1: tensor<*xi32>, %arg2: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf32>, tensor<*xi32>, tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
@@ -1010,9 +1026,9 @@ func.func @test_gather_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>, %arg1: tensor<13x26
 
 // -----
 // CHECK-LABEL: scatter_f8E5M2
-func.func @test_scatter_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf8E5M2>) -> tensor<13x21x3xf8E5M2> {
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf8E5M2>, tensor<13x26xi32>, tensor<13x26x3xf8E5M2>) -> tensor<13x21x3xf8E5M2>
-  return %0 : tensor<13x21x3xf8E5M2>
+func.func @test_scatter_f8E5M2(%arg0: tensor<13x52x3xf8E5M2>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf8E5M2>) -> tensor<13x52x3xf8E5M2> {
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x52x3xf8E5M2>, tensor<13x26xi32>, tensor<13x26x3xf8E5M2>) -> tensor<13x52x3xf8E5M2>
+  return %0 : tensor<13x52x3xf8E5M2>
 }
 
 // -----
@@ -1155,7 +1171,7 @@ func.func @test_gather_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>, %arg1: tensor<1
 
 // -----
 // CHECK-LABEL: scatter_f8E4M3FN
-func.func @test_scatter_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN> {
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf8E4M3FN>, tensor<13x26xi32>, tensor<13x26x3xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN>
-  return %0 : tensor<13x21x3xf8E4M3FN>
+func.func @test_scatter_f8E4M3FN(%arg0: tensor<13x29x3xf8E4M3FN>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf8E4M3FN>) -> tensor<13x29x3xf8E4M3FN> {
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x29x3xf8E4M3FN>, tensor<13x26xi32>, tensor<13x26x3xf8E4M3FN>) -> tensor<13x29x3xf8E4M3FN>
+  return %0 : tensor<13x29x3xf8E4M3FN>
 }
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir
index 72669c62c95c..fad485935125 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir
@@ -310,10 +310,10 @@ func.func @test_gather(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>) ->
 }
 
 // -----
-func.func @test_scatter(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x21x3xf32> {
+func.func @test_scatter(%arg0: tensor<13x28x3xf32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x28x3xf32> {
   // expected-error@+1 {{'tosa.scatter' op illegal: requires [pro_fp] but not enabled in target}}
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf32>, tensor<13x26xi32>, tensor<13x26x3xf32>) -> tensor<13x21x3xf32>
-  return %0 : tensor<13x21x3xf32>
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x28x3xf32>, tensor<13x26xi32>, tensor<13x26x3xf32>) -> tensor<13x28x3xf32>
+  return %0 : tensor<13x28x3xf32>
 }
 
 // -----
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir
index e98b906377b2..9438179622aa 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir
@@ -242,10 +242,10 @@ func.func @test_gather(%arg0: tensor<13x21x3xi32>, %arg1: tensor<13x26xi32>) ->
 }
 
 // -----
-func.func @test_scatter(%arg0: tensor<13x21x3xi32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xi32>) -> tensor<13x21x3xi32> {
+func.func @test_scatter(%arg0: tensor<13x27x3xi32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xi32>) -> tensor<13x27x3xi32> {
   // expected-error@+1 {{'tosa.scatter' op illegal: requires [pro_int] but not enabled in target}}
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xi32>, tensor<13x26xi32>, tensor<13x26x3xi32>) -> tensor<13x21x3xi32>
-  return %0 : tensor<13x21x3xi32>
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x27x3xi32>, tensor<13x26xi32>, tensor<13x26x3xi32>) -> tensor<13x27x3xi32>
+  return %0 : tensor<13x27x3xi32>
 }
 
 // -----
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
index 1ad1e6c76c29..591a3f0acf65 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
@@ -656,9 +656,9 @@ func.func @gather_minimum_info(%arg0 : tensor<3x?x5xi32>, %arg1 : tensor<?x6xi32
 // -----
 
 // CHECK-LABEL: @scatter_static
-func.func @scatter_static(%arg0 : tensor<3x4x5xi32>, %arg1 : tensor<3x6xi32>, %arg2 : tensor<3x6x5xi32>) {
-  // CHECK: tosa.scatter %arg0, %arg1, %arg2 : (tensor<3x4x5xi32>, tensor<3x6xi32>, tensor<3x6x5xi32>) -> tensor<3x4x5xi32>
-  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<3x4x5xi32>, tensor<3x6xi32>, tensor<3x6x5xi32>) -> tensor<?x?x?xi32>
+func.func @scatter_static(%arg0 : tensor<3x8x5xi32>, %arg1 : tensor<3x6xi32>, %arg2 : tensor<3x6x5xi32>) {
+  // CHECK: tosa.scatter %arg0, %arg1, %arg2 : (tensor<3x8x5xi32>, tensor<3x6xi32>, tensor<3x6x5xi32>) -> tensor<3x8x5xi32>
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<3x8x5xi32>, tensor<3x6xi32>, tensor<3x6x5xi32>) -> tensor<?x?x?xi32>
   return
 }
 
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/variables.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/variables.mlir
index 25f63331f39d..9953eb375d3a 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/variables.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/variables.mlir
@@ -31,3 +31,48 @@ func.func @test_variable_tensor(%arg0: tensor<2x4x8xi32>) -> () {
   tosa.variable_write @stored_var, %1 : tensor<2x4x8xi32>
   return
 }
+
+// -----
+// CHECK-LABEL:   @test_variable_scalar_no_initial_value(
+// CHECK-SAME:                        %[[ADD_VAL:.*]]: tensor<f32>) {
+func.func @test_variable_scalar_no_initial_value(%arg0: tensor<f32>) -> () {
+  // CHECK:           tosa.variable @stored_var : tensor<f32>
+  tosa.variable @stored_var : tensor<f32>
+  // CHECK:           %[[STORED_VAL:.*]] = tosa.variable_read @stored_var : tensor<f32>
+  %0 = tosa.variable_read @stored_var : tensor<f32>
+  // CHECK:           %[[RESULT_ADD:.*]] = tosa.add %[[ADD_VAL]], %[[STORED_VAL]] : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  %1 = "tosa.add"(%arg0, %0) : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  // CHECK:           tosa.variable_write @stored_var, %[[RESULT_ADD]] : tensor<f32>
+  tosa.variable_write @stored_var, %1 : tensor<f32>
+  return
+}
+
+// -----
+// CHECK-LABEL:   @test_variable_tensor_no_initial_value(
+// CHECK-SAME:                        %[[ADD_VAL:.*]]: tensor<2x4x8xi32>) {
+func.func @test_variable_tensor_no_initial_value(%arg0: tensor<2x4x8xi32>) -> () {
+  // CHECK:           tosa.variable @stored_var : tensor<2x4x8xi32>
+  tosa.variable @stored_var : tensor<2x4x8xi32>
+  // CHECK:           %[[STORED_VAL:.*]] = tosa.variable_read @stored_var : tensor<2x4x8xi32>
+  %0 = tosa.variable_read @stored_var : tensor<2x4x8xi32>
+  // CHECK:           %[[RESULT_ADD:.*]] = tosa.add %[[ADD_VAL]], %[[STORED_VAL]] : (tensor<2x4x8xi32>, tensor<2x4x8xi32>) -> tensor<2x4x8xi32>
+  %1 = "tosa.add"(%arg0, %0) : (tensor<2x4x8xi32>, tensor<2x4x8xi32>) -> tensor<2x4x8xi32>
+  // CHECK:           tosa.variable_write @stored_var, %[[RESULT_ADD]] : tensor<2x4x8xi32>
+  tosa.variable_write @stored_var, %1 : tensor<2x4x8xi32>
+  return
+}
+
+// -----
+// CHECK-LABEL:   @test_variable_tensor_with_unknowns(
+// CHECK-SAME:                        %[[ADD_VAL:.*]]: tensor<2x4x8xi32>) {
+func.func @test_variable_tensor_with_unknowns(%arg0: tensor<2x4x8xi32>) -> () {
+  // CHECK:           tosa.variable @stored_var : tensor<2x?x8xi32>
+  tosa.variable @stored_var : tensor<2x?x8xi32>
+  // CHECK:           %[[STORED_VAL:.*]] = tosa.variable_read @stored_var : tensor<2x4x8xi32>
+  %0 = tosa.variable_read @stored_var : tensor<2x4x8xi32>
+  // CHECK:           %[[RESULT_ADD:.*]] = tosa.add %[[ADD_VAL]], %[[STORED_VAL]] : (tensor<2x4x8xi32>, tensor<2x4x8xi32>) -> tensor<2x4x8xi32>
+  %1 = "tosa.add"(%arg0, %0) : (tensor<2x4x8xi32>, tensor<2x4x8xi32>) -> tensor<2x4x8xi32>
+  // CHECK:           tosa.variable_write @stored_var, %[[RESULT_ADD]] : tensor<2x4x8xi32>
+  tosa.variable_write @stored_var, %1 : tensor<2x4x8xi32>
+  return
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Tosa/verifier.mlir b/external/llvm-project/mlir/test/Dialect/Tosa/verifier.mlir
index 434243712ba2..91b150a85e51 100644
--- a/external/llvm-project/mlir/test/Dialect/Tosa/verifier.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Tosa/verifier.mlir
@@ -774,3 +774,75 @@ func.func @test_variable_write_shape_mismatch(%arg0: tensor<2x4x8xf32>) -> () {
   tosa.variable_write @stored_var, %arg0 : tensor<2x4x8xf32>
   return
 }
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_indices_N
+func.func @scatter_invalid_indices_N(%arg0 : tensor<2x4x5xi32>, %arg1 : tensor<3x2xi32>, %arg2 : tensor<2x2x5xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires indices dimension 0 to have size 2, got 3}}
+  %1 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<2x4x5xi32>, tensor<3x2xi32>, tensor<2x2x5xi32>) -> tensor<2x4x5xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_input_N
+func.func @scatter_invalid_input_N(%arg0 : tensor<?x4x5xi32>, %arg1 : tensor<2x2xi32>, %arg2 : tensor<3x2x5xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires input dimension 0 to have size 2, got 3}}
+  %2 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<?x4x5xi32>, tensor<2x2xi32>, tensor<3x2x5xi32>) -> tensor<2x4x5xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_out_N
+func.func @scatter_invalid_out_N(%arg0 : tensor<?x4x5xi32>, %arg1 : tensor<?x2xi32>, %arg2 : tensor<2x2x5xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires values_out dimension 0 to have size 2, got 3}}
+  %2 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<?x4x5xi32>, tensor<?x2xi32>, tensor<2x2x5xi32>) -> tensor<3x4x5xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_out_K
+func.func @scatter_invalid_out_K(%arg0 : tensor<?x4x5xi32>, %arg1 : tensor<?x2xi32>, %arg2 : tensor<2x2x5xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires values_out dimension 1 to have size 4, got 3}}
+  %2 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<?x4x5xi32>, tensor<?x2xi32>, tensor<2x2x5xi32>) -> tensor<2x3x5xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_input_W
+func.func @scatter_invalid_input_W(%arg0 : tensor<?x4x5xi32>, %arg1 : tensor<?x2xi32>, %arg2 : tensor<2x3x5xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires input dimension 1 to have size 2, got 3}}
+  %2 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<?x4x5xi32>, tensor<?x2xi32>, tensor<2x3x5xi32>) -> tensor<2x4x5xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_input_C
+func.func @scatter_invalid_input_C(%arg0 : tensor<?x4x5xi32>, %arg1 : tensor<?x2xi32>, %arg2 : tensor<2x2x6xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires input dimension 2 to have size 5, got 6}}
+  %2 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<?x4x5xi32>, tensor<?x2xi32>, tensor<2x2x6xi32>) -> tensor<2x4x5xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_out_C
+func.func @scatter_invalid_out_C(%arg0 : tensor<?x4x5xi32>, %arg1 : tensor<?x2xi32>, %arg2 : tensor<2x2x5xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires values_out dimension 2 to have size 5, got 6}}
+  %2 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<?x4x5xi32>, tensor<?x2xi32>, tensor<2x2x5xi32>) -> tensor<2x4x6xi32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @scatter_invalid_K_W
+func.func @scatter_invalid_K_W(%arg0 : tensor<2x4x5xi32>, %arg1 : tensor<2x6xi32>, %arg2 : tensor<2x6x5xi32>) {
+  // expected-error@+1 {{'tosa.scatter' op requires dimensions K >= W, got K=4 and W=6}}
+  %2 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<2x4x5xi32>, tensor<2x6xi32>, tensor<2x6x5xi32>) -> tensor<2x4x5xi32>
+  return
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Transform/test-pass-application.mlir b/external/llvm-project/mlir/test/Dialect/Transform/test-pass-application.mlir
index 3a40b462b827..ce8f69c58701 100644
--- a/external/llvm-project/mlir/test/Dialect/Transform/test-pass-application.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Transform/test-pass-application.mlir
@@ -79,7 +79,9 @@ module attributes {transform.with_named_sequence} {
     %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     // expected-error @below {{failed to add pass or pass pipeline to pipeline: canonicalize}}
     // expected-error @below {{<Pass-Options-Parser>: no such option invalid-option}}
-    transform.apply_registered_pass "canonicalize" to %1 {options = "invalid-option=1"} : (!transform.any_op) -> !transform.any_op
+    transform.apply_registered_pass "canonicalize"
+        with options = { "invalid-option" = 1 } to %1
+        : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
 }
@@ -94,7 +96,289 @@ func.func @valid_pass_option() {
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
     %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.apply_registered_pass "canonicalize" to %1 {options = "top-down=false"} : (!transform.any_op) -> !transform.any_op
+    transform.apply_registered_pass "canonicalize"
+        with options = { "top-down" = false } to %1
+        : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @valid_pass_options()
+func.func @valid_pass_options() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    //transform.apply_registered_pass "canonicalize" with options = "top-down=false,max-iterations=10" to %1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_registered_pass "canonicalize"
+        with options = { "top-down" = false, "test-convergence" =true } to %1
+        : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @valid_pass_options_as_list()
+func.func @valid_pass_options_as_list() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_registered_pass "canonicalize"
+        with options = { "top-down" = false, "max-iterations" = 0 } to %1
+        : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @valid_dynamic_pass_options()
+func.func @valid_dynamic_pass_options() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %max_iter = transform.param.constant 10 -> !transform.any_param
+    %max_rewrites = transform.param.constant 1 -> !transform.any_param
+    %2 = transform.apply_registered_pass
+        "canonicalize"
+        with options = { "top-down" = false,
+                         "max-iterations" = %max_iter,
+                         "test-convergence" = true,
+                         "max-num-rewrites" =  %max_rewrites }
+        to %1
+        : (!transform.any_op, !transform.any_param, !transform.any_param) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func private @valid_multiple_values_as_list_option_single_param()
+module {
+  func.func @valid_multiple_values_as_list_option_single_param() {
+    return
+  }
+
+  // CHECK: func @a()
+  func.func @a() {
+    return
+  }
+  // CHECK: func @b()
+  func.func @b() {
+    return
+  }
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.get_parent_op %1 { deduplicate } : (!transform.any_op) -> !transform.any_op
+    %symbol_a = transform.param.constant "a" -> !transform.any_param
+    %symbol_b = transform.param.constant "b" -> !transform.any_param
+    %multiple_symbol_names = transform.merge_handles %symbol_a, %symbol_b : !transform.any_param
+    transform.apply_registered_pass "symbol-privatize"
+        with options = { exclude = %multiple_symbol_names } to %2
+        : (!transform.any_op, !transform.any_param) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func private @valid_array_attr_as_list_option()
+module {
+  func.func @valid_array_attr_as_list_option() {
+    return
+  }
+
+  // CHECK: func @a()
+  func.func @a() {
+    return
+  }
+  // CHECK: func @b()
+  func.func @b() {
+    return
+  }
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.get_parent_op %1 { deduplicate } : (!transform.any_op) -> !transform.any_op
+    transform.apply_registered_pass "symbol-privatize"
+        with options = { exclude = ["a", "b"] } to %2
+        : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func private @valid_array_attr_param_as_list_option()
+module {
+  func.func @valid_array_attr_param_as_list_option() {
+    return
+  }
+
+  // CHECK: func @a()
+  func.func @a() {
+    return
+  }
+  // CHECK: func @b()
+  func.func @b() {
+    return
+  }
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.get_parent_op %1 { deduplicate } : (!transform.any_op) -> !transform.any_op
+    %multiple_symbol_names = transform.param.constant ["a","b"] -> !transform.any_param
+    transform.apply_registered_pass "symbol-privatize"
+        with options = { exclude = %multiple_symbol_names } to %2
+        : (!transform.any_op, !transform.any_param) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func private @valid_multiple_params_as_single_list_option()
+module {
+  func.func @valid_multiple_params_as_single_list_option() {
+    return
+  }
+
+  // CHECK: func @a()
+  func.func @a() {
+    return
+  }
+  // CHECK: func @b()
+  func.func @b() {
+    return
+  }
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.get_parent_op %1 { deduplicate } : (!transform.any_op) -> !transform.any_op
+    %symbol_a = transform.param.constant "a" -> !transform.any_param
+    %symbol_b = transform.param.constant "b" -> !transform.any_param
+    transform.apply_registered_pass "symbol-privatize"
+        with options = { exclude = [%symbol_a, %symbol_b] } to %2
+        : (!transform.any_op, !transform.any_param, !transform.any_param) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @invalid_options_as_str() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    // expected-error @+2 {{expected '{' in options dictionary}}
+    %2 = transform.apply_registered_pass "canonicalize"
+        with options = "top-down=false" to %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @invalid_options_as_pairs_without_braces() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    // expected-error @+2 {{expected '{' in options dictionary}}
+    %2 = transform.apply_registered_pass "canonicalize"
+        with options = "top-down"=false to %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @invalid_options_due_to_reserved_attr() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    // expected-error @+3 {{the param_operand attribute is a marker reserved for indicating a value will be passed via params and is only used in the generic print format}}
+    // expected-error @+2 {{expected a valid attribute or operand as value associated to key 'top-down'}}
+    %2 = transform.apply_registered_pass "canonicalize"
+        with options = { "top-down" = #transform.param_operand<index=0> } to %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @invalid_options_due_duplicated_key() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    // expected-error @+2 {{duplicate keys found in options dictionary}}
+    %2 = transform.apply_registered_pass "canonicalize"
+        with options = {"top-down"=false,"top-down"=true} to %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @invalid_options_due_invalid_key() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    // expected-error @+2 {{expected key to either be an identifier or a string}}
+    %2 = transform.apply_registered_pass "canonicalize"
+        with options = { @label = 0 } to %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @invalid_pass_option_bare_param() {
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op) {
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %pass_options = transform.param.constant 42 -> !transform.any_param
+    // expected-error @+2 {{expected '{' in options dictionary}}
+    transform.apply_registered_pass "canonicalize"
+        with options = %pass_options to %1
+        : (!transform.any_op, !transform.any_param) -> !transform.any_op
     transform.yield
   }
 }
@@ -117,3 +401,77 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+// -----
+
+/////////////////////////////////////////////////////////////////////
+// Check that the following cases are caugh in the generic format. //
+/////////////////////////////////////////////////////////////////////
+
+// Invalid due to param_operand occurences in options dict not being
+// one-to-one with the dynamic options provided as params:
+//   param_operand_index out of bounds w.r.t. the number of options provided via params.
+
+"builtin.module"() ({
+  "transform.named_sequence"() <{function_type = (!transform.any_op) -> (), sym_name = "__transform_main"}> ({
+  ^bb0(%arg0: !transform.any_op):
+    %0 = "transform.structured.match"(%arg0) <{ops = ["func.func"]}> : (!transform.any_op) -> !transform.any_op
+    %1 = "transform.param.constant"() <{value = 10 : i64}> : () -> !transform.any_param
+    // expected-error @below {{dynamic option index 1 is out of bounds for the number of dynamic options: 1}}
+    %2 = "transform.apply_registered_pass"(%0, %1) <{
+      options = {"max-iterations" = #transform.param_operand<index=1 : i64>,
+                 "test-convergence" = true,
+                 "top-down" = false},
+      pass_name = "canonicalize"}>
+    : (!transform.any_op, !transform.any_param) -> !transform.any_op
+    "transform.yield"() : () -> ()
+  }) : () -> ()
+}) {transform.with_named_sequence} : () -> ()
+
+// -----
+
+// Invalid due to param_operand occurences in options dict not being
+// one-to-one with the dynamic options provided as params:
+//   the first option-param is referred to twice and the second one not at all.
+// (In the pretty-printed format, if you want to refer to a param SSA-value twice, it counts as two param arguments.)
+
+"builtin.module"() ({
+  "transform.named_sequence"() <{function_type = (!transform.any_op) -> (), sym_name = "__transform_main"}> ({
+  ^bb0(%arg0: !transform.any_op):
+    %0 = "transform.structured.match"(%arg0) <{ops = ["func.func"]}> : (!transform.any_op) -> !transform.any_op
+    %1 = "transform.param.constant"() <{value = 10 : i64}> : () -> !transform.any_param
+    %2 = "transform.param.constant"() <{value = 1 : i64}> : () -> !transform.any_param
+    // expected-error @below {{dynamic option index 0 is already used in options}}
+    %3 = "transform.apply_registered_pass"(%0, %1, %2) <{
+      options = {"max-iterations" = #transform.param_operand<index=0 : i64>,
+                 "max-num-rewrites" = #transform.param_operand<index=0 : i64>,
+                 "test-convergence" = true,
+                 "top-down" = false},
+      pass_name = "canonicalize"}>
+    : (!transform.any_op, !transform.any_param, !transform.any_param) -> !transform.any_op
+    "transform.yield"() : () -> ()
+  }) : () -> ()
+}) {transform.with_named_sequence} : () -> ()
+
+// -----
+
+// Invalid due to param_operand occurences in options dict not being
+// one-to-one with the dynamic options provided as params:
+//   two option-params are provide though only the first one is referred to from the options-dict.
+
+"builtin.module"() ({
+  "transform.named_sequence"() <{function_type = (!transform.any_op) -> (), sym_name = "__transform_main"}> ({
+  ^bb0(%arg0: !transform.any_op):
+    %0 = "transform.structured.match"(%arg0) <{ops = ["func.func"]}> : (!transform.any_op) -> !transform.any_op
+    %1 = "transform.param.constant"() <{value = 10 : i64}> : () -> !transform.any_param
+    %2 = "transform.param.constant"() <{value = 1 : i64}> : () -> !transform.any_param
+    // expected-error @below {{a param operand does not have a corresponding param_operand attr in the options dict}}
+    %3 = "transform.apply_registered_pass"(%0, %1, %2) <{
+      options = {"max-iterations" = #transform.param_operand<index=0 : i64>,
+                 "test-convergence" = true,
+                 "top-down" = false},
+      pass_name = "canonicalize"}>
+    : (!transform.any_op, !transform.any_param, !transform.any_param) -> !transform.any_op
+    "transform.yield"() : () -> ()
+  }) : () -> ()
+}) {transform.with_named_sequence} : () -> ()
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-smmla.mlir b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-smmla.mlir
new file mode 100644
index 000000000000..404ef87380c9
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-smmla.mlir
@@ -0,0 +1,204 @@
+// RUN:  mlir-opt %s --transform-interpreter | FileCheck %s
+
+#attrs = {
+  indexing_maps = [
+    affine_map<(d0, d1, d2) -> (d0, d2)>,
+    affine_map<(d0, d1, d2) -> (d1, d2)>,
+    affine_map<(d0, d1, d2) -> (d0, d1)>
+  ],
+  iterator_types = ["parallel", "parallel", "reduction"],
+  kind = #vector.kind<add>
+}
+
+// CHECK-LABEL: @test_vector_contract_to_smmla
+
+// CHECK-SAME: %[[LHS:arg0]]: vector<4x8xi8>
+// CHECK-SAME: %[[RHS:arg1]]: vector<[4]x8xi8>
+// CHECK-SAME: %[[ACC:arg2]]: vector<4x[4]xi32>
+
+// CHECK:         [[P0:[0-9]+]] = ub.poison : vector<[8]xi32>
+// CHECK-NEXT:    [[P1:[0-9]+]] = ub.poison : vector<4x[4]xi32>
+// CHECK-NEXT:    [[P2:[0-9]+]] = ub.poison : vector<[16]xi8>
+
+// Extract LHS rows 0 and 1, concatenate, turn into scalable vector
+// CHECK:         %[[T3:[0-9]+]] = vector.extract %[[LHS]][0] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T4:[0-9]+]] = vector.extract %[[LHS]][1] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T5:[0-9]+]] = vector.shuffle %[[T3]], %[[T4]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T6:[0-9]+]] = vector.scalable.insert %[[T5]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+
+// Replicate across the entire length of the scalable vector
+// CHECK-NEXT:    %[[LHS_0:[0-9]+]] = arm_sve.dupq_lane %[[T6]][0] : vector<[16]xi8>
+
+// Same for LHS rows 2 and 3
+// CHECK-NEXT:    %[[T8:[0-9]+]] = vector.extract %[[LHS]][2] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T9:[0-9]+]] = vector.extract %[[LHS]][3] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T10:[0-9]+]] = vector.shuffle %[[T8]], %[[T9]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T11:[0-9]+]] = vector.scalable.insert %[[T10]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+// CHECK-NEXT:    %[[LHS_1:[0-9]+]] = arm_sve.dupq_lane %[[T11]][0] : vector<[16]xi8>
+
+// Extract sub-tiles from the RHS
+// CHECK-NEXT:    %[[T13:[0-9]+]] = vector.shape_cast %[[RHS]] : vector<[4]x8xi8> to vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_0:[0-9]+]] = vector.scalable.extract %[[T13]][0] : vector<[16]xi8> from vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_1:[0-9]+]] = vector.scalable.extract %[[T13]][16] : vector<[16]xi8> from vector<[32]xi8>
+
+// Extract accumulator rows 0 and 1 and pack (into "registers")
+// CHECK-NEXT:    %[[T16:[0-9]+]] = vector.extract %[[ACC]][0] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T17:[0-9]+]] = vector.extract %[[ACC]][1] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T18:[0-9]+]] = vector.bitcast %[[T16]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T19:[0-9]+]] = vector.bitcast %[[T17]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T20:[0-9]+]] = vector.interleave %[[T18]], %[[T19]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T21:[0-9]+]] = vector.bitcast %[[T20]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_0:[0-9]+]] = vector.scalable.extract %[[T21]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_1:[0-9]+]] = vector.scalable.extract %[[T21]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Same for accumulator rows 2 and 3
+// CHECK-NEXT:    %[[T24:[0-9]+]] = vector.extract %[[ACC]][2] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T25:[0-9]+]] = vector.extract %[[ACC]][3] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T26:[0-9]+]] = vector.bitcast %[[T24]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T27:[0-9]+]] = vector.bitcast %[[T25]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T28:[0-9]+]] = vector.interleave %[[T26]], %[[T27]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T29:[0-9]+]] = vector.bitcast %[[T28]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_2:[0-9]+]] = vector.scalable.extract %[[T29]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_3:[0-9]+]] = vector.scalable.extract %[[T29]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Do the sub-tile matrix multiplications
+// CHECK-NEXT:    %[[PACK_RES_00:[0-9]+]] = arm_sve.smmla %[[ACC_0]], %[[LHS_0]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_01:[0-9]+]] = arm_sve.smmla %[[ACC_1]], %[[LHS_0]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_10:[0-9]+]] = arm_sve.smmla %[[ACC_2]], %[[LHS_1]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_11:[0-9]+]] = arm_sve.smmla %[[ACC_3]], %[[LHS_1]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+
+// Unpack (from "registers") and insert in the output result rows 0 and 1
+// CHECK-NEXT:    %[[T36:[0-9]+]] = vector.scalable.insert %[[PACK_RES_00]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T37:[0-9]+]] = vector.scalable.insert %[[PACK_RES_01]], %[[T36]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T38:[0-9]+]] = vector.bitcast %[[T37]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1, %res2 = vector.deinterleave %[[T38]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_0:[0-9]+]] = vector.bitcast %res1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_1:[0-9]+]] = vector.bitcast %res2 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_0:[0-9]+]] = vector.insert %[[UNPACK_RES_0]], %[[P1]] [0] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_1:[0-9]+]] = vector.insert %[[UNPACK_RES_1]], %[[TMP_OUT_0]] [1] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// Same for result rows 2 and 3
+// CHECK-NEXT:    %[[T43:[0-9]+]] = vector.scalable.insert %[[PACK_RES_10]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T44:[0-9]+]] = vector.scalable.insert %[[PACK_RES_11]], %[[T43]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T45:[0-9]+]] = vector.bitcast %[[T44]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1_0, %res2_1 = vector.deinterleave %[[T45]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_2:[0-9]+]] = vector.bitcast %res1_0 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_3:[0-9]+]] = vector.bitcast %res2_1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_2:[0-9]+]] = vector.insert %[[UNPACK_RES_2]], %[[TMP_OUT_1]] [2] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[OUT:[0-9]+]] = vector.insert %[[UNPACK_RES_3]], %[[TMP_OUT_2]] [3] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// CHECK-NEXT:    return %[[OUT]] : vector<4x[4]xi32>
+func.func @test_vector_contract_to_smmla(%lhs: vector<4x8xi8>,
+              %rhs: vector<[4]x8xi8>,
+              %acc: vector<4x[4]xi32>) -> vector<4x[4]xi32> {
+
+  %0 = arith.extsi %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extsi %rhs : vector<[4]x8xi8> to vector<[4]x8xi32>
+  %2 = vector.contract #attrs %0, %1, %acc
+    : vector<4x8xi32>, vector<[4]x8xi32> into vector<4x[4]xi32>
+
+  return %2 : vector<4x[4]xi32>
+}
+
+// CHECK-LABEL: @test_vector_contract_to_smmla_implicit_sext
+
+// CHECK-SAME: %[[LHS:arg0]]: vector<4x8xi8>
+// CHECK-SAME: %[[RHS:arg1]]: vector<[4]x8xi8>
+// CHECK-SAME: %[[ACC:arg2]]: vector<4x[4]xi32>
+
+// CHECK:         [[P0:[0-9]+]] = ub.poison : vector<[8]xi32>
+// CHECK-NEXT:    [[P1:[0-9]+]] = ub.poison : vector<4x[4]xi32>
+// CHECK-NEXT:    [[P2:[0-9]+]] = ub.poison : vector<[16]xi8>
+
+// Extract LHS rows 0 and 1, concatenate, turn into scalable vector
+// CHECK:         %[[T3:[0-9]+]] = vector.extract %[[LHS]][0] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T4:[0-9]+]] = vector.extract %[[LHS]][1] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T5:[0-9]+]] = vector.shuffle %[[T3]], %[[T4]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T6:[0-9]+]] = vector.scalable.insert %[[T5]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+
+// Replicate across the entire length of the scalable vector
+// CHECK-NEXT:    %[[LHS_0:[0-9]+]] = arm_sve.dupq_lane %[[T6]][0] : vector<[16]xi8>
+
+// Same for LHS rows 2 and 3
+// CHECK-NEXT:    %[[T8:[0-9]+]] = vector.extract %[[LHS]][2] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T9:[0-9]+]] = vector.extract %[[LHS]][3] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T10:[0-9]+]] = vector.shuffle %[[T8]], %[[T9]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T11:[0-9]+]] = vector.scalable.insert %[[T10]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+// CHECK-NEXT:    %[[LHS_1:[0-9]+]] = arm_sve.dupq_lane %[[T11]][0] : vector<[16]xi8>
+
+// Extract sub-tiles from the RHS
+// CHECK-NEXT:    %[[T13:[0-9]+]] = vector.shape_cast %[[RHS]] : vector<[4]x8xi8> to vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_0:[0-9]+]] = vector.scalable.extract %[[T13]][0] : vector<[16]xi8> from vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_1:[0-9]+]] = vector.scalable.extract %[[T13]][16] : vector<[16]xi8> from vector<[32]xi8>
+
+// Extract accumulator rows 0 and 1 and pack (into "registers")
+// CHECK-NEXT:    %[[T16:[0-9]+]] = vector.extract %[[ACC]][0] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T17:[0-9]+]] = vector.extract %[[ACC]][1] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T18:[0-9]+]] = vector.bitcast %[[T16]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T19:[0-9]+]] = vector.bitcast %[[T17]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T20:[0-9]+]] = vector.interleave %[[T18]], %[[T19]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T21:[0-9]+]] = vector.bitcast %[[T20]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_0:[0-9]+]] = vector.scalable.extract %[[T21]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_1:[0-9]+]] = vector.scalable.extract %[[T21]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Same for accumulator rows 2 and 3
+// CHECK-NEXT:    %[[T24:[0-9]+]] = vector.extract %[[ACC]][2] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T25:[0-9]+]] = vector.extract %[[ACC]][3] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T26:[0-9]+]] = vector.bitcast %[[T24]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T27:[0-9]+]] = vector.bitcast %[[T25]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T28:[0-9]+]] = vector.interleave %[[T26]], %[[T27]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T29:[0-9]+]] = vector.bitcast %[[T28]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_2:[0-9]+]] = vector.scalable.extract %[[T29]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_3:[0-9]+]] = vector.scalable.extract %[[T29]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Do the sub-tile matrix multiplications
+// CHECK-NEXT:    %[[PACK_RES_00:[0-9]+]] = arm_sve.smmla %[[ACC_0]], %[[LHS_0]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_01:[0-9]+]] = arm_sve.smmla %[[ACC_1]], %[[LHS_0]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_10:[0-9]+]] = arm_sve.smmla %[[ACC_2]], %[[LHS_1]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_11:[0-9]+]] = arm_sve.smmla %[[ACC_3]], %[[LHS_1]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+
+// Unpack (from "registers") and insert in the output result rows 0 and 1
+// CHECK-NEXT:    %[[T36:[0-9]+]] = vector.scalable.insert %[[PACK_RES_00]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T37:[0-9]+]] = vector.scalable.insert %[[PACK_RES_01]], %[[T36]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T38:[0-9]+]] = vector.bitcast %[[T37]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1, %res2 = vector.deinterleave %[[T38]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_0:[0-9]+]] = vector.bitcast %res1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_1:[0-9]+]] = vector.bitcast %res2 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_0:[0-9]+]] = vector.insert %[[UNPACK_RES_0]], %[[P1]] [0] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_1:[0-9]+]] = vector.insert %[[UNPACK_RES_1]], %[[TMP_OUT_0]] [1] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// Same for result rows 2 and 3
+// CHECK-NEXT:    %[[T43:[0-9]+]] = vector.scalable.insert %[[PACK_RES_10]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T44:[0-9]+]] = vector.scalable.insert %[[PACK_RES_11]], %[[T43]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T45:[0-9]+]] = vector.bitcast %[[T44]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1_0, %res2_1 = vector.deinterleave %[[T45]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_2:[0-9]+]] = vector.bitcast %res1_0 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_3:[0-9]+]] = vector.bitcast %res2_1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_2:[0-9]+]] = vector.insert %[[UNPACK_RES_2]], %[[TMP_OUT_1]] [2] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[OUT:[0-9]+]] = vector.insert %[[UNPACK_RES_3]], %[[TMP_OUT_2]] [3] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// CHECK-NEXT:    return %[[OUT]] : vector<4x[4]xi32>
+
+// Test a variant where the sign-extension of the operands is
+// implicit. The output is identical to the one of the previous test.
+func.func @test_vector_contract_to_smmla_implicit_sext(%lhs: vector<4x8xi8>,
+              %rhs: vector<[4]x8xi8>,
+              %acc: vector<4x[4]xi32>) -> vector<4x[4]xi32> {
+
+  %0 = vector.contract #attrs %lhs, %rhs, %acc
+    : vector<4x8xi8>, vector<[4]x8xi8> into vector<4x[4]xi32>
+
+  return %0 : vector<4x[4]xi32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func {
+      transform.apply_patterns.arm_sve.vector_contract_to_i8mm
+    } : !transform.op<"func.func">
+
+    transform.yield
+  }
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-summla.mlir b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-summla.mlir
new file mode 100644
index 000000000000..518e995b698a
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-summla.mlir
@@ -0,0 +1,108 @@
+// RUN:  mlir-opt %s -transform-interpreter | FileCheck %s
+
+#packed_maps = [
+  affine_map<(d0, d1, d2) -> (d0, d2)>,
+  affine_map<(d0, d1, d2) -> (d1, d2)>,
+  affine_map<(d0, d1, d2) -> (d0, d1)>
+]
+
+// CHECK-LABEL: @test_vector_contract_to_usmmla_rev
+
+// CHECK-SAME: %[[LHS:arg0]]: vector<4x8xi8>
+// CHECK-SAME: %[[RHS:arg1]]: vector<[4]x8xi8>
+// CHECK-SAME: %[[ACC:arg2]]: vector<4x[4]xi32>
+
+// CHECK:         [[P0:[0-9]+]] = ub.poison : vector<[8]xi32>
+// CHECK-NEXT:    [[P1:[0-9]+]] = ub.poison : vector<4x[4]xi32>
+// CHECK-NEXT:    [[P2:[0-9]+]] = ub.poison : vector<[16]xi8>
+
+// Extract LHS rows 0 and 1, concatenate, turn into scalable vector
+// CHECK:         %[[T3:[0-9]+]] = vector.extract %[[LHS]][0] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T4:[0-9]+]] = vector.extract %[[LHS]][1] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T5:[0-9]+]] = vector.shuffle %[[T3]], %[[T4]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T6:[0-9]+]] = vector.scalable.insert %[[T5]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+
+// Replicate across the entire length of the scalable vector
+// CHECK-NEXT:    %[[LHS_0:[0-9]+]] = arm_sve.dupq_lane %[[T6]][0] : vector<[16]xi8>
+
+// Same for LHS rows 2 and 3
+// CHECK-NEXT:    %[[T8:[0-9]+]] = vector.extract %[[LHS]][2] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T9:[0-9]+]] = vector.extract %[[LHS]][3] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T10:[0-9]+]] = vector.shuffle %[[T8]], %[[T9]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T11:[0-9]+]] = vector.scalable.insert %[[T10]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+// CHECK-NEXT:    %[[LHS_1:[0-9]+]] = arm_sve.dupq_lane %[[T11]][0] : vector<[16]xi8>
+
+// Extract sub-tiles from the RHS
+// CHECK-NEXT:    %[[T13:[0-9]+]] = vector.shape_cast %[[RHS]] : vector<[4]x8xi8> to vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_0:[0-9]+]] = vector.scalable.extract %[[T13]][0] : vector<[16]xi8> from vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_1:[0-9]+]] = vector.scalable.extract %[[T13]][16] : vector<[16]xi8> from vector<[32]xi8>
+
+// Extract accumulator rows 0 and 1 and pack (into "registers")
+// Note the lack of bitcasts (i.e. interleave by pairs). This has the effect of transposing the tile.
+// CHECK-NEXT:    %[[T16:[0-9]+]] = vector.extract %[[ACC]][0] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T17:[0-9]+]] = vector.extract %[[ACC]][1] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T20:[0-9]+]] = vector.interleave %[[T16]], %[[T17]] : vector<[4]xi32> -> vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_0:[0-9]+]] = vector.scalable.extract %[[T20]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_1:[0-9]+]] = vector.scalable.extract %[[T20]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Same for accumulator rows 2 and 3
+// CHECK-NEXT:    %[[T24:[0-9]+]] = vector.extract %[[ACC]][2] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T25:[0-9]+]] = vector.extract %[[ACC]][3] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T28:[0-9]+]] = vector.interleave %[[T24]], %[[T25]] : vector<[4]xi32> -> vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_2:[0-9]+]] = vector.scalable.extract %[[T28]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_3:[0-9]+]] = vector.scalable.extract %[[T28]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Do the sub-tile matrix multiplications
+// The accumulator is transposed, the operands are in the opposite order,
+// thus the result is obtained transposed too.
+//
+// CHECK-NEXT:    %[[PACK_RES_00:[0-9]+]] = arm_sve.usmmla %[[ACC_0]], %[[RHS_0]], %[[LHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_01:[0-9]+]] = arm_sve.usmmla %[[ACC_1]], %[[RHS_1]], %[[LHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_10:[0-9]+]] = arm_sve.usmmla %[[ACC_2]], %[[RHS_0]], %[[LHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_11:[0-9]+]] = arm_sve.usmmla %[[ACC_3]], %[[RHS_1]], %[[LHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+
+// Unpack (from "registers") and insert in the output result rows 0 and 1
+// Again, no bitcast/interleave-by-pairs, i.e. the transposed result of sub-tile
+// multiplications is unpacked into the correct output layout.
+//
+// CHECK-NEXT:    %[[T36:[0-9]+]] = vector.scalable.insert %[[PACK_RES_00]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T37:[0-9]+]] = vector.scalable.insert %[[PACK_RES_01]], %[[T36]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_0:[_a-z0-9]+]], %[[UNPACK_RES_1:[_a-z0-9]+]] = vector.deinterleave %[[T37]] : vector<[8]xi32> -> vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_0:[0-9]+]] = vector.insert %[[UNPACK_RES_0]], %[[P1]] [0] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_1:[0-9]+]] = vector.insert %[[UNPACK_RES_1]], %[[TMP_OUT_0]] [1] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// Same for result rows 2 and 3
+// CHECK-NEXT:    %[[T43:[0-9]+]] = vector.scalable.insert %[[PACK_RES_10]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T44:[0-9]+]] = vector.scalable.insert %[[PACK_RES_11]], %[[T43]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_2:[_a-z0-9]+]], %[[UNPACK_RES_3:[_a-z0-9]+]] = vector.deinterleave %[[T44]] : vector<[8]xi32> -> vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_2:[0-9]+]] = vector.insert %[[UNPACK_RES_2]], %[[TMP_OUT_1]] [2] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[OUT:[0-9]+]] = vector.insert %[[UNPACK_RES_3]], %[[TMP_OUT_2]] [3] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// CHECK-NEXT:    return %[[OUT]] : vector<4x[4]xi32>
+
+func.func @test_vector_contract_to_usmmla_rev(
+  %lhs: vector<4x8xi8>,
+  %rhs: vector<[4]x8xi8>,
+  %acc: vector<4x[4]xi32>) -> vector<4x[4]xi32> {
+
+  %0 = arith.extsi %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extui %rhs : vector<[4]x8xi8> to vector<[4]x8xi32>
+  %2 = vector.contract {indexing_maps = #packed_maps,
+                        iterator_types = ["parallel", "parallel", "reduction"],
+                        kind = #vector.kind<add>} %0, %1, %acc
+    : vector<4x8xi32>, vector<[4]x8xi32> into vector<4x[4]xi32>
+
+  return %2 : vector<4x[4]xi32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func {
+      transform.apply_patterns.arm_sve.vector_contract_to_i8mm
+    } : !transform.op<"func.func">
+
+    transform.yield
+  }
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-ummla.mlir b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-ummla.mlir
new file mode 100644
index 000000000000..703182490f66
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-ummla.mlir
@@ -0,0 +1,112 @@
+// RUN:  mlir-opt %s -transform-interpreter --split-input-file | FileCheck %s
+
+#packed_maps = [
+  affine_map<(d0, d1, d2) -> (d0, d2)>,
+  affine_map<(d0, d1, d2) -> (d1, d2)>,
+  affine_map<(d0, d1, d2) -> (d0, d1)>
+]
+
+// CHECK-LABEL: @test_vector_contract_to_ummla
+
+// CHECK-SAME: %[[LHS:arg0]]: vector<4x8xi8>
+// CHECK-SAME: %[[RHS:arg1]]: vector<[4]x8xi8>
+// CHECK-SAME: %[[ACC:arg2]]: vector<4x[4]xi32>
+
+// CHECK:         [[P0:[0-9]+]] = ub.poison : vector<[8]xi32>
+// CHECK-NEXT:    [[P1:[0-9]+]] = ub.poison : vector<4x[4]xi32>
+// CHECK-NEXT:    [[P2:[0-9]+]] = ub.poison : vector<[16]xi8>
+
+// Extract LHS rows 0 and 1, concatenate, turn into scalable vector
+// CHECK:         %[[T3:[0-9]+]] = vector.extract %[[LHS]][0] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T4:[0-9]+]] = vector.extract %[[LHS]][1] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T5:[0-9]+]] = vector.shuffle %[[T3]], %[[T4]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T6:[0-9]+]] = vector.scalable.insert %[[T5]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+
+// Replicate across the entire length of the scalable vector
+// CHECK-NEXT:    %[[LHS_0:[0-9]+]] = arm_sve.dupq_lane %[[T6]][0] : vector<[16]xi8>
+
+// Same for LHS rows 2 and 3
+// CHECK-NEXT:    %[[T8:[0-9]+]] = vector.extract %[[LHS]][2] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T9:[0-9]+]] = vector.extract %[[LHS]][3] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T10:[0-9]+]] = vector.shuffle %[[T8]], %[[T9]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T11:[0-9]+]] = vector.scalable.insert %[[T10]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+// CHECK-NEXT:    %[[LHS_1:[0-9]+]] = arm_sve.dupq_lane %[[T11]][0] : vector<[16]xi8>
+
+// Extract sub-tiles from the RHS
+// CHECK-NEXT:    %[[T13:[0-9]+]] = vector.shape_cast %[[RHS]] : vector<[4]x8xi8> to vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_0:[0-9]+]] = vector.scalable.extract %[[T13]][0] : vector<[16]xi8> from vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_1:[0-9]+]] = vector.scalable.extract %[[T13]][16] : vector<[16]xi8> from vector<[32]xi8>
+
+// Extract accumulator rows 0 and 1 and pack (into "registers")
+// CHECK-NEXT:    %[[T16:[0-9]+]] = vector.extract %[[ACC]][0] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T17:[0-9]+]] = vector.extract %[[ACC]][1] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T18:[0-9]+]] = vector.bitcast %[[T16]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T19:[0-9]+]] = vector.bitcast %[[T17]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T20:[0-9]+]] = vector.interleave %[[T18]], %[[T19]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T21:[0-9]+]] = vector.bitcast %[[T20]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_0:[0-9]+]] = vector.scalable.extract %[[T21]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_1:[0-9]+]] = vector.scalable.extract %[[T21]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Same for accumulator rows 2 and 3
+// CHECK-NEXT:    %[[T24:[0-9]+]] = vector.extract %[[ACC]][2] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T25:[0-9]+]] = vector.extract %[[ACC]][3] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T26:[0-9]+]] = vector.bitcast %[[T24]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T27:[0-9]+]] = vector.bitcast %[[T25]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T28:[0-9]+]] = vector.interleave %[[T26]], %[[T27]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T29:[0-9]+]] = vector.bitcast %[[T28]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_2:[0-9]+]] = vector.scalable.extract %[[T29]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_3:[0-9]+]] = vector.scalable.extract %[[T29]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Do the sub-tile matrix multiplications
+// CHECK-NEXT:    %[[PACK_RES_00:[0-9]+]] = arm_sve.ummla %[[ACC_0]], %[[LHS_0]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_01:[0-9]+]] = arm_sve.ummla %[[ACC_1]], %[[LHS_0]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_10:[0-9]+]] = arm_sve.ummla %[[ACC_2]], %[[LHS_1]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_11:[0-9]+]] = arm_sve.ummla %[[ACC_3]], %[[LHS_1]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+
+// Unpack (from "registers") and insert in the output result rows 0 and 1
+// CHECK-NEXT:    %[[T36:[0-9]+]] = vector.scalable.insert %[[PACK_RES_00]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T37:[0-9]+]] = vector.scalable.insert %[[PACK_RES_01]], %[[T36]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T38:[0-9]+]] = vector.bitcast %[[T37]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1, %res2 = vector.deinterleave %[[T38]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_0:[0-9]+]] = vector.bitcast %res1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_1:[0-9]+]] = vector.bitcast %res2 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_0:[0-9]+]] = vector.insert %[[UNPACK_RES_0]], %[[P1]] [0] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_1:[0-9]+]] = vector.insert %[[UNPACK_RES_1]], %[[TMP_OUT_0]] [1] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// Same for result rows 2 and 3
+// CHECK-NEXT:    %[[T43:[0-9]+]] = vector.scalable.insert %[[PACK_RES_10]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T44:[0-9]+]] = vector.scalable.insert %[[PACK_RES_11]], %[[T43]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T45:[0-9]+]] = vector.bitcast %[[T44]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1_0, %res2_1 = vector.deinterleave %[[T45]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_2:[0-9]+]] = vector.bitcast %res1_0 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_3:[0-9]+]] = vector.bitcast %res2_1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_2:[0-9]+]] = vector.insert %[[UNPACK_RES_2]], %[[TMP_OUT_1]] [2] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[OUT:[0-9]+]] = vector.insert %[[UNPACK_RES_3]], %[[TMP_OUT_2]] [3] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// CHECK-NEXT:    return %[[OUT]] : vector<4x[4]xi32>
+
+func.func @test_vector_contract_to_ummla(%lhs: vector<4x8xi8>,
+              %rhs: vector<[4]x8xi8>,
+              %acc: vector<4x[4]xi32>) -> vector<4x[4]xi32> {
+
+  %0 = arith.extui %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extui %rhs : vector<[4]x8xi8> to vector<[4]x8xi32>
+  %2 = vector.contract {indexing_maps = #packed_maps,
+                        iterator_types = ["parallel", "parallel", "reduction"],
+                        kind = #vector.kind<add>} %0, %1, %acc
+    : vector<4x8xi32>, vector<[4]x8xi32> into vector<4x[4]xi32>
+
+  return %2 : vector<4x[4]xi32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func {
+      transform.apply_patterns.arm_sve.vector_contract_to_i8mm
+    } : !transform.op<"func.func">
+
+    transform.yield
+  }
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-usmmla.mlir b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-usmmla.mlir
new file mode 100644
index 000000000000..28156f6ef270
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/Vector/CPU/ArmSVE/vector-usmmla.mlir
@@ -0,0 +1,113 @@
+// RUN:  mlir-opt %s -transform-interpreter | FileCheck %s
+
+#packed_maps = [
+  affine_map<(d0, d1, d2) -> (d0, d2)>,
+  affine_map<(d0, d1, d2) -> (d1, d2)>,
+  affine_map<(d0, d1, d2) -> (d0, d1)>
+]
+
+// CHECK-LABEL: @test_vector_contract_to_usmmla
+
+// CHECK-SAME: %[[LHS:arg0]]: vector<4x8xi8>
+// CHECK-SAME: %[[RHS:arg1]]: vector<[4]x8xi8>
+// CHECK-SAME: %[[ACC:arg2]]: vector<4x[4]xi32>
+
+// CHECK:         [[P0:[0-9]+]] = ub.poison : vector<[8]xi32>
+// CHECK-NEXT:    [[P1:[0-9]+]] = ub.poison : vector<4x[4]xi32>
+// CHECK-NEXT:    [[P2:[0-9]+]] = ub.poison : vector<[16]xi8>
+
+// Extract LHS rows 0 and 1, concatenate, turn into scalable vector
+// CHECK:         %[[T3:[0-9]+]] = vector.extract %[[LHS]][0] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T4:[0-9]+]] = vector.extract %[[LHS]][1] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T5:[0-9]+]] = vector.shuffle %[[T3]], %[[T4]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T6:[0-9]+]] = vector.scalable.insert %[[T5]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+
+// Replicate across the entire length of the scalable vector
+// CHECK-NEXT:    %[[LHS_0:[0-9]+]] = arm_sve.dupq_lane %[[T6]][0] : vector<[16]xi8>
+
+// Same for LHS rows 2 and 3
+// CHECK-NEXT:    %[[T8:[0-9]+]] = vector.extract %[[LHS]][2] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T9:[0-9]+]] = vector.extract %[[LHS]][3] : vector<8xi8> from vector<4x8xi8>
+// CHECK-NEXT:    %[[T10:[0-9]+]] = vector.shuffle %[[T8]], %[[T9]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : vector<8xi8>, vector<8xi8>
+// CHECK-NEXT:    %[[T11:[0-9]+]] = vector.scalable.insert %[[T10]], %[[P2]][0] : vector<16xi8> into vector<[16]xi8>
+// CHECK-NEXT:    %[[LHS_1:[0-9]+]] = arm_sve.dupq_lane %[[T11]][0] : vector<[16]xi8>
+
+// Extract sub-tiles from the RHS
+// CHECK-NEXT:    %[[T13:[0-9]+]] = vector.shape_cast %[[RHS]] : vector<[4]x8xi8> to vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_0:[0-9]+]] = vector.scalable.extract %[[T13]][0] : vector<[16]xi8> from vector<[32]xi8>
+// CHECK-NEXT:    %[[RHS_1:[0-9]+]] = vector.scalable.extract %[[T13]][16] : vector<[16]xi8> from vector<[32]xi8>
+
+// Extract accumulator rows 0 and 1 and pack (into "registers")
+// CHECK-NEXT:    %[[T16:[0-9]+]] = vector.extract %[[ACC]][0] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T17:[0-9]+]] = vector.extract %[[ACC]][1] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T18:[0-9]+]] = vector.bitcast %[[T16]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T19:[0-9]+]] = vector.bitcast %[[T17]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T20:[0-9]+]] = vector.interleave %[[T18]], %[[T19]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T21:[0-9]+]] = vector.bitcast %[[T20]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_0:[0-9]+]] = vector.scalable.extract %[[T21]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_1:[0-9]+]] = vector.scalable.extract %[[T21]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Same for accumulator rows 2 and 3
+// CHECK-NEXT:    %[[T24:[0-9]+]] = vector.extract %[[ACC]][2] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T25:[0-9]+]] = vector.extract %[[ACC]][3] : vector<[4]xi32> from vector<4x[4]xi32>
+// CHECK-NEXT:    %[[T26:[0-9]+]] = vector.bitcast %[[T24]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T27:[0-9]+]] = vector.bitcast %[[T25]] : vector<[4]xi32> to vector<[2]xi64>
+// CHECK-NEXT:    %[[T28:[0-9]+]] = vector.interleave %[[T26]], %[[T27]] : vector<[2]xi64> -> vector<[4]xi64>
+// CHECK-NEXT:    %[[T29:[0-9]+]] = vector.bitcast %[[T28]] : vector<[4]xi64> to vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_2:[0-9]+]] = vector.scalable.extract %[[T29]][0] : vector<[4]xi32> from vector<[8]xi32>
+// CHECK-NEXT:    %[[ACC_3:[0-9]+]] = vector.scalable.extract %[[T29]][4] : vector<[4]xi32> from vector<[8]xi32>
+
+// Do the sub-tile matrix multiplications
+// CHECK-NEXT:    %[[PACK_RES_00:[0-9]+]] = arm_sve.usmmla %[[ACC_0]], %[[LHS_0]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_01:[0-9]+]] = arm_sve.usmmla %[[ACC_1]], %[[LHS_0]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_10:[0-9]+]] = arm_sve.usmmla %[[ACC_2]], %[[LHS_1]], %[[RHS_0]] : vector<[16]xi8> to vector<[4]xi32>
+// CHECK-NEXT:    %[[PACK_RES_11:[0-9]+]] = arm_sve.usmmla %[[ACC_3]], %[[LHS_1]], %[[RHS_1]] : vector<[16]xi8> to vector<[4]xi32>
+
+// Unpack (from "registers") and insert in the output result rows 0 and 1
+// CHECK-NEXT:    %[[T36:[0-9]+]] = vector.scalable.insert %[[PACK_RES_00]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T37:[0-9]+]] = vector.scalable.insert %[[PACK_RES_01]], %[[T36]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T38:[0-9]+]] = vector.bitcast %[[T37]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1, %res2 = vector.deinterleave %[[T38]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_0:[0-9]+]] = vector.bitcast %res1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_1:[0-9]+]] = vector.bitcast %res2 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_0:[0-9]+]] = vector.insert %[[UNPACK_RES_0]], %[[P1]] [0] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_1:[0-9]+]] = vector.insert %[[UNPACK_RES_1]], %[[TMP_OUT_0]] [1] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// Same for result rows 2 and 3
+// CHECK-NEXT:    %[[T43:[0-9]+]] = vector.scalable.insert %[[PACK_RES_10]], %[[P0]][0] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T44:[0-9]+]] = vector.scalable.insert %[[PACK_RES_11]], %[[T43]][4] : vector<[4]xi32> into vector<[8]xi32>
+// CHECK-NEXT:    %[[T45:[0-9]+]] = vector.bitcast %[[T44]] : vector<[8]xi32> to vector<[4]xi64>
+// CHECK-NEXT:    %res1_0, %res2_1 = vector.deinterleave %[[T45]] : vector<[4]xi64> -> vector<[2]xi64>
+// CHECK-NEXT:    %[[UNPACK_RES_2:[0-9]+]] = vector.bitcast %res1_0 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[UNPACK_RES_3:[0-9]+]] = vector.bitcast %res2_1 : vector<[2]xi64> to vector<[4]xi32>
+// CHECK-NEXT:    %[[TMP_OUT_2:[0-9]+]] = vector.insert %[[UNPACK_RES_2]], %[[TMP_OUT_1]] [2] : vector<[4]xi32> into vector<4x[4]xi32>
+// CHECK-NEXT:    %[[OUT:[0-9]+]] = vector.insert %[[UNPACK_RES_3]], %[[TMP_OUT_2]] [3] : vector<[4]xi32> into vector<4x[4]xi32>
+
+// CHECK-NEXT:    return %[[OUT]] : vector<4x[4]xi32>
+
+func.func @test_vector_contract_to_usmmla(
+  %lhs: vector<4x8xi8>,
+  %rhs: vector<[4]x8xi8>,
+  %acc: vector<4x[4]xi32>) -> vector<4x[4]xi32> {
+
+  %0 = arith.extui %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extsi %rhs : vector<[4]x8xi8> to vector<[4]x8xi32>
+  %2 = vector.contract {indexing_maps = #packed_maps,
+                        iterator_types = ["parallel", "parallel", "reduction"],
+                        kind = #vector.kind<add>} %0, %1, %acc
+    : vector<4x8xi32>, vector<[4]x8xi32> into vector<4x[4]xi32>
+
+  return %2 : vector<4x[4]xi32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func {
+      transform.apply_patterns.arm_sve.vector_contract_to_i8mm
+    } : !transform.op<"func.func">
+
+    transform.yield
+  }
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/canonicalize.mlir b/external/llvm-project/mlir/test/Dialect/Vector/canonicalize.mlir
index a6543aafd1c7..6691cb52acdc 100644
--- a/external/llvm-project/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -408,7 +408,7 @@ func.func @extract_strided_fold_insert(%a: vector<6x4xf32>, %b: vector<8x16xf32>
 // -----
 
 // Negative test where the extract is not a subset of the element inserted.
-// CHECK-LABEL: extract_strided_fold_negative
+// CHECK-LABEL: negative_extract_strided_fold
 //  CHECK-SAME: (%[[ARG0:.*]]: vector<4x4xf32>, %[[ARG1:.*]]: vector<8x16xf32>
 //       CHECK:   %[[INS:.*]] = vector.insert_strided_slice %[[ARG0]], %[[ARG1]]
 //  CHECK-SAME:     {offsets = [2, 2], strides = [1, 1]}
@@ -417,7 +417,7 @@ func.func @extract_strided_fold_insert(%a: vector<6x4xf32>, %b: vector<8x16xf32>
 //  CHECK-SAME:     {offsets = [2, 2], sizes = [6, 4], strides = [1, 1]}
 //  CHECK-SAME:       : vector<8x16xf32> to vector<6x4xf32>
 //  CHECK-NEXT:   return %[[EXT]] : vector<6x4xf32>
-func.func @extract_strided_fold_negative(%a: vector<4x4xf32>, %b: vector<8x16xf32>)
+func.func @negative_extract_strided_fold(%a: vector<4x4xf32>, %b: vector<8x16xf32>)
   -> (vector<6x4xf32>) {
   %0 = vector.insert_strided_slice %a, %b {offsets = [2, 2], strides = [1, 1]}
     : vector<4x4xf32> into vector<8x16xf32>
@@ -753,10 +753,10 @@ func.func @fold_extract_broadcast_0dvec_input_scalar_output(%a : vector<f32>,
 
 // -----
 
-// CHECK-LABEL: fold_extract_broadcast_negative
+// CHECK-LABEL: negative_fold_extract_broadcast
 //       CHECK:   vector.broadcast %{{.*}} : vector<1x1xf32> to vector<1x1x4xf32>
 //       CHECK:   vector.extract %{{.*}}[0, 0] : vector<4xf32> from vector<1x1x4xf32>
-func.func @fold_extract_broadcast_negative(%a : vector<1x1xf32>) -> vector<4xf32> {
+func.func @negative_fold_extract_broadcast(%a : vector<1x1xf32>) -> vector<4xf32> {
   %b = vector.broadcast %a : vector<1x1xf32> to vector<1x1x4xf32>
   %r = vector.extract %b[0, 0] : vector<4xf32> from vector<1x1x4xf32>
   return %r : vector<4xf32>
@@ -895,11 +895,11 @@ func.func @fold_extract_shapecast_0d_source(%arg0 : vector<f32>) -> f32 {
 
 // -----
 
-// CHECK-LABEL: fold_extract_shapecast_negative
+// CHECK-LABEL: negative_fold_extract_shapecast
 //       CHECK:   %[[V:.*]] = vector.shape_cast %{{.*}} : vector<16xf32> to vector<2x4x2xf32>
 //       CHECK:   %[[R:.*]] = vector.extract %[[V]][1] : vector<4x2xf32> from vector<2x4x2xf32>
 //       CHECK:   return %[[R]] : vector<4x2xf32>
-func.func @fold_extract_shapecast_negative(%arg0 : vector<16xf32>) -> vector<4x2xf32> {
+func.func @negative_fold_extract_shapecast(%arg0 : vector<16xf32>) -> vector<4x2xf32> {
   %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<2x4x2xf32>
   %r = vector.extract %0[1] : vector<4x2xf32> from vector<2x4x2xf32>
   return %r : vector<4x2xf32>
@@ -1460,11 +1460,11 @@ func.func @store_after_load_tensor(%arg0 : tensor<4x4xf32>) -> tensor<4x4xf32> {
 
 // -----
 
-// CHECK-LABEL: func @store_after_load_tensor_negative
+// CHECK-LABEL: func @negative_store_after_load_tensor
 //       CHECK:   vector.transfer_read
 //       CHECK:   vector.transfer_write
 //       CHECK:   return
-func.func @store_after_load_tensor_negative(%arg0 : tensor<4x4xf32>) -> tensor<4x4xf32> {
+func.func @negative_store_after_load_tensor(%arg0 : tensor<4x4xf32>) -> tensor<4x4xf32> {
   %c1 = arith.constant 1 : index
   %c0 = arith.constant 0 : index
   %cf0 = arith.constant 0.0 : f32
@@ -1499,12 +1499,12 @@ func.func @store_to_load_tensor(%arg0 : tensor<4x4xf32>,
 
 // -----
 
-// CHECK-LABEL: func @store_to_load_negative_tensor
+// CHECK-LABEL: func @negative_store_to_load_tensor
 //       CHECK:   vector.transfer_write
 //       CHECK:   vector.transfer_write
 //       CHECK:   %[[V:.*]] = vector.transfer_read
 //       CHECK:   return %[[V]] : vector<1x4xf32>
-func.func @store_to_load_negative_tensor(%arg0 : tensor<4x4xf32>,
+func.func @negative_store_to_load_tensor(%arg0 : tensor<4x4xf32>,
   %v0 : vector<1x4xf32>, %v1 : vector<1x4xf32>, %i : index) -> vector<1x4xf32> {
   %c1 = arith.constant 1 : index
   %c2 = arith.constant 2 : index
@@ -1540,6 +1540,86 @@ func.func @store_to_load_tensor_broadcast(%arg0 : tensor<4x4xf32>,
 
 // -----
 
+// CHECK-LABEL: func @negative_store_to_load_tensor_memref
+//   CHECK-NOT:   vector.broadcast
+//   CHECK-NOT:   vector.transpose
+//       CHECK:   vector.transfer_write
+//       CHECK:   vector.transfer_read
+func.func @negative_store_to_load_tensor_memref(
+    %arg0 : tensor<?x?xf32>,
+    %arg1 : memref<?x?xf32>,
+    %v0 : vector<4x2xf32>
+  ) -> vector<4x2xf32> 
+{
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  vector.transfer_write %v0, %arg1[%c0, %c0] {in_bounds = [true, true]} :
+    vector<4x2xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %arg0[%c0, %c0], %cf0 {in_bounds = [true, true]} :
+    tensor<?x?xf32>, vector<4x2xf32>
+  return %0 : vector<4x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_store_to_load_tensor_no_actual_broadcast
+//   CHECK-NOT:   vector.broadcast
+//   CHECK-NOT:   vector.transpose
+//       CHECK:   vector.transfer_write
+//       CHECK:   vector.transfer_read
+func.func @negative_store_to_load_tensor_no_actual_broadcast(%arg0 : tensor<?x?xf32>,
+  %v0 : vector<4x2xf32>) -> vector<4x2xf32> {
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  %w0 = vector.transfer_write %v0, %arg0[%c0, %c0] :
+    vector<4x2xf32>, tensor<?x?xf32>
+  %0 = vector.transfer_read %w0[%c0, %c0], %cf0 {in_bounds = [true, true]} :
+    tensor<?x?xf32>, vector<4x2xf32>
+  return %0 : vector<4x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_store_to_load_tensor_broadcast_out_of_bounds
+//   CHECK-NOT:   vector.broadcast
+//   CHECK-NOT:   vector.transpose
+//       CHECK:   vector.transfer_write
+//       CHECK:   vector.transfer_read
+func.func @negative_store_to_load_tensor_broadcast_out_of_bounds(%arg0 : tensor<?x?xf32>,
+  %v0 : vector<4x2xf32>) -> vector<4x2x6xf32> {
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  %w0 = vector.transfer_write %v0, %arg0[%c0, %c0] :
+    vector<4x2xf32>, tensor<?x?xf32>
+  %0 = vector.transfer_read %w0[%c0, %c0], %cf0 {in_bounds = [true, true, true],
+  permutation_map = affine_map<(d0, d1) -> (d0, d1, 0)>} :
+    tensor<?x?xf32>, vector<4x2x6xf32>
+  return %0 : vector<4x2x6xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_store_to_load_tensor_broadcast_masked
+//   CHECK-NOT:   vector.broadcast
+//   CHECK-NOT:   vector.transpose
+//       CHECK:   vector.transfer_write
+//       CHECK:   vector.transfer_read
+func.func @negative_store_to_load_tensor_broadcast_masked(
+    %arg0 : tensor<?x?xf32>, %v0 : vector<4x2xf32>, %mask : vector<4x2xi1>)
+  -> vector<4x2x6xf32> 
+{
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  %w0 = vector.transfer_write %v0, %arg0[%c0, %c0], %mask {in_bounds = [true, true]} :
+    vector<4x2xf32>, tensor<?x?xf32>
+  %0 = vector.transfer_read %w0[%c0, %c0], %cf0 {in_bounds = [true, true, true],
+  permutation_map = affine_map<(d0, d1) -> (d0, d1, 0)>} :
+    tensor<?x?xf32>, vector<4x2x6xf32>
+  return %0 : vector<4x2x6xf32>
+}
+
+// -----
+
 // CHECK-LABEL: func @store_to_load_tensor_broadcast_scalable
 //  CHECK-SAME: (%[[ARG:.*]]: tensor<?xf32>, %[[V0:.*]]: vector<[4]xf32>)
 //       CHECK:   %[[B:.*]] = vector.broadcast %[[V0]] : vector<[4]xf32> to vector<6x[4]xf32>
@@ -1604,7 +1684,7 @@ func.func @dead_store_tensor(%arg0 : tensor<4x4xf32>,
 
 // -----
 
-// CHECK-LABEL: func @dead_store_tensor_negative
+// CHECK-LABEL: func @negative_dead_store_tensor
 //   CHECK-DAG:      %[[C0:.*]] = arith.constant 0 : index
 //   CHECK-DAG:      %[[C1:.*]] = arith.constant 1 : index
 //       CHECK:   vector.transfer_write
@@ -1612,7 +1692,7 @@ func.func @dead_store_tensor(%arg0 : tensor<4x4xf32>,
 //       CHECK:   vector.transfer_read
 //       CHECK:   %[[VTW:.*]] = vector.transfer_write {{.*}}, {{.*}}[%[[C1]], %[[C0]]]
 //       CHECK:   return %[[VTW]] : tensor<4x4xf32>
-func.func @dead_store_tensor_negative(%arg0 : tensor<4x4xf32>,
+func.func @negative_dead_store_tensor(%arg0 : tensor<4x4xf32>,
   %v0 : vector<1x4xf32>, %v1 : vector<1x4xf32>, %i : index) -> tensor<4x4xf32> {
   %c1 = arith.constant 1 : index
   %c2 = arith.constant 2 : index
@@ -2063,10 +2143,10 @@ func.func @extract_insert_rank_reduce(%a: vector<4xf32>, %b: vector<8x16xf32>)
 
 // -----
 
-// CHECK-LABEL: extract_insert_negative
+// CHECK-LABEL: negative_extract_insert
 //       CHECK: vector.insert_strided_slice
 //       CHECK: vector.extract
-func.func @extract_insert_negative(%a: vector<2x15xf32>, %b: vector<12x8x16xf32>)
+func.func @negative_extract_insert(%a: vector<2x15xf32>, %b: vector<12x8x16xf32>)
   -> vector<16xf32> {
   %0 = vector.insert_strided_slice %a, %b {offsets = [4, 2, 0], strides = [1, 1]}
     : vector<2x15xf32> into vector<12x8x16xf32>
@@ -2943,75 +3023,6 @@ func.func @extract_from_0d_splat_broadcast_regression(%a: f32, %b: vector<f32>,
 
 // -----
 
-// CHECK-LABEL: func @extract_scalar_from_from_elements(
-//  CHECK-SAME:     %[[a:.*]]: f32, %[[b:.*]]: f32)
-func.func @extract_scalar_from_from_elements(%a: f32, %b: f32) -> (f32, f32, f32, f32, f32, f32, f32) {
-  // Extract from 0D.
-  %0 = vector.from_elements %a : vector<f32>
-  %1 = vector.extract %0[] : f32 from vector<f32>
-
-  // Extract from 1D.
-  %2 = vector.from_elements %a : vector<1xf32>
-  %3 = vector.extract %2[0] : f32 from vector<1xf32>
-  %4 = vector.from_elements %a, %b, %a, %a, %b : vector<5xf32>
-  %5 = vector.extract %4[4] : f32 from vector<5xf32>
-
-  // Extract from 2D.
-  %6 = vector.from_elements %a, %a, %a, %b, %b, %b : vector<2x3xf32>
-  %7 = vector.extract %6[0, 0] : f32 from vector<2x3xf32>
-  %8 = vector.extract %6[0, 1] : f32 from vector<2x3xf32>
-  %9 = vector.extract %6[1, 1] : f32 from vector<2x3xf32>
-  %10 = vector.extract %6[1, 2] : f32 from vector<2x3xf32>
-
-  // CHECK: return %[[a]], %[[a]], %[[b]], %[[a]], %[[a]], %[[b]], %[[b]]
-  return %1, %3, %5, %7, %8, %9, %10 : f32, f32, f32, f32, f32, f32, f32
-}
-
-// -----
-
-// CHECK-LABEL: func @extract_1d_from_from_elements(
-//  CHECK-SAME:     %[[a:.*]]: f32, %[[b:.*]]: f32)
-func.func @extract_1d_from_from_elements(%a: f32, %b: f32) -> (vector<3xf32>, vector<3xf32>) {
-  %0 = vector.from_elements %a, %a, %a, %b, %b, %b : vector<2x3xf32>
-  // CHECK: %[[splat1:.*]] = vector.splat %[[a]] : vector<3xf32>
-  %1 = vector.extract %0[0] : vector<3xf32> from vector<2x3xf32>
-  // CHECK: %[[splat2:.*]] = vector.splat %[[b]] : vector<3xf32>
-  %2 = vector.extract %0[1] : vector<3xf32> from vector<2x3xf32>
-  // CHECK: return %[[splat1]], %[[splat2]]
-  return %1, %2 : vector<3xf32>, vector<3xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @extract_2d_from_from_elements(
-//  CHECK-SAME:     %[[a:.*]]: f32, %[[b:.*]]: f32)
-func.func @extract_2d_from_from_elements(%a: f32, %b: f32) -> (vector<2x2xf32>, vector<2x2xf32>) {
-  %0 = vector.from_elements %a, %a, %a, %b, %b, %b, %b, %a, %b, %a, %a, %b : vector<3x2x2xf32>
-  // CHECK: %[[splat1:.*]] = vector.from_elements %[[a]], %[[a]], %[[a]], %[[b]] : vector<2x2xf32>
-  %1 = vector.extract %0[0] : vector<2x2xf32> from vector<3x2x2xf32>
-  // CHECK: %[[splat2:.*]] = vector.from_elements %[[b]], %[[b]], %[[b]], %[[a]] : vector<2x2xf32>
-  %2 = vector.extract %0[1] : vector<2x2xf32> from vector<3x2x2xf32>
-  // CHECK: return %[[splat1]], %[[splat2]]
-  return %1, %2 : vector<2x2xf32>, vector<2x2xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @from_elements_to_splat(
-//  CHECK-SAME:     %[[a:.*]]: f32, %[[b:.*]]: f32)
-func.func @from_elements_to_splat(%a: f32, %b: f32) -> (vector<2x3xf32>, vector<2x3xf32>, vector<f32>) {
-  // CHECK: %[[splat:.*]] = vector.splat %[[a]] : vector<2x3xf32>
-  %0 = vector.from_elements %a, %a, %a, %a, %a, %a : vector<2x3xf32>
-  // CHECK: %[[from_el:.*]] = vector.from_elements {{.*}} : vector<2x3xf32>
-  %1 = vector.from_elements %a, %a, %a, %a, %b, %a : vector<2x3xf32>
-  // CHECK: %[[splat2:.*]] = vector.splat %[[a]] : vector<f32>
-  %2 = vector.from_elements %a : vector<f32>
-  // CHECK: return %[[splat]], %[[from_el]], %[[splat2]]
-  return %0, %1, %2 : vector<2x3xf32>, vector<2x3xf32>, vector<f32>
-}
-
-// -----
-
 // CHECK-LABEL: func @vector_insert_const_regression(
 //       CHECK:   llvm.mlir.undef
 //       CHECK:   vector.insert
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/canonicalize/vector-from-elements.mlir b/external/llvm-project/mlir/test/Dialect/Vector/canonicalize/vector-from-elements.mlir
new file mode 100644
index 000000000000..fdab2a8918a2
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/Vector/canonicalize/vector-from-elements.mlir
@@ -0,0 +1,268 @@
+// RUN: mlir-opt %s -canonicalize="test-convergence" -split-input-file -allow-unregistered-dialect | FileCheck %s
+
+// This file contains some tests of folding/canonicalizing vector.from_elements
+
+///===----------------------------------------------===//
+///  Tests of `rewriteFromElementsAsSplat`
+///===----------------------------------------------===//
+
+// CHECK-LABEL: func @extract_scalar_from_from_elements(
+//  CHECK-SAME:      %[[A:.*]]: f32, %[[B:.*]]: f32)
+func.func @extract_scalar_from_from_elements(%a: f32, %b: f32) -> (f32, f32, f32, f32, f32, f32, f32) {
+  // Extract from 0D.
+  %0 = vector.from_elements %a : vector<f32>
+  %1 = vector.extract %0[] : f32 from vector<f32>
+
+  // Extract from 1D.
+  %2 = vector.from_elements %a : vector<1xf32>
+  %3 = vector.extract %2[0] : f32 from vector<1xf32>
+  %4 = vector.from_elements %a, %b, %a, %a, %b : vector<5xf32>
+  %5 = vector.extract %4[4] : f32 from vector<5xf32>
+
+  // Extract from 2D.
+  %6 = vector.from_elements %a, %a, %a, %b, %b, %b : vector<2x3xf32>
+  %7 = vector.extract %6[0, 0] : f32 from vector<2x3xf32>
+  %8 = vector.extract %6[0, 1] : f32 from vector<2x3xf32>
+  %9 = vector.extract %6[1, 1] : f32 from vector<2x3xf32>
+  %10 = vector.extract %6[1, 2] : f32 from vector<2x3xf32>
+
+  // CHECK: return %[[A]], %[[A]], %[[B]], %[[A]], %[[A]], %[[B]], %[[B]]
+  return %1, %3, %5, %7, %8, %9, %10 : f32, f32, f32, f32, f32, f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: func @extract_1d_from_from_elements(
+//  CHECK-SAME:      %[[A:.*]]: f32, %[[B:.*]]: f32)
+func.func @extract_1d_from_from_elements(%a: f32, %b: f32) -> (vector<3xf32>, vector<3xf32>) {
+  %0 = vector.from_elements %a, %a, %a, %b, %b, %b : vector<2x3xf32>
+  // CHECK: %[[SPLAT1:.*]] = vector.splat %[[A]] : vector<3xf32>
+  %1 = vector.extract %0[0] : vector<3xf32> from vector<2x3xf32>
+  // CHECK: %[[SPLAT2:.*]] = vector.splat %[[B]] : vector<3xf32>
+  %2 = vector.extract %0[1] : vector<3xf32> from vector<2x3xf32>
+  // CHECK: return %[[SPLAT1]], %[[SPLAT2]]
+  return %1, %2 : vector<3xf32>, vector<3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @extract_2d_from_from_elements(
+//  CHECK-SAME:      %[[A:.*]]: f32, %[[B:.*]]: f32)
+func.func @extract_2d_from_from_elements(%a: f32, %b: f32) -> (vector<2x2xf32>, vector<2x2xf32>) {
+  %0 = vector.from_elements %a, %a, %a, %b, %b, %b, %b, %a, %b, %a, %a, %b : vector<3x2x2xf32>
+  // CHECK: %[[SPLAT1:.*]] = vector.from_elements %[[A]], %[[A]], %[[A]], %[[B]] : vector<2x2xf32>
+  %1 = vector.extract %0[0] : vector<2x2xf32> from vector<3x2x2xf32>
+  // CHECK: %[[SPLAT2:.*]] = vector.from_elements %[[B]], %[[B]], %[[B]], %[[A]] : vector<2x2xf32>
+  %2 = vector.extract %0[1] : vector<2x2xf32> from vector<3x2x2xf32>
+  // CHECK: return %[[SPLAT1]], %[[SPLAT2]]
+  return %1, %2 : vector<2x2xf32>, vector<2x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @from_elements_to_splat(
+//  CHECK-SAME:      %[[A:.*]]: f32, %[[B:.*]]: f32)
+func.func @from_elements_to_splat(%a: f32, %b: f32) -> (vector<2x3xf32>, vector<2x3xf32>, vector<f32>) {
+  // CHECK: %[[SPLAT:.*]] = vector.splat %[[A]] : vector<2x3xf32>
+  %0 = vector.from_elements %a, %a, %a, %a, %a, %a : vector<2x3xf32>
+  // CHECK: %[[FROM_EL:.*]] = vector.from_elements {{.*}} : vector<2x3xf32>
+  %1 = vector.from_elements %a, %a, %a, %a, %b, %a : vector<2x3xf32>
+  // CHECK: %[[SPLAT2:.*]] = vector.splat %[[A]] : vector<f32>
+  %2 = vector.from_elements %a : vector<f32>
+  // CHECK: return %[[SPLAT]], %[[FROM_EL]], %[[SPLAT2]]
+  return %0, %1, %2 : vector<2x3xf32>, vector<2x3xf32>, vector<f32>
+}
+
+// -----
+
+///===----------------------------------------------===//
+///  Tests of `FromElementsToShapeCast`
+///===----------------------------------------------===//
+
+// CHECK-LABEL: func @to_shape_cast_rank2_to_rank1(
+//  CHECK-SAME:       %[[A:.*]]: vector<1x2xi8>)
+//       CHECK:       %[[EXTRACT:.*]] = vector.extract %[[A]][0] : vector<2xi8> from vector<1x2xi8>
+//       CHECK:       return %[[EXTRACT]] : vector<2xi8>
+func.func @to_shape_cast_rank2_to_rank1(%arg0: vector<1x2xi8>) -> vector<2xi8> {
+  %0 = vector.extract %arg0[0, 0] : i8 from vector<1x2xi8>
+  %1 = vector.extract %arg0[0, 1] : i8 from vector<1x2xi8>
+  %4 = vector.from_elements %0, %1 : vector<2xi8>
+  return %4 : vector<2xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @to_shape_cast_rank1_to_rank3(
+//  CHECK-SAME:       %[[A:.*]]: vector<8xi8>)
+//       CHECK:       %[[SHAPE_CAST:.*]] = vector.shape_cast %[[A]] : vector<8xi8> to vector<2x2x2xi8>
+//       CHECK:       return %[[SHAPE_CAST]] : vector<2x2x2xi8>
+func.func @to_shape_cast_rank1_to_rank3(%arg0: vector<8xi8>) -> vector<2x2x2xi8> {
+  %0 = vector.extract %arg0[0] : i8 from vector<8xi8>
+  %1 = vector.extract %arg0[1] : i8 from vector<8xi8>
+  %2 = vector.extract %arg0[2] : i8 from vector<8xi8>
+  %3 = vector.extract %arg0[3] : i8 from vector<8xi8>
+  %4 = vector.extract %arg0[4] : i8 from vector<8xi8>
+  %5 = vector.extract %arg0[5] : i8 from vector<8xi8>
+  %6 = vector.extract %arg0[6] : i8 from vector<8xi8>
+  %7 = vector.extract %arg0[7] : i8 from vector<8xi8>
+  %8 = vector.from_elements %0, %1, %2, %3, %4, %5, %6, %7 : vector<2x2x2xi8>
+  return %8 : vector<2x2x2xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @source_larger_than_out(
+//  CHECK-SAME:      %[[A:.*]]: vector<2x3x4xi8>)
+//       CHECK:      %[[EXTRACT:.*]] = vector.extract %[[A]][1] : vector<3x4xi8> from vector<2x3x4xi8>
+//       CHECK:      %[[SHAPE_CAST:.*]] = vector.shape_cast %[[EXTRACT]] : vector<3x4xi8> to vector<12xi8>
+//       CHECK:      return %[[SHAPE_CAST]] : vector<12xi8>
+func.func @source_larger_than_out(%arg0: vector<2x3x4xi8>) -> vector<12xi8> {
+  %0 = vector.extract %arg0[1, 0, 0] : i8 from vector<2x3x4xi8>
+  %1 = vector.extract %arg0[1, 0, 1] : i8 from vector<2x3x4xi8>
+  %2 = vector.extract %arg0[1, 0, 2] : i8 from vector<2x3x4xi8>
+  %3 = vector.extract %arg0[1, 0, 3] : i8 from vector<2x3x4xi8>
+  %4 = vector.extract %arg0[1, 1, 0] : i8 from vector<2x3x4xi8>
+  %5 = vector.extract %arg0[1, 1, 1] : i8 from vector<2x3x4xi8>
+  %6 = vector.extract %arg0[1, 1, 2] : i8 from vector<2x3x4xi8>
+  %7 = vector.extract %arg0[1, 1, 3] : i8 from vector<2x3x4xi8>
+  %8 = vector.extract %arg0[1, 2, 0] : i8 from vector<2x3x4xi8>
+  %9 = vector.extract %arg0[1, 2, 1] : i8 from vector<2x3x4xi8>
+  %10 = vector.extract %arg0[1, 2, 2] : i8 from vector<2x3x4xi8>
+  %11 = vector.extract %arg0[1, 2, 3] : i8 from vector<2x3x4xi8>
+  %12 = vector.from_elements %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11 : vector<12xi8>
+  return %12 : vector<12xi8>
+}
+
+// -----
+
+// This test is similar to `source_larger_than_out` except here the number of elements
+// extracted contigously starting from the first position [0,0] could be 6 instead of 3
+// and the pattern would still match.
+// CHECK-LABEL: func @suffix_with_excess_zeros(
+//       CHECK:      %[[EXT:.*]] = vector.extract {{.*}}[0] : vector<3xi8> from vector<2x3xi8>
+//       CHECK:      return %[[EXT]] : vector<3xi8>
+func.func @suffix_with_excess_zeros(%arg0: vector<2x3xi8>) -> vector<3xi8> {
+  %0 = vector.extract %arg0[0, 0] : i8 from vector<2x3xi8>
+  %1 = vector.extract %arg0[0, 1] : i8 from vector<2x3xi8>
+  %2 = vector.extract %arg0[0, 2] : i8 from vector<2x3xi8>
+  %3 = vector.from_elements %0, %1, %2 : vector<3xi8>
+  return %3 : vector<3xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @large_source_with_shape_cast_required(
+//  CHECK-SAME:      %[[A:.*]]: vector<2x2x2x2xi8>)
+//       CHECK:      %[[EXTRACT:.*]] = vector.extract %[[A]][0, 1] : vector<2x2xi8> from vector<2x2x2x2xi8>
+//       CHECK:      %[[SHAPE_CAST:.*]] = vector.shape_cast %[[EXTRACT]] : vector<2x2xi8> to vector<1x4x1xi8>
+//       CHECK:      return %[[SHAPE_CAST]] : vector<1x4x1xi8>
+func.func @large_source_with_shape_cast_required(%arg0: vector<2x2x2x2xi8>) -> vector<1x4x1xi8> {
+  %0 = vector.extract %arg0[0, 1, 0, 0] : i8 from vector<2x2x2x2xi8>
+  %1 = vector.extract %arg0[0, 1, 0, 1] : i8 from vector<2x2x2x2xi8>
+  %2 = vector.extract %arg0[0, 1, 1, 0] : i8 from vector<2x2x2x2xi8>
+  %3 = vector.extract %arg0[0, 1, 1, 1] : i8 from vector<2x2x2x2xi8>
+  %4 = vector.from_elements %0, %1, %2, %3 : vector<1x4x1xi8>
+  return %4 : vector<1x4x1xi8>
+}
+
+//  -----
+
+// Could match, but handled by `rewriteFromElementsAsSplat`.
+// CHECK-LABEL: func @extract_single_elm(
+//  CHECK-NEXT:      vector.extract
+//  CHECK-NEXT:      vector.splat
+//  CHECK-NEXT:      return
+func.func @extract_single_elm(%arg0 : vector<2x3xi8>) -> vector<1xi8> {
+  %0 = vector.extract %arg0[0, 0] : i8 from vector<2x3xi8>
+  %1 = vector.from_elements %0 : vector<1xi8>
+  return %1 : vector<1xi8>
+}
+
+// -----
+
+//   CHECK-LABEL: func @negative_source_contiguous_but_not_suffix(
+//     CHECK-NOT:      shape_cast
+//         CHECK:      from_elements
+func.func @negative_source_contiguous_but_not_suffix(%arg0: vector<2x3xi8>) -> vector<3xi8> {
+  %0 = vector.extract %arg0[0, 1] : i8 from vector<2x3xi8>
+  %1 = vector.extract %arg0[0, 2] : i8 from vector<2x3xi8>
+  %2 = vector.extract %arg0[1, 0] : i8 from vector<2x3xi8>
+  %3 = vector.from_elements %0, %1, %2 : vector<3xi8>
+  return %3 : vector<3xi8>
+}
+
+// -----
+
+// The extracted elements are recombined into a single vector, but in a new order.
+// CHECK-LABEL: func @negative_nonascending_order(
+//   CHECK-NOT:      shape_cast
+//       CHECK:      from_elements
+func.func @negative_nonascending_order(%arg0: vector<1x2xi8>) -> vector<2xi8> {
+  %0 = vector.extract %arg0[0, 1] : i8 from vector<1x2xi8>
+  %1 = vector.extract %arg0[0, 0] : i8 from vector<1x2xi8>
+  %2 = vector.from_elements %0, %1 : vector<2xi8>
+  return %2 : vector<2xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_nonstatic_extract(
+//   CHECK-NOT:      shape_cast
+//       CHECK:      from_elements
+func.func @negative_nonstatic_extract(%arg0: vector<1x2xi8>, %i0 : index, %i1 : index) -> vector<2xi8> {
+  %0 = vector.extract %arg0[0, %i0] : i8 from vector<1x2xi8>
+  %1 = vector.extract %arg0[0, %i1] : i8 from vector<1x2xi8>
+  %2 = vector.from_elements %0, %1 : vector<2xi8>
+  return %2 : vector<2xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_different_sources(
+//   CHECK-NOT:      shape_cast
+//       CHECK:      from_elements
+func.func @negative_different_sources(%arg0: vector<1x2xi8>, %arg1: vector<1x2xi8>) -> vector<2xi8> {
+  %0 = vector.extract %arg0[0, 0] : i8 from vector<1x2xi8>
+  %1 = vector.extract %arg1[0, 1] : i8 from vector<1x2xi8>
+  %2 = vector.from_elements %0, %1 : vector<2xi8>
+  return %2 : vector<2xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_source_not_suffix(
+//   CHECK-NOT:      shape_cast
+//       CHECK:      from_elements
+func.func @negative_source_not_suffix(%arg0: vector<1x3xi8>) -> vector<2xi8> {
+  %0 = vector.extract %arg0[0, 0] : i8 from vector<1x3xi8>
+  %1 = vector.extract %arg0[0, 1] : i8 from vector<1x3xi8>
+  %2 = vector.from_elements %0, %1 : vector<2xi8>
+  return %2 : vector<2xi8>
+}
+
+// -----
+
+// The inserted elements are a subset of the extracted elements.
+// [0, 1, 2] -> [1, 1, 2]
+// CHECK-LABEL: func @negative_nobijection_order(
+//   CHECK-NOT:      shape_cast
+//       CHECK:      from_elements
+func.func @negative_nobijection_order(%arg0: vector<1x3xi8>) -> vector<3xi8> {
+  %0 = vector.extract %arg0[0, 1] : i8 from vector<1x3xi8>
+  %1 = vector.extract %arg0[0, 2] : i8 from vector<1x3xi8>
+  %2 = vector.from_elements %0, %0, %1 : vector<3xi8>
+  return %2 : vector<3xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_source_too_small(
+//   CHECK-NOT:      shape_cast
+//       CHECK:      from_elements
+func.func @negative_source_too_small(%arg0: vector<2xi8>) -> vector<4xi8> {
+  %0 = vector.extract %arg0[0] : i8 from vector<2xi8>
+  %1 = vector.extract %arg0[1] : i8 from vector<2xi8>
+  %2 = vector.from_elements %0, %1, %1, %1 : vector<4xi8>
+  return %2 : vector<4xi8>
+}
+
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/side-effects.mlir b/external/llvm-project/mlir/test/Dialect/Vector/side-effects.mlir
new file mode 100644
index 000000000000..54c274a1a2a0
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/Vector/side-effects.mlir
@@ -0,0 +1,15 @@
+// RUN: mlir-opt %s --test-side-effects --verify-diagnostics
+
+func.func @test_side_effects(%arg0: memref<8xf32>) {
+  // expected-remark @below {{operation has no memory effects}}
+  %c0 = arith.constant 0 : index
+  // expected-remark @below {{operation has no memory effects}}
+  %c4 = arith.constant 4 : index
+  // expected-remark @below {{operation has no memory effects}}
+  %cst = arith.constant 0.0 : f32
+  // expected-remark @below {{found an instance of 'read' on op operand 0, on resource '<Default>'}}
+  %0 = vector.transfer_read %arg0[%c0], %cst : memref<8xf32>, vector<4xf32>
+  // expected-remark @below {{found an instance of 'write' on op operand 1, on resource '<Default>'}}
+  vector.transfer_write %0, %arg0[%c4] : vector<4xf32>, memref<8xf32>
+  return
+}
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir b/external/llvm-project/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir
index ef32f8c6a1cd..5011d8b2b2ef 100644
--- a/external/llvm-project/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Vector/vector-shape-cast-lowering-transforms.mlir
@@ -1,145 +1,392 @@
 // RUN: mlir-opt %s --transform-interpreter | FileCheck %s
 
 // CHECK-LABEL: func @nop_shape_cast
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>
-// CHECK:      return %[[A]] : vector<16xf32>
+//  CHECK-SAME:    %[[A:.*]]: vector<16xf32>
+//       CHECK: return %[[A]] : vector<16xf32>
 func.func @nop_shape_cast(%arg0: vector<16xf32>) -> vector<16xf32> {
   %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<16xf32>
   return %0 : vector<16xf32>
 }
 
 // CHECK-LABEL: func @cancel_shape_cast
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>
-// CHECK:      return %[[A]] : vector<16xf32>
-
+//  CHECK-SAME:    %[[A:.*]]: vector<16xf32>
+//       CHECK: return %[[A]] : vector<16xf32>
 func.func @cancel_shape_cast(%arg0: vector<16xf32>) -> vector<16xf32> {
   %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<4x4xf32>
   %1 = vector.shape_cast %0 : vector<4x4xf32> to vector<16xf32>
   return %1 : vector<16xf32>
 }
 
-// Shape up and downcasts for 2-D vectors, for supporting conversion to
-// llvm.matrix operations
-// CHECK-LABEL: func @shape_casts
-func.func @shape_casts(%a: vector<2x2xf32>) -> (vector<4xf32>, vector<2x2xf32>) {
-  // CHECK-DAG: %[[ub22:.*]] = ub.poison : vector<2x2xf32>
-  // CHECK-DAG: %[[ub:.*]] = ub.poison : vector<4xf32>
-  // CHECK: %[[ex0:.*]] = vector.extract %{{.*}}[0] : vector<2xf32> from vector<2x2xf32>
-  //
-  // CHECK: %[[in0:.*]] = vector.insert_strided_slice %[[ex0]], %[[ub]]
-  // CHECK-SAME: {offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
-  //
-  // CHECK: %[[ex1:.*]] = vector.extract %{{.*}}[1] : vector<2xf32> from vector<2x2xf32>
-  //
-  // CHECK: %[[in2:.*]] = vector.insert_strided_slice %[[ex1]], %[[in0]]
-  // CHECK-SAME: {offsets = [2], strides = [1]} : vector<2xf32> into vector<4xf32>
-  //
+// Collapse 2-D to 1-D.
+// CHECK-LABEL: func @shape_cast_2d1d
+//  CHECK-SAME:    %[[A:.*]]: vector<2x2xf32>) -> vector<4xf32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<4xf32>
+//
+//       CHECK: %[[EX0:.*]] = vector.extract %[[A]][0] : vector<2xf32> from vector<2x2xf32>
+//       CHECK: %[[IN0:.*]] = vector.insert_strided_slice %[[EX0]], %[[UB]]
+//  CHECK-SAME:    {offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
+//
+//       CHECK: %[[EX1:.*]] = vector.extract %{{.*}}[1] : vector<2xf32> from vector<2x2xf32>
+//       CHECK: %[[IN2:.*]] = vector.insert_strided_slice %[[EX1]], %[[IN0]]
+//  CHECK-SAME:    {offsets = [2], strides = [1]} : vector<2xf32> into vector<4xf32>
+//       CHECK: return %[[IN2]] : vector<4xf32>
+func.func @shape_cast_2d1d(%a: vector<2x2xf32>) -> (vector<4xf32>) {
   %0 = vector.shape_cast %a : vector<2x2xf32> to vector<4xf32>
-  // CHECK: %[[add:.*]] = arith.addf %[[in2]], %[[in2]] : vector<4xf32>
-  %r0 = arith.addf %0, %0: vector<4xf32>
-  //
-  // CHECK: %[[ss0:.*]] = vector.extract_strided_slice %[[add]]
-  // CHECK-SAME: {offsets = [0], sizes = [2], strides = [1]} :
-  // CHECK-SAME: vector<4xf32> to vector<2xf32>
-  //
-  // CHECK: %[[res0:.*]] = vector.insert %[[ss0]], %[[ub22]] [0] :
-  // CHECK-SAME: vector<2xf32> into vector<2x2xf32>
-  //
-  // CHECK: %[[s2:.*]] = vector.extract_strided_slice %[[add]]
-  // CHECK-SAME: {offsets = [2], sizes = [2], strides = [1]} :
-  // CHECK-SAME: vector<4xf32> to vector<2xf32>
-  //
-  // CHECK: %[[res1:.*]] = vector.insert %[[s2]], %[[res0]] [1] :
-  // CHECK-SAME: vector<2xf32> into vector<2x2xf32>
-  //
-  %1 = vector.shape_cast %r0  : vector<4xf32> to vector<2x2xf32>
-  // CHECK: return %[[add]], %[[res1]] : vector<4xf32>, vector<2x2xf32>
-  return %r0, %1 : vector<4xf32>, vector<2x2xf32>
-}
-
-// CHECK-LABEL: func @shape_cast_2d2d
-// CHECK-SAME: %[[A:.*]]: vector<3x2xf32>
-// CHECK: %[[UB:.*]] = ub.poison : vector<2x3xf32>
-// CHECK: %[[T0:.*]] = vector.extract %[[A]][0, 0] : f32 from vector<3x2xf32>
-// CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[UB]] [0, 0] : f32 into vector<2x3xf32>
-// CHECK: %[[T2:.*]] = vector.extract %[[A]][0, 1] : f32 from vector<3x2xf32>
-// CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [0, 1] : f32 into vector<2x3xf32>
-// CHECK: %[[T4:.*]] = vector.extract %[[A]][1, 0] : f32 from vector<3x2xf32>
-// CHECK: %[[T5:.*]] = vector.insert %[[T4]], %[[T3]] [0, 2] : f32 into vector<2x3xf32>
-// CHECK: %[[T6:.*]] = vector.extract %[[A]][1, 1] : f32 from vector<3x2xf32>
-// CHECK: %[[T7:.*]] = vector.insert %[[T6]], %[[T5]] [1, 0] : f32 into vector<2x3xf32>
-// CHECK: %[[T8:.*]] = vector.extract %[[A]][2, 0] : f32 from vector<3x2xf32>
-// CHECK: %[[T9:.*]] = vector.insert %[[T8]], %[[T7]] [1, 1] : f32 into vector<2x3xf32>
-// CHECK: %[[T10:.*]] = vector.extract %[[A]][2, 1] : f32 from vector<3x2xf32>
-// CHECK: %[[T11:.*]] = vector.insert %[[T10]], %[[T9]] [1, 2] : f32 into vector<2x3xf32>
-// CHECK: return %[[T11]] : vector<2x3xf32>
-
-func.func @shape_cast_2d2d(%arg0 : vector<3x2xf32>) -> vector<2x3xf32> {
-  %s = vector.shape_cast %arg0: vector<3x2xf32> to vector<2x3xf32>
-  return %s : vector<2x3xf32>
+  return %0 : vector<4xf32>
 }
 
+// Collapse 3-D to 1-D.
 // CHECK-LABEL: func @shape_cast_3d1d
-// CHECK-SAME: %[[A:.*]]: vector<1x3x2xf32>
-// CHECK: %[[UB:.*]] = ub.poison : vector<6xf32>
-// CHECK: %[[T0:.*]] = vector.extract %[[A]][0, 0] : vector<2xf32> from vector<1x3x2xf32>
-// CHECK: %[[T1:.*]] = vector.insert_strided_slice %[[T0]], %[[UB]]
-// CHECK-SAME:           {offsets = [0], strides = [1]} : vector<2xf32> into vector<6xf32>
-// CHECK: %[[T2:.*]] = vector.extract %[[A]][0, 1] : vector<2xf32> from vector<1x3x2xf32>
-// CHECK: %[[T3:.*]] = vector.insert_strided_slice %[[T2]], %[[T1]]
-// CHECK-SAME:           {offsets = [2], strides = [1]} : vector<2xf32> into vector<6xf32>
-// CHECK: %[[T4:.*]] = vector.extract %[[A]][0, 2] : vector<2xf32> from vector<1x3x2xf32>
-// CHECK: %[[T5:.*]] = vector.insert_strided_slice %[[T4]], %[[T3]]
-// CHECK-SAME:           {offsets = [4], strides = [1]} : vector<2xf32> into vector<6xf32>
-// CHECK: return %[[T5]] : vector<6xf32>
-
+//  CHECK-SAME:    %[[A:.*]]: vector<1x3x2xf32>
+//       CHECK: %[[UB:.*]] = ub.poison : vector<6xf32>
+//
+//       CHECK: %[[T0:.*]] = vector.extract %[[A]][0, 0] : vector<2xf32> from vector<1x3x2xf32>
+//       CHECK: %[[T1:.*]] = vector.insert_strided_slice %[[T0]], %[[UB]]
+//  CHECK-SAME:    {offsets = [0], strides = [1]} : vector<2xf32> into vector<6xf32>
+//
+//       CHECK: %[[T2:.*]] = vector.extract %[[A]][0, 1] : vector<2xf32> from vector<1x3x2xf32>
+//       CHECK: %[[T3:.*]] = vector.insert_strided_slice %[[T2]], %[[T1]]
+//  CHECK-SAME:    {offsets = [2], strides = [1]} : vector<2xf32> into vector<6xf32>
+//
+//       CHECK: %[[T4:.*]] = vector.extract %[[A]][0, 2] : vector<2xf32> from vector<1x3x2xf32>
+//       CHECK: %[[T5:.*]] = vector.insert_strided_slice %[[T4]], %[[T3]]
+//  CHECK-SAME:    {offsets = [4], strides = [1]} : vector<2xf32> into vector<6xf32>
+//       CHECK: return %[[T5]] : vector<6xf32>
 func.func @shape_cast_3d1d(%arg0 : vector<1x3x2xf32>) -> vector<6xf32> {
   %s = vector.shape_cast %arg0 : vector<1x3x2xf32> to vector<6xf32>
   return %s : vector<6xf32>
 }
 
-// CHECK-LABEL: func @shape_cast_1d3d
-// CHECK-SAME: %[[A:.*]]: vector<6xf32>
-// CHECK: %[[UB:.*]] = ub.poison : vector<2x1x3xf32>
-// CHECK: %[[T0:.*]] = vector.extract_strided_slice %[[A]]
-// CHECK-SAME:           {offsets = [0], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
-// CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[UB]] [0, 0] : vector<3xf32> into vector<2x1x3xf32>
-// CHECK: %[[T2:.*]] = vector.extract_strided_slice %[[A]]
-// CHECK:                {offsets = [3], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
-// CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [1, 0] : vector<3xf32> into vector<2x1x3xf32>
-// CHECK: return %[[T3]] : vector<2x1x3xf32>
+// Expand 1-D to 2-D.
+// CHECK-LABEL: func.func @shape_cast_1d2d(
+//  CHECK-SAME:    %[[A:.*]]: vector<4xf32>) -> vector<2x2xf32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<2x2xf32>
+//
+//       CHECK: %[[SS0:.*]] = vector.extract_strided_slice %[[A]]
+//  CHECK-SAME:    {offsets = [0], sizes = [2], strides = [1]} :
+//  CHECK-SAME:    vector<4xf32> to vector<2xf32>
+//       CHECK: %[[res0:.*]] = vector.insert %[[SS0]], %[[UB]] [0] :
+//  CHECK-SAME:    vector<2xf32> into vector<2x2xf32>
+//
+//       CHECK: %[[SS2:.*]] = vector.extract_strided_slice %[[A]]
+//  CHECK-SAME:    {offsets = [2], sizes = [2], strides = [1]} :
+//  CHECK-SAME:    vector<4xf32> to vector<2xf32>
+//       CHECK: %[[res1:.*]] = vector.insert %[[SS2]], %[[res0]] [1] :
+//  CHECK-SAME:    vector<2xf32> into vector<2x2xf32>
+//       CHECK: return  %[[res1]] :  vector<2x2xf32>
+func.func @shape_cast_1d2d(%a: vector<4xf32>) -> (vector<2x2xf32>) {
+  %1 = vector.shape_cast %a: vector<4xf32> to vector<2x2xf32>
+  return %1 : vector<2x2xf32>
+}
 
+// Expand 1-D to 3-D.
+// CHECK-LABEL: func @shape_cast_1d3d
+//  CHECK-SAME:    %[[A:.*]]: vector<6xf32>
+//       CHECK: %[[UB:.*]] = ub.poison : vector<2x1x3xf32>
+//
+//       CHECK: %[[T0:.*]] = vector.extract_strided_slice %[[A]]
+//  CHECK-SAME:    {offsets = [0], sizes = [3], strides = [1]} :
+//  CHECK-SAME:    vector<6xf32> to vector<3xf32>
+//       CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[UB]] [0, 0] :
+//  CHECK-SAME:    vector<3xf32> into vector<2x1x3xf32>
+//
+//       CHECK: %[[T2:.*]] = vector.extract_strided_slice %[[A]]
+//  CHECK-SAME:    {offsets = [3], sizes = [3], strides = [1]} :
+//  CHECK-SAME:    vector<6xf32> to vector<3xf32>
+//       CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [1, 0] :
+//  CHECK-SAME:    vector<3xf32> into vector<2x1x3xf32>
+//       CHECK: return %[[T3]] : vector<2x1x3xf32>
 func.func @shape_cast_1d3d(%arg0 : vector<6xf32>) -> vector<2x1x3xf32> {
   %s = vector.shape_cast %arg0 : vector<6xf32> to vector<2x1x3xf32>
   return %s : vector<2x1x3xf32>
 }
 
-// CHECK-LABEL:   func.func @shape_cast_0d1d(
-// CHECK-SAME:                               %[[ARG0:.*]]: vector<f32>) -> vector<1xf32> {
-// CHECK:           %[[UB:.*]] = ub.poison : vector<1xf32>
-// CHECK:           %[[EXTRACT0:.*]] = vector.extract %[[ARG0]][] : f32 from vector<f32>
-// CHECK:           %[[RES:.*]] = vector.insert %[[EXTRACT0]], %[[UB]] [0] : f32 into vector<1xf32>
-// CHECK:           return %[[RES]] : vector<1xf32>
-// CHECK:         }
+// 2-D to 2-D where the inner-most dimensions have no common factors. This
+// case requires scalar element by element extraction and insertion.
+// CHECK-LABEL: func @shape_cast_2d2d
+//  CHECK-SAME:    %[[A:.*]]: vector<3x2xf32>
+//       CHECK: %[[UB:.*]] = ub.poison : vector<2x3xf32>
+//
+//       CHECK: %[[T0:.*]] = vector.extract %[[A]][0, 0] : f32 from vector<3x2xf32>
+//       CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[UB]] [0, 0] :
+//
+//       CHECK: %[[T2:.*]] = vector.extract %[[A]][0, 1] : f32 from vector<3x2xf32>
+//       CHECK: %[[T3:.*]] = vector.insert %[[T2]], %[[T1]] [0, 1] :
+//
+//       CHECK: %[[T4:.*]] = vector.extract %[[A]][1, 0] : f32 from vector<3x2xf32>
+//       CHECK: %[[T5:.*]] = vector.insert %[[T4]], %[[T3]] [0, 2] :
+//
+//       CHECK: %[[T6:.*]] = vector.extract %[[A]][1, 1] : f32 from vector<3x2xf32>
+//       CHECK: %[[T7:.*]] = vector.insert %[[T6]], %[[T5]] [1, 0] :
+//
+//       CHECK: %[[T8:.*]] = vector.extract %[[A]][2, 0] : f32 from vector<3x2xf32>
+//       CHECK: %[[T9:.*]] = vector.insert %[[T8]], %[[T7]] [1, 1] :
+//
+//       CHECK: %[[T10:.*]] = vector.extract %[[A]][2, 1] : f32 from vector<3x2xf32>
+//       CHECK: %[[T11:.*]] = vector.insert %[[T10]], %[[T9]] [1, 2] :
+//
+//       CHECK: return %[[T11]] : vector<2x3xf32>
+func.func @shape_cast_2d2d(%arg0 : vector<3x2xf32>) -> vector<2x3xf32> {
+  %s = vector.shape_cast %arg0: vector<3x2xf32> to vector<2x3xf32>
+  return %s : vector<2x3xf32>
+}
 
+// CHECK-LABEL: func.func @shape_cast_0d1d(
+//  CHECK-SAME:    %[[A:.*]]: vector<f32>) -> vector<1xf32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<1xf32>
+//
+//       CHECK: %[[EXTRACT0:.*]] = vector.extract %[[A]][] : f32 from vector<f32>
+//       CHECK: %[[RES:.*]] = vector.insert %[[EXTRACT0]], %[[UB]] [0] :
+//       CHECK: return %[[RES]] : vector<1xf32>
 func.func @shape_cast_0d1d(%arg0 : vector<f32>) -> vector<1xf32> {
   %s = vector.shape_cast %arg0 : vector<f32> to vector<1xf32>
   return %s : vector<1xf32>
 }
 
-// CHECK-LABEL:   func.func @shape_cast_1d0d(
-// CHECK-SAME:                               %[[ARG0:.*]]: vector<1xf32>) -> vector<f32> {
-// CHECK:           %[[UB:.*]] = ub.poison : vector<f32>
-// CHECK:           %[[EXTRACT0:.*]] = vector.extract %[[ARG0]][0] : f32 from vector<1xf32>
-// CHECK:           %[[RES:.*]] = vector.insert %[[EXTRACT0]], %[[UB]] [] : f32 into vector<f32>
-// CHECK:           return %[[RES]] : vector<f32>
-// CHECK:         }
-
+// CHECK-LABEL: func.func @shape_cast_1d0d(
+//  CHECK-SAME:    %[[A:.*]]: vector<1xf32>) -> vector<f32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<f32>
+//
+//       CHECK: %[[EXTRACT0:.*]] = vector.extract %[[A]][0] : f32 from vector<1xf32>
+//       CHECK: %[[RES:.*]] = vector.insert %[[EXTRACT0]], %[[UB]] [] :
+//       CHECK: return %[[RES]] : vector<f32>
 func.func @shape_cast_1d0d(%arg0 : vector<1xf32>) -> vector<f32> {
   %s = vector.shape_cast %arg0 : vector<1xf32> to vector<f32>
   return %s : vector<f32>
 }
 
+// The shapes have 2 inner dimension sizes in common, so the extract result is rank-2.
+// CHECK-LABEL: func.func @squeeze_out_prefix_unit_dim(
+//  CHECK-SAME:    %[[A:.*]]: vector<1x2x3xf32>) -> vector<2x3xf32> {
+//
+//       CHECK: %[[EXTRACTED:.*]] = vector.extract %[[A]][0] :
+//  CHECK-SAME:    vector<2x3xf32> from vector<1x2x3xf32>
+//       CHECK: return %[[EXTRACTED]] : vector<2x3xf32>
+func.func @squeeze_out_prefix_unit_dim(%arg0 : vector<1x2x3xf32>) -> vector<2x3xf32> {
+  %s = vector.shape_cast %arg0 : vector<1x2x3xf32> to vector<2x3xf32>
+  return %s : vector<2x3xf32>
+}
+
+// The shapes have 1 inner dimension size in common, so the extract results are rank-1.
+// CHECK-LABEL: func.func @squeeze_out_middle_unit_dim(
+//  CHECK-SAME:    %[[A:.*]]: vector<2x1x3xf32>) -> vector<2x3xf32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<2x3xf32>
+//
+//       CHECK: %[[E0:.*]] = vector.extract %[[A]][0, 0] : vector<3xf32>
+//       CHECK: %[[I0:.*]] = vector.insert %[[E0]], %[[UB]] [0] :
+//
+//       CHECK: %[[E1:.*]] = vector.extract %[[A]][1, 0] : vector<3xf32>
+//       CHECK: %[[I1:.*]] = vector.insert %[[E1]], %[[I0]] [1] :
+//       CHECK: return %[[I1]] : vector<2x3xf32>
+func.func @squeeze_out_middle_unit_dim(%arg0 : vector<2x1x3xf32>) -> vector<2x3xf32> {
+  %s = vector.shape_cast %arg0 : vector<2x1x3xf32> to vector<2x3xf32>
+  return %s : vector<2x3xf32>
+}
+
+// CHECK-LABEL: func.func @prepend_unit_dim(
+//  CHECK-SAME:    %[[A:.*]]: vector<2x3xf32>) -> vector<1x2x3xf32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<1x2x3xf32>
+//
+//       CHECK: %[[I0:.*]] = vector.insert %[[A]], %[[UB]] [0]
+//       CHECK: return %[[I0]] : vector<1x2x3xf32>
+func.func @prepend_unit_dim(%arg0 : vector<2x3xf32>) -> vector<1x2x3xf32> {
+  %s = vector.shape_cast %arg0 : vector<2x3xf32> to vector<1x2x3xf32>
+  return %s : vector<1x2x3xf32>
+}
+
+// CHECK-LABEL: func.func @insert_middle_unit_dim(
+//  CHECK-SAME:    %[[A:.*]]: vector<2x3xf32>) -> vector<2x1x3xf32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<2x1x3xf32>
+//
+//       CHECK: %[[E0:.*]] = vector.extract %[[A]][0] : vector<3xf32>
+//       CHECK: %[[I0:.*]] = vector.insert %[[E0]], %[[UB]] [0, 0] : vector<3xf32>
+//
+//       CHECK: %[[E1:.*]] = vector.extract %[[A]][1] : vector<3xf32>
+//       CHECK: %[[I1:.*]] = vector.insert %[[E1]], %[[I0]] [1, 0] : vector<3xf32>
+//       CHECK: return %[[I1]] : vector<2x1x3xf32>
+func.func @insert_middle_unit_dim(%arg0 : vector<2x3xf32>) -> vector<2x1x3xf32> {
+  %s = vector.shape_cast %arg0 : vector<2x3xf32> to vector<2x1x3xf32>
+  return %s : vector<2x1x3xf32>
+}
+
+//   CHECK-LABEL: func.func @postpend_unit_dims(
+//    CHECK-SAME:    %[[A:.*]]: vector<4xf32>) -> vector<4x1x1xf32> {
+//         CHECK: %[[UB:.*]] = ub.poison : vector<4x1x1xf32>
+//         CHECK: %[[E0:.*]] = vector.extract %[[A]][0] : f32 from vector<4xf32>
+//         CHECK: %[[I0:.*]] = vector.insert %[[E0]], %[[UB]] [0, 0, 0] : f32 into vector<4x1x1xf32>
+//         CHECK: %[[E1:.*]] = vector.extract %[[A]][1] : f32 from vector<4xf32>
+//         CHECK: %[[I1:.*]] = vector.insert %[[E1]], %[[I0]] [1, 0, 0] : f32 into vector<4x1x1xf32>
+//         CHECK: vector.extract %[[A]][2]
+//         CHECK: vector.insert {{.*}} [2, 0, 0]
+//         CHECK: vector.extract %[[A]][3]
+//         CHECK: vector.insert {{.*}} [3, 0, 0]
+//         CHECK: return
+func.func @postpend_unit_dims(%arg0 : vector<4xf32>) -> vector<4x1x1xf32> {
+  %s = vector.shape_cast %arg0 : vector<4xf32> to vector<4x1x1xf32>
+  return %s : vector<4x1x1xf32>
+}
+
+// CHECK-LABEL: func.func @expand_inner_dims(
+//  CHECK-SAME:    %[[A:.*]]: vector<2x10xf32>) -> vector<2x2x5xf32> {
+//       CHECK: %[[UB:.*]] = ub.poison : vector<2x2x5xf32>
+//
+//       CHECK: %[[E0:.*]] = vector.extract %[[A]][0] : vector<10xf32>
+//       CHECK: %[[S0:.*]] = vector.extract_strided_slice %[[E0]]
+//  CHECK-SAME:    {offsets = [0], sizes = [5], {{.*}} to vector<5xf32>
+//       CHECK: %[[I0:.*]] = vector.insert %[[S0]], %[[UB]] [0, 0]
+//
+//       CHECK: %[[S1:.*]] = vector.extract_strided_slice %[[E0]]
+//  CHECK-SAME:    {offsets = [5], sizes = [5], {{.*}} to vector<5xf32>
+//       CHECK: %[[I1:.*]] = vector.insert %[[S1]], %[[I0]] [0, 1]
+//
+//       CHECK: %[[E1:.*]] = vector.extract %[[A]][1] : vector<10xf32>
+//       CHECK: %[[S2:.*]] = vector.extract_strided_slice %[[E1]]
+//  CHECK-SAME:    {offsets = [0], sizes = [5], {{.*}} to vector<5xf32>
+//       CHECK: %[[I2:.*]] = vector.insert %[[S2]], %[[I1]] [1, 0]
+//
+//       CHECK: %[[S3:.*]] = vector.extract_strided_slice %[[E1]]
+//  CHECK-SAME:    {offsets = [5], sizes = [5], {{.*}} to vector<5xf32>
+//       CHECK: %[[I3:.*]] = vector.insert %[[S3]], %[[I2]] [1, 1]
+//       CHECK: return %[[I3]] : vector<2x2x5xf32>
+func.func @expand_inner_dims(%arg0 : vector<2x10xf32>) -> vector<2x2x5xf32> {
+  %s = vector.shape_cast %arg0 : vector<2x10xf32> to vector<2x2x5xf32>
+  return %s : vector<2x2x5xf32>
+}
+
+
+// Some pseudocode describing how this function is lowered:
+//
+// func collapse_inner_dims(A : vector<2x2x5xi8>) -> vector<1x2x1x10xi8> {
+//   v0          = empty of shape (10)
+//   v1          = empty of shape (1,2,1,10)
+//   v0[0:5]     = A[0,0,:]
+//   v0[5:10]    = A[0,1,:]
+//   v1[0,0,0,:] = v0
+//   v0[0:5]     = A[1,0,:]
+//   v0[5:10]    = A[1,1,:]
+//   v1[0,1,0,:] = v0
+//   return v1;
+// }
+// CHECK-LABEL: func.func @collapse_inner_dims(
+//  CHECK-SAME:    %[[A:.*]]: vector<2x2x5xi8>) -> vector<1x2x1x10xi8> {
+//   CHECK-DAG: %[[UBSMALL:.*]] = ub.poison : vector<10xi8>
+//   CHECK-DAG: %[[UBLARGE:.*]] = ub.poison : vector<1x2x1x10xi8>
+//
+//       CHECK: %[[EX0:.*]] = vector.extract %[[A]][0, 0]
+//       CHECK: %[[IN0:.*]] = vector.insert_strided_slice %[[EX0]], %[[UBSMALL]]
+//  CHECK-SAME:    {offsets = [0], {{.*}}
+//       CHECK: %[[EX1:.*]] = vector.extract %[[A]][0, 1]
+//       CHECK: %[[IN1:.*]] = vector.insert_strided_slice %[[EX1]], %[[IN0]]
+//  CHECK-SAME:    {offsets = [5], {{.*}}
+//       CHECK: %[[IN2:.*]] = vector.insert %[[IN1]], %[[UBLARGE]] [0, 0, 0]
+//
+//       CHECK: %[[EX2:.*]] = vector.extract %[[A]][1, 0]
+//       CHECK: %[[IN3:.*]] = vector.insert_strided_slice %[[EX2]], %[[UBSMALL]]
+//  CHECK-SAME:    {offsets = [0], {{.*}}
+//       CHECK: %[[EX3:.*]] = vector.extract %[[A]][1, 1]
+//       CHECK: %[[IN4:.*]] = vector.insert_strided_slice %[[EX3]], %[[IN3]]
+//  CHECK-SAME:    {offsets = [5], {{.*}}
+//       CHECK: %[[IN5:.*]] = vector.insert %[[IN4]], %[[IN2]] [0, 1, 0]
+//       CHECK: return %[[IN5]] : vector<1x2x1x10xi8>
+func.func @collapse_inner_dims(%arg0 : vector<2x2x5xi8>) -> vector<1x2x1x10xi8> {
+  %s = vector.shape_cast %arg0 : vector<2x2x5xi8> to vector<1x2x1x10xi8>
+  return %s : vector<1x2x1x10xi8>
+}
+
+// Some alternative pseudocode describing how this function is lowered:
+//
+// func non_dividing_gcd_decreasing(A : vector<2x15xi8>) -> vector<3x10xi8> {
+//   v0          = empty of shape (10)
+//   v1          = empty of shape (3,10)
+//   e0          = A[0,:]
+//   v0[0:5]     = e0[0:5]
+//   v0[5:10]    = e0[5:10]
+//   v1[0,:]     = v0
+//   v0[0,0:5]   = e0[10:15]
+//   e1          = A[1,:]
+//   v0[0,5:10]  = e1[0:5]
+//   v1[1,:]     = v0
+//   v0[0,0:5]   = e1[5:10]
+//   v0[0,5:10]  = e1[10:15]
+//   v1[2,:]     = v0
+//   return v1;
+// }
+// CHECK-LABEL: func.func @non_dividing_gcd_decreasing(
+//  CHECK-SAME:    %[[A:.*]]: vector<2x15xi8>) -> vector<3x10xi8> {
+//   CHECK-DAG: %[[UB0:.*]] = ub.poison : vector<10xi8>
+//   CHECK-DAG: %[[UB1:.*]] = ub.poison : vector<3x10xi8>
+//
+// First 10 elements:
+//       CHECK: %[[EX0:.*]] = vector.extract %[[A]][0] : vector<15xi8> from vector<2x15xi8>
+//       CHECK: %[[SS0:.*]] = vector.extract_strided_slice %[[EX0]]
+//  CHECK-SAME:    {offsets = [0], {{.*}} to vector<5xi8>
+//       CHECK: %[[IN0:.*]] = vector.insert_strided_slice %[[SS0]], %[[UB0]]
+//  CHECK-SAME:    {offsets = [0], {{.*}}
+//       CHECK: %[[SS1:.*]] = vector.extract_strided_slice %[[EX0]]
+//  CHECK-SAME:    {offsets = [5], {{.*}} to vector<5xi8>
+//       CHECK: %[[IN1:.*]] = vector.insert_strided_slice %[[SS1]], %[[IN0]]
+//  CHECK-SAME:    {offsets = [5], {{.*}}
+//       CHECK: %[[IN2:.*]] = vector.insert %[[IN1]], %[[UB1]] [0] : vector<10xi8> into vector<3x10xi8>
+//
+// Next 10 elements:
+//       CHECK: %[[SS2:.*]] = vector.extract_strided_slice %[[EX0]]
+//  CHECK-SAME:    {offsets = [10], {{.*}} to vector<5xi8>
+//       CHECK: %[[IN3:.*]] = vector.insert_strided_slice %[[SS2]], %[[UB0]]
+//  CHECK-SAME:    {offsets = [0], {{.*}}
+//       CHECK: %[[EX1:.*]] = vector.extract %[[A]][1] : vector<15xi8> from vector<2x15xi8>
+//       CHECK: %[[SS3:.*]] = vector.extract_strided_slice %[[EX1]]
+//  CHECK-SAME:    {offsets = [0], {{.*}} to vector<5xi8>
+//       CHECK: %[[IN4:.*]] = vector.insert_strided_slice %[[SS3]], %[[IN3]]
+//  CHECK-SAME:    {offsets = [5], {{.*}}
+//       CHECK: %[[IN5:.*]] = vector.insert %[[IN4]], %[[IN2]] [1] : vector<10xi8> into vector<3x10xi8>
+//
+// Final 10 elements:
+//       CHECK: %[[SS4:.*]] = vector.extract_strided_slice %[[EX1]]
+//  CHECK-SAME:    {offsets = [5], {{.*}} to vector<5xi8>
+//       CHECK: %[[IN6:.*]] = vector.insert_strided_slice %[[SS4]], %[[UB0]]
+//  CHECK-SAME:    {offsets = [0], {{.*}}
+//       CHECK: %[[SS5:.*]] = vector.extract_strided_slice %[[EX1]]
+//  CHECK-SAME:    {offsets = [10], {{.*}} to vector<5xi8>
+//       CHECK: %[[IN7:.*]] = vector.insert_strided_slice %[[SS5]], %[[IN6]]
+//  CHECK-SAME:    {offsets = [5], {{.*}}
+//       CHECK: %[[IN8:.*]] = vector.insert %[[IN7]], %[[IN5]] [2] : vector<10xi8> into vector<3x10xi8>
+//       CHECK: return %[[IN8]] : vector<3x10xi8>
+func.func @non_dividing_gcd_decreasing(%arg0 : vector<2x15xi8>) -> vector<3x10xi8> {
+  %0 = vector.shape_cast %arg0 : vector<2x15xi8> to vector<3x10xi8>
+  return %0 : vector<3x10xi8>
+}
+
+// CHECK-LABEL: func.func @non_dividing_gcd_increasing(
+//  CHECK-SAME:    %[[A:.*]]: vector<3x10xi8>) -> vector<2x15xi8> {
+//
+//   CHECK-DAG: ub.poison : vector<15xi8>
+//   CHECK-DAG: ub.poison : vector<2x15xi8>
+//
+// Collect the first 15 elements, and insert into the first row of the result.
+//       CHECK: vector.extract %[[A]][0]
+//       CHECK: extract_strided_slice
+//       CHECK: insert_strided_slice
+//       CHECK: extract_strided_slice
+//       CHECK: insert_strided_slice
+//       CHECK: vector.extract %[[A]][1]
+//       CHECK: extract_strided_slice
+//       CHECK: insert_strided_slice
+//       CHECK: vector.insert {{.*}} [0] : vector<15xi8> into vector<2x15xi8>
+//
+// Collect the next 15 elements, and insert into the second row of the result.
+//       CHECK: extract_strided_slice
+//       CHECK: insert_strided_slice
+//       CHECK: vector.extract %[[A]][2]
+//       CHECK: extract_strided_slice
+//       CHECK: insert_strided_slice
+//       CHECK: extract_strided_slice
+//       CHECK: insert_strided_slice
+//       CHECK: vector.insert {{.*}} [1] : vector<15xi8> into vector<2x15xi8>
+func.func @non_dividing_gcd_increasing(%arg0 : vector<3x10xi8>) -> vector<2x15xi8> {
+  %0 = vector.shape_cast %arg0 : vector<3x10xi8> to vector<2x15xi8>
+  return %0 : vector<2x15xi8>
+}
+
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
     %f = transform.structured.match ops{["func.func"]} in %module_op
diff --git a/external/llvm-project/mlir/test/Dialect/Vector/vector-unroll-options.mlir b/external/llvm-project/mlir/test/Dialect/Vector/vector-unroll-options.mlir
index 9c158d05b723..fbb178fb49d8 100644
--- a/external/llvm-project/mlir/test/Dialect/Vector/vector-unroll-options.mlir
+++ b/external/llvm-project/mlir/test/Dialect/Vector/vector-unroll-options.mlir
@@ -196,7 +196,7 @@ func.func @negative_vector_fma_3d(%a: vector<3x2x2xf32>) -> vector<3x2x2xf32>{
 // CHECK-LABEL: func @negative_vector_fma_3d
 //   CHECK-NOT: vector.extract_strided_slice
 //       CHECK: %[[R0:.*]] = vector.fma %{{.+}} : vector<3x2x2xf32>
-//       CHECK: return 
+//       CHECK: return
 
 func.func @vector_multi_reduction(%v : vector<4x6xf32>, %acc: vector<4xf32>) -> vector<4xf32> {
   %0 = vector.multi_reduction #vector.kind<add>, %v, %acc [1] : vector<4x6xf32> to vector<4xf32>
@@ -311,3 +311,70 @@ func.func @vector_contract_batched(%lhs: vector<8x8x4xf32>, %rhs: vector<8x8x4xf
 // BATCHED-COUNT-16: vector.contract
 //      BATCHED-NOT: vector.contract
 //          BATCHED: return
+
+
+func.func @vector_broadcast(%v: vector<4xf32>) -> vector<4x4xf32> {
+  %0 = vector.broadcast %v : vector<4xf32> to vector<4x4xf32>
+  return %0 : vector<4x4xf32>
+}
+
+// CHECK-LABEL: func @vector_broadcast
+//  CHECK-SAME: [[arg0:%.+]]: vector<4xf32>
+//       CHECK: [[c:%.+]] = arith.constant dense<0.000000e+00> : vector<4x4xf32>
+//       CHECK: [[s0:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
+//       CHECK: [[b0:%.+]] = vector.broadcast [[s0]] : vector<2xf32> to vector<2x2xf32>
+//       CHECK: [[r0:%.+]] = vector.insert_strided_slice [[b0]], [[c]] {offsets = [0, 0], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s1:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [2], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
+//       CHECK: [[b1:%.+]] = vector.broadcast [[s1]] : vector<2xf32> to vector<2x2xf32>
+//       CHECK: [[r1:%.+]] = vector.insert_strided_slice [[b1]], [[r0]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s2:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
+//       CHECK: [[b2:%.+]] = vector.broadcast [[s2]] : vector<2xf32> to vector<2x2xf32>
+//       CHECK: [[r2:%.+]] = vector.insert_strided_slice [[b2]], [[r1]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s3:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [2], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
+//       CHECK: [[b3:%.+]] = vector.broadcast [[s3]] : vector<2xf32> to vector<2x2xf32>
+//       CHECK: [[r3:%.+]] = vector.insert_strided_slice [[b3]], [[r2]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: return [[r3]] : vector<4x4xf32>
+
+func.func @vector_broadcast_with_leading_unit_dim(%v: vector<1x4xf32>) -> vector<4x4xf32> {
+  %0 = vector.broadcast %v : vector<1x4xf32> to vector<4x4xf32>
+  return %0 : vector<4x4xf32>
+}
+
+// CHECK-LABEL: func.func @vector_broadcast_with_leading_unit_dim
+//  CHECK-SAME: ([[arg0:%.+]]: vector<1x4xf32>) -> vector<4x4xf32> {
+//       CHECK: [[c:%.+]] = arith.constant dense<0.000000e+00> : vector<4x4xf32>
+//       CHECK: [[s0:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0, 0], sizes = [1, 2], strides = [1, 1]} : vector<1x4xf32> to vector<1x2xf32>
+//       CHECK: [[b0:%.+]] = vector.broadcast [[s0]] : vector<1x2xf32> to vector<2x2xf32>
+//       CHECK: [[r0:%.+]] = vector.insert_strided_slice [[b0]], [[c]] {offsets = [0, 0], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s1:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0, 2], sizes = [1, 2], strides = [1, 1]} : vector<1x4xf32> to vector<1x2xf32>
+//       CHECK: [[b1:%.+]] = vector.broadcast [[s1]] : vector<1x2xf32> to vector<2x2xf32>
+//       CHECK: [[r1:%.+]] = vector.insert_strided_slice [[b1]], [[r0]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s2:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0, 0], sizes = [1, 2], strides = [1, 1]} : vector<1x4xf32> to vector<1x2xf32>
+//       CHECK: [[b2:%.+]] = vector.broadcast [[s2]] : vector<1x2xf32> to vector<2x2xf32>
+//       CHECK: [[r2:%.+]] = vector.insert_strided_slice [[b2]], [[r1]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s3:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0, 2], sizes = [1, 2], strides = [1, 1]} : vector<1x4xf32> to vector<1x2xf32>
+//       CHECK: [[b3:%.+]] = vector.broadcast [[s3]] : vector<1x2xf32> to vector<2x2xf32>
+//       CHECK: [[r3:%.+]] = vector.insert_strided_slice [[b3]], [[r2]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: return [[r3]] : vector<4x4xf32>
+
+func.func @vector_broadcast_with_tailing_unit_dim(%v: vector<4x1xf32>) -> vector<4x4xf32> {
+  %0 = vector.broadcast %v : vector<4x1xf32> to vector<4x4xf32>
+  return %0 : vector<4x4xf32>
+}
+
+// CHECK-LABEL: func.func @vector_broadcast_with_tailing_unit_dim
+//  CHECK-SAME: ([[arg0:%.+]]: vector<4x1xf32>) -> vector<4x4xf32> {
+//       CHECK: [[c:%.+]] = arith.constant dense<0.000000e+00> : vector<4x4xf32>
+//       CHECK: [[s0:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0, 0], sizes = [2, 1], strides = [1, 1]} : vector<4x1xf32> to vector<2x1xf32>
+//       CHECK: [[b0:%.+]] = vector.broadcast [[s0]] : vector<2x1xf32> to vector<2x2xf32>
+//       CHECK: [[r0:%.+]] = vector.insert_strided_slice [[b0]], [[c]] {offsets = [0, 0], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s1:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [0, 0], sizes = [2, 1], strides = [1, 1]} : vector<4x1xf32> to vector<2x1xf32>
+//       CHECK: [[b1:%.+]] = vector.broadcast [[s1]] : vector<2x1xf32> to vector<2x2xf32>
+//       CHECK: [[r1:%.+]] = vector.insert_strided_slice [[b1]], [[r0]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s2:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [2, 0], sizes = [2, 1], strides = [1, 1]} : vector<4x1xf32> to vector<2x1xf32>
+//       CHECK: [[b2:%.+]] = vector.broadcast [[s2]] : vector<2x1xf32> to vector<2x2xf32>
+//       CHECK: [[r2:%.+]] = vector.insert_strided_slice [[b2]], [[r1]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: [[s3:%.+]] = vector.extract_strided_slice [[arg0]] {offsets = [2, 0], sizes = [2, 1], strides = [1, 1]} : vector<4x1xf32> to vector<2x1xf32>
+//       CHECK: [[b3:%.+]] = vector.broadcast [[s3]] : vector<2x1xf32> to vector<2x2xf32>
+//       CHECK: [[r3:%.+]] = vector.insert_strided_slice [[b3]], [[r2]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
+//       CHECK: return [[r3]] : vector<4x4xf32>
diff --git a/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir b/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
new file mode 100644
index 000000000000..8e3673d04eac
--- /dev/null
+++ b/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
@@ -0,0 +1,358 @@
+// RUN: mlir-opt --xegpu-blocking -split-input-file %s | FileCheck %s
+
+#a = #xegpu.layout<inst_data = [8, 16], lane_layout = [1, 16], lane_data = [8, 1]>
+#b = #xegpu.layout<inst_data = [16, 16], lane_layout = [1, 16], lane_data = [16, 1]>
+#c = #xegpu.layout<inst_data = [8, 16], lane_layout = [1, 16], lane_data = [8, 1]>
+gpu.module @test_kernel {
+  gpu.func @test_gemm_with_one_to_n_lowering(%A: memref<1024x1024xf16>, %B: memref<1024x1024xf16>, %C: memref<1024x1024xf32>) {
+    %c0 = arith.constant 0 : index
+    %c16 = arith.constant 16 : index
+    %c32 = arith.constant 32 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id x
+    %block_id_y = gpu.block_id y
+    %m = arith.muli %block_id_x, %c16 : index
+    %n = arith.muli %block_id_y, %c32 : index
+
+    %c_tdesc = xegpu.create_nd_tdesc %C[%m, %n] : memref<1024x1024xf32> -> !xegpu.tensor_desc<16x32xf32, #c>
+    %c_init = xegpu.load_nd %c_tdesc : !xegpu.tensor_desc<16x32xf32, #c> -> vector<16x32xf32>
+
+    %a_tdesc = xegpu.create_nd_tdesc %A[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x32xf16, #a>
+    %b_tdesc = xegpu.create_nd_tdesc %B[%c0, %n] : memref<1024x1024xf16> -> !xegpu.tensor_desc<32x32xf16, #b>
+    %out:3 = scf.for %k = %c0 to %c1024 step %c32
+      iter_args(%arg0 = %a_tdesc, %arg1 = %b_tdesc, %arg2 = %c_init)
+      -> (!xegpu.tensor_desc<16x32xf16, #a>, !xegpu.tensor_desc<32x32xf16, #b>, vector<16x32xf32>) {
+      //CHECK-COUNT-4: xegpu.load_nd {{.*}} -> vector<8x16xf16>
+      %a = xegpu.load_nd %arg0 : !xegpu.tensor_desc<16x32xf16, #a> -> vector<16x32xf16>
+      //CHECK-COUNT-4: xegpu.load_nd {{.*}} -> vector<16x16xf16>
+      %b = xegpu.load_nd %arg1 : !xegpu.tensor_desc<32x32xf16, #b> -> vector<32x32xf16>
+      //CHECK-COUNT-8: xegpu.dpas {{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [8, 1]>} : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+      %c = xegpu.dpas %a, %b, %arg2 {layout_result_0 = #c}: vector<16x32xf16>, vector<32x32xf16>, vector<16x32xf32> -> vector<16x32xf32>
+      //CHECK-COUNT-4: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [8, 1]>>
+      %a_next_tdesc = xegpu.update_nd_offset %arg0, [%c0, %c32] : !xegpu.tensor_desc<16x32xf16, #a>
+      //CHECK-COUNT-4: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [16, 1]>>
+      %b_next_tdesc = xegpu.update_nd_offset %arg1, [%c32, %c0] : !xegpu.tensor_desc<32x32xf16, #b>
+      scf.yield %a_next_tdesc, %b_next_tdesc, %c
+        : !xegpu.tensor_desc<16x32xf16, #a>, !xegpu.tensor_desc<32x32xf16, #b>, vector<16x32xf32>
+    }
+    //CHECK-COUNT-4: xegpu.store_nd {{.*}} : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [8, 1]>>
+    xegpu.store_nd %out#2, %c_tdesc: vector<16x32xf32>, !xegpu.tensor_desc<16x32xf32, #c>
+    gpu.return
+  }
+}
+
+// -----
+#l1 = #xegpu.layout<inst_data = [8, 16]>
+#l2 = #xegpu.layout<inst_data = [16, 16]>
+gpu.module @test_kernel {
+  gpu.func @test_gemm_with_inst_data_only_attribute(%A: memref<1024x1024xf16>, %B: memref<1024x1024xf16>, %C: memref<1024x1024xf32>) {
+    %c0 = arith.constant 0 : index
+    %c16 = arith.constant 16 : index
+    %c32 = arith.constant 32 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id x
+    %block_id_y = gpu.block_id y
+    %m = arith.muli %block_id_x, %c16 : index
+    %n = arith.muli %block_id_y, %c32 : index
+
+    %c_tdesc = xegpu.create_nd_tdesc %C[%m, %n] : memref<1024x1024xf32> -> !xegpu.tensor_desc<16x32xf32, #l1>
+    %c_init = xegpu.load_nd %c_tdesc : !xegpu.tensor_desc<16x32xf32, #l1> -> vector<16x32xf32>
+
+    %a_tdesc = xegpu.create_nd_tdesc %A[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x32xf16, #l1>
+    %b_tdesc = xegpu.create_nd_tdesc %B[%c0, %n] : memref<1024x1024xf16> -> !xegpu.tensor_desc<32x32xf16, #l2>
+    %out:3 = scf.for %k = %c0 to %c1024 step %c32
+      iter_args(%arg0 = %a_tdesc, %arg1 = %b_tdesc, %arg2 = %c_init)
+      -> (!xegpu.tensor_desc<16x32xf16, #l1>, !xegpu.tensor_desc<32x32xf16, #l2>, vector<16x32xf32>) {
+      //CHECK-COUNT-4: xegpu.load_nd {{.*}} -> vector<8x16xf16>
+      %a = xegpu.load_nd %arg0 : !xegpu.tensor_desc<16x32xf16, #l1> -> vector<16x32xf16>
+      //CHECK-COUNT-4: xegpu.load_nd {{.*}} -> vector<16x16xf16>
+      %b = xegpu.load_nd %arg1 : !xegpu.tensor_desc<32x32xf16, #l2> -> vector<32x32xf16>
+      //CHECK-COUNT-8: xegpu.dpas {{.*}} : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+      %c = xegpu.dpas %a, %b, %arg2 {layout_result_0 = #l1}: vector<16x32xf16>, vector<32x32xf16>, vector<16x32xf32> -> vector<16x32xf32>
+      //CHECK-COUNT-4: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<8x16xf16>
+      %a_next_tdesc = xegpu.update_nd_offset %arg0, [%c0, %c32] : !xegpu.tensor_desc<16x32xf16, #l1>
+      //CHECK-COUNT-4: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<16x16xf16>
+      %b_next_tdesc = xegpu.update_nd_offset %arg1, [%c32, %c0] : !xegpu.tensor_desc<32x32xf16, #l2>
+      scf.yield %a_next_tdesc, %b_next_tdesc, %c
+        : !xegpu.tensor_desc<16x32xf16, #l1>, !xegpu.tensor_desc<32x32xf16, #l2>, vector<16x32xf32>
+    }
+    //CHECK-COUNT-4: xegpu.store_nd {{.*}} : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+    xegpu.store_nd %out#2, %c_tdesc: vector<16x32xf32>, !xegpu.tensor_desc<16x32xf32, #l1>
+    gpu.return
+  }
+}
+
+// -----
+#l1 = #xegpu.layout<inst_data = [8, 16]>
+#l2 = #xegpu.layout<inst_data = [16, 16]>
+gpu.module @test_kernel {
+  gpu.func @test_gemm_with_one_to_one_lowering(%A: memref<1024x1024xf16>, %B: memref<1024x1024xf16>, %C: memref<1024x1024xf32>) {
+    %c0 = arith.constant 0 : index
+    %c8 = arith.constant 8 : index
+    %c16 = arith.constant 16 : index
+    %c32 = arith.constant 32 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id x
+    %block_id_y = gpu.block_id y
+    %m = arith.muli %block_id_x, %c8 : index
+    %n = arith.muli %block_id_y, %c32 : index
+
+    %c_tdesc = xegpu.create_nd_tdesc %C[%m, %n] : memref<1024x1024xf32> -> !xegpu.tensor_desc<8x32xf32, #l1>
+
+    //CHECK-COUNT-2: xegpu.load_nd {{.*}} : !xegpu.tensor_desc<8x16xf32> -> vector<8x16xf32>
+    %c_init = xegpu.load_nd %c_tdesc : !xegpu.tensor_desc<8x32xf32, #l1> -> vector<8x32xf32>
+
+    %a_tdesc = xegpu.create_nd_tdesc %A[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<8x16xf16, #l1>
+    %b_tdesc = xegpu.create_nd_tdesc %B[%c0, %n] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x32xf16, #l2>
+    %out:3 = scf.for %k = %c0 to %c1024 step %c16
+      iter_args(%arg0 = %a_tdesc, %arg1 = %b_tdesc, %arg2 = %c_init)
+      -> (!xegpu.tensor_desc<8x16xf16, #l1>, !xegpu.tensor_desc<16x32xf16, #l2>, vector<8x32xf32>) {
+      //CHECK: xegpu.load_nd {{.*}} : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+      %a = xegpu.load_nd %arg0 : !xegpu.tensor_desc<8x16xf16, #l1> -> vector<8x16xf16>
+      //CHECK-COUNT-2: xegpu.load_nd {{.*}} : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+      %b = xegpu.load_nd %arg1 : !xegpu.tensor_desc<16x32xf16, #l2> -> vector<16x32xf16>
+      %c = xegpu.dpas %a, %b, %arg2 {layout_result_0 = #l1}: vector<8x16xf16>, vector<16x32xf16>, vector<8x32xf32> -> vector<8x32xf32>
+      //CHECK: xegpu.update_nd_offset {{.*}} [%c0, %c32] : !xegpu.tensor_desc<8x16xf16>
+      %a_next_tdesc = xegpu.update_nd_offset %arg0, [%c0, %c32] : !xegpu.tensor_desc<8x16xf16, #l1>
+      //CHECK-COUNT-2: xegpu.update_nd_offset {{.*}} [%c32, %c0] : !xegpu.tensor_desc<16x16xf16>
+      %b_next_tdesc = xegpu.update_nd_offset %arg1, [%c32, %c0] : !xegpu.tensor_desc<16x32xf16, #l2>
+      scf.yield %a_next_tdesc, %b_next_tdesc, %c
+        : !xegpu.tensor_desc<8x16xf16, #l1>, !xegpu.tensor_desc<16x32xf16, #l2>, vector<8x32xf32>
+    }
+    //CHECK-COUNT-2: xegpu.store_nd {{.*}} : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+    xegpu.store_nd %out#2, %c_tdesc: vector<8x32xf32>, !xegpu.tensor_desc<8x32xf32, #l1>
+    gpu.return
+  }
+}
+
+// -----
+#a = #xegpu.layout<inst_data = [8, 16], lane_layout = [1, 16], lane_data = [8, 1]>
+#b = #xegpu.layout<inst_data = [16, 16], lane_layout = [1, 16], lane_data = [16, 1]>
+#c = #xegpu.layout<inst_data = [8, 16], lane_layout = [1, 16], lane_data = [8, 1]>
+gpu.module @test_kernel {
+  gpu.func @test_gemm_with_elemwise_preop(%A: memref<1024x1024xf16>, %B: memref<1024x1024xf16>, %C: memref<1024x1024xf32>) {
+    %c0 = arith.constant 0 : index
+    %c16 = arith.constant 16 : index
+    %c32 = arith.constant 32 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id x
+    %block_id_y = gpu.block_id y
+    %m = arith.muli %block_id_x, %c16 : index
+    %n = arith.muli %block_id_y, %c32 : index
+
+    %c_tdesc = xegpu.create_nd_tdesc %C[%m, %n] : memref<1024x1024xf32> -> !xegpu.tensor_desc<16x32xf32, #c>
+    %c_init = xegpu.load_nd %c_tdesc : !xegpu.tensor_desc<16x32xf32, #c> -> vector<16x32xf32>
+
+    %a_tdesc = xegpu.create_nd_tdesc %A[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x32xf16, #a>
+    %b_tdesc = xegpu.create_nd_tdesc %B[%c0, %n] : memref<1024x1024xf16> -> !xegpu.tensor_desc<32x32xf16, #b>
+    %out:3 = scf.for %k = %c0 to %c1024 step %c32
+      iter_args(%arg0 = %a_tdesc, %arg1 = %b_tdesc, %arg2 = %c_init)
+      -> (!xegpu.tensor_desc<16x32xf16, #a>, !xegpu.tensor_desc<32x32xf16, #b>, vector<16x32xf32>) {
+      //CHECK-COUNT-4: xegpu.load_nd {{.*}} -> vector<8x16xf16>
+      %a = xegpu.load_nd %arg0 : !xegpu.tensor_desc<16x32xf16, #a> -> vector<16x32xf16>
+      //CHECK-COUNT-4: xegpu.load_nd {{.*}} -> vector<16x16xf16>
+      %b = xegpu.load_nd %arg1 : !xegpu.tensor_desc<32x32xf16, #b> -> vector<32x32xf16>
+      //CHECK-COUNT-4: math.exp {{.*}} : vector<8x16xf16>
+      %e = math.exp %a {layout_result_0 = #a} : vector<16x32xf16>
+      //CHECK-COUNT-8: xegpu.dpas {{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [8, 1]>} : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+      %c = xegpu.dpas %e, %b, %arg2 {layout_result_0 = #c}: vector<16x32xf16>, vector<32x32xf16>, vector<16x32xf32> -> vector<16x32xf32>
+      //CHECK-COUNT-4: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [8, 1]>>
+      %a_next_tdesc = xegpu.update_nd_offset %arg0, [%c0, %c32] : !xegpu.tensor_desc<16x32xf16, #a>
+      //CHECK-COUNT-4: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [16, 1]>>
+      %b_next_tdesc = xegpu.update_nd_offset %arg1, [%c32, %c0] : !xegpu.tensor_desc<32x32xf16, #b>
+      scf.yield %a_next_tdesc, %b_next_tdesc, %c
+        : !xegpu.tensor_desc<16x32xf16, #a>, !xegpu.tensor_desc<32x32xf16, #b>, vector<16x32xf32>
+    }
+    //CHECK-COUNT-4: xegpu.store_nd {{.*}} : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [8, 1]>>
+    xegpu.store_nd %out#2, %c_tdesc: vector<16x32xf32>, !xegpu.tensor_desc<16x32xf32, #c>
+    gpu.return
+  }
+}
+
+// -----
+#l = #xegpu.layout<inst_data = [8, 16]>
+gpu.module @test_kernel {
+  gpu.func @test_elementwise_with_inst_data_only(%A: memref<1024x1024xf16>, %B: memref<1024x1024xf16>, %C: memref<1024x1024xf16>) {
+    %c0 = arith.constant 0 : index
+    %c32 = arith.constant 32 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id x
+    %block_id_y = gpu.block_id y
+    %m = arith.muli %block_id_x, %c32 : index
+
+    %a_tdesc = xegpu.create_nd_tdesc %A[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x32xf16, #l>
+    %b_tdesc = xegpu.create_nd_tdesc %B[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x32xf16, #l>
+    %c_tdesc = xegpu.create_nd_tdesc %C[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x32xf16, #l>
+
+    %out:3 = scf.for %k = %c0 to %c1024 step %c32
+      iter_args(%arg0 = %a_tdesc, %arg1 = %b_tdesc, %arg2 = %c_tdesc)
+      -> (!xegpu.tensor_desc<16x32xf16, #l>, !xegpu.tensor_desc<16x32xf16, #l>, !xegpu.tensor_desc<16x32xf16, #l>) {
+      //CHECK-COUNT-8: xegpu.load_nd {{.*}}  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+      %a = xegpu.load_nd %arg0 : !xegpu.tensor_desc<16x32xf16, #l> -> vector<16x32xf16>
+      %b = xegpu.load_nd %arg1 : !xegpu.tensor_desc<16x32xf16, #l> -> vector<16x32xf16>
+
+      //CHECK-COUNT-4: arith.addf {{.*}} : vector<8x16xf16>
+      %c = arith.addf %a, %b {layout_result_0 = #l} : vector<16x32xf16>
+
+      //CHECK-COUNT-4: xegpu.store_nd {{.*}} : vector<8x16xf16>, !xegpu.tensor_desc<8x16xf16>
+      xegpu.store_nd %c, %arg2: vector<16x32xf16>, !xegpu.tensor_desc<16x32xf16, #l>
+
+      //CHECK-COUNT-12: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<8x16xf16>
+      %a_next_tdesc = xegpu.update_nd_offset %arg0, [%c0, %c32] : !xegpu.tensor_desc<16x32xf16, #l>
+      %b_next_tdesc = xegpu.update_nd_offset %arg1, [%c0, %c32] : !xegpu.tensor_desc<16x32xf16, #l>
+      %c_next_tdesc = xegpu.update_nd_offset %arg2, [%c0, %c32] : !xegpu.tensor_desc<16x32xf16, #l>
+      scf.yield %a_next_tdesc, %b_next_tdesc, %c_next_tdesc
+        : !xegpu.tensor_desc<16x32xf16, #l>, !xegpu.tensor_desc<16x32xf16, #l>, !xegpu.tensor_desc<16x32xf16, #l>
+    }
+    gpu.return
+  }
+}
+
+// -----
+#l = #xegpu.layout<inst_data = [8]>
+gpu.module @test_kernel {
+  gpu.func @test_elementwise_1D(%A: memref<1024x1024xf16>, %B: memref<1024x1024xf16>, %C: memref<1024x1024xf16>) {
+    %c0 = arith.constant 0 : index
+    %c32 = arith.constant 32 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id x
+    %block_id_y = gpu.block_id y
+    %m = arith.muli %block_id_x, %c32 : index
+
+    %a_tdesc = xegpu.create_nd_tdesc %A[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<32xf16, #l>
+    %b_tdesc = xegpu.create_nd_tdesc %B[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<32xf16, #l>
+    %c_tdesc = xegpu.create_nd_tdesc %C[%m, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<32xf16, #l>
+
+    %out:3 = scf.for %k = %c0 to %c1024 step %c32
+      iter_args(%arg0 = %a_tdesc, %arg1 = %b_tdesc, %arg2 = %c_tdesc)
+      -> (!xegpu.tensor_desc<32xf16, #l>, !xegpu.tensor_desc<32xf16, #l>, !xegpu.tensor_desc<32xf16, #l>) {
+      //CHECK-COUNT-8: xegpu.load_nd {{.*}}  : !xegpu.tensor_desc<8xf16> -> vector<8xf16>
+      %a = xegpu.load_nd %arg0 : !xegpu.tensor_desc<32xf16, #l> -> vector<32xf16>
+      %b = xegpu.load_nd %arg1 : !xegpu.tensor_desc<32xf16, #l> -> vector<32xf16>
+
+      //CHECK-COUNT-4: arith.addf {{.*}} : vector<8xf16>
+      %c = arith.addf %a, %b {layout_result_0 = #l} : vector<32xf16>
+
+      //CHECK-COUNT-4: xegpu.store_nd {{.*}} : vector<8xf16>, !xegpu.tensor_desc<8xf16>
+      xegpu.store_nd %c, %arg2: vector<32xf16>, !xegpu.tensor_desc<32xf16, #l>
+
+      //CHECK-COUNT-12: xegpu.update_nd_offset {{.*}} : !xegpu.tensor_desc<8xf16>
+      %a_next_tdesc = xegpu.update_nd_offset %arg0, [%c32] : !xegpu.tensor_desc<32xf16, #l>
+      %b_next_tdesc = xegpu.update_nd_offset %arg1, [%c32] : !xegpu.tensor_desc<32xf16, #l>
+      %c_next_tdesc = xegpu.update_nd_offset %arg2, [%c32] : !xegpu.tensor_desc<32xf16, #l>
+      scf.yield %a_next_tdesc, %b_next_tdesc, %c_next_tdesc
+        : !xegpu.tensor_desc<32xf16, #l>, !xegpu.tensor_desc<32xf16, #l>, !xegpu.tensor_desc<32xf16, #l>
+    }
+    gpu.return
+  }
+}
+
+// -----
+#l = #xegpu.layout<inst_data = [16, 16]>
+#r = #xegpu.layout<inst_data = [16]>
+
+gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+  gpu.func @reduce_dim_0(%a: memref<16x512xf32>, %b: memref<512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+    %acc = arith.constant dense<0.0> : vector<64xf32>
+    %c64 = arith.constant 64 : index
+    %block_id_x = gpu.block_id x
+    %m = arith.muli %block_id_x, %c64 : index
+    %0 = xegpu.create_nd_tdesc %a[0, %m] : memref<16x512xf32> -> !xegpu.tensor_desc<16x64xf32, #l>
+    %1 = xegpu.load_nd %0: !xegpu.tensor_desc<16x64xf32, #l> -> vector<16x64xf32>
+    // CHECK: vector.multi_reduction <add>, {{.*}}, [[ACC:%[0-9A-Za-z]+]] [0] : vector<16x16xf32> to vector<16xf32>
+    // CHECK-COUNT-3: vector.multi_reduction <add>, {{.*}}, [[ACC]] [0] : vector<16x16xf32> to vector<16xf32>
+    %2 = vector.multi_reduction <add>, %1, %acc {layout_result_0 = #r} [0]: vector<16x64xf32> to vector<64xf32>
+    %3 = xegpu.create_nd_tdesc %b[%m] : memref<512xf32> -> !xegpu.tensor_desc<64xf32, #r>
+    xegpu.store_nd %2, %3: vector<64xf32>, !xegpu.tensor_desc<64xf32, #r>
+    gpu.return
+  }
+}
+
+// -----
+#l = #xegpu.layout<inst_data = [16, 16]>
+#r = #xegpu.layout<inst_data = [16]>
+
+gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+  gpu.func @reduce_dim_1(%a: memref<512x32xf32>, %b: memref<512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+    %c1 = arith.constant 1 : index
+    %c32 = arith.constant 32 : index
+    %acc = arith.constant dense<0.0> : vector<32xf32>
+
+    %block_id_x = gpu.block_id x
+    %block_id_y = gpu.block_id y
+
+    %m = arith.muli %block_id_x, %c32 : index
+    %n = arith.muli %block_id_y, %c32 : index
+    %0 = xegpu.create_nd_tdesc %a[%m, %n] : memref<512x32xf32> -> !xegpu.tensor_desc<32x128xf32, #l>
+    %1 = xegpu.load_nd %0: !xegpu.tensor_desc<32x128xf32, #l> -> vector<32x128xf32>
+
+    // CHECK: vector.multi_reduction <add>, {{.*}}, [[INIT:%[0-9A-Za-z]+]] [1] : vector<16x16xf32> to vector<16xf32>
+    // CHECK-COUNT-1: vector.multi_reduction <add>, {{.*}}, [[INIT]] [1] : vector<16x16xf32> to vector<16xf32>
+
+    %2 = vector.multi_reduction <add>, %1, %acc {layout_result_0 = #r} [1]: vector<32x128xf32> to vector<32xf32>
+    %3 = xegpu.create_nd_tdesc %b[%n] : memref<512xf32> -> !xegpu.tensor_desc<32xf32, #r>
+    xegpu.store_nd %2, %3: vector<32xf32>, !xegpu.tensor_desc<32xf32, #r>
+    gpu.return
+  }
+}
+
+// -----
+#r = #xegpu.layout<inst_data = [16]>
+#l = #xegpu.layout<inst_data = [16, 16]>
+
+gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+  gpu.func @broadcast_dim_0(%a: memref<512xf32>, %b: memref<16x512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+
+    %c64 = arith.constant 64 : index
+    %block_id_x = gpu.block_id x
+    %m = arith.muli %block_id_x, %c64 : index
+    %0 = xegpu.create_nd_tdesc %a[%m] : memref<512xf32> -> !xegpu.tensor_desc<64xf32, #r>
+    %1 = xegpu.load_nd %0: !xegpu.tensor_desc<64xf32, #r> -> vector<64xf32>
+    // CHECK-COUNT-4: vector.broadcast {{.*}} : vector<16xf32> to vector<16x16xf32>
+    %2 = vector.broadcast  %1 {layout_result_0 = #l} : vector<64xf32> to vector<16x64xf32>
+    %3 = xegpu.create_nd_tdesc %b[0, %m] : memref<16x512xf32> -> !xegpu.tensor_desc<16x64xf32, #l>
+    xegpu.store_nd %2, %3: vector<16x64xf32>, !xegpu.tensor_desc<16x64xf32, #l>
+    gpu.return
+  }
+}
+
+// -----
+#r = #xegpu.layout<inst_data = [16]>
+#l = #xegpu.layout<inst_data = [16, 16]>
+
+gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+  gpu.func @broadcast_dim_1(%a: memref<512xf32>, %b: memref<16x512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+
+    %c32 = arith.constant 32 : index
+    %block_id_x = gpu.block_id x
+    %m = arith.muli %block_id_x, %c32 : index
+    %0 = xegpu.create_nd_tdesc %a[%m] : memref<512xf32> -> !xegpu.tensor_desc<32xf32, #r>
+    %1 = xegpu.load_nd %0: !xegpu.tensor_desc<32xf32, #r> -> vector<32xf32>
+    %11 = vector.shape_cast %1 :  vector<32xf32> to vector<32x1xf32>
+    // CHECK-COUNT-8: vector.broadcast {{.*}}: vector<16x1xf32> to vector<16x16xf32>
+    %2 = vector.broadcast  %11 {layout_result_0 = #l} : vector<32x1xf32> to vector<32x64xf32>
+    %3 = xegpu.create_nd_tdesc %b[0, %m] : memref<16x512xf32> -> !xegpu.tensor_desc<32x64xf32, #l>
+    xegpu.store_nd %2, %3: vector<32x64xf32>, !xegpu.tensor_desc<32x64xf32, #l>
+    gpu.return
+  }
+}
+
+// -----
+#l = #xegpu.layout<inst_data = [16, 8]>
+#t = #xegpu.layout<inst_data = [8, 16]>
+
+gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+  gpu.func @transpose(%a: memref<512x8xf32>, %b: memref<8x512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+
+    %c32 = arith.constant 32 : index
+    %block_id_x = gpu.block_id x
+    %m = arith.muli %block_id_x, %c32 : index
+    %0 = xegpu.create_nd_tdesc %a[%m, 0] : memref<512x8xf32> -> !xegpu.tensor_desc<32x8xf32, #l>
+    %1 = xegpu.load_nd %0: !xegpu.tensor_desc<32x8xf32, #l> -> vector<32x8xf32>
+    // CHECK-COUNT-2: vector.transpose {{.*}}  [1, 0] : vector<16x8xf32> to vector<8x16xf32>
+    %2 = vector.transpose  %1, [1, 0] {layout_result_0 = #t} : vector<32x8xf32> to vector<8x32xf32>
+    %3 = xegpu.create_nd_tdesc %b[0, %m] : memref<8x512xf32> -> !xegpu.tensor_desc<8x32xf32, #t>
+    xegpu.store_nd %2, %3: vector<8x32xf32>, !xegpu.tensor_desc<8x32xf32, #t>
+    gpu.return
+  }
+}
\ No newline at end of file
diff --git a/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir b/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir
index bee026eb2084..35ad16d8cd9a 100644
--- a/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir
+++ b/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir
@@ -85,7 +85,7 @@ gpu.module @test_round_robin_assignment {
     %tdesc_c = xegpu.create_nd_tdesc %c[0, 0] : memref<8x8xf32>
       -> !xegpu.tensor_desc<8x8xf32, #xegpu.layout<sg_layout = [2, 2], sg_data = [2, 2], lane_layout = [2, 2], lane_data = [1, 1]>>
     %dpas = xegpu.dpas %load_a, %load_b
-      {layout =  #xegpu.layout<sg_layout = [2, 2], sg_data = [2, 2], lane_layout = [2, 2], lane_data = [1, 1]>}
+      {layout_result_0 =  #xegpu.layout<sg_layout = [2, 2], sg_data = [2, 2], lane_layout = [2, 2], lane_data = [1, 1]>}
       : vector<8x8xf32>, vector<8x8xf32> -> vector<8x8xf32>
     gpu.return
   }
@@ -102,4 +102,100 @@ gpu.module @test_round_robin_assignment {
       : !xegpu.tensor_desc<24x32xf32, #xegpu.layout<sg_layout = [4, 4], sg_data = [2, 2], lane_layout = [2, 2], lane_data = [1, 1]>>
     gpu.return
   }
+
+  gpu.func @test_scf_for(%arg0: memref<1024xf32>, %arg1: memref<1024xf32>) {
+    %c1 = arith.constant 1 : index
+    %c10 = arith.constant 10 : index
+    %c0 = arith.constant 0 : index
+    %c256 = arith.constant 256 : index
+    %c1024 = arith.constant 1024 : index
+    %0 = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    %1 = xegpu.create_nd_tdesc %arg1[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    // CHECK-LABEL: scf.for
+    // CHECK-SAME: (!xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>)
+    %2:2 = scf.for %arg2 = %c0 to %c1024 step %c256 iter_args(%arg3 = %0, %arg4 = %1)
+        -> (!xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>) {
+      %3 = xegpu.load_nd %0  : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>> -> vector<256xf32>
+      xegpu.store_nd %3, %arg3  : vector<256xf32>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+      %4 = xegpu.update_nd_offset %arg3, [256] : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+      %5 = xegpu.update_nd_offset %arg4, [256] : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+      // CHECK-LABEL: scf.yield
+      // CHECK-SAME: !xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>
+      scf.yield %4, %5 : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    }
+    gpu.return
+  }
+
+  gpu.func @test_scf_while_and_condition(%arg0: memref<1024xf32>, %arg1: memref<1024xf32>) {
+    %c1_i32 = arith.constant 1 : i32
+    %c10_i32 = arith.constant 10 : i32
+    %c0_i32 = arith.constant 0 : i32
+    %0 = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    %1 = xegpu.load_nd %0  : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>> -> vector<256xf32>
+    %2 = xegpu.create_nd_tdesc %arg1[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    //CHECK: scf.while ({{.*}}) : (vector<16xf32>, vector<16xf32>, i32) -> (vector<16xf32>, vector<16xf32>, i32)
+    %3:2 = scf.while (%arg2 = %1, %arg3 = %c0_i32) : (vector<256xf32>, i32) -> (vector<256xf32>, i32) {
+      %4 = arith.cmpi slt, %arg3, %c10_i32 : i32
+      //CHECK: scf.condition{{.*}} : vector<16xf32>, vector<16xf32>, i32
+      scf.condition(%4) %arg2, %arg3 : vector<256xf32>, i32
+    } do {
+    // CHECK: ([[arg2:%.+]]: vector<16xf32>, [[arg3:%.+]]: vector<16xf32>, [[arg4:%.+]]: i32)
+    ^bb0(%arg2: vector<256xf32>, %arg3: i32):
+      xegpu.store_nd %arg2, %2  : vector<256xf32>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+      %4 = arith.addi %arg3, %c1_i32 : i32
+      %5 = xegpu.update_nd_offset %0, [256] : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+      %6 = xegpu.load_nd %5  : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>> -> vector<256xf32>
+      scf.yield %6, %4 : vector<256xf32>, i32
+    }
+    gpu.return
+  }
+
+  gpu.func @test_scf_if(%arg0: memref<1024xf32>, %arg1: memref<1024xf32>) {
+    %c10 = arith.constant 10 : index
+    %0 = gpu.subgroup_id : index
+    %1 = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    %2 = xegpu.create_nd_tdesc %arg1[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    %3 = arith.cmpi eq, %0, %c10 : index
+    // CHECK-LABEL: scf.if
+    //  CHECK-SAME: (vector<16xf32>, vector<16xf32>)
+    %4 = scf.if %3 -> (vector<256xf32>) {
+      %5 = xegpu.load_nd %1  : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>> -> vector<256xf32>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: vector<16xf32>, vector<16xf32>
+      scf.yield %5 : vector<256xf32>
+    } else {
+      %5 = xegpu.load_nd %2  : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>> -> vector<256xf32>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: vector<16xf32>, vector<16xf32>
+      scf.yield %5 : vector<256xf32>
+    } {layout_result_0 = #xegpu.layout<sg_layout = [8], sg_data = [16]>}
+    xegpu.store_nd %4, %1  : vector<256xf32>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    gpu.return
+  }
+
+  gpu.func @test_scf_if_tensor_desc(%arg0: memref<1024xf32>, %arg1: memref<1024xf32>) {
+    %c10 = arith.constant 10 : index
+    %id = gpu.subgroup_id : index
+
+    %t = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    %d = xegpu.load_nd %t : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>> -> vector<256xf32>
+
+    %0 = arith.cmpi eq, %id, %c10 : index
+    // CHECK-LABEL: scf.if
+    //  CHECK-SAME: (!xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>)
+    %1 = scf.if %0 -> (!xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>) {
+      %2 = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: !xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>
+      scf.yield %2 : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    } else {
+      %3 = xegpu.create_nd_tdesc %arg1[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: !xegpu.tensor_desc<16xf32>, !xegpu.tensor_desc<16xf32>
+      scf.yield %3 : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    }
+    xegpu.store_nd %d, %1 : vector<256xf32>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [8], sg_data = [16]>>
+    gpu.return
+  }
+
 }
diff --git a/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir b/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir
index 7e89ada93407..466842c96844 100644
--- a/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir
+++ b/external/llvm-project/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir
@@ -5,7 +5,7 @@
 gpu.module @test_1_1_assignment {
   // CHECK-LABEL: test_create_nd_tdesc
   // CHECK-SAME: %[[ARG_0:.*]]: memref<24x32xf32>
-  gpu.func @test_create_nd_tdesc(%src: memref<24x32xf32>) {  
+  gpu.func @test_create_nd_tdesc(%src: memref<24x32xf32>) {
   // CHECK: %[[SGID:.*]] = gpu.subgroup_id
   // CHECK: %[[C12:.*]] = arith.constant 12 : index
   // CHECK: %[[C4:.*]] = arith.constant 4 : index
@@ -108,7 +108,7 @@ gpu.func @test_dpas(%a: memref<24x32xf32>, %b: memref<32x24xf32>) {
       : !xegpu.tensor_desc<32x24xf32, #xegpu.layout<sg_layout = [4, 2], sg_data = [8, 12], lane_layout = [8, 2], lane_data = [1, 1]>>
       -> vector<32x24xf32>
     %dpas = xegpu.dpas %load_a, %load_b
-      {layout =  #xegpu.layout<sg_layout = [2, 2], sg_data = [12, 12], lane_layout = [2, 2], lane_data = [1, 1]>}
+      {layout_result_0 =  #xegpu.layout<sg_layout = [2, 2], sg_data = [12, 12], lane_layout = [2, 2], lane_data = [1, 1]>}
       : vector<24x32xf32>, vector<32x24xf32> -> vector<24x24xf32>
     gpu.return
   }
@@ -142,7 +142,7 @@ gpu.func @test_dpas_no_sg_data(%a: memref<24x32xf32>, %b: memref<32x24xf32>) {
       : !xegpu.tensor_desc<32x24xf32, #xegpu.layout<sg_layout = [4, 2], lane_layout = [8, 2], lane_data = [1, 1]>>
       -> vector<32x24xf32>
     %dpas = xegpu.dpas %load_a, %load_b
-      {layout =  #xegpu.layout<sg_layout = [2, 2], lane_layout = [2, 2], lane_data = [1, 1]>}
+      {layout_result_0 =  #xegpu.layout<sg_layout = [2, 2], lane_layout = [2, 2], lane_data = [1, 1]>}
       : vector<24x32xf32>, vector<32x24xf32> -> vector<24x24xf32>
     gpu.return
   }
@@ -169,4 +169,132 @@ gpu.func @test_dpas_no_sg_data(%a: memref<24x32xf32>, %b: memref<32x24xf32>) {
       : vector<24x32xf32>, vector<32x24xf32> -> vector<24x24xf32>
     gpu.return
   }
+
+  gpu.func @test_scf_for(%arg0: memref<1024x1024xf16>, %arg1: memref<1024x1024xf16>, %arg2: memref<1024x1024xf32>) {
+    //CHECK: [[c0:%.+]] = arith.constant 0 : index
+    //CHECK: [[c128:%.+]] = arith.constant 128 : index
+    //CHECK: [[c1024:%.+]] = arith.constant 1024 : index
+    %c0 = arith.constant 0 : index
+    %c128 = arith.constant 128 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id  x
+    %block_id_y = gpu.block_id  y
+    %0 = arith.muli %block_id_x, %c128 : index
+    %1 = arith.muli %block_id_y, %c128 : index
+    %2 = xegpu.create_nd_tdesc %arg2[%0, %1] : memref<1024x1024xf32> -> !xegpu.tensor_desc<128x128xf32, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>>
+    %3 = xegpu.load_nd %2 : !xegpu.tensor_desc<128x128xf32, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>> -> vector<128x128xf32>
+    %4 = xegpu.create_nd_tdesc %arg0[%0, %c0] : memref<1024x1024xf16> -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>>
+    %5 = xegpu.create_nd_tdesc %arg1[%c0, %1] : memref<1024x1024xf16> -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>>
+
+    //      CHECK: [[scf:%.+]]:3 = scf.for [[arg3:%.+]] = [[c0]] to [[c1024]] step [[c128]]
+    // CHECK-SAME: iter_args([[arg4:%.+]] = {{.*}}, [[arg5:%.+]] = {{.*}}, [[arg6:%.+]] = {{.*}}) ->
+    // CHECK-SAME: (!xegpu.tensor_desc<16x128xf16>, !xegpu.tensor_desc<128x16xf16>, vector<16x16xf32>)
+    //      CHECK: [[a:%.+]] = xegpu.load_nd [[arg4]] : !xegpu.tensor_desc<16x128xf16> -> vector<16x128xf16>
+    //      CHECK: [[b:%.+]] = xegpu.load_nd [[arg5]] : !xegpu.tensor_desc<128x16xf16> -> vector<128x16xf16>
+    //      CHECK: [[c:%.+]] = xegpu.dpas [[a]], [[b]], [[arg6]] : vector<16x128xf16>, vector<128x16xf16>, vector<16x16xf32> -> vector<16x16xf32>
+    //      CHECK: [[at:%.+]] = xegpu.update_nd_offset [[arg4]], [[[c0]], [[c128]]] : !xegpu.tensor_desc<16x128xf16>
+    //      CHECK: [[bt:%.+]] = xegpu.update_nd_offset [[arg5]], [[[c128]], [[c0]]] : !xegpu.tensor_desc<128x16xf16>
+    //      CHECK: scf.yield [[at]], [[bt]], [[c]] : !xegpu.tensor_desc<16x128xf16>, !xegpu.tensor_desc<128x16xf16>, vector<16x16xf32>
+    %6:3 = scf.for %arg3 = %c0 to %c1024 step %c128 iter_args(%arg4 = %4, %arg5 = %5, %arg6 = %3)
+        -> (!xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>>,
+            !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>>, vector<128x128xf32>) {
+      %8 = xegpu.load_nd %arg4  : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>> -> vector<128x128xf16>
+      %9 = xegpu.load_nd %arg5  : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>> -> vector<128x128xf16>
+      %10 = xegpu.dpas %8, %9, %arg6 {layout_result_0 = #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>}
+                          : vector<128x128xf16>, vector<128x128xf16>, vector<128x128xf32> -> vector<128x128xf32>
+      %11 = xegpu.update_nd_offset %arg4, [%c0, %c128] : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>>
+      %12 = xegpu.update_nd_offset %arg5, [%c128, %c0] : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>>
+      scf.yield %11, %12, %10 : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>>,
+                                !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>>, vector<128x128xf32>
+    }
+    %7 = xegpu.create_nd_tdesc %arg2[%0, %1] : memref<1024x1024xf32>
+            -> !xegpu.tensor_desc<128x128xf32, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>>
+    xegpu.store_nd %6#2, %7  : vector<128x128xf32>, !xegpu.tensor_desc<128x128xf32, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>>
+    gpu.return
+  }
+
+  gpu.func @test_scf_while_and_condition(%arg0: memref<1024xf32>, %arg1: memref<1024xf32>) {
+    %c1_i32 = arith.constant 1 : i32
+    %c10_i32 = arith.constant 10 : i32
+    %c0_i32 = arith.constant 0 : i32
+    %0 = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+    %1 = xegpu.load_nd %0  : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>> -> vector<256xf32>
+    %2 = xegpu.create_nd_tdesc %arg1[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+
+    // CHECK: scf.while {{.*}} : (vector<16xf32>, i32) -> (vector<16xf32>, i32)
+    %3:2 = scf.while (%arg2 = %1, %arg3 = %c0_i32) : (vector<256xf32>, i32) -> (vector<256xf32>, i32) {
+      %4 = arith.cmpi slt, %arg3, %c10_i32 : i32
+      // CHECK: scf.condition{{.*}} : vector<16xf32>, i32
+      scf.condition(%4) %arg2, %arg3 : vector<256xf32>, i32
+    } do {
+    // CHECK: ([[arg2:%.+]]: vector<16xf32>, [[arg3:%.+]]: i32)
+    ^bb0(%arg2: vector<256xf32>, %arg3: i32):
+      xegpu.store_nd %arg2, %2  : vector<256xf32>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+      %4 = arith.addi %arg3, %c1_i32 : i32
+      %5 = xegpu.update_nd_offset %0, [256] : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+      %6 = xegpu.load_nd %5  : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>> -> vector<256xf32>
+      scf.yield %6, %4 : vector<256xf32>, i32
+    }
+    gpu.return
+  }
+
+  gpu.func @test_scf_if(%arg0: memref<1024xf32>, %arg1: memref<1024xf32>) {
+    %c10 = arith.constant 10 : index
+    %id = gpu.subgroup_id : index
+
+    %0 = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+    %1 = xegpu.create_nd_tdesc %arg1[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+
+    %4 = arith.cmpi eq, %id, %c10 : index
+    // CHECK-LABEL: scf.if
+    //  CHECK-SAME: (vector<16xf32>)
+    %5 = scf.if %4 -> (vector<256xf32>) {
+      // CHECK-LABEL: xegpu.load_nd
+      //  CHECK-SAME: !xegpu.tensor_desc<16xf32> -> vector<16xf32>
+      %2 = xegpu.load_nd %0 : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>> -> vector<256xf32>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: vector<16xf32>
+      scf.yield %2 : vector<256xf32>
+    } else {
+      // CHECK-LABEL: xegpu.load_nd
+      //  CHECK-SAME: !xegpu.tensor_desc<16xf32> -> vector<16xf32>
+      %3 = xegpu.load_nd %1 : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>> -> vector<256xf32>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: vector<16xf32>
+      scf.yield %3 : vector<256xf32>
+    } {layout_result_0 = #xegpu.layout<sg_layout = [16], sg_data = [16]>}
+    xegpu.store_nd %5, %0 : vector<256xf32>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+    gpu.return
+  }
+
+  gpu.func @test_scf_if_tensor_desc(%arg0: memref<1024xf32>, %arg1: memref<1024xf32>) {
+    %c10 = arith.constant 10 : index
+    %id = gpu.subgroup_id : index
+
+    %t = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+    %d = xegpu.load_nd %t : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>> -> vector<256xf32>
+
+    %0 = arith.cmpi eq, %id, %c10 : index
+    // CHECK-LABEL: scf.if
+    //  CHECK-SAME: (!xegpu.tensor_desc<16xf32>)
+    %1 = scf.if %0 -> (!xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>) {
+      // CHECK-LABEL: xegpu.create_nd_tdesc
+      //  CHECK-SAME: memref<1024xf32> -> !xegpu.tensor_desc<16xf32>
+      %2 = xegpu.create_nd_tdesc %arg0[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: !xegpu.tensor_desc<16xf32>
+      scf.yield %2 : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+    } else {
+      // CHECK-LABEL: xegpu.create_nd_tdesc
+      //  CHECK-SAME: memref<1024xf32> -> !xegpu.tensor_desc<16xf32>
+      %3 = xegpu.create_nd_tdesc %arg1[0] : memref<1024xf32> -> !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+      // CHECK-LABEL: scf.yield
+      //  CHECK-SAME: !xegpu.tensor_desc<16xf32>
+      scf.yield %3 : !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+    }
+    xegpu.store_nd %d, %1 : vector<256xf32>, !xegpu.tensor_desc<256xf32, #xegpu.layout<sg_layout = [16], sg_data = [16]>>
+    gpu.return
+  }
+
+
 }
diff --git a/external/llvm-project/mlir/test/IR/test-side-effects.mlir b/external/llvm-project/mlir/test/IR/test-side-effects.mlir
index efce4856041a..b652ecb7dad1 100644
--- a/external/llvm-project/mlir/test/IR/test-side-effects.mlir
+++ b/external/llvm-project/mlir/test/IR/test-side-effects.mlir
@@ -15,7 +15,7 @@ func.func @side_effect(%arg : index) {
     {effect="write", test_resource}
   ]} : () -> i32
   
-  // expected-remark@+1 {{found an instance of 'allocate' on a op result, on resource '<Test>'}}
+  // expected-remark@+1 {{found an instance of 'allocate' on op result 0, on resource '<Test>'}}
   %3 = "test.side_effect_op"() {effects = [
     {effect="allocate", on_result, test_resource}
   ]} : () -> i32
@@ -38,19 +38,19 @@ func.func @side_effect(%arg : index) {
     effect_parameter = affine_map<(i, j) -> (j, i)>
   } : () -> i32
 
-  // expected-remark@+1 {{found an instance of 'allocate' on a op operand, on resource '<Test>'}}
+  // expected-remark@+1 {{found an instance of 'allocate' on op operand 0, on resource '<Test>'}}
   %6 = test.side_effect_with_region_op (%arg) {
   ^bb0(%arg0 : index):
     test.region_yield %arg0 : index 
   } {effects = [ {effect="allocate", on_operand, test_resource} ]} : index -> index
 
-  // expected-remark@+1 {{found an instance of 'allocate' on a op result, on resource '<Test>'}}
+  // expected-remark@+1 {{found an instance of 'allocate' on op result 0, on resource '<Test>'}}
   %7 = test.side_effect_with_region_op (%arg) {
   ^bb0(%arg0 : index):
     test.region_yield %arg0 : index 
   } {effects = [ {effect="allocate", on_result, test_resource} ]} : index -> index
 
-  // expected-remark@+1 {{found an instance of 'allocate' on a block argument, on resource '<Test>'}}
+  // expected-remark@+1 {{found an instance of 'allocate' on block argument 0, on resource '<Test>'}}
   %8 = test.side_effect_with_region_op (%arg) {
   ^bb0(%arg0 : index):
     test.region_yield %arg0 : index 
diff --git a/external/llvm-project/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir b/external/llvm-project/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir
index 1361d21e7d94..63db0def1cbc 100644
--- a/external/llvm-project/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir
+++ b/external/llvm-project/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-linalg-transform-patterns=test-linalg-to-vector-patterns \
+// RUN: mlir-opt %s -transform-interpreter -test-transform-dialect-erase-schedule \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
 // RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref \
 // RUN: -convert-linalg-to-loops -convert-scf-to-cf -expand-strided-metadata \
@@ -34,4 +34,15 @@ func.func @main() {
   return
 }
 
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.op<"func.func">
+
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.linalg.pad_vectorization
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
+
 func.func private @printMemrefF32(%ptr : tensor<*xf32>)
diff --git a/external/llvm-project/mlir/test/Integration/Dialect/Tensor/cast-runtime-verification.mlir b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/cast-runtime-verification.mlir
new file mode 100644
index 000000000000..e4aab32d4a39
--- /dev/null
+++ b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/cast-runtime-verification.mlir
@@ -0,0 +1,50 @@
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-scf-to-cf \
+// RUN:     -convert-to-llvm | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
+func.func private @cast_to_static_dim(%t: tensor<?xf32>) -> tensor<10xf32> {
+  %0 = tensor.cast %t : tensor<?xf32> to tensor<10xf32>
+  return %0 : tensor<10xf32>
+}
+
+func.func private @cast_to_ranked(%t: tensor<*xf32>) -> tensor<f32> {
+  %0 = tensor.cast %t : tensor<*xf32> to tensor<f32>
+  return %0 : tensor<f32>
+}
+
+func.func private @valid_cast(%t: tensor<*xf32>) -> tensor<?xf32> {
+  %0 = tensor.cast %t : tensor<*xf32> to tensor<?xf32>
+  return %0 : tensor<?xf32>
+}
+
+func.func @main() {
+  // All casts inside the called functions are invalid at runtime, except for
+  // the last one.
+  %alloc = tensor.empty() : tensor<5xf32>
+
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.cast"(%{{.*}}) : (tensor<?xf32>) -> tensor<10xf32>
+  // CHECK-NEXT: ^ size mismatch of dim 0
+  // CHECK-NEXT: Location: loc({{.*}})
+  %1 = tensor.cast %alloc : tensor<5xf32> to tensor<?xf32>
+  func.call @cast_to_static_dim(%1) : (tensor<?xf32>) -> (tensor<10xf32>)
+
+  // CHECK-NEXT: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.cast"(%{{.*}}) : (tensor<*xf32>) -> tensor<f32>
+  // CHECK-NEXT: ^ rank mismatch
+  // CHECK-NEXT: Location: loc({{.*}})
+  %3 = tensor.cast %alloc : tensor<5xf32> to tensor<*xf32>
+  func.call @cast_to_ranked(%3) : (tensor<*xf32>) -> (tensor<f32>)
+
+  // A last cast that actually succeeds.
+  // CHECK-NOT: ERROR: Runtime op verification failed
+  func.call @valid_cast(%3) : (tensor<*xf32>) -> (tensor<?xf32>)
+
+  return
+}
diff --git a/external/llvm-project/mlir/test/Integration/Dialect/Tensor/dim-runtime-verification.mlir b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/dim-runtime-verification.mlir
new file mode 100644
index 000000000000..c6d8f698b943
--- /dev/null
+++ b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/dim-runtime-verification.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize \
+// RUN:     -buffer-deallocation-pipeline \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
+func.func @main() {
+  %c4 = arith.constant 4 : index
+  %tensor = tensor.empty() : tensor<1xf32>
+
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.dim"(%{{.*}}, %{{.*}}) : (tensor<1xf32>, index) -> index
+  // CHECK-NEXT: ^ index is out of bounds
+  // CHECK-NEXT: Location: loc({{.*}})
+  %dim = tensor.dim %tensor, %c4 : tensor<1xf32>
+
+  return
+}
diff --git a/external/llvm-project/mlir/test/Integration/Dialect/Tensor/extract-runtime-verification.mlir b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/extract-runtime-verification.mlir
new file mode 100644
index 000000000000..8e3cab7be704
--- /dev/null
+++ b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/extract-runtime-verification.mlir
@@ -0,0 +1,64 @@
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-scf-to-cf \
+// RUN:     -convert-to-llvm | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
+func.func @extract(%tensor: tensor<1xf32>, %index: index) {
+    tensor.extract %tensor[%index] :  tensor<1xf32>
+    return
+}
+
+func.func @extract_dynamic(%tensor: tensor<?xf32>, %index: index) {
+    tensor.extract %tensor[%index] :  tensor<?xf32>
+    return
+}
+
+func.func @extract_nd_dynamic(%tensor: tensor<?x?x?xf32>, %index0: index, %index1: index, %index2: index) {
+    tensor.extract %tensor[%index0, %index1, %index2] :  tensor<?x?x?xf32>
+    return
+}
+
+func.func @main() {
+  %0 = arith.constant 0 : index
+  %1 = arith.constant 1 : index
+  %n1 = arith.constant -1 : index
+  %2 = arith.constant 2 : index
+  %alloca_1 = tensor.empty() : tensor<1xf32>
+  %alloc_1 = tensor.empty(%1) : tensor<?xf32>
+  %alloc_2x2x2 = tensor.empty(%2, %2, %2) : tensor<?x?x?xf32>
+
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract"(%{{.*}}, %{{.*}}) : (tensor<1xf32>, index) -> f32
+  // CHECK-NEXT: ^ out-of-bounds access
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract(%alloca_1, %1) : (tensor<1xf32>, index) -> ()
+
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract"(%{{.*}}, %{{.*}}) : (tensor<?xf32>, index) -> f32
+  // CHECK-NEXT: ^ out-of-bounds access
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract_dynamic(%alloc_1, %1) : (tensor<?xf32>, index) -> ()
+
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract"(%{{.*}}, %{{.*}}) : (tensor<?x?x?xf32>, index, index, index) -> f32
+  // CHECK-NEXT: ^ out-of-bounds access
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract_nd_dynamic(%alloc_2x2x2, %1, %n1, %0) : (tensor<?x?x?xf32>, index, index, index) -> ()
+
+  // CHECK-NOT: ERROR: Runtime op verification failed
+  func.call @extract(%alloca_1, %0) : (tensor<1xf32>, index) -> ()
+
+  // CHECK-NOT: ERROR: Runtime op verification failed
+  func.call @extract_dynamic(%alloc_1, %0) : (tensor<?xf32>, index) -> ()
+
+  // CHECK-NOT: ERROR: Runtime op verification failed
+  func.call @extract_nd_dynamic(%alloc_2x2x2, %1, %1, %0) : (tensor<?x?x?xf32>, index, index, index) -> ()
+
+  return
+}
+
diff --git a/external/llvm-project/mlir/test/Integration/Dialect/Tensor/extract_slice-runtime-verification.mlir b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/extract_slice-runtime-verification.mlir
new file mode 100644
index 000000000000..28f9be0fffe6
--- /dev/null
+++ b/external/llvm-project/mlir/test/Integration/Dialect/Tensor/extract_slice-runtime-verification.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-scf-to-cf \
+// RUN:     -convert-to-llvm | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
+func.func @extract_slice(%tensor: tensor<1xf32>, %offset: index) {
+    tensor.extract_slice %tensor[%offset] [1] [1] : tensor<1xf32> to tensor<1xf32>
+    return
+}
+
+func.func @extract_slice_dynamic(%tensor: tensor<?x4xf32>, %offset: index, %size: index, %stride: index) {
+    tensor.extract_slice %tensor[%offset, 0] [%size, 4] [%stride, 1] : tensor<?x4xf32> to tensor<?x4xf32>
+    return
+}
+
+func.func @extract_slice_dynamic_rank_reduce(%tensor: tensor<?x4xf32>, %offset: index, %size: index, %stride: index) {
+    tensor.extract_slice %tensor[%offset, 0] [%size, 1] [%stride, 1] : tensor<?x4xf32> to tensor<?xf32>
+    return
+}
+
+func.func @main() {
+  %0 = arith.constant 0 : index
+  %1 = arith.constant 1 : index
+  %n1 = arith.constant -1 : index
+  %4 = arith.constant 4 : index
+  %5 = arith.constant 5 : index
+
+  %alloca = tensor.empty() : tensor<1xf32>
+  %alloca_4 = tensor.empty() : tensor<4x4xf32>
+  %alloca_4_dyn = tensor.cast %alloca_4 : tensor<4x4xf32> to tensor<?x4xf32>
+
+  // Offset is out-of-bounds and slice runs out-of-bounds
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1, %arg2, %arg3) <{operandSegmentSizes = array<i32: 1, 1, 1, 1>, static_offsets = array<i64: -9223372036854775808, 0>, static_sizes = array<i64: -9223372036854775808, 1>, static_strides = array<i64: -9223372036854775808, 1>}> : (tensor<?x4xf32>, index, index, index) -> tensor<?xf32>
+  // CHECK-NEXT: ^ offset 0 is out-of-bounds
+  // CHECK-NEXT: Location: loc({{.*}})
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1, %arg2, %arg3) <{operandSegmentSizes = array<i32: 1, 1, 1, 1>, static_offsets = array<i64: -9223372036854775808, 0>, static_sizes = array<i64: -9223372036854775808, 1>, static_strides = array<i64: -9223372036854775808, 1>}> : (tensor<?x4xf32>, index, index, index) -> tensor<?xf32>
+  // CHECK-NEXT: ^ extract_slice runs out-of-bounds along dimension 0
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract_slice_dynamic_rank_reduce(%alloca_4_dyn, %5, %5, %1) : (tensor<?x4xf32>, index, index, index) -> ()
+
+  // Offset is out-of-bounds and slice runs out-of-bounds
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1) <{operandSegmentSizes = array<i32: 1, 1, 0, 0>, static_offsets = array<i64: -9223372036854775808>, static_sizes = array<i64: 1>, static_strides = array<i64: 1>}> : (tensor<1xf32>, index) -> tensor<1xf32>
+  // CHECK-NEXT: ^ offset 0 is out-of-bounds
+  // CHECK-NEXT: Location: loc({{.*}})
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1) <{operandSegmentSizes = array<i32: 1, 1, 0, 0>, static_offsets = array<i64: -9223372036854775808>, static_sizes = array<i64: 1>, static_strides = array<i64: 1>}> : (tensor<1xf32>, index) -> tensor<1xf32>
+  // CHECK-NEXT: ^ extract_slice runs out-of-bounds along dimension 0
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract_slice(%alloca, %1) : (tensor<1xf32>, index) -> ()
+
+  // Offset is out-of-bounds and slice runs out-of-bounds
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1) <{operandSegmentSizes = array<i32: 1, 1, 0, 0>, static_offsets = array<i64: -9223372036854775808>, static_sizes = array<i64: 1>, static_strides = array<i64: 1>}> : (tensor<1xf32>, index) -> tensor<1xf32>
+  // CHECK-NEXT: ^ offset 0 is out-of-bounds
+  // CHECK-NEXT: Location: loc({{.*}})
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1) <{operandSegmentSizes = array<i32: 1, 1, 0, 0>, static_offsets = array<i64: -9223372036854775808>, static_sizes = array<i64: 1>, static_strides = array<i64: 1>}> : (tensor<1xf32>, index) -> tensor<1xf32>
+  // CHECK-NEXT: ^ extract_slice runs out-of-bounds along dimension 0
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract_slice(%alloca, %n1) : (tensor<1xf32>, index) -> ()
+
+  // Slice runs out-of-bounds due to size
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1, %arg2, %arg3) <{operandSegmentSizes = array<i32: 1, 1, 1, 1>, static_offsets = array<i64: -9223372036854775808, 0>, static_sizes = array<i64: -9223372036854775808, 4>, static_strides = array<i64: -9223372036854775808, 1>}> : (tensor<?x4xf32>, index, index, index) -> tensor<?x4xf32>
+  // CHECK-NEXT: ^ extract_slice runs out-of-bounds along dimension 0
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract_slice_dynamic(%alloca_4_dyn, %0, %5, %1) : (tensor<?x4xf32>, index, index, index) -> ()
+
+  // Slice runs out-of-bounds due to stride
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "tensor.extract_slice"(%arg0, %arg1, %arg2, %arg3) <{operandSegmentSizes = array<i32: 1, 1, 1, 1>, static_offsets = array<i64: -9223372036854775808, 0>, static_sizes = array<i64: -9223372036854775808, 4>, static_strides = array<i64: -9223372036854775808, 1>}> : (tensor<?x4xf32>, index, index, index) -> tensor<?x4xf32>
+  // CHECK-NEXT: ^ extract_slice runs out-of-bounds along dimension 0
+  // CHECK-NEXT: Location: loc({{.*}})
+  func.call @extract_slice_dynamic(%alloca_4_dyn, %0, %4, %4) : (tensor<?x4xf32>, index, index, index) -> ()
+
+  // CHECK-NOT: ERROR: Runtime op verification failed
+  func.call @extract_slice(%alloca, %0) : (tensor<1xf32>, index) -> ()
+
+  // CHECK-NOT: ERROR: Runtime op verification failed
+  func.call @extract_slice_dynamic(%alloca_4_dyn, %0, %4, %1) : (tensor<?x4xf32>, index, index, index) -> ()
+
+  // CHECK-NOT: ERROR: Runtime op verification failed
+  func.call @extract_slice_dynamic_rank_reduce(%alloca_4_dyn, %0, %1, %0) : (tensor<?x4xf32>, index, index, index) -> ()
+
+
+  return
+}
diff --git a/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir b/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir
index aaa3aff5350a..a5653f395a2c 100644
--- a/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir
+++ b/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir
@@ -120,7 +120,7 @@ func.func @main() {
             threads(%arg3, %arg4, %arg5) in (%arg9 = %hc128, %arg10 = %hc1, %arg11 = %hc1) 
             dynamic_shared_memory_size %shmemSize 
   {  
-    memref.assume_alignment %matrixD, 16 : memref<128x128xf32>
+    %align_matrixD = memref.assume_alignment %matrixD, 16 : memref<128x128xf32>
 
     %c256 = arith.constant 256 : index
     %c10000000 = arith.constant 10000000 : index
@@ -226,7 +226,7 @@ func.func @main() {
     scf.for %arg12 = %17 to %c128 step %c4 {
       %19 = arith.muli %18, %c4 : index
       %20 = vector.load %accShmemPtr[%arg12, %19] : memref<128x128xf32, 3>, vector<4xf32>
-      vector.store %20, %matrixD[%arg12, %19] : memref<128x128xf32>, vector<4xf32>
+      vector.store %20, %align_matrixD[%arg12, %19] : memref<128x128xf32>, vector<4xf32>
     }
     gpu.terminator
   }
diff --git a/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir b/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir
index b257d2b0f1e3..197351f1921e 100644
--- a/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir
+++ b/external/llvm-project/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir
@@ -120,7 +120,7 @@ func.func @main() {
             threads(%arg3, %arg4, %arg5) in (%arg9 = %hc128, %arg10 = %hc1, %arg11 = %hc1) 
             dynamic_shared_memory_size %shmemSize 
   {  
-    memref.assume_alignment %matrixD, 16 : memref<128x128xf32>
+    %align_matrixD = memref.assume_alignment %matrixD, 16 : memref<128x128xf32>
 
     %c256 = arith.constant 256 : index
     %c10000000 = arith.constant 10000000 : index
@@ -234,7 +234,7 @@ func.func @main() {
     scf.for %arg12 = %17 to %c128 step %c4 {
       %19 = arith.muli %18, %c4 : index
       %20 = vector.load %accShmemPtr[%arg12, %19] : memref<128x128xf32, 3>, vector<4xf32>
-      vector.store %20, %matrixD[%arg12, %19] : memref<128x128xf32>, vector<4xf32>
+      vector.store %20, %align_matrixD[%arg12, %19] : memref<128x128xf32>, vector<4xf32>
     }
     gpu.terminator
   }
diff --git a/external/llvm-project/mlir/test/Integration/lit.local.cfg b/external/llvm-project/mlir/test/Integration/lit.local.cfg
index 1c828fb22808..5f16b1cc3cc4 100644
--- a/external/llvm-project/mlir/test/Integration/lit.local.cfg
+++ b/external/llvm-project/mlir/test/Integration/lit.local.cfg
@@ -1,5 +1,5 @@
 from lit.llvm import llvm_config
-llvm_config.with_environment("AMD_LOG_LEVEL", "4")
+
 if not config.mlir_include_integration_tests:
     config.unsupported = True
 
diff --git a/external/llvm-project/mlir/test/Pass/pipeline-options-parsing.mlir b/external/llvm-project/mlir/test/Pass/pipeline-options-parsing.mlir
index b8cd605a83a2..9385d353faf9 100644
--- a/external/llvm-project/mlir/test/Pass/pipeline-options-parsing.mlir
+++ b/external/llvm-project/mlir/test/Pass/pipeline-options-parsing.mlir
@@ -38,11 +38,71 @@
 
 // CHECK_1: test-options-pass{enum=zero list={1,2,3,4,5} string=nested_pipeline{arg1=10 arg2=" {} " arg3=true} string-list={a,b,c,d}}
 // CHECK_2: test-options-pass{enum=one list={1} string= string-list={a,b}}
-// CHECK_3: builtin.module(builtin.module(func.func(test-options-pass{enum=zero list={3} string= }),func.func(test-options-pass{enum=one list={1,2,3,4} string= })))
-// CHECK_4: builtin.module(builtin.module(func.func(test-options-pass{enum=zero list={3} string= }),func.func(test-options-pass{enum=one list={1,2,3,4} string=foobar })))
-// CHECK_5: builtin.module(builtin.module(func.func(test-options-pass{enum=zero list={3} string= }),func.func(test-options-pass{enum=one list={1,2,3,4} string={foo bar baz} })))
-// CHECK_6: builtin.module(builtin.module(func.func(test-options-pass{enum=zero list={3} string= }),func.func(test-options-pass{enum=one list={1,2,3,4} string=foo"bar"baz })))
-// CHECK_7{LITERAL}: builtin.module(func.func(test-options-super-pass{list={{enum=zero list={1} string=foo },{enum=one list={2} string=bar },{enum=two list={3} string=baz }}}))
-// CHECK_8{LITERAL}: builtin.module(func.func(test-options-super-pass{list={{enum=zero list={1} string=foo },{enum=one string=bar }}}))
-// CHECK_9: builtin.module(func.func(test-options-pass{enum=zero  string= string-list={}}))
-// CHECK_10: builtin.module(func.func(test-options-pass{enum=zero  string= string-list={,}}))
+
+// CHECK_3:      builtin.module(
+// CHECK_3-NEXT:   builtin.module(
+// CHECK_3-NEXT:     func.func(
+// CHECK_3-NEXT:       test-options-pass{enum=zero list={3} string= }
+// CHECK_3-NEXT:     ),
+// CHECK_3-NEXT:     func.func(
+// CHECK_3-NEXT:       test-options-pass{enum=one list={1,2,3,4} string= }
+// CHECK_3-NEXT:     )
+// CHECK_3-NEXT:   )
+// CHECK_3-NEXT: )
+
+// CHECK_4:      builtin.module(
+// CHECK_4-NEXT:   builtin.module(
+// CHECK_4-NEXT:     func.func(
+// CHECK_4-NEXT:       test-options-pass{enum=zero list={3} string= }
+// CHECK_4-NEXT:     ),
+// CHECK_4-NEXT:     func.func(
+// CHECK_4-NEXT:       test-options-pass{enum=one list={1,2,3,4} string=foobar }
+// CHECK_4-NEXT:     )
+// CHECK_4-NEXT:   )
+// CHECK_4-NEXT: )
+
+// CHECK_5:      builtin.module(
+// CHECK_5-NEXT:   builtin.module(
+// CHECK_5-NEXT:     func.func(
+// CHECK_5-NEXT:       test-options-pass{enum=zero list={3} string= }
+// CHECK_5-NEXT:     ),
+// CHECK_5-NEXT:     func.func(
+// CHECK_5-NEXT:       test-options-pass{enum=one list={1,2,3,4} string={foo bar baz} }
+// CHECK_5-NEXT:     )
+// CHECK_5-NEXT:   )
+// CHECK_5-NEXT: )
+
+// CHECK_6:      builtin.module(
+// CHECK_6-NEXT:   builtin.module(
+// CHECK_6-NEXT:     func.func(
+// CHECK_6-NEXT:       test-options-pass{enum=zero list={3} string= }
+// CHECK_6-NEXT:     ),
+// CHECK_6-NEXT:     func.func(
+// CHECK_6-NEXT:       test-options-pass{enum=one list={1,2,3,4} string=foo"bar"baz }
+// CHECK_6-NEXT:     )
+// CHECK_6-NEXT:   )
+// CHECK_6-NEXT: )
+
+// CHECK_7{LITERAL}:      builtin.module(
+// CHECK_7{LITERAL}-NEXT:   func.func(
+// CHECK_7{LITERAL}-NEXT:     test-options-super-pass{list={{enum=zero list={1} string=foo },{enum=one list={2} string=bar },{enum=two list={3} string=baz }}}
+// CHECK_7{LITERAL}-NEXT:   )
+// CHECK_7{LITERAL}-NEXT: )
+
+// CHECK_8{LITERAL}:      builtin.module(
+// CHECK_8{LITERAL}-NEXT:   func.func(
+// CHECK_8{LITERAL}-NEXT:     test-options-super-pass{list={{enum=zero list={1} string=foo },{enum=one string=bar }}}
+// CHECK_8{LITERAL}-NEXT:   )
+// CHECK_8{LITERAL}-NEXT: )
+
+// CHECK_9:      builtin.module(
+// CHECK_9-NEXT:   func.func(
+// CHECK_9-NEXT:     test-options-pass{enum=zero  string= string-list={}}
+// CHECK_9-NEXT:   )
+// CHECK_9-NEXT: )
+
+// CHECK_10:      builtin.module(
+// CHECK_10-NEXT:   func.func(
+// CHECK_10-NEXT:     test-options-pass{enum=zero  string= string-list={,}}
+// CHECK_10-NEXT:   )
+// CHECK_10-NEXT: )
diff --git a/external/llvm-project/mlir/test/Pass/pipeline-parsing.mlir b/external/llvm-project/mlir/test/Pass/pipeline-parsing.mlir
index f41553d2669f..7f368c6df2af 100644
--- a/external/llvm-project/mlir/test/Pass/pipeline-parsing.mlir
+++ b/external/llvm-project/mlir/test/Pass/pipeline-parsing.mlir
@@ -21,7 +21,9 @@
 // CHECK_ERROR_7: can't run 'wrong-op' pass manager on 'builtin.module' op
 
 // RUN: mlir-opt %s -pass-pipeline='any(cse)' -dump-pass-pipeline 2>&1 | FileCheck %s -check-prefix=CHECK_ROUNDTRIP
-// CHECK_ROUNDTRIP: any(cse)
+// CHECK_ROUNDTRIP: any(
+// CHECK_ROUNDTRIP-NEXT: cse
+// CHECK_ROUNDTRIP-NEXT: )
 
 func.func @foo() {
   return
diff --git a/external/llvm-project/mlir/test/Pass/run-reproducer.mlir b/external/llvm-project/mlir/test/Pass/run-reproducer.mlir
index bf3ab2dae2ff..d2daceab163d 100644
--- a/external/llvm-project/mlir/test/Pass/run-reproducer.mlir
+++ b/external/llvm-project/mlir/test/Pass/run-reproducer.mlir
@@ -14,7 +14,12 @@ func.func @bar() {
   external_resources: {
     mlir_reproducer: {
       verify_each: true,
-      // CHECK:  builtin.module(func.func(cse,canonicalize{ max-iterations=1 max-num-rewrites=-1 region-simplify=normal test-convergence=false top-down=false}))
+      // CHECK:       builtin.module(
+      // CHECK-NEXT:    func.func(
+      // CHECK-NEXT:      cse,
+      // CHECK-NEXT:      canonicalize{ max-iterations=1 max-num-rewrites=-1 region-simplify=normal test-convergence=false top-down=false}
+      // CHECK-NEXT:    )
+      // CHECK-NEXT:  )
       pipeline: "builtin.module(func.func(cse,canonicalize{max-iterations=1 max-num-rewrites=-1 region-simplify=normal top-down=false}))",
       disable_threading: true
     }
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/Import/mrecip.ll b/external/llvm-project/mlir/test/Target/LLVMIR/Import/mrecip.ll
new file mode 100644
index 000000000000..01814cce7005
--- /dev/null
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/Import/mrecip.ll
@@ -0,0 +1,9 @@
+; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s
+
+; CHECK-LABEL: llvm.func @reciprocal_estimates()
+; CHECK-SAME: reciprocal_estimates = "all"
+define void @reciprocal_estimates() #0 {
+  ret void
+}
+
+attributes #0 = { "reciprocal-estimates" = "all" }
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/amx.mlir b/external/llvm-project/mlir/test/Target/LLVMIR/amx.mlir
index 094475040436..abdf2fe3bd53 100644
--- a/external/llvm-project/mlir/test/Target/LLVMIR/amx.mlir
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/amx.mlir
@@ -88,3 +88,23 @@ func.func @amx_tile_muli(%matA: memref<?x?xi8>, %matB: memref<?x?xi8>,
   amx.tile_store %out[%c16, %c16], %res3 : memref<?x?xi8>, !amx.tile<16x16xi32>
   return
 }
+
+// CHECK-LABEL: define void @amx_tile_type_through_cf
+func.func @amx_tile_type_through_cf(%src: memref<?x?xi8>, %out: memref<?x?xi8>,
+    %idx: index, %cond: i1) {
+  cf.cond_br %cond, ^bb1, ^bb2
+^bb1:  // pred: ^bb0
+  // CHECK: call x86_amx @llvm.x86.tileloadd64.internal
+  %0 = amx.tile_load %src[%idx, %idx] : memref<?x?xi8> into !amx.tile<16x64xi8>
+  cf.br ^bb3(%0 : !amx.tile<16x64xi8>)
+^bb2:  // pred: ^bb0
+  // CHECK: call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+  %1 = amx.tile_zero : !amx.tile<16x64xi8>
+  cf.br ^bb3(%1 : !amx.tile<16x64xi8>)
+^bb3(%2: !amx.tile<16x64xi8>):  // 2 preds: ^bb1, ^bb2
+  cf.br ^bb4
+^bb4:  // pred: ^bb3
+  // CHECK: call void @llvm.x86.tilestored64.internal
+  amx.tile_store %out[%idx, %idx], %2 : memref<?x?xi8>, !amx.tile<16x64xi8>
+  return
+}
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/mrecip.mlir b/external/llvm-project/mlir/test/Target/LLVMIR/mrecip.mlir
new file mode 100644
index 000000000000..e0bc66c272f6
--- /dev/null
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/mrecip.mlir
@@ -0,0 +1,8 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK: define void @reciprocal_estimates() #[[ATTRS:.*]] {
+// CHECK: attributes #[[ATTRS]] = { "reciprocal-estimates"="all" }
+
+llvm.func @reciprocal_estimates() attributes {reciprocal_estimates = "all"} {
+  llvm.return
+}
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/nvvm/prefetch.mlir b/external/llvm-project/mlir/test/Target/LLVMIR/nvvm/prefetch.mlir
new file mode 100644
index 000000000000..f38b7529a723
--- /dev/null
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/nvvm/prefetch.mlir
@@ -0,0 +1,47 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+llvm.func @prefetch_L1(%gen_ptr: !llvm.ptr, %local_ptr: !llvm.ptr<5>, %global_ptr: !llvm.ptr<1>) {
+  // CHECK-LABEL: define void @prefetch_L1(ptr %0, ptr addrspace(5) %1, ptr addrspace(1) %2) {
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.L1(ptr %0)
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.local.L1(ptr addrspace(5) %1)
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.global.L1(ptr addrspace(1) %2)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
+  nvvm.prefetch level = L1, %gen_ptr : !llvm.ptr<0>
+  nvvm.prefetch level = L1, %local_ptr : !llvm.ptr<5>
+  nvvm.prefetch level = L1, %global_ptr : !llvm.ptr<1>
+  llvm.return
+}
+
+llvm.func @prefetch_L2(%gen_ptr: !llvm.ptr, %local_ptr: !llvm.ptr<5>, %global_ptr: !llvm.ptr<1>) {
+  // CHECK-LABEL: define void @prefetch_L2(ptr %0, ptr addrspace(5) %1, ptr addrspace(1) %2) {
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.L2(ptr %0)
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.local.L2(ptr addrspace(5) %1)
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.global.L2(ptr addrspace(1) %2)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
+  nvvm.prefetch level = L2, %gen_ptr : !llvm.ptr<0>
+  nvvm.prefetch level = L2, %local_ptr : !llvm.ptr<5>
+  nvvm.prefetch level = L2, %global_ptr : !llvm.ptr<1>
+  llvm.return
+}
+
+llvm.func @prefetch_L2_eviction_priority(%global_ptr: !llvm.ptr<1>) {
+  // CHECK-LABEL: define void @prefetch_L2_eviction_priority(ptr addrspace(1) %0) {
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.global.L2.evict.last(ptr addrspace(1) %0)
+  // CHECK-NEXT: call void @llvm.nvvm.prefetch.global.L2.evict.normal(ptr addrspace(1) %0)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
+  nvvm.prefetch level = L2, %global_ptr, evict_priority = evict_last : !llvm.ptr<1>
+  nvvm.prefetch level = L2, %global_ptr, evict_priority = evict_normal : !llvm.ptr<1>
+  llvm.return
+}
+
+llvm.func @prefetch_L1_uniform(%gen_ptr: !llvm.ptr) {
+  // CHECK-LABEL: define void @prefetch_L1_uniform(ptr %0) {
+  // CHECK-NEXT: call void @llvm.nvvm.prefetchu.L1(ptr %0)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
+  nvvm.prefetch level = L1 uniform, %gen_ptr : !llvm.ptr
+  llvm.return
+}
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir b/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
index 3d63434f310b..8c4f0aafd36a 100644
--- a/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
@@ -248,3 +248,67 @@ llvm.func @nvvm_cvt_bf16x2_to_f8x2_invalid_rounding(%src : vector<2xbf16>) {
   %res = nvvm.convert.bf16x2.to.f8x2 <ue8m0> %src {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xbf16> -> i16
   llvm.return
 }
+
+// -----
+
+llvm.func @nvvm_prefetch_L1_with_evict_priority(%global_ptr: !llvm.ptr<1>) {
+  // expected-error @below {{cache eviction priority supported only for cache level L2}}
+  nvvm.prefetch level = L1, %global_ptr, evict_priority = evict_last : !llvm.ptr<1>
+  llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_prefetch_L2_with_evict_last_invalid_addr_space(%local_ptr: !llvm.ptr<5>) {
+  // expected-error @below {{cache eviction priority requires a global pointer}}
+  nvvm.prefetch level = L2, %local_ptr, evict_priority = evict_last : !llvm.ptr<5>
+  llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_prefetch_L2_with_evict_normal_invalid_addr_space(%local_ptr: !llvm.ptr<5>) {
+  // expected-error @below {{cache eviction priority requires a global pointer}}
+  nvvm.prefetch level = L2, %local_ptr, evict_priority = evict_normal : !llvm.ptr<5>
+  llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_prefetch_L2_with_invalid_evict_first(%global_ptr: !llvm.ptr<1>) {
+  // expected-error @below {{unsupported cache eviction priority, only evict_last and evict_normal are supported}}
+  nvvm.prefetch level = L2, %global_ptr, evict_priority = evict_first : !llvm.ptr<1>
+  llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_prefetch_L2_with_invalid_evict_unchanged(%global_ptr: !llvm.ptr<1>) {
+  // expected-error @below {{unsupported cache eviction priority, only evict_last and evict_normal are supported}}
+  nvvm.prefetch level = L2, %global_ptr, evict_priority = evict_unchanged : !llvm.ptr<1>
+  llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_prefetch_L2_with_invalid_no_allocate(%global_ptr: !llvm.ptr<1>) {
+  // expected-error @below {{unsupported cache eviction priority, only evict_last and evict_normal are supported}}
+  nvvm.prefetch level = L2, %global_ptr, evict_priority = no_allocate : !llvm.ptr<1>
+  llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_prefetch_uniform_with_L2(%gen_ptr: !llvm.ptr) {
+  // expected-error @below {{unsupported cache level, the only supported uniform cache level is L1}}
+  nvvm.prefetch level = L2 uniform, %gen_ptr : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_prefetch_uniform_with_invalid_addr_space(%global_ptr: !llvm.ptr<1>) {
+  // expected-error @below {{prefetch to uniform cache requires a generic pointer}}
+  nvvm.prefetch level = L1 uniform, %global_ptr : !llvm.ptr<1>
+  llvm.return
+}
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir.mlir b/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir.mlir
index e892fc43f4a3..660d0a22dce9 100644
--- a/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir.mlir
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/nvvmir.mlir
@@ -866,3 +866,41 @@ llvm.func @nvvm_dot_accumulate_4way(%a: vector<4xi8>, %b: vector<4xi8>, %c: i32)
   %3 = nvvm.dot.accumulate.4way %a <signed>, %b <signed>, %c: vector<4xi8>, vector<4xi8>
   llvm.return
 }
+
+// -----
+// CHECK-LABEL: @nvvm_dot_accumulate_2way
+llvm.func @nvvm_dot_accumulate_2way(%a: vector<2xi16>, %b: vector<4xi8>, %c: i32) {
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.u.u(i32 %[[a_cast]], i32 %[[b_cast]], i1 false, i32 %{{.*}})
+  %0 = nvvm.dot.accumulate.2way %a <unsigned>, %b <unsigned>, %c {b_hi = false} : vector<2xi16>, vector<4xi8>
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.u.u(i32 %[[a_cast]], i32 %[[b_cast]], i1 true, i32 %{{.*}})
+  %1 = nvvm.dot.accumulate.2way %a <unsigned>, %b <unsigned>, %c {b_hi = true}: vector<2xi16>, vector<4xi8>
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.s.u(i32 %[[a_cast]], i32 %[[b_cast]], i1 false, i32 %{{.*}})
+  %2 = nvvm.dot.accumulate.2way %a <signed>, %b <unsigned>, %c {b_hi = false}: vector<2xi16>, vector<4xi8>
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.s.u(i32 %[[a_cast]], i32 %[[b_cast]], i1 true, i32 %{{.*}})
+  %3 = nvvm.dot.accumulate.2way %a <signed>, %b <unsigned>, %c {b_hi = true}: vector<2xi16>, vector<4xi8>
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.u.s(i32 %[[a_cast]], i32 %[[b_cast]], i1 false, i32 %{{.*}})
+  %4 = nvvm.dot.accumulate.2way %a <unsigned>, %b <signed>, %c {b_hi = false}: vector<2xi16>, vector<4xi8>
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.u.s(i32 %[[a_cast]], i32 %[[b_cast]], i1 true, i32 %{{.*}})
+  %5 = nvvm.dot.accumulate.2way %a <unsigned>, %b <signed>, %c {b_hi = true}: vector<2xi16>, vector<4xi8>
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.s.s(i32 %[[a_cast]], i32 %[[b_cast]], i1 false, i32 %{{.*}})
+  %6 = nvvm.dot.accumulate.2way %a <signed>, %b <signed>, %c {b_hi = false}: vector<2xi16>, vector<4xi8>
+  // CHECK: %[[a_cast:.*]] = bitcast <2 x i16> %{{.*}} to i32
+  // CHECK: %[[b_cast:.*]] = bitcast <4 x i8> %{{.*}} to i32
+  // CHECK: call i32 @llvm.nvvm.idp2a.s.s(i32 %[[a_cast]], i32 %[[b_cast]], i1 true, i32 %{{.*}})
+  %7 = nvvm.dot.accumulate.2way %a <signed>, %b <signed>, %c {b_hi = true}: vector<2xi16>, vector<4xi8>
+  llvm.return
+}
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/openmp-barrier-cancel.mlir b/external/llvm-project/mlir/test/Target/LLVMIR/openmp-barrier-cancel.mlir
new file mode 100644
index 000000000000..c4b245667a1f
--- /dev/null
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/openmp-barrier-cancel.mlir
@@ -0,0 +1,50 @@
+// RUN: mlir-translate --mlir-to-llvmir %s | FileCheck %s
+
+// Regression test for a compiler crash. Ensure that the insertion point is set
+// correctly after the barrier's cancel check
+
+llvm.func @test() {
+  omp.parallel {
+    omp.cancel cancellation_construct_type(parallel)
+    omp.barrier
+    omp.terminator
+  }
+  llvm.return
+}
+
+// CHECK-LABEL: define internal void @test..omp_par
+// CHECK:       omp.par.entry:
+// CHECK:         %[[VAL_4:.*]] = alloca i32, align 4
+// CHECK:         %[[VAL_5:.*]] = load i32, ptr %[[VAL_6:.*]], align 4
+// CHECK:         store i32 %[[VAL_5]], ptr %[[VAL_4]], align 4
+// CHECK:         %[[VAL_7:.*]] = load i32, ptr %[[VAL_4]], align 4
+// CHECK:         br label %[[VAL_8:.*]]
+// CHECK:       omp.region.after_alloca:                          ; preds = %[[VAL_9:.*]]
+// CHECK:         br label %[[VAL_10:.*]]
+// CHECK:       omp.par.region:                                   ; preds = %[[VAL_8]]
+// CHECK:         br label %[[VAL_11:.*]]
+// CHECK:       omp.par.region1:                                  ; preds = %[[VAL_10]]
+// CHECK:         %[[VAL_12:.*]] = call i32 @__kmpc_global_thread_num(ptr @1)
+// CHECK:         %[[VAL_13:.*]] = call i32 @__kmpc_cancel(ptr @1, i32 %[[VAL_12]], i32 1)
+// CHECK:         %[[VAL_14:.*]] = icmp eq i32 %[[VAL_13]], 0
+// CHECK:         br i1 %[[VAL_14]], label %[[VAL_15:.*]], label %[[VAL_16:.*]]
+// CHECK:       omp.par.region1.cncl:                             ; preds = %[[VAL_11]]
+// CHECK:         %[[VAL_17:.*]] = call i32 @__kmpc_global_thread_num(ptr @1)
+// CHECK:         %[[VAL_18:.*]] = call i32 @__kmpc_cancel_barrier(ptr @2, i32 %[[VAL_17]])
+// CHECK:         br label %[[VAL_19:.*]]
+// CHECK:       omp.par.region1.split:                            ; preds = %[[VAL_11]]
+// CHECK:         %[[VAL_20:.*]] = call i32 @__kmpc_global_thread_num(ptr @1)
+// CHECK:         %[[VAL_21:.*]] = call i32 @__kmpc_cancel_barrier(ptr @3, i32 %[[VAL_20]])
+// CHECK:         %[[VAL_22:.*]] = icmp eq i32 %[[VAL_21]], 0
+// CHECK:         br i1 %[[VAL_22]], label %[[VAL_23:.*]], label %[[VAL_24:.*]]
+// CHECK:       omp.par.region1.split.cncl:                       ; preds = %[[VAL_15]]
+// CHECK:         br label %[[VAL_19]]
+// CHECK:       omp.par.region1.split.cont:                       ; preds = %[[VAL_15]]
+// CHECK:         br label %[[VAL_25:.*]]
+// CHECK:       omp.region.cont:                                  ; preds = %[[VAL_23]]
+// CHECK:         br label %[[VAL_26:.*]]
+// CHECK:       omp.par.pre_finalize:                             ; preds = %[[VAL_25]]
+// CHECK:         br label %[[VAL_19]]
+// CHECK:       omp.par.exit.exitStub:                            ; preds = %[[VAL_26]], %[[VAL_24]], %[[VAL_16]]
+// CHECK:         ret void
+
diff --git a/external/llvm-project/mlir/test/Target/LLVMIR/openmp-todo.mlir b/external/llvm-project/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 19063cf16c92..d6baa4db0047 100644
--- a/external/llvm-project/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/external/llvm-project/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -519,7 +519,6 @@ llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
 }
 
 // -----
-
 llvm.func @wsloop_order(%lb : i32, %ub : i32, %step : i32) {
   // expected-error@below {{not yet implemented: Unhandled clause order in omp.wsloop operation}}
   // expected-error@below {{LLVM Translation failed for operation: omp.wsloop}}
diff --git a/external/llvm-project/mlir/test/Target/SPIRV/cast-ops.mlir b/external/llvm-project/mlir/test/Target/SPIRV/cast-ops.mlir
index ede0bf30511e..04a468b39b64 100644
--- a/external/llvm-project/mlir/test/Target/SPIRV/cast-ops.mlir
+++ b/external/llvm-project/mlir/test/Target/SPIRV/cast-ops.mlir
@@ -25,6 +25,11 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %0 = spirv.ConvertFToS %arg0 : f64 to i32
     spirv.ReturnValue %0 : i32
   }
+  spirv.func @convert_bf16_to_s32(%arg0 : bf16) -> i32 "None" {
+    // CHECK: {{%.*}} = spirv.ConvertFToS {{%.*}} : bf16 to i32
+    %0 = spirv.ConvertFToS %arg0 : bf16 to i32
+    spirv.ReturnValue %0 : i32
+  }
   spirv.func @convert_f_to_u(%arg0 : f32) -> i32 "None" {
     // CHECK: {{%.*}} = spirv.ConvertFToU {{%.*}} : f32 to i32
     %0 = spirv.ConvertFToU %arg0 : f32 to i32
@@ -35,6 +40,11 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %0 = spirv.ConvertFToU %arg0 : f64 to i32
     spirv.ReturnValue %0 : i32
   }
+  spirv.func @convert_bf16_to_u32(%arg0 : bf16) -> i32 "None" {
+    // CHECK: {{%.*}} = spirv.ConvertFToU {{%.*}} : bf16 to i32
+    %0 = spirv.ConvertFToU %arg0 : bf16 to i32
+    spirv.ReturnValue %0 : i32
+  }
   spirv.func @convert_s_to_f(%arg0 : i32) -> f32 "None" {
     // CHECK: {{%.*}} = spirv.ConvertSToF {{%.*}} : i32 to f32
     %0 = spirv.ConvertSToF %arg0 : i32 to f32
@@ -45,6 +55,11 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %0 = spirv.ConvertSToF %arg0 : i64 to f32
     spirv.ReturnValue %0 : f32
   }
+  spirv.func @convert_s64_to_bf16(%arg0 : i64) -> bf16 "None" {
+    // CHECK: {{%.*}} = spirv.ConvertSToF {{%.*}} : i64 to bf16
+    %0 = spirv.ConvertSToF %arg0 : i64 to bf16
+    spirv.ReturnValue %0 : bf16
+  }
   spirv.func @convert_u_to_f(%arg0 : i32) -> f32 "None" {
     // CHECK: {{%.*}} = spirv.ConvertUToF {{%.*}} : i32 to f32
     %0 = spirv.ConvertUToF %arg0 : i32 to f32
@@ -55,11 +70,26 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %0 = spirv.ConvertUToF %arg0 : i64 to f32
     spirv.ReturnValue %0 : f32
   }
-  spirv.func @f_convert(%arg0 : f32) -> f64 "None" {
+  spirv.func @convert_u64_to_bf16(%arg0 : i64) -> bf16 "None" {
+    // CHECK: {{%.*}} = spirv.ConvertUToF {{%.*}} : i64 to bf16
+    %0 = spirv.ConvertUToF %arg0 : i64 to bf16
+    spirv.ReturnValue %0 : bf16
+  }
+  spirv.func @convert_f32_to_f64(%arg0 : f32) -> f64 "None" {
     // CHECK: {{%.*}} = spirv.FConvert {{%.*}} : f32 to f64
     %0 = spirv.FConvert %arg0 : f32 to f64
     spirv.ReturnValue %0 : f64
   }
+  spirv.func @convert_f32_to_bf16(%arg0 : f32) -> bf16 "None" {
+    // CHECK: {{%.*}} = spirv.FConvert {{%.*}} : f32 to bf16
+    %0 = spirv.FConvert %arg0 : f32 to bf16
+    spirv.ReturnValue %0 : bf16
+  }
+  spirv.func @convert_bf16_to_f32(%arg0 : bf16) -> f32 "None" {
+    // CHECK: {{%.*}} = spirv.FConvert {{%.*}} : bf16 to f32
+    %0 = spirv.FConvert %arg0 : bf16 to f32
+    spirv.ReturnValue %0 : f32
+  }
   spirv.func @s_convert(%arg0 : i32) -> i64 "None" {
     // CHECK: {{%.*}} = spirv.SConvert {{%.*}} : i32 to i64
     %0 = spirv.SConvert %arg0 : i32 to i64
diff --git a/external/llvm-project/mlir/test/Target/SPIRV/constant.mlir b/external/llvm-project/mlir/test/Target/SPIRV/constant.mlir
index f3950214a7f0..8d4e53418b70 100644
--- a/external/llvm-project/mlir/test/Target/SPIRV/constant.mlir
+++ b/external/llvm-project/mlir/test/Target/SPIRV/constant.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-translate -no-implicit-module -test-spirv-roundtrip %s | FileCheck %s
+// RUN: mlir-translate --no-implicit-module --test-spirv-roundtrip %s | FileCheck %s
 
 spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
   // CHECK-LABEL: @bool_const
@@ -277,4 +277,32 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %signed_minus_one = spirv.Constant -1 : si16
     spirv.ReturnValue %signed_minus_one : si16
   }
+
+  // CHECK-LABEL: @coop_matrix_const_zero_f32
+  spirv.func @coop_matrix_const_zero_f32() -> (!spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<0.000000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+    %coop = spirv.Constant dense<0.000000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+    spirv.ReturnValue %coop : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+  }
+
+  // CHECK-LABEL: @coop_matrix_const_non_zero_f32
+  spirv.func @coop_matrix_const_non_zero_f32() -> (!spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<4.200000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+    %coop = spirv.Constant dense<4.200000e+00> : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+    spirv.ReturnValue %coop : !spirv.coopmatrix<16x16xf32, Subgroup, MatrixAcc>
+  }
+
+  // CHECK-LABEL: @coop_matrix_const_zero_i8
+  spirv.func @coop_matrix_const_zero_i8() -> (!spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<0> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+    %coop = spirv.Constant dense<0> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+    spirv.ReturnValue %coop : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+  }
+
+  // CHECK-LABEL: @coop_matrix_const_non_zero_i8
+  spirv.func @coop_matrix_const_non_zero_i8() -> (!spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<4> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+    %coop = spirv.Constant dense<4> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+    spirv.ReturnValue %coop : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
+  }
 }
diff --git a/external/llvm-project/mlir/test/Target/SPIRV/gl-ops.mlir b/external/llvm-project/mlir/test/Target/SPIRV/gl-ops.mlir
index 7f9771220b75..eacf36bfba9c 100644
--- a/external/llvm-project/mlir/test/Target/SPIRV/gl-ops.mlir
+++ b/external/llvm-project/mlir/test/Target/SPIRV/gl-ops.mlir
@@ -34,6 +34,18 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %15 = spirv.GL.FMix %arg0 : f32, %arg1 : f32, %arg0 : f32 -> f32
     // CHECK: {{%.*}} = spirv.GL.Fract {{%.*}} : f32
     %16 = spirv.GL.Fract %arg0 : f32
+    // CHECK: {{%.*}} = spirv.GL.Asinh {{%.*}} : f32
+    %17 = spirv.GL.Asinh %arg0 : f32
+    // CHECK: {{%.*}} = spirv.GL.Acosh {{%.*}} : f32
+    %18 = spirv.GL.Acosh %arg0 : f32
+    // CHECK: {{%.*}} = spirv.GL.Atanh {{%.*}} : f32
+    %19 = spirv.GL.Atanh %arg0 : f32
+    // CHECK: {{%.*}} = spirv.GL.Log2 {{%.*}} : f32
+    %20 = spirv.GL.Log2 %arg0 : f32
+    // CHECK: {{%.*}} = spirv.GL.Tanh {{%.*}} : f32
+    %21 = spirv.GL.Tanh %arg0 : f32
+    // CHECK: {{%.*}} = spirv.GL.Exp2 {{%.*}} : f32
+    %22 = spirv.GL.Exp2 %arg0 : f32
     spirv.Return
   }
 
@@ -78,13 +90,24 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     spirv.Return
   }
 
+  spirv.func @findilsb(%arg0 : i32) "None" {
+    // CHECK: spirv.GL.FindILsb {{%.*}} : i32
+    %2 = spirv.GL.FindILsb %arg0 : i32
+    spirv.Return
+  }
+  spirv.func @findsmsb(%arg0 : i32) "None" {
+    // CHECK: spirv.GL.FindSMsb {{%.*}} : i32
+    %2 = spirv.GL.FindSMsb %arg0 : i32
+    spirv.Return
+  }
+
   spirv.func @findumsb(%arg0 : i32) "None" {
     // CHECK: spirv.GL.FindUMsb {{%.*}} : i32
     %2 = spirv.GL.FindUMsb %arg0 : i32
     spirv.Return
   }
 
-spirv.func @vector(%arg0 : f32, %arg1 : vector<3xf32>, %arg2 : vector<3xf32>) "None" {
+  spirv.func @vector(%arg0 : f32, %arg1 : vector<3xf32>, %arg2 : vector<3xf32>, %arg3: vector<3xi32>) "None" {
     // CHECK: {{%.*}} = spirv.GL.Cross {{%.*}}, {{%.*}} : vector<3xf32>
     %0 = spirv.GL.Cross %arg1, %arg2 : vector<3xf32>
     // CHECK: {{%.*}} = spirv.GL.Normalize {{%.*}} : f32
@@ -99,8 +122,20 @@ spirv.func @vector(%arg0 : f32, %arg1 : vector<3xf32>, %arg2 : vector<3xf32>) "N
     %5 = spirv.GL.Distance %arg0, %arg0 : f32, f32 -> f32
     // CHECK: {{%.*}} = spirv.GL.Distance {{%.*}}, {{%.*}} : vector<3xf32>, vector<3xf32> -> f32
     %6 = spirv.GL.Distance %arg1, %arg2 : vector<3xf32>, vector<3xf32> -> f32
+    // CHECK: {{%.*}} = spirv.GL.FindILsb {{%.*}} : vector<3xi32>
+    %7 = spirv.GL.FindILsb %arg3 : vector<3xi32>
+    // CHECK: {{%.*}} = spirv.GL.FindSMsb {{%.*}} : vector<3xi32>
+    %8 = spirv.GL.FindSMsb %arg3 : vector<3xi32>
+    // CHECK: {{%.*}} = spirv.GL.FindUMsb {{%.*}} : vector<3xi32>
+    %9 = spirv.GL.FindUMsb %arg3 : vector<3xi32>
     spirv.Return
   }
 
-
+  spirv.func @pack_half_2x16(%arg0 : i32) "None" {
+    // CHECK: {{%.*}} = spirv.GL.UnpackHalf2x16 {{%.*}} : i32 -> vector<2xf32>
+    %0 = spirv.GL.UnpackHalf2x16 %arg0 : i32 -> vector<2xf32>
+    // CHECK: {{%.*}} = spirv.GL.PackHalf2x16 {{%.*}} : vector<2xf32> -> i32
+    %1 = spirv.GL.PackHalf2x16 %0 : vector<2xf32> -> i32
+    spirv.Return
+  }
 }
diff --git a/external/llvm-project/mlir/test/Target/SPIRV/image-ops.mlir b/external/llvm-project/mlir/test/Target/SPIRV/image-ops.mlir
index 6dd23844d46a..b8d19f0f9a7d 100644
--- a/external/llvm-project/mlir/test/Target/SPIRV/image-ops.mlir
+++ b/external/llvm-project/mlir/test/Target/SPIRV/image-ops.mlir
@@ -13,6 +13,11 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, ImageQuery, Link
     %0 = spirv.ImageQuerySize %arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown> -> vector<2xi32>
     spirv.Return
   }
+  spirv.func @image_read(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, %arg1 : vector<2xsi32>) "None" {
+    // CHECK: {{.*}} = spirv.ImageRead {{%.*}}, {{%.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, vector<2xsi32> -> vector<4xf32>
+    %0 = spirv.ImageRead %arg0, %arg1 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, vector<2xsi32> -> vector<4xf32>
+    spirv.Return
+  }
   spirv.func @image_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>) "None" {
     // CHECK: spirv.ImageWrite {{%.*}}, {{%.*}}, {{%.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, vector<2xsi32>, vector<4xf32>
     spirv.ImageWrite %arg0, %arg1, %arg2 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Rgba8>, vector<2xsi32>, vector<4xf32>
@@ -38,9 +43,11 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, ImageQuery, Link
 // -----
 
 spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, StorageImageWriteWithoutFormat, Linkage], []> {
-  spirv.func @image_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, %arg1 : vector<2xsi32>, %arg2 : vector<4xf32>) "None" {
+  spirv.func @image_read_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, %arg1 : vector<2xsi32>) "None" {
+    // CHECK: spirv.ImageRead {{%.*}}, {{%.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32> -> vector<4xf32>
+    %0 = spirv.ImageRead %arg0, %arg1 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32> -> vector<4xf32>
     // CHECK: spirv.ImageWrite {{%.*}}, {{%.*}}, {{%.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32>, vector<4xf32>
-    spirv.ImageWrite %arg0, %arg1, %arg2 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32>, vector<4xf32>
+    spirv.ImageWrite %arg0, %arg1, %0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32>, vector<4xf32>
     spirv.Return
   }
 }
diff --git a/external/llvm-project/mlir/test/Target/SPIRV/logical-ops.mlir b/external/llvm-project/mlir/test/Target/SPIRV/logical-ops.mlir
index 16846ac84e38..b2008719b021 100644
--- a/external/llvm-project/mlir/test/Target/SPIRV/logical-ops.mlir
+++ b/external/llvm-project/mlir/test/Target/SPIRV/logical-ops.mlir
@@ -108,3 +108,26 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     spirv.Return
   }
 }
+
+// -----
+
+// Test select works with bf16 scalar and vectors.
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
+  spirv.SpecConstant @condition_scalar = true
+  spirv.func @select_bf16() -> () "None" {
+    %0 = spirv.Constant 4.0 : bf16
+    %1 = spirv.Constant 5.0 : bf16
+    %2 = spirv.mlir.referenceof @condition_scalar : i1
+    // CHECK: spirv.Select {{.*}}, {{.*}}, {{.*}} : i1, bf16
+    %3 = spirv.Select %2, %0, %1 : i1, bf16
+    %4 = spirv.Constant dense<[2.0, 3.0, 4.0, 5.0]> : vector<4xbf16>
+    %5 = spirv.Constant dense<[6.0, 7.0, 8.0, 9.0]> : vector<4xbf16>
+    // CHECK: spirv.Select {{.*}}, {{.*}}, {{.*}} : i1, vector<4xbf16>
+    %6 = spirv.Select %2, %4, %5 : i1, vector<4xbf16>
+    %7 = spirv.Constant dense<[true, true, true, true]> : vector<4xi1>
+    // CHECK: spirv.Select {{.*}}, {{.*}}, {{.*}} : vector<4xi1>, vector<4xbf16>
+    %8 = spirv.Select %7, %4, %5 : vector<4xi1>, vector<4xbf16>
+    spirv.Return
+  }
+}
diff --git a/external/llvm-project/mlir/test/Transforms/composite-pass.mlir b/external/llvm-project/mlir/test/Transforms/composite-pass.mlir
index 75587edd5b96..65764a96a401 100644
--- a/external/llvm-project/mlir/test/Transforms/composite-pass.mlir
+++ b/external/llvm-project/mlir/test/Transforms/composite-pass.mlir
@@ -2,8 +2,9 @@
 // RUN: mlir-opt %s --log-actions-to=- --composite-fixed-point-pass='name=TestCompositePass pipeline=any(canonicalize,cse)' -split-input-file | FileCheck %s
 
 // Ensure the composite pass correctly prints its options.
-// PIPELINE:      builtin.module(composite-fixed-point-pass{max-iterations=10 name=TestCompositePass
-// PIPELINE-SAME: pipeline=canonicalize{ max-iterations=10 max-num-rewrites=-1 region-simplify=normal test-convergence=false top-down=true},cse})
+// PIPELINE:      builtin.module(
+// PIPELINE-NEXT:    composite-fixed-point-pass{max-iterations=10 name=TestCompositePass
+// PIPELINE-SAME: pipeline=canonicalize{ max-iterations=10 max-num-rewrites=-1 region-simplify=normal test-convergence=false top-down=true},cse}
 
 // CHECK-LABEL: running `TestCompositePass`
 //       CHECK: running `Canonicalizer`
diff --git a/external/llvm-project/mlir/test/Transforms/inlining-dump-default-pipeline.mlir b/external/llvm-project/mlir/test/Transforms/inlining-dump-default-pipeline.mlir
index 4f8638054206..ef5fac49c6ce 100644
--- a/external/llvm-project/mlir/test/Transforms/inlining-dump-default-pipeline.mlir
+++ b/external/llvm-project/mlir/test/Transforms/inlining-dump-default-pipeline.mlir
@@ -1,2 +1,4 @@
 // RUN: mlir-opt %s -pass-pipeline="builtin.module(inline)" -dump-pass-pipeline 2>&1 | FileCheck %s
-// CHECK: builtin.module(inline{default-pipeline=canonicalize inlining-threshold=4294967295 max-iterations=4 })
+// CHECK:      builtin.module(
+// CHECK-NEXT:   inline{default-pipeline=canonicalize inlining-threshold=4294967295 max-iterations=4 }
+// CHECK-NEXT: )
diff --git a/external/llvm-project/mlir/test/lib/Conversion/CMakeLists.txt b/external/llvm-project/mlir/test/lib/Conversion/CMakeLists.txt
index c09496be729b..167cce225595 100644
--- a/external/llvm-project/mlir/test/lib/Conversion/CMakeLists.txt
+++ b/external/llvm-project/mlir/test/lib/Conversion/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(ConvertToSPIRV)
 add_subdirectory(FuncToLLVM)
 add_subdirectory(MathToVCIX)
+add_subdirectory(MemRefToLLVM)
 add_subdirectory(VectorToSPIRV)
diff --git a/external/llvm-project/mlir/test/lib/Conversion/MemRefToLLVM/CMakeLists.txt b/external/llvm-project/mlir/test/lib/Conversion/MemRefToLLVM/CMakeLists.txt
new file mode 100644
index 000000000000..580c9ca4a604
--- /dev/null
+++ b/external/llvm-project/mlir/test/lib/Conversion/MemRefToLLVM/CMakeLists.txt
@@ -0,0 +1,22 @@
+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRTestMemRefToLLVMWithTransforms
+  TestMemRefToLLVMWithTransforms.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRTestDialect
+  )
+mlir_target_link_libraries(MLIRTestMemRefToLLVMWithTransforms PUBLIC
+  MLIRFuncToLLVM
+  MLIRLLVMCommonConversion
+  MLIRLLVMDialect
+  MLIRMemRefTransforms
+  MLIRPass
+  )
+
+target_include_directories(MLIRTestMemRefToLLVMWithTransforms
+  PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../Dialect/Test
+  ${CMAKE_CURRENT_BINARY_DIR}/../../Dialect/Test
+  )
diff --git a/external/llvm-project/mlir/test/lib/Conversion/MemRefToLLVM/TestMemRefToLLVMWithTransforms.cpp b/external/llvm-project/mlir/test/lib/Conversion/MemRefToLLVM/TestMemRefToLLVMWithTransforms.cpp
new file mode 100644
index 000000000000..af3b6608aea1
--- /dev/null
+++ b/external/llvm-project/mlir/test/lib/Conversion/MemRefToLLVM/TestMemRefToLLVMWithTransforms.cpp
@@ -0,0 +1,62 @@
+//===- TestMemRefToLLVMWithTransforms.cpp ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
+#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
+#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
+#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
+#include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+
+namespace {
+
+struct TestMemRefToLLVMWithTransforms
+    : public PassWrapper<TestMemRefToLLVMWithTransforms,
+                         OperationPass<ModuleOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestMemRefToLLVMWithTransforms)
+
+  void getDependentDialects(DialectRegistry &registry) const final {
+    registry.insert<LLVM::LLVMDialect>();
+  }
+
+  StringRef getArgument() const final {
+    return "test-memref-to-llvm-with-transforms";
+  }
+
+  StringRef getDescription() const final {
+    return "Tests conversion of MemRef dialects + `func.func` to LLVM dialect "
+           "with MemRef transforms.";
+  }
+
+  void runOnOperation() override {
+    MLIRContext *ctx = &getContext();
+    LowerToLLVMOptions options(ctx);
+    LLVMTypeConverter typeConverter(ctx, options);
+    RewritePatternSet patterns(ctx);
+    memref::populateExpandStridedMetadataPatterns(patterns);
+    populateFuncToLLVMConversionPatterns(typeConverter, patterns);
+    LLVMConversionTarget target(getContext());
+    if (failed(applyPartialConversion(getOperation(), target,
+                                      std::move(patterns))))
+      signalPassFailure();
+  }
+};
+
+} // namespace
+
+namespace mlir::test {
+void registerTestMemRefToLLVMWithTransforms() {
+  PassRegistration<TestMemRefToLLVMWithTransforms>();
+}
+} // namespace mlir::test
diff --git a/external/llvm-project/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt b/external/llvm-project/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
deleted file mode 100644
index 460842d23853..000000000000
--- a/external/llvm-project/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# Exclude tests from libMLIR.so
-add_mlir_library(MLIRArmNeonTestPasses
-  TestLowerToArmNeon.cpp
-
-  EXCLUDE_FROM_LIBMLIR
-  )
-mlir_target_link_libraries(MLIRArmNeonTestPasses PUBLIC
-  MLIRArmNeonDialect
-  MLIRArmNeonTransforms
-  MLIRIR
-  MLIRPass
-  MLIRTransforms
-  )
diff --git a/external/llvm-project/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp b/external/llvm-project/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp
deleted file mode 100644
index 03c80b601a34..000000000000
--- a/external/llvm-project/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-//===- TestLowerToArmNeon.cpp - Test lowering to ArmNeon as a sink pass -===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a pass for testing the lowering to ArmNeon as a
-// generally usable sink pass.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
-#include "mlir/Dialect/ArmNeon/Transforms.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassManager.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-
-#define PASS_NAME "test-lower-to-arm-neon"
-
-using namespace mlir;
-using namespace mlir::arm_neon;
-
-namespace {
-struct TestLowerToArmNeon
-    : public PassWrapper<TestLowerToArmNeon, OperationPass<func::FuncOp>> {
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLowerToArmNeon)
-
-  StringRef getArgument() const final { return PASS_NAME; }
-  StringRef getDescription() const final { return "Tests lower to arm Neon."; }
-  TestLowerToArmNeon() = default;
-  TestLowerToArmNeon(const TestLowerToArmNeon &pass) = default;
-
-  void getDependentDialects(DialectRegistry &registry) const override {
-    registry.insert<arm_neon::ArmNeonDialect>();
-  }
-
-  void runOnOperation() override;
-};
-
-} // namespace
-
-void TestLowerToArmNeon::runOnOperation() {
-  MLIRContext *context = &getContext();
-  RewritePatternSet patterns(context);
-  populateLowerContractionToSMMLAPatternPatterns(patterns);
-  if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
-    return signalPassFailure();
-}
-
-namespace mlir {
-namespace test {
-
-void registerTestLowerToArmNeon() { PassRegistration<TestLowerToArmNeon>(); }
-
-} // namespace test
-} // namespace mlir
diff --git a/external/llvm-project/mlir/test/lib/Dialect/CMakeLists.txt b/external/llvm-project/mlir/test/lib/Dialect/CMakeLists.txt
index 2a3b2dce360b..eb2f74e8aeca 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/CMakeLists.txt
+++ b/external/llvm-project/mlir/test/lib/Dialect/CMakeLists.txt
@@ -1,6 +1,5 @@
 add_subdirectory(Affine)
 add_subdirectory(Arith)
-add_subdirectory(ArmNeon)
 add_subdirectory(ArmSME)
 add_subdirectory(Bufferization)
 add_subdirectory(ControlFlow)
diff --git a/external/llvm-project/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp b/external/llvm-project/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
index 046b9a65f335..738648b8ccdc 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
+++ b/external/llvm-project/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
@@ -65,11 +65,6 @@ struct TestLinalgTransforms
       llvm::cl::desc(
           "Test a fused pass that forwards memref.copy to vector.transfer"),
       llvm::cl::init(false)};
-  Option<bool> testGenericToVectorPattern{
-      *this, "test-linalg-to-vector-patterns",
-      llvm::cl::desc("Test a set of patterns that rewrite a linalg contraction "
-                     "in vector.contract form"),
-      llvm::cl::init(false)};
   Option<bool> testDecomposePadTensor{
       *this, "test-decompose-pad-tensor",
       llvm::cl::desc("Test transform pad tensor by copying with generic ops"),
@@ -166,15 +161,6 @@ static void applyVectorTransferForwardingPatterns(func::FuncOp funcOp) {
   (void)applyPatternsGreedily(funcOp, std::move(forwardPattern));
 }
 
-static void applyLinalgToVectorPatterns(func::FuncOp funcOp) {
-  RewritePatternSet patterns(funcOp.getContext());
-  auto *ctx = funcOp.getContext();
-  patterns.add<CopyVectorizationPattern>(ctx);
-  populatePadOpVectorizationPatterns(patterns);
-  populateConvolutionVectorizationPatterns(patterns);
-  (void)applyPatternsGreedily(funcOp, std::move(patterns));
-}
-
 static void applyDecomposePadPatterns(func::FuncOp funcOp) {
   RewritePatternSet patterns(funcOp.getContext());
   patterns.add<DecomposePadOpPattern>(funcOp.getContext());
@@ -254,8 +240,6 @@ void TestLinalgTransforms::runOnOperation() {
     return applyPatterns(getOperation());
   if (testVectorTransferForwardingPatterns)
     return applyVectorTransferForwardingPatterns(getOperation());
-  if (testGenericToVectorPattern)
-    return applyLinalgToVectorPatterns(getOperation());
   if (testDecomposePadTensor)
     return applyDecomposePadPatterns(getOperation());
   if (testDecomposeTensorPackOp)
diff --git a/external/llvm-project/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp b/external/llvm-project/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp
index e435130c2a41..0e191c32f009 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp
+++ b/external/llvm-project/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp
@@ -77,6 +77,11 @@ struct TestTensorTransforms
       llvm::cl::desc("Test folding of expand_shape/collapse_shape"),
       llvm::cl::init(false)};
 
+  Option<bool> testFoldExtractFromCollapseShape{
+      *this, "test-fold-extract-from-collapse-shape",
+      llvm::cl::desc("Test folding of extract from collapse_shape"),
+      llvm::cl::init(false)};
+
   Option<bool> useForeach{
       *this, "use-foreach",
       llvm::cl::desc(
@@ -132,6 +137,12 @@ applyDropRedundantInsertSliceRankExpansionPatterns(Operation *rootOp) {
   (void)applyPatternsGreedily(rootOp, std::move(patterns));
 }
 
+static void applyFoldExtractFromCollapseShapePatterns(Operation *rootOp) {
+  RewritePatternSet patterns(rootOp->getContext());
+  tensor::populateFoldCollapseExtractPatterns(patterns);
+  (void)applyPatternsGreedily(rootOp, std::move(patterns));
+}
+
 namespace {
 /// Base pattern to rewrite  a `tensor.collapse_shape -> tensor.extract_slice`.
 /// The `tensor.extract_slice` is replaced by a loop or gather operation that
@@ -380,6 +391,8 @@ void TestTensorTransforms::runOnOperation() {
             applyRewriteExtractFromCollapseShapePatterns(rootOp, useForeach)))
       return signalPassFailure();
   }
+  if (testFoldExtractFromCollapseShape)
+    applyFoldExtractFromCollapseShapePatterns(rootOp);
   if (testTrackingListener)
     if (failed(testTrackingListenerReplacements(rootOp)))
       return signalPassFailure();
diff --git a/external/llvm-project/mlir/test/lib/Dialect/TestIRDLToCpp/CMakeLists.txt b/external/llvm-project/mlir/test/lib/Dialect/TestIRDLToCpp/CMakeLists.txt
index 98a4bd3058a2..103bc94d8692 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/TestIRDLToCpp/CMakeLists.txt
+++ b/external/llvm-project/mlir/test/lib/Dialect/TestIRDLToCpp/CMakeLists.txt
@@ -7,8 +7,9 @@ add_mlir_library(MLIRTestIRDLToCppDialect
 
   DEPENDS
   TestIRDLToCppGen
+)
 
-  LINK_LIBS PUBLIC
+mlir_target_link_libraries(MLIRTestIRDLToCppDialect PUBLIC
   MLIRIR
   MLIRPass
   MLIRTransforms
diff --git a/external/llvm-project/mlir/test/lib/Dialect/Tosa/CMakeLists.txt b/external/llvm-project/mlir/test/lib/Dialect/Tosa/CMakeLists.txt
index 49b169c4e6fd..43f0d0d21c1c 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/Tosa/CMakeLists.txt
+++ b/external/llvm-project/mlir/test/lib/Dialect/Tosa/CMakeLists.txt
@@ -15,5 +15,4 @@ mlir_target_link_libraries(MLIRTosaTestPasses PUBLIC
   MLIRPass
   MLIRTosaDialect
   MLIRTransformUtils
-  MLIRTosaTransforms
   )
diff --git a/external/llvm-project/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp b/external/llvm-project/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
index 58faad51bd6c..3389a1ccfe98 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
+++ b/external/llvm-project/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
@@ -18,9 +18,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/SmallVector.h"
 
 #define PASS_NAME "tosa-test-quant-utils"
 
@@ -148,12 +146,10 @@ ConvertTosaConv2DOp::matchAndRewrite(Operation *op,
       RankedTensorType::get(outputType.getShape(), rewriter.getIntegerType(32));
 
   auto newTosaConv2DOp = rewriter.create<tosa::Conv2DOp>(
-    op->getLoc(), newTosaConv2DOpType, tosaConv2DOp.getInput(),
-    tosaConv2DOp.getWeight(), tosaConv2DOp.getBias(),
-    tosaConv2DOp.getPadAttr(), tosaConv2DOp.getStrideAttr(),
-    tosaConv2DOp.getDilationAttr(), tosaConv2DOp.getAccTypeAttr());
-
-  newTosaConv2DOp.setGroupAttr(tosaConv2DOp.getGroupAttr());
+      op->getLoc(), newTosaConv2DOpType, tosaConv2DOp.getInput(),
+      tosaConv2DOp.getWeight(), tosaConv2DOp.getBias(),
+      tosaConv2DOp.getPadAttr(), tosaConv2DOp.getStrideAttr(),
+      tosaConv2DOp.getDilationAttr(), tosaConv2DOp.getAccTypeAttr());
 
   // Create rescale to quantized type
   double inputScale = inputQType.getScale();
diff --git a/external/llvm-project/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/external/llvm-project/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index f4f32e933987..54aa96ba89a0 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/external/llvm-project/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -157,12 +157,14 @@ struct TestVectorUnrollingPatterns
     MLIRContext *ctx = &getContext();
     RewritePatternSet patterns(ctx);
     populateVectorUnrollPatterns(
-        patterns, UnrollVectorOptions()
-                      .setNativeShape(ArrayRef<int64_t>{2, 2})
-                      .setFilterConstraint([](Operation *op) {
-                        return success(isa<arith::AddFOp, vector::FMAOp,
-                                           vector::MultiDimReductionOp>(op));
-                      }));
+        patterns,
+        UnrollVectorOptions()
+            .setNativeShape(ArrayRef<int64_t>{2, 2})
+            .setFilterConstraint([](Operation *op) {
+              return success(
+                  isa<arith::AddFOp, vector::FMAOp, vector::MultiDimReductionOp,
+                      vector::BroadcastOp>(op));
+            }));
     populateVectorUnrollPatterns(
         patterns, UnrollVectorOptions()
                       .setNativeShape(ArrayRef<int64_t>{2})
diff --git a/external/llvm-project/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp b/external/llvm-project/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
index eaa3b988cad8..3f3461e92bc0 100644
--- a/external/llvm-project/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
+++ b/external/llvm-project/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
@@ -121,4 +121,4 @@ void registerTestXeGPULowerings() {
   PassRegistration<TestXeGPUUnrollingPatterns>();
 }
 } // namespace test
-} // namespace mlir
\ No newline at end of file
+} // namespace mlir
diff --git a/external/llvm-project/mlir/test/lib/IR/TestSideEffects.cpp b/external/llvm-project/mlir/test/lib/IR/TestSideEffects.cpp
index 7e01509d5568..000e7c204fd5 100644
--- a/external/llvm-project/mlir/test/lib/IR/TestSideEffects.cpp
+++ b/external/llvm-project/mlir/test/lib/IR/TestSideEffects.cpp
@@ -52,12 +52,12 @@ struct SideEffectsPass
           diag << "'write'";
 
         if (instance.getValue()) {
-          if (instance.getEffectValue<OpOperand *>())
-            diag << " on a op operand,";
-          else if (instance.getEffectValue<OpResult>())
-            diag << " on a op result,";
-          else if (instance.getEffectValue<BlockArgument>())
-            diag << " on a block argument,";
+          if (auto *opOpd = instance.getEffectValue<OpOperand *>())
+            diag << " on op operand " << opOpd->getOperandNumber() << ",";
+          else if (auto opRes = instance.getEffectValue<OpResult>())
+            diag << " on op result " << opRes.getResultNumber() << ",";
+          else if (auto opBlk = instance.getEffectValue<BlockArgument>())
+            diag << " on block argument " << opBlk.getArgNumber() << ",";
         } else if (SymbolRefAttr symbolRef = instance.getSymbolRef())
           diag << " on a symbol '" << symbolRef << "',";
 
diff --git a/external/llvm-project/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp b/external/llvm-project/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
index 45d6ae382015..9971f0cde4ed 100644
--- a/external/llvm-project/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
+++ b/external/llvm-project/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
@@ -260,8 +260,7 @@ applyTileToAll(RewriterBase &rewriter, Operation *transformOp,
       return failure();
 
     // Perform the replacement of tiled and fused values.
-    rewriter.replaceOp(tilingInterfaceOp,
-                       tiledResults->mergeResult.replacements);
+    rewriter.replaceOp(tilingInterfaceOp, tiledResults->replacements);
 
     // Report back the relevant handles to the transform op.
     tiledOps.push_back(tiledResults->tiledOps.front());
diff --git a/external/llvm-project/mlir/test/lit.cfg.py b/external/llvm-project/mlir/test/lit.cfg.py
index 16051d1492e8..9b5cadd62bef 100644
--- a/external/llvm-project/mlir/test/lit.cfg.py
+++ b/external/llvm-project/mlir/test/lit.cfg.py
@@ -164,7 +164,6 @@ def find_real_python_interpreter():
 
 
 llvm_config.with_system_environment(["HOME", "INCLUDE", "LIB", "TMP", "TEMP"])
-llvm_config.with_environment("AMD_LOG_LEVEL", "4")
 
 llvm_config.use_default_substitutions()
 
diff --git a/external/llvm-project/mlir/test/mlir-lsp-server/uri-based-registration.test b/external/llvm-project/mlir/test/mlir-lsp-server/uri-based-registration.test
new file mode 100644
index 000000000000..d6d06692a8d7
--- /dev/null
+++ b/external/llvm-project/mlir/test/mlir-lsp-server/uri-based-registration.test
@@ -0,0 +1,23 @@
+// RUN: not mlir-lsp-server -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"mlir","capabilities":{},"trace":"off"}}
+// -----
+// Just regular parse, successful.
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{
+  "uri":"test:///foo-regular-registration.mlir",
+  "languageId":"mlir",
+  "version":1,
+  "text":"func.func @fail_with_empty_registry() { return }"
+}}}
+// CHECK: "method": "textDocument/publishDiagnostics",
+// CHECK:     "diagnostics": []
+// -----
+// Just regular parse, successful.
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{
+  "uri":"test:///foo-disable-lsp-registration.mlir",
+  "languageId":"mlir",
+  "version":1,
+  "text":"func.func @fail_with_empty_registry() { return }"
+}}}
+// CHECK: "method": "textDocument/publishDiagnostics",
+// CHECK: "message": "Dialect `func' not found for custom op 'func.func'
+
diff --git a/external/llvm-project/mlir/test/mlir-tblgen/attr-constraints.td b/external/llvm-project/mlir/test/mlir-tblgen/attr-constraints.td
new file mode 100644
index 000000000000..59bc5f252660
--- /dev/null
+++ b/external/llvm-project/mlir/test/mlir-tblgen/attr-constraints.td
@@ -0,0 +1,14 @@
+// RUN: mlir-tblgen -gen-attr-constraint-decls -I %S/../../include %s | FileCheck %s --check-prefix=DECL
+// RUN: mlir-tblgen -gen-attr-constraint-defs -I %S/../../include %s | FileCheck %s --check-prefix=DEF
+
+include "mlir/IR/CommonAttrConstraints.td"
+
+def DummyConstraint : AnyAttrOf<[APIntAttr, ArrayAttr, UnitAttr]> {
+  let cppFunctionName = "isValidDummy";
+}
+
+// DECL: bool isValidDummy(::mlir::Attribute attr);
+
+// DEF: bool isValidDummy(::mlir::Attribute attr) {
+// DEF:   return (((::llvm::isa<::mlir::IntegerAttr>(attr))) || ((::llvm::isa<::mlir::ArrayAttr>(attr))) || ((::llvm::isa<::mlir::UnitAttr>(attr))));
+// DEF: }
diff --git a/external/llvm-project/mlir/test/mlir-tblgen/directive-common.td b/external/llvm-project/mlir/test/mlir-tblgen/directive-common.td
index 9429238a03f0..78747b82f1f4 100644
--- a/external/llvm-project/mlir/test/mlir-tblgen/directive-common.td
+++ b/external/llvm-project/mlir/test/mlir-tblgen/directive-common.td
@@ -7,11 +7,11 @@ def TestDirectiveLanguage : DirectiveLanguage {
   let cppNamespace = "tdl";
 }
 
-def TDLCV_vala : ClauseVal<"vala",1,1> {}
-def TDLCV_valb : ClauseVal<"valb",2,1> {}
-def TDLCV_valc : ClauseVal<"valc",3,0> { let isDefault = 1; }
+def TDLCV_vala : EnumVal<"vala",1,1> {}
+def TDLCV_valb : EnumVal<"valb",2,1> {}
+def TDLCV_valc : EnumVal<"valc",3,0> { let isDefault = 1; }
 
-def TDLC_ClauseA : Clause<"clausea"> {
+def TDLC_ClauseA : Clause<[Spelling<"clausea">]> {
   let flangClass = "TdlClauseA";
   let enumClauseValue = "AKind";
   let allowedClauseValues = [
diff --git a/external/llvm-project/mlir/test/python/dialects/transform.py b/external/llvm-project/mlir/test/python/dialects/transform.py
index 6ed4818fc9d2..6c5e4e5505b1 100644
--- a/external/llvm-project/mlir/test/python/dialects/transform.py
+++ b/external/llvm-project/mlir/test/python/dialects/transform.py
@@ -254,3 +254,81 @@ def testReplicateOp(module: Module):
     # CHECK: %[[FIRST:.+]] = pdl_match
     # CHECK: %[[SECOND:.+]] = pdl_match
     # CHECK: %{{.*}} = replicate num(%[[FIRST]]) %[[SECOND]]
+
+
+# CHECK-LABEL: TEST: testApplyRegisteredPassOp
+@run
+def testApplyRegisteredPassOp(module: Module):
+    # CHECK: transform.sequence
+    sequence = transform.SequenceOp(
+        transform.FailurePropagationMode.Propagate, [], transform.AnyOpType.get()
+    )
+    with InsertionPoint(sequence.body):
+        # CHECK:   %{{.*}} = apply_registered_pass "canonicalize" to {{.*}} : (!transform.any_op) -> !transform.any_op
+        mod = transform.ApplyRegisteredPassOp(
+            transform.AnyOpType.get(), sequence.bodyTarget, "canonicalize"
+        )
+        # CHECK:   %{{.*}} = apply_registered_pass "canonicalize"
+        # CHECK-SAME:    with options = {"top-down" = false}
+        # CHECK-SAME:    to {{.*}} : (!transform.any_op) -> !transform.any_op
+        mod = transform.ApplyRegisteredPassOp(
+            transform.AnyOpType.get(),
+            mod.result,
+            "canonicalize",
+            options={"top-down": BoolAttr.get(False)},
+        )
+        # CHECK:   %[[MAX_ITER:.+]] = transform.param.constant
+        max_iter = transform.param_constant(
+            transform.AnyParamType.get(),
+            IntegerAttr.get(IntegerType.get_signless(64), 10),
+        )
+        # CHECK:   %[[MAX_REWRITE:.+]] = transform.param.constant
+        max_rewrites = transform.param_constant(
+            transform.AnyParamType.get(),
+            IntegerAttr.get(IntegerType.get_signless(64), 1),
+        )
+        # CHECK:   %{{.*}} = apply_registered_pass "canonicalize"
+        # NB: MLIR has sorted the dict lexicographically by key:
+        # CHECK-SAME:    with options = {"max-iterations" = %[[MAX_ITER]],
+        # CHECK-SAME:                    "max-rewrites" =  %[[MAX_REWRITE]],
+        # CHECK-SAME:                    "test-convergence" = true,
+        # CHECK-SAME:                    "top-down" = false}
+        # CHECK-SAME:    to %{{.*}} : (!transform.any_op, !transform.any_param, !transform.any_param) -> !transform.any_op
+        mod = transform.apply_registered_pass(
+            transform.AnyOpType.get(),
+            mod,
+            "canonicalize",
+            options={
+                "top-down": BoolAttr.get(False),
+                "max-iterations": max_iter,
+                "test-convergence": True,
+                "max-rewrites": max_rewrites,
+            },
+        )
+        # CHECK:   %{{.*}} = apply_registered_pass "symbol-privatize"
+        # CHECK-SAME:    with options = {"exclude" = ["a", "b"]}
+        # CHECK-SAME:    to %{{.*}} : (!transform.any_op) -> !transform.any_op
+        mod = transform.apply_registered_pass(
+            transform.AnyOpType.get(),
+            mod,
+            "symbol-privatize",
+            options={"exclude": ("a", "b")},
+        )
+        # CHECK:   %[[SYMBOL_A:.+]] = transform.param.constant
+        symbol_a = transform.param_constant(
+            transform.AnyParamType.get(), StringAttr.get("a")
+        )
+        # CHECK:   %[[SYMBOL_B:.+]] = transform.param.constant
+        symbol_b = transform.param_constant(
+            transform.AnyParamType.get(), StringAttr.get("b")
+        )
+        # CHECK:   %{{.*}} = apply_registered_pass "symbol-privatize"
+        # CHECK-SAME:    with options = {"exclude" = [%[[SYMBOL_A]], %[[SYMBOL_B]]]}
+        # CHECK-SAME:    to %{{.*}} : (!transform.any_op, !transform.any_param, !transform.any_param) -> !transform.any_op
+        mod = transform.apply_registered_pass(
+            transform.AnyOpType.get(),
+            mod,
+            "symbol-privatize",
+            options={"exclude": (symbol_a, symbol_b)},
+        )
+        transform.YieldOp()
diff --git a/external/llvm-project/mlir/tools/mlir-lsp-server/mlir-lsp-server.cpp b/external/llvm-project/mlir/tools/mlir-lsp-server/mlir-lsp-server.cpp
index f0ecc5adc68b..6a759d9e0d60 100644
--- a/external/llvm-project/mlir/tools/mlir-lsp-server/mlir-lsp-server.cpp
+++ b/external/llvm-project/mlir/tools/mlir-lsp-server/mlir-lsp-server.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/InitAllDialects.h"
 #include "mlir/InitAllExtensions.h"
+#include "mlir/Tools/lsp-server-support/Protocol.h"
 #include "mlir/Tools/mlir-lsp-server/MlirLspServerMain.h"
 
 using namespace mlir;
@@ -23,7 +23,7 @@ void registerTestTransformDialectExtension(DialectRegistry &);
 #endif
 
 int main(int argc, char **argv) {
-  DialectRegistry registry;
+  DialectRegistry registry, empty;
   registerAllDialects(registry);
   registerAllExtensions(registry);
 
@@ -32,5 +32,18 @@ int main(int argc, char **argv) {
   ::test::registerTestTransformDialectExtension(registry);
   ::test::registerTestDynDialect(registry);
 #endif
-  return failed(MlirLspServerMain(argc, argv, registry));
+
+  // Returns the registry, except in testing mode when the URI contains
+  // "-disable-lsp-registration". Testing for/example of registering dialects
+  // based on URI.
+  auto registryFn = [&registry,
+                     &empty](const lsp::URIForFile &uri) -> DialectRegistry & {
+    (void)empty;
+#ifdef MLIR_INCLUDE_TESTS
+    if (uri.uri().contains("-disable-lsp-registration"))
+      return empty;
+#endif
+    return registry;
+  };
+  return failed(MlirLspServerMain(argc, argv, registryFn));
 }
diff --git a/external/llvm-project/mlir/tools/mlir-opt/CMakeLists.txt b/external/llvm-project/mlir/tools/mlir-opt/CMakeLists.txt
index 67fea55d141c..26d7597347a8 100644
--- a/external/llvm-project/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/external/llvm-project/mlir/tools/mlir-opt/CMakeLists.txt
@@ -17,7 +17,6 @@ if(MLIR_INCLUDE_TESTS)
     MLIRTestFuncToLLVM
     MLIRAffineTransformsTestPasses
     MLIRArithTestPasses
-    MLIRArmNeonTestPasses
     MLIRArmSMETestPasses
     MLIRBufferizationTestPasses
     MLIRControlFlowTestPasses
@@ -29,6 +28,7 @@ if(MLIR_INCLUDE_TESTS)
     MLIRMathTestPasses
     MLIRTestMathToVCIX
     MLIRMemRefTestPasses
+    MLIRTestMemRefToLLVMWithTransforms
     MLIRMeshTest
     MLIRNVGPUTestPasses
     MLIRSCFTestPasses
diff --git a/external/llvm-project/mlir/tools/mlir-opt/mlir-opt.cpp b/external/llvm-project/mlir/tools/mlir-opt/mlir-opt.cpp
index b51da0e0383d..6ef9ff8e8454 100644
--- a/external/llvm-project/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/external/llvm-project/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -120,7 +120,6 @@ void registerTestLLVMLegalizePatternsPass();
 void registerTestLoopFusion();
 void registerTestLoopMappingPass();
 void registerTestLoopUnrollingPass();
-void registerTestLowerToArmNeon();
 void registerTestLowerToArmSME();
 void registerTestLowerToLLVM();
 void registerTestMakeIsolatedFromAbovePass();
@@ -131,6 +130,7 @@ void registerTestMathToVCIXPass();
 void registerTestIrdlTestDialectConversionPass();
 void registerTestMemRefDependenceCheck();
 void registerTestMemRefStrideCalculation();
+void registerTestMemRefToLLVMWithTransforms();
 void registerTestMeshReshardingSpmdizationPass();
 void registerTestMeshSimplificationsPass();
 void registerTestMultiBuffering();
@@ -267,7 +267,6 @@ void registerTestPasses() {
   mlir::test::registerTestLoopFusion();
   mlir::test::registerTestLoopMappingPass();
   mlir::test::registerTestLoopUnrollingPass();
-  mlir::test::registerTestLowerToArmNeon();
   mlir::test::registerTestLowerToArmSME();
   mlir::test::registerTestLowerToLLVM();
   mlir::test::registerTestMakeIsolatedFromAbovePass();
@@ -277,6 +276,7 @@ void registerTestPasses() {
   mlir::test::registerTestMathToVCIXPass();
   mlir::test::registerTestMemRefDependenceCheck();
   mlir::test::registerTestMemRefStrideCalculation();
+  mlir::test::registerTestMemRefToLLVMWithTransforms();
   mlir::test::registerTestMeshReshardingSpmdizationPass();
   mlir::test::registerTestMeshSimplificationsPass();
   mlir::test::registerTestMultiBuffering();
diff --git a/external/llvm-project/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp b/external/llvm-project/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
index 2a6071602fa4..defd1fa12ca1 100644
--- a/external/llvm-project/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
+++ b/external/llvm-project/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
@@ -1083,15 +1083,15 @@ bool DefGenerator::emitDefs(StringRef selectedDialect) {
 }
 
 //===----------------------------------------------------------------------===//
-// Type Constraints
+// Constraints
 //===----------------------------------------------------------------------===//
 
 /// Find all type constraints for which a C++ function should be generated.
-static std::vector<Constraint>
-getAllTypeConstraints(const RecordKeeper &records) {
+static std::vector<Constraint> getAllCppConstraints(const RecordKeeper &records,
+                                                    StringRef constraintKind) {
   std::vector<Constraint> result;
   for (const Record *def :
-       records.getAllDerivedDefinitionsIfDefined("TypeConstraint")) {
+       records.getAllDerivedDefinitionsIfDefined(constraintKind)) {
     // Ignore constraints defined outside of the top-level file.
     if (llvm::SrcMgr.FindBufferContainingLoc(def->getLoc()[0]) !=
         llvm::SrcMgr.getMainFileID())
@@ -1105,32 +1105,74 @@ getAllTypeConstraints(const RecordKeeper &records) {
   return result;
 }
 
+static std::vector<Constraint>
+getAllCppTypeConstraints(const RecordKeeper &records) {
+  return getAllCppConstraints(records, "TypeConstraint");
+}
+
+static std::vector<Constraint>
+getAllCppAttrConstraints(const RecordKeeper &records) {
+  return getAllCppConstraints(records, "AttrConstraint");
+}
+
+/// Emit the declarations for the given constraints, of the form:
+/// `bool <constraintCppFunctionName>(<parameterTypeName> <parameterName>);`
+static void emitConstraintDecls(const std::vector<Constraint> &constraints,
+                                raw_ostream &os, StringRef parameterTypeName,
+                                StringRef parameterName) {
+  static const char *const constraintDecl = "bool {0}({1} {2});\n";
+  for (Constraint constr : constraints)
+    os << strfmt(constraintDecl, *constr.getCppFunctionName(),
+                 parameterTypeName, parameterName);
+}
+
 static void emitTypeConstraintDecls(const RecordKeeper &records,
                                     raw_ostream &os) {
-  static const char *const typeConstraintDecl = R"(
-bool {0}(::mlir::Type type);
-)";
+  emitConstraintDecls(getAllCppTypeConstraints(records), os, "::mlir::Type",
+                      "type");
+}
 
-  for (Constraint constr : getAllTypeConstraints(records))
-    os << strfmt(typeConstraintDecl, *constr.getCppFunctionName());
+static void emitAttrConstraintDecls(const RecordKeeper &records,
+                                    raw_ostream &os) {
+  emitConstraintDecls(getAllCppAttrConstraints(records), os,
+                      "::mlir::Attribute", "attr");
 }
 
-static void emitTypeConstraintDefs(const RecordKeeper &records,
-                                   raw_ostream &os) {
-  static const char *const typeConstraintDef = R"(
-bool {0}(::mlir::Type type) {
-  return ({1});
+/// Emit the definitions for the given constraints, of the form:
+/// `bool <constraintCppFunctionName>(<parameterTypeName> <parameterName>) {
+///   return (<condition>); }`
+/// where `<condition>` is the condition template with the `self` variable
+/// replaced with the `selfName` parameter.
+static void emitConstraintDefs(const std::vector<Constraint> &constraints,
+                               raw_ostream &os, StringRef parameterTypeName,
+                               StringRef selfName) {
+  static const char *const constraintDef = R"(
+bool {0}({1} {2}) {
+return ({3});
 }
 )";
 
-  for (Constraint constr : getAllTypeConstraints(records)) {
+  for (Constraint constr : constraints) {
     FmtContext ctx;
-    ctx.withSelf("type");
+    ctx.withSelf(selfName);
     std::string condition = tgfmt(constr.getConditionTemplate(), &ctx);
-    os << strfmt(typeConstraintDef, *constr.getCppFunctionName(), condition);
+    os << strfmt(constraintDef, *constr.getCppFunctionName(), parameterTypeName,
+                 selfName, condition);
   }
 }
 
+static void emitTypeConstraintDefs(const RecordKeeper &records,
+                                   raw_ostream &os) {
+  emitConstraintDefs(getAllCppTypeConstraints(records), os, "::mlir::Type",
+                     "type");
+}
+
+static void emitAttrConstraintDefs(const RecordKeeper &records,
+                                   raw_ostream &os) {
+  emitConstraintDefs(getAllCppAttrConstraints(records), os, "::mlir::Attribute",
+                     "attr");
+}
+
 //===----------------------------------------------------------------------===//
 // GEN: Registration hooks
 //===----------------------------------------------------------------------===//
@@ -1158,6 +1200,21 @@ static mlir::GenRegistration
                    return generator.emitDecls(attrDialect);
                  });
 
+static mlir::GenRegistration
+    genAttrConstrDefs("gen-attr-constraint-defs",
+                      "Generate attribute constraint definitions",
+                      [](const RecordKeeper &records, raw_ostream &os) {
+                        emitAttrConstraintDefs(records, os);
+                        return false;
+                      });
+static mlir::GenRegistration
+    genAttrConstrDecls("gen-attr-constraint-decls",
+                       "Generate attribute constraint declarations",
+                       [](const RecordKeeper &records, raw_ostream &os) {
+                         emitAttrConstraintDecls(records, os);
+                         return false;
+                       });
+
 //===----------------------------------------------------------------------===//
 // TypeDef
 //===----------------------------------------------------------------------===//
diff --git a/external/llvm-project/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp b/external/llvm-project/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp
index 602b3df3094a..09e1bdafe7bc 100644
--- a/external/llvm-project/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp
+++ b/external/llvm-project/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp
@@ -21,7 +21,7 @@
 #include "llvm/TableGen/Record.h"
 
 using llvm::Clause;
-using llvm::ClauseVal;
+using llvm::EnumVal;
 using llvm::raw_ostream;
 using llvm::RecordKeeper;
 
@@ -63,11 +63,11 @@ static bool emitDecls(const RecordKeeper &records, llvm::StringRef dialect,
 
     std::vector<std::string> cvDefs;
     for (const auto &it : llvm::enumerate(clauseVals)) {
-      const ClauseVal cval{it.value()};
-      if (!cval.isUserVisible())
+      const EnumVal val{it.value()};
+      if (!val.isUserVisible())
         continue;
 
-      std::string name = cval.getFormattedName();
+      std::string name = val.getFormattedName();
       std::string enumValName(name.length(), ' ');
       llvm::transform(name, enumValName.begin(), llvm::toLower);
       enumValName[0] = llvm::toUpper(enumValName[0]);
diff --git a/external/llvm-project/mlir/unittests/Analysis/Presburger/QuasiPolynomialTest.cpp b/external/llvm-project/mlir/unittests/Analysis/Presburger/QuasiPolynomialTest.cpp
index a84f0234067a..e9f3c577af97 100644
--- a/external/llvm-project/mlir/unittests/Analysis/Presburger/QuasiPolynomialTest.cpp
+++ b/external/llvm-project/mlir/unittests/Analysis/Presburger/QuasiPolynomialTest.cpp
@@ -137,4 +137,4 @@ TEST(QuasiPolynomialTest, simplify) {
                       {{{Fraction(1, 1), Fraction(3, 4), Fraction(5, 3)},
                         {Fraction(2, 1), Fraction(0, 1), Fraction(0, 1)}},
                        {{Fraction(1, 1), Fraction(4, 5), Fraction(6, 5)}}}));
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/mlir/unittests/Dialect/Utils/CMakeLists.txt b/external/llvm-project/mlir/unittests/Dialect/Utils/CMakeLists.txt
index 61b9cdcb3b8f..e921c8bcfb4e 100644
--- a/external/llvm-project/mlir/unittests/Dialect/Utils/CMakeLists.txt
+++ b/external/llvm-project/mlir/unittests/Dialect/Utils/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_mlir_unittest(MLIRDialectUtilsTests
   StructuredOpsUtilsTest.cpp
+  ReshapeOpsUtilsTest.cpp
   IndexingUtilsTest.cpp
 )
 mlir_target_link_libraries(MLIRDialectUtilsTests
diff --git a/external/llvm-project/mlir/unittests/Dialect/Utils/ReshapeOpsUtilsTest.cpp b/external/llvm-project/mlir/unittests/Dialect/Utils/ReshapeOpsUtilsTest.cpp
new file mode 100644
index 000000000000..05f97e875e2d
--- /dev/null
+++ b/external/llvm-project/mlir/unittests/Dialect/Utils/ReshapeOpsUtilsTest.cpp
@@ -0,0 +1,203 @@
+//===- ReshapeOpsUtilsTest.cpp - ReshapeOpsUtils unit tests ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
+#include "mlir/IR/BuiltinTypeInterfaces.h"
+#include "llvm/ADT/STLExtras.h"
+#include "gtest/gtest.h"
+#include <optional>
+
+using namespace mlir;
+
+/// Helper to make constructing
+/// `std::optional<SmallVector<ReassociationIndices>>` more readable.
+static std::optional<SmallVector<ReassociationIndices>>
+makeOptionalIndices(std::initializer_list<ReassociationIndices> list) {
+  return std::optional<SmallVector<ReassociationIndices>>(list);
+}
+
+TEST(ReassociationIndicesForCollapse, ScalarTest) {
+  EXPECT_EQ(getReassociationIndicesForCollapse({1}, {}),
+            makeOptionalIndices({}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({1, 1}, {}),
+            makeOptionalIndices({}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({ShapedType::kDynamic}, {}),
+            makeOptionalIndices({}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({1, ShapedType::kDynamic,
+                                                ShapedType::kDynamic, 1,
+                                                ShapedType::kDynamic},
+                                               {}),
+            makeOptionalIndices({}));
+}
+
+TEST(ReassociationIndicesForCollapse, ScalarTestFailure) {
+  EXPECT_EQ(getReassociationIndicesForCollapse({}, {}), std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse({}, {1}), std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse({2}, {}), std::nullopt);
+  EXPECT_EQ(
+      getReassociationIndicesForCollapse({1, 2, ShapedType::kDynamic, 1}, {}),
+      std::nullopt);
+}
+
+TEST(ReassociationIndicesForCollapse, StaticTest) {
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 20}, {200}),
+            makeOptionalIndices({{0, 1}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 20, 30}, {10, 600}),
+            makeOptionalIndices({{0}, {1, 2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 20, 30}, {200, 30}),
+            makeOptionalIndices({{0, 1}, {2}}));
+}
+
+TEST(ReassociationIndicesForCollapse, StaticTestFailure) {
+  // No-op reassociation
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 20}, {10, 20}),
+            std::nullopt);
+  // Invalid static reassociations
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 20}, {10}), std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 20, 30}, {200, 300}),
+            std::nullopt);
+  // Non-collapsing (expanding) reassociation
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 20, 30}, {1, 10, 20, 30}),
+            std::nullopt);
+}
+
+TEST(ReassociationIndicesForCollapse, StaticTestUnitDims) {
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, 1}, {10}),
+            makeOptionalIndices({{0, 1}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({1, 20, 30}, {600}),
+            makeOptionalIndices({{0, 1, 2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({1, 1, 1}, {1}),
+            makeOptionalIndices({{0, 1, 2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({1, 1, 1, 1}, {1, 1, 1}),
+            makeOptionalIndices({{0}, {1}, {2, 3}}));
+}
+
+TEST(ReassociationIndicesForCollapse, DynamicTest) {
+  EXPECT_EQ(getReassociationIndicesForCollapse({ShapedType::kDynamic, 1},
+                                               {ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({ShapedType::kDynamic, 1, 1},
+                                               {ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1, 2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {1, ShapedType::kDynamic, 1, ShapedType::kDynamic, 1},
+                {ShapedType::kDynamic, ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1}, {2, 3, 4}}));
+  EXPECT_EQ(
+      getReassociationIndicesForCollapse(
+          {ShapedType::kDynamic, ShapedType::kDynamic}, {ShapedType::kDynamic}),
+      makeOptionalIndices({{0, 1}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {1, ShapedType::kDynamic, ShapedType::kDynamic},
+                {1, ShapedType::kDynamic}),
+            makeOptionalIndices({{0}, {1, 2}}));
+
+  EXPECT_EQ(getReassociationIndicesForCollapse({ShapedType::kDynamic, 10},
+                                               {ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {1, ShapedType::kDynamic, ShapedType::kDynamic},
+                {ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1, 2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, ShapedType::kDynamic},
+                                               {ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 1, 2, ShapedType::kDynamic, 10},
+                {ShapedType::kDynamic, 10}),
+            makeOptionalIndices({{0, 1, 2, 3}, {4}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({ShapedType::kDynamic, 10, 20},
+                                               {ShapedType::kDynamic, 20}),
+            makeOptionalIndices({{0, 1}, {2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({10, ShapedType::kDynamic, 20},
+                                               {ShapedType::kDynamic, 20}),
+            makeOptionalIndices({{0, 1}, {2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 3, 2, 5, 2}, {ShapedType::kDynamic, 20}),
+            makeOptionalIndices({{0, 1}, {2, 3, 4}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {10, ShapedType::kDynamic, 20, ShapedType::kDynamic, 1},
+                {ShapedType::kDynamic, 20, ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1}, {2}, {3, 4}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse({1, ShapedType::kDynamic, 1},
+                                               {ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1, 2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, ShapedType::kDynamic, 1},
+                {ShapedType::kDynamic, ShapedType::kDynamic}),
+            makeOptionalIndices({{0}, {1, 2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {1, ShapedType::kDynamic, ShapedType::kDynamic},
+                {ShapedType::kDynamic, ShapedType::kDynamic}),
+            makeOptionalIndices({{0, 1}, {2}}));
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 1, ShapedType::kDynamic},
+                {ShapedType::kDynamic, ShapedType::kDynamic}),
+            makeOptionalIndices({{0}, {1, 2}}));
+}
+
+TEST(ReassociationIndicesForCollapse, DynamicTestFailure) {
+  EXPECT_EQ(getReassociationIndicesForCollapse({ShapedType::kDynamic, 10, 20},
+                                               {ShapedType::kDynamic, 10}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 10, ShapedType::kDynamic},
+                {ShapedType::kDynamic, ShapedType::kDynamic}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {20, ShapedType::kDynamic, 10, ShapedType::kDynamic},
+                {ShapedType::kDynamic, ShapedType::kDynamic}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 5, 3, 2, 2}, {ShapedType::kDynamic, 20}),
+            std::nullopt);
+  EXPECT_EQ(
+      getReassociationIndicesForCollapse(
+          {ShapedType::kDynamic, ShapedType::kDynamic, ShapedType::kDynamic},
+          {ShapedType::kDynamic, ShapedType::kDynamic}),
+      std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, ShapedType::kDynamic, 10, 1,
+                 ShapedType::kDynamic},
+                {ShapedType::kDynamic, ShapedType::kDynamic}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 10, 10, 10, ShapedType::kDynamic},
+                {ShapedType::kDynamic, 10, ShapedType::kDynamic}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 10, 10, 10, ShapedType::kDynamic},
+                {ShapedType::kDynamic, 2, 2, ShapedType::kDynamic}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 3, 4, 3, ShapedType::kDynamic},
+                {ShapedType::kDynamic, 12, ShapedType::kDynamic}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 8, 4, 2, 16, ShapedType::kDynamic},
+                {ShapedType::kDynamic, 32, ShapedType::kDynamic}),
+            std::nullopt);
+
+  //===----------------------------------------------------------------------===//
+  // TODO: Reassociation for the following examples can be computed, but isn't
+  // supported by `getReassociationIndicesForCollapse`.
+  //===----------------------------------------------------------------------===//
+
+  // TODO: Fails because there's no backtracking when some source dimensions
+  // remain unmatched at either edge.
+  EXPECT_EQ(getReassociationIndicesForCollapse(
+                {ShapedType::kDynamic, 10, ShapedType::kDynamic, 10},
+                {ShapedType::kDynamic, 10}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse({1, ShapedType::kDynamic, 2, 2},
+                                               {1, ShapedType::kDynamic, 2}),
+            std::nullopt);
+  EXPECT_EQ(getReassociationIndicesForCollapse({2, 2, ShapedType::kDynamic, 1},
+                                               {2, ShapedType::kDynamic}),
+            std::nullopt);
+}
diff --git a/external/llvm-project/mlir/unittests/IR/OperationSupportTest.cpp b/external/llvm-project/mlir/unittests/IR/OperationSupportTest.cpp
index bac2b72b68de..18ee9d71cb9f 100644
--- a/external/llvm-project/mlir/unittests/IR/OperationSupportTest.cpp
+++ b/external/llvm-project/mlir/unittests/IR/OperationSupportTest.cpp
@@ -315,6 +315,7 @@ TEST(OperandStorageTest, PopulateDefaultAttrs) {
 TEST(OperationEquivalenceTest, HashWorksWithFlags) {
   MLIRContext context;
   context.getOrLoadDialect<test::TestDialect>();
+  OpBuilder b(&context);
 
   auto *op1 = createOp(&context);
   // `op1` has an unknown loc.
@@ -325,12 +326,36 @@ TEST(OperationEquivalenceTest, HashWorksWithFlags) {
         op, OperationEquivalence::ignoreHashValue,
         OperationEquivalence::ignoreHashValue, flags);
   };
+  // Check ignore location.
   EXPECT_EQ(getHash(op1, OperationEquivalence::IgnoreLocations),
             getHash(op2, OperationEquivalence::IgnoreLocations));
   EXPECT_NE(getHash(op1, OperationEquivalence::None),
             getHash(op2, OperationEquivalence::None));
+  op1->setLoc(NameLoc::get(StringAttr::get(&context, "foo")));
+  // Check ignore discardable dictionary attributes.
+  SmallVector<NamedAttribute> newAttrs = {
+      b.getNamedAttr("foo", b.getStringAttr("f"))};
+  op1->setAttrs(newAttrs);
+  EXPECT_EQ(getHash(op1, OperationEquivalence::IgnoreDiscardableAttrs),
+            getHash(op2, OperationEquivalence::IgnoreDiscardableAttrs));
+  EXPECT_NE(getHash(op1, OperationEquivalence::None),
+            getHash(op2, OperationEquivalence::None));
   op1->destroy();
   op2->destroy();
+
+  // Check ignore properties.
+  auto req1 = b.getI32IntegerAttr(10);
+  Operation *opWithProperty1 = b.create<test::OpAttrMatch1>(
+      b.getUnknownLoc(), req1, nullptr, nullptr, req1);
+  auto req2 = b.getI32IntegerAttr(60);
+  Operation *opWithProperty2 = b.create<test::OpAttrMatch1>(
+      b.getUnknownLoc(), req2, nullptr, nullptr, req2);
+  EXPECT_EQ(getHash(opWithProperty1, OperationEquivalence::IgnoreProperties),
+            getHash(opWithProperty2, OperationEquivalence::IgnoreProperties));
+  EXPECT_NE(getHash(opWithProperty1, OperationEquivalence::None),
+            getHash(opWithProperty2, OperationEquivalence::None));
+  opWithProperty1->destroy();
+  opWithProperty2->destroy();
 }
 
 } // namespace
diff --git a/external/llvm-project/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp b/external/llvm-project/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp
index 314f25833f38..3c880edee4ff 100644
--- a/external/llvm-project/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp
+++ b/external/llvm-project/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp
@@ -239,4 +239,4 @@ TEST_F(MLIRTargetLLVM,
   ASSERT_TRUE(serializedBinary != std::nullopt);
   ASSERT_TRUE(!serializedBinary->empty());
   ASSERT_TRUE(!optimizedLLVMIR.empty());
-}
\ No newline at end of file
+}
diff --git a/external/llvm-project/mlir/utils/generate-test-checks.py b/external/llvm-project/mlir/utils/generate-test-checks.py
index 11fb4e40072e..14a790e6d0e6 100755
--- a/external/llvm-project/mlir/utils/generate-test-checks.py
+++ b/external/llvm-project/mlir/utils/generate-test-checks.py
@@ -208,7 +208,7 @@ def process_source_lines(source_lines, note, args):
     source_segments = [[]]
     for line in source_lines:
         # Remove previous note.
-        if line == note:
+        if line in note:
             continue
         # Remove previous CHECK lines.
         if line.find(args.check_prefix) != -1:
@@ -220,12 +220,19 @@ def process_source_lines(source_lines, note, args):
         source_segments[-1].append(line + "\n")
     return source_segments
 
-def process_attribute_definition(line, attribute_namer, output):
+
+def process_attribute_definition(line, attribute_namer):
     m = ATTR_DEF_RE.match(line)
     if m:
         attribute_name = attribute_namer.generate_name(m.group(1))
-        line = '// CHECK: #[[' + attribute_name + ':.+]] =' + line[len(m.group(0)):] + '\n'
-        output.write(line)
+        return (
+            "// CHECK: #[["
+            + attribute_name
+            + ":.+]] ="
+            + line[len(m.group(0)) :]
+            + "\n"
+        )
+    return None
 
 def process_attribute_references(line, attribute_namer):
 
@@ -340,6 +347,9 @@ def main():
     variable_namer = VariableNamer(args.variable_names)
     attribute_namer = AttributeNamer(args.attribute_names)
 
+    # Store attribute definitions to emit at appropriate scope
+    pending_attr_defs = []
+
     # Process lines
     for input_line in input_lines:
         if not input_line:
@@ -350,8 +360,9 @@ def main():
         if input_line.startswith("// -----"):
             continue
 
-        # Check if this is an attribute definition and process it
-        process_attribute_definition(input_line, attribute_namer, output)
+        if ATTR_DEF_RE.match(input_line):
+            pending_attr_defs.append(input_line)
+            continue
 
         # Lines with blocks begin with a ^. These lines have a trailing comment
         # that needs to be stripped.
@@ -407,6 +418,13 @@ def main():
             output_line += process_line(ssa_split[1:], variable_namer)
 
         else:
+            # Emit any pending attribute definitions at the start of this scope
+            for attr in pending_attr_defs:
+                attr_line = process_attribute_definition(attr, attribute_namer)
+                if attr_line:
+                    output_segments[-1].append(attr_line)
+            pending_attr_defs.clear()
+
             # Output the first line chunk that does not contain an SSA name for the
             # label.
             output_line = "// " + args.check_prefix + "-LABEL: " + ssa_split[0] + "\n"
diff --git a/external/llvm-project/offload/CMakeLists.txt b/external/llvm-project/offload/CMakeLists.txt
index c13904f8301b..aaf4658601fe 100644
--- a/external/llvm-project/offload/CMakeLists.txt
+++ b/external/llvm-project/offload/CMakeLists.txt
@@ -57,6 +57,8 @@ endif()
 
 set(LLVM_COMMON_CMAKE_UTILS ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
 
+option(OFFLOAD_INCLUDE_TESTS "Generate and build offload tests." ${LLVM_INCLUDE_TESTS})
+
 # Add path for custom modules
 list(INSERT CMAKE_MODULE_PATH 0
   "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
@@ -445,6 +447,8 @@ if(OFFLOAD_ENABLE_EMISSARY_APIS)
     DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH})
 endif()
 
+add_subdirectory(tools/offload-tblgen)
+
 # Build offloading plugins and device RTLs if they are available.
 add_subdirectory(plugins-nextgen)
 
@@ -455,19 +459,20 @@ add_subdirectory(DeviceRTL)
 add_subdirectory(libomptarget)
 
 # FIXME: Re-enable once OMPT design allows
-# add_subdirectory(tools/offload-tblgen)
 # add_subdirectory(liboffload)
 
 # Add tests.
-add_subdirectory(test)
+if(OFFLOAD_INCLUDE_TESTS)
+  add_subdirectory(test)
 
-# Add unit tests if GMock/GTest is present
-if(NOT LLVM_THIRD_PARTY_DIR)
-  set(LLVM_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../third-party")
-endif()
-if(EXISTS ${LLVM_THIRD_PARTY_DIR}/unittest AND NOT TARGET llvm_gtest)
-  add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest)
-endif()
-if(TARGET llvm_gtest)
-  add_subdirectory(unittests)
+  # Add unit tests if GMock/GTest is present
+  if(NOT LLVM_THIRD_PARTY_DIR)
+    set(LLVM_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../third-party")
+  endif()
+  if(EXISTS ${LLVM_THIRD_PARTY_DIR}/unittest AND NOT TARGET llvm_gtest)
+    add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest)
+  endif()
+  if(TARGET llvm_gtest)
+    add_subdirectory(unittests)
+  endif()
 endif()
diff --git a/external/llvm-project/offload/include/Shared/OffloadErrcodes.inc b/external/llvm-project/offload/include/Shared/OffloadErrcodes.inc
deleted file mode 100644
index 130e553aa70a..000000000000
--- a/external/llvm-project/offload/include/Shared/OffloadErrcodes.inc
+++ /dev/null
@@ -1,51 +0,0 @@
-//===- Auto-generated file, part of the LLVM/Offload project --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef OFFLOAD_ERRC
-#error Please define the macro OFFLOAD_ERRCODE(Name, Desc, Value)
-#endif
-
-// Error codes are shared between PluginInterface and liboffload.
-// To add new error codes, add them to offload/liboffload/API/Common.td and run
-// the GenerateOffload target.
-
-OFFLOAD_ERRC(SUCCESS, "success", 0)
-OFFLOAD_ERRC(UNKNOWN, "unknown or internal error", 1)
-OFFLOAD_ERRC(HOST_IO, "I/O error on host", 2)
-OFFLOAD_ERRC(INVALID_BINARY, "a provided binary image is malformed", 3)
-OFFLOAD_ERRC(INVALID_NULL_POINTER,
-             "a pointer argument is null when it should not be", 4)
-OFFLOAD_ERRC(INVALID_ARGUMENT, "an argument is invalid", 5)
-OFFLOAD_ERRC(NOT_FOUND, "requested object was not found in the binary image", 6)
-OFFLOAD_ERRC(OUT_OF_RESOURCES, "out of resources", 7)
-OFFLOAD_ERRC(
-    INVALID_SIZE,
-    "invalid size or dimensions (e.g., must not be zero, or is out of bounds)",
-    8)
-OFFLOAD_ERRC(INVALID_ENUMERATION, "enumerator argument is not valid", 9)
-OFFLOAD_ERRC(HOST_TOOL_NOT_FOUND,
-             "a required binary (linker, etc.) was not found on the host", 10)
-OFFLOAD_ERRC(INVALID_VALUE, "invalid value", 11)
-OFFLOAD_ERRC(UNIMPLEMENTED,
-             "generic error code for features currently unimplemented by the "
-             "device/backend",
-             12)
-OFFLOAD_ERRC(
-    UNSUPPORTED,
-    "generic error code for features unsupported by the device/backend", 13)
-OFFLOAD_ERRC(ASSEMBLE_FAILURE,
-             "assembler failure while processing binary image", 14)
-OFFLOAD_ERRC(LINK_FAILURE, "linker failure while processing binary image", 15)
-OFFLOAD_ERRC(BACKEND_FAILURE,
-             "the plugin backend is in an invalid or unsupported state", 16)
-OFFLOAD_ERRC(INVALID_NULL_HANDLE,
-             "a handle argument is null when it should not be", 17)
-OFFLOAD_ERRC(INVALID_PLATFORM, "invalid platform", 18)
-OFFLOAD_ERRC(INVALID_DEVICE, "invalid device", 19)
-OFFLOAD_ERRC(INVALID_QUEUE, "invalid queue", 20)
-OFFLOAD_ERRC(INVALID_EVENT, "invalid event", 21)
diff --git a/external/llvm-project/offload/liboffload/API/CMakeLists.txt b/external/llvm-project/offload/liboffload/API/CMakeLists.txt
index 5f8d1435d141..e4baa4772a1e 100644
--- a/external/llvm-project/offload/liboffload/API/CMakeLists.txt
+++ b/external/llvm-project/offload/liboffload/API/CMakeLists.txt
@@ -1,29 +1,49 @@
-# The OffloadGenerate target is used to regenerate the generated files in the
-# include directory. These files are checked in with the rest of the source,
-# therefore it is only needed when making changes to the API.
+# We want to clang-format the generated files if possible, since OffloadAPI.h is
+# the main public header for liboffload. Generate them in a temporary location,
+# then clang-format and copy them to the proper location. If clang-format is
+# missing just copy them.
+# Ideally we'd just clang-format them in place and avoid the copy but cmake
+# gets confused about the same path being a byproduct of two custom commands.
 
-find_program(CLANG_FORMAT clang-format PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
-if (CLANG_FORMAT)
-    set(LLVM_TARGET_DEFINITIONS ${CMAKE_CURRENT_SOURCE_DIR}/OffloadAPI.td)
+set(LLVM_TARGET_DEFINITIONS ${CMAKE_CURRENT_SOURCE_DIR}/OffloadAPI.td)
+set(files_to_copy "")
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/generated)
 
-    tablegen(OFFLOAD OffloadAPI.h -gen-api)
-    tablegen(OFFLOAD OffloadEntryPoints.inc -gen-entry-points)
-    tablegen(OFFLOAD OffloadFuncs.inc -gen-func-names)
-    tablegen(OFFLOAD OffloadImplFuncDecls.inc -gen-impl-func-decls)
-    tablegen(OFFLOAD OffloadPrint.hpp -gen-print-header)
-    tablegen(OFFLOAD OffloadErrcodes.inc -gen-errcodes)
+macro(offload_tablegen file)
+    tablegen(OFFLOAD generated/${file}.gen ${ARGN})
+    list(APPEND files_to_copy ${file})
+endmacro()
 
-    set(FILES_TO_COPY "OffloadAPI.h;OffloadEntryPoints.inc;OffloadFuncs.inc;OffloadImplFuncDecls.inc;OffloadPrint.hpp")
-    set(GEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include/generated)
-    add_public_tablegen_target(OffloadGenerate)
-    add_custom_command(TARGET OffloadGenerate POST_BUILD COMMAND ${CLANG_FORMAT}
-        -i ${TABLEGEN_OUTPUT})
-    add_custom_command(TARGET OffloadGenerate POST_BUILD COMMAND ${CMAKE_COMMAND}
-        -E copy_if_different ${FILES_TO_COPY} ${GEN_DIR})
-    add_custom_command(TARGET OffloadGenerate POST_BUILD COMMAND ${CMAKE_COMMAND}
-        -E copy_if_different OffloadErrcodes.inc "${LIBOMPTARGET_INCLUDE_DIR}/Shared/OffloadErrcodes.inc")
+offload_tablegen(OffloadAPI.h -gen-api)
+offload_tablegen(OffloadEntryPoints.inc -gen-entry-points)
+offload_tablegen(OffloadFuncs.inc -gen-func-names)
+offload_tablegen(OffloadImplFuncDecls.inc -gen-impl-func-decls)
+offload_tablegen(OffloadPrint.hpp -gen-print-header)
+
+add_public_tablegen_target(OffloadGenerate)
+
+add_custom_target(OffloadAPI DEPENDS OffloadGenerate)
+find_program(clang_format clang-format PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
+if (clang_format)
+    foreach(file IN LISTS files_to_copy)
+      add_custom_command(
+          OUTPUT ${file}
+          COMMAND ${clang_format} -i generated/${file}.gen
+          COMMAND ${CMAKE_COMMAND} -E copy_if_different generated/${file}.gen ${CMAKE_CURRENT_BINARY_DIR}/${file}
+          DEPENDS generated/${file}.gen
+      )
+      add_custom_target(OffloadAPI.${file} DEPENDS ${file})
+      add_dependencies(OffloadAPI OffloadAPI.${file})
+    endforeach()
 else()
-    message(WARNING "clang-format was not found, so the OffloadGenerate target\
-        will not be available. Offload will still build, but you will not be\
-        able to make changes to the API.")
+    message(WARNING "clang-format not found, the generated Offload API headers will not be formatted")
+    foreach(file IN LISTS files_to_copy)
+        add_custom_command(
+            OUTPUT ${file}
+            COMMAND ${CMAKE_COMMAND} -E copy_if_different generated/${file}.gen ${CMAKE_CURRENT_BINARY_DIR}/${file}
+            DEPENDS generated/${file}.gen
+        )
+        add_custom_target(OffloadAPI.${file} DEPENDS ${file})
+        add_dependencies(OffloadAPI OffloadAPI.${file})
+    endforeach()
 endif()
diff --git a/external/llvm-project/offload/liboffload/API/Common.td b/external/llvm-project/offload/liboffload/API/Common.td
index 7674da0438c2..8a2ecd6c6e8f 100644
--- a/external/llvm-project/offload/liboffload/API/Common.td
+++ b/external/llvm-project/offload/liboffload/API/Common.td
@@ -148,6 +148,16 @@ def : Struct {
   ];
 }
 
+def : Struct {
+  let name = "ol_dimensions_t";
+  let desc = "A three element vector";
+  let members = [
+    StructMember<"uint32_t", "x", "X">,
+    StructMember<"uint32_t", "y", "Y">,
+    StructMember<"uint32_t", "z", "Z">,
+  ];
+}
+
 def : Function {
   let name = "olInit";
   let desc = "Perform initialization of the Offload library and plugins";
diff --git a/external/llvm-project/offload/liboffload/API/Kernel.td b/external/llvm-project/offload/liboffload/API/Kernel.td
index 247f9c1bf5b6..0913a036fa04 100644
--- a/external/llvm-project/offload/liboffload/API/Kernel.td
+++ b/external/llvm-project/offload/liboffload/API/Kernel.td
@@ -29,12 +29,8 @@ def : Struct {
     let desc = "Size-related arguments for a kernel launch.";
     let members = [
         StructMember<"size_t", "Dimensions", "Number of work dimensions">,
-        StructMember<"size_t", "NumGroupsX", "Number of work groups on the X dimension">,
-        StructMember<"size_t", "NumGroupsY", "Number of work groups on the Y dimension">,
-        StructMember<"size_t", "NumGroupsZ", "Number of work groups on the Z dimension">,
-        StructMember<"size_t", "GroupSizeX", "Size of a work group on the X dimension.">,
-        StructMember<"size_t", "GroupSizeY", "Size of a work group on the Y dimension.">,
-        StructMember<"size_t", "GroupSizeZ", "Size of a work group on the Z dimension.">,
+        StructMember<"struct ol_dimensions_t", "NumGroups", "Number of work groups in each dimension">,
+        StructMember<"struct ol_dimensions_t", "GroupSize", "Size of a work group in each dimension">,
         StructMember<"size_t", "DynSharedMemory", "Size of dynamic shared memory in bytes.">
     ];
 }
@@ -43,19 +39,21 @@ def : Function {
     let name = "olLaunchKernel";
     let desc = "Enqueue a kernel launch with the specified size and parameters.";
     let details = [
-        "If a queue is not specified, kernel execution happens synchronously"
+        "If a queue is not specified, kernel execution happens synchronously",
+        "ArgumentsData may be set to NULL (to indicate no parameters)"
     ];
     let params = [
         Param<"ol_queue_handle_t", "Queue", "handle of the queue", PARAM_IN_OPTIONAL>,
         Param<"ol_device_handle_t", "Device", "handle of the device to execute on", PARAM_IN>,
         Param<"ol_kernel_handle_t", "Kernel", "handle of the kernel", PARAM_IN>,
-        Param<"const void*", "ArgumentsData", "pointer to the kernel argument struct", PARAM_IN>,
+        Param<"const void*", "ArgumentsData", "pointer to the kernel argument struct", PARAM_IN_OPTIONAL>,
         Param<"size_t", "ArgumentsSize", "size of the kernel argument struct", PARAM_IN>,
         Param<"const ol_kernel_launch_size_args_t*", "LaunchSizeArgs", "pointer to the struct containing launch size parameters", PARAM_IN>,
         Param<"ol_event_handle_t*", "EventOut", "optional recorded event for the enqueued operation", PARAM_OUT_OPTIONAL>
     ];
     let returns = [
         Return<"OL_ERRC_INVALID_ARGUMENT", ["`Queue == NULL && EventOut != NULL`"]>,
+        Return<"OL_ERRC_INVALID_ARGUMENT", ["`ArgumentsSize > 0 && ArgumentsData == NULL`"]>,
         Return<"OL_ERRC_INVALID_DEVICE", ["If Queue is non-null but does not belong to Device"]>,
     ];
 }
diff --git a/external/llvm-project/offload/liboffload/API/Memory.td b/external/llvm-project/offload/liboffload/API/Memory.td
index 9cd1ef6362e1..029975c44829 100644
--- a/external/llvm-project/offload/liboffload/API/Memory.td
+++ b/external/llvm-project/offload/liboffload/API/Memory.td
@@ -57,7 +57,7 @@ def : Function {
         Param<"ol_queue_handle_t", "Queue", "handle of the queue.", PARAM_IN_OPTIONAL>,
         Param<"void*", "DstPtr", "pointer to copy to", PARAM_IN>,
         Param<"ol_device_handle_t", "DstDevice", "device that DstPtr belongs to", PARAM_IN>,
-        Param<"void*", "SrcPtr", "pointer to copy from", PARAM_IN>,
+        Param<"const void*", "SrcPtr", "pointer to copy from", PARAM_IN>,
         Param<"ol_device_handle_t", "SrcDevice", "device that SrcPtr belongs to", PARAM_IN>,
         Param<"size_t", "Size", "size in bytes of data to copy", PARAM_IN>,
         Param<"ol_event_handle_t*", "EventOut", "optional recorded event for the enqueued operation", PARAM_OUT_OPTIONAL>
diff --git a/external/llvm-project/offload/liboffload/CMakeLists.txt b/external/llvm-project/offload/liboffload/CMakeLists.txt
index db12236ddfc7..efb800f2495f 100644
--- a/external/llvm-project/offload/liboffload/CMakeLists.txt
+++ b/external/llvm-project/offload/liboffload/CMakeLists.txt
@@ -8,6 +8,10 @@ add_llvm_library(
   LINK_COMPONENTS
   FrontendOpenMP
   Support
+
+  DEPENDS
+  OffloadAPI
+  PluginErrcodes
   )
 
 foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
@@ -19,11 +23,13 @@ if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
 endif()
 
 target_include_directories(LLVMOffload PUBLIC
+                            ${CMAKE_CURRENT_BINARY_DIR}/API
                             ${CMAKE_CURRENT_BINARY_DIR}/../include
                             ${CMAKE_CURRENT_SOURCE_DIR}/include
-                            ${CMAKE_CURRENT_SOURCE_DIR}/include/generated
                             ${CMAKE_CURRENT_SOURCE_DIR}/../include
-                            ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include)
+                            ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include
+                            ${CMAKE_CURRENT_BINARY_DIR}/../plugins-nextgen/common/include
+                            )
 
 target_compile_options(LLVMOffload PRIVATE ${offload_compile_flags})
 target_link_options(LLVMOffload PRIVATE ${offload_link_flags})
@@ -39,5 +45,5 @@ set_target_properties(LLVMOffload PROPERTIES
                       BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
 install(TARGETS LLVMOffload LIBRARY COMPONENT LLVMOffload DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
 
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/generated/OffloadAPI.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload)
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/generated/OffloadPrint.hpp DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/API/OffloadAPI.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/API/OffloadPrint.hpp DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload)
diff --git a/external/llvm-project/offload/liboffload/include/generated/OffloadAPI.h b/external/llvm-project/offload/liboffload/include/generated/OffloadAPI.h
deleted file mode 100644
index a1d7540519e3..000000000000
--- a/external/llvm-project/offload/liboffload/include/generated/OffloadAPI.h
+++ /dev/null
@@ -1,1021 +0,0 @@
-//===- Auto-generated file, part of the LLVM/Offload project --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// Auto-generated file, do not manually edit.
-
-#pragma once
-
-#include <stddef.h>
-#include <stdint.h>
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Defines Return/Error codes
-typedef enum ol_errc_t {
-  /// success
-  OL_ERRC_SUCCESS = 0,
-  /// unknown or internal error
-  OL_ERRC_UNKNOWN = 1,
-  /// I/O error on host
-  OL_ERRC_HOST_IO = 2,
-  /// a provided binary image is malformed
-  OL_ERRC_INVALID_BINARY = 3,
-  /// a pointer argument is null when it should not be
-  OL_ERRC_INVALID_NULL_POINTER = 4,
-  /// an argument is invalid
-  OL_ERRC_INVALID_ARGUMENT = 5,
-  /// requested object was not found in the binary image
-  OL_ERRC_NOT_FOUND = 6,
-  /// out of resources
-  OL_ERRC_OUT_OF_RESOURCES = 7,
-  /// invalid size or dimensions (e.g., must not be zero, or is out of bounds)
-  OL_ERRC_INVALID_SIZE = 8,
-  /// enumerator argument is not valid
-  OL_ERRC_INVALID_ENUMERATION = 9,
-  /// a required binary (linker, etc.) was not found on the host
-  OL_ERRC_HOST_TOOL_NOT_FOUND = 10,
-  /// invalid value
-  OL_ERRC_INVALID_VALUE = 11,
-  /// generic error code for features currently unimplemented by the
-  /// device/backend
-  OL_ERRC_UNIMPLEMENTED = 12,
-  /// generic error code for features unsupported by the device/backend
-  OL_ERRC_UNSUPPORTED = 13,
-  /// assembler failure while processing binary image
-  OL_ERRC_ASSEMBLE_FAILURE = 14,
-  /// linker failure while processing binary image
-  OL_ERRC_LINK_FAILURE = 15,
-  /// the plugin backend is in an invalid or unsupported state
-  OL_ERRC_BACKEND_FAILURE = 16,
-  /// a handle argument is null when it should not be
-  OL_ERRC_INVALID_NULL_HANDLE = 17,
-  /// invalid platform
-  OL_ERRC_INVALID_PLATFORM = 18,
-  /// invalid device
-  OL_ERRC_INVALID_DEVICE = 19,
-  /// invalid queue
-  OL_ERRC_INVALID_QUEUE = 20,
-  /// invalid event
-  OL_ERRC_INVALID_EVENT = 21,
-  /// @cond
-  OL_ERRC_FORCE_UINT32 = 0x7fffffff
-  /// @endcond
-
-} ol_errc_t;
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_VERSION_MAJOR
-/// @brief Major version of the Offload API
-#define OL_VERSION_MAJOR 0
-#endif // OL_VERSION_MAJOR
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_VERSION_MINOR
-/// @brief Minor version of the Offload API
-#define OL_VERSION_MINOR 0
-#endif // OL_VERSION_MINOR
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_VERSION_PATCH
-/// @brief Patch version of the Offload API
-#define OL_VERSION_PATCH 1
-#endif // OL_VERSION_PATCH
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_APICALL
-#if defined(_WIN32)
-/// @brief Calling convention for all API functions
-#define OL_APICALL __cdecl
-#else
-#define OL_APICALL
-#endif // defined(_WIN32)
-#endif // OL_APICALL
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_APIEXPORT
-#if defined(_WIN32)
-/// @brief Microsoft-specific dllexport storage-class attribute
-#define OL_APIEXPORT __declspec(dllexport)
-#else
-#define OL_APIEXPORT
-#endif // defined(_WIN32)
-#endif // OL_APIEXPORT
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_DLLEXPORT
-#if defined(_WIN32)
-/// @brief Microsoft-specific dllexport storage-class attribute
-#define OL_DLLEXPORT __declspec(dllexport)
-#endif // defined(_WIN32)
-#endif // OL_DLLEXPORT
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_DLLEXPORT
-#if __GNUC__ >= 4
-/// @brief GCC-specific dllexport storage-class attribute
-#define OL_DLLEXPORT __attribute__((visibility("default")))
-#else
-#define OL_DLLEXPORT
-#endif // __GNUC__ >= 4
-#endif // OL_DLLEXPORT
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Handle of a platform instance
-typedef struct ol_platform_impl_t *ol_platform_handle_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Handle of platform's device object
-typedef struct ol_device_impl_t *ol_device_handle_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Handle of context object
-typedef struct ol_context_impl_t *ol_context_handle_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Handle of queue object
-typedef struct ol_queue_impl_t *ol_queue_handle_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Handle of event object
-typedef struct ol_event_impl_t *ol_event_handle_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Handle of program object
-typedef struct ol_program_impl_t *ol_program_handle_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Handle of kernel object
-typedef void *ol_kernel_handle_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Details of the error condition returned by an API call
-typedef struct ol_error_struct_t {
-  ol_errc_t Code;      /// The error code
-  const char *Details; /// String containing error details
-} ol_error_struct_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Result type returned by all entry points.
-typedef const ol_error_struct_t *ol_result_t;
-
-///////////////////////////////////////////////////////////////////////////////
-#ifndef OL_SUCCESS
-/// @brief Success condition
-#define OL_SUCCESS NULL
-#endif // OL_SUCCESS
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Code location information that can optionally be associated with an
-/// API call
-typedef struct ol_code_location_t {
-  const char *FunctionName; /// Function name
-  const char *SourceFile;   /// Source code file
-  uint32_t LineNumber;      /// Source code line number
-  uint32_t ColumnNumber;    /// Source code column number
-} ol_code_location_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Perform initialization of the Offload library and plugins
-///
-/// @details
-///    - This must be the first API call made by a user of the Offload library
-///    - Each call will increment an internal reference count that is
-///    decremented by `olShutDown`
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olInit();
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Release the resources in use by Offload
-///
-/// @details
-///    - This decrements an internal reference count. When this reaches 0, all
-///    resources will be released
-///    - Subsequent API calls made after this are not valid
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olShutDown();
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Supported platform info.
-typedef enum ol_platform_info_t {
-  /// [char[]] The string denoting name of the platform. The size of the info
-  /// needs to be dynamically queried.
-  OL_PLATFORM_INFO_NAME = 0,
-  /// [char[]] The string denoting name of the vendor of the platform. The size
-  /// of the info needs to be dynamically queried.
-  OL_PLATFORM_INFO_VENDOR_NAME = 1,
-  /// [char[]] The string denoting the version of the platform. The size of the
-  /// info needs to be dynamically queried.
-  OL_PLATFORM_INFO_VERSION = 2,
-  /// [ol_platform_backend_t] The native backend of the platform.
-  OL_PLATFORM_INFO_BACKEND = 3,
-  /// @cond
-  OL_PLATFORM_INFO_FORCE_UINT32 = 0x7fffffff
-  /// @endcond
-
-} ol_platform_info_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Identifies the native backend of the platform.
-typedef enum ol_platform_backend_t {
-  /// The backend is not recognized
-  OL_PLATFORM_BACKEND_UNKNOWN = 0,
-  /// The backend is CUDA
-  OL_PLATFORM_BACKEND_CUDA = 1,
-  /// The backend is AMDGPU
-  OL_PLATFORM_BACKEND_AMDGPU = 2,
-  /// The backend is the host
-  OL_PLATFORM_BACKEND_HOST = 3,
-  /// @cond
-  OL_PLATFORM_BACKEND_FORCE_UINT32 = 0x7fffffff
-  /// @endcond
-
-} ol_platform_backend_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Queries the given property of the platform.
-///
-/// @details
-///    - `olGetPlatformInfoSize` can be used to query the storage size required
-///    for the given query.
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
-///         + If `PropName` is not supported by the platform.
-///     - ::OL_ERRC_INVALID_SIZE
-///         + `PropSize == 0`
-///         + If `PropSize` is less than the real number of bytes needed to
-///         return the info.
-///     - ::OL_ERRC_INVALID_PLATFORM
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Platform`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == PropValue`
-OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfo(
-    // [in] handle of the platform
-    ol_platform_handle_t Platform,
-    // [in] type of the info to retrieve
-    ol_platform_info_t PropName,
-    // [in] the number of bytes pointed to by pPlatformInfo.
-    size_t PropSize,
-    // [out] array of bytes holding the info. If Size is not equal to or greater
-    // to the real number of bytes needed to return the info then the
-    // OL_ERRC_INVALID_SIZE error is returned and pPlatformInfo is not used.
-    void *PropValue);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Returns the storage size of the given platform query.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
-///         + If `PropName` is not supported by the platform.
-///     - ::OL_ERRC_INVALID_PLATFORM
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Platform`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == PropSizeRet`
-OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfoSize(
-    // [in] handle of the platform
-    ol_platform_handle_t Platform,
-    // [in] type of the info to query
-    ol_platform_info_t PropName,
-    // [out] pointer to the number of bytes required to store the query
-    size_t *PropSizeRet);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Supported device types.
-typedef enum ol_device_type_t {
-  /// The default device type as preferred by the runtime
-  OL_DEVICE_TYPE_DEFAULT = 0,
-  /// Devices of all types
-  OL_DEVICE_TYPE_ALL = 1,
-  /// GPU device type
-  OL_DEVICE_TYPE_GPU = 2,
-  /// CPU device type
-  OL_DEVICE_TYPE_CPU = 3,
-  /// Host device type
-  OL_DEVICE_TYPE_HOST = 4,
-  /// @cond
-  OL_DEVICE_TYPE_FORCE_UINT32 = 0x7fffffff
-  /// @endcond
-
-} ol_device_type_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Supported device info.
-typedef enum ol_device_info_t {
-  /// [ol_device_type_t] type of the device
-  OL_DEVICE_INFO_TYPE = 0,
-  /// [ol_platform_handle_t] the platform associated with the device
-  OL_DEVICE_INFO_PLATFORM = 1,
-  /// [char[]] Device name
-  OL_DEVICE_INFO_NAME = 2,
-  /// [char[]] Device vendor
-  OL_DEVICE_INFO_VENDOR = 3,
-  /// [char[]] Driver version
-  OL_DEVICE_INFO_DRIVER_VERSION = 4,
-  /// @cond
-  OL_DEVICE_INFO_FORCE_UINT32 = 0x7fffffff
-  /// @endcond
-
-} ol_device_info_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief User-provided function to be used with `olIterateDevices`
-typedef bool (*ol_device_iterate_cb_t)(
-    // the device handle of the current iteration
-    ol_device_handle_t Device,
-    // optional user data
-    void *UserData);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Iterates over all available devices, calling the callback for each
-/// device.
-///
-/// @details
-///    - If the user-provided callback returns `false`, the iteration is
-///    stopped.
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_DEVICE
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olIterateDevices(
-    // [in] User-provided function called for each available device
-    ol_device_iterate_cb_t Callback,
-    // [in][optional] Optional user data to pass to the callback
-    void *UserData);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Queries the given property of the device.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
-///         + If `PropName` is not supported by the device.
-///     - ::OL_ERRC_INVALID_SIZE
-///         + `PropSize == 0`
-///         + If `PropSize` is less than the real number of bytes needed to
-///         return the info.
-///     - ::OL_ERRC_INVALID_DEVICE
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Device`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == PropValue`
-OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfo(
-    // [in] handle of the device instance
-    ol_device_handle_t Device,
-    // [in] type of the info to retrieve
-    ol_device_info_t PropName,
-    // [in] the number of bytes pointed to by PropValue.
-    size_t PropSize,
-    // [out] array of bytes holding the info. If PropSize is not equal to or
-    // greater than the real number of bytes needed to return the info then the
-    // OL_ERRC_INVALID_SIZE error is returned and PropValue is not used.
-    void *PropValue);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Returns the storage size of the given device query.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_UNSUPPORTED_ENUMERATION
-///         + If `PropName` is not supported by the device.
-///     - ::OL_ERRC_INVALID_DEVICE
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Device`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == PropSizeRet`
-OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoSize(
-    // [in] handle of the device instance
-    ol_device_handle_t Device,
-    // [in] type of the info to retrieve
-    ol_device_info_t PropName,
-    // [out] pointer to the number of bytes required to store the query
-    size_t *PropSizeRet);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Represents the type of allocation made with olMemAlloc.
-typedef enum ol_alloc_type_t {
-  /// Host allocation
-  OL_ALLOC_TYPE_HOST = 0,
-  /// Device allocation
-  OL_ALLOC_TYPE_DEVICE = 1,
-  /// Managed allocation
-  OL_ALLOC_TYPE_MANAGED = 2,
-  /// @cond
-  OL_ALLOC_TYPE_FORCE_UINT32 = 0x7fffffff
-  /// @endcond
-
-} ol_alloc_type_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Creates a memory allocation on the specified device.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_SIZE
-///         + `Size == 0`
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Device`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == AllocationOut`
-OL_APIEXPORT ol_result_t OL_APICALL olMemAlloc(
-    // [in] handle of the device to allocate on
-    ol_device_handle_t Device,
-    // [in] type of the allocation
-    ol_alloc_type_t Type,
-    // [in] size of the allocation in bytes
-    size_t Size,
-    // [out] output for the allocated pointer
-    void **AllocationOut);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Frees a memory allocation previously made by olMemAlloc.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == Address`
-OL_APIEXPORT ol_result_t OL_APICALL olMemFree(
-    // [in] address of the allocation to free
-    void *Address);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Enqueue a memcpy operation.
-///
-/// @details
-///    - For host pointers, use the host device belonging to the
-///    OL_PLATFORM_BACKEND_HOST platform.
-///    - If a queue is specified, at least one device must be a non-host device
-///    - If a queue is not specified, the memcpy happens synchronously
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_ARGUMENT
-///         + `Queue == NULL && EventOut != NULL`
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == DstDevice`
-///         + `NULL == SrcDevice`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == DstPtr`
-///         + `NULL == SrcPtr`
-OL_APIEXPORT ol_result_t OL_APICALL olMemcpy(
-    // [in][optional] handle of the queue.
-    ol_queue_handle_t Queue,
-    // [in] pointer to copy to
-    void *DstPtr,
-    // [in] device that DstPtr belongs to
-    ol_device_handle_t DstDevice,
-    // [in] pointer to copy from
-    void *SrcPtr,
-    // [in] device that SrcPtr belongs to
-    ol_device_handle_t SrcDevice,
-    // [in] size in bytes of data to copy
-    size_t Size,
-    // [out][optional] optional recorded event for the enqueued operation
-    ol_event_handle_t *EventOut);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Create a queue for the given device.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Device`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == Queue`
-OL_APIEXPORT ol_result_t OL_APICALL olCreateQueue(
-    // [in] handle of the device
-    ol_device_handle_t Device,
-    // [out] output pointer for the created queue
-    ol_queue_handle_t *Queue);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Destroy the queue and free all underlying resources.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Queue`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyQueue(
-    // [in] handle of the queue
-    ol_queue_handle_t Queue);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Wait for the enqueued work on a queue to complete.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Queue`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olWaitQueue(
-    // [in] handle of the queue
-    ol_queue_handle_t Queue);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Destroy the event and free all underlying resources.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Event`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyEvent(
-    // [in] handle of the event
-    ol_event_handle_t Event);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Wait for the event to be complete.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Event`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olWaitEvent(
-    // [in] handle of the event
-    ol_event_handle_t Event);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Create a program for the device from the binary image pointed to by
-/// `ProgData`.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Device`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == ProgData`
-///         + `NULL == Program`
-OL_APIEXPORT ol_result_t OL_APICALL olCreateProgram(
-    // [in] handle of the device
-    ol_device_handle_t Device,
-    // [in] pointer to the program binary data
-    const void *ProgData,
-    // [in] size of the program binary in bytes
-    size_t ProgDataSize,
-    // [out] output pointer for the created program
-    ol_program_handle_t *Program);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Destroy the program and free all underlying resources.
-///
-/// @details
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Program`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyProgram(
-    // [in] handle of the program
-    ol_program_handle_t Program);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Get a kernel from the function identified by `KernelName` in the
-/// given program.
-///
-/// @details
-///    - The kernel handle returned is owned by the device so does not need to
-///    be destroyed.
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Program`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == KernelName`
-///         + `NULL == Kernel`
-OL_APIEXPORT ol_result_t OL_APICALL olGetKernel(
-    // [in] handle of the program
-    ol_program_handle_t Program,
-    // [in] name of the kernel entry point in the program
-    const char *KernelName,
-    // [out] output pointer for the fetched kernel
-    ol_kernel_handle_t *Kernel);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Size-related arguments for a kernel launch.
-typedef struct ol_kernel_launch_size_args_t {
-  size_t Dimensions;      /// Number of work dimensions
-  size_t NumGroupsX;      /// Number of work groups on the X dimension
-  size_t NumGroupsY;      /// Number of work groups on the Y dimension
-  size_t NumGroupsZ;      /// Number of work groups on the Z dimension
-  size_t GroupSizeX;      /// Size of a work group on the X dimension.
-  size_t GroupSizeY;      /// Size of a work group on the Y dimension.
-  size_t GroupSizeZ;      /// Size of a work group on the Z dimension.
-  size_t DynSharedMemory; /// Size of dynamic shared memory in bytes.
-} ol_kernel_launch_size_args_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Enqueue a kernel launch with the specified size and parameters.
-///
-/// @details
-///    - If a queue is not specified, kernel execution happens synchronously
-///
-/// @returns
-///     - ::OL_RESULT_SUCCESS
-///     - ::OL_ERRC_UNINITIALIZED
-///     - ::OL_ERRC_DEVICE_LOST
-///     - ::OL_ERRC_INVALID_ARGUMENT
-///         + `Queue == NULL && EventOut != NULL`
-///     - ::OL_ERRC_INVALID_DEVICE
-///         + If Queue is non-null but does not belong to Device
-///     - ::OL_ERRC_INVALID_NULL_HANDLE
-///         + `NULL == Device`
-///         + `NULL == Kernel`
-///     - ::OL_ERRC_INVALID_NULL_POINTER
-///         + `NULL == ArgumentsData`
-///         + `NULL == LaunchSizeArgs`
-OL_APIEXPORT ol_result_t OL_APICALL olLaunchKernel(
-    // [in][optional] handle of the queue
-    ol_queue_handle_t Queue,
-    // [in] handle of the device to execute on
-    ol_device_handle_t Device,
-    // [in] handle of the kernel
-    ol_kernel_handle_t Kernel,
-    // [in] pointer to the kernel argument struct
-    const void *ArgumentsData,
-    // [in] size of the kernel argument struct
-    size_t ArgumentsSize,
-    // [in] pointer to the struct containing launch size parameters
-    const ol_kernel_launch_size_args_t *LaunchSizeArgs,
-    // [out][optional] optional recorded event for the enqueued operation
-    ol_event_handle_t *EventOut);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olGetPlatformInfo
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_get_platform_info_params_t {
-  ol_platform_handle_t *pPlatform;
-  ol_platform_info_t *pPropName;
-  size_t *pPropSize;
-  void **pPropValue;
-} ol_get_platform_info_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olGetPlatformInfoSize
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_get_platform_info_size_params_t {
-  ol_platform_handle_t *pPlatform;
-  ol_platform_info_t *pPropName;
-  size_t **pPropSizeRet;
-} ol_get_platform_info_size_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olIterateDevices
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_iterate_devices_params_t {
-  ol_device_iterate_cb_t *pCallback;
-  void **pUserData;
-} ol_iterate_devices_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olGetDeviceInfo
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_get_device_info_params_t {
-  ol_device_handle_t *pDevice;
-  ol_device_info_t *pPropName;
-  size_t *pPropSize;
-  void **pPropValue;
-} ol_get_device_info_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olGetDeviceInfoSize
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_get_device_info_size_params_t {
-  ol_device_handle_t *pDevice;
-  ol_device_info_t *pPropName;
-  size_t **pPropSizeRet;
-} ol_get_device_info_size_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olMemAlloc
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_mem_alloc_params_t {
-  ol_device_handle_t *pDevice;
-  ol_alloc_type_t *pType;
-  size_t *pSize;
-  void ***pAllocationOut;
-} ol_mem_alloc_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olMemFree
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_mem_free_params_t {
-  void **pAddress;
-} ol_mem_free_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olMemcpy
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_memcpy_params_t {
-  ol_queue_handle_t *pQueue;
-  void **pDstPtr;
-  ol_device_handle_t *pDstDevice;
-  void **pSrcPtr;
-  ol_device_handle_t *pSrcDevice;
-  size_t *pSize;
-  ol_event_handle_t **pEventOut;
-} ol_memcpy_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olCreateQueue
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_create_queue_params_t {
-  ol_device_handle_t *pDevice;
-  ol_queue_handle_t **pQueue;
-} ol_create_queue_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olDestroyQueue
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_destroy_queue_params_t {
-  ol_queue_handle_t *pQueue;
-} ol_destroy_queue_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olWaitQueue
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_wait_queue_params_t {
-  ol_queue_handle_t *pQueue;
-} ol_wait_queue_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olDestroyEvent
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_destroy_event_params_t {
-  ol_event_handle_t *pEvent;
-} ol_destroy_event_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olWaitEvent
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_wait_event_params_t {
-  ol_event_handle_t *pEvent;
-} ol_wait_event_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olCreateProgram
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_create_program_params_t {
-  ol_device_handle_t *pDevice;
-  const void **pProgData;
-  size_t *pProgDataSize;
-  ol_program_handle_t **pProgram;
-} ol_create_program_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olDestroyProgram
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_destroy_program_params_t {
-  ol_program_handle_t *pProgram;
-} ol_destroy_program_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olGetKernel
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_get_kernel_params_t {
-  ol_program_handle_t *pProgram;
-  const char **pKernelName;
-  ol_kernel_handle_t **pKernel;
-} ol_get_kernel_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for olLaunchKernel
-/// @details Each entry is a pointer to the parameter passed to the function;
-typedef struct ol_launch_kernel_params_t {
-  ol_queue_handle_t *pQueue;
-  ol_device_handle_t *pDevice;
-  ol_kernel_handle_t *pKernel;
-  const void **pArgumentsData;
-  size_t *pArgumentsSize;
-  const ol_kernel_launch_size_args_t **pLaunchSizeArgs;
-  ol_event_handle_t **pEventOut;
-} ol_launch_kernel_params_t;
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olInit that also sets source code location information
-/// @details See also ::olInit
-OL_APIEXPORT ol_result_t OL_APICALL
-olInitWithCodeLoc(ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olShutDown that also sets source code location information
-/// @details See also ::olShutDown
-OL_APIEXPORT ol_result_t OL_APICALL
-olShutDownWithCodeLoc(ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olGetPlatformInfo that also sets source code location
-/// information
-/// @details See also ::olGetPlatformInfo
-OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfoWithCodeLoc(
-    ol_platform_handle_t Platform, ol_platform_info_t PropName, size_t PropSize,
-    void *PropValue, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olGetPlatformInfoSize that also sets source code location
-/// information
-/// @details See also ::olGetPlatformInfoSize
-OL_APIEXPORT ol_result_t OL_APICALL olGetPlatformInfoSizeWithCodeLoc(
-    ol_platform_handle_t Platform, ol_platform_info_t PropName,
-    size_t *PropSizeRet, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olIterateDevices that also sets source code location
-/// information
-/// @details See also ::olIterateDevices
-OL_APIEXPORT ol_result_t OL_APICALL
-olIterateDevicesWithCodeLoc(ol_device_iterate_cb_t Callback, void *UserData,
-                            ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olGetDeviceInfo that also sets source code location
-/// information
-/// @details See also ::olGetDeviceInfo
-OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoWithCodeLoc(
-    ol_device_handle_t Device, ol_device_info_t PropName, size_t PropSize,
-    void *PropValue, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olGetDeviceInfoSize that also sets source code location
-/// information
-/// @details See also ::olGetDeviceInfoSize
-OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoSizeWithCodeLoc(
-    ol_device_handle_t Device, ol_device_info_t PropName, size_t *PropSizeRet,
-    ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olMemAlloc that also sets source code location information
-/// @details See also ::olMemAlloc
-OL_APIEXPORT ol_result_t OL_APICALL olMemAllocWithCodeLoc(
-    ol_device_handle_t Device, ol_alloc_type_t Type, size_t Size,
-    void **AllocationOut, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olMemFree that also sets source code location information
-/// @details See also ::olMemFree
-OL_APIEXPORT ol_result_t OL_APICALL
-olMemFreeWithCodeLoc(void *Address, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olMemcpy that also sets source code location information
-/// @details See also ::olMemcpy
-OL_APIEXPORT ol_result_t OL_APICALL olMemcpyWithCodeLoc(
-    ol_queue_handle_t Queue, void *DstPtr, ol_device_handle_t DstDevice,
-    void *SrcPtr, ol_device_handle_t SrcDevice, size_t Size,
-    ol_event_handle_t *EventOut, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olCreateQueue that also sets source code location
-/// information
-/// @details See also ::olCreateQueue
-OL_APIEXPORT ol_result_t OL_APICALL
-olCreateQueueWithCodeLoc(ol_device_handle_t Device, ol_queue_handle_t *Queue,
-                         ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olDestroyQueue that also sets source code location
-/// information
-/// @details See also ::olDestroyQueue
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyQueueWithCodeLoc(
-    ol_queue_handle_t Queue, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olWaitQueue that also sets source code location
-/// information
-/// @details See also ::olWaitQueue
-OL_APIEXPORT ol_result_t OL_APICALL olWaitQueueWithCodeLoc(
-    ol_queue_handle_t Queue, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olDestroyEvent that also sets source code location
-/// information
-/// @details See also ::olDestroyEvent
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyEventWithCodeLoc(
-    ol_event_handle_t Event, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olWaitEvent that also sets source code location
-/// information
-/// @details See also ::olWaitEvent
-OL_APIEXPORT ol_result_t OL_APICALL olWaitEventWithCodeLoc(
-    ol_event_handle_t Event, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olCreateProgram that also sets source code location
-/// information
-/// @details See also ::olCreateProgram
-OL_APIEXPORT ol_result_t OL_APICALL olCreateProgramWithCodeLoc(
-    ol_device_handle_t Device, const void *ProgData, size_t ProgDataSize,
-    ol_program_handle_t *Program, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olDestroyProgram that also sets source code location
-/// information
-/// @details See also ::olDestroyProgram
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyProgramWithCodeLoc(
-    ol_program_handle_t Program, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olGetKernel that also sets source code location
-/// information
-/// @details See also ::olGetKernel
-OL_APIEXPORT ol_result_t OL_APICALL olGetKernelWithCodeLoc(
-    ol_program_handle_t Program, const char *KernelName,
-    ol_kernel_handle_t *Kernel, ol_code_location_t *CodeLocation);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Variant of olLaunchKernel that also sets source code location
-/// information
-/// @details See also ::olLaunchKernel
-OL_APIEXPORT ol_result_t OL_APICALL olLaunchKernelWithCodeLoc(
-    ol_queue_handle_t Queue, ol_device_handle_t Device,
-    ol_kernel_handle_t Kernel, const void *ArgumentsData, size_t ArgumentsSize,
-    const ol_kernel_launch_size_args_t *LaunchSizeArgs,
-    ol_event_handle_t *EventOut, ol_code_location_t *CodeLocation);
-
-#if defined(__cplusplus)
-} // extern "C"
-#endif
diff --git a/external/llvm-project/offload/liboffload/include/generated/OffloadEntryPoints.inc b/external/llvm-project/offload/liboffload/include/generated/OffloadEntryPoints.inc
deleted file mode 100644
index 9feebeea09ec..000000000000
--- a/external/llvm-project/offload/liboffload/include/generated/OffloadEntryPoints.inc
+++ /dev/null
@@ -1,903 +0,0 @@
-//===- Auto-generated file, part of the LLVM/Offload project --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olInit_val() {
-  if (offloadConfig().ValidationEnabled) {
-  }
-
-  return llvm::offload::olInit_impl();
-}
-OL_APIEXPORT ol_result_t OL_APICALL olInit() {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olInit";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olInit_val());
-
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "()";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olInitWithCodeLoc(ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olInit();
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olShutDown_val() {
-  if (offloadConfig().ValidationEnabled) {
-  }
-
-  return llvm::offload::olShutDown_impl();
-}
-OL_APIEXPORT ol_result_t OL_APICALL olShutDown() {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olShutDown";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olShutDown_val());
-
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "()";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olShutDownWithCodeLoc(ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olShutDown();
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olGetPlatformInfo_val(ol_platform_handle_t Platform,
-                                  ol_platform_info_t PropName, size_t PropSize,
-                                  void *PropValue) {
-  if (offloadConfig().ValidationEnabled) {
-    if (PropSize == 0) {
-      return createOffloadError(error::ErrorCode::INVALID_SIZE,
-                                "validation failure: PropSize == 0");
-    }
-
-    if (NULL == Platform) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Platform");
-    }
-
-    if (NULL == PropValue) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == PropValue");
-    }
-  }
-
-  return llvm::offload::olGetPlatformInfo_impl(Platform, PropName, PropSize,
-                                               PropValue);
-}
-OL_APIEXPORT ol_result_t OL_APICALL
-olGetPlatformInfo(ol_platform_handle_t Platform, ol_platform_info_t PropName,
-                  size_t PropSize, void *PropValue) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olGetPlatformInfo";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(
-      olGetPlatformInfo_val(Platform, PropName, PropSize, PropValue));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_get_platform_info_params_t Params = {&Platform, &PropName, &PropSize,
-                                            &PropValue};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olGetPlatformInfoWithCodeLoc(ol_platform_handle_t Platform,
-                                         ol_platform_info_t PropName,
-                                         size_t PropSize, void *PropValue,
-                                         ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result =
-      ::olGetPlatformInfo(Platform, PropName, PropSize, PropValue);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olGetPlatformInfoSize_val(ol_platform_handle_t Platform,
-                                      ol_platform_info_t PropName,
-                                      size_t *PropSizeRet) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Platform) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Platform");
-    }
-
-    if (NULL == PropSizeRet) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == PropSizeRet");
-    }
-  }
-
-  return llvm::offload::olGetPlatformInfoSize_impl(Platform, PropName,
-                                                   PropSizeRet);
-}
-OL_APIEXPORT ol_result_t OL_APICALL
-olGetPlatformInfoSize(ol_platform_handle_t Platform,
-                      ol_platform_info_t PropName, size_t *PropSizeRet) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olGetPlatformInfoSize";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(
-      olGetPlatformInfoSize_val(Platform, PropName, PropSizeRet));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_get_platform_info_size_params_t Params = {&Platform, &PropName,
-                                                 &PropSizeRet};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olGetPlatformInfoSizeWithCodeLoc(ol_platform_handle_t Platform,
-                                             ol_platform_info_t PropName,
-                                             size_t *PropSizeRet,
-                                             ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olGetPlatformInfoSize(Platform, PropName, PropSizeRet);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olIterateDevices_val(ol_device_iterate_cb_t Callback,
-                                 void *UserData) {
-  if (offloadConfig().ValidationEnabled) {
-  }
-
-  return llvm::offload::olIterateDevices_impl(Callback, UserData);
-}
-OL_APIEXPORT ol_result_t OL_APICALL
-olIterateDevices(ol_device_iterate_cb_t Callback, void *UserData) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olIterateDevices";
-  }
-
-  ol_result_t Result =
-      llvmErrorToOffloadError(olIterateDevices_val(Callback, UserData));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_iterate_devices_params_t Params = {&Callback, &UserData};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olIterateDevicesWithCodeLoc(ol_device_iterate_cb_t Callback,
-                                        void *UserData,
-                                        ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olIterateDevices(Callback, UserData);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olGetDeviceInfo_val(ol_device_handle_t Device,
-                                ol_device_info_t PropName, size_t PropSize,
-                                void *PropValue) {
-  if (offloadConfig().ValidationEnabled) {
-    if (PropSize == 0) {
-      return createOffloadError(error::ErrorCode::INVALID_SIZE,
-                                "validation failure: PropSize == 0");
-    }
-
-    if (NULL == Device) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Device");
-    }
-
-    if (NULL == PropValue) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == PropValue");
-    }
-  }
-
-  return llvm::offload::olGetDeviceInfo_impl(Device, PropName, PropSize,
-                                             PropValue);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfo(ol_device_handle_t Device,
-                                                    ol_device_info_t PropName,
-                                                    size_t PropSize,
-                                                    void *PropValue) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olGetDeviceInfo";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(
-      olGetDeviceInfo_val(Device, PropName, PropSize, PropValue));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_get_device_info_params_t Params = {&Device, &PropName, &PropSize,
-                                          &PropValue};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olGetDeviceInfoWithCodeLoc(ol_device_handle_t Device,
-                                       ol_device_info_t PropName,
-                                       size_t PropSize, void *PropValue,
-                                       ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olGetDeviceInfo(Device, PropName, PropSize, PropValue);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olGetDeviceInfoSize_val(ol_device_handle_t Device,
-                                    ol_device_info_t PropName,
-                                    size_t *PropSizeRet) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Device) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Device");
-    }
-
-    if (NULL == PropSizeRet) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == PropSizeRet");
-    }
-  }
-
-  return llvm::offload::olGetDeviceInfoSize_impl(Device, PropName, PropSizeRet);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olGetDeviceInfoSize(
-    ol_device_handle_t Device, ol_device_info_t PropName, size_t *PropSizeRet) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olGetDeviceInfoSize";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(
-      olGetDeviceInfoSize_val(Device, PropName, PropSizeRet));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_get_device_info_size_params_t Params = {&Device, &PropName,
-                                               &PropSizeRet};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olGetDeviceInfoSizeWithCodeLoc(ol_device_handle_t Device,
-                                           ol_device_info_t PropName,
-                                           size_t *PropSizeRet,
-                                           ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olGetDeviceInfoSize(Device, PropName, PropSizeRet);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olMemAlloc_val(ol_device_handle_t Device, ol_alloc_type_t Type,
-                           size_t Size, void **AllocationOut) {
-  if (offloadConfig().ValidationEnabled) {
-    if (Size == 0) {
-      return createOffloadError(error::ErrorCode::INVALID_SIZE,
-                                "validation failure: Size == 0");
-    }
-
-    if (NULL == Device) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Device");
-    }
-
-    if (NULL == AllocationOut) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == AllocationOut");
-    }
-  }
-
-  return llvm::offload::olMemAlloc_impl(Device, Type, Size, AllocationOut);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olMemAlloc(ol_device_handle_t Device,
-                                               ol_alloc_type_t Type,
-                                               size_t Size,
-                                               void **AllocationOut) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olMemAlloc";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(
-      olMemAlloc_val(Device, Type, Size, AllocationOut));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_mem_alloc_params_t Params = {&Device, &Type, &Size, &AllocationOut};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olMemAllocWithCodeLoc(ol_device_handle_t Device,
-                                  ol_alloc_type_t Type, size_t Size,
-                                  void **AllocationOut,
-                                  ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olMemAlloc(Device, Type, Size, AllocationOut);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olMemFree_val(void *Address) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Address) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == Address");
-    }
-  }
-
-  return llvm::offload::olMemFree_impl(Address);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olMemFree(void *Address) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olMemFree";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olMemFree_val(Address));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_mem_free_params_t Params = {&Address};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olMemFreeWithCodeLoc(void *Address,
-                                 ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olMemFree(Address);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olMemcpy_val(ol_queue_handle_t Queue, void *DstPtr,
-                         ol_device_handle_t DstDevice, void *SrcPtr,
-                         ol_device_handle_t SrcDevice, size_t Size,
-                         ol_event_handle_t *EventOut) {
-  if (offloadConfig().ValidationEnabled) {
-    if (Queue == NULL && EventOut != NULL) {
-      return createOffloadError(
-          error::ErrorCode::INVALID_ARGUMENT,
-          "validation failure: Queue == NULL && EventOut != NULL");
-    }
-
-    if (NULL == DstDevice) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == DstDevice");
-    }
-
-    if (NULL == SrcDevice) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == SrcDevice");
-    }
-
-    if (NULL == DstPtr) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == DstPtr");
-    }
-
-    if (NULL == SrcPtr) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == SrcPtr");
-    }
-  }
-
-  return llvm::offload::olMemcpy_impl(Queue, DstPtr, DstDevice, SrcPtr,
-                                      SrcDevice, Size, EventOut);
-}
-OL_APIEXPORT ol_result_t OL_APICALL
-olMemcpy(ol_queue_handle_t Queue, void *DstPtr, ol_device_handle_t DstDevice,
-         void *SrcPtr, ol_device_handle_t SrcDevice, size_t Size,
-         ol_event_handle_t *EventOut) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olMemcpy";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olMemcpy_val(
-      Queue, DstPtr, DstDevice, SrcPtr, SrcDevice, Size, EventOut));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_memcpy_params_t Params = {&Queue,     &DstPtr, &DstDevice, &SrcPtr,
-                                 &SrcDevice, &Size,   &EventOut};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olMemcpyWithCodeLoc(ol_queue_handle_t Queue, void *DstPtr,
-                                ol_device_handle_t DstDevice, void *SrcPtr,
-                                ol_device_handle_t SrcDevice, size_t Size,
-                                ol_event_handle_t *EventOut,
-                                ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result =
-      ::olMemcpy(Queue, DstPtr, DstDevice, SrcPtr, SrcDevice, Size, EventOut);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olCreateQueue_val(ol_device_handle_t Device,
-                              ol_queue_handle_t *Queue) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Device) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Device");
-    }
-
-    if (NULL == Queue) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == Queue");
-    }
-  }
-
-  return llvm::offload::olCreateQueue_impl(Device, Queue);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olCreateQueue(ol_device_handle_t Device,
-                                                  ol_queue_handle_t *Queue) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olCreateQueue";
-  }
-
-  ol_result_t Result =
-      llvmErrorToOffloadError(olCreateQueue_val(Device, Queue));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_create_queue_params_t Params = {&Device, &Queue};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olCreateQueueWithCodeLoc(ol_device_handle_t Device,
-                                     ol_queue_handle_t *Queue,
-                                     ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olCreateQueue(Device, Queue);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olDestroyQueue_val(ol_queue_handle_t Queue) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Queue) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Queue");
-    }
-  }
-
-  return llvm::offload::olDestroyQueue_impl(Queue);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyQueue(ol_queue_handle_t Queue) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olDestroyQueue";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olDestroyQueue_val(Queue));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_destroy_queue_params_t Params = {&Queue};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olDestroyQueueWithCodeLoc(ol_queue_handle_t Queue,
-                                      ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olDestroyQueue(Queue);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olWaitQueue_val(ol_queue_handle_t Queue) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Queue) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Queue");
-    }
-  }
-
-  return llvm::offload::olWaitQueue_impl(Queue);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olWaitQueue(ol_queue_handle_t Queue) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olWaitQueue";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olWaitQueue_val(Queue));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_wait_queue_params_t Params = {&Queue};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olWaitQueueWithCodeLoc(ol_queue_handle_t Queue,
-                                   ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olWaitQueue(Queue);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olDestroyEvent_val(ol_event_handle_t Event) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Event) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Event");
-    }
-  }
-
-  return llvm::offload::olDestroyEvent_impl(Event);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olDestroyEvent(ol_event_handle_t Event) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olDestroyEvent";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olDestroyEvent_val(Event));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_destroy_event_params_t Params = {&Event};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olDestroyEventWithCodeLoc(ol_event_handle_t Event,
-                                      ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olDestroyEvent(Event);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olWaitEvent_val(ol_event_handle_t Event) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Event) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Event");
-    }
-  }
-
-  return llvm::offload::olWaitEvent_impl(Event);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olWaitEvent(ol_event_handle_t Event) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olWaitEvent";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olWaitEvent_val(Event));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_wait_event_params_t Params = {&Event};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olWaitEventWithCodeLoc(ol_event_handle_t Event,
-                                   ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olWaitEvent(Event);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olCreateProgram_val(ol_device_handle_t Device, const void *ProgData,
-                                size_t ProgDataSize,
-                                ol_program_handle_t *Program) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Device) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Device");
-    }
-
-    if (NULL == ProgData) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == ProgData");
-    }
-
-    if (NULL == Program) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == Program");
-    }
-  }
-
-  return llvm::offload::olCreateProgram_impl(Device, ProgData, ProgDataSize,
-                                             Program);
-}
-OL_APIEXPORT ol_result_t OL_APICALL
-olCreateProgram(ol_device_handle_t Device, const void *ProgData,
-                size_t ProgDataSize, ol_program_handle_t *Program) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olCreateProgram";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(
-      olCreateProgram_val(Device, ProgData, ProgDataSize, Program));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_create_program_params_t Params = {&Device, &ProgData, &ProgDataSize,
-                                         &Program};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olCreateProgramWithCodeLoc(ol_device_handle_t Device,
-                                       const void *ProgData,
-                                       size_t ProgDataSize,
-                                       ol_program_handle_t *Program,
-                                       ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result =
-      ::olCreateProgram(Device, ProgData, ProgDataSize, Program);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olDestroyProgram_val(ol_program_handle_t Program) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Program) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Program");
-    }
-  }
-
-  return llvm::offload::olDestroyProgram_impl(Program);
-}
-OL_APIEXPORT ol_result_t OL_APICALL
-olDestroyProgram(ol_program_handle_t Program) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olDestroyProgram";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(olDestroyProgram_val(Program));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_destroy_program_params_t Params = {&Program};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olDestroyProgramWithCodeLoc(ol_program_handle_t Program,
-                                        ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olDestroyProgram(Program);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error olGetKernel_val(ol_program_handle_t Program, const char *KernelName,
-                            ol_kernel_handle_t *Kernel) {
-  if (offloadConfig().ValidationEnabled) {
-    if (NULL == Program) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Program");
-    }
-
-    if (NULL == KernelName) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == KernelName");
-    }
-
-    if (NULL == Kernel) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == Kernel");
-    }
-  }
-
-  return llvm::offload::olGetKernel_impl(Program, KernelName, Kernel);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olGetKernel(ol_program_handle_t Program,
-                                                const char *KernelName,
-                                                ol_kernel_handle_t *Kernel) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olGetKernel";
-  }
-
-  ol_result_t Result =
-      llvmErrorToOffloadError(olGetKernel_val(Program, KernelName, Kernel));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_get_kernel_params_t Params = {&Program, &KernelName, &Kernel};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olGetKernelWithCodeLoc(ol_program_handle_t Program,
-                                   const char *KernelName,
-                                   ol_kernel_handle_t *Kernel,
-                                   ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result = ::olGetKernel(Program, KernelName, Kernel);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-llvm::Error
-olLaunchKernel_val(ol_queue_handle_t Queue, ol_device_handle_t Device,
-                   ol_kernel_handle_t Kernel, const void *ArgumentsData,
-                   size_t ArgumentsSize,
-                   const ol_kernel_launch_size_args_t *LaunchSizeArgs,
-                   ol_event_handle_t *EventOut) {
-  if (offloadConfig().ValidationEnabled) {
-    if (Queue == NULL && EventOut != NULL) {
-      return createOffloadError(
-          error::ErrorCode::INVALID_ARGUMENT,
-          "validation failure: Queue == NULL && EventOut != NULL");
-    }
-
-    if (NULL == Device) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Device");
-    }
-
-    if (NULL == Kernel) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_HANDLE,
-                                "validation failure: NULL == Kernel");
-    }
-
-    if (NULL == ArgumentsData) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == ArgumentsData");
-    }
-
-    if (NULL == LaunchSizeArgs) {
-      return createOffloadError(error::ErrorCode::INVALID_NULL_POINTER,
-                                "validation failure: NULL == LaunchSizeArgs");
-    }
-  }
-
-  return llvm::offload::olLaunchKernel_impl(Queue, Device, Kernel,
-                                            ArgumentsData, ArgumentsSize,
-                                            LaunchSizeArgs, EventOut);
-}
-OL_APIEXPORT ol_result_t OL_APICALL olLaunchKernel(
-    ol_queue_handle_t Queue, ol_device_handle_t Device,
-    ol_kernel_handle_t Kernel, const void *ArgumentsData, size_t ArgumentsSize,
-    const ol_kernel_launch_size_args_t *LaunchSizeArgs,
-    ol_event_handle_t *EventOut) {
-  if (offloadConfig().TracingEnabled) {
-    llvm::errs() << "---> olLaunchKernel";
-  }
-
-  ol_result_t Result = llvmErrorToOffloadError(
-      olLaunchKernel_val(Queue, Device, Kernel, ArgumentsData, ArgumentsSize,
-                         LaunchSizeArgs, EventOut));
-
-  if (offloadConfig().TracingEnabled) {
-    ol_launch_kernel_params_t Params = {
-        &Queue,         &Device,         &Kernel,  &ArgumentsData,
-        &ArgumentsSize, &LaunchSizeArgs, &EventOut};
-    llvm::errs() << "(" << &Params << ")";
-    llvm::errs() << "-> " << Result << "\n";
-    if (Result && Result->Details) {
-      llvm::errs() << "     *Error Details* " << Result->Details << " \n";
-    }
-  }
-  return Result;
-}
-ol_result_t olLaunchKernelWithCodeLoc(
-    ol_queue_handle_t Queue, ol_device_handle_t Device,
-    ol_kernel_handle_t Kernel, const void *ArgumentsData, size_t ArgumentsSize,
-    const ol_kernel_launch_size_args_t *LaunchSizeArgs,
-    ol_event_handle_t *EventOut, ol_code_location_t *CodeLocation) {
-  currentCodeLocation() = CodeLocation;
-  ol_result_t Result =
-      ::olLaunchKernel(Queue, Device, Kernel, ArgumentsData, ArgumentsSize,
-                       LaunchSizeArgs, EventOut);
-
-  currentCodeLocation() = nullptr;
-  return Result;
-}
diff --git a/external/llvm-project/offload/liboffload/include/generated/OffloadFuncs.inc b/external/llvm-project/offload/liboffload/include/generated/OffloadFuncs.inc
deleted file mode 100644
index 78ff9ddb8279..000000000000
--- a/external/llvm-project/offload/liboffload/include/generated/OffloadFuncs.inc
+++ /dev/null
@@ -1,52 +0,0 @@
-//===- Auto-generated file, part of the LLVM/Offload project --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef OFFLOAD_FUNC
-#error Please define the macro OFFLOAD_FUNC(Function)
-#endif
-
-OFFLOAD_FUNC(olInit)
-OFFLOAD_FUNC(olShutDown)
-OFFLOAD_FUNC(olGetPlatformInfo)
-OFFLOAD_FUNC(olGetPlatformInfoSize)
-OFFLOAD_FUNC(olIterateDevices)
-OFFLOAD_FUNC(olGetDeviceInfo)
-OFFLOAD_FUNC(olGetDeviceInfoSize)
-OFFLOAD_FUNC(olMemAlloc)
-OFFLOAD_FUNC(olMemFree)
-OFFLOAD_FUNC(olMemcpy)
-OFFLOAD_FUNC(olCreateQueue)
-OFFLOAD_FUNC(olDestroyQueue)
-OFFLOAD_FUNC(olWaitQueue)
-OFFLOAD_FUNC(olDestroyEvent)
-OFFLOAD_FUNC(olWaitEvent)
-OFFLOAD_FUNC(olCreateProgram)
-OFFLOAD_FUNC(olDestroyProgram)
-OFFLOAD_FUNC(olGetKernel)
-OFFLOAD_FUNC(olLaunchKernel)
-OFFLOAD_FUNC(olInitWithCodeLoc)
-OFFLOAD_FUNC(olShutDownWithCodeLoc)
-OFFLOAD_FUNC(olGetPlatformInfoWithCodeLoc)
-OFFLOAD_FUNC(olGetPlatformInfoSizeWithCodeLoc)
-OFFLOAD_FUNC(olIterateDevicesWithCodeLoc)
-OFFLOAD_FUNC(olGetDeviceInfoWithCodeLoc)
-OFFLOAD_FUNC(olGetDeviceInfoSizeWithCodeLoc)
-OFFLOAD_FUNC(olMemAllocWithCodeLoc)
-OFFLOAD_FUNC(olMemFreeWithCodeLoc)
-OFFLOAD_FUNC(olMemcpyWithCodeLoc)
-OFFLOAD_FUNC(olCreateQueueWithCodeLoc)
-OFFLOAD_FUNC(olDestroyQueueWithCodeLoc)
-OFFLOAD_FUNC(olWaitQueueWithCodeLoc)
-OFFLOAD_FUNC(olDestroyEventWithCodeLoc)
-OFFLOAD_FUNC(olWaitEventWithCodeLoc)
-OFFLOAD_FUNC(olCreateProgramWithCodeLoc)
-OFFLOAD_FUNC(olDestroyProgramWithCodeLoc)
-OFFLOAD_FUNC(olGetKernelWithCodeLoc)
-OFFLOAD_FUNC(olLaunchKernelWithCodeLoc)
-
-#undef OFFLOAD_FUNC
diff --git a/external/llvm-project/offload/liboffload/include/generated/OffloadImplFuncDecls.inc b/external/llvm-project/offload/liboffload/include/generated/OffloadImplFuncDecls.inc
deleted file mode 100644
index 71d25dee8786..000000000000
--- a/external/llvm-project/offload/liboffload/include/generated/OffloadImplFuncDecls.inc
+++ /dev/null
@@ -1,60 +0,0 @@
-//===- Auto-generated file, part of the LLVM/Offload project --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-Error olInit_impl();
-
-Error olShutDown_impl();
-
-Error olGetPlatformInfo_impl(ol_platform_handle_t Platform,
-                             ol_platform_info_t PropName, size_t PropSize,
-                             void *PropValue);
-
-Error olGetPlatformInfoSize_impl(ol_platform_handle_t Platform,
-                                 ol_platform_info_t PropName,
-                                 size_t *PropSizeRet);
-
-Error olIterateDevices_impl(ol_device_iterate_cb_t Callback, void *UserData);
-
-Error olGetDeviceInfo_impl(ol_device_handle_t Device, ol_device_info_t PropName,
-                           size_t PropSize, void *PropValue);
-
-Error olGetDeviceInfoSize_impl(ol_device_handle_t Device,
-                               ol_device_info_t PropName, size_t *PropSizeRet);
-
-Error olMemAlloc_impl(ol_device_handle_t Device, ol_alloc_type_t Type,
-                      size_t Size, void **AllocationOut);
-
-Error olMemFree_impl(void *Address);
-
-Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,
-                    ol_device_handle_t DstDevice, void *SrcPtr,
-                    ol_device_handle_t SrcDevice, size_t Size,
-                    ol_event_handle_t *EventOut);
-
-Error olCreateQueue_impl(ol_device_handle_t Device, ol_queue_handle_t *Queue);
-
-Error olDestroyQueue_impl(ol_queue_handle_t Queue);
-
-Error olWaitQueue_impl(ol_queue_handle_t Queue);
-
-Error olDestroyEvent_impl(ol_event_handle_t Event);
-
-Error olWaitEvent_impl(ol_event_handle_t Event);
-
-Error olCreateProgram_impl(ol_device_handle_t Device, const void *ProgData,
-                           size_t ProgDataSize, ol_program_handle_t *Program);
-
-Error olDestroyProgram_impl(ol_program_handle_t Program);
-
-Error olGetKernel_impl(ol_program_handle_t Program, const char *KernelName,
-                       ol_kernel_handle_t *Kernel);
-
-Error olLaunchKernel_impl(ol_queue_handle_t Queue, ol_device_handle_t Device,
-                          ol_kernel_handle_t Kernel, const void *ArgumentsData,
-                          size_t ArgumentsSize,
-                          const ol_kernel_launch_size_args_t *LaunchSizeArgs,
-                          ol_event_handle_t *EventOut);
diff --git a/external/llvm-project/offload/liboffload/include/generated/OffloadPrint.hpp b/external/llvm-project/offload/liboffload/include/generated/OffloadPrint.hpp
deleted file mode 100644
index 3aad6223d4de..000000000000
--- a/external/llvm-project/offload/liboffload/include/generated/OffloadPrint.hpp
+++ /dev/null
@@ -1,645 +0,0 @@
-//===- Auto-generated file, part of the LLVM/Offload project --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// Auto-generated file, do not manually edit.
-
-#pragma once
-
-#include <OffloadAPI.h>
-#include <llvm/Support/raw_ostream.h>
-
-template <typename T>
-inline ol_result_t printPtr(llvm::raw_ostream &os, const T *ptr);
-template <typename T>
-inline void printTagged(llvm::raw_ostream &os, const void *ptr, T value,
-                        size_t size);
-template <typename T> struct is_handle : std::false_type {};
-template <> struct is_handle<ol_platform_handle_t> : std::true_type {};
-template <> struct is_handle<ol_device_handle_t> : std::true_type {};
-template <> struct is_handle<ol_context_handle_t> : std::true_type {};
-template <> struct is_handle<ol_queue_handle_t> : std::true_type {};
-template <> struct is_handle<ol_event_handle_t> : std::true_type {};
-template <> struct is_handle<ol_program_handle_t> : std::true_type {};
-template <typename T> inline constexpr bool is_handle_v = is_handle<T>::value;
-
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_errc_t value);
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_platform_info_t value);
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_platform_backend_t value);
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_device_type_t value);
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_device_info_t value);
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_alloc_type_t value);
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_errc_t type
-/// @returns llvm::raw_ostream &
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_errc_t value) {
-  switch (value) {
-  case OL_ERRC_SUCCESS:
-    os << "OL_ERRC_SUCCESS";
-    break;
-  case OL_ERRC_UNKNOWN:
-    os << "OL_ERRC_UNKNOWN";
-    break;
-  case OL_ERRC_HOST_IO:
-    os << "OL_ERRC_HOST_IO";
-    break;
-  case OL_ERRC_INVALID_BINARY:
-    os << "OL_ERRC_INVALID_BINARY";
-    break;
-  case OL_ERRC_INVALID_NULL_POINTER:
-    os << "OL_ERRC_INVALID_NULL_POINTER";
-    break;
-  case OL_ERRC_INVALID_ARGUMENT:
-    os << "OL_ERRC_INVALID_ARGUMENT";
-    break;
-  case OL_ERRC_NOT_FOUND:
-    os << "OL_ERRC_NOT_FOUND";
-    break;
-  case OL_ERRC_OUT_OF_RESOURCES:
-    os << "OL_ERRC_OUT_OF_RESOURCES";
-    break;
-  case OL_ERRC_INVALID_SIZE:
-    os << "OL_ERRC_INVALID_SIZE";
-    break;
-  case OL_ERRC_INVALID_ENUMERATION:
-    os << "OL_ERRC_INVALID_ENUMERATION";
-    break;
-  case OL_ERRC_HOST_TOOL_NOT_FOUND:
-    os << "OL_ERRC_HOST_TOOL_NOT_FOUND";
-    break;
-  case OL_ERRC_INVALID_VALUE:
-    os << "OL_ERRC_INVALID_VALUE";
-    break;
-  case OL_ERRC_UNIMPLEMENTED:
-    os << "OL_ERRC_UNIMPLEMENTED";
-    break;
-  case OL_ERRC_UNSUPPORTED:
-    os << "OL_ERRC_UNSUPPORTED";
-    break;
-  case OL_ERRC_ASSEMBLE_FAILURE:
-    os << "OL_ERRC_ASSEMBLE_FAILURE";
-    break;
-  case OL_ERRC_LINK_FAILURE:
-    os << "OL_ERRC_LINK_FAILURE";
-    break;
-  case OL_ERRC_BACKEND_FAILURE:
-    os << "OL_ERRC_BACKEND_FAILURE";
-    break;
-  case OL_ERRC_INVALID_NULL_HANDLE:
-    os << "OL_ERRC_INVALID_NULL_HANDLE";
-    break;
-  case OL_ERRC_INVALID_PLATFORM:
-    os << "OL_ERRC_INVALID_PLATFORM";
-    break;
-  case OL_ERRC_INVALID_DEVICE:
-    os << "OL_ERRC_INVALID_DEVICE";
-    break;
-  case OL_ERRC_INVALID_QUEUE:
-    os << "OL_ERRC_INVALID_QUEUE";
-    break;
-  case OL_ERRC_INVALID_EVENT:
-    os << "OL_ERRC_INVALID_EVENT";
-    break;
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-  return os;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_platform_info_t type
-/// @returns llvm::raw_ostream &
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_platform_info_t value) {
-  switch (value) {
-  case OL_PLATFORM_INFO_NAME:
-    os << "OL_PLATFORM_INFO_NAME";
-    break;
-  case OL_PLATFORM_INFO_VENDOR_NAME:
-    os << "OL_PLATFORM_INFO_VENDOR_NAME";
-    break;
-  case OL_PLATFORM_INFO_VERSION:
-    os << "OL_PLATFORM_INFO_VERSION";
-    break;
-  case OL_PLATFORM_INFO_BACKEND:
-    os << "OL_PLATFORM_INFO_BACKEND";
-    break;
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-  return os;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print type-tagged ol_platform_info_t enum value
-/// @returns llvm::raw_ostream &
-template <>
-inline void printTagged(llvm::raw_ostream &os, const void *ptr,
-                        ol_platform_info_t value, size_t size) {
-  if (ptr == NULL) {
-    printPtr(os, ptr);
-    return;
-  }
-
-  switch (value) {
-  case OL_PLATFORM_INFO_NAME: {
-    printPtr(os, (const char *)ptr);
-    break;
-  }
-  case OL_PLATFORM_INFO_VENDOR_NAME: {
-    printPtr(os, (const char *)ptr);
-    break;
-  }
-  case OL_PLATFORM_INFO_VERSION: {
-    printPtr(os, (const char *)ptr);
-    break;
-  }
-  case OL_PLATFORM_INFO_BACKEND: {
-    const ol_platform_backend_t *const tptr =
-        (const ol_platform_backend_t *const)ptr;
-    os << (const void *)tptr << " (";
-    os << *tptr;
-    os << ")";
-    break;
-  }
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-}
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_platform_backend_t type
-/// @returns llvm::raw_ostream &
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_platform_backend_t value) {
-  switch (value) {
-  case OL_PLATFORM_BACKEND_UNKNOWN:
-    os << "OL_PLATFORM_BACKEND_UNKNOWN";
-    break;
-  case OL_PLATFORM_BACKEND_CUDA:
-    os << "OL_PLATFORM_BACKEND_CUDA";
-    break;
-  case OL_PLATFORM_BACKEND_AMDGPU:
-    os << "OL_PLATFORM_BACKEND_AMDGPU";
-    break;
-  case OL_PLATFORM_BACKEND_HOST:
-    os << "OL_PLATFORM_BACKEND_HOST";
-    break;
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-  return os;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_device_type_t type
-/// @returns llvm::raw_ostream &
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_device_type_t value) {
-  switch (value) {
-  case OL_DEVICE_TYPE_DEFAULT:
-    os << "OL_DEVICE_TYPE_DEFAULT";
-    break;
-  case OL_DEVICE_TYPE_ALL:
-    os << "OL_DEVICE_TYPE_ALL";
-    break;
-  case OL_DEVICE_TYPE_GPU:
-    os << "OL_DEVICE_TYPE_GPU";
-    break;
-  case OL_DEVICE_TYPE_CPU:
-    os << "OL_DEVICE_TYPE_CPU";
-    break;
-  case OL_DEVICE_TYPE_HOST:
-    os << "OL_DEVICE_TYPE_HOST";
-    break;
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-  return os;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_device_info_t type
-/// @returns llvm::raw_ostream &
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_device_info_t value) {
-  switch (value) {
-  case OL_DEVICE_INFO_TYPE:
-    os << "OL_DEVICE_INFO_TYPE";
-    break;
-  case OL_DEVICE_INFO_PLATFORM:
-    os << "OL_DEVICE_INFO_PLATFORM";
-    break;
-  case OL_DEVICE_INFO_NAME:
-    os << "OL_DEVICE_INFO_NAME";
-    break;
-  case OL_DEVICE_INFO_VENDOR:
-    os << "OL_DEVICE_INFO_VENDOR";
-    break;
-  case OL_DEVICE_INFO_DRIVER_VERSION:
-    os << "OL_DEVICE_INFO_DRIVER_VERSION";
-    break;
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-  return os;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print type-tagged ol_device_info_t enum value
-/// @returns llvm::raw_ostream &
-template <>
-inline void printTagged(llvm::raw_ostream &os, const void *ptr,
-                        ol_device_info_t value, size_t size) {
-  if (ptr == NULL) {
-    printPtr(os, ptr);
-    return;
-  }
-
-  switch (value) {
-  case OL_DEVICE_INFO_TYPE: {
-    const ol_device_type_t *const tptr = (const ol_device_type_t *const)ptr;
-    os << (const void *)tptr << " (";
-    os << *tptr;
-    os << ")";
-    break;
-  }
-  case OL_DEVICE_INFO_PLATFORM: {
-    const ol_platform_handle_t *const tptr =
-        (const ol_platform_handle_t *const)ptr;
-    os << (const void *)tptr << " (";
-    os << *tptr;
-    os << ")";
-    break;
-  }
-  case OL_DEVICE_INFO_NAME: {
-    printPtr(os, (const char *)ptr);
-    break;
-  }
-  case OL_DEVICE_INFO_VENDOR: {
-    printPtr(os, (const char *)ptr);
-    break;
-  }
-  case OL_DEVICE_INFO_DRIVER_VERSION: {
-    printPtr(os, (const char *)ptr);
-    break;
-  }
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-}
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_alloc_type_t type
-/// @returns llvm::raw_ostream &
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     enum ol_alloc_type_t value) {
-  switch (value) {
-  case OL_ALLOC_TYPE_HOST:
-    os << "OL_ALLOC_TYPE_HOST";
-    break;
-  case OL_ALLOC_TYPE_DEVICE:
-    os << "OL_ALLOC_TYPE_DEVICE";
-    break;
-  case OL_ALLOC_TYPE_MANAGED:
-    os << "OL_ALLOC_TYPE_MANAGED";
-    break;
-  default:
-    os << "unknown enumerator";
-    break;
-  }
-  return os;
-}
-
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     const ol_error_struct_t *Err) {
-  if (Err == nullptr) {
-    os << "OL_SUCCESS";
-  } else {
-    os << Err->Code;
-  }
-  return os;
-}
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_code_location_t type
-/// @returns llvm::raw_ostream &
-
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     const struct ol_code_location_t params) {
-  os << "(struct ol_code_location_t){";
-  os << ".FunctionName = ";
-  printPtr(os, params.FunctionName);
-  os << ", ";
-  os << ".SourceFile = ";
-  printPtr(os, params.SourceFile);
-  os << ", ";
-  os << ".LineNumber = ";
-  os << params.LineNumber;
-  os << ", ";
-  os << ".ColumnNumber = ";
-  os << params.ColumnNumber;
-  os << "}";
-  return os;
-}
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ol_kernel_launch_size_args_t type
-/// @returns llvm::raw_ostream &
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_kernel_launch_size_args_t params) {
-  os << "(struct ol_kernel_launch_size_args_t){";
-  os << ".Dimensions = ";
-  os << params.Dimensions;
-  os << ", ";
-  os << ".NumGroupsX = ";
-  os << params.NumGroupsX;
-  os << ", ";
-  os << ".NumGroupsY = ";
-  os << params.NumGroupsY;
-  os << ", ";
-  os << ".NumGroupsZ = ";
-  os << params.NumGroupsZ;
-  os << ", ";
-  os << ".GroupSizeX = ";
-  os << params.GroupSizeX;
-  os << ", ";
-  os << ".GroupSizeY = ";
-  os << params.GroupSizeY;
-  os << ", ";
-  os << ".GroupSizeZ = ";
-  os << params.GroupSizeZ;
-  os << ", ";
-  os << ".DynSharedMemory = ";
-  os << params.DynSharedMemory;
-  os << "}";
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_get_platform_info_params_t *params) {
-  os << ".Platform = ";
-  printPtr(os, *params->pPlatform);
-  os << ", ";
-  os << ".PropName = ";
-  os << *params->pPropName;
-  os << ", ";
-  os << ".PropSize = ";
-  os << *params->pPropSize;
-  os << ", ";
-  os << ".PropValue = ";
-  printTagged(os, *params->pPropValue, *params->pPropName, *params->pPropSize);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_get_platform_info_size_params_t *params) {
-  os << ".Platform = ";
-  printPtr(os, *params->pPlatform);
-  os << ", ";
-  os << ".PropName = ";
-  os << *params->pPropName;
-  os << ", ";
-  os << ".PropSizeRet = ";
-  printPtr(os, *params->pPropSizeRet);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_iterate_devices_params_t *params) {
-  os << ".Callback = ";
-  os << reinterpret_cast<void *>(*params->pCallback);
-  os << ", ";
-  os << ".UserData = ";
-  printPtr(os, *params->pUserData);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_get_device_info_params_t *params) {
-  os << ".Device = ";
-  printPtr(os, *params->pDevice);
-  os << ", ";
-  os << ".PropName = ";
-  os << *params->pPropName;
-  os << ", ";
-  os << ".PropSize = ";
-  os << *params->pPropSize;
-  os << ", ";
-  os << ".PropValue = ";
-  printTagged(os, *params->pPropValue, *params->pPropName, *params->pPropSize);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_get_device_info_size_params_t *params) {
-  os << ".Device = ";
-  printPtr(os, *params->pDevice);
-  os << ", ";
-  os << ".PropName = ";
-  os << *params->pPropName;
-  os << ", ";
-  os << ".PropSizeRet = ";
-  printPtr(os, *params->pPropSizeRet);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os, const struct ol_mem_alloc_params_t *params) {
-  os << ".Device = ";
-  printPtr(os, *params->pDevice);
-  os << ", ";
-  os << ".Type = ";
-  os << *params->pType;
-  os << ", ";
-  os << ".Size = ";
-  os << *params->pSize;
-  os << ", ";
-  os << ".AllocationOut = ";
-  printPtr(os, *params->pAllocationOut);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os, const struct ol_mem_free_params_t *params) {
-  os << ".Address = ";
-  printPtr(os, *params->pAddress);
-  return os;
-}
-
-inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                     const struct ol_memcpy_params_t *params) {
-  os << ".Queue = ";
-  printPtr(os, *params->pQueue);
-  os << ", ";
-  os << ".DstPtr = ";
-  printPtr(os, *params->pDstPtr);
-  os << ", ";
-  os << ".DstDevice = ";
-  printPtr(os, *params->pDstDevice);
-  os << ", ";
-  os << ".SrcPtr = ";
-  printPtr(os, *params->pSrcPtr);
-  os << ", ";
-  os << ".SrcDevice = ";
-  printPtr(os, *params->pSrcDevice);
-  os << ", ";
-  os << ".Size = ";
-  os << *params->pSize;
-  os << ", ";
-  os << ".EventOut = ";
-  printPtr(os, *params->pEventOut);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_create_queue_params_t *params) {
-  os << ".Device = ";
-  printPtr(os, *params->pDevice);
-  os << ", ";
-  os << ".Queue = ";
-  printPtr(os, *params->pQueue);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_destroy_queue_params_t *params) {
-  os << ".Queue = ";
-  printPtr(os, *params->pQueue);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os, const struct ol_wait_queue_params_t *params) {
-  os << ".Queue = ";
-  printPtr(os, *params->pQueue);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_destroy_event_params_t *params) {
-  os << ".Event = ";
-  printPtr(os, *params->pEvent);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os, const struct ol_wait_event_params_t *params) {
-  os << ".Event = ";
-  printPtr(os, *params->pEvent);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_create_program_params_t *params) {
-  os << ".Device = ";
-  printPtr(os, *params->pDevice);
-  os << ", ";
-  os << ".ProgData = ";
-  printPtr(os, *params->pProgData);
-  os << ", ";
-  os << ".ProgDataSize = ";
-  os << *params->pProgDataSize;
-  os << ", ";
-  os << ".Program = ";
-  printPtr(os, *params->pProgram);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_destroy_program_params_t *params) {
-  os << ".Program = ";
-  printPtr(os, *params->pProgram);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os, const struct ol_get_kernel_params_t *params) {
-  os << ".Program = ";
-  printPtr(os, *params->pProgram);
-  os << ", ";
-  os << ".KernelName = ";
-  printPtr(os, *params->pKernelName);
-  os << ", ";
-  os << ".Kernel = ";
-  printPtr(os, *params->pKernel);
-  return os;
-}
-
-inline llvm::raw_ostream &
-operator<<(llvm::raw_ostream &os,
-           const struct ol_launch_kernel_params_t *params) {
-  os << ".Queue = ";
-  printPtr(os, *params->pQueue);
-  os << ", ";
-  os << ".Device = ";
-  printPtr(os, *params->pDevice);
-  os << ", ";
-  os << ".Kernel = ";
-  printPtr(os, *params->pKernel);
-  os << ", ";
-  os << ".ArgumentsData = ";
-  printPtr(os, *params->pArgumentsData);
-  os << ", ";
-  os << ".ArgumentsSize = ";
-  os << *params->pArgumentsSize;
-  os << ", ";
-  os << ".LaunchSizeArgs = ";
-  printPtr(os, *params->pLaunchSizeArgs);
-  os << ", ";
-  os << ".EventOut = ";
-  printPtr(os, *params->pEventOut);
-  return os;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// @brief Print pointer value
-template <typename T>
-inline ol_result_t printPtr(llvm::raw_ostream &os, const T *ptr) {
-  if (ptr == nullptr) {
-    os << "nullptr";
-  } else if constexpr (std::is_pointer_v<T>) {
-    os << (const void *)(ptr) << " (";
-    printPtr(os, *ptr);
-    os << ")";
-  } else if constexpr (std::is_void_v<T> || is_handle_v<T *>) {
-    os << (const void *)ptr;
-  } else if constexpr (std::is_same_v<std::remove_cv_t<T>, char>) {
-    os << (const void *)(ptr) << " (";
-    os << ptr;
-    os << ")";
-  } else {
-    os << (const void *)(ptr) << " (";
-    os << *ptr;
-    os << ")";
-  }
-
-  return OL_SUCCESS;
-}
diff --git a/external/llvm-project/offload/liboffload/src/Helpers.hpp b/external/llvm-project/offload/liboffload/src/Helpers.hpp
index bb8245c12e19..425934b6760d 100644
--- a/external/llvm-project/offload/liboffload/src/Helpers.hpp
+++ b/external/llvm-project/offload/liboffload/src/Helpers.hpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "OffloadAPI.h"
-#include "Shared/OffloadError.h"
+#include "OffloadError.h"
 #include "llvm/Support/Error.h"
 
 #include <cstring>
diff --git a/external/llvm-project/offload/liboffload/src/OffloadImpl.cpp b/external/llvm-project/offload/liboffload/src/OffloadImpl.cpp
index 7b67cbba43e6..770c212d804d 100644
--- a/external/llvm-project/offload/liboffload/src/OffloadImpl.cpp
+++ b/external/llvm-project/offload/liboffload/src/OffloadImpl.cpp
@@ -229,26 +229,19 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
 
   // Find the info if it exists under any of the given names
   auto GetInfo = [&](std::vector<std::string> Names) {
-    InfoQueueTy DevInfo;
     if (Device == HostDevice())
       return std::string("Host");
 
     if (!Device->Device)
       return std::string("");
 
-    if (auto Err = Device->Device->obtainInfoImpl(DevInfo))
+    auto Info = Device->Device->obtainInfoImpl();
+    if (auto Err = Info.takeError())
       return std::string("");
 
     for (auto Name : Names) {
-      auto InfoKeyMatches = [&](const InfoQueueTy::InfoQueueEntryTy &Info) {
-        return Info.Key == Name;
-      };
-      auto Item = std::find_if(DevInfo.getQueue().begin(),
-                               DevInfo.getQueue().end(), InfoKeyMatches);
-
-      if (Item != std::end(DevInfo.getQueue())) {
-        return Item->Value;
-      }
+      if (auto Entry = Info->get(Name))
+        return (*Entry)->Value;
     }
 
     return std::string("");
@@ -399,7 +392,7 @@ ol_event_handle_t makeEvent(ol_queue_handle_t Queue) {
 }
 
 Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,
-                    ol_device_handle_t DstDevice, void *SrcPtr,
+                    ol_device_handle_t DstDevice, const void *SrcPtr,
                     ol_device_handle_t SrcDevice, size_t Size,
                     ol_event_handle_t *EventOut) {
   if (DstDevice == HostDevice() && SrcDevice == HostDevice()) {
@@ -499,12 +492,12 @@ Error olLaunchKernel_impl(ol_queue_handle_t Queue, ol_device_handle_t Device,
   auto *QueueImpl = Queue ? Queue->AsyncInfo : nullptr;
   AsyncInfoWrapperTy AsyncInfoWrapper(*DeviceImpl, QueueImpl);
   KernelArgsTy LaunchArgs{};
-  LaunchArgs.NumTeams[0] = LaunchSizeArgs->NumGroupsX;
-  LaunchArgs.NumTeams[1] = LaunchSizeArgs->NumGroupsY;
-  LaunchArgs.NumTeams[2] = LaunchSizeArgs->NumGroupsZ;
-  LaunchArgs.ThreadLimit[0] = LaunchSizeArgs->GroupSizeX;
-  LaunchArgs.ThreadLimit[1] = LaunchSizeArgs->GroupSizeY;
-  LaunchArgs.ThreadLimit[2] = LaunchSizeArgs->GroupSizeZ;
+  LaunchArgs.NumTeams[0] = LaunchSizeArgs->NumGroups.x;
+  LaunchArgs.NumTeams[1] = LaunchSizeArgs->NumGroups.y;
+  LaunchArgs.NumTeams[2] = LaunchSizeArgs->NumGroups.z;
+  LaunchArgs.ThreadLimit[0] = LaunchSizeArgs->GroupSize.x;
+  LaunchArgs.ThreadLimit[1] = LaunchSizeArgs->GroupSize.y;
+  LaunchArgs.ThreadLimit[2] = LaunchSizeArgs->GroupSize.z;
   LaunchArgs.DynCGroupMem = LaunchSizeArgs->DynSharedMemory;
 
   KernelLaunchParamsTy Params;
diff --git a/external/llvm-project/offload/libomptarget/PluginManager.cpp b/external/llvm-project/offload/libomptarget/PluginManager.cpp
index b7e76e0194a0..0d889fd964bc 100644
--- a/external/llvm-project/offload/libomptarget/PluginManager.cpp
+++ b/external/llvm-project/offload/libomptarget/PluginManager.cpp
@@ -329,19 +329,17 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
 
   PM->RTLsMtx.unlock();
 
-  bool UseAutoZeroCopy = Plugins.size() > 0;
   bool IsAPU = Plugins.size() > 0;
 
+  bool UseAutoZeroCopy = false;
   auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
-  for (const auto &Device : *ExclusiveDevicesAccessor) {
-    UseAutoZeroCopy &= Device->useAutoZeroCopy();
-    IsAPU &= Device->checkIfAPU();
+  // APUs are homogeneous set of GPUs. Check the first device for
+  // configuring Auto Zero-Copy.
+  if (ExclusiveDevicesAccessor->size() > 0) {
+    auto &Device = *(*ExclusiveDevicesAccessor)[0];
+    UseAutoZeroCopy = Device.useAutoZeroCopy();
   }
 
-  // Auto Zero-Copy can only be currently triggered when the system is an
-  // homogeneous APU architecture without attached discrete GPUs.
-  // If all devices suggest to use it, change requirement flags to trigger
-  // zero-copy behavior when mapping memory.
   if (UseAutoZeroCopy)
     addRequirements(OMPX_REQ_AUTO_ZERO_COPY);
 
diff --git a/external/llvm-project/offload/plugins-nextgen/CMakeLists.txt b/external/llvm-project/offload/plugins-nextgen/CMakeLists.txt
index ad95137ec987..4a3a5ee0ddb1 100644
--- a/external/llvm-project/offload/plugins-nextgen/CMakeLists.txt
+++ b/external/llvm-project/offload/plugins-nextgen/CMakeLists.txt
@@ -1,5 +1,6 @@
 # Common interface to handle creating a plugin library.
 set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common)
+set(common_bin_dir ${CMAKE_CURRENT_BINARY_DIR}/common)
 add_subdirectory(common)
 function(add_target_library target_name lib_name)
   add_llvm_library(${target_name} STATIC
@@ -36,7 +37,8 @@ function(add_target_library target_name lib_name)
   )
 
   llvm_update_compile_flags(${target_name})
-  target_include_directories(${target_name} PUBLIC ${common_dir}/include)
+  target_include_directories(${target_name} PUBLIC ${common_dir}/include
+                             ${common_bin_dir}/include)
   if(OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
     target_include_directories(${target_name} PUBLIC ${common_dir}/OMPT)
   endif()
diff --git a/external/llvm-project/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/external/llvm-project/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index f16f1fb93014..a33f00f7bbf8 100644
--- a/external/llvm-project/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/external/llvm-project/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -632,7 +632,7 @@ struct AMDGPUMemoryManagerTy : public DeviceAllocatorTy {
     assert(PtrStorage && "Invalid pointer storage");
 
     *PtrStorage = MemoryManager->allocate(Size, nullptr);
-    if (*PtrStorage == nullptr)
+    if (Size && *PtrStorage == nullptr)
       return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                            "failure to allocate from AMDGPU memory manager");
 
@@ -641,8 +641,6 @@ struct AMDGPUMemoryManagerTy : public DeviceAllocatorTy {
 
   /// Release an allocation to be reused.
   Error deallocate(void *Ptr) {
-    assert(Ptr && "Invalid pointer");
-
     if (MemoryManager->free(Ptr))
       return Plugin::error(ErrorCode::UNKNOWN,
                            "failure to deallocate from AMDGPU memory manager");
@@ -2061,7 +2059,6 @@ struct AMDGPUStreamTy {
     ReleaseBufferArgsTy *Args = reinterpret_cast<ReleaseBufferArgsTy *>(Data);
     assert(Args && "Invalid arguments");
     assert(Args->MemoryManager && "Invalid memory manager");
-    assert(Args->Buffer && "Invalid buffer");
 
     // Release the allocation to the memory manager.
     return Args->MemoryManager->deallocate(Args->Buffer);
@@ -3970,7 +3967,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   }
 
   /// Print information about the device.
-  Error obtainInfoImpl(InfoQueueTy &Info) override {
+  Expected<InfoTreeNode> obtainInfoImpl() override {
     char TmpChar[1000];
     const char *TmpCharPtr = "Unknown";
     uint16_t Major, Minor;
@@ -3981,6 +3978,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     uint16_t WorkgrpMaxDim[3];
     hsa_dim3_t GridMaxDim;
     hsa_status_t Status, Status2;
+    InfoTreeNode Info;
 
     Status = hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MAJOR, &Major);
     Status2 = hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MINOR, &Minor);
@@ -4036,11 +4034,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     // runtime.
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_CACHE_SIZE, CacheSize);
     if (Status == HSA_STATUS_SUCCESS) {
-      Info.add("Cache");
+      auto &Cache = *Info.add("Cache");
 
       for (int I = 0; I < 4; I++)
         if (CacheSize[I])
-          Info.add<InfoLevel2>("L" + std::to_string(I), CacheSize[I]);
+          Cache.add("L" + std::to_string(I), CacheSize[I]);
     }
 
     Status = getDeviceAttrRaw(HSA_AMD_AGENT_INFO_CACHELINE_SIZE, TmpUInt);
@@ -4073,10 +4071,10 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_WORKGROUP_MAX_DIM, WorkgrpMaxDim);
     if (Status == HSA_STATUS_SUCCESS) {
-      Info.add("Workgroup Max Size per Dimension");
-      Info.add<InfoLevel2>("x", WorkgrpMaxDim[0]);
-      Info.add<InfoLevel2>("y", WorkgrpMaxDim[1]);
-      Info.add<InfoLevel2>("z", WorkgrpMaxDim[2]);
+      auto &MaxSize = *Info.add("Workgroup Max Size per Dimension");
+      MaxSize.add("x", WorkgrpMaxDim[0]);
+      MaxSize.add("y", WorkgrpMaxDim[1]);
+      MaxSize.add("z", WorkgrpMaxDim[2]);
     }
 
     Status = getDeviceAttrRaw(
@@ -4092,17 +4090,17 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_GRID_MAX_DIM, GridMaxDim);
     if (Status == HSA_STATUS_SUCCESS) {
-      Info.add("Grid Max Size per Dimension");
-      Info.add<InfoLevel2>("x", GridMaxDim.x);
-      Info.add<InfoLevel2>("y", GridMaxDim.y);
-      Info.add<InfoLevel2>("z", GridMaxDim.z);
+      auto &MaxDim = *Info.add("Grid Max Size per Dimension");
+      MaxDim.add("x", GridMaxDim.x);
+      MaxDim.add("y", GridMaxDim.y);
+      MaxDim.add("z", GridMaxDim.z);
     }
 
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_FBARRIER_MAX_SIZE, TmpUInt);
     if (Status == HSA_STATUS_SUCCESS)
       Info.add("Max fbarriers/Workgrp", TmpUInt);
 
-    Info.add("Memory Pools");
+    auto &RootPool = *Info.add("Memory Pools");
     for (AMDGPUMemoryPoolTy *Pool : AllMemoryPools) {
       std::string TmpStr, TmpStr2;
 
@@ -4117,7 +4115,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       else
         TmpStr = "Unknown";
 
-      Info.add<InfoLevel2>(std::string("Pool ") + TmpStr);
+      auto &PoolNode = *RootPool.add(std::string("Pool ") + TmpStr);
 
       if (Pool->isGlobal()) {
         if (Pool->isFineGrained())
@@ -4127,39 +4125,39 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
         if (Pool->supportsKernelArgs())
           TmpStr2 += "Kernarg ";
 
-        Info.add<InfoLevel3>("Flags", TmpStr2);
+        PoolNode.add("Flags", TmpStr2);
       }
 
       Status = Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_SIZE, TmpSt);
       if (Status == HSA_STATUS_SUCCESS)
-        Info.add<InfoLevel3>("Size", TmpSt, "bytes");
+        PoolNode.add("Size", TmpSt, "bytes");
 
       Status = Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALLOWED,
                                 TmpBool);
       if (Status == HSA_STATUS_SUCCESS)
-        Info.add<InfoLevel3>("Allocatable", TmpBool);
+        PoolNode.add("Allocatable", TmpBool);
 
       Status = Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
                                 TmpSt);
       if (Status == HSA_STATUS_SUCCESS)
-        Info.add<InfoLevel3>("Runtime Alloc Granule", TmpSt, "bytes");
+        PoolNode.add("Runtime Alloc Granule", TmpSt, "bytes");
 
       Status = Pool->getAttrRaw(
           HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALIGNMENT, TmpSt);
       if (Status == HSA_STATUS_SUCCESS)
-        Info.add<InfoLevel3>("Runtime Alloc Alignment", TmpSt, "bytes");
+        PoolNode.add("Runtime Alloc Alignment", TmpSt, "bytes");
 
       Status =
           Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_ACCESSIBLE_BY_ALL, TmpBool);
       if (Status == HSA_STATUS_SUCCESS)
-        Info.add<InfoLevel3>("Accessible by all", TmpBool);
+        PoolNode.add("Accessible by all", TmpBool);
     }
 
-    Info.add("ISAs");
+    auto &ISAs = *Info.add("ISAs");
     auto Err = hsa_utils::iterateAgentISAs(getAgent(), [&](hsa_isa_t ISA) {
       Status = hsa_isa_get_info_alt(ISA, HSA_ISA_INFO_NAME, TmpChar);
       if (Status == HSA_STATUS_SUCCESS)
-        Info.add<InfoLevel2>("Name", TmpChar);
+        ISAs.add("Name", TmpChar);
 
       return Status;
     });
@@ -4168,7 +4166,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (Err)
       consumeError(std::move(Err));
 
-    return Plugin::success();
+    return Info;
   }
 
   /// Returns true if auto zero-copy the best configuration for the current
@@ -5222,9 +5220,9 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
                                  KernelLaunchParamsTy LaunchParams,
                                  AsyncInfoWrapperTy &AsyncInfoWrapper) const {
   if (ArgsSize != LaunchParams.Size &&
-      ArgsSize != LaunchParams.Size + getImplicitArgsSize())
+      ArgsSize > LaunchParams.Size + getImplicitArgsSize())
     return Plugin::error(ErrorCode::INVALID_ARGUMENT,
-                         "mismatch of kernel arguments size");
+                         "invalid kernel arguments size");
 
   AMDGPUPluginTy &AMDGPUPlugin =
       static_cast<AMDGPUPluginTy &>(GenericDevice.Plugin);
@@ -5257,27 +5255,40 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   if (auto Err = AMDGPUDevice.getStream(AsyncInfoWrapper, Stream))
     return Err;
 
-  // Only COV5 implicitargs needs to be set. COV4 implicitargs are not used.
-  hsa_utils::AMDGPUImplicitArgsTy *ImplArgs = nullptr;
-  if (ArgsSize == LaunchParams.Size + getImplicitArgsSize()) {
-    ImplArgs = reinterpret_cast<hsa_utils::AMDGPUImplicitArgsTy *>(
-        utils::advancePtr(AllArgs, LaunchParams.Size));
+  uint64_t ImplArgsOffset = utils::roundUp(
+      LaunchParams.Size, alignof(hsa_utils::AMDGPUImplicitArgsTy));
+  if (ArgsSize > ImplArgsOffset) {
+    hsa_utils::AMDGPUImplicitArgsTy *ImplArgs =
+        reinterpret_cast<hsa_utils::AMDGPUImplicitArgsTy *>(
+            utils::advancePtr(AllArgs, ImplArgsOffset));
 
     DP("Setting fields of ImplicitArgs for COV5\n");
-    // Set the COV5+ implicit arguments to the appropriate values.
-    std::memset(ImplArgs, 0, getImplicitArgsSize());
-    ImplArgs->BlockCountX = NumBlocks[0];
-    ImplArgs->BlockCountY = NumBlocks[1];
-    ImplArgs->BlockCountZ = NumBlocks[2];
-    ImplArgs->GroupSizeX = NumThreads[0];
-    ImplArgs->GroupSizeY = NumThreads[1];
-    ImplArgs->GroupSizeZ = NumThreads[2];
-    ImplArgs->GridDims = NumBlocks[2] * NumThreads[2] > 1
-                             ? 3
-                             : 1 + (NumBlocks[1] * NumThreads[1] != 1);
-    ImplArgs->HeapV1Ptr =
-        (uint64_t)AMDGPUDevice.getPreAllocatedDeviceMemoryPool();
-    ImplArgs->DynamicLdsSize = KernelArgs.DynCGroupMem;
+    // Set the COV5+ implicit arguments to the appropriate values if present.
+    uint64_t ImplArgsSize = ArgsSize - ImplArgsOffset;
+    std::memset(ImplArgs, 0, ImplArgsSize);
+
+    using ImplArgsTy = hsa_utils::AMDGPUImplicitArgsTy;
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountX, ImplArgsSize,
+                           NumBlocks[0]);
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountY, ImplArgsSize,
+                           NumBlocks[1]);
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountZ, ImplArgsSize,
+                           NumBlocks[2]);
+
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeX, ImplArgsSize,
+                           NumThreads[0]);
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeY, ImplArgsSize,
+                           NumThreads[1]);
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeZ, ImplArgsSize,
+                           NumThreads[2]);
+
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GridDims, ImplArgsSize,
+                           NumBlocks[2] * NumThreads[2] > 1
+                               ? 3
+                               : 1 + (NumBlocks[1] * NumThreads[1] != 1));
+
+    hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::DynamicLdsSize, ImplArgsSize,
+                           KernelArgs.DynCGroupMem);
   }
 
   // Get required OMPT-related data
diff --git a/external/llvm-project/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h b/external/llvm-project/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h
index 903c93d5905f..bc0eee89e61e 100644
--- a/external/llvm-project/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h
+++ b/external/llvm-project/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h
@@ -13,6 +13,7 @@
 #include <cstdint>
 
 #include "Shared/Debug.h"
+#include "Shared/Utils.h"
 #include "Utils/ELF.h"
 
 #include "omptarget.h"
@@ -34,7 +35,7 @@ enum COV_OFFSETS : uint32_t {
 typedef unsigned XnackBuildMode;
 
 // The implicit arguments of COV5 AMDGPU kernels.
-struct AMDGPUImplicitArgsTy {
+struct alignas(alignof(void *)) AMDGPUImplicitArgsTy {
   uint32_t BlockCountX;
   uint32_t BlockCountY;
   uint32_t BlockCountZ;
@@ -119,6 +120,18 @@ inline Error readAMDGPUMetaDataFromImage(
   return Err;
 }
 
+/// Initializes the HSA implicit argument if the struct size permits it. This is
+/// necessary because optimizations can modify the size of the struct if
+/// portions of it are unused.
+template <typename MemberTy, typename T>
+void initImplArg(AMDGPUImplicitArgsTy *Base,
+                 MemberTy AMDGPUImplicitArgsTy::*Member, size_t AvailableSize,
+                 T Value) {
+  uint64_t Offset = utils::getPtrDiff(&(Base->*Member), Base);
+  if (Offset + sizeof(MemberTy) <= AvailableSize)
+    Base->*Member = static_cast<MemberTy>(Value);
+}
+
 } // namespace hsa_utils
 } // namespace plugin
 } // namespace target
diff --git a/external/llvm-project/offload/plugins-nextgen/common/CMakeLists.txt b/external/llvm-project/offload/plugins-nextgen/common/CMakeLists.txt
index 27197d20b2af..edb444cf29c1 100644
--- a/external/llvm-project/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/external/llvm-project/offload/plugins-nextgen/common/CMakeLists.txt
@@ -1,3 +1,10 @@
+# The error codes in this header are shared with liboffload, so need to be
+# generated from the same source.
+include(TableGen)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include)
+set(LLVM_TARGET_DEFINITIONS ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/API/OffloadAPI.td)
+tablegen(OFFLOAD include/OffloadErrcodes.inc -gen-errcodes -I ${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/API)
+add_public_tablegen_target(PluginErrcodes)
 
 if(OFFLOAD_ENABLE_EMISSARY_APIS)
   set(emissary_sources
@@ -18,7 +25,7 @@ add_library(PluginCommon OBJECT
   src/Utils/ELF.cpp
   ${emissary_sources}
 )
-add_dependencies(PluginCommon intrinsics_gen)
+add_dependencies(PluginCommon intrinsics_gen PluginErrcodes)
 
 # Only enable JIT for those targets that LLVM can support.
 set(supported_jit_targets AMDGPU NVPTX)
@@ -60,6 +67,7 @@ endif()
 
 target_include_directories(PluginCommon PUBLIC
   ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${CMAKE_CURRENT_BINARY_DIR}/include
   ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
   ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
   ${LIBOMPTARGET_INCLUDE_DIR}
diff --git a/external/llvm-project/offload/plugins-nextgen/common/include/GlobalHandler.h b/external/llvm-project/offload/plugins-nextgen/common/include/GlobalHandler.h
index cfaa53c7d98f..3d1af7824e6e 100644
--- a/external/llvm-project/offload/plugins-nextgen/common/include/GlobalHandler.h
+++ b/external/llvm-project/offload/plugins-nextgen/common/include/GlobalHandler.h
@@ -80,6 +80,7 @@ struct GPUProfGlobals {
 
   void dump() const;
   Error write() const;
+  bool empty() const;
 };
 
 /// Subclass of GlobalTy that holds the memory for a global of \p Ty.
@@ -192,9 +193,6 @@ class GenericGlobalHandlerTy {
                                           /*D2H=*/false);
   }
 
-  /// Checks whether a given image contains profiling globals.
-  bool hasProfilingGlobals(GenericDeviceTy &Device, DeviceImageTy &Image);
-
   /// Reads profiling data from a GPU image to supplied profdata struct.
   /// Iterates through the image symbol table and stores global values
   /// with profiling prefixes.
diff --git a/external/llvm-project/offload/include/Shared/OffloadError.h b/external/llvm-project/offload/plugins-nextgen/common/include/OffloadError.h
similarity index 98%
rename from external/llvm-project/offload/include/Shared/OffloadError.h
rename to external/llvm-project/offload/plugins-nextgen/common/include/OffloadError.h
index 01b2c59d7f6d..f2a56d2c4111 100644
--- a/external/llvm-project/offload/include/Shared/OffloadError.h
+++ b/external/llvm-project/offload/plugins-nextgen/common/include/OffloadError.h
@@ -18,7 +18,7 @@ namespace error {
 
 enum class ErrorCode {
 #define OFFLOAD_ERRC(Name, _, Value) Name = Value,
-#include "Shared/OffloadErrcodes.inc"
+#include "OffloadErrcodes.inc"
 #undef OFFLOAD_ERRC
 };
 
diff --git a/external/llvm-project/offload/plugins-nextgen/common/include/PluginInterface.h b/external/llvm-project/offload/plugins-nextgen/common/include/PluginInterface.h
index 2f496d2d33e4..1dcc2c7096c1 100644
--- a/external/llvm-project/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/external/llvm-project/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -26,13 +26,13 @@
 #include "Shared/Debug.h"
 #include "Shared/Environment.h"
 #include "Shared/EnvironmentVar.h"
-#include "Shared/OffloadError.h"
 #include "Shared/Requirements.h"
 #include "Shared/Utils.h"
 
 #include "GlobalHandler.h"
 #include "JIT.h"
 #include "MemoryManager.h"
+#include "OffloadError.h"
 #include "RPC.h"
 #include "omptarget.h"
 
@@ -116,77 +116,100 @@ struct AsyncInfoWrapperTy {
   __tgt_async_info *AsyncInfoPtr;
 };
 
-/// The information level represents the level of a key-value property in the
-/// info tree print (i.e. indentation). The first level should be the default.
-enum InfoLevelKind { InfoLevel1 = 1, InfoLevel2, InfoLevel3 };
-
-/// Class for storing device information and later be printed. An object of this
-/// type acts as a queue of key-value properties. Each property has a key, a
-/// a value, and an optional unit for the value. For printing purposes, the
-/// information can be classified into several levels. These levels are useful
-/// for defining sections and subsections. Thus, each key-value property also
-/// has an additional field indicating to which level belongs to. Notice that
-/// we use the level to determine the indentation of the key-value property at
-/// printing time. See the enum InfoLevelKind for the list of accepted levels.
-class InfoQueueTy {
-public:
-  struct InfoQueueEntryTy {
-    std::string Key;
-    std::string Value;
-    std::string Units;
-    uint64_t Level;
-  };
-
-private:
-  std::deque<InfoQueueEntryTy> Queue;
-
-public:
-  /// Add a new info entry to the queue. The entry requires at least a key
-  /// string in \p Key. The value in \p Value is optional and can be any type
-  /// that is representable as a string. The units in \p Units is optional and
-  /// must be a string. The info level is a template parameter that defaults to
-  /// the first level (top level).
-  template <InfoLevelKind L = InfoLevel1, typename T = std::string>
-  void add(const std::string &Key, T Value = T(),
-           const std::string &Units = std::string()) {
+/// Tree node for device information
+///
+/// This information is either printed or used by liboffload to extract certain
+/// device queries. Each property has an optional key, an optional value
+/// and optional children. The children can be used to store additional
+/// information (such as x, y and z components of ranges).
+struct InfoTreeNode {
+  static constexpr uint64_t IndentSize = 4;
+
+  std::string Key;
+  std::string Value;
+  std::string Units;
+  // Need to specify a default value number of elements here as `InfoTreeNode`'s
+  // size is unknown. This is a vector (rather than a Key->Value map) since:
+  // * The keys need to be owned and thus `std::string`s
+  // * The order of keys is important
+  // * The same key can appear multiple times
+  std::unique_ptr<llvm::SmallVector<InfoTreeNode, 8>> Children;
+
+  InfoTreeNode() : InfoTreeNode("", "", "") {}
+  InfoTreeNode(std::string Key, std::string Value, std::string Units)
+      : Key(Key), Value(Value), Units(Units) {}
+
+  /// Add a new info entry as a child of this node. The entry requires at least
+  /// a key string in \p Key. The value in \p Value is optional and can be any
+  /// type that is representable as a string. The units in \p Units is optional
+  /// and must be a string.
+  template <typename T = std::string>
+  InfoTreeNode *add(std::string Key, T Value = T(),
+                    const std::string &Units = std::string()) {
     assert(!Key.empty() && "Invalid info key");
 
-    // Convert the value to a string depending on its type.
+    if (!Children)
+      Children = std::make_unique<llvm::SmallVector<InfoTreeNode, 8>>();
+
+    std::string ValueStr;
     if constexpr (std::is_same_v<T, bool>)
-      Queue.push_back({Key, Value ? "Yes" : "No", Units, L});
+      ValueStr = Value ? "Yes" : "No";
     else if constexpr (std::is_arithmetic_v<T>)
-      Queue.push_back({Key, std::to_string(Value), Units, L});
+      ValueStr = std::to_string(Value);
     else
-      Queue.push_back({Key, Value, Units, L});
+      ValueStr = Value;
+
+    return &Children->emplace_back(Key, ValueStr, Units);
   }
 
-  const std::deque<InfoQueueEntryTy> &getQueue() const { return Queue; }
+  std::optional<InfoTreeNode *> get(StringRef Key) {
+    if (!Children)
+      return std::nullopt;
 
-  /// Print all info entries added to the queue.
-  void print() const {
-    // We print four spances for each level.
-    constexpr uint64_t IndentSize = 4;
+    auto It = std::find_if(Children->begin(), Children->end(),
+                           [&](auto &V) { return V.Key == Key; });
+    if (It == Children->end())
+      return std::nullopt;
+    return It;
+  }
 
-    // Find the maximum key length (level + key) to compute the individual
-    // indentation of each entry.
-    uint64_t MaxKeySize = 0;
-    for (const auto &Entry : Queue) {
-      uint64_t KeySize = Entry.Key.size() + Entry.Level * IndentSize;
-      if (KeySize > MaxKeySize)
-        MaxKeySize = KeySize;
-    }
+  /// Print all info entries in the tree
+  void print() const {
+    // Fake an additional indent so that values are offset from the keys
+    doPrint(0, maxKeySize(1));
+  }
 
-    // Print all info entries.
-    for (const auto &Entry : Queue) {
+private:
+  void doPrint(int Level, uint64_t MaxKeySize) const {
+    if (Key.size()) {
       // Compute the indentations for the current entry.
-      uint64_t KeyIndentSize = Entry.Level * IndentSize;
+      uint64_t KeyIndentSize = Level * IndentSize;
       uint64_t ValIndentSize =
-          MaxKeySize - (Entry.Key.size() + KeyIndentSize) + IndentSize;
+          MaxKeySize - (Key.size() + KeyIndentSize) + IndentSize;
 
-      llvm::outs() << std::string(KeyIndentSize, ' ') << Entry.Key
-                   << std::string(ValIndentSize, ' ') << Entry.Value
-                   << (Entry.Units.empty() ? "" : " ") << Entry.Units << "\n";
+      llvm::outs() << std::string(KeyIndentSize, ' ') << Key
+                   << std::string(ValIndentSize, ' ') << Value
+                   << (Units.empty() ? "" : " ") << Units << "\n";
     }
+
+    // Print children
+    if (Children)
+      for (const auto &Entry : *Children)
+        Entry.doPrint(Level + 1, MaxKeySize);
+  }
+
+  // Recursively calculates the maximum width of each key, including indentation
+  uint64_t maxKeySize(int Level) const {
+    uint64_t MaxKeySize = 0;
+
+    if (Children)
+      for (const auto &Entry : *Children) {
+        uint64_t KeySize = Entry.Key.size() + Level * IndentSize;
+        MaxKeySize = std::max(MaxKeySize, KeySize);
+        MaxKeySize = std::max(MaxKeySize, Entry.maxKeySize(Level + 1));
+      }
+
+    return MaxKeySize;
   }
 };
 
@@ -994,7 +1017,7 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
 
   /// Print information about the device.
   Error printInfo();
-  virtual Error obtainInfoImpl(InfoQueueTy &Info) = 0;
+  virtual Expected<InfoTreeNode> obtainInfoImpl() = 0;
 
   /// Getters of the grid values.
   uint32_t getWarpSize() const { return GridValues.GV_Warp_Size; }
diff --git a/external/llvm-project/offload/plugins-nextgen/common/src/GlobalHandler.cpp b/external/llvm-project/offload/plugins-nextgen/common/src/GlobalHandler.cpp
index 27d7e8ee2fdf..5464c197dba7 100644
--- a/external/llvm-project/offload/plugins-nextgen/common/src/GlobalHandler.cpp
+++ b/external/llvm-project/offload/plugins-nextgen/common/src/GlobalHandler.cpp
@@ -173,16 +173,6 @@ Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device,
   return Plugin::success();
 }
 
-bool GenericGlobalHandlerTy::hasProfilingGlobals(GenericDeviceTy &Device,
-                                                 DeviceImageTy &Image) {
-  GlobalTy global(getInstrProfNamesVarName().str(), 0);
-  if (auto Err = getGlobalMetadataFromImage(Device, Image, global)) {
-    consumeError(std::move(Err));
-    return false;
-  }
-  return true;
-}
-
 Expected<GPUProfGlobals>
 GenericGlobalHandlerTy::readProfilingGlobals(GenericDeviceTy &Device,
                                              DeviceImageTy &Image) {
@@ -204,12 +194,17 @@ GenericGlobalHandlerTy::readProfilingGlobals(GenericDeviceTy &Device,
     // Check if given current global is a profiling global based
     // on name
     if (*NameOrErr == getInstrProfNamesVarName()) {
-      // Read in profiled function names
-      DeviceProfileData.NamesData = SmallVector<uint8_t>(Sym.getSize(), 0);
-      GlobalTy NamesGlobal(NameOrErr->str(), Sym.getSize(),
-                           DeviceProfileData.NamesData.data());
-      if (auto Err = readGlobalFromDevice(Device, Image, NamesGlobal))
-        return Err;
+      // Read in profiled function names from ELF
+      auto SectionOrErr = Sym.getSection();
+      if (!SectionOrErr)
+        return SectionOrErr.takeError();
+
+      auto ContentsOrErr = (*SectionOrErr)->getContents();
+      if (!ContentsOrErr)
+        return ContentsOrErr.takeError();
+
+      SmallVector<uint8_t> NameBytes(ContentsOrErr->bytes());
+      DeviceProfileData.NamesData = NameBytes;
     } else if (NameOrErr->starts_with(getInstrProfCountersVarPrefix())) {
       // Read global variable profiling counts
       SmallVector<int64_t> Counts(Sym.getSize() / sizeof(int64_t), 0);
@@ -322,3 +317,7 @@ Error GPUProfGlobals::write() const {
 
   return Plugin::success();
 }
+
+bool GPUProfGlobals::empty() const {
+  return Counts.empty() && Data.empty() && NamesData.empty();
+}
diff --git a/external/llvm-project/offload/plugins-nextgen/common/src/OffloadError.cpp b/external/llvm-project/offload/plugins-nextgen/common/src/OffloadError.cpp
index ad151cf26d04..78d49047d729 100644
--- a/external/llvm-project/offload/plugins-nextgen/common/src/OffloadError.cpp
+++ b/external/llvm-project/offload/plugins-nextgen/common/src/OffloadError.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Shared/OffloadError.h"
+#include "OffloadError.h"
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace llvm;
@@ -24,7 +24,7 @@ class OffloadErrorCategory : public std::error_category {
 #define OFFLOAD_ERRC(Name, Desc, Value)                                        \
   case ErrorCode::Name:                                                        \
     return #Desc;
-#include "Shared/OffloadErrcodes.inc"
+#include "OffloadErrcodes.inc"
 #undef OFFLOAD_ERRC
     }
     llvm_unreachable("Unrecognized offload ErrorCode");
diff --git a/external/llvm-project/offload/plugins-nextgen/common/src/PluginInterface.cpp b/external/llvm-project/offload/plugins-nextgen/common/src/PluginInterface.cpp
index f1fcac0e0d8b..3dad6a845096 100644
--- a/external/llvm-project/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/external/llvm-project/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -1077,14 +1077,13 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
 
   for (auto *Image : LoadedImages) {
     GenericGlobalHandlerTy &Handler = Plugin.getGlobalHandler();
-    if (!Handler.hasProfilingGlobals(*this, *Image))
-      continue;
-
-    GPUProfGlobals profdata;
     auto ProfOrErr = Handler.readProfilingGlobals(*this, *Image);
     if (!ProfOrErr)
       return ProfOrErr.takeError();
 
+    if (ProfOrErr->empty())
+      continue;
+
     // Dump out profdata
     if ((OMPX_DebugKind.get() & uint32_t(DeviceDebugKind::PGODump)) ==
         uint32_t(DeviceDebugKind::PGODump))
@@ -1852,14 +1851,14 @@ Error GenericDeviceTy::prepopulatePageTable(void *ptr, int64_t size) {
 }
 
 Error GenericDeviceTy::printInfo() {
-  InfoQueueTy InfoQueue;
+  auto Info = obtainInfoImpl();
 
   // Get the vendor-specific info entries describing the device properties.
-  if (auto Err = obtainInfoImpl(InfoQueue))
+  if (auto Err = Info.takeError())
     return Err;
 
   // Print all info entries.
-  InfoQueue.print();
+  Info->print();
 
   return Plugin::success();
 }
diff --git a/external/llvm-project/offload/plugins-nextgen/common/src/Utils/ELF.cpp b/external/llvm-project/offload/plugins-nextgen/common/src/Utils/ELF.cpp
index bd0ff2892817..84ebae0cd160 100644
--- a/external/llvm-project/offload/plugins-nextgen/common/src/Utils/ELF.cpp
+++ b/external/llvm-project/offload/plugins-nextgen/common/src/Utils/ELF.cpp
@@ -79,8 +79,7 @@ checkMachineImpl(const object::ELFObjectFile<ELFT> &ELFObj, uint16_t EMachine) {
   } else if (Header.e_machine == EM_CUDA) {
     if (~Header.e_flags & EF_CUDA_64BIT_ADDRESS)
       return createError("Invalid CUDA addressing mode");
-    if ((Header.e_flags & EF_CUDA_SM) < EF_CUDA_SM35 ||
-        (Header.e_flags & EF_CUDA_SM) > EF_CUDA_SM90)
+    if ((Header.e_flags & EF_CUDA_SM) < EF_CUDA_SM35)
       return createError("Unsupported NVPTX architecture");
   }
 
diff --git a/external/llvm-project/offload/plugins-nextgen/cuda/src/rtl.cpp b/external/llvm-project/offload/plugins-nextgen/cuda/src/rtl.cpp
index c411a6d80df0..af8d323aa585 100644
--- a/external/llvm-project/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/external/llvm-project/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -922,11 +922,12 @@ struct CUDADeviceTy : public GenericDeviceTy {
   }
 
   /// Print information about the device.
-  Error obtainInfoImpl(InfoQueueTy &Info) override {
+  Expected<InfoTreeNode> obtainInfoImpl() override {
     char TmpChar[1000];
     const char *TmpCharPtr;
     size_t TmpSt;
     int TmpInt;
+    InfoTreeNode Info;
 
     CUresult Res = cuDriverGetVersion(&TmpInt);
     if (Res == CUDA_SUCCESS)
@@ -971,27 +972,27 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       Info.add("Maximum Threads per Block", TmpInt);
 
-    Info.add("Maximum Block Dimensions", "");
+    auto &MaxBlock = *Info.add("Maximum Block Dimensions", "");
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add<InfoLevel2>("x", TmpInt);
+      MaxBlock.add("x", TmpInt);
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add<InfoLevel2>("y", TmpInt);
+      MaxBlock.add("y", TmpInt);
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add<InfoLevel2>("z", TmpInt);
+      MaxBlock.add("z", TmpInt);
 
-    Info.add("Maximum Grid Dimensions", "");
+    auto &MaxGrid = *Info.add("Maximum Grid Dimensions", "");
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add<InfoLevel2>("x", TmpInt);
+      MaxGrid.add("x", TmpInt);
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add<InfoLevel2>("y", TmpInt);
+      MaxGrid.add("y", TmpInt);
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add<InfoLevel2>("z", TmpInt);
+      MaxGrid.add("z", TmpInt);
 
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_PITCH, TmpInt);
     if (Res == CUDA_SUCCESS)
@@ -1087,7 +1088,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
     Info.add("Compute Capabilities", ComputeCapability.str());
 
-    return Plugin::success();
+    return Info;
   }
 
   virtual bool shouldSetupDeviceMemoryPool() const override {
diff --git a/external/llvm-project/offload/plugins-nextgen/host/src/rtl.cpp b/external/llvm-project/offload/plugins-nextgen/host/src/rtl.cpp
index a396ec12953c..fe8b2b64b8e2 100644
--- a/external/llvm-project/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/external/llvm-project/offload/plugins-nextgen/host/src/rtl.cpp
@@ -336,9 +336,10 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
   Error syncEventImpl(void *EventPtr) override { return Plugin::success(); }
 
   /// Print information about the device.
-  Error obtainInfoImpl(InfoQueueTy &Info) override {
+  Expected<InfoTreeNode> obtainInfoImpl() override {
+    InfoTreeNode Info;
     Info.add("Device Type", "Generic-elf-64bit");
-    return Plugin::success();
+    return Info;
   }
 
   /// This plugin should not setup the device environment or memory pool.
diff --git a/external/llvm-project/offload/test/api/omp_device_memory.c b/external/llvm-project/offload/test/api/omp_device_memory.c
index 60f47e94f90d..8876fc91ba40 100644
--- a/external/llvm-project/offload/test/api/omp_device_memory.c
+++ b/external/llvm-project/offload/test/api/omp_device_memory.c
@@ -2,6 +2,7 @@
 
 #include <omp.h>
 #include <stdio.h>
+#include <assert.h>
 
 int main() {
   const int N = 64;
@@ -24,4 +25,9 @@ int main() {
     printf("PASS\n");
 
   omp_free(device_ptr, llvm_omp_target_device_mem_alloc);
+
+  // Make sure this interface works.
+  void *ptr = omp_alloc(0, llvm_omp_target_device_mem_alloc);
+  assert(!ptr && "Ptr not (nullptr)");
+  omp_free(ptr, llvm_omp_target_device_mem_alloc);
 }
diff --git a/external/llvm-project/offload/test/lit.cfg b/external/llvm-project/offload/test/lit.cfg
index f0123e07dbea..e764afe64c78 100644
--- a/external/llvm-project/offload/test/lit.cfg
+++ b/external/llvm-project/offload/test/lit.cfg
@@ -36,8 +36,7 @@ if 'HSA_ENABLE_SDMA' in os.environ:
 
 # Architectures like gfx942 may or may not be APUs so an additional environment
 # variable is required as some tests can be APU specific.
-if 'IS_APU' in os.environ:
-    config.environment['IS_APU'] = os.environ['IS_APU']
+config.environment['IS_APU'] = os.environ.get('IS_APU', '0')
 
 # set default environment variables for test
 if 'CHECK_OPENMP_ENV' in os.environ:
diff --git a/external/llvm-project/offload/test/mapping/declare_mapper_nested_default_mappers_array.cpp b/external/llvm-project/offload/test/mapping/declare_mapper_nested_default_mappers_array.cpp
index d545e98ef6c3..93695d1b388f 100644
--- a/external/llvm-project/offload/test/mapping/declare_mapper_nested_default_mappers_array.cpp
+++ b/external/llvm-project/offload/test/mapping/declare_mapper_nested_default_mappers_array.cpp
@@ -4,8 +4,6 @@
 // RUN: %libomptarget-compilexx-run-and-check-x86_64-unknown-linux-gnu
 // RUN: %libomptarget-compilexx-run-and-check-nvptx64-nvidia-cuda
 
-// UNSUPPORTED: clang
-
 #include <cstdio>
 #include <cstdlib>
 
@@ -50,7 +48,7 @@ int main() {
   sa[1].h = N;
 
   printf("%d %d %d %4.5f %d\n", sa[1].e, sa[1].f.a, sa[1].f.c.a, sa[1].f.b[1],
-         sa[1].f.b == &x[0] ? 1 : 0);
+         sa[1].f.b == &y[0] ? 1 : 0);
   // CHECK: 111 222 777 20.00000 1
 
   __intptr_t p = reinterpret_cast<__intptr_t>(&y[0]);
@@ -65,6 +63,6 @@ int main() {
     sa[1].f.b[1] = 40;
   }
   printf("%d %d %d %4.5f %d\n", sa[1].e, sa[1].f.a, sa[1].f.c.a, sa[1].f.b[1],
-         sa[1].f.b == &x[0] ? 1 : 0);
+         sa[1].f.b == &y[0] ? 1 : 0);
   // CHECK: 333 222 777 40.00000 1
 }
diff --git a/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_teams.c b/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_teams.c
index 7bf3b1c11f28..b3b72db08039 100644
--- a/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_teams.c
+++ b/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_teams.c
@@ -3,7 +3,7 @@
 // RUN:     -Xarch_device -fprofile-update=atomic
 // RUN: env LLVM_PROFILE_FILE=%basename_t.llvm.profraw \
 // RUN:     %libomptarget-run-generic 2>&1
-// RUN: llvm-profdata show --all-functions --counts \
+// RUN: %profdata show --all-functions --counts \
 // RUN:     %target_triple.%basename_t.llvm.profraw | \
 // RUN:     %fcheck-generic --check-prefix="LLVM-PGO"
 
@@ -12,11 +12,11 @@
 // RUN:     -Xarch_device -fprofile-update=atomic
 // RUN: env LLVM_PROFILE_FILE=%basename_t.clang.profraw \
 // RUN:     %libomptarget-run-generic 2>&1
-// RUN: llvm-profdata show --all-functions --counts \
+// RUN: %profdata show --all-functions --counts \
 // RUN:     %target_triple.%basename_t.clang.profraw | \
 // RUN:     %fcheck-generic --check-prefix="CLANG-PGO"
 
-// REQUIRES: gpu
+// REQUIRES: amdgpu
 // REQUIRES: pgo
 
 int test1(int a) { return a / 2; }
diff --git a/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_threads.c b/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_threads.c
index f0e7111f7a64..440a6b533317 100644
--- a/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_threads.c
+++ b/external/llvm-project/offload/test/offloading/gpupgo/pgo_atomic_threads.c
@@ -3,7 +3,7 @@
 // RUN:     -Xarch_device -fprofile-update=atomic
 // RUN: env LLVM_PROFILE_FILE=%basename_t.llvm.profraw \
 // RUN:     %libomptarget-run-generic 2>&1
-// RUN: llvm-profdata show --all-functions --counts \
+// RUN: %profdata show --all-functions --counts \
 // RUN:     %target_triple.%basename_t.llvm.profraw | \
 // RUN:     %fcheck-generic --check-prefix="LLVM-PGO"
 
@@ -12,11 +12,11 @@
 // RUN:     -Xarch_device -fprofile-update=atomic
 // RUN: env LLVM_PROFILE_FILE=%basename_t.clang.profraw \
 // RUN:     %libomptarget-run-generic 2>&1
-// RUN: llvm-profdata show --all-functions --counts \
+// RUN: %profdata show --all-functions --counts \
 // RUN:     %target_triple.%basename_t.clang.profraw | \
 // RUN:     %fcheck-generic --check-prefix="CLANG-PGO"
 
-// REQUIRES: gpu
+// REQUIRES: amdgpu
 // REQUIRES: pgo
 
 int test1(int a) { return a / 2; }
diff --git a/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_1.c b/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_1.c
index 0e72941afde8..bdd028352e40 100644
--- a/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_1.c
+++ b/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_1.c
@@ -27,7 +27,7 @@ int main() {
     *P = 3;
   }
   // clang-format off
-// CHECK: OFFLOAD ERROR: Memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
+// CHECK: OFFLOAD ERROR: memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
 // NTRCE: Use 'OFFLOAD_TRACK_ALLOCATION_TRACES=true' to track device allocations
 // TRACE: Device pointer [[PTR]] does not point into any (current or prior) host-issued allocation.
 // TRACE: Closest host-issued allocation (distance 4096 bytes; might be by page):
diff --git a/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_2.c b/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_2.c
index 750d54b86317..6a67962f9eb3 100644
--- a/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_2.c
+++ b/external/llvm-project/offload/test/sanitizer/ptr_outside_alloc_2.c
@@ -18,7 +18,7 @@ int main() {
 #pragma omp target map(A[ : N])
   { A[N] = 3; }
   // clang-format off
-// CHECK: OFFLOAD ERROR: Memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
+// CHECK: OFFLOAD ERROR: memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
 // CHECK: Device pointer [[PTR]] does not point into any (current or prior) host-issued allocation.
 // CHECK: Closest host-issued allocation (distance 1 byte; might be by page):
 // CHECK: Last allocation of size 1073741824
diff --git a/external/llvm-project/offload/test/sanitizer/use_after_free_1.c b/external/llvm-project/offload/test/sanitizer/use_after_free_1.c
index 0ed86f66332c..c4783c5c36df 100644
--- a/external/llvm-project/offload/test/sanitizer/use_after_free_1.c
+++ b/external/llvm-project/offload/test/sanitizer/use_after_free_1.c
@@ -28,7 +28,7 @@ int main() {
   }
   llvm_omp_target_free_host(A, omp_get_default_device());
   // clang-format off
-// CHECK: OFFLOAD ERROR: Memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
+// CHECK: OFFLOAD ERROR: memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
 // NTRCE: Use 'OFFLOAD_TRACK_ALLOCATION_TRACES=true' to track device allocations
 // TRACE: Device pointer [[PTR]] points into prior host-issued allocation:
 // TRACE: Last deallocation:
diff --git a/external/llvm-project/offload/test/sanitizer/use_after_free_2.c b/external/llvm-project/offload/test/sanitizer/use_after_free_2.c
index f4e7f0fa0184..02aa453d0a97 100644
--- a/external/llvm-project/offload/test/sanitizer/use_after_free_2.c
+++ b/external/llvm-project/offload/test/sanitizer/use_after_free_2.c
@@ -22,7 +22,7 @@ int main() {
     *P = 3;
   }
   // clang-format off
-// CHECK: OFFLOAD ERROR: Memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
+// CHECK: OFFLOAD ERROR: memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
 // CHECK: Device pointer [[PTR]] points into prior host-issued allocation:
 // CHECK: Last deallocation:
 // CHECK: Last allocation of size 1073741824
diff --git a/external/llvm-project/offload/unittests/CMakeLists.txt b/external/llvm-project/offload/unittests/CMakeLists.txt
index 7ebfcd15bfc9..3d720f143920 100644
--- a/external/llvm-project/offload/unittests/CMakeLists.txt
+++ b/external/llvm-project/offload/unittests/CMakeLists.txt
@@ -2,7 +2,17 @@ add_custom_target(OffloadUnitTests)
 set_target_properties(OffloadUnitTests PROPERTIES FOLDER "Tests/UnitTests")
 
 function(add_offload_unittest test_dirname)
-  add_unittest(OffloadUnitTests ${test_dirname} ${ARGN})
+  set(target_name "${test_dirname}.unittests")
+
+  list(TRANSFORM ARGN PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" OUTPUT_VARIABLE files)
+
+  add_unittest(OffloadUnitTests "${target_name}"
+    ${CMAKE_CURRENT_SOURCE_DIR}/common/Environment.cpp
+    ${files})
+  add_dependencies(${target_name} ${PLUGINS_TEST_COMMON} OffloadUnitTestsDeviceBins)
+  target_compile_definitions(${target_name} PRIVATE DEVICE_CODE_PATH="${OFFLOAD_TEST_DEVICE_CODE_PATH}")
+  target_link_libraries(${target_name} PRIVATE ${PLUGINS_TEST_COMMON})
+  target_include_directories(${target_name} PRIVATE ${PLUGINS_TEST_INCLUDE})
 endfunction()
 
 # FIXME: Re-enable once OMPT design allows
diff --git a/external/llvm-project/offload/unittests/OffloadAPI/CMakeLists.txt b/external/llvm-project/offload/unittests/OffloadAPI/CMakeLists.txt
index 54b5c4e245e6..2844b675e5de 100644
--- a/external/llvm-project/offload/unittests/OffloadAPI/CMakeLists.txt
+++ b/external/llvm-project/offload/unittests/OffloadAPI/CMakeLists.txt
@@ -3,27 +3,33 @@ set(PLUGINS_TEST_INCLUDE ${LIBOMPTARGET_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}
 
 add_subdirectory(device_code)
 
-add_offload_unittest("offload.unittests"
-    ${CMAKE_CURRENT_SOURCE_DIR}/common/Environment.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/platform/olGetPlatformInfo.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/platform/olGetPlatformInfoSize.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/device/olIterateDevices.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/device/olGetDeviceInfo.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/device/olGetDeviceInfoSize.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/queue/olCreateQueue.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/queue/olWaitQueue.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/queue/olDestroyQueue.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/memory/olMemAlloc.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/memory/olMemFree.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/memory/olMemcpy.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/program/olCreateProgram.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/program/olDestroyProgram.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/kernel/olGetKernel.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/kernel/olLaunchKernel.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/event/olDestroyEvent.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/event/olWaitEvent.cpp
-    )
-add_dependencies("offload.unittests" ${PLUGINS_TEST_COMMON} OffloadUnitTestsDeviceBins)
-target_compile_definitions("offload.unittests" PRIVATE DEVICE_CODE_PATH="${OFFLOAD_TEST_DEVICE_CODE_PATH}")
-target_link_libraries("offload.unittests" PRIVATE ${PLUGINS_TEST_COMMON})
-target_include_directories("offload.unittests" PRIVATE ${PLUGINS_TEST_INCLUDE})
+add_offload_unittest("device"
+    device/olIterateDevices.cpp
+    device/olGetDeviceInfo.cpp
+    device/olGetDeviceInfoSize.cpp)
+
+add_offload_unittest("event"
+    event/olDestroyEvent.cpp
+    event/olWaitEvent.cpp)
+
+add_offload_unittest("kernel"
+    kernel/olGetKernel.cpp
+    kernel/olLaunchKernel.cpp)
+
+add_offload_unittest("memory"
+    memory/olMemAlloc.cpp
+    memory/olMemFree.cpp
+    memory/olMemcpy.cpp)
+
+add_offload_unittest("platform"
+    platform/olGetPlatformInfo.cpp
+    platform/olGetPlatformInfoSize.cpp)
+
+add_offload_unittest("program"
+    program/olCreateProgram.cpp
+    program/olDestroyProgram.cpp)
+
+add_offload_unittest("queue"
+    queue/olCreateQueue.cpp
+    queue/olWaitQueue.cpp
+    queue/olDestroyQueue.cpp)
diff --git a/external/llvm-project/offload/unittests/OffloadAPI/device_code/CMakeLists.txt b/external/llvm-project/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
index 5814943e4aaa..c2e4d0cb24e6 100644
--- a/external/llvm-project/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
+++ b/external/llvm-project/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
@@ -7,6 +7,7 @@ macro(add_offload_test_device_code test_filename test_name)
         add_custom_command(OUTPUT ${BIN_PATH}
             COMMAND
             ${CMAKE_C_COMPILER} --target=nvptx64-nvidia-cuda
+            ${ARGN}
             -march=${LIBOMPTARGET_DEP_CUDA_ARCH}
             --cuda-path=${CUDA_ROOT}
             ${SRC_PATH} -o ${BIN_PATH}
@@ -21,6 +22,7 @@ macro(add_offload_test_device_code test_filename test_name)
         add_custom_command(OUTPUT ${BIN_PATH}
             COMMAND
             ${CMAKE_C_COMPILER} --target=amdgcn-amd-amdhsa -nogpulib
+            ${ARGN}
             -mcpu=${LIBOMPTARGET_DEP_AMDGPU_ARCH}
             ${SRC_PATH} -o ${BIN_PATH}
             DEPENDS ${SRC_PATH}
@@ -61,6 +63,9 @@ endif()
 
 add_offload_test_device_code(foo.c foo)
 add_offload_test_device_code(bar.c bar)
+# By default, amdhsa will add a number of "hidden" arguments to the kernel defintion
+# O3 disables this, and results in a kernel function with actually no arguments as seen by liboffload
+add_offload_test_device_code(noargs.c noargs -O3)
 
 add_custom_target(OffloadUnitTestsDeviceBins DEPENDS ${BIN_PATHS})
 
diff --git a/external/llvm-project/offload/unittests/OffloadAPI/device_code/foo.c b/external/llvm-project/offload/unittests/OffloadAPI/device_code/foo.c
index 5bc893961d49..83cdc53cddd8 100644
--- a/external/llvm-project/offload/unittests/OffloadAPI/device_code/foo.c
+++ b/external/llvm-project/offload/unittests/OffloadAPI/device_code/foo.c
@@ -1,5 +1,6 @@
 #include <gpuintrin.h>
+#include <stdint.h>
 
-__gpu_kernel void foo(int *out) {
+__gpu_kernel void foo(uint32_t *out) {
   out[__gpu_thread_id(0)] = __gpu_thread_id(0);
 }
diff --git a/external/llvm-project/offload/unittests/OffloadAPI/device_code/noargs.c b/external/llvm-project/offload/unittests/OffloadAPI/device_code/noargs.c
new file mode 100644
index 000000000000..36e609aa26a0
--- /dev/null
+++ b/external/llvm-project/offload/unittests/OffloadAPI/device_code/noargs.c
@@ -0,0 +1,3 @@
+#include <gpuintrin.h>
+
+__gpu_kernel void noargs() { (void)0; }
diff --git a/external/llvm-project/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp b/external/llvm-project/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
index 20462e22fd73..157f33a36370 100644
--- a/external/llvm-project/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
+++ b/external/llvm-project/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
@@ -10,22 +10,17 @@
 #include <OffloadAPI.h>
 #include <gtest/gtest.h>
 
-struct olLaunchKernelTest : OffloadQueueTest {
-  void SetUp() override {
+struct LaunchKernelTestBase : OffloadQueueTest {
+  void SetUpKernel(const char *kernel) {
     RETURN_ON_FATAL_FAILURE(OffloadQueueTest::SetUp());
-    ASSERT_TRUE(TestEnvironment::loadDeviceBinary("foo", Device, DeviceBin));
+    ASSERT_TRUE(TestEnvironment::loadDeviceBinary(kernel, Device, DeviceBin));
     ASSERT_GE(DeviceBin->getBufferSize(), 0lu);
     ASSERT_SUCCESS(olCreateProgram(Device, DeviceBin->getBufferStart(),
                                    DeviceBin->getBufferSize(), &Program));
-    ASSERT_SUCCESS(olGetKernel(Program, "foo", &Kernel));
+    ASSERT_SUCCESS(olGetKernel(Program, kernel, &Kernel));
     LaunchArgs.Dimensions = 1;
-    LaunchArgs.GroupSizeX = 64;
-    LaunchArgs.GroupSizeY = 1;
-    LaunchArgs.GroupSizeZ = 1;
-
-    LaunchArgs.NumGroupsX = 1;
-    LaunchArgs.NumGroupsY = 1;
-    LaunchArgs.NumGroupsZ = 1;
+    LaunchArgs.GroupSize = {64, 1, 1};
+    LaunchArgs.NumGroups = {1, 1, 1};
 
     LaunchArgs.DynSharedMemory = 0;
   }
@@ -43,11 +38,24 @@ struct olLaunchKernelTest : OffloadQueueTest {
   ol_kernel_launch_size_args_t LaunchArgs{};
 };
 
+struct olLaunchKernelTest : LaunchKernelTestBase {
+  void SetUp() override {
+    RETURN_ON_FATAL_FAILURE(LaunchKernelTestBase::SetUpKernel("foo"));
+  }
+};
 OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernelTest);
 
+struct olLaunchKernelNoArgsTest : LaunchKernelTestBase {
+  void SetUp() override {
+    RETURN_ON_FATAL_FAILURE(LaunchKernelTestBase::SetUpKernel("noargs"));
+  }
+};
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernelNoArgsTest);
+
 TEST_P(olLaunchKernelTest, Success) {
   void *Mem;
-  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, 64, &Mem));
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
   struct {
     void *Mem;
   } Args{Mem};
@@ -57,7 +65,7 @@ TEST_P(olLaunchKernelTest, Success) {
 
   ASSERT_SUCCESS(olWaitQueue(Queue));
 
-  int *Data = (int *)Mem;
+  uint32_t *Data = (uint32_t *)Mem;
   for (int i = 0; i < 64; i++) {
     ASSERT_EQ(Data[i], i);
   }
@@ -65,9 +73,17 @@ TEST_P(olLaunchKernelTest, Success) {
   ASSERT_SUCCESS(olMemFree(Mem));
 }
 
+TEST_P(olLaunchKernelNoArgsTest, Success) {
+  ASSERT_SUCCESS(
+      olLaunchKernel(Queue, Device, Kernel, nullptr, 0, &LaunchArgs, nullptr));
+
+  ASSERT_SUCCESS(olWaitQueue(Queue));
+}
+
 TEST_P(olLaunchKernelTest, SuccessSynchronous) {
   void *Mem;
-  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, 64, &Mem));
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
 
   struct {
     void *Mem;
@@ -76,7 +92,7 @@ TEST_P(olLaunchKernelTest, SuccessSynchronous) {
   ASSERT_SUCCESS(olLaunchKernel(nullptr, Device, Kernel, &Args, sizeof(Args),
                                 &LaunchArgs, nullptr));
 
-  int *Data = (int *)Mem;
+  uint32_t *Data = (uint32_t *)Mem;
   for (int i = 0; i < 64; i++) {
     ASSERT_EQ(Data[i], i);
   }
diff --git a/external/llvm-project/openmp/docs/SupportAndFAQ.rst b/external/llvm-project/openmp/docs/SupportAndFAQ.rst
index a1f1a1759c2f..6e883531e75a 100644
--- a/external/llvm-project/openmp/docs/SupportAndFAQ.rst
+++ b/external/llvm-project/openmp/docs/SupportAndFAQ.rst
@@ -92,22 +92,18 @@ For AMDGPU offload, please see :ref:`build_amdgpu_offload_capable_compiler`.
 
 Q: How to build an OpenMP Nvidia offload capable compiler?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-The Cuda SDK is required on the machine that will execute the openmp application.
-
-If your build machine is not the target machine or automatic detection of the
-available GPUs failed, you should also set:
-
-- ``LIBOMPTARGET_DEVICE_ARCHITECTURES='sm_<xy>;...'`` where ``<xy>`` is the numeric
-  compute capability of your GPU. For instance, set 
-  ``LIBOMPTARGET_DEVICE_ARCHITECTURES='sm_70;sm_80'`` to target the Nvidia Volta
-  and Ampere architectures. 
-
+The CUDA SDK is required on the machine that will build and execute the
+offloading application. Normally this is only required at runtime by dynamically
+opening the CUDA driver API. This can be disabled in the build by omitting
+``cuda`` from the ``LIBOMPTARGET_DLOPEN_PLUGINS`` list which is present by
+default. With this setting we will instead find the CUDA library at LLVM build
+time and link against it directly.
 
 .. _build_amdgpu_offload_capable_compiler:
 
 Q: How to build an OpenMP AMDGPU offload capable compiler?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-A subset of the `ROCm <https://github.com/radeonopencompute>`_ toolchain is
+A subset of the `ROCm <https://github.com/ROCm>`_ toolchain is
 required to build the LLVM toolchain and to execute the openmp application.
 Either install ROCm somewhere that cmake's find_package can locate it, or
 build the required subcomponents ROCt and ROCr from source.
@@ -125,9 +121,9 @@ OpenMP's perspective. Rocr is an implementation of `HSA
   INSTALL_PREFIX=same-as-llvm-install
 
   cd $SOURCE_DIR
-  git clone git@github.com:RadeonOpenCompute/ROCT-Thunk-Interface.git -b roc-4.2.x \
+  git clone git@github.com:ROCm/ROCT-Thunk-Interface.git -b roc-4.2.x \
     --single-branch
-  git clone git@github.com:RadeonOpenCompute/ROCR-Runtime.git -b rocm-4.2.x \
+  git clone git@github.com:ROCm/ROCR-Runtime.git -b rocm-4.2.x \
     --single-branch
 
   cd $BUILD_DIR && mkdir roct && cd roct
@@ -164,31 +160,25 @@ available GPUs failed, you should also set:
 
 Q: What are the known limitations of OpenMP AMDGPU offload?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-LD_LIBRARY_PATH or rpath/runpath are required to find libomp.so and libomptarget.so
 
-There is no libc. That is, malloc and printf do not exist. Libm is implemented in terms
-of the rocm device library, which will be searched for if linking with '-lm'.
+LD_LIBRARY_PATH or rpath/runpath are required to find libomp.so and
+libomptarget.so correctly. The recommended way to configure this is with the
+``-frtlib-add-rpath`` option. Alternatively, set the ``LD_LIBRARY_PATH``
+environment variable to point to the installation. Normally, these libraries are
+installed in the target specific runtime directory. For example, a typical
+installation will have
+``<install>/lib/x86_64-unknown-linux-gnu/llibomptarget.so``
 
 Some versions of the driver for the radeon vii (gfx906) will error unless the
 environment variable 'export HSA_IGNORE_SRAMECC_MISREPORT=1' is set.
 
-It is a recent addition to LLVM and the implementation differs from that which
-has been shipping in ROCm and AOMP for some time. Early adopters will encounter
-bugs.
-
 Q: What are the LLVM components used in offloading and how are they found?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 The libraries used by an executable compiled for target offloading are:
 
 - ``libomp.so`` (or similar), the host openmp runtime
 - ``libomptarget.so``, the target-agnostic target offloading openmp runtime
-- plugins loaded by libomptarget.so:
-
-  - ``libomptarget.rtl.amdgpu.so``
-  - ``libomptarget.rtl.cuda.so``
-  - ``libomptarget.rtl.x86_64.so``
-  - and others
-
+- ``libompdevice.a``, the device-side OpenMP runtime.
 - dependencies of those plugins, e.g. cuda/rocr for nvptx/amdgpu
 
 The compiled executable is dynamically linked against a host runtime, e.g.
@@ -244,7 +234,6 @@ Q: Does OpenMP offloading support work in packages distributed as part of my OS?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 For now, the answer is most likely *no*. Please see :ref:`build_offload_capable_compiler`.
 
-
 .. _math_and_complex_in_target_regions:
 
 Q: Does Clang support `<math.h>` and `<complex.h>` operations in OpenMP target on GPUs?
@@ -273,21 +262,13 @@ through a similar mechanism. It is worth noting that this support requires
 <https://clang.llvm.org/docs/AttributeReference.html#pragma-omp-declare-variant>`__
 that are exposed through LLVM/Clang to the user as well.
 
-Q: What is a way to debug errors from mapping memory to a target device?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-An experimental way to debug these errors is to use :ref:`remote process
-offloading <remote_offloading_plugin>`.
-By using ``libomptarget.rtl.rpc.so`` and ``openmp-offloading-server``, it is
-possible to explicitly perform memory transfers between processes on the host
-CPU and run sanitizers while doing so in order to catch these errors.
-
 Q: Can I use dynamically linked libraries with OpenMP offloading?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Dynamically linked libraries can be only used if there is no device code split
+Dynamically linked libraries can be used if there is no device code shared
 between the library and application. Anything declared on the device inside the
-shared library will not be visible to the application when it's linked.
+shared library will not be visible to the application when it's linked. This is
+because device code only supports static linking.
 
 Q: How to build an OpenMP offload capable compiler with an outdated host compiler?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -302,38 +283,6 @@ For example, if your system-wide GCC installation is too old to build LLVM and
 you would like to use a newer GCC, set ``--gcc-install-dir=``
 to inform clang of the GCC installation you would like to use in the second stage.
 
-Q: How can I include OpenMP offloading support in my CMake project?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Currently, there is an experimental CMake find module for OpenMP target
-offloading provided by LLVM. It will attempt to find OpenMP target offloading
-support for your compiler. The flags necessary for OpenMP target offloading will
-be loaded into the ``OpenMPTarget::OpenMPTarget_<device>`` target or the
-``OpenMPTarget_<device>_FLAGS`` variable if successful. Currently supported
-devices are ``AMDGPU`` and ``NVPTX``.
-
-To use this module, simply add the path to CMake's current module path and call
-``find_package``. The module will be installed with your OpenMP installation by
-default. Including OpenMP offloading support in an application should now only
-require a few additions.
-
-.. code-block:: cmake
-
-  cmake_minimum_required(VERSION 3.20.0)
-  project(offloadTest VERSION 1.0 LANGUAGES CXX)
-
-  list(APPEND CMAKE_MODULE_PATH "${PATH_TO_OPENMP_INSTALL}/lib/cmake/openmp")
-
-  find_package(OpenMPTarget REQUIRED NVPTX)
-
-  add_executable(offload)
-  target_link_libraries(offload PRIVATE OpenMPTarget::OpenMPTarget_NVPTX)
-  target_sources(offload PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/Main.cpp)
-
-Using this module requires at least CMake version 3.20.0. Supported languages
-are C and C++ with Fortran support planned in the future. Compiler support is
-best for Clang but this module should work for other compiler vendors such as
-IBM, GNU.
 
 Q: What does 'Stack size for entry function cannot be statically determined' mean?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -359,11 +308,11 @@ will only extract archive members if an architecture is used, allowing users to
 create generic libraries.
 
 The architecture can either be specified manually using ``--offload-arch=``. If
-``--offload-arch=`` is present no ``-fopenmp-targets=`` flag is present then the
-targets will be inferred from the architectures. Conversely, if
+``--offload-arch=`` is present and no ``-fopenmp-targets=`` flag is present then
+the targets will be inferred from the architectures. Conversely, if
 ``--fopenmp-targets=`` is present with no ``--offload-arch`` then the target
 architecture will be set to a default value, usually the architecture supported
-by the system LLVM was built on.
+by the system LLVM was built on by executing the ``offload-arch`` utility.
 
 For example, an executable can be built that runs on AMDGPU and NVIDIA hardware
 given that the necessary build tools are installed for both.
@@ -433,7 +382,7 @@ linkable device image.
    clang++ openmp.o cuda.o --offload-link -o app
 
 Q: Are libomptarget and plugins backward compatible?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 No. libomptarget and plugins are now built as LLVM libraries starting from LLVM
 15. Because LLVM libraries are not backward compatible, libomptarget and plugins
@@ -459,7 +408,7 @@ with OpenMP.
 
 .. code-block:: shell
 
-   clang++ openmp.cpp -fopenmp --offload-arch=gfx90a -lcgpu
+   clang++ openmp.cpp -fopenmp --offload-arch=gfx90a -Xoffload-linker -lc
 
 For more information on how this is implemented in LLVM/OpenMP's offloading 
 runtime, refer to the `runtime documentation <libomptarget_libc>`_.
diff --git a/external/llvm-project/openmp/docs/index.rst b/external/llvm-project/openmp/docs/index.rst
index 5d39a1c8f080..97ded910782d 100644
--- a/external/llvm-project/openmp/docs/index.rst
+++ b/external/llvm-project/openmp/docs/index.rst
@@ -13,6 +13,24 @@
    LLVM/OpenMP Documentation <self>
 
 
+Building LLVM/OpenMP Offloading
+===============================
+
+Building LLVM/OpenMP with offloading support is fully documented in the
+:doc:`Support and FAQ <SupportAndFAQ>` page. For a quick start, we recommend
+the following template.
+
+.. code-block:: sh
+
+  $> cd llvm-project  # The llvm-project checkout
+  $> mkdir build
+  $> cd build
+  $> cmake ../llvm -G Ninja                                                 \
+     -C ../offload/cmake/caches/Offload.cmake \ # The preset cache file
+     -DCMAKE_BUILD_TYPE=<Debug|Release>   \ # Select build type
+     -DCMAKE_INSTALL_PREFIX=<PATH>        \ # Where the libraries will live
+  $> ninja install
+
 LLVM/OpenMP Design & Overview
 =============================
 
diff --git a/external/llvm-project/openmp/libompd/test/lit.site.cfg.in b/external/llvm-project/openmp/libompd/test/lit.site.cfg.in
index 63b8c97637d6..52e63f48bdc6 100644
--- a/external/llvm-project/openmp/libompd/test/lit.site.cfg.in
+++ b/external/llvm-project/openmp/libompd/test/lit.site.cfg.in
@@ -12,10 +12,14 @@ config.library_dir = "@LIBOMP_LIBRARY_DIR@"
 config.ompd_library_dir = "@CMAKE_CURRENT_BINARY_DIR@/../src/"
 config.omp_header_directory = "@LIBOMP_BINARY_DIR@/src"
 config.operating_system = "@CMAKE_SYSTEM_NAME@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
 
 config.ompt_plugin = "@OMPT_PLUGIN@"
 config.ompt_include_dir = "@LIBOMP_INCLUDE_DIR@"
 config.ompd_module = "@CMAKE_CURRENT_BINARY_DIR@/../gdb-plugin/python-module/ompd/"
 
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
 # Let the main config do the real work.
 lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")
diff --git a/external/llvm-project/openmp/runtime/CMakeLists.txt b/external/llvm-project/openmp/runtime/CMakeLists.txt
index 33afc7de96fd..10ebdc359a30 100644
--- a/external/llvm-project/openmp/runtime/CMakeLists.txt
+++ b/external/llvm-project/openmp/runtime/CMakeLists.txt
@@ -190,6 +190,8 @@ set(VE FALSE)
 set(S390X FALSE)
 set(WASM FALSE)
 set(PPC FALSE)
+set(SPARC FALSE)
+set(SPARCV9 FALSE)
 if("${LIBOMP_ARCH}" STREQUAL "i386" OR "${LIBOMP_ARCH}" STREQUAL "32")    # IA-32 architecture
   set(IA32 TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "x86_64" OR "${LIBOMP_ARCH}" STREQUAL "32e") # Intel(R) 64 architecture
@@ -226,6 +228,10 @@ elseif("${LIBOMP_ARCH}" STREQUAL "s390x") # S390x (Z) architecture
     set(S390X TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "wasm32") # WebAssembly architecture
     set(WASM TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "sparc") # SPARC architecture
+    set(SPARC TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "sparcv9") # SPARC V9 architecture
+    set(SPARCV9 TRUE)
 endif()
 
 # Set some flags based on build_type
diff --git a/external/llvm-project/openmp/runtime/cmake/LibompHandleFlags.cmake b/external/llvm-project/openmp/runtime/cmake/LibompHandleFlags.cmake
index 9618cda8fc4d..a41dc1dbaca0 100644
--- a/external/llvm-project/openmp/runtime/cmake/LibompHandleFlags.cmake
+++ b/external/llvm-project/openmp/runtime/cmake/LibompHandleFlags.cmake
@@ -60,10 +60,10 @@ function(libomp_get_cxxflags cxxflags)
     libomp_append(flags_local -Qinline-min-size=1 LIBOMP_HAVE_INLINE_MIN_SIZE_FLAG)
   endif()
   # Architectural C and C++ flags
+  if(CMAKE_SIZEOF_VOID_P EQUAL 4)
+    libomp_append(flags_local -m32 LIBOMP_HAVE_M32_FLAG)
+  endif()
   if(${IA32})
-    if(CMAKE_SIZEOF_VOID_P EQUAL 8)
-      libomp_append(flags_local -m32 LIBOMP_HAVE_M32_FLAG)
-    endif()
     libomp_append(flags_local /arch:SSE2 LIBOMP_HAVE_ARCH_SSE2_FLAG)
     libomp_append(flags_local -msse2 LIBOMP_HAVE_MSSE2_FLAG)
     libomp_append(flags_local -falign-stack=maintain-16-byte LIBOMP_HAVE_FALIGN_STACK_FLAG)
@@ -81,10 +81,10 @@ endfunction()
 function(libomp_get_asmflags asmflags)
   set(asmflags_local)
   # Architectural assembler flags
+  if(CMAKE_SIZEOF_VOID_P EQUAL 4)
+    libomp_append(asmflags_local -m32 LIBOMP_HAVE_M32_FLAG)
+  endif()
   if(${IA32})
-    if(CMAKE_SIZEOF_VOID_P EQUAL 8)
-      libomp_append(asmflags_local -m32 LIBOMP_HAVE_M32_FLAG)
-    endif()
     libomp_append(asmflags_local /safeseh LIBOMP_HAVE_SAFESEH_MASM_FLAG)
     libomp_append(asmflags_local /coff LIBOMP_HAVE_COFF_MASM_FLAG)
   elseif(${MIC})
@@ -113,10 +113,10 @@ function(libomp_get_ldflags ldflags)
   libomp_append(ldflags_local -static-intel LIBOMP_HAVE_STATIC_INTEL_FLAG)
   libomp_append(ldflags_local /SAFESEH LIBOMP_HAVE_SAFESEH_FLAG)
   # Architectural linker flags
+  if(CMAKE_SIZEOF_VOID_P EQUAL 4)
+    libomp_append(ldflags_local -m32 LIBOMP_HAVE_M32_FLAG)
+  endif()
   if(${IA32})
-    if(CMAKE_SIZEOF_VOID_P EQUAL 8)
-      libomp_append(ldflags_local -m32 LIBOMP_HAVE_M32_FLAG)
-    endif()
     libomp_append(ldflags_local -msse2 LIBOMP_HAVE_MSSE2_FLAG)
   elseif(${MIC})
     libomp_append(ldflags_local -mmic LIBOMP_HAVE_MMIC_FLAG)
@@ -150,9 +150,6 @@ function(libomp_get_libflags libflags)
     endif()
   elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux|NetBSD|SunOS")
     libomp_append(libflags_local -lm)
-    if (${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
-      libomp_append(libflags_local "-lproc")
-    endif()
   endif()
   set(libflags_local ${libflags_local} ${LIBOMP_LIBFLAGS})
   libomp_setup_flags(libflags_local)
@@ -163,7 +160,7 @@ endfunction()
 # Fortran flags
 function(libomp_get_fflags fflags)
   set(fflags_local)
-  if(${IA32})
+  if(CMAKE_SIZEOF_VOID_P EQUAL 4)
     libomp_append(fflags_local -m32 LIBOMP_HAVE_M32_FORTRAN_FLAG)
   endif()
   set(fflags_local ${fflags_local} ${LIBOMP_FFLAGS})
diff --git a/external/llvm-project/openmp/runtime/cmake/LibompUtils.cmake b/external/llvm-project/openmp/runtime/cmake/LibompUtils.cmake
index 58b93b384a76..a810881c2544 100644
--- a/external/llvm-project/openmp/runtime/cmake/LibompUtils.cmake
+++ b/external/llvm-project/openmp/runtime/cmake/LibompUtils.cmake
@@ -117,6 +117,10 @@ function(libomp_get_legal_arch return_arch_string)
     set(${return_arch_string} "VE" PARENT_SCOPE)
   elseif(${S390X})
     set(${return_arch_string} "S390X" PARENT_SCOPE)
+  elseif(${SPARC})
+    set(${return_arch_string} "SPARC" PARENT_SCOPE)
+  elseif(${SPARCV9})
+    set(${return_arch_string} "SPARCV9" PARENT_SCOPE)
   else()
     set(${return_arch_string} "${LIBOMP_ARCH}" PARENT_SCOPE)
     libomp_warning_say("libomp_get_legal_arch(): Warning: Unknown architecture: Using ${LIBOMP_ARCH}")
diff --git a/external/llvm-project/openmp/runtime/src/include/omp_lib.F90.var b/external/llvm-project/openmp/runtime/src/include/omp_lib.F90.var
index 57b764b01f92..e5651777dd65 100644
--- a/external/llvm-project/openmp/runtime/src/include/omp_lib.F90.var
+++ b/external/llvm-project/openmp/runtime/src/include/omp_lib.F90.var
@@ -431,7 +431,7 @@
 
           function omp_get_cancellation() bind(c)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_cancellation
+            logical (kind=omp_logical_kind) omp_get_cancellation
           end function omp_get_cancellation
 
           function omp_is_initial_device() bind(c)
diff --git a/external/llvm-project/openmp/runtime/src/include/ompx.h.var b/external/llvm-project/openmp/runtime/src/include/ompx.h.var
index 623f0b9c315b..6884745f4240 100644
--- a/external/llvm-project/openmp/runtime/src/include/ompx.h.var
+++ b/external/llvm-project/openmp/runtime/src/include/ompx.h.var
@@ -9,13 +9,21 @@
 #ifndef __OMPX_H
 #define __OMPX_H
 
-#ifdef __AMDGCN_WAVEFRONT_SIZE
-#define __WARP_SIZE __AMDGCN_WAVEFRONT_SIZE
-#else
-#define __WARP_SIZE 32
+#if (defined(__NVPTX__) || defined(__AMDGPU__))
+#include <gpuintrin.h>
+#define __OMPX_TARGET_IS_GPU
 #endif
 
 typedef unsigned long uint64_t;
+typedef unsigned int uint32_t;
+
+static inline uint32_t __warpSize(void) {
+#ifdef __OMPX_TARGET_IS_GPU
+  return __gpu_num_lanes();
+#else
+  __builtin_trap();
+#endif
+}
 
 #ifdef __cplusplus
 extern "C" {
@@ -212,7 +220,7 @@ static inline uint64_t ballot_sync(uint64_t mask, int pred) {
 ///{
 #define _TGT_KERNEL_LANGUAGE_SHFL_DOWN_SYNC(TYPE, TY)                          \
   static inline TYPE shfl_down_sync(uint64_t mask, TYPE var, unsigned delta,   \
-                                    int width = __WARP_SIZE) {                 \
+                                    int width = __warpSize()) {                \
     return ompx_shfl_down_sync_##TY(mask, var, delta, width);                  \
   }
 
diff --git a/external/llvm-project/openmp/runtime/src/z_Linux_util.cpp b/external/llvm-project/openmp/runtime/src/z_Linux_util.cpp
index 3b82b45e2440..368c0b6e872c 100644
--- a/external/llvm-project/openmp/runtime/src/z_Linux_util.cpp
+++ b/external/llvm-project/openmp/runtime/src/z_Linux_util.cpp
@@ -75,7 +75,6 @@
 #include <pthread_np.h>
 #endif
 #elif KMP_OS_SOLARIS
-#include <libproc.h>
 #include <procfs.h>
 #include <thread.h>
 #include <sys/loadavg.h>
@@ -2232,43 +2231,34 @@ int __kmp_is_address_mapped(void *addr) {
 
   kvm_close(fd);
 #elif KMP_OS_SOLARIS
-  prmap_t *cur, *map;
+  prxmap_t *cur, *map;
   void *buf;
   uintptr_t uaddr;
   ssize_t rd;
-  int err;
-  int file;
-
+  int fd;
   pid_t pid = getpid();
-  struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
-  ;
-
-  if (!fd) {
-    return 0;
-  }
-
-  char *name = __kmp_str_format("/proc/%d/map", pid);
-  size_t sz = (1 << 20);
-  file = open(name, O_RDONLY);
-  if (file == -1) {
+  char *name = __kmp_str_format("/proc/%d/xmap", pid);
+  fd = open(name, O_RDONLY);
+  if (fd == -1) {
     KMP_INTERNAL_FREE(name);
     return 0;
   }
 
+  size_t sz = (1 << 20);
   buf = KMP_INTERNAL_MALLOC(sz);
 
-  while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
+  while (sz > 0 && (rd = pread(fd, buf, sz, 0)) == sz) {
     void *newbuf;
     sz <<= 1;
     newbuf = KMP_INTERNAL_REALLOC(buf, sz);
     buf = newbuf;
   }
 
-  map = reinterpret_cast<prmap_t *>(buf);
+  map = reinterpret_cast<prxmap_t *>(buf);
   uaddr = reinterpret_cast<uintptr_t>(addr);
 
   for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
-    if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
+    if (uaddr >= cur->pr_vaddr && uaddr < cur->pr_vaddr) {
       if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
         found = 1;
         break;
@@ -2277,7 +2267,7 @@ int __kmp_is_address_mapped(void *addr) {
   }
 
   KMP_INTERNAL_FREE(map);
-  close(file);
+  close(fd);
   KMP_INTERNAL_FREE(name);
 #elif KMP_OS_DARWIN
 
diff --git a/external/llvm-project/openmp/runtime/test/lit.cfg b/external/llvm-project/openmp/runtime/test/lit.cfg
index 931029b105ec..3755a8d44e4e 100644
--- a/external/llvm-project/openmp/runtime/test/lit.cfg
+++ b/external/llvm-project/openmp/runtime/test/lit.cfg
@@ -70,7 +70,6 @@ if config.operating_system != 'Haiku':
         libs += " -latomic"
 
 # Allow REQUIRES / UNSUPPORTED / XFAIL to work
-config.target_triple = [ ]
 for feature in config.test_compiler_features:
     config.available_features.add(feature)
 
diff --git a/external/llvm-project/openmp/runtime/test/lit.site.cfg.in b/external/llvm-project/openmp/runtime/test/lit.site.cfg.in
index d0e54eac0dc3..fc65289e4ce6 100644
--- a/external/llvm-project/openmp/runtime/test/lit.site.cfg.in
+++ b/external/llvm-project/openmp/runtime/test/lit.site.cfg.in
@@ -12,6 +12,7 @@ config.libomp_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
 config.library_dir = "@LIBOMP_LIBRARY_DIR@"
 config.omp_header_directory = "@LIBOMP_BINARY_DIR@/src"
 config.operating_system = "@CMAKE_SYSTEM_NAME@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.hwloc_library_dir = "@LIBOMP_HWLOC_LIBRARY_DIR@"
 config.using_hwloc = @LIBOMP_USE_HWLOC@
 config.has_ompt = @LIBOMP_OMPT_SUPPORT@ and @LIBOMP_OMPT_OPTIONAL@
@@ -24,5 +25,8 @@ config.target_arch = "@LIBOMP_ARCH@"
 config.compiler_frontend_variant = "@CMAKE_C_COMPILER_FRONTEND_VARIANT@"
 config.compiler_simulate_id = "@CMAKE_C_SIMULATE_ID@"
 
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
 # Let the main config do the real work.
 lit_config.load_config(config, "@LIBOMP_BASE_DIR@/test/lit.cfg")
diff --git a/external/llvm-project/openmp/runtime/test/worksharing/for/omp_for_private_reduction.cpp b/external/llvm-project/openmp/runtime/test/worksharing/for/omp_for_private_reduction.cpp
new file mode 100644
index 000000000000..4520755a8a30
--- /dev/null
+++ b/external/llvm-project/openmp/runtime/test/worksharing/for/omp_for_private_reduction.cpp
@@ -0,0 +1,166 @@
+// RUN: %libomp-cxx-compile -fopenmp-version=60  && %libomp-run
+#include <stdio.h>
+#include <omp.h>
+#include <limits.h>
+#include <complex.h>
+#include <math.h>
+#include "omp_testsuite.h"
+
+#define N 10
+class Sum {
+  int val;
+
+public:
+  Sum(int v = 0) : val(v) {}
+  Sum operator+(const Sum &rhs) const { return Sum(val + rhs.val); }
+  Sum &operator+=(const Sum &rhs) {
+    val += rhs.val;
+    return *this;
+  }
+  int getValue() const { return val; }
+};
+
+// Declare OpenMP reduction
+#pragma omp declare reduction(sum_reduction:Sum : omp_out += omp_in)           \
+    initializer(omp_priv = Sum(0))
+
+#pragma omp declare reduction(sum_pctor_reduction:Sum : omp_out += omp_in)     \
+    initializer(omp_priv = Sum(1)) // non-default ctor
+
+int checkUserDefinedReduction() {
+  Sum final_result_udr(0);
+  Sum final_result_udr_pctor(1);
+  Sum array_sum[N];
+  int error_flag = 0;
+  int expected_value = 0;
+  int expected_value_pctor = 0;
+  for (int i = 0; i < N; ++i) {
+    array_sum[i] = Sum(i);
+    expected_value += i; // Calculate expected sum: 0 + 1 + ... + (N-1)
+    expected_value_pctor += i;
+  }
+  int num_threads_for_pctor_calc = 4; //  num_threads(4)
+  int priv_initializer_val_pctor = 1; //  initializer(omp_priv = Sum(1))
+  expected_value_pctor +=
+      num_threads_for_pctor_calc + priv_initializer_val_pctor;
+#pragma omp parallel num_threads(4) private(final_result_udr) private(         \
+        final_result_udr_pctor)
+  {
+#pragma omp for reduction(sum_reduction : final_result_udr)                    \
+    reduction(sum_pctor_reduction : final_result_udr_pctor)
+    for (int i = 0; i < N; ++i) {
+      final_result_udr += array_sum[i];
+      final_result_udr_pctor += array_sum[i];
+    }
+
+    if (final_result_udr.getValue() != expected_value ||
+        final_result_udr_pctor.getValue() != expected_value_pctor)
+      error_flag += 1;
+  }
+  return error_flag;
+}
+void performMinMaxRed(int &min_val, int &max_val) {
+  int input_data[] = {7, 3, 12, 5, 8};
+  int n_size = sizeof(input_data) / sizeof(input_data[0]);
+  min_val = INT_MAX;
+  max_val = INT_MIN;
+#pragma omp for reduction(original(private), min : min_val)                    \
+    reduction(original(private), max : max_val)
+  for (int i = 0; i < n_size; ++i) {
+    if (input_data[i] < min_val)
+      min_val = input_data[i];
+    if (input_data[i] > max_val)
+      max_val = input_data[i];
+  }
+}
+std::complex<double> doComplexReduction(std::complex<double> *arr) {
+  std::complex<double> result(1, 0);
+
+#pragma omp declare reduction(* : std::complex<double> : omp_out *= omp_in)    \
+    initializer(omp_priv = std::complex<double>(1, 0))
+
+#pragma omp for reduction(original(private), * : result)
+  for (int i = 0; i < N; ++i)
+    result *= arr[i];
+
+  return result;
+}
+
+void performReductions(int n_elements, const int *input_values,
+                       int &sum_val_out, int &prod_val_out,
+                       float &float_sum_val_out) {
+  // private variables for this thread's reduction.
+  sum_val_out = 0;
+  prod_val_out = 1;
+  float_sum_val_out = 0.0f;
+
+  const float kPiValue = 3.14f;
+#pragma omp for reduction(original(private), + : sum_val_out)                  \
+    reduction(original(private), * : prod_val_out)                             \
+    reduction(original(private), + : float_sum_val_out)
+  for (int i = 0; i < n_elements; ++i) {
+    sum_val_out += input_values[i];
+    prod_val_out *= (i + 1);
+    float_sum_val_out += kPiValue;
+  }
+}
+int main(void) {
+  int input_array[N];
+  int total_errors = 0;
+  const float kPiVal = 3.14f;
+  const int kExpectedSum = 45; // Sum of 0..9
+  const int kExpectedProd = 3628800; // 10!
+  const float kExpectedFsum = 31.400000f; // 3.14f * 10
+  const float kTolerance = 1e-4f;
+  const int kExpectedMin = 3;
+  const int kExpectedMax = 12;
+  std::complex<double> arr[N];
+  std::complex<double> kExpectedComplex(1, 0);
+  // Initialize the array
+  for (int i = 1; i <= N; ++i) {
+    arr[i - 1] = std::complex<double>(
+        1.0 + 0.1 * i, 0.5 * i); // Avoid zero to prevent multiplication by zero
+    kExpectedComplex *= arr[i - 1];
+  }
+
+  for (int i = 0; i < N; i++)
+    input_array[i] = i;
+#pragma omp parallel num_threads(4)
+  {
+
+    int t_sum_v;
+    int t_prod_v;
+    float t_fsum_v;
+    performReductions(N, input_array, t_sum_v, t_prod_v, t_fsum_v);
+    if (t_sum_v != kExpectedSum)
+      total_errors++;
+    if (t_prod_v != kExpectedProd)
+      total_errors++;
+    if (std::abs(t_fsum_v - kExpectedFsum) > kTolerance)
+      total_errors++;
+  }
+#pragma omp parallel num_threads(4)
+  {
+    int t_min_v;
+    int t_max_v;
+    performMinMaxRed(t_min_v, t_max_v);
+    if (t_min_v != kExpectedMin)
+      total_errors++;
+    if (t_max_v != kExpectedMax)
+      total_errors++;
+  }
+  total_errors += checkUserDefinedReduction();
+#pragma omp parallel num_threads(4)
+  {
+    std::complex<double> result(1, 0);
+    result = doComplexReduction(arr);
+    if (std::abs(result.real() - kExpectedComplex.real()) > 1e-6 ||
+        std::abs(result.imag() - kExpectedComplex.imag()) > 1e-6) {
+      total_errors++;
+    }
+  }
+  if (total_errors != 0)
+    fprintf(stderr, "ERROR: reduction on private variable  %d\n", total_errors);
+
+  return total_errors;
+}
diff --git a/external/llvm-project/openmp/tools/archer/ompt-tsan.cpp b/external/llvm-project/openmp/tools/archer/ompt-tsan.cpp
index bb60fc6b603f..c315999af432 100644
--- a/external/llvm-project/openmp/tools/archer/ompt-tsan.cpp
+++ b/external/llvm-project/openmp/tools/archer/ompt-tsan.cpp
@@ -1224,7 +1224,7 @@ static void ompt_tsan_finalize(ompt_data_t *tool_data) {
   if (archer_flags->print_max_rss) {
     struct rusage end;
     getrusage(RUSAGE_SELF, &end);
-    printf("MAX RSS[KBytes] during execution: %ld\n", end.ru_maxrss);
+    printf("MAX RSS[KiB] during execution: %ld\n", end.ru_maxrss);
   }
 
   if (archer_flags)
diff --git a/external/llvm-project/openmp/tools/archer/tests/lit.cfg b/external/llvm-project/openmp/tools/archer/tests/lit.cfg
index b22646a4defc..419c335bf860 100644
--- a/external/llvm-project/openmp/tools/archer/tests/lit.cfg
+++ b/external/llvm-project/openmp/tools/archer/tests/lit.cfg
@@ -58,7 +58,6 @@ if config.has_libatomic:
     libs += " -latomic"
 
 # Allow XFAIL to work
-config.target_triple = [ ]
 for feature in config.test_compiler_features:
     config.available_features.add(feature)
 
diff --git a/external/llvm-project/openmp/tools/archer/tests/lit.site.cfg.in b/external/llvm-project/openmp/tools/archer/tests/lit.site.cfg.in
index ddcb7b8bc3a5..dc7293972f08 100644
--- a/external/llvm-project/openmp/tools/archer/tests/lit.site.cfg.in
+++ b/external/llvm-project/openmp/tools/archer/tests/lit.site.cfg.in
@@ -11,11 +11,15 @@ config.libomp_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
 config.omp_library_dir = "@LIBOMP_LIBRARY_DIR@"
 config.omp_header_dir = "@LIBOMP_INCLUDE_DIR@"
 config.operating_system = "@CMAKE_SYSTEM_NAME@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.has_libatomic = @LIBARCHER_HAVE_LIBATOMIC@
 config.has_tsan = @OPENMP_TEST_ENABLE_TSAN@
 
 config.test_archer_flags = "@LIBARCHER_TEST_FLAGS@"
 config.libarcher_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
 
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
 # Let the main config do the real work.
 lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")
diff --git a/external/llvm-project/openmp/tools/multiplex/tests/lit.cfg b/external/llvm-project/openmp/tools/multiplex/tests/lit.cfg
index c399ab9cf415..6e89b3aba6f9 100644
--- a/external/llvm-project/openmp/tools/multiplex/tests/lit.cfg
+++ b/external/llvm-project/openmp/tools/multiplex/tests/lit.cfg
@@ -55,7 +55,6 @@ if 'INTEL_LICENSE_FILE' in os.environ:
     config.environment['INTEL_LICENSE_FILE'] = os.environ['INTEL_LICENSE_FILE']
 
 # Allow XFAIL to work
-config.target_triple = [ ]
 for feature in config.test_compiler_features:
     config.available_features.add(feature)
 
diff --git a/external/llvm-project/openmp/tools/multiplex/tests/lit.site.cfg.in b/external/llvm-project/openmp/tools/multiplex/tests/lit.site.cfg.in
index dbe7a33291ec..b6aff0de444e 100644
--- a/external/llvm-project/openmp/tools/multiplex/tests/lit.site.cfg.in
+++ b/external/llvm-project/openmp/tools/multiplex/tests/lit.site.cfg.in
@@ -11,6 +11,10 @@ config.omp_library_dir = "@LIBOMP_LIBRARY_DIR@"
 config.omp_header_dir = "@LIBOMP_INCLUDE_DIR@"
 config.ompt_print_callback_dir = "@OMPT_PRINT_CALLBACKS_DIR@"
 config.operating_system = "@CMAKE_SYSTEM_NAME@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")
diff --git a/external/llvm-project/polly/lib/Support/RegisterPasses.cpp b/external/llvm-project/polly/lib/Support/RegisterPasses.cpp
index 503c3ae1e07c..56cb8aadce3b 100644
--- a/external/llvm-project/polly/lib/Support/RegisterPasses.cpp
+++ b/external/llvm-project/polly/lib/Support/RegisterPasses.cpp
@@ -541,7 +541,6 @@ static bool
 parseTopLevelPipeline(llvm::ModulePassManager &MPM,
                       PassInstrumentationCallbacks *PIC,
                       ArrayRef<PassBuilder::PipelineElement> Pipeline) {
-  std::vector<PassBuilder::PipelineElement> FullPipeline;
   StringRef FirstName = Pipeline.front().Name;
 
   if (!isScopPassName(FirstName))
diff --git a/external/llvm-project/runtimes/CMakeLists.txt b/external/llvm-project/runtimes/CMakeLists.txt
index 7f1e2ae065d6..878b2eee3861 100644
--- a/external/llvm-project/runtimes/CMakeLists.txt
+++ b/external/llvm-project/runtimes/CMakeLists.txt
@@ -35,7 +35,7 @@ list(INSERT CMAKE_MODULE_PATH 0
 
 # We order libraries to mirror roughly how they are layered, except that compiler-rt can depend
 # on libc++, so we put it after.
-set(LLVM_DEFAULT_RUNTIMES "libc;libunwind;libcxxabi;pstl;libcxx;compiler-rt;openmp;offload")
+set(LLVM_DEFAULT_RUNTIMES "libc;libunwind;libcxxabi;pstl;libcxx;compiler-rt;libclc;openmp;offload")
 set(LLVM_SUPPORTED_RUNTIMES "${LLVM_DEFAULT_RUNTIMES};llvm-libgcc;flang-rt")
 set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
   "Semicolon-separated list of runtimes to build, or \"all\" (${LLVM_DEFAULT_RUNTIMES}). Supported runtimes are ${LLVM_SUPPORTED_RUNTIMES}.")
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index d9a02a2fd85c..8e629270c89d 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -344,6 +344,24 @@ libc_support_library(
     hdrs = ["hdr/types/struct_epoll_event.h"],
 )
 
+libc_support_library(
+    name = "types_struct_f_owner_ex",
+    hdrs = ["hdr/types/struct_f_owner_ex.h"],
+    deps = [":hdr_fcntl_overlay"],
+)
+
+libc_support_library(
+    name = "types_struct_flock",
+    hdrs = ["hdr/types/struct_flock.h"],
+    deps = [":hdr_fcntl_overlay"],
+)
+
+libc_support_library(
+    name = "types_struct_flock64",
+    hdrs = ["hdr/types/struct_flock64.h"],
+    deps = [":hdr_fcntl_overlay"],
+)
+
 libc_support_library(
     name = "types_struct_timespec",
     hdrs = ["hdr/types/struct_timespec.h"],
@@ -1380,6 +1398,28 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_osutil_fcntl",
+    srcs = ["src/__support/OSUtil/linux/fcntl.cpp"],
+    hdrs = ["src/__support/OSUtil/fcntl.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_common",
+        ":__support_error_or",
+        ":__support_osutil_syscall",
+        ":hdr_errno_macros",
+        ":hdr_fcntl_macros",
+        ":types_mode_t",
+        ":types_off_t",
+        ":types_struct_f_owner_ex",
+        ":types_struct_flock",
+        ":types_struct_flock64",
+    ],
+)
+
 libc_support_library(
     name = "__support_osutil_exit",
     srcs = ["src/__support/OSUtil/linux/exit.cpp"],
@@ -1602,6 +1642,7 @@ libc_header_library(
     name = "libcxx_shared_headers",
     hdrs = [
         "shared/fp_bits.h",
+        "shared/libc_common.h",
         "shared/str_to_float.h",
         "shared/str_to_integer.h",
     ],
@@ -1615,13 +1656,22 @@ libc_header_library(
 
 ############################### errno ########################################
 
+libc_support_library(
+    name = "__support_libc_errno",
+    hdrs = ["src/__support/libc_errno.h"],
+    deps = [
+        ":__support_macros_config",
+        ":hdr_errno_macros",
+    ],
+)
+
 libc_support_library(
     name = "errno",
     srcs = ["src/errno/libc_errno.cpp"],
-    hdrs = ["src/errno/libc_errno.h"],
     deps = [
         ":__support_common",
         ":__support_cpp_atomic",
+        ":__support_libc_errno",
         ":__support_macros_attributes",
         ":__support_macros_properties_architectures",
         ":hdr_errno_macros",
@@ -1972,6 +2022,32 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_math_exp_float_constants",
+    hdrs = ["src/__support/math/exp_float_constants.h"],
+    deps = [
+        ":__support_macros_config",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_expf",
+    hdrs = ["src/__support/math/expf.h"],
+    deps = [
+        ":__support_common",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+        ":__support_fputil_rounding_mode",
+        ":__support_libc_errno",
+        ":__support_macros_config",
+        ":__support_macros_optimization",
+        ":__support_math_exp_float_constants",
+    ],
+)
+
 ############################### complex targets ################################
 
 libc_function(
@@ -2569,13 +2645,8 @@ libc_math_function(
 libc_math_function(
     name = "expf",
     additional_deps = [
-        ":__support_fputil_fma",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_nearest_integer",
-        ":__support_fputil_polyeval",
-        ":__support_fputil_rounding_mode",
-        ":__support_macros_optimization",
-        ":common_constants",
+        ":__support_math_expf",
+        ":errno",
     ],
 )
 
@@ -4188,9 +4259,10 @@ libc_support_library(
     deps = [
         ":__support_common",
         ":__support_cpp_bitset",
+        ":__support_cpp_type_traits",
         ":__support_macros_optimization",
+        ":hdr_limits_macros",
         ":llvm_libc_types_size_t",
-        ":string_memory_utils",
         ":types_size_t",
     ],
 )
@@ -4201,6 +4273,7 @@ libc_function(
     hdrs = ["src/string/memchr.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_utils",
     ],
 )
@@ -4211,6 +4284,7 @@ libc_function(
     hdrs = ["src/string/memcpy.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_memory_utils",
     ],
 )
@@ -4221,6 +4295,7 @@ libc_function(
     hdrs = ["src/string/memset.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_memory_utils",
     ],
 )
@@ -4231,6 +4306,7 @@ libc_function(
     hdrs = ["src/string/memmove.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_memory_utils",
     ],
 )
@@ -4241,6 +4317,7 @@ libc_function(
     hdrs = ["src/string/mempcpy.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_memory_utils",
     ],
 )
@@ -4262,6 +4339,7 @@ libc_function(
     deps = [
         ":__support_common",
         ":__support_integer_operations",
+        ":__support_macros_null_check",
         ":string_memory_utils",
     ],
 )
@@ -4292,6 +4370,7 @@ libc_function(
     hdrs = ["src/string/memrchr.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_utils",
     ],
 )
@@ -4302,6 +4381,7 @@ libc_function(
     hdrs = ["src/string/strlen.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_utils",
     ],
 )
@@ -4312,6 +4392,7 @@ libc_function(
     hdrs = ["src/string/strcpy.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_memory_utils",
         ":string_utils",
     ],
@@ -4323,6 +4404,7 @@ libc_function(
     hdrs = ["src/string/strncpy.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
     ],
 )
 
@@ -4363,6 +4445,7 @@ libc_function(
     hdrs = ["src/string/strstr.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_null_check",
         ":string_memory_utils",
         ":string_utils",
     ],
@@ -4395,6 +4478,7 @@ libc_function(
     deps = [
         ":__support_common",
         ":__support_cpp_bitset",
+        ":__support_macros_null_check",
         ":string_utils",
     ],
 )
@@ -4431,13 +4515,28 @@ libc_function(
     }),
     deps = [
         ":__support_common",
-        ":__support_osutil_syscall",
+        ":__support_osutil_fcntl",
         ":errno",
         ":hdr_fcntl_macros",
         ":types_mode_t",
     ],
 )
 
+libc_function(
+    name = "fcntl",
+    srcs = ["src/fcntl/linux/fcntl.cpp"],
+    hdrs = ["src/fcntl/fcntl.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_common",
+        ":__support_osutil_fcntl",
+        ":errno",
+    ],
+)
+
 libc_function(
     name = "openat",
     srcs = ["src/fcntl/linux/openat.cpp"],
@@ -4498,7 +4597,7 @@ libc_function(
     hdrs = ["src/unistd/close.h"],
     deps = [
         ":__support_common",
-        ":__support_osutil_syscall",
+        ":__support_osutil_fcntl",
         ":errno",
     ],
 )
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index b37ec1933023..2c5ad7d27ce8 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -58,6 +58,7 @@ libc_test_library(
         "//libc:__support_fputil_fp_bits",
         "//libc:__support_fputil_fpbits_str",
         "//libc:__support_fputil_rounding_mode",
+        "//libc:__support_libc_errno",
         "//libc:__support_macros_config",
         "//libc:__support_macros_properties_architectures",
         "//libc:__support_macros_properties_types",
@@ -80,6 +81,7 @@ libc_test_library(
     ],
     deps = [
         ":LibcUnitTest",
+        "//libc:__support_libc_errno",
         "//libc:__support_macros_config",
         "//libc:errno",
     ],
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
index 123e05727aef..ba8753a17a85 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
@@ -31,6 +31,7 @@ def libc_test(name, copts = [], deps = [], local_defines = [], **kwargs):
         deps = [
             "//libc/test/UnitTest:LibcUnitTest",
             "//libc:__support_macros_config",
+            "//libc:__support_libc_errno",
             "//libc:errno",
             "//libc:func_aligned_alloc",
             "//libc:func_free",
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
index 484d3e5e0a24..505b73fd7711 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
@@ -122,6 +122,7 @@ libc_test(
         "//libc:mkdirat",
         "//libc:open",
         "//libc:remove",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
 
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
index 40f672d8099f..4f66793d44df 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
@@ -107,6 +107,7 @@ libc_test(
     deps = [
         "//libc:__support_fputil_fp_bits",
         "//libc:atof",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
 
@@ -203,9 +204,11 @@ libc_test_library(
         "//libc:__support_cpp_limits",
         "//libc:__support_cpp_type_traits",
         "//libc:__support_ctype_utils",
+        "//libc:__support_libc_errno",
         "//libc:__support_macros_properties_architectures",
         "//libc:errno",
         "//libc/test/UnitTest:LibcUnitTest",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
 
@@ -251,6 +254,7 @@ libc_test(
     deps = [
         "//libc:__support_fputil_fp_bits",
         "//libc:strtof",
+        "//libc/test/UnitTest:errno_test_helpers",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
 )
@@ -261,6 +265,7 @@ libc_test(
     deps = [
         "//libc:__support_fputil_fp_bits",
         "//libc:strtod",
+        "//libc/test/UnitTest:errno_test_helpers",
         "//libc/test/UnitTest:fp_test_helpers",
     ],
 )
@@ -272,5 +277,6 @@ libc_test(
         "//libc:__support_fputil_fp_bits",
         "//libc:__support_uint128",
         "//libc:strtold",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
index 7555c9f35e3a..f16570efa9c7 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
@@ -22,6 +22,7 @@ libc_test(
     name = "strcpy_test",
     srcs = ["strcpy_test.cpp"],
     deps = [
+        "//libc:hdr_signal_macros",
         "//libc:strcpy",
     ],
 )
@@ -38,6 +39,7 @@ libc_test(
     name = "memchr_test",
     srcs = ["memchr_test.cpp"],
     deps = [
+        "//libc:hdr_signal_macros",
         "//libc:memchr",
     ],
 )
@@ -77,6 +79,7 @@ libc_test(
     name = "memrchr_test",
     srcs = ["memrchr_test.cpp"],
     deps = [
+        "//libc:hdr_signal_macros",
         "//libc:memrchr",
     ],
 )
@@ -102,6 +105,7 @@ libc_test(
     name = "strspn_test",
     srcs = ["strspn_test.cpp"],
     deps = [
+        "//libc:hdr_signal_macros",
         "//libc:strspn",
     ],
 )
@@ -142,6 +146,7 @@ libc_test(
         ":memory_check_utils",
         ":protected_pages",
         "//libc:__support_macros_properties_os",
+        "//libc:hdr_signal_macros",
         "//libc:memcpy",
     ],
 )
@@ -150,6 +155,7 @@ libc_test(
     name = "mempcpy_test",
     srcs = ["mempcpy_test.cpp"],
     deps = [
+        "//libc:hdr_signal_macros",
         "//libc:mempcpy",
     ],
 )
@@ -161,6 +167,7 @@ libc_test(
         ":memory_check_utils",
         ":protected_pages",
         "//libc:__support_macros_properties_os",
+        "//libc:hdr_signal_macros",
         "//libc:memset",
     ],
 )
@@ -171,6 +178,7 @@ libc_test(
     deps = [
         ":memory_check_utils",
         "//libc:__support_cpp_span",
+        "//libc:hdr_signal_macros",
         "//libc:memcmp",
         "//libc:memmove",
         "//libc/test/UnitTest:memory_matcher",
@@ -182,6 +190,7 @@ libc_test(
     srcs = ["memcmp_test.cpp"],
     deps = [
         ":memory_check_utils",
+        "//libc:hdr_signal_macros",
         "//libc:memcmp",
         "//libc/test/UnitTest:test_logger",
     ],
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index edc39766774e..31855cd5444c 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -378,6 +378,7 @@ cc_library(
     hdrs = glob(["include/llvm/TableGen/*.h"]),
     copts = llvm_copts,
     deps = [
+        ":FrontendDirective",
         ":Support",
         ":config",
     ],
@@ -960,7 +961,10 @@ cc_library(
         "include/llvm/Analysis/WithCache.h",
     ] + [":llvm_intrinsics_headers"],
     copts = llvm_copts,
-    textual_hdrs = glob(["include/llvm/IR/*.def"]),
+    textual_hdrs = glob([
+        "include/llvm/IR/*.def",
+        "lib/IR/*.def",
+    ]),
     deps = [
         ":BinaryFormat",
         ":Demangle",
@@ -1627,6 +1631,8 @@ cc_library(
     copts = llvm_copts,
     deps = [
         ":Analysis",
+        ":ProfileData",
+        ":Support",
         ":TargetParser",
     ],
 )
@@ -1727,6 +1733,7 @@ cc_library(
         ":Core",
         ":Demangle",
         ":FrontendAtomic",
+        ":FrontendDirective",
         ":FrontendOffloading",
         ":MC",
         ":Scalar",
@@ -1762,6 +1769,13 @@ gentbl_filegroup(
     deps = [":AccTdFiles"],
 )
 
+cc_library(
+    name = "FrontendDirective",
+    srcs = glob(["lib/Frontend/Directive/*.cpp"]),
+    hdrs = glob(["include/llvm/Frontend/Directive/*.h"]),
+    deps = [":Support"],
+)
+
 cc_library(
     name = "FrontendOpenACC",
     srcs = glob([
@@ -1774,6 +1788,7 @@ cc_library(
     deps = [
         ":Analysis",
         ":Core",
+        ":FrontendDirective",
         ":Support",
         ":TransformUtils",
     ],
@@ -2112,7 +2127,6 @@ llvm_target_lib_list = [lib for lib in [
             "lib/Target/AArch64/AArch64GenSubtargetInfo.inc": ["-gen-subtarget"],
             "lib/Target/AArch64/AArch64GenDisassemblerTables.inc": [
                 "-gen-disassembler",
-                "--large-decoder-table",
             ],
             "lib/Target/AArch64/AArch64GenSystemOperands.inc": ["-gen-searchable-tables"],
             "lib/Target/AArch64/AArch64GenExegesis.inc": ["-gen-exegesis"],
@@ -5163,6 +5177,13 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "TestingDemangle",
+    testonly = True,
+    copts = llvm_copts,
+    textual_hdrs = glob(["include/llvm/Testing/Demangle/*.inc"]),
+)
+
 cc_library(
     name = "TestingSupport",
     testonly = True,
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/config.bzl b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/config.bzl
index fa616bcb9a8c..d9d3666a3ecc 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/config.bzl
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/config.bzl
@@ -32,6 +32,7 @@ posix_defines = [
     "BACKTRACE_HEADER=<execinfo.h>",
     r'LTDL_SHLIB_EXT=\".so\"',
     r'LLVM_PLUGIN_EXT=\".so\"',
+    "LLVM_ENABLE_PLUGINS=1",
     "LLVM_ENABLE_THREADS=1",
     "HAVE_DEREGISTER_FRAME=1",
     "HAVE_LIBPTHREAD=1",
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h
index 613d026de370..5dd53cffb7bd 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h
@@ -124,11 +124,18 @@
 /* Define to 1 if you have the DIA SDK installed, and to 0 if you don't. */
 #define LLVM_ENABLE_DIA_SDK 0
 
+/* Define if plugins enabled */
+/* LLVM_ENABLE_PLUGINS defined in Bazel */
+
 /* Define if building LLVM with LLVM_ENABLE_TELEMETRY */
 #define LLVM_ENABLE_TELEMETRY 1
 
 /* Define to 1 to enable expensive checks for debug location coverage checking,
    and to 0 otherwise. */
-#define LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING 1
+#define LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING 0
+
+/* Define to 1 to enable expensive tracking of the origin of debug location
+   coverage bugs, and to 0 otherwise. */
+#define LLVM_ENABLE_DEBUGLOC_ORIGIN_TRACKING 0
 
 #endif
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index 8ae197a64620..142b39823ed6 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -33,6 +33,80 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "inline_advisor_plugin",
+    srcs = glob(["Analysis/InlineAdvisorPlugin/*.cpp"]),
+    deps = [
+        "//llvm:Analysis",
+        "//llvm:Core",
+        "//llvm:Passes",
+        "//llvm:Support",
+    ],
+)
+
+cc_shared_library(
+    name = "inline_advisor_plugin_shared",
+    shared_lib_name = "InlineAdvisorPlugin.so",
+    deps = [":inline_advisor_plugin"],
+)
+
+cc_test(
+    name = "plugin_inline_advisor_analysis_test",
+    srcs = ["Analysis/PluginInlineAdvisorAnalysisTest.cpp"],
+    data = [
+        ":inline_advisor_plugin_shared",
+    ],
+    deps = [
+        "//llvm:Analysis",
+        "//llvm:AsmParser",
+        "//llvm:Core",
+        "//llvm:Passes",
+        "//llvm:Support",
+        "//llvm:TestingSupport",
+        "//llvm:attributes_gen",
+        "//llvm:config",
+        "//third-party/unittest:gtest",
+        "//third-party/unittest:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "inline_order_plugin",
+    srcs = glob(["Analysis/InlineOrderPlugin/*.cpp"]),
+    deps = [
+        "//llvm:Analysis",
+        "//llvm:Core",
+        "//llvm:Passes",
+        "//llvm:Support",
+    ],
+)
+
+cc_shared_library(
+    name = "inline_order_plugin_shared",
+    shared_lib_name = "InlineOrderPlugin.so",
+    deps = [":inline_order_plugin"],
+)
+
+cc_test(
+    name = "plugin_inline_order_analysis_test",
+    srcs = ["Analysis/PluginInlineOrderAnalysisTest.cpp"],
+    data = [
+        ":inline_order_plugin_shared",
+    ],
+    deps = [
+        "//llvm:Analysis",
+        "//llvm:AsmParser",
+        "//llvm:Core",
+        "//llvm:Passes",
+        "//llvm:Support",
+        "//llvm:TestingSupport",
+        "//llvm:attributes_gen",
+        "//llvm:config",
+        "//third-party/unittest:gtest",
+        "//third-party/unittest:gtest_main",
+    ],
+)
+
 cc_test(
     name = "analysis_tests",
     size = "small",
@@ -45,6 +119,12 @@ cc_test(
             "Analysis/TFUtilsTest.cpp",
             "Analysis/TrainingLoggerTest.cpp",
             "Analysis/MLModelRunnerTest.cpp",
+            # These tests dynamically load Plugins which both pull in
+            # llvm/lib/Analysis/ProfileSummaryInfo.cpp, which registers flags;
+            # if built into the same cc_test target, those flags will be
+            # registered twice and cause runtime failures.
+            "Analysis/PluginInlineAdvisorAnalysisTest.cpp",
+            "Analysis/PluginInlineOrderAnalysisTest.cpp",
         ],
     ),
     deps = [
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index e23fcacce472..e7398a696bea 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -535,6 +535,23 @@ mlir_c_api_cc_library(
     ],
 )
 
+mlir_c_api_cc_library(
+    name = "CAPIIndex",
+    srcs = [
+        "lib/CAPI/Dialect/Index.cpp",
+    ],
+    hdrs = [
+        "include/mlir-c/Dialect/Index.h",
+    ],
+    capi_deps = [
+        ":CAPIIR",
+    ],
+    includes = ["include"],
+    deps = [
+        ":IndexDialect",
+    ],
+)
+
 mlir_c_api_cc_library(
     name = "CAPILinalg",
     srcs = [
@@ -1842,6 +1859,38 @@ gentbl_cc_library(
     deps = [":ArmNeonTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "ArmNeonVectorTransformOpsIncGen",
+    tbl_outs = {
+        "include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.h.inc": ["-gen-op-decls"],
+        "include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp.inc": ["-gen-op-defs"],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.td",
+    deps = [
+        ":ArmNeonTdFiles",
+        ":TransformDialectTdFiles",
+    ],
+)
+
+cc_library(
+    name = "ArmNeonTransformOps",
+    srcs = glob(["lib/Dialect/ArmNeon/TransformOps/*.cpp"]),
+    hdrs = glob(["include/mlir/Dialect/ArmNeon/TransformOps/*.h"]),
+    includes = ["include"],
+    deps = [
+        ":ArmNeonDialect",
+        ":ArmNeonTransforms",
+        ":ArmNeonVectorTransformOpsIncGen",
+        ":IR",
+        ":LLVMCommonConversion",
+        ":LLVMDialect",
+        ":TransformDialect",
+        ":TransformDialectInterfaces",
+        ":VectorDialect",
+    ],
+)
+
 cc_library(
     name = "ArmNeon2dToIntr",
     srcs = glob([
@@ -2161,6 +2210,7 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/ArmSVE/Transforms/*.h"]),
     includes = ["include"],
     deps = [
+        ":ArithDialect",
         ":ArmSVEDialect",
         ":ArmSVEPassIncGen",
         ":DialectUtils",
@@ -2171,6 +2221,7 @@ cc_library(
         ":MemRefDialect",
         ":Pass",
         ":TransformUtils",
+        ":UBDialect",
         ":VectorDialect",
     ],
 )
@@ -2183,6 +2234,38 @@ gentbl_cc_library(
     deps = [":ArmSVETdFiles"],
 )
 
+gentbl_cc_library(
+    name = "ArmSVEVectorTransformOpsIncGen",
+    tbl_outs = {
+        "include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.h.inc": ["-gen-op-decls"],
+        "include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.cpp.inc": ["-gen-op-defs"],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/ArmSVE/TransformOps/ArmSVEVectorTransformOps.td",
+    deps = [
+        ":ArmSVETdFiles",
+        ":TransformDialectTdFiles",
+    ],
+)
+
+cc_library(
+    name = "ArmSVEVectorTransformOps",
+    srcs = glob(["lib/Dialect/ArmSVE/TransformOps/*.cpp"]),
+    hdrs = glob(["include/mlir/Dialect/ArmSVE/TransformOps/*.h"]),
+    includes = ["include"],
+    deps = [
+        ":ArmSVEDialect",
+        ":ArmSVETransforms",
+        ":ArmSVEVectorTransformOpsIncGen",
+        ":IR",
+        ":LLVMCommonConversion",
+        ":LLVMDialect",
+        ":TransformDialect",
+        ":TransformDialectInterfaces",
+        ":VectorDialect",
+    ],
+)
+
 ##---------------------------------------------------------------------------##
 # AMX dialect.
 ##---------------------------------------------------------------------------##
@@ -3437,8 +3520,10 @@ cc_library(
         ":IR",
         ":IndexDialect",
         ":InliningUtils",
+        ":LoopLikeInterface",
         ":MemRefDialect",
         ":Pass",
+        ":SCFTransforms",
         ":TransformUtils",
         ":VectorDialect",
         ":VectorTransforms",
@@ -3455,8 +3540,13 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/XeGPU/Utils/*.h"]),
     includes = ["include"],
     deps = [
+        ":DialectUtils",
         ":IR",
+        ":LoopLikeInterface",
+        ":SCFTransforms",
+        ":TransformUtils",
         ":XeGPUDialect",
+        "//llvm:Support",
     ],
 )
 
@@ -4429,6 +4519,8 @@ cc_library(
         ":AffineTransformOps",
         ":ArithToEmitC",
         ":ArithToLLVM",
+        ":ArmNeonTransformOps",
+        ":ArmSVEVectorTransformOps",
         ":BufferizationTransformOps",
         ":BuiltinToLLVMIRTranslation",
         ":ComplexToLLVM",
@@ -6807,6 +6899,7 @@ cc_library(
     deps = [
         ":SPIRVDialect",
         ":Support",
+        "//llvm:Support",
         "//llvm:config",
     ],
 )
@@ -6953,6 +7046,7 @@ cc_library(
         ":LoopLikeInterface",
         ":ParallelCombiningOpInterface",
         ":RelayoutOpInterface",
+        ":RuntimeVerifiableOpInterface",
         ":ShapedOpInterfaces",
         ":SideEffectInterfaces",
         ":SubsetOpInterface",
@@ -7060,11 +7154,13 @@ cc_library(
         ":ArithUtils",
         ":BufferizationDialect",
         ":BufferizationInterfaces",
+        ":ControlFlowDialect",
         ":DialectUtils",
         ":IR",
         ":LinalgDialect",
         ":MemRefDialect",
         ":Pass",
+        ":RuntimeVerifiableOpInterface",
         ":SCFDialect",
         ":SubsetOpInterface",
         ":TensorDialect",
@@ -8904,11 +9000,50 @@ cc_binary(
     name = "mlir-lsp-server",
     srcs = ["tools/mlir-lsp-server/mlir-lsp-server.cpp"],
     includes = ["include"],
+    local_defines = ["MLIR_INCLUDE_TESTS"],
     deps = [
         ":AllExtensions",
         ":AllPassesAndDialects",
         ":IR",
         ":MlirLspServerLib",
+        ":MlirLspServerSupportLib",
+        "//mlir/test:TestAffine",
+        "//mlir/test:TestAnalysis",
+        "//mlir/test:TestArith",
+        "//mlir/test:TestArmSME",
+        "//mlir/test:TestBufferization",
+        "//mlir/test:TestControlFlow",
+        "//mlir/test:TestConvertToSPIRV",
+        "//mlir/test:TestDLTI",
+        "//mlir/test:TestDialect",
+        "//mlir/test:TestFunc",
+        "//mlir/test:TestFuncToLLVM",
+        "//mlir/test:TestGPU",
+        "//mlir/test:TestIR",
+        "//mlir/test:TestLLVM",
+        "//mlir/test:TestLinalg",
+        "//mlir/test:TestLoopLikeInterface",
+        "//mlir/test:TestMath",
+        "//mlir/test:TestMathToVCIX",
+        "//mlir/test:TestMemRef",
+        "//mlir/test:TestMesh",
+        "//mlir/test:TestNVGPU",
+        "//mlir/test:TestPDLL",
+        "//mlir/test:TestPass",
+        "//mlir/test:TestReducer",
+        "//mlir/test:TestRewrite",
+        "//mlir/test:TestSCF",
+        "//mlir/test:TestSPIRV",
+        "//mlir/test:TestShapeDialect",
+        "//mlir/test:TestTensor",
+        "//mlir/test:TestTestDynDialect",
+        "//mlir/test:TestTilingInterface",
+        "//mlir/test:TestTosaDialect",
+        "//mlir/test:TestTransformDialect",
+        "//mlir/test:TestTransforms",
+        "//mlir/test:TestVector",
+        "//mlir/test:TestVectorToSPIRV",
+        "//mlir/test:TestXeGPU",
     ],
 )
 
@@ -8935,7 +9070,6 @@ cc_binary(
         "//mlir/test:TestAffine",
         "//mlir/test:TestAnalysis",
         "//mlir/test:TestArith",
-        "//mlir/test:TestArmNeon",
         "//mlir/test:TestArmSME",
         "//mlir/test:TestBufferization",
         "//mlir/test:TestControlFlow",
@@ -8953,6 +9087,7 @@ cc_binary(
         "//mlir/test:TestMath",
         "//mlir/test:TestMathToVCIX",
         "//mlir/test:TestMemRef",
+        "//mlir/test:TestMemRefToLLVMWithTransforms",
         "//mlir/test:TestMesh",
         "//mlir/test:TestNVGPU",
         "//mlir/test:TestPDLL",
@@ -10735,6 +10870,7 @@ cc_library(
         ":ArithDialect",
         ":ArithUtils",
         ":ArmNeonDialect",
+        ":ArmNeonTransforms",
         ":ArmSVEDialect",
         ":ArmSVETransforms",
         ":ConversionPassIncGen",
@@ -11131,7 +11267,7 @@ td_library(
 )
 
 gentbl_cc_library(
-    name = "TransformDialectEnumsIncGen",
+    name = "TransformAttrsIncGen",
     tbl_outs = {
         "include/mlir/Dialect/Transform/IR/TransformDialectEnums.h.inc": [
             "-gen-enum-decls",
@@ -11139,6 +11275,12 @@ gentbl_cc_library(
         "include/mlir/Dialect/Transform/IR/TransformDialectEnums.cpp.inc": [
             "-gen-enum-defs",
         ],
+        "include/mlir/Dialect/Transform/IR/TransformAttrs.h.inc": [
+            "-gen-attrdef-decls",
+        ],
+        "include/mlir/Dialect/Transform/IR/TransformAttrs.cpp.inc": [
+            "-gen-attrdef-defs",
+        ],
     },
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Dialect/Transform/IR/TransformAttrs.td",
@@ -11264,7 +11406,7 @@ cc_library(
         ":Rewrite",
         ":SideEffectInterfaces",
         ":Support",
-        ":TransformDialectEnumsIncGen",
+        ":TransformAttrsIncGen",
         ":TransformDialectIncGen",
         ":TransformDialectInterfaces",
         ":TransformDialectUtils",
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl b/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl
index 16a7ecadeaff..89b17735e005 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl
@@ -484,7 +484,8 @@ def gentbl_sharded_ops(
         test = False,
         includes = [],
         strip_include_prefix = None,
-        deps = []):
+        deps = [],
+        **kwargs):
     """Generate sharded op declarations and definitions.
 
     This special build rule shards op definitions in a TableGen file and generates multiple copies
@@ -524,6 +525,7 @@ def gentbl_sharded_ops(
         td_file = td_file,
         test = test,
         deps = deps,
+        **kwargs
     )
     all_files = [hdr_out, src_out]
     for i in range(0, shard_count):
@@ -535,9 +537,14 @@ def gentbl_sharded_ops(
             out = out_file,
             sharder = sharder,
             src_file = src_file,
+            **kwargs
         )
         all_files.append(out_file)
-    native.filegroup(name = name, srcs = all_files)
+    native.filegroup(
+        name = name,
+        srcs = all_files,
+        **kwargs
+    )
 
 def gentbl_sharded_op_defs(name, source_file, shard_count):
     """Generates multiple copies of a source file that includes sharded op definitions.
diff --git a/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 23d89f41a3a4..53405a0dea24 100644
--- a/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/external/llvm-project/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -882,6 +882,21 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "TestMemRefToLLVMWithTransforms",
+    srcs = glob(["lib/Conversion/MemRefToLLVM/*.cpp"]),
+    deps = [
+        "//mlir:FuncDialect",
+        "//mlir:FuncToLLVM",
+        "//mlir:IR",
+        "//mlir:LLVMCommonConversion",
+        "//mlir:LLVMDialect",
+        "//mlir:MemRefToLLVM",
+        "//mlir:MemRefTransforms",
+        "//mlir:Pass",
+    ],
+)
+
 cc_library(
     name = "TestMesh",
     srcs = glob(["lib/Dialect/Mesh/**/*.cpp"]),
@@ -963,22 +978,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "TestArmNeon",
-    srcs = glob(["lib/Dialect/ArmNeon/*.cpp"]),
-    includes = ["lib/Dialect/Test"],
-    deps = [
-        "//mlir:ArmNeonDialect",
-        "//mlir:ArmNeonTransforms",
-        "//mlir:FuncDialect",
-        "//mlir:IR",
-        "//mlir:Pass",
-        "//mlir:Support",
-        "//mlir:TransformUtils",
-        "//mlir:Transforms",
-    ],
-)
-
 cc_library(
     name = "TestArmSME",
     srcs = glob(["lib/Dialect/ArmSME/*.cpp"]),
diff --git a/external/llvm-project/utils/bazel/llvm_configs/llvm-config.h.cmake b/external/llvm-project/utils/bazel/llvm_configs/llvm-config.h.cmake
index dbc882937b4f..6d3c37cc8b19 100644
--- a/external/llvm-project/utils/bazel/llvm_configs/llvm-config.h.cmake
+++ b/external/llvm-project/utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -133,4 +133,8 @@
    and to 0 otherwise. */
 #cmakedefine01 LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
 
+/* Define to 1 to enable expensive tracking of the origin of debug location
+   coverage bugs, and to 0 otherwise. */
+#cmakedefine01 LLVM_ENABLE_DEBUGLOC_ORIGIN_TRACKING
+
 #endif
diff --git a/mlir/test/rocmlir-driver/runner-pipelines.mlir b/mlir/test/rocmlir-driver/runner-pipelines.mlir
index cd3101ab402b..3b380e0a47c2 100644
--- a/mlir/test/rocmlir-driver/runner-pipelines.mlir
+++ b/mlir/test/rocmlir-driver/runner-pipelines.mlir
@@ -15,7 +15,7 @@
 // RUNNER-SAME: convert-arith-to-llvm{index-bitwidth=0},
 // RUNNER-SAME: convert-math-to-llvm{approximate-log1p=true}),
 // RUNNER-SAME: convert-math-to-libm,
-// RUNNER-SAME: convert-vector-to-llvm{enable-amx=false enable-arm-neon=false enable-arm-sve=false enable-x86vector=false force-32bit-vector-indices=true reassociate-fp-reductions=false use-vector-alignment=false vector-contract-lowering=dot vector-transpose-lowering=eltwise},
+// RUNNER-SAME: convert-vector-to-llvm{enable-amx=false enable-arm-i8mm=false enable-arm-neon=false enable-arm-sve=false enable-x86vector=false force-32bit-vector-indices=true reassociate-fp-reductions=false use-vector-alignment=false vector-contract-lowering=dot vector-transpose-lowering=eltwise}
 // RUNNER-SAME: finalize-memref-to-llvm{index-bitwidth=0 use-aligned-alloc=false use-generic-functions=false},
 // RUNNER-SAME: async-to-async-runtime,
 // RUNNER-SAME: func.func(async-runtime-ref-counting,
diff --git a/mlir/tools/rocmlir-lib/librockcompiler_deps.cmake b/mlir/tools/rocmlir-lib/librockcompiler_deps.cmake
index e8631aeb07da..27be68e6f4ee 100644
--- a/mlir/tools/rocmlir-lib/librockcompiler_deps.cmake
+++ b/mlir/tools/rocmlir-lib/librockcompiler_deps.cmake
@@ -30,6 +30,7 @@ LLVMDebugInfoPDB
 LLVMDemangle
 LLVMExtensions
 LLVMFrontendAtomic
+LLVMFrontendDirective
 LLVMFrontendOffloading
 LLVMFrontendOpenMP
 LLVMGlobalISel